From 05c7ec032db32d16855181cd57eac6d3db298478 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Mon, 12 Oct 2026 10:37:11 +0200 Subject: [PATCH 001/100] fix(sensing): recover saved empty reference without renewing authority --- ...DR-355-local-empty-room-bootstrap-prior.md | 21 ++ .../src/bearer_auth.rs | 1 + .../src/field_bridge.rs | 7 +- .../wifi-densepose-sensing-server/src/main.rs | 223 +++++++++++++++++- 4 files changed, 250 insertions(+), 2 deletions(-) diff --git a/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md b/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md index e03ef06d7e..b47082dbf3 100644 --- a/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md +++ b/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md @@ -72,6 +72,27 @@ host to provide a protected state directory. ## Consequences +### Explicit recovery without restarting, 2026-10-12 + +`POST /api/v1/calibration/bootstrap/recover` requires administrator authority, +the current server boot epoch and the stored payload digest. It validates the +installation-bound file and original expiry before changing live state, and +requires at least ten seconds of eligible raw CSI on its original source/grid. +An active unfinished capture must be cancelled first. + +Recovery clears old source sequence faults and runtime windows, withdraws any +full room session/receipt, and restores only `bootstrap_only`. New packets must +build a complete matching runtime window before negative suppression applies. +Duplicates and backward discontinuities remain rejected. No calibration frame +count, training timestamp, threshold or expiry is renewed, and the stored file +is not rewritten. Full calibrated evidence and numeric vitals remain disabled. + +The twelve-hour stale boundary still withdraws full calibrated authority. A +restored negative-only prior may match fresh runtime windows while stale until +its original twenty-four-hour hard expiry. Expiry stops matching and recovery. +This resolves the mismatch between the advertised twenty-four-hour prior and +the previous fresh-only runtime gate; it makes no room-selectivity claim. + Startup can use recent measured background structure instead of only generic heuristics. No accuracy lift is claimed until occupied and empty held out data from multiple installations is recorded. diff --git a/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs b/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs index d8a9cbfa5a..9516c80a32 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs @@ -1975,6 +1975,7 @@ mod scope_gate_polarity_tests { (Method::DELETE, "/api/v1/recording/r1"), (Method::POST, "/api/v1/config/ground-truth"), (Method::POST, "/api/v1/calibration/reset"), + (Method::POST, "/api/v1/calibration/bootstrap/recover"), ] { assert_eq!( required_scope_for(&m, p), diff --git a/v2/crates/wifi-densepose-sensing-server/src/field_bridge.rs b/v2/crates/wifi-densepose-sensing-server/src/field_bridge.rs index 6793cc26f3..7d6a3539b9 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/field_bridge.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/field_bridge.rs @@ -264,7 +264,7 @@ pub fn bootstrap_background_match( observed_at_us: u64, ) -> Option { if frame_history.len() < OCCUPANCY_WINDOW - || field.check_freshness(observed_at_us) != CalibrationStatus::Fresh + || !matches!(field.check_freshness(observed_at_us), CalibrationStatus::Fresh | CalibrationStatus::Stale) { return None; } @@ -923,6 +923,11 @@ mod tests { .is_some_and(|score| (0.5..=1.0).contains(&score))); let short_history: VecDeque> = empty_history.iter().take(49).cloned().collect(); + // Staleness withdraws full calibrated authority after half the lifetime, + // but the persisted negative-only reference remains useful until expiry. + assert!(bootstrap_background_match(&field, &empty_history, 32_000_000).is_some()); + assert!(calibrated_occupancy(&field, &empty_history, 32_000_000).is_none()); + assert!(bootstrap_background_match(&field, &empty_history, 62_000_000).is_none()); assert!(!bootstrap_empty_prior_applies( &field, &short_history, diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index d35f8830fe..1e0d6afb27 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -2646,6 +2646,11 @@ impl AppStateInner { (chrono::Utc::now().timestamp_millis().max(0) as u64) .saturating_mul(1_000), ) == CalibrationStatus::Fresh + || (self.bootstrap_baseline_active + && field.check_freshness( + (chrono::Utc::now().timestamp_millis().max(0) as u64) + .saturating_mul(1_000), + ) == CalibrationStatus::Stale) } }); if accepted { @@ -3109,7 +3114,7 @@ mod calibration_expiry_tests { use super::*; use wifi_densepose_signal::ruvsense::field_model::FieldModelConfig; - fn finalized_model(calibrated_at_unix_ms: u64) -> FieldModel { + pub(super) fn finalized_model(calibrated_at_unix_ms: u64) -> FieldModel { let mut model = FieldModel::new(FieldModelConfig { n_links: 1, n_subcarriers: 56, @@ -9486,6 +9491,221 @@ struct BootstrapRefineQuery { source_node_id: Option, } +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct BootstrapRecoverRequest { + boot_epoch: String, + content_sha256: String, +} + +/// Explicitly restore the verified negative-only prior, just as a clean boot +/// would. This never renews its lifetime or restores a full room receipt. +async fn calibration_recover_bootstrap( + State(state): State, + Json(request): Json, +) -> Json { + let mut s = state.write().await; + if request.boot_epoch != s.calibration_boot_epoch + || !valid_binding_digest(&request.content_sha256) + || s.bootstrap_baseline.as_ref().is_none_or(|metadata| { + metadata.content_sha256 != request.content_sha256 + }) + { + return Json(serde_json::json!({ + "success": false, "error_code": "bootstrap_recovery_identity_mismatch", + "error": "The saved reference or server boot changed. Refresh calibration status before restoring it.", + })); + } + if s.field_model.as_ref().is_some_and(|model| matches!( + model.status(), CalibrationStatus::Uncalibrated | CalibrationStatus::Collecting + )) && s.calibration_session_id.is_some() { + return Json(serde_json::json!({ + "success": false, "error_code": "calibration_capture_active", + "error": "Cancel the active capture before restoring the saved empty reference.", + })); + } + let Some(installation_id) = s.installation_id.as_deref() else { + return Json(serde_json::json!({ + "success": false, "error_code": "installation_id_required", + "error": "A stable installation identity is required to restore the saved reference.", + })); + }; + let path = bootstrap_baseline::path_in(&s.data_dir); + // Validate disk content before touching the live model, histories or faults. + let (model, metadata) = match bootstrap_baseline::load( + &path, installation_id, chrono::Utc::now().timestamp_millis().max(0) as u64, + ) { + Ok(value) => value, + Err(_) => return Json(serde_json::json!({ + "success": false, "error_code": "bootstrap_reference_unavailable", + "error": "The saved empty reference is expired or failed validation. Collect a new baseline.", + })), + }; + if metadata.content_sha256 != request.content_sha256 { + return Json(serde_json::json!({ + "success": false, "error_code": "bootstrap_recovery_identity_mismatch", + "error": "The saved reference changed. Refresh calibration status before restoring it.", + })); + } + let [source_node_id] = metadata.source_node_ids.as_slice() else { + return Json(serde_json::json!({ + "success": false, "error_code": "bootstrap_source_invalid", + "error": "The saved reference must have one source radio.", + })); + }; + let grid = CsiGridKey::from_bootstrap_grid(metadata.source_grid); + let now = std::time::Instant::now(); + let evidence = s.node_states.get(source_node_id).and_then(|node| { + node.calibration_grid_candidates(now).into_iter().find_map(|(candidate, evidence)| { + (candidate == grid + && evidence.span_s >= CALIBRATION_GRID_MIN_SPAN_S + && evidence.rate_hz >= CALIBRATION_GRID_MIN_RATE_HZ + && evidence.max_gap_s < CALIBRATION_GRID_MAX_GAP_S + && evidence.latest_age_s < CALIBRATION_GRID_MAX_GAP_S).then_some(evidence) + }) + }); + let Some(evidence) = evidence else { + return Json(serde_json::json!({ + "success": false, "error_code": "bootstrap_source_unavailable", + "error": "The saved source radio and CSI grid need at least ten seconds of fresh live data before recovery.", + "source_node_id": source_node_id, + })); + }; + let binding = CalibrationGridBinding { source_node_id: *source_node_id, grid, evidence: Some(evidence) }; + s.clear_field_model_binding(); + s.clear_calibration_sequence_state(); + s.calibration_source_node_ids.clear(); + s.calibration_source_node_ids.insert(binding.source_node_id); + s.calibration_observed_source_node_ids.clear(); + s.calibration_model_id = None; + s.calibration_session_id = None; + s.calibration_binding_digest = None; + s.calibration_model_receipt = None; + s.field_model = Some(model); + s.calibration_grid_binding = Some(binding); + s.bootstrap_baseline = Some(metadata.clone()); + s.bootstrap_baseline_active = true; + Json(serde_json::json!({ + "success": true, "boot_epoch": s.calibration_boot_epoch, + "status": s.field_model_status_at(chrono::Utc::now().timestamp_millis().max(0) as u64).map(|value| format!("{value:?}").to_lowercase()), + "bootstrap_baseline": metadata, "restored_frames": 0, + "full_calibration_required": true, + "calibrated_evidence_authorized": false, "numeric_vitals_authorized": false, + })) +} + +#[cfg(test)] +mod bootstrap_recovery_tests { + use super::*; + + fn fixture() -> (tempfile::TempDir, SharedState, BootstrapRecoverRequest) { + let directory = tempfile::tempdir().unwrap(); + let now_ms = chrono::Utc::now().timestamp_millis() as u64; + let model = super::calibration_expiry_tests::finalized_model(now_ms); + let mut snapshot = model.export_snapshot().unwrap(); + snapshot.config.baseline_expiry_s = 86_400.0; + snapshot.modes.calibrated_at_us = (now_ms - 13 * 3_600_000) * 1_000; + let model = FieldModel::from_snapshot(snapshot, now_ms * 1_000).unwrap(); + let grid = CsiGridKey { n_subcarriers: 64, ppdu_type: 0 }; + let metadata = bootstrap_baseline::store( + &bootstrap_baseline::path_in(directory.path()), "recovery-test", &[5], + grid.as_bootstrap_grid(), "cal-model-saved", now_ms, &model, + ).unwrap(); + let mut state = AppStateInner::minimal(); + state.installation_id = Some("recovery-test".into()); + state.data_dir = directory.path().to_path_buf(); + state.bootstrap_baseline = Some(metadata.clone()); + state.field_model = Some(model); + state.calibration_session_id = Some("old-session".into()); + state.calibration_model_id = Some("old-model".into()); + state.calibration_binding_digest = Some("ab".repeat(32)); + state.calibration_sequence_fault_node_ids.insert(5); + state.calibration_last_sequences.insert(5, 50_000); + let mut node = NodeState::new(); + let now = std::time::Instant::now(); + for index in 0..=240 { + node.observe_raw_grid(grid, now - Duration::from_millis(12_000 - index * 50)); + } + node.field_model_history.push_back(vec![999.0; 56]); + node.field_model_latest_sequence = Some(50_000); + node.field_model_latest_seen = Some(now); + state.node_states.insert(5, node); + let request = BootstrapRecoverRequest { + boot_epoch: state.calibration_boot_epoch.clone(), content_sha256: metadata.content_sha256, + }; + (directory, Arc::new(RwLock::new(state)), request) + } + + #[tokio::test] + async fn stale_recovery_retains_expiry_withdraws_full_receipt_and_requires_new_frames() { + let (_directory, state, request) = fixture(); + let expiry = state.read().await.bootstrap_baseline.as_ref().unwrap().expires_at_unix_ms; + let Json(response) = calibration_recover_bootstrap(State(state.clone()), Json(request)).await; + assert_eq!(response["success"], true); + assert_eq!(response["bootstrap_baseline"]["expires_at_unix_ms"], expiry); + assert_eq!(response["restored_frames"], 0); + assert_eq!(response["calibrated_evidence_authorized"], false); + let mut s = state.write().await; + assert!(s.bootstrap_baseline_active); + assert!(s.calibration_model_receipt.is_none()); + assert!(s.calibration_session_id.is_none()); + assert!(s.calibration_sequence_fault_node_ids.is_empty()); + assert!(s.node_states[&5].field_model_history.is_empty()); + assert!(s.node_states[&5].field_model_latest_seen.is_none()); + assert!(!s.bootstrap_empty_prior_applies(chrono::Utc::now().timestamp_millis() as u64)); + let frame = s.field_model.as_ref().unwrap().modes().unwrap().baseline[0].clone(); + let grid = s.calibration_grid_binding.unwrap().grid; + assert!(s.maybe_feed_calibration_frame(5, grid, 1, &frame, std::time::Instant::now())); + assert!(!s.maybe_feed_calibration_frame(5, grid, 1, &frame, std::time::Instant::now())); + assert!(s.maybe_feed_calibration_frame(5, grid, 100, &frame, std::time::Instant::now())); + assert!(!s.maybe_feed_calibration_frame(5, grid, 2, &frame, std::time::Instant::now())); + assert!(s.calibration_sequence_fault_node_ids.contains(&5)); + assert!(!s.explicit_calibration_fresh_at(chrono::Utc::now().timestamp_millis() as u64)); + } + + #[tokio::test] + async fn rejected_recovery_preserves_live_state_and_saved_image() { + for invalid_case in ["boot", "digest", "tamper", "source", "grid", "capture", "installation", "expired"] { + let (directory, state, mut request) = fixture(); + let path = bootstrap_baseline::path_in(directory.path()); + match invalid_case { + "boot" => request.boot_epoch = "old-boot".into(), + "digest" => request.content_sha256 = "cd".repeat(32), + "tamper" => std::fs::write(&path, b"{}").unwrap(), + "source" => state.write().await.node_states.clear(), + "grid" => { + state.write().await.node_states.get_mut(&5).unwrap().raw_grid_observations.clear(); + } + "capture" => { + state.write().await.field_model = Some(FieldModel::new(field_bridge::single_link_config()).unwrap()); + } + "installation" => state.write().await.installation_id = Some("another-installation".into()), + "expired" => { + let mut s = state.write().await; + let now_ms = chrono::Utc::now().timestamp_millis() as u64; + let old = s.field_model.as_ref().unwrap().export_snapshot().unwrap(); + let mut old = old; + old.modes.calibrated_at_us = (now_ms - 25 * 3_600_000) * 1_000; + // Store a valid image when it was still young, then load now. + let model = FieldModel::from_snapshot(old, (now_ms - 24 * 3_600_000) * 1_000).unwrap(); + let metadata = bootstrap_baseline::store(&path, "recovery-test", &[5], + s.calibration_grid_binding.map_or(CsiGridKey { n_subcarriers:64, ppdu_type:0 }, |b| b.grid).as_bootstrap_grid(), + "cal-model-saved", now_ms - 24 * 3_600_000, &model).unwrap(); + request.content_sha256 = metadata.content_sha256.clone(); s.bootstrap_baseline = Some(metadata); + } + _ => unreachable!(), + } + let bytes = std::fs::read(&path).unwrap(); + let Json(response) = calibration_recover_bootstrap(State(state.clone()), Json(request)).await; + assert_eq!(response["success"], false, "{invalid_case}"); + let s = state.read().await; + assert_eq!(s.calibration_session_id.as_deref(), Some("old-session"), "{invalid_case}"); + assert!(s.calibration_sequence_fault_node_ids.contains(&5), "{invalid_case}"); + assert_eq!(std::fs::read(&path).unwrap(), bytes); + } + } +} + async fn calibration_start( State(state): State, Query(query): Query, @@ -15236,6 +15456,7 @@ async fn main() { .route("/api/v1/calibration/cancel", post(calibration_cancel)) .route("/api/v1/calibration/status", get(calibration_status)) .route("/api/v1/calibration/reset", post(calibration_reset)) + .route("/api/v1/calibration/bootstrap/recover", post(calibration_recover_bootstrap)) // ADR-044 §5.3: runtime-configurable dedup factor .route( "/api/v1/config/dedup-factor", From 33aadc9b9a58d9a5677e5e8460c0cc8f3681ee4d Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Mon, 12 Oct 2026 18:10:51 +0200 Subject: [PATCH 002/100] feat(companion): add owner-approved pinned TLS pairing and shared rooms --- .github/workflows/ci.yml | 7 + .../ADR-383-secure-local-companion-pairing.md | 76 ++ scripts/tests/companion_socket_probe.py | 94 ++ v2/Cargo.lock | 5 + .../wifi-densepose-sensing-server/Cargo.toml | 5 + .../src/bearer_auth.rs | 9 + .../src/companion.rs | 840 ++++++++++++++++++ .../src/companion_integration.rs | 289 ++++++ .../src/companion_room.rs | 395 ++++++++ .../src/companion_tls.rs | 141 +++ .../src/discovery.rs | 9 + .../wifi-densepose-sensing-server/src/lib.rs | 3 + .../wifi-densepose-sensing-server/src/main.rs | 36 +- 13 files changed, 1906 insertions(+), 3 deletions(-) create mode 100644 docs/adr/ADR-383-secure-local-companion-pairing.md create mode 100644 scripts/tests/companion_socket_probe.py create mode 100644 v2/crates/wifi-densepose-sensing-server/src/companion.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/companion_room.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/companion_tls.rs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 205ba5c270..0674804f59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,6 +139,13 @@ jobs: CARGO_PROFILE_TEST_DEBUG: "0" run: cargo test --workspace --no-default-features + - name: Verify secure companion over real sockets + env: + CARGO_PROFILE_DEV_DEBUG: "0" + run: | + cargo build --manifest-path v2/Cargo.toml --locked -p wifi-densepose-sensing-server --bin sensing-server + python3 scripts/tests/companion_socket_probe.py v2/target/debug/sensing-server + - name: Run ADR-147 worldmodel tests working-directory: v2 env: diff --git a/docs/adr/ADR-383-secure-local-companion-pairing.md b/docs/adr/ADR-383-secure-local-companion-pairing.md new file mode 100644 index 0000000000..85d903670c --- /dev/null +++ b/docs/adr/ADR-383-secure-local-companion-pairing.md @@ -0,0 +1,76 @@ +# ADR-383: Secure local Mac–iPhone companion pairing + +- Status: Accepted decision (2026-10-12); implementation and verification in progress +- Decision owner: product owner, approved in the implementation session +- Related: ADR-271 (OAuth), ADR-272 (WebSocket tickets); ruview-mobile ADR-046 + +## Context + +The managed Mac helper uses an ephemeral, full-authority owner token. Earlier +phone testing connected to an unauthenticated LAN helper. Exporting the owner +token or disabling authentication would not establish a safe public companion +workflow. An IP address and Bonjour identity are routing hints, not trust anchors. + +## Decision + +Use a separate opt-in TLS companion listener (managed Mac port 3443), with a +persistent installation certificate. The owner app remains on authenticated +loopback HTTP. Its token never leaves that owner channel. Companion management +routes require both the actual loopback socket peer and the exact owner token; +OAuth administrator authority and forwarded headers do not substitute for either. + +The Mac displays a five-minute, one-use QR invitation containing version, +installation ID, HTTPS local endpoint, certificate DER SHA-256 fingerprint, +invitation ID, and a 256-bit secret. The phone pins the certificate before sending +any secret. It generates an independent 256-bit device secret, submits enrollment, +and displays a comparison code. The Mac operator approves the matching request. +Observing the invitation alone cannot retrieve an approved phone's credential. + +The phone stores its device secret and certificate pin in device-only Keychain. +The server persists only device-secret hashes. Pairing lasts at most 30 days; +expiry requires explicit pairing again. A paired phone obtains a 12-hour access +token, capped by pairing expiry; access tokens and calibration grants are not +persisted on the server. Restart refresh uses the durable device proof over pinned +TLS. Permission to calibrate is a separate owner-granted 15-minute capability. +Read authority never authorizes model, recording, configuration or administrator +mutations. Existing OAuth scope semantics are unchanged. + +Companion WebSockets use native authenticated TLS upgrades, not legacy browser +WS tickets. The initial supported stream is `/ws/sensing`. Unsupported streams +fail closed. Revocation and access expiry close established streams and prevent +new requests. Corrupt private identity/grant state or failed grant persistence +fails closed. No authentication disablement, certificate trust fallback, redirect +following or plaintext fallback is allowed. + +Bonjour may rediscover a route, but the persisted certificate pin and installation +must still match. DHCP changes do not change pairing identity. There is no cloud +relay or automatic certificate rotation in this decision; lost identity requires +explicit re-pairing. This is distinct from Finder/Xcode device pairing and Cognitum +account login. + +## Shared calibration authority + +The Mac explicitly publishes one bounded current room geometry and RF placement +revision. Publication uses compare-and-swap; phone import is explicit. It does not +transfer raw CSI, photos, recordings, teacher samples, models, historical checks +or full-evidence receipts. Both clients derive the same installation/room/revision +binding from canonical integer millimetres and heading tenths, independently of +loopback, LAN origin and transport. Geometry changes require republishing. +Companion calibration must match the published digest and complete node set. +Existing server boot/session, accepted-grid and evidence checks remain mandatory. +A phone cannot label an old receipt current or shorten the required baseline. + +## Deployment, rollback and verification + +Land producer and consumer PRs separately and bundle the identified helper only +after both protocols are verified. An older helper must leave pairing unavailable +without weakening authentication. Roll back the app/helper together while retaining +private pairing files; no rollback silently reenables unauthenticated LAN access. + +Required verification: exact compiled helper over real sockets; owner peer/token +isolation; QR expiry/replay/duplicate enrollment; unknown and oversized input; +wrong certificate/installation and redirect rejection; read-versus-calibration +scope; revocation of live streams; durable restart and corrupt-state behavior; +shared-room digest parity, revision conflicts and no stale-evidence import; native +iOS and Mac builds; actual phone–Mac pairing and reconnect. Synthetic tests and +authenticated transport are not physical sensing or room-selectivity acceptance. diff --git a/scripts/tests/companion_socket_probe.py b/scripts/tests/companion_socket_probe.py new file mode 100644 index 0000000000..5939845643 --- /dev/null +++ b/scripts/tests/companion_socket_probe.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Real helper socket contract. Synthetic isolated state, never the live room.""" +import argparse, base64, hashlib, http.client, json, os, pathlib, secrets, socket, ssl, subprocess, tempfile, time + +def port(): + with socket.socket() as s: s.bind(('127.0.0.1',0)); return s.getsockname()[1] + +def main(): + parser=argparse.ArgumentParser(); parser.add_argument('binary'); args=parser.parse_args() + ports=[port() for _ in range(4)]; http_port,ws,udp,tls=ports + owner=secrets.token_urlsafe(32); device=secrets.token_urlsafe(32); proof=secrets.token_urlsafe(32) + context=ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT); context.check_hostname=False; context.verify_mode=ssl.CERT_NONE + count=0 + def check(condition,label): + nonlocal count + assert condition,label + count+=1; print('PASS',label,flush=True) + with tempfile.TemporaryDirectory(prefix='ruview-companion-probe-') as directory: + root=pathlib.Path(directory); proc=None; log=(root/'server.log').open('w') + def start(): + env=os.environ.copy();env['RUVIEW_API_TOKEN']=owner + p=subprocess.Popen([str(pathlib.Path(args.binary).resolve()),'--source','simulated','--bind-addr','127.0.0.1','--http-port',str(http_port),'--ws-port',str(ws),'--udp-port',str(udp),'--companion-port',str(tls),'--installation-id','companion-probe','--data-dir',directory,'--no-mdns','--no-edge-registry'],env=env,stdout=log,stderr=subprocess.STDOUT) + for _ in range(200): + if p.poll() is not None: raise RuntimeError('helper exited; '+(root/'server.log').read_text()[-1600:]) + try: + with socket.create_connection(('127.0.0.1',http_port),timeout=.2):return p + except OSError:time.sleep(.05) + raise RuntimeError('helper did not bind') + def request(path,method='GET',body=None,token=None,secure=False,pin=None): + conn=http.client.HTTPSConnection('127.0.0.1',tls,context=context,timeout=5) if secure else http.client.HTTPConnection('127.0.0.1',http_port,timeout=5) + conn.connect() + if secure and pin: assert hashlib.sha256(conn.sock.getpeercert(binary_form=True)).hexdigest()==pin,'TLS pin changed' + headers={'Content-Type':'application/json'} + if token:headers['Authorization']='Bearer '+token + if secure:headers['X-RuView-Installation-ID']='companion-probe' + conn.request(method,path,json.dumps(body) if body is not None else None,headers) + res=conn.getresponse(); status=res.status;raw=res.read();conn.close() + try:value=json.loads(raw) + except ValueError:value={} + return status,value + try: + proc=start() + check(request('/api/v1/companion')[0]==403,'owner endpoint rejects absent credential') + status,invite=request('/api/v1/companion/invites','POST',{},owner) + check(status==200 and invite['schema']=='ruview.companion.invite.v1','owner creates bounded invitation') + pin=invite['cert_sha256'] + payload={k:invite[k] for k in ['installation_id','invite_id','invite_secret']};payload.update(device_id=device,device_name='Contract probe',pairing_secret=proof) + status,enrolled=request('/companion/v1/enroll','POST',payload,secure=True,pin=pin) + check(status==200 and enrolled['status']=='pending','pinned enrollment pending owner') + session={'installation_id':'companion-probe','device_id':device,'pairing_secret':proof} + check(request('/companion/v1/session','POST',session,secure=True,pin=pin)[1]['status']=='pending','QR cannot autoapprove') + check(request('/companion/v1/enroll','POST',payload,secure=True,pin=pin)[0]>=400,'invitation cannot replay') + check(request('/api/v1/companion/requests/'+enrolled['request_id']+'/approve','POST',{'comparison_code':enrolled['comparison_code']},owner)[0]==200,'owner approves matched request') + status,access=request('/companion/v1/session','POST',session,secure=True,pin=pin);token=access['access_token'] + check(status==200,'device obtains read session') + check(request('/api/v1/status',token=token,secure=True,pin=pin)[0]==200,'read token works over pinned TLS') + check(request('/api/v1/status',token=token)[0]==401,'companion token rejected over plaintext') + check(request('/api/v1/companion',token=owner,secure=True,pin=pin)[0] in (403,404),'owner token cannot manage over TLS LAN listener') + check(request('/api/v1/ws-ticket','POST',{},token,True,pin)[0]==403,'legacy browser ticket denied') + check(request('/api/v1/calibration/start','POST',{},token,True,pin)[0]==403,'read grant cannot calibrate') + check(request('/api/v1/companion/devices/'+device+'/calibration','POST',{},owner)[0]==200,'owner grants calibration separately') + status,result=request('/api/v1/calibration/start','POST',{'binding_digest':'a'*64,'source_node_ids':[1]},token,True,pin) + check(status==200 and result.get('error_code')=='shared_room_binding_required','calibration rejects unshared room') + check(request('/api/v1/calibration/reset','POST',{},token,True,pin)[0]==403,'calibration grant is not administrator') + room={'schema':'ruview.companion.room.v1','id':'room-1','name':'Probe room','widthM':5,'lengthM':6,'heightM':2.6,'nodes':[{'nodeId':1,'slot':'north_west','xM':.45,'yM':.45,'role':'receiver','headingDeg':90,'orientationConfirmed':True,'confirmedAtUnixMs':1}],'accessPoint':{'slot':'north_east','xM':4.55,'yM':.45,'provenance':'operator_entered','confirmedAtUnixMs':1}} + status,published=request('/api/v1/companion/room','PUT',{'expected_revision':0,'room':room},owner) + check(status==200 and published['revision']==1,'owner publishes canonical room') + check(request('/companion/v1/room',token=token,secure=True,pin=pin)[1]==published,'paired phone reads exact shared room') + check(request('/api/v1/companion/room','PUT',{'expected_revision':0,'room':room},owner)[0]==409,'room publication rejects stale revision') + raw=socket.create_connection(('127.0.0.1',tls),timeout=5);stream=context.wrap_socket(raw,server_hostname='localhost') + check(hashlib.sha256(stream.getpeercert(binary_form=True)).hexdigest()==pin,'stream server pin matches') + key=base64.b64encode(secrets.token_bytes(16)).decode() + stream.sendall(('GET /ws/sensing HTTP/1.1\r\nHost: localhost\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: '+key+'\r\nAuthorization: Bearer '+token+'\r\nX-RuView-Installation-ID: companion-probe\r\n\r\n').encode()) + check(b'101 Switching Protocols' in stream.recv(4096),'authenticated live WebSocket upgrades') + proc.terminate();proc.wait(timeout=10);proc=start() + check(request('/api/v1/status',token=token,secure=True,pin=pin)[0]==401,'restart invalidates ephemeral access token') + status,access=request('/companion/v1/session','POST',session,secure=True,pin=pin);token=access['access_token'] + check(status==200 and request('/api/v1/status',token=token,secure=True,pin=pin)[0]==200,'durable device proof refreshes after restart') + stream.close() + raw=socket.create_connection(('127.0.0.1',tls),timeout=5);stream=context.wrap_socket(raw,server_hostname='localhost');stream.sendall(('GET /ws/sensing HTTP/1.1\r\nHost: localhost\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: '+key+'\r\nAuthorization: Bearer '+token+'\r\nX-RuView-Installation-ID: companion-probe\r\n\r\n').encode());stream.recv(4096) + check(request('/api/v1/companion/devices/'+device,'DELETE',token=owner)[0]==200,'owner revokes phone') + check(request('/api/v1/status',token=token,secure=True,pin=pin)[0]==401,'revocation blocks existing read token') + check(request('/companion/v1/session','POST',session,secure=True,pin=pin)[0]==401,'revocation blocks durable refresh') + stream.settimeout(3);closed=False + try: + for _ in range(100): + if not stream.recv(65536):closed=True;break + except (ssl.SSLEOFError,ConnectionResetError):closed=True + check(closed,'revocation closes already-open WebSocket');stream.close() + print('COMPANION SOCKET CONTRACT PASS',count) + finally: + if proc and proc.poll() is None:proc.terminate();proc.wait(timeout=10) + log.close() +if __name__=='__main__':main() diff --git a/v2/Cargo.lock b/v2/Cargo.lock index ab4f302e9a..edfd982b29 100644 --- a/v2/Cargo.lock +++ b/v2/Cargo.lock @@ -13481,6 +13481,8 @@ dependencies = [ "criterion", "futures-util", "hmac", + "hyper", + "hyper-util", "jsonwebtoken", "mdns-sd", "midstreamer-attractor", @@ -13491,7 +13493,9 @@ dependencies = [ "p256", "proptest", "rand 0.8.5", + "rcgen", "rumqttc-v4-next", + "rustls", "ruvector-mincut", "ruview-auth", "serde", @@ -13501,6 +13505,7 @@ dependencies = [ "tempfile", "thiserror 1.0.69", "tokio", + "tokio-rustls", "tokio-tungstenite", "tower 0.4.13", "tower-http", diff --git a/v2/crates/wifi-densepose-sensing-server/Cargo.toml b/v2/crates/wifi-densepose-sensing-server/Cargo.toml index eaae6dfc34..ab2a191001 100644 --- a/v2/crates/wifi-densepose-sensing-server/Cargo.toml +++ b/v2/crates/wifi-densepose-sensing-server/Cargo.toml @@ -27,6 +27,11 @@ tokio = { workspace = true, features = ["full", "process"] } futures-util = "0.3" ruvector-mincut = { workspace = true } mdns-sd = "0.11" +rcgen = "0.12.1" +rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } +tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } +hyper = { workspace = true } +hyper-util = { version = "0.1", features = ["server-auto", "tokio", "service"] } # Serialization serde = { workspace = true } diff --git a/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs b/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs index 9516c80a32..49667f3f18 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs @@ -424,6 +424,11 @@ impl AuthState { self.oauth.is_some() } + /// Exact owner capability check; caller must also verify a loopback socket peer. + pub fn is_owner_token(&self, supplied: &str) -> bool { + self.token.as_ref().is_some_and(|expected| ct_eq(supplied.as_bytes(), expected.as_bytes())) + } + /// Whether the legacy static `RUVIEW_API_TOKEN` is configured. pub fn static_token_enabled(&self) -> bool { self.token.is_some() @@ -521,6 +526,10 @@ pub async fn require_bearer( mut request: Request, next: Next, ) -> Response { + // Only the dedicated TLS router can insert this verified principal. + if request.extensions().get::().is_some() { + return next.run(request).await; + } if !auth.is_enabled() { return next.run(request).await; } diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion.rs b/v2/crates/wifi-densepose-sensing-server/src/companion.rs new file mode 100644 index 0000000000..55f0fcfdff --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/companion.rs @@ -0,0 +1,840 @@ +//! Owner-approved, QR-pinned companion pairing; durable secrets are hashed. +use axum::{ + extract::{DefaultBodyLimit, Path as RoutePath, State}, + http::StatusCode, + routing::{delete, get, post}, + Json, Router, +}; +use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; +use rand::{rngs::OsRng, RngCore}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +use std::{ + collections::HashMap, + io::{self, Write}, + path::{Path, PathBuf}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, + }, + time::{SystemTime, UNIX_EPOCH}, +}; +use tokio::sync::watch; +pub fn secret() -> String { + let mut b = [0; 32]; + OsRng.fill_bytes(&mut b); + URL_SAFE_NO_PAD.encode(b) +} +fn hash(s: &str) -> String { + format!("{:x}", Sha256::digest(s.as_bytes())) +} +fn now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() +} +fn valid_secret(s: &str) -> bool { + s.len() == 43 && URL_SAFE_NO_PAD.decode(s).is_ok_and(|b| b.len() == 32) +} +fn valid_id(s: &str) -> bool { + !s.is_empty() + && s.len() <= 64 + && s.bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') +} +pub(crate) fn private_dir(path: &Path) -> io::Result<()> { + std::fs::create_dir_all(path)?; + if std::fs::symlink_metadata(path)?.file_type().is_symlink() { + return Err(io::Error::other("symlink state directory")); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o700))?; + } + Ok(()) +} +pub(crate) fn private_read(path: &Path) -> io::Result> { + let meta = std::fs::symlink_metadata(path)?; + if !meta.is_file() || meta.len() > 65536 { + return Err(io::Error::other("invalid private state file")); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if meta.permissions().mode() & 0o077 != 0 { + return Err(io::Error::other("insecure private state permissions")); + } + } + std::fs::read(path) +} +pub(crate) fn atomic_private_write(path: &Path, data: &[u8]) -> io::Result<()> { + let temp = path.with_extension(format!("tmp-{}", secret())); + let mut options = std::fs::OpenOptions::new(); + options.create_new(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut f = options.open(&temp)?; + f.write_all(data)?; + f.sync_all()?; + std::fs::rename(&temp, path)?; + std::fs::File::open(path.parent().ok_or_else(|| io::Error::other("parent"))?)?.sync_all()?; + Ok(()) +} +#[derive(Clone, Serialize, Deserialize)] +struct Device { + device_id: String, + device_name: String, + pairing_hash: String, + expires_at: u64, +} +#[derive(Serialize, Deserialize)] +struct Disk { + version: u32, + installation_id: String, + devices: Vec, +} +struct Invite { + hash: String, + expires: u64, +} +struct Pending { + device: Device, + code: String, + expires: u64, +} +struct Access { + device_id: String, + expires: u64, +} +struct Inner { + devices: HashMap, + invites: HashMap, + pending: HashMap, + access: HashMap, + calibration: HashMap, + revocations: HashMap>, +} +pub struct Companion { + pub installation_id: String, + pub cert_sha256: String, + pub origin: String, + path: PathBuf, + fault: AtomicBool, + inner: Mutex, +} +pub type Shared = Arc; +pub type ApiResult = Result, StatusCode>; +#[derive(Clone)] +pub struct Grant { + pub device_id: String, + pub expires_at: u64, + pub revoked: watch::Receiver, +} +impl Companion { + pub fn load( + dir: &Path, + installation_id: String, + cert_sha256: String, + origin: String, + ) -> io::Result { + private_dir(dir)?; + if !origin.starts_with("https://") { + return Err(io::Error::other("https required")); + } + let path = dir.join("companion-grants.json"); + let devices = if path.exists() { + let disk: Disk = + serde_json::from_slice(&private_read(&path)?).map_err(io::Error::other)?; + if disk.version != 1 + || disk.installation_id != installation_id + || disk.devices.len() > 16 + { + return Err(io::Error::other("invalid grants")); + } + let mut m = HashMap::new(); + for d in disk.devices { + if !valid_id(&d.device_id) + || d.pairing_hash.len() != 64 + || m.insert(d.device_id.clone(), d).is_some() + { + return Err(io::Error::other("invalid device")); + } + } + m + } else { + HashMap::new() + }; + Ok(Arc::new(Self { + installation_id, + cert_sha256, + origin, + path, + fault: AtomicBool::new(false), + inner: Mutex::new(Inner { + devices, + invites: HashMap::new(), + pending: HashMap::new(), + access: HashMap::new(), + calibration: HashMap::new(), + revocations: HashMap::new(), + }), + })) + } + fn persist(&self, devices: &HashMap, inner: &Inner) -> Result<(), StatusCode> { + let disk = Disk { + version: 1, + installation_id: self.installation_id.clone(), + devices: devices.values().cloned().collect(), + }; + atomic_private_write( + &self.path, + &serde_json::to_vec(&disk).map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?, + ) + .map_err(|_| { + self.fault.store(true, Ordering::SeqCst); + for tx in inner.revocations.values() { + let _ = tx.send(true); + } + StatusCode::SERVICE_UNAVAILABLE + }) + } + pub fn authorize( + &self, + installation: &str, + token: &str, + calibration: bool, + ) -> Result { + if self.fault.load(Ordering::SeqCst) { + return Err(StatusCode::SERVICE_UNAVAILABLE); + } + if installation != self.installation_id || !valid_secret(token) { + return Err(StatusCode::UNAUTHORIZED); + } + let mut i = self + .inner + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + let t = now(); + let a = i.access.get(&hash(token)).ok_or(StatusCode::UNAUTHORIZED)?; + let d = i + .devices + .get(&a.device_id) + .ok_or(StatusCode::UNAUTHORIZED)?; + if a.expires <= t || d.expires_at <= t { + return Err(StatusCode::UNAUTHORIZED); + } + if calibration + && i.calibration + .get(&d.device_id) + .copied() + .filter(|until| *until > now()) + .unwrap_or(0) + <= t + { + return Err(StatusCode::FORBIDDEN); + } + let device_id = d.device_id.clone(); + let expires_at = a.expires.min(d.expires_at); + let revoked = i + .revocations + .entry(device_id.clone()) + .or_insert_with(|| watch::channel(false).0) + .subscribe(); + Ok(Grant { + device_id, + expires_at, + revoked, + }) + } +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Enroll { + installation_id: String, + invite_id: String, + invite_secret: String, + device_id: String, + device_name: String, + pairing_secret: String, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Session { + installation_id: String, + device_id: String, + pairing_secret: String, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Approve { + comparison_code: String, +} +pub fn public_router(state: Shared) -> Router { + Router::new() + .route("/companion/v1/enroll", post(enroll)) + .route("/companion/v1/session", post(session)) + .layer(DefaultBodyLimit::max(4096)) + .with_state(state) +} +/// Caller must wrap these endpoints in loopback-only owner authentication. +pub fn owner_router(state: Shared) -> Router { + Router::new() + .route("/api/v1/companion", get(snapshot)) + .route("/api/v1/companion/invites", post(invite)) + .route("/api/v1/companion/requests/:id/approve", post(approve)) + .route("/api/v1/companion/devices/:id", delete(revoke)) + .route( + "/api/v1/companion/devices/:id/calibration", + post(calibration), + ) + .layer(DefaultBodyLimit::max(4096)) + .with_state(state) +} +async fn invite(State(s): State) -> ApiResult { + let mut i = s + .inner + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + i.invites.retain(|_, v| v.expires > now()); + if i.invites.len() >= 4 { + return Err(StatusCode::TOO_MANY_REQUESTS); + } + let id = secret(); + let token = secret(); + let expires = now() + 300; + i.invites.insert( + id.clone(), + Invite { + hash: hash(&token), + expires, + }, + ); + Ok(Json( + json!({"schema":"ruview.companion.invite.v1","installation_id":s.installation_id,"cert_sha256":s.cert_sha256,"origin":s.origin,"invite_id":id,"invite_secret":token,"expires_at":expires}), + )) +} +async fn enroll(State(s): State, Json(r): Json) -> ApiResult { + if r.installation_id != s.installation_id + || !valid_id(&r.device_id) + || !valid_secret(&r.pairing_secret) + || r.device_name.trim().is_empty() + || r.device_name.len() > 80 + || r.device_name.chars().any(char::is_control) + { + return Err(StatusCode::BAD_REQUEST); + } + let mut i = s + .inner + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + let t = now(); + i.pending.retain(|_, p| p.expires > t); + if i.pending.len() >= 8 || i.devices.len() >= 16 { + return Err(StatusCode::TOO_MANY_REQUESTS); + } + if i.devices.contains_key(&r.device_id) + || i.pending + .values() + .any(|p| p.device.device_id == r.device_id) + { + return Err(StatusCode::CONFLICT); + } + let inv = i + .invites + .get(&r.invite_id) + .ok_or(StatusCode::UNAUTHORIZED)?; + if inv.expires <= t || inv.hash != hash(&r.invite_secret) { + return Err(StatusCode::UNAUTHORIZED); + } + i.invites.remove(&r.invite_id); + let request_id = secret(); + let code = format!("{:06}", OsRng.next_u32() % 1_000_000); + i.pending.insert( + request_id.clone(), + Pending { + device: Device { + device_id: r.device_id, + device_name: r.device_name, + pairing_hash: hash(&r.pairing_secret), + expires_at: t + 30 * 86400, + }, + code: code.clone(), + expires: t + 300, + }, + ); + Ok(Json( + json!({"request_id":request_id,"comparison_code":code,"status":"pending"}), + )) +} +async fn approve( + State(s): State, + RoutePath(id): RoutePath, + Json(r): Json, +) -> ApiResult { + let mut i = s + .inner + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + let p = i.pending.get(&id).ok_or(StatusCode::NOT_FOUND)?; + if p.expires <= now() || p.code != r.comparison_code { + return Err(StatusCode::FORBIDDEN); + } + let mut devices = i.devices.clone(); + devices.insert(p.device.device_id.clone(), p.device.clone()); + s.persist(&devices, &i)?; + i.devices = devices; + i.pending.remove(&id); + Ok(Json(json!({"status":"approved"}))) +} +async fn session(State(s): State, Json(r): Json) -> ApiResult { + if s.fault.load(Ordering::SeqCst) { + return Err(StatusCode::SERVICE_UNAVAILABLE); + } + if r.installation_id != s.installation_id || !valid_secret(&r.pairing_secret) { + return Err(StatusCode::UNAUTHORIZED); + } + let mut i = s + .inner + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + let t = now(); + let h = hash(&r.pairing_secret); + if i.pending + .values() + .any(|p| p.device.device_id == r.device_id && p.device.pairing_hash == h && p.expires > t) + { + return Ok(Json(json!({"status":"pending"}))); + } + let d = i + .devices + .get(&r.device_id) + .ok_or(StatusCode::UNAUTHORIZED)?; + if d.pairing_hash != h || d.expires_at <= t { + return Err(StatusCode::UNAUTHORIZED); + } + let expires_at = (t + 12 * 3600).min(d.expires_at); + i.access + .retain(|_, a| a.expires > t && a.device_id != r.device_id); + let token = secret(); + i.access.insert( + hash(&token), + Access { + device_id: r.device_id.clone(), + expires: expires_at, + }, + ); + Ok(Json( + json!({"status":"approved","access_token":token,"expires_at":expires_at,"installation_id":s.installation_id,"device_id":r.device_id,"calibration_until":i.calibration.get(&r.device_id).copied().filter(|until| *until > now()).unwrap_or(0)}), + )) +} +async fn snapshot(State(s): State) -> ApiResult { + let i = s + .inner + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + let requests:Vec<_>=i.pending.iter().filter(|(_,p)|p.expires>now()).map(|(id,p)|json!({"request_id":id,"device_id":p.device.device_id,"device_name":p.device.device_name,"comparison_code":p.code,"expires_at":p.expires})).collect(); + let devices:Vec<_>=i.devices.values().map(|d|json!({"device_id":d.device_id,"device_name":d.device_name,"expires_at":d.expires_at,"calibration_until":i.calibration.get(&d.device_id).copied().filter(|until| *until > now()).unwrap_or(0)})).collect(); + Ok(Json( + json!({"installation_id":s.installation_id,"origin":s.origin,"cert_sha256":s.cert_sha256,"requests":requests,"devices":devices}), + )) +} +async fn revoke(State(s): State, RoutePath(id): RoutePath) -> ApiResult { + let mut i = s + .inner + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + let mut devices = i.devices.clone(); + if devices.remove(&id).is_none() { + return Err(StatusCode::NOT_FOUND); + } + s.persist(&devices, &i)?; + i.devices = devices; + i.access.retain(|_, a| a.device_id != id); + i.calibration.remove(&id); + if let Some(tx) = i.revocations.remove(&id) { + let _ = tx.send(true); + } + Ok(Json(json!({"status":"revoked"}))) +} +async fn calibration(State(s): State, RoutePath(id): RoutePath) -> ApiResult { + let mut i = s + .inner + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + if !i.devices.get(&id).is_some_and(|d| d.expires_at > now()) { + return Err(StatusCode::NOT_FOUND); + } + // An owner grant is an explicit controller handoff, never a second controller. + let until = now() + 900; + i.calibration.clear(); + i.calibration.insert(id, until); + Ok(Json(json!({"calibration_until":until}))) +} +#[cfg(test)] +mod tests { + use super::*; + fn setup() -> (tempfile::TempDir, Shared) { + let d = tempfile::tempdir().unwrap(); + let s = Companion::load( + d.path(), + "installation".into(), + "a".repeat(64), + "https://ruview.local:3443".into(), + ) + .unwrap(); + (d, s) + } + async fn pending(s: &Shared) -> (String, String, String) { + let v = invite(State(s.clone())).await.unwrap().0; + let proof = secret(); + let v = enroll( + State(s.clone()), + Json(Enroll { + installation_id: s.installation_id.clone(), + invite_id: v["invite_id"].as_str().unwrap().into(), + invite_secret: v["invite_secret"].as_str().unwrap().into(), + device_id: "phone".into(), + device_name: "Phone".into(), + pairing_secret: proof.clone(), + }), + ) + .await + .unwrap() + .0; + ( + v["request_id"].as_str().unwrap().into(), + v["comparison_code"].as_str().unwrap().into(), + proof, + ) + } + async fn access(s: &Shared, proof: &str) -> ApiResult { + session( + State(s.clone()), + Json(Session { + installation_id: s.installation_id.clone(), + device_id: "phone".into(), + pairing_secret: proof.into(), + }), + ) + .await + } + #[tokio::test] + async fn approval_read_scope_calibration_revocation() { + let (_d, s) = setup(); + let (id, code, proof) = pending(&s).await; + assert_eq!(access(&s, &proof).await.unwrap().0["status"], "pending"); + assert!(approve( + State(s.clone()), + RoutePath(id.clone()), + Json(Approve { + comparison_code: "wrong".into() + }) + ) + .await + .is_err()); + approve( + State(s.clone()), + RoutePath(id), + Json(Approve { + comparison_code: code, + }), + ) + .await + .unwrap(); + let v = access(&s, &proof).await.unwrap().0; + let token = v["access_token"].as_str().unwrap(); + let grant = s.authorize("installation", token, false).unwrap(); + assert!(s.authorize("other", token, false).is_err()); + assert!(s.authorize("installation", token, true).is_err()); + calibration(State(s.clone()), RoutePath("phone".into())) + .await + .unwrap(); + assert!(s.authorize("installation", token, true).is_ok()); + revoke(State(s.clone()), RoutePath("phone".into())) + .await + .unwrap(); + assert!(*grant.revoked.borrow()); + assert!(s.authorize("installation", token, false).is_err()); + assert!(access(&s, &proof).await.is_err()); + } + #[tokio::test] + async fn durable_pair_not_access_and_no_plaintext() { + let (d, s) = setup(); + let (id, code, proof) = pending(&s).await; + approve( + State(s.clone()), + RoutePath(id), + Json(Approve { + comparison_code: code, + }), + ) + .await + .unwrap(); + let v = access(&s, &proof).await.unwrap().0; + let token = v["access_token"].as_str().unwrap(); + let disk = std::fs::read_to_string(&s.path).unwrap(); + assert!(!disk.contains(&proof)); + assert!(!disk.contains(token)); + let loaded = Companion::load( + d.path(), + "installation".into(), + "a".repeat(64), + s.origin.clone(), + ) + .unwrap(); + assert!(loaded.authorize("installation", token, false).is_err()); + assert_eq!( + access(&loaded, &proof).await.unwrap().0["status"], + "approved" + ); + assert!( + Companion::load(d.path(), "wrong".into(), "a".repeat(64), s.origin.clone()).is_err() + ); + } + #[tokio::test] + async fn expires_and_cannot_replay_invite() { + let (_d, s) = setup(); + let v = invite(State(s.clone())).await.unwrap().0; + let make = || Enroll { + installation_id: s.installation_id.clone(), + invite_id: v["invite_id"].as_str().unwrap().into(), + invite_secret: v["invite_secret"].as_str().unwrap().into(), + device_id: secret(), + device_name: "Phone".into(), + pairing_secret: secret(), + }; + enroll(State(s.clone()), Json(make())).await.unwrap(); + assert!(enroll(State(s.clone()), Json(make())).await.is_err()); + let (id, code, proof) = pending(&s).await; + s.inner + .lock() + .unwrap() + .pending + .get_mut(&id) + .unwrap() + .expires = 0; + assert!(approve( + State(s.clone()), + RoutePath(id), + Json(Approve { + comparison_code: code + }) + ) + .await + .is_err()); + assert!(access(&s, &proof).await.is_err()); + } + #[tokio::test] + async fn persistence_failure_never_approves() { + let (_d, s) = setup(); + let (id, code, proof) = pending(&s).await; + std::fs::create_dir(&s.path).unwrap(); + assert_eq!( + approve( + State(s.clone()), + RoutePath(id), + Json(Approve { + comparison_code: code + }) + ) + .await + .unwrap_err(), + StatusCode::SERVICE_UNAVAILABLE + ); + assert!(s.inner.lock().unwrap().devices.is_empty()); + assert!(access(&s, &proof).await.is_err()); + } + #[tokio::test] + async fn renewal_invalidates_previous_and_pair_expiry() { + let (_d, s) = setup(); + let (id, code, proof) = pending(&s).await; + approve( + State(s.clone()), + RoutePath(id), + Json(Approve { + comparison_code: code, + }), + ) + .await + .unwrap(); + let first = access(&s, &proof).await.unwrap().0; + let second = access(&s, &proof).await.unwrap().0; + assert!(s + .authorize( + "installation", + first["access_token"].as_str().unwrap(), + false + ) + .is_err()); + assert!(s + .authorize( + "installation", + second["access_token"].as_str().unwrap(), + false + ) + .is_ok()); + s.inner + .lock() + .unwrap() + .devices + .get_mut("phone") + .unwrap() + .expires_at = 0; + assert!(access(&s, &proof).await.is_err()); + assert!(s + .authorize( + "installation", + second["access_token"].as_str().unwrap(), + false + ) + .is_err()); + } + #[tokio::test] + async fn router_rejects_unknown_fields_and_oversized_body() { + use tower::ServiceExt; + let (_d, s) = setup(); + let app = public_router(s); + let req=axum::http::Request::builder().method("POST").uri("/companion/v1/session").header("content-type","application/json").body(axum::body::Body::from(serde_json::to_vec(&json!({"installation_id":"installation","device_id":"phone","pairing_secret":secret(),"admin":true})).unwrap())).unwrap(); + assert_eq!( + app.clone().oneshot(req).await.unwrap().status(), + StatusCode::UNPROCESSABLE_ENTITY + ); + let req = axum::http::Request::builder() + .method("POST") + .uri("/companion/v1/session") + .header("content-type", "application/json") + .body(axum::body::Body::from("x".repeat(5000))) + .unwrap(); + assert_eq!( + app.oneshot(req).await.unwrap().status(), + StatusCode::PAYLOAD_TOO_LARGE + ); + } + #[tokio::test] + async fn invite_bound_and_duplicate_device_cannot_replace_proof() { + let (_d, s) = setup(); + let (id, code, proof) = pending(&s).await; + approve( + State(s.clone()), + RoutePath(id), + Json(Approve { + comparison_code: code, + }), + ) + .await + .unwrap(); + let v = invite(State(s.clone())).await.unwrap().0; + let attempted = enroll( + State(s.clone()), + Json(Enroll { + installation_id: s.installation_id.clone(), + invite_id: v["invite_id"].as_str().unwrap().into(), + invite_secret: v["invite_secret"].as_str().unwrap().into(), + device_id: "phone".into(), + device_name: "attacker".into(), + pairing_secret: secret(), + }), + ) + .await; + assert_eq!(attempted.unwrap_err(), StatusCode::CONFLICT); + assert_eq!(access(&s, &proof).await.unwrap().0["status"], "approved"); + for _ in 0..3 { + invite(State(s.clone())).await.unwrap(); + } + assert_eq!( + invite(State(s)).await.unwrap_err(), + StatusCode::TOO_MANY_REQUESTS + ); + } + #[tokio::test] + async fn calibration_handoff_is_exclusive_and_snapshot_expires() { + let (_dir, s) = setup(); + let (id, code, proof) = pending(&s).await; + approve( + State(s.clone()), + RoutePath(id), + Json(Approve { + comparison_code: code, + }), + ) + .await + .unwrap(); + let first = access(&s, &proof).await.unwrap().0; + let first_token = first["access_token"].as_str().unwrap(); + // A second approved device has its own credential and access token. + let second_proof = secret(); + { + let mut i = s.inner.lock().unwrap(); + i.devices.insert( + "phone-two".into(), + Device { + device_id: "phone-two".into(), + device_name: "Second phone".into(), + pairing_hash: hash(&second_proof), + expires_at: now() + 86400, + }, + ); + } + let second = session( + State(s.clone()), + Json(Session { + installation_id: s.installation_id.clone(), + device_id: "phone-two".into(), + pairing_secret: second_proof, + }), + ) + .await + .unwrap() + .0; + let second_token = second["access_token"].as_str().unwrap(); + calibration(State(s.clone()), RoutePath("phone".into())) + .await + .unwrap(); + assert!(s.authorize("installation", first_token, true).is_ok()); + calibration(State(s.clone()), RoutePath("phone-two".into())) + .await + .unwrap(); + assert!(matches!( + s.authorize("installation", first_token, true), + Err(StatusCode::FORBIDDEN) + )); + assert!(s.authorize("installation", second_token, true).is_ok()); + assert!(s.authorize("installation", first_token, false).is_ok()); + let view = snapshot(State(s.clone())).await.unwrap().0; + let devices = view["devices"].as_array().unwrap(); + assert_eq!( + devices.iter().find(|d| d["device_id"] == "phone").unwrap()["calibration_until"], + 0 + ); + assert!( + devices + .iter() + .find(|d| d["device_id"] == "phone-two") + .unwrap()["calibration_until"] + .as_u64() + .unwrap() + > now() + ); + s.inner + .lock() + .unwrap() + .calibration + .insert("phone-two".into(), now() - 1); + assert!(matches!( + s.authorize("installation", second_token, true), + Err(StatusCode::FORBIDDEN) + )); + let view = snapshot(State(s)).await.unwrap().0; + assert!(view["devices"] + .as_array() + .unwrap() + .iter() + .all(|d| d["calibration_until"] == 0)); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs new file mode 100644 index 0000000000..b2a3a45154 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs @@ -0,0 +1,289 @@ +//! Dedicated TLS authority: never infer trust from headers or LAN reachability. +use super::*; +use axum::{ + extract::{ConnectInfo, Request}, + middleware::Next, + response::Response, +}; +use wifi_densepose_sensing_server::{ + bearer_auth::AuthState, + companion::{self, Companion, Grant}, + companion_tls, +}; + +pub struct Integration { + service: Arc, + auth: AuthState, + listener: tokio::net::TcpListener, + tls: Arc, + room: Arc, + control: Control, + advertiser: Option, +} +impl Integration { + pub async fn prepare( + args: &Args, + state: &SharedState, + auth: AuthState, + ) -> std::io::Result> { + let Some(port) = args.companion_port else { + return Ok(None); + }; + if port == 0 + || port == args.http_port + || port == args.ws_port + || !auth.static_token_enabled() + { + return Err(std::io::Error::other( + "companion requires a dedicated port and owner credential", + )); + } + let dir = args.data_dir.join("companion"); + let identity = + companion_tls::Identity::load_or_create(&dir, args.installation_id.as_deref())?; + let hostname = + wifi_densepose_sensing_server::discovery::discovery_hostname(&identity.installation_id); + let origin = format!("https://{}:{port}", hostname.trim_end_matches('.')); + let service = Companion::load( + &dir, + identity.installation_id.clone(), + identity.cert_sha256, + origin, + )?; + state.write().await.installation_id = Some(identity.installation_id.clone()); + let ip: std::net::IpAddr = args.bind_addr.parse().map_err(std::io::Error::other)?; + let listener = tokio::net::TcpListener::bind(SocketAddr::from((ip, port))).await?; + let advertiser = if args.no_mdns || ip.is_loopback() { + None + } else { + Some( + wifi_densepose_sensing_server::discovery::start_tls_advertiser( + &format!("{} Companion", args.installation_name), + &identity.installation_id, + &hostname, + port, + ) + .map_err(std::io::Error::other)?, + ) + }; + let room = companion_room::SharedRoomStore::load(&dir, &identity.installation_id)?; + let control = Control { + state: state.clone(), + serial: Arc::new(tokio::sync::Mutex::new(())), + }; + Ok(Some(Self { + service, + auth, + listener, + tls: identity.config, + advertiser, + room, + control, + })) + } + pub fn owner_router(&self) -> Router { + companion::owner_router(self.service.clone()) + .merge(companion_room::owner_router(self.room.clone())) + .layer(axum::middleware::from_fn_with_state( + self.control.clone(), + serialize_control, + )) + .layer(axum::middleware::from_fn_with_state( + self.auth.clone(), + require_owner, + )) + } + pub fn protect_control(&self, app: Router) -> Router { + app.layer(Extension(self.room.clone())) + .layer(axum::middleware::from_fn_with_state( + self.control.clone(), + serialize_control, + )) + } + pub async fn start(self, app: Router) { + let app = app + .merge(companion_room::read_router(self.room.clone())) + .layer(axum::middleware::from_fn_with_state( + self.service.clone(), + require_companion, + )) + .merge(companion::public_router(self.service)) + .layer(axum::extract::DefaultBodyLimit::max(64 * 1024)); + tokio::spawn(async move { + let _advertiser = self.advertiser; + if let Err(error) = companion_tls::serve(self.listener, self.tls, app).await { + error!("Companion listener stopped: {error}"); + } + }); + } +} +#[derive(Clone)] +struct Control { + state: SharedState, + serial: Arc>, +} +async fn serialize_control( + State(control): State, + request: Request, + next: Next, +) -> Response { + let publish = request.method() == axum::http::Method::PUT + && request.uri().path() == "/api/v1/companion/room"; + let start = request.method() == axum::http::Method::POST + && matches!( + request.uri().path(), + "/api/v1/calibration/start" | "/api/v1/pose/calibrate" + ); + if !publish && !start { + return next.run(request).await; + } + let _guard = control.serial.lock().await; + if publish { + let state = control.state.read().await; + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + if state.field_model_status_at(now).is_some_and(|status| { + matches!( + status, + CalibrationStatus::Collecting | CalibrationStatus::Uncalibrated + ) + }) { + return ( + StatusCode::CONFLICT, + "Calibration is active; stop it before publishing another room revision", + ) + .into_response(); + } + } + next.run(request).await +} + +fn bearer(request: &Request) -> Option<&str> { + let (scheme, token) = request + .headers() + .get(axum::http::header::AUTHORIZATION)? + .to_str() + .ok()? + .split_once(' ')?; + if !scheme.eq_ignore_ascii_case("Bearer") { + return None; + }; + Some(token) +} +async fn require_owner(State(auth): State, request: Request, next: Next) -> Response { + let local = request + .extensions() + .get::>() + .is_some_and(|p| p.0.ip().is_loopback()); + if !local || !bearer(&request).is_some_and(|t| auth.is_owner_token(t)) { + return StatusCode::FORBIDDEN.into_response(); + } + next.run(request).await +} +fn policy(method: &axum::http::Method, path: &str) -> Option { + use axum::http::Method; + if *method == Method::GET || *method == Method::HEAD { + if matches!( + path, + "/health" + | "/api/v1/status" + | "/api/v1/info" + | "/api/v1/sensing/latest" + | "/api/v1/nodes" + | "/api/v1/stream/status" + | "/api/v1/vital-signs" + | "/api/v1/edge-vitals" + | "/api/v1/pose/current" + | "/api/v1/pose/stats" + | "/api/v1/pose/zones/summary" + | "/api/v1/pose/activities" + | "/api/v1/calibration/status" + | "/api/v1/pose/calibration/status" + | "/ws/sensing" + | "/companion/v1/room" + ) { + return Some(false); + } + } + if *method == Method::POST + && matches!( + path, + "/api/v1/calibration/start" + | "/api/v1/calibration/stop" + | "/api/v1/calibration/cancel" + | "/api/v1/calibration/bootstrap/promote" + | "/api/v1/calibration/bootstrap/refine" + ) + { + return Some(true); + } + None +} +async fn require_companion( + State(service): State>, + mut request: Request, + next: Next, +) -> Response { + let Some(calibration) = policy(request.method(), request.uri().path()) else { + return StatusCode::FORBIDDEN.into_response(); + }; + // Ticket and credential query parameters are never accepted on this listener. + if request.uri().query().is_some_and(|q| { + q.split('&').any(|part| { + matches!( + part.split('=').next(), + Some("ticket" | "token" | "access_token") + ) + }) + }) { + return StatusCode::BAD_REQUEST.into_response(); + } + let installation = request + .headers() + .get("x-ruview-installation-id") + .and_then(|v| v.to_str().ok()) + .unwrap_or(""); + let grant = match service.authorize(installation, bearer(&request).unwrap_or(""), calibration) { + Ok(grant) => grant, + Err(status) => return status.into_response(), + }; + request.extensions_mut().insert(grant); + next.run(request).await +} +pub async fn guard_stream(grant: Grant, stream: impl std::future::Future) { + let mut revoked = grant.revoked; + if *revoked.borrow() { + return; + } + let now = chrono::Utc::now().timestamp().max(0) as u64; + if grant.expires_at <= now { + return; + } + tokio::select! { + _ = stream => {}, + _ = revoked.changed() => {}, + _ = tokio::time::sleep(Duration::from_secs(grant.expires_at-now)) => {}, + } +} +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn device_policy_is_not_the_legacy_read_policy() { + use axum::http::Method; + assert_eq!(policy(&Method::GET, "/api/v1/sensing/latest"), Some(false)); + assert_eq!( + policy(&Method::POST, "/api/v1/calibration/start"), + Some(true) + ); + for path in [ + "/api/v1/ws-ticket", + "/api/v1/models/load", + "/api/v1/recording/start", + "/api/v1/calibration/reset", + "/api/v1/companion/invites", + ] { + assert_eq!(policy(&Method::POST, path), None, "{path}"); + } + assert_eq!(policy(&Method::GET, "/ws/train/progress"), None); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_room.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_room.rs new file mode 100644 index 0000000000..f91fde656d --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_room.rs @@ -0,0 +1,395 @@ +//! Installation-owned room publication. Authentication is supplied by the caller. +use axum::{extract::State, http::StatusCode, routing::get, Json, Router}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use sha2::{Digest, Sha256}; +#[cfg(unix)] +use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; +use std::{ + collections::HashSet, + fs::{self, OpenOptions}, + io::Write, + path::{Path, PathBuf}, + sync::{Arc, Mutex}, +}; +const MAX_SAFE: u64 = 9_007_199_254_740_991; +const SLOTS: &[&str] = &[ + "north_west", + "north_center", + "north_east", + "west_center", + "east_center", + "south_west", + "south_center", + "south_east", +]; +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Envelope { + pub installation_id: String, + pub revision: u64, + pub room: Option, + pub binding_digest: Option, +} +pub struct SharedRoomStore { + path: PathBuf, + current: Mutex, +} +fn invalid() -> std::io::Error { + std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid shared room state") +} +fn keys(v: &Value, allowed: &[&str]) -> bool { + v.as_object() + .is_some_and(|o| o.keys().all(|k| allowed.contains(&k.as_str()))) +} +fn text(v: &Value, min: usize, max: usize) -> bool { + v.as_str().is_some_and(|s| { + (min..=max).contains(&s.encode_utf16().count()) + && s.chars().all(|c| c >= ' ' && c != '\u{7f}') + }) +} +fn number(v: &Value, min: f64, max: f64, scale: f64) -> bool { + v.as_f64().is_some_and(|n| { + n.is_finite() && n >= min && n <= max && (n * scale - (n * scale).round()).abs() < 1e-8 + }) +} +fn position(v: &Value, width: f64, length: f64, slots: &HashSet) -> bool { + v["slot"] + .as_str() + .is_some_and(|s| SLOTS.contains(&s) && !slots.contains(s)) + && number(&v["xM"], 0., width, 1000.) + && number(&v["yM"], 0., length, 1000.) + && v["confirmedAtUnixMs"] + .as_u64() + .is_some_and(|n| n > 0 && n <= MAX_SAFE) +} +fn validate(mut r: Value) -> Result { + if !keys( + &r, + &[ + "schema", + "id", + "name", + "widthM", + "lengthM", + "heightM", + "nodes", + "accessPoint", + ], + ) || r["schema"] != "ruview.companion.room.v1" + || !text(&r["id"], 4, 96) + || !text(&r["name"], 1, 48) + || !number(&r["widthM"], 1.5, 30., 1000.) + || !number(&r["lengthM"], 1.5, 30., 1000.) + || !number(&r["heightM"], 1.8, 6., 1000.) + || r.get("accessPoint").is_none() + { + return Err(invalid()); + } + let name = r["name"].as_str().unwrap(); + if name.split_whitespace().collect::>().join(" ") != name { + return Err(invalid()); + } + let width = r["widthM"].as_f64().unwrap(); + let length = r["lengthM"].as_f64().unwrap(); + let nodes = r["nodes"].as_array().ok_or_else(invalid)?; + if nodes.is_empty() || nodes.len() > 16 { + return Err(invalid()); + } + let mut ids = HashSet::new(); + let mut slots = HashSet::new(); + let mut base = false; + let mut receiver = false; + for n in nodes { + if !keys( + n, + &[ + "nodeId", + "slot", + "xM", + "yM", + "role", + "headingDeg", + "orientationConfirmed", + "confirmedAtUnixMs", + ], + ) || !number(&n["nodeId"], 0., 255., 1.) + || !position(n, width, length, &slots) + || n["orientationConfirmed"] != true + || !number(&n["headingDeg"], 0., 360., 10.) + { + return Err(invalid()); + } + let id = n["nodeId"].as_f64().unwrap() as u8; + if !ids.insert(id) { + return Err(invalid()); + } + slots.insert(n["slot"].as_str().unwrap().to_string()); + match n["role"].as_str() { + Some("receiver") => receiver = true, + Some("base_station") => base = true, + _ => return Err(invalid()), + } + } + let ap = &r["accessPoint"]; + if !ap.is_null() + && (!keys(ap, &["slot", "xM", "yM", "provenance", "confirmedAtUnixMs"]) + || !position(ap, width, length, &slots) + || ap["provenance"] != "operator_entered") + { + return Err(invalid()); + } + if (receiver && ap.is_null() && !base) || (base && !receiver) { + return Err(invalid()); + } + r["nodes"] + .as_array_mut() + .unwrap() + .sort_by_key(|n| n["nodeId"].as_f64().unwrap() as u8); + Ok(r) +} +fn scaled(v: &Value, scale: f64) -> i64 { + (v.as_f64().unwrap() * scale).round() as i64 +} +fn digest(room: &Value, installation: &str, revision: u64) -> String { + let nodes: Vec = room["nodes"] + .as_array() + .unwrap() + .iter() + .map(|n| { + json!([ + scaled(&n["nodeId"], 1.), + n["slot"], + scaled(&n["xM"], 1000.), + scaled(&n["yM"], 1000.), + n["role"], + scaled(&n["headingDeg"], 10.) + ]) + }) + .collect(); + let ap = &room["accessPoint"]; + let ap = if ap.is_null() { + Value::Null + } else { + json!([ + ap["slot"], + scaled(&ap["xM"], 1000.), + scaled(&ap["yM"], 1000.), + "operator_entered" + ]) + }; + let canonical = json!([ + "ruview.room-calibration-binding.v2", + installation, + revision, + room["id"], + [ + scaled(&room["widthM"], 1000.), + scaled(&room["lengthM"], 1000.), + scaled(&room["heightM"], 1000.) + ], + nodes, + ap + ]); + format!( + "{:x}", + Sha256::digest(serde_json::to_vec(&canonical).unwrap()) + ) +} +impl SharedRoomStore { + pub fn load(dir: &Path, installation_id: &str) -> std::io::Result> { + if !text(&json!(installation_id), 1, 128) { + return Err(invalid()); + } + fs::create_dir_all(dir)?; + let path = dir.join("companion-room.json"); + let current = match fs::symlink_metadata(&path) { + Ok(meta) => { + if !meta.file_type().is_file() || meta.len() > 65536 { + return Err(invalid()); + } + #[cfg(unix)] + if meta.permissions().mode() & 0o077 != 0 { + return Err(invalid()); + } + let e: Envelope = + serde_json::from_slice(&fs::read(&path)?).map_err(|_| invalid())?; + if e.installation_id != installation_id || e.revision == 0 || e.revision > MAX_SAFE + { + return Err(invalid()); + } + let room = validate(e.room.clone().ok_or_else(invalid)?)?; + if e.binding_digest.as_deref() + != Some(digest(&room, installation_id, e.revision).as_str()) + { + return Err(invalid()); + } + Envelope { + room: Some(room), + ..e + } + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Envelope { + installation_id: installation_id.into(), + revision: 0, + room: None, + binding_digest: None, + }, + Err(e) => return Err(e), + }; + Ok(Arc::new(Self { + path, + current: Mutex::new(current), + })) + } + pub fn matches_binding(&self, binding: &str, node_ids: &[u8]) -> bool { + let Ok(e) = self.current.lock() else { + return false; + }; + let Some(room) = &e.room else { return false }; + let ids: HashSet = node_ids.iter().copied().collect(); + let expected: HashSet = room["nodes"] + .as_array() + .unwrap() + .iter() + .map(|n| scaled(&n["nodeId"], 1.) as u8) + .collect(); + e.binding_digest.as_deref() == Some(binding) + && ids.len() == node_ids.len() + && ids == expected + } + fn publish(&self, expected: u64, room: Value) -> Result { + let room = validate(room).map_err(|_| StatusCode::BAD_REQUEST)?; + let mut current = self + .current + .lock() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + if current.revision != expected { + return Err(StatusCode::CONFLICT); + } + if expected >= MAX_SAFE { + return Err(StatusCode::CONFLICT); + } + let next = Envelope { + installation_id: current.installation_id.clone(), + revision: expected + 1, + binding_digest: Some(digest(&room, ¤t.installation_id, expected + 1)), + room: Some(room), + }; + let persist = || -> std::io::Result<()> { + let tmp = self + .path + .with_extension(format!("{}.tmp", rand::random::())); + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + options.mode(0o600); + let mut file = options.open(&tmp)?; + file.write_all(&serde_json::to_vec(&next)?)?; + file.sync_all()?; + fs::rename(&tmp, &self.path)?; + fs::File::open(self.path.parent().unwrap())?.sync_all()?; + Ok(()) + }; + persist().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + *current = next.clone(); + Ok(next) + } +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Publication { + expected_revision: u64, + room: Value, +} +async fn read(State(s): State>) -> Result, StatusCode> { + Ok(Json( + s.current + .lock() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)? + .clone(), + )) +} +async fn publish( + State(s): State>, + Json(p): Json, +) -> Result, StatusCode> { + s.publish(p.expected_revision, p.room).map(Json) +} +pub fn owner_router(store: Arc) -> Router { + Router::new() + .route("/api/v1/companion/room", get(read).put(publish)) + .with_state(store) +} +pub fn read_router(store: Arc) -> Router { + Router::new() + .route("/companion/v1/room", get(read)) + .with_state(store) +} +#[cfg(test)] +mod tests { + use super::*; + fn room() -> Value { + json!({"schema":"ruview.companion.room.v1","id":"room-1","name":"Living room","widthM":2.5,"lengthM":2.5,"heightM":2.4,"nodes":[{"nodeId":17,"slot":"north_west","xM":0,"yM":0,"role":"receiver","headingDeg":90,"orientationConfirmed":true,"confirmedAtUnixMs":1}],"accessPoint":{"slot":"south_east","xM":2.5,"yM":2.5,"provenance":"operator_entered","confirmedAtUnixMs":1}}) + } + #[test] + fn cross_language_fixture() { + let mut r = room(); + r["widthM"] = json!(5); + r["lengthM"] = json!(6); + r["heightM"] = json!(2.6); + r["nodes"][0]["nodeId"] = json!(1); + r["nodes"][0]["xM"] = json!(0.45); + r["nodes"][0]["yM"] = json!(0.45); + r["accessPoint"]["slot"] = json!("north_east"); + r["accessPoint"]["xM"] = json!(4.55); + r["accessPoint"]["yM"] = json!(0.45); + let r = validate(r).unwrap(); + assert_eq!( + digest(&r, "installation-1", 1), + "4b9276991f7d676f220ea053c3474e120f52698682c246cc92f2c66109a6be28" + ); + } + #[test] + fn persistence_cas_and_binding_are_installation_scoped() { + let dir = tempfile::tempdir().unwrap(); + let s = SharedRoomStore::load(dir.path(), "install-1").unwrap(); + assert_eq!(s.current.lock().unwrap().revision, 0); + let e = s.publish(0, room()).unwrap(); + assert!(s.matches_binding(e.binding_digest.as_ref().unwrap(), &[17])); + assert!(!s.matches_binding(e.binding_digest.as_ref().unwrap(), &[17, 17])); + assert!(!s.matches_binding(e.binding_digest.as_ref().unwrap(), &[18])); + assert!(matches!(s.publish(0, room()), Err(StatusCode::CONFLICT))); + let restored = SharedRoomStore::load(dir.path(), "install-1").unwrap(); + assert!(restored.matches_binding(e.binding_digest.as_ref().unwrap(), &[17])); + assert!(SharedRoomStore::load(dir.path(), "other").is_err()); + } + #[test] + fn malformed_geometry_unknown_evidence_and_duplicate_slots_rejected() { + for (key, value) in [ + ("widthM", json!(2.5001)), + ("baseline", json!({})), + ("name", json!(" bad")), + ] { + let mut r = room(); + r[key] = value; + assert!(validate(r).is_err()); + } + let mut r = room(); + r["accessPoint"]["slot"] = json!("north_west"); + assert!(validate(r).is_err()); + let mut r = room(); + r["nodes"][0]["orientationConfirmed"] = json!(false); + assert!(validate(r).is_err()); + } + #[test] + fn disk_tampering_fails_closed() { + let dir = tempfile::tempdir().unwrap(); + let s = SharedRoomStore::load(dir.path(), "install-1").unwrap(); + s.publish(0, room()).unwrap(); + let mut e: Value = serde_json::from_slice(&fs::read(&s.path).unwrap()).unwrap(); + e["room"]["widthM"] = json!(3); + fs::write(&s.path, serde_json::to_vec(&e).unwrap()).unwrap(); + assert!(SharedRoomStore::load(dir.path(), "install-1").is_err()); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_tls.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_tls.rs new file mode 100644 index 0000000000..371f7e4ae8 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_tls.rs @@ -0,0 +1,141 @@ +//! Private, persistent TLS identity for the QR-pinned companion listener. +use rustls::pki_types::{CertificateDer, PrivatePkcs8KeyDer}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{io, path::Path, sync::Arc}; +#[derive(Serialize, Deserialize)] +struct DiskIdentity { + installation_id: String, + certificate: Vec, + private_key: Vec, +} +// Publish a complete first identity without replacing another process's identity. +fn create_identity_exclusively(path: &Path, bytes: &[u8]) -> io::Result<()> { + let temporary = path.with_extension(format!("create-{}", super::companion::secret())); + super::companion::atomic_private_write(&temporary, bytes)?; + let result = std::fs::hard_link(&temporary, path); + let cleanup = std::fs::remove_file(&temporary); + result?; + cleanup?; + std::fs::File::open( + path.parent() + .ok_or_else(|| io::Error::other("identity parent"))?, + )? + .sync_all() +} +pub struct Identity { + pub installation_id: String, + pub cert_sha256: String, + pub config: Arc, +} +impl Identity { + pub fn load_or_create(dir: &Path, preferred_installation_id: Option<&str>) -> io::Result { + super::companion::private_dir(dir)?; + if preferred_installation_id.is_some_and(|id| { + id.is_empty() + || id.len() > 128 + || !id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') + }) { + return Err(io::Error::other("invalid configured installation")); + } + let path = dir.join("companion-identity.json"); + let data: DiskIdentity = if path.exists() { + serde_json::from_slice(&super::companion::private_read(&path)?) + .map_err(io::Error::other)? + } else { + let cert = rcgen::generate_simple_self_signed(vec!["ruview.local".into()]) + .map_err(io::Error::other)?; + let data = DiskIdentity { + installation_id: preferred_installation_id + .map(str::to_owned) + .unwrap_or_else(super::companion::secret), + certificate: cert.serialize_der().map_err(io::Error::other)?, + private_key: cert.serialize_private_key_der(), + }; + create_identity_exclusively( + &path, + &serde_json::to_vec(&data).map_err(io::Error::other)?, + )?; + data + }; + if data.installation_id.is_empty() + || data.installation_id.len() > 128 + || preferred_installation_id.is_some_and(|id| id != data.installation_id) + { + return Err(io::Error::other("invalid companion installation")); + } + let pin = format!("{:x}", Sha256::digest(&data.certificate)); + let config = rustls::ServerConfig::builder_with_provider(Arc::new( + rustls::crypto::ring::default_provider(), + )) + .with_safe_default_protocol_versions() + .map_err(io::Error::other)? + .with_no_client_auth() + .with_single_cert( + vec![CertificateDer::from(data.certificate)], + PrivatePkcs8KeyDer::from(data.private_key).into(), + ) + .map_err(io::Error::other)?; + Ok(Self { + installation_id: data.installation_id, + cert_sha256: pin, + config: Arc::new(config), + }) + } +} +pub async fn serve( + listener: tokio::net::TcpListener, + config: Arc, + app: axum::Router, +) -> io::Result<()> { + let acceptor = tokio_rustls::TlsAcceptor::from(config); + let slots = Arc::new(tokio::sync::Semaphore::new(64)); + loop { + let (stream, _) = listener.accept().await?; + let Ok(permit) = slots.clone().try_acquire_owned() else { + continue; + }; + let acceptor = acceptor.clone(); + let app = app.clone(); + tokio::spawn(async move { + let _permit = permit; + let Ok(Ok(tls)) = + tokio::time::timeout(std::time::Duration::from_secs(10), acceptor.accept(stream)) + .await + else { + return; + }; + let service = hyper_util::service::TowerToHyperService::new(app); + let builder = + hyper_util::server::conn::auto::Builder::new(hyper_util::rt::TokioExecutor::new()); + let _ = builder + .serve_connection_with_upgrades(hyper_util::rt::TokioIo::new(tls), service) + .await; + }); + } +} +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn stable_identity_and_configured_mismatch() { + let d = tempfile::tempdir().unwrap(); + let first = Identity::load_or_create(d.path(), Some("mac-installation")).unwrap(); + let second = Identity::load_or_create(d.path(), Some("mac-installation")).unwrap(); + assert_eq!(first.cert_sha256, second.cert_sha256); + assert_eq!(first.installation_id, "mac-installation"); + assert!(Identity::load_or_create(d.path(), Some("other")).is_err()); + std::fs::write(d.path().join("companion-identity.json"), b"corrupt").unwrap(); + assert!(Identity::load_or_create(d.path(), None).is_err()); + } + #[test] + fn first_identity_publication_never_replaces_existing() { + let d = tempfile::tempdir().unwrap(); + let p = d.path().join("identity"); + create_identity_exclusively(&p, b"first").unwrap(); + assert!(create_identity_exclusively(&p, b"second").is_err()); + assert_eq!(std::fs::read(&p).unwrap(), b"first"); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/discovery.rs b/v2/crates/wifi-densepose-sensing-server/src/discovery.rs index 10eda76441..61245c8205 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/discovery.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/discovery.rs @@ -87,6 +87,15 @@ pub fn start_advertiser( Ok(DiscoveryAdvertiser { daemon, fullname }) } +/// Advertise the QR-pinned companion TLS endpoint independently of legacy HTTP. +pub fn start_tls_advertiser(instance_name: &str, installation_id: &str, hostname: &str, port: u16) -> Result { + let daemon = ServiceDaemon::new()?; + let service = build_service(instance_name, installation_id, hostname, port, true)?; + let fullname = service.get_fullname().to_owned(); + daemon.register(service)?; + Ok(DiscoveryAdvertiser { daemon, fullname }) +} + #[cfg(test)] mod tests { use super::*; diff --git a/v2/crates/wifi-densepose-sensing-server/src/lib.rs b/v2/crates/wifi-densepose-sensing-server/src/lib.rs index bc0b730211..c7e54ab8ec 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/lib.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/lib.rs @@ -69,3 +69,6 @@ pub mod vendor_rf; // `crate::sona`) and public API (`wifi_densepose_sensing_server::embedding`, …) // are unchanged. pub use wifi_densepose_aether::{embedding, graph_transformer, sona, sparse_inference}; + +pub mod companion; +pub mod companion_tls; diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index 1e0d6afb27..81f4012dd1 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -10,6 +10,8 @@ #![allow(dead_code)] mod adaptive_classifier; +mod companion_integration; +mod companion_room; pub mod cli; pub mod csi; mod engine_bridge; @@ -169,6 +171,10 @@ struct Args { #[arg(long, default_value = "127.0.0.1", env = "SENSING_BIND_ADDR")] bind_addr: String, + /// Enable QR-pinned HTTPS companion service on this separate port. + #[arg(long, env = "RUVIEW_COMPANION_PORT")] + companion_port: Option, + /// Disable local `_ruview._tcp` discovery. Discovery is automatically /// skipped for loopback-only binds and never carries sensor data. #[arg(long, env = "RUVIEW_NO_MDNS", value_parser = cli::parse_env_bool)] @@ -5805,8 +5811,13 @@ fn generate_simulated_frame(tick: u64) -> Esp32Frame { async fn ws_sensing_handler( ws: WebSocketUpgrade, State(state): State, + grant: Option>, ) -> impl IntoResponse { - ws.on_upgrade(|socket| handle_ws_client(socket, state)) + ws.on_upgrade(move |socket| async move { + if let Some(Extension(grant)) = grant { + companion_integration::guard_stream(grant, handle_ws_client(socket, state)).await; + } else { handle_ws_client(socket, state).await; } + }) } async fn handle_ws_client(mut socket: WebSocket, state: SharedState) { @@ -8853,6 +8864,8 @@ mod bootstrap_vital_publication_tests { Query(CalibrationStartQuery { source_node_id: Some(5), }), + None, + None, Json(test_binding_request(vec![5])), ) .await; @@ -8883,6 +8896,8 @@ mod bootstrap_vital_publication_tests { Query(CalibrationStartQuery { source_node_id: Some(5), }), + None, + None, Json(test_binding_request(vec![5])), ) .await; @@ -9709,8 +9724,13 @@ mod bootstrap_recovery_tests { async fn calibration_start( State(state): State, Query(query): Query, + companion_grant: Option>, + shared_room: Option>>, Json(request): Json, ) -> Json { + if companion_grant.is_some() && !shared_room.as_ref().is_some_and(|Extension(room)| room.matches_binding(&request.binding_digest, &request.source_node_ids)) { + return Json(serde_json::json!({"success":false,"error_code":"shared_room_binding_required","error":"Import the current Mac room revision before starting calibration."})); + } let mut s = state.write().await; if !valid_binding_digest(&request.binding_digest) || !valid_source_node_ids(&request.source_node_ids) @@ -15250,6 +15270,9 @@ async fn main() { } } + let companion = companion_integration::Integration::prepare(&args, &state, bearer_auth_state.clone()).await + .unwrap_or_else(|error| { error!("Companion startup refused: {error}"); std::process::exit(1) }); + // DNS-rebinding defense: validate the `Host` header against an allowlist // before any handler runs. Default is loopback-only (`localhost`, // `127.0.0.1`, `[::1]`, each with or without a port). Operators extend @@ -15323,7 +15346,7 @@ async fn main() { // HTTP server (serves UI + full DensePose-compatible REST API) let ui_path = args.ui_path.clone(); - let http_app = Router::new() + let mut http_app = Router::new() .route("/", get(info_page)) // Health endpoints (DensePose-compatible) .route("/health", get(health)) @@ -15513,6 +15536,13 @@ async fn main() { wifi_densepose_sensing_server::host_validation::require_allowed_host, )); + if let Some(companion) = companion { + let owner_router = companion.owner_router(); + http_app = companion.protect_control(http_app); + companion.start(http_app.clone()).await; + http_app = http_app.merge(owner_router); + } + let http_addr = SocketAddr::from((bind_ip, args.http_port)); let http_listener = tokio::net::TcpListener::bind(http_addr) .await @@ -15551,7 +15581,7 @@ async fn main() { // Run the HTTP server with graceful shutdown support let shutdown_state = state.clone(); - let server = axum::serve(http_listener, http_app).with_graceful_shutdown(async { + let server = axum::serve(http_listener, http_app.into_make_service_with_connect_info::()).with_graceful_shutdown(async { tokio::signal::ctrl_c() .await .expect("failed to install CTRL+C handler"); From 7ba5456a3d1f814f7e8c1d012eaa636f7759aca2 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Mon, 12 Oct 2026 18:25:13 +0200 Subject: [PATCH 003/100] docs(companion): record implementation and verification boundaries --- .../adr/ADR-383-secure-local-companion-pairing.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/adr/ADR-383-secure-local-companion-pairing.md b/docs/adr/ADR-383-secure-local-companion-pairing.md index 85d903670c..0a976c3bcb 100644 --- a/docs/adr/ADR-383-secure-local-companion-pairing.md +++ b/docs/adr/ADR-383-secure-local-companion-pairing.md @@ -1,6 +1,6 @@ # ADR-383: Secure local Mac–iPhone companion pairing -- Status: Accepted decision (2026-10-12); implementation and verification in progress +- Status: Accepted decision (2026-10-12); implemented; software verified, physical qualification pending - Decision owner: product owner, approved in the implementation session - Related: ADR-271 (OAuth), ADR-272 (WebSocket tickets); ruview-mobile ADR-046 @@ -32,6 +32,9 @@ expiry requires explicit pairing again. A paired phone obtains a 12-hour access token, capped by pairing expiry; access tokens and calibration grants are not persisted on the server. Restart refresh uses the durable device proof over pinned TLS. Permission to calibrate is a separate owner-granted 15-minute capability. +Only one phone holds calibration authority at a time; granting another phone +explicitly transfers control and removes the previous grant, while read access +remains. Read authority never authorizes model, recording, configuration or administrator mutations. Existing OAuth scope semantics are unchanged. @@ -74,3 +77,13 @@ scope; revocation of live streams; durable restart and corrupt-state behavior; shared-room digest parity, revision conflicts and no stale-evidence import; native iOS and Mac builds; actual phone–Mac pairing and reconnect. Synthetic tests and authenticated transport are not physical sensing or room-selectivity acceptance. + +## Implementation evidence (2026-10-12) + +Producer PR #2249 and consumer `cognitum-one/ruview-mobile#57` implement this +contract. The producer package passed 1,143 tests (three existing ignores); the +compiled debug and release helpers each passed the 26-check real-socket probe. +The consumer passed 1,547 Jest tests, iOS/Mac builds and production Swift TLS, +path, recovery and disconnect-race checks. Adversarial review has no remaining +evidence-backed blocker for PR review. These results do not establish physical +QR scanning, room controls or distribution acceptance. From 7a04840b0299bd0df611a6ee60b4d8f35eb7ddf6 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Mon, 19 Oct 2026 11:56:05 +0200 Subject: [PATCH 004/100] docs(adr): resolve secure companion pairing number collision Move the companion pairing design to ADR-419 so upstream ADR-383 remains the HTTP request-target validation decision. --- ...ion-pairing.md => ADR-419-secure-local-companion-pairing.md} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename docs/adr/{ADR-383-secure-local-companion-pairing.md => ADR-419-secure-local-companion-pairing.md} (98%) diff --git a/docs/adr/ADR-383-secure-local-companion-pairing.md b/docs/adr/ADR-419-secure-local-companion-pairing.md similarity index 98% rename from docs/adr/ADR-383-secure-local-companion-pairing.md rename to docs/adr/ADR-419-secure-local-companion-pairing.md index 0a976c3bcb..c6377f1e85 100644 --- a/docs/adr/ADR-383-secure-local-companion-pairing.md +++ b/docs/adr/ADR-419-secure-local-companion-pairing.md @@ -1,4 +1,4 @@ -# ADR-383: Secure local Mac–iPhone companion pairing +# ADR-419: Secure local Mac–iPhone companion pairing - Status: Accepted decision (2026-10-12); implemented; software verified, physical qualification pending - Decision owner: product owner, approved in the implementation session From a13463bc32076a3f24c9c7e323e85d4687eb355b Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Mon, 19 Oct 2026 15:55:43 +0200 Subject: [PATCH 005/100] fix(companion): enforce durable capacity and prepared host authority Recheck approved-device capacity under the approval lock before persistence, retaining pending requests when full. Build the exact Host allowlist entry from the prepared identity authority used by the QR, including generated and reloaded identities. Regressions reproduced the old failures and now pass. Verified actual workspace binary check, nine companion tests and the real binary authority test, plus 28 isolated module tests and independent fix-delta review. No Host wildcard or validation bypass. --- .../src/companion.rs | 88 ++++++++++++++++++- .../src/companion_integration.rs | 61 ++++++++++++- .../wifi-densepose-sensing-server/src/main.rs | 14 +-- 3 files changed, 151 insertions(+), 12 deletions(-) diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion.rs b/v2/crates/wifi-densepose-sensing-server/src/companion.rs index 55f0fcfdff..43df457469 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/companion.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/companion.rs @@ -385,6 +385,11 @@ async fn approve( if p.expires <= now() || p.code != r.comparison_code { return Err(StatusCode::FORBIDDEN); } + // Pending requests can queue while a slot is available. Recheck under the + // approval lock so another approval cannot exceed the persisted load limit. + if i.devices.len() >= 16 { + return Err(StatusCode::TOO_MANY_REQUESTS); + } let mut devices = i.devices.clone(); devices.insert(p.device.device_id.clone(), p.device.clone()); s.persist(&devices, &i)?; @@ -491,6 +496,9 @@ mod tests { (d, s) } async fn pending(s: &Shared) -> (String, String, String) { + pending_device(s, "phone").await + } + async fn pending_device(s: &Shared, device_id: &str) -> (String, String, String) { let v = invite(State(s.clone())).await.unwrap().0; let proof = secret(); let v = enroll( @@ -499,7 +507,7 @@ mod tests { installation_id: s.installation_id.clone(), invite_id: v["invite_id"].as_str().unwrap().into(), invite_secret: v["invite_secret"].as_str().unwrap().into(), - device_id: "phone".into(), + device_id: device_id.into(), device_name: "Phone".into(), pairing_secret: proof.clone(), }), @@ -524,6 +532,84 @@ mod tests { ) .await } + #[tokio::test] + async fn queued_approvals_cannot_exceed_durable_device_capacity() { + let (dir, s) = setup(); + for index in 0..15 { + let (id, code, _) = pending_device(&s, &format!("phone-{index}")).await; + let _ = approve( + State(s.clone()), + RoutePath(id), + Json(Approve { + comparison_code: code, + }), + ) + .await + .unwrap(); + } + let (first_id, first_code, _) = pending_device(&s, "last-slot").await; + let (waiting_id, waiting_code, _) = pending_device(&s, "waiting").await; + let _ = approve( + State(s.clone()), + RoutePath(first_id), + Json(Approve { + comparison_code: first_code, + }), + ) + .await + .unwrap(); + let persisted = std::fs::read(&s.path).unwrap(); + assert_eq!( + approve( + State(s.clone()), + RoutePath(waiting_id.clone()), + Json(Approve { + comparison_code: waiting_code.clone(), + }) + ) + .await + .unwrap_err(), + StatusCode::TOO_MANY_REQUESTS, + ); + assert_eq!(std::fs::read(&s.path).unwrap(), persisted); + { + let inner = s.inner.lock().unwrap(); + assert_eq!(inner.devices.len(), 16); + assert!(inner.pending.contains_key(&waiting_id)); + } + let reloaded = Companion::load( + dir.path(), + s.installation_id.clone(), + s.cert_sha256.clone(), + s.origin.clone(), + ) + .unwrap(); + assert_eq!(reloaded.inner.lock().unwrap().devices.len(), 16); + let _ = revoke(State(s.clone()), RoutePath("last-slot".into())) + .await + .unwrap(); + let _ = approve( + State(s.clone()), + RoutePath(waiting_id.clone()), + Json(Approve { + comparison_code: waiting_code, + }), + ) + .await + .unwrap(); + assert!(!s.inner.lock().unwrap().pending.contains_key(&waiting_id)); + let reloaded = Companion::load( + dir.path(), + s.installation_id.clone(), + s.cert_sha256.clone(), + s.origin.clone(), + ) + .unwrap(); + let inner = reloaded.inner.lock().unwrap(); + assert_eq!(inner.devices.len(), 16); + assert!(inner.devices.contains_key("waiting")); + } + #[tokio::test] async fn approval_read_scope_calibration_revocation() { let (_d, s) = setup(); diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs index b2a3a45154..b5ad6ad0d8 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs @@ -11,8 +11,31 @@ use wifi_densepose_sensing_server::{ companion_tls, }; +// Share the QR authority with Host validation; DNS-SD names retain their terminal dot. +fn companion_authority(identity: &companion_tls::Identity, port: u16) -> String { + let hostname = + wifi_densepose_sensing_server::discovery::discovery_hostname(&identity.installation_id); + format!("{}:{port}", hostname.trim_end_matches('.')) +} + +pub fn host_allowlist( + args: &Args, + companion_authority: Option<&str>, +) -> wifi_densepose_sensing_server::host_validation::HostAllowlist { + let discovery_label = args.installation_id.as_deref().unwrap_or("installation"); + let discovery_host = + wifi_densepose_sensing_server::discovery::discovery_hostname(discovery_label); + wifi_densepose_sensing_server::host_validation::HostAllowlist::from_cli_and_env( + args.allowed_hosts + .iter() + .map(String::as_str) + .chain(std::iter::once(discovery_host.as_str())) + .chain(companion_authority), + ) +} pub struct Integration { service: Arc, + authority: String, auth: AuthState, listener: tokio::net::TcpListener, tls: Arc, @@ -43,7 +66,8 @@ impl Integration { companion_tls::Identity::load_or_create(&dir, args.installation_id.as_deref())?; let hostname = wifi_densepose_sensing_server::discovery::discovery_hostname(&identity.installation_id); - let origin = format!("https://{}:{port}", hostname.trim_end_matches('.')); + let authority = companion_authority(&identity, port); + let origin = format!("https://{authority}"); let service = Companion::load( &dir, identity.installation_id.clone(), @@ -73,6 +97,7 @@ impl Integration { }; Ok(Some(Self { service, + authority, auth, listener, tls: identity.config, @@ -81,6 +106,9 @@ impl Integration { control, })) } + pub fn authority(&self) -> &str { + &self.authority + } pub fn owner_router(&self) -> Router { companion::owner_router(self.service.clone()) .merge(companion_room::owner_router(self.room.clone())) @@ -267,6 +295,37 @@ pub async fn guard_stream(grant: Grant, stream: impl std::future::Future Date: Tue, 20 Oct 2026 14:11:04 +0200 Subject: [PATCH 006/100] feat(companion): add scoped transient phone sensor preview --- .../ADR-419-secure-local-companion-pairing.md | 29 + .../src/companion.rs | 13 + .../src/companion_integration.rs | 30 +- .../src/companion_sensors.rs | 644 ++++++++++++++++++ .../wifi-densepose-sensing-server/src/lib.rs | 2 + 5 files changed, 717 insertions(+), 1 deletion(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/companion_sensors.rs diff --git a/docs/adr/ADR-419-secure-local-companion-pairing.md b/docs/adr/ADR-419-secure-local-companion-pairing.md index c6377f1e85..aeaa5ef994 100644 --- a/docs/adr/ADR-419-secure-local-companion-pairing.md +++ b/docs/adr/ADR-419-secure-local-companion-pairing.md @@ -3,6 +3,7 @@ - Status: Accepted decision (2026-10-12); implemented; software verified, physical qualification pending - Decision owner: product owner, approved in the implementation session - Related: ADR-271 (OAuth), ADR-272 (WebSocket tickets); ruview-mobile ADR-046 +- Updated: 2026-10-20 — bounded transient sensor preview; physical qualification pending. ## Context @@ -87,3 +88,31 @@ The consumer passed 1,547 Jest tests, iOS/Mac builds and production Swift TLS, path, recovery and disconnect-race checks. Adversarial review has no remaining evidence-backed blocker for PR review. These results do not establish physical QR scanning, room controls or distribution acceptance. + +## Sensor preview amendment (2026-10-20) + +Sensor sharing is a separate transient owner-granted capability, lasting fifteen +minutes and never implied by read or calibration authority. Owner grant/revoke +routes are `/api/v1/companion/devices/:id/sensors`; snapshot readback is +`/api/v1/companion/sensors`. Existing actual-loopback plus exact-owner-token +protection applies to all three. No companion may call owner routes. + +The dedicated pinned TLS `/companion/v1/sensors` WebSocket authenticates the +existing pairing/access proof plus current sharing permission and immediately +sends `sensor_ready`. Each exact-schema geometry frame is bounded to 64 KiB, +1–512 integer-millimetre points, a rigid finite camera transform, explicit capture +lineage, monotonic sequence and bounded source age. Unknown fields, images, +empty/malformed geometry, replay, lineage substitution and excess rate fail +closed. Device identity comes from the authenticated grant, never packet data. + +Only the latest frame exists in transient memory. Pairing revocation, access or +sharing expiry, owner revocation, supersession and socket closure suppress it; +owner readback marks frames stale after three seconds. A superseded socket cannot +clear its replacement's preview. The receiver additionally checks both source +and receive age before drawing geometry. No storage, camera feed, legacy relay +admin token, calibration promotion or ARKit-to-room alignment is added. + +The consumer independently opts in to foreground sharing and renders remote +geometry separately from local sensing/calibration authority. Software protocol, +real socket and owner/companion middleware checks do not establish physical +phone-to-Mac capture accuracy or calibration acceptance. diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion.rs b/v2/crates/wifi-densepose-sensing-server/src/companion.rs index 43df457469..0ae02c75bf 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/companion.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/companion.rs @@ -204,6 +204,19 @@ impl Companion { StatusCode::SERVICE_UNAVAILABLE }) } + /// Metadata only; sharing never receives durable pairing credentials. + pub fn paired_device(&self, id: &str) -> Option<(String, u64)> { + if self.fault.load(Ordering::SeqCst) { return None; } + let inner = self.inner.lock().ok()?; + let device = inner.devices.get(id)?; + (device.expires_at > now()).then(|| (device.device_name.clone(), device.expires_at)) + } + pub fn device_revocation(&self, id: &str) -> Option> { + if self.fault.load(Ordering::SeqCst) { return None; } + let mut inner = self.inner.lock().ok()?; + if !inner.devices.get(id).is_some_and(|d| d.expires_at > now()) { return None; } + Some(inner.revocations.entry(id.into()).or_insert_with(|| watch::channel(false).0).subscribe()) + } pub fn authorize( &self, installation: &str, diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs index b5ad6ad0d8..949b6eeda3 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs @@ -8,7 +8,7 @@ use axum::{ use wifi_densepose_sensing_server::{ bearer_auth::AuthState, companion::{self, Companion, Grant}, - companion_tls, + companion_tls, companion_sensors, }; // Share the QR authority with Host validation; DNS-SD names retain their terminal dot. @@ -41,6 +41,7 @@ pub struct Integration { tls: Arc, room: Arc, control: Control, + sensors: companion_sensors::Shared, advertiser: Option, } impl Integration { @@ -95,6 +96,7 @@ impl Integration { state: state.clone(), serial: Arc::new(tokio::sync::Mutex::new(())), }; + let sensors = companion_sensors::SensorStore::new(service.clone()); Ok(Some(Self { service, authority, @@ -104,6 +106,7 @@ impl Integration { advertiser, room, control, + sensors, })) } pub fn authority(&self) -> &str { @@ -111,6 +114,7 @@ impl Integration { } pub fn owner_router(&self) -> Router { companion::owner_router(self.service.clone()) + .merge(companion_sensors::owner_router(self.sensors.clone())) .merge(companion_room::owner_router(self.room.clone())) .layer(axum::middleware::from_fn_with_state( self.control.clone(), @@ -130,6 +134,7 @@ impl Integration { } pub async fn start(self, app: Router) { let app = app + .merge(companion_sensors::companion_router(self.sensors.clone())) .merge(companion_room::read_router(self.room.clone())) .layer(axum::middleware::from_fn_with_state( self.service.clone(), @@ -228,6 +233,7 @@ fn policy(method: &axum::http::Method, path: &str) -> Option { | "/api/v1/pose/calibration/status" | "/ws/sensing" | "/companion/v1/room" + | "/companion/v1/sensors" ) { return Some(false); } @@ -345,4 +351,26 @@ mod tests { } assert_eq!(policy(&Method::GET, "/ws/train/progress"), None); } + #[tokio::test] + async fn sensor_routes_preserve_owner_peer_and_companion_isolation() { + use tower::ServiceExt; + use axum::{body::Body, http::{Request, Method}}; + let dir = tempfile::tempdir().unwrap(); + let service = Companion::load(dir.path(), "test".into(), "a".repeat(64), "https://test.local:3443".into()).unwrap(); + let sensors = companion_sensors::SensorStore::new(service.clone()); + let owner = companion_sensors::owner_router(sensors).layer(axum::middleware::from_fn_with_state(AuthState::from_token("owner"), require_owner)); + for (peer, credential, expected) in [("127.0.0.1:1", "owner", 200), ("127.0.0.1:1", "phone", 403), ("10.0.0.1:1", "owner", 403)] { + let mut req=Request::builder().uri("/api/v1/companion/sensors").header("authorization",format!("Bearer {credential}")).body(Body::empty()).unwrap(); + req.extensions_mut().insert(ConnectInfo(peer.parse::().unwrap())); + assert_eq!(owner.clone().oneshot(req).await.unwrap().status().as_u16(), expected); + } + assert_eq!(policy(&Method::GET,"/companion/v1/sensors"),Some(false)); + for path in ["/api/v1/companion/sensors", "/api/v1/companion/devices/phone/sensors"] { + assert_eq!(policy(&Method::GET,path),None); assert_eq!(policy(&Method::POST,path),None); assert_eq!(policy(&Method::DELETE,path),None); + } + let denied=Router::new().route("/companion/v1/sensors",get(||async {StatusCode::OK})).layer(axum::middleware::from_fn_with_state(service,require_companion)); + let response=denied.oneshot(Request::builder().uri("/companion/v1/sensors").body(Body::empty()).unwrap()).await.unwrap(); + assert_eq!(response.status(),StatusCode::UNAUTHORIZED); + } + } diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_sensors.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_sensors.rs new file mode 100644 index 0000000000..147d0ad314 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_sensors.rs @@ -0,0 +1,644 @@ +//! Transient geometry preview. A room digest never implies spatial registration. +use crate::companion::{Companion, Grant}; +use axum::{ + extract::{ + ws::{Message, WebSocket, WebSocketUpgrade}, + Path, State, + }, + http::StatusCode, + response::{IntoResponse, Response}, + routing::{get, post}, + Extension, Json, Router, +}; +use serde::{Deserialize, Serialize}; +use serde_json::{json, Value}; +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + time::{Duration, Instant}, +}; +use tokio::sync::watch; +const SAFE: u64 = 9_007_199_254_740_991; +const MAX_BYTES: usize = 65_536; +fn now_ms() -> u64 { + chrono::Utc::now().timestamp_millis().max(0) as u64 +} +fn identifier(s: &str) -> bool { + !s.is_empty() + && s.len() <= 128 + && s.bytes() + .all(|b| b.is_ascii_alphanumeric() || b == b'-' || b == b'_') +} +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +pub struct SensorFrame { + #[serde(rename = "type")] + pub kind: String, + pub schema: String, + pub share_session_id: String, + pub capture_session_id: String, + pub coordinate_frame_id: String, + pub sequence: u64, + pub captured_at_unix_ms: u64, + pub tracking_state: String, + pub points_mm: Vec, + pub confidences: Option>, + pub camera_transform: [f64; 16], + pub room_binding_digest: Option, +} +impl SensorFrame { + fn valid(&self, now: u64) -> bool { + let tracking = self.tracking_state.as_str(); + self.kind == "sensor_frame" + && self.schema == "ruview.companion.sensor-frame.v1" + && identifier(&self.share_session_id) + && identifier(&self.capture_session_id) + && identifier(&self.coordinate_frame_id) + && self.sequence <= SAFE + && self.captured_at_unix_ms > 0 + && self.captured_at_unix_ms <= SAFE + && self.captured_at_unix_ms <= now.saturating_add(1000) + && now <= self.captured_at_unix_ms.saturating_add(3000) + && matches!( + tracking, + "normal" + | "unavailable" + | "limited_initializing" + | "limited_excessiveMotion" + | "limited_insufficientFeatures" + | "limited_relocalizing" + ) + && !self.points_mm.is_empty() + && self.points_mm.len() <= 512 * 3 + && self.points_mm.len() % 3 == 0 + && self + .points_mm + .iter() + .all(|x| (-30_000..=30_000).contains(x)) + && self + .confidences + .as_ref() + .is_none_or(|c| c.len() == self.points_mm.len() / 3 && c.iter().all(|x| *x <= 2)) + && self.room_binding_digest.as_ref().is_none_or(|d| { + d.len() == 64 + && d.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }) + && rigid(&self.camera_transform) + } + fn lineage(&self) -> (&str, &str, &str) { + ( + &self.share_session_id, + &self.capture_session_id, + &self.coordinate_frame_id, + ) + } +} +fn decode(text: &str) -> Result { + if text.len() > MAX_BYTES { + return Err(()); + } + let value: Value = serde_json::from_str(text).map_err(|_| ())?; + if !value.as_object().is_some_and(|m| { + m.len() == 12 && m.contains_key("confidences") && m.contains_key("roomBindingDigest") + }) { + return Err(()); + } + serde_json::from_str(text).map_err(|_| ()) +} +fn rigid(m: &[f64; 16]) -> bool { + if !m.iter().all(|v| v.is_finite()) + || [3, 7, 11].iter().any(|i| m[*i].abs() > 1e-5) + || (m[15] - 1.).abs() > 1e-5 + || [12, 13, 14].iter().any(|i| m[*i].abs() > 30.) + { + return false; + } + for a in 0..3 { + for b in 0..3 { + let dot: f64 = (0..3).map(|r| m[a * 4 + r] * m[b * 4 + r]).sum(); + if (dot - if a == b { 1. } else { 0. }).abs() > 1e-3 { + return false; + } + } + } + let det = m[0] * (m[5] * m[10] - m[9] * m[6]) - m[4] * (m[1] * m[10] - m[9] * m[2]) + + m[8] * (m[1] * m[6] - m[5] * m[2]); + (det - 1.).abs() < 1e-3 +} +struct Entry { + permission_revoked: watch::Receiver, + until: u64, + generation: u64, + stop: watch::Sender, + received: Option, + frame: Option, + access_until: u64, + access_revoked: Option>, + state: &'static str, +} +pub struct SensorStore { + companion: Arc, + entries: Mutex>, +} +pub type Shared = Arc; +impl SensorStore { + pub fn new(companion: Arc) -> Shared { + Arc::new(Self { + companion, + entries: Mutex::new(HashMap::new()), + }) + } + fn permit(&self, id: &str) -> Result { + let (_, expires) = self + .companion + .paired_device(id) + .ok_or(StatusCode::NOT_FOUND)?; + let permission_revoked = self + .companion + .device_revocation(id) + .ok_or(StatusCode::NOT_FOUND)?; + let mut entries = self + .entries + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + // Pairing membership is bounded. Forget departed devices rather than + // accumulating historical sensor grants across repeated enrollments. + entries.retain(|device_id, _| self.companion.paired_device(device_id).is_some()); + let entry = entries.entry(id.into()).or_insert_with(|| Entry { + permission_revoked: permission_revoked.clone(), + until: 0, + generation: 0, + stop: watch::channel(0).0, + received: None, + frame: None, + access_until: 0, + access_revoked: None, + state: "waiting", + }); + entry.permission_revoked = permission_revoked; + entry.generation = entry.generation.wrapping_add(1); + entry.stop.send_replace(entry.generation); + entry.until = (now_ms() / 1000 + 900).min(expires); + entry.frame = None; + entry.received = None; + entry.state = "waiting"; + entry.access_until = 0; + entry.access_revoked = None; + Ok(entry.until) + } + fn revoke(&self, id: &str) -> Result<(), StatusCode> { + let mut entries = self + .entries + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + let entry = entries.get_mut(id).ok_or(StatusCode::NOT_FOUND)?; + entry.until = 0; + entry.generation = entry.generation.wrapping_add(1); + entry.stop.send_replace(entry.generation); + entry.frame = None; + entry.received = None; + entry.state = "stopped"; + Ok(()) + } + fn begin(&self, grant: &Grant) -> Result<(u64, u64, watch::Receiver), StatusCode> { + if self.companion.paired_device(&grant.device_id).is_none() + || *grant.revoked.borrow() + || grant.expires_at <= now_ms() / 1000 + { + return Err(StatusCode::UNAUTHORIZED); + } + let mut entries = self + .entries + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + let e = entries + .get_mut(&grant.device_id) + .filter(|e| e.until > now_ms() / 1000 && !*e.permission_revoked.borrow()) + .ok_or(StatusCode::FORBIDDEN)?; + e.generation = e.generation.wrapping_add(1); + e.stop.send_replace(e.generation); + e.frame = None; + e.received = None; + e.state = "waiting"; + e.access_until = grant.expires_at; + e.access_revoked = Some(grant.revoked.clone()); + Ok(( + e.generation, + e.until.min(grant.expires_at), + e.stop.subscribe(), + )) + } + fn stop(&self, id: &str, generation: u64) { + if let Ok(mut entries) = self.entries.lock() { + if let Some(e) = entries.get_mut(id).filter(|e| e.generation == generation) { + e.frame = None; + e.received = None; + e.state = "stopped"; + } + } + } + fn accept(&self, id: &str, generation: u64, frame: SensorFrame) -> bool { + let Ok(mut entries) = self.entries.lock() else { + return false; + }; + let Some(e) = entries.get_mut(id).filter(|e| { + e.generation == generation + && !*e.permission_revoked.borrow() + && e.until > now_ms() / 1000 + && e.access_until > now_ms() / 1000 + && !e.access_revoked.as_ref().is_some_and(|r| *r.borrow()) + }) else { + return false; + }; + e.frame = Some(frame); + e.received = Some(now_ms()); + e.state = "live"; + true + } + fn snapshot(&self) -> Result { + let entries = self + .entries + .lock() + .map_err(|_| StatusCode::SERVICE_UNAVAILABLE)?; + let now = now_ms(); + let mut ids: Vec<_> = entries.keys().collect(); + ids.sort(); + let devices:Vec<_>=ids.into_iter().filter_map(|id| { + let (name,_)=self.companion.paired_device(id)?;let e=&entries[id]; + let authorized=!*e.permission_revoked.borrow() && e.until>now/1000 && (e.access_until==0 || e.access_until>now/1000) && !e.access_revoked.as_ref().is_some_and(|r| *r.borrow()); + let state=if !authorized {"stopped"} else if e.state=="live" && e.received.is_some_and(|t| now.saturating_sub(t)>3000) {"stale"} else {e.state}; + Some(json!({"deviceId":id,"deviceName":name,"sensorUntil":e.until,"state":state,"receivedAtUnixMs":if authorized {e.received} else {None},"frame":if state=="live" {e.frame.as_ref()} else {None}})) + }).collect(); + Ok(json!({"schema":"ruview.companion.sensor-snapshot.v1","devices":devices})) + } +} +async fn snapshot(State(s): State) -> Result, StatusCode> { + s.snapshot().map(Json) +} +async fn permit( + State(s): State, + Path(id): Path, +) -> Result, StatusCode> { + s.permit(&id) + .map(|until| Json(json!({"sensor_until":until}))) +} +async fn revoke( + State(s): State, + Path(id): Path, +) -> Result, StatusCode> { + s.revoke(&id)?; + Ok(Json(json!({"sensor_until":0}))) +} +/// Must be wrapped by the exact-owner-token and loopback-peer middleware. +pub fn owner_router(s: Shared) -> Router { + Router::new() + .route("/api/v1/companion/sensors", get(snapshot)) + .route( + "/api/v1/companion/devices/:id/sensors", + post(permit).delete(revoke), + ) + .with_state(s) +} +/// Must be wrapped by pinned companion authentication, which supplies Grant. +pub fn companion_router(s: Shared) -> Router { + Router::new() + .route("/companion/v1/sensors", get(upgrade)) + .with_state(s) +} +async fn upgrade( + State(s): State, + Extension(grant): Extension, + ws: WebSocketUpgrade, +) -> Response { + let (generation, until, stop) = match s.begin(&grant) { + Ok(v) => v, + Err(code) => return code.into_response(), + }; + ws.max_message_size(MAX_BYTES) + .max_frame_size(MAX_BYTES) + .on_upgrade(move |socket| run(s, grant, generation, until, stop, socket)) + .into_response() +} +async fn send(socket: &mut WebSocket, text: String) -> bool { + matches!( + tokio::time::timeout(Duration::from_millis(500), socket.send(Message::Text(text))).await, + Ok(Ok(())) + ) +} +async fn run( + s: Shared, + grant: Grant, + generation: u64, + until: u64, + mut stop: watch::Receiver, + mut socket: WebSocket, +) { + let id = grant.device_id; + let mut revoked = grant.revoked; + let ready = + json!({"type":"sensor_ready","schema":"ruview.companion.sensor-stream.v1"}).to_string(); + let mut previous: Option = None; + let mut last_at: Option = None; + let deadline = tokio::time::sleep(Duration::from_millis( + (until * 1000).saturating_sub(now_ms()), + )); + tokio::pin!(deadline); + if send(&mut socket, ready).await { + loop { + tokio::select! { + _=&mut deadline=>break, _=stop.changed()=>break, _=revoked.changed()=>break, + message=socket.recv()=>{ + if *revoked.borrow() || *stop.borrow()!=generation || until<=now_ms()/1000 {break;} + let text=match message {Some(Ok(Message::Text(t)))=>t,Some(Ok(Message::Ping(_)|Message::Pong(_)))=>continue,_=>break}; + if text.len()>MAX_BYTES {break;} + let frame=match decode(&text){Ok(f)=>f,Err(_)=>break}; + if !frame.valid(now_ms()) || last_at.is_some_and(|t|t.elapsed() (tempfile::TempDir, Shared, watch::Sender, Grant) { + let dir = tempfile::tempdir().unwrap(); + let proof = crate::companion::secret(); + let disk = json!({"version":1,"installation_id":"test","devices":[{"device_id":"phone","device_name":"Test phone","pairing_hash":format!("{:x}",Sha256::digest(proof.as_bytes())),"expires_at":now_ms()/1000+3600}]}); + crate::companion::private_dir(dir.path()).unwrap(); + crate::companion::atomic_private_write( + &dir.path().join("companion-grants.json"), + &serde_json::to_vec(&disk).unwrap(), + ) + .unwrap(); + let companion = Companion::load( + dir.path(), + "test".into(), + "a".repeat(64), + "https://test.local:3443".into(), + ) + .unwrap(); + let store = SensorStore::new(companion); + let (tx, rx) = watch::channel(false); + let grant = Grant { + device_id: "phone".into(), + expires_at: now_ms() / 1000 + 3600, + revoked: rx, + }; + (dir, store, tx, grant) + } + fn frame() -> SensorFrame { + serde_json::from_value(json!({"type":"sensor_frame","schema":"ruview.companion.sensor-frame.v1","shareSessionId":"share","captureSessionId":"capture","coordinateFrameId":"frame","sequence":1,"capturedAtUnixMs":now_ms(),"trackingState":"normal","pointsMm":[1,2,3],"confidences":[2],"cameraTransform":[1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1],"roomBindingDigest":null})).unwrap() + } + #[test] + fn geometry_validation_and_exact_nullable_keys() { + let f = frame(); + assert!(f.valid(now_ms())); + let mut value = serde_json::to_value(&f).unwrap(); + value.as_object_mut().unwrap().remove("confidences"); + assert!(decode(&value.to_string()).is_err()); + let mut empty = f.clone(); + empty.points_mm.clear(); + empty.confidences = Some(vec![]); + assert!(!empty.valid(now_ms())); + let mut uppercase = f.clone(); + uppercase.room_binding_digest = Some("A".repeat(64)); + assert!(!uppercase.valid(now_ms())); + let mut bad = f.clone(); + bad.points_mm = vec![1, 2]; + assert!(!bad.valid(now_ms())); + bad = f.clone(); + bad.camera_transform[0] = -1.; + assert!(!bad.valid(now_ms())); + bad = f.clone(); + bad.camera_transform[3] = 1.; + assert!(!bad.valid(now_ms())); + bad = f.clone(); + bad.captured_at_unix_ms = now_ms() - 3001; + assert!(!bad.valid(now_ms())); + bad = f.clone(); + bad.captured_at_unix_ms = now_ms() + 1001; + assert!(!bad.valid(now_ms())); + bad = f.clone(); + bad.confidences = Some(vec![]); + assert!(!bad.valid(now_ms())); + bad = f.clone(); + bad.points_mm = vec![1; 1539]; + assert!(!bad.valid(now_ms())); + bad = f.clone(); + bad.room_binding_digest = Some("x".repeat(64)); + assert!(!bad.valid(now_ms())); + value = serde_json::to_value(&f).unwrap(); + value["deviceId"] = json!("spoof"); + assert!(decode(&value.to_string()).is_err()); + } + #[test] + fn permission_expiry_revocation_supersession_and_stale() { + let (_dir, s, tx, g) = setup(); + assert!(matches!(s.begin(&g), Err(StatusCode::FORBIDDEN))); + s.permit("phone").unwrap(); + let (old, _, _) = s.begin(&g).unwrap(); + assert!(s.accept("phone", old, frame())); + let (current, _, _) = s.begin(&g).unwrap(); + s.stop("phone", old); + assert!(s.accept("phone", current, frame())); + assert_eq!(s.snapshot().unwrap()["devices"][0]["state"], "live"); + s.entries.lock().unwrap().get_mut("phone").unwrap().received = Some(now_ms() - 3001); + assert_eq!(s.snapshot().unwrap()["devices"][0]["state"], "stale"); + assert!(s.snapshot().unwrap()["devices"][0]["frame"].is_null()); + tx.send_replace(true); + assert!(s.snapshot().unwrap()["devices"][0]["frame"].is_null()); + assert!(!s.accept("phone", current, frame())); + s.permit("phone").unwrap(); + s.entries.lock().unwrap().get_mut("phone").unwrap().until = now_ms() / 1000; + assert_eq!(s.snapshot().unwrap()["devices"][0]["state"], "stopped"); + assert!(matches!(s.begin(&g), Err(StatusCode::UNAUTHORIZED))); + s.revoke("phone").unwrap(); + assert_eq!(s.snapshot().unwrap()["devices"][0]["sensorUntil"], 0); + assert!(matches!(s.permit("unknown"), Err(StatusCode::NOT_FOUND))); + } + async fn server(s: Shared, grant: Option) -> (String, tokio::task::JoinHandle<()>) { + let mut router = companion_router(s); + if let Some(grant) = grant { + router = router.layer(Extension(grant)); + } + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let task = tokio::spawn(async move { + axum::serve(listener, router).await.unwrap(); + }); + (format!("ws://{addr}/companion/v1/sensors"), task) + } + #[tokio::test] + async fn actual_upgrade_requires_permission_and_immediate_ready_then_revoke() { + let (_d, s, _tx, g) = setup(); + let (url, task) = server(s.clone(), Some(g)).await; + assert!(connect_async(&url).await.is_err()); + s.permit("phone").unwrap(); + let (mut socket, _) = connect_async(&url).await.unwrap(); + let ready = tokio::time::timeout(Duration::from_secs(1), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(ready.to_text().unwrap().contains("sensor_ready")); + socket + .send(ClientMessage::Text( + serde_json::to_string(&frame()).unwrap(), + )) + .await + .unwrap(); + assert!(socket + .next() + .await + .unwrap() + .unwrap() + .to_text() + .unwrap() + .contains("sensor_ack")); + assert_eq!(s.snapshot().unwrap()["devices"][0]["deviceId"], "phone"); + use tower::ServiceExt; + let response = owner_router(s.clone()) + .oneshot( + axum::http::Request::builder() + .method("DELETE") + .uri("/api/v1/companion/devices/phone/sensors") + .body(axum::body::Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let receipt: Value = serde_json::from_slice( + &axum::body::to_bytes(response.into_body(), 1024) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!(receipt, json!({"sensor_until":0})); + assert!(s.snapshot().unwrap()["devices"][0]["frame"].is_null()); + let stopped = tokio::time::timeout(Duration::from_secs(1), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(stopped.to_text().unwrap().contains("sensor_stopped")); + task.abort(); + } + #[tokio::test] + async fn actual_socket_rejects_replay_lineage_rate_and_unknown_input() { + for mode in ["replay", "lineage", "rate", "unknown"] { + let (_d, s, _tx, g) = setup(); + s.permit("phone").unwrap(); + let (url, task) = server(s.clone(), Some(g)).await; + let (mut socket, _) = connect_async(&url).await.unwrap(); + socket.next().await.unwrap().unwrap(); + socket + .send(ClientMessage::Text( + serde_json::to_string(&frame()).unwrap(), + )) + .await + .unwrap(); + socket.next().await.unwrap().unwrap(); + if mode != "rate" { + tokio::time::sleep(Duration::from_millis(210)).await; + } + let mut next = frame(); + if mode != "replay" { + next.sequence = 2; + } + if mode == "lineage" { + next.coordinate_frame_id = "changed".into(); + } + let mut value = serde_json::to_value(next).unwrap(); + if mode == "unknown" { + value["rgb"] = json!("forbidden"); + } + socket + .send(ClientMessage::Text(value.to_string())) + .await + .unwrap(); + let stopped = tokio::time::timeout(Duration::from_secs(1), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!( + stopped.to_text().unwrap().contains("sensor_stopped"), + "{mode}" + ); + assert!(s.snapshot().unwrap()["devices"][0]["frame"].is_null()); + task.abort(); + } + } + #[tokio::test] + async fn pairing_revocation_removes_preview_and_prevents_regrant() { + use tower::ServiceExt; + let (_d, s, _tx, g) = setup(); + s.permit("phone").unwrap(); + let (generation, _, _) = s.begin(&g).unwrap(); + assert!(s.accept("phone", generation, frame())); + let response = crate::companion::owner_router(s.companion.clone()) + .oneshot( + axum::http::Request::builder() + .method("DELETE") + .uri("/api/v1/companion/devices/phone") + .body(axum::body::Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert!(s.snapshot().unwrap()["devices"] + .as_array() + .unwrap() + .is_empty()); + assert!(matches!(s.permit("phone"), Err(StatusCode::NOT_FOUND))); + assert!(!s.accept("phone", generation, frame())); + } + #[tokio::test] + async fn established_socket_expires_sensor_and_access_permission() { + for sensor_expiry in [true, false] { + let (_d, s, _tx, mut g) = setup(); + s.permit("phone").unwrap(); + if sensor_expiry { + s.entries.lock().unwrap().get_mut("phone").unwrap().until = now_ms() / 1000 + 1; + } else { + g.expires_at = now_ms() / 1000 + 1; + } + let (url, task) = server(s.clone(), Some(g)).await; + let (mut socket, _) = connect_async(&url).await.unwrap(); + socket.next().await.unwrap().unwrap(); + socket + .send(ClientMessage::Text( + serde_json::to_string(&frame()).unwrap(), + )) + .await + .unwrap(); + socket.next().await.unwrap().unwrap(); + let stopped = tokio::time::timeout(Duration::from_secs(2), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + assert!(stopped.to_text().unwrap().contains("sensor_stopped")); + assert!(s.snapshot().unwrap()["devices"][0]["frame"].is_null()); + task.abort(); + } + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/lib.rs b/v2/crates/wifi-densepose-sensing-server/src/lib.rs index c7e54ab8ec..3e3a5460c1 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/lib.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/lib.rs @@ -72,3 +72,5 @@ pub use wifi_densepose_aether::{embedding, graph_transformer, sona, sparse_infer pub mod companion; pub mod companion_tls; + +pub mod companion_sensors; From a686cca09653bafcd8b75d297e6c40cda6dc1d68 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Wed, 21 Oct 2026 08:46:27 +0200 Subject: [PATCH 007/100] fix(security): patch TLS dependencies and migrate ZIP consumers safely --- .github/workflows/security-scan.yml | 3 + scripts/verify_zip_parent_patches.py | 23 + v2/Cargo.lock | 593 +- v2/Cargo.toml | 11 +- .../tests/zip9_tensor_roundtrip.rs | 14 + .../tests/zip9_tensor_roundtrip.rs | 15 + v2/dependency-patches/README.md | 21 + .../candle-core-0.4.1/Cargo.toml | 143 + .../candle-core-0.4.1/Cargo.toml.orig | 50 + .../candle-core-0.4.1/LICENSE | 201 + .../candle-core-0.4.1/README.md | 2 + .../candle-core-0.4.1/benches/bench_main.rs | 9 + .../benches/benchmarks/affine.rs | 43 + .../benches/benchmarks/matmul.rs | 44 + .../benches/benchmarks/mod.rs | 66 + .../benches/benchmarks/random.rs | 63 + .../benches/benchmarks/where_cond.rs | 64 + .../candle-core-0.4.1/examples/basics.rs | 17 + .../candle-core-0.4.1/examples/cuda_basics.rs | 36 + .../examples/cuda_sum_benchmark.rs | 54 + .../examples/tensor-tools.rs | 389 + .../candle-core-0.4.1/src/accelerate.rs | 476 + .../candle-core-0.4.1/src/backend.rs | 124 + .../candle-core-0.4.1/src/backprop.rs | 728 + .../candle-core-0.4.1/src/conv.rs | 358 + .../candle-core-0.4.1/src/convert.rs | 145 + .../candle-core-0.4.1/src/cpu/avx.rs | 148 + .../candle-core-0.4.1/src/cpu/erf.rs | 763 + .../candle-core-0.4.1/src/cpu/kernels.rs | 191 + .../candle-core-0.4.1/src/cpu/mod.rs | 180 + .../candle-core-0.4.1/src/cpu/neon.rs | 74 + .../candle-core-0.4.1/src/cpu/simd128.rs | 64 + .../candle-core-0.4.1/src/cpu_backend.rs | 2819 ++ .../candle-core-0.4.1/src/cuda_backend.rs | 2249 + .../candle-core-0.4.1/src/cudnn.rs | 123 + .../candle-core-0.4.1/src/device.rs | 323 + .../candle-core-0.4.1/src/display.rs | 495 + .../candle-core-0.4.1/src/dtype.rs | 201 + .../src/dummy_cuda_backend.rs | 211 + .../src/dummy_metal_backend.rs | 223 + .../candle-core-0.4.1/src/error.rs | 265 + .../candle-core-0.4.1/src/indexer.rs | 168 + .../candle-core-0.4.1/src/layout.rs | 271 + .../candle-core-0.4.1/src/lib.rs | 151 + .../candle-core-0.4.1/src/metal_backend.rs | 1699 + .../candle-core-0.4.1/src/mkl.rs | 418 + .../candle-core-0.4.1/src/npy.rs | 450 + .../candle-core-0.4.1/src/op.rs | 1069 + .../candle-core-0.4.1/src/pickle.rs | 817 + .../candle-core-0.4.1/src/quantized/avx.rs | 667 + .../candle-core-0.4.1/src/quantized/cuda.rs | 321 + .../src/quantized/dummy_cuda.rs | 50 + .../src/quantized/dummy_metal.rs | 50 + .../src/quantized/ggml_file.rs | 265 + .../src/quantized/gguf_file.rs | 534 + .../src/quantized/k_quants.rs | 1965 + .../candle-core-0.4.1/src/quantized/metal.rs | 217 + .../candle-core-0.4.1/src/quantized/mod.rs | 491 + .../candle-core-0.4.1/src/quantized/neon.rs | 613 + .../src/quantized/simd128.rs | 419 + .../candle-core-0.4.1/src/quantized/utils.rs | 326 + .../candle-core-0.4.1/src/safetensors.rs | 441 + .../candle-core-0.4.1/src/scalar.rs | 23 + .../candle-core-0.4.1/src/shape.rs | 629 + .../candle-core-0.4.1/src/storage.rs | 700 + .../candle-core-0.4.1/src/strided_index.rs | 82 + .../candle-core-0.4.1/src/tensor.rs | 2765 ++ .../candle-core-0.4.1/src/test_utils.rs | 62 + .../candle-core-0.4.1/src/utils.rs | 44 + .../candle-core-0.4.1/src/variable.rs | 147 + .../candle-core-0.4.1/tests/conv_tests.rs | 609 + .../tests/custom_op_tests.rs | 114 + .../candle-core-0.4.1/tests/display_tests.rs | 84 + .../tests/fortran_tensor_3d.pth | Bin 0 -> 1486 bytes .../candle-core-0.4.1/tests/grad_tests.rs | 531 + .../candle-core-0.4.1/tests/indexing_tests.rs | 122 + .../candle-core-0.4.1/tests/layout_tests.rs | 138 + .../candle-core-0.4.1/tests/npy.py | 9 + .../candle-core-0.4.1/tests/pool_tests.rs | 114 + .../candle-core-0.4.1/tests/pth.py | 37 + .../candle-core-0.4.1/tests/pth_tests.rs | 31 + .../tests/quantized_tests.rs | 1002 + .../tests/serialization_tests.rs | 24 + .../candle-core-0.4.1/tests/tensor_tests.rs | 1274 + .../candle-core-0.4.1/tests/test.npy | Bin 0 -> 208 bytes .../candle-core-0.4.1/tests/test.npz | Bin 0 -> 668 bytes .../candle-core-0.4.1/tests/test.pt | Bin 0 -> 1165 bytes .../candle-core-0.4.1/tests/test_with_key.pt | Bin 0 -> 1338 bytes .../candle-core-0.9.2/Cargo.toml | 262 + .../candle-core-0.9.2/Cargo.toml.orig | 70 + .../candle-core-0.9.2/LICENSE | 201 + .../candle-core-0.9.2/README.md | 2 + .../candle-core-0.9.2/benches/bench_main.rs | 17 + .../benches/benchmarks/affine.rs | 46 + .../benches/benchmarks/binary.rs | 57 + .../benches/benchmarks/broadcast.rs | 47 + .../benches/benchmarks/conv_transpose2d.rs | 60 + .../benches/benchmarks/copy.rs | 39 + .../benches/benchmarks/matmul.rs | 70 + .../benches/benchmarks/mod.rs | 77 + .../benches/benchmarks/qmatmul.rs | 73 + .../benches/benchmarks/random.rs | 64 + .../benches/benchmarks/reduce.rs | 159 + .../benches/benchmarks/unary.rs | 98 + .../benches/benchmarks/where_cond.rs | 65 + .../candle-core-0.9.2/examples/basics.rs | 20 + .../candle-core-0.9.2/examples/cuda_basics.rs | 23 + .../examples/cuda_sum_benchmark.rs | 54 + .../examples/metal_basics.rs | 28 + .../candle-core-0.9.2/src/accelerate.rs | 476 + .../candle-core-0.9.2/src/backend.rs | 174 + .../candle-core-0.9.2/src/backprop.rs | 784 + .../candle-core-0.9.2/src/conv.rs | 387 + .../candle-core-0.9.2/src/convert.rs | 166 + .../candle-core-0.9.2/src/cpu/avx.rs | 227 + .../candle-core-0.9.2/src/cpu/erf.rs | 325 + .../candle-core-0.9.2/src/cpu/kernels.rs | 232 + .../candle-core-0.9.2/src/cpu/mod.rs | 242 + .../candle-core-0.9.2/src/cpu/neon.rs | 74 + .../candle-core-0.9.2/src/cpu/simd128.rs | 64 + .../src/cpu_backend/conv2d.rs | 432 + .../candle-core-0.9.2/src/cpu_backend/mod.rs | 3284 ++ .../src/cpu_backend/utils.rs | 409 + .../src/cuda_backend/cudnn.rs | 225 + .../src/cuda_backend/device.rs | 710 + .../src/cuda_backend/error.rs | 62 + .../candle-core-0.9.2/src/cuda_backend/mod.rs | 2682 ++ .../src/cuda_backend/utils.rs | 194 + .../candle-core-0.9.2/src/custom_op.rs | 489 + .../candle-core-0.9.2/src/device.rs | 502 + .../candle-core-0.9.2/src/display.rs | 558 + .../candle-core-0.9.2/src/dtype.rs | 294 + .../src/dummy_cuda_backend.rs | 317 + .../candle-core-0.9.2/src/dummy_dtype.rs | 268 + .../src/dummy_metal_backend.rs | 280 + .../candle-core-0.9.2/src/error.rs | 394 + .../candle-core-0.9.2/src/indexer.rs | 257 + .../candle-core-0.9.2/src/layout.rs | 278 + .../candle-core-0.9.2/src/lib.rs | 177 + .../src/metal_backend/device.rs | 287 + .../src/metal_backend/mod.rs | 2137 + .../candle-core-0.9.2/src/mkl.rs | 418 + .../candle-core-0.9.2/src/npy.rs | 477 + .../candle-core-0.9.2/src/op.rs | 1148 + .../candle-core-0.9.2/src/pickle.rs | 841 + .../candle-core-0.9.2/src/quantized/avx.rs | 669 + .../candle-core-0.9.2/src/quantized/cuda.rs | 983 + .../src/quantized/dummy_cuda.rs | 95 + .../src/quantized/dummy_metal.rs | 87 + .../src/quantized/ggml_file.rs | 266 + .../src/quantized/gguf_file.rs | 539 + .../src/quantized/imatrix_file.rs | 85 + .../src/quantized/k_quants.rs | 2505 + .../candle-core-0.9.2/src/quantized/metal.rs | 391 + .../candle-core-0.9.2/src/quantized/mod.rs | 881 + .../candle-core-0.9.2/src/quantized/neon.rs | 615 + .../src/quantized/simd128.rs | 418 + .../candle-core-0.9.2/src/quantized/utils.rs | 562 + .../candle-core-0.9.2/src/safetensors.rs | 644 + .../candle-core-0.9.2/src/scalar.rs | 115 + .../candle-core-0.9.2/src/shape.rs | 634 + .../candle-core-0.9.2/src/sort.rs | 290 + .../candle-core-0.9.2/src/storage.rs | 849 + .../candle-core-0.9.2/src/streaming.rs | 208 + .../candle-core-0.9.2/src/strided_index.rs | 94 + .../candle-core-0.9.2/src/tensor.rs | 3116 ++ .../candle-core-0.9.2/src/tensor_cat.rs | 303 + .../candle-core-0.9.2/src/test_utils.rs | 71 + .../candle-core-0.9.2/src/utils.rs | 45 + .../candle-core-0.9.2/src/variable.rs | 152 + .../candle-core-0.9.2/tests/bilinear_tests.rs | 525 + .../candle-core-0.9.2/tests/conv_tests.rs | 894 + .../tests/custom_op_tests.rs | 182 + .../candle-core-0.9.2/tests/display_tests.rs | 84 + .../tests/fortran_tensor_3d.pth | Bin 0 -> 1486 bytes .../candle-core-0.9.2/tests/grad_tests.rs | 563 + .../candle-core-0.9.2/tests/indexing_tests.rs | 122 + .../candle-core-0.9.2/tests/layout_tests.rs | 152 + .../candle-core-0.9.2/tests/matmul_tests.rs | 146 + .../candle-core-0.9.2/tests/npy.py | 9 + .../candle-core-0.9.2/tests/pool_tests.rs | 117 + .../candle-core-0.9.2/tests/pth.py | 37 + .../candle-core-0.9.2/tests/pth_tests.rs | 31 + .../tests/quantized_tests.rs | 1402 + .../tests/serialization_tests.rs | 71 + .../candle-core-0.9.2/tests/tensor_tests.rs | 2062 + .../candle-core-0.9.2/tests/test.npy | Bin 0 -> 208 bytes .../candle-core-0.9.2/tests/test.npz | Bin 0 -> 668 bytes .../candle-core-0.9.2/tests/test.pt | Bin 0 -> 1165 bytes .../candle-core-0.9.2/tests/test_with_key.pt | Bin 0 -> 1338 bytes v2/dependency-patches/compat-tests/Cargo.lock | 887 + v2/dependency-patches/compat-tests/Cargo.toml | 13 + v2/dependency-patches/compat-tests/src/lib.rs | 14 + v2/dependency-patches/compatibility.patch | 185 + v2/dependency-patches/patches.json | 1877 + .../tch-0.24.0/.cargo/config.toml | 2 + .../tch-0.24.0/.github/workflows/rust-ci.yml | 74 + v2/dependency-patches/tch-0.24.0/.gitignore | 19 + .../tch-0.24.0/.python-version | 1 + v2/dependency-patches/tch-0.24.0/CHANGELOG.md | 143 + v2/dependency-patches/tch-0.24.0/Cargo.toml | 246 + .../tch-0.24.0/Cargo.toml.orig | 67 + .../tch-0.24.0/LICENSE-APACHE | 201 + v2/dependency-patches/tch-0.24.0/LICENSE-MIT | 23 + v2/dependency-patches/tch-0.24.0/Makefile | 13 + v2/dependency-patches/tch-0.24.0/README.md | 372 + v2/dependency-patches/tch-0.24.0/build.rs | 13 + v2/dependency-patches/tch-0.24.0/clippy.toml | 1 + v2/dependency-patches/tch-0.24.0/dune-project | 1 + .../tch-0.24.0/examples/basics.rs | 36 + .../tch-0.24.0/examples/char-rnn/README.md | 43 + .../tch-0.24.0/examples/char-rnn/main.rs | 68 + .../tch-0.24.0/examples/cifar/main.rs | 72 + .../examples/custom-optimizer/README.md | 21 + .../examples/custom-optimizer/main.rs | 50 + .../examples/custom-optimizer/sparse_adam.rs | 147 + .../tch-0.24.0/examples/gan/main.rs | 148 + .../examples/jit-quantized/README.md | 40 + .../tch-0.24.0/examples/jit-quantized/main.rs | 51 + .../examples/jit-quantized/resnet.py | 22 + .../tch-0.24.0/examples/jit-trace/README.md | 27 + .../tch-0.24.0/examples/jit-trace/main.rs | 63 + .../tch-0.24.0/examples/jit-trace/test.py | 45 + .../tch-0.24.0/examples/jit-train/README.md | 136 + .../tch-0.24.0/examples/jit-train/main.rs | 57 + .../tch-0.24.0/examples/jit-train/resnet.py | 27 + .../tch-0.24.0/examples/jit/README.md | 116 + .../tch-0.24.0/examples/jit/main.rs | 27 + .../tch-0.24.0/examples/jit/resnet.py | 8 + .../examples/llama/convert_checkpoint.py | 63 + .../tch-0.24.0/examples/llama/main.rs | 397 + .../examples/llama/sentencepiece.rs | 149 + .../tch-0.24.0/examples/memory_test.rs | 19 + .../tch-0.24.0/examples/min-gpt/README.md | 78 + .../tch-0.24.0/examples/min-gpt/main.rs | 212 + .../tch-0.24.0/examples/mnist/README.md | 100 + .../tch-0.24.0/examples/mnist/main.rs | 25 + .../tch-0.24.0/examples/mnist/mnist_conv.rs | 54 + .../tch-0.24.0/examples/mnist/mnist_linear.rs | 42 + .../tch-0.24.0/examples/mnist/mnist_nn.rs | 34 + .../examples/neural-style-transfer/README.md | 162 + .../examples/neural-style-transfer/main.rs | 72 + .../examples/pretrained-models/main.rs | 81 + .../examples/reinforcement-learning/README.md | 36 + .../examples/reinforcement-learning/a2c.rs | 171 + .../reinforcement-learning/atari_wrappers.py | 308 + .../examples/reinforcement-learning/ddpg.rs | 343 + .../reinforcement-learning/gym_env.rs | 78 + .../examples/reinforcement-learning/main.rs | 20 + .../reinforcement-learning/policy_gradient.rs | 84 + .../examples/reinforcement-learning/ppo.rs | 181 + .../reinforcement-learning/vec_gym_env.rs | 66 + .../examples/stable-diffusion/README.md | 11 + .../examples/stable-diffusion/main.rs | 2512 + .../tch-0.24.0/examples/tensor-tools.rs | 79 + .../examples/transfer-learning/README.md | 141 + .../examples/transfer-learning/main.rs | 42 + .../examples/translation/dataset.rs | 115 + .../tch-0.24.0/examples/translation/lang.rs | 73 + .../tch-0.24.0/examples/translation/main.rs | 224 + .../tch-0.24.0/examples/vae/main.rs | 95 + .../tch-0.24.0/examples/yolo/coco_classes.rs | 82 + .../tch-0.24.0/examples/yolo/darknet.rs | 286 + .../tch-0.24.0/examples/yolo/main.rs | 153 + .../tch-0.24.0/examples/yolo/yolo-v3.cfg | 790 + .../tch-0.24.0/gen/.ocamlformat | 2 + v2/dependency-patches/tch-0.24.0/gen/dune | 3 + v2/dependency-patches/tch-0.24.0/gen/gen.ml | 892 + v2/dependency-patches/tch-0.24.0/gen/gen.mli | 1 + v2/dependency-patches/tch-0.24.0/main.py | 6 + .../tch-0.24.0/pyproject.toml | 9 + v2/dependency-patches/tch-0.24.0/rustfmt.toml | 2 + v2/dependency-patches/tch-0.24.0/src/data.rs | 203 + v2/dependency-patches/tch-0.24.0/src/error.rs | 74 + v2/dependency-patches/tch-0.24.0/src/lib.rs | 32 + .../tch-0.24.0/src/nn/batch_norm.rs | 132 + .../tch-0.24.0/src/nn/conv.rs | 242 + .../tch-0.24.0/src/nn/conv_transpose.rs | 189 + .../tch-0.24.0/src/nn/func.rs | 51 + .../tch-0.24.0/src/nn/group_norm.rs | 66 + .../tch-0.24.0/src/nn/init.rs | 212 + .../tch-0.24.0/src/nn/layer_norm.rs | 65 + .../tch-0.24.0/src/nn/linear.rs | 74 + .../tch-0.24.0/src/nn/mod.rs | 69 + .../tch-0.24.0/src/nn/module.rs | 67 + .../tch-0.24.0/src/nn/optimizer.rs | 324 + .../tch-0.24.0/src/nn/rnn.rs | 285 + .../tch-0.24.0/src/nn/sequential.rs | 149 + .../tch-0.24.0/src/nn/sparse.rs | 55 + .../tch-0.24.0/src/nn/var_store.rs | 935 + .../tch-0.24.0/src/tensor/convert.rs | 182 + .../tch-0.24.0/src/tensor/display.rs | 434 + .../tch-0.24.0/src/tensor/index.rs | 372 + .../tch-0.24.0/src/tensor/iter.rs | 62 + .../tch-0.24.0/src/tensor/mod.rs | 253 + .../tch-0.24.0/src/tensor/npy.rs | 277 + .../tch-0.24.0/src/tensor/ops.rs | 307 + .../tch-0.24.0/src/tensor/safetensors.rs | 183 + .../tch-0.24.0/src/vision/alexnet.rs | 44 + .../tch-0.24.0/src/vision/cifar.rs | 64 + .../tch-0.24.0/src/vision/convmixer.rs | 49 + .../tch-0.24.0/src/vision/dataset.rs | 95 + .../tch-0.24.0/src/vision/densenet.rs | 96 + .../tch-0.24.0/src/vision/dinov2.rs | 234 + .../tch-0.24.0/src/vision/efficientnet.rs | 329 + .../tch-0.24.0/src/vision/export_dinov2.py | 16 + .../tch-0.24.0/src/vision/export_model.py | 16 + .../tch-0.24.0/src/vision/image.rs | 146 + .../tch-0.24.0/src/vision/imagenet.rs | 1170 + .../tch-0.24.0/src/vision/inception.rs | 154 + .../tch-0.24.0/src/vision/mnist.rs | 69 + .../tch-0.24.0/src/vision/mobilenet.rs | 83 + .../tch-0.24.0/src/vision/mod.rs | 34 + .../tch-0.24.0/src/vision/resnet.rs | 183 + .../tch-0.24.0/src/vision/rust_image.rs | 145 + .../tch-0.24.0/src/vision/squeezenet.rs | 68 + .../tch-0.24.0/src/vision/vgg.rs | 100 + .../tch-0.24.0/src/wrappers/device.rs | 120 + .../tch-0.24.0/src/wrappers/image.rs | 35 + .../tch-0.24.0/src/wrappers/jit.rs | 879 + .../tch-0.24.0/src/wrappers/kind.rs | 176 + .../tch-0.24.0/src/wrappers/layout.rs | 29 + .../tch-0.24.0/src/wrappers/mod.rs | 21 + .../tch-0.24.0/src/wrappers/optimizer.rs | 129 + .../tch-0.24.0/src/wrappers/python.rs | 34 + .../tch-0.24.0/src/wrappers/scalar.rs | 111 + .../tch-0.24.0/src/wrappers/stream.rs | 26 + .../tch-0.24.0/src/wrappers/tensor.rs | 883 + .../src/wrappers/tensor_fallible_generated.rs | 38995 ++++++++++++++++ .../src/wrappers/tensor_generated.rs | 19703 ++++++++ .../tch-0.24.0/src/wrappers/utils.rs | 157 + .../tch-0.24.0/tests/autocast.rs | 22 + .../tch-0.24.0/tests/create_jit_models.py | 119 + .../tch-0.24.0/tests/data_tests.rs | 60 + .../tch-0.24.0/tests/device_tests.rs | 7 + .../tch-0.24.0/tests/display_tests.rs | 79 + v2/dependency-patches/tch-0.24.0/tests/foo.pt | Bin 0 -> 2030 bytes .../tch-0.24.0/tests/foo1.pt | Bin 0 -> 1659 bytes .../tch-0.24.0/tests/foo2.pt | Bin 0 -> 1723 bytes .../tch-0.24.0/tests/foo3.pt | Bin 0 -> 1723 bytes .../tch-0.24.0/tests/foo4.pt | Bin 0 -> 1595 bytes .../tch-0.24.0/tests/foo5.pt | Bin 0 -> 1531 bytes .../tch-0.24.0/tests/foo6.pt | Bin 0 -> 1659 bytes .../tch-0.24.0/tests/foo7.pt | Bin 0 -> 1787 bytes .../tch-0.24.0/tests/foo8.pt | Bin 0 -> 1595 bytes .../tch-0.24.0/tests/jit_tests.rs | 190 + .../tch-0.24.0/tests/linear/bs.npy | Bin 0 -> 144 bytes .../tch-0.24.0/tests/linear/gen-test-data.py | 16 + .../tch-0.24.0/tests/linear/in.npy | Bin 0 -> 2184 bytes .../tch-0.24.0/tests/linear/out.npy | Bin 0 -> 160 bytes .../tch-0.24.0/tests/linear/ws.npy | Bin 0 -> 4240 bytes .../tch-0.24.0/tests/nn_tests.rs | 412 + .../tch-0.24.0/tests/serialization_tests.rs | 195 + .../tch-0.24.0/tests/tensor_indexing.rs | 229 + .../tch-0.24.0/tests/tensor_tests.rs | 508 + .../tch-0.24.0/tests/test_utils.rs | 44 + .../tch-0.24.0/tests/var_store.rs | 634 + .../tch-0.24.0/tests/vision_tests.rs | 34 + v2/dependency-patches/tch-0.24.0/uv.lock | 433 + .../torch-sys-0.24.0/Cargo.toml | 79 + .../torch-sys-0.24.0/Cargo.toml.orig | 32 + .../torch-sys-0.24.0/build.rs | 493 + .../torch-sys-0.24.0/libtch/CMakeLists.txt | 9 + .../torch-sys-0.24.0/libtch/stb_image.h | 7530 +++ .../libtch/stb_image_resize.h | 2627 ++ .../torch-sys-0.24.0/libtch/stb_image_write.h | 1621 + .../torch-sys-0.24.0/libtch/torch_api.cpp | 1356 + .../torch-sys-0.24.0/libtch/torch_api.h | 295 + .../libtch/torch_api_generated.cpp | 19081 ++++++++ .../libtch/torch_api_generated.h | 2648 ++ .../torch-sys-0.24.0/libtch/torch_python.cpp | 22 + .../torch-sys-0.24.0/libtch/torch_python.h | 19 + .../torch-sys-0.24.0/src/c_generated.rs | 16055 +++++++ .../torch-sys-0.24.0/src/cuda.rs | 30 + .../torch-sys-0.24.0/src/io.rs | 111 + .../torch-sys-0.24.0/src/lib.rs | 336 + .../torch-sys-0.24.0/src/python.rs | 12 + .../torch-sys-0.24.0/src/traits.rs | 119 + 378 files changed, 213334 insertions(+), 240 deletions(-) create mode 100644 scripts/verify_zip_parent_patches.py create mode 100644 v2/crates/cog-person-count/tests/zip9_tensor_roundtrip.rs create mode 100644 v2/crates/wifi-densepose-nn/tests/zip9_tensor_roundtrip.rs create mode 100644 v2/dependency-patches/README.md create mode 100644 v2/dependency-patches/candle-core-0.4.1/Cargo.toml create mode 100644 v2/dependency-patches/candle-core-0.4.1/Cargo.toml.orig create mode 100644 v2/dependency-patches/candle-core-0.4.1/LICENSE create mode 100644 v2/dependency-patches/candle-core-0.4.1/README.md create mode 100644 v2/dependency-patches/candle-core-0.4.1/benches/bench_main.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/affine.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/matmul.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/mod.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/random.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/where_cond.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/examples/basics.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/examples/cuda_basics.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/examples/cuda_sum_benchmark.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/examples/tensor-tools.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/accelerate.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/backend.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/backprop.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/conv.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/convert.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/cpu/avx.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/cpu/erf.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/cpu/kernels.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/cpu/mod.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/cpu/neon.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/cpu/simd128.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/cpu_backend.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/cuda_backend.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/cudnn.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/device.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/display.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/dtype.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/dummy_cuda_backend.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/dummy_metal_backend.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/error.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/indexer.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/layout.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/lib.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/metal_backend.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/mkl.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/npy.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/op.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/pickle.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/avx.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/cuda.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/dummy_cuda.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/dummy_metal.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/ggml_file.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/gguf_file.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/k_quants.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/metal.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/mod.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/neon.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/simd128.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/utils.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/safetensors.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/scalar.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/shape.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/storage.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/strided_index.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/tensor.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/test_utils.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/utils.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/src/variable.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/conv_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/custom_op_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/display_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/fortran_tensor_3d.pth create mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/grad_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/indexing_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/layout_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/npy.py create mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/pool_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/pth.py create mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/pth_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/quantized_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/serialization_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/tensor_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/test.npy create mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/test.npz create mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/test.pt create mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/test_with_key.pt create mode 100644 v2/dependency-patches/candle-core-0.9.2/Cargo.toml create mode 100644 v2/dependency-patches/candle-core-0.9.2/Cargo.toml.orig create mode 100644 v2/dependency-patches/candle-core-0.9.2/LICENSE create mode 100644 v2/dependency-patches/candle-core-0.9.2/README.md create mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/bench_main.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/affine.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/binary.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/broadcast.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/conv_transpose2d.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/copy.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/matmul.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/mod.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/qmatmul.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/random.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/reduce.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/unary.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/where_cond.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/examples/basics.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/examples/cuda_basics.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/examples/cuda_sum_benchmark.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/examples/metal_basics.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/accelerate.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/backend.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/backprop.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/conv.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/convert.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cpu/avx.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cpu/erf.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cpu/kernels.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cpu/mod.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cpu/neon.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cpu/simd128.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/conv2d.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/mod.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/utils.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/cudnn.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/device.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/error.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/mod.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/utils.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/custom_op.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/device.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/display.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/dtype.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/dummy_cuda_backend.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/dummy_dtype.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/dummy_metal_backend.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/error.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/indexer.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/layout.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/lib.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/metal_backend/device.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/metal_backend/mod.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/mkl.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/npy.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/op.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/pickle.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/avx.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/cuda.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/dummy_cuda.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/dummy_metal.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/ggml_file.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/gguf_file.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/imatrix_file.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/k_quants.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/metal.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/mod.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/neon.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/simd128.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/utils.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/safetensors.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/scalar.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/shape.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/sort.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/storage.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/streaming.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/strided_index.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/tensor.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/tensor_cat.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/test_utils.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/utils.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/src/variable.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/bilinear_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/conv_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/custom_op_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/display_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/fortran_tensor_3d.pth create mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/grad_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/indexing_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/layout_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/matmul_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/npy.py create mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/pool_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/pth.py create mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/pth_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/quantized_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/serialization_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/tensor_tests.rs create mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/test.npy create mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/test.npz create mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/test.pt create mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/test_with_key.pt create mode 100644 v2/dependency-patches/compat-tests/Cargo.lock create mode 100644 v2/dependency-patches/compat-tests/Cargo.toml create mode 100644 v2/dependency-patches/compat-tests/src/lib.rs create mode 100644 v2/dependency-patches/compatibility.patch create mode 100644 v2/dependency-patches/patches.json create mode 100644 v2/dependency-patches/tch-0.24.0/.cargo/config.toml create mode 100644 v2/dependency-patches/tch-0.24.0/.github/workflows/rust-ci.yml create mode 100644 v2/dependency-patches/tch-0.24.0/.gitignore create mode 100644 v2/dependency-patches/tch-0.24.0/.python-version create mode 100644 v2/dependency-patches/tch-0.24.0/CHANGELOG.md create mode 100644 v2/dependency-patches/tch-0.24.0/Cargo.toml create mode 100644 v2/dependency-patches/tch-0.24.0/Cargo.toml.orig create mode 100644 v2/dependency-patches/tch-0.24.0/LICENSE-APACHE create mode 100644 v2/dependency-patches/tch-0.24.0/LICENSE-MIT create mode 100644 v2/dependency-patches/tch-0.24.0/Makefile create mode 100644 v2/dependency-patches/tch-0.24.0/README.md create mode 100644 v2/dependency-patches/tch-0.24.0/build.rs create mode 100644 v2/dependency-patches/tch-0.24.0/clippy.toml create mode 100644 v2/dependency-patches/tch-0.24.0/dune-project create mode 100644 v2/dependency-patches/tch-0.24.0/examples/basics.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/char-rnn/README.md create mode 100644 v2/dependency-patches/tch-0.24.0/examples/char-rnn/main.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/cifar/main.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/README.md create mode 100644 v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/main.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/sparse_adam.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/gan/main.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit-quantized/README.md create mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit-quantized/main.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit-quantized/resnet.py create mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit-trace/README.md create mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit-trace/main.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit-trace/test.py create mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit-train/README.md create mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit-train/main.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit-train/resnet.py create mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit/README.md create mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit/main.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit/resnet.py create mode 100644 v2/dependency-patches/tch-0.24.0/examples/llama/convert_checkpoint.py create mode 100644 v2/dependency-patches/tch-0.24.0/examples/llama/main.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/llama/sentencepiece.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/memory_test.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/min-gpt/README.md create mode 100644 v2/dependency-patches/tch-0.24.0/examples/min-gpt/main.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/mnist/README.md create mode 100644 v2/dependency-patches/tch-0.24.0/examples/mnist/main.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_conv.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_linear.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_nn.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/neural-style-transfer/README.md create mode 100644 v2/dependency-patches/tch-0.24.0/examples/neural-style-transfer/main.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/pretrained-models/main.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/README.md create mode 100644 v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/a2c.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/atari_wrappers.py create mode 100644 v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/ddpg.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/gym_env.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/main.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/policy_gradient.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/ppo.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/vec_gym_env.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/stable-diffusion/README.md create mode 100644 v2/dependency-patches/tch-0.24.0/examples/stable-diffusion/main.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/tensor-tools.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/transfer-learning/README.md create mode 100644 v2/dependency-patches/tch-0.24.0/examples/transfer-learning/main.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/translation/dataset.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/translation/lang.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/translation/main.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/vae/main.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/yolo/coco_classes.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/yolo/darknet.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/yolo/main.rs create mode 100644 v2/dependency-patches/tch-0.24.0/examples/yolo/yolo-v3.cfg create mode 100644 v2/dependency-patches/tch-0.24.0/gen/.ocamlformat create mode 100644 v2/dependency-patches/tch-0.24.0/gen/dune create mode 100644 v2/dependency-patches/tch-0.24.0/gen/gen.ml create mode 100644 v2/dependency-patches/tch-0.24.0/gen/gen.mli create mode 100644 v2/dependency-patches/tch-0.24.0/main.py create mode 100644 v2/dependency-patches/tch-0.24.0/pyproject.toml create mode 100644 v2/dependency-patches/tch-0.24.0/rustfmt.toml create mode 100644 v2/dependency-patches/tch-0.24.0/src/data.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/error.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/lib.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/batch_norm.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/conv.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/conv_transpose.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/func.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/group_norm.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/init.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/layer_norm.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/linear.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/mod.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/module.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/optimizer.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/rnn.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/sequential.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/sparse.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/var_store.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/tensor/convert.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/tensor/display.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/tensor/index.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/tensor/iter.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/tensor/mod.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/tensor/npy.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/tensor/ops.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/tensor/safetensors.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/alexnet.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/cifar.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/convmixer.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/dataset.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/densenet.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/dinov2.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/efficientnet.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/export_dinov2.py create mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/export_model.py create mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/image.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/imagenet.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/inception.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/mnist.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/mobilenet.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/mod.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/resnet.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/rust_image.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/squeezenet.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/vgg.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/device.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/image.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/jit.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/kind.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/layout.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/mod.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/optimizer.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/python.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/scalar.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/stream.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/tensor.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/tensor_fallible_generated.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/tensor_generated.rs create mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/utils.rs create mode 100644 v2/dependency-patches/tch-0.24.0/tests/autocast.rs create mode 100644 v2/dependency-patches/tch-0.24.0/tests/create_jit_models.py create mode 100644 v2/dependency-patches/tch-0.24.0/tests/data_tests.rs create mode 100644 v2/dependency-patches/tch-0.24.0/tests/device_tests.rs create mode 100644 v2/dependency-patches/tch-0.24.0/tests/display_tests.rs create mode 100644 v2/dependency-patches/tch-0.24.0/tests/foo.pt create mode 100644 v2/dependency-patches/tch-0.24.0/tests/foo1.pt create mode 100644 v2/dependency-patches/tch-0.24.0/tests/foo2.pt create mode 100644 v2/dependency-patches/tch-0.24.0/tests/foo3.pt create mode 100644 v2/dependency-patches/tch-0.24.0/tests/foo4.pt create mode 100644 v2/dependency-patches/tch-0.24.0/tests/foo5.pt create mode 100644 v2/dependency-patches/tch-0.24.0/tests/foo6.pt create mode 100644 v2/dependency-patches/tch-0.24.0/tests/foo7.pt create mode 100644 v2/dependency-patches/tch-0.24.0/tests/foo8.pt create mode 100644 v2/dependency-patches/tch-0.24.0/tests/jit_tests.rs create mode 100644 v2/dependency-patches/tch-0.24.0/tests/linear/bs.npy create mode 100644 v2/dependency-patches/tch-0.24.0/tests/linear/gen-test-data.py create mode 100644 v2/dependency-patches/tch-0.24.0/tests/linear/in.npy create mode 100644 v2/dependency-patches/tch-0.24.0/tests/linear/out.npy create mode 100644 v2/dependency-patches/tch-0.24.0/tests/linear/ws.npy create mode 100644 v2/dependency-patches/tch-0.24.0/tests/nn_tests.rs create mode 100644 v2/dependency-patches/tch-0.24.0/tests/serialization_tests.rs create mode 100644 v2/dependency-patches/tch-0.24.0/tests/tensor_indexing.rs create mode 100644 v2/dependency-patches/tch-0.24.0/tests/tensor_tests.rs create mode 100644 v2/dependency-patches/tch-0.24.0/tests/test_utils.rs create mode 100644 v2/dependency-patches/tch-0.24.0/tests/var_store.rs create mode 100644 v2/dependency-patches/tch-0.24.0/tests/vision_tests.rs create mode 100644 v2/dependency-patches/tch-0.24.0/uv.lock create mode 100644 v2/dependency-patches/torch-sys-0.24.0/Cargo.toml create mode 100644 v2/dependency-patches/torch-sys-0.24.0/Cargo.toml.orig create mode 100644 v2/dependency-patches/torch-sys-0.24.0/build.rs create mode 100644 v2/dependency-patches/torch-sys-0.24.0/libtch/CMakeLists.txt create mode 100644 v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image.h create mode 100644 v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image_resize.h create mode 100644 v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image_write.h create mode 100644 v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api.cpp create mode 100644 v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api.h create mode 100644 v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api_generated.cpp create mode 100644 v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api_generated.h create mode 100644 v2/dependency-patches/torch-sys-0.24.0/libtch/torch_python.cpp create mode 100644 v2/dependency-patches/torch-sys-0.24.0/libtch/torch_python.h create mode 100644 v2/dependency-patches/torch-sys-0.24.0/src/c_generated.rs create mode 100644 v2/dependency-patches/torch-sys-0.24.0/src/cuda.rs create mode 100644 v2/dependency-patches/torch-sys-0.24.0/src/io.rs create mode 100644 v2/dependency-patches/torch-sys-0.24.0/src/lib.rs create mode 100644 v2/dependency-patches/torch-sys-0.24.0/src/python.rs create mode 100644 v2/dependency-patches/torch-sys-0.24.0/src/traits.rs diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 68375748f2..b6c73e7e31 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -25,6 +25,9 @@ jobs: - name: Checkout code uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + - name: Verify ZIP parent patch integrity + run: python3 scripts/verify_zip_parent_patches.py + - name: Install cargo-audit run: cargo install cargo-audit --locked --version 0.22.2 diff --git a/scripts/verify_zip_parent_patches.py b/scripts/verify_zip_parent_patches.py new file mode 100644 index 0000000000..7c40015a34 --- /dev/null +++ b/scripts/verify_zip_parent_patches.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Verify checked-in ZIP parent patch provenance/integrity; no network or writes.""" +import hashlib +import json +from pathlib import Path + +root = Path(__file__).resolve().parent.parent / "v2" / "dependency-patches" +manifest = json.loads((root / "patches.json").read_text()) +for package in manifest["packages"]: + directory = root / package["package"] + expected = {entry["path"]: entry for entry in package["files"]} + actual = {str(p.relative_to(directory)) for p in directory.rglob("*") if p.is_file()} + assert actual == set(expected), "Patched package inventory changed" + changed = [] + for name, entry in expected.items(): + path = directory / name + assert not path.is_symlink() and directory.resolve() in path.resolve().parents + digest = hashlib.sha256(path.read_bytes()).hexdigest() + assert digest == entry["sha256"], "Patched package digest changed: " + name + if digest != entry["upstreamSha256"]: + changed.append(name) + assert sorted(changed) == sorted(package["changedFiles"]) +print("ZIP parent patch inventory and SHA256: PASS") diff --git a/v2/Cargo.lock b/v2/Cargo.lock index edfd982b29..ff7e381171 100644 --- a/v2/Cargo.lock +++ b/v2/Cargo.lock @@ -29,13 +29,13 @@ dependencies = [ [[package]] name = "aes" -version = "0.8.4" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32" dependencies = [ - "cfg-if", - "cipher", - "cpufeatures 0.2.17", + "cipher 0.5.2", + "cpubits", + "cpufeatures 0.3.0", ] [[package]] @@ -234,9 +234,6 @@ name = "arbitrary" version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" -dependencies = [ - "derive_arbitrary", -] [[package]] name = "arrayref" @@ -428,7 +425,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sha1", + "sha1 0.10.6", "sync_wrapper", "tokio", "tokio-tungstenite", @@ -630,6 +627,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ "hybrid-array", + "zeroize", ] [[package]] @@ -1055,22 +1053,11 @@ dependencies = [ [[package]] name = "bzip2" -version = "0.4.4" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" dependencies = [ - "bzip2-sys", - "libc", -] - -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" -dependencies = [ - "cc", - "pkg-config", + "libbz2-rs-sys", ] [[package]] @@ -1110,8 +1097,6 @@ dependencies = [ [[package]] name = "candle-core" version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f1b20174c1707e20f4cb364a355b449803c03e9b0c9193324623cf9787a4e00" dependencies = [ "byteorder", "gemm 0.17.1", @@ -1125,14 +1110,12 @@ dependencies = [ "safetensors 0.4.5", "thiserror 1.0.69", "yoke 0.7.5", - "zip 0.6.6", + "zip", ] [[package]] name = "candle-core" version = "0.9.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c15b675b80d994b2eadb20a4bbe434eabeb454eac3ee5e2b4cf6f147ee9be091" dependencies = [ "byteorder", "candle-kernels", @@ -1151,7 +1134,7 @@ dependencies = [ "safetensors 0.7.0", "thiserror 2.0.18", "yoke 0.8.1", - "zip 7.2.0", + "zip", ] [[package]] @@ -1322,7 +1305,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ "cfg-if", - "cipher", + "cipher 0.4.4", "cpufeatures 0.2.17", ] @@ -1345,7 +1328,7 @@ checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" dependencies = [ "aead", "chacha20 0.9.1", - "cipher", + "cipher 0.4.4", "poly1305", "zeroize", ] @@ -1398,10 +1381,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common 0.1.7", - "inout", + "inout 0.1.4", "zeroize", ] +[[package]] +name = "cipher" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" +dependencies = [ + "crypto-common 0.2.2", + "inout 0.2.5", +] + [[package]] name = "clap" version = "4.5.60" @@ -1544,7 +1537,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" dependencies = [ "lazy_static", - "windows-sys 0.59.0", + "windows-sys 0.48.0", ] [[package]] @@ -1553,7 +1546,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -1661,15 +1654,18 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.1.5" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" [[package]] name = "constant_time_eq" -version = "0.4.2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +checksum = "9eb6feb175df65d0a3ccd3e73484e940d0ac664e8034f0519fd49a93db3071d9" +dependencies = [ + "rustversion", +] [[package]] name = "convert_case" @@ -1791,36 +1787,36 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4353b283b6c76205abc8eab20b41838e47510c9669ed83c27f7bca8cd1c262f" +checksum = "b3741b9ad77e89c710586b2fcdd619baf28482d92b70d88cecb2a081dd912ae6" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61f6d6eb342f6c48d97278c7549c5b91756f03db80bce9cdeddaa536c356c937" +checksum = "8243e149085051b23faebd352ef7031aeebc5f45e63aaefd63ee7f810c22c7df" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c746ef625773869efaffda6918a2b73f973c73e419e1de20833f3a1a9d614006" +checksum = "c538af3aee13eb6aa8a85d2ef85b4d0aa41623dae8b19f59bbacfd5d6c114920" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-bitset" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5447cd11203bcb5b71bb91e1e8b8470f350870ca837847bf82ec2e25f55fcd7b" +checksum = "2292ea3964d7d751031da1a1261d64feb09e89bd4fbebf3c78f43f9c34c20a11" dependencies = [ "serde", "serde_derive", @@ -1828,9 +1824,9 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ac26ac02abded3bb8223f0cf8a0055c6284be55ca9a6c380f78494bdf686adc" +checksum = "8967a699a4982d02c32246a00f2aa4cee5bccfaaa042608d5e151cb99cba210e" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -1855,9 +1851,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c4a9197c03f7b75c228a66ecaf0b7469e0f9b86fe3cf2cf7ebfbcaacfce0a9b" +checksum = "e258eeeb6903b4e91e314a3667ab2a1e2924738ef2df08bdb50355e38decc749" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -1868,24 +1864,24 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3279a717ec8773811c148650b77cb7c554c6898c858cb547d3da477a8bca21b3" +checksum = "e1d36ff0de30460c01dad152ca2097bcabef2f16c659b9ffbca2f67e9bbdb321" [[package]] name = "cranelift-control" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e2a7ad0219d6f4a4899e5e93d5a4944a79dd6cffe2f6d6fba4d6b2ac165a154" +checksum = "acc880d91877efb6199031bda872726ca3287dc1e18ccf3f61c07bfdab62930b" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "554aaa52367f0396570d031aa0bdedfff4f63794022dc418d0cdcf8f6d321b60" +checksum = "5f7af321c4b6e5be4729e4a5b1ed0fc78fc31695c1ecd3c9b45b91908bf3d0a3" dependencies = [ "cranelift-bitset", "serde", @@ -1894,9 +1890,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72db97f8a7ea1626ecfcac1adf664cba1f1d68448abbb36b7dbaa0a6b78bc11f" +checksum = "3463f1b26faac3491008cede8e9cdf7ceb83139143bd92ba86c14b0882aaf1bb" dependencies = [ "cranelift-codegen", "log", @@ -1906,15 +1902,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17a237c16305f95ff07031be4a5226a4c0753761f40bbad2105c758a33aee692" +checksum = "90b76554b329634c4241c5670a96058affca99ac3b33daa14fb221bd68c0038e" [[package]] name = "cranelift-native" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd6f62b06973b3e5fd813167b01958b09f634bfa59a8f5db2cd49b930d93aa6f" +checksum = "36e7c67bcfd30a8160187a6d52b2ec0cbe45698a9d61f12d1292bd24701f7ed2" dependencies = [ "cranelift-codegen", "libc", @@ -1923,9 +1919,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd76efe2edc25d9f062aaf0a16c7528a30c8ba5782afe6296237ed48d8d6127c" +checksum = "1ccca83c45ddfe0412b1b2e4fe509d28d1bdbc014e306574a848519741bebc36" [[package]] name = "crc" @@ -2743,6 +2739,12 @@ dependencies = [ "uuid", ] +[[package]] +name = "deflate64" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2" + [[package]] name = "der" version = "0.7.10" @@ -2775,17 +2777,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "derive_arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "derive_more" version = "0.99.20" @@ -2849,6 +2840,8 @@ dependencies = [ "block-buffer 0.12.1", "const-oid 0.10.2", "crypto-common 0.2.2", + "ctutils", + "zeroize", ] [[package]] @@ -3333,7 +3326,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3364,13 +3357,13 @@ checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" [[package]] name = "fastbloom" -version = "0.14.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7f34442dbe69c60fe8eaf58a8cafff81a1f278816d8ab4db255b3bef4ac3c4" +checksum = "ef975e30683b2d965054bb0a836f8973857c4ebf6acf274fe46617cd285060d8" dependencies = [ - "getrandom 0.3.4", + "foldhash 0.2.0", "libm", - "rand 0.9.2", + "portable-atomic", "siphasher 1.0.2", ] @@ -4304,11 +4297,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi", "wasip2", - "wasm-bindgen", ] [[package]] @@ -4318,11 +4309,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "rand_core 0.10.1", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -4899,7 +4892,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac", + "hmac 0.12.1", ] [[package]] @@ -4911,6 +4904,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "hmac" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "hmac-sha256" version = "1.1.14" @@ -5231,7 +5233,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.7", + "webpki-roots 1.0.10", ] [[package]] @@ -5506,6 +5508,15 @@ dependencies = [ "generic-array 0.14.7", ] +[[package]] +name = "inout" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b5cf4372d022092674a83b8febf5865f7a573cc0c5baec5e49c879b6a20978f" +dependencies = [ + "hybrid-array", +] + [[package]] name = "io-kit-sys" version = "0.4.1" @@ -5549,7 +5560,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5677,19 +5688,68 @@ dependencies = [ "cesu8", "cfg-if", "combine", - "jni-sys", + "jni-sys 0.3.0", "log", "thiserror 1.0.69", "walkdir", "windows-sys 0.45.0", ] +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys 0.4.1", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link 0.2.1", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + [[package]] name = "jni-sys" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] + [[package]] name = "jobserver" version = "0.1.34" @@ -5857,11 +5917,17 @@ dependencies = [ "once_cell", ] +[[package]] +name = "libbz2-rs-sys" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" + [[package]] name = "libc" -version = "0.2.182" +version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" [[package]] name = "libloading" @@ -5996,6 +6062,15 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1670343e58806300d87950e3401e820b519b9384281bbabfb15e3636689ffd69" +[[package]] +name = "lzma-rust2" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7277c9742229dc25243236d578b6471a798c554512c6f6a04365ec2e87c2a3bb" +dependencies = [ + "sha2 0.11.0", +] + [[package]] name = "mac" version = "0.1.1" @@ -6178,7 +6253,7 @@ dependencies = [ "js-sys", "quinn", "rcgen", - "rustls-platform-verifier", + "rustls-platform-verifier 0.6.2", "serde", "thiserror 2.0.18", "tokio", @@ -6599,7 +6674,6 @@ dependencies = [ "num-complex", "num-traits", "py_literal", - "zip 6.0.0", ] [[package]] @@ -6609,7 +6683,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3f42e7bbe13d351b6bead8286a43aac9534b82bd3cc43e47037f012ebfd62d4" dependencies = [ "bitflags 2.11.0", - "jni-sys", + "jni-sys 0.3.0", "log", "ndk-sys", "num_enum", @@ -6629,7 +6703,7 @@ version = "0.6.0+11769913" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee6cda3051665f1fb8d9e08fc35c96d5a244fb1be711a03b71118828afc9a873" dependencies = [ - "jni-sys", + "jni-sys 0.3.0", ] [[package]] @@ -6673,6 +6747,18 @@ dependencies = [ "libc", ] +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.11.0", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nodrop" version = "0.1.14" @@ -7261,7 +7347,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06503bb33f294c5f1ba484011e053bfa6ae227074bdb841e9863492dc5960d4b" dependencies = [ "hmac-sha256", - "lzma-rust2", + "lzma-rust2 0.15.7", "ureq 3.2.0", ] @@ -7272,7 +7358,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.45.0", ] [[package]] @@ -7383,17 +7469,6 @@ dependencies = [ "thiserror 2.0.18", ] -[[package]] -name = "password-hash" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" -dependencies = [ - "base64ct", - "rand_core 0.6.4", - "subtle", -] - [[package]] name = "paste" version = "1.0.15" @@ -7408,14 +7483,12 @@ checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" [[package]] name = "pbkdf2" -version = "0.11.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" dependencies = [ - "digest 0.10.7", - "hmac", - "password-hash", - "sha2 0.10.9", + "digest 0.11.3", + "hmac 0.13.0", ] [[package]] @@ -7828,6 +7901,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "ppmd-rust" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "494b4379484fc92217cc7d9e3a532375b1d01a61889ec8333025dc3b0377af56" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -8051,7 +8130,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.117", @@ -8088,9 +8167,9 @@ dependencies = [ [[package]] name = "pulley-interpreter" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09f3fdea932cc327d051b2aaccfc82fddcd49997e8579f3d91ae3468a954b94" +checksum = "c956c7140270b72e3bf03381a51f2eb79c70fd6881fc17cd027fa7cd3f56ae5a" dependencies = [ "cranelift-bitset", "log", @@ -8100,9 +8179,9 @@ dependencies = [ [[package]] name = "pulley-macros" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96da696c8995a3033eb1287ee418895592da41a5d549b8615b5f1e5851318b48" +checksum = "74db420d7ce2bb03eed5b81c1f778bda5771b3fd818e2b496dfc3a2783ceb60e" dependencies = [ "proc-macro2", "quote", @@ -8208,20 +8287,21 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.15" +version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" +checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc" dependencies = [ "bytes", "fastbloom", - "getrandom 0.3.4", + "getrandom 0.4.1", "lru-slab", - "rand 0.9.2", + "rand 0.10.2", + "rand_pcg 0.10.2", "ring", "rustc-hash 2.1.1", "rustls", "rustls-pki-types", - "rustls-platform-verifier", + "rustls-platform-verifier 0.7.1", "slab", "thiserror 2.0.18", "tinyvec", @@ -8240,7 +8320,7 @@ dependencies = [ "once_cell", "socket2 0.6.2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -8278,7 +8358,7 @@ dependencies = [ "rand_chacha 0.2.2", "rand_core 0.5.1", "rand_hc", - "rand_pcg", + "rand_pcg 0.2.1", ] [[package]] @@ -8424,6 +8504,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "rand_xorshift" version = "0.4.0" @@ -8717,7 +8806,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 1.0.7", + "webpki-roots 1.0.10", ] [[package]] @@ -8760,7 +8849,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "hmac", + "hmac 0.12.1", "subtle", ] @@ -9104,14 +9193,14 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "log", "once_cell", @@ -9152,17 +9241,38 @@ checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" dependencies = [ "core-foundation 0.10.1", "core-foundation-sys", - "jni", + "jni 0.21.1", "log", "once_cell", "rustls", "rustls-native-certs", - "rustls-platform-verifier-android", + "rustls-platform-verifier-android 0.1.1", "rustls-webpki", "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni 0.22.4", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android 0.2.0", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.59.0", ] [[package]] @@ -9171,11 +9281,17 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" +[[package]] +name = "rustls-platform-verifier-android" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" + [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "ring", "rustls-pki-types", @@ -9600,7 +9716,7 @@ dependencies = [ "candle-core 0.9.2", "candle-nn 0.9.2", "criterion", - "hmac", + "hmac 0.12.1", "nalgebra 0.33.2", "ort", "rand 0.8.5", @@ -10126,6 +10242,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + [[package]] name = "sha2" version = "0.10.9" @@ -10247,6 +10374,16 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -10768,7 +10905,7 @@ dependencies = [ "gdkwayland-sys", "gdkx11-sys", "gtk", - "jni", + "jni 0.21.1", "libc", "log", "ndk", @@ -10840,7 +10977,7 @@ dependencies = [ "gtk", "heck 0.5.0", "http", - "jni", + "jni 0.21.1", "libc", "log", "mime", @@ -11027,7 +11164,7 @@ dependencies = [ "dpi", "gtk", "http", - "jni", + "jni 0.21.1", "objc2", "objc2-ui-kit", "objc2-web-kit", @@ -11050,7 +11187,7 @@ checksum = "e11ea2e6f801d275fdd890d6c9603736012742a1c33b96d0db788c9cdebf7f9e" dependencies = [ "gtk", "http", - "jni", + "jni 0.21.1", "log", "objc2", "objc2-app-kit", @@ -11120,8 +11257,6 @@ dependencies = [ [[package]] name = "tch" version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d3f84a069d8ba16dbf720b61e8bf131d90ffb8e958a664eae8e4993c5c2fa6f" dependencies = [ "half", "lazy_static", @@ -11131,7 +11266,7 @@ dependencies = [ "safetensors 0.3.3", "thiserror 1.0.69", "torch-sys", - "zip 0.6.6", + "zip", ] [[package]] @@ -11144,7 +11279,7 @@ dependencies = [ "getrandom 0.4.1", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -11251,6 +11386,7 @@ checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", "itoa", + "js-sys", "num-conv", "powerfmt", "serde_core", @@ -11618,13 +11754,11 @@ dependencies = [ [[package]] name = "torch-sys" version = "0.24.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4ba78777379cf09aaa79708c63e477cf0f95e021d04360c6821f1a9f56173f7" dependencies = [ "anyhow", "cc", "libc", - "zip 0.6.6", + "zip", ] [[package]] @@ -11847,7 +11981,7 @@ dependencies = [ "httparse", "log", "rand 0.8.5", - "sha1", + "sha1 0.10.6", "thiserror 1.0.69", "utf-8", ] @@ -11866,7 +12000,7 @@ dependencies = [ "rand 0.9.2", "rustls", "rustls-pki-types", - "sha1", + "sha1 0.10.6", "thiserror 2.0.18", ] @@ -12510,9 +12644,9 @@ dependencies = [ [[package]] name = "wasmtime" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc1374519a6d4c3eb99e0598f2ce069e6feb63de65bed7d0240a5f84b58c3b" +checksum = "60e5f244a2c9d40afa9e9a607ee7f048c0648bb1c22cc28b130c5fc71e23d71d" dependencies = [ "addr2line", "anyhow", @@ -12565,9 +12699,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7e05a9ebefb914d4df7928652e27aa1e89e1ca23c5ab73c647316783f3bcbc2" +checksum = "3e246cd5107aa406cfa4acf6942e864bd189ae8136bab853365147da131c9ca3" dependencies = [ "anyhow", "cpp_demangle", @@ -12592,18 +12726,18 @@ dependencies = [ [[package]] name = "wasmtime-internal-asm-macros" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fcf4b431fb653aefd2cc3c3a65ea78988acf5a7626cb825c54b5aab5aa1a8e" +checksum = "5380a8106fea8bfedb7208bb11ced9e7488ee44394ca889ede73acd3bdfd9d0b" dependencies = [ "cfg-if", ] [[package]] name = "wasmtime-internal-cache" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65b514e39cd914f34a135ce7696373287ccb9611b083cbe244a48c4903380fb5" +checksum = "4108e1964b3a02b4b5604ea1a2918a0cbcab512e5de0ccf4c062fc6c06b2eb8e" dependencies = [ "anyhow", "base64 0.22.1", @@ -12621,9 +12755,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-macro" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92fe9cf6ca766adc68b5ed4474155e88cc97e79d13c63450fe91779de9c41a3" +checksum = "8278aa2bb237788ac9e3a70c5bea3ea3de6b4f95fb003df303858941871b8c4e" dependencies = [ "anyhow", "proc-macro2", @@ -12636,15 +12770,15 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-util" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36b8553647a6ae550ec3bca45759565204af763c37b1b51d5e45d6c5a26a1e32" +checksum = "838c652b6cafd82a432a3a2c7af5bea4c6957d218aafbe50018e0297486ab7cc" [[package]] name = "wasmtime-internal-cranelift" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bd306f478b4a68d94a80e64d1a92871b697188eafe8bfc387f865085c818a04" +checksum = "7339fdd64a3fde574505adea23244e065d5b1de0e72c8dd87e150f5d4ab63a63" dependencies = [ "anyhow", "cfg-if", @@ -12669,9 +12803,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-fiber" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f9bb63081293450d0dfb439f246339e91e34d51c09b1310bdd297b67be5933" +checksum = "6f3afc72d3c72f71d4ebc72e83f479ba8ca24409c41d43e5e0228f0bf82b873e" dependencies = [ "anyhow", "cc", @@ -12685,9 +12819,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-debug" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c04edbaf79fb10460ee0867a829f831b7afa8d5d8c9d32287d88b3f1aaef79" +checksum = "21d33adf2d1eae36df9b6b033a1d549c7553ea607f4f9a81f75d51627725c354" dependencies = [ "cc", "object", @@ -12697,9 +12831,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e2b3b397afafb7e6b3603dd3f867cc626d3bde726bd65ce01443a70fe66e1c0" +checksum = "5bde2fcf14cc76084242c6fe792e49778b5a9008c7df28ca4fe21c35c69d5f55" dependencies = [ "anyhow", "cfg-if", @@ -12709,24 +12843,24 @@ dependencies = [ [[package]] name = "wasmtime-internal-math" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2509a3a45fbdc79eab62680df8f8d5ffe5420c3bf7f0c2083d49110c09c61b34" +checksum = "4bd982249e933e730859f84c17254c7e95f38c3b0cdf81695bfc6a270e237852" dependencies = [ "libm", ] [[package]] name = "wasmtime-internal-slab" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0985eb32142c197f1aa5d50e94498a769f55095e8ed2838eff386c735f9da72f" +checksum = "8280b359384cc412bddf0023193eeaffadcb0d7fbbf860f2b7e0580e8806a344" [[package]] name = "wasmtime-internal-unwinder" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b88fa7f08080eeb4926d6062c2ae8c38c19a2b1a6d0f01c569b827d987f4909" +checksum = "bb43477c41769fec0d031d32cb3236195acfb1e6aa4efdf463e5fd001c23f6cc" dependencies = [ "anyhow", "cfg-if", @@ -12737,9 +12871,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-versioned-export-macros" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfff26e96f2539205ff7c049e4dff8343a479722bbbae88bc4da54adde12c1fa" +checksum = "b33afac17819f2f767ea280b4b87a6c95458ef1ae43da410cfc5b26813ecb78f" dependencies = [ "proc-macro2", "quote", @@ -12748,9 +12882,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-winch" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "176d188460e507807f9f8b7521d7c1dc62b356fa221b4e2ce566e88652294d9d" +checksum = "1757c8b0626385c93994c7a3c7a23ee5b469061465b601c1fdce71ff61b11c82" dependencies = [ "anyhow", "cranelift-codegen", @@ -12765,9 +12899,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-wit-bindgen" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3c06e67956fabd2fa3b1aa94200779cc0df8100dc45c6a9b80fda67f565697" +checksum = "061dd5bae2e618b32a8ed2acda5953d07dd308ac0e3f7960afc62ccdf6df7c44" dependencies = [ "anyhow", "bitflags 2.11.0", @@ -12876,9 +13010,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.6" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca" +checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb" dependencies = [ "rustls-pki-types", ] @@ -12889,14 +13023,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.7", + "webpki-roots 1.0.10", ] [[package]] name = "webpki-roots" -version = "1.0.7" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" dependencies = [ "rustls-pki-types", ] @@ -13230,7 +13364,7 @@ dependencies = [ "flume 0.11.1", "futures", "hex", - "hmac", + "hmac 0.12.1", "libc", "mdns-sd", "regex", @@ -13287,7 +13421,7 @@ dependencies = [ "chrono", "clap", "criterion", - "hmac", + "hmac 0.12.1", "midstreamer-quic", "midstreamer-scheduler", "serde", @@ -13480,7 +13614,7 @@ dependencies = [ "clap", "criterion", "futures-util", - "hmac", + "hmac 0.12.1", "hyper", "hyper-util", "jsonwebtoken", @@ -13669,7 +13803,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -13680,9 +13814,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winch-codegen" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde193a101057fce046a3fff773932419b7861f6bb95632d01bca90843c9ce51" +checksum = "69940a29fc6b01b0e6b141912b700c17eaf956d58d63bc0c6e733435caf020e4" dependencies = [ "anyhow", "cranelift-assembler-x64", @@ -14455,7 +14589,7 @@ dependencies = [ "html5ever", "http", "javascriptcore-rs", - "jni", + "jni 0.21.1", "kuchikiki", "libc", "ndk", @@ -14689,55 +14823,37 @@ dependencies = [ [[package]] name = "zip" -version = "0.6.6" +version = "9.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e" dependencies = [ "aes", - "byteorder", "bzip2", - "constant_time_eq 0.1.5", + "constant_time_eq 0.6.1", "crc32fast", - "crossbeam-utils", + "deflate64", "flate2", - "hmac", - "pbkdf2", - "sha1", - "time", - "zstd 0.11.2+zstd.1.5.2", -] - -[[package]] -name = "zip" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" -dependencies = [ - "arbitrary", - "crc32fast", - "flate2", - "indexmap 2.13.0", - "memchr", - "zopfli", -] - -[[package]] -name = "zip" -version = "7.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0" -dependencies = [ - "crc32fast", + "getrandom 0.4.1", + "hmac 0.13.0", "indexmap 2.13.0", + "lzma-rust2 0.22.0", "memchr", + "nix 0.31.3", + "pbkdf2", + "ppmd-rust", + "sha1 0.11.0", + "time", "typed-path", + "zeroize", + "zopfli", + "zstd 0.14.3", ] [[package]] name = "zlib-rs" -version = "0.6.3" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" +checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" [[package]] name = "zmij" @@ -14759,46 +14875,45 @@ dependencies = [ [[package]] name = "zstd" -version = "0.11.2+zstd.1.5.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ - "zstd-safe 5.0.2+zstd.1.5.2", + "zstd-safe 7.2.4", ] [[package]] name = "zstd" -version = "0.13.3" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" dependencies = [ - "zstd-safe 7.2.4", + "zstd-safe 8.2.0", ] [[package]] name = "zstd-safe" -version = "5.0.2+zstd.1.5.2" +version = "7.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" dependencies = [ - "libc", "zstd-sys", ] [[package]] name = "zstd-safe" -version = "7.2.4" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "2.1.2+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" dependencies = [ "cc", "pkg-config", diff --git a/v2/Cargo.toml b/v2/Cargo.toml index 87cd12a0b6..19d505998c 100644 --- a/v2/Cargo.toml +++ b/v2/Cargo.toml @@ -235,7 +235,8 @@ pcap = "1.1" petgraph = "0.6" # Data loading -ndarray-npy = "0.10" +# Training uses NPY only; keep complex arrays without the unused NPZ/ZIP backend. +ndarray-npy = { version = "0.10", default-features = false, features = ["num-complex-0_4"] } walkdir = "2.4" # Hashing (for proof) @@ -318,3 +319,11 @@ strip = false [profile.bench] inherits = "release" debug = true + +# Narrow ZIP9 compatibility patches retain existing model/framework versions. +# Provenance and review scope: dependency-patches/README.md. +[patch.crates-io] +candle-core-04 = { package = "candle-core", path = "dependency-patches/candle-core-0.4.1" } +candle-core-09 = { package = "candle-core", path = "dependency-patches/candle-core-0.9.2" } +tch = { path = "dependency-patches/tch-0.24.0" } +torch-sys = { path = "dependency-patches/torch-sys-0.24.0" } diff --git a/v2/crates/cog-person-count/tests/zip9_tensor_roundtrip.rs b/v2/crates/cog-person-count/tests/zip9_tensor_roundtrip.rs new file mode 100644 index 0000000000..e9386bad6a --- /dev/null +++ b/v2/crates/cog-person-count/tests/zip9_tensor_roundtrip.rs @@ -0,0 +1,14 @@ +// ZIP9 compatibility must preserve actual tensor names and values, not just compile. +use candle_core::{Device, Tensor}; +#[test] +fn npz_roundtrip_retains_unicode_and_nested_tensor_names() -> candle_core::Result<()> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("tensors.npz"); + let tensor = Tensor::new(&[1f32, -2.5, 3.25], &Device::Cpu)?; + Tensor::write_npz(&[("nested/room-α", &tensor)], &path)?; + let restored = Tensor::read_npz(&path)?; + assert_eq!(restored.len(), 1); + assert_eq!(restored[0].0, "nested/room-α"); + assert_eq!(restored[0].1.to_vec1::()?, vec![1., -2.5, 3.25]); + Ok(()) +} diff --git a/v2/crates/wifi-densepose-nn/tests/zip9_tensor_roundtrip.rs b/v2/crates/wifi-densepose-nn/tests/zip9_tensor_roundtrip.rs new file mode 100644 index 0000000000..90b6b243cd --- /dev/null +++ b/v2/crates/wifi-densepose-nn/tests/zip9_tensor_roundtrip.rs @@ -0,0 +1,15 @@ +#![cfg(feature = "candle-backend")] +// ZIP9 compatibility must preserve actual tensor names and values, not just compile. +use candle_core::{Device, Tensor}; +#[test] +fn npz_roundtrip_retains_unicode_and_nested_tensor_names() -> candle_core::Result<()> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("tensors.npz"); + let tensor = Tensor::new(&[1f32, -2.5, 3.25], &Device::Cpu)?; + Tensor::write_npz(&[("nested/room-α", &tensor)], &path)?; + let restored = Tensor::read_npz(&path)?; + assert_eq!(restored.len(), 1); + assert_eq!(restored[0].0, "nested/room-α"); + assert_eq!(restored[0].1.to_vec1::()?, vec![1., -2.5, 3.25]); + Ok(()) +} diff --git a/v2/dependency-patches/README.md b/v2/dependency-patches/README.md new file mode 100644 index 0000000000..639a4aad30 --- /dev/null +++ b/v2/dependency-patches/README.md @@ -0,0 +1,21 @@ +# ZIP9 compatibility patches + +These are narrow source patches to the existing crates.io Candle0.4.1, Candle0.9.2, tch0.24.0 and torch-sys0.24.0 releases. The model/framework versions, numerical operators and public APIs are retained. Root Cargo patches redirect those exact package versions to these sources; every ZIP edge resolves to patched ZIP9.0.5 in the root lockfile. + +`patches.json` records downloaded `.crate` SHA256, each retained upstream-file hash, patched-file hash and the changed-file list. `compatibility.patch` contains the complete review delta. Sources came from `https://static.crates.io/crates//-.crate`. MIT/Apache licences remain in each directory. Cargo.lock and cargo VCS metadata were omitted; tch's 87MB `third_party` model/PyTorch test snapshot was omitted because it is not a Rust library or build input. Its asset-dependent upstream tests are not represented as passed. These snapshots are not independently published packages. + +## Changes + +- Four normalized Cargo manifests request ZIP9; old manifests are retained as upstream provenance only. +- NPZ name decoding propagates ZIP9's checked UTF8 result; names are borrowed from Cow without truncation. NPZ writers use SimpleFileOptions. +- PTH file-name iteration collects checked results before reading entries. +- The optional libtorch download build script uses ZIP9's checked extraction instead of joining mangled names manually. This is a build-time path, not a default server runtime. +- No advisory ignore, audit gate change, framework-major migration or tensor algorithm change. + +## Verification + +Run `python3 scripts/verify_zip_parent_patches.py`, then root `cargo audit --file v2/Cargo.lock`. Focused tests include the actual current Candle consumer and NPZ Unicode/nested-name/value roundtrip, sensing-server companion/TLS, NPY loader and Wasmtime sandbox tests. `cargo check -p wifi-densepose-train --features tch-backend,tch/doc-only` checks tch and torch-sys Rust/build-script compatibility without downloading or claiming libtorch execution. + +The legacy optional Candle0.4 backend cannot currently compile against the main workspace's half2.7/rand0.9 combination: its unchanged random operators require rand0.8. Pinning half2.3 in the full workspace conflicts with burn's half2.7 requirement, so no such pin is applied. `cargo test --manifest-path v2/dependency-patches/compat-tests/Cargo.toml` exercises the patched Candle0.4 NPZ API in an isolated compatible half2.3 environment. Its own lockfile is retained. This does not claim that the legacy optional backend is qualified in the main workspace. Full PyTorch/CUDA/model numerical qualification remains unexecuted; the default Mac server does not include those optional backends. + +Remove each snapshot when an upstream compatible package release supplies ZIP>=9. Source changes require the same focused format, build and integrity review. The separate Apple helper binary must be rebuilt from the clean source commit and verified against its embedded manifest before distribution. diff --git a/v2/dependency-patches/candle-core-0.4.1/Cargo.toml b/v2/dependency-patches/candle-core-0.4.1/Cargo.toml new file mode 100644 index 0000000000..3b359c646c --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/Cargo.toml @@ -0,0 +1,143 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +name = "candle-core" +version = "0.4.1" +description = "Minimalist ML framework." +readme = "README.md" +keywords = [ + "blas", + "tensor", + "machine-learning", +] +categories = ["science"] +license = "MIT OR Apache-2.0" +repository = "https://github.com/huggingface/candle" + +[[bench]] +name = "bench_main" +harness = false + +[dependencies.accelerate-src] +version = "0.3.2" +optional = true + +[dependencies.byteorder] +version = "1.4.3" + +[dependencies.candle-kernels] +version = "0.4.1" +optional = true + +[dependencies.candle-metal-kernels] +version = "0.4.1" +optional = true + +[dependencies.cudarc] +version = "0.10.0" +features = ["f16"] +optional = true + +[dependencies.gemm] +version = "0.17.0" +features = ["wasm-simd128-enable"] + +[dependencies.half] +version = "2.3.1" +features = [ + "num-traits", + "use-intrinsics", + "rand_distr", +] + +[dependencies.intel-mkl-src] +version = "0.8.1" +features = ["mkl-static-lp64-iomp"] +optional = true + +[dependencies.libc] +version = "0.2.147" +optional = true + +[dependencies.memmap2] +version = "0.9.3" +features = ["stable_deref_trait"] + +[dependencies.metal] +version = "0.27.0" +features = ["mps"] +optional = true + +[dependencies.num-traits] +version = "0.2.15" + +[dependencies.num_cpus] +version = "1.15.0" + +[dependencies.rand] +version = "0.8.5" + +[dependencies.rand_distr] +version = "0.4.3" + +[dependencies.rayon] +version = "1.7.0" + +[dependencies.safetensors] +version = "0.4.1" + +[dependencies.thiserror] +version = "1" + +[dependencies.yoke] +version = "0.7.2" +features = ["derive"] + +[dependencies.zip] +version = "9.0.1" +default-features = false + +[dev-dependencies.anyhow] +version = "1" +features = ["backtrace"] + +[dev-dependencies.clap] +version = "4.2.4" +features = ["derive"] + +[dev-dependencies.criterion] +version = "0.5.1" +default-features = false + +[features] +accelerate = [ + "dep:libc", + "dep:accelerate-src", +] +cuda = [ + "cudarc", + "dep:candle-kernels", +] +cudnn = [ + "cuda", + "cudarc/cudnn", +] +default = [] +metal = [ + "dep:metal", + "dep:candle-metal-kernels", +] +mkl = [ + "dep:libc", + "dep:intel-mkl-src", +] diff --git a/v2/dependency-patches/candle-core-0.4.1/Cargo.toml.orig b/v2/dependency-patches/candle-core-0.4.1/Cargo.toml.orig new file mode 100644 index 0000000000..92a04917db --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/Cargo.toml.orig @@ -0,0 +1,50 @@ +[package] +name = "candle-core" +version.workspace = true +edition.workspace = true +description.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true +license.workspace = true +readme = "README.md" + +[dependencies] +accelerate-src = { workspace = true, optional = true } +byteorder = { workspace = true } +candle-kernels = { workspace = true, optional = true } +candle-metal-kernels = { workspace = true, optional = true } +metal = { workspace = true, optional = true} +cudarc = { workspace = true, optional = true } +gemm = { workspace = true } +half = { workspace = true } +intel-mkl-src = { workspace = true, optional = true } +libc = { workspace = true, optional = true } +memmap2 = { workspace = true } +num-traits = { workspace = true } +num_cpus = { workspace = true } +rand = { workspace = true } +rand_distr = { workspace = true } +rayon = { workspace = true } +safetensors = { workspace = true } +thiserror = { workspace = true } +yoke = { workspace = true } +zip = { workspace = true } + +[dev-dependencies] +anyhow = { workspace = true } +clap = { workspace = true } +criterion = { workspace = true } + + +[features] +default = [] +cuda = ["cudarc", "dep:candle-kernels"] +cudnn = ["cuda", "cudarc/cudnn"] +mkl = ["dep:libc", "dep:intel-mkl-src"] +accelerate = ["dep:libc", "dep:accelerate-src"] +metal = ["dep:metal", "dep:candle-metal-kernels"] + +[[bench]] +name = "bench_main" +harness = false diff --git a/v2/dependency-patches/candle-core-0.4.1/LICENSE b/v2/dependency-patches/candle-core-0.4.1/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/v2/dependency-patches/candle-core-0.4.1/README.md b/v2/dependency-patches/candle-core-0.4.1/README.md new file mode 100644 index 0000000000..246f5bae61 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/README.md @@ -0,0 +1,2 @@ +# candle +Minimalist ML framework for Rust diff --git a/v2/dependency-patches/candle-core-0.4.1/benches/bench_main.rs b/v2/dependency-patches/candle-core-0.4.1/benches/bench_main.rs new file mode 100644 index 0000000000..162e3f2be1 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/benches/bench_main.rs @@ -0,0 +1,9 @@ +mod benchmarks; + +use criterion::criterion_main; +criterion_main!( + benchmarks::affine::benches, + benchmarks::matmul::benches, + benchmarks::random::benches, + benchmarks::where_cond::benches +); diff --git a/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/affine.rs b/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/affine.rs new file mode 100644 index 0000000000..eded9f5787 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/affine.rs @@ -0,0 +1,43 @@ +use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; +use candle_core::{DType, Device, Tensor}; +use criterion::{black_box, criterion_group, Criterion, Throughput}; +use std::time::Instant; + +fn run(a: &Tensor) { + a.affine(12.34, 56.78).unwrap(); +} + +fn run_affine_benchmark(c: &mut Criterion, device: &Device, dtype: DType, name: &str) { + let b = 1; + let m = 1024; + let k = 1024; + + let tensor = Tensor::zeros((b, m, k), dtype, &device).unwrap(); + + let flops = b * m * k * dtype.size_in_bytes(); + + let mut group = c.benchmark_group(device.bench_name(name)); + group.throughput(Throughput::Bytes(flops as u64)); + group.bench_function("iter", move |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + for _i in 0..iters { + run(black_box(&tensor)); + } + device.sync().unwrap(); + start.elapsed() + }) + }); + group.finish(); +} + +fn criterion_benchmark(c: &mut Criterion) { + let handler = BenchDeviceHandler::new().unwrap(); + for device in handler.devices { + run_affine_benchmark(c, &device, DType::F32, "affine_f32"); + run_affine_benchmark(c, &device, DType::F16, "affine_f16"); + run_affine_benchmark(c, &device, DType::BF16, "affine_bf16"); + } +} + +criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/matmul.rs b/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/matmul.rs new file mode 100644 index 0000000000..9d67e642cd --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/matmul.rs @@ -0,0 +1,44 @@ +use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; +use candle_core::{DType, Device, Tensor}; +use criterion::{black_box, criterion_group, Criterion, Throughput}; +use std::time::Instant; + +fn run(a: &Tensor, b: &Tensor) { + a.matmul(&b.t().unwrap()).unwrap(); +} + +fn run_bench(c: &mut Criterion, device: &Device) { + let b = 1; + let m = 1; + let n = 2048; + let k = 2048; + + let dtype = DType::F32; + let lhs = Tensor::zeros((b, m, k), dtype, device).unwrap(); + let rhs = Tensor::zeros((b, n, k), dtype, device).unwrap(); + + let flops = b * m * n * k; + + let mut group = c.benchmark_group(device.bench_name("matmul")); + group.throughput(Throughput::Bytes(flops as u64)); + group.bench_function("iter", move |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + for _i in 0..iters { + run(black_box(&lhs), black_box(&rhs)); + } + device.sync().unwrap(); + start.elapsed() + }) + }); + group.finish(); +} + +fn criterion_benchmark(c: &mut Criterion) { + let handler = BenchDeviceHandler::new().unwrap(); + for device in handler.devices { + run_bench(c, &device); + } +} + +criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/mod.rs b/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/mod.rs new file mode 100644 index 0000000000..c45effee20 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/mod.rs @@ -0,0 +1,66 @@ +pub(crate) mod affine; +pub(crate) mod matmul; +pub(crate) mod random; +pub(crate) mod where_cond; + +use candle_core::{Device, Result}; + +pub(crate) trait BenchDevice { + fn sync(&self) -> Result<()>; + + fn bench_name>(&self, name: S) -> String; +} + +impl BenchDevice for Device { + fn sync(&self) -> Result<()> { + match self { + Device::Cpu => Ok(()), + Device::Cuda(device) => { + #[cfg(feature = "cuda")] + return Ok(device.synchronize()?); + #[cfg(not(feature = "cuda"))] + panic!("Cuda device without cuda feature enabled: {:?}", device) + } + Device::Metal(device) => { + #[cfg(feature = "metal")] + return Ok(device.wait_until_completed()?); + #[cfg(not(feature = "metal"))] + panic!("Metal device without metal feature enabled: {:?}", device) + } + } + } + + fn bench_name>(&self, name: S) -> String { + match self { + Device::Cpu => { + let cpu_type = if cfg!(feature = "accelerate") { + "accelerate" + } else if cfg!(feature = "mkl") { + "mkl" + } else { + "cpu" + }; + format!("{}_{}", cpu_type, name.into()) + } + Device::Cuda(_) => format!("cuda_{}", name.into()), + Device::Metal(_) => format!("metal_{}", name.into()), + } + } +} + +struct BenchDeviceHandler { + devices: Vec, +} + +impl BenchDeviceHandler { + pub fn new() -> Result { + let mut devices = Vec::new(); + if cfg!(feature = "metal") { + devices.push(Device::new_metal(0)?); + } else if cfg!(feature = "cuda") { + devices.push(Device::new_cuda(0)?); + } + devices.push(Device::Cpu); + Ok(Self { devices }) + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/random.rs b/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/random.rs new file mode 100644 index 0000000000..22c60ef18c --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/random.rs @@ -0,0 +1,63 @@ +use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; +use candle_core::{DType, Device, Tensor}; +use criterion::{black_box, criterion_group, Criterion, Throughput}; +use std::time::Instant; + +fn rand_uniform(a: &Tensor) { + a.rand_like(-1.0, 123.0).unwrap(); +} + +fn rand_normal(a: &Tensor) { + a.randn_like(100.0, 15.0).unwrap(); +} + +fn run_random_bench(c: &mut Criterion, device: &Device) { + let b = 1; + + let rows = 2048; + let cols = 2048; + + let dtype = DType::F32; + let tensor = Tensor::zeros((b, rows, cols), dtype, device).unwrap(); + + let flops = b * rows * cols * dtype.size_in_bytes(); + + let mut group = c.benchmark_group(device.bench_name("random_uniform")); + group.throughput(Throughput::Bytes(flops as u64)); + group.bench_function("iter", move |benches| { + benches.iter_custom(|iters| { + let start = Instant::now(); + for _i in 0..iters { + rand_uniform(black_box(&tensor)); + } + device.sync().unwrap(); + start.elapsed() + }) + }); + group.finish(); + + let tensor = Tensor::zeros((b, rows, cols), dtype, device).unwrap(); + + let mut group = c.benchmark_group(device.bench_name("random_normal")); + group.throughput(Throughput::Bytes(flops as u64)); + group.bench_function("iter", move |benches| { + benches.iter_custom(|iters| { + let start = Instant::now(); + for _i in 0..iters { + rand_normal(black_box(&tensor)); + } + device.sync().unwrap(); + start.elapsed() + }) + }); + group.finish(); +} + +fn criterion_benchmark(c: &mut Criterion) { + let handler = BenchDeviceHandler::new().unwrap(); + for device in handler.devices { + run_random_bench(c, &device); + } +} + +criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/where_cond.rs b/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/where_cond.rs new file mode 100644 index 0000000000..c517dcf590 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/where_cond.rs @@ -0,0 +1,64 @@ +use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; +use candle_core::{DType, Device, Tensor}; +use criterion::{black_box, criterion_group, Criterion, Throughput}; +use std::time::Instant; + +fn run(a: &Tensor, b: &Tensor, c: &Tensor) { + a.where_cond(b, c).unwrap(); +} + +const fn create_cond_arr() -> [u8; N] { + let mut arr = [0u8; N]; + let mut i = 0; + while i < N { + arr[i] = (i % 2) as u8; + i += 1; + } + arr +} + +const B: usize = 1; +const M: usize = 1024; +const K: usize = 1024; +const SIZE: usize = B * M * K; + +const DATA: [u8; SIZE] = create_cond_arr::(); + +fn run_where_cond_benchmark(c: &mut Criterion, device: &Device, dtype: DType, name: &str) { + let tensor = Tensor::from_slice(DATA.as_slice(), (B, M, K), &device).unwrap(); + let on_true = Tensor::ones((B, M, K), dtype, &device).unwrap(); + let on_false = Tensor::zeros((B, M, K), dtype, &device).unwrap(); + + let elements = B * M * K; + // E.g. 2 f32 tensors + 1 u8 tensor + let flops = (2 * elements * dtype.size_in_bytes()) + elements; + + let mut group = c.benchmark_group(device.bench_name(name)); + group.throughput(Throughput::Bytes(flops as u64)); + group.bench_function("iter", move |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + for _i in 0..iters { + run( + black_box(&tensor), + black_box(&on_true), + black_box(&on_false), + ); + } + device.sync().unwrap(); + start.elapsed() + }) + }); + group.finish(); +} + +fn criterion_benchmark(c: &mut Criterion) { + let device = BenchDeviceHandler::new().unwrap(); + for d in device.devices { + run_where_cond_benchmark(c, &d, DType::F32, "where_cond_f32"); + run_where_cond_benchmark(c, &d, DType::BF16, "where_cond_bf16"); + run_where_cond_benchmark(c, &d, DType::F16, "where_cond_f16"); + } +} + +criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.4.1/examples/basics.rs b/v2/dependency-patches/candle-core-0.4.1/examples/basics.rs new file mode 100644 index 0000000000..fe15187b5a --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/examples/basics.rs @@ -0,0 +1,17 @@ +#[cfg(feature = "mkl")] +extern crate intel_mkl_src; + +#[cfg(feature = "accelerate")] +extern crate accelerate_src; + +use anyhow::Result; +use candle_core::{Device, Tensor}; + +fn main() -> Result<()> { + let a = Tensor::new(&[[0.0f32, 1.0, 2.0], [3.0, 4.0, 5.0]], &Device::Cpu)?; + let b = Tensor::new(&[[88.0f32, 99.0]], &Device::Cpu)?; + let new_a = a.slice_scatter(&b, 1, 2)?; + assert_eq!(a.to_vec2::()?, [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + assert_eq!(new_a.to_vec2::()?, [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.4.1/examples/cuda_basics.rs b/v2/dependency-patches/candle-core-0.4.1/examples/cuda_basics.rs new file mode 100644 index 0000000000..7929fba601 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/examples/cuda_basics.rs @@ -0,0 +1,36 @@ +#[cfg(feature = "accelerate")] +extern crate accelerate_src; + +#[cfg(feature = "mkl")] +extern crate intel_mkl_src; + +use anyhow::Result; +use candle_core::{Device, Module, Tensor}; + +use candle_core::quantized::{QMatMul, QTensor}; + +fn main() -> Result<()> { + let device = Device::new_cuda(0)?; + let q = Tensor::randn(0f32, 1.0, (72, 256), &device)?; + let q_cpu = q.to_device(&Device::Cpu)?; + let q = QTensor::quantize(&q, candle_core::quantized::GgmlDType::Q8K)?; + let q = QMatMul::from_qtensor(q)?; + let x = Tensor::randn(0f32, 1.0, (5, 256), &device)?; + let res_q_cuda = q.forward(&x)?; + println!("{res_q_cuda}"); + + let q_cpu = QTensor::quantize(&q_cpu, candle_core::quantized::GgmlDType::Q8K)?; + let q_cpu_tensor = q_cpu.dequantize(&Device::Cpu)?; + let q_cpu = QMatMul::from_qtensor(q_cpu)?; + let x_cpu = x.to_device(&Device::Cpu)?; + let res_q_cpu = q_cpu.forward(&x_cpu)?; + println!("{res_q_cpu}"); + + let res_mm = x_cpu.matmul(&q_cpu_tensor.t()?)?; + let diff = (res_mm - res_q_cuda.to_device(&Device::Cpu))? + .abs()? + .flatten_all()? + .max(0)?; + println!("{diff}"); + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.4.1/examples/cuda_sum_benchmark.rs b/v2/dependency-patches/candle-core-0.4.1/examples/cuda_sum_benchmark.rs new file mode 100644 index 0000000000..d6d182e8fc --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/examples/cuda_sum_benchmark.rs @@ -0,0 +1,54 @@ +#[cfg(feature = "mkl")] +extern crate intel_mkl_src; + +#[cfg(feature = "accelerate")] +extern crate accelerate_src; + +use std::str::FromStr; + +use anyhow::Result; +use candle_core::{Device, Tensor}; + +fn cos_sin(n: usize, device: &Device) -> Result { + let thetas: Vec<_> = (0..n).map(|i| (i as f32 / n as f32)).collect(); + let xs: Vec<_> = thetas.iter().map(|t| t.cos().abs()).collect(); + let ys: Vec<_> = thetas.iter().map(|t| t.sin().abs()).collect(); + let xs = Tensor::from_vec(xs, (n, 1), device)?; + let ys = Tensor::from_vec(ys, (1, n), device)?; + let ys = Tensor::cat(&[&ys, &ys, &ys, &ys, &ys, &ys], 1)?; + Ok(xs.matmul(&ys)?) +} + +fn main() -> Result<()> { + let device = Device::new_cuda(0)?; + let args = std::env::args().collect::>(); + let n = if args.len() < 2 { + 2000usize + } else { + usize::from_str(&args[1])? + }; + let xys_cpu = cos_sin(n, &Device::Cpu)?; + let xys = cos_sin(n, &device)?; + println!("{xys_cpu:?} {xys:?}"); + let sum_keepdim_cpu = xys_cpu.sum_keepdim(1)?; + println!("{sum_keepdim_cpu}"); + let sum_keepdim = xys.sum_keepdim(1)?; + println!("{sum_keepdim}"); + let start = std::time::Instant::now(); + let n_iters = 100; + let mut v = 0f32; + for _i in 0..n_iters { + let sum_keepdim = xys.sum_keepdim(1)?; + let sum_keepdim = sum_keepdim.sum_keepdim(0)?; + let sum_keepdim: f32 = sum_keepdim.reshape(&[])?.to_scalar()?; + v += sum_keepdim; + } + let elapsed = start.elapsed(); + if v > 0. { + println!( + "ran {n_iters} iterations, time per iter: {:?} ({v})", + elapsed.div_f64(n_iters as f64) + ); + } + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.4.1/examples/tensor-tools.rs b/v2/dependency-patches/candle-core-0.4.1/examples/tensor-tools.rs new file mode 100644 index 0000000000..1801ac58ad --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/examples/tensor-tools.rs @@ -0,0 +1,389 @@ +use candle_core::quantized::{gguf_file, GgmlDType, QTensor}; +use candle_core::{Device, Result}; +use clap::{Parser, Subcommand, ValueEnum}; +use rayon::prelude::*; + +#[derive(ValueEnum, Debug, Clone)] +enum QuantizationMode { + /// The default quantization includes all 2d tensors, except the output tensor which always + /// uses Q6_K. + Llama, +} + +impl QuantizationMode { + fn quantize(&self, name: &str, tensor: QTensor, dtype: GgmlDType) -> Result { + match self { + Self::Llama => { + // Same behavior as the llama.cpp quantization. + let should_quantize = name.ends_with(".weight") && tensor.rank() == 2; + if should_quantize { + let tensor = tensor.dequantize(&Device::Cpu)?; + if name == "output.weight" { + QTensor::quantize(&tensor, GgmlDType::Q6K) + } else { + QTensor::quantize(&tensor, dtype) + } + } else { + Ok(tensor) + } + } + } + } +} + +#[derive(ValueEnum, Debug, Clone)] +enum Quantization { + #[value(name = "q4_0")] + Q4_0, + #[value(name = "q4_1")] + Q4_1, + #[value(name = "q5_0")] + Q5_0, + #[value(name = "q5_1")] + Q5_1, + #[value(name = "q8_0")] + Q8_0, + #[value(name = "q8_1")] + Q8_1, + Q2k, + Q3k, + Q4k, + Q5k, + Q6k, + Q8k, + F16, + F32, +} + +impl Quantization { + fn dtype(&self) -> GgmlDType { + match self { + Quantization::Q4_0 => GgmlDType::Q4_0, + Quantization::Q4_1 => GgmlDType::Q4_1, + Quantization::Q5_0 => GgmlDType::Q5_0, + Quantization::Q5_1 => GgmlDType::Q5_1, + Quantization::Q8_0 => GgmlDType::Q8_0, + Quantization::Q8_1 => GgmlDType::Q8_1, + Quantization::Q2k => GgmlDType::Q2K, + Quantization::Q3k => GgmlDType::Q3K, + Quantization::Q4k => GgmlDType::Q4K, + Quantization::Q5k => GgmlDType::Q5K, + Quantization::Q6k => GgmlDType::Q6K, + Quantization::Q8k => GgmlDType::Q8K, + Quantization::F16 => GgmlDType::F16, + Quantization::F32 => GgmlDType::F32, + } + } +} + +#[derive(ValueEnum, Debug, Clone)] +enum Format { + Safetensors, + Npz, + Ggml, + Gguf, + Pth, + Pickle, +} + +impl Format { + fn infer>(p: P) -> Option { + p.as_ref() + .extension() + .and_then(|e| e.to_str()) + .and_then(|e| match e { + // We don't infer any format for .bin as it can be used for ggml/gguf or pytorch. + "safetensors" | "safetensor" => Some(Self::Safetensors), + "npz" => Some(Self::Npz), + "pth" | "pt" => Some(Self::Pth), + "ggml" => Some(Self::Ggml), + "gguf" => Some(Self::Gguf), + _ => None, + }) + } +} + +#[derive(Subcommand, Debug, Clone)] +enum Command { + Ls { + files: Vec, + + /// The file format to use, if unspecified infer from the file extension. + #[arg(long, value_enum)] + format: Option, + + /// Enable verbose mode. + #[arg(short, long)] + verbose: bool, + }, + + Quantize { + /// The input file(s), in safetensors format. + in_file: Vec, + + /// The output file, in gguf format. + #[arg(long)] + out_file: std::path::PathBuf, + + /// The quantization schema to apply. + #[arg(long, value_enum)] + quantization: Quantization, + + /// Which tensor to quantize. + #[arg(long, value_enum, default_value_t = QuantizationMode::Llama)] + mode: QuantizationMode, + }, + + Dequantize { + /// The input file, in gguf format. + in_file: std::path::PathBuf, + + /// The output file, in safetensors format. + #[arg(long)] + out_file: std::path::PathBuf, + }, +} + +#[derive(Parser, Debug, Clone)] +struct Args { + #[command(subcommand)] + command: Command, +} + +fn run_ls( + file: &std::path::PathBuf, + format: Option, + verbose: bool, + device: &Device, +) -> Result<()> { + let format = match format { + Some(format) => format, + None => match Format::infer(file) { + Some(format) => format, + None => { + println!( + "{file:?}: cannot infer format from file extension, use the --format flag" + ); + return Ok(()); + } + }, + }; + match format { + Format::Npz => { + let tensors = candle_core::npy::NpzTensors::new(file)?; + let mut names = tensors.names(); + names.sort(); + for name in names { + let shape_dtype = match tensors.get_shape_and_dtype(name) { + Ok((shape, dtype)) => format!("[{shape:?}; {dtype:?}]"), + Err(err) => err.to_string(), + }; + println!("{name}: {shape_dtype}") + } + } + Format::Safetensors => { + let tensors = unsafe { candle_core::safetensors::MmapedSafetensors::new(file)? }; + let mut tensors = tensors.tensors(); + tensors.sort_by(|a, b| a.0.cmp(&b.0)); + for (name, view) in tensors.iter() { + let dtype = view.dtype(); + let dtype = match candle_core::DType::try_from(dtype) { + Ok(dtype) => format!("{dtype:?}"), + Err(_) => format!("{dtype:?}"), + }; + let shape = view.shape(); + println!("{name}: [{shape:?}; {dtype}]") + } + } + Format::Pth => { + let mut tensors = candle_core::pickle::read_pth_tensor_info(file, verbose, None)?; + tensors.sort_by(|a, b| a.name.cmp(&b.name)); + for tensor_info in tensors.iter() { + println!( + "{}: [{:?}; {:?}]", + tensor_info.name, + tensor_info.layout.shape(), + tensor_info.dtype, + ); + if verbose { + println!(" {:?}", tensor_info); + } + } + } + Format::Pickle => { + let file = std::fs::File::open(file)?; + let mut reader = std::io::BufReader::new(file); + let mut stack = candle_core::pickle::Stack::empty(); + stack.read_loop(&mut reader)?; + for (i, obj) in stack.stack().iter().enumerate() { + println!("{i} {obj:?}"); + } + } + Format::Ggml => { + let mut file = std::fs::File::open(file)?; + let content = candle_core::quantized::ggml_file::Content::read(&mut file, device)?; + let mut tensors = content.tensors.into_iter().collect::>(); + tensors.sort_by(|a, b| a.0.cmp(&b.0)); + for (name, qtensor) in tensors.iter() { + println!("{name}: [{:?}; {:?}]", qtensor.shape(), qtensor.dtype()); + } + } + Format::Gguf => { + let mut file = std::fs::File::open(file)?; + let content = gguf_file::Content::read(&mut file)?; + if verbose { + let mut metadata = content.metadata.into_iter().collect::>(); + metadata.sort_by(|a, b| a.0.cmp(&b.0)); + println!("metadata entries ({})", metadata.len()); + for (key, value) in metadata.iter() { + println!(" {key}: {value:?}"); + } + } + let mut tensors = content.tensor_infos.into_iter().collect::>(); + tensors.sort_by(|a, b| a.0.cmp(&b.0)); + for (name, info) in tensors.iter() { + println!("{name}: [{:?}; {:?}]", info.shape, info.ggml_dtype); + } + } + } + Ok(()) +} + +fn run_quantize_safetensors( + in_files: &[std::path::PathBuf], + out_file: std::path::PathBuf, + q: Quantization, +) -> Result<()> { + let mut out_file = std::fs::File::create(out_file)?; + let mut tensors = std::collections::HashMap::new(); + for in_file in in_files.iter() { + let in_tensors = candle_core::safetensors::load(in_file, &Device::Cpu)?; + tensors.extend(in_tensors) + } + println!("tensors: {}", tensors.len()); + + let dtype = q.dtype(); + let block_size = dtype.block_size(); + + let qtensors = tensors + .into_par_iter() + .map(|(name, tensor)| { + let should_quantize = tensor.rank() == 2 && tensor.dim(1)? % block_size == 0; + println!(" quantizing {name} {tensor:?} {should_quantize}"); + let tensor = if should_quantize { + QTensor::quantize(&tensor, dtype)? + } else { + QTensor::quantize(&tensor, GgmlDType::F32)? + }; + Ok((name, tensor)) + }) + .collect::>>()?; + let qtensors = qtensors + .iter() + .map(|(k, v)| (k.as_str(), v)) + .collect::>(); + gguf_file::write(&mut out_file, &[], &qtensors)?; + Ok(()) +} + +fn run_dequantize( + in_file: std::path::PathBuf, + out_file: std::path::PathBuf, + device: &Device, +) -> Result<()> { + let mut in_file = std::fs::File::open(in_file)?; + let content = gguf_file::Content::read(&mut in_file)?; + let mut tensors = std::collections::HashMap::new(); + for (tensor_name, _) in content.tensor_infos.iter() { + let tensor = content.tensor(&mut in_file, tensor_name, device)?; + let tensor = tensor.dequantize(device)?; + tensors.insert(tensor_name.to_string(), tensor); + } + candle_core::safetensors::save(&tensors, out_file)?; + Ok(()) +} + +fn run_quantize( + in_files: &[std::path::PathBuf], + out_file: std::path::PathBuf, + q: Quantization, + qmode: QuantizationMode, + device: &Device, +) -> Result<()> { + if in_files.is_empty() { + candle_core::bail!("no specified input files") + } + if let Some(extension) = out_file.extension() { + if extension == "safetensors" { + candle_core::bail!("the generated file cannot use the safetensors extension") + } + } + if let Some(extension) = in_files[0].extension() { + if extension == "safetensors" { + return run_quantize_safetensors(in_files, out_file, q); + } + } + + if in_files.len() != 1 { + candle_core::bail!("only a single in-file can be used when quantizing gguf files") + } + + // Open the out file early so as to fail directly on missing directories etc. + let mut out_file = std::fs::File::create(out_file)?; + let mut in_ = std::fs::File::open(&in_files[0])?; + let content = gguf_file::Content::read(&mut in_)?; + println!("tensors: {}", content.tensor_infos.len()); + + let dtype = q.dtype(); + let qtensors = content + .tensor_infos + .par_iter() + .map(|(name, _)| { + println!(" quantizing {name}"); + let mut in_file = std::fs::File::open(&in_files[0])?; + let tensor = content.tensor(&mut in_file, name, device)?; + let tensor = qmode.quantize(name, tensor, dtype)?; + Ok((name, tensor)) + }) + .collect::>>()?; + let qtensors = qtensors + .iter() + .map(|(k, v)| (k.as_str(), v)) + .collect::>(); + + let metadata = content + .metadata + .iter() + .map(|(k, v)| (k.as_str(), v)) + .collect::>(); + gguf_file::write(&mut out_file, metadata.as_slice(), &qtensors)?; + Ok(()) +} + +fn main() -> anyhow::Result<()> { + let args = Args::parse(); + let device = Device::Cpu; + match args.command { + Command::Ls { + files, + format, + verbose, + } => { + let multiple_files = files.len() > 1; + for file in files.iter() { + if multiple_files { + println!("--- {file:?} ---"); + } + run_ls(file, format.clone(), verbose, &device)? + } + } + Command::Quantize { + in_file, + out_file, + quantization, + mode, + } => run_quantize(&in_file, out_file, quantization, mode, &device)?, + Command::Dequantize { in_file, out_file } => run_dequantize(in_file, out_file, &device)?, + } + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/accelerate.rs b/v2/dependency-patches/candle-core-0.4.1/src/accelerate.rs new file mode 100644 index 0000000000..d371d3b33b --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/accelerate.rs @@ -0,0 +1,476 @@ +#![allow(dead_code)] +use libc::{c_char, c_double, c_float, c_int, c_long, c_ulong}; + +mod ffi { + use super::*; + extern "C" { + // It would be nice to be able to switch to the NEWLAPACK version of the function but this + // seems to trigger some link error. Available function names can be seen here: + // /Library/Developer/CommandLineTools/SDKs/MacOSX13.3.sdk/System/Library/Frameworks/Accelerate.framework/Versions/A/Accelerate.tbd + #[link_name = "sgemm_"] + pub fn sgemm_ffi( + transa: *const c_char, + transb: *const c_char, + m: *const c_int, + n: *const c_int, + k: *const c_int, + alpha: *const c_float, + a: *const c_float, + lda: *const c_int, + b: *const c_float, + ldb: *const c_int, + beta: *const c_float, + c: *mut c_float, + ldc: *const c_int, + ); + #[link_name = "dgemm_"] + pub fn dgemm_ffi( + transa: *const c_char, + transb: *const c_char, + m: *const c_int, + n: *const c_int, + k: *const c_int, + alpha: *const c_double, + a: *const c_double, + lda: *const c_int, + b: *const c_double, + ldb: *const c_int, + beta: *const c_double, + c: *mut c_double, + ldc: *const c_int, + ); + + pub fn vvexpf(dst: *mut c_float, src: *const c_float, len: *const c_int); + pub fn vvexp(dst: *mut c_double, src: *const c_double, len: *const c_int); + pub fn vvsqrtf(dst: *mut c_float, src: *const c_float, len: *const c_int); + pub fn vvsqrt(dst: *mut c_double, src: *const c_double, len: *const c_int); + pub fn vvsinf(dst: *mut c_float, src: *const c_float, len: *const c_int); + pub fn vvsin(dst: *mut c_double, src: *const c_double, len: *const c_int); + pub fn vvcosf(dst: *mut c_float, src: *const c_float, len: *const c_int); + pub fn vvcos(dst: *mut c_double, src: *const c_double, len: *const c_int); + pub fn vvlogf(dst: *mut c_float, src: *const c_float, len: *const c_int); + pub fn vvlog(dst: *mut c_double, src: *const c_double, len: *const c_int); + pub fn vvtanhf(dst: *mut c_float, src: *const c_float, len: *const c_int); + pub fn vvtanh(dst: *mut c_double, src: *const c_double, len: *const c_int); + + pub fn vDSP_vaddD( + _: *const c_double, + _: c_long, + _: *const c_double, + _: c_long, + _: *mut c_double, + _: c_long, + _: c_ulong, + ); + pub fn vDSP_vadd( + _: *const c_float, + _: c_long, + _: *const c_float, + _: c_long, + _: *mut c_float, + _: c_long, + _: c_ulong, + ); + pub fn vDSP_vsubD( + _: *const c_double, + _: c_long, + _: *const c_double, + _: c_long, + _: *mut c_double, + _: c_long, + _: c_ulong, + ); + pub fn vDSP_vsub( + _: *const c_float, + _: c_long, + _: *const c_float, + _: c_long, + _: *mut c_float, + _: c_long, + _: c_ulong, + ); + pub fn vDSP_vmulD( + _: *const c_double, + _: c_long, + _: *const c_double, + _: c_long, + _: *mut c_double, + _: c_long, + _: c_ulong, + ); + pub fn vDSP_vmul( + _: *const c_float, + _: c_long, + _: *const c_float, + _: c_long, + _: *mut c_float, + _: c_long, + _: c_ulong, + ); + pub fn vDSP_vdivD( + _: *const c_double, + _: c_long, + _: *const c_double, + _: c_long, + _: *mut c_double, + _: c_long, + _: c_ulong, + ); + pub fn vDSP_vdiv( + _: *const c_float, + _: c_long, + _: *const c_float, + _: c_long, + _: *mut c_float, + _: c_long, + _: c_ulong, + ); + pub fn vDSP_vminD( + _: *const c_double, + _: c_long, + _: *const c_double, + _: c_long, + _: *mut c_double, + _: c_long, + _: c_ulong, + ); + pub fn vDSP_vmin( + _: *const c_float, + _: c_long, + _: *const c_float, + _: c_long, + _: *mut c_float, + _: c_long, + _: c_ulong, + ); + pub fn vDSP_vmaxD( + _: *const c_double, + _: c_long, + _: *const c_double, + _: c_long, + _: *mut c_double, + _: c_long, + _: c_ulong, + ); + pub fn vDSP_vmax( + _: *const c_float, + _: c_long, + _: *const c_float, + _: c_long, + _: *mut c_float, + _: c_long, + _: c_ulong, + ); + } +} + +#[allow(clippy::too_many_arguments)] +#[inline] +pub unsafe fn sgemm( + transa: u8, + transb: u8, + m: i32, + n: i32, + k: i32, + alpha: f32, + a: &[f32], + lda: i32, + b: &[f32], + ldb: i32, + beta: f32, + c: &mut [f32], + ldc: i32, +) { + ffi::sgemm_ffi( + &(transa as c_char), + &(transb as c_char), + &m, + &n, + &k, + &alpha, + a.as_ptr(), + &lda, + b.as_ptr(), + &ldb, + &beta, + c.as_mut_ptr(), + &ldc, + ) +} + +#[allow(clippy::too_many_arguments)] +#[inline] +pub unsafe fn dgemm( + transa: u8, + transb: u8, + m: i32, + n: i32, + k: i32, + alpha: f64, + a: &[f64], + lda: i32, + b: &[f64], + ldb: i32, + beta: f64, + c: &mut [f64], + ldc: i32, +) { + ffi::dgemm_ffi( + &(transa as c_char), + &(transb as c_char), + &m, + &n, + &k, + &alpha, + a.as_ptr(), + &lda, + b.as_ptr(), + &ldb, + &beta, + c.as_mut_ptr(), + &ldc, + ) +} + +#[inline] +pub fn vs_exp(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvexpf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} + +#[inline] +pub fn vd_exp(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvexp(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} + +#[inline] +pub fn vs_sqrt(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvsqrtf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} + +#[inline] +pub fn vd_sqrt(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvsqrt(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} + +#[inline] +pub fn vs_sin(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvsinf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} + +#[inline] +pub fn vd_sin(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvsin(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} +#[inline] +pub fn vs_cos(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvcosf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} + +#[inline] +pub fn vd_cos(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvcos(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} +#[inline] +pub fn vs_tanh(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvtanhf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} + +#[inline] +pub fn vd_tanh(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvtanh(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} + +#[inline] +pub fn vs_ln(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvlogf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} + +#[inline] +pub fn vd_ln(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvlog(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} + +#[inline] +pub fn vs_sqr(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + y.iter_mut().zip(a.iter()).for_each(|(y, a)| *y = *a * *a) +} + +#[inline] +pub fn vd_sqr(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + y.iter_mut().zip(a.iter()).for_each(|(y, a)| *y = *a * *a) +} + +#[inline] +pub fn vs_tanh_inplace(y: &mut [f32]) { + unsafe { ffi::vvtanhf(y.as_mut_ptr(), y.as_ptr(), &(y.len() as i32)) } +} + +#[inline] +pub fn vd_tanh_inplace(y: &mut [f64]) { + unsafe { ffi::vvtanh(y.as_mut_ptr(), y.as_ptr(), &(y.len() as i32)) } +} + +#[inline] +pub fn vs_exp_inplace(y: &mut [f32]) { + unsafe { ffi::vvexpf(y.as_mut_ptr(), y.as_ptr(), &(y.len() as i32)) } +} + +#[inline] +pub fn vd_exp_inplace(y: &mut [f64]) { + unsafe { ffi::vvexp(y.as_mut_ptr(), y.as_ptr(), &(y.len() as i32)) } +} + +#[inline] +pub fn vs_gelu(vs: &[f32], ys: &mut [f32]) { + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = (2.0f32 / std::f32::consts::PI).sqrt() * v * (1.0 + 0.044715 * v * v) + } + vs_tanh_inplace(ys); + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = 0.5 * v * (1.0 + *y) + } +} + +#[inline] +pub fn vd_gelu(vs: &[f64], ys: &mut [f64]) { + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = (2.0f64 / std::f64::consts::PI).sqrt() * v * (1.0 + 0.044715 * v * v) + } + vd_tanh_inplace(ys); + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = 0.5 * v * (1.0 + *y) + } +} + +#[inline] +pub fn vs_silu(vs: &[f32], ys: &mut [f32]) { + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = -v + } + vs_exp_inplace(ys); + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = v / (1.0 + *y) + } +} + +#[inline] +pub fn vd_silu(vs: &[f64], ys: &mut [f64]) { + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = -v + } + vd_exp_inplace(ys); + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = v / (1.0 + *y) + } +} + +macro_rules! binary_op { + ($fn_name:ident, $ty:ty, $accelerate_name:ident) => { + #[inline] + pub fn $fn_name(a: &[$ty], b: &[$ty], y: &mut [$ty]) { + let a_len = a.len(); + let b_len = b.len(); + let y_len = y.len(); + if a_len != y_len || b_len != y_len { + panic!( + "{} a,b,y len mismatch {a_len} {b_len} {y_len}", + stringify!($fn_name) + ); + } + unsafe { + // Weird quirk of accelerate, the rhs comes before the lhs. + ffi::$accelerate_name( + b.as_ptr(), + 1, + a.as_ptr(), + 1, + y.as_mut_ptr(), + 1, + a_len as u64, + ) + } + } + }; +} +binary_op!(vs_add, f32, vDSP_vadd); +binary_op!(vd_add, f64, vDSP_vaddD); +binary_op!(vs_sub, f32, vDSP_vsub); +binary_op!(vd_sub, f64, vDSP_vsubD); +binary_op!(vs_mul, f32, vDSP_vmul); +binary_op!(vd_mul, f64, vDSP_vmulD); +binary_op!(vs_div, f32, vDSP_vdiv); +binary_op!(vd_div, f64, vDSP_vdivD); +binary_op!(vs_max, f32, vDSP_vmax); +binary_op!(vd_max, f64, vDSP_vmaxD); +binary_op!(vs_min, f32, vDSP_vmin); +binary_op!(vd_min, f64, vDSP_vminD); diff --git a/v2/dependency-patches/candle-core-0.4.1/src/backend.rs b/v2/dependency-patches/candle-core-0.4.1/src/backend.rs new file mode 100644 index 0000000000..2125af69e7 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/backend.rs @@ -0,0 +1,124 @@ +use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; +use crate::{CpuStorage, DType, Layout, Result, Shape}; + +pub trait BackendStorage: Sized { + type Device: BackendDevice; + + fn try_clone(&self, _: &Layout) -> Result; + + fn dtype(&self) -> DType; + + fn device(&self) -> &Self::Device; + + // Maybe this should return a Cow instead so that no copy is done on the cpu case. + fn to_cpu_storage(&self) -> Result; + + fn affine(&self, _: &Layout, _: f64, _: f64) -> Result; + + fn powf(&self, _: &Layout, _: f64) -> Result; + + fn elu(&self, _: &Layout, _: f64) -> Result; + + fn reduce_op(&self, _: ReduceOp, _: &Layout, _: &[usize]) -> Result; + + fn cmp(&self, _: CmpOp, _: &Self, _: &Layout, _: &Layout) -> Result; + + fn to_dtype(&self, _: &Layout, _: DType) -> Result; + + fn unary_impl(&self, _: &Layout) -> Result; + + fn binary_impl(&self, _: &Self, _: &Layout, _: &Layout) -> Result; + + fn where_cond(&self, _: &Layout, _: &Self, _: &Layout, _: &Self, _: &Layout) -> Result; + + fn conv1d( + &self, + _l: &Layout, + _kernel: &Self, + _kernel_l: &Layout, + _params: &crate::conv::ParamsConv1D, + ) -> Result; + + fn conv_transpose1d( + &self, + _l: &Layout, + _kernel: &Self, + _kernel_l: &Layout, + _params: &crate::conv::ParamsConvTranspose1D, + ) -> Result; + + fn conv2d( + &self, + _l: &Layout, + _kernel: &Self, + _kernel_l: &Layout, + _params: &crate::conv::ParamsConv2D, + ) -> Result; + + fn conv_transpose2d( + &self, + _l: &Layout, + _kernel: &Self, + _kernel_l: &Layout, + _params: &crate::conv::ParamsConvTranspose2D, + ) -> Result; + + fn avg_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result; + fn max_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result; + fn upsample_nearest1d(&self, _: &Layout, _: usize) -> Result; + fn upsample_nearest2d(&self, _: &Layout, _: usize, _: usize) -> Result; + + fn gather(&self, _: &Layout, _: &Self, _: &Layout, _: usize) -> Result; + fn scatter_add( + &self, + _: &Layout, + _: &Self, + _: &Layout, + _: &Self, + _: &Layout, + _: usize, + ) -> Result; + fn index_select(&self, _: &Self, _: &Layout, _: &Layout, _: usize) -> Result; + fn index_add( + &self, + _: &Layout, + _: &Self, + _: &Layout, + _: &Self, + _: &Layout, + _: usize, + ) -> Result; + + fn matmul( + &self, + _: &Self, + _: (usize, usize, usize, usize), + _: &Layout, + _: &Layout, + ) -> Result; + + fn copy_strided_src(&self, _: &mut Self, _: usize, _: &Layout) -> Result<()>; +} + +pub trait BackendDevice: Sized + std::fmt::Debug + Clone { + type Storage: BackendStorage; + + // TODO: Make the usize generic and part of a generic DeviceLocation. + fn new(_: usize) -> Result; + + fn location(&self) -> crate::DeviceLocation; + + fn same_device(&self, _: &Self) -> bool; + + fn zeros_impl(&self, _shape: &Shape, _dtype: DType) -> Result; + + fn ones_impl(&self, _shape: &Shape, _dtype: DType) -> Result; + + fn storage_from_cpu_storage(&self, _: &CpuStorage) -> Result; + + fn rand_uniform(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result; + + fn rand_normal(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result; + + fn set_seed(&self, _: u64) -> Result<()>; +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/backprop.rs b/v2/dependency-patches/candle-core-0.4.1/src/backprop.rs new file mode 100644 index 0000000000..2a1db58a5e --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/backprop.rs @@ -0,0 +1,728 @@ +use crate::op::{BinaryOp, Op, ReduceOp, UnaryOp}; +use crate::{Error, Result, Tensor, TensorId}; +use std::collections::HashMap; + +// arg has been reduced to node via reduce_dims, expand it back to arg. +// This has to handle keepdims. +fn broadcast_back(arg: &Tensor, node: &Tensor, reduced_dims: &[usize]) -> Result { + if arg.rank() == node.rank() { + // keepdim = true + node.broadcast_as(arg.shape()) + } else { + // keepdim = false + // first expand the reduced dims. + node.reshape(reduced_dims)?.broadcast_as(arg.shape()) + } +} + +thread_local! { + static CANDLE_GRAD_DO_NOT_DETACH: bool = { + match std::env::var("CANDLE_GRAD_DO_NOT_DETACH") { + Ok(s) => { + !s.is_empty() && s != "0" + }, + Err(_) => false, + } + } +} + +impl Tensor { + /// Return all the nodes that lead to this value in a topologically sorted vec, the first + /// elements having dependencies on the latter ones, e.g. the first element if any is the + /// argument. + /// This assumes that the op graph is a DAG. + fn sorted_nodes(&self) -> Vec<&Tensor> { + // The vec of sorted nodes is passed as an owned value rather than a mutable reference + // to get around some lifetime limitations. + fn walk<'a>( + node: &'a Tensor, + nodes: Vec<&'a Tensor>, + already_seen: &mut HashMap, + ) -> (bool, Vec<&'a Tensor>) { + if let Some(&tg) = already_seen.get(&node.id()) { + return (tg, nodes); + } + let mut track_grad = false; + let mut nodes = if node.is_variable() { + // Do not call recursively on the "leaf" nodes. + track_grad = true; + nodes + } else if node.dtype().is_int() { + nodes + } else if let Some(op) = node.op() { + match op { + Op::IndexAdd(t1, t2, t3, _) + | Op::ScatterAdd(t1, t2, t3, _) + | Op::CustomOp3(t1, t2, t3, _) + | Op::WhereCond(t1, t2, t3) => { + let (tg, nodes) = walk(t1, nodes, already_seen); + track_grad |= tg; + let (tg, nodes) = walk(t2, nodes, already_seen); + track_grad |= tg; + let (tg, nodes) = walk(t3, nodes, already_seen); + track_grad |= tg; + nodes + } + Op::Conv1D { + arg: lhs, + kernel: rhs, + .. + } + | Op::ConvTranspose1D { + arg: lhs, + kernel: rhs, + .. + } + | Op::Conv2D { + arg: lhs, + kernel: rhs, + .. + } + | Op::ConvTranspose2D { + arg: lhs, + kernel: rhs, + .. + } + | Op::CustomOp2(lhs, rhs, _) + | Op::Binary(lhs, rhs, _) + | Op::Gather(lhs, rhs, _) + | Op::IndexSelect(lhs, rhs, _) + | Op::Matmul(lhs, rhs) + | Op::SliceScatter0(lhs, rhs, _) => { + let (tg, nodes) = walk(lhs, nodes, already_seen); + track_grad |= tg; + let (tg, nodes) = walk(rhs, nodes, already_seen); + track_grad |= tg; + nodes + } + Op::Cat(args, _) => args.iter().fold(nodes, |nodes, arg| { + let (tg, nodes) = walk(arg, nodes, already_seen); + track_grad |= tg; + nodes + }), + Op::Affine { arg, mul, .. } => { + if *mul == 0. { + nodes + } else { + let (tg, nodes) = walk(arg, nodes, already_seen); + track_grad |= tg; + nodes + } + } + Op::Unary(_node, UnaryOp::Ceil) + | Op::Unary(_node, UnaryOp::Floor) + | Op::Unary(_node, UnaryOp::Round) => nodes, + Op::Reshape(node) + | Op::UpsampleNearest1D { arg: node, .. } + | Op::UpsampleNearest2D { arg: node, .. } + | Op::AvgPool2D { arg: node, .. } + | Op::MaxPool2D { arg: node, .. } + | Op::Copy(node) + | Op::Broadcast(node) + | Op::Cmp(node, _) + | Op::Reduce(node, ReduceOp::Min | ReduceOp::Sum | ReduceOp::Max, _) + | Op::ToDevice(node) + | Op::Transpose(node, _, _) + | Op::Permute(node, _) + | Op::Narrow(node, _, _, _) + | Op::Unary(node, _) + | Op::Elu(node, _) + | Op::Powf(node, _) + | Op::CustomOp1(node, _) => { + let (tg, nodes) = walk(node, nodes, already_seen); + track_grad |= tg; + nodes + } + Op::ToDType(node) => { + if node.dtype().is_float() { + let (tg, nodes) = walk(node, nodes, already_seen); + track_grad |= tg; + nodes + } else { + nodes + } + } + Op::Reduce(_, ReduceOp::ArgMin | ReduceOp::ArgMax, _) => nodes, + } + } else { + nodes + }; + already_seen.insert(node.id(), track_grad); + if track_grad { + nodes.push(node); + } + (track_grad, nodes) + } + let (_tg, mut nodes) = walk(self, vec![], &mut HashMap::new()); + nodes.reverse(); + nodes + } + + pub fn backward(&self) -> Result { + let sorted_nodes = self.sorted_nodes(); + let mut grads = GradStore::new(); + grads.insert(self, self.ones_like()?.contiguous()?); + for node in sorted_nodes.iter() { + if node.is_variable() { + continue; + } + let grad = grads + .remove(node) + .expect("candle internal error - grad not populated"); + // https://github.com/huggingface/candle/issues/1241 + // Ideally, we would make these operations in place where possible to ensure that we + // do not have to allocate too often. Here we just call `.detach` to avoid computing + // the backprop graph of the backprop itself. This would be an issue for second order + // derivatives but these are out of scope at the moment. + let do_not_detach = CANDLE_GRAD_DO_NOT_DETACH.with(|b| *b); + let grad = if do_not_detach { grad } else { grad.detach() }; + if let Some(op) = node.op() { + match op { + Op::Binary(lhs, rhs, BinaryOp::Add) => { + let lhs_sum_grad = grads.or_insert(lhs)?; + *lhs_sum_grad = lhs_sum_grad.add(&grad)?; + let rhs_sum_grad = grads.or_insert(rhs)?; + *rhs_sum_grad = rhs_sum_grad.add(&grad)?; + } + Op::Binary(lhs, rhs, BinaryOp::Sub) => { + let lhs_sum_grad = grads.or_insert(lhs)?; + *lhs_sum_grad = lhs_sum_grad.add(&grad)?; + let rhs_sum_grad = grads.or_insert(rhs)?; + *rhs_sum_grad = rhs_sum_grad.sub(&grad)?; + } + Op::Binary(lhs, rhs, BinaryOp::Mul) => { + let lhs_grad = grad.mul(rhs)?; + let lhs_sum_grad = grads.or_insert(lhs)?; + *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?; + let rhs_grad = grad.mul(lhs)?; + let rhs_sum_grad = grads.or_insert(rhs)?; + *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?; + } + Op::Binary(lhs, rhs, BinaryOp::Div) => { + let lhs_grad = grad.div(rhs)?; + let lhs_sum_grad = grads.or_insert(lhs)?; + *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?; + let rhs_grad = grad.mul(lhs)?.div(&rhs.sqr()?)?; + let rhs_sum_grad = grads.or_insert(rhs)?; + *rhs_sum_grad = rhs_sum_grad.sub(&rhs_grad)?; + } + Op::Binary(lhs, rhs, BinaryOp::Minimum) + | Op::Binary(lhs, rhs, BinaryOp::Maximum) => { + let mask_lhs = node.eq(lhs)?.to_dtype(grad.dtype())?; + let mask_rhs = node.eq(rhs)?.to_dtype(grad.dtype())?; + + // If both masks are 1 one the same point, we want to scale the + // gradient by 0.5 rather than 1. + let lhs_grad = mask_lhs.mul(&grad)?.div(&(&mask_rhs + 1.)?)?; + let lhs_sum_grad = grads.or_insert(lhs)?; + *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?; + + let rhs_grad = mask_rhs.mul(&grad)?.div(&(&mask_lhs + 1.)?)?; + let rhs_sum_grad = grads.or_insert(rhs)?; + *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?; + } + Op::WhereCond(pred, t, f) => { + let zeros = grad.zeros_like()?; + let t_sum_grad = grads.or_insert(t)?; + let t_grad = pred.where_cond(&grad, &zeros)?; + *t_sum_grad = t_sum_grad.add(&t_grad)?; + let f_sum_grad = grads.or_insert(f)?; + let f_grad = pred.where_cond(&zeros, &grad)?; + *f_sum_grad = f_sum_grad.add(&f_grad)?; + } + Op::Conv1D { + arg, + kernel, + padding, + stride, + dilation, + } => { + // The output height for conv_transpose1d is: + // (l_in - 1) * stride - 2 * padding + dilation * (k_size - 1) + out_padding + 1 + let grad_l_in = grad.dim(2)?; + let k_size = kernel.dim(2)?; + let out_size = + (grad_l_in - 1) * stride + dilation * (k_size - 1) + 1 - 2 * padding; + let out_padding = arg.dim(2)? - out_size; + let grad_arg = grad.conv_transpose1d( + kernel, + *padding, + out_padding, + *stride, + *dilation, + /* groups */ 1, + )?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&grad_arg)?; + + let grad_kernel = arg + .transpose(0, 1)? + .conv1d(&grad.transpose(0, 1)?, *padding, *dilation, *stride, 1)? + .transpose(0, 1)?; + let sum_grad = grads.or_insert(kernel)?; + let (_, _, k0) = kernel.dims3()?; + let (_, _, g_k0) = grad_kernel.dims3()?; + let grad_kernel = if g_k0 != k0 { + grad_kernel.narrow(2, 0, k0)? + } else { + grad_kernel + }; + *sum_grad = sum_grad.add(&grad_kernel)?; + } + Op::Conv2D { + arg, + kernel, + padding, + stride, + dilation, + } => { + // The output height for conv_transpose2d is: + // (i_h - 1) * stride - 2 * padding + dilation * (k_h - 1) + out_padding + 1 + let grad_h = grad.dim(2)?; + let k_h = kernel.dim(2)?; + let out_size = + (grad_h - 1) * stride + dilation * (k_h - 1) + 1 - 2 * padding; + let out_padding = arg.dim(2)? - out_size; + let grad_arg = grad.conv_transpose2d( + kernel, + *padding, + out_padding, + *stride, + *dilation, + )?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&grad_arg)?; + + let grad_kernel = arg + .transpose(0, 1)? + .conv2d(&grad.transpose(0, 1)?, *padding, *dilation, *stride, 1)? + .transpose(0, 1)?; + let sum_grad = grads.or_insert(kernel)?; + let (_, _, k0, k1) = kernel.dims4()?; + let (_, _, g_k0, g_k1) = grad_kernel.dims4()?; + let grad_kernel = if g_k0 != k0 || g_k1 != k1 { + grad_kernel.narrow(2, 0, k0)?.narrow(3, 0, k1)? + } else { + grad_kernel + }; + *sum_grad = sum_grad.add(&grad_kernel)?; + } + Op::ConvTranspose1D { .. } => Err(Error::BackwardNotSupported { + op: "conv-transpose1d", + })?, + Op::ConvTranspose2D { .. } => Err(Error::BackwardNotSupported { + op: "conv-transpose2d", + })?, + Op::AvgPool2D { + arg, + kernel_size, + stride, + } => { + if kernel_size != stride { + crate::bail!("backward not supported for avgpool2d if ksize {kernel_size:?} != stride {stride:?}") + } + let (_n, _c, h, w) = arg.dims4()?; + let grad_arg = grad.upsample_nearest2d(h, w)?; + let grad_arg = + (grad_arg * (1f64 / (kernel_size.0 * kernel_size.1) as f64))?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&grad_arg)?; + } + Op::MaxPool2D { + arg, + kernel_size, + stride, + } => { + if kernel_size != stride { + crate::bail!("backward not supported for maxpool2d if ksize {kernel_size:?} != stride {stride:?}") + } + let (_n, _c, h, w) = arg.dims4()?; + // For computing the max-pool gradient, we compute a mask where a 1 means + // that the element is the maximum, then we apply this mask to the + // upsampled gradient (taking into account that multiple max may exist so + // we scale the gradient for this case). + let node_upsampled = node.upsample_nearest2d(h, w)?; + let mask = arg.eq(&node_upsampled)?.to_dtype(arg.dtype())?; + let avg = mask.avg_pool2d_with_stride(*kernel_size, *stride)?; + let grad_arg = ((grad * avg)?.upsample_nearest2d(h, w)? * mask)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&grad_arg)?; + } + Op::UpsampleNearest1D { arg, target_size } => { + let (_n, c, size) = arg.dims3()?; + if target_size % size != 0 { + crate::bail!("backward not supported for non integer upscaling factors") + } + let scale = target_size / size; + + let kernel = Tensor::ones((c, 1, scale), arg.dtype(), arg.device())?; + let conv_sum = grad.conv1d(&kernel, 0, scale, 1, c)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = conv_sum; + } + Op::UpsampleNearest2D { + arg, + target_h, + target_w, + } => { + let (_n, c, h, w) = arg.dims4()?; + if target_h % h != 0 || target_w % w != 0 { + crate::bail!("backward not supported for non integer upscaling factors") + } + let scale_h = target_h / h; + let scale_w = target_w / w; + + if scale_h != scale_w { + crate::bail!("backward not supported for non uniform upscaling factors") + }; + let kernel = + Tensor::ones((c, 1, scale_h, scale_w), arg.dtype(), arg.device())?; + let conv_sum = grad.conv2d(&kernel, 0, scale_h, 1, c)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = conv_sum; + } + Op::SliceScatter0(lhs, rhs, start_rhs) => { + let rhs_sum_grad = grads.or_insert(rhs)?; + let rhs_grad = grad.narrow(0, *start_rhs, rhs.dim(0)?)?; + *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?; + + let lhs_sum_grad = grads.or_insert(lhs)?; + let lhs_grad = grad.slice_scatter0(&rhs.zeros_like()?, *start_rhs)?; + *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)? + } + Op::Gather(arg, indexes, dim) => { + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.scatter_add(indexes, &grad, *dim)?; + } + Op::ScatterAdd(init, indexes, src, dim) => { + let init_sum_grad = grads.or_insert(init)?; + *init_sum_grad = init_sum_grad.add(&grad)?; + + let src_grad = grad.gather(indexes, *dim)?; + let src_sum_grad = grads.or_insert(src)?; + *src_sum_grad = src_sum_grad.add(&src_grad)?; + } + Op::IndexAdd(init, indexes, src, dim) => { + let init_sum_grad = grads.or_insert(init)?; + *init_sum_grad = init_sum_grad.add(&grad)?; + + let src_grad = grad.index_select(indexes, *dim)?; + let src_sum_grad = grads.or_insert(src)?; + *src_sum_grad = src_sum_grad.add(&src_grad)?; + } + Op::IndexSelect(arg, indexes, dim) => { + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.index_add(indexes, &grad, *dim)?; + } + Op::Matmul(lhs, rhs) => { + // Skipping checks, the op went ok, we can skip + // the matmul size checks for now. + + let lhs_grad = grad.matmul(&rhs.t()?)?; + let lhs_sum_grad = grads.or_insert(lhs)?; + *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?; + + let rhs_grad = lhs.t()?.matmul(&grad)?; + let rhs_sum_grad = grads.or_insert(rhs)?; + *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?; + } + Op::Cat(args, dim) => { + let mut start_idx = 0; + for arg in args { + let len = arg.dims()[*dim]; + let arg_grad = grad.narrow(*dim, start_idx, len)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&arg_grad)?; + start_idx += len; + } + } + Op::Broadcast(arg) => { + let arg_dims = arg.dims(); + let node_dims = node.dims(); + // The number of dims that have been inserted on the left. + let left_dims = node_dims.len() - arg_dims.len(); + let mut sum_dims: Vec = (0..left_dims).collect(); + for (dim, (node_dim, arg_dim)) in node_dims[left_dims..] + .iter() + .zip(arg_dims.iter()) + .enumerate() + { + if node_dim != arg_dim { + sum_dims.push(dim + left_dims) + } + } + + let mut arg_grad = grad.sum_keepdim(sum_dims.as_slice())?; + for _i in 0..left_dims { + arg_grad = arg_grad.squeeze(0)? + } + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&arg_grad.broadcast_as(sum_grad.dims())?)?; + } + Op::Reduce(arg, ReduceOp::Sum, reduced_dims) => { + let grad = broadcast_back(arg, &grad, reduced_dims)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&grad)?; + } + Op::Cmp(_args, _) => {} + Op::Reduce(arg, ReduceOp::Max, reduced_dims) => { + let node = broadcast_back(arg, node, reduced_dims)?; + let grad = broadcast_back(arg, &grad, reduced_dims)?; + let grad = node.eq(arg)?.to_dtype(grad.dtype())?.mul(&grad)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&grad.broadcast_as(sum_grad.dims())?)?; + } + Op::Reduce(arg, ReduceOp::Min, reduced_dims) => { + let node = broadcast_back(arg, node, reduced_dims)?; + let grad = broadcast_back(arg, &grad, reduced_dims)?; + let grad = node.eq(arg)?.to_dtype(grad.dtype())?.mul(&grad)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&grad.broadcast_as(sum_grad.dims())?)?; + } + Op::ToDType(arg) => { + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&grad.to_dtype(arg.dtype())?)? + } + Op::Copy(arg) => { + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&grad)? + } + Op::Affine { arg, mul, .. } => { + let arg_grad = grad.affine(*mul, 0.)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&arg_grad)? + } + Op::Unary(arg, UnaryOp::Log) => { + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&(grad / arg)?)? + } + Op::Unary(arg, UnaryOp::Sin) => { + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&(&grad * arg.cos())?)? + } + Op::Unary(arg, UnaryOp::Cos) => { + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.sub(&(&grad * arg.sin())?)? + } + Op::Unary(arg, UnaryOp::Tanh) => { + let sum_grad = grads.or_insert(arg)?; + let minus_dtanh = (node.sqr()? - 1.)?; + *sum_grad = sum_grad.sub(&(&grad * &minus_dtanh)?)? + } + Op::Unary(arg, UnaryOp::Abs) => { + let sum_grad = grads.or_insert(arg)?; + let ones = arg.ones_like()?; + let abs_grad = arg.ge(&arg.zeros_like()?)?.where_cond(&ones, &ones.neg()?); + *sum_grad = sum_grad.add(&(&grad * abs_grad)?)? + } + Op::Unary(arg, UnaryOp::Exp) => { + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&(&grad * *node)?)? + } + Op::Unary(arg, UnaryOp::Neg) => { + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.sub(&grad)? + } + Op::Unary(arg, UnaryOp::Recip) => { + let sum_grad = grads.or_insert(arg)?; + let grad = (grad / arg.sqr()?)?; + *sum_grad = sum_grad.sub(&grad)? + } + &Op::Narrow(ref arg, dim, start_idx, len) => { + let arg_dims = arg.dims(); + let left_pad = if start_idx == 0 { + None + } else { + let mut dims = arg_dims.to_vec(); + dims[dim] = start_idx; + Some(Tensor::zeros(dims, grad.dtype(), grad.device())?) + }; + let right_pad = arg_dims[dim] - start_idx - len; + let right_pad = if right_pad == 0 { + None + } else { + let mut dims = arg_dims.to_vec(); + dims[dim] = right_pad; + Some(Tensor::zeros(dims, grad.dtype(), grad.device())?) + }; + let arg_grad = match (left_pad, right_pad) { + (None, None) => grad, + (Some(l), None) => Tensor::cat(&[&l, &grad], dim)?, + (None, Some(r)) => Tensor::cat(&[&grad, &r], dim)?, + (Some(l), Some(r)) => Tensor::cat(&[&l, &grad, &r], dim)?, + }; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&arg_grad)? + } + Op::Reduce(_, ReduceOp::ArgMin, _) => {} + Op::Reduce(_, ReduceOp::ArgMax, _) => {} + Op::Reshape(arg) => { + let arg_grad = grad.reshape(arg.dims())?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&arg_grad)? + } + Op::Unary(_, UnaryOp::Ceil) => Err(Error::BackwardNotSupported { op: "ceil" })?, + Op::Unary(_, UnaryOp::Floor) => { + Err(Error::BackwardNotSupported { op: "floor" })? + } + Op::Unary(_, UnaryOp::Round) => { + Err(Error::BackwardNotSupported { op: "round" })? + } + Op::Unary(arg, UnaryOp::Gelu) => { + let sum_grad = grads.or_insert(arg)?; + let cube = arg.powf(3.)?; + let tanh = (0.0356774 * &cube + (0.797885 * arg)?)?.tanh()?; + let gelu_grad = (((0.5 * &tanh)? + + (0.0535161 * cube + (0.398942 * arg)?)? * (1. - tanh.powf(2.)?))? + + 0.5)?; + *sum_grad = sum_grad.add(&(&grad * gelu_grad)?)? + } + Op::Unary(arg, UnaryOp::Erf) => { + let sum_grad = grads.or_insert(arg)?; + // d/dx erf(x) = 2/sqrt(pi) * e^(-x^2) + let erf_grad = + (2. / std::f64::consts::PI.sqrt()) * (arg.sqr()?.neg()?).exp()?; + *sum_grad = sum_grad.add(&(&grad * erf_grad)?)? + } + Op::Unary(arg, UnaryOp::GeluErf) => { + let sum_grad = grads.or_insert(arg)?; + // d/dx gelu_erf(x) = 0.5 + 0.398942 e^(-x^2/2) x + 0.5 erf(x/sqrt(2)) + let neg_half_square = (arg.sqr()?.neg()? / 2.)?; + let scaled_exp_arg = (0.398942 * neg_half_square.exp()? * arg)?; + let arg_scaled_sqrt = (arg / 2f64.sqrt())?; + let erf_scaled_sqrt = (0.5 * arg_scaled_sqrt.erf()?)?; + let gelu_erf_grad = (0.5 + scaled_exp_arg + erf_scaled_sqrt)?; + *sum_grad = sum_grad.add(&(&grad * gelu_erf_grad)?)?; + } + Op::Unary(arg, UnaryOp::Relu) => { + let sum_grad = grads.or_insert(arg)?; + let relu_grad = arg.ge(&arg.zeros_like()?)?.to_dtype(arg.dtype())?; + *sum_grad = sum_grad.add(&(&grad * relu_grad)?)? + } + Op::Unary(arg, UnaryOp::Silu) => { + let sum_grad = grads.or_insert(arg)?; + // d/dx silu = sigmoid(x) * (1 + x * (1 - sigmoid(x))) + let sigmoid_arg = (*node / arg)?; + let silu_grad = (&sigmoid_arg * (1. + (arg * (1. - &sigmoid_arg)?)?)?)?; + *sum_grad = sum_grad.add(&(&grad * silu_grad)?)? + } + Op::Elu(arg, alpha) => { + // d/dx elu(x) = 1 for x > 0, alpha * e^x for x <= 0 + let sum_grad = grads.or_insert(arg)?; + let zeros = arg.zeros_like()?; + let positive_mask = arg.gt(&zeros)?.to_dtype(arg.dtype())?; + let negative_mask = arg.le(&zeros)?.to_dtype(arg.dtype())?; + let negative_exp_mask = ((negative_mask * arg.exp())? * *alpha)?; + let combined_mask = (positive_mask + negative_exp_mask)?; + *sum_grad = sum_grad.add(&(grad * combined_mask)?)? + } + Op::Powf(arg, e) => { + let arg_grad = (&(grad * arg.powf(e - 1.)?)? * *e)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&arg_grad)? + } + Op::CustomOp1(arg, c) => { + if let Some(arg_grad) = c.bwd(arg, node, &grad)? { + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&arg_grad)? + } + } + Op::CustomOp2(arg1, arg2, c) => { + let (arg_grad1, arg_grad2) = c.bwd(arg1, arg2, node, &grad)?; + if let Some(arg_grad1) = arg_grad1 { + let sum_grad = grads.or_insert(arg1)?; + *sum_grad = sum_grad.add(&arg_grad1)? + } + if let Some(arg_grad2) = arg_grad2 { + let sum_grad = grads.or_insert(arg2)?; + *sum_grad = sum_grad.add(&arg_grad2)? + } + } + Op::CustomOp3(arg1, arg2, arg3, c) => { + let (arg_grad1, arg_grad2, arg_grad3) = + c.bwd(arg1, arg2, arg3, node, &grad)?; + if let Some(arg_grad1) = arg_grad1 { + let sum_grad = grads.or_insert(arg1)?; + *sum_grad = sum_grad.add(&arg_grad1)? + } + if let Some(arg_grad2) = arg_grad2 { + let sum_grad = grads.or_insert(arg2)?; + *sum_grad = sum_grad.add(&arg_grad2)? + } + if let Some(arg_grad3) = arg_grad3 { + let sum_grad = grads.or_insert(arg3)?; + *sum_grad = sum_grad.add(&arg_grad3)? + } + } + Op::Unary(arg, UnaryOp::Sqr) => { + let arg_grad = arg.mul(&grad)?.affine(2., 0.)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&arg_grad)? + } + Op::Unary(arg, UnaryOp::Sqrt) => { + let arg_grad = grad.div(node)?.affine(0.5, 0.)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&arg_grad)? + } + Op::ToDevice(arg) => { + let sum_grad = grads.or_insert(arg)?; + let arg_grad = grad.to_device(sum_grad.device())?; + *sum_grad = sum_grad.add(&arg_grad)? + } + Op::Transpose(arg, dim1, dim2) => { + let arg_grad = grad.transpose(*dim1, *dim2)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&arg_grad)? + } + Op::Permute(arg, dims) => { + let mut inv_dims = vec![0; dims.len()]; + for (i, &dim_idx) in dims.iter().enumerate() { + inv_dims[dim_idx] = i + } + let arg_grad = grad.permute(inv_dims)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&arg_grad)? + } + }; + } + } + Ok(grads) + } +} + +#[derive(Debug)] +pub struct GradStore(HashMap); + +impl GradStore { + fn new() -> Self { + GradStore(HashMap::new()) + } + + pub fn get_id(&self, id: TensorId) -> Option<&Tensor> { + self.0.get(&id) + } + + pub fn get(&self, tensor: &Tensor) -> Option<&Tensor> { + self.0.get(&tensor.id()) + } + + pub fn remove(&mut self, tensor: &Tensor) -> Option { + self.0.remove(&tensor.id()) + } + + pub fn insert(&mut self, tensor: &Tensor, grad: Tensor) -> Option { + self.0.insert(tensor.id(), grad) + } + + fn or_insert(&mut self, tensor: &Tensor) -> Result<&mut Tensor> { + use std::collections::hash_map::Entry; + let grad = match self.0.entry(tensor.id()) { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(entry) => { + let grad = tensor.zeros_like()?; + entry.insert(grad) + } + }; + Ok(grad) + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/conv.rs b/v2/dependency-patches/candle-core-0.4.1/src/conv.rs new file mode 100644 index 0000000000..7b3922dd73 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/conv.rs @@ -0,0 +1,358 @@ +use crate::{op::BackpropOp, op::Op, Error, Result, Tensor}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParamsConv1D { + pub(crate) b_size: usize, + // Maybe we should have a version without l_in as this bit depends on the input and not only on + // the weights. + pub(crate) l_in: usize, + pub(crate) c_out: usize, + pub(crate) c_in: usize, + pub(crate) k_size: usize, + pub(crate) padding: usize, + pub(crate) stride: usize, + pub(crate) dilation: usize, +} + +impl ParamsConv1D { + pub(crate) fn l_out(&self) -> usize { + (self.l_in + 2 * self.padding - self.dilation * (self.k_size - 1) - 1) / self.stride + 1 + } + + pub(crate) fn out_dims(&self) -> Vec { + let l_out = self.l_out(); + vec![self.b_size, self.c_out, l_out] + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParamsConvTranspose1D { + pub(crate) b_size: usize, + pub(crate) l_in: usize, + pub(crate) c_out: usize, + pub(crate) c_in: usize, + pub(crate) k_size: usize, + pub(crate) padding: usize, + pub(crate) output_padding: usize, + pub(crate) stride: usize, + pub(crate) dilation: usize, +} + +impl ParamsConvTranspose1D { + pub(crate) fn l_out(&self) -> usize { + (self.l_in - 1) * self.stride - 2 * self.padding + + self.dilation * (self.k_size - 1) + + self.output_padding + + 1 + } + + pub(crate) fn out_dims(&self) -> Vec { + let l_out = self.l_out(); + vec![self.b_size, self.c_out, l_out] + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum CudnnFwdAlgo { + ImplicitGemm, + ImplicitPrecompGemm, + Gemm, + Direct, + Fft, + FftTiling, + Winograd, + WinogradNonFused, + Count, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParamsConv2D { + pub(crate) b_size: usize, + pub(crate) i_h: usize, + pub(crate) i_w: usize, + pub(crate) k_h: usize, + pub(crate) k_w: usize, + pub(crate) c_out: usize, + pub(crate) c_in: usize, + pub(crate) padding: usize, + pub(crate) stride: usize, + pub(crate) dilation: usize, + pub cudnn_fwd_algo: Option, +} + +impl ParamsConv2D { + pub(crate) fn out_h(&self) -> usize { + (self.i_h + 2 * self.padding - self.dilation * (self.k_h - 1) - 1) / self.stride + 1 + } + + pub(crate) fn out_w(&self) -> usize { + (self.i_w + 2 * self.padding - self.dilation * (self.k_w - 1) - 1) / self.stride + 1 + } + + pub(crate) fn out_dims(&self) -> Vec { + vec![self.b_size, self.c_out, self.out_h(), self.out_w()] + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParamsConvTranspose2D { + pub(crate) b_size: usize, + pub(crate) i_h: usize, + pub(crate) i_w: usize, + pub(crate) k_h: usize, + pub(crate) k_w: usize, + pub(crate) c_out: usize, + pub(crate) c_in: usize, + pub(crate) padding: usize, + pub(crate) output_padding: usize, + pub(crate) stride: usize, + pub(crate) dilation: usize, +} + +impl ParamsConvTranspose2D { + pub(crate) fn out_h(&self) -> usize { + (self.i_h - 1) * self.stride + self.dilation * (self.k_h - 1) + self.output_padding + 1 + - 2 * self.padding + } + + pub(crate) fn out_w(&self) -> usize { + (self.i_w - 1) * self.stride + self.dilation * (self.k_w - 1) + self.output_padding + 1 + - 2 * self.padding + } + + pub(crate) fn out_dims(&self) -> Vec { + vec![self.b_size, self.c_out, self.out_h(), self.out_w()] + } +} + +impl Tensor { + fn conv1d_single_group(&self, kernel: &Self, params: &ParamsConv1D) -> Result { + let storage = + self.storage() + .conv1d(self.layout(), &kernel.storage(), kernel.layout(), params)?; + let op = BackpropOp::new2(self, kernel, |arg, kernel| Op::Conv1D { + arg, + kernel, + padding: params.padding, + stride: params.stride, + dilation: params.dilation, + }); + let out_dims = params.out_dims(); + Ok(crate::tensor::from_storage(storage, out_dims, op, false)) + } + + /// Applies a 1D convolution over the input tensor. + pub fn conv1d( + &self, + kernel: &Self, + padding: usize, + stride: usize, + dilation: usize, + groups: usize, + ) -> Result { + let (c_out, c_in_k, k_size) = kernel.dims3()?; + let (b_size, c_in, l_in) = self.dims3()?; + if c_in != c_in_k * groups { + Err(Error::Conv1dInvalidArgs { + inp_shape: self.shape().clone(), + k_shape: kernel.shape().clone(), + padding, + stride, + msg: "the number of in-channels on the input doesn't match the kernel size", + } + .bt())? + } + + let params = ParamsConv1D { + b_size, + l_in, + c_out: c_out / groups, + c_in: c_in / groups, + k_size, + padding, + stride, + dilation, + }; + if groups == 1 { + self.conv1d_single_group(kernel, ¶ms) + } else { + let blocks = self.chunk(groups, 1)?; + let kernel = kernel.chunk(groups, 0)?; + let blocks = blocks + .iter() + .zip(&kernel) + .map(|(block, kernel)| block.conv1d_single_group(kernel, ¶ms)) + .collect::>>()?; + Tensor::cat(&blocks, 1) + } + } + + fn conv_transpose1d_single_group( + &self, + kernel: &Self, + params: &ParamsConvTranspose1D, + ) -> Result { + let storage = self.storage().conv_transpose1d( + self.layout(), + &kernel.storage(), + kernel.layout(), + params, + )?; + let op = BackpropOp::new2(self, kernel, |arg, kernel| Op::ConvTranspose1D { + arg, + kernel, + padding: params.padding, + output_padding: params.output_padding, + stride: params.stride, + dilation: params.dilation, + }); + let out_dims = params.out_dims(); + Ok(crate::tensor::from_storage(storage, out_dims, op, false)) + } + + /// Applies a 1D transposed convolution over the input tensor. + pub fn conv_transpose1d( + &self, + kernel: &Self, + padding: usize, + output_padding: usize, + stride: usize, + dilation: usize, + groups: usize, + ) -> Result { + let (c_in_k, c_out, k_size) = kernel.dims3()?; + let (b_size, c_in, l_in) = self.dims3()?; + if c_in != c_in_k { + crate::bail!("in_channel mismatch between input ({c_in}) and kernel ({c_in_k})") + } + if c_in % groups != 0 { + crate::bail!("in_channel {c_in} is not divisible by the number of groups") + } + let params = ParamsConvTranspose1D { + b_size, + l_in, + k_size, + c_out, + c_in: c_in / groups, + padding, + output_padding, + stride, + dilation, + }; + if groups == 1 { + self.conv_transpose1d_single_group(kernel, ¶ms) + } else { + let blocks = self.chunk(groups, 1)?; + let kernel = kernel.chunk(groups, 0)?; + let blocks = blocks + .iter() + .zip(&kernel) + .map(|(block, kernel)| block.conv_transpose1d_single_group(kernel, ¶ms)) + .collect::>>()?; + Tensor::cat(&blocks, 1) + } + } + + fn conv2d_single_group(&self, kernel: &Self, params: &ParamsConv2D) -> Result { + let storage = + self.storage() + .conv2d(self.layout(), &kernel.storage(), kernel.layout(), params)?; + let op = BackpropOp::new2(self, kernel, |arg, kernel| Op::Conv2D { + arg, + kernel, + padding: params.padding, + stride: params.stride, + dilation: params.dilation, + }); + let out_dims = params.out_dims(); + Ok(crate::tensor::from_storage(storage, out_dims, op, false)) + } + + /// Applies a 2D convolution over the input tensor. + pub fn conv2d( + &self, + kernel: &Self, + padding: usize, + stride: usize, + dilation: usize, + groups: usize, + ) -> Result { + let (b_size, c_in, i_h, i_w) = self.dims4()?; + let (c_out, c_in_k, k_h, k_w) = kernel.dims4()?; + if c_in != c_in_k * groups { + crate::bail!( + "in_channel mismatch between input ({c_in}, groups {groups}) and kernel ({c_in_k})" + ) + } + let params = ParamsConv2D { + b_size, + i_h, + i_w, + k_h, + k_w, + c_out: c_out / groups, + c_in: c_in / groups, + padding, + stride, + dilation, + cudnn_fwd_algo: None, + }; + if groups == 1 { + self.conv2d_single_group(kernel, ¶ms) + } else { + let blocks = self.chunk(groups, 1)?; + let kernel = kernel.chunk(groups, 0)?; + let blocks = blocks + .iter() + .zip(&kernel) + .map(|(block, kernel)| block.conv2d_single_group(kernel, ¶ms)) + .collect::>>()?; + Tensor::cat(&blocks, 1) + } + } + + /// Applies a 2D transposed convolution over the input tensor. + pub fn conv_transpose2d( + &self, + kernel: &Self, + padding: usize, + output_padding: usize, + stride: usize, + dilation: usize, + ) -> Result { + let (b_size, c_in, i_h, i_w) = self.dims4()?; + let (c_in_k, c_out, k_h, k_w) = kernel.dims4()?; + if c_in != c_in_k { + crate::bail!("in_channel mismatch between input ({c_in}) and kernel ({c_in_k})") + } + let params = ParamsConvTranspose2D { + b_size, + i_h, + i_w, + k_h, + k_w, + c_out, + c_in, + padding, + output_padding, + stride, + dilation, + }; + let storage = self.storage().conv_transpose2d( + self.layout(), + &kernel.storage(), + kernel.layout(), + ¶ms, + )?; + let op = BackpropOp::new2(self, kernel, |arg, kernel| Op::ConvTranspose2D { + arg, + kernel, + padding: params.padding, + output_padding: params.output_padding, + stride: params.stride, + dilation: params.dilation, + }); + let out_dims = params.out_dims(); + Ok(crate::tensor::from_storage(storage, out_dims, op, false)) + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/convert.rs b/v2/dependency-patches/candle-core-0.4.1/src/convert.rs new file mode 100644 index 0000000000..5ea5612a7c --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/convert.rs @@ -0,0 +1,145 @@ +//! Implement conversion traits for tensors +use crate::{DType, Device, Error, Tensor, WithDType}; +use half::{bf16, f16, slice::HalfFloatSliceExt}; +use std::convert::TryFrom; + +impl TryFrom<&Tensor> for Vec { + type Error = Error; + fn try_from(tensor: &Tensor) -> Result { + tensor.to_vec1::() + } +} + +impl TryFrom<&Tensor> for Vec> { + type Error = Error; + fn try_from(tensor: &Tensor) -> Result { + tensor.to_vec2::() + } +} + +impl TryFrom<&Tensor> for Vec>> { + type Error = Error; + fn try_from(tensor: &Tensor) -> Result { + tensor.to_vec3::() + } +} + +impl TryFrom for Vec { + type Error = Error; + fn try_from(tensor: Tensor) -> Result { + Vec::::try_from(&tensor) + } +} + +impl TryFrom for Vec> { + type Error = Error; + fn try_from(tensor: Tensor) -> Result { + Vec::>::try_from(&tensor) + } +} + +impl TryFrom for Vec>> { + type Error = Error; + fn try_from(tensor: Tensor) -> Result { + Vec::>>::try_from(&tensor) + } +} + +impl TryFrom<&[T]> for Tensor { + type Error = Error; + fn try_from(v: &[T]) -> Result { + Tensor::from_slice(v, v.len(), &Device::Cpu) + } +} + +impl TryFrom> for Tensor { + type Error = Error; + fn try_from(v: Vec) -> Result { + let len = v.len(); + Tensor::from_vec(v, len, &Device::Cpu) + } +} + +macro_rules! from_tensor { + ($typ:ident) => { + impl TryFrom<&Tensor> for $typ { + type Error = Error; + + fn try_from(tensor: &Tensor) -> Result { + tensor.to_scalar::<$typ>() + } + } + + impl TryFrom for $typ { + type Error = Error; + + fn try_from(tensor: Tensor) -> Result { + $typ::try_from(&tensor) + } + } + + impl TryFrom<$typ> for Tensor { + type Error = Error; + + fn try_from(v: $typ) -> Result { + Tensor::new(v, &Device::Cpu) + } + } + }; +} + +from_tensor!(f64); +from_tensor!(f32); +from_tensor!(f16); +from_tensor!(bf16); +from_tensor!(i64); +from_tensor!(u32); +from_tensor!(u8); + +impl Tensor { + pub fn write_bytes(&self, f: &mut W) -> crate::Result<()> { + use byteorder::{LittleEndian, WriteBytesExt}; + + let vs = self.flatten_all()?; + match self.dtype() { + DType::BF16 => { + let vs = vs.to_vec1::()?; + for &v in vs.reinterpret_cast() { + f.write_u16::(v)? + } + } + DType::F16 => { + let vs = vs.to_vec1::()?; + for &v in vs.reinterpret_cast() { + f.write_u16::(v)? + } + } + DType::F32 => { + // TODO: Avoid using a buffer when data is already on the CPU. + for v in vs.to_vec1::()? { + f.write_f32::(v)? + } + } + DType::F64 => { + for v in vs.to_vec1::()? { + f.write_f64::(v)? + } + } + DType::U32 => { + for v in vs.to_vec1::()? { + f.write_u32::(v)? + } + } + DType::I64 => { + for v in vs.to_vec1::()? { + f.write_i64::(v)? + } + } + DType::U8 => { + let vs = vs.to_vec1::()?; + f.write_all(&vs)?; + } + } + Ok(()) + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/cpu/avx.rs b/v2/dependency-patches/candle-core-0.4.1/src/cpu/avx.rs new file mode 100644 index 0000000000..9398a3460a --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/cpu/avx.rs @@ -0,0 +1,148 @@ +use super::{Cpu, CpuF16}; +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + +use half::f16; + +pub struct CurrentCpu {} + +const STEP: usize = 32; +const EPR: usize = 8; +const ARR: usize = STEP / EPR; + +impl Cpu for CurrentCpu { + type Unit = __m256; + type Array = [__m256; ARR]; + + const STEP: usize = STEP; + const EPR: usize = EPR; + + fn n() -> usize { + ARR + } + + unsafe fn zero() -> Self::Unit { + _mm256_setzero_ps() + } + + unsafe fn zero_array() -> Self::Array { + [Self::zero(); ARR] + } + + unsafe fn from_f32(v: f32) -> Self::Unit { + _mm256_set1_ps(v) + } + + unsafe fn load(mem_addr: *const f32) -> Self::Unit { + _mm256_loadu_ps(mem_addr) + } + + unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit { + _mm256_add_ps(a, b) + } + + unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit { + _mm256_add_ps(_mm256_mul_ps(b, c), a) + } + + unsafe fn vec_store(mem_addr: *mut f32, a: Self::Unit) { + _mm256_storeu_ps(mem_addr, a); + } + + unsafe fn vec_reduce(mut x: Self::Array, y: *mut f32) { + for i in 0..ARR / 2 { + x[2 * i] = _mm256_add_ps(x[2 * i], x[2 * i + 1]); + } + for i in 0..ARR / 4 { + x[4 * i] = _mm256_add_ps(x[4 * i], x[4 * i + 2]); + } + #[allow(clippy::reversed_empty_ranges)] + for i in 0..ARR / 8 { + x[8 * i] = _mm256_add_ps(x[8 * i], x[8 * i + 4]); + } + let t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]), _mm256_extractf128_ps(x[0], 1)); + let t1 = _mm_hadd_ps(t0, t0); + *y = _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); + } +} + +pub struct CurrentCpuF16 {} +impl CpuF16 for CurrentCpuF16 { + type Unit = __m256; + type Array = [__m256; ARR]; + + const STEP: usize = STEP; + const EPR: usize = EPR; + + fn n() -> usize { + ARR + } + + unsafe fn zero() -> Self::Unit { + _mm256_setzero_ps() + } + + unsafe fn zero_array() -> Self::Array { + [Self::zero(); ARR] + } + + unsafe fn from_f32(v: f32) -> Self::Unit { + _mm256_set1_ps(v) + } + + #[cfg(target_feature = "f16c")] + unsafe fn load(mem_addr: *const f16) -> Self::Unit { + _mm256_cvtph_ps(_mm_loadu_si128(mem_addr as *const __m128i)) + } + + #[cfg(not(target_feature = "f16c"))] + unsafe fn load(mem_addr: *const f16) -> Self::Unit { + let mut tmp = [0.0f32; 8]; + for i in 0..8 { + tmp[i] = (*mem_addr.add(i)).to_f32(); + } + _mm256_loadu_ps(tmp.as_ptr()) + } + + unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit { + _mm256_add_ps(a, b) + } + + unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit { + _mm256_add_ps(_mm256_mul_ps(b, c), a) + } + + #[cfg(target_feature = "f16c")] + unsafe fn vec_store(mem_addr: *mut f16, a: Self::Unit) { + _mm_storeu_si128(mem_addr as *mut __m128i, _mm256_cvtps_ph(a, 0)) + } + + #[cfg(not(target_feature = "f16c"))] + unsafe fn vec_store(mem_addr: *mut f16, a: Self::Unit) { + let mut tmp = [0.0f32; 8]; + _mm256_storeu_ps(tmp.as_mut_ptr(), a); + for i in 0..8 { + *mem_addr.add(i) = f16::from_f32(tmp[i]); + } + } + + unsafe fn vec_reduce(mut x: Self::Array, y: *mut f32) { + let mut offset = ARR >> 1; + for i in 0..offset { + x[i] = _mm256_add_ps(x[i], x[offset + i]); + } + offset >>= 1; + for i in 0..offset { + x[i] = _mm256_add_ps(x[i], x[offset + i]); + } + offset >>= 1; + for i in 0..offset { + x[i] = _mm256_add_ps(x[i], x[offset + i]); + } + let t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]), _mm256_extractf128_ps(x[0], 1)); + let t1 = _mm_hadd_ps(t0, t0); + *y = _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/cpu/erf.rs b/v2/dependency-patches/candle-core-0.4.1/src/cpu/erf.rs new file mode 100644 index 0000000000..ca6be53fd4 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/cpu/erf.rs @@ -0,0 +1,763 @@ +#![allow(clippy::excessive_precision)] +// Code taken from https://github.com/statrs-dev/statrs +//! Provides the [error](https://en.wikipedia.org/wiki/Error_function) and +//! related functions + +mod evaluate { + //! Provides functions that don't have a numerical solution and must + //! be solved computationally (e.g. evaluation of a polynomial) + + /// evaluates a polynomial at `z` where `coeff` are the coeffecients + /// to a polynomial of order `k` where `k` is the length of `coeff` and the + /// coeffecient + /// to the `k`th power is the `k`th element in coeff. E.g. [3,-1,2] equates to + /// `2z^2 - z + 3` + /// + /// # Remarks + /// + /// Returns 0 for a 0 length coefficient slice + pub fn polynomial(z: f64, coeff: &[f64]) -> f64 { + let n = coeff.len(); + if n == 0 { + return 0.0; + } + + let mut sum = *coeff.last().unwrap(); + for c in coeff[0..n - 1].iter().rev() { + sum = *c + z * sum; + } + sum + } +} +use std::f64; + +/// `erf` calculates the error function at `x`. +pub fn erf(x: f64) -> f64 { + if x.is_nan() { + f64::NAN + } else if x >= 0.0 && x.is_infinite() { + 1.0 + } else if x <= 0.0 && x.is_infinite() { + -1.0 + } else if x == 0. { + 0.0 + } else { + erf_impl(x, false) + } +} + +/// `erf_inv` calculates the inverse error function +/// at `x`. +pub fn erf_inv(x: f64) -> f64 { + if x == 0.0 { + 0.0 + } else if x >= 1.0 { + f64::INFINITY + } else if x <= -1.0 { + f64::NEG_INFINITY + } else if x < 0.0 { + erf_inv_impl(-x, 1.0 + x, -1.0) + } else { + erf_inv_impl(x, 1.0 - x, 1.0) + } +} + +/// `erfc` calculates the complementary error function +/// at `x`. +pub fn erfc(x: f64) -> f64 { + if x.is_nan() { + f64::NAN + } else if x == f64::INFINITY { + 0.0 + } else if x == f64::NEG_INFINITY { + 2.0 + } else { + erf_impl(x, true) + } +} + +/// `erfc_inv` calculates the complementary inverse +/// error function at `x`. +pub fn erfc_inv(x: f64) -> f64 { + if x <= 0.0 { + f64::INFINITY + } else if x >= 2.0 { + f64::NEG_INFINITY + } else if x > 1.0 { + erf_inv_impl(-1.0 + x, 2.0 - x, -1.0) + } else { + erf_inv_impl(1.0 - x, x, 1.0) + } +} + +// ********************************************************** +// ********** Coefficients for erf_impl polynomial ********** +// ********************************************************** + +/// Polynomial coefficients for a numerator of `erf_impl` +/// in the interval [1e-10, 0.5]. +const ERF_IMPL_AN: &[f64] = &[ + 0.00337916709551257388990745, + -0.00073695653048167948530905, + -0.374732337392919607868241, + 0.0817442448733587196071743, + -0.0421089319936548595203468, + 0.0070165709512095756344528, + -0.00495091255982435110337458, + 0.000871646599037922480317225, +]; + +/// Polynomial coefficients for a denominator of `erf_impl` +/// in the interval [1e-10, 0.5] +const ERF_IMPL_AD: &[f64] = &[ + 1.0, + -0.218088218087924645390535, + 0.412542972725442099083918, + -0.0841891147873106755410271, + 0.0655338856400241519690695, + -0.0120019604454941768171266, + 0.00408165558926174048329689, + -0.000615900721557769691924509, +]; + +/// Polynomial coefficients for a numerator in `erf_impl` +/// in the interval [0.5, 0.75]. +const ERF_IMPL_BN: &[f64] = &[ + -0.0361790390718262471360258, + 0.292251883444882683221149, + 0.281447041797604512774415, + 0.125610208862766947294894, + 0.0274135028268930549240776, + 0.00250839672168065762786937, +]; + +/// Polynomial coefficients for a denominator in `erf_impl` +/// in the interval [0.5, 0.75]. +const ERF_IMPL_BD: &[f64] = &[ + 1.0, + 1.8545005897903486499845, + 1.43575803037831418074962, + 0.582827658753036572454135, + 0.124810476932949746447682, + 0.0113724176546353285778481, +]; + +/// Polynomial coefficients for a numerator in `erf_impl` +/// in the interval [0.75, 1.25]. +const ERF_IMPL_CN: &[f64] = &[ + -0.0397876892611136856954425, + 0.153165212467878293257683, + 0.191260295600936245503129, + 0.10276327061989304213645, + 0.029637090615738836726027, + 0.0046093486780275489468812, + 0.000307607820348680180548455, +]; + +/// Polynomial coefficients for a denominator in `erf_impl` +/// in the interval [0.75, 1.25]. +const ERF_IMPL_CD: &[f64] = &[ + 1.0, + 1.95520072987627704987886, + 1.64762317199384860109595, + 0.768238607022126250082483, + 0.209793185936509782784315, + 0.0319569316899913392596356, + 0.00213363160895785378615014, +]; + +/// Polynomial coefficients for a numerator in `erf_impl` +/// in the interval [1.25, 2.25]. +const ERF_IMPL_DN: &[f64] = &[ + -0.0300838560557949717328341, + 0.0538578829844454508530552, + 0.0726211541651914182692959, + 0.0367628469888049348429018, + 0.00964629015572527529605267, + 0.00133453480075291076745275, + 0.778087599782504251917881e-4, +]; + +/// Polynomial coefficients for a denominator in `erf_impl` +/// in the interval [1.25, 2.25]. +const ERF_IMPL_DD: &[f64] = &[ + 1.0, + 1.75967098147167528287343, + 1.32883571437961120556307, + 0.552528596508757581287907, + 0.133793056941332861912279, + 0.0179509645176280768640766, + 0.00104712440019937356634038, + -0.106640381820357337177643e-7, +]; + +/// Polynomial coefficients for a numerator in `erf_impl` +/// in the interval [2.25, 3.5]. +const ERF_IMPL_EN: &[f64] = &[ + -0.0117907570137227847827732, + 0.014262132090538809896674, + 0.0202234435902960820020765, + 0.00930668299990432009042239, + 0.00213357802422065994322516, + 0.00025022987386460102395382, + 0.120534912219588189822126e-4, +]; + +/// Polynomial coefficients for a denominator in `erf_impl` +/// in the interval [2.25, 3.5]. +const ERF_IMPL_ED: &[f64] = &[ + 1.0, + 1.50376225203620482047419, + 0.965397786204462896346934, + 0.339265230476796681555511, + 0.0689740649541569716897427, + 0.00771060262491768307365526, + 0.000371421101531069302990367, +]; + +/// Polynomial coefficients for a numerator in `erf_impl` +/// in the interval [3.5, 5.25]. +const ERF_IMPL_FN: &[f64] = &[ + -0.00546954795538729307482955, + 0.00404190278731707110245394, + 0.0054963369553161170521356, + 0.00212616472603945399437862, + 0.000394984014495083900689956, + 0.365565477064442377259271e-4, + 0.135485897109932323253786e-5, +]; + +/// Polynomial coefficients for a denominator in `erf_impl` +/// in the interval [3.5, 5.25]. +const ERF_IMPL_FD: &[f64] = &[ + 1.0, + 1.21019697773630784832251, + 0.620914668221143886601045, + 0.173038430661142762569515, + 0.0276550813773432047594539, + 0.00240625974424309709745382, + 0.891811817251336577241006e-4, + -0.465528836283382684461025e-11, +]; + +/// Polynomial coefficients for a numerator in `erf_impl` +/// in the interval [5.25, 8]. +const ERF_IMPL_GN: &[f64] = &[ + -0.00270722535905778347999196, + 0.0013187563425029400461378, + 0.00119925933261002333923989, + 0.00027849619811344664248235, + 0.267822988218331849989363e-4, + 0.923043672315028197865066e-6, +]; + +/// Polynomial coefficients for a denominator in `erf_impl` +/// in the interval [5.25, 8]. +const ERF_IMPL_GD: &[f64] = &[ + 1.0, + 0.814632808543141591118279, + 0.268901665856299542168425, + 0.0449877216103041118694989, + 0.00381759663320248459168994, + 0.000131571897888596914350697, + 0.404815359675764138445257e-11, +]; + +/// Polynomial coefficients for a numerator in `erf_impl` +/// in the interval [8, 11.5]. +const ERF_IMPL_HN: &[f64] = &[ + -0.00109946720691742196814323, + 0.000406425442750422675169153, + 0.000274499489416900707787024, + 0.465293770646659383436343e-4, + 0.320955425395767463401993e-5, + 0.778286018145020892261936e-7, +]; + +/// Polynomial coefficients for a denominator in `erf_impl` +/// in the interval [8, 11.5]. +const ERF_IMPL_HD: &[f64] = &[ + 1.0, + 0.588173710611846046373373, + 0.139363331289409746077541, + 0.0166329340417083678763028, + 0.00100023921310234908642639, + 0.24254837521587225125068e-4, +]; + +/// Polynomial coefficients for a numerator in `erf_impl` +/// in the interval [11.5, 17]. +const ERF_IMPL_IN: &[f64] = &[ + -0.00056907993601094962855594, + 0.000169498540373762264416984, + 0.518472354581100890120501e-4, + 0.382819312231928859704678e-5, + 0.824989931281894431781794e-7, +]; + +/// Polynomial coefficients for a denominator in `erf_impl` +/// in the interval [11.5, 17]. +const ERF_IMPL_ID: &[f64] = &[ + 1.0, + 0.339637250051139347430323, + 0.043472647870310663055044, + 0.00248549335224637114641629, + 0.535633305337152900549536e-4, + -0.117490944405459578783846e-12, +]; + +/// Polynomial coefficients for a numerator in `erf_impl` +/// in the interval [17, 24]. +const ERF_IMPL_JN: &[f64] = &[ + -0.000241313599483991337479091, + 0.574224975202501512365975e-4, + 0.115998962927383778460557e-4, + 0.581762134402593739370875e-6, + 0.853971555085673614607418e-8, +]; + +/// Polynomial coefficients for a denominator in `erf_impl` +/// in the interval [17, 24]. +const ERF_IMPL_JD: &[f64] = &[ + 1.0, + 0.233044138299687841018015, + 0.0204186940546440312625597, + 0.000797185647564398289151125, + 0.117019281670172327758019e-4, +]; + +/// Polynomial coefficients for a numerator in `erf_impl` +/// in the interval [24, 38]. +const ERF_IMPL_KN: &[f64] = &[ + -0.000146674699277760365803642, + 0.162666552112280519955647e-4, + 0.269116248509165239294897e-5, + 0.979584479468091935086972e-7, + 0.101994647625723465722285e-8, +]; + +/// Polynomial coefficients for a denominator in `erf_impl` +/// in the interval [24, 38]. +const ERF_IMPL_KD: &[f64] = &[ + 1.0, + 0.165907812944847226546036, + 0.0103361716191505884359634, + 0.000286593026373868366935721, + 0.298401570840900340874568e-5, +]; + +/// Polynomial coefficients for a numerator in `erf_impl` +/// in the interval [38, 60]. +const ERF_IMPL_LN: &[f64] = &[ + -0.583905797629771786720406e-4, + 0.412510325105496173512992e-5, + 0.431790922420250949096906e-6, + 0.993365155590013193345569e-8, + 0.653480510020104699270084e-10, +]; + +/// Polynomial coefficients for a denominator in `erf_impl` +/// in the interval [38, 60]. +const ERF_IMPL_LD: &[f64] = &[ + 1.0, + 0.105077086072039915406159, + 0.00414278428675475620830226, + 0.726338754644523769144108e-4, + 0.477818471047398785369849e-6, +]; + +/// Polynomial coefficients for a numerator in `erf_impl` +/// in the interval [60, 85]. +const ERF_IMPL_MN: &[f64] = &[ + -0.196457797609229579459841e-4, + 0.157243887666800692441195e-5, + 0.543902511192700878690335e-7, + 0.317472492369117710852685e-9, +]; + +/// Polynomial coefficients for a denominator in `erf_impl` +/// in the interval [60, 85]. +const ERF_IMPL_MD: &[f64] = &[ + 1.0, + 0.052803989240957632204885, + 0.000926876069151753290378112, + 0.541011723226630257077328e-5, + 0.535093845803642394908747e-15, +]; + +/// Polynomial coefficients for a numerator in `erf_impl` +/// in the interval [85, 110]. +const ERF_IMPL_NN: &[f64] = &[ + -0.789224703978722689089794e-5, + 0.622088451660986955124162e-6, + 0.145728445676882396797184e-7, + 0.603715505542715364529243e-10, +]; + +/// Polynomial coefficients for a denominator in `erf_impl` +/// in the interval [85, 110]. +const ERF_IMPL_ND: &[f64] = &[ + 1.0, + 0.0375328846356293715248719, + 0.000467919535974625308126054, + 0.193847039275845656900547e-5, +]; + +// ********************************************************** +// ********** Coefficients for erf_inv_impl polynomial ****** +// ********************************************************** + +/// Polynomial coefficients for a numerator of `erf_inv_impl` +/// in the interval [0, 0.5]. +const ERF_INV_IMPL_AN: &[f64] = &[ + -0.000508781949658280665617, + -0.00836874819741736770379, + 0.0334806625409744615033, + -0.0126926147662974029034, + -0.0365637971411762664006, + 0.0219878681111168899165, + 0.00822687874676915743155, + -0.00538772965071242932965, +]; + +/// Polynomial coefficients for a denominator of `erf_inv_impl` +/// in the interval [0, 0.5]. +const ERF_INV_IMPL_AD: &[f64] = &[ + 1.0, + -0.970005043303290640362, + -1.56574558234175846809, + 1.56221558398423026363, + 0.662328840472002992063, + -0.71228902341542847553, + -0.0527396382340099713954, + 0.0795283687341571680018, + -0.00233393759374190016776, + 0.000886216390456424707504, +]; + +/// Polynomial coefficients for a numerator of `erf_inv_impl` +/// in the interval [0.5, 0.75]. +const ERF_INV_IMPL_BN: &[f64] = &[ + -0.202433508355938759655, + 0.105264680699391713268, + 8.37050328343119927838, + 17.6447298408374015486, + -18.8510648058714251895, + -44.6382324441786960818, + 17.445385985570866523, + 21.1294655448340526258, + -3.67192254707729348546, +]; + +/// Polynomial coefficients for a denominator of `erf_inv_impl` +/// in the interval [0.5, 0.75]. +const ERF_INV_IMPL_BD: &[f64] = &[ + 1.0, + 6.24264124854247537712, + 3.9713437953343869095, + -28.6608180499800029974, + -20.1432634680485188801, + 48.5609213108739935468, + 10.8268667355460159008, + -22.6436933413139721736, + 1.72114765761200282724, +]; + +/// Polynomial coefficients for a numerator of `erf_inv_impl` +/// in the interval [0.75, 1] with x less than 3. +const ERF_INV_IMPL_CN: &[f64] = &[ + -0.131102781679951906451, + -0.163794047193317060787, + 0.117030156341995252019, + 0.387079738972604337464, + 0.337785538912035898924, + 0.142869534408157156766, + 0.0290157910005329060432, + 0.00214558995388805277169, + -0.679465575181126350155e-6, + 0.285225331782217055858e-7, + -0.681149956853776992068e-9, +]; + +/// Polynomial coefficients for a denominator of `erf_inv_impl` +/// in the interval [0.75, 1] with x less than 3. +const ERF_INV_IMPL_CD: &[f64] = &[ + 1.0, + 3.46625407242567245975, + 5.38168345707006855425, + 4.77846592945843778382, + 2.59301921623620271374, + 0.848854343457902036425, + 0.152264338295331783612, + 0.01105924229346489121, +]; + +/// Polynomial coefficients for a numerator of `erf_inv_impl` +/// in the interval [0.75, 1] with x between 3 and 6. +const ERF_INV_IMPL_DN: &[f64] = &[ + -0.0350353787183177984712, + -0.00222426529213447927281, + 0.0185573306514231072324, + 0.00950804701325919603619, + 0.00187123492819559223345, + 0.000157544617424960554631, + 0.460469890584317994083e-5, + -0.230404776911882601748e-9, + 0.266339227425782031962e-11, +]; + +/// Polynomial coefficients for a denominator of `erf_inv_impl` +/// in the interval [0.75, 1] with x between 3 and 6. +const ERF_INV_IMPL_DD: &[f64] = &[ + 1.0, + 1.3653349817554063097, + 0.762059164553623404043, + 0.220091105764131249824, + 0.0341589143670947727934, + 0.00263861676657015992959, + 0.764675292302794483503e-4, +]; + +/// Polynomial coefficients for a numerator of `erf_inv_impl` +/// in the interval [0.75, 1] with x between 6 and 18. +const ERF_INV_IMPL_EN: &[f64] = &[ + -0.0167431005076633737133, + -0.00112951438745580278863, + 0.00105628862152492910091, + 0.000209386317487588078668, + 0.149624783758342370182e-4, + 0.449696789927706453732e-6, + 0.462596163522878599135e-8, + -0.281128735628831791805e-13, + 0.99055709973310326855e-16, +]; + +/// Polynomial coefficients for a denominator of `erf_inv_impl` +/// in the interval [0.75, 1] with x between 6 and 18. +const ERF_INV_IMPL_ED: &[f64] = &[ + 1.0, + 0.591429344886417493481, + 0.138151865749083321638, + 0.0160746087093676504695, + 0.000964011807005165528527, + 0.275335474764726041141e-4, + 0.282243172016108031869e-6, +]; + +/// Polynomial coefficients for a numerator of `erf_inv_impl` +/// in the interval [0.75, 1] with x between 18 and 44. +const ERF_INV_IMPL_FN: &[f64] = &[ + -0.0024978212791898131227, + -0.779190719229053954292e-5, + 0.254723037413027451751e-4, + 0.162397777342510920873e-5, + 0.396341011304801168516e-7, + 0.411632831190944208473e-9, + 0.145596286718675035587e-11, + -0.116765012397184275695e-17, +]; + +/// Polynomial coefficients for a denominator of `erf_inv_impl` +/// in the interval [0.75, 1] with x between 18 and 44. +const ERF_INV_IMPL_FD: &[f64] = &[ + 1.0, + 0.207123112214422517181, + 0.0169410838120975906478, + 0.000690538265622684595676, + 0.145007359818232637924e-4, + 0.144437756628144157666e-6, + 0.509761276599778486139e-9, +]; + +/// Polynomial coefficients for a numerator of `erf_inv_impl` +/// in the interval [0.75, 1] with x greater than 44. +const ERF_INV_IMPL_GN: &[f64] = &[ + -0.000539042911019078575891, + -0.28398759004727721098e-6, + 0.899465114892291446442e-6, + 0.229345859265920864296e-7, + 0.225561444863500149219e-9, + 0.947846627503022684216e-12, + 0.135880130108924861008e-14, + -0.348890393399948882918e-21, +]; + +/// Polynomial coefficients for a denominator of `erf_inv_impl` +/// in the interval [0.75, 1] with x greater than 44. +const ERF_INV_IMPL_GD: &[f64] = &[ + 1.0, + 0.0845746234001899436914, + 0.00282092984726264681981, + 0.468292921940894236786e-4, + 0.399968812193862100054e-6, + 0.161809290887904476097e-8, + 0.231558608310259605225e-11, +]; + +/// `erf_impl` computes the error function at `z`. +/// If `inv` is true, `1 - erf` is calculated as opposed to `erf` +fn erf_impl(z: f64, inv: bool) -> f64 { + if z < 0.0 { + if !inv { + return -erf_impl(-z, false); + } + if z < -0.5 { + return 2.0 - erf_impl(-z, true); + } + return 1.0 + erf_impl(-z, false); + } + + let result = if z < 0.5 { + if z < 1e-10 { + z * 1.125 + z * 0.003379167095512573896158903121545171688 + } else { + z * 1.125 + + z * evaluate::polynomial(z, ERF_IMPL_AN) / evaluate::polynomial(z, ERF_IMPL_AD) + } + } else if z < 110.0 { + let (r, b) = if z < 0.75 { + ( + evaluate::polynomial(z - 0.5, ERF_IMPL_BN) + / evaluate::polynomial(z - 0.5, ERF_IMPL_BD), + 0.3440242112, + ) + } else if z < 1.25 { + ( + evaluate::polynomial(z - 0.75, ERF_IMPL_CN) + / evaluate::polynomial(z - 0.75, ERF_IMPL_CD), + 0.419990927, + ) + } else if z < 2.25 { + ( + evaluate::polynomial(z - 1.25, ERF_IMPL_DN) + / evaluate::polynomial(z - 1.25, ERF_IMPL_DD), + 0.4898625016, + ) + } else if z < 3.5 { + ( + evaluate::polynomial(z - 2.25, ERF_IMPL_EN) + / evaluate::polynomial(z - 2.25, ERF_IMPL_ED), + 0.5317370892, + ) + } else if z < 5.25 { + ( + evaluate::polynomial(z - 3.5, ERF_IMPL_FN) + / evaluate::polynomial(z - 3.5, ERF_IMPL_FD), + 0.5489973426, + ) + } else if z < 8.0 { + ( + evaluate::polynomial(z - 5.25, ERF_IMPL_GN) + / evaluate::polynomial(z - 5.25, ERF_IMPL_GD), + 0.5571740866, + ) + } else if z < 11.5 { + ( + evaluate::polynomial(z - 8.0, ERF_IMPL_HN) + / evaluate::polynomial(z - 8.0, ERF_IMPL_HD), + 0.5609807968, + ) + } else if z < 17.0 { + ( + evaluate::polynomial(z - 11.5, ERF_IMPL_IN) + / evaluate::polynomial(z - 11.5, ERF_IMPL_ID), + 0.5626493692, + ) + } else if z < 24.0 { + ( + evaluate::polynomial(z - 17.0, ERF_IMPL_JN) + / evaluate::polynomial(z - 17.0, ERF_IMPL_JD), + 0.5634598136, + ) + } else if z < 38.0 { + ( + evaluate::polynomial(z - 24.0, ERF_IMPL_KN) + / evaluate::polynomial(z - 24.0, ERF_IMPL_KD), + 0.5638477802, + ) + } else if z < 60.0 { + ( + evaluate::polynomial(z - 38.0, ERF_IMPL_LN) + / evaluate::polynomial(z - 38.0, ERF_IMPL_LD), + 0.5640528202, + ) + } else if z < 85.0 { + ( + evaluate::polynomial(z - 60.0, ERF_IMPL_MN) + / evaluate::polynomial(z - 60.0, ERF_IMPL_MD), + 0.5641309023, + ) + } else { + ( + evaluate::polynomial(z - 85.0, ERF_IMPL_NN) + / evaluate::polynomial(z - 85.0, ERF_IMPL_ND), + 0.5641584396, + ) + }; + let g = (-z * z).exp() / z; + g * b + g * r + } else { + 0.0 + }; + + if inv && z >= 0.5 { + result + } else if z >= 0.5 || inv { + 1.0 - result + } else { + result + } +} + +// `erf_inv_impl` computes the inverse error function where +// `p`,`q`, and `s` are the first, second, and third intermediate +// parameters respectively +fn erf_inv_impl(p: f64, q: f64, s: f64) -> f64 { + let result = if p <= 0.5 { + let y = 0.0891314744949340820313; + let g = p * (p + 10.0); + let r = evaluate::polynomial(p, ERF_INV_IMPL_AN) / evaluate::polynomial(p, ERF_INV_IMPL_AD); + g * y + g * r + } else if q >= 0.25 { + let y = 2.249481201171875; + let g = (-2.0 * q.ln()).sqrt(); + let xs = q - 0.25; + let r = + evaluate::polynomial(xs, ERF_INV_IMPL_BN) / evaluate::polynomial(xs, ERF_INV_IMPL_BD); + g / (y + r) + } else { + let x = (-q.ln()).sqrt(); + if x < 3.0 { + let y = 0.807220458984375; + let xs = x - 1.125; + let r = evaluate::polynomial(xs, ERF_INV_IMPL_CN) + / evaluate::polynomial(xs, ERF_INV_IMPL_CD); + y * x + r * x + } else if x < 6.0 { + let y = 0.93995571136474609375; + let xs = x - 3.0; + let r = evaluate::polynomial(xs, ERF_INV_IMPL_DN) + / evaluate::polynomial(xs, ERF_INV_IMPL_DD); + y * x + r * x + } else if x < 18.0 { + let y = 0.98362827301025390625; + let xs = x - 6.0; + let r = evaluate::polynomial(xs, ERF_INV_IMPL_EN) + / evaluate::polynomial(xs, ERF_INV_IMPL_ED); + y * x + r * x + } else if x < 44.0 { + let y = 0.99714565277099609375; + let xs = x - 18.0; + let r = evaluate::polynomial(xs, ERF_INV_IMPL_FN) + / evaluate::polynomial(xs, ERF_INV_IMPL_FD); + y * x + r * x + } else { + let y = 0.99941349029541015625; + let xs = x - 44.0; + let r = evaluate::polynomial(xs, ERF_INV_IMPL_GN) + / evaluate::polynomial(xs, ERF_INV_IMPL_GD); + y * x + r * x + } + }; + s * result +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/cpu/kernels.rs b/v2/dependency-patches/candle-core-0.4.1/src/cpu/kernels.rs new file mode 100644 index 0000000000..527646d62b --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/cpu/kernels.rs @@ -0,0 +1,191 @@ +pub trait VecOps: num_traits::NumAssign + Copy { + fn min(self, rhs: Self) -> Self; + fn max(self, rhs: Self) -> Self; + + /// Dot-product of two vectors. + /// + /// # Safety + /// + /// The length of `lhs` and `rhs` have to be at least `len`. `res` has to point to a valid + /// element. + #[inline(always)] + unsafe fn vec_dot(lhs: *const Self, rhs: *const Self, res: *mut Self, len: usize) { + *res = Self::zero(); + for i in 0..len { + *res += *lhs.add(i) * *rhs.add(i) + } + } + + /// Sum of all elements in a vector. + /// + /// # Safety + /// + /// The length of `xs` must be at least `len`. `res` has to point to a valid + /// element. + #[inline(always)] + unsafe fn vec_reduce_sum(xs: *const Self, res: *mut Self, len: usize) { + *res = Self::zero(); + for i in 0..len { + *res += *xs.add(i) + } + } + + /// Maximum element in a non-empty vector. + /// + /// # Safety + /// + /// The length of `xs` must be at least `len` and positive. `res` has to point to a valid + /// element. + #[inline(always)] + unsafe fn vec_reduce_max(xs: *const Self, res: *mut Self, len: usize) { + *res = *xs; + for i in 1..len { + *res = (*res).max(*xs.add(i)) + } + } + + /// Minimum element in a non-empty vector. + /// + /// # Safety + /// + /// The length of `xs` must be at least `len` and positive. `res` has to point to a valid + /// element. + #[inline(always)] + unsafe fn vec_reduce_min(xs: *const Self, res: *mut Self, len: usize) { + *res = *xs; + for i in 1..len { + *res = (*res).min(*xs.add(i)) + } + } +} + +impl VecOps for f32 { + #[inline(always)] + fn min(self, other: Self) -> Self { + Self::min(self, other) + } + + #[inline(always)] + fn max(self, other: Self) -> Self { + Self::max(self, other) + } + + #[inline(always)] + unsafe fn vec_dot(lhs: *const Self, rhs: *const Self, res: *mut Self, len: usize) { + super::vec_dot_f32(lhs, rhs, res, len) + } + + #[inline(always)] + unsafe fn vec_reduce_sum(xs: *const Self, res: *mut Self, len: usize) { + super::vec_sum(xs, res, len) + } +} + +impl VecOps for half::f16 { + #[inline(always)] + fn min(self, other: Self) -> Self { + Self::min(self, other) + } + + #[inline(always)] + fn max(self, other: Self) -> Self { + Self::max(self, other) + } + + #[inline(always)] + unsafe fn vec_dot(lhs: *const Self, rhs: *const Self, res: *mut Self, len: usize) { + let mut res_f32 = 0f32; + super::vec_dot_f16(lhs, rhs, &mut res_f32, len); + *res = half::f16::from_f32(res_f32); + } +} + +impl VecOps for f64 { + #[inline(always)] + fn min(self, other: Self) -> Self { + Self::min(self, other) + } + + #[inline(always)] + fn max(self, other: Self) -> Self { + Self::max(self, other) + } +} +impl VecOps for half::bf16 { + #[inline(always)] + fn min(self, other: Self) -> Self { + Self::min(self, other) + } + + #[inline(always)] + fn max(self, other: Self) -> Self { + Self::max(self, other) + } +} +impl VecOps for u8 { + #[inline(always)] + fn min(self, other: Self) -> Self { + ::min(self, other) + } + + #[inline(always)] + fn max(self, other: Self) -> Self { + ::max(self, other) + } +} +impl VecOps for u32 { + #[inline(always)] + fn min(self, other: Self) -> Self { + ::min(self, other) + } + + #[inline(always)] + fn max(self, other: Self) -> Self { + ::max(self, other) + } +} +impl VecOps for i64 { + #[inline(always)] + fn min(self, other: Self) -> Self { + ::min(self, other) + } + + #[inline(always)] + fn max(self, other: Self) -> Self { + ::max(self, other) + } +} + +#[inline(always)] +pub fn par_for_each(n_threads: usize, func: impl Fn(usize) + Send + Sync) { + if n_threads == 1 { + func(0) + } else { + rayon::scope(|s| { + for thread_idx in 0..n_threads { + let func = &func; + s.spawn(move |_| func(thread_idx)); + } + }) + } +} + +#[inline(always)] +pub fn par_range(lo: usize, up: usize, n_threads: usize, func: impl Fn(usize) + Send + Sync) { + if n_threads == 1 { + for i in lo..up { + func(i) + } + } else { + rayon::scope(|s| { + for thread_idx in 0..n_threads { + let func = &func; + s.spawn(move |_| { + for i in (thread_idx..up).step_by(n_threads) { + func(i) + } + }); + } + }) + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/cpu/mod.rs b/v2/dependency-patches/candle-core-0.4.1/src/cpu/mod.rs new file mode 100644 index 0000000000..50afb30fe6 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/cpu/mod.rs @@ -0,0 +1,180 @@ +pub mod erf; +pub mod kernels; + +trait Cpu { + type Unit; + type Array; + const STEP: usize; + const EPR: usize; + + fn n() -> usize; + unsafe fn zero() -> Self::Unit; + unsafe fn zero_array() -> Self::Array; + unsafe fn load(mem_addr: *const f32) -> Self::Unit; + unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit; + unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit; + unsafe fn vec_reduce(x: Self::Array, y: *mut f32); + unsafe fn from_f32(v: f32) -> Self::Unit; + unsafe fn vec_store(mem_addr: *mut f32, a: Self::Unit); +} + +trait CpuF16 { + type Unit; + type Array; + const STEP: usize; + const EPR: usize; + + fn n() -> usize; + unsafe fn zero() -> Self::Unit; + unsafe fn zero_array() -> Self::Array; + unsafe fn load(mem_addr: *const f16) -> Self::Unit; + unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit; + unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit; + unsafe fn vec_reduce(x: Self::Array, y: *mut f32); + unsafe fn from_f32(v: f32) -> Self::Unit; + unsafe fn vec_store(mem_addr: *mut f16, a: Self::Unit); +} +use half::f16; + +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[cfg(target_feature = "avx")] +pub mod avx; +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[cfg(target_feature = "avx")] +pub use avx::{CurrentCpu, CurrentCpuF16}; + +#[cfg(target_arch = "wasm32")] +#[cfg(target_feature = "simd128")] +pub mod simd128; +#[cfg(target_arch = "wasm32")] +#[cfg(target_feature = "simd128")] +pub use simd128::CurrentCpu; + +#[cfg(any(target_arch = "arm", target_arch = "aarch64"))] +#[cfg(target_feature = "neon")] +pub mod neon; +#[cfg(any(target_arch = "arm", target_arch = "aarch64"))] +#[cfg(target_feature = "neon")] +pub use neon::CurrentCpu; + +#[cfg(any( + target_feature = "neon", + target_feature = "avx", + target_feature = "simd128" +))] +#[inline(always)] +pub(crate) unsafe fn vec_dot_f32(a_row: *const f32, b_row: *const f32, c: *mut f32, k: usize) { + let np = k & !(CurrentCpu::STEP - 1); + + let mut sum = CurrentCpu::zero_array(); + let mut ax = CurrentCpu::zero_array(); + let mut ay = CurrentCpu::zero_array(); + + for i in (0..np).step_by(CurrentCpu::STEP) { + for j in 0..CurrentCpu::n() { + ax[j] = CurrentCpu::load(a_row.add(i + j * CurrentCpu::EPR)); + ay[j] = CurrentCpu::load(b_row.add(i + j * CurrentCpu::EPR)); + + sum[j] = CurrentCpu::vec_fma(sum[j], ax[j], ay[j]); + } + } + + CurrentCpu::vec_reduce(sum, c); + + // leftovers + for i in np..k { + *c += *a_row.add(i) * (*b_row.add(i)); + } +} + +#[cfg(not(any( + target_feature = "neon", + target_feature = "avx", + target_feature = "simd128" +)))] +#[inline(always)] +pub(crate) unsafe fn vec_dot_f32(a_row: *const f32, b_row: *const f32, c: *mut f32, k: usize) { + // leftovers + for i in 0..k { + *c += *a_row.add(i) * (*b_row.add(i)); + } +} + +#[cfg(any( + target_feature = "neon", + target_feature = "avx", + target_feature = "simd128" +))] +#[inline(always)] +pub(crate) unsafe fn vec_sum(row: *const f32, b: *mut f32, k: usize) { + let np = k & !(CurrentCpu::STEP - 1); + + let mut sum = CurrentCpu::zero_array(); + let mut x = CurrentCpu::zero_array(); + + for i in (0..np).step_by(CurrentCpu::STEP) { + for j in 0..CurrentCpu::n() { + x[j] = CurrentCpu::load(row.add(i + j * CurrentCpu::EPR)); + sum[j] = CurrentCpu::vec_add(sum[j], x[j]); + } + } + + CurrentCpu::vec_reduce(sum, b); + + // leftovers + for i in np..k { + *b += *row.add(i) + } +} + +#[cfg(not(any( + target_feature = "neon", + target_feature = "avx", + target_feature = "simd128" +)))] +#[inline(always)] +pub(crate) unsafe fn vec_sum(row: *const f32, b: *mut f32, k: usize) { + *b = 0f32; + for i in 0..k { + *b += *row.add(i) + } +} + +#[cfg(target_feature = "avx")] +#[inline(always)] +pub(crate) unsafe fn vec_dot_f16(a_row: *const f16, b_row: *const f16, c: *mut f32, k: usize) { + let mut sumf = 0.0f32; + let np = k & !(CurrentCpuF16::STEP - 1); + + let mut sum = CurrentCpuF16::zero_array(); + let mut ax = CurrentCpuF16::zero_array(); + let mut ay = CurrentCpuF16::zero_array(); + + for i in (0..np).step_by(CurrentCpuF16::STEP) { + for j in 0..CurrentCpuF16::n() { + ax[j] = CurrentCpuF16::load(a_row.add(i + j * CurrentCpuF16::EPR)); + ay[j] = CurrentCpuF16::load(b_row.add(i + j * CurrentCpuF16::EPR)); + + sum[j] = CurrentCpuF16::vec_fma(sum[j], ax[j], ay[j]); + } + } + + CurrentCpuF16::vec_reduce(sum, &mut sumf); + + // leftovers + for i in np..k { + sumf += (*a_row.add(i)).to_f32() * (*b_row.add(i)).to_f32(); + } + *c = sumf; +} + +#[cfg(not(target_feature = "avx"))] +#[inline(always)] +pub(crate) unsafe fn vec_dot_f16(a_row: *const f16, b_row: *const f16, c: *mut f32, k: usize) { + // leftovers + let mut sum = 0.0; + for i in 0..k { + sum += (*a_row.add(i)).to_f32() * (*b_row.add(i)).to_f32(); + } + *c = sum; +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/cpu/neon.rs b/v2/dependency-patches/candle-core-0.4.1/src/cpu/neon.rs new file mode 100644 index 0000000000..66b8b45e15 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/cpu/neon.rs @@ -0,0 +1,74 @@ +use super::Cpu; +#[cfg(target_arch = "arm")] +use core::arch::arm::*; + +#[cfg(target_arch = "aarch64")] +use core::arch::aarch64::*; + +pub struct CurrentCpu {} + +const STEP: usize = 16; +const EPR: usize = 4; +const ARR: usize = STEP / EPR; + +impl CurrentCpu { + #[cfg(target_arch = "aarch64")] + unsafe fn reduce_one(x: float32x4_t) -> f32 { + vaddvq_f32(x) + } + + #[cfg(target_arch = "arm")] + unsafe fn reduce_one(x: float32x4_t) -> f32 { + vgetq_lane_f32(x, 0) + vgetq_lane_f32(x, 1) + vgetq_lane_f32(x, 2) + vgetq_lane_f32(x, 3) + } +} + +impl Cpu for CurrentCpu { + type Unit = float32x4_t; + type Array = [float32x4_t; ARR]; + + const STEP: usize = STEP; + const EPR: usize = EPR; + + fn n() -> usize { + ARR + } + + unsafe fn zero() -> Self::Unit { + vdupq_n_f32(0.0) + } + + unsafe fn from_f32(x: f32) -> Self::Unit { + vdupq_n_f32(x) + } + + unsafe fn zero_array() -> Self::Array { + [Self::zero(); ARR] + } + + unsafe fn load(mem_addr: *const f32) -> Self::Unit { + vld1q_f32(mem_addr) + } + + unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit { + vaddq_f32(a, b) + } + + unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit { + vfmaq_f32(a, b, c) + } + + unsafe fn vec_store(mem_addr: *mut f32, a: Self::Unit) { + vst1q_f32(mem_addr, a); + } + + unsafe fn vec_reduce(mut x: Self::Array, y: *mut f32) { + for i in 0..ARR / 2 { + x[2 * i] = vaddq_f32(x[2 * i], x[2 * i + 1]); + } + for i in 0..ARR / 4 { + x[4 * i] = vaddq_f32(x[4 * i], x[4 * i + 2]); + } + *y = Self::reduce_one(x[0]); + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/cpu/simd128.rs b/v2/dependency-patches/candle-core-0.4.1/src/cpu/simd128.rs new file mode 100644 index 0000000000..64b94588bd --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/cpu/simd128.rs @@ -0,0 +1,64 @@ +use super::Cpu; +use core::arch::wasm32::*; + +pub struct CurrentCpu {} + +const STEP: usize = 16; +const EPR: usize = 4; +const ARR: usize = STEP / EPR; + +impl Cpu for CurrentCpu { + type Unit = v128; + type Array = [v128; ARR]; + + const STEP: usize = STEP; + const EPR: usize = EPR; + + fn n() -> usize { + ARR + } + + unsafe fn zero() -> Self::Unit { + f32x4_splat(0.0) + } + + unsafe fn zero_array() -> Self::Array { + [Self::zero(); ARR] + } + + unsafe fn from_f32(v: f32) -> Self::Unit { + f32x4_splat(v) + } + + unsafe fn load(mem_addr: *const f32) -> Self::Unit { + v128_load(mem_addr as *mut v128) + } + + unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit { + f32x4_add(a, b) + } + + unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit { + f32x4_add(f32x4_mul(b, c), a) + } + + unsafe fn vec_store(mem_addr: *mut f32, a: Self::Unit) { + v128_store(mem_addr as *mut v128, a); + } + + unsafe fn vec_reduce(mut x: Self::Array, y: *mut f32) { + for i in 0..ARR / 2 { + x[2 * i] = f32x4_add(x[2 * i], x[2 * i + 1]); + } + for i in 0..ARR / 4 { + x[4 * i] = f32x4_add(x[4 * i], x[4 * i + 2]); + } + for i in 0..ARR / 8 { + x[8 * i] = f32x4_add(x[8 * i], x[8 * i + 4]); + } + *y = f32x4_extract_lane::<0>(x[0]) + + f32x4_extract_lane::<1>(x[0]) + + f32x4_extract_lane::<2>(x[0]) + + f32x4_extract_lane::<3>(x[0]); + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/cpu_backend.rs b/v2/dependency-patches/candle-core-0.4.1/src/cpu_backend.rs new file mode 100644 index 0000000000..8ae39020fd --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/cpu_backend.rs @@ -0,0 +1,2819 @@ +use crate::backend::{BackendDevice, BackendStorage}; +use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; +use crate::{DType, Error, IntDType, Layout, Result, Shape, WithDType}; +use half::{bf16, f16}; +use rayon::prelude::*; + +const USE_IM2COL_CONV1D: bool = true; +const USE_IM2COL_CONV2D: bool = true; + +// TODO: Maybe we should not implement [Clone] here and instead have an explicit allocator + +// intercept the oom errors to avoid panicking and provide a proper error. +#[derive(Debug, Clone)] +pub enum CpuStorage { + U8(Vec), + U32(Vec), + I64(Vec), + BF16(Vec), + F16(Vec), + F32(Vec), + F64(Vec), +} + +#[derive(Debug, Clone)] +pub struct CpuDevice; + +pub trait Map1 { + fn f(&self, vs: &[T], layout: &Layout) -> Result>; + + fn map(&self, vs: &CpuStorage, layout: &Layout) -> Result { + match vs { + CpuStorage::U8(vs) => Ok(CpuStorage::U8(self.f(vs, layout)?)), + CpuStorage::U32(vs) => Ok(CpuStorage::U32(self.f(vs, layout)?)), + CpuStorage::I64(vs) => Ok(CpuStorage::I64(self.f(vs, layout)?)), + CpuStorage::BF16(vs) => Ok(CpuStorage::BF16(self.f(vs, layout)?)), + CpuStorage::F16(vs) => Ok(CpuStorage::F16(self.f(vs, layout)?)), + CpuStorage::F32(vs) => Ok(CpuStorage::F32(self.f(vs, layout)?)), + CpuStorage::F64(vs) => Ok(CpuStorage::F64(self.f(vs, layout)?)), + } + } +} + +pub trait Map1Any { + fn f) -> CpuStorage>( + &self, + vs: &[T], + layout: &Layout, + wrap: W, + ) -> Result; + + fn map(&self, vs: &CpuStorage, layout: &Layout) -> Result { + match vs { + CpuStorage::U8(vs) => Ok(self.f(vs, layout, CpuStorage::U8)?), + CpuStorage::U32(vs) => Ok(self.f(vs, layout, CpuStorage::U32)?), + CpuStorage::I64(vs) => Ok(self.f(vs, layout, CpuStorage::I64)?), + CpuStorage::BF16(vs) => Ok(self.f(vs, layout, CpuStorage::BF16)?), + CpuStorage::F16(vs) => Ok(self.f(vs, layout, CpuStorage::F16)?), + CpuStorage::F32(vs) => Ok(self.f(vs, layout, CpuStorage::F32)?), + CpuStorage::F64(vs) => Ok(self.f(vs, layout, CpuStorage::F64)?), + } + } +} + +type C = CpuStorage; +pub trait Map2 { + const OP: &'static str; + fn f(&self, v1: &[T], l1: &Layout, v2: &[T], l2: &Layout) -> Result>; + + fn map( + &self, + v1: &CpuStorage, + l1: &Layout, + v2: &CpuStorage, + l2: &Layout, + ) -> Result { + match (v1, v2) { + (C::U8(v1), C::U8(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), + (C::U32(v1), C::U32(v2)) => Ok(C::U32(self.f(v1, l1, v2, l2)?)), + (C::I64(v1), C::I64(v2)) => Ok(C::I64(self.f(v1, l1, v2, l2)?)), + (C::BF16(v1), C::BF16(v2)) => Ok(C::BF16(self.f(v1, l1, v2, l2)?)), + (C::F16(v1), C::F16(v2)) => Ok(C::F16(self.f(v1, l1, v2, l2)?)), + (C::F32(v1), C::F32(v2)) => Ok(C::F32(self.f(v1, l1, v2, l2)?)), + (C::F64(v1), C::F64(v2)) => Ok(C::F64(self.f(v1, l1, v2, l2)?)), + _ => Err(Error::DTypeMismatchBinaryOp { + lhs: v1.dtype(), + rhs: v2.dtype(), + op: Self::OP, + } + .bt()), + } + } +} + +pub trait Map2U8 { + const OP: &'static str; + fn f(&self, v1: &[T], l1: &Layout, v2: &[T], l2: &Layout) -> Result>; + + fn map( + &self, + v1: &CpuStorage, + l1: &Layout, + v2: &CpuStorage, + l2: &Layout, + ) -> Result { + match (v1, v2) { + (C::U8(v1), C::U8(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), + (C::U32(v1), C::U32(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), + (C::I64(v1), C::I64(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), + (C::BF16(v1), C::BF16(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), + (C::F16(v1), C::F16(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), + (C::F32(v1), C::F32(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), + (C::F64(v1), C::F64(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), + _ => Err(Error::DTypeMismatchBinaryOp { + lhs: v1.dtype(), + rhs: v2.dtype(), + op: Self::OP, + } + .bt()), + } + } +} + +struct Cmp(CmpOp); +impl Map2U8 for Cmp { + const OP: &'static str = "cmp"; + #[inline(always)] + fn f( + &self, + lhs: &[T], + lhs_l: &Layout, + rhs: &[T], + rhs_l: &Layout, + ) -> Result> { + let dst = match self.0 { + CmpOp::Eq => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x == y)), + CmpOp::Ne => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x != y)), + CmpOp::Lt => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x < y)), + CmpOp::Le => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x <= y)), + CmpOp::Gt => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x > y)), + CmpOp::Ge => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x >= y)), + }; + Ok(dst) + } +} + +struct WCond<'a, T: IntDType>(&'a [T], &'a Layout); + +impl<'a, I: IntDType> Map2 for WCond<'a, I> { + const OP: &'static str = "where"; + #[inline(always)] + fn f(&self, t: &[T], t_l: &Layout, f: &[T], f_l: &Layout) -> Result> { + let vs = match ( + self.1.contiguous_offsets(), + t_l.contiguous_offsets(), + f_l.contiguous_offsets(), + ) { + (Some((o1, o2)), Some((o_t1, o_t2)), Some((o_f1, o_f2))) => { + let pred = &self.0[o1..o2]; + let t = &t[o_t1..o_t2]; + let f = &f[o_f1..o_f2]; + pred.iter() + .zip(t.iter().zip(f.iter())) + .map(|(p, (&t, &f))| if p.is_true() { t } else { f }) + .collect::>() + } + _ => self + .1 + .strided_index() + .zip(t_l.strided_index().zip(f_l.strided_index())) + .map(|(i_p, (i_t, i_f))| { + if self.0[i_p].is_true() { + t[i_t] + } else { + f[i_f] + } + }) + .collect::>(), + }; + Ok(vs) + } +} + +struct ReduceIndex { + reduce_dim_index: usize, + use_min: bool, + return_index: bool, +} + +impl ReduceIndex { + // The value gets replaced if f(s[current_acc], s[i]) returns true. + #[inline(always)] + fn fold_impl(&self, src: &[T], src_l: &Layout, f: F, g: G) -> Result> + where + T: Clone + Copy, + U: Clone + Copy, + F: Fn(T, T) -> bool, + G: Fn(T, usize) -> U, + { + let reduce_dim_size = src_l.dims()[self.reduce_dim_index]; + let reduce_dim_stride = src_l.stride()[self.reduce_dim_index]; + let dst_len = src_l.shape().elem_count() / reduce_dim_size; + let mut dst: Vec = Vec::with_capacity(dst_len); + let dst_to_set = dst.spare_capacity_mut(); + let dst_to_set = unsafe { std::mem::transmute::<_, &mut [U]>(dst_to_set) }; + match src_l.contiguous_offsets() { + Some((o1, o2)) => { + let src = &src[o1..o2]; + if reduce_dim_stride == 1 { + for (start_src_i, dst_v) in dst_to_set.iter_mut().enumerate() { + let start_src_i = start_src_i * reduce_dim_size; + let src = &src[start_src_i..start_src_i + reduce_dim_size]; + let mut acc = 0; + let mut val = src[0]; + for (src_i, &s) in src.iter().enumerate() { + if f(val, s) { + acc = src_i; + val = s + } + } + *dst_v = g(val, acc) + } + } else { + for (start_src_i, dst_v) in dst_to_set.iter_mut().enumerate() { + let (p, q) = ( + start_src_i / reduce_dim_stride, + start_src_i % reduce_dim_stride, + ); + // start_src_i = p * reduce_dim_stride + q + let start_src_i = p * reduce_dim_stride * reduce_dim_size + q; + let src = &src[start_src_i..]; + let mut acc = 0; + let mut val = src[0]; + for src_i in 0..reduce_dim_size { + let s = src[src_i * reduce_dim_stride]; + if f(val, s) { + acc = src_i; + val = s + } + } + *dst_v = g(val, acc) + } + } + } + None => { + let l = src_l.narrow(self.reduce_dim_index, 0, 1)?; + for (unstr_index, src_index) in l.strided_index().enumerate() { + let src = &src[src_index..]; + let mut acc = 0; + let mut val = src[0]; + for src_i in 0..reduce_dim_size { + let s = src[src_i * reduce_dim_stride]; + if f(val, s) { + acc = src_i; + val = s + } + } + dst_to_set[unstr_index] = g(val, acc) + } + } + } + unsafe { dst.set_len(dst_len) }; + Ok(dst) + } +} + +impl Map1Any for ReduceIndex { + #[inline(always)] + fn f) -> CpuStorage>( + &self, + src: &[T], + src_l: &Layout, + wrap: W, + ) -> Result { + if src_l.shape().elem_count() == 0 { + Err(Error::EmptyTensor { op: "reduce" }.bt())? + } + let dst = match (self.return_index, self.use_min) { + (false, true) => wrap(self.fold_impl(src, src_l, |x, y| x > y, |v, _i| v)?), + (false, false) => wrap(self.fold_impl(src, src_l, |x, y| x < y, |v, _i| v)?), + (true, true) => { + CpuStorage::U32(self.fold_impl(src, src_l, |x, y| x > y, |_v, i| i as u32)?) + } + (true, false) => { + CpuStorage::U32(self.fold_impl(src, src_l, |x, y| x < y, |_v, i| i as u32)?) + } + }; + Ok(dst) + } +} + +struct ReduceSum<'a> { + dst_shape: &'a Shape, + reduce_dims: &'a [usize], + reduce_dims_and_stride: Vec<(usize, usize)>, +} + +impl<'a> ReduceSum<'a> { + #[inline(always)] + fn fold_impl(&self, src: &[T], src_l: &Layout, start_elt: T) -> Result> + where + T: WithDType, + { + let mut dst = vec![start_elt; self.dst_shape.elem_count()]; + match src_l.contiguous_offsets() { + Some((o1, o2)) => { + let src = &src[o1..o2]; + // Handle the case where we reduce over the last dimensions separately as it is + // fairly common and easy to optimize. This rely on the layout being contiguous! + // reduce_dims is sorted, check if it is ranging from a to n-1. + let reduce_over_last_dims = self + .reduce_dims + .iter() + .rev() + .enumerate() + .all(|(i, &v)| v == src_l.shape().rank() - 1 - i); + if reduce_over_last_dims { + let reduce_sz = self + .reduce_dims_and_stride + .iter() + .map(|(u, _)| u) + .product::(); + for (dst_i, dst_v) in dst.iter_mut().enumerate() { + let src_i = dst_i * reduce_sz; + unsafe { + T::vec_reduce_sum( + src[src_i..src_i + reduce_sz].as_ptr(), + dst_v, + reduce_sz, + ) + }; + } + return Ok(dst); + }; + for (unstr_index, &src) in src.iter().enumerate() { + let mut dst_index = unstr_index; + // Set the reduce_dims indexes to 0. + for &(dim, stride) in self.reduce_dims_and_stride.iter() { + // The compiler is able to optimize the following in a single divmod op. + let (pre, post) = (dst_index / stride, dst_index % stride); + dst_index = (pre / dim) * stride + post; + } + dst[dst_index] += src; + } + } + None => { + for (unstr_index, src_index) in src_l.strided_index().enumerate() { + let mut dst_index = unstr_index; + // Set the reduce_dims indexes to 0. + for &(dim, stride) in self.reduce_dims_and_stride.iter() { + // The compiler is able to optimize the following in a single divmod op. + let (pre, post) = (dst_index / stride, dst_index % stride); + dst_index = (pre / dim) * stride + post; + } + dst[dst_index] += src[src_index]; + } + } + } + Ok(dst) + } +} + +impl<'a> Map1 for ReduceSum<'a> { + #[inline(always)] + fn f(&self, src: &[T], src_l: &Layout) -> Result> { + self.fold_impl(src, src_l, T::zero()) + } +} + +pub fn unary_map U>( + vs: &[T], + layout: &Layout, + mut f: F, +) -> Vec { + match layout.strided_blocks() { + crate::StridedBlocks::SingleBlock { start_offset, len } => vs + [start_offset..start_offset + len] + .iter() + .map(|&v| f(v)) + .collect(), + crate::StridedBlocks::MultipleBlocks { + block_start_index, + block_len, + } => { + let mut result = Vec::with_capacity(layout.shape().elem_count()); + // Specialize the case where block_len is one to avoid the second loop. + if block_len == 1 { + for index in block_start_index { + let v = unsafe { vs.get_unchecked(index) }; + result.push(f(*v)) + } + } else { + for index in block_start_index { + for offset in 0..block_len { + let v = unsafe { vs.get_unchecked(index + offset) }; + result.push(f(*v)) + } + } + } + result + } + } +} + +pub fn unary_map_vec U, FV: FnMut(&[T], &mut [U])>( + vs: &[T], + layout: &Layout, + mut f: F, + mut f_vec: FV, +) -> Vec { + match layout.strided_blocks() { + crate::StridedBlocks::SingleBlock { start_offset, len } => { + let mut ys: Vec = Vec::with_capacity(len); + let ys_to_set = ys.spare_capacity_mut(); + let ys_to_set = unsafe { std::mem::transmute::<_, &mut [U]>(ys_to_set) }; + f_vec(&vs[start_offset..start_offset + len], ys_to_set); + // SAFETY: values are all set by f_vec. + unsafe { ys.set_len(len) }; + ys + } + crate::StridedBlocks::MultipleBlocks { + block_start_index, + block_len, + } => { + let el_count = layout.shape().elem_count(); + // Specialize the case where block_len is one to avoid the second loop. + if block_len == 1 { + let mut result = Vec::with_capacity(el_count); + for index in block_start_index { + let v = unsafe { vs.get_unchecked(index) }; + result.push(f(*v)) + } + result + } else { + let mut ys: Vec = Vec::with_capacity(el_count); + let ys_to_set = ys.spare_capacity_mut(); + let ys_to_set = unsafe { std::mem::transmute::<_, &mut [U]>(ys_to_set) }; + let mut dst_index = 0; + for src_index in block_start_index { + let vs = &vs[src_index..src_index + block_len]; + let ys = &mut ys_to_set[dst_index..dst_index + block_len]; + f_vec(vs, ys); + dst_index += block_len; + } + // SAFETY: values are all set by f_vec. + unsafe { ys.set_len(el_count) }; + ys + } + } + } +} + +// This function maps over two strided index sequences. +pub fn binary_map U>( + lhs_l: &Layout, + rhs_l: &Layout, + lhs: &[T], + rhs: &[T], + mut f: F, +) -> Vec { + match (lhs_l.contiguous_offsets(), rhs_l.contiguous_offsets()) { + (Some((o_l1, o_l2)), Some((o_r1, o_r2))) => lhs[o_l1..o_l2] + .iter() + .zip(rhs[o_r1..o_r2].iter()) + .map(|(&l, &r)| f(l, r)) + .collect(), + (Some((o_l1, o_l2)), None) => { + // TODO: Maybe we want to avoid going through the layout twice. + match rhs_l.offsets_b() { + Some(ob) => { + let mut i_in_block = 0; + let mut i_right_broadcast = 0; + lhs[o_l1..o_l2] + .iter() + .map(|&l| { + let r = unsafe { rhs.get_unchecked(i_in_block + ob.start) }; + i_right_broadcast += 1; + if i_right_broadcast >= ob.right_broadcast { + i_in_block += 1; + i_right_broadcast = 0; + } + if i_in_block >= ob.len { + i_in_block = 0 + } + f(l, *r) + }) + .collect() + } + None => lhs_l + .strided_index() + .zip(rhs_l.strided_index()) + .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) + .collect(), + } + } + (None, Some((o_r1, o_r2))) => { + // TODO: Maybe we want to avoid going through the layout twice. + match lhs_l.offsets_b() { + Some(ob) => { + let mut i_in_block = 0; + let mut i_right_broadcast = 0; + rhs[o_r1..o_r2] + .iter() + .map(|&r| { + let l = unsafe { lhs.get_unchecked(i_in_block + ob.start) }; + i_right_broadcast += 1; + if i_right_broadcast >= ob.right_broadcast { + i_in_block += 1; + i_right_broadcast = 0; + } + if i_in_block >= ob.len { + i_in_block = 0 + } + f(*l, r) + }) + .collect() + } + None => lhs_l + .strided_index() + .zip(rhs_l.strided_index()) + .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) + .collect(), + } + } + _ => lhs_l + .strided_index() + .zip(rhs_l.strided_index()) + .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) + .collect(), + } +} + +// Similar to binary_map but with vectorized variants. +pub fn binary_map_vec T, FV: FnMut(&[T], &[T], &mut [T])>( + lhs_l: &Layout, + rhs_l: &Layout, + lhs: &[T], + rhs: &[T], + mut f: F, + mut f_vec: FV, +) -> Vec { + let el_count = lhs_l.shape().elem_count(); + match (lhs_l.contiguous_offsets(), rhs_l.contiguous_offsets()) { + (Some((o_l1, o_l2)), Some((o_r1, o_r2))) => { + let mut ys: Vec = Vec::with_capacity(el_count); + let ys_to_set = ys.spare_capacity_mut(); + let ys_to_set = unsafe { std::mem::transmute::<_, &mut [T]>(ys_to_set) }; + f_vec(&lhs[o_l1..o_l2], &rhs[o_r1..o_r2], ys_to_set); + // SAFETY: values are all set by f_vec. + unsafe { ys.set_len(el_count) }; + ys + } + (Some((o_l1, o_l2)), None) => match rhs_l.offsets_b() { + Some(ob) if ob.right_broadcast == 1 => { + let rhs = &rhs[ob.start..ob.start + ob.len]; + let mut ys: Vec = Vec::with_capacity(el_count); + let ys_to_set = ys.spare_capacity_mut(); + let ys_to_set = unsafe { std::mem::transmute::<_, &mut [T]>(ys_to_set) }; + let mut dst_i = 0; + for src_i in (o_l1..o_l2).step_by(ob.len) { + f_vec( + &lhs[src_i..src_i + ob.len], + rhs, + &mut ys_to_set[dst_i..dst_i + ob.len], + ); + dst_i += ob.len; + } + // SAFETY: values are all set by f_vec. + unsafe { ys.set_len(el_count) }; + ys + } + Some(ob) => { + let rhs = &rhs[ob.start..ob.start + ob.len]; + let mut ys = lhs[o_l1..o_l2].to_vec(); + for idx_l in 0..ob.left_broadcast { + let start = idx_l * ob.len * ob.right_broadcast; + for (i, &r) in rhs.iter().enumerate() { + let start = start + i * ob.right_broadcast; + for v in ys[start..start + ob.right_broadcast].iter_mut() { + *v = f(*v, r) + } + } + } + ys + } + None => lhs_l + .strided_index() + .zip(rhs_l.strided_index()) + .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) + .collect(), + }, + (None, Some((o_r1, o_r2))) => match lhs_l.offsets_b() { + Some(ob) if ob.right_broadcast == 1 => { + let lhs = &lhs[ob.start..ob.start + ob.len]; + let mut ys: Vec = Vec::with_capacity(el_count); + let ys_to_set = ys.spare_capacity_mut(); + let ys_to_set = unsafe { std::mem::transmute::<_, &mut [T]>(ys_to_set) }; + let mut dst_i = 0; + for src_i in (o_r1..o_r2).step_by(ob.len) { + f_vec( + lhs, + &rhs[src_i..src_i + ob.len], + &mut ys_to_set[dst_i..dst_i + ob.len], + ); + dst_i += ob.len; + } + // SAFETY: values are all set by f_vec. + unsafe { ys.set_len(el_count) }; + ys + } + Some(ob) => { + let lhs = &lhs[ob.start..ob.start + ob.len]; + let mut ys = rhs[o_r1..o_r2].to_vec(); + for idx_l in 0..ob.left_broadcast { + let start = idx_l * ob.len * ob.right_broadcast; + for (i, &l) in lhs.iter().enumerate() { + let start = start + i * ob.right_broadcast; + for v in ys[start..start + ob.right_broadcast].iter_mut() { + *v = f(l, *v) + } + } + } + ys + } + None => lhs_l + .strided_index() + .zip(rhs_l.strided_index()) + .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) + .collect(), + }, + _ => lhs_l + .strided_index() + .zip(rhs_l.strided_index()) + .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) + .collect(), + } +} + +struct Affine(f64, f64); + +impl Map1 for Affine { + fn f(&self, vs: &[T], layout: &Layout) -> Result> { + let mul = T::from_f64(self.0); + let add = T::from_f64(self.1); + Ok(unary_map(vs, layout, |v| v * mul + add)) + } +} + +struct AvgPool2D((usize, usize), (usize, usize)); + +impl Map1 for AvgPool2D { + fn f(&self, src: &[T], layout: &Layout) -> Result> { + // https://pytorch.org/docs/stable/generated/torch.nn.AvgPool2d.html + let (k_h, k_w) = self.0; + let (s_h, s_w) = self.1; + let (b_sz, c, h, w) = layout.shape().dims4()?; + let stride = layout.stride(); + let (stride_h, stride_w) = (stride[2], stride[3]); + let h_out = (h - k_h) / s_h + 1; + let w_out = (w - k_w) / s_w + 1; + let src_index = layout.start_offset(); + let mut dst = vec![T::zero(); b_sz * c * h_out * w_out]; + let scale = 1f64 / (k_h * k_w) as f64; + let scale = T::from_f64(scale); + for b_idx in 0..b_sz { + let dst = &mut dst[b_idx * c * h_out * w_out..]; + let src_index = src_index + b_idx * stride[0]; + for c_idx in 0..c { + let dst = &mut dst[c_idx * h_out * w_out..]; + let src_index = src_index + c_idx * stride[1]; + for h_idx in 0..h_out { + for w_idx in 0..w_out { + let mut sum = T::zero(); + for m in 0..k_h { + for n in 0..k_w { + let m = s_h * h_idx + m; + let n = s_w * w_idx + n; + sum += src[src_index + m * stride_h + n * stride_w] + } + } + dst[h_idx * w_out + w_idx] = sum * scale; + } + } + } + } + Ok(dst) + } +} + +struct MaxPool2D((usize, usize), (usize, usize)); + +impl Map1 for MaxPool2D { + fn f(&self, src: &[T], layout: &Layout) -> Result> { + // https://pytorch.org/docs/stable/generated/torch.nn.MaxPool2d.html + let (k_h, k_w) = self.0; + let (s_h, s_w) = self.1; + let (b_sz, c, h, w) = layout.shape().dims4()?; + let stride = layout.stride(); + let (stride_h, stride_w) = (stride[2], stride[3]); + let h_out = (h - k_h) / s_h + 1; + let w_out = (w - k_w) / s_w + 1; + let src_index = layout.start_offset(); + let mut dst = vec![T::zero(); b_sz * c * h_out * w_out]; + for b_idx in 0..b_sz { + let dst = &mut dst[b_idx * c * h_out * w_out..]; + let src_index = src_index + b_idx * stride[0]; + for c_idx in 0..c { + let dst = &mut dst[c_idx * h_out * w_out..]; + let src_index = src_index + c_idx * stride[1]; + for h_idx in 0..h_out { + for w_idx in 0..w_out { + let mut largest = + src[src_index + s_h * h_idx * stride_h + s_w * w_idx * stride_w]; + for m in 0..k_h { + for n in 0..k_w { + let m = s_h * h_idx + m; + let n = s_w * w_idx + n; + if largest < src[src_index + m * stride_h + n * stride_w] { + largest = src[src_index + m * stride_h + n * stride_w] + } + } + } + dst[h_idx * w_out + w_idx] = largest; + } + } + } + } + Ok(dst) + } +} + +struct UpsampleNearest1D(usize); + +impl Map1 for UpsampleNearest1D { + fn f(&self, src: &[T], layout: &Layout) -> Result> { + // TODO: Specialized implementation for the case 2*sz? + let dst_sz = self.0; + let (b_sz, c, src_sz) = layout.shape().dims3()?; + let stride = layout.stride(); + let stride_sz = stride[2]; + let src_index = layout.start_offset(); + let scale_sz = src_sz as f64 / dst_sz as f64; + let mut dst = vec![T::zero(); b_sz * c * dst_sz]; + let src_idxs = (0..dst_sz) + .map(|idx| usize::min(src_sz - 1, (idx as f64 * scale_sz) as usize)) + .collect::>(); + for b_idx in 0..b_sz { + let dst = &mut dst[b_idx * c * dst_sz..]; + let src_index = src_index + b_idx * stride[0]; + for c_idx in 0..c { + let dst = &mut dst[c_idx * dst_sz..]; + let src_index = src_index + c_idx * stride[1]; + for (idx, src_idx) in src_idxs.iter().enumerate() { + dst[idx] = src[src_index + src_idx * stride_sz] + } + } + } + Ok(dst) + } +} + +struct UpsampleNearest2D(usize, usize); + +impl Map1 for UpsampleNearest2D { + fn f(&self, src: &[T], layout: &Layout) -> Result> { + // TODO: Specialized implementation for the case 2*h, 2*w? + let (dst_h, dst_w) = (self.0, self.1); + let (b_sz, c, src_h, src_w) = layout.shape().dims4()?; + let stride = layout.stride(); + let (stride_h, stride_w) = (stride[2], stride[3]); + let src_index = layout.start_offset(); + let scale_h = src_h as f64 / dst_h as f64; + let scale_w = src_w as f64 / dst_w as f64; + let mut dst = vec![T::zero(); b_sz * c * dst_h * dst_w]; + let src_h_idxs = (0..dst_h) + .map(|h_idx| usize::min(src_h - 1, (h_idx as f64 * scale_h) as usize)) + .collect::>(); + let src_w_idxs = (0..dst_w) + .map(|w_idx| usize::min(src_w - 1, (w_idx as f64 * scale_w) as usize)) + .collect::>(); + for b_idx in 0..b_sz { + let dst = &mut dst[b_idx * c * dst_h * dst_w..]; + let src_index = src_index + b_idx * stride[0]; + for c_idx in 0..c { + let dst = &mut dst[c_idx * dst_h * dst_w..]; + let src_index = src_index + c_idx * stride[1]; + for (h_idx, src_h_idx) in src_h_idxs.iter().enumerate() { + for (w_idx, src_w_idx) in src_w_idxs.iter().enumerate() { + let src_index = src_index + src_h_idx * stride_h + src_w_idx * stride_w; + dst[h_idx * dst_w + w_idx] = src[src_index] + } + } + } + } + Ok(dst) + } +} + +struct Gather<'a, I: IntDType> { + ids: &'a [I], + ids_l: &'a Layout, + dim: usize, +} + +impl<'a, I: IntDType> Map1 for Gather<'a, I> { + fn f(&self, src: &[T], src_l: &Layout) -> Result> { + let ids = match self.ids_l.contiguous_offsets() { + Some((a, b)) => &self.ids[a..b], + None => Err(Error::RequiresContiguous { op: "gather" }.bt())?, + }; + let src = match src_l.contiguous_offsets() { + Some((a, b)) => &src[a..b], + None => Err(Error::RequiresContiguous { op: "gather" }.bt())?, + }; + let dim = self.dim; + let ids_dims = self.ids_l.dims(); + let src_dims = src_l.dims(); + let dst_len: usize = ids_dims.iter().product(); + let dst_left_len: usize = ids_dims[..dim].iter().product(); + let dst_dim_len = ids_dims[dim]; + let dst_right_len: usize = ids_dims[dim + 1..].iter().product(); + + let src_dim_len = src_dims[dim]; + let src_right_len: usize = src_dims[dim + 1..].iter().product(); + + let mut dst = vec![T::zero(); dst_len]; + for left_i in 0..dst_left_len { + let start_src_idx = left_i * src_right_len * src_dim_len; + let start_dst_idx = left_i * dst_right_len * dst_dim_len; + for i in 0..dst_dim_len { + let start_dst_idx = start_dst_idx + i * dst_right_len; + for right_i in 0..dst_right_len { + let dst_idx = start_dst_idx + right_i; + let index = ids[dst_idx].as_usize(); + if index >= src_dim_len { + Err(Error::InvalidIndex { + index, + size: src_dim_len, + op: "gather", + } + .bt())? + } + let src_idx = start_src_idx + index * src_right_len + right_i; + dst[dst_idx] = src[src_idx] + } + } + } + Ok(dst) + } +} + +struct IndexSelect<'a, T: IntDType> { + ids: &'a [T], + ids_l: &'a Layout, + dim: usize, +} + +impl<'a, I: IntDType> Map1 for IndexSelect<'a, I> { + fn f(&self, src: &[T], layout: &Layout) -> Result> { + let src = match layout.contiguous_offsets() { + Some((a, b)) => &src[a..b], + None => Err(Error::RequiresContiguous { op: "index-select" }.bt())?, + }; + let dim = self.dim; + let n_ids = match self.ids_l.dims() { + [n_ids] => *n_ids, + d => Err(Error::UnexpectedNumberOfDims { + expected: 1, + got: d.len(), + shape: self.ids_l.shape().clone(), + } + .bt())?, + }; + let stride_ids = self.ids_l.stride()[0]; + let mut dst_dims = layout.dims().to_vec(); + let src_dim = dst_dims[dim]; + dst_dims[dim] = n_ids; + let dst_len: usize = dst_dims.iter().product(); + let left_len: usize = dst_dims[..dim].iter().product(); + let right_len: usize = dst_dims[dim + 1..].iter().product(); + let mut dst = vec![T::zero(); dst_len]; + for left_i in 0..left_len { + let start_src_idx = left_i * right_len * src_dim; + let start_dst_idx = left_i * right_len * n_ids; + for i in 0..n_ids { + let index = self.ids[self.ids_l.start_offset() + stride_ids * i].as_usize(); + if index >= src_dim { + Err(Error::InvalidIndex { + index, + size: src_dim, + op: "index-select", + } + .bt())? + } + let start_src_idx = start_src_idx + index * right_len; + let start_dst_idx = start_dst_idx + i * right_len; + dst[start_dst_idx..start_dst_idx + right_len] + .copy_from_slice(&src[start_src_idx..start_src_idx + right_len]) + } + } + Ok(dst) + } +} + +struct ScatterAdd<'a, I: IntDType> { + ids: &'a [I], + ids_l: &'a Layout, + dim: usize, +} + +impl<'a, I: IntDType> Map2 for ScatterAdd<'a, I> { + const OP: &'static str = "scatter-add"; + fn f(&self, v1: &[T], l1: &Layout, src: &[T], src_l: &Layout) -> Result> { + let dst_len = l1.shape().elem_count(); + let mut dst = vec![T::zero(); dst_len]; + copy_strided_src_(v1, &mut dst, 0, l1); + let src = match src_l.contiguous_offsets() { + None => Err(Error::RequiresContiguous { op: "scatter-add" }.bt())?, + Some((o1, o2)) => &src[o1..o2], + }; + + let dim = self.dim; + let ids_dims = self.ids_l.dims(); + let dst_dims = l1.dims(); + let dst_dim_len = dst_dims[dim]; + let dst_right_len: usize = dst_dims[dim + 1..].iter().product(); + + let ids_left_len: usize = ids_dims[..dim].iter().product(); + let ids_dim_len = ids_dims[dim]; + let ids_right_len: usize = ids_dims[dim + 1..].iter().product(); + + let ids = match self.ids_l.contiguous_offsets() { + Some((a, b)) => &self.ids[a..b], + None => Err(Error::RequiresContiguous { op: "gather" }.bt())?, + }; + for left_i in 0..ids_left_len { + let start_ids_idx = left_i * ids_right_len * ids_dim_len; + let start_dst_idx = left_i * dst_right_len * dst_dim_len; + for i in 0..ids_dim_len { + let start_ids_idx = start_ids_idx + i * ids_right_len; + for right_i in 0..dst_right_len { + let ids_idx = start_ids_idx + right_i; + let index = ids[ids_idx].as_usize(); + if index >= dst_dim_len { + Err(Error::InvalidIndex { + index, + size: dst_dim_len, + op: "gather", + } + .bt())? + } + let dst_idx = start_dst_idx + index * dst_right_len + right_i; + dst[dst_idx] += src[ids_idx] + } + } + } + + Ok(dst) + } +} + +struct IndexAdd<'a, I: IntDType> { + ids: &'a [I], + dim: usize, +} + +impl<'a, I: IntDType> Map2 for IndexAdd<'a, I> { + const OP: &'static str = "index-add"; + // https://pytorch.org/docs/stable/generated/torch.Tensor.index_add_.html#torch.Tensor.index_add_ + // v1, l1 -> self + fn f(&self, v1: &[T], l1: &Layout, src: &[T], src_l: &Layout) -> Result> { + let dst_len = l1.shape().elem_count(); + let mut dst = vec![T::zero(); dst_len]; + copy_strided_src_(v1, &mut dst, 0, l1); + let src = match src_l.contiguous_offsets() { + None => Err(Error::RequiresContiguous { op: "index-add" }.bt())?, + Some((o1, o2)) => &src[o1..o2], + }; + let dim = self.dim; + let max_idx = l1.dims()[dim]; + let pre_dim = src_l.dims()[..dim].iter().product::(); + let src_dim_sz = src_l.dims()[dim]; + let post_dim = src_l.dims()[dim + 1..].iter().product::(); + if dim == 0 { + for (src_idx, dst_idx) in self.ids.iter().enumerate() { + let dst_idx = dst_idx.as_usize(); + if dst_idx >= max_idx { + Err(Error::InvalidIndex { + index: dst_idx, + op: "index-add", + size: max_idx, + })? + } + let src_idx = src_idx * post_dim; + let dst_idx = dst_idx * post_dim; + let src = &src[src_idx..src_idx + post_dim]; + let dst = &mut dst[dst_idx..dst_idx + post_dim]; + for (d, &s) in dst.iter_mut().zip(src.iter()) { + *d += s + } + } + } else { + for (src_idx, dst_idx) in self.ids.iter().enumerate() { + let dst_idx = dst_idx.as_usize(); + if dst_idx >= max_idx { + Err(Error::InvalidIndex { + index: dst_idx, + op: "index-add", + size: max_idx, + })? + } + for pre_i in 0..pre_dim { + let pre_src_i = (pre_i * src_dim_sz + src_idx) * post_dim; + let pre_dst_i = (pre_i * max_idx + dst_idx) * post_dim; + let src = &src[pre_src_i..pre_src_i + post_dim]; + let dst = &mut dst[pre_dst_i..pre_dst_i + post_dim]; + for (d, &s) in dst.iter_mut().zip(src.iter()) { + *d += s + } + } + } + } + Ok(dst) + } +} + +fn copy_strided_src_(src: &[T], dst: &mut [T], dst_offset: usize, src_l: &Layout) { + match src_l.strided_blocks() { + crate::StridedBlocks::SingleBlock { start_offset, len } => { + let to_copy = (dst.len() - dst_offset).min(len); + dst[dst_offset..dst_offset + to_copy] + .copy_from_slice(&src[start_offset..start_offset + to_copy]) + } + crate::StridedBlocks::MultipleBlocks { + block_start_index, + block_len: 1, + } => { + for (dst_index, src_index) in block_start_index.enumerate() { + let dst_index = dst_index + dst_offset; + if dst_index >= dst.len() { + break; + } + dst[dst_index] = src[src_index] + } + } + crate::StridedBlocks::MultipleBlocks { + block_start_index, + block_len, + } => { + let mut dst_index = dst_offset; + for src_index in block_start_index { + let next_dst_index = dst_index + block_len; + if dst_index >= dst.len() { + break; + } + let to_copy = usize::min(block_len, dst.len() - dst_index); + dst[dst_index..dst_index + to_copy] + .copy_from_slice(&src[src_index..src_index + to_copy]); + dst_index = next_dst_index + } + } + } +} + +struct Conv1D<'a>(&'a crate::conv::ParamsConv1D); + +impl<'a> Map2 for Conv1D<'a> { + const OP: &'static str = "conv1d"; + fn f(&self, inp: &[T], inp_l: &Layout, k: &[T], k_l: &Layout) -> Result> { + let p = self.0; + let inp = &inp[inp_l.start_offset()..]; + let k = &k[k_l.start_offset()..]; + let (inp_s0, inp_s1, inp_s2) = crate::shape::dims3(inp_l.stride())?; + let (k_s0, k_s1, k_s2) = crate::shape::dims3(k_l.stride())?; + let l_out = p.l_out(); + let dst_elems = p.c_out * l_out * p.b_size; + // The output shape is [b_size, c_out, l_out] + let dst = vec![T::zero(); dst_elems]; + + // TODO: Avoid making this copy if `inp` already has the appropriate layout. + let mut inp_cont = vec![T::zero(); p.b_size * p.c_in * p.l_in]; + for b_idx in 0..p.b_size { + for src_l in 0..p.l_in { + for src_c_idx in 0..p.c_in { + let inp_idx = b_idx * inp_s0 + src_c_idx * inp_s1 + src_l * inp_s2; + inp_cont[b_idx * p.l_in * p.c_in + src_l * p.c_in + src_c_idx] = inp[inp_idx] + } + } + } + + for offset in 0..p.k_size { + (0..p.c_out).into_par_iter().for_each(|dst_c_idx| { + let dst_idx = dst_c_idx * l_out; + let k_cont = (0..p.c_in) + .map(|c_in_idx| k[dst_c_idx * k_s0 + c_in_idx * k_s1 + offset * k_s2]) + .collect::>(); + for b_idx in 0..p.b_size { + let dst_idx = dst_idx + b_idx * p.c_out * l_out; + for dst_l in 0..l_out { + let dst_idx = dst_idx + dst_l; + let src_l = p.stride * dst_l + offset * p.dilation; + if src_l < p.padding || src_l >= p.padding + p.l_in { + continue; + } + let src_l = src_l - p.padding; + let inp_cont = &inp_cont[b_idx * p.l_in * p.c_in + src_l * p.c_in..]; + assert!(inp_cont.len() >= p.c_in); + assert!(k_cont.len() >= p.c_in); + let mut d = T::zero(); + unsafe { T::vec_dot(inp_cont.as_ptr(), k_cont.as_ptr(), &mut d, p.c_in) } + let dst_p = dst.as_ptr(); + // Safety: dst_idx are uniques per dst_c_idx which is used to parallelise + // the different tasks so no two threads can try to write at the same + // location. + unsafe { + let ptr = dst_p.add(dst_idx) as *mut T; + *ptr += d + } + } + } + }) + } + Ok(dst) + } +} + +struct Im2Col1D { + l_k: usize, + stride: usize, + dilation: usize, + padding: usize, +} + +impl Im2Col1D { + fn l_out(&self, l: usize) -> usize { + (l + 2 * self.padding - self.dilation * (self.l_k - 1) - 1) / self.stride + 1 + } +} + +impl Map1 for Im2Col1D { + fn f(&self, vs: &[T], layout: &Layout) -> Result> { + let &Self { + l_k, + stride, + dilation, + padding, + } = self; + let (b, c, l) = layout.shape().dims3()?; + let l_out = self.l_out(l); + let src = &vs[layout.start_offset()..]; + let mut dst = vec![T::zero(); b * l_out * c * l_k]; + let (src_s0, src_s1, src_s2) = { + let s = layout.stride(); + (s[0], s[1], s[2]) + }; + // TODO: provide specialized kernels for the common use cases. + // - l_k = 1 + // - padding = 0 + // - stride = 1 + // - dilation = 1 + for b_idx in 0..b { + let src_idx = b_idx * src_s0; + let dst_idx = b_idx * l_out * c * l_k; + for l_idx in 0..l_out { + let dst_idx = dst_idx + l_idx * c * l_k; + for c_idx in 0..c { + let dst_idx = dst_idx + c_idx * l_k; + let src_idx = c_idx * src_s1 + src_idx; + for l_k_idx in 0..l_k { + let src_l = l_idx * stride + l_k_idx * dilation; + if padding != 0 && (src_l < padding || src_l >= l + padding) { + continue; + } + let src_l = src_l - padding; + let src_idx = src_idx + src_l * src_s2; + let dst_idx = dst_idx + l_k_idx; + dst[dst_idx] = src[src_idx] + } + } + } + } + Ok(dst) + } +} + +struct Im2Col { + h_k: usize, + w_k: usize, + stride: usize, + dilation: usize, + padding: usize, +} + +impl Im2Col { + fn hw_out(&self, h: usize, w: usize) -> (usize, usize) { + let h_out = (h + 2 * self.padding - self.dilation * (self.h_k - 1) - 1) / self.stride + 1; + let w_out = (w + 2 * self.padding - self.dilation * (self.w_k - 1) - 1) / self.stride + 1; + (h_out, w_out) + } +} + +impl Map1 for Im2Col { + fn f(&self, vs: &[T], layout: &Layout) -> Result> { + let &Self { + h_k, + w_k, + stride, + dilation, + padding, + } = self; + let (b, c, h, w) = layout.shape().dims4()?; + let (h_out, w_out) = self.hw_out(h, w); + let src = &vs[layout.start_offset()..]; + let mut dst = vec![T::zero(); b * h_out * w_out * c * h_k * w_k]; + let (src_s0, src_s1, src_s2, src_s3) = { + let s = layout.stride(); + (s[0], s[1], s[2], s[3]) + }; + // TODO: provide specialized kernels for the common use cases. + // - h_k = w_k = 1 + // - padding = 0 + // - stride = 1 + // - dilation = 1 + for b_idx in 0..b { + let src_idx = b_idx * src_s0; + let dst_idx = b_idx * h_out * w_out * c * h_k * w_k; + for h_idx in 0..h_out { + let dst_idx = dst_idx + h_idx * w_out * c * h_k * w_k; + for w_idx in 0..w_out { + let dst_idx = dst_idx + w_idx * c * h_k * w_k; + for c_idx in 0..c { + let dst_idx = dst_idx + c_idx * h_k * w_k; + let src_idx = c_idx * src_s1 + src_idx; + for h_k_idx in 0..h_k { + let src_h = h_idx * stride + h_k_idx * dilation; + if padding != 0 && (src_h < padding || src_h >= h + padding) { + continue; + } + let src_h = src_h - padding; + let src_idx = src_idx + src_h * src_s2; + let dst_idx = dst_idx + h_k_idx * w_k; + for w_k_idx in 0..w_k { + let src_w = w_idx * stride + w_k_idx * dilation; + if padding != 0 && (src_w < padding || src_w >= w + padding) { + continue; + } + let src_w = src_w - padding; + let src_idx = src_idx + src_w * src_s3; + let dst_idx = dst_idx + w_k_idx; + dst[dst_idx] = src[src_idx] + } + } + } + } + } + } + Ok(dst) + } +} + +struct ConvTranspose1D<'a>(&'a crate::conv::ParamsConvTranspose1D); + +impl<'a> Map2 for ConvTranspose1D<'a> { + const OP: &'static str = "conv_transpose1d"; + fn f(&self, inp: &[T], inp_l: &Layout, k: &[T], k_l: &Layout) -> Result> { + let p = self.0; + let inp = &inp[inp_l.start_offset()..]; + let k = &k[k_l.start_offset()..]; + let (inp_s0, inp_s1, inp_s2) = crate::shape::dims3(inp_l.stride())?; + let (k_s0, k_s1, k_s2) = crate::shape::dims3(k_l.stride())?; + let l_out = p.l_out(); + + // Output shape: [b_size, c_out, l_out]. + let dst_elems = p.c_out * l_out * p.b_size; + let dst = vec![T::zero(); dst_elems]; + let dst_s0 = p.c_out * l_out; + let dst_s1 = l_out; + let dst_s2 = 1; + + // TODO: Avoid making this copy if `inp` already has the appropriate layout. + let mut inp_cont = vec![T::zero(); p.b_size * p.c_in * p.l_in]; + let cont_s0 = p.l_in * p.c_in; + let cont_s1 = p.c_in; + for b_idx in 0..p.b_size { + for l_idx in 0..p.l_in { + for c_idx in 0..p.c_in { + let src_idx = b_idx * inp_s0 + c_idx * inp_s1 + l_idx * inp_s2; + let dst_idx = b_idx * cont_s0 + l_idx * cont_s1 + c_idx; + inp_cont[dst_idx] = inp[src_idx] + } + } + } + + for k_idx in 0..p.k_size { + (0..p.c_out).into_par_iter().for_each(|dst_c_idx| { + let k_cont = (0..p.c_in) + .map(|c_in_idx| k[c_in_idx * k_s0 + dst_c_idx * k_s1 + k_idx * k_s2]) + .collect::>(); + for b_idx in 0..p.b_size { + for l_idx in 0..p.l_in { + let out_idx = l_idx * p.stride + k_idx * p.dilation; + if out_idx < p.padding { + continue; + } + let out_idx = out_idx - p.padding; + if out_idx < l_out { + let inp_cont = &inp_cont[b_idx * cont_s0 + l_idx * cont_s1..]; + let dst_idx = b_idx * dst_s0 + out_idx * dst_s2 + dst_c_idx * dst_s1; + let mut d = T::zero(); + unsafe { + T::vec_dot(inp_cont.as_ptr(), k_cont.as_ptr(), &mut d, p.c_in) + } + let dst_p = dst.as_ptr(); + // Safety: dst_idx are uniques per dst_c_idx which is used to + // parallelise the different tasks so no two threads can try to + // write at the same location. + unsafe { + let ptr = dst_p.add(dst_idx) as *mut T; + *ptr += d + } + } + } + } + }) + } + Ok(dst) + } +} + +struct Conv2D<'a>(&'a crate::conv::ParamsConv2D); + +impl<'a> Map2 for Conv2D<'a> { + const OP: &'static str = "conv2d"; + fn f(&self, inp: &[T], inp_l: &Layout, k: &[T], k_l: &Layout) -> Result> { + let p = self.0; + let inp = &inp[inp_l.start_offset()..]; + let (inp_s0, inp_s1, inp_s2, inp_s3) = crate::shape::dims4(inp_l.stride())?; + let k = &k[k_l.start_offset()..]; + let (k_s0, k_s1, k_s2, k_s3) = crate::shape::dims4(k_l.stride())?; + let (out_h, out_w) = (p.out_h(), p.out_w()); + + // Output shape: [b_size, c_out, out_h, out_w]. + let dst = vec![T::zero(); p.b_size * p.c_out * out_h * out_w]; + + // TODO: Avoid making this copy if `inp` already has the appropriate layout. + let mut inp_cont = vec![T::zero(); p.b_size * p.c_in * p.i_h * p.i_w]; + let cont_s0 = p.i_h * p.i_w * p.c_in; + let cont_s1 = p.i_w * p.c_in; + let cont_s2 = p.c_in; + for b_idx in 0..p.b_size { + for h_idx in 0..p.i_h { + for w_idx in 0..p.i_w { + for c_idx in 0..p.c_in { + let src_idx = + b_idx * inp_s0 + c_idx * inp_s1 + h_idx * inp_s2 + w_idx * inp_s3; + let dst_idx = b_idx * cont_s0 + h_idx * cont_s1 + w_idx * cont_s2 + c_idx; + inp_cont[dst_idx] = inp[src_idx] + } + } + } + } + + for offset_h in 0..p.k_h { + for offset_w in 0..p.k_w { + (0..p.c_out).into_par_iter().for_each(|dst_c_idx| { + let dst_idx = dst_c_idx * out_w * out_h; + let k_cont = (0..p.c_in) + .map(|c_in_idx| { + k[dst_c_idx * k_s0 + + c_in_idx * k_s1 + + offset_h * k_s2 + + offset_w * k_s3] + }) + .collect::>(); + for b_idx in 0..p.b_size { + let dst_idx = dst_idx + b_idx * p.c_out * out_h * out_w; + for dst_h in 0..out_h { + let dst_idx = dst_idx + dst_h * out_w; + let src_h = p.stride * dst_h + offset_h * p.dilation; + if src_h < p.padding || src_h >= p.i_h + p.padding { + continue; + } + let src_h = src_h - p.padding; + for dst_w in 0..out_w { + let dst_idx = dst_idx + dst_w; + let src_w = p.stride * dst_w + offset_w * p.dilation; + if src_w < p.padding || src_w >= p.i_w + p.padding { + continue; + } + let src_w = src_w - p.padding; + let inp_cont = &inp_cont + [b_idx * cont_s0 + src_h * cont_s1 + src_w * cont_s2..]; + assert!(inp_cont.len() >= p.c_in); + assert!(k_cont.len() >= p.c_in); + let mut d = T::zero(); + unsafe { + T::vec_dot(inp_cont.as_ptr(), k_cont.as_ptr(), &mut d, p.c_in) + } + let dst_p = dst.as_ptr(); + // Safety: dst_idx are uniques per dst_c_idx which is used to parallelise + // the different tasks so no two threads can try to write at the same + // location. + unsafe { + let ptr = dst_p.add(dst_idx) as *mut T; + *ptr += d + } + } + } + } + }); + } + } + + Ok(dst) + } +} + +struct ConvTranspose2D<'a>(&'a crate::conv::ParamsConvTranspose2D); + +impl<'a> Map2 for ConvTranspose2D<'a> { + const OP: &'static str = "conv_transpose2d"; + fn f(&self, inp: &[T], inp_l: &Layout, k: &[T], k_l: &Layout) -> Result> { + let p = self.0; + let inp = &inp[inp_l.start_offset()..]; + let (inp_s0, inp_s1, inp_s2, inp_s3) = crate::shape::dims4(inp_l.stride())?; + let k = &k[k_l.start_offset()..]; + let (k_s0, k_s1, k_s2, k_s3) = crate::shape::dims4(k_l.stride())?; + let (out_h, out_w) = (p.out_h(), p.out_w()); + + // Output shape: [b_size, c_out, out_h, out_w]. + let dst = vec![T::zero(); p.b_size * p.c_out * out_h * out_w]; + let dst_s0 = p.c_out * out_h * out_w; + let dst_s1 = out_h * out_w; + let dst_s2 = out_w; + let dst_s3 = 1; + + // TODO: Avoid making this copy if `inp` already has the appropriate layout. + let mut inp_cont = vec![T::zero(); p.b_size * p.c_in * p.i_h * p.i_w]; + let cont_s0 = p.i_h * p.i_w * p.c_in; + let cont_s1 = p.i_w * p.c_in; + let cont_s2 = p.c_in; + for b_idx in 0..p.b_size { + for h_idx in 0..p.i_h { + for w_idx in 0..p.i_w { + for c_idx in 0..p.c_in { + let src_idx = + b_idx * inp_s0 + c_idx * inp_s1 + h_idx * inp_s2 + w_idx * inp_s3; + let dst_idx = b_idx * cont_s0 + h_idx * cont_s1 + w_idx * cont_s2 + c_idx; + inp_cont[dst_idx] = inp[src_idx] + } + } + } + } + + for k_y in 0..p.k_h { + for k_x in 0..p.k_w { + (0..p.c_out).into_par_iter().for_each(|dst_c_idx| { + let k_cont = (0..p.c_in) + .map(|c_in_idx| { + k[c_in_idx * k_s0 + dst_c_idx * k_s1 + k_y * k_s2 + k_x * k_s3] + }) + .collect::>(); + for b_idx in 0..p.b_size { + for inp_y in 0..p.i_h { + for inp_x in 0..p.i_w { + let out_x = inp_x * p.stride + k_x * p.dilation; + let out_y = inp_y * p.stride + k_y * p.dilation; + if out_x < p.padding || out_y < p.padding { + continue; + } + let out_x = out_x - p.padding; + let out_y = out_y - p.padding; + if out_x < out_w && out_y < out_h { + let inp_cont = &inp_cont + [b_idx * cont_s0 + inp_y * cont_s1 + inp_x * cont_s2..]; + let dst_idx = b_idx * dst_s0 + + out_y * dst_s2 + + out_x * dst_s3 + + dst_c_idx * dst_s1; + let mut d = T::zero(); + unsafe { + T::vec_dot( + inp_cont.as_ptr(), + k_cont.as_ptr(), + &mut d, + p.c_in, + ) + } + let dst_p = dst.as_ptr(); + // Safety: dst_idx are uniques per dst_c_idx which is used to + // parallelise the different tasks so no two threads can try to + // write at the same location. + unsafe { + let ptr = dst_p.add(dst_idx) as *mut T; + *ptr += d + } + } + } + } + } + }) + } + } + Ok(dst) + } +} + +struct MatMul((usize, usize, usize, usize)); + +impl MatMul { + fn striding_error(&self, lhs_l: &Layout, rhs_l: &Layout, msg: &'static str) -> Error { + Error::MatMulUnexpectedStriding(Box::new(crate::error::MatMulUnexpectedStriding { + lhs_l: lhs_l.clone(), + rhs_l: rhs_l.clone(), + bmnk: self.0, + msg, + })) + .bt() + } +} + +impl Map2 for MatMul { + const OP: &'static str = "mat_mul"; + + #[cfg(all(not(feature = "mkl"), not(feature = "accelerate")))] + fn f( + &self, + lhs: &[T], + lhs_l: &Layout, + rhs: &[T], + rhs_l: &Layout, + ) -> Result> { + use gemm::{gemm, Parallelism}; + + match T::DTYPE { + DType::F16 | DType::F32 | DType::F64 => {} + _ => Err(Error::UnsupportedDTypeForOp(T::DTYPE, "matmul").bt())?, + } + + let (b, m, n, k) = self.0; + let lhs = &lhs[lhs_l.start_offset()..]; + let rhs = &rhs[rhs_l.start_offset()..]; + + let lhs_stride = lhs_l.stride(); + let rhs_stride = rhs_l.stride(); + let rank = lhs_stride.len(); + let lhs_cs = lhs_stride[rank - 1]; + let lhs_rs = lhs_stride[rank - 2]; + + let rhs_cs = rhs_stride[rank - 1]; + let rhs_rs = rhs_stride[rank - 2]; + + let a_skip: usize = match lhs_stride[..rank - 2] { + [s1, stride] if s1 == stride * lhs_l.dims()[1] => stride, + [stride] => stride, + [] => m * k, + _ => Err(self.striding_error(lhs_l, rhs_l, "non-contiguous lhs"))?, + }; + let b_skip: usize = match rhs_stride[..rank - 2] { + [s1, stride] if s1 == stride * rhs_l.dims()[1] => stride, + [stride] => stride, + [] => n * k, + _ => Err(self.striding_error(lhs_l, rhs_l, "non-contiguous rhs"))?, + }; + let c_skip: usize = m * n; + + let dst_shape: Shape = (m, n).into(); + let dst_strides = dst_shape.stride_contiguous(); + let dst_rs = dst_strides[0]; + let dst_cs = dst_strides[1]; + + let mut dst = vec![T::zero(); b * m * n]; + let num_threads = crate::utils::get_num_threads(); + let parallelism = if num_threads > 1 { + Parallelism::Rayon(num_threads) + } else { + Parallelism::None + }; + for step in 0..b { + let lhs_p = &lhs[step * a_skip..]; + let rhs_p = &rhs[step * b_skip..]; + let dst_p = &mut dst[step * c_skip..]; + unsafe { + gemm( + /* m: usize = */ m, + /* n: usize = */ n, + /* k: usize = */ k, + /* dst: *mut T = */ dst_p.as_mut_ptr(), + /* dst_cs: isize = */ dst_cs as isize, + /* dst_rs: isize = */ dst_rs as isize, + /* read_dst: bool = */ false, + /* lhs: *const T = */ lhs_p.as_ptr(), + /* lhs_cs: isize = */ lhs_cs as isize, + /* lhs_rs: isize = */ lhs_rs as isize, + /* rhs: *const T = */ rhs_p.as_ptr(), + /* rhs_cs: isize = */ rhs_cs as isize, + /* rhs_rs: isize = */ rhs_rs as isize, + /* alpha: T = */ T::zero(), + /* beta: T = */ T::one(), + /* conj_dst: bool = */ false, + /* conj_lhs: bool = */ false, + /* conj_rhs: bool = */ false, + parallelism, + ) + } + } + Ok(dst) + } + + #[cfg(feature = "accelerate")] + fn f( + &self, + lhs: &[T], + lhs_l: &Layout, + rhs: &[T], + rhs_l: &Layout, + ) -> Result> { + let (b, m, n, k) = self.0; + let lhs = &lhs[lhs_l.start_offset()..]; + let rhs = &rhs[rhs_l.start_offset()..]; + + let lhs_stride = lhs_l.stride(); + let rhs_stride = rhs_l.stride(); + let rank = lhs_stride.len(); + + let a_skip: usize = match lhs_stride[..rank - 2] { + [s1, stride] if s1 == stride * lhs_l.dims()[1] => stride, + [stride] => stride, + [] => m * k, + _ => Err(self.striding_error(lhs_l, rhs_l, "non-contiguous lhs"))?, + }; + let b_skip: usize = match rhs_stride[..rank - 2] { + [s1, stride] if s1 == stride * rhs_l.dims()[1] => stride, + [stride] => stride, + [] => n * k, + _ => Err(self.striding_error(lhs_l, rhs_l, "non-contiguous rhs"))?, + }; + let c_skip: usize = m * n; + + let rhs_m1 = rhs_stride[rhs_stride.len() - 1]; + let rhs_m2 = rhs_stride[rhs_stride.len() - 2]; + let lhs_m1 = lhs_stride[lhs_stride.len() - 1]; + let lhs_m2 = lhs_stride[lhs_stride.len() - 2]; + + let (lda, transa) = if rhs_m1 == 1 && rhs_m2 == n { + (n as i32, b'N') + } else if rhs_m1 == k && rhs_m2 == 1 { + (k as i32, b'T') + } else { + Err(self.striding_error(lhs_l, rhs_l, "non-contiguous rhs"))? + }; + // The b tensor has dims batching, m, k (lhs) + let (ldb, transb) = if lhs_m1 == 1 && lhs_m2 == k { + (k as i32, b'N') + } else if lhs_m1 == m && lhs_m2 == 1 { + (m as i32, b'T') + } else { + Err(self.striding_error(lhs_l, rhs_l, "non-contiguous lhs"))? + }; + + let mut dst = vec![T::zero(); b * m * n]; + match T::DTYPE { + DType::F16 => { + crate::bail!("the accelerate backend does not support f16 matmul") + } + DType::F32 => { + for step in 0..b { + let lhs_p = &lhs[step * a_skip..]; + let rhs_p = &rhs[step * b_skip..]; + let dst_p = &mut dst[step * c_skip..]; + unsafe { + let a = rhs_p.as_ptr() as *const f32; + let b = lhs_p.as_ptr() as *const f32; + let c = dst_p.as_mut_ptr() as *mut f32; + let a = std::slice::from_raw_parts(a, a_skip); + let b = std::slice::from_raw_parts(b, b_skip); + let c = std::slice::from_raw_parts_mut(c, c_skip); + crate::accelerate::sgemm( + transa, transb, /* m= */ n as i32, /* n= */ m as i32, + /* k= */ k as i32, /* alpha= */ 1., /* a= */ a, + /* lda= */ lda, /* b= */ b, /* ldb= */ ldb, + /* beta= */ 0., /* c= */ c, /* ldc= */ n as i32, + ) + } + } + } + DType::F64 => { + for step in 0..b { + let lhs_p = &lhs[step * a_skip..]; + let rhs_p = &rhs[step * b_skip..]; + let dst_p = &mut dst[step * c_skip..]; + unsafe { + let a = rhs_p.as_ptr() as *const f64; + let b = lhs_p.as_ptr() as *const f64; + let c = dst_p.as_mut_ptr() as *mut f64; + let a = std::slice::from_raw_parts(a, a_skip); + let b = std::slice::from_raw_parts(b, b_skip); + let c = std::slice::from_raw_parts_mut(c, c_skip); + crate::accelerate::dgemm( + transa, transb, /* m= */ n as i32, /* n= */ m as i32, + /* k= */ k as i32, /* alpha= */ 1., /* a= */ a, + /* lda= */ lda, /* b= */ b, /* ldb= */ ldb, + /* beta= */ 0., /* c= */ c, /* ldc= */ n as i32, + ) + } + } + } + dtype => Err(Error::UnsupportedDTypeForOp(dtype, "matmul").bt())?, + } + Ok(dst) + } + + #[cfg(feature = "mkl")] + fn f( + &self, + lhs: &[T], + lhs_l: &Layout, + rhs: &[T], + rhs_l: &Layout, + ) -> Result> { + let (b, m, n, k) = self.0; + let lhs = &lhs[lhs_l.start_offset()..]; + let rhs = &rhs[rhs_l.start_offset()..]; + + let lhs_stride = lhs_l.stride(); + let rhs_stride = rhs_l.stride(); + let rank = lhs_stride.len(); + + let a_skip: usize = match lhs_stride[..rank - 2] { + [s1, stride] if s1 == stride * lhs_l.dims()[1] => stride, + [stride] => stride, + [] => m * k, + _ => Err(self.striding_error(lhs_l, rhs_l, "non-contiguous lhs"))?, + }; + let b_skip: usize = match rhs_stride[..rank - 2] { + [s1, stride] if s1 == stride * rhs_l.dims()[1] => stride, + [stride] => stride, + [] => n * k, + _ => Err(self.striding_error(lhs_l, rhs_l, "non-contiguous rhs"))?, + }; + let c_skip: usize = m * n; + + let rhs_m1 = rhs_stride[rhs_stride.len() - 1]; + let rhs_m2 = rhs_stride[rhs_stride.len() - 2]; + let lhs_m1 = lhs_stride[lhs_stride.len() - 1]; + let lhs_m2 = lhs_stride[lhs_stride.len() - 2]; + + let (lda, transa) = if rhs_m1 == 1 && rhs_m2 == n { + (n as i32, b'N') + } else if rhs_m1 == k && rhs_m2 == 1 { + (k as i32, b'T') + } else { + Err(self.striding_error(lhs_l, rhs_l, "non-contiguous rhs"))? + }; + // The b tensor has dims batching, m, k (lhs) + let (ldb, transb) = if lhs_m1 == 1 && lhs_m2 == k { + (k as i32, b'N') + } else if lhs_m1 == m && lhs_m2 == 1 { + (m as i32, b'T') + } else { + Err(self.striding_error(lhs_l, rhs_l, "non-contiguous lhs"))? + }; + + let mut dst = vec![T::zero(); b * m * n]; + match T::DTYPE { + DType::F16 => { + for step in 0..b { + let lhs_p = &lhs[step * a_skip..]; + let rhs_p = &rhs[step * b_skip..]; + let dst_p = &mut dst[step * c_skip..]; + unsafe { + let a = rhs_p.as_ptr() as *const f16; + let b = lhs_p.as_ptr() as *const f16; + let c = dst_p.as_mut_ptr() as *mut f16; + let a = std::slice::from_raw_parts(a, a_skip); + let b = std::slice::from_raw_parts(b, b_skip); + let c = std::slice::from_raw_parts_mut(c, c_skip); + crate::mkl::hgemm( + transa, + transb, + /* m= */ n as i32, + /* n= */ m as i32, + /* k= */ k as i32, + /* alpha= */ f16::ONE, + /* a= */ a, + /* lda= */ lda, + /* b= */ b, + /* ldb= */ ldb, + /* beta= */ f16::ZERO, + /* c= */ c, + /* ldc= */ n as i32, + ) + } + } + } + DType::F32 => { + for step in 0..b { + let lhs_p = &lhs[step * a_skip..]; + let rhs_p = &rhs[step * b_skip..]; + let dst_p = &mut dst[step * c_skip..]; + unsafe { + let a = rhs_p.as_ptr() as *const f32; + let b = lhs_p.as_ptr() as *const f32; + let c = dst_p.as_mut_ptr() as *mut f32; + let a = std::slice::from_raw_parts(a, a_skip); + let b = std::slice::from_raw_parts(b, b_skip); + let c = std::slice::from_raw_parts_mut(c, c_skip); + crate::mkl::sgemm( + transa, transb, /* m= */ n as i32, /* n= */ m as i32, + /* k= */ k as i32, /* alpha= */ 1., /* a= */ a, + /* lda= */ lda, /* b= */ b, /* ldb= */ ldb, + /* beta= */ 0., /* c= */ c, /* ldc= */ n as i32, + ) + } + } + } + DType::F64 => { + for step in 0..b { + let lhs_p = &lhs[step * a_skip..]; + let rhs_p = &rhs[step * b_skip..]; + let dst_p = &mut dst[step * c_skip..]; + unsafe { + let a = rhs_p.as_ptr() as *const f64; + let b = lhs_p.as_ptr() as *const f64; + let c = dst_p.as_mut_ptr() as *mut f64; + let a = std::slice::from_raw_parts(a, a_skip); + let b = std::slice::from_raw_parts(b, b_skip); + let c = std::slice::from_raw_parts_mut(c, c_skip); + crate::mkl::dgemm( + transa, transb, /* m= */ n as i32, /* n= */ m as i32, + /* k= */ k as i32, /* alpha= */ 1., /* a= */ a, + /* lda= */ lda, /* b= */ b, /* ldb= */ ldb, + /* beta= */ 0., /* c= */ c, /* ldc= */ n as i32, + ) + } + } + } + dtype => Err(Error::UnsupportedDTypeForOp(dtype, "matmul").bt())?, + } + Ok(dst) + } +} + +fn elu(v: T, alpha: T) -> T { + if v.is_sign_positive() { + v + } else { + (v.exp() - T::one()) * alpha + } +} + +impl CpuStorage { + pub fn as_slice(&self) -> Result<&[D]> { + D::cpu_storage_as_slice(self) + } + + pub fn concat(storages: &[CpuStorage]) -> Result { + let storage0 = &storages[0]; + let s = match storage0 { + Self::U8(_) => { + let storages = storages + .iter() + .map(|s| match s { + Self::U8(s) => Ok(s.as_slice()), + _ => crate::bail!("dtype mismatch"), + }) + .collect::>>()? + .concat(); + Self::U8(storages) + } + Self::U32(_) => { + let storages = storages + .iter() + .map(|s| match s { + Self::U32(s) => Ok(s.as_slice()), + _ => crate::bail!("dtype mismatch"), + }) + .collect::>>()? + .concat(); + Self::U32(storages) + } + Self::I64(_) => { + let storages = storages + .iter() + .map(|s| match s { + Self::I64(s) => Ok(s.as_slice()), + _ => crate::bail!("dtype mismatch"), + }) + .collect::>>()? + .concat(); + Self::I64(storages) + } + Self::BF16(_) => { + let storages = storages + .iter() + .map(|s| match s { + Self::BF16(s) => Ok(s.as_slice()), + _ => crate::bail!("dtype mismatch"), + }) + .collect::>>()? + .concat(); + Self::BF16(storages) + } + Self::F16(_) => { + let storages = storages + .iter() + .map(|s| match s { + Self::F16(s) => Ok(s.as_slice()), + _ => crate::bail!("dtype mismatch"), + }) + .collect::>>()? + .concat(); + Self::F16(storages) + } + Self::F32(_) => { + let storages = storages + .iter() + .map(|s| match s { + Self::F32(s) => Ok(s.as_slice()), + _ => crate::bail!("dtype mismatch"), + }) + .collect::>>()? + .concat(); + Self::F32(storages) + } + Self::F64(_) => { + let storages = storages + .iter() + .map(|s| match s { + Self::F64(s) => Ok(s.as_slice()), + _ => crate::bail!("dtype mismatch"), + }) + .collect::>>()? + .concat(); + Self::F64(storages) + } + }; + Ok(s) + } +} + +impl BackendStorage for CpuStorage { + type Device = CpuDevice; + + fn dtype(&self) -> DType { + match self { + Self::U8(_) => DType::U8, + Self::U32(_) => DType::U32, + Self::I64(_) => DType::I64, + Self::BF16(_) => DType::BF16, + Self::F16(_) => DType::F16, + Self::F32(_) => DType::F32, + Self::F64(_) => DType::F64, + } + } + + fn to_dtype(&self, layout: &Layout, dtype: DType) -> Result { + // TODO: find a way around the quadratic number of cases below. + match (self, dtype) { + (Self::U8(storage), DType::BF16) => { + let data = unary_map(storage, layout, |v| bf16::from_f32(v as f32)); + Ok(Self::BF16(data)) + } + (Self::U32(storage), DType::BF16) => { + let data = unary_map(storage, layout, |v| bf16::from_f32(v as f32)); + Ok(Self::BF16(data)) + } + (Self::I64(storage), DType::BF16) => { + let data = unary_map(storage, layout, |v| bf16::from_f32(v as f32)); + Ok(Self::BF16(data)) + } + (Self::BF16(storage), DType::BF16) => { + let data = unary_map(storage, layout, |v| v); + Ok(Self::BF16(data)) + } + (Self::F16(storage), DType::BF16) => { + let data = unary_map(storage, layout, |v| bf16::from_f32(v.to_f32())); + Ok(Self::BF16(data)) + } + (Self::F32(storage), DType::BF16) => { + let data = unary_map(storage, layout, bf16::from_f32); + Ok(Self::BF16(data)) + } + (Self::F64(storage), DType::BF16) => { + let data = unary_map(storage, layout, bf16::from_f64); + Ok(Self::BF16(data)) + } + (Self::U8(storage), DType::F16) => { + let data = unary_map(storage, layout, |v| f16::from_f32(v as f32)); + Ok(Self::F16(data)) + } + (Self::U32(storage), DType::F16) => { + let data = unary_map(storage, layout, |v| f16::from_f32(v as f32)); + Ok(Self::F16(data)) + } + (Self::I64(storage), DType::F16) => { + let data = unary_map(storage, layout, |v| f16::from_f32(v as f32)); + Ok(Self::F16(data)) + } + (Self::BF16(storage), DType::F16) => { + let data = unary_map(storage, layout, |v| f16::from_f32(v.to_f32())); + Ok(Self::F16(data)) + } + (Self::F16(storage), DType::F16) => { + let data = unary_map(storage, layout, |v| v); + Ok(Self::F16(data)) + } + (Self::F32(storage), DType::F16) => { + let data = unary_map(storage, layout, f16::from_f32); + Ok(Self::F16(data)) + } + (Self::F64(storage), DType::F16) => { + let data = unary_map(storage, layout, f16::from_f64); + Ok(Self::F16(data)) + } + (Self::U8(storage), DType::F32) => { + let data = unary_map(storage, layout, |v| v as f32); + Ok(Self::F32(data)) + } + (Self::U32(storage), DType::F32) => { + let data = unary_map(storage, layout, |v| v as f32); + Ok(Self::F32(data)) + } + (Self::I64(storage), DType::F32) => { + let data = unary_map(storage, layout, |v| v as f32); + Ok(Self::F32(data)) + } + (Self::BF16(storage), DType::F32) => { + let data = unary_map(storage, layout, |v| v.to_f32()); + Ok(Self::F32(data)) + } + (Self::F16(storage), DType::F32) => { + let data = unary_map(storage, layout, |v| v.to_f32()); + Ok(Self::F32(data)) + } + (Self::F32(storage), DType::F32) => { + let data = unary_map(storage, layout, |v| v); + Ok(Self::F32(data)) + } + (Self::F64(storage), DType::F32) => { + let data = unary_map(storage, layout, |v| v as f32); + Ok(Self::F32(data)) + } + (Self::U8(storage), DType::U8) => { + let data = unary_map(storage, layout, |v| v); + Ok(Self::U8(data)) + } + (Self::BF16(storage), DType::U8) => { + let data = unary_map(storage, layout, |v| v.to_f32() as u8); + Ok(Self::U8(data)) + } + (Self::F16(storage), DType::U8) => { + let data = unary_map(storage, layout, |v| v.to_f32() as u8); + Ok(Self::U8(data)) + } + (Self::F32(storage), DType::U8) => { + let data = unary_map(storage, layout, |v| v as u8); + Ok(Self::U8(data)) + } + (Self::F64(storage), DType::U8) => { + let data = unary_map(storage, layout, |v| v as u8); + Ok(Self::U8(data)) + } + (Self::U32(storage), DType::U8) => { + let data = unary_map(storage, layout, |v| v as u8); + Ok(Self::U8(data)) + } + (Self::I64(storage), DType::U8) => { + let data = unary_map(storage, layout, |v| v as u8); + Ok(Self::U8(data)) + } + (Self::U8(storage), DType::U32) => { + let data = unary_map(storage, layout, |v| v as u32); + Ok(Self::U32(data)) + } + (Self::U32(storage), DType::U32) => { + let data = unary_map(storage, layout, |v| v); + Ok(Self::U32(data)) + } + (Self::I64(storage), DType::U32) => { + let data = unary_map(storage, layout, |v| v as u32); + Ok(Self::U32(data)) + } + (Self::BF16(storage), DType::U32) => { + let data = unary_map(storage, layout, |v| v.to_f32() as u32); + Ok(Self::U32(data)) + } + (Self::F16(storage), DType::U32) => { + let data = unary_map(storage, layout, |v| v.to_f32() as u32); + Ok(Self::U32(data)) + } + (Self::F32(storage), DType::U32) => { + let data = unary_map(storage, layout, |v| v as u32); + Ok(Self::U32(data)) + } + (Self::F64(storage), DType::U32) => { + let data = unary_map(storage, layout, |v| v as u32); + Ok(Self::U32(data)) + } + (Self::U8(storage), DType::I64) => { + let data = unary_map(storage, layout, |v| v as i64); + Ok(Self::I64(data)) + } + (Self::U32(storage), DType::I64) => { + let data = unary_map(storage, layout, |v| v as i64); + Ok(Self::I64(data)) + } + (Self::I64(storage), DType::I64) => { + let data = unary_map(storage, layout, |v| v); + Ok(Self::I64(data)) + } + (Self::BF16(storage), DType::I64) => { + let data = unary_map(storage, layout, |v| v.to_f32() as i64); + Ok(Self::I64(data)) + } + (Self::F16(storage), DType::I64) => { + let data = unary_map(storage, layout, |v| v.to_f32() as i64); + Ok(Self::I64(data)) + } + (Self::F32(storage), DType::I64) => { + let data = unary_map(storage, layout, |v| v as i64); + Ok(Self::I64(data)) + } + (Self::F64(storage), DType::I64) => { + let data = unary_map(storage, layout, |v| v as i64); + Ok(Self::I64(data)) + } + (Self::U8(storage), DType::F64) => { + let data = unary_map(storage, layout, |v| v as f64); + Ok(Self::F64(data)) + } + (Self::U32(storage), DType::F64) => { + let data = unary_map(storage, layout, |v| v as f64); + Ok(Self::F64(data)) + } + (Self::I64(storage), DType::F64) => { + let data = unary_map(storage, layout, |v| v as f64); + Ok(Self::F64(data)) + } + (Self::BF16(storage), DType::F64) => { + let data = unary_map(storage, layout, |v| v.to_f64()); + Ok(Self::F64(data)) + } + (Self::F16(storage), DType::F64) => { + let data = unary_map(storage, layout, |v| v.to_f64()); + Ok(Self::F64(data)) + } + (Self::F32(storage), DType::F64) => { + let data = unary_map(storage, layout, |v| v as f64); + Ok(Self::F64(data)) + } + (Self::F64(storage), DType::F64) => { + let data = unary_map(storage, layout, |v| v); + Ok(Self::F64(data)) + } + } + } + + fn reduce_op(&self, op: ReduceOp, layout: &Layout, reduce_dims: &[usize]) -> Result { + match op { + ReduceOp::Sum => { + let src_dims = layout.dims(); + let mut dst_dims = src_dims.to_vec(); + for &dim in reduce_dims.iter() { + dst_dims[dim] = 1; + } + let dst_shape = Shape::from(dst_dims); + let mut reduce_dims = reduce_dims.to_vec(); + // Sort the reduce_dims as they have to be processed from left to right when converting the + // indexes. + reduce_dims.sort(); + let reduce_dims_and_stride: Vec<_> = reduce_dims + .iter() + .map(|&d| (src_dims[d], src_dims[d + 1..].iter().product::())) + .collect(); + ReduceSum { + dst_shape: &dst_shape, + reduce_dims: &reduce_dims, + reduce_dims_and_stride, + } + .map(self, layout) + } + ReduceOp::Min | ReduceOp::ArgMin | ReduceOp::Max | ReduceOp::ArgMax => { + let reduce_dim_index = match reduce_dims { + [reduce_dim_index] => *reduce_dim_index, + _ => { + let op = match op { + ReduceOp::Min => "min", + ReduceOp::ArgMin => "argmin", + ReduceOp::Max => "max", + ReduceOp::ArgMax => "argmax", + _ => unreachable!(), + }; + let dims = reduce_dims.to_vec(); + Err(Error::OnlySingleDimension { op, dims })? + } + }; + let (use_min, return_index) = match op { + ReduceOp::Min => (true, false), + ReduceOp::ArgMin => (true, true), + ReduceOp::Max => (false, false), + ReduceOp::ArgMax => (false, true), + _ => unreachable!(), + }; + ReduceIndex { + reduce_dim_index, + use_min, + return_index, + } + .map(self, layout) + } + } + } + + fn cmp(&self, op: CmpOp, rhs: &Self, lhs_l: &Layout, rhs_l: &Layout) -> Result { + Cmp(op).map(self, lhs_l, rhs, rhs_l) + } + + fn affine(&self, layout: &Layout, mul: f64, add: f64) -> Result { + Affine(mul, add).map(self, layout) + } + + fn avg_pool2d( + &self, + layout: &Layout, + kernel_size: (usize, usize), + stride: (usize, usize), + ) -> Result { + AvgPool2D(kernel_size, stride).map(self, layout) + } + + fn max_pool2d( + &self, + layout: &Layout, + kernel_size: (usize, usize), + stride: (usize, usize), + ) -> Result { + MaxPool2D(kernel_size, stride).map(self, layout) + } + + fn upsample_nearest1d(&self, layout: &Layout, sz: usize) -> Result { + UpsampleNearest1D(sz).map(self, layout) + } + + fn upsample_nearest2d(&self, layout: &Layout, h: usize, w: usize) -> Result { + UpsampleNearest2D(h, w).map(self, layout) + } + + fn powf(&self, layout: &Layout, e: f64) -> Result { + use num_traits::Float; + // TODO: Have some generic map for functions that apply on num_traits::Float elements. + match self { + Self::BF16(storage) => { + let data = unary_map(storage, layout, |v| v.powf(bf16::from_f64(e))); + Ok(Self::BF16(data)) + } + Self::F16(storage) => { + let data = unary_map(storage, layout, |v| v.powf(f16::from_f64(e))); + Ok(Self::F16(data)) + } + Self::F32(storage) => { + let data = unary_map(storage, layout, |v| v.powf(e as f32)); + Ok(Self::F32(data)) + } + Self::F64(storage) => { + let data = unary_map(storage, layout, |v| v.powf(e)); + Ok(Self::F64(data)) + } + Self::U8(_) => Err(Error::UnsupportedDTypeForOp(DType::U8, "elu").bt()), + Self::U32(_) => Err(Error::UnsupportedDTypeForOp(DType::U32, "elu").bt()), + Self::I64(_) => Err(Error::UnsupportedDTypeForOp(DType::I64, "elu").bt()), + } + } + + fn elu(&self, layout: &Layout, alpha: f64) -> Result { + // TODO: Have some generic map for functions that apply on num_traits::Float elements. + match self { + Self::BF16(storage) => { + let data = unary_map(storage, layout, |v| elu(v, bf16::from_f64(alpha))); + Ok(Self::BF16(data)) + } + Self::F16(storage) => { + let data = unary_map(storage, layout, |v| elu(v, f16::from_f64(alpha))); + Ok(Self::F16(data)) + } + Self::F32(storage) => { + let data = unary_map(storage, layout, |v| elu(v, f32::from_f64(alpha))); + Ok(Self::F32(data)) + } + Self::F64(storage) => { + let data = unary_map(storage, layout, |v| elu(v, alpha)); + Ok(Self::F64(data)) + } + Self::U8(_) => Err(Error::UnsupportedDTypeForOp(DType::U8, "elu").bt()), + Self::U32(_) => Err(Error::UnsupportedDTypeForOp(DType::U32, "elu").bt()), + Self::I64(_) => Err(Error::UnsupportedDTypeForOp(DType::I64, "elu").bt()), + } + } + + fn unary_impl(&self, layout: &Layout) -> Result { + match self { + Self::BF16(storage) => { + if B::BF16_VEC { + let data = unary_map_vec(storage, layout, B::bf16, B::bf16_vec); + Ok(Self::BF16(data)) + } else { + let data = unary_map(storage, layout, B::bf16); + Ok(Self::BF16(data)) + } + } + Self::F16(storage) => { + if B::F16_VEC { + let data = unary_map_vec(storage, layout, B::f16, B::f16_vec); + Ok(Self::F16(data)) + } else { + let data = unary_map(storage, layout, B::f16); + Ok(Self::F16(data)) + } + } + Self::F32(storage) => { + if B::F32_VEC { + let data = unary_map_vec(storage, layout, B::f32, B::f32_vec); + Ok(Self::F32(data)) + } else { + let data = unary_map(storage, layout, B::f32); + Ok(Self::F32(data)) + } + } + Self::F64(storage) => { + if B::F64_VEC { + let data = unary_map_vec(storage, layout, B::f64, B::f64_vec); + Ok(Self::F64(data)) + } else { + let data = unary_map(storage, layout, B::f64); + Ok(Self::F64(data)) + } + } + Self::U8(storage) => { + let data = unary_map(storage, layout, B::u8); + Ok(Self::U8(data)) + } + Self::U32(storage) => { + let data = unary_map(storage, layout, B::u32); + Ok(Self::U32(data)) + } + Self::I64(storage) => { + let data = unary_map(storage, layout, B::i64); + Ok(Self::I64(data)) + } + } + } + + fn binary_impl( + &self, + rhs: &Self, + lhs_l: &Layout, + rhs_l: &Layout, + ) -> Result { + match (self, rhs) { + (Self::BF16(lhs), Self::BF16(rhs)) => { + let data = if B::BF16_VEC { + binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::bf16, B::bf16_vec) + } else { + binary_map(lhs_l, rhs_l, lhs, rhs, B::bf16) + }; + Ok(Self::BF16(data)) + } + (Self::F16(lhs), Self::F16(rhs)) => { + let data = if B::F16_VEC { + binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::f16, B::f16_vec) + } else { + binary_map(lhs_l, rhs_l, lhs, rhs, B::f16) + }; + Ok(Self::F16(data)) + } + (Self::F32(lhs), Self::F32(rhs)) => { + let data = if B::F32_VEC { + binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::f32, B::f32_vec) + } else { + binary_map(lhs_l, rhs_l, lhs, rhs, B::f32) + }; + Ok(Self::F32(data)) + } + (Self::F64(lhs), Self::F64(rhs)) => { + let data = if B::F64_VEC { + binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::f64, B::f64_vec) + } else { + binary_map(lhs_l, rhs_l, lhs, rhs, B::f64) + }; + Ok(Self::F64(data)) + } + (Self::U32(lhs), Self::U32(rhs)) => { + let data = if B::U32_VEC { + binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::u32, B::u32_vec) + } else { + binary_map(lhs_l, rhs_l, lhs, rhs, B::u32) + }; + Ok(Self::U32(data)) + } + (Self::I64(lhs), Self::I64(rhs)) => { + let data = if B::I64_VEC { + binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::i64, B::i64_vec) + } else { + binary_map(lhs_l, rhs_l, lhs, rhs, B::i64) + }; + Ok(Self::I64(data)) + } + (Self::U8(lhs), Self::U8(rhs)) => { + let data = if B::U8_VEC { + binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::u8, B::u8_vec) + } else { + binary_map(lhs_l, rhs_l, lhs, rhs, B::u8) + }; + Ok(Self::U8(data)) + } + _ => { + // This should be covered by the dtype check above. + Err(Error::DTypeMismatchBinaryOp { + lhs: self.dtype(), + rhs: rhs.dtype(), + op: B::NAME, + } + .bt()) + } + } + } + + fn copy_strided_src(&self, dst: &mut Self, dst_offset: usize, src_l: &Layout) -> Result<()> { + match (self, dst) { + (Self::U8(src), Self::U8(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), + (Self::U32(src), Self::U32(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), + (Self::I64(src), Self::I64(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), + (Self::BF16(src), Self::BF16(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), + (Self::F16(src), Self::F16(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), + (Self::F32(src), Self::F32(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), + (Self::F64(src), Self::F64(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), + (_, dst) => { + // This should be covered by the dtype check above. + return Err(Error::DTypeMismatchBinaryOp { + lhs: self.dtype(), + rhs: dst.dtype(), + op: "copy_strided", + } + .bt()); + } + } + Ok(()) + } + + fn where_cond( + &self, + layout: &Layout, + t: &Self, + t_l: &Layout, + f: &Self, + f_l: &Layout, + ) -> Result { + match self { + Self::U8(pred) => WCond(pred, layout).map(t, t_l, f, f_l), + Self::U32(pred) => WCond(pred, layout).map(t, t_l, f, f_l), + Self::I64(pred) => WCond(pred, layout).map(t, t_l, f, f_l), + _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), "where-cond")), + } + } + + fn conv1d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConv1D, + ) -> Result { + if !USE_IM2COL_CONV1D { + return Conv1D(params).map(self, l, kernel, kernel_l); + } + let op = Im2Col1D { + l_k: params.k_size, + padding: params.padding, + stride: params.stride, + dilation: params.dilation, + }; + let col = op.map(self, l)?; + let b = params.b_size; + let n = params.c_out; + let l_out = params.l_out(); + let k = op.l_k * params.c_in; + let m = l_out; + let col_l = Layout::contiguous((b, m, k)); + let res = if kernel_l.is_contiguous() { + let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) + .transpose(1, 2)? + .broadcast_as((b, k, n))?; + col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? + } else { + // Make the kernel contiguous if not already the case. + let mut kernel_c = self.device().zeros_impl(kernel_l.shape(), kernel.dtype())?; + kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?; + let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) + .transpose(1, 2)? + .broadcast_as((b, k, n))?; + col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? + }; + let res_l = Layout::contiguous((b, l_out, params.c_out)).transpose(1, 2)?; + let mut res_t = self.device().zeros_impl(res_l.shape(), res.dtype())?; + res.copy_strided_src(&mut res_t, 0, &res_l)?; + Ok(res_t) + } + + fn conv_transpose1d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConvTranspose1D, + ) -> Result { + ConvTranspose1D(params).map(self, l, kernel, kernel_l) + } + + fn conv2d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConv2D, + ) -> Result { + if !USE_IM2COL_CONV2D { + return Conv2D(params).map(self, l, kernel, kernel_l); + } + let op = Im2Col { + h_k: params.k_h, + w_k: params.k_w, + padding: params.padding, + stride: params.stride, + dilation: params.dilation, + }; + let col = op.map(self, l)?; + let b = params.b_size; + let n = params.c_out; + let (h_out, w_out) = (params.out_h(), params.out_w()); + let k = op.h_k * op.w_k * params.c_in; + let m = h_out * w_out; + let col_l = Layout::contiguous((b, m, k)); + let res = if kernel_l.is_contiguous() { + let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) + .transpose(1, 2)? + .broadcast_as((b, k, n))?; + col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? + } else { + // Make the kernel contiguous if not already the case. + let mut kernel_c = self.device().zeros_impl(kernel_l.shape(), kernel.dtype())?; + kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?; + let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) + .transpose(1, 2)? + .broadcast_as((b, k, n))?; + col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? + }; + let res_l = Layout::contiguous((b, h_out, w_out, params.c_out)) + .transpose(1, 2)? + .transpose(1, 3)?; + let mut res_t = self.device().zeros_impl(res_l.shape(), res.dtype())?; + res.copy_strided_src(&mut res_t, 0, &res_l)?; + Ok(res_t) + } + + fn conv_transpose2d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConvTranspose2D, + ) -> Result { + ConvTranspose2D(params).map(self, l, kernel, kernel_l) + } + + fn index_select(&self, ids: &Self, l: &Layout, ids_l: &Layout, dim: usize) -> Result { + match ids { + Self::U8(ids) => IndexSelect { ids, ids_l, dim }.map(self, l), + Self::U32(ids) => IndexSelect { ids, ids_l, dim }.map(self, l), + Self::I64(ids) => IndexSelect { ids, ids_l, dim }.map(self, l), + _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), "index-select").bt()), + } + } + + fn gather(&self, l: &Layout, ids: &Self, ids_l: &Layout, dim: usize) -> Result { + match ids { + Self::U8(ids) => Gather { ids, ids_l, dim }.map(self, l), + Self::U32(ids) => Gather { ids, ids_l, dim }.map(self, l), + Self::I64(ids) => Gather { ids, ids_l, dim }.map(self, l), + _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), "gather").bt()), + } + } + + fn scatter_add( + &self, + l: &Layout, + ids: &Self, + ids_l: &Layout, + src: &Self, + src_l: &Layout, + dim: usize, + ) -> Result { + match ids { + Self::U8(ids) => ScatterAdd { ids, ids_l, dim }.map(self, l, src, src_l), + Self::U32(ids) => ScatterAdd { ids, ids_l, dim }.map(self, l, src, src_l), + Self::I64(ids) => ScatterAdd { ids, ids_l, dim }.map(self, l, src, src_l), + _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), "scatter-add").bt()), + } + } + + fn index_add( + &self, + l: &Layout, + ids: &Self, + ids_l: &Layout, + src: &Self, + src_l: &Layout, + dim: usize, + ) -> Result { + match ids { + Self::U8(ids) => { + let ids = match ids_l.contiguous_offsets() { + Some((a, b)) => &ids[a..b], + None => Err(Error::RequiresContiguous { op: "index-add" }.bt())?, + }; + IndexAdd { ids, dim }.map(self, l, src, src_l) + } + Self::U32(ids) => { + let ids = match ids_l.contiguous_offsets() { + Some((a, b)) => &ids[a..b], + None => Err(Error::RequiresContiguous { op: "index-add" }.bt())?, + }; + IndexAdd { ids, dim }.map(self, l, src, src_l) + } + Self::I64(ids) => { + let ids = match ids_l.contiguous_offsets() { + Some((a, b)) => &ids[a..b], + None => Err(Error::RequiresContiguous { op: "index-add" }.bt())?, + }; + IndexAdd { ids, dim }.map(self, l, src, src_l) + } + _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), "index-add").bt()), + } + } + + fn matmul( + &self, + rhs: &Self, + bmnk: (usize, usize, usize, usize), + lhs_l: &Layout, + rhs_l: &Layout, + ) -> Result { + MatMul(bmnk).map(self, lhs_l, rhs, rhs_l) + } + + fn device(&self) -> &Self::Device { + &CpuDevice + } + + fn try_clone(&self, _: &Layout) -> Result { + Ok(self.clone()) + } + + fn to_cpu_storage(&self) -> Result { + Ok(self.clone()) + } +} + +impl BackendDevice for CpuDevice { + type Storage = CpuStorage; + + fn location(&self) -> crate::DeviceLocation { + crate::DeviceLocation::Cpu + } + + fn same_device(&self, _: &Self) -> bool { + true + } + + fn storage_from_cpu_storage(&self, s: &CpuStorage) -> Result { + Ok(s.clone()) + } + + fn new(_: usize) -> Result { + Ok(Self) + } + + fn set_seed(&self, _seed: u64) -> Result<()> { + crate::bail!("cannot seed the CPU rng with set_seed") + } + + fn rand_uniform(&self, shape: &Shape, dtype: DType, min: f64, max: f64) -> Result { + use rand::prelude::*; + + let elem_count = shape.elem_count(); + let mut rng = rand::thread_rng(); + match dtype { + DType::U8 | DType::U32 | DType::I64 => { + Err(Error::UnsupportedDTypeForOp(dtype, "rand_uniform").bt()) + } + DType::BF16 => { + let mut data = Vec::with_capacity(elem_count); + let uniform = + rand::distributions::Uniform::new(bf16::from_f64(min), bf16::from_f64(max)); + for _i in 0..elem_count { + data.push(rng.sample::(uniform)) + } + Ok(CpuStorage::BF16(data)) + } + DType::F16 => { + let mut data = Vec::with_capacity(elem_count); + let uniform = + rand::distributions::Uniform::new(f16::from_f64(min), f16::from_f64(max)); + for _i in 0..elem_count { + data.push(rng.sample::(uniform)) + } + Ok(CpuStorage::F16(data)) + } + DType::F32 => { + let mut data = Vec::with_capacity(elem_count); + let uniform = rand::distributions::Uniform::new(min as f32, max as f32); + for _i in 0..elem_count { + data.push(rng.sample::(uniform)) + } + Ok(CpuStorage::F32(data)) + } + DType::F64 => { + let mut data = Vec::with_capacity(elem_count); + let uniform = rand::distributions::Uniform::new(min, max); + for _i in 0..elem_count { + data.push(rng.sample::(uniform)) + } + Ok(CpuStorage::F64(data)) + } + } + } + + fn rand_normal(&self, shape: &Shape, dtype: DType, mean: f64, std: f64) -> Result { + use rand::prelude::*; + + let elem_count = shape.elem_count(); + let mut rng = rand::thread_rng(); + match dtype { + DType::U8 | DType::U32 | DType::I64 => { + Err(Error::UnsupportedDTypeForOp(dtype, "rand_normal").bt()) + } + DType::BF16 => { + let mut data = Vec::with_capacity(elem_count); + let normal = rand_distr::Normal::new(bf16::from_f64(mean), bf16::from_f64(std)) + .map_err(Error::wrap)?; + for _i in 0..elem_count { + data.push(normal.sample(&mut rng)) + } + Ok(CpuStorage::BF16(data)) + } + DType::F16 => { + let mut data = Vec::with_capacity(elem_count); + let normal = rand_distr::Normal::new(f16::from_f64(mean), f16::from_f64(std)) + .map_err(Error::wrap)?; + for _i in 0..elem_count { + data.push(normal.sample(&mut rng)) + } + Ok(CpuStorage::F16(data)) + } + DType::F32 => { + let mut data = Vec::with_capacity(elem_count); + let normal = + rand_distr::Normal::new(mean as f32, std as f32).map_err(Error::wrap)?; + for _i in 0..elem_count { + data.push(normal.sample(&mut rng)) + } + Ok(CpuStorage::F32(data)) + } + DType::F64 => { + let mut data = Vec::with_capacity(elem_count); + let normal = rand_distr::Normal::new(mean, std).map_err(Error::wrap)?; + for _i in 0..elem_count { + data.push(normal.sample(&mut rng)) + } + Ok(CpuStorage::F64(data)) + } + } + } + + fn ones_impl(&self, shape: &Shape, dtype: DType) -> Result { + let elem_count = shape.elem_count(); + let storage = match dtype { + DType::U8 => CpuStorage::U8(vec![1u8; elem_count]), + DType::U32 => CpuStorage::U32(vec![1u32; elem_count]), + DType::I64 => CpuStorage::I64(vec![1i64; elem_count]), + DType::BF16 => CpuStorage::BF16(vec![bf16::ONE; elem_count]), + DType::F16 => CpuStorage::F16(vec![f16::ONE; elem_count]), + DType::F32 => CpuStorage::F32(vec![1f32; elem_count]), + DType::F64 => CpuStorage::F64(vec![1f64; elem_count]), + }; + Ok(storage) + } + + fn zeros_impl(&self, shape: &Shape, dtype: DType) -> Result { + let elem_count = shape.elem_count(); + let storage = match dtype { + DType::U8 => CpuStorage::U8(vec![0u8; elem_count]), + DType::U32 => CpuStorage::U32(vec![0u32; elem_count]), + DType::I64 => CpuStorage::I64(vec![0i64; elem_count]), + DType::BF16 => CpuStorage::BF16(vec![bf16::ZERO; elem_count]), + DType::F16 => CpuStorage::F16(vec![f16::ZERO; elem_count]), + DType::F32 => CpuStorage::F32(vec![0f32; elem_count]), + DType::F64 => CpuStorage::F64(vec![0f64; elem_count]), + }; + Ok(storage) + } +} + +#[macro_export] +macro_rules! map_dtype { + ($name:expr, $storage:ident, $fn:expr, ($($dtypes:ident),+)) => { + match $storage { + $(CpuStorage::$dtypes(__e) => CpuStorage::$dtypes($fn(__e)),)* + s => Err(Error::UnsupportedDTypeForOp(s.dtype(), $name).bt())?, + } + }; +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/cuda_backend.rs b/v2/dependency-patches/candle-core-0.4.1/src/cuda_backend.rs new file mode 100644 index 0000000000..b7756fa6a4 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/cuda_backend.rs @@ -0,0 +1,2249 @@ +use crate::backend::{BackendDevice, BackendStorage}; +use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; +use crate::{CpuStorage, DType, Layout, Result, Shape, WithDType}; +pub use candle_kernels as kernels; +pub use cudarc; +use cudarc::cublas::{Gemm, GemmConfig, StridedBatchedConfig}; +use cudarc::driver::{ + CudaFunction, CudaSlice, DevicePtr, DeviceRepr, DeviceSlice, LaunchAsync, LaunchConfig, + ValidAsZeroBits, +}; +use half::{bf16, f16}; +use std::sync::{Arc, Mutex}; + +/// cudarc related errors +#[derive(thiserror::Error, Debug)] +pub enum CudaError { + #[error(transparent)] + Cuda(#[from] cudarc::driver::DriverError), + + #[error(transparent)] + Compiler(#[from] cudarc::nvrtc::CompileError), + + #[error(transparent)] + Cublas(#[from] cudarc::cublas::result::CublasError), + + #[error(transparent)] + Curand(#[from] cudarc::curand::result::CurandError), + + #[error("missing kernel '{module_name}'")] + MissingKernel { module_name: String }, + + #[error("unsupported dtype {dtype:?} for {op}")] + UnsupportedDtype { dtype: DType, op: &'static str }, + + #[error("internal error '{0}'")] + InternalError(&'static str), + + #[error("matmul is only supported for contiguous tensors lstride: {lhs_stride:?} rstride: {rhs_stride:?} mnk: {mnk:?}")] + MatMulNonContiguous { + lhs_stride: Vec, + rhs_stride: Vec, + mnk: (usize, usize, usize), + }, + + #[error("{msg}, expected: {expected:?}, got: {got:?}")] + UnexpectedDType { + msg: &'static str, + expected: DType, + got: DType, + }, + + #[error("{cuda} when loading {module_name}")] + Load { + cuda: cudarc::driver::DriverError, + module_name: String, + }, +} + +impl From for crate::Error { + fn from(val: CudaError) -> Self { + crate::Error::Cuda(Box::new(val)).bt() + } +} + +/// Unique identifier for cuda devices. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct DeviceId(usize); + +impl DeviceId { + fn new() -> Self { + // https://users.rust-lang.org/t/idiomatic-rust-way-to-generate-unique-id/33805 + use std::sync::atomic; + static COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::new(1); + Self(COUNTER.fetch_add(1, atomic::Ordering::Relaxed)) + } +} + +struct CudaRng(cudarc::curand::CudaRng); +unsafe impl Send for CudaRng {} + +#[derive(Clone)] +pub struct CudaDevice { + id: DeviceId, + device: Arc, + blas: Arc, + curand: Arc>, +} + +impl std::fmt::Debug for CudaDevice { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "CudaDevice({:?})", self.id) + } +} + +impl std::ops::Deref for CudaDevice { + type Target = Arc; + + fn deref(&self) -> &Self::Target { + &self.device + } +} + +pub trait WrapErr { + fn w(self) -> std::result::Result; +} + +impl> WrapErr for std::result::Result { + fn w(self) -> std::result::Result { + self.map_err(|e| crate::Error::Cuda(Box::new(e.into()))) + } +} + +impl CudaDevice { + pub fn cuda_device(&self) -> Arc { + self.device.clone() + } + + pub fn id(&self) -> DeviceId { + self.id + } + + fn const_impl(&self, v: f64, shape: &Shape, dtype: DType) -> Result { + let elem_count = shape.elem_count(); + let cfg = LaunchConfig::for_num_elems(elem_count as u32); + let slice = match dtype { + DType::U8 => { + // SAFETY: Set later by running the fill kernel. + let data = unsafe { self.alloc::(elem_count) }.w()?; + let func = self.get_or_load_func("fill_u8", kernels::FILL)?; + let params = (&data, v as u8, elem_count); + unsafe { func.launch(cfg, params) }.w()?; + CudaStorageSlice::U8(data) + } + DType::U32 => { + // SAFETY: Set later by running the fill kernel. + let data = unsafe { self.alloc::(elem_count) }.w()?; + let func = self.get_or_load_func("fill_u32", kernels::FILL)?; + let params = (&data, v as u32, elem_count); + unsafe { func.launch(cfg, params) }.w()?; + CudaStorageSlice::U32(data) + } + DType::I64 => { + // SAFETY: Set later by running the fill kernel. + let data = unsafe { self.alloc::(elem_count) }.w()?; + let func = self.get_or_load_func("fill_i64", kernels::FILL)?; + let params = (&data, v as i64, elem_count); + unsafe { func.launch(cfg, params) }.w()?; + CudaStorageSlice::I64(data) + } + DType::BF16 => { + // SAFETY: Set later by running the fill kernel. + let data = unsafe { self.alloc::(elem_count) }.w()?; + let func = self.get_or_load_func("fill_bf16", kernels::FILL)?; + let params = (&data, bf16::from_f64(v), elem_count); + unsafe { func.launch(cfg, params) }.w()?; + CudaStorageSlice::BF16(data) + } + DType::F16 => { + // SAFETY: Set later by running the fill kernel. + let data = unsafe { self.alloc::(elem_count) }.w()?; + let func = self.get_or_load_func("fill_f16", kernels::FILL)?; + let params = (&data, f16::from_f64(v), elem_count); + unsafe { func.launch(cfg, params) }.w()?; + CudaStorageSlice::F16(data) + } + DType::F32 => { + // SAFETY: Set later by running the fill kernel. + let data = unsafe { self.alloc::(elem_count) }.w()?; + let func = self.get_or_load_func("fill_f32", kernels::FILL)?; + let params = (&data, v as f32, elem_count); + unsafe { func.launch(cfg, params) }.w()?; + CudaStorageSlice::F32(data) + } + DType::F64 => { + // SAFETY: Set later by running the fill kernel. + let data = unsafe { self.alloc::(elem_count) }.w()?; + let func = self.get_or_load_func("fill_f64", kernels::FILL)?; + let params = (&data, v, elem_count); + unsafe { func.launch(cfg, params) }.w()?; + CudaStorageSlice::F64(data) + } + }; + Ok(CudaStorage { + slice, + device: self.clone(), + }) + } + + pub fn get_or_load_func(&self, module_name: &str, ptx: &'static str) -> Result { + if !self.has_func(module_name, module_name) { + // Leaking the string here is a bit sad but we need a &'static str and this is only + // done once per kernel name. + let static_module_name = Box::leak(module_name.to_string().into_boxed_str()); + self.load_ptx(ptx.into(), module_name, &[static_module_name]) + .map_err(|cuda| CudaError::Load { + cuda, + module_name: module_name.to_string(), + }) + .w()?; + } + self.get_func(module_name, module_name) + // Clippy recommends this `ok_or` rather than `ok_or_else` so hopefully the compiler is + // able to only build the error value if needed. + .ok_or(CudaError::MissingKernel { + module_name: module_name.to_string(), + }) + .w() + } +} + +impl BackendDevice for CudaDevice { + type Storage = CudaStorage; + + fn new(ordinal: usize) -> Result { + let device = cudarc::driver::CudaDevice::new(ordinal).w()?; + let blas = cudarc::cublas::CudaBlas::new(device.clone()).w()?; + let curand = cudarc::curand::CudaRng::new(299792458, device.clone()).w()?; + Ok(Self { + id: DeviceId::new(), + device, + blas: Arc::new(blas), + curand: Arc::new(Mutex::new(CudaRng(curand))), + }) + } + + fn set_seed(&self, seed: u64) -> Result<()> { + // We do not call set_seed but instead create a new curand object. This ensures that the + // state will be identical and the same random numbers will be generated. + let mut curand = self.curand.lock().unwrap(); + curand.0 = cudarc::curand::CudaRng::new(seed, self.device.clone()).w()?; + Ok(()) + } + + fn location(&self) -> crate::DeviceLocation { + crate::DeviceLocation::Cuda { + gpu_id: self.device.ordinal(), + } + } + + fn same_device(&self, rhs: &Self) -> bool { + self.id == rhs.id + } + + fn zeros_impl(&self, shape: &Shape, dtype: DType) -> Result { + let elem_count = shape.elem_count(); + let slice = match dtype { + DType::U8 => { + let data = self.alloc_zeros::(elem_count).w()?; + CudaStorageSlice::U8(data) + } + DType::U32 => { + let data = self.alloc_zeros::(elem_count).w()?; + CudaStorageSlice::U32(data) + } + DType::I64 => { + let data = self.alloc_zeros::(elem_count).w()?; + CudaStorageSlice::I64(data) + } + DType::BF16 => { + let data = self.alloc_zeros::(elem_count).w()?; + CudaStorageSlice::BF16(data) + } + DType::F16 => { + let data = self.alloc_zeros::(elem_count).w()?; + CudaStorageSlice::F16(data) + } + DType::F32 => { + let data = self.alloc_zeros::(elem_count).w()?; + CudaStorageSlice::F32(data) + } + DType::F64 => { + let data = self.alloc_zeros::(elem_count).w()?; + CudaStorageSlice::F64(data) + } + }; + Ok(CudaStorage { + slice, + device: self.clone(), + }) + } + + fn rand_uniform(&self, shape: &Shape, dtype: DType, lo: f64, up: f64) -> Result { + let elem_count = shape.elem_count(); + let curand = self.curand.lock().unwrap(); + let slice = match dtype { + // TODO: Add support for F16 and BF16 though this is likely to require some upstream + // cudarc changes. + DType::U8 | DType::U32 | DType::I64 | DType::F16 | DType::BF16 => { + Err(CudaError::UnsupportedDtype { + dtype, + op: "rand_uniform", + }) + .w()? + } + DType::F32 => { + let mut data = unsafe { self.alloc::(elem_count) }.w()?; + curand.0.fill_with_uniform(&mut data).w()?; + CudaStorageSlice::F32(data) + } + DType::F64 => { + let mut data = unsafe { self.alloc::(elem_count) }.w()?; + curand.0.fill_with_uniform(&mut data).w()?; + CudaStorageSlice::F64(data) + } + }; + let slice = if lo == 0. && up == 1.0 { + slice + } else { + let layout = Layout::contiguous(shape); + Affine(up - lo, lo).map(&slice, self, &layout)? + }; + Ok(CudaStorage { + slice, + device: self.clone(), + }) + } + + fn rand_normal(&self, shape: &Shape, dtype: DType, mean: f64, std: f64) -> Result { + // TODO: Add support for F16 and BF16 though this is likely to require some upstream + // cudarc changes. + let elem_count = shape.elem_count(); + let curand = self.curand.lock().unwrap(); + // curand can only generate an odd number of values. + // https://github.com/huggingface/candle/issues/734 + let elem_count_round = if elem_count % 2 == 1 { + elem_count + 1 + } else { + elem_count + }; + let slice = match dtype { + DType::U8 | DType::U32 | DType::I64 | DType::F16 | DType::BF16 => { + Err(CudaError::UnsupportedDtype { + dtype, + op: "rand_normal", + }) + .w()? + } + DType::F32 => { + let mut data = unsafe { self.alloc::(elem_count_round) }.w()?; + curand + .0 + .fill_with_normal(&mut data, mean as f32, std as f32) + .w()?; + CudaStorageSlice::F32(data) + } + DType::F64 => { + let mut data = unsafe { self.alloc::(elem_count_round) }.w()?; + curand.0.fill_with_normal(&mut data, mean, std).w()?; + CudaStorageSlice::F64(data) + } + }; + Ok(CudaStorage { + slice, + device: self.clone(), + }) + } + + fn ones_impl(&self, shape: &Shape, dtype: DType) -> Result { + self.const_impl(1., shape, dtype) + } + + fn storage_from_cpu_storage(&self, storage: &CpuStorage) -> Result { + let slice = match storage { + CpuStorage::U8(storage) => { + let data = self.htod_sync_copy(storage).w()?; + CudaStorageSlice::U8(data) + } + CpuStorage::U32(storage) => { + let data = self.htod_sync_copy(storage).w()?; + CudaStorageSlice::U32(data) + } + CpuStorage::I64(storage) => { + let data = self.htod_sync_copy(storage).w()?; + CudaStorageSlice::I64(data) + } + CpuStorage::BF16(storage) => { + let data = self.htod_sync_copy(storage).w()?; + CudaStorageSlice::BF16(data) + } + CpuStorage::F16(storage) => { + let data = self.htod_sync_copy(storage).w()?; + CudaStorageSlice::F16(data) + } + CpuStorage::F32(storage) => { + let data = self.htod_sync_copy(storage).w()?; + CudaStorageSlice::F32(data) + } + CpuStorage::F64(storage) => { + let data = self.htod_sync_copy(storage).w()?; + CudaStorageSlice::F64(data) + } + }; + Ok(CudaStorage { + slice, + device: self.clone(), + }) + } +} + +#[derive(Debug)] +pub enum CudaStorageSlice { + U8(CudaSlice), + U32(CudaSlice), + I64(CudaSlice), + BF16(CudaSlice), + F16(CudaSlice), + F32(CudaSlice), + F64(CudaSlice), +} +type S = CudaStorageSlice; + +pub trait Map1 { + fn f( + &self, + src: &CudaSlice, + dev: &CudaDevice, + layout: &Layout, + ) -> Result>; + + fn map(&self, s: &S, d: &CudaDevice, l: &Layout) -> Result { + let out = match s { + S::U8(s) => S::U8(self.f(s, d, l)?), + S::U32(s) => S::U32(self.f(s, d, l)?), + S::I64(s) => S::I64(self.f(s, d, l)?), + S::BF16(s) => S::BF16(self.f(s, d, l)?), + S::F16(s) => S::F16(self.f(s, d, l)?), + S::F32(s) => S::F32(self.f(s, d, l)?), + S::F64(s) => S::F64(self.f(s, d, l)?), + }; + Ok(out) + } +} + +pub trait Map2 { + fn f( + &self, + src1: &CudaSlice, + layout1: &Layout, + src2: &CudaSlice, + layout2: &Layout, + dev: &CudaDevice, + ) -> Result>; + + fn map(&self, s1: &S, l1: &Layout, s2: &S, l2: &Layout, d: &CudaDevice) -> Result { + let out = match (s1, s2) { + (S::U8(s1), S::U8(s2)) => S::U8(self.f(s1, l1, s2, l2, d)?), + (S::U32(s1), S::U32(s2)) => S::U32(self.f(s1, l1, s2, l2, d)?), + (S::I64(s1), S::I64(s2)) => S::I64(self.f(s1, l1, s2, l2, d)?), + (S::BF16(s1), S::BF16(s2)) => S::BF16(self.f(s1, l1, s2, l2, d)?), + (S::F16(s1), S::F16(s2)) => S::F16(self.f(s1, l1, s2, l2, d)?), + (S::F32(s1), S::F32(s2)) => S::F32(self.f(s1, l1, s2, l2, d)?), + (S::F64(s1), S::F64(s2)) => S::F64(self.f(s1, l1, s2, l2, d)?), + _ => Err(CudaError::InternalError("dtype mismatch in binary op"))?, + }; + Ok(out) + } +} + +pub trait Map2InPlace { + fn f( + &self, + dst: &mut CudaSlice, + dst_shape: &Shape, + src: &CudaSlice, + src_l: &Layout, + dev: &CudaDevice, + ) -> Result<()>; + + fn map( + &self, + dst: &mut S, + dst_s: &Shape, + src: &S, + src_l: &Layout, + d: &CudaDevice, + ) -> Result<()> { + match (dst, src) { + (S::U8(dst), S::U8(src)) => self.f(dst, dst_s, src, src_l, d), + (S::U32(dst), S::U32(src)) => self.f(dst, dst_s, src, src_l, d), + (S::I64(dst), S::I64(src)) => self.f(dst, dst_s, src, src_l, d), + (S::BF16(dst), S::BF16(src)) => self.f(dst, dst_s, src, src_l, d), + (S::F16(dst), S::F16(src)) => self.f(dst, dst_s, src, src_l, d), + (S::F32(dst), S::F32(src)) => self.f(dst, dst_s, src, src_l, d), + (S::F64(dst), S::F64(src)) => self.f(dst, dst_s, src, src_l, d), + _ => Err(CudaError::InternalError("dtype mismatch in binary op"))?, + } + } +} + +pub trait Map1Any { + fn f) -> S>( + &self, + src: &CudaSlice, + dev: &CudaDevice, + layout: &Layout, + wrap: W, + ) -> Result; + + fn map(&self, s: &S, d: &CudaDevice, l: &Layout) -> Result { + let out = match s { + S::U8(s) => self.f(s, d, l, S::U8)?, + S::U32(s) => self.f(s, d, l, S::U32)?, + S::I64(s) => self.f(s, d, l, S::I64)?, + S::BF16(s) => self.f(s, d, l, S::BF16)?, + S::F16(s) => self.f(s, d, l, S::F16)?, + S::F32(s) => self.f(s, d, l, S::F32)?, + S::F64(s) => self.f(s, d, l, S::F64)?, + }; + Ok(out) + } +} + +pub trait Map2Any { + fn f( + &self, + src1: &CudaSlice, + layout1: &Layout, + src2: &CudaSlice, + layout2: &Layout, + dev: &CudaDevice, + ) -> Result; + + fn map(&self, s1: &S, l1: &Layout, s2: &S, l2: &Layout, d: &CudaDevice) -> Result { + let out = match (s1, s2) { + (S::U8(s1), S::U8(s2)) => self.f(s1, l1, s2, l2, d)?, + (S::U32(s1), S::U32(s2)) => self.f(s1, l1, s2, l2, d)?, + (S::I64(s1), S::I64(s2)) => self.f(s1, l1, s2, l2, d)?, + (S::BF16(s1), S::BF16(s2)) => self.f(s1, l1, s2, l2, d)?, + (S::F16(s1), S::F16(s2)) => self.f(s1, l1, s2, l2, d)?, + (S::F32(s1), S::F32(s2)) => self.f(s1, l1, s2, l2, d)?, + (S::F64(s1), S::F64(s2)) => self.f(s1, l1, s2, l2, d)?, + _ => Err(CudaError::InternalError("dtype mismatch in binary op")).w()?, + }; + Ok(out) + } +} + +struct Clone; +impl Map1 for Clone { + fn f( + &self, + s: &CudaSlice, + _: &CudaDevice, + _: &Layout, + ) -> Result> { + s.try_clone().w() + } +} + +pub fn kernel_name(root: &str) -> String { + let dtype = T::DTYPE.as_str(); + format!("{root}_{dtype}") +} + +struct Affine(f64, f64); +impl Map1 for Affine { + fn f( + &self, + src: &CudaSlice, + dev: &CudaDevice, + layout: &Layout, + ) -> Result> { + let shape = layout.shape(); + let dims = shape.dims(); + let el = shape.elem_count(); + let cfg = LaunchConfig::for_num_elems(el as u32); + let ds = dev.htod_copy([dims, layout.stride()].concat()).w()?; + let src = &src.slice(layout.start_offset()..); + let func = dev.get_or_load_func(&kernel_name::("affine"), kernels::AFFINE)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(el) }.w()?; + let params = ( + el, + dims.len(), + &ds, + src, + &out, + T::from_f64(self.0), + T::from_f64(self.1), + ); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()?; + Ok(out) + } +} + +struct Elu(f64); +impl Map1 for Elu { + fn f( + &self, + src: &CudaSlice, + dev: &CudaDevice, + layout: &Layout, + ) -> Result> { + let shape = layout.shape(); + let dims = shape.dims(); + let el = shape.elem_count(); + let cfg = LaunchConfig::for_num_elems(el as u32); + let ds = dev.htod_copy([dims, layout.stride()].concat()).w()?; + let src = &src.slice(layout.start_offset()..); + let func = dev.get_or_load_func(&kernel_name::("uelu"), kernels::UNARY)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(el) }.w()?; + let params = (el, dims.len(), &ds, T::from_f64(self.0), src, &out); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()?; + Ok(out) + } +} + +struct Im2Col1D { + l_k: usize, + stride: usize, + dilation: usize, + padding: usize, +} + +impl Im2Col1D { + fn l_out(&self, l: usize) -> usize { + (l + 2 * self.padding - self.dilation * (self.l_k - 1) - 1) / self.stride + 1 + } +} + +impl Map1 for Im2Col1D { + fn f( + &self, + src: &CudaSlice, + dev: &CudaDevice, + layout: &Layout, + ) -> Result> { + let shape = layout.shape(); + let dims = shape.dims(); + let l_out = self.l_out(dims[2]); + let dst_el = dims[0] * l_out * dims[1] * self.l_k; + let cfg = LaunchConfig::for_num_elems(dst_el as u32); + let ds = dev.htod_copy([dims, layout.stride()].concat()).w()?; + let src = &src.slice(layout.start_offset()..); + let func = dev.get_or_load_func(&kernel_name::("im2col1d"), kernels::CONV)?; + // SAFETY: Set later by running the kernel. + let dst = unsafe { dev.alloc::(dst_el) }.w()?; + let params = ( + dst_el, + l_out, + self.l_k, + self.stride, + self.padding, + self.dilation, + &ds, + src, + &dst, + ); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()?; + Ok(dst) + } +} + +struct Im2Col { + h_k: usize, + w_k: usize, + stride: usize, + dilation: usize, + padding: usize, +} + +impl Im2Col { + fn hw_out(&self, h: usize, w: usize) -> (usize, usize) { + let h_out = (h + 2 * self.padding - self.dilation * (self.h_k - 1) - 1) / self.stride + 1; + let w_out = (w + 2 * self.padding - self.dilation * (self.w_k - 1) - 1) / self.stride + 1; + (h_out, w_out) + } +} + +impl Map1 for Im2Col { + fn f( + &self, + src: &CudaSlice, + dev: &CudaDevice, + layout: &Layout, + ) -> Result> { + let shape = layout.shape(); + let dims = shape.dims(); + let (h_out, w_out) = self.hw_out(dims[2], dims[3]); + let dst_el = dims[0] * h_out * w_out * dims[1] * self.h_k * self.w_k; + let cfg = LaunchConfig::for_num_elems(dst_el as u32); + let ds = dev.htod_copy([dims, layout.stride()].concat()).w()?; + let src = &src.slice(layout.start_offset()..); + let func = dev.get_or_load_func(&kernel_name::("im2col"), kernels::CONV)?; + // SAFETY: Set later by running the kernel. + let dst = unsafe { dev.alloc::(dst_el) }.w()?; + let params = ( + dst_el, + h_out, + w_out, + self.h_k, + self.w_k, + self.stride, + self.padding, + self.dilation, + &ds, + src, + &dst, + ); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()?; + Ok(dst) + } +} + +struct Powf(f64); +impl Map1 for Powf { + fn f( + &self, + src: &CudaSlice, + dev: &CudaDevice, + layout: &Layout, + ) -> Result> { + let shape = layout.shape(); + let dims = shape.dims(); + let el = shape.elem_count(); + let cfg = LaunchConfig::for_num_elems(el as u32); + let ds = dev.htod_copy([dims, layout.stride()].concat()).w()?; + let src = &src.slice(layout.start_offset()..); + let func = dev.get_or_load_func(&kernel_name::("upowf"), kernels::UNARY)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(el) }.w()?; + let params = (el, dims.len(), &ds, T::from_f64(self.0), src, &out); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()?; + Ok(out) + } +} + +struct Sum<'a>(&'a [usize]); +impl<'a> Map1 for Sum<'a> { + fn f( + &self, + src: &CudaSlice, + dev: &CudaDevice, + layout: &Layout, + ) -> Result> { + let shape = layout.shape(); + let src_dims = shape.dims(); + let el = shape.elem_count(); + let mut dst_el = el; + for &sum_dim in self.0.iter() { + dst_el /= src_dims[sum_dim]; + } + let mut sum_dims = self.0.to_vec(); + // Sort the sum_dims as they have to be processed from left to right when converting the + // indexes. + sum_dims.sort(); + let sum_dims_l: Vec = sum_dims.iter().map(|&d| src_dims[d]).collect(); + let sum_dims_s: Vec = sum_dims + .iter() + .map(|&d| src_dims[d + 1..].iter().product::()) + .collect(); + let cfg = LaunchConfig::for_num_elems(el as u32); + let ds = dev + .htod_copy([src_dims, layout.stride(), &sum_dims_l, &sum_dims_s].concat()) + .w()?; + let src = &src.slice(layout.start_offset()..); + let func = dev.get_or_load_func(&kernel_name::("sum"), kernels::REDUCE)?; + let out = dev.alloc_zeros::(dst_el).w()?; + let params = (el, src_dims.len(), sum_dims.len(), &ds, src, &out); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()?; + Ok(out) + } +} + +struct FastReduce<'a>(&'a [usize], ReduceOp); +impl<'a> Map1Any for FastReduce<'a> { + fn f) -> S>( + &self, + src: &CudaSlice, + dev: &CudaDevice, + layout: &Layout, + wrap: W, + ) -> Result { + let src_stride = layout.stride(); + let src_dims = layout.shape().dims(); + let src_el: usize = src_dims.iter().product(); + // Source dims and strides with the sum dims at the end. + let mut dims = vec![]; + let mut stride = vec![]; + let mut dst_el: usize = 1; + for (dim_idx, &d) in src_dims.iter().enumerate() { + if !self.0.contains(&dim_idx) { + dst_el *= d; + dims.push(d); + stride.push(src_stride[dim_idx]); + } + } + for &dim_idx in self.0.iter() { + dims.push(src_dims[dim_idx]); + stride.push(src_stride[dim_idx]); + } + let el_to_sum_per_block = src_el / dst_el; + // The reduction loop requires the shared array to be properly initialized and for + // this we want the number of threads to be a power of two. + let block_dim = usize::min(1024, el_to_sum_per_block).next_power_of_two(); + let cfg = LaunchConfig { + // TODO: Maybe use grid_y if the output is too large? + // TODO: Specialized implementation when reducing on no or all dimensions or when + // reducing only aggregate a small number of elements together. + grid_dim: (dst_el as u32, 1, 1), + block_dim: (block_dim as u32, 1, 1), + shared_mem_bytes: 0, + }; + let ds = dev + .htod_copy([dims.as_slice(), stride.as_slice()].concat()) + .w()?; + let src = &src.slice(layout.start_offset()..); + let (name, check_empty, return_index) = match self.1 { + ReduceOp::Sum => ("fast_sum", false, false), + ReduceOp::Min => ("fast_min", true, false), + ReduceOp::Max => ("fast_max", true, false), + ReduceOp::ArgMin => ("fast_argmin", true, true), + ReduceOp::ArgMax => ("fast_argmax", true, true), + }; + if check_empty && layout.shape().elem_count() == 0 { + Err(crate::Error::EmptyTensor { op: "reduce" }.bt())? + } + let func = dev.get_or_load_func(&kernel_name::(name), kernels::REDUCE)?; + if return_index { + // SAFETY: filled in by the follow up kernel. + let out = unsafe { dev.alloc::(dst_el) }.w()?; + let params = (src_el, el_to_sum_per_block, src_dims.len(), &ds, src, &out); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()?; + Ok(S::U32(out)) + } else { + // SAFETY: filled in by the follow up kernel. + let out = unsafe { dev.alloc::(dst_el) }.w()?; + let params = (src_el, el_to_sum_per_block, src_dims.len(), &ds, src, &out); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()?; + Ok(wrap(out)) + } + } +} + +impl Map1 for U { + fn f( + &self, + src: &CudaSlice, + dev: &CudaDevice, + layout: &Layout, + ) -> Result> { + let shape = layout.shape(); + let dims = shape.dims(); + let el_count = shape.elem_count(); + let cfg = LaunchConfig::for_num_elems(el_count as u32); + let ds = dev.htod_copy([dims, layout.stride()].concat()).w()?; + let src = &src.slice(layout.start_offset()..); + let func = dev.get_or_load_func(&kernel_name::(U::KERNEL), kernels::UNARY)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(el_count) }.w()?; + let params = (el_count, dims.len(), &ds, src, &out); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()?; + Ok(out) + } +} + +struct IndexSelect<'a>(&'a CudaStorage, &'a Layout, usize); +impl<'a> Map1 for IndexSelect<'a> { + fn f( + &self, + src: &CudaSlice, + dev: &CudaDevice, + src_l: &Layout, + ) -> Result> { + let ids_l = &self.1; + let (name, ids) = match &self.0.slice { + CudaStorageSlice::U32(slice) => { + ("is_u32", *slice.slice(ids_l.start_offset()..).device_ptr()) + } + CudaStorageSlice::U8(slice) => { + ("is_u8", *slice.slice(ids_l.start_offset()..).device_ptr()) + } + CudaStorageSlice::I64(slice) => { + ("is_i64", *slice.slice(ids_l.start_offset()..).device_ptr()) + } + _ => Err(CudaError::UnexpectedDType { + msg: "index_select ids should be u8 or u32", + expected: DType::U32, + got: self.0.dtype(), + }) + .w()?, + }; + let ids_shape = ids_l.shape(); + let ids_dims = ids_shape.dims(); + let ds = dev.htod_copy([ids_dims, ids_l.stride()].concat()).w()?; + let src = match src_l.contiguous_offsets() { + Some((o1, o2)) => src.slice(o1..o2), + None => Err(crate::Error::RequiresContiguous { op: "index-select" }.bt())?, + }; + let left_size: usize = src_l.dims()[..self.2].iter().product(); + let right_size: usize = src_l.dims()[self.2 + 1..].iter().product(); + let src_dim_size = src_l.dims()[self.2]; + let ids_dim_size = ids_shape.elem_count(); + let dst_el = ids_shape.elem_count() * left_size * right_size; + let cfg = LaunchConfig::for_num_elems(dst_el as u32); + let func = dev.get_or_load_func(&kernel_name::(name), kernels::INDEXING)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(dst_el) }.w()?; + let params = ( + dst_el, + ids_dims.len(), + &ds, + ids, + &src, + &out, + left_size, + src_dim_size, + ids_dim_size, + right_size, + ); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()?; + Ok(out) + } +} + +struct Gather<'a>(&'a CudaStorage, &'a Layout, usize); +impl<'a> Map1 for Gather<'a> { + fn f( + &self, + src: &CudaSlice, + dev: &CudaDevice, + src_l: &Layout, + ) -> Result> { + let ids = &self.0; + let ids_l = &self.1; + let dim = self.2; + let (ids_o1, ids_o2) = match ids_l.contiguous_offsets() { + Some(o12) => o12, + None => Err(crate::Error::RequiresContiguous { op: "gather" }.bt())?, + }; + let (name, ids) = match &ids.slice { + CudaStorageSlice::U32(slice) => { + ("gather_u32", *slice.slice(ids_o1..ids_o2).device_ptr()) + } + CudaStorageSlice::U8(slice) => ("gather_u8", *slice.slice(ids_o1..ids_o2).device_ptr()), + CudaStorageSlice::I64(slice) => { + ("gather_i64", *slice.slice(ids_o1..ids_o2).device_ptr()) + } + _ => Err(CudaError::UnexpectedDType { + msg: "gather ids should be u8/u32/i64", + expected: DType::U32, + got: ids.dtype(), + })?, + }; + let el = ids_l.shape().elem_count(); + let cfg = LaunchConfig::for_num_elems(el as u32); + let src = match src_l.contiguous_offsets() { + Some((o1, o2)) => src.slice(o1..o2), + None => Err(crate::Error::RequiresContiguous { op: "gather" }.bt())?, + }; + let left_sz: usize = src_l.dims()[..dim].iter().product(); + let right_sz: usize = src_l.dims()[dim + 1..].iter().product(); + let src_dim_sz = src_l.dims()[dim]; + let ids_dim_sz = ids_l.dims()[dim]; + let func = dev.get_or_load_func(&kernel_name::(name), kernels::INDEXING)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(el) }.w()?; + let params = ( + el, ids, &src, &out, left_sz, src_dim_sz, ids_dim_sz, right_sz, + ); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()?; + Ok(out) + } +} + +struct IndexAdd<'a>(&'a CudaStorage, &'a Layout, usize); +impl<'a> Map2InPlace for IndexAdd<'a> { + fn f( + &self, + dst: &mut CudaSlice, + dst_shape: &Shape, + src: &CudaSlice, + src_l: &Layout, + dev: &CudaDevice, + ) -> Result<()> { + let ids = &self.0; + let ids_l = &self.1; + let dim = self.2; + let (ids_o1, ids_o2) = match ids_l.contiguous_offsets() { + Some(o12) => o12, + None => Err(crate::Error::RequiresContiguous { op: "index-add" }.bt())?, + }; + let (name, ids) = match &ids.slice { + CudaStorageSlice::U32(slice) => ("ia_u32", *slice.slice(ids_o1..ids_o2).device_ptr()), + CudaStorageSlice::I64(slice) => ("ia_i64", *slice.slice(ids_o1..ids_o2).device_ptr()), + CudaStorageSlice::U8(slice) => ("ia_u8", *slice.slice(ids_o1..ids_o2).device_ptr()), + _ => Err(CudaError::UnexpectedDType { + msg: "index-add ids should be u8/u32/i64", + expected: DType::U32, + got: ids.dtype(), + })?, + }; + let src = match src_l.contiguous_offsets() { + Some((o1, o2)) => src.slice(o1..o2), + None => Err(crate::Error::RequiresContiguous { op: "index-add" }.bt())?, + }; + let left_sz: usize = src_l.dims()[..dim].iter().product(); + let right_sz: usize = src_l.dims()[dim + 1..].iter().product(); + let src_dim_sz = src_l.dims()[dim]; + let dst_dim_sz = dst_shape.dims()[dim]; + let ids_dim_sz = ids_l.dims()[0]; + let cfg = LaunchConfig::for_num_elems((left_sz * right_sz) as u32); + let func = dev.get_or_load_func(&kernel_name::(name), kernels::INDEXING)?; + // SAFETY: Set later by running the kernel. + let params = ( + ids, ids_dim_sz, &src, dst, left_sz, src_dim_sz, dst_dim_sz, right_sz, + ); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()?; + Ok(()) + } +} + +struct ScatterAdd<'a>(&'a CudaStorage, &'a Layout, usize); +impl<'a> Map2InPlace for ScatterAdd<'a> { + fn f( + &self, + dst: &mut CudaSlice, + dst_shape: &Shape, + src: &CudaSlice, + src_l: &Layout, + dev: &CudaDevice, + ) -> Result<()> { + let ids = &self.0; + let ids_l = &self.1; + let dim = self.2; + let (ids_o1, ids_o2) = match ids_l.contiguous_offsets() { + Some(o12) => o12, + None => Err(crate::Error::RequiresContiguous { op: "scatter-add" }.bt())?, + }; + let (name, ids) = match &ids.slice { + CudaStorageSlice::U32(slice) => ("sa_u32", *slice.slice(ids_o1..ids_o2).device_ptr()), + CudaStorageSlice::I64(slice) => ("sa_i64", *slice.slice(ids_o1..ids_o2).device_ptr()), + CudaStorageSlice::U8(slice) => ("sa_u8", *slice.slice(ids_o1..ids_o2).device_ptr()), + _ => Err(CudaError::UnexpectedDType { + msg: "scatter-add ids should be u8/u32/i64", + expected: DType::U32, + got: ids.dtype(), + })?, + }; + let src = match src_l.contiguous_offsets() { + Some((o1, o2)) => src.slice(o1..o2), + None => Err(crate::Error::RequiresContiguous { op: "scatter-add" }.bt())?, + }; + let left_sz: usize = src_l.dims()[..dim].iter().product(); + let right_sz: usize = src_l.dims()[dim + 1..].iter().product(); + let src_dim_sz = src_l.dims()[dim]; + let dst_dim_sz = dst_shape.dims()[dim]; + let cfg = LaunchConfig::for_num_elems((left_sz * right_sz) as u32); + let func = dev.get_or_load_func(&kernel_name::(name), kernels::INDEXING)?; + // SAFETY: Set later by running the kernel. + let params = (ids, &src, dst, left_sz, src_dim_sz, dst_dim_sz, right_sz); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()?; + Ok(()) + } +} + +struct Conv1D<'a>(&'a crate::conv::ParamsConv1D); +impl<'a> Map2 for Conv1D<'a> { + fn f( + &self, + inp: &CudaSlice, + inp_l: &Layout, + k: &CudaSlice, + k_l: &Layout, + dev: &CudaDevice, + ) -> Result> { + // Kernel shape: (c_out, c_in_k, k_size) + // Input shape: (b_size, c_in, l_in) or (c_in, l_in) + let p = &self.0; + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(k_l.start_offset()..); + let shape = inp_l.shape(); + let dims = shape.dims(); + let el = shape.elem_count(); + let l_out = p.l_out(); + let dst_el = p.c_out * l_out * p.b_size; + let cfg = LaunchConfig::for_num_elems(dst_el as u32); + let func = dev.get_or_load_func(&kernel_name::("conv1d"), kernels::CONV)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(dst_el) }.w()?; + let ds = if dims.len() == 3 { + [dims, inp_l.stride(), k_l.dims(), k_l.stride()].concat() + } else if dims.len() == 2 { + [&[1], dims, &[1], inp_l.stride(), k_l.dims(), k_l.stride()].concat() + } else { + crate::bail!("unexpected input shape for conv1d {dims:?}") + }; + let ds = dev.htod_copy(ds).w()?; + let params = ( + el, l_out, p.stride, p.padding, p.dilation, &ds, inp, k, &out, + ); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()?; + Ok(out) + } +} + +struct Conv2D<'a>(&'a crate::conv::ParamsConv2D); +impl<'a> Map2 for Conv2D<'a> { + fn f( + &self, + inp: &CudaSlice, + inp_l: &Layout, + k: &CudaSlice, + k_l: &Layout, + dev: &CudaDevice, + ) -> Result> { + // Kernel shape: (c_out, c_in_k, h_k, w_k) + // Input shape: (b_size, c_in, h_in, w_in) + let p = &self.0; + let (out_w, out_h) = (p.out_w(), p.out_h()); + let dst_el = p.c_out * out_w * out_h * p.b_size; + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(k_l.start_offset()..); + let shape = inp_l.shape(); + let dims = shape.dims(); + let el = shape.elem_count(); + + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(dst_el) }.w()?; + let cfg = LaunchConfig::for_num_elems(dst_el as u32); + let func = dev.get_or_load_func(&kernel_name::("conv2d"), kernels::CONV)?; + let ds = if dims.len() == 4 { + [dims, inp_l.stride(), k_l.dims(), k_l.stride()].concat() + } else { + crate::bail!("unexpected input shape for conv2d {dims:?}") + }; + let ds = dev.htod_copy(ds).w()?; + let params = ( + el, out_w, out_h, p.stride, p.padding, p.dilation, &ds, inp, k, &out, + ); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()?; + Ok(out) + } +} + +struct ConvTranspose1D<'a>(&'a crate::conv::ParamsConvTranspose1D); +impl<'a> Map2 for ConvTranspose1D<'a> { + fn f( + &self, + inp: &CudaSlice, + inp_l: &Layout, + k: &CudaSlice, + k_l: &Layout, + dev: &CudaDevice, + ) -> Result> { + // Kernel shape: (c_in_k, c_out, l_k) + // Input shape: (b_size, c_in, l_in) + let p = &self.0; + let l_out = p.l_out(); + let dst_el = p.c_out * l_out * p.b_size; + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(k_l.start_offset()..); + let shape = inp_l.shape(); + let dims = shape.dims(); + let el = shape.elem_count(); + + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(dst_el) }.w()?; + let cfg = LaunchConfig::for_num_elems(dst_el as u32); + let func = dev.get_or_load_func(&kernel_name::("conv_transpose1d"), kernels::CONV)?; + let ds = if dims.len() == 3 { + [dims, inp_l.stride(), k_l.dims(), k_l.stride()].concat() + } else { + crate::bail!("unexpected input shape for conv_transpose1d {dims:?}") + }; + let ds = dev.htod_copy(ds).w()?; + let params = ( + el, + l_out, + p.stride, + p.padding, + p.output_padding, + p.dilation, + &ds, + inp, + k, + &out, + ); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()?; + Ok(out) + } +} + +struct ConvTranspose2D<'a>(&'a crate::conv::ParamsConvTranspose2D); +impl<'a> Map2 for ConvTranspose2D<'a> { + fn f( + &self, + inp: &CudaSlice, + inp_l: &Layout, + k: &CudaSlice, + k_l: &Layout, + dev: &CudaDevice, + ) -> Result> { + // Kernel shape: (c_in_k, c_out, h_k, w_k) + // Input shape: (b_size, c_in, h_in, w_in) + let p = &self.0; + let (out_w, out_h) = (p.out_w(), p.out_h()); + let dst_el = p.c_out * out_w * out_h * p.b_size; + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(k_l.start_offset()..); + let shape = inp_l.shape(); + let dims = shape.dims(); + let el = shape.elem_count(); + + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(dst_el) }.w()?; + let cfg = LaunchConfig::for_num_elems(dst_el as u32); + let func = dev.get_or_load_func(&kernel_name::("conv_transpose2d"), kernels::CONV)?; + let ds = if dims.len() == 4 { + [dims, inp_l.stride(), k_l.dims(), k_l.stride()].concat() + } else { + crate::bail!("unexpected input shape for conv_transpose2d {dims:?}") + }; + let ds = dev.htod_copy(ds).w()?; + let params = ( + el, + out_w, + out_h, + p.stride, + p.padding, + p.output_padding, + p.dilation, + &ds, + inp, + k, + &out, + ); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()?; + Ok(out) + } +} + +enum PoolOp { + Max, + Avg, +} + +struct Pool2D { + w_k: usize, + h_k: usize, + w_stride: usize, + h_stride: usize, + op: PoolOp, +} + +impl Map1 for Pool2D { + fn f( + &self, + inp: &CudaSlice, + dev: &CudaDevice, + inp_l: &Layout, + ) -> Result> { + // Input shape: (b_size, c, h, w) + let inp = &inp.slice(inp_l.start_offset()..); + let shape = inp_l.shape(); + let dims = shape.dims(); + let ds = if dims.len() == 4 { + [dims, inp_l.stride()].concat() + } else { + crate::bail!("unexpected input shape for pool {dims:?}") + }; + let el = shape.elem_count(); + let out_w = (dims[2] - self.w_k) / self.w_stride + 1; + let out_h = (dims[3] - self.h_k) / self.h_stride + 1; + let dst_el = out_w * out_h * dims[0] * dims[1]; + let cfg = LaunchConfig::for_num_elems(dst_el as u32); + let kname = match self.op { + PoolOp::Max => "max_pool2d", + PoolOp::Avg => "avg_pool2d", + }; + let func = dev.get_or_load_func(&kernel_name::(kname), kernels::CONV)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(dst_el) }.w()?; + let ds = dev.htod_copy(ds).w()?; + let params = ( + el, + self.w_k, + self.h_k, + self.w_stride, + self.h_stride, + &ds, + inp, + &out, + ); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()?; + Ok(out) + } +} + +struct UpsampleNearest2D(usize, usize); +impl Map1 for UpsampleNearest2D { + fn f( + &self, + inp: &CudaSlice, + dev: &CudaDevice, + inp_l: &Layout, + ) -> Result> { + // Input shape: (b_size, c, h, w) + let inp = &inp.slice(inp_l.start_offset()..); + let shape = inp_l.shape(); + let dims = shape.dims(); + let ds = if dims.len() == 4 { + [dims, inp_l.stride()].concat() + } else { + crate::bail!("unexpected input shape for upsample {dims:?}") + }; + let (out_w, out_h) = (self.0, self.1); + let dst_el = out_w * out_h * dims[0] * dims[1]; + let cfg = LaunchConfig::for_num_elems(dst_el as u32); + let func = dev.get_or_load_func(&kernel_name::("upsample_nearest2d"), kernels::CONV)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(dst_el) }.w()?; + let ds = dev.htod_copy(ds).w()?; + let scale_w = dims[2] as f64 / out_w as f64; + let scale_h = dims[3] as f64 / out_h as f64; + let params = (out_w, out_h, scale_w, scale_h, &ds, inp, &out); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()?; + Ok(out) + } +} + +struct WhereCond<'a>(&'a CudaStorage, &'a Layout); +impl<'a> Map2 for WhereCond<'a> { + fn f( + &self, + t: &CudaSlice, + layout_t: &Layout, + f: &CudaSlice, + layout_f: &Layout, + dev: &CudaDevice, + ) -> Result> { + let ids_l = &self.1; + let (ids, name) = match &self.0.slice { + CudaStorageSlice::U8(slice) => { + let ptr = *slice.slice(ids_l.start_offset()..).device_ptr(); + (ptr, "where_u8") + } + CudaStorageSlice::U32(slice) => { + let ptr = *slice.slice(ids_l.start_offset()..).device_ptr(); + (ptr, "where_u32") + } + CudaStorageSlice::I64(slice) => { + let ptr = *slice.slice(ids_l.start_offset()..).device_ptr(); + (ptr, "where_i64") + } + _ => Err(CudaError::UnexpectedDType { + msg: "where conditions should be u8/u32/i64", + expected: DType::U32, + got: self.0.dtype(), + }) + .w()?, + }; + let shape = ids_l.shape(); + let dims = shape.dims(); + let el = shape.elem_count(); + let cfg = LaunchConfig::for_num_elems(el as u32); + let ds = dev + .htod_copy([dims, ids_l.stride(), layout_t.stride(), layout_f.stride()].concat()) + .w()?; + let t = &t.slice(layout_t.start_offset()..); + let f = &f.slice(layout_f.start_offset()..); + let func = dev.get_or_load_func(&kernel_name::(name), kernels::TERNARY)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(el) }.w()?; + let params = (el, dims.len(), &ds, ids, t, f, &out); + // SAFETY: ffi + unsafe { func.launch(cfg, params) }.w()?; + Ok(out) + } +} + +impl Map2 for U { + fn f( + &self, + lhs: &CudaSlice, + lhs_l: &Layout, + rhs: &CudaSlice, + rhs_l: &Layout, + dev: &CudaDevice, + ) -> Result> { + let shape = lhs_l.shape(); + let dims = shape.dims(); + let elem_count = shape.elem_count(); + let cfg = LaunchConfig::for_num_elems(elem_count as u32); + let dims_and_strides = dev + .htod_copy([dims, lhs_l.stride(), rhs_l.stride()].concat()) + .w()?; + let lhs = &lhs.slice(lhs_l.start_offset()..); + let rhs = &rhs.slice(rhs_l.start_offset()..); + let func = dev.get_or_load_func(&kernel_name::(U::KERNEL), kernels::BINARY)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(elem_count) }.w()?; + let params = (elem_count, dims.len(), &dims_and_strides, lhs, rhs, &out); + // SAFETY: ffi + unsafe { func.launch(cfg, params) }.w()?; + Ok(out) + } +} + +struct Cmp(CmpOp); +impl Map2Any for Cmp { + fn f( + &self, + lhs: &CudaSlice, + lhs_l: &Layout, + rhs: &CudaSlice, + rhs_l: &Layout, + dev: &CudaDevice, + ) -> Result { + let shape = lhs_l.shape(); + let dims = shape.dims(); + let elem_count = shape.elem_count(); + let cfg = LaunchConfig::for_num_elems(elem_count as u32); + let dims_and_strides = dev + .htod_copy([dims, lhs_l.stride(), rhs_l.stride()].concat()) + .w()?; + let lhs = &lhs.slice(lhs_l.start_offset()..); + let rhs = &rhs.slice(rhs_l.start_offset()..); + let name = match self.0 { + CmpOp::Eq => "eq", + CmpOp::Ne => "ne", + CmpOp::Lt => "lt", + CmpOp::Le => "le", + CmpOp::Gt => "gt", + CmpOp::Ge => "ge", + }; + let func = dev.get_or_load_func(&kernel_name::(name), kernels::BINARY)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(elem_count) }.w()?; + let params = (elem_count, dims.len(), &dims_and_strides, lhs, rhs, &out); + // SAFETY: ffi + unsafe { func.launch(cfg, params) }.w()?; + Ok(S::U8(out)) + } +} + +fn slice_src_and_dst<'a, T>( + src: &'a CudaSlice, + src_l: &Layout, + dst: &'a mut CudaSlice, + dst_offset: usize, +) -> ( + cudarc::driver::CudaView<'a, T>, + cudarc::driver::CudaViewMut<'a, T>, +) { + let src_offset = src_l.start_offset(); + let to_copy = dst + .len() + .saturating_sub(dst_offset) + .min(src.len().saturating_sub(src_offset)); + let src = src.slice(src_offset..src_offset + to_copy); + let dst = dst.slice_mut(dst_offset..dst_offset + to_copy); + (src, dst) +} + +#[derive(Debug)] +pub struct CudaStorage { + pub slice: CudaStorageSlice, + pub device: CudaDevice, +} + +pub trait CudaDType: Sized { + fn as_cuda_slice(s: &CudaStorage) -> Result<&CudaSlice>; + fn wrap_cuda_slice(s: CudaSlice, dev: CudaDevice) -> CudaStorage; +} + +macro_rules! cuda_dtype { + ($ty:ty, $dtype:ident) => { + impl CudaDType for $ty { + fn as_cuda_slice(s: &CudaStorage) -> Result<&CudaSlice> { + match &s.slice { + CudaStorageSlice::$dtype(data) => Ok(&data), + _ => Err(crate::Error::UnexpectedDType { + expected: DType::$dtype, + got: s.dtype(), + msg: "unexpected dtype", + } + .bt()), + } + } + + fn wrap_cuda_slice(slice: CudaSlice, device: CudaDevice) -> CudaStorage { + let slice = CudaStorageSlice::$dtype(slice); + CudaStorage { slice, device } + } + } + }; +} +cuda_dtype!(u8, U8); +cuda_dtype!(u32, U32); +cuda_dtype!(i64, I64); +cuda_dtype!(f16, F16); +cuda_dtype!(bf16, BF16); +cuda_dtype!(f32, F32); +cuda_dtype!(f64, F64); + +impl CudaStorage { + pub fn wrap_cuda_slice(slice: CudaSlice, device: CudaDevice) -> CudaStorage { + T::wrap_cuda_slice(slice, device) + } + + pub fn as_cuda_slice(&self) -> Result<&CudaSlice> { + T::as_cuda_slice(self) + } +} + +fn gemm_config( + alpha: T, + beta: T, + (b, m, n, k): (usize, usize, usize, usize), + lhs_l: &Layout, + rhs_l: &Layout, +) -> Result> { + // https://docs.nvidia.com/cuda/cublas/index.html#cublas-t-gemm + use cudarc::cublas::sys::cublasOperation_t; + + let lhs_stride = lhs_l.stride(); + let rhs_stride = rhs_l.stride(); + let rhs_m1 = rhs_stride[rhs_stride.len() - 1]; + let rhs_m2 = rhs_stride[rhs_stride.len() - 2]; + let lhs_m1 = lhs_stride[lhs_stride.len() - 1]; + let lhs_m2 = lhs_stride[lhs_stride.len() - 2]; + // The a tensor has dims batching, k, n (rhs) + let (lda, transa) = if rhs_m1 == 1 && rhs_m2 == n { + (n as i32, cublasOperation_t::CUBLAS_OP_N) + } else if rhs_m1 == k && rhs_m2 == 1 { + (k as i32, cublasOperation_t::CUBLAS_OP_T) + } else { + Err(CudaError::MatMulNonContiguous { + lhs_stride: lhs_stride.to_vec(), + rhs_stride: rhs_stride.to_vec(), + mnk: (m, n, k), + })? + }; + // The b tensor has dims batching, m, k (lhs) + let (ldb, transb) = if lhs_m1 == 1 && lhs_m2 == k { + (k as i32, cublasOperation_t::CUBLAS_OP_N) + } else if lhs_m1 == m && lhs_m2 == 1 { + (m as i32, cublasOperation_t::CUBLAS_OP_T) + } else { + Err(CudaError::MatMulNonContiguous { + lhs_stride: lhs_stride.to_vec(), + rhs_stride: rhs_stride.to_vec(), + mnk: (m, n, k), + })? + }; + // The setup below was copied from: + // https://github.com/lebedov/scikit-cuda/blob/7e7300474286019c917a6c8a4bca59405c64fbce/tests/test_cublas.py#L531 + let gemm = GemmConfig { + alpha, + beta, + m: n as i32, + n: m as i32, + k: k as i32, + lda, + ldb, + ldc: n as i32, + transa, + transb, + }; + + let stride_b: usize = match lhs_stride[..lhs_stride.len() - 2] { + [s1, stride] if s1 == stride * lhs_l.dims()[1] => stride, + [stride] => stride, + [] => m * k, + _ => Err(CudaError::MatMulNonContiguous { + lhs_stride: lhs_stride.to_vec(), + rhs_stride: rhs_stride.to_vec(), + mnk: (m, n, k), + })?, + }; + let stride_a: usize = match rhs_stride[..rhs_stride.len() - 2] { + [s1, stride] if s1 == stride * rhs_l.dims()[1] => stride, + [stride] => stride, + [] => n * k, + _ => Err(CudaError::MatMulNonContiguous { + lhs_stride: lhs_stride.to_vec(), + rhs_stride: rhs_stride.to_vec(), + mnk: (m, n, k), + })?, + }; + + Ok(StridedBatchedConfig { + batch_size: b as i32, + gemm, + stride_a: stride_a as i64, + stride_b: stride_b as i64, + stride_c: (m * n) as i64, + }) +} + +impl BackendStorage for CudaStorage { + type Device = CudaDevice; + + fn try_clone(&self, layout: &Layout) -> Result { + let slice = Clone.map(&self.slice, self.device(), layout)?; + let device = self.device.clone(); + Ok(Self { slice, device }) + } + + fn dtype(&self) -> DType { + match self.slice { + CudaStorageSlice::U8(_) => DType::U8, + CudaStorageSlice::U32(_) => DType::U32, + CudaStorageSlice::I64(_) => DType::I64, + CudaStorageSlice::BF16(_) => DType::BF16, + CudaStorageSlice::F16(_) => DType::F16, + CudaStorageSlice::F32(_) => DType::F32, + CudaStorageSlice::F64(_) => DType::F64, + } + } + + fn device(&self) -> &CudaDevice { + &self.device + } + + fn to_dtype(&self, layout: &Layout, dtype: DType) -> Result { + let shape = layout.shape(); + let dims = shape.dims(); + let el = shape.elem_count(); + let cfg = LaunchConfig::for_num_elems(el as u32); + let dev = self.device(); + let ds = dev.htod_copy([dims, layout.stride()].concat()).w()?; + let start_o = layout.start_offset(); + // This returns an i64 rather than a &i64, this is useful to get around some temporary + // lifetime issue and is safe as long as self.slice does not go out of scope before inp + // is used. + let inp = match &self.slice { + CudaStorageSlice::U8(inp) => *inp.slice(start_o..).device_ptr(), + CudaStorageSlice::U32(inp) => *inp.slice(start_o..).device_ptr(), + CudaStorageSlice::I64(inp) => *inp.slice(start_o..).device_ptr(), + CudaStorageSlice::BF16(inp) => *inp.slice(start_o..).device_ptr(), + CudaStorageSlice::F16(inp) => *inp.slice(start_o..).device_ptr(), + CudaStorageSlice::F32(inp) => *inp.slice(start_o..).device_ptr(), + CudaStorageSlice::F64(inp) => *inp.slice(start_o..).device_ptr(), + }; + let inp = &inp; + + let kernel_name = format!("cast_{}_{}", self.dtype().as_str(), dtype.as_str()); + let func = dev.get_or_load_func(&kernel_name, kernels::CAST)?; + let slice = match dtype { + DType::U8 => { + let out = unsafe { dev.alloc::(el) }.w()?; + let params = (el, dims.len(), &ds, *inp, &out); + unsafe { func.launch(cfg, params) }.w()?; + CudaStorageSlice::U8(out) + } + DType::U32 => { + let out = unsafe { dev.alloc::(el) }.w()?; + let params = (el, dims.len(), &ds, *inp, &out); + unsafe { func.launch(cfg, params) }.w()?; + CudaStorageSlice::U32(out) + } + DType::I64 => { + let out = unsafe { dev.alloc::(el) }.w()?; + let params = (el, dims.len(), &ds, *inp, &out); + unsafe { func.launch(cfg, params) }.w()?; + CudaStorageSlice::I64(out) + } + DType::BF16 => { + let out = unsafe { dev.alloc::(el) }.w()?; + let params = (el, dims.len(), &ds, *inp, &out); + unsafe { func.launch(cfg, params) }.w()?; + CudaStorageSlice::BF16(out) + } + DType::F16 => { + let out = unsafe { dev.alloc::(el) }.w()?; + let params = (el, dims.len(), &ds, *inp, &out); + unsafe { func.launch(cfg, params) }.w()?; + CudaStorageSlice::F16(out) + } + DType::F32 => { + let out = unsafe { dev.alloc::(el) }.w()?; + let params = (el, dims.len(), &ds, *inp, &out); + unsafe { func.launch(cfg, params) }.w()?; + CudaStorageSlice::F32(out) + } + DType::F64 => { + let out = unsafe { dev.alloc::(el) }.w()?; + let params = (el, dims.len(), &ds, *inp, &out); + unsafe { func.launch(cfg, params) }.w()?; + CudaStorageSlice::F64(out) + } + }; + Ok(Self { + slice, + device: dev.clone(), + }) + } + + fn affine(&self, layout: &Layout, mul: f64, add: f64) -> Result { + let device = self.device().clone(); + let slice = Affine(mul, add).map(&self.slice, &device, layout)?; + Ok(Self { slice, device }) + } + + fn powf(&self, layout: &Layout, e: f64) -> Result { + let device = self.device().clone(); + let slice = Powf(e).map(&self.slice, &device, layout)?; + Ok(Self { slice, device }) + } + + fn elu(&self, layout: &Layout, alpha: f64) -> Result { + let device = self.device().clone(); + let slice = Elu(alpha).map(&self.slice, &device, layout)?; + Ok(Self { slice, device }) + } + + fn reduce_op(&self, op: ReduceOp, layout: &Layout, sum_dims: &[usize]) -> Result { + let device = self.device().clone(); + let slice = FastReduce(sum_dims, op).map(&self.slice, &device, layout)?; + Ok(Self { slice, device }) + } + + fn cmp(&self, op: CmpOp, rhs: &Self, lhs_l: &Layout, rhs_l: &Layout) -> Result { + let device = self.device().clone(); + let slice = Cmp(op).map(&self.slice, lhs_l, &rhs.slice, rhs_l, &device)?; + Ok(Self { slice, device }) + } + + fn unary_impl(&self, layout: &Layout) -> Result { + let device = self.device().clone(); + let slice = U::V.map(&self.slice, &device, layout)?; + Ok(Self { slice, device }) + } + + fn binary_impl( + &self, + rhs: &Self, + lhs_l: &Layout, + rhs_l: &Layout, + ) -> Result { + let device = self.device().clone(); + let slice = B::V.map(&self.slice, lhs_l, &rhs.slice, rhs_l, &device)?; + Ok(Self { slice, device }) + } + + fn to_cpu_storage(&self) -> Result { + match &self.slice { + CudaStorageSlice::U8(slice) => { + let dev = slice.device(); + let cpu_storage = dev.dtoh_sync_copy(slice).w()?; + Ok(CpuStorage::U8(cpu_storage)) + } + CudaStorageSlice::U32(slice) => { + let dev = slice.device(); + let cpu_storage = dev.dtoh_sync_copy(slice).w()?; + Ok(CpuStorage::U32(cpu_storage)) + } + CudaStorageSlice::I64(slice) => { + let dev = slice.device(); + let cpu_storage = dev.dtoh_sync_copy(slice).w()?; + Ok(CpuStorage::I64(cpu_storage)) + } + CudaStorageSlice::BF16(slice) => { + let dev = slice.device(); + let cpu_storage = dev.dtoh_sync_copy(slice).w()?; + Ok(CpuStorage::BF16(cpu_storage)) + } + CudaStorageSlice::F16(slice) => { + let dev = slice.device(); + let cpu_storage = dev.dtoh_sync_copy(slice).w()?; + Ok(CpuStorage::F16(cpu_storage)) + } + CudaStorageSlice::F32(slice) => { + let dev = slice.device(); + let cpu_storage = dev.dtoh_sync_copy(slice).w()?; + Ok(CpuStorage::F32(cpu_storage)) + } + CudaStorageSlice::F64(slice) => { + let dev = slice.device(); + let cpu_storage = dev.dtoh_sync_copy(slice).w()?; + Ok(CpuStorage::F64(cpu_storage)) + } + } + } + + fn where_cond( + &self, + layout: &Layout, + t: &Self, + t_l: &Layout, + f: &Self, + f_l: &Layout, + ) -> Result { + let device = self.device().clone(); + let slice = WhereCond(self, layout).map(&t.slice, t_l, &f.slice, f_l, &device)?; + Ok(Self { slice, device }) + } + + fn conv1d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConv1D, + ) -> Result { + const USE_IM2COL_CONV1D: bool = true; + + let device = self.device().clone(); + if !USE_IM2COL_CONV1D { + let slice = Conv1D(params).map(&self.slice, l, &kernel.slice, kernel_l, &device)?; + return Ok(Self { slice, device }); + } + + let col = Im2Col1D { + l_k: params.k_size, + stride: params.stride, + dilation: params.dilation, + padding: params.padding, + } + .map(&self.slice, &device, l)?; + let col = Self { slice: col, device }; + let l_out = params.l_out(); + let b = params.b_size; + let n = params.c_out; + let k = params.k_size * params.c_in; + let m = l_out; + let col_l = Layout::contiguous((b, m, k)); + let res = if kernel_l.is_contiguous() { + let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) + .transpose(1, 2)? + .broadcast_as((b, k, n))?; + col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? + } else { + // Make the kernel contiguous if not already the case. + let mut kernel_c = self.device().zeros_impl(kernel_l.shape(), kernel.dtype())?; + kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?; + let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) + .transpose(1, 2)? + .broadcast_as((b, k, n))?; + col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? + }; + let res_l = Layout::contiguous((b, l_out, n)).transpose(1, 2)?; + let mut res_t = self.device().zeros_impl(res_l.shape(), res.dtype())?; + res.copy_strided_src(&mut res_t, 0, &res_l)?; + Ok(res_t) + } + + fn conv_transpose1d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConvTranspose1D, + ) -> Result { + let device = self.device().clone(); + let slice = + ConvTranspose1D(params).map(&self.slice, l, &kernel.slice, kernel_l, &device)?; + Ok(Self { slice, device }) + } + + #[cfg(not(feature = "cudnn"))] + fn conv2d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConv2D, + ) -> Result { + const USE_IM2COL_CONV2D: bool = true; + + let device = self.device().clone(); + if !USE_IM2COL_CONV2D { + let slice = Conv2D(params).map(&self.slice, l, &kernel.slice, kernel_l, &device)?; + return Ok(Self { slice, device }); + } + + let col = Im2Col { + h_k: params.k_h, + w_k: params.k_w, + stride: params.stride, + dilation: params.dilation, + padding: params.padding, + } + .map(&self.slice, &device, l)?; + let col = Self { slice: col, device }; + let h_out = params.out_h(); + let w_out = params.out_w(); + let b = params.b_size; + let n = params.c_out; + let k = params.k_h * params.k_w * params.c_in; + let m = h_out * w_out; + let col_l = Layout::contiguous((b, m, k)); + let res = if kernel_l.is_contiguous() { + let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) + .transpose(1, 2)? + .broadcast_as((b, k, n))?; + col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? + } else { + // Make the kernel contiguous if not already the case. + let mut kernel_c = self.device().zeros_impl(kernel_l.shape(), kernel.dtype())?; + kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?; + let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) + .transpose(1, 2)? + .broadcast_as((b, k, n))?; + col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? + }; + let res_l = Layout::contiguous((b, h_out, w_out, n)) + .transpose(1, 2)? + .transpose(1, 3)?; + let mut res_t = self.device().zeros_impl(res_l.shape(), res.dtype())?; + res.copy_strided_src(&mut res_t, 0, &res_l)?; + Ok(res_t) + } + + #[cfg(feature = "cudnn")] + fn conv2d( + &self, + inp_l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConv2D, + ) -> Result { + let device = self.device().clone(); + if !kernel_l.is_contiguous() { + let slice = Conv2D(params).map(&self.slice, inp_l, &kernel.slice, kernel_l, &device)?; + return Ok(Self { slice, device }); + } + let (out_w, out_h) = (params.out_w(), params.out_h()); + let dst_el = params.c_out * out_w * out_h * params.b_size; + let slice = match (&self.slice, &kernel.slice) { + (S::U8(inp), S::U8(k)) => { + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(kernel_l.start_offset()..); + let mut out = unsafe { device.alloc::(dst_el) }.w()?; + crate::cudnn::launch_conv2d::(inp, inp_l, k, &mut out, params, &device) + .map_err(crate::Error::wrap)?; + S::U8(out) + } + (S::BF16(inp), S::BF16(k)) => { + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(kernel_l.start_offset()..); + let mut out = unsafe { device.alloc::(dst_el) }.w()?; + crate::cudnn::launch_conv2d::(inp, inp_l, k, &mut out, params, &device) + .map_err(crate::Error::wrap)?; + S::BF16(out) + } + (S::F16(inp), S::F16(k)) => { + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(kernel_l.start_offset()..); + let mut out = unsafe { device.alloc::(dst_el) }.w()?; + crate::cudnn::launch_conv2d::(inp, inp_l, k, &mut out, params, &device) + .map_err(crate::Error::wrap)?; + S::F16(out) + } + (S::F32(inp), S::F32(k)) => { + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(kernel_l.start_offset()..); + let mut out = unsafe { device.alloc::(dst_el) }.w()?; + crate::cudnn::launch_conv2d::(inp, inp_l, k, &mut out, params, &device) + .map_err(crate::Error::wrap)?; + S::F32(out) + } + (S::F64(inp), S::F64(k)) => { + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(kernel_l.start_offset()..); + let mut out = unsafe { device.alloc::(dst_el) }.w()?; + crate::cudnn::launch_conv2d::(inp, inp_l, k, &mut out, params, &device) + .map_err(crate::Error::wrap)?; + S::F64(out) + } + (S::U32(_), S::U32(_)) => Err(CudaError::InternalError("conv2d does not support u32"))?, + (S::I64(_), S::I64(_)) => Err(CudaError::InternalError("conv2d does not support i64"))?, + _ => Err(CudaError::InternalError("dtype mismatch in conv2d"))?, + }; + Ok(Self { slice, device }) + } + + fn conv_transpose2d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConvTranspose2D, + ) -> Result { + let device = self.device().clone(); + let slice = + ConvTranspose2D(params).map(&self.slice, l, &kernel.slice, kernel_l, &device)?; + Ok(Self { slice, device }) + } + + fn avg_pool2d(&self, l: &Layout, k: (usize, usize), stride: (usize, usize)) -> Result { + let device = self.device().clone(); + let slice = Pool2D { + w_k: k.0, + h_k: k.1, + w_stride: stride.0, + h_stride: stride.1, + op: PoolOp::Avg, + } + .map(&self.slice, &device, l)?; + Ok(Self { slice, device }) + } + + fn max_pool2d(&self, l: &Layout, k: (usize, usize), stride: (usize, usize)) -> Result { + let device = self.device().clone(); + let slice = Pool2D { + w_k: k.0, + h_k: k.1, + w_stride: stride.0, + h_stride: stride.1, + op: PoolOp::Max, + } + .map(&self.slice, &device, l)?; + Ok(Self { slice, device }) + } + + fn upsample_nearest1d(&self, _: &Layout, _out_sz: usize) -> Result { + crate::bail!("upsample-nearest1d is not supported on cuda") + } + + fn upsample_nearest2d(&self, l: &Layout, out_w: usize, out_h: usize) -> Result { + let device = self.device().clone(); + let slice = UpsampleNearest2D(out_w, out_h).map(&self.slice, &device, l)?; + Ok(Self { slice, device }) + } + + fn index_select(&self, ids: &Self, l: &Layout, ids_l: &Layout, dim: usize) -> Result { + let device = self.device().clone(); + let slice = IndexSelect(ids, ids_l, dim).map(&self.slice, &device, l)?; + Ok(Self { slice, device }) + } + fn gather(&self, l: &Layout, ids: &Self, ids_l: &Layout, dim: usize) -> Result { + let device = self.device().clone(); + let slice = Gather(ids, ids_l, dim).map(&self.slice, &device, l)?; + Ok(Self { slice, device }) + } + fn scatter_add( + &self, + l: &Layout, + ids: &Self, + ids_l: &Layout, + src: &Self, + src_l: &Layout, + dim: usize, + ) -> Result { + let device = self.device().clone(); + let mut acc = device.zeros_impl(l.shape(), self.dtype())?; + self.copy_strided_src(&mut acc, 0, l)?; + ScatterAdd(ids, ids_l, dim).map(&mut acc.slice, l.shape(), &src.slice, src_l, &device)?; + Ok(acc) + } + fn index_add( + &self, + l: &Layout, + ids: &Self, + ids_l: &Layout, + src: &Self, + src_l: &Layout, + dim: usize, + ) -> Result { + let device = self.device().clone(); + let mut acc = device.zeros_impl(l.shape(), self.dtype())?; + self.copy_strided_src(&mut acc, 0, l)?; + IndexAdd(ids, ids_l, dim).map(&mut acc.slice, l.shape(), &src.slice, src_l, &device)?; + Ok(acc) + } + + fn matmul( + &self, + rhs: &Self, + (b, m, n, k): (usize, usize, usize, usize), + lhs_l: &Layout, + rhs_l: &Layout, + ) -> Result { + let elem_count = b * m * n; + let dev = &self.device; + let slice = match (&self.slice, &rhs.slice) { + (CudaStorageSlice::BF16(lhs), CudaStorageSlice::BF16(rhs)) => { + let lhs = &lhs.slice(lhs_l.start_offset()..); + let rhs = &rhs.slice(rhs_l.start_offset()..); + let cfg = gemm_config(bf16::ONE, bf16::ZERO, (b, m, n, k), lhs_l, rhs_l)?; + let mut out = unsafe { dev.alloc::(elem_count) }.w()?; + unsafe { + self.device + .blas + .gemm_strided_batched(cfg, rhs, lhs, &mut out) + } + .w()?; + CudaStorageSlice::BF16(out) + } + (CudaStorageSlice::F16(lhs), CudaStorageSlice::F16(rhs)) => { + let lhs = &lhs.slice(lhs_l.start_offset()..); + let rhs = &rhs.slice(rhs_l.start_offset()..); + let cfg = gemm_config(f16::ONE, f16::ZERO, (b, m, n, k), lhs_l, rhs_l)?; + let mut out = unsafe { dev.alloc::(elem_count) }.w()?; + unsafe { + self.device + .blas + .gemm_strided_batched(cfg, rhs, lhs, &mut out) + } + .w()?; + CudaStorageSlice::F16(out) + } + (CudaStorageSlice::F32(lhs), CudaStorageSlice::F32(rhs)) => { + let lhs = &lhs.slice(lhs_l.start_offset()..); + let rhs = &rhs.slice(rhs_l.start_offset()..); + let cfg = gemm_config(1., 0., (b, m, n, k), lhs_l, rhs_l)?; + let mut out = unsafe { dev.alloc::(elem_count) }.w()?; + unsafe { + self.device + .blas + .gemm_strided_batched(cfg, rhs, lhs, &mut out) + } + .w()?; + CudaStorageSlice::F32(out) + } + (CudaStorageSlice::F64(lhs), CudaStorageSlice::F64(rhs)) => { + let lhs = &lhs.slice(lhs_l.start_offset()..); + let rhs = &rhs.slice(rhs_l.start_offset()..); + let cfg = gemm_config(1., 0., (b, m, n, k), lhs_l, rhs_l)?; + let mut out = unsafe { dev.alloc::(elem_count) }.w()?; + unsafe { + self.device + .blas + .gemm_strided_batched(cfg, rhs, lhs, &mut out) + } + .w()?; + CudaStorageSlice::F64(out) + } + _ => Err(CudaError::InternalError("dtype mismatch in matmul op"))?, + }; + let device = dev.clone(); + Ok(Self { slice, device }) + } + + fn copy_strided_src(&self, dst: &mut Self, dst_offset: usize, src_l: &Layout) -> Result<()> { + let src_shape = src_l.shape(); + let dims = src_shape.dims(); + let el_count = src_shape.elem_count(); + if el_count == 0 { + return Ok(()); + } + let cfg = LaunchConfig::for_num_elems(el_count as u32); + let dev = &self.device; + let ds = dev.htod_copy([dims, src_l.stride()].concat()).w()?; + match (&self.slice, &mut dst.slice) { + (CudaStorageSlice::BF16(src), CudaStorageSlice::BF16(dst)) => { + let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); + if src_l.is_contiguous() { + dev.dtod_copy(&src, &mut dst).w()? + } else { + let func = dev.get_or_load_func("ucopy_bf16", kernels::UNARY)?; + // SAFETY: Set later by running the kernel. + let params = (el_count, dims.len(), &ds, &src, &mut dst); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()? + } + } + (CudaStorageSlice::F16(src), CudaStorageSlice::F16(dst)) => { + let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); + if src_l.is_contiguous() { + dev.dtod_copy(&src, &mut dst).w()? + } else { + let func = dev.get_or_load_func("ucopy_f16", kernels::UNARY)?; + // SAFETY: Set later by running the kernel. + let params = (el_count, dims.len(), &ds, &src, &mut dst); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()? + } + } + (CudaStorageSlice::F32(src), CudaStorageSlice::F32(dst)) => { + let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); + if src_l.is_contiguous() { + dev.dtod_copy(&src, &mut dst).w()? + } else { + let func = dev.get_or_load_func("ucopy_f32", kernels::UNARY)?; + // SAFETY: Set later by running the kernel. + let params = (el_count, dims.len(), &ds, &src, &mut dst); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()? + } + } + (CudaStorageSlice::U8(src), CudaStorageSlice::U8(dst)) => { + let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); + if src_l.is_contiguous() { + dev.dtod_copy(&src, &mut dst).w()? + } else { + let func = dev.get_or_load_func("ucopy_u8", kernels::UNARY)?; + // SAFETY: Set later by running the kernel. + let params = (el_count, dims.len(), &ds, &src, &mut dst); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()? + } + } + (CudaStorageSlice::U32(src), CudaStorageSlice::U32(dst)) => { + let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); + if src_l.is_contiguous() { + dev.dtod_copy(&src, &mut dst).w()? + } else { + let func = dev.get_or_load_func("ucopy_u32", kernels::UNARY)?; + // SAFETY: Set later by running the kernel. + let params = (el_count, dims.len(), &ds, &src, &mut dst); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()? + } + } + (CudaStorageSlice::I64(src), CudaStorageSlice::I64(dst)) => { + let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); + if src_l.is_contiguous() { + dev.dtod_copy(&src, &mut dst).w()? + } else { + let func = dev.get_or_load_func("ucopy_i64", kernels::UNARY)?; + // SAFETY: Set later by running the kernel. + let params = (el_count, dims.len(), &ds, &src, &mut dst); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()? + } + } + (CudaStorageSlice::F64(src), CudaStorageSlice::F64(dst)) => { + let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); + if src_l.is_contiguous() { + dev.dtod_copy(&src, &mut dst).w()? + } else { + let func = dev.get_or_load_func("ucopy_f64", kernels::UNARY)?; + // SAFETY: Set later by running the kernel. + let params = (el_count, dims.len(), &ds, &src, &mut dst); + // SAFETY: ffi. + unsafe { func.launch(cfg, params) }.w()?; + } + } + _ => Err(CudaError::InternalError( + "dtype mismatch in copy_strided op", + ))?, + } + Ok(()) + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/cudnn.rs b/v2/dependency-patches/candle-core-0.4.1/src/cudnn.rs new file mode 100644 index 0000000000..0c149cd0d0 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/cudnn.rs @@ -0,0 +1,123 @@ +use crate::WithDType; +use cudarc; +use cudarc::cudnn::safe::{Conv2dForward, Cudnn}; +use cudarc::driver::{CudaSlice, CudaView, DeviceRepr, ValidAsZeroBits}; +use std::cell::RefCell; +use std::collections::HashMap; +use std::sync::Arc; + +// The cudnn handles are stored per thread here rather than on the CudaDevice as they are neither +// send nor sync. +thread_local! { + static CUDNN: RefCell>> = HashMap::new().into(); +} + +impl From for crate::Error { + fn from(err: cudarc::cudnn::CudnnError) -> Self { + crate::Error::wrap(err) + } +} + +impl From for crate::Error { + fn from(err: cudarc::driver::DriverError) -> Self { + crate::Error::wrap(err) + } +} + +pub(crate) fn launch_conv2d< + T: DeviceRepr + WithDType + ValidAsZeroBits + cudarc::cudnn::CudnnDataType, +>( + src: &CudaView, + src_l: &crate::Layout, + filter: &CudaView, + dst: &mut CudaSlice, + params: &crate::conv::ParamsConv2D, + dev: &crate::cuda_backend::CudaDevice, +) -> crate::Result<()> { + use crate::conv::CudnnFwdAlgo as CandleAlgo; + use cudarc::cudnn::sys::cudnnConvolutionFwdAlgo_t as A; + + let device_id = dev.id(); + let cudnn = CUDNN.with(|cudnn| { + if let Some(cudnn) = cudnn.borrow().get(&device_id) { + return Ok(cudnn.clone()); + } + let c = Cudnn::new(dev.cuda_device()); + if let Ok(c) = &c { + cudnn.borrow_mut().insert(device_id, c.clone()); + } + c + })?; + let conv = cudnn.create_conv2d::( + /* pad */ [params.padding as i32, params.padding as i32], + /* stride */ [params.stride as i32, params.stride as i32], + /* dilation */ [params.dilation as i32, params.dilation as i32], + cudarc::cudnn::sys::cudnnConvolutionMode_t::CUDNN_CROSS_CORRELATION, + )?; + let x_shape = [ + params.b_size as i32, + params.c_in as i32, + params.i_h as i32, + params.i_w as i32, + ]; + // Note that `src` already starts at the proper offset. + let x = if src_l.is_contiguous() { + cudnn.create_4d_tensor( + cudarc::cudnn::sys::cudnnTensorFormat_t::CUDNN_TENSOR_NCHW, + x_shape, + )? + } else { + let s = src_l.stride(); + cudnn.create_4d_tensor_ex( + x_shape, + [s[0] as i32, s[1] as i32, s[2] as i32, s[3] as i32], + )? + }; + let w = cudnn.create_4d_filter( + cudarc::cudnn::sys::cudnnTensorFormat_t::CUDNN_TENSOR_NCHW, + [ + params.c_out as i32, + params.c_in as i32, + params.k_h as i32, + params.k_w as i32, + ], + )?; + let (w_out, h_out) = (params.out_w() as i32, params.out_h() as i32); + let y = cudnn.create_4d_tensor( + cudarc::cudnn::sys::cudnnTensorFormat_t::CUDNN_TENSOR_NCHW, + [params.b_size as i32, params.c_out as i32, h_out, w_out], + )?; + let conv2d = Conv2dForward { + conv: &conv, + x: &x, + w: &w, + y: &y, + }; + let alg = match params.cudnn_fwd_algo { + None => conv2d.pick_algorithm()?, + Some(CandleAlgo::ImplicitGemm) => A::CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM, + Some(CandleAlgo::ImplicitPrecompGemm) => { + A::CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM + } + Some(CandleAlgo::Gemm) => A::CUDNN_CONVOLUTION_FWD_ALGO_GEMM, + Some(CandleAlgo::Direct) => A::CUDNN_CONVOLUTION_FWD_ALGO_DIRECT, + Some(CandleAlgo::Fft) => A::CUDNN_CONVOLUTION_FWD_ALGO_FFT, + Some(CandleAlgo::FftTiling) => A::CUDNN_CONVOLUTION_FWD_ALGO_FFT_TILING, + Some(CandleAlgo::Winograd) => A::CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD, + Some(CandleAlgo::WinogradNonFused) => A::CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED, + Some(CandleAlgo::Count) => A::CUDNN_CONVOLUTION_FWD_ALGO_COUNT, + }; + let workspace_size = conv2d.get_workspace_size(alg)?; + let mut workspace = dev.cuda_device().alloc_zeros::(workspace_size)?; + unsafe { + conv2d.launch::, _, _, _>( + alg, + Some(&mut workspace), + (T::one(), T::zero()), + src, + filter, + dst, + )?; + } + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/device.rs b/v2/dependency-patches/candle-core-0.4.1/src/device.rs new file mode 100644 index 0000000000..1e33021bf2 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/device.rs @@ -0,0 +1,323 @@ +use crate::backend::BackendDevice; +use crate::cpu_backend::CpuDevice; +use crate::{CpuStorage, DType, Result, Shape, Storage, WithDType}; + +/// A `DeviceLocation` represents a physical device whereas multiple `Device` +/// can live on the same location (typically for cuda devices). +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub enum DeviceLocation { + Cpu, + Cuda { gpu_id: usize }, + Metal { gpu_id: usize }, +} + +#[derive(Debug, Clone)] +pub enum Device { + Cpu, + Cuda(crate::CudaDevice), + Metal(crate::MetalDevice), +} + +pub trait NdArray { + fn shape(&self) -> Result; + + fn to_cpu_storage(&self) -> CpuStorage; +} + +impl NdArray for S { + fn shape(&self) -> Result { + Ok(Shape::from(())) + } + + fn to_cpu_storage(&self) -> CpuStorage { + S::to_cpu_storage(&[*self]) + } +} + +impl NdArray for &[S; N] { + fn shape(&self) -> Result { + Ok(Shape::from(self.len())) + } + + fn to_cpu_storage(&self) -> CpuStorage { + S::to_cpu_storage(self.as_slice()) + } +} + +impl NdArray for &[S] { + fn shape(&self) -> Result { + Ok(Shape::from(self.len())) + } + + fn to_cpu_storage(&self) -> CpuStorage { + S::to_cpu_storage(self) + } +} + +impl NdArray for &[[S; N]; M] { + fn shape(&self) -> Result { + Ok(Shape::from((M, N))) + } + + fn to_cpu_storage(&self) -> CpuStorage { + S::to_cpu_storage_owned(self.concat()) + } +} + +impl NdArray + for &[[[S; N3]; N2]; N1] +{ + fn shape(&self) -> Result { + Ok(Shape::from((N1, N2, N3))) + } + + fn to_cpu_storage(&self) -> CpuStorage { + let mut vec = Vec::with_capacity(N1 * N2 * N3); + for i1 in 0..N1 { + for i2 in 0..N2 { + vec.extend(self[i1][i2]) + } + } + S::to_cpu_storage_owned(vec) + } +} + +impl NdArray + for &[[[[S; N4]; N3]; N2]; N1] +{ + fn shape(&self) -> Result { + Ok(Shape::from((N1, N2, N3, N4))) + } + + fn to_cpu_storage(&self) -> CpuStorage { + let mut vec = Vec::with_capacity(N1 * N2 * N3 * N4); + for i1 in 0..N1 { + for i2 in 0..N2 { + for i3 in 0..N3 { + vec.extend(self[i1][i2][i3]) + } + } + } + S::to_cpu_storage_owned(vec) + } +} + +impl NdArray for Vec { + fn shape(&self) -> Result { + if self.is_empty() { + crate::bail!("empty array") + } + let shape0 = self[0].shape()?; + let n = self.len(); + for v in self.iter() { + let shape = v.shape()?; + if shape != shape0 { + crate::bail!("two elements have different shapes {shape:?} {shape0:?}") + } + } + Ok(Shape::from([[n].as_slice(), shape0.dims()].concat())) + } + + fn to_cpu_storage(&self) -> CpuStorage { + // This allocates intermediary memory and shouldn't be necessary. + let storages = self.iter().map(|v| v.to_cpu_storage()).collect::>(); + CpuStorage::concat(storages.as_slice()).unwrap() + } +} + +impl Device { + pub fn new_cuda(ordinal: usize) -> Result { + Ok(Self::Cuda(crate::CudaDevice::new(ordinal)?)) + } + + pub fn new_metal(ordinal: usize) -> Result { + Ok(Self::Metal(crate::MetalDevice::new(ordinal)?)) + } + + pub fn set_seed(&self, seed: u64) -> Result<()> { + match self { + Self::Cpu => CpuDevice.set_seed(seed), + Self::Cuda(c) => c.set_seed(seed), + Self::Metal(m) => m.set_seed(seed), + } + } + + pub fn same_device(&self, rhs: &Self) -> bool { + match (self, rhs) { + (Self::Cpu, Self::Cpu) => true, + (Self::Cuda(lhs), Self::Cuda(rhs)) => lhs.same_device(rhs), + (Self::Metal(lhs), Self::Metal(rhs)) => lhs.same_device(rhs), + _ => false, + } + } + + pub fn location(&self) -> DeviceLocation { + match self { + Self::Cpu => DeviceLocation::Cpu, + Self::Cuda(device) => device.location(), + Device::Metal(device) => device.location(), + } + } + + pub fn is_cpu(&self) -> bool { + matches!(self, Self::Cpu) + } + + pub fn is_cuda(&self) -> bool { + matches!(self, Self::Cuda(_)) + } + + pub fn is_metal(&self) -> bool { + matches!(self, Self::Metal(_)) + } + + pub fn cuda_if_available(ordinal: usize) -> Result { + if crate::utils::cuda_is_available() { + Self::new_cuda(ordinal) + } else { + Ok(Self::Cpu) + } + } + + pub(crate) fn rand_uniform_f64( + &self, + lo: f64, + up: f64, + shape: &Shape, + dtype: DType, + ) -> Result { + match self { + Device::Cpu => { + let storage = CpuDevice.rand_uniform(shape, dtype, lo, up)?; + Ok(Storage::Cpu(storage)) + } + Device::Cuda(device) => { + // TODO: Remove the special case if we start supporting generating f16/bf16 directly. + if dtype == DType::F16 || dtype == DType::BF16 { + let storage = device.rand_uniform(shape, DType::F32, lo, up)?; + Storage::Cuda(storage).to_dtype(&crate::Layout::contiguous(shape), dtype) + } else { + let storage = device.rand_uniform(shape, dtype, lo, up)?; + Ok(Storage::Cuda(storage)) + } + } + Device::Metal(device) => { + let storage = device.rand_uniform(shape, dtype, lo, up)?; + Ok(Storage::Metal(storage)) + } + } + } + + pub(crate) fn rand_uniform( + &self, + lo: T, + up: T, + shape: &Shape, + ) -> Result { + self.rand_uniform_f64(lo.to_f64(), up.to_f64(), shape, T::DTYPE) + } + + pub(crate) fn rand_normal_f64( + &self, + mean: f64, + std: f64, + shape: &Shape, + dtype: DType, + ) -> Result { + match self { + Device::Cpu => { + let storage = CpuDevice.rand_normal(shape, dtype, mean, std)?; + Ok(Storage::Cpu(storage)) + } + Device::Cuda(device) => { + // TODO: Remove the special case if we start supporting generating f16/bf16 directly. + if dtype == DType::F16 || dtype == DType::BF16 { + let storage = device.rand_normal(shape, DType::F32, mean, std)?; + Storage::Cuda(storage).to_dtype(&crate::Layout::contiguous(shape), dtype) + } else { + let storage = device.rand_normal(shape, dtype, mean, std)?; + Ok(Storage::Cuda(storage)) + } + } + Device::Metal(device) => { + let storage = device.rand_normal(shape, dtype, mean, std)?; + Ok(Storage::Metal(storage)) + } + } + } + + pub(crate) fn rand_normal( + &self, + mean: T, + std: T, + shape: &Shape, + ) -> Result { + self.rand_normal_f64(mean.to_f64(), std.to_f64(), shape, T::DTYPE) + } + + pub(crate) fn ones(&self, shape: &Shape, dtype: DType) -> Result { + match self { + Device::Cpu => { + let storage = CpuDevice.ones_impl(shape, dtype)?; + Ok(Storage::Cpu(storage)) + } + Device::Cuda(device) => { + let storage = device.ones_impl(shape, dtype)?; + Ok(Storage::Cuda(storage)) + } + Device::Metal(device) => { + let storage = device.ones_impl(shape, dtype)?; + Ok(Storage::Metal(storage)) + } + } + } + + pub(crate) fn zeros(&self, shape: &Shape, dtype: DType) -> Result { + match self { + Device::Cpu => { + let storage = CpuDevice.zeros_impl(shape, dtype)?; + Ok(Storage::Cpu(storage)) + } + Device::Cuda(device) => { + let storage = device.zeros_impl(shape, dtype)?; + Ok(Storage::Cuda(storage)) + } + Device::Metal(device) => { + let storage = device.zeros_impl(shape, dtype)?; + Ok(Storage::Metal(storage)) + } + } + } + + pub(crate) fn storage(&self, array: A) -> Result { + match self { + Device::Cpu => Ok(Storage::Cpu(array.to_cpu_storage())), + Device::Cuda(device) => { + let storage = array.to_cpu_storage(); + let storage = device.storage_from_cpu_storage(&storage)?; + Ok(Storage::Cuda(storage)) + } + Device::Metal(device) => { + let storage = array.to_cpu_storage(); + let storage = device.storage_from_cpu_storage(&storage)?; + Ok(Storage::Metal(storage)) + } + } + } + + pub(crate) fn storage_owned(&self, data: Vec) -> Result { + match self { + Device::Cpu => Ok(Storage::Cpu(S::to_cpu_storage_owned(data))), + Device::Cuda(device) => { + let storage = S::to_cpu_storage_owned(data); + let storage = device.storage_from_cpu_storage(&storage)?; + Ok(Storage::Cuda(storage)) + } + Device::Metal(device) => { + let storage = S::to_cpu_storage_owned(data); + let storage = device.storage_from_cpu_storage(&storage)?; + Ok(Storage::Metal(storage)) + } + } + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/display.rs b/v2/dependency-patches/candle-core-0.4.1/src/display.rs new file mode 100644 index 0000000000..4f5a390e80 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/display.rs @@ -0,0 +1,495 @@ +/// Pretty printing of tensors +/// This implementation should be in line with the PyTorch version. +/// https://github.com/pytorch/pytorch/blob/7b419e8513a024e172eae767e24ec1b849976b13/torch/_tensor_str.py +use crate::{DType, Result, Tensor, WithDType}; +use half::{bf16, f16}; + +impl Tensor { + fn fmt_dt( + &self, + f: &mut std::fmt::Formatter, + ) -> std::fmt::Result { + let device_str = match self.device().location() { + crate::DeviceLocation::Cpu => "".to_owned(), + crate::DeviceLocation::Cuda { gpu_id } => { + format!(", cuda:{}", gpu_id) + } + crate::DeviceLocation::Metal { gpu_id } => { + format!(", metal:{}", gpu_id) + } + }; + + write!(f, "Tensor[")?; + match self.dims() { + [] => { + if let Ok(v) = self.to_scalar::() { + write!(f, "{v}")? + } + } + [s] if *s < 10 => { + if let Ok(vs) = self.to_vec1::() { + for (i, v) in vs.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{v}")?; + } + } + } + dims => { + write!(f, "dims ")?; + for (i, d) in dims.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{d}")?; + } + } + } + write!(f, "; {}{}]", self.dtype().as_str(), device_str) + } +} + +impl std::fmt::Debug for Tensor { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self.dtype() { + DType::U8 => self.fmt_dt::(f), + DType::U32 => self.fmt_dt::(f), + DType::I64 => self.fmt_dt::(f), + DType::BF16 => self.fmt_dt::(f), + DType::F16 => self.fmt_dt::(f), + DType::F32 => self.fmt_dt::(f), + DType::F64 => self.fmt_dt::(f), + } + } +} + +/// Options for Tensor pretty printing +pub struct PrinterOptions { + precision: usize, + threshold: usize, + edge_items: usize, + line_width: usize, + sci_mode: Option, +} + +static PRINT_OPTS: std::sync::Mutex = + std::sync::Mutex::new(PrinterOptions::const_default()); + +impl PrinterOptions { + // We cannot use the default trait as it's not const. + const fn const_default() -> Self { + Self { + precision: 4, + threshold: 1000, + edge_items: 3, + line_width: 80, + sci_mode: None, + } + } +} + +pub fn set_print_options(options: PrinterOptions) { + *PRINT_OPTS.lock().unwrap() = options +} + +pub fn set_print_options_default() { + *PRINT_OPTS.lock().unwrap() = PrinterOptions::const_default() +} + +pub fn set_print_options_short() { + *PRINT_OPTS.lock().unwrap() = PrinterOptions { + precision: 2, + threshold: 1000, + edge_items: 2, + line_width: 80, + sci_mode: None, + } +} + +pub fn set_print_options_full() { + *PRINT_OPTS.lock().unwrap() = PrinterOptions { + precision: 4, + threshold: usize::MAX, + edge_items: 3, + line_width: 80, + sci_mode: None, + } +} + +struct FmtSize { + current_size: usize, +} + +impl FmtSize { + fn new() -> Self { + Self { current_size: 0 } + } + + fn final_size(self) -> usize { + self.current_size + } +} + +impl std::fmt::Write for FmtSize { + fn write_str(&mut self, s: &str) -> std::fmt::Result { + self.current_size += s.len(); + Ok(()) + } +} + +trait TensorFormatter { + type Elem: WithDType; + + fn fmt(&self, v: Self::Elem, max_w: usize, f: &mut T) -> std::fmt::Result; + + fn max_width(&self, to_display: &Tensor) -> usize { + let mut max_width = 1; + if let Ok(vs) = to_display.flatten_all().and_then(|t| t.to_vec1()) { + for &v in vs.iter() { + let mut fmt_size = FmtSize::new(); + let _res = self.fmt(v, 1, &mut fmt_size); + max_width = usize::max(max_width, fmt_size.final_size()) + } + } + max_width + } + + fn write_newline_indent(i: usize, f: &mut std::fmt::Formatter) -> std::fmt::Result { + writeln!(f)?; + for _ in 0..i { + write!(f, " ")? + } + Ok(()) + } + + fn fmt_tensor( + &self, + t: &Tensor, + indent: usize, + max_w: usize, + summarize: bool, + po: &PrinterOptions, + f: &mut std::fmt::Formatter, + ) -> std::fmt::Result { + let dims = t.dims(); + let edge_items = po.edge_items; + write!(f, "[")?; + match dims { + [] => { + if let Ok(v) = t.to_scalar::() { + self.fmt(v, max_w, f)? + } + } + [v] if summarize && *v > 2 * edge_items => { + if let Ok(vs) = t + .narrow(0, 0, edge_items) + .and_then(|t| t.to_vec1::()) + { + for v in vs.into_iter() { + self.fmt(v, max_w, f)?; + write!(f, ", ")?; + } + } + write!(f, "...")?; + if let Ok(vs) = t + .narrow(0, v - edge_items, edge_items) + .and_then(|t| t.to_vec1::()) + { + for v in vs.into_iter() { + write!(f, ", ")?; + self.fmt(v, max_w, f)?; + } + } + } + [_] => { + let elements_per_line = usize::max(1, po.line_width / (max_w + 2)); + if let Ok(vs) = t.to_vec1::() { + for (i, v) in vs.into_iter().enumerate() { + if i > 0 { + if i % elements_per_line == 0 { + write!(f, ",")?; + Self::write_newline_indent(indent, f)? + } else { + write!(f, ", ")?; + } + } + self.fmt(v, max_w, f)? + } + } + } + _ => { + if summarize && dims[0] > 2 * edge_items { + for i in 0..edge_items { + match t.get(i) { + Ok(t) => self.fmt_tensor(&t, indent + 1, max_w, summarize, po, f)?, + Err(e) => write!(f, "{e:?}")?, + } + write!(f, ",")?; + Self::write_newline_indent(indent, f)? + } + write!(f, "...")?; + Self::write_newline_indent(indent, f)?; + for i in dims[0] - edge_items..dims[0] { + match t.get(i) { + Ok(t) => self.fmt_tensor(&t, indent + 1, max_w, summarize, po, f)?, + Err(e) => write!(f, "{e:?}")?, + } + if i + 1 != dims[0] { + write!(f, ",")?; + Self::write_newline_indent(indent, f)? + } + } + } else { + for i in 0..dims[0] { + match t.get(i) { + Ok(t) => self.fmt_tensor(&t, indent + 1, max_w, summarize, po, f)?, + Err(e) => write!(f, "{e:?}")?, + } + if i + 1 != dims[0] { + write!(f, ",")?; + Self::write_newline_indent(indent, f)? + } + } + } + } + } + write!(f, "]")?; + Ok(()) + } +} + +struct FloatFormatter { + int_mode: bool, + sci_mode: bool, + precision: usize, + _phantom: std::marker::PhantomData, +} + +impl FloatFormatter +where + S: WithDType + num_traits::Float + std::fmt::Display, +{ + fn new(t: &Tensor, po: &PrinterOptions) -> Result { + let mut int_mode = true; + let mut sci_mode = false; + + // Rather than containing all values, this should only include + // values that end up being displayed according to [threshold]. + let values = t + .flatten_all()? + .to_vec1()? + .into_iter() + .filter(|v: &S| v.is_finite() && !v.is_zero()) + .collect::>(); + if !values.is_empty() { + let mut nonzero_finite_min = S::max_value(); + let mut nonzero_finite_max = S::min_value(); + for &v in values.iter() { + let v = v.abs(); + if v < nonzero_finite_min { + nonzero_finite_min = v + } + if v > nonzero_finite_max { + nonzero_finite_max = v + } + } + + for &value in values.iter() { + if value.ceil() != value { + int_mode = false; + break; + } + } + if let Some(v1) = S::from(1000.) { + if let Some(v2) = S::from(1e8) { + if let Some(v3) = S::from(1e-4) { + sci_mode = nonzero_finite_max / nonzero_finite_min > v1 + || nonzero_finite_max > v2 + || nonzero_finite_min < v3 + } + } + } + } + + match po.sci_mode { + None => {} + Some(v) => sci_mode = v, + } + Ok(Self { + int_mode, + sci_mode, + precision: po.precision, + _phantom: std::marker::PhantomData, + }) + } +} + +impl TensorFormatter for FloatFormatter +where + S: WithDType + num_traits::Float + std::fmt::Display + std::fmt::LowerExp, +{ + type Elem = S; + + fn fmt(&self, v: Self::Elem, max_w: usize, f: &mut T) -> std::fmt::Result { + if self.sci_mode { + write!( + f, + "{v:width$.prec$e}", + v = v, + width = max_w, + prec = self.precision + ) + } else if self.int_mode { + if v.is_finite() { + write!(f, "{v:width$.0}.", v = v, width = max_w - 1) + } else { + write!(f, "{v:max_w$.0}") + } + } else { + write!( + f, + "{v:width$.prec$}", + v = v, + width = max_w, + prec = self.precision + ) + } + } +} + +struct IntFormatter { + _phantom: std::marker::PhantomData, +} + +impl IntFormatter { + fn new() -> Self { + Self { + _phantom: std::marker::PhantomData, + } + } +} + +impl TensorFormatter for IntFormatter +where + S: WithDType + std::fmt::Display, +{ + type Elem = S; + + fn fmt(&self, v: Self::Elem, max_w: usize, f: &mut T) -> std::fmt::Result { + write!(f, "{v:max_w$}") + } +} + +fn get_summarized_data(t: &Tensor, edge_items: usize) -> Result { + let dims = t.dims(); + if dims.is_empty() { + Ok(t.clone()) + } else if dims.len() == 1 { + if dims[0] > 2 * edge_items { + Tensor::cat( + &[ + t.narrow(0, 0, edge_items)?, + t.narrow(0, dims[0] - edge_items, edge_items)?, + ], + 0, + ) + } else { + Ok(t.clone()) + } + } else if dims[0] > 2 * edge_items { + let mut vs: Vec<_> = (0..edge_items) + .map(|i| get_summarized_data(&t.get(i)?, edge_items)) + .collect::>>()?; + for i in (dims[0] - edge_items)..dims[0] { + vs.push(get_summarized_data(&t.get(i)?, edge_items)?) + } + Tensor::cat(&vs, 0) + } else { + let vs: Vec<_> = (0..dims[0]) + .map(|i| get_summarized_data(&t.get(i)?, edge_items)) + .collect::>>()?; + Tensor::cat(&vs, 0) + } +} + +impl std::fmt::Display for Tensor { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let po = PRINT_OPTS.lock().unwrap(); + let summarize = self.elem_count() > po.threshold; + let to_display = if summarize { + match get_summarized_data(self, po.edge_items) { + Ok(v) => v, + Err(err) => return write!(f, "{err:?}"), + } + } else { + self.clone() + }; + match self.dtype() { + DType::U8 => { + let tf: IntFormatter = IntFormatter::new(); + let max_w = tf.max_width(&to_display); + tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; + writeln!(f)?; + } + DType::U32 => { + let tf: IntFormatter = IntFormatter::new(); + let max_w = tf.max_width(&to_display); + tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; + writeln!(f)?; + } + DType::I64 => { + let tf: IntFormatter = IntFormatter::new(); + let max_w = tf.max_width(&to_display); + tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; + writeln!(f)?; + } + DType::BF16 => { + if let Ok(tf) = FloatFormatter::::new(&to_display, &po) { + let max_w = tf.max_width(&to_display); + tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; + writeln!(f)?; + } + } + DType::F16 => { + if let Ok(tf) = FloatFormatter::::new(&to_display, &po) { + let max_w = tf.max_width(&to_display); + tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; + writeln!(f)?; + } + } + DType::F64 => { + if let Ok(tf) = FloatFormatter::::new(&to_display, &po) { + let max_w = tf.max_width(&to_display); + tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; + writeln!(f)?; + } + } + DType::F32 => { + if let Ok(tf) = FloatFormatter::::new(&to_display, &po) { + let max_w = tf.max_width(&to_display); + tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; + writeln!(f)?; + } + } + }; + + let device_str = match self.device().location() { + crate::DeviceLocation::Cpu => "".to_owned(), + crate::DeviceLocation::Cuda { gpu_id } => { + format!(", cuda:{}", gpu_id) + } + crate::DeviceLocation::Metal { gpu_id } => { + format!(", metal:{}", gpu_id) + } + }; + + write!( + f, + "Tensor[{:?}, {}{}]", + self.dims(), + self.dtype().as_str(), + device_str + ) + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/dtype.rs b/v2/dependency-patches/candle-core-0.4.1/src/dtype.rs new file mode 100644 index 0000000000..94ca57d8fd --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/dtype.rs @@ -0,0 +1,201 @@ +//! Types for elements that can be stored and manipulated using tensors. +#![allow(clippy::redundant_closure_call)] +use crate::backend::BackendStorage; +use crate::{CpuStorage, Error, Result}; + +/// The different types of elements allowed in tensors. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub enum DType { + // Unsigned 8 bits integer. + U8, + // Unsigned 32 bits integer. + U32, + // Signed 64 bits integer. + I64, + // Brain floating-point using half precision (16 bits). + BF16, + // Floating-point using half precision (16 bits). + F16, + // Floating-point using single precision (32 bits). + F32, + // Floating-point using double precision (64 bits). + F64, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct DTypeParseError; + +impl std::str::FromStr for DType { + type Err = DTypeParseError; + fn from_str(s: &str) -> std::result::Result { + match s { + "u8" => Ok(Self::U8), + "u32" => Ok(Self::U32), + "i64" => Ok(Self::I64), + "bf16" => Ok(Self::BF16), + "f16" => Ok(Self::F16), + "f32" => Ok(Self::F32), + "f64" => Ok(Self::F64), + _ => Err(DTypeParseError), + } + } +} + +impl DType { + /// String representation for dtypes. + pub fn as_str(&self) -> &'static str { + match self { + Self::U8 => "u8", + Self::U32 => "u32", + Self::I64 => "i64", + Self::BF16 => "bf16", + Self::F16 => "f16", + Self::F32 => "f32", + Self::F64 => "f64", + } + } + + /// The size used by each element in bytes, i.e. 1 for `U8`, 4 for `F32`. + pub fn size_in_bytes(&self) -> usize { + match self { + Self::U8 => 1, + Self::U32 => 4, + Self::I64 => 8, + Self::BF16 => 2, + Self::F16 => 2, + Self::F32 => 4, + Self::F64 => 8, + } + } + + pub fn is_int(&self) -> bool { + match self { + Self::U8 | Self::U32 | Self::I64 => true, + Self::BF16 | Self::F16 | Self::F32 | Self::F64 => false, + } + } + + pub fn is_float(&self) -> bool { + match self { + Self::U8 | Self::U32 | Self::I64 => false, + Self::BF16 | Self::F16 | Self::F32 | Self::F64 => true, + } + } +} + +pub trait WithDType: + Sized + + Copy + + num_traits::NumAssign + + std::cmp::PartialOrd + + std::fmt::Display + + 'static + + Send + + Sync + + crate::cpu::kernels::VecOps +{ + const DTYPE: DType; + + fn from_f64(v: f64) -> Self; + fn to_f64(self) -> f64; + fn to_cpu_storage_owned(data: Vec) -> CpuStorage; + + fn to_cpu_storage(data: &[Self]) -> CpuStorage { + Self::to_cpu_storage_owned(data.to_vec()) + } + + fn cpu_storage_as_slice(s: &CpuStorage) -> Result<&[Self]>; + fn cpu_storage_data(s: CpuStorage) -> Result>; +} + +macro_rules! with_dtype { + ($ty:ty, $dtype:ident, $from_f64:expr, $to_f64:expr) => { + impl WithDType for $ty { + const DTYPE: DType = DType::$dtype; + + fn from_f64(v: f64) -> Self { + $from_f64(v) + } + + fn to_f64(self) -> f64 { + $to_f64(self) + } + + fn to_cpu_storage_owned(data: Vec) -> CpuStorage { + CpuStorage::$dtype(data) + } + + fn cpu_storage_data(s: CpuStorage) -> Result> { + match s { + CpuStorage::$dtype(data) => Ok(data), + _ => Err(Error::UnexpectedDType { + expected: DType::$dtype, + got: s.dtype(), + msg: "unexpected dtype", + } + .bt()), + } + } + + fn cpu_storage_as_slice(s: &CpuStorage) -> Result<&[Self]> { + match s { + CpuStorage::$dtype(data) => Ok(data), + _ => Err(Error::UnexpectedDType { + expected: DType::$dtype, + got: s.dtype(), + msg: "unexpected dtype", + } + .bt()), + } + } + } + }; +} +use half::{bf16, f16}; + +with_dtype!(u8, U8, |v: f64| v as u8, |v: u8| v as f64); +with_dtype!(u32, U32, |v: f64| v as u32, |v: u32| v as f64); +with_dtype!(i64, I64, |v: f64| v as i64, |v: i64| v as f64); +with_dtype!(f16, F16, f16::from_f64, f16::to_f64); +with_dtype!(bf16, BF16, bf16::from_f64, bf16::to_f64); +with_dtype!(f32, F32, |v: f64| v as f32, |v: f32| v as f64); +with_dtype!(f64, F64, |v: f64| v, |v: f64| v); + +pub trait IntDType: WithDType { + fn is_true(&self) -> bool; + fn as_usize(&self) -> usize; +} + +impl IntDType for i64 { + fn is_true(&self) -> bool { + *self != 0 + } + fn as_usize(&self) -> usize { + *self as usize + } +} + +impl IntDType for u32 { + fn is_true(&self) -> bool { + *self != 0 + } + fn as_usize(&self) -> usize { + *self as usize + } +} + +impl IntDType for u8 { + fn is_true(&self) -> bool { + *self != 0 + } + fn as_usize(&self) -> usize { + *self as usize + } +} + +pub trait FloatDType: WithDType {} + +impl FloatDType for f16 {} +impl FloatDType for bf16 {} +impl FloatDType for f32 {} +impl FloatDType for f64 {} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/dummy_cuda_backend.rs b/v2/dependency-patches/candle-core-0.4.1/src/dummy_cuda_backend.rs new file mode 100644 index 0000000000..34c5d97f8c --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/dummy_cuda_backend.rs @@ -0,0 +1,211 @@ +#![allow(dead_code)] +use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; +use crate::{CpuStorage, DType, Error, Layout, Result, Shape}; + +#[derive(Debug, Clone)] +pub struct CudaDevice; + +#[derive(Debug)] +pub struct CudaStorage; + +macro_rules! fail { + () => { + unimplemented!("cuda support has not been enabled, add `cuda` feature to enable.") + }; +} + +impl crate::backend::BackendStorage for CudaStorage { + type Device = CudaDevice; + + fn try_clone(&self, _: &Layout) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn dtype(&self) -> DType { + fail!() + } + + fn device(&self) -> &Self::Device { + fail!() + } + + fn to_cpu_storage(&self) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn affine(&self, _: &Layout, _: f64, _: f64) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn powf(&self, _: &Layout, _: f64) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn elu(&self, _: &Layout, _: f64) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn reduce_op(&self, _: ReduceOp, _: &Layout, _: &[usize]) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn cmp(&self, _: CmpOp, _: &Self, _: &Layout, _: &Layout) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn to_dtype(&self, _: &Layout, _: DType) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn unary_impl(&self, _: &Layout) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn binary_impl(&self, _: &Self, _: &Layout, _: &Layout) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn where_cond(&self, _: &Layout, _: &Self, _: &Layout, _: &Self, _: &Layout) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn conv1d( + &self, + _: &Layout, + _: &Self, + _: &Layout, + _: &crate::conv::ParamsConv1D, + ) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn conv_transpose1d( + &self, + _: &Layout, + _: &Self, + _: &Layout, + _: &crate::conv::ParamsConvTranspose1D, + ) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn conv2d( + &self, + _: &Layout, + _: &Self, + _: &Layout, + _: &crate::conv::ParamsConv2D, + ) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn conv_transpose2d( + &self, + _l: &Layout, + _kernel: &Self, + _kernel_l: &Layout, + _params: &crate::conv::ParamsConvTranspose2D, + ) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn index_select(&self, _: &Self, _: &Layout, _: &Layout, _: usize) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + fn gather(&self, _: &Layout, _: &Self, _: &Layout, _: usize) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn scatter_add( + &self, + _: &Layout, + _: &Self, + _: &Layout, + _: &Self, + _: &Layout, + _: usize, + ) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn index_add( + &self, + _: &Layout, + _: &Self, + _: &Layout, + _: &Self, + _: &Layout, + _: usize, + ) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn matmul( + &self, + _: &Self, + _: (usize, usize, usize, usize), + _: &Layout, + _: &Layout, + ) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn copy_strided_src(&self, _: &mut Self, _: usize, _: &Layout) -> Result<()> { + Err(Error::NotCompiledWithCudaSupport) + } + + fn avg_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn max_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn upsample_nearest1d(&self, _: &Layout, _: usize) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn upsample_nearest2d(&self, _: &Layout, _: usize, _: usize) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } +} + +impl crate::backend::BackendDevice for CudaDevice { + type Storage = CudaStorage; + fn new(_: usize) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn set_seed(&self, _: u64) -> Result<()> { + Err(Error::NotCompiledWithCudaSupport) + } + + fn location(&self) -> crate::DeviceLocation { + fail!() + } + + fn same_device(&self, _: &Self) -> bool { + fail!() + } + + fn zeros_impl(&self, _shape: &Shape, _dtype: DType) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn ones_impl(&self, _shape: &Shape, _dtype: DType) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn storage_from_cpu_storage(&self, _: &CpuStorage) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn rand_uniform(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn rand_normal(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/dummy_metal_backend.rs b/v2/dependency-patches/candle-core-0.4.1/src/dummy_metal_backend.rs new file mode 100644 index 0000000000..e9d923310b --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/dummy_metal_backend.rs @@ -0,0 +1,223 @@ +#![allow(dead_code)] +use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; +use crate::{CpuStorage, DType, Error, Layout, Result, Shape}; + +#[derive(Debug, Clone)] +pub struct MetalDevice; + +#[derive(Debug)] +pub struct MetalStorage; + +#[derive(thiserror::Error, Debug)] +pub enum MetalError { + #[error("{0}")] + Message(String), +} + +impl From for MetalError { + fn from(e: String) -> Self { + MetalError::Message(e) + } +} + +macro_rules! fail { + () => { + unimplemented!("metal support has not been enabled, add `metal` feature to enable.") + }; +} + +impl crate::backend::BackendStorage for MetalStorage { + type Device = MetalDevice; + + fn try_clone(&self, _: &Layout) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn dtype(&self) -> DType { + fail!() + } + + fn device(&self) -> &Self::Device { + fail!() + } + + fn to_cpu_storage(&self) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn affine(&self, _: &Layout, _: f64, _: f64) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn powf(&self, _: &Layout, _: f64) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn elu(&self, _: &Layout, _: f64) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn reduce_op(&self, _: ReduceOp, _: &Layout, _: &[usize]) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn cmp(&self, _: CmpOp, _: &Self, _: &Layout, _: &Layout) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn to_dtype(&self, _: &Layout, _: DType) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn unary_impl(&self, _: &Layout) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn binary_impl(&self, _: &Self, _: &Layout, _: &Layout) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn where_cond(&self, _: &Layout, _: &Self, _: &Layout, _: &Self, _: &Layout) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn conv1d( + &self, + _: &Layout, + _: &Self, + _: &Layout, + _: &crate::conv::ParamsConv1D, + ) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn conv_transpose1d( + &self, + _l: &Layout, + _kernel: &Self, + _kernel_l: &Layout, + _params: &crate::conv::ParamsConvTranspose1D, + ) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn conv2d( + &self, + _: &Layout, + _: &Self, + _: &Layout, + _: &crate::conv::ParamsConv2D, + ) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn conv_transpose2d( + &self, + _l: &Layout, + _kernel: &Self, + _kernel_l: &Layout, + _params: &crate::conv::ParamsConvTranspose2D, + ) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn index_select(&self, _: &Self, _: &Layout, _: &Layout, _: usize) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + fn gather(&self, _: &Layout, _: &Self, _: &Layout, _: usize) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn scatter_add( + &self, + _: &Layout, + _: &Self, + _: &Layout, + _: &Self, + _: &Layout, + _: usize, + ) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn index_add( + &self, + _: &Layout, + _: &Self, + _: &Layout, + _: &Self, + _: &Layout, + _: usize, + ) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn matmul( + &self, + _: &Self, + _: (usize, usize, usize, usize), + _: &Layout, + _: &Layout, + ) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn copy_strided_src(&self, _: &mut Self, _: usize, _: &Layout) -> Result<()> { + Err(Error::NotCompiledWithMetalSupport) + } + + fn avg_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn max_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn upsample_nearest1d(&self, _: &Layout, _: usize) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn upsample_nearest2d(&self, _: &Layout, _: usize, _: usize) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } +} + +impl crate::backend::BackendDevice for MetalDevice { + type Storage = MetalStorage; + fn new(_: usize) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn set_seed(&self, _: u64) -> Result<()> { + Err(Error::NotCompiledWithMetalSupport) + } + + fn location(&self) -> crate::DeviceLocation { + fail!() + } + + fn same_device(&self, _: &Self) -> bool { + fail!() + } + + fn zeros_impl(&self, _shape: &Shape, _dtype: DType) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn ones_impl(&self, _shape: &Shape, _dtype: DType) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn storage_from_cpu_storage(&self, _: &CpuStorage) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn rand_uniform(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn rand_normal(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/error.rs b/v2/dependency-patches/candle-core-0.4.1/src/error.rs new file mode 100644 index 0000000000..60ddea1149 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/error.rs @@ -0,0 +1,265 @@ +use crate::{DType, DeviceLocation, Layout, MetalError, Shape}; + +#[derive(Debug, Clone)] +pub struct MatMulUnexpectedStriding { + pub lhs_l: Layout, + pub rhs_l: Layout, + pub bmnk: (usize, usize, usize, usize), + pub msg: &'static str, +} + +/// Main library error type. +#[derive(thiserror::Error, Debug)] +pub enum Error { + // === DType Errors === + #[error("{msg}, expected: {expected:?}, got: {got:?}")] + UnexpectedDType { + msg: &'static str, + expected: DType, + got: DType, + }, + + #[error("dtype mismatch in {op}, lhs: {lhs:?}, rhs: {rhs:?}")] + DTypeMismatchBinaryOp { + lhs: DType, + rhs: DType, + op: &'static str, + }, + + #[error("unsupported dtype {0:?} for op {1}")] + UnsupportedDTypeForOp(DType, &'static str), + + // === Dimension Index Errors === + #[error("{op}: dimension index {dim} out of range for shape {shape:?}")] + DimOutOfRange { + shape: Shape, + dim: i32, + op: &'static str, + }, + + #[error("{op}: duplicate dim index {dims:?} for shape {shape:?}")] + DuplicateDimIndex { + shape: Shape, + dims: Vec, + op: &'static str, + }, + + // === Shape Errors === + #[error("unexpected rank, expected: {expected}, got: {got} ({shape:?})")] + UnexpectedNumberOfDims { + expected: usize, + got: usize, + shape: Shape, + }, + + #[error("{msg}, expected: {expected:?}, got: {got:?}")] + UnexpectedShape { + msg: String, + expected: Shape, + got: Shape, + }, + + #[error( + "Shape mismatch, got buffer of size {buffer_size} which is compatible with shape {shape:?}" + )] + ShapeMismatch { buffer_size: usize, shape: Shape }, + + #[error("shape mismatch in {op}, lhs: {lhs:?}, rhs: {rhs:?}")] + ShapeMismatchBinaryOp { + lhs: Shape, + rhs: Shape, + op: &'static str, + }, + + #[error("shape mismatch in cat for dim {dim}, shape for arg 1: {first_shape:?} shape for arg {n}: {nth_shape:?}")] + ShapeMismatchCat { + dim: usize, + first_shape: Shape, + n: usize, + nth_shape: Shape, + }, + + #[error("Cannot divide tensor of shape {shape:?} equally along dim {dim} into {n_parts}")] + ShapeMismatchSplit { + shape: Shape, + dim: usize, + n_parts: usize, + }, + + #[error("{op} can only be performed on a single dimension")] + OnlySingleDimension { op: &'static str, dims: Vec }, + + #[error("empty tensor for {op}")] + EmptyTensor { op: &'static str }, + + // === Device Errors === + #[error("device mismatch in {op}, lhs: {lhs:?}, rhs: {rhs:?}")] + DeviceMismatchBinaryOp { + lhs: DeviceLocation, + rhs: DeviceLocation, + op: &'static str, + }, + + // === Op Specific Errors === + #[error("narrow invalid args {msg}: {shape:?}, dim: {dim}, start: {start}, len:{len}")] + NarrowInvalidArgs { + shape: Shape, + dim: usize, + start: usize, + len: usize, + msg: &'static str, + }, + + #[error("conv1d invalid args {msg}: inp: {inp_shape:?}, k: {k_shape:?}, pad: {padding}, stride: {stride}")] + Conv1dInvalidArgs { + inp_shape: Shape, + k_shape: Shape, + padding: usize, + stride: usize, + msg: &'static str, + }, + + #[error("{op} invalid index {index} with dim size {size}")] + InvalidIndex { + op: &'static str, + index: usize, + size: usize, + }, + + #[error("cannot broadcast {src_shape:?} to {dst_shape:?}")] + BroadcastIncompatibleShapes { src_shape: Shape, dst_shape: Shape }, + + #[error("cannot set variable {msg}")] + CannotSetVar { msg: &'static str }, + + // Box indirection to avoid large variant. + #[error("{0:?}")] + MatMulUnexpectedStriding(Box), + + #[error("{op} only supports contiguous tensors")] + RequiresContiguous { op: &'static str }, + + #[error("{op} expects at least one tensor")] + OpRequiresAtLeastOneTensor { op: &'static str }, + + #[error("{op} expects at least two tensors")] + OpRequiresAtLeastTwoTensors { op: &'static str }, + + #[error("backward is not supported for {op}")] + BackwardNotSupported { op: &'static str }, + + // === Other Errors === + #[error("the candle crate has not been built with cuda support")] + NotCompiledWithCudaSupport, + + #[error("the candle crate has not been built with metal support")] + NotCompiledWithMetalSupport, + + #[error("cannot find tensor {path}")] + CannotFindTensor { path: String }, + + // === Wrapped Errors === + #[error(transparent)] + Cuda(Box), + + #[error("Metal error {0}")] + Metal(#[from] MetalError), + + #[error(transparent)] + TryFromIntError(#[from] core::num::TryFromIntError), + + #[error("npy/npz error {0}")] + Npy(String), + + /// Zip file format error. + #[error(transparent)] + Zip(#[from] zip::result::ZipError), + + /// Integer parse error. + #[error(transparent)] + ParseInt(#[from] std::num::ParseIntError), + + /// I/O error. + #[error(transparent)] + Io(#[from] std::io::Error), + + /// SafeTensor error. + #[error(transparent)] + SafeTensor(#[from] safetensors::SafeTensorError), + + #[error("unsupported safetensor dtype {0:?}")] + UnsupportedSafeTensorDtype(safetensors::Dtype), + + /// Arbitrary errors wrapping. + #[error(transparent)] + Wrapped(Box), + + /// Adding path information to an error. + #[error("path: {path:?} {inner}")] + WithPath { + inner: Box, + path: std::path::PathBuf, + }, + + #[error("{inner}\n{backtrace}")] + WithBacktrace { + inner: Box, + backtrace: Box, + }, + + /// User generated error message, typically created via `bail!`. + #[error("{0}")] + Msg(String), +} + +pub type Result = std::result::Result; + +impl Error { + pub fn wrap(err: impl std::error::Error + Send + Sync + 'static) -> Self { + Self::Wrapped(Box::new(err)).bt() + } + + pub fn msg(err: impl std::error::Error + Send + Sync + 'static) -> Self { + Self::Msg(err.to_string()).bt() + } + + pub fn bt(self) -> Self { + let backtrace = std::backtrace::Backtrace::capture(); + match backtrace.status() { + std::backtrace::BacktraceStatus::Disabled + | std::backtrace::BacktraceStatus::Unsupported => self, + _ => Self::WithBacktrace { + inner: Box::new(self), + backtrace: Box::new(backtrace), + }, + } + } + + pub fn with_path>(self, p: P) -> Self { + Self::WithPath { + inner: Box::new(self), + path: p.as_ref().to_path_buf(), + } + } +} + +#[macro_export] +macro_rules! bail { + ($msg:literal $(,)?) => { + return Err($crate::Error::Msg(format!($msg).into()).bt()) + }; + ($err:expr $(,)?) => { + return Err($crate::Error::Msg(format!($err).into()).bt()) + }; + ($fmt:expr, $($arg:tt)*) => { + return Err($crate::Error::Msg(format!($fmt, $($arg)*).into()).bt()) + }; +} + +pub fn zip(r1: Result, r2: Result) -> Result<(T, U)> { + match (r1, r2) { + (Ok(r1), Ok(r2)) => Ok((r1, r2)), + (Err(e), _) => Err(e), + (_, Err(e)) => Err(e), + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/indexer.rs b/v2/dependency-patches/candle-core-0.4.1/src/indexer.rs new file mode 100644 index 0000000000..e3ed41e52b --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/indexer.rs @@ -0,0 +1,168 @@ +use crate::{Error, Tensor}; +use std::ops::{ + Bound, Range, RangeBounds, RangeFrom, RangeFull, RangeInclusive, RangeTo, RangeToInclusive, +}; + +impl Tensor { + /// Intended to be use by the trait `.i()` + /// + /// ``` + /// # use candle_core::{Tensor, DType, Device, IndexOp}; + /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; + /// + /// let c = a.i(0..1)?; + /// assert_eq!(c.shape().dims(), &[1, 3]); + /// + /// let c = a.i(0)?; + /// assert_eq!(c.shape().dims(), &[3]); + /// + /// let c = a.i((.., ..2) )?; + /// assert_eq!(c.shape().dims(), &[2, 2]); + /// + /// let c = a.i((.., ..=2))?; + /// assert_eq!(c.shape().dims(), &[2, 3]); + /// + /// # Ok::<(), candle_core::Error>(()) + /// ``` + fn index(&self, indexers: &[TensorIndexer]) -> Result { + let mut x = self.clone(); + let dims = self.shape().dims(); + let mut current_dim = 0; + for (i, indexer) in indexers.iter().enumerate() { + x = match indexer { + TensorIndexer::Select(n) => x.narrow(current_dim, *n, 1)?.squeeze(current_dim)?, + TensorIndexer::Narrow(left_bound, right_bound) => { + let start = match left_bound { + Bound::Included(n) => *n, + Bound::Excluded(n) => *n + 1, + Bound::Unbounded => 0, + }; + let stop = match right_bound { + Bound::Included(n) => *n + 1, + Bound::Excluded(n) => *n, + Bound::Unbounded => dims[i], + }; + let out = x.narrow(current_dim, start, stop.saturating_sub(start))?; + current_dim += 1; + out + } + TensorIndexer::IndexSelect(indexes) => { + if indexes.rank() != 1 { + crate::bail!("multi-dimensional tensor indexing is not supported") + } + let out = x.index_select(&indexes.to_device(x.device())?, current_dim)?; + current_dim += 1; + out + } + TensorIndexer::Err(e) => crate::bail!("indexing error {e:?}"), + }; + } + Ok(x) + } +} + +#[derive(Debug)] +/// Generic structure used to index a slice of the tensor +pub enum TensorIndexer { + /// This selects the elements for which an index has some specific value. + Select(usize), + /// This is a regular slice, purely indexing a chunk of the tensor + Narrow(Bound, Bound), + /// Indexing via a 1d tensor + IndexSelect(Tensor), + Err(Error), +} + +impl From for TensorIndexer { + fn from(index: usize) -> Self { + TensorIndexer::Select(index) + } +} + +impl From<&[u32]> for TensorIndexer { + fn from(index: &[u32]) -> Self { + match Tensor::new(index, &crate::Device::Cpu) { + Ok(tensor) => TensorIndexer::IndexSelect(tensor), + Err(e) => TensorIndexer::Err(e), + } + } +} + +impl From> for TensorIndexer { + fn from(index: Vec) -> Self { + let len = index.len(); + match Tensor::from_vec(index, len, &crate::Device::Cpu) { + Ok(tensor) => TensorIndexer::IndexSelect(tensor), + Err(e) => TensorIndexer::Err(e), + } + } +} + +impl From<&Tensor> for TensorIndexer { + fn from(tensor: &Tensor) -> Self { + TensorIndexer::IndexSelect(tensor.clone()) + } +} + +trait RB: RangeBounds {} +impl RB for Range {} +impl RB for RangeFrom {} +impl RB for RangeFull {} +impl RB for RangeInclusive {} +impl RB for RangeTo {} +impl RB for RangeToInclusive {} + +impl From for TensorIndexer { + fn from(range: T) -> Self { + use std::ops::Bound::*; + let start = match range.start_bound() { + Included(idx) => Included(*idx), + Excluded(idx) => Excluded(*idx), + Unbounded => Unbounded, + }; + let end = match range.end_bound() { + Included(idx) => Included(*idx), + Excluded(idx) => Excluded(*idx), + Unbounded => Unbounded, + }; + TensorIndexer::Narrow(start, end) + } +} + +/// Trait used to implement multiple signatures for ease of use of the slicing +/// of a tensor +pub trait IndexOp { + /// Returns a slicing iterator which are the chunks of data necessary to + /// reconstruct the desired tensor. + fn i(&self, index: T) -> Result; +} + +impl IndexOp for Tensor +where + T: Into, +{ + fn i(&self, index: T) -> Result { + self.index(&[index.into()]) + } +} + +macro_rules! index_op_tuple { + ($($t:ident),+) => { + #[allow(non_snake_case)] + impl<$($t),*> IndexOp<($($t,)*)> for Tensor + where + $($t: Into,)* + { + fn i(&self, ($($t,)*): ($($t,)*)) -> Result { + self.index(&[$($t.into(),)*]) + } + } + }; +} +index_op_tuple!(A); +index_op_tuple!(A, B); +index_op_tuple!(A, B, C); +index_op_tuple!(A, B, C, D); +index_op_tuple!(A, B, C, D, E); +index_op_tuple!(A, B, C, D, E, F); +index_op_tuple!(A, B, C, D, E, F, G); diff --git a/v2/dependency-patches/candle-core-0.4.1/src/layout.rs b/v2/dependency-patches/candle-core-0.4.1/src/layout.rs new file mode 100644 index 0000000000..bf346cf270 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/layout.rs @@ -0,0 +1,271 @@ +use crate::{Error, Result, Shape}; + +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct Layout { + shape: Shape, + // The strides are given in number of elements and not in bytes. + stride: Vec, + start_offset: usize, +} + +impl Layout { + pub fn new(shape: Shape, stride: Vec, start_offset: usize) -> Self { + Self { + shape, + stride, + start_offset, + } + } + + pub fn contiguous_with_offset>(shape: S, start_offset: usize) -> Self { + let shape = shape.into(); + let stride = shape.stride_contiguous(); + Self { + shape, + stride, + start_offset, + } + } + + pub fn contiguous>(shape: S) -> Self { + Self::contiguous_with_offset(shape, 0) + } + + pub fn dims(&self) -> &[usize] { + self.shape.dims() + } + + pub fn shape(&self) -> &Shape { + &self.shape + } + + pub fn stride(&self) -> &[usize] { + &self.stride + } + + pub fn start_offset(&self) -> usize { + self.start_offset + } + + /// Returns the appropriate start and stop offset if the data is stored in a C + /// contiguous (aka row major) way. + pub fn contiguous_offsets(&self) -> Option<(usize, usize)> { + if self.is_contiguous() { + let start_o = self.start_offset; + Some((start_o, start_o + self.shape.elem_count())) + } else { + None + } + } + + /// Returns true if the data is stored in a C contiguous (aka row major) way. + /// Note that this does not implies that the start offset is 0 or that there are no extra + /// elements at the end of the storage. + pub fn is_contiguous(&self) -> bool { + self.shape.is_contiguous(&self.stride) + } + + /// Returns true if the data is stored in a Fortran contiguous (aka column major) way. + pub fn is_fortran_contiguous(&self) -> bool { + self.shape.is_fortran_contiguous(&self.stride) + } + + pub(crate) fn narrow(&self, dim: usize, start: usize, len: usize) -> Result { + let dims = self.shape().dims(); + if dim >= dims.len() { + Err(Error::DimOutOfRange { + shape: self.shape().clone(), + dim: dim as i32, + op: "narrow", + } + .bt())? + } + if start + len > dims[dim] { + Err(Error::NarrowInvalidArgs { + shape: self.shape.clone(), + dim, + start, + len, + msg: "start + len > dim_len", + } + .bt())? + } + let mut dims = dims.to_vec(); + dims[dim] = len; + Ok(Self { + shape: Shape::from(dims), + stride: self.stride.clone(), + start_offset: self.start_offset + self.stride[dim] * start, + }) + } + + pub(crate) fn transpose(&self, dim1: usize, dim2: usize) -> Result { + let rank = self.shape.rank(); + if rank <= dim1 || rank <= dim2 { + Err(Error::UnexpectedNumberOfDims { + expected: usize::max(dim1, dim2), + got: rank, + shape: self.shape().clone(), + } + .bt())? + } + let mut stride = self.stride().to_vec(); + let mut dims = self.shape().dims().to_vec(); + dims.swap(dim1, dim2); + stride.swap(dim1, dim2); + Ok(Self { + shape: Shape::from(dims), + stride, + start_offset: self.start_offset, + }) + } + + pub(crate) fn permute(&self, idxs: &[usize]) -> Result { + let is_permutation = + idxs.len() == self.shape.rank() && (0..idxs.len()).all(|i| idxs.contains(&i)); + if !is_permutation { + crate::bail!( + "dimension mismatch in permute, tensor {:?}, dims: {:?}", + self.dims(), + idxs + ) + } + let stride = self.stride(); + let dims = self.shape().dims(); + let mut perm_stride = stride.to_vec(); + let mut perm_dims = dims.to_vec(); + for (i, &idx) in idxs.iter().enumerate() { + perm_stride[i] = stride[idx]; + perm_dims[i] = dims[idx]; + } + Ok(Self { + shape: Shape::from(perm_dims), + stride: perm_stride, + start_offset: self.start_offset, + }) + } + + pub fn broadcast_as>(&self, shape: S) -> Result { + let shape = shape.into(); + if shape.rank() < self.shape().rank() { + return Err(Error::BroadcastIncompatibleShapes { + src_shape: self.shape().clone(), + dst_shape: shape, + } + .bt()); + } + let added_dims = shape.rank() - self.shape().rank(); + let mut stride = vec![0; added_dims]; + for (&dst_dim, (&src_dim, &src_stride)) in shape.dims()[added_dims..] + .iter() + .zip(self.dims().iter().zip(self.stride())) + { + let s = if dst_dim == src_dim { + src_stride + } else if src_dim != 1 { + return Err(Error::BroadcastIncompatibleShapes { + src_shape: self.shape().clone(), + dst_shape: shape, + } + .bt()); + } else { + 0 + }; + stride.push(s) + } + Ok(Self { + shape, + stride, + start_offset: self.start_offset, + }) + } + + pub(crate) fn strided_index(&self) -> crate::StridedIndex { + crate::StridedIndex::from_layout(self) + } + + pub(crate) fn strided_blocks(&self) -> crate::StridedBlocks { + let mut block_len = 1; + let mut contiguous_dims = 0; // These are counted from the right. + for (&stride, &dim) in self.stride().iter().zip(self.dims().iter()).rev() { + if stride != block_len { + break; + } + block_len *= dim; + contiguous_dims += 1; + } + let index_dims = self.dims().len() - contiguous_dims; + if index_dims == 0 { + crate::StridedBlocks::SingleBlock { + start_offset: self.start_offset, + len: block_len, + } + } else { + let block_start_index = crate::StridedIndex::new( + &self.dims()[..index_dims], + &self.stride[..index_dims], + self.start_offset, + ); + crate::StridedBlocks::MultipleBlocks { + block_start_index, + block_len, + } + } + } + + // Returns the contiguous offsets with broadcast if applicable. + pub(crate) fn offsets_b(&self) -> Option { + let mut left_broadcast = 1; + let mut right_broadcast = 1; + let strides = self.stride(); + let dims = self.dims(); + let mut start_cont = 0; + let mut end_cont = dims.len(); + for (&s, &d) in strides.iter().zip(dims.iter()) { + if s != 0 { + break; + } + start_cont += 1; + left_broadcast *= d; + } + if start_cont == dims.len() { + return Some(ContiguousOffsetsWithBroadcast { + start: self.start_offset, + len: 1, + left_broadcast, + right_broadcast: 1, + }); + } + for (&s, &d) in strides.iter().zip(dims.iter()).rev() { + if s != 0 { + break; + } + end_cont -= 1; + right_broadcast *= d; + } + // Check that the inner dims are contiguous + let strides = &strides[start_cont..end_cont]; + let dims = &dims[start_cont..end_cont]; + let mut len = 1; + for (&stride, &dim) in strides.iter().zip(dims.iter()).rev() { + if stride != len { + return None; + } + len *= dim; + } + Some(ContiguousOffsetsWithBroadcast { + start: self.start_offset, + len, + left_broadcast, + right_broadcast, + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ContiguousOffsetsWithBroadcast { + pub start: usize, + pub len: usize, + pub left_broadcast: usize, + pub right_broadcast: usize, +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/lib.rs b/v2/dependency-patches/candle-core-0.4.1/src/lib.rs new file mode 100644 index 0000000000..fcc17afcd7 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/lib.rs @@ -0,0 +1,151 @@ +//! ML framework for Rust +//! +//! ```rust +//! use candle_core::{Tensor, DType, Device}; +//! # use candle_core::Error; +//! # fn main() -> Result<(), Error>{ +//! +//! let a = Tensor::arange(0f32, 6f32, &Device::Cpu)?.reshape((2, 3))?; +//! let b = Tensor::arange(0f32, 12f32, &Device::Cpu)?.reshape((3, 4))?; +//! +//! let c = a.matmul(&b)?; +//! # Ok(())} +//! ``` +//! +//! ## Features +//! +//! - Simple syntax (looks and like PyTorch) +//! - CPU and Cuda backends (and M1 support) +//! - Enable serverless (CPU) small and fast deployments +//! - Model training +//! - Distributed computing (NCCL). +//! - Models out of the box (Llama, Whisper, Falcon, ...) +//! +//! ## FAQ +//! +//! - Why Candle? +//! +//! Candle stems from the need to reduce binary size in order to *enable serverless* +//! possible by making the whole engine smaller than PyTorch very large library volume +//! +//! And simply *removing Python* from production workloads. +//! Python can really add overhead in more complex workflows and the [GIL](https://www.backblaze.com/blog/the-python-gil-past-present-and-future/) is a notorious source of headaches. +//! +//! Rust is cool, and a lot of the HF ecosystem already has Rust crates [safetensors](https://github.com/huggingface/safetensors) and [tokenizers](https://github.com/huggingface/tokenizers) + +#[cfg(feature = "accelerate")] +mod accelerate; +pub mod backend; +pub mod backprop; +mod conv; +mod convert; +pub mod cpu; +pub mod cpu_backend; +#[cfg(feature = "cuda")] +pub mod cuda_backend; +#[cfg(feature = "cudnn")] +pub mod cudnn; +mod device; +pub mod display; +mod dtype; +mod dummy_cuda_backend; +mod dummy_metal_backend; +pub mod error; +mod indexer; +pub mod layout; +#[cfg(feature = "metal")] +pub mod metal_backend; +#[cfg(feature = "mkl")] +mod mkl; +pub mod npy; +mod op; +pub mod pickle; +pub mod quantized; +pub mod safetensors; +pub mod scalar; +pub mod shape; +mod storage; +mod strided_index; +mod tensor; +pub mod test_utils; +pub mod utils; +mod variable; + +pub use cpu_backend::CpuStorage; +pub use device::{Device, DeviceLocation, NdArray}; +pub use dtype::{DType, FloatDType, IntDType, WithDType}; +pub use error::{Error, Result}; +pub use indexer::IndexOp; +pub use layout::Layout; +pub use op::{CustomOp1, CustomOp2, CustomOp3}; +pub use shape::{Shape, D}; +pub use storage::Storage; +pub use strided_index::{StridedBlocks, StridedIndex}; +pub use tensor::{Tensor, TensorId}; +pub use variable::Var; + +#[cfg(feature = "cuda")] +pub use cuda_backend::{CudaDevice, CudaStorage}; + +#[cfg(not(feature = "cuda"))] +pub use dummy_cuda_backend::{CudaDevice, CudaStorage}; + +#[cfg(feature = "metal")] +pub use metal_backend::{MetalDevice, MetalError, MetalStorage}; + +#[cfg(not(feature = "metal"))] +pub use dummy_metal_backend::{MetalDevice, MetalError, MetalStorage}; + +#[cfg(feature = "mkl")] +extern crate intel_mkl_src; + +#[cfg(feature = "accelerate")] +extern crate accelerate_src; + +pub trait ToUsize2 { + fn to_usize2(self) -> (usize, usize); +} + +impl ToUsize2 for usize { + fn to_usize2(self) -> (usize, usize) { + (self, self) + } +} + +impl ToUsize2 for (usize, usize) { + fn to_usize2(self) -> (usize, usize) { + self + } +} + +// A simple trait defining a module with forward method using a single argument. +pub trait Module { + fn forward(&self, xs: &Tensor) -> Result; +} + +impl Result> Module for T { + fn forward(&self, xs: &Tensor) -> Result { + self(xs) + } +} + +impl Module for Option<&M> { + fn forward(&self, xs: &Tensor) -> Result { + match self { + None => Ok(xs.clone()), + Some(m) => m.forward(xs), + } + } +} + +// A trait defining a module with forward method using a single tensor argument and a flag to +// separate the training and evaluation behaviors. +pub trait ModuleT { + fn forward_t(&self, xs: &Tensor, train: bool) -> Result; +} + +impl ModuleT for M { + fn forward_t(&self, xs: &Tensor, _train: bool) -> Result { + self.forward(xs) + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/metal_backend.rs b/v2/dependency-patches/candle-core-0.4.1/src/metal_backend.rs new file mode 100644 index 0000000000..959f0f310c --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/metal_backend.rs @@ -0,0 +1,1699 @@ +use crate::backend::{BackendDevice, BackendStorage}; +use crate::conv::{ParamsConv1D, ParamsConv2D, ParamsConvTranspose1D, ParamsConvTranspose2D}; +use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; +use crate::{CpuStorage, DType, Layout, Result, Shape}; +use candle_metal_kernels; +use candle_metal_kernels::Kernels; +use metal; +use metal::{Buffer, CommandBuffer, CommandQueue, MTLResourceOptions, NSUInteger}; +use std::collections::HashMap; +use std::ffi::c_void; +use std::path::Path; +use std::sync::{Arc, Mutex, RwLock, TryLockError}; + +/// Simple way to catch lock error without +/// depending on T +#[derive(thiserror::Error, Debug)] +pub enum LockError { + #[error("{0}")] + Poisoned(String), + #[error("Would block")] + WouldBlock, +} + +impl From> for MetalError { + fn from(value: TryLockError) -> Self { + match value { + TryLockError::Poisoned(p) => MetalError::LockError(LockError::Poisoned(p.to_string())), + TryLockError::WouldBlock => MetalError::LockError(LockError::WouldBlock), + } + } +} + +/// Metal related errors +#[derive(thiserror::Error, Debug)] +pub enum MetalError { + #[error("{0}")] + Message(String), + #[error(transparent)] + KernelError(#[from] candle_metal_kernels::MetalKernelError), + + #[error("matmul is only supported for contiguous tensors lstride: {lhs_stride:?} rstride: {rhs_stride:?} mnk: {mnk:?}")] + MatMulNonContiguous { + lhs_stride: Vec, + rhs_stride: Vec, + mnk: (usize, usize, usize), + }, + #[error("{0:?}")] + LockError(LockError), + #[error("{msg}, expected: {expected:?}, got: {got:?}")] + UnexpectedDType { + msg: &'static str, + expected: DType, + got: DType, + }, +} + +impl From for MetalError { + fn from(e: String) -> Self { + MetalError::Message(e) + } +} + +type AllocatedBuffers = Arc>>>>; + +#[derive(Clone)] +pub struct MetalDevice { + /// Raw metal device: + device: metal::Device, + + /// Single command queue for the entire device. + command_queue: metal::CommandQueue, + /// One command buffer at a time. + /// The scheduler works by allowing multiple + /// [ComputeCommandEncoder](https://developer.apple.com/documentation/metal/mtlcomputecommandencoder?language=objc) + /// on a single command buffer. Using a single command buffer would be fastest on the GPU but + /// prevents overlapping of CPU and GPU commands (because command buffer needs to be committed + /// to start to work). + /// Despite what the documentation says, command buffers are NOT ordered. They are ordered + /// for their START time, but there's no guarantee that command buffer1 will finish before + /// command buffer2 starts (or there are metal bugs there) + command_buffer: Arc>, + /// Keeps track of the current amount of compute command encoders on the current + /// command buffer + /// Arc, RwLock because of the interior mutability. + command_buffer_index: Arc>, + /// The maximum amount of [compute command encoder](https://developer.apple.com/documentation/metal/mtlcomputecommandencoder?language=objc) per [command buffer](https://developer.apple.com/documentation/metal/mtlcommandbuffer?language=objc) + compute_per_buffer: usize, + /// Simple keeper struct to keep track of the already compiled kernels so we can reuse them. + /// Heavily used by [`candle_metal_kernels`] + kernels: Arc, + /// Simple allocator struct. + /// The buffers are stored in size buckets since ML tends to use similar shapes over and over. + /// We store the buffers in [`Arc`] because it's much faster than Obj-c internal ref counting + /// (could be linked to FFI communication overhead). + /// + /// Whenever a buffer has a strong_count==1, we can reuse it, it means it was dropped in the + /// graph calculation, and only we the allocator kept a reference to it, therefore it's free + /// to be reused. However, in order for this to work, we need to guarantee the order of + /// operation, so that this buffer is not being used by another kernel at the same time. + /// Arc is the CPU reference count, it doesn't mean anything on the GPU side of things. + /// + /// Whenever we actually allocate a new buffer, we make a full sweep to cleanup unused buffers + /// (strong_count = 1). + buffers: AllocatedBuffers, + /// Seed for random number generation. + seed: Arc>, +} + +impl std::fmt::Debug for MetalDevice { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "MetalDevice({:?})", self.device.registry_id()) + } +} + +impl std::ops::Deref for MetalDevice { + type Target = metal::DeviceRef; + + fn deref(&self) -> &Self::Target { + &self.device + } +} + +impl MetalDevice { + pub fn id(&self) -> NSUInteger { + self.registry_id() + } + + pub fn metal_device(&self) -> &metal::Device { + &self.device + } + + pub fn command_queue(&self) -> &CommandQueue { + &self.command_queue + } + + pub fn command_buffer(&self) -> Result { + let mut command_buffer_lock = self.command_buffer.try_write().map_err(MetalError::from)?; + let mut command_buffer = command_buffer_lock.to_owned(); + let mut index = self + .command_buffer_index + .try_write() + .map_err(MetalError::from)?; + if *index > self.compute_per_buffer { + command_buffer.commit(); + command_buffer = self.command_queue.new_command_buffer().to_owned(); + *command_buffer_lock = command_buffer.clone(); + *index = 0; + } + *index += 1; + Ok(command_buffer) + } + + pub fn wait_until_completed(&self) -> Result<()> { + let mut command_buffer = self.command_buffer.try_write().map_err(MetalError::from)?; + match command_buffer.status() { + metal::MTLCommandBufferStatus::Committed + | metal::MTLCommandBufferStatus::Scheduled + | metal::MTLCommandBufferStatus::Completed => { + panic!("Already committed"); + } + _ => {} + } + command_buffer.commit(); + command_buffer.wait_until_completed(); + *command_buffer = self.command_queue.new_command_buffer().to_owned(); + Ok(()) + } + + pub fn kernels(&self) -> &Kernels { + &self.kernels + } + + pub fn device(&self) -> &metal::Device { + &self.device + } + + /// Creates a new buffer (not necessarily zeroed). + /// The buffer is [MTLPrivate](https://developer.apple.com/documentation/metal/mtlstoragemode) + /// This means the buffer data cannot be read on the CPU directly. + /// + /// [`name`] is only used to keep track of the resource origin in case of bugs + pub fn new_buffer( + &self, + element_count: usize, + dtype: DType, + name: &str, + ) -> Result> { + let size = (element_count * dtype.size_in_bytes()) as NSUInteger; + self.allocate_buffer(size, MTLResourceOptions::StorageModePrivate, name) + } + + /// Creates a new buffer (not necessarily zeroed). + /// The buffer is [MTLManaged](https://developer.apple.com/documentation/metal/mtlstoragemode) + /// This means the buffer can be read on the CPU but will require manual + /// synchronization when the CPU memory is modified + /// Used as a bridge to gather data back from the GPU + pub fn new_buffer_managed(&self, size: NSUInteger) -> Result> { + self.allocate_buffer(size, MTLResourceOptions::StorageModeManaged, "managed") + } + + /// Creates a new buffer from data. + /// The buffer is [MTLPrivate](https://developer.apple.com/documentation/metal/mtlstoragemode) + /// + /// This method will block the computation because of the + /// lack of lifetime management through the GPU. + /// Internal comment for technical details. + pub fn new_buffer_with_data(&self, data: &[T]) -> Result> { + let size = core::mem::size_of_val(data) as NSUInteger; + let tmp = self.device.new_buffer_with_data( + data.as_ptr() as *const core::ffi::c_void, + size, + metal::MTLResourceOptions::StorageModeManaged, + ); + let real = self.allocate_buffer( + size, + metal::MTLResourceOptions::StorageModePrivate, + "with_data", + )?; + let command_buffer = self.command_buffer()?; + command_buffer.set_label("with_data"); + let blit = command_buffer.new_blit_command_encoder(); + blit.set_label("with_data_blit"); + blit.copy_from_buffer(&tmp, 0, &real, 0, tmp.length()); + blit.end_encoding(); + + // This is necessary, for mmaped safetensors + // Because of the unsafe slice cast we're doing. + // The slice might not live long enough for metal + // To actually fill the GPU buffer. + // Putting this wait forces the GPU buffer to be filled + // with the actual data allowing the CPU storage to do + // deallocate properly. + self.wait_until_completed()?; + Ok(real) + } + + pub fn allocate_zeros(&self, size_in_bytes: usize) -> Result> { + let buffer = self.allocate_buffer( + size_in_bytes as NSUInteger, + MTLResourceOptions::StorageModePrivate, + "allocate_zeros", + )?; + let command_buffer = self.command_buffer()?; + command_buffer.set_label("zeros"); + let blit = command_buffer.new_blit_command_encoder(); + blit.fill_buffer( + &buffer, + metal::NSRange { + location: 0, + length: buffer.length(), + }, + 0, + ); + blit.end_encoding(); + Ok(buffer) + } + + /// The critical allocator algorithm + fn allocate_buffer( + &self, + size: NSUInteger, + option: MTLResourceOptions, + _name: &str, + ) -> Result> { + let mut buffers = self.buffers.try_write().map_err(MetalError::from)?; + let subbuffers = buffers.entry((size, option)).or_insert(vec![]); + + for sub in &mut *subbuffers { + if Arc::strong_count(sub) == 1 { + return Ok(sub.clone()); + } + } + let new_buffer = self.device.new_buffer(size as NSUInteger, option); + let new_buffer = Arc::new(new_buffer); + subbuffers.push(new_buffer.clone()); + for subbuffers in buffers.values_mut() { + let newbuffers = subbuffers + .iter() + .filter(|s| Arc::strong_count(s) > 1) + .map(Arc::clone) + .collect(); + *subbuffers = newbuffers; + } + Ok(new_buffer) + } + + /// Create a metal GPU capture trace on [`path`]. + pub fn capture>(&self, path: P) -> Result<()> { + let capture = metal::CaptureManager::shared(); + let descriptor = metal::CaptureDescriptor::new(); + descriptor.set_destination(metal::MTLCaptureDestination::GpuTraceDocument); + descriptor.set_capture_device(self); + descriptor.set_output_url(path); + + capture + .start_capture(&descriptor) + .map_err(MetalError::from)?; + Ok(()) + } +} + +#[derive(Debug, Clone)] +pub struct MetalStorage { + /// The actual buffer containing the data. + buffer: Arc, + /// a reference to the device owning this buffer + device: MetalDevice, + /// The dtype is kept since buffers are untyped. + dtype: DType, +} + +impl BackendStorage for MetalStorage { + type Device = MetalDevice; + + fn try_clone(&self, _: &Layout) -> Result { + Ok(self.clone()) + } + + fn dtype(&self) -> DType { + self.dtype + } + + fn device(&self) -> &Self::Device { + &self.device + } + + fn to_cpu_storage(&self) -> Result { + match self.dtype { + DType::U8 => Ok(CpuStorage::U8(self.to_cpu()?)), + DType::U32 => Ok(CpuStorage::U32(self.to_cpu()?)), + DType::I64 => Ok(CpuStorage::I64(self.to_cpu()?)), + DType::F16 => Ok(CpuStorage::F16(self.to_cpu()?)), + DType::BF16 => Ok(CpuStorage::BF16(self.to_cpu()?)), + DType::F32 => Ok(CpuStorage::F32(self.to_cpu()?)), + DType::F64 => Ok(CpuStorage::F64(self.to_cpu()?)), + } + } + + fn affine(&self, layout: &Layout, mul: f64, add: f64) -> Result { + let device = self.device().clone(); + + let shape = layout.shape(); + let el = shape.elem_count(); + let dtype = self.dtype; + + let buffer = device.new_buffer(el, self.dtype, "affine")?; + let command_buffer = self.device.command_buffer()?; + if layout.is_contiguous() && layout.start_offset() == 0 { + let name = match self.dtype { + DType::F32 => "affine_f32", + DType::F16 => "affine_f16", + DType::BF16 => "affine_bf16", + dtype => crate::bail!("Metal contiguous affine {dtype:?} not implemented"), + }; + candle_metal_kernels::call_affine( + &device.device, + &command_buffer, + &device.kernels, + name, + el, + &self.buffer, + &buffer, + mul as f32, + add as f32, + ) + .map_err(MetalError::from)?; + } else { + let name = match self.dtype { + DType::F32 => "affine_f32_strided", + DType::F16 => "affine_f16_strided", + DType::BF16 => "affine_bf16_strided", + dtype => crate::bail!("Metal strided affine {dtype:?} not implemented"), + }; + candle_metal_kernels::call_affine_strided( + &device.device, + &command_buffer, + &device.kernels, + name, + layout.dims(), + &self.buffer, + layout.stride(), + layout.start_offset() * dtype.size_in_bytes(), + &buffer, + mul as f32, + add as f32, + ) + .map_err(MetalError::from)?; + } + Ok(Self::new(buffer, device.clone(), dtype)) + } + + fn powf(&self, layout: &Layout, pow: f64) -> Result { + let device = self.device().clone(); + + let shape = layout.shape(); + let el = shape.elem_count(); + let dtype = self.dtype; + + let buffer = device.new_buffer(el, self.dtype, "powf")?; + let command_buffer = self.device.command_buffer()?; + if layout.is_contiguous() && layout.start_offset() == 0 { + let name = match self.dtype { + DType::F32 => "powf_f32", + DType::F16 => "powf_f16", + dtype => crate::bail!("Metal contiguous powf {dtype:?} not implemented"), + }; + candle_metal_kernels::call_powf( + &device.device, + &command_buffer, + &device.kernels, + name, + el, + &self.buffer, + &buffer, + pow as f32, + ) + .map_err(MetalError::from)?; + } else { + let name = match self.dtype { + DType::F32 => "powf_f32_strided", + DType::F16 => "powf_f16_strided", + dtype => crate::bail!("Metal strided powf {dtype:?} not implemented"), + }; + candle_metal_kernels::call_powf_strided( + &device.device, + &command_buffer, + &device.kernels, + name, + layout.dims(), + &self.buffer, + layout.stride(), + layout.start_offset() * dtype.size_in_bytes(), + &buffer, + pow as f32, + ) + .map_err(MetalError::from)?; + } + Ok(Self::new(buffer, device.clone(), dtype)) + } + + fn elu(&self, layout: &Layout, alpha: f64) -> Result { + let device = self.device().clone(); + + let shape = layout.shape(); + let el = shape.elem_count(); + let dtype = self.dtype; + + let buffer = device.new_buffer(el, self.dtype, "elu")?; + let command_buffer = self.device.command_buffer()?; + if layout.is_contiguous() && layout.start_offset() == 0 { + let name = match self.dtype { + DType::F32 => "elu_f32", + DType::F16 => "elu_f16", + dtype => crate::bail!("Metal contiguous elu {dtype:?} not implemented"), + }; + candle_metal_kernels::call_elu( + &device.device, + &command_buffer, + &device.kernels, + name, + el, + &self.buffer, + &buffer, + alpha as f32, + ) + .map_err(MetalError::from)?; + } else { + let name = match self.dtype { + DType::F32 => "elu_f32_strided", + DType::F16 => "elu_f16_strided", + dtype => crate::bail!("Metal strided elu {dtype:?} not implemented"), + }; + candle_metal_kernels::call_elu_strided( + &device.device, + &command_buffer, + &device.kernels, + name, + layout.dims(), + &self.buffer, + layout.stride(), + layout.start_offset() * dtype.size_in_bytes(), + &buffer, + alpha as f32, + ) + .map_err(MetalError::from)?; + } + Ok(Self::new(buffer, device.clone(), dtype)) + } + + fn reduce_op(&self, op: ReduceOp, layout: &Layout, sum_dims: &[usize]) -> Result { + let device = self.device.clone(); + let src_stride = layout.stride(); + let src_dims = layout.shape().dims(); + // Source dims and strides with the sum dims at the end. + let mut dims = vec![]; + let mut stride = vec![]; + let mut dst_el: usize = 1; + for (dim_idx, &d) in src_dims.iter().enumerate() { + if !sum_dims.contains(&dim_idx) { + dst_el *= d; + dims.push(d); + stride.push(src_stride[dim_idx]); + } + } + for &dim_idx in sum_dims.iter() { + dims.push(src_dims[dim_idx]); + stride.push(src_stride[dim_idx]); + } + + // The reduction loop requires the shared array to be properly initialized and for + // this we want the number of threads to be a power of two. + let (name, check_empty, return_index) = match (op, self.dtype) { + (ReduceOp::Sum, DType::F32) => ("fast_sum_f32_strided", false, false), + (ReduceOp::Min, DType::F32) => ("fast_min_f32_strided", true, false), + (ReduceOp::Max, DType::F32) => ("fast_max_f32_strided", true, false), + (ReduceOp::ArgMin, DType::F32) => ("fast_argmin_f32_strided", true, true), + (ReduceOp::ArgMax, DType::F32) => ("fast_argmax_f32_strided", true, true), + (ReduceOp::Sum, DType::U32) => ("fast_sum_u32_strided", false, false), + (ReduceOp::Min, DType::U32) => ("fast_min_u32_strided", true, false), + (ReduceOp::Max, DType::U32) => ("fast_max_u32_strided", true, false), + (ReduceOp::ArgMin, DType::U32) => ("fast_argmin_u32_strided", true, true), + (ReduceOp::ArgMax, DType::U32) => ("fast_argmax_u32_strided", true, true), + (ReduceOp::Sum, DType::F16) => ("fast_sum_f16_strided", false, false), + (ReduceOp::Min, DType::F16) => ("fast_min_f16_strided", true, false), + (ReduceOp::Max, DType::F16) => ("fast_max_f16_strided", true, false), + (ReduceOp::ArgMin, DType::F16) => ("fast_argmin_f16_strided", true, true), + (ReduceOp::ArgMax, DType::F16) => ("fast_argmax_f16_strided", true, true), + (ReduceOp::Sum, DType::BF16) => ("fast_sum_bf16_strided", false, false), + (ReduceOp::Min, DType::BF16) => ("fast_min_bf16_strided", true, false), + (ReduceOp::Max, DType::BF16) => ("fast_max_bf16_strided", true, false), + (ReduceOp::ArgMin, DType::BF16) => ("fast_argmin_bf16_strided", true, true), + (ReduceOp::ArgMax, DType::BF16) => ("fast_argmax_bf16_strided", true, true), + (ReduceOp::Sum, DType::I64) => ("fast_sum_i64_strided", false, false), + (ReduceOp::Min, DType::I64) => ("fast_min_i64_strided", true, false), + (ReduceOp::Max, DType::I64) => ("fast_max_i64_strided", true, false), + (ReduceOp::ArgMin, DType::I64) => ("fast_argmin_i64_strided", true, true), + (ReduceOp::ArgMax, DType::I64) => ("fast_argmax_i64_strided", true, true), + (ReduceOp::Sum, DType::U8) => ("fast_sum_u8_strided", false, false), + (ReduceOp::Min, DType::U8) => ("fast_min_u8_strided", true, false), + (ReduceOp::Max, DType::U8) => ("fast_max_u8_strided", true, false), + (ReduceOp::ArgMin, DType::U8) => ("fast_argmin_u8_strided", true, true), + (ReduceOp::ArgMax, DType::U8) => ("fast_argmax_u8_strided", true, true), + (k, dtype) => crate::bail!("Metal reduce op {k:?} {dtype:?} not implemented"), + }; + if check_empty && layout.shape().elem_count() == 0 { + Err(crate::Error::EmptyTensor { op: "reduce" }.bt())? + } + let dtype = if return_index { DType::U32 } else { self.dtype }; + let buffer = device.new_buffer(dst_el, dtype, "reduce")?; + let command_buffer = self.device.command_buffer()?; + candle_metal_kernels::call_reduce_strided( + &device.device, + &command_buffer, + &device.kernels, + name, + &dims, + &stride, + dst_el, + &self.buffer, + layout.start_offset() * self.dtype.size_in_bytes(), + &buffer, + ) + .map_err(MetalError::from)?; + + Ok(Self::new(buffer, device, dtype)) + } + + fn cmp(&self, op: CmpOp, rhs: &Self, lhs_l: &Layout, rhs_l: &Layout) -> Result { + let name = match op { + CmpOp::Eq => "eq", + CmpOp::Ne => "ne", + CmpOp::Le => "le", + CmpOp::Ge => "ge", + CmpOp::Lt => "lt", + CmpOp::Gt => "gt", + }; + self.binary(name, rhs, lhs_l, rhs_l) + } + + fn to_dtype(&self, layout: &Layout, dtype: DType) -> Result { + let device = self.device(); + let shape = layout.shape(); + let el_count = shape.elem_count(); + let buffer = device.new_buffer(el_count, dtype, "todtype")?; + let command_buffer = device.command_buffer()?; + if layout.is_contiguous() && layout.start_offset() == 0 { + let kernel_name = match (self.dtype, dtype) { + (DType::U32, DType::F32) => "cast_u32_f32", + (DType::U32, DType::U8) => "cast_u32_u8", + (DType::U32, DType::I64) => "cast_u32_i64", + (DType::U32, DType::BF16) => "cast_u32_bf16", + + (DType::U8, DType::U32) => "cast_u8_u32", + (DType::U8, DType::F32) => "cast_u8_f32", + (DType::U8, DType::I64) => "cast_u8_i64", + (DType::U8, DType::BF16) => "cast_u8_bf16", + + (DType::F32, DType::F16) => "cast_f32_f16", + (DType::F32, DType::BF16) => "cast_f32_bf16", + + (DType::I64, DType::F32) => "cast_i64_f32", + + (DType::F16, DType::BF16) => "cast_f16_bf16", + (DType::F16, DType::F32) => "cast_f16_f32", + + (DType::BF16, DType::U8) => "cast_bf16_u8", + (DType::BF16, DType::U32) => "cast_bf16_u32", + (DType::BF16, DType::F16) => "cast_bf16_f16", + (DType::BF16, DType::F32) => "cast_bf16_f32", + + (left, right) => { + crate::bail!("Metal contiguous to_dtype {left:?} {right:?} not implemented") + } + }; + candle_metal_kernels::call_cast_contiguous( + &device.device, + &command_buffer, + &device.kernels, + kernel_name, + el_count, + &self.buffer, + layout.start_offset() * self.dtype.size_in_bytes(), + &buffer, + ) + .map_err(MetalError::from)?; + } else { + let kernel_name = match (self.dtype, dtype) { + (DType::U32, DType::F32) => "cast_u32_f32_strided", + (DType::U32, DType::U8) => "cast_u32_u8_strided", + (DType::U32, DType::I64) => "cast_u32_i64_strided", + (DType::U8, DType::U32) => "cast_u8_u32_strided", + (DType::U8, DType::F32) => "cast_u8_f32_strided", + (DType::U8, DType::I64) => "cast_u8_i64_strided", + (DType::F32, DType::F16) => "cast_f32_f16_strided", + (DType::F16, DType::F32) => "cast_f16_f32_strided", + (DType::I64, DType::F32) => "cast_i64_f32_strided", + (DType::F32, DType::BF16) => "cast_f32_bf16_strided", + (DType::BF16, DType::F32) => "cast_bf16_f32_strided", + (left, right) => { + crate::bail!("Metal strided to_dtype {left:?} {right:?} not implemented") + } + }; + candle_metal_kernels::call_cast_strided( + &device.device, + &command_buffer, + &device.kernels, + kernel_name, + layout.dims(), + &self.buffer, + layout.stride(), + layout.start_offset() * self.dtype.size_in_bytes(), + &buffer, + ) + .map_err(MetalError::from)?; + } + command_buffer.set_label("to_dtype"); + Ok(Self::new(buffer, device.clone(), dtype)) + } + + fn unary_impl(&self, layout: &Layout) -> Result { + let device = self.device(); + let dtype = self.dtype; + let shape = layout.shape(); + let el_count = shape.elem_count(); + let buffer = device.new_buffer(el_count, dtype, B::KERNEL)?; + let command_buffer = device.command_buffer()?; + command_buffer.set_label(B::KERNEL); + if layout.is_contiguous() && layout.start_offset() == 0 { + use candle_metal_kernels::unary::contiguous; + + let kernel_name = match (B::KERNEL, dtype) { + ("ucos", DType::F32) => contiguous::cos::FLOAT, + ("usin", DType::F32) => contiguous::sin::FLOAT, + ("usqr", DType::F32) => contiguous::sqr::FLOAT, + ("usqrt", DType::F32) => contiguous::sqrt::FLOAT, + ("uneg", DType::F32) => contiguous::neg::FLOAT, + ("uexp", DType::F32) => contiguous::exp::FLOAT, + ("ulog", DType::F32) => contiguous::log::FLOAT, + ("ugelu", DType::F32) => contiguous::gelu::FLOAT, + ("ugelu_erf", DType::F32) => contiguous::gelu_erf::FLOAT, + ("uerf", DType::F32) => contiguous::erf::FLOAT, + ("usilu", DType::F32) => contiguous::silu::FLOAT, + ("uabs", DType::F32) => contiguous::abs::FLOAT, + ("uceil", DType::F32) => contiguous::ceil::FLOAT, + ("ufloor", DType::F32) => contiguous::floor::FLOAT, + ("uround", DType::F32) => contiguous::round::FLOAT, + ("urecip", DType::F32) => contiguous::recip::FLOAT, + ("utanh", DType::F32) => contiguous::tanh::FLOAT, + ("urelu", DType::F32) => contiguous::relu::FLOAT, + ("ucos", DType::F16) => contiguous::cos::HALF, + ("usin", DType::F16) => contiguous::sin::HALF, + ("usqr", DType::F16) => contiguous::sqr::HALF, + ("usqrt", DType::F16) => contiguous::sqrt::HALF, + ("uneg", DType::F16) => contiguous::neg::HALF, + ("uexp", DType::F16) => contiguous::exp::HALF, + ("ulog", DType::F16) => contiguous::log::HALF, + ("ugelu", DType::F16) => contiguous::gelu::HALF, + ("ugelu_erf", DType::F16) => contiguous::gelu_erf::HALF, + ("uerf", DType::F16) => contiguous::erf::HALF, + ("usilu", DType::F16) => contiguous::silu::HALF, + ("uabs", DType::F16) => contiguous::abs::HALF, + ("uceil", DType::F16) => contiguous::ceil::HALF, + ("ufloor", DType::F16) => contiguous::floor::HALF, + ("uround", DType::F16) => contiguous::round::HALF, + ("urecip", DType::F16) => contiguous::recip::HALF, + ("utanh", DType::F16) => contiguous::tanh::HALF, + ("urelu", DType::F16) => contiguous::relu::HALF, + (name, dtype) => { + crate::bail!("Metal contiguous unary {name} {dtype:?} not implemented") + } + }; + candle_metal_kernels::call_unary_contiguous( + &device.device, + &command_buffer, + &device.kernels, + kernel_name, + el_count, + &self.buffer, + &buffer, + ) + .map_err(MetalError::from)?; + } else { + use candle_metal_kernels::unary::strided; + let kernel_name = match (B::KERNEL, dtype) { + ("ucos", DType::F32) => strided::cos::FLOAT, + ("usin", DType::F32) => strided::sin::FLOAT, + ("usqr", DType::F32) => strided::sqr::FLOAT, + ("usqrt", DType::F32) => strided::sqrt::FLOAT, + ("uneg", DType::F32) => strided::neg::FLOAT, + ("uexp", DType::F32) => strided::exp::FLOAT, + ("ulog", DType::F32) => strided::log::FLOAT, + ("ugelu", DType::F32) => strided::gelu::FLOAT, + ("ugelu_erf", DType::F32) => strided::gelu_erf::FLOAT, + ("uerf", DType::F32) => strided::erf::FLOAT, + ("usilu", DType::F32) => strided::silu::FLOAT, + ("uabs", DType::F32) => strided::abs::FLOAT, + ("uceil", DType::F32) => strided::ceil::FLOAT, + ("ufloor", DType::F32) => strided::floor::FLOAT, + ("urelu", DType::F32) => strided::relu::FLOAT, + ("uround", DType::F32) => strided::round::FLOAT, + ("ucos", DType::F16) => strided::cos::HALF, + ("usin", DType::F16) => strided::sin::HALF, + ("usqr", DType::F16) => strided::sqr::HALF, + ("usqrt", DType::F16) => strided::sqrt::HALF, + ("uneg", DType::F16) => strided::neg::HALF, + ("uexp", DType::F16) => strided::exp::HALF, + ("ulog", DType::F16) => strided::log::HALF, + ("ugelu", DType::F16) => strided::gelu::HALF, + ("ugelu_erf", DType::F16) => strided::gelu_erf::HALF, + ("uerf", DType::F16) => strided::erf::HALF, + ("usilu", DType::F16) => strided::silu::HALF, + ("uabs", DType::F16) => strided::abs::HALF, + ("uceil", DType::F16) => strided::ceil::HALF, + ("ufloor", DType::F16) => strided::floor::HALF, + ("urelu", DType::F16) => strided::relu::HALF, + ("uround", DType::F16) => strided::round::HALF, + (name, dtype) => { + crate::bail!("Metal strided unary {name} {dtype:?} not implemented") + } + }; + candle_metal_kernels::call_unary_strided( + &device.device, + &command_buffer, + &device.kernels, + kernel_name, + layout.dims(), + &self.buffer, + layout.stride(), + layout.start_offset() * self.dtype.size_in_bytes(), + &buffer, + 0, + ) + .map_err(MetalError::from)?; + } + Ok(Self::new(buffer, device.clone(), dtype)) + } + + fn binary_impl( + &self, + rhs: &Self, + lhs_l: &Layout, + rhs_l: &Layout, + ) -> Result { + self.binary(B::KERNEL, rhs, lhs_l, rhs_l) + } + + fn where_cond( + &self, + layout: &Layout, + t: &Self, + t_l: &Layout, + f: &Self, + f_l: &Layout, + ) -> Result { + let device = self.device.clone(); + let shape = t_l.shape(); + let dims = shape.dims(); + let el = shape.elem_count(); + let dtype = t.dtype; + let buffer = self.device.new_buffer(el, dtype, "where")?; + let command_buffer = self.device.command_buffer()?; + if t.dtype() != f.dtype() { + crate::bail!( + "Invalid where: different dtypes for values {:?} != {:?}", + t.dtype(), + f.dtype() + ); + } + let name = match (self.dtype, t.dtype()) { + (DType::U8, DType::F32) => "where_u8_f32", + (DType::U8, DType::BF16) => "where_u8_bf16", + (DType::U8, DType::F16) => "where_u8_f16", + (DType::U8, DType::I64) => "where_u8_i64", + (DType::U8, DType::U32) => "where_u8_u32", + (DType::U8, DType::U8) => "where_u8_u8", + (left, right) => crate::bail!("Metal where_cond {left:?} {right:?} not implemented"), + }; + candle_metal_kernels::call_where_cond_strided( + &device.device, + &command_buffer, + &device.kernels, + name, + dims, + &self.buffer, + ( + layout.stride(), + layout.start_offset() * self.dtype.size_in_bytes(), + ), + &t.buffer, + (t_l.stride(), t_l.start_offset() * t.dtype.size_in_bytes()), + &f.buffer, + (f_l.stride(), f_l.start_offset() * f.dtype.size_in_bytes()), + &buffer, + ) + .map_err(MetalError::from)?; + Ok(Self::new(buffer, device, dtype)) + } + + fn conv1d( + &self, + layout: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &ParamsConv1D, + ) -> Result { + let device = self.device().clone(); + let shape = layout.shape(); + let dims = shape.dims(); + let strides = layout.stride(); + + let stride = params.stride; + let dilation = params.dilation; + let padding = params.padding; + let k_size = params.k_size; + let l_out = (dims[2] + 2 * padding - dilation * (k_size - 1) - 1) / stride + 1; + let dst_el = dims[0] * l_out * dims[1] * k_size; + let dst = self + .device + .new_buffer(dst_el, self.dtype, "conv1d_im2col")?; + let command_buffer = self.device.command_buffer()?; + let name = match self.dtype { + DType::F32 => "im2col1d_f32", + dtype => crate::bail!("Metal conv1d {dtype:?} not implemented"), + }; + candle_metal_kernels::call_im2col1d_strided( + &self.device.device, + &command_buffer, + &self.device.kernels, + name, + layout.shape().dims(), + strides, + (k_size, stride, padding, dilation), + &self.buffer, + layout.start_offset() * self.dtype.size_in_bytes(), + &dst, + ) + .map_err(MetalError::from)?; + let col = Self { + buffer: dst, + device, + dtype: self.dtype, + }; + let l_out = params.l_out(); + let b = params.b_size; + let n = params.c_out; + let k = params.k_size * params.c_in; + let m = l_out; + let col_l = Layout::contiguous((b, m, k)); + let res = if kernel_l.is_contiguous() { + let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) + .transpose(1, 2)? + .broadcast_as((b, k, n))?; + col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? + } else { + // Make the kernel contiguous if not already the case. + let mut kernel_c = self.device().zeros_impl(kernel_l.shape(), kernel.dtype())?; + kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?; + let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) + .transpose(1, 2)? + .broadcast_as((b, k, n))?; + col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? + }; + let res_l = Layout::contiguous((b, l_out, n)).transpose(1, 2)?; + let mut res_t = self.device().zeros_impl(res_l.shape(), res.dtype())?; + res.copy_strided_src(&mut res_t, 0, &res_l)?; + Ok(res_t) + } + + fn conv_transpose1d( + &self, + _l: &Layout, + _kernel: &Self, + _kernel_l: &Layout, + _params: &ParamsConvTranspose1D, + ) -> Result { + crate::bail!("Metal conv_transpose1d not implemented") + } + + fn conv2d( + &self, + layout: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &ParamsConv2D, + ) -> Result { + let device = self.device().clone(); + let shape = layout.shape(); + let dims = shape.dims(); + + let stride = params.stride; + let dilation = params.dilation; + let padding = params.padding; + let h_k = params.k_h; + let w_k = params.k_w; + let h = dims[2]; + let w = dims[3]; + let h_out = (h + 2 * padding - dilation * (h_k - 1) - 1) / stride + 1; + let w_out = (w + 2 * padding - dilation * (w_k - 1) - 1) / stride + 1; + let dst_el = dims[0] * h_out * w_out * dims[1] * h_k * w_k; + + let dst = self + .device + .new_buffer(dst_el, self.dtype, "conv2d_im2col")?; + let command_buffer = self.device.command_buffer()?; + let name = match self.dtype { + DType::F32 => "im2col_f32", + dtype => crate::bail!("Metal conv2d {dtype:?} not implemented"), + }; + candle_metal_kernels::call_im2col_strided( + &self.device.device, + &command_buffer, + &self.device.kernels, + name, + layout.shape().dims(), + layout.stride(), + (h_k, w_k, stride, padding, dilation), + &self.buffer, + layout.start_offset() * self.dtype.size_in_bytes(), + &dst, + ) + .map_err(MetalError::from)?; + let col = Self { + buffer: dst, + device, + dtype: self.dtype, + }; + let h_out = params.out_h(); + let w_out = params.out_w(); + let b = params.b_size; + let n = params.c_out; + let k = params.k_h * params.k_w * params.c_in; + let m = h_out * w_out; + let col_l = Layout::contiguous((b, m, k)); + let res = if kernel_l.is_contiguous() { + let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) + .transpose(1, 2)? + .broadcast_as((b, k, n))?; + col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? + } else { + // Make the kernel contiguous if not already the case. + let mut kernel_c = self.device().zeros_impl(kernel_l.shape(), kernel.dtype())?; + kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?; + let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) + .transpose(1, 2)? + .broadcast_as((b, k, n))?; + col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? + }; + let res_l = Layout::contiguous((b, h_out, w_out, n)) + .transpose(1, 2)? + .transpose(1, 3)?; + let mut res_t = self.device().zeros_impl(res_l.shape(), res.dtype())?; + res.copy_strided_src(&mut res_t, 0, &res_l)?; + Ok(res_t) + } + + fn conv_transpose2d( + &self, + _l: &Layout, + _kernel: &Self, + _kernel_l: &Layout, + _params: &ParamsConvTranspose2D, + ) -> Result { + crate::bail!("Metal conv_tranpose2d not implemented") + } + + fn avg_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result { + crate::bail!("Metal avg_pool2d not implemented") + } + + fn max_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result { + crate::bail!("Metal max_pool2d not implemented") + } + + fn upsample_nearest1d(&self, _: &Layout, _: usize) -> Result { + crate::bail!("Metal upsample_nearest1d not implemented") + } + + fn upsample_nearest2d(&self, inp_l: &Layout, out_w: usize, out_h: usize) -> Result { + // let inp = &inp.slice(inp_l.start_offset()..); + let shape = inp_l.shape(); + let dims = shape.dims(); + let strides = inp_l.stride(); + if dims.len() != 4 { + crate::bail!("unexpected input shape for upsample {dims:?}") + } + let name = match self.dtype { + DType::F32 => "upsample_nearest2d_f32", + dtype => crate::bail!("Metal upsample_nearest2d {dtype:?} not implemented"), + }; + + let dst_el = out_w * out_h * dims[0] * dims[1]; + let buffer = self + .device + .new_buffer(dst_el, self.dtype, "upsample_nearest2d")?; + let command_buffer = self.device.command_buffer()?; + candle_metal_kernels::call_upsample_nearest_2d( + &self.device.device, + &command_buffer, + &self.device.kernels, + name, + dims, + strides, + out_w, + out_h, + &self.buffer, + inp_l.start_offset() * self.dtype.size_in_bytes(), + &buffer, + ) + .map_err(MetalError::from)?; + Ok(Self::new(buffer, self.device.clone(), self.dtype)) + } + + fn gather(&self, src_l: &Layout, ids: &Self, ids_l: &Layout, dim: usize) -> Result { + let (ids_o1, _) = match ids_l.contiguous_offsets() { + Some(o12) => o12, + None => Err(crate::Error::RequiresContiguous { op: "gather" }.bt())?, + }; + let ids_el = ids_l.dims()[dim]; + let dst_el = ids_l.shape().elem_count(); + let dtype = self.dtype; + let device = self.device(); + let buffer = device.new_buffer(dst_el, dtype, "index_select")?; + let name = match (ids.dtype, self.dtype) { + (DType::U32, DType::F32) => "gather_u32_f32", + (DType::U32, DType::F16) => "gather_u32_f16", + (left, right) => crate::bail!("Metal gather {left:?} {right:?} not implemented"), + }; + let command_buffer = self.device.command_buffer()?; + candle_metal_kernels::call_gather( + &device.device, + &command_buffer, + &self.device.kernels, + name, + src_l.dims(), + ids_el, + dim, + &self.buffer, + src_l.start_offset() * dtype.size_in_bytes(), + &ids.buffer, + ids_o1 * ids.dtype.size_in_bytes(), + &buffer, + ) + .map_err(MetalError::from)?; + Ok(Self::new(buffer, device.clone(), dtype)) + } + + fn scatter_add( + &self, + l: &Layout, + ids: &Self, + ids_l: &Layout, + src: &Self, + src_l: &Layout, + dim: usize, + ) -> Result { + let mut acc = self.device.zeros_impl(l.shape(), self.dtype())?; + self.copy_strided_src(&mut acc, 0, l)?; + let (ids_offset, _) = match ids_l.contiguous_offsets() { + Some(o12) => o12, + None => Err(crate::Error::RequiresContiguous { op: "scatter-add" }.bt())?, + }; + let src_offset = match src_l.contiguous_offsets() { + Some((o1, _)) => o1, + None => Err(crate::Error::RequiresContiguous { op: "scatter-add" }.bt())?, + }; + let name = match (ids.dtype, self.dtype) { + (DType::U32, DType::F32) => "sa_u32_f32", + _ => Err(MetalError::UnexpectedDType { + msg: "scatter-add ids should be u8/u32/i64", + expected: DType::U32, + got: ids.dtype(), + })?, + }; + let command_buffer = self.device.command_buffer()?; + candle_metal_kernels::call_scatter_add( + &self.device.device, + &command_buffer, + &self.device.kernels, + name, + src_l.dims(), + l.dims(), + dim, + &src.buffer, + src_offset * src.dtype.size_in_bytes(), + &ids.buffer, + ids_offset * ids.dtype.size_in_bytes(), + &acc.buffer, + ) + .map_err(MetalError::from)?; + Ok(acc) + } + + fn index_select(&self, ids: &Self, src_l: &Layout, ids_l: &Layout, dim: usize) -> Result { + if !(src_l.is_contiguous() + && src_l.start_offset() == 0 + && ids_l.is_contiguous() + && ids_l.start_offset() == 0) + { + crate::bail!("Metal strided index_select not implemented"); + } + let left_size: usize = src_l.dims()[..dim].iter().product(); + let right_size: usize = src_l.dims()[dim + 1..].iter().product(); + let ids_el = ids_l.shape().elem_count(); + let dst_el = ids_el * left_size * right_size; + let dtype = self.dtype; + let device = self.device(); + let buffer = device.new_buffer(dst_el, dtype, "index_select")?; + let name = match (ids.dtype, self.dtype) { + (DType::U8, DType::BF16) => "is_u8_bf16", + + (DType::U32, DType::F32) => "is_u32_f32", + (DType::U32, DType::F16) => "is_u32_f16", + (DType::U32, DType::BF16) => "is_u32_bf16", + + (left, right) => { + crate::bail!("Metal contiguous index_select {left:?} {right:?} not implemented") + } + }; + let command_buffer = self.device.command_buffer()?; + candle_metal_kernels::call_index_select( + &device.device, + &command_buffer, + &self.device.kernels, + name, + src_l.dims(), + ids_el, + dim, + &self.buffer, + &ids.buffer, + &buffer, + ) + .map_err(MetalError::from)?; + Ok(Self::new(buffer, device.clone(), dtype)) + } + + fn index_add( + &self, + l: &Layout, + ids: &Self, + ids_l: &Layout, + src: &Self, + src_l: &Layout, + dim: usize, + ) -> Result { + let mut acc = self.device.zeros_impl(l.shape(), self.dtype())?; + self.copy_strided_src(&mut acc, 0, l)?; + let (ids_offset, _) = match ids_l.contiguous_offsets() { + Some(o12) => o12, + None => Err(crate::Error::RequiresContiguous { op: "index-add" }.bt())?, + }; + let src_offset = match src_l.contiguous_offsets() { + Some((o1, _)) => o1, + None => Err(crate::Error::RequiresContiguous { op: "index-add" }.bt())?, + }; + let name = match (ids.dtype, self.dtype) { + (DType::U32, DType::F32) => "ia_u32_f32", + _ => Err(MetalError::UnexpectedDType { + msg: "index-add ids should be u32", + expected: DType::U32, + got: ids.dtype(), + })?, + }; + let command_buffer = self.device.command_buffer()?; + candle_metal_kernels::call_index_add( + &self.device.device, + &command_buffer, + &self.device.kernels, + name, + src_l.dims(), + l.dims(), + ids_l.dims(), + dim, + &src.buffer, + src_offset * src.dtype.size_in_bytes(), + &ids.buffer, + ids_offset * ids.dtype.size_in_bytes(), + &acc.buffer, + ) + .map_err(MetalError::from)?; + Ok(acc) + } + fn matmul( + &self, + rhs: &Self, + (b, m, n, k): (usize, usize, usize, usize), + lhs_l: &Layout, + rhs_l: &Layout, + ) -> Result { + let buffer = self.device.new_buffer(b * m * n, self.dtype, "matmul")?; + let name = match self.dtype { + DType::F32 => "sgemm", + DType::F16 => "hgemm", + dtype => { + return Err(MetalError::Message(format!("matmul doesn't support {dtype:?}")).into()) + } + }; + + let command_buffer = self.device.command_buffer()?; + command_buffer.set_label("matmul"); + candle_metal_kernels::call_gemm( + &self.device.device, + &command_buffer, + &self.device.kernels, + name, + (b, m, n, k), + lhs_l.stride(), + lhs_l.start_offset() * self.dtype.size_in_bytes(), + &self.buffer, + rhs_l.stride(), + rhs_l.start_offset() * rhs.dtype.size_in_bytes(), + &rhs.buffer, + &buffer, + ) + .map_err(MetalError::from)?; + Ok(Self::new(buffer, self.device.clone(), self.dtype())) + } + + fn copy_strided_src(&self, dst: &mut Self, dst_offset: usize, src_l: &Layout) -> Result<()> { + let command_buffer = self.device.command_buffer()?; + if src_l.is_contiguous() && self.dtype == dst.dtype() { + command_buffer.set_label("copy_contiguous"); + let blit = command_buffer.new_blit_command_encoder(); + blit.set_label("copy_contiguous"); + let src_offset = (src_l.start_offset() * self.dtype.size_in_bytes()) as NSUInteger; + let length = (src_l.shape().elem_count() * self.dtype.size_in_bytes()) as NSUInteger; + let dst_offset = (dst_offset * dst.dtype().size_in_bytes()) as NSUInteger; + blit.copy_from_buffer(&self.buffer, src_offset, dst.buffer(), dst_offset, length); + blit.end_encoding(); + } else { + let src_shape = src_l.shape(); + let el_count = src_shape.elem_count(); + if el_count == 0 { + return Ok(()); + } + let kernel_name = match self.dtype { + DType::F32 => candle_metal_kernels::unary::strided::copy::FLOAT, + DType::F16 => candle_metal_kernels::unary::strided::copy::HALF, + DType::BF16 => candle_metal_kernels::unary::strided::copy::BFLOAT, + DType::I64 => candle_metal_kernels::unary::strided::copy::I64, + DType::U32 => candle_metal_kernels::unary::strided::copy::U32, + DType::U8 => candle_metal_kernels::unary::strided::copy::U8, + dtype => crate::bail!("Metal copy_strided {dtype:?} not implemented"), + }; + candle_metal_kernels::call_unary_strided( + &self.device.device, + &command_buffer, + &self.device.kernels, + kernel_name, + src_l.dims(), + &self.buffer, + src_l.stride(), + src_l.start_offset() * self.dtype.size_in_bytes(), + &dst.buffer, + dst_offset * dst.dtype.size_in_bytes(), + ) + .map_err(MetalError::from)?; + command_buffer.set_label("copy_strided"); + } + Ok(()) + } +} + +impl MetalStorage { + pub fn new(buffer: Arc, device: MetalDevice, dtype: DType) -> Self { + Self { + buffer, + device, + dtype, + } + } + + pub fn buffer(&self) -> &Buffer { + &self.buffer + } + + pub fn binary( + &self, + op: &'static str, + rhs: &Self, + lhs_l: &Layout, + rhs_l: &Layout, + ) -> Result { + let device = self.device(); + let shape = lhs_l.shape(); + let el_count = shape.elem_count(); + let command_buffer = device.command_buffer()?; + let (buffer, dtype) = if (lhs_l.is_contiguous() && lhs_l.start_offset() == 0) + && (rhs_l.is_contiguous() && rhs_l.start_offset() == 0) + && &op[..1] != "b" + { + use candle_metal_kernels::binary::contiguous; + + let (kernel_name, dtype) = match (op, self.dtype) { + ("add", DType::F32) => (contiguous::add::FLOAT, self.dtype), + ("sub", DType::F32) => (contiguous::sub::FLOAT, self.dtype), + ("mul", DType::F32) => (contiguous::mul::FLOAT, self.dtype), + ("div", DType::F32) => (contiguous::div::FLOAT, self.dtype), + ("eq", DType::F32) => (contiguous::eq::FLOAT, DType::U8), + ("ne", DType::F32) => (contiguous::ne::FLOAT, DType::U8), + ("le", DType::F32) => (contiguous::le::FLOAT, DType::U8), + ("lt", DType::F32) => (contiguous::lt::FLOAT, DType::U8), + ("ge", DType::F32) => (contiguous::ge::FLOAT, DType::U8), + ("gt", DType::F32) => (contiguous::gt::FLOAT, DType::U8), + + ("add", DType::F16) => (contiguous::add::HALF, self.dtype), + ("sub", DType::F16) => (contiguous::sub::HALF, self.dtype), + ("mul", DType::F16) => (contiguous::mul::HALF, self.dtype), + ("div", DType::F16) => (contiguous::div::HALF, self.dtype), + ("eq", DType::F16) => (contiguous::eq::HALF, DType::U8), + ("ne", DType::F16) => (contiguous::ne::HALF, DType::U8), + ("le", DType::F16) => (contiguous::le::HALF, DType::U8), + ("lt", DType::F16) => (contiguous::lt::HALF, DType::U8), + ("ge", DType::F16) => (contiguous::ge::HALF, DType::U8), + ("gt", DType::F16) => (contiguous::gt::HALF, DType::U8), + + ("add", DType::BF16) => (contiguous::add::BFLOAT, self.dtype), + ("sub", DType::BF16) => (contiguous::sub::BFLOAT, self.dtype), + ("mul", DType::BF16) => (contiguous::mul::BFLOAT, self.dtype), + ("div", DType::BF16) => (contiguous::div::BFLOAT, self.dtype), + ("eq", DType::BF16) => (contiguous::eq::BFLOAT, DType::U8), + ("ne", DType::BF16) => (contiguous::ne::BFLOAT, DType::U8), + ("le", DType::BF16) => (contiguous::le::BFLOAT, DType::U8), + ("lt", DType::BF16) => (contiguous::lt::BFLOAT, DType::U8), + ("ge", DType::BF16) => (contiguous::ge::BFLOAT, DType::U8), + ("gt", DType::BF16) => (contiguous::gt::BFLOAT, DType::U8), + + ("add", DType::I64) => (contiguous::add::I64, self.dtype), + ("sub", DType::I64) => (contiguous::sub::I64, self.dtype), + ("mul", DType::I64) => (contiguous::mul::I64, self.dtype), + ("div", DType::I64) => (contiguous::div::I64, self.dtype), + ("eq", DType::I64) => (contiguous::eq::I64, DType::U8), + ("ne", DType::I64) => (contiguous::ne::I64, DType::U8), + ("le", DType::I64) => (contiguous::le::I64, DType::U8), + ("lt", DType::I64) => (contiguous::lt::I64, DType::U8), + ("ge", DType::I64) => (contiguous::ge::I64, DType::U8), + ("gt", DType::I64) => (contiguous::gt::I64, DType::U8), + + ("add", DType::U32) => (contiguous::add::U32, self.dtype), + ("sub", DType::U32) => (contiguous::sub::U32, self.dtype), + ("mul", DType::U32) => (contiguous::mul::U32, self.dtype), + ("div", DType::U32) => (contiguous::div::U32, self.dtype), + ("eq", DType::U32) => (contiguous::eq::U32, DType::U8), + ("ne", DType::U32) => (contiguous::ne::U32, DType::U8), + ("le", DType::U32) => (contiguous::le::U32, DType::U8), + ("lt", DType::U32) => (contiguous::lt::U32, DType::U8), + ("ge", DType::U32) => (contiguous::ge::U32, DType::U8), + ("gt", DType::U32) => (contiguous::gt::U32, DType::U8), + + ("add", DType::U8) => (contiguous::add::U8, self.dtype), + ("sub", DType::U8) => (contiguous::sub::U8, self.dtype), + ("mul", DType::U8) => (contiguous::mul::U8, self.dtype), + ("div", DType::U8) => (contiguous::div::U8, self.dtype), + ("eq", DType::U8) => (contiguous::eq::U8, DType::U8), + ("ne", DType::U8) => (contiguous::ne::U8, DType::U8), + ("le", DType::U8) => (contiguous::le::U8, DType::U8), + ("lt", DType::U8) => (contiguous::lt::U8, DType::U8), + ("ge", DType::U8) => (contiguous::ge::U8, DType::U8), + ("gt", DType::U8) => (contiguous::gt::U8, DType::U8), + + (name, dtype) => { + crate::bail!("Metal contiguous binary {name} {dtype:?} not implemented") + } + }; + let buffer = device.new_buffer(el_count, dtype, op)?; + candle_metal_kernels::call_binary_contiguous( + &device.device, + &command_buffer, + &device.kernels, + kernel_name, + el_count, + &self.buffer, + &rhs.buffer, + &buffer, + ) + .map_err(MetalError::from)?; + (buffer, dtype) + } else { + use candle_metal_kernels::binary::strided; + + let (kernel_name, dtype) = match (op, self.dtype) { + ("badd", DType::F32) => (strided::add::FLOAT, self.dtype), + ("bsub", DType::F32) => (strided::sub::FLOAT, self.dtype), + ("bmul", DType::F32) => (strided::mul::FLOAT, self.dtype), + ("bdiv", DType::F32) => (strided::div::FLOAT, self.dtype), + ("bminimum", DType::F32) => (strided::min::FLOAT, self.dtype), + ("bmaximum", DType::F32) => (strided::max::FLOAT, self.dtype), + ("eq", DType::F32) => (strided::eq::FLOAT, DType::U8), + ("ne", DType::F32) => (strided::ne::FLOAT, DType::U8), + ("le", DType::F32) => (strided::le::FLOAT, DType::U8), + ("lt", DType::F32) => (strided::lt::FLOAT, DType::U8), + ("ge", DType::F32) => (strided::ge::FLOAT, DType::U8), + ("gt", DType::F32) => (strided::gt::FLOAT, DType::U8), + + ("badd", DType::F16) => (strided::add::HALF, self.dtype), + ("bsub", DType::F16) => (strided::sub::HALF, self.dtype), + ("bmul", DType::F16) => (strided::mul::HALF, self.dtype), + ("bdiv", DType::F16) => (strided::div::HALF, self.dtype), + ("bminimum", DType::F16) => (strided::min::HALF, self.dtype), + ("bmaximum", DType::F16) => (strided::max::HALF, self.dtype), + ("eq", DType::F16) => (strided::eq::HALF, DType::U8), + ("ne", DType::F16) => (strided::ne::HALF, DType::U8), + ("le", DType::F16) => (strided::le::HALF, DType::U8), + ("lt", DType::F16) => (strided::lt::HALF, DType::U8), + ("ge", DType::F16) => (strided::ge::HALF, DType::U8), + ("gt", DType::F16) => (strided::gt::HALF, DType::U8), + + ("badd", DType::BF16) => (strided::add::BFLOAT, self.dtype), + ("bsub", DType::BF16) => (strided::sub::BFLOAT, self.dtype), + ("bmul", DType::BF16) => (strided::mul::BFLOAT, self.dtype), + ("bdiv", DType::BF16) => (strided::div::BFLOAT, self.dtype), + ("bminimum", DType::BF16) => (strided::min::BFLOAT, self.dtype), + ("bmaximum", DType::BF16) => (strided::max::BFLOAT, self.dtype), + ("eq", DType::BF16) => (strided::eq::BFLOAT, DType::U8), + ("ne", DType::BF16) => (strided::ne::BFLOAT, DType::U8), + ("le", DType::BF16) => (strided::le::BFLOAT, DType::U8), + ("lt", DType::BF16) => (strided::lt::BFLOAT, DType::U8), + ("ge", DType::BF16) => (strided::ge::BFLOAT, DType::U8), + ("gt", DType::BF16) => (strided::gt::BFLOAT, DType::U8), + + ("badd", DType::I64) => (strided::add::I64, self.dtype), + ("bsub", DType::I64) => (strided::sub::I64, self.dtype), + ("bmul", DType::I64) => (strided::mul::I64, self.dtype), + ("bdiv", DType::I64) => (strided::div::I64, self.dtype), + ("bminimum", DType::I64) => (strided::min::I64, self.dtype), + ("bmaximum", DType::I64) => (strided::max::I64, self.dtype), + ("eq", DType::I64) => (strided::eq::I64, DType::U8), + ("ne", DType::I64) => (strided::ne::I64, DType::U8), + ("le", DType::I64) => (strided::le::I64, DType::U8), + ("lt", DType::I64) => (strided::lt::I64, DType::U8), + ("ge", DType::I64) => (strided::ge::I64, DType::U8), + ("gt", DType::I64) => (strided::gt::I64, DType::U8), + + ("badd", DType::U32) => (strided::add::U32, self.dtype), + ("bsub", DType::U32) => (strided::sub::U32, self.dtype), + ("bmul", DType::U32) => (strided::mul::U32, self.dtype), + ("bdiv", DType::U32) => (strided::div::U32, self.dtype), + ("bminimum", DType::U32) => (strided::min::U32, self.dtype), + ("bmaximum", DType::U32) => (strided::max::U32, self.dtype), + ("eq", DType::U32) => (strided::eq::U32, DType::U8), + ("ne", DType::U32) => (strided::ne::U32, DType::U8), + ("le", DType::U32) => (strided::le::U32, DType::U8), + ("lt", DType::U32) => (strided::lt::U32, DType::U8), + ("ge", DType::U32) => (strided::ge::U32, DType::U8), + ("gt", DType::U32) => (strided::gt::U32, DType::U8), + + ("badd", DType::U8) => (strided::add::U8, self.dtype), + ("bsub", DType::U8) => (strided::sub::U8, self.dtype), + ("bmul", DType::U8) => (strided::mul::U8, self.dtype), + ("bdiv", DType::U8) => (strided::div::U8, self.dtype), + ("bminimum", DType::U8) => (strided::min::U8, self.dtype), + ("bmaximum", DType::U8) => (strided::max::U8, self.dtype), + ("eq", DType::U8) => (strided::eq::U8, DType::U8), + ("ne", DType::U8) => (strided::ne::U8, DType::U8), + ("le", DType::U8) => (strided::le::U8, DType::U8), + ("lt", DType::U8) => (strided::lt::U8, DType::U8), + ("ge", DType::U8) => (strided::ge::U8, DType::U8), + ("gt", DType::U8) => (strided::gt::U8, DType::U8), + + (name, dtype) => { + crate::bail!("Metal strided binary {name} {dtype:?} not implemented") + } + }; + let buffer = device.new_buffer(el_count, dtype, op)?; + candle_metal_kernels::call_binary_strided( + &device.device, + &command_buffer, + &device.kernels, + kernel_name, + lhs_l.dims(), + &self.buffer, + lhs_l.stride(), + lhs_l.start_offset() * self.dtype.size_in_bytes(), + &rhs.buffer, + rhs_l.stride(), + rhs_l.start_offset() * rhs.dtype.size_in_bytes(), + &buffer, + ) + .map_err(MetalError::from)?; + (buffer, dtype) + }; + command_buffer.set_label("binary"); + Ok(Self::new(buffer, device.clone(), dtype)) + } + + pub(crate) fn to_cpu(&self) -> Result> { + let length = self.buffer.length() as usize; + let size = self.dtype.size_in_bytes(); + if length % size != 0 { + crate::bail!( + "The Metal buffer length is not aligned with dtype {:?}", + self.dtype + ); + } + let buffer = self.device.new_buffer_managed(self.buffer.length())?; + { + let command_buffer = self.device.command_buffer()?; + command_buffer.set_label("to_cpu"); + let blit = command_buffer.new_blit_command_encoder(); + blit.set_label("blit_to_cpu"); + blit.copy_from_buffer(&self.buffer, 0, &buffer, 0, self.buffer.length()); + blit.end_encoding(); + } + self.device.wait_until_completed()?; + Ok(read_to_vec(&buffer, length / size)) + } +} + +impl BackendDevice for MetalDevice { + type Storage = MetalStorage; + + fn new(ordinal: usize) -> Result { + let device = metal::Device::all().swap_remove(ordinal); + let command_queue = device.new_command_queue(); + let command_buffer = command_queue.new_command_buffer().to_owned(); + command_buffer.enqueue(); + let command_buffer = Arc::new(RwLock::new(command_buffer)); + let command_buffer_index = Arc::new(RwLock::new(0)); + let kernels = Arc::new(Kernels::new()); + let buffers = Arc::new(RwLock::new(HashMap::new())); + let compute_per_buffer = match std::env::var("CANDLE_METAL_COMPUTE_PER_BUFFER") { + Ok(val) => val.parse()?, + _ => 10, + }; + let seed = Arc::new(Mutex::new(device.new_buffer_with_data( + [299792458].as_ptr() as *const c_void, + 4, + MTLResourceOptions::StorageModeManaged, + ))); + Ok(Self { + device, + command_queue, + command_buffer, + command_buffer_index, + compute_per_buffer, + buffers, + kernels, + seed, + }) + } + + fn location(&self) -> crate::DeviceLocation { + crate::DeviceLocation::Metal { + gpu_id: self.registry_id() as usize, + } + } + + fn same_device(&self, rhs: &Self) -> bool { + self.device.registry_id() == rhs.device.registry_id() + } + + fn zeros_impl(&self, shape: &Shape, dtype: DType) -> Result { + let size = shape.elem_count() * dtype.size_in_bytes(); + let buffer = self.allocate_zeros(size)?; + Ok(MetalStorage::new(buffer, self.clone(), dtype)) + } + + fn ones_impl(&self, shape: &Shape, dtype: DType) -> Result { + // TODO Is there a faster way ? + let cpu_storage = crate::cpu_backend::CpuDevice.ones_impl(shape, dtype)?; + self.storage_from_cpu_storage(&cpu_storage) + } + + fn storage_from_cpu_storage(&self, storage: &CpuStorage) -> Result { + let buffer = match storage { + CpuStorage::U8(storage) => self.new_buffer_with_data(storage), + CpuStorage::U32(storage) => self.new_buffer_with_data(storage), + CpuStorage::I64(storage) => self.new_buffer_with_data(storage), + CpuStorage::BF16(storage) => self.new_buffer_with_data(storage), + CpuStorage::F16(storage) => self.new_buffer_with_data(storage), + CpuStorage::F32(storage) => self.new_buffer_with_data(storage), + CpuStorage::F64(storage) => self.new_buffer_with_data(storage), + }?; + Ok(Self::Storage::new(buffer, self.clone(), storage.dtype())) + } + + fn rand_uniform( + &self, + shape: &Shape, + dtype: DType, + min: f64, + max: f64, + ) -> Result { + let name = match dtype { + DType::F32 => "rand_uniform_f32", + DType::F16 => "rand_uniform_f16", + DType::BF16 => "rand_uniform_bf16", + dtype => crate::bail!("rand_uniform not implemented for {dtype:?}"), + }; + let buffer = self.new_buffer(shape.elem_count(), dtype, "rand_uniform")?; + let command_buffer = self.command_buffer()?; + candle_metal_kernels::call_random_uniform( + &self.device, + &command_buffer, + &self.kernels, + name, + min as f32, + max as f32, + shape.elem_count(), + &self.seed.lock().unwrap(), + &buffer, + ) + .map_err(MetalError::from)?; + + Ok(Self::Storage::new(buffer, self.clone(), dtype)) + } + + fn rand_normal( + &self, + shape: &Shape, + dtype: DType, + mean: f64, + stddev: f64, + ) -> Result { + let name = match dtype { + DType::F32 => "rand_normal_f32", + DType::F16 => "rand_normal_f16", + DType::BF16 => "rand_normal_bf16", + dtype => crate::bail!("rand_uniform not implemented for {dtype:?}"), + }; + let buffer = self.new_buffer(shape.elem_count(), dtype, "rand_normal")?; + let command_buffer = self.command_buffer()?; + candle_metal_kernels::call_random_normal( + &self.device, + &command_buffer, + &self.kernels, + name, + mean as f32, + stddev as f32, + shape.elem_count(), + &self.seed.lock().unwrap(), + &buffer, + ) + .map_err(MetalError::from)?; + + Ok(Self::Storage::new(buffer, self.clone(), dtype)) + } + + fn set_seed(&self, seed: u64) -> Result<()> { + let seed: u32 = seed.try_into().map_err(|_| { + MetalError::Message("Metal seed must be less than or equal to u32::MAX".to_string()) + })?; + + let seed_buffer = self.seed.try_lock().map_err(MetalError::from)?; + let contents = seed_buffer.contents(); + unsafe { + std::ptr::copy([seed].as_ptr(), contents as *mut u32, 4); + } + seed_buffer.did_modify_range(metal::NSRange::new(0, 4)); + + Ok(()) + } +} + +fn read_to_vec(buffer: &Buffer, n: usize) -> Vec { + let ptr = buffer.contents() as *const T; + assert!(!ptr.is_null()); + let slice = unsafe { std::slice::from_raw_parts(ptr, n) }; + slice.to_vec() +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/mkl.rs b/v2/dependency-patches/candle-core-0.4.1/src/mkl.rs new file mode 100644 index 0000000000..359add741c --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/mkl.rs @@ -0,0 +1,418 @@ +#![allow(dead_code)] +use libc::{c_char, c_double, c_float, c_int}; + +mod ffi { + use super::*; + extern "C" { + pub fn vsTanh(n: c_int, a: *const c_float, y: *mut c_float); + pub fn vdTanh(n: c_int, a: *const c_double, y: *mut c_double); + pub fn vsExp(n: c_int, a: *const c_float, y: *mut c_float); + pub fn vdExp(n: c_int, a: *const c_double, y: *mut c_double); + pub fn vsLn(n: c_int, a: *const c_float, y: *mut c_float); + pub fn vdLn(n: c_int, a: *const c_double, y: *mut c_double); + pub fn vsSin(n: c_int, a: *const c_float, y: *mut c_float); + pub fn vdSin(n: c_int, a: *const c_double, y: *mut c_double); + pub fn vsCos(n: c_int, a: *const c_float, y: *mut c_float); + pub fn vdCos(n: c_int, a: *const c_double, y: *mut c_double); + pub fn vsSqrt(n: c_int, a: *const c_float, y: *mut c_float); + pub fn vdSqrt(n: c_int, a: *const c_double, y: *mut c_double); + + pub fn vsAdd(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); + pub fn vdAdd(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); + pub fn vsSub(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); + pub fn vdSub(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); + pub fn vsMul(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); + pub fn vdMul(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); + pub fn vsDiv(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); + pub fn vdDiv(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); + pub fn vsFmax(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); + pub fn vdFmax(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); + pub fn vsFmin(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); + pub fn vdFmin(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); + + pub fn sgemm_( + transa: *const c_char, + transb: *const c_char, + m: *const c_int, + n: *const c_int, + k: *const c_int, + alpha: *const c_float, + a: *const c_float, + lda: *const c_int, + b: *const c_float, + ldb: *const c_int, + beta: *const c_float, + c: *mut c_float, + ldc: *const c_int, + ); + pub fn dgemm_( + transa: *const c_char, + transb: *const c_char, + m: *const c_int, + n: *const c_int, + k: *const c_int, + alpha: *const c_double, + a: *const c_double, + lda: *const c_int, + b: *const c_double, + ldb: *const c_int, + beta: *const c_double, + c: *mut c_double, + ldc: *const c_int, + ); + pub fn hgemm_( + transa: *const c_char, + transb: *const c_char, + m: *const c_int, + n: *const c_int, + k: *const c_int, + alpha: *const half::f16, + a: *const half::f16, + lda: *const c_int, + b: *const half::f16, + ldb: *const c_int, + beta: *const half::f16, + c: *mut half::f16, + ldc: *const c_int, + ); + } +} + +#[allow(clippy::too_many_arguments)] +#[inline] +pub unsafe fn sgemm( + transa: u8, + transb: u8, + m: i32, + n: i32, + k: i32, + alpha: f32, + a: &[f32], + lda: i32, + b: &[f32], + ldb: i32, + beta: f32, + c: &mut [f32], + ldc: i32, +) { + ffi::sgemm_( + &(transa as c_char), + &(transb as c_char), + &m, + &n, + &k, + &alpha, + a.as_ptr(), + &lda, + b.as_ptr(), + &ldb, + &beta, + c.as_mut_ptr(), + &ldc, + ) +} + +#[allow(clippy::too_many_arguments)] +#[inline] +pub unsafe fn dgemm( + transa: u8, + transb: u8, + m: i32, + n: i32, + k: i32, + alpha: f64, + a: &[f64], + lda: i32, + b: &[f64], + ldb: i32, + beta: f64, + c: &mut [f64], + ldc: i32, +) { + ffi::dgemm_( + &(transa as c_char), + &(transb as c_char), + &m, + &n, + &k, + &alpha, + a.as_ptr(), + &lda, + b.as_ptr(), + &ldb, + &beta, + c.as_mut_ptr(), + &ldc, + ) +} + +#[allow(clippy::too_many_arguments)] +#[inline] +pub unsafe fn hgemm( + transa: u8, + transb: u8, + m: i32, + n: i32, + k: i32, + alpha: half::f16, + a: &[half::f16], + lda: i32, + b: &[half::f16], + ldb: i32, + beta: half::f16, + c: &mut [half::f16], + ldc: i32, +) { + ffi::hgemm_( + &(transa as c_char), + &(transb as c_char), + &m, + &n, + &k, + &alpha, + a.as_ptr(), + &lda, + b.as_ptr(), + &ldb, + &beta, + c.as_mut_ptr(), + &ldc, + ) +} + +#[inline] +pub fn vs_exp(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vsExp(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vd_exp(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vdExp(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vs_ln(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vsLn(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vd_ln(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vdLn(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vs_sin(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vsSin(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vd_sin(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vdSin(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vs_cos(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vsCos(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vd_cos(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vdCos(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vs_sqrt(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vsSqrt(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vd_sqrt(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vdSqrt(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vs_sqr(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vsMul(a_len as i32, a.as_ptr(), a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vd_sqr(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vdMul(a_len as i32, a.as_ptr(), a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vs_tanh(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vsTanh(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vd_tanh(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vdTanh(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +// The vector functions from mkl can be performed in place by using the same array for input and +// output. +// https://www.intel.com/content/www/us/en/docs/onemkl/developer-reference-c/2023-2/vector-mathematical-functions.html +#[inline] +pub fn vs_tanh_inplace(y: &mut [f32]) { + unsafe { ffi::vsTanh(y.len() as i32, y.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vd_tanh_inplace(y: &mut [f64]) { + unsafe { ffi::vdTanh(y.len() as i32, y.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vs_exp_inplace(y: &mut [f32]) { + unsafe { ffi::vsExp(y.len() as i32, y.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vd_exp_inplace(y: &mut [f64]) { + unsafe { ffi::vdExp(y.len() as i32, y.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vs_gelu(vs: &[f32], ys: &mut [f32]) { + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = (2.0f32 / std::f32::consts::PI).sqrt() * v * (1.0 + 0.044715 * v * v) + } + vs_tanh_inplace(ys); + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = 0.5 * v * (1.0 + *y) + } +} + +#[inline] +pub fn vd_gelu(vs: &[f64], ys: &mut [f64]) { + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = (2.0f64 / std::f64::consts::PI).sqrt() * v * (1.0 + 0.044715 * v * v) + } + vd_tanh_inplace(ys); + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = 0.5 * v * (1.0 + *y) + } +} + +#[inline] +pub fn vs_silu(vs: &[f32], ys: &mut [f32]) { + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = -v + } + vs_exp_inplace(ys); + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = v / (1.0 + *y) + } +} + +#[inline] +pub fn vd_silu(vs: &[f64], ys: &mut [f64]) { + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = -v + } + vd_exp_inplace(ys); + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = v / (1.0 + *y) + } +} + +macro_rules! binary_op { + ($fn_name:ident, $ty:ty, $mkl_name:ident) => { + #[inline] + pub fn $fn_name(a: &[$ty], b: &[$ty], y: &mut [$ty]) { + let a_len = a.len(); + let b_len = b.len(); + let y_len = y.len(); + if a_len != y_len || b_len != y_len { + panic!( + "{} a,b,y len mismatch {a_len} {b_len} {y_len}", + stringify!($fn_name) + ); + } + unsafe { ffi::$mkl_name(a_len as i32, a.as_ptr(), b.as_ptr(), y.as_mut_ptr()) } + } + }; +} +binary_op!(vs_add, f32, vsAdd); +binary_op!(vd_add, f64, vdAdd); +binary_op!(vs_sub, f32, vsSub); +binary_op!(vd_sub, f64, vdSub); +binary_op!(vs_mul, f32, vsMul); +binary_op!(vd_mul, f64, vdMul); +binary_op!(vs_div, f32, vsDiv); +binary_op!(vd_div, f64, vdDiv); +binary_op!(vs_max, f32, vsFmax); +binary_op!(vd_max, f64, vdFmax); +binary_op!(vs_min, f32, vsFmin); +binary_op!(vd_min, f64, vdFmin); diff --git a/v2/dependency-patches/candle-core-0.4.1/src/npy.rs b/v2/dependency-patches/candle-core-0.4.1/src/npy.rs new file mode 100644 index 0000000000..b07c48e9fe --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/npy.rs @@ -0,0 +1,450 @@ +//! Numpy support for tensors. +//! +//! The spec for the npy format can be found in +//! [npy-format](https://docs.scipy.org/doc/numpy-1.14.2/neps/npy-format.html). +//! The functions from this module can be used to read tensors from npy/npz files +//! or write tensors to these files. A npy file contains a single tensor (unnamed) +//! whereas a npz file can contain multiple named tensors. npz files are also compressed. +//! +//! These two formats are easy to use in Python using the numpy library. +//! +//! ```python +//! import numpy as np +//! x = np.arange(10) +//! +//! # Write a npy file. +//! np.save("test.npy", x) +//! +//! # Read a value from the npy file. +//! x = np.load("test.npy") +//! +//! # Write multiple values to a npz file. +//! values = { "x": x, "x_plus_one": x + 1 } +//! np.savez("test.npz", **values) +//! +//! # Load multiple values from a npz file. +//! values = np.loadz("test.npz") +//! ``` +use crate::{DType, Device, Error, Result, Shape, Tensor}; +use byteorder::{LittleEndian, ReadBytesExt}; +use half::{bf16, f16, slice::HalfFloatSliceExt}; +use std::collections::HashMap; +use std::fs::File; +use std::io::{BufReader, Read, Write}; +use std::path::Path; + +const NPY_MAGIC_STRING: &[u8] = b"\x93NUMPY"; +const NPY_SUFFIX: &str = ".npy"; + +fn read_header(reader: &mut R) -> Result { + let mut magic_string = vec![0u8; NPY_MAGIC_STRING.len()]; + reader.read_exact(&mut magic_string)?; + if magic_string != NPY_MAGIC_STRING { + return Err(Error::Npy("magic string mismatch".to_string())); + } + let mut version = [0u8; 2]; + reader.read_exact(&mut version)?; + let header_len_len = match version[0] { + 1 => 2, + 2 => 4, + otherwise => return Err(Error::Npy(format!("unsupported version {otherwise}"))), + }; + let mut header_len = vec![0u8; header_len_len]; + reader.read_exact(&mut header_len)?; + let header_len = header_len + .iter() + .rev() + .fold(0_usize, |acc, &v| 256 * acc + v as usize); + let mut header = vec![0u8; header_len]; + reader.read_exact(&mut header)?; + Ok(String::from_utf8_lossy(&header).to_string()) +} + +#[derive(Debug, PartialEq)] +struct Header { + descr: DType, + fortran_order: bool, + shape: Vec, +} + +impl Header { + fn shape(&self) -> Shape { + Shape::from(self.shape.as_slice()) + } + + fn to_string(&self) -> Result { + let fortran_order = if self.fortran_order { "True" } else { "False" }; + let mut shape = self + .shape + .iter() + .map(|x| x.to_string()) + .collect::>() + .join(","); + let descr = match self.descr { + DType::BF16 => Err(Error::Npy("bf16 is not supported".into()))?, + DType::F16 => "f2", + DType::F32 => "f4", + DType::F64 => "f8", + DType::I64 => "i8", + DType::U32 => "u4", + DType::U8 => "u1", + }; + if !shape.is_empty() { + shape.push(',') + } + Ok(format!( + "{{'descr': '<{descr}', 'fortran_order': {fortran_order}, 'shape': ({shape}), }}" + )) + } + + // Hacky parser for the npy header, a typical example would be: + // {'descr': ' Result
{ + let header = + header.trim_matches(|c: char| c == '{' || c == '}' || c == ',' || c.is_whitespace()); + + let mut parts: Vec = vec![]; + let mut start_index = 0usize; + let mut cnt_parenthesis = 0i64; + for (index, c) in header.chars().enumerate() { + match c { + '(' => cnt_parenthesis += 1, + ')' => cnt_parenthesis -= 1, + ',' => { + if cnt_parenthesis == 0 { + parts.push(header[start_index..index].to_owned()); + start_index = index + 1; + } + } + _ => {} + } + } + parts.push(header[start_index..].to_owned()); + let mut part_map: HashMap = HashMap::new(); + for part in parts.iter() { + let part = part.trim(); + if !part.is_empty() { + match part.split(':').collect::>().as_slice() { + [key, value] => { + let key = key.trim_matches(|c: char| c == '\'' || c.is_whitespace()); + let value = value.trim_matches(|c: char| c == '\'' || c.is_whitespace()); + let _ = part_map.insert(key.to_owned(), value.to_owned()); + } + _ => return Err(Error::Npy(format!("unable to parse header {header}"))), + } + } + } + let fortran_order = match part_map.get("fortran_order") { + None => false, + Some(fortran_order) => match fortran_order.as_ref() { + "False" => false, + "True" => true, + _ => return Err(Error::Npy(format!("unknown fortran_order {fortran_order}"))), + }, + }; + let descr = match part_map.get("descr") { + None => return Err(Error::Npy("no descr in header".to_string())), + Some(descr) => { + if descr.is_empty() { + return Err(Error::Npy("empty descr".to_string())); + } + if descr.starts_with('>') { + return Err(Error::Npy(format!("little-endian descr {descr}"))); + } + // the only supported types in tensor are: + // float64, float32, float16, + // complex64, complex128, + // int64, int32, int16, int8, + // uint8, and bool. + match descr.trim_matches(|c: char| c == '=' || c == '<' || c == '|') { + "e" | "f2" => DType::F16, + "f" | "f4" => DType::F32, + "d" | "f8" => DType::F64, + // "i" | "i4" => DType::S32, + "q" | "i8" => DType::I64, + // "h" | "i2" => DType::S16, + // "b" | "i1" => DType::S8, + "B" | "u1" => DType::U8, + "I" | "u4" => DType::U32, + "?" | "b1" => DType::U8, + // "F" | "F4" => DType::C64, + // "D" | "F8" => DType::C128, + descr => return Err(Error::Npy(format!("unrecognized descr {descr}"))), + } + } + }; + let shape = match part_map.get("shape") { + None => return Err(Error::Npy("no shape in header".to_string())), + Some(shape) => { + let shape = shape.trim_matches(|c: char| c == '(' || c == ')' || c == ','); + if shape.is_empty() { + vec![] + } else { + shape + .split(',') + .map(|v| v.trim().parse::()) + .collect::, _>>()? + } + } + }; + Ok(Header { + descr, + fortran_order, + shape, + }) + } +} + +impl Tensor { + // TODO: Add the possibility to read directly to a device? + pub(crate) fn from_reader( + shape: Shape, + dtype: DType, + reader: &mut R, + ) -> Result { + let elem_count = shape.elem_count(); + match dtype { + DType::BF16 => { + let mut data_t = vec![bf16::ZERO; elem_count]; + reader.read_u16_into::(data_t.reinterpret_cast_mut())?; + Tensor::from_vec(data_t, shape, &Device::Cpu) + } + DType::F16 => { + let mut data_t = vec![f16::ZERO; elem_count]; + reader.read_u16_into::(data_t.reinterpret_cast_mut())?; + Tensor::from_vec(data_t, shape, &Device::Cpu) + } + DType::F32 => { + let mut data_t = vec![0f32; elem_count]; + reader.read_f32_into::(&mut data_t)?; + Tensor::from_vec(data_t, shape, &Device::Cpu) + } + DType::F64 => { + let mut data_t = vec![0f64; elem_count]; + reader.read_f64_into::(&mut data_t)?; + Tensor::from_vec(data_t, shape, &Device::Cpu) + } + DType::U8 => { + let mut data_t = vec![0u8; elem_count]; + reader.read_exact(&mut data_t)?; + Tensor::from_vec(data_t, shape, &Device::Cpu) + } + DType::U32 => { + let mut data_t = vec![0u32; elem_count]; + reader.read_u32_into::(&mut data_t)?; + Tensor::from_vec(data_t, shape, &Device::Cpu) + } + DType::I64 => { + let mut data_t = vec![0i64; elem_count]; + reader.read_i64_into::(&mut data_t)?; + Tensor::from_vec(data_t, shape, &Device::Cpu) + } + } + } + + /// Reads a npy file and return the stored multi-dimensional array as a tensor. + pub fn read_npy>(path: T) -> Result { + let mut reader = File::open(path.as_ref())?; + let header = read_header(&mut reader)?; + let header = Header::parse(&header)?; + if header.fortran_order { + return Err(Error::Npy("fortran order not supported".to_string())); + } + Self::from_reader(header.shape(), header.descr, &mut reader) + } + + /// Reads a npz file and returns the stored multi-dimensional arrays together with their names. + pub fn read_npz>(path: T) -> Result> { + let zip_reader = BufReader::new(File::open(path.as_ref())?); + let mut zip = zip::ZipArchive::new(zip_reader)?; + let mut result = vec![]; + for i in 0..zip.len() { + let mut reader = zip.by_index(i)?; + let name = { + let name = reader.name()?; + name.strip_suffix(NPY_SUFFIX).unwrap_or(&name).to_owned() + }; + let header = read_header(&mut reader)?; + let header = Header::parse(&header)?; + if header.fortran_order { + return Err(Error::Npy("fortran order not supported".to_string())); + } + let s = Self::from_reader(header.shape(), header.descr, &mut reader)?; + result.push((name, s)) + } + Ok(result) + } + + /// Reads a npz file and returns the stored multi-dimensional arrays for some specified names. + pub fn read_npz_by_name>(path: T, names: &[&str]) -> Result> { + let zip_reader = BufReader::new(File::open(path.as_ref())?); + let mut zip = zip::ZipArchive::new(zip_reader)?; + let mut result = vec![]; + for name in names.iter() { + let mut reader = match zip.by_name(&format!("{name}{NPY_SUFFIX}")) { + Ok(reader) => reader, + Err(_) => Err(Error::Npy(format!( + "no array for {name} in {:?}", + path.as_ref() + )))?, + }; + let header = read_header(&mut reader)?; + let header = Header::parse(&header)?; + if header.fortran_order { + return Err(Error::Npy("fortran order not supported".to_string())); + } + let s = Self::from_reader(header.shape(), header.descr, &mut reader)?; + result.push(s) + } + Ok(result) + } + + fn write(&self, f: &mut T) -> Result<()> { + f.write_all(NPY_MAGIC_STRING)?; + f.write_all(&[1u8, 0u8])?; + let header = Header { + descr: self.dtype(), + fortran_order: false, + shape: self.dims().to_vec(), + }; + let mut header = header.to_string()?; + let pad = 16 - (NPY_MAGIC_STRING.len() + 5 + header.len()) % 16; + for _ in 0..pad % 16 { + header.push(' ') + } + header.push('\n'); + f.write_all(&[(header.len() % 256) as u8, (header.len() / 256) as u8])?; + f.write_all(header.as_bytes())?; + self.write_bytes(f) + } + + /// Writes a multi-dimensional array in the npy format. + pub fn write_npy>(&self, path: T) -> Result<()> { + let mut f = File::create(path.as_ref())?; + self.write(&mut f) + } + + /// Writes multiple multi-dimensional arrays using the npz format. + pub fn write_npz, T: AsRef, P: AsRef>( + ts: &[(S, T)], + path: P, + ) -> Result<()> { + let mut zip = zip::ZipWriter::new(File::create(path.as_ref())?); + let options = + zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + + for (name, tensor) in ts.iter() { + zip.start_file(format!("{}.npy", name.as_ref()), options)?; + tensor.as_ref().write(&mut zip)? + } + Ok(()) + } +} + +/// Lazy tensor loader. +pub struct NpzTensors { + index_per_name: HashMap, + path: std::path::PathBuf, + // We do not store a zip reader as it needs mutable access to extract data. Instead we + // re-create a zip reader for each tensor. +} + +impl NpzTensors { + pub fn new>(path: T) -> Result { + let path = path.as_ref().to_owned(); + let zip_reader = BufReader::new(File::open(&path)?); + let mut zip = zip::ZipArchive::new(zip_reader)?; + let mut index_per_name = HashMap::new(); + for i in 0..zip.len() { + let file = zip.by_index(i)?; + let name = { + let name = file.name()?; + name.strip_suffix(NPY_SUFFIX).unwrap_or(&name).to_owned() + }; + index_per_name.insert(name, i); + } + Ok(Self { + index_per_name, + path, + }) + } + + pub fn names(&self) -> Vec<&String> { + self.index_per_name.keys().collect() + } + + /// This only returns the shape and dtype for a named tensor. Compared to `get`, this avoids + /// reading the whole tensor data. + pub fn get_shape_and_dtype(&self, name: &str) -> Result<(Shape, DType)> { + let index = match self.index_per_name.get(name) { + None => crate::bail!("cannot find tensor {name}"), + Some(index) => *index, + }; + let zip_reader = BufReader::new(File::open(&self.path)?); + let mut zip = zip::ZipArchive::new(zip_reader)?; + let mut reader = zip.by_index(index)?; + let header = read_header(&mut reader)?; + let header = Header::parse(&header)?; + Ok((header.shape(), header.descr)) + } + + pub fn get(&self, name: &str) -> Result> { + let index = match self.index_per_name.get(name) { + None => return Ok(None), + Some(index) => *index, + }; + // We hope that the file has not changed since first reading it. + let zip_reader = BufReader::new(File::open(&self.path)?); + let mut zip = zip::ZipArchive::new(zip_reader)?; + let mut reader = zip.by_index(index)?; + let header = read_header(&mut reader)?; + let header = Header::parse(&header)?; + if header.fortran_order { + return Err(Error::Npy("fortran order not supported".to_string())); + } + let tensor = Tensor::from_reader(header.shape(), header.descr, &mut reader)?; + Ok(Some(tensor)) + } +} + +#[cfg(test)] +mod tests { + use super::Header; + + #[test] + fn parse() { + let h = "{'descr': ' &'static str { + match self { + Self::ArgMax => "argmax", + Self::ArgMin => "argmin", + Self::Min => "min", + Self::Max => "max", + Self::Sum => "sum", + } + } +} + +// These ops return the same type as their input type. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BinaryOp { + Add, + Mul, + Sub, + Div, + Maximum, + Minimum, +} + +// Unary ops with no argument +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UnaryOp { + Exp, + Log, + Sin, + Cos, + Abs, + Neg, + Recip, + Sqr, + Sqrt, + Gelu, + GeluErf, + Erf, + Relu, + Silu, + Tanh, + Floor, + Ceil, + Round, +} + +#[derive(Clone)] +pub enum Op { + Binary(Tensor, Tensor, BinaryOp), + Unary(Tensor, UnaryOp), + Cmp(Tensor, CmpOp), + // The third argument is the reduced shape with `keepdim=true`. + Reduce(Tensor, ReduceOp, Vec), + Matmul(Tensor, Tensor), + Gather(Tensor, Tensor, usize), + ScatterAdd(Tensor, Tensor, Tensor, usize), + IndexSelect(Tensor, Tensor, usize), + IndexAdd(Tensor, Tensor, Tensor, usize), + WhereCond(Tensor, Tensor, Tensor), + + #[allow(dead_code)] + Conv1D { + arg: Tensor, + kernel: Tensor, + padding: usize, + stride: usize, + dilation: usize, + }, + + #[allow(dead_code)] + ConvTranspose1D { + arg: Tensor, + kernel: Tensor, + padding: usize, + output_padding: usize, + stride: usize, + dilation: usize, + }, + + #[allow(dead_code)] + Conv2D { + arg: Tensor, + kernel: Tensor, + padding: usize, + stride: usize, + dilation: usize, + }, + + #[allow(dead_code)] + ConvTranspose2D { + arg: Tensor, + kernel: Tensor, + padding: usize, + output_padding: usize, + stride: usize, + dilation: usize, + }, + + AvgPool2D { + arg: Tensor, + kernel_size: (usize, usize), + stride: (usize, usize), + }, + + MaxPool2D { + arg: Tensor, + kernel_size: (usize, usize), + stride: (usize, usize), + }, + + UpsampleNearest1D { + arg: Tensor, + target_size: usize, + }, + UpsampleNearest2D { + arg: Tensor, + target_h: usize, + target_w: usize, + }, + + Cat(Vec, usize), + + #[allow(dead_code)] // add is currently unused. + Affine { + arg: Tensor, + mul: f64, + add: f64, + }, + ToDType(Tensor), + Copy(Tensor), + Broadcast(Tensor), + Narrow(Tensor, usize, usize, usize), + SliceScatter0(Tensor, Tensor, usize), + Reshape(Tensor), + ToDevice(Tensor), + Transpose(Tensor, usize, usize), + Permute(Tensor, Vec), + Elu(Tensor, f64), + Powf(Tensor, f64), + CustomOp1(Tensor, std::sync::Arc>), + CustomOp2( + Tensor, + Tensor, + std::sync::Arc>, + ), + CustomOp3( + Tensor, + Tensor, + Tensor, + std::sync::Arc>, + ), +} + +/// Unary ops that can be defined in user-land. +pub trait CustomOp1 { + // Box does not support const yet, so use a function to get the name. + fn name(&self) -> &'static str; + + /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn cpu_fwd(&self, storage: &CpuStorage, layout: &Layout) -> Result<(CpuStorage, Shape)>; + + /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn cuda_fwd(&self, _storage: &CudaStorage, _layout: &Layout) -> Result<(CudaStorage, Shape)> { + Err(crate::Error::Cuda( + format!("no cuda implementation for {}", self.name()).into(), + )) + } + + /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn metal_fwd( + &self, + _storage: &MetalStorage, + _layout: &Layout, + ) -> Result<(MetalStorage, Shape)> { + Err(crate::Error::Metal( + format!("no metal implementation for {}", self.name()).into(), + )) + } + + /// This function takes as argument the argument `arg` used in the forward pass, the result + /// produced by the forward operation `res` and the gradient of the result `grad_res`. + /// The function should return the gradient of the argument. + fn bwd(&self, _arg: &Tensor, _res: &Tensor, _grad_res: &Tensor) -> Result> { + Err(crate::Error::BackwardNotSupported { op: self.name() }) + } +} + +pub trait CustomOp2 { + fn name(&self) -> &'static str; + + /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn cpu_fwd( + &self, + s1: &CpuStorage, + l1: &Layout, + s2: &CpuStorage, + l2: &Layout, + ) -> Result<(CpuStorage, Shape)>; + + /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn cuda_fwd( + &self, + _: &CudaStorage, + _: &Layout, + _: &CudaStorage, + _: &Layout, + ) -> Result<(CudaStorage, Shape)> { + Err(crate::Error::Cuda( + format!("no cuda implementation for {}", self.name()).into(), + )) + } + + /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn metal_fwd( + &self, + _: &MetalStorage, + _: &Layout, + _: &MetalStorage, + _: &Layout, + ) -> Result<(MetalStorage, Shape)> { + Err(crate::Error::Metal( + format!("no metal implementation for {}", self.name()).into(), + )) + } + + fn bwd( + &self, + _arg1: &Tensor, + _arg2: &Tensor, + _res: &Tensor, + _grad_res: &Tensor, + ) -> Result<(Option, Option)> { + Err(crate::Error::BackwardNotSupported { op: self.name() }) + } +} + +pub trait CustomOp3 { + fn name(&self) -> &'static str; + + /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn cpu_fwd( + &self, + s1: &CpuStorage, + l1: &Layout, + s2: &CpuStorage, + l2: &Layout, + s3: &CpuStorage, + l3: &Layout, + ) -> Result<(CpuStorage, Shape)>; + + /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn cuda_fwd( + &self, + _: &CudaStorage, + _: &Layout, + _: &CudaStorage, + _: &Layout, + _: &CudaStorage, + _: &Layout, + ) -> Result<(CudaStorage, Shape)> { + Err(crate::Error::Cuda( + format!("no cuda implementation for {}", self.name()).into(), + )) + } + + /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn metal_fwd( + &self, + _: &MetalStorage, + _: &Layout, + _: &MetalStorage, + _: &Layout, + _: &MetalStorage, + _: &Layout, + ) -> Result<(MetalStorage, Shape)> { + Err(crate::Error::Metal( + format!("no metal implementation for {}", self.name()).into(), + )) + } + + fn bwd( + &self, + _arg1: &Tensor, + _arg2: &Tensor, + _arg3: &Tensor, + _res: &Tensor, + _grad_res: &Tensor, + ) -> Result<(Option, Option, Option)> { + Err(crate::Error::BackwardNotSupported { op: self.name() }) + } +} + +pub trait UnaryOpT { + const NAME: &'static str; + const KERNEL: &'static str; + const V: Self; + fn bf16(v1: bf16) -> bf16; + fn f16(v1: f16) -> f16; + fn f32(v1: f32) -> f32; + fn f64(v1: f64) -> f64; + fn u8(v1: u8) -> u8; + fn u32(v1: u32) -> u32; + fn i64(v1: i64) -> i64; + + // There is no very good way to represent optional function in traits so we go for an explicit + // boolean flag to mark the function as existing. + const BF16_VEC: bool = false; + fn bf16_vec(_xs: &[bf16], _ys: &mut [bf16]) {} + const F16_VEC: bool = false; + fn f16_vec(_xs: &[f16], _ys: &mut [f16]) {} + const F32_VEC: bool = false; + fn f32_vec(_xs: &[f32], _ys: &mut [f32]) {} + const F64_VEC: bool = false; + fn f64_vec(_xs: &[f64], _ys: &mut [f64]) {} +} + +pub trait BinaryOpT { + const NAME: &'static str; + const KERNEL: &'static str; + const V: Self; + fn bf16(v1: bf16, v2: bf16) -> bf16; + fn f16(v1: f16, v2: f16) -> f16; + fn f32(v1: f32, v2: f32) -> f32; + fn f64(v1: f64, v2: f64) -> f64; + fn u8(v1: u8, v2: u8) -> u8; + fn u32(v1: u32, v2: u32) -> u32; + fn i64(v1: i64, v2: i64) -> i64; + + const BF16_VEC: bool = false; + fn bf16_vec(_xs1: &[bf16], _xs2: &[bf16], _ys: &mut [bf16]) {} + const F16_VEC: bool = false; + fn f16_vec(_xs1: &[f16], _xs2: &[f16], _ys: &mut [f16]) {} + const F32_VEC: bool = false; + fn f32_vec(_xs1: &[f32], _xs2: &[f32], _ys: &mut [f32]) {} + const F64_VEC: bool = false; + fn f64_vec(_xs1: &[f64], _xs2: &[f64], _ys: &mut [f64]) {} + const U8_VEC: bool = false; + fn u8_vec(_xs1: &[u8], _xs2: &[u8], _ys: &mut [u8]) {} + const U32_VEC: bool = false; + fn u32_vec(_xs1: &[u32], _xs2: &[u32], _ys: &mut [u32]) {} + const I64_VEC: bool = false; + fn i64_vec(_xs1: &[i64], _xs2: &[i64], _ys: &mut [i64]) {} +} + +pub(crate) struct Add; +pub(crate) struct Div; +pub(crate) struct Mul; +pub(crate) struct Sub; +pub(crate) struct Maximum; +pub(crate) struct Minimum; +pub(crate) struct Exp; +pub(crate) struct Log; +pub(crate) struct Sin; +pub(crate) struct Cos; +pub(crate) struct Abs; +pub(crate) struct Neg; +pub(crate) struct Recip; +pub(crate) struct Sqr; +pub(crate) struct Sqrt; +pub(crate) struct Gelu; +pub(crate) struct GeluErf; +pub(crate) struct Erf; +pub(crate) struct Relu; +pub(crate) struct Silu; +pub(crate) struct Tanh; +pub(crate) struct Floor; +pub(crate) struct Ceil; +pub(crate) struct Round; + +macro_rules! bin_op { + ($op:ident, $name: literal, $e: expr, $f32_vec: ident, $f64_vec: ident) => { + impl BinaryOpT for $op { + const NAME: &'static str = $name; + const KERNEL: &'static str = concat!("b", $name); + const V: Self = $op; + #[inline(always)] + fn bf16(v1: bf16, v2: bf16) -> bf16 { + $e(v1, v2) + } + #[inline(always)] + fn f16(v1: f16, v2: f16) -> f16 { + $e(v1, v2) + } + #[inline(always)] + fn f32(v1: f32, v2: f32) -> f32 { + $e(v1, v2) + } + #[inline(always)] + fn f64(v1: f64, v2: f64) -> f64 { + $e(v1, v2) + } + #[inline(always)] + fn u8(v1: u8, v2: u8) -> u8 { + $e(v1, v2) + } + #[inline(always)] + fn u32(v1: u32, v2: u32) -> u32 { + $e(v1, v2) + } + #[inline(always)] + fn i64(v1: i64, v2: i64) -> i64 { + $e(v1, v2) + } + + #[cfg(feature = "mkl")] + const F32_VEC: bool = true; + #[cfg(feature = "mkl")] + const F64_VEC: bool = true; + #[cfg(feature = "mkl")] + #[inline(always)] + fn f32_vec(xs1: &[f32], xs2: &[f32], ys: &mut [f32]) { + crate::mkl::$f32_vec(xs1, xs2, ys) + } + #[cfg(feature = "mkl")] + #[inline(always)] + fn f64_vec(xs1: &[f64], xs2: &[f64], ys: &mut [f64]) { + crate::mkl::$f64_vec(xs1, xs2, ys) + } + + #[cfg(feature = "accelerate")] + const F32_VEC: bool = true; + #[cfg(feature = "accelerate")] + const F64_VEC: bool = true; + #[cfg(feature = "accelerate")] + #[inline(always)] + fn f32_vec(xs1: &[f32], xs2: &[f32], ys: &mut [f32]) { + crate::accelerate::$f32_vec(xs1, xs2, ys) + } + #[cfg(feature = "accelerate")] + #[inline(always)] + fn f64_vec(xs1: &[f64], xs2: &[f64], ys: &mut [f64]) { + crate::accelerate::$f64_vec(xs1, xs2, ys) + } + } + }; +} + +bin_op!(Add, "add", |v1, v2| v1 + v2, vs_add, vd_add); +bin_op!(Sub, "sub", |v1, v2| v1 - v2, vs_sub, vd_sub); +bin_op!(Mul, "mul", |v1, v2| v1 * v2, vs_mul, vd_mul); +bin_op!(Div, "div", |v1, v2| v1 / v2, vs_div, vd_div); +bin_op!( + Minimum, + "minimum", + |v1, v2| if v1 > v2 { v2 } else { v1 }, + vs_min, + vd_min +); +bin_op!( + Maximum, + "maximum", + |v1, v2| if v1 < v2 { v2 } else { v1 }, + vs_max, + vd_max +); + +#[allow(clippy::redundant_closure_call)] +macro_rules! unary_op { + ($op: ident, $name: literal, $a: ident, $e: expr) => { + impl UnaryOpT for $op { + const NAME: &'static str = $name; + const KERNEL: &'static str = concat!("u", $name); + const V: Self = $op; + #[inline(always)] + fn bf16($a: bf16) -> bf16 { + $e + } + #[inline(always)] + fn f16($a: f16) -> f16 { + $e + } + #[inline(always)] + fn f32($a: f32) -> f32 { + $e + } + #[inline(always)] + fn f64($a: f64) -> f64 { + $e + } + #[inline(always)] + fn u8(_: u8) -> u8 { + todo!("no unary function for u8") + } + #[inline(always)] + fn u32(_: u32) -> u32 { + todo!("no unary function for u32") + } + #[inline(always)] + fn i64(_: i64) -> i64 { + todo!("no unary function for i64") + } + } + }; + + ($op: ident, $name: literal, $a: ident, $e: expr, $f32_vec:ident, $f64_vec:ident) => { + impl UnaryOpT for $op { + const NAME: &'static str = $name; + const KERNEL: &'static str = concat!("u", $name); + const V: Self = $op; + #[inline(always)] + fn bf16($a: bf16) -> bf16 { + $e + } + #[inline(always)] + fn f16($a: f16) -> f16 { + $e + } + #[inline(always)] + fn f32($a: f32) -> f32 { + $e + } + #[inline(always)] + fn f64($a: f64) -> f64 { + $e + } + #[inline(always)] + fn u8(_: u8) -> u8 { + todo!("no unary function for u8") + } + #[inline(always)] + fn u32(_: u32) -> u32 { + todo!("no unary function for u32") + } + #[inline(always)] + fn i64(_: i64) -> i64 { + todo!("no unary function for i64") + } + + #[cfg(feature = "mkl")] + const F32_VEC: bool = true; + #[cfg(feature = "mkl")] + const F64_VEC: bool = true; + #[cfg(feature = "mkl")] + #[inline(always)] + fn f32_vec(xs: &[f32], ys: &mut [f32]) { + crate::mkl::$f32_vec(xs, ys) + } + #[cfg(feature = "mkl")] + #[inline(always)] + fn f64_vec(xs: &[f64], ys: &mut [f64]) { + crate::mkl::$f64_vec(xs, ys) + } + + #[cfg(feature = "accelerate")] + const F32_VEC: bool = true; + #[cfg(feature = "accelerate")] + const F64_VEC: bool = true; + #[cfg(feature = "accelerate")] + #[inline(always)] + fn f32_vec(xs: &[f32], ys: &mut [f32]) { + crate::accelerate::$f32_vec(xs, ys) + } + #[cfg(feature = "accelerate")] + #[inline(always)] + fn f64_vec(xs: &[f64], ys: &mut [f64]) { + crate::accelerate::$f64_vec(xs, ys) + } + } + }; +} + +unary_op!(Exp, "exp", v, v.exp(), vs_exp, vd_exp); +unary_op!(Log, "log", v, v.ln(), vs_ln, vd_ln); +unary_op!(Sin, "sin", v, v.sin(), vs_sin, vd_sin); +unary_op!(Cos, "cos", v, v.cos(), vs_cos, vd_cos); +unary_op!(Tanh, "tanh", v, v.tanh(), vs_tanh, vd_tanh); +unary_op!(Neg, "neg", v, -v); +unary_op!(Recip, "recip", v, v.recip()); +unary_op!(Sqr, "sqr", v, v * v, vs_sqr, vd_sqr); +unary_op!(Sqrt, "sqrt", v, v.sqrt(), vs_sqrt, vd_sqrt); + +/// Tanh based approximation of the `gelu` operation +/// GeluErf is the more precise one. +/// +impl UnaryOpT for Gelu { + const NAME: &'static str = "gelu"; + const V: Self = Gelu; + #[inline(always)] + fn bf16(v: bf16) -> bf16 { + bf16::from_f32_const(0.5) + * v + * (bf16::ONE + + bf16::tanh( + (bf16::from_f32_const(2.0) / bf16::PI).sqrt() + * v + * (bf16::ONE + bf16::from_f32_const(0.044715) * v * v), + )) + } + #[inline(always)] + fn f16(v: f16) -> f16 { + f16::from_f32_const(0.5) + * v + * (f16::ONE + + f16::tanh( + (f16::from_f32_const(2.0) / f16::PI).sqrt() + * v + * (f16::ONE + f16::from_f32_const(0.044715) * v * v), + )) + } + #[inline(always)] + fn f32(v: f32) -> f32 { + 0.5 * v + * (1.0 + + f32::tanh((2.0f32 / std::f32::consts::PI).sqrt() * v * (1.0 + 0.044715 * v * v))) + } + #[inline(always)] + fn f64(v: f64) -> f64 { + 0.5 * v + * (1.0 + + f64::tanh((2.0f64 / std::f64::consts::PI).sqrt() * v * (1.0 + 0.044715 * v * v))) + } + #[inline(always)] + fn u8(_: u8) -> u8 { + 0 + } + #[inline(always)] + fn u32(_: u32) -> u32 { + 0 + } + #[inline(always)] + fn i64(_: i64) -> i64 { + 0 + } + const KERNEL: &'static str = "ugelu"; + + #[cfg(feature = "mkl")] + const F32_VEC: bool = true; + + #[cfg(feature = "mkl")] + #[inline(always)] + fn f32_vec(xs: &[f32], ys: &mut [f32]) { + crate::mkl::vs_gelu(xs, ys) + } + + #[cfg(feature = "mkl")] + const F64_VEC: bool = true; + + #[cfg(feature = "mkl")] + #[inline(always)] + fn f64_vec(xs: &[f64], ys: &mut [f64]) { + crate::mkl::vd_gelu(xs, ys) + } + + #[cfg(feature = "accelerate")] + const F32_VEC: bool = true; + + #[cfg(feature = "accelerate")] + #[inline(always)] + fn f32_vec(xs: &[f32], ys: &mut [f32]) { + crate::accelerate::vs_gelu(xs, ys) + } + + #[cfg(feature = "accelerate")] + const F64_VEC: bool = true; + + #[cfg(feature = "accelerate")] + #[inline(always)] + fn f64_vec(xs: &[f64], ys: &mut [f64]) { + crate::accelerate::vd_gelu(xs, ys) + } +} + +/// `erf` operation +/// +impl UnaryOpT for Erf { + const NAME: &'static str = "erf"; + const KERNEL: &'static str = "uerf"; + const V: Self = Erf; + #[inline(always)] + fn bf16(v: bf16) -> bf16 { + bf16::from_f64(Self::f64(v.to_f64())) + } + #[inline(always)] + fn f16(v: f16) -> f16 { + f16::from_f64(Self::f64(v.to_f64())) + } + #[inline(always)] + fn f32(v: f32) -> f32 { + Self::f64(v as f64) as f32 + } + #[inline(always)] + fn f64(v: f64) -> f64 { + crate::cpu::erf::erf(v) + } + #[inline(always)] + fn u8(_: u8) -> u8 { + 0 + } + #[inline(always)] + fn u32(_: u32) -> u32 { + 0 + } + #[inline(always)] + fn i64(_: i64) -> i64 { + 0 + } +} + +/// Silu operation +impl UnaryOpT for Silu { + const NAME: &'static str = "silu"; + const V: Self = Silu; + #[inline(always)] + fn bf16(v: bf16) -> bf16 { + v / (bf16::ONE + (-v).exp()) + } + #[inline(always)] + fn f16(v: f16) -> f16 { + v / (f16::ONE + (-v).exp()) + } + #[inline(always)] + fn f32(v: f32) -> f32 { + v / (1.0 + (-v).exp()) + } + #[inline(always)] + fn f64(v: f64) -> f64 { + v / (1.0 + (-v).exp()) + } + #[inline(always)] + fn u8(_: u8) -> u8 { + 0 + } + #[inline(always)] + fn u32(_: u32) -> u32 { + 0 + } + #[inline(always)] + fn i64(_: i64) -> i64 { + 0 + } + const KERNEL: &'static str = "usilu"; + + #[cfg(feature = "mkl")] + const F32_VEC: bool = true; + + #[cfg(feature = "mkl")] + #[inline(always)] + fn f32_vec(xs: &[f32], ys: &mut [f32]) { + crate::mkl::vs_silu(xs, ys) + } + + #[cfg(feature = "mkl")] + const F64_VEC: bool = true; + + #[cfg(feature = "mkl")] + #[inline(always)] + fn f64_vec(xs: &[f64], ys: &mut [f64]) { + crate::mkl::vd_silu(xs, ys) + } + + #[cfg(feature = "accelerate")] + const F32_VEC: bool = true; + + #[cfg(feature = "accelerate")] + #[inline(always)] + fn f32_vec(xs: &[f32], ys: &mut [f32]) { + crate::accelerate::vs_silu(xs, ys) + } + + #[cfg(feature = "accelerate")] + const F64_VEC: bool = true; + + #[cfg(feature = "accelerate")] + #[inline(always)] + fn f64_vec(xs: &[f64], ys: &mut [f64]) { + crate::accelerate::vd_silu(xs, ys) + } +} + +impl UnaryOpT for Abs { + const NAME: &'static str = "abs"; + const KERNEL: &'static str = "uabs"; + const V: Self = Abs; + #[inline(always)] + fn bf16(v: bf16) -> bf16 { + v.abs() + } + #[inline(always)] + fn f16(v: f16) -> f16 { + v.abs() + } + #[inline(always)] + fn f32(v: f32) -> f32 { + v.abs() + } + #[inline(always)] + fn f64(v: f64) -> f64 { + v.abs() + } + #[inline(always)] + fn u8(v: u8) -> u8 { + v + } + #[inline(always)] + fn u32(v: u32) -> u32 { + v + } + #[inline(always)] + fn i64(v: i64) -> i64 { + v.abs() + } +} + +impl UnaryOpT for Ceil { + const NAME: &'static str = "ceil"; + const KERNEL: &'static str = "uceil"; + const V: Self = Ceil; + #[inline(always)] + fn bf16(v: bf16) -> bf16 { + v.ceil() + } + #[inline(always)] + fn f16(v: f16) -> f16 { + v.ceil() + } + #[inline(always)] + fn f32(v: f32) -> f32 { + v.ceil() + } + #[inline(always)] + fn f64(v: f64) -> f64 { + v.ceil() + } + #[inline(always)] + fn u8(v: u8) -> u8 { + v + } + #[inline(always)] + fn u32(v: u32) -> u32 { + v + } + #[inline(always)] + fn i64(v: i64) -> i64 { + v + } +} + +impl UnaryOpT for Floor { + const NAME: &'static str = "floor"; + const KERNEL: &'static str = "ufloor"; + const V: Self = Floor; + #[inline(always)] + fn bf16(v: bf16) -> bf16 { + v.floor() + } + #[inline(always)] + fn f16(v: f16) -> f16 { + v.floor() + } + #[inline(always)] + fn f32(v: f32) -> f32 { + v.floor() + } + #[inline(always)] + fn f64(v: f64) -> f64 { + v.floor() + } + #[inline(always)] + fn u8(v: u8) -> u8 { + v + } + #[inline(always)] + fn u32(v: u32) -> u32 { + v + } + #[inline(always)] + fn i64(v: i64) -> i64 { + v + } +} + +impl UnaryOpT for Round { + const NAME: &'static str = "round"; + const KERNEL: &'static str = "uround"; + const V: Self = Round; + #[inline(always)] + fn bf16(v: bf16) -> bf16 { + v.round() + } + #[inline(always)] + fn f16(v: f16) -> f16 { + v.round() + } + #[inline(always)] + fn f32(v: f32) -> f32 { + v.round() + } + #[inline(always)] + fn f64(v: f64) -> f64 { + v.round() + } + #[inline(always)] + fn u8(v: u8) -> u8 { + v + } + #[inline(always)] + fn u32(v: u32) -> u32 { + v + } + #[inline(always)] + fn i64(v: i64) -> i64 { + v + } +} + +impl UnaryOpT for GeluErf { + const NAME: &'static str = "gelu_erf"; + const KERNEL: &'static str = "ugelu_erf"; + const V: Self = GeluErf; + #[inline(always)] + fn bf16(v: bf16) -> bf16 { + bf16::from_f64(Self::f64(v.to_f64())) + } + #[inline(always)] + fn f16(v: f16) -> f16 { + f16::from_f64(Self::f64(v.to_f64())) + } + #[inline(always)] + fn f32(v: f32) -> f32 { + Self::f64(v as f64) as f32 + } + #[inline(always)] + fn f64(v: f64) -> f64 { + (crate::cpu::erf::erf(v / 2f64.sqrt()) + 1.) * 0.5 * v + } + #[inline(always)] + fn u8(_: u8) -> u8 { + 0 + } + #[inline(always)] + fn u32(_: u32) -> u32 { + 0 + } + #[inline(always)] + fn i64(_: i64) -> i64 { + 0 + } +} + +impl UnaryOpT for Relu { + const NAME: &'static str = "relu"; + const KERNEL: &'static str = "urelu"; + const V: Self = Relu; + #[inline(always)] + fn bf16(v: bf16) -> bf16 { + v.max(bf16::ZERO) + } + #[inline(always)] + fn f16(v: f16) -> f16 { + v.max(f16::ZERO) + } + #[inline(always)] + fn f32(v: f32) -> f32 { + v.max(0f32) + } + #[inline(always)] + fn f64(v: f64) -> f64 { + v.max(0f64) + } + #[inline(always)] + fn u8(v: u8) -> u8 { + v + } + #[inline(always)] + fn u32(v: u32) -> u32 { + v + } + #[inline(always)] + fn i64(v: i64) -> i64 { + v + } +} + +/// `BackpropOp` is a wrapper around `Option`. The main goal is to ensure that dependencies are +/// properly checked when creating a new value +#[derive(Clone)] +pub struct BackpropOp(Option); + +impl BackpropOp { + pub(crate) fn none() -> Self { + BackpropOp(None) + } + + pub(crate) fn new1(arg: &Tensor, f: impl Fn(Tensor) -> Op) -> Self { + let op = if arg.track_op() { + Some(f(arg.clone())) + } else { + None + }; + Self(op) + } + + pub(crate) fn new2(arg1: &Tensor, arg2: &Tensor, f: impl Fn(Tensor, Tensor) -> Op) -> Self { + let op = if arg1.track_op() || arg2.track_op() { + Some(f(arg1.clone(), arg2.clone())) + } else { + None + }; + Self(op) + } + + pub(crate) fn new3( + arg1: &Tensor, + arg2: &Tensor, + arg3: &Tensor, + f: impl Fn(Tensor, Tensor, Tensor) -> Op, + ) -> Self { + let op = if arg1.track_op() || arg2.track_op() || arg3.track_op() { + Some(f(arg1.clone(), arg2.clone(), arg3.clone())) + } else { + None + }; + Self(op) + } + + pub(crate) fn new>(args: &[A], f: impl Fn(Vec) -> Op) -> Self { + let op = if args.iter().any(|arg| arg.as_ref().track_op()) { + let args: Vec = args.iter().map(|arg| arg.as_ref().clone()).collect(); + Some(f(args)) + } else { + None + }; + Self(op) + } + + pub(crate) fn is_none(&self) -> bool { + self.0.is_none() + } +} + +impl std::ops::Deref for BackpropOp { + type Target = Option; + fn deref(&self) -> &Self::Target { + &self.0 + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/pickle.rs b/v2/dependency-patches/candle-core-0.4.1/src/pickle.rs new file mode 100644 index 0000000000..7a4182e1ea --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/pickle.rs @@ -0,0 +1,817 @@ +// Just enough pickle support to be able to read PyTorch checkpoints. +// This hardcodes objects that are required for tensor reading, we may want to make this a bit more +// composable/tensor agnostic at some point. +use crate::{DType, Error as E, Layout, Result, Tensor}; +use byteorder::{LittleEndian, ReadBytesExt}; +use std::collections::HashMap; +use std::io::BufRead; + +const VERBOSE: bool = false; + +// https://docs.juliahub.com/Pickle/LAUNc/0.1.0/opcode/ +#[repr(u8)] +#[derive(Debug, Eq, PartialEq, Clone)] +pub enum OpCode { + // https://github.com/python/cpython/blob/ed25f097160b5cbb0c9a1f9a746d2f1bbc96515a/Lib/pickletools.py#L2123 + Proto = 0x80, + Global = b'c', + BinPut = b'q', + LongBinPut = b'r', + EmptyTuple = b')', + Reduce = b'R', + Mark = b'(', + BinUnicode = b'X', + BinInt = b'J', + Tuple = b't', + BinPersId = b'Q', + BinInt1 = b'K', + BinInt2 = b'M', + Tuple1 = 0x85, + Tuple2 = 0x86, + Tuple3 = 0x87, + NewTrue = 0x88, + NewFalse = 0x89, + None = b'N', + BinGet = b'h', + LongBinGet = b'j', + SetItem = b's', + SetItems = b'u', + EmptyDict = b'}', + Dict = b'd', + Build = b'b', + Stop = b'.', + NewObj = 0x81, + EmptyList = b']', + BinFloat = b'G', + Append = b'a', + Appends = b'e', +} + +// Avoid using FromPrimitive so as not to drag another dependency. +impl TryFrom for OpCode { + type Error = u8; + fn try_from(value: u8) -> std::result::Result { + match value { + 0x80 => Ok(Self::Proto), + b'c' => Ok(Self::Global), + b'q' => Ok(Self::BinPut), + b'r' => Ok(Self::LongBinPut), + b')' => Ok(Self::EmptyTuple), + b'R' => Ok(Self::Reduce), + b'(' => Ok(Self::Mark), + b'X' => Ok(Self::BinUnicode), + b'J' => Ok(Self::BinInt), + b't' => Ok(Self::Tuple), + b'Q' => Ok(Self::BinPersId), + b'K' => Ok(Self::BinInt1), + b'M' => Ok(Self::BinInt2), + b'N' => Ok(Self::None), + 0x85 => Ok(Self::Tuple1), + 0x86 => Ok(Self::Tuple2), + 0x87 => Ok(Self::Tuple3), + 0x88 => Ok(Self::NewTrue), + 0x89 => Ok(Self::NewFalse), + b'h' => Ok(Self::BinGet), + b'j' => Ok(Self::LongBinGet), + b's' => Ok(Self::SetItem), + b'u' => Ok(Self::SetItems), + b'}' => Ok(Self::EmptyDict), + b'd' => Ok(Self::EmptyDict), + b'b' => Ok(Self::Build), + b'.' => Ok(Self::Stop), + 0x81 => Ok(Self::NewObj), + b']' => Ok(Self::EmptyList), + b'G' => Ok(Self::BinFloat), + b'a' => Ok(Self::Append), + b'e' => Ok(Self::Appends), + value => Err(value), + } + } +} + +fn read_to_newline(r: &mut R) -> Result> { + let mut data: Vec = Vec::with_capacity(32); + r.read_until(b'\n', &mut data)?; + data.pop(); + if data.last() == Some(&b'\r') { + data.pop(); + } + Ok(data) +} + +#[derive(Debug, Clone, PartialEq)] +pub enum Object { + Class { + module_name: String, + class_name: String, + }, + Int(i32), + Float(f64), + Unicode(String), + Bool(bool), + None, + Tuple(Vec), + List(Vec), + Mark, + Dict(Vec<(Object, Object)>), + Reduce { + callable: Box, + args: Box, + }, + Build { + callable: Box, + args: Box, + }, + PersistentLoad(Box), +} + +type OResult = std::result::Result; + +impl Object { + pub fn unicode(self) -> OResult { + match self { + Self::Unicode(t) => Ok(t), + _ => Err(self), + } + } + + pub fn reduce(self) -> OResult<(Self, Self)> { + match self { + Self::Reduce { callable, args } => Ok((*callable, *args)), + _ => Err(self), + } + } + + pub fn none(self) -> OResult<()> { + match self { + Self::None => Ok(()), + _ => Err(self), + } + } + + pub fn persistent_load(self) -> OResult { + match self { + Self::PersistentLoad(t) => Ok(*t), + _ => Err(self), + } + } + + pub fn bool(self) -> OResult { + match self { + Self::Bool(t) => Ok(t), + _ => Err(self), + } + } + + pub fn int(self) -> OResult { + match self { + Self::Int(t) => Ok(t), + _ => Err(self), + } + } + + pub fn tuple(self) -> OResult> { + match self { + Self::Tuple(t) => Ok(t), + _ => Err(self), + } + } + + pub fn dict(self) -> OResult> { + match self { + Self::Dict(t) => Ok(t), + _ => Err(self), + } + } + + pub fn class(self) -> OResult<(String, String)> { + match self { + Self::Class { + module_name, + class_name, + } => Ok((module_name, class_name)), + _ => Err(self), + } + } + + pub fn into_tensor_info( + self, + name: Self, + dir_name: &std::path::Path, + ) -> Result> { + let name = match name.unicode() { + Ok(name) => name, + Err(_) => return Ok(None), + }; + let (callable, args) = match self.reduce() { + Ok(callable_args) => callable_args, + _ => return Ok(None), + }; + let (callable, args) = match callable { + Object::Class { + module_name, + class_name, + } if module_name == "torch._tensor" && class_name == "_rebuild_from_type_v2" => { + let mut args = args.tuple()?; + let callable = args.remove(0); + let args = args.remove(1); + (callable, args) + } + Object::Class { + module_name, + class_name, + } if module_name == "torch._utils" && class_name == "_rebuild_parameter" => { + let mut args = args.tuple()?; + args.remove(0).reduce()? + } + _ => (callable, args), + }; + match callable { + Object::Class { + module_name, + class_name, + } if module_name == "torch._utils" && class_name == "_rebuild_tensor_v2" => {} + _ => return Ok(None), + }; + let (layout, dtype, file_path, storage_size) = rebuild_args(args)?; + Ok(Some(TensorInfo { + name, + dtype, + layout, + path: format!("{}/{}", dir_name.to_string_lossy(), file_path), + storage_size, + })) + } +} + +impl TryFrom for String { + type Error = Object; + fn try_from(value: Object) -> std::result::Result { + match value { + Object::Unicode(s) => Ok(s), + other => Err(other), + } + } +} + +impl TryFrom for usize { + type Error = Object; + fn try_from(value: Object) -> std::result::Result { + match value { + Object::Int(s) if s >= 0 => Ok(s as usize), + other => Err(other), + } + } +} + +impl> TryFrom for Vec { + type Error = Object; + fn try_from(value: Object) -> std::result::Result { + match value { + Object::Tuple(values) => { + // This does not return the appropriate value in the error case but instead return + // the object related to the first error. + values + .into_iter() + .map(|v| T::try_from(v)) + .collect::, Self::Error>>() + } + other => Err(other), + } + } +} + +#[derive(Debug)] +pub struct Stack { + stack: Vec, + memo: HashMap, +} + +impl Stack { + pub fn empty() -> Self { + Self { + stack: Vec::with_capacity(512), + memo: HashMap::new(), + } + } + + pub fn stack(&self) -> &[Object] { + self.stack.as_slice() + } + + pub fn read_loop(&mut self, r: &mut R) -> Result<()> { + loop { + if self.read(r)? { + break; + } + } + Ok(()) + } + + pub fn finalize(mut self) -> Result { + self.pop() + } + + fn push(&mut self, obj: Object) { + self.stack.push(obj) + } + + fn pop(&mut self) -> Result { + match self.stack.pop() { + None => crate::bail!("unexpected empty stack"), + Some(obj) => Ok(obj), + } + } + + // https://docs.juliahub.com/Pickle/LAUNc/0.1.0/opcode/#Pickle.OpCodes.BUILD + fn build(&mut self) -> Result<()> { + let args = self.pop()?; + let obj = self.pop()?; + let obj = match (obj, args) { + (Object::Dict(mut obj), Object::Dict(mut args)) => { + obj.append(&mut args); + Object::Dict(obj) + } + (obj, args) => Object::Build { + callable: Box::new(obj), + args: Box::new(args), + }, + }; + self.push(obj); + Ok(()) + } + + fn reduce(&mut self) -> Result<()> { + let args = self.pop()?; + let callable = self.pop()?; + #[allow(clippy::single_match)] + let reduced = match &callable { + Object::Class { + module_name, + class_name, + } => { + if module_name == "collections" + && (class_name == "OrderedDict" || class_name == "defaultdict") + { + // TODO: have a separate ordered dict and a separate default dict. + Some(Object::Dict(vec![])) + } else { + None + } + } + _ => None, + }; + let reduced = reduced.unwrap_or_else(|| Object::Reduce { + callable: Box::new(callable), + args: Box::new(args), + }); + self.push(reduced); + Ok(()) + } + + fn last(&mut self) -> Result<&mut Object> { + match self.stack.last_mut() { + None => crate::bail!("unexpected empty stack"), + Some(obj) => Ok(obj), + } + } + + fn memo_get(&self, id: u32) -> Result { + match self.memo.get(&id) { + None => crate::bail!("missing object in memo {id}"), + Some(obj) => { + // Maybe we should use refcounting rather than doing potential large clones here. + Ok(obj.clone()) + } + } + } + + fn memo_put(&mut self, id: u32) -> Result<()> { + let obj = self.last()?.clone(); + self.memo.insert(id, obj); + Ok(()) + } + + fn persistent_load(&self, id: Object) -> Result { + Ok(Object::PersistentLoad(Box::new(id))) + } + + fn new_obj(&self, class: Object, args: Object) -> Result { + Ok(Object::Reduce { + callable: Box::new(class), + args: Box::new(args), + }) + } + + fn pop_to_marker(&mut self) -> Result> { + let mut mark_idx = None; + for (idx, obj) in self.stack.iter().enumerate().rev() { + if obj == &Object::Mark { + mark_idx = Some(idx); + break; + } + } + match mark_idx { + Some(mark_idx) => { + let objs = self.stack.split_off(mark_idx + 1); + self.stack.pop(); + Ok(objs) + } + None => { + crate::bail!("marker object not found") + } + } + } + + pub fn read(&mut self, r: &mut R) -> Result { + let op_code = match OpCode::try_from(r.read_u8()?) { + Ok(op_code) => op_code, + Err(op_code) => { + crate::bail!("unknown op-code {op_code}") + } + }; + // println!("op: {op_code:?}"); + // println!("{:?}", self.stack); + match op_code { + OpCode::Proto => { + let version = r.read_u8()?; + if VERBOSE { + println!("proto {version}"); + } + } + OpCode::Global => { + let module_name = read_to_newline(r)?; + let class_name = read_to_newline(r)?; + let module_name = String::from_utf8_lossy(&module_name).to_string(); + let class_name = String::from_utf8_lossy(&class_name).to_string(); + self.push(Object::Class { + module_name, + class_name, + }) + } + OpCode::BinInt1 => { + let arg = r.read_u8()?; + self.push(Object::Int(arg as i32)) + } + OpCode::BinInt2 => { + let arg = r.read_u16::()?; + self.push(Object::Int(arg as i32)) + } + OpCode::BinInt => { + let arg = r.read_i32::()?; + self.push(Object::Int(arg)) + } + OpCode::BinFloat => { + // Somehow floats are encoded using BigEndian whereas int types use LittleEndian. + // https://github.com/python/cpython/blob/0c80da4c14d904a367968955544dd6ae58c8101c/Lib/pickletools.py#L855 + // https://github.com/pytorch/pytorch/blob/372d078f361e726bb4ac0884ac334b04c58179ef/torch/_weights_only_unpickler.py#L243 + let arg = r.read_f64::()?; + self.push(Object::Float(arg)) + } + OpCode::BinUnicode => { + let len = r.read_u32::()?; + let mut data = vec![0u8; len as usize]; + r.read_exact(&mut data)?; + let data = String::from_utf8(data).map_err(E::wrap)?; + self.push(Object::Unicode(data)) + } + OpCode::BinPersId => { + let id = self.pop()?; + let obj = self.persistent_load(id)?; + self.push(obj) + } + OpCode::Tuple => { + let objs = self.pop_to_marker()?; + self.push(Object::Tuple(objs)) + } + OpCode::Tuple1 => { + let obj = self.pop()?; + self.push(Object::Tuple(vec![obj])) + } + OpCode::Tuple2 => { + let obj2 = self.pop()?; + let obj1 = self.pop()?; + self.push(Object::Tuple(vec![obj1, obj2])) + } + OpCode::Tuple3 => { + let obj3 = self.pop()?; + let obj2 = self.pop()?; + let obj1 = self.pop()?; + self.push(Object::Tuple(vec![obj1, obj2, obj3])) + } + OpCode::NewTrue => self.push(Object::Bool(true)), + OpCode::NewFalse => self.push(Object::Bool(false)), + OpCode::Append => { + let value = self.pop()?; + let pylist = self.last()?; + if let Object::List(d) = pylist { + d.push(value) + } else { + crate::bail!("expected a list, got {pylist:?}") + } + } + OpCode::Appends => { + let objs = self.pop_to_marker()?; + let pylist = self.last()?; + if let Object::List(d) = pylist { + d.extend(objs) + } else { + crate::bail!("expected a list, got {pylist:?}") + } + } + OpCode::SetItem => { + let value = self.pop()?; + let key = self.pop()?; + let pydict = self.last()?; + if let Object::Dict(d) = pydict { + d.push((key, value)) + } else { + crate::bail!("expected a dict, got {pydict:?}") + } + } + OpCode::SetItems => { + let mut objs = self.pop_to_marker()?; + let pydict = self.last()?; + if let Object::Dict(d) = pydict { + if objs.len() % 2 != 0 { + crate::bail!("setitems: not an even number of objects") + } + while let Some(value) = objs.pop() { + let key = objs.pop().unwrap(); + d.push((key, value)) + } + } else { + crate::bail!("expected a dict, got {pydict:?}") + } + } + OpCode::None => self.push(Object::None), + OpCode::Stop => { + return Ok(true); + } + OpCode::Build => self.build()?, + OpCode::EmptyDict => self.push(Object::Dict(vec![])), + OpCode::Dict => { + let mut objs = self.pop_to_marker()?; + let mut pydict = vec![]; + if objs.len() % 2 != 0 { + crate::bail!("setitems: not an even number of objects") + } + while let Some(value) = objs.pop() { + let key = objs.pop().unwrap(); + pydict.push((key, value)) + } + self.push(Object::Dict(pydict)) + } + OpCode::Mark => self.push(Object::Mark), + OpCode::Reduce => self.reduce()?, + OpCode::EmptyTuple => self.push(Object::Tuple(vec![])), + OpCode::EmptyList => self.push(Object::List(vec![])), + OpCode::BinGet => { + let arg = r.read_u8()?; + let obj = self.memo_get(arg as u32)?; + self.push(obj) + } + OpCode::LongBinGet => { + let arg = r.read_u32::()?; + let obj = self.memo_get(arg)?; + self.push(obj) + } + OpCode::BinPut => { + let arg = r.read_u8()?; + self.memo_put(arg as u32)? + } + OpCode::LongBinPut => { + let arg = r.read_u32::()?; + self.memo_put(arg)? + } + OpCode::NewObj => { + let args = self.pop()?; + let class = self.pop()?; + let obj = self.new_obj(class, args)?; + self.push(obj) + } + } + Ok(false) + } +} + +impl From for E { + fn from(value: Object) -> Self { + E::Msg(format!("conversion error on {value:?}")) + } +} + +// https://github.com/pytorch/pytorch/blob/4eac43d046ded0f0a5a5fa8db03eb40f45bf656e/torch/_utils.py#L198 +// Arguments: storage, storage_offset, size, stride, requires_grad, backward_hooks +fn rebuild_args(args: Object) -> Result<(Layout, DType, String, usize)> { + let mut args = args.tuple()?; + let stride = Vec::::try_from(args.remove(3))?; + let size = Vec::::try_from(args.remove(2))?; + let offset = args.remove(1).int()? as usize; + let storage = args.remove(0).persistent_load()?; + let mut storage = storage.tuple()?; + let storage_size = storage.remove(4).int()? as usize; + let path = storage.remove(2).unicode()?; + let (_module_name, class_name) = storage.remove(1).class()?; + let dtype = match class_name.as_str() { + "FloatStorage" => DType::F32, + "DoubleStorage" => DType::F64, + "HalfStorage" => DType::F16, + "BFloat16Storage" => DType::BF16, + "ByteStorage" => DType::U8, + "LongStorage" => DType::I64, + other => { + crate::bail!("unsupported storage type {other}") + } + }; + let layout = Layout::new(crate::Shape::from(size), stride, offset); + Ok((layout, dtype, path, storage_size)) +} + +#[derive(Debug, Clone)] +pub struct TensorInfo { + pub name: String, + pub dtype: DType, + pub layout: Layout, + pub path: String, + pub storage_size: usize, +} + +/// Read the tensor info from a .pth file. +/// +/// # Arguments +/// * `file` - The path to the .pth file. +/// * `verbose` - Whether to print debug information. +/// * `key` - Optional key to retrieve `state_dict` from the pth file. +pub fn read_pth_tensor_info>( + file: P, + verbose: bool, + key: Option<&str>, +) -> Result> { + let file = std::fs::File::open(file)?; + let zip_reader = std::io::BufReader::new(file); + let mut zip = zip::ZipArchive::new(zip_reader)?; + let zip_file_names = zip + .file_names() + .map(|name| name.map(|name| name.into_owned())) + .collect::>>()?; + + let mut tensor_infos = vec![]; + for file_name in zip_file_names.iter() { + if !file_name.ends_with("data.pkl") { + continue; + } + let dir_name = std::path::PathBuf::from(file_name.strip_suffix(".pkl").unwrap()); + let reader = zip.by_name(file_name)?; + let mut reader = std::io::BufReader::new(reader); + let mut stack = Stack::empty(); + stack.read_loop(&mut reader)?; + let obj = stack.finalize()?; + if VERBOSE || verbose { + println!("{obj:#?}"); + } + + let obj = match obj { + Object::Build { callable, args } => match *callable { + Object::Reduce { callable, args: _ } => match *callable { + Object::Class { + module_name, + class_name, + } if module_name == "__torch__" && class_name == "Module" => *args, + _ => continue, + }, + _ => continue, + }, + obj => obj, + }; + + // If key is provided, then we need to extract the state_dict from the object. + let obj = if let Some(key) = key { + if let Object::Dict(key_values) = obj { + key_values + .into_iter() + .find(|(k, _)| *k == Object::Unicode(key.to_owned())) + .map(|(_, v)| v) + .ok_or_else(|| E::Msg(format!("key {key} not found")))? + } else { + obj + } + } else { + obj + }; + + // If the object is a dict, then we can extract the tensor info from it. + // NOTE: We are assuming that the `obj` is state_dict by this stage. + if let Object::Dict(key_values) = obj { + for (name, value) in key_values.into_iter() { + match value.into_tensor_info(name, &dir_name) { + Ok(Some(tensor_info)) => tensor_infos.push(tensor_info), + Ok(None) => {} + Err(err) => eprintln!("skipping: {err:?}"), + } + } + } + } + Ok(tensor_infos) +} + +/// Lazy tensor loader. +pub struct PthTensors { + tensor_infos: HashMap, + path: std::path::PathBuf, + // We do not store a zip reader as it needs mutable access to extract data. Instead we + // re-create a zip reader for each tensor. +} + +impl PthTensors { + pub fn new>(path: P, key: Option<&str>) -> Result { + let tensor_infos = read_pth_tensor_info(path.as_ref(), false, key)?; + let tensor_infos = tensor_infos + .into_iter() + .map(|ti| (ti.name.to_string(), ti)) + .collect(); + let path = path.as_ref().to_owned(); + Ok(Self { tensor_infos, path }) + } + + pub fn tensor_infos(&self) -> &HashMap { + &self.tensor_infos + } + + pub fn get(&self, name: &str) -> Result> { + use std::io::Read; + let tensor_info = match self.tensor_infos.get(name) { + None => return Ok(None), + Some(tensor_info) => tensor_info, + }; + // We hope that the file has not changed since first reading it. + let zip_reader = std::io::BufReader::new(std::fs::File::open(&self.path)?); + let mut zip = zip::ZipArchive::new(zip_reader)?; + let mut reader = zip.by_name(&tensor_info.path)?; + let is_fortran_contiguous = tensor_info.layout.is_fortran_contiguous(); + let rank = tensor_info.layout.shape().rank(); + + // Reading the data is a bit tricky as it can be strided, for now only support the basic + // case and when the tensor is fortran contiguous. + if !tensor_info.layout.is_contiguous() && !is_fortran_contiguous { + crate::bail!( + "cannot retrieve non-contiguous tensors {:?}", + tensor_info.layout + ) + } + let start_offset = tensor_info.layout.start_offset(); + if start_offset > 0 { + std::io::copy( + &mut reader.by_ref().take(start_offset as u64), + &mut std::io::sink(), + )?; + } + let tensor = Tensor::from_reader( + tensor_info.layout.shape().clone(), + tensor_info.dtype, + &mut reader, + )?; + + if rank > 1 && is_fortran_contiguous { + // Reverse the shape, e.g. Shape(2, 3, 4) -> Shape(4, 3, 2) + let shape_reversed: Vec<_> = tensor_info.layout.dims().iter().rev().cloned().collect(); + let tensor = tensor.reshape(shape_reversed)?; + + // Permute (transpose) the dimensions, e.g. Shape(4, 3, 2) -> Shape(2, 3, 4) + let dim_indeces_reversed: Vec<_> = (0..rank).rev().collect(); + let tensor = tensor.permute(dim_indeces_reversed)?; + Ok(Some(tensor)) + } else { + Ok(Some(tensor)) + } + } +} + +/// Read all the tensors from a PyTorch pth file with a given key. +/// +/// # Arguments +/// * `path` - Path to the pth file. +/// * `key` - Optional key to retrieve `state_dict` from the pth file. Sometimes the pth file +/// contains multiple objects and the state_dict is the one we are interested in. +pub fn read_all_with_key>( + path: P, + key: Option<&str>, +) -> Result> { + let pth = PthTensors::new(path, key)?; + let tensor_names = pth.tensor_infos.keys(); + let mut tensors = Vec::with_capacity(tensor_names.len()); + for name in tensor_names { + if let Some(tensor) = pth.get(name)? { + tensors.push((name.to_string(), tensor)) + } + } + Ok(tensors) +} + +/// Read all the tensors from a PyTorch pth file. +/// +/// # Arguments +/// * `path` - Path to the pth file. +pub fn read_all>(path: P) -> Result> { + read_all_with_key(path, None) +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/avx.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/avx.rs new file mode 100644 index 0000000000..664f7653ee --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/quantized/avx.rs @@ -0,0 +1,667 @@ +use super::k_quants::{ + BlockQ2K, BlockQ3K, BlockQ4K, BlockQ4_0, BlockQ5K, BlockQ6K, BlockQ8K, BlockQ8_0, QK8_0, QK_K, +}; +use crate::Result; +use byteorder::{ByteOrder, LittleEndian}; +use half::f16; + +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + +#[inline(always)] +pub(crate) unsafe fn sum_i16_pairs_float(x: __m256i) -> __m256 { + let ones = _mm256_set1_epi16(1); + let summed_pairs = _mm256_madd_epi16(ones, x); + _mm256_cvtepi32_ps(summed_pairs) +} + +#[inline(always)] +pub(crate) unsafe fn mul_sum_us8_pairs_float(ax: __m256i, sy: __m256i) -> __m256 { + let dot = _mm256_maddubs_epi16(ax, sy); + sum_i16_pairs_float(dot) +} + +#[inline(always)] +pub(crate) unsafe fn hsum_float_8(x: __m256) -> f32 { + let res = _mm256_extractf128_ps(x, 1); + let res = _mm_add_ps(res, _mm256_castps256_ps128(x)); + let res = _mm_add_ps(res, _mm_movehl_ps(res, res)); + let res = _mm_add_ss(res, _mm_movehdup_ps(res)); + _mm_cvtss_f32(res) +} + +#[inline(always)] +pub(crate) unsafe fn bytes_from_nibbles_32(rsi: *const u8) -> __m256i { + let tmp = _mm_loadu_si128(rsi as *const __m128i); + let bytes = _mm256_insertf128_si256::<1>(_mm256_castsi128_si256(tmp), _mm_srli_epi16(tmp, 4)); + let low_mask = _mm256_set1_epi8(0xF); + _mm256_and_si256(low_mask, bytes) +} + +#[inline(always)] +pub(crate) unsafe fn mul_sum_i8_pairs_float(x: __m256i, y: __m256i) -> __m256 { + let ax = _mm256_sign_epi8(x, x); + let sy = _mm256_sign_epi8(y, x); + mul_sum_us8_pairs_float(ax, sy) +} + +#[inline(always)] +pub(crate) fn vec_dot_q4_0_q8_0(n: usize, xs: &[BlockQ4_0], ys: &[BlockQ8_0]) -> Result { + let qk = QK8_0; + if n % QK8_0 != 0 { + crate::bail!("vec_dot_q4_0_q8_0: {n} is not divisible by {qk}") + } + unsafe { + let mut acc = _mm256_setzero_ps(); + for (x, y) in xs.iter().zip(ys.iter()) { + let d = _mm256_set1_ps(f16::to_f32(x.d) * f16::to_f32(y.d)); + let bx = bytes_from_nibbles_32(x.qs.as_ptr()); + let off = _mm256_set1_epi8(8); + let bx = _mm256_sub_epi8(bx, off); + let by = _mm256_loadu_si256(y.qs.as_ptr() as *const __m256i); + let q = mul_sum_i8_pairs_float(bx, by); + acc = _mm256_fmadd_ps(d, q, acc); + } + Ok(hsum_float_8(acc)) + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q8_0_q8_0(n: usize, xs: &[BlockQ8_0], ys: &[BlockQ8_0]) -> Result { + let qk = QK8_0; + if n % QK8_0 != 0 { + crate::bail!("vec_dot_q8_0_q8_0: {n} is not divisible by {qk}") + } + unsafe { + let mut acc = _mm256_setzero_ps(); + for (x, y) in xs.iter().zip(ys.iter()) { + let d = _mm256_set1_ps(f16::to_f32(x.d) * f16::to_f32(y.d)); + let bx = _mm256_loadu_si256(x.qs.as_ptr() as *const __m256i); + let by = _mm256_loadu_si256(y.qs.as_ptr() as *const __m256i); + let q = mul_sum_i8_pairs_float(bx, by); + acc = _mm256_fmadd_ps(d, q, acc); + } + Ok(hsum_float_8(acc)) + } +} + +#[inline(always)] +unsafe fn get_scale_shuffle(i: usize) -> __m128i { + const K_SHUFFLE: [u8; 128] = [ + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, + 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, + 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, + 11, 11, 11, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 13, 13, 13, 13, 13, 13, 13, + 13, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, + ]; + _mm_loadu_si128((K_SHUFFLE.as_ptr() as *const __m128i).add(i)) +} + +#[inline(always)] +unsafe fn get_scale_shuffle_k4(i: usize) -> __m256i { + const K_SHUFFLE: [u8; 256] = [ + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, + 2, 3, 2, 3, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, + 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, + 6, 7, 6, 7, 6, 7, 6, 7, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, + 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, + 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 12, 13, 12, 13, 12, 13, + 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, + 13, 12, 13, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, + 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, + ]; + _mm256_loadu_si256((K_SHUFFLE.as_ptr() as *const __m256i).add(i)) +} + +#[inline(always)] +unsafe fn get_scale_shuffle_q3k(i: usize) -> __m256i { + const K_SHUFFLE: [u8; 128] = [ + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, + 2, 3, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, + 6, 7, 6, 7, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 10, 11, 10, 11, 10, 11, 10, 11, + 10, 11, 10, 11, 10, 11, 10, 11, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, + 13, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, + ]; + _mm256_loadu_si256((K_SHUFFLE.as_ptr() as *const __m256i).add(i)) +} + +#[inline(always)] +pub(crate) fn vec_dot_q6k_q8k(n: usize, xs: &[BlockQ6K], ys: &[BlockQ8K]) -> Result { + let qk = QK_K; + if n % qk != 0 { + crate::bail!("vec_dot_q6k_8k: {n} is not divisible by {qk}") + } + + unsafe { + let m4 = _mm256_set1_epi8(0xF); + let m2 = _mm256_set1_epi8(3); + let m32s = _mm256_set1_epi8(32); + let mut acc = _mm256_setzero_ps(); + for (x, y) in xs.iter().zip(ys.iter()) { + let d = y.d * x.d.to_f32(); + let mut q4 = x.ql.as_ptr(); + let mut qh = x.qh.as_ptr(); + let mut q8 = y.qs.as_ptr(); + + let scales = _mm_loadu_si128(x.scales.as_ptr() as *const __m128i); + let mut sumi = _mm256_setzero_si256(); + + for j in 0..QK_K / 128 { + let is = j * 4; + let scale_0 = _mm_shuffle_epi8(scales, get_scale_shuffle(is)); + let scale_1 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 1)); + let scale_2 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 2)); + let scale_3 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 3)); + + let q4bits1 = _mm256_loadu_si256(q4 as *const __m256i); + q4 = q4.add(32); + let q4bits2 = _mm256_loadu_si256(q4 as *const __m256i); + q4 = q4.add(32); + let q4bits_h = _mm256_loadu_si256(qh as *const __m256i); + qh = qh.add(32); + + let q4h_0 = _mm256_slli_epi16(_mm256_and_si256(q4bits_h, m2), 4); + let q4h_1 = + _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bits_h, 2), m2), 4); + let q4h_2 = + _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bits_h, 4), m2), 4); + let q4h_3 = + _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bits_h, 6), m2), 4); + + let q4_0 = _mm256_or_si256(_mm256_and_si256(q4bits1, m4), q4h_0); + let q4_1 = _mm256_or_si256(_mm256_and_si256(q4bits2, m4), q4h_1); + let q4_2 = + _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits1, 4), m4), q4h_2); + let q4_3 = + _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits2, 4), m4), q4h_3); + + let q8_0 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let q8_1 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let q8_2 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let q8_3 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + + let q8s_0 = _mm256_maddubs_epi16(m32s, q8_0); + let q8s_1 = _mm256_maddubs_epi16(m32s, q8_1); + let q8s_2 = _mm256_maddubs_epi16(m32s, q8_2); + let q8s_3 = _mm256_maddubs_epi16(m32s, q8_3); + + let p16_0 = _mm256_maddubs_epi16(q4_0, q8_0); + let p16_1 = _mm256_maddubs_epi16(q4_1, q8_1); + let p16_2 = _mm256_maddubs_epi16(q4_2, q8_2); + let p16_3 = _mm256_maddubs_epi16(q4_3, q8_3); + + let p16_0 = _mm256_sub_epi16(p16_0, q8s_0); + let p16_1 = _mm256_sub_epi16(p16_1, q8s_1); + let p16_2 = _mm256_sub_epi16(p16_2, q8s_2); + let p16_3 = _mm256_sub_epi16(p16_3, q8s_3); + + let p16_0 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_0), p16_0); + let p16_1 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_1), p16_1); + let p16_2 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_2), p16_2); + let p16_3 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_3), p16_3); + + sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1)); + sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_2, p16_3)); + } + acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc); + } + Ok(hsum_float_8(acc)) + } +} + +#[inline(always)] +unsafe fn mm256_set_m128i(a: __m128i, b: __m128i) -> __m256i { + _mm256_insertf128_si256(_mm256_castsi128_si256(b), a, 1) +} + +#[inline(always)] +pub(crate) fn vec_dot_q2k_q8k(n: usize, xs: &[BlockQ2K], ys: &[BlockQ8K]) -> Result { + if n % QK_K != 0 { + crate::bail!("vec_dot_q2k_q8k: {n} is not divisible by {QK_K}") + } + + unsafe { + let m3 = _mm256_set1_epi8(3); + let m4 = _mm_set1_epi8(0xF); + + let mut acc = _mm256_setzero_ps(); + + for (x, y) in xs.iter().zip(ys.iter()) { + let d = y.d * x.d.to_f32(); + let dmin = -y.d * x.dmin.to_f32(); + + let mut q2 = x.qs.as_ptr(); + let mut q8 = y.qs.as_ptr(); + + let mins_and_scales = _mm_loadu_si128(x.scales.as_ptr() as *const __m128i); + let scales8 = _mm_and_si128(mins_and_scales, m4); + let mins8 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4); + let mins = _mm256_cvtepi8_epi16(mins8); + let prod = + _mm256_madd_epi16(mins, _mm256_loadu_si256(y.bsums.as_ptr() as *const __m256i)); + + acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&dmin), _mm256_cvtepi32_ps(prod), acc); + + let all_scales = _mm256_cvtepi8_epi16(scales8); + let l_scales = _mm256_extracti128_si256(all_scales, 0); + let h_scales = _mm256_extracti128_si256(all_scales, 1); + let scales = [ + mm256_set_m128i(l_scales, l_scales), + mm256_set_m128i(h_scales, h_scales), + ]; + + let mut sumi = _mm256_setzero_si256(); + + for scale in scales { + let q2bits = _mm256_loadu_si256(q2 as *const __m256i); + q2 = q2.add(32); + + let q8_0 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let q8_1 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let q8_2 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let q8_3 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + + let q2_0 = _mm256_and_si256(q2bits, m3); + let q2_1 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 2), m3); + let q2_2 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 4), m3); + let q2_3 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 6), m3); + + let p0 = _mm256_maddubs_epi16(q2_0, q8_0); + let p1 = _mm256_maddubs_epi16(q2_1, q8_1); + let p2 = _mm256_maddubs_epi16(q2_2, q8_2); + let p3 = _mm256_maddubs_epi16(q2_3, q8_3); + + let p0 = + _mm256_madd_epi16(_mm256_shuffle_epi8(scale, get_scale_shuffle_q3k(0)), p0); + let p1 = + _mm256_madd_epi16(_mm256_shuffle_epi8(scale, get_scale_shuffle_q3k(1)), p1); + let p2 = + _mm256_madd_epi16(_mm256_shuffle_epi8(scale, get_scale_shuffle_q3k(2)), p2); + let p3 = + _mm256_madd_epi16(_mm256_shuffle_epi8(scale, get_scale_shuffle_q3k(3)), p3); + + let p0 = _mm256_add_epi32(p0, p1); + let p2 = _mm256_add_epi32(p2, p3); + + sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p0, p2)); + } + acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc); + } + + Ok(hsum_float_8(acc)) + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q3k_q8k(n: usize, xs: &[BlockQ3K], ys: &[BlockQ8K]) -> Result { + if n % QK_K != 0 { + crate::bail!("vec_dot_q3k_q8k: {n} is not divisible by {QK_K}") + } + + const KMASK1: u32 = 0x03030303; + const KMASK2: u32 = 0x0f0f0f0f; + + let mut aux = [0u32; 3]; + + unsafe { + let m3 = _mm256_set1_epi8(3); + let mone = _mm256_set1_epi8(1); + let m32 = _mm_set1_epi8(32); + + let mut acc = _mm256_setzero_ps(); + for (x, y) in xs.iter().zip(ys.iter()) { + let d = y.d * x.d.to_f32(); + + let mut q3 = x.qs.as_ptr(); + let mut q8 = y.qs.as_ptr(); + + LittleEndian::read_u32_into(&x.scales, &mut aux); + let scales128 = _mm_set_epi32( + (((aux[1] >> 4) & KMASK2) | (((aux[2] >> 6) & KMASK1) << 4)) as i32, + (((aux[0] >> 4) & KMASK2) | (((aux[2] >> 4) & KMASK1) << 4)) as i32, + ((aux[1] & KMASK2) | (((aux[2] >> 2) & KMASK1) << 4)) as i32, + ((aux[0] & KMASK2) | (((aux[2]) & KMASK1) << 4)) as i32, + ); + let scales128 = _mm_sub_epi8(scales128, m32); + let all_scales = _mm256_cvtepi8_epi16(scales128); + let l_scales = _mm256_extracti128_si256(all_scales, 0); + let h_scales = _mm256_extracti128_si256(all_scales, 1); + let scales = [ + mm256_set_m128i(l_scales, l_scales), + mm256_set_m128i(h_scales, h_scales), + ]; + + // high bit + let hbits = _mm256_loadu_si256(x.hmask.as_ptr() as *const __m256i); + + let mut sumi = _mm256_setzero_si256(); + + for (j, scale) in scales.iter().enumerate() { + // load low 2 bits + let q3bits = _mm256_loadu_si256(q3 as *const __m256i); + q3 = q3.add(32); + + // Prepare low and high bits + // We hardcode the shifts here to avoid loading them into a separate register + let q3l_0 = _mm256_and_si256(q3bits, m3); + let q3h_0 = if j == 0 { + _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 0)), 0) + } else { + _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 4)), 4) + }; + let q3h_0 = _mm256_slli_epi16(q3h_0, 2); + + let q3l_1 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 2), m3); + let q3h_1 = if j == 0 { + _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 1)), 1) + } else { + _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 5)), 5) + }; + let q3h_1 = _mm256_slli_epi16(q3h_1, 2); + + let q3l_2 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 4), m3); + let q3h_2 = if j == 0 { + _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 2)), 2) + } else { + _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 6)), 6) + }; + let q3h_2 = _mm256_slli_epi16(q3h_2, 2); + + let q3l_3 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 6), m3); + let q3h_3 = if j == 0 { + _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 3)), 3) + } else { + _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 7)), 7) + }; + let q3h_3 = _mm256_slli_epi16(q3h_3, 2); + + // load Q8 quants + let q8_0 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let q8_1 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let q8_2 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let q8_3 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + + // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we + // can use _mm256_maddubs_epi16, and then subtract. The high bit part has the 2 + // already subtracted (and so, it is zero if the high bit was not set, and 2 if the + // high bit was set) + let q8s_0 = _mm256_maddubs_epi16(q3h_0, q8_0); + let q8s_1 = _mm256_maddubs_epi16(q3h_1, q8_1); + let q8s_2 = _mm256_maddubs_epi16(q3h_2, q8_2); + let q8s_3 = _mm256_maddubs_epi16(q3h_3, q8_3); + + let p16_0 = _mm256_maddubs_epi16(q3l_0, q8_0); + let p16_1 = _mm256_maddubs_epi16(q3l_1, q8_1); + let p16_2 = _mm256_maddubs_epi16(q3l_2, q8_2); + let p16_3 = _mm256_maddubs_epi16(q3l_3, q8_3); + + let p16_0 = _mm256_sub_epi16(p16_0, q8s_0); + let p16_1 = _mm256_sub_epi16(p16_1, q8s_1); + let p16_2 = _mm256_sub_epi16(p16_2, q8s_2); + let p16_3 = _mm256_sub_epi16(p16_3, q8s_3); + + // multiply with scales + let p16_0 = + _mm256_madd_epi16(_mm256_shuffle_epi8(*scale, get_scale_shuffle_q3k(0)), p16_0); + let p16_1 = + _mm256_madd_epi16(_mm256_shuffle_epi8(*scale, get_scale_shuffle_q3k(1)), p16_1); + let p16_2 = + _mm256_madd_epi16(_mm256_shuffle_epi8(*scale, get_scale_shuffle_q3k(2)), p16_2); + let p16_3 = + _mm256_madd_epi16(_mm256_shuffle_epi8(*scale, get_scale_shuffle_q3k(3)), p16_3); + + // accumulate + let p16_0 = _mm256_add_epi32(p16_0, p16_1); + let p16_2 = _mm256_add_epi32(p16_2, p16_3); + sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_2)); + } + + // multiply with block scale and accumulate + acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc); + } + Ok(hsum_float_8(acc)) + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q4k_q8k(n: usize, xs: &[BlockQ4K], ys: &[BlockQ8K]) -> Result { + if n % QK_K != 0 { + crate::bail!("vec_dot_q4k_q8k: {n} is not divisible by {QK_K}") + } + let mut utmp = [0u32; 4]; + const KMASK1: u32 = 0x3f3f3f3f; + const KMASK2: u32 = 0x0f0f0f0f; + const KMASK3: u32 = 0x03030303; + + unsafe { + let m4 = _mm256_set1_epi8(0xF); + + let mut acc = _mm256_setzero_ps(); + let mut acc_m = _mm_setzero_ps(); + + for (x, y) in xs.iter().zip(ys.iter()) { + let d = y.d * x.d.to_f32(); + let dmin = -y.d * x.dmin.to_f32(); + + LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); + + utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); + let uaux = utmp[1] & KMASK1; + utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); + utmp[2] = uaux; + utmp[0] &= KMASK1; + + let mut q4 = x.qs.as_ptr(); + let mut q8 = y.qs.as_ptr(); + + let mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32( + utmp[3] as i32, + utmp[2] as i32, + utmp[1] as i32, + utmp[0] as i32, + )); + + let q8sums = _mm256_loadu_si256(y.bsums.as_ptr() as *const __m256i); + let q8s = _mm_hadd_epi16( + _mm256_extracti128_si256(q8sums, 0), + _mm256_extracti128_si256(q8sums, 1), + ); + let prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s); + acc_m = _mm_fmadd_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod), acc_m); + + let sc128 = _mm256_extracti128_si256(mins_and_scales, 0); + let scales = mm256_set_m128i(sc128, sc128); + + let mut sumi = _mm256_setzero_si256(); + + for j in 0..QK_K / 64 { + let scale_l = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2 * j)); + let scale_h = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2 * j + 1)); + + let q4bits = _mm256_loadu_si256(q4 as *const __m256i); + q4 = q4.add(32); + let q4l = _mm256_and_si256(q4bits, m4); + let q4h = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), m4); + + let q8l = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let p16l = _mm256_maddubs_epi16(q4l, q8l); + let p16l = _mm256_madd_epi16(scale_l, p16l); + sumi = _mm256_add_epi32(sumi, p16l); + + let q8h = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let p16h = _mm256_maddubs_epi16(q4h, q8h); + let p16h = _mm256_madd_epi16(scale_h, p16h); + sumi = _mm256_add_epi32(sumi, p16h); + } + + let vd = _mm256_set1_ps(d); + acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc); + } + + let acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m)); + let acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m)); + + Ok(hsum_float_8(acc) + _mm_cvtss_f32(acc_m)) + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q5k_q8k(n: usize, xs: &[BlockQ5K], ys: &[BlockQ8K]) -> Result { + if n % QK_K != 0 { + crate::bail!("vec_dot_q5k_q8k: {n} is not divisible by {QK_K}") + } + let mut utmp = [0u32; 4]; + const KMASK1: u32 = 0x3f3f3f3f; + const KMASK2: u32 = 0x0f0f0f0f; + const KMASK3: u32 = 0x03030303; + + unsafe { + let m4 = _mm256_set1_epi8(0xF); + let mzero = _mm_setzero_si128(); + let mone = _mm256_set1_epi8(1); + + let mut acc = _mm256_setzero_ps(); + let mut summs = 0.0; + + for (x, y) in xs.iter().zip(ys.iter()) { + let d = y.d * x.d.to_f32(); + let dmin = -y.d * x.dmin.to_f32(); + + LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); + + utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); + let uaux = utmp[1] & KMASK1; + utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); + utmp[2] = uaux; + utmp[0] &= KMASK1; + + let mut q5 = x.qs.as_ptr(); + let mut q8 = y.qs.as_ptr(); + + let mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32( + utmp[3] as i32, + utmp[2] as i32, + utmp[1] as i32, + utmp[0] as i32, + )); + + let q8sums = _mm256_loadu_si256(y.bsums.as_ptr() as *const __m256i); + let q8s = _mm_hadd_epi16( + _mm256_extracti128_si256(q8sums, 0), + _mm256_extracti128_si256(q8sums, 1), + ); + let prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s); + let hsum = _mm_hadd_epi32(_mm_hadd_epi32(prod, mzero), mzero); + summs += dmin * _mm_extract_epi32(hsum, 0) as f32; + + let sc128 = _mm256_extracti128_si256(mins_and_scales, 0); + let scales = mm256_set_m128i(sc128, sc128); + + let hbits = _mm256_loadu_si256(x.qh.as_ptr() as *const __m256i); + let mut hmask = mone; + + let mut sumi = _mm256_setzero_si256(); + + for j in 0..QK_K / 64 { + let scale_0 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2 * j)); + let scale_1 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2 * j + 1)); + + let q5bits = _mm256_loadu_si256(q5 as *const __m256i); + q5 = q5.add(32); + + //Similar to q3k we hardcode the shifts here to avoid loading them into a separate register + let q5l_0 = _mm256_and_si256(q5bits, m4); + let q5l_0_shift_input = _mm256_and_si256(hbits, hmask); + let q5l_0_right_shift = match j { + 0 => _mm256_srli_epi16(q5l_0_shift_input, 0), + 1 => _mm256_srli_epi16(q5l_0_shift_input, 2), + 2 => _mm256_srli_epi16(q5l_0_shift_input, 4), + 3 => _mm256_srli_epi16(q5l_0_shift_input, 6), + _ => unreachable!(), + }; + let q5h_0 = _mm256_slli_epi16(q5l_0_right_shift, 4); + let q5_0 = _mm256_add_epi8(q5l_0, q5h_0); + hmask = _mm256_slli_epi16(hmask, 1); + + let q5l_1 = _mm256_and_si256(_mm256_srli_epi16(q5bits, 4), m4); + let q5l_1_shift_input = _mm256_and_si256(hbits, hmask); + let q5l_1_right_shift = match j { + 0 => _mm256_srli_epi16(q5l_1_shift_input, 1), + 1 => _mm256_srli_epi16(q5l_1_shift_input, 3), + 2 => _mm256_srli_epi16(q5l_1_shift_input, 5), + 3 => _mm256_srli_epi16(q5l_1_shift_input, 7), + _ => unreachable!(), + }; + + let q5h_1 = _mm256_slli_epi16(q5l_1_right_shift, 4); + let q5_1 = _mm256_add_epi8(q5l_1, q5h_1); + hmask = _mm256_slli_epi16(hmask, 1); + + let q8_0 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let q8_1 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + + let p16_0 = _mm256_maddubs_epi16(q5_0, q8_0); + let p16_1 = _mm256_maddubs_epi16(q5_1, q8_1); + + let p16_0 = _mm256_madd_epi16(scale_0, p16_0); + let p16_1 = _mm256_madd_epi16(scale_1, p16_1); + + sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1)); + } + let vd = _mm256_set1_ps(d); + acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc); + } + Ok(hsum_float_8(acc) + summs) + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q8k_q8k(n: usize, xs: &[BlockQ8K], ys: &[BlockQ8K]) -> Result { + let qk = QK_K; + if n % qk != 0 { + crate::bail!("vec_dot_q8k_8k: {n} is not divisible by {qk}") + } + + unsafe { + let mut acc = _mm256_setzero_ps(); + for (xs, ys) in xs.iter().zip(ys.iter()) { + let mut sumi = _mm256_setzero_si256(); + let x_qs = xs.qs.as_ptr(); + let y_qs = ys.qs.as_ptr(); + for j in (0..QK_K).step_by(32) { + let xs = _mm256_loadu_si256(x_qs.add(j) as *const __m256i); + let ys = _mm256_loadu_si256(y_qs.add(j) as *const __m256i); + + let xs0 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(xs, 0)); + let ys0 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(ys, 0)); + sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(xs0, ys0)); + + let xs1 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(xs, 1)); + let ys1 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(ys, 1)); + sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(xs1, ys1)); + } + let d = _mm256_set1_ps(xs.d * ys.d); + acc = _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(sumi), acc); + } + Ok(hsum_float_8(acc)) + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/cuda.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/cuda.rs new file mode 100644 index 0000000000..84af483d32 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/quantized/cuda.rs @@ -0,0 +1,321 @@ +use super::{GgmlDType, QStorage}; +use crate::{backend::BackendDevice, cuda_backend::WrapErr}; +use crate::{CudaDevice, CudaStorage, Result}; + +use cudarc::driver::{CudaSlice, DeviceSlice}; + +pub struct QCudaStorage { + data: CudaSlice, + dtype: GgmlDType, + device: CudaDevice, +} + +pub const WARP_SIZE: usize = 32; +pub const MMQ_X_Q4_0_AMPERE: usize = 4; +pub const MMQ_Y_Q4_0_AMPERE: usize = 32; +pub const NWARPS_Q4_0_AMPERE: usize = 4; +pub const GGML_CUDA_MMV_X: usize = 32; +pub const GGML_CUDA_MMV_Y: usize = 1; + +fn dequantize( + data: &CudaSlice, + dtype: GgmlDType, + elem_count: usize, + dev: &CudaDevice, +) -> Result { + use cudarc::driver::LaunchAsync; + + let (kernel_name, is_k, block_dim) = match dtype { + GgmlDType::Q4_0 => ("dequantize_block_q4_0", false, 32), + GgmlDType::Q4_1 => ("dequantize_block_q4_1", false, 32), + GgmlDType::Q5_0 => ("dequantize_block_q5_0", false, 32), + GgmlDType::Q5_1 => ("dequantize_block_q5_1", false, 32), + GgmlDType::Q8_0 => ("dequantize_block_q8_0", false, 32), + GgmlDType::Q2K => ("dequantize_block_q2_K", true, 64), + GgmlDType::Q3K => ("dequantize_block_q3_K", true, 64), + GgmlDType::Q4K => ("dequantize_block_q4_K", true, 32), + GgmlDType::Q5K => ("dequantize_block_q5_K", true, 64), + GgmlDType::Q6K => ("dequantize_block_q6_K", true, 64), + GgmlDType::Q8K => ("dequantize_block_q8_K", true, 32), + _ => crate::bail!("unsupported dtype for dequantize {dtype:?}"), + }; + let func = dev.get_or_load_func(kernel_name, candle_kernels::QUANTIZED)?; + let dst = dev.alloc_zeros::(elem_count).w()?; + let nb = (elem_count + 255) / 256; + // See e.g. + // https://github.com/ggerganov/llama.cpp/blob/cbbd1efa06f8c09f9dff58ff9d9af509cc4c152b/ggml-cuda.cu#L7270 + let cfg = cudarc::driver::LaunchConfig { + grid_dim: (nb as u32, 1, 1), + block_dim: (block_dim, 1, 1), + shared_mem_bytes: 0, + }; + + if is_k { + let params = (data, &dst); + unsafe { func.launch(cfg, params) }.w()?; + } else { + let nb32 = elem_count / 32; + let params = (data, &dst, nb32 as i32); + unsafe { func.launch(cfg, params) }.w()?; + } + Ok(CudaStorage::wrap_cuda_slice(dst, dev.clone())) +} + +fn dequantize_mut_mal_vec( + data: &CudaSlice, + y: &cudarc::driver::CudaView, + dtype: GgmlDType, + ncols: usize, + nrows: usize, + dev: &CudaDevice, +) -> Result { + use cudarc::driver::LaunchAsync; + + let kernel_name = match dtype { + GgmlDType::Q4_0 => "dequantize_mul_mat_vec_q4_0_cuda", + GgmlDType::Q4_1 => "dequantize_mul_mat_vec_q4_1_cuda", + GgmlDType::Q5_0 => "dequantize_mul_mat_vec_q5_0_cuda", + GgmlDType::Q5_1 => "dequantize_mul_mat_vec_q5_1_cuda", + GgmlDType::Q8_0 => "dequantize_mul_mat_vec_q8_0_cuda", + GgmlDType::Q2K => "dequantize_mul_mat_vec_q2_k", + GgmlDType::Q3K => "dequantize_mul_mat_vec_q3_k", + GgmlDType::Q4K => "dequantize_mul_mat_vec_q4_k", + GgmlDType::Q5K => "dequantize_mul_mat_vec_q5_k", + GgmlDType::Q6K => "dequantize_mul_mat_vec_q6_k", + _ => crate::bail!("unsupported dtype for quantized matmul {dtype:?}"), + }; + let func = dev.get_or_load_func(kernel_name, candle_kernels::QUANTIZED)?; + let dst = dev.alloc_zeros::(nrows).w()?; + let block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; + let cfg = cudarc::driver::LaunchConfig { + grid_dim: (block_num_y as u32, 1, 1), + block_dim: (WARP_SIZE as u32, GGML_CUDA_MMV_Y as u32, 1), + shared_mem_bytes: 0, + }; + + let params = (data, y, &dst, ncols as i32, nrows as i32); + unsafe { func.launch(cfg, params) }.w()?; + Ok(CudaStorage::wrap_cuda_slice(dst, dev.clone())) +} + +impl QCudaStorage { + pub fn zeros(device: &CudaDevice, el_count: usize, dtype: GgmlDType) -> Result { + let size_in_bytes = el_count * dtype.type_size() / dtype.block_size(); + let data = device.alloc_zeros::(size_in_bytes).w()?; + Ok(QCudaStorage { + data, + device: device.clone(), + dtype, + }) + } + + pub fn dtype(&self) -> GgmlDType { + self.dtype + } + + pub fn device(&self) -> &CudaDevice { + &self.device + } + + pub fn dequantize(&self, elem_count: usize) -> Result { + let fast_kernel = matches!( + self.dtype, + GgmlDType::Q4_0 + | GgmlDType::Q4_1 + | GgmlDType::Q5_0 + | GgmlDType::Q5_1 + | GgmlDType::Q8_0 + | GgmlDType::Q2K + | GgmlDType::Q3K + | GgmlDType::Q4K + | GgmlDType::Q5K + | GgmlDType::Q6K + | GgmlDType::Q8K + ); + if fast_kernel { + return dequantize(&self.data, self.dtype, elem_count, self.device()); + } + // Run the dequantization on cpu. + use crate::quantized::k_quants::GgmlType; + + let buffer = self.device.dtoh_sync_copy(&self.data).w()?; + let mut out = vec![0.0; elem_count]; + let block_len = elem_count / self.dtype.block_size(); + match self.dtype { + GgmlDType::F32 => { + let slice = + unsafe { std::slice::from_raw_parts(buffer.as_ptr() as *const f32, block_len) }; + out.copy_from_slice(slice) + } + GgmlDType::F16 => { + let vec: Vec = read_to_vec(&buffer, block_len); + half::f16::to_float(&vec, &mut out)?; + } + GgmlDType::Q4_0 => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ4_0::to_float(&vec, &mut out)?; + } + GgmlDType::Q4_1 => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ4_1::to_float(&vec, &mut out)?; + } + GgmlDType::Q5_0 => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ5_0::to_float(&vec, &mut out)?; + } + GgmlDType::Q5_1 => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ5_1::to_float(&vec, &mut out)?; + } + GgmlDType::Q8_0 => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ8_0::to_float(&vec, &mut out)?; + } + GgmlDType::Q8_1 => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ8_1::to_float(&vec, &mut out)?; + } + GgmlDType::Q2K => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ2K::to_float(&vec, &mut out)?; + } + GgmlDType::Q3K => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ3K::to_float(&vec, &mut out)?; + } + GgmlDType::Q4K => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ4K::to_float(&vec, &mut out)?; + } + GgmlDType::Q5K => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ5K::to_float(&vec, &mut out)?; + } + GgmlDType::Q6K => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ6K::to_float(&vec, &mut out)?; + } + GgmlDType::Q8K => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ8K::to_float(&vec, &mut out)?; + } + } + + self.device + .storage_from_cpu_storage(&crate::CpuStorage::F32(out)) + } + + pub fn quantize(&mut self, src: &CudaStorage) -> Result<()> { + // Run the quantization on cpu. + let src = match &src.slice { + crate::cuda_backend::CudaStorageSlice::F32(data) => { + self.device.dtoh_sync_copy(data).w()? + } + _ => crate::bail!("only f32 can be quantized"), + }; + let src_len = src.len(); + let src = crate::Storage::Cpu(crate::CpuStorage::F32(src)); + let mut qcpu_storage = crate::Device::Cpu.qzeros(src_len, self.dtype)?; + qcpu_storage.quantize(&src)?; + let data = qcpu_storage.data()?; + let data = self.device.htod_sync_copy(data.as_ref()).w()?; + self.data = data; + Ok(()) + } + + pub fn storage_size_in_bytes(&self) -> usize { + self.data.len() + } + + pub fn fwd( + &self, + self_shape: &crate::Shape, + storage: &CudaStorage, + layout: &crate::Layout, + ) -> Result<(CudaStorage, crate::Shape)> { + if matches!(layout.shape().dims(), [1, 1, _] | [1, _]) { + self.dequantize_matmul_vec(self_shape, storage, layout) + } else { + self.dequantize_matmul(self_shape, storage, layout) + } + } +} + +impl QCudaStorage { + fn dequantize_matmul_vec( + &self, + self_shape: &crate::Shape, + rhs: &CudaStorage, + rhs_l: &crate::Layout, + ) -> Result<(CudaStorage, crate::Shape)> { + let (nrows, ncols) = self_shape.dims2()?; + let rhs = rhs.as_cuda_slice::()?; + let rhs = match rhs_l.contiguous_offsets() { + Some((o1, o2)) => rhs.slice(o1..o2), + None => Err(crate::Error::RequiresContiguous { op: "dmmv" }.bt())?, + }; + let (with_batch, k) = match rhs_l.shape().dims() { + [1, 1, k] => (true, k), + [1, k] => (false, k), + _ => crate::bail!("unexpected rhs shape in dmmv {:?}", rhs_l.shape()), + }; + if ncols != *k { + crate::bail!("mismatch on matmul dim {self_shape:?} {:?}", rhs_l.shape()) + } + + let out = + dequantize_mut_mal_vec(&self.data, &rhs, self.dtype, ncols, nrows, self.device())?; + let out_shape = if with_batch { + vec![1, 1, nrows] + } else { + vec![1, nrows] + }; + Ok((out, out_shape.into())) + } + + fn dequantize_matmul( + &self, + self_shape: &crate::Shape, + storage: &CudaStorage, + layout: &crate::Layout, + ) -> Result<(CudaStorage, crate::Shape)> { + use crate::backend::BackendStorage; + let (n, k) = self_shape.dims2()?; + let (b, m, k2) = match layout.shape().dims() { + &[b, m, k2] => (b, m, k2), + &[m, k2] => (1, m, k2), + s => crate::bail!("unexpected shape for input {s:?}"), + }; + if k2 != k { + crate::bail!("mismatch on matmul dim {self_shape:?} {:?}", layout.shape()) + } + + let data_f32 = self.dequantize(n * k)?; + let rhs_l = crate::Layout::new((k, n).into(), vec![1, k], 0); + let out = storage.matmul(&data_f32, (b, m, n, k), layout, &rhs_l)?; + let mut out_shape = layout.shape().dims().to_vec(); + out_shape.pop(); + out_shape.push(n); + Ok((out, out_shape.into())) + } +} + +fn read_to_vec(buffer: &[u8], n: usize) -> Vec { + let slice = unsafe { std::slice::from_raw_parts(buffer.as_ptr() as *const T, n) }; + slice.to_vec() +} + +pub fn load_quantized( + device: &CudaDevice, + data: &[T], +) -> Result { + let data = unsafe { + std::slice::from_raw_parts(data.as_ptr() as *const u8, core::mem::size_of_val(data)) + }; + let data = device.htod_sync_copy(data).w()?; + Ok(QStorage::Cuda(QCudaStorage { + data, + device: device.clone(), + dtype: T::DTYPE, + })) +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/dummy_cuda.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/dummy_cuda.rs new file mode 100644 index 0000000000..598c5cd131 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/quantized/dummy_cuda.rs @@ -0,0 +1,50 @@ +#![allow(unused)] +use super::GgmlDType; +use crate::{CudaDevice, CudaStorage, Error, Result}; + +pub struct QCudaStorage { + dtype: GgmlDType, + device: CudaDevice, +} + +impl QCudaStorage { + pub fn zeros(_: &CudaDevice, _: usize, _: GgmlDType) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + pub fn dtype(&self) -> GgmlDType { + self.dtype + } + + pub fn device(&self) -> &CudaDevice { + &self.device + } + + pub fn dequantize(&self, _elem_count: usize) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + pub fn quantize(&mut self, _src: &CudaStorage) -> Result<()> { + Err(Error::NotCompiledWithCudaSupport) + } + + pub fn storage_size_in_bytes(&self) -> usize { + 0 + } + + pub fn fwd( + &self, + _self_shape: &crate::Shape, + _storage: &CudaStorage, + _layout: &crate::Layout, + ) -> Result<(CudaStorage, crate::Shape)> { + Err(Error::NotCompiledWithCudaSupport) + } +} + +pub fn load_quantized( + _device: &CudaDevice, + _data: &[T], +) -> Result { + Err(Error::NotCompiledWithCudaSupport) +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/dummy_metal.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/dummy_metal.rs new file mode 100644 index 0000000000..520d0ed49a --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/quantized/dummy_metal.rs @@ -0,0 +1,50 @@ +#![allow(unused)] +use super::GgmlDType; +use crate::{Error, MetalDevice, MetalStorage, Result}; + +pub struct QMetalStorage { + dtype: GgmlDType, + device: MetalDevice, +} + +impl QMetalStorage { + pub fn zeros(_: &MetalDevice, _: usize, _: GgmlDType) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + pub fn dtype(&self) -> GgmlDType { + self.dtype + } + + pub fn device(&self) -> &MetalDevice { + &self.device + } + + pub fn dequantize(&self, _elem_count: usize) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + pub fn quantize(&mut self, _src: &MetalStorage) -> Result<()> { + Err(Error::NotCompiledWithMetalSupport) + } + + pub fn storage_size_in_bytes(&self) -> usize { + 0 + } + + pub fn fwd( + &self, + _self_shape: &crate::Shape, + _storage: &MetalStorage, + _layout: &crate::Layout, + ) -> Result<(MetalStorage, crate::Shape)> { + Err(Error::NotCompiledWithMetalSupport) + } +} + +pub fn load_quantized( + _device: &MetalDevice, + _data: &[T], +) -> Result { + Err(Error::NotCompiledWithMetalSupport) +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/ggml_file.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/ggml_file.rs new file mode 100644 index 0000000000..99200bbd06 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/quantized/ggml_file.rs @@ -0,0 +1,265 @@ +//! Support for the GGML file format. + +use super::{k_quants, GgmlDType, QStorage}; +use crate::{Device, Result}; +use byteorder::{LittleEndian, ReadBytesExt}; +use std::collections::HashMap; + +// https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/llama.h#L37 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Magic { + Ggjt, + Ggla, + Ggmf, + Ggml, + Ggsn, +} + +impl TryFrom for Magic { + type Error = crate::Error; + fn try_from(value: u32) -> Result { + let magic = match value { + 0x67676a74 => Self::Ggjt, + 0x67676c61 => Self::Ggla, + 0x67676d66 => Self::Ggmf, + 0x67676d6c => Self::Ggml, + 0x6767736e => Self::Ggsn, + _ => crate::bail!("unknown magic {value:08x}"), + }; + Ok(magic) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VersionedMagic { + GgmlUnversioned, + GgmfV1, + GgjtV1, + GgjtV2, + GgjtV3, +} + +impl VersionedMagic { + fn read(reader: &mut R) -> Result { + let magic = reader.read_u32::()?; + let magic = Magic::try_from(magic)?; + if magic == Magic::Ggml { + return Ok(Self::GgmlUnversioned); + } + let version = reader.read_u32::()?; + let versioned_magic = match (magic, version) { + (Magic::Ggmf, 1) => Self::GgmfV1, + (Magic::Ggjt, 1) => Self::GgjtV1, + (Magic::Ggjt, 2) => Self::GgjtV2, + (Magic::Ggjt, 3) => Self::GgjtV3, + _ => crate::bail!("ggml: unsupported magic/version {magic:?}/{version}"), + }; + Ok(versioned_magic) + } + + fn align32(&self) -> bool { + match self { + Self::GgmlUnversioned | Self::GgmfV1 => false, + Self::GgjtV1 | Self::GgjtV2 | Self::GgjtV3 => true, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HParams { + pub n_vocab: u32, + pub n_embd: u32, + pub n_mult: u32, + pub n_head: u32, + pub n_layer: u32, + pub n_rot: u32, + pub ftype: u32, +} + +impl HParams { + fn read(reader: &mut R) -> Result { + let n_vocab = reader.read_u32::()?; + let n_embd = reader.read_u32::()?; + let n_mult = reader.read_u32::()?; + let n_head = reader.read_u32::()?; + let n_layer = reader.read_u32::()?; + let n_rot = reader.read_u32::()?; + let ftype = reader.read_u32::()?; + Ok(Self { + n_vocab, + n_embd, + n_mult, + n_head, + n_layer, + n_rot, + ftype, + }) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Vocab { + pub token_score_pairs: Vec<(Vec, f32)>, +} + +impl Vocab { + fn read(reader: &mut R, n_vocab: usize) -> Result { + // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/llama.cpp#L556 + let mut token_score_pairs = Vec::with_capacity(n_vocab); + for _index in 0..n_vocab { + let len = reader.read_u32::()? as usize; + let mut word = vec![0u8; len]; + reader.read_exact(&mut word)?; + let score = reader.read_f32::()?; + token_score_pairs.push((word, score)) + } + Ok(Self { token_score_pairs }) + } +} + +fn from_raw_data( + raw_data: &[u8], + size_in_bytes: usize, + dims: Vec, + device: &Device, +) -> Result { + let raw_data_ptr = raw_data.as_ptr(); + let n_blocks = size_in_bytes / std::mem::size_of::(); + let data = unsafe { std::slice::from_raw_parts(raw_data_ptr as *const T, n_blocks) }; + let data: QStorage = match device { + Device::Cpu => QStorage::Cpu(Box::new(data.to_vec())), + Device::Metal(metal) => super::metal::load_quantized(metal, data)?, + Device::Cuda(cuda) => super::cuda::load_quantized(cuda, data)?, + }; + super::QTensor::new(data, dims) +} + +/// Creates a [Tensor] from a raw GGML tensor. +pub fn qtensor_from_ggml( + ggml_dtype: GgmlDType, + raw_data: &[u8], + dims: Vec, + device: &Device, +) -> Result { + let tensor_elems = dims.iter().product::(); + let block_size = ggml_dtype.block_size(); + if tensor_elems % block_size != 0 { + crate::bail!( + "the number of elements {tensor_elems} is not divisible by the block size {block_size}" + ) + } + let size_in_bytes = tensor_elems / block_size * ggml_dtype.type_size(); + + match ggml_dtype { + GgmlDType::F32 => from_raw_data::(raw_data, size_in_bytes, dims, device), + GgmlDType::F16 => from_raw_data::(raw_data, size_in_bytes, dims, device), + GgmlDType::Q4_0 => { + from_raw_data::(raw_data, size_in_bytes, dims, device) + } + GgmlDType::Q4_1 => { + from_raw_data::(raw_data, size_in_bytes, dims, device) + } + GgmlDType::Q5_0 => { + from_raw_data::(raw_data, size_in_bytes, dims, device) + } + GgmlDType::Q5_1 => { + from_raw_data::(raw_data, size_in_bytes, dims, device) + } + GgmlDType::Q8_0 => { + from_raw_data::(raw_data, size_in_bytes, dims, device) + } + GgmlDType::Q2K => { + from_raw_data::(raw_data, size_in_bytes, dims, device) + } + GgmlDType::Q3K => { + from_raw_data::(raw_data, size_in_bytes, dims, device) + } + GgmlDType::Q4K => { + from_raw_data::(raw_data, size_in_bytes, dims, device) + } + GgmlDType::Q5K => { + from_raw_data::(raw_data, size_in_bytes, dims, device) + } + GgmlDType::Q6K => { + from_raw_data::(raw_data, size_in_bytes, dims, device) + } + _ => crate::bail!("quantized type {ggml_dtype:?} is not supported yet"), + } +} + +fn read_one_tensor( + reader: &mut R, + magic: VersionedMagic, + device: &Device, +) -> Result<(String, super::QTensor)> { + let n_dims = reader.read_u32::()?; + let name_len = reader.read_u32::()?; + let ggml_dtype = reader.read_u32::()?; + let ggml_dtype = GgmlDType::from_u32(ggml_dtype)?; + let mut dims = vec![0u32; n_dims as usize]; + reader.read_u32_into::(&mut dims)?; + // The dimensions are stored in reverse order, see for example: + // https://github.com/ggerganov/llama.cpp/blob/b5ffb2849d23afe73647f68eec7b68187af09be6/convert.py#L969 + dims.reverse(); + let mut name = vec![0u8; name_len as usize]; + reader.read_exact(&mut name)?; + let name = String::from_utf8_lossy(&name).into_owned(); + + if magic.align32() { + let pos = reader.stream_position()?; + reader.seek(std::io::SeekFrom::Current(((32 - pos % 32) % 32) as i64))?; + } + let dims = dims.iter().map(|&u| u as usize).collect::>(); + let tensor_elems = dims.iter().product::(); + let size_in_bytes = tensor_elems * ggml_dtype.type_size() / ggml_dtype.block_size(); + // TODO: Mmap version to avoid copying the data around? + let mut raw_data = vec![0u8; size_in_bytes]; + reader.read_exact(&mut raw_data)?; + match qtensor_from_ggml(ggml_dtype, &raw_data, dims, device) { + Ok(tensor) => Ok((name, tensor)), + Err(e) => crate::bail!("Error creating tensor {name}: {e}"), + } +} + +pub struct Content { + pub magic: VersionedMagic, + pub hparams: HParams, + pub vocab: Vocab, + pub tensors: HashMap, + pub device: Device, +} + +impl Content { + pub fn read( + reader: &mut R, + device: &Device, + ) -> Result { + // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/llama.cpp#L505 + let last_position = reader.seek(std::io::SeekFrom::End(0))?; + reader.seek(std::io::SeekFrom::Start(0))?; + let magic = VersionedMagic::read(reader)?; + let hparams = HParams::read(reader)?; + let vocab = Vocab::read(reader, hparams.n_vocab as usize)?; + let mut tensors = HashMap::new(); + + while reader.stream_position()? != last_position { + let (name, tensor) = read_one_tensor(reader, magic, device)?; + tensors.insert(name, tensor); + } + let device = device.clone(); + Ok(Self { + magic, + hparams, + vocab, + tensors, + device, + }) + } + + pub fn remove(&mut self, name: &str) -> Result { + match self.tensors.remove(name) { + None => crate::bail!("cannot find tensor with name '{name}'"), + Some(tensor) => Ok(tensor), + } + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/gguf_file.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/gguf_file.rs new file mode 100644 index 0000000000..b729d4a0fd --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/quantized/gguf_file.rs @@ -0,0 +1,534 @@ +//! Support for the GGUF file format. +//! +//! Spec: https://github.com/philpax/ggml/blob/gguf-spec/docs/gguf.md + +use super::{GgmlDType, QTensor}; +use crate::{Device, Result}; +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; +use std::collections::HashMap; + +pub const DEFAULT_ALIGNMENT: u64 = 32; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Magic { + Gguf, +} + +impl TryFrom for Magic { + type Error = crate::Error; + fn try_from(value: u32) -> Result { + let magic = match value { + 0x46554747 | 0x47475546 => Self::Gguf, + _ => crate::bail!("unknown magic 0x{value:08x}"), + }; + Ok(magic) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VersionedMagic { + GgufV1, + GgufV2, + GgufV3, +} + +impl VersionedMagic { + fn read(reader: &mut R) -> Result { + let magic = reader.read_u32::()?; + let magic = Magic::try_from(magic)?; + let version = reader.read_u32::()?; + let versioned_magic = match (magic, version) { + (Magic::Gguf, 1) => Self::GgufV1, + (Magic::Gguf, 2) => Self::GgufV2, + (Magic::Gguf, 3) => Self::GgufV3, + _ => crate::bail!("gguf: unsupported magic/version {magic:?}/{version}"), + }; + Ok(versioned_magic) + } +} + +#[derive(Debug)] +pub struct TensorInfo { + pub ggml_dtype: GgmlDType, + pub shape: crate::Shape, + pub offset: u64, +} + +impl TensorInfo { + pub fn read( + &self, + reader: &mut R, + tensor_data_offset: u64, + device: &Device, + ) -> Result { + let tensor_elems = self.shape.elem_count(); + let block_size = self.ggml_dtype.block_size(); + if tensor_elems % block_size != 0 { + crate::bail!( + "the number of elements {tensor_elems} is not divisible by the block size {block_size}" + ) + } + let size_in_bytes = tensor_elems / block_size * self.ggml_dtype.type_size(); + let mut raw_data = vec![0u8; size_in_bytes]; + reader.seek(std::io::SeekFrom::Start(tensor_data_offset + self.offset))?; + reader.read_exact(&mut raw_data)?; + super::ggml_file::qtensor_from_ggml( + self.ggml_dtype, + &raw_data, + self.shape.dims().to_vec(), + device, + ) + } +} + +#[derive(Debug)] +pub struct Content { + pub magic: VersionedMagic, + pub metadata: HashMap, + pub tensor_infos: HashMap, + pub tensor_data_offset: u64, +} + +fn read_string(reader: &mut R, magic: &VersionedMagic) -> Result { + let len = match magic { + VersionedMagic::GgufV1 => reader.read_u32::()? as usize, + VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => { + reader.read_u64::()? as usize + } + }; + let mut v = vec![0u8; len]; + reader.read_exact(&mut v)?; + // GGUF strings are supposed to be non-null terminated but in practice this happens. + while let Some(0) = v.last() { + v.pop(); + } + // GGUF strings are utf8 encoded but there are cases that don't seem to be valid. + Ok(String::from_utf8_lossy(&v).into_owned()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ValueType { + // The value is a 8-bit unsigned integer. + U8, + // The value is a 8-bit signed integer. + I8, + // The value is a 16-bit unsigned little-endian integer. + U16, + // The value is a 16-bit signed little-endian integer. + I16, + // The value is a 32-bit unsigned little-endian integer. + U32, + // The value is a 32-bit signed little-endian integer. + I32, + // The value is a 64-bit unsigned little-endian integer. + U64, + // The value is a 64-bit signed little-endian integer. + I64, + // The value is a 32-bit IEEE754 floating point number. + F32, + // The value is a 64-bit IEEE754 floating point number. + F64, + // The value is a boolean. + // 1-byte value where 0 is false and 1 is true. + // Anything else is invalid, and should be treated as either the model being invalid or the reader being buggy. + Bool, + // The value is a UTF-8 non-null-terminated string, with length prepended. + String, + // The value is an array of other values, with the length and type prepended. + /// + // Arrays can be nested, and the length of the array is the number of elements in the array, not the number of bytes. + Array, +} + +#[derive(Debug, Clone)] +pub enum Value { + U8(u8), + I8(i8), + U16(u16), + I16(i16), + U32(u32), + I32(i32), + U64(u64), + I64(i64), + F32(f32), + F64(f64), + Bool(bool), + String(String), + Array(Vec), +} + +impl Value { + pub fn value_type(&self) -> ValueType { + match self { + Self::U8(_) => ValueType::U8, + Self::I8(_) => ValueType::I8, + Self::U16(_) => ValueType::U16, + Self::I16(_) => ValueType::I16, + Self::U32(_) => ValueType::U32, + Self::I32(_) => ValueType::I32, + Self::U64(_) => ValueType::U64, + Self::I64(_) => ValueType::I64, + Self::F32(_) => ValueType::F32, + Self::F64(_) => ValueType::F64, + Self::Bool(_) => ValueType::Bool, + Self::String(_) => ValueType::String, + Self::Array(_) => ValueType::Array, + } + } + + pub fn to_u8(&self) -> Result { + match self { + Self::U8(v) => Ok(*v), + v => crate::bail!("not a u8 {v:?}"), + } + } + + pub fn to_i8(&self) -> Result { + match self { + Self::I8(v) => Ok(*v), + v => crate::bail!("not a i8 {v:?}"), + } + } + + pub fn to_u16(&self) -> Result { + match self { + Self::U16(v) => Ok(*v), + v => crate::bail!("not a u16 {v:?}"), + } + } + + pub fn to_i16(&self) -> Result { + match self { + Self::I16(v) => Ok(*v), + v => crate::bail!("not a i16 {v:?}"), + } + } + + pub fn to_u32(&self) -> Result { + match self { + Self::U32(v) => Ok(*v), + v => crate::bail!("not a u32 {v:?}"), + } + } + + pub fn to_i32(&self) -> Result { + match self { + Self::I32(v) => Ok(*v), + v => crate::bail!("not a i32 {v:?}"), + } + } + + pub fn to_u64(&self) -> Result { + match self { + Self::U64(v) => Ok(*v), + v => crate::bail!("not a u64 {v:?}"), + } + } + + pub fn to_i64(&self) -> Result { + match self { + Self::I64(v) => Ok(*v), + v => crate::bail!("not a i64 {v:?}"), + } + } + + pub fn to_f32(&self) -> Result { + match self { + Self::F32(v) => Ok(*v), + v => crate::bail!("not a f32 {v:?}"), + } + } + + pub fn to_f64(&self) -> Result { + match self { + Self::F64(v) => Ok(*v), + v => crate::bail!("not a f64 {v:?}"), + } + } + + pub fn to_bool(&self) -> Result { + match self { + Self::Bool(v) => Ok(*v), + v => crate::bail!("not a bool {v:?}"), + } + } + + pub fn to_vec(&self) -> Result<&Vec> { + match self { + Self::Array(v) => Ok(v), + v => crate::bail!("not a vec {v:?}"), + } + } + + pub fn to_string(&self) -> Result<&String> { + match self { + Self::String(v) => Ok(v), + v => crate::bail!("not a string {v:?}"), + } + } + + fn read( + reader: &mut R, + value_type: ValueType, + magic: &VersionedMagic, + ) -> Result { + let v = match value_type { + ValueType::U8 => Self::U8(reader.read_u8()?), + ValueType::I8 => Self::I8(reader.read_i8()?), + ValueType::U16 => Self::U16(reader.read_u16::()?), + ValueType::I16 => Self::I16(reader.read_i16::()?), + ValueType::U32 => Self::U32(reader.read_u32::()?), + ValueType::I32 => Self::I32(reader.read_i32::()?), + ValueType::U64 => Self::U64(reader.read_u64::()?), + ValueType::I64 => Self::I64(reader.read_i64::()?), + ValueType::F32 => Self::F32(reader.read_f32::()?), + ValueType::F64 => Self::F64(reader.read_f64::()?), + ValueType::Bool => match reader.read_u8()? { + 0 => Self::Bool(false), + 1 => Self::Bool(true), + b => crate::bail!("unexpected bool value {b}"), + }, + ValueType::String => Self::String(read_string(reader, magic)?), + ValueType::Array => { + let value_type = reader.read_u32::()?; + let value_type = ValueType::from_u32(value_type)?; + let len = match magic { + VersionedMagic::GgufV1 => reader.read_u32::()? as usize, + VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => { + reader.read_u64::()? as usize + } + }; + let mut vs = Vec::with_capacity(len); + for _ in 0..len { + vs.push(Value::read(reader, value_type, magic)?) + } + Self::Array(vs) + } + }; + Ok(v) + } + + fn write(&self, w: &mut W) -> Result<()> { + match self { + &Self::U8(v) => w.write_u8(v)?, + &Self::I8(v) => w.write_i8(v)?, + &Self::U16(v) => w.write_u16::(v)?, + &Self::I16(v) => w.write_i16::(v)?, + &Self::U32(v) => w.write_u32::(v)?, + &Self::I32(v) => w.write_i32::(v)?, + &Self::U64(v) => w.write_u64::(v)?, + &Self::I64(v) => w.write_i64::(v)?, + &Self::F32(v) => w.write_f32::(v)?, + &Self::F64(v) => w.write_f64::(v)?, + &Self::Bool(v) => w.write_u8(u8::from(v))?, + Self::String(v) => write_string(w, v.as_str())?, + Self::Array(v) => { + // The `Value` type does not enforce that all the values in an Array have the same + // type. + let value_type = if v.is_empty() { + // Doesn't matter, the array is empty. + ValueType::U32 + } else { + let value_type: std::collections::HashSet<_> = + v.iter().map(|elem| elem.value_type()).collect(); + if value_type.len() != 1 { + crate::bail!("multiple value-types in the same array {value_type:?}") + } + value_type.into_iter().next().unwrap() + }; + w.write_u32::(value_type.to_u32())?; + w.write_u64::(v.len() as u64)?; + for elem in v.iter() { + elem.write(w)? + } + } + } + Ok(()) + } +} + +impl ValueType { + fn from_u32(v: u32) -> Result { + let v = match v { + 0 => Self::U8, + 1 => Self::I8, + 2 => Self::U16, + 3 => Self::I16, + 4 => Self::U32, + 5 => Self::I32, + 6 => Self::F32, + 7 => Self::Bool, + 8 => Self::String, + 9 => Self::Array, + 10 => Self::U64, + 11 => Self::I64, + 12 => Self::F64, + v => crate::bail!("unrecognized value-type {v:#08x}"), + }; + Ok(v) + } + + fn to_u32(self) -> u32 { + match self { + Self::U8 => 0, + Self::I8 => 1, + Self::U16 => 2, + Self::I16 => 3, + Self::U32 => 4, + Self::I32 => 5, + Self::F32 => 6, + Self::Bool => 7, + Self::String => 8, + Self::Array => 9, + Self::U64 => 10, + Self::I64 => 11, + Self::F64 => 12, + } + } +} + +impl Content { + pub fn read(reader: &mut R) -> Result { + let magic = VersionedMagic::read(reader)?; + + let tensor_count = match magic { + VersionedMagic::GgufV1 => reader.read_u32::()? as usize, + VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => { + reader.read_u64::()? as usize + } + }; + let metadata_kv_count = match magic { + VersionedMagic::GgufV1 => reader.read_u32::()? as usize, + VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => { + reader.read_u64::()? as usize + } + }; + + let mut metadata = HashMap::new(); + for _idx in 0..metadata_kv_count { + let key = read_string(reader, &magic)?; + let value_type = reader.read_u32::()?; + let value_type = ValueType::from_u32(value_type)?; + let value = Value::read(reader, value_type, &magic)?; + metadata.insert(key, value); + } + let mut tensor_infos = HashMap::new(); + for _idx in 0..tensor_count { + let tensor_name = read_string(reader, &magic)?; + let n_dimensions = reader.read_u32::()?; + + let mut dimensions: Vec = match magic { + VersionedMagic::GgufV1 => { + let mut dimensions = vec![0; n_dimensions as usize]; + reader.read_u32_into::(&mut dimensions)?; + dimensions.into_iter().map(|c| c as usize).collect() + } + VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => { + let mut dimensions = vec![0; n_dimensions as usize]; + reader.read_u64_into::(&mut dimensions)?; + dimensions.into_iter().map(|c| c as usize).collect() + } + }; + + dimensions.reverse(); + let ggml_dtype = reader.read_u32::()?; + let ggml_dtype = GgmlDType::from_u32(ggml_dtype)?; + let offset = reader.read_u64::()?; + tensor_infos.insert( + tensor_name, + TensorInfo { + shape: crate::Shape::from(dimensions), + offset, + ggml_dtype, + }, + ); + } + let position = reader.stream_position()?; + let alignment = match metadata.get("general.alignment") { + Some(Value::U8(v)) => *v as u64, + Some(Value::U16(v)) => *v as u64, + Some(Value::U32(v)) => *v as u64, + Some(Value::I8(v)) if *v >= 0 => *v as u64, + Some(Value::I16(v)) if *v >= 0 => *v as u64, + Some(Value::I32(v)) if *v >= 0 => *v as u64, + _ => DEFAULT_ALIGNMENT, + }; + let tensor_data_offset = (position + alignment - 1) / alignment * alignment; + Ok(Self { + magic, + metadata, + tensor_infos, + tensor_data_offset, + }) + } + + pub fn tensor( + &self, + reader: &mut R, + name: &str, + device: &Device, + ) -> Result { + let tensor_info = match self.tensor_infos.get(name) { + Some(tensor_info) => tensor_info, + None => crate::bail!("cannot find tensor info for {name}"), + }; + tensor_info.read(reader, self.tensor_data_offset, device) + } +} + +fn write_string(w: &mut W, str: &str) -> Result<()> { + let bytes = str.as_bytes(); + w.write_u64::(bytes.len() as u64)?; + w.write_all(bytes)?; + Ok(()) +} + +pub fn write( + w: &mut W, + metadata: &[(&str, &Value)], + tensors: &[(&str, &QTensor)], +) -> Result<()> { + w.write_u32::(0x46554747)?; + w.write_u32::(2)?; // version 2. + w.write_u64::(tensors.len() as u64)?; + w.write_u64::(metadata.len() as u64)?; + for (name, value) in metadata.iter() { + write_string(w, name)?; + w.write_u32::(value.value_type().to_u32())?; + value.write(w)?; + } + let mut offset = 0usize; + let mut offsets = Vec::with_capacity(tensors.len()); + for (name, tensor) in tensors.iter() { + write_string(w, name)?; + let dims = tensor.shape().dims(); + w.write_u32::(dims.len() as u32)?; + for &dim in dims.iter().rev() { + w.write_u64::(dim as u64)?; + } + w.write_u32::(tensor.dtype().to_u32())?; + w.write_u64::(offset as u64)?; + offsets.push(offset); + let size_in_bytes = tensor.storage_size_in_bytes(); + let padding = 31 - (31 + size_in_bytes) % 32; + offset += size_in_bytes + padding; + } + let pos = w.stream_position()? as usize; + let padding = 31 - (31 + pos) % 32; + w.write_all(&vec![0u8; padding])?; + let tensor_start_pos = w.stream_position()? as usize; + for (offset, (_name, tensor)) in offsets.iter().zip(tensors.iter()) { + let pos = w.stream_position()? as usize; + if tensor_start_pos + offset != pos { + crate::bail!( + "internal error, unexpected current position {tensor_start_pos} {offset} {pos}" + ) + } + let data = tensor.data()?; + let size_in_bytes = data.len(); + w.write_all(&data)?; + let padding = 31 - (31 + size_in_bytes) % 32; + w.write_all(&vec![0u8; padding])?; + } + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/k_quants.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/k_quants.rs new file mode 100644 index 0000000000..6210ac1e9f --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/quantized/k_quants.rs @@ -0,0 +1,1965 @@ +use super::utils::{ + get_scale_min_k4, group_for_dequantization, group_for_quantization, make_q3_quants, + make_qkx1_quants, make_qx_quants, nearest_int, +}; +use super::GgmlDType; +use crate::Result; +use byteorder::{ByteOrder, LittleEndian}; +use half::f16; +use rayon::prelude::*; + +// Default to QK_K 256 rather than 64. +pub const QK_K: usize = 256; +pub const K_SCALE_SIZE: usize = 12; + +pub const QK4_0: usize = 32; +pub const QK4_1: usize = 32; +pub const QK5_0: usize = 32; +pub const QK5_1: usize = 32; +pub const QK8_0: usize = 32; +pub const QK8_1: usize = 32; + +pub trait GgmlType: Sized + Clone + Send + Sync { + const DTYPE: GgmlDType; + const BLCK_SIZE: usize; + type VecDotType: GgmlType; + + // This is only safe for types that include immediate values such as float/int/... + fn zeros() -> Self { + unsafe { std::mem::MaybeUninit::zeroed().assume_init() } + } + fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()>; + fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()>; + + /// Dot product used as a building block for quantized mat-mul. + /// n is the number of elements to be considered. + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result; + + /// Generic implementation of the dot product without simd optimizations. + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result; +} + +#[derive(Debug, Clone, PartialEq)] +#[repr(C)] +pub struct BlockQ4_0 { + pub(crate) d: f16, + pub(crate) qs: [u8; QK4_0 / 2], +} +const _: () = assert!(std::mem::size_of::() == 18); + +#[derive(Debug, Clone, PartialEq)] +#[repr(C)] +pub struct BlockQ4_1 { + pub(crate) d: f16, + pub(crate) m: f16, + pub(crate) qs: [u8; QK4_1 / 2], +} +const _: () = assert!(std::mem::size_of::() == 20); + +#[derive(Debug, Clone, PartialEq)] +#[repr(C)] +pub struct BlockQ5_0 { + pub(crate) d: f16, + pub(crate) qh: [u8; 4], + pub(crate) qs: [u8; QK5_0 / 2], +} +const _: () = assert!(std::mem::size_of::() == 22); + +#[derive(Debug, Clone, PartialEq)] +#[repr(C)] +pub struct BlockQ5_1 { + pub(crate) d: f16, + pub(crate) m: f16, + pub(crate) qh: [u8; 4], + pub(crate) qs: [u8; QK5_1 / 2], +} +const _: () = assert!(std::mem::size_of::() == 24); + +#[derive(Debug, Clone, PartialEq)] +#[repr(C)] +pub struct BlockQ8_0 { + pub(crate) d: f16, + pub(crate) qs: [i8; QK8_0], +} +const _: () = assert!(std::mem::size_of::() == 34); + +#[derive(Debug, Clone, PartialEq)] +#[repr(C)] +pub struct BlockQ8_1 { + pub(crate) d: f16, + pub(crate) s: f16, + pub(crate) qs: [i8; QK8_1], +} +const _: () = assert!(std::mem::size_of::() == 36); + +#[derive(Debug, Clone, PartialEq)] +#[repr(C)] +pub struct BlockQ2K { + pub(crate) scales: [u8; QK_K / 16], + pub(crate) qs: [u8; QK_K / 4], + pub(crate) d: f16, + pub(crate) dmin: f16, +} +const _: () = assert!(QK_K / 16 + QK_K / 4 + 2 * 2 == std::mem::size_of::()); + +#[derive(Debug, Clone, PartialEq)] +#[repr(C)] +pub struct BlockQ3K { + pub(crate) hmask: [u8; QK_K / 8], + pub(crate) qs: [u8; QK_K / 4], + pub(crate) scales: [u8; 12], + pub(crate) d: f16, +} +const _: () = assert!(QK_K / 8 + QK_K / 4 + 12 + 2 == std::mem::size_of::()); + +#[derive(Debug, Clone, PartialEq)] +// https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/k_quants.h#L82 +#[repr(C)] +pub struct BlockQ4K { + pub(crate) d: f16, + pub(crate) dmin: f16, + pub(crate) scales: [u8; K_SCALE_SIZE], + pub(crate) qs: [u8; QK_K / 2], +} +const _: () = assert!(QK_K / 2 + K_SCALE_SIZE + 2 * 2 == std::mem::size_of::()); + +#[derive(Debug, Clone, PartialEq)] +#[repr(C)] +pub struct BlockQ5K { + pub(crate) d: f16, + pub(crate) dmin: f16, + pub(crate) scales: [u8; K_SCALE_SIZE], + pub(crate) qh: [u8; QK_K / 8], + pub(crate) qs: [u8; QK_K / 2], +} +const _: () = + assert!(QK_K / 8 + QK_K / 2 + 2 * 2 + K_SCALE_SIZE == std::mem::size_of::()); + +#[derive(Debug, Clone, PartialEq)] +#[repr(C)] +pub struct BlockQ6K { + pub(crate) ql: [u8; QK_K / 2], + pub(crate) qh: [u8; QK_K / 4], + pub(crate) scales: [i8; QK_K / 16], + pub(crate) d: f16, +} +const _: () = assert!(3 * QK_K / 4 + QK_K / 16 + 2 == std::mem::size_of::()); + +#[derive(Debug, Clone, PartialEq)] +#[repr(C)] +pub struct BlockQ8K { + pub(crate) d: f32, + pub(crate) qs: [i8; QK_K], + pub(crate) bsums: [i16; QK_K / 16], +} +const _: () = assert!(4 + QK_K + QK_K / 16 * 2 == std::mem::size_of::()); + +impl GgmlType for BlockQ4_0 { + const DTYPE: GgmlDType = GgmlDType::Q4_0; + const BLCK_SIZE: usize = QK4_0; + type VecDotType = BlockQ8_0; + + // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1525 + fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { + let k = ys.len(); + let qk = Self::BLCK_SIZE; + if k % qk != 0 { + crate::bail!("dequantize_row_q4_0: {k} is not divisible by {qk}") + } + + let nb = k / qk; + for i in 0..nb { + let d = xs[i].d.to_f32(); + + for j in 0..(qk / 2) { + let x0 = (xs[i].qs[j] & 0x0F) as i16 - 8; + let x1 = (xs[i].qs[j] >> 4) as i16 - 8; + + ys[i * qk + j] = (x0 as f32) * d; + ys[i * qk + j + qk / 2] = (x1 as f32) * d; + } + } + Ok(()) + } + + fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { + // quantize_row_q4_0 + let qk = Self::BLCK_SIZE; + let k = xs.len(); + if k % qk != 0 { + crate::bail!("{k} is not divisible by {}", qk); + }; + let nb = k / qk; + if ys.len() != nb { + crate::bail!("size mismatch {} {} {}", xs.len(), ys.len(), qk,) + } + for (i, ys) in ys.iter_mut().enumerate() { + let mut amax = 0f32; + let mut max = 0f32; + + let xs = &xs[i * qk..(i + 1) * qk]; + for &x in xs.iter() { + if amax < x.abs() { + amax = x.abs(); + max = x; + } + } + let d = max / -8.0; + let id = if d != 0f32 { 1. / d } else { 0. }; + ys.d = f16::from_f32(d); + + for (j, q) in ys.qs.iter_mut().enumerate() { + let x0 = xs[j] * id; + let x1 = xs[qk / 2 + j] * id; + let xi0 = u8::min(15, (x0 + 8.5) as u8); + let xi1 = u8::min(15, (x1 + 8.5) as u8); + *q = xi0 | (xi1 << 4) + } + } + Ok(()) + } + + // https://github.com/ggerganov/llama.cpp/blob/b5ffb2849d23afe73647f68eec7b68187af09be6/ggml.c#L2361C10-L2361C122 + #[allow(unreachable_code)] + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + #[cfg(target_feature = "avx")] + return super::avx::vec_dot_q4_0_q8_0(n, xs, ys); + + #[cfg(target_feature = "neon")] + return super::neon::vec_dot_q4_0_q8_0(n, xs, ys); + + #[cfg(target_feature = "simd128")] + return super::simd128::vec_dot_q4_0_q8_0(n, xs, ys); + + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + let qk = QK8_0; + if n % QK8_0 != 0 { + crate::bail!("vec_dot_q4_0_q8_0: {n} is not divisible by {qk}") + } + // Generic implementation. + let mut sumf = 0f32; + for (xs, ys) in xs.iter().zip(ys.iter()) { + let mut sum_i = 0; + for j in 0..qk / 2 { + let v0 = (xs.qs[j] & 0x0F) as i32 - 8; + let v1 = (xs.qs[j] >> 4) as i32 - 8; + sum_i += v0 * ys.qs[j] as i32 + v1 * ys.qs[j + qk / 2] as i32 + } + sumf += sum_i as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d) + } + Ok(sumf) + } +} + +impl GgmlType for BlockQ4_1 { + const DTYPE: GgmlDType = GgmlDType::Q4_1; + const BLCK_SIZE: usize = QK4_1; + type VecDotType = BlockQ8_1; + + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + // ggml_vec_dot_q4_1_q8_1 + let qk = QK8_1; + if n % qk != 0 { + crate::bail!("vec_dot_q4_1_q8_1: {n} is not divisible by {qk}") + } + let nb = n / qk; + if nb % 2 != 0 { + crate::bail!("vec_dot_q4_1_q8_1: {n}, nb is not divisible by 2") + } + + // Generic implementation. + let mut sumf = 0f32; + + for (xs, ys) in xs.iter().zip(ys.iter()) { + let mut sumi = 0i32; + + for j in 0..qk / 2 { + let v0 = xs.qs[j] as i32 & 0x0F; + let v1 = xs.qs[j] as i32 >> 4; + sumi += (v0 * ys.qs[j] as i32) + (v1 * ys.qs[j + qk / 2] as i32); + } + + sumf += sumi as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d) + + f16::to_f32(xs.m) * f16::to_f32(ys.s) + } + Ok(sumf) + } + + fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { + // quantize_row_q4_1 + let qk = Self::BLCK_SIZE; + if ys.len() * qk != xs.len() { + crate::bail!("size mismatch {} {} {}", xs.len(), ys.len(), qk,) + } + for (i, ys) in ys.iter_mut().enumerate() { + let xs = &xs[i * qk..(i + 1) * qk]; + + let mut min = f32::INFINITY; + let mut max = f32::NEG_INFINITY; + for &x in xs.iter() { + min = f32::min(x, min); + max = f32::max(x, max); + } + let d = (max - min) / ((1 << 4) - 1) as f32; + let id = if d != 0f32 { 1. / d } else { 0. }; + ys.d = f16::from_f32(d); + ys.m = f16::from_f32(min); + + for (j, q) in ys.qs.iter_mut().take(qk / 2).enumerate() { + let x0 = (xs[j] - min) * id; + let x1 = (xs[qk / 2 + j] - min) * id; + + let xi0 = u8::min(15, (x0 + 0.5) as u8); + let xi1 = u8::min(15, (x1 + 0.5) as u8); + + *q = xi0 | (xi1 << 4); + } + } + Ok(()) + } + + // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1545 + fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { + let k = ys.len(); + if k % QK4_1 != 0 { + crate::bail!("dequantize_row_q4_1: {k} is not divisible by {QK4_1}"); + } + + let nb = k / QK4_1; + for i in 0..nb { + let d = xs[i].d.to_f32(); + let m = xs[i].m.to_f32(); + + for j in 0..(QK4_1 / 2) { + let x0 = xs[i].qs[j] & 0x0F; + let x1 = xs[i].qs[j] >> 4; + + ys[i * QK4_1 + j] = (x0 as f32) * d + m; + ys[i * QK4_1 + j + QK4_1 / 2] = (x1 as f32) * d + m; + } + } + Ok(()) + } +} + +impl GgmlType for BlockQ5_0 { + const DTYPE: GgmlDType = GgmlDType::Q5_0; + const BLCK_SIZE: usize = QK5_0; + type VecDotType = BlockQ8_0; + + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + let qk = Self::BLCK_SIZE; + if n % Self::BLCK_SIZE != 0 { + crate::bail!("vec_dot_q5_0_q8_0: {n} is not divisible by {qk}") + } + let nb = n / qk; + if nb % 2 != 0 { + crate::bail!("vec_dot_q5_0_q8_0: {n}, nb is not divisible by 2") + } + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(_n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + // Generic implementation. + let mut sumf = 0f32; + + for (xs, ys) in xs.iter().zip(ys.iter()) { + let qh = LittleEndian::read_u32(&xs.qh); + let mut sumi = 0i32; + + for j in 0..Self::BLCK_SIZE / 2 { + let xh_0 = (((qh & (1u32 << j)) >> j) << 4) as u8; + let xh_1 = ((qh & (1u32 << (j + 16))) >> (j + 12)) as u8; + + let x0 = ((xs.qs[j] & 0x0F) as i32 | xh_0 as i32) - 16; + let x1 = ((xs.qs[j] >> 4) as i32 | xh_1 as i32) - 16; + + sumi += (x0 * ys.qs[j] as i32) + (x1 * ys.qs[j + Self::BLCK_SIZE / 2] as i32); + } + + sumf += sumi as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d) + } + Ok(sumf) + } + + fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { + // quantize_row_q5_0 + let k = xs.len(); + if ys.len() * Self::BLCK_SIZE != k { + crate::bail!("size mismatch {k} {} {}", ys.len(), Self::BLCK_SIZE) + } + for (i, ys) in ys.iter_mut().enumerate() { + let xs = &xs[i * Self::BLCK_SIZE..(i + 1) * Self::BLCK_SIZE]; + + let mut amax = 0f32; + let mut max = 0f32; + for &x in xs.iter() { + if amax < x.abs() { + amax = x.abs(); + max = x; + } + } + let d = max / -16.; + let id = if d != 0f32 { 1. / d } else { 0. }; + ys.d = f16::from_f32(d); + let mut qh = 0u32; + for j in 0..Self::BLCK_SIZE / 2 { + let x0 = xs[j] * id; + let x1 = xs[j + Self::BLCK_SIZE / 2] * id; + let xi0 = ((x0 + 16.5) as i8).min(31) as u8; + let xi1 = ((x1 + 16.5) as i8).min(31) as u8; + ys.qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4); + qh |= ((xi0 as u32 & 0x10) >> 4) << j; + qh |= ((xi1 as u32 & 0x10) >> 4) << (j + Self::BLCK_SIZE / 2); + } + LittleEndian::write_u32(&mut ys.qh, qh) + } + Ok(()) + } + + // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1566 + fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { + let k = ys.len(); + if k % QK5_0 != 0 { + crate::bail!("dequantize_row_q5_0: {k} is not divisible by {QK5_0}"); + } + + let nb = k / QK5_0; + for i in 0..nb { + let d = xs[i].d.to_f32(); + let qh: u32 = LittleEndian::read_u32(&xs[i].qh); + + for j in 0..(QK5_0 / 2) { + let xh_0 = (((qh >> j) << 4) & 0x10) as u8; + let xh_1 = ((qh >> (j + 12)) & 0x10) as u8; + + let x0 = ((xs[i].qs[j] & 0x0F) | xh_0) as i32 - 16; + let x1 = ((xs[i].qs[j] >> 4) | xh_1) as i32 - 16; + + ys[i * QK5_0 + j] = (x0 as f32) * d; + ys[i * QK5_0 + j + QK5_0 / 2] = (x1 as f32) * d; + } + } + Ok(()) + } +} + +impl GgmlType for BlockQ5_1 { + const DTYPE: GgmlDType = GgmlDType::Q5_1; + const BLCK_SIZE: usize = QK5_1; + type VecDotType = BlockQ8_1; + + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + let qk = Self::BLCK_SIZE; + if n % Self::BLCK_SIZE != 0 { + crate::bail!("vec_dot_q5_1_q8_1: {n} is not divisible by {qk}") + } + let nb = n / qk; + if nb % 2 != 0 { + crate::bail!("vec_dot_q5_1_q8_1: {n}, nb is not divisible by 2") + } + + // Generic implementation. + let mut sumf = 0f32; + + for (xs, ys) in xs.iter().zip(ys.iter()) { + let qh = LittleEndian::read_u32(&xs.qh); + let mut sumi = 0i32; + + for j in 0..Self::BLCK_SIZE / 2 { + let xh_0 = ((qh >> j) << 4) & 0x10; + let xh_1 = (qh >> (j + 12)) & 0x10; + + let x0 = (xs.qs[j] as i32 & 0xF) | xh_0 as i32; + let x1 = (xs.qs[j] as i32 >> 4) | xh_1 as i32; + + sumi += (x0 * ys.qs[j] as i32) + (x1 * ys.qs[j + Self::BLCK_SIZE / 2] as i32); + } + + sumf += sumi as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d) + + f16::to_f32(xs.m) * f16::to_f32(ys.s) + } + Ok(sumf) + } + + fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { + // quantize_row_q5_1 + let qk = Self::BLCK_SIZE; + if ys.len() * qk != xs.len() { + crate::bail!("size mismatch {} {} {}", xs.len(), ys.len(), qk,) + } + for (i, ys) in ys.iter_mut().enumerate() { + let xs = &xs[i * qk..(i + 1) * qk]; + + let mut min = f32::INFINITY; + let mut max = f32::NEG_INFINITY; + for &x in xs.iter() { + min = f32::min(x, min); + max = f32::max(x, max); + } + let d = (max - min) / ((1 << 5) - 1) as f32; + let id = if d != 0f32 { 1. / d } else { 0. }; + ys.d = f16::from_f32(d); + ys.m = f16::from_f32(min); + + let mut qh = 0u32; + for (j, q) in ys.qs.iter_mut().take(qk / 2).enumerate() { + let x0 = (xs[j] - min) * id; + let x1 = (xs[qk / 2 + j] - min) * id; + + let xi0 = (x0 + 0.5) as u8; + let xi1 = (x1 + 0.5) as u8; + + *q = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4); + // get the 5-th bit and store it in qh at the right position + qh |= ((xi0 as u32 & 0x10) >> 4) << j; + qh |= ((xi1 as u32 & 0x10) >> 4) << (j + qk / 2); + } + LittleEndian::write_u32(&mut ys.qh, qh); + } + Ok(()) + } + + // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1592 + fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { + let k = ys.len(); + if k % QK5_1 != 0 { + crate::bail!("dequantize_row_q5_1: {k} is not divisible by {QK5_1}"); + } + + let nb = k / QK5_1; + for i in 0..nb { + let d = xs[i].d.to_f32(); + let m = xs[i].m.to_f32(); + let qh: u32 = LittleEndian::read_u32(&xs[i].qh); + + for j in 0..(QK5_1 / 2) { + let xh_0 = (((qh >> j) << 4) & 0x10) as u8; + let xh_1 = ((qh >> (j + 12)) & 0x10) as u8; + + let x0 = (xs[i].qs[j] & 0x0F) | xh_0; + let x1 = (xs[i].qs[j] >> 4) | xh_1; + + ys[i * QK5_1 + j] = (x0 as f32) * d + m; + ys[i * QK5_1 + j + QK5_1 / 2] = (x1 as f32) * d + m; + } + } + Ok(()) + } +} + +impl GgmlType for BlockQ8_0 { + const DTYPE: GgmlDType = GgmlDType::Q8_0; + const BLCK_SIZE: usize = QK8_0; + type VecDotType = BlockQ8_0; + + // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1619 + fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { + let k = ys.len(); + if k % QK8_0 != 0 { + crate::bail!("dequantize_row_q8_0: {k} is not divisible by {QK8_0}"); + } + + let nb = k / QK8_0; + + for i in 0..nb { + let d = xs[i].d.to_f32(); + + for j in 0..QK8_0 { + ys[i * QK8_0 + j] = xs[i].qs[j] as f32 * d; + } + } + Ok(()) + } + + fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { + // quantize_row_q8_0 + let k = xs.len(); + if k % Self::BLCK_SIZE != 0 { + crate::bail!("{k} is not divisible by {}", Self::BLCK_SIZE); + }; + let nb = k / Self::BLCK_SIZE; + if ys.len() != nb { + crate::bail!( + "size mismatch {} {} {}", + xs.len(), + ys.len(), + Self::BLCK_SIZE + ) + } + for (i, ys) in ys.iter_mut().enumerate() { + let mut amax = 0f32; + let xs = &xs[i * Self::BLCK_SIZE..(i + 1) * Self::BLCK_SIZE]; + for &x in xs.iter() { + amax = amax.max(x.abs()) + } + let d = amax / ((1 << 7) - 1) as f32; + let id = if d != 0f32 { 1. / d } else { 0. }; + ys.d = f16::from_f32(d); + for (y, &x) in ys.qs.iter_mut().zip(xs.iter()) { + *y = f32::round(x * id) as i8 + } + } + Ok(()) + } + + #[allow(unreachable_code)] + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + #[cfg(target_feature = "avx")] + return super::avx::vec_dot_q8_0_q8_0(n, xs, ys); + + #[cfg(target_feature = "neon")] + return super::neon::vec_dot_q8_0_q8_0(n, xs, ys); + + #[cfg(target_feature = "simd128")] + return super::simd128::vec_dot_q8_0_q8_0(n, xs, ys); + + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + let qk = QK8_0; + if n % QK8_0 != 0 { + crate::bail!("vec_dot_q8_0_q8_0: {n} is not divisible by {qk}") + } + + // Generic implementation. + let mut sumf = 0f32; + for (xs, ys) in xs.iter().zip(ys.iter()) { + let sum_i = xs + .qs + .iter() + .zip(ys.qs.iter()) + .map(|(&x, &y)| x as i32 * y as i32) + .sum::(); + sumf += sum_i as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d) + } + Ok(sumf) + } +} + +impl GgmlType for BlockQ8_1 { + const DTYPE: GgmlDType = GgmlDType::Q8_1; + const BLCK_SIZE: usize = QK8_1; + type VecDotType = BlockQ8_1; + + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(_n: usize, _xs: &[Self], _ys: &[Self::VecDotType]) -> Result { + unimplemented!("no support for vec-dot on Q8_1") + } + + fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { + // quantize_row_q8_1 + let k = xs.len(); + if ys.len() * Self::BLCK_SIZE != k { + crate::bail!("size mismatch {k} {} {}", ys.len(), Self::BLCK_SIZE) + } + for (i, ys) in ys.iter_mut().enumerate() { + let mut amax = 0f32; + let xs = &xs[i * Self::BLCK_SIZE..(i + 1) * Self::BLCK_SIZE]; + for &x in xs.iter() { + amax = amax.max(x.abs()) + } + let d = amax / ((1 << 7) - 1) as f32; + let id = if d != 0f32 { 1. / d } else { 0. }; + ys.d = f16::from_f32(d); + let mut sum = 0i32; + for j in 0..Self::BLCK_SIZE / 2 { + let v0 = xs[j] * id; + let v1 = xs[j + Self::BLCK_SIZE / 2] * id; + ys.qs[j] = f32::round(v0) as i8; + ys.qs[j + Self::BLCK_SIZE / 2] = f32::round(v1) as i8; + sum += ys.qs[j] as i32 + ys.qs[j + Self::BLCK_SIZE / 2] as i32; + } + ys.s = f16::from_f32(sum as f32) * ys.d; + } + Ok(()) + } + + fn to_float(_xs: &[Self], _ys: &mut [f32]) -> Result<()> { + unimplemented!("no support for vec-dot on Q8_1") + } +} + +impl GgmlType for BlockQ2K { + const DTYPE: GgmlDType = GgmlDType::Q2K; + const BLCK_SIZE: usize = QK_K; + type VecDotType = BlockQ8K; + + #[allow(unreachable_code)] + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + #[cfg(target_feature = "avx")] + return super::avx::vec_dot_q2k_q8k(n, xs, ys); + + #[cfg(target_feature = "neon")] + return super::neon::vec_dot_q2k_q8k(n, xs, ys); + + #[cfg(target_feature = "simd128")] + return super::simd128::vec_dot_q2k_q8k(n, xs, ys); + + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + if n % QK_K != 0 { + crate::bail!("vec_dot_q2k_q8k: {n} is not divisible by {QK_K}") + } + + let mut sumf = 0.0; + for (x, y) in xs.iter().zip(ys.iter()) { + let mut q2: &[_] = &x.qs; + let mut q8: &[_] = &y.qs; + let sc = &x.scales; + + let mut summs = 0; + for (bsum, scale) in y.bsums.iter().zip(sc) { + summs += *bsum as i32 * ((scale >> 4) as i32); + } + + let dall = y.d * x.d.to_f32(); + let dmin = y.d * x.dmin.to_f32(); + + let mut isum = 0; + let mut is = 0; + for _ in 0..(QK_K / 128) { + let mut shift = 0; + for _ in 0..4 { + let d = (sc[is] & 0xF) as i32; + is += 1; + let mut isuml = 0; + for l in 0..16 { + isuml += q8[l] as i32 * (((q2[l] >> shift) & 3) as i32); + } + isum += d * isuml; + let d = (sc[is] & 0xF) as i32; + is += 1; + isuml = 0; + for l in 16..32 { + isuml += q8[l] as i32 * (((q2[l] >> shift) & 3) as i32); + } + isum += d * isuml; + shift += 2; + // adjust the indexing + q8 = &q8[32..]; + } + // adjust the indexing + q2 = &q2[32..]; + } + sumf += dall * isum as f32 - dmin * summs as f32; + } + + Ok(sumf) + } + + // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L279 + fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { + const Q4SCALE: f32 = 15.0; + + for (block, x) in group_for_quantization(xs, ys)? { + //calculate scales and mins + let mut mins: [f32; QK_K / 16] = [0.0; QK_K / 16]; + let mut scales: [f32; QK_K / 16] = [0.0; QK_K / 16]; + + for (j, x_scale_slice) in x.chunks(16).enumerate() { + (scales[j], mins[j]) = make_qkx1_quants(3, 5, x_scale_slice); + } + // get max scale and max min and ensure they are >= 0.0 + let max_scale = scales.iter().fold(0.0, |max, &val| val.max(max)); + let max_min = mins.iter().fold(0.0, |max, &val| val.max(max)); + + if max_scale > 0.0 { + let iscale = Q4SCALE / max_scale; + for (j, scale) in scales.iter().enumerate().take(QK_K / 16) { + block.scales[j] = nearest_int(iscale * scale) as u8; + } + block.d = f16::from_f32(max_scale / Q4SCALE); + } else { + for j in 0..QK_K / 16 { + block.scales[j] = 0; + } + block.d = f16::from_f32(0.0); + } + + if max_min > 0.0 { + let iscale = Q4SCALE / max_min; + for (j, scale) in block.scales.iter_mut().enumerate() { + let l = nearest_int(iscale * mins[j]) as u8; + *scale |= l << 4; + } + block.dmin = f16::from_f32(max_min / Q4SCALE); + } else { + block.dmin = f16::from_f32(0.0); + } + + let mut big_l: [u8; QK_K] = [0; QK_K]; + + for j in 0..QK_K / 16 { + let d = block.d.to_f32() * (block.scales[j] & 0xF) as f32; + if d == 0.0 { + continue; + } + let dm = block.dmin.to_f32() * (block.scales[j] >> 4) as f32; + for ii in 0..16 { + let ll = nearest_int((x[16 * j + ii] + dm) / d).clamp(0, 3); + big_l[16 * j + ii] = ll as u8; + } + } + + for j in (0..QK_K).step_by(128) { + for ll in 0..32 { + block.qs[j / 4 + ll] = big_l[j + ll] + | (big_l[j + ll + 32] << 2) + | (big_l[j + ll + 64] << 4) + | (big_l[j + ll + 96] << 6); + } + } + } + Ok(()) + } + // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L354 + fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { + for (block, y) in group_for_dequantization(xs, ys)? { + let d = block.d.to_f32(); + let min = block.dmin.to_f32(); + + let mut is = 0; + + for (y_block, qs) in y.chunks_exact_mut(128).zip(block.qs.chunks_exact(32)) { + // Step by 32 over q. + let mut shift = 0; + let mut y_block_index = 0; + for _j in 0..4 { + let sc = block.scales[is]; + is += 1; + let dl = d * (sc & 0xF) as f32; + let ml = min * (sc >> 4) as f32; + for q in &qs[..16] { + let y = dl * ((q >> shift) & 3) as f32 - ml; + y_block[y_block_index] = y; + y_block_index += 1; + } + + let sc = block.scales[is]; + is += 1; + let dl = d * (sc & 0xF) as f32; + let ml = min * (sc >> 4) as f32; + for q in &qs[16..] { + let y = dl * ((q >> shift) & 3) as f32 - ml; + y_block[y_block_index] = y; + y_block_index += 1; + } + + shift += 2; + } + } + } + Ok(()) + } +} + +impl GgmlType for BlockQ3K { + const DTYPE: GgmlDType = GgmlDType::Q3K; + const BLCK_SIZE: usize = QK_K; + type VecDotType = BlockQ8K; + + #[allow(unreachable_code)] + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + #[cfg(target_feature = "avx")] + return super::avx::vec_dot_q3k_q8k(n, xs, ys); + + #[cfg(target_feature = "neon")] + return super::neon::vec_dot_q3k_q8k(n, xs, ys); + + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + if n % QK_K != 0 { + crate::bail!("vec_dot_q3k_q8k: {n} is not divisible by {QK_K}") + } + + const KMASK1: u32 = 0x03030303; + const KMASK2: u32 = 0x0f0f0f0f; + + let mut aux8: [i8; QK_K] = [0; QK_K]; + let mut aux16: [i16; 8] = [0; 8]; + let mut sums: [f32; 8] = [0.0; 8]; + let mut aux32: [i32; 8] = [0; 8]; + + let mut auxs: [u32; 4] = [0; 4]; + + for (x, y) in xs.iter().zip(ys.iter()) { + let mut q3: &[u8] = &x.qs; + let hmask: &[u8] = &x.hmask; + let mut q8: &[i8] = &y.qs; + + aux32.fill(0); + let mut a = &mut aux8[..]; + + let mut m = 1; + //Like the GGML original this is written this way to enable the compiler to vectorize it. + for _ in 0..QK_K / 128 { + a.iter_mut() + .take(32) + .zip(q3) + .for_each(|(a_val, q3_val)| *a_val = (q3_val & 3) as i8); + a.iter_mut() + .take(32) + .zip(hmask) + .for_each(|(a_val, hmask_val)| { + *a_val -= if hmask_val & m != 0 { 0 } else { 4 } + }); + a = &mut a[32..]; + m <<= 1; + + a.iter_mut() + .take(32) + .zip(q3) + .for_each(|(a_val, q3_val)| *a_val = ((q3_val >> 2) & 3) as i8); + a.iter_mut() + .take(32) + .zip(hmask) + .for_each(|(a_val, hmask_val)| { + *a_val -= if hmask_val & m != 0 { 0 } else { 4 } + }); + a = &mut a[32..]; + m <<= 1; + + a.iter_mut() + .take(32) + .zip(q3) + .for_each(|(a_val, q3_val)| *a_val = ((q3_val >> 4) & 3) as i8); + a.iter_mut() + .take(32) + .zip(hmask) + .for_each(|(a_val, hmask_val)| { + *a_val -= if hmask_val & m != 0 { 0 } else { 4 } + }); + a = &mut a[32..]; + m <<= 1; + + a.iter_mut() + .take(32) + .zip(q3) + .for_each(|(a_val, q3_val)| *a_val = ((q3_val >> 6) & 3) as i8); + a.iter_mut() + .take(32) + .zip(hmask) + .for_each(|(a_val, hmask_val)| { + *a_val -= if hmask_val & m != 0 { 0 } else { 4 } + }); + a = &mut a[32..]; + m <<= 1; + q3 = &q3[32..]; + } + + a = &mut aux8[..]; + + LittleEndian::read_u32_into(&x.scales, &mut auxs[0..3]); + + let tmp = auxs[2]; + auxs[2] = ((auxs[0] >> 4) & KMASK2) | (((tmp >> 4) & KMASK1) << 4); + auxs[3] = ((auxs[1] >> 4) & KMASK2) | (((tmp >> 6) & KMASK1) << 4); + auxs[0] = (auxs[0] & KMASK2) | (((tmp) & KMASK1) << 4); + auxs[1] = (auxs[1] & KMASK2) | (((tmp >> 2) & KMASK1) << 4); + + for aux in auxs { + for scale in aux.to_le_bytes() { + let scale = i8::from_be_bytes([scale]); + for l in 0..8 { + aux16[l] = q8[l] as i16 * a[l] as i16; + } + for l in 0..8 { + aux32[l] += (scale as i32 - 32) * aux16[l] as i32; + } + q8 = &q8[8..]; + a = &mut a[8..]; + + for l in 0..8 { + aux16[l] = q8[l] as i16 * a[l] as i16; + } + for l in 0..8 { + aux32[l] += (scale as i32 - 32) * aux16[l] as i32; + } + q8 = &q8[8..]; + a = &mut a[8..]; + } + } + let d = x.d.to_f32() * y.d; + for l in 0..8 { + sums[l] += d * aux32[l] as f32; + } + } + + Ok(sums.iter().sum()) + } + + fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { + for (block, x) in group_for_quantization(xs, ys)? { + let mut scales: [f32; QK_K / 16] = [0.0; QK_K / 16]; + for (j, x_scale_slice) in x.chunks_exact(16).enumerate() { + scales[j] = make_q3_quants(x_scale_slice, 4, true); + } + + // Get max scale by absolute value. + let mut max_scale: f32 = 0.0; + for &scale in scales.iter() { + if scale.abs() > max_scale.abs() { + max_scale = scale; + } + } + + block.scales.fill(0); + + if max_scale != 0.0 { + let iscale = -32.0 / max_scale; + for (j, scale) in scales.iter().enumerate() { + let l_val = nearest_int(iscale * scale); + let l_val = l_val.clamp(-32, 31) + 32; + if j < 8 { + block.scales[j] = (l_val & 0xF) as u8; + } else { + block.scales[j - 8] |= ((l_val & 0xF) << 4) as u8; + } + let l_val = l_val >> 4; + block.scales[j % 4 + 8] |= (l_val << (2 * (j / 4))) as u8; + } + block.d = f16::from_f32(1.0 / iscale); + } else { + block.d = f16::from_f32(0.0); + } + + let mut l: [i8; QK_K] = [0; QK_K]; + + for j in 0..QK_K / 16 { + let sc = if j < 8 { + block.scales[j] & 0xF + } else { + block.scales[j - 8] >> 4 + }; + let sc = (sc | (((block.scales[8 + j % 4] >> (2 * (j / 4))) & 3) << 4)) as i8 - 32; + let d = block.d.to_f32() * sc as f32; + if d != 0.0 { + for ii in 0..16 { + let l_val = nearest_int(x[16 * j + ii] / d); + l[16 * j + ii] = (l_val.clamp(-4, 3) + 4) as i8; + } + } + } + + block.hmask.fill(0); + let mut m = 0; + let mut hm = 1; + + for ll in l.iter_mut() { + if *ll > 3 { + block.hmask[m] |= hm; + *ll -= 4; + } + m += 1; + if m == QK_K / 8 { + m = 0; + hm <<= 1; + } + } + + for j in (0..QK_K).step_by(128) { + for l_val in 0..32 { + block.qs[j / 4 + l_val] = (l[j + l_val] + | (l[j + l_val + 32] << 2) + | (l[j + l_val + 64] << 4) + | (l[j + l_val + 96] << 6)) + as u8; + } + } + } + + Ok(()) + } + + // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L533 + fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { + const KMASK1: u32 = 0x03030303; + const KMASK2: u32 = 0x0f0f0f0f; + + for (block, y) in group_for_dequantization(xs, ys)? { + //Reconstruct the scales + let mut aux = [0; 4]; + LittleEndian::read_u32_into(&block.scales, &mut aux[0..3]); + + let tmp = aux[2]; + aux[2] = ((aux[0] >> 4) & KMASK2) | (((tmp >> 4) & KMASK1) << 4); + aux[3] = ((aux[1] >> 4) & KMASK2) | (((tmp >> 6) & KMASK1) << 4); + aux[0] = (aux[0] & KMASK2) | (((tmp) & KMASK1) << 4); + aux[1] = (aux[1] & KMASK2) | (((tmp >> 2) & KMASK1) << 4); + + //Transfer the scales into an i8 array + let scales: &mut [i8] = + unsafe { std::slice::from_raw_parts_mut(aux.as_mut_ptr() as *mut i8, 16) }; + + let d_all = block.d.to_f32(); + let mut m = 1; + let mut is = 0; + + // Dequantize both 128 long blocks + // 32 qs values per 128 long block + // Each 16 elements get a scale + for (y, qs) in y.chunks_exact_mut(128).zip(block.qs.chunks_exact(32)) { + let mut shift = 0; + for shift_scoped_y in y.chunks_exact_mut(32) { + for (scale_index, scale_scoped_y) in + shift_scoped_y.chunks_exact_mut(16).enumerate() + { + let dl = d_all * (scales[is] as f32 - 32.0); + for (i, inner_y) in scale_scoped_y.iter_mut().enumerate() { + let new_y = dl + * (((qs[i + 16 * scale_index] >> shift) & 3) as i8 + - if (block.hmask[i + 16 * scale_index] & m) == 0 { + 4 + } else { + 0 + }) as f32; + *inner_y = new_y; + } + // 16 block finished => advance scale index + is += 1; + } + // 32 block finished => increase shift and m + shift += 2; + m <<= 1; + } + } + } + + Ok(()) + } +} + +impl GgmlType for BlockQ4K { + const DTYPE: GgmlDType = GgmlDType::Q4K; + const BLCK_SIZE: usize = QK_K; + type VecDotType = BlockQ8K; + + #[allow(unreachable_code)] + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + #[cfg(target_feature = "avx")] + return super::avx::vec_dot_q4k_q8k(n, xs, ys); + + #[cfg(target_feature = "neon")] + return super::neon::vec_dot_q4k_q8k(n, xs, ys); + + #[cfg(target_feature = "simd128")] + return super::simd128::vec_dot_q4k_q8k(n, xs, ys); + + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + if n % QK_K != 0 { + crate::bail!("vec_dot_q4k_q8k: {n} is not divisible by {QK_K}") + } + + const KMASK1: u32 = 0x3f3f3f3f; + const KMASK2: u32 = 0x0f0f0f0f; + const KMASK3: u32 = 0x03030303; + + let mut utmp: [u32; 4] = [0; 4]; + let mut scales: [u8; 8] = [0; 8]; + let mut mins: [u8; 8] = [0; 8]; + + let mut aux8: [i8; QK_K] = [0; QK_K]; + let mut aux16: [i16; 8] = [0; 8]; + let mut sums: [f32; 8] = [0.0; 8]; + let mut aux32: [i32; 8] = [0; 8]; + + let mut sumf = 0.0; + for (y, x) in ys.iter().zip(xs.iter()) { + let q4 = &x.qs; + let q8 = &y.qs; + aux32.fill(0); + + let mut a = &mut aux8[..]; + let mut q4 = &q4[..]; + for _ in 0..QK_K / 64 { + for l in 0..32 { + a[l] = (q4[l] & 0xF) as i8; + } + a = &mut a[32..]; + for l in 0..32 { + a[l] = (q4[l] >> 4) as i8; + } + a = &mut a[32..]; + q4 = &q4[32..]; + } + + LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); + + utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); + let uaux = utmp[1] & KMASK1; + utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); + utmp[2] = uaux; + utmp[0] &= KMASK1; + + //extract scales and mins + LittleEndian::write_u32_into(&utmp[0..2], &mut scales); + LittleEndian::write_u32_into(&utmp[2..4], &mut mins); + + let mut sumi = 0; + for j in 0..QK_K / 16 { + sumi += y.bsums[j] as i32 * mins[j / 2] as i32; + } + + let mut a = &mut aux8[..]; + let mut q8 = &q8[..]; + + for scale in scales { + let scale = scale as i32; + for _ in 0..4 { + for l in 0..8 { + aux16[l] = q8[l] as i16 * a[l] as i16; + } + for l in 0..8 { + aux32[l] += scale * aux16[l] as i32; + } + q8 = &q8[8..]; + a = &mut a[8..]; + } + } + let d = x.d.to_f32() * y.d; + for l in 0..8 { + sums[l] += d * aux32[l] as f32; + } + let dmin = x.dmin.to_f32() * y.d; + sumf -= dmin * sumi as f32; + } + Ok(sumf + sums.iter().sum::()) + } + + fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { + for (block, x) in group_for_quantization(xs, ys)? { + let mut mins: [f32; QK_K / 32] = [0.0; QK_K / 32]; + let mut scales: [f32; QK_K / 32] = [0.0; QK_K / 32]; + + for (j, x_scale_slice) in x.chunks_exact(32).enumerate() { + (scales[j], mins[j]) = make_qkx1_quants(15, 5, x_scale_slice); + } + + // get max scale and max min and ensure they are >= 0.0 + let max_scale = scales.iter().fold(0.0, |max, &val| val.max(max)); + let max_min = mins.iter().fold(0.0, |max, &val| val.max(max)); + + let inv_scale = if max_scale > 0.0 { + 63.0 / max_scale + } else { + 0.0 + }; + let inv_min = if max_min > 0.0 { 63.0 / max_min } else { 0.0 }; + + for j in 0..QK_K / 32 { + let ls = nearest_int(inv_scale * scales[j]).min(63) as u8; + let lm = nearest_int(inv_min * mins[j]).min(63) as u8; + if j < 4 { + block.scales[j] = ls; + block.scales[j + 4] = lm; + } else { + block.scales[j + 4] = (ls & 0xF) | ((lm & 0xF) << 4); + block.scales[j - 4] |= (ls >> 4) << 6; + block.scales[j] |= (lm >> 4) << 6; + } + } + + block.d = f16::from_f32(max_scale / 63.0); + block.dmin = f16::from_f32(max_min / 63.0); + + let mut l: [u8; QK_K] = [0; QK_K]; + + for j in 0..QK_K / 32 { + let (sc, m) = get_scale_min_k4(j, &block.scales); + let d = block.d.to_f32() * sc as f32; + if d != 0.0 { + let dm = block.dmin.to_f32() * m as f32; + for ii in 0..32 { + let l_val = nearest_int((x[32 * j + ii] + dm) / d); + l[32 * j + ii] = l_val.clamp(0, 15) as u8; + } + } + } + + let q = &mut block.qs; + for j in (0..QK_K).step_by(64) { + for l_val in 0..32 { + let offset_index = (j / 64) * 32 + l_val; + q[offset_index] = l[j + l_val] | (l[j + l_val + 32] << 4); + } + } + } + Ok(()) + } + // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L735 + fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { + for (block, y) in group_for_dequantization(xs, ys)? { + let d = block.d.to_f32(); + let min = block.dmin.to_f32(); + let q = &block.qs; + let mut is = 0; + let mut ys_index = 0; + + for j in (0..QK_K).step_by(64) { + let q = &q[j / 2..j / 2 + 32]; + let (sc, m) = get_scale_min_k4(is, &block.scales); + let d1 = d * sc as f32; + let m1 = min * m as f32; + let (sc, m) = get_scale_min_k4(is + 1, &block.scales); + let d2 = d * sc as f32; + let m2 = min * m as f32; + for q in q { + y[ys_index] = d1 * (q & 0xF) as f32 - m1; + ys_index += 1; + } + for q in q { + y[ys_index] = d2 * (q >> 4) as f32 - m2; + ys_index += 1; + } + is += 2; + } + } + Ok(()) + } +} + +// https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L928 +impl GgmlType for BlockQ5K { + const DTYPE: GgmlDType = GgmlDType::Q5K; + const BLCK_SIZE: usize = QK_K; + type VecDotType = BlockQ8K; + + #[allow(unreachable_code)] + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + #[cfg(target_feature = "avx")] + return super::avx::vec_dot_q5k_q8k(n, xs, ys); + + #[cfg(target_feature = "neon")] + return super::neon::vec_dot_q5k_q8k(n, xs, ys); + + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + if n % QK_K != 0 { + crate::bail!("vec_dot_q5k_q8k: {n} is not divisible by {QK_K}") + } + + const KMASK1: u32 = 0x3f3f3f3f; + const KMASK2: u32 = 0x0f0f0f0f; + const KMASK3: u32 = 0x03030303; + + let mut utmp: [u32; 4] = [0; 4]; + let mut scales: [u8; 8] = [0; 8]; + let mut mins: [u8; 8] = [0; 8]; + + let mut aux8: [i8; QK_K] = [0; QK_K]; + let mut aux16: [i16; 8] = [0; 8]; + let mut sums: [f32; 8] = [0.0; 8]; + let mut aux32: [i32; 8] = [0; 8]; + + let mut sumf = 0.0; + for (y, x) in ys.iter().zip(xs.iter()) { + let q5 = &x.qs; + let hm = &x.qh; + let q8 = &y.qs; + aux32.fill(0); + + let mut a = &mut aux8[..]; + let mut q5 = &q5[..]; + let mut m = 1u8; + + for _ in 0..QK_K / 64 { + for l in 0..32 { + a[l] = (q5[l] & 0xF) as i8; + a[l] += if hm[l] & m != 0 { 16 } else { 0 }; + } + a = &mut a[32..]; + m <<= 1; + for l in 0..32 { + a[l] = (q5[l] >> 4) as i8; + a[l] += if hm[l] & m != 0 { 16 } else { 0 }; + } + a = &mut a[32..]; + m <<= 1; + q5 = &q5[32..]; + } + + LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); + + utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); + let uaux = utmp[1] & KMASK1; + utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); + utmp[2] = uaux; + utmp[0] &= KMASK1; + + //extract scales and mins + LittleEndian::write_u32_into(&utmp[0..2], &mut scales); + LittleEndian::write_u32_into(&utmp[2..4], &mut mins); + + let mut sumi = 0; + for j in 0..QK_K / 16 { + sumi += y.bsums[j] as i32 * mins[j / 2] as i32; + } + + let mut a = &mut aux8[..]; + let mut q8 = &q8[..]; + + for scale in scales { + let scale = scale as i32; + for _ in 0..4 { + for l in 0..8 { + aux16[l] = q8[l] as i16 * a[l] as i16; + } + for l in 0..8 { + aux32[l] += scale * aux16[l] as i32; + } + q8 = &q8[8..]; + a = &mut a[8..]; + } + } + let d = x.d.to_f32() * y.d; + for l in 0..8 { + sums[l] += d * aux32[l] as f32; + } + let dmin = x.dmin.to_f32() * y.d; + sumf -= dmin * sumi as f32; + } + Ok(sumf + sums.iter().sum::()) + } + + // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L793 + fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { + for (block, x) in group_for_quantization(xs, ys)? { + let mut mins: [f32; QK_K / 32] = [0.0; QK_K / 32]; + let mut scales: [f32; QK_K / 32] = [0.0; QK_K / 32]; + + for (j, x_scale_slice) in x.chunks_exact(32).enumerate() { + (scales[j], mins[j]) = make_qkx1_quants(31, 5, x_scale_slice); + } + + // get max scale and max min and ensure they are >= 0.0 + let max_scale = scales.iter().fold(0.0, |max, &val| val.max(max)); + let max_min = mins.iter().fold(0.0, |max, &val| val.max(max)); + + let inv_scale = if max_scale > 0.0 { + 63.0 / max_scale + } else { + 0.0 + }; + let inv_min = if max_min > 0.0 { 63.0 / max_min } else { 0.0 }; + for j in 0..QK_K / 32 { + let ls = nearest_int(inv_scale * scales[j]).min(63) as u8; + let lm = nearest_int(inv_min * mins[j]).min(63) as u8; + if j < 4 { + block.scales[j] = ls; + block.scales[j + 4] = lm; + } else { + block.scales[j + 4] = (ls & 0xF) | ((lm & 0xF) << 4); + block.scales[j - 4] |= (ls >> 4) << 6; + block.scales[j] |= (lm >> 4) << 6; + } + } + block.d = f16::from_f32(max_scale / 63.0); + block.dmin = f16::from_f32(max_min / 63.0); + + let mut l: [u8; QK_K] = [0; QK_K]; + for j in 0..QK_K / 32 { + let (sc, m) = get_scale_min_k4(j, &block.scales); + let d = block.d.to_f32() * sc as f32; + if d == 0.0 { + continue; + } + let dm = block.dmin.to_f32() * m as f32; + for ii in 0..32 { + let ll = nearest_int((x[32 * j + ii] + dm) / d); + l[32 * j + ii] = ll.clamp(0, 31) as u8; + } + } + + let qh = &mut block.qh; + let ql = &mut block.qs; + qh.fill(0); + + let mut m1 = 1; + let mut m2 = 2; + for n in (0..QK_K).step_by(64) { + let offset = (n / 64) * 32; + for j in 0..32 { + let mut l1 = l[n + j]; + if l1 > 15 { + l1 -= 16; + qh[j] |= m1; + } + let mut l2 = l[n + j + 32]; + if l2 > 15 { + l2 -= 16; + qh[j] |= m2; + } + ql[offset + j] = l1 | (l2 << 4); + } + m1 <<= 2; + m2 <<= 2; + } + } + + Ok(()) + } + + // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L928 + fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { + for (block, y) in group_for_dequantization(xs, ys)? { + let d = block.d.to_f32(); + let min = block.dmin.to_f32(); + let ql = &block.qs; + let qh = &block.qh; + let mut is = 0; + let mut u1 = 1; + let mut u2 = 2; + let mut ys_index = 0; + + for j in (0..QK_K).step_by(64) { + let ql = &ql[j / 2..j / 2 + 32]; + let (sc, m) = get_scale_min_k4(is, &block.scales); + let d1 = d * sc as f32; + let m1 = min * m as f32; + let (sc, m) = get_scale_min_k4(is + 1, &block.scales); + let d2 = d * sc as f32; + let m2 = min * m as f32; + for (ql, qh) in ql.iter().zip(qh) { + let to_add = if qh & u1 != 0 { 16f32 } else { 0f32 }; + y[ys_index] = d1 * ((ql & 0xF) as f32 + to_add) - m1; + ys_index += 1; + } + for (ql, qh) in ql.iter().zip(qh) { + let to_add = if qh & u2 != 0 { 16f32 } else { 0f32 }; + y[ys_index] = d2 * ((ql >> 4) as f32 + to_add) - m2; + ys_index += 1; + } + is += 2; + u1 <<= 2; + u2 <<= 2; + } + } + Ok(()) + } +} + +impl GgmlType for BlockQ6K { + const DTYPE: GgmlDType = GgmlDType::Q6K; + const BLCK_SIZE: usize = QK_K; + type VecDotType = BlockQ8K; + + #[allow(unreachable_code)] + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + #[cfg(target_feature = "avx")] + return super::avx::vec_dot_q6k_q8k(n, xs, ys); + + #[cfg(target_feature = "neon")] + return super::neon::vec_dot_q6k_q8k(n, xs, ys); + + #[cfg(target_feature = "simd128")] + return super::simd128::vec_dot_q6k_q8k(n, xs, ys); + + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + if n % QK_K != 0 { + crate::bail!("vec_dot_q6k_q8k: {n} is not divisible by {QK_K}") + } + + let mut aux8 = [0i8; QK_K]; + let mut aux16 = [0i16; 8]; + let mut sums = [0f32; 8]; + let mut aux32 = [0f32; 8]; + + for (x, y) in xs.iter().zip(ys.iter()) { + let q4 = &x.ql; + let qh = &x.qh; + let q8 = &y.qs; + aux32.fill(0f32); + + for j in (0..QK_K).step_by(128) { + let aux8 = &mut aux8[j..]; + let q4 = &q4[j / 2..]; + let qh = &qh[j / 4..]; + for l in 0..32 { + aux8[l] = (((q4[l] & 0xF) | ((qh[l] & 3) << 4)) as i32 - 32) as i8; + aux8[l + 32] = + (((q4[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) as i32 - 32) as i8; + aux8[l + 64] = (((q4[l] >> 4) | (((qh[l] >> 4) & 3) << 4)) as i32 - 32) as i8; + aux8[l + 96] = + (((q4[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) as i32 - 32) as i8; + } + } + + for (j, &scale) in x.scales.iter().enumerate() { + let scale = scale as f32; + let q8 = &q8[16 * j..]; + let aux8 = &aux8[16 * j..]; + for l in 0..8 { + aux16[l] = q8[l] as i16 * aux8[l] as i16; + } + for l in 0..8 { + aux32[l] += scale * aux16[l] as f32 + } + let q8 = &q8[8..]; + let aux8 = &aux8[8..]; + for l in 0..8 { + aux16[l] = q8[l] as i16 * aux8[l] as i16; + } + for l in 0..8 { + aux32[l] += scale * aux16[l] as f32 + } + } + + let d = x.d.to_f32() * y.d; + for (sum, &a) in sums.iter_mut().zip(aux32.iter()) { + *sum += a * d; + } + } + Ok(sums.iter().sum()) + } + + fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { + if xs.len() != ys.len() * Self::BLCK_SIZE { + crate::bail!( + "quantize_row_q6k: size mismatch {} {} {}", + xs.len(), + ys.len(), + Self::BLCK_SIZE + ) + } + let mut l = [0i8; QK_K]; + let mut scales = [0f32; QK_K / 16]; + let mut x = xs.as_ptr(); + let l = l.as_mut_ptr(); + unsafe { + for y in ys.iter_mut() { + let mut max_scale = 0f32; + let mut max_abs_scale = 0f32; + for (ib, scale_) in scales.iter_mut().enumerate() { + let scale = make_qx_quants(16, 32, x.add(16 * ib), l.add(16 * ib), 1); + *scale_ = scale; + let abs_scale = scale.abs(); + if abs_scale > max_abs_scale { + max_abs_scale = abs_scale; + max_scale = scale + } + } + + let iscale = -128f32 / max_scale; + y.d = f16::from_f32(1.0 / iscale); + + for (y_scale, scale) in y.scales.iter_mut().zip(scales.iter()) { + *y_scale = nearest_int(iscale * scale).min(127) as i8 + } + + for (j, &y_scale) in y.scales.iter().enumerate() { + let d = y.d.to_f32() * y_scale as f32; + if d == 0. { + continue; + } + for ii in 0..16 { + let ll = nearest_int(*x.add(16 * j + ii) / d).clamp(-32, 31); + *l.add(16 * j + ii) = (ll + 32) as i8 + } + } + + let mut ql = y.ql.as_mut_ptr(); + let mut qh = y.qh.as_mut_ptr(); + + for j in (0..QK_K).step_by(128) { + for l_idx in 0..32 { + let q1 = *l.add(j + l_idx) & 0xF; + let q2 = *l.add(j + l_idx + 32) & 0xF; + let q3 = *l.add(j + l_idx + 64) & 0xF; + let q4 = *l.add(j + l_idx + 96) & 0xF; + *ql.add(l_idx) = (q1 | (q3 << 4)) as u8; + *ql.add(l_idx + 32) = (q2 | (q4 << 4)) as u8; + *qh.add(l_idx) = ((*l.add(j + l_idx) >> 4) + | ((*l.add(j + l_idx + 32) >> 4) << 2) + | ((*l.add(j + l_idx + 64) >> 4) << 4) + | ((*l.add(j + l_idx + 96) >> 4) << 6)) + as u8; + } + ql = ql.add(64); + qh = qh.add(32); + } + + x = x.add(QK_K) + } + } + Ok(()) + } + + // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L1067 + fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { + let k = ys.len(); + if k % QK_K != 0 { + crate::bail!("dequantize_row_q6k: {k} is not divisible by {QK_K}") + } + for (idx_x, x) in xs.iter().enumerate() { + let d = x.d.to_f32(); + let ql = &x.ql; + let qh = &x.qh; + let sc = &x.scales; + for n in (0..QK_K).step_by(128) { + let idx = n / 128; + let ys = &mut ys[idx_x * QK_K + n..]; + let sc = &sc[8 * idx..]; + let ql = &ql[64 * idx..]; + let qh = &qh[32 * idx..]; + for l in 0..32 { + let is = l / 16; + let q1 = ((ql[l] & 0xF) | ((qh[l] & 3) << 4)) as i8 - 32; + let q2 = ((ql[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) as i8 - 32; + let q3 = ((ql[l] >> 4) | (((qh[l] >> 4) & 3) << 4)) as i8 - 32; + let q4 = ((ql[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) as i8 - 32; + ys[l] = d * sc[is] as f32 * q1 as f32; + ys[l + 32] = d * sc[is + 2] as f32 * q2 as f32; + ys[l + 64] = d * sc[is + 4] as f32 * q3 as f32; + ys[l + 96] = d * sc[is + 6] as f32 * q4 as f32; + } + } + } + Ok(()) + } +} + +impl GgmlType for BlockQ8K { + const DTYPE: GgmlDType = GgmlDType::Q8K; + const BLCK_SIZE: usize = QK_K; + type VecDotType = BlockQ8K; + + #[allow(unreachable_code)] + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + #[cfg(target_feature = "avx")] + return super::avx::vec_dot_q8k_q8k(n, xs, ys); + + #[cfg(target_feature = "neon")] + return super::neon::vec_dot_q8k_q8k(n, xs, ys); + + #[cfg(target_feature = "simd128")] + return super::simd128::vec_dot_q8k_q8k(n, xs, ys); + + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + let qk = QK_K; + if n % QK_K != 0 { + crate::bail!("vec_dot_q8k_q8k: {n} is not divisible by {qk}") + } + + // Generic implementation. + let mut sumf = 0f32; + for (xs, ys) in xs.iter().zip(ys.iter()) { + let sum_i = xs + .qs + .iter() + .zip(ys.qs.iter()) + .map(|(&x, &y)| x as i32 * y as i32) + .sum::(); + sumf += sum_i as f32 * xs.d * ys.d + } + Ok(sumf) + } + + fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { + let k = xs.len(); + if k % QK_K != 0 { + crate::bail!("quantize_row_q8k: {k} is not divisible by {QK_K}") + } + for (i, y) in ys.iter_mut().enumerate() { + let mut max = 0f32; + let mut amax = 0f32; + let xs = &xs[i * QK_K..(i + 1) * QK_K]; + for &x in xs.iter() { + if amax < x.abs() { + amax = x.abs(); + max = x; + } + } + if amax == 0f32 { + y.d = 0f32; + y.qs.fill(0) + } else { + let iscale = -128f32 / max; + for (j, q) in y.qs.iter_mut().enumerate() { + // ggml uses nearest_int with bit magic here, maybe we want the same + // but we would have to test and benchmark it. + let v = (iscale * xs[j]).round(); + *q = v.min(127.) as i8 + } + for j in 0..QK_K / 16 { + let mut sum = 0i32; + for ii in 0..16 { + sum += y.qs[j * 16 + ii] as i32 + } + y.bsums[j] = sum as i16 + } + y.d = 1.0 / iscale + } + } + Ok(()) + } + + fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { + let k = ys.len(); + if k % QK_K != 0 { + crate::bail!("dequantize_row_q8k: {k} is not divisible by {QK_K}") + } + for (i, x) in xs.iter().enumerate() { + for (j, &q) in x.qs.iter().enumerate() { + ys[i * QK_K + j] = x.d * q as f32 + } + } + Ok(()) + } +} + +// https://github.com/ggerganov/llama.cpp/blob/b5ffb2849d23afe73647f68eec7b68187af09be6/ggml.c#L10605 +pub fn matmul( + mkn: (usize, usize, usize), + lhs: &[f32], + rhs_t: &[T], + dst: &mut [f32], +) -> Result<()> { + let (m, k, n) = mkn; + if m * k != lhs.len() { + crate::bail!("unexpected lhs length {} {mkn:?}", lhs.len()); + } + + let k_in_lhs_blocks = (k + T::BLCK_SIZE - 1) / T::BLCK_SIZE; + let k_in_rhs_blocks = (k + T::VecDotType::BLCK_SIZE - 1) / T::VecDotType::BLCK_SIZE; + // TODO: Do not make this copy if the DotType is f32. + // TODO: Pre-allocate this. + let mut lhs_b = vec![T::VecDotType::zeros(); m * k_in_lhs_blocks]; + for row_idx in 0..m { + let lhs_b = &mut lhs_b[row_idx * k_in_lhs_blocks..(row_idx + 1) * k_in_lhs_blocks]; + let lhs = &lhs[row_idx * k..(row_idx + 1) * k]; + T::VecDotType::from_float(lhs, lhs_b)? + } + let lhs_b = lhs_b.as_slice(); + + for row_idx in 0..m { + let lhs_row = &lhs_b[row_idx * k_in_lhs_blocks..(row_idx + 1) * k_in_lhs_blocks]; + let dst_row = &mut dst[row_idx * n..(row_idx + 1) * n]; + + let result: Result> = dst_row + .into_par_iter() + .enumerate() + .with_min_len(128) + .with_max_len(512) + .map(|(col_idx, dst)| { + let rhs_col = &rhs_t[col_idx * k_in_rhs_blocks..(col_idx + 1) * k_in_rhs_blocks]; + T::vec_dot(k, rhs_col, lhs_row).map(|value| *dst = value) + }) + .collect(); + + result?; + } + Ok(()) +} + +impl GgmlType for f32 { + const DTYPE: GgmlDType = GgmlDType::F32; + const BLCK_SIZE: usize = 1; + type VecDotType = f32; + + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + if xs.len() < n { + crate::bail!("size mismatch {} < {n}", xs.len()) + } + if ys.len() < n { + crate::bail!("size mismatch {} < {n}", ys.len()) + } + let mut res = 0f32; + unsafe { crate::cpu::vec_dot_f32(xs.as_ptr(), ys.as_ptr(), &mut res, n) }; + Ok(res) + } + + fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { + if xs.len() != ys.len() { + crate::bail!("size mismatch {} {}", xs.len(), ys.len()); + } + ys.copy_from_slice(xs); + Ok(()) + } + + fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { + if xs.len() != ys.len() { + crate::bail!("size mismatch {} {}", xs.len(), ys.len()); + } + ys.copy_from_slice(xs); + Ok(()) + } +} + +impl GgmlType for f16 { + const DTYPE: GgmlDType = GgmlDType::F16; + const BLCK_SIZE: usize = 1; + type VecDotType = f16; + + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { + if xs.len() < n { + crate::bail!("size mismatch {} < {n}", xs.len()) + } + if ys.len() < n { + crate::bail!("size mismatch {} < {n}", ys.len()) + } + let mut res = 0f32; + unsafe { crate::cpu::vec_dot_f16(xs.as_ptr(), ys.as_ptr(), &mut res, n) }; + Ok(res) + } + + fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { + if xs.len() != ys.len() { + crate::bail!("size mismatch {} {}", xs.len(), ys.len()); + } + // TODO: vectorize + for (x, y) in xs.iter().zip(ys.iter_mut()) { + *y = f16::from_f32(*x) + } + Ok(()) + } + + fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { + if xs.len() != ys.len() { + crate::bail!("size mismatch {} {}", xs.len(), ys.len()); + } + // TODO: vectorize + for (x, y) in xs.iter().zip(ys.iter_mut()) { + *y = x.to_f32() + } + Ok(()) + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/metal.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/metal.rs new file mode 100644 index 0000000000..af1cf36973 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/quantized/metal.rs @@ -0,0 +1,217 @@ +use super::{GgmlDType, QStorage}; +use crate::backend::BackendStorage; +use crate::{DType, MetalDevice, MetalStorage, Result, Shape}; +use metal::Buffer; +use std::sync::Arc; + +pub struct QMetalStorage { + dtype: GgmlDType, + device: MetalDevice, + buffer: Arc, +} + +impl QMetalStorage { + pub fn zeros(device: &MetalDevice, elem_count: usize, dtype: GgmlDType) -> Result { + let size = elem_count * dtype.type_size() / dtype.block_size(); + let buffer = device.allocate_zeros(size)?; + Ok(Self { + buffer, + device: device.clone(), + dtype, + }) + } + + pub fn dtype(&self) -> GgmlDType { + self.dtype + } + + pub fn device(&self) -> &MetalDevice { + &self.device + } + + pub fn buffer(&self) -> &Buffer { + &self.buffer + } + + pub fn dequantize(&self, elem_count: usize) -> Result { + use crate::quantized::k_quants::GgmlType; + + let buffer = self.device.new_buffer_managed(self.buffer.length())?; + let command_buffer = self.device.command_buffer()?; + command_buffer.set_label("to_cpu"); + let blit = command_buffer.new_blit_command_encoder(); + blit.set_label("blit_to_cpu"); + blit.copy_from_buffer(&self.buffer, 0, &buffer, 0, self.buffer.length()); + blit.end_encoding(); + self.device.wait_until_completed()?; + let mut out = vec![0.0; elem_count]; + let block_len = elem_count / self.dtype.block_size(); + match self.dtype { + GgmlDType::F32 => { + let vec: Vec = read_to_vec(&buffer, block_len); + f32::to_float(&vec, &mut out)?; + } + GgmlDType::F16 => { + let vec: Vec = read_to_vec(&buffer, block_len); + half::f16::to_float(&vec, &mut out)?; + } + GgmlDType::Q4_0 => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ4_0::to_float(&vec, &mut out)?; + } + GgmlDType::Q4_1 => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ4_1::to_float(&vec, &mut out)?; + } + GgmlDType::Q5_0 => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ5_0::to_float(&vec, &mut out)?; + } + GgmlDType::Q5_1 => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ5_1::to_float(&vec, &mut out)?; + } + GgmlDType::Q8_0 => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ8_0::to_float(&vec, &mut out)?; + } + GgmlDType::Q8_1 => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ8_1::to_float(&vec, &mut out)?; + } + GgmlDType::Q2K => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ2K::to_float(&vec, &mut out)?; + } + GgmlDType::Q3K => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ3K::to_float(&vec, &mut out)?; + } + GgmlDType::Q4K => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ4K::to_float(&vec, &mut out)?; + } + GgmlDType::Q5K => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ5K::to_float(&vec, &mut out)?; + } + GgmlDType::Q6K => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ6K::to_float(&vec, &mut out)?; + } + GgmlDType::Q8K => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ8K::to_float(&vec, &mut out)?; + } + } + + let buffer = self.device.new_buffer_with_data(&out)?; + Ok(MetalStorage::new(buffer, self.device.clone(), DType::F32)) + } + + pub fn quantize(&mut self, src: &MetalStorage) -> Result<()> { + // Quantization only happens on CPU for now. + let src = src.to_cpu::()?; + let elem_count = src.len(); + let src = crate::Storage::Cpu(crate::CpuStorage::F32(src)); + let mut qcpu_storage = crate::Device::Cpu.qzeros(elem_count, self.dtype)?; + qcpu_storage.quantize(&src)?; + let buffer = self.device.new_buffer_with_data(&qcpu_storage.data()?)?; + self.buffer = buffer; + Ok(()) + } + + pub fn storage_size_in_bytes(&self) -> usize { + self.buffer.length() as usize + } + + pub fn fwd( + &self, + self_shape: &Shape, + storage: &MetalStorage, + layout: &crate::Layout, + ) -> Result<(MetalStorage, Shape)> { + use crate::MetalError; + + if !layout.is_contiguous() { + crate::bail!("input tensor is not contiguous {layout:?}") + } + let src_shape = layout.shape(); + // self is transposed so n is first then k. + if src_shape.rank() < 2 { + crate::bail!("input tensor has only one dimension {layout:?}") + } + let (n, k) = self_shape.dims2()?; + let mut dst_shape = src_shape.dims().to_vec(); + + let (b, m) = match dst_shape.len() { + 3 => (dst_shape[0], dst_shape[1]), + 2 => (1, dst_shape[0]), + n => crate::bail!("Invalid rank {n} for quantized matmul metal"), + }; + let last_k = dst_shape.pop().unwrap(); + if last_k != k { + crate::bail!("input tensor {layout:?} incompatible with {:?}", self_shape) + } + dst_shape.push(n); + let dst_shape = Shape::from(dst_shape); + let device = storage.device().clone(); + let dst = device.new_buffer(dst_shape.elem_count(), DType::F32, "qmatmul")?; + let command_buffer = device.command_buffer()?; + candle_metal_kernels::call_quantized_matmul_t( + device.device(), + &command_buffer, + device.kernels(), + self.dtype.into(), + (b, m, n, k), + storage.buffer(), + layout.start_offset() * storage.dtype().size_in_bytes(), + &self.buffer, + &dst, + ) + .map_err(MetalError::from)?; + let dst_storage = crate::MetalStorage::new(dst, device, DType::F32); + Ok((dst_storage, dst_shape)) + } +} + +pub fn load_quantized( + device: &MetalDevice, + data: &[T], +) -> Result { + let buffer = device.new_buffer_with_data(data)?; + let device = device.clone(); + Ok(QStorage::Metal(QMetalStorage { + dtype: T::DTYPE, + device, + buffer, + })) +} + +fn read_to_vec(buffer: &Buffer, n: usize) -> Vec { + let ptr = buffer.contents() as *const T; + assert!(!ptr.is_null()); + let slice = unsafe { std::slice::from_raw_parts(ptr, n) }; + slice.to_vec() +} + +impl From for candle_metal_kernels::GgmlDType { + fn from(value: GgmlDType) -> Self { + match value { + GgmlDType::Q4_0 => candle_metal_kernels::GgmlDType::Q4_0, + GgmlDType::Q4_1 => candle_metal_kernels::GgmlDType::Q4_1, + GgmlDType::Q5_0 => candle_metal_kernels::GgmlDType::Q5_0, + GgmlDType::Q5_1 => candle_metal_kernels::GgmlDType::Q5_1, + GgmlDType::Q8_0 => candle_metal_kernels::GgmlDType::Q8_0, + GgmlDType::Q8_1 => candle_metal_kernels::GgmlDType::Q8_1, + GgmlDType::Q2K => candle_metal_kernels::GgmlDType::Q2K, + GgmlDType::Q3K => candle_metal_kernels::GgmlDType::Q3K, + GgmlDType::Q4K => candle_metal_kernels::GgmlDType::Q4K, + GgmlDType::Q5K => candle_metal_kernels::GgmlDType::Q5K, + GgmlDType::Q6K => candle_metal_kernels::GgmlDType::Q6K, + GgmlDType::Q8K => candle_metal_kernels::GgmlDType::Q8K, + GgmlDType::F16 => candle_metal_kernels::GgmlDType::F16, + GgmlDType::F32 => candle_metal_kernels::GgmlDType::F32, + } + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/mod.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/mod.rs new file mode 100644 index 0000000000..f7abcd93d3 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/quantized/mod.rs @@ -0,0 +1,491 @@ +use crate::{CpuStorage, Device, Result, Shape, Storage, Tensor}; +use k_quants::*; +use std::borrow::Cow; + +#[cfg(target_feature = "avx")] +pub mod avx; +mod dummy_cuda; +mod dummy_metal; +pub mod ggml_file; +pub mod gguf_file; +pub mod k_quants; +#[cfg(feature = "metal")] +pub mod metal; +#[cfg(not(feature = "metal"))] +mod metal { + pub use super::dummy_metal::*; +} +#[cfg(feature = "cuda")] +pub mod cuda; +#[cfg(not(feature = "cuda"))] +mod cuda { + pub use super::dummy_cuda::*; +} + +#[cfg(target_feature = "neon")] +pub mod neon; +#[cfg(target_feature = "simd128")] +pub mod simd128; +pub mod utils; +use half::f16; + +pub use k_quants::GgmlType; + +pub struct QTensor { + storage: QStorage, + shape: Shape, +} + +impl Device { + fn qzeros(&self, elem_count: usize, dtype: GgmlDType) -> Result { + match self { + Device::Cpu => { + let storage = dtype.cpu_zeros(elem_count); + Ok(QStorage::Cpu(storage)) + } + Device::Metal(metal) => { + let storage = metal::QMetalStorage::zeros(metal, elem_count, dtype)?; + Ok(QStorage::Metal(storage)) + } + Device::Cuda(cuda) => { + let storage = cuda::QCudaStorage::zeros(cuda, elem_count, dtype)?; + Ok(QStorage::Cuda(storage)) + } + } + } +} + +pub enum QStorage { + Cpu(Box), + Metal(metal::QMetalStorage), + Cuda(cuda::QCudaStorage), +} + +impl QStorage { + fn block_size(&self) -> usize { + match self { + QStorage::Cpu(storage) => storage.block_size(), + QStorage::Metal(storage) => storage.dtype().block_size(), + QStorage::Cuda(storage) => storage.dtype().block_size(), + } + } + + fn dtype(&self) -> GgmlDType { + match self { + QStorage::Cpu(storage) => storage.dtype(), + QStorage::Metal(storage) => storage.dtype(), + QStorage::Cuda(storage) => storage.dtype(), + } + } + + fn device(&self) -> Device { + match self { + QStorage::Cpu(_storage) => Device::Cpu, + QStorage::Metal(storage) => Device::Metal(storage.device().clone()), + QStorage::Cuda(storage) => Device::Cuda(storage.device().clone()), + } + } + + fn size_in_bytes(&self) -> usize { + match self { + QStorage::Cpu(storage) => storage.storage_size_in_bytes(), + QStorage::Metal(storage) => storage.storage_size_in_bytes(), + QStorage::Cuda(storage) => storage.storage_size_in_bytes(), + } + } + + fn quantize(&mut self, src: &Storage) -> Result<()> { + match (self, src) { + (QStorage::Cpu(storage), Storage::Cpu(src)) => { + storage.from_float(src.as_slice::()?)?; + } + (QStorage::Metal(storage), Storage::Metal(src)) => storage.quantize(src)?, + (QStorage::Cuda(storage), Storage::Cuda(src)) => storage.quantize(src)?, + _ => crate::bail!("Invalid dequantize storage locations do not match"), + } + Ok(()) + } + + fn dequantize(&self, elem_count: usize) -> Result { + match self { + QStorage::Cpu(storage) => Ok(Storage::Cpu(storage.dequantize(elem_count)?)), + QStorage::Metal(storage) => Ok(Storage::Metal(storage.dequantize(elem_count)?)), + QStorage::Cuda(storage) => Ok(Storage::Cuda(storage.dequantize(elem_count)?)), + } + } + + fn data(&self) -> Result> { + match self { + QStorage::Cpu(storage) => { + let data_ptr = storage.as_ptr(); + let size_in_bytes = storage.storage_size_in_bytes(); + let data = unsafe { std::slice::from_raw_parts(data_ptr, size_in_bytes) }; + Ok(Cow::from(data)) + } + QStorage::Metal(_) | QStorage::Cuda(_) => { + crate::bail!("not implemented"); + } + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum GgmlDType { + F32, + F16, + Q4_0, + Q4_1, + Q5_0, + Q5_1, + Q8_0, + Q8_1, + Q2K, + Q3K, + Q4K, + Q5K, + Q6K, + Q8K, +} + +impl GgmlDType { + pub(crate) fn from_u32(u: u32) -> Result { + let dtype = match u { + 0 => Self::F32, + 1 => Self::F16, + 2 => Self::Q4_0, + 3 => Self::Q4_1, + 6 => Self::Q5_0, + 7 => Self::Q5_1, + 8 => Self::Q8_0, + 9 => Self::Q8_1, + 10 => Self::Q2K, + 11 => Self::Q3K, + 12 => Self::Q4K, + 13 => Self::Q5K, + 14 => Self::Q6K, + 15 => Self::Q8K, + _ => crate::bail!("unknown dtype for tensor {u}"), + }; + Ok(dtype) + } + + pub(crate) fn to_u32(self) -> u32 { + match self { + Self::F32 => 0, + Self::F16 => 1, + Self::Q4_0 => 2, + Self::Q4_1 => 3, + Self::Q5_0 => 6, + Self::Q5_1 => 7, + Self::Q8_0 => 8, + Self::Q8_1 => 9, + Self::Q2K => 10, + Self::Q3K => 11, + Self::Q4K => 12, + Self::Q5K => 13, + Self::Q6K => 14, + Self::Q8K => 15, + } + } + + /// The block dtype + pub fn cpu_zeros(&self, elem_count: usize) -> Box { + match self { + Self::F32 => Box::new(vec![f32::zeros(); elem_count]), + Self::F16 => Box::new(vec![f16::zeros(); elem_count]), + Self::Q4_0 => Box::new(vec![BlockQ4_0::zeros(); elem_count / BlockQ4_0::BLCK_SIZE]), + Self::Q4_1 => Box::new(vec![BlockQ4_1::zeros(); elem_count / BlockQ4_1::BLCK_SIZE]), + Self::Q5_0 => Box::new(vec![BlockQ5_0::zeros(); elem_count / BlockQ5_0::BLCK_SIZE]), + Self::Q5_1 => Box::new(vec![BlockQ5_1::zeros(); elem_count / BlockQ5_1::BLCK_SIZE]), + Self::Q8_0 => Box::new(vec![BlockQ8_0::zeros(); elem_count / BlockQ8_0::BLCK_SIZE]), + Self::Q8_1 => Box::new(vec![BlockQ8_1::zeros(); elem_count / BlockQ8_1::BLCK_SIZE]), + Self::Q2K => Box::new(vec![BlockQ2K::zeros(); elem_count / BlockQ2K::BLCK_SIZE]), + Self::Q3K => Box::new(vec![BlockQ3K::zeros(); elem_count / BlockQ3K::BLCK_SIZE]), + Self::Q4K => Box::new(vec![BlockQ4K::zeros(); elem_count / BlockQ4K::BLCK_SIZE]), + Self::Q5K => Box::new(vec![BlockQ5K::zeros(); elem_count / BlockQ5K::BLCK_SIZE]), + Self::Q6K => Box::new(vec![BlockQ6K::zeros(); elem_count / BlockQ6K::BLCK_SIZE]), + Self::Q8K => Box::new(vec![BlockQ8K::zeros(); elem_count / BlockQ8K::BLCK_SIZE]), + } + } + /// The type size for blocks in bytes. + pub fn type_size(&self) -> usize { + use k_quants::*; + match self { + Self::F32 => 4, + Self::F16 => 2, + Self::Q4_0 => std::mem::size_of::(), + Self::Q4_1 => std::mem::size_of::(), + Self::Q5_0 => std::mem::size_of::(), + Self::Q5_1 => std::mem::size_of::(), + // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L932 + Self::Q8_0 => std::mem::size_of::(), + Self::Q8_1 => std::mem::size_of::(), + Self::Q2K => std::mem::size_of::(), + Self::Q3K => std::mem::size_of::(), + Self::Q4K => std::mem::size_of::(), + Self::Q5K => std::mem::size_of::(), + Self::Q6K => std::mem::size_of::(), + Self::Q8K => std::mem::size_of::(), + } + } + + /// The block size, i.e. the number of elements stored in each block. + pub fn block_size(&self) -> usize { + match self { + Self::F32 => 1, + Self::F16 => 1, + Self::Q4_0 => k_quants::QK4_0, + Self::Q4_1 => k_quants::QK4_1, + Self::Q5_0 => k_quants::QK5_0, + Self::Q5_1 => k_quants::QK5_1, + Self::Q8_0 => k_quants::QK8_0, + Self::Q8_1 => k_quants::QK8_1, + Self::Q2K | Self::Q3K | Self::Q4K | Self::Q5K | Self::Q6K | Self::Q8K => k_quants::QK_K, + } + } +} + +// A version of GgmlType without `vec_dot` so that it can be dyn boxed. +pub trait QuantizedType: Send + Sync { + fn dtype(&self) -> GgmlDType; + fn matmul_t(&self, mkn: (usize, usize, usize), lhs: &[f32], dst: &mut [f32]) -> Result<()>; + fn dequantize(&self, elem_count: usize) -> Result; + fn storage_size_in_bytes(&self) -> usize; + fn as_ptr(&self) -> *const u8; + fn block_size(&self) -> usize; + #[allow(clippy::wrong_self_convention)] + fn from_float(&mut self, xs: &[f32]) -> Result<()>; + fn size(&self) -> usize; +} + +impl QuantizedType for Vec { + fn matmul_t(&self, mkn: (usize, usize, usize), lhs: &[f32], dst: &mut [f32]) -> Result<()> { + k_quants::matmul(mkn, lhs, self.as_slice(), dst) + } + + fn size(&self) -> usize { + self.len() * core::mem::size_of::() + } + + fn from_float(&mut self, xs: &[f32]) -> Result<()> { + T::from_float(xs, self) + } + + fn dtype(&self) -> GgmlDType { + T::DTYPE + } + + fn block_size(&self) -> usize { + T::BLCK_SIZE + } + + fn dequantize(&self, elem_count: usize) -> Result { + let mut ys = vec![0.0f32; elem_count]; + T::to_float(self.as_slice(), &mut ys)?; + Ok(CpuStorage::F32(ys)) + } + + fn storage_size_in_bytes(&self) -> usize { + self.len() * std::mem::size_of::() + } + + fn as_ptr(&self) -> *const u8 { + self.as_ptr() as *const u8 + } +} + +impl std::fmt::Debug for QTensor { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "QTensor[{:?}; {:?}]", self.shape, self.dtype()) + } +} + +fn check_shape(shape: &Shape, block_size: usize) -> Result<()> { + let dims = shape.dims(); + if dims.is_empty() { + crate::bail!("scalar tensor cannot be quantized {shape:?}") + } + if dims[dims.len() - 1] % block_size != 0 { + crate::bail!( + "quantized tensor must have their last dim divisible by block size {shape:?} {}", + block_size + ) + } + Ok(()) +} + +impl QTensor { + pub fn new>(storage: QStorage, shape: S) -> Result { + let shape = shape.into(); + check_shape(&shape, storage.block_size())?; + Ok(Self { storage, shape }) + } + + pub fn quantize(src: &Tensor, dtype: GgmlDType) -> Result { + let shape = src.shape(); + let block_size = dtype.block_size(); + check_shape(shape, block_size)?; + let src = src.to_dtype(crate::DType::F32)?.flatten_all()?; + let elem_count = shape.elem_count(); + if elem_count % block_size != 0 { + crate::bail!( + "tensor size ({shape:?}) is not divisible by block size {}", + block_size + ) + } + let mut storage = src.device().qzeros(elem_count, dtype)?; + storage.quantize(&src.storage())?; + Ok(Self { + storage, + shape: shape.clone(), + }) + } + + pub fn dtype(&self) -> GgmlDType { + self.storage.dtype() + } + + pub fn device(&self) -> Device { + self.storage.device() + } + + pub fn rank(&self) -> usize { + self.shape.rank() + } + + pub fn shape(&self) -> &Shape { + &self.shape + } + + pub fn dequantize(&self, device: &Device) -> Result { + let storage = self.storage.dequantize(self.shape.elem_count())?; + let none = crate::op::BackpropOp::none(); + let is_variable = false; + crate::tensor::from_storage(storage, self.shape.clone(), none, is_variable) + .to_device(device) + } + + pub fn storage_size_in_bytes(&self) -> usize { + self.storage.size_in_bytes() + } + + pub fn data(&self) -> Result> { + self.storage.data() + } +} + +#[derive(Clone, Debug)] +pub enum QMatMul { + QTensor(std::sync::Arc), + Tensor(Tensor), +} + +thread_local! { + static DEQUANTIZE_ALL: bool = { + match std::env::var("CANDLE_DEQUANTIZE_ALL") { + Ok(s) => { + !s.is_empty() && s != "0" + }, + Err(_) => false, + } + } +} + +impl QMatMul { + pub fn from_arc(qtensor: std::sync::Arc) -> Result { + let dequantize = match qtensor.dtype() { + GgmlDType::F32 | GgmlDType::F16 => true, + _ => DEQUANTIZE_ALL.with(|b| *b), + }; + let t = if dequantize { + let tensor = qtensor.dequantize(&Device::Cpu)?; + Self::Tensor(tensor) + } else { + Self::QTensor(qtensor) + }; + Ok(t) + } + + pub fn from_qtensor(qtensor: QTensor) -> Result { + Self::from_arc(std::sync::Arc::new(qtensor)) + } +} + +impl crate::CustomOp1 for QTensor { + fn name(&self) -> &'static str { + "qmatmul" + } + + fn cpu_fwd( + &self, + storage: &crate::CpuStorage, + layout: &crate::Layout, + ) -> Result<(crate::CpuStorage, Shape)> { + if !layout.is_contiguous() { + crate::bail!("input tensor is not contiguous {layout:?}") + } + let src_shape = layout.shape(); + // self is transposed so n is first then k. + let (n, k) = self.shape.dims2()?; + if src_shape.rank() < 2 { + crate::bail!("input tensor has only one dimension {layout:?}") + } + let mut dst_shape = src_shape.dims().to_vec(); + let last_k = dst_shape.pop().unwrap(); + if last_k != k { + crate::bail!("input tensor {layout:?} incompatible with {:?}", self.shape) + } + dst_shape.push(n); + let dst_shape = Shape::from(dst_shape); + #[allow(clippy::infallible_destructuring_match)] + let self_storage = match &self.storage { + QStorage::Cpu(storage) => storage, + QStorage::Metal(_) | QStorage::Cuda(_) => crate::bail!("Invalid storage"), + }; + let slice = storage.as_slice::()?; + let slice = &slice[layout.start_offset()..layout.start_offset() + src_shape.elem_count()]; + let mut dst_storage = vec![0f32; dst_shape.elem_count()]; + self_storage.matmul_t((dst_shape.elem_count() / n, k, n), slice, &mut dst_storage)?; + Ok((crate::CpuStorage::F32(dst_storage), dst_shape)) + } + + fn metal_fwd( + &self, + storage: &crate::MetalStorage, + layout: &crate::Layout, + ) -> Result<(crate::MetalStorage, Shape)> { + let self_storage = match &self.storage { + QStorage::Metal(metal) => metal, + _ => unreachable!("Cannot call metal matmul on non metal QTensor"), + }; + self_storage.fwd(&self.shape, storage, layout) + } + + fn cuda_fwd( + &self, + storage: &crate::CudaStorage, + layout: &crate::Layout, + ) -> Result<(crate::CudaStorage, Shape)> { + let self_storage = match &self.storage { + QStorage::Cuda(cuda) => cuda, + _ => unreachable!("Cannot call cuda matmul on non cuda QTensor"), + }; + self_storage.fwd(&self.shape, storage, layout) + } +} + +impl crate::Module for QMatMul { + fn forward(&self, xs: &Tensor) -> Result { + match self { + Self::QTensor(t) => xs.apply_op1_no_bwd(t.as_ref()), + Self::Tensor(w) => { + let w = match *xs.dims() { + [b1, b2, _, _] => w.broadcast_left((b1, b2))?.t()?, + [bsize, _, _] => w.broadcast_left(bsize)?.t()?, + _ => w.t()?, + }; + xs.matmul(&w) + } + } + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/neon.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/neon.rs new file mode 100644 index 0000000000..c4d5d6f41a --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/quantized/neon.rs @@ -0,0 +1,613 @@ +use super::k_quants::{ + BlockQ2K, BlockQ3K, BlockQ4K, BlockQ4_0, BlockQ5K, BlockQ6K, BlockQ8K, BlockQ8_0, QK8_0, QK_K, +}; +use crate::Result; +use byteorder::{ByteOrder, LittleEndian}; + +#[allow(unused_imports)] +#[cfg(target_arch = "arm")] +use core::arch::arm::*; + +#[allow(unused_imports)] +#[cfg(target_arch = "aarch64")] +use core::arch::aarch64::*; + +#[inline(always)] +unsafe fn vdotq_s32(a: int8x16_t, b: int8x16_t) -> int32x4_t { + // TODO: dotprod + let p0 = vmull_s8(vget_low_s8(a), vget_low_s8(b)); + let p1 = vmull_s8(vget_high_s8(a), vget_high_s8(b)); + vaddq_s32(vpaddlq_s16(p0), vpaddlq_s16(p1)) +} + +#[inline(always)] +pub(crate) fn vec_dot_q4_0_q8_0(n: usize, xs: &[BlockQ4_0], ys: &[BlockQ8_0]) -> Result { + let qk = QK8_0; + let nb = n / qk; + if n % QK8_0 != 0 { + crate::bail!("vec_dot_q4_0_q8_0: {n} is not divisible by {qk}") + } + + unsafe { + let mut sumv0 = vdupq_n_f32(0.0f32); + for i in 0..nb { + let x0 = &xs[i]; + let y0 = &ys[i]; + + let m4b = vdupq_n_u8(0x0F); + let s8b = vdupq_n_s8(0x8); + + let v0_0 = vld1q_u8(x0.qs.as_ptr()); + + // 4-bit -> 8-bit + let v0_0l = vreinterpretq_s8_u8(vandq_u8(v0_0, m4b)); + let v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4)); + + // sub 8 + let v0_0ls = vsubq_s8(v0_0l, s8b); + let v0_0hs = vsubq_s8(v0_0h, s8b); + + // load y + let v1_0l = vld1q_s8(y0.qs.as_ptr()); + let v1_0h = vld1q_s8(y0.qs.as_ptr().add(16)); + + let pl0 = vdotq_s32(v0_0ls, v1_0l); + let ph0 = vdotq_s32(v0_0hs, v1_0h); + sumv0 = vmlaq_n_f32( + sumv0, + vcvtq_f32_s32(vaddq_s32(pl0, ph0)), + x0.d.to_f32() * y0.d.to_f32(), + ); + } + Ok(vaddvq_f32(sumv0)) + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q8_0_q8_0(n: usize, xs: &[BlockQ8_0], ys: &[BlockQ8_0]) -> Result { + let qk = QK8_0; + if n % QK8_0 != 0 { + crate::bail!("vec_dot_q8_0_q8_0: {n} is not divisible by {qk}") + } + let nb = n / QK8_0; + unsafe { + let mut sumv0 = vdupq_n_f32(0.0f32); + for i in 0..nb { + let x0 = &xs[i]; + let y0 = &ys[i]; + + let x0_0 = vld1q_s8(x0.qs.as_ptr()); + let x0_1 = vld1q_s8(x0.qs.as_ptr().add(16)); + + // load y + let y0_0 = vld1q_s8(y0.qs.as_ptr()); + let y0_1 = vld1q_s8(y0.qs.as_ptr().add(16)); + + let p0 = vdotq_s32(x0_0, y0_0); + let p1 = vdotq_s32(x0_1, y0_1); + + sumv0 = vmlaq_n_f32( + sumv0, + vcvtq_f32_s32(vaddq_s32(p0, p1)), + x0.d.to_f32() * y0.d.to_f32(), + ); + } + Ok(vaddvq_f32(sumv0)) + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q8k_q8k(n: usize, xs: &[BlockQ8K], ys: &[BlockQ8K]) -> Result { + let qk = QK_K; + if n % QK_K != 0 { + crate::bail!("vec_dot_q8k_q8k: {n} is not divisible by {qk}") + } + + let mut sumf = 0f32; + for (xs, ys) in xs.iter().zip(ys.iter()) { + unsafe { + let mut sum_i = vdupq_n_s32(0); + let scale = xs.d * ys.d; + let xs = xs.qs.as_ptr(); + let ys = ys.qs.as_ptr(); + for i in (0..QK_K).step_by(16) { + let xs = vld1q_s8(xs.add(i)); + let ys = vld1q_s8(ys.add(i)); + let xy = vdotq_s32(xs, ys); + sum_i = vaddq_s32(sum_i, xy) + } + sumf += vaddvq_s32(sum_i) as f32 * scale + } + } + Ok(sumf) +} + +#[inline(always)] +pub(crate) fn vec_dot_q6k_q8k(n: usize, xs: &[BlockQ6K], ys: &[BlockQ8K]) -> Result { + if n % QK_K != 0 { + crate::bail!("vec_dot_q6k_q8k: {n} is not divisible by {QK_K}") + } + let mut sum = 0f32; + unsafe { + let m4b = vdupq_n_u8(0xF); + + let mone = vdupq_n_u8(3); + + for (x, y) in xs.iter().zip(ys.iter()) { + let d_all = x.d.to_f32(); + + let mut q6 = x.ql.as_ptr(); + let mut qh = x.qh.as_ptr(); + let mut q8 = y.qs.as_ptr(); + + let mut scale = x.scales.as_ptr(); + + let q8sums = vld1q_s16_x2(y.bsums.as_ptr()); + let scales = vld1q_s8(scale); + let q6scales = int16x8x2_t( + vmovl_s8(vget_low_s8(scales)), + vmovl_s8(vget_high_s8(scales)), + ); + + let prod = vaddq_s32( + vaddq_s32( + vmull_s16(vget_low_s16(q8sums.0), vget_low_s16(q6scales.0)), + vmull_s16(vget_high_s16(q8sums.0), vget_high_s16(q6scales.0)), + ), + vaddq_s32( + vmull_s16(vget_low_s16(q8sums.1), vget_low_s16(q6scales.1)), + vmull_s16(vget_high_s16(q8sums.1), vget_high_s16(q6scales.1)), + ), + ); + let isum_mins = vaddvq_s32(prod); + + let mut isum = 0i32; + + for _j in 0..QK_K / 128 { + let qhbits = vld1q_u8_x2(qh); + qh = qh.add(32); + let q6bits = vld1q_u8_x4(q6); + q6 = q6.add(64); + let q8bytes = vld1q_s8_x4(q8); + q8 = q8.add(64); + + let q6h_0 = vshlq_n_u8(vandq_u8(mone, qhbits.0), 4); + let q6h_1 = vshlq_n_u8(vandq_u8(mone, qhbits.1), 4); + let shifted = vshrq_n_u8(qhbits.0, 2); + let q6h_2 = vshlq_n_u8(vandq_u8(mone, shifted), 4); + let shifted = vshrq_n_u8(qhbits.1, 2); + let q6h_3 = vshlq_n_u8(vandq_u8(mone, shifted), 4); + + let q6bytes_0 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.0, m4b), q6h_0)); + let q6bytes_1 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.1, m4b), q6h_1)); + let q6bytes_2 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.2, m4b), q6h_2)); + let q6bytes_3 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.3, m4b), q6h_3)); + + let p0 = vdotq_s32(q6bytes_0, q8bytes.0); + let p1 = vdotq_s32(q6bytes_1, q8bytes.1); + let (scale0, scale1) = (*scale as i32, *scale.add(1) as i32); + isum += vaddvq_s32(p0) * scale0 + vaddvq_s32(p1) * scale1; + scale = scale.add(2); + + let p2 = vdotq_s32(q6bytes_2, q8bytes.2); + let p3 = vdotq_s32(q6bytes_3, q8bytes.3); + let (scale0, scale1) = (*scale as i32, *scale.add(1) as i32); + isum += vaddvq_s32(p2) * scale0 + vaddvq_s32(p3) * scale1; + scale = scale.add(2); + + let q8bytes = vld1q_s8_x4(q8); + q8 = q8.add(64); + + let shifted = vshrq_n_u8(qhbits.0, 4); + let q6h_0 = vshlq_n_u8(vandq_u8(mone, shifted), 4); + let shifted = vshrq_n_u8(qhbits.1, 4); + let q6h_1 = vshlq_n_u8(vandq_u8(mone, shifted), 4); + let shifted = vshrq_n_u8(qhbits.0, 6); + let q6h_2 = vshlq_n_u8(vandq_u8(mone, shifted), 4); + let shifted = vshrq_n_u8(qhbits.1, 6); + let q6h_3 = vshlq_n_u8(vandq_u8(mone, shifted), 4); + + let q6bytes_0 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.0, 4), q6h_0)); + let q6bytes_1 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.1, 4), q6h_1)); + let q6bytes_2 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.2, 4), q6h_2)); + let q6bytes_3 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.3, 4), q6h_3)); + + let p0 = vdotq_s32(q6bytes_0, q8bytes.0); + let p1 = vdotq_s32(q6bytes_1, q8bytes.1); + let (scale0, scale1) = (*scale as i32, *scale.add(1) as i32); + isum += vaddvq_s32(p0) * scale0 + vaddvq_s32(p1) * scale1; + scale = scale.add(2); + + let p2 = vdotq_s32(q6bytes_2, q8bytes.2); + let p3 = vdotq_s32(q6bytes_3, q8bytes.3); + let (scale0, scale1) = (*scale as i32, *scale.add(1) as i32); + isum += vaddvq_s32(p2) * scale0 + vaddvq_s32(p3) * scale1; + scale = scale.add(2); + } + sum += d_all * y.d * ((isum - 32 * isum_mins) as f32); + } + } + Ok(sum) +} + +#[inline(always)] +pub(crate) fn vec_dot_q5k_q8k(n: usize, xs: &[BlockQ5K], ys: &[BlockQ8K]) -> Result { + if n % QK_K != 0 { + crate::bail!("vec_dot_q5k_q8k: {n} is not divisible by {QK_K}") + } + let mut sumf = 0f32; + let mut utmp = [0u32; 4]; + const KMASK1: u32 = 0x3f3f3f3f; + const KMASK2: u32 = 0x0f0f0f0f; + const KMASK3: u32 = 0x03030303; + + unsafe { + let m4b = vdupq_n_u8(0xF); + let mone = vdupq_n_u8(1); + let mtwo = vdupq_n_u8(2); + + for (x, y) in xs.iter().zip(ys.iter()) { + let d = y.d * x.d.to_f32(); + let dmin = y.d * x.dmin.to_f32(); + + let q8sums = vpaddq_s16( + vld1q_s16(y.bsums.as_ptr()), + vld1q_s16(y.bsums.as_ptr().add(8)), + ); + + LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); + + utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); + let uaux = utmp[1] & KMASK1; + utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); + utmp[2] = uaux; + utmp[0] &= KMASK1; + + let mins8 = vld1_u8((utmp.as_ptr() as *const u8).add(8)); + let mins = vreinterpretq_s16_u16(vmovl_u8(mins8)); + let prod = vaddq_s32( + vmull_s16(vget_low_s16(q8sums), vget_low_s16(mins)), + vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)), + ); + let sumi_mins = vaddvq_s32(prod); + + let mut scales = utmp.as_ptr() as *const u8; + + let mut q5 = x.qs.as_ptr(); + let mut q8 = y.qs.as_ptr(); + + let mut qhbits = vld1q_u8_x2(x.qh.as_ptr()); + + let mut sumi = 0i32; + + for _j in 0..QK_K / 64 { + let q5bits = vld1q_u8_x2(q5); + q5 = q5.add(32); + let q8bytes = vld1q_s8_x4(q8); + q8 = q8.add(64); + + let q5h_0 = vshlq_n_u8(vandq_u8(mone, qhbits.0), 4); + let q5h_1 = vshlq_n_u8(vandq_u8(mone, qhbits.1), 4); + let q5h_2 = vshlq_n_u8(vandq_u8(mtwo, qhbits.0), 3); + let q5h_3 = vshlq_n_u8(vandq_u8(mtwo, qhbits.1), 3); + qhbits.0 = vshrq_n_u8(qhbits.0, 2); + qhbits.1 = vshrq_n_u8(qhbits.1, 2); + + let q5bytes_0 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q5bits.0, m4b), q5h_0)); + let q5bytes_1 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q5bits.1, m4b), q5h_1)); + let q5bytes_2 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.0, 4), q5h_2)); + let q5bytes_3 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.1, 4), q5h_3)); + + let p0 = vdotq_s32(q5bytes_0, q8bytes.0); + let p1 = vdotq_s32(q5bytes_1, q8bytes.1); + sumi += vaddvq_s32(vaddq_s32(p0, p1)) * *scales as i32; + scales = scales.add(1); + + let p2 = vdotq_s32(q5bytes_2, q8bytes.2); + let p3 = vdotq_s32(q5bytes_3, q8bytes.3); + sumi += vaddvq_s32(vaddq_s32(p2, p3)) * *scales as i32; + scales = scales.add(1); + } + sumf += d * sumi as f32 - dmin * sumi_mins as f32; + } + } + Ok(sumf) +} + +#[inline(always)] +pub(crate) fn vec_dot_q4k_q8k(n: usize, xs: &[BlockQ4K], ys: &[BlockQ8K]) -> Result { + if n % QK_K != 0 { + crate::bail!("vec_dot_q4k_q8k: {n} is not divisible by {QK_K}") + } + let mut sumf = 0f32; + let mut utmp = [0u32; 4]; + let mut scales = [0u8; 16]; + const KMASK1: u32 = 0x3f3f3f3f; + const KMASK2: u32 = 0x0f0f0f0f; + const KMASK3: u32 = 0x03030303; + + unsafe { + let m4b = vdupq_n_u8(0xF); + + for (x, y) in xs.iter().zip(ys.iter()) { + let d = y.d * x.d.to_f32(); + let dmin = y.d * x.dmin.to_f32(); + + let q8sums = vpaddq_s16( + vld1q_s16(y.bsums.as_ptr()), + vld1q_s16(y.bsums.as_ptr().add(8)), + ); + + LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); + + let mins8 = vld1_u32( + [ + utmp[1] & KMASK1, + ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4), + ] + .as_ptr(), + ); + utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); + utmp[0] &= KMASK1; + + let mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins8))); + let prod = vaddq_s32( + vmull_s16(vget_low_s16(q8sums), vget_low_s16(mins)), + vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)), + ); + sumf -= dmin * vaddvq_s32(prod) as f32; + + LittleEndian::write_u32_into(&utmp, &mut scales); + + let mut q4 = x.qs.as_ptr(); + let mut q8 = y.qs.as_ptr(); + + let mut sumi1 = 0i32; + let mut sumi2 = 0i32; + + for j in 0..QK_K / 64 { + let q4bits = vld1q_u8_x2(q4); + q4 = q4.add(32); + let q8bytes = vld1q_s8_x2(q8); + q8 = q8.add(32); + let q4bytes = int8x16x2_t( + vreinterpretq_s8_u8(vandq_u8(q4bits.0, m4b)), + vreinterpretq_s8_u8(vandq_u8(q4bits.1, m4b)), + ); + let p0 = vdotq_s32(q4bytes.0, q8bytes.0); + let p1 = vdotq_s32(q4bytes.1, q8bytes.1); + sumi1 += vaddvq_s32(vaddq_s32(p0, p1)) * scales[2 * j] as i32; + + let q8bytes = vld1q_s8_x2(q8); + q8 = q8.add(32); + let q4bytes = int8x16x2_t( + vreinterpretq_s8_u8(vshrq_n_u8(q4bits.0, 4)), + vreinterpretq_s8_u8(vshrq_n_u8(q4bits.1, 4)), + ); + let p2 = vdotq_s32(q4bytes.0, q8bytes.0); + let p3 = vdotq_s32(q4bytes.1, q8bytes.1); + sumi2 += vaddvq_s32(vaddq_s32(p2, p3)) * scales[2 * j + 1] as i32; + } + sumf += d * (sumi1 + sumi2) as f32; + } + } + Ok(sumf) +} + +#[inline(always)] +pub(crate) fn vec_dot_q3k_q8k(n: usize, xs: &[BlockQ3K], ys: &[BlockQ8K]) -> Result { + if n % QK_K != 0 { + crate::bail!("vec_dot_q3k_q8k: {n} is not divisible by {QK_K}") + } + let mut sumf = 0f32; + let mut utmp = [0u32; 4]; + let mut aux = [0u32; 3]; + const KMASK1: u32 = 0x03030303; + const KMASK2: u32 = 0x0f0f0f0f; + + unsafe { + let m3b = vdupq_n_u8(0x3); + let m0 = vdupq_n_u8(1); + let m1 = vshlq_n_u8(m0, 1); + let m2 = vshlq_n_u8(m0, 2); + let m3 = vshlq_n_u8(m0, 3); + for (x, y) in xs.iter().zip(ys.iter()) { + let d = y.d * x.d.to_f32(); + let mut q3 = x.qs.as_ptr(); + let qh = x.hmask.as_ptr(); + let mut q8 = y.qs.as_ptr(); + + let mut qhbits = vld1q_u8_x2(qh); + + let mut isum = 0i32; + + // Set up scales + LittleEndian::read_u32_into(&x.scales, &mut aux); + + utmp[3] = ((aux[1] >> 4) & KMASK2) | (((aux[2] >> 6) & KMASK1) << 4); + utmp[2] = ((aux[0] >> 4) & KMASK2) | (((aux[2] >> 4) & KMASK1) << 4); + utmp[1] = (aux[1] & KMASK2) | (((aux[2] >> 2) & KMASK1) << 4); + utmp[0] = (aux[0] & KMASK2) | ((aux[2] & KMASK1) << 4); + + let mut scale = utmp.as_mut_ptr() as *mut i8; + for j in 0..16 { + *scale.add(j) -= 32i8 + } + + for j in 0..QK_K / 128 { + let q3bits = vld1q_u8_x2(q3); + q3 = q3.add(32); + let q8bytes_1 = vld1q_s8_x4(q8); + q8 = q8.add(64); + let q8bytes_2 = vld1q_s8_x4(q8); + q8 = q8.add(64); + + let q3h_0 = vshlq_n_u8(vbicq_u8(m0, qhbits.0), 2); + let q3h_1 = vshlq_n_u8(vbicq_u8(m0, qhbits.1), 2); + let q3h_2 = vshlq_n_u8(vbicq_u8(m1, qhbits.0), 1); + let q3h_3 = vshlq_n_u8(vbicq_u8(m1, qhbits.1), 1); + + let q3bytes_0 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(q3bits.0, m3b)), + vreinterpretq_s8_u8(q3h_0), + ); + let q3bytes_1 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(q3bits.1, m3b)), + vreinterpretq_s8_u8(q3h_1), + ); + let q3bytes_2 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.0, 2), m3b)), + vreinterpretq_s8_u8(q3h_2), + ); + let q3bytes_3 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.1, 2), m3b)), + vreinterpretq_s8_u8(q3h_3), + ); + + let p0 = vdotq_s32(q3bytes_0, q8bytes_1.0); + let p1 = vdotq_s32(q3bytes_1, q8bytes_1.1); + let p2 = vdotq_s32(q3bytes_2, q8bytes_1.2); + let p3 = vdotq_s32(q3bytes_3, q8bytes_1.3); + isum += vaddvq_s32(p0) * *scale as i32 + + vaddvq_s32(p1) * *scale.add(1) as i32 + + vaddvq_s32(p2) * *scale.add(2) as i32 + + vaddvq_s32(p3) * *scale.add(3) as i32; + scale = scale.add(4); + + let q3h_0 = vbicq_u8(m2, qhbits.0); + let q3h_1 = vbicq_u8(m2, qhbits.1); + let q3h_2 = vshrq_n_u8(vbicq_u8(m3, qhbits.0), 1); + let q3h_3 = vshrq_n_u8(vbicq_u8(m3, qhbits.1), 1); + + let q3bytes_0 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.0, 4), m3b)), + vreinterpretq_s8_u8(q3h_0), + ); + let q3bytes_1 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.1, 4), m3b)), + vreinterpretq_s8_u8(q3h_1), + ); + let q3bytes_2 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.0, 6), m3b)), + vreinterpretq_s8_u8(q3h_2), + ); + let q3bytes_3 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.1, 6), m3b)), + vreinterpretq_s8_u8(q3h_3), + ); + + let p0 = vdotq_s32(q3bytes_0, q8bytes_2.0); + let p1 = vdotq_s32(q3bytes_1, q8bytes_2.1); + let p2 = vdotq_s32(q3bytes_2, q8bytes_2.2); + let p3 = vdotq_s32(q3bytes_3, q8bytes_2.3); + isum += vaddvq_s32(p0) * *scale as i32 + + vaddvq_s32(p1) * *scale.add(1) as i32 + + vaddvq_s32(p2) * *scale.add(2) as i32 + + vaddvq_s32(p3) * *scale.add(3) as i32; + scale = scale.add(4); + + if j == 0 { + qhbits.0 = vshrq_n_u8(qhbits.0, 4); + qhbits.1 = vshrq_n_u8(qhbits.1, 4); + } + } + sumf += d * isum as f32; + } + } + Ok(sumf) +} + +#[inline(always)] +pub(crate) fn vec_dot_q2k_q8k(n: usize, xs: &[BlockQ2K], ys: &[BlockQ8K]) -> Result { + if n % QK_K != 0 { + crate::bail!("vec_dot_q2k_q8k: {n} is not divisible by {QK_K}") + } + let mut sumf = 0f32; + let mut aux = [0u8; 16]; + + unsafe { + let m3 = vdupq_n_u8(0x3); + let m4 = vdupq_n_u8(0xF); + + for (x, y) in xs.iter().zip(ys.iter()) { + let d = y.d * x.d.to_f32(); + let dmin = -y.d * x.dmin.to_f32(); + + let mut q2 = x.qs.as_ptr(); + let mut q8 = y.qs.as_ptr(); + let sc = x.scales.as_ptr(); + + let mins_and_scales = vld1q_u8(sc); + let scales = vandq_u8(mins_and_scales, m4); + vst1q_u8(aux.as_mut_ptr(), scales); + + let mins = vshrq_n_u8(mins_and_scales, 4); + let q8sums = vld1q_s16_x2(y.bsums.as_ptr()); + let mins16 = int16x8x2_t( + vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(mins))), + vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(mins))), + ); + let s0 = vaddq_s32( + vmull_s16(vget_low_s16(mins16.0), vget_low_s16(q8sums.0)), + vmull_s16(vget_high_s16(mins16.0), vget_high_s16(q8sums.0)), + ); + let s1 = vaddq_s32( + vmull_s16(vget_low_s16(mins16.1), vget_low_s16(q8sums.1)), + vmull_s16(vget_high_s16(mins16.1), vget_high_s16(q8sums.1)), + ); + sumf += dmin * vaddvq_s32(vaddq_s32(s0, s1)) as f32; + + let mut isum = 0i32; + let mut is = 0usize; + + // TODO: dotprod + for _j in 0..QK_K / 128 { + let q2bits = vld1q_u8_x2(q2); + q2 = q2.add(32); + + let q8bytes = vld1q_s8_x2(q8); + q8 = q8.add(32); + let mut q2bytes = int8x16x2_t( + vreinterpretq_s8_u8(vandq_u8(q2bits.0, m3)), + vreinterpretq_s8_u8(vandq_u8(q2bits.1, m3)), + ); + isum += multiply_accum_with_scale(&aux, is, 0, q2bytes, q8bytes); + + let q8bytes = vld1q_s8_x2(q8); + q8 = q8.add(32); + q2bytes.0 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.0, 2), m3)); + q2bytes.1 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.1, 2), m3)); + isum += multiply_accum_with_scale(&aux, is, 2, q2bytes, q8bytes); + + let q8bytes = vld1q_s8_x2(q8); + q8 = q8.add(32); + q2bytes.0 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.0, 4), m3)); + q2bytes.1 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.1, 4), m3)); + isum += multiply_accum_with_scale(&aux, is, 4, q2bytes, q8bytes); + + let q8bytes = vld1q_s8_x2(q8); + q8 = q8.add(32); + q2bytes.0 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.0, 6), m3)); + q2bytes.1 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.1, 6), m3)); + isum += multiply_accum_with_scale(&aux, is, 6, q2bytes, q8bytes); + + is += 8; + } + sumf += d * isum as f32; + } + } + Ok(sumf) +} + +#[inline(always)] +unsafe fn multiply_accum_with_scale( + aux: &[u8; 16], + is: usize, + index: usize, + q2bytes: int8x16x2_t, + q8bytes: int8x16x2_t, +) -> i32 { + let p1 = vdotq_s32(q2bytes.0, q8bytes.0); + let p2 = vdotq_s32(q2bytes.1, q8bytes.1); + vaddvq_s32(p1) * aux[is + index] as i32 + vaddvq_s32(p2) * aux[is + 1 + index] as i32 +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/simd128.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/simd128.rs new file mode 100644 index 0000000000..1c8c0f2068 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/quantized/simd128.rs @@ -0,0 +1,419 @@ +use super::k_quants::{BlockQ2K, BlockQ4K, BlockQ4_0, BlockQ6K, BlockQ8K, BlockQ8_0, QK8_0, QK_K}; +use crate::Result; +use byteorder::{ByteOrder, LittleEndian}; +use half::f16; + +use core::arch::wasm32::*; + +#[inline(always)] +pub(crate) fn vec_dot_q4_0_q8_0(n: usize, xs: &[BlockQ4_0], ys: &[BlockQ8_0]) -> Result { + let qk = QK8_0; + if n % QK8_0 != 0 { + crate::bail!("vec_dot_q4_0_q8_0: {n} is not divisible by {qk}") + } + unsafe { + let mut acc = f32x4_splat(0.0f32); + for (x, y) in xs.iter().zip(ys.iter()) { + let x1234 = v128_load(x.qs.as_ptr() as *const v128); + let x12 = v128_and(x1234, u8x16_splat(0x0F)); + let x12 = i8x16_sub(x12, i8x16_splat(8)); + let x34 = u8x16_shr(x1234, 4); + let x34 = i8x16_sub(x34, i8x16_splat(8)); + + let x1 = i16x8_extend_low_i8x16(x12); + let y1 = i16x8_load_extend_i8x8(y.qs.as_ptr()); + let sum_xy = i32x4_dot_i16x8(x1, y1); + + let x2 = i16x8_extend_high_i8x16(x12); + let y2 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(8)); + let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x2, y2)); + + let x3 = i16x8_extend_low_i8x16(x34); + let y3 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(16)); + let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x3, y3)); + + let x4 = i16x8_extend_high_i8x16(x34); + let y4 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(24)); + let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x4, y4)); + + let sum_xy = f32x4_convert_i32x4(sum_xy); + + // f32x4_relaxed_madd is nightly only. + let d = f32x4_splat(f16::to_f32(x.d) * f16::to_f32(y.d)); + let scaled = f32x4_mul(sum_xy, d); + acc = f32x4_add(acc, scaled) + } + let res = f32x4_extract_lane::<0>(acc) + + f32x4_extract_lane::<1>(acc) + + f32x4_extract_lane::<2>(acc) + + f32x4_extract_lane::<3>(acc); + Ok(res) + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q8_0_q8_0(n: usize, xs: &[BlockQ8_0], ys: &[BlockQ8_0]) -> Result { + let qk = QK8_0; + if n % QK8_0 != 0 { + crate::bail!("vec_dot_q8_0_q8_0: {n} is not divisible by {qk}") + } + unsafe { + let mut acc = f32x4_splat(0.0f32); + for (x, y) in xs.iter().zip(ys.iter()) { + let x1 = i16x8_load_extend_i8x8(x.qs.as_ptr()); + let y1 = i16x8_load_extend_i8x8(y.qs.as_ptr()); + let sum_xy = i32x4_dot_i16x8(x1, y1); + + let x2 = i16x8_load_extend_i8x8(x.qs.as_ptr().add(8)); + let y2 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(8)); + let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x2, y2)); + + let x3 = i16x8_load_extend_i8x8(x.qs.as_ptr().add(16)); + let y3 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(16)); + let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x3, y3)); + + let x4 = i16x8_load_extend_i8x8(x.qs.as_ptr().add(24)); + let y4 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(24)); + let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x4, y4)); + + let sum_xy = f32x4_convert_i32x4(sum_xy); + + // f32x4_relaxed_madd is nightly only. + let d = f32x4_splat(f16::to_f32(x.d) * f16::to_f32(y.d)); + let scaled = f32x4_mul(sum_xy, d); + acc = f32x4_add(acc, scaled) + } + let res = f32x4_extract_lane::<0>(acc) + + f32x4_extract_lane::<1>(acc) + + f32x4_extract_lane::<2>(acc) + + f32x4_extract_lane::<3>(acc); + Ok(res) + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q2k_q8k(n: usize, xs: &[BlockQ2K], ys: &[BlockQ8K]) -> Result { + if n % QK_K != 0 { + crate::bail!("vec_dot_q2k_q8k: {n} is not divisible by {QK_K}") + } + unsafe { + let mut sumf = f32x4_splat(0f32); + for (x, y) in xs.iter().zip(ys.iter()) { + let mut q2: &[_] = &x.qs; + let mut q8: &[_] = &y.qs; + let sc = &x.scales; + + let mut summs = i32x4_splat(0); + for i in (0..(QK_K / 16)).step_by(4) { + let bsums = i32x4_load_extend_i16x4(y.bsums.as_ptr().add(i)); + let scales = i32x4_shr( + i32x4( + sc[i] as i32, + sc[i + 1] as i32, + sc[i + 2] as i32, + sc[i + 3] as i32, + ), + 4, + ); + summs = i32x4_add(summs, i32x4_mul(bsums, scales)) + } + let summs = f32x4_convert_i32x4(summs); + + let dall = y.d * x.d.to_f32(); + let dmin = y.d * x.dmin.to_f32(); + + let mut isum = i32x4_splat(0); + let mut is = 0; + for _ in 0..(QK_K / 128) { + let mut shift = 0; + for _ in 0..4 { + let d = (sc[is] & 0xF) as i32; + is += 1; + let mut isuml = i16x8_splat(0); + for l in (0..16).step_by(8) { + let q8 = i16x8_load_extend_i8x8(q8.as_ptr().add(l)); + let q2 = i16x8_load_extend_u8x8(q2.as_ptr().add(l)); + let q2 = v128_and(i16x8_shr(q2, shift), i16x8_splat(3)); + isuml = i16x8_add(isuml, i16x8_mul(q2, q8)) + } + let dd = i32x4_splat(d); + isum = i32x4_add(isum, i32x4_mul(i32x4_extend_low_i16x8(isuml), dd)); + isum = i32x4_add(isum, i32x4_mul(i32x4_extend_high_i16x8(isuml), dd)); + let d = (sc[is] & 0xF) as i32; + is += 1; + let mut isuml = i16x8_splat(0); + for l in (16..32).step_by(8) { + let q8 = i16x8_load_extend_i8x8(q8.as_ptr().add(l)); + let q2 = i16x8_load_extend_u8x8(q2.as_ptr().add(l)); + let q2 = v128_and(i16x8_shr(q2, shift), i16x8_splat(3)); + isuml = i16x8_add(isuml, i16x8_mul(q2, q8)) + } + let dd = i32x4_splat(d); + isum = i32x4_add(isum, i32x4_mul(i32x4_extend_low_i16x8(isuml), dd)); + isum = i32x4_add(isum, i32x4_mul(i32x4_extend_high_i16x8(isuml), dd)); + shift += 2; + // adjust the indexing + q8 = &q8[32..]; + } + // adjust the indexing + q2 = &q2[32..]; + } + let isum = f32x4_convert_i32x4(isum); + sumf = f32x4_add( + sumf, + f32x4_sub( + f32x4_mul(isum, f32x4_splat(dall)), + f32x4_mul(summs, f32x4_splat(dmin)), + ), + ); + } + let sumf = f32x4_extract_lane::<0>(sumf) + + f32x4_extract_lane::<1>(sumf) + + f32x4_extract_lane::<2>(sumf) + + f32x4_extract_lane::<3>(sumf); + Ok(sumf) + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q4k_q8k(n: usize, xs: &[BlockQ4K], ys: &[BlockQ8K]) -> Result { + if n % QK_K != 0 { + crate::bail!("vec_dot_q4k_q8k: {n} is not divisible by {QK_K}") + } + + const KMASK1: u32 = 0x3f3f3f3f; + const KMASK2: u32 = 0x0f0f0f0f; + const KMASK3: u32 = 0x03030303; + + let mut utmp: [u32; 4] = [0; 4]; + let mut scales: [u8; 8] = [0; 8]; + let mut mins: [u8; 8] = [0; 8]; + + let mut aux8: [u8; QK_K] = [0; QK_K]; + let mut sums = f32x4_splat(0f32); + unsafe { + for (y, x) in ys.iter().zip(xs.iter()) { + let q4 = &x.qs; + let q8 = &y.qs; + + for j in 0..QK_K / 64 { + let q4_1 = v128_load(q4.as_ptr().add(32 * j) as *const v128); + let q4_2 = v128_load(q4.as_ptr().add(32 * j + 16) as *const v128); + v128_store( + aux8.as_mut_ptr().add(64 * j) as *mut v128, + v128_and(q4_1, u8x16_splat(0x0F)), + ); + v128_store( + aux8.as_mut_ptr().add(64 * j + 16) as *mut v128, + v128_and(q4_2, u8x16_splat(0x0F)), + ); + v128_store( + aux8.as_mut_ptr().add(64 * j + 32) as *mut v128, + u8x16_shr(q4_1, 4), + ); + v128_store( + aux8.as_mut_ptr().add(64 * j + 48) as *mut v128, + u8x16_shr(q4_2, 4), + ); + } + + LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); + + utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); + let uaux = utmp[1] & KMASK1; + utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); + utmp[2] = uaux; + utmp[0] &= KMASK1; + + //extract scales and mins + LittleEndian::write_u32_into(&utmp[0..2], &mut scales); + LittleEndian::write_u32_into(&utmp[2..4], &mut mins); + + let mut sumi = i32x4_splat(0); + for j in (0..QK_K / 16).step_by(4) { + let bsums = i32x4_load_extend_i16x4(y.bsums.as_ptr().add(j)); + let (m1, m2) = (mins[j / 2] as i32, mins[j / 2 + 1] as i32); + let mins = i32x4(m1, m1, m2, m2); + sumi = i32x4_add(sumi, i32x4_mul(bsums, mins)); + } + + let mut aux32 = i32x4_splat(0i32); + for (scale_i, scale) in scales.iter().enumerate() { + let scale = i32x4_splat(*scale as i32); + for j in 0..4 { + let i = 32 * scale_i + 8 * j; + let q8 = i16x8_load_extend_i8x8(q8.as_ptr().add(i)); + let aux8 = i16x8_load_extend_u8x8(aux8.as_ptr().add(i)); + let aux16 = i16x8_mul(q8, aux8); + aux32 = i32x4_add(aux32, i32x4_mul(scale, i32x4_extend_low_i16x8(aux16))); + aux32 = i32x4_add(aux32, i32x4_mul(scale, i32x4_extend_high_i16x8(aux16))); + } + } + let aux32 = f32x4_convert_i32x4(aux32); + let d = f32x4_splat(x.d.to_f32() * y.d); + sums = f32x4_add(sums, f32x4_mul(aux32, d)); + let dmin = x.dmin.to_f32() * y.d; + let dmin = f32x4_splat(dmin); + let sumi = f32x4_convert_i32x4(sumi); + sums = f32x4_sub(sums, f32x4_mul(sumi, dmin)); + } + let sums = f32x4_extract_lane::<0>(sums) + + f32x4_extract_lane::<1>(sums) + + f32x4_extract_lane::<2>(sums) + + f32x4_extract_lane::<3>(sums); + Ok(sums) + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q6k_q8k(n: usize, xs: &[BlockQ6K], ys: &[BlockQ8K]) -> Result { + if n % QK_K != 0 { + crate::bail!("vec_dot_q6k_q8k: {n} is not divisible by {QK_K}") + } + + let mut aux8 = [0i8; QK_K]; + unsafe { + let mut sums = f32x4_splat(0f32); + + for (x, y) in xs.iter().zip(ys.iter()) { + let q4 = &x.ql; + let qh = &x.qh; + let q8 = &y.qs; + let mut aux32 = f32x4_splat(0f32); + + for j in (0..QK_K).step_by(128) { + let aux8 = aux8.as_mut_ptr().add(j); + let q4 = &q4.as_ptr().add(j / 2); + let qh = &qh.as_ptr().add(j / 4); + for l in (0..32).step_by(16) { + // aux8[l] = (((q4[l] & 0xF) | ((qh[l] & 3) << 4)) as i32 - 32) as i8; + let a8 = v128_or( + v128_and(v128_load(q4.add(l) as *const v128), u8x16_splat(0xF)), + u8x16_shl( + v128_and(v128_load(qh.add(l) as *const v128), u8x16_splat(3)), + 4, + ), + ); + let a8_low = i16x8_sub(i16x8_extend_low_u8x16(a8), i16x8_splat(32)); + let a8_high = i16x8_sub(i16x8_extend_high_u8x16(a8), i16x8_splat(32)); + v128_store( + aux8.add(l) as *mut v128, + i8x16_narrow_i16x8(a8_low, a8_high), + ); + + // aux8[l + 32] = + // (((q4[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) as i32 - 32) as i8; + let a8 = v128_or( + v128_and(v128_load(q4.add(l + 32) as *const v128), u8x16_splat(0xF)), + u8x16_shl( + v128_and( + u8x16_shr(v128_load(qh.add(l) as *const v128), 2), + u8x16_splat(3), + ), + 4, + ), + ); + let a8_low = i16x8_sub(i16x8_extend_low_u8x16(a8), i16x8_splat(32)); + let a8_high = i16x8_sub(i16x8_extend_high_u8x16(a8), i16x8_splat(32)); + v128_store( + aux8.add(l + 32) as *mut v128, + i8x16_narrow_i16x8(a8_low, a8_high), + ); + + // aux8[l + 64] = (((q4[l] >> 4) | (((qh[l] >> 4) & 3) << 4)) as i32 - 32) as i8; + let a8 = v128_or( + u8x16_shr(v128_load(q4.add(l) as *const v128), 4), + u8x16_shl( + v128_and( + u8x16_shr(v128_load(qh.add(l) as *const v128), 4), + u8x16_splat(3), + ), + 4, + ), + ); + let a8_low = i16x8_sub(i16x8_extend_low_u8x16(a8), i16x8_splat(32)); + let a8_high = i16x8_sub(i16x8_extend_high_u8x16(a8), i16x8_splat(32)); + v128_store( + aux8.add(l + 64) as *mut v128, + i8x16_narrow_i16x8(a8_low, a8_high), + ); + + // aux8[l + 96] = + // (((q4[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) as i32 - 32) as i8; + let a8 = v128_or( + u8x16_shr(v128_load(q4.add(l + 32) as *const v128), 4), + u8x16_shl( + v128_and( + u8x16_shr(v128_load(qh.add(l) as *const v128), 6), + u8x16_splat(3), + ), + 4, + ), + ); + let a8_low = i16x8_sub(i16x8_extend_low_u8x16(a8), i16x8_splat(32)); + let a8_high = i16x8_sub(i16x8_extend_high_u8x16(a8), i16x8_splat(32)); + v128_store( + aux8.add(l + 96) as *mut v128, + i8x16_narrow_i16x8(a8_low, a8_high), + ); + } + } + + for (j, &scale) in x.scales.iter().enumerate() { + let scale = f32x4_splat(scale as f32); + for offset in [0, 8] { + let aux16 = i16x8_mul( + i16x8_load_extend_i8x8(q8.as_ptr().add(16 * j + offset)), + i16x8_load_extend_i8x8(aux8.as_ptr().add(16 * j + offset)), + ); + aux32 = f32x4_add( + aux32, + f32x4_mul(f32x4_convert_i32x4(i32x4_extend_low_i16x8(aux16)), scale), + ); + aux32 = f32x4_add( + aux32, + f32x4_mul(f32x4_convert_i32x4(i32x4_extend_high_i16x8(aux16)), scale), + ); + } + } + + let d = f32x4_splat(x.d.to_f32() * y.d); + sums = f32x4_add(sums, f32x4_mul(aux32, d)); + } + let sums = f32x4_extract_lane::<0>(sums) + + f32x4_extract_lane::<1>(sums) + + f32x4_extract_lane::<2>(sums) + + f32x4_extract_lane::<3>(sums); + Ok(sums) + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q8k_q8k(n: usize, xs: &[BlockQ8K], ys: &[BlockQ8K]) -> Result { + let qk = QK_K; + if n % QK_K != 0 { + crate::bail!("vec_dot_q8k_q8k: {n} is not divisible by {qk}") + } + + unsafe { + let mut acc = f32x4_splat(0.0f32); + for (xs, ys) in xs.iter().zip(ys.iter()) { + let x_qs = xs.qs.as_ptr(); + let y_qs = ys.qs.as_ptr(); + let mut sumi = i32x4_splat(0); + for j in (0..QK_K).step_by(8) { + let xs = i16x8_load_extend_i8x8(x_qs.add(j)); + let ys = i16x8_load_extend_i8x8(y_qs.add(j)); + let sum_xy = i32x4_dot_i16x8(xs, ys); + sumi = i32x4_add(sumi, sum_xy) + } + let d = f32x4_splat(xs.d * ys.d); + acc = f32x4_add(acc, f32x4_mul(f32x4_convert_i32x4(sumi), d)) + } + let res = f32x4_extract_lane::<0>(acc) + + f32x4_extract_lane::<1>(acc) + + f32x4_extract_lane::<2>(acc) + + f32x4_extract_lane::<3>(acc); + Ok(res) + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/utils.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/utils.rs new file mode 100644 index 0000000000..fa6eff51d3 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/quantized/utils.rs @@ -0,0 +1,326 @@ +use crate::Result; + +pub(super) fn nearest_int(v: f32) -> i32 { + v.round() as i32 +} + +/// Validates that the input and output are the right size and returns an iterator which maps each +/// input region `xs` to its corresponding output block in `ys`. Each output region is guaranteed +/// to be `T::BLCK_SIZE` long. +pub(super) fn group_for_quantization<'a, 'b, T: super::k_quants::GgmlType>( + xs: &'b [f32], + ys: &'a mut [T], +) -> Result> { + let block_size = T::BLCK_SIZE; + let dtype = T::DTYPE; + + let expected_blocks = xs.len() / block_size; + let actual_blocks = ys.len(); + + // Validate that the input is the right size + if expected_blocks != actual_blocks { + crate::bail!("quantize {dtype:?}: expected {expected_blocks} blocks but only {actual_blocks} were provided!") + } + + Ok(ys.iter_mut().zip(xs.chunks_exact(block_size)).collect()) +} + +/// Validates that the input and output are the right size and returns an iterator which maps each +/// input block `xs` to its corresponding output region in `ys`. Each output region is guaranteed +/// to be `T::BLCK_SIZE` long. +pub(super) fn group_for_dequantization<'a, 'b, T: super::k_quants::GgmlType>( + xs: &'a [T], + ys: &'b mut [f32], +) -> Result> { + let block_size = T::BLCK_SIZE; + let dtype = T::DTYPE; + + let actual_output_len = ys.len(); + let expected_output_len = xs.len() * block_size; + // Validate that the output is the right size + if expected_output_len != actual_output_len { + crate::bail!("dequantize {dtype:?}: ys (len = {actual_output_len}) does not match the expected length of {expected_output_len}!") + } + + // Zip the blocks and outputs together + Ok(xs.iter().zip(ys.chunks_exact_mut(block_size)).collect()) +} + +pub(super) fn get_scale_min_k4(j: usize, q: &[u8]) -> (u8, u8) { + if j < 4 { + let d = q[j] & 63; + let m = q[j + 4] & 63; + (d, m) + } else { + let d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4); + let m = (q[j + 4] >> 4) | ((q[j] >> 6) << 4); + (d, m) + } +} + +pub(super) unsafe fn make_qx_quants( + n: usize, + nmax: i32, + x: *const f32, + ls: *mut i8, + rmse_type: i32, +) -> f32 { + let mut max = 0f32; + let mut amax = 0f32; + for i in 0..n { + let x = *x.add(i); + let ax = x.abs(); + if ax > amax { + amax = ax; + max = x; + } + } + if amax == 0. { + // all zero + for i in 0..n { + *ls.add(i) = 0; + } + return 0.; + } + let mut iscale = -(nmax as f32) / max; + if rmse_type == 0 { + for i in 0..n { + let x = *x.add(i); + let l = nearest_int(iscale * x); + *ls.add(i) = (nmax + l.clamp(-nmax, nmax - 1)) as i8; + } + return 1.0 / iscale; + } + let weight_type = rmse_type % 2; + let mut sumlx = 0f32; + let mut suml2 = 0f32; + for i in 0..n { + let x = *x.add(i); + let l = nearest_int(iscale * x); + let l = l.clamp(-nmax, nmax - 1); + *ls.add(i) = (l + nmax) as i8; + let w = if weight_type == 1 { x * x } else { 1.0 }; + let l = l as f32; + sumlx += w * x * l; + suml2 += w * l * l; + } + let mut scale = sumlx / suml2; + let mut best = scale * sumlx; + for _itry in 0..3 { + let iscale = 1.0 / scale; + let mut slx = 0f32; + let mut sl2 = 0f32; + let mut changed = false; + for i in 0..n { + let x = *x.add(i); + let l = nearest_int(iscale * x); + let l = l.clamp(-nmax, nmax - 1); + if l + nmax != *ls.add(i) as i32 { + changed = true; + } + let w = if weight_type == 1 { x * x } else { 1f32 }; + let l = l as f32; + slx += w * x * l; + sl2 += w * l * l; + } + if !changed || sl2 == 0.0 || slx * slx <= best * sl2 { + break; + } + for i in 0..n { + let x = *x.add(i); + let l = nearest_int(iscale * x); + *ls.add(i) = (nmax + l.clamp(-nmax, nmax - 1)) as i8; + } + sumlx = slx; + suml2 = sl2; + scale = sumlx / suml2; + best = scale * sumlx; + } + for _itry in 0..5 { + let mut n_changed = 0; + for i in 0..n { + let x = *x.add(i); + let w = if weight_type == 1 { x * x } else { 1. }; + let l = *ls.add(i) as i32 - nmax; + let mut slx = sumlx - w * x * l as f32; + if slx > 0. { + let mut sl2 = suml2 - w * l as f32 * l as f32; + let new_l = nearest_int(x * sl2 / slx); + let new_l = new_l.clamp(-nmax, nmax - 1); + if new_l != l { + slx += w * x * new_l as f32; + sl2 += w * new_l as f32 * new_l as f32; + if sl2 > 0. && slx * slx * suml2 > sumlx * sumlx * sl2 { + *ls.add(i) = (nmax + new_l) as i8; + sumlx = slx; + suml2 = sl2; + scale = sumlx / suml2; + best = scale * sumlx; + n_changed += 1; + } + } + } + } + if n_changed == 0 { + break; + } + } + if rmse_type < 3 { + return scale; + } + for is in -4..4 { + if is == 0 { + continue; + } + iscale = -(nmax as f32 + 0.1f32 * is as f32) / max; + let mut sumlx = 0.; + let mut suml2 = 0.; + for i in 0..n { + let x = *x.add(i); + let l = nearest_int(iscale * x); + let l = l.clamp(-nmax, nmax - 1); + let w = if weight_type == 1 { x * x } else { 1. }; + let l = l as f32; + sumlx += w * x * l; + suml2 += w * l * l; + } + if suml2 > 0. && sumlx * sumlx > best * suml2 { + for i in 0..n { + let x = *x.add(i); + let l = nearest_int(iscale * x); + *ls.add(i) = (nmax + l.clamp(-nmax, nmax - 1)) as i8; + } + scale = sumlx / suml2; + best = scale * sumlx; + } + } + scale +} + +// https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L224 +pub(super) fn make_qkx1_quants(nmax: i32, ntry: usize, x: &[f32]) -> (f32, f32) { + let n = x.len(); + let mut l = vec![0; n]; + // Get min/max + let min = *x + .iter() + .take(n) + .min_by(|a, b| a.total_cmp(b)) + .unwrap_or(&x[0]); + let max = *x.iter().max_by(|a, b| a.total_cmp(b)).unwrap_or(&x[0]); + + // If min == max, all values are the same => nothing to do here + if max == min { + return (0.0, 0.0); + } + + // Ensure min <= 0.0 + let mut min = min.min(0.); + + // Compute scale and inverse scale + let mut iscale = nmax as f32 / (max - min); + let mut scale = 1.0 / iscale; + + for _ in 0..ntry { + let mut sumlx = 0.0; + let mut suml2 = 0; + let mut did_change = false; + + for (i, value) in x.iter().enumerate().take(n) { + let li = nearest_int(iscale * (value - min)).clamp(0, nmax); + let clamped_li = li as u8; + if clamped_li != l[i] { + l[i] = clamped_li; + did_change = true; + } + sumlx += (value - min) * li as f32; + suml2 += li * li; + } + scale = sumlx / suml2 as f32; + + let sum: f32 = x + .iter() + .take(n) + .zip(l.iter().take(n)) + .map(|(xi, &li)| xi - scale * li as f32) + .sum(); + + min = sum / n as f32; + if min > 0.0 { + min = 0.0; + } + iscale = 1.0 / scale; + if !did_change { + break; + } + } + (scale, -min) +} + +// https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L165 +pub(super) fn make_q3_quants(x: &[f32], nmax: i32, do_rmse: bool) -> f32 { + let n = x.len(); + let mut l = vec![0i8; n]; + + let mut max = 0.0; + let mut amax = 0.0; + for &xi in x.iter().take(n) { + let ax = xi.abs(); + if ax > amax { + amax = ax; + max = xi; + } + } + + if amax == 0.0 { + return 0.0; + } + + let iscale = -(nmax as f32) / max; + if do_rmse { + let mut sumlx = 0.0; + let mut suml2 = 0.0; + for i in 0..n { + let li = (iscale * x[i]).round() as i32; + let li = li.clamp(-nmax, nmax - 1); + l[i] = li as i8; + let w = x[i] * x[i]; + sumlx += w * x[i] * li as f32; + suml2 += w * (li * li) as f32; + } + for _ in 0..5 { + let mut n_changed = 0; + for i in 0..n { + let w = x[i] * x[i]; + let mut slx = sumlx - w * x[i] * l[i] as f32; + if slx > 0.0 { + let mut sl2 = suml2 - w * (l[i] as i32 * l[i] as i32) as f32; + let mut new_l = (x[i] * sl2 / slx).round() as i32; + new_l = new_l.clamp(-nmax, nmax - 1); + if new_l != l[i] as i32 { + slx += w * x[i] * new_l as f32; + sl2 += w * (new_l * new_l) as f32; + if sl2 > 0.0 && slx * slx * suml2 > sumlx * sumlx * sl2 { + l[i] = new_l as i8; + sumlx = slx; + suml2 = sl2; + n_changed += 1; + } + } + } + } + if n_changed == 0 { + break; + } + } + for li in l.iter_mut() { + *li += nmax as i8; + } + return sumlx / suml2; + } + for i in 0..n { + let li = (iscale * x[i]).round() as i32; + l[i] = (li.clamp(-nmax, nmax - 1) + nmax) as i8; + } + 1.0 / iscale +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/safetensors.rs b/v2/dependency-patches/candle-core-0.4.1/src/safetensors.rs new file mode 100644 index 0000000000..02d54e6895 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/safetensors.rs @@ -0,0 +1,441 @@ +use crate::{DType, Device, Error, Result, Tensor, WithDType}; +use safetensors::tensor as st; +use safetensors::tensor::SafeTensors; +use std::borrow::Cow; +use std::collections::HashMap; +use std::path::Path; + +impl From for st::Dtype { + fn from(value: DType) -> Self { + match value { + DType::U8 => st::Dtype::U8, + DType::U32 => st::Dtype::U32, + DType::I64 => st::Dtype::I64, + DType::BF16 => st::Dtype::BF16, + DType::F16 => st::Dtype::F16, + DType::F32 => st::Dtype::F32, + DType::F64 => st::Dtype::F64, + } + } +} + +impl TryFrom for DType { + type Error = Error; + fn try_from(value: st::Dtype) -> Result { + match value { + st::Dtype::U8 => Ok(DType::U8), + st::Dtype::U32 => Ok(DType::U32), + st::Dtype::I64 => Ok(DType::I64), + st::Dtype::BF16 => Ok(DType::BF16), + st::Dtype::F16 => Ok(DType::F16), + st::Dtype::F32 => Ok(DType::F32), + st::Dtype::F64 => Ok(DType::F64), + dtype => Err(Error::UnsupportedSafeTensorDtype(dtype)), + } + } +} + +impl st::View for Tensor { + fn dtype(&self) -> st::Dtype { + self.dtype().into() + } + fn shape(&self) -> &[usize] { + self.shape().dims() + } + + fn data(&self) -> Cow<[u8]> { + // This copies data from GPU to CPU. + // TODO: Avoid the unwrap here. + Cow::Owned(convert_back(self).unwrap()) + } + + fn data_len(&self) -> usize { + let n: usize = self.shape().elem_count(); + let bytes_per_element = self.dtype().size_in_bytes(); + n * bytes_per_element + } +} + +impl st::View for &Tensor { + fn dtype(&self) -> st::Dtype { + (*self).dtype().into() + } + fn shape(&self) -> &[usize] { + self.dims() + } + + fn data(&self) -> Cow<[u8]> { + // This copies data from GPU to CPU. + // TODO: Avoid the unwrap here. + Cow::Owned(convert_back(self).unwrap()) + } + + fn data_len(&self) -> usize { + let n: usize = self.dims().iter().product(); + let bytes_per_element = (*self).dtype().size_in_bytes(); + n * bytes_per_element + } +} + +impl Tensor { + pub fn save_safetensors>(&self, name: &str, filename: P) -> Result<()> { + let data = [(name, self.clone())]; + Ok(st::serialize_to_file(data, &None, filename.as_ref())?) + } +} + +fn convert_slice(data: &[u8], shape: &[usize], device: &Device) -> Result { + let size_in_bytes = T::DTYPE.size_in_bytes(); + let elem_count = data.len() / size_in_bytes; + if (data.as_ptr() as usize) % size_in_bytes == 0 { + // SAFETY This is safe because we just checked that this + // was correctly aligned. + let data: &[T] = + unsafe { std::slice::from_raw_parts(data.as_ptr() as *const T, elem_count) }; + Tensor::from_slice(data, shape, device) + } else { + // XXX: We need to specify `T` here, otherwise the compiler will infer u8 because of the following cast + // Making this vector too small to fit a full f16/f32/f64 weights, resulting in out-of-bounds access + let mut c: Vec = Vec::with_capacity(elem_count); + // SAFETY: We just created c, so the allocated memory is necessarily + // contiguous and non overlapping with the view's data. + // We're downgrading the `c` pointer from T to u8, which removes alignment + // constraints. + unsafe { + std::ptr::copy_nonoverlapping(data.as_ptr(), c.as_mut_ptr() as *mut u8, data.len()); + c.set_len(elem_count) + } + Tensor::from_slice(&c, shape, device) + } +} + +fn convert_slice_with_cast Result>( + data: &[u8], + shape: &[usize], + device: &Device, + conv: F, +) -> Result { + let size_in_bytes = std::mem::size_of::(); + let elem_count = data.len() / size_in_bytes; + if (data.as_ptr() as usize) % size_in_bytes == 0 { + // SAFETY This is safe because we just checked that this + // was correctly aligned. + let data: &[T] = + unsafe { std::slice::from_raw_parts(data.as_ptr() as *const T, elem_count) }; + let data = data.iter().map(|t| conv(*t)).collect::>>()?; + Tensor::from_vec(data, shape, device) + } else { + // XXX: We need to specify `T` here, otherwise the compiler will infer u8 because of the following cast + // Making this vector too small to fit a full f16/f32/f64 weights, resulting in out-of-bounds access + let mut c: Vec = Vec::with_capacity(elem_count); + // SAFETY: We just created c, so the allocated memory is necessarily + // contiguous and non overlapping with the view's data. + // We're downgrading the `c` pointer from T to u8, which removes alignment + // constraints. + unsafe { + std::ptr::copy_nonoverlapping(data.as_ptr(), c.as_mut_ptr() as *mut u8, data.len()); + c.set_len(elem_count) + } + let c = c.into_iter().map(conv).collect::>>()?; + Tensor::from_vec(c, shape, device) + } +} + +fn convert_with_cast_ Result>( + view: &st::TensorView<'_>, + device: &Device, + conv: F, +) -> Result { + convert_slice_with_cast::(view.data(), view.shape(), device, conv) +} + +fn convert_(view: &st::TensorView<'_>, device: &Device) -> Result { + convert_slice::(view.data(), view.shape(), device) +} + +fn convert_back_(mut vs: Vec) -> Vec { + let size_in_bytes = T::DTYPE.size_in_bytes(); + let length = vs.len() * size_in_bytes; + let capacity = vs.capacity() * size_in_bytes; + let ptr = vs.as_mut_ptr() as *mut u8; + // Don't run the destructor for Vec + std::mem::forget(vs); + // SAFETY: + // + // Every T is larger than u8, so there is no issue regarding alignment. + // This re-interpret the Vec as a Vec. + unsafe { Vec::from_raw_parts(ptr, length, capacity) } +} + +pub trait Load { + fn load(&self, device: &Device) -> Result; +} + +impl<'a> Load for st::TensorView<'a> { + fn load(&self, device: &Device) -> Result { + convert(self, device) + } +} + +impl Tensor { + pub fn from_raw_buffer( + data: &[u8], + dtype: DType, + shape: &[usize], + device: &Device, + ) -> Result { + match dtype { + DType::U8 => convert_slice::(data, shape, device), + DType::U32 => convert_slice::(data, shape, device), + DType::I64 => convert_slice::(data, shape, device), + DType::BF16 => convert_slice::(data, shape, device), + DType::F16 => convert_slice::(data, shape, device), + DType::F32 => convert_slice::(data, shape, device), + DType::F64 => convert_slice::(data, shape, device), + } + } +} + +fn convert(view: &st::TensorView<'_>, device: &Device) -> Result { + match view.dtype() { + st::Dtype::U8 => convert_::(view, device), + st::Dtype::U16 => { + let conv = |x| Ok(u32::from(x)); + convert_with_cast_::(view, device, conv) + } + st::Dtype::U32 => convert_::(view, device), + st::Dtype::I32 => { + let conv = |x| Ok(i64::from(x)); + convert_with_cast_::(view, device, conv) + } + st::Dtype::I64 => convert_::(view, device), + st::Dtype::BF16 => convert_::(view, device), + st::Dtype::F16 => convert_::(view, device), + st::Dtype::F32 => convert_::(view, device), + st::Dtype::F64 => convert_::(view, device), + dtype => Err(Error::UnsupportedSafeTensorDtype(dtype)), + } +} + +fn convert_back(tensor: &Tensor) -> Result> { + // TODO: This makes an unnecessary copy when the tensor is on the cpu. + let tensor = tensor.flatten_all()?; + match tensor.dtype() { + DType::U8 => Ok(convert_back_::(tensor.to_vec1()?)), + DType::U32 => Ok(convert_back_::(tensor.to_vec1()?)), + DType::I64 => Ok(convert_back_::(tensor.to_vec1()?)), + DType::F16 => Ok(convert_back_::(tensor.to_vec1()?)), + DType::BF16 => Ok(convert_back_::(tensor.to_vec1()?)), + DType::F32 => Ok(convert_back_::(tensor.to_vec1()?)), + DType::F64 => Ok(convert_back_::(tensor.to_vec1()?)), + } +} + +pub fn load>(filename: P, device: &Device) -> Result> { + let data = std::fs::read(filename.as_ref())?; + load_buffer(&data[..], device) +} + +pub fn load_buffer(data: &[u8], device: &Device) -> Result> { + let st = safetensors::SafeTensors::deserialize(data)?; + st.tensors() + .into_iter() + .map(|(name, view)| Ok((name, view.load(device)?))) + .collect() +} + +pub fn save + Ord + std::fmt::Display, P: AsRef>( + tensors: &HashMap, + filename: P, +) -> Result<()> { + Ok(st::serialize_to_file(tensors, &None, filename.as_ref())?) +} + +#[derive(yoke::Yokeable)] +struct SafeTensors_<'a>(SafeTensors<'a>); + +pub struct MmapedSafetensors { + safetensors: Vec, memmap2::Mmap>>, + routing: Option>, +} + +impl MmapedSafetensors { + /// Creates a wrapper around a memory mapped file and deserialize the safetensors header. + /// + /// # Safety + /// + /// The unsafe is inherited from [`memmap2::MmapOptions`]. + pub unsafe fn new>(p: P) -> Result { + let p = p.as_ref(); + let file = std::fs::File::open(p).map_err(|e| Error::from(e).with_path(p))?; + let file = memmap2::MmapOptions::new() + .map(&file) + .map_err(|e| Error::from(e).with_path(p))?; + let safetensors = yoke::Yoke::, memmap2::Mmap>::try_attach_to_cart( + file, + |data: &[u8]| { + let st = safetensors::SafeTensors::deserialize(data) + .map_err(|e| Error::from(e).with_path(p))?; + Ok::<_, Error>(SafeTensors_(st)) + }, + )?; + Ok(Self { + safetensors: vec![safetensors], + routing: None, + }) + } + + /// Creates a wrapper around multiple memory mapped file and deserialize the safetensors headers. + /// + /// If a tensor name appears in multiple files, the last entry is returned. + /// + /// # Safety + /// + /// The unsafe is inherited from [`memmap2::MmapOptions`]. + pub unsafe fn multi>(paths: &[P]) -> Result { + let mut routing = HashMap::new(); + let mut safetensors = vec![]; + for (index, p) in paths.iter().enumerate() { + let p = p.as_ref(); + let file = std::fs::File::open(p).map_err(|e| Error::from(e).with_path(p))?; + let file = memmap2::MmapOptions::new() + .map(&file) + .map_err(|e| Error::from(e).with_path(p))?; + let data = yoke::Yoke::, memmap2::Mmap>::try_attach_to_cart( + file, + |data: &[u8]| { + let st = safetensors::SafeTensors::deserialize(data) + .map_err(|e| Error::from(e).with_path(p))?; + Ok::<_, Error>(SafeTensors_(st)) + }, + )?; + for k in data.get().0.names() { + routing.insert(k.to_string(), index); + } + safetensors.push(data) + } + Ok(Self { + safetensors, + routing: Some(routing), + }) + } + + pub fn load(&self, name: &str, dev: &Device) -> Result { + self.get(name)?.load(dev) + } + + pub fn tensors(&self) -> Vec<(String, st::TensorView<'_>)> { + let mut tensors = vec![]; + for safetensors in self.safetensors.iter() { + tensors.push(safetensors.get().0.tensors()) + } + tensors.into_iter().flatten().collect() + } + + pub fn get(&self, name: &str) -> Result> { + let index = match &self.routing { + None => 0, + Some(routing) => { + let index = routing.get(name).ok_or_else(|| { + Error::CannotFindTensor { + path: name.to_string(), + } + .bt() + })?; + *index + } + }; + Ok(self.safetensors[index].get().0.tensor(name)?) + } +} + +pub struct BufferedSafetensors { + safetensors: yoke::Yoke, Vec>, +} + +impl BufferedSafetensors { + /// Creates a wrapper around a binary buffer and deserialize the safetensors header. + pub fn new(buffer: Vec) -> Result { + let safetensors = yoke::Yoke::, Vec>::try_attach_to_cart( + buffer, + |data: &[u8]| { + let st = safetensors::SafeTensors::deserialize(data)?; + Ok::<_, Error>(SafeTensors_(st)) + }, + )?; + Ok(Self { safetensors }) + } + + pub fn load(&self, name: &str, dev: &Device) -> Result { + self.get(name)?.load(dev) + } + + pub fn tensors(&self) -> Vec<(String, st::TensorView<'_>)> { + self.safetensors.get().0.tensors() + } + + pub fn get(&self, name: &str) -> Result> { + Ok(self.safetensors.get().0.tensor(name)?) + } +} + +pub struct MmapedFile { + path: std::path::PathBuf, + inner: memmap2::Mmap, +} + +impl MmapedFile { + /// Creates a wrapper around a memory mapped file from which you can retrieve + /// tensors using [`MmapedFile::deserialize`] + /// + /// # Safety + /// + /// The unsafe is inherited from [`memmap2::MmapOptions`]. + pub unsafe fn new>(p: P) -> Result { + let p = p.as_ref(); + let file = std::fs::File::open(p).map_err(|e| Error::from(e).with_path(p))?; + let inner = memmap2::MmapOptions::new() + .map(&file) + .map_err(|e| Error::from(e).with_path(p))?; + Ok(Self { + inner, + path: p.to_path_buf(), + }) + } + + pub fn deserialize(&self) -> Result> { + let st = safetensors::SafeTensors::deserialize(&self.inner) + .map_err(|e| Error::from(e).with_path(&self.path))?; + Ok(st) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + #[test] + fn save_single_tensor() { + let t = Tensor::zeros((2, 2), DType::F32, &Device::Cpu).unwrap(); + t.save_safetensors("t", "t.safetensors").unwrap(); + let bytes = std::fs::read("t.safetensors").unwrap(); + assert_eq!(bytes, b"@\0\0\0\0\0\0\0{\"t\":{\"dtype\":\"F32\",\"shape\":[2,2],\"data_offsets\":[0,16]}} \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"); + std::fs::remove_file("t.safetensors").unwrap(); + } + + #[test] + fn save_load_multiple_tensors() { + let t = Tensor::zeros((2, 2), DType::F32, &Device::Cpu).unwrap(); + let u = Tensor::zeros((1, 2), DType::F32, &Device::Cpu).unwrap(); + let map: HashMap<_, _> = [("t", t), ("u", u)].into_iter().collect(); + save(&map, "multi.safetensors").unwrap(); + + let weights = load("multi.safetensors", &Device::Cpu).unwrap(); + assert_eq!(weights.get("t").unwrap().dims(), &[2, 2]); + assert_eq!(weights.get("u").unwrap().dims(), &[1, 2]); + let bytes = std::fs::read("multi.safetensors").unwrap(); + assert_eq!(bytes, b"x\0\0\0\0\0\0\0{\"t\":{\"dtype\":\"F32\",\"shape\":[2,2],\"data_offsets\":[0,16]},\"u\":{\"dtype\":\"F32\",\"shape\":[1,2],\"data_offsets\":[16,24]}} \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"); + std::fs::remove_file("multi.safetensors").unwrap(); + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/scalar.rs b/v2/dependency-patches/candle-core-0.4.1/src/scalar.rs new file mode 100644 index 0000000000..43e1f4c8c5 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/scalar.rs @@ -0,0 +1,23 @@ +use crate::{Result, Tensor, WithDType}; + +pub enum TensorScalar { + Tensor(Tensor), + Scalar(Tensor), +} + +pub trait TensorOrScalar { + fn to_tensor_scalar(self) -> Result; +} + +impl TensorOrScalar for &Tensor { + fn to_tensor_scalar(self) -> Result { + Ok(TensorScalar::Tensor(self.clone())) + } +} + +impl TensorOrScalar for T { + fn to_tensor_scalar(self) -> Result { + let scalar = Tensor::new(self, &crate::Device::Cpu)?; + Ok(TensorScalar::Scalar(scalar)) + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/shape.rs b/v2/dependency-patches/candle-core-0.4.1/src/shape.rs new file mode 100644 index 0000000000..32ebb23f24 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/shape.rs @@ -0,0 +1,629 @@ +//! The shape of a tensor is a tuple with the size of each of its dimensions. +#![allow(clippy::redundant_closure_call)] +use crate::{Error, Result}; + +#[derive(Clone, PartialEq, Eq)] +pub struct Shape(Vec); + +pub const SCALAR: Shape = Shape(vec![]); + +impl std::fmt::Debug for Shape { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", &self.dims()) + } +} + +impl From<&[usize; C]> for Shape { + fn from(dims: &[usize; C]) -> Self { + Self(dims.to_vec()) + } +} + +impl From<&[usize]> for Shape { + fn from(dims: &[usize]) -> Self { + Self(dims.to_vec()) + } +} + +impl From<&Shape> for Shape { + fn from(shape: &Shape) -> Self { + Self(shape.0.to_vec()) + } +} + +impl From<()> for Shape { + fn from(_: ()) -> Self { + Self(vec![]) + } +} + +impl From for Shape { + fn from(d1: usize) -> Self { + Self(vec![d1]) + } +} + +impl From<(usize,)> for Shape { + fn from(d1: (usize,)) -> Self { + Self(vec![d1.0]) + } +} + +impl From<(usize, usize)> for Shape { + fn from(d12: (usize, usize)) -> Self { + Self(vec![d12.0, d12.1]) + } +} + +impl From<(usize, usize, usize)> for Shape { + fn from(d123: (usize, usize, usize)) -> Self { + Self(vec![d123.0, d123.1, d123.2]) + } +} + +impl From<(usize, usize, usize, usize)> for Shape { + fn from(d1234: (usize, usize, usize, usize)) -> Self { + Self(vec![d1234.0, d1234.1, d1234.2, d1234.3]) + } +} + +impl From<(usize, usize, usize, usize, usize)> for Shape { + fn from(d12345: (usize, usize, usize, usize, usize)) -> Self { + Self(vec![d12345.0, d12345.1, d12345.2, d12345.3, d12345.4]) + } +} + +impl From<(usize, usize, usize, usize, usize, usize)> for Shape { + fn from(d123456: (usize, usize, usize, usize, usize, usize)) -> Self { + Self(vec![ + d123456.0, d123456.1, d123456.2, d123456.3, d123456.4, d123456.5, + ]) + } +} + +impl From> for Shape { + fn from(dims: Vec) -> Self { + Self(dims) + } +} + +macro_rules! extract_dims { + ($fn_name:ident, $cnt:tt, $dims:expr, $out_type:ty) => { + pub fn $fn_name(dims: &[usize]) -> Result<$out_type> { + if dims.len() != $cnt { + Err(Error::UnexpectedNumberOfDims { + expected: $cnt, + got: dims.len(), + shape: Shape::from(dims), + } + .bt()) + } else { + Ok($dims(dims)) + } + } + + impl Shape { + pub fn $fn_name(&self) -> Result<$out_type> { + $fn_name(self.0.as_slice()) + } + } + + impl crate::Tensor { + pub fn $fn_name(&self) -> Result<$out_type> { + self.shape().$fn_name() + } + } + + impl std::convert::TryInto<$out_type> for Shape { + type Error = crate::Error; + fn try_into(self) -> std::result::Result<$out_type, Self::Error> { + self.$fn_name() + } + } + }; +} + +impl Shape { + pub fn from_dims(dims: &[usize]) -> Self { + Self(dims.to_vec()) + } + + /// The rank is the number of dimensions, 0 for a scalar value, 1 for a vector, etc. + pub fn rank(&self) -> usize { + self.0.len() + } + + pub fn into_dims(self) -> Vec { + self.0 + } + + /// The dimensions as a slice of `usize`. + pub fn dims(&self) -> &[usize] { + &self.0 + } + + /// The total number of elements, this is the product of all dimension sizes. + pub fn elem_count(&self) -> usize { + self.0.iter().product() + } + + /// The strides given in number of elements for a contiguous n-dimensional + /// arrays using this shape. + pub(crate) fn stride_contiguous(&self) -> Vec { + let mut stride: Vec<_> = self + .0 + .iter() + .rev() + .scan(1, |prod, u| { + let prod_pre_mult = *prod; + *prod *= u; + Some(prod_pre_mult) + }) + .collect(); + stride.reverse(); + stride + } + + /// Returns true if the strides are C contiguous (aka row major). + pub fn is_contiguous(&self, stride: &[usize]) -> bool { + if self.0.len() != stride.len() { + return false; + } + let mut acc = 1; + for (&stride, &dim) in stride.iter().zip(self.0.iter()).rev() { + if stride != acc { + return false; + } + acc *= dim; + } + true + } + + /// Returns true if the strides are Fortran contiguous (aka column major). + pub fn is_fortran_contiguous(&self, stride: &[usize]) -> bool { + if self.0.len() != stride.len() { + return false; + } + let mut acc = 1; + for (&stride, &dim) in stride.iter().zip(self.0.iter()) { + if stride != acc { + return false; + } + acc *= dim; + } + true + } + + /// Modifies the shape by adding a list of additional dimensions at the end of the existing + /// dimensions. + pub fn extend(mut self, additional_dims: &[usize]) -> Self { + self.0.extend(additional_dims); + self + } + + /// Check whether the two shapes are compatible for broadcast, and if it is the case return the + /// broadcasted shape. This is to be used for binary pointwise ops. + pub fn broadcast_shape_binary_op(&self, rhs: &Self, op: &'static str) -> Result { + let lhs = self; + let lhs_dims = lhs.dims(); + let rhs_dims = rhs.dims(); + let lhs_ndims = lhs_dims.len(); + let rhs_ndims = rhs_dims.len(); + let bcast_ndims = usize::max(lhs_ndims, rhs_ndims); + let mut bcast_dims = vec![0; bcast_ndims]; + for (idx, bcast_value) in bcast_dims.iter_mut().enumerate() { + let rev_idx = bcast_ndims - idx; + let l_value = if lhs_ndims < rev_idx { + 1 + } else { + lhs_dims[lhs_ndims - rev_idx] + }; + let r_value = if rhs_ndims < rev_idx { + 1 + } else { + rhs_dims[rhs_ndims - rev_idx] + }; + *bcast_value = if l_value == r_value { + l_value + } else if l_value == 1 { + r_value + } else if r_value == 1 { + l_value + } else { + Err(Error::ShapeMismatchBinaryOp { + lhs: lhs.clone(), + rhs: rhs.clone(), + op, + } + .bt())? + } + } + Ok(Shape::from(bcast_dims)) + } + + pub(crate) fn broadcast_shape_matmul(&self, rhs: &Self) -> Result<(Shape, Shape)> { + let lhs = self; + let lhs_dims = lhs.dims(); + let rhs_dims = rhs.dims(); + if lhs_dims.len() < 2 || rhs_dims.len() < 2 { + crate::bail!("only 2d matrixes are supported {lhs:?} {rhs:?}") + } + let (m, lhs_k) = (lhs_dims[lhs_dims.len() - 2], lhs_dims[lhs_dims.len() - 1]); + let (rhs_k, n) = (rhs_dims[rhs_dims.len() - 2], rhs_dims[rhs_dims.len() - 1]); + if lhs_k != rhs_k { + crate::bail!("different inner dimensions in broadcast matmul {lhs:?} {rhs:?}") + } + + let lhs_b = Self::from(&lhs_dims[..lhs_dims.len() - 2]); + let rhs_b = Self::from(&rhs_dims[..rhs_dims.len() - 2]); + let bcast = lhs_b.broadcast_shape_binary_op(&rhs_b, "broadcast_matmul")?; + let bcast_dims = bcast.dims(); + + let bcast_lhs = [bcast_dims, &[m, lhs_k]].concat(); + let bcast_rhs = [bcast_dims, &[rhs_k, n]].concat(); + Ok((Shape::from(bcast_lhs), Shape::from(bcast_rhs))) + } +} + +pub trait Dim { + fn to_index(&self, shape: &Shape, op: &'static str) -> Result; + fn to_index_plus_one(&self, shape: &Shape, op: &'static str) -> Result; +} + +impl Dim for usize { + fn to_index(&self, shape: &Shape, op: &'static str) -> Result { + let dim = *self; + if dim >= shape.dims().len() { + Err(Error::DimOutOfRange { + shape: shape.clone(), + dim: dim as i32, + op, + } + .bt())? + } else { + Ok(dim) + } + } + + fn to_index_plus_one(&self, shape: &Shape, op: &'static str) -> Result { + let dim = *self; + if dim > shape.dims().len() { + Err(Error::DimOutOfRange { + shape: shape.clone(), + dim: dim as i32, + op, + } + .bt())? + } else { + Ok(dim) + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub enum D { + Minus1, + Minus2, +} + +impl D { + fn out_of_range(&self, shape: &Shape, op: &'static str) -> Error { + let dim = match self { + Self::Minus1 => -1, + Self::Minus2 => -2, + }; + Error::DimOutOfRange { + shape: shape.clone(), + dim, + op, + } + .bt() + } +} + +impl Dim for D { + fn to_index(&self, shape: &Shape, op: &'static str) -> Result { + let rank = shape.rank(); + match self { + Self::Minus1 if rank >= 1 => Ok(rank - 1), + Self::Minus2 if rank >= 2 => Ok(rank - 2), + _ => Err(self.out_of_range(shape, op)), + } + } + + fn to_index_plus_one(&self, shape: &Shape, op: &'static str) -> Result { + let rank = shape.rank(); + match self { + Self::Minus1 => Ok(rank), + Self::Minus2 if rank >= 1 => Ok(rank - 1), + _ => Err(self.out_of_range(shape, op)), + } + } +} + +pub trait Dims: Sized { + fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result>; + + fn to_indexes(self, shape: &Shape, op: &'static str) -> Result> { + let dims = self.to_indexes_internal(shape, op)?; + for (i, &dim) in dims.iter().enumerate() { + if dims[..i].contains(&dim) { + Err(Error::DuplicateDimIndex { + shape: shape.clone(), + dims: dims.clone(), + op, + } + .bt())? + } + if dim >= shape.rank() { + Err(Error::DimOutOfRange { + shape: shape.clone(), + dim: dim as i32, + op, + } + .bt())? + } + } + Ok(dims) + } +} + +impl Dims for Vec { + fn to_indexes_internal(self, _: &Shape, _: &'static str) -> Result> { + Ok(self) + } +} + +impl Dims for [usize; N] { + fn to_indexes_internal(self, _: &Shape, _: &'static str) -> Result> { + Ok(self.to_vec()) + } +} + +impl Dims for &[usize] { + fn to_indexes_internal(self, _: &Shape, _: &'static str) -> Result> { + Ok(self.to_vec()) + } +} + +impl Dims for () { + fn to_indexes_internal(self, _: &Shape, _: &'static str) -> Result> { + Ok(vec![]) + } +} + +impl Dims for D { + fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { + let dim = self.to_index(shape, op)?; + Ok(vec![dim]) + } +} + +impl Dims for (D,) { + fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { + let dim = self.0.to_index(shape, op)?; + Ok(vec![dim]) + } +} + +impl Dims for (D1, D2) { + fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { + let d0 = self.0.to_index(shape, op)?; + let d1 = self.1.to_index(shape, op)?; + Ok(vec![d0, d1]) + } +} + +impl Dims for (D1, D2, D3) { + fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { + let d0 = self.0.to_index(shape, op)?; + let d1 = self.1.to_index(shape, op)?; + let d2 = self.2.to_index(shape, op)?; + Ok(vec![d0, d1, d2]) + } +} + +impl Dims for (D1, D2, D3, D4) { + fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { + let d0 = self.0.to_index(shape, op)?; + let d1 = self.1.to_index(shape, op)?; + let d2 = self.2.to_index(shape, op)?; + let d3 = self.3.to_index(shape, op)?; + Ok(vec![d0, d1, d2, d3]) + } +} + +impl Dims for (D1, D2, D3, D4, D5) { + fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { + let d0 = self.0.to_index(shape, op)?; + let d1 = self.1.to_index(shape, op)?; + let d2 = self.2.to_index(shape, op)?; + let d3 = self.3.to_index(shape, op)?; + let d4 = self.4.to_index(shape, op)?; + Ok(vec![d0, d1, d2, d3, d4]) + } +} + +impl Dims for (D1, D2, D3, D4, D5, D6) { + fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { + let d0 = self.0.to_index(shape, op)?; + let d1 = self.1.to_index(shape, op)?; + let d2 = self.2.to_index(shape, op)?; + let d3 = self.3.to_index(shape, op)?; + let d4 = self.4.to_index(shape, op)?; + let d5 = self.5.to_index(shape, op)?; + Ok(vec![d0, d1, d2, d3, d4, d5]) + } +} + +extract_dims!(dims0, 0, |_: &[usize]| (), ()); +extract_dims!(dims1, 1, |d: &[usize]| d[0], usize); +extract_dims!(dims2, 2, |d: &[usize]| (d[0], d[1]), (usize, usize)); +extract_dims!( + dims3, + 3, + |d: &[usize]| (d[0], d[1], d[2]), + (usize, usize, usize) +); +extract_dims!( + dims4, + 4, + |d: &[usize]| (d[0], d[1], d[2], d[3]), + (usize, usize, usize, usize) +); +extract_dims!( + dims5, + 5, + |d: &[usize]| (d[0], d[1], d[2], d[3], d[4]), + (usize, usize, usize, usize, usize) +); + +pub trait ShapeWithOneHole { + fn into_shape(self, el_count: usize) -> Result; +} + +impl> ShapeWithOneHole for S { + fn into_shape(self, _el_count: usize) -> Result { + Ok(self.into()) + } +} + +impl ShapeWithOneHole for ((),) { + fn into_shape(self, el_count: usize) -> Result { + Ok(el_count.into()) + } +} + +fn hole_size(el_count: usize, prod_d: usize, s: &dyn std::fmt::Debug) -> Result { + if prod_d == 0 { + crate::bail!("cannot reshape tensor of {el_count} elements to {s:?}") + } + if el_count % prod_d != 0 { + crate::bail!("cannot reshape tensor with {el_count} elements to {s:?}") + } + Ok(el_count / prod_d) +} + +impl ShapeWithOneHole for ((), usize) { + fn into_shape(self, el_count: usize) -> Result { + let ((), d1) = self; + Ok((hole_size(el_count, d1, &self)?, d1).into()) + } +} + +impl ShapeWithOneHole for (usize, ()) { + fn into_shape(self, el_count: usize) -> Result { + let (d1, ()) = self; + Ok((d1, hole_size(el_count, d1, &self)?).into()) + } +} + +impl ShapeWithOneHole for ((), usize, usize) { + fn into_shape(self, el_count: usize) -> Result { + let ((), d1, d2) = self; + Ok((hole_size(el_count, d1 * d2, &self)?, d1, d2).into()) + } +} + +impl ShapeWithOneHole for (usize, (), usize) { + fn into_shape(self, el_count: usize) -> Result { + let (d1, (), d2) = self; + Ok((d1, hole_size(el_count, d1 * d2, &self)?, d2).into()) + } +} + +impl ShapeWithOneHole for (usize, usize, ()) { + fn into_shape(self, el_count: usize) -> Result { + let (d1, d2, ()) = self; + Ok((d1, d2, hole_size(el_count, d1 * d2, &self)?).into()) + } +} + +impl ShapeWithOneHole for ((), usize, usize, usize) { + fn into_shape(self, el_count: usize) -> Result { + let ((), d1, d2, d3) = self; + let d = hole_size(el_count, d1 * d2 * d3, &self)?; + Ok((d, d1, d2, d3).into()) + } +} + +impl ShapeWithOneHole for (usize, (), usize, usize) { + fn into_shape(self, el_count: usize) -> Result { + let (d1, (), d2, d3) = self; + let d = hole_size(el_count, d1 * d2 * d3, &self)?; + Ok((d1, d, d2, d3).into()) + } +} + +impl ShapeWithOneHole for (usize, usize, (), usize) { + fn into_shape(self, el_count: usize) -> Result { + let (d1, d2, (), d3) = self; + let d = hole_size(el_count, d1 * d2 * d3, &self)?; + Ok((d1, d2, d, d3).into()) + } +} + +impl ShapeWithOneHole for (usize, usize, usize, ()) { + fn into_shape(self, el_count: usize) -> Result { + let (d1, d2, d3, ()) = self; + let d = hole_size(el_count, d1 * d2 * d3, &self)?; + Ok((d1, d2, d3, d).into()) + } +} + +impl ShapeWithOneHole for ((), usize, usize, usize, usize) { + fn into_shape(self, el_count: usize) -> Result { + let ((), d1, d2, d3, d4) = self; + let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?; + Ok((d, d1, d2, d3, d4).into()) + } +} + +impl ShapeWithOneHole for (usize, (), usize, usize, usize) { + fn into_shape(self, el_count: usize) -> Result { + let (d1, (), d2, d3, d4) = self; + let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?; + Ok((d1, d, d2, d3, d4).into()) + } +} + +impl ShapeWithOneHole for (usize, usize, (), usize, usize) { + fn into_shape(self, el_count: usize) -> Result { + let (d1, d2, (), d3, d4) = self; + let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?; + Ok((d1, d2, d, d3, d4).into()) + } +} + +impl ShapeWithOneHole for (usize, usize, usize, (), usize) { + fn into_shape(self, el_count: usize) -> Result { + let (d1, d2, d3, (), d4) = self; + let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?; + Ok((d1, d2, d3, d, d4).into()) + } +} + +impl ShapeWithOneHole for (usize, usize, usize, usize, ()) { + fn into_shape(self, el_count: usize) -> Result { + let (d1, d2, d3, d4, ()) = self; + let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?; + Ok((d1, d2, d3, d4, d).into()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stride() { + let shape = Shape::from(()); + assert_eq!(shape.stride_contiguous(), Vec::::new()); + let shape = Shape::from(42); + assert_eq!(shape.stride_contiguous(), [1]); + let shape = Shape::from((42, 1337)); + assert_eq!(shape.stride_contiguous(), [1337, 1]); + let shape = Shape::from((299, 792, 458)); + assert_eq!(shape.stride_contiguous(), [458 * 792, 458, 1]); + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/storage.rs b/v2/dependency-patches/candle-core-0.4.1/src/storage.rs new file mode 100644 index 0000000000..9e1a2c1dee --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/storage.rs @@ -0,0 +1,700 @@ +use crate::backend::BackendStorage; +use crate::op::{self, CmpOp, CustomOp1, CustomOp2, CustomOp3, ReduceOp}; +use crate::{CpuStorage, CudaStorage, DType, Device, Error, Layout, MetalStorage, Result, Shape}; + +// We do not want to implement Clone on Storage as cloning may fail because of +// out of memory. Instead try_clone should be used. +#[derive(Debug)] +pub enum Storage { + Cpu(CpuStorage), + Cuda(CudaStorage), + Metal(MetalStorage), +} + +impl Storage { + pub fn try_clone(&self, layout: &Layout) -> Result { + match self { + Self::Cpu(storage) => Ok(Self::Cpu(storage.clone())), + Self::Cuda(storage) => { + let storage = storage.try_clone(layout)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = storage.try_clone(layout)?; + Ok(Self::Metal(storage)) + } + } + } + + pub fn device(&self) -> Device { + match self { + Self::Cpu(_) => Device::Cpu, + Self::Cuda(storage) => Device::Cuda(storage.device().clone()), + Self::Metal(storage) => Device::Metal(storage.device().clone()), + } + } + + pub fn dtype(&self) -> DType { + match self { + Self::Cpu(storage) => storage.dtype(), + Self::Cuda(storage) => storage.dtype(), + Self::Metal(storage) => storage.dtype(), + } + } + + pub(crate) fn same_device(&self, rhs: &Self, op: &'static str) -> Result<()> { + let lhs = self.device().location(); + let rhs = rhs.device().location(); + if lhs != rhs { + Err(Error::DeviceMismatchBinaryOp { lhs, rhs, op }.bt()) + } else { + Ok(()) + } + } + + pub(crate) fn same_dtype(&self, rhs: &Self, op: &'static str) -> Result<()> { + let lhs = self.dtype(); + let rhs = rhs.dtype(); + if lhs != rhs { + Err(Error::DTypeMismatchBinaryOp { lhs, rhs, op }.bt()) + } else { + Ok(()) + } + } + + pub(crate) fn affine(&self, layout: &Layout, mul: f64, add: f64) -> Result { + match self { + Storage::Cpu(storage) => { + let storage = storage.affine(layout, mul, add)?; + Ok(Self::Cpu(storage)) + } + Self::Cuda(storage) => { + let storage = storage.affine(layout, mul, add)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = storage.affine(layout, mul, add)?; + Ok(Self::Metal(storage)) + } + } + } + + pub(crate) fn powf(&self, layout: &Layout, alpha: f64) -> Result { + match self { + Storage::Cpu(storage) => { + let storage = storage.powf(layout, alpha)?; + Ok(Self::Cpu(storage)) + } + Self::Cuda(storage) => { + let storage = storage.powf(layout, alpha)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = storage.powf(layout, alpha)?; + Ok(Self::Metal(storage)) + } + } + } + + pub(crate) fn elu(&self, layout: &Layout, alpha: f64) -> Result { + match self { + Storage::Cpu(storage) => { + let storage = storage.elu(layout, alpha)?; + Ok(Self::Cpu(storage)) + } + Self::Cuda(storage) => { + let storage = storage.elu(layout, alpha)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = storage.elu(layout, alpha)?; + Ok(Self::Metal(storage)) + } + } + } + + pub(crate) fn cmp( + &self, + op: CmpOp, + rhs: &Self, + lhs_layout: &Layout, + rhs_layout: &Layout, + ) -> Result { + self.same_device(rhs, "cmp")?; + self.same_dtype(rhs, "cmp")?; + match (self, rhs) { + (Storage::Cpu(lhs), Storage::Cpu(rhs)) => { + let storage = lhs.cmp(op, rhs, lhs_layout, rhs_layout)?; + Ok(Self::Cpu(storage)) + } + (Self::Cuda(lhs), Self::Cuda(rhs)) => { + let storage = lhs.cmp(op, rhs, lhs_layout, rhs_layout)?; + Ok(Self::Cuda(storage)) + } + (Self::Metal(lhs), Self::Metal(rhs)) => { + let storage = lhs.cmp(op, rhs, lhs_layout, rhs_layout)?; + Ok(Self::Metal(storage)) + } + (lhs, rhs) => { + // Should not happen because of the same device check above but we're defensive + // anyway. + Err(Error::DeviceMismatchBinaryOp { + lhs: lhs.device().location(), + rhs: rhs.device().location(), + op: "cmp", + } + .bt()) + } + } + } + + pub(crate) fn reduce_op(&self, op: ReduceOp, layout: &Layout, s: &[usize]) -> Result { + match self { + Storage::Cpu(storage) => { + let storage = storage.reduce_op(op, layout, s)?; + Ok(Self::Cpu(storage)) + } + Self::Cuda(storage) => { + let storage = storage.reduce_op(op, layout, s)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = storage.reduce_op(op, layout, s)?; + Ok(Self::Metal(storage)) + } + } + } + + pub(crate) fn to_dtype(&self, layout: &Layout, dtype: DType) -> Result { + match self { + Storage::Cpu(storage) => { + let storage = storage.to_dtype(layout, dtype)?; + Ok(Self::Cpu(storage)) + } + Self::Cuda(storage) => { + let storage = storage.to_dtype(layout, dtype)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = storage.to_dtype(layout, dtype)?; + Ok(Self::Metal(storage)) + } + } + } + + pub(crate) fn apply_op1(&self, l: &Layout, c: &dyn CustomOp1) -> Result<(Self, Shape)> { + match self { + Self::Cpu(storage) => { + let (storage, shape) = c.cpu_fwd(storage, l)?; + Ok((Self::Cpu(storage), shape)) + } + Self::Cuda(storage) => { + let (storage, shape) = c.cuda_fwd(storage, l)?; + Ok((Self::Cuda(storage), shape)) + } + Self::Metal(storage) => { + let (storage, shape) = c.metal_fwd(storage, l)?; + Ok((Self::Metal(storage), shape)) + } + } + } + + pub(crate) fn apply_op2( + &self, + l1: &Layout, + t2: &Self, + l2: &Layout, + c: &dyn CustomOp2, + ) -> Result<(Self, Shape)> { + self.same_device(t2, c.name())?; + match (self, t2) { + (Self::Cpu(s1), Self::Cpu(s2)) => { + let (s, shape) = c.cpu_fwd(s1, l1, s2, l2)?; + Ok((Self::Cpu(s), shape)) + } + (Self::Cuda(s1), Self::Cuda(s2)) => { + let (s, shape) = c.cuda_fwd(s1, l1, s2, l2)?; + Ok((Self::Cuda(s), shape)) + } + (Self::Metal(s1), Self::Metal(s2)) => { + let (s, shape) = c.metal_fwd(s1, l1, s2, l2)?; + Ok((Self::Metal(s), shape)) + } + _ => unreachable!(), + } + } + + pub(crate) fn apply_op3( + &self, + l1: &Layout, + t2: &Self, + l2: &Layout, + t3: &Self, + l3: &Layout, + c: &dyn CustomOp3, + ) -> Result<(Self, Shape)> { + self.same_device(t2, c.name())?; + self.same_device(t3, c.name())?; + match (self, t2, t3) { + (Self::Cpu(s1), Self::Cpu(s2), Self::Cpu(s3)) => { + let (s, shape) = c.cpu_fwd(s1, l1, s2, l2, s3, l3)?; + Ok((Self::Cpu(s), shape)) + } + (Self::Cuda(s1), Self::Cuda(s2), Self::Cuda(s3)) => { + let (s, shape) = c.cuda_fwd(s1, l1, s2, l2, s3, l3)?; + Ok((Self::Cuda(s), shape)) + } + (Self::Metal(s1), Self::Metal(s2), Self::Metal(s3)) => { + let (s, shape) = c.metal_fwd(s1, l1, s2, l2, s3, l3)?; + Ok((Self::Metal(s), shape)) + } + _ => unreachable!(), + } + } + + pub(crate) fn unary_impl(&self, layout: &Layout) -> Result { + match self { + Storage::Cpu(storage) => { + let storage = storage.unary_impl::(layout)?; + Ok(Self::Cpu(storage)) + } + Self::Cuda(storage) => { + let storage = storage.unary_impl::(layout)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = storage.unary_impl::(layout)?; + Ok(Self::Metal(storage)) + } + } + } + + pub(crate) fn binary_impl( + &self, + rhs: &Self, + lhs_layout: &Layout, + rhs_layout: &Layout, + ) -> Result { + self.same_device(rhs, B::NAME)?; + self.same_dtype(rhs, B::NAME)?; + match (self, rhs) { + (Storage::Cpu(lhs), Storage::Cpu(rhs)) => { + let storage = lhs.binary_impl::(rhs, lhs_layout, rhs_layout)?; + Ok(Self::Cpu(storage)) + } + (Self::Cuda(lhs), Self::Cuda(rhs)) => { + let storage = lhs.binary_impl::(rhs, lhs_layout, rhs_layout)?; + Ok(Self::Cuda(storage)) + } + (Self::Metal(lhs), Self::Metal(rhs)) => { + let storage = lhs.binary_impl::(rhs, lhs_layout, rhs_layout)?; + Ok(Self::Metal(storage)) + } + (lhs, rhs) => { + // Should not happen because of the same device check above but we're defensive + // anyway. + Err(Error::DeviceMismatchBinaryOp { + lhs: lhs.device().location(), + rhs: rhs.device().location(), + op: B::NAME, + } + .bt()) + } + } + } + + pub(crate) fn conv1d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConv1D, + ) -> Result { + self.same_device(kernel, "conv1d")?; + self.same_dtype(kernel, "conv1d")?; + match (self, &kernel) { + (Storage::Cpu(inp), Storage::Cpu(kernel)) => { + let s = inp.conv1d(l, kernel, kernel_l, params)?; + Ok(Self::Cpu(s)) + } + (Storage::Cuda(inp), Storage::Cuda(kernel)) => { + let s = inp.conv1d(l, kernel, kernel_l, params)?; + Ok(Self::Cuda(s)) + } + (Storage::Metal(inp), Storage::Metal(kernel)) => { + let s = inp.conv1d(l, kernel, kernel_l, params)?; + Ok(Self::Metal(s)) + } + (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { + lhs: lhs.device().location(), + rhs: rhs.device().location(), + op: "conv1d", + } + .bt()), + } + } + + pub(crate) fn conv_transpose1d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConvTranspose1D, + ) -> Result { + self.same_device(kernel, "conv-transpose1d")?; + self.same_dtype(kernel, "conv-transpose1d")?; + match (self, &kernel) { + (Storage::Cpu(inp), Storage::Cpu(kernel)) => { + let s = inp.conv_transpose1d(l, kernel, kernel_l, params)?; + Ok(Self::Cpu(s)) + } + (Storage::Cuda(inp), Storage::Cuda(kernel)) => { + let s = inp.conv_transpose1d(l, kernel, kernel_l, params)?; + Ok(Self::Cuda(s)) + } + (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { + lhs: lhs.device().location(), + rhs: rhs.device().location(), + op: "conv-transpose1d", + } + .bt()), + } + } + + pub(crate) fn conv2d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConv2D, + ) -> Result { + self.same_device(kernel, "conv2d")?; + self.same_dtype(kernel, "conv2d")?; + match (self, &kernel) { + (Storage::Cpu(inp), Storage::Cpu(kernel)) => { + let s = inp.conv2d(l, kernel, kernel_l, params)?; + Ok(Self::Cpu(s)) + } + (Storage::Cuda(inp), Storage::Cuda(kernel)) => { + let s = inp.conv2d(l, kernel, kernel_l, params)?; + Ok(Self::Cuda(s)) + } + (Storage::Metal(inp), Storage::Metal(kernel)) => { + let s = inp.conv2d(l, kernel, kernel_l, params)?; + Ok(Self::Metal(s)) + } + (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { + lhs: lhs.device().location(), + rhs: rhs.device().location(), + op: "conv2d", + } + .bt()), + } + } + + pub(crate) fn conv_transpose2d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConvTranspose2D, + ) -> Result { + self.same_device(kernel, "conv_transpose2d")?; + self.same_dtype(kernel, "conv_transpose2d")?; + match (self, &kernel) { + (Storage::Cpu(inp), Storage::Cpu(kernel)) => { + let s = inp.conv_transpose2d(l, kernel, kernel_l, params)?; + Ok(Self::Cpu(s)) + } + (Storage::Cuda(inp), Storage::Cuda(kernel)) => { + let s = inp.conv_transpose2d(l, kernel, kernel_l, params)?; + Ok(Self::Cuda(s)) + } + (Storage::Metal(inp), Storage::Metal(kernel)) => { + let s = inp.conv_transpose2d(l, kernel, kernel_l, params)?; + Ok(Self::Metal(s)) + } + (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { + lhs: lhs.device().location(), + rhs: rhs.device().location(), + op: "conv_transpose2d", + } + .bt()), + } + } + + pub(crate) fn avg_pool2d( + &self, + layout: &Layout, + kernel_size: (usize, usize), + stride: (usize, usize), + ) -> Result { + match self { + Storage::Cpu(storage) => { + let storage = storage.avg_pool2d(layout, kernel_size, stride)?; + Ok(Self::Cpu(storage)) + } + Self::Cuda(storage) => { + let storage = storage.avg_pool2d(layout, kernel_size, stride)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = storage.avg_pool2d(layout, kernel_size, stride)?; + Ok(Self::Metal(storage)) + } + } + } + + pub(crate) fn max_pool2d( + &self, + layout: &Layout, + kernel_size: (usize, usize), + stride: (usize, usize), + ) -> Result { + match self { + Storage::Cpu(storage) => { + let storage = storage.max_pool2d(layout, kernel_size, stride)?; + Ok(Self::Cpu(storage)) + } + Self::Cuda(storage) => { + let storage = storage.max_pool2d(layout, kernel_size, stride)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = storage.max_pool2d(layout, kernel_size, stride)?; + Ok(Self::Metal(storage)) + } + } + } + + pub(crate) fn upsample_nearest1d(&self, layout: &Layout, sz: usize) -> Result { + match self { + Storage::Cpu(storage) => { + let storage = storage.upsample_nearest1d(layout, sz)?; + Ok(Self::Cpu(storage)) + } + Self::Cuda(storage) => { + let storage = storage.upsample_nearest1d(layout, sz)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = storage.upsample_nearest1d(layout, sz)?; + Ok(Self::Metal(storage)) + } + } + } + + pub(crate) fn upsample_nearest2d(&self, layout: &Layout, h: usize, w: usize) -> Result { + match self { + Storage::Cpu(storage) => { + let storage = storage.upsample_nearest2d(layout, h, w)?; + Ok(Self::Cpu(storage)) + } + Self::Cuda(storage) => { + let storage = storage.upsample_nearest2d(layout, h, w)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = storage.upsample_nearest2d(layout, h, w)?; + Ok(Self::Metal(storage)) + } + } + } + + pub(crate) fn where_cond( + &self, + layout: &Layout, + t: &Self, + layout_t: &Layout, + f: &Self, + layout_f: &Layout, + ) -> Result { + self.same_device(t, "where")?; + self.same_device(f, "where")?; + t.same_dtype(f, "where")?; + match (self, t, f) { + (Storage::Cpu(cond), Storage::Cpu(t), Storage::Cpu(f)) => { + let storage = cond.where_cond(layout, t, layout_t, f, layout_f)?; + Ok(Self::Cpu(storage)) + } + (Self::Cuda(cond), Self::Cuda(t), Self::Cuda(f)) => { + let storage = cond.where_cond(layout, t, layout_t, f, layout_f)?; + Ok(Self::Cuda(storage)) + } + (Self::Metal(cond), Self::Metal(t), Self::Metal(f)) => { + let storage = cond.where_cond(layout, t, layout_t, f, layout_f)?; + Ok(Self::Metal(storage)) + } + (_, lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { + lhs: lhs.device().location(), + rhs: rhs.device().location(), + op: "where", + } + .bt()), + } + } + + pub(crate) fn gather( + &self, + l: &Layout, + indexes: &Self, + indexes_l: &Layout, + d: usize, + ) -> Result { + self.same_device(indexes, "index-add")?; + match (self, indexes) { + (Self::Cpu(s), Self::Cpu(indexes)) => { + let storage = s.gather(l, indexes, indexes_l, d)?; + Ok(Self::Cpu(storage)) + } + (Self::Cuda(s), Self::Cuda(indexes)) => { + let storage = s.gather(l, indexes, indexes_l, d)?; + Ok(Self::Cuda(storage)) + } + (Self::Metal(s), Self::Metal(indexes)) => { + let storage = s.gather(l, indexes, indexes_l, d)?; + Ok(Self::Metal(storage)) + } + _ => unreachable!(), + } + } + + pub(crate) fn scatter_add( + &self, + l: &Layout, + indexes: &Self, + indexes_l: &Layout, + source: &Self, + source_l: &Layout, + d: usize, + ) -> Result { + self.same_device(indexes, "scatter-add")?; + self.same_device(source, "scatter-add")?; + match (self, indexes, source) { + (Self::Cpu(s), Self::Cpu(indexes), Self::Cpu(source)) => { + let storage = s.scatter_add(l, indexes, indexes_l, source, source_l, d)?; + Ok(Self::Cpu(storage)) + } + (Self::Cuda(s), Self::Cuda(indexes), Self::Cuda(source)) => { + let storage = s.scatter_add(l, indexes, indexes_l, source, source_l, d)?; + Ok(Self::Cuda(storage)) + } + (Self::Metal(s), Self::Metal(indexes), Self::Metal(source)) => { + let storage = s.scatter_add(l, indexes, indexes_l, source, source_l, d)?; + Ok(Self::Metal(storage)) + } + _ => unreachable!(), + } + } + + pub(crate) fn index_add( + &self, + l: &Layout, + indexes: &Self, + indexes_l: &Layout, + source: &Self, + source_l: &Layout, + d: usize, + ) -> Result { + self.same_device(indexes, "index-add")?; + self.same_device(source, "index-add")?; + match (self, indexes, source) { + (Self::Cpu(s), Self::Cpu(indexes), Self::Cpu(source)) => { + let storage = s.index_add(l, indexes, indexes_l, source, source_l, d)?; + Ok(Self::Cpu(storage)) + } + (Self::Cuda(s), Self::Cuda(indexes), Self::Cuda(source)) => { + let storage = s.index_add(l, indexes, indexes_l, source, source_l, d)?; + Ok(Self::Cuda(storage)) + } + (Self::Metal(s), Self::Metal(indexes), Self::Metal(source)) => { + let storage = s.index_add(l, indexes, indexes_l, source, source_l, d)?; + Ok(Self::Metal(storage)) + } + _ => unreachable!(), + } + } + + pub(crate) fn index_select( + &self, + rhs: &Self, + lhs_l: &Layout, + rhs_l: &Layout, + d: usize, + ) -> Result { + self.same_device(rhs, "index-select")?; + match (self, rhs) { + (Self::Cpu(lhs), Self::Cpu(rhs)) => { + let storage = lhs.index_select(rhs, lhs_l, rhs_l, d)?; + Ok(Self::Cpu(storage)) + } + (Self::Cuda(lhs), Self::Cuda(rhs)) => { + let storage = lhs.index_select(rhs, lhs_l, rhs_l, d)?; + Ok(Self::Cuda(storage)) + } + (Self::Metal(lhs), Self::Metal(rhs)) => { + let storage = lhs.index_select(rhs, lhs_l, rhs_l, d)?; + Ok(Self::Metal(storage)) + } + (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { + lhs: lhs.device().location(), + rhs: rhs.device().location(), + op: "index-select", + } + .bt()), + } + } + + pub(crate) fn matmul( + &self, + rhs: &Self, + bmnk: (usize, usize, usize, usize), + lhs_layout: &Layout, + rhs_layout: &Layout, + ) -> Result { + self.same_device(rhs, "matmul")?; + self.same_dtype(rhs, "matmul")?; + match (self, rhs) { + (Self::Cpu(lhs), Self::Cpu(rhs)) => { + let storage = lhs.matmul(rhs, bmnk, lhs_layout, rhs_layout)?; + Ok(Self::Cpu(storage)) + } + (Self::Cuda(lhs), Self::Cuda(rhs)) => { + let storage = lhs.matmul(rhs, bmnk, lhs_layout, rhs_layout)?; + Ok(Self::Cuda(storage)) + } + (Self::Metal(lhs), Self::Metal(rhs)) => { + let storage = lhs.matmul(rhs, bmnk, lhs_layout, rhs_layout)?; + Ok(Self::Metal(storage)) + } + (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { + lhs: lhs.device().location(), + rhs: rhs.device().location(), + op: "matmul", + } + .bt()), + } + } + + // self, the source can be strided whereas dst is contiguous. + pub(crate) fn copy_strided_src( + &self, + dst: &mut Self, + dst_offset: usize, + src_l: &Layout, + ) -> Result<()> { + match (self, dst) { + (Self::Cpu(src), Self::Cpu(dst)) => src.copy_strided_src(dst, dst_offset, src_l), + (Self::Cuda(src), Self::Cuda(dst)) => Ok(src.copy_strided_src(dst, dst_offset, src_l)?), + (Self::Metal(src), Self::Metal(dst)) => { + Ok(src.copy_strided_src(dst, dst_offset, src_l)?) + } + (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { + lhs: lhs.device().location(), + rhs: rhs.device().location(), + op: "copy", + } + .bt()), + } + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/strided_index.rs b/v2/dependency-patches/candle-core-0.4.1/src/strided_index.rs new file mode 100644 index 0000000000..eb6a736f83 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/strided_index.rs @@ -0,0 +1,82 @@ +use crate::Layout; + +/// An iterator over offset position for items of an N-dimensional arrays stored in a +/// flat buffer using some potential strides. +#[derive(Debug)] +pub struct StridedIndex<'a> { + next_storage_index: Option, + multi_index: Vec, + dims: &'a [usize], + stride: &'a [usize], +} + +impl<'a> StridedIndex<'a> { + pub(crate) fn new(dims: &'a [usize], stride: &'a [usize], start_offset: usize) -> Self { + let elem_count: usize = dims.iter().product(); + let next_storage_index = if elem_count == 0 { + None + } else { + // This applies to the scalar case. + Some(start_offset) + }; + StridedIndex { + next_storage_index, + multi_index: vec![0; dims.len()], + dims, + stride, + } + } + + pub(crate) fn from_layout(l: &'a Layout) -> Self { + Self::new(l.dims(), l.stride(), l.start_offset()) + } +} + +impl<'a> Iterator for StridedIndex<'a> { + type Item = usize; + + fn next(&mut self) -> Option { + let storage_index = match self.next_storage_index { + None => return None, + Some(storage_index) => storage_index, + }; + let mut updated = false; + let mut next_storage_index = storage_index; + for ((multi_i, max_i), stride_i) in self + .multi_index + .iter_mut() + .zip(self.dims.iter()) + .zip(self.stride.iter()) + .rev() + { + let next_i = *multi_i + 1; + if next_i < *max_i { + *multi_i = next_i; + updated = true; + next_storage_index += stride_i; + break; + } else { + next_storage_index -= *multi_i * stride_i; + *multi_i = 0 + } + } + self.next_storage_index = if updated { + Some(next_storage_index) + } else { + None + }; + Some(storage_index) + } +} + +#[derive(Debug)] +pub enum StridedBlocks<'a> { + SingleBlock { + start_offset: usize, + len: usize, + }, + MultipleBlocks { + block_start_index: StridedIndex<'a>, + block_len: usize, + }, +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/tensor.rs b/v2/dependency-patches/candle-core-0.4.1/src/tensor.rs new file mode 100644 index 0000000000..0e2c3e8f32 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/tensor.rs @@ -0,0 +1,2765 @@ +//! Tensors are N-dimensional matrixes of elements using a single data type. +#![allow(clippy::redundant_closure_call)] +use crate::backend::{BackendDevice, BackendStorage}; +use crate::op::{ + BackpropOp, BinaryOp, CmpOp, CustomOp1, CustomOp2, CustomOp3, Op, ReduceOp, UnaryOp, +}; +use crate::scalar::TensorOrScalar; +use crate::shape::{Dim, Dims}; +use crate::{bail, storage::Storage, DType, Device, Error, Layout, Result, Shape}; +use std::sync::{Arc, RwLock}; + +/// Unique identifier for tensors. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct TensorId(usize); + +impl TensorId { + fn new() -> Self { + // https://users.rust-lang.org/t/idiomatic-rust-way-to-generate-unique-id/33805 + use std::sync::atomic; + static COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::new(1); + Self(COUNTER.fetch_add(1, atomic::Ordering::Relaxed)) + } +} + +pub struct Tensor_ { + id: TensorId, + // As we provide inner mutability on the tensor content, the alternatives are: + // - Using a mutex, this would have the highest cost when retrieving the storage but would + // prevent errors when concurrent access takes place. Mutex would also be subject to + // deadlocks for example using the current code if the same tensor is used twice by a single + // binary op. + // - Using a refcell unsafe cell would have some intermediary cost, borrow checking would be + // verified dynamically, but the resulting tensors would not be send or sync. + // - Using an unsafe cell would have the lowest cost but undefined behavior on concurrent + // accesses. + // Ideally, we would use Arc for tensors on which we don't plan on modifying the data + // and Arc> for tensors where the data could be modified, e.g. variables but + // that's tricky to encode in the current setup. + storage: Arc>, + layout: Layout, + op: BackpropOp, + is_variable: bool, + dtype: DType, + device: Device, +} + +impl AsRef for Tensor { + fn as_ref(&self) -> &Tensor { + self + } +} + +// Tensors are refcounted so that cloning is cheap when building the op graph. +// Storages are also refcounted independently so that its possible to avoid +// copying the storage for operations that only modify the shape or stride. +#[derive(Clone)] +/// The core struct for manipulating tensors. +/// +/// ```rust +/// use candle_core::{Tensor, DType, Device}; +/// +/// let a = Tensor::arange(0f32, 6f32, &Device::Cpu)?.reshape((2, 3))?; +/// let b = Tensor::arange(0f32, 12f32, &Device::Cpu)?.reshape((3, 4))?; +/// +/// let c = a.matmul(&b)?; +/// # Ok::<(), candle_core::Error>(()) +/// ``` +/// +/// Tensors are reference counted with [`Arc`] so cloning them is cheap. +pub struct Tensor(Arc); + +impl std::ops::Deref for Tensor { + type Target = Tensor_; + + fn deref(&self) -> &Self::Target { + self.0.as_ref() + } +} + +macro_rules! unary_op { + ($fn_name:ident, $op_name:ident) => { + pub fn $fn_name(&self) -> Result { + let shape = self.shape(); + let storage = self + .storage() + .unary_impl::(self.layout())?; + let op = BackpropOp::new1(self, |s| Op::Unary(s, UnaryOp::$op_name)); + Ok(from_storage(storage, shape.clone(), op, false)) + } + }; +} + +macro_rules! binary_op { + ($fn_name:ident, $op_name:ident) => { + pub fn $fn_name(&self, rhs: &Self) -> Result { + let shape = self.same_shape_binary_op(rhs, stringify!($fn_name))?; + let storage = self.storage().binary_impl::( + &*rhs.storage(), + self.layout(), + rhs.layout(), + )?; + let op = BackpropOp::new2(self, rhs, |t1, t2| Op::Binary(t1, t2, BinaryOp::$op_name)); + Ok(from_storage(storage, shape.clone(), op, false)) + } + }; +} + +macro_rules! binary_op_scalar { + ($fn_name:ident, $op_name:ident) => { + pub fn $fn_name(&self, rhs: T) -> Result { + let rhs = match rhs.to_tensor_scalar()? { + crate::scalar::TensorScalar::Tensor(rhs) => rhs, + crate::scalar::TensorScalar::Scalar(rhs) => rhs + .to_dtype(self.dtype())? + .to_device(self.device())? + .broadcast_as(self.shape())?, + }; + let shape = self.same_shape_binary_op(&rhs, stringify!($fn_name))?; + let storage = self.storage().binary_impl::( + &*rhs.storage(), + self.layout(), + rhs.layout(), + )?; + let op = BackpropOp::new2(self, &rhs, |t1, t2| Op::Binary(t1, t2, BinaryOp::$op_name)); + Ok(from_storage(storage, shape.clone(), op, false)) + } + }; +} + +macro_rules! broadcast_binary_op { + ($fn_name:ident, $inner_fn_name:ident) => { + pub fn $fn_name(&self, rhs: &Self) -> Result { + let lhs = self; + let shape = lhs + .shape() + .broadcast_shape_binary_op(rhs.shape(), stringify!($fn_name))?; + let l_broadcast = shape != *lhs.shape(); + let r_broadcast = shape != *rhs.shape(); + match (l_broadcast, r_broadcast) { + (true, true) => lhs + .broadcast_as(&shape)? + .$inner_fn_name(&rhs.broadcast_as(&shape)?), + (false, true) => lhs.$inner_fn_name(&rhs.broadcast_as(&shape)?), + (true, false) => lhs.broadcast_as(&shape)?.$inner_fn_name(rhs), + (false, false) => lhs.$inner_fn_name(rhs), + } + } + }; +} + +/// Creates a fresh tensor structure based on a storage and a shape, this uses contiguous strides. +pub(crate) fn from_storage>( + storage: Storage, + shape: S, + op: BackpropOp, + is_variable: bool, +) -> Tensor { + let dtype = storage.dtype(); + let device = storage.device(); + let tensor_ = Tensor_ { + id: TensorId::new(), + storage: Arc::new(RwLock::new(storage)), + layout: Layout::contiguous(shape), + op, + is_variable, + dtype, + device, + }; + Tensor(Arc::new(tensor_)) +} + +impl Tensor { + pub(crate) fn ones_impl>( + shape: S, + dtype: DType, + device: &Device, + is_variable: bool, + ) -> Result { + let none = BackpropOp::none(); + let shape = shape.into(); + let storage = device.ones(&shape, dtype)?; + Ok(from_storage(storage, shape, none, is_variable)) + } + + /// Creates a new tensor filled with ones. + /// + /// ```rust + /// use candle_core::{Tensor, DType, Device}; + /// let a = Tensor::ones((2, 3), DType::F32, &Device::Cpu)?; + /// let b = Tensor::from_slice(&[1.0f32, 1.0, 1.0, 1.0, 1.0, 1.0], (2, 3), &Device::Cpu)?; + /// // a == b + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn ones>(shape: S, dtype: DType, device: &Device) -> Result { + Self::ones_impl(shape, dtype, device, false) + } + + /// Creates a new tensor filled with ones with same shape, dtype, and device as the other tensor. + /// + /// ```rust + /// use candle_core::{Tensor, DType, Device}; + /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; + /// let b = a.ones_like()?; + /// // b == a + 1 + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn ones_like(&self) -> Result { + Tensor::ones(self.shape(), self.dtype(), self.device()) + } + + // Do not expose outside of the crate, the `is_variable=true` case should only be accessed from + // the variable module. + pub(crate) fn zeros_impl>( + shape: S, + dtype: DType, + device: &Device, + is_variable: bool, + ) -> Result { + let none = BackpropOp::none(); + let shape = shape.into(); + let storage = device.zeros(&shape, dtype)?; + Ok(from_storage(storage, shape, none, is_variable)) + } + + /// Creates a new tensor filled with zeros. + /// + /// ```rust + /// use candle_core::{Tensor, DType, Device}; + /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; + /// let b = Tensor::from_slice(&[0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0], (2, 3), &Device::Cpu)?; + /// // a == b + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn zeros>(shape: S, dtype: DType, device: &Device) -> Result { + Self::zeros_impl(shape, dtype, device, false) + } + + /// Creates a new tensor filled with ones with same shape, dtype, and device as the other + /// tensor. + /// + /// ```rust + /// use candle_core::{Tensor, DType, Device}; + /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; + /// let b = a.zeros_like()?; + /// // b is on CPU f32. + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn zeros_like(&self) -> Result { + Tensor::zeros(self.shape(), self.dtype(), self.device()) + } + + pub(crate) fn rand_impl, T: crate::FloatDType>( + lo: T, + up: T, + s: S, + device: &Device, + is_variable: bool, + ) -> Result { + let s = s.into(); + let storage = device.rand_uniform(lo, up, &s)?; + let none = BackpropOp::none(); + Ok(from_storage(storage, s, none, is_variable)) + } + + pub(crate) fn rand_f64_impl>( + lo: f64, + up: f64, + s: S, + dtype: DType, + device: &Device, + is_variable: bool, + ) -> Result { + let s = s.into(); + let storage = device.rand_uniform_f64(lo, up, &s, dtype)?; + let none = BackpropOp::none(); + Ok(from_storage(storage, s, none, is_variable)) + } + + /// Creates a new tensor initialized with values sampled uniformly between `lo` and `up`. + pub fn rand, T: crate::FloatDType>( + lo: T, + up: T, + s: S, + device: &Device, + ) -> Result { + Self::rand_impl(lo, up, s, device, false) + } + + pub fn rand_like(&self, lo: f64, up: f64) -> Result { + Tensor::rand_f64_impl(lo, up, self.shape(), self.dtype(), self.device(), false) + } + + pub(crate) fn randn_impl, T: crate::FloatDType>( + mean: T, + std: T, + s: S, + device: &Device, + is_variable: bool, + ) -> Result { + let s = s.into(); + let storage = device.rand_normal(mean, std, &s)?; + let none = BackpropOp::none(); + Ok(from_storage(storage, s, none, is_variable)) + } + + pub(crate) fn randn_f64_impl>( + mean: f64, + std: f64, + s: S, + dtype: DType, + device: &Device, + is_variable: bool, + ) -> Result { + let s = s.into(); + let storage = device.rand_normal_f64(mean, std, &s, dtype)?; + let none = BackpropOp::none(); + Ok(from_storage(storage, s, none, is_variable)) + } + + pub fn randn_like(&self, mean: f64, stdev: f64) -> Result { + Tensor::randn_f64_impl( + mean, + stdev, + self.shape(), + self.dtype(), + self.device(), + false, + ) + } + + /// Creates a new tensor initialized with values sampled from a normal distribution with the + /// specified `mean` and standard deviation `std`. + pub fn randn, T: crate::FloatDType>( + mean: T, + std: T, + s: S, + device: &Device, + ) -> Result { + Self::randn_impl(mean, std, s, device, false) + } + + pub(crate) fn new_impl( + array: A, + shape: Shape, + device: &Device, + is_variable: bool, + ) -> Result { + let n: usize = shape.elem_count(); + let buffer_size: usize = array.shape()?.elem_count(); + if buffer_size != n { + return Err(Error::ShapeMismatch { buffer_size, shape }.bt()); + } + let storage = device.storage(array)?; + let none = BackpropOp::none(); + Ok(from_storage(storage, shape, none, is_variable)) + } + + /// Creates a new tensor on the specified device using the content and shape of the input. + pub fn new(array: A, device: &Device) -> Result { + let shape = array.shape()?; + Self::new_impl(array, shape, device, false) + } + + /// Returns a new tensor with all the elements having the same specified value. Note that + /// the tensor is not contiguous so you would have to call `.contiguous()` on it if needed. + pub fn full>( + value: D, + shape: S, + device: &Device, + ) -> Result { + Self::from_vec_impl(vec![value], (), device, false)?.broadcast_as(shape) + } + + /// Creates a new 1D tensor from an iterator. + pub fn from_iter( + iter: impl IntoIterator, + device: &Device, + ) -> Result { + let data = iter.into_iter().collect::>(); + let len = data.len(); + Self::from_vec_impl(data, len, device, false) + } + + /// Creates a new 1D tensor with values from the interval `[start, end)` taken with a common + /// difference `1` from `start`. + pub fn arange(start: D, end: D, device: &Device) -> Result { + Self::arange_step(start, end, D::one(), device) + } + + /// Creates a new 1D tensor with values from the interval `[start, end)` taken with a common + /// difference `step` from `start`. + pub fn arange_step( + start: D, + end: D, + step: D, + device: &Device, + ) -> Result { + if D::is_zero(&step) { + bail!("step cannot be zero") + } + let mut data = vec![]; + let mut current = start; + if step >= D::zero() { + while current < end { + data.push(current); + current += step; + } + } else { + while current > end { + data.push(current); + current += step; + } + } + let len = data.len(); + Self::from_vec_impl(data, len, device, false) + } + + pub(crate) fn from_vec_impl, D: crate::WithDType>( + data: Vec, + shape: S, + device: &Device, + is_variable: bool, + ) -> Result { + let shape = shape.into(); + let buffer_size = data.len(); + if buffer_size != shape.elem_count() { + return Err(Error::ShapeMismatch { buffer_size, shape }.bt()); + } + let storage = device.storage_owned(data)?; + let none = BackpropOp::none(); + Ok(from_storage(storage, shape, none, is_variable)) + } + + /// Creates a new tensor initialized with values from the input vector. The number of elements + /// in this vector must be the same as the number of elements defined by the shape. + /// If the device is cpu, no data copy is made. + pub fn from_vec, D: crate::WithDType>( + data: Vec, + shape: S, + device: &Device, + ) -> Result { + Self::from_vec_impl(data, shape, device, false) + } + + /// Creates a new tensor initialized with values from the input slice. The number of elements + /// in this vector must be the same as the number of elements defined by the shape. + pub fn from_slice, D: crate::WithDType>( + array: &[D], + shape: S, + device: &Device, + ) -> Result { + Self::new_impl(array, shape.into(), device, false) + } + + pub(crate) fn same_shape_binary_op(&self, rhs: &Self, op: &'static str) -> Result<&Shape> { + let lhs = self.shape(); + let rhs = rhs.shape(); + if lhs != rhs { + Err(Error::ShapeMismatchBinaryOp { + lhs: lhs.clone(), + rhs: rhs.clone(), + op, + } + .bt()) + } else { + Ok(lhs) + } + } + + /// Returns true if the computation graph should track this op, that is if it is + /// a variable or if it has some variable as dependencies. + pub fn track_op(&self) -> bool { + self.is_variable || self.op.is_some() + } + + // TODO: Also make an inplace version or a pre-allocated? This could be tricky + // if this can create cycles in the compute graph. + binary_op!(add, Add); + binary_op!(mul, Mul); + binary_op!(sub, Sub); + binary_op!(div, Div); + binary_op_scalar!(maximum, Maximum); + binary_op_scalar!(minimum, Minimum); + broadcast_binary_op!(broadcast_add, add); + broadcast_binary_op!(broadcast_mul, mul); + broadcast_binary_op!(broadcast_sub, sub); + broadcast_binary_op!(broadcast_div, div); + broadcast_binary_op!(broadcast_maximum, maximum); + broadcast_binary_op!(broadcast_minimum, minimum); + broadcast_binary_op!(broadcast_eq, eq); + broadcast_binary_op!(broadcast_ne, ne); + broadcast_binary_op!(broadcast_lt, lt); + broadcast_binary_op!(broadcast_le, le); + broadcast_binary_op!(broadcast_gt, gt); + broadcast_binary_op!(broadcast_ge, ge); + + unary_op!(recip, Recip); + unary_op!(neg, Neg); + unary_op!(exp, Exp); + unary_op!(log, Log); + unary_op!(sin, Sin); + unary_op!(cos, Cos); + unary_op!(tanh, Tanh); + unary_op!(abs, Abs); + unary_op!(sqr, Sqr); + unary_op!(sqrt, Sqrt); + unary_op!(gelu, Gelu); + unary_op!(gelu_erf, GeluErf); + unary_op!(erf, Erf); + unary_op!(relu, Relu); + unary_op!(silu, Silu); + unary_op!(ceil, Ceil); + unary_op!(floor, Floor); + unary_op!(round, Round); + + /// Round element of the input tensor to the nearest integer. + /// + /// If the number of decimals is negative, it specifies the number of positions to the left of + /// the decimal point. + pub fn round_to(&self, decimals: i32) -> Result { + let mult = 10f64.powi(decimals); + (self * mult)?.round()? * (1f64 / mult) + } + + /// Retrieves the single scalar value hold in the tensor. If the tensor contains multiple + /// dimensions, an error is returned instead. + pub fn to_scalar(&self) -> Result { + if self.rank() != 0 { + Err(Error::UnexpectedNumberOfDims { + expected: 0, + got: self.rank(), + shape: self.shape().clone(), + } + .bt())? + } + let from_cpu_storage = |cpu_storage: &crate::CpuStorage| { + let data = S::cpu_storage_as_slice(cpu_storage)?; + Ok::<_, Error>(data[self.layout().start_offset()]) + }; + match &*self.storage() { + Storage::Cpu(cpu_storage) => from_cpu_storage(cpu_storage), + Storage::Cuda(storage) => from_cpu_storage(&storage.to_cpu_storage()?), + Storage::Metal(storage) => from_cpu_storage(&storage.to_cpu_storage()?), + } + } + + /// An alias for `to_scalar`. + pub fn to_vec0(&self) -> Result { + self.to_scalar::() + } + + /// Repeat this tensor along the specified dimensions. + pub fn repeat>(&self, shape: S) -> Result { + // Similar to PyTorch, we extend the number of dimensions of self if needed. + let repeats = shape.into(); + let repeats = repeats.dims(); + let mut inp = if self.rank() < repeats.len() { + let shape = [vec![1; repeats.len() - self.rank()], self.dims().to_vec()].concat(); + self.reshape(shape)? + } else { + self.clone() + }; + for (idx, &repeat) in repeats.iter().enumerate() { + if repeat > 1 { + inp = Tensor::cat(&vec![&inp; repeat], idx)? + } + } + Ok(inp) + } + + /// Creates grids of coordinates specified by the 1D inputs. + /// + /// # Arguments + /// + /// * `args` - A slice of 1D tensors. + /// * `xy_indexing` - Whether to use xy indexing or ij indexing. If xy is selected, the + /// first dimension corresponds to the cardinality of the second input and the second + /// dimension corresponds to the cardinality of the first input. If ij is selected, the + /// dimensions are in the same order as the cardinality of the inputs. + /// + /// # Examples + /// + /// ```rust + /// use candle_core::{Tensor, Device, Shape}; + /// let x = Tensor::new(&[1f32, 2., 3.], &Device::Cpu)?; + /// let y = Tensor::new(&[4f32, 5., 6.], &Device::Cpu)?; + /// + /// let grids_xy = Tensor::meshgrid(&[&x, &y], true)?; + /// + /// assert_eq!(grids_xy.len(), 2); + /// assert_eq!(grids_xy[0].dims(), &[3, 3]); + /// + /// assert_eq!(grids_xy[0].to_vec2::()?, &[[1., 2., 3.], [1., 2., 3.], [1., 2., 3.]]); + /// assert_eq!(grids_xy[1].to_vec2::()?, &[[4., 4., 4.], [5., 5., 5.], [6., 6., 6.]]); + /// + /// let grids_ij = Tensor::meshgrid(&[&x, &y], false)?; + /// + /// assert_eq!(grids_ij[0].to_vec2::()?, &[[1., 1., 1.], [2., 2., 2.], [3., 3., 3.]]); + /// assert_eq!(grids_ij[1].to_vec2::()?, &[[4., 5., 6.], [4., 5., 6.], [4., 5., 6.]]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + /// + /// # Errors + /// + /// * Will return `Err` if `args` contains less than 2 tensors. + /// + pub fn meshgrid>(args: &[A], xy_indexing: bool) -> Result> { + if args.len() <= 1 { + Err(Error::OpRequiresAtLeastTwoTensors { op: "meshgrid" }.bt())? + } + let args: Vec<_> = if xy_indexing { + args.iter().rev().collect() + } else { + args.iter().collect() + }; + + let mut shape = Vec::with_capacity(args.len()); + for arg in args.iter() { + shape.push(arg.as_ref().dims1()?) + } + + let mut grids = Vec::with_capacity(args.len()); + for idx in 0..args.len() { + let mut ones = vec![1usize; args.len()]; + ones[idx] = shape[idx]; + let arg = args[idx].as_ref().reshape(ones)?; + let mut repeats = shape.clone(); + repeats[idx] = 1; + let repeated_tensor = arg.repeat(repeats)?; + grids.push(repeated_tensor); + } + if xy_indexing { + grids.reverse(); + } + Ok(grids) + } + + /// This operation multiplies the input tensor by `mul` then adds `add` and return the result. + /// The input values `mul` and `add` are casted to the appropriate type so some rounding might + /// be performed. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let a = Tensor::new(&[[0f32, 1.], [2., 3.]], &Device::Cpu)?; + /// let a = a.affine(4., -2.)?; + /// assert_eq!(a.to_vec2::()?, &[[-2.0, 2.0], [6.0, 10.0]]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn affine(&self, mul: f64, add: f64) -> Result { + let storage = self.storage().affine(self.layout(), mul, add)?; + let op = BackpropOp::new1(self, |arg| Op::Affine { arg, mul, add }); + Ok(from_storage(storage, self.shape(), op, false)) + } + + /// Applies the Exponential Linear Unit (ELU) function on each element of the input tensor. + pub fn elu(&self, alpha: f64) -> Result { + let storage = self.storage().elu(self.layout(), alpha)?; + let op = BackpropOp::new1(self, |t| Op::Elu(t, alpha)); + Ok(from_storage(storage, self.shape(), op, false)) + } + + /// Raise the tensor to some float exponent `e`. + pub fn powf(&self, e: f64) -> Result { + let storage = self.storage().powf(self.layout(), e)?; + let op = BackpropOp::new1(self, |t| Op::Powf(t, e)); + Ok(from_storage(storage, self.shape(), op, false)) + } + + fn check_dim(&self, dim: usize, op: &'static str) -> Result<()> { + if dim >= self.dims().len() { + Err(Error::DimOutOfRange { + shape: self.shape().clone(), + dim: dim as i32, + op, + } + .bt())? + } else { + Ok(()) + } + } + + /// Split a tensor into the specified number of chunks, this may return less chunks than + /// specified. + pub fn chunk(&self, chunks: usize, dim: D) -> Result> { + let dim = dim.to_index(self.shape(), "chunk")?; + let size = self.dim(dim)?; + if size < chunks { + (0..size).map(|i| self.narrow(dim, i, 1)).collect() + } else { + let chunk_size = size / chunks; + let cnt_additional = size % chunks; + let mut tensors = vec![]; + let mut sum_chunk_size = 0; + for i in 0..chunks { + let chunk_size = if i < cnt_additional { + chunk_size + 1 + } else { + chunk_size + }; + let tensor = self.narrow(dim, sum_chunk_size, chunk_size)?; + tensors.push(tensor); + sum_chunk_size += chunk_size + } + Ok(tensors) + } + } + + /// Returns a new tensor that is a narrowed version of the input, the dimension `dim` + /// ranges from `start` to `start + len`. + pub fn narrow(&self, dim: D, start: usize, len: usize) -> Result { + let dims = self.dims(); + let dim = dim.to_index(self.shape(), "narrow")?; + let err = |msg| { + Err::<(), _>( + Error::NarrowInvalidArgs { + shape: self.shape().clone(), + dim, + start, + len, + msg, + } + .bt(), + ) + }; + if start > dims[dim] { + err("start > dim_len")? + } + if start.saturating_add(len) > dims[dim] { + err("start + len > dim_len")? + } + if start == 0 && dims[dim] == len { + Ok(self.clone()) + } else { + let op = BackpropOp::new1(self, |t| Op::Narrow(t, dim, start, len)); + let layout = self.layout().narrow(dim, start, len)?; + let tensor_ = Tensor_ { + id: TensorId::new(), + storage: self.storage.clone(), + layout, + op, + is_variable: false, + dtype: self.dtype, + device: self.device.clone(), + }; + Ok(Tensor(Arc::new(tensor_))) + } + } + + fn squeeze_dims(self, dims: &[usize]) -> Result { + match dims { + [] => Ok(self), + [i] => self.squeeze(*i), + dims => { + let dims = self + .dims() + .iter() + .enumerate() + .filter_map(|(dim_idx, &v)| { + if dims.contains(&dim_idx) { + None + } else { + Some(v) + } + }) + .collect::>(); + self.reshape(dims) + } + } + } + + fn reduce_impl(&self, dim: D, keepdim: bool, op: ReduceOp) -> Result { + let dim = dim.to_index(self.shape(), op.name())?; + let storage = self.storage().reduce_op(op, self.layout(), &[dim])?; + let mut dims = self.dims().to_vec(); + dims[dim] = 1; + let op = match op { + ReduceOp::Sum | ReduceOp::Min | ReduceOp::Max => { + BackpropOp::new1(self, |arg| Op::Reduce(arg, op, dims.to_vec())) + } + ReduceOp::ArgMin | ReduceOp::ArgMax => BackpropOp::none(), + }; + let res = from_storage(storage, dims, op, false); + if keepdim { + Ok(res) + } else { + res.squeeze_dims(&[dim]) + } + } + + fn sum_impl(&self, sum_dims: D, keepdim: bool) -> Result { + let sum_dims = sum_dims.to_indexes(self.shape(), "sum")?; + let storage = self + .storage() + .reduce_op(ReduceOp::Sum, self.layout(), &sum_dims)?; + let mut dims = self.dims().to_vec(); + for &sum_dim in sum_dims.iter() { + dims[sum_dim] = 1 + } + let op = BackpropOp::new1(self, |a| Op::Reduce(a, ReduceOp::Sum, dims.to_vec())); + let sum = from_storage(storage, dims, op, false); + if keepdim { + Ok(sum) + } else { + sum.squeeze_dims(&sum_dims) + } + } + + /// Roll the tensor input along the given dimension. + /// Elements that are shifted beyond the last position are re-introduced at the first position. + /// + /// ```rust + /// # use candle_core::{Tensor, Device}; + /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; + /// let tensor = tensor.roll(1, 0)?; + /// assert_eq!(tensor.to_vec2::()?, &[[4., 5.], [0., 1.], [2., 3.]]); + /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; + /// let tensor = tensor.roll(-1, 0)?; + /// assert_eq!(tensor.to_vec2::()?, &[[2., 3.], [4., 5.], [0., 1.]]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn roll(&self, shift: i32, dim: D) -> Result + where + D: Dim + Clone, + { + let dim = dim.to_index(self.shape(), "roll")?; + let dim_size = self.dim(dim)?; + let shift = shift.rem_euclid(dim_size as i32) as usize; + if shift == 0 { + Ok(self.clone()) + } else { + let a = self.narrow(dim, 0, dim_size - shift)?; + let b = self.narrow(dim, dim_size - shift, shift)?; + Tensor::cat(&[&b, &a], dim) + } + } + + /// Returns the sum of all elements in the input tensor. The sum is performed over all the + /// input dimensions. + /// + /// The resulting tensor has a shape that is similar to the shape of the input tensor, except + /// that the number of elements for each dimension index in `sum_dims` is 1. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let a = Tensor::new(&[[0f32, 1.], [2., 3.]], &Device::Cpu)?; + /// let s = a.sum_keepdim(0)?; + /// assert_eq!(s.to_vec2::()?, &[[2., 4.]]); + /// let s = a.sum_keepdim(1)?; + /// assert_eq!(s.to_vec2::()?, &[[1.], [5.]]); + /// let s = a.sum_keepdim((0, 1))?; + /// assert_eq!(s.to_vec2::()?, &[[6.]]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn sum_keepdim(&self, sum_dims: D) -> Result { + self.sum_impl(sum_dims, true) + } + + /// Returns the sum of all elements in the input tensor. The sum is performed over all the + /// input dimensions and compared to `sum_keepdim` these dimensions are squeezed rather than + /// kept. + pub fn sum(&self, sum_dims: D) -> Result { + self.sum_impl(sum_dims, false) + } + + /// Returns the mean of all elements in the input tensor. The mean is performed over all the + /// input dimensions. + /// + /// The resulting tensor has a shape that is similar to the shape of the input tensor, except + /// that the number of elements for each dimension index in `mean_dims` is 1. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let a = Tensor::new(&[[0f32, 1.], [2., 3.]], &Device::Cpu)?; + /// let s = a.mean_keepdim(0)?; + /// assert_eq!(s.to_vec2::()?, &[[1., 2.]]); + /// let s = a.mean_keepdim(1)?; + /// assert_eq!(s.to_vec2::()?, &[[0.5], [2.5]]); + /// let s = a.mean_keepdim((0, 1))?; + /// assert_eq!(s.to_vec2::()?, &[[1.5]]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn mean_keepdim(&self, mean_dims: D) -> Result { + let mean_dims = mean_dims.to_indexes(self.shape(), "mean-keepdim")?; + let reduced_dim: usize = mean_dims.iter().map(|i| self.dims()[*i]).product(); + let scale = 1f64 / (reduced_dim as f64); + self.sum_impl(mean_dims, true)? * scale + } + + /// Returns the mean of all elements in the input tensor. The mean is performed over all the + /// input dimensions and compared to `mean_keepdim` these dimensions are squeezed rather than + /// kept. + pub fn mean(&self, mean_dims: D) -> Result { + let mean_dims = mean_dims.to_indexes(self.shape(), "mean")?; + let reduced_dim: usize = mean_dims.iter().map(|i| self.dims()[*i]).product(); + let scale = 1f64 / (reduced_dim as f64); + self.sum_impl(mean_dims, false)? * scale + } + + /// Returns the unbiased variance over the selected dimension. + pub fn var_keepdim(&self, dim: D) -> Result { + let dim = dim.to_index(self.shape(), "var")?; + let mean = self.mean_keepdim(dim)?; + let squares = self.broadcast_sub(&mean)?.sqr()?; + squares.sum_impl(dim, true)? / (self.dim(dim)? - 1) as f64 + } + + /// Returns the unbiased variance over the selected dimension. + pub fn var(&self, dim: D) -> Result { + let dim = dim.to_index(self.shape(), "var")?; + self.var_keepdim(dim)?.squeeze(dim) + } + + /// Gathers the maximum value across the selected dimension. The resulting shape has the same + /// number of dimensions as the original tensor and the select dimension has a single element. + pub fn max_keepdim(&self, dim: D) -> Result { + self.reduce_impl(dim, true, ReduceOp::Max) + } + + /// Similar to `max_keepdim` but the target dimension is squeezed. + pub fn max(&self, dim: D) -> Result { + self.reduce_impl(dim, false, ReduceOp::Max) + } + + /// Gathers the minimum value across the selected dimension. The resulting shape has the same + /// number of dimensions as the original tensor and the select dimension has a single element. + pub fn min_keepdim(&self, dim: D) -> Result { + self.reduce_impl(dim, true, ReduceOp::Min) + } + + /// Similar to `min_keepdim` but the target dimension is squeezed. + pub fn min(&self, dim: D) -> Result { + self.reduce_impl(dim, false, ReduceOp::Min) + } + + pub fn argmax_keepdim(&self, dim: D) -> Result { + self.reduce_impl(dim, true, ReduceOp::ArgMax) + } + + /// Similar to `argmax_keepdim` but the target dimension is squeezed. + pub fn argmax(&self, dim: D) -> Result { + self.reduce_impl(dim, false, ReduceOp::ArgMax) + } + + pub fn argmin_keepdim(&self, dim: D) -> Result { + self.reduce_impl(dim, true, ReduceOp::ArgMin) + } + + /// Similar to `argmin_keepdim` but the target dimension is squeezed. + pub fn argmin(&self, dim: D) -> Result { + self.reduce_impl(dim, false, ReduceOp::ArgMin) + } + + /// Element-wise comparison between two tensors, e.g. equality, greater than, ... The actual + /// comparison operation is specified by the `op` argument. + /// + /// The returned tensor has the same shape as the original tensors and uses `u8` elements. + pub fn cmp(&self, rhs: T, op: CmpOp) -> Result { + let rhs = match rhs.to_tensor_scalar()? { + crate::scalar::TensorScalar::Tensor(rhs) => rhs, + crate::scalar::TensorScalar::Scalar(rhs) => rhs + .to_dtype(self.dtype())? + .to_device(self.device())? + .broadcast_as(self.shape())?, + }; + let shape = self.same_shape_binary_op(&rhs, "cmp")?; + let storage = self + .storage() + .cmp(op, &rhs.storage(), self.layout(), rhs.layout())?; + let op = BackpropOp::new1(self, |a| Op::Cmp(a, op)); + Ok(from_storage(storage, shape.dims(), op, false)) + } + + /// Element-wise equality. + pub fn eq(&self, rhs: T) -> Result { + self.cmp(rhs, CmpOp::Eq) + } + + /// Element-wise non-equality. + pub fn ne(&self, rhs: T) -> Result { + self.cmp(rhs, CmpOp::Ne) + } + + /// Element-wise comparison with lower-than, the returned tensor uses value 1 where `self < + /// rhs` and 0 otherwise. + pub fn lt(&self, rhs: T) -> Result { + self.cmp(rhs, CmpOp::Lt) + } + + /// Element-wise comparison with greater-than, the returned tensor uses value 1 where `self > + /// rhs` and 0 otherwise. + pub fn gt(&self, rhs: T) -> Result { + self.cmp(rhs, CmpOp::Gt) + } + + /// Element-wise comparison with greater-equal, the returned tensor uses value 1 where `self >= + /// rhs` and 0 otherwise. + pub fn ge(&self, rhs: T) -> Result { + self.cmp(rhs, CmpOp::Ge) + } + + /// Element-wise comparison with lower-equal, the returned tensor uses value 1 where `self <= + /// rhs` and 0 otherwise. + pub fn le(&self, rhs: T) -> Result { + self.cmp(rhs, CmpOp::Le) + } + + /// Clamp the tensor values to be between `min` and `max`. + pub fn clamp(&self, min: T1, max: T2) -> Result { + self.maximum(min)?.minimum(max) + } + + /// Interpolate the input tensor to the `target_size` size, taking the value of the nearest element. + /// + /// The input tensor should have three dimensions, `(batch, channels, l)`, the returned + /// tensor also has three dimensions, `(batch, channels, target_size)`. + pub fn interpolate1d(&self, target_size: usize) -> Result { + let (n, c, _l) = self.dims3()?; + let op = BackpropOp::new1(self, |arg| Op::UpsampleNearest1D { arg, target_size }); + let storage = self + .storage() + .upsample_nearest1d(self.layout(), target_size)?; + Ok(from_storage(storage, (n, c, target_size), op, false)) + } + + /// Alias for `interpolate1d`. + pub fn upsample_nearest1d(&self, target_size: usize) -> Result { + self.interpolate1d(target_size) + } + + /// Interpolate the input tensor to the `(target_h, target_w)` size, taking the value of the + /// nearest element. + /// + /// The input tensor should have four dimensions, `(batch, channels, h, w)`, the returned + /// tensor also has four dimensions, `(batch, channels, target_h, target_w)`. + pub fn interpolate2d(&self, target_h: usize, target_w: usize) -> Result { + let (n, c, _h, _w) = self.dims4()?; + let op = BackpropOp::new1(self, |arg| Op::UpsampleNearest2D { + arg, + target_h, + target_w, + }); + let storage = self + .storage() + .upsample_nearest2d(self.layout(), target_h, target_w)?; + Ok(from_storage(storage, (n, c, target_h, target_w), op, false)) + } + + /// Alias for `interpolate2d`. + pub fn upsample_nearest2d(&self, target_h: usize, target_w: usize) -> Result { + self.interpolate2d(target_h, target_w) + } + + /// 2D average pooling over an input tensor with multiple channels. + /// + /// The input tensor should have four dimensions, `(batch, channels, h, w)`, the returned + /// tensor also has four dimensions, `(batch, channels, h', w')`. The pooling is performed on + /// the two last dimensions using a kernel of size `sz`. The returned element is the average + /// value over the kernel window. + pub fn avg_pool2d(&self, sz: T) -> Result { + let sz = sz.to_usize2(); + self.avg_pool2d_with_stride(sz, sz) + } + + /// Same as `avg_pool2d` but with a `stride` that can be set to a value different from the + /// kernel size. + pub fn avg_pool2d_with_stride( + &self, + kernel_size: T, + stride: T, + ) -> Result { + let kernel_size = kernel_size.to_usize2(); + let stride = stride.to_usize2(); + let (n, c, h, w) = self.dims4()?; + if h < kernel_size.0 || w < kernel_size.1 { + bail!("kernel-size {kernel_size:?} is larger than the input size {h},{w}") + } + // https://pytorch.org/docs/stable/generated/torch.nn.AvgPool2d.html#torch.nn.AvgPool2d + let h_out = (h - kernel_size.0) / stride.0 + 1; + let w_out = (w - kernel_size.1) / stride.1 + 1; + let op = BackpropOp::new1(self, |arg| Op::AvgPool2D { + arg, + kernel_size, + stride, + }); + let storage = self + .storage() + .avg_pool2d(self.layout(), kernel_size, stride)?; + Ok(from_storage(storage, (n, c, h_out, w_out), op, false)) + } + + /// 2D max pooling over an input tensor with multiple channels. + /// + /// The input tensor should have four dimensions, `(batch, channels, h, w)`, the returned + /// tensor also has four dimensions, `(batch, channels, h', w')`. The pooling is performed on + /// the two last dimensions using a kernel of size `sz`, the returned element is the maximum + /// value over the kernel window. + pub fn max_pool2d(&self, sz: T) -> Result { + let sz = sz.to_usize2(); + self.max_pool2d_with_stride(sz, sz) + } + + /// Same as `max_pool2d` but with a `stride` that can be set to a value different from the + /// kernel size. + pub fn max_pool2d_with_stride( + &self, + kernel_size: T, + stride: T, + ) -> Result { + let kernel_size = kernel_size.to_usize2(); + let stride = stride.to_usize2(); + let (n, c, h, w) = self.dims4()?; + if h < kernel_size.0 || w < kernel_size.1 { + bail!("kernel-size {kernel_size:?} is larger than the input size {h},{w}") + } + // https://pytorch.org/docs/stable/generated/torch.nn.MaxPool2d.html#torch.nn.MaxPool2d + let h_out = (h - kernel_size.0) / stride.0 + 1; + let w_out = (w - kernel_size.1) / stride.1 + 1; + let op = BackpropOp::new1(self, |arg| Op::MaxPool2D { + arg, + kernel_size, + stride, + }); + let storage = self + .storage() + .max_pool2d(self.layout(), kernel_size, stride)?; + Ok(from_storage(storage, (n, c, h_out, w_out), op, false)) + } + + /// Returns the matrix-multiplication of the input tensor with the other provided tensor. + /// + /// # Arguments + /// + /// * `self` - A tensor with dimensions `b1, b2, ..., bi, m, k`. + /// * `rhs` - A tensor with dimensions `b1, b2, ..., bi, k, n`. + /// + /// The resulting tensor has dimensions `b1, b2, ..., bi, m, n`. + pub fn matmul(&self, rhs: &Self) -> Result { + let a_dims = self.shape().dims(); + let b_dims = rhs.shape().dims(); + + let dim = a_dims.len(); + + if dim < 2 || b_dims.len() != dim { + Err(Error::ShapeMismatchBinaryOp { + lhs: self.shape().clone(), + rhs: rhs.shape().clone(), + op: "matmul", + } + .bt())? + } + + let m = a_dims[dim - 2]; + let k = a_dims[dim - 1]; + let k2 = b_dims[dim - 2]; + let n = b_dims[dim - 1]; + + let c_shape = Shape::from(&a_dims[..dim - 2]).extend(&[m, n]); + let batching: usize = a_dims[..dim - 2].iter().product(); + let batching_b: usize = b_dims[..dim - 2].iter().product(); + if k != k2 || batching != batching_b { + Err(Error::ShapeMismatchBinaryOp { + lhs: self.shape().clone(), + rhs: rhs.shape().clone(), + op: "matmul", + } + .bt())? + } + + let storage = self.storage().matmul( + &rhs.storage(), + (batching, m, n, k), + self.layout(), + rhs.layout(), + )?; + let op = BackpropOp::new2(self, rhs, Op::Matmul); + Ok(from_storage(storage, c_shape, op, false)) + } + + /// Matrix-multiplication with broadcasting support. + /// + /// Compared to `matmul` the two matrixes are allowed to have different dimensions as long as + /// they are compatible for broadcast. E.g. if `self` has shape `(j, 1, n, k)` and `rhs` has + /// shape `(l, k, m)`, the output will have shape `(j, l, n, m)`. + pub fn broadcast_matmul(&self, rhs: &Self) -> Result { + let lhs = self; + let (l_shape, r_shape) = lhs.shape().broadcast_shape_matmul(rhs.shape())?; + let l_broadcast = l_shape != *lhs.shape(); + let r_broadcast = r_shape != *rhs.shape(); + // TODO: Avoid concretising the broadcasted matrixes via contiguous. + match (l_broadcast, r_broadcast) { + (true, true) => lhs + .broadcast_as(&l_shape)? + .contiguous()? + .matmul(&rhs.broadcast_as(&r_shape)?.contiguous()?), + (false, true) => lhs.matmul(&rhs.broadcast_as(&r_shape)?.contiguous()?), + (true, false) => lhs.broadcast_as(&l_shape)?.contiguous()?.matmul(rhs), + (false, false) => lhs.matmul(rhs), + } + } + + /// Returns a tensor with the same shape as the input tensor, the values are taken from + /// `on_true` if the input tensor value is not zero, and `on_false` at the positions where the + /// input tensor is equal to zero. + pub fn where_cond(&self, on_true: &Self, on_false: &Self) -> Result { + let _shap = self.same_shape_binary_op(on_true, "where_cond")?; + let shape = self.same_shape_binary_op(on_false, "where_cond")?; + let storage = self.storage().where_cond( + self.layout(), + &on_true.storage(), + on_true.layout(), + &on_false.storage(), + on_false.layout(), + )?; + let op = BackpropOp::new3(self, on_true, on_false, Op::WhereCond); + Ok(from_storage(storage, shape, op, false)) + } + + /// Returns a tensor with the values from the `self` tensor at the index corresponding to the + /// values hold in the `ids` tensor. + /// + /// # Arguments + /// + /// * `self` - A tensor with dimensions `v, h`. + /// * `ids` - A tensor with dimensions `s` and with integer values between 0 and v (exclusive). + /// + /// The resulting tensor has dimensions `s, h`. `s` is called the sequence length, `v` the + /// vocabulary size, and `h` the hidden size. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let values = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; + /// let ids = Tensor::new(&[2u32, 1u32, 2u32], &Device::Cpu)?; + /// let emb = values.embedding(&ids)?; + /// assert_eq!(emb.to_vec2::()?, &[[4., 5.], [2., 3.], [4., 5.]]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn embedding(&self, ids: &Self) -> Result { + if self.rank() != 2 || ids.rank() != 1 { + Err(Error::ShapeMismatchBinaryOp { + lhs: self.shape().clone(), + rhs: ids.shape().clone(), + op: "embedding", + } + .bt())? + } + self.index_select(ids, 0) + } + + pub fn scatter_add(&self, indexes: &Self, source: &Self, dim: D) -> Result { + let dim = dim.to_index(self.shape(), "scatter-add")?; + let source_dims = source.dims(); + let self_dims = self.dims(); + let mismatch = if source_dims.len() != self_dims.len() { + true + } else { + let mut mismatch = false; + for (i, (&d1, &d2)) in self_dims.iter().zip(source_dims.iter()).enumerate() { + if i != dim && d1 != d2 { + mismatch = true; + break; + } + } + mismatch + }; + if mismatch { + Err(Error::ShapeMismatchBinaryOp { + op: "scatter-add (self, src)", + lhs: self.shape().clone(), + rhs: source.shape().clone(), + } + .bt())? + } + if indexes.dims() != source.dims() { + Err(Error::ShapeMismatchBinaryOp { + op: "scatter-add (indexes, src)", + lhs: indexes.shape().clone(), + rhs: source.shape().clone(), + } + .bt())? + } + let storage = self.storage().scatter_add( + self.layout(), + &indexes.storage(), + indexes.layout(), + &source.storage(), + source.layout(), + dim, + )?; + let op = BackpropOp::new3(self, indexes, source, |t1, t2, t3| { + Op::ScatterAdd(t1, t2, t3, dim) + }); + Ok(from_storage(storage, self.shape(), op, false)) + } + + /// Embeds the values of the `src` tensor into the `self` tensor on the specified dimension. + pub fn slice_scatter(&self, src: &Self, dim: D, start: usize) -> Result { + let dim = dim.to_index(self.shape(), "slice-scatter")?; + if dim == 0 { + self.slice_scatter0(src, start) + } else { + // TODO: Maybe we want to add a more efficient implementation at some point. + self.transpose(0, dim)? + .slice_scatter0(&src.transpose(0, dim)?, start)? + .transpose(0, dim) + } + } + + /// Embeds the values of the `src` tensor into the `self` tensor on the first dimension. + pub fn slice_scatter0(&self, src: &Self, start: usize) -> Result { + if self.dtype() != src.dtype() { + Err(Error::DTypeMismatchBinaryOp { + lhs: self.dtype(), + rhs: src.dtype(), + op: "slice-scatter", + } + .bt())? + } + if self.device().location() != src.device.location() { + Err(Error::DeviceMismatchBinaryOp { + lhs: self.device().location(), + rhs: src.device().location(), + op: "slice-scatter", + } + .bt())? + } + if self.rank() != src.rank() { + Err(Error::UnexpectedNumberOfDims { + expected: self.rank(), + got: src.rank(), + shape: src.shape().clone(), + } + .bt())? + } + let shape_ok = + self.dims() + .iter() + .zip(src.dims().iter()) + .enumerate() + .all(|(dim_idx, (&d1, &d2))| { + if 0 == dim_idx { + d2 + start <= d1 + } else { + d1 == d2 + } + }); + if !shape_ok { + Err(Error::ShapeMismatchBinaryOp { + op: "slice-scatter (self, src)", + lhs: self.shape().clone(), + rhs: src.shape().clone(), + } + .bt())? + } + let mut storage = self.device().zeros(self.shape(), self.dtype())?; + self.storage() + .copy_strided_src(&mut storage, 0, self.layout())?; + let offset = start * src.dims()[1..].iter().product::(); + src.storage() + .copy_strided_src(&mut storage, offset, src.layout())?; + let op = BackpropOp::new2(self, src, |t1, t2| Op::SliceScatter0(t1, t2, start)); + Ok(from_storage(storage, self.shape(), op, false)) + } + + /// Accumulate element from `source` at indexes `indexes` and add them to `self`. + pub fn index_add(&self, indexes: &Self, source: &Self, dim: D) -> Result { + let dim = dim.to_index(self.shape(), "index-add")?; + let source_dims = source.dims(); + let self_dims = self.dims(); + let mismatch = if source_dims.len() != self_dims.len() { + true + } else { + let mut mismatch = false; + for (i, (&d1, &d2)) in self_dims.iter().zip(source_dims.iter()).enumerate() { + if i != dim && d1 != d2 { + mismatch = true; + break; + } + } + mismatch + }; + if mismatch { + Err(Error::ShapeMismatchBinaryOp { + op: "index-add (self, source)", + lhs: self.shape().clone(), + rhs: source.shape().clone(), + } + .bt())? + } + // The number of element in indexes must match the dimension on which the add is + // performed on the source tensor (and the index values from `indexes` are taken from + // the target tensor self) + let indexes_len = indexes.dims1()?; + if source_dims[dim] != indexes_len { + Err(Error::ShapeMismatchBinaryOp { + op: "index-add (ids, source))", + lhs: indexes.shape().clone(), + rhs: source.shape().clone(), + } + .bt())? + } + let storage = self.storage().index_add( + self.layout(), + &indexes.storage(), + indexes.layout(), + &source.storage(), + source.layout(), + dim, + )?; + let op = BackpropOp::new3(self, indexes, source, |t1, t2, t3| { + Op::IndexAdd(t1, t2, t3, dim) + }); + Ok(from_storage(storage, self.shape(), op, false)) + } + + /// Gather values across the target dimension. + /// + /// # Arguments + /// + /// * `self` - The input tensor. + /// * `indexes` - The indices of elements to gather, this should have the same shape as `self` + /// but can have a different number of elements on the target dimension. + /// * `dim` - the target dimension. + /// + /// The resulting tensor has the same shape as `indexes` and use values from `self` indexed on + /// dimension `dim` by the values in `indexes`. + pub fn gather(&self, indexes: &Self, dim: D) -> Result { + let dim = dim.to_index(self.shape(), "gather")?; + let self_dims = self.dims(); + let indexes_dims = indexes.dims(); + let mismatch = if indexes_dims.len() != self_dims.len() { + true + } else { + let mut mismatch = false; + for (i, (&d1, &d2)) in self_dims.iter().zip(indexes_dims.iter()).enumerate() { + if i != dim && d1 != d2 { + mismatch = true; + break; + } + } + mismatch + }; + if mismatch { + Err(Error::ShapeMismatchBinaryOp { + op: "gather", + lhs: self.shape().clone(), + rhs: indexes.shape().clone(), + } + .bt())? + } + let storage = + self.storage() + .gather(self.layout(), &indexes.storage(), indexes.layout(), dim)?; + let op = BackpropOp::new2(self, indexes, |t1, t2| Op::Gather(t1, t2, dim)); + Ok(from_storage(storage, indexes.shape(), op, false)) + } + + /// Select values for the input tensor at the target indexes across the specified dimension. + /// + /// The `indexes` is argument is an int tensor with a single dimension. + /// The output has the same number of dimension as the `self` input. The target dimension of + /// the output has length the length of `indexes` and the values are taken from `self` using + /// the index from `indexes`. Other dimensions have the same number of elements as the input + /// tensor. + pub fn index_select(&self, indexes: &Self, dim: D) -> Result { + let dim = dim.to_index(self.shape(), "index-select")?; + let indexes_len = match indexes.dims() { + [l] => *l, + _ => Err(Error::ShapeMismatchBinaryOp { + lhs: self.shape().clone(), + rhs: indexes.shape().clone(), + op: "index-select", + } + .bt())?, + }; + let storage = self.storage().index_select( + &indexes.storage(), + self.layout(), + indexes.layout(), + dim, + )?; + let mut dims = self.dims().to_vec(); + dims[dim] = indexes_len; + let op = BackpropOp::new2(self, indexes, |t1, t2| Op::IndexSelect(t1, t2, dim)); + Ok(from_storage(storage, dims, op, false)) + } + + /// Returns an iterator over position of the elements in the storage when ranging over the + /// index tuples in lexicographic order. + pub fn strided_index(&self) -> crate::StridedIndex { + self.layout.strided_index() + } + + /// Similar to `strided_index` but returns the position of the start of each contiguous block + /// as well as the length of the contiguous blocks. For a contiguous tensor, the index iterator + /// will only return the start offset and the size would be the number of elements in the + /// tensor. + pub fn strided_blocks(&self) -> crate::StridedBlocks { + self.layout.strided_blocks() + } + + /// Returns the data contained in a 1D tensor as a vector of scalar values. + pub fn to_vec1(&self) -> Result> { + if self.rank() != 1 { + Err(Error::UnexpectedNumberOfDims { + expected: 1, + got: self.rank(), + shape: self.shape().clone(), + } + .bt())? + } + let from_cpu_storage = |cpu_storage: &crate::CpuStorage| { + let data = S::cpu_storage_as_slice(cpu_storage)?; + let data = match self.layout.contiguous_offsets() { + Some((o1, o2)) => data[o1..o2].to_vec(), + None => self.strided_index().map(|i| data[i]).collect(), + }; + Ok::, Error>(data) + }; + match &*self.storage() { + Storage::Cpu(storage) => from_cpu_storage(storage), + Storage::Cuda(storage) => from_cpu_storage(&storage.to_cpu_storage()?), + Storage::Metal(storage) => from_cpu_storage(&storage.to_cpu_storage()?), + } + } + + /// Returns the data contained in a 2D tensor as a vector of vector of scalar values. + pub fn to_vec2(&self) -> Result>> { + let (dim1, dim2) = self.dims2()?; + let from_cpu_storage = |cpu_storage: &crate::CpuStorage| { + let data = S::cpu_storage_as_slice(cpu_storage)?; + let mut rows = vec![]; + match self.layout.contiguous_offsets() { + Some((o1, o2)) => { + let data = &data[o1..o2]; + for idx_row in 0..dim1 { + rows.push(data[idx_row * dim2..(idx_row + 1) * dim2].to_vec()) + } + } + None => { + let mut src_index = self.strided_index(); + for _idx_row in 0..dim1 { + let row = (0..dim2).map(|_| data[src_index.next().unwrap()]).collect(); + rows.push(row) + } + assert!(src_index.next().is_none()); + } + } + Ok(rows) + }; + match &*self.storage() { + Storage::Cpu(storage) => from_cpu_storage(storage), + Storage::Cuda(storage) => from_cpu_storage(&storage.to_cpu_storage()?), + Storage::Metal(storage) => from_cpu_storage(&storage.to_cpu_storage()?), + } + } + + /// Returns the data contained in a 3D tensor. + pub fn to_vec3(&self) -> Result>>> { + let (dim1, dim2, dim3) = self.dims3()?; + let from_cpu_storage = |cpu_storage: &crate::CpuStorage| { + let data = S::cpu_storage_as_slice(cpu_storage)?; + let mut top_rows = vec![]; + match self.layout.contiguous_offsets() { + Some((o1, o2)) => { + let data = &data[o1..o2]; + let dim23 = dim2 * dim3; + for idx1 in 0..dim1 { + let data = &data[idx1 * dim23..(idx1 + 1) * dim23]; + let mut rows = vec![]; + for idx2 in 0..dim2 { + rows.push(data[idx2 * dim3..(idx2 + 1) * dim3].to_vec()) + } + top_rows.push(rows); + } + } + None => { + let mut src_index = self.strided_index(); + for _idx in 0..dim1 { + let mut rows = vec![]; + for _jdx in 0..dim2 { + let row = (0..dim3).map(|_| data[src_index.next().unwrap()]).collect(); + rows.push(row) + } + top_rows.push(rows); + } + assert!(src_index.next().is_none()); + } + } + Ok(top_rows) + }; + match &*self.storage() { + Storage::Cpu(storage) => from_cpu_storage(storage), + Storage::Cuda(storage) => from_cpu_storage(&storage.to_cpu_storage()?), + Storage::Metal(storage) => from_cpu_storage(&storage.to_cpu_storage()?), + } + } + + /// The dtype for the elements stored in the input tensor. + pub fn dtype(&self) -> DType { + self.dtype + } + + /// The device on which the input tensor is located. + pub fn device(&self) -> &Device { + &self.device + } + + /// The tensor shape, i.e. dimension sizes on each axis. + pub fn shape(&self) -> &Shape { + self.layout().shape() + } + + /// The dimension size for this tensor on each axis. + pub fn dims(&self) -> &[usize] { + self.shape().dims() + } + + /// The dimension size for a specified dimension index. + pub fn dim(&self, dim: D) -> Result { + let dim = dim.to_index(self.shape(), "dim")?; + Ok(self.dims()[dim]) + } + + /// The layout of the input tensor, this stores both the shape of the tensor as well as the + /// strides and the start offset to apply to the underlying storage. + pub fn layout(&self) -> &Layout { + &self.layout + } + + pub fn stride(&self) -> &[usize] { + self.layout.stride() + } + + /// The number of dimensions for this tensor, 0 for a scalar tensor, 1 for a 1D tensor, etc. + pub fn rank(&self) -> usize { + self.shape().rank() + } + + /// The number of elements stored in this tensor. + pub fn elem_count(&self) -> usize { + self.shape().elem_count() + } + + /// The unique identifier for this tensor. + pub fn id(&self) -> TensorId { + self.id + } + + /// Whether this tensor is a variable or not. A variable is a tensor for which gradient is + /// tracked and on which backpropagation can be performed. + pub fn is_variable(&self) -> bool { + self.is_variable + } + + pub(crate) fn op(&self) -> &Option { + &self.op + } + + /// Computes the sum of all the elements in this tensor and returns a tensor holding this + /// scalar with zero dimensions. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; + /// let tensor = tensor.sum_all()?; + /// assert_eq!(tensor.to_scalar::()?, 15.); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn sum_all(&self) -> Result { + let dims: Vec<_> = (0..self.rank()).collect(); + self.sum(dims) + } + + pub fn mean_all(&self) -> Result { + self.sum_all()? / self.elem_count() as f64 + } + + fn flatten_( + &self, + start_dim: Option, + end_dim: Option, + ) -> Result { + if self.rank() == 0 { + self.reshape(1) + } else { + let start_dim = match start_dim { + None => 0, + Some(dim) => dim.to_index(self.shape(), "flatten")?, + }; + let end_dim = match end_dim { + None => self.rank() - 1, + Some(dim) => dim.to_index(self.shape(), "flatten")?, + }; + if start_dim < end_dim { + let dims = self.dims(); + let mut dst_dims = dims[..start_dim].to_vec(); + dst_dims.push(dims[start_dim..end_dim + 1].iter().product::()); + if end_dim + 1 < dims.len() { + dst_dims.extend(&dims[end_dim + 1..]); + } + self.reshape(dst_dims) + } else { + Ok(self.clone()) + } + } + } + + /// Flattens the input tensor on the dimension indexes from `start_dim` to `end_dim` (both + /// inclusive). + pub fn flatten(&self, start_dim: D1, end_dim: D2) -> Result { + self.flatten_(Some(start_dim), Some(end_dim)) + } + + /// Flattens the input tensor on the dimension indexes from `0` to `end_dim` (inclusive). + pub fn flatten_to(&self, end_dim: D) -> Result { + self.flatten_(None::, Some(end_dim)) + } + + /// Flattens the input tensor on the dimension indexes from `start_dim` (inclusive) to the last + /// dimension. + pub fn flatten_from(&self, start_dim: D) -> Result { + self.flatten_(Some(start_dim), None::) + } + + /// Flattens the input tensor by reshaping it into a one dimension tensor. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; + /// let tensor = tensor.flatten_all()?; + /// assert_eq!(tensor.to_vec1::()?, &[0., 1., 2., 3., 4., 5.]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn flatten_all(&self) -> Result { + self.flatten_(None::, None::) + } + + /// Returns the sub-tensor fixing the index at `i` on the first dimension. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; + /// let t = tensor.get(0)?; + /// assert_eq!(t.to_vec1::()?, &[0., 1.]); + /// let t = tensor.get(1)?; + /// assert_eq!(t.to_vec1::()?, &[2., 3.]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn get(&self, i: usize) -> Result { + let dims = self.dims(); + if dims.is_empty() { + Ok(self.clone()) + } else { + self.narrow(0, i, 1)?.reshape(&dims[1..]) + } + } + + /// Returns the sub-tensor fixing the index at `index` on the dimension `dim`. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; + /// let t = tensor.get_on_dim(1, 0)?; + /// assert_eq!(t.to_vec1::()?, &[0., 2., 4.]); + /// let t = tensor.get_on_dim(1, 1)?; + /// assert_eq!(t.to_vec1::()?, &[1., 3., 5.]); + /// let t = tensor.get_on_dim(0, 1)?; + /// assert_eq!(t.to_vec1::()?, &[2., 3.]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn get_on_dim(&self, dim: D, index: usize) -> Result { + let dim = dim.to_index(self.shape(), "get_on_dim")?; + self.narrow(dim, index, 1)?.squeeze(dim) + } + + /// Returns a tensor that is a transposed version of the input, the two last dimensions of the + /// input are swapped. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; + /// let tensor = tensor.t()?; + /// assert_eq!(tensor.to_vec2::()?, &[[0.0, 2.0, 4.0], [1.0, 3.0, 5.0]]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn t(&self) -> Result { + let rank = self.rank(); + if rank < 2 { + Err(Error::UnexpectedNumberOfDims { + expected: 2, + got: rank, + shape: self.shape().clone(), + } + .bt())? + } + self.transpose(rank - 2, rank - 1) + } + + /// Returns a tensor that is a transposed version of the input, the given dimensions are + /// swapped. + pub fn transpose(&self, dim1: D1, dim2: D2) -> Result { + let dim1 = dim1.to_index(self.shape(), "transpose")?; + let dim2 = dim2.to_index(self.shape(), "transpose")?; + if dim1 == dim2 { + return Ok(self.clone()); + } + let op = BackpropOp::new1(self, |t| Op::Transpose(t, dim1, dim2)); + let tensor_ = Tensor_ { + id: TensorId::new(), + storage: self.storage.clone(), + layout: self.layout.transpose(dim1, dim2)?, + op, + is_variable: false, + dtype: self.dtype, + device: self.device.clone(), + }; + Ok(Tensor(Arc::new(tensor_))) + } + + /// Returns a tensor with the same data as the input where the dimensions have been permuted. + /// dims must be a permutation, i.e. include each dimension index exactly once. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let tensor = Tensor::arange(0u32, 120u32, &Device::Cpu)?.reshape((2, 3, 4, 5))?; + /// assert_eq!(tensor.dims(), &[2, 3, 4, 5]); + /// let tensor = tensor.permute((2, 3, 1, 0))?; + /// assert_eq!(tensor.dims(), &[4, 5, 3, 2]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn permute(&self, dims: D) -> Result { + let dims = dims.to_indexes(self.shape(), "permute")?; + // O(n^2) permutation check but these arrays are small. + let is_permutation = + dims.len() == self.rank() && (0..dims.len()).all(|i| dims.contains(&i)); + if !is_permutation { + bail!( + "dimension mismatch in permute, tensor {:?}, dims: {:?}", + self.dims(), + dims + ) + } + let op = BackpropOp::new1(self, |t| Op::Permute(t, dims.clone())); + let tensor_ = Tensor_ { + id: TensorId::new(), + storage: self.storage.clone(), + layout: self.layout.permute(&dims)?, + op, + is_variable: false, + dtype: self.dtype, + device: self.device.clone(), + }; + Ok(Tensor(Arc::new(tensor_))) + } + + /// Returns true if the data is stored in a C contiguous (aka row major) way. + pub fn is_contiguous(&self) -> bool { + self.layout.is_contiguous() + } + + /// Returns true if the data is stored in a Fortran contiguous (aka column major) way. + pub fn is_fortran_contiguous(&self) -> bool { + self.layout.is_fortran_contiguous() + } + + /// Compared to clone, this copies the actual storage but may fail because of running out of + /// memory. + pub fn copy(&self) -> Result { + let op = BackpropOp::new1(self, Op::Copy); + let tensor_ = Tensor_ { + id: TensorId::new(), + storage: Arc::new(RwLock::new(self.storage().try_clone(self.layout())?)), + layout: self.layout.clone(), + op, + is_variable: false, + dtype: self.dtype, + device: self.device.clone(), + }; + Ok(Tensor(Arc::new(tensor_))) + } + + /// Returns a new tensor detached from the current graph, gradient are not propagated through + /// this new node. The storage of this tensor is shared with the initial tensor. + /// + /// If the tensor is already detached from the computation graph, the same tensor is returned. + pub fn detach(&self) -> Tensor { + if self.op.is_none() && !self.is_variable { + self.clone() + } else { + let tensor_ = Tensor_ { + id: TensorId::new(), + storage: self.storage.clone(), + layout: self.layout.clone(), + op: BackpropOp::none(), + is_variable: false, + dtype: self.dtype, + device: self.device.clone(), + }; + Tensor(Arc::new(tensor_)) + } + } + + /// If the target device is the same as the tensor device, only a shallow copy is performed. + pub fn to_device(&self, device: &Device) -> Result { + if self.device().same_device(device) { + Ok(self.clone()) + } else { + let storage = match (&*self.storage(), device) { + (Storage::Cpu(storage), Device::Cuda(cuda)) => { + Storage::Cuda(cuda.storage_from_cpu_storage(storage)?) + } + (Storage::Cpu(storage), Device::Metal(metal)) => { + Storage::Metal(metal.storage_from_cpu_storage(storage)?) + } + (Storage::Cuda(storage), Device::Cpu) => Storage::Cpu(storage.to_cpu_storage()?), + (Storage::Metal(storage), Device::Cpu) => Storage::Cpu(storage.to_cpu_storage()?), + (Storage::Cuda(storage), Device::Cuda(cuda)) => { + // TODO: Avoid passing through the cpu storage here, especially if the gpu ids + // are the same. + let cpu_storage = storage.to_cpu_storage()?; + Storage::Cuda(cuda.storage_from_cpu_storage(&cpu_storage)?) + } + (Storage::Cpu(storage), Device::Cpu) => Storage::Cpu(storage.clone()), + _ => { + bail!("not implemented yet") + } + }; + let op = BackpropOp::new1(self, Op::ToDevice); + let tensor_ = Tensor_ { + id: TensorId::new(), + storage: Arc::new(RwLock::new(storage)), + layout: self.layout.clone(), + op, + is_variable: false, + dtype: self.dtype, + device: device.clone(), + }; + Ok(Tensor(Arc::new(tensor_))) + } + } + + /// Returns a new tensor duplicating data from the original tensor. New dimensions are inserted + /// on the left. + pub fn broadcast_left>(&self, left_shape: S) -> Result { + let left_shape = left_shape.into(); + let mut dims = left_shape.into_dims(); + dims.extend(self.dims()); + self.broadcast_as(dims) + } + + /// Broadcast the input tensor to the target shape. This returns an error if the input shape is + /// not compatible with the target shape. + /// + /// If the input shape is `i_1, i_2, ... i_k`, the target shape has to have `k` dimensions or + /// more and shape `j_1, ..., j_l, t_1, t_2, ..., t_k`. The dimensions `j_1` to `j_l` can have + /// any value, the dimension `t_a` must be equal to `i_a` if `i_a` is different from 1. If + /// `i_a` is equal to 1, any value can be used. + pub fn broadcast_as>(&self, shape: S) -> Result { + let tensor_ = Tensor_ { + id: TensorId::new(), + storage: self.storage.clone(), + layout: self.layout.broadcast_as(shape)?, + op: BackpropOp::new1(self, Op::Broadcast), + is_variable: false, + dtype: self.dtype, + device: self.device.clone(), + }; + Ok(Tensor(Arc::new(tensor_))) + } + + /// An alias for broadcast_as. + pub fn expand>(&self, shape: S) -> Result { + self.broadcast_as(shape) + } + + /// Casts the input tensor to the target `dtype`. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let tensor = Tensor::new(3.14159265358979f64, &Device::Cpu)?; + /// assert_eq!(tensor.to_scalar::()?, 3.14159265358979); + /// let tensor = tensor.to_dtype(candle_core::DType::F32)?; + /// assert_eq!(tensor.to_scalar::()?, 3.1415927); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn to_dtype(&self, dtype: DType) -> Result { + if self.dtype() == dtype { + Ok(self.clone()) + } else { + let shape = self.shape(); + let storage = self.storage().to_dtype(self.layout(), dtype)?; + let op = BackpropOp::new1(self, Op::ToDType); + Ok(from_storage(storage, shape.clone(), op, false)) + } + } + + /// Returns a tensor that is in row major order. This is the same as the original tensor if it + /// was already contiguous, otherwise a copy is triggered. + pub fn contiguous(&self) -> Result { + if self.is_contiguous() { + Ok(self.clone()) + } else { + let shape = self.shape(); + let mut storage = self.device().zeros(shape, self.dtype())?; + self.storage() + .copy_strided_src(&mut storage, 0, self.layout())?; + let op = BackpropOp::new1(self, Op::Copy); + Ok(from_storage(storage, shape.clone(), op, false)) + } + } + + /// Create a variable based on the values currently stored in a tensor. The storage is always + /// copied. + pub(crate) fn make_var(&self) -> Result { + let shape = self.shape().clone(); + let mut storage = self.device().zeros(&shape, self.dtype())?; + self.storage() + .copy_strided_src(&mut storage, 0, self.layout())?; + Ok(from_storage(storage, shape, BackpropOp::none(), true)) + } + + /// Reshape returns a tensor with the target shape provided that the number of elements of the + /// original tensor is the same. + /// If the input tensor is contiguous, this is a view on the original data. Otherwise this uses + /// a new storage and copies the data over, the returned tensor is always contiguous. + /// + /// The shape can be specified using a tuple of `usize` and at most one `()` in which case + /// the behavior is the same as when using `-1` in PyTorch: this dimension size is adjusted so + /// as to match the number of elements in the tensor. + /// + /// ```rust + /// # use candle_core::{Tensor, DType, Device, D}; + /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; + /// + /// let c = a.reshape((1, 6))?; + /// assert_eq!(c.shape().dims(), &[1, 6]); + /// + /// let c = a.reshape((3, 2))?; + /// assert_eq!(c.shape().dims(), &[3, 2]); + /// + /// let c = a.reshape((2, (), 1))?; + /// assert_eq!(c.shape().dims(), &[2, 3, 1]); + /// + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn reshape(&self, s: S) -> Result { + let shape = s.into_shape(self.elem_count())?; + if shape.elem_count() != self.elem_count() { + return Err(Error::ShapeMismatchBinaryOp { + lhs: self.shape().clone(), + rhs: shape, + op: "reshape", + } + .bt()); + } + let op = BackpropOp::new1(self, Op::Reshape); + if self.is_contiguous() { + let tensor_ = Tensor_ { + id: TensorId::new(), + storage: self.storage.clone(), + layout: Layout::contiguous_with_offset(shape, self.layout.start_offset()), + op, + is_variable: false, + dtype: self.dtype, + device: self.device.clone(), + }; + Ok(Tensor(Arc::new(tensor_))) + } else { + let mut storage = self.device().zeros(&shape, self.dtype())?; + self.storage() + .copy_strided_src(&mut storage, 0, self.layout())?; + Ok(from_storage(storage, shape, op, false)) + } + } + + /// Creates a new tensor with the specified dimension removed if its size was one. + /// + /// ```rust + /// # use candle_core::{Tensor, DType, Device, D}; + /// let a = Tensor::zeros((2, 3, 1), DType::F32, &Device::Cpu)?; + /// + /// let c = a.squeeze(2)?; + /// assert_eq!(c.shape().dims(), &[2, 3]); + /// + /// let c = a.squeeze(D::Minus1)?; + /// assert_eq!(c.shape().dims(), &[2, 3]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn squeeze(&self, dim: D) -> Result { + // The PyTorch semantics are to return the same tensor if the target dimension + // does not have a size of 1. + let dims = self.dims(); + let dim = dim.to_index(self.shape(), "squeeze")?; + if dims[dim] == 1 { + let mut dims = dims.to_vec(); + dims.remove(dim); + self.reshape(dims) + } else { + Ok(self.clone()) + } + } + + /// Creates a new tensor with a dimension of size one inserted at the specified position. + /// + /// ```rust + /// # use candle_core::{Tensor, DType, Device, D}; + /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; + /// + /// let c = a.unsqueeze(0)?; + /// assert_eq!(c.shape().dims(), &[1, 2, 3]); + /// + /// let c = a.unsqueeze(D::Minus1)?; + /// assert_eq!(c.shape().dims(), &[2, 3, 1]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn unsqueeze(&self, dim: D) -> Result { + let mut dims = self.dims().to_vec(); + let dim = dim.to_index_plus_one(self.shape(), "unsqueeze")?; + // Cannot panic because to_index_plus_one already checks dimensions + dims.insert(dim, 1); + self.reshape(dims) + } + + /// Stacks two or more tensors along a particular dimension. + /// + /// All tensors must have the same rank, and the output has one additional rank + /// + /// ```rust + /// # use candle_core::{Tensor, DType, Device}; + /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; + /// let b = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; + /// + /// let c = Tensor::stack(&[&a, &b], 0)?; + /// assert_eq!(c.shape().dims(), &[2, 2, 3]); + /// + /// let c = Tensor::stack(&[&a, &b], 2)?; + /// assert_eq!(c.shape().dims(), &[2, 3, 2]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn stack, D: Dim>(args: &[A], dim: D) -> Result { + if args.is_empty() { + Err(Error::OpRequiresAtLeastOneTensor { op: "stack" }.bt())? + } + let dim = dim.to_index_plus_one(args[0].as_ref().shape(), "stack")?; + let args = args + .iter() + .map(|t| t.as_ref().unsqueeze(dim)) + .collect::>>()?; + Self::cat(&args, dim) + } + + /// Concatenates two or more tensors along a particular dimension. + /// + /// All tensors must of the same rank, and the output will have + /// the same rank + /// + /// ```rust + /// # use candle_core::{Tensor, DType, Device}; + /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; + /// let b = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; + /// + /// let c = Tensor::cat(&[&a, &b], 0)?; + /// assert_eq!(c.shape().dims(), &[4, 3]); + /// + /// let c = Tensor::cat(&[&a, &b], 1)?; + /// assert_eq!(c.shape().dims(), &[2, 6]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn cat, D: Dim>(args: &[A], dim: D) -> Result { + if args.is_empty() { + Err(Error::OpRequiresAtLeastOneTensor { op: "cat" }.bt())? + } + let arg0 = args[0].as_ref(); + if args.len() == 1 { + return Ok(arg0.clone()); + } + let dim = dim.to_index(arg0.shape(), "cat")?; + for arg in args { + arg.as_ref().check_dim(dim, "cat")?; + } + for (arg_idx, arg) in args.iter().enumerate() { + let arg = arg.as_ref(); + if arg0.rank() != arg.rank() { + Err(Error::UnexpectedNumberOfDims { + expected: arg0.rank(), + got: arg.rank(), + shape: arg.shape().clone(), + } + .bt())? + } + for (dim_idx, (v1, v2)) in arg0 + .shape() + .dims() + .iter() + .zip(arg.shape().dims().iter()) + .enumerate() + { + if dim_idx != dim && v1 != v2 { + Err(Error::ShapeMismatchCat { + dim: dim_idx, + first_shape: arg0.shape().clone(), + n: arg_idx + 1, + nth_shape: arg.shape().clone(), + } + .bt())? + } + } + } + if dim == 0 { + Self::cat0(args) + } else { + // TODO: Avoid these transpositions and have an implementation that works + // for dim != 0... + let args: Vec = args + .iter() + .map(|a| a.as_ref().transpose(0, dim)) + .collect::>>()?; + let cat = Self::cat0(&args)?; + cat.transpose(0, dim) + } + } + + fn cat0>(args: &[A]) -> Result { + if args.is_empty() { + Err(Error::OpRequiresAtLeastOneTensor { op: "cat" }.bt())? + } + let arg0 = args[0].as_ref(); + if args.len() == 1 { + return Ok(arg0.clone()); + } + let rank = arg0.rank(); + let device = arg0.device(); + let dtype = arg0.dtype(); + let first_dims = arg0.shape().dims(); + let mut cat_dims = first_dims.to_vec(); + cat_dims[0] = 0; + let mut offsets = vec![0usize]; + for (arg_idx, arg) in args.iter().enumerate() { + let arg = arg.as_ref(); + if arg.dtype() != dtype { + Err(Error::DTypeMismatchBinaryOp { + lhs: dtype, + rhs: arg.dtype(), + op: "cat", + } + .bt())? + } + if arg.device().location() != device.location() { + Err(Error::DeviceMismatchBinaryOp { + lhs: device.location(), + rhs: arg.device().location(), + op: "cat", + } + .bt())? + } + if rank != arg.rank() { + Err(Error::UnexpectedNumberOfDims { + expected: rank, + got: arg.rank(), + shape: arg.shape().clone(), + } + .bt())? + } + for (dim_idx, (v1, v2)) in arg0 + .shape() + .dims() + .iter() + .zip(arg.shape().dims().iter()) + .enumerate() + { + if dim_idx == 0 { + cat_dims[0] += v2; + } + if dim_idx != 0 && v1 != v2 { + Err(Error::ShapeMismatchCat { + dim: dim_idx, + first_shape: arg0.shape().clone(), + n: arg_idx + 1, + nth_shape: arg.shape().clone(), + } + .bt())? + } + } + let next_offset = offsets.last().unwrap() + arg.elem_count(); + offsets.push(next_offset); + } + let shape = Shape::from(cat_dims); + let op = BackpropOp::new(args, |args| Op::Cat(args, 0)); + let mut storage = device.zeros(&shape, dtype)?; + for (arg, &offset) in args.iter().zip(offsets.iter()) { + let arg = arg.as_ref(); + arg.storage() + .copy_strided_src(&mut storage, offset, arg.layout())?; + } + Ok(from_storage(storage, shape, op, false)) + } + + /// Pad the input tensor using 0s along dimension `dim`. This adds `left` elements before the + /// input tensor values and `right` elements after. + pub fn pad_with_zeros(&self, dim: D, left: usize, right: usize) -> Result { + if left == 0 && right == 0 { + Ok(self.clone()) + } else if left == 0 { + let dim = dim.to_index(self.shape(), "pad_with_zeros")?; + let mut dims = self.dims().to_vec(); + dims[dim] = right; + let right = Tensor::zeros(dims.as_slice(), self.dtype, self.device())?; + Tensor::cat(&[self, &right], dim) + } else if right == 0 { + let dim = dim.to_index(self.shape(), "pad_with_zeros")?; + let mut dims = self.dims().to_vec(); + dims[dim] = left; + let left = Tensor::zeros(dims.as_slice(), self.dtype, self.device())?; + Tensor::cat(&[&left, self], dim) + } else { + let dim = dim.to_index(self.shape(), "pad_with_zeros")?; + let mut dims = self.dims().to_vec(); + dims[dim] = left; + let left = Tensor::zeros(dims.as_slice(), self.dtype, self.device())?; + dims[dim] = right; + let right = Tensor::zeros(dims.as_slice(), self.dtype, self.device())?; + Tensor::cat(&[&left, self, &right], dim) + } + } + + /// Pad the input tensor using same values along dimension `dim`. This adds `left` elements before the + /// input tensor values and `right` elements after. + pub fn pad_with_same(&self, dim: D, left: usize, right: usize) -> Result { + if left == 0 && right == 0 { + Ok(self.clone()) + } else if self.elem_count() == 0 { + bail!("cannot use pad_with_same on an empty tensor") + } else if left == 0 { + let dim = dim.to_index(self.shape(), "pad_with_same")?; + let r = self.narrow(dim, self.dim(dim)? - 1, 1)?; + let mut v = vec![self]; + for _ in 0..right { + v.push(&r) + } + Tensor::cat(&v, dim) + } else if right == 0 { + let dim = dim.to_index(self.shape(), "pad_with_same")?; + let l = self.narrow(dim, 0, 1)?; + let mut v = vec![]; + for _ in 0..left { + v.push(&l) + } + v.push(self); + Tensor::cat(&v, dim) + } else { + let dim = dim.to_index(self.shape(), "pad_with_same")?; + let l = self.narrow(dim, 0, 1)?; + let r = self.narrow(dim, self.dim(dim)? - 1, 1)?; + let mut v = vec![]; + for _ in 0..left { + v.push(&l) + } + v.push(self); + for _ in 0..right { + v.push(&r) + } + Tensor::cat(&v, dim) + } + } + + /// Run the `forward` method of `m` on `self`. + pub fn apply(&self, m: &M) -> Result { + m.forward(self) + } + + /// Run the `forward` method of `m` on `self`. + pub fn apply_t(&self, m: &M, train: bool) -> Result { + m.forward_t(self, train) + } + + pub(crate) fn storage(&self) -> std::sync::RwLockReadGuard<'_, Storage> { + self.storage.read().unwrap() + } + + // If we extend the visibility of this function to be usable outside of this crate, we should + // make it unsafe. + pub(crate) fn storage_mut_and_layout( + &self, + ) -> (std::sync::RwLockWriteGuard<'_, Storage>, &Layout) { + let storage = self.storage.write().unwrap(); + (storage, &self.layout) + } + + /// The storage used by this tensor, together with the layout to use to access it safely. + pub fn storage_and_layout(&self) -> (std::sync::RwLockReadGuard<'_, Storage>, &Layout) { + let storage = self.storage.read().unwrap(); + (storage, &self.layout) + } + + pub(crate) fn same_storage(&self, rhs: &Self) -> bool { + let lhs: &RwLock = self.storage.as_ref(); + let rhs: &RwLock = rhs.storage.as_ref(); + std::ptr::eq(lhs, rhs) + } + + /// Applies a unary custom op without backward support + pub fn apply_op1_no_bwd(&self, c: &C) -> Result { + let (storage, shape) = self.storage().apply_op1(self.layout(), c)?; + Ok(from_storage(storage, shape, BackpropOp::none(), false)) + } + + /// Applies a binary custom op without backward support + pub fn apply_op2_no_bwd(&self, rhs: &Self, c: &C) -> Result { + let (storage, shape) = + self.storage() + .apply_op2(self.layout(), &rhs.storage(), rhs.layout(), c)?; + Ok(from_storage(storage, shape, BackpropOp::none(), false)) + } + + /// Applies a ternary custom op without backward support + pub fn apply_op3_no_bwd(&self, t2: &Self, t3: &Self, c: &C) -> Result { + let (storage, shape) = self.storage().apply_op3( + self.layout(), + &t2.storage(), + t2.layout(), + &t3.storage(), + t3.layout(), + c, + )?; + Ok(from_storage(storage, shape, BackpropOp::none(), false)) + } + + /// Applies a unary custom op. + pub fn apply_op1_arc(&self, c: Arc>) -> Result { + let (storage, shape) = self + .storage() + .apply_op1(self.layout(), c.as_ref().as_ref())?; + let op = BackpropOp::new1(self, |s| Op::CustomOp1(s, c.clone())); + Ok(from_storage(storage, shape, op, false)) + } + + pub fn apply_op1(&self, c: C) -> Result { + self.apply_op1_arc(Arc::new(Box::new(c))) + } + + /// Applies a binary custom op. + pub fn apply_op2_arc( + &self, + rhs: &Self, + c: Arc>, + ) -> Result { + let (storage, shape) = self.storage().apply_op2( + self.layout(), + &rhs.storage(), + rhs.layout(), + c.as_ref().as_ref(), + )?; + let op = BackpropOp::new2(self, rhs, |t1, t2| Op::CustomOp2(t1, t2, c.clone())); + Ok(from_storage(storage, shape, op, false)) + } + + pub fn apply_op2(&self, r: &Self, c: C) -> Result { + self.apply_op2_arc(r, Arc::new(Box::new(c))) + } + + /// Applies a ternary custom op. + pub fn apply_op3_arc( + &self, + t2: &Self, + t3: &Self, + c: Arc>, + ) -> Result { + let (storage, shape) = self.storage().apply_op3( + self.layout(), + &t2.storage(), + t2.layout(), + &t3.storage(), + t3.layout(), + c.as_ref().as_ref(), + )?; + let op = BackpropOp::new3(self, t2, t3, |t1, t2, t3| { + Op::CustomOp3(t1, t2, t3, c.clone()) + }); + Ok(from_storage(storage, shape, op, false)) + } + + pub fn apply_op3( + &self, + t2: &Self, + t3: &Self, + c: C, + ) -> Result { + self.apply_op3_arc(t2, t3, Arc::new(Box::new(c))) + } + + /// Normalize a 'relative' axis value: positive values are kept, negative + /// values means counting the dimensions from the back. + pub fn normalize_axis(&self, axis: i64) -> Result { + let rank = self.rank() as i64; + if rank <= axis { + bail!("axis {axis} is too large, tensor rank {rank}") + } else if 0 <= axis { + Ok(axis as usize) + } else { + let naxis = rank + axis; + if naxis < 0 { + bail!("axis {axis} is too small, tensor rank {rank}") + } + Ok(naxis as usize) + } + } + + /// Returns a lower triangular matrix of ones of size n by n. + pub fn tril2(n: usize, dtype: DType, device: &Device) -> Result { + let t = Tensor::arange(0u32, n as u32, device)?; + let t1 = t.reshape((1, n))?.broadcast_as((n, n))?; + let t2 = t.reshape((n, 1))?.broadcast_as((n, n))?; + t1.le(&t2)?.to_dtype(dtype) + } + + /// Returns an upper triangular matrix of ones of size n by n. + pub fn triu2(n: usize, dtype: DType, device: &Device) -> Result { + let t = Tensor::arange(0u32, n as u32, device)?; + let t1 = t.reshape((1, n))?.broadcast_as((n, n))?; + let t2 = t.reshape((n, 1))?.broadcast_as((n, n))?; + t1.ge(&t2)?.to_dtype(dtype) + } + + /// Returns a matrix with a diagonal of ones of size n by n. + pub fn eye(n: usize, dtype: DType, device: &Device) -> Result { + let t = Tensor::arange(0u32, n as u32, device)?; + let t1 = t.reshape((1, n))?.broadcast_as((n, n))?; + let t2 = t.reshape((n, 1))?.broadcast_as((n, n))?; + t1.eq(&t2)?.to_dtype(dtype) + } + + /// Returns the cumulative sum of elements of the input tensor summed over the specified + /// dimension. + /// + /// This operation is most efficient when dim is the last dimension of the tensor. + pub fn cumsum(&self, dim: D) -> Result { + let dim = dim.to_index(self.shape(), "cumsum")?; + let rank = self.rank(); + if rank == 0 { + return Ok(self.clone()); + } + let n_axis = self.dim(dim)?; + let triu = Tensor::triu2(n_axis, self.dtype(), self.device())?; + if rank == 1 { + self.unsqueeze(0)?.matmul(&triu)?.squeeze(0) + } else { + let last = rank - 1; + let t = self.transpose(dim, last)?; + let t = t.broadcast_matmul(&triu)?; + t.transpose(dim, last) + } + } + + /// Returns a copy of `self` where the values within `ranges` have been replaced with the + /// content of `src`. + pub fn slice_assign>( + &self, + ranges: &[D], + src: &Tensor, + ) -> Result { + let src_dims = src.dims(); + let self_dims = self.dims(); + if self_dims.len() != src_dims.len() { + bail!( + "slice-assign requires input with the same rank {} <> {}", + self_dims.len(), + src_dims.len() + ) + } + if self_dims.len() != ranges.len() { + bail!( + "slice-assign requires input with the same rank as there are ranges {} <> {}", + self_dims.len(), + ranges.len() + ) + } + let mut src = src.clone(); + let mut mask = Self::ones(src.shape(), DType::U8, src.device())?; + for (i, range) in ranges.iter().enumerate() { + let start_included = match range.start_bound() { + std::ops::Bound::Unbounded => 0, + std::ops::Bound::Included(v) => *v, + std::ops::Bound::Excluded(v) => *v + 1, + }; + let end_excluded = match range.end_bound() { + std::ops::Bound::Unbounded => self_dims[i], + std::ops::Bound::Included(v) => *v + 1, + std::ops::Bound::Excluded(v) => *v, + }; + if end_excluded <= start_included { + bail!("slice-assign: empty range for dim {i}, {start_included} {end_excluded}") + } + if self_dims[i] < end_excluded { + bail!( + "slice-assign: upper bound is out of range for dim {i}, {end_excluded} {}", + self_dims[i] + ) + } + if end_excluded - start_included != src_dims[i] { + bail!( + "slice-assign: the range for dim {i} ({start_included}..{end_excluded}) does not match the size of src {}", src_dims[i] + ) + } + src = src.pad_with_zeros(i, start_included, self_dims[i] - end_excluded)?; + mask = mask.pad_with_zeros(i, start_included, self_dims[i] - end_excluded)? + } + mask.where_cond(/* on_true= */ &src, /* on_false= */ self) + } + + /// Returns log(sum(exp(tensor), dim)). + pub fn log_sum_exp(&self, sum_dims: D) -> Result { + let exp = self.exp()?; + let sum = exp.sum(sum_dims)?; + sum.log() + } + + /// Pointwise pow operation. + pub fn pow(&self, rhs: &Tensor) -> Result { + rhs.mul(&self.log()?)?.exp() + } + + /// Broadcasting version of `pow`. + pub fn broadcast_pow(&self, rhs: &Tensor) -> Result { + rhs.broadcast_mul(&self.log()?)?.exp() + } +} + +macro_rules! bin_trait { + ($trait:ident, $fn1:ident, $mul:expr, $add:expr) => { + impl> std::ops::$trait for Tensor { + type Output = Result; + + fn $fn1(self, rhs: B) -> Self::Output { + Tensor::$fn1(&self, rhs.borrow()) + } + } + + impl> std::ops::$trait for &Tensor { + type Output = Result; + + fn $fn1(self, rhs: B) -> Self::Output { + Tensor::$fn1(&self, rhs.borrow()) + } + } + + impl> std::ops::$trait for Result { + type Output = Result; + + fn $fn1(self, rhs: Tensor) -> Self::Output { + Tensor::$fn1(self?.borrow(), &rhs) + } + } + + impl> std::ops::$trait<&Tensor> for Result { + type Output = Result; + + fn $fn1(self, rhs: &Tensor) -> Self::Output { + Tensor::$fn1(self?.borrow(), rhs) + } + } + + impl> std::ops::$trait> for Tensor { + type Output = Result; + + fn $fn1(self, rhs: Result) -> Self::Output { + Tensor::$fn1(&self, rhs?.borrow()) + } + } + + impl> std::ops::$trait> for &Tensor { + type Output = Result; + + fn $fn1(self, rhs: Result) -> Self::Output { + Tensor::$fn1(&self, rhs?.borrow()) + } + } + + impl std::ops::$trait for Tensor { + type Output = Result; + + fn $fn1(self, rhs: f64) -> Self::Output { + self.affine($mul(rhs), $add(rhs)) + } + } + + impl std::ops::$trait for &Tensor { + type Output = Result; + + fn $fn1(self, rhs: f64) -> Self::Output { + self.affine($mul(rhs), $add(rhs)) + } + } + }; +} + +bin_trait!(Add, add, |_| 1., |v| v); +bin_trait!(Sub, sub, |_| 1., |v: f64| -v); +bin_trait!(Mul, mul, |v| v, |_| 0.); +bin_trait!(Div, div, |v| 1. / v, |_| 0.); + +impl std::ops::Add for f64 { + type Output = Result; + + fn add(self, rhs: Tensor) -> Self::Output { + rhs + self + } +} + +impl std::ops::Add<&Tensor> for f64 { + type Output = Result; + + fn add(self, rhs: &Tensor) -> Self::Output { + rhs + self + } +} + +impl std::ops::Mul for f64 { + type Output = Result; + + fn mul(self, rhs: Tensor) -> Self::Output { + rhs * self + } +} + +impl std::ops::Mul<&Tensor> for f64 { + type Output = Result; + + fn mul(self, rhs: &Tensor) -> Self::Output { + rhs * self + } +} + +impl std::ops::Sub for f64 { + type Output = Result; + + fn sub(self, rhs: Tensor) -> Self::Output { + rhs.affine(-1., self) + } +} + +impl std::ops::Sub<&Tensor> for f64 { + type Output = Result; + + fn sub(self, rhs: &Tensor) -> Self::Output { + rhs.affine(-1., self) + } +} + +impl std::ops::Div for f64 { + type Output = Result; + + #[allow(clippy::suspicious_arithmetic_impl)] + fn div(self, rhs: Tensor) -> Self::Output { + rhs.recip()? * self + } +} + +impl std::ops::Div<&Tensor> for f64 { + type Output = Result; + + #[allow(clippy::suspicious_arithmetic_impl)] + fn div(self, rhs: &Tensor) -> Self::Output { + rhs.recip()? * self + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/test_utils.rs b/v2/dependency-patches/candle-core-0.4.1/src/test_utils.rs new file mode 100644 index 0000000000..3b8fb904c0 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/test_utils.rs @@ -0,0 +1,62 @@ +use crate::{Result, Tensor}; + +#[macro_export] +macro_rules! test_device { + // TODO: Switch to generating the two last arguments automatically once concat_idents is + // stable. https://github.com/rust-lang/rust/issues/29599 + ($fn_name: ident, $test_cpu: ident, $test_cuda: ident, $test_metal: ident) => { + #[test] + fn $test_cpu() -> Result<()> { + $fn_name(&Device::Cpu) + } + + #[cfg(feature = "cuda")] + #[test] + fn $test_cuda() -> Result<()> { + $fn_name(&Device::new_cuda(0)?) + } + + #[cfg(feature = "metal")] + #[test] + fn $test_metal() -> Result<()> { + $fn_name(&Device::new_metal(0)?) + } + }; +} + +pub fn to_vec0_round(t: &Tensor, digits: i32) -> Result { + let b = 10f32.powi(digits); + let t = t.to_vec0::()?; + Ok(f32::round(t * b) / b) +} + +pub fn to_vec1_round(t: &Tensor, digits: i32) -> Result> { + let b = 10f32.powi(digits); + let t = t.to_vec1::()?; + let t = t.iter().map(|t| f32::round(t * b) / b).collect(); + Ok(t) +} + +pub fn to_vec2_round(t: &Tensor, digits: i32) -> Result>> { + let b = 10f32.powi(digits); + let t = t.to_vec2::()?; + let t = t + .iter() + .map(|t| t.iter().map(|t| f32::round(t * b) / b).collect()) + .collect(); + Ok(t) +} + +pub fn to_vec3_round(t: &Tensor, digits: i32) -> Result>>> { + let b = 10f32.powi(digits); + let t = t.to_vec3::()?; + let t = t + .iter() + .map(|t| { + t.iter() + .map(|t| t.iter().map(|t| f32::round(t * b) / b).collect()) + .collect() + }) + .collect(); + Ok(t) +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/utils.rs b/v2/dependency-patches/candle-core-0.4.1/src/utils.rs new file mode 100644 index 0000000000..78c45a9a9d --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/utils.rs @@ -0,0 +1,44 @@ +use std::str::FromStr; + +pub fn get_num_threads() -> usize { + // Respond to the same environment variable as rayon. + match std::env::var("RAYON_NUM_THREADS") + .ok() + .and_then(|s| usize::from_str(&s).ok()) + { + Some(x) if x > 0 => x, + Some(_) | None => num_cpus::get(), + } +} + +pub fn has_accelerate() -> bool { + cfg!(feature = "accelerate") +} + +pub fn has_mkl() -> bool { + cfg!(feature = "mkl") +} + +pub fn cuda_is_available() -> bool { + cfg!(feature = "cuda") +} + +pub fn metal_is_available() -> bool { + cfg!(feature = "metal") +} + +pub fn with_avx() -> bool { + cfg!(target_feature = "avx") +} + +pub fn with_neon() -> bool { + cfg!(target_feature = "neon") +} + +pub fn with_simd128() -> bool { + cfg!(target_feature = "simd128") +} + +pub fn with_f16c() -> bool { + cfg!(target_feature = "f16c") +} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/variable.rs b/v2/dependency-patches/candle-core-0.4.1/src/variable.rs new file mode 100644 index 0000000000..bdf8da4aed --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/src/variable.rs @@ -0,0 +1,147 @@ +// Variables are wrappers around tensors that can be modified, they are typically used for holding +// weights and being modified by gradient descent. +// We do not expose a public way to create variables as this would break the invariant that the +// tensor within a variable is actually with `is_variable` set to `true`. +use crate::{DType, Device, Error, Result, Shape, Tensor}; + +/// A variable is a wrapper around a tensor, however variables can have their content modified +/// whereas tensors are immutable. +#[derive(Clone, Debug)] +pub struct Var(Tensor); + +impl std::fmt::Display for Var { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(&self.0, f) + } +} + +impl std::ops::Deref for Var { + type Target = Tensor; + + fn deref(&self) -> &Self::Target { + self.0.as_ref() + } +} + +impl Var { + pub fn zeros>(shape: S, dtype: DType, device: &Device) -> Result { + let inner = Tensor::zeros_impl(shape, dtype, device, true)?; + Ok(Self(inner)) + } + + pub fn ones>(shape: S, dtype: DType, device: &Device) -> Result { + let inner = Tensor::ones_impl(shape, dtype, device, true)?; + Ok(Self(inner)) + } + + pub fn from_tensor(t: &Tensor) -> Result { + let inner = t.make_var()?; + Ok(Self(inner)) + } + + pub fn rand_f64>( + lo: f64, + up: f64, + s: S, + dtype: DType, + device: &Device, + ) -> Result { + let inner = Tensor::rand_f64_impl(lo, up, s, dtype, device, true)?; + Ok(Self(inner)) + } + + pub fn randn_f64>( + mean: f64, + std: f64, + s: S, + dtype: DType, + device: &Device, + ) -> Result { + let inner = Tensor::randn_f64_impl(mean, std, s, dtype, device, true)?; + Ok(Self(inner)) + } + + pub fn rand, T: crate::FloatDType>( + lo: T, + up: T, + s: S, + device: &Device, + ) -> Result { + let inner = Tensor::rand_impl(lo, up, s, device, true)?; + Ok(Self(inner)) + } + + pub fn randn, T: crate::FloatDType>( + mean: T, + std: T, + s: S, + device: &Device, + ) -> Result { + let inner = Tensor::randn_impl(mean, std, s, device, true)?; + Ok(Self(inner)) + } + + /// Creates a new tensor on the specified device using the content and shape of the input. + /// This is similar to `new` but the resulting tensor is a variable. + pub fn new(array: A, device: &Device) -> Result { + let shape = array.shape()?; + let inner = Tensor::new_impl(array, shape, device, true)?; + Ok(Self(inner)) + } + + pub fn from_vec, D: crate::WithDType>( + data: Vec, + shape: S, + device: &Device, + ) -> Result { + let inner = Tensor::from_vec_impl(data, shape, device, true)?; + Ok(Self(inner)) + } + + pub fn from_slice, D: crate::WithDType>( + array: &[D], + shape: S, + device: &Device, + ) -> Result { + let inner = Tensor::new_impl(array, shape.into(), device, true)?; + Ok(Self(inner)) + } + + pub fn as_detached_tensor(&self) -> Tensor { + self.0.detach() + } + + pub fn as_tensor(&self) -> &Tensor { + &self.0 + } + + /// Consumes this `Var` and return the underlying tensor. + pub fn into_inner(self) -> Tensor { + self.0 + } + + /// Sets the content of the inner tensor, this does not require a mutable reference as inner + /// mutability is used. + pub fn set(&self, src: &Tensor) -> Result<()> { + if self.same_storage(src) { + let msg = "cannot set a variable to a tensor that is derived from its value"; + Err(Error::CannotSetVar { msg }.bt())? + } + let (mut dst, layout) = self.storage_mut_and_layout(); + if !layout.is_contiguous() { + let msg = "cannot set a non-contiguous variable"; + Err(Error::CannotSetVar { msg }.bt())? + } + let (src, src_l) = src.storage_and_layout(); + if layout.shape() != src_l.shape() { + Err(Error::ShapeMismatchBinaryOp { + lhs: layout.shape().clone(), + rhs: src_l.shape().clone(), + op: "set", + } + .bt())? + } + src.copy_strided_src(&mut dst, layout.start_offset(), src_l)?; + Ok(()) + } +} diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/conv_tests.rs b/v2/dependency-patches/candle-core-0.4.1/tests/conv_tests.rs new file mode 100644 index 0000000000..b967515dd9 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/tests/conv_tests.rs @@ -0,0 +1,609 @@ +use anyhow::Result; +use candle_core::{test_device, test_utils, Device, IndexOp, Tensor}; + +/* This test is based on the following script. +import torch +torch.manual_seed(4242) + +t = torch.randn((1, 4, 5)) +w = torch.randn((2, 4, 3)) +print(t.flatten()) +print(w.flatten()) +res = torch.nn.functional.conv1d(t, w) +print(res.flatten()) +res = torch.nn.functional.conv1d(t, w, padding=1) +print(res.flatten()) + +w_t = w.transpose(0, 1) +res = torch.nn.functional.conv_transpose1d(t, w_t) +print(res.shape) +print(res) +res = torch.nn.functional.conv_transpose1d(t, w_t, groups=2) +print(res.shape) +print(res) +*/ +fn conv1d(dev: &Device) -> Result<()> { + let t = Tensor::new( + &[ + 0.4056f32, -0.8689, -0.0773, -1.5630, 1.2279, -0.9287, -1.7030, 0.1370, 0.1866, 0.4145, + 1.8025, -0.1536, 2.2013, -0.6836, 0.2477, 1.3127, -0.6957, 0.3278, -1.0124, 0.5599, + ], + dev, + )? + .reshape((1, 4, 5))?; + let w = Tensor::new( + &[ + -0.8404f32, -0.3490, 0.0130, 1.3123, 0.1763, -1.9249, 1.4270, 0.9421, 0.8670, -0.7181, + -1.1111, 0.8869, -1.2429, 1.8357, 1.6052, -1.3844, 0.3951, -1.2036, 0.6686, 1.6261, + -0.6451, -0.0840, -1.4247, 0.5512, + ], + dev, + )? + .reshape((2, 4, 3))?; + let res = t.conv1d(&w, 0, 1, 1, 1)?; + assert_eq!(res.dims(), [1, 2, 3]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [2.6357, -1.3336, 4.1393, -1.1784, 3.5675, 0.5069] + ); + let res = t.conv1d(&w, /*padding*/ 1, 1, 1, 1)?; + assert_eq!(res.dims(), [1, 2, 5]); + // Same as pytorch default padding: use zeros. + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [2.4509, 2.6357, -1.3336, 4.1393, 0.5657, 1.8091, -1.1784, 3.5675, 0.5069, 3.3352] + ); + let res = t.conv_transpose1d(&w.transpose(0, 1)?, 0, 0, 1, 1, 1)?; + assert_eq!(res.dims(), [1, 2, 7]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [ + 0.0699, -1.2899, 8.3018, 5.5873, 2.4572, -2.6143, -0.0706, 1.8765, 4.8318, 1.1538, + 4.7076, -5.9745, -0.8276, 1.621 + ], + ); + let res = t.conv_transpose1d(&w.transpose(0, 1)?, 0, 0, 1, 1, 2)?; + assert_eq!(res.dims(), [1, 4, 7]); + assert_eq!( + test_utils::to_vec2_round(&res.squeeze(0)?, 4)?, + [ + [-1.5596, -1.8099, 2.0407, 4.8764, -0.1743, -0.735, -0.7819], + [0.7816, 3.8152, -0.5926, 2.2515, -5.1844, -0.3157, 1.4721], + [1.6295, 0.52, 6.2611, 0.7109, 2.6315, -1.8793, 0.7113], + [1.0949, 1.0166, 1.7464, 2.4561, -0.79, -0.5119, 0.1488] + ] + ); + Ok(()) +} + +fn conv1d_small(dev: &Device) -> Result<()> { + let t = Tensor::new(&[0.4056f32, -0.8689, -0.0773, -1.5630], dev)?.reshape((1, 1, 4))?; + let w = Tensor::new(&[1f32, 0., 0.], dev)?.reshape((1, 1, 3))?; + let res = t.conv1d(&w, 0, 1, 1, 1)?; + assert_eq!(res.dims(), [1, 1, 2]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [0.4056, -0.8689] + ); + let res = t.conv1d(&w, /*padding*/ 1, 1, 1, 1)?; + assert_eq!(res.dims(), [1, 1, 4]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [0.0, 0.4056, -0.8689, -0.0773], + ); + Ok(()) +} + +/* This test is based on the following script. +import torch +torch.manual_seed(4242) + +t = torch.randn((1, 4, 5, 5)) +w = torch.randn((2, 4, 3, 3)) +print(t.flatten()) +print(w.flatten()) +res = torch.nn.functional.conv2d(t, w) +print(res.flatten()) + +w_t = w.transpose(0, 1) +res = torch.nn.functional.conv_transpose2d(t, w_t) +print(res.shape) +print(res) + +res = torch.nn.functional.conv2d(t, w, dilation=2) +print(res.shape) +print(res[0]) + +res = torch.nn.functional.conv_transpose2d(t, w_t, dilation=2) +print(res.shape) +print(res) +*/ +fn conv2d(dev: &Device) -> Result<()> { + let t = Tensor::new( + &[ + 0.4056f32, -0.8689, -0.0773, -1.5630, -2.8012, -1.5059, 0.3972, 1.0852, 0.4997, 3.0616, + 1.6541, 0.0964, -0.8338, -1.6523, -0.8323, -0.1699, 0.0823, 0.3526, 0.6843, 0.2395, + 1.2279, -0.9287, -1.7030, 0.1370, 0.6047, 0.3770, -0.6266, 0.3529, 2.2013, -0.6836, + 0.2477, 1.3127, -0.2260, 0.2622, -1.2974, -0.8140, -0.8404, -0.3490, 0.0130, 1.3123, + 1.7569, -0.3956, -1.8255, 0.1727, -0.3538, 2.6941, 1.0529, 0.4219, -0.2071, 1.1586, + 0.4717, 0.3865, -0.5690, -0.5010, -0.1310, 0.7796, 0.6630, -0.2021, 2.6090, 0.2049, + 0.6466, -0.5042, -0.0603, -1.6538, -1.2429, 1.8357, 1.6052, -1.3844, 0.3323, -1.3712, + 0.9634, -0.4799, -0.6451, -0.0840, -1.4247, 0.5512, -0.1747, -0.5509, -0.3742, 0.3790, + -0.4431, -0.4720, -0.7890, 0.2620, 0.7875, 0.5377, -0.6779, -0.8088, 1.9098, 1.2006, + -0.8000, -0.4983, 1.5480, 0.8265, -0.1025, 0.5138, 0.5748, 0.3821, -0.4607, 0.0085, + ], + dev, + )?; + let w = Tensor::new( + &[ + -0.9325f32, 0.6451, -0.8537, 0.2378, 0.8764, -0.1832, 0.2987, -0.6488, -0.2273, + -2.4184, -0.1192, -0.4821, -0.5079, -0.5766, -2.4729, 1.6734, 0.4558, 0.2851, 1.1514, + -0.9013, 1.0662, -0.1817, -0.0259, 0.1709, 0.5367, 0.7513, 0.8086, -2.2586, -0.5027, + 0.9141, -1.3086, -1.3343, -1.5669, -0.1657, 0.7958, 0.1432, 0.3896, -0.4501, 0.1667, + 0.0714, -0.0952, 1.2970, -0.1674, -0.3178, 1.0677, 0.3060, 0.7080, 0.1914, 1.1679, + -0.3602, 1.9265, -1.8626, -0.5112, -0.0982, 0.2621, 0.6565, 0.5908, 1.0089, -0.1646, + 1.8032, -0.6286, 0.2016, -0.3370, 1.2555, 0.8009, -0.6488, -0.4652, -1.5685, 1.5860, + 0.5583, 0.4623, 0.6026, + ], + dev, + )?; + let t = t.reshape((1, 4, 5, 5))?; + let w = w.reshape((2, 4, 3, 3))?; + let res = t.conv2d(&w, 0, 1, 1, 1)?; + assert_eq!(res.dims(), [1, 2, 3, 3]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [ + -4.2812, 2.0923, 5.2187, 7.5184, 0.752, -14.9426, 10.0087, 4.391, 0.2918, 1.6715, + 10.389, 3.6023, -4.2808, 0.2672, 5.3646, -5.2023, -2.1955, -9.4075 + ] + ); + let res = t.conv_transpose2d(&w.transpose(0, 1)?, 0, 0, 1, 1)?; + assert_eq!(res.dims(), [1, 2, 7, 7]); + assert_eq!( + test_utils::to_vec3_round(&res.i(0)?, 4)?, + [ + [ + [-1.9918, 2.6797, -0.4599, -1.6037, 1.4131, -2.4012, 2.9277], + [1.8016, -3.5361, 1.0757, 3.5395, -8.2168, -3.2023, 0.5375], + [0.8243, 1.8675, 7.8929, -4.0746, -6.4415, 5.1139, 1.6889], + [0.2722, 8.9679, 3.3477, 1.8514, -4.2896, -3.8228, -7.5632], + [-8.5412, -5.8142, -7.1587, -1.6095, 0.4651, 0.2748, -2.0985], + [2.0833, -0.6482, -12.1692, -4.1284, -2.9765, -0.0656, -4.5114], + [5.307, 2.6957, 2.3087, 1.0478, 0.7808, -1.1519, -0.9579] + ], + [ + [1.089, 0.1872, -0.6408, -0.9897, 0.8503, 1.1019, -0.9211], + [-0.1741, -0.2915, 4.2472, 1.9417, 1.65, 0.6303, -4.7131], + [1.6555, 2.4026, -2.9293, 2.9953, 0.5328, 3.5873, -0.9621], + [-1.4289, -3.2787, 4.1747, -6.0341, -4.6341, -5.7945, 4.142], + [7.5973, 6.4431, 5.9872, 2.1639, -8.6566, 3.3143, -3.4059], + [-0.8775, -3.048, 11.6543, 0.6442, 2.3218, -0.4765, 1.1516], + [-5.5423, -2.5188, 1.0754, -0.0563, -2.9386, -1.1504, 1.0171] + ] + ] + ); + // Dilations. + let res = t.conv2d(&w, 0, 1, 2, 1)?; + assert_eq!(res.dims(), [1, 2, 1, 1]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [2.45, -2.3504], + ); + + // Transpose and dilations. + let res = t.conv_transpose2d(&w.transpose(0, 1)?, 0, 0, 1, 2)?; + assert_eq!(res.dims(), [1, 2, 9, 9]); + assert_eq!( + test_utils::to_vec3_round(&res.i(0)?, 4)?, + [ + [ + [-1.9918, 3.1652, -0.6778, -4.3442, 4.4351, 0.6652, -3.0124, -0.6031, 2.9277], + [2.7036, -1.7156, -0.3969, 1.0516, 1.6381, -2.8886, -0.205, 2.4682, -1.0499], + [-0.9459, 3.1631, 3.707, -4.8369, -8.5166, -1.4496, -2.7559, -3.2698, 1.4376], + [-0.2157, 3.7786, -2.0252, -4.2633, 3.6731, -1.5142, 5.9391, -0.2622, -0.141], + [-6.8121, -3.1744, 1.5945, 3.0637, -9.6088, 1.4446, 2.9489, -3.0082, -7.3822], + [0.2371, 3.3303, 0.3861, 2.2646, -4.6784, 4.1235, -0.0109, 0.3176, -0.03], + [-2.5339, -2.9564, -3.4518, -4.4594, -9.1873, -1.9709, -0.4676, 0.51, -3.5024], + [4.007, 0.3067, -2.2954, 1.1105, -0.1992, 1.6372, -2.9268, 0.2807, -1.2787], + [5.307, 1.1317, 1.3518, 0.9049, 3.8116, -0.4075, -0.8874, -0.2241, -0.9579] + ], + [ + [1.089, -0.6483, 0.0726, -0.4752, -1.3283, 1.7103, 1.0703, 0.1076, -0.9211], + [-0.8629, 0.1376, 0.3202, 2.0955, 0.9696, 2.8988, -1.0012, 1.5049, -0.1278], + [1.9286, -1.5255, -2.9563, 2.4589, 3.3611, -0.6951, 0.3525, -1.7724, -5.9861], + [1.1226, 2.1561, 3.6417, 4.7546, -0.692, 4.4126, -5.1902, 6.0805, 2.3185], + [1.0111, 0.3604, 0.6432, -3.6605, 7.9517, -9.2955, -5.2988, -3.7803, -2.0642], + [3.3172, -1.7967, -3.6576, -2.0942, 1.3158, 0.112, -1.7405, 2.9167, 0.7957], + [5.1001, 1.8995, -1.8639, 1.1262, 9.9629, 2.683, -3.6319, -1.1607, 0.5856], + [-4.8445, -0.5642, 4.2317, 0.0856, 1.2267, -0.5712, 1.736, 1.0997, 0.6908], + [-5.5423, -1.1831, -1.2176, 0.0843, 0.0446, -0.7545, -2.4798, -0.0827, 1.0171] + ] + ] + ); + Ok(()) +} + +/* This test is based on the following script. +import torch +torch.manual_seed(4242) + +t = torch.randn((1, 2, 3, 3)) +w = torch.randn((1, 2, 1, 1)) +print(t.flatten()) +print(w.flatten()) +res = torch.nn.functional.conv2d(t, w) +print(res.flatten()) + +w_t = w.transpose(0, 1) +res = torch.nn.functional.conv_transpose2d(t, w_t) +print(res.shape) +print(res.flatten()) + +t_t = w.transpose(0, 1) +res = torch.nn.functional.conv_transpose2d(t_t, w) +print(res.shape) +print(res.flatten()) +*/ +fn conv2d_small(dev: &Device) -> Result<()> { + let t = Tensor::new( + &[ + 0.4056f32, -0.8689, 0.6843, 0.2395, 1.2279, -0.9287, -1.7030, 0.1370, 0.1866, 0.4145, + -0.6266, 0.3529, 2.2013, -0.6836, 0.2477, 1.3127, -0.6957, 0.3278, + ], + dev, + )?; + let w = Tensor::new(&[-0.9259f32, 1.3017], dev)?; + let t = t.reshape((1, 2, 3, 3))?; + let w = w.reshape((1, 2, 1, 1))?; + let res = t.conv2d(&w, 0, 1, 1, 1)?; + assert_eq!(res.dims(), [1, 1, 3, 3]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [0.164, -0.0111, -0.1742, 2.6437, -2.0268, 1.1823, 3.2855, -1.0324, 0.2539] + ); + let res = t.conv2d(&w, 2, 1, 1, 1)?; + assert_eq!(res.dims(), [1, 1, 7, 7]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [ + 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, + 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.1640, -0.0111, -0.1742, 0.0000, 0.0000, + 0.0000, 0.0000, 2.6437, -2.0268, 1.1823, 0.0000, 0.0000, 0.0000, 0.0000, 3.2855, + -1.0324, 0.2539, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, + 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000 + ] + ); + let res = t.conv_transpose2d(&w.transpose(0, 1)?, 0, 0, 1, 1)?; + assert_eq!(res.dims(), [1, 1, 3, 3]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [0.164, -0.0111, -0.1742, 2.6437, -2.0268, 1.1823, 3.2855, -1.0324, 0.2539], + ); + let res = t.transpose(0, 1)?.conv_transpose2d(&w, 0, 0, 1, 1)?; + assert_eq!(res.dims(), [2, 2, 3, 3]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [ + -0.3755, 0.8045, -0.6336, -0.2218, -1.1369, 0.8599, 1.5768, -0.1268, -0.1728, 0.528, + -1.131, 0.8908, 0.3118, 1.5984, -1.2089, -2.2168, 0.1783, 0.2429, -0.3838, 0.5802, + -0.3268, -2.0382, 0.6329, -0.2293, -1.2154, 0.6441, -0.3035, 0.5396, -0.8156, 0.4594, + 2.8654, -0.8898, 0.3224, 1.7087, -0.9056, 0.4267 + ] + ); + Ok(()) +} + +fn conv2d_smaller(dev: &Device) -> Result<()> { + let t = Tensor::new( + &[ + 0.4056f32, -0.8689, 0.6843, 0.2395, 1.2279, -0.9287, -1.7030, 0.1370, 0.1866, + ], + dev, + )?; + let w = Tensor::new(&[1f32, 1., 1., 1., 1., 1., 1., 1., 1.], dev)?; + let t = t.reshape((1, 1, 3, 3))?; + let w = w.reshape((1, 1, 3, 3))?; + let res = t.conv2d(&w, 0, 1, 1, 1)?; + assert_eq!(res.dims(), [1, 1, 1, 1]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [-0.6197] + ); + Ok(()) +} + +/* This test is based on the following script. +import torch +torch.manual_seed(4242) + +t = torch.randn((1, 2, 4, 2)) +w = torch.randn((1, 2, 1, 1)) +print(t.flatten()) +print(w.flatten()) +res = torch.nn.functional.conv2d(t, w) +print(res.flatten()) +*/ +fn conv2d_non_square(dev: &Device) -> Result<()> { + let t = Tensor::new( + &[ + 0.4056f32, -0.8689, -0.0773, -1.5630, -2.8012, -1.5059, 0.3972, 1.0852, 0.4997, 3.0616, + 1.6541, 0.0964, -0.8338, -1.6523, -0.8323, -0.1699, + ], + dev, + )?; + let w = Tensor::new(&[-1.1351f32, 1.3841], dev)?; + let t = t.reshape((1, 2, 4, 2))?; + let w = w.reshape((1, 2, 1, 1))?; + let res = t.conv2d(&w, 0, 1, 1, 1)?; + assert_eq!(res.dims(), [1, 1, 4, 2]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [0.2312, 5.2238, 2.3772, 1.9076, 2.0256, -0.5776, -1.6028, -1.467] + ); + Ok(()) +} + +/* +import torch +torch.manual_seed(4242) + +t = torch.randn((1, 4, 5, 5), requires_grad=True) +w = torch.randn((2, 4, 3, 3), requires_grad=True) +print(t.flatten()) +print(w.flatten()) +res = torch.nn.functional.conv2d(t, w) +print(res.flatten()) +loss = (res ** 2).sum() +print(loss) +loss.backward() +print(t.grad.shape) +print(t.grad.flatten()) +print(w.grad.shape) +print(w.grad.flatten()) + +t.grad.zero_() +w.grad.zero_() +res = torch.nn.functional.conv2d(t, w, stride=2) +print(res.flatten()) +loss = (res ** 2).sum() +print(loss) +loss.backward() +print(t.grad.shape) +print(t.grad[0]) +print(w.grad.shape) +print(w.grad[0]) +*/ +fn conv2d_grad(dev: &Device) -> Result<()> { + use candle_core::Var; + let t = Var::from_slice( + &[ + 0.4056f32, -0.8689, -0.0773, -1.5630, -2.8012, -1.5059, 0.3972, 1.0852, 0.4997, 3.0616, + 1.6541, 0.0964, -0.8338, -1.6523, -0.8323, -0.1699, 0.0823, 0.3526, 0.6843, 0.2395, + 1.2279, -0.9287, -1.7030, 0.1370, 0.6047, 0.3770, -0.6266, 0.3529, 2.2013, -0.6836, + 0.2477, 1.3127, -0.2260, 0.2622, -1.2974, -0.8140, -0.8404, -0.3490, 0.0130, 1.3123, + 1.7569, -0.3956, -1.8255, 0.1727, -0.3538, 2.6941, 1.0529, 0.4219, -0.2071, 1.1586, + 0.4717, 0.3865, -0.5690, -0.5010, -0.1310, 0.7796, 0.6630, -0.2021, 2.6090, 0.2049, + 0.6466, -0.5042, -0.0603, -1.6538, -1.2429, 1.8357, 1.6052, -1.3844, 0.3323, -1.3712, + 0.9634, -0.4799, -0.6451, -0.0840, -1.4247, 0.5512, -0.1747, -0.5509, -0.3742, 0.3790, + -0.4431, -0.4720, -0.7890, 0.2620, 0.7875, 0.5377, -0.6779, -0.8088, 1.9098, 1.2006, + -0.8000, -0.4983, 1.5480, 0.8265, -0.1025, 0.5138, 0.5748, 0.3821, -0.4607, 0.0085, + ], + (1, 4, 5, 5), + dev, + )?; + let w = Var::from_slice( + &[ + -0.9325f32, 0.6451, -0.8537, 0.2378, 0.8764, -0.1832, 0.2987, -0.6488, -0.2273, + -2.4184, -0.1192, -0.4821, -0.5079, -0.5766, -2.4729, 1.6734, 0.4558, 0.2851, 1.1514, + -0.9013, 1.0662, -0.1817, -0.0259, 0.1709, 0.5367, 0.7513, 0.8086, -2.2586, -0.5027, + 0.9141, -1.3086, -1.3343, -1.5669, -0.1657, 0.7958, 0.1432, 0.3896, -0.4501, 0.1667, + 0.0714, -0.0952, 1.2970, -0.1674, -0.3178, 1.0677, 0.3060, 0.7080, 0.1914, 1.1679, + -0.3602, 1.9265, -1.8626, -0.5112, -0.0982, 0.2621, 0.6565, 0.5908, 1.0089, -0.1646, + 1.8032, -0.6286, 0.2016, -0.3370, 1.2555, 0.8009, -0.6488, -0.4652, -1.5685, 1.5860, + 0.5583, 0.4623, 0.6026, + ], + (2, 4, 3, 3), + dev, + )?; + let res = t.conv2d(&w, 0, 1, 1, 1)?; + let loss = res.sqr()?.sum_all()?; + assert_eq!(test_utils::to_vec0_round(&loss, 2)?, 741.12f32); + let grads = loss.backward()?; + let grad_t = grads.get(&t).unwrap(); + let grad_w = grads.get(&w).unwrap(); + assert_eq!(grad_t.dims(), [1, 4, 5, 5]); + assert_eq!(grad_w.dims(), [2, 4, 3, 3]); + assert_eq!( + test_utils::to_vec1_round(&grad_t.flatten_all()?, 2)?, + [ + 9.29, -2.84, -5.71, 3.38, -7.71, -19.15, 7.02, 29.1, 9.34, 34.73, -22.87, 24.35, + -39.88, -14.01, 21.08, 9.94, 13.63, -34.68, 11.21, -6.26, 7.72, -6.32, -16.64, -1.08, + -20.22, 21.73, -0.37, -4.06, 5.82, -3.65, -30.73, 14.55, 87.7, 31.6, 4.53, -89.78, + -75.37, -57.43, -7.56, 92.96, 18.79, -4.63, -159.75, -42.47, -47.26, 52.88, 37.32, + 49.0, 12.82, 2.01, -8.98, 20.18, 16.62, 12.06, 15.38, 20.0, 2.57, -15.22, 72.62, + -10.75, 2.25, -31.2, 3.75, -0.2, 9.76, -0.68, 5.21, -40.44, -22.59, -61.61, 17.28, + 20.41, 37.55, 5.23, 6.81, 23.54, 23.62, -9.99, -9.13, 4.87, -35.06, -26.1, 63.48, + 25.81, -39.21, -70.68, -46.96, 2.33, 41.81, 82.42, -28.63, -11.78, -35.33, -10.28, + -28.57, -9.13, 7.21, -9.05, -9.62, -11.25 + ] + ); + assert_eq!( + test_utils::to_vec1_round(&grad_w.flatten_all()?, 2)?, + [ + -28.92, -22.88, -141.23, 73.35, 61.07, 47.81, -20.0, -73.71, -41.82, -13.59, 21.5, + 28.72, 28.57, -46.85, -90.19, 143.61, 16.68, 7.43, 18.88, -90.81, -20.29, 54.79, 82.63, + 22.94, 77.81, -16.39, -13.2, 9.34, -40.39, -26.62, 5.33, -60.91, 9.09, -59.37, 7.08, + 58.64, 5.55, 20.52, 2.5, -17.25, -6.8, 22.21, 30.15, -7.52, -37.46, 5.67, 22.58, 9.03, + 47.05, 17.61, 37.31, -98.13, -14.61, -4.8, -6.36, 44.69, 23.34, 8.37, -13.52, 80.05, + -34.24, -16.36, -12.31, 1.92, -33.62, -14.1, -49.23, -7.39, 11.5, -9.98, 9.66, 29.6 + ] + ); + + // Same as before but with stride. + let res = t.conv2d(&w, 0, 2, 1, 1)?; + let loss = res.sqr()?.sum_all()?; + assert_eq!(test_utils::to_vec0_round(&loss, 2)?, 277.16f32); + let grads = loss.backward()?; + let grad_t = grads.get(&t).unwrap(); + let grad_w = grads.get(&w).unwrap(); + assert_eq!(grad_t.dims(), [1, 4, 5, 5]); + assert_eq!(grad_w.dims(), [2, 4, 3, 3]); + assert_eq!( + test_utils::to_vec3_round(&grad_t.i(0)?, 2)?, + [ + [ + [9.29, -7.03, 0.94, 3.49, -7.71], + [-1.8, -7.82, 8.9, 8.46, 7.43], + [-25.84, 22.09, -19.27, -0.22, 1.69], + [4.02, 18.53, -18.37, 2.3, -24.51], + [7.72, -9.68, -12.34, 5.6, -20.22] + ], + [ + [21.73, 3.39, -18.27, 3.86, -3.65], + [8.25, 3.73, 30.73, -8.61, -11.93], + [-72.15, -15.36, -17.53, -12.32, -1.61], + [-22.32, -7.79, -91.82, 6.44, -37.69], + [52.88, 14.44, 42.75, 9.88, 2.01] + ], + [ + [-8.98, 9.91, 6.75, -4.68, 15.38], + [4.93, -0.33, 9.94, -1.46, 14.78], + [13.62, -30.63, 3.96, -3.58, -4.48], + [-14.13, 1.19, -34.43, 3.08, -33.83], + [17.28, 12.94, 31.83, -3.35, 6.81] + ], + [ + [23.54, 6.98, -24.52, 0.52, 4.87], + [9.65, 6.18, 1.71, -25.23, -4.93], + [-54.99, -23.66, 3.19, -3.73, 18.58], + [-21.35, -10.39, -39.88, 28.73, -30.76], + [-9.13, 11.12, -14.0, -8.23, -11.25] + ] + ] + ); + assert_eq!( + test_utils::to_vec3_round(&grad_w.i(0)?, 2)?, + [ + [ + [28.34, -7.91, -45.75], + [21.03, 3.86, 29.86], + [0.72, -36.58, -35.28] + ], + [ + [-16.04, 11.53, -16.38], + [29.62, -16.32, -48.35], + [57.5, 28.29, 25.81] + ], + [ + [2.93, -19.6, 1.57], + [27.15, 53.88, -24.64], + [12.74, -22.6, -26.2] + ], + [ + [-0.18, -14.86, -6.82], + [-19.55, -2.72, 45.9], + [-2.54, 36.97, 27.11] + ] + ] + ); + + // Replicate the issue from https://github.com/huggingface/candle/issues/1212 + let res = t.i((.., .., 0..4, 0..4))?.conv2d(&w, 0, 2, 1, 1)?; + let loss = res.sqr()?.sum_all()?; + assert_eq!(test_utils::to_vec0_round(&loss, 2)?, 21.12f32); + let grads = loss.backward()?; + let grad_t = grads.get(&t).unwrap(); + let grad_w = grads.get(&w).unwrap(); + assert_eq!(grad_t.dims(), [1, 4, 5, 5]); + assert_eq!(grad_w.dims(), [2, 4, 3, 3]); + assert_eq!( + test_utils::to_vec3_round(&grad_t.i(0)?, 2)?, + [ + [ + [9.29, -7.03, 7.87, 0.0, 0.0], + [-1.8, -7.82, 5.9, 0.0, 0.0], + [-3.12, 4.49, 5.52, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0] + ], + [ + [21.73, 3.39, 4.77, 0.0, 0.0], + [8.25, 3.73, 27.61, 0.0, 0.0], + [-20.55, -5.61, -2.77, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0] + ], + [ + [-8.98, 9.91, -7.15, 0.0, 0.0], + [4.93, -0.33, 4.56, 0.0, 0.0], + [-6.7, -5.76, -8.05, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0] + ], + [ + [23.54, 6.98, -10.0, 0.0, 0.0], + [9.65, 6.18, 18.72, 0.0, 0.0], + [3.29, -5.27, 0.79, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0] + ] + ] + ); + assert_eq!( + test_utils::to_vec3_round(&grad_w.i(0)?, 2)?, + [ + [ + [-3.47, 7.44, 0.66], + [12.89, -3.4, -9.29], + [-14.16, -0.83, 7.14] + ], + [ + [-3.23, 5.37, -3.02], + [-2.12, -11.24, 1.94], + [6.97, 7.2, 2.99] + ], + [ + [-4.04, -3.31, 4.87], + [-6.68, -5.68, 1.73], + [-5.54, 4.32, 0.52] + ], + [[-4.72, 1.5, 4.72], [3.79, 4.04, 6.76], [-4.6, 5.8, 6.93]] + ] + ); + + Ok(()) +} + +test_device!(conv1d, conv1d_cpu, conv1d_gpu, conv1d_metal); +test_device!( + conv1d_small, + conv1d_small_cpu, + conv1d_small_gpu, + conv1d_small_metal +); +test_device!(conv2d, conv2d_cpu, conv2d_gpu, conv2d_metal); +test_device!( + conv2d_non_square, + conv2d_non_square_cpu, + conv2d_non_square_gpu, + conv2d_non_square_metal +); +test_device!( + conv2d_small, + conv2d_small_cpu, + conv2d_small_gpu, + conv2d_small_metal +); +test_device!( + conv2d_smaller, + conv2d_smaller_cpu, + conv2d_smaller_gpu, + conv2d_smaller_metal +); +test_device!( + conv2d_grad, + conv2d_grad_cpu, + conv2d_grad_gpu, + conv2_grad_metal +); diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/custom_op_tests.rs b/v2/dependency-patches/candle-core-0.4.1/tests/custom_op_tests.rs new file mode 100644 index 0000000000..cff0aebe78 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/tests/custom_op_tests.rs @@ -0,0 +1,114 @@ +use candle_core::backend::BackendStorage; +use candle_core::cpu_backend; +use candle_core::test_utils::to_vec1_round; +use candle_core::{CpuStorage, CustomOp1, DType, Device, Error, Layout, Result, Shape, Tensor}; + +fn fwd(v: T, alpha: f64) -> T { + if v.is_sign_positive() { + v + } else { + let alpha = T::from(alpha).unwrap_or(T::nan()); + (v.exp() - T::one()) * alpha + } +} + +struct Elu { + alpha: f64, +} + +impl CustomOp1 for Elu { + fn name(&self) -> &'static str { + "elu" + } + + fn cpu_fwd(&self, s: &CpuStorage, l: &Layout) -> Result<(CpuStorage, Shape)> { + let storage = candle_core::map_dtype!( + "elu", + s, + |s| cpu_backend::unary_map(s, l, |v| fwd(v, self.alpha)), + (BF16, F16, F32, F64) + ); + Ok((storage, l.shape().clone())) + } +} + +#[test] +fn custom_op1_no_backward() -> Result<()> { + let cpu = &Device::Cpu; + let t = Tensor::arange(0u32, 12u32, cpu)?.to_dtype(DType::F32)?; + let t = (t - 5.)?; + let elu_t = t.apply_op1_no_bwd(&Elu { alpha: 1. })?; + assert_eq!( + to_vec1_round(&elu_t, 4)?, + &[-0.9933, -0.9817, -0.9502, -0.8647, -0.6321, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0] + ); + Ok(()) +} + +// Define a similar struct as Elu but with backward support. +fn bwd(v: T, alpha: f64) -> T { + if v.is_sign_positive() { + T::one() + } else { + let alpha = T::from(alpha).unwrap_or(T::nan()); + v.exp() * alpha + } +} + +struct EluBackward { + alpha: f64, +} + +impl CustomOp1 for EluBackward { + fn name(&self) -> &'static str { + "elu-bwd" + } + + fn cpu_fwd(&self, s: &CpuStorage, l: &Layout) -> Result<(CpuStorage, Shape)> { + let storage = candle_core::map_dtype!( + "elu-bwd", + s, + |s| cpu_backend::unary_map(s, l, |v| bwd(v, self.alpha)), + (BF16, F16, F32, F64) + ); + Ok((storage, l.shape().clone())) + } +} + +struct EluWithBackward(Elu); + +impl EluWithBackward { + fn new(alpha: f64) -> Self { + Self(Elu { alpha }) + } +} + +impl CustomOp1 for EluWithBackward { + fn name(&self) -> &'static str { + "elu" + } + + fn cpu_fwd(&self, s: &CpuStorage, l: &Layout) -> Result<(CpuStorage, Shape)> { + self.0.cpu_fwd(s, l) + } + + fn bwd(&self, arg: &Tensor, _res: &Tensor, grad_res: &Tensor) -> Result> { + let alpha = self.0.alpha; + let bwd = arg.apply_op1(EluBackward { alpha })?; + Ok(Some(grad_res.mul(&bwd)?)) + } +} + +#[test] +fn custom_op1_with_backward() -> Result<()> { + let cpu = &Device::Cpu; + let t = candle_core::Var::new(&[-2f32, 0f32, 2f32], cpu)?; + let elu_t = t.apply_op1(EluWithBackward::new(2.))?; + assert_eq!(to_vec1_round(&elu_t, 4)?, &[-1.7293, 0.0, 2.0]); + + let grads = elu_t.backward()?; + let grad_x = grads.get(&t).unwrap(); + assert_eq!(to_vec1_round(grad_x, 4)?, [0.2707, 1.0, 1.0]); + + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/display_tests.rs b/v2/dependency-patches/candle-core-0.4.1/tests/display_tests.rs new file mode 100644 index 0000000000..8ab8bb6ab7 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/tests/display_tests.rs @@ -0,0 +1,84 @@ +use anyhow::Result; +use candle_core::{DType, Device::Cpu, Tensor}; + +#[test] +fn display_scalar() -> Result<()> { + let t = Tensor::new(1234u32, &Cpu)?; + let s = format!("{t}"); + assert_eq!(&s, "[1234]\nTensor[[], u32]"); + let t = t.to_dtype(DType::F32)?.neg()?; + let s = format!("{}", (&t / 10.0)?); + assert_eq!(&s, "[-123.4000]\nTensor[[], f32]"); + let s = format!("{}", (&t / 1e8)?); + assert_eq!(&s, "[-1.2340e-5]\nTensor[[], f32]"); + let s = format!("{}", (&t * 1e8)?); + assert_eq!(&s, "[-1.2340e11]\nTensor[[], f32]"); + let s = format!("{}", (&t * 0.)?); + assert_eq!(&s, "[0.]\nTensor[[], f32]"); + Ok(()) +} + +#[test] +fn display_vector() -> Result<()> { + let t = Tensor::new::<&[u32; 0]>(&[], &Cpu)?; + let s = format!("{t}"); + assert_eq!(&s, "[]\nTensor[[0], u32]"); + let t = Tensor::new(&[0.1234567, 1.0, -1.2, 4.1, f64::NAN], &Cpu)?; + let s = format!("{t}"); + assert_eq!( + &s, + "[ 0.1235, 1.0000, -1.2000, 4.1000, NaN]\nTensor[[5], f64]" + ); + let t = (Tensor::ones(50, DType::F32, &Cpu)? * 42.)?; + let s = format!("\n{t}"); + let expected = r#" +[42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., + 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., + 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., + 42., 42.] +Tensor[[50], f32]"#; + assert_eq!(&s, expected); + let t = (Tensor::ones(11000, DType::F32, &Cpu)? * 42.)?; + let s = format!("{t}"); + assert_eq!( + &s, + "[42., 42., 42., ..., 42., 42., 42.]\nTensor[[11000], f32]" + ); + Ok(()) +} + +#[test] +fn display_multi_dim() -> Result<()> { + let t = (Tensor::ones((200, 100), DType::F32, &Cpu)? * 42.)?; + let s = format!("\n{t}"); + let expected = r#" +[[42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + ... + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.]] +Tensor[[200, 100], f32]"#; + assert_eq!(&s, expected); + let t = t.reshape(&[2, 1, 1, 100, 100])?; + let t = format!("\n{t}"); + let expected = r#" +[[[[[42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + ... + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.]]]], + [[[[42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + ... + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.]]]]] +Tensor[[2, 1, 1, 100, 100], f32]"#; + assert_eq!(&t, expected); + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/fortran_tensor_3d.pth b/v2/dependency-patches/candle-core-0.4.1/tests/fortran_tensor_3d.pth new file mode 100644 index 0000000000000000000000000000000000000000..bd50b03d9cd74135fa6f53e2bf39da2f7ac0879d GIT binary patch literal 1486 zcmbVMON-M`6uxcRw3C_f{b*;vl}H8cq_1fM4rq~Vu3&5tikp&2uAN|L<|elT3Zfes z`~xoi8LnNr)ZgHra3gq5ax?akg5!b1*QDoseErTjwWv}cq+BK|u|vw_We|$co{j~Z zMnO1kxqGfH?E3L5-|QF23yAmmx15YFK)0-sEj;IlAasuEF|D;O?2<9xF#&?}#<-dQ*_3!wA}nJZmn5c(c}+fVNvudj z=N{boHv9_DnCqRZ5+lR86Bb6WEM~uRZtLQCw=`7nI)k}hL5W)3v?BD(;LAu`VHkz$&w_)rOI+dOSykWlHOd3LLTKWc|6b;pN%gI@N0_} N Result<()> { + let x = Var::new(&[3f32, 1., 4.], device)?; + let x = x.as_tensor(); + let y = (((x * x)? + x * 5f64)? + 4f64)?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!(x.to_vec1::()?, [3., 1., 4.]); + // y = x^2 + 5.x + 4 + assert_eq!(y.to_vec1::()?, [28., 10., 40.]); + // dy/dx = 2.x + 5 + assert_eq!(grad_x.to_vec1::()?, [11., 7., 13.]); + Ok(()) +} + +fn sum_grad(device: &Device) -> Result<()> { + let x = Var::new(&[3f32, 1., 4.], device)?; + let x = x.as_tensor(); + let y = (x.sqr()?.sum_keepdim(0)? * 2.)?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!(y.to_vec1::()?, [52.]); + // y = 2.x^2 so dy/dx = 4.x + assert_eq!(grad_x.to_vec1::()?, &[12., 4., 16.]); + + // Same test as before but squeezing on the last dimension. + let y = (x.sqr()?.sum_keepdim(0)? * 2.)?.squeeze(0)?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!(y.to_scalar::()?, 52.); + // y = 2.x^2 so dy/dx = 4.x + assert_eq!(grad_x.to_vec1::()?, &[12., 4., 16.]); + Ok(()) +} + +fn matmul_grad(device: &Device) -> Result<()> { + let data: Vec<_> = (0..12).map(|i| i as f32).collect(); + let x = Var::from_slice(&data, (2, 2, 3), device)?; + let data: Vec<_> = (0..12).map(|i| i as f32).collect(); + let y = Var::from_slice(&data, (2, 3, 2), device)?; + let c = x.matmul(&y)?; + let grads = c.backward()?; + let grad_x = grads.get(&x).context("no grad for x")?; + let grad_y = grads.get(&y).context("no grad for y")?; + assert_eq!(grad_x.shape(), &Shape::from((2, 2, 3))); + assert_eq!(grad_y.shape(), &Shape::from((2, 3, 2))); + assert_eq!( + &*grad_x.to_vec3::()?, + &[ + [[1., 5., 9.], [1., 5., 9.]], + [[13., 17., 21.], [13., 17., 21.]] + ] + ); + assert_eq!( + &*grad_y.to_vec3::()?, + &[ + [[3., 3.], [5., 5.], [7., 7.]], + [[15., 15.], [17., 17.], [19., 19.]] + ] + ); + Ok(()) +} + +// The simplest gradient descent, using scalar variable. +fn grad_descent(device: &Device) -> Result<()> { + let x = Var::new(0f32, device)?; + let learning_rate = 0.1; + for _step in 0..100 { + let xt = x.as_tensor(); + let c = ((xt - 4.2)? * (xt - 4.2)?)?; + let grads = c.backward()?; + let x_grad = grads.get(&x).context("no grad for x")?; + x.set(&(xt - x_grad * learning_rate)?)? + } + assert_eq!(x.to_scalar::()?, 4.199999); + Ok(()) +} + +fn unary_grad(device: &Device) -> Result<()> { + let x = Var::new(&[3f32, 1., 4., 0.15], device)?; + let x = x.as_tensor(); + let y = (x.log()? + 1.)?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!( + test_utils::to_vec1_round(&y, 4)?, + [2.0986, 1.0, 2.3863, -0.8971] + ); + assert_eq!( + test_utils::to_vec1_round(grad_x, 4)?, + [0.3333, 1.0, 0.25, 6.6667] + ); + let y = x.exp()?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!( + y.to_vec1::()?, + [20.085537, 2.7182817, 54.59815, 1.1618342] + ); + assert_eq!( + grad_x.to_vec1::()?, + [20.085537, 2.7182817, 54.59815, 1.1618342] + ); + let y = x.exp()?.sqr()?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!( + y.to_vec1::()?, + [403.4288, 7.3890557, 2980.9578, 1.3498588] + ); + // exp(x)^2 = exp(2*x) + assert_eq!( + grad_x.to_vec1::()?, + [806.8576, 14.778111, 5961.9155, 2.6997175] + ); + let y = x.sin()?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!( + test_utils::to_vec1_round(&y, 4)?, + [0.1411, 0.8415, -0.7568, 0.1494], + ); + assert_eq!( + test_utils::to_vec1_round(grad_x, 4)?, + [-0.99, 0.5403, -0.6536, 0.9888], + ); + let y = x.cos()?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!( + test_utils::to_vec1_round(&y, 4)?, + [-0.99, 0.5403, -0.6536, 0.9888], + ); + assert_eq!( + test_utils::to_vec1_round(grad_x, 4)?, + [-0.1411, -0.8415, 0.7568, -0.1494], + ); + let y = x.sqr()?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!(y.to_vec1::()?, [9.0, 1.0, 16.0, 0.0225]); + assert_eq!(grad_x.to_vec1::()?, [6.0, 2.0, 8.0, 0.3]); + let y = x.sqr()?.sqrt()?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!(y.to_vec1::()?, [3.0, 1.0, 4.0, 0.15]); + assert_eq!(test_utils::to_vec1_round(grad_x, 4)?, [1.0, 1.0, 1.0, 1.0]); + let y = x.neg()?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!(y.to_vec1::()?, [-3.0, -1.0, -4.0, -0.15]); + assert_eq!(grad_x.to_vec1::()?, [-1.0, -1.0, -1.0, -1.0]); + let y = x.affine(0.2, 1.)?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!(y.to_vec1::()?, [1.6, 1.2, 1.8, 1.03]); + assert_eq!(grad_x.to_vec1::()?, [0.2, 0.2, 0.2, 0.2]); + let y = Tensor::new(1f32, device)?.broadcast_div(x)?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!( + test_utils::to_vec1_round(&y, 4)?, + [0.3333, 1.0, 0.25, 6.6667] + ); + assert_eq!( + grad_x.to_vec1::()?, + [-0.11111111, -1.0, -0.0625, -44.444443], + ); + let y = x.broadcast_div(&Tensor::new(0.5f32, device)?)?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!(y.to_vec1::()?, [6., 2., 8., 0.3]); + assert_eq!(grad_x.to_vec1::()?, [2., 2., 2., 2.]); + + let x = Var::new(&[3f32, 1., 4., 0.15], device)?; + let y = x.powf(2.5)?; + let grads = y.backward()?; + let grad_x = grads.get(&x).context("no grad for x")?; + assert_eq!(test_utils::to_vec1_round(&y, 2)?, [15.59, 1.0, 32.0, 0.01]); + assert_eq!( + test_utils::to_vec1_round(grad_x, 2)?, + [12.99, 2.5, 20.0, 0.15] + ); + + let y = x.tanh()?; + let grads = y.backward()?; + let grad_x = grads.get(&x).context("no grad for x")?; + assert_eq!(test_utils::to_vec1_round(&y, 2)?, [1.0, 0.76, 1.0, 0.15]); + assert_eq!( + test_utils::to_vec1_round(grad_x, 2)?, + [0.01, 0.42, 0.0, 0.98], + ); + + // testing compared to pytorch nn.GELU(approximate = 'tanh') + let y = x.gelu()?; + let grads = y.backward()?; + let grad_x = grads.get(&x).context("no grad for x")?; + assert_eq!( + test_utils::to_vec1_round(&y, 4)?, + [2.9964, 0.8412, 3.9999, 0.0839] + ); + assert_eq!( + test_utils::to_vec1_round(grad_x, 4)?, + [1.0116, 1.0830, 1.0003, 0.6188], + ); + + // Testing compared to pytorch torch.erf + // + // import torch + // x = torch.tensor([3.0, 1.0, 4.0, 0.15], requires_grad=True) + // y = x.erf() + // print(y) + // loss = y.sum() + // loss.backward() + // print(x.grad) + let y = x.erf()?; + let grads = y.backward()?; + let grad_x = grads.get(&x).context("no grad for x")?; + assert_eq!(test_utils::to_vec1_round(&y, 4)?, [1.0, 0.8427, 1.0, 0.168]); + assert_eq!( + test_utils::to_vec1_round(grad_x, 4)?, + [0.0001, 0.4151, 0.0, 1.1033], + ); + + // Testing compared to pytorch nn.GELU(approximate = 'none') + // + // import torch + // import torch.nn.functional as F + // x = torch.tensor([3.0, 1.0, 4.0, 0.15], requires_grad=True) + // y = F.gelu(x, approximate='none') + // print(y) + // loss = y.sum() + // loss.backward() + // print(x.grad) + let y = x.gelu_erf()?; + let grads = y.backward()?; + let grad_x = grads.get(&x).context("no grad for x")?; + assert_eq!( + test_utils::to_vec1_round(&y, 4)?, + [2.9960, 0.8413, 3.9999, 0.0839] + ); + assert_eq!( + test_utils::to_vec1_round(grad_x, 4)?, + [1.0119, 1.0833, 1.0005, 0.6188], + ); + + // Testing compared to pytorch elu + // + // import torch + // import torch.nn.functional as F + // x = torch.tensor([-1.0, 0.0, -2.0, 3.0], requires_grad=True) + // y = F.elu(x, alpha=2.0) + // print(y) + // loss = y.min + // loss = y.sum() + // loss.backward() + // print(x.grad) + let elu_x = Var::new(&[-1.0f32, 0., -2., 3.], device)?; + let y = elu_x.elu(2.)?; + let grads = y.backward()?; + let grad_x = grads.get(&elu_x).context("no grad for x")?; + assert_eq!( + test_utils::to_vec1_round(&y, 4)?, + [-1.2642, 0.0000, -1.7293, 3.0000] + ); + assert_eq!( + test_utils::to_vec1_round(grad_x, 4)?, + [0.7358, 2.0000, 0.2707, 1.0000] + ); + + // testing compared to pytorch nn.Silu() + let y = x.silu()?; + let grads = y.backward()?; + let grad_x = grads.get(&x).context("no grad for x")?; + assert_eq!( + test_utils::to_vec1_round(&y, 4)?, + [2.8577, 0.7311, 3.9281, 0.0806] + ); + assert_eq!( + test_utils::to_vec1_round(grad_x, 4)?, + [1.0881, 0.9277, 1.0527, 0.5747], + ); + + if device.is_cpu() { + let x = Var::new(&[[[1f32, 2., 3.], [4., 5., 6.], [7., 8., 9.]]], device)?; + let y = x.interpolate1d(12)?.reshape(36)?; + + let z = Tensor::new( + &[ + 1_f32, 02., 03., 04., 05., 06., 07., 08., 09., 10., 11., 12., 13., 14., 15., 16., + 17., 18., 19., 20., 21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., + 33., 34., 35., 36., + ], + device, + )?; + + let loss = y.unsqueeze(1)?.transpose(0, 1)?.matmul(&z.unsqueeze(1)?)?; + let grads = loss.backward()?; + let grad_x = grads.get(&x).context("no grad for x")?; + + assert_eq!( + test_utils::to_vec3_round(grad_x, 4)?, + [[[10_f32, 26., 42.], [58., 74., 90.], [106., 122., 138.]]] + ); + } + + // manually checked: see comments + let x = Var::new(&[[[[1f32, 2., 3.], [4., 5., 6.], [7., 8., 9.]]]], device)?; + let y = x.interpolate2d(6, 6)?.reshape(36)?; + + let z = Tensor::new( + &[ + 1_f32, 02., 03., 04., 05., 06., 07., 08., 09., 10., 11., 12., 13., 14., 15., 16., 17., + 18., 19., 20., 21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., 33., 34., + 35., 36., + ], + device, + )?; + // gradient should be + // row 1 + // 1+2+7+8 = 18 + // 3+4+9+10 = 26 + // 5+6+11+12 = 34 + // row 2 + // 13+14+19+20 = 66 + // 15+16+21+22 = 74 + // 17+18+23+24 = 82 + // row 3 + // 25+26+31+32 = 114 + // 27+28+33+34 = 122 + // 29+30+35+36 = 130 + let loss = y.unsqueeze(1)?.transpose(0, 1)?.matmul(&z.unsqueeze(1)?)?; + + let grads = loss.backward()?; + + let grad_x = grads.get(&x).context("no grad for x")?; + assert_eq!( + test_utils::to_vec2_round(&grad_x.flatten(0, 2)?, 4)?, + [[18_f32, 26., 34.], [66., 74., 82.], [114., 122., 130.]] + ); + + // manually checked: see comments + let x = Var::new(&[[[[1f32, 2.], [4., 5.]]]], device)?; + let y = x.interpolate2d(6, 6)?.reshape(36)?; + + let z = Tensor::new( + &[ + 1_f32, 02., 03., 04., 05., 06., 07., 08., 09., 10., 11., 12., 13., 14., 15., 16., 17., + 18., 19., 20., 21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., 33., 34., + 35., 36., + ], + device, + )?; + // gradient should be + // row 1 + // 1+2+3+7+8+9+13+14+15 = 72 + // 4+5+6+10+11+12+16+17+18 = 99 + // row 2 + // 19+20+21+25+26+27+31+32+33 = 234 + // 22+23+24+28+29+30+34+35+36 = 243 + let loss = y.unsqueeze(1)?.transpose(0, 1)?.matmul(&z.unsqueeze(1)?)?; + + let grads = loss.backward()?; + + let grad_x = grads.get(&x).context("no grad for x")?; + assert_eq!( + test_utils::to_vec2_round(&grad_x.flatten(0, 2)?, 4)?, + [[72_f32, 99.], [234., 261.]] + ); + + // manually checked: see comments + let x = Var::new(&[[[[1f32, 2.], [4., 5.]], [[6f32, 7.], [8., 9.]]]], device)?; + + let y = x.interpolate2d(4, 4)?.reshape(32)?; + + #[rustfmt::skip] + let z = Tensor::new( + &[ + 1_f32, 02., 03., 04., + 05., 06., 07., 08., + 09., 10., 11., 12., + 13., 14., 15., 16., + 17., 18., 19., 20., + 21., 22., 23., 24., + 25., 26., 27., 28., + 29., 30., 31., 32. + ], + device, + )?; + // gradient should be + // m1r1 + // 1+2+5+6=14 + // 3+4+7+8=22 + // m1r2 + // 9+10+13+14=46 + // 11+12+15+16=54 + // m2r1 + // 17+18+21+22=78 + // 19+20+23+24=86 + // m2r2 + // 25+26+29+30=110 + // 27+28+31+32=118 + let loss = y.unsqueeze(1)?.transpose(0, 1)?.matmul(&z.unsqueeze(1)?)?; + + let grads = loss.backward()?; + + let grad_x = grads.get(&x).context("no grad for x")?; + + assert_eq!( + test_utils::to_vec3_round(&grad_x.flatten(0, 1)?, 4)?, + [[[14_f32, 22.], [46., 54.]], [[78., 86.], [110., 118.]]] + ); + + // manually checked: see comments + let x = Var::new( + &[[[[1f32, 2.], [4., 5.]]], [[[6f32, 7.], [8., 9.]]]], + device, + )?; + + let y = x.interpolate2d(4, 4)?.reshape(32)?; + + #[rustfmt::skip] + let z = Tensor::new( + &[ + 1_f32, 02., 03., 04., + 05., 06., 07., 08., + 09., 10., 11., 12., + 13., 14., 15., 16., + 17., 18., 19., 20., + 21., 22., 23., 24., + 25., 26., 27., 28., + 29., 30., 31., 32. + ], + device, + )?; + // gradient should be + // m1r1 + // 1+2+5+6=14 + // 3+4+7+8=22 + // m1r2 + // 9+10+13+14=46 + // 11+12+15+16=54 + // m2r1 + // 17+18+21+22=78 + // 19+20+23+24=86 + // m2r2 + // 25+26+29+30=110 + // 27+28+31+32=118 + let loss = y.unsqueeze(1)?.transpose(0, 1)?.matmul(&z.unsqueeze(1)?)?; + + let grads = loss.backward()?; + + let grad_x = grads.get(&x).context("no grad for x")?; + + assert_eq!( + test_utils::to_vec3_round(&grad_x.flatten(0, 1)?, 4)?, + [[[14_f32, 22.], [46., 54.]], [[78., 86.], [110., 118.]]] + ); + Ok(()) +} + +fn binary_grad(device: &Device) -> Result<()> { + let x = Var::new(&[3f32, 1., -4., -1.], device)?; + let x = x.as_tensor(); + // leaky relu + let y = x.maximum(&(x * 0.1)?)?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!(x.to_vec1::()?, [3., 1., -4., -1.]); + assert_eq!(y.to_vec1::()?, [3., 1., -0.4, -0.1]); + assert_eq!(grad_x.to_vec1::()?, [1., 1., 0.1, 0.1]); + + let y = x.minimum(&(x * 0.1)?)?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!(y.to_vec1::()?, [0.3, 0.1, -4., -1.]); + assert_eq!(grad_x.to_vec1::()?, [0.1, 0.1, 1., 1.]); + + // This one is easy to mess up, we want the gradient to be one as it is the identity function. + let y = x.minimum(x)?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!(y.to_vec1::()?, [3., 1., -4., -1.]); + assert_eq!(grad_x.to_vec1::()?, [1., 1., 1., 1.]); + + let x_var = Var::new(&[3f32, 1., -4., -1., 5., 9.], device)?; + let x = x_var.as_tensor(); + let y_var = Var::new(&[2f32, 7., 1.], device)?; + let y = y_var.as_tensor(); + + let ss = x + .reshape((2, 3))? + .slice_scatter0(&y.reshape((1, 3))?, 1)? + .sqr()?; + let grads = ss.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + let grad_y = grads.get(y).context("no grad for y")?; + assert_eq!(ss.to_vec2::()?, [[9., 1., 16.], [4., 49., 1.]]); + assert_eq!(grad_x.to_vec1::()?, [6.0, 2.0, -8.0, 0.0, 0.0, 0.0]); + assert_eq!(grad_y.to_vec1::()?, [4.0, 14.0, 2.0]); + Ok(()) +} + +test_device!( + simple_grad, + simple_grad_cpu, + simple_grad_gpu, + simple_grad_metal +); +test_device!(sum_grad, sum_grad_cpu, sum_grad_gpu, sum_grad_metal); +test_device!( + matmul_grad, + matmul_grad_cpu, + matmul_grad_gpu, + matmul_grad_metal +); +test_device!( + grad_descent, + grad_descent_cpu, + grad_descent_gpu, + grad_descent_metal +); +test_device!(unary_grad, unary_grad_cpu, unary_grad_gpu, unary_grad_metal); +test_device!( + binary_grad, + binary_grad_cpu, + binary_grad_gpu, + binary_grad_metal +); diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/indexing_tests.rs b/v2/dependency-patches/candle-core-0.4.1/tests/indexing_tests.rs new file mode 100644 index 0000000000..047205a31f --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/tests/indexing_tests.rs @@ -0,0 +1,122 @@ +use anyhow::Result; +use candle_core::{Device, IndexOp, Tensor}; + +#[test] +fn integer_index() -> Result<()> { + let dev = Device::Cpu; + + let tensor = Tensor::arange(0u32, 2 * 3, &dev)?.reshape((2, 3))?; + let result = tensor.i(1)?; + assert_eq!(result.dims(), &[3]); + assert_eq!(result.to_vec1::()?, &[3, 4, 5]); + + let result = tensor.i((.., 2))?; + assert_eq!(result.dims(), &[2]); + assert_eq!(result.to_vec1::()?, &[2, 5]); + + Ok(()) +} + +#[test] +fn range_index() -> Result<()> { + let dev = Device::Cpu; + // RangeFull + let tensor = Tensor::arange(0u32, 2 * 3, &dev)?.reshape((2, 3))?; + let result = tensor.i(..)?; + assert_eq!(result.dims(), &[2, 3]); + assert_eq!(result.to_vec2::()?, &[[0, 1, 2], [3, 4, 5]]); + + // Range + let tensor = Tensor::arange(0u32, 4 * 3, &dev)?.reshape((4, 3))?; + let result = tensor.i(1..3)?; + assert_eq!(result.dims(), &[2, 3]); + assert_eq!(result.to_vec2::()?, &[[3, 4, 5], [6, 7, 8]]); + + // RangeFrom + let result = tensor.i(2..)?; + assert_eq!(result.dims(), &[2, 3]); + assert_eq!(result.to_vec2::()?, &[[6, 7, 8], [9, 10, 11]]); + + // RangeTo + let result = tensor.i(..2)?; + assert_eq!(result.dims(), &[2, 3]); + assert_eq!(result.to_vec2::()?, &[[0, 1, 2], [3, 4, 5]]); + + // RangeInclusive + let result = tensor.i(1..=2)?; + assert_eq!(result.dims(), &[2, 3]); + assert_eq!(result.to_vec2::()?, &[[3, 4, 5], [6, 7, 8]]); + + // RangeTo + let result = tensor.i(..1)?; + assert_eq!(result.dims(), &[1, 3]); + assert_eq!(result.to_vec2::()?, &[[0, 1, 2]]); + + // RangeToInclusive + let result = tensor.i(..=1)?; + assert_eq!(result.dims(), &[2, 3]); + assert_eq!(result.to_vec2::()?, &[[0, 1, 2], [3, 4, 5]]); + + // Empty range + let result = tensor.i(1..1)?; + assert_eq!(result.dims(), &[0, 3]); + let empty: [[u32; 3]; 0] = []; + assert_eq!(result.to_vec2::()?, &empty); + + // Similar to PyTorch, allow empty ranges when the computed length is negative. + #[allow(clippy::reversed_empty_ranges)] + let result = tensor.i(1..0)?; + assert_eq!(result.dims(), &[0, 3]); + let empty: [[u32; 3]; 0] = []; + assert_eq!(result.to_vec2::()?, &empty); + Ok(()) +} + +#[test] +fn index_3d() -> Result<()> { + let tensor = Tensor::from_iter(0..24u32, &Device::Cpu)?.reshape((2, 3, 4))?; + assert_eq!(tensor.i((0, 0, 0))?.to_scalar::()?, 0); + assert_eq!(tensor.i((1, 0, 0))?.to_scalar::()?, 12); + assert_eq!(tensor.i((0, 1, 0))?.to_scalar::()?, 4); + assert_eq!(tensor.i((0, 1, 3))?.to_scalar::()?, 7); + assert_eq!(tensor.i((0..2, 0, 0))?.to_vec1::()?, &[0, 12]); + assert_eq!( + tensor.i((0..2, .., 0))?.to_vec2::()?, + &[[0, 4, 8], [12, 16, 20]] + ); + assert_eq!( + tensor.i((..2, .., 3))?.to_vec2::()?, + &[[3, 7, 11], [15, 19, 23]] + ); + assert_eq!(tensor.i((1, .., 3))?.to_vec1::()?, &[15, 19, 23]); + Ok(()) +} + +#[test] +fn slice_assign() -> Result<()> { + let dev = Device::Cpu; + + let tensor = Tensor::arange(0u32, 4 * 5, &dev)?.reshape((4, 5))?; + let src = Tensor::arange(0u32, 2 * 3, &dev)?.reshape((3, 2))?; + let out = tensor.slice_assign(&[1..4, 3..5], &src)?; + assert_eq!( + out.to_vec2::()?, + &[ + [0, 1, 2, 3, 4], + [5, 6, 7, 0, 1], + [10, 11, 12, 2, 3], + [15, 16, 17, 4, 5] + ] + ); + let out = tensor.slice_assign(&[0..3, 0..2], &src)?; + assert_eq!( + out.to_vec2::()?, + &[ + [0, 1, 2, 3, 4], + [2, 3, 7, 8, 9], + [4, 5, 12, 13, 14], + [15, 16, 17, 18, 19] + ] + ); + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/layout_tests.rs b/v2/dependency-patches/candle-core-0.4.1/tests/layout_tests.rs new file mode 100644 index 0000000000..e06188509c --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/tests/layout_tests.rs @@ -0,0 +1,138 @@ +use candle::{test_device, Device, IndexOp, Result, Tensor}; +use candle_core as candle; + +fn contiguous(device: &Device) -> Result<()> { + let tensor = Tensor::arange(0u32, 24u32, device)?.reshape((2, 3, 4))?; + assert_eq!( + tensor.to_vec3::()?, + &[ + [[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]], + [[12, 13, 14, 15], [16, 17, 18, 19], [20, 21, 22, 23]] + ] + ); + assert_eq!( + tensor.t()?.contiguous()?.to_vec3::()?, + &[ + [[0, 4, 8], [1, 5, 9], [2, 6, 10], [3, 7, 11]], + [[12, 16, 20], [13, 17, 21], [14, 18, 22], [15, 19, 23]] + ] + ); + assert_eq!( + tensor.transpose(0, 1)?.contiguous()?.to_vec3::()?, + &[ + [[0, 1, 2, 3], [12, 13, 14, 15]], + [[4, 5, 6, 7], [16, 17, 18, 19]], + [[8, 9, 10, 11], [20, 21, 22, 23]] + ] + ); + assert_eq!( + tensor.transpose(0, 1)?.flatten_all()?.to_vec1::()?, + &[0, 1, 2, 3, 12, 13, 14, 15, 4, 5, 6, 7, 16, 17, 18, 19, 8, 9, 10, 11, 20, 21, 22, 23] + ); + assert_eq!( + tensor + .i(1..)? + .transpose(0, 1)? + .contiguous()? + .to_vec3::()?, + &[[[12, 13, 14, 15]], [[16, 17, 18, 19]], [[20, 21, 22, 23]]] + ); + assert_eq!( + tensor.transpose(0, 2)?.contiguous()?.to_vec3::()?, + &[ + [[0, 12], [4, 16], [8, 20]], + [[1, 13], [5, 17], [9, 21]], + [[2, 14], [6, 18], [10, 22]], + [[3, 15], [7, 19], [11, 23]] + ] + ); + Ok(()) +} + +test_device!(contiguous, contiguous_cpu, contiguous_gpu, contiguous_metal); + +#[test] +fn strided_blocks() -> Result<()> { + use candle::Device::Cpu; + let tensor = Tensor::arange(0u32, 24u32, &Cpu)?.reshape((2, 3, 4))?; + match tensor.strided_blocks() { + candle::StridedBlocks::SingleBlock { start_offset, len } => { + assert_eq!(start_offset, 0); + assert_eq!(len, 24); + } + candle::StridedBlocks::MultipleBlocks { .. } => { + panic!("unexpected block structure") + } + }; + let tensor = Tensor::arange(0u32, 26u32, &Cpu)? + .i(2..)? + .reshape((2, 3, 4))?; + match tensor.strided_blocks() { + candle::StridedBlocks::SingleBlock { start_offset, len } => { + assert_eq!(start_offset, 2); + assert_eq!(len, 24); + } + candle::StridedBlocks::MultipleBlocks { .. } => { + panic!("unexpected block structure") + } + }; + let tensor = Tensor::arange(0u32, 24u32, &Cpu)?.reshape((2, 3, 4))?; + let tensor = tensor.i(1)?; + match tensor.strided_blocks() { + candle::StridedBlocks::SingleBlock { start_offset, len } => { + assert_eq!(start_offset, 12); + assert_eq!(len, 12); + } + candle::StridedBlocks::MultipleBlocks { .. } => { + panic!("unexpected block structure") + } + }; + let tensor = Tensor::arange(0u32, 24u32, &Cpu)?.reshape((2, 3, 4))?; + let tensor = tensor.i((.., 1))?; + match tensor.strided_blocks() { + candle::StridedBlocks::SingleBlock { start_offset, len } => { + assert_eq!(start_offset, 0); + assert_eq!(len, 8); + assert_eq!(tensor.to_vec2::()?, &[[4, 5, 6, 7], [16, 17, 18, 19]]); + } + candle::StridedBlocks::MultipleBlocks { .. } => { + panic!("unexpected block structure") + } + }; + let tensor = Tensor::arange(0u32, 24u32, &Cpu)?.reshape((2, 3, 4))?; + match tensor.t()?.strided_blocks() { + candle::StridedBlocks::SingleBlock { .. } => { + panic!("unexpected block structure") + } + candle::StridedBlocks::MultipleBlocks { + block_start_index, + block_len, + } => { + assert_eq!(block_len, 1); + assert_eq!( + block_start_index.collect::>(), + &[ + 0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11, 12, 16, 20, 13, 17, 21, 14, 18, 22, 15, + 19, 23 + ] + ) + } + }; + let tensor = Tensor::arange(0u32, 24u32, &Cpu)?.reshape((2, 3, 4))?; + match tensor.transpose(0, 1)?.strided_blocks() { + candle::StridedBlocks::SingleBlock { .. } => { + panic!("unexpected block structure") + } + candle::StridedBlocks::MultipleBlocks { + block_start_index, + block_len, + } => { + assert_eq!(block_len, 4); + assert_eq!( + block_start_index.collect::>(), + &[0, 12, 4, 16, 8, 20] + ) + } + }; + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/npy.py b/v2/dependency-patches/candle-core-0.4.1/tests/npy.py new file mode 100644 index 0000000000..0fd2778a15 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/tests/npy.py @@ -0,0 +1,9 @@ +import numpy as np +x = np.arange(10) + +# Write a npy file. +np.save("test.npy", x) + +# Write multiple values to a npz file. +values = { "x": x, "x_plus_one": x + 1 } +np.savez("test.npz", **values) diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/pool_tests.rs b/v2/dependency-patches/candle-core-0.4.1/tests/pool_tests.rs new file mode 100644 index 0000000000..a3708ec488 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/tests/pool_tests.rs @@ -0,0 +1,114 @@ +use candle_core::{test_device, test_utils, Device, IndexOp, Result, Tensor}; + +// https://github.com/huggingface/candle/issues/364 +fn avg_pool2d(dev: &Device) -> Result<()> { + let data: Vec = vec![ + 1., 1., 1., 1., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., + ]; + let t = Tensor::from_vec(data, (1, 1, 4, 4), dev)?; + let pool = t.avg_pool2d(2)?.squeeze(0)?.squeeze(0)?; + assert_eq!(pool.to_vec2::()?, [[0.5f32, 1.], [1., 1.]]); + + let data: Vec = vec![ + 1., 2., 1., 3., 0., 0., 1., 1., 1., 1., 1., 1., 5., 1., 1., 1., + ]; + let t = Tensor::from_vec(data, (1, 1, 2, 8), dev)?; + let pool = t.avg_pool2d(2)?.squeeze(0)?.squeeze(0)?; + assert_eq!(pool.to_vec2::()?, [[5. / 4., 6. / 4., 6. / 4., 1.]]); + Ok(()) +} + +fn max_pool2d(dev: &Device) -> Result<()> { + let data: Vec = vec![ + 1., 2., 1., 3., 0., 0., 1., 1., 1., 1., 1., 1., 5., 1., 1., 1., + ]; + let t = Tensor::from_vec(data, (1, 1, 4, 4), dev)?; + + let pool = t.max_pool2d(2)?.squeeze(0)?.squeeze(0)?; + assert_eq!(pool.to_vec2::()?, [[2f32, 3.], [5., 1.]]); + + let t = t.reshape((1, 1, 2, 8))?; + let pool = t.max_pool2d(2)?.squeeze(0)?.squeeze(0)?; + assert_eq!(pool.to_vec2::()?, [[2.0, 3.0, 5.0, 1.0]]); + Ok(()) +} + +/* This test corresponds to the following PyTorch script. +import torch +torch.manual_seed(4242) + +t = torch.randn((1, 2, 4, 4)) +print(t.flatten()) +res = torch.nn.functional.avg_pool2d(t, 2) +print(res) +*/ +fn avg_pool2d_pytorch(dev: &Device) -> Result<()> { + let t = Tensor::new( + &[ + 0.4056f32, -0.8689, -0.0773, -1.5630, -2.8012, -1.5059, 0.3972, 1.0852, 0.4997, 3.0616, + 1.6541, 0.0964, -0.8338, -1.6523, -0.8323, -0.1699, 0.0823, 0.3526, 0.6843, 0.2395, + 1.2279, -0.9287, -1.7030, 0.1370, 0.6047, 0.3770, -0.6266, 0.3529, 2.2013, -0.6836, + 0.2477, 1.3127, + ], + dev, + )? + .reshape((1, 2, 4, 4))?; + let pool = t.avg_pool2d(2)?.squeeze(0)?; + assert_eq!( + test_utils::to_vec3_round(&pool, 4)?, + [ + [[-1.1926, -0.0395], [0.2688, 0.1871]], + [[0.1835, -0.1606], [0.6249, 0.3217]] + ] + ); + let pool = t.avg_pool2d(3)?.squeeze(0)?; + assert_eq!( + test_utils::to_vec3_round(&pool, 4)?, + [[[0.085]], [[0.0078]]] + ); + + let t = t.reshape((1, 1, 4, 8))?; + let pool = t.avg_pool2d(2)?.squeeze(0)?.squeeze(0)?; + assert_eq!( + test_utils::to_vec2_round(&pool, 4)?, + [ + [0.7745, 0.0276, -1.6983, 0.12], + [0.3542, 0.1625, 0.4542, -0.0014] + ] + ); + Ok(()) +} + +fn upsample_nearest2d(dev: &Device) -> Result<()> { + let t = Tensor::arange(0f32, 6f32, dev)?.reshape((1, 1, 2, 3))?; + let upsampled = t.upsample_nearest2d(4, 6)?.i(0)?.i(0)?; + assert_eq!( + t.i(0)?.i(0)?.to_vec2::()?, + [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]] + ); + assert_eq!( + upsampled.to_vec2::()?, + [ + [0.0, 0.0, 1.0, 1.0, 2.0, 2.0], + [0.0, 0.0, 1.0, 1.0, 2.0, 2.0], + [3.0, 3.0, 4.0, 4.0, 5.0, 5.0], + [3.0, 3.0, 4.0, 4.0, 5.0, 5.0] + ] + ); + Ok(()) +} + +test_device!(avg_pool2d, avg_pool2d_cpu, avg_pool2d_gpu, avg_pool2d_metal); +test_device!( + avg_pool2d_pytorch, + avg_pool2d_pytorch_cpu, + avg_pool2d_pytorch_gpu, + avg_pool2d_pytorch_metal +); +test_device!(max_pool2d, max_pool2d_cpu, max_pool2d_gpu, max_pool2d_metal); +test_device!( + upsample_nearest2d, + upsample_nearest2d_cpu, + upsample_nearest2d_gpu, + upsample_nearest2d_metal +); diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/pth.py b/v2/dependency-patches/candle-core-0.4.1/tests/pth.py new file mode 100644 index 0000000000..5c787c2036 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/tests/pth.py @@ -0,0 +1,37 @@ +import torch +from collections import OrderedDict + +# Write a trivial tensor to a pt file +a= torch.tensor([[1,2,3,4], [5,6,7,8]]) +o = OrderedDict() +o["test"] = a + +# Write a trivial tensor to a pt file +torch.save(o, "test.pt") + +############################################################################################################ +# Write a trivial tensor to a pt file with a key +torch.save({"model_state_dict": o}, "test_with_key.pt") + +############################################################################################################ +# Create a tensor with fortran contiguous memory layout +import numpy as np + +# Step 1: Create a 3D NumPy array with Fortran order using a range of numbers +# For example, creating a 2x3x4 array +array_fortran = np.asfortranarray(np.arange(1, 2*3*4 + 1).reshape(2, 3, 4)) + +# Verify the memory order +print("Is Fortran contiguous (F order):", array_fortran.flags['F_CONTIGUOUS']) # Should be True +print("Is C contiguous (C order):", array_fortran.flags['C_CONTIGUOUS']) # Should be False + +# Step 2: Convert the NumPy array to a PyTorch tensor +tensor_fortran = torch.from_numpy(array_fortran) + +# Verify the tensor layout +print("Tensor stride:", tensor_fortran.stride()) # Stride will reflect the Fortran memory layout + +# Step 3: Save the PyTorch tensor to a .pth file +torch.save({"tensor_fortran": tensor_fortran}, 'fortran_tensor_3d.pth') + +print("3D Tensor saved with Fortran layout.") diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/pth_tests.rs b/v2/dependency-patches/candle-core-0.4.1/tests/pth_tests.rs new file mode 100644 index 0000000000..9521f9a05d --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/tests/pth_tests.rs @@ -0,0 +1,31 @@ +/// Regression test for pth files not loading on Windows. +#[test] +fn test_pth() { + let tensors = candle_core::pickle::PthTensors::new("tests/test.pt", None).unwrap(); + tensors.get("test").unwrap().unwrap(); +} + +#[test] +fn test_pth_with_key() { + let tensors = + candle_core::pickle::PthTensors::new("tests/test_with_key.pt", Some("model_state_dict")) + .unwrap(); + tensors.get("test").unwrap().unwrap(); +} + +#[test] +fn test_pth_fortran_congiguous() { + let tensors = + candle_core::pickle::PthTensors::new("tests/fortran_tensor_3d.pth", None).unwrap(); + let tensor = tensors.get("tensor_fortran").unwrap().unwrap(); + + assert_eq!(tensor.dims3().unwrap(), (2, 3, 4)); + + assert_eq!( + tensor.to_vec3::().unwrap(), + [ + [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]], + [[13, 14, 15, 16], [17, 18, 19, 20], [21, 22, 23, 24]] + ] + ); +} diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/quantized_tests.rs b/v2/dependency-patches/candle-core-0.4.1/tests/quantized_tests.rs new file mode 100644 index 0000000000..5f7e482516 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/tests/quantized_tests.rs @@ -0,0 +1,1002 @@ +use candle_core::{ + bail, + quantized::{self, GgmlDType}, + test_device, + test_utils::to_vec2_round, + Device, Module, Result, Tensor, +}; +use quantized::{k_quants, GgmlType}; +use rand::prelude::*; + +const GGML_TEST_SIZE: usize = 32 * 128; + +const GGML_MAX_QUANTIZATION_TOTAL_ERROR: f32 = 0.002; +const GGML_MAX_QUANTIZATION_TOTAL_ERROR_2BITS: f32 = 0.0075; +const GGML_MAX_QUANTIZATION_TOTAL_ERROR_3BITS: f32 = 0.0040; +const GGML_MAX_DOT_PRODUCT_ERROR: f32 = 0.02; + +fn test_matmul( + device: &Device, + (b, m, n, k): (usize, usize, usize, usize), + dtype: GgmlDType, +) -> Result<()> { + let lhs = (0..(m * k)) + .map(|v| v as f32 / (m * k) as f32) + .collect::>(); + let rhs = (0..(k * n)) + .map(|v| v as f32 / (n * k) as f32) + .collect::>(); + + let lhs = Tensor::from_slice(&lhs, (m, k), device)?; + let rhs = Tensor::from_slice(&rhs, (k, n), device)?; + let mm = lhs.matmul(&rhs)?; + let qtensor = quantized::QTensor::quantize(&rhs.t()?, dtype)?; + let matmul = quantized::QMatMul::from_qtensor(qtensor)?; + let res = matmul.forward(&lhs)?; + + let error: f32 = ((&mm - &res)?.abs()? / &mm.abs()?)? + .sum_all()? + .to_scalar()?; + let error = error / (b * m * n) as f32; + assert!( + error <= 0.02, + "Error {error} is too big. \nExpected:\n {mm} \nFound:\n {res}\n for {dtype:?}" + ); + + Ok(()) +} + +fn quantized_matmul(device: &Device) -> Result<()> { + // TODO Enable this later when we enable cuda. + if device.is_cuda() { + return Ok(()); + } + let (m, k, n) = (3, 64, 4); + let lhs = (0..(m * k)).map(|v| v as f32).collect::>(); + let tensor_lhs = Tensor::from_slice(&lhs, (m, k), device)?; + let mut dst = vec![42.; 3 * 4]; + let mut rhs_t = vec![k_quants::BlockQ4_0::zeros(); 8]; + let rhs = (0..(k * n)).map(|v| v as f32).collect::>(); + k_quants::BlockQ4_0::from_float(&rhs, &mut rhs_t)?; + k_quants::matmul((m, k, n), &lhs, &rhs_t, &mut dst)?; + assert_eq!( + dst.iter().map(|x| x.round()).collect::>(), + &[ + 85120.0, 214562.0, 345455.0, 474748.0, 213475.0, 604465.0, 1000686.0, 1388317.0, + 341876.0, 994283.0, 1655709.0, 2301518.0 + ] + ); + let tensor_rhs = Tensor::from_slice(&rhs, (n, k), device)?.t()?; + let mm = tensor_lhs.matmul(&tensor_rhs)?; + assert_eq!( + mm.to_vec2::()?, + &[ + [85344.0, 214368.0, 343392.0, 472416.0], + [214368.0, 605536.0, 996704.0, 1387872.0], + [343392.0, 996704.0, 1650016.0, 2303328.0] + ] + ); + + let qtensor = quantized::QTensor::quantize(&tensor_rhs.t()?, GgmlDType::Q4_0)?; + let matmul = quantized::QMatMul::from_qtensor(qtensor)?; + let res = matmul.forward(&tensor_lhs)?; + match device { + Device::Metal(_) => assert_eq!( + to_vec2_round(&res, 0)?, + &[ + [84946.0, 214126.0, 344757.0, 473798.0], + [213458.0, 604350.0, 1000469.0, 1387990.0], + [341970.0, 994574.0, 1656181.0, 2302182.0] + ] + ), + _ => assert_eq!( + to_vec2_round(&res, 0)?, + &[ + [85120.0, 214562.0, 345455.0, 474748.0], + [213475.0, 604465.0, 1000686.0, 1388317.0], + [341876.0, 994283.0, 1655709.0, 2301518.0] + ] + ), + } + + test_matmul(device, (1, 3, 4, 256), GgmlDType::Q4_0)?; + + Ok(()) +} + +fn quantized_matmul_neg(device: &Device) -> Result<()> { + // TODO Enable this later when we enable cuda. + if device.is_cuda() { + return Ok(()); + } + let (m, k, n) = (3, 64, 4); + let lhs = (0..(m * k)) + .map(|v| v as f32 - (m * k) as f32 / 2.0) + .collect::>(); + let tensor_lhs = Tensor::from_slice(&lhs, (m, k), device)?; + let mut dst = vec![42.; 3 * 4]; + let mut rhs_t = vec![k_quants::BlockQ4_0::zeros(); 8]; + let rhs = (0..k * n) + .map(|v| v as f32 - (k * n) as f32 / 3.0) + .collect::>(); + let tensor_rhs = Tensor::from_slice(&rhs, (n, k), device)?.t()?; + k_quants::BlockQ4_0::from_float(&rhs, &mut rhs_t)?; + k_quants::matmul((m, k, n), &lhs, &rhs_t, &mut dst)?; + assert_eq!( + dst.iter().map(|x| x.round()).collect::>(), + &[ + 243524.0, -19596.0, -285051.0, -549815.0, 23777.0, 21651.0, 19398.0, 18367.0, + -196472.0, 63012.0, 324585.0, 587902.0 + ] + ); + let mm = tensor_lhs.matmul(&tensor_rhs)?; + assert_eq!( + to_vec2_round(&mm, 0)?, + &[ + [244064.0, -20128.0, -284320.0, -548512.0], + [23563.0, 21515.0, 19467.0, 17419.0], + [-196939.0, 63157.0, 323253.0, 583349.0] + ] + ); + + let qtensor = quantized::QTensor::quantize(&tensor_rhs.t()?, GgmlDType::Q4_0)?; + let matmul = quantized::QMatMul::from_qtensor(qtensor)?; + let res = matmul.forward(&tensor_lhs)?; + match device { + Device::Metal(_) => assert_eq!( + to_vec2_round(&res, 0)?, + &[ + [243666.0, -19714.0, -285433.0, -550453.0], + [23782.0, 21654.0, 19400.0, 18369.0], + [-196102.0, 63022.0, 324233.0, 587191.0] + ] + ), + _ => assert_eq!( + to_vec2_round(&res, 0)?, + &[ + [243524.0, -19596.0, -285051.0, -549815.0], + [23777.0, 21651.0, 19398.0, 18367.0], + [-196472.0, 63012.0, 324585.0, 587902.0] + ] + ), + } + + Ok(()) +} + +test_device!( + quantized_matmul, + quantized_matmul_cpu, + quantized_matmul_cuda, + quantized_matmul_metal +); +test_device!( + quantized_matmul_neg, + quantized_matmul_neg_cpu, + quantized_matmul_neg_cuda, + quantized_matmul_neg_metal +); + +fn quantize_q4_0(device: &Device) -> Result<()> { + let src = (0..32 * 4).map(|v| v as f32).collect::>(); + + let src = Tensor::from_slice(&src, (32 * 4,), device)?; + let quant = quantized::QTensor::quantize(&src, GgmlDType::Q4_0)?; + let dst = quant.dequantize(device)?; + assert_eq!( + dst.to_vec1::()?, + &[ + -0.0, -0.0, 3.875, 3.875, 3.875, 3.875, 7.75, 7.75, 7.75, 7.75, 11.625, 11.625, 11.625, + 11.625, 15.5, 15.5, 15.5, 15.5, 19.375, 19.375, 19.375, 19.375, 23.25, 23.25, 23.25, + 23.25, 27.125, 27.125, 27.125, 27.125, 31.0, 31.0, 31.5, 31.5, 31.5, 31.5, 39.375, + 39.375, 39.375, 39.375, 39.375, 39.375, 39.375, 39.375, 47.25, 47.25, 47.25, 47.25, + 47.25, 47.25, 47.25, 47.25, 55.125, 55.125, 55.125, 55.125, 55.125, 55.125, 55.125, + 55.125, 63.0, 63.0, 63.0, 63.0, 59.375, 59.375, 71.25, 71.25, 71.25, 71.25, 71.25, + 71.25, 71.25, 71.25, 71.25, 71.25, 71.25, 71.25, 83.125, 83.125, 83.125, 83.125, + 83.125, 83.125, 83.125, 83.125, 83.125, 83.125, 83.125, 83.125, 95.0, 95.0, 95.0, 95.0, + 95.0, 95.0, 95.25, 95.25, 95.25, 95.25, 95.25, 95.25, 95.25, 95.25, 111.125, 111.125, + 111.125, 111.125, 111.125, 111.125, 111.125, 111.125, 111.125, 111.125, 111.125, + 111.125, 111.125, 111.125, 111.125, 111.125, 127.0, 127.0, 127.0, 127.0, 127.0, 127.0, + 127.0, 127.0 + ] + ); + ggml_quantization_error_test(GgmlDType::Q4_0, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; + Ok(()) +} + +fn quantize_q4_1(device: &Device) -> Result<()> { + let src = (0..32 * 4).map(|v| v as f32).collect::>(); + let src = Tensor::from_slice(&src, (32 * 4,), device)?; + let quant = quantized::QTensor::quantize(&src, GgmlDType::Q4_1)?; + let dst = quant.dequantize(device)?; + assert_eq!( + round_vector(&dst.to_vec1::()?), + &[ + 0.0, 0.0, 2.066, 2.066, 4.133, 4.133, 6.199, 6.199, 8.266, 8.266, 10.332, 10.332, + 12.398, 12.398, 14.465, 14.465, 16.531, 16.531, 18.598, 18.598, 20.664, 20.664, 22.73, + 22.73, 24.797, 24.797, 26.863, 26.863, 28.93, 28.93, 30.996, 30.996, 32.0, 32.0, + 34.066, 34.066, 36.133, 36.133, 38.199, 38.199, 40.266, 40.266, 42.332, 42.332, 44.398, + 44.398, 46.465, 46.465, 48.531, 48.531, 50.598, 50.598, 52.664, 52.664, 54.73, 54.73, + 56.797, 56.797, 58.863, 58.863, 60.93, 60.93, 62.996, 62.996, 64.0, 64.0, 66.066, + 66.066, 68.133, 68.133, 70.199, 70.199, 72.266, 72.266, 74.332, 74.332, 76.398, 76.398, + 78.465, 78.465, 80.531, 80.531, 82.598, 82.598, 84.664, 84.664, 86.73, 86.73, 88.797, + 88.797, 90.863, 90.863, 92.93, 92.93, 94.996, 94.996, 96.0, 96.0, 98.066, 98.066, + 100.133, 100.133, 102.199, 102.199, 104.266, 104.266, 106.332, 106.332, 108.398, + 108.398, 110.465, 110.465, 112.531, 112.531, 114.598, 114.598, 116.664, 116.664, + 118.73, 118.73, 120.797, 120.797, 122.863, 122.863, 124.93, 124.93, 126.996, 126.996 + ] + ); + ggml_quantization_error_test(GgmlDType::Q4_1, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; + Ok(()) +} + +fn quantize_q5_0(device: &Device) -> Result<()> { + // TODO Enable this later when we enable cuda. + if device.is_cuda() { + return Ok(()); + } + let src = (0..32 * 4).map(|v| v as f32).collect::>(); + let src = Tensor::from_slice(&src, (32 * 4,), device)?; + let quant = quantized::QTensor::quantize(&src, GgmlDType::Q5_0)?; + let dst = quant.dequantize(device)?; + assert_eq!( + round_vector(&dst.to_vec1::()?), + &[ + -0.0, 1.938, 1.938, 3.875, 3.875, 5.813, 5.813, 7.75, 7.75, 9.688, 9.688, 11.625, + 11.625, 13.563, 13.563, 15.5, 15.5, 17.438, 17.438, 19.375, 19.375, 21.313, 21.313, + 23.25, 23.25, 25.188, 25.188, 27.125, 27.125, 29.063, 29.063, 31.0, 31.5, 31.5, 35.438, + 35.438, 35.438, 35.438, 39.375, 39.375, 39.375, 39.375, 43.313, 43.313, 43.313, 43.313, + 47.25, 47.25, 47.25, 47.25, 51.188, 51.188, 51.188, 51.188, 55.125, 55.125, 55.125, + 55.125, 59.063, 59.063, 59.063, 59.063, 63.0, 63.0, 65.313, 65.313, 65.313, 65.313, + 65.313, 71.25, 71.25, 71.25, 71.25, 71.25, 71.25, 77.188, 77.188, 77.188, 77.188, + 77.188, 77.188, 83.125, 83.125, 83.125, 83.125, 83.125, 83.125, 89.063, 89.063, 89.063, + 89.063, 89.063, 89.063, 95.0, 95.0, 95.0, 95.25, 95.25, 95.25, 95.25, 103.188, 103.188, + 103.188, 103.188, 103.188, 103.188, 103.188, 103.188, 111.125, 111.125, 111.125, + 111.125, 111.125, 111.125, 111.125, 111.125, 119.063, 119.063, 119.063, 119.063, + 119.063, 119.063, 119.063, 119.063, 127.0, 127.0, 127.0, 127.0 + ] + ); + ggml_quantization_error_test(GgmlDType::Q5_0, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; + Ok(()) +} + +fn quantize_q5_1(device: &Device) -> Result<()> { + // TODO Enable this later when we enable cuda. + if device.is_cuda() { + return Ok(()); + } + let src = (0..32 * 4).map(|v| v as f32).collect::>(); + let src = Tensor::from_slice(&src, (32 * 4,), device)?; + let quant = quantized::QTensor::quantize(&src, GgmlDType::Q5_1)?; + let dst = quant.dequantize(device)?; + assert_eq!( + round_vector(&dst.to_vec1::()?), + &[ + 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, + 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, + 30.0, 31.0, 32.0, 33.0, 34.0, 35.0, 36.0, 37.0, 38.0, 39.0, 40.0, 41.0, 42.0, 43.0, + 44.0, 45.0, 46.0, 47.0, 48.0, 49.0, 50.0, 51.0, 52.0, 53.0, 54.0, 55.0, 56.0, 57.0, + 58.0, 59.0, 60.0, 61.0, 62.0, 63.0, 64.0, 65.0, 66.0, 67.0, 68.0, 69.0, 70.0, 71.0, + 72.0, 73.0, 74.0, 75.0, 76.0, 77.0, 78.0, 79.0, 80.0, 81.0, 82.0, 83.0, 84.0, 85.0, + 86.0, 87.0, 88.0, 89.0, 90.0, 91.0, 92.0, 93.0, 94.0, 95.0, 96.0, 97.0, 98.0, 99.0, + 100.0, 101.0, 102.0, 103.0, 104.0, 105.0, 106.0, 107.0, 108.0, 109.0, 110.0, 111.0, + 112.0, 113.0, 114.0, 115.0, 116.0, 117.0, 118.0, 119.0, 120.0, 121.0, 122.0, 123.0, + 124.0, 125.0, 126.0, 127.0 + ] + ); + ggml_quantization_error_test(GgmlDType::Q5_1, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; + Ok(()) +} + +fn get_test_vector2(bound: f32, size: usize, device: &Device) -> Result { + assert!( + size % crate::quantized::k_quants::QK_K == 0, + "size must be a multiple of {}", + crate::quantized::k_quants::QK_K + ); + + let src = (0..size) + .map(|v| (v as f32 - size as f32 / 2.) * bound / (size as f32 / 2.)) + .collect::>(); + assert_eq!([src[0], src[size / 2]], [-bound, 0.0]); + Tensor::from_vec(src, (size,), device) +} + +/// Round a vector +fn round_vector(values: &[f32]) -> Vec { + values + .iter() + .map(|x| (1000. * x).round() / 1000.) + .collect::>() +} + +fn compare_with_error(values: &[f32], expected: &[f32], tolerance: f32) { + for (i, (value, expected_value)) in values.iter().zip(expected.iter()).enumerate() { + let difference = (value - expected_value).abs(); + + assert!( + difference < tolerance, + "Error at index {}: value = {}, expected = {}. Difference = {} exceeds tolerance = {}.", + i, + value, + expected_value, + difference, + tolerance + ); + } +} + +/// Creates a vector similar to the ones used in GGML unit tests: +/// https://github.com/ggerganov/llama.cpp/blob/master/tests/test-quantize-fns.cpp#L26-L30 +fn create_ggml_like_vector(offset: f32) -> Vec { + (0..GGML_TEST_SIZE) + .map(|i| 0.1 + 2.0 * (i as f32 + offset).cos()) + .collect() +} + +/// Calculates the root mean square error between two vectors +fn calculate_rmse(a: &[f32], b: &[f32]) -> f32 { + assert_eq!(a.len(), b.len()); + let sum = a + .iter() + .zip(b) + .map(|(a, b)| (a - b).powi(2)) + .sum::() + .sqrt(); + sum / a.len() as f32 +} + +/// Similar to the GGML quantization unit test: +/// https://github.com/ggerganov/llama.cpp/blob/master/tests/test-quantize-fns.cpp#L43-L50 +fn ggml_quantization_error_test(dtype: GgmlDType, device: &Device, max_error: f32) -> Result<()> { + let src = create_ggml_like_vector(0.0); + let src = Tensor::from_slice(&src, (GGML_TEST_SIZE,), device)?; + let quant = quantized::QTensor::quantize(&src, dtype)?; + let dst = quant.dequantize(device)?; + let error = calculate_rmse(&src.to_vec1::()?, &dst.to_vec1::()?); + if error > max_error { + bail!( + "Quantization error {} exceeds max error {}", + error, + max_error + ); + } + Ok(()) +} + +fn quantize_q2k(device: &Device) -> Result<()> { + let dtype = GgmlDType::Q2K; + + let src = get_test_vector2(0.5, 1024, device)?; + let quant = quantized::QTensor::quantize(&src, dtype)?; + let dst = quant.dequantize(device)?; + + let src = src.to_vec1::()?; + let dst = dst.to_vec1::()?; + compare_with_error(dst.as_slice(), src.as_slice(), 0.1); + + // Test some specific values + assert_eq!( + [src[0], src[128], src[256], src[512], src[800], src[1023]], + [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] + ); + let dst = round_vector(&dst); + assert_eq!( + [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], + [-0.499, -0.366, -0.249, 0.0, 0.295, 0.492] + ); + + let src_big = get_test_vector2(128.0, 1024, device)?; + let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; + let dst_big = quant_big.dequantize(device)?; + + let src_big = src_big.to_vec1::()?; + let dst_big = dst_big.to_vec1::()?; + compare_with_error(dst_big.as_slice(), src_big.as_slice(), 6.0); + + ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR_2BITS)?; + Ok(()) +} + +fn quantize_q3k(device: &Device) -> Result<()> { + let dtype = GgmlDType::Q3K; + let src = get_test_vector2(0.5, 1024, device)?; + let quant = quantized::QTensor::quantize(&src, dtype)?; + let dst = quant.dequantize(device)?; + + let src = src.to_vec1::()?; + let dst = dst.to_vec1::()?; + compare_with_error(dst.as_slice(), src.as_slice(), 0.03); + + // Test some specific values + assert_eq!( + [src[0], src[128], src[256], src[512], src[800], src[1023]], + [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] + ); + let dst = round_vector(&dst); + assert_eq!( + [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], + [-0.493, -0.37, -0.243, -0.0, 0.292, 0.492] + ); + + let src_big = get_test_vector2(128.0, 1024, device)?; + let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; + let dst_big = quant_big.dequantize(device)?; + + let src_big = src_big.to_vec1::()?; + let dst_big = dst_big.to_vec1::()?; + compare_with_error(dst_big.as_slice(), src_big.as_slice(), 3.5); + + ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR_3BITS)?; + Ok(()) +} + +fn quantize_q4k(device: &Device) -> Result<()> { + let dtype = GgmlDType::Q4K; + let src = get_test_vector2(0.5, 1024, device)?; + let quant = quantized::QTensor::quantize(&src, dtype)?; + let dst = quant.dequantize(device)?; + + let src = src.to_vec1::()?; + let dst = dst.to_vec1::()?; + compare_with_error(dst.as_slice(), src.as_slice(), 0.017); + + // Test some specific values + assert_eq!( + [src[0], src[128], src[256], src[512], src[800], src[1023]], + [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] + ); + let dst = round_vector(&dst); + assert_eq!( + [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], + [-0.5, -0.373, -0.25, 0.0, 0.288, 0.498] + ); + + let src_big = get_test_vector2(128.0, 1024, device)?; + let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; + let dst_big = quant_big.dequantize(device)?; + + let src_big = src_big.to_vec1::()?; + let dst_big = dst_big.to_vec1::()?; + compare_with_error(dst_big.as_slice(), src_big.as_slice(), 4.5); + + ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; + Ok(()) +} + +fn quantize_q5k(device: &Device) -> Result<()> { + let dtype = GgmlDType::Q5K; + let src = get_test_vector2(0.5, 1024, device)?; + let quant = quantized::QTensor::quantize(&src, dtype)?; + let dst = quant.dequantize(device)?; + + let src = src.to_vec1::()?; + let dst = dst.to_vec1::()?; + compare_with_error(dst.as_slice(), src.as_slice(), 0.009); + + // Test some specific values + assert_eq!( + [src[0], src[128], src[256], src[512], src[800], src[1023]], + [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] + ); + let dst = round_vector(&dst); + assert_eq!( + [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], + [-0.5, -0.373, -0.25, 0.0, 0.279, 0.499] + ); + + let src_big = get_test_vector2(128.0, 1024, device)?; + let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; + let dst_big = quant_big.dequantize(device)?; + + let src_big = src_big.to_vec1::()?; + let dst_big = dst_big.to_vec1::()?; + compare_with_error(dst_big.as_slice(), src_big.as_slice(), 2.5); + + ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; + Ok(()) +} + +fn quantize_q6k(device: &Device) -> Result<()> { + let dtype = GgmlDType::Q6K; + let src = get_test_vector2(0.5, 1024, device)?; + let quant = quantized::QTensor::quantize(&src, dtype)?; + let dst = quant.dequantize(device)?; + + let src = src.to_vec1::()?; + let dst = dst.to_vec1::()?; + compare_with_error(dst.as_slice(), src.as_slice(), 0.008); + + // Test some specific values + assert_eq!( + [src[0], src[128], src[256], src[512], src[800], src[1023]], + [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] + ); + let dst = round_vector(&dst); + assert_eq!( + [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], + [-0.497, -0.372, -0.25, -0.0, 0.284, 0.5] + ); + + let src_big = get_test_vector2(128.0, 1024, device)?; + let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; + let dst_big = quant_big.dequantize(device)?; + + let src_big = src_big.to_vec1::()?; + let dst_big = dst_big.to_vec1::()?; + compare_with_error(dst_big.as_slice(), src_big.as_slice(), 2.0); + + ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; + Ok(()) +} + +fn quantize_q8k(device: &Device) -> Result<()> { + let dtype = GgmlDType::Q8K; + let src = get_test_vector2(0.5, 1024, device)?; + let quant = quantized::QTensor::quantize(&src, dtype)?; + let dst = quant.dequantize(device)?; + + let src = src.to_vec1::()?; + let dst = dst.to_vec1::()?; + compare_with_error(dst.as_slice(), src.as_slice(), 0.008); + + // Test some specific values + assert_eq!( + [src[0], src[128], src[256], src[512], src[800], src[1023]], + [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] + ); + let dst = round_vector(&dst); + assert_eq!( + [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], + [-0.5, -0.375, -0.25, -0.0, 0.281, 0.499] + ); + + let src_big = get_test_vector2(128.0, 1024, device)?; + let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; + let dst_big = quant_big.dequantize(device)?; + + let src_big = src_big.to_vec1::()?; + let dst_big = dst_big.to_vec1::()?; + compare_with_error(dst_big.as_slice(), src_big.as_slice(), 0.6); + + ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; + Ok(()) +} + +test_device!( + quantize_q4_0, + quantize_q4_0_cpu, + quantize_q4_0_cuda, + quantize_q4_0_metal +); +test_device!( + quantize_q4_1, + quantize_q4_1_cpu, + quantize_q4_1_cuda, + quantize_q4_1_metal +); +test_device!( + quantize_q5_0, + quantize_q5_0_cpu, + quantize_q5_0_cuda, + quantize_q5_0_metal +); +test_device!( + quantize_q5_1, + quantize_q5_1_cpu, + quantize_q5_1_cuda, + quantize_q5_1_metal +); +test_device!( + quantize_q2k, + quantize_q2k_cpu, + quantize_q2k_cuda, + quantize_q2k_metal +); +test_device!( + quantize_q3k, + quantize_q3k_cpu, + quantize_q3k_cuda, + quantize_q3k_metal +); +test_device!( + quantize_q4k, + quantize_q4k_cpu, + quantize_q4k_cuda, + quantize_q4k_metal +); +test_device!( + quantize_q5k, + quantize_q5k_cpu, + quantize_q5k_cuda, + quantize_q5k_metal +); +test_device!( + quantize_q6k, + quantize_q6k_cpu, + quantize_q6k_cuda, + quantize_q6k_metal +); +test_device!( + quantize_q8k, + quantize_q8k_cpu, + quantize_q8k_cuda, + quantize_q8k_metal +); + +/// Very simple dot product implementation +fn vec_dot_reference(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b).map(|(a, b)| a * b).sum() +} + +/// Returns the error achieved by the GGML matmul unit test. +fn ggml_reference_matmul_error(dtype: GgmlDType) -> Result { + let err = match dtype { + GgmlDType::F16 => 0.000010, + GgmlDType::Q2K => 0.004086, + GgmlDType::Q3K => 0.016148, + GgmlDType::Q4K => 0.002425, + GgmlDType::Q5K => 0.000740, + GgmlDType::Q6K => 0.000952, + GgmlDType::Q4_0 => 0.001143, + GgmlDType::Q4_1 => 0.008, + GgmlDType::Q5_0 => 0.001353, + GgmlDType::Q5_1 => 0.00149, + GgmlDType::Q8_0 => 0.000092, + + // Not from the ggml repo. + GgmlDType::Q8K => 0.00065, + _ => bail!("No GGML results for quantization type {dtype:?}",), + }; + Ok(err) +} + +/// Similar to the GGML matmul unit test: +/// https://github.com/ggerganov/llama.cpp/blob/master/tests/test-quantize-fns.cpp#L76-L91 +fn ggml_matmul_error_test() -> Result<()> { + let a = create_ggml_like_vector(0.0); + let b = create_ggml_like_vector(1.0); + ggml_matmul_error_test_::(a.as_slice(), b.as_slice(), 1.0)?; + // Another example that is more likely to trigger the overflow reported in #1526 + let a = (0..GGML_TEST_SIZE) + .map(|i| i as f32 / GGML_TEST_SIZE as f32) + .collect::>(); + let b = (0..GGML_TEST_SIZE) + .map(|i| i as f32 / GGML_TEST_SIZE as f32) + .collect::>(); + ggml_matmul_error_test_::(a.as_slice(), b.as_slice(), 2.0)?; + Ok(()) +} + +fn ggml_matmul_error_test_(a: &[f32], b: &[f32], err_m: f32) -> Result<()> { + let length = a.len(); + + let mut a_quant = vec![T::zeros(); length / T::BLCK_SIZE]; + let mut b_quant = vec![T::VecDotType::zeros(); length / T::VecDotType::BLCK_SIZE]; + T::from_float(a, &mut a_quant)?; + T::VecDotType::from_float(b, &mut b_quant)?; + + let result = T::vec_dot(length, &a_quant, &b_quant)?; + let result_unopt = T::vec_dot_unopt(length, &a_quant, &b_quant)?; + let reference_result = vec_dot_reference(a, b); + + if (result - result_unopt).abs() / length as f32 > 1e-6 { + bail!( + "the opt and unopt vec-dot returned different values, opt {result}, unopt {result_unopt}" + ) + } + + let error = (result - reference_result).abs() / length as f32; + + let ggml_error = ggml_reference_matmul_error(T::DTYPE)? * err_m; + + if !error.is_finite() || error > GGML_MAX_DOT_PRODUCT_ERROR { + bail!("Dot product error {error} exceeds max error {GGML_MAX_DOT_PRODUCT_ERROR}",); + } + + // We diverge slightly due to different rounding behavior / f16 to f32 conversions in GGML + // => we use a slightly higher error threshold + const ERROR_LENIENCY: f32 = 0.00001; + if error - ERROR_LENIENCY > ggml_error { + bail!( + "Dot product error {} exceeds ggml reference error {}", + error, + ggml_error + ); + } + Ok(()) +} + +#[test] +fn quantized_mm() -> Result<()> { + ggml_matmul_error_test::()?; + ggml_matmul_error_test::()?; + ggml_matmul_error_test::()?; + ggml_matmul_error_test::()?; + ggml_matmul_error_test::()?; + Ok(()) +} + +/// generates random tensors of size `m x k` and `n x k` and calculates their expected matrix multiplication result. +fn get_random_tensors( + m: usize, + k: usize, + n: usize, + device: &Device, +) -> Result<(Tensor, Tensor, Tensor)> { + let mut rng = StdRng::seed_from_u64(314159265358979); + + let lhs = (0..m * k) + .map(|_| rng.gen::() - 0.5) + .collect::>(); + let rhs = (0..n * k) + .map(|_| rng.gen::() - 0.5) + .collect::>(); + + let lhs = Tensor::from_vec(lhs, (m, k), device)?; + let rhs = Tensor::from_vec(rhs, (n, k), device)?; + + let mm = lhs.matmul(&rhs.t()?)?; + Ok((lhs, rhs, mm)) +} + +#[macro_export] +macro_rules! quantized_matmul { + // TODO: Switch to generating the two last arguments automatically once concat_idents is + // stable. https://github.com/rust-lang/rust/issues/29599 + ($fn_name: ident, $fn_name_cpu: ident, $fn_name_cuda: ident, $fn_name_metal: ident, $dtype: expr) => { + fn $fn_name(device: &Device) -> Result<()> { + if device.is_cuda() { + // TODO Enable Cuda GGML sometime maybe. + return Ok(()); + } + test_matmul(device, (1, 3, 4, 256), $dtype)?; + Ok(()) + } + + test_device!($fn_name, $fn_name_cpu, $fn_name_cuda, $fn_name_metal); + }; +} + +quantized_matmul!( + quantized_matmul_q4_0_bis, + quantized_matmul_q4_0_cpu, + quantized_matmul_q4_0_cuda, + quantized_matmul_q4_0_metal, + GgmlDType::Q4_0 +); +quantized_matmul!( + quantized_matmul_q4_1_bis, + quantized_matmul_q4_1_cpu, + quantized_matmul_q4_1_cuda, + quantized_matmul_q4_1_metal, + GgmlDType::Q4_1 +); +quantized_matmul!( + quantized_matmul_q5_0_bis, + quantized_matmul_q5_0_cpu, + quantized_matmul_q5_0_cuda, + quantized_matmul_q5_0_metal, + GgmlDType::Q5_0 +); +quantized_matmul!( + quantized_matmul_q5_1_bis, + quantized_matmul_q5_1_cpu, + quantized_matmul_q5_1_cuda, + quantized_matmul_q5_1_metal, + GgmlDType::Q5_1 +); +quantized_matmul!( + quantized_matmul_q8_0_bis, + quantized_matmul_q8_0_cpu, + quantized_matmul_q8_0_cuda, + quantized_matmul_q8_0_metal, + GgmlDType::Q8_0 +); +// Not implemented in Ggml +// quantized_matmul!( +// quantized_matmul_q8_1_bis, +// quantized_matmul_q8_1_cpu, +// quantized_matmul_q8_1_cuda, +// quantized_matmul_q8_1_metal, +// GgmlDType::Q8_1 +// ); +// TODO This is bugged (also bugged in GGML +quantized_matmul!( + quantized_matmul_q2k_bis, + quantized_matmul_q2k_cpu, + quantized_matmul_q2k_cuda, + quantized_matmul_q2k_metal, + GgmlDType::Q2K +); +quantized_matmul!( + quantized_matmul_q3k_bis, + quantized_matmul_q3k_cpu, + quantized_matmul_q3k_cuda, + quantized_matmul_q3k_metal, + GgmlDType::Q3K +); +quantized_matmul!( + quantized_matmul_q4k_bis, + quantized_matmul_q4k_cpu, + quantized_matmul_q4k_cuda, + quantized_matmul_q4k_metal, + GgmlDType::Q4K +); +quantized_matmul!( + quantized_matmul_q5k_bis, + quantized_matmul_q5k_cpu, + quantized_matmul_q5k_cuda, + quantized_matmul_q5k_metal, + GgmlDType::Q5K +); +quantized_matmul!( + quantized_matmul_q6k_bis, + quantized_matmul_q6k_cpu, + quantized_matmul_q6k_cuda, + quantized_matmul_q6k_metal, + GgmlDType::Q6K +); +// Not implemented on metal +// quantized_matmul!( +// quantized_matmul_q8k_bis, +// quantized_matmul_q8k_cpu, +// quantized_matmul_q8k_cuda, +// quantized_matmul_q8k_metal, +// GgmlDType::Q8K +// ); + +#[test] +fn quantized_matmul_q2k() -> Result<()> { + use k_quants::BlockQ2K; + + let cpu = &Device::Cpu; + let (m, k, n) = (11, 512, 21); + let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); + + let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q2K)?; + let rhs = quantized::QMatMul::from_qtensor(rhs)?; + let mm = rhs.forward(&lhs)?; + + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [0.916, 0.422, 0.215, 1.668]); + + ggml_matmul_error_test::()?; + + Ok(()) +} + +#[test] +fn quantized_matmul_q3k() -> Result<()> { + use k_quants::BlockQ3K; + + let cpu = &Device::Cpu; + let (m, k, n) = (11, 512, 21); + let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); + + let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q3K)?; + let rhs = quantized::QMatMul::from_qtensor(rhs)?; + let mm = rhs.forward(&lhs)?; + + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [1.029, 1.418, -0.314, 1.495]); + + ggml_matmul_error_test::()?; + + Ok(()) +} + +#[test] +fn quantized_matmul_q4k() -> Result<()> { + use k_quants::BlockQ4K; + + let cpu = &Device::Cpu; + let (m, k, n) = (11, 512, 21); + let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); + + let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q4K)?; + let rhs = quantized::QMatMul::from_qtensor(rhs)?; + let mm = rhs.forward(&lhs)?; + + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [1.125, 1.435, -0.201, 1.589]); + + ggml_matmul_error_test::()?; + + Ok(()) +} + +#[test] +fn quantized_matmul_q5k() -> Result<()> { + use k_quants::BlockQ5K; + + let cpu = &Device::Cpu; + let (m, k, n) = (11, 512, 21); + let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); + + let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q5K)?; + let rhs = quantized::QMatMul::from_qtensor(rhs)?; + let mm = rhs.forward(&lhs)?; + + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [1.192, 1.491, -0.18, 1.743]); + + //Expected: 0.000740408897 + ggml_matmul_error_test::()?; + + Ok(()) +} + +#[test] +fn quantized_matmul_q6k() -> Result<()> { + use k_quants::BlockQ6K; + + let cpu = &Device::Cpu; + let (m, k, n) = (11, 512, 21); + let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); + + let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q6K)?; + let rhs = quantized::QMatMul::from_qtensor(rhs)?; + let mm = rhs.forward(&lhs)?; + + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [1.324, 1.49, -0.164, 1.741]); + + ggml_matmul_error_test::()?; + Ok(()) +} + +#[test] +fn quantized_matmul_q8k() -> Result<()> { + use k_quants::BlockQ8K; + + let cpu = &Device::Cpu; + let (m, k, n) = (11, 512, 21); + let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); + + let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q8K)?; + let rhs = quantized::QMatMul::from_qtensor(rhs)?; + let mm = rhs.forward(&lhs)?; + + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [1.266, 1.504, -0.204, 1.7]); + + ggml_matmul_error_test::()?; + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/serialization_tests.rs b/v2/dependency-patches/candle-core-0.4.1/tests/serialization_tests.rs new file mode 100644 index 0000000000..415306f444 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/tests/serialization_tests.rs @@ -0,0 +1,24 @@ +use candle_core::{DType, Result, Tensor}; + +#[test] +fn npy() -> Result<()> { + let npy = Tensor::read_npy("tests/test.npy")?; + assert_eq!( + npy.to_dtype(DType::U8)?.to_vec1::()?, + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + ); + Ok(()) +} + +#[test] +fn npz() -> Result<()> { + let npz = Tensor::read_npz("tests/test.npz")?; + assert_eq!(npz.len(), 2); + assert_eq!(npz[0].0, "x"); + assert_eq!(npz[1].0, "x_plus_one"); + assert_eq!( + npz[1].1.to_dtype(DType::U8)?.to_vec1::()?, + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + ); + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/tensor_tests.rs b/v2/dependency-patches/candle-core-0.4.1/tests/tensor_tests.rs new file mode 100644 index 0000000000..40737e7b4d --- /dev/null +++ b/v2/dependency-patches/candle-core-0.4.1/tests/tensor_tests.rs @@ -0,0 +1,1274 @@ +use candle_core::{test_device, test_utils, DType, Device, IndexOp, Result, Tensor, D}; + +fn zeros(device: &Device) -> Result<()> { + let tensor = Tensor::zeros((5, 2), DType::F32, device)?; + let (dim1, dim2) = tensor.dims2()?; + assert_eq!(dim1, 5); + assert_eq!(dim2, 2); + Ok(()) +} + +fn ones(device: &Device) -> Result<()> { + assert_eq!( + Tensor::ones((2, 3), DType::U8, device)?.to_vec2::()?, + [[1, 1, 1], [1, 1, 1]], + ); + assert_eq!( + Tensor::ones((2, 3), DType::U32, device)?.to_vec2::()?, + [[1, 1, 1], [1, 1, 1]], + ); + assert_eq!( + Tensor::ones((2, 3), DType::I64, device)?.to_vec2::()?, + [[1, 1, 1], [1, 1, 1]], + ); + assert_eq!( + Tensor::ones((2, 3), DType::F32, device)?.to_vec2::()?, + [[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]], + ); + assert_eq!( + Tensor::ones((2, 3), DType::F64, device)?.to_vec2::()?, + [[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]], + ); + Ok(()) +} + +fn full(device: &Device) -> Result<()> { + assert_eq!( + Tensor::full(42u32, (2, 3), device)?.to_vec2::()?, + [[42, 42, 42], [42, 42, 42]], + ); + Ok(()) +} + +fn arange(device: &Device) -> Result<()> { + assert_eq!( + Tensor::arange(0u8, 5u8, device)?.to_vec1::()?, + [0, 1, 2, 3, 4], + ); + assert_eq!( + Tensor::arange_step(0u8, 5u8, 2, device)?.to_vec1::()?, + [0, 2, 4], + ); + assert_eq!( + Tensor::arange_step(0u8, 5u8, 3, device)?.to_vec1::()?, + [0, 3], + ); + assert_eq!( + Tensor::arange_step(5i64, 0i64, -1, device)?.to_vec1::()?, + [5, 4, 3, 2, 1], + ); + Ok(()) +} + +fn add_mul(device: &Device) -> Result<()> { + let tensor = Tensor::new(&[3f32, 1., 4.], device)?; + let dim1 = tensor.dims1()?; + assert_eq!(dim1, 3); + let content: Vec = tensor.to_vec1()?; + assert_eq!(content, [3., 1., 4.]); + let tensor = Tensor::add(&tensor, &tensor)?; + let content: Vec = tensor.to_vec1()?; + assert_eq!(content, [6., 2., 8.]); + let tensor = Tensor::mul(&tensor, &tensor)?; + let content: Vec = tensor.to_vec1()?; + assert_eq!(content, [36., 4., 64.]); + Ok(()) +} + +fn tensor_2d(device: &Device) -> Result<()> { + let data = &[[3f32, 1., 4., 1., 5.], [2., 1., 7., 8., 2.]]; + let tensor = Tensor::new(data, device)?; + let dims = tensor.dims2()?; + assert_eq!(dims, (2, 5)); + let content: Vec> = tensor.to_vec2()?; + assert_eq!(content, data); + Ok(()) +} + +fn clamp(device: &Device) -> Result<()> { + let data = &[[3f32, 1., 4., 1., 5.], [2., 1., 7., 8., 2.]]; + let tensor = Tensor::new(data, device)?; + let tensor = tensor.clamp(1.5, 6.2)?; + assert_eq!( + tensor.to_vec2::()?, + [[3.0, 1.5, 4.0, 1.5, 5.0], [2.0, 1.5, 6.2, 6.2, 2.0]], + ); + Ok(()) +} + +fn unary_op(device: &Device) -> Result<()> { + let data = &[[-3f32, 1., 4., -0.1, 0.5], [2.7, -1.8, -0.28, 1.8, 2.8]]; + let tensor = Tensor::new(data, device)?; + assert_eq!( + test_utils::to_vec2_round(&tensor.gelu()?, 4)?, + [ + [-0.0036, 0.8412, 3.9999, -0.046, 0.3457], + [2.6911, -0.0647, -0.1091, 1.7353, 2.7933] + ] + ); + assert_eq!( + test_utils::to_vec2_round(&tensor.gelu_erf()?, 4)?, + [ + [-0.004, 0.8413, 3.9999, -0.046, 0.3457], + [2.6906, -0.0647, -0.1091, 1.7353, 2.7928] + ] + ); + assert_eq!( + test_utils::to_vec2_round(&tensor.erf()?, 4)?, + [ + [-1.0, 0.8427, 1.0, -0.1125, 0.5205], + [0.9999, -0.9891, -0.3079, 0.9891, 0.9999] + ] + ); + assert_eq!( + test_utils::to_vec2_round(&tensor.silu()?, 4)?, + [ + [-0.1423, 0.7311, 3.9281, -0.0475, 0.3112], + [2.53, -0.2553, -0.1205, 1.5447, 2.6395] + ] + ); + assert_eq!( + test_utils::to_vec2_round(&tensor.ceil()?, 4)?, + [[-3.0, 1.0, 4.0, -0.0, 1.0], [3.0, -1.0, -0.0, 2.0, 3.0]] + ); + assert_eq!( + test_utils::to_vec2_round(&tensor.floor()?, 4)?, + [[-3.0, 1.0, 4.0, -1.0, 0.0], [2.0, -2.0, -1.0, 1.0, 2.0]] + ); + assert_eq!( + test_utils::to_vec2_round(&tensor.round()?, 4)?, + [[-3.0, 1.0, 4.0, -0.0, 1.0], [3.0, -2.0, -0.0, 2.0, 3.0]] + ); + let tensor = Tensor::new(&[2997.9246, 314.15926f32], device)?; + assert_eq!( + test_utils::to_vec1_round(&tensor.round_to(2)?, 4)?, + [2997.92, 314.16] + ); + assert_eq!( + test_utils::to_vec1_round(&tensor.round_to(-2)?, 4)?, + [3000.0, 300.] + ); + Ok(()) +} + +fn binary_op(device: &Device) -> Result<()> { + let data = &[[3f32, 1., 4., 1., 5.], [2., 1., 7., 8., 2.]]; + let tensor1 = Tensor::new(data, device)?; + let data2 = &[[5f32, 5., 5., 5., 5.], [2., 1., 7., 8., 2.]]; + let tensor2 = Tensor::new(data2, device)?; + let tensor = (&tensor1 + (&tensor1 * &tensor1)? / (&tensor1 + &tensor2))?; + let dims = tensor.dims2()?; + assert_eq!(dims, (2, 5)); + let content: Vec> = tensor.to_vec2()?; + assert_eq!(content[0], [4.125, 1.1666666, 5.7777777, 1.1666666, 7.5]); + assert_eq!(content[1], [3.0, 1.5, 10.5, 12.0, 3.0]); + #[allow(clippy::eq_op)] + let tensor = (&tensor - &tensor)?; + let content: Vec> = tensor.to_vec2()?; + assert_eq!(content[0], [0., 0., 0., 0., 0.]); + + let min = tensor1.minimum(&(&tensor2 * 0.5)?)?; + let max = tensor1.maximum(&(&tensor2 * 0.5)?)?; + assert_eq!( + min.to_vec2::()?, + [[2.5, 1.0, 2.5, 1.0, 2.5], [1.0, 0.5, 3.5, 4.0, 1.0]], + ); + assert_eq!( + max.to_vec2::()?, + [[3.0, 2.5, 4.0, 2.5, 5.0], [2.0, 1.0, 7.0, 8.0, 2.0]] + ); + Ok(()) +} + +fn transpose(device: &Device) -> Result<()> { + let data = &[[3f32, 1., 4., 1., 5.], [2., 1., 7., 8., 2.]]; + let tensor = Tensor::new(data, device)?.t()?; + let dims = tensor.dims2()?; + assert_eq!(dims, (5, 2)); + assert_eq!( + tensor.to_vec2::()?, + &[[3f32, 2.], [1., 1.], [4., 7.], [1., 8.], [5., 2.]] + ); + assert_eq!(tensor.t()?.to_vec2::()?, data); + assert_eq!(tensor.contiguous()?.t()?.to_vec2::()?, data); + assert_eq!(((tensor + 1.)?.t()? - 1.)?.to_vec2::()?, data); + Ok(()) +} + +fn var(device: &Device) -> Result<()> { + // Values taken from https://pytorch.org/docs/stable/generated/torch.var.html + let data = &[ + [0.2035f32, 1.2959, 1.8101, -0.4644], + [1.5027, -0.3270, 0.5905, 0.6538], + [-1.5745, 1.3330, -0.5596, -0.6548], + [0.1264, -0.5080, 1.6420, 0.1992], + ]; + let tensor = Tensor::new(data, device)?; + assert_eq!( + test_utils::to_vec2_round(&tensor.var_keepdim(1)?, 4)?, + &[[1.0631], [0.559], [1.4893], [0.8258]] + ); + Ok(()) +} + +fn sum(device: &Device) -> Result<()> { + let data = &[[[3u32, 1, 4], [1, 5, 9]], [[2, 1, 7], [8, 2, 8]]]; + let tensor = Tensor::new(data, device)?; + assert_eq!( + tensor.sum_keepdim(2)?.to_vec3::()?, + &[[[8], [15]], [[10], [18]]] + ); + assert_eq!( + tensor.sum_keepdim(0)?.to_vec3::()?, + &[[[5, 2, 11], [9, 7, 17]]], + ); + assert_eq!(tensor.sum_keepdim((0, 2, 1))?.to_vec3::()?, &[[[51]]],); + assert_eq!( + tensor.t()?.sum_keepdim(1)?.t()?.to_vec3::()?, + &[[[8], [15]], [[10], [18]]] + ); + assert_eq!( + tensor.sum_keepdim((2, 1))?.to_vec3::()?, + &[[[8 + 15]], [[10 + 18]]] + ); + let data: Vec = (0..4000u32).collect(); + let tensor = Tensor::new(data.as_slice(), device)?; + assert_eq!(tensor.sum_keepdim(0)?.to_vec1::()?, &[7998000]); + let tensor = tensor.reshape((2000, 2))?; + assert_eq!(tensor.sum_keepdim((0, 1))?.to_vec2::()?, &[[7998000]]); + assert_eq!( + tensor.sum_keepdim(0)?.sum_keepdim(1)?.to_vec2::()?, + &[[7998000]] + ); + assert_eq!( + tensor.sum_keepdim(1)?.sum_keepdim(0)?.to_vec2::()?, + &[[7998000]] + ); + assert_eq!( + tensor.sum_keepdim(0)?.to_vec2::()?, + &[[3998000, 4000000]] + ); + + // Make the tensor non contiguous. + let tensor = tensor.t()?.contiguous()?.t()?; + assert_eq!(tensor.sum_keepdim((0, 1))?.to_vec2::()?, &[[7998000]]); + assert_eq!( + tensor.sum_keepdim(0)?.sum_keepdim(1)?.to_vec2::()?, + &[[7998000]] + ); + assert_eq!( + tensor.sum_keepdim(1)?.sum_keepdim(0)?.to_vec2::()?, + &[[7998000]] + ); + assert_eq!( + tensor.sum_keepdim(0)?.to_vec2::()?, + &[[3998000, 4000000]] + ); + + let t1 = tensor.reshape((200, 5, 4))?; + let t2 = t1.transpose(0, 2)?.contiguous()?.transpose(0, 2)?; + for tensor in [t1, t2] { + assert_eq!( + tensor.sum_keepdim((0, 1, 2))?.to_vec3::()?, + &[[[7998000]]] + ); + assert_eq!( + tensor + .sum_keepdim(0)? + .sum_keepdim(2)? + .sum_keepdim(1)? + .to_vec3::()?, + &[[[7998000]]] + ); + assert_eq!( + tensor + .sum_keepdim(0)? + .sum_keepdim((1, 2))? + .to_vec3::()?, + &[[[7998000]]] + ); + assert_eq!( + tensor + .sum_keepdim(1)? + .sum_keepdim((0, 2))? + .to_vec3::()?, + &[[[7998000]]] + ); + assert_eq!( + tensor.sum_keepdim(0)?.to_vec3::()?, + &[[ + [398000, 398200, 398400, 398600], + [398800, 399000, 399200, 399400], + [399600, 399800, 400000, 400200], + [400400, 400600, 400800, 401000], + [401200, 401400, 401600, 401800] + ]] + ); + } + Ok(()) +} + +fn min(device: &Device) -> Result<()> { + let data = &[[[3u32, 1, 4], [1, 5, 9]], [[2, 1, 7], [8, 2, 8]]]; + let tensor = Tensor::new(data, device)?; + assert_eq!( + tensor.min_keepdim(2)?.to_vec3::()?, + &[[[1], [1]], [[1], [2]]] + ); + assert_eq!( + tensor.min_keepdim(0)?.to_vec3::()?, + &[[[2, 1, 4], [1, 2, 8]]], + ); + let data: Vec = (200..4000u32).collect(); + let tensor = Tensor::new(data.as_slice(), device)?; + assert_eq!(tensor.min_keepdim(0)?.to_vec1::()?, &[200]); + let tensor = tensor.reshape((1900, 2))?; + assert_eq!( + tensor.min_keepdim(0)?.min_keepdim(1)?.to_vec2::()?, + &[[200]] + ); + assert_eq!( + tensor.min_keepdim(1)?.min_keepdim(0)?.to_vec2::()?, + &[[200]] + ); + assert_eq!(tensor.min_keepdim(0)?.to_vec2::()?, &[[200, 201]]); + + // Make the tensor non contiguous. + let tensor = tensor.t()?.contiguous()?.t()?; + assert_eq!( + tensor.min_keepdim(0)?.min_keepdim(1)?.to_vec2::()?, + &[[200]] + ); + assert_eq!( + tensor.min_keepdim(1)?.min_keepdim(0)?.to_vec2::()?, + &[[200]] + ); + assert_eq!(tensor.min_keepdim(0)?.to_vec2::()?, &[[200, 201]]); + + let t1 = tensor.reshape((190, 5, 4))?; + let t2 = t1.transpose(0, 2)?.contiguous()?.transpose(0, 2)?; + for tensor in [t1, t2] { + assert_eq!( + tensor + .min_keepdim(0)? + .min_keepdim(2)? + .min_keepdim(1)? + .to_vec3::()?, + &[[[200]]] + ); + assert_eq!( + tensor.min_keepdim(0)?.to_vec3::()?, + &[[ + [200, 201, 202, 203], + [204, 205, 206, 207], + [208, 209, 210, 211], + [212, 213, 214, 215], + [216, 217, 218, 219] + ]] + ); + } + Ok(()) +} + +fn max(device: &Device) -> Result<()> { + let data = &[[[3u32, 1, 4], [1, 5, 9]], [[2, 1, 7], [8, 2, 8]]]; + let tensor = Tensor::new(data, device)?; + assert_eq!( + tensor.max_keepdim(2)?.to_vec3::()?, + &[[[4], [9]], [[7], [8]]] + ); + assert_eq!( + tensor.max_keepdim(0)?.to_vec3::()?, + &[[[3, 1, 7], [8, 5, 9]]], + ); + let data: Vec = (200..4000u32).collect(); + let tensor = Tensor::new(data.as_slice(), device)?; + assert_eq!(tensor.max_keepdim(0)?.to_vec1::()?, &[3999]); + let tensor = tensor.reshape((1900, 2))?; + assert_eq!( + tensor.max_keepdim(0)?.max_keepdim(1)?.to_vec2::()?, + &[[3999]] + ); + assert_eq!( + tensor.max_keepdim(1)?.max_keepdim(0)?.to_vec2::()?, + &[[3999]] + ); + assert_eq!(tensor.max_keepdim(0)?.to_vec2::()?, &[[3998, 3999]]); + + // Make the tensor non contiguous. + let tensor = tensor.t()?.contiguous()?.t()?; + assert_eq!( + tensor.max_keepdim(0)?.max_keepdim(1)?.to_vec2::()?, + &[[3999]] + ); + assert_eq!( + tensor.max_keepdim(1)?.max_keepdim(0)?.to_vec2::()?, + &[[3999]] + ); + assert_eq!(tensor.max_keepdim(0)?.to_vec2::()?, &[[3998, 3999]]); + + let t1 = tensor.reshape((190, 5, 4))?; + let t2 = t1.transpose(0, 2)?.contiguous()?.transpose(0, 2)?; + for tensor in [t1, t2] { + assert_eq!( + tensor + .max_keepdim(0)? + .max_keepdim(2)? + .max_keepdim(1)? + .to_vec3::()?, + &[[[3999]]] + ); + assert_eq!( + tensor.max_keepdim(0)?.to_vec3::()?, + &[[ + [3980, 3981, 3982, 3983], + [3984, 3985, 3986, 3987], + [3988, 3989, 3990, 3991], + [3992, 3993, 3994, 3995], + [3996, 3997, 3998, 3999] + ]] + ); + } + Ok(()) +} + +fn argmin(device: &Device) -> Result<()> { + let data = &[[[3u32, 1, 4], [1, 5, 9]], [[2, 1, 7], [8, 2, 8]]]; + let tensor = Tensor::new(data, device)?; + assert_eq!( + tensor.argmin_keepdim(2)?.to_vec3::()?, + &[[[1], [0]], [[1], [1]]] + ); + assert_eq!( + tensor.argmin_keepdim(0)?.to_vec3::()?, + &[[[1, 0, 0], [0, 1, 1]]], + ); + let data: Vec = (200..4000u32).collect(); + let tensor = Tensor::new(data.as_slice(), device)?; + assert_eq!(tensor.argmin_keepdim(0)?.to_vec1::()?, &[0]); + let tensor = tensor.reshape((1900, 2))?; + assert_eq!( + tensor + .argmin_keepdim(0)? + .argmin_keepdim(1)? + .to_vec2::()?, + &[[0]] + ); + assert_eq!( + tensor + .argmin_keepdim(1)? + .argmin_keepdim(0)? + .to_vec2::()?, + &[[0]] + ); + assert_eq!(tensor.argmin_keepdim(0)?.to_vec2::()?, &[[0, 0]]); + + // Make the tensor non contiguous. + let tensor = tensor.t()?.contiguous()?.t()?; + assert_eq!( + tensor + .argmin_keepdim(0)? + .argmin_keepdim(1)? + .to_vec2::()?, + &[[0]] + ); + assert_eq!( + tensor + .argmin_keepdim(1)? + .argmin_keepdim(0)? + .to_vec2::()?, + &[[0]] + ); + assert_eq!(tensor.argmin_keepdim(0)?.to_vec2::()?, &[[0, 0]]); + + let t1 = tensor.reshape((190, 5, 4))?; + let t2 = t1.transpose(0, 2)?.contiguous()?.transpose(0, 2)?; + for tensor in [t1, t2] { + assert_eq!( + tensor + .argmin_keepdim(0)? + .argmin_keepdim(2)? + .argmin_keepdim(1)? + .to_vec3::()?, + &[[[0]]] + ); + assert_eq!( + tensor.argmin_keepdim(0)?.to_vec3::()?, + &[[ + [0, 0, 0, 0], + [0, 0, 0, 0], + [0, 0, 0, 0], + [0, 0, 0, 0], + [0, 0, 0, 0], + ]] + ); + } + Ok(()) +} + +fn argmax(device: &Device) -> Result<()> { + let data = &[[[3u32, 1, 4], [1, 5, 9]], [[2, 1, 7], [8, 2, 8]]]; + let tensor = Tensor::new(data, device)?; + assert_eq!( + tensor.argmax_keepdim(2)?.to_vec3::()?, + &[[[2], [2]], [[2], [0]]] + ); + assert_eq!( + tensor.argmax_keepdim(0)?.to_vec3::()?, + &[[[0, 0, 1], [1, 0, 0]]], + ); + let data: Vec = (200..4000u32).collect(); + let tensor = Tensor::new(data.as_slice(), device)?; + assert_eq!(tensor.argmax_keepdim(0)?.to_vec1::()?, &[3799]); + let tensor = tensor.reshape((1900, 2))?; + assert_eq!( + tensor + .argmax_keepdim(0)? + .argmax_keepdim(1)? + .to_vec2::()?, + &[[0]] + ); + assert_eq!( + tensor + .argmax_keepdim(1)? + .argmax_keepdim(0)? + .to_vec2::()?, + &[[0]] + ); + assert_eq!(tensor.argmax_keepdim(0)?.to_vec2::()?, &[[1899, 1899]]); + + // Make the tensor non contiguous. + let tensor = tensor.t()?.contiguous()?.t()?; + assert_eq!( + tensor + .argmax_keepdim(0)? + .argmax_keepdim(1)? + .to_vec2::()?, + &[[0]] + ); + assert_eq!( + tensor + .argmax_keepdim(1)? + .argmax_keepdim(0)? + .to_vec2::()?, + &[[0]] + ); + assert_eq!(tensor.argmax_keepdim(0)?.to_vec2::()?, &[[1899, 1899]]); + + let t1 = tensor.reshape((190, 5, 4))?; + let t2 = t1.transpose(0, 2)?.contiguous()?.transpose(0, 2)?; + for tensor in [t1, t2] { + assert_eq!( + tensor + .argmax_keepdim(0)? + .argmax_keepdim(2)? + .argmax_keepdim(1)? + .to_vec3::()?, + &[[[0]]] + ); + assert_eq!( + tensor.argmax_keepdim(0)?.to_vec3::()?, + &[[ + [189, 189, 189, 189], + [189, 189, 189, 189], + [189, 189, 189, 189], + [189, 189, 189, 189], + [189, 189, 189, 189], + ]] + ); + } + Ok(()) +} + +fn narrow(device: &Device) -> Result<()> { + let data = &[[[3f32, 1., 4.], [1., 5., 9.]], [[2., 1., 7.], [8., 2., 8.]]]; + let tensor = Tensor::new(data, device)?; + assert_eq!( + tensor.narrow(2, 1, 2)?.to_vec3::()?, + &[[[1.0, 4.0], [5.0, 9.0]], [[1.0, 7.0], [2.0, 8.0]]], + ); + assert_eq!( + tensor.narrow(1, 1, 1)?.to_vec3::()?, + &[[[1.0, 5.0, 9.0]], [[8.0, 2.0, 8.0]]], + ); + assert_eq!( + tensor.narrow(0, 0, 1)?.to_vec3::()?, + &[[[3.0, 1.0, 4.0], [1.0, 5.0, 9.0]]], + ); + assert_eq!( + tensor.narrow(0, 1, 1)?.to_vec3::()?, + &[[[2.0, 1.0, 7.0], [8.0, 2.0, 8.0]]], + ); + // The following has been checked against PyTorch via: + // import torch + // t = torch.tensor([[[3., 1., 4.], [1., 5., 9.]], [[2., 1., 7.], [8., 2., 8.]]]) + // t.transpose(-1, -2).narrow(1, 1, 2) + assert_eq!( + tensor.t()?.narrow(1, 1, 2)?.to_vec3::()?, + &[[[1.0, 5.0], [4.0, 9.0]], [[1.0, 2.0], [7.0, 8.0]]], + ); + Ok(()) +} + +fn broadcast(device: &Device) -> Result<()> { + let data = &[3f32, 1., 4.]; + let tensor = Tensor::new(data, device)?; + assert_eq!( + tensor.broadcast_left((3, 1))?.to_vec3::()?, + &[[[3.0, 1.0, 4.0]], [[3.0, 1.0, 4.0]], [[3.0, 1.0, 4.0]]] + ); + Ok(()) +} + +fn cat(device: &Device) -> Result<()> { + // 1D + let t1 = Tensor::new(&[3f32, 1., 4.], device)?; + let t2 = Tensor::new(&[1f32, 5., 9., 2.], device)?; + let t3 = Tensor::new(&[6f32, 5., 3., 5., 8., 9.], device)?; + assert_eq!(Tensor::cat(&[&t1], 0)?.to_vec1::()?, [3f32, 1., 4.],); + assert_eq!( + Tensor::cat(&[&t1, &t2], 0)?.to_vec1::()?, + [3f32, 1., 4., 1., 5., 9., 2.], + ); + assert_eq!( + Tensor::cat(&[&t1, &t2, &t3], 0)?.to_vec1::()?, + [3f32, 1., 4., 1., 5., 9., 2., 6., 5., 3., 5., 8., 9.], + ); + + // 2D + let data = &[[3f32, 1., 4., 1., 5.], [2., 7., 1., 8., 2.]]; + let t1 = Tensor::new(data, device)?; + let data2 = &[[5f32, 5., 5., 5., 5.], [2., 7., 1., 8., 2.]]; + let t2 = Tensor::new(data2, device)?; + assert_eq!( + Tensor::cat(&[&t1, &t2], 0)?.to_vec2::()?, + [ + [3.0, 1.0, 4.0, 1.0, 5.0], + [2.0, 7.0, 1.0, 8.0, 2.0], + [5.0, 5.0, 5.0, 5.0, 5.0], + [2.0, 7.0, 1.0, 8.0, 2.0] + ] + ); + // PyTorch equivalent: + // import torch + // t1 = torch.tensor([[3, 1, 4, 1, 5], [2, 7, 1, 8, 2]]) + // t2 = torch.tensor([[5]*5, [2, 7, 1, 8, 2]]) + // torch.cat([t1.t(), t2.t()], dim=1).t() + assert_eq!( + Tensor::cat(&[&t1.t()?, &t2.t()?], 1)? + .t()? + .to_vec2::()?, + [ + [3.0, 1.0, 4.0, 1.0, 5.0], + [2.0, 7.0, 1.0, 8.0, 2.0], + [5.0, 5.0, 5.0, 5.0, 5.0], + [2.0, 7.0, 1.0, 8.0, 2.0] + ] + ); + assert_eq!( + Tensor::cat(&[&t1, &t2], 1)?.to_vec2::()?, + [ + [3.0, 1.0, 4.0, 1.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0], + [2.0, 7.0, 1.0, 8.0, 2.0, 2.0, 7.0, 1.0, 8.0, 2.0] + ] + ); + Ok(()) +} + +fn embeddings(device: &Device) -> Result<()> { + let ids = Tensor::new(&[0u32, 2u32, 1u32], device)?; + let t = Tensor::new(&[[0f32, 1f32], [2f32, 3f32], [4f32, 5f32]], device)?; + let hs = t.embedding(&ids)?; + assert_eq!(hs.to_vec2::()?, &[[0.0, 1.0], [4.0, 5.0], [2.0, 3.0]]); + let hs = t.index_select(&ids, 0)?; + assert_eq!(hs.to_vec2::()?, &[[0.0, 1.0], [4.0, 5.0], [2.0, 3.0]]); + Ok(()) +} + +fn cmp(device: &Device) -> Result<()> { + let t1 = Tensor::new(&[[0f32, 1f32], [2f32, 3f32], [4f32, 5f32]], device)?; + let t2 = Tensor::new(&[[1f32, 0f32], [3f32, 3f32], [4f32, 7f32]], device)?; + assert_eq!(t1.eq(&t2)?.to_vec2::()?, &[[0, 0], [0, 1], [1, 0]]); + assert_eq!(t1.ne(&t2)?.to_vec2::()?, &[[1, 1], [1, 0], [0, 1]]); + assert_eq!(t1.le(&t2)?.to_vec2::()?, &[[1, 0], [1, 1], [1, 1]]); + assert_eq!(t1.lt(&t2)?.to_vec2::()?, &[[1, 0], [1, 0], [0, 1]]); + assert_eq!(t1.gt(&t2)?.to_vec2::()?, &[[0, 1], [0, 0], [0, 0]]); + assert_eq!(t1.ge(&t2)?.to_vec2::()?, &[[0, 1], [0, 1], [1, 0]]); + Ok(()) +} + +fn index_select(device: &Device) -> Result<()> { + let ids = Tensor::new(&[0u32, 2u32, 1u32], device)?; + let t = Tensor::arange(0f32, 12f32, device)?.reshape((4, 3))?; + assert_eq!( + t.to_vec2::()?, + &[ + [0.0, 1.0, 2.0], + [3.0, 4.0, 5.0], + [6.0, 7.0, 8.0], + [9.0, 10.0, 11.0] + ] + ); + let hs = t.index_select(&ids, 1)?; + assert_eq!( + hs.to_vec2::()?, + &[ + [0.0, 2.0, 1.0], + [3.0, 5.0, 4.0], + [6.0, 8.0, 7.0], + [9.0, 11.0, 10.0] + ] + ); + let hs = t.index_select(&ids, 0)?; + assert_eq!( + hs.to_vec2::()?, + &[[0.0, 1.0, 2.0], [6.0, 7.0, 8.0], [3.0, 4.0, 5.0]] + ); + // Prior to https://github.com/huggingface/candle/pull/1022 + // There would be a bug where the last values in the result tensor would be set to 0. + let ids = Tensor::new(&[0u32, 2u32, 1u32, 0u32, 2u32, 1u32], device)?; + let hs = t.index_select(&ids, 0)?; + assert_eq!( + hs.to_vec2::()?, + &[ + [0.0, 1.0, 2.0], + [6.0, 7.0, 8.0], + [3.0, 4.0, 5.0], + [0.0, 1.0, 2.0], + [6.0, 7.0, 8.0], + [3.0, 4.0, 5.0], + ] + ); + + // Test when selecting dim > 0 with ids size different from elem count of + // target dim in source/input. + let ids = Tensor::new(&[1u32, 0u32, 1u32], device)?; + let t = Tensor::arange(1f32, 5f32, device)?.reshape((2, 2))?; + assert_eq!(t.to_vec2::()?, &[[1.0, 2.0], [3.0, 4.0]]); + let hs = t.index_select(&ids, 1)?; + assert_eq!(hs.to_vec2::()?, &[[2.0, 1.0, 2.0], [4.0, 3.0, 4.0]]); + + Ok(()) +} + +fn index_add(device: &Device) -> Result<()> { + let ids = Tensor::new(&[0u32, 1u32, 1u32], device)?; + let t = Tensor::arange(0f32, 12f32, device)?.reshape((4, 3))?; + assert_eq!( + t.to_vec2::()?, + &[ + [0.0, 1.0, 2.0], + [3.0, 4.0, 5.0], + [6.0, 7.0, 8.0], + [9.0, 10.0, 11.0] + ] + ); + let init = Tensor::ones((4, 2), DType::F32, device)?; + let hs = init.index_add(&ids, &t, 1)?; + assert_eq!( + hs.to_vec2::()?, + &[[1.0, 4.0], [4.0, 10.0], [7.0, 16.0], [10.0, 22.0]], + ); + let init = Tensor::zeros((4, 2), DType::F32, device)?; + let ids = Tensor::new(&[1u32, 0u32, 0u32], device)?; + let hs = init.index_add(&ids, &t, 1)?; + assert_eq!( + hs.to_vec2::()?, + &[[3.0, 0.0], [9.0, 3.0], [15.0, 6.0], [21.0, 9.0]], + ); + + let init = Tensor::zeros((6, 3), DType::F32, device)?; + let ids = Tensor::new(&[5u32, 0u32, 1u32, 0u32], device)?; + let hs = init.index_add(&ids, &t, 0)?; + assert_eq!( + hs.to_vec2::()?, + &[ + [12.0, 14.0, 16.0], + [6.0, 7.0, 8.0], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + [0.0, 1.0, 2.0] + ] + ); + Ok(()) +} + +fn slice_scatter(device: &Device) -> Result<()> { + let t = Tensor::arange(0f32, 12f32, device)?.reshape((4, 3))?; + assert_eq!( + t.to_vec2::()?, + &[ + [0.0, 1.0, 2.0], + [3.0, 4.0, 5.0], + [6.0, 7.0, 8.0], + [9.0, 10.0, 11.0] + ] + ); + let src = Tensor::arange(100f32, 106f32, device)?.reshape((2, 3))?; + assert_eq!( + t.slice_scatter0(&src, 0)?.to_vec2::()?, + &[ + [100.0, 101.0, 102.0], + [103.0, 104.0, 105.0], + [6.0, 7.0, 8.0], + [9.0, 10.0, 11.0] + ] + ); + assert_eq!( + t.slice_scatter0(&src, 1)?.to_vec2::()?, + &[ + [0.0, 1.0, 2.0], + [100.0, 101.0, 102.0], + [103.0, 104.0, 105.0], + [9.0, 10.0, 11.0] + ] + ); + assert_eq!( + t.slice_scatter0(&src, 2)?.to_vec2::()?, + &[ + [0.0, 1.0, 2.0], + [3.0, 4.0, 5.0], + [100.0, 101.0, 102.0], + [103.0, 104.0, 105.0], + ] + ); + Ok(()) +} + +fn scatter_add(device: &Device) -> Result<()> { + let t = Tensor::arange(0f32, 12f32, device)?.reshape((4, 3))?; + assert_eq!( + t.to_vec2::()?, + &[ + [0.0, 1.0, 2.0], + [3.0, 4.0, 5.0], + [6.0, 7.0, 8.0], + [9.0, 10.0, 11.0] + ] + ); + let ids = Tensor::new(&[[0u32, 1, 2], [3, 4, 0], [3, 3, 1], [2, 0, 4]], device)?; + let init = Tensor::ones((4, 5), DType::F32, device)?; + let hs = init.scatter_add(&ids, &t, 1)?; + assert_eq!( + hs.to_vec2::()?, + &[ + [1.0, 2.0, 3.0, 1.0, 1.0], + [6.0, 1.0, 1.0, 4.0, 5.0], + [1.0, 9.0, 1.0, 14.0, 1.0], + [11.0, 1.0, 10.0, 1.0, 12.0] + ] + ); + + let init = Tensor::ones((6, 3), DType::F32, device)?; + let hs = init.scatter_add(&ids, &t, 0)?; + assert_eq!( + hs.to_vec2::()?, + &[ + [1.0, 11.0, 6.0], + [1.0, 2.0, 9.0], + [10.0, 1.0, 3.0], + [10.0, 8.0, 1.0], + [1.0, 5.0, 12.0], + [1.0, 1.0, 1.0] + ] + ); + Ok(()) +} + +fn gather(device: &Device) -> Result<()> { + let ids = Tensor::new(&[[0u32], [2u32], [1u32], [0u32]], device)?; + let t = Tensor::arange(0f32, 12f32, device)?.reshape((4, 3))?; + assert_eq!( + t.to_vec2::()?, + &[ + [0.0, 1.0, 2.0], + [3.0, 4.0, 5.0], + [6.0, 7.0, 8.0], + [9.0, 10.0, 11.0] + ] + ); + let hs = t.gather(&ids, 1)?; + assert_eq!(hs.to_vec2::()?, &[[0.0], [5.0], [7.0], [9.0]]); + let ids = Tensor::new( + &[[0u32, 0u32], [2u32, 0u32], [1u32, 1u32], [0u32, 2u32]], + device, + )?; + let hs = t.gather(&ids, 1)?; + assert_eq!( + hs.to_vec2::()?, + &[[0.0, 0.0], [5.0, 3.0], [7.0, 7.0], [9.0, 11.0]] + ); + let ids = Tensor::new(&[[0u32, 2u32, 0u32]], device)?; + let hs = t.gather(&ids, 0)?; + assert_eq!(hs.to_vec2::()?, &[[0.0, 7.0, 2.0]]); + let ids = Tensor::new(&[[0u32, 2u32, 0u32], [0u32, 1u32, 1u32]], device)?; + let hs = t.gather(&ids, 0)?; + assert_eq!(hs.to_vec2::()?, &[[0.0, 7.0, 2.0], [0.0, 4.0, 5.0]]); + Ok(()) +} + +fn matmul(device: &Device) -> Result<()> { + let data = vec![1.0f32, 2.0, 3.0, 4.0]; + let a = Tensor::from_slice(&data, (2, 2), device)?; + let data = vec![1.0f32, 2.0, 3.0, 4.0]; + let b = Tensor::from_slice(&data, (2, 2), device)?; + + let c = a.matmul(&b)?; + assert_eq!(c.to_vec2::()?, &[[7.0f32, 10.0], [15.0, 22.0]]); + + let data = vec![1.0f32, 2.0]; + let a = Tensor::from_slice(&data, (2, 1), device)?; + let data = vec![3.0f32, 4.0]; + let b = Tensor::from_slice(&data, (1, 2), device)?; + let c = a.matmul(&b)?; + assert_eq!(c.to_vec2::()?, &[&[3.0, 4.0], &[6.0, 8.0]]); + + let data: Vec<_> = (0..6).map(|i| i as f32).collect(); + let a = Tensor::from_slice(&data, (2, 3), device)?; + let data: Vec<_> = (0..6).map(|i| (i + 2) as f32).collect(); + let b = Tensor::from_slice(&data, (3, 2), device)?; + let c = a.matmul(&b)?; + assert_eq!(c.to_vec2::()?, &[&[16., 19.], &[52., 64.]]); + + let data: Vec<_> = (0..12).map(|i| i as f32).collect(); + let a = Tensor::from_slice(&data, (2, 2, 3), device)?; + let data: Vec<_> = (0..12).map(|i| (i + 2) as f32).collect(); + let b = Tensor::from_slice(&data, (2, 3, 2), device)?; + let expected = [[[16., 19.], [52., 64.]], [[214., 235.], [304., 334.]]]; + + let c = a.matmul(&b)?; + assert_eq!(c.to_vec3::()?, &expected); + + // Also perform the matmul on contiguous transposed versions. + let a_tt = a.t()?.contiguous()?.t()?; + assert!(!a_tt.is_contiguous()); + assert_eq!(a.dims(), a_tt.dims()); + assert_eq!(a_tt.stride(), &[6, 1, 2]); + + let b_tt = b.t()?.contiguous()?.t()?; + assert!(!b_tt.is_contiguous()); + assert_eq!(b.dims(), b_tt.dims()); + assert_eq!(b_tt.stride(), &[6, 1, 3]); + + assert_eq!(a_tt.matmul(&b)?.to_vec3::()?, &expected); + assert_eq!(a.matmul(&b_tt)?.to_vec3::()?, &expected); + assert_eq!(a_tt.matmul(&b_tt)?.to_vec3::()?, &expected); + Ok(()) +} + +fn broadcast_matmul(device: &Device) -> Result<()> { + let lhs = Tensor::randn(0f32, 1f32, (3, 1, 4, 5), device)?; + let rhs = Tensor::randn(0f32, 1f32, (6, 5, 2), device)?; + let out = lhs.broadcast_matmul(&rhs)?; + assert_eq!(out.dims(), &[3, 6, 4, 2]); + for idx1 in 0..3 { + for idx2 in 0..6 { + let out = out.i((idx1, idx2))?; + let lhs = lhs.i((idx1, 0))?; + let rhs = rhs.i(idx2)?; + let out2 = lhs.matmul(&rhs); + let sum_diff2 = (out - out2)?.sqr()?.sum_all()?; + // With cuda, we see errors of up to ~1e-12. + assert!(sum_diff2.to_vec0::()? < 1e-6) + } + } + Ok(()) +} + +fn broadcasting(device: &Device) -> Result<()> { + let t1 = Tensor::arange(0f32, 24f32, device)?.reshape((4, 2, 3))?; + let t2 = Tensor::new(&[100f32, 200f32], device)?; + let s = t1.broadcast_add(&t2.reshape((2, 1))?)?; + assert_eq!( + s.to_vec3::()?, + &[ + [[100.0, 101.0, 102.0], [203.0, 204.0, 205.0]], + [[106.0, 107.0, 108.0], [209.0, 210.0, 211.0]], + [[112.0, 113.0, 114.0], [215.0, 216.0, 217.0]], + [[118.0, 119.0, 120.0], [221.0, 222.0, 223.0]] + ] + ); + let s = t1.t()?.broadcast_add(&t2)?; + assert_eq!( + s.to_vec3::()?, + &[ + [[100.0, 203.0], [101.0, 204.0], [102.0, 205.0]], + [[106.0, 209.0], [107.0, 210.0], [108.0, 211.0]], + [[112.0, 215.0], [113.0, 216.0], [114.0, 217.0]], + [[118.0, 221.0], [119.0, 222.0], [120.0, 223.0]] + ] + ); + let s = t1.broadcast_sub(&t2.reshape((2, 1))?)?; + assert_eq!( + s.to_vec3::()?, + &[ + [[-100.0, -99.0, -98.0], [-197.0, -196.0, -195.0]], + [[-94.0, -93.0, -92.0], [-191.0, -190.0, -189.0]], + [[-88.0, -87.0, -86.0], [-185.0, -184.0, -183.0]], + [[-82.0, -81.0, -80.0], [-179.0, -178.0, -177.0]] + ] + ); + let s = t1.t()?.broadcast_sub(&t2)?; + assert_eq!( + s.to_vec3::()?, + &[ + [[-100.0, -197.0], [-99.0, -196.0], [-98.0, -195.0]], + [[-94.0, -191.0], [-93.0, -190.0], [-92.0, -189.0]], + [[-88.0, -185.0], [-87.0, -184.0], [-86.0, -183.0]], + [[-82.0, -179.0], [-81.0, -178.0], [-80.0, -177.0]] + ] + ); + // Test a narrowed version as this uses a layout start_offset. + let t1 = t1.i(2..)?; + let s = t1.broadcast_add(&t2.reshape((2, 1))?)?; + assert_eq!( + s.to_vec3::()?, + &[ + [[112.0, 113.0, 114.0], [215.0, 216.0, 217.0]], + [[118.0, 119.0, 120.0], [221.0, 222.0, 223.0]] + ] + ); + let s = t1.t()?.broadcast_add(&t2)?; + assert_eq!( + s.to_vec3::()?, + &[ + [[112.0, 215.0], [113.0, 216.0], [114.0, 217.0]], + [[118.0, 221.0], [119.0, 222.0], [120.0, 223.0]] + ] + ); + let s = t1.broadcast_sub(&t2.reshape((2, 1))?)?; + assert_eq!( + s.to_vec3::()?, + &[ + [[-88.0, -87.0, -86.0], [-185.0, -184.0, -183.0]], + [[-82.0, -81.0, -80.0], [-179.0, -178.0, -177.0]] + ] + ); + let s = t1.t()?.broadcast_sub(&t2)?; + assert_eq!( + s.to_vec3::()?, + &[ + [[-88.0, -185.0], [-87.0, -184.0], [-86.0, -183.0]], + [[-82.0, -179.0], [-81.0, -178.0], [-80.0, -177.0]] + ] + ); + let t3 = Tensor::new(1f32, device)?.broadcast_div(&t2)?; + let s = t1.broadcast_mul(&t2.reshape((2, 1))?)?; + let s_div = t1.broadcast_div(&t3.reshape((2, 1))?)?; + assert_eq!( + s.to_vec3::()?, + &[ + [[1200.0, 1300.0, 1400.0], [3000.0, 3200.0, 3400.0]], + [[1800.0, 1900.0, 2000.0], [4200.0, 4400.0, 4600.0]] + ] + ); + assert_eq!(s.to_vec3::()?, s_div.to_vec3::()?,); + let s = t1.t()?.broadcast_mul(&t2)?; + let s_div = t1.t()?.broadcast_div(&t3)?; + assert_eq!( + s.to_vec3::()?, + &[ + [[1200.0, 3000.0], [1300.0, 3200.0], [1400.0, 3400.0]], + [[1800.0, 4200.0], [1900.0, 4400.0], [2000.0, 4600.0]] + ] + ); + assert_eq!(s.to_vec3::()?, s_div.to_vec3::()?,); + Ok(()) +} + +fn randn(device: &Device) -> Result<()> { + let tensor = Tensor::randn(0f32, 1f32, (5, 3), device)?; + assert_eq!(tensor.dims(), [5, 3]); + let tensor = Tensor::rand(0f32, 1f32, (5, 3), device)?; + assert_eq!(tensor.dims(), [5, 3]); + Ok(()) +} + +test_device!(zeros, zeros_cpu, zeros_gpu, zeros_metal); +test_device!(ones, ones_cpu, ones_gpu, ones_metal); +test_device!(full, full_cpu, full_gpu, full_metal); +test_device!(arange, arange_cpu, arange_gpu, arange_metal); +test_device!(add_mul, add_mul_cpu, add_mul_gpu, add_mul_metal); +test_device!(tensor_2d, tensor_2d_cpu, tensor_2d_gpu, tensor_2d_metal); +test_device!(narrow, narrow_cpu, narrow_gpu, narrow_metal); +test_device!(broadcast, broadcast_cpu, broadcast_gpu, broadcast_metal); +test_device!(cat, cat_cpu, cat_gpu, cat_metal); +test_device!(sum, sum_cpu, sum_gpu, sum_metal); +test_device!(min, min_cpu, min_gpu, min_metal); +test_device!(max, max_cpu, max_gpu, max_metal); +test_device!(argmax, argmax_cpu, argmax_gpu, argmax_metal); +test_device!(argmin, argmin_cpu, argmin_gpu, argmin_metal); +test_device!(transpose, transpose_cpu, transpose_gpu, transpose_metal); +test_device!(unary_op, unary_op_cpu, unary_op_gpu, unary_op_metal); +test_device!(binary_op, binary_op_cpu, binary_op_gpu, binary_op_metal); +test_device!(embeddings, embeddings_cpu, embeddings_gpu, embeddings_metal); +test_device!(cmp, cmp_cpu, cmp_gpu, cmp_metal); +test_device!(matmul, matmul_cpu, matmul_gpu, matmul_metal); +test_device!( + broadcast_matmul, + broadcast_matmul_cpu, + broadcast_matmul_gpu, + broadcast_matmul_metal +); +test_device!( + broadcasting, + broadcasting_cpu, + broadcasting_gpu, + broadcasting_metal +); +test_device!( + index_select, + index_select_cpu, + index_select_gpu, + index_select_metal +); +test_device!(index_add, index_add_cpu, index_add_gpu, index_add_metal); +test_device!(gather, gather_cpu, gather_gpu, gather_metal); +test_device!( + scatter_add, + scatter_add_cpu, + scatter_add_gpu, + scatter_add_metal +); +test_device!( + slice_scatter, + slice_scatter_cpu, + slice_scatter_gpu, + slice_scatter_metal +); +test_device!(randn, randn_cpu, randn_gpu, randn_metal); +test_device!(clamp, clamp_cpu, clamp_gpu, clamp_metal); +test_device!(var, var_cpu, var_gpu, var_metal); + +// There was originally a bug on the CPU implementation for randn +// https://github.com/huggingface/candle/issues/381 +#[test] +fn randn_hasneg() -> Result<()> { + let t = Tensor::randn(0f32, 1f32, 200, &Device::Cpu)?.to_vec1::()?; + if t.iter().all(|&v| v >= 0.) { + candle_core::bail!("all values in tensors are non-negative") + } + Ok(()) +} + +#[test] +fn pad_with_same() -> Result<()> { + let t = Tensor::arange(1f32, 5f32, &Device::Cpu)?.reshape((2, 2))?; + let t0 = t.pad_with_same(0, 1, 2)?; + assert_eq!( + t0.to_vec2::()?, + [[1.0, 2.0], [1.0, 2.0], [3.0, 4.0], [3.0, 4.0], [3.0, 4.0]] + ); + let t1 = t.pad_with_same(1, 1, 2)?; + assert_eq!( + t1.to_vec2::()?, + [[1.0, 1.0, 2.0, 2.0, 2.0], [3.0, 3.0, 4.0, 4.0, 4.0]] + ); + Ok(()) +} + +#[test] +fn i64_abs() -> Result<()> { + let t = Tensor::new(&[-42i64, 1337], &Device::Cpu)?; + let t = t.abs()?; + assert_eq!(t.to_vec1::()?, [42, 1337]); + Ok(()) +} + +#[test] +fn tril_triu_eye() -> Result<()> { + let t = Tensor::tril2(4, DType::F32, &Device::Cpu)?; + assert_eq!( + t.to_vec2::()?, + [ + [1.0, 0.0, 0.0, 0.0], + [1.0, 1.0, 0.0, 0.0], + [1.0, 1.0, 1.0, 0.0], + [1.0, 1.0, 1.0, 1.0] + ], + ); + let t = Tensor::triu2(4, DType::F32, &Device::Cpu)?; + assert_eq!( + t.to_vec2::()?, + [ + [1.0, 1.0, 1.0, 1.0], + [0.0, 1.0, 1.0, 1.0], + [0.0, 0.0, 1.0, 1.0], + [0.0, 0.0, 0.0, 1.0] + ] + ); + let t = Tensor::eye(4, DType::F32, &Device::Cpu)?; + assert_eq!( + t.to_vec2::()?, + [ + [1.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 1.0] + ] + ); + Ok(()) +} + +#[test] +fn cumsum() -> Result<()> { + let t = &[3f32, 1., 4., 1., 5.]; + let t = Tensor::new(t, &Device::Cpu)?; + assert_eq!(t.cumsum(0)?.to_vec1::()?, [3., 4., 8., 9., 14.]); + let t = t.unsqueeze(1)?; + assert_eq!( + t.cumsum(0)?.to_vec2::()?, + [[3.0], [4.0], [8.0], [9.0], [14.0]] + ); + assert_eq!( + t.cumsum(1)?.to_vec2::()?, + [[3.0], [1.0], [4.0], [1.0], [5.0]] + ); + let t = &[[3f32, 1., 4., 1., 5.], [2., 1., 7., 8., 2.]]; + let t = Tensor::new(t, &Device::Cpu)?; + assert_eq!( + t.cumsum(1)?.to_vec2::()?, + [[3.0, 4.0, 8.0, 9.0, 14.0], [2.0, 3.0, 10.0, 18.0, 20.0]], + ); + assert_eq!( + t.cumsum(0)?.to_vec2::()?, + [[3.0, 1.0, 4.0, 1.0, 5.0], [5.0, 2.0, 11.0, 9.0, 7.0]] + ); + Ok(()) +} + +/// A helper function for floating point comparison. Both a and b must be 1D Tensor and contains the same amount of data. +/// Assertion passes if the difference of all pairs of a and b is smaller than epsilon. +fn assert_close(a: &Tensor, b: &Tensor, epsilon: f64) -> Result<()> { + let a_vec: Vec = a.to_vec1()?; + let b_vec: Vec = b.to_vec1()?; + + assert_eq!(a_vec.len(), b_vec.len()); + for (a, b) in a_vec.iter().zip(b_vec.iter()) { + assert!((a - b).abs() < epsilon); + } + Ok(()) +} + +#[test] +fn log_sum_exp() -> Result<()> { + let input = Tensor::new(&[[1f64, 2., 3.], [4., 5., 6.]], &Device::Cpu)?; + let output = input.log_sum_exp(D::Minus1)?; + // The expectations obtained from pytorch. + let expected = Tensor::new(&[3.4076, 6.4076], &Device::Cpu)?; + assert_close(&output, &expected, 0.00001)?; + Ok(()) +} + +#[test] +fn pow() -> Result<()> { + let lhs = Tensor::new(&[[1f32, 2., 3.], [4., 5., 6.]], &Device::Cpu)?; + let rhs = (&lhs - 2.)?; + let res = lhs.pow(&rhs)?; + assert_eq!( + test_utils::to_vec2_round(&res, 4)?, + [[1.0, 1.0, 3.0], [16.0, 125.0, 1296.0001]] + ); + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/test.npy b/v2/dependency-patches/candle-core-0.4.1/tests/test.npy new file mode 100644 index 0000000000000000000000000000000000000000..a3ff5af9540510ac6518b5a2478bc1a40f992371 GIT binary patch literal 208 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= wXCxM+0{I$-20EHL3bhL411<(AV1&|4P?{M^vp{K9D9r|?*`YKCl;(ue0P!XpoB#j- literal 0 HcmV?d00001 diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/test.npz b/v2/dependency-patches/candle-core-0.4.1/tests/test.npz new file mode 100644 index 0000000000000000000000000000000000000000..b6683caab0e0e9e66cb5ca649aab4980a1e650df GIT binary patch literal 668 zcmWIWW@Zs#fB;1X8xBpr3qTGCvoeSRzLK#d@ni)#7KxtMe%?72}p)?1S=7iD#D8ZokVdgfgc5ZyS5W1OUF4@IgfaSj3!+p!yZC;|&vLW_h60tGIXYp+tbR@}8WmEgpM zO8o`Bapb}U_$P4S5AY|DxUgf#!FKBdBaPM{^PXqkJlnY0LP(O(mFS~w#7M$MzRTR< z=}BmrTj)(S-V2@=g(2}+5G@kTjR$rRn5(#DL_r#w{fSQL9lLubi?Fp-^@ z94>>866e`Y|{Pa$jw1Xu^ zrRNx70k_WS4cx+wb1GjPK?hw%dsEs^hBod<4~{Qif95DiyLR*qw3Cf~_?D3fMzYN9 ztFwa^VZc~OfcnG1uN{%oLe^7v(W8H=jugv0xSqGFbKN~i0U=}u0!bi1b%=I$q+P)! zs0)!V96JB>^~;lo{263@=RMAm`r(Yk317Bl3cU5P3=H2t&R;?Yko63^$jBImV5w0; z;=m1qcP?1R34F^`|6dDL)%SE=n`qfpi3(kQ`+4)n zfOiOK<3=i3?);)L`uu%4O8-F>T3q}_3HOKJrO>sHs@S6R-4gZ{-|YIcBc%!~D&8vr zH~Cz}JynrK6?RL=X^y;pKQcsFZQ}Y+30Wu-S|m&nlEnez+N;E>iM#fu4OV8_A{%=Ua&@QUk(LBL$$hm%--75a)d;S;$UxT1w^n387>AHq2KYo>>4lqPFX(eg-Z1eK!Yr}cbF5km4 zMqWW11%Nh;xri}WZ5bp#%(Vi@y9JO`dS!(CFxCqQCOo-FBa=WGBHyXa0>t+Ph|6ba t(q?lj%e+;)Gsl?ec^02%3viYRAMz53|Hu{qk4@UZrG#B*5WJt>{S7%wAvgd4 literal 0 HcmV?d00001 diff --git a/v2/dependency-patches/candle-core-0.9.2/Cargo.toml b/v2/dependency-patches/candle-core-0.9.2/Cargo.toml new file mode 100644 index 0000000000..d4278ccc0a --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/Cargo.toml @@ -0,0 +1,262 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +name = "candle-core" +version = "0.9.2" +build = false +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Minimalist ML framework." +readme = "README.md" +keywords = [ + "blas", + "tensor", + "machine-learning", +] +categories = ["science"] +license = "MIT OR Apache-2.0" +repository = "https://github.com/huggingface/candle" + +[features] +accelerate = [ + "dep:libc", + "dep:accelerate-src", +] +cuda = [ + "cudarc", + "dep:candle-kernels", + "candle-ug?/cuda", + "float8/cuda", +] +cudnn = [ + "cuda", + "cudarc/cudnn", +] +default = [] +metal = [ + "dep:objc2-metal", + "dep:objc2-foundation", + "dep:candle-metal-kernels", + "candle-ug?/metal", +] +mkl = [ + "dep:libc", + "dep:intel-mkl-src", +] +nccl = [ + "cuda", + "cudarc/nccl", +] +ug = ["dep:candle-ug"] + +[lib] +name = "candle_core" +path = "src/lib.rs" + +[[example]] +name = "basics" +path = "examples/basics.rs" + +[[example]] +name = "cuda_basics" +path = "examples/cuda_basics.rs" +required-features = ["cuda"] + +[[example]] +name = "cuda_sum_benchmark" +path = "examples/cuda_sum_benchmark.rs" + +[[example]] +name = "metal_basics" +path = "examples/metal_basics.rs" +required-features = ["metal"] + +[[test]] +name = "bilinear_tests" +path = "tests/bilinear_tests.rs" + +[[test]] +name = "conv_tests" +path = "tests/conv_tests.rs" + +[[test]] +name = "custom_op_tests" +path = "tests/custom_op_tests.rs" + +[[test]] +name = "display_tests" +path = "tests/display_tests.rs" + +[[test]] +name = "grad_tests" +path = "tests/grad_tests.rs" + +[[test]] +name = "indexing_tests" +path = "tests/indexing_tests.rs" + +[[test]] +name = "layout_tests" +path = "tests/layout_tests.rs" + +[[test]] +name = "matmul_tests" +path = "tests/matmul_tests.rs" + +[[test]] +name = "pool_tests" +path = "tests/pool_tests.rs" + +[[test]] +name = "pth_tests" +path = "tests/pth_tests.rs" + +[[test]] +name = "quantized_tests" +path = "tests/quantized_tests.rs" + +[[test]] +name = "serialization_tests" +path = "tests/serialization_tests.rs" + +[[test]] +name = "tensor_tests" +path = "tests/tensor_tests.rs" + +[[bench]] +name = "bench_main" +path = "benches/bench_main.rs" +harness = false + +[dependencies.accelerate-src] +version = "0.3.2" +optional = true + +[dependencies.byteorder] +version = "1.4.3" + +[dependencies.candle-kernels] +version = "0.9.2" +optional = true + +[dependencies.candle-metal-kernels] +version = "0.9.2" +optional = true + +[dependencies.cudarc] +version = "0.19.0" +features = [ + "std", + "cublas", + "cublaslt", + "curand", + "driver", + "nvrtc", + "f16", + "f8", + "cuda-version-from-build-system", + "dynamic-linking", +] +optional = true +default-features = false + +[dependencies.float8] +version = "0.6.0" +features = [ + "num-traits", + "rand_distr", +] + +[dependencies.gemm] +version = "0.19.0" +features = ["wasm-simd128-enable"] + +[dependencies.half] +version = "2.5.0" +features = [ + "num-traits", + "use-intrinsics", + "rand_distr", +] + +[dependencies.intel-mkl-src] +version = "0.8.1" +features = ["mkl-static-lp64-iomp"] +optional = true + +[dependencies.libc] +version = "0.2.147" +optional = true + +[dependencies.libm] +version = "0.2.15" + +[dependencies.memmap2] +version = "0.9.3" +features = ["stable_deref_trait"] + +[dependencies.num-traits] +version = "0.2.15" + +[dependencies.num_cpus] +version = "1.15.0" + +[dependencies.objc2-foundation] +version = "0.3.1" +optional = true + +[dependencies.objc2-metal] +version = "0.3.1" +optional = true + +[dependencies.rand] +version = "0.9.0" + +[dependencies.rand_distr] +version = "0.5.1" + +[dependencies.rayon] +version = "1.7.0" + +[dependencies.safetensors] +version = "0.7.0" + +[dependencies.thiserror] +version = "2" + +[dependencies.yoke] +version = "0.8.1" +features = ["derive"] + +[dependencies.zip] +version = "9.0.1" +default-features = false + +[dev-dependencies.anyhow] +version = "1" +features = ["backtrace"] + +[dev-dependencies.clap] +version = "4.2.4" +features = ["derive"] + +[dev-dependencies.criterion] +version = "0.8" +default-features = false + +[target.'cfg(all(not(target_arch = "wasm32"), not(target_os = "ios")))'.dependencies.candle-ug] +version = "0.9.2" +optional = true diff --git a/v2/dependency-patches/candle-core-0.9.2/Cargo.toml.orig b/v2/dependency-patches/candle-core-0.9.2/Cargo.toml.orig new file mode 100644 index 0000000000..f02d15c318 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/Cargo.toml.orig @@ -0,0 +1,70 @@ +[package] +name = "candle-core" +version.workspace = true +edition.workspace = true +description.workspace = true +repository.workspace = true +keywords.workspace = true +categories.workspace = true +license.workspace = true +readme = "README.md" + +[dependencies] +accelerate-src = { workspace = true, optional = true } +byteorder = { workspace = true } +candle-kernels = { workspace = true, optional = true } +candle-metal-kernels = { workspace = true, optional = true } +objc2-metal = { workspace = true, optional = true } +objc2-foundation = { workspace = true, optional = true } +cudarc = { workspace = true, optional = true } +gemm = { workspace = true } +half = { workspace = true } +float8 = { workspace = true } +intel-mkl-src = { workspace = true, optional = true } +libc = { workspace = true, optional = true } +libm = { workspace = true } +memmap2 = { workspace = true } +num-traits = { workspace = true } +num_cpus = { workspace = true } +rand = { workspace = true } +rand_distr = { workspace = true } +rayon = { workspace = true } +safetensors = { workspace = true } +thiserror = { workspace = true } +yoke = { workspace = true } +zip = { workspace = true } + +[target.'cfg(all(not(target_arch = "wasm32"), not(target_os = "ios")))'.dependencies] +candle-ug = { workspace = true, optional = true } + +[dev-dependencies] +anyhow = { workspace = true } +clap = { workspace = true } +criterion = { workspace = true } + +[features] +default = [] +cuda = ["cudarc", "dep:candle-kernels", "candle-ug?/cuda", "float8/cuda"] +cudnn = ["cuda", "cudarc/cudnn"] +nccl = ["cuda", "cudarc/nccl"] +mkl = ["dep:libc", "dep:intel-mkl-src"] +accelerate = ["dep:libc", "dep:accelerate-src"] +metal = [ + "dep:objc2-metal", + "dep:objc2-foundation", + "dep:candle-metal-kernels", + "candle-ug?/metal", +] +ug = ["dep:candle-ug"] + +[[bench]] +name = "bench_main" +harness = false + +[[example]] +name = "metal_basics" +required-features = ["metal"] + +[[example]] +name = "cuda_basics" +required-features = ["cuda"] diff --git a/v2/dependency-patches/candle-core-0.9.2/LICENSE b/v2/dependency-patches/candle-core-0.9.2/LICENSE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/v2/dependency-patches/candle-core-0.9.2/README.md b/v2/dependency-patches/candle-core-0.9.2/README.md new file mode 100644 index 0000000000..246f5bae61 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/README.md @@ -0,0 +1,2 @@ +# candle +Minimalist ML framework for Rust diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/bench_main.rs b/v2/dependency-patches/candle-core-0.9.2/benches/bench_main.rs new file mode 100644 index 0000000000..ec02e4bddb --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/benches/bench_main.rs @@ -0,0 +1,17 @@ +mod benchmarks; + +use criterion::criterion_main; + +criterion_main!( + benchmarks::affine::benches, + benchmarks::binary::benches, + benchmarks::broadcast::benches, + benchmarks::copy::benches, + benchmarks::conv_transpose2d::benches, + benchmarks::matmul::benches, + benchmarks::qmatmul::benches, + benchmarks::random::benches, + benchmarks::reduce::benches, + benchmarks::unary::benches, + benchmarks::where_cond::benches, +); diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/affine.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/affine.rs new file mode 100644 index 0000000000..1d6edd4c69 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/affine.rs @@ -0,0 +1,46 @@ +use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; +use candle_core::{DType, Device, Tensor}; +use criterion::{criterion_group, Criterion, Throughput}; +use std::hint::black_box; +use std::time::Instant; + +fn run(a: &Tensor) { + a.affine(12.34, 56.78).unwrap(); +} + +fn run_affine_benchmark(c: &mut Criterion, device: &Device, dtype: DType, name: &str) { + let b = 1; + let m = 1024; + let k = 1024; + + let tensor = Tensor::zeros((b, m, k), dtype, device).unwrap(); + + let flops = b * m * k * dtype.size_in_bytes(); + + let mut group = c.benchmark_group(device.bench_name(name)); + group.throughput(Throughput::Bytes(flops as u64)); + group.bench_function("iter", move |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + for _i in 0..iters { + run(black_box(&tensor)); + } + device.sync().unwrap(); + start.elapsed() + }) + }); + group.finish(); +} + +fn criterion_benchmark(c: &mut Criterion) { + let handler = BenchDeviceHandler::new().unwrap(); + for device in handler.devices { + run_affine_benchmark(c, &device, DType::F32, "affine_f32"); + run_affine_benchmark(c, &device, DType::F16, "affine_f16"); + run_affine_benchmark(c, &device, DType::BF16, "affine_bf16"); + #[cfg(not(feature = "metal"))] + run_affine_benchmark(c, &device, DType::F8E4M3, "affine_fp8"); + } +} + +criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/binary.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/binary.rs new file mode 100644 index 0000000000..a4953c6332 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/binary.rs @@ -0,0 +1,57 @@ +use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; +use candle_core::{DType, Device, Tensor}; +use criterion::{criterion_group, Criterion, Throughput}; +use std::hint::black_box; +use std::time::Instant; + +fn run(lhs: &Tensor, rhs: &Tensor) -> Tensor { + lhs.mul(rhs).unwrap() +} + +fn run_unary_benchmark(c: &mut Criterion, device: &Device, dtype: DType, name: &str) { + let b = 1; + let m = 1024; + let k = 1024; + + let lhs = Tensor::arange(0.0f32, (b * m * k) as f32, device) + .unwrap() + .to_dtype(dtype) + .unwrap() + .reshape((b, m, k)) + .unwrap(); + + let rhs = Tensor::arange(0.0f32, (b * m * k) as f32, device) + .unwrap() + .to_dtype(dtype) + .unwrap() + .reshape((b, m, k)) + .unwrap(); + + let flops = 2 * b * m * k * dtype.size_in_bytes(); + + let mut group = c.benchmark_group(device.bench_name(name)); + group.throughput(Throughput::Bytes(flops as u64)); + group.bench_function("iter", move |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + for _i in 0..iters { + run(black_box(&lhs), black_box(&rhs)); + } + device.sync().unwrap(); + start.elapsed() + }) + }); + group.finish(); +} + +fn criterion_benchmark(c: &mut Criterion) { + let handler = BenchDeviceHandler::new().unwrap(); + for device in handler.devices { + for dtype in [DType::F32, DType::BF16, DType::F16] { + let name = format!("binary_mul_{dtype:?}"); + run_unary_benchmark(c, &device, dtype, &name); + } + } +} + +criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/broadcast.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/broadcast.rs new file mode 100644 index 0000000000..020ed5b259 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/broadcast.rs @@ -0,0 +1,47 @@ +use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; +use candle_core::{DType, Device, Tensor}; +use criterion::{criterion_group, Criterion, Throughput}; +use std::hint::black_box; +use std::time::Instant; + +fn run(w: &Tensor, bias: &Tensor) { + w.broadcast_add(bias).unwrap(); +} + +fn run_benchmark(c: &mut Criterion, device: &Device, dtype: DType, name: &str) { + // We simulate a candle-nn style conv2d + bias forward pass. + let batch_size = 1; + let ch = 1; + let m = 126; + let bias_size = 128; + + let x = Tensor::ones((batch_size, ch, m, m), dtype, device).unwrap(); + let bias = Tensor::ones((1, bias_size, 1, 1), dtype, device).unwrap(); + + let flops = batch_size * ch * m * bias_size * dtype.size_in_bytes(); + + let mut group = c.benchmark_group(device.bench_name(name)); + group.throughput(Throughput::Bytes(flops as u64)); + group.bench_function("iter", move |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + for _i in 0..iters { + run(black_box(&x), black_box(&bias)); + } + device.sync().unwrap(); + start.elapsed() + }) + }); + group.finish(); +} + +fn criterion_benchmark(c: &mut Criterion) { + let handler = BenchDeviceHandler::new().unwrap(); + for device in handler.devices { + run_benchmark(c, &device, DType::F32, "broadcast_add_f32"); + run_benchmark(c, &device, DType::F16, "broadcast_add_f16"); + run_benchmark(c, &device, DType::BF16, "broadcast_add_bf16"); + } +} + +criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/conv_transpose2d.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/conv_transpose2d.rs new file mode 100644 index 0000000000..7bd5c5bf06 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/conv_transpose2d.rs @@ -0,0 +1,60 @@ +use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; +use candle_core::{DType, Device, Tensor}; +use criterion::{criterion_group, Criterion, Throughput}; +use std::hint::black_box; +use std::time::Instant; + +fn run( + x: &Tensor, + k: &Tensor, + padding: usize, + output_padding: usize, + stride: usize, + dilation: usize, +) { + x.conv_transpose2d(k, padding, output_padding, stride, dilation) + .unwrap(); +} + +fn run_benchmark(c: &mut Criterion, device: &Device, dtype: DType, name: &str) { + let t = Tensor::arange(0.0f32, 10000.0, device) + .unwrap() + .reshape((1, 4, 50, 50)) + .unwrap() + .to_dtype(dtype) + .unwrap(); + + let kernel = Tensor::arange(0.0f32, 100.0, device) + .unwrap() + .reshape((4, 1, 5, 5)) + .unwrap() + .to_dtype(dtype) + .unwrap(); + + let flops = t.dims().iter().product::() * dtype.size_in_bytes(); + + let mut group = c.benchmark_group(device.bench_name(name)); + group.throughput(Throughput::Bytes(flops as u64)); + group.bench_function("iter", move |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + for _i in 0..iters { + run(black_box(&t), black_box(&kernel), 1, 0, 1, 2); + } + device.sync().unwrap(); + start.elapsed() + }) + }); + group.finish(); +} + +fn criterion_benchmark(c: &mut Criterion) { + let handler = BenchDeviceHandler::new().unwrap(); + for device in handler.devices { + run_benchmark(c, &device, DType::F32, "conv_transpose2d_f32"); + run_benchmark(c, &device, DType::F16, "conv_transpose2d_f16"); + run_benchmark(c, &device, DType::BF16, "conv_transpose2d_bf16"); + } +} + +criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/copy.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/copy.rs new file mode 100644 index 0000000000..00eff3dca6 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/copy.rs @@ -0,0 +1,39 @@ +use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; +use candle_core::{Device, Tensor, WithDType}; +use criterion::{criterion_group, Criterion, Throughput}; +use std::hint::black_box; +use std::time::Instant; + +fn run_copy_mask_benchmark(c: &mut Criterion, device: &Device, name: &str) { + let batch_size = 128; + let in_seq_len = 1; + let kv_seq_len = 1024; + + let attn_mask = vec![vec![vec![D::zero(); kv_seq_len]; in_seq_len]; batch_size]; + let size_in_bytes = batch_size * in_seq_len * kv_seq_len * D::DTYPE.size_in_bytes(); + + let mut group = c.benchmark_group(device.bench_name(name)); + group.throughput(Throughput::Bytes(size_in_bytes as u64)); + group.bench_function("iter", move |b| { + b.iter_custom(|iters| { + let attn_masks = vec![attn_mask.clone(); iters as usize]; + let start = Instant::now(); + for attn_mask in attn_masks.into_iter() { + let tensor = Tensor::new(black_box(attn_mask), device).unwrap(); + black_box(tensor); + } + device.sync().unwrap(); + start.elapsed() + }) + }); + group.finish(); +} + +fn criterion_benchmark(c: &mut Criterion) { + let handler = BenchDeviceHandler::new().unwrap(); + for device in handler.devices { + run_copy_mask_benchmark::(c, &device, "copy_mask"); + } +} + +criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/matmul.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/matmul.rs new file mode 100644 index 0000000000..08732d2b32 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/matmul.rs @@ -0,0 +1,70 @@ +use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; +use candle_core::{DType, Device, Tensor}; +use criterion::{criterion_group, Criterion, Throughput}; +use std::hint::black_box; +use std::time::Instant; + +/// Matmul benchmark shapes covering common GEMM scenarios +const MATMUL_SHAPES: &[(&str, &[usize], &[usize])] = &[ + // Original GEMV test + ("gemv", &[1, 1, 2048], &[1, 2048, 2048]), + // 4D Attention scenarios (multi-head attention) + ("attn_4d_small", &[484, 6, 144, 32], &[484, 6, 32, 144]), + ("attn_4d_large", &[121, 24, 144, 32], &[121, 24, 32, 144]), + // Square matrix tests + ("square_512", &[512, 512], &[512, 512]), + ("square_1024", &[1024, 1024], &[1024, 1024]), + // 3D Batch matmul (attention patterns) + ("batch_1000", &[1000, 144, 32], &[1000, 32, 144]), + // 2D Linear layer scenarios (transformer FFN) + ("linear_large", &[17424, 768], &[768, 3072]), +]; + +fn run(a: &Tensor, b: &Tensor) { + a.broadcast_matmul(b).unwrap(); +} + +fn calculate_flops(shape_a: &[usize], shape_b: &[usize]) -> usize { + let batch: usize = shape_a + .iter() + .take(shape_a.len().saturating_sub(2)) + .product(); + let batch = if batch == 0 { 1 } else { batch }; + let m = shape_a[shape_a.len() - 2]; + let k = shape_a[shape_a.len() - 1]; + let n = shape_b[shape_b.len() - 1]; + 2 * batch * m * k * n +} + +fn run_bench(c: &mut Criterion, device: &Device, name: &str, shape_a: &[usize], shape_b: &[usize]) { + let dtype = DType::F32; + let lhs = Tensor::zeros(shape_a, dtype, device).unwrap(); + let rhs = Tensor::zeros(shape_b, dtype, device).unwrap(); + + let flops = calculate_flops(shape_a, shape_b); + + let mut group = c.benchmark_group(device.bench_name(format!("matmul_{name}"))); + group.throughput(Throughput::Bytes(flops as u64)); + group.bench_function("iter", move |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + for _i in 0..iters { + run(black_box(&lhs), black_box(&rhs)); + } + device.sync().unwrap(); + start.elapsed() + }) + }); + group.finish(); +} + +fn criterion_benchmark(c: &mut Criterion) { + let handler = BenchDeviceHandler::new().unwrap(); + for device in handler.devices { + for (name, shape_a, shape_b) in MATMUL_SHAPES { + run_bench(c, &device, name, shape_a, shape_b); + } + } +} + +criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/mod.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/mod.rs new file mode 100644 index 0000000000..9cc6767a4d --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/mod.rs @@ -0,0 +1,77 @@ +pub(crate) mod affine; +pub(crate) mod binary; +pub(crate) mod broadcast; +pub(crate) mod conv_transpose2d; +pub(crate) mod copy; +pub(crate) mod matmul; +pub(crate) mod qmatmul; +pub(crate) mod random; +pub(crate) mod reduce; +pub(crate) mod unary; +pub(crate) mod where_cond; + +use candle_core::{Device, Result}; + +pub(crate) trait BenchDevice { + fn sync(&self) -> Result<()>; + + fn bench_name>(&self, name: S) -> String; +} + +impl BenchDevice for Device { + fn sync(&self) -> Result<()> { + match self { + Device::Cpu => Ok(()), + Device::Cuda(device) => { + #[cfg(feature = "cuda")] + { + use candle_core::backend::BackendDevice; + return Ok(device.synchronize()?); + } + #[cfg(not(feature = "cuda"))] + panic!("Cuda device without cuda feature enabled: {device:?}") + } + Device::Metal(device) => { + #[cfg(feature = "metal")] + return device.wait_until_completed(); + #[cfg(not(feature = "metal"))] + panic!("Metal device without metal feature enabled: {device:?}") + } + } + } + + fn bench_name>(&self, name: S) -> String { + match self { + Device::Cpu => { + let cpu_type = if cfg!(feature = "accelerate") { + "accelerate" + } else if cfg!(feature = "mkl") { + "mkl" + } else { + "cpu" + }; + format!("{}_{}", cpu_type, name.into()) + } + Device::Cuda(_) => format!("cuda_{}", name.into()), + Device::Metal(_) => format!("metal_{}", name.into()), + } + } +} + +struct BenchDeviceHandler { + devices: Vec, +} + +impl BenchDeviceHandler { + pub fn new() -> Result { + let mut devices = Vec::new(); + if cfg!(feature = "metal") { + devices.push(Device::new_metal(0)?); + } else if cfg!(feature = "cuda") { + devices.push(Device::new_cuda(0)?); + } else { + devices.push(Device::Cpu); + } + Ok(Self { devices }) + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/qmatmul.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/qmatmul.rs new file mode 100644 index 0000000000..be1d2ad021 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/qmatmul.rs @@ -0,0 +1,73 @@ +use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; +use candle_core::{ + quantized::{self, GgmlDType, QMatMul}, + Device, Module, Tensor, +}; +use criterion::{criterion_group, Criterion, Throughput}; +use std::hint::black_box; +use std::time::Instant; + +fn run(matmul: &QMatMul, x: &Tensor) { + matmul.forward(x).unwrap(); +} + +fn run_bench(c: &mut Criterion, device: &Device, dtype: GgmlDType) { + let b = 1; + let m = 1; + let n = 1024; + let k = 1024; + + let lhs = (0..(m * k)) + .map(|v| v as f32 / (m * k) as f32) + .collect::>(); + let rhs = (0..(k * n)) + .map(|v| v as f32 / (n * k) as f32) + .collect::>(); + + let lhs = Tensor::from_slice(&lhs, (m, k), device).unwrap(); + let rhs = Tensor::from_slice(&rhs, (k, n), device).unwrap(); + + let qtensor = quantized::QTensor::quantize(&rhs.t().unwrap(), dtype).unwrap(); + let matmul = quantized::QMatMul::from_qtensor(qtensor).unwrap(); + + let flops = b * m * n * k; + + let mut group = c.benchmark_group(device.bench_name(format!("qmatmul_{dtype:?}"))); + group.sample_size(200); + group.throughput(Throughput::Bytes(flops as u64)); + group.bench_function("iter", move |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + for _i in 0..iters { + run(black_box(&matmul), black_box(&lhs)); + } + device.sync().unwrap(); + start.elapsed() + }) + }); + group.finish(); +} + +fn criterion_benchmark(c: &mut Criterion) { + let handler = BenchDeviceHandler::new().unwrap(); + for device in handler.devices { + for dtype in [ + GgmlDType::F32, + GgmlDType::F16, + GgmlDType::Q4_0, + GgmlDType::Q4_1, + GgmlDType::Q5_0, + GgmlDType::Q5_1, + GgmlDType::Q8_0, + GgmlDType::Q2K, + GgmlDType::Q3K, + GgmlDType::Q4K, + GgmlDType::Q5K, + GgmlDType::Q6K, + ] { + run_bench(c, &device, dtype); + } + } +} + +criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/random.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/random.rs new file mode 100644 index 0000000000..365e051c0e --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/random.rs @@ -0,0 +1,64 @@ +use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; +use candle_core::{DType, Device, Tensor}; +use criterion::{criterion_group, Criterion, Throughput}; +use std::hint::black_box; +use std::time::Instant; + +fn rand_uniform(a: &Tensor) { + a.rand_like(-1.0, 123.0).unwrap(); +} + +fn rand_normal(a: &Tensor) { + a.randn_like(100.0, 15.0).unwrap(); +} + +fn run_random_bench(c: &mut Criterion, device: &Device) { + let b = 1; + + let rows = 2048; + let cols = 2048; + + let dtype = DType::F32; + let tensor = Tensor::zeros((b, rows, cols), dtype, device).unwrap(); + + let flops = b * rows * cols * dtype.size_in_bytes(); + + let mut group = c.benchmark_group(device.bench_name("random_uniform")); + group.throughput(Throughput::Bytes(flops as u64)); + group.bench_function("iter", move |benches| { + benches.iter_custom(|iters| { + let start = Instant::now(); + for _i in 0..iters { + rand_uniform(black_box(&tensor)); + } + device.sync().unwrap(); + start.elapsed() + }) + }); + group.finish(); + + let tensor = Tensor::zeros((b, rows, cols), dtype, device).unwrap(); + + let mut group = c.benchmark_group(device.bench_name("random_normal")); + group.throughput(Throughput::Bytes(flops as u64)); + group.bench_function("iter", move |benches| { + benches.iter_custom(|iters| { + let start = Instant::now(); + for _i in 0..iters { + rand_normal(black_box(&tensor)); + } + device.sync().unwrap(); + start.elapsed() + }) + }); + group.finish(); +} + +fn criterion_benchmark(c: &mut Criterion) { + let handler = BenchDeviceHandler::new().unwrap(); + for device in handler.devices { + run_random_bench(c, &device); + } +} + +criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/reduce.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/reduce.rs new file mode 100644 index 0000000000..6020633c00 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/reduce.rs @@ -0,0 +1,159 @@ +use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; +use candle_core::{DType, Device, Tensor}; +use criterion::{criterion_group, Criterion, Throughput}; +use half::{bf16, f16}; +use std::hint::black_box; +use std::time::Instant; + +fn run_sum(a: &Tensor) { + a.sum_keepdim(2).unwrap(); +} +fn run_arg_min(a: &Tensor) { + a.argmin_keepdim(2).unwrap(); +} + +fn criterion_benchmark(c: &mut Criterion) { + let handler = BenchDeviceHandler::new().unwrap(); + let (lo, up) = (-1000.0f32, 1000.0f32); + for device in handler.devices { + run_reduce(c, &device, (lo, up), false); + run_reduce(c, &device, (f16::from_f32(lo), f16::from_f32(up)), false); + run_reduce(c, &device, (bf16::from_f32(lo), bf16::from_f32(up)), false); + + run_arg_reduce(c, &device, (lo, up), false); + run_arg_reduce(c, &device, (f16::from_f32(lo), f16::from_f32(up)), false); + run_arg_reduce(c, &device, (bf16::from_f32(lo), bf16::from_f32(up)), false); + + run_reduce(c, &device, (lo, up), true); + run_reduce(c, &device, (f16::from_f32(lo), f16::from_f32(up)), true); + run_reduce(c, &device, (bf16::from_f32(lo), bf16::from_f32(up)), true); + + run_arg_reduce(c, &device, (lo, up), true); + run_arg_reduce(c, &device, (f16::from_f32(lo), f16::from_f32(up)), true); + run_arg_reduce(c, &device, (bf16::from_f32(lo), bf16::from_f32(up)), true); + } +} + +fn run_reduce( + c: &mut Criterion, + device: &Device, + (lo, up): (T, T), + strided: bool, +) { + let b = 1; + let m = 1024; + let k = 1024; + + let a = if strided { + Tensor::rand(lo, up, (b, m, k), device) + .unwrap() + .transpose(0, 2) + .unwrap() + } else { + Tensor::rand(lo, up, (b, m, k), device).unwrap() + }; + + let flops = b * m * k * T::DTYPE.size_in_bytes(); + + let name = match T::DTYPE { + DType::F32 => { + if strided { + "reduce_f32_strided" + } else { + "reduce_f32" + } + } + DType::F16 => { + if strided { + "reduce_f16_strided" + } else { + "reduce_f16" + } + } + DType::BF16 => { + if strided { + "reduce_bf16_strided" + } else { + "reduce_bf16" + } + } + _ => "unknown", + }; + + let mut group = c.benchmark_group(device.bench_name(name)); + group.throughput(Throughput::Bytes(flops as u64)); + group.bench_function("iter", move |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + for _i in 0..iters { + run_sum(black_box(&a)); + } + device.sync().unwrap(); + start.elapsed() + }) + }); + group.finish(); +} + +fn run_arg_reduce( + c: &mut Criterion, + device: &Device, + (lo, up): (T, T), + strided: bool, +) { + let b = 1; + let m = 1024; + let k = 1024; + + let a = if strided { + Tensor::rand(lo, up, (b, m, k), device) + .unwrap() + .transpose(0, 2) + .unwrap() + } else { + Tensor::rand(lo, up, (b, m, k), device).unwrap() + }; + + let flops = b * m * k * T::DTYPE.size_in_bytes(); + + let name = match T::DTYPE { + DType::F32 => { + if strided { + "arg_reduce_f32_strided" + } else { + "arg_reduce_f32" + } + } + DType::F16 => { + if strided { + "arg_reduce_f16_strided" + } else { + "arg_reduce_f16" + } + } + DType::BF16 => { + if strided { + "arg_reduce_bf16_strided" + } else { + "arg_reduce_bf16" + } + } + _ => "unknown", + }; + + let mut group = c.benchmark_group(device.bench_name(name)); + group.throughput(Throughput::Bytes(flops as u64)); + group.bench_function("iter", move |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + for _i in 0..iters { + run_arg_min(black_box(&a)); + } + device.sync().unwrap(); + start.elapsed() + }) + }); + group.finish(); +} + +criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/unary.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/unary.rs new file mode 100644 index 0000000000..65723bb3fd --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/unary.rs @@ -0,0 +1,98 @@ +use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; +use candle_core::{DType, Device, Tensor}; +use criterion::{criterion_group, Criterion, Throughput}; +use std::hint::black_box; +use std::time::Instant; + +fn run_sqrt(a: &Tensor) { + a.sqrt().unwrap(); +} + +fn run_unary_benchmark(c: &mut Criterion, device: &Device, dtype: DType, name: &str) { + let b = 1; + let m = 1024; + let k = 1024; + + let tensor = Tensor::arange(0.0f32, (b * m * k) as f32, device) + .unwrap() + .to_dtype(dtype) + .unwrap() + .reshape((b, m, k)) + .unwrap(); + + let flops = b * m * k * dtype.size_in_bytes(); + + let mut group = c.benchmark_group(device.bench_name(name)); + group.throughput(Throughput::Bytes(flops as u64)); + group.bench_function("iter", move |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + for _i in 0..iters { + run_sqrt(black_box(&tensor)); + } + device.sync().unwrap(); + start.elapsed() + }) + }); + group.finish(); +} + +fn run_cast(a: &Tensor, dtype: DType) { + a.to_dtype(dtype).unwrap(); +} + +fn run_cast_benchmark( + c: &mut Criterion, + device: &Device, + dtype: DType, + to_dtype: DType, + name: &str, +) { + let b = 1; + let m = 1024; + let k = 1024; + + let tensor = Tensor::arange(0.0f32, (b * m * k) as f32, device) + .unwrap() + .to_dtype(dtype) + .unwrap() + .reshape((b, m, k)) + .unwrap(); + + let flops = b * m * k * dtype.size_in_bytes(); + + let mut group = c.benchmark_group(device.bench_name(name)); + group.throughput(Throughput::Bytes(flops as u64)); + group.bench_function("iter", move |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + for _i in 0..iters { + run_cast(black_box(&tensor), black_box(to_dtype)); + } + device.sync().unwrap(); + start.elapsed() + }) + }); + group.finish(); +} + +fn criterion_benchmark(c: &mut Criterion) { + let handler = BenchDeviceHandler::new().unwrap(); + for device in handler.devices { + for dtype in [DType::F32, DType::BF16, DType::F16] { + let to_dtype = if matches!(dtype, DType::F32) { + DType::F16 + } else { + DType::F32 + }; + let name = format!("cast_{}_{}", dtype.as_str(), to_dtype.as_str()); + run_cast_benchmark(c, &device, dtype, to_dtype, &name); + } + for dtype in [DType::F32, DType::BF16, DType::F16] { + let name = format!("sqrt_{dtype:?}"); + run_unary_benchmark(c, &device, dtype, &name); + } + } +} + +criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/where_cond.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/where_cond.rs new file mode 100644 index 0000000000..348c1d8cba --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/where_cond.rs @@ -0,0 +1,65 @@ +use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; +use candle_core::{DType, Device, Tensor}; +use criterion::{criterion_group, Criterion, Throughput}; +use std::hint::black_box; +use std::time::Instant; + +fn run(a: &Tensor, b: &Tensor, c: &Tensor) { + a.where_cond(b, c).unwrap(); +} + +const fn create_cond_arr() -> [u8; N] { + let mut arr = [0u8; N]; + let mut i = 0; + while i < N { + arr[i] = (i % 2) as u8; + i += 1; + } + arr +} + +const B: usize = 1; +const M: usize = 1024; +const K: usize = 1024; +const SIZE: usize = B * M * K; + +static DATA: [u8; SIZE] = create_cond_arr::(); + +fn run_where_cond_benchmark(c: &mut Criterion, device: &Device, dtype: DType, name: &str) { + let tensor = Tensor::from_slice(DATA.as_slice(), (B, M, K), device).unwrap(); + let on_true = Tensor::ones((B, M, K), dtype, device).unwrap(); + let on_false = Tensor::zeros((B, M, K), dtype, device).unwrap(); + + let elements = B * M * K; + // E.g. 2 f32 tensors + 1 u8 tensor + let flops = (2 * elements * dtype.size_in_bytes()) + elements; + + let mut group = c.benchmark_group(device.bench_name(name)); + group.throughput(Throughput::Bytes(flops as u64)); + group.bench_function("iter", move |b| { + b.iter_custom(|iters| { + let start = Instant::now(); + for _i in 0..iters { + run( + black_box(&tensor), + black_box(&on_true), + black_box(&on_false), + ); + } + device.sync().unwrap(); + start.elapsed() + }) + }); + group.finish(); +} + +fn criterion_benchmark(c: &mut Criterion) { + let device = BenchDeviceHandler::new().unwrap(); + for d in device.devices { + run_where_cond_benchmark(c, &d, DType::F32, "where_cond_f32"); + run_where_cond_benchmark(c, &d, DType::BF16, "where_cond_bf16"); + run_where_cond_benchmark(c, &d, DType::F16, "where_cond_f16"); + } +} + +criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.9.2/examples/basics.rs b/v2/dependency-patches/candle-core-0.9.2/examples/basics.rs new file mode 100644 index 0000000000..e991403441 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/examples/basics.rs @@ -0,0 +1,20 @@ +#[cfg(feature = "mkl")] +extern crate intel_mkl_src; + +#[cfg(feature = "accelerate")] +extern crate accelerate_src; + +use anyhow::Result; +use candle_core::{Device, Tensor}; + +fn main() -> Result<()> { + let a = Tensor::new(&[[0.0f32, 1.0, 2.0], [3.0, 4.0, 5.0]], &Device::Cpu)?; + let b = Tensor::new(&[[88.0f32], [99.0]], &Device::Cpu)?; + let new_a = a.slice_scatter(&b, 1, 2)?; + assert_eq!(a.to_vec2::()?, [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + assert_eq!( + new_a.to_vec2::()?, + [[0.0, 1.0, 88.0], [3.0, 4.0, 99.0]] + ); + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.9.2/examples/cuda_basics.rs b/v2/dependency-patches/candle-core-0.9.2/examples/cuda_basics.rs new file mode 100644 index 0000000000..4eadcdeb82 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/examples/cuda_basics.rs @@ -0,0 +1,23 @@ +#[cfg(feature = "accelerate")] +extern crate accelerate_src; + +#[cfg(feature = "mkl")] +extern crate intel_mkl_src; + +use anyhow::Result; +use candle_core::{Device, Tensor}; +// xs: [1024, 64, 1924], c Tensor[dims 128, 64, 8; f32, cuda:0] Conv1dConfig { padding: 0, stride: 4, dilation: 1, groups: 1 } +fn main() -> Result<()> { + let device = Device::new_cuda(0)?; + let x = Tensor::randn(0f32, 1.0, (1024, 64, 1924), &device)?; + let c = Tensor::randn(0f32, 1.0, (128, 64, 8), &device)?; + let _x1 = x.conv1d(&c, 0, 4, 1, 1)?; + drop(_x1); + for _ in 0..20 { + let start_time = std::time::Instant::now(); + let _x1 = x.conv1d(&c, 0, 4, 1, 1)?; + device.synchronize()?; + println!("conv1d: {:?}", start_time.elapsed()); + } + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.9.2/examples/cuda_sum_benchmark.rs b/v2/dependency-patches/candle-core-0.9.2/examples/cuda_sum_benchmark.rs new file mode 100644 index 0000000000..5bd4b4eefe --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/examples/cuda_sum_benchmark.rs @@ -0,0 +1,54 @@ +#[cfg(feature = "mkl")] +extern crate intel_mkl_src; + +#[cfg(feature = "accelerate")] +extern crate accelerate_src; + +use std::str::FromStr; + +use anyhow::Result; +use candle_core::{Device, Tensor}; + +fn cos_sin(n: usize, device: &Device) -> Result { + let thetas: Vec<_> = (0..n).map(|i| i as f32 / n as f32).collect(); + let xs: Vec<_> = thetas.iter().map(|t| t.cos().abs()).collect(); + let ys: Vec<_> = thetas.iter().map(|t| t.sin().abs()).collect(); + let xs = Tensor::from_vec(xs, (n, 1), device)?; + let ys = Tensor::from_vec(ys, (1, n), device)?; + let ys = Tensor::cat(&[&ys, &ys, &ys, &ys, &ys, &ys], 1)?; + Ok(xs.matmul(&ys)?) +} + +fn main() -> Result<()> { + let device = Device::new_cuda(0)?; + let args = std::env::args().collect::>(); + let n = if args.len() < 2 { + 2000usize + } else { + usize::from_str(&args[1])? + }; + let xys_cpu = cos_sin(n, &Device::Cpu)?; + let xys = cos_sin(n, &device)?; + println!("{xys_cpu:?} {xys:?}"); + let sum_keepdim_cpu = xys_cpu.sum_keepdim(1)?; + println!("{sum_keepdim_cpu}"); + let sum_keepdim = xys.sum_keepdim(1)?; + println!("{sum_keepdim}"); + let start = std::time::Instant::now(); + let n_iters = 100; + let mut v = 0f32; + for _i in 0..n_iters { + let sum_keepdim = xys.sum_keepdim(1)?; + let sum_keepdim = sum_keepdim.sum_keepdim(0)?; + let sum_keepdim: f32 = sum_keepdim.reshape(&[])?.to_scalar()?; + v += sum_keepdim; + } + let elapsed = start.elapsed(); + if v > 0. { + println!( + "ran {n_iters} iterations, time per iter: {:?} ({v})", + elapsed.div_f64(n_iters as f64) + ); + } + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.9.2/examples/metal_basics.rs b/v2/dependency-patches/candle-core-0.9.2/examples/metal_basics.rs new file mode 100644 index 0000000000..3f433d9c26 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/examples/metal_basics.rs @@ -0,0 +1,28 @@ +#[cfg(feature = "accelerate")] +extern crate accelerate_src; + +#[cfg(feature = "mkl")] +extern crate intel_mkl_src; + +use anyhow::Result; +use candle_core::{Device, Tensor}; + +fn main() -> Result<()> { + // This requires the code to be run with MTL_CAPTURE_ENABLED=1 + let device = Device::new_metal(0)?; + let metal_device = match &device { + Device::Metal(m) => m, + _ => anyhow::bail!("unexpected device"), + }; + metal_device.capture("/tmp/candle.gputrace")?; + // This first synchronize ensures that a new command buffer gets created after setting up the + // capture scope. + device.synchronize()?; + let x = Tensor::randn(0f32, 1.0, (128, 128), &device)?; + let x1 = x.add(&x)?; + println!("{x1:?}"); + // This second synchronize ensures that the command buffer gets committed before the end of the + // capture scope. + device.synchronize()?; + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/accelerate.rs b/v2/dependency-patches/candle-core-0.9.2/src/accelerate.rs new file mode 100644 index 0000000000..d371d3b33b --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/accelerate.rs @@ -0,0 +1,476 @@ +#![allow(dead_code)] +use libc::{c_char, c_double, c_float, c_int, c_long, c_ulong}; + +mod ffi { + use super::*; + extern "C" { + // It would be nice to be able to switch to the NEWLAPACK version of the function but this + // seems to trigger some link error. Available function names can be seen here: + // /Library/Developer/CommandLineTools/SDKs/MacOSX13.3.sdk/System/Library/Frameworks/Accelerate.framework/Versions/A/Accelerate.tbd + #[link_name = "sgemm_"] + pub fn sgemm_ffi( + transa: *const c_char, + transb: *const c_char, + m: *const c_int, + n: *const c_int, + k: *const c_int, + alpha: *const c_float, + a: *const c_float, + lda: *const c_int, + b: *const c_float, + ldb: *const c_int, + beta: *const c_float, + c: *mut c_float, + ldc: *const c_int, + ); + #[link_name = "dgemm_"] + pub fn dgemm_ffi( + transa: *const c_char, + transb: *const c_char, + m: *const c_int, + n: *const c_int, + k: *const c_int, + alpha: *const c_double, + a: *const c_double, + lda: *const c_int, + b: *const c_double, + ldb: *const c_int, + beta: *const c_double, + c: *mut c_double, + ldc: *const c_int, + ); + + pub fn vvexpf(dst: *mut c_float, src: *const c_float, len: *const c_int); + pub fn vvexp(dst: *mut c_double, src: *const c_double, len: *const c_int); + pub fn vvsqrtf(dst: *mut c_float, src: *const c_float, len: *const c_int); + pub fn vvsqrt(dst: *mut c_double, src: *const c_double, len: *const c_int); + pub fn vvsinf(dst: *mut c_float, src: *const c_float, len: *const c_int); + pub fn vvsin(dst: *mut c_double, src: *const c_double, len: *const c_int); + pub fn vvcosf(dst: *mut c_float, src: *const c_float, len: *const c_int); + pub fn vvcos(dst: *mut c_double, src: *const c_double, len: *const c_int); + pub fn vvlogf(dst: *mut c_float, src: *const c_float, len: *const c_int); + pub fn vvlog(dst: *mut c_double, src: *const c_double, len: *const c_int); + pub fn vvtanhf(dst: *mut c_float, src: *const c_float, len: *const c_int); + pub fn vvtanh(dst: *mut c_double, src: *const c_double, len: *const c_int); + + pub fn vDSP_vaddD( + _: *const c_double, + _: c_long, + _: *const c_double, + _: c_long, + _: *mut c_double, + _: c_long, + _: c_ulong, + ); + pub fn vDSP_vadd( + _: *const c_float, + _: c_long, + _: *const c_float, + _: c_long, + _: *mut c_float, + _: c_long, + _: c_ulong, + ); + pub fn vDSP_vsubD( + _: *const c_double, + _: c_long, + _: *const c_double, + _: c_long, + _: *mut c_double, + _: c_long, + _: c_ulong, + ); + pub fn vDSP_vsub( + _: *const c_float, + _: c_long, + _: *const c_float, + _: c_long, + _: *mut c_float, + _: c_long, + _: c_ulong, + ); + pub fn vDSP_vmulD( + _: *const c_double, + _: c_long, + _: *const c_double, + _: c_long, + _: *mut c_double, + _: c_long, + _: c_ulong, + ); + pub fn vDSP_vmul( + _: *const c_float, + _: c_long, + _: *const c_float, + _: c_long, + _: *mut c_float, + _: c_long, + _: c_ulong, + ); + pub fn vDSP_vdivD( + _: *const c_double, + _: c_long, + _: *const c_double, + _: c_long, + _: *mut c_double, + _: c_long, + _: c_ulong, + ); + pub fn vDSP_vdiv( + _: *const c_float, + _: c_long, + _: *const c_float, + _: c_long, + _: *mut c_float, + _: c_long, + _: c_ulong, + ); + pub fn vDSP_vminD( + _: *const c_double, + _: c_long, + _: *const c_double, + _: c_long, + _: *mut c_double, + _: c_long, + _: c_ulong, + ); + pub fn vDSP_vmin( + _: *const c_float, + _: c_long, + _: *const c_float, + _: c_long, + _: *mut c_float, + _: c_long, + _: c_ulong, + ); + pub fn vDSP_vmaxD( + _: *const c_double, + _: c_long, + _: *const c_double, + _: c_long, + _: *mut c_double, + _: c_long, + _: c_ulong, + ); + pub fn vDSP_vmax( + _: *const c_float, + _: c_long, + _: *const c_float, + _: c_long, + _: *mut c_float, + _: c_long, + _: c_ulong, + ); + } +} + +#[allow(clippy::too_many_arguments)] +#[inline] +pub unsafe fn sgemm( + transa: u8, + transb: u8, + m: i32, + n: i32, + k: i32, + alpha: f32, + a: &[f32], + lda: i32, + b: &[f32], + ldb: i32, + beta: f32, + c: &mut [f32], + ldc: i32, +) { + ffi::sgemm_ffi( + &(transa as c_char), + &(transb as c_char), + &m, + &n, + &k, + &alpha, + a.as_ptr(), + &lda, + b.as_ptr(), + &ldb, + &beta, + c.as_mut_ptr(), + &ldc, + ) +} + +#[allow(clippy::too_many_arguments)] +#[inline] +pub unsafe fn dgemm( + transa: u8, + transb: u8, + m: i32, + n: i32, + k: i32, + alpha: f64, + a: &[f64], + lda: i32, + b: &[f64], + ldb: i32, + beta: f64, + c: &mut [f64], + ldc: i32, +) { + ffi::dgemm_ffi( + &(transa as c_char), + &(transb as c_char), + &m, + &n, + &k, + &alpha, + a.as_ptr(), + &lda, + b.as_ptr(), + &ldb, + &beta, + c.as_mut_ptr(), + &ldc, + ) +} + +#[inline] +pub fn vs_exp(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvexpf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} + +#[inline] +pub fn vd_exp(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvexp(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} + +#[inline] +pub fn vs_sqrt(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvsqrtf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} + +#[inline] +pub fn vd_sqrt(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvsqrt(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} + +#[inline] +pub fn vs_sin(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvsinf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} + +#[inline] +pub fn vd_sin(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvsin(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} +#[inline] +pub fn vs_cos(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvcosf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} + +#[inline] +pub fn vd_cos(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvcos(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} +#[inline] +pub fn vs_tanh(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvtanhf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} + +#[inline] +pub fn vd_tanh(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvtanh(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} + +#[inline] +pub fn vs_ln(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvlogf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} + +#[inline] +pub fn vd_ln(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vvlog(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } +} + +#[inline] +pub fn vs_sqr(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + y.iter_mut().zip(a.iter()).for_each(|(y, a)| *y = *a * *a) +} + +#[inline] +pub fn vd_sqr(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + y.iter_mut().zip(a.iter()).for_each(|(y, a)| *y = *a * *a) +} + +#[inline] +pub fn vs_tanh_inplace(y: &mut [f32]) { + unsafe { ffi::vvtanhf(y.as_mut_ptr(), y.as_ptr(), &(y.len() as i32)) } +} + +#[inline] +pub fn vd_tanh_inplace(y: &mut [f64]) { + unsafe { ffi::vvtanh(y.as_mut_ptr(), y.as_ptr(), &(y.len() as i32)) } +} + +#[inline] +pub fn vs_exp_inplace(y: &mut [f32]) { + unsafe { ffi::vvexpf(y.as_mut_ptr(), y.as_ptr(), &(y.len() as i32)) } +} + +#[inline] +pub fn vd_exp_inplace(y: &mut [f64]) { + unsafe { ffi::vvexp(y.as_mut_ptr(), y.as_ptr(), &(y.len() as i32)) } +} + +#[inline] +pub fn vs_gelu(vs: &[f32], ys: &mut [f32]) { + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = (2.0f32 / std::f32::consts::PI).sqrt() * v * (1.0 + 0.044715 * v * v) + } + vs_tanh_inplace(ys); + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = 0.5 * v * (1.0 + *y) + } +} + +#[inline] +pub fn vd_gelu(vs: &[f64], ys: &mut [f64]) { + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = (2.0f64 / std::f64::consts::PI).sqrt() * v * (1.0 + 0.044715 * v * v) + } + vd_tanh_inplace(ys); + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = 0.5 * v * (1.0 + *y) + } +} + +#[inline] +pub fn vs_silu(vs: &[f32], ys: &mut [f32]) { + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = -v + } + vs_exp_inplace(ys); + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = v / (1.0 + *y) + } +} + +#[inline] +pub fn vd_silu(vs: &[f64], ys: &mut [f64]) { + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = -v + } + vd_exp_inplace(ys); + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = v / (1.0 + *y) + } +} + +macro_rules! binary_op { + ($fn_name:ident, $ty:ty, $accelerate_name:ident) => { + #[inline] + pub fn $fn_name(a: &[$ty], b: &[$ty], y: &mut [$ty]) { + let a_len = a.len(); + let b_len = b.len(); + let y_len = y.len(); + if a_len != y_len || b_len != y_len { + panic!( + "{} a,b,y len mismatch {a_len} {b_len} {y_len}", + stringify!($fn_name) + ); + } + unsafe { + // Weird quirk of accelerate, the rhs comes before the lhs. + ffi::$accelerate_name( + b.as_ptr(), + 1, + a.as_ptr(), + 1, + y.as_mut_ptr(), + 1, + a_len as u64, + ) + } + } + }; +} +binary_op!(vs_add, f32, vDSP_vadd); +binary_op!(vd_add, f64, vDSP_vaddD); +binary_op!(vs_sub, f32, vDSP_vsub); +binary_op!(vd_sub, f64, vDSP_vsubD); +binary_op!(vs_mul, f32, vDSP_vmul); +binary_op!(vd_mul, f64, vDSP_vmulD); +binary_op!(vs_div, f32, vDSP_vdiv); +binary_op!(vd_div, f64, vDSP_vdivD); +binary_op!(vs_max, f32, vDSP_vmax); +binary_op!(vd_max, f64, vDSP_vmaxD); +binary_op!(vs_min, f32, vDSP_vmin); +binary_op!(vd_min, f64, vDSP_vminD); diff --git a/v2/dependency-patches/candle-core-0.9.2/src/backend.rs b/v2/dependency-patches/candle-core-0.9.2/src/backend.rs new file mode 100644 index 0000000000..d8ab2b5629 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/backend.rs @@ -0,0 +1,174 @@ +//! Traits to Define Backend Behavior +//! +use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; +use crate::{CpuStorage, DType, Layout, Result, Shape}; + +pub trait BackendStorage: Sized { + type Device: BackendDevice; + + fn try_clone(&self, _: &Layout) -> Result; + + fn dtype(&self) -> DType; + + fn device(&self) -> &Self::Device; + + // Maybe this should return a Cow instead so that no copy is done on the cpu case. + fn to_cpu_storage(&self) -> Result; + + fn affine(&self, _: &Layout, _: f64, _: f64) -> Result; + + fn powf(&self, _: &Layout, _: f64) -> Result; + + fn elu(&self, _: &Layout, _: f64) -> Result; + + fn reduce_op(&self, _: ReduceOp, _: &Layout, _: &[usize]) -> Result; + + fn cmp(&self, _: CmpOp, _: &Self, _: &Layout, _: &Layout) -> Result; + + fn to_dtype(&self, _: &Layout, _: DType) -> Result; + + fn unary_impl(&self, _: &Layout) -> Result; + + fn binary_impl(&self, _: &Self, _: &Layout, _: &Layout) -> Result; + + fn where_cond(&self, _: &Layout, _: &Self, _: &Layout, _: &Self, _: &Layout) -> Result; + + fn conv1d( + &self, + _l: &Layout, + _kernel: &Self, + _kernel_l: &Layout, + _params: &crate::conv::ParamsConv1D, + ) -> Result; + + fn conv_transpose1d( + &self, + _l: &Layout, + _kernel: &Self, + _kernel_l: &Layout, + _params: &crate::conv::ParamsConvTranspose1D, + ) -> Result; + + fn conv2d( + &self, + _l: &Layout, + _kernel: &Self, + _kernel_l: &Layout, + _params: &crate::conv::ParamsConv2D, + ) -> Result; + + fn conv_transpose2d( + &self, + _l: &Layout, + _kernel: &Self, + _kernel_l: &Layout, + _params: &crate::conv::ParamsConvTranspose2D, + ) -> Result; + + fn avg_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result; + fn max_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result; + fn upsample_nearest1d(&self, _: &Layout, _: usize) -> Result; + fn upsample_nearest2d(&self, _: &Layout, _: usize, _: usize) -> Result; + fn upsample_bilinear2d( + &self, + _: &Layout, + _: usize, + _: usize, + _: bool, + _: Option, + _: Option, + ) -> Result; + + fn gather(&self, _: &Layout, _: &Self, _: &Layout, _: usize) -> Result; + + fn scatter_set( + &mut self, + _: &Layout, + _: &Self, + _: &Layout, + _: &Self, + _: &Layout, + _: usize, + ) -> Result<()>; + + fn scatter_add_set( + &mut self, + _: &Layout, + _: &Self, + _: &Layout, + _: &Self, + _: &Layout, + _: usize, + ) -> Result<()>; + + fn index_select(&self, _: &Self, _: &Layout, _: &Layout, _: usize) -> Result; + fn index_add( + &self, + _: &Layout, + _: &Self, + _: &Layout, + _: &Self, + _: &Layout, + _: usize, + ) -> Result; + + fn matmul( + &self, + _: &Self, + _: (usize, usize, usize, usize), + _: &Layout, + _: &Layout, + ) -> Result; + + fn copy_strided_src(&self, _: &mut Self, _: usize, _: &Layout) -> Result<()>; + + #[allow(clippy::too_many_arguments)] + // Similar to cudaMemcpy2D, though values are in elements and not in bytes. + fn copy2d( + &self, + _: &mut Self, + _d1: usize, + _d2: usize, + _src_stride1: usize, + _dst_stride1: usize, + _src_offset: usize, + _dst_offset: usize, + ) -> Result<()>; + + fn const_set(&mut self, _: crate::scalar::Scalar, _: &Layout) -> Result<()>; +} + +pub trait BackendDevice: Sized + std::fmt::Debug + Clone { + type Storage: BackendStorage; + + // TODO: Make the usize generic and part of a generic DeviceLocation. + fn new(_: usize) -> Result; + + fn location(&self) -> crate::DeviceLocation; + + fn same_device(&self, _: &Self) -> bool; + + fn zeros_impl(&self, _shape: &Shape, _dtype: DType) -> Result; + + /// # Safety + /// This function is unsafe as it doesn't initialize the underlying data store. + /// The caller should ensure that the data is properly initialized as early as possible + /// after this call. + unsafe fn alloc_uninit(&self, _shape: &Shape, _dtype: DType) -> Result; + + fn storage_from_slice(&self, _: &[T]) -> Result; + + fn storage_from_cpu_storage(&self, _: &CpuStorage) -> Result; + + fn storage_from_cpu_storage_owned(&self, _: CpuStorage) -> Result; + + fn rand_uniform(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result; + + fn rand_normal(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result; + + fn set_seed(&self, _: u64) -> Result<()>; + fn get_current_seed(&self) -> Result; + + /// Synchronize should block until all the operations on the device are completed. + fn synchronize(&self) -> Result<()>; +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/backprop.rs b/v2/dependency-patches/candle-core-0.9.2/src/backprop.rs new file mode 100644 index 0000000000..d2310cbe28 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/backprop.rs @@ -0,0 +1,784 @@ +//! Methods for backpropagation of gradients. +use crate::op::{BinaryOp, Op, ReduceOp, UnaryOp}; +use crate::{Error, Result, Tensor, TensorId}; +use std::collections::HashMap; + +// arg has been reduced to node via reduce_dims, expand it back to arg. +// This has to handle keepdims. +fn broadcast_back(arg: &Tensor, node: &Tensor, reduced_dims: &[usize]) -> Result { + if arg.rank() == node.rank() { + // keepdim = true + node.broadcast_as(arg.shape()) + } else { + // keepdim = false + // first expand the reduced dims. + node.reshape(reduced_dims)?.broadcast_as(arg.shape()) + } +} + +thread_local! { + static CANDLE_GRAD_DO_NOT_DETACH: bool = { + match std::env::var("CANDLE_GRAD_DO_NOT_DETACH") { + Ok(s) => { + !s.is_empty() && s != "0" + }, + Err(_) => false, + } + } +} + +impl Tensor { + /// Return all the nodes that lead to this value in a topologically sorted vec, the first + /// elements having dependencies on the latter ones, e.g. the first element if any is the + /// argument. + /// This assumes that the op graph is a DAG. + pub fn sorted_nodes(&self) -> Vec<&Tensor> { + // The vec of sorted nodes is passed as an owned value rather than a mutable reference + // to get around some lifetime limitations. + fn walk<'a>( + node: &'a Tensor, + nodes: Vec<&'a Tensor>, + already_seen: &mut HashMap, + ) -> (bool, Vec<&'a Tensor>) { + if let Some(&tg) = already_seen.get(&node.id()) { + return (tg, nodes); + } + let mut track_grad = false; + let mut nodes = if node.is_variable() { + // Do not call recursively on the "leaf" nodes. + track_grad = true; + nodes + } else if node.dtype().is_int() { + nodes + } else if let Some(op) = node.op() { + match op { + Op::IndexAdd(t1, t2, t3, _) + | Op::Scatter(t1, t2, t3, _) + | Op::ScatterAdd(t1, t2, t3, _) + | Op::CustomOp3(t1, t2, t3, _) + | Op::WhereCond(t1, t2, t3) => { + let (tg, nodes) = walk(t1, nodes, already_seen); + track_grad |= tg; + let (tg, nodes) = walk(t2, nodes, already_seen); + track_grad |= tg; + let (tg, nodes) = walk(t3, nodes, already_seen); + track_grad |= tg; + nodes + } + Op::Conv1D { + arg: lhs, + kernel: rhs, + .. + } + | Op::ConvTranspose1D { + arg: lhs, + kernel: rhs, + .. + } + | Op::Conv2D { + arg: lhs, + kernel: rhs, + .. + } + | Op::ConvTranspose2D { + arg: lhs, + kernel: rhs, + .. + } + | Op::CustomOp2(lhs, rhs, _) + | Op::Binary(lhs, rhs, _) + | Op::Gather(lhs, rhs, _) + | Op::IndexSelect(lhs, rhs, _) + | Op::Matmul(lhs, rhs) + | Op::SliceScatter0(lhs, rhs, _) => { + let (tg, nodes) = walk(lhs, nodes, already_seen); + track_grad |= tg; + let (tg, nodes) = walk(rhs, nodes, already_seen); + track_grad |= tg; + nodes + } + Op::Cat(args, _) => args.iter().fold(nodes, |nodes, arg| { + let (tg, nodes) = walk(arg, nodes, already_seen); + track_grad |= tg; + nodes + }), + Op::Affine { arg, mul, .. } => { + if *mul == 0. { + nodes + } else { + let (tg, nodes) = walk(arg, nodes, already_seen); + track_grad |= tg; + nodes + } + } + Op::Unary(_node, UnaryOp::Ceil) + | Op::Unary(_node, UnaryOp::Floor) + | Op::Unary(_node, UnaryOp::Round) + | Op::Unary(_node, UnaryOp::Sign) => nodes, + Op::Reshape(node) + | Op::UpsampleNearest1D { arg: node, .. } + | Op::UpsampleNearest2D { arg: node, .. } + | Op::UpsampleBilinear2D { arg: node, .. } + | Op::AvgPool2D { arg: node, .. } + | Op::MaxPool2D { arg: node, .. } + | Op::Copy(node) + | Op::Broadcast(node) + | Op::Cmp(node, _) + | Op::Reduce(node, ReduceOp::Min | ReduceOp::Sum | ReduceOp::Max, _) + | Op::ToDevice(node) + | Op::Transpose(node, _, _) + | Op::Permute(node, _) + | Op::Narrow(node, _, _, _) + | Op::Unary(node, _) + | Op::Elu(node, _) + | Op::Powf(node, _) + | Op::CustomOp1(node, _) => { + let (tg, nodes) = walk(node, nodes, already_seen); + track_grad |= tg; + nodes + } + Op::ToDType(node) => { + if node.dtype().is_float() { + let (tg, nodes) = walk(node, nodes, already_seen); + track_grad |= tg; + nodes + } else { + nodes + } + } + Op::Reduce(_, ReduceOp::ArgMin | ReduceOp::ArgMax, _) => nodes, + } + } else { + nodes + }; + already_seen.insert(node.id(), track_grad); + if track_grad { + nodes.push(node); + } + (track_grad, nodes) + } + let (_tg, mut nodes) = walk(self, vec![], &mut HashMap::new()); + nodes.reverse(); + nodes + } + + pub fn backward(&self) -> Result { + let sorted_nodes = self.sorted_nodes(); + let mut grads = GradStore::new(); + grads.insert(self, self.ones_like()?.contiguous()?); + for node in sorted_nodes.iter() { + if node.is_variable() { + continue; + } + let grad = grads + .remove(node) + .expect("candle internal error - grad not populated"); + // https://github.com/huggingface/candle/issues/1241 + // Ideally, we would make these operations in place where possible to ensure that we + // do not have to allocate too often. Here we just call `.detach` to avoid computing + // the backprop graph of the backprop itself. This would be an issue for second order + // derivatives but these are out of scope at the moment. + let do_not_detach = CANDLE_GRAD_DO_NOT_DETACH.with(|b| *b); + let grad = if do_not_detach { grad } else { grad.detach() }; + if let Some(op) = node.op() { + match op { + Op::Binary(lhs, rhs, BinaryOp::Add) => { + let lhs_sum_grad = grads.or_insert(lhs)?; + *lhs_sum_grad = lhs_sum_grad.add(&grad)?; + let rhs_sum_grad = grads.or_insert(rhs)?; + *rhs_sum_grad = rhs_sum_grad.add(&grad)?; + } + Op::Binary(lhs, rhs, BinaryOp::Sub) => { + let lhs_sum_grad = grads.or_insert(lhs)?; + *lhs_sum_grad = lhs_sum_grad.add(&grad)?; + let rhs_sum_grad = grads.or_insert(rhs)?; + *rhs_sum_grad = rhs_sum_grad.sub(&grad)?; + } + Op::Binary(lhs, rhs, BinaryOp::Mul) => { + let lhs_grad = grad.mul(rhs)?; + let lhs_sum_grad = grads.or_insert(lhs)?; + *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?; + let rhs_grad = grad.mul(lhs)?; + let rhs_sum_grad = grads.or_insert(rhs)?; + *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?; + } + Op::Binary(lhs, rhs, BinaryOp::Div) => { + let lhs_grad = grad.div(rhs)?; + let lhs_sum_grad = grads.or_insert(lhs)?; + *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?; + let rhs_grad = grad.mul(lhs)?.div(&rhs.sqr()?)?; + let rhs_sum_grad = grads.or_insert(rhs)?; + *rhs_sum_grad = rhs_sum_grad.sub(&rhs_grad)?; + } + Op::Binary(lhs, rhs, BinaryOp::Minimum) + | Op::Binary(lhs, rhs, BinaryOp::Maximum) => { + let mask_lhs = node.eq(lhs)?.to_dtype(grad.dtype())?; + let mask_rhs = node.eq(rhs)?.to_dtype(grad.dtype())?; + + // If both masks are 1 one the same point, we want to scale the + // gradient by 0.5 rather than 1. + let lhs_grad = mask_lhs.mul(&grad)?.div(&(&mask_rhs + 1.)?)?; + let lhs_sum_grad = grads.or_insert(lhs)?; + *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?; + + let rhs_grad = mask_rhs.mul(&grad)?.div(&(&mask_lhs + 1.)?)?; + let rhs_sum_grad = grads.or_insert(rhs)?; + *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?; + } + Op::WhereCond(pred, t, f) => { + let zeros = grad.zeros_like()?; + let t_sum_grad = grads.or_insert(t)?; + let t_grad = pred.where_cond(&grad, &zeros)?; + *t_sum_grad = t_sum_grad.add(&t_grad)?; + let f_sum_grad = grads.or_insert(f)?; + let f_grad = pred.where_cond(&zeros, &grad)?; + *f_sum_grad = f_sum_grad.add(&f_grad)?; + } + Op::Conv1D { + arg, + kernel, + padding, + stride, + dilation, + } => { + // The output height for conv_transpose1d is: + // (l_in - 1) * stride - 2 * padding + dilation * (k_size - 1) + out_padding + 1 + let grad_l_in = grad.dim(2)?; + let k_size = kernel.dim(2)?; + let out_size = + (grad_l_in - 1) * stride + dilation * (k_size - 1) + 1 - 2 * padding; + let out_padding = arg.dim(2)? - out_size; + let grad_arg = grad.conv_transpose1d( + kernel, + *padding, + out_padding, + *stride, + *dilation, + /* groups */ 1, + )?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&grad_arg)?; + + let grad_kernel = arg + .transpose(0, 1)? + .conv1d(&grad.transpose(0, 1)?, *padding, *dilation, *stride, 1)? + .transpose(0, 1)?; + let sum_grad = grads.or_insert(kernel)?; + let (_, _, k0) = kernel.dims3()?; + let (_, _, g_k0) = grad_kernel.dims3()?; + let grad_kernel = if g_k0 != k0 { + grad_kernel.narrow(2, 0, k0)? + } else { + grad_kernel + }; + *sum_grad = sum_grad.add(&grad_kernel)?; + } + Op::Conv2D { + arg, + kernel, + padding, + stride, + dilation, + } => { + // The output height for conv_transpose2d is: + // (i_h - 1) * stride - 2 * padding + dilation * (k_h - 1) + out_padding + 1 + let grad_h = grad.dim(2)?; + let k_h = kernel.dim(2)?; + let out_size = + (grad_h - 1) * stride + dilation * (k_h - 1) + 1 - 2 * padding; + let out_padding = arg.dim(2)? - out_size; + let grad_arg = grad.conv_transpose2d( + kernel, + *padding, + out_padding, + *stride, + *dilation, + )?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&grad_arg)?; + + let grad_kernel = arg + .transpose(0, 1)? + .conv2d(&grad.transpose(0, 1)?, *padding, *dilation, *stride, 1)? + .transpose(0, 1)?; + let sum_grad = grads.or_insert(kernel)?; + let (_, _, k0, k1) = kernel.dims4()?; + let (_, _, g_k0, g_k1) = grad_kernel.dims4()?; + let grad_kernel = if g_k0 != k0 || g_k1 != k1 { + grad_kernel.narrow(2, 0, k0)?.narrow(3, 0, k1)? + } else { + grad_kernel + }; + *sum_grad = sum_grad.add(&grad_kernel)?; + } + Op::ConvTranspose1D { .. } => Err(Error::BackwardNotSupported { + op: "conv-transpose1d", + })?, + Op::ConvTranspose2D { + arg, + kernel, + padding, + stride, + dilation, + output_padding: _output_padding, + } => { + let grad_arg = grad.conv2d(kernel, *padding, *stride, *dilation, 1)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&grad_arg)?; + + let grad_kernel = grad + .transpose(0, 1)? + .conv2d(&arg.transpose(0, 1)?, *padding, *dilation, *stride, 1)? + .transpose(0, 1)?; + let sum_grad = grads.or_insert(kernel)?; + let (_, _, k0, k1) = kernel.dims4()?; + let (_, _, g_k0, g_k1) = grad_kernel.dims4()?; + let grad_kernel = if g_k0 != k0 || g_k1 != k1 { + grad_kernel.narrow(2, 0, k0)?.narrow(3, 0, k1)? + } else { + grad_kernel + }; + *sum_grad = sum_grad.add(&grad_kernel)?; + } + Op::AvgPool2D { + arg, + kernel_size, + stride, + } => { + if kernel_size != stride { + crate::bail!("backward not supported for avgpool2d if ksize {kernel_size:?} != stride {stride:?}") + } + let (_n, _c, h, w) = arg.dims4()?; + let grad_arg = grad.upsample_nearest2d(h, w)?; + let grad_arg = + (grad_arg * (1f64 / (kernel_size.0 * kernel_size.1) as f64))?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&grad_arg)?; + } + Op::MaxPool2D { + arg, + kernel_size, + stride, + } => { + if kernel_size != stride { + crate::bail!("backward not supported for maxpool2d if ksize {kernel_size:?} != stride {stride:?}") + } + let (_n, _c, h, w) = arg.dims4()?; + // For computing the max-pool gradient, we compute a mask where a 1 means + // that the element is the maximum, then we apply this mask to the + // upsampled gradient (taking into account that multiple max may exist so + // we scale the gradient for this case). + let node_upsampled = node.upsample_nearest2d(h, w)?; + let mask = arg.eq(&node_upsampled)?.to_dtype(arg.dtype())?; + let avg = mask.avg_pool2d_with_stride(*kernel_size, *stride)?; + let grad_arg = ((grad * avg)?.upsample_nearest2d(h, w)? * mask)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&grad_arg)?; + } + Op::UpsampleNearest1D { arg, target_size } => { + let (_n, c, size) = arg.dims3()?; + if target_size % size != 0 { + crate::bail!("backward not supported for non integer upscaling factors") + } + let scale = target_size / size; + + let kernel = Tensor::ones((c, 1, scale), arg.dtype(), arg.device())?; + let conv_sum = grad.conv1d(&kernel, 0, scale, 1, c)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = conv_sum; + } + Op::UpsampleNearest2D { + arg, + target_h, + target_w, + } => { + let (_n, c, h, w) = arg.dims4()?; + if target_h % h != 0 || target_w % w != 0 { + crate::bail!("backward not supported for non integer upscaling factors") + } + let scale_h = target_h / h; + let scale_w = target_w / w; + + if scale_h != scale_w { + crate::bail!("backward not supported for non uniform upscaling factors") + }; + let kernel = + Tensor::ones((c, 1, scale_h, scale_w), arg.dtype(), arg.device())?; + let conv_sum = grad.conv2d(&kernel, 0, scale_h, 1, c)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = conv_sum; + } + Op::UpsampleBilinear2D { .. } => { + crate::bail!("backward not supported for upsample_bilinear2d") + } + Op::SliceScatter0(lhs, rhs, start_rhs) => { + let rhs_sum_grad = grads.or_insert(rhs)?; + let rhs_grad = grad.narrow(0, *start_rhs, rhs.dim(0)?)?; + *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?; + + let lhs_sum_grad = grads.or_insert(lhs)?; + let lhs_grad = grad.slice_scatter0(&rhs.zeros_like()?, *start_rhs)?; + *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)? + } + Op::Gather(arg, indexes, dim) => { + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.scatter_add(indexes, &grad, *dim)?; + } + Op::Scatter(init, indexes, src, dim) => { + let init_sum_grad = grads.or_insert(init)?; + *init_sum_grad = init_sum_grad.add(&grad)?; + + let src_grad = grad.gather(indexes, *dim)?; + let src_sum_grad = grads.or_insert(src)?; + *src_sum_grad = src_sum_grad.add(&src_grad)?; + } + Op::ScatterAdd(init, indexes, src, dim) => { + let init_sum_grad = grads.or_insert(init)?; + let mask = init.ones_like()?; + let mask = mask.scatter(indexes, &mask.zeros_like()?, *dim)?; + *init_sum_grad = init_sum_grad.add(&grad.mul(&mask)?)?; + + let src_grad = grad.gather(indexes, *dim)?; + let src_sum_grad = grads.or_insert(src)?; + *src_sum_grad = src_sum_grad.add(&src_grad)?; + } + Op::IndexAdd(init, indexes, src, dim) => { + let init_sum_grad = grads.or_insert(init)?; + *init_sum_grad = init_sum_grad.add(&grad)?; + + let src_grad = grad.index_select(indexes, *dim)?; + let src_sum_grad = grads.or_insert(src)?; + *src_sum_grad = src_sum_grad.add(&src_grad)?; + } + Op::IndexSelect(arg, indexes, dim) => { + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.index_add(indexes, &grad, *dim)?; + } + Op::Matmul(lhs, rhs) => { + // Skipping checks, the op went ok, we can skip + // the matmul size checks for now. + + let lhs_grad = grad.matmul(&rhs.t()?)?; + let lhs_sum_grad = grads.or_insert(lhs)?; + *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?; + + let rhs_grad = lhs.t()?.matmul(&grad)?; + let rhs_sum_grad = grads.or_insert(rhs)?; + *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?; + } + Op::Cat(args, dim) => { + let mut start_idx = 0; + for arg in args { + let len = arg.dims()[*dim]; + let arg_grad = grad.narrow(*dim, start_idx, len)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&arg_grad)?; + start_idx += len; + } + } + Op::Broadcast(arg) => { + let arg_dims = arg.dims(); + let node_dims = node.dims(); + // The number of dims that have been inserted on the left. + let left_dims = node_dims.len() - arg_dims.len(); + let mut sum_dims: Vec = (0..left_dims).collect(); + for (dim, (node_dim, arg_dim)) in node_dims[left_dims..] + .iter() + .zip(arg_dims.iter()) + .enumerate() + { + if node_dim != arg_dim { + sum_dims.push(dim + left_dims) + } + } + + let mut arg_grad = grad.sum_keepdim(sum_dims.as_slice())?; + for _i in 0..left_dims { + arg_grad = arg_grad.squeeze(0)? + } + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&arg_grad.broadcast_as(sum_grad.dims())?)?; + } + Op::Reduce(arg, ReduceOp::Sum, reduced_dims) => { + let grad = broadcast_back(arg, &grad, reduced_dims)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&grad)?; + } + Op::Reduce(arg, ReduceOp::Max, reduced_dims) => { + let node = broadcast_back(arg, node, reduced_dims)?; + let grad = broadcast_back(arg, &grad, reduced_dims)?; + let grad = node.eq(arg)?.to_dtype(grad.dtype())?.mul(&grad)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&grad.broadcast_as(sum_grad.dims())?)?; + } + Op::Reduce(arg, ReduceOp::Min, reduced_dims) => { + let node = broadcast_back(arg, node, reduced_dims)?; + let grad = broadcast_back(arg, &grad, reduced_dims)?; + let grad = node.eq(arg)?.to_dtype(grad.dtype())?.mul(&grad)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&grad.broadcast_as(sum_grad.dims())?)?; + } + Op::ToDType(arg) => { + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&grad.to_dtype(arg.dtype())?)? + } + Op::Copy(arg) => { + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&grad)? + } + Op::Affine { arg, mul, .. } => { + let arg_grad = grad.affine(*mul, 0.)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&arg_grad)? + } + Op::Unary(arg, UnaryOp::Log) => { + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&(grad / arg)?)? + } + Op::Unary(arg, UnaryOp::Sin) => { + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&(&grad * arg.cos())?)? + } + Op::Unary(arg, UnaryOp::Cos) => { + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.sub(&(&grad * arg.sin())?)? + } + Op::Unary(arg, UnaryOp::Tanh) => { + let sum_grad = grads.or_insert(arg)?; + let minus_dtanh = (node.sqr()? - 1.)?; + *sum_grad = sum_grad.sub(&(&grad * &minus_dtanh)?)? + } + Op::Unary(arg, UnaryOp::Abs) => { + let sum_grad = grads.or_insert(arg)?; + let ones = arg.ones_like()?; + let abs_grad = arg.ge(&arg.zeros_like()?)?.where_cond(&ones, &ones.neg()?); + *sum_grad = sum_grad.add(&(&grad * abs_grad)?)? + } + Op::Unary(arg, UnaryOp::Exp) => { + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&(&grad * *node)?)? + } + Op::Unary(arg, UnaryOp::Neg) => { + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.sub(&grad)? + } + Op::Unary(arg, UnaryOp::Recip) => { + let sum_grad = grads.or_insert(arg)?; + let grad = (grad / arg.sqr()?)?; + *sum_grad = sum_grad.sub(&grad)? + } + &Op::Narrow(ref arg, dim, start_idx, len) => { + let arg_dims = arg.dims(); + let left_pad = if start_idx == 0 { + None + } else { + let mut dims = arg_dims.to_vec(); + dims[dim] = start_idx; + Some(Tensor::zeros(dims, grad.dtype(), grad.device())?) + }; + let right_pad = arg_dims[dim] - start_idx - len; + let right_pad = if right_pad == 0 { + None + } else { + let mut dims = arg_dims.to_vec(); + dims[dim] = right_pad; + Some(Tensor::zeros(dims, grad.dtype(), grad.device())?) + }; + let arg_grad = match (left_pad, right_pad) { + (None, None) => grad, + (Some(l), None) => Tensor::cat(&[&l, &grad], dim)?, + (None, Some(r)) => Tensor::cat(&[&grad, &r], dim)?, + (Some(l), Some(r)) => Tensor::cat(&[&l, &grad, &r], dim)?, + }; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&arg_grad)? + } + Op::Unary(_, UnaryOp::Floor) + | Op::Unary(_, UnaryOp::Round) + | Op::Reduce(_, ReduceOp::ArgMin, _) + | Op::Reduce(_, ReduceOp::ArgMax, _) + | Op::Unary(_, UnaryOp::Sign) + | Op::Cmp(_, _) => {} + Op::Reshape(arg) => { + let arg_grad = grad.reshape(arg.dims())?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&arg_grad)? + } + Op::Unary(_, UnaryOp::Ceil) => Err(Error::BackwardNotSupported { op: "ceil" })?, + Op::Unary(arg, UnaryOp::Gelu) => { + let sum_grad = grads.or_insert(arg)?; + let cube = arg.powf(3.)?; + let tanh = (0.0356774 * &cube + (0.797885 * arg)?)?.tanh()?; + let gelu_grad = (((0.5 * &tanh)? + + (0.0535161 * cube + (0.398942 * arg)?)? * (1. - tanh.powf(2.)?))? + + 0.5)?; + *sum_grad = sum_grad.add(&(&grad * gelu_grad)?)? + } + Op::Unary(arg, UnaryOp::Erf) => { + let sum_grad = grads.or_insert(arg)?; + // d/dx erf(x) = 2/sqrt(pi) * e^(-x^2) + let erf_grad = + (2. / std::f64::consts::PI.sqrt()) * (arg.sqr()?.neg()?).exp()?; + *sum_grad = sum_grad.add(&(&grad * erf_grad)?)? + } + Op::Unary(arg, UnaryOp::GeluErf) => { + let sum_grad = grads.or_insert(arg)?; + // d/dx gelu_erf(x) = 0.5 + 0.398942 e^(-x^2/2) x + 0.5 erf(x/sqrt(2)) + let neg_half_square = (arg.sqr()?.neg()? / 2.)?; + let scaled_exp_arg = (0.398942 * neg_half_square.exp()? * arg)?; + let arg_scaled_sqrt = (arg / 2f64.sqrt())?; + let erf_scaled_sqrt = (0.5 * arg_scaled_sqrt.erf()?)?; + let gelu_erf_grad = (0.5 + scaled_exp_arg + erf_scaled_sqrt)?; + *sum_grad = sum_grad.add(&(&grad * gelu_erf_grad)?)?; + } + Op::Unary(arg, UnaryOp::Relu) => { + let sum_grad = grads.or_insert(arg)?; + let relu_grad = arg.ge(&arg.zeros_like()?)?.to_dtype(arg.dtype())?; + *sum_grad = sum_grad.add(&(&grad * relu_grad)?)? + } + Op::Unary(arg, UnaryOp::Silu) => { + let sum_grad = grads.or_insert(arg)?; + // d/dx silu = sigmoid(x) * (1 + x * (1 - sigmoid(x))) = sigmoid(x) * (1 - node) + node + let sigmoid_arg = (arg.neg()?.exp()? + 1.)?.recip()?; + let silu_grad = &sigmoid_arg * (1. - *node) + *node; + *sum_grad = sum_grad.add(&(&grad * silu_grad)?)? + } + Op::Elu(arg, alpha) => { + // d/dx elu(x) = 1 for x > 0, alpha * e^x for x <= 0 + let sum_grad = grads.or_insert(arg)?; + let zeros = arg.zeros_like()?; + let positive_mask = arg.gt(&zeros)?.to_dtype(arg.dtype())?; + let negative_mask = arg.le(&zeros)?.to_dtype(arg.dtype())?; + // node == alpha * (e^x - 1) for x <= 0, reuse it + let negative_exp_mask = (negative_mask * (*node + *alpha))?; + let combined_mask = (positive_mask + negative_exp_mask)?; + *sum_grad = sum_grad.add(&(grad * combined_mask)?)? + } + Op::Powf(arg, e) => { + let arg_grad = (&(grad * arg.powf(e - 1.)?)? * *e)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&arg_grad)? + } + Op::CustomOp1(arg, c) => { + if let Some(arg_grad) = c.bwd(arg, node, &grad)? { + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&arg_grad)? + } + } + Op::CustomOp2(arg1, arg2, c) => { + let (arg_grad1, arg_grad2) = c.bwd(arg1, arg2, node, &grad)?; + if let Some(arg_grad1) = arg_grad1 { + let sum_grad = grads.or_insert(arg1)?; + *sum_grad = sum_grad.add(&arg_grad1)? + } + if let Some(arg_grad2) = arg_grad2 { + let sum_grad = grads.or_insert(arg2)?; + *sum_grad = sum_grad.add(&arg_grad2)? + } + } + Op::CustomOp3(arg1, arg2, arg3, c) => { + let (arg_grad1, arg_grad2, arg_grad3) = + c.bwd(arg1, arg2, arg3, node, &grad)?; + if let Some(arg_grad1) = arg_grad1 { + let sum_grad = grads.or_insert(arg1)?; + *sum_grad = sum_grad.add(&arg_grad1)? + } + if let Some(arg_grad2) = arg_grad2 { + let sum_grad = grads.or_insert(arg2)?; + *sum_grad = sum_grad.add(&arg_grad2)? + } + if let Some(arg_grad3) = arg_grad3 { + let sum_grad = grads.or_insert(arg3)?; + *sum_grad = sum_grad.add(&arg_grad3)? + } + } + Op::Unary(arg, UnaryOp::Sqr) => { + let arg_grad = arg.mul(&grad)?.affine(2., 0.)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&arg_grad)? + } + Op::Unary(arg, UnaryOp::Sqrt) => { + let arg_grad = grad.div(node)?.affine(0.5, 0.)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&arg_grad)? + } + Op::ToDevice(arg) => { + let sum_grad = grads.or_insert(arg)?; + let arg_grad = grad.to_device(sum_grad.device())?; + *sum_grad = sum_grad.add(&arg_grad)? + } + Op::Transpose(arg, dim1, dim2) => { + let arg_grad = grad.transpose(*dim1, *dim2)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&arg_grad)? + } + Op::Permute(arg, dims) => { + let mut inv_dims = vec![0; dims.len()]; + for (i, &dim_idx) in dims.iter().enumerate() { + inv_dims[dim_idx] = i + } + let arg_grad = grad.permute(inv_dims)?; + let sum_grad = grads.or_insert(arg)?; + *sum_grad = sum_grad.add(&arg_grad)? + } + }; + } + } + Ok(grads) + } +} + +/// A store for gradients, associating a tensor id to the corresponding gradient tensor, used for back propagation. +#[derive(Debug)] +pub struct GradStore(HashMap); + +impl GradStore { + /// Create a new gradient store + fn new() -> Self { + GradStore(HashMap::new()) + } + + /// Get the gradient tensor corresponding to the given tensor id + pub fn get_id(&self, id: TensorId) -> Option<&Tensor> { + self.0.get(&id) + } + + /// Get the gradient tensor associated with the given tensor + pub fn get(&self, tensor: &Tensor) -> Option<&Tensor> { + self.0.get(&tensor.id()) + } + + /// Remove the gradient tensor associated with the given tensor, returning it if it exists + pub fn remove(&mut self, tensor: &Tensor) -> Option { + self.0.remove(&tensor.id()) + } + + /// Insert a gradient tensor associated with the given tensor, returning the previous gradient tensor if it existed + pub fn insert(&mut self, tensor: &Tensor, grad: Tensor) -> Option { + self.0.insert(tensor.id(), grad) + } + + /// Insert a gradient tensor associated with the given tensor id, returning the previous gradient tensor if it existed + pub fn insert_id(&mut self, id: TensorId, grad: Tensor) -> Option { + self.0.insert(id, grad) + } + + /// Get the gradient tensor associated with the given tensor, or, if it does not exist, + /// insert a tensor of zeroes, with the same shape and type as the given tensors and return it + fn or_insert(&mut self, tensor: &Tensor) -> Result<&mut Tensor> { + use std::collections::hash_map::Entry; + let grad = match self.0.entry(tensor.id()) { + Entry::Occupied(entry) => entry.into_mut(), + Entry::Vacant(entry) => { + let grad = tensor.zeros_like()?; + entry.insert(grad) + } + }; + Ok(grad) + } + + /// Get the tensor ids of the stored gradient tensors + pub fn get_ids(&self) -> impl Iterator { + self.0.keys() + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/conv.rs b/v2/dependency-patches/candle-core-0.9.2/src/conv.rs new file mode 100644 index 0000000000..115035ef1c --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/conv.rs @@ -0,0 +1,387 @@ +//! 1D and 2D Convolutions +//! +use crate::{op::BackpropOp, op::Op, Error, Result, Tensor}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParamsConv1D { + pub(crate) b_size: usize, + // Maybe we should have a version without l_in as this bit depends on the input and not only on + // the weights. + pub(crate) l_in: usize, + pub(crate) c_out: usize, + pub(crate) c_in: usize, + pub(crate) k_size: usize, + pub(crate) padding: usize, + pub(crate) stride: usize, + pub(crate) dilation: usize, + pub(crate) cudnn_fwd_algo: Option, +} + +impl ParamsConv1D { + pub(crate) fn l_out(&self) -> usize { + (self.l_in + 2 * self.padding - self.dilation * (self.k_size - 1) - 1) / self.stride + 1 + } + + pub(crate) fn out_dims(&self) -> Vec { + let l_out = self.l_out(); + vec![self.b_size, self.c_out, l_out] + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParamsConvTranspose1D { + pub(crate) b_size: usize, + pub(crate) l_in: usize, + pub(crate) c_out: usize, + pub(crate) c_in: usize, + pub(crate) k_size: usize, + pub(crate) padding: usize, + pub(crate) output_padding: usize, + pub(crate) stride: usize, + pub(crate) dilation: usize, +} + +impl ParamsConvTranspose1D { + pub(crate) fn l_out(&self) -> usize { + (self.l_in - 1) * self.stride - 2 * self.padding + + self.dilation * (self.k_size - 1) + + self.output_padding + + 1 + } + + pub(crate) fn out_dims(&self) -> Vec { + let l_out = self.l_out(); + vec![self.b_size, self.c_out, l_out] + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum CudnnFwdAlgo { + ImplicitGemm, + ImplicitPrecompGemm, + Gemm, + Direct, + Fft, + FftTiling, + Winograd, + WinogradNonFused, + Count, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParamsConv2D { + pub(crate) b_size: usize, + pub(crate) i_h: usize, + pub(crate) i_w: usize, + pub(crate) k_h: usize, + pub(crate) k_w: usize, + pub(crate) c_out: usize, + pub(crate) c_in: usize, + pub(crate) padding: usize, + pub(crate) stride: usize, + pub(crate) dilation: usize, + pub cudnn_fwd_algo: Option, +} + +impl ParamsConv2D { + pub(crate) fn out_h(&self) -> usize { + (self.i_h + 2 * self.padding - self.dilation * (self.k_h - 1) - 1) / self.stride + 1 + } + + pub(crate) fn out_w(&self) -> usize { + (self.i_w + 2 * self.padding - self.dilation * (self.k_w - 1) - 1) / self.stride + 1 + } + + pub(crate) fn out_dims(&self) -> Vec { + vec![self.b_size, self.c_out, self.out_h(), self.out_w()] + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParamsConvTranspose2D { + pub(crate) b_size: usize, + pub(crate) i_h: usize, + pub(crate) i_w: usize, + pub(crate) k_h: usize, + pub(crate) k_w: usize, + pub(crate) c_out: usize, + pub(crate) c_in: usize, + pub(crate) padding: usize, + pub(crate) output_padding: usize, + pub(crate) stride: usize, + pub(crate) dilation: usize, +} + +impl ParamsConvTranspose2D { + pub(crate) fn out_h(&self) -> usize { + (self.i_h - 1) * self.stride + self.dilation * (self.k_h - 1) + self.output_padding + 1 + - 2 * self.padding + } + + pub(crate) fn out_w(&self) -> usize { + (self.i_w - 1) * self.stride + self.dilation * (self.k_w - 1) + self.output_padding + 1 + - 2 * self.padding + } + + pub(crate) fn out_dims(&self) -> Vec { + vec![self.b_size, self.c_out, self.out_h(), self.out_w()] + } +} + +impl Tensor { + fn conv1d_single_group(&self, kernel: &Self, params: &ParamsConv1D) -> Result { + let storage = + self.storage() + .conv1d(self.layout(), &kernel.storage(), kernel.layout(), params)?; + let op = BackpropOp::new2(self, kernel, |arg, kernel| Op::Conv1D { + arg, + kernel, + padding: params.padding, + stride: params.stride, + dilation: params.dilation, + }); + let out_dims = params.out_dims(); + Ok(crate::tensor::from_storage(storage, out_dims, op, false)) + } + + /// Applies a 1D convolution over the input tensor. + pub fn conv1d( + &self, + kernel: &Self, + padding: usize, + stride: usize, + dilation: usize, + groups: usize, + ) -> Result { + self.conv1d_with_algo(kernel, padding, stride, dilation, groups, None) + } + + /// Applies a 1D convolution over the input tensor. + pub fn conv1d_with_algo( + &self, + kernel: &Self, + padding: usize, + stride: usize, + dilation: usize, + groups: usize, + cudnn_fwd_algo: Option, + ) -> Result { + let (c_out, c_in_k, k_size) = kernel.dims3()?; + let (b_size, c_in, l_in) = self.dims3()?; + if c_in != c_in_k * groups { + Err(Error::Conv1dInvalidArgs { + inp_shape: self.shape().clone(), + k_shape: kernel.shape().clone(), + padding, + stride, + msg: "the number of in-channels on the input doesn't match the kernel size", + } + .bt())? + } + + let params = ParamsConv1D { + b_size, + l_in, + c_out: c_out / groups, + c_in: c_in / groups, + k_size, + padding, + stride, + dilation, + cudnn_fwd_algo, + }; + if groups == 1 { + self.conv1d_single_group(kernel, ¶ms) + } else { + let blocks = self.chunk(groups, 1)?; + let kernel = kernel.chunk(groups, 0)?; + let blocks = blocks + .iter() + .zip(&kernel) + .map(|(block, kernel)| block.conv1d_single_group(kernel, ¶ms)) + .collect::>>()?; + Tensor::cat(&blocks, 1) + } + } + + fn conv_transpose1d_single_group( + &self, + kernel: &Self, + params: &ParamsConvTranspose1D, + ) -> Result { + let storage = self.storage().conv_transpose1d( + self.layout(), + &kernel.storage(), + kernel.layout(), + params, + )?; + let op = BackpropOp::new2(self, kernel, |arg, kernel| Op::ConvTranspose1D { + arg, + kernel, + padding: params.padding, + output_padding: params.output_padding, + stride: params.stride, + dilation: params.dilation, + }); + let out_dims = params.out_dims(); + Ok(crate::tensor::from_storage(storage, out_dims, op, false)) + } + + /// Applies a 1D transposed convolution over the input tensor. + pub fn conv_transpose1d( + &self, + kernel: &Self, + padding: usize, + output_padding: usize, + stride: usize, + dilation: usize, + groups: usize, + ) -> Result { + let (c_in_k, c_out, k_size) = kernel.dims3()?; + let (b_size, c_in, l_in) = self.dims3()?; + if c_in != c_in_k { + crate::bail!("in_channel mismatch between input ({c_in}) and kernel ({c_in_k})") + } + if c_in % groups != 0 { + crate::bail!("in_channel {c_in} is not divisible by the number of groups") + } + let params = ParamsConvTranspose1D { + b_size, + l_in, + k_size, + c_out, + c_in: c_in / groups, + padding, + output_padding, + stride, + dilation, + }; + if groups == 1 { + self.conv_transpose1d_single_group(kernel, ¶ms) + } else { + let blocks = self.chunk(groups, 1)?; + let kernel = kernel.chunk(groups, 0)?; + let blocks = blocks + .iter() + .zip(&kernel) + .map(|(block, kernel)| block.conv_transpose1d_single_group(kernel, ¶ms)) + .collect::>>()?; + Tensor::cat(&blocks, 1) + } + } + + fn conv2d_single_group(&self, kernel: &Self, params: &ParamsConv2D) -> Result { + let storage = + self.storage() + .conv2d(self.layout(), &kernel.storage(), kernel.layout(), params)?; + let op = BackpropOp::new2(self, kernel, |arg, kernel| Op::Conv2D { + arg, + kernel, + padding: params.padding, + stride: params.stride, + dilation: params.dilation, + }); + let out_dims = params.out_dims(); + Ok(crate::tensor::from_storage(storage, out_dims, op, false)) + } + + /// Applies a 2D convolution over the input tensor. + pub fn conv2d( + &self, + kernel: &Self, + padding: usize, + stride: usize, + dilation: usize, + groups: usize, + ) -> Result { + self.conv2d_with_algo(kernel, padding, stride, dilation, groups, None) + } + + pub fn conv2d_with_algo( + &self, + kernel: &Self, + padding: usize, + stride: usize, + dilation: usize, + groups: usize, + cudnn_fwd_algo: Option, + ) -> Result { + let (b_size, c_in, i_h, i_w) = self.dims4()?; + let (c_out, c_in_k, k_h, k_w) = kernel.dims4()?; + if c_in != c_in_k * groups { + crate::bail!( + "in_channel mismatch between input ({c_in}, groups {groups}) and kernel ({c_in_k})" + ) + } + let params = ParamsConv2D { + b_size, + i_h, + i_w, + k_h, + k_w, + c_out: c_out / groups, + c_in: c_in / groups, + padding, + stride, + dilation, + cudnn_fwd_algo, + }; + if groups == 1 { + self.conv2d_single_group(kernel, ¶ms) + } else { + let blocks = self.chunk(groups, 1)?; + let kernel = kernel.chunk(groups, 0)?; + let blocks = blocks + .iter() + .zip(&kernel) + .map(|(block, kernel)| block.conv2d_single_group(kernel, ¶ms)) + .collect::>>()?; + Tensor::cat(&blocks, 1) + } + } + + /// Applies a 2D transposed convolution over the input tensor. + pub fn conv_transpose2d( + &self, + kernel: &Self, + padding: usize, + output_padding: usize, + stride: usize, + dilation: usize, + ) -> Result { + let (b_size, c_in, i_h, i_w) = self.dims4()?; + let (c_in_k, c_out, k_h, k_w) = kernel.dims4()?; + if c_in != c_in_k { + crate::bail!("in_channel mismatch between input ({c_in}) and kernel ({c_in_k})") + } + let params = ParamsConvTranspose2D { + b_size, + i_h, + i_w, + k_h, + k_w, + c_out, + c_in, + padding, + output_padding, + stride, + dilation, + }; + let storage = self.storage().conv_transpose2d( + self.layout(), + &kernel.storage(), + kernel.layout(), + ¶ms, + )?; + let op = BackpropOp::new2(self, kernel, |arg, kernel| Op::ConvTranspose2D { + arg, + kernel, + padding: params.padding, + output_padding: params.output_padding, + stride: params.stride, + dilation: params.dilation, + }); + let out_dims = params.out_dims(); + Ok(crate::tensor::from_storage(storage, out_dims, op, false)) + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/convert.rs b/v2/dependency-patches/candle-core-0.9.2/src/convert.rs new file mode 100644 index 0000000000..38e7a7c9a6 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/convert.rs @@ -0,0 +1,166 @@ +//! Implement conversion traits for tensors +use crate::{DType, Device, Error, Tensor, WithDType}; +use half::{bf16, f16, slice::HalfFloatSliceExt}; +use std::convert::TryFrom; + +impl TryFrom<&Tensor> for Vec { + type Error = Error; + fn try_from(tensor: &Tensor) -> Result { + tensor.to_vec1::() + } +} + +impl TryFrom<&Tensor> for Vec> { + type Error = Error; + fn try_from(tensor: &Tensor) -> Result { + tensor.to_vec2::() + } +} + +impl TryFrom<&Tensor> for Vec>> { + type Error = Error; + fn try_from(tensor: &Tensor) -> Result { + tensor.to_vec3::() + } +} + +impl TryFrom for Vec { + type Error = Error; + fn try_from(tensor: Tensor) -> Result { + Vec::::try_from(&tensor) + } +} + +impl TryFrom for Vec> { + type Error = Error; + fn try_from(tensor: Tensor) -> Result { + Vec::>::try_from(&tensor) + } +} + +impl TryFrom for Vec>> { + type Error = Error; + fn try_from(tensor: Tensor) -> Result { + Vec::>>::try_from(&tensor) + } +} + +impl TryFrom<&[T]> for Tensor { + type Error = Error; + fn try_from(v: &[T]) -> Result { + Tensor::from_slice(v, v.len(), &Device::Cpu) + } +} + +impl TryFrom> for Tensor { + type Error = Error; + fn try_from(v: Vec) -> Result { + let len = v.len(); + Tensor::from_vec(v, len, &Device::Cpu) + } +} + +macro_rules! from_tensor { + ($typ:ident) => { + impl TryFrom<&Tensor> for $typ { + type Error = Error; + + fn try_from(tensor: &Tensor) -> Result { + tensor.to_scalar::<$typ>() + } + } + + impl TryFrom for $typ { + type Error = Error; + + fn try_from(tensor: Tensor) -> Result { + $typ::try_from(&tensor) + } + } + + impl TryFrom<$typ> for Tensor { + type Error = Error; + + fn try_from(v: $typ) -> Result { + Tensor::new(v, &Device::Cpu) + } + } + }; +} + +from_tensor!(f64); +from_tensor!(f32); +from_tensor!(f16); +from_tensor!(bf16); +from_tensor!(i64); +from_tensor!(i32); +from_tensor!(i16); +from_tensor!(u32); +from_tensor!(u8); + +impl Tensor { + pub fn write_bytes(&self, f: &mut W) -> crate::Result<()> { + use byteorder::{LittleEndian, WriteBytesExt}; + + let vs = self.flatten_all()?; + match self.dtype() { + DType::BF16 => { + let vs = vs.to_vec1::()?; + for &v in vs.reinterpret_cast() { + f.write_u16::(v)? + } + } + DType::F16 => { + let vs = vs.to_vec1::()?; + for &v in vs.reinterpret_cast() { + f.write_u16::(v)? + } + } + DType::F32 => { + // TODO: Avoid using a buffer when data is already on the CPU. + for v in vs.to_vec1::()? { + f.write_f32::(v)? + } + } + DType::F64 => { + for v in vs.to_vec1::()? { + f.write_f64::(v)? + } + } + DType::U32 => { + for v in vs.to_vec1::()? { + f.write_u32::(v)? + } + } + DType::I16 => { + for v in vs.to_vec1::()? { + f.write_i16::(v)? + } + } + DType::I32 => { + for v in vs.to_vec1::()? { + f.write_i32::(v)? + } + } + DType::I64 => { + for v in vs.to_vec1::()? { + f.write_i64::(v)? + } + } + DType::U8 => { + let vs = vs.to_vec1::()?; + f.write_all(&vs)?; + } + DType::F8E4M3 => { + let vs = vs.to_vec1::()?; + for v in vs { + f.write_u8(v.to_bits())? + } + } + DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { + return Err(crate::Error::UnsupportedDTypeForOp(self.dtype(), "write_bytes").bt()) + } + } + Ok(()) + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cpu/avx.rs b/v2/dependency-patches/candle-core-0.9.2/src/cpu/avx.rs new file mode 100644 index 0000000000..113fc14ced --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/cpu/avx.rs @@ -0,0 +1,227 @@ +use super::{Cpu, CpuBF16, CpuF16}; +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + +use half::{bf16, f16}; + +pub struct CurrentCpu {} + +const STEP: usize = 32; +const EPR: usize = 8; +const ARR: usize = STEP / EPR; + +impl Cpu for CurrentCpu { + type Unit = __m256; + type Array = [__m256; ARR]; + + const STEP: usize = STEP; + const EPR: usize = EPR; + + fn n() -> usize { + ARR + } + + unsafe fn zero() -> Self::Unit { + _mm256_setzero_ps() + } + + unsafe fn zero_array() -> Self::Array { + [Self::zero(); ARR] + } + + unsafe fn from_f32(v: f32) -> Self::Unit { + _mm256_set1_ps(v) + } + + unsafe fn load(mem_addr: *const f32) -> Self::Unit { + _mm256_loadu_ps(mem_addr) + } + + unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit { + _mm256_add_ps(a, b) + } + + unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit { + _mm256_add_ps(_mm256_mul_ps(b, c), a) + } + + unsafe fn vec_store(mem_addr: *mut f32, a: Self::Unit) { + _mm256_storeu_ps(mem_addr, a); + } + + unsafe fn vec_reduce(mut x: Self::Array, y: *mut f32) { + for i in 0..ARR / 2 { + x[2 * i] = _mm256_add_ps(x[2 * i], x[2 * i + 1]); + } + for i in 0..ARR / 4 { + x[4 * i] = _mm256_add_ps(x[4 * i], x[4 * i + 2]); + } + #[allow(clippy::reversed_empty_ranges)] + for i in 0..ARR / 8 { + x[8 * i] = _mm256_add_ps(x[8 * i], x[8 * i + 4]); + } + let t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]), _mm256_extractf128_ps(x[0], 1)); + let t1 = _mm_hadd_ps(t0, t0); + *y = _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); + } +} + +pub struct CurrentCpuF16 {} +impl CpuF16 for CurrentCpuF16 { + type Unit = __m256; + type Array = [__m256; ARR]; + + const STEP: usize = STEP; + const EPR: usize = EPR; + + fn n() -> usize { + ARR + } + + unsafe fn zero() -> Self::Unit { + _mm256_setzero_ps() + } + + unsafe fn zero_array() -> Self::Array { + [Self::zero(); ARR] + } + + unsafe fn from_f32(v: f32) -> Self::Unit { + _mm256_set1_ps(v) + } + + #[cfg(target_feature = "f16c")] + unsafe fn load(mem_addr: *const f16) -> Self::Unit { + _mm256_cvtph_ps(_mm_loadu_si128(mem_addr as *const __m128i)) + } + + #[cfg(not(target_feature = "f16c"))] + unsafe fn load(mem_addr: *const f16) -> Self::Unit { + let mut tmp = [0.0f32; 8]; + for i in 0..8 { + tmp[i] = (*mem_addr.add(i)).to_f32(); + } + _mm256_loadu_ps(tmp.as_ptr()) + } + + unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit { + _mm256_add_ps(a, b) + } + + unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit { + _mm256_add_ps(_mm256_mul_ps(b, c), a) + } + + #[cfg(target_feature = "f16c")] + unsafe fn vec_store(mem_addr: *mut f16, a: Self::Unit) { + _mm_storeu_si128(mem_addr as *mut __m128i, _mm256_cvtps_ph(a, 0)) + } + + #[cfg(not(target_feature = "f16c"))] + unsafe fn vec_store(mem_addr: *mut f16, a: Self::Unit) { + let mut tmp = [0.0f32; 8]; + _mm256_storeu_ps(tmp.as_mut_ptr(), a); + for i in 0..8 { + *mem_addr.add(i) = f16::from_f32(tmp[i]); + } + } + + unsafe fn vec_reduce(mut x: Self::Array, y: *mut f32) { + let mut offset = ARR >> 1; + for i in 0..offset { + x[i] = _mm256_add_ps(x[i], x[offset + i]); + } + offset >>= 1; + for i in 0..offset { + x[i] = _mm256_add_ps(x[i], x[offset + i]); + } + offset >>= 1; + for i in 0..offset { + x[i] = _mm256_add_ps(x[i], x[offset + i]); + } + let t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]), _mm256_extractf128_ps(x[0], 1)); + let t1 = _mm_hadd_ps(t0, t0); + *y = _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); + } +} + +pub struct CurrentCpuBF16 {} +impl CpuBF16 for CurrentCpuBF16 { + type Unit = __m256; + type Array = [__m256; ARR]; + + const STEP: usize = STEP; + const EPR: usize = EPR; + + fn n() -> usize { + ARR + } + + unsafe fn zero() -> Self::Unit { + _mm256_setzero_ps() + } + + unsafe fn zero_array() -> Self::Array { + [Self::zero(); ARR] + } + + unsafe fn from_f32(v: f32) -> Self::Unit { + _mm256_set1_ps(v) + } + + #[cfg(target_feature = "f16c")] + unsafe fn load(mem_addr: *const bf16) -> Self::Unit { + _mm256_cvtph_ps(_mm_loadu_si128(mem_addr as *const __m128i)) + } + + #[cfg(not(target_feature = "f16c"))] + unsafe fn load(mem_addr: *const bf16) -> Self::Unit { + let mut tmp = [0.0f32; 8]; + for i in 0..8 { + tmp[i] = (*mem_addr.add(i)).to_f32(); + } + _mm256_loadu_ps(tmp.as_ptr()) + } + + unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit { + _mm256_add_ps(a, b) + } + + unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit { + _mm256_add_ps(_mm256_mul_ps(b, c), a) + } + + #[cfg(target_feature = "f16c")] + unsafe fn vec_store(mem_addr: *mut bf16, a: Self::Unit) { + _mm_storeu_si128(mem_addr as *mut __m128i, _mm256_cvtps_ph(a, 0)) + } + + #[cfg(not(target_feature = "f16c"))] + unsafe fn vec_store(mem_addr: *mut bf16, a: Self::Unit) { + let mut tmp = [0.0f32; 8]; + _mm256_storeu_ps(tmp.as_mut_ptr(), a); + for i in 0..8 { + *mem_addr.add(i) = bf16::from_f32(tmp[i]); + } + } + + unsafe fn vec_reduce(mut x: Self::Array, y: *mut f32) { + let mut offset = ARR >> 1; + for i in 0..offset { + x[i] = _mm256_add_ps(x[i], x[offset + i]); + } + offset >>= 1; + for i in 0..offset { + x[i] = _mm256_add_ps(x[i], x[offset + i]); + } + offset >>= 1; + for i in 0..offset { + x[i] = _mm256_add_ps(x[i], x[offset + i]); + } + let t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]), _mm256_extractf128_ps(x[0], 1)); + let t1 = _mm_hadd_ps(t0, t0); + *y = _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cpu/erf.rs b/v2/dependency-patches/candle-core-0.9.2/src/cpu/erf.rs new file mode 100644 index 0000000000..4d05728b0d --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/cpu/erf.rs @@ -0,0 +1,325 @@ +#![allow(clippy::excessive_precision)] +// Code taken from https://github.com/statrs-dev/statrs +//! Provides the [error](https://en.wikipedia.org/wiki/Error_function) and +//! related functions + +mod evaluate { + //! Provides functions that don't have a numerical solution and must + //! be solved computationally (e.g. evaluation of a polynomial) + + /// evaluates a polynomial at `z` where `coeff` are the coefficients + /// to a polynomial of order `k` where `k` is the length of `coeff` and the + /// coeffecient + /// to the `k`th power is the `k`th element in coeff. E.g. [3,-1,2] equates to + /// `2z^2 - z + 3` + /// + /// # Remarks + /// + /// Returns 0 for a 0 length coefficient slice + pub fn polynomial(z: f64, coeff: &[f64]) -> f64 { + let n = coeff.len(); + if n == 0 { + return 0.0; + } + + let mut sum = *coeff.last().unwrap(); + for c in coeff[0..n - 1].iter().rev() { + sum = *c + z * sum; + } + sum + } +} +use std::f64; + +/// `erf` calculates the error function at `x`. +pub fn erf_f64(x: f64) -> f64 { + libm::erf(x) +} + +pub fn erf_f32(x: f32) -> f32 { + libm::erff(x) +} + +/// `erf_inv` calculates the inverse error function +/// at `x`. +pub fn erf_inv(x: f64) -> f64 { + if x == 0.0 { + 0.0 + } else if x >= 1.0 { + f64::INFINITY + } else if x <= -1.0 { + f64::NEG_INFINITY + } else if x < 0.0 { + erf_inv_impl(-x, 1.0 + x, -1.0) + } else { + erf_inv_impl(x, 1.0 - x, 1.0) + } +} + +/// `erfc` calculates the complementary error function +/// at `x`. +pub fn erfc_f64(x: f64) -> f64 { + libm::erfc(x) +} + +pub fn erfc_f32(x: f32) -> f32 { + libm::erfcf(x) +} + +/// `erfc_inv` calculates the complementary inverse +/// error function at `x`. +pub fn erfc_inv(x: f64) -> f64 { + if x <= 0.0 { + f64::INFINITY + } else if x >= 2.0 { + f64::NEG_INFINITY + } else if x > 1.0 { + erf_inv_impl(-1.0 + x, 2.0 - x, -1.0) + } else { + erf_inv_impl(1.0 - x, x, 1.0) + } +} + +// ********************************************************** +// ********** Coefficients for erf_inv_impl polynomial ****** +// ********************************************************** + +/// Polynomial coefficients for a numerator of `erf_inv_impl` +/// in the interval [0, 0.5]. +const ERF_INV_IMPL_AN: &[f64] = &[ + -0.000508781949658280665617, + -0.00836874819741736770379, + 0.0334806625409744615033, + -0.0126926147662974029034, + -0.0365637971411762664006, + 0.0219878681111168899165, + 0.00822687874676915743155, + -0.00538772965071242932965, +]; + +/// Polynomial coefficients for a denominator of `erf_inv_impl` +/// in the interval [0, 0.5]. +const ERF_INV_IMPL_AD: &[f64] = &[ + 1.0, + -0.970005043303290640362, + -1.56574558234175846809, + 1.56221558398423026363, + 0.662328840472002992063, + -0.71228902341542847553, + -0.0527396382340099713954, + 0.0795283687341571680018, + -0.00233393759374190016776, + 0.000886216390456424707504, +]; + +/// Polynomial coefficients for a numerator of `erf_inv_impl` +/// in the interval [0.5, 0.75]. +const ERF_INV_IMPL_BN: &[f64] = &[ + -0.202433508355938759655, + 0.105264680699391713268, + 8.37050328343119927838, + 17.6447298408374015486, + -18.8510648058714251895, + -44.6382324441786960818, + 17.445385985570866523, + 21.1294655448340526258, + -3.67192254707729348546, +]; + +/// Polynomial coefficients for a denominator of `erf_inv_impl` +/// in the interval [0.5, 0.75]. +const ERF_INV_IMPL_BD: &[f64] = &[ + 1.0, + 6.24264124854247537712, + 3.9713437953343869095, + -28.6608180499800029974, + -20.1432634680485188801, + 48.5609213108739935468, + 10.8268667355460159008, + -22.6436933413139721736, + 1.72114765761200282724, +]; + +/// Polynomial coefficients for a numerator of `erf_inv_impl` +/// in the interval [0.75, 1] with x less than 3. +const ERF_INV_IMPL_CN: &[f64] = &[ + -0.131102781679951906451, + -0.163794047193317060787, + 0.117030156341995252019, + 0.387079738972604337464, + 0.337785538912035898924, + 0.142869534408157156766, + 0.0290157910005329060432, + 0.00214558995388805277169, + -0.679465575181126350155e-6, + 0.285225331782217055858e-7, + -0.681149956853776992068e-9, +]; + +/// Polynomial coefficients for a denominator of `erf_inv_impl` +/// in the interval [0.75, 1] with x less than 3. +const ERF_INV_IMPL_CD: &[f64] = &[ + 1.0, + 3.46625407242567245975, + 5.38168345707006855425, + 4.77846592945843778382, + 2.59301921623620271374, + 0.848854343457902036425, + 0.152264338295331783612, + 0.01105924229346489121, +]; + +/// Polynomial coefficients for a numerator of `erf_inv_impl` +/// in the interval [0.75, 1] with x between 3 and 6. +const ERF_INV_IMPL_DN: &[f64] = &[ + -0.0350353787183177984712, + -0.00222426529213447927281, + 0.0185573306514231072324, + 0.00950804701325919603619, + 0.00187123492819559223345, + 0.000157544617424960554631, + 0.460469890584317994083e-5, + -0.230404776911882601748e-9, + 0.266339227425782031962e-11, +]; + +/// Polynomial coefficients for a denominator of `erf_inv_impl` +/// in the interval [0.75, 1] with x between 3 and 6. +const ERF_INV_IMPL_DD: &[f64] = &[ + 1.0, + 1.3653349817554063097, + 0.762059164553623404043, + 0.220091105764131249824, + 0.0341589143670947727934, + 0.00263861676657015992959, + 0.764675292302794483503e-4, +]; + +/// Polynomial coefficients for a numerator of `erf_inv_impl` +/// in the interval [0.75, 1] with x between 6 and 18. +const ERF_INV_IMPL_EN: &[f64] = &[ + -0.0167431005076633737133, + -0.00112951438745580278863, + 0.00105628862152492910091, + 0.000209386317487588078668, + 0.149624783758342370182e-4, + 0.449696789927706453732e-6, + 0.462596163522878599135e-8, + -0.281128735628831791805e-13, + 0.99055709973310326855e-16, +]; + +/// Polynomial coefficients for a denominator of `erf_inv_impl` +/// in the interval [0.75, 1] with x between 6 and 18. +const ERF_INV_IMPL_ED: &[f64] = &[ + 1.0, + 0.591429344886417493481, + 0.138151865749083321638, + 0.0160746087093676504695, + 0.000964011807005165528527, + 0.275335474764726041141e-4, + 0.282243172016108031869e-6, +]; + +/// Polynomial coefficients for a numerator of `erf_inv_impl` +/// in the interval [0.75, 1] with x between 18 and 44. +const ERF_INV_IMPL_FN: &[f64] = &[ + -0.0024978212791898131227, + -0.779190719229053954292e-5, + 0.254723037413027451751e-4, + 0.162397777342510920873e-5, + 0.396341011304801168516e-7, + 0.411632831190944208473e-9, + 0.145596286718675035587e-11, + -0.116765012397184275695e-17, +]; + +/// Polynomial coefficients for a denominator of `erf_inv_impl` +/// in the interval [0.75, 1] with x between 18 and 44. +const ERF_INV_IMPL_FD: &[f64] = &[ + 1.0, + 0.207123112214422517181, + 0.0169410838120975906478, + 0.000690538265622684595676, + 0.145007359818232637924e-4, + 0.144437756628144157666e-6, + 0.509761276599778486139e-9, +]; + +/// Polynomial coefficients for a numerator of `erf_inv_impl` +/// in the interval [0.75, 1] with x greater than 44. +const ERF_INV_IMPL_GN: &[f64] = &[ + -0.000539042911019078575891, + -0.28398759004727721098e-6, + 0.899465114892291446442e-6, + 0.229345859265920864296e-7, + 0.225561444863500149219e-9, + 0.947846627503022684216e-12, + 0.135880130108924861008e-14, + -0.348890393399948882918e-21, +]; + +/// Polynomial coefficients for a denominator of `erf_inv_impl` +/// in the interval [0.75, 1] with x greater than 44. +const ERF_INV_IMPL_GD: &[f64] = &[ + 1.0, + 0.0845746234001899436914, + 0.00282092984726264681981, + 0.468292921940894236786e-4, + 0.399968812193862100054e-6, + 0.161809290887904476097e-8, + 0.231558608310259605225e-11, +]; + +// `erf_inv_impl` computes the inverse error function where +// `p`,`q`, and `s` are the first, second, and third intermediate +// parameters respectively +fn erf_inv_impl(p: f64, q: f64, s: f64) -> f64 { + let result = if p <= 0.5 { + let y = 0.0891314744949340820313; + let g = p * (p + 10.0); + let r = evaluate::polynomial(p, ERF_INV_IMPL_AN) / evaluate::polynomial(p, ERF_INV_IMPL_AD); + g * y + g * r + } else if q >= 0.25 { + let y = 2.249481201171875; + let g = (-2.0 * q.ln()).sqrt(); + let xs = q - 0.25; + let r = + evaluate::polynomial(xs, ERF_INV_IMPL_BN) / evaluate::polynomial(xs, ERF_INV_IMPL_BD); + g / (y + r) + } else { + let x = (-q.ln()).sqrt(); + if x < 3.0 { + let y = 0.807220458984375; + let xs = x - 1.125; + let r = evaluate::polynomial(xs, ERF_INV_IMPL_CN) + / evaluate::polynomial(xs, ERF_INV_IMPL_CD); + y * x + r * x + } else if x < 6.0 { + let y = 0.93995571136474609375; + let xs = x - 3.0; + let r = evaluate::polynomial(xs, ERF_INV_IMPL_DN) + / evaluate::polynomial(xs, ERF_INV_IMPL_DD); + y * x + r * x + } else if x < 18.0 { + let y = 0.98362827301025390625; + let xs = x - 6.0; + let r = evaluate::polynomial(xs, ERF_INV_IMPL_EN) + / evaluate::polynomial(xs, ERF_INV_IMPL_ED); + y * x + r * x + } else if x < 44.0 { + let y = 0.99714565277099609375; + let xs = x - 18.0; + let r = evaluate::polynomial(xs, ERF_INV_IMPL_FN) + / evaluate::polynomial(xs, ERF_INV_IMPL_FD); + y * x + r * x + } else { + let y = 0.99941349029541015625; + let xs = x - 44.0; + let r = evaluate::polynomial(xs, ERF_INV_IMPL_GN) + / evaluate::polynomial(xs, ERF_INV_IMPL_GD); + y * x + r * x + } + }; + s * result +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cpu/kernels.rs b/v2/dependency-patches/candle-core-0.9.2/src/cpu/kernels.rs new file mode 100644 index 0000000000..bca76adcc8 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/cpu/kernels.rs @@ -0,0 +1,232 @@ +pub trait VecOps: num_traits::NumAssign + Copy { + fn min(self, rhs: Self) -> Self; + fn max(self, rhs: Self) -> Self; + + /// Dot-product of two vectors. + /// + /// # Safety + /// + /// The length of `lhs` and `rhs` have to be at least `len`. `res` has to point to a valid + /// element. + #[inline(always)] + unsafe fn vec_dot(lhs: *const Self, rhs: *const Self, res: *mut Self, len: usize) { + *res = Self::zero(); + for i in 0..len { + *res += *lhs.add(i) * *rhs.add(i) + } + } + + /// Sum of all elements in a vector. + /// + /// # Safety + /// + /// The length of `xs` must be at least `len`. `res` has to point to a valid + /// element. + #[inline(always)] + unsafe fn vec_reduce_sum(xs: *const Self, res: *mut Self, len: usize) { + *res = Self::zero(); + for i in 0..len { + *res += *xs.add(i) + } + } + + /// Maximum element in a non-empty vector. + /// + /// # Safety + /// + /// The length of `xs` must be at least `len` and positive. `res` has to point to a valid + /// element. + #[inline(always)] + unsafe fn vec_reduce_max(xs: *const Self, res: *mut Self, len: usize) { + *res = *xs; + for i in 1..len { + *res = (*res).max(*xs.add(i)) + } + } + + /// Minimum element in a non-empty vector. + /// + /// # Safety + /// + /// The length of `xs` must be at least `len` and positive. `res` has to point to a valid + /// element. + #[inline(always)] + unsafe fn vec_reduce_min(xs: *const Self, res: *mut Self, len: usize) { + *res = *xs; + for i in 1..len { + *res = (*res).min(*xs.add(i)) + } + } +} + +impl VecOps for f32 { + #[inline(always)] + fn min(self, other: Self) -> Self { + Self::min(self, other) + } + + #[inline(always)] + fn max(self, other: Self) -> Self { + Self::max(self, other) + } + + #[inline(always)] + unsafe fn vec_dot(lhs: *const Self, rhs: *const Self, res: *mut Self, len: usize) { + super::vec_dot_f32(lhs, rhs, res, len) + } + + #[inline(always)] + unsafe fn vec_reduce_sum(xs: *const Self, res: *mut Self, len: usize) { + super::vec_sum(xs, res, len) + } +} + +impl VecOps for half::f16 { + #[inline(always)] + fn min(self, other: Self) -> Self { + Self::min(self, other) + } + + #[inline(always)] + fn max(self, other: Self) -> Self { + Self::max(self, other) + } + + #[inline(always)] + unsafe fn vec_dot(lhs: *const Self, rhs: *const Self, res: *mut Self, len: usize) { + let mut res_f32 = 0f32; + super::vec_dot_f16(lhs, rhs, &mut res_f32, len); + *res = half::f16::from_f32(res_f32); + } +} + +impl VecOps for f64 { + #[inline(always)] + fn min(self, other: Self) -> Self { + Self::min(self, other) + } + + #[inline(always)] + fn max(self, other: Self) -> Self { + Self::max(self, other) + } +} +impl VecOps for half::bf16 { + #[inline(always)] + fn min(self, other: Self) -> Self { + Self::min(self, other) + } + + #[inline(always)] + fn max(self, other: Self) -> Self { + Self::max(self, other) + } + + #[inline(always)] + unsafe fn vec_dot(lhs: *const Self, rhs: *const Self, res: *mut Self, len: usize) { + let mut res_f32 = 0f32; + super::vec_dot_bf16(lhs, rhs, &mut res_f32, len); + *res = half::bf16::from_f32(res_f32); + } +} +impl VecOps for u8 { + #[inline(always)] + fn min(self, other: Self) -> Self { + ::min(self, other) + } + + #[inline(always)] + fn max(self, other: Self) -> Self { + ::max(self, other) + } +} +impl VecOps for u32 { + #[inline(always)] + fn min(self, other: Self) -> Self { + ::min(self, other) + } + + #[inline(always)] + fn max(self, other: Self) -> Self { + ::max(self, other) + } +} +impl VecOps for i16 { + #[inline(always)] + fn min(self, other: Self) -> Self { + ::min(self, other) + } + + #[inline(always)] + fn max(self, other: Self) -> Self { + ::max(self, other) + } +} +impl VecOps for i32 { + #[inline(always)] + fn min(self, other: Self) -> Self { + ::min(self, other) + } + + #[inline(always)] + fn max(self, other: Self) -> Self { + ::max(self, other) + } +} +impl VecOps for i64 { + #[inline(always)] + fn min(self, other: Self) -> Self { + ::min(self, other) + } + + #[inline(always)] + fn max(self, other: Self) -> Self { + ::max(self, other) + } +} + +impl VecOps for float8::F8E4M3 { + #[inline(always)] + fn min(self, other: Self) -> Self { + Self::min(self, other) + } + + #[inline(always)] + fn max(self, other: Self) -> Self { + Self::max(self, other) + } +} + +#[inline(always)] +pub fn par_for_each(n_threads: usize, func: impl Fn(usize) + Send + Sync) { + if n_threads == 1 { + func(0) + } else { + rayon::scope(|s| { + for thread_idx in 0..n_threads { + let func = &func; + s.spawn(move |_| func(thread_idx)); + } + }) + } +} + +#[inline(always)] +pub fn par_range(lo: usize, up: usize, n_threads: usize, func: impl Fn(usize) + Send + Sync) { + if n_threads == 1 { + for i in lo..up { + func(i) + } + } else { + rayon::scope(|s| { + for thread_idx in 0..n_threads { + let func = &func; + s.spawn(move |_| { + for i in (thread_idx..up).step_by(n_threads) { + func(i) + } + }); + } + }) + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cpu/mod.rs b/v2/dependency-patches/candle-core-0.9.2/src/cpu/mod.rs new file mode 100644 index 0000000000..c4864b7a81 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/cpu/mod.rs @@ -0,0 +1,242 @@ +//! Traits and methods for CPU-backed Tensors + +pub mod erf; +pub mod kernels; + +#[allow(unused)] +trait Cpu { + type Unit; + type Array; + const STEP: usize; + const EPR: usize; + + fn n() -> usize; + unsafe fn zero() -> Self::Unit; + unsafe fn zero_array() -> Self::Array; + unsafe fn load(mem_addr: *const f32) -> Self::Unit; + unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit; + unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit; + unsafe fn vec_reduce(x: Self::Array, y: *mut f32); + unsafe fn from_f32(v: f32) -> Self::Unit; + unsafe fn vec_store(mem_addr: *mut f32, a: Self::Unit); +} + +#[allow(unused)] +trait CpuF16 { + type Unit; + type Array; + const STEP: usize; + const EPR: usize; + + fn n() -> usize; + unsafe fn zero() -> Self::Unit; + unsafe fn zero_array() -> Self::Array; + unsafe fn load(mem_addr: *const f16) -> Self::Unit; + unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit; + unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit; + unsafe fn vec_reduce(x: Self::Array, y: *mut f32); + unsafe fn from_f32(v: f32) -> Self::Unit; + unsafe fn vec_store(mem_addr: *mut f16, a: Self::Unit); +} + +#[allow(unused)] +trait CpuBF16 { + type Unit; + type Array; + const STEP: usize; + const EPR: usize; + + fn n() -> usize; + unsafe fn zero() -> Self::Unit; + unsafe fn zero_array() -> Self::Array; + unsafe fn load(mem_addr: *const bf16) -> Self::Unit; + unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit; + unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit; + unsafe fn vec_reduce(x: Self::Array, y: *mut f32); + unsafe fn from_f32(v: f32) -> Self::Unit; + unsafe fn vec_store(mem_addr: *mut bf16, a: Self::Unit); +} + +use half::{bf16, f16}; + +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[cfg(target_feature = "avx2")] +pub mod avx; +#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] +#[cfg(target_feature = "avx2")] +pub use avx::{CurrentCpu, CurrentCpuBF16, CurrentCpuF16}; + +#[cfg(target_arch = "wasm32")] +#[cfg(target_feature = "simd128")] +pub mod simd128; +#[cfg(target_arch = "wasm32")] +#[cfg(target_feature = "simd128")] +pub use simd128::CurrentCpu; + +#[cfg(any(target_arch = "arm", target_arch = "aarch64"))] +#[cfg(target_feature = "neon")] +pub mod neon; +#[cfg(any(target_arch = "arm", target_arch = "aarch64"))] +#[cfg(target_feature = "neon")] +pub use neon::CurrentCpu; + +#[cfg(any( + target_feature = "neon", + target_feature = "avx2", + target_feature = "simd128" +))] +#[inline(always)] +pub(crate) unsafe fn vec_dot_f32(a_row: *const f32, b_row: *const f32, c: *mut f32, k: usize) { + let np = k & !(CurrentCpu::STEP - 1); + + let mut sum = CurrentCpu::zero_array(); + let mut ax = CurrentCpu::zero_array(); + let mut ay = CurrentCpu::zero_array(); + + for i in (0..np).step_by(CurrentCpu::STEP) { + for j in 0..CurrentCpu::n() { + ax[j] = CurrentCpu::load(a_row.add(i + j * CurrentCpu::EPR)); + ay[j] = CurrentCpu::load(b_row.add(i + j * CurrentCpu::EPR)); + + sum[j] = CurrentCpu::vec_fma(sum[j], ax[j], ay[j]); + } + } + + CurrentCpu::vec_reduce(sum, c); + + // leftovers + for i in np..k { + *c += *a_row.add(i) * (*b_row.add(i)); + } +} + +#[cfg(not(any( + target_feature = "neon", + target_feature = "avx2", + target_feature = "simd128" +)))] +#[inline(always)] +pub(crate) unsafe fn vec_dot_f32(a_row: *const f32, b_row: *const f32, c: *mut f32, k: usize) { + // leftovers + for i in 0..k { + *c += *a_row.add(i) * (*b_row.add(i)); + } +} + +#[cfg(any( + target_feature = "neon", + target_feature = "avx2", + target_feature = "simd128" +))] +#[inline(always)] +pub(crate) unsafe fn vec_sum(row: *const f32, b: *mut f32, k: usize) { + let np = k & !(CurrentCpu::STEP - 1); + + let mut sum = CurrentCpu::zero_array(); + let mut x = CurrentCpu::zero_array(); + + for i in (0..np).step_by(CurrentCpu::STEP) { + for j in 0..CurrentCpu::n() { + x[j] = CurrentCpu::load(row.add(i + j * CurrentCpu::EPR)); + sum[j] = CurrentCpu::vec_add(sum[j], x[j]); + } + } + + CurrentCpu::vec_reduce(sum, b); + + // leftovers + for i in np..k { + *b += *row.add(i) + } +} + +#[cfg(not(any( + target_feature = "neon", + target_feature = "avx2", + target_feature = "simd128" +)))] +#[inline(always)] +pub(crate) unsafe fn vec_sum(row: *const f32, b: *mut f32, k: usize) { + *b = 0f32; + for i in 0..k { + *b += *row.add(i) + } +} + +#[cfg(target_feature = "avx2")] +#[inline(always)] +pub(crate) unsafe fn vec_dot_f16(a_row: *const f16, b_row: *const f16, c: *mut f32, k: usize) { + let mut sumf = 0.0f32; + let np = k & !(CurrentCpuF16::STEP - 1); + + let mut sum = CurrentCpuF16::zero_array(); + let mut ax = CurrentCpuF16::zero_array(); + let mut ay = CurrentCpuF16::zero_array(); + + for i in (0..np).step_by(CurrentCpuF16::STEP) { + for j in 0..CurrentCpuF16::n() { + ax[j] = CurrentCpuF16::load(a_row.add(i + j * CurrentCpuF16::EPR)); + ay[j] = CurrentCpuF16::load(b_row.add(i + j * CurrentCpuF16::EPR)); + + sum[j] = CurrentCpuF16::vec_fma(sum[j], ax[j], ay[j]); + } + } + + CurrentCpuF16::vec_reduce(sum, &mut sumf); + + // leftovers + for i in np..k { + sumf += (*a_row.add(i)).to_f32() * (*b_row.add(i)).to_f32(); + } + *c = sumf; +} + +#[cfg(target_feature = "avx2")] +#[inline(always)] +pub(crate) unsafe fn vec_dot_bf16(a_row: *const bf16, b_row: *const bf16, c: *mut f32, k: usize) { + let mut sumf = 0.0f32; + let np = k & !(CurrentCpuBF16::STEP - 1); + + let mut sum = CurrentCpuBF16::zero_array(); + let mut ax = CurrentCpuBF16::zero_array(); + let mut ay = CurrentCpuBF16::zero_array(); + + for i in (0..np).step_by(CurrentCpuBF16::STEP) { + for j in 0..CurrentCpuBF16::n() { + ax[j] = CurrentCpuBF16::load(a_row.add(i + j * CurrentCpuBF16::EPR)); + ay[j] = CurrentCpuBF16::load(b_row.add(i + j * CurrentCpuBF16::EPR)); + + sum[j] = CurrentCpuBF16::vec_fma(sum[j], ax[j], ay[j]); + } + } + + CurrentCpuBF16::vec_reduce(sum, &mut sumf); + + // leftovers + for i in np..k { + sumf += (*a_row.add(i)).to_f32() * (*b_row.add(i)).to_f32(); + } + *c = sumf; +} + +#[cfg(not(target_feature = "avx2"))] +#[inline(always)] +pub(crate) unsafe fn vec_dot_f16(a_row: *const f16, b_row: *const f16, c: *mut f32, k: usize) { + // leftovers + let mut sum = 0.0; + for i in 0..k { + sum += (*a_row.add(i)).to_f32() * (*b_row.add(i)).to_f32(); + } + *c = sum; +} + +#[cfg(not(target_feature = "avx2"))] +#[inline(always)] +pub(crate) unsafe fn vec_dot_bf16(a_row: *const bf16, b_row: *const bf16, c: *mut f32, k: usize) { + // leftovers + let mut sum = 0.0; + for i in 0..k { + sum += (*a_row.add(i)).to_f32() * (*b_row.add(i)).to_f32(); + } + *c = sum; +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cpu/neon.rs b/v2/dependency-patches/candle-core-0.9.2/src/cpu/neon.rs new file mode 100644 index 0000000000..66b8b45e15 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/cpu/neon.rs @@ -0,0 +1,74 @@ +use super::Cpu; +#[cfg(target_arch = "arm")] +use core::arch::arm::*; + +#[cfg(target_arch = "aarch64")] +use core::arch::aarch64::*; + +pub struct CurrentCpu {} + +const STEP: usize = 16; +const EPR: usize = 4; +const ARR: usize = STEP / EPR; + +impl CurrentCpu { + #[cfg(target_arch = "aarch64")] + unsafe fn reduce_one(x: float32x4_t) -> f32 { + vaddvq_f32(x) + } + + #[cfg(target_arch = "arm")] + unsafe fn reduce_one(x: float32x4_t) -> f32 { + vgetq_lane_f32(x, 0) + vgetq_lane_f32(x, 1) + vgetq_lane_f32(x, 2) + vgetq_lane_f32(x, 3) + } +} + +impl Cpu for CurrentCpu { + type Unit = float32x4_t; + type Array = [float32x4_t; ARR]; + + const STEP: usize = STEP; + const EPR: usize = EPR; + + fn n() -> usize { + ARR + } + + unsafe fn zero() -> Self::Unit { + vdupq_n_f32(0.0) + } + + unsafe fn from_f32(x: f32) -> Self::Unit { + vdupq_n_f32(x) + } + + unsafe fn zero_array() -> Self::Array { + [Self::zero(); ARR] + } + + unsafe fn load(mem_addr: *const f32) -> Self::Unit { + vld1q_f32(mem_addr) + } + + unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit { + vaddq_f32(a, b) + } + + unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit { + vfmaq_f32(a, b, c) + } + + unsafe fn vec_store(mem_addr: *mut f32, a: Self::Unit) { + vst1q_f32(mem_addr, a); + } + + unsafe fn vec_reduce(mut x: Self::Array, y: *mut f32) { + for i in 0..ARR / 2 { + x[2 * i] = vaddq_f32(x[2 * i], x[2 * i + 1]); + } + for i in 0..ARR / 4 { + x[4 * i] = vaddq_f32(x[4 * i], x[4 * i + 2]); + } + *y = Self::reduce_one(x[0]); + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cpu/simd128.rs b/v2/dependency-patches/candle-core-0.9.2/src/cpu/simd128.rs new file mode 100644 index 0000000000..64b94588bd --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/cpu/simd128.rs @@ -0,0 +1,64 @@ +use super::Cpu; +use core::arch::wasm32::*; + +pub struct CurrentCpu {} + +const STEP: usize = 16; +const EPR: usize = 4; +const ARR: usize = STEP / EPR; + +impl Cpu for CurrentCpu { + type Unit = v128; + type Array = [v128; ARR]; + + const STEP: usize = STEP; + const EPR: usize = EPR; + + fn n() -> usize { + ARR + } + + unsafe fn zero() -> Self::Unit { + f32x4_splat(0.0) + } + + unsafe fn zero_array() -> Self::Array { + [Self::zero(); ARR] + } + + unsafe fn from_f32(v: f32) -> Self::Unit { + f32x4_splat(v) + } + + unsafe fn load(mem_addr: *const f32) -> Self::Unit { + v128_load(mem_addr as *mut v128) + } + + unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit { + f32x4_add(a, b) + } + + unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit { + f32x4_add(f32x4_mul(b, c), a) + } + + unsafe fn vec_store(mem_addr: *mut f32, a: Self::Unit) { + v128_store(mem_addr as *mut v128, a); + } + + unsafe fn vec_reduce(mut x: Self::Array, y: *mut f32) { + for i in 0..ARR / 2 { + x[2 * i] = f32x4_add(x[2 * i], x[2 * i + 1]); + } + for i in 0..ARR / 4 { + x[4 * i] = f32x4_add(x[4 * i], x[4 * i + 2]); + } + for i in 0..ARR / 8 { + x[8 * i] = f32x4_add(x[8 * i], x[8 * i + 4]); + } + *y = f32x4_extract_lane::<0>(x[0]) + + f32x4_extract_lane::<1>(x[0]) + + f32x4_extract_lane::<2>(x[0]) + + f32x4_extract_lane::<3>(x[0]); + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/conv2d.rs b/v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/conv2d.rs new file mode 100644 index 0000000000..70c5ea75fa --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/conv2d.rs @@ -0,0 +1,432 @@ +use std::borrow::Cow; + +use rayon::iter::{IntoParallelIterator, ParallelIterator}; + +use crate::{ + conv::ParamsConv2D, + cpu_backend::{copy_strided_src_, Im2Col, Map1, Map2, MatMul}, + shape::dims4, + Layout, Result, WithDType, +}; + +pub(super) struct Conv2D<'a>(pub(super) &'a crate::conv::ParamsConv2D); + +#[allow(dead_code)] +enum Conv2dImpl { + TiledIm2Col, + FullIm2Col, + Direct, +} + +const DEFAULT_CONV2D_IMPL: Conv2dImpl = Conv2dImpl::TiledIm2Col; + +impl Map2 for Conv2D<'_> { + const OP: &'static str = "conv2d"; + fn f( + &self, + inp: &[T], + inp_l: &Layout, + k: &[T], + k_l: &Layout, + ) -> Result> { + let p = self.0; + + // Specialization: pick the best algorithm based on parameters. + // 1x1 convolutions with stride=1, padding=0, dilation=1 + if p.k_h == 1 && p.k_w == 1 && p.stride == 1 && p.padding == 0 && p.dilation == 1 { + return conv2d_1x1(p, inp, inp_l, k, k_l); + } else if p.k_h == 1 && p.k_w == 1 { + // Other 1x1 convolutions for now are assumed faster with full im2col, + // although with large enough input size, tiled will start beating it. + return conv2d_im2col_gemm(p, inp, inp_l, k, k_l); + } + // TODO other cases + + // No fast path, fallback to default general impl. + match DEFAULT_CONV2D_IMPL { + Conv2dImpl::TiledIm2Col => conv2d_tiled(p, inp, inp_l, k, k_l), + Conv2dImpl::Direct => conv2d_direct(p, inp, inp_l, k, k_l), + Conv2dImpl::FullIm2Col => conv2d_im2col_gemm(p, inp, inp_l, k, k_l), + } + } +} + +/// Fast kernel for 1x1 convolutions with stride=1, padding=0, dilation=1 +/// These are just matrix multiplications: [c_out, c_in] @ [c_in, b*h*w] -> [c_out, b*h*w]. +fn conv2d_1x1( + p: &ParamsConv2D, + inp: &[T], + inp_l: &Layout, + k: &[T], + k_l: &Layout, +) -> Result> { + let inp = &inp[inp_l.start_offset()..]; + let inp_stride = inp_l.stride(); + let (inp_s0, inp_s1, inp_s2, inp_s3) = + (inp_stride[0], inp_stride[1], inp_stride[2], inp_stride[3]); + let k = &k[k_l.start_offset()..]; + let k_stride = k_l.stride(); + let (k_s0, k_s1) = (k_stride[0], k_stride[1]); + let (out_h, out_w) = (p.out_h(), p.out_w()); + + let spatial_size = out_h * out_w; + let dst = vec![T::zero(); p.b_size * p.c_out * spatial_size]; + let k_reshaped: Cow<[T]> = if k_s0 == p.c_in && k_s1 == 1 { + // Already contiguous, use slice directly + Cow::Borrowed(&k[..p.c_out * p.c_in]) + } else { + // Reshape kernel to [c_out, c_in] + let mut k_reshaped = Vec::with_capacity(p.c_out * p.c_in); + (0..p.c_out).for_each(|c_out_idx| { + (0..p.c_in).for_each(|c_in_idx| { + let k_idx = c_out_idx * k_s0 + c_in_idx * k_s1; + k_reshaped.push(k[k_idx]); + }); + }); + Cow::Owned(k_reshaped) + }; + let k_layout = Layout::contiguous((p.c_out, p.c_in)); + + // Process each batch + (0..p.b_size).into_par_iter().try_for_each(|b_idx| { + // Reshape input to [c_in, h*w] for this batch + let mut inp_reshaped = Vec::with_capacity(p.c_in * spatial_size); + for c_in_idx in 0..p.c_in { + for h_idx in 0..p.i_h { + for w_idx in 0..p.i_w { + let inp_idx = + b_idx * inp_s0 + c_in_idx * inp_s1 + h_idx * inp_s2 + w_idx * inp_s3; + inp_reshaped.push(inp[inp_idx]); + } + } + } + let inp_layout = Layout::contiguous((p.c_in, spatial_size)); + + // Perform matmul: [c_out, c_in] @ [c_in, spatial_size] -> [c_out, spatial_size] + let matmul = MatMul((1, p.c_out, spatial_size, p.c_in)); + let result = matmul.f(&k_reshaped, &k_layout, &inp_reshaped, &inp_layout)?; + + // Copy result to output + let out_offset = b_idx * p.c_out * spatial_size; + for (i, r) in result.iter().enumerate() { + unsafe { + let ptr = dst.as_ptr().add(out_offset + i) as *mut T; + *ptr = *r; + } + } + Ok::<(), crate::Error>(()) + })?; + + Ok(dst) +} + +/// General tiled convolution implementation using gemm. +/// +/// Similar to full im2col, but instead of materializing the full matrix, we process input/output in tiles, in parallel. +fn conv2d_tiled( + p: &ParamsConv2D, + inp: &[T], + inp_l: &Layout, + k: &[T], + k_l: &Layout, +) -> Result> { + let inp = &inp[inp_l.start_offset()..]; + let (inp_s0, inp_s1, inp_s2, inp_s3) = dims4(inp_l.stride())?; + let k = &k[k_l.start_offset()..]; + let (k_s0, k_s1, k_s2, k_s3) = dims4(k_l.stride())?; + let (out_h, out_w) = (p.out_h(), p.out_w()); + + // Output shape: [b_size, c_out, out_h, out_w]. + let dst = vec![T::zero(); p.b_size * p.c_out * out_h * out_w]; + + // Make contiguous input copy if needed. + let cont_s0 = p.i_h * p.i_w * p.c_in; + let cont_s1 = p.i_w * p.c_in; + let cont_s2 = p.c_in; + let layout_is_valid = inp_l.stride() == [cont_s0, cont_s1, cont_s2, 1]; + let inp_cont: Cow<[T]> = if layout_is_valid { + Cow::Borrowed(inp) + } else { + let mut inp_cont = vec![T::zero(); p.b_size * p.c_in * p.i_h * p.i_w]; + for b_idx in 0..p.b_size { + for h_idx in 0..p.i_h { + for w_idx in 0..p.i_w { + for c_idx in 0..p.c_in { + let src_idx = + b_idx * inp_s0 + c_idx * inp_s1 + h_idx * inp_s2 + w_idx * inp_s3; + let dst_idx = b_idx * cont_s0 + h_idx * cont_s1 + w_idx * cont_s2 + c_idx; + inp_cont[dst_idx] = inp[src_idx] + } + } + } + } + Cow::Owned(inp_cont) + }; + + // shape of k: [c_out, c_in, k_h, k_w] + // strides of k: [k_s0, k_s1, k_s2, k_s3] + // For matmul, we need flattened k in shape [c_out, k_h * k_w * c_in] + // with stride [k_h * k_w * c_in, 1] + let k_size = p.c_in * p.k_h * p.k_w; + let mut k_flat = Vec::with_capacity(p.c_out * k_size); + for dst_c_idx in 0..p.c_out { + for kh in 0..p.k_h { + for kw in 0..p.k_w { + for c_in_idx in 0..p.c_in { + let k_idx = dst_c_idx * k_s0 + c_in_idx * k_s1 + kh * k_s2 + kw * k_s3; + k_flat.push(k[k_idx]); + } + } + } + } + // k_layout: [c_out, k_size] with stride [k_size, 1] + let k_layout = Layout::contiguous((p.c_out, k_size)); + + // TILE_SIZE is number of output pixels (out_h * out_w) per tile. + // Higher tile size can be faster due to better usage of gemm, + // but lower tile sizes enable bigger parallelism across tiles. + // This parameter is impactful and may be dynamic or even runtime tunable in the future. + const TILE_SIZE: usize = 512; + + let total_out_pixels = out_h * out_w; + + // Process batches and tiles in parallel using rayon. + (0..p.b_size).into_par_iter().try_for_each(|b_idx| { + let inp_offset = b_idx * cont_s0; + let out_batch_offset = b_idx * (p.c_out * out_h * out_w); + + let num_tiles = total_out_pixels.div_ceil(TILE_SIZE); + (0..num_tiles).into_par_iter().try_for_each(|tile_idx| { + // Determine actual tile size (may be smaller at the end) { + let tile_start = tile_idx * TILE_SIZE; + let tile_end = (tile_start + TILE_SIZE).min(total_out_pixels); + let tile_size = tile_end - tile_start; + + // Precompute output coordinates. + // Used in both im2col extraction and writing output. + let out_coords: Vec<_> = (tile_start..tile_end) + .map(|idx| (idx / out_w, idx % out_w)) + .collect(); + + // Build im2col tile: [k_size, tile_size] + // This represents the input patches needed for this tile of outputs + let mut col_tile = vec![T::zero(); k_size * tile_size]; + + for (tile_idx, (out_y, out_x)) in out_coords.iter().enumerate() { + // Extract the im2col patch for this output position + for c_in in 0..p.c_in { + let mut patch_offset = c_in; + for kh in 0..p.k_h { + let in_y = + (out_y * p.stride + kh * p.dilation) as isize - p.padding as isize; + if in_y < 0 || in_y >= p.i_h as isize { + // Padding: already zero + patch_offset += p.c_in * p.k_w; + continue; + } + for kw in 0..p.k_w { + let in_x = + (out_x * p.stride + kw * p.dilation) as isize - p.padding as isize; + + if in_x >= 0 && in_x < p.i_w as isize { + let in_y = in_y as usize; + let in_x = in_x as usize; + let inp_idx = inp_offset + in_y * cont_s1 + in_x * cont_s2 + c_in; + let col_idx = patch_offset * tile_size + tile_idx; + col_tile[col_idx] = inp_cont[inp_idx]; + } + // Move to next position (skip c_in channels) + patch_offset += p.c_in; + } + } + } + } + + // Now perform matmul: k_cache [c_out, k_size] @ col_tile [k_size, tile_size] + let matmul = MatMul((1, p.c_out, tile_size, k_size)); + + // Layouts for matmul + // k_flat layout: [c_out, k_size] with stride [k_size, 1] + // col_tile layout: [k_size, tile_size] with stride [tile_size, 1] + let col_layout = Layout::contiguous((k_size, tile_size)); + + // Perform matmul + let result = matmul.f(&k_flat, &k_layout, &col_tile, &col_layout)?; + + // Copy results to output: result is [c_out, tile_size] + for (tile_idx, (out_y, out_x)) in out_coords.iter().enumerate() { + let dst_base = out_batch_offset + out_y * out_w + out_x; + + for c_out_idx in 0..p.c_out { + let dst_idx = dst_base + c_out_idx * (out_h * out_w); + let result_idx = c_out_idx * tile_size + tile_idx; + // SAFETY: Each batch processes a distinct region of the output buffer. + // Within each batch, tiles process non-overlapping output positions. + // Therefore, no two threads will write to the same dst_idx. + unsafe { + let ptr = dst.as_ptr().add(dst_idx) as *mut T; + *ptr = result[result_idx]; + } + } + } + Ok::<(), crate::Error>(()) + }) + })?; + + Ok(dst) +} + +/// General direct convolution impl. Decently fast for small inputs and kernels, but loses to full/tiled gemm. +fn conv2d_direct( + p: &ParamsConv2D, + inp: &[T], + inp_l: &Layout, + k: &[T], + k_l: &Layout, +) -> Result> { + let inp = &inp[inp_l.start_offset()..]; + let (inp_s0, inp_s1, inp_s2, inp_s3) = crate::shape::dims4(inp_l.stride())?; + let k = &k[k_l.start_offset()..]; + let (k_s0, k_s1, k_s2, k_s3) = crate::shape::dims4(k_l.stride())?; + let (out_h, out_w) = (p.out_h(), p.out_w()); + + // Output shape: [b_size, c_out, out_h, out_w]. + let dst = vec![T::zero(); p.b_size * p.c_out * out_h * out_w]; + + // Make contiguous input copy if needed. + let cont_s0 = p.i_h * p.i_w * p.c_in; + let cont_s1 = p.i_w * p.c_in; + let cont_s2 = p.c_in; + let layout_is_valid = inp_l.stride() == [cont_s0, cont_s1, cont_s2, 1]; + let inp_cont: Cow<[T]> = if layout_is_valid { + Cow::Borrowed(inp) + } else { + let mut inp_cont = vec![T::zero(); p.b_size * p.c_in * p.i_h * p.i_w]; + for b_idx in 0..p.b_size { + for h_idx in 0..p.i_h { + for w_idx in 0..p.i_w { + for c_idx in 0..p.c_in { + let src_idx = + b_idx * inp_s0 + c_idx * inp_s1 + h_idx * inp_s2 + w_idx * inp_s3; + let dst_idx = b_idx * cont_s0 + h_idx * cont_s1 + w_idx * cont_s2 + c_idx; + inp_cont[dst_idx] = inp[src_idx] + } + } + } + } + Cow::Owned(inp_cont) + }; + let inp_cont_len = inp_cont.len(); + + let k_cache: Vec> = (0..p.c_out) + .map(|dst_c_idx| { + (0..p.k_h * p.k_w) + .flat_map(|kw_kh| { + let offset_h = kw_kh / p.k_w; + let offset_w = kw_kh % p.k_w; + (0..p.c_in).map(move |c_in_idx| { + k[dst_c_idx * k_s0 + c_in_idx * k_s1 + offset_h * k_s2 + offset_w * k_s3] + }) + }) + .collect() + }) + .collect(); + + for b_idx in 0..p.b_size { + for offset_h in 0..p.k_h { + for offset_w in 0..p.k_w { + let k_offset = offset_h * p.k_w + offset_w; + + (0..p.c_out).into_par_iter().for_each(|dst_c_idx| { + let k_cont = &k_cache[dst_c_idx][k_offset * p.c_in..(k_offset + 1) * p.c_in]; + let base_dst_idx = dst_c_idx * out_w * out_h; + let batch_dst_idx = base_dst_idx + b_idx * p.c_out * out_h * out_w; + let batch_src_idx = b_idx * cont_s0; + + for dst_h in 0..out_h { + let src_h = p.stride * dst_h + offset_h * p.dilation; + if src_h < p.padding || src_h >= p.i_h + p.padding { + continue; + } + let src_h = src_h - p.padding; + let h_dst_idx = batch_dst_idx + dst_h * out_w; + let h_src_idx = batch_src_idx + src_h * cont_s1; + + for dst_w in 0..out_w { + let src_w = p.stride * dst_w + offset_w * p.dilation; + if src_w < p.padding || src_w >= p.i_w + p.padding { + continue; + } + let src_w = src_w - p.padding; + let dst_idx = h_dst_idx + dst_w; + let inp_idx_1 = h_src_idx + src_w * cont_s2; + let inp_idx_2 = (inp_idx_1 + p.c_in).min(inp_cont_len); + let inp_cont = &inp_cont[inp_idx_1..inp_idx_2]; + let mut d = T::zero(); + unsafe { + T::vec_dot(inp_cont.as_ptr(), k_cont.as_ptr(), &mut d, p.c_in); + let ptr = dst.as_ptr().add(dst_idx) as *mut T; + *ptr += d; + } + } + } + }); + } + } + } + + Ok(dst) +} + +#[allow(clippy::uninit_vec)] +fn alloc_uninit_vec(size: usize) -> Vec { + let mut v = Vec::with_capacity(size); + unsafe { v.set_len(size) }; + v +} + +/// Full im2col + gemm convolution implementation. +/// +/// For large inputs im2col and copy_strided_src for output gets expensive. +fn conv2d_im2col_gemm( + p: &ParamsConv2D, + inp: &[T], + inp_l: &Layout, + kernel: &[T], + kernel_l: &Layout, +) -> Result> { + let op = Im2Col { + h_k: p.k_h, + w_k: p.k_w, + padding: p.padding, + stride: p.stride, + dilation: p.dilation, + }; + let col = op.f(inp, inp_l)?; + let b = p.b_size; + let n = p.c_out; + let (h_out, w_out) = (p.out_h(), p.out_w()); + let k = op.h_k * op.w_k * p.c_in; + let m = h_out * w_out; + let col_l = Layout::contiguous((b, m, k)); + let res: Vec = if kernel_l.is_contiguous() { + let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) + .transpose(1, 2)? + .broadcast_as((b, k, n))?; + MatMul((b, m, n, k)).f(&col, &col_l, kernel, &kernel_l)? + } else { + // Make the kernel contiguous if not already the case. + let mut kernel_c = alloc_uninit_vec(kernel_l.shape().elem_count()); + copy_strided_src_(kernel, &mut kernel_c, 0, kernel_l); + let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) + .transpose(1, 2)? + .broadcast_as((b, k, n))?; + MatMul((b, m, n, k)).f(&col, &col_l, &kernel_c, &kernel_l)? + }; + let res_l = Layout::contiguous((b, h_out, w_out, p.c_out)) + .transpose(1, 2)? + .transpose(1, 3)?; + let mut res_t = alloc_uninit_vec(res_l.shape().elem_count()); + copy_strided_src_(&res, &mut res_t, 0, &res_l); + Ok(res_t) +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/mod.rs b/v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/mod.rs new file mode 100644 index 0000000000..afb93024ac --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/mod.rs @@ -0,0 +1,3284 @@ +//! Implementation of Backend Fns for CPU +use crate::backend::{BackendDevice, BackendStorage}; +use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; +use crate::{DType, Error, IntDType, Layout, Result, Shape, WithDType}; +use float8::F8E4M3; +use half::{bf16, f16}; +use rayon::prelude::*; + +mod utils; +pub use utils::{ + binary_map, binary_map_vec, unary_map, unary_map_vec, Map1, Map1Any, Map2, Map2InPlace, Map2U8, +}; +mod conv2d; +use conv2d::Conv2D; + +const USE_IM2COL_CONV1D: bool = true; +const USE_COL2IM_CONV1D_TR: bool = true; + +// TODO: Maybe we should not implement [Clone] here and instead have an explicit allocator + +// intercept the oom errors to avoid panicking and provide a proper error. +#[derive(Debug, Clone)] +pub enum CpuStorage { + U8(Vec), + U32(Vec), + I16(Vec), + I32(Vec), + I64(Vec), + BF16(Vec), + F16(Vec), + F32(Vec), + F64(Vec), + F8E4M3(Vec), + // Dummy types that store raw bytes + F6E2M3(Vec), + F6E3M2(Vec), + F4(Vec), + F8E8M0(Vec), +} + +#[derive(Debug, Clone)] +pub enum CpuStorageRef<'a> { + U8(&'a [u8]), + U32(&'a [u32]), + I16(&'a [i16]), + I32(&'a [i32]), + I64(&'a [i64]), + BF16(&'a [bf16]), + F16(&'a [f16]), + F32(&'a [f32]), + F64(&'a [f64]), + F8E4M3(&'a [F8E4M3]), + // Dummy types that store raw bytes + F6E2M3(&'a [u8]), + F6E3M2(&'a [u8]), + F4(&'a [u8]), + F8E8M0(&'a [u8]), +} + +#[derive(Debug, Clone)] +pub struct CpuDevice; + +struct Cmp(CmpOp); +impl Map2U8 for Cmp { + const OP: &'static str = "cmp"; + #[inline(always)] + fn f( + &self, + lhs: &[T], + lhs_l: &Layout, + rhs: &[T], + rhs_l: &Layout, + ) -> Result> { + let dst = match self.0 { + CmpOp::Eq => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x == y)), + CmpOp::Ne => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x != y)), + CmpOp::Lt => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x < y)), + CmpOp::Le => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x <= y)), + CmpOp::Gt => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x > y)), + CmpOp::Ge => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x >= y)), + }; + Ok(dst) + } +} + +struct WCond<'a, T: IntDType>(&'a [T], &'a Layout); + +impl Map2 for WCond<'_, I> { + const OP: &'static str = "where"; + #[inline(always)] + fn f(&self, t: &[T], t_l: &Layout, f: &[T], f_l: &Layout) -> Result> { + let vs = match ( + self.1.contiguous_offsets(), + t_l.contiguous_offsets(), + f_l.contiguous_offsets(), + ) { + (Some((o1, o2)), Some((o_t1, o_t2)), Some((o_f1, o_f2))) => { + let pred = &self.0[o1..o2]; + let t = &t[o_t1..o_t2]; + let f = &f[o_f1..o_f2]; + pred.iter() + .zip(t.iter().zip(f.iter())) + .map(|(p, (&t, &f))| if p.is_true() { t } else { f }) + .collect::>() + } + _ => self + .1 + .strided_index() + .zip(t_l.strided_index().zip(f_l.strided_index())) + .map(|(i_p, (i_t, i_f))| { + if self.0[i_p].is_true() { + t[i_t] + } else { + f[i_f] + } + }) + .collect::>(), + }; + Ok(vs) + } +} + +struct ReduceIndex { + reduce_dim_index: usize, + use_min: bool, + return_index: bool, +} + +impl ReduceIndex { + // The value gets replaced if f(s[current_acc], s[i]) returns true. + #[inline(always)] + fn fold_impl(&self, src: &[T], src_l: &Layout, f: F, g: G) -> Result> + where + T: Clone + Copy, + U: Clone + Copy, + F: Fn(T, T) -> bool, + G: Fn(T, usize) -> U, + { + let reduce_dim_size = src_l.dims()[self.reduce_dim_index]; + let reduce_dim_stride = src_l.stride()[self.reduce_dim_index]; + let dst_len = src_l.shape().elem_count() / reduce_dim_size; + let mut dst: Vec = Vec::with_capacity(dst_len); + let dst_to_set = dst.spare_capacity_mut(); + let dst_to_set = + unsafe { std::mem::transmute::<&mut [std::mem::MaybeUninit], &mut [U]>(dst_to_set) }; + match src_l.contiguous_offsets() { + Some((o1, o2)) => { + let src = &src[o1..o2]; + if reduce_dim_stride == 1 { + for (start_src_i, dst_v) in dst_to_set.iter_mut().enumerate() { + let start_src_i = start_src_i * reduce_dim_size; + let src = &src[start_src_i..start_src_i + reduce_dim_size]; + let mut acc = 0; + let mut val = src[0]; + for (src_i, &s) in src.iter().enumerate() { + if f(val, s) { + acc = src_i; + val = s + } + } + *dst_v = g(val, acc) + } + } else { + for (start_src_i, dst_v) in dst_to_set.iter_mut().enumerate() { + let (p, q) = ( + start_src_i / reduce_dim_stride, + start_src_i % reduce_dim_stride, + ); + // start_src_i = p * reduce_dim_stride + q + let start_src_i = p * reduce_dim_stride * reduce_dim_size + q; + let src = &src[start_src_i..]; + let mut acc = 0; + let mut val = src[0]; + for src_i in 0..reduce_dim_size { + let s = src[src_i * reduce_dim_stride]; + if f(val, s) { + acc = src_i; + val = s + } + } + *dst_v = g(val, acc) + } + } + } + None => { + let l = src_l.narrow(self.reduce_dim_index, 0, 1)?; + for (unstr_index, src_index) in l.strided_index().enumerate() { + let src = &src[src_index..]; + let mut acc = 0; + let mut val = src[0]; + for src_i in 0..reduce_dim_size { + let s = src[src_i * reduce_dim_stride]; + if f(val, s) { + acc = src_i; + val = s + } + } + dst_to_set[unstr_index] = g(val, acc) + } + } + } + unsafe { dst.set_len(dst_len) }; + Ok(dst) + } +} + +impl Map1Any for ReduceIndex { + #[inline(always)] + fn f) -> CpuStorage>( + &self, + src: &[T], + src_l: &Layout, + wrap: W, + ) -> Result { + if src_l.shape().elem_count() == 0 { + Err(Error::EmptyTensor { op: "reduce" }.bt())? + } + let dst = match (self.return_index, self.use_min) { + (false, true) => wrap(self.fold_impl(src, src_l, |x, y| x > y, |v, _i| v)?), + (false, false) => wrap(self.fold_impl(src, src_l, |x, y| x < y, |v, _i| v)?), + (true, true) => { + CpuStorage::U32(self.fold_impl(src, src_l, |x, y| x > y, |_v, i| i as u32)?) + } + (true, false) => { + CpuStorage::U32(self.fold_impl(src, src_l, |x, y| x < y, |_v, i| i as u32)?) + } + }; + Ok(dst) + } +} + +struct ReduceSum<'a> { + dst_shape: &'a Shape, + reduce_dims: &'a [usize], + reduce_dims_and_stride: Vec<(usize, usize)>, +} + +impl ReduceSum<'_> { + #[inline(always)] + fn fold_impl(&self, src: &[T], src_l: &Layout, start_elt: T) -> Result> + where + T: WithDType, + { + let mut dst = vec![start_elt; self.dst_shape.elem_count()]; + match src_l.contiguous_offsets() { + Some((o1, o2)) => { + let src = &src[o1..o2]; + // Handle the case where we reduce over the last dimensions separately as it is + // fairly common and easy to optimize. This rely on the layout being contiguous! + // reduce_dims is sorted, check if it is ranging from a to n-1. + let reduce_over_last_dims = self + .reduce_dims + .iter() + .rev() + .enumerate() + .all(|(i, &v)| v == src_l.shape().rank() - 1 - i); + if reduce_over_last_dims { + let reduce_sz = self + .reduce_dims_and_stride + .iter() + .map(|(u, _)| u) + .product::(); + for (dst_i, dst_v) in dst.iter_mut().enumerate() { + let src_i = dst_i * reduce_sz; + unsafe { + T::vec_reduce_sum( + src[src_i..src_i + reduce_sz].as_ptr(), + dst_v, + reduce_sz, + ) + }; + } + return Ok(dst); + }; + for (unstr_index, &src) in src.iter().enumerate() { + let mut dst_index = unstr_index; + // Set the reduce_dims indexes to 0. + for &(dim, stride) in self.reduce_dims_and_stride.iter() { + // The compiler is able to optimize the following in a single divmod op. + let (pre, post) = (dst_index / stride, dst_index % stride); + dst_index = (pre / dim) * stride + post; + } + dst[dst_index] += src; + } + } + None => { + for (unstr_index, src_index) in src_l.strided_index().enumerate() { + let mut dst_index = unstr_index; + // Set the reduce_dims indexes to 0. + for &(dim, stride) in self.reduce_dims_and_stride.iter() { + // The compiler is able to optimize the following in a single divmod op. + let (pre, post) = (dst_index / stride, dst_index % stride); + dst_index = (pre / dim) * stride + post; + } + dst[dst_index] += src[src_index]; + } + } + } + Ok(dst) + } +} + +impl Map1 for ReduceSum<'_> { + #[inline(always)] + fn f(&self, src: &[T], src_l: &Layout) -> Result> { + self.fold_impl(src, src_l, T::zero()) + } +} + +struct Affine(f64, f64); + +impl Map1 for Affine { + fn f(&self, vs: &[T], layout: &Layout) -> Result> { + let mul = T::from_f64(self.0); + let add = T::from_f64(self.1); + Ok(unary_map(vs, layout, |v| v * mul + add)) + } +} + +struct AvgPool2D((usize, usize), (usize, usize)); + +impl Map1 for AvgPool2D { + fn f(&self, src: &[T], layout: &Layout) -> Result> { + // https://pytorch.org/docs/stable/generated/torch.nn.AvgPool2d.html + let (k_h, k_w) = self.0; + let (s_h, s_w) = self.1; + let (b_sz, c, h, w) = layout.shape().dims4()?; + let stride = layout.stride(); + let (stride_h, stride_w) = (stride[2], stride[3]); + let h_out = (h - k_h) / s_h + 1; + let w_out = (w - k_w) / s_w + 1; + let src_index = layout.start_offset(); + let mut dst = vec![T::zero(); b_sz * c * h_out * w_out]; + let scale = 1f64 / (k_h * k_w) as f64; + let scale = T::from_f64(scale); + for b_idx in 0..b_sz { + let dst = &mut dst[b_idx * c * h_out * w_out..]; + let src_index = src_index + b_idx * stride[0]; + for c_idx in 0..c { + let dst = &mut dst[c_idx * h_out * w_out..]; + let src_index = src_index + c_idx * stride[1]; + for h_idx in 0..h_out { + for w_idx in 0..w_out { + let mut sum = T::zero(); + for m in 0..k_h { + for n in 0..k_w { + let m = s_h * h_idx + m; + let n = s_w * w_idx + n; + sum += src[src_index + m * stride_h + n * stride_w] + } + } + dst[h_idx * w_out + w_idx] = sum * scale; + } + } + } + } + Ok(dst) + } +} + +struct MaxPool2D((usize, usize), (usize, usize)); + +impl Map1 for MaxPool2D { + fn f(&self, src: &[T], layout: &Layout) -> Result> { + // https://pytorch.org/docs/stable/generated/torch.nn.MaxPool2d.html + let (k_h, k_w) = self.0; + let (s_h, s_w) = self.1; + let (b_sz, c, h, w) = layout.shape().dims4()?; + let stride = layout.stride(); + let (stride_h, stride_w) = (stride[2], stride[3]); + let h_out = (h - k_h) / s_h + 1; + let w_out = (w - k_w) / s_w + 1; + let src_index = layout.start_offset(); + let mut dst = vec![T::zero(); b_sz * c * h_out * w_out]; + for b_idx in 0..b_sz { + let dst = &mut dst[b_idx * c * h_out * w_out..]; + let src_index = src_index + b_idx * stride[0]; + for c_idx in 0..c { + let dst = &mut dst[c_idx * h_out * w_out..]; + let src_index = src_index + c_idx * stride[1]; + for h_idx in 0..h_out { + for w_idx in 0..w_out { + let mut largest = + src[src_index + s_h * h_idx * stride_h + s_w * w_idx * stride_w]; + for m in 0..k_h { + for n in 0..k_w { + let m = s_h * h_idx + m; + let n = s_w * w_idx + n; + if largest < src[src_index + m * stride_h + n * stride_w] { + largest = src[src_index + m * stride_h + n * stride_w] + } + } + } + dst[h_idx * w_out + w_idx] = largest; + } + } + } + } + Ok(dst) + } +} + +struct UpsampleNearest1D(usize); + +impl Map1 for UpsampleNearest1D { + fn f(&self, src: &[T], layout: &Layout) -> Result> { + // TODO: Specialized implementation for the case 2*sz? + let dst_sz = self.0; + let (b_sz, c, src_sz) = layout.shape().dims3()?; + let stride = layout.stride(); + let stride_sz = stride[2]; + let src_index = layout.start_offset(); + let scale_sz = src_sz as f64 / dst_sz as f64; + let mut dst = vec![T::zero(); b_sz * c * dst_sz]; + let src_idxs = (0..dst_sz) + .map(|idx| usize::min(src_sz - 1, (idx as f64 * scale_sz) as usize)) + .collect::>(); + for b_idx in 0..b_sz { + let dst = &mut dst[b_idx * c * dst_sz..]; + let src_index = src_index + b_idx * stride[0]; + for c_idx in 0..c { + let dst = &mut dst[c_idx * dst_sz..]; + let src_index = src_index + c_idx * stride[1]; + for (idx, src_idx) in src_idxs.iter().enumerate() { + dst[idx] = src[src_index + src_idx * stride_sz] + } + } + } + Ok(dst) + } +} + +struct UpsampleNearest2D(usize, usize); + +impl Map1 for UpsampleNearest2D { + fn f(&self, src: &[T], layout: &Layout) -> Result> { + // TODO: Specialized implementation for the case 2*h, 2*w? + let (dst_h, dst_w) = (self.0, self.1); + let (b_sz, c, src_h, src_w) = layout.shape().dims4()?; + let stride = layout.stride(); + let (stride_h, stride_w) = (stride[2], stride[3]); + let src_index = layout.start_offset(); + let scale_h = src_h as f64 / dst_h as f64; + let scale_w = src_w as f64 / dst_w as f64; + let mut dst = vec![T::zero(); b_sz * c * dst_h * dst_w]; + let src_h_idxs = (0..dst_h) + .map(|h_idx| usize::min(src_h - 1, (h_idx as f64 * scale_h) as usize)) + .collect::>(); + let src_w_idxs = (0..dst_w) + .map(|w_idx| usize::min(src_w - 1, (w_idx as f64 * scale_w) as usize)) + .collect::>(); + for b_idx in 0..b_sz { + let dst = &mut dst[b_idx * c * dst_h * dst_w..]; + let src_index = src_index + b_idx * stride[0]; + for c_idx in 0..c { + let dst = &mut dst[c_idx * dst_h * dst_w..]; + let src_index = src_index + c_idx * stride[1]; + for (h_idx, src_h_idx) in src_h_idxs.iter().enumerate() { + for (w_idx, src_w_idx) in src_w_idxs.iter().enumerate() { + let src_index = src_index + src_h_idx * stride_h + src_w_idx * stride_w; + dst[h_idx * dst_w + w_idx] = src[src_index] + } + } + } + } + Ok(dst) + } +} + +struct UpsampleBilinear2D { + target_h: usize, + target_w: usize, + align_corners: bool, + scale_h_factor: Option, + scale_w_factor: Option, +} + +impl Map1 for UpsampleBilinear2D { + fn f(&self, src: &[T], layout: &Layout) -> Result> { + let (batch, channels, height_in, width_in) = layout.shape().dims4()?; + let height_out = self.target_h; + let width_out = self.target_w; + + // Early return for identity case + if height_in == height_out && width_in == width_out { + return Ok(src.to_vec()); + } + + let stride = layout.stride(); + let src_offset = layout.start_offset(); + + // Calculate scale factors following PyTorch's area_pixel_compute_scale logic + let scale_h = if self.align_corners { + if height_out > 1 { + (height_in - 1) as f64 / (height_out - 1) as f64 + } else { + 0.0 + } + } else { + // PyTorch's compute_scales_value logic: + // If scale_factor was provided, use 1.0 / scale_factor + // Otherwise, use input_size / output_size + if let Some(scale_factor) = self.scale_h_factor { + 1.0 / scale_factor + } else { + height_in as f64 / height_out as f64 + } + }; + + let scale_w = if self.align_corners { + if width_out > 1 { + (width_in - 1) as f64 / (width_out - 1) as f64 + } else { + 0.0 + } + } else if let Some(scale_factor) = self.scale_w_factor { + 1.0 / scale_factor + } else { + width_in as f64 / width_out as f64 + }; + + // Precompute indices and weights for height + let mut h_indices = Vec::with_capacity(height_out); + for h_out in 0..height_out { + let src_h = if self.align_corners { + scale_h * h_out as f64 + } else { + scale_h * (h_out as f64 + 0.5) - 0.5 + }; + let src_h_clamped = src_h.max(0.0); + let h0 = src_h_clamped.floor() as usize; + let h1 = (h0 + 1).min(height_in - 1); + let weight_h = (src_h_clamped - h0 as f64).clamp(0.0, 1.0); + h_indices.push((h0, h1, weight_h)); + } + + // Precompute indices and weights for width + let mut w_indices = Vec::with_capacity(width_out); + for w_out in 0..width_out { + let src_w = if self.align_corners { + scale_w * w_out as f64 + } else { + scale_w * (w_out as f64 + 0.5) - 0.5 + }; + let src_w_clamped = src_w.max(0.0); + let w0 = src_w_clamped.floor() as usize; + let w1 = (w0 + 1).min(width_in - 1); + let weight_w = (src_w_clamped - w0 as f64).clamp(0.0, 1.0); + w_indices.push((w0, w1, weight_w)); + } + + // Allocate output + let mut dst = vec![T::zero(); batch * channels * height_out * width_out]; + + // Perform bilinear interpolation + for b in 0..batch { + for c in 0..channels { + let base_idx = src_offset + b * stride[0] + c * stride[1]; + let dst_base = (b * channels + c) * height_out * width_out; + + for (h_out, &(h0, h1, weight_h)) in h_indices.iter().enumerate() { + for (w_out, &(w0, w1, weight_w)) in w_indices.iter().enumerate() { + // Get four neighboring pixels + let idx_00 = base_idx + h0 * stride[2] + w0 * stride[3]; + let idx_10 = base_idx + h0 * stride[2] + w1 * stride[3]; + let idx_01 = base_idx + h1 * stride[2] + w0 * stride[3]; + let idx_11 = base_idx + h1 * stride[2] + w1 * stride[3]; + + let v00 = src[idx_00].to_f64(); + let v10 = src[idx_10].to_f64(); + let v01 = src[idx_01].to_f64(); + let v11 = src[idx_11].to_f64(); + + // Bilinear interpolation + let v_top = v00 * (1.0 - weight_w) + v10 * weight_w; + let v_bottom = v01 * (1.0 - weight_w) + v11 * weight_w; + let value = v_top * (1.0 - weight_h) + v_bottom * weight_h; + + dst[dst_base + h_out * width_out + w_out] = T::from_f64(value); + } + } + } + } + + Ok(dst) + } +} + +struct Gather<'a, I: IntDType> { + ids: &'a [I], + ids_l: &'a Layout, + dim: usize, +} + +impl Map1 for Gather<'_, I> { + fn f(&self, src: &[T], src_l: &Layout) -> Result> { + let ids = match self.ids_l.contiguous_offsets() { + Some((a, b)) => &self.ids[a..b], + None => Err(Error::RequiresContiguous { op: "gather" }.bt())?, + }; + let src = match src_l.contiguous_offsets() { + Some((a, b)) => &src[a..b], + None => Err(Error::RequiresContiguous { op: "gather" }.bt())?, + }; + let dim = self.dim; + let ids_dims = self.ids_l.dims(); + let src_dims = src_l.dims(); + let dst_len: usize = ids_dims.iter().product(); + let dst_left_len: usize = ids_dims[..dim].iter().product(); + let dst_dim_len = ids_dims[dim]; + let dst_right_len: usize = ids_dims[dim + 1..].iter().product(); + + let src_dim_len = src_dims[dim]; + let src_right_len: usize = src_dims[dim + 1..].iter().product(); + + let mut dst = vec![T::zero(); dst_len]; + for left_i in 0..dst_left_len { + let start_src_idx = left_i * src_right_len * src_dim_len; + let start_dst_idx = left_i * dst_right_len * dst_dim_len; + for i in 0..dst_dim_len { + let start_dst_idx = start_dst_idx + i * dst_right_len; + for right_i in 0..dst_right_len { + let dst_idx = start_dst_idx + right_i; + let index = ids[dst_idx]; + if index == I::max_value() { + dst[dst_idx] = T::zero(); + } else { + let index = index.as_usize(); + if index >= src_dim_len { + Err(Error::InvalidIndex { + index, + size: src_dim_len, + op: "gather", + } + .bt())? + } + let src_idx = start_src_idx + index * src_right_len + right_i; + dst[dst_idx] = src[src_idx] + } + } + } + } + Ok(dst) + } +} + +struct IndexSelect<'a, T: IntDType> { + ids: &'a [T], + ids_l: &'a Layout, + dim: usize, +} + +impl Map1 for IndexSelect<'_, I> { + fn f(&self, src: &[T], layout: &Layout) -> Result> { + let src = match layout.contiguous_offsets() { + Some((a, b)) => &src[a..b], + None => Err(Error::RequiresContiguous { op: "index-select" }.bt())?, + }; + let dim = self.dim; + let n_ids = match self.ids_l.dims() { + [n_ids] => *n_ids, + d => Err(Error::UnexpectedNumberOfDims { + expected: 1, + got: d.len(), + shape: self.ids_l.shape().clone(), + } + .bt())?, + }; + let stride_ids = self.ids_l.stride()[0]; + let mut dst_dims = layout.dims().to_vec(); + let src_dim = dst_dims[dim]; + dst_dims[dim] = n_ids; + let dst_len: usize = dst_dims.iter().product(); + let left_len: usize = dst_dims[..dim].iter().product(); + let right_len: usize = dst_dims[dim + 1..].iter().product(); + let mut dst = vec![T::zero(); dst_len]; + for left_i in 0..left_len { + let start_src_idx = left_i * right_len * src_dim; + let start_dst_idx = left_i * right_len * n_ids; + for i in 0..n_ids { + let start_dst_idx = start_dst_idx + i * right_len; + let index = self.ids[self.ids_l.start_offset() + stride_ids * i]; + if index == I::max_value() { + dst[start_dst_idx..start_dst_idx + right_len].fill(T::zero()); + } else { + let index = index.as_usize(); + if index >= src_dim { + Err(Error::InvalidIndex { + index, + size: src_dim, + op: "index-select", + } + .bt())? + } + let start_src_idx = start_src_idx + index * right_len; + dst[start_dst_idx..start_dst_idx + right_len] + .copy_from_slice(&src[start_src_idx..start_src_idx + right_len]) + } + } + } + Ok(dst) + } +} + +trait ElemUpdate { + fn f(dst: &mut T, src: T); +} + +struct Set; +struct Add; + +impl ElemUpdate for Set { + fn f(dst: &mut T, src: T) { + *dst = src + } +} + +impl ElemUpdate for Add { + fn f(dst: &mut T, src: T) { + *dst += src + } +} + +struct Scatter<'a, I: IntDType, M: ElemUpdate> { + ids: &'a [I], + ids_l: &'a Layout, + dim: usize, + _phantom: std::marker::PhantomData, +} + +impl<'a, I: IntDType, M: ElemUpdate> Scatter<'a, I, M> { + fn new(ids: &'a [I], ids_l: &'a Layout, dim: usize) -> Self { + Self { + ids, + ids_l, + dim, + _phantom: Default::default(), + } + } +} + +impl Map2InPlace for Scatter<'_, I, M> { + const OP: &'static str = "scatter"; + fn f( + &self, + dst: &mut [T], + dst_l: &Layout, + src: &[T], + src_l: &Layout, + ) -> Result<()> { + let dst = match dst_l.contiguous_offsets() { + None => Err(Error::RequiresContiguous { op: "scatter" }.bt())?, + Some((o1, o2)) => &mut dst[o1..o2], + }; + + let src = match src_l.contiguous_offsets() { + None => Err(Error::RequiresContiguous { op: "scatter" }.bt())?, + Some((o1, o2)) => &src[o1..o2], + }; + + let dim = self.dim; + let ids_dims = self.ids_l.dims(); + let dst_dims = dst_l.dims(); + let dst_dim_len = dst_dims[dim]; + let dst_right_len: usize = dst_dims[dim + 1..].iter().product(); + + let ids_left_len: usize = ids_dims[..dim].iter().product(); + let ids_dim_len = ids_dims[dim]; + let ids_right_len: usize = ids_dims[dim + 1..].iter().product(); + + let ids = match self.ids_l.contiguous_offsets() { + Some((a, b)) => &self.ids[a..b], + None => Err(Error::RequiresContiguous { op: "gather" }.bt())?, + }; + for left_i in 0..ids_left_len { + let start_ids_idx = left_i * ids_right_len * ids_dim_len; + let start_dst_idx = left_i * dst_right_len * dst_dim_len; + for i in 0..ids_dim_len { + let start_ids_idx = start_ids_idx + i * ids_right_len; + for right_i in 0..dst_right_len { + let ids_idx = start_ids_idx + right_i; + let index = ids[ids_idx]; + if index == I::max_value() { + continue; + } + let index = index.as_usize(); + if index >= dst_dim_len { + Err(Error::InvalidIndex { + index, + size: dst_dim_len, + op: "gather", + } + .bt())? + } + let dst_idx = start_dst_idx + index * dst_right_len + right_i; + M::f(&mut dst[dst_idx], src[ids_idx]) + } + } + } + + Ok(()) + } +} + +struct IndexAdd<'a, I: IntDType> { + ids: &'a [I], + dim: usize, +} + +impl Map2 for IndexAdd<'_, I> { + const OP: &'static str = "index-add"; + // https://pytorch.org/docs/stable/generated/torch.Tensor.index_add_.html#torch.Tensor.index_add_ + // v1, l1 -> self + fn f(&self, v1: &[T], l1: &Layout, src: &[T], src_l: &Layout) -> Result> { + let dst_len = l1.shape().elem_count(); + let mut dst = vec![T::zero(); dst_len]; + copy_strided_src_(v1, &mut dst, 0, l1); + let src = match src_l.contiguous_offsets() { + None => Err(Error::RequiresContiguous { op: "index-add" }.bt())?, + Some((o1, o2)) => &src[o1..o2], + }; + let dim = self.dim; + let max_idx = l1.dims()[dim]; + let pre_dim = src_l.dims()[..dim].iter().product::(); + let src_dim_sz = src_l.dims()[dim]; + let post_dim = src_l.dims()[dim + 1..].iter().product::(); + if dim == 0 { + for (src_idx, dst_idx) in self.ids.iter().enumerate() { + if *dst_idx == I::max_value() { + continue; + } + let dst_idx = dst_idx.as_usize(); + if dst_idx >= max_idx { + Err(Error::InvalidIndex { + index: dst_idx, + op: "index-add", + size: max_idx, + })? + } + let src_idx = src_idx * post_dim; + let dst_idx = dst_idx * post_dim; + let src = &src[src_idx..src_idx + post_dim]; + let dst = &mut dst[dst_idx..dst_idx + post_dim]; + for (d, &s) in dst.iter_mut().zip(src.iter()) { + *d += s + } + } + } else { + for (src_idx, dst_idx) in self.ids.iter().enumerate() { + if *dst_idx == I::max_value() { + continue; + } + let dst_idx = dst_idx.as_usize(); + if dst_idx >= max_idx { + Err(Error::InvalidIndex { + index: dst_idx, + op: "index-add", + size: max_idx, + })? + } + for pre_i in 0..pre_dim { + let pre_src_i = (pre_i * src_dim_sz + src_idx) * post_dim; + let pre_dst_i = (pre_i * max_idx + dst_idx) * post_dim; + let src = &src[pre_src_i..pre_src_i + post_dim]; + let dst = &mut dst[pre_dst_i..pre_dst_i + post_dim]; + for (d, &s) in dst.iter_mut().zip(src.iter()) { + *d += s + } + } + } + } + Ok(dst) + } +} + +#[allow(clippy::too_many_arguments)] +fn copy2d_( + src: &[T], + dst: &mut [T], + d1: usize, + d2: usize, + src_stride1: usize, + dst_stride1: usize, + src_offset: usize, + dst_offset: usize, +) { + for i1 in 0..d1 { + let dst_idx = i1 * dst_stride1 + dst_offset; + let src_idx = i1 * src_stride1 + src_offset; + let dst = &mut dst[dst_idx..dst_idx + d2]; + let src = &src[src_idx..src_idx + d2]; + dst.copy_from_slice(src) + } +} + +fn copy_strided_src_(src: &[T], dst: &mut [T], dst_offset: usize, src_l: &Layout) { + match src_l.strided_blocks() { + crate::StridedBlocks::SingleBlock { start_offset, len } => { + let to_copy = (dst.len() - dst_offset).min(len); + dst[dst_offset..dst_offset + to_copy] + .copy_from_slice(&src[start_offset..start_offset + to_copy]) + } + crate::StridedBlocks::MultipleBlocks { + block_start_index, + block_len: 1, + } => { + for (dst_index, src_index) in block_start_index.enumerate() { + let dst_index = dst_index + dst_offset; + if dst_index >= dst.len() { + break; + } + dst[dst_index] = src[src_index] + } + } + crate::StridedBlocks::MultipleBlocks { + block_start_index, + block_len, + } => { + let mut dst_index = dst_offset; + for src_index in block_start_index { + let next_dst_index = dst_index + block_len; + if dst_index >= dst.len() { + break; + } + let to_copy = usize::min(block_len, dst.len() - dst_index); + dst[dst_index..dst_index + to_copy] + .copy_from_slice(&src[src_index..src_index + to_copy]); + dst_index = next_dst_index + } + } + } +} + +struct Conv1D<'a>(&'a crate::conv::ParamsConv1D); + +impl Map2 for Conv1D<'_> { + const OP: &'static str = "conv1d"; + fn f(&self, inp: &[T], inp_l: &Layout, k: &[T], k_l: &Layout) -> Result> { + let p = self.0; + let inp = &inp[inp_l.start_offset()..]; + let k = &k[k_l.start_offset()..]; + let (inp_s0, inp_s1, inp_s2) = crate::shape::dims3(inp_l.stride())?; + let (k_s0, k_s1, k_s2) = crate::shape::dims3(k_l.stride())?; + let l_out = p.l_out(); + let dst_elems = p.c_out * l_out * p.b_size; + // The output shape is [b_size, c_out, l_out] + let dst = vec![T::zero(); dst_elems]; + + // TODO: Avoid making this copy if `inp` already has the appropriate layout. + let mut inp_cont = vec![T::zero(); p.b_size * p.c_in * p.l_in]; + for b_idx in 0..p.b_size { + for src_l in 0..p.l_in { + for src_c_idx in 0..p.c_in { + let inp_idx = b_idx * inp_s0 + src_c_idx * inp_s1 + src_l * inp_s2; + inp_cont[b_idx * p.l_in * p.c_in + src_l * p.c_in + src_c_idx] = inp[inp_idx] + } + } + } + + for offset in 0..p.k_size { + (0..p.c_out).into_par_iter().for_each(|dst_c_idx| { + let dst_idx = dst_c_idx * l_out; + let k_cont = (0..p.c_in) + .map(|c_in_idx| k[dst_c_idx * k_s0 + c_in_idx * k_s1 + offset * k_s2]) + .collect::>(); + for b_idx in 0..p.b_size { + let dst_idx = dst_idx + b_idx * p.c_out * l_out; + for dst_l in 0..l_out { + let dst_idx = dst_idx + dst_l; + let src_l = p.stride * dst_l + offset * p.dilation; + if src_l < p.padding || src_l >= p.padding + p.l_in { + continue; + } + let src_l = src_l - p.padding; + let inp_cont = &inp_cont[b_idx * p.l_in * p.c_in + src_l * p.c_in..]; + assert!(inp_cont.len() >= p.c_in); + assert!(k_cont.len() >= p.c_in); + let mut d = T::zero(); + unsafe { T::vec_dot(inp_cont.as_ptr(), k_cont.as_ptr(), &mut d, p.c_in) } + let dst_p = dst.as_ptr(); + // Safety: dst_idx are uniques per dst_c_idx which is used to parallelise + // the different tasks so no two threads can try to write at the same + // location. + unsafe { + let ptr = dst_p.add(dst_idx) as *mut T; + *ptr += d + } + } + } + }) + } + Ok(dst) + } +} + +struct Im2Col1D { + l_k: usize, + stride: usize, + dilation: usize, + padding: usize, +} + +impl Im2Col1D { + fn l_out(&self, l: usize) -> usize { + (l + 2 * self.padding - self.dilation * (self.l_k - 1) - 1) / self.stride + 1 + } +} + +impl Map1 for Im2Col1D { + fn f(&self, vs: &[T], layout: &Layout) -> Result> { + let &Self { + l_k, + stride, + dilation, + padding, + } = self; + let (b, c, l) = layout.shape().dims3()?; + let l_out = self.l_out(l); + let src = &vs[layout.start_offset()..]; + let mut dst = vec![T::zero(); b * l_out * c * l_k]; + let (src_s0, src_s1, src_s2) = { + let s = layout.stride(); + (s[0], s[1], s[2]) + }; + // TODO: provide specialized kernels for the common use cases. + // - l_k = 1 + // - padding = 0 + // - stride = 1 + // - dilation = 1 + for b_idx in 0..b { + let src_idx = b_idx * src_s0; + let dst_idx = b_idx * l_out * c * l_k; + for l_idx in 0..l_out { + let dst_idx = dst_idx + l_idx * c * l_k; + for c_idx in 0..c { + let dst_idx = dst_idx + c_idx * l_k; + let src_idx = c_idx * src_s1 + src_idx; + for l_k_idx in 0..l_k { + let src_l = l_idx * stride + l_k_idx * dilation; + if padding != 0 && (src_l < padding || src_l >= l + padding) { + continue; + } + let src_l = src_l - padding; + let src_idx = src_idx + src_l * src_s2; + let dst_idx = dst_idx + l_k_idx; + dst[dst_idx] = src[src_idx] + } + } + } + } + Ok(dst) + } +} + +struct Im2Col { + h_k: usize, + w_k: usize, + stride: usize, + dilation: usize, + padding: usize, +} + +impl Im2Col { + fn hw_out(&self, h: usize, w: usize) -> (usize, usize) { + let h_out = (h + 2 * self.padding - self.dilation * (self.h_k - 1) - 1) / self.stride + 1; + let w_out = (w + 2 * self.padding - self.dilation * (self.w_k - 1) - 1) / self.stride + 1; + (h_out, w_out) + } +} + +impl Map1 for Im2Col { + fn f(&self, vs: &[T], layout: &Layout) -> Result> { + let &Self { + h_k, + w_k, + stride, + dilation, + padding, + } = self; + let (b, c, h, w) = layout.shape().dims4()?; + let (h_out, w_out) = self.hw_out(h, w); + let src = &vs[layout.start_offset()..]; + let mut dst = vec![T::zero(); b * h_out * w_out * c * h_k * w_k]; + let (src_s0, src_s1, src_s2, src_s3) = { + let s = layout.stride(); + (s[0], s[1], s[2], s[3]) + }; + // TODO: provide specialized kernels for the common use cases. + // - h_k = w_k = 1 + // - padding = 0 + // - stride = 1 + // - dilation = 1 + for b_idx in 0..b { + let src_idx = b_idx * src_s0; + let dst_idx = b_idx * h_out * w_out * c * h_k * w_k; + for h_idx in 0..h_out { + let dst_idx = dst_idx + h_idx * w_out * c * h_k * w_k; + for w_idx in 0..w_out { + let dst_idx = dst_idx + w_idx * c * h_k * w_k; + for c_idx in 0..c { + let dst_idx = dst_idx + c_idx * h_k * w_k; + let src_idx = c_idx * src_s1 + src_idx; + for h_k_idx in 0..h_k { + let src_h = h_idx * stride + h_k_idx * dilation; + if padding != 0 && (src_h < padding || src_h >= h + padding) { + continue; + } + let src_h = src_h - padding; + let src_idx = src_idx + src_h * src_s2; + let dst_idx = dst_idx + h_k_idx * w_k; + for w_k_idx in 0..w_k { + let src_w = w_idx * stride + w_k_idx * dilation; + if padding != 0 && (src_w < padding || src_w >= w + padding) { + continue; + } + let src_w = src_w - padding; + let src_idx = src_idx + src_w * src_s3; + let dst_idx = dst_idx + w_k_idx; + dst[dst_idx] = src[src_idx] + } + } + } + } + } + } + Ok(dst) + } +} + +struct Col2Im1D { + stride: usize, +} + +impl Map1 for Col2Im1D { + fn f(&self, col: &[T], l: &Layout) -> Result> { + let (b_size, l_in, c_out, k_size) = l.shape().dims4()?; + let stride = self.stride; + let l_out = (l_in - 1) * stride + k_size; + let mut im = vec![T::zero(); b_size * c_out * l_out]; + let (dst_s0, dst_s1) = (c_out * l_out, l_out); + let (src_s0, src_s1, src_s2) = (c_out * k_size * l_in, c_out * k_size, k_size); + for l_in_i in 0..l_in { + for k_i in 0..k_size { + let l_out_i = l_in_i * stride + k_i; + for b_i in 0..b_size { + for c_i in 0..c_out { + let dst_idx = b_i * dst_s0 + c_i * dst_s1 + l_out_i; + let src_idx = b_i * src_s0 + l_in_i * src_s1 + c_i * src_s2 + k_i; + im[dst_idx] += col[src_idx] + } + } + } + } + Ok(im) + } +} + +struct ConvTranspose1D<'a>(&'a crate::conv::ParamsConvTranspose1D); + +impl Map2 for ConvTranspose1D<'_> { + const OP: &'static str = "conv_transpose1d"; + fn f(&self, inp: &[T], inp_l: &Layout, k: &[T], k_l: &Layout) -> Result> { + let p = self.0; + let inp = &inp[inp_l.start_offset()..]; + let k = &k[k_l.start_offset()..]; + let (inp_s0, inp_s1, inp_s2) = crate::shape::dims3(inp_l.stride())?; + let (k_s0, k_s1, k_s2) = crate::shape::dims3(k_l.stride())?; + let l_out = p.l_out(); + + // Output shape: [b_size, c_out, l_out]. + let dst_elems = p.c_out * l_out * p.b_size; + let dst = vec![T::zero(); dst_elems]; + let dst_s0 = p.c_out * l_out; + let dst_s1 = l_out; + let dst_s2 = 1; + + // TODO: Avoid making this copy if `inp` already has the appropriate layout. + let mut inp_cont = vec![T::zero(); p.b_size * p.c_in * p.l_in]; + let cont_s0 = p.l_in * p.c_in; + let cont_s1 = p.c_in; + for b_idx in 0..p.b_size { + for l_idx in 0..p.l_in { + for c_idx in 0..p.c_in { + let src_idx = b_idx * inp_s0 + c_idx * inp_s1 + l_idx * inp_s2; + let dst_idx = b_idx * cont_s0 + l_idx * cont_s1 + c_idx; + inp_cont[dst_idx] = inp[src_idx] + } + } + } + + for k_idx in 0..p.k_size { + (0..p.c_out).into_par_iter().for_each(|dst_c_idx| { + let k_cont = (0..p.c_in) + .map(|c_in_idx| k[c_in_idx * k_s0 + dst_c_idx * k_s1 + k_idx * k_s2]) + .collect::>(); + for b_idx in 0..p.b_size { + for l_idx in 0..p.l_in { + let out_idx = l_idx * p.stride + k_idx * p.dilation; + if out_idx < p.padding { + continue; + } + let out_idx = out_idx - p.padding; + if out_idx < l_out { + let inp_cont = &inp_cont[b_idx * cont_s0 + l_idx * cont_s1..]; + let dst_idx = b_idx * dst_s0 + out_idx * dst_s2 + dst_c_idx * dst_s1; + let mut d = T::zero(); + unsafe { + T::vec_dot(inp_cont.as_ptr(), k_cont.as_ptr(), &mut d, p.c_in) + } + let dst_p = dst.as_ptr(); + // Safety: dst_idx are uniques per dst_c_idx which is used to + // parallelise the different tasks so no two threads can try to + // write at the same location. + unsafe { + let ptr = dst_p.add(dst_idx) as *mut T; + *ptr += d + } + } + } + } + }) + } + Ok(dst) + } +} + +struct ConvTranspose2D<'a>(&'a crate::conv::ParamsConvTranspose2D); + +impl Map2 for ConvTranspose2D<'_> { + const OP: &'static str = "conv_transpose2d"; + fn f(&self, inp: &[T], inp_l: &Layout, k: &[T], k_l: &Layout) -> Result> { + let p = self.0; + let inp = &inp[inp_l.start_offset()..]; + let (inp_s0, inp_s1, inp_s2, inp_s3) = crate::shape::dims4(inp_l.stride())?; + let k = &k[k_l.start_offset()..]; + let (k_s0, k_s1, k_s2, k_s3) = crate::shape::dims4(k_l.stride())?; + let (out_h, out_w) = (p.out_h(), p.out_w()); + + // Output shape: [b_size, c_out, out_h, out_w]. + let dst = vec![T::zero(); p.b_size * p.c_out * out_h * out_w]; + let dst_s0 = p.c_out * out_h * out_w; + let dst_s1 = out_h * out_w; + let dst_s2 = out_w; + let dst_s3 = 1; + + // TODO: Avoid making this copy if `inp` already has the appropriate layout. + let mut inp_cont = vec![T::zero(); p.b_size * p.c_in * p.i_h * p.i_w]; + let cont_s0 = p.i_h * p.i_w * p.c_in; + let cont_s1 = p.i_w * p.c_in; + let cont_s2 = p.c_in; + for b_idx in 0..p.b_size { + for h_idx in 0..p.i_h { + for w_idx in 0..p.i_w { + for c_idx in 0..p.c_in { + let src_idx = + b_idx * inp_s0 + c_idx * inp_s1 + h_idx * inp_s2 + w_idx * inp_s3; + let dst_idx = b_idx * cont_s0 + h_idx * cont_s1 + w_idx * cont_s2 + c_idx; + inp_cont[dst_idx] = inp[src_idx] + } + } + } + } + + for k_y in 0..p.k_h { + for k_x in 0..p.k_w { + (0..p.c_out).into_par_iter().for_each(|dst_c_idx| { + let k_cont = (0..p.c_in) + .map(|c_in_idx| { + k[c_in_idx * k_s0 + dst_c_idx * k_s1 + k_y * k_s2 + k_x * k_s3] + }) + .collect::>(); + for b_idx in 0..p.b_size { + for inp_y in 0..p.i_h { + for inp_x in 0..p.i_w { + let out_x = inp_x * p.stride + k_x * p.dilation; + let out_y = inp_y * p.stride + k_y * p.dilation; + if out_x < p.padding || out_y < p.padding { + continue; + } + let out_x = out_x - p.padding; + let out_y = out_y - p.padding; + if out_x < out_w && out_y < out_h { + let inp_cont = &inp_cont + [b_idx * cont_s0 + inp_y * cont_s1 + inp_x * cont_s2..]; + let dst_idx = b_idx * dst_s0 + + out_y * dst_s2 + + out_x * dst_s3 + + dst_c_idx * dst_s1; + let mut d = T::zero(); + unsafe { + T::vec_dot( + inp_cont.as_ptr(), + k_cont.as_ptr(), + &mut d, + p.c_in, + ) + } + let dst_p = dst.as_ptr(); + // Safety: dst_idx are uniques per dst_c_idx which is used to + // parallelise the different tasks so no two threads can try to + // write at the same location. + unsafe { + let ptr = dst_p.add(dst_idx) as *mut T; + *ptr += d + } + } + } + } + } + }) + } + } + Ok(dst) + } +} + +struct MatMul((usize, usize, usize, usize)); + +impl MatMul { + fn striding_error(&self, lhs_l: &Layout, rhs_l: &Layout, msg: &'static str) -> Error { + Error::MatMulUnexpectedStriding(Box::new(crate::error::MatMulUnexpectedStriding { + lhs_l: lhs_l.clone(), + rhs_l: rhs_l.clone(), + bmnk: self.0, + msg, + })) + .bt() + } + + fn ab_skip(&self, lhs_l: &Layout, rhs_l: &Layout) -> Result<(usize, usize)> { + let lhs_stride = lhs_l.stride(); + let rhs_stride = rhs_l.stride(); + let rank = lhs_stride.len(); + let (_b, m, n, k) = self.0; + let a_skip: usize = match lhs_stride[..rank - 2] { + [s1, stride] if s1 == stride * lhs_l.dims()[1] => stride, + [_, stride] if lhs_l.dims()[0] == 1 => stride, + [stride, _] if lhs_l.dims()[1] == 1 => stride, + [stride] => stride, + [] => m * k, + _ => Err(self.striding_error(lhs_l, rhs_l, "non-contiguous lhs"))?, + }; + let b_skip: usize = match rhs_stride[..rank - 2] { + [s1, stride] if s1 == stride * rhs_l.dims()[1] => stride, + [_, stride] if rhs_l.dims()[0] == 1 => stride, + [stride, _] if rhs_l.dims()[1] == 1 => stride, + [stride] => stride, + [] => n * k, + _ => Err(self.striding_error(lhs_l, rhs_l, "non-contiguous rhs"))?, + }; + Ok((a_skip, b_skip)) + } +} + +impl Map2 for MatMul { + const OP: &'static str = "mat_mul"; + + #[cfg(all(not(feature = "mkl"), not(feature = "accelerate")))] + fn f( + &self, + lhs: &[T], + lhs_l: &Layout, + rhs: &[T], + rhs_l: &Layout, + ) -> Result> { + use gemm::{gemm, Parallelism}; + + match T::DTYPE { + DType::F16 | DType::F32 | DType::F64 => {} + _ => Err(Error::UnsupportedDTypeForOp(T::DTYPE, "matmul").bt())?, + } + + let (b, m, n, k) = self.0; + let lhs = &lhs[lhs_l.start_offset()..]; + let rhs = &rhs[rhs_l.start_offset()..]; + + let lhs_stride = lhs_l.stride(); + let rhs_stride = rhs_l.stride(); + let rank = lhs_stride.len(); + let lhs_cs = lhs_stride[rank - 1]; + let lhs_rs = lhs_stride[rank - 2]; + + let rhs_cs = rhs_stride[rank - 1]; + let rhs_rs = rhs_stride[rank - 2]; + + let (a_skip, b_skip) = self.ab_skip(lhs_l, rhs_l)?; + let c_skip: usize = m * n; + + let dst_shape: Shape = (m, n).into(); + let dst_strides = dst_shape.stride_contiguous(); + let dst_rs = dst_strides[0]; + let dst_cs = dst_strides[1]; + + let mut dst = vec![T::zero(); b * m * n]; + let num_threads = crate::utils::get_num_threads(); + let parallelism = if num_threads > 1 { + Parallelism::Rayon(num_threads) + } else { + Parallelism::None + }; + let (b, m, n, k) = if b_skip == 0 && a_skip == m * k { + // a_skip and c_skip should be updated but step is always 0 so + // it wouldn't matter. + (1, b * m, n, k) + } else if a_skip == 0 && b_skip == n * k { + (1, m, b * n, k) + } else { + (b, m, n, k) + }; + for step in 0..b { + let lhs_p = &lhs[step * a_skip..]; + let rhs_p = &rhs[step * b_skip..]; + let dst_p = &mut dst[step * c_skip..]; + unsafe { + gemm( + /* m: usize = */ m, + /* n: usize = */ n, + /* k: usize = */ k, + /* dst: *mut T = */ dst_p.as_mut_ptr(), + /* dst_cs: isize = */ dst_cs as isize, + /* dst_rs: isize = */ dst_rs as isize, + /* read_dst: bool = */ false, + /* lhs: *const T = */ lhs_p.as_ptr(), + /* lhs_cs: isize = */ lhs_cs as isize, + /* lhs_rs: isize = */ lhs_rs as isize, + /* rhs: *const T = */ rhs_p.as_ptr(), + /* rhs_cs: isize = */ rhs_cs as isize, + /* rhs_rs: isize = */ rhs_rs as isize, + /* alpha: T = */ T::zero(), + /* beta: T = */ T::one(), + /* conj_dst: bool = */ false, + /* conj_lhs: bool = */ false, + /* conj_rhs: bool = */ false, + parallelism, + ) + } + } + Ok(dst) + } + + #[cfg(feature = "accelerate")] + fn f( + &self, + lhs: &[T], + lhs_l: &Layout, + rhs: &[T], + rhs_l: &Layout, + ) -> Result> { + let (b, m, n, k) = self.0; + let lhs = &lhs[lhs_l.start_offset()..]; + let rhs = &rhs[rhs_l.start_offset()..]; + + let lhs_stride = lhs_l.stride(); + let rhs_stride = rhs_l.stride(); + + let (a_skip, b_skip) = self.ab_skip(lhs_l, rhs_l)?; + let c_skip: usize = m * n; + + let rhs_m1 = rhs_stride[rhs_stride.len() - 1]; + let rhs_m2 = rhs_stride[rhs_stride.len() - 2]; + let lhs_m1 = lhs_stride[lhs_stride.len() - 1]; + let lhs_m2 = lhs_stride[lhs_stride.len() - 2]; + + let (lda, transa) = if (rhs_m1 == 1 || n == 1) && (rhs_m2 == n || k == 1) { + (n as i32, b'N') + } else if rhs_m1 == k && rhs_m2 == 1 { + (k as i32, b'T') + } else { + Err(self.striding_error(lhs_l, rhs_l, "non-contiguous rhs"))? + }; + // The b tensor has dims batching, m, k (lhs) + let (ldb, transb) = if (lhs_m1 == 1 || k == 1) && (lhs_m2 == k || m == 1) { + (k as i32, b'N') + } else if lhs_m1 == m && lhs_m2 == 1 { + (m as i32, b'T') + } else { + Err(self.striding_error(lhs_l, rhs_l, "non-contiguous lhs"))? + }; + + let mut dst = vec![T::zero(); b * m * n]; + match T::DTYPE { + DType::F16 => { + crate::bail!("the accelerate backend does not support f16 matmul") + } + DType::F32 => { + for step in 0..b { + let lhs_p = &lhs[step * a_skip..]; + let rhs_p = &rhs[step * b_skip..]; + let dst_p = &mut dst[step * c_skip..]; + unsafe { + let a = rhs_p.as_ptr() as *const f32; + let b = lhs_p.as_ptr() as *const f32; + let c = dst_p.as_mut_ptr() as *mut f32; + let a = std::slice::from_raw_parts(a, a_skip); + let b = std::slice::from_raw_parts(b, b_skip); + let c = std::slice::from_raw_parts_mut(c, c_skip); + crate::accelerate::sgemm( + transa, transb, /* m= */ n as i32, /* n= */ m as i32, + /* k= */ k as i32, /* alpha= */ 1., /* a= */ a, + /* lda= */ lda, /* b= */ b, /* ldb= */ ldb, + /* beta= */ 0., /* c= */ c, /* ldc= */ n as i32, + ) + } + } + } + DType::F64 => { + for step in 0..b { + let lhs_p = &lhs[step * a_skip..]; + let rhs_p = &rhs[step * b_skip..]; + let dst_p = &mut dst[step * c_skip..]; + unsafe { + let a = rhs_p.as_ptr() as *const f64; + let b = lhs_p.as_ptr() as *const f64; + let c = dst_p.as_mut_ptr() as *mut f64; + let a = std::slice::from_raw_parts(a, a_skip); + let b = std::slice::from_raw_parts(b, b_skip); + let c = std::slice::from_raw_parts_mut(c, c_skip); + crate::accelerate::dgemm( + transa, transb, /* m= */ n as i32, /* n= */ m as i32, + /* k= */ k as i32, /* alpha= */ 1., /* a= */ a, + /* lda= */ lda, /* b= */ b, /* ldb= */ ldb, + /* beta= */ 0., /* c= */ c, /* ldc= */ n as i32, + ) + } + } + } + dtype => Err(Error::UnsupportedDTypeForOp(dtype, "matmul").bt())?, + } + Ok(dst) + } + + #[cfg(feature = "mkl")] + fn f( + &self, + lhs: &[T], + lhs_l: &Layout, + rhs: &[T], + rhs_l: &Layout, + ) -> Result> { + let (b, m, n, k) = self.0; + let lhs = &lhs[lhs_l.start_offset()..]; + let rhs = &rhs[rhs_l.start_offset()..]; + + let lhs_stride = lhs_l.stride(); + let rhs_stride = rhs_l.stride(); + + let (a_skip, b_skip) = self.ab_skip(lhs_l, rhs_l)?; + let c_skip: usize = m * n; + + let rhs_m1 = rhs_stride[rhs_stride.len() - 1]; + let rhs_m2 = rhs_stride[rhs_stride.len() - 2]; + let lhs_m1 = lhs_stride[lhs_stride.len() - 1]; + let lhs_m2 = lhs_stride[lhs_stride.len() - 2]; + + let (lda, transa) = if (rhs_m1 == 1 || n == 1) && (rhs_m2 == n || k == 1) { + (n as i32, b'N') + } else if rhs_m1 == k && rhs_m2 == 1 { + (k as i32, b'T') + } else { + Err(self.striding_error(lhs_l, rhs_l, "non-contiguous rhs"))? + }; + // The b tensor has dims batching, m, k (lhs) + let (ldb, transb) = if (lhs_m1 == 1 || k == 1) && (lhs_m2 == k || m == 1) { + (k as i32, b'N') + } else if lhs_m1 == m && lhs_m2 == 1 { + (m as i32, b'T') + } else { + Err(self.striding_error(lhs_l, rhs_l, "non-contiguous lhs"))? + }; + + let mut dst = vec![T::zero(); b * m * n]; + match T::DTYPE { + DType::F16 => { + for step in 0..b { + let lhs_p = &lhs[step * a_skip..]; + let rhs_p = &rhs[step * b_skip..]; + let dst_p = &mut dst[step * c_skip..]; + unsafe { + let a = rhs_p.as_ptr() as *const f16; + let b = lhs_p.as_ptr() as *const f16; + let c = dst_p.as_mut_ptr() as *mut f16; + let a = std::slice::from_raw_parts(a, a_skip); + let b = std::slice::from_raw_parts(b, b_skip); + let c = std::slice::from_raw_parts_mut(c, c_skip); + crate::mkl::hgemm( + transa, + transb, + /* m= */ n as i32, + /* n= */ m as i32, + /* k= */ k as i32, + /* alpha= */ f16::ONE, + /* a= */ a, + /* lda= */ lda, + /* b= */ b, + /* ldb= */ ldb, + /* beta= */ f16::ZERO, + /* c= */ c, + /* ldc= */ n as i32, + ) + } + } + } + DType::F32 => { + for step in 0..b { + let lhs_p = &lhs[step * a_skip..]; + let rhs_p = &rhs[step * b_skip..]; + let dst_p = &mut dst[step * c_skip..]; + unsafe { + let a = rhs_p.as_ptr() as *const f32; + let b = lhs_p.as_ptr() as *const f32; + let c = dst_p.as_mut_ptr() as *mut f32; + let a = std::slice::from_raw_parts(a, a_skip); + let b = std::slice::from_raw_parts(b, b_skip); + let c = std::slice::from_raw_parts_mut(c, c_skip); + crate::mkl::sgemm( + transa, transb, /* m= */ n as i32, /* n= */ m as i32, + /* k= */ k as i32, /* alpha= */ 1., /* a= */ a, + /* lda= */ lda, /* b= */ b, /* ldb= */ ldb, + /* beta= */ 0., /* c= */ c, /* ldc= */ n as i32, + ) + } + } + } + DType::F64 => { + for step in 0..b { + let lhs_p = &lhs[step * a_skip..]; + let rhs_p = &rhs[step * b_skip..]; + let dst_p = &mut dst[step * c_skip..]; + unsafe { + let a = rhs_p.as_ptr() as *const f64; + let b = lhs_p.as_ptr() as *const f64; + let c = dst_p.as_mut_ptr() as *mut f64; + let a = std::slice::from_raw_parts(a, a_skip); + let b = std::slice::from_raw_parts(b, b_skip); + let c = std::slice::from_raw_parts_mut(c, c_skip); + crate::mkl::dgemm( + transa, transb, /* m= */ n as i32, /* n= */ m as i32, + /* k= */ k as i32, /* alpha= */ 1., /* a= */ a, + /* lda= */ lda, /* b= */ b, /* ldb= */ ldb, + /* beta= */ 0., /* c= */ c, /* ldc= */ n as i32, + ) + } + } + } + dtype => Err(Error::UnsupportedDTypeForOp(dtype, "matmul").bt())?, + } + Ok(dst) + } +} + +fn elu(v: T, alpha: T) -> T { + if v.is_sign_positive() { + v + } else { + (v.exp() - T::one()) * alpha + } +} + +impl CpuStorage { + pub fn as_slice(&self) -> Result<&[D]> { + D::cpu_storage_as_slice(self) + } + + pub fn concat(storages: &[CpuStorage]) -> Result { + let storage0 = &storages[0]; + let s = match storage0 { + Self::U8(_) => { + let storages = storages + .iter() + .map(|s| match s { + Self::U8(s) => Ok(s.as_slice()), + _ => crate::bail!("dtype mismatch"), + }) + .collect::>>()? + .concat(); + Self::U8(storages) + } + Self::U32(_) => { + let storages = storages + .iter() + .map(|s| match s { + Self::U32(s) => Ok(s.as_slice()), + _ => crate::bail!("dtype mismatch"), + }) + .collect::>>()? + .concat(); + Self::U32(storages) + } + Self::I16(_) => { + let storages = storages + .iter() + .map(|s| match s { + Self::I16(s) => Ok(s.as_slice()), + _ => crate::bail!("dtype mismatch"), + }) + .collect::>>()? + .concat(); + Self::I16(storages) + } + Self::I32(_) => { + let storages = storages + .iter() + .map(|s| match s { + Self::I32(s) => Ok(s.as_slice()), + _ => crate::bail!("dtype mismatch"), + }) + .collect::>>()? + .concat(); + Self::I32(storages) + } + Self::I64(_) => { + let storages = storages + .iter() + .map(|s| match s { + Self::I64(s) => Ok(s.as_slice()), + _ => crate::bail!("dtype mismatch"), + }) + .collect::>>()? + .concat(); + Self::I64(storages) + } + Self::BF16(_) => { + let storages = storages + .iter() + .map(|s| match s { + Self::BF16(s) => Ok(s.as_slice()), + _ => crate::bail!("dtype mismatch"), + }) + .collect::>>()? + .concat(); + Self::BF16(storages) + } + Self::F16(_) => { + let storages = storages + .iter() + .map(|s| match s { + Self::F16(s) => Ok(s.as_slice()), + _ => crate::bail!("dtype mismatch"), + }) + .collect::>>()? + .concat(); + Self::F16(storages) + } + Self::F32(_) => { + let storages = storages + .iter() + .map(|s| match s { + Self::F32(s) => Ok(s.as_slice()), + _ => crate::bail!("dtype mismatch"), + }) + .collect::>>()? + .concat(); + Self::F32(storages) + } + Self::F64(_) => { + let storages = storages + .iter() + .map(|s| match s { + Self::F64(s) => Ok(s.as_slice()), + _ => crate::bail!("dtype mismatch"), + }) + .collect::>>()? + .concat(); + Self::F64(storages) + } + Self::F8E4M3(_) => { + let storages = storages + .iter() + .map(|s| match s { + Self::F8E4M3(s) => Ok(s.as_slice()), + _ => crate::bail!("dtype mismatch"), + }) + .collect::>>()? + .concat(); + Self::F8E4M3(storages) + } + Self::F6E2M3(_) => { + let storages = storages + .iter() + .map(|s| match s { + Self::F6E2M3(s) => Ok(s.as_slice()), + _ => crate::bail!("dtype mismatch"), + }) + .collect::>>()? + .concat(); + Self::F6E2M3(storages) + } + Self::F6E3M2(_) => { + let storages = storages + .iter() + .map(|s| match s { + Self::F6E3M2(s) => Ok(s.as_slice()), + _ => crate::bail!("dtype mismatch"), + }) + .collect::>>()? + .concat(); + Self::F6E3M2(storages) + } + Self::F4(_) => { + let storages = storages + .iter() + .map(|s| match s { + Self::F4(s) => Ok(s.as_slice()), + _ => crate::bail!("dtype mismatch"), + }) + .collect::>>()? + .concat(); + Self::F4(storages) + } + Self::F8E8M0(_) => { + let storages = storages + .iter() + .map(|s| match s { + Self::F8E8M0(s) => Ok(s.as_slice()), + _ => crate::bail!("dtype mismatch"), + }) + .collect::>>()? + .concat(); + Self::F8E8M0(storages) + } + }; + Ok(s) + } +} + +impl BackendStorage for CpuStorage { + type Device = CpuDevice; + + fn dtype(&self) -> DType { + match self { + Self::U8(_) => DType::U8, + Self::U32(_) => DType::U32, + Self::I16(_) => DType::I16, + Self::I32(_) => DType::I32, + Self::I64(_) => DType::I64, + Self::BF16(_) => DType::BF16, + Self::F16(_) => DType::F16, + Self::F32(_) => DType::F32, + Self::F64(_) => DType::F64, + Self::F8E4M3(_) => DType::F8E4M3, + Self::F6E2M3(_) => DType::F6E2M3, + Self::F6E3M2(_) => DType::F6E3M2, + Self::F4(_) => DType::F4, + Self::F8E8M0(_) => DType::F8E8M0, + } + } + + fn to_dtype(&self, layout: &Layout, dtype: DType) -> Result { + // TODO: find a way around the quadratic number of cases below. + match (self, dtype) { + (Self::U8(storage), DType::BF16) => { + let data = unary_map(storage, layout, |v| bf16::from_f32(v as f32)); + Ok(Self::BF16(data)) + } + (Self::U32(storage), DType::BF16) => { + let data = unary_map(storage, layout, |v| bf16::from_f32(v as f32)); + Ok(Self::BF16(data)) + } + (Self::I64(storage), DType::BF16) => { + let data = unary_map(storage, layout, |v| bf16::from_f32(v as f32)); + Ok(Self::BF16(data)) + } + (Self::BF16(storage), DType::BF16) => { + let data = unary_map(storage, layout, |v| v); + Ok(Self::BF16(data)) + } + (Self::F16(storage), DType::BF16) => { + let data = unary_map(storage, layout, |v| bf16::from_f32(v.to_f32())); + Ok(Self::BF16(data)) + } + (Self::F32(storage), DType::BF16) => { + let data = unary_map(storage, layout, bf16::from_f32); + Ok(Self::BF16(data)) + } + (Self::F64(storage), DType::BF16) => { + let data = unary_map(storage, layout, bf16::from_f64); + Ok(Self::BF16(data)) + } + (Self::U8(storage), DType::F16) => { + let data = unary_map(storage, layout, |v| f16::from_f32(v as f32)); + Ok(Self::F16(data)) + } + (Self::U32(storage), DType::F16) => { + let data = unary_map(storage, layout, |v| f16::from_f32(v as f32)); + Ok(Self::F16(data)) + } + (Self::I64(storage), DType::F16) => { + let data = unary_map(storage, layout, |v| f16::from_f32(v as f32)); + Ok(Self::F16(data)) + } + (Self::BF16(storage), DType::F16) => { + let data = unary_map(storage, layout, |v| f16::from_f32(v.to_f32())); + Ok(Self::F16(data)) + } + (Self::F16(storage), DType::F16) => { + let data = unary_map(storage, layout, |v| v); + Ok(Self::F16(data)) + } + (Self::F32(storage), DType::F16) => { + let data = unary_map(storage, layout, f16::from_f32); + Ok(Self::F16(data)) + } + (Self::F64(storage), DType::F16) => { + let data = unary_map(storage, layout, f16::from_f64); + Ok(Self::F16(data)) + } + (Self::U8(storage), DType::F32) => { + let data = unary_map(storage, layout, |v| v as f32); + Ok(Self::F32(data)) + } + (Self::U32(storage), DType::F32) => { + let data = unary_map(storage, layout, |v| v as f32); + Ok(Self::F32(data)) + } + (Self::I64(storage), DType::F32) => { + let data = unary_map(storage, layout, |v| v as f32); + Ok(Self::F32(data)) + } + (Self::BF16(storage), DType::F32) => { + let data = unary_map(storage, layout, |v| v.to_f32()); + Ok(Self::F32(data)) + } + (Self::F16(storage), DType::F32) => { + let data = unary_map(storage, layout, |v| v.to_f32()); + Ok(Self::F32(data)) + } + (Self::F32(storage), DType::F32) => { + let data = unary_map(storage, layout, |v| v); + Ok(Self::F32(data)) + } + (Self::F64(storage), DType::F32) => { + let data = unary_map(storage, layout, |v| v as f32); + Ok(Self::F32(data)) + } + (Self::U8(storage), DType::U8) => { + let data = unary_map(storage, layout, |v| v); + Ok(Self::U8(data)) + } + (Self::BF16(storage), DType::U8) => { + let data = unary_map(storage, layout, |v| v.to_f32() as u8); + Ok(Self::U8(data)) + } + (Self::F16(storage), DType::U8) => { + let data = unary_map(storage, layout, |v| v.to_f32() as u8); + Ok(Self::U8(data)) + } + (Self::F32(storage), DType::U8) => { + let data = unary_map(storage, layout, |v| v as u8); + Ok(Self::U8(data)) + } + (Self::F64(storage), DType::U8) => { + let data = unary_map(storage, layout, |v| v as u8); + Ok(Self::U8(data)) + } + (Self::U32(storage), DType::U8) => { + let data = unary_map(storage, layout, |v| v as u8); + Ok(Self::U8(data)) + } + (Self::I64(storage), DType::U8) => { + let data = unary_map(storage, layout, |v| v as u8); + Ok(Self::U8(data)) + } + (Self::U8(storage), DType::U32) => { + let data = unary_map(storage, layout, |v| v as u32); + Ok(Self::U32(data)) + } + (Self::U32(storage), DType::U32) => { + let data = unary_map(storage, layout, |v| v); + Ok(Self::U32(data)) + } + (Self::I64(storage), DType::U32) => { + let data = unary_map(storage, layout, |v| v as u32); + Ok(Self::U32(data)) + } + (Self::BF16(storage), DType::U32) => { + let data = unary_map(storage, layout, |v| v.to_f32() as u32); + Ok(Self::U32(data)) + } + (Self::F16(storage), DType::U32) => { + let data = unary_map(storage, layout, |v| v.to_f32() as u32); + Ok(Self::U32(data)) + } + (Self::F32(storage), DType::U32) => { + let data = unary_map(storage, layout, |v| v as u32); + Ok(Self::U32(data)) + } + (Self::F64(storage), DType::U32) => { + let data = unary_map(storage, layout, |v| v as u32); + Ok(Self::U32(data)) + } + (Self::U8(storage), DType::I64) => { + let data = unary_map(storage, layout, |v| v as i64); + Ok(Self::I64(data)) + } + (Self::U32(storage), DType::I64) => { + let data = unary_map(storage, layout, |v| v as i64); + Ok(Self::I64(data)) + } + (Self::I64(storage), DType::I64) => { + let data = unary_map(storage, layout, |v| v); + Ok(Self::I64(data)) + } + (Self::BF16(storage), DType::I64) => { + let data = unary_map(storage, layout, |v| v.to_f32() as i64); + Ok(Self::I64(data)) + } + (Self::F16(storage), DType::I64) => { + let data = unary_map(storage, layout, |v| v.to_f32() as i64); + Ok(Self::I64(data)) + } + (Self::F32(storage), DType::I64) => { + let data = unary_map(storage, layout, |v| v as i64); + Ok(Self::I64(data)) + } + (Self::F64(storage), DType::I64) => { + let data = unary_map(storage, layout, |v| v as i64); + Ok(Self::I64(data)) + } + (Self::U8(storage), DType::F64) => { + let data = unary_map(storage, layout, |v| v as f64); + Ok(Self::F64(data)) + } + (Self::U32(storage), DType::F64) => { + let data = unary_map(storage, layout, |v| v as f64); + Ok(Self::F64(data)) + } + (Self::I64(storage), DType::F64) => { + let data = unary_map(storage, layout, |v| v as f64); + Ok(Self::F64(data)) + } + (Self::BF16(storage), DType::F64) => { + let data = unary_map(storage, layout, |v| v.to_f64()); + Ok(Self::F64(data)) + } + (Self::F16(storage), DType::F64) => { + let data = unary_map(storage, layout, |v| v.to_f64()); + Ok(Self::F64(data)) + } + (Self::F32(storage), DType::F64) => { + let data = unary_map(storage, layout, |v| v as f64); + Ok(Self::F64(data)) + } + (Self::F64(storage), DType::F64) => { + let data = unary_map(storage, layout, |v| v); + Ok(Self::F64(data)) + } + // Conversions to F8E4M3 + (Self::U8(storage), DType::F8E4M3) => { + let data = unary_map(storage, layout, |v| F8E4M3::from_f32(v as f32)); + Ok(Self::F8E4M3(data)) + } + (Self::U32(storage), DType::F8E4M3) => { + let data = unary_map(storage, layout, |v| F8E4M3::from_f32(v as f32)); + Ok(Self::F8E4M3(data)) + } + (Self::I64(storage), DType::F8E4M3) => { + let data = unary_map(storage, layout, |v| F8E4M3::from_f32(v as f32)); + Ok(Self::F8E4M3(data)) + } + (Self::BF16(storage), DType::F8E4M3) => { + let data = unary_map(storage, layout, |v| F8E4M3::from_f32(v.to_f32())); + Ok(Self::F8E4M3(data)) + } + (Self::F16(storage), DType::F8E4M3) => { + let data = unary_map(storage, layout, |v| F8E4M3::from_f32(v.to_f32())); + Ok(Self::F8E4M3(data)) + } + (Self::F32(storage), DType::F8E4M3) => { + let data = unary_map(storage, layout, F8E4M3::from_f32); + Ok(Self::F8E4M3(data)) + } + (Self::F64(storage), DType::F8E4M3) => { + let data = unary_map(storage, layout, F8E4M3::from_f64); + Ok(Self::F8E4M3(data)) + } + (Self::F8E4M3(storage), DType::F8E4M3) => { + let data = unary_map(storage, layout, |v| v); + Ok(Self::F8E4M3(data)) + } + // Conversions from F8E4M3 + (Self::F8E4M3(storage), DType::U8) => { + let data = unary_map(storage, layout, |v| v.to_f32() as u8); + Ok(Self::U8(data)) + } + (Self::F8E4M3(storage), DType::U32) => { + let data = unary_map(storage, layout, |v| v.to_f32() as u32); + Ok(Self::U32(data)) + } + (Self::F8E4M3(storage), DType::I64) => { + let data = unary_map(storage, layout, |v| v.to_f32() as i64); + Ok(Self::I64(data)) + } + (Self::F8E4M3(storage), DType::BF16) => { + let data = unary_map(storage, layout, |v| bf16::from_f32(v.to_f32())); + Ok(Self::BF16(data)) + } + (Self::F8E4M3(storage), DType::F16) => { + let data = unary_map(storage, layout, |v| f16::from_f32(v.to_f32())); + Ok(Self::F16(data)) + } + (Self::F8E4M3(storage), DType::F32) => { + let data = unary_map(storage, layout, |v| v.to_f32()); + Ok(Self::F32(data)) + } + (Self::F8E4M3(storage), DType::F64) => { + let data = unary_map(storage, layout, |v| v.to_f64()); + Ok(Self::F64(data)) + } + // Conversions to I16 + (Self::U8(storage), DType::I16) => { + let data = unary_map(storage, layout, |v| v as i16); + Ok(Self::I16(data)) + } + (Self::U32(storage), DType::I16) => { + let data = unary_map(storage, layout, |v| v as i16); + Ok(Self::I16(data)) + } + (Self::I16(storage), DType::I16) => { + let data = unary_map(storage, layout, |v| v); + Ok(Self::I16(data)) + } + (Self::I32(storage), DType::I16) => { + let data = unary_map(storage, layout, |v| v as i16); + Ok(Self::I16(data)) + } + (Self::I64(storage), DType::I16) => { + let data = unary_map(storage, layout, |v| v as i16); + Ok(Self::I16(data)) + } + (Self::BF16(storage), DType::I16) => { + let data = unary_map(storage, layout, |v| v.to_f32() as i16); + Ok(Self::I16(data)) + } + (Self::F16(storage), DType::I16) => { + let data = unary_map(storage, layout, |v| v.to_f32() as i16); + Ok(Self::I16(data)) + } + (Self::F32(storage), DType::I16) => { + let data = unary_map(storage, layout, |v| v as i16); + Ok(Self::I16(data)) + } + (Self::F64(storage), DType::I16) => { + let data = unary_map(storage, layout, |v| v as i16); + Ok(Self::I16(data)) + } + (Self::F8E4M3(storage), DType::I16) => { + let data = unary_map(storage, layout, |v| v.to_f32() as i16); + Ok(Self::I16(data)) + } + // Conversions to I32 + (Self::U8(storage), DType::I32) => { + let data = unary_map(storage, layout, |v| v as i32); + Ok(Self::I32(data)) + } + (Self::U32(storage), DType::I32) => { + let data = unary_map(storage, layout, |v| v as i32); + Ok(Self::I32(data)) + } + (Self::I16(storage), DType::I32) => { + let data = unary_map(storage, layout, |v| v as i32); + Ok(Self::I32(data)) + } + (Self::I32(storage), DType::I32) => { + let data = unary_map(storage, layout, |v| v); + Ok(Self::I32(data)) + } + (Self::I64(storage), DType::I32) => { + let data = unary_map(storage, layout, |v| v as i32); + Ok(Self::I32(data)) + } + (Self::BF16(storage), DType::I32) => { + let data = unary_map(storage, layout, |v| v.to_f32() as i32); + Ok(Self::I32(data)) + } + (Self::F16(storage), DType::I32) => { + let data = unary_map(storage, layout, |v| v.to_f32() as i32); + Ok(Self::I32(data)) + } + (Self::F32(storage), DType::I32) => { + let data = unary_map(storage, layout, |v| v as i32); + Ok(Self::I32(data)) + } + (Self::F64(storage), DType::I32) => { + let data = unary_map(storage, layout, |v| v as i32); + Ok(Self::I32(data)) + } + (Self::F8E4M3(storage), DType::I32) => { + let data = unary_map(storage, layout, |v| v.to_f32() as i32); + Ok(Self::I32(data)) + } + // Conversions from I16 + (Self::I16(storage), DType::U8) => { + let data = unary_map(storage, layout, |v| v as u8); + Ok(Self::U8(data)) + } + (Self::I16(storage), DType::U32) => { + let data = unary_map(storage, layout, |v| v as u32); + Ok(Self::U32(data)) + } + (Self::I16(storage), DType::I64) => { + let data = unary_map(storage, layout, |v| v as i64); + Ok(Self::I64(data)) + } + (Self::I16(storage), DType::BF16) => { + let data = unary_map(storage, layout, |v| bf16::from_f32(v as f32)); + Ok(Self::BF16(data)) + } + (Self::I16(storage), DType::F16) => { + let data = unary_map(storage, layout, |v| f16::from_f32(v as f32)); + Ok(Self::F16(data)) + } + (Self::I16(storage), DType::F32) => { + let data = unary_map(storage, layout, |v| v as f32); + Ok(Self::F32(data)) + } + (Self::I16(storage), DType::F64) => { + let data = unary_map(storage, layout, |v| v as f64); + Ok(Self::F64(data)) + } + (Self::I16(storage), DType::F8E4M3) => { + let data = unary_map(storage, layout, |v| F8E4M3::from_f32(v as f32)); + Ok(Self::F8E4M3(data)) + } + // Conversions from I32 + (Self::I32(storage), DType::U8) => { + let data = unary_map(storage, layout, |v| v as u8); + Ok(Self::U8(data)) + } + (Self::I32(storage), DType::U32) => { + let data = unary_map(storage, layout, |v| v as u32); + Ok(Self::U32(data)) + } + (Self::I32(storage), DType::I64) => { + let data = unary_map(storage, layout, |v| v as i64); + Ok(Self::I64(data)) + } + (Self::I32(storage), DType::BF16) => { + let data = unary_map(storage, layout, |v| bf16::from_f32(v as f32)); + Ok(Self::BF16(data)) + } + (Self::I32(storage), DType::F16) => { + let data = unary_map(storage, layout, |v| f16::from_f32(v as f32)); + Ok(Self::F16(data)) + } + (Self::I32(storage), DType::F32) => { + let data = unary_map(storage, layout, |v| v as f32); + Ok(Self::F32(data)) + } + (Self::I32(storage), DType::F64) => { + let data = unary_map(storage, layout, |v| v as f64); + Ok(Self::F64(data)) + } + (Self::I32(storage), DType::F8E4M3) => { + let data = unary_map(storage, layout, |v| F8E4M3::from_f32(v as f32)); + Ok(Self::F8E4M3(data)) + } + // Dummy types - return error for all conversions to/from dummy types + (_, DType::F6E2M3) | (_, DType::F6E3M2) | (_, DType::F4) | (_, DType::F8E8M0) => { + Err(Error::UnsupportedDTypeForOp(dtype, "to_dtype").bt()) + } + (Self::F6E2M3(_), _) + | (Self::F6E3M2(_), _) + | (Self::F4(_), _) + | (Self::F8E8M0(_), _) => { + Err(Error::UnsupportedDTypeForOp(self.dtype(), "to_dtype").bt()) + } + } + } + + fn reduce_op(&self, op: ReduceOp, layout: &Layout, reduce_dims: &[usize]) -> Result { + match op { + ReduceOp::Sum => { + let src_dims = layout.dims(); + let mut dst_dims = src_dims.to_vec(); + for &dim in reduce_dims.iter() { + dst_dims[dim] = 1; + } + let dst_shape = Shape::from(dst_dims); + let mut reduce_dims = reduce_dims.to_vec(); + // Sort the reduce_dims as they have to be processed from left to right when converting the + // indexes. + reduce_dims.sort(); + let reduce_dims_and_stride: Vec<_> = reduce_dims + .iter() + .map(|&d| (src_dims[d], src_dims[d + 1..].iter().product::())) + .collect(); + ReduceSum { + dst_shape: &dst_shape, + reduce_dims: &reduce_dims, + reduce_dims_and_stride, + } + .map(self, layout) + } + ReduceOp::Min | ReduceOp::ArgMin | ReduceOp::Max | ReduceOp::ArgMax => { + let reduce_dim_index = match reduce_dims { + [reduce_dim_index] => *reduce_dim_index, + _ => { + let op = match op { + ReduceOp::Min => "min", + ReduceOp::ArgMin => "argmin", + ReduceOp::Max => "max", + ReduceOp::ArgMax => "argmax", + _ => unreachable!(), + }; + let dims = reduce_dims.to_vec(); + Err(Error::OnlySingleDimension { op, dims })? + } + }; + let (use_min, return_index) = match op { + ReduceOp::Min => (true, false), + ReduceOp::ArgMin => (true, true), + ReduceOp::Max => (false, false), + ReduceOp::ArgMax => (false, true), + _ => unreachable!(), + }; + ReduceIndex { + reduce_dim_index, + use_min, + return_index, + } + .map(self, layout) + } + } + } + + fn cmp(&self, op: CmpOp, rhs: &Self, lhs_l: &Layout, rhs_l: &Layout) -> Result { + Cmp(op).map(self, lhs_l, rhs, rhs_l) + } + + fn affine(&self, layout: &Layout, mul: f64, add: f64) -> Result { + Affine(mul, add).map(self, layout) + } + + fn avg_pool2d( + &self, + layout: &Layout, + kernel_size: (usize, usize), + stride: (usize, usize), + ) -> Result { + AvgPool2D(kernel_size, stride).map(self, layout) + } + + fn max_pool2d( + &self, + layout: &Layout, + kernel_size: (usize, usize), + stride: (usize, usize), + ) -> Result { + MaxPool2D(kernel_size, stride).map(self, layout) + } + + fn upsample_nearest1d(&self, layout: &Layout, sz: usize) -> Result { + UpsampleNearest1D(sz).map(self, layout) + } + + fn upsample_nearest2d(&self, layout: &Layout, h: usize, w: usize) -> Result { + UpsampleNearest2D(h, w).map(self, layout) + } + + fn upsample_bilinear2d( + &self, + layout: &Layout, + h: usize, + w: usize, + align_corners: bool, + scale_h: Option, + scale_w: Option, + ) -> Result { + UpsampleBilinear2D { + target_h: h, + target_w: w, + align_corners, + scale_h_factor: scale_h, + scale_w_factor: scale_w, + } + .map(self, layout) + } + + fn powf(&self, layout: &Layout, e: f64) -> Result { + use num_traits::Float; + // TODO: Have some generic map for functions that apply on num_traits::Float elements. + match self { + Self::BF16(storage) => { + let data = unary_map(storage, layout, |v| v.powf(bf16::from_f64(e))); + Ok(Self::BF16(data)) + } + Self::F16(storage) => { + let data = unary_map(storage, layout, |v| v.powf(f16::from_f64(e))); + Ok(Self::F16(data)) + } + Self::F32(storage) => { + let data = unary_map(storage, layout, |v| v.powf(e as f32)); + Ok(Self::F32(data)) + } + Self::F64(storage) => { + let data = unary_map(storage, layout, |v| v.powf(e)); + Ok(Self::F64(data)) + } + Self::F8E4M3(storage) => { + let data = unary_map(storage, layout, |v| v.powf(F8E4M3::from_f64(e))); + Ok(Self::F8E4M3(data)) + } + Self::U8(_) => Err(Error::UnsupportedDTypeForOp(DType::U8, "powf").bt()), + Self::U32(_) => Err(Error::UnsupportedDTypeForOp(DType::U32, "powf").bt()), + Self::I16(_) => Err(Error::UnsupportedDTypeForOp(DType::I16, "powf").bt()), + Self::I32(_) => Err(Error::UnsupportedDTypeForOp(DType::I32, "powf").bt()), + Self::I64(_) => Err(Error::UnsupportedDTypeForOp(DType::I64, "powf").bt()), + Self::F6E2M3(_) => Err(Error::UnsupportedDTypeForOp(DType::F6E2M3, "powf").bt()), + Self::F6E3M2(_) => Err(Error::UnsupportedDTypeForOp(DType::F6E3M2, "powf").bt()), + Self::F4(_) => Err(Error::UnsupportedDTypeForOp(DType::F4, "powf").bt()), + Self::F8E8M0(_) => Err(Error::UnsupportedDTypeForOp(DType::F8E8M0, "powf").bt()), + } + } + + fn elu(&self, layout: &Layout, alpha: f64) -> Result { + // TODO: Have some generic map for functions that apply on num_traits::Float elements. + match self { + Self::BF16(storage) => { + let data = unary_map(storage, layout, |v| elu(v, bf16::from_f64(alpha))); + Ok(Self::BF16(data)) + } + Self::F16(storage) => { + let data = unary_map(storage, layout, |v| elu(v, f16::from_f64(alpha))); + Ok(Self::F16(data)) + } + Self::F32(storage) => { + let data = unary_map(storage, layout, |v| elu(v, f32::from_f64(alpha))); + Ok(Self::F32(data)) + } + Self::F64(storage) => { + let data = unary_map(storage, layout, |v| elu(v, alpha)); + Ok(Self::F64(data)) + } + Self::F8E4M3(storage) => { + let data = unary_map(storage, layout, |v| elu(v, F8E4M3::from_f64(alpha))); + Ok(Self::F8E4M3(data)) + } + Self::U8(_) => Err(Error::UnsupportedDTypeForOp(DType::U8, "elu").bt()), + Self::U32(_) => Err(Error::UnsupportedDTypeForOp(DType::U32, "elu").bt()), + Self::I16(_) => Err(Error::UnsupportedDTypeForOp(DType::I16, "elu").bt()), + Self::I32(_) => Err(Error::UnsupportedDTypeForOp(DType::I32, "elu").bt()), + Self::I64(_) => Err(Error::UnsupportedDTypeForOp(DType::I64, "elu").bt()), + Self::F6E2M3(_) => Err(Error::UnsupportedDTypeForOp(DType::F6E2M3, "elu").bt()), + Self::F6E3M2(_) => Err(Error::UnsupportedDTypeForOp(DType::F6E3M2, "elu").bt()), + Self::F4(_) => Err(Error::UnsupportedDTypeForOp(DType::F4, "elu").bt()), + Self::F8E8M0(_) => Err(Error::UnsupportedDTypeForOp(DType::F8E8M0, "elu").bt()), + } + } + + fn unary_impl(&self, layout: &Layout) -> Result { + match self { + Self::BF16(storage) => { + if B::BF16_VEC { + let data = unary_map_vec(storage, layout, B::bf16, B::bf16_vec); + Ok(Self::BF16(data)) + } else { + let data = unary_map(storage, layout, B::bf16); + Ok(Self::BF16(data)) + } + } + Self::F16(storage) => { + if B::F16_VEC { + let data = unary_map_vec(storage, layout, B::f16, B::f16_vec); + Ok(Self::F16(data)) + } else { + let data = unary_map(storage, layout, B::f16); + Ok(Self::F16(data)) + } + } + Self::F32(storage) => { + if B::F32_VEC { + let data = unary_map_vec(storage, layout, B::f32, B::f32_vec); + Ok(Self::F32(data)) + } else { + let data = unary_map(storage, layout, B::f32); + Ok(Self::F32(data)) + } + } + Self::F64(storage) => { + if B::F64_VEC { + let data = unary_map_vec(storage, layout, B::f64, B::f64_vec); + Ok(Self::F64(data)) + } else { + let data = unary_map(storage, layout, B::f64); + Ok(Self::F64(data)) + } + } + Self::U8(storage) => { + let data = unary_map(storage, layout, B::u8); + Ok(Self::U8(data)) + } + Self::U32(storage) => { + let data = unary_map(storage, layout, B::u32); + Ok(Self::U32(data)) + } + Self::I16(storage) => { + let data = unary_map(storage, layout, B::i16); + Ok(Self::I16(data)) + } + Self::I32(storage) => { + let data = unary_map(storage, layout, B::i32); + Ok(Self::I32(data)) + } + Self::I64(storage) => { + let data = unary_map(storage, layout, B::i64); + Ok(Self::I64(data)) + } + Self::F8E4M3(storage) => { + let data = unary_map(storage, layout, B::f8e4m3); + Ok(Self::F8E4M3(data)) + } + Self::F6E2M3(_) => Err(Error::UnsupportedDTypeForOp(DType::F6E2M3, "unary").bt()), + Self::F6E3M2(_) => Err(Error::UnsupportedDTypeForOp(DType::F6E3M2, "unary").bt()), + Self::F4(_) => Err(Error::UnsupportedDTypeForOp(DType::F4, "unary").bt()), + Self::F8E8M0(_) => Err(Error::UnsupportedDTypeForOp(DType::F8E8M0, "unary").bt()), + } + } + + fn binary_impl( + &self, + rhs: &Self, + lhs_l: &Layout, + rhs_l: &Layout, + ) -> Result { + match (self, rhs) { + (Self::BF16(lhs), Self::BF16(rhs)) => { + let data = if B::BF16_VEC { + binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::bf16, B::bf16_vec) + } else { + binary_map(lhs_l, rhs_l, lhs, rhs, B::bf16) + }; + Ok(Self::BF16(data)) + } + (Self::F16(lhs), Self::F16(rhs)) => { + let data = if B::F16_VEC { + binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::f16, B::f16_vec) + } else { + binary_map(lhs_l, rhs_l, lhs, rhs, B::f16) + }; + Ok(Self::F16(data)) + } + (Self::F32(lhs), Self::F32(rhs)) => { + let data = if B::F32_VEC { + binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::f32, B::f32_vec) + } else { + binary_map(lhs_l, rhs_l, lhs, rhs, B::f32) + }; + Ok(Self::F32(data)) + } + (Self::F64(lhs), Self::F64(rhs)) => { + let data = if B::F64_VEC { + binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::f64, B::f64_vec) + } else { + binary_map(lhs_l, rhs_l, lhs, rhs, B::f64) + }; + Ok(Self::F64(data)) + } + (Self::U32(lhs), Self::U32(rhs)) => { + let data = if B::U32_VEC { + binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::u32, B::u32_vec) + } else { + binary_map(lhs_l, rhs_l, lhs, rhs, B::u32) + }; + Ok(Self::U32(data)) + } + (Self::I16(lhs), Self::I16(rhs)) => { + let data = binary_map(lhs_l, rhs_l, lhs, rhs, B::i16); + Ok(Self::I16(data)) + } + (Self::I32(lhs), Self::I32(rhs)) => { + let data = binary_map(lhs_l, rhs_l, lhs, rhs, B::i32); + Ok(Self::I32(data)) + } + (Self::I64(lhs), Self::I64(rhs)) => { + let data = if B::I64_VEC { + binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::i64, B::i64_vec) + } else { + binary_map(lhs_l, rhs_l, lhs, rhs, B::i64) + }; + Ok(Self::I64(data)) + } + (Self::U8(lhs), Self::U8(rhs)) => { + let data = if B::U8_VEC { + binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::u8, B::u8_vec) + } else { + binary_map(lhs_l, rhs_l, lhs, rhs, B::u8) + }; + Ok(Self::U8(data)) + } + (Self::F8E4M3(lhs), Self::F8E4M3(rhs)) => { + let data = binary_map(lhs_l, rhs_l, lhs, rhs, B::f8e4m3); + Ok(Self::F8E4M3(data)) + } + _ => { + // This should be covered by the dtype check above. + Err(Error::DTypeMismatchBinaryOp { + lhs: self.dtype(), + rhs: rhs.dtype(), + op: B::NAME, + } + .bt()) + } + } + } + + fn copy2d( + &self, + dst: &mut Self, + d1: usize, + d2: usize, + src_s: usize, + dst_s: usize, + src_o: usize, + dst_o: usize, + ) -> Result<()> { + match (self, dst) { + (Self::U8(src), Self::U8(dst)) => copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o), + (Self::U32(src), Self::U32(dst)) => { + copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) + } + (Self::I16(src), Self::I16(dst)) => { + copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) + } + (Self::I32(src), Self::I32(dst)) => { + copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) + } + (Self::I64(src), Self::I64(dst)) => { + copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) + } + (Self::BF16(src), Self::BF16(dst)) => { + copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) + } + (Self::F16(src), Self::F16(dst)) => { + copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) + } + (Self::F32(src), Self::F32(dst)) => { + copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) + } + (Self::F64(src), Self::F64(dst)) => { + copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) + } + (Self::F8E4M3(src), Self::F8E4M3(dst)) => { + copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) + } + (Self::F6E2M3(src), Self::F6E2M3(dst)) => { + copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) + } + (Self::F6E3M2(src), Self::F6E3M2(dst)) => { + copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) + } + (Self::F4(src), Self::F4(dst)) => copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o), + (Self::F8E8M0(src), Self::F8E8M0(dst)) => { + copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) + } + (_, dst) => { + return Err(Error::DTypeMismatchBinaryOp { + lhs: self.dtype(), + rhs: dst.dtype(), + op: "copy2d", + } + .bt()); + } + } + Ok(()) + } + + fn copy_strided_src(&self, dst: &mut Self, dst_offset: usize, src_l: &Layout) -> Result<()> { + match (self, dst) { + (Self::U8(src), Self::U8(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), + (Self::U32(src), Self::U32(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), + (Self::I16(src), Self::I16(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), + (Self::I32(src), Self::I32(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), + (Self::I64(src), Self::I64(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), + (Self::BF16(src), Self::BF16(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), + (Self::F16(src), Self::F16(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), + (Self::F32(src), Self::F32(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), + (Self::F64(src), Self::F64(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), + (Self::F8E4M3(src), Self::F8E4M3(dst)) => { + copy_strided_src_(src, dst, dst_offset, src_l) + } + (Self::F6E2M3(src), Self::F6E2M3(dst)) => { + copy_strided_src_(src, dst, dst_offset, src_l) + } + (Self::F6E3M2(src), Self::F6E3M2(dst)) => { + copy_strided_src_(src, dst, dst_offset, src_l) + } + (Self::F4(src), Self::F4(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), + (Self::F8E8M0(src), Self::F8E8M0(dst)) => { + copy_strided_src_(src, dst, dst_offset, src_l) + } + (_, dst) => { + // This should be covered by the dtype check above. + return Err(Error::DTypeMismatchBinaryOp { + lhs: self.dtype(), + rhs: dst.dtype(), + op: "copy_strided", + } + .bt()); + } + } + Ok(()) + } + + fn where_cond( + &self, + layout: &Layout, + t: &Self, + t_l: &Layout, + f: &Self, + f_l: &Layout, + ) -> Result { + match self { + Self::U8(pred) => WCond(pred, layout).map(t, t_l, f, f_l), + Self::U32(pred) => WCond(pred, layout).map(t, t_l, f, f_l), + Self::I16(pred) => WCond(pred, layout).map(t, t_l, f, f_l), + Self::I32(pred) => WCond(pred, layout).map(t, t_l, f, f_l), + Self::I64(pred) => WCond(pred, layout).map(t, t_l, f, f_l), + _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), "where-cond")), + } + } + + fn conv1d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConv1D, + ) -> Result { + if !USE_IM2COL_CONV1D { + return Conv1D(params).map(self, l, kernel, kernel_l); + } + let op = Im2Col1D { + l_k: params.k_size, + padding: params.padding, + stride: params.stride, + dilation: params.dilation, + }; + let col = op.map(self, l)?; + let b = params.b_size; + let n = params.c_out; + let l_out = params.l_out(); + let k = op.l_k * params.c_in; + let m = l_out; + let col_l = Layout::contiguous((b, m, k)); + let res = if kernel_l.is_contiguous() { + let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) + .transpose(1, 2)? + .broadcast_as((b, k, n))?; + col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? + } else { + // Make the kernel contiguous if not already the case. + let mut kernel_c = unsafe { + self.device() + .alloc_uninit(kernel_l.shape(), kernel.dtype())? + }; + kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?; + let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) + .transpose(1, 2)? + .broadcast_as((b, k, n))?; + col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? + }; + let res_l = Layout::contiguous((b, l_out, params.c_out)).transpose(1, 2)?; + let mut res_t = unsafe { self.device().alloc_uninit(res_l.shape(), res.dtype())? }; + res.copy_strided_src(&mut res_t, 0, &res_l)?; + Ok(res_t) + } + + fn conv_transpose1d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConvTranspose1D, + ) -> Result { + let can_use_col2im = kernel_l.is_contiguous() + && params.dilation == 1 + && params.padding == 0 + && params.output_padding == 0; + if USE_COL2IM_CONV1D_TR && can_use_col2im { + let (b_size, c_in, l_in) = l.shape().dims3()?; + let (c_in2, c_out, k_size) = kernel_l.shape().dims3()?; + if !kernel_l.is_contiguous() { + crate::bail!( + "convtr1d: the second argument (kernel) has to be contiguous {kernel_l:?}" + ) + } + if c_in != c_in2 { + crate::bail!( + "convtr1d: shape mismatch on c_in {:?} {:?}", + l.shape(), + kernel_l.shape() + ) + } + let col = { + // This merges the last two dimensions of the kernel together. + let kernel_l_mm = Layout::new( + (b_size, c_in, k_size * c_out).into(), + vec![0, k_size * c_out, 1], + kernel_l.start_offset(), + ); + self.matmul( + kernel, + ( + b_size, + /* m */ l_in, + /* n */ c_out * k_size, + /* k */ c_in, + ), + &l.transpose(1, 2)?, + &kernel_l_mm, + )? + }; + let col_l = Layout::contiguous((b_size, l_in, c_out, k_size)); + Col2Im1D { + stride: params.stride, + } + .map(&col, &col_l) + } else { + ConvTranspose1D(params).map(self, l, kernel, kernel_l) + } + } + + fn conv2d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConv2D, + ) -> Result { + Conv2D(params).map(self, l, kernel, kernel_l) + } + + fn conv_transpose2d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConvTranspose2D, + ) -> Result { + ConvTranspose2D(params).map(self, l, kernel, kernel_l) + } + + fn index_select(&self, ids: &Self, l: &Layout, ids_l: &Layout, dim: usize) -> Result { + match ids { + Self::U8(ids) => IndexSelect { ids, ids_l, dim }.map(self, l), + Self::U32(ids) => IndexSelect { ids, ids_l, dim }.map(self, l), + Self::I64(ids) => IndexSelect { ids, ids_l, dim }.map(self, l), + _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), "index-select").bt()), + } + } + + fn gather(&self, l: &Layout, ids: &Self, ids_l: &Layout, dim: usize) -> Result { + match ids { + Self::U8(ids) => Gather { ids, ids_l, dim }.map(self, l), + Self::U32(ids) => Gather { ids, ids_l, dim }.map(self, l), + Self::I64(ids) => Gather { ids, ids_l, dim }.map(self, l), + _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), "gather").bt()), + } + } + + fn scatter_set( + &mut self, + l: &Layout, + ids: &Self, + ids_l: &Layout, + src: &Self, + src_l: &Layout, + dim: usize, + ) -> Result<()> { + match ids { + Self::U8(ids) => Scatter::<_, Set>::new(ids, ids_l, dim).map(self, l, src, src_l), + Self::U32(ids) => Scatter::<_, Set>::new(ids, ids_l, dim).map(self, l, src, src_l), + Self::I64(ids) => Scatter::<_, Set>::new(ids, ids_l, dim).map(self, l, src, src_l), + _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), "scatter").bt()), + } + } + + fn scatter_add_set( + &mut self, + l: &Layout, + ids: &Self, + ids_l: &Layout, + src: &Self, + src_l: &Layout, + dim: usize, + ) -> Result<()> { + match ids { + Self::U8(ids) => Scatter::<_, Add>::new(ids, ids_l, dim).map(self, l, src, src_l), + Self::U32(ids) => Scatter::<_, Add>::new(ids, ids_l, dim).map(self, l, src, src_l), + Self::I16(ids) => Scatter::<_, Add>::new(ids, ids_l, dim).map(self, l, src, src_l), + Self::I32(ids) => Scatter::<_, Add>::new(ids, ids_l, dim).map(self, l, src, src_l), + Self::I64(ids) => Scatter::<_, Add>::new(ids, ids_l, dim).map(self, l, src, src_l), + _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), "scatter-add").bt()), + } + } + + fn index_add( + &self, + l: &Layout, + ids: &Self, + ids_l: &Layout, + src: &Self, + src_l: &Layout, + dim: usize, + ) -> Result { + match ids { + Self::U8(ids) => { + let ids = match ids_l.contiguous_offsets() { + Some((a, b)) => &ids[a..b], + None => Err(Error::RequiresContiguous { op: "index-add" }.bt())?, + }; + IndexAdd { ids, dim }.map(self, l, src, src_l) + } + Self::U32(ids) => { + let ids = match ids_l.contiguous_offsets() { + Some((a, b)) => &ids[a..b], + None => Err(Error::RequiresContiguous { op: "index-add" }.bt())?, + }; + IndexAdd { ids, dim }.map(self, l, src, src_l) + } + Self::I16(ids) => { + let ids = match ids_l.contiguous_offsets() { + Some((a, b)) => &ids[a..b], + None => Err(Error::RequiresContiguous { op: "index-add" }.bt())?, + }; + IndexAdd { ids, dim }.map(self, l, src, src_l) + } + Self::I32(ids) => { + let ids = match ids_l.contiguous_offsets() { + Some((a, b)) => &ids[a..b], + None => Err(Error::RequiresContiguous { op: "index-add" }.bt())?, + }; + IndexAdd { ids, dim }.map(self, l, src, src_l) + } + Self::I64(ids) => { + let ids = match ids_l.contiguous_offsets() { + Some((a, b)) => &ids[a..b], + None => Err(Error::RequiresContiguous { op: "index-add" }.bt())?, + }; + IndexAdd { ids, dim }.map(self, l, src, src_l) + } + _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), "index-add").bt()), + } + } + + fn matmul( + &self, + rhs: &Self, + bmnk: (usize, usize, usize, usize), + lhs_l: &Layout, + rhs_l: &Layout, + ) -> Result { + MatMul(bmnk).map(self, lhs_l, rhs, rhs_l) + } + + fn device(&self) -> &Self::Device { + &CpuDevice + } + + fn try_clone(&self, _: &Layout) -> Result { + Ok(self.clone()) + } + + fn to_cpu_storage(&self) -> Result { + Ok(self.clone()) + } + + fn const_set(&mut self, s: crate::scalar::Scalar, l: &Layout) -> Result<()> { + use crate::scalar::Scalar; + fn set(src: &mut [T], l: &Layout, s: T) { + match l.strided_blocks() { + crate::StridedBlocks::SingleBlock { start_offset, len } => { + src[start_offset..start_offset + len].fill(s) + } + crate::StridedBlocks::MultipleBlocks { + block_start_index, + block_len: 1, + } => { + for src_index in block_start_index { + src[src_index] = s + } + } + crate::StridedBlocks::MultipleBlocks { + block_start_index, + block_len, + } => { + for src_index in block_start_index { + src[src_index..src_index + block_len].fill(s) + } + } + } + } + match (self, s) { + (Self::BF16(storage), Scalar::BF16(v)) => set(storage, l, v), + (Self::F16(storage), Scalar::F16(v)) => set(storage, l, v), + (Self::F32(storage), Scalar::F32(v)) => set(storage, l, v), + (Self::F64(storage), Scalar::F64(v)) => set(storage, l, v), + (Self::U8(storage), Scalar::U8(v)) => set(storage, l, v), + (Self::U32(storage), Scalar::U32(v)) => set(storage, l, v), + (Self::I16(storage), Scalar::I16(v)) => set(storage, l, v), + (Self::I32(storage), Scalar::I32(v)) => set(storage, l, v), + (Self::I64(storage), Scalar::I64(v)) => set(storage, l, v), + (Self::F8E4M3(storage), Scalar::F8E4M3(v)) => set(storage, l, v), + // Dummy types don't support scalar operations + (Self::F6E2M3(_), _) => { + crate::bail!("const_set not supported for dummy type F6E2M3") + } + (Self::F6E3M2(_), _) => { + crate::bail!("const_set not supported for dummy type F6E3M2") + } + (Self::F4(_), _) => { + crate::bail!("const_set not supported for dummy type F4") + } + (Self::F8E8M0(_), _) => { + crate::bail!("const_set not supported for dummy type F8E8M0") + } + (st, s) => crate::bail!( + "const_set dtype mismatch, expected {:?} but got {:?}", + st.dtype(), + s + ), + } + Ok(()) + } +} + +impl BackendDevice for CpuDevice { + type Storage = CpuStorage; + + fn location(&self) -> crate::DeviceLocation { + crate::DeviceLocation::Cpu + } + + fn same_device(&self, _: &Self) -> bool { + true + } + + fn storage_from_slice(&self, s: &[T]) -> Result { + Ok(T::to_cpu_storage(s)) + } + + fn storage_from_cpu_storage(&self, s: &CpuStorage) -> Result { + Ok(s.clone()) + } + + fn storage_from_cpu_storage_owned(&self, s: CpuStorage) -> Result { + Ok(s) + } + + fn new(_: usize) -> Result { + Ok(Self) + } + + fn set_seed(&self, _seed: u64) -> Result<()> { + crate::bail!("cannot seed the CPU rng with set_seed") + } + + fn get_current_seed(&self) -> Result { + crate::bail!("cannot get the CPU rng seed with get_current_seed") + } + + fn rand_uniform(&self, shape: &Shape, dtype: DType, min: f64, max: f64) -> Result { + use rand::prelude::*; + + let elem_count = shape.elem_count(); + let mut rng = rand::rng(); + match dtype { + DType::U8 + | DType::U32 + | DType::I16 + | DType::I32 + | DType::I64 + | DType::F6E2M3 + | DType::F6E3M2 + | DType::F4 + | DType::F8E8M0 => Err(Error::UnsupportedDTypeForOp(dtype, "rand_uniform").bt()), + DType::BF16 => { + let mut data = Vec::with_capacity(elem_count); + let uniform = rand::distr::Uniform::new(bf16::from_f64(min), bf16::from_f64(max)) + .map_err(Error::wrap)?; + for _i in 0..elem_count { + data.push(rng.sample::(uniform)) + } + Ok(CpuStorage::BF16(data)) + } + DType::F16 => { + let mut data = Vec::with_capacity(elem_count); + let uniform = rand::distr::Uniform::new(f16::from_f64(min), f16::from_f64(max)) + .map_err(Error::wrap)?; + for _i in 0..elem_count { + data.push(rng.sample::(uniform)) + } + Ok(CpuStorage::F16(data)) + } + DType::F8E4M3 => { + let mut data = Vec::with_capacity(elem_count); + let uniform = + rand::distr::Uniform::new(F8E4M3::from_f64(min), F8E4M3::from_f64(max)) + .map_err(Error::wrap)?; + for _i in 0..elem_count { + data.push(rng.sample::(uniform)) + } + Ok(CpuStorage::F8E4M3(data)) + } + DType::F32 => { + let mut data = Vec::with_capacity(elem_count); + let uniform = + rand::distr::Uniform::new(min as f32, max as f32).map_err(Error::wrap)?; + for _i in 0..elem_count { + data.push(rng.sample::(uniform)) + } + Ok(CpuStorage::F32(data)) + } + DType::F64 => { + let mut data = Vec::with_capacity(elem_count); + let uniform = rand::distr::Uniform::new(min, max).map_err(Error::wrap)?; + for _i in 0..elem_count { + data.push(rng.sample::(uniform)) + } + Ok(CpuStorage::F64(data)) + } + } + } + + fn rand_normal(&self, shape: &Shape, dtype: DType, mean: f64, std: f64) -> Result { + use rand::prelude::*; + + let elem_count = shape.elem_count(); + let mut rng = rand::rng(); + match dtype { + DType::U8 + | DType::U32 + | DType::I16 + | DType::I32 + | DType::I64 + | DType::F6E2M3 + | DType::F6E3M2 + | DType::F4 + | DType::F8E8M0 => Err(Error::UnsupportedDTypeForOp(dtype, "rand_normal").bt()), + DType::BF16 => { + let mut data = Vec::with_capacity(elem_count); + let normal = rand_distr::Normal::new(bf16::from_f64(mean), bf16::from_f64(std)) + .map_err(Error::wrap)?; + for _i in 0..elem_count { + data.push(normal.sample(&mut rng)) + } + Ok(CpuStorage::BF16(data)) + } + DType::F16 => { + let mut data = Vec::with_capacity(elem_count); + let normal = rand_distr::Normal::new(f16::from_f64(mean), f16::from_f64(std)) + .map_err(Error::wrap)?; + for _i in 0..elem_count { + data.push(normal.sample(&mut rng)) + } + Ok(CpuStorage::F16(data)) + } + DType::F8E4M3 => { + let mut data = Vec::with_capacity(elem_count); + let normal = rand_distr::Normal::new(F8E4M3::from_f64(mean), F8E4M3::from_f64(std)) + .map_err(Error::wrap)?; + for _i in 0..elem_count { + data.push(normal.sample(&mut rng)) + } + Ok(CpuStorage::F8E4M3(data)) + } + DType::F32 => { + let mut data = Vec::with_capacity(elem_count); + let normal = + rand_distr::Normal::new(mean as f32, std as f32).map_err(Error::wrap)?; + for _i in 0..elem_count { + data.push(normal.sample(&mut rng)) + } + Ok(CpuStorage::F32(data)) + } + DType::F64 => { + let mut data = Vec::with_capacity(elem_count); + let normal = rand_distr::Normal::new(mean, std).map_err(Error::wrap)?; + for _i in 0..elem_count { + data.push(normal.sample(&mut rng)) + } + Ok(CpuStorage::F64(data)) + } + } + } + + #[allow(clippy::uninit_vec)] + unsafe fn alloc_uninit(&self, shape: &Shape, dtype: DType) -> Result { + let elem_count = shape.elem_count(); + // The code below is highly unsafe but hopefully not directly unsound as we only consider + // types that are Copy, not Drop, and for which all bit patterns are proper values. + // It's still pretty risky, see the following for more details: + // https://github.com/rust-lang/rust-clippy/issues/4483 + let storage = match dtype { + DType::U8 => { + let mut v = Vec::with_capacity(elem_count); + v.set_len(elem_count); + CpuStorage::U8(v) + } + DType::U32 => { + let mut v = Vec::with_capacity(elem_count); + v.set_len(elem_count); + CpuStorage::U32(v) + } + DType::I16 => { + let mut v = Vec::with_capacity(elem_count); + v.set_len(elem_count); + CpuStorage::I16(v) + } + DType::I32 => { + let mut v = Vec::with_capacity(elem_count); + v.set_len(elem_count); + CpuStorage::I32(v) + } + DType::I64 => { + let mut v = Vec::with_capacity(elem_count); + v.set_len(elem_count); + CpuStorage::I64(v) + } + DType::BF16 => { + let mut v = Vec::with_capacity(elem_count); + v.set_len(elem_count); + CpuStorage::BF16(v) + } + DType::F16 => { + let mut v = Vec::with_capacity(elem_count); + v.set_len(elem_count); + CpuStorage::F16(v) + } + DType::F32 => { + let mut v = Vec::with_capacity(elem_count); + v.set_len(elem_count); + CpuStorage::F32(v) + } + DType::F64 => { + let mut v = Vec::with_capacity(elem_count); + v.set_len(elem_count); + CpuStorage::F64(v) + } + DType::F8E4M3 => { + let mut v = Vec::with_capacity(elem_count); + v.set_len(elem_count); + CpuStorage::F8E4M3(v) + } + DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { + return Err(Error::UnsupportedDTypeForOp(dtype, "alloc_uninit").bt()) + } + }; + Ok(storage) + } + + fn zeros_impl(&self, shape: &Shape, dtype: DType) -> Result { + let elem_count = shape.elem_count(); + let storage = match dtype { + DType::U8 => CpuStorage::U8(vec![0u8; elem_count]), + DType::U32 => CpuStorage::U32(vec![0u32; elem_count]), + DType::I16 => CpuStorage::I16(vec![0i16; elem_count]), + DType::I32 => CpuStorage::I32(vec![0i32; elem_count]), + DType::I64 => CpuStorage::I64(vec![0i64; elem_count]), + DType::BF16 => CpuStorage::BF16(vec![bf16::ZERO; elem_count]), + DType::F16 => CpuStorage::F16(vec![f16::ZERO; elem_count]), + DType::F32 => CpuStorage::F32(vec![0f32; elem_count]), + DType::F64 => CpuStorage::F64(vec![0f64; elem_count]), + DType::F8E4M3 => CpuStorage::F8E4M3(vec![F8E4M3::ZERO; elem_count]), + DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { + return Err(Error::UnsupportedDTypeForOp(dtype, "zeros").bt()) + } + }; + Ok(storage) + } + + fn synchronize(&self) -> Result<()> { + Ok(()) + } +} + +#[macro_export] +macro_rules! map_dtype { + ($name:expr, $storage:ident, $fn:expr, ($($dtypes:ident),+)) => { + match $storage { + $(CpuStorage::$dtypes(__e) => CpuStorage::$dtypes($fn(__e)),)* + s => Err(Error::UnsupportedDTypeForOp(s.dtype(), $name).bt())?, + } + }; +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/utils.rs b/v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/utils.rs new file mode 100644 index 0000000000..1f800a928b --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/utils.rs @@ -0,0 +1,409 @@ +/// Helper functions to write CPU kernels. +use crate::backend::BackendStorage; +use crate::{Error, Layout, Result, WithDType}; + +type C = super::CpuStorage; +pub trait Map1 { + fn f(&self, vs: &[T], layout: &Layout) -> Result>; + + fn map(&self, vs: &C, layout: &Layout) -> Result { + match vs { + C::U8(vs) => Ok(C::U8(self.f(vs, layout)?)), + C::U32(vs) => Ok(C::U32(self.f(vs, layout)?)), + C::I16(vs) => Ok(C::I16(self.f(vs, layout)?)), + C::I32(vs) => Ok(C::I32(self.f(vs, layout)?)), + C::I64(vs) => Ok(C::I64(self.f(vs, layout)?)), + C::BF16(vs) => Ok(C::BF16(self.f(vs, layout)?)), + C::F16(vs) => Ok(C::F16(self.f(vs, layout)?)), + C::F32(vs) => Ok(C::F32(self.f(vs, layout)?)), + C::F64(vs) => Ok(C::F64(self.f(vs, layout)?)), + C::F8E4M3(vs) => Ok(C::F8E4M3(self.f(vs, layout)?)), + // Dummy types don't support Map1 operations + C::F6E2M3(_) => Err(Error::UnsupportedDTypeForOp(vs.dtype(), "map1").bt()), + C::F6E3M2(_) => Err(Error::UnsupportedDTypeForOp(vs.dtype(), "map1").bt()), + C::F4(_) => Err(Error::UnsupportedDTypeForOp(vs.dtype(), "map1").bt()), + C::F8E8M0(_) => Err(Error::UnsupportedDTypeForOp(vs.dtype(), "map1").bt()), + } + } +} + +pub trait Map1Any { + fn f) -> C>(&self, vs: &[T], layout: &Layout, wrap: W) -> Result; + + fn map(&self, vs: &C, layout: &Layout) -> Result { + match vs { + C::U8(vs) => Ok(self.f(vs, layout, C::U8)?), + C::U32(vs) => Ok(self.f(vs, layout, C::U32)?), + C::I16(vs) => Ok(self.f(vs, layout, C::I16)?), + C::I32(vs) => Ok(self.f(vs, layout, C::I32)?), + C::I64(vs) => Ok(self.f(vs, layout, C::I64)?), + C::BF16(vs) => Ok(self.f(vs, layout, C::BF16)?), + C::F16(vs) => Ok(self.f(vs, layout, C::F16)?), + C::F32(vs) => Ok(self.f(vs, layout, C::F32)?), + C::F64(vs) => Ok(self.f(vs, layout, C::F64)?), + C::F8E4M3(vs) => Ok(self.f(vs, layout, C::F8E4M3)?), + // Dummy types don't support Map1Any operations + C::F6E2M3(_) => Err(Error::UnsupportedDTypeForOp(vs.dtype(), "map1any").bt()), + C::F6E3M2(_) => Err(Error::UnsupportedDTypeForOp(vs.dtype(), "map1any").bt()), + C::F4(_) => Err(Error::UnsupportedDTypeForOp(vs.dtype(), "map1any").bt()), + C::F8E8M0(_) => Err(Error::UnsupportedDTypeForOp(vs.dtype(), "map1any").bt()), + } + } +} + +pub trait Map2 { + const OP: &'static str; + fn f(&self, v1: &[T], l1: &Layout, v2: &[T], l2: &Layout) -> Result>; + + fn map(&self, v1: &C, l1: &Layout, v2: &C, l2: &Layout) -> Result { + match (v1, v2) { + (C::U8(v1), C::U8(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), + (C::U32(v1), C::U32(v2)) => Ok(C::U32(self.f(v1, l1, v2, l2)?)), + (C::I16(v1), C::I16(v2)) => Ok(C::I16(self.f(v1, l1, v2, l2)?)), + (C::I32(v1), C::I32(v2)) => Ok(C::I32(self.f(v1, l1, v2, l2)?)), + (C::I64(v1), C::I64(v2)) => Ok(C::I64(self.f(v1, l1, v2, l2)?)), + (C::BF16(v1), C::BF16(v2)) => Ok(C::BF16(self.f(v1, l1, v2, l2)?)), + (C::F16(v1), C::F16(v2)) => Ok(C::F16(self.f(v1, l1, v2, l2)?)), + (C::F32(v1), C::F32(v2)) => Ok(C::F32(self.f(v1, l1, v2, l2)?)), + (C::F64(v1), C::F64(v2)) => Ok(C::F64(self.f(v1, l1, v2, l2)?)), + (C::F8E4M3(v1), C::F8E4M3(v2)) => Ok(C::F8E4M3(self.f(v1, l1, v2, l2)?)), + _ => Err(Error::DTypeMismatchBinaryOp { + lhs: v1.dtype(), + rhs: v2.dtype(), + op: Self::OP, + } + .bt()), + } + } +} + +pub trait Map2InPlace { + const OP: &'static str; + fn f(&self, v1: &mut [T], l1: &Layout, v2: &[T], l2: &Layout) -> Result<()>; + + fn map(&self, v1: &mut C, l1: &Layout, v2: &C, l2: &Layout) -> Result<()> { + match (v1, v2) { + (C::U8(v1), C::U8(v2)) => self.f(v1, l1, v2, l2)?, + (C::U32(v1), C::U32(v2)) => self.f(v1, l1, v2, l2)?, + (C::I16(v1), C::I16(v2)) => self.f(v1, l1, v2, l2)?, + (C::I32(v1), C::I32(v2)) => self.f(v1, l1, v2, l2)?, + (C::I64(v1), C::I64(v2)) => self.f(v1, l1, v2, l2)?, + (C::BF16(v1), C::BF16(v2)) => self.f(v1, l1, v2, l2)?, + (C::F16(v1), C::F16(v2)) => self.f(v1, l1, v2, l2)?, + (C::F32(v1), C::F32(v2)) => self.f(v1, l1, v2, l2)?, + (C::F64(v1), C::F64(v2)) => self.f(v1, l1, v2, l2)?, + (C::F8E4M3(v1), C::F8E4M3(v2)) => self.f(v1, l1, v2, l2)?, + (v1, v2) => Err(Error::DTypeMismatchBinaryOp { + lhs: v1.dtype(), + rhs: v2.dtype(), + op: Self::OP, + } + .bt())?, + }; + Ok(()) + } +} + +pub trait Map2U8 { + const OP: &'static str; + fn f(&self, v1: &[T], l1: &Layout, v2: &[T], l2: &Layout) -> Result>; + + fn map(&self, v1: &C, l1: &Layout, v2: &C, l2: &Layout) -> Result { + match (v1, v2) { + (C::U8(v1), C::U8(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), + (C::U32(v1), C::U32(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), + (C::I16(v1), C::I16(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), + (C::I32(v1), C::I32(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), + (C::I64(v1), C::I64(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), + (C::BF16(v1), C::BF16(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), + (C::F16(v1), C::F16(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), + (C::F32(v1), C::F32(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), + (C::F64(v1), C::F64(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), + (C::F8E4M3(v1), C::F8E4M3(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), + _ => Err(Error::DTypeMismatchBinaryOp { + lhs: v1.dtype(), + rhs: v2.dtype(), + op: Self::OP, + } + .bt()), + } + } +} + +pub fn binary_map U>( + lhs_l: &Layout, + rhs_l: &Layout, + lhs: &[T], + rhs: &[T], + mut f: F, +) -> Vec { + match (lhs_l.contiguous_offsets(), rhs_l.contiguous_offsets()) { + (Some((o_l1, o_l2)), Some((o_r1, o_r2))) => lhs[o_l1..o_l2] + .iter() + .zip(rhs[o_r1..o_r2].iter()) + .map(|(&l, &r)| f(l, r)) + .collect(), + (Some((o_l1, o_l2)), None) => { + // TODO: Maybe we want to avoid going through the layout twice. + match rhs_l.offsets_b() { + Some(ob) => { + let mut i_in_block = 0; + let mut i_right_broadcast = 0; + lhs[o_l1..o_l2] + .iter() + .map(|&l| { + let r = unsafe { rhs.get_unchecked(i_in_block + ob.start) }; + i_right_broadcast += 1; + if i_right_broadcast >= ob.right_broadcast { + i_in_block += 1; + i_right_broadcast = 0; + } + if i_in_block >= ob.len { + i_in_block = 0 + } + f(l, *r) + }) + .collect() + } + None => lhs_l + .strided_index() + .zip(rhs_l.strided_index()) + .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) + .collect(), + } + } + (None, Some((o_r1, o_r2))) => { + // TODO: Maybe we want to avoid going through the layout twice. + match lhs_l.offsets_b() { + Some(ob) => { + let mut i_in_block = 0; + let mut i_right_broadcast = 0; + rhs[o_r1..o_r2] + .iter() + .map(|&r| { + let l = unsafe { lhs.get_unchecked(i_in_block + ob.start) }; + i_right_broadcast += 1; + if i_right_broadcast >= ob.right_broadcast { + i_in_block += 1; + i_right_broadcast = 0; + } + if i_in_block >= ob.len { + i_in_block = 0 + } + f(*l, r) + }) + .collect() + } + None => lhs_l + .strided_index() + .zip(rhs_l.strided_index()) + .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) + .collect(), + } + } + _ => lhs_l + .strided_index() + .zip(rhs_l.strided_index()) + .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) + .collect(), + } +} + +// Similar to binary_map but with vectorized variants. +pub fn binary_map_vec T, FV: FnMut(&[T], &[T], &mut [T])>( + lhs_l: &Layout, + rhs_l: &Layout, + lhs: &[T], + rhs: &[T], + mut f: F, + mut f_vec: FV, +) -> Vec { + let el_count = lhs_l.shape().elem_count(); + match (lhs_l.contiguous_offsets(), rhs_l.contiguous_offsets()) { + (Some((o_l1, o_l2)), Some((o_r1, o_r2))) => { + let mut ys: Vec = Vec::with_capacity(el_count); + let ys_to_set = ys.spare_capacity_mut(); + let ys_to_set = unsafe { + std::mem::transmute::<&mut [std::mem::MaybeUninit], &mut [T]>(ys_to_set) + }; + f_vec(&lhs[o_l1..o_l2], &rhs[o_r1..o_r2], ys_to_set); + // SAFETY: values are all set by f_vec. + unsafe { ys.set_len(el_count) }; + ys + } + (Some((o_l1, o_l2)), None) => match rhs_l.offsets_b() { + Some(ob) if ob.right_broadcast == 1 => { + let rhs = &rhs[ob.start..ob.start + ob.len]; + let mut ys: Vec = Vec::with_capacity(el_count); + let ys_to_set = ys.spare_capacity_mut(); + let ys_to_set = unsafe { + std::mem::transmute::<&mut [std::mem::MaybeUninit], &mut [T]>(ys_to_set) + }; + let mut dst_i = 0; + for src_i in (o_l1..o_l2).step_by(ob.len) { + f_vec( + &lhs[src_i..src_i + ob.len], + rhs, + &mut ys_to_set[dst_i..dst_i + ob.len], + ); + dst_i += ob.len; + } + // SAFETY: values are all set by f_vec. + unsafe { ys.set_len(el_count) }; + ys + } + Some(ob) => { + let rhs = &rhs[ob.start..ob.start + ob.len]; + let mut ys = lhs[o_l1..o_l2].to_vec(); + for idx_l in 0..ob.left_broadcast { + let start = idx_l * ob.len * ob.right_broadcast; + for (i, &r) in rhs.iter().enumerate() { + let start = start + i * ob.right_broadcast; + for v in ys[start..start + ob.right_broadcast].iter_mut() { + *v = f(*v, r) + } + } + } + ys + } + None => lhs_l + .strided_index() + .zip(rhs_l.strided_index()) + .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) + .collect(), + }, + (None, Some((o_r1, o_r2))) => match lhs_l.offsets_b() { + Some(ob) if ob.right_broadcast == 1 => { + let lhs = &lhs[ob.start..ob.start + ob.len]; + let mut ys: Vec = Vec::with_capacity(el_count); + let ys_to_set = ys.spare_capacity_mut(); + let ys_to_set = unsafe { + std::mem::transmute::<&mut [std::mem::MaybeUninit], &mut [T]>(ys_to_set) + }; + let mut dst_i = 0; + for src_i in (o_r1..o_r2).step_by(ob.len) { + f_vec( + lhs, + &rhs[src_i..src_i + ob.len], + &mut ys_to_set[dst_i..dst_i + ob.len], + ); + dst_i += ob.len; + } + // SAFETY: values are all set by f_vec. + unsafe { ys.set_len(el_count) }; + ys + } + Some(ob) => { + let lhs = &lhs[ob.start..ob.start + ob.len]; + let mut ys = rhs[o_r1..o_r2].to_vec(); + for idx_l in 0..ob.left_broadcast { + let start = idx_l * ob.len * ob.right_broadcast; + for (i, &l) in lhs.iter().enumerate() { + let start = start + i * ob.right_broadcast; + for v in ys[start..start + ob.right_broadcast].iter_mut() { + *v = f(l, *v) + } + } + } + ys + } + None => lhs_l + .strided_index() + .zip(rhs_l.strided_index()) + .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) + .collect(), + }, + _ => lhs_l + .strided_index() + .zip(rhs_l.strided_index()) + .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) + .collect(), + } +} + +pub fn unary_map U>( + vs: &[T], + layout: &Layout, + mut f: F, +) -> Vec { + match layout.strided_blocks() { + crate::StridedBlocks::SingleBlock { start_offset, len } => vs + [start_offset..start_offset + len] + .iter() + .map(|&v| f(v)) + .collect(), + crate::StridedBlocks::MultipleBlocks { + block_start_index, + block_len, + } => { + let mut result = Vec::with_capacity(layout.shape().elem_count()); + // Specialize the case where block_len is one to avoid the second loop. + if block_len == 1 { + for index in block_start_index { + let v = unsafe { vs.get_unchecked(index) }; + result.push(f(*v)) + } + } else { + for index in block_start_index { + for offset in 0..block_len { + let v = unsafe { vs.get_unchecked(index + offset) }; + result.push(f(*v)) + } + } + } + result + } + } +} + +pub fn unary_map_vec U, FV: FnMut(&[T], &mut [U])>( + vs: &[T], + layout: &Layout, + mut f: F, + mut f_vec: FV, +) -> Vec { + match layout.strided_blocks() { + crate::StridedBlocks::SingleBlock { start_offset, len } => { + let mut ys: Vec = Vec::with_capacity(len); + let ys_to_set = ys.spare_capacity_mut(); + let ys_to_set = unsafe { + std::mem::transmute::<&mut [std::mem::MaybeUninit], &mut [U]>(ys_to_set) + }; + f_vec(&vs[start_offset..start_offset + len], ys_to_set); + // SAFETY: values are all set by f_vec. + unsafe { ys.set_len(len) }; + ys + } + crate::StridedBlocks::MultipleBlocks { + block_start_index, + block_len, + } => { + let el_count = layout.shape().elem_count(); + // Specialize the case where block_len is one to avoid the second loop. + if block_len == 1 { + let mut result = Vec::with_capacity(el_count); + for index in block_start_index { + let v = unsafe { vs.get_unchecked(index) }; + result.push(f(*v)) + } + result + } else { + let mut ys: Vec = Vec::with_capacity(el_count); + let ys_to_set = ys.spare_capacity_mut(); + let ys_to_set = unsafe { + std::mem::transmute::<&mut [std::mem::MaybeUninit], &mut [U]>(ys_to_set) + }; + let mut dst_index = 0; + for src_index in block_start_index { + let vs = &vs[src_index..src_index + block_len]; + let ys = &mut ys_to_set[dst_index..dst_index + block_len]; + f_vec(vs, ys); + dst_index += block_len; + } + // SAFETY: values are all set by f_vec. + unsafe { ys.set_len(el_count) }; + ys + } + } + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/cudnn.rs b/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/cudnn.rs new file mode 100644 index 0000000000..d7d8770587 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/cudnn.rs @@ -0,0 +1,225 @@ +use crate::WithDType; +use cudarc; +use cudarc::cudnn::safe::{ConvForward, Cudnn}; +use cudarc::driver::{CudaSlice, CudaView, DeviceRepr, ValidAsZeroBits}; +use std::cell::RefCell; +use std::collections::HashMap; +use std::sync::Arc; + +// The cudnn handles are stored per thread here rather than on the CudaDevice as they are neither +// send nor sync. +thread_local! { + static CUDNN: RefCell>> = HashMap::new().into(); +} + +impl From for crate::Error { + fn from(err: cudarc::cudnn::CudnnError) -> Self { + crate::Error::wrap(err) + } +} + +impl From for crate::Error { + fn from(err: cudarc::driver::DriverError) -> Self { + crate::Error::wrap(err) + } +} + +pub(crate) fn launch_conv2d< + T: DeviceRepr + WithDType + ValidAsZeroBits + cudarc::cudnn::CudnnDataType, + Y: cudarc::cudnn::CudnnDataType, +>( + src: &CudaView, + src_l: &crate::Layout, + filter: &CudaView, + dst: &mut CudaSlice, + params: &crate::conv::ParamsConv2D, + dev: &crate::cuda_backend::CudaDevice, +) -> crate::Result<()> { + use crate::conv::CudnnFwdAlgo as CandleAlgo; + use cudarc::cudnn::sys::cudnnConvolutionFwdAlgo_t as A; + + let device_id = dev.id(); + let cudnn = CUDNN.with(|cudnn| { + if let Some(cudnn) = cudnn.borrow().get(&device_id) { + return Ok(cudnn.clone()); + } + let c = Cudnn::new(dev.cuda_stream()); + if let Ok(c) = &c { + cudnn.borrow_mut().insert(device_id, c.clone()); + } + c + })?; + let conv = cudnn.create_conv2d::( + /* pad */ [params.padding as i32, params.padding as i32], + /* stride */ [params.stride as i32, params.stride as i32], + /* dilation */ [params.dilation as i32, params.dilation as i32], + cudarc::cudnn::sys::cudnnConvolutionMode_t::CUDNN_CROSS_CORRELATION, + )?; + let x_shape = [ + params.b_size as i32, + params.c_in as i32, + params.i_h as i32, + params.i_w as i32, + ]; + // Note that `src` already starts at the proper offset. + let x = if src_l.is_contiguous() { + cudnn.create_4d_tensor::( + cudarc::cudnn::sys::cudnnTensorFormat_t::CUDNN_TENSOR_NCHW, + x_shape, + )? + } else { + let s = src_l.stride(); + cudnn.create_4d_tensor_ex::( + x_shape, + [s[0] as i32, s[1] as i32, s[2] as i32, s[3] as i32], + )? + }; + let w = cudnn.create_4d_filter::( + cudarc::cudnn::sys::cudnnTensorFormat_t::CUDNN_TENSOR_NCHW, + [ + params.c_out as i32, + params.c_in as i32, + params.k_h as i32, + params.k_w as i32, + ], + )?; + let (w_out, h_out) = (params.out_w() as i32, params.out_h() as i32); + let y = cudnn.create_4d_tensor::( + cudarc::cudnn::sys::cudnnTensorFormat_t::CUDNN_TENSOR_NCHW, + [params.b_size as i32, params.c_out as i32, h_out, w_out], + )?; + let conv2d = ConvForward { + conv: &conv, + x: &x, + w: &w, + y: &y, + }; + let alg = match params.cudnn_fwd_algo { + None => conv2d.pick_algorithm()?, + Some(CandleAlgo::ImplicitGemm) => A::CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM, + Some(CandleAlgo::ImplicitPrecompGemm) => { + A::CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM + } + Some(CandleAlgo::Gemm) => A::CUDNN_CONVOLUTION_FWD_ALGO_GEMM, + Some(CandleAlgo::Direct) => A::CUDNN_CONVOLUTION_FWD_ALGO_DIRECT, + Some(CandleAlgo::Fft) => A::CUDNN_CONVOLUTION_FWD_ALGO_FFT, + Some(CandleAlgo::FftTiling) => A::CUDNN_CONVOLUTION_FWD_ALGO_FFT_TILING, + Some(CandleAlgo::Winograd) => A::CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD, + Some(CandleAlgo::WinogradNonFused) => A::CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED, + Some(CandleAlgo::Count) => A::CUDNN_CONVOLUTION_FWD_ALGO_COUNT, + }; + let workspace_size = conv2d.get_workspace_size(alg)?; + let mut workspace = dev.cuda_stream().alloc_zeros::(workspace_size)?; + unsafe { + conv2d.launch::, _, _, _>( + alg, + Some(&mut workspace), + (T::one(), T::zero()), + src, + filter, + dst, + )?; + } + Ok(()) +} + +pub(crate) fn launch_conv1d< + T: DeviceRepr + WithDType + ValidAsZeroBits + cudarc::cudnn::CudnnDataType, + Y: cudarc::cudnn::CudnnDataType, +>( + src: &CudaView, + src_l: &crate::Layout, + filter: &CudaView, + dst: &mut CudaSlice, + params: &crate::conv::ParamsConv1D, + dev: &crate::cuda_backend::CudaDevice, +) -> crate::Result<()> { + use crate::conv::CudnnFwdAlgo as CandleAlgo; + use cudarc::cudnn::sys::cudnnConvolutionFwdAlgo_t as A; + + let device_id = dev.id(); + let cudnn = CUDNN.with(|cudnn| { + if let Some(cudnn) = cudnn.borrow().get(&device_id) { + return Ok(cudnn.clone()); + } + let c = Cudnn::new(dev.cuda_stream()); + if let Ok(c) = &c { + cudnn.borrow_mut().insert(device_id, c.clone()); + } + c + })?; + let conv = cudnn.create_conv2d::( + /* pad */ [params.padding as i32, 0], + /* stride */ [params.stride as i32, 1], + /* dilation */ [params.dilation as i32, 1], + cudarc::cudnn::sys::cudnnConvolutionMode_t::CUDNN_CROSS_CORRELATION, + )?; + // https://docs.nvidia.com/deeplearning/cudnn/backend/latest/api/cudnn-ops-library.html#cudnnsettensornddescriptor + // > Tensors are restricted to having at least 4 dimensions, and at most CUDNN_DIM_MAX + // > dimensions (defined in cudnn.h). When working with lower dimensional data, it is + // > recommended that the user create a 4D tensor, and set the size along unused dimensions + // > to 1. + let x_shape = [ + params.b_size as i32, + params.c_in as i32, + params.l_in as i32, + 1, + ]; + // Note that `src` already starts at the proper offset. + let x = if src_l.is_contiguous() { + cudnn.create_4d_tensor::( + cudarc::cudnn::sys::cudnnTensorFormat_t::CUDNN_TENSOR_NCHW, + x_shape, + )? + } else { + let s = src_l.stride(); + cudnn.create_4d_tensor_ex::(x_shape, [s[0] as i32, s[1] as i32, s[2] as i32, 1i32])? + }; + let w = cudnn.create_4d_filter::( + cudarc::cudnn::sys::cudnnTensorFormat_t::CUDNN_TENSOR_NCHW, + [ + params.c_out as i32, + params.c_in as i32, + params.k_size as i32, + 1, + ], + )?; + let l_out = params.l_out() as i32; + let y = cudnn.create_4d_tensor::( + cudarc::cudnn::sys::cudnnTensorFormat_t::CUDNN_TENSOR_NCHW, + [params.b_size as i32, params.c_out as i32, l_out, 1], + )?; + let conv1d = ConvForward { + conv: &conv, + x: &x, + w: &w, + y: &y, + }; + let alg = match params.cudnn_fwd_algo { + None => conv1d.pick_algorithm()?, + Some(CandleAlgo::ImplicitGemm) => A::CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM, + Some(CandleAlgo::ImplicitPrecompGemm) => { + A::CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM + } + Some(CandleAlgo::Gemm) => A::CUDNN_CONVOLUTION_FWD_ALGO_GEMM, + Some(CandleAlgo::Direct) => A::CUDNN_CONVOLUTION_FWD_ALGO_DIRECT, + Some(CandleAlgo::Fft) => A::CUDNN_CONVOLUTION_FWD_ALGO_FFT, + Some(CandleAlgo::FftTiling) => A::CUDNN_CONVOLUTION_FWD_ALGO_FFT_TILING, + Some(CandleAlgo::Winograd) => A::CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD, + Some(CandleAlgo::WinogradNonFused) => A::CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED, + Some(CandleAlgo::Count) => A::CUDNN_CONVOLUTION_FWD_ALGO_COUNT, + }; + let workspace_size = conv1d.get_workspace_size(alg)?; + let mut workspace = dev.cuda_stream().alloc_zeros::(workspace_size)?; + unsafe { + conv1d.launch::, _, _, _>( + alg, + Some(&mut workspace), + (T::one(), T::zero()), + src, + filter, + dst, + )?; + } + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/device.rs b/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/device.rs new file mode 100644 index 0000000000..425fd74f76 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/device.rs @@ -0,0 +1,710 @@ +use crate::backend::{BackendDevice, BackendStorage}; +use crate::{CpuStorage, CpuStorageRef, DType, Layout, Result, Shape}; +pub use candle_kernels as kernels; +pub use cudarc; +use cudarc::driver::CudaFunction; +use float8::F8E4M3; +use half::{bf16, f16}; +use std::collections::HashMap; +use std::sync::{Arc, Mutex, RwLock}; + +use super::{CudaError, CudaStorage, CudaStorageSlice, WrapErr}; + +/// Unique identifier for cuda devices. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct DeviceId(usize); + +impl DeviceId { + fn new() -> Self { + // https://users.rust-lang.org/t/idiomatic-rust-way-to-generate-unique-id/33805 + use std::sync::atomic; + static COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::new(1); + Self(COUNTER.fetch_add(1, atomic::Ordering::Relaxed)) + } +} + +struct CudaRng(cudarc::curand::CudaRng); +unsafe impl Send for CudaRng {} + +pub struct ModuleStore { + mdls: [Option>; kernels::ALL_IDS.len()], +} + +#[derive(Clone)] +pub struct CudaDevice { + id: DeviceId, + context: Arc, + modules: Arc>, + custom_modules: Arc>>>, + stream: Arc, + pub(crate) blas: Arc, + curand: Arc>, + seed_value: Arc>, +} + +impl std::fmt::Debug for CudaDevice { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "CudaDevice({:?})", self.id) + } +} + +impl CudaDevice { + #[allow(clippy::missing_safety_doc)] + pub unsafe fn alloc( + &self, + len: usize, + ) -> Result> { + self.stream.alloc::(len).w() + } + + pub fn alloc_zeros( + &self, + len: usize, + ) -> Result> { + self.stream.alloc_zeros::(len).w() + } + + pub fn memcpy_htod< + T: cudarc::driver::DeviceRepr, + Src: cudarc::driver::HostSlice + ?Sized, + Dst: cudarc::driver::DevicePtrMut, + >( + &self, + src: &Src, + dst: &mut Dst, + ) -> Result<()> { + self.stream.memcpy_htod(src, dst).w() + } + + pub fn clone_dtoh>( + &self, + src: &Src, + ) -> Result> { + self.stream.clone_dtoh(src).w() + } + + pub fn memcpy_dtod< + T, + Src: cudarc::driver::DevicePtr, + Dst: cudarc::driver::DevicePtrMut, + >( + &self, + src: &Src, + dst: &mut Dst, + ) -> Result<()> { + self.stream.memcpy_dtod(src, dst).w() + } + + pub fn memcpy_dtoh< + T: cudarc::driver::DeviceRepr, + Src: cudarc::driver::DevicePtr, + Dst: cudarc::driver::HostSlice, + >( + &self, + src: &Src, + dst: &mut Dst, + ) -> Result<()> { + self.stream.memcpy_dtoh(src, dst).w() + } + + pub fn clone_htod + ?Sized>( + &self, + src: &Src, + ) -> Result> { + self.stream.clone_htod(src).w() + } +} + +pub struct CudaFunc { + func: CudaFunction, + stream: Arc, +} + +impl std::ops::Deref for CudaFunc { + type Target = CudaFunction; + + fn deref(&self) -> &Self::Target { + &self.func + } +} + +impl CudaFunc { + pub fn into_cuda_function(self) -> CudaFunction { + self.func + } +} + +#[macro_export] +macro_rules! builder_arg { + ($b:ident, $($arg:expr),*) => { + $( + let __arg = $arg; + $b.arg(&__arg); + )* + }; +} + +impl CudaFunc { + pub fn builder(&self) -> cudarc::driver::LaunchArgs<'_> { + self.stream.launch_builder(&self.func) + } +} + +impl CudaDevice { + pub fn cuda_stream(&self) -> Arc { + self.stream.clone() + } + + /// When turned on, all cuda tensors **created after calling this function** will + /// not track uses via cuda events. + /// + /// # Safety + /// + /// It is up to the user to ensure proper synchronization between multiple streams: + /// - Ensure that no tensor is freed before a use on another stream is finished. + /// - Ensure that a tensor is not used on another stream before allocation on the + /// allocating stream finishes. + /// - Ensure that a tensor is not written two concurrently by multiple streams. + pub unsafe fn disable_event_tracking(&self) { + self.context.disable_event_tracking() + } + + pub fn is_event_tracking(&self) -> bool { + self.context.is_event_tracking() + } + + #[cfg(all(feature = "ug", not(target_arch = "wasm32")))] + pub fn compile( + &self, + func_name: &'static str, + kernel: candle_ug::lang::ssa::Kernel, + ) -> Result { + let mut buf = vec![]; + candle_ug::cuda::code_gen::gen(&mut buf, func_name, &kernel)?; + let cuda_code = String::from_utf8(buf)?; + let opts = cudarc::nvrtc::CompileOptions { + use_fast_math: Some(true), + ..Default::default() + }; + let ptx = cudarc::nvrtc::safe::compile_ptx_with_opts(cuda_code, opts).w()?; + let module = self.context.load_module(ptx).w()?; + let func = module.load_function(func_name).w()?; + Ok(CudaFunc { + func, + stream: self.stream.clone(), + }) + } + + pub fn id(&self) -> DeviceId { + self.id + } + + pub fn get_or_load_custom_func( + &self, + fn_name: &str, + module_name: &str, + ptx: &str, + ) -> Result { + let ms = self.custom_modules.read().unwrap(); + if let Some(mdl) = ms.get(module_name).as_ref() { + let func = mdl.load_function(fn_name).w()?; + return Ok(CudaFunc { + func, + stream: self.stream.clone(), + }); + } + drop(ms); + let mut ms = self.custom_modules.write().unwrap(); + let cuda_module = self.context.load_module(ptx.into()).w()?; + ms.insert(module_name.to_string(), cuda_module.clone()); + let func = cuda_module.load_function(fn_name).w()?; + Ok(CudaFunc { + func, + stream: self.stream.clone(), + }) + } + + pub fn get_or_load_func(&self, fn_name: &str, mdl: &kernels::Module) -> Result { + let ms = self.modules.read().unwrap(); + if let Some(mdl) = ms.mdls[mdl.index()].as_ref() { + let func = mdl.load_function(fn_name).w()?; + return Ok(CudaFunc { + func, + stream: self.stream.clone(), + }); + } + drop(ms); + let mut ms = self.modules.write().unwrap(); + let cuda_module = self.context.load_module(mdl.ptx().into()).w()?; + ms.mdls[mdl.index()] = Some(cuda_module.clone()); + let func = cuda_module.load_function(fn_name).w()?; + Ok(CudaFunc { + func, + stream: self.stream.clone(), + }) + } + + pub fn cublas_handle(&self) -> Arc { + self.blas.clone() + } +} + +impl CudaDevice { + pub fn new_with_stream(ordinal: usize) -> Result { + let context = cudarc::driver::CudaContext::new(ordinal).w()?; + let stream = context.new_stream().w()?; + let blas = cudarc::cublas::CudaBlas::new(stream.clone()).w()?; + let curand = cudarc::curand::CudaRng::new(299792458, stream.clone()).w()?; + let module_store = ModuleStore { + mdls: [const { None }; kernels::ALL_IDS.len()], + }; + Ok(Self { + id: DeviceId::new(), + context, + stream, + blas: Arc::new(blas), + curand: Arc::new(Mutex::new(CudaRng(curand))), + modules: Arc::new(std::sync::RwLock::new(module_store)), + custom_modules: Arc::new(std::sync::RwLock::new(HashMap::new())), + seed_value: Arc::new(RwLock::new(299792458)), + }) + } +} + +impl BackendDevice for CudaDevice { + type Storage = CudaStorage; + + fn new(ordinal: usize) -> Result { + let context = cudarc::driver::CudaContext::new(ordinal).w()?; + let stream = context.default_stream(); + let blas = cudarc::cublas::CudaBlas::new(stream.clone()).w()?; + let curand = cudarc::curand::CudaRng::new(299792458, stream.clone()).w()?; + let module_store = ModuleStore { + mdls: [const { None }; kernels::ALL_IDS.len()], + }; + Ok(Self { + id: DeviceId::new(), + context, + stream, + blas: Arc::new(blas), + curand: Arc::new(Mutex::new(CudaRng(curand))), + modules: Arc::new(std::sync::RwLock::new(module_store)), + custom_modules: Arc::new(std::sync::RwLock::new(HashMap::new())), + seed_value: Arc::new(RwLock::new(299792458)), + }) + } + + fn set_seed(&self, seed: u64) -> Result<()> { + // We do not call set_seed but instead create a new curand object. This ensures that the + // state will be identical and the same random numbers will be generated. + let mut curand = self.curand.lock().unwrap(); + curand.0 = cudarc::curand::CudaRng::new(seed, self.stream.clone()).w()?; + *self.seed_value.write().unwrap() = seed; + Ok(()) + } + + fn get_current_seed(&self) -> Result { + Ok(*self.seed_value.read().unwrap()) + } + + fn location(&self) -> crate::DeviceLocation { + crate::DeviceLocation::Cuda { + gpu_id: self.context.ordinal(), + } + } + + fn same_device(&self, rhs: &Self) -> bool { + self.id == rhs.id + } + + fn zeros_impl(&self, shape: &Shape, dtype: DType) -> Result { + let elem_count = shape.elem_count(); + let slice = match dtype { + DType::U8 => { + let data = self.alloc_zeros::(elem_count)?; + CudaStorageSlice::U8(data) + } + DType::U32 => { + let data = self.alloc_zeros::(elem_count)?; + CudaStorageSlice::U32(data) + } + DType::I16 => { + let data = self.alloc_zeros::(elem_count)?; + CudaStorageSlice::I16(data) + } + DType::I32 => { + let data = self.alloc_zeros::(elem_count)?; + CudaStorageSlice::I32(data) + } + DType::I64 => { + let data = self.alloc_zeros::(elem_count)?; + CudaStorageSlice::I64(data) + } + DType::BF16 => { + let data = self.alloc_zeros::(elem_count)?; + CudaStorageSlice::BF16(data) + } + DType::F16 => { + let data = self.alloc_zeros::(elem_count)?; + CudaStorageSlice::F16(data) + } + DType::F32 => { + let data = self.alloc_zeros::(elem_count)?; + CudaStorageSlice::F32(data) + } + DType::F64 => { + let data = self.alloc_zeros::(elem_count)?; + CudaStorageSlice::F64(data) + } + DType::F8E4M3 => { + let data = self.alloc_zeros::(elem_count)?; + CudaStorageSlice::F8E4M3(data) + } + DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { + return Err( + CudaError::InternalError("Dummy types not supported in CUDA backend").into(), + ) + } + }; + Ok(CudaStorage { + slice, + device: self.clone(), + }) + } + + fn rand_uniform(&self, shape: &Shape, dtype: DType, lo: f64, up: f64) -> Result { + let elem_count = shape.elem_count(); + let curand = self.curand.lock().unwrap(); + let slice = match dtype { + // TODO: Add support for F16 and BF16 though this is likely to require some upstream + // cudarc changes. + DType::U8 + | DType::U32 + | DType::I16 + | DType::I32 + | DType::I64 + | DType::F16 + | DType::BF16 => Err(CudaError::UnsupportedDtype { + dtype, + op: "rand_uniform", + }) + .w()?, + DType::F32 => { + let mut data = unsafe { self.alloc::(elem_count)? }; + curand.0.fill_with_uniform(&mut data).w()?; + CudaStorageSlice::F32(data) + } + DType::F64 => { + let mut data = unsafe { self.alloc::(elem_count)? }; + curand.0.fill_with_uniform(&mut data).w()?; + CudaStorageSlice::F64(data) + } + DType::F8E4M3 | DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { + Err(CudaError::UnsupportedDtype { + dtype, + op: "rand_uniform", + }) + .w()? + } + }; + let slice = if lo == 0. && up == 1.0 { + slice + } else { + use super::utils::Map1; + let layout = Layout::contiguous(shape); + super::Affine(up - lo, lo).map(&slice, self, &layout)? + }; + Ok(CudaStorage { + slice, + device: self.clone(), + }) + } + + fn rand_normal(&self, shape: &Shape, dtype: DType, mean: f64, std: f64) -> Result { + // TODO: Add support for F16 and BF16 though this is likely to require some upstream + // cudarc changes. + let elem_count = shape.elem_count(); + let curand = self.curand.lock().unwrap(); + // curand can only generate an odd number of values. + // https://github.com/huggingface/candle/issues/734 + let elem_count_round = if elem_count % 2 == 1 { + elem_count + 1 + } else { + elem_count + }; + let slice = match dtype { + DType::U8 + | DType::U32 + | DType::I16 + | DType::I32 + | DType::I64 + | DType::F16 + | DType::BF16 => Err(CudaError::UnsupportedDtype { + dtype, + op: "rand_normal", + }) + .w()?, + DType::F32 => { + let mut data = unsafe { self.alloc::(elem_count_round)? }; + curand + .0 + .fill_with_normal(&mut data, mean as f32, std as f32) + .w()?; + CudaStorageSlice::F32(data) + } + DType::F64 => { + let mut data = unsafe { self.alloc::(elem_count_round)? }; + curand.0.fill_with_normal(&mut data, mean, std).w()?; + CudaStorageSlice::F64(data) + } + DType::F8E4M3 | DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { + Err(CudaError::UnsupportedDtype { + dtype, + op: "rand_normal", + }) + .w()? + } + }; + Ok(CudaStorage { + slice, + device: self.clone(), + }) + } + + unsafe fn alloc_uninit(&self, shape: &Shape, dtype: DType) -> Result { + let elem_count = shape.elem_count(); + let slice = match dtype { + DType::U8 => { + let data = self.alloc::(elem_count)?; + CudaStorageSlice::U8(data) + } + DType::U32 => { + let data = self.alloc::(elem_count)?; + CudaStorageSlice::U32(data) + } + DType::I16 => { + let data = self.alloc::(elem_count)?; + CudaStorageSlice::I16(data) + } + DType::I32 => { + let data = self.alloc::(elem_count)?; + CudaStorageSlice::I32(data) + } + DType::I64 => { + let data = self.alloc::(elem_count)?; + CudaStorageSlice::I64(data) + } + DType::BF16 => { + let data = self.alloc::(elem_count)?; + CudaStorageSlice::BF16(data) + } + DType::F16 => { + let data = self.alloc::(elem_count)?; + CudaStorageSlice::F16(data) + } + DType::F32 => { + let data = self.alloc::(elem_count)?; + CudaStorageSlice::F32(data) + } + DType::F64 => { + let data = self.alloc::(elem_count)?; + CudaStorageSlice::F64(data) + } + DType::F8E4M3 => { + let data = self.alloc::(elem_count)?; + CudaStorageSlice::F8E4M3(data) + } + DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { + return Err( + CudaError::InternalError("Dummy types not supported in CUDA backend").into(), + ) + } + }; + Ok(CudaStorage { + slice, + device: self.clone(), + }) + } + + fn storage_from_slice(&self, s: &[T]) -> Result { + let slice = match T::cpu_storage_ref(s) { + CpuStorageRef::U8(storage) => { + let data = self.clone_htod(storage)?; + CudaStorageSlice::U8(data) + } + CpuStorageRef::U32(storage) => { + let data = self.clone_htod(storage)?; + CudaStorageSlice::U32(data) + } + CpuStorageRef::I16(storage) => { + let data = self.clone_htod(storage)?; + CudaStorageSlice::I16(data) + } + CpuStorageRef::I32(storage) => { + let data = self.clone_htod(storage)?; + CudaStorageSlice::I32(data) + } + CpuStorageRef::I64(storage) => { + let data = self.clone_htod(storage)?; + CudaStorageSlice::I64(data) + } + CpuStorageRef::BF16(storage) => { + let data = self.clone_htod(storage)?; + CudaStorageSlice::BF16(data) + } + CpuStorageRef::F16(storage) => { + let data = self.clone_htod(storage)?; + CudaStorageSlice::F16(data) + } + CpuStorageRef::F32(storage) => { + let data = self.clone_htod(storage)?; + CudaStorageSlice::F32(data) + } + CpuStorageRef::F64(storage) => { + let data = self.clone_htod(storage)?; + CudaStorageSlice::F64(data) + } + CpuStorageRef::F8E4M3(storage) => { + let data = self.clone_htod(storage)?; + CudaStorageSlice::F8E4M3(data) + } + CpuStorageRef::F4(_) + | CpuStorageRef::F6E2M3(_) + | CpuStorageRef::F6E3M2(_) + | CpuStorageRef::F8E8M0(_) => { + return Err(CudaError::UnsupportedDtype { + dtype: T::DTYPE, + op: "storage_from_slice", + } + .into()); + } + }; + Ok(CudaStorage { + slice, + device: self.clone(), + }) + } + + fn storage_from_cpu_storage(&self, storage: &CpuStorage) -> Result { + let slice = match storage { + CpuStorage::U8(storage) => { + let data = self.clone_htod(storage)?; + CudaStorageSlice::U8(data) + } + CpuStorage::U32(storage) => { + let data = self.clone_htod(storage)?; + CudaStorageSlice::U32(data) + } + CpuStorage::I16(storage) => { + let data = self.clone_htod(storage)?; + CudaStorageSlice::I16(data) + } + CpuStorage::I32(storage) => { + let data = self.clone_htod(storage)?; + CudaStorageSlice::I32(data) + } + CpuStorage::I64(storage) => { + let data = self.clone_htod(storage)?; + CudaStorageSlice::I64(data) + } + CpuStorage::BF16(storage) => { + let data = self.clone_htod(storage)?; + CudaStorageSlice::BF16(data) + } + CpuStorage::F16(storage) => { + let data = self.clone_htod(storage)?; + CudaStorageSlice::F16(data) + } + CpuStorage::F32(storage) => { + let data = self.clone_htod(storage)?; + CudaStorageSlice::F32(data) + } + CpuStorage::F64(storage) => { + let data = self.clone_htod(storage)?; + CudaStorageSlice::F64(data) + } + CpuStorage::F8E4M3(storage) => { + let data = self.clone_htod(storage)?; + CudaStorageSlice::F8E4M3(data) + } + CpuStorage::F4(_) + | CpuStorage::F6E2M3(_) + | CpuStorage::F6E3M2(_) + | CpuStorage::F8E8M0(_) => { + return Err(CudaError::UnsupportedDtype { + dtype: storage.dtype(), + op: "storage_from_cpu_storage", + } + .into()); + } + }; + Ok(CudaStorage { + slice, + device: self.clone(), + }) + } + + fn storage_from_cpu_storage_owned(&self, storage: CpuStorage) -> Result { + let slice = match storage { + CpuStorage::U8(storage) => { + let data = self.clone_htod(&storage)?; + CudaStorageSlice::U8(data) + } + CpuStorage::U32(storage) => { + let data = self.clone_htod(&storage)?; + CudaStorageSlice::U32(data) + } + CpuStorage::I16(storage) => { + let data = self.clone_htod(&storage)?; + CudaStorageSlice::I16(data) + } + CpuStorage::I32(storage) => { + let data = self.clone_htod(&storage)?; + CudaStorageSlice::I32(data) + } + CpuStorage::I64(storage) => { + let data = self.clone_htod(&storage)?; + CudaStorageSlice::I64(data) + } + CpuStorage::BF16(storage) => { + let data = self.clone_htod(&storage)?; + CudaStorageSlice::BF16(data) + } + CpuStorage::F16(storage) => { + let data = self.clone_htod(&storage)?; + CudaStorageSlice::F16(data) + } + CpuStorage::F32(storage) => { + let data = self.clone_htod(&storage)?; + CudaStorageSlice::F32(data) + } + CpuStorage::F64(storage) => { + let data = self.clone_htod(&storage)?; + CudaStorageSlice::F64(data) + } + CpuStorage::F8E4M3(storage) => { + let data = self.clone_htod(&storage)?; + CudaStorageSlice::F8E4M3(data) + } + CpuStorage::F4(_) + | CpuStorage::F6E2M3(_) + | CpuStorage::F6E3M2(_) + | CpuStorage::F8E8M0(_) => { + return Err(CudaError::UnsupportedDtype { + dtype: storage.dtype(), + op: "storage_from_cpu_storage_owned", + } + .into()); + } + }; + Ok(CudaStorage { + slice, + device: self.clone(), + }) + } + + fn synchronize(&self) -> Result<()> { + self.stream.synchronize().map_err(crate::Error::wrap)?; + Ok(()) + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/error.rs b/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/error.rs new file mode 100644 index 0000000000..bd6f8ac6da --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/error.rs @@ -0,0 +1,62 @@ +use crate::{DType, Layout}; + +/// cudarc related errors +#[derive(thiserror::Error, Debug)] +pub enum CudaError { + #[error(transparent)] + Cuda(#[from] cudarc::driver::DriverError), + + #[error(transparent)] + Compiler(#[from] cudarc::nvrtc::CompileError), + + #[error(transparent)] + Cublas(#[from] cudarc::cublas::result::CublasError), + + #[error(transparent)] + Curand(#[from] cudarc::curand::result::CurandError), + + #[error("missing kernel '{module_name}'")] + MissingKernel { module_name: String }, + + #[error("unsupported dtype {dtype:?} for {op}")] + UnsupportedDtype { dtype: DType, op: &'static str }, + + #[error("internal error '{0}'")] + InternalError(&'static str), + + #[error("matmul is only supported for contiguous tensors lstride: {lhs_stride:?} rstride: {rhs_stride:?} mnk: {mnk:?}")] + MatMulNonContiguous { + lhs_stride: Layout, + rhs_stride: Layout, + mnk: (usize, usize, usize), + }, + + #[error("{msg}, expected: {expected:?}, got: {got:?}")] + UnexpectedDType { + msg: &'static str, + expected: DType, + got: DType, + }, + + #[error("{cuda} when loading {module_name}")] + Load { + cuda: cudarc::driver::DriverError, + module_name: String, + }, +} + +impl From for crate::Error { + fn from(val: CudaError) -> Self { + crate::Error::Cuda(Box::new(val)).bt() + } +} + +pub trait WrapErr { + fn w(self) -> std::result::Result; +} + +impl> WrapErr for std::result::Result { + fn w(self) -> std::result::Result { + self.map_err(|e| crate::Error::Cuda(Box::new(e.into())).bt()) + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/mod.rs b/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/mod.rs new file mode 100644 index 0000000000..d2a6fd56d6 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/mod.rs @@ -0,0 +1,2682 @@ +//! Implementation of Backend traits for CUDA device +//! +use crate::backend::{BackendDevice, BackendStorage}; +use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; +use crate::{builder_arg as barg, CpuStorage, DType, Layout, Result, WithDType}; +pub use candle_kernels as kernels; +pub use cudarc; +use cudarc::cublas::{Gemm, GemmConfig, StridedBatchedConfig}; +use cudarc::driver::{ + CudaSlice, DevicePtr, DeviceRepr, LaunchConfig, PushKernelArg, ValidAsZeroBits, +}; +use half::{bf16, f16}; + +#[cfg(feature = "cudnn")] +pub mod cudnn; +mod device; +mod error; +mod utils; +pub use device::{CudaDevice, DeviceId}; +pub use error::{CudaError, WrapErr}; +pub use utils::{Map1, Map1Any, Map2, Map2Any, Map2InPlace, Map3, S}; + +pub enum SlicePtrOrNull { + Ptr(CudaSlice), + Null, +} + +impl SlicePtrOrNull { + pub fn builder_arg<'a, 'b: 'a>(&'b self, builder: &mut cudarc::driver::LaunchArgs<'a>) { + match self { + SlicePtrOrNull::Ptr(slice) => builder.arg(slice), + SlicePtrOrNull::Null => builder.arg(&0usize), + }; + } +} + +impl crate::scalar::Scalar { + pub fn builder_arg<'a, 'b: 'a>(&'b self, builder: &mut cudarc::driver::LaunchArgs<'a>) { + use crate::scalar::Scalar; + match self { + Scalar::U8(v) => builder.arg(v), + Scalar::U32(v) => builder.arg(v), + Scalar::I16(v) => builder.arg(v), + Scalar::I32(v) => builder.arg(v), + Scalar::I64(v) => builder.arg(v), + Scalar::F32(v) => builder.arg(v), + Scalar::F64(v) => builder.arg(v), + Scalar::F16(v) => builder.arg(v), + Scalar::BF16(v) => builder.arg(v), + Scalar::F8E4M3(v) => builder.arg(v), + }; + } +} + +impl SlicePtrOrNull { + pub fn params_from_layout(dev: &CudaDevice, l: &Layout) -> Result { + let ds = if l.is_contiguous() { + SlicePtrOrNull::Null + } else { + SlicePtrOrNull::Ptr(dev.clone_htod(&[l.dims(), l.stride()].concat())?) + }; + Ok(ds) + } +} + +#[derive(Debug)] +pub enum CudaStorageSlice { + U8(CudaSlice), + U32(CudaSlice), + I16(CudaSlice), + I32(CudaSlice), + I64(CudaSlice), + BF16(CudaSlice), + F16(CudaSlice), + F32(CudaSlice), + F64(CudaSlice), + F8E4M3(CudaSlice), + // Dummy types that store raw bytes + F6E2M3(CudaSlice), + F6E3M2(CudaSlice), + F4(CudaSlice), + F8E8M0(CudaSlice), +} + +struct Clone; +impl Map1 for Clone { + fn f( + &self, + s: &CudaSlice, + _: &CudaDevice, + _: &Layout, + ) -> Result> { + s.try_clone().w() + } +} + +pub fn kernel_name(root: &str) -> String { + let dtype = T::DTYPE.as_str(); + format!("{root}_{dtype}") +} + +struct Affine(f64, f64); +impl Map1 for Affine { + fn f( + &self, + src: &CudaSlice, + dev: &CudaDevice, + layout: &Layout, + ) -> Result> { + let shape = layout.shape(); + let dims = shape.dims(); + let el = shape.elem_count(); + let cfg = LaunchConfig::for_num_elems(el as u32); + let ds = SlicePtrOrNull::params_from_layout(dev, layout)?; + let src = &src.slice(layout.start_offset()..); + let func = dev.get_or_load_func(&kernel_name::("affine"), &kernels::AFFINE)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(el)? }; + let mut builder = func.builder(); + barg!(builder, el); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + builder.arg(src); + builder.arg(&out); + barg!(builder, T::from_f64(self.0)); + barg!(builder, T::from_f64(self.1)); + // SAFETY: ffi. + unsafe { builder.launch(cfg).w() }?; + Ok(out) + } +} + +struct Elu(f64); +impl Map1 for Elu { + fn f( + &self, + src: &CudaSlice, + dev: &CudaDevice, + layout: &Layout, + ) -> Result> { + let shape = layout.shape(); + let dims = shape.dims(); + let el = shape.elem_count(); + let cfg = LaunchConfig::for_num_elems(el as u32); + let ds = SlicePtrOrNull::params_from_layout(dev, layout)?; + let src = &src.slice(layout.start_offset()..); + let func = dev.get_or_load_func(&kernel_name::("uelu"), &kernels::UNARY)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(el)? }; + let mut builder = func.builder(); + barg!(builder, el); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + barg!(builder, T::from_f64(self.0)); + builder.arg(src); + builder.arg(&out); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + Ok(out) + } +} + +#[allow(unused)] +struct Im2Col1D { + l_k: usize, + stride: usize, + dilation: usize, + padding: usize, +} + +impl Im2Col1D { + #[allow(unused)] + fn l_out(&self, l: usize) -> usize { + (l + 2 * self.padding - self.dilation * (self.l_k - 1) - 1) / self.stride + 1 + } +} + +impl Map1 for Im2Col1D { + fn f( + &self, + src: &CudaSlice, + dev: &CudaDevice, + layout: &Layout, + ) -> Result> { + let shape = layout.shape(); + let dims = shape.dims(); + let l_out = self.l_out(dims[2]); + let threads = dims[0] * l_out * dims[1]; + let cfg = LaunchConfig::for_num_elems(threads as u32); + let ds = dev.clone_htod(&[dims, layout.stride()].concat())?; + let src = &src.slice(layout.start_offset()..); + let func = dev.get_or_load_func(&kernel_name::("im2col1d"), &kernels::CONV)?; + // SAFETY: Set later by running the kernel. + let dst = unsafe { dev.alloc::(threads * self.l_k)? }; + let mut builder = func.builder(); + barg!(builder, threads); + barg!(builder, l_out); + barg!(builder, self.l_k); + barg!(builder, self.stride); + barg!(builder, self.padding); + barg!(builder, self.dilation); + builder.arg(&ds); + builder.arg(src); + builder.arg(&dst); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + Ok(dst) + } +} + +#[allow(unused)] +struct Im2Col { + h_k: usize, + w_k: usize, + stride: usize, + dilation: usize, + padding: usize, +} + +impl Im2Col { + #[allow(unused)] + fn hw_out(&self, h: usize, w: usize) -> (usize, usize) { + let h_out = (h + 2 * self.padding - self.dilation * (self.h_k - 1) - 1) / self.stride + 1; + let w_out = (w + 2 * self.padding - self.dilation * (self.w_k - 1) - 1) / self.stride + 1; + (h_out, w_out) + } +} + +impl Map1 for Im2Col { + fn f( + &self, + src: &CudaSlice, + dev: &CudaDevice, + layout: &Layout, + ) -> Result> { + let shape = layout.shape(); + let dims = shape.dims(); + let (h_out, w_out) = self.hw_out(dims[2], dims[3]); + let dst_el = dims[0] * h_out * w_out * dims[1] * self.h_k * self.w_k; + let cfg = LaunchConfig::for_num_elems(dst_el as u32); + let ds = dev.clone_htod(&[dims, layout.stride()].concat())?; + let src = &src.slice(layout.start_offset()..); + let func = dev.get_or_load_func(&kernel_name::("im2col"), &kernels::CONV)?; + // SAFETY: Set later by running the kernel. + let dst = unsafe { dev.alloc::(dst_el)? }; + let mut builder = func.builder(); + barg!(builder, dst_el); + barg!(builder, h_out); + barg!(builder, w_out); + barg!(builder, self.h_k); + barg!(builder, self.w_k); + barg!(builder, self.stride); + barg!(builder, self.padding); + barg!(builder, self.dilation); + builder.arg(&ds); + builder.arg(src); + builder.arg(&dst); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + Ok(dst) + } +} + +struct Powf(f64); +impl Map1 for Powf { + fn f( + &self, + src: &CudaSlice, + dev: &CudaDevice, + layout: &Layout, + ) -> Result> { + let shape = layout.shape(); + let dims = shape.dims(); + let el = shape.elem_count(); + let cfg = LaunchConfig::for_num_elems(el as u32); + let ds = SlicePtrOrNull::params_from_layout(dev, layout)?; + let src = &src.slice(layout.start_offset()..); + let func = dev.get_or_load_func(&kernel_name::("upowf"), &kernels::UNARY)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(el)? }; + let mut builder = func.builder(); + barg!(builder, el); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + barg!(builder, T::from_f64(self.0)); + builder.arg(src); + builder.arg(&out); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + Ok(out) + } +} + +struct FastReduce<'a>(&'a [usize], ReduceOp); +impl Map1Any for FastReduce<'_> { + fn f) -> S>( + &self, + src: &CudaSlice, + dev: &CudaDevice, + layout: &Layout, + wrap: W, + ) -> Result { + let src_stride = layout.stride(); + let src_dims = layout.shape().dims(); + let src_el: usize = src_dims.iter().product(); + // Source dims and strides with the sum dims at the end. + let mut dims = vec![]; + let mut stride = vec![]; + let mut dst_el: usize = 1; + for (dim_idx, &d) in src_dims.iter().enumerate() { + if !self.0.contains(&dim_idx) { + dst_el *= d; + dims.push(d); + stride.push(src_stride[dim_idx]); + } + } + for &dim_idx in self.0.iter() { + dims.push(src_dims[dim_idx]); + stride.push(src_stride[dim_idx]); + } + let el_to_sum_per_block = src_el / dst_el; + // The reduction loop requires the shared array to be properly initialized and for + // this we want the number of threads to be a power of two. + let block_dim = usize::min(1024, el_to_sum_per_block).next_power_of_two(); + let cfg = LaunchConfig { + // TODO: Maybe use grid_y if the output is too large? + // TODO: Specialized implementation when reducing on no or all dimensions or when + // reducing only aggregate a small number of elements together. + grid_dim: (dst_el as u32, 1, 1), + block_dim: (block_dim as u32, 1, 1), + shared_mem_bytes: 0, + }; + let ds = dev.clone_htod(&[dims.as_slice(), stride.as_slice()].concat())?; + let src = &src.slice(layout.start_offset()..); + let (name, check_empty, return_index) = match self.1 { + ReduceOp::Sum => ("fast_sum", false, false), + ReduceOp::Min => ("fast_min", true, false), + ReduceOp::Max => ("fast_max", true, false), + ReduceOp::ArgMin => ("fast_argmin", true, true), + ReduceOp::ArgMax => ("fast_argmax", true, true), + }; + if check_empty && layout.shape().elem_count() == 0 { + Err(crate::Error::EmptyTensor { op: "reduce" }.bt())? + } + let func = dev.get_or_load_func(&kernel_name::(name), &kernels::REDUCE)?; + if return_index { + // SAFETY: filled in by the follow up kernel. + let out = unsafe { dev.alloc::(dst_el)? }; + let mut builder = func.builder(); + barg!(builder, src_el); + barg!(builder, el_to_sum_per_block); + barg!(builder, src_dims.len()); + builder.arg(&ds); + builder.arg(src); + builder.arg(&out); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + Ok(S::U32(out)) + } else { + // SAFETY: filled in by the follow up kernel. + let out = unsafe { dev.alloc::(dst_el)? }; + let mut builder = func.builder(); + barg!(builder, src_el); + barg!(builder, el_to_sum_per_block); + barg!(builder, src_dims.len()); + builder.arg(&ds); + builder.arg(src); + builder.arg(&out); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + Ok(wrap(out)) + } + } +} + +impl Map1 for U { + fn f( + &self, + src: &CudaSlice, + dev: &CudaDevice, + layout: &Layout, + ) -> Result> { + let shape = layout.shape(); + let dims = shape.dims(); + let el_count = shape.elem_count(); + let cfg = LaunchConfig::for_num_elems(el_count as u32); + let ds = SlicePtrOrNull::params_from_layout(dev, layout)?; + let src = &src.slice(layout.start_offset()..); + let func = dev.get_or_load_func(&kernel_name::(U::KERNEL), &kernels::UNARY)?; + // SAFETY: Set later by running the kernel. + let mut out = unsafe { dev.alloc::(el_count)? }; + let mut builder = func.builder(); + barg!(builder, el_count); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + builder.arg(src); + builder.arg(&mut out); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + Ok(out) + } +} + +fn slice_ptr(v: &CudaSlice, lo: usize) -> (u64, cudarc::driver::SyncOnDrop<'_>) { + let (_, guard) = v.device_ptr(v.stream()); + let (ptr, _) = v.slice(lo..).device_ptr(v.stream()); + (ptr, guard) +} + +struct IndexSelect<'a>(&'a CudaStorage, &'a Layout, usize); +impl Map1 for IndexSelect<'_> { + fn f( + &self, + src: &CudaSlice, + dev: &CudaDevice, + src_l: &Layout, + ) -> Result> { + let ids_l = &self.1; + let (name, (ids, _guard)) = match &self.0.slice { + CudaStorageSlice::U32(slice) => ("is_u32", slice_ptr(slice, ids_l.start_offset())), + CudaStorageSlice::U8(slice) => ("is_u8", slice_ptr(slice, ids_l.start_offset())), + CudaStorageSlice::I64(slice) => ("is_i64", slice_ptr(slice, ids_l.start_offset())), + _ => Err(CudaError::UnexpectedDType { + msg: "index_select ids should be u8, u32, or i64", + expected: DType::U32, + got: self.0.dtype(), + }) + .w()?, + }; + let ids_shape = ids_l.shape(); + let ids_dims = ids_shape.dims(); + let ds = dev.clone_htod(&[ids_dims, ids_l.stride()].concat())?; + let src = match src_l.contiguous_offsets() { + Some((o1, o2)) => src.slice(o1..o2), + None => Err(crate::Error::RequiresContiguous { op: "index-select" }.bt())?, + }; + let left_size: usize = src_l.dims()[..self.2].iter().product(); + let right_size: usize = src_l.dims()[self.2 + 1..].iter().product(); + let src_dim_size = src_l.dims()[self.2]; + let ids_dim_size = ids_shape.elem_count(); + let dst_el = ids_shape.elem_count() * left_size * right_size; + let cfg = LaunchConfig::for_num_elems(dst_el as u32); + let func = dev.get_or_load_func(&kernel_name::(name), &kernels::INDEXING)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(dst_el)? }; + let mut builder = func.builder(); + barg!(builder, dst_el); + barg!(builder, ids_dims.len()); + builder.arg(&ds); + barg!(builder, ids); + builder.arg(&src); + builder.arg(&out); + barg!(builder, left_size); + barg!(builder, src_dim_size); + barg!(builder, ids_dim_size); + barg!(builder, right_size); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + Ok(out) + } +} + +struct Gather<'a>(&'a CudaStorage, &'a Layout, usize); +impl Map1 for Gather<'_> { + fn f( + &self, + src: &CudaSlice, + dev: &CudaDevice, + src_l: &Layout, + ) -> Result> { + let ids = &self.0; + let ids_l = &self.1; + let dim = self.2; + let (ids_o1, _) = match ids_l.contiguous_offsets() { + Some(o12) => o12, + None => Err(crate::Error::RequiresContiguous { op: "gather" }.bt())?, + }; + let (name, (ids, _guard)) = match &ids.slice { + CudaStorageSlice::U32(slice) => ("gather_u32", slice_ptr(slice, ids_o1)), + CudaStorageSlice::U8(slice) => ("gather_u8", slice_ptr(slice, ids_o1)), + CudaStorageSlice::I64(slice) => ("gather_i64", slice_ptr(slice, ids_o1)), + _ => Err(CudaError::UnexpectedDType { + msg: "gather ids should be u8/u32/i64", + expected: DType::U32, + got: ids.dtype(), + })?, + }; + let el = ids_l.shape().elem_count(); + let cfg = LaunchConfig::for_num_elems(el as u32); + let src = match src_l.contiguous_offsets() { + Some((o1, o2)) => src.slice(o1..o2), + None => Err(crate::Error::RequiresContiguous { op: "gather" }.bt())?, + }; + let left_sz: usize = src_l.dims()[..dim].iter().product(); + let right_sz: usize = src_l.dims()[dim + 1..].iter().product(); + let src_dim_sz = src_l.dims()[dim]; + let ids_dim_sz = ids_l.dims()[dim]; + let func = dev.get_or_load_func(&kernel_name::(name), &kernels::INDEXING)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(el)? }; + let mut builder = func.builder(); + barg!(builder, el); + barg!(builder, ids); + builder.arg(&src); + builder.arg(&out); + barg!(builder, left_sz); + barg!(builder, src_dim_sz); + barg!(builder, ids_dim_sz); + barg!(builder, right_sz); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + Ok(out) + } +} + +struct IndexAdd<'a>(&'a CudaStorage, &'a Layout, usize); +impl Map2InPlace for IndexAdd<'_> { + fn f( + &self, + dst: &mut CudaSlice, + dst_l: &Layout, + src: &CudaSlice, + src_l: &Layout, + dev: &CudaDevice, + ) -> Result<()> { + let ids = &self.0; + let ids_l = &self.1; + let dim = self.2; + let (ids_o1, _) = match ids_l.contiguous_offsets() { + Some(o12) => o12, + None => Err(crate::Error::RequiresContiguous { op: "index-add" }.bt())?, + }; + let (name, (ids, _guard)) = match &ids.slice { + CudaStorageSlice::U32(slice) => ("ia_u32", slice_ptr(slice, ids_o1)), + CudaStorageSlice::I64(slice) => ("ia_i64", slice_ptr(slice, ids_o1)), + CudaStorageSlice::U8(slice) => ("ia_u8", slice_ptr(slice, ids_o1)), + _ => Err(CudaError::UnexpectedDType { + msg: "index-add ids should be u8/u32/i64", + expected: DType::U32, + got: ids.dtype(), + })?, + }; + let dst = match dst_l.contiguous_offsets() { + Some((o1, o2)) => dst.slice(o1..o2), + None => Err(crate::Error::RequiresContiguous { op: "index-add" }.bt())?, + }; + let src = match src_l.contiguous_offsets() { + Some((o1, o2)) => src.slice(o1..o2), + None => Err(crate::Error::RequiresContiguous { op: "index-add" }.bt())?, + }; + let left_sz: usize = src_l.dims()[..dim].iter().product(); + let right_sz: usize = src_l.dims()[dim + 1..].iter().product(); + let src_dim_sz = src_l.dims()[dim]; + let dst_dim_sz = dst_l.dims()[dim]; + let ids_dim_sz = ids_l.dims()[0]; + let cfg = LaunchConfig::for_num_elems((left_sz * right_sz) as u32); + let func = dev.get_or_load_func(&kernel_name::(name), &kernels::INDEXING)?; + let mut builder = func.builder(); + barg!(builder, ids); + barg!(builder, ids_dim_sz); + builder.arg(&src); + builder.arg(&dst); + barg!(builder, left_sz, src_dim_sz, dst_dim_sz, right_sz); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + Ok(()) + } +} + +struct Scatter<'a>(&'a CudaStorage, &'a Layout, usize); +impl Map2InPlace for Scatter<'_> { + fn f( + &self, + dst: &mut CudaSlice, + dst_l: &Layout, + src: &CudaSlice, + src_l: &Layout, + dev: &CudaDevice, + ) -> Result<()> { + let ids = &self.0; + let ids_l = &self.1; + let dim = self.2; + let (ids_o1, _) = match ids_l.contiguous_offsets() { + Some(o12) => o12, + None => Err(crate::Error::RequiresContiguous { op: "scatter" }.bt())?, + }; + let (name, (ids, _guard)) = match &ids.slice { + CudaStorageSlice::U32(slice) => ("s_u32", slice_ptr(slice, ids_o1)), + CudaStorageSlice::I64(slice) => ("s_i64", slice_ptr(slice, ids_o1)), + CudaStorageSlice::U8(slice) => ("s_u8", slice_ptr(slice, ids_o1)), + _ => Err(CudaError::UnexpectedDType { + msg: "scatter ids should be u8/u32/i64", + expected: DType::U32, + got: ids.dtype(), + })?, + }; + let dst = match dst_l.contiguous_offsets() { + Some((o1, o2)) => dst.slice(o1..o2), + None => Err(crate::Error::RequiresContiguous { op: "scatter" }.bt())?, + }; + let src = match src_l.contiguous_offsets() { + Some((o1, o2)) => src.slice(o1..o2), + None => Err(crate::Error::RequiresContiguous { op: "scatter" }.bt())?, + }; + let left_sz: usize = src_l.dims()[..dim].iter().product(); + let right_sz: usize = src_l.dims()[dim + 1..].iter().product(); + let src_dim_sz = src_l.dims()[dim]; + let dst_dim_sz = dst_l.dims()[dim]; + let cfg = LaunchConfig::for_num_elems((left_sz * right_sz) as u32); + let func = dev.get_or_load_func(&kernel_name::(name), &kernels::INDEXING)?; + let mut builder = func.builder(); + barg!(builder, ids); + builder.arg(&src); + builder.arg(&dst); + barg!(builder, left_sz, src_dim_sz, dst_dim_sz, right_sz); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + Ok(()) + } +} + +struct ScatterAdd<'a>(&'a CudaStorage, &'a Layout, usize); +impl Map2InPlace for ScatterAdd<'_> { + fn f( + &self, + dst: &mut CudaSlice, + dst_l: &Layout, + src: &CudaSlice, + src_l: &Layout, + dev: &CudaDevice, + ) -> Result<()> { + let ids = &self.0; + let ids_l = &self.1; + let dim = self.2; + let (ids_o1, _) = match ids_l.contiguous_offsets() { + Some(o12) => o12, + None => Err(crate::Error::RequiresContiguous { op: "scatter-add" }.bt())?, + }; + let (name, (ids, _guard)) = match &ids.slice { + CudaStorageSlice::U32(slice) => ("sa_u32", slice_ptr(slice, ids_o1)), + CudaStorageSlice::I64(slice) => ("sa_i64", slice_ptr(slice, ids_o1)), + CudaStorageSlice::U8(slice) => ("sa_u8", slice_ptr(slice, ids_o1)), + _ => Err(CudaError::UnexpectedDType { + msg: "scatter-add ids should be u8/u32/i64", + expected: DType::U32, + got: ids.dtype(), + })?, + }; + let dst = match dst_l.contiguous_offsets() { + Some((o1, o2)) => dst.slice(o1..o2), + None => Err(crate::Error::RequiresContiguous { op: "scatter-add" }.bt())?, + }; + let src = match src_l.contiguous_offsets() { + Some((o1, o2)) => src.slice(o1..o2), + None => Err(crate::Error::RequiresContiguous { op: "scatter-add" }.bt())?, + }; + let left_sz: usize = src_l.dims()[..dim].iter().product(); + let right_sz: usize = src_l.dims()[dim + 1..].iter().product(); + let src_dim_sz = src_l.dims()[dim]; + let dst_dim_sz = dst_l.dims()[dim]; + let cfg = LaunchConfig::for_num_elems((left_sz * right_sz) as u32); + let func = dev.get_or_load_func(&kernel_name::(name), &kernels::INDEXING)?; + let mut builder = func.builder(); + barg!(builder, ids); + builder.arg(&src); + builder.arg(&dst); + barg!(builder, left_sz, src_dim_sz, dst_dim_sz, right_sz); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + Ok(()) + } +} + +struct Conv1D<'a>(&'a crate::conv::ParamsConv1D); +impl Map2 for Conv1D<'_> { + fn f( + &self, + inp: &CudaSlice, + inp_l: &Layout, + k: &CudaSlice, + k_l: &Layout, + dev: &CudaDevice, + ) -> Result> { + // Kernel shape: (c_out, c_in_k, k_size) + // Input shape: (b_size, c_in, l_in) or (c_in, l_in) + let p = &self.0; + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(k_l.start_offset()..); + let shape = inp_l.shape(); + let dims = shape.dims(); + let el = shape.elem_count(); + let l_out = p.l_out(); + let dst_el = p.c_out * l_out * p.b_size; + let cfg = LaunchConfig::for_num_elems(dst_el as u32); + let func = dev.get_or_load_func(&kernel_name::("conv1d"), &kernels::CONV)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(dst_el)? }; + let ds = if dims.len() == 3 { + [dims, inp_l.stride(), k_l.dims(), k_l.stride()].concat() + } else if dims.len() == 2 { + [&[1], dims, &[1], inp_l.stride(), k_l.dims(), k_l.stride()].concat() + } else { + crate::bail!("unexpected input shape for conv1d {dims:?}") + }; + let ds = dev.clone_htod(&ds)?; + let mut builder = func.builder(); + barg!(builder, el, l_out, p.stride, p.padding, p.dilation); + builder.arg(&ds); + builder.arg(inp); + builder.arg(k); + builder.arg(&out); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + Ok(out) + } +} + +struct Conv2D<'a>(&'a crate::conv::ParamsConv2D); +impl Map2 for Conv2D<'_> { + fn f( + &self, + inp: &CudaSlice, + inp_l: &Layout, + k: &CudaSlice, + k_l: &Layout, + dev: &CudaDevice, + ) -> Result> { + // Kernel shape: (c_out, c_in_k, h_k, w_k) + // Input shape: (b_size, c_in, h_in, w_in) + let p = &self.0; + let (out_w, out_h) = (p.out_w(), p.out_h()); + let dst_el = p.c_out * out_w * out_h * p.b_size; + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(k_l.start_offset()..); + let shape = inp_l.shape(); + let dims = shape.dims(); + let el = shape.elem_count(); + + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(dst_el)? }; + let cfg = LaunchConfig::for_num_elems(dst_el as u32); + let func = dev.get_or_load_func(&kernel_name::("conv2d"), &kernels::CONV)?; + let ds = if dims.len() == 4 { + [dims, inp_l.stride(), k_l.dims(), k_l.stride()].concat() + } else { + crate::bail!("unexpected input shape for conv2d {dims:?}") + }; + let ds = dev.clone_htod(&ds)?; + let mut builder = func.builder(); + barg!(builder, el, out_w, out_h, p.stride, p.padding, p.dilation); + builder.arg(&ds); + builder.arg(inp); + builder.arg(k); + builder.arg(&out); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + Ok(out) + } +} + +struct Col2Im1D { + stride: usize, +} + +impl Map1 for Col2Im1D { + fn f( + &self, + col: &CudaSlice, + dev: &CudaDevice, + l: &Layout, + ) -> Result> { + let (b_size, l_in, c_out, k_size) = l.shape().dims4()?; + let stride = self.stride; + let l_out = (l_in - 1) * stride + k_size; + let dst_el = b_size * c_out * l_out; + let mut im = unsafe { dev.alloc::(dst_el)? }; + + let cfg = LaunchConfig::for_num_elems(dst_el as u32); + let func = dev.get_or_load_func(&kernel_name::("col2im1d"), &kernels::CONV)?; + let mut builder = func.builder(); + barg!(builder, dst_el, l_out, l_in, c_out, k_size, stride); + builder.arg(col); + builder.arg(&mut im); + unsafe { builder.launch(cfg) }.w()?; + Ok(im) + } +} + +struct ConvTranspose1D<'a>(&'a crate::conv::ParamsConvTranspose1D); +impl Map2 for ConvTranspose1D<'_> { + fn f( + &self, + inp: &CudaSlice, + inp_l: &Layout, + k: &CudaSlice, + k_l: &Layout, + dev: &CudaDevice, + ) -> Result> { + // Kernel shape: (c_in_k, c_out, l_k) + // Input shape: (b_size, c_in, l_in) + let p = &self.0; + let l_out = p.l_out(); + let dst_el = p.c_out * l_out * p.b_size; + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(k_l.start_offset()..); + let shape = inp_l.shape(); + let dims = shape.dims(); + let el = shape.elem_count(); + + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(dst_el)? }; + let cfg = LaunchConfig::for_num_elems(dst_el as u32); + let func = dev.get_or_load_func(&kernel_name::("conv_transpose1d"), &kernels::CONV)?; + let ds = if dims.len() == 3 { + [dims, inp_l.stride(), k_l.dims(), k_l.stride()].concat() + } else { + crate::bail!("unexpected input shape for conv_transpose1d {dims:?}") + }; + let ds = dev.clone_htod(&ds)?; + let mut builder = func.builder(); + barg!(builder, el); + barg!(builder, l_out); + barg!(builder, p.stride); + barg!(builder, p.padding); + barg!(builder, p.output_padding); + barg!(builder, p.dilation); + builder.arg(&ds); + builder.arg(inp); + builder.arg(k); + builder.arg(&out); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + Ok(out) + } +} + +struct ConvTranspose2D<'a>(&'a crate::conv::ParamsConvTranspose2D); +impl Map2 for ConvTranspose2D<'_> { + fn f( + &self, + inp: &CudaSlice, + inp_l: &Layout, + k: &CudaSlice, + k_l: &Layout, + dev: &CudaDevice, + ) -> Result> { + // Kernel shape: (c_in_k, c_out, h_k, w_k) + // Input shape: (b_size, c_in, h_in, w_in) + let p = &self.0; + let (out_w, out_h) = (p.out_w(), p.out_h()); + let dst_el = p.c_out * out_w * out_h * p.b_size; + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(k_l.start_offset()..); + let shape = inp_l.shape(); + let dims = shape.dims(); + let el = shape.elem_count(); + + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(dst_el)? }; + let cfg = LaunchConfig::for_num_elems(dst_el as u32); + let func = dev.get_or_load_func(&kernel_name::("conv_transpose2d"), &kernels::CONV)?; + let ds = if dims.len() == 4 { + [dims, inp_l.stride(), k_l.dims(), k_l.stride()].concat() + } else { + crate::bail!("unexpected input shape for conv_transpose2d {dims:?}") + }; + let ds = dev.clone_htod(&ds)?; + let mut builder = func.builder(); + barg!(builder, el); + barg!(builder, out_w); + barg!(builder, out_h); + barg!(builder, p.stride); + barg!(builder, p.padding); + barg!(builder, p.output_padding); + barg!(builder, p.dilation); + builder.arg(&ds); + builder.arg(inp); + builder.arg(k); + builder.arg(&out); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + Ok(out) + } +} + +enum PoolOp { + Max, + Avg, +} + +struct Pool2D { + w_k: usize, + h_k: usize, + w_stride: usize, + h_stride: usize, + op: PoolOp, +} + +impl Map1 for Pool2D { + fn f( + &self, + inp: &CudaSlice, + dev: &CudaDevice, + inp_l: &Layout, + ) -> Result> { + // Input shape: (b_size, c, h, w) + let inp = &inp.slice(inp_l.start_offset()..); + let shape = inp_l.shape(); + let dims = shape.dims(); + let ds = if dims.len() == 4 { + [dims, inp_l.stride()].concat() + } else { + crate::bail!("unexpected input shape for pool {dims:?}") + }; + let el = shape.elem_count(); + let out_w = (dims[2] - self.w_k) / self.w_stride + 1; + let out_h = (dims[3] - self.h_k) / self.h_stride + 1; + let dst_el = out_w * out_h * dims[0] * dims[1]; + let cfg = LaunchConfig::for_num_elems(dst_el as u32); + let kname = match self.op { + PoolOp::Max => "max_pool2d", + PoolOp::Avg => "avg_pool2d", + }; + let func = dev.get_or_load_func(&kernel_name::(kname), &kernels::CONV)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(dst_el)? }; + let ds = dev.clone_htod(&ds)?; + let mut builder = func.builder(); + barg!(builder, el); + barg!(builder, self.w_k); + barg!(builder, self.h_k); + barg!(builder, self.w_stride); + barg!(builder, self.h_stride); + builder.arg(&ds); + builder.arg(inp); + builder.arg(&out); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + Ok(out) + } +} + +struct UpsampleNearest2D(usize, usize); +impl Map1 for UpsampleNearest2D { + fn f( + &self, + inp: &CudaSlice, + dev: &CudaDevice, + inp_l: &Layout, + ) -> Result> { + // Input shape: (b_size, c, h, w) + let inp = &inp.slice(inp_l.start_offset()..); + let shape = inp_l.shape(); + let dims = shape.dims(); + let ds = if dims.len() == 4 { + [dims, inp_l.stride()].concat() + } else { + crate::bail!("unexpected input shape for upsample {dims:?}") + }; + let (out_w, out_h) = (self.0, self.1); + let dst_el = out_w * out_h * dims[0] * dims[1]; + let cfg = LaunchConfig::for_num_elems(dst_el as u32); + let func = dev.get_or_load_func(&kernel_name::("upsample_nearest2d"), &kernels::CONV)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(dst_el)? }; + let ds = dev.clone_htod(&ds)?; + let scale_w = dims[2] as f64 / out_w as f64; + let scale_h = dims[3] as f64 / out_h as f64; + let mut builder = func.builder(); + barg!(builder, out_w); + barg!(builder, out_h); + barg!(builder, scale_w); + barg!(builder, scale_h); + builder.arg(&ds); + builder.arg(inp); + builder.arg(&out); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + Ok(out) + } +} + +struct UpsampleBilinear2D { + out_w: usize, + out_h: usize, + align_corners: bool, + scale_h_factor: Option, + scale_w_factor: Option, +} + +impl Map1 for UpsampleBilinear2D { + fn f( + &self, + inp: &CudaSlice, + dev: &CudaDevice, + inp_l: &Layout, + ) -> Result> { + let inp = &inp.slice(inp_l.start_offset()..); + let shape = inp_l.shape(); + let dims = shape.dims(); + let ds = if dims.len() == 4 { + [dims, inp_l.stride()].concat() + } else { + crate::bail!("unexpected input shape for upsample_bilinear2d {dims:?}") + }; + + let (out_w, out_h) = (self.out_w, self.out_h); + let dst_el = out_w * out_h * dims[0] * dims[1]; + let cfg = LaunchConfig::for_num_elems(dst_el as u32); + let func = + dev.get_or_load_func(&kernel_name::("upsample_bilinear2d"), &kernels::CONV)?; + + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(dst_el)? }; + let ds = dev.clone_htod(&ds)?; + + let mut builder = func.builder(); + barg!(builder, out_w); + barg!(builder, out_h); + barg!(builder, self.align_corners); + barg!(builder, self.scale_h_factor.is_some()); + barg!(builder, self.scale_h_factor.unwrap_or(0.0)); + barg!(builder, self.scale_w_factor.is_some()); + barg!(builder, self.scale_w_factor.unwrap_or(0.0)); + builder.arg(&ds); + builder.arg(inp); + builder.arg(&out); + + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + Ok(out) + } +} + +struct WhereCond<'a>(&'a CudaStorage, &'a Layout); +impl Map2 for WhereCond<'_> { + fn f( + &self, + t: &CudaSlice, + layout_t: &Layout, + f: &CudaSlice, + layout_f: &Layout, + dev: &CudaDevice, + ) -> Result> { + let ids_l = &self.1; + let ((ids, _guard), name) = match &self.0.slice { + CudaStorageSlice::U8(slice) => { + let ptr = slice_ptr(slice, ids_l.start_offset()); + (ptr, "where_u8") + } + CudaStorageSlice::U32(slice) => { + let ptr = slice_ptr(slice, ids_l.start_offset()); + (ptr, "where_u32") + } + CudaStorageSlice::I64(slice) => { + let ptr = slice_ptr(slice, ids_l.start_offset()); + (ptr, "where_i64") + } + _ => Err(CudaError::UnexpectedDType { + msg: "where conditions should be u8/u32/i64", + expected: DType::U32, + got: self.0.dtype(), + }) + .w()?, + }; + let shape = ids_l.shape(); + let dims = shape.dims(); + let el = shape.elem_count(); + let cfg = LaunchConfig::for_num_elems(el as u32); + let ds = + dev.clone_htod(&[dims, ids_l.stride(), layout_t.stride(), layout_f.stride()].concat())?; + let t = &t.slice(layout_t.start_offset()..); + let f = &f.slice(layout_f.start_offset()..); + let func = dev.get_or_load_func(&kernel_name::(name), &kernels::TERNARY)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(el)? }; + let mut builder = func.builder(); + barg!(builder, el); + barg!(builder, dims.len()); + builder.arg(&ds); + barg!(builder, ids); + builder.arg(t); + builder.arg(f); + builder.arg(&out); + // SAFETY: ffi + unsafe { builder.launch(cfg) }.w()?; + Ok(out) + } +} + +impl Map2 for U { + fn f( + &self, + lhs: &CudaSlice, + lhs_l: &Layout, + rhs: &CudaSlice, + rhs_l: &Layout, + dev: &CudaDevice, + ) -> Result> { + let shape = lhs_l.shape(); + let dims = shape.dims(); + let elem_count = shape.elem_count(); + let cfg = LaunchConfig::for_num_elems(elem_count as u32); + let dims_and_strides = if lhs_l.is_contiguous() && rhs_l.is_contiguous() { + SlicePtrOrNull::Null + } else { + SlicePtrOrNull::Ptr(dev.clone_htod(&[dims, lhs_l.stride(), rhs_l.stride()].concat())?) + }; + let lhs = &lhs.slice(lhs_l.start_offset()..); + let rhs = &rhs.slice(rhs_l.start_offset()..); + let func = dev.get_or_load_func(&kernel_name::(U::KERNEL), &kernels::BINARY)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(elem_count)? }; + let mut builder = func.builder(); + barg!(builder, elem_count); + barg!(builder, dims.len()); + dims_and_strides.builder_arg(&mut builder); + builder.arg(lhs); + builder.arg(rhs); + builder.arg(&out); + // SAFETY: ffi + unsafe { builder.launch(cfg) }.w()?; + Ok(out) + } +} + +struct Cmp(CmpOp); +impl Map2Any for Cmp { + fn f( + &self, + lhs: &CudaSlice, + lhs_l: &Layout, + rhs: &CudaSlice, + rhs_l: &Layout, + dev: &CudaDevice, + ) -> Result { + let shape = lhs_l.shape(); + let dims = shape.dims(); + let elem_count = shape.elem_count(); + let cfg = LaunchConfig::for_num_elems(elem_count as u32); + let dims_and_strides = if lhs_l.is_contiguous() && rhs_l.is_contiguous() { + SlicePtrOrNull::Null + } else { + SlicePtrOrNull::Ptr(dev.clone_htod(&[dims, lhs_l.stride(), rhs_l.stride()].concat())?) + }; + let lhs = &lhs.slice(lhs_l.start_offset()..); + let rhs = &rhs.slice(rhs_l.start_offset()..); + let name = match self.0 { + CmpOp::Eq => "eq", + CmpOp::Ne => "ne", + CmpOp::Lt => "lt", + CmpOp::Le => "le", + CmpOp::Gt => "gt", + CmpOp::Ge => "ge", + }; + let func = dev.get_or_load_func(&kernel_name::(name), &kernels::BINARY)?; + // SAFETY: Set later by running the kernel. + let out = unsafe { dev.alloc::(elem_count)? }; + let mut builder = func.builder(); + barg!(builder, elem_count); + barg!(builder, dims.len()); + dims_and_strides.builder_arg(&mut builder); + builder.arg(lhs); + builder.arg(rhs); + builder.arg(&out); + // SAFETY: ffi + unsafe { builder.launch(cfg) }.w()?; + Ok(S::U8(out)) + } +} + +fn slice_src_and_dst<'a, T>( + src: &'a CudaSlice, + src_l: &Layout, + dst: &'a mut CudaSlice, + dst_offset: usize, +) -> ( + cudarc::driver::CudaView<'a, T>, + cudarc::driver::CudaViewMut<'a, T>, +) { + let src_offset = src_l.start_offset(); + let to_copy = dst + .len() + .saturating_sub(dst_offset) + .min(src.len().saturating_sub(src_offset)); + let src = src.slice(src_offset..src_offset + to_copy); + let dst = dst.slice_mut(dst_offset..dst_offset + to_copy); + (src, dst) +} + +#[derive(Debug)] +pub struct CudaStorage { + pub slice: CudaStorageSlice, + pub device: CudaDevice, +} + +pub trait CudaDType: Sized { + fn as_cuda_slice(s: &CudaStorage) -> Result<&CudaSlice>; + fn as_cuda_slice_mut(s: &mut CudaStorage) -> Result<&mut CudaSlice>; + fn wrap_cuda_slice(s: CudaSlice, dev: CudaDevice) -> CudaStorage; +} + +macro_rules! cuda_dtype { + ($ty:ty, $dtype:ident) => { + impl CudaDType for $ty { + fn as_cuda_slice(s: &CudaStorage) -> Result<&CudaSlice> { + match &s.slice { + CudaStorageSlice::$dtype(data) => Ok(&data), + _ => Err(crate::Error::UnexpectedDType { + expected: DType::$dtype, + got: s.dtype(), + msg: "unexpected dtype", + } + .bt()), + } + } + + fn as_cuda_slice_mut(s: &mut CudaStorage) -> Result<&mut CudaSlice> { + match s.slice { + CudaStorageSlice::$dtype(ref mut data) => Ok(data), + _ => Err(crate::Error::UnexpectedDType { + expected: DType::$dtype, + got: s.dtype(), + msg: "unexpected dtype", + } + .bt()), + } + } + + fn wrap_cuda_slice(slice: CudaSlice, device: CudaDevice) -> CudaStorage { + let slice = CudaStorageSlice::$dtype(slice); + CudaStorage { slice, device } + } + } + }; +} +cuda_dtype!(u8, U8); +cuda_dtype!(u32, U32); +cuda_dtype!(i16, I16); +cuda_dtype!(i32, I32); +cuda_dtype!(i64, I64); +cuda_dtype!(f16, F16); +cuda_dtype!(bf16, BF16); +cuda_dtype!(f32, F32); +cuda_dtype!(f64, F64); +cuda_dtype!(float8::F8E4M3, F8E4M3); + +impl CudaStorage { + pub fn wrap_cuda_slice(slice: CudaSlice, device: CudaDevice) -> CudaStorage { + T::wrap_cuda_slice(slice, device) + } + + pub fn as_cuda_slice(&self) -> Result<&CudaSlice> { + T::as_cuda_slice(self) + } + + pub fn as_cuda_slice_mut(&mut self) -> Result<&mut CudaSlice> { + T::as_cuda_slice_mut(self) + } + + pub fn transfer_to_device(&self, dst: &CudaDevice) -> Result { + let dst_stream = dst.cuda_stream(); + let storage_slice = match self.dtype() { + DType::U8 => { + let cuda_slice = self.as_cuda_slice::()?; + let result = dst_stream.clone_dtod(cuda_slice).w()?; + CudaStorageSlice::U8(result) + } + DType::U32 => { + let cuda_slice = self.as_cuda_slice::()?; + let result = dst_stream.clone_dtod(cuda_slice).w()?; + CudaStorageSlice::U32(result) + } + DType::I16 => { + let cuda_slice = self.as_cuda_slice::()?; + let result = dst_stream.clone_dtod(cuda_slice).w()?; + CudaStorageSlice::I16(result) + } + DType::I32 => { + let cuda_slice = self.as_cuda_slice::()?; + let result = dst_stream.clone_dtod(cuda_slice).w()?; + CudaStorageSlice::I32(result) + } + DType::I64 => { + let cuda_slice = self.as_cuda_slice::()?; + let result = dst_stream.clone_dtod(cuda_slice).w()?; + CudaStorageSlice::I64(result) + } + DType::BF16 => { + let cuda_slice = self.as_cuda_slice::()?; + let result = dst_stream.clone_dtod(cuda_slice).w()?; + CudaStorageSlice::BF16(result) + } + DType::F16 => { + let cuda_slice = self.as_cuda_slice::()?; + let result = dst_stream.clone_dtod(cuda_slice).w()?; + CudaStorageSlice::F16(result) + } + DType::F32 => { + let cuda_slice = self.as_cuda_slice::()?; + let result = dst_stream.clone_dtod(cuda_slice).w()?; + CudaStorageSlice::F32(result) + } + DType::F64 => { + let cuda_slice = self.as_cuda_slice::()?; + let result = dst_stream.clone_dtod(cuda_slice).w()?; + CudaStorageSlice::F64(result) + } + DType::F8E4M3 => { + let cuda_slice = self.as_cuda_slice::()?; + let result = dst_stream.clone_dtod(cuda_slice).w()?; + CudaStorageSlice::F8E4M3(result) + } + DType::F6E2M3 => { + let cuda_slice = self.as_cuda_slice::()?; + let result = dst_stream.clone_dtod(cuda_slice).w()?; + CudaStorageSlice::F6E2M3(result) + } + DType::F6E3M2 => { + let cuda_slice = self.as_cuda_slice::()?; + let result = dst_stream.clone_dtod(cuda_slice).w()?; + CudaStorageSlice::F6E3M2(result) + } + DType::F4 => { + let cuda_slice = self.as_cuda_slice::()?; + let result = dst_stream.clone_dtod(cuda_slice).w()?; + CudaStorageSlice::F4(result) + } + DType::F8E8M0 => { + let cuda_slice = self.as_cuda_slice::()?; + let result = dst_stream.clone_dtod(cuda_slice).w()?; + CudaStorageSlice::F8E8M0(result) + } + }; + + Ok(Self { + slice: storage_slice, + device: dst.clone(), + }) + } +} + +fn gemm_config( + alpha: T, + beta: T, + (b, m, n, k): (usize, usize, usize, usize), + lhs_l: &Layout, + rhs_l: &Layout, +) -> Result> { + // https://docs.nvidia.com/cuda/cublas/index.html#cublas-t-gemm + use cudarc::cublas::sys::cublasOperation_t; + + let lhs_stride = lhs_l.stride(); + let rhs_stride = rhs_l.stride(); + let rhs_m1 = rhs_stride[rhs_stride.len() - 1]; + let rhs_m2 = rhs_stride[rhs_stride.len() - 2]; + let lhs_m1 = lhs_stride[lhs_stride.len() - 1]; + let lhs_m2 = lhs_stride[lhs_stride.len() - 2]; + // The a tensor has dims batching, k, n (rhs) + // We also allow for the case where the stride on the minor dimension is not as expected but + // there is a single element. + let (lda, transa) = if (rhs_m1 == 1 || n == 1) && (rhs_m2 == n || k == 1) { + (n as i32, cublasOperation_t::CUBLAS_OP_N) + } else if (rhs_m1 == k || n == 1) && (rhs_m2 == 1 || k == 1) { + (k as i32, cublasOperation_t::CUBLAS_OP_T) + } else { + Err(CudaError::MatMulNonContiguous { + lhs_stride: lhs_l.clone(), + rhs_stride: rhs_l.clone(), + mnk: (m, n, k), + })? + }; + // The b tensor has dims batching, m, k (lhs) + // We also allow for the case where the stride on the minor dimension is not as expected but + // there is a single element. + let (ldb, transb) = if (lhs_m1 == 1 || k == 1) && (lhs_m2 == k || m == 1) { + (k as i32, cublasOperation_t::CUBLAS_OP_N) + } else if (lhs_m1 == m || k == 1) && (lhs_m2 == 1 || m == 1) { + (m as i32, cublasOperation_t::CUBLAS_OP_T) + } else { + Err(CudaError::MatMulNonContiguous { + lhs_stride: lhs_l.clone(), + rhs_stride: rhs_l.clone(), + mnk: (m, n, k), + })? + }; + // The setup below was copied from: + // https://github.com/lebedov/scikit-cuda/blob/7e7300474286019c917a6c8a4bca59405c64fbce/tests/test_cublas.py#L531 + let gemm = GemmConfig { + alpha, + beta, + m: n as i32, + n: m as i32, + k: k as i32, + lda, + ldb, + ldc: n as i32, + transa, + transb, + }; + + let stride_b: usize = match lhs_stride[..lhs_stride.len() - 2] { + [s1, stride] if s1 == stride * lhs_l.dims()[1] => stride, + [_, stride] if lhs_l.dims()[0] == 1 => stride, + [stride, _] if lhs_l.dims()[1] == 1 => stride, + [stride] => stride, + [] => m * k, + _ => Err(CudaError::MatMulNonContiguous { + lhs_stride: lhs_l.clone(), + rhs_stride: rhs_l.clone(), + mnk: (m, n, k), + })?, + }; + let stride_a: usize = match rhs_stride[..rhs_stride.len() - 2] { + [s1, stride] if s1 == stride * rhs_l.dims()[1] => stride, + [_, stride] if rhs_l.dims()[0] == 1 => stride, + [stride, _] if rhs_l.dims()[1] == 1 => stride, + [stride] => stride, + [] => n * k, + _ => Err(CudaError::MatMulNonContiguous { + lhs_stride: lhs_l.clone(), + rhs_stride: rhs_l.clone(), + mnk: (m, n, k), + })?, + }; + Ok(StridedBatchedConfig { + batch_size: b as i32, + gemm, + stride_a: stride_a as i64, + stride_b: stride_b as i64, + stride_c: (m * n) as i64, + }) +} + +impl BackendStorage for CudaStorage { + type Device = CudaDevice; + + fn try_clone(&self, layout: &Layout) -> Result { + let slice = Clone.map(&self.slice, self.device(), layout)?; + let device = self.device.clone(); + Ok(Self { slice, device }) + } + + fn dtype(&self) -> DType { + match self.slice { + CudaStorageSlice::U8(_) => DType::U8, + CudaStorageSlice::U32(_) => DType::U32, + CudaStorageSlice::I16(_) => DType::I16, + CudaStorageSlice::I32(_) => DType::I32, + CudaStorageSlice::I64(_) => DType::I64, + CudaStorageSlice::BF16(_) => DType::BF16, + CudaStorageSlice::F16(_) => DType::F16, + CudaStorageSlice::F32(_) => DType::F32, + CudaStorageSlice::F64(_) => DType::F64, + CudaStorageSlice::F8E4M3(_) => DType::F8E4M3, + CudaStorageSlice::F6E2M3(_) => DType::F6E2M3, + CudaStorageSlice::F6E3M2(_) => DType::F6E3M2, + CudaStorageSlice::F4(_) => DType::F4, + CudaStorageSlice::F8E8M0(_) => DType::F8E8M0, + } + } + + fn device(&self) -> &CudaDevice { + &self.device + } + + fn const_set(&mut self, s: crate::scalar::Scalar, layout: &Layout) -> Result<()> { + let dev = &self.device; + let shape = layout.shape(); + let dims = shape.dims(); + let el_count = shape.elem_count(); + let cfg = LaunchConfig::for_num_elems(el_count as u32); + let ds = SlicePtrOrNull::params_from_layout(dev, layout)?; + let src_o = layout.start_offset(); + let ((src, _guard_src), kernel_name) = match &mut self.slice { + S::U8(s) => (slice_ptr(s, src_o), "const_set_u8"), + S::U32(s) => (slice_ptr(s, src_o), "const_set_u32"), + S::I16(s) => (slice_ptr(s, src_o), "const_set_i16"), + S::I32(s) => (slice_ptr(s, src_o), "const_set_i32"), + S::I64(s) => (slice_ptr(s, src_o), "const_set_i64"), + S::BF16(s) => (slice_ptr(s, src_o), "const_set_bf16"), + S::F16(s) => (slice_ptr(s, src_o), "const_set_f16"), + S::F32(s) => (slice_ptr(s, src_o), "const_set_f32"), + S::F64(s) => (slice_ptr(s, src_o), "const_set_f64"), + S::F8E4M3(s) => (slice_ptr(s, src_o), "const_set_f8_e4m3"), + S::F4(_) | S::F6E2M3(_) | S::F6E3M2(_) | S::F8E8M0(_) => { + return Err(CudaError::UnsupportedDtype { + dtype: self.dtype(), + op: "const_set", + } + .into()); + } + }; + + let func = dev.get_or_load_func(kernel_name, &kernels::FILL)?; + let mut builder = func.builder(); + barg!(builder, el_count); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + s.builder_arg(&mut builder); + barg!(builder, src); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + Ok(()) + } + + fn to_dtype(&self, layout: &Layout, dtype: DType) -> Result { + let shape = layout.shape(); + let dims = shape.dims(); + let el = shape.elem_count(); + let cfg = LaunchConfig::for_num_elems(el as u32); + let dev = self.device(); + let ds = SlicePtrOrNull::params_from_layout(dev, layout)?; + let start_o = layout.start_offset(); + // This returns an i64 rather than a &i64, this is useful to get around some temporary + // lifetime issue and is safe as long as self.slice does not go out of scope before inp + // is used. + let (inp, _guard) = match &self.slice { + CudaStorageSlice::U8(inp) => slice_ptr(inp, start_o), + CudaStorageSlice::U32(inp) => slice_ptr(inp, start_o), + CudaStorageSlice::I16(inp) => slice_ptr(inp, start_o), + CudaStorageSlice::I32(inp) => slice_ptr(inp, start_o), + CudaStorageSlice::I64(inp) => slice_ptr(inp, start_o), + CudaStorageSlice::BF16(inp) => slice_ptr(inp, start_o), + CudaStorageSlice::F16(inp) => slice_ptr(inp, start_o), + CudaStorageSlice::F32(inp) => slice_ptr(inp, start_o), + CudaStorageSlice::F64(inp) => slice_ptr(inp, start_o), + CudaStorageSlice::F8E4M3(inp) => slice_ptr(inp, start_o), + CudaStorageSlice::F4(_) + | CudaStorageSlice::F6E2M3(_) + | CudaStorageSlice::F6E3M2(_) + | CudaStorageSlice::F8E8M0(_) => { + return Err(CudaError::UnsupportedDtype { + dtype: self.dtype(), + op: "to_dtype", + } + .into()); + } + }; + let inp = &inp; + + let kernel_name = format!("cast_{}_{}", self.dtype().as_str(), dtype.as_str()); + let func = dev.get_or_load_func(&kernel_name, &kernels::CAST)?; + let slice = match dtype { + DType::U8 => { + let out = unsafe { dev.alloc::(el)? }; + let mut builder = func.builder(); + barg!(builder, el); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + barg!(builder, *inp); + builder.arg(&out); + unsafe { builder.launch(cfg) }.w()?; + CudaStorageSlice::U8(out) + } + DType::U32 => { + let out = unsafe { dev.alloc::(el)? }; + let mut builder = func.builder(); + barg!(builder, el); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + barg!(builder, *inp); + builder.arg(&out); + unsafe { builder.launch(cfg) }.w()?; + CudaStorageSlice::U32(out) + } + DType::I64 => { + let out = unsafe { dev.alloc::(el)? }; + let mut builder = func.builder(); + barg!(builder, el); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + barg!(builder, *inp); + builder.arg(&out); + unsafe { builder.launch(cfg) }.w()?; + CudaStorageSlice::I64(out) + } + DType::BF16 => { + let out = unsafe { dev.alloc::(el)? }; + let mut builder = func.builder(); + barg!(builder, el); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + barg!(builder, *inp); + builder.arg(&out); + unsafe { builder.launch(cfg) }.w()?; + CudaStorageSlice::BF16(out) + } + DType::F16 => { + let out = unsafe { dev.alloc::(el)? }; + let mut builder = func.builder(); + barg!(builder, el); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + barg!(builder, *inp); + builder.arg(&out); + unsafe { builder.launch(cfg) }.w()?; + CudaStorageSlice::F16(out) + } + DType::F32 => { + let out = unsafe { dev.alloc::(el)? }; + let mut builder = func.builder(); + barg!(builder, el); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + barg!(builder, *inp); + builder.arg(&out); + unsafe { builder.launch(cfg) }.w()?; + CudaStorageSlice::F32(out) + } + DType::F64 => { + let out = unsafe { dev.alloc::(el)? }; + let mut builder = func.builder(); + barg!(builder, el); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + barg!(builder, *inp); + builder.arg(&out); + unsafe { builder.launch(cfg) }.w()?; + CudaStorageSlice::F64(out) + } + DType::F8E4M3 => { + let out = unsafe { dev.alloc::(el)? }; + let mut builder = func.builder(); + barg!(builder, el); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + barg!(builder, *inp); + builder.arg(&out); + unsafe { builder.launch(cfg) }.w()?; + CudaStorageSlice::F8E4M3(out) + } + DType::I16 | DType::I32 => { + return Err(CudaError::InternalError("i16,i32 dtypes are not supported").into()) + } + DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { + return Err( + CudaError::InternalError("Dummy types not supported in CUDA backend").into(), + ) + } + }; + Ok(Self { + slice, + device: dev.clone(), + }) + } + + fn affine(&self, layout: &Layout, mul: f64, add: f64) -> Result { + let device = self.device().clone(); + let slice = Affine(mul, add).map(&self.slice, &device, layout)?; + Ok(Self { slice, device }) + } + + fn powf(&self, layout: &Layout, e: f64) -> Result { + let device = self.device().clone(); + let slice = Powf(e).map(&self.slice, &device, layout)?; + Ok(Self { slice, device }) + } + + fn elu(&self, layout: &Layout, alpha: f64) -> Result { + let device = self.device().clone(); + let slice = Elu(alpha).map(&self.slice, &device, layout)?; + Ok(Self { slice, device }) + } + + fn reduce_op(&self, op: ReduceOp, layout: &Layout, sum_dims: &[usize]) -> Result { + let device = self.device().clone(); + let slice = FastReduce(sum_dims, op).map(&self.slice, &device, layout)?; + Ok(Self { slice, device }) + } + + fn cmp(&self, op: CmpOp, rhs: &Self, lhs_l: &Layout, rhs_l: &Layout) -> Result { + let device = self.device().clone(); + let slice = Cmp(op).map(&self.slice, lhs_l, &rhs.slice, rhs_l, &device)?; + Ok(Self { slice, device }) + } + + fn unary_impl(&self, layout: &Layout) -> Result { + let device = self.device().clone(); + let slice = U::V.map(&self.slice, &device, layout)?; + Ok(Self { slice, device }) + } + + fn binary_impl( + &self, + rhs: &Self, + lhs_l: &Layout, + rhs_l: &Layout, + ) -> Result { + let device = self.device().clone(); + let slice = B::V.map(&self.slice, lhs_l, &rhs.slice, rhs_l, &device)?; + Ok(Self { slice, device }) + } + + fn to_cpu_storage(&self) -> Result { + match &self.slice { + CudaStorageSlice::U8(slice) => { + let cpu_storage = slice.stream().clone_dtoh(slice).w()?; + Ok(CpuStorage::U8(cpu_storage)) + } + CudaStorageSlice::U32(slice) => { + let cpu_storage = slice.stream().clone_dtoh(slice).w()?; + Ok(CpuStorage::U32(cpu_storage)) + } + CudaStorageSlice::I16(slice) => { + let cpu_storage = slice.stream().clone_dtoh(slice).w()?; + Ok(CpuStorage::I16(cpu_storage)) + } + CudaStorageSlice::I32(slice) => { + let cpu_storage = slice.stream().clone_dtoh(slice).w()?; + Ok(CpuStorage::I32(cpu_storage)) + } + CudaStorageSlice::I64(slice) => { + let cpu_storage = slice.stream().clone_dtoh(slice).w()?; + Ok(CpuStorage::I64(cpu_storage)) + } + CudaStorageSlice::BF16(slice) => { + let cpu_storage = slice.stream().clone_dtoh(slice).w()?; + Ok(CpuStorage::BF16(cpu_storage)) + } + CudaStorageSlice::F16(slice) => { + let cpu_storage = slice.stream().clone_dtoh(slice).w()?; + Ok(CpuStorage::F16(cpu_storage)) + } + CudaStorageSlice::F32(slice) => { + let cpu_storage = slice.stream().clone_dtoh(slice).w()?; + Ok(CpuStorage::F32(cpu_storage)) + } + CudaStorageSlice::F64(slice) => { + let cpu_storage = slice.stream().clone_dtoh(slice).w()?; + Ok(CpuStorage::F64(cpu_storage)) + } + CudaStorageSlice::F8E4M3(slice) => { + let cpu_storage = slice.stream().clone_dtoh(slice).w()?; + Ok(CpuStorage::F8E4M3(cpu_storage)) + } + CudaStorageSlice::F4(_) + | CudaStorageSlice::F6E2M3(_) + | CudaStorageSlice::F6E3M2(_) + | CudaStorageSlice::F8E8M0(_) => Err(CudaError::UnsupportedDtype { + dtype: self.dtype(), + op: "to_cpu_storage", + } + .into()), + } + } + + fn where_cond( + &self, + layout: &Layout, + t: &Self, + t_l: &Layout, + f: &Self, + f_l: &Layout, + ) -> Result { + let device = self.device().clone(); + let slice = WhereCond(self, layout).map(&t.slice, t_l, &f.slice, f_l, &device)?; + Ok(Self { slice, device }) + } + + #[cfg(not(feature = "cudnn"))] + fn conv1d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConv1D, + ) -> Result { + const USE_IM2COL_CONV1D: bool = true; + + let device = self.device().clone(); + if !USE_IM2COL_CONV1D { + let slice = Conv1D(params).map(&self.slice, l, &kernel.slice, kernel_l, &device)?; + return Ok(Self { slice, device }); + } + + let col = Im2Col1D { + l_k: params.k_size, + stride: params.stride, + dilation: params.dilation, + padding: params.padding, + } + .map(&self.slice, &device, l)?; + let col = Self { slice: col, device }; + let l_out = params.l_out(); + let b = params.b_size; + let n = params.c_out; + let k = params.k_size * params.c_in; + let m = l_out; + let col_l = Layout::contiguous((b * m, k)); + let res = if kernel_l.is_contiguous() { + let kernel_l = + Layout::contiguous_with_offset((n, k), kernel_l.start_offset()).transpose(0, 1)?; + col.matmul(kernel, (1, b * m, n, k), &col_l, &kernel_l)? + } else { + // Make the kernel contiguous if not already the case. + let mut kernel_c = unsafe { + self.device() + .alloc_uninit(kernel_l.shape(), kernel.dtype())? + }; + kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?; + let kernel_l = + Layout::contiguous_with_offset((n, k), kernel_l.start_offset()).transpose(0, 1)?; + col.matmul(kernel, (1, b * m, n, k), &col_l, &kernel_l)? + }; + let res_l = Layout::contiguous((b, l_out, n)).transpose(1, 2)?; + let mut res_t = unsafe { self.device().alloc_uninit(res_l.shape(), res.dtype())? }; + res.copy_strided_src(&mut res_t, 0, &res_l)?; + Ok(res_t) + } + + #[cfg(feature = "cudnn")] + fn conv1d( + &self, + inp_l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConv1D, + ) -> Result { + let device = self.device().clone(); + if !kernel_l.is_contiguous() { + let slice = Conv1D(params).map(&self.slice, inp_l, &kernel.slice, kernel_l, &device)?; + return Ok(Self { slice, device }); + } + let l_out = params.l_out(); + let dst_el = params.c_out * l_out * params.b_size; + let slice = match (&self.slice, &kernel.slice) { + (S::U8(inp), S::U8(k)) => { + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(kernel_l.start_offset()..); + let mut out = unsafe { device.alloc::(dst_el)? }; + crate::cudnn::launch_conv1d::(inp, inp_l, k, &mut out, params, &device) + .map_err(crate::Error::wrap)?; + S::U8(out) + } + (S::BF16(inp), S::BF16(k)) => { + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(kernel_l.start_offset()..); + let mut out = unsafe { device.alloc::(dst_el)? }; + // Only PSEUDO_BFLOAT16_CONFIG is supported in cudnn, there is no "true bfloat16" + // version. + // https://docs.nvidia.com/deeplearning/cudnn/latest/api/cudnn-cnn-library.html#id88 + crate::cudnn::launch_conv1d::(inp, inp_l, k, &mut out, params, &device) + .map_err(crate::Error::wrap)?; + S::BF16(out) + } + (S::F16(inp), S::F16(k)) => { + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(kernel_l.start_offset()..); + let mut out = unsafe { device.alloc::(dst_el)? }; + crate::cudnn::launch_conv1d::(inp, inp_l, k, &mut out, params, &device) + .map_err(crate::Error::wrap)?; + S::F16(out) + } + (S::F32(inp), S::F32(k)) => { + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(kernel_l.start_offset()..); + let mut out = unsafe { device.alloc::(dst_el)? }; + crate::cudnn::launch_conv1d::(inp, inp_l, k, &mut out, params, &device) + .map_err(crate::Error::wrap)?; + S::F32(out) + } + (S::F64(inp), S::F64(k)) => { + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(kernel_l.start_offset()..); + let mut out = unsafe { device.alloc::(dst_el)? }; + crate::cudnn::launch_conv1d::(inp, inp_l, k, &mut out, params, &device) + .map_err(crate::Error::wrap)?; + S::F64(out) + } + (S::U32(_), S::U32(_)) => Err(CudaError::InternalError("conv1d does not support u32"))?, + (S::I16(_), S::I16(_)) => Err(CudaError::InternalError("conv1d does not support i16"))?, + (S::I32(_), S::I32(_)) => Err(CudaError::InternalError("conv1d does not support i32"))?, + (S::I64(_), S::I64(_)) => Err(CudaError::InternalError("conv1d does not support i64"))?, + (S::F8E4M3(_), S::F8E4M3(_)) => { + Err(CudaError::InternalError("conv1d does not support f8e4m3"))? + } + _ => Err(CudaError::InternalError("dtype mismatch in conv1d"))?, + }; + Ok(Self { slice, device }) + } + + fn conv_transpose1d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConvTranspose1D, + ) -> Result { + const USE_COL2IM_CONV1D_TR: bool = true; + + let device = self.device().clone(); + let can_use_col2im = kernel_l.is_contiguous() + && params.dilation == 1 + && params.padding == 0 + && params.output_padding == 0; + let slice = if USE_COL2IM_CONV1D_TR && can_use_col2im { + let (b_size, c_in, l_in) = l.shape().dims3()?; + let (c_in2, c_out, k_size) = kernel_l.shape().dims3()?; + if !kernel_l.is_contiguous() { + crate::bail!( + "convtr1d: the second argument (kernel) has to be contiguous {kernel_l:?}" + ) + } + if c_in != c_in2 { + crate::bail!( + "convtr1d: shape mismatch on c_in {:?} {:?}", + l.shape(), + kernel_l.shape() + ) + } + let col = { + // This merges the last two dimensions of the kernel together. + let kernel_l_mm = Layout::new( + (b_size, c_in, k_size * c_out).into(), + vec![0, k_size * c_out, 1], + kernel_l.start_offset(), + ); + self.matmul( + kernel, + ( + b_size, + /* m */ l_in, + /* n */ c_out * k_size, + /* k */ c_in, + ), + &l.transpose(1, 2)?, + &kernel_l_mm, + )? + }; + let col_l = Layout::contiguous((b_size, l_in, c_out, k_size)); + Col2Im1D { + stride: params.stride, + } + .map(&col.slice, &device, &col_l)? + } else { + ConvTranspose1D(params).map(&self.slice, l, &kernel.slice, kernel_l, &device)? + }; + Ok(Self { slice, device }) + } + + #[cfg(not(feature = "cudnn"))] + fn conv2d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConv2D, + ) -> Result { + const USE_IM2COL_CONV2D: bool = true; + + let device = self.device().clone(); + if !USE_IM2COL_CONV2D { + let slice = Conv2D(params).map(&self.slice, l, &kernel.slice, kernel_l, &device)?; + return Ok(Self { slice, device }); + } + + let col = Im2Col { + h_k: params.k_h, + w_k: params.k_w, + stride: params.stride, + dilation: params.dilation, + padding: params.padding, + } + .map(&self.slice, &device, l)?; + let col = Self { slice: col, device }; + let h_out = params.out_h(); + let w_out = params.out_w(); + let b = params.b_size; + let n = params.c_out; + let k = params.k_h * params.k_w * params.c_in; + let m = h_out * w_out; + let col_l = Layout::contiguous((b * m, k)); + let res = if kernel_l.is_contiguous() { + let kernel_l = + Layout::contiguous_with_offset((n, k), kernel_l.start_offset()).transpose(0, 1)?; + col.matmul(kernel, (1, b * m, n, k), &col_l, &kernel_l)? + } else { + // Make the kernel contiguous if not already the case. + let mut kernel_c = unsafe { + self.device() + .alloc_uninit(kernel_l.shape(), kernel.dtype())? + }; + kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?; + let kernel_l = + Layout::contiguous_with_offset((n, k), kernel_l.start_offset()).transpose(0, 1)?; + col.matmul(kernel, (1, b * m, n, k), &col_l, &kernel_l)? + }; + let res_l = Layout::contiguous((b, h_out, w_out, n)) + .transpose(1, 2)? + .transpose(1, 3)?; + let mut res_t = unsafe { self.device().alloc_uninit(res_l.shape(), res.dtype())? }; + res.copy_strided_src(&mut res_t, 0, &res_l)?; + Ok(res_t) + } + + #[cfg(feature = "cudnn")] + fn conv2d( + &self, + inp_l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConv2D, + ) -> Result { + let device = self.device().clone(); + if !kernel_l.is_contiguous() { + let slice = Conv2D(params).map(&self.slice, inp_l, &kernel.slice, kernel_l, &device)?; + return Ok(Self { slice, device }); + } + let (out_w, out_h) = (params.out_w(), params.out_h()); + let dst_el = params.c_out * out_w * out_h * params.b_size; + let slice = match (&self.slice, &kernel.slice) { + (S::U8(inp), S::U8(k)) => { + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(kernel_l.start_offset()..); + let mut out = unsafe { device.alloc::(dst_el)? }; + crate::cudnn::launch_conv2d::(inp, inp_l, k, &mut out, params, &device) + .map_err(crate::Error::wrap)?; + S::U8(out) + } + (S::BF16(inp), S::BF16(k)) => { + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(kernel_l.start_offset()..); + let mut out = unsafe { device.alloc::(dst_el)? }; + // Only PSEUDO_BFLOAT16_CONFIG is supported in cudnn, there is no "true bfloat16" + // version. + // https://docs.nvidia.com/deeplearning/cudnn/latest/api/cudnn-cnn-library.html#id88 + crate::cudnn::launch_conv2d::(inp, inp_l, k, &mut out, params, &device) + .map_err(crate::Error::wrap)?; + S::BF16(out) + } + (S::F16(inp), S::F16(k)) => { + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(kernel_l.start_offset()..); + let mut out = unsafe { device.alloc::(dst_el)? }; + crate::cudnn::launch_conv2d::(inp, inp_l, k, &mut out, params, &device) + .map_err(crate::Error::wrap)?; + S::F16(out) + } + (S::F32(inp), S::F32(k)) => { + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(kernel_l.start_offset()..); + let mut out = unsafe { device.alloc::(dst_el)? }; + crate::cudnn::launch_conv2d::(inp, inp_l, k, &mut out, params, &device) + .map_err(crate::Error::wrap)?; + S::F32(out) + } + (S::F64(inp), S::F64(k)) => { + let inp = &inp.slice(inp_l.start_offset()..); + let k = &k.slice(kernel_l.start_offset()..); + let mut out = unsafe { device.alloc::(dst_el)? }; + crate::cudnn::launch_conv2d::(inp, inp_l, k, &mut out, params, &device) + .map_err(crate::Error::wrap)?; + S::F64(out) + } + (S::U32(_), S::U32(_)) => Err(CudaError::InternalError("conv2d does not support u32"))?, + (S::I16(_), S::I16(_)) => Err(CudaError::InternalError("conv2d does not support i16"))?, + (S::I32(_), S::I32(_)) => Err(CudaError::InternalError("conv2d does not support i32"))?, + (S::I64(_), S::I64(_)) => Err(CudaError::InternalError("conv2d does not support i64"))?, + (S::F8E4M3(_), S::F8E4M3(_)) => { + Err(CudaError::InternalError("conv2d does not support f8e4m3"))? + } + _ => Err(CudaError::InternalError("dtype mismatch in conv2d"))?, + }; + Ok(Self { slice, device }) + } + + fn conv_transpose2d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConvTranspose2D, + ) -> Result { + let device = self.device().clone(); + let slice = + ConvTranspose2D(params).map(&self.slice, l, &kernel.slice, kernel_l, &device)?; + Ok(Self { slice, device }) + } + + fn avg_pool2d(&self, l: &Layout, k: (usize, usize), stride: (usize, usize)) -> Result { + let device = self.device().clone(); + let slice = Pool2D { + w_k: k.0, + h_k: k.1, + w_stride: stride.0, + h_stride: stride.1, + op: PoolOp::Avg, + } + .map(&self.slice, &device, l)?; + Ok(Self { slice, device }) + } + + fn max_pool2d(&self, l: &Layout, k: (usize, usize), stride: (usize, usize)) -> Result { + let device = self.device().clone(); + let slice = Pool2D { + w_k: k.0, + h_k: k.1, + w_stride: stride.0, + h_stride: stride.1, + op: PoolOp::Max, + } + .map(&self.slice, &device, l)?; + Ok(Self { slice, device }) + } + + fn upsample_nearest1d(&self, _: &Layout, _out_sz: usize) -> Result { + crate::bail!("upsample-nearest1d is not supported on cuda") + } + + fn upsample_nearest2d(&self, l: &Layout, out_w: usize, out_h: usize) -> Result { + let device = self.device().clone(); + let slice = UpsampleNearest2D(out_w, out_h).map(&self.slice, &device, l)?; + Ok(Self { slice, device }) + } + + fn upsample_bilinear2d( + &self, + l: &Layout, + out_h: usize, + out_w: usize, + align_corners: bool, + scale_h: Option, + scale_w: Option, + ) -> Result { + let device = self.device().clone(); + let slice = UpsampleBilinear2D { + out_w, + out_h, + align_corners, + scale_h_factor: scale_h, + scale_w_factor: scale_w, + } + .map(&self.slice, &device, l)?; + Ok(Self { slice, device }) + } + + fn index_select(&self, ids: &Self, l: &Layout, ids_l: &Layout, dim: usize) -> Result { + let device = self.device().clone(); + let slice = IndexSelect(ids, ids_l, dim).map(&self.slice, &device, l)?; + Ok(Self { slice, device }) + } + fn gather(&self, l: &Layout, ids: &Self, ids_l: &Layout, dim: usize) -> Result { + let device = self.device().clone(); + let slice = Gather(ids, ids_l, dim).map(&self.slice, &device, l)?; + Ok(Self { slice, device }) + } + fn scatter_set( + &mut self, + l: &Layout, + ids: &Self, + ids_l: &Layout, + src: &Self, + src_l: &Layout, + dim: usize, + ) -> Result<()> { + let device = self.device().clone(); + Scatter(ids, ids_l, dim).map(&mut self.slice, l, &src.slice, src_l, &device) + } + fn scatter_add_set( + &mut self, + l: &Layout, + ids: &Self, + ids_l: &Layout, + src: &Self, + src_l: &Layout, + dim: usize, + ) -> Result<()> { + let device = self.device().clone(); + ScatterAdd(ids, ids_l, dim).map(&mut self.slice, l, &src.slice, src_l, &device) + } + fn index_add( + &self, + l: &Layout, + ids: &Self, + ids_l: &Layout, + src: &Self, + src_l: &Layout, + dim: usize, + ) -> Result { + let device = self.device().clone(); + let mut acc = unsafe { device.alloc_uninit(l.shape(), self.dtype())? }; + self.copy_strided_src(&mut acc, 0, l)?; + IndexAdd(ids, ids_l, dim).map(&mut acc.slice, l, &src.slice, src_l, &device)?; + Ok(acc) + } + + fn matmul( + &self, + rhs: &Self, + (b, m, n, k): (usize, usize, usize, usize), + lhs_l: &Layout, + rhs_l: &Layout, + ) -> Result { + let elem_count = b * m * n; + let dev = &self.device; + let slice = match (&self.slice, &rhs.slice) { + (CudaStorageSlice::BF16(lhs), CudaStorageSlice::BF16(rhs)) => { + let lhs = &lhs.slice(lhs_l.start_offset()..); + let rhs = &rhs.slice(rhs_l.start_offset()..); + let cfg = gemm_config(bf16::ONE, bf16::ZERO, (b, m, n, k), lhs_l, rhs_l)?; + let mut out = unsafe { dev.alloc::(elem_count)? }; + unsafe { gemm_strided_batched_bf16(&self.device.blas, cfg, rhs, lhs, &mut out) } + .w()?; + CudaStorageSlice::BF16(out) + } + (CudaStorageSlice::F16(lhs), CudaStorageSlice::F16(rhs)) => { + let lhs = &lhs.slice(lhs_l.start_offset()..); + let rhs = &rhs.slice(rhs_l.start_offset()..); + let cfg = gemm_config(f16::ONE, f16::ZERO, (b, m, n, k), lhs_l, rhs_l)?; + let mut out = unsafe { dev.alloc::(elem_count)? }; + unsafe { gemm_strided_batched_f16(&self.device.blas, cfg, rhs, lhs, &mut out) } + .w()?; + CudaStorageSlice::F16(out) + } + (CudaStorageSlice::F32(lhs), CudaStorageSlice::F32(rhs)) => { + let lhs = &lhs.slice(lhs_l.start_offset()..); + let rhs = &rhs.slice(rhs_l.start_offset()..); + let cfg = gemm_config(1., 0., (b, m, n, k), lhs_l, rhs_l)?; + let mut out = unsafe { dev.alloc::(elem_count)? }; + unsafe { gemm_strided_batched_f32(&self.device.blas, cfg, rhs, lhs, &mut out) } + .w()?; + CudaStorageSlice::F32(out) + } + (CudaStorageSlice::F64(lhs), CudaStorageSlice::F64(rhs)) => { + let lhs = &lhs.slice(lhs_l.start_offset()..); + let rhs = &rhs.slice(rhs_l.start_offset()..); + let cfg = gemm_config(1., 0., (b, m, n, k), lhs_l, rhs_l)?; + let mut out = unsafe { dev.alloc::(elem_count)? }; + unsafe { + self.device + .blas + .gemm_strided_batched(cfg, rhs, lhs, &mut out) + } + .w()?; + CudaStorageSlice::F64(out) + } + _ => Err(CudaError::InternalError("dtype mismatch in matmul op"))?, + }; + let device = dev.clone(); + Ok(Self { slice, device }) + } + + fn copy2d( + &self, + dst: &mut Self, + d1: usize, + d2: usize, + src_s: usize, + dst_s: usize, + src_o: usize, + dst_o: usize, + ) -> Result<()> { + let dev = &self.device; + let d1 = d1 as u32; + let d2 = d2 as u32; + // Nothing to copy so we exit early to avoid launching a kernel and some potential invalid + // argument with a null pointer. + if d1 == 0 || d2 == 0 { + return Ok(()); + } + let dst_s = dst_s as u32; + let src_s = src_s as u32; + let ((src, _guard_src), (dst, _guard_dst), kname) = match (&self.slice, &mut dst.slice) { + (S::U8(s), S::U8(d)) => (slice_ptr(s, src_o), slice_ptr(d, dst_o), "copy2d_u8"), + (S::U32(s), S::U32(d)) => (slice_ptr(s, src_o), slice_ptr(d, dst_o), "copy2d_u32"), + (S::I16(s), S::I16(d)) => (slice_ptr(s, src_o), slice_ptr(d, dst_o), "copy2d_i16"), + (S::I32(s), S::I32(d)) => (slice_ptr(s, src_o), slice_ptr(d, dst_o), "copy2d_i32"), + (S::I64(s), S::I64(d)) => (slice_ptr(s, src_o), slice_ptr(d, dst_o), "copy2d_i64"), + (S::BF16(s), S::BF16(d)) => (slice_ptr(s, src_o), slice_ptr(d, dst_o), "copy2d_bf16"), + (S::F16(s), S::F16(d)) => (slice_ptr(s, src_o), slice_ptr(d, dst_o), "copy2d_f16"), + (S::F32(s), S::F32(d)) => (slice_ptr(s, src_o), slice_ptr(d, dst_o), "copy2d_f32"), + (S::F64(s), S::F64(d)) => (slice_ptr(s, src_o), slice_ptr(d, dst_o), "copy2d_f64"), + (S::F8E4M3(s), S::F8E4M3(d)) => (slice_ptr(s, src_o), slice_ptr(d, dst_o), "copy2d_u8"), + (S::F8E8M0(s), S::F8E8M0(d)) => (slice_ptr(s, src_o), slice_ptr(d, dst_o), "copy2d_u8"), + _ => Err(CudaError::InternalError("dtype mismatch in copy2d"))?, + }; + let func = dev.get_or_load_func(kname, &kernels::FILL)?; + let cfg = LaunchConfig::for_num_elems(d1 * d2); + let mut builder = func.builder(); + barg!(builder, src); + barg!(builder, dst); + barg!(builder, d1); + barg!(builder, d2); + builder.arg(&src_s); + builder.arg(&dst_s); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + Ok(()) + } + + fn copy_strided_src(&self, dst: &mut Self, dst_offset: usize, src_l: &Layout) -> Result<()> { + let src_shape = src_l.shape(); + let dims = src_shape.dims(); + let el_count = src_shape.elem_count(); + if el_count == 0 { + return Ok(()); + } + let cfg = LaunchConfig::for_num_elems(el_count as u32); + let dev = &self.device; + let ds = SlicePtrOrNull::params_from_layout(dev, src_l)?; + match (&self.slice, &mut dst.slice) { + (CudaStorageSlice::BF16(src), CudaStorageSlice::BF16(dst)) => { + let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); + if src_l.is_contiguous() { + dev.memcpy_dtod(&src, &mut dst)? + } else { + let func = dev.get_or_load_func("ucopy_bf16", &kernels::UNARY)?; + let mut builder = func.builder(); + barg!(builder, el_count); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + builder.arg(&src); + builder.arg(&mut dst); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + } + } + (CudaStorageSlice::F16(src), CudaStorageSlice::F16(dst)) => { + let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); + if src_l.is_contiguous() { + dev.memcpy_dtod(&src, &mut dst)? + } else { + let func = dev.get_or_load_func("ucopy_f16", &kernels::UNARY)?; + let mut builder = func.builder(); + barg!(builder, el_count); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + builder.arg(&src); + builder.arg(&mut dst); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + } + } + (CudaStorageSlice::F32(src), CudaStorageSlice::F32(dst)) => { + let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); + if src_l.is_contiguous() { + dev.memcpy_dtod(&src, &mut dst)? + } else { + let func = dev.get_or_load_func("ucopy_f32", &kernels::UNARY)?; + let mut builder = func.builder(); + barg!(builder, el_count); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + builder.arg(&src); + builder.arg(&mut dst); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + } + } + (CudaStorageSlice::U8(src), CudaStorageSlice::U8(dst)) => { + let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); + if src_l.is_contiguous() { + dev.memcpy_dtod(&src, &mut dst)? + } else { + let func = dev.get_or_load_func("ucopy_u8", &kernels::UNARY)?; + let mut builder = func.builder(); + barg!(builder, el_count); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + builder.arg(&src); + builder.arg(&mut dst); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + } + } + (CudaStorageSlice::U32(src), CudaStorageSlice::U32(dst)) => { + let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); + if src_l.is_contiguous() { + dev.memcpy_dtod(&src, &mut dst)? + } else { + let func = dev.get_or_load_func("ucopy_u32", &kernels::UNARY)?; + let mut builder = func.builder(); + barg!(builder, el_count); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + builder.arg(&src); + builder.arg(&mut dst); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + } + } + (CudaStorageSlice::I16(src), CudaStorageSlice::I16(dst)) => { + let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); + if src_l.is_contiguous() { + dev.memcpy_dtod(&src, &mut dst)? + } else { + let func = dev.get_or_load_func("ucopy_i16", &kernels::UNARY)?; + let mut builder = func.builder(); + barg!(builder, el_count); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + builder.arg(&src); + builder.arg(&mut dst); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + } + } + (CudaStorageSlice::I32(src), CudaStorageSlice::I32(dst)) => { + let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); + if src_l.is_contiguous() { + dev.memcpy_dtod(&src, &mut dst)? + } else { + let func = dev.get_or_load_func("ucopy_i32", &kernels::UNARY)?; + let mut builder = func.builder(); + barg!(builder, el_count); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + builder.arg(&src); + builder.arg(&mut dst); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + } + } + (CudaStorageSlice::I64(src), CudaStorageSlice::I64(dst)) => { + let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); + if src_l.is_contiguous() { + dev.memcpy_dtod(&src, &mut dst)? + } else { + let func = dev.get_or_load_func("ucopy_i64", &kernels::UNARY)?; + let mut builder = func.builder(); + barg!(builder, el_count); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + builder.arg(&src); + builder.arg(&mut dst); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + } + } + (CudaStorageSlice::F64(src), CudaStorageSlice::F64(dst)) => { + let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); + if src_l.is_contiguous() { + dev.memcpy_dtod(&src, &mut dst)? + } else { + let func = dev.get_or_load_func("ucopy_f64", &kernels::UNARY)?; + let mut builder = func.builder(); + barg!(builder, el_count); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + builder.arg(&src); + builder.arg(&mut dst); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + } + } + (CudaStorageSlice::F8E4M3(src), CudaStorageSlice::F8E4M3(dst)) => { + let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); + if src_l.is_contiguous() { + dev.memcpy_dtod(&src, &mut dst)? + } else { + let func = dev.get_or_load_func("ucopy_f8e4m3", &kernels::UNARY)?; + let mut builder = func.builder(); + barg!(builder, el_count); + barg!(builder, dims.len()); + ds.builder_arg(&mut builder); + builder.arg(&src); + builder.arg(&mut dst); + // SAFETY: ffi. + unsafe { builder.launch(cfg) }.w()?; + } + } + _ => Err(CudaError::InternalError( + "dtype mismatch in copy_strided op", + ))?, + } + Ok(()) + } +} + +// Default for the reduced precision setting is false, similar to pytorch. +// https://github.com/pytorch/pytorch/issues/123157 +static MM_F16_REDUCED_PRECISION: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); +static MM_BF16_REDUCED_PRECISION: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); +static MM_F32_REDUCED_PRECISION: std::sync::atomic::AtomicBool = + std::sync::atomic::AtomicBool::new(false); + +/// This bool controls whether reduced precision reductions (e.g., with tf32 accumulation type) are +/// allowed with f32 GEMMs. +pub fn gemm_reduced_precision_f32() -> bool { + MM_F32_REDUCED_PRECISION.load(std::sync::atomic::Ordering::Relaxed) +} + +/// This bool controls whether reduced precision reductions (e.g., with tf32 accumulation type) are +/// allowed with f32 GEMMs. +pub fn set_gemm_reduced_precision_f32(b: bool) { + MM_F32_REDUCED_PRECISION.store(b, std::sync::atomic::Ordering::Relaxed) +} + +/// This bool controls whether reduced precision reductions (e.g., with fp16 accumulation type) are +/// allowed with f16 GEMMs. +pub fn gemm_reduced_precision_f16() -> bool { + MM_F16_REDUCED_PRECISION.load(std::sync::atomic::Ordering::Relaxed) +} + +/// This bool controls whether reduced precision reductions (e.g., with fp16 accumulation type) are +/// allowed with f16 GEMMs. +pub fn set_gemm_reduced_precision_f16(b: bool) { + MM_F16_REDUCED_PRECISION.store(b, std::sync::atomic::Ordering::Relaxed) +} + +/// This bool controls whether reduced precision reductions (e.g., with fp16 accumulation type) are +/// allowed with bf16 GEMMs. +pub fn gemm_reduced_precision_bf16() -> bool { + MM_BF16_REDUCED_PRECISION.load(std::sync::atomic::Ordering::Relaxed) +} + +/// This bool controls whether reduced precision reductions (e.g., with fp16 accumulation type) are +/// allowed with bf16 GEMMs. +pub fn set_gemm_reduced_precision_bf16(b: bool) { + MM_BF16_REDUCED_PRECISION.store(b, std::sync::atomic::Ordering::Relaxed) +} + +unsafe fn gemm_strided_batched_f32( + cublas: &cudarc::cublas::CudaBlas, + cfg: StridedBatchedConfig, + a: &cudarc::driver::CudaView, + b: &cudarc::driver::CudaView, + c: &mut CudaSlice, +) -> std::result::Result<(), cudarc::cublas::result::CublasError> { + use cudarc::cublas::sys; + use cudarc::driver::DevicePtrMut; + + let compute_type = if gemm_reduced_precision_f32() { + sys::cublasComputeType_t::CUBLAS_COMPUTE_32F_FAST_TF32 + } else { + sys::cublasComputeType_t::CUBLAS_COMPUTE_32F + }; + let alpha = &cfg.gemm.alpha as *const f32 as *const _; + let beta = &cfg.gemm.beta as *const f32 as *const _; + + let stream = c.stream().clone(); + let (a, _guard_a) = a.device_ptr(&stream); + let (b, _guard_b) = b.device_ptr(&stream); + let (c, _guard_c) = c.device_ptr_mut(&stream); + + cudarc::cublas::result::gemm_strided_batched_ex( + *cublas.handle(), + cfg.gemm.transa, + cfg.gemm.transb, + cfg.gemm.m, + cfg.gemm.n, + cfg.gemm.k, + alpha, + a as *const _, + sys::cudaDataType_t::CUDA_R_32F, + cfg.gemm.lda, + cfg.stride_a, + b as *const _, + sys::cudaDataType_t::CUDA_R_32F, + cfg.gemm.ldb, + cfg.stride_b, + beta, + c as *mut _, + sys::cudaDataType_t::CUDA_R_32F, + cfg.gemm.ldc, + cfg.stride_c, + cfg.batch_size, + compute_type, + sys::cublasGemmAlgo_t::CUBLAS_GEMM_DEFAULT_TENSOR_OP, + ) +} + +unsafe fn gemm_strided_batched_f16( + cublas: &cudarc::cublas::CudaBlas, + cfg: StridedBatchedConfig, + a: &cudarc::driver::CudaView, + b: &cudarc::driver::CudaView, + c: &mut CudaSlice, +) -> std::result::Result<(), cudarc::cublas::result::CublasError> { + use cudarc::cublas::sys; + use cudarc::driver::DevicePtrMut; + + let alpha = cfg.gemm.alpha; + let beta = cfg.gemm.beta; + let alpha_f32: f32 = cfg.gemm.alpha.to_f32(); + let beta_f32: f32 = cfg.gemm.beta.to_f32(); + let (compute_type, alpha, beta) = if gemm_reduced_precision_f16() { + ( + sys::cublasComputeType_t::CUBLAS_COMPUTE_16F, + (&alpha) as *const f16 as *const _, + (&beta) as *const f16 as *const _, + ) + } else { + ( + sys::cublasComputeType_t::CUBLAS_COMPUTE_32F, + (&alpha_f32) as *const f32 as *const _, + (&beta_f32) as *const f32 as *const _, + ) + }; + + let stream = c.stream().clone(); + let (a, _guard_a) = a.device_ptr(&stream); + let (b, _guard_b) = b.device_ptr(&stream); + let (c, _guard_c) = c.device_ptr_mut(&stream); + cudarc::cublas::result::gemm_strided_batched_ex( + *cublas.handle(), + cfg.gemm.transa, + cfg.gemm.transb, + cfg.gemm.m, + cfg.gemm.n, + cfg.gemm.k, + alpha, + a as *const _, + sys::cudaDataType_t::CUDA_R_16F, + cfg.gemm.lda, + cfg.stride_a, + b as *const _, + sys::cudaDataType_t::CUDA_R_16F, + cfg.gemm.ldb, + cfg.stride_b, + beta, + c as *mut _, + sys::cudaDataType_t::CUDA_R_16F, + cfg.gemm.ldc, + cfg.stride_c, + cfg.batch_size, + compute_type, + sys::cublasGemmAlgo_t::CUBLAS_GEMM_DEFAULT_TENSOR_OP, + ) +} + +unsafe fn gemm_strided_batched_bf16( + cublas: &cudarc::cublas::CudaBlas, + cfg: StridedBatchedConfig, + a: &cudarc::driver::CudaView, + b: &cudarc::driver::CudaView, + c: &mut CudaSlice, +) -> std::result::Result<(), cudarc::cublas::result::CublasError> { + use cudarc::cublas::sys; + use cudarc::driver::DevicePtrMut; + + let alpha_f32: f32 = cfg.gemm.alpha.to_f32(); + let beta_f32: f32 = cfg.gemm.beta.to_f32(); + // The type for alpha and beta depends on the computeType. + // https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmstridedbatchedex + let (compute_type, alpha, beta) = if gemm_reduced_precision_bf16() { + ( + sys::cublasComputeType_t::CUBLAS_COMPUTE_32F_FAST_16BF, + (&alpha_f32) as *const f32 as *const _, + (&beta_f32) as *const f32 as *const _, + ) + } else { + ( + sys::cublasComputeType_t::CUBLAS_COMPUTE_32F, + (&alpha_f32) as *const f32 as *const _, + (&beta_f32) as *const f32 as *const _, + ) + }; + + let stream = c.stream().clone(); + let (a, _guard_a) = a.device_ptr(&stream); + let (b, _guard_b) = b.device_ptr(&stream); + let (c, _guard_c) = c.device_ptr_mut(&stream); + cudarc::cublas::result::gemm_strided_batched_ex( + *cublas.handle(), + cfg.gemm.transa, + cfg.gemm.transb, + cfg.gemm.m, + cfg.gemm.n, + cfg.gemm.k, + alpha, + a as *const _, + sys::cudaDataType_t::CUDA_R_16BF, + cfg.gemm.lda, + cfg.stride_a, + b as *const _, + sys::cudaDataType_t::CUDA_R_16BF, + cfg.gemm.ldb, + cfg.stride_b, + beta, + c as *mut _, + sys::cudaDataType_t::CUDA_R_16BF, + cfg.gemm.ldc, + cfg.stride_c, + cfg.batch_size, + compute_type, + sys::cublasGemmAlgo_t::CUBLAS_GEMM_DEFAULT_TENSOR_OP, + ) +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/utils.rs b/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/utils.rs new file mode 100644 index 0000000000..10f8876ab5 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/utils.rs @@ -0,0 +1,194 @@ +/// Helper functions to plug cuda kernels in candle. +use crate::{Layout, Result, WithDType}; +pub use cudarc; +use cudarc::driver::{CudaSlice, DeviceRepr, ValidAsZeroBits}; + +use super::{CudaDevice, CudaError, WrapErr}; + +pub type S = super::CudaStorageSlice; + +pub trait Map1 { + fn f( + &self, + src: &CudaSlice, + dev: &CudaDevice, + layout: &Layout, + ) -> Result>; + + fn map(&self, s: &S, d: &CudaDevice, l: &Layout) -> Result { + let out = match s { + S::U8(s) => S::U8(self.f(s, d, l)?), + S::U32(s) => S::U32(self.f(s, d, l)?), + S::I16(s) => S::I16(self.f(s, d, l)?), + S::I32(s) => S::I32(self.f(s, d, l)?), + S::I64(s) => S::I64(self.f(s, d, l)?), + S::BF16(s) => S::BF16(self.f(s, d, l)?), + S::F16(s) => S::F16(self.f(s, d, l)?), + S::F32(s) => S::F32(self.f(s, d, l)?), + S::F64(s) => S::F64(self.f(s, d, l)?), + S::F8E4M3(s) => S::F8E4M3(self.f(s, d, l)?), + S::F4(_) | S::F6E2M3(_) | S::F6E3M2(_) | S::F8E8M0(_) => { + crate::bail!("Map1 does not uspport this dtype."); + } + }; + Ok(out) + } +} + +pub trait Map2 { + fn f( + &self, + src1: &CudaSlice, + layout1: &Layout, + src2: &CudaSlice, + layout2: &Layout, + dev: &CudaDevice, + ) -> Result>; + + fn map(&self, s1: &S, l1: &Layout, s2: &S, l2: &Layout, d: &CudaDevice) -> Result { + let out = match (s1, s2) { + (S::U8(s1), S::U8(s2)) => S::U8(self.f(s1, l1, s2, l2, d)?), + (S::U32(s1), S::U32(s2)) => S::U32(self.f(s1, l1, s2, l2, d)?), + (S::I16(s1), S::I16(s2)) => S::I16(self.f(s1, l1, s2, l2, d)?), + (S::I32(s1), S::I32(s2)) => S::I32(self.f(s1, l1, s2, l2, d)?), + (S::I64(s1), S::I64(s2)) => S::I64(self.f(s1, l1, s2, l2, d)?), + (S::BF16(s1), S::BF16(s2)) => S::BF16(self.f(s1, l1, s2, l2, d)?), + (S::F16(s1), S::F16(s2)) => S::F16(self.f(s1, l1, s2, l2, d)?), + (S::F32(s1), S::F32(s2)) => S::F32(self.f(s1, l1, s2, l2, d)?), + (S::F64(s1), S::F64(s2)) => S::F64(self.f(s1, l1, s2, l2, d)?), + (S::F8E4M3(s1), S::F8E4M3(s2)) => S::F8E4M3(self.f(s1, l1, s2, l2, d)?), + _ => Err(CudaError::InternalError("dtype mismatch in binary op"))?, + }; + Ok(out) + } +} + +pub trait Map3 { + #[allow(clippy::too_many_arguments)] + fn f( + &self, + src1: &CudaSlice, + layout1: &Layout, + src2: &CudaSlice, + layout2: &Layout, + src3: &CudaSlice, + layout3: &Layout, + dev: &CudaDevice, + ) -> Result>; + + #[allow(clippy::too_many_arguments)] + fn map( + &self, + s1: &S, + l1: &Layout, + s2: &S, + l2: &Layout, + s3: &S, + l3: &Layout, + d: &CudaDevice, + ) -> Result { + let out = match (s1, s2, s3) { + (S::U8(s1), S::U8(s2), S::U8(s3)) => S::U8(self.f(s1, l1, s2, l2, s3, l3, d)?), + (S::U32(s1), S::U32(s2), S::U32(s3)) => S::U32(self.f(s1, l1, s2, l2, s3, l3, d)?), + (S::I64(s1), S::I64(s2), S::I64(s3)) => S::I64(self.f(s1, l1, s2, l2, s3, l3, d)?), + (S::BF16(s1), S::BF16(s2), S::BF16(s3)) => S::BF16(self.f(s1, l1, s2, l2, s3, l3, d)?), + (S::F16(s1), S::F16(s2), S::F16(s3)) => S::F16(self.f(s1, l1, s2, l2, s3, l3, d)?), + (S::F32(s1), S::F32(s2), S::F32(s3)) => S::F32(self.f(s1, l1, s2, l2, s3, l3, d)?), + (S::F64(s1), S::F64(s2), S::F64(s3)) => S::F64(self.f(s1, l1, s2, l2, s3, l3, d)?), + (S::F8E4M3(s1), S::F8E4M3(s2), S::F8E4M3(s3)) => { + S::F8E4M3(self.f(s1, l1, s2, l2, s3, l3, d)?) + } + _ => Err(CudaError::InternalError("dtype mismatch in ternary op"))?, + }; + Ok(out) + } +} + +pub trait Map2InPlace { + fn f( + &self, + dst: &mut CudaSlice, + dst_l: &Layout, + src: &CudaSlice, + src_l: &Layout, + dev: &CudaDevice, + ) -> Result<()>; + + fn map( + &self, + dst: &mut S, + dst_l: &Layout, + src: &S, + src_l: &Layout, + d: &CudaDevice, + ) -> Result<()> { + match (dst, src) { + (S::U8(dst), S::U8(src)) => self.f(dst, dst_l, src, src_l, d), + (S::U32(dst), S::U32(src)) => self.f(dst, dst_l, src, src_l, d), + (S::I16(dst), S::I16(src)) => self.f(dst, dst_l, src, src_l, d), + (S::I32(dst), S::I32(src)) => self.f(dst, dst_l, src, src_l, d), + (S::I64(dst), S::I64(src)) => self.f(dst, dst_l, src, src_l, d), + (S::BF16(dst), S::BF16(src)) => self.f(dst, dst_l, src, src_l, d), + (S::F16(dst), S::F16(src)) => self.f(dst, dst_l, src, src_l, d), + (S::F32(dst), S::F32(src)) => self.f(dst, dst_l, src, src_l, d), + (S::F64(dst), S::F64(src)) => self.f(dst, dst_l, src, src_l, d), + (S::F8E4M3(dst), S::F8E4M3(src)) => self.f(dst, dst_l, src, src_l, d), + _ => Err(CudaError::InternalError("dtype mismatch in binary op"))?, + } + } +} + +pub trait Map1Any { + fn f) -> S>( + &self, + src: &CudaSlice, + dev: &CudaDevice, + layout: &Layout, + wrap: W, + ) -> Result; + + fn map(&self, s: &S, d: &CudaDevice, l: &Layout) -> Result { + let out = match s { + S::U8(s) => self.f(s, d, l, S::U8)?, + S::U32(s) => self.f(s, d, l, S::U32)?, + S::I16(s) => self.f(s, d, l, S::I16)?, + S::I32(s) => self.f(s, d, l, S::I32)?, + S::I64(s) => self.f(s, d, l, S::I64)?, + S::BF16(s) => self.f(s, d, l, S::BF16)?, + S::F16(s) => self.f(s, d, l, S::F16)?, + S::F32(s) => self.f(s, d, l, S::F32)?, + S::F64(s) => self.f(s, d, l, S::F64)?, + S::F8E4M3(s) => self.f(s, d, l, S::F8E4M3)?, + S::F4(_) | S::F6E2M3(_) | S::F6E3M2(_) | S::F8E8M0(_) => { + crate::bail!("Map1 does not uspport this dtype."); + } + }; + Ok(out) + } +} + +pub trait Map2Any { + fn f( + &self, + src1: &CudaSlice, + layout1: &Layout, + src2: &CudaSlice, + layout2: &Layout, + dev: &CudaDevice, + ) -> Result; + + fn map(&self, s1: &S, l1: &Layout, s2: &S, l2: &Layout, d: &CudaDevice) -> Result { + let out = match (s1, s2) { + (S::U8(s1), S::U8(s2)) => self.f(s1, l1, s2, l2, d)?, + (S::U32(s1), S::U32(s2)) => self.f(s1, l1, s2, l2, d)?, + (S::I64(s1), S::I64(s2)) => self.f(s1, l1, s2, l2, d)?, + (S::BF16(s1), S::BF16(s2)) => self.f(s1, l1, s2, l2, d)?, + (S::F16(s1), S::F16(s2)) => self.f(s1, l1, s2, l2, d)?, + (S::F32(s1), S::F32(s2)) => self.f(s1, l1, s2, l2, d)?, + (S::F64(s1), S::F64(s2)) => self.f(s1, l1, s2, l2, d)?, + (S::F8E4M3(s1), S::F8E4M3(s2)) => self.f(s1, l1, s2, l2, d)?, + _ => Err(CudaError::InternalError("dtype mismatch in binary op")).w()?, + }; + Ok(out) + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/custom_op.rs b/v2/dependency-patches/candle-core-0.9.2/src/custom_op.rs new file mode 100644 index 0000000000..e76f96806e --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/custom_op.rs @@ -0,0 +1,489 @@ +use crate::op::{BackpropOp, Op}; +use crate::tensor::from_storage; +use crate::{CpuStorage, CudaStorage, Layout, MetalStorage, Result, Shape, Tensor}; +use std::sync::Arc; + +/// Unary ops that can be defined in user-land. +pub trait CustomOp1 { + // Box does not support const yet, so use a function to get the name. + fn name(&self) -> &'static str; + + /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn cpu_fwd(&self, storage: &CpuStorage, layout: &Layout) -> Result<(CpuStorage, Shape)>; + + /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn cuda_fwd(&self, _storage: &CudaStorage, _layout: &Layout) -> Result<(CudaStorage, Shape)> { + Err(crate::Error::Cuda( + format!("no cuda implementation for {}", self.name()).into(), + )) + } + + /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn metal_fwd( + &self, + _storage: &MetalStorage, + _layout: &Layout, + ) -> Result<(MetalStorage, Shape)> { + Err(crate::Error::Metal( + format!("no metal implementation for {}", self.name()).into(), + )) + } + + /// This function takes as argument the argument `arg` used in the forward pass, the result + /// produced by the forward operation `res` and the gradient of the result `grad_res`. + /// The function should return the gradient of the argument. + fn bwd(&self, _arg: &Tensor, _res: &Tensor, _grad_res: &Tensor) -> Result> { + Err(crate::Error::BackwardNotSupported { op: self.name() }) + } +} + +pub trait CustomOp2 { + fn name(&self) -> &'static str; + + /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn cpu_fwd( + &self, + s1: &CpuStorage, + l1: &Layout, + s2: &CpuStorage, + l2: &Layout, + ) -> Result<(CpuStorage, Shape)>; + + /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn cuda_fwd( + &self, + _: &CudaStorage, + _: &Layout, + _: &CudaStorage, + _: &Layout, + ) -> Result<(CudaStorage, Shape)> { + Err(crate::Error::Cuda( + format!("no cuda implementation for {}", self.name()).into(), + )) + } + + /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn metal_fwd( + &self, + _: &MetalStorage, + _: &Layout, + _: &MetalStorage, + _: &Layout, + ) -> Result<(MetalStorage, Shape)> { + Err(crate::Error::Metal( + format!("no metal implementation for {}", self.name()).into(), + )) + } + + fn bwd( + &self, + _arg1: &Tensor, + _arg2: &Tensor, + _res: &Tensor, + _grad_res: &Tensor, + ) -> Result<(Option, Option)> { + Err(crate::Error::BackwardNotSupported { op: self.name() }) + } +} + +pub trait CustomOp3 { + fn name(&self) -> &'static str; + + /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn cpu_fwd( + &self, + s1: &CpuStorage, + l1: &Layout, + s2: &CpuStorage, + l2: &Layout, + s3: &CpuStorage, + l3: &Layout, + ) -> Result<(CpuStorage, Shape)>; + + /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn cuda_fwd( + &self, + _: &CudaStorage, + _: &Layout, + _: &CudaStorage, + _: &Layout, + _: &CudaStorage, + _: &Layout, + ) -> Result<(CudaStorage, Shape)> { + Err(crate::Error::Cuda( + format!("no cuda implementation for {}", self.name()).into(), + )) + } + + /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn metal_fwd( + &self, + _: &MetalStorage, + _: &Layout, + _: &MetalStorage, + _: &Layout, + _: &MetalStorage, + _: &Layout, + ) -> Result<(MetalStorage, Shape)> { + Err(crate::Error::Metal( + format!("no metal implementation for {}", self.name()).into(), + )) + } + + fn bwd( + &self, + _arg1: &Tensor, + _arg2: &Tensor, + _arg3: &Tensor, + _res: &Tensor, + _grad_res: &Tensor, + ) -> Result<(Option, Option, Option)> { + Err(crate::Error::BackwardNotSupported { op: self.name() }) + } +} + +impl Tensor { + /// Applies a unary custom op without backward support + pub fn apply_op1_no_bwd(&self, c: &C) -> Result { + let (storage, shape) = self.storage().apply_op1(self.layout(), c)?; + Ok(from_storage(storage, shape, BackpropOp::none(), false)) + } + + /// Applies a binary custom op without backward support + pub fn apply_op2_no_bwd(&self, rhs: &Self, c: &C) -> Result { + let (storage, shape) = + self.storage() + .apply_op2(self.layout(), &rhs.storage(), rhs.layout(), c)?; + Ok(from_storage(storage, shape, BackpropOp::none(), false)) + } + + /// Applies a ternary custom op without backward support + pub fn apply_op3_no_bwd(&self, t2: &Self, t3: &Self, c: &C) -> Result { + let (storage, shape) = self.storage().apply_op3( + self.layout(), + &t2.storage(), + t2.layout(), + &t3.storage(), + t3.layout(), + c, + )?; + Ok(from_storage(storage, shape, BackpropOp::none(), false)) + } + + /// Applies a unary custom op. + pub fn apply_op1_arc(&self, c: Arc>) -> Result { + let (storage, shape) = self + .storage() + .apply_op1(self.layout(), c.as_ref().as_ref())?; + let op = BackpropOp::new1(self, |s| Op::CustomOp1(s, c.clone())); + Ok(from_storage(storage, shape, op, false)) + } + + pub fn apply_op1(&self, c: C) -> Result { + self.apply_op1_arc(Arc::new(Box::new(c))) + } + + /// Applies a binary custom op. + pub fn apply_op2_arc( + &self, + rhs: &Self, + c: Arc>, + ) -> Result { + let (storage, shape) = self.storage().apply_op2( + self.layout(), + &rhs.storage(), + rhs.layout(), + c.as_ref().as_ref(), + )?; + let op = BackpropOp::new2(self, rhs, |t1, t2| Op::CustomOp2(t1, t2, c.clone())); + Ok(from_storage(storage, shape, op, false)) + } + + pub fn apply_op2(&self, r: &Self, c: C) -> Result { + self.apply_op2_arc(r, Arc::new(Box::new(c))) + } + + /// Applies a ternary custom op. + pub fn apply_op3_arc( + &self, + t2: &Self, + t3: &Self, + c: Arc>, + ) -> Result { + let (storage, shape) = self.storage().apply_op3( + self.layout(), + &t2.storage(), + t2.layout(), + &t3.storage(), + t3.layout(), + c.as_ref().as_ref(), + )?; + let op = BackpropOp::new3(self, t2, t3, |t1, t2, t3| { + Op::CustomOp3(t1, t2, t3, c.clone()) + }); + Ok(from_storage(storage, shape, op, false)) + } + + pub fn apply_op3( + &self, + t2: &Self, + t3: &Self, + c: C, + ) -> Result { + self.apply_op3_arc(t2, t3, Arc::new(Box::new(c))) + } +} + +// In place ops. + +/// Unary ops that can be defined in user-land. +/// These ops work in place and as such back-prop is unsupported. +pub trait InplaceOp1 { + // Box does not support const yet, so use a function to get the name. + fn name(&self) -> &'static str; + + /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn cpu_fwd(&self, storage: &mut CpuStorage, layout: &Layout) -> Result<()>; + + /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn cuda_fwd(&self, _storage: &mut CudaStorage, _layout: &Layout) -> Result<()> { + Err(crate::Error::Cuda( + format!("no cuda implementation for {}", self.name()).into(), + )) + } + + /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn metal_fwd(&self, _storage: &mut MetalStorage, _layout: &Layout) -> Result<()> { + Err(crate::Error::Metal( + format!("no metal implementation for {}", self.name()).into(), + )) + } +} + +pub trait InplaceOp2 { + fn name(&self) -> &'static str; + + /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn cpu_fwd(&self, s1: &mut CpuStorage, l1: &Layout, s2: &CpuStorage, l2: &Layout) + -> Result<()>; + + /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn cuda_fwd(&self, _: &mut CudaStorage, _: &Layout, _: &CudaStorage, _: &Layout) -> Result<()> { + Err(crate::Error::Cuda( + format!("no cuda implementation for {}", self.name()).into(), + )) + } + + /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn metal_fwd( + &self, + _: &mut MetalStorage, + _: &Layout, + _: &MetalStorage, + _: &Layout, + ) -> Result<()> { + Err(crate::Error::Metal( + format!("no metal implementation for {}", self.name()).into(), + )) + } +} + +pub trait InplaceOp3 { + fn name(&self) -> &'static str; + + /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn cpu_fwd( + &self, + s1: &mut CpuStorage, + l1: &Layout, + s2: &CpuStorage, + l2: &Layout, + s3: &CpuStorage, + l3: &Layout, + ) -> Result<()>; + + /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn cuda_fwd( + &self, + _: &mut CudaStorage, + _: &Layout, + _: &CudaStorage, + _: &Layout, + _: &CudaStorage, + _: &Layout, + ) -> Result<()> { + Err(crate::Error::Cuda( + format!("no cuda implementation for {}", self.name()).into(), + )) + } + + /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides, + /// offsets etc so the associated layout should be used to access it. + fn metal_fwd( + &self, + _: &mut MetalStorage, + _: &Layout, + _: &MetalStorage, + _: &Layout, + _: &MetalStorage, + _: &Layout, + ) -> Result<()> { + Err(crate::Error::Metal( + format!("no metal implementation for {}", self.name()).into(), + )) + } +} + +impl Tensor { + /// Applies a unary custom op in place. + pub fn inplace_op1(&self, c: &C) -> Result<()> { + self.storage_mut().inplace_op1(self.layout(), c) + } + + /// Applies a unary custom op in place (for the first tensor). + pub fn inplace_op2(&self, rhs: &Self, c: &C) -> Result<()> { + self.storage_mut() + .inplace_op2(self.layout(), &rhs.storage(), rhs.layout(), c) + } + + /// Applies a ternary custom op in place (for the first tensor). + pub fn inplace_op3(&self, t2: &Self, t3: &Self, c: &C) -> Result<()> { + self.storage_mut().inplace_op3( + self.layout(), + &t2.storage(), + t2.layout(), + &t3.storage(), + t3.layout(), + c, + ) + } +} + +#[cfg(feature = "ug")] +pub struct UgIOp1 { + name: &'static str, + #[cfg(feature = "cuda")] + func: cudarc::driver::CudaFunction, + #[cfg(feature = "metal")] + func: candle_metal_kernels::metal::ComputePipeline, +} + +#[cfg(feature = "ug")] +impl UgIOp1 { + #[allow(unused)] + #[cfg(all(not(target_arch = "wasm32"), not(target_os = "ios")))] + pub fn new( + name: &'static str, + kernel: candle_ug::lang::ssa::Kernel, + device: &crate::Device, + ) -> Result { + #[cfg(feature = "cuda")] + { + let device = device.as_cuda_device()?; + let func = device.compile(name, kernel)?; + Ok(Self { + name, + func: func.into_cuda_function(), + }) + } + #[cfg(feature = "metal")] + { + let device = device.as_metal_device()?; + let func = device.compile(name, kernel)?; + Ok(Self { name, func }) + } + #[cfg(not(any(feature = "cuda", feature = "metal")))] + { + Ok(Self { name }) + } + } +} + +#[cfg(feature = "ug")] +impl InplaceOp1 for UgIOp1 { + fn name(&self) -> &'static str { + self.name + } + + fn cpu_fwd(&self, _: &mut CpuStorage, _: &Layout) -> Result<()> { + crate::bail!("ug ops are only supported on metal/cuda at the moment") + } + + #[cfg(feature = "metal")] + fn metal_fwd(&self, sto: &mut MetalStorage, layout: &Layout) -> Result<()> { + use crate::backend::BackendStorage; + use objc2_metal; + + let elem_count = layout.shape().elem_count(); + if sto.dtype() != crate::DType::F32 { + // TODO: support more dtypes. + crate::bail!("input is not a f32 tensor") + } + let device = sto.device(); + let encoder = device.command_encoder()?; + encoder.set_compute_pipeline_state(&self.func); + let (g, b) = if elem_count.is_multiple_of(32) { + (elem_count / 32, 32) + } else { + (elem_count, 1) + }; + let grid_dims = objc2_metal::MTLSize { + width: g, + height: 1, + depth: 1, + }; + let group_dims = candle_metal_kernels::utils::get_block_dims(b, 1, 1); + candle_metal_kernels::utils::set_param(&encoder, 0, (sto.buffer(), 0usize)); + + encoder.use_resource(sto.buffer(), objc2_metal::MTLResourceUsage::Write); + encoder.dispatch_threads(grid_dims, group_dims); + + Ok(()) + } + + #[cfg(feature = "cuda")] + fn cuda_fwd(&self, sto: &mut CudaStorage, layout: &Layout) -> Result<()> { + use crate::cuda_backend::WrapErr; + use cudarc::driver::PushKernelArg; + + let elem_count = layout.shape().elem_count(); + let stream = sto.device.cuda_stream(); + // TODO: support more dtypes. + let sto = sto.as_cuda_slice::()?; + let sto = match layout.contiguous_offsets() { + None => crate::bail!("input has to be contiguous"), + Some((o1, o2)) => sto.slice(o1..o2), + }; + let (g, b) = if elem_count % 32 == 0 { + (elem_count / 32, 32) + } else { + (elem_count, 1) + }; + let cfg = cudarc::driver::LaunchConfig { + grid_dim: (g as u32, 1, 1), + block_dim: (b as u32, 1, 1), + shared_mem_bytes: 0, + }; + let mut builder = stream.launch_builder(&self.func); + builder.arg(&sto); + unsafe { builder.launch(cfg) }.w()?; + Ok(()) + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/device.rs b/v2/dependency-patches/candle-core-0.9.2/src/device.rs new file mode 100644 index 0000000000..d0167c61e9 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/device.rs @@ -0,0 +1,502 @@ +use crate::backend::BackendDevice; +use crate::cpu_backend::CpuDevice; +use crate::{CpuStorage, DType, Result, Shape, Storage, WithDType}; + +/// A `DeviceLocation` represents a physical device whereas multiple `Device` +/// can live on the same location (typically for cuda devices). +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub enum DeviceLocation { + Cpu, + Cuda { gpu_id: usize }, + Metal { gpu_id: usize }, +} + +/// Cpu, Cuda, or Metal +#[derive(Debug, Clone)] +pub enum Device { + Cpu, + Cuda(crate::CudaDevice), + Metal(crate::MetalDevice), +} + +pub trait NdArray { + fn shape(&self) -> Result; + + fn to_cpu_storage(&self) -> CpuStorage; +} + +impl NdArray for S { + fn shape(&self) -> Result { + Ok(Shape::from(())) + } + + fn to_cpu_storage(&self) -> CpuStorage { + S::to_cpu_storage(&[*self]) + } +} + +impl NdArray for &[S; N] { + fn shape(&self) -> Result { + Ok(Shape::from(self.len())) + } + + fn to_cpu_storage(&self) -> CpuStorage { + S::to_cpu_storage(self.as_slice()) + } +} + +impl NdArray for &[S] { + fn shape(&self) -> Result { + Ok(Shape::from(self.len())) + } + + fn to_cpu_storage(&self) -> CpuStorage { + S::to_cpu_storage(self) + } +} + +impl NdArray for &[[S; N]; M] { + fn shape(&self) -> Result { + Ok(Shape::from((M, N))) + } + + fn to_cpu_storage(&self) -> CpuStorage { + S::to_cpu_storage_owned(self.concat()) + } +} + +impl NdArray + for &[[[S; N3]; N2]; N1] +{ + fn shape(&self) -> Result { + Ok(Shape::from((N1, N2, N3))) + } + + fn to_cpu_storage(&self) -> CpuStorage { + let mut vec = Vec::with_capacity(N1 * N2 * N3); + for i1 in 0..N1 { + for i2 in 0..N2 { + vec.extend(self[i1][i2]) + } + } + S::to_cpu_storage_owned(vec) + } +} + +impl NdArray + for &[[[[S; N4]; N3]; N2]; N1] +{ + fn shape(&self) -> Result { + Ok(Shape::from((N1, N2, N3, N4))) + } + + fn to_cpu_storage(&self) -> CpuStorage { + let mut vec = Vec::with_capacity(N1 * N2 * N3 * N4); + for i1 in 0..N1 { + for i2 in 0..N2 { + for i3 in 0..N3 { + vec.extend(self[i1][i2][i3]) + } + } + } + S::to_cpu_storage_owned(vec) + } +} + +impl NdArray for Vec { + fn shape(&self) -> Result { + Ok(Shape::from(self.len())) + } + + fn to_cpu_storage(&self) -> CpuStorage { + S::to_cpu_storage(self.as_slice()) + } +} + +impl NdArray for Vec<&[S]> { + fn shape(&self) -> Result { + if self.is_empty() { + crate::bail!("empty array") + } + let n = self.len(); + let m = self[0].len(); + for v in self.iter() { + if v.len() != m { + crate::bail!("two elements have different len {m} {}", v.len()) + } + } + Ok(Shape::from((n, m))) + } + + fn to_cpu_storage(&self) -> CpuStorage { + let data = self.iter().copied().flatten().copied().collect::>(); + S::to_cpu_storage_owned(data) + } +} + +impl NdArray for Vec> { + fn shape(&self) -> Result { + if self.is_empty() { + crate::bail!("empty array") + } + let n = self.len(); + let m = self[0].len(); + for v in self.iter() { + if v.len() != m { + crate::bail!("two elements have different len {m} {}", v.len()) + } + } + Ok(Shape::from((n, m))) + } + + fn to_cpu_storage(&self) -> CpuStorage { + let len: usize = self.iter().map(|v| v.len()).sum(); + let mut dst = Vec::with_capacity(len); + for v in self.iter() { + dst.extend(v.iter().copied()); + } + S::to_cpu_storage_owned(dst) + } +} + +impl NdArray for Vec>> { + fn shape(&self) -> Result { + if self.is_empty() { + crate::bail!("empty array") + } + let shape0 = self[0].shape()?; + let n = self.len(); + for v in self.iter() { + let shape = v.shape()?; + if shape != shape0 { + crate::bail!("two elements have different shapes {shape:?} {shape0:?}") + } + } + Ok(Shape::from([[n].as_slice(), shape0.dims()].concat())) + } + + fn to_cpu_storage(&self) -> CpuStorage { + if self.is_empty() { + return S::to_cpu_storage_owned(vec![]); + } + let len: usize = self + .iter() + .map(|v| v.iter().map(|v| v.len()).sum::()) + .sum(); + let mut dst = Vec::with_capacity(len); + for v1 in self.iter() { + for v2 in v1.iter() { + dst.extend(v2.iter().copied()); + } + } + S::to_cpu_storage_owned(dst) + } +} + +impl NdArray for Vec>>> { + fn shape(&self) -> Result { + if self.is_empty() { + crate::bail!("empty array") + } + let shape0 = self[0].shape()?; + let n = self.len(); + for v in self.iter() { + let shape = v.shape()?; + if shape != shape0 { + crate::bail!("two elements have different shapes {shape:?} {shape0:?}") + } + } + Ok(Shape::from([[n].as_slice(), shape0.dims()].concat())) + } + + fn to_cpu_storage(&self) -> CpuStorage { + let len: usize = self + .iter() + .map(|v| { + v.iter() + .map(|v| v.iter().map(|v| v.len()).sum::()) + .sum::() + }) + .sum(); + let mut dst = Vec::with_capacity(len); + for v1 in self.iter() { + for v2 in v1.iter() { + for v3 in v2.iter() { + dst.extend(v3.iter().copied()); + } + } + } + S::to_cpu_storage_owned(dst) + } +} + +impl Device { + pub fn new_cuda(ordinal: usize) -> Result { + Ok(Self::Cuda(crate::CudaDevice::new(ordinal)?)) + } + + pub fn as_cuda_device(&self) -> Result<&crate::CudaDevice> { + match self { + Self::Cuda(d) => Ok(d), + Self::Cpu => crate::bail!("expected a cuda device, got cpu"), + Self::Metal(_) => crate::bail!("expected a cuda device, got Metal"), + } + } + + pub fn as_metal_device(&self) -> Result<&crate::MetalDevice> { + match self { + Self::Cuda(_) => crate::bail!("expected a metal device, got cuda"), + Self::Cpu => crate::bail!("expected a metal device, got cpu"), + Self::Metal(d) => Ok(d), + } + } + + pub fn new_cuda_with_stream(ordinal: usize) -> Result { + Ok(Self::Cuda(crate::CudaDevice::new_with_stream(ordinal)?)) + } + + pub fn new_metal(ordinal: usize) -> Result { + Ok(Self::Metal(crate::MetalDevice::new(ordinal)?)) + } + + pub fn set_seed(&self, seed: u64) -> Result<()> { + match self { + Self::Cpu => CpuDevice.set_seed(seed), + Self::Cuda(c) => c.set_seed(seed), + Self::Metal(m) => m.set_seed(seed), + } + } + + pub fn get_current_seed(&self) -> Result { + match self { + Self::Cpu => CpuDevice.get_current_seed(), + Self::Cuda(c) => c.get_current_seed(), + Self::Metal(m) => m.get_current_seed(), + } + } + + pub fn same_device(&self, rhs: &Self) -> bool { + match (self, rhs) { + (Self::Cpu, Self::Cpu) => true, + (Self::Cuda(lhs), Self::Cuda(rhs)) => lhs.same_device(rhs), + (Self::Metal(lhs), Self::Metal(rhs)) => lhs.same_device(rhs), + _ => false, + } + } + + pub fn location(&self) -> DeviceLocation { + match self { + Self::Cpu => DeviceLocation::Cpu, + Self::Cuda(device) => device.location(), + Device::Metal(device) => device.location(), + } + } + + pub fn is_cpu(&self) -> bool { + matches!(self, Self::Cpu) + } + + pub fn is_cuda(&self) -> bool { + matches!(self, Self::Cuda(_)) + } + + pub fn is_metal(&self) -> bool { + matches!(self, Self::Metal(_)) + } + + pub fn supports_bf16(&self) -> bool { + match self { + Self::Cuda(_) | Self::Metal(_) => true, + Self::Cpu => false, + } + } + + /// Return `BF16` for devices that support it, otherwise default to `F32`. + pub fn bf16_default_to_f32(&self) -> DType { + if self.supports_bf16() { + DType::BF16 + } else { + DType::F32 + } + } + + pub fn cuda_if_available(ordinal: usize) -> Result { + if crate::utils::cuda_is_available() { + Self::new_cuda(ordinal) + } else { + Ok(Self::Cpu) + } + } + + pub fn metal_if_available(ordinal: usize) -> Result { + if crate::utils::metal_is_available() { + Self::new_metal(ordinal) + } else { + Ok(Self::Cpu) + } + } + + pub(crate) fn rand_uniform_f64( + &self, + lo: f64, + up: f64, + shape: &Shape, + dtype: DType, + ) -> Result { + match self { + Device::Cpu => { + let storage = CpuDevice.rand_uniform(shape, dtype, lo, up)?; + Ok(Storage::Cpu(storage)) + } + Device::Cuda(device) => { + // TODO: Remove the special case if we start supporting generating f16/bf16 directly. + if dtype == DType::F16 || dtype == DType::BF16 { + let storage = device.rand_uniform(shape, DType::F32, lo, up)?; + Storage::Cuda(storage).to_dtype(&crate::Layout::contiguous(shape), dtype) + } else { + let storage = device.rand_uniform(shape, dtype, lo, up)?; + Ok(Storage::Cuda(storage)) + } + } + Device::Metal(device) => { + let storage = device.rand_uniform(shape, dtype, lo, up)?; + Ok(Storage::Metal(storage)) + } + } + } + + pub(crate) fn rand_uniform( + &self, + lo: T, + up: T, + shape: &Shape, + ) -> Result { + self.rand_uniform_f64(lo.to_f64(), up.to_f64(), shape, T::DTYPE) + } + + pub(crate) fn rand_normal_f64( + &self, + mean: f64, + std: f64, + shape: &Shape, + dtype: DType, + ) -> Result { + match self { + Device::Cpu => { + let storage = CpuDevice.rand_normal(shape, dtype, mean, std)?; + Ok(Storage::Cpu(storage)) + } + Device::Cuda(device) => { + // TODO: Remove the special case if we start supporting generating f16/bf16 directly. + if dtype == DType::F16 || dtype == DType::BF16 { + let storage = device.rand_normal(shape, DType::F32, mean, std)?; + Storage::Cuda(storage).to_dtype(&crate::Layout::contiguous(shape), dtype) + } else { + let storage = device.rand_normal(shape, dtype, mean, std)?; + Ok(Storage::Cuda(storage)) + } + } + Device::Metal(device) => { + let storage = device.rand_normal(shape, dtype, mean, std)?; + Ok(Storage::Metal(storage)) + } + } + } + + pub(crate) fn rand_normal( + &self, + mean: T, + std: T, + shape: &Shape, + ) -> Result { + self.rand_normal_f64(mean.to_f64(), std.to_f64(), shape, T::DTYPE) + } + + pub(crate) fn zeros(&self, shape: &Shape, dtype: DType) -> Result { + match self { + Device::Cpu => { + let storage = CpuDevice.zeros_impl(shape, dtype)?; + Ok(Storage::Cpu(storage)) + } + Device::Cuda(device) => { + let storage = device.zeros_impl(shape, dtype)?; + Ok(Storage::Cuda(storage)) + } + Device::Metal(device) => { + let storage = device.zeros_impl(shape, dtype)?; + Ok(Storage::Metal(storage)) + } + } + } + + pub(crate) unsafe fn alloc_uninit(&self, shape: &Shape, dtype: DType) -> Result { + match self { + Device::Cpu => { + let storage = CpuDevice.alloc_uninit(shape, dtype)?; + Ok(Storage::Cpu(storage)) + } + Device::Cuda(device) => { + let storage = device.alloc_uninit(shape, dtype)?; + Ok(Storage::Cuda(storage)) + } + Device::Metal(device) => { + let storage = device.alloc_uninit(shape, dtype)?; + Ok(Storage::Metal(storage)) + } + } + } + + pub(crate) fn storage_from_slice(&self, data: &[D]) -> Result { + match self { + Device::Cpu => Ok(Storage::Cpu(data.to_cpu_storage())), + Device::Cuda(device) => { + let storage = device.storage_from_slice(data)?; + Ok(Storage::Cuda(storage)) + } + Device::Metal(device) => { + let storage = device.storage_from_slice(data)?; + Ok(Storage::Metal(storage)) + } + } + } + + pub(crate) fn storage(&self, array: A) -> Result { + match self { + Device::Cpu => Ok(Storage::Cpu(array.to_cpu_storage())), + Device::Cuda(device) => { + let storage = array.to_cpu_storage(); + let storage = device.storage_from_cpu_storage_owned(storage)?; + Ok(Storage::Cuda(storage)) + } + Device::Metal(device) => { + let storage = array.to_cpu_storage(); + let storage = device.storage_from_cpu_storage_owned(storage)?; + Ok(Storage::Metal(storage)) + } + } + } + + pub(crate) fn storage_owned(&self, data: Vec) -> Result { + match self { + Device::Cpu => Ok(Storage::Cpu(S::to_cpu_storage_owned(data))), + Device::Cuda(device) => { + let storage = S::to_cpu_storage_owned(data); + let storage = device.storage_from_cpu_storage_owned(storage)?; + Ok(Storage::Cuda(storage)) + } + Device::Metal(device) => { + let storage = S::to_cpu_storage_owned(data); + let storage = device.storage_from_cpu_storage_owned(storage)?; + Ok(Storage::Metal(storage)) + } + } + } + + pub fn synchronize(&self) -> Result<()> { + match self { + Self::Cpu => Ok(()), + Self::Cuda(d) => d.synchronize(), + Self::Metal(d) => d.synchronize(), + } + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/display.rs b/v2/dependency-patches/candle-core-0.9.2/src/display.rs new file mode 100644 index 0000000000..a9b53947f3 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/display.rs @@ -0,0 +1,558 @@ +//! Pretty printing of tensors +//! +//! This implementation should be in line with the [PyTorch version](https://github.com/pytorch/pytorch/blob/7b419e8513a024e172eae767e24ec1b849976b13/torch/_tensor_str.py). +//! +use crate::{DType, Result, Tensor, WithDType}; +use half::{bf16, f16}; + +impl Tensor { + fn fmt_dt( + &self, + f: &mut std::fmt::Formatter, + ) -> std::fmt::Result { + let device_str = match self.device().location() { + crate::DeviceLocation::Cpu => "".to_owned(), + crate::DeviceLocation::Cuda { gpu_id } => { + format!(", cuda:{gpu_id}") + } + crate::DeviceLocation::Metal { gpu_id } => { + format!(", metal:{gpu_id}") + } + }; + + write!(f, "Tensor[")?; + match self.dims() { + [] => { + if let Ok(v) = self.to_scalar::() { + write!(f, "{v}")? + } + } + [s] if *s < 10 => { + if let Ok(vs) = self.to_vec1::() { + for (i, v) in vs.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{v}")?; + } + } + } + dims => { + write!(f, "dims ")?; + for (i, d) in dims.iter().enumerate() { + if i > 0 { + write!(f, ", ")?; + } + write!(f, "{d}")?; + } + } + } + write!(f, "; {}{}]", self.dtype().as_str(), device_str) + } +} + +impl std::fmt::Debug for Tensor { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self.dtype() { + DType::U8 => self.fmt_dt::(f), + DType::U32 => self.fmt_dt::(f), + DType::I16 => self.fmt_dt::(f), + DType::I32 => self.fmt_dt::(f), + DType::I64 => self.fmt_dt::(f), + DType::BF16 => self.fmt_dt::(f), + DType::F16 => self.fmt_dt::(f), + DType::F32 => self.fmt_dt::(f), + DType::F64 => self.fmt_dt::(f), + DType::F8E4M3 => self.fmt_dt::(f), + DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { + write!( + f, + "Tensor[{:?}; dtype={}, unsupported dummy type]", + self.shape(), + self.dtype().as_str() + ) + } + } + } +} + +/// Options for Tensor pretty printing +#[derive(Debug, Clone)] +pub struct PrinterOptions { + pub precision: usize, + pub threshold: usize, + pub edge_items: usize, + pub line_width: usize, + pub sci_mode: Option, +} + +static PRINT_OPTS: std::sync::Mutex = + std::sync::Mutex::new(PrinterOptions::const_default()); + +impl PrinterOptions { + // We cannot use the default trait as it's not const. + const fn const_default() -> Self { + Self { + precision: 4, + threshold: 1000, + edge_items: 3, + line_width: 80, + sci_mode: None, + } + } +} + +pub fn print_options() -> &'static std::sync::Mutex { + &PRINT_OPTS +} + +pub fn set_print_options(options: PrinterOptions) { + *PRINT_OPTS.lock().unwrap() = options +} + +pub fn set_print_options_default() { + *PRINT_OPTS.lock().unwrap() = PrinterOptions::const_default() +} + +pub fn set_print_options_short() { + *PRINT_OPTS.lock().unwrap() = PrinterOptions { + precision: 2, + threshold: 1000, + edge_items: 2, + line_width: 80, + sci_mode: None, + } +} + +pub fn set_print_options_full() { + *PRINT_OPTS.lock().unwrap() = PrinterOptions { + precision: 4, + threshold: usize::MAX, + edge_items: 3, + line_width: 80, + sci_mode: None, + } +} + +pub fn set_line_width(line_width: usize) { + PRINT_OPTS.lock().unwrap().line_width = line_width +} + +pub fn set_precision(precision: usize) { + PRINT_OPTS.lock().unwrap().precision = precision +} + +pub fn set_edge_items(edge_items: usize) { + PRINT_OPTS.lock().unwrap().edge_items = edge_items +} + +pub fn set_threshold(threshold: usize) { + PRINT_OPTS.lock().unwrap().threshold = threshold +} + +pub fn set_sci_mode(sci_mode: Option) { + PRINT_OPTS.lock().unwrap().sci_mode = sci_mode +} + +struct FmtSize { + current_size: usize, +} + +impl FmtSize { + fn new() -> Self { + Self { current_size: 0 } + } + + fn final_size(self) -> usize { + self.current_size + } +} + +impl std::fmt::Write for FmtSize { + fn write_str(&mut self, s: &str) -> std::fmt::Result { + self.current_size += s.len(); + Ok(()) + } +} + +trait TensorFormatter { + type Elem: WithDType; + + fn fmt(&self, v: Self::Elem, max_w: usize, f: &mut T) -> std::fmt::Result; + + fn max_width(&self, to_display: &Tensor) -> usize { + let mut max_width = 1; + if let Ok(vs) = to_display.flatten_all().and_then(|t| t.to_vec1()) { + for &v in vs.iter() { + let mut fmt_size = FmtSize::new(); + let _res = self.fmt(v, 1, &mut fmt_size); + max_width = usize::max(max_width, fmt_size.final_size()) + } + } + max_width + } + + fn write_newline_indent(i: usize, f: &mut std::fmt::Formatter) -> std::fmt::Result { + writeln!(f)?; + for _ in 0..i { + write!(f, " ")? + } + Ok(()) + } + + fn fmt_tensor( + &self, + t: &Tensor, + indent: usize, + max_w: usize, + summarize: bool, + po: &PrinterOptions, + f: &mut std::fmt::Formatter, + ) -> std::fmt::Result { + let dims = t.dims(); + let edge_items = po.edge_items; + write!(f, "[")?; + match dims { + [] => { + if let Ok(v) = t.to_scalar::() { + self.fmt(v, max_w, f)? + } + } + [v] if summarize && *v > 2 * edge_items => { + if let Ok(vs) = t + .narrow(0, 0, edge_items) + .and_then(|t| t.to_vec1::()) + { + for v in vs.into_iter() { + self.fmt(v, max_w, f)?; + write!(f, ", ")?; + } + } + write!(f, "...")?; + if let Ok(vs) = t + .narrow(0, v - edge_items, edge_items) + .and_then(|t| t.to_vec1::()) + { + for v in vs.into_iter() { + write!(f, ", ")?; + self.fmt(v, max_w, f)?; + } + } + } + [_] => { + let elements_per_line = usize::max(1, po.line_width / (max_w + 2)); + if let Ok(vs) = t.to_vec1::() { + for (i, v) in vs.into_iter().enumerate() { + if i > 0 { + if i % elements_per_line == 0 { + write!(f, ",")?; + Self::write_newline_indent(indent, f)? + } else { + write!(f, ", ")?; + } + } + self.fmt(v, max_w, f)? + } + } + } + _ => { + if summarize && dims[0] > 2 * edge_items { + for i in 0..edge_items { + match t.get(i) { + Ok(t) => self.fmt_tensor(&t, indent + 1, max_w, summarize, po, f)?, + Err(e) => write!(f, "{e:?}")?, + } + write!(f, ",")?; + Self::write_newline_indent(indent, f)? + } + write!(f, "...")?; + Self::write_newline_indent(indent, f)?; + for i in dims[0] - edge_items..dims[0] { + match t.get(i) { + Ok(t) => self.fmt_tensor(&t, indent + 1, max_w, summarize, po, f)?, + Err(e) => write!(f, "{e:?}")?, + } + if i + 1 != dims[0] { + write!(f, ",")?; + Self::write_newline_indent(indent, f)? + } + } + } else { + for i in 0..dims[0] { + match t.get(i) { + Ok(t) => self.fmt_tensor(&t, indent + 1, max_w, summarize, po, f)?, + Err(e) => write!(f, "{e:?}")?, + } + if i + 1 != dims[0] { + write!(f, ",")?; + Self::write_newline_indent(indent, f)? + } + } + } + } + } + write!(f, "]")?; + Ok(()) + } +} + +struct FloatFormatter { + int_mode: bool, + sci_mode: bool, + precision: usize, + _phantom: std::marker::PhantomData, +} + +impl FloatFormatter +where + S: WithDType + num_traits::Float + std::fmt::Display, +{ + fn new(t: &Tensor, po: &PrinterOptions) -> Result { + let mut int_mode = true; + let mut sci_mode = false; + + // Rather than containing all values, this should only include + // values that end up being displayed according to [threshold]. + let values = t + .flatten_all()? + .to_vec1()? + .into_iter() + .filter(|v: &S| v.is_finite() && !v.is_zero()) + .collect::>(); + if !values.is_empty() { + let mut nonzero_finite_min = S::max_value(); + let mut nonzero_finite_max = S::min_value(); + for &v in values.iter() { + let v = v.abs(); + if v < nonzero_finite_min { + nonzero_finite_min = v + } + if v > nonzero_finite_max { + nonzero_finite_max = v + } + } + + for &value in values.iter() { + if value.ceil() != value { + int_mode = false; + break; + } + } + if let Some(v1) = S::from(1000.) { + if let Some(v2) = S::from(1e8) { + if let Some(v3) = S::from(1e-4) { + sci_mode = nonzero_finite_max / nonzero_finite_min > v1 + || nonzero_finite_max > v2 + || nonzero_finite_min < v3 + } + } + } + } + + match po.sci_mode { + None => {} + Some(v) => sci_mode = v, + } + Ok(Self { + int_mode, + sci_mode, + precision: po.precision, + _phantom: std::marker::PhantomData, + }) + } +} + +impl TensorFormatter for FloatFormatter +where + S: WithDType + num_traits::Float + std::fmt::Display + std::fmt::LowerExp, +{ + type Elem = S; + + fn fmt(&self, v: Self::Elem, max_w: usize, f: &mut T) -> std::fmt::Result { + if self.sci_mode { + write!( + f, + "{v:width$.prec$e}", + v = v, + width = max_w, + prec = self.precision + ) + } else if self.int_mode { + if v.is_finite() { + write!(f, "{v:width$.0}.", v = v, width = max_w - 1) + } else { + write!(f, "{v:max_w$.0}") + } + } else { + write!( + f, + "{v:width$.prec$}", + v = v, + width = max_w, + prec = self.precision + ) + } + } +} + +struct IntFormatter { + _phantom: std::marker::PhantomData, +} + +impl IntFormatter { + fn new() -> Self { + Self { + _phantom: std::marker::PhantomData, + } + } +} + +impl TensorFormatter for IntFormatter +where + S: WithDType + std::fmt::Display, +{ + type Elem = S; + + fn fmt(&self, v: Self::Elem, max_w: usize, f: &mut T) -> std::fmt::Result { + write!(f, "{v:max_w$}") + } +} + +fn get_summarized_data(t: &Tensor, edge_items: usize) -> Result { + let dims = t.dims(); + if dims.is_empty() { + Ok(t.clone()) + } else if dims.len() == 1 { + if dims[0] > 2 * edge_items { + Tensor::cat( + &[ + t.narrow(0, 0, edge_items)?, + t.narrow(0, dims[0] - edge_items, edge_items)?, + ], + 0, + ) + } else { + Ok(t.clone()) + } + } else if dims[0] > 2 * edge_items { + let mut vs: Vec<_> = (0..edge_items) + .map(|i| get_summarized_data(&t.get(i)?, edge_items)) + .collect::>>()?; + for i in (dims[0] - edge_items)..dims[0] { + vs.push(get_summarized_data(&t.get(i)?, edge_items)?) + } + Tensor::cat(&vs, 0) + } else { + let vs: Vec<_> = (0..dims[0]) + .map(|i| get_summarized_data(&t.get(i)?, edge_items)) + .collect::>>()?; + Tensor::cat(&vs, 0) + } +} + +impl std::fmt::Display for Tensor { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + let po = PRINT_OPTS.lock().unwrap(); + let summarize = self.elem_count() > po.threshold; + let to_display = if summarize { + match get_summarized_data(self, po.edge_items) { + Ok(v) => v, + Err(err) => return write!(f, "{err:?}"), + } + } else { + self.clone() + }; + match self.dtype() { + DType::U8 => { + let tf: IntFormatter = IntFormatter::new(); + let max_w = tf.max_width(&to_display); + tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; + writeln!(f)?; + } + DType::U32 => { + let tf: IntFormatter = IntFormatter::new(); + let max_w = tf.max_width(&to_display); + tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; + writeln!(f)?; + } + DType::I16 => { + let tf: IntFormatter = IntFormatter::new(); + let max_w = tf.max_width(&to_display); + tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; + writeln!(f)?; + } + DType::I32 => { + let tf: IntFormatter = IntFormatter::new(); + let max_w = tf.max_width(&to_display); + tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; + writeln!(f)?; + } + DType::I64 => { + let tf: IntFormatter = IntFormatter::new(); + let max_w = tf.max_width(&to_display); + tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; + writeln!(f)?; + } + DType::BF16 => { + if let Ok(tf) = FloatFormatter::::new(&to_display, &po) { + let max_w = tf.max_width(&to_display); + tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; + writeln!(f)?; + } + } + DType::F16 => { + if let Ok(tf) = FloatFormatter::::new(&to_display, &po) { + let max_w = tf.max_width(&to_display); + tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; + writeln!(f)?; + } + } + DType::F64 => { + if let Ok(tf) = FloatFormatter::::new(&to_display, &po) { + let max_w = tf.max_width(&to_display); + tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; + writeln!(f)?; + } + } + DType::F32 => { + if let Ok(tf) = FloatFormatter::::new(&to_display, &po) { + let max_w = tf.max_width(&to_display); + tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; + writeln!(f)?; + } + } + DType::F8E4M3 => { + if let Ok(tf) = FloatFormatter::::new(&to_display, &po) { + let max_w = tf.max_width(&to_display); + tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; + writeln!(f)?; + } + } + DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { + writeln!( + f, + "Dummy type {} (not supported for display)", + self.dtype().as_str() + )?; + } + }; + + let device_str = match self.device().location() { + crate::DeviceLocation::Cpu => "".to_owned(), + crate::DeviceLocation::Cuda { gpu_id } => { + format!(", cuda:{gpu_id}") + } + crate::DeviceLocation::Metal { gpu_id } => { + format!(", metal:{gpu_id}") + } + }; + + write!( + f, + "Tensor[{:?}, {}{}]", + self.dims(), + self.dtype().as_str(), + device_str + ) + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/dtype.rs b/v2/dependency-patches/candle-core-0.9.2/src/dtype.rs new file mode 100644 index 0000000000..035ca6d503 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/dtype.rs @@ -0,0 +1,294 @@ +//! Types for elements that can be stored and manipulated using tensors. +#![allow(clippy::redundant_closure_call)] +use crate::backend::BackendStorage; +use crate::{CpuStorage, CpuStorageRef, Error, Result}; + +/// The different types of elements allowed in tensors. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub enum DType { + // Unsigned 8 bits integer. + U8, + // Unsigned 32 bits integer. + U32, + // Signed 16 bits integer. + I16, + // Signed 32 bits integer. + I32, + // Signed 64 bits integer. + I64, + // Brain floating-point using half precision (16 bits). + BF16, + // Floating-point using half precision (16 bits). + F16, + // Floating-point using single precision (32 bits). + F32, + // Floating-point using double precision (64 bits). + F64, + // 8-bit floating point with 4-bit exponent and 3-bit mantissa. + F8E4M3, + /// 6-bit float with 2 exponent bits and 3 mantissa bits (MX6 format) + F6E2M3, + /// 6-bit float with 3 exponent bits and 2 mantissa bits (MX6 format) + F6E3M2, + /// 4-bit float (MX4 format) + F4, + /// 8-bit float with 8 exponent bits and 0 mantissa bits + F8E8M0, +} + +#[derive(Debug, PartialEq, Eq)] +pub struct DTypeParseError(String); + +impl std::fmt::Display for DTypeParseError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "cannot parse '{}' as a dtype", self.0) + } +} + +impl std::error::Error for DTypeParseError {} + +impl std::str::FromStr for DType { + type Err = DTypeParseError; + fn from_str(s: &str) -> std::result::Result { + match s { + "u8" => Ok(Self::U8), + "u32" => Ok(Self::U32), + "i16" => Ok(Self::I16), + "i32" => Ok(Self::I32), + "i64" => Ok(Self::I64), + "bf16" => Ok(Self::BF16), + "f16" => Ok(Self::F16), + "f32" => Ok(Self::F32), + "f64" => Ok(Self::F64), + "f8e4m3" => Ok(Self::F8E4M3), + "f6e2m3" => Ok(Self::F6E2M3), + "f6e3m2" => Ok(Self::F6E3M2), + "f4" => Ok(Self::F4), + "f8e8m0" => Ok(Self::F8E8M0), + _ => Err(DTypeParseError(s.to_string())), + } + } +} + +impl DType { + /// String representation for dtypes. + pub fn as_str(&self) -> &'static str { + match self { + Self::U8 => "u8", + Self::U32 => "u32", + Self::I16 => "i16", + Self::I32 => "i32", + Self::I64 => "i64", + Self::BF16 => "bf16", + Self::F16 => "f16", + Self::F32 => "f32", + Self::F64 => "f64", + Self::F8E4M3 => "f8e4m3", + Self::F6E2M3 => "f6e2m3", + Self::F6E3M2 => "f6e3m2", + Self::F4 => "f4", + Self::F8E8M0 => "f8e8m0", + } + } + + /// The size used by each element in bytes, i.e. 1 for `U8`, 4 for `F32`. + pub fn size_in_bytes(&self) -> usize { + match self { + Self::U8 => 1, + Self::U32 => 4, + Self::I16 => 2, + Self::I32 => 4, + Self::I64 => 8, + Self::BF16 => 2, + Self::F16 => 2, + Self::F32 => 4, + Self::F64 => 8, + Self::F8E4M3 => 1, + Self::F6E2M3 => 0, // 6 bits + Self::F6E3M2 => 0, // 6 bits + Self::F4 => 0, // 4 bits + Self::F8E8M0 => 1, + } + } + + pub fn is_int(&self) -> bool { + match self { + Self::U8 | Self::U32 | Self::I16 | Self::I32 | Self::I64 => true, + Self::BF16 + | Self::F16 + | Self::F32 + | Self::F64 + | Self::F8E4M3 + | Self::F6E2M3 + | Self::F6E3M2 + | Self::F4 + | Self::F8E8M0 => false, + } + } + + pub fn is_float(&self) -> bool { + match self { + Self::U8 | Self::U32 | Self::I16 | Self::I32 | Self::I64 => false, + Self::BF16 + | Self::F16 + | Self::F32 + | Self::F64 + | Self::F8E4M3 + | Self::F6E2M3 + | Self::F6E3M2 + | Self::F4 + | Self::F8E8M0 => true, + } + } +} + +pub trait WithDType: + Sized + + Copy + + num_traits::NumAssign + + std::cmp::PartialOrd + + std::fmt::Display + + 'static + + Send + + Sync + + std::any::Any + + crate::cpu::kernels::VecOps +{ + const DTYPE: DType; + + fn from_f64(v: f64) -> Self; + fn to_f64(self) -> f64; + fn to_scalar(self) -> crate::scalar::Scalar; + fn cpu_storage_ref(data: &[Self]) -> CpuStorageRef<'_>; + fn to_cpu_storage_owned(data: Vec) -> CpuStorage; + + fn to_cpu_storage(data: &[Self]) -> CpuStorage { + Self::to_cpu_storage_owned(data.to_vec()) + } + + fn cpu_storage_as_slice(s: &CpuStorage) -> Result<&[Self]>; + fn cpu_storage_data(s: CpuStorage) -> Result>; +} + +macro_rules! with_dtype { + ($ty:ty, $dtype:ident, $from_f64:expr, $to_f64:expr) => { + impl WithDType for $ty { + const DTYPE: DType = DType::$dtype; + + fn from_f64(v: f64) -> Self { + $from_f64(v) + } + + fn to_f64(self) -> f64 { + $to_f64(self) + } + + fn to_scalar(self) -> crate::scalar::Scalar { + crate::scalar::Scalar::$dtype(self) + } + + fn cpu_storage_ref(data: &[Self]) -> CpuStorageRef<'_> { + CpuStorageRef::$dtype(data) + } + + fn to_cpu_storage_owned(data: Vec) -> CpuStorage { + CpuStorage::$dtype(data) + } + + fn cpu_storage_data(s: CpuStorage) -> Result> { + match s { + CpuStorage::$dtype(data) => Ok(data), + _ => Err(Error::UnexpectedDType { + expected: DType::$dtype, + got: s.dtype(), + msg: "unexpected dtype", + } + .bt()), + } + } + + fn cpu_storage_as_slice(s: &CpuStorage) -> Result<&[Self]> { + match s { + CpuStorage::$dtype(data) => Ok(data), + _ => Err(Error::UnexpectedDType { + expected: DType::$dtype, + got: s.dtype(), + msg: "unexpected dtype", + } + .bt()), + } + } + } + }; +} +use float8::F8E4M3 as f8e4m3; +use half::{bf16, f16}; + +with_dtype!(u8, U8, |v: f64| v as u8, |v: u8| v as f64); +with_dtype!(u32, U32, |v: f64| v as u32, |v: u32| v as f64); +with_dtype!(i16, I16, |v: f64| v as i16, |v: i16| v as f64); +with_dtype!(i32, I32, |v: f64| v as i32, |v: i32| v as f64); +with_dtype!(i64, I64, |v: f64| v as i64, |v: i64| v as f64); +with_dtype!(f16, F16, f16::from_f64, f16::to_f64); +with_dtype!(bf16, BF16, bf16::from_f64, bf16::to_f64); +with_dtype!(f32, F32, |v: f64| v as f32, |v: f32| v as f64); +with_dtype!(f64, F64, |v: f64| v, |v: f64| v); +with_dtype!(f8e4m3, F8E4M3, f8e4m3::from_f64, |v: f8e4m3| v.to_f64()); + +pub trait IntDType: WithDType + num_traits::Bounded { + fn is_true(&self) -> bool; + fn as_usize(&self) -> usize; +} + +impl IntDType for i64 { + fn is_true(&self) -> bool { + *self != 0 + } + fn as_usize(&self) -> usize { + *self as usize + } +} + +impl IntDType for u32 { + fn is_true(&self) -> bool { + *self != 0 + } + fn as_usize(&self) -> usize { + *self as usize + } +} + +impl IntDType for u8 { + fn is_true(&self) -> bool { + *self != 0 + } + fn as_usize(&self) -> usize { + *self as usize + } +} + +impl IntDType for i16 { + fn is_true(&self) -> bool { + *self != 0 + } + fn as_usize(&self) -> usize { + *self as usize + } +} + +impl IntDType for i32 { + fn is_true(&self) -> bool { + *self != 0 + } + fn as_usize(&self) -> usize { + *self as usize + } +} + +pub trait FloatDType: WithDType {} + +impl FloatDType for f16 {} +impl FloatDType for bf16 {} +impl FloatDType for f32 {} +impl FloatDType for f64 {} +impl FloatDType for f8e4m3 {} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/dummy_cuda_backend.rs b/v2/dependency-patches/candle-core-0.9.2/src/dummy_cuda_backend.rs new file mode 100644 index 0000000000..2aa4585966 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/dummy_cuda_backend.rs @@ -0,0 +1,317 @@ +//! Implementation of the Cuda backend when Cuda support has not been compiled in. +//! +#![allow(dead_code)] +use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; +use crate::{CpuStorage, DType, Error, Layout, Result, Shape}; + +#[derive(Debug, Clone)] +pub struct CudaDevice; + +#[derive(Debug)] +pub struct CudaStorage; + +impl CudaStorage { + pub fn transfer_to_device(&self, _dst: &CudaDevice) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } +} + +macro_rules! fail { + () => { + unimplemented!("cuda support has not been enabled, add `cuda` feature to enable.") + }; +} +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct DeviceId(usize); + +impl CudaDevice { + pub fn new_with_stream(_: usize) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + pub fn id(&self) -> DeviceId { + DeviceId(0) + } +} + +impl crate::backend::BackendStorage for CudaStorage { + type Device = CudaDevice; + + fn try_clone(&self, _: &Layout) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn dtype(&self) -> DType { + fail!() + } + + fn device(&self) -> &Self::Device { + fail!() + } + + fn const_set(&mut self, _: crate::scalar::Scalar, _: &Layout) -> Result<()> { + Err(Error::NotCompiledWithCudaSupport) + } + + fn to_cpu_storage(&self) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn affine(&self, _: &Layout, _: f64, _: f64) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn powf(&self, _: &Layout, _: f64) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn elu(&self, _: &Layout, _: f64) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn reduce_op(&self, _: ReduceOp, _: &Layout, _: &[usize]) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn cmp(&self, _: CmpOp, _: &Self, _: &Layout, _: &Layout) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn to_dtype(&self, _: &Layout, _: DType) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn unary_impl(&self, _: &Layout) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn binary_impl(&self, _: &Self, _: &Layout, _: &Layout) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn where_cond(&self, _: &Layout, _: &Self, _: &Layout, _: &Self, _: &Layout) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn conv1d( + &self, + _: &Layout, + _: &Self, + _: &Layout, + _: &crate::conv::ParamsConv1D, + ) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn conv_transpose1d( + &self, + _: &Layout, + _: &Self, + _: &Layout, + _: &crate::conv::ParamsConvTranspose1D, + ) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn conv2d( + &self, + _: &Layout, + _: &Self, + _: &Layout, + _: &crate::conv::ParamsConv2D, + ) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn conv_transpose2d( + &self, + _l: &Layout, + _kernel: &Self, + _kernel_l: &Layout, + _params: &crate::conv::ParamsConvTranspose2D, + ) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn index_select(&self, _: &Self, _: &Layout, _: &Layout, _: usize) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + fn gather(&self, _: &Layout, _: &Self, _: &Layout, _: usize) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn scatter_set( + &mut self, + _: &Layout, + _: &Self, + _: &Layout, + _: &Self, + _: &Layout, + _: usize, + ) -> Result<()> { + Err(Error::NotCompiledWithCudaSupport) + } + + fn scatter_add_set( + &mut self, + _: &Layout, + _: &Self, + _: &Layout, + _: &Self, + _: &Layout, + _: usize, + ) -> Result<()> { + Err(Error::NotCompiledWithCudaSupport) + } + + fn index_add( + &self, + _: &Layout, + _: &Self, + _: &Layout, + _: &Self, + _: &Layout, + _: usize, + ) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn matmul( + &self, + _: &Self, + _: (usize, usize, usize, usize), + _: &Layout, + _: &Layout, + ) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn copy_strided_src(&self, _: &mut Self, _: usize, _: &Layout) -> Result<()> { + Err(Error::NotCompiledWithCudaSupport) + } + + fn copy2d( + &self, + _: &mut Self, + _: usize, + _: usize, + _: usize, + _: usize, + _: usize, + _: usize, + ) -> Result<()> { + Err(Error::NotCompiledWithCudaSupport) + } + + fn avg_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn max_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn upsample_nearest1d(&self, _: &Layout, _: usize) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn upsample_nearest2d(&self, _: &Layout, _: usize, _: usize) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn upsample_bilinear2d( + &self, + _: &Layout, + _: usize, + _: usize, + _: bool, + _: Option, + _: Option, + ) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } +} + +impl crate::backend::BackendDevice for CudaDevice { + type Storage = CudaStorage; + fn new(_: usize) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn set_seed(&self, _: u64) -> Result<()> { + Err(Error::NotCompiledWithCudaSupport) + } + + fn get_current_seed(&self) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn location(&self) -> crate::DeviceLocation { + fail!() + } + + fn same_device(&self, _: &Self) -> bool { + fail!() + } + + fn zeros_impl(&self, _shape: &Shape, _dtype: DType) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + unsafe fn alloc_uninit(&self, _shape: &Shape, _dtype: DType) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn storage_from_slice(&self, _: &[T]) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn storage_from_cpu_storage(&self, _: &CpuStorage) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn storage_from_cpu_storage_owned(&self, _: CpuStorage) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn rand_uniform(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn rand_normal(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + fn synchronize(&self) -> Result<()> { + Ok(()) + } +} + +/// This bool controls whether reduced precision reductions (e.g., with fp16 accumulation type) are +/// allowed with f16 GEMMs. +pub fn gemm_reduced_precision_f16() -> bool { + true +} + +/// This bool controls whether reduced precision reductions (e.g., with fp16 accumulation type) are +/// allowed with f16 GEMMs. +pub fn set_gemm_reduced_precision_f16(_: bool) {} + +/// This bool controls whether reduced precision reductions (e.g., with fp16 accumulation type) are +/// allowed with bf16 GEMMs. +pub fn gemm_reduced_precision_bf16() -> bool { + true +} + +/// This bool controls whether reduced precision reductions (e.g., with fp16 accumulation type) are +/// allowed with bf16 GEMMs. +pub fn set_gemm_reduced_precision_bf16(_: bool) {} + +/// This bool controls whether reduced precision reductions (e.g., with tf32 accumulation type) are +/// allowed with f32 GEMMs. +pub fn gemm_reduced_precision_f32() -> bool { + true +} + +/// This bool controls whether reduced precision reductions (e.g., with tf32 accumulation type) are +/// allowed with f32 GEMMs. +pub fn set_gemm_reduced_precision_f32(_b: bool) {} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/dummy_dtype.rs b/v2/dependency-patches/candle-core-0.9.2/src/dummy_dtype.rs new file mode 100644 index 0000000000..5fdb0961a8 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/dummy_dtype.rs @@ -0,0 +1,268 @@ +//! Dummy data types for experimental/future float formats +//! +//! These are placeholder types for experimental floating-point formats +//! that are defined in the safetensors spec but not yet fully implemented. + +use crate::{DType, Error, Result, WithDType}; + +/// 6-bit float with 2 exponent bits and 3 mantissa bits (MX6 format) +/// This is a dummy type. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct F6E2M3; + +/// 6-bit float with 3 exponent bits and 2 mantissa bits (MX6 format) +/// This is a dummy type. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct F6E3M2; + +/// 4-bit float (MX4 format) +/// This is a dummy type. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct F4; + +/// 8-bit float with 8 exponent bits and 0 mantissa bits +/// This is a dummy type. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct F8E8M0; + +// Implement WithDType for dummy types +macro_rules! dummy_with_dtype { + ($ty:ty, $dtype:ident) => { + impl WithDType for $ty { + const DTYPE: DType = DType::$dtype; + + fn from_f64(_v: f64) -> Self { + panic!( + "{} is a dummy type and cannot be constructed", + stringify!($ty) + ) + } + + fn to_f64(self) -> f64 { + panic!( + "{} is a dummy type and cannot be converted", + stringify!($ty) + ) + } + + fn to_scalar(self) -> crate::scalar::Scalar { + panic!( + "{} is a dummy type and cannot be converted to scalar", + stringify!($ty) + ) + } + + fn cpu_storage_ref(_data: &[Self]) -> crate::CpuStorageRef<'_> { + panic!( + "{} is a dummy type and does not support storage", + stringify!($ty) + ) + } + + fn to_cpu_storage_owned(_data: Vec) -> crate::CpuStorage { + panic!( + "{} is a dummy type and does not support storage", + stringify!($ty) + ) + } + + fn cpu_storage_data(_s: crate::CpuStorage) -> Result> { + Err(Error::UnsupportedDTypeForOp(DType::$dtype, "cpu_storage_data").bt()) + } + + fn cpu_storage_as_slice(_s: &crate::CpuStorage) -> Result<&[Self]> { + Err(Error::UnsupportedDTypeForOp(DType::$dtype, "cpu_storage_as_slice").bt()) + } + } + }; +} + +dummy_with_dtype!(F6E2M3, F6E2M3); +dummy_with_dtype!(F6E3M2, F6E3M2); +dummy_with_dtype!(F4, F4); +dummy_with_dtype!(F8E8M0, F8E8M0); + +// Implement NumAssign traits for dummy types +macro_rules! dummy_num_assign { + ($ty:ty) => { + impl std::ops::AddAssign for $ty { + fn add_assign(&mut self, _other: Self) { + panic!( + "{} is a dummy type and does not support operations", + stringify!($ty) + ) + } + } + + impl std::ops::SubAssign for $ty { + fn sub_assign(&mut self, _other: Self) { + panic!( + "{} is a dummy type and does not support operations", + stringify!($ty) + ) + } + } + + impl std::ops::MulAssign for $ty { + fn mul_assign(&mut self, _other: Self) { + panic!( + "{} is a dummy type and does not support operations", + stringify!($ty) + ) + } + } + + impl std::ops::DivAssign for $ty { + fn div_assign(&mut self, _other: Self) { + panic!( + "{} is a dummy type and does not support operations", + stringify!($ty) + ) + } + } + + impl std::ops::RemAssign for $ty { + fn rem_assign(&mut self, _other: Self) { + panic!( + "{} is a dummy type and does not support operations", + stringify!($ty) + ) + } + } + + impl std::ops::Add for $ty { + type Output = Self; + fn add(self, _other: Self) -> Self { + panic!( + "{} is a dummy type and does not support operations", + stringify!($ty) + ) + } + } + + impl std::ops::Sub for $ty { + type Output = Self; + fn sub(self, _other: Self) -> Self { + panic!( + "{} is a dummy type and does not support operations", + stringify!($ty) + ) + } + } + + impl std::ops::Mul for $ty { + type Output = Self; + fn mul(self, _other: Self) -> Self { + panic!( + "{} is a dummy type and does not support operations", + stringify!($ty) + ) + } + } + + impl std::ops::Div for $ty { + type Output = Self; + fn div(self, _other: Self) -> Self { + panic!( + "{} is a dummy type and does not support operations", + stringify!($ty) + ) + } + } + + impl std::ops::Rem for $ty { + type Output = Self; + fn rem(self, _other: Self) -> Self { + panic!( + "{} is a dummy type and does not support operations", + stringify!($ty) + ) + } + } + + impl num_traits::Zero for $ty { + fn zero() -> Self { + panic!( + "{} is a dummy type and does not support operations", + stringify!($ty) + ) + } + + fn is_zero(&self) -> bool { + panic!( + "{} is a dummy type and does not support operations", + stringify!($ty) + ) + } + } + + impl num_traits::One for $ty { + fn one() -> Self { + panic!( + "{} is a dummy type and does not support operations", + stringify!($ty) + ) + } + } + + impl num_traits::Num for $ty { + type FromStrRadixErr = std::num::ParseFloatError; + + fn from_str_radix( + _str: &str, + _radix: u32, + ) -> std::result::Result { + panic!( + "{} is a dummy type and does not support parsing", + stringify!($ty) + ) + } + } + + impl crate::cpu::kernels::VecOps for $ty { + fn min(self, _other: Self) -> Self { + panic!( + "{} is a dummy type and does not support operations", + stringify!($ty) + ) + } + + fn max(self, _other: Self) -> Self { + panic!( + "{} is a dummy type and does not support operations", + stringify!($ty) + ) + } + } + }; +} + +dummy_num_assign!(F6E2M3); +dummy_num_assign!(F6E3M2); +dummy_num_assign!(F4); +dummy_num_assign!(F8E8M0); + +// Display implementations +impl std::fmt::Display for F6E2M3 { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "F6E2M3") + } +} + +impl std::fmt::Display for F6E3M2 { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "F6E3M2") + } +} + +impl std::fmt::Display for F4 { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "F4") + } +} + +impl std::fmt::Display for F8E8M0 { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "F8E8M0") + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/dummy_metal_backend.rs b/v2/dependency-patches/candle-core-0.9.2/src/dummy_metal_backend.rs new file mode 100644 index 0000000000..8c23b580fc --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/dummy_metal_backend.rs @@ -0,0 +1,280 @@ +#![allow(dead_code)] +use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; +use crate::{CpuStorage, DType, Error, Layout, Result, Shape}; + +#[derive(Debug, Clone)] +pub struct MetalDevice; + +#[derive(Debug)] +pub struct MetalStorage; + +#[derive(thiserror::Error, Debug)] +pub enum MetalError { + #[error("{0}")] + Message(String), +} + +impl From for MetalError { + fn from(e: String) -> Self { + MetalError::Message(e) + } +} + +macro_rules! fail { + () => { + unimplemented!("metal support has not been enabled, add `metal` feature to enable.") + }; +} + +impl crate::backend::BackendStorage for MetalStorage { + type Device = MetalDevice; + + fn try_clone(&self, _: &Layout) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn dtype(&self) -> DType { + fail!() + } + + fn device(&self) -> &Self::Device { + fail!() + } + + fn const_set(&mut self, _: crate::scalar::Scalar, _: &Layout) -> Result<()> { + Err(Error::NotCompiledWithMetalSupport) + } + + fn to_cpu_storage(&self) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn affine(&self, _: &Layout, _: f64, _: f64) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn powf(&self, _: &Layout, _: f64) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn elu(&self, _: &Layout, _: f64) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn reduce_op(&self, _: ReduceOp, _: &Layout, _: &[usize]) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn cmp(&self, _: CmpOp, _: &Self, _: &Layout, _: &Layout) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn to_dtype(&self, _: &Layout, _: DType) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn unary_impl(&self, _: &Layout) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn binary_impl(&self, _: &Self, _: &Layout, _: &Layout) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn where_cond(&self, _: &Layout, _: &Self, _: &Layout, _: &Self, _: &Layout) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn conv1d( + &self, + _: &Layout, + _: &Self, + _: &Layout, + _: &crate::conv::ParamsConv1D, + ) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn conv_transpose1d( + &self, + _l: &Layout, + _kernel: &Self, + _kernel_l: &Layout, + _params: &crate::conv::ParamsConvTranspose1D, + ) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn conv2d( + &self, + _: &Layout, + _: &Self, + _: &Layout, + _: &crate::conv::ParamsConv2D, + ) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn conv_transpose2d( + &self, + _l: &Layout, + _kernel: &Self, + _kernel_l: &Layout, + _params: &crate::conv::ParamsConvTranspose2D, + ) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn index_select(&self, _: &Self, _: &Layout, _: &Layout, _: usize) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + fn gather(&self, _: &Layout, _: &Self, _: &Layout, _: usize) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn scatter_set( + &mut self, + _: &Layout, + _: &Self, + _: &Layout, + _: &Self, + _: &Layout, + _: usize, + ) -> Result<()> { + Err(Error::NotCompiledWithMetalSupport) + } + + fn scatter_add_set( + &mut self, + _: &Layout, + _: &Self, + _: &Layout, + _: &Self, + _: &Layout, + _: usize, + ) -> Result<()> { + Err(Error::NotCompiledWithMetalSupport) + } + + fn index_add( + &self, + _: &Layout, + _: &Self, + _: &Layout, + _: &Self, + _: &Layout, + _: usize, + ) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn matmul( + &self, + _: &Self, + _: (usize, usize, usize, usize), + _: &Layout, + _: &Layout, + ) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn copy_strided_src(&self, _: &mut Self, _: usize, _: &Layout) -> Result<()> { + Err(Error::NotCompiledWithMetalSupport) + } + + fn copy2d( + &self, + _: &mut Self, + _: usize, + _: usize, + _: usize, + _: usize, + _: usize, + _: usize, + ) -> Result<()> { + Err(Error::NotCompiledWithMetalSupport) + } + + fn avg_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn max_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn upsample_nearest1d(&self, _: &Layout, _: usize) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn upsample_nearest2d(&self, _: &Layout, _: usize, _: usize) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn upsample_bilinear2d( + &self, + _: &Layout, + _: usize, + _: usize, + _: bool, + _: Option, + _: Option, + ) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } +} + +impl crate::backend::BackendDevice for MetalDevice { + type Storage = MetalStorage; + fn new(_: usize) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn set_seed(&self, _: u64) -> Result<()> { + Err(Error::NotCompiledWithMetalSupport) + } + + fn get_current_seed(&self) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn location(&self) -> crate::DeviceLocation { + fail!() + } + + fn same_device(&self, _: &Self) -> bool { + fail!() + } + + fn zeros_impl(&self, _shape: &Shape, _dtype: DType) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + unsafe fn alloc_uninit(&self, _shape: &Shape, _dtype: DType) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn storage_from_slice(&self, _: &[T]) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn storage_from_cpu_storage(&self, _: &CpuStorage) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn storage_from_cpu_storage_owned(&self, _: CpuStorage) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn rand_uniform(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn rand_normal(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + fn synchronize(&self) -> Result<()> { + Ok(()) + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/error.rs b/v2/dependency-patches/candle-core-0.9.2/src/error.rs new file mode 100644 index 0000000000..9f10ca29cf --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/error.rs @@ -0,0 +1,394 @@ +//! Candle-specific Error and Result +use std::{convert::Infallible, fmt::Display}; + +use crate::{DType, DeviceLocation, Layout, MetalError, Shape}; + +#[derive(Debug, Clone)] +pub struct MatMulUnexpectedStriding { + pub lhs_l: Layout, + pub rhs_l: Layout, + pub bmnk: (usize, usize, usize, usize), + pub msg: &'static str, +} + +impl std::fmt::Debug for Error { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{self}") + } +} + +/// Main library error type. +#[derive(thiserror::Error)] +pub enum Error { + // === DType Errors === + #[error("{msg}, expected: {expected:?}, got: {got:?}")] + UnexpectedDType { + msg: &'static str, + expected: DType, + got: DType, + }, + + #[error("dtype mismatch in {op}, lhs: {lhs:?}, rhs: {rhs:?}")] + DTypeMismatchBinaryOp { + lhs: DType, + rhs: DType, + op: &'static str, + }, + + #[error("unsupported dtype {0:?} for op {1}")] + UnsupportedDTypeForOp(DType, &'static str), + + // === Dimension Index Errors === + #[error("{op}: dimension index {dim} out of range for shape {shape:?}")] + DimOutOfRange { + shape: Shape, + dim: i32, + op: &'static str, + }, + + #[error("{op}: duplicate dim index {dims:?} for shape {shape:?}")] + DuplicateDimIndex { + shape: Shape, + dims: Vec, + op: &'static str, + }, + + // === Shape Errors === + #[error("unexpected rank, expected: {expected}, got: {got} ({shape:?})")] + UnexpectedNumberOfDims { + expected: usize, + got: usize, + shape: Shape, + }, + + #[error("{msg}, expected: {expected:?}, got: {got:?}")] + UnexpectedShape { + msg: String, + expected: Shape, + got: Shape, + }, + + #[error( + "Shape mismatch, got buffer of size {buffer_size} which is compatible with shape {shape:?}" + )] + ShapeMismatch { buffer_size: usize, shape: Shape }, + + #[error("shape mismatch in {op}, lhs: {lhs:?}, rhs: {rhs:?}")] + ShapeMismatchBinaryOp { + lhs: Shape, + rhs: Shape, + op: &'static str, + }, + + #[error("shape mismatch in cat for dim {dim}, shape for arg 1: {first_shape:?} shape for arg {n}: {nth_shape:?}")] + ShapeMismatchCat { + dim: usize, + first_shape: Shape, + n: usize, + nth_shape: Shape, + }, + + #[error("Cannot divide tensor of shape {shape:?} equally along dim {dim} into {n_parts}")] + ShapeMismatchSplit { + shape: Shape, + dim: usize, + n_parts: usize, + }, + + #[error("{op} can only be performed on a single dimension")] + OnlySingleDimension { op: &'static str, dims: Vec }, + + #[error("empty tensor for {op}")] + EmptyTensor { op: &'static str }, + + // === Device Errors === + #[error("device mismatch in {op}, lhs: {lhs:?}, rhs: {rhs:?}")] + DeviceMismatchBinaryOp { + lhs: DeviceLocation, + rhs: DeviceLocation, + op: &'static str, + }, + + // === Op Specific Errors === + #[error("narrow invalid args {msg}: {shape:?}, dim: {dim}, start: {start}, len:{len}")] + NarrowInvalidArgs { + shape: Shape, + dim: usize, + start: usize, + len: usize, + msg: &'static str, + }, + + #[error("conv1d invalid args {msg}: inp: {inp_shape:?}, k: {k_shape:?}, pad: {padding}, stride: {stride}")] + Conv1dInvalidArgs { + inp_shape: Shape, + k_shape: Shape, + padding: usize, + stride: usize, + msg: &'static str, + }, + + #[error("{op} invalid index {index} with dim size {size}")] + InvalidIndex { + op: &'static str, + index: usize, + size: usize, + }, + + #[error("cannot broadcast {src_shape:?} to {dst_shape:?}")] + BroadcastIncompatibleShapes { src_shape: Shape, dst_shape: Shape }, + + #[error("cannot set variable {msg}")] + CannotSetVar { msg: &'static str }, + + // Box indirection to avoid large variant. + #[error("{0:?}")] + MatMulUnexpectedStriding(Box), + + #[error("{op} only supports contiguous tensors")] + RequiresContiguous { op: &'static str }, + + #[error("{op} expects at least one tensor")] + OpRequiresAtLeastOneTensor { op: &'static str }, + + #[error("{op} expects at least two tensors")] + OpRequiresAtLeastTwoTensors { op: &'static str }, + + #[error("backward is not supported for {op}")] + BackwardNotSupported { op: &'static str }, + + // === Other Errors === + #[error("the candle crate has not been built with cuda support")] + NotCompiledWithCudaSupport, + + #[error("the candle crate has not been built with metal support")] + NotCompiledWithMetalSupport, + + #[error("cannot find tensor {path}")] + CannotFindTensor { path: String }, + + // === Wrapped Errors === + #[error(transparent)] + Cuda(Box), + + #[error("Metal error {0}")] + Metal(#[from] MetalError), + + #[cfg(all(not(target_arch = "wasm32"), not(target_os = "ios"), feature = "ug"))] + #[error(transparent)] + Ug(#[from] candle_ug::Error), + + #[error(transparent)] + TryFromIntError(#[from] core::num::TryFromIntError), + + #[error("npy/npz error {0}")] + Npy(String), + + /// Zip file format error. + #[error(transparent)] + Zip(#[from] zip::result::ZipError), + + /// Integer parse error. + #[error(transparent)] + ParseInt(#[from] std::num::ParseIntError), + + /// Utf8 parse error. + #[error(transparent)] + FromUtf8(#[from] std::string::FromUtf8Error), + + /// I/O error. + #[error(transparent)] + Io(#[from] std::io::Error), + + /// SafeTensor error. + #[error(transparent)] + SafeTensor(#[from] safetensors::SafeTensorError), + + #[error("unsupported safetensor dtype {0:?}")] + UnsupportedSafeTensorDtype(safetensors::Dtype), + + /// Arbitrary errors wrapping. + #[error("{0}")] + Wrapped(Box), + + /// Arbitrary errors wrapping with context. + #[error("{wrapped:?}\n{context:?}")] + WrappedContext { + wrapped: Box, + context: String, + }, + + #[error("{context}\n{inner}")] + Context { + inner: Box, + context: Box, + }, + + /// Adding path information to an error. + #[error("path: {path:?} {inner}")] + WithPath { + inner: Box, + path: std::path::PathBuf, + }, + + #[error("{inner}\n{backtrace}")] + WithBacktrace { + inner: Box, + backtrace: Box, + }, + + /// User generated error message, typically created via `bail!`. + #[error("{0}")] + Msg(String), + + #[error("unwrap none")] + UnwrapNone, +} + +pub type Result = std::result::Result; + +impl Error { + pub fn wrap(err: impl std::fmt::Display + Send + Sync + 'static) -> Self { + Self::Wrapped(Box::new(err)).bt() + } + + pub fn msg(err: impl std::fmt::Display) -> Self { + Self::Msg(err.to_string()).bt() + } + + pub fn debug(err: impl std::fmt::Debug) -> Self { + Self::Msg(format!("{err:?}")).bt() + } + + pub fn bt(self) -> Self { + let backtrace = std::backtrace::Backtrace::capture(); + match backtrace.status() { + std::backtrace::BacktraceStatus::Disabled + | std::backtrace::BacktraceStatus::Unsupported => self, + _ => Self::WithBacktrace { + inner: Box::new(self), + backtrace: Box::new(backtrace), + }, + } + } + + pub fn with_path>(self, p: P) -> Self { + Self::WithPath { + inner: Box::new(self), + path: p.as_ref().to_path_buf(), + } + } + + pub fn context(self, c: impl std::fmt::Display + Send + Sync + 'static) -> Self { + Self::Context { + inner: Box::new(self), + context: Box::new(c), + } + } +} + +#[macro_export] +macro_rules! bail { + ($msg:literal $(,)?) => { + return Err($crate::Error::Msg(format!($msg).into()).bt()) + }; + ($err:expr $(,)?) => { + return Err($crate::Error::Msg(format!($err).into()).bt()) + }; + ($fmt:expr, $($arg:tt)*) => { + return Err($crate::Error::Msg(format!($fmt, $($arg)*).into()).bt()) + }; +} + +pub fn zip(r1: Result, r2: Result) -> Result<(T, U)> { + match (r1, r2) { + (Ok(r1), Ok(r2)) => Ok((r1, r2)), + (Err(e), _) => Err(e), + (_, Err(e)) => Err(e), + } +} + +pub(crate) mod private { + pub trait Sealed {} + + impl Sealed for std::result::Result where E: std::error::Error {} + impl Sealed for Option {} +} + +/// Attach more context to an error. +/// +/// Inspired by [`anyhow::Context`]. +pub trait Context: private::Sealed { + /// Wrap the error value with additional context. + fn context(self, context: C) -> std::result::Result + where + C: Display + Send + Sync + 'static; + + /// Wrap the error value with additional context that is evaluated lazily + /// only once an error does occur. + fn with_context(self, f: F) -> std::result::Result + where + C: Display + Send + Sync + 'static, + F: FnOnce() -> C; +} + +impl Context for std::result::Result +where + E: std::error::Error + Send + Sync + 'static, +{ + fn context(self, context: C) -> std::result::Result + where + C: Display + Send + Sync + 'static, + { + // Not using map_err to save 2 useless frames off the captured backtrace + // in ext_context. + match self { + Ok(ok) => Ok(ok), + Err(error) => Err(Error::WrappedContext { + wrapped: Box::new(error), + context: context.to_string(), + } + .bt()), + } + } + + fn with_context(self, context: F) -> std::result::Result + where + C: Display + Send + Sync + 'static, + F: FnOnce() -> C, + { + match self { + Ok(ok) => Ok(ok), + Err(error) => Err(Error::WrappedContext { + wrapped: Box::new(error), + context: context().to_string(), + } + .bt()), + } + } +} + +impl Context for Option { + fn context(self, context: C) -> std::result::Result + where + C: Display + Send + Sync + 'static, + { + // Not using ok_or_else to save 2 useless frames off the captured + // backtrace. + match self { + Some(ok) => Ok(ok), + None => Err(Error::msg(context).bt()), + } + } + + fn with_context(self, context: F) -> std::result::Result + where + C: Display + Send + Sync + 'static, + F: FnOnce() -> C, + { + match self { + Some(v) => Ok(v), + None => Err(Error::UnwrapNone.context(context()).bt()), + } + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/indexer.rs b/v2/dependency-patches/candle-core-0.9.2/src/indexer.rs new file mode 100644 index 0000000000..d6cd6debf8 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/indexer.rs @@ -0,0 +1,257 @@ +use crate::{Error, Tensor}; +use std::ops::{ + Bound, Range, RangeBounds, RangeFrom, RangeFull, RangeInclusive, RangeTo, RangeToInclusive, +}; + +impl Tensor { + /// Intended to be use by the trait `.i()` + /// + /// ``` + /// # use candle_core::{Tensor, DType, Device, IndexOp}; + /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; + /// + /// let c = a.i(0..1)?; + /// assert_eq!(c.shape().dims(), &[1, 3]); + /// + /// let c = a.i(0)?; + /// assert_eq!(c.shape().dims(), &[3]); + /// + /// let c = a.i((.., ..2) )?; + /// assert_eq!(c.shape().dims(), &[2, 2]); + /// + /// let c = a.i((.., ..=2))?; + /// assert_eq!(c.shape().dims(), &[2, 3]); + /// + /// # Ok::<(), candle_core::Error>(()) + /// ``` + fn index(&self, indexers: &[TensorIndexer]) -> Result { + let mut x = self.clone(); + let dims = self.shape().dims(); + let mut current_dim = 0; + for (i, indexer) in indexers.iter().enumerate() { + x = match indexer { + TensorIndexer::Select(n) => x.narrow(current_dim, *n, 1)?.squeeze(current_dim)?, + TensorIndexer::Narrow(left_bound, right_bound) => { + let start = match left_bound { + Bound::Included(n) => *n, + Bound::Excluded(n) => *n + 1, + Bound::Unbounded => 0, + }; + let stop = match right_bound { + Bound::Included(n) => *n + 1, + Bound::Excluded(n) => *n, + Bound::Unbounded => dims[i], + }; + let out = x.narrow(current_dim, start, stop.saturating_sub(start))?; + current_dim += 1; + out + } + TensorIndexer::IndexSelect(indexes) => { + if indexes.rank() != 1 { + crate::bail!("multi-dimensional tensor indexing is not supported") + } + let out = x.index_select(&indexes.to_device(x.device())?, current_dim)?; + current_dim += 1; + out + } + TensorIndexer::Err(e) => crate::bail!("indexing error {e:?}"), + }; + } + Ok(x) + } +} + +#[derive(Debug)] +/// Generic structure used to index a slice of the tensor +pub enum TensorIndexer { + /// This selects the elements for which an index has some specific value. + Select(usize), + /// This is a regular slice, purely indexing a chunk of the tensor + Narrow(Bound, Bound), + /// Indexing via a 1d tensor + IndexSelect(Tensor), + Err(Error), +} + +impl From for TensorIndexer { + fn from(index: usize) -> Self { + TensorIndexer::Select(index) + } +} + +impl From<&[u32]> for TensorIndexer { + fn from(index: &[u32]) -> Self { + match Tensor::new(index, &crate::Device::Cpu) { + Ok(tensor) => TensorIndexer::IndexSelect(tensor), + Err(e) => TensorIndexer::Err(e), + } + } +} + +impl From> for TensorIndexer { + fn from(index: Vec) -> Self { + let len = index.len(); + match Tensor::from_vec(index, len, &crate::Device::Cpu) { + Ok(tensor) => TensorIndexer::IndexSelect(tensor), + Err(e) => TensorIndexer::Err(e), + } + } +} + +impl From<&Tensor> for TensorIndexer { + fn from(tensor: &Tensor) -> Self { + TensorIndexer::IndexSelect(tensor.clone()) + } +} + +trait RB: RangeBounds {} +impl RB for Range {} +impl RB for RangeFrom {} +impl RB for RangeFull {} +impl RB for RangeInclusive {} +impl RB for RangeTo {} +impl RB for RangeToInclusive {} + +impl From for TensorIndexer { + fn from(range: T) -> Self { + use std::ops::Bound::*; + let start = match range.start_bound() { + Included(idx) => Included(*idx), + Excluded(idx) => Excluded(*idx), + Unbounded => Unbounded, + }; + let end = match range.end_bound() { + Included(idx) => Included(*idx), + Excluded(idx) => Excluded(*idx), + Unbounded => Unbounded, + }; + TensorIndexer::Narrow(start, end) + } +} + +/// Trait used to implement multiple signatures for ease of use of the slicing +/// of a tensor +pub trait IndexOp { + /// Returns a slicing iterator which are the chunks of data necessary to + /// reconstruct the desired tensor. + fn i(&self, index: T) -> Result; +} + +impl IndexOp for Tensor +where + T: Into, +{ + ///```rust + /// use candle_core::{Tensor, DType, Device, IndexOp}; + /// let a = Tensor::new(&[ + /// [0., 1.], + /// [2., 3.], + /// [4., 5.] + /// ], &Device::Cpu)?; + /// + /// let b = a.i(0)?; + /// assert_eq!(b.shape().dims(), &[2]); + /// assert_eq!(b.to_vec1::()?, &[0., 1.]); + /// + /// let c = a.i(..2)?; + /// assert_eq!(c.shape().dims(), &[2, 2]); + /// assert_eq!(c.to_vec2::()?, &[ + /// [0., 1.], + /// [2., 3.] + /// ]); + /// + /// let d = a.i(1..)?; + /// assert_eq!(d.shape().dims(), &[2, 2]); + /// assert_eq!(d.to_vec2::()?, &[ + /// [2., 3.], + /// [4., 5.] + /// ]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + fn i(&self, index: T) -> Result { + self.index(&[index.into()]) + } +} + +impl IndexOp<(A,)> for Tensor +where + A: Into, +{ + ///```rust + /// use candle_core::{Tensor, DType, Device, IndexOp}; + /// let a = Tensor::new(&[ + /// [0f32, 1.], + /// [2. , 3.], + /// [4. , 5.] + /// ], &Device::Cpu)?; + /// + /// let b = a.i((0,))?; + /// assert_eq!(b.shape().dims(), &[2]); + /// assert_eq!(b.to_vec1::()?, &[0., 1.]); + /// + /// let c = a.i((..2,))?; + /// assert_eq!(c.shape().dims(), &[2, 2]); + /// assert_eq!(c.to_vec2::()?, &[ + /// [0., 1.], + /// [2., 3.] + /// ]); + /// + /// let d = a.i((1..,))?; + /// assert_eq!(d.shape().dims(), &[2, 2]); + /// assert_eq!(d.to_vec2::()?, &[ + /// [2., 3.], + /// [4., 5.] + /// ]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + fn i(&self, (a,): (A,)) -> Result { + self.index(&[a.into()]) + } +} +#[allow(non_snake_case)] +impl IndexOp<(A, B)> for Tensor +where + A: Into, + B: Into, +{ + ///```rust + /// use candle_core::{Tensor, DType, Device, IndexOp}; + /// let a = Tensor::new(&[[0f32, 1., 2.], [3., 4., 5.], [6., 7., 8.]], &Device::Cpu)?; + /// + /// let b = a.i((1, 0))?; + /// assert_eq!(b.to_vec0::()?, 3.); + /// + /// let c = a.i((..2, 1))?; + /// assert_eq!(c.shape().dims(), &[2]); + /// assert_eq!(c.to_vec1::()?, &[1., 4.]); + /// + /// let d = a.i((2.., ..))?; + /// assert_eq!(d.shape().dims(), &[1, 3]); + /// assert_eq!(d.to_vec2::()?, &[[6., 7., 8.]]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + fn i(&self, (a, b): (A, B)) -> Result { + self.index(&[a.into(), b.into()]) + } +} + +macro_rules! index_op_tuple { + ($doc:tt, $($t:ident),+) => { + #[allow(non_snake_case)] + impl<$($t),*> IndexOp<($($t,)*)> for Tensor + where + $($t: Into,)* + { + #[doc=$doc] + fn i(&self, ($($t,)*): ($($t,)*)) -> Result { + self.index(&[$($t.into(),)*]) + } + } + }; +} + +index_op_tuple!("see [TensorIndex#method.i]", A, B, C); +index_op_tuple!("see [TensorIndex#method.i]", A, B, C, D); +index_op_tuple!("see [TensorIndex#method.i]", A, B, C, D, E); +index_op_tuple!("see [TensorIndex#method.i]", A, B, C, D, E, F); +index_op_tuple!("see [TensorIndex#method.i]", A, B, C, D, E, F, G); diff --git a/v2/dependency-patches/candle-core-0.9.2/src/layout.rs b/v2/dependency-patches/candle-core-0.9.2/src/layout.rs new file mode 100644 index 0000000000..91e50481ec --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/layout.rs @@ -0,0 +1,278 @@ +//! Tensor Layouts including contiguous or sparse strides +use crate::{Error, Result, Shape}; + +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct Layout { + shape: Shape, + // The strides are given in number of elements and not in bytes. + stride: Vec, + start_offset: usize, +} + +impl Layout { + pub fn new(shape: Shape, stride: Vec, start_offset: usize) -> Self { + Self { + shape, + stride, + start_offset, + } + } + + pub fn contiguous_with_offset>(shape: S, start_offset: usize) -> Self { + let shape = shape.into(); + let stride = shape.stride_contiguous(); + Self { + shape, + stride, + start_offset, + } + } + + pub fn contiguous>(shape: S) -> Self { + Self::contiguous_with_offset(shape, 0) + } + + pub fn dims(&self) -> &[usize] { + self.shape.dims() + } + + /// The dimension size for a specified dimension index. + pub fn dim(&self, dim: D) -> Result { + let dim = dim.to_index(&self.shape, "dim")?; + Ok(self.dims()[dim]) + } + + pub fn shape(&self) -> &Shape { + &self.shape + } + + pub fn stride(&self) -> &[usize] { + &self.stride + } + + pub fn start_offset(&self) -> usize { + self.start_offset + } + + /// Returns the appropriate start and stop offset if the data is stored in a C + /// contiguous (aka row major) way. + pub fn contiguous_offsets(&self) -> Option<(usize, usize)> { + if self.is_contiguous() { + let start_o = self.start_offset; + Some((start_o, start_o + self.shape.elem_count())) + } else { + None + } + } + + /// Returns true if the data is stored in a C contiguous (aka row major) way. + /// Note that this does not implies that the start offset is 0 or that there are no extra + /// elements at the end of the storage. + pub fn is_contiguous(&self) -> bool { + self.shape.is_contiguous(&self.stride) + } + + /// Returns true if the data is stored in a Fortran contiguous (aka column major) way. + pub fn is_fortran_contiguous(&self) -> bool { + self.shape.is_fortran_contiguous(&self.stride) + } + + pub fn narrow(&self, dim: usize, start: usize, len: usize) -> Result { + let dims = self.shape().dims(); + if dim >= dims.len() { + Err(Error::DimOutOfRange { + shape: self.shape().clone(), + dim: dim as i32, + op: "narrow", + } + .bt())? + } + if start + len > dims[dim] { + Err(Error::NarrowInvalidArgs { + shape: self.shape.clone(), + dim, + start, + len, + msg: "start + len > dim_len", + } + .bt())? + } + let mut dims = dims.to_vec(); + dims[dim] = len; + Ok(Self { + shape: Shape::from(dims), + stride: self.stride.clone(), + start_offset: self.start_offset + self.stride[dim] * start, + }) + } + + pub fn transpose(&self, dim1: usize, dim2: usize) -> Result { + let rank = self.shape.rank(); + if rank <= dim1 || rank <= dim2 { + Err(Error::UnexpectedNumberOfDims { + expected: usize::max(dim1, dim2), + got: rank, + shape: self.shape().clone(), + } + .bt())? + } + let mut stride = self.stride().to_vec(); + let mut dims = self.shape().dims().to_vec(); + dims.swap(dim1, dim2); + stride.swap(dim1, dim2); + Ok(Self { + shape: Shape::from(dims), + stride, + start_offset: self.start_offset, + }) + } + + pub fn permute(&self, idxs: &[usize]) -> Result { + let is_permutation = + idxs.len() == self.shape.rank() && (0..idxs.len()).all(|i| idxs.contains(&i)); + if !is_permutation { + crate::bail!( + "dimension mismatch in permute, tensor {:?}, dims: {:?}", + self.dims(), + idxs + ) + } + let stride = self.stride(); + let dims = self.shape().dims(); + let mut perm_stride = stride.to_vec(); + let mut perm_dims = dims.to_vec(); + for (i, &idx) in idxs.iter().enumerate() { + perm_stride[i] = stride[idx]; + perm_dims[i] = dims[idx]; + } + Ok(Self { + shape: Shape::from(perm_dims), + stride: perm_stride, + start_offset: self.start_offset, + }) + } + + pub fn broadcast_as>(&self, shape: S) -> Result { + let shape = shape.into(); + if shape.rank() < self.shape().rank() { + return Err(Error::BroadcastIncompatibleShapes { + src_shape: self.shape().clone(), + dst_shape: shape, + } + .bt()); + } + let added_dims = shape.rank() - self.shape().rank(); + let mut stride = vec![0; added_dims]; + for (&dst_dim, (&src_dim, &src_stride)) in shape.dims()[added_dims..] + .iter() + .zip(self.dims().iter().zip(self.stride())) + { + let s = if dst_dim == src_dim { + src_stride + } else if src_dim != 1 { + return Err(Error::BroadcastIncompatibleShapes { + src_shape: self.shape().clone(), + dst_shape: shape, + } + .bt()); + } else { + 0 + }; + stride.push(s) + } + Ok(Self { + shape, + stride, + start_offset: self.start_offset, + }) + } + + pub(crate) fn strided_index(&self) -> crate::StridedIndex<'_> { + crate::StridedIndex::from_layout(self) + } + + pub(crate) fn strided_blocks(&self) -> crate::StridedBlocks<'_> { + let mut block_len = 1; + let mut contiguous_dims = 0; // These are counted from the right. + for (&stride, &dim) in self.stride().iter().zip(self.dims().iter()).rev() { + if stride != block_len { + break; + } + block_len *= dim; + contiguous_dims += 1; + } + let index_dims = self.dims().len() - contiguous_dims; + if index_dims == 0 { + crate::StridedBlocks::SingleBlock { + start_offset: self.start_offset, + len: block_len, + } + } else { + let block_start_index = crate::StridedIndex::new( + &self.dims()[..index_dims], + &self.stride[..index_dims], + self.start_offset, + ); + crate::StridedBlocks::MultipleBlocks { + block_start_index, + block_len, + } + } + } + + // Returns the contiguous offsets with broadcast if applicable. + pub(crate) fn offsets_b(&self) -> Option { + let mut left_broadcast = 1; + let mut right_broadcast = 1; + let strides = self.stride(); + let dims = self.dims(); + let mut start_cont = 0; + let mut end_cont = dims.len(); + for (&s, &d) in strides.iter().zip(dims.iter()) { + if s != 0 { + break; + } + start_cont += 1; + left_broadcast *= d; + } + if start_cont == dims.len() { + return Some(ContiguousOffsetsWithBroadcast { + start: self.start_offset, + len: 1, + left_broadcast, + right_broadcast: 1, + }); + } + for (&s, &d) in strides.iter().zip(dims.iter()).rev() { + if s != 0 { + break; + } + end_cont -= 1; + right_broadcast *= d; + } + // Check that the inner dims are contiguous + let strides = &strides[start_cont..end_cont]; + let dims = &dims[start_cont..end_cont]; + let mut len = 1; + for (&stride, &dim) in strides.iter().zip(dims.iter()).rev() { + if stride != len { + return None; + } + len *= dim; + } + Some(ContiguousOffsetsWithBroadcast { + start: self.start_offset, + len, + left_broadcast, + right_broadcast, + }) + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ContiguousOffsetsWithBroadcast { + pub start: usize, + pub len: usize, + pub left_broadcast: usize, + pub right_broadcast: usize, +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/lib.rs b/v2/dependency-patches/candle-core-0.9.2/src/lib.rs new file mode 100644 index 0000000000..791933d25a --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/lib.rs @@ -0,0 +1,177 @@ +//! ML framework for Rust +//! +//! ```rust +//! use candle_core::{Tensor, DType, Device}; +//! # use candle_core::Error; +//! # fn main() -> Result<(), Error>{ +//! +//! let a = Tensor::arange(0f32, 6f32, &Device::Cpu)?.reshape((2, 3))?; +//! let b = Tensor::arange(0f32, 12f32, &Device::Cpu)?.reshape((3, 4))?; +//! let c = a.matmul(&b)?; +//! +//! # Ok(())} +//! ``` +//! +//! ## Features +//! +//! - Simple syntax (looks and feels like PyTorch) +//! - CPU and Cuda backends (and M1 support) +//! - Enable serverless (CPU) small and fast deployments +//! - Model training +//! - Distributed computing (NCCL). +//! - Models out of the box (Llama, Whisper, Falcon, ...) +//! +//! ## FAQ +//! +//! - Why Candle? +//! +//! Candle stems from the need to reduce binary size in order to *enable serverless* +//! possible by making the whole engine smaller than PyTorch very large library volume +//! +//! And simply *removing Python* from production workloads. +//! Python can really add overhead in more complex workflows and the [GIL](https://www.backblaze.com/blog/the-python-gil-past-present-and-future/) is a notorious source of headaches. +//! +//! Rust is cool, and a lot of the HF ecosystem already has Rust crates [safetensors](https://github.com/huggingface/safetensors) and [tokenizers](https://github.com/huggingface/tokenizers) +//! +//! ## Other Crates +//! +//! Candle consists of a number of crates. This crate holds core the common data structures but you may wish +//! to look at the docs for the other crates which can be found here: +//! +//! - [candle-core](https://docs.rs/candle-core/). Core Datastructures and DataTypes. +//! - [candle-nn](https://docs.rs/candle-nn/). Building blocks for Neural Nets. +//! - [candle-datasets](https://docs.rs/candle-datasets/). Rust access to commonly used Datasets like MNIST. +//! - [candle-examples](https://docs.rs/candle-examples/). Examples of Candle in Use. +//! - [candle-onnx](https://docs.rs/candle-onnx/). Loading and using ONNX models. +//! - [candle-pyo3](https://docs.rs/candle-pyo3/). Access to Candle from Python. +//! - [candle-transformers](https://docs.rs/candle-transformers/). Candle implementation of many published transformer models. +//! + +#[cfg(feature = "accelerate")] +mod accelerate; +pub mod backend; +pub mod backprop; +pub mod conv; +mod convert; +pub mod cpu; +pub mod cpu_backend; +#[cfg(feature = "cuda")] +pub mod cuda_backend; +mod custom_op; +mod device; +pub mod display; +mod dtype; +pub mod dummy_cuda_backend; +pub mod dummy_dtype; +mod dummy_metal_backend; +pub mod error; +mod indexer; +pub mod layout; +#[cfg(feature = "metal")] +pub mod metal_backend; +#[cfg(feature = "mkl")] +mod mkl; +pub mod npy; +pub mod op; +pub mod pickle; +pub mod quantized; +pub mod safetensors; +pub mod scalar; +pub mod shape; +mod sort; +mod storage; +pub mod streaming; +mod strided_index; +mod tensor; +mod tensor_cat; +pub mod test_utils; +pub mod utils; +mod variable; + +#[cfg(feature = "cudnn")] +pub use cuda_backend::cudnn; + +pub use cpu_backend::{CpuStorage, CpuStorageRef}; +#[cfg(feature = "ug")] +pub use custom_op::UgIOp1; +pub use custom_op::{CustomOp1, CustomOp2, CustomOp3, InplaceOp1, InplaceOp2, InplaceOp3}; +pub use device::{Device, DeviceLocation, NdArray}; +pub use dtype::{DType, DTypeParseError, FloatDType, IntDType, WithDType}; +pub use dummy_dtype::{F4, F6E2M3, F6E3M2, F8E8M0}; +pub use error::{Context, Error, Result}; +pub use indexer::{IndexOp, TensorIndexer}; +pub use layout::Layout; +pub use shape::{Shape, D}; +pub use storage::Storage; +pub use streaming::{StreamTensor, StreamingBinOp, StreamingModule}; +pub use strided_index::{StridedBlocks, StridedIndex}; +pub use tensor::{Tensor, TensorId}; +pub use variable::Var; + +#[cfg(feature = "cuda")] +pub use cuda_backend as cuda; + +#[cfg(not(feature = "cuda"))] +pub use dummy_cuda_backend as cuda; + +pub use cuda::{CudaDevice, CudaStorage}; + +#[cfg(feature = "metal")] +pub use metal_backend::{MetalDevice, MetalError, MetalStorage}; + +#[cfg(not(feature = "metal"))] +pub use dummy_metal_backend::{MetalDevice, MetalError, MetalStorage}; + +#[cfg(feature = "mkl")] +extern crate intel_mkl_src; + +#[cfg(feature = "accelerate")] +extern crate accelerate_src; + +pub trait ToUsize2 { + fn to_usize2(self) -> (usize, usize); +} + +impl ToUsize2 for usize { + fn to_usize2(self) -> (usize, usize) { + (self, self) + } +} + +impl ToUsize2 for (usize, usize) { + fn to_usize2(self) -> (usize, usize) { + self + } +} + +/// Defining a module with forward method using a single argument. +pub trait Module { + fn forward(&self, xs: &Tensor) -> Result; +} + +impl Result> Module for T { + fn forward(&self, xs: &Tensor) -> Result { + self(xs) + } +} + +impl Module for Option<&M> { + fn forward(&self, xs: &Tensor) -> Result { + match self { + None => Ok(xs.clone()), + Some(m) => m.forward(xs), + } + } +} + +/// A single forward method using a single single tensor argument and a flag to +/// separate the training and evaluation behaviors. +pub trait ModuleT { + fn forward_t(&self, xs: &Tensor, train: bool) -> Result; +} + +impl ModuleT for M { + fn forward_t(&self, xs: &Tensor, _train: bool) -> Result { + self.forward(xs) + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/metal_backend/device.rs b/v2/dependency-patches/candle-core-0.9.2/src/metal_backend/device.rs new file mode 100644 index 0000000000..1728c5a4e0 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/metal_backend/device.rs @@ -0,0 +1,287 @@ +use crate::{DType, Result}; + +#[cfg(feature = "ug")] +use candle_metal_kernels::metal::ComputePipeline; +use candle_metal_kernels::{ + metal::{ + BlitCommandEncoder, Buffer, BufferMap, Commands, ComputeCommandEncoder, Device, + MTLResourceOptions, + }, + Kernels, +}; +use objc2_foundation::NSURL; +use objc2_metal::{MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager}; + +use std::path::Path; +use std::sync::{Arc, Mutex, RwLock}; + +use super::MetalError; + +/// Unique identifier for metal devices. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct DeviceId(usize); + +impl DeviceId { + pub(crate) fn new() -> Self { + // https://users.rust-lang.org/t/idiomatic-rust-way-to-generate-unique-id/33805 + use std::sync::atomic; + static COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::new(1); + Self(COUNTER.fetch_add(1, atomic::Ordering::Relaxed)) + } +} + +#[derive(Clone)] +pub struct MetalDevice { + /// Unique identifier, the registryID is not sufficient as it identifies the GPU rather than + /// the device itself. + pub(crate) id: DeviceId, + + /// Raw metal device: + pub(crate) device: Device, + + pub(crate) commands: Arc>, + + /// Simple allocator struct. + /// The buffers are stored in size buckets since ML tends to use similar shapes over and over. + /// We store the buffers in [`Arc`] because it's much faster than Obj-c internal ref counting + /// (could be linked to FFI communication overhead). + /// + /// Whenever a buffer has a strong_count==1, we can reuse it, it means it was dropped in the + /// graph calculation, and only we the allocator kept a reference to it, therefore it's free + /// to be reused. However, in order for this to work, we need to guarantee the order of + /// operation, so that this buffer is not being used by another kernel at the same time. + /// Arc is the CPU reference count, it doesn't mean anything on the GPU side of things. + /// + /// Whenever we actually allocate a new buffer, we make a full sweep to clean up unused buffers + /// (strong_count = 1). + pub(crate) buffers: Arc>, + + /// Simple keeper struct to keep track of the already compiled kernels so we can reuse them. + /// Heavily used by [`candle_metal_kernels`] + pub(crate) kernels: Arc, + /// Seed for random number generation. + pub(crate) seed: Arc>, + /// Last seed value set on this device. + pub(crate) seed_value: Arc>, +} + +// Resource options used for creating buffers. Shared storage mode allows both CPU and GPU to access the buffer. +pub const RESOURCE_OPTIONS: MTLResourceOptions = + objc2_metal::MTLResourceOptions(MTLResourceOptions::StorageModeShared.bits()); +//| MTLResourceOptions::HazardTrackingModeUntracked.bits(), +//); + +// Resource options used for `new_private_buffer`. This uses `private` where supported. +#[cfg(target_os = "ios")] +pub const PRIVATE_RESOURCE_OPTIONS: MTLResourceOptions = MTLResourceOptions::StorageModeShared; +#[cfg(not(target_os = "ios"))] +pub const PRIVATE_RESOURCE_OPTIONS: MTLResourceOptions = MTLResourceOptions::StorageModePrivate; + +impl std::fmt::Debug for MetalDevice { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "MetalDevice({:?})", self.id) + } +} + +impl std::ops::Deref for MetalDevice { + type Target = Device; + + fn deref(&self) -> &Self::Target { + &self.device + } +} + +impl MetalDevice { + #[cfg(all(feature = "ug", not(target_arch = "wasm32"), not(target_os = "ios")))] + pub fn compile( + &self, + func_name: &'static str, + kernel: candle_ug::lang::ssa::Kernel, + ) -> Result { + let mut buf = vec![]; + candle_ug::metal::code_gen::gen(&mut buf, func_name, &kernel)?; + let metal_code = String::from_utf8(buf)?; + let lib = self + .device + .new_library_with_source(&metal_code, None) + .map_err(MetalError::from)?; + let func = lib + .get_function(func_name, None) + .map_err(MetalError::from)?; + let pl = self + .device + .new_compute_pipeline_state_with_function(&func) + .map_err(MetalError::from)?; + Ok(pl) + } + + pub fn id(&self) -> DeviceId { + self.id + } + + pub fn metal_device(&self) -> &Device { + &self.device + } + + fn drop_unused_buffers(&self) -> Result<()> { + let mut buffers = self.buffers.write().map_err(MetalError::from)?; + for subbuffers in buffers.values_mut() { + let newbuffers = subbuffers + .iter() + .filter(|s| Arc::strong_count(*s) > 1) + .map(Arc::clone) + .collect(); + *subbuffers = newbuffers; + } + Ok(()) + } + + pub fn command_encoder(&self) -> Result { + let commands = self.commands.write().map_err(MetalError::from)?; + let (flush, command_encoder) = commands.command_encoder().map_err(MetalError::from)?; + if flush { + self.drop_unused_buffers()? + } + Ok(command_encoder) + } + + pub fn blit_command_encoder(&self) -> Result { + let commands = self.commands.write().map_err(MetalError::from)?; + let (flush, command_encoder) = commands.blit_command_encoder().map_err(MetalError::from)?; + if flush { + self.drop_unused_buffers()? + } + Ok(command_encoder) + } + + pub fn wait_until_completed(&self) -> Result<()> { + let commands = self.commands.write().map_err(MetalError::from)?; + commands.wait_until_completed().map_err(MetalError::from)?; + Ok(()) + } + + pub fn kernels(&self) -> &Kernels { + &self.kernels + } + + pub fn device(&self) -> &Device { + &self.device + } + + /// Creates a new buffer (not necessarily zeroed). + pub fn new_buffer( + &self, + element_count: usize, + dtype: DType, + _name: &str, + ) -> Result> { + let size = element_count * dtype.size_in_bytes(); + self.allocate_buffer(size) + } + + /// Creates a new private buffer (not necessarily zeroed). + /// + /// This is intentionally not in the Metal buffer pool to allow the efficient implementation of persistent buffers. + pub fn new_private_buffer( + &self, + element_count: usize, + dtype: DType, + _name: &str, + ) -> Result> { + let size = element_count * dtype.size_in_bytes(); + let buffer = self + .device + .new_buffer(size, PRIVATE_RESOURCE_OPTIONS) + .map_err(MetalError::from)?; + Ok(Arc::new(buffer)) + } + + /// Creates a new buffer from data. + /// + /// Does not require synchronization, as [newBufferWithBytes](https://developer.apple.com/documentation/metal/mtldevice/1433429-newbufferwithbytes) + /// allocates the buffer and copies over the existing data before returning the MTLBuffer. + pub fn new_buffer_with_data(&self, data: &[T]) -> Result> { + let size = core::mem::size_of_val(data); + let new_buffer = self + .device + .new_buffer_with_data(data.as_ptr().cast(), size, RESOURCE_OPTIONS) + .map_err(MetalError::from)?; + let mut buffers = self.buffers.write().map_err(MetalError::from)?; + + let subbuffers = buffers.entry(size).or_insert(vec![]); + + let new_buffer = Arc::new(new_buffer); + subbuffers.push(new_buffer.clone()); + Ok(new_buffer) + } + + pub fn allocate_zeros(&self, size_in_bytes: usize) -> Result> { + let buffer = self.allocate_buffer(size_in_bytes)?; + let blit = self.blit_command_encoder()?; + blit.set_label("zeros"); + blit.fill_buffer(&buffer, (0, buffer.length()), 0); + blit.end_encoding(); + Ok(buffer) + } + + /// The critical allocator algorithm + pub fn allocate_buffer(&self, size: usize) -> Result> { + let mut buffers = self.buffers.write().map_err(MetalError::from)?; + if let Some(b) = find_available_buffer(size, &buffers) { + // Cloning also ensures we increment the strong count + return Ok(b.clone()); + } + let size = buf_size(size); + let subbuffers = buffers.entry(size).or_insert(vec![]); + + let new_buffer = self + .device + .new_buffer(size, RESOURCE_OPTIONS) + .map_err(MetalError::from)?; + let new_buffer = Arc::new(new_buffer); + subbuffers.push(new_buffer.clone()); + Ok(new_buffer) + } + + /// Create a metal GPU capture trace on [`path`]. + pub fn capture>(&self, path: P) -> Result<()> { + let capture = unsafe { MTLCaptureManager::sharedCaptureManager() }; + let descriptor = MTLCaptureDescriptor::new(); + descriptor.setDestination(MTLCaptureDestination::GPUTraceDocument); + descriptor.set_capture_device(self.device().as_ref()); + // The [set_output_url] call requires an absolute path so we convert it if needed. + if path.as_ref().is_absolute() { + let url = NSURL::from_file_path(path); + descriptor.setOutputURL(url.as_deref()); + } else { + let path = std::env::current_dir()?.join(path); + let url = NSURL::from_file_path(path); + descriptor.setOutputURL(url.as_deref()); + } + + capture + .startCaptureWithDescriptor_error(&descriptor) + .map_err(|e| MetalError::from(e.to_string()))?; + Ok(()) + } +} + +fn buf_size(size: usize) -> usize { + size.saturating_sub(1).next_power_of_two() +} + +fn find_available_buffer(size: usize, buffers: &BufferMap) -> Option> { + let mut best_buffer: Option<&Arc> = None; + let mut best_buffer_size = usize::MAX; + for (buffer_size, subbuffers) in buffers.iter() { + if buffer_size >= &size && buffer_size < &best_buffer_size { + for sub in subbuffers { + if Arc::strong_count(sub) == 1 { + best_buffer = Some(sub); + best_buffer_size = *buffer_size; + } + } + } + } + best_buffer.cloned() +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/metal_backend/mod.rs b/v2/dependency-patches/candle-core-0.9.2/src/metal_backend/mod.rs new file mode 100644 index 0000000000..363ffa9f7a --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/metal_backend/mod.rs @@ -0,0 +1,2137 @@ +//! Implementation of Backend traits for Metal +//! +use crate::backend::{BackendDevice, BackendStorage}; +use crate::conv::{ParamsConv1D, ParamsConv2D, ParamsConvTranspose1D, ParamsConvTranspose2D}; +use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; +use crate::{CpuStorage, CpuStorageRef, DType, Error, Layout, Result, Shape}; +use candle_metal_kernels::{ + metal::{Buffer, Commands, Device}, + BufferOffset, CallConvTranspose2dCfg, Kernels, RESOURCE_OPTIONS, +}; +use objc2_foundation::NSRange; +use std::collections::HashMap; +use std::ffi::c_void; +use std::sync::{Arc, Mutex, PoisonError, RwLock, TryLockError}; + +mod device; +pub use device::{DeviceId, MetalDevice}; + +pub fn buffer_o<'a>(buffer: &'a Buffer, l: &Layout, dtype: DType) -> BufferOffset<'a> { + BufferOffset { + buffer, + offset_in_bytes: l.start_offset() * dtype.size_in_bytes(), + } +} +/// Simple way to catch lock error without +/// depending on T +#[derive(thiserror::Error, Debug)] +pub enum LockError { + #[error("{0}")] + Poisoned(String), + #[error("Would block")] + WouldBlock, +} + +impl From> for MetalError { + fn from(value: TryLockError) -> Self { + match value { + TryLockError::Poisoned(p) => MetalError::LockError(LockError::Poisoned(p.to_string())), + TryLockError::WouldBlock => MetalError::LockError(LockError::WouldBlock), + } + } +} + +impl From> for MetalError { + fn from(p: PoisonError) -> Self { + MetalError::LockError(LockError::Poisoned(p.to_string())) + } +} + +/// Metal related errors +#[derive(thiserror::Error, Debug)] +pub enum MetalError { + #[error("{0}")] + Message(String), + #[error(transparent)] + KernelError(#[from] candle_metal_kernels::MetalKernelError), + #[error("{0:?}")] + LockError(LockError), + #[error("{msg}, expected: {expected:?}, got: {got:?}")] + UnexpectedDType { + msg: &'static str, + expected: DType, + got: DType, + }, +} + +impl From for MetalError { + fn from(e: String) -> Self { + MetalError::Message(e) + } +} + +#[derive(Debug, Clone)] +pub struct MetalStorage { + /// The actual buffer containing the data. + buffer: Arc, + /// a reference to the device owning this buffer + device: MetalDevice, + /// The count of allocated elements in the buffer + count: usize, + /// The dtype is kept since buffers are untyped. + dtype: DType, +} + +impl BackendStorage for MetalStorage { + type Device = MetalDevice; + + fn try_clone(&self, _: &Layout) -> Result { + Ok(self.clone()) + } + + fn dtype(&self) -> DType { + self.dtype + } + + fn device(&self) -> &Self::Device { + &self.device + } + + fn to_cpu_storage(&self) -> Result { + match self.dtype { + DType::U8 => Ok(CpuStorage::U8(self.to_cpu()?)), + DType::U32 => Ok(CpuStorage::U32(self.to_cpu()?)), + DType::I16 => Ok(CpuStorage::I16(self.to_cpu()?)), + DType::I32 => Ok(CpuStorage::I32(self.to_cpu()?)), + DType::I64 => Ok(CpuStorage::I64(self.to_cpu()?)), + DType::F16 => Ok(CpuStorage::F16(self.to_cpu()?)), + DType::BF16 => Ok(CpuStorage::BF16(self.to_cpu()?)), + DType::F32 => Ok(CpuStorage::F32(self.to_cpu()?)), + DType::F64 => Ok(CpuStorage::F64(self.to_cpu()?)), + DType::F8E4M3 => Ok(CpuStorage::F8E4M3(self.to_cpu()?)), + DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { + Err(crate::Error::UnsupportedDTypeForOp(self.dtype, "to_cpu_storage").bt()) + } + } + } + + fn affine(&self, layout: &Layout, mul: f64, add: f64) -> Result { + let device = self.device().clone(); + + let shape = layout.shape(); + let el = shape.elem_count(); + let dtype = self.dtype; + + let buffer = device.new_buffer(el, self.dtype, "affine")?; + let encoder = self.device.command_encoder()?; + encoder.set_label("affine"); + let src = buffer_o(&self.buffer, layout, dtype); + if layout.is_contiguous() { + let name = match self.dtype { + DType::F32 => "affine_f32", + DType::F16 => "affine_f16", + DType::BF16 => "affine_bf16", + DType::U8 => "affine_u8", + DType::U32 => "affine_u32", + DType::I64 => "affine_i64", + dtype => crate::bail!("Metal contiguous affine {dtype:?} not implemented"), + }; + candle_metal_kernels::call_affine( + &device.device, + &encoder, + &device.kernels, + name, + self.dtype.size_in_bytes(), + el, + src, + &buffer, + mul as f32, + add as f32, + ) + .map_err(MetalError::from)?; + } else { + let name = match self.dtype { + DType::F32 => "affine_f32_strided", + DType::F16 => "affine_f16_strided", + DType::BF16 => "affine_bf16_strided", + DType::U8 => "affine_u8_strided", + DType::U32 => "affine_u32_strided", + DType::I64 => "affine_i64_strided", + dtype => crate::bail!("Metal strided affine {dtype:?} not implemented"), + }; + candle_metal_kernels::call_affine_strided( + &device.device, + &encoder, + &device.kernels, + name, + layout.dims(), + src, + layout.stride(), + &buffer, + mul as f32, + add as f32, + ) + .map_err(MetalError::from)?; + } + Ok(Self::new(buffer, device.clone(), el, dtype)) + } + + fn powf(&self, layout: &Layout, pow: f64) -> Result { + let device = self.device().clone(); + + let shape = layout.shape(); + let el = shape.elem_count(); + let dtype = self.dtype; + + let buffer = device.new_buffer(el, self.dtype, "powf")?; + let encoder = self.device.command_encoder()?; + encoder.set_label("powf"); + let src = buffer_o(&self.buffer, layout, dtype); + if layout.is_contiguous() { + let name = match self.dtype { + DType::F32 => "powf_f32", + DType::F16 => "powf_f16", + DType::BF16 => "powf_bf16", + dtype => crate::bail!("Metal contiguous powf {dtype:?} not implemented"), + }; + candle_metal_kernels::call_powf( + &device.device, + &encoder, + &device.kernels, + name, + self.dtype.size_in_bytes(), + el, + src, + &buffer, + pow as f32, + ) + .map_err(MetalError::from)?; + } else { + let name = match self.dtype { + DType::F32 => "powf_f32_strided", + DType::F16 => "powf_f16_strided", + DType::BF16 => "powf_bf16_strided", + dtype => crate::bail!("Metal strided powf {dtype:?} not implemented"), + }; + candle_metal_kernels::call_powf_strided( + &device.device, + &encoder, + &device.kernels, + name, + layout.dims(), + src, + layout.stride(), + &buffer, + pow as f32, + ) + .map_err(MetalError::from)?; + } + Ok(Self::new(buffer, device.clone(), el, dtype)) + } + + fn elu(&self, layout: &Layout, alpha: f64) -> Result { + let device = self.device().clone(); + + let shape = layout.shape(); + let el = shape.elem_count(); + let dtype = self.dtype; + + let buffer = device.new_buffer(el, self.dtype, "elu")?; + let encoder = self.device.command_encoder()?; + encoder.set_label("elu"); + let src = buffer_o(&self.buffer, layout, self.dtype); + if layout.is_contiguous() { + let name = match self.dtype { + DType::F32 => "elu_f32", + DType::F16 => "elu_f16", + DType::BF16 => "elu_bf16", + dtype => crate::bail!("Metal contiguous elu {dtype:?} not implemented"), + }; + candle_metal_kernels::call_elu( + &device.device, + &encoder, + &device.kernels, + name, + self.dtype.size_in_bytes(), + el, + src, + &buffer, + alpha as f32, + ) + .map_err(MetalError::from)?; + } else { + let name = match self.dtype { + DType::F32 => "elu_f32_strided", + DType::F16 => "elu_f16_strided", + DType::BF16 => "elu_bf16_strided", + dtype => crate::bail!("Metal strided elu {dtype:?} not implemented"), + }; + candle_metal_kernels::call_elu_strided( + &device.device, + &encoder, + &device.kernels, + name, + layout.dims(), + src, + layout.stride(), + &buffer, + alpha as f32, + ) + .map_err(MetalError::from)?; + } + Ok(Self::new(buffer, device.clone(), el, dtype)) + } + + fn reduce_op(&self, op: ReduceOp, layout: &Layout, sum_dims: &[usize]) -> Result { + let device = self.device.clone(); + + let src_stride = layout.stride(); + let src_dims = layout.shape().dims(); + // Source dims and strides with the sum dims at the end. + let mut dims = vec![]; + let mut stride = vec![]; + let mut dst_el: usize = 1; + for (dim_idx, &d) in src_dims.iter().enumerate() { + if !sum_dims.contains(&dim_idx) { + dst_el *= d; + dims.push(d); + stride.push(src_stride[dim_idx]); + } + } + + for &dim_idx in sum_dims.iter() { + dims.push(src_dims[dim_idx]); + stride.push(src_stride[dim_idx]); + } + + let reduction_shape = Shape::from(dims.clone()); + + if layout.is_contiguous() && reduction_shape.is_contiguous(&stride) { + let (name, check_empty, return_index) = match (op, self.dtype) { + (ReduceOp::Sum, DType::F32) => ("fast_sum_f32", false, false), + (ReduceOp::Min, DType::F32) => ("fast_min_f32", true, false), + (ReduceOp::Max, DType::F32) => ("fast_max_f32", true, false), + (ReduceOp::ArgMin, DType::F32) => ("fast_argmin_f32", true, true), + (ReduceOp::ArgMax, DType::F32) => ("fast_argmax_f32", true, true), + (ReduceOp::Sum, DType::U32) => ("fast_sum_u32", false, false), + (ReduceOp::Min, DType::U32) => ("fast_min_u32", true, false), + (ReduceOp::Max, DType::U32) => ("fast_max_u32", true, false), + (ReduceOp::ArgMin, DType::U32) => ("fast_argmin_u32", true, true), + (ReduceOp::ArgMax, DType::U32) => ("fast_argmax_u32", true, true), + (ReduceOp::Sum, DType::F16) => ("fast_sum_f16", false, false), + (ReduceOp::Min, DType::F16) => ("fast_min_f16", true, false), + (ReduceOp::Max, DType::F16) => ("fast_max_f16", true, false), + (ReduceOp::ArgMin, DType::F16) => ("fast_argmin_f16", true, true), + (ReduceOp::ArgMax, DType::F16) => ("fast_argmax_f16", true, true), + (ReduceOp::Sum, DType::BF16) => ("fast_sum_bf16", false, false), + (ReduceOp::Min, DType::BF16) => ("fast_min_bf16", true, false), + (ReduceOp::Max, DType::BF16) => ("fast_max_bf16", true, false), + (ReduceOp::ArgMin, DType::BF16) => ("fast_argmin_bf16", true, true), + (ReduceOp::ArgMax, DType::BF16) => ("fast_argmax_bf16", true, true), + (ReduceOp::Sum, DType::I64) => ("fast_sum_i64", false, false), + (ReduceOp::Min, DType::I64) => ("fast_min_i64", true, false), + (ReduceOp::Max, DType::I64) => ("fast_max_i64", true, false), + (ReduceOp::ArgMin, DType::I64) => ("fast_argmin_i64", true, true), + (ReduceOp::ArgMax, DType::I64) => ("fast_argmax_i64", true, true), + (ReduceOp::Sum, DType::U8) => ("fast_sum_u8", false, false), + (ReduceOp::Min, DType::U8) => ("fast_min_u8", true, false), + (ReduceOp::Max, DType::U8) => ("fast_max_u8", true, false), + (ReduceOp::ArgMin, DType::U8) => ("fast_argmin_u8", true, true), + (ReduceOp::ArgMax, DType::U8) => ("fast_argmax_u8", true, true), + (k, dtype) => { + crate::bail!("Metal contiguous reduce op {k:?} {dtype:?} not implemented") + } + }; + if check_empty && layout.shape().elem_count() == 0 { + Err(crate::Error::EmptyTensor { op: "reduce" }.bt())? + } + let dtype = if return_index { DType::U32 } else { self.dtype }; + let buffer = device.new_buffer(dst_el, dtype, "reduce")?; + let encoder = self.device.command_encoder()?; + encoder.set_label("reduce"); + let src = buffer_o(&self.buffer, layout, self.dtype); + candle_metal_kernels::call_reduce_contiguous( + &device.device, + &encoder, + &device.kernels, + name, + src_dims, + dst_el, + src, + &buffer, + ) + .map_err(MetalError::from)?; + + return Ok(Self::new(buffer, device, dst_el, dtype)); + } + + let (name, check_empty, return_index) = match (op, self.dtype) { + (ReduceOp::Sum, DType::F32) => ("fast_sum_f32_strided", false, false), + (ReduceOp::Min, DType::F32) => ("fast_min_f32_strided", true, false), + (ReduceOp::Max, DType::F32) => ("fast_max_f32_strided", true, false), + (ReduceOp::ArgMin, DType::F32) => ("fast_argmin_f32_strided", true, true), + (ReduceOp::ArgMax, DType::F32) => ("fast_argmax_f32_strided", true, true), + (ReduceOp::Sum, DType::U32) => ("fast_sum_u32_strided", false, false), + (ReduceOp::Min, DType::U32) => ("fast_min_u32_strided", true, false), + (ReduceOp::Max, DType::U32) => ("fast_max_u32_strided", true, false), + (ReduceOp::ArgMin, DType::U32) => ("fast_argmin_u32_strided", true, true), + (ReduceOp::ArgMax, DType::U32) => ("fast_argmax_u32_strided", true, true), + (ReduceOp::Sum, DType::F16) => ("fast_sum_f16_strided", false, false), + (ReduceOp::Min, DType::F16) => ("fast_min_f16_strided", true, false), + (ReduceOp::Max, DType::F16) => ("fast_max_f16_strided", true, false), + (ReduceOp::ArgMin, DType::F16) => ("fast_argmin_f16_strided", true, true), + (ReduceOp::ArgMax, DType::F16) => ("fast_argmax_f16_strided", true, true), + (ReduceOp::Sum, DType::BF16) => ("fast_sum_bf16_strided", false, false), + (ReduceOp::Min, DType::BF16) => ("fast_min_bf16_strided", true, false), + (ReduceOp::Max, DType::BF16) => ("fast_max_bf16_strided", true, false), + (ReduceOp::ArgMin, DType::BF16) => ("fast_argmin_bf16_strided", true, true), + (ReduceOp::ArgMax, DType::BF16) => ("fast_argmax_bf16_strided", true, true), + (ReduceOp::Sum, DType::I64) => ("fast_sum_i64_strided", false, false), + (ReduceOp::Min, DType::I64) => ("fast_min_i64_strided", true, false), + (ReduceOp::Max, DType::I64) => ("fast_max_i64_strided", true, false), + (ReduceOp::ArgMin, DType::I64) => ("fast_argmin_i64_strided", true, true), + (ReduceOp::ArgMax, DType::I64) => ("fast_argmax_i64_strided", true, true), + (ReduceOp::Sum, DType::U8) => ("fast_sum_u8_strided", false, false), + (ReduceOp::Min, DType::U8) => ("fast_min_u8_strided", true, false), + (ReduceOp::Max, DType::U8) => ("fast_max_u8_strided", true, false), + (ReduceOp::ArgMin, DType::U8) => ("fast_argmin_u8_strided", true, true), + (ReduceOp::ArgMax, DType::U8) => ("fast_argmax_u8_strided", true, true), + (k, dtype) => crate::bail!("Metal strided reduce op {k:?} {dtype:?} not implemented"), + }; + if check_empty && layout.shape().elem_count() == 0 { + Err(crate::Error::EmptyTensor { op: "reduce" }.bt())? + } + let dtype = if return_index { DType::U32 } else { self.dtype }; + let buffer = device.new_buffer(dst_el, dtype, "reduce")?; + let encoder = self.device.command_encoder()?; + encoder.set_label("reduce"); + let src = buffer_o(&self.buffer, layout, self.dtype); + candle_metal_kernels::call_reduce_strided( + &device.device, + &encoder, + &device.kernels, + name, + &dims, + &stride, + dst_el, + src, + &buffer, + ) + .map_err(MetalError::from)?; + + Ok(Self::new(buffer, device, dst_el, dtype)) + } + + fn cmp(&self, op: CmpOp, rhs: &Self, lhs_l: &Layout, rhs_l: &Layout) -> Result { + let name = match op { + CmpOp::Eq => "eq", + CmpOp::Ne => "ne", + CmpOp::Le => "le", + CmpOp::Ge => "ge", + CmpOp::Lt => "lt", + CmpOp::Gt => "gt", + }; + self.binary(name, rhs, lhs_l, rhs_l) + } + + fn const_set(&mut self, s: crate::scalar::Scalar, l: &Layout) -> Result<()> { + use crate::scalar::Scalar; + fn set( + self_: &mut MetalStorage, + s: S, + l: &Layout, + ) -> Result<()> { + let device = self_.device(); + let dtype = self_.dtype; + let shape = l.shape(); + let el_count = shape.elem_count(); + let encoder = device.command_encoder()?; + encoder.set_label("const-set"); + let dst = buffer_o(&self_.buffer, l, self_.dtype); + + if l.is_contiguous() { + use candle_metal_kernels::unary::contiguous; + let kernel_name = match dtype { + DType::F16 => contiguous::const_set::HALF, + DType::BF16 => contiguous::const_set::BFLOAT, + DType::F32 => contiguous::const_set::FLOAT, + DType::I64 => contiguous::const_set::I64, + DType::U32 => contiguous::const_set::U32, + DType::U8 => contiguous::const_set::U8, + DType::F8E4M3 => crate::bail!("unsupported const-set f8e4m3"), + DType::F64 => crate::bail!("unsupported const-set f64"), + DType::F4 + | DType::F6E2M3 + | DType::F6E3M2 + | DType::F8E8M0 + | DType::I16 + | DType::I32 => { + return Err(Error::UnsupportedDTypeForOp(dtype, "const-set").bt()) + } + }; + candle_metal_kernels::call_const_set_contiguous( + &device.device, + &encoder, + &device.kernels, + kernel_name, + dtype.size_in_bytes(), + el_count, + s, + dst, + ) + .map_err(MetalError::from)?; + } else { + use candle_metal_kernels::unary::strided; + let kernel_name = match dtype { + DType::F16 => strided::const_set::HALF, + DType::BF16 => strided::const_set::BFLOAT, + DType::F32 => strided::const_set::FLOAT, + DType::I64 => strided::const_set::I64, + DType::U32 => strided::const_set::U32, + DType::U8 => strided::const_set::U8, + DType::F8E4M3 => crate::bail!("unsupported const-set f8e4m3"), + DType::F64 => crate::bail!("unsupported const-set f64"), + DType::F4 + | DType::F6E2M3 + | DType::F6E3M2 + | DType::F8E8M0 + | DType::I16 + | DType::I32 => { + return Err(Error::UnsupportedDTypeForOp(dtype, "const-set").bt()) + } + }; + candle_metal_kernels::call_const_set_strided( + &device.device, + &encoder, + &device.kernels, + kernel_name, + l.dims(), + s, + l.stride(), + dst, + ) + .map_err(MetalError::from)?; + } + Ok(()) + } + match (self.dtype, s) { + (DType::U8, Scalar::U8(s)) => set(self, s, l), + (DType::U32, Scalar::U32(s)) => set(self, s, l), + (DType::I64, Scalar::I64(s)) => set(self, s, l), + (DType::F16, Scalar::F16(s)) => set(self, s, l), + (DType::BF16, Scalar::BF16(s)) => set(self, s, l), + (DType::F32, Scalar::F32(s)) => set(self, s, l), + (DType::F64, Scalar::F64(s)) => set(self, s, l), + _ => crate::bail!("dtype mismatch, expected {:?}, got {:?}", self.dtype, s), + } + } + + fn to_dtype(&self, layout: &Layout, dtype: DType) -> Result { + let device = self.device(); + let shape = layout.shape(); + let el_count = shape.elem_count(); + let buffer = device.new_buffer(el_count, dtype, "to_dtype")?; + let encoder = device.command_encoder()?; + encoder.set_label("to_dtype"); + let src = buffer_o(&self.buffer, layout, self.dtype); + if layout.is_contiguous() { + let kernel_name = match (self.dtype, dtype) { + (DType::U32, DType::BF16) => "cast_u32_bf16", + (DType::U32, DType::F16) => "cast_u32_f16", + (DType::U32, DType::F32) => "cast_u32_f32", + (DType::U32, DType::I64) => "cast_u32_i64", + (DType::U32, DType::U8) => "cast_u32_u8", + + (DType::U8, DType::BF16) => "cast_u8_bf16", + (DType::U8, DType::F16) => "cast_u8_f16", + (DType::U8, DType::F32) => "cast_u8_f32", + (DType::U8, DType::I64) => "cast_u8_i64", + (DType::U8, DType::U32) => "cast_u8_u32", + + (DType::F32, DType::BF16) => "cast_f32_bf16", + (DType::F32, DType::F16) => "cast_f32_f16", + (DType::F32, DType::I64) => "cast_f32_i64", + (DType::F32, DType::U32) => "cast_f32_u32", + (DType::F32, DType::U8) => "cast_f32_u8", + + (DType::I64, DType::BF16) => "cast_i64_bf16", + (DType::I64, DType::F16) => "cast_i64_f16", + (DType::I64, DType::F32) => "cast_i64_f32", + (DType::I64, DType::U32) => "cast_i64_u32", + (DType::I64, DType::U8) => "cast_i64_u8", + + (DType::F16, DType::BF16) => "cast_f16_bf16", + (DType::F16, DType::F32) => "cast_f16_f32", + (DType::F16, DType::I64) => "cast_f16_i64", + (DType::F16, DType::U32) => "cast_f16_u32", + (DType::F16, DType::U8) => "cast_f16_u8", + + (DType::BF16, DType::F16) => "cast_bf16_f16", + (DType::BF16, DType::F32) => "cast_bf16_f32", + (DType::BF16, DType::I64) => "cast_bf16_i64", + (DType::BF16, DType::U32) => "cast_bf16_u32", + (DType::BF16, DType::U8) => "cast_bf16_u8", + + (left, right) => { + crate::bail!("Metal contiguous to_dtype {left:?} {right:?} not implemented") + } + }; + candle_metal_kernels::call_cast_contiguous( + &device.device, + &encoder, + &device.kernels, + kernel_name, + self.dtype.size_in_bytes(), + el_count, + src, + &buffer, + ) + .map_err(MetalError::from)?; + } else { + let kernel_name = match (self.dtype, dtype) { + (DType::BF16, DType::F16) => "cast_bf16_f16_strided", + (DType::BF16, DType::F32) => "cast_bf16_f32_strided", + (DType::BF16, DType::I64) => "cast_bf16_i64_strided", + (DType::BF16, DType::U32) => "cast_bf16_u32_strided", + (DType::BF16, DType::U8) => "cast_bf16_u8_strided", + + (DType::F16, DType::BF16) => "cast_f16_bf16_strided", + (DType::F16, DType::F32) => "cast_f16_f32_strided", + (DType::F16, DType::I64) => "cast_f16_i64_strided", + (DType::F16, DType::U32) => "cast_f16_u32_strided", + (DType::F16, DType::U8) => "cast_f16_u8_strided", + + (DType::F32, DType::BF16) => "cast_f32_bf16_strided", + (DType::F32, DType::F16) => "cast_f32_f16_strided", + (DType::F32, DType::I64) => "cast_f32_i64_strided", + (DType::F32, DType::U32) => "cast_f32_u32_strided", + (DType::F32, DType::U8) => "cast_f32_u8_strided", + + (DType::I64, DType::F32) => "cast_i64_f32_strided", + (DType::I64, DType::BF16) => "cast_i64_bf16_strided", + (DType::I64, DType::F16) => "cast_i64_f16_strided", + (DType::I64, DType::U32) => "cast_i64_u32_strided", + (DType::I64, DType::U8) => "cast_i64_u8_strided", + + (DType::U32, DType::BF16) => "cast_u32_bf16_strided", + (DType::U32, DType::F16) => "cast_u32_f16_strided", + (DType::U32, DType::F32) => "cast_u32_f32_strided", + (DType::U32, DType::I64) => "cast_u32_i64_strided", + (DType::U32, DType::U8) => "cast_u32_u8_strided", + + (DType::U8, DType::BF16) => "cast_u8_bf16_strided", + (DType::U8, DType::F16) => "cast_u8_f16_strided", + (DType::U8, DType::F32) => "cast_u8_f32_strided", + (DType::U8, DType::I64) => "cast_u8_i64_strided", + (DType::U8, DType::U32) => "cast_u8_u32_strided", + + (left, right) => { + crate::bail!("Metal strided to_dtype {left:?} {right:?} not implemented") + } + }; + candle_metal_kernels::call_cast_strided( + &device.device, + &encoder, + &device.kernels, + kernel_name, + layout.dims(), + src, + layout.stride(), + &buffer, + ) + .map_err(MetalError::from)?; + } + Ok(Self::new(buffer, device.clone(), el_count, dtype)) + } + + fn unary_impl(&self, layout: &Layout) -> Result { + let device = self.device(); + let dtype = self.dtype; + let shape = layout.shape(); + let el_count = shape.elem_count(); + let buffer = device.new_buffer(el_count, dtype, B::KERNEL)?; + let encoder = device.command_encoder()?; + encoder.set_label(B::KERNEL); + let src = buffer_o(&self.buffer, layout, self.dtype); + + if layout.is_contiguous() { + use candle_metal_kernels::unary::contiguous; + let kernel_name = match (B::KERNEL, dtype) { + ("uabs", DType::F16) => contiguous::abs::HALF, + ("uabs", DType::F32) => contiguous::abs::FLOAT, + ("uabs", DType::BF16) => contiguous::abs::BFLOAT, + ("uceil", DType::F16) => contiguous::ceil::HALF, + ("uceil", DType::F32) => contiguous::ceil::FLOAT, + ("uceil", DType::BF16) => contiguous::ceil::BFLOAT, + ("ucos", DType::F16) => contiguous::cos::HALF, + ("ucos", DType::F32) => contiguous::cos::FLOAT, + ("ucos", DType::BF16) => contiguous::cos::BFLOAT, + ("uerf", DType::F16) => contiguous::erf::HALF, + ("uerf", DType::F32) => contiguous::erf::FLOAT, + ("uerf", DType::BF16) => contiguous::erf::BFLOAT, + ("uexp", DType::F16) => contiguous::exp::HALF, + ("uexp", DType::F32) => contiguous::exp::FLOAT, + ("uexp", DType::BF16) => contiguous::exp::BFLOAT, + ("ufloor", DType::F16) => contiguous::floor::HALF, + ("ufloor", DType::F32) => contiguous::floor::FLOAT, + ("ufloor", DType::BF16) => contiguous::floor::BFLOAT, + ("ugelu_erf", DType::F16) => contiguous::gelu_erf::HALF, + ("ugelu_erf", DType::F32) => contiguous::gelu_erf::FLOAT, + ("ugelu_erf", DType::BF16) => contiguous::gelu_erf::BFLOAT, + ("ugelu", DType::F16) => contiguous::gelu::HALF, + ("ugelu", DType::F32) => contiguous::gelu::FLOAT, + ("ugelu", DType::BF16) => contiguous::gelu::BFLOAT, + ("ulog", DType::F16) => contiguous::log::HALF, + ("ulog", DType::F32) => contiguous::log::FLOAT, + ("ulog", DType::BF16) => contiguous::log::BFLOAT, + ("uneg", DType::F16) => contiguous::neg::HALF, + ("uneg", DType::F32) => contiguous::neg::FLOAT, + ("uneg", DType::BF16) => contiguous::neg::BFLOAT, + ("urecip", DType::F16) => contiguous::recip::HALF, + ("urecip", DType::F32) => contiguous::recip::FLOAT, + ("urecip", DType::BF16) => contiguous::recip::BFLOAT, + ("urelu", DType::F16) => contiguous::relu::HALF, + ("urelu", DType::F32) => contiguous::relu::FLOAT, + ("urelu", DType::BF16) => contiguous::relu::BFLOAT, + ("uround", DType::F16) => contiguous::round::HALF, + ("uround", DType::F32) => contiguous::round::FLOAT, + ("uround", DType::BF16) => contiguous::round::BFLOAT, + ("usilu", DType::F16) => contiguous::silu::HALF, + ("usilu", DType::F32) => contiguous::silu::FLOAT, + ("usilu", DType::BF16) => contiguous::silu::BFLOAT, + ("usin", DType::F16) => contiguous::sin::HALF, + ("usin", DType::F32) => contiguous::sin::FLOAT, + ("usin", DType::BF16) => contiguous::sin::BFLOAT, + ("usqr", DType::F16) => contiguous::sqr::HALF, + ("usqr", DType::F32) => contiguous::sqr::FLOAT, + ("usqr", DType::BF16) => contiguous::sqr::BFLOAT, + ("usqrt", DType::F16) => contiguous::sqrt::HALF, + ("usqrt", DType::F32) => contiguous::sqrt::FLOAT, + ("usqrt", DType::BF16) => contiguous::sqrt::BFLOAT, + ("utanh", DType::F16) => contiguous::tanh::HALF, + ("utanh", DType::F32) => contiguous::tanh::FLOAT, + ("utanh", DType::BF16) => contiguous::tanh::BFLOAT, + ("usign", DType::F16) => contiguous::sign::HALF, + ("usign", DType::F32) => contiguous::sign::FLOAT, + ("usign", DType::BF16) => contiguous::sign::BFLOAT, + ("usign", DType::I64) => contiguous::sign::I64, + (name, dtype) => { + crate::bail!("Metal contiguous unary {name} {dtype:?} not implemented") + } + }; + + candle_metal_kernels::call_unary_contiguous( + &device.device, + &encoder, + &device.kernels, + kernel_name, + dtype.size_in_bytes(), + el_count, + src, + &buffer, + ) + .map_err(MetalError::from)?; + } else { + use candle_metal_kernels::unary::strided; + let kernel_name = match (B::KERNEL, dtype) { + ("ucos", DType::F32) => strided::cos::FLOAT, + ("usin", DType::F32) => strided::sin::FLOAT, + ("usqr", DType::F32) => strided::sqr::FLOAT, + ("usqrt", DType::F32) => strided::sqrt::FLOAT, + ("uneg", DType::F32) => strided::neg::FLOAT, + ("uexp", DType::F32) => strided::exp::FLOAT, + ("ulog", DType::F32) => strided::log::FLOAT, + ("ugelu", DType::F32) => strided::gelu::FLOAT, + ("ugelu_erf", DType::F32) => strided::gelu_erf::FLOAT, + ("uerf", DType::F32) => strided::erf::FLOAT, + ("usilu", DType::F32) => strided::silu::FLOAT, + ("uabs", DType::F32) => strided::abs::FLOAT, + ("uceil", DType::F32) => strided::ceil::FLOAT, + ("ufloor", DType::F32) => strided::floor::FLOAT, + ("urelu", DType::F32) => strided::relu::FLOAT, + ("uround", DType::F32) => strided::round::FLOAT, + ("utanh", DType::F32) => strided::tanh::FLOAT, + + ("ucos", DType::F16) => strided::cos::HALF, + ("usin", DType::F16) => strided::sin::HALF, + ("usqr", DType::F16) => strided::sqr::HALF, + ("usqrt", DType::F16) => strided::sqrt::HALF, + ("uneg", DType::F16) => strided::neg::HALF, + ("uexp", DType::F16) => strided::exp::HALF, + ("ulog", DType::F16) => strided::log::HALF, + ("ugelu", DType::F16) => strided::gelu::HALF, + ("ugelu_erf", DType::F16) => strided::gelu_erf::HALF, + ("uerf", DType::F16) => strided::erf::HALF, + ("usilu", DType::F16) => strided::silu::HALF, + ("uabs", DType::F16) => strided::abs::HALF, + ("uceil", DType::F16) => strided::ceil::HALF, + ("ufloor", DType::F16) => strided::floor::HALF, + ("urelu", DType::F16) => strided::relu::HALF, + ("uround", DType::F16) => strided::round::HALF, + ("utanh", DType::F16) => strided::tanh::HALF, + + ("ucos", DType::BF16) => strided::cos::BFLOAT, + ("usin", DType::BF16) => strided::sin::BFLOAT, + ("usqr", DType::BF16) => strided::sqr::BFLOAT, + ("usqrt", DType::BF16) => strided::sqrt::BFLOAT, + ("uneg", DType::BF16) => strided::neg::BFLOAT, + ("uexp", DType::BF16) => strided::exp::BFLOAT, + ("ulog", DType::BF16) => strided::log::BFLOAT, + ("ugelu", DType::BF16) => strided::gelu::BFLOAT, + ("ugelu_erf", DType::BF16) => strided::gelu_erf::BFLOAT, + ("uerf", DType::BF16) => strided::erf::BFLOAT, + ("usilu", DType::BF16) => strided::silu::BFLOAT, + ("uabs", DType::BF16) => strided::abs::BFLOAT, + ("uceil", DType::BF16) => strided::ceil::BFLOAT, + ("ufloor", DType::BF16) => strided::floor::BFLOAT, + ("urelu", DType::BF16) => strided::relu::BFLOAT, + ("uround", DType::BF16) => strided::round::BFLOAT, + ("utanh", DType::BF16) => strided::tanh::BFLOAT, + + (name, dtype) => { + crate::bail!("Metal strided unary {name} {dtype:?} not implemented") + } + }; + let dst = BufferOffset::zero_offset(&buffer); + candle_metal_kernels::call_unary_strided( + &device.device, + &encoder, + &device.kernels, + kernel_name, + layout.dims(), + src, + layout.stride(), + dst, + ) + .map_err(MetalError::from)?; + } + + Ok(Self::new(buffer, device.clone(), el_count, dtype)) + } + + fn binary_impl( + &self, + rhs: &Self, + lhs_l: &Layout, + rhs_l: &Layout, + ) -> Result { + self.binary(B::KERNEL, rhs, lhs_l, rhs_l) + } + + fn where_cond( + &self, + layout: &Layout, + t: &Self, + t_l: &Layout, + f: &Self, + f_l: &Layout, + ) -> Result { + let device = self.device.clone(); + let shape = t_l.shape(); + let dims = shape.dims(); + let el = shape.elem_count(); + let dtype = t.dtype; + let buffer = self.device.new_buffer(el, dtype, "where")?; + let encoder = self.device.command_encoder()?; + encoder.set_label("where"); + if t.dtype() != f.dtype() { + crate::bail!( + "Invalid where: different dtypes for values {:?} != {:?}", + t.dtype(), + f.dtype() + ); + } + let name = match (self.dtype, t.dtype()) { + (DType::U8, DType::F32) => "where_u8_f32", + (DType::U32, DType::F32) => "where_u32_f32", + (DType::U8, DType::BF16) => "where_u8_bf16", + (DType::U8, DType::F16) => "where_u8_f16", + (DType::U8, DType::I64) => "where_u8_i64", + (DType::U8, DType::U32) => "where_u8_u32", + (DType::U8, DType::U8) => "where_u8_u8", + (left, right) => crate::bail!("Metal where_cond {left:?} {right:?} not implemented"), + }; + let src = buffer_o(&self.buffer, layout, self.dtype); + let t = buffer_o(&t.buffer, t_l, t.dtype); + let f = buffer_o(&f.buffer, f_l, f.dtype); + candle_metal_kernels::call_where_cond( + &device.device, + &encoder, + &device.kernels, + name, + dtype.size_in_bytes(), + dims, + src, + layout.stride(), + layout.is_contiguous(), + t, + t_l.stride(), + t_l.is_contiguous(), + f, + f_l.stride(), + f_l.is_contiguous(), + &buffer, + ) + .map_err(MetalError::from)?; + Ok(Self::new(buffer, device, el, dtype)) + } + + fn conv1d( + &self, + layout: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &ParamsConv1D, + ) -> Result { + let device = self.device().clone(); + let shape = layout.shape(); + let dims = shape.dims(); + let strides = layout.stride(); + + let stride = params.stride; + let dilation = params.dilation; + let padding = params.padding; + let k_size = params.k_size; + let l_out = (dims[2] + 2 * padding - dilation * (k_size - 1) - 1) / stride + 1; + let dst_el = dims[0] * l_out * dims[1] * k_size; + let dst = self + .device + .new_buffer(dst_el, self.dtype, "conv1d_im2col")?; + let encoder = self.device.command_encoder()?; + encoder.set_label("conv1d_im2col"); + let name = match self.dtype { + DType::F32 => "im2col1d_f32", + DType::F16 => "im2col1d_f16", + DType::BF16 => "im2col1d_bf16", + DType::U8 => "im2col1d_u8", + DType::U32 => "im2col1d_u32", + dtype => crate::bail!("Metal conv1d {dtype:?} not implemented"), + }; + let src = buffer_o(&self.buffer, layout, self.dtype); + candle_metal_kernels::call_im2col1d_strided( + &self.device.device, + &encoder, + &self.device.kernels, + name, + layout.shape().dims(), + strides, + (k_size, stride, padding, dilation), + src, + &dst, + ) + .map_err(MetalError::from)?; + drop(encoder); + let col = Self { + buffer: dst, + device, + count: dst_el, + dtype: self.dtype, + }; + let l_out = params.l_out(); + let b = params.b_size; + let n = params.c_out; + let k = params.k_size * params.c_in; + let m = l_out; + let col_l = Layout::contiguous((b, m, k)); + let res = if kernel_l.is_contiguous() { + let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) + .transpose(1, 2)? + .broadcast_as((b, k, n))?; + col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? + } else { + // Make the kernel contiguous if not already the case. + let mut kernel_c = self.device().zeros_impl(kernel_l.shape(), kernel.dtype())?; + kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?; + let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) + .transpose(1, 2)? + .broadcast_as((b, k, n))?; + col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? + }; + let res_l = Layout::contiguous((b, l_out, n)).transpose(1, 2)?; + let mut res_t = self.device().zeros_impl(res_l.shape(), res.dtype())?; + res.copy_strided_src(&mut res_t, 0, &res_l)?; + Ok(res_t) + } + + fn conv_transpose1d( + &self, + layout: &Layout, + k: &Self, + k_layout: &Layout, + params: &ParamsConvTranspose1D, + ) -> Result { + const USE_COL2IM_CONV1D_TR: bool = true; + + let can_use_col2im = k_layout.is_contiguous() + && params.dilation == 1 + && params.padding == 0 + && params.output_padding == 0; + let l_out = params.l_out(); + let dst_el = params.c_out * l_out * params.b_size; + + let buffer = if USE_COL2IM_CONV1D_TR && can_use_col2im { + let (b_size, c_in, l_in) = layout.shape().dims3()?; + let (c_in2, c_out, k_size) = k_layout.shape().dims3()?; + if c_in != c_in2 { + crate::bail!( + "convtr1d: shape mismatch on c_in {:?} {:?}", + layout.shape(), + k_layout.shape() + ) + } + let buffer = self + .device + .new_buffer(dst_el, self.dtype, "conv_transpose1d")?; + + let name = match self.dtype { + DType::F32 => "col2im1d_f32", + DType::F16 => "col2im1d_f16", + DType::BF16 => "col2im1d_bf16", + DType::U32 => "col2im1d_u32", + DType::U8 => "col2im1d_u8", + dtype => crate::bail!("metal col2im1d {dtype:?} not implemented"), + }; + let col = { + // This merges the last two dimensions of the kernel together. + let kernel_l_mm = Layout::new( + (b_size, c_in, k_size * c_out).into(), + vec![0, k_size * c_out, 1], + k_layout.start_offset(), + ); + self.matmul( + k, + (b_size, l_in, c_out * k_size, c_in), + &layout.transpose(1, 2)?, + &kernel_l_mm, + )? + }; + // It is important for the command encoder to be obtained *after* the matmul + // kernel has run, otherwise we might use a command-buffer that has been committed + // already resulting in the following error. + // _status < MTLCommandBufferStatusCommitted > + // -[IOGPUMetalCommandBuffer setCurrentCommandEncoder:] + let encoder = self.device.command_encoder()?; + encoder.set_label("col2im1d"); + candle_metal_kernels::call_col2im1d( + &self.device.device, + &encoder, + &self.device.kernels, + name, + &[b_size, l_in, c_out, k_size], + params.k_size, + params.stride, + BufferOffset::zero_offset(&col.buffer), + &buffer, + ) + .map_err(MetalError::from)?; + buffer + } else { + let buffer = self + .device + .new_buffer(dst_el, self.dtype, "conv_transpose1d")?; + + let encoder = self.device.command_encoder()?; + encoder.set_label("conv_transpose1d"); + let name = match self.dtype { + DType::F32 => "conv_transpose1d_f32", + DType::F16 => "conv_transpose1d_f16", + DType::BF16 => "conv_transpose1d_bf16", + DType::U32 => "conv_transpose1d_u32", + DType::U8 => "conv_transpose1d_u8", + dtype => crate::bail!("Metal conv_transpose1d {dtype:?} not implemented"), + }; + candle_metal_kernels::call_conv_transpose1d( + &self.device.device, + &encoder, + &self.device.kernels, + name, + params.dilation, + params.stride, + params.padding, + params.output_padding, + params.c_out, + l_out, + params.b_size, + layout.dims(), + layout.stride(), + k_layout.dims(), + k_layout.stride(), + &self.buffer, + layout.start_offset() * self.dtype.size_in_bytes(), + &k.buffer, + k_layout.start_offset() * k.dtype.size_in_bytes(), + &buffer, + ) + .map_err(MetalError::from)?; + buffer + }; + Ok(Self::new(buffer, self.device.clone(), dst_el, self.dtype)) + } + + fn conv2d( + &self, + layout: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &ParamsConv2D, + ) -> Result { + let device = self.device().clone(); + let shape = layout.shape(); + let dims = shape.dims(); + + let stride = params.stride; + let dilation = params.dilation; + let padding = params.padding; + let h_k = params.k_h; + let w_k = params.k_w; + let h = dims[2]; + let w = dims[3]; + let h_out = (h + 2 * padding - dilation * (h_k - 1) - 1) / stride + 1; + let w_out = (w + 2 * padding - dilation * (w_k - 1) - 1) / stride + 1; + let dst_el = dims[0] * h_out * w_out * dims[1] * h_k * w_k; + + let dst = self + .device + .new_buffer(dst_el, self.dtype, "conv2d_im2col")?; + let encoder = self.device.command_encoder()?; + encoder.set_label("conv2d_im2col"); + let name = match self.dtype { + DType::F32 => "im2col_f32", + DType::F16 => "im2col_f16", + DType::BF16 => "im2col_bf16", + DType::U8 => "im2col_u8", + DType::U32 => "im2col_u32", + dtype => crate::bail!("Metal conv2d {dtype:?} not implemented"), + }; + let src = buffer_o(&self.buffer, layout, self.dtype); + candle_metal_kernels::call_im2col_strided( + &self.device.device, + &encoder, + &self.device.kernels, + name, + layout.shape().dims(), + layout.stride(), + (h_k, w_k, stride, padding, dilation), + src, + &dst, + ) + .map_err(MetalError::from)?; + drop(encoder); + let col = Self { + buffer: dst, + device, + count: dst_el, + dtype: self.dtype, + }; + let h_out = params.out_h(); + let w_out = params.out_w(); + let b = params.b_size; + let n = params.c_out; + let k = params.k_h * params.k_w * params.c_in; + let m = h_out * w_out; + let col_l = Layout::contiguous((b, m, k)); + let res = if kernel_l.is_contiguous() { + let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) + .transpose(1, 2)? + .broadcast_as((b, k, n))?; + col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? + } else { + // Make the kernel contiguous if not already the case. + let mut kernel_c = self.device().zeros_impl(kernel_l.shape(), kernel.dtype())?; + kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?; + let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) + .transpose(1, 2)? + .broadcast_as((b, k, n))?; + col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? + }; + let res_l = Layout::contiguous((b, h_out, w_out, n)) + .transpose(1, 2)? + .transpose(1, 3)?; + let mut res_t = self.device().zeros_impl(res_l.shape(), res.dtype())?; + res.copy_strided_src(&mut res_t, 0, &res_l)?; + Ok(res_t) + } + + fn conv_transpose2d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &ParamsConvTranspose2D, + ) -> Result { + // Kernel shape: (c_in_k, c_out, h_k, w_k) + // Input shape: (b_size, c_in, h_in, w_in) + let (out_w, out_h) = (params.out_w(), params.out_h()); + let dst_el = params.c_out * out_w * out_h * params.b_size; + + let dims = l.dims(); + if dims.len() != 4 { + crate::bail!("unexpected input shape for conv_transpose2d {dims:?}, expected 4") + } + + let k_dims = kernel_l.dims(); + if k_dims.len() != 4 { + crate::bail!("unexpected kernel shape for conv_transpose2d {k_dims:?}, expected 4") + } + + let buffer = self + .device + .new_buffer(dst_el, self.dtype, "conv_transpose2d")?; + + let encoder = self.device.command_encoder()?; + encoder.set_label("conv_transpose2d"); + + let name = match self.dtype { + DType::F32 => "conv_transpose2d_f32", + DType::F16 => "conv_transpose2d_f16", + DType::BF16 => "conv_transpose2d_bf16", + dtype => crate::bail!("Metal conv_transpose2d {dtype:?} not implemented"), + }; + + candle_metal_kernels::call_conv_transpose2d( + &self.device.device, + &encoder, + &self.device.kernels, + name, + CallConvTranspose2dCfg { + dilation: params.dilation, + stride: params.stride, + padding: params.padding, + output_padding: params.output_padding, + c_out: params.c_out, + out_h, + out_w, + b_size: params.b_size, + input_dims: l.dims(), + input_stride: l.stride(), + kernel_dims: kernel_l.dims(), + kernel_stride: kernel_l.stride(), + input_offset: l.start_offset() * self.dtype.size_in_bytes(), + kernel_offset: kernel_l.start_offset() * kernel.dtype.size_in_bytes(), + }, + &self.buffer, + &kernel.buffer, + &buffer, + ) + .map_err(MetalError::from)?; + Ok(Self::new(buffer, self.device.clone(), dst_el, self.dtype)) + } + + fn avg_pool2d( + &self, + inp_l: &Layout, + (w_k, h_k): (usize, usize), + (w_stride, h_stride): (usize, usize), + ) -> Result { + let shape = inp_l.shape(); + let (b_size, channels, width, height) = shape.dims4()?; + let strides = inp_l.stride(); + let name = match self.dtype { + DType::F32 => "avg_pool2d_f32", + DType::F16 => "avg_pool2d_f16", + DType::BF16 => "avg_pool2d_bf16", + DType::U8 => "avg_pool2d_u8", + DType::U32 => "avg_pool2d_u32", + dtype => crate::bail!("Metal avg_pool2d {dtype:?} not implemented"), + }; + let out_w = (width - w_k) / w_stride + 1; + let out_h = (height - h_k) / h_stride + 1; + let dst_el = out_w * out_h * b_size * channels; + let buffer = self.device.new_buffer(dst_el, self.dtype, "avg_pool2d")?; + let encoder = self.device.command_encoder()?; + encoder.set_label("avg_pool2d"); + candle_metal_kernels::call_pool2d( + &self.device.device, + &encoder, + &self.device.kernels, + name, + inp_l.dims(), + strides, + out_w, + out_h, + w_k, + h_k, + w_stride, + h_stride, + &self.buffer, + &buffer, + ) + .map_err(MetalError::from)?; + Ok(Self::new(buffer, self.device.clone(), dst_el, self.dtype)) + } + + fn max_pool2d( + &self, + inp_l: &Layout, + (w_k, h_k): (usize, usize), + (w_stride, h_stride): (usize, usize), + ) -> Result { + let shape = inp_l.shape(); + let (b_size, channels, width, height) = shape.dims4()?; + let strides = inp_l.stride(); + let name = match self.dtype { + DType::F32 => "max_pool2d_f32", + DType::F16 => "max_pool2d_f16", + DType::BF16 => "max_pool2d_bf16", + DType::U8 => "max_pool2d_u8", + DType::U32 => "max_pool2d_u32", + dtype => crate::bail!("Metal max_pool2d {dtype:?} not implemented"), + }; + let out_w = (width - w_k) / w_stride + 1; + let out_h = (height - h_k) / h_stride + 1; + let dst_el = out_w * out_h * b_size * channels; + let buffer = self.device.new_buffer(dst_el, self.dtype, "max_pool2d")?; + let encoder = self.device.command_encoder()?; + encoder.set_label("max_pool2d"); + candle_metal_kernels::call_pool2d( + &self.device.device, + &encoder, + &self.device.kernels, + name, + inp_l.dims(), + strides, + out_w, + out_h, + w_k, + h_k, + w_stride, + h_stride, + &self.buffer, + &buffer, + ) + .map_err(MetalError::from)?; + Ok(Self::new(buffer, self.device.clone(), dst_el, self.dtype)) + } + + fn upsample_nearest1d(&self, _: &Layout, _: usize) -> Result { + crate::bail!("Metal upsample_nearest1d not implemented") + } + + fn upsample_nearest2d(&self, inp_l: &Layout, out_w: usize, out_h: usize) -> Result { + // let inp = &inp.slice(inp_l.start_offset()..); + let shape = inp_l.shape(); + let dims = shape.dims(); + let strides = inp_l.stride(); + if dims.len() != 4 { + crate::bail!("unexpected input shape for upsample {dims:?}") + } + let name = match self.dtype { + DType::F32 => "upsample_nearest2d_f32", + DType::F16 => "upsample_nearest2d_f16", + DType::BF16 => "upsample_nearest2d_bf16", + DType::U8 => "upsample_nearest2d_u8", + DType::U32 => "upsample_nearest2d_u32", + dtype => crate::bail!("Metal upsample_nearest2d {dtype:?} not implemented"), + }; + + let dst_el = out_w * out_h * dims[0] * dims[1]; + let buffer = self + .device + .new_buffer(dst_el, self.dtype, "upsample_nearest2d")?; + let encoder = self.device.command_encoder()?; + encoder.set_label("upsample_nearest2d"); + let src = buffer_o(&self.buffer, inp_l, self.dtype); + candle_metal_kernels::call_upsample_nearest_2d( + &self.device.device, + &encoder, + &self.device.kernels, + name, + dims, + strides, + out_w, + out_h, + src, + &buffer, + ) + .map_err(MetalError::from)?; + Ok(Self::new(buffer, self.device.clone(), dst_el, self.dtype)) + } + + fn upsample_bilinear2d( + &self, + inp_l: &Layout, + out_h: usize, + out_w: usize, + align_corners: bool, + scale_h: Option, + scale_w: Option, + ) -> Result { + let shape = inp_l.shape(); + let dims = shape.dims(); + let strides = inp_l.stride(); + + if dims.len() != 4 { + crate::bail!("unexpected input shape for upsample_bilinear2d {dims:?}") + } + + let name = match self.dtype { + DType::F32 => "upsample_bilinear2d_f32", + DType::F16 => "upsample_bilinear2d_f16", + DType::BF16 => "upsample_bilinear2d_bf16", + DType::U8 => "upsample_bilinear2d_u8", + DType::U32 => "upsample_bilinear2d_u32", + dtype => crate::bail!("Metal upsample_bilinear2d {dtype:?} not implemented"), + }; + + let dst_el = out_w * out_h * dims[0] * dims[1]; + let buffer = self + .device + .new_buffer(dst_el, self.dtype, "upsample_bilinear2d")?; + + let encoder = self.device.command_encoder()?; + encoder.set_label("upsample_bilinear2d"); + + let src = buffer_o(&self.buffer, inp_l, self.dtype); + candle_metal_kernels::call_upsample_bilinear_2d( + &self.device.device, + &encoder, + &self.device.kernels, + name, + dims, + strides, + out_w, + out_h, + align_corners, + scale_h, + scale_w, + src, + &buffer, + ) + .map_err(MetalError::from)?; + + Ok(Self::new(buffer, self.device.clone(), dst_el, self.dtype)) + } + + fn gather(&self, src_l: &Layout, ids: &Self, ids_l: &Layout, dim: usize) -> Result { + if !ids_l.is_contiguous() { + return Err(crate::Error::RequiresContiguous { op: "gather" }.bt()); + }; + let ids_el = ids_l.dims()[dim]; + let dst_el = ids_l.shape().elem_count(); + let dtype = self.dtype; + let device = self.device(); + let buffer = device.new_buffer(dst_el, dtype, "gather")?; + let name = match (ids.dtype, self.dtype) { + (DType::U8, DType::U8) => "gather_u8_u8", + (DType::U8, DType::F32) => "gather_u8_f32", + (DType::U8, DType::F16) => "gather_u8_f16", + (DType::U8, DType::BF16) => "gather_u8_bf16", + (DType::U8, DType::U32) => "gather_u8_u32", + (DType::U8, DType::I64) => "gather_u8_i64", + (DType::U32, DType::F32) => "gather_u32_f32", + (DType::U32, DType::F16) => "gather_u32_f16", + (DType::U32, DType::BF16) => "gather_u32_bf16", + (DType::U32, DType::U32) => "gather_u32_u32", + (DType::U32, DType::I64) => "gather_u32_i64", + (DType::I64, DType::F32) => "gather_i64_f32", + (DType::I64, DType::F16) => "gather_i64_f16", + (DType::I64, DType::BF16) => "gather_i64_bf16", + (DType::I64, DType::U32) => "gather_i64_u32", + (DType::I64, DType::I64) => "gather_i64_i64", + (left, right) => crate::bail!("Metal gather {left:?} {right:?} not implemented"), + }; + let encoder = self.device.command_encoder()?; + encoder.set_label("gather"); + let src = buffer_o(&self.buffer, src_l, dtype); + let ids = buffer_o(&ids.buffer, ids_l, ids.dtype); + candle_metal_kernels::call_gather( + &device.device, + &encoder, + &self.device.kernels, + name, + src_l.dims(), + ids_el, + dim, + src, + ids, + &buffer, + ) + .map_err(MetalError::from)?; + Ok(Self::new(buffer, device.clone(), dst_el, dtype)) + } + + fn scatter_set( + &mut self, + l: &Layout, + ids: &Self, + ids_l: &Layout, + src: &Self, + src_l: &Layout, + dim: usize, + ) -> Result<()> { + if !l.is_contiguous() || !ids_l.is_contiguous() || !src_l.is_contiguous() { + return Err(crate::Error::RequiresContiguous { op: "scatter" }.bt()); + }; + let name = match (ids.dtype, self.dtype) { + (DType::U8, DType::F32) => "s_u8_f32", + (DType::U8, DType::F16) => "s_u8_f16", + (DType::U8, DType::BF16) => "s_u8_bf16", + (DType::U32, DType::U32) => "s_u32_u32", + (DType::U32, DType::F32) => "s_u32_f32", + (DType::U32, DType::F16) => "s_u32_f16", + (DType::U32, DType::BF16) => "s_u32_bf16", + (DType::I64, DType::F32) => "s_i64_f32", + (DType::I64, DType::F16) => "s_i64_f16", + (DType::I64, DType::BF16) => "s_i64_bf16", + _ => Err(MetalError::UnexpectedDType { + msg: "scatter ids should be u8/u32/i64", + expected: DType::U32, + got: ids.dtype(), + })?, + }; + let encoder = self.device.command_encoder()?; + encoder.set_label("scatter"); + let dst = buffer_o(&self.buffer, l, self.dtype); + let src = buffer_o(&src.buffer, src_l, src.dtype); + let ids = buffer_o(&ids.buffer, ids_l, ids.dtype); + candle_metal_kernels::call_scatter( + &self.device.device, + &encoder, + &self.device.kernels, + name, + src_l.dims(), + l.dims(), + dim, + src, + ids, + dst, + ) + .map_err(MetalError::from)?; + Ok(()) + } + + fn scatter_add_set( + &mut self, + l: &Layout, + ids: &Self, + ids_l: &Layout, + src: &Self, + src_l: &Layout, + dim: usize, + ) -> Result<()> { + if !l.is_contiguous() || !ids_l.is_contiguous() || !src_l.is_contiguous() { + return Err(crate::Error::RequiresContiguous { op: "scatter-add" }.bt()); + }; + let name = match (ids.dtype, self.dtype) { + (DType::U8, DType::F32) => "sa_u8_f32", + (DType::U8, DType::F16) => "sa_u8_f16", + (DType::U8, DType::BF16) => "sa_u8_bf16", + (DType::U32, DType::U32) => "sa_u32_u32", + (DType::U32, DType::F32) => "sa_u32_f32", + (DType::U32, DType::F16) => "sa_u32_f16", + (DType::U32, DType::BF16) => "sa_u32_bf16", + (DType::I64, DType::F32) => "sa_i64_f32", + (DType::I64, DType::F16) => "sa_i64_f16", + (DType::I64, DType::BF16) => "sa_i64_bf16", + _ => Err(MetalError::UnexpectedDType { + msg: "scatter-add ids should be u8/u32/i64", + expected: DType::U32, + got: ids.dtype(), + })?, + }; + let encoder = self.device.command_encoder()?; + encoder.set_label("scatter_add"); + let dst = buffer_o(&self.buffer, l, self.dtype); + let src = buffer_o(&src.buffer, src_l, src.dtype); + let ids = buffer_o(&ids.buffer, ids_l, ids.dtype); + candle_metal_kernels::call_scatter( + &self.device.device, + &encoder, + &self.device.kernels, + name, + src_l.dims(), + l.dims(), + dim, + src, + ids, + dst, + ) + .map_err(MetalError::from)?; + Ok(()) + } + + fn index_select(&self, ids: &Self, src_l: &Layout, ids_l: &Layout, dim: usize) -> Result { + if !ids_l.is_contiguous() { + crate::bail!("Metal index_select requires contiguous ids") + } + let left_size: usize = src_l.dims()[..dim].iter().product(); + let right_size: usize = src_l.dims()[dim + 1..].iter().product(); + let ids_el = ids_l.shape().elem_count(); + let dst_el = ids_el * left_size * right_size; + let dtype = self.dtype; + let device = self.device(); + let buffer = device.new_buffer(dst_el, dtype, "index_select")?; + let name = match (ids.dtype, self.dtype) { + (DType::U8, DType::U8) => "is_u8_u8", + (DType::U8, DType::U32) => "is_u8_u32", + (DType::U8, DType::I64) => "is_u8_i64", + (DType::U8, DType::BF16) => "is_u8_bf16", + (DType::U8, DType::F32) => "is_u8_f32", + (DType::U8, DType::F16) => "is_u8_f16", + + (DType::U32, DType::U8) => "is_u32_u8", + (DType::U32, DType::U32) => "is_u32_u32", + (DType::U32, DType::I64) => "is_u32_i64", + (DType::U32, DType::F32) => "is_u32_f32", + (DType::U32, DType::F16) => "is_u32_f16", + (DType::U32, DType::BF16) => "is_u32_bf16", + + (DType::I64, DType::U8) => "is_i64_u8", + (DType::I64, DType::U32) => "is_i64_u32", + (DType::I64, DType::I64) => "is_i64_i64", + (DType::I64, DType::F32) => "is_i64_f32", + (DType::I64, DType::F16) => "is_i64_f16", + (DType::I64, DType::BF16) => "is_i64_bf16", + + (left, right) => { + crate::bail!("Metal contiguous index_select {left:?} {right:?} not implemented") + } + }; + let encoder = self.device.command_encoder()?; + let src = buffer_o(&self.buffer, src_l, dtype); + let ids = buffer_o(&ids.buffer, ids_l, ids.dtype); + candle_metal_kernels::call_index_select( + &device.device, + &encoder, + &self.device.kernels, + name, + src_l.dims(), + ids_el, + dim, + src_l.is_contiguous(), + src_l.dims(), + src_l.stride(), + src, + ids, + &buffer, + ) + .map_err(MetalError::from)?; + Ok(Self::new(buffer, device.clone(), dst_el, dtype)) + } + + fn index_add( + &self, + l: &Layout, + ids: &Self, + ids_l: &Layout, + src: &Self, + src_l: &Layout, + dim: usize, + ) -> Result { + let mut acc = self.device.zeros_impl(l.shape(), self.dtype())?; + self.copy_strided_src(&mut acc, 0, l)?; + if !ids_l.is_contiguous() || !src_l.is_contiguous() { + return Err(crate::Error::RequiresContiguous { op: "index-add" }.bt()); + }; + let name = match (ids.dtype, self.dtype) { + (DType::I64, DType::BF16) => "ia_i64_bf16", + (DType::I64, DType::F16) => "ia_i64_f16", + (DType::I64, DType::F32) => "ia_i64_f32", + (DType::I64, DType::I64) => "ia_i64_i64", + (DType::I64, DType::U32) => "ia_i64_u32", + (DType::I64, DType::U8) => "ia_i64_u8", + + (DType::U32, DType::BF16) => "ia_u32_bf16", + (DType::U32, DType::F16) => "ia_u32_f16", + (DType::U32, DType::F32) => "ia_u32_f32", + (DType::U32, DType::I64) => "ia_u32_i64", + (DType::U32, DType::U32) => "ia_u32_u32", + (DType::U32, DType::U8) => "ia_u32_u8", + + (DType::U8, DType::BF16) => "ia_u8_bf16", + (DType::U8, DType::F16) => "ia_u8_f16", + (DType::U8, DType::F32) => "ia_u8_f32", + (DType::U8, DType::I64) => "ia_u8_i64", + (DType::U8, DType::U32) => "ia_u8_u32", + (DType::U8, DType::U8) => "ia_u8_u8", + + _ => Err(MetalError::UnexpectedDType { + msg: "index-add ids should be u8/u32/i64", + expected: DType::U32, + got: ids.dtype(), + })?, + }; + let encoder = self.device.command_encoder()?; + encoder.set_label("index_add"); + let src = buffer_o(&src.buffer, src_l, src.dtype); + let ids = buffer_o(&ids.buffer, ids_l, ids.dtype); + candle_metal_kernels::call_index_add( + &self.device.device, + &encoder, + &self.device.kernels, + name, + src_l.dims(), + l.dims(), + ids_l.dims(), + dim, + src, + ids, + &acc.buffer, + ) + .map_err(MetalError::from)?; + Ok(acc) + } + + fn matmul( + &self, + rhs: &Self, + (b, m, n, k): (usize, usize, usize, usize), + lhs_l: &Layout, + rhs_l: &Layout, + ) -> Result { + let buffer = self.device.new_buffer(b * m * n, self.dtype, "matmul")?; + let encoder = self.device.command_encoder()?; + encoder.set_label("matmul"); + let dtype = match self.dtype { + DType::F32 => candle_metal_kernels::GemmDType::F32, + DType::F16 => candle_metal_kernels::GemmDType::F16, + DType::BF16 => candle_metal_kernels::GemmDType::BF16, + dtype => { + return Err( + MetalError::Message(format!("mlx matmul doesn't support {dtype:?}")).into(), + ) + } + }; + candle_metal_kernels::call_mlx_gemm( + &self.device.device, + &encoder, + &self.device.kernels, + dtype, + (b, m, n, k), + lhs_l.stride(), + lhs_l.start_offset() * self.dtype.size_in_bytes(), + &self.buffer, + rhs_l.stride(), + rhs_l.start_offset() * rhs.dtype.size_in_bytes(), + &rhs.buffer, + &buffer, + ) + .map_err(MetalError::from)?; + + Ok(Self::new( + buffer, + self.device.clone(), + b * m * n, + self.dtype(), + )) + } + + fn copy2d( + &self, + dst: &mut Self, + d1: usize, + d2: usize, + src_s: usize, + dst_s: usize, + src_o: usize, + dst_o: usize, + ) -> Result<()> { + if self.dtype() != dst.dtype() { + crate::bail!( + "copy2d with inconsistent dtypes {:?} {:?}", + self.dtype(), + dst.dtype() + ) + } + if src_s == d2 && dst_s == d2 { + let blit = self.device.blit_command_encoder()?; + blit.set_label("copy2d_contiguous"); + let src_offset = src_o * self.dtype.size_in_bytes(); + let length = d1 * d2 * self.dtype.size_in_bytes(); + let dst_offset = dst_o * dst.dtype().size_in_bytes(); + blit.copy_from_buffer(&self.buffer, src_offset, dst.buffer(), dst_offset, length); + blit.end_encoding(); + } else { + let el_count = d1 * d2; + if el_count == 0 { + return Ok(()); + } + let kernel_name = match self.dtype { + DType::F32 => candle_metal_kernels::copy2d::FLOAT, + DType::F16 => candle_metal_kernels::copy2d::HALF, + DType::BF16 => candle_metal_kernels::copy2d::BFLOAT, + DType::I64 => candle_metal_kernels::copy2d::I64, + DType::U32 => candle_metal_kernels::copy2d::U32, + DType::U8 => candle_metal_kernels::copy2d::U8, + dtype => crate::bail!("Metal copy2d {dtype:?} not implemented"), + }; + let encoder = self.device.command_encoder()?; + encoder.set_label("copy2d"); + candle_metal_kernels::call_copy2d( + &self.device.device, + &encoder, + &self.device.kernels, + kernel_name, + &self.buffer, + &dst.buffer, + d1, + d2, + src_s, + dst_s, + src_o * self.dtype.size_in_bytes(), + dst_o * self.dtype.size_in_bytes(), + ) + .map_err(MetalError::from)?; + } + Ok(()) + } + + fn copy_strided_src(&self, dst: &mut Self, dst_offset: usize, src_l: &Layout) -> Result<()> { + if src_l.is_contiguous() && self.dtype == dst.dtype() { + let blit = self.device.blit_command_encoder()?; + blit.set_label("copy_contiguous"); + let src_offset = src_l.start_offset() * self.dtype.size_in_bytes(); + let length = src_l.shape().elem_count() * self.dtype.size_in_bytes(); + let dst_offset = dst_offset * dst.dtype().size_in_bytes(); + blit.copy_from_buffer(&self.buffer, src_offset, dst.buffer(), dst_offset, length); + blit.end_encoding(); + } else { + let src_shape = src_l.shape(); + let el_count = src_shape.elem_count(); + if el_count == 0 { + return Ok(()); + } + let kernel_name = match self.dtype { + DType::F32 => candle_metal_kernels::unary::strided::copy::FLOAT, + DType::F16 => candle_metal_kernels::unary::strided::copy::HALF, + DType::BF16 => candle_metal_kernels::unary::strided::copy::BFLOAT, + DType::I64 => candle_metal_kernels::unary::strided::copy::I64, + DType::U32 => candle_metal_kernels::unary::strided::copy::U32, + DType::U8 => candle_metal_kernels::unary::strided::copy::U8, + dtype => crate::bail!("Metal copy_strided {dtype:?} not implemented"), + }; + let src = buffer_o(&self.buffer, src_l, self.dtype); + let dst = BufferOffset { + buffer: &dst.buffer, + offset_in_bytes: dst_offset * dst.dtype.size_in_bytes(), + }; + let encoder = self.device.command_encoder()?; + encoder.set_label("copy_strided"); + candle_metal_kernels::call_unary_strided( + &self.device.device, + &encoder, + &self.device.kernels, + kernel_name, + src_l.dims(), + src, + src_l.stride(), + dst, + ) + .map_err(MetalError::from)?; + } + Ok(()) + } +} + +impl MetalStorage { + pub fn new(buffer: Arc, device: MetalDevice, count: usize, dtype: DType) -> Self { + Self { + buffer, + device, + count, + dtype, + } + } + + pub fn buffer(&self) -> &Buffer { + &self.buffer + } + + pub fn binary( + &self, + op: &'static str, + rhs: &Self, + lhs_l: &Layout, + rhs_l: &Layout, + ) -> Result { + fn kernel_name(op: &'static str, dtype: &DType, suffix: &str) -> String { + format!("{op}_{}{}", dtype.as_str(), suffix) + } + let device = self.device(); + let shape = lhs_l.shape(); + let el_count = shape.elem_count(); + let encoder = device.command_encoder()?; + let lhs = buffer_o(&self.buffer, lhs_l, self.dtype); + let rhs = buffer_o(&rhs.buffer, rhs_l, rhs.dtype); + + let dtype = match op { + "eq" | "ne" | "le" | "lt" | "ge" | "gt" => DType::U8, + _ => self.dtype, + }; + let lhs_contiguous = lhs_l.is_contiguous(); + let rhs_contiguous = rhs_l.is_contiguous(); + + let buffer = if lhs_contiguous && rhs_contiguous { + let kernel = kernel_name(op, &self.dtype, ""); + let buffer = device.new_buffer(el_count, dtype, op)?; + candle_metal_kernels::call_binary_contiguous( + &device.device, + &encoder, + &device.kernels, + kernel, + self.dtype.size_in_bytes(), + el_count, + lhs, + rhs, + &buffer, + ) + .map_err(MetalError::from)?; + buffer + } else { + let strided_suffix = if lhs_contiguous { + "_rstrided" + } else if rhs_contiguous { + "_lstrided" + } else { + "_strided" + }; + let kernel = kernel_name(op, &self.dtype, strided_suffix); + let buffer = device.new_buffer(el_count, dtype, op)?; + candle_metal_kernels::call_binary_strided( + &device.device, + &encoder, + &device.kernels, + kernel, + self.dtype.size_in_bytes(), + lhs_l.dims(), + lhs, + lhs_l.stride(), + rhs, + rhs_l.stride(), + &buffer, + ) + .map_err(MetalError::from)?; + buffer + }; + encoder.set_label("binary"); + Ok(Self::new(buffer, device.clone(), el_count, dtype)) + } + + pub(crate) fn to_cpu(&self) -> Result> { + let size = self.count * self.dtype.size_in_bytes(); + let buffer = self.device.allocate_buffer(size)?; + { + let blit = self.device.blit_command_encoder()?; + blit.set_label("blit_to_cpu"); + blit.copy_from_buffer(&self.buffer, 0, &buffer, 0, size); + blit.end_encoding(); + } + self.device.wait_until_completed()?; + Ok(read_to_vec(&buffer, self.count)) + } +} + +impl BackendDevice for MetalDevice { + type Storage = MetalStorage; + + fn new(ordinal: usize) -> Result { + let device = Device::all().swap_remove(ordinal); + let command_queue = device.new_command_queue().map_err(MetalError::from)?; + let kernels = Arc::new(Kernels::new()); + let seed = Arc::new(Mutex::new( + device + .new_buffer_with_data( + [299792458u64].as_ptr() as *const c_void, + 4, + RESOURCE_OPTIONS, + ) + .map_err(MetalError::from)?, + )); + let commands = Commands::new(command_queue).map_err(MetalError::from)?; + Ok(Self { + id: DeviceId::new(), + device, + commands: Arc::new(RwLock::new(commands)), + buffers: Arc::new(RwLock::new(HashMap::new())), + kernels, + seed, + seed_value: Arc::new(RwLock::new(299792458)), + }) + } + + fn location(&self) -> crate::DeviceLocation { + crate::DeviceLocation::Metal { + gpu_id: self.registry_id() as usize, + } + } + + fn same_device(&self, rhs: &Self) -> bool { + self.id == rhs.id + } + + unsafe fn alloc_uninit(&self, shape: &Shape, dtype: DType) -> Result { + let buffer = self.new_buffer(shape.elem_count(), dtype, "alloc-uninit")?; + Ok(MetalStorage::new( + buffer, + self.clone(), + shape.elem_count(), + dtype, + )) + } + + fn zeros_impl(&self, shape: &Shape, dtype: DType) -> Result { + let size = shape.elem_count() * dtype.size_in_bytes(); + let buffer = self.allocate_zeros(size)?; + Ok(MetalStorage::new( + buffer, + self.clone(), + shape.elem_count(), + dtype, + )) + } + + fn storage_from_slice(&self, s: &[T]) -> Result { + let (count, buffer) = match T::cpu_storage_ref(s) { + CpuStorageRef::U8(storage) => (storage.len(), self.new_buffer_with_data(storage)), + CpuStorageRef::U32(storage) => (storage.len(), self.new_buffer_with_data(storage)), + CpuStorageRef::I16(storage) => (storage.len(), self.new_buffer_with_data(storage)), + CpuStorageRef::I32(storage) => (storage.len(), self.new_buffer_with_data(storage)), + CpuStorageRef::I64(storage) => (storage.len(), self.new_buffer_with_data(storage)), + CpuStorageRef::BF16(storage) => (storage.len(), self.new_buffer_with_data(storage)), + CpuStorageRef::F16(storage) => (storage.len(), self.new_buffer_with_data(storage)), + CpuStorageRef::F32(storage) => (storage.len(), self.new_buffer_with_data(storage)), + CpuStorageRef::F64(storage) => (storage.len(), self.new_buffer_with_data(storage)), + CpuStorageRef::F8E4M3(storage) => (storage.len(), self.new_buffer_with_data(storage)), + CpuStorageRef::F6E2M3(_) + | CpuStorageRef::F6E3M2(_) + | CpuStorageRef::F4(_) + | CpuStorageRef::F8E8M0(_) => { + return Err(Error::UnsupportedDTypeForOp(T::DTYPE, "to_dtype").bt()) + } + }; + Ok(Self::Storage::new(buffer?, self.clone(), count, T::DTYPE)) + } + + fn storage_from_cpu_storage(&self, storage: &CpuStorage) -> Result { + let (count, buffer) = match storage { + CpuStorage::U8(storage) => (storage.len(), self.new_buffer_with_data(storage)), + CpuStorage::U32(storage) => (storage.len(), self.new_buffer_with_data(storage)), + CpuStorage::I16(storage) => (storage.len(), self.new_buffer_with_data(storage)), + CpuStorage::I32(storage) => (storage.len(), self.new_buffer_with_data(storage)), + CpuStorage::I64(storage) => (storage.len(), self.new_buffer_with_data(storage)), + CpuStorage::BF16(storage) => (storage.len(), self.new_buffer_with_data(storage)), + CpuStorage::F16(storage) => (storage.len(), self.new_buffer_with_data(storage)), + CpuStorage::F32(storage) => (storage.len(), self.new_buffer_with_data(storage)), + CpuStorage::F64(storage) => (storage.len(), self.new_buffer_with_data(storage)), + CpuStorage::F8E4M3(storage) => (storage.len(), self.new_buffer_with_data(storage)), + CpuStorage::F6E2M3(_) + | CpuStorage::F6E3M2(_) + | CpuStorage::F4(_) + | CpuStorage::F8E8M0(_) => { + return Err(Error::UnsupportedDTypeForOp(storage.dtype(), "to_dtype").bt()) + } + }; + Ok(Self::Storage::new( + buffer?, + self.clone(), + count, + storage.dtype(), + )) + } + + fn storage_from_cpu_storage_owned(&self, storage: CpuStorage) -> Result { + self.storage_from_cpu_storage(&storage) + } + + fn rand_uniform( + &self, + shape: &Shape, + dtype: DType, + min: f64, + max: f64, + ) -> Result { + let name = match dtype { + DType::F32 => "rand_uniform_f32", + DType::F16 => "rand_uniform_f16", + DType::BF16 => "rand_uniform_bf16", + dtype => crate::bail!("rand_uniform not implemented for {dtype:?}"), + }; + let buffer = self.new_buffer(shape.elem_count(), dtype, "rand_uniform")?; + let encoder = self.command_encoder()?; + encoder.set_label("rand_uniform"); + candle_metal_kernels::call_random_uniform( + &self.device, + &encoder, + &self.kernels, + name, + min as f32, + max as f32, + shape.elem_count(), + &self.seed.lock().unwrap(), + &buffer, + ) + .map_err(MetalError::from)?; + + Ok(Self::Storage::new( + buffer, + self.clone(), + shape.elem_count(), + dtype, + )) + } + + fn rand_normal( + &self, + shape: &Shape, + dtype: DType, + mean: f64, + stddev: f64, + ) -> Result { + let name = match dtype { + DType::F32 => "rand_normal_f32", + DType::F16 => "rand_normal_f16", + DType::BF16 => "rand_normal_bf16", + dtype => crate::bail!("rand_uniform not implemented for {dtype:?}"), + }; + let buffer = self.new_buffer(shape.elem_count(), dtype, "rand_normal")?; + let encoder = self.command_encoder()?; + encoder.set_label("rand_normal"); + candle_metal_kernels::call_random_normal( + &self.device, + &encoder, + &self.kernels, + name, + mean as f32, + stddev as f32, + shape.elem_count(), + &self.seed.lock().unwrap(), + &buffer, + ) + .map_err(MetalError::from)?; + + Ok(Self::Storage::new( + buffer, + self.clone(), + shape.elem_count(), + dtype, + )) + } + + fn set_seed(&self, seed: u64) -> Result<()> { + *self.seed_value.write().unwrap() = seed; + + let seed_buffer = self.seed.try_lock().map_err(MetalError::from)?; + let contents = seed_buffer.data(); + unsafe { + std::ptr::copy_nonoverlapping([seed].as_ptr(), contents as *mut u64, 1); + } + seed_buffer.did_modify_range(NSRange::new(0, 8)); + + Ok(()) + } + + fn get_current_seed(&self) -> Result { + Ok(*self.seed_value.read().unwrap()) + } + + fn synchronize(&self) -> Result<()> { + self.wait_until_completed() + } +} + +fn read_to_vec(buffer: &Buffer, n: usize) -> Vec { + let ptr = buffer.contents() as *const T; + assert!(!ptr.is_null()); + let slice = unsafe { std::slice::from_raw_parts(ptr, n) }; + slice.to_vec() +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/mkl.rs b/v2/dependency-patches/candle-core-0.9.2/src/mkl.rs new file mode 100644 index 0000000000..359add741c --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/mkl.rs @@ -0,0 +1,418 @@ +#![allow(dead_code)] +use libc::{c_char, c_double, c_float, c_int}; + +mod ffi { + use super::*; + extern "C" { + pub fn vsTanh(n: c_int, a: *const c_float, y: *mut c_float); + pub fn vdTanh(n: c_int, a: *const c_double, y: *mut c_double); + pub fn vsExp(n: c_int, a: *const c_float, y: *mut c_float); + pub fn vdExp(n: c_int, a: *const c_double, y: *mut c_double); + pub fn vsLn(n: c_int, a: *const c_float, y: *mut c_float); + pub fn vdLn(n: c_int, a: *const c_double, y: *mut c_double); + pub fn vsSin(n: c_int, a: *const c_float, y: *mut c_float); + pub fn vdSin(n: c_int, a: *const c_double, y: *mut c_double); + pub fn vsCos(n: c_int, a: *const c_float, y: *mut c_float); + pub fn vdCos(n: c_int, a: *const c_double, y: *mut c_double); + pub fn vsSqrt(n: c_int, a: *const c_float, y: *mut c_float); + pub fn vdSqrt(n: c_int, a: *const c_double, y: *mut c_double); + + pub fn vsAdd(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); + pub fn vdAdd(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); + pub fn vsSub(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); + pub fn vdSub(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); + pub fn vsMul(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); + pub fn vdMul(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); + pub fn vsDiv(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); + pub fn vdDiv(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); + pub fn vsFmax(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); + pub fn vdFmax(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); + pub fn vsFmin(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); + pub fn vdFmin(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); + + pub fn sgemm_( + transa: *const c_char, + transb: *const c_char, + m: *const c_int, + n: *const c_int, + k: *const c_int, + alpha: *const c_float, + a: *const c_float, + lda: *const c_int, + b: *const c_float, + ldb: *const c_int, + beta: *const c_float, + c: *mut c_float, + ldc: *const c_int, + ); + pub fn dgemm_( + transa: *const c_char, + transb: *const c_char, + m: *const c_int, + n: *const c_int, + k: *const c_int, + alpha: *const c_double, + a: *const c_double, + lda: *const c_int, + b: *const c_double, + ldb: *const c_int, + beta: *const c_double, + c: *mut c_double, + ldc: *const c_int, + ); + pub fn hgemm_( + transa: *const c_char, + transb: *const c_char, + m: *const c_int, + n: *const c_int, + k: *const c_int, + alpha: *const half::f16, + a: *const half::f16, + lda: *const c_int, + b: *const half::f16, + ldb: *const c_int, + beta: *const half::f16, + c: *mut half::f16, + ldc: *const c_int, + ); + } +} + +#[allow(clippy::too_many_arguments)] +#[inline] +pub unsafe fn sgemm( + transa: u8, + transb: u8, + m: i32, + n: i32, + k: i32, + alpha: f32, + a: &[f32], + lda: i32, + b: &[f32], + ldb: i32, + beta: f32, + c: &mut [f32], + ldc: i32, +) { + ffi::sgemm_( + &(transa as c_char), + &(transb as c_char), + &m, + &n, + &k, + &alpha, + a.as_ptr(), + &lda, + b.as_ptr(), + &ldb, + &beta, + c.as_mut_ptr(), + &ldc, + ) +} + +#[allow(clippy::too_many_arguments)] +#[inline] +pub unsafe fn dgemm( + transa: u8, + transb: u8, + m: i32, + n: i32, + k: i32, + alpha: f64, + a: &[f64], + lda: i32, + b: &[f64], + ldb: i32, + beta: f64, + c: &mut [f64], + ldc: i32, +) { + ffi::dgemm_( + &(transa as c_char), + &(transb as c_char), + &m, + &n, + &k, + &alpha, + a.as_ptr(), + &lda, + b.as_ptr(), + &ldb, + &beta, + c.as_mut_ptr(), + &ldc, + ) +} + +#[allow(clippy::too_many_arguments)] +#[inline] +pub unsafe fn hgemm( + transa: u8, + transb: u8, + m: i32, + n: i32, + k: i32, + alpha: half::f16, + a: &[half::f16], + lda: i32, + b: &[half::f16], + ldb: i32, + beta: half::f16, + c: &mut [half::f16], + ldc: i32, +) { + ffi::hgemm_( + &(transa as c_char), + &(transb as c_char), + &m, + &n, + &k, + &alpha, + a.as_ptr(), + &lda, + b.as_ptr(), + &ldb, + &beta, + c.as_mut_ptr(), + &ldc, + ) +} + +#[inline] +pub fn vs_exp(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vsExp(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vd_exp(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vdExp(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vs_ln(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vsLn(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vd_ln(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vdLn(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vs_sin(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vsSin(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vd_sin(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vdSin(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vs_cos(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vsCos(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vd_cos(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vdCos(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vs_sqrt(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vsSqrt(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vd_sqrt(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vdSqrt(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vs_sqr(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vsMul(a_len as i32, a.as_ptr(), a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vd_sqr(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vdMul(a_len as i32, a.as_ptr(), a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vs_tanh(a: &[f32], y: &mut [f32]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vsTanh(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vd_tanh(a: &[f64], y: &mut [f64]) { + let a_len = a.len(); + let y_len = y.len(); + if a_len != y_len { + panic!("a and y have different lengths {a_len} <> {y_len}") + } + unsafe { ffi::vdTanh(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } +} + +// The vector functions from mkl can be performed in place by using the same array for input and +// output. +// https://www.intel.com/content/www/us/en/docs/onemkl/developer-reference-c/2023-2/vector-mathematical-functions.html +#[inline] +pub fn vs_tanh_inplace(y: &mut [f32]) { + unsafe { ffi::vsTanh(y.len() as i32, y.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vd_tanh_inplace(y: &mut [f64]) { + unsafe { ffi::vdTanh(y.len() as i32, y.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vs_exp_inplace(y: &mut [f32]) { + unsafe { ffi::vsExp(y.len() as i32, y.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vd_exp_inplace(y: &mut [f64]) { + unsafe { ffi::vdExp(y.len() as i32, y.as_ptr(), y.as_mut_ptr()) } +} + +#[inline] +pub fn vs_gelu(vs: &[f32], ys: &mut [f32]) { + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = (2.0f32 / std::f32::consts::PI).sqrt() * v * (1.0 + 0.044715 * v * v) + } + vs_tanh_inplace(ys); + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = 0.5 * v * (1.0 + *y) + } +} + +#[inline] +pub fn vd_gelu(vs: &[f64], ys: &mut [f64]) { + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = (2.0f64 / std::f64::consts::PI).sqrt() * v * (1.0 + 0.044715 * v * v) + } + vd_tanh_inplace(ys); + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = 0.5 * v * (1.0 + *y) + } +} + +#[inline] +pub fn vs_silu(vs: &[f32], ys: &mut [f32]) { + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = -v + } + vs_exp_inplace(ys); + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = v / (1.0 + *y) + } +} + +#[inline] +pub fn vd_silu(vs: &[f64], ys: &mut [f64]) { + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = -v + } + vd_exp_inplace(ys); + for (&v, y) in vs.iter().zip(ys.iter_mut()) { + *y = v / (1.0 + *y) + } +} + +macro_rules! binary_op { + ($fn_name:ident, $ty:ty, $mkl_name:ident) => { + #[inline] + pub fn $fn_name(a: &[$ty], b: &[$ty], y: &mut [$ty]) { + let a_len = a.len(); + let b_len = b.len(); + let y_len = y.len(); + if a_len != y_len || b_len != y_len { + panic!( + "{} a,b,y len mismatch {a_len} {b_len} {y_len}", + stringify!($fn_name) + ); + } + unsafe { ffi::$mkl_name(a_len as i32, a.as_ptr(), b.as_ptr(), y.as_mut_ptr()) } + } + }; +} +binary_op!(vs_add, f32, vsAdd); +binary_op!(vd_add, f64, vdAdd); +binary_op!(vs_sub, f32, vsSub); +binary_op!(vd_sub, f64, vdSub); +binary_op!(vs_mul, f32, vsMul); +binary_op!(vd_mul, f64, vdMul); +binary_op!(vs_div, f32, vsDiv); +binary_op!(vd_div, f64, vdDiv); +binary_op!(vs_max, f32, vsFmax); +binary_op!(vd_max, f64, vdFmax); +binary_op!(vs_min, f32, vsFmin); +binary_op!(vd_min, f64, vdFmin); diff --git a/v2/dependency-patches/candle-core-0.9.2/src/npy.rs b/v2/dependency-patches/candle-core-0.9.2/src/npy.rs new file mode 100644 index 0000000000..6035f32734 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/npy.rs @@ -0,0 +1,477 @@ +//! Numpy support for tensors. +//! +//! The spec for the npy format can be found in +//! [npy-format](https://docs.scipy.org/doc/numpy-1.14.2/neps/npy-format.html). +//! The functions from this module can be used to read tensors from npy/npz files +//! or write tensors to these files. A npy file contains a single tensor (unnamed) +//! whereas a npz file can contain multiple named tensors. npz files are also compressed. +//! +//! These two formats are easy to use in Python using the numpy library. +//! +//! ```python +//! import numpy as np +//! x = np.arange(10) +//! +//! # Write a npy file. +//! np.save("test.npy", x) +//! +//! # Read a value from the npy file. +//! x = np.load("test.npy") +//! +//! # Write multiple values to a npz file. +//! values = { "x": x, "x_plus_one": x + 1 } +//! np.savez("test.npz", **values) +//! +//! # Load multiple values from a npz file. +//! values = np.loadz("test.npz") +//! ``` +use crate::{DType, Device, Error, Result, Shape, Tensor}; +use byteorder::{LittleEndian, ReadBytesExt}; +use half::{bf16, f16, slice::HalfFloatSliceExt}; +use std::collections::HashMap; +use std::fs::File; +use std::io::{BufReader, Read, Write}; +use std::path::Path; + +const NPY_MAGIC_STRING: &[u8] = b"\x93NUMPY"; +const NPY_SUFFIX: &str = ".npy"; + +fn read_header(reader: &mut R) -> Result { + let mut magic_string = vec![0u8; NPY_MAGIC_STRING.len()]; + reader.read_exact(&mut magic_string)?; + if magic_string != NPY_MAGIC_STRING { + return Err(Error::Npy("magic string mismatch".to_string())); + } + let mut version = [0u8; 2]; + reader.read_exact(&mut version)?; + let header_len_len = match version[0] { + 1 => 2, + 2 => 4, + otherwise => return Err(Error::Npy(format!("unsupported version {otherwise}"))), + }; + let mut header_len = vec![0u8; header_len_len]; + reader.read_exact(&mut header_len)?; + let header_len = header_len + .iter() + .rev() + .fold(0_usize, |acc, &v| 256 * acc + v as usize); + let mut header = vec![0u8; header_len]; + reader.read_exact(&mut header)?; + Ok(String::from_utf8_lossy(&header).to_string()) +} + +#[derive(Debug, PartialEq)] +struct Header { + descr: DType, + fortran_order: bool, + shape: Vec, +} + +impl Header { + fn shape(&self) -> Shape { + Shape::from(self.shape.as_slice()) + } + + fn to_string(&self) -> Result { + let fortran_order = if self.fortran_order { "True" } else { "False" }; + let mut shape = self + .shape + .iter() + .map(|x| x.to_string()) + .collect::>() + .join(","); + let descr = match self.descr { + DType::BF16 => Err(Error::Npy("bf16 is not supported".into()))?, + DType::F16 => "f2", + DType::F32 => "f4", + DType::F64 => "f8", + DType::I16 => "i2", + DType::I32 => "i4", + DType::I64 => "i8", + DType::U32 => "u4", + DType::U8 => "u1", + DType::F8E4M3 => Err(Error::Npy("f8e4m3 is not supported".into()))?, + DType::F6E2M3 => Err(Error::Npy("f6e2m3 is not supported".into()))?, + DType::F6E3M2 => Err(Error::Npy("f6e3m2 is not supported".into()))?, + DType::F4 => Err(Error::Npy("f4 is not supported".into()))?, + DType::F8E8M0 => Err(Error::Npy("f8e8m0 is not supported".into()))?, + }; + if !shape.is_empty() { + shape.push(',') + } + Ok(format!( + "{{'descr': '<{descr}', 'fortran_order': {fortran_order}, 'shape': ({shape}), }}" + )) + } + + // Hacky parser for the npy header, a typical example would be: + // {'descr': ' Result
{ + let header = + header.trim_matches(|c: char| c == '{' || c == '}' || c == ',' || c.is_whitespace()); + + let mut parts: Vec = vec![]; + let mut start_index = 0usize; + let mut cnt_parenthesis = 0i64; + for (index, c) in header.char_indices() { + match c { + '(' => cnt_parenthesis += 1, + ')' => cnt_parenthesis -= 1, + ',' => { + if cnt_parenthesis == 0 { + parts.push(header[start_index..index].to_owned()); + start_index = index + 1; + } + } + _ => {} + } + } + parts.push(header[start_index..].to_owned()); + let mut part_map: HashMap = HashMap::new(); + for part in parts.iter() { + let part = part.trim(); + if !part.is_empty() { + match part.split(':').collect::>().as_slice() { + [key, value] => { + let key = key.trim_matches(|c: char| c == '\'' || c.is_whitespace()); + let value = value.trim_matches(|c: char| c == '\'' || c.is_whitespace()); + let _ = part_map.insert(key.to_owned(), value.to_owned()); + } + _ => return Err(Error::Npy(format!("unable to parse header {header}"))), + } + } + } + let fortran_order = match part_map.get("fortran_order") { + None => false, + Some(fortran_order) => match fortran_order.as_ref() { + "False" => false, + "True" => true, + _ => return Err(Error::Npy(format!("unknown fortran_order {fortran_order}"))), + }, + }; + let descr = match part_map.get("descr") { + None => return Err(Error::Npy("no descr in header".to_string())), + Some(descr) => { + if descr.is_empty() { + return Err(Error::Npy("empty descr".to_string())); + } + if descr.starts_with('>') { + return Err(Error::Npy(format!("little-endian descr {descr}"))); + } + // the only supported types in tensor are: + // float64, float32, float16, + // complex64, complex128, + // int64, int32, int16, int8, + // uint8, and bool. + match descr.trim_matches(|c: char| c == '=' || c == '<' || c == '|') { + "e" | "f2" => DType::F16, + "f" | "f4" => DType::F32, + "d" | "f8" => DType::F64, + "i" | "i4" => DType::I32, + "q" | "i8" => DType::I64, + "h" | "i2" => DType::I16, + // "b" | "i1" => DType::S8, + "B" | "u1" => DType::U8, + "I" | "u4" => DType::U32, + "?" | "b1" => DType::U8, + // "F" | "F4" => DType::C64, + // "D" | "F8" => DType::C128, + descr => return Err(Error::Npy(format!("unrecognized descr {descr}"))), + } + } + }; + let shape = match part_map.get("shape") { + None => return Err(Error::Npy("no shape in header".to_string())), + Some(shape) => { + let shape = shape.trim_matches(|c: char| c == '(' || c == ')' || c == ','); + if shape.is_empty() { + vec![] + } else { + shape + .split(',') + .map(|v| v.trim().parse::()) + .collect::, _>>()? + } + } + }; + Ok(Header { + descr, + fortran_order, + shape, + }) + } +} + +impl Tensor { + // TODO: Add the possibility to read directly to a device? + pub(crate) fn from_reader( + shape: Shape, + dtype: DType, + reader: &mut R, + ) -> Result { + let elem_count = shape.elem_count(); + match dtype { + DType::BF16 => { + let mut data_t = vec![bf16::ZERO; elem_count]; + reader.read_u16_into::(data_t.reinterpret_cast_mut())?; + Tensor::from_vec(data_t, shape, &Device::Cpu) + } + DType::F16 => { + let mut data_t = vec![f16::ZERO; elem_count]; + reader.read_u16_into::(data_t.reinterpret_cast_mut())?; + Tensor::from_vec(data_t, shape, &Device::Cpu) + } + DType::F32 => { + let mut data_t = vec![0f32; elem_count]; + reader.read_f32_into::(&mut data_t)?; + Tensor::from_vec(data_t, shape, &Device::Cpu) + } + DType::F64 => { + let mut data_t = vec![0f64; elem_count]; + reader.read_f64_into::(&mut data_t)?; + Tensor::from_vec(data_t, shape, &Device::Cpu) + } + DType::U8 => { + let mut data_t = vec![0u8; elem_count]; + reader.read_exact(&mut data_t)?; + Tensor::from_vec(data_t, shape, &Device::Cpu) + } + DType::U32 => { + let mut data_t = vec![0u32; elem_count]; + reader.read_u32_into::(&mut data_t)?; + Tensor::from_vec(data_t, shape, &Device::Cpu) + } + DType::I16 => { + let mut data_t = vec![0i16; elem_count]; + reader.read_i16_into::(&mut data_t)?; + Tensor::from_vec(data_t, shape, &Device::Cpu) + } + DType::I32 => { + let mut data_t = vec![0i32; elem_count]; + reader.read_i32_into::(&mut data_t)?; + Tensor::from_vec(data_t, shape, &Device::Cpu) + } + DType::I64 => { + let mut data_t = vec![0i64; elem_count]; + reader.read_i64_into::(&mut data_t)?; + Tensor::from_vec(data_t, shape, &Device::Cpu) + } + DType::F8E4M3 => { + let mut data_t = vec![0u8; elem_count]; + reader.read_exact(&mut data_t)?; + let data_f8: Vec = + data_t.into_iter().map(float8::F8E4M3::from_bits).collect(); + Tensor::from_vec(data_f8, shape, &Device::Cpu) + } + DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { + Err(Error::UnsupportedDTypeForOp(dtype, "from_reader").bt()) + } + } + } + + /// Reads a npy file and return the stored multi-dimensional array as a tensor. + pub fn read_npy>(path: T) -> Result { + let mut reader = File::open(path.as_ref())?; + let header = read_header(&mut reader)?; + let header = Header::parse(&header)?; + if header.fortran_order { + return Err(Error::Npy("fortran order not supported".to_string())); + } + Self::from_reader(header.shape(), header.descr, &mut reader) + } + + /// Reads a npz file and returns the stored multi-dimensional arrays together with their names. + pub fn read_npz>(path: T) -> Result> { + let zip_reader = BufReader::new(File::open(path.as_ref())?); + let mut zip = zip::ZipArchive::new(zip_reader)?; + let mut result = vec![]; + for i in 0..zip.len() { + let mut reader = zip.by_index(i)?; + let name = { + let name = reader.name()?; + name.strip_suffix(NPY_SUFFIX).unwrap_or(&name).to_owned() + }; + let header = read_header(&mut reader)?; + let header = Header::parse(&header)?; + if header.fortran_order { + return Err(Error::Npy("fortran order not supported".to_string())); + } + let s = Self::from_reader(header.shape(), header.descr, &mut reader)?; + result.push((name, s)) + } + Ok(result) + } + + /// Reads a npz file and returns the stored multi-dimensional arrays for some specified names. + pub fn read_npz_by_name>(path: T, names: &[&str]) -> Result> { + let zip_reader = BufReader::new(File::open(path.as_ref())?); + let mut zip = zip::ZipArchive::new(zip_reader)?; + let mut result = vec![]; + for name in names.iter() { + let mut reader = match zip.by_name(&format!("{name}{NPY_SUFFIX}")) { + Ok(reader) => reader, + Err(_) => Err(Error::Npy(format!( + "no array for {name} in {:?}", + path.as_ref() + )))?, + }; + let header = read_header(&mut reader)?; + let header = Header::parse(&header)?; + if header.fortran_order { + return Err(Error::Npy("fortran order not supported".to_string())); + } + let s = Self::from_reader(header.shape(), header.descr, &mut reader)?; + result.push(s) + } + Ok(result) + } + + fn write(&self, f: &mut T) -> Result<()> { + f.write_all(NPY_MAGIC_STRING)?; + f.write_all(&[1u8, 0u8])?; + let header = Header { + descr: self.dtype(), + fortran_order: false, + shape: self.dims().to_vec(), + }; + let mut header = header.to_string()?; + let pad = 16 - (NPY_MAGIC_STRING.len() + 5 + header.len()) % 16; + for _ in 0..pad % 16 { + header.push(' ') + } + header.push('\n'); + f.write_all(&[(header.len() % 256) as u8, (header.len() / 256) as u8])?; + f.write_all(header.as_bytes())?; + self.write_bytes(f) + } + + /// Writes a multi-dimensional array in the npy format. + pub fn write_npy>(&self, path: T) -> Result<()> { + let mut f = File::create(path.as_ref())?; + self.write(&mut f) + } + + /// Writes multiple multi-dimensional arrays using the npz format. + pub fn write_npz, T: AsRef, P: AsRef>( + ts: &[(S, T)], + path: P, + ) -> Result<()> { + let mut zip = zip::ZipWriter::new(File::create(path.as_ref())?); + let options: zip::write::FileOptions<()> = + zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + + for (name, tensor) in ts.iter() { + zip.start_file(format!("{}.npy", name.as_ref()), options)?; + tensor.as_ref().write(&mut zip)? + } + Ok(()) + } +} + +/// Lazy tensor loader. +pub struct NpzTensors { + index_per_name: HashMap, + path: std::path::PathBuf, + // We do not store a zip reader as it needs mutable access to extract data. Instead we + // re-create a zip reader for each tensor. +} + +impl NpzTensors { + pub fn new>(path: T) -> Result { + let path = path.as_ref().to_owned(); + let zip_reader = BufReader::new(File::open(&path)?); + let mut zip = zip::ZipArchive::new(zip_reader)?; + let mut index_per_name = HashMap::new(); + for i in 0..zip.len() { + let file = zip.by_index(i)?; + let name = { + let name = file.name()?; + name.strip_suffix(NPY_SUFFIX).unwrap_or(&name).to_owned() + }; + index_per_name.insert(name, i); + } + Ok(Self { + index_per_name, + path, + }) + } + + pub fn names(&self) -> Vec<&String> { + self.index_per_name.keys().collect() + } + + /// This only returns the shape and dtype for a named tensor. Compared to `get`, this avoids + /// reading the whole tensor data. + pub fn get_shape_and_dtype(&self, name: &str) -> Result<(Shape, DType)> { + let index = match self.index_per_name.get(name) { + None => crate::bail!("cannot find tensor {name}"), + Some(index) => *index, + }; + let zip_reader = BufReader::new(File::open(&self.path)?); + let mut zip = zip::ZipArchive::new(zip_reader)?; + let mut reader = zip.by_index(index)?; + let header = read_header(&mut reader)?; + let header = Header::parse(&header)?; + Ok((header.shape(), header.descr)) + } + + pub fn get(&self, name: &str) -> Result> { + let index = match self.index_per_name.get(name) { + None => return Ok(None), + Some(index) => *index, + }; + // We hope that the file has not changed since first reading it. + let zip_reader = BufReader::new(File::open(&self.path)?); + let mut zip = zip::ZipArchive::new(zip_reader)?; + let mut reader = zip.by_index(index)?; + let header = read_header(&mut reader)?; + let header = Header::parse(&header)?; + if header.fortran_order { + return Err(Error::Npy("fortran order not supported".to_string())); + } + let tensor = Tensor::from_reader(header.shape(), header.descr, &mut reader)?; + Ok(Some(tensor)) + } +} + +#[cfg(test)] +mod tests { + use super::Header; + + #[test] + fn parse() { + let h = "{'descr': ' &'static str { + match self { + Self::ArgMax => "argmax", + Self::ArgMin => "argmin", + Self::Min => "min", + Self::Max => "max", + Self::Sum => "sum", + } + } +} + +// These ops return the same type as their input type. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BinaryOp { + Add, + Mul, + Sub, + Div, + Maximum, + Minimum, +} + +// Unary ops with no argument +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum UnaryOp { + Exp, + Log, + Sin, + Cos, + Abs, + Neg, + Recip, + Sqr, + Sqrt, + Gelu, + GeluErf, + Erf, + Relu, + Silu, + Tanh, + Floor, + Ceil, + Round, + Sign, +} + +#[derive(Clone)] +pub enum Op { + Binary(Tensor, Tensor, BinaryOp), + Unary(Tensor, UnaryOp), + Cmp(Tensor, CmpOp), + // The third argument is the reduced shape with `keepdim=true`. + Reduce(Tensor, ReduceOp, Vec), + Matmul(Tensor, Tensor), + Gather(Tensor, Tensor, usize), + Scatter(Tensor, Tensor, Tensor, usize), + ScatterAdd(Tensor, Tensor, Tensor, usize), + IndexSelect(Tensor, Tensor, usize), + IndexAdd(Tensor, Tensor, Tensor, usize), + WhereCond(Tensor, Tensor, Tensor), + + #[allow(dead_code)] + Conv1D { + arg: Tensor, + kernel: Tensor, + padding: usize, + stride: usize, + dilation: usize, + }, + + #[allow(dead_code)] + ConvTranspose1D { + arg: Tensor, + kernel: Tensor, + padding: usize, + output_padding: usize, + stride: usize, + dilation: usize, + }, + + #[allow(dead_code)] + Conv2D { + arg: Tensor, + kernel: Tensor, + padding: usize, + stride: usize, + dilation: usize, + }, + + #[allow(dead_code)] + ConvTranspose2D { + arg: Tensor, + kernel: Tensor, + padding: usize, + output_padding: usize, + stride: usize, + dilation: usize, + }, + + AvgPool2D { + arg: Tensor, + kernel_size: (usize, usize), + stride: (usize, usize), + }, + + MaxPool2D { + arg: Tensor, + kernel_size: (usize, usize), + stride: (usize, usize), + }, + + UpsampleNearest1D { + arg: Tensor, + target_size: usize, + }, + UpsampleNearest2D { + arg: Tensor, + target_h: usize, + target_w: usize, + }, + UpsampleBilinear2D { + arg: Tensor, + target_h: usize, + target_w: usize, + align_corners: bool, + }, + + Cat(Vec, usize), + + #[allow(dead_code)] // add is currently unused. + Affine { + arg: Tensor, + mul: f64, + add: f64, + }, + ToDType(Tensor), + Copy(Tensor), + Broadcast(Tensor), + Narrow(Tensor, usize, usize, usize), + SliceScatter0(Tensor, Tensor, usize), + Reshape(Tensor), + ToDevice(Tensor), + Transpose(Tensor, usize, usize), + Permute(Tensor, Vec), + Elu(Tensor, f64), + Powf(Tensor, f64), + CustomOp1( + Tensor, + std::sync::Arc>, + ), + CustomOp2( + Tensor, + Tensor, + std::sync::Arc>, + ), + CustomOp3( + Tensor, + Tensor, + Tensor, + std::sync::Arc>, + ), +} + +pub trait UnaryOpT { + const NAME: &'static str; + const KERNEL: &'static str; + const V: Self; + fn bf16(v1: bf16) -> bf16; + fn f16(v1: f16) -> f16; + fn f32(v1: f32) -> f32; + fn f64(v1: f64) -> f64; + fn u8(v1: u8) -> u8; + fn u32(v1: u32) -> u32; + fn i16(v1: i16) -> i16; + fn i32(v1: i32) -> i32; + fn i64(v1: i64) -> i64; + fn f8e4m3(v1: f8e4m3) -> f8e4m3; + + // There is no very good way to represent optional function in traits so we go for an explicit + // boolean flag to mark the function as existing. + const BF16_VEC: bool = false; + fn bf16_vec(_xs: &[bf16], _ys: &mut [bf16]) {} + const F16_VEC: bool = false; + fn f16_vec(_xs: &[f16], _ys: &mut [f16]) {} + const F32_VEC: bool = false; + fn f32_vec(_xs: &[f32], _ys: &mut [f32]) {} + const F64_VEC: bool = false; + fn f64_vec(_xs: &[f64], _ys: &mut [f64]) {} +} + +pub trait BinaryOpT { + const NAME: &'static str; + const KERNEL: &'static str; + const V: Self; + fn bf16(v1: bf16, v2: bf16) -> bf16; + fn f16(v1: f16, v2: f16) -> f16; + fn f32(v1: f32, v2: f32) -> f32; + fn f64(v1: f64, v2: f64) -> f64; + fn u8(v1: u8, v2: u8) -> u8; + fn u32(v1: u32, v2: u32) -> u32; + fn i16(v1: i16, v2: i16) -> i16; + fn i32(v1: i32, v2: i32) -> i32; + fn i64(v1: i64, v2: i64) -> i64; + fn f8e4m3(v1: f8e4m3, v2: f8e4m3) -> f8e4m3; + + const BF16_VEC: bool = false; + fn bf16_vec(_xs1: &[bf16], _xs2: &[bf16], _ys: &mut [bf16]) {} + const F16_VEC: bool = false; + fn f16_vec(_xs1: &[f16], _xs2: &[f16], _ys: &mut [f16]) {} + const F32_VEC: bool = false; + fn f32_vec(_xs1: &[f32], _xs2: &[f32], _ys: &mut [f32]) {} + const F64_VEC: bool = false; + fn f64_vec(_xs1: &[f64], _xs2: &[f64], _ys: &mut [f64]) {} + const U8_VEC: bool = false; + fn u8_vec(_xs1: &[u8], _xs2: &[u8], _ys: &mut [u8]) {} + const U32_VEC: bool = false; + fn u32_vec(_xs1: &[u32], _xs2: &[u32], _ys: &mut [u32]) {} + const I64_VEC: bool = false; + fn i64_vec(_xs1: &[i64], _xs2: &[i64], _ys: &mut [i64]) {} +} + +pub struct Add; +pub struct Div; +pub struct Mul; +pub struct Sub; +pub struct Maximum; +pub struct Minimum; +pub struct Exp; +pub struct Log; +pub struct Sin; +pub struct Cos; +pub struct Abs; +pub struct Neg; +pub struct Recip; +pub struct Sqr; +pub struct Sqrt; +pub struct Gelu; +pub struct GeluErf; +pub struct Erf; +pub struct Relu; +pub struct Silu; +pub struct Tanh; +pub struct Floor; +pub struct Ceil; +pub struct Round; +pub struct Sign; + +macro_rules! bin_op { + ($op:ident, $name: literal, $e: expr, $f32_vec: ident, $f64_vec: ident) => { + impl BinaryOpT for $op { + const NAME: &'static str = $name; + const KERNEL: &'static str = concat!("b", $name); + const V: Self = $op; + #[inline(always)] + fn bf16(v1: bf16, v2: bf16) -> bf16 { + $e(v1, v2) + } + #[inline(always)] + fn f16(v1: f16, v2: f16) -> f16 { + $e(v1, v2) + } + #[inline(always)] + fn f32(v1: f32, v2: f32) -> f32 { + $e(v1, v2) + } + #[inline(always)] + fn f64(v1: f64, v2: f64) -> f64 { + $e(v1, v2) + } + #[inline(always)] + fn u8(v1: u8, v2: u8) -> u8 { + $e(v1, v2) + } + #[inline(always)] + fn u32(v1: u32, v2: u32) -> u32 { + $e(v1, v2) + } + #[inline(always)] + fn i16(v1: i16, v2: i16) -> i16 { + $e(v1, v2) + } + #[inline(always)] + fn i32(v1: i32, v2: i32) -> i32 { + $e(v1, v2) + } + #[inline(always)] + fn i64(v1: i64, v2: i64) -> i64 { + $e(v1, v2) + } + #[inline(always)] + fn f8e4m3(v1: f8e4m3, v2: f8e4m3) -> f8e4m3 { + $e(v1, v2) + } + + #[cfg(feature = "mkl")] + const F32_VEC: bool = true; + #[cfg(feature = "mkl")] + const F64_VEC: bool = true; + #[cfg(feature = "mkl")] + #[inline(always)] + fn f32_vec(xs1: &[f32], xs2: &[f32], ys: &mut [f32]) { + crate::mkl::$f32_vec(xs1, xs2, ys) + } + #[cfg(feature = "mkl")] + #[inline(always)] + fn f64_vec(xs1: &[f64], xs2: &[f64], ys: &mut [f64]) { + crate::mkl::$f64_vec(xs1, xs2, ys) + } + + #[cfg(feature = "accelerate")] + const F32_VEC: bool = true; + #[cfg(feature = "accelerate")] + const F64_VEC: bool = true; + #[cfg(feature = "accelerate")] + #[inline(always)] + fn f32_vec(xs1: &[f32], xs2: &[f32], ys: &mut [f32]) { + crate::accelerate::$f32_vec(xs1, xs2, ys) + } + #[cfg(feature = "accelerate")] + #[inline(always)] + fn f64_vec(xs1: &[f64], xs2: &[f64], ys: &mut [f64]) { + crate::accelerate::$f64_vec(xs1, xs2, ys) + } + } + }; +} + +bin_op!(Add, "add", |v1, v2| v1 + v2, vs_add, vd_add); +bin_op!(Sub, "sub", |v1, v2| v1 - v2, vs_sub, vd_sub); +bin_op!(Mul, "mul", |v1, v2| v1 * v2, vs_mul, vd_mul); +bin_op!(Div, "div", |v1, v2| v1 / v2, vs_div, vd_div); +bin_op!( + Minimum, + "minimum", + |v1, v2| if v1 > v2 { v2 } else { v1 }, + vs_min, + vd_min +); +bin_op!( + Maximum, + "maximum", + |v1, v2| if v1 < v2 { v2 } else { v1 }, + vs_max, + vd_max +); + +#[allow(clippy::redundant_closure_call)] +macro_rules! unary_op { + ($op: ident, $name: literal, $a: ident, $e: expr) => { + impl UnaryOpT for $op { + const NAME: &'static str = $name; + const KERNEL: &'static str = concat!("u", $name); + const V: Self = $op; + #[inline(always)] + fn bf16($a: bf16) -> bf16 { + $e + } + #[inline(always)] + fn f16($a: f16) -> f16 { + $e + } + #[inline(always)] + fn f32($a: f32) -> f32 { + $e + } + #[inline(always)] + fn f64($a: f64) -> f64 { + $e + } + #[inline(always)] + fn u8(_: u8) -> u8 { + todo!("no unary function for u8") + } + #[inline(always)] + fn u32(_: u32) -> u32 { + todo!("no unary function for u32") + } + #[inline(always)] + fn i16(_: i16) -> i16 { + todo!("no unary function for i16") + } + #[inline(always)] + fn i32(_: i32) -> i32 { + todo!("no unary function for i32") + } + #[inline(always)] + fn i64(_: i64) -> i64 { + todo!("no unary function for i64") + } + #[inline(always)] + fn f8e4m3($a: f8e4m3) -> f8e4m3 { + $e + } + } + }; + + ($op: ident, $name: literal, $a: ident, $e: expr, $f32_vec:ident, $f64_vec:ident) => { + impl UnaryOpT for $op { + const NAME: &'static str = $name; + const KERNEL: &'static str = concat!("u", $name); + const V: Self = $op; + #[inline(always)] + fn bf16($a: bf16) -> bf16 { + $e + } + #[inline(always)] + fn f16($a: f16) -> f16 { + $e + } + #[inline(always)] + fn f32($a: f32) -> f32 { + $e + } + #[inline(always)] + fn f64($a: f64) -> f64 { + $e + } + #[inline(always)] + fn u8(_: u8) -> u8 { + todo!("no unary function for u8") + } + #[inline(always)] + fn u32(_: u32) -> u32 { + todo!("no unary function for u32") + } + #[inline(always)] + fn i16(_: i16) -> i16 { + todo!("no unary function for i16") + } + #[inline(always)] + fn i32(_: i32) -> i32 { + todo!("no unary function for i32") + } + #[inline(always)] + fn i64(_: i64) -> i64 { + todo!("no unary function for i64") + } + #[inline(always)] + fn f8e4m3($a: f8e4m3) -> f8e4m3 { + $e + } + + #[cfg(feature = "mkl")] + const F32_VEC: bool = true; + #[cfg(feature = "mkl")] + const F64_VEC: bool = true; + #[cfg(feature = "mkl")] + #[inline(always)] + fn f32_vec(xs: &[f32], ys: &mut [f32]) { + crate::mkl::$f32_vec(xs, ys) + } + #[cfg(feature = "mkl")] + #[inline(always)] + fn f64_vec(xs: &[f64], ys: &mut [f64]) { + crate::mkl::$f64_vec(xs, ys) + } + + #[cfg(feature = "accelerate")] + const F32_VEC: bool = true; + #[cfg(feature = "accelerate")] + const F64_VEC: bool = true; + #[cfg(feature = "accelerate")] + #[inline(always)] + fn f32_vec(xs: &[f32], ys: &mut [f32]) { + crate::accelerate::$f32_vec(xs, ys) + } + #[cfg(feature = "accelerate")] + #[inline(always)] + fn f64_vec(xs: &[f64], ys: &mut [f64]) { + crate::accelerate::$f64_vec(xs, ys) + } + } + }; +} + +unary_op!(Exp, "exp", v, v.exp(), vs_exp, vd_exp); +unary_op!(Log, "log", v, v.ln(), vs_ln, vd_ln); +unary_op!(Sin, "sin", v, v.sin(), vs_sin, vd_sin); +unary_op!(Cos, "cos", v, v.cos(), vs_cos, vd_cos); +unary_op!(Tanh, "tanh", v, v.tanh(), vs_tanh, vd_tanh); +unary_op!(Neg, "neg", v, -v); +unary_op!(Recip, "recip", v, v.recip()); +unary_op!(Sqr, "sqr", v, v * v, vs_sqr, vd_sqr); +unary_op!(Sqrt, "sqrt", v, v.sqrt(), vs_sqrt, vd_sqrt); + +// Hardcode the value for sqrt(2/pi) +// https://github.com/huggingface/candle/issues/1982 +#[allow(clippy::excessive_precision)] +const SQRT_TWO_OVER_PI_F32: f32 = 0.79788456080286535587989211986876373; +#[allow(clippy::excessive_precision)] +const SQRT_TWO_OVER_PI_F64: f64 = 0.79788456080286535587989211986876373; + +/// Tanh based approximation of the `gelu` operation +/// GeluErf is the more precise one. +/// +impl UnaryOpT for Gelu { + const NAME: &'static str = "gelu"; + const V: Self = Gelu; + #[inline(always)] + fn bf16(v: bf16) -> bf16 { + bf16::from_f32_const(0.5) + * v + * (bf16::ONE + + bf16::tanh( + bf16::from_f32_const(SQRT_TWO_OVER_PI_F32) + * v + * (bf16::ONE + bf16::from_f32_const(0.044715) * v * v), + )) + } + #[inline(always)] + fn f16(v: f16) -> f16 { + f16::from_f32_const(0.5) + * v + * (f16::ONE + + f16::tanh( + f16::from_f32_const(SQRT_TWO_OVER_PI_F32) + * v + * (f16::ONE + f16::from_f32_const(0.044715) * v * v), + )) + } + #[inline(always)] + fn f32(v: f32) -> f32 { + 0.5 * v * (1.0 + f32::tanh(SQRT_TWO_OVER_PI_F32 * v * (1.0 + 0.044715 * v * v))) + } + #[inline(always)] + fn f64(v: f64) -> f64 { + 0.5 * v * (1.0 + f64::tanh(SQRT_TWO_OVER_PI_F64 * v * (1.0 + 0.044715 * v * v))) + } + #[inline(always)] + fn u8(_: u8) -> u8 { + 0 + } + #[inline(always)] + fn u32(_: u32) -> u32 { + 0 + } + #[inline(always)] + fn i16(_: i16) -> i16 { + 0 + } + #[inline(always)] + fn i32(_: i32) -> i32 { + 0 + } + #[inline(always)] + fn i64(_: i64) -> i64 { + 0 + } + #[inline(always)] + fn f8e4m3(v: f8e4m3) -> f8e4m3 { + f8e4m3::from_f32(0.5) + * v + * (f8e4m3::ONE + + f8e4m3::tanh( + f8e4m3::from_f32(SQRT_TWO_OVER_PI_F32) + * v + * (f8e4m3::ONE + f8e4m3::from_f32(0.044715) * v * v), + )) + } + const KERNEL: &'static str = "ugelu"; + + #[cfg(feature = "mkl")] + const F32_VEC: bool = true; + + #[cfg(feature = "mkl")] + #[inline(always)] + fn f32_vec(xs: &[f32], ys: &mut [f32]) { + crate::mkl::vs_gelu(xs, ys) + } + + #[cfg(feature = "mkl")] + const F64_VEC: bool = true; + + #[cfg(feature = "mkl")] + #[inline(always)] + fn f64_vec(xs: &[f64], ys: &mut [f64]) { + crate::mkl::vd_gelu(xs, ys) + } + + #[cfg(feature = "accelerate")] + const F32_VEC: bool = true; + + #[cfg(feature = "accelerate")] + #[inline(always)] + fn f32_vec(xs: &[f32], ys: &mut [f32]) { + crate::accelerate::vs_gelu(xs, ys) + } + + #[cfg(feature = "accelerate")] + const F64_VEC: bool = true; + + #[cfg(feature = "accelerate")] + #[inline(always)] + fn f64_vec(xs: &[f64], ys: &mut [f64]) { + crate::accelerate::vd_gelu(xs, ys) + } +} + +/// `erf` operation +/// +impl UnaryOpT for Erf { + const NAME: &'static str = "erf"; + const KERNEL: &'static str = "uerf"; + const V: Self = Erf; + #[inline(always)] + fn bf16(v: bf16) -> bf16 { + bf16::from_f64(Self::f64(v.to_f64())) + } + #[inline(always)] + fn f16(v: f16) -> f16 { + f16::from_f64(Self::f64(v.to_f64())) + } + #[inline(always)] + fn f32(v: f32) -> f32 { + crate::cpu::erf::erf_f32(v) + } + #[inline(always)] + fn f64(v: f64) -> f64 { + crate::cpu::erf::erf_f64(v) + } + #[inline(always)] + fn u8(_: u8) -> u8 { + 0 + } + #[inline(always)] + fn u32(_: u32) -> u32 { + 0 + } + #[inline(always)] + fn i16(_: i16) -> i16 { + 0 + } + #[inline(always)] + fn i32(_: i32) -> i32 { + 0 + } + #[inline(always)] + fn i64(_: i64) -> i64 { + 0 + } + #[inline(always)] + fn f8e4m3(v: f8e4m3) -> f8e4m3 { + f8e4m3::from_f64(Self::f64(v.to_f64())) + } +} + +/// Silu operation +impl UnaryOpT for Silu { + const NAME: &'static str = "silu"; + const V: Self = Silu; + #[inline(always)] + fn bf16(v: bf16) -> bf16 { + v / (bf16::ONE + (-v).exp()) + } + #[inline(always)] + fn f16(v: f16) -> f16 { + v / (f16::ONE + (-v).exp()) + } + #[inline(always)] + fn f32(v: f32) -> f32 { + v / (1.0 + (-v).exp()) + } + #[inline(always)] + fn f64(v: f64) -> f64 { + v / (1.0 + (-v).exp()) + } + #[inline(always)] + fn u8(_: u8) -> u8 { + 0 + } + #[inline(always)] + fn u32(_: u32) -> u32 { + 0 + } + #[inline(always)] + fn i16(_: i16) -> i16 { + 0 + } + #[inline(always)] + fn i32(_: i32) -> i32 { + 0 + } + #[inline(always)] + fn i64(_: i64) -> i64 { + 0 + } + #[inline(always)] + fn f8e4m3(v: f8e4m3) -> f8e4m3 { + v / (f8e4m3::ONE + (-v).exp()) + } + const KERNEL: &'static str = "usilu"; + + #[cfg(feature = "mkl")] + const F32_VEC: bool = true; + + #[cfg(feature = "mkl")] + #[inline(always)] + fn f32_vec(xs: &[f32], ys: &mut [f32]) { + crate::mkl::vs_silu(xs, ys) + } + + #[cfg(feature = "mkl")] + const F64_VEC: bool = true; + + #[cfg(feature = "mkl")] + #[inline(always)] + fn f64_vec(xs: &[f64], ys: &mut [f64]) { + crate::mkl::vd_silu(xs, ys) + } + + #[cfg(feature = "accelerate")] + const F32_VEC: bool = true; + + #[cfg(feature = "accelerate")] + #[inline(always)] + fn f32_vec(xs: &[f32], ys: &mut [f32]) { + crate::accelerate::vs_silu(xs, ys) + } + + #[cfg(feature = "accelerate")] + const F64_VEC: bool = true; + + #[cfg(feature = "accelerate")] + #[inline(always)] + fn f64_vec(xs: &[f64], ys: &mut [f64]) { + crate::accelerate::vd_silu(xs, ys) + } +} + +impl UnaryOpT for Abs { + const NAME: &'static str = "abs"; + const KERNEL: &'static str = "uabs"; + const V: Self = Abs; + #[inline(always)] + fn bf16(v: bf16) -> bf16 { + v.abs() + } + #[inline(always)] + fn f16(v: f16) -> f16 { + v.abs() + } + #[inline(always)] + fn f32(v: f32) -> f32 { + v.abs() + } + #[inline(always)] + fn f64(v: f64) -> f64 { + v.abs() + } + #[inline(always)] + fn u8(v: u8) -> u8 { + v + } + #[inline(always)] + fn u32(v: u32) -> u32 { + v + } + #[inline(always)] + fn i16(v: i16) -> i16 { + v.abs() + } + #[inline(always)] + fn i32(v: i32) -> i32 { + v.abs() + } + #[inline(always)] + fn i64(v: i64) -> i64 { + v.abs() + } + #[inline(always)] + fn f8e4m3(v: f8e4m3) -> f8e4m3 { + v.abs() + } +} + +impl UnaryOpT for Ceil { + const NAME: &'static str = "ceil"; + const KERNEL: &'static str = "uceil"; + const V: Self = Ceil; + #[inline(always)] + fn bf16(v: bf16) -> bf16 { + v.ceil() + } + #[inline(always)] + fn f16(v: f16) -> f16 { + v.ceil() + } + #[inline(always)] + fn f32(v: f32) -> f32 { + v.ceil() + } + #[inline(always)] + fn f64(v: f64) -> f64 { + v.ceil() + } + #[inline(always)] + fn u8(v: u8) -> u8 { + v + } + #[inline(always)] + fn u32(v: u32) -> u32 { + v + } + #[inline(always)] + fn i16(v: i16) -> i16 { + v + } + #[inline(always)] + fn i32(v: i32) -> i32 { + v + } + #[inline(always)] + fn i64(v: i64) -> i64 { + v + } + #[inline(always)] + fn f8e4m3(v: f8e4m3) -> f8e4m3 { + v.ceil() + } +} + +impl UnaryOpT for Floor { + const NAME: &'static str = "floor"; + const KERNEL: &'static str = "ufloor"; + const V: Self = Floor; + #[inline(always)] + fn bf16(v: bf16) -> bf16 { + v.floor() + } + #[inline(always)] + fn f16(v: f16) -> f16 { + v.floor() + } + #[inline(always)] + fn f32(v: f32) -> f32 { + v.floor() + } + #[inline(always)] + fn f64(v: f64) -> f64 { + v.floor() + } + #[inline(always)] + fn u8(v: u8) -> u8 { + v + } + #[inline(always)] + fn u32(v: u32) -> u32 { + v + } + #[inline(always)] + fn i16(v: i16) -> i16 { + v + } + #[inline(always)] + fn i32(v: i32) -> i32 { + v + } + #[inline(always)] + fn i64(v: i64) -> i64 { + v + } + #[inline(always)] + fn f8e4m3(v: f8e4m3) -> f8e4m3 { + v.floor() + } +} + +impl UnaryOpT for Round { + const NAME: &'static str = "round"; + const KERNEL: &'static str = "uround"; + const V: Self = Round; + #[inline(always)] + fn bf16(v: bf16) -> bf16 { + v.round() + } + #[inline(always)] + fn f16(v: f16) -> f16 { + v.round() + } + #[inline(always)] + fn f32(v: f32) -> f32 { + v.round() + } + #[inline(always)] + fn f64(v: f64) -> f64 { + v.round() + } + #[inline(always)] + fn u8(v: u8) -> u8 { + v + } + #[inline(always)] + fn u32(v: u32) -> u32 { + v + } + #[inline(always)] + fn i16(v: i16) -> i16 { + v + } + #[inline(always)] + fn i32(v: i32) -> i32 { + v + } + #[inline(always)] + fn i64(v: i64) -> i64 { + v + } + #[inline(always)] + fn f8e4m3(v: f8e4m3) -> f8e4m3 { + v.round() + } +} + +impl UnaryOpT for GeluErf { + const NAME: &'static str = "gelu_erf"; + const KERNEL: &'static str = "ugelu_erf"; + const V: Self = GeluErf; + #[inline(always)] + fn bf16(v: bf16) -> bf16 { + bf16::from_f64(Self::f64(v.to_f64())) + } + #[inline(always)] + fn f16(v: f16) -> f16 { + f16::from_f64(Self::f64(v.to_f64())) + } + #[inline(always)] + fn f32(v: f32) -> f32 { + (crate::cpu::erf::erf_f32(v * std::f32::consts::FRAC_1_SQRT_2) + 1.) * 0.5 * v + } + #[inline(always)] + fn f64(v: f64) -> f64 { + (crate::cpu::erf::erf_f64(v * std::f64::consts::FRAC_1_SQRT_2) + 1.) * 0.5 * v + } + #[inline(always)] + fn u8(_: u8) -> u8 { + 0 + } + #[inline(always)] + fn u32(_: u32) -> u32 { + 0 + } + #[inline(always)] + fn i16(_: i16) -> i16 { + 0 + } + #[inline(always)] + fn i32(_: i32) -> i32 { + 0 + } + #[inline(always)] + fn i64(_: i64) -> i64 { + 0 + } + #[inline(always)] + fn f8e4m3(v: f8e4m3) -> f8e4m3 { + f8e4m3::from_f32(Self::f32(v.to_f32())) + } +} + +impl UnaryOpT for Relu { + const NAME: &'static str = "relu"; + const KERNEL: &'static str = "urelu"; + const V: Self = Relu; + #[inline(always)] + fn bf16(v: bf16) -> bf16 { + v.max(bf16::ZERO) + } + #[inline(always)] + fn f16(v: f16) -> f16 { + v.max(f16::ZERO) + } + #[inline(always)] + fn f32(v: f32) -> f32 { + v.max(0f32) + } + #[inline(always)] + fn f64(v: f64) -> f64 { + v.max(0f64) + } + #[inline(always)] + fn u8(v: u8) -> u8 { + v + } + #[inline(always)] + fn u32(v: u32) -> u32 { + v + } + #[inline(always)] + fn i16(v: i16) -> i16 { + v.max(0) + } + #[inline(always)] + fn i32(v: i32) -> i32 { + v.max(0) + } + #[inline(always)] + fn i64(v: i64) -> i64 { + v.max(0) + } + #[inline(always)] + fn f8e4m3(v: f8e4m3) -> f8e4m3 { + v.max(f8e4m3::ZERO) + } +} + +/// `BackpropOp` is a wrapper around `Option`. The main goal is to ensure that dependencies are +/// properly checked when creating a new value +#[derive(Clone)] +pub struct BackpropOp(Option); + +impl BackpropOp { + pub fn none() -> Self { + BackpropOp(None) + } + + pub(crate) fn new1(arg: &Tensor, f: impl Fn(Tensor) -> Op) -> Self { + let op = if arg.track_op() { + Some(f(arg.clone())) + } else { + None + }; + Self(op) + } + + pub(crate) fn new2(arg1: &Tensor, arg2: &Tensor, f: impl Fn(Tensor, Tensor) -> Op) -> Self { + let op = if arg1.track_op() || arg2.track_op() { + Some(f(arg1.clone(), arg2.clone())) + } else { + None + }; + Self(op) + } + + pub(crate) fn new3( + arg1: &Tensor, + arg2: &Tensor, + arg3: &Tensor, + f: impl Fn(Tensor, Tensor, Tensor) -> Op, + ) -> Self { + let op = if arg1.track_op() || arg2.track_op() || arg3.track_op() { + Some(f(arg1.clone(), arg2.clone(), arg3.clone())) + } else { + None + }; + Self(op) + } + + pub(crate) fn new>(args: &[A], f: impl Fn(Vec) -> Op) -> Self { + let op = if args.iter().any(|arg| arg.as_ref().track_op()) { + let args: Vec = args.iter().map(|arg| arg.as_ref().clone()).collect(); + Some(f(args)) + } else { + None + }; + Self(op) + } + + pub(crate) fn is_none(&self) -> bool { + self.0.is_none() + } +} + +impl std::ops::Deref for BackpropOp { + type Target = Option; + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl UnaryOpT for Sign { + const NAME: &'static str = "sign"; + const KERNEL: &'static str = "usign"; + const V: Self = Sign; + #[inline(always)] + fn bf16(v: bf16) -> bf16 { + bf16::from((v > bf16::ZERO) as i8) - bf16::from((v < bf16::ZERO) as i8) + } + #[inline(always)] + fn f16(v: f16) -> f16 { + f16::from((v > f16::ZERO) as i8) - f16::from((v < f16::ZERO) as i8) + } + #[inline(always)] + fn f32(v: f32) -> f32 { + f32::from(v > 0.) - f32::from(v < 0.) + } + #[inline(always)] + fn f64(v: f64) -> f64 { + f64::from(v > 0.) - f64::from(v < 0.) + } + #[inline(always)] + fn u8(v: u8) -> u8 { + u8::min(1, v) + } + #[inline(always)] + fn u32(v: u32) -> u32 { + u32::min(1, v) + } + #[inline(always)] + fn i16(v: i16) -> i16 { + (v > 0) as i16 - (v < 0) as i16 + } + #[inline(always)] + fn i32(v: i32) -> i32 { + (v > 0) as i32 - (v < 0) as i32 + } + #[inline(always)] + fn i64(v: i64) -> i64 { + (v > 0) as i64 - (v < 0) as i64 + } + #[inline(always)] + fn f8e4m3(v: f8e4m3) -> f8e4m3 { + if v > f8e4m3::ZERO { + f8e4m3::ONE + } else if v < f8e4m3::ZERO { + -f8e4m3::ONE + } else { + f8e4m3::ZERO + } + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/pickle.rs b/v2/dependency-patches/candle-core-0.9.2/src/pickle.rs new file mode 100644 index 0000000000..8a41edfbc6 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/pickle.rs @@ -0,0 +1,841 @@ +//! Just enough pickle support to be able to read PyTorch checkpoints. +// This hardcodes objects that are required for tensor reading, we may want to make this a bit more +// composable/tensor agnostic at some point. +use crate::{Context, DType, Error as E, Layout, Result, Tensor}; +use byteorder::{LittleEndian, ReadBytesExt}; +use std::collections::HashMap; +use std::io::BufRead; + +const VERBOSE: bool = false; + +// https://docs.juliahub.com/Pickle/LAUNc/0.1.0/opcode/ +#[repr(u8)] +#[derive(Debug, Eq, PartialEq, Clone)] +pub enum OpCode { + // https://github.com/python/cpython/blob/ed25f097160b5cbb0c9a1f9a746d2f1bbc96515a/Lib/pickletools.py#L2123 + Proto = 0x80, + Global = b'c', + BinPut = b'q', + LongBinPut = b'r', + EmptyTuple = b')', + Reduce = b'R', + Mark = b'(', + BinUnicode = b'X', + BinInt = b'J', + Tuple = b't', + BinPersId = b'Q', + BinInt1 = b'K', + BinInt2 = b'M', + Tuple1 = 0x85, + Tuple2 = 0x86, + Tuple3 = 0x87, + NewTrue = 0x88, + NewFalse = 0x89, + None = b'N', + BinGet = b'h', + LongBinGet = b'j', + SetItem = b's', + SetItems = b'u', + EmptyDict = b'}', + Dict = b'd', + Build = b'b', + Stop = b'.', + NewObj = 0x81, + EmptyList = b']', + BinFloat = b'G', + Append = b'a', + Appends = b'e', + Long1 = 0x8a, +} + +// Avoid using FromPrimitive so as not to drag another dependency. +impl TryFrom for OpCode { + type Error = u8; + fn try_from(value: u8) -> std::result::Result { + match value { + 0x80 => Ok(Self::Proto), + b'c' => Ok(Self::Global), + b'q' => Ok(Self::BinPut), + b'r' => Ok(Self::LongBinPut), + b')' => Ok(Self::EmptyTuple), + b'R' => Ok(Self::Reduce), + b'(' => Ok(Self::Mark), + b'X' => Ok(Self::BinUnicode), + b'J' => Ok(Self::BinInt), + b't' => Ok(Self::Tuple), + b'Q' => Ok(Self::BinPersId), + b'K' => Ok(Self::BinInt1), + b'M' => Ok(Self::BinInt2), + b'N' => Ok(Self::None), + 0x85 => Ok(Self::Tuple1), + 0x86 => Ok(Self::Tuple2), + 0x87 => Ok(Self::Tuple3), + 0x88 => Ok(Self::NewTrue), + 0x89 => Ok(Self::NewFalse), + b'h' => Ok(Self::BinGet), + b'j' => Ok(Self::LongBinGet), + b's' => Ok(Self::SetItem), + b'u' => Ok(Self::SetItems), + b'}' => Ok(Self::EmptyDict), + b'd' => Ok(Self::EmptyDict), + b'b' => Ok(Self::Build), + b'.' => Ok(Self::Stop), + 0x81 => Ok(Self::NewObj), + b']' => Ok(Self::EmptyList), + b'G' => Ok(Self::BinFloat), + b'a' => Ok(Self::Append), + b'e' => Ok(Self::Appends), + 0x8a => Ok(Self::Long1), + value => Err(value), + } + } +} + +fn read_to_newline(r: &mut R) -> Result> { + let mut data: Vec = Vec::with_capacity(32); + r.read_until(b'\n', &mut data)?; + data.pop(); + if data.last() == Some(&b'\r') { + data.pop(); + } + Ok(data) +} + +#[derive(Debug, Clone, PartialEq)] +pub enum Object { + Class { + module_name: String, + class_name: String, + }, + Int(i32), + Long(i64), + Float(f64), + Unicode(String), + Bool(bool), + None, + Tuple(Vec), + List(Vec), + Mark, + Dict(Vec<(Object, Object)>), + Reduce { + callable: Box, + args: Box, + }, + Build { + callable: Box, + args: Box, + }, + PersistentLoad(Box), +} + +type OResult = std::result::Result; + +impl Object { + pub fn unicode(self) -> OResult { + match self { + Self::Unicode(t) => Ok(t), + _ => Err(self), + } + } + + pub fn reduce(self) -> OResult<(Self, Self)> { + match self { + Self::Reduce { callable, args } => Ok((*callable, *args)), + _ => Err(self), + } + } + + pub fn none(self) -> OResult<()> { + match self { + Self::None => Ok(()), + _ => Err(self), + } + } + + pub fn persistent_load(self) -> OResult { + match self { + Self::PersistentLoad(t) => Ok(*t), + _ => Err(self), + } + } + + pub fn bool(self) -> OResult { + match self { + Self::Bool(t) => Ok(t), + _ => Err(self), + } + } + + pub fn int(self) -> OResult { + match self { + Self::Int(t) => Ok(t), + _ => Err(self), + } + } + + pub fn int_or_long(self) -> OResult { + match self { + Self::Int(t) => Ok(t as i64), + Self::Long(t) => Ok(t), + _ => Err(self), + } + } + + pub fn tuple(self) -> OResult> { + match self { + Self::Tuple(t) => Ok(t), + _ => Err(self), + } + } + + pub fn dict(self) -> OResult> { + match self { + Self::Dict(t) => Ok(t), + _ => Err(self), + } + } + + pub fn class(self) -> OResult<(String, String)> { + match self { + Self::Class { + module_name, + class_name, + } => Ok((module_name, class_name)), + _ => Err(self), + } + } + + pub fn into_tensor_info( + self, + name: Self, + dir_name: &std::path::Path, + ) -> Result> { + let name = match name.unicode() { + Ok(name) => name, + Err(_) => return Ok(None), + }; + let (callable, args) = match self.reduce() { + Ok(callable_args) => callable_args, + _ => return Ok(None), + }; + let (callable, args) = match callable { + Object::Class { + module_name, + class_name, + } if module_name == "torch._tensor" && class_name == "_rebuild_from_type_v2" => { + let mut args = args.tuple()?; + let callable = args.remove(0); + let args = args.remove(1); + (callable, args) + } + Object::Class { + module_name, + class_name, + } if module_name == "torch._utils" && class_name == "_rebuild_parameter" => { + let mut args = args.tuple()?; + args.remove(0).reduce()? + } + _ => (callable, args), + }; + match callable { + Object::Class { + module_name, + class_name, + } if module_name == "torch._utils" && class_name == "_rebuild_tensor_v2" => {} + _ => return Ok(None), + }; + let (layout, dtype, file_path, storage_size) = rebuild_args(args)?; + Ok(Some(TensorInfo { + name, + dtype, + layout, + path: format!("{}/{}", dir_name.to_string_lossy(), file_path), + storage_size, + })) + } +} + +impl TryFrom for String { + type Error = Object; + fn try_from(value: Object) -> std::result::Result { + match value { + Object::Unicode(s) => Ok(s), + other => Err(other), + } + } +} + +impl TryFrom for usize { + type Error = Object; + fn try_from(value: Object) -> std::result::Result { + match value { + Object::Int(s) if s >= 0 => Ok(s as usize), + other => Err(other), + } + } +} + +impl> TryFrom for Vec { + type Error = Object; + fn try_from(value: Object) -> std::result::Result { + match value { + Object::Tuple(values) => { + // This does not return the appropriate value in the error case but instead return + // the object related to the first error. + values + .into_iter() + .map(|v| T::try_from(v)) + .collect::, Self::Error>>() + } + other => Err(other), + } + } +} + +#[derive(Debug)] +pub struct Stack { + stack: Vec, + memo: HashMap, +} + +impl Stack { + pub fn empty() -> Self { + Self { + stack: Vec::with_capacity(512), + memo: HashMap::new(), + } + } + + pub fn stack(&self) -> &[Object] { + self.stack.as_slice() + } + + pub fn read_loop(&mut self, r: &mut R) -> Result<()> { + loop { + if self.read(r)? { + break; + } + } + Ok(()) + } + + pub fn finalize(mut self) -> Result { + self.pop() + } + + fn push(&mut self, obj: Object) { + self.stack.push(obj) + } + + fn pop(&mut self) -> Result { + match self.stack.pop() { + None => crate::bail!("unexpected empty stack"), + Some(obj) => Ok(obj), + } + } + + // https://docs.juliahub.com/Pickle/LAUNc/0.1.0/opcode/#Pickle.OpCodes.BUILD + fn build(&mut self) -> Result<()> { + let args = self.pop()?; + let obj = self.pop()?; + let obj = match (obj, args) { + (Object::Dict(mut obj), Object::Dict(mut args)) => { + obj.append(&mut args); + Object::Dict(obj) + } + (obj, args) => Object::Build { + callable: Box::new(obj), + args: Box::new(args), + }, + }; + self.push(obj); + Ok(()) + } + + fn reduce(&mut self) -> Result<()> { + let args = self.pop()?; + let callable = self.pop()?; + #[allow(clippy::single_match)] + let reduced = match &callable { + Object::Class { + module_name, + class_name, + } => { + if module_name == "collections" + && (class_name == "OrderedDict" || class_name == "defaultdict") + { + // TODO: have a separate ordered dict and a separate default dict. + Some(Object::Dict(vec![])) + } else { + None + } + } + _ => None, + }; + let reduced = reduced.unwrap_or_else(|| Object::Reduce { + callable: Box::new(callable), + args: Box::new(args), + }); + self.push(reduced); + Ok(()) + } + + fn last(&mut self) -> Result<&mut Object> { + match self.stack.last_mut() { + None => crate::bail!("unexpected empty stack"), + Some(obj) => Ok(obj), + } + } + + fn memo_get(&self, id: u32) -> Result { + match self.memo.get(&id) { + None => crate::bail!("missing object in memo {id}"), + Some(obj) => { + // Maybe we should use refcounting rather than doing potential large clones here. + Ok(obj.clone()) + } + } + } + + fn memo_put(&mut self, id: u32) -> Result<()> { + let obj = self.last()?.clone(); + self.memo.insert(id, obj); + Ok(()) + } + + fn persistent_load(&self, id: Object) -> Result { + Ok(Object::PersistentLoad(Box::new(id))) + } + + fn new_obj(&self, class: Object, args: Object) -> Result { + Ok(Object::Reduce { + callable: Box::new(class), + args: Box::new(args), + }) + } + + fn pop_to_marker(&mut self) -> Result> { + let mut mark_idx = None; + for (idx, obj) in self.stack.iter().enumerate().rev() { + if obj == &Object::Mark { + mark_idx = Some(idx); + break; + } + } + match mark_idx { + Some(mark_idx) => { + let objs = self.stack.split_off(mark_idx + 1); + self.stack.pop(); + Ok(objs) + } + None => { + crate::bail!("marker object not found") + } + } + } + + pub fn read(&mut self, r: &mut R) -> Result { + let op_code = match OpCode::try_from(r.read_u8()?) { + Ok(op_code) => op_code, + Err(op_code) => { + crate::bail!("unknown op-code {op_code}") + } + }; + // println!("op: {op_code:?}"); + // println!("{:?}", self.stack); + match op_code { + OpCode::Proto => { + let version = r.read_u8()?; + if VERBOSE { + println!("proto {version}"); + } + } + OpCode::Global => { + let module_name = read_to_newline(r)?; + let class_name = read_to_newline(r)?; + let module_name = String::from_utf8_lossy(&module_name).to_string(); + let class_name = String::from_utf8_lossy(&class_name).to_string(); + self.push(Object::Class { + module_name, + class_name, + }) + } + OpCode::BinInt1 => { + let arg = r.read_u8()?; + self.push(Object::Int(arg as i32)) + } + OpCode::BinInt2 => { + let arg = r.read_u16::()?; + self.push(Object::Int(arg as i32)) + } + OpCode::BinInt => { + let arg = r.read_i32::()?; + self.push(Object::Int(arg)) + } + OpCode::BinFloat => { + // Somehow floats are encoded using BigEndian whereas int types use LittleEndian. + // https://github.com/python/cpython/blob/0c80da4c14d904a367968955544dd6ae58c8101c/Lib/pickletools.py#L855 + // https://github.com/pytorch/pytorch/blob/372d078f361e726bb4ac0884ac334b04c58179ef/torch/_weights_only_unpickler.py#L243 + let arg = r.read_f64::()?; + self.push(Object::Float(arg)) + } + OpCode::BinUnicode => { + let len = r.read_u32::()?; + let mut data = vec![0u8; len as usize]; + r.read_exact(&mut data)?; + let data = String::from_utf8(data).map_err(E::wrap)?; + self.push(Object::Unicode(data)) + } + OpCode::BinPersId => { + let id = self.pop()?; + let obj = self.persistent_load(id)?; + self.push(obj) + } + OpCode::Tuple => { + let objs = self.pop_to_marker()?; + self.push(Object::Tuple(objs)) + } + OpCode::Tuple1 => { + let obj = self.pop()?; + self.push(Object::Tuple(vec![obj])) + } + OpCode::Tuple2 => { + let obj2 = self.pop()?; + let obj1 = self.pop()?; + self.push(Object::Tuple(vec![obj1, obj2])) + } + OpCode::Tuple3 => { + let obj3 = self.pop()?; + let obj2 = self.pop()?; + let obj1 = self.pop()?; + self.push(Object::Tuple(vec![obj1, obj2, obj3])) + } + OpCode::NewTrue => self.push(Object::Bool(true)), + OpCode::NewFalse => self.push(Object::Bool(false)), + OpCode::Append => { + let value = self.pop()?; + let pylist = self.last()?; + if let Object::List(d) = pylist { + d.push(value) + } else { + crate::bail!("expected a list, got {pylist:?}") + } + } + OpCode::Appends => { + let objs = self.pop_to_marker()?; + let pylist = self.last()?; + if let Object::List(d) = pylist { + d.extend(objs) + } else { + crate::bail!("expected a list, got {pylist:?}") + } + } + OpCode::SetItem => { + let value = self.pop()?; + let key = self.pop()?; + let pydict = self.last()?; + if let Object::Dict(d) = pydict { + d.push((key, value)) + } else { + crate::bail!("expected a dict, got {pydict:?}") + } + } + OpCode::SetItems => { + let mut objs = self.pop_to_marker()?; + let pydict = self.last()?; + if let Object::Dict(d) = pydict { + if objs.len() % 2 != 0 { + crate::bail!("setitems: not an even number of objects") + } + while let Some(value) = objs.pop() { + let key = objs.pop().context("empty objs")?; + d.push((key, value)) + } + } else { + crate::bail!("expected a dict, got {pydict:?}") + } + } + OpCode::None => self.push(Object::None), + OpCode::Stop => { + return Ok(true); + } + OpCode::Build => self.build()?, + OpCode::EmptyDict => self.push(Object::Dict(vec![])), + OpCode::Dict => { + let mut objs = self.pop_to_marker()?; + let mut pydict = vec![]; + if objs.len() % 2 != 0 { + crate::bail!("setitems: not an even number of objects") + } + while let Some(value) = objs.pop() { + let key = objs.pop().context("empty objs")?; + pydict.push((key, value)) + } + self.push(Object::Dict(pydict)) + } + OpCode::Mark => self.push(Object::Mark), + OpCode::Reduce => self.reduce()?, + OpCode::EmptyTuple => self.push(Object::Tuple(vec![])), + OpCode::EmptyList => self.push(Object::List(vec![])), + OpCode::BinGet => { + let arg = r.read_u8()?; + let obj = self.memo_get(arg as u32)?; + self.push(obj) + } + OpCode::LongBinGet => { + let arg = r.read_u32::()?; + let obj = self.memo_get(arg)?; + self.push(obj) + } + OpCode::BinPut => { + let arg = r.read_u8()?; + self.memo_put(arg as u32)? + } + OpCode::LongBinPut => { + let arg = r.read_u32::()?; + self.memo_put(arg)? + } + OpCode::NewObj => { + let args = self.pop()?; + let class = self.pop()?; + let obj = self.new_obj(class, args)?; + self.push(obj) + } + OpCode::Long1 => { + let n_bytes = r.read_u8()?; + let mut v = 0; + // Decode the next n bytes in little endian + for i in 0..n_bytes { + v |= (r.read_u8()? as i64) << (i * 8); + } + self.push(Object::Long(v)) + } + } + Ok(false) + } +} + +impl From for E { + fn from(value: Object) -> Self { + E::Msg(format!("conversion error on {value:?}")) + } +} + +// https://github.com/pytorch/pytorch/blob/4eac43d046ded0f0a5a5fa8db03eb40f45bf656e/torch/_utils.py#L198 +// Arguments: storage, storage_offset, size, stride, requires_grad, backward_hooks +fn rebuild_args(args: Object) -> Result<(Layout, DType, String, usize)> { + let mut args = args.tuple()?; + let stride = Vec::::try_from(args.remove(3))?; + let size = Vec::::try_from(args.remove(2))?; + let offset = args.remove(1).int_or_long()? as usize; + let storage = args.remove(0).persistent_load()?; + let mut storage = storage.tuple()?; + let storage_size = storage.remove(4).int_or_long()? as usize; + let path = storage.remove(2).unicode()?; + let (_module_name, class_name) = storage.remove(1).class()?; + let dtype = match class_name.as_str() { + "FloatStorage" => DType::F32, + "DoubleStorage" => DType::F64, + "HalfStorage" => DType::F16, + "BFloat16Storage" => DType::BF16, + "ByteStorage" => DType::U8, + "LongStorage" => DType::I64, + other => { + crate::bail!("unsupported storage type {other}") + } + }; + let layout = Layout::new( + crate::Shape::from(size), + stride, + offset * dtype.size_in_bytes(), + ); + Ok((layout, dtype, path, storage_size)) +} + +#[derive(Debug, Clone)] +pub struct TensorInfo { + pub name: String, + pub dtype: DType, + pub layout: Layout, + pub path: String, + pub storage_size: usize, +} + +/// Read the tensor info from a .pth file. +/// +/// # Arguments +/// * `file` - The path to the .pth file. +/// * `verbose` - Whether to print debug information. +/// * `key` - Optional key to retrieve `state_dict` from the pth file. +pub fn read_pth_tensor_info>( + file: P, + verbose: bool, + key: Option<&str>, +) -> Result> { + let file = std::fs::File::open(file)?; + let zip_reader = std::io::BufReader::new(file); + let mut zip = zip::ZipArchive::new(zip_reader)?; + let zip_file_names = zip + .file_names() + .map(|name| name.map(|name| name.into_owned())) + .collect::>>()?; + + let mut tensor_infos = vec![]; + for file_name in zip_file_names.iter() { + if !file_name.ends_with("data.pkl") { + continue; + } + let dir_name = std::path::PathBuf::from(file_name.strip_suffix(".pkl").context("no .pkl")?); + let reader = zip.by_name(file_name)?; + let mut reader = std::io::BufReader::new(reader); + let mut stack = Stack::empty(); + stack.read_loop(&mut reader)?; + let obj = stack.finalize()?; + if VERBOSE || verbose { + println!("{obj:#?}"); + } + + let obj = match obj { + Object::Build { callable, args } => match *callable { + Object::Reduce { callable, args: _ } => match *callable { + Object::Class { + module_name, + class_name, + } if module_name == "__torch__" && class_name == "Module" => *args, + _ => continue, + }, + _ => continue, + }, + obj => obj, + }; + + // If key is provided, then we need to extract the state_dict from the object. + let obj = if let Some(key) = key { + if let Object::Dict(key_values) = obj { + key_values + .into_iter() + .find(|(k, _)| *k == Object::Unicode(key.to_owned())) + .map(|(_, v)| v) + .ok_or_else(|| E::Msg(format!("key {key} not found")))? + } else { + obj + } + } else { + obj + }; + + // If the object is a dict, then we can extract the tensor info from it. + // NOTE: We are assuming that the `obj` is state_dict by this stage. + if let Object::Dict(key_values) = obj { + for (name, value) in key_values.into_iter() { + match value.into_tensor_info(name, &dir_name) { + Ok(Some(tensor_info)) => tensor_infos.push(tensor_info), + Ok(None) => {} + Err(err) => eprintln!("skipping: {err:?}"), + } + } + } + } + Ok(tensor_infos) +} + +/// Lazy tensor loader. +pub struct PthTensors { + tensor_infos: HashMap, + path: std::path::PathBuf, + // We do not store a zip reader as it needs mutable access to extract data. Instead we + // re-create a zip reader for each tensor. +} + +impl PthTensors { + pub fn new>(path: P, key: Option<&str>) -> Result { + let tensor_infos = read_pth_tensor_info(path.as_ref(), false, key)?; + let tensor_infos = tensor_infos + .into_iter() + .map(|ti| (ti.name.to_string(), ti)) + .collect(); + let path = path.as_ref().to_owned(); + Ok(Self { tensor_infos, path }) + } + + pub fn tensor_infos(&self) -> &HashMap { + &self.tensor_infos + } + + pub fn get(&self, name: &str) -> Result> { + use std::io::Read; + let tensor_info = match self.tensor_infos.get(name) { + None => return Ok(None), + Some(tensor_info) => tensor_info, + }; + // We hope that the file has not changed since first reading it. + let zip_reader = std::io::BufReader::new(std::fs::File::open(&self.path)?); + let mut zip = zip::ZipArchive::new(zip_reader)?; + let mut reader = zip.by_name(&tensor_info.path)?; + let is_fortran_contiguous = tensor_info.layout.is_fortran_contiguous(); + let rank = tensor_info.layout.shape().rank(); + + // Reading the data is a bit tricky as it can be strided, for now only support the basic + // case and when the tensor is fortran contiguous. + if !tensor_info.layout.is_contiguous() && !is_fortran_contiguous { + crate::bail!( + "cannot retrieve non-contiguous tensors {:?}", + tensor_info.layout + ) + } + let start_offset = tensor_info.layout.start_offset(); + if start_offset > 0 { + std::io::copy( + &mut reader.by_ref().take(start_offset as u64), + &mut std::io::sink(), + )?; + } + let tensor = Tensor::from_reader( + tensor_info.layout.shape().clone(), + tensor_info.dtype, + &mut reader, + )?; + + if rank > 1 && is_fortran_contiguous { + // Reverse the shape, e.g. Shape(2, 3, 4) -> Shape(4, 3, 2) + let shape_reversed: Vec<_> = tensor_info.layout.dims().iter().rev().cloned().collect(); + let tensor = tensor.reshape(shape_reversed)?; + + // Permute (transpose) the dimensions, e.g. Shape(4, 3, 2) -> Shape(2, 3, 4) + let dim_indices_reversed: Vec<_> = (0..rank).rev().collect(); + let tensor = tensor.permute(dim_indices_reversed)?; + Ok(Some(tensor)) + } else { + Ok(Some(tensor)) + } + } +} + +/// Read all the tensors from a PyTorch pth file with a given key. +/// +/// # Arguments +/// * `path` - Path to the pth file. +/// * `key` - Optional key to retrieve `state_dict` from the pth file. Sometimes the pth file +/// contains multiple objects and the state_dict is the one we are interested in. +pub fn read_all_with_key>( + path: P, + key: Option<&str>, +) -> Result> { + let pth = PthTensors::new(path, key)?; + let tensor_names = pth.tensor_infos.keys(); + let mut tensors = Vec::with_capacity(tensor_names.len()); + for name in tensor_names { + if let Some(tensor) = pth.get(name)? { + tensors.push((name.to_string(), tensor)) + } + } + Ok(tensors) +} + +/// Read all the tensors from a PyTorch pth file. +/// +/// # Arguments +/// * `path` - Path to the pth file. +pub fn read_all>(path: P) -> Result> { + read_all_with_key(path, None) +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/avx.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/avx.rs new file mode 100644 index 0000000000..527941eb6d --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/quantized/avx.rs @@ -0,0 +1,669 @@ +use super::k_quants::{ + BlockQ2K, BlockQ3K, BlockQ4K, BlockQ4_0, BlockQ5K, BlockQ6K, BlockQ8K, BlockQ8_0, QK8_0, QK_K, +}; +use byteorder::{ByteOrder, LittleEndian}; +use half::f16; + +#[cfg(target_arch = "x86")] +use core::arch::x86::*; +#[cfg(target_arch = "x86_64")] +use core::arch::x86_64::*; + +#[inline(always)] +pub(crate) unsafe fn sum_i16_pairs_float(x: __m256i) -> __m256 { + let ones = _mm256_set1_epi16(1); + let summed_pairs = _mm256_madd_epi16(ones, x); + _mm256_cvtepi32_ps(summed_pairs) +} + +#[inline(always)] +pub(crate) unsafe fn mul_sum_us8_pairs_float(ax: __m256i, sy: __m256i) -> __m256 { + let dot = _mm256_maddubs_epi16(ax, sy); + sum_i16_pairs_float(dot) +} + +#[inline(always)] +pub(crate) unsafe fn hsum_float_8(x: __m256) -> f32 { + let res = _mm256_extractf128_ps(x, 1); + let res = _mm_add_ps(res, _mm256_castps256_ps128(x)); + let res = _mm_add_ps(res, _mm_movehl_ps(res, res)); + let res = _mm_add_ss(res, _mm_movehdup_ps(res)); + _mm_cvtss_f32(res) +} + +#[inline(always)] +pub(crate) unsafe fn bytes_from_nibbles_32(rsi: *const u8) -> __m256i { + let tmp = _mm_loadu_si128(rsi as *const __m128i); + let bytes = _mm256_insertf128_si256::<1>(_mm256_castsi128_si256(tmp), _mm_srli_epi16(tmp, 4)); + let low_mask = _mm256_set1_epi8(0xF); + _mm256_and_si256(low_mask, bytes) +} + +#[inline(always)] +pub(crate) unsafe fn mul_sum_i8_pairs_float(x: __m256i, y: __m256i) -> __m256 { + let ax = _mm256_sign_epi8(x, x); + let sy = _mm256_sign_epi8(y, x); + mul_sum_us8_pairs_float(ax, sy) +} + +#[inline(always)] +pub(crate) fn vec_dot_q4_0_q8_0(n: usize, xs: &[BlockQ4_0], ys: &[BlockQ8_0]) -> f32 { + debug_assert!( + n.is_multiple_of(QK8_0), + "vec_dot_q4_0_q8_0: {n} is not divisible by {QK8_0}" + ); + unsafe { + let mut acc = _mm256_setzero_ps(); + for (x, y) in xs.iter().zip(ys.iter()) { + let d = _mm256_set1_ps(f16::to_f32(x.d) * f16::to_f32(y.d)); + let bx = bytes_from_nibbles_32(x.qs.as_ptr()); + let off = _mm256_set1_epi8(8); + let bx = _mm256_sub_epi8(bx, off); + let by = _mm256_loadu_si256(y.qs.as_ptr() as *const __m256i); + let q = mul_sum_i8_pairs_float(bx, by); + acc = _mm256_fmadd_ps(d, q, acc); + } + hsum_float_8(acc) + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q8_0_q8_0(n: usize, xs: &[BlockQ8_0], ys: &[BlockQ8_0]) -> f32 { + debug_assert!( + n.is_multiple_of(QK8_0), + "vec_dot_q8_0_q8_0: {n} is not divisible by {QK8_0}" + ); + unsafe { + let mut acc = _mm256_setzero_ps(); + for (x, y) in xs.iter().zip(ys.iter()) { + let d = _mm256_set1_ps(f16::to_f32(x.d) * f16::to_f32(y.d)); + let bx = _mm256_loadu_si256(x.qs.as_ptr() as *const __m256i); + let by = _mm256_loadu_si256(y.qs.as_ptr() as *const __m256i); + let q = mul_sum_i8_pairs_float(bx, by); + acc = _mm256_fmadd_ps(d, q, acc); + } + hsum_float_8(acc) + } +} + +#[inline(always)] +unsafe fn get_scale_shuffle(i: usize) -> __m128i { + const K_SHUFFLE: [u8; 128] = [ + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, + 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, + 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, + 11, 11, 11, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 13, 13, 13, 13, 13, 13, 13, + 13, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, + ]; + _mm_loadu_si128((K_SHUFFLE.as_ptr() as *const __m128i).add(i)) +} + +#[inline(always)] +unsafe fn get_scale_shuffle_k4(i: usize) -> __m256i { + const K_SHUFFLE: [u8; 256] = [ + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, + 2, 3, 2, 3, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, + 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, + 6, 7, 6, 7, 6, 7, 6, 7, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, + 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, + 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 12, 13, 12, 13, 12, 13, + 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, + 13, 12, 13, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, + 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, + ]; + _mm256_loadu_si256((K_SHUFFLE.as_ptr() as *const __m256i).add(i)) +} + +#[inline(always)] +unsafe fn get_scale_shuffle_q3k(i: usize) -> __m256i { + const K_SHUFFLE: [u8; 128] = [ + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, + 2, 3, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, + 6, 7, 6, 7, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 10, 11, 10, 11, 10, 11, 10, 11, + 10, 11, 10, 11, 10, 11, 10, 11, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, + 13, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, + ]; + _mm256_loadu_si256((K_SHUFFLE.as_ptr() as *const __m256i).add(i)) +} + +#[inline(always)] +pub(crate) fn vec_dot_q6k_q8k(n: usize, xs: &[BlockQ6K], ys: &[BlockQ8K]) -> f32 { + debug_assert!( + n.is_multiple_of(QK_K), + "vec_dot_q6k_8k: {n} is not divisible by {QK_K}" + ); + + unsafe { + let m4 = _mm256_set1_epi8(0xF); + let m2 = _mm256_set1_epi8(3); + let m32s = _mm256_set1_epi8(32); + let mut acc = _mm256_setzero_ps(); + for (x, y) in xs.iter().zip(ys.iter()) { + let d = y.d * x.d.to_f32(); + let mut q4 = x.ql.as_ptr(); + let mut qh = x.qh.as_ptr(); + let mut q8 = y.qs.as_ptr(); + + let scales = _mm_loadu_si128(x.scales.as_ptr() as *const __m128i); + let mut sumi = _mm256_setzero_si256(); + + for j in 0..QK_K / 128 { + let is = j * 4; + let scale_0 = _mm_shuffle_epi8(scales, get_scale_shuffle(is)); + let scale_1 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 1)); + let scale_2 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 2)); + let scale_3 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 3)); + + let q4bits1 = _mm256_loadu_si256(q4 as *const __m256i); + q4 = q4.add(32); + let q4bits2 = _mm256_loadu_si256(q4 as *const __m256i); + q4 = q4.add(32); + let q4bits_h = _mm256_loadu_si256(qh as *const __m256i); + qh = qh.add(32); + + let q4h_0 = _mm256_slli_epi16(_mm256_and_si256(q4bits_h, m2), 4); + let q4h_1 = + _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bits_h, 2), m2), 4); + let q4h_2 = + _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bits_h, 4), m2), 4); + let q4h_3 = + _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bits_h, 6), m2), 4); + + let q4_0 = _mm256_or_si256(_mm256_and_si256(q4bits1, m4), q4h_0); + let q4_1 = _mm256_or_si256(_mm256_and_si256(q4bits2, m4), q4h_1); + let q4_2 = + _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits1, 4), m4), q4h_2); + let q4_3 = + _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits2, 4), m4), q4h_3); + + let q8_0 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let q8_1 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let q8_2 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let q8_3 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + + let q8s_0 = _mm256_maddubs_epi16(m32s, q8_0); + let q8s_1 = _mm256_maddubs_epi16(m32s, q8_1); + let q8s_2 = _mm256_maddubs_epi16(m32s, q8_2); + let q8s_3 = _mm256_maddubs_epi16(m32s, q8_3); + + let p16_0 = _mm256_maddubs_epi16(q4_0, q8_0); + let p16_1 = _mm256_maddubs_epi16(q4_1, q8_1); + let p16_2 = _mm256_maddubs_epi16(q4_2, q8_2); + let p16_3 = _mm256_maddubs_epi16(q4_3, q8_3); + + let p16_0 = _mm256_sub_epi16(p16_0, q8s_0); + let p16_1 = _mm256_sub_epi16(p16_1, q8s_1); + let p16_2 = _mm256_sub_epi16(p16_2, q8s_2); + let p16_3 = _mm256_sub_epi16(p16_3, q8s_3); + + let p16_0 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_0), p16_0); + let p16_1 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_1), p16_1); + let p16_2 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_2), p16_2); + let p16_3 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_3), p16_3); + + sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1)); + sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_2, p16_3)); + } + acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc); + } + hsum_float_8(acc) + } +} + +#[inline(always)] +unsafe fn mm256_set_m128i(a: __m128i, b: __m128i) -> __m256i { + _mm256_insertf128_si256(_mm256_castsi128_si256(b), a, 1) +} + +#[inline(always)] +pub(crate) fn vec_dot_q2k_q8k(n: usize, xs: &[BlockQ2K], ys: &[BlockQ8K]) -> f32 { + debug_assert!( + n.is_multiple_of(QK_K), + "vec_dot_q2k_q8k: {n} is not divisible by {QK_K}" + ); + + unsafe { + let m3 = _mm256_set1_epi8(3); + let m4 = _mm_set1_epi8(0xF); + + let mut acc = _mm256_setzero_ps(); + + for (x, y) in xs.iter().zip(ys.iter()) { + let d = y.d * x.d.to_f32(); + let dmin = -y.d * x.dmin.to_f32(); + + let mut q2 = x.qs.as_ptr(); + let mut q8 = y.qs.as_ptr(); + + let mins_and_scales = _mm_loadu_si128(x.scales.as_ptr() as *const __m128i); + let scales8 = _mm_and_si128(mins_and_scales, m4); + let mins8 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4); + let mins = _mm256_cvtepi8_epi16(mins8); + let prod = + _mm256_madd_epi16(mins, _mm256_loadu_si256(y.bsums.as_ptr() as *const __m256i)); + + acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&dmin), _mm256_cvtepi32_ps(prod), acc); + + let all_scales = _mm256_cvtepi8_epi16(scales8); + let l_scales = _mm256_extracti128_si256(all_scales, 0); + let h_scales = _mm256_extracti128_si256(all_scales, 1); + let scales = [ + mm256_set_m128i(l_scales, l_scales), + mm256_set_m128i(h_scales, h_scales), + ]; + + let mut sumi = _mm256_setzero_si256(); + + for scale in scales { + let q2bits = _mm256_loadu_si256(q2 as *const __m256i); + q2 = q2.add(32); + + let q8_0 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let q8_1 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let q8_2 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let q8_3 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + + let q2_0 = _mm256_and_si256(q2bits, m3); + let q2_1 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 2), m3); + let q2_2 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 4), m3); + let q2_3 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 6), m3); + + let p0 = _mm256_maddubs_epi16(q2_0, q8_0); + let p1 = _mm256_maddubs_epi16(q2_1, q8_1); + let p2 = _mm256_maddubs_epi16(q2_2, q8_2); + let p3 = _mm256_maddubs_epi16(q2_3, q8_3); + + let p0 = + _mm256_madd_epi16(_mm256_shuffle_epi8(scale, get_scale_shuffle_q3k(0)), p0); + let p1 = + _mm256_madd_epi16(_mm256_shuffle_epi8(scale, get_scale_shuffle_q3k(1)), p1); + let p2 = + _mm256_madd_epi16(_mm256_shuffle_epi8(scale, get_scale_shuffle_q3k(2)), p2); + let p3 = + _mm256_madd_epi16(_mm256_shuffle_epi8(scale, get_scale_shuffle_q3k(3)), p3); + + let p0 = _mm256_add_epi32(p0, p1); + let p2 = _mm256_add_epi32(p2, p3); + + sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p0, p2)); + } + acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc); + } + + hsum_float_8(acc) + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q3k_q8k(n: usize, xs: &[BlockQ3K], ys: &[BlockQ8K]) -> f32 { + debug_assert!( + n.is_multiple_of(QK_K), + "vec_dot_q3k_q8k: {n} is not divisible by {QK_K}" + ); + + const KMASK1: u32 = 0x03030303; + const KMASK2: u32 = 0x0f0f0f0f; + + let mut aux = [0u32; 3]; + + unsafe { + let m3 = _mm256_set1_epi8(3); + let mone = _mm256_set1_epi8(1); + let m32 = _mm_set1_epi8(32); + + let mut acc = _mm256_setzero_ps(); + for (x, y) in xs.iter().zip(ys.iter()) { + let d = y.d * x.d.to_f32(); + + let mut q3 = x.qs.as_ptr(); + let mut q8 = y.qs.as_ptr(); + + LittleEndian::read_u32_into(&x.scales, &mut aux); + let scales128 = _mm_set_epi32( + (((aux[1] >> 4) & KMASK2) | (((aux[2] >> 6) & KMASK1) << 4)) as i32, + (((aux[0] >> 4) & KMASK2) | (((aux[2] >> 4) & KMASK1) << 4)) as i32, + ((aux[1] & KMASK2) | (((aux[2] >> 2) & KMASK1) << 4)) as i32, + ((aux[0] & KMASK2) | (((aux[2]) & KMASK1) << 4)) as i32, + ); + let scales128 = _mm_sub_epi8(scales128, m32); + let all_scales = _mm256_cvtepi8_epi16(scales128); + let l_scales = _mm256_extracti128_si256(all_scales, 0); + let h_scales = _mm256_extracti128_si256(all_scales, 1); + let scales = [ + mm256_set_m128i(l_scales, l_scales), + mm256_set_m128i(h_scales, h_scales), + ]; + + // high bit + let hbits = _mm256_loadu_si256(x.hmask.as_ptr() as *const __m256i); + + let mut sumi = _mm256_setzero_si256(); + + for (j, scale) in scales.iter().enumerate() { + // load low 2 bits + let q3bits = _mm256_loadu_si256(q3 as *const __m256i); + q3 = q3.add(32); + + // Prepare low and high bits + // We hardcode the shifts here to avoid loading them into a separate register + let q3l_0 = _mm256_and_si256(q3bits, m3); + let q3h_0 = if j == 0 { + _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 0)), 0) + } else { + _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 4)), 4) + }; + let q3h_0 = _mm256_slli_epi16(q3h_0, 2); + + let q3l_1 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 2), m3); + let q3h_1 = if j == 0 { + _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 1)), 1) + } else { + _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 5)), 5) + }; + let q3h_1 = _mm256_slli_epi16(q3h_1, 2); + + let q3l_2 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 4), m3); + let q3h_2 = if j == 0 { + _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 2)), 2) + } else { + _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 6)), 6) + }; + let q3h_2 = _mm256_slli_epi16(q3h_2, 2); + + let q3l_3 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 6), m3); + let q3h_3 = if j == 0 { + _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 3)), 3) + } else { + _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 7)), 7) + }; + let q3h_3 = _mm256_slli_epi16(q3h_3, 2); + + // load Q8 quants + let q8_0 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let q8_1 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let q8_2 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let q8_3 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + + // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we + // can use _mm256_maddubs_epi16, and then subtract. The high bit part has the 2 + // already subtracted (and so, it is zero if the high bit was not set, and 2 if the + // high bit was set) + let q8s_0 = _mm256_maddubs_epi16(q3h_0, q8_0); + let q8s_1 = _mm256_maddubs_epi16(q3h_1, q8_1); + let q8s_2 = _mm256_maddubs_epi16(q3h_2, q8_2); + let q8s_3 = _mm256_maddubs_epi16(q3h_3, q8_3); + + let p16_0 = _mm256_maddubs_epi16(q3l_0, q8_0); + let p16_1 = _mm256_maddubs_epi16(q3l_1, q8_1); + let p16_2 = _mm256_maddubs_epi16(q3l_2, q8_2); + let p16_3 = _mm256_maddubs_epi16(q3l_3, q8_3); + + let p16_0 = _mm256_sub_epi16(p16_0, q8s_0); + let p16_1 = _mm256_sub_epi16(p16_1, q8s_1); + let p16_2 = _mm256_sub_epi16(p16_2, q8s_2); + let p16_3 = _mm256_sub_epi16(p16_3, q8s_3); + + // multiply with scales + let p16_0 = + _mm256_madd_epi16(_mm256_shuffle_epi8(*scale, get_scale_shuffle_q3k(0)), p16_0); + let p16_1 = + _mm256_madd_epi16(_mm256_shuffle_epi8(*scale, get_scale_shuffle_q3k(1)), p16_1); + let p16_2 = + _mm256_madd_epi16(_mm256_shuffle_epi8(*scale, get_scale_shuffle_q3k(2)), p16_2); + let p16_3 = + _mm256_madd_epi16(_mm256_shuffle_epi8(*scale, get_scale_shuffle_q3k(3)), p16_3); + + // accumulate + let p16_0 = _mm256_add_epi32(p16_0, p16_1); + let p16_2 = _mm256_add_epi32(p16_2, p16_3); + sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_2)); + } + + // multiply with block scale and accumulate + acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc); + } + hsum_float_8(acc) + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q4k_q8k(n: usize, xs: &[BlockQ4K], ys: &[BlockQ8K]) -> f32 { + debug_assert!( + n.is_multiple_of(QK_K), + "vec_dot_q4k_q8k: {n} is not divisible by {QK_K}" + ); + let mut utmp = [0u32; 4]; + const KMASK1: u32 = 0x3f3f3f3f; + const KMASK2: u32 = 0x0f0f0f0f; + const KMASK3: u32 = 0x03030303; + + unsafe { + let m4 = _mm256_set1_epi8(0xF); + + let mut acc = _mm256_setzero_ps(); + let mut acc_m = _mm_setzero_ps(); + + for (x, y) in xs.iter().zip(ys.iter()) { + let d = y.d * x.d.to_f32(); + let dmin = -y.d * x.dmin.to_f32(); + + LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); + + utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); + let uaux = utmp[1] & KMASK1; + utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); + utmp[2] = uaux; + utmp[0] &= KMASK1; + + let mut q4 = x.qs.as_ptr(); + let mut q8 = y.qs.as_ptr(); + + let mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32( + utmp[3] as i32, + utmp[2] as i32, + utmp[1] as i32, + utmp[0] as i32, + )); + + let q8sums = _mm256_loadu_si256(y.bsums.as_ptr() as *const __m256i); + let q8s = _mm_hadd_epi16( + _mm256_extracti128_si256(q8sums, 0), + _mm256_extracti128_si256(q8sums, 1), + ); + let prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s); + acc_m = _mm_fmadd_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod), acc_m); + + let sc128 = _mm256_extracti128_si256(mins_and_scales, 0); + let scales = mm256_set_m128i(sc128, sc128); + + let mut sumi = _mm256_setzero_si256(); + + for j in 0..QK_K / 64 { + let scale_l = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2 * j)); + let scale_h = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2 * j + 1)); + + let q4bits = _mm256_loadu_si256(q4 as *const __m256i); + q4 = q4.add(32); + let q4l = _mm256_and_si256(q4bits, m4); + let q4h = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), m4); + + let q8l = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let p16l = _mm256_maddubs_epi16(q4l, q8l); + let p16l = _mm256_madd_epi16(scale_l, p16l); + sumi = _mm256_add_epi32(sumi, p16l); + + let q8h = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let p16h = _mm256_maddubs_epi16(q4h, q8h); + let p16h = _mm256_madd_epi16(scale_h, p16h); + sumi = _mm256_add_epi32(sumi, p16h); + } + + let vd = _mm256_set1_ps(d); + acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc); + } + + let acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m)); + let acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m)); + + hsum_float_8(acc) + _mm_cvtss_f32(acc_m) + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q5k_q8k(n: usize, xs: &[BlockQ5K], ys: &[BlockQ8K]) -> f32 { + debug_assert!( + n.is_multiple_of(QK_K), + "vec_dot_q5k_q8k: {n} is not divisible by {QK_K}" + ); + let mut utmp = [0u32; 4]; + const KMASK1: u32 = 0x3f3f3f3f; + const KMASK2: u32 = 0x0f0f0f0f; + const KMASK3: u32 = 0x03030303; + + unsafe { + let m4 = _mm256_set1_epi8(0xF); + let mzero = _mm_setzero_si128(); + let mone = _mm256_set1_epi8(1); + + let mut acc = _mm256_setzero_ps(); + let mut summs = 0.0; + + for (x, y) in xs.iter().zip(ys.iter()) { + let d = y.d * x.d.to_f32(); + let dmin = -y.d * x.dmin.to_f32(); + + LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); + + utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); + let uaux = utmp[1] & KMASK1; + utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); + utmp[2] = uaux; + utmp[0] &= KMASK1; + + let mut q5 = x.qs.as_ptr(); + let mut q8 = y.qs.as_ptr(); + + let mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32( + utmp[3] as i32, + utmp[2] as i32, + utmp[1] as i32, + utmp[0] as i32, + )); + + let q8sums = _mm256_loadu_si256(y.bsums.as_ptr() as *const __m256i); + let q8s = _mm_hadd_epi16( + _mm256_extracti128_si256(q8sums, 0), + _mm256_extracti128_si256(q8sums, 1), + ); + let prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s); + let hsum = _mm_hadd_epi32(_mm_hadd_epi32(prod, mzero), mzero); + summs += dmin * _mm_extract_epi32(hsum, 0) as f32; + + let sc128 = _mm256_extracti128_si256(mins_and_scales, 0); + let scales = mm256_set_m128i(sc128, sc128); + + let hbits = _mm256_loadu_si256(x.qh.as_ptr() as *const __m256i); + let mut hmask = mone; + + let mut sumi = _mm256_setzero_si256(); + + for j in 0..QK_K / 64 { + let scale_0 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2 * j)); + let scale_1 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2 * j + 1)); + + let q5bits = _mm256_loadu_si256(q5 as *const __m256i); + q5 = q5.add(32); + + //Similar to q3k we hardcode the shifts here to avoid loading them into a separate register + let q5l_0 = _mm256_and_si256(q5bits, m4); + let q5l_0_shift_input = _mm256_and_si256(hbits, hmask); + let q5l_0_right_shift = match j { + 0 => _mm256_srli_epi16(q5l_0_shift_input, 0), + 1 => _mm256_srli_epi16(q5l_0_shift_input, 2), + 2 => _mm256_srli_epi16(q5l_0_shift_input, 4), + 3 => _mm256_srli_epi16(q5l_0_shift_input, 6), + _ => unreachable!(), + }; + let q5h_0 = _mm256_slli_epi16(q5l_0_right_shift, 4); + let q5_0 = _mm256_add_epi8(q5l_0, q5h_0); + hmask = _mm256_slli_epi16(hmask, 1); + + let q5l_1 = _mm256_and_si256(_mm256_srli_epi16(q5bits, 4), m4); + let q5l_1_shift_input = _mm256_and_si256(hbits, hmask); + let q5l_1_right_shift = match j { + 0 => _mm256_srli_epi16(q5l_1_shift_input, 1), + 1 => _mm256_srli_epi16(q5l_1_shift_input, 3), + 2 => _mm256_srli_epi16(q5l_1_shift_input, 5), + 3 => _mm256_srli_epi16(q5l_1_shift_input, 7), + _ => unreachable!(), + }; + + let q5h_1 = _mm256_slli_epi16(q5l_1_right_shift, 4); + let q5_1 = _mm256_add_epi8(q5l_1, q5h_1); + hmask = _mm256_slli_epi16(hmask, 1); + + let q8_0 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + let q8_1 = _mm256_loadu_si256(q8 as *const __m256i); + q8 = q8.add(32); + + let p16_0 = _mm256_maddubs_epi16(q5_0, q8_0); + let p16_1 = _mm256_maddubs_epi16(q5_1, q8_1); + + let p16_0 = _mm256_madd_epi16(scale_0, p16_0); + let p16_1 = _mm256_madd_epi16(scale_1, p16_1); + + sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1)); + } + let vd = _mm256_set1_ps(d); + acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc); + } + hsum_float_8(acc) + summs + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q8k_q8k(n: usize, xs: &[BlockQ8K], ys: &[BlockQ8K]) -> f32 { + debug_assert!( + n.is_multiple_of(QK_K), + "vec_dot_q8k_8k: {n} is not divisible by {QK_K}" + ); + unsafe { + let mut acc = _mm256_setzero_ps(); + for (xs, ys) in xs.iter().zip(ys.iter()) { + let mut sumi = _mm256_setzero_si256(); + let x_qs = xs.qs.as_ptr(); + let y_qs = ys.qs.as_ptr(); + for j in (0..QK_K).step_by(32) { + let xs = _mm256_loadu_si256(x_qs.add(j) as *const __m256i); + let ys = _mm256_loadu_si256(y_qs.add(j) as *const __m256i); + + let xs0 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(xs, 0)); + let ys0 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(ys, 0)); + sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(xs0, ys0)); + + let xs1 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(xs, 1)); + let ys1 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(ys, 1)); + sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(xs1, ys1)); + } + let d = _mm256_set1_ps(xs.d * ys.d); + acc = _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(sumi), acc); + } + hsum_float_8(acc) + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/cuda.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/cuda.rs new file mode 100644 index 0000000000..c47aaf9d24 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/quantized/cuda.rs @@ -0,0 +1,983 @@ +use super::{GgmlDType, QStorage}; +use crate::quantized::k_quants::GgmlType; +use crate::{backend::BackendDevice, cuda_backend::WrapErr}; +use crate::{builder_arg as barg, CudaDevice, CudaStorage, Result}; +use half::f16; + +use cudarc::driver::{CudaSlice, CudaView, PushKernelArg}; + +#[derive(Clone, Debug)] +struct PaddedCudaSlice { + inner: CudaSlice, + len: usize, +} + +#[derive(Clone, Debug)] +pub struct QCudaStorage { + data: PaddedCudaSlice, + dtype: GgmlDType, + device: CudaDevice, +} + +static FORCE_DMMV: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); + +pub fn set_force_dmmv(f: bool) { + FORCE_DMMV.store(f, std::sync::atomic::Ordering::Relaxed) +} + +pub const WARP_SIZE: usize = 32; +pub const MMQ_X_Q4_0_AMPERE: usize = 4; +pub const MMQ_Y_Q4_0_AMPERE: usize = 32; +pub const NWARPS_Q4_0_AMPERE: usize = 4; +pub const GGML_CUDA_MMV_X: usize = 32; +pub const GGML_CUDA_MMV_Y: usize = 1; +pub const CUDA_QUANTIZE_BLOCK_SIZE: usize = 256; +pub const CUDA_DEQUANTIZE_BLOCK_SIZE: usize = 256; +pub const MATRIX_ROW_PADDING: usize = 512; + +fn ceil_div(p: usize, q: usize) -> usize { + p.div_ceil(q) +} + +fn pad(p: usize, q: usize) -> usize { + ceil_div(p, q) * q +} + +fn quantize_q8_1( + src: &CudaView, + dst: &mut CudaSlice, + k: usize, + ky: usize, + dev: &CudaDevice, +) -> Result<()> { + let kx_padded = pad(k, MATRIX_ROW_PADDING); + let num_blocks = ceil_div(kx_padded, CUDA_QUANTIZE_BLOCK_SIZE); + + let total_rows = ky; + // Get Q8_1 metadata. + let q8_1_block_size = GgmlDType::Q8_1.block_size(); + let q8_1_type_size = GgmlDType::Q8_1.type_size(); + + // Calculate the size of the output buffer in bytes. + let num_blocks_per_row = kx_padded / q8_1_block_size; + let dst_row_size_bytes = num_blocks_per_row * q8_1_type_size; + + const CHUNK_SIZE: usize = 65535; // gridDim.y limit + let func = dev.get_or_load_func("quantize_q8_1", &candle_kernels::QUANTIZED)?; + + let mut rows_processed = 0; + while rows_processed < total_rows { + // --- calculate the number of rows for this chunk --- + let remaining_rows = total_rows - rows_processed; + // This is our gridDim.y, now <= 65535 + let rows_in_chunk = std::cmp::min(CHUNK_SIZE, remaining_rows); + + // --- slice the source (f32) tensor by elements --- + let src_start_elem = rows_processed * k; + let src_num_elems = rows_in_chunk * k; + let src_chunk = src.slice(src_start_elem..(src_start_elem + src_num_elems)); + + // --- slice the destination (u8) tensor by bytes --- + let dst_start_byte = rows_processed * dst_row_size_bytes; + let dst_num_bytes = rows_in_chunk * dst_row_size_bytes; + let dst_chunk = dst.slice(dst_start_byte..(dst_start_byte + dst_num_bytes)); + + let cfg = cudarc::driver::LaunchConfig { + grid_dim: (num_blocks as u32, rows_in_chunk as u32, 1), + block_dim: (CUDA_QUANTIZE_BLOCK_SIZE as u32, 1, 1), + shared_mem_bytes: 0, + }; + + let mut builder = func.builder(); + builder.arg(&src_chunk); + builder.arg(&dst_chunk); + barg!(builder, k as i32, kx_padded as i32); + unsafe { builder.launch(cfg) }.w()?; + + rows_processed += rows_in_chunk; + } + + Ok(()) +} + +fn dequantize_f32( + data: &PaddedCudaSlice, + dtype: GgmlDType, + elem_count: usize, + dev: &CudaDevice, +) -> Result { + let nb = elem_count.div_ceil(256); + let (kernel_name, is_k, block_dim, num_blocks) = match dtype { + GgmlDType::Q4_0 => ("dequantize_block_q4_0_f32", false, 32, nb), + GgmlDType::Q4_1 => ("dequantize_block_q4_1_f32", false, 32, nb), + GgmlDType::Q5_0 => ( + "dequantize_block_q5_0_f32", + false, + CUDA_DEQUANTIZE_BLOCK_SIZE, + ceil_div(elem_count, 2 * CUDA_DEQUANTIZE_BLOCK_SIZE), + ), + GgmlDType::Q5_1 => ( + "dequantize_block_q5_1_f32", + false, + CUDA_DEQUANTIZE_BLOCK_SIZE, + ceil_div(elem_count, 2 * CUDA_DEQUANTIZE_BLOCK_SIZE), + ), + GgmlDType::Q8_0 => ("dequantize_block_q8_0_f32", false, 32, nb), + GgmlDType::Q2K => ("dequantize_block_q2_K_f32", true, 64, nb), + GgmlDType::Q3K => ("dequantize_block_q3_K_f32", true, 64, nb), + GgmlDType::Q4K => ("dequantize_block_q4_K_f32", true, 32, nb), + GgmlDType::Q5K => ("dequantize_block_q5_K_f32", true, 64, nb), + GgmlDType::Q6K => ("dequantize_block_q6_K_f32", true, 64, nb), + GgmlDType::Q8K => ("dequantize_block_q8_K_f32", true, 32, nb), + _ => crate::bail!("unsupported dtype for dequantize {dtype:?}"), + }; + let func = dev.get_or_load_func(kernel_name, &candle_kernels::QUANTIZED)?; + let dst = unsafe { dev.alloc::(elem_count)? }; + // See e.g. + // https://github.com/ggerganov/llama.cpp/blob/cbbd1efa06f8c09f9dff58ff9d9af509cc4c152b/ggml-cuda.cu#L7270 + let cfg = cudarc::driver::LaunchConfig { + grid_dim: (num_blocks as u32, 1, 1), + block_dim: (block_dim as u32, 1, 1), + shared_mem_bytes: 0, + }; + + if is_k { + let mut builder = func.builder(); + builder.arg(&data.inner); + builder.arg(&dst); + unsafe { builder.launch(cfg) }.w()?; + } else { + let nb32 = match dtype { + GgmlDType::Q5_0 | GgmlDType::Q5_1 => elem_count, + _ => elem_count / 32, + }; + let mut builder = func.builder(); + builder.arg(&data.inner); + builder.arg(&dst); + barg!(builder, nb32 as i32); + unsafe { builder.launch(cfg) }.w()?; + } + Ok(CudaStorage::wrap_cuda_slice(dst, dev.clone())) +} + +fn dequantize_f16( + data: &PaddedCudaSlice, + dtype: GgmlDType, + elem_count: usize, + dev: &CudaDevice, +) -> Result { + let nb = elem_count.div_ceil(256); + let (kernel_name, is_k, block_dim, num_blocks) = match dtype { + GgmlDType::Q4_0 => ("dequantize_block_q4_0_f16", false, 32, nb), + GgmlDType::Q4_1 => ("dequantize_block_q4_1_f16", false, 32, nb), + GgmlDType::Q5_0 => ( + "dequantize_block_q5_0_f16", + false, + CUDA_DEQUANTIZE_BLOCK_SIZE, + ceil_div(elem_count, 2 * CUDA_DEQUANTIZE_BLOCK_SIZE), + ), + GgmlDType::Q5_1 => ( + "dequantize_block_q5_1_f16", + false, + CUDA_DEQUANTIZE_BLOCK_SIZE, + ceil_div(elem_count, 2 * CUDA_DEQUANTIZE_BLOCK_SIZE), + ), + GgmlDType::Q8_0 => ("dequantize_block_q8_0_f16", false, 32, nb), + GgmlDType::Q2K => ("dequantize_block_q2_K_f16", true, 64, nb), + GgmlDType::Q3K => ("dequantize_block_q3_K_f16", true, 64, nb), + GgmlDType::Q4K => ("dequantize_block_q4_K_f16", true, 32, nb), + GgmlDType::Q5K => ("dequantize_block_q5_K_f16", true, 64, nb), + GgmlDType::Q6K => ("dequantize_block_q6_K_f16", true, 64, nb), + GgmlDType::Q8K => ("dequantize_block_q8_K_f16", true, 32, nb), + _ => crate::bail!("unsupported dtype for dequantize {dtype:?}"), + }; + let func = dev.get_or_load_func(kernel_name, &candle_kernels::QUANTIZED)?; + let dst = unsafe { dev.alloc::(elem_count)? }; + // See e.g. + // https://github.com/ggerganov/llama.cpp/blob/cbbd1efa06f8c09f9dff58ff9d9af509cc4c152b/ggml-cuda.cu#L7270 + let cfg = cudarc::driver::LaunchConfig { + grid_dim: (num_blocks as u32, 1, 1), + block_dim: (block_dim as u32, 1, 1), + shared_mem_bytes: 0, + }; + + if is_k { + let mut builder = func.builder(); + builder.arg(&data.inner); + builder.arg(&dst); + unsafe { builder.launch(cfg) }.w()?; + } else { + let nb32 = match dtype { + GgmlDType::Q5_0 | GgmlDType::Q5_1 => elem_count, + _ => elem_count / 32, + }; + let mut builder = func.builder(); + builder.arg(&data.inner); + builder.arg(&dst); + barg!(builder, nb32 as i32); + unsafe { builder.launch(cfg) }.w()?; + } + Ok(CudaStorage::wrap_cuda_slice(dst, dev.clone())) +} + +fn dequantize_mul_mat_vec( + data: &PaddedCudaSlice, + y: &CudaView, + dtype: GgmlDType, + ncols: usize, + nrows: usize, + dev: &CudaDevice, +) -> Result { + let data_elems = data.len / dtype.type_size() * dtype.block_size(); + if data_elems < ncols * nrows { + crate::bail!("unexpected data size {}, ncols {ncols} {nrows}", data_elems) + } + if y.len() != ncols { + crate::bail!("unexpected y size {}, ncols {ncols} {nrows}", y.len()) + } + let kernel_name = match dtype { + GgmlDType::Q4_0 => "dequantize_mul_mat_vec_q4_0_cuda", + GgmlDType::Q4_1 => "dequantize_mul_mat_vec_q4_1_cuda", + GgmlDType::Q5_0 => "dequantize_mul_mat_vec_q5_0_cuda", + GgmlDType::Q5_1 => "dequantize_mul_mat_vec_q5_1_cuda", + GgmlDType::Q8_0 => "dequantize_mul_mat_vec_q8_0_cuda", + GgmlDType::Q2K => "dequantize_mul_mat_vec_q2_k", + GgmlDType::Q3K => "dequantize_mul_mat_vec_q3_k", + GgmlDType::Q4K => "dequantize_mul_mat_vec_q4_k", + GgmlDType::Q5K => "dequantize_mul_mat_vec_q5_k", + GgmlDType::Q6K => "dequantize_mul_mat_vec_q6_k", + _ => crate::bail!("unsupported dtype for quantized matmul {dtype:?}"), + }; + let func = dev.get_or_load_func(kernel_name, &candle_kernels::QUANTIZED)?; + let dst = unsafe { dev.alloc::(nrows)? }; + let block_num_y = ceil_div(nrows, GGML_CUDA_MMV_Y); + let cfg = cudarc::driver::LaunchConfig { + grid_dim: (block_num_y as u32, 1, 1), + block_dim: (WARP_SIZE as u32, GGML_CUDA_MMV_Y as u32, 1), + shared_mem_bytes: 0, + }; + + let mut builder = func.builder(); + builder.arg(&data.inner); + builder.arg(y); + builder.arg(&dst); + barg!(builder, ncols as i32, nrows as i32); + unsafe { builder.launch(cfg) }.w()?; + Ok(CudaStorage::wrap_cuda_slice(dst, dev.clone())) +} + +fn mul_mat_vec_via_q8_1( + data: &PaddedCudaSlice, + y: &CudaView, + dtype: GgmlDType, + ncols: usize, + nrows: usize, + b_size: usize, + dev: &CudaDevice, +) -> Result { + let data_elems = data.len / dtype.type_size() * dtype.block_size(); + if data_elems < ncols * nrows { + crate::bail!("unexpected data size {}, ncols {ncols} {nrows}", data_elems) + } + if y.len() != ncols * b_size { + crate::bail!("unexpected y size {}, ncols {ncols} {nrows}", y.len()) + } + if b_size == 0 || b_size > 8 { + crate::bail!("only bsize between 1 and 8 are supported, got {b_size}") + } + // Start by quantizing y + let ncols_padded = pad(ncols, MATRIX_ROW_PADDING); + let y_size_in_bytes = + b_size * ncols_padded * GgmlDType::Q8_1.type_size() / GgmlDType::Q8_1.block_size(); + let mut y_q8_1 = unsafe { dev.alloc::(y_size_in_bytes)? }; + quantize_q8_1(y, &mut y_q8_1, ncols, b_size, dev)?; + + let kernel_name = match dtype { + GgmlDType::Q4_0 => "mul_mat_vec_q4_0_q8_1_cuda", + GgmlDType::Q4_1 => "mul_mat_vec_q4_1_q8_1_cuda", + GgmlDType::Q5_0 => "mul_mat_vec_q5_0_q8_1_cuda", + GgmlDType::Q5_1 => "mul_mat_vec_q5_1_q8_1_cuda", + GgmlDType::Q8_0 => "mul_mat_vec_q8_0_q8_1_cuda", + GgmlDType::Q2K => "mul_mat_vec_q2_K_q8_1_cuda", + GgmlDType::Q3K => "mul_mat_vec_q3_K_q8_1_cuda", + GgmlDType::Q4K => "mul_mat_vec_q4_K_q8_1_cuda", + GgmlDType::Q5K => "mul_mat_vec_q5_K_q8_1_cuda", + GgmlDType::Q6K => "mul_mat_vec_q6_K_q8_1_cuda", + _ => crate::bail!("unsupported dtype for quantized matmul {dtype:?}"), + }; + let kernel_name = format!("{kernel_name}{b_size}"); + let func = dev.get_or_load_func(&kernel_name, &candle_kernels::QUANTIZED)?; + let dst = unsafe { dev.alloc::(nrows * b_size)? }; + // https://github.com/ggerganov/llama.cpp/blob/facb8b56f8fd3bb10a693bf0943ae9d69d0828ef/ggml-cuda/mmvq.cu#L98 + let (nblocks, nwarps) = match b_size { + 1 => (nrows as u32, 4), + 2..=4 => ((nrows as u32).div_ceil(2), 4), + 5..=8 => ((nrows as u32).div_ceil(2), 2), + _ => crate::bail!("unexpected bsize {b_size}"), + }; + let cfg = cudarc::driver::LaunchConfig { + grid_dim: (nblocks, 1, 1), + block_dim: (WARP_SIZE as u32, nwarps, 1), + shared_mem_bytes: 0, + }; + + let mut builder = func.builder(); + builder.arg(&data.inner); + builder.arg(&y_q8_1); + builder.arg(&dst); + barg!( + builder, + /* ncols_x */ ncols as i32, + /* nrows_x */ nrows as i32, + /* nrows_y */ ncols_padded as i32, + /* nrows_dst */ nrows as i32 + ); + unsafe { builder.launch(cfg) }.w()?; + Ok(CudaStorage::wrap_cuda_slice(dst, dev.clone())) +} + +#[allow(clippy::too_many_arguments)] +fn mul_mat_via_q8_1( + data: &PaddedCudaSlice, + y: &CudaView, + dtype: GgmlDType, + x_rows: usize, + x_cols: usize, + y_rows: usize, + y_cols: usize, + dev: &CudaDevice, +) -> Result { + let data_elems = data.len / dtype.type_size() * dtype.block_size(); + if data_elems < x_rows * x_cols { + crate::bail!("unexpected lhs size {}, {x_rows} {x_cols}", data_elems) + } + if y.len() != y_rows * y_cols { + crate::bail!("unexpected y size {}, {y_rows} {y_cols}", y.len()) + } + if x_cols != y_rows { + crate::bail!("unexpected x/y size {x_rows} {x_cols} {y_rows} {y_cols}") + } + let k = x_cols; + // Start by quantizing y + let k_padded = pad(k, MATRIX_ROW_PADDING); + let y_size_in_bytes = + k_padded * y_cols * GgmlDType::Q8_1.type_size() / GgmlDType::Q8_1.block_size(); + let mut y_q8_1 = unsafe { dev.alloc::(y_size_in_bytes)? }; + quantize_q8_1(y, &mut y_q8_1, k, y_cols, dev)?; + + let (kernel_name, mmq_x, mmq_y) = match dtype { + GgmlDType::Q4_0 => ("mul_mat_q4_0", 64, 128), + GgmlDType::Q4_1 => ("mul_mat_q4_1", 64, 128), + GgmlDType::Q5_0 => ("mul_mat_q5_0", 128, 64), + GgmlDType::Q5_1 => ("mul_mat_q5_1", 128, 64), + GgmlDType::Q8_0 => ("mul_mat_q8_0", 128, 64), + GgmlDType::Q2K => ("mul_mat_q2_K", 64, 128), + GgmlDType::Q3K => ("mul_mat_q3_K", 128, 128), + GgmlDType::Q4K => ("mul_mat_q4_K", 64, 128), + GgmlDType::Q5K => ("mul_mat_q5_K", 64, 128), + GgmlDType::Q6K => ("mul_mat_q6_K", 64, 64), + _ => crate::bail!("unsupported dtype for quantized matmul {dtype:?}"), + }; + let func = dev.get_or_load_func(kernel_name, &candle_kernels::QUANTIZED)?; + let dst = unsafe { dev.alloc::(x_rows * y_cols)? }; + let cfg = cudarc::driver::LaunchConfig { + grid_dim: ( + ceil_div(x_rows, mmq_y) as u32, + ceil_div(y_cols, mmq_x) as u32, + 1, + ), + block_dim: (WARP_SIZE as u32, 4, 1), + shared_mem_bytes: 0, + }; + + let mut builder = func.builder(); + builder.arg(/* vx */ &data.inner); + builder.arg(/* vy */ &y_q8_1); + builder.arg(/* dst */ &dst); + barg!( + builder, + /* ncols_x */ x_cols as i32, + /* nrows_x */ x_rows as i32, + /* ncols_y */ y_cols as i32, + /* nrows_y */ k_padded as i32, + /* nrows_dst */ x_rows as i32 + ); + unsafe { builder.launch(cfg) }.w()?; + Ok(CudaStorage::wrap_cuda_slice(dst, dev.clone())) +} + +#[allow(clippy::too_many_arguments)] +fn indexed_moe_forward_fused_q8_1_input( + weight: &CudaView, + w_shape: &crate::Shape, //[num_experts, n, k] + w_dtype: GgmlDType, + input: &CudaSlice, + in_shape: &crate::Shape, //[batch, topk or 1, k] + ids: &CudaView, + idx_shape: &crate::Shape, //[batch, topk] + dev: &CudaDevice, +) -> Result<(CudaStorage, crate::Shape)> { + let (_, n, k) = w_shape.dims3()?; + let batch = in_shape.dims()[0]; + let input_dim1 = in_shape.dims()[1]; + + let topk = idx_shape.dims()[1]; + assert!(batch == idx_shape.dims()[0], "batch dim not match!"); + + // Quantize input into q8_1. + let total_rows = batch * input_dim1; + let k_padded = pad(k, MATRIX_ROW_PADDING); + // Get Q8_1 metadata. + let q8_1_block_size = GgmlDType::Q8_1.block_size(); + let q8_1_type_size = GgmlDType::Q8_1.type_size(); + + // Calculate the size of the output buffer in bytes. + let num_blocks_per_row = k_padded / q8_1_block_size; + let dst_row_size_bytes = num_blocks_per_row * q8_1_type_size; + let y_size_in_bytes = total_rows * dst_row_size_bytes; + let mut input_quant = unsafe { dev.alloc::(y_size_in_bytes)? }; + + let input_view = input.slice(0..); + quantize_q8_1(&input_view, &mut input_quant, k, total_rows, dev)?; + + // output buffer + let outsize = batch * topk * n; + let out = unsafe { dev.alloc::(outsize)? }; + + let kernel_name = match w_dtype { + GgmlDType::Q2K => "indexed_moe_forward_q2k_q8_1", + GgmlDType::Q3K => "indexed_moe_forward_q3k_q8_1", + GgmlDType::Q4K => "indexed_moe_forward_q4k_q8_1", + GgmlDType::Q5K => "indexed_moe_forward_q5k_q8_1", + GgmlDType::Q6K => "indexed_moe_forward_q6k_q8_1", + GgmlDType::Q8_0 => "indexed_moe_forward_q8_0_q8_1", + _ => crate::bail!("unsupported dtype for indexed_moe_forward {w_dtype:?}"), + }; + let func = dev.get_or_load_func(kernel_name, &candle_kernels::QUANTIZED)?; + let (nblocks, nwarps) = (n as u32, 4); + let cfg = cudarc::driver::LaunchConfig { + grid_dim: (nblocks, batch as u32, topk as u32), + block_dim: (WARP_SIZE as u32, nwarps, 1), + shared_mem_bytes: 0, + }; + + let mut builder = func.builder(); + builder.arg(weight); + builder.arg(&input_quant); + builder.arg(ids); + builder.arg(&out); + + barg!( + builder, + n as i32, + k as i32, + batch as i32, + topk as i32, + k_padded as i32, + input_dim1 as i32 + ); + unsafe { builder.launch(cfg) }.w()?; + + let mut out_shape = in_shape.dims().to_vec(); + out_shape.pop(); + out_shape.push(n); + out_shape[1] = topk; + Ok(( + CudaStorage::wrap_cuda_slice(out, dev.clone()), + out_shape.into(), + )) +} + +impl QCudaStorage { + pub fn indexed_moe_forward( + &self, + self_shape: &crate::Shape, //[num_experts, n, k] + input: &CudaStorage, //[batch, topk or 1, k] + input_l: &crate::Layout, + ids: &CudaStorage, //[batch, topk] + ids_l: &crate::Layout, + ) -> Result<(CudaStorage, crate::Shape)> { + if matches!( + self.dtype(), + GgmlDType::Q8_0 + | GgmlDType::Q2K + | GgmlDType::Q3K + | GgmlDType::Q4K + | GgmlDType::Q5K + | GgmlDType::Q6K + ) { + let input_storage = input.as_cuda_slice::()?; + let ids_storage = ids.as_cuda_slice::()?; + indexed_moe_forward_fused_q8_1_input( + &self.data.inner.slice(0..), + self_shape, //[num_experts, n, k] + self.dtype(), + input_storage, + input_l.shape(), //[batch, topk or 1, k] + &ids_storage.slice(0..), + ids_l.shape(), //[batch, topk] + &self.device, + ) + } else { + crate::bail!( + "The given quantized dtype {:?} is not supported for indexed_moe_forward!", + self.dtype() + ); + } + } + + pub fn zeros(device: &CudaDevice, el_count: usize, dtype: GgmlDType) -> Result { + let size_in_bytes = ceil_div(el_count, dtype.block_size()) * dtype.type_size(); + let padded_size_in_bytes = + ceil_div(el_count + MATRIX_ROW_PADDING, dtype.block_size()) * dtype.type_size(); + let inner = device.alloc_zeros::(padded_size_in_bytes)?; + Ok(QCudaStorage { + data: PaddedCudaSlice { + inner, + len: size_in_bytes, + }, + device: device.clone(), + dtype, + }) + } + + pub fn dtype(&self) -> GgmlDType { + self.dtype + } + + pub fn device(&self) -> &CudaDevice { + &self.device + } + + pub fn dequantize(&self, elem_count: usize) -> Result { + fn deq(buffer: &[u8], n: usize, dst: &mut [f32]) { + let slice = unsafe { std::slice::from_raw_parts(buffer.as_ptr() as *const T, n) }; + let vec = slice.to_vec(); + T::to_float(&vec, dst) + } + + let fast_kernel = matches!( + self.dtype, + GgmlDType::Q4_0 + | GgmlDType::Q4_1 + | GgmlDType::Q5_0 + | GgmlDType::Q5_1 + | GgmlDType::Q8_0 + | GgmlDType::Q2K + | GgmlDType::Q3K + | GgmlDType::Q4K + | GgmlDType::Q5K + | GgmlDType::Q6K + | GgmlDType::Q8K + ); + if fast_kernel { + return dequantize_f32(&self.data, self.dtype, elem_count, self.device()); + } + // Run the dequantization on cpu. + + let buffer = self + .device + .clone_dtoh(&self.data.inner.slice(..self.data.len))?; + let mut out = vec![0.0; elem_count]; + let block_len = elem_count / self.dtype.block_size(); + match self.dtype { + GgmlDType::F32 => deq::(&buffer, block_len, &mut out), + GgmlDType::F16 => deq::(&buffer, block_len, &mut out), + GgmlDType::BF16 => deq::(&buffer, block_len, &mut out), + GgmlDType::Q4_0 => deq::(&buffer, block_len, &mut out), + GgmlDType::Q4_1 => deq::(&buffer, block_len, &mut out), + GgmlDType::Q5_0 => deq::(&buffer, block_len, &mut out), + GgmlDType::Q5_1 => deq::(&buffer, block_len, &mut out), + GgmlDType::Q8_0 => deq::(&buffer, block_len, &mut out), + GgmlDType::Q8_1 => deq::(&buffer, block_len, &mut out), + GgmlDType::Q2K => deq::(&buffer, block_len, &mut out), + GgmlDType::Q3K => deq::(&buffer, block_len, &mut out), + GgmlDType::Q4K => deq::(&buffer, block_len, &mut out), + GgmlDType::Q5K => deq::(&buffer, block_len, &mut out), + GgmlDType::Q6K => deq::(&buffer, block_len, &mut out), + GgmlDType::Q8K => deq::(&buffer, block_len, &mut out), + } + + self.device + .storage_from_cpu_storage(&crate::CpuStorage::F32(out)) + } + + pub fn dequantize_f16(&self, elem_count: usize) -> Result { + dequantize_f16(&self.data, self.dtype, elem_count, self.device()) + } + + pub fn quantize(&mut self, src: &CudaStorage) -> Result<()> { + // Run the quantization on cpu. + let src = match &src.slice { + crate::cuda_backend::CudaStorageSlice::F32(data) => self.device.clone_dtoh(data)?, + _ => crate::bail!("only f32 can be quantized"), + }; + let src_len = src.len(); + let src = crate::Storage::Cpu(crate::CpuStorage::F32(src)); + let mut qcpu_storage = crate::Device::Cpu.qzeros(src_len, self.dtype)?; + qcpu_storage.quantize(&src)?; + let data = qcpu_storage.data()?; + let padded_len = + data.len() + MATRIX_ROW_PADDING * self.dtype.type_size() / self.dtype.block_size(); + let mut inner = unsafe { self.device.alloc::(padded_len)? }; + self.device + .memcpy_htod(&*data, &mut inner.slice_mut(..data.len()))?; + self.data = PaddedCudaSlice { + inner, + len: data.len(), + }; + Ok(()) + } + + pub fn quantize_imatrix( + &mut self, + src: &CudaStorage, + imatrix_weights: &[f32], + n_per_row: usize, + ) -> Result<()> { + // Run the quantization on cpu. + let src = match &src.slice { + crate::cuda_backend::CudaStorageSlice::F32(data) => self.device.clone_dtoh(data)?, + _ => crate::bail!("only f32 can be quantized"), + }; + let src_len = src.len(); + let src = crate::Storage::Cpu(crate::CpuStorage::F32(src)); + let mut qcpu_storage = crate::Device::Cpu.qzeros(src_len, self.dtype)?; + qcpu_storage.quantize_imatrix(&src, imatrix_weights, n_per_row)?; + let data = qcpu_storage.data()?; + let padded_len = + data.len() + MATRIX_ROW_PADDING * self.dtype.type_size() / self.dtype.block_size(); + let mut inner = unsafe { self.device.alloc::(padded_len)? }; + self.device + .memcpy_htod(&*data, &mut inner.slice_mut(..data.len()))?; + self.data = PaddedCudaSlice { + inner, + len: data.len(), + }; + Ok(()) + } + + pub fn quantize_imatrix_onto( + &mut self, + src: &crate::CpuStorage, + imatrix_weights: &[f32], + n_per_row: usize, + ) -> Result<()> { + // Run the quantization on cpu. + let src_len = src.as_slice::()?.len(); + let mut qcpu_storage = crate::Device::Cpu.qzeros(src_len, self.dtype)?; + + if let QStorage::Cpu(storage) = &mut qcpu_storage { + storage.from_float_imatrix(src.as_slice::()?, imatrix_weights, n_per_row); + } else { + unreachable!() + } + + let data = qcpu_storage.data()?; + let padded_len = + data.len() + MATRIX_ROW_PADDING * self.dtype.type_size() / self.dtype.block_size(); + let mut inner = unsafe { self.device.alloc::(padded_len)? }; + self.device + .memcpy_htod(&*data, &mut inner.slice_mut(..data.len()))?; + self.data = PaddedCudaSlice { + inner, + len: data.len(), + }; + Ok(()) + } + + pub fn quantize_onto(&mut self, src: &crate::CpuStorage) -> Result<()> { + // Run the quantization on cpu. + let src_len = src.as_slice::()?.len(); + let mut qcpu_storage = crate::Device::Cpu.qzeros(src_len, self.dtype)?; + + if let QStorage::Cpu(storage) = &mut qcpu_storage { + storage.from_float(src.as_slice::()?); + } else { + unreachable!() + } + + let data = qcpu_storage.data()?; + let padded_len = + data.len() + MATRIX_ROW_PADDING * self.dtype.type_size() / self.dtype.block_size(); + let mut inner = unsafe { self.device.alloc::(padded_len)? }; + self.device + .memcpy_htod(&*data, &mut inner.slice_mut(..data.len()))?; + self.data = PaddedCudaSlice { + inner, + len: data.len(), + }; + Ok(()) + } + + pub fn storage_size_in_bytes(&self) -> usize { + self.data.len + } + + pub fn fwd( + &self, + self_shape: &crate::Shape, + storage: &CudaStorage, + layout: &crate::Layout, + ) -> Result<(CudaStorage, crate::Shape)> { + let max_bm = if FORCE_DMMV.load(std::sync::atomic::Ordering::Relaxed) { + 1 + } else { + 8 + }; + let use_vec_kernel = match layout.shape().dims() { + [b, m, _k] => b * m <= max_bm, + [b, _k] => *b <= max_bm, + _ => false, + }; + if use_vec_kernel { + self.dequantize_matmul_vec(self_shape, storage, layout) + } else { + self.dequantize_matmul(self_shape, storage, layout) + } + } + + pub fn data(&self) -> Result> { + let mut out = vec![0u8; self.data.len]; + self.device + .memcpy_dtoh(&self.data.inner.slice(..self.data.len), &mut out)?; + Ok(out) + } + + pub fn device_ptr(&self) -> Result<*const u8> { + use cudarc::driver::DevicePtr; + Ok(self.data.inner.device_ptr(self.data.inner.stream()).0 as *const u8) + } +} + +impl QCudaStorage { + fn dequantize_matmul_vec( + &self, + self_shape: &crate::Shape, + rhs: &CudaStorage, + rhs_l: &crate::Layout, + ) -> Result<(CudaStorage, crate::Shape)> { + let (nrows, ncols) = self_shape.dims2()?; + let rhs = rhs.as_cuda_slice::()?; + let rhs = match rhs_l.contiguous_offsets() { + Some((o1, o2)) => rhs.slice(o1..o2), + None => Err(crate::Error::RequiresContiguous { op: "dmmv" }.bt())?, + }; + let (b_size, k) = match rhs_l.shape().dims() { + [b, m, k] => (b * m, *k), + [b, k] => (*b, *k), + _ => crate::bail!("unexpected rhs shape in dmmv {:?}", rhs_l.shape()), + }; + if ncols != k { + crate::bail!("mismatch on matmul dim {self_shape:?} {:?}", rhs_l.shape()) + } + + let out = if FORCE_DMMV.load(std::sync::atomic::Ordering::Relaxed) { + dequantize_mul_mat_vec(&self.data, &rhs, self.dtype, ncols, nrows, self.device())? + } else { + mul_mat_vec_via_q8_1( + &self.data, + &rhs, + self.dtype, + ncols, + nrows, + b_size, + self.device(), + )? + }; + let mut out_shape = rhs_l.shape().dims().to_vec(); + out_shape.pop(); + out_shape.push(nrows); + Ok((out, out_shape.into())) + } + + fn dequantize_matmul( + &self, + self_shape: &crate::Shape, + storage: &CudaStorage, + layout: &crate::Layout, + ) -> Result<(CudaStorage, crate::Shape)> { + use crate::backend::BackendStorage; + let (n, k) = self_shape.dims2()?; + let (b, m, k2) = match layout.shape().dims() { + &[b, m, k2] => (b, m, k2), + &[m, k2] => (1, m, k2), + s => crate::bail!("unexpected shape for input {s:?}"), + }; + if k2 != k { + crate::bail!("mismatch on matmul dim {self_shape:?} {:?}", layout.shape()) + } + + let out = if FORCE_DMMV.load(std::sync::atomic::Ordering::Relaxed) { + let data_f32 = self.dequantize(n * k)?; + let rhs_l = crate::Layout::new((k, n).into(), vec![1, k], 0).broadcast_as((b, k, n))?; + storage.matmul(&data_f32, (b, m, n, k), layout, &rhs_l)? + } else { + let storage = storage.as_cuda_slice::()?; + let storage = match layout.contiguous_offsets() { + Some((o1, o2)) => storage.slice(o1..o2), + None => Err(crate::Error::RequiresContiguous { + op: "quantized-matmul", + } + .bt())?, + }; + mul_mat_via_q8_1( + &self.data, + &storage, + self.dtype, + /* x_rows */ n, + /* x_cols */ k, + /* y_rows */ k, + /* y_cols */ b * m, + self.device(), + )? + }; + let mut out_shape = layout.shape().dims().to_vec(); + out_shape.pop(); + out_shape.push(n); + Ok((out, out_shape.into())) + } +} + +pub fn load_quantized( + device: &CudaDevice, + data: &[T], +) -> Result { + let data = unsafe { + std::slice::from_raw_parts(data.as_ptr() as *const u8, core::mem::size_of_val(data)) + }; + let dtype = T::DTYPE; + let padded_len = data.len() + MATRIX_ROW_PADDING * dtype.type_size() / dtype.block_size(); + let mut inner = unsafe { device.alloc::(padded_len)? }; + device.memcpy_htod(data, &mut inner.slice_mut(..data.len()))?; + Ok(QStorage::Cuda(QCudaStorage { + data: PaddedCudaSlice { + inner, + len: data.len(), + }, + device: device.clone(), + dtype, + })) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn cuda_quantize_q8_1() -> Result<()> { + let dev = CudaDevice::new(0)?; + let el = 256; + let el_padded = pad(el, MATRIX_ROW_PADDING); + let y_size_in_bytes = + el_padded * GgmlDType::Q8_1.type_size() / GgmlDType::Q8_1.block_size(); + let mut y_q8_1 = unsafe { dev.alloc::(y_size_in_bytes)? }; + let vs: Vec = (0..el).map(|v| v as f32).collect(); + let y = dev.clone_htod(&vs)?; + quantize_q8_1(&y.as_view(), &mut y_q8_1, el, 1, &dev)?; + Ok(()) + } + + #[test] + fn cuda_mmv_q8_1() -> Result<()> { + let dev = CudaDevice::new(0)?; + let ncols = 256; + let vs: Vec = (0..ncols).map(|v| v as f32).collect(); + let y = dev.clone_htod(&vs)?; + let mut xs = QCudaStorage::zeros(&dev, ncols, GgmlDType::Q4_0)?; + xs.quantize(&CudaStorage::wrap_cuda_slice(y.clone(), dev.clone()))?; + let cuda_storage = mul_mat_vec_via_q8_1( + &xs.data, + &y.as_view(), + /* dtype */ GgmlDType::Q4_0, + /* ncols */ ncols, + /* nrows */ 1, + /* b_size */ 1, + &dev, + )?; + let vs = cuda_storage.as_cuda_slice::()?; + let vs = dev.clone_dtoh(&vs.as_view())?; + assert_eq!(vs.len(), 1); + // for n = 255, n.(n+1).(2n+1) / 6 = 5559680 + // Q8 means 1/256 precision. + assert_eq!(vs[0], 5561664.5); + + let cuda_storage = dequantize_mul_mat_vec( + &xs.data, + &y.as_view(), + /* dtype */ GgmlDType::Q4_0, + /* ncols */ ncols, + /* nrows */ 1, + &dev, + )?; + let vs = cuda_storage.as_cuda_slice::()?; + let vs = dev.clone_dtoh(&vs.as_view())?; + assert_eq!(vs.len(), 1); + assert_eq!(vs[0], 5561851.0); + Ok(()) + } + + #[test] + fn cuda_mm_q8_1() -> Result<()> { + let dev = CudaDevice::new(0)?; + let ncols = 256; + let vs: Vec = (0..ncols * 4).map(|v| v as f32 / 4.).collect(); + let y = dev.clone_htod(&vs)?; + let mut xs = QCudaStorage::zeros(&dev, ncols * 4, GgmlDType::Q4_0)?; + xs.quantize(&CudaStorage::wrap_cuda_slice(y.clone(), dev.clone()))?; + let cuda_storage = mul_mat_via_q8_1( + &xs.data, + &y.as_view(), + /* dtype */ GgmlDType::Q4_0, + /* x_rows */ 4, + /* x_cols */ ncols, + /* y_rows */ ncols, + /* y_cols */ 4, + &dev, + )?; + let vs = cuda_storage.as_cuda_slice::()?; + let vs = dev.clone_dtoh(&vs.as_view())?; + + /* + x = torch.tensor([float(v) for v in range(1024)]).reshape(4, 256) + x @ x.t() / 16 + tensor([[ 347480.0000, 869720.0000, 1391960.0000, 1914200.0000], + [ 869720.0000, 2440536.0000, 4011352.0000, 5582166.5000], + [ 1391960.0000, 4011352.0000, 6630742.0000, 9250132.0000], + [ 1914200.0000, 5582166.5000, 9250132.0000, 12918099.0000]]) + */ + assert_eq!(vs.len(), 16); + assert_eq!(vs[0], 347604.0); + assert_eq!(vs[1], 888153.06); + assert_eq!(vs[4], 869780.7); + assert_eq!(vs[5], 2483145.0); + assert_eq!(vs[11], 9407368.0); + assert_eq!(vs[14], 9470856.0); + assert_eq!(vs[15], 13138824.0); + Ok(()) + } + + // The following test used to fail under compute-sanitizer until #2526. + #[test] + fn cuda_mm_q8_1_pad() -> Result<()> { + let dev = CudaDevice::new(0)?; + let (x_rows, ncols, y_cols) = (4, 16, 2048); + let vs: Vec = (0..ncols * y_cols).map(|v| v as f32 / 256.).collect(); + let y = dev.clone_htod(&vs)?; + let mut xs = QCudaStorage::zeros(&dev, ncols * x_rows, GgmlDType::Q4_0)?; + xs.quantize(&CudaStorage::wrap_cuda_slice(y.clone(), dev.clone()))?; + let cuda_storage = mul_mat_via_q8_1( + &xs.data, + &y.as_view(), + /* dtype */ GgmlDType::Q4_0, + /* x_rows */ x_rows, + /* x_cols */ ncols, + /* y_rows */ ncols, + /* y_cols */ y_cols, + &dev, + )?; + let vs = cuda_storage.as_cuda_slice::()?; + let _vs = dev.clone_dtoh(&vs.as_view())?; + Ok(()) + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/dummy_cuda.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/dummy_cuda.rs new file mode 100644 index 0000000000..04f19f9fcb --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/quantized/dummy_cuda.rs @@ -0,0 +1,95 @@ +#![allow(unused)] +use super::GgmlDType; +use crate::{CudaDevice, CudaStorage, Error, Result}; + +pub struct QCudaStorage { + dtype: GgmlDType, + device: CudaDevice, +} + +impl QCudaStorage { + pub fn zeros(_: &CudaDevice, _: usize, _: GgmlDType) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + pub fn dtype(&self) -> GgmlDType { + self.dtype + } + + pub fn device(&self) -> &CudaDevice { + &self.device + } + + pub fn dequantize(&self, _elem_count: usize) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + pub fn dequantize_f16(&self, _elem_count: usize) -> Result { + Err(Error::NotCompiledWithCudaSupport) + } + + pub fn quantize(&mut self, _src: &CudaStorage) -> Result<()> { + Err(Error::NotCompiledWithCudaSupport) + } + + pub fn quantize_imatrix( + &mut self, + _src: &CudaStorage, + _imatrix_weights: &[f32], + _n_per_row: usize, + ) -> Result<()> { + Err(Error::NotCompiledWithCudaSupport) + } + + pub fn quantize_imatrix_onto( + &mut self, + _src: &crate::CpuStorage, + _imatrix_weights: &[f32], + _n_per_row: usize, + ) -> Result<()> { + Err(Error::NotCompiledWithCudaSupport) + } + + pub fn quantize_onto(&mut self, _src: &crate::CpuStorage) -> Result<()> { + Err(Error::NotCompiledWithCudaSupport) + } + + pub fn device_ptr(&self) -> Result<*const u8> { + Err(Error::NotCompiledWithCudaSupport) + } + + pub fn storage_size_in_bytes(&self) -> usize { + 0 + } + + pub fn fwd( + &self, + _self_shape: &crate::Shape, + _storage: &CudaStorage, + _layout: &crate::Layout, + ) -> Result<(CudaStorage, crate::Shape)> { + Err(Error::NotCompiledWithCudaSupport) + } + + pub fn data(&self) -> Result> { + Err(Error::NotCompiledWithCudaSupport) + } + + pub fn indexed_moe_forward( + &self, + _: &crate::Shape, + _: &CudaStorage, + _: &crate::Layout, + _: &CudaStorage, + _: &crate::Layout, + ) -> Result<(CudaStorage, crate::Shape)> { + Err(Error::NotCompiledWithCudaSupport) + } +} + +pub fn load_quantized( + _device: &CudaDevice, + _data: &[T], +) -> Result { + Err(Error::NotCompiledWithCudaSupport) +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/dummy_metal.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/dummy_metal.rs new file mode 100644 index 0000000000..6f470e9099 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/quantized/dummy_metal.rs @@ -0,0 +1,87 @@ +#![allow(unused)] +use super::GgmlDType; +use crate::{Error, MetalDevice, MetalStorage, Result}; + +pub struct QMetalStorage { + dtype: GgmlDType, + device: MetalDevice, +} + +impl QMetalStorage { + pub fn zeros(_: &MetalDevice, _: usize, _: GgmlDType) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + pub fn dtype(&self) -> GgmlDType { + self.dtype + } + + pub fn device(&self) -> &MetalDevice { + &self.device + } + + pub fn dequantize(&self, _elem_count: usize) -> Result { + Err(Error::NotCompiledWithMetalSupport) + } + + pub fn quantize(&mut self, _src: &MetalStorage) -> Result<()> { + Err(Error::NotCompiledWithMetalSupport) + } + + pub fn quantize_imatrix( + &mut self, + _src: &MetalStorage, + _imatrix_weights: &[f32], + _n_per_row: usize, + ) -> Result<()> { + Err(Error::NotCompiledWithMetalSupport) + } + + pub fn quantize_imatrix_onto( + &mut self, + _src: &crate::CpuStorage, + _imatrix_weights: &[f32], + _n_per_row: usize, + ) -> Result<()> { + Err(Error::NotCompiledWithMetalSupport) + } + + pub fn quantize_onto(&mut self, _src: &crate::CpuStorage) -> Result<()> { + Err(Error::NotCompiledWithCudaSupport) + } + + pub fn storage_size_in_bytes(&self) -> usize { + 0 + } + + pub fn fwd( + &self, + _self_shape: &crate::Shape, + _storage: &MetalStorage, + _layout: &crate::Layout, + ) -> Result<(MetalStorage, crate::Shape)> { + Err(Error::NotCompiledWithMetalSupport) + } + + pub fn data(&self) -> Result> { + Err(Error::NotCompiledWithMetalSupport) + } + + pub fn indexed_moe_forward( + &self, + _: &crate::Shape, + _: &MetalStorage, + _: &crate::Layout, + _: &MetalStorage, + _: &crate::Layout, + ) -> Result<(MetalStorage, crate::Shape)> { + Err(Error::NotCompiledWithMetalSupport) + } +} + +pub fn load_quantized( + _device: &MetalDevice, + _data: &[T], +) -> Result { + Err(Error::NotCompiledWithMetalSupport) +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/ggml_file.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/ggml_file.rs new file mode 100644 index 0000000000..ea5ec02578 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/quantized/ggml_file.rs @@ -0,0 +1,266 @@ +//! Support for the GGML file format. + +use super::{k_quants, GgmlDType, QStorage}; +use crate::{Device, Result}; +use byteorder::{LittleEndian, ReadBytesExt}; +use std::collections::HashMap; + +// https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/llama.h#L37 +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Magic { + Ggjt, + Ggla, + Ggmf, + Ggml, + Ggsn, +} + +impl TryFrom for Magic { + type Error = crate::Error; + fn try_from(value: u32) -> Result { + let magic = match value { + 0x67676a74 => Self::Ggjt, + 0x67676c61 => Self::Ggla, + 0x67676d66 => Self::Ggmf, + 0x67676d6c => Self::Ggml, + 0x6767736e => Self::Ggsn, + _ => crate::bail!("unknown magic {value:08x}"), + }; + Ok(magic) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VersionedMagic { + GgmlUnversioned, + GgmfV1, + GgjtV1, + GgjtV2, + GgjtV3, +} + +impl VersionedMagic { + fn read(reader: &mut R) -> Result { + let magic = reader.read_u32::()?; + let magic = Magic::try_from(magic)?; + if magic == Magic::Ggml { + return Ok(Self::GgmlUnversioned); + } + let version = reader.read_u32::()?; + let versioned_magic = match (magic, version) { + (Magic::Ggmf, 1) => Self::GgmfV1, + (Magic::Ggjt, 1) => Self::GgjtV1, + (Magic::Ggjt, 2) => Self::GgjtV2, + (Magic::Ggjt, 3) => Self::GgjtV3, + _ => crate::bail!("ggml: unsupported magic/version {magic:?}/{version}"), + }; + Ok(versioned_magic) + } + + fn align32(&self) -> bool { + match self { + Self::GgmlUnversioned | Self::GgmfV1 => false, + Self::GgjtV1 | Self::GgjtV2 | Self::GgjtV3 => true, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HParams { + pub n_vocab: u32, + pub n_embd: u32, + pub n_mult: u32, + pub n_head: u32, + pub n_layer: u32, + pub n_rot: u32, + pub ftype: u32, +} + +impl HParams { + fn read(reader: &mut R) -> Result { + let n_vocab = reader.read_u32::()?; + let n_embd = reader.read_u32::()?; + let n_mult = reader.read_u32::()?; + let n_head = reader.read_u32::()?; + let n_layer = reader.read_u32::()?; + let n_rot = reader.read_u32::()?; + let ftype = reader.read_u32::()?; + Ok(Self { + n_vocab, + n_embd, + n_mult, + n_head, + n_layer, + n_rot, + ftype, + }) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct Vocab { + pub token_score_pairs: Vec<(Vec, f32)>, +} + +impl Vocab { + fn read(reader: &mut R, n_vocab: usize) -> Result { + // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/llama.cpp#L556 + let mut token_score_pairs = Vec::with_capacity(n_vocab); + for _index in 0..n_vocab { + let len = reader.read_u32::()? as usize; + let mut word = vec![0u8; len]; + reader.read_exact(&mut word)?; + let score = reader.read_f32::()?; + token_score_pairs.push((word, score)) + } + Ok(Self { token_score_pairs }) + } +} + +fn from_raw_data( + raw_data: &[u8], + size_in_bytes: usize, + dims: Vec, + device: &Device, +) -> Result { + let raw_data_ptr = raw_data.as_ptr(); + let n_blocks = size_in_bytes / std::mem::size_of::(); + let data = unsafe { std::slice::from_raw_parts(raw_data_ptr as *const T, n_blocks) }; + let data: QStorage = match device { + Device::Cpu => QStorage::Cpu(Box::new(data.to_vec())), + Device::Metal(metal) => super::metal::load_quantized(metal, data)?, + Device::Cuda(cuda) => super::cuda::load_quantized(cuda, data)?, + }; + super::QTensor::new(data, dims) +} + +/// Creates a [Tensor] from a raw GGML tensor. +pub fn qtensor_from_ggml( + ggml_dtype: GgmlDType, + raw_data: &[u8], + dims: Vec, + device: &Device, +) -> Result { + let tensor_elems = dims.iter().product::(); + let block_size = ggml_dtype.block_size(); + if tensor_elems % block_size != 0 { + crate::bail!( + "the number of elements {tensor_elems} is not divisible by the block size {block_size}" + ) + } + let size_in_bytes = tensor_elems / block_size * ggml_dtype.type_size(); + + match ggml_dtype { + GgmlDType::F32 => from_raw_data::(raw_data, size_in_bytes, dims, device), + GgmlDType::F16 => from_raw_data::(raw_data, size_in_bytes, dims, device), + GgmlDType::BF16 => from_raw_data::(raw_data, size_in_bytes, dims, device), + GgmlDType::Q4_0 => { + from_raw_data::(raw_data, size_in_bytes, dims, device) + } + GgmlDType::Q4_1 => { + from_raw_data::(raw_data, size_in_bytes, dims, device) + } + GgmlDType::Q5_0 => { + from_raw_data::(raw_data, size_in_bytes, dims, device) + } + GgmlDType::Q5_1 => { + from_raw_data::(raw_data, size_in_bytes, dims, device) + } + GgmlDType::Q8_0 => { + from_raw_data::(raw_data, size_in_bytes, dims, device) + } + GgmlDType::Q2K => { + from_raw_data::(raw_data, size_in_bytes, dims, device) + } + GgmlDType::Q3K => { + from_raw_data::(raw_data, size_in_bytes, dims, device) + } + GgmlDType::Q4K => { + from_raw_data::(raw_data, size_in_bytes, dims, device) + } + GgmlDType::Q5K => { + from_raw_data::(raw_data, size_in_bytes, dims, device) + } + GgmlDType::Q6K => { + from_raw_data::(raw_data, size_in_bytes, dims, device) + } + _ => crate::bail!("quantized type {ggml_dtype:?} is not supported yet"), + } +} + +fn read_one_tensor( + reader: &mut R, + magic: VersionedMagic, + device: &Device, +) -> Result<(String, super::QTensor)> { + let n_dims = reader.read_u32::()?; + let name_len = reader.read_u32::()?; + let ggml_dtype = reader.read_u32::()?; + let ggml_dtype = GgmlDType::from_u32(ggml_dtype)?; + let mut dims = vec![0u32; n_dims as usize]; + reader.read_u32_into::(&mut dims)?; + // The dimensions are stored in reverse order, see for example: + // https://github.com/ggerganov/llama.cpp/blob/b5ffb2849d23afe73647f68eec7b68187af09be6/convert.py#L969 + dims.reverse(); + let mut name = vec![0u8; name_len as usize]; + reader.read_exact(&mut name)?; + let name = String::from_utf8_lossy(&name).into_owned(); + + if magic.align32() { + let pos = reader.stream_position()?; + reader.seek(std::io::SeekFrom::Current(((32 - pos % 32) % 32) as i64))?; + } + let dims = dims.iter().map(|&u| u as usize).collect::>(); + let tensor_elems = dims.iter().product::(); + let size_in_bytes = tensor_elems * ggml_dtype.type_size() / ggml_dtype.block_size(); + // TODO: Mmap version to avoid copying the data around? + let mut raw_data = vec![0u8; size_in_bytes]; + reader.read_exact(&mut raw_data)?; + match qtensor_from_ggml(ggml_dtype, &raw_data, dims, device) { + Ok(tensor) => Ok((name, tensor)), + Err(e) => crate::bail!("Error creating tensor {name}: {e}"), + } +} + +pub struct Content { + pub magic: VersionedMagic, + pub hparams: HParams, + pub vocab: Vocab, + pub tensors: HashMap, + pub device: Device, +} + +impl Content { + pub fn read( + reader: &mut R, + device: &Device, + ) -> Result { + // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/llama.cpp#L505 + let last_position = reader.seek(std::io::SeekFrom::End(0))?; + reader.seek(std::io::SeekFrom::Start(0))?; + let magic = VersionedMagic::read(reader)?; + let hparams = HParams::read(reader)?; + let vocab = Vocab::read(reader, hparams.n_vocab as usize)?; + let mut tensors = HashMap::new(); + + while reader.stream_position()? != last_position { + let (name, tensor) = read_one_tensor(reader, magic, device)?; + tensors.insert(name, tensor); + } + let device = device.clone(); + Ok(Self { + magic, + hparams, + vocab, + tensors, + device, + }) + } + + pub fn remove(&mut self, name: &str) -> Result { + match self.tensors.remove(name) { + None => crate::bail!("cannot find tensor with name '{name}'"), + Some(tensor) => Ok(tensor), + } + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/gguf_file.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/gguf_file.rs new file mode 100644 index 0000000000..197e43cfe3 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/quantized/gguf_file.rs @@ -0,0 +1,539 @@ +//! Support for the [GGUF file format](https://github.com/philpax/ggml/blob/gguf-spec/docs/gguf.md). +//! +//! Spec: https://github.com/ggml-org/ggml/blob/master/docs/gguf.md + +use super::{GgmlDType, QTensor}; +use crate::{Context, Device, Result}; +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; +use std::collections::HashMap; + +pub const DEFAULT_ALIGNMENT: u64 = 32; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Magic { + Gguf, +} + +impl TryFrom for Magic { + type Error = crate::Error; + fn try_from(value: u32) -> Result { + let magic = match value { + 0x46554747 | 0x47475546 => Self::Gguf, + _ => crate::bail!("unknown magic 0x{value:08x}"), + }; + Ok(magic) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum VersionedMagic { + GgufV1, + GgufV2, + GgufV3, +} + +impl VersionedMagic { + fn read(reader: &mut R) -> Result { + let magic = reader.read_u32::()?; + let magic = Magic::try_from(magic)?; + let version = reader.read_u32::()?; + let versioned_magic = match (magic, version) { + (Magic::Gguf, 1) => Self::GgufV1, + (Magic::Gguf, 2) => Self::GgufV2, + (Magic::Gguf, 3) => Self::GgufV3, + _ => crate::bail!("gguf: unsupported magic/version {magic:?}/{version}"), + }; + Ok(versioned_magic) + } +} + +#[derive(Debug)] +pub struct TensorInfo { + pub ggml_dtype: GgmlDType, + pub shape: crate::Shape, + pub offset: u64, +} + +impl TensorInfo { + pub fn read( + &self, + reader: &mut R, + tensor_data_offset: u64, + device: &Device, + ) -> Result { + let tensor_elems = self.shape.elem_count(); + let block_size = self.ggml_dtype.block_size(); + if !tensor_elems.is_multiple_of(block_size) { + crate::bail!( + "the number of elements {tensor_elems} is not divisible by the block size {block_size}" + ) + } + let size_in_bytes = tensor_elems / block_size * self.ggml_dtype.type_size(); + let mut raw_data = vec![0u8; size_in_bytes]; + reader.seek(std::io::SeekFrom::Start(tensor_data_offset + self.offset))?; + reader.read_exact(&mut raw_data)?; + super::ggml_file::qtensor_from_ggml( + self.ggml_dtype, + &raw_data, + self.shape.dims().to_vec(), + device, + ) + } +} + +#[derive(Debug)] +pub struct Content { + pub magic: VersionedMagic, + pub metadata: HashMap, + pub tensor_infos: HashMap, + pub tensor_data_offset: u64, +} + +fn read_string(reader: &mut R, magic: &VersionedMagic) -> Result { + let len = match magic { + VersionedMagic::GgufV1 => reader.read_u32::()? as usize, + VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => { + reader.read_u64::()? as usize + } + }; + let mut v = vec![0u8; len]; + reader.read_exact(&mut v)?; + // GGUF strings are supposed to be non-null terminated but in practice this happens. + while let Some(0) = v.last() { + v.pop(); + } + // GGUF strings are utf8 encoded but there are cases that don't seem to be valid. + Ok(String::from_utf8_lossy(&v).into_owned()) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ValueType { + // The value is a 8-bit unsigned integer. + U8, + // The value is a 8-bit signed integer. + I8, + // The value is a 16-bit unsigned little-endian integer. + U16, + // The value is a 16-bit signed little-endian integer. + I16, + // The value is a 32-bit unsigned little-endian integer. + U32, + // The value is a 32-bit signed little-endian integer. + I32, + // The value is a 64-bit unsigned little-endian integer. + U64, + // The value is a 64-bit signed little-endian integer. + I64, + // The value is a 32-bit IEEE754 floating point number. + F32, + // The value is a 64-bit IEEE754 floating point number. + F64, + // The value is a boolean. + // 1-byte value where 0 is false and 1 is true. + // Anything else is invalid, and should be treated as either the model being invalid or the reader being buggy. + Bool, + // The value is a UTF-8 non-null-terminated string, with length prepended. + String, + // The value is an array of other values, with the length and type prepended. + // Arrays can be nested, and the length of the array is the number of elements in the array, not the number of bytes. + Array, +} + +#[derive(Debug, Clone)] +pub enum Value { + U8(u8), + I8(i8), + U16(u16), + I16(i16), + U32(u32), + I32(i32), + U64(u64), + I64(i64), + F32(f32), + F64(f64), + Bool(bool), + String(String), + Array(Vec), +} + +impl Value { + pub fn value_type(&self) -> ValueType { + match self { + Self::U8(_) => ValueType::U8, + Self::I8(_) => ValueType::I8, + Self::U16(_) => ValueType::U16, + Self::I16(_) => ValueType::I16, + Self::U32(_) => ValueType::U32, + Self::I32(_) => ValueType::I32, + Self::U64(_) => ValueType::U64, + Self::I64(_) => ValueType::I64, + Self::F32(_) => ValueType::F32, + Self::F64(_) => ValueType::F64, + Self::Bool(_) => ValueType::Bool, + Self::String(_) => ValueType::String, + Self::Array(_) => ValueType::Array, + } + } + + pub fn to_u8(&self) -> Result { + match self { + Self::U8(v) => Ok(*v), + v => crate::bail!("not a u8 {v:?}"), + } + } + + pub fn to_i8(&self) -> Result { + match self { + Self::I8(v) => Ok(*v), + v => crate::bail!("not a i8 {v:?}"), + } + } + + pub fn to_u16(&self) -> Result { + match self { + Self::U16(v) => Ok(*v), + v => crate::bail!("not a u16 {v:?}"), + } + } + + pub fn to_i16(&self) -> Result { + match self { + Self::I16(v) => Ok(*v), + v => crate::bail!("not a i16 {v:?}"), + } + } + + pub fn to_u32(&self) -> Result { + match self { + Self::U32(v) => Ok(*v), + v => crate::bail!("not a u32 {v:?}"), + } + } + + pub fn to_i32(&self) -> Result { + match self { + Self::I32(v) => Ok(*v), + v => crate::bail!("not a i32 {v:?}"), + } + } + + /// This will also automatically upcast any integral types which will not truncate. + pub fn to_u64(&self) -> Result { + match self { + Self::U64(v) => Ok(*v), + // Autoupcast cases here + Self::U8(v) => Ok(*v as u64), + Self::U16(v) => Ok(*v as u64), + Self::U32(v) => Ok(*v as u64), + Self::Bool(v) => Ok(*v as u64), + v => crate::bail!("not a u64 or upcastable to u64 {v:?}"), + } + } + + pub fn to_i64(&self) -> Result { + match self { + Self::I64(v) => Ok(*v), + v => crate::bail!("not a i64 {v:?}"), + } + } + + pub fn to_f32(&self) -> Result { + match self { + Self::F32(v) => Ok(*v), + v => crate::bail!("not a f32 {v:?}"), + } + } + + pub fn to_f64(&self) -> Result { + match self { + Self::F64(v) => Ok(*v), + v => crate::bail!("not a f64 {v:?}"), + } + } + + pub fn to_bool(&self) -> Result { + match self { + Self::Bool(v) => Ok(*v), + v => crate::bail!("not a bool {v:?}"), + } + } + + pub fn to_vec(&self) -> Result<&Vec> { + match self { + Self::Array(v) => Ok(v), + v => crate::bail!("not a vec {v:?}"), + } + } + + pub fn to_string(&self) -> Result<&String> { + match self { + Self::String(v) => Ok(v), + v => crate::bail!("not a string {v:?}"), + } + } + + fn read( + reader: &mut R, + value_type: ValueType, + magic: &VersionedMagic, + ) -> Result { + let v = match value_type { + ValueType::U8 => Self::U8(reader.read_u8()?), + ValueType::I8 => Self::I8(reader.read_i8()?), + ValueType::U16 => Self::U16(reader.read_u16::()?), + ValueType::I16 => Self::I16(reader.read_i16::()?), + ValueType::U32 => Self::U32(reader.read_u32::()?), + ValueType::I32 => Self::I32(reader.read_i32::()?), + ValueType::U64 => Self::U64(reader.read_u64::()?), + ValueType::I64 => Self::I64(reader.read_i64::()?), + ValueType::F32 => Self::F32(reader.read_f32::()?), + ValueType::F64 => Self::F64(reader.read_f64::()?), + ValueType::Bool => match reader.read_u8()? { + 0 => Self::Bool(false), + 1 => Self::Bool(true), + b => crate::bail!("unexpected bool value {b}"), + }, + ValueType::String => Self::String(read_string(reader, magic)?), + ValueType::Array => { + let value_type = reader.read_u32::()?; + let value_type = ValueType::from_u32(value_type)?; + let len = match magic { + VersionedMagic::GgufV1 => reader.read_u32::()? as usize, + VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => { + reader.read_u64::()? as usize + } + }; + let mut vs = Vec::with_capacity(len); + for _ in 0..len { + vs.push(Value::read(reader, value_type, magic)?) + } + Self::Array(vs) + } + }; + Ok(v) + } + + fn write(&self, w: &mut W) -> Result<()> { + match self { + &Self::U8(v) => w.write_u8(v)?, + &Self::I8(v) => w.write_i8(v)?, + &Self::U16(v) => w.write_u16::(v)?, + &Self::I16(v) => w.write_i16::(v)?, + &Self::U32(v) => w.write_u32::(v)?, + &Self::I32(v) => w.write_i32::(v)?, + &Self::U64(v) => w.write_u64::(v)?, + &Self::I64(v) => w.write_i64::(v)?, + &Self::F32(v) => w.write_f32::(v)?, + &Self::F64(v) => w.write_f64::(v)?, + &Self::Bool(v) => w.write_u8(u8::from(v))?, + Self::String(v) => write_string(w, v.as_str())?, + Self::Array(v) => { + // The `Value` type does not enforce that all the values in an Array have the same + // type. + let value_type = if v.is_empty() { + // Doesn't matter, the array is empty. + ValueType::U32 + } else { + let value_type: std::collections::HashSet<_> = + v.iter().map(|elem| elem.value_type()).collect(); + if value_type.len() != 1 { + crate::bail!("multiple value-types in the same array {value_type:?}") + } + value_type.into_iter().next().context("empty value_type")? + }; + w.write_u32::(value_type.to_u32())?; + w.write_u64::(v.len() as u64)?; + for elem in v.iter() { + elem.write(w)? + } + } + } + Ok(()) + } +} + +impl ValueType { + fn from_u32(v: u32) -> Result { + let v = match v { + 0 => Self::U8, + 1 => Self::I8, + 2 => Self::U16, + 3 => Self::I16, + 4 => Self::U32, + 5 => Self::I32, + 6 => Self::F32, + 7 => Self::Bool, + 8 => Self::String, + 9 => Self::Array, + 10 => Self::U64, + 11 => Self::I64, + 12 => Self::F64, + v => crate::bail!("unrecognized value-type {v:#08x}"), + }; + Ok(v) + } + + fn to_u32(self) -> u32 { + match self { + Self::U8 => 0, + Self::I8 => 1, + Self::U16 => 2, + Self::I16 => 3, + Self::U32 => 4, + Self::I32 => 5, + Self::F32 => 6, + Self::Bool => 7, + Self::String => 8, + Self::Array => 9, + Self::U64 => 10, + Self::I64 => 11, + Self::F64 => 12, + } + } +} + +impl Content { + pub fn read(reader: &mut R) -> Result { + let magic = VersionedMagic::read(reader)?; + + let tensor_count = match magic { + VersionedMagic::GgufV1 => reader.read_u32::()? as usize, + VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => { + reader.read_u64::()? as usize + } + }; + let metadata_kv_count = match magic { + VersionedMagic::GgufV1 => reader.read_u32::()? as usize, + VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => { + reader.read_u64::()? as usize + } + }; + + let mut metadata = HashMap::new(); + for _idx in 0..metadata_kv_count { + let key = read_string(reader, &magic)?; + let value_type = reader.read_u32::()?; + let value_type = ValueType::from_u32(value_type)?; + let value = Value::read(reader, value_type, &magic)?; + metadata.insert(key, value); + } + let mut tensor_infos = HashMap::new(); + for _idx in 0..tensor_count { + let tensor_name = read_string(reader, &magic)?; + let n_dimensions = reader.read_u32::()?; + + let mut dimensions: Vec = match magic { + VersionedMagic::GgufV1 => { + let mut dimensions = vec![0; n_dimensions as usize]; + reader.read_u32_into::(&mut dimensions)?; + dimensions.into_iter().map(|c| c as usize).collect() + } + VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => { + let mut dimensions = vec![0; n_dimensions as usize]; + reader.read_u64_into::(&mut dimensions)?; + dimensions.into_iter().map(|c| c as usize).collect() + } + }; + + dimensions.reverse(); + let ggml_dtype = reader.read_u32::()?; + let ggml_dtype = GgmlDType::from_u32(ggml_dtype)?; + let offset = reader.read_u64::()?; + tensor_infos.insert( + tensor_name, + TensorInfo { + shape: crate::Shape::from(dimensions), + offset, + ggml_dtype, + }, + ); + } + let position = reader.stream_position()?; + let alignment = match metadata.get("general.alignment") { + Some(Value::U8(v)) => *v as u64, + Some(Value::U16(v)) => *v as u64, + Some(Value::U32(v)) => *v as u64, + Some(Value::I8(v)) if *v >= 0 => *v as u64, + Some(Value::I16(v)) if *v >= 0 => *v as u64, + Some(Value::I32(v)) if *v >= 0 => *v as u64, + _ => DEFAULT_ALIGNMENT, + }; + let tensor_data_offset = position.div_ceil(alignment) * alignment; + Ok(Self { + magic, + metadata, + tensor_infos, + tensor_data_offset, + }) + } + + pub fn tensor( + &self, + reader: &mut R, + name: &str, + device: &Device, + ) -> Result { + let tensor_info = match self.tensor_infos.get(name) { + Some(tensor_info) => tensor_info, + None => crate::bail!("cannot find tensor info for {name}"), + }; + tensor_info.read(reader, self.tensor_data_offset, device) + } +} + +fn write_string(w: &mut W, str: &str) -> Result<()> { + let bytes = str.as_bytes(); + w.write_u64::(bytes.len() as u64)?; + w.write_all(bytes)?; + Ok(()) +} + +pub fn write( + w: &mut W, + metadata: &[(&str, &Value)], + tensors: &[(&str, &QTensor)], +) -> Result<()> { + w.write_u32::(0x46554747)?; + w.write_u32::(2)?; // version 2. + w.write_u64::(tensors.len() as u64)?; + w.write_u64::(metadata.len() as u64)?; + for (name, value) in metadata.iter() { + write_string(w, name)?; + w.write_u32::(value.value_type().to_u32())?; + value.write(w)?; + } + let mut offset = 0usize; + let mut offsets = Vec::with_capacity(tensors.len()); + for (name, tensor) in tensors.iter() { + write_string(w, name)?; + let dims = tensor.shape().dims(); + w.write_u32::(dims.len() as u32)?; + for &dim in dims.iter().rev() { + w.write_u64::(dim as u64)?; + } + w.write_u32::(tensor.dtype().to_u32())?; + w.write_u64::(offset as u64)?; + offsets.push(offset); + let size_in_bytes = tensor.storage_size_in_bytes(); + let padding = 31 - (31 + size_in_bytes) % 32; + offset += size_in_bytes + padding; + } + let pos = w.stream_position()? as usize; + let padding = 31 - (31 + pos) % 32; + w.write_all(&vec![0u8; padding])?; + let tensor_start_pos = w.stream_position()? as usize; + for (offset, (_name, tensor)) in offsets.iter().zip(tensors.iter()) { + let pos = w.stream_position()? as usize; + if tensor_start_pos + offset != pos { + crate::bail!( + "internal error, unexpected current position {tensor_start_pos} {offset} {pos}" + ) + } + let data = tensor.data()?; + let size_in_bytes = data.len(); + w.write_all(&data)?; + let padding = 31 - (31 + size_in_bytes) % 32; + w.write_all(&vec![0u8; padding])?; + } + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/imatrix_file.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/imatrix_file.rs new file mode 100644 index 0000000000..ed228b74ce --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/quantized/imatrix_file.rs @@ -0,0 +1,85 @@ +use std::collections::HashMap; +use std::fs::File; +use std::io::{Cursor, Read}; +use std::path::Path; + +use byteorder::{LittleEndian, ReadBytesExt}; + +use crate::Result; + +pub fn load_imatrix>(fname: P) -> Result>> { + let mut all_data = HashMap::new(); + + let mut file = File::open(&fname).map_err(|e| { + crate::Error::msg(format!( + "Failed to open {}: {}", + fname.as_ref().display(), + e + )) + })?; + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer).map_err(|e| { + crate::Error::msg(format!( + "Failed to read file {}: {}", + fname.as_ref().display(), + e + )) + })?; + + let mut cursor = Cursor::new(buffer); + + let n_entries = cursor + .read_i32::() + .map_err(|e| crate::Error::msg(format!("Failed to read number of entries: {e}")))? + as usize; + + if n_entries < 1 { + crate::bail!("No data in file {}", fname.as_ref().display()); + } + + for i in 0..n_entries { + // Read length of the name + let len = cursor.read_i32::().map_err(|e| { + crate::Error::msg(format!( + "Failed to read name length for entry {}: {}", + i + 1, + e + )) + })? as usize; + + // Read the name + let mut name_buf = vec![0u8; len]; + cursor.read_exact(&mut name_buf).map_err(|e| { + crate::Error::msg(format!("Failed to read name for entry {}: {}", i + 1, e)) + })?; + let name = String::from_utf8(name_buf).map_err(|e| { + crate::Error::msg(format!("Invalid UTF-8 name for entry {}: {}", i + 1, e)) + })?; + + // Read ncall and nval + let ncall = cursor.read_i32::().map_err(|e| { + crate::Error::msg(format!("Failed to read ncall for entry {}: {}", i + 1, e)) + })? as usize; + + let nval = cursor.read_i32::().map_err(|e| { + crate::Error::msg(format!("Failed to read nval for entry {}: {}", i + 1, e)) + })? as usize; + + if nval < 1 { + crate::bail!("Invalid nval for entry {}: {}", i + 1, nval); + } + + let mut data = Vec::with_capacity(nval); + for _ in 0..nval { + let v = cursor.read_f32::().unwrap(); + if ncall == 0 { + data.push(v); + } else { + data.push(v / ncall as f32); + } + } + all_data.insert(name, data); + } + + Ok(all_data) +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/k_quants.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/k_quants.rs new file mode 100644 index 0000000000..9069b23667 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/quantized/k_quants.rs @@ -0,0 +1,2505 @@ +use super::utils::{ + get_scale_min_k4, group_for_dequantization, group_for_quantization, make_q3_quants, + make_qkx1_quants, make_qx_quants, nearest_int, +}; +use super::GgmlDType; +use crate::quantized::utils::{make_qkx3_quants, make_qp_quants}; +use crate::Result; +use byteorder::{ByteOrder, LittleEndian}; +use half::{bf16, f16, slice::HalfFloatSliceExt}; +use rayon::prelude::*; + +// Default to QK_K 256 rather than 64. +pub const QK_K: usize = 256; +pub const K_SCALE_SIZE: usize = 12; + +pub const QK4_0: usize = 32; +pub const QK4_1: usize = 32; +pub const QK5_0: usize = 32; +pub const QK5_1: usize = 32; +pub const QK8_0: usize = 32; +pub const QK8_1: usize = 32; + +pub trait GgmlType: Sized + Clone + Send + Sync { + const DTYPE: GgmlDType; + const BLCK_SIZE: usize; + const DIRECT_COPY: bool = false; + type VecDotType: GgmlType; + + // This is only safe for types that include immediate values such as float/int/... + fn zeros() -> Self { + unsafe { std::mem::MaybeUninit::zeroed().assume_init() } + } + fn to_float(xs: &[Self], ys: &mut [f32]); + fn from_float(xs: &[f32], ys: &mut [Self]); + fn from_float_imatrix( + _xs: &[f32], + _ys: &mut [Self], + _imatrix_weights: &[f32], + _n_per_row: usize, + ) { + panic!( + "`from_float_imatrix` is unimplemented for {:?}", + Self::DTYPE + ); + } + + fn direct_copy(_xs: &[f32], _ys: &mut [Self]) {} + + /// Dot product used as a building block for quantized mat-mul. + /// n is the number of elements to be considered. + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32; + + /// Generic implementation of the dot product without simd optimizations. + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32; +} + +#[derive(Debug, Clone, PartialEq)] +#[repr(C)] +pub struct BlockQ4_0 { + pub(crate) d: f16, + pub(crate) qs: [u8; QK4_0 / 2], +} +const _: () = assert!(std::mem::size_of::() == 18); + +#[derive(Debug, Clone, PartialEq)] +#[repr(C)] +pub struct BlockQ4_1 { + pub(crate) d: f16, + pub(crate) m: f16, + pub(crate) qs: [u8; QK4_1 / 2], +} +const _: () = assert!(std::mem::size_of::() == 20); + +#[derive(Debug, Clone, PartialEq)] +#[repr(C)] +pub struct BlockQ5_0 { + pub(crate) d: f16, + pub(crate) qh: [u8; 4], + pub(crate) qs: [u8; QK5_0 / 2], +} +const _: () = assert!(std::mem::size_of::() == 22); + +#[derive(Debug, Clone, PartialEq)] +#[repr(C)] +pub struct BlockQ5_1 { + pub(crate) d: f16, + pub(crate) m: f16, + pub(crate) qh: [u8; 4], + pub(crate) qs: [u8; QK5_1 / 2], +} +const _: () = assert!(std::mem::size_of::() == 24); + +#[derive(Debug, Clone, PartialEq)] +#[repr(C)] +pub struct BlockQ8_0 { + pub(crate) d: f16, + pub(crate) qs: [i8; QK8_0], +} +const _: () = assert!(std::mem::size_of::() == 34); + +#[derive(Debug, Clone, PartialEq)] +#[repr(C)] +pub struct BlockQ8_1 { + pub(crate) d: f16, + pub(crate) s: f16, + pub(crate) qs: [i8; QK8_1], +} +const _: () = assert!(std::mem::size_of::() == 36); + +#[derive(Debug, Clone, PartialEq)] +#[repr(C)] +pub struct BlockQ2K { + pub(crate) scales: [u8; QK_K / 16], + pub(crate) qs: [u8; QK_K / 4], + pub(crate) d: f16, + pub(crate) dmin: f16, +} +const _: () = assert!(QK_K / 16 + QK_K / 4 + 2 * 2 == std::mem::size_of::()); + +#[derive(Debug, Clone, PartialEq)] +#[repr(C)] +pub struct BlockQ3K { + pub(crate) hmask: [u8; QK_K / 8], + pub(crate) qs: [u8; QK_K / 4], + pub(crate) scales: [u8; 12], + pub(crate) d: f16, +} +const _: () = assert!(QK_K / 8 + QK_K / 4 + 12 + 2 == std::mem::size_of::()); + +#[derive(Debug, Clone, PartialEq)] +// https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/k_quants.h#L82 +#[repr(C)] +pub struct BlockQ4K { + pub(crate) d: f16, + pub(crate) dmin: f16, + pub(crate) scales: [u8; K_SCALE_SIZE], + pub(crate) qs: [u8; QK_K / 2], +} +const _: () = assert!(QK_K / 2 + K_SCALE_SIZE + 2 * 2 == std::mem::size_of::()); + +#[derive(Debug, Clone, PartialEq)] +#[repr(C)] +pub struct BlockQ5K { + pub(crate) d: f16, + pub(crate) dmin: f16, + pub(crate) scales: [u8; K_SCALE_SIZE], + pub(crate) qh: [u8; QK_K / 8], + pub(crate) qs: [u8; QK_K / 2], +} +const _: () = + assert!(QK_K / 8 + QK_K / 2 + 2 * 2 + K_SCALE_SIZE == std::mem::size_of::()); + +#[derive(Debug, Clone, PartialEq)] +#[repr(C)] +pub struct BlockQ6K { + pub(crate) ql: [u8; QK_K / 2], + pub(crate) qh: [u8; QK_K / 4], + pub(crate) scales: [i8; QK_K / 16], + pub(crate) d: f16, +} +const _: () = assert!(3 * QK_K / 4 + QK_K / 16 + 2 == std::mem::size_of::()); + +#[derive(Debug, Clone, PartialEq)] +#[repr(C)] +pub struct BlockQ8K { + pub(crate) d: f32, + pub(crate) qs: [i8; QK_K], + pub(crate) bsums: [i16; QK_K / 16], +} +const _: () = assert!(4 + QK_K + QK_K / 16 * 2 == std::mem::size_of::()); + +impl GgmlType for BlockQ4_0 { + const DTYPE: GgmlDType = GgmlDType::Q4_0; + const BLCK_SIZE: usize = QK4_0; + type VecDotType = BlockQ8_0; + + // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1525 + fn to_float(xs: &[Self], ys: &mut [f32]) { + let k = ys.len(); + let qk = Self::BLCK_SIZE; + debug_assert!( + k.is_multiple_of(qk), + "dequantize_row_q4_0: {k} is not divisible by {qk}" + ); + + let nb = k / qk; + for i in 0..nb { + let d = xs[i].d.to_f32(); + + for j in 0..(qk / 2) { + let x0 = (xs[i].qs[j] & 0x0F) as i16 - 8; + let x1 = (xs[i].qs[j] >> 4) as i16 - 8; + + ys[i * qk + j] = (x0 as f32) * d; + ys[i * qk + j + qk / 2] = (x1 as f32) * d; + } + } + } + + fn from_float(xs: &[f32], ys: &mut [Self]) { + // quantize_row_q4_0 + let qk = Self::BLCK_SIZE; + let k = xs.len(); + debug_assert!(k.is_multiple_of(qk), "{k} is not divisible by {qk}"); + debug_assert_eq!( + ys.len(), + k / qk, + "size mismatch {} {} {}", + xs.len(), + ys.len(), + qk, + ); + for (i, ys) in ys.iter_mut().enumerate() { + let mut amax = 0f32; + let mut max = 0f32; + + let xs = &xs[i * qk..(i + 1) * qk]; + for &x in xs.iter() { + if amax < x.abs() { + amax = x.abs(); + max = x; + } + } + let d = max / -8.0; + let id = if d != 0f32 { 1. / d } else { 0. }; + ys.d = f16::from_f32(d); + + for (j, q) in ys.qs.iter_mut().enumerate() { + let x0 = xs[j] * id; + let x1 = xs[qk / 2 + j] * id; + let xi0 = u8::min(15, (x0 + 8.5) as u8); + let xi1 = u8::min(15, (x1 + 8.5) as u8); + *q = xi0 | (xi1 << 4) + } + } + } + + // https://github.com/ggerganov/llama.cpp/blob/b5ffb2849d23afe73647f68eec7b68187af09be6/ggml.c#L2361C10-L2361C122 + #[allow(unreachable_code)] + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + #[cfg(target_feature = "avx2")] + return super::avx::vec_dot_q4_0_q8_0(n, xs, ys); + + #[cfg(target_feature = "neon")] + return super::neon::vec_dot_q4_0_q8_0(n, xs, ys); + + #[cfg(target_feature = "simd128")] + return super::simd128::vec_dot_q4_0_q8_0(n, xs, ys); + + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + debug_assert!( + n.is_multiple_of(QK8_0), + "vec_dot_q4_0_q8_0: {n} is not divisible by {QK8_0}" + ); + // Generic implementation. + let mut sumf = 0f32; + for (xs, ys) in xs.iter().zip(ys.iter()) { + let mut sum_i = 0; + for j in 0..QK8_0 / 2 { + let v0 = (xs.qs[j] & 0x0F) as i32 - 8; + let v1 = (xs.qs[j] >> 4) as i32 - 8; + sum_i += v0 * ys.qs[j] as i32 + v1 * ys.qs[j + QK8_0 / 2] as i32 + } + sumf += sum_i as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d) + } + sumf + } +} + +impl GgmlType for BlockQ4_1 { + const DTYPE: GgmlDType = GgmlDType::Q4_1; + const BLCK_SIZE: usize = QK4_1; + type VecDotType = BlockQ8_1; + + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + // ggml_vec_dot_q4_1_q8_1 + let qk = QK8_1; + debug_assert!( + n.is_multiple_of(qk), + "vec_dot_q4_1_q8_1: {n} is not divisible by {qk}" + ); + debug_assert!( + (n / qk).is_multiple_of(2), + "vec_dot_q4_1_q8_1: {n}, nb is not divisible by 2" + ); + + // Generic implementation. + let mut sumf = 0f32; + + for (xs, ys) in xs.iter().zip(ys.iter()) { + let mut sumi = 0i32; + + for j in 0..qk / 2 { + let v0 = xs.qs[j] as i32 & 0x0F; + let v1 = xs.qs[j] as i32 >> 4; + sumi += (v0 * ys.qs[j] as i32) + (v1 * ys.qs[j + qk / 2] as i32); + } + + sumf += sumi as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d) + + f16::to_f32(xs.m) * f16::to_f32(ys.s) + } + sumf + } + + fn from_float(xs: &[f32], ys: &mut [Self]) { + // quantize_row_q4_1 + let qk = Self::BLCK_SIZE; + + debug_assert_eq!( + ys.len() * qk, + xs.len(), + "size mismatch {} {} {}", + xs.len(), + ys.len(), + qk, + ); + for (i, ys) in ys.iter_mut().enumerate() { + let xs = &xs[i * qk..(i + 1) * qk]; + + let mut min = f32::INFINITY; + let mut max = f32::NEG_INFINITY; + for &x in xs.iter() { + min = f32::min(x, min); + max = f32::max(x, max); + } + let d = (max - min) / ((1 << 4) - 1) as f32; + let id = if d != 0f32 { 1. / d } else { 0. }; + ys.d = f16::from_f32(d); + ys.m = f16::from_f32(min); + + for (j, q) in ys.qs.iter_mut().take(qk / 2).enumerate() { + let x0 = (xs[j] - min) * id; + let x1 = (xs[qk / 2 + j] - min) * id; + + let xi0 = u8::min(15, (x0 + 0.5) as u8); + let xi1 = u8::min(15, (x1 + 0.5) as u8); + + *q = xi0 | (xi1 << 4); + } + } + } + + // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1545 + fn to_float(xs: &[Self], ys: &mut [f32]) { + let k = ys.len(); + debug_assert!( + k.is_multiple_of(QK4_1), + "dequantize_row_q4_1: {k} is not divisible by {QK4_1}" + ); + + let nb = k / QK4_1; + for i in 0..nb { + let d = xs[i].d.to_f32(); + let m = xs[i].m.to_f32(); + + for j in 0..(QK4_1 / 2) { + let x0 = xs[i].qs[j] & 0x0F; + let x1 = xs[i].qs[j] >> 4; + + ys[i * QK4_1 + j] = (x0 as f32) * d + m; + ys[i * QK4_1 + j + QK4_1 / 2] = (x1 as f32) * d + m; + } + } + } +} + +impl GgmlType for BlockQ5_0 { + const DTYPE: GgmlDType = GgmlDType::Q5_0; + const BLCK_SIZE: usize = QK5_0; + type VecDotType = BlockQ8_0; + + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + let qk = Self::BLCK_SIZE; + + debug_assert!( + n.is_multiple_of(qk), + "vec_dot_q5_0_q8_0: {n} is not divisible by {qk}" + ); + debug_assert!( + (n / qk).is_multiple_of(2), + "vec_dot_q5_0_q8_0: {n}, nb is not divisible by 2" + ); + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(_n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + // Generic implementation. + let mut sumf = 0f32; + + for (xs, ys) in xs.iter().zip(ys.iter()) { + let qh = LittleEndian::read_u32(&xs.qh); + let mut sumi = 0i32; + + for j in 0..Self::BLCK_SIZE / 2 { + let xh_0 = (((qh & (1u32 << j)) >> j) << 4) as u8; + let xh_1 = ((qh & (1u32 << (j + 16))) >> (j + 12)) as u8; + + let x0 = ((xs.qs[j] & 0x0F) as i32 | xh_0 as i32) - 16; + let x1 = ((xs.qs[j] >> 4) as i32 | xh_1 as i32) - 16; + + sumi += (x0 * ys.qs[j] as i32) + (x1 * ys.qs[j + Self::BLCK_SIZE / 2] as i32); + } + + sumf += sumi as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d) + } + sumf + } + + fn from_float(xs: &[f32], ys: &mut [Self]) { + // quantize_row_q5_0 + debug_assert_eq!( + ys.len() * Self::BLCK_SIZE, + xs.len(), + "size mismatch {} {} {}", + xs.len(), + ys.len(), + Self::BLCK_SIZE, + ); + for (i, ys) in ys.iter_mut().enumerate() { + let xs = &xs[i * Self::BLCK_SIZE..(i + 1) * Self::BLCK_SIZE]; + + let mut amax = 0f32; + let mut max = 0f32; + for &x in xs.iter() { + if amax < x.abs() { + amax = x.abs(); + max = x; + } + } + let d = max / -16.; + let id = if d != 0f32 { 1. / d } else { 0. }; + ys.d = f16::from_f32(d); + let mut qh = 0u32; + for j in 0..Self::BLCK_SIZE / 2 { + let x0 = xs[j] * id; + let x1 = xs[j + Self::BLCK_SIZE / 2] * id; + let xi0 = ((x0 + 16.5) as i8).min(31) as u8; + let xi1 = ((x1 + 16.5) as i8).min(31) as u8; + ys.qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4); + qh |= ((xi0 as u32 & 0x10) >> 4) << j; + qh |= ((xi1 as u32 & 0x10) >> 4) << (j + Self::BLCK_SIZE / 2); + } + LittleEndian::write_u32(&mut ys.qh, qh) + } + } + + // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1566 + fn to_float(xs: &[Self], ys: &mut [f32]) { + let k = ys.len(); + debug_assert!( + k.is_multiple_of(QK5_0), + "dequantize_row_q5_0: {k} is not divisible by {QK5_0}" + ); + let nb = k / QK5_0; + for i in 0..nb { + let d = xs[i].d.to_f32(); + let qh: u32 = LittleEndian::read_u32(&xs[i].qh); + + for j in 0..(QK5_0 / 2) { + let xh_0 = (((qh >> j) << 4) & 0x10) as u8; + let xh_1 = ((qh >> (j + 12)) & 0x10) as u8; + + let x0 = ((xs[i].qs[j] & 0x0F) | xh_0) as i32 - 16; + let x1 = ((xs[i].qs[j] >> 4) | xh_1) as i32 - 16; + + ys[i * QK5_0 + j] = (x0 as f32) * d; + ys[i * QK5_0 + j + QK5_0 / 2] = (x1 as f32) * d; + } + } + } +} + +impl GgmlType for BlockQ5_1 { + const DTYPE: GgmlDType = GgmlDType::Q5_1; + const BLCK_SIZE: usize = QK5_1; + type VecDotType = BlockQ8_1; + + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + let qk = Self::BLCK_SIZE; + debug_assert!( + n.is_multiple_of(qk), + "vec_dot_q5_1_q8_1: {n} is not divisible by {qk}" + ); + debug_assert!( + (n / qk).is_multiple_of(2), + "vec_dot_q5_1_q8_1: {n}, nb is not divisible by 2" + ); + + // Generic implementation. + let mut sumf = 0f32; + + for (xs, ys) in xs.iter().zip(ys.iter()) { + let qh = LittleEndian::read_u32(&xs.qh); + let mut sumi = 0i32; + + for j in 0..Self::BLCK_SIZE / 2 { + let xh_0 = ((qh >> j) << 4) & 0x10; + let xh_1 = (qh >> (j + 12)) & 0x10; + + let x0 = (xs.qs[j] as i32 & 0xF) | xh_0 as i32; + let x1 = (xs.qs[j] as i32 >> 4) | xh_1 as i32; + + sumi += (x0 * ys.qs[j] as i32) + (x1 * ys.qs[j + Self::BLCK_SIZE / 2] as i32); + } + + sumf += sumi as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d) + + f16::to_f32(xs.m) * f16::to_f32(ys.s) + } + sumf + } + + fn from_float(xs: &[f32], ys: &mut [Self]) { + // quantize_row_q5_1 + let qk = Self::BLCK_SIZE; + debug_assert_eq!( + ys.len() * qk, + xs.len(), + "size mismatch {} {} {}", + xs.len(), + ys.len(), + qk, + ); + for (i, ys) in ys.iter_mut().enumerate() { + let xs = &xs[i * qk..(i + 1) * qk]; + + let mut min = f32::INFINITY; + let mut max = f32::NEG_INFINITY; + for &x in xs.iter() { + min = f32::min(x, min); + max = f32::max(x, max); + } + let d = (max - min) / ((1 << 5) - 1) as f32; + let id = if d != 0f32 { 1. / d } else { 0. }; + ys.d = f16::from_f32(d); + ys.m = f16::from_f32(min); + + let mut qh = 0u32; + for (j, q) in ys.qs.iter_mut().take(qk / 2).enumerate() { + let x0 = (xs[j] - min) * id; + let x1 = (xs[qk / 2 + j] - min) * id; + + let xi0 = (x0 + 0.5) as u8; + let xi1 = (x1 + 0.5) as u8; + + *q = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4); + // get the 5-th bit and store it in qh at the right position + qh |= ((xi0 as u32 & 0x10) >> 4) << j; + qh |= ((xi1 as u32 & 0x10) >> 4) << (j + qk / 2); + } + LittleEndian::write_u32(&mut ys.qh, qh); + } + } + + // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1592 + fn to_float(xs: &[Self], ys: &mut [f32]) { + let k = ys.len(); + debug_assert!( + k.is_multiple_of(QK5_1), + "dequantize_row_q5_1: {k} is not divisible by {QK5_1}" + ); + + let nb = k / QK5_1; + for i in 0..nb { + let d = xs[i].d.to_f32(); + let m = xs[i].m.to_f32(); + let qh: u32 = LittleEndian::read_u32(&xs[i].qh); + + for j in 0..(QK5_1 / 2) { + let xh_0 = (((qh >> j) << 4) & 0x10) as u8; + let xh_1 = ((qh >> (j + 12)) & 0x10) as u8; + + let x0 = (xs[i].qs[j] & 0x0F) | xh_0; + let x1 = (xs[i].qs[j] >> 4) | xh_1; + + ys[i * QK5_1 + j] = (x0 as f32) * d + m; + ys[i * QK5_1 + j + QK5_1 / 2] = (x1 as f32) * d + m; + } + } + } +} + +impl GgmlType for BlockQ8_0 { + const DTYPE: GgmlDType = GgmlDType::Q8_0; + const BLCK_SIZE: usize = QK8_0; + type VecDotType = BlockQ8_0; + + // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1619 + fn to_float(xs: &[Self], ys: &mut [f32]) { + let k = ys.len(); + debug_assert!( + k.is_multiple_of(QK8_0), + "dequantize_row_q8_0: {k} is not divisible by {QK8_0}" + ); + + let nb = k / QK8_0; + + for i in 0..nb { + let d = xs[i].d.to_f32(); + + for j in 0..QK8_0 { + ys[i * QK8_0 + j] = xs[i].qs[j] as f32 * d; + } + } + } + + fn from_float(xs: &[f32], ys: &mut [Self]) { + // quantize_row_q8_0 + let k = xs.len(); + debug_assert!( + k.is_multiple_of(Self::BLCK_SIZE), + "{k} is not divisible by {}", + Self::BLCK_SIZE + ); + debug_assert_eq!( + ys.len(), + k / Self::BLCK_SIZE, + "size mismatch {} {} {}", + xs.len(), + ys.len(), + Self::BLCK_SIZE + ); + for (i, ys) in ys.iter_mut().enumerate() { + let mut amax = 0f32; + let xs = &xs[i * Self::BLCK_SIZE..(i + 1) * Self::BLCK_SIZE]; + for &x in xs.iter() { + amax = amax.max(x.abs()) + } + let d = amax / ((1 << 7) - 1) as f32; + let id = if d != 0f32 { 1. / d } else { 0. }; + ys.d = f16::from_f32(d); + for (y, &x) in ys.qs.iter_mut().zip(xs.iter()) { + *y = f32::round(x * id) as i8 + } + } + } + + #[allow(unreachable_code)] + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + #[cfg(target_feature = "avx2")] + return super::avx::vec_dot_q8_0_q8_0(n, xs, ys); + + #[cfg(target_feature = "neon")] + return super::neon::vec_dot_q8_0_q8_0(n, xs, ys); + + #[cfg(target_feature = "simd128")] + return super::simd128::vec_dot_q8_0_q8_0(n, xs, ys); + + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + debug_assert!( + n.is_multiple_of(QK8_0), + "vec_dot_q8_0_q8_0: {n} is not divisible by {QK8_0}" + ); + + // Generic implementation. + let mut sumf = 0f32; + for (xs, ys) in xs.iter().zip(ys.iter()) { + let sum_i = xs + .qs + .iter() + .zip(ys.qs.iter()) + .map(|(&x, &y)| x as i32 * y as i32) + .sum::(); + sumf += sum_i as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d) + } + sumf + } +} + +impl GgmlType for BlockQ8_1 { + const DTYPE: GgmlDType = GgmlDType::Q8_1; + const BLCK_SIZE: usize = QK8_1; + type VecDotType = BlockQ8_1; + + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + debug_assert!( + n.is_multiple_of(QK8_1), + "vec_dot_q8_1_q8_1: {n} is not divisible by {QK8_1}" + ); + + // Generic implementation. + let mut sumf = 0f32; + for (xs, ys) in xs.iter().zip(ys.iter()) { + let sum_i = xs + .qs + .iter() + .zip(ys.qs.iter()) + .map(|(&x, &y)| x as i32 * y as i32) + .sum::(); + sumf += sum_i as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d) + } + sumf + } + + fn from_float(xs: &[f32], ys: &mut [Self]) { + // quantize_row_q8_1 + debug_assert_eq!( + ys.len() * Self::BLCK_SIZE, + xs.len(), + "size mismatch {} {} {}", + xs.len(), + ys.len(), + Self::BLCK_SIZE + ); + for (i, ys) in ys.iter_mut().enumerate() { + let mut amax = 0f32; + let xs = &xs[i * Self::BLCK_SIZE..(i + 1) * Self::BLCK_SIZE]; + for &x in xs.iter() { + amax = amax.max(x.abs()) + } + let d = amax / ((1 << 7) - 1) as f32; + let id = if d != 0f32 { 1. / d } else { 0. }; + ys.d = f16::from_f32(d); + let mut sum = 0i32; + for j in 0..Self::BLCK_SIZE / 2 { + let v0 = xs[j] * id; + let v1 = xs[j + Self::BLCK_SIZE / 2] * id; + ys.qs[j] = f32::round(v0) as i8; + ys.qs[j + Self::BLCK_SIZE / 2] = f32::round(v1) as i8; + sum += ys.qs[j] as i32 + ys.qs[j + Self::BLCK_SIZE / 2] as i32; + } + ys.s = f16::from_f32(sum as f32) * ys.d; + } + } + + fn to_float(_xs: &[Self], _ys: &mut [f32]) { + unimplemented!("no support for vec-dot on Q8_1") + } +} + +impl GgmlType for BlockQ2K { + const DTYPE: GgmlDType = GgmlDType::Q2K; + const BLCK_SIZE: usize = QK_K; + type VecDotType = BlockQ8K; + + #[allow(unreachable_code)] + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + #[cfg(target_feature = "avx2")] + return super::avx::vec_dot_q2k_q8k(n, xs, ys); + + #[cfg(target_feature = "neon")] + return super::neon::vec_dot_q2k_q8k(n, xs, ys); + + #[cfg(target_feature = "simd128")] + return super::simd128::vec_dot_q2k_q8k(n, xs, ys); + + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + debug_assert!( + n.is_multiple_of(QK_K), + "vec_dot_q2k_q8k: {n} is not divisible by {QK_K}" + ); + + let mut sumf = 0.0; + for (x, y) in xs.iter().zip(ys.iter()) { + let mut q2: &[_] = &x.qs; + let mut q8: &[_] = &y.qs; + let sc = &x.scales; + + let mut summs = 0; + for (bsum, scale) in y.bsums.iter().zip(sc) { + summs += *bsum as i32 * ((scale >> 4) as i32); + } + + let dall = y.d * x.d.to_f32(); + let dmin = y.d * x.dmin.to_f32(); + + let mut isum = 0; + let mut is = 0; + for _ in 0..(QK_K / 128) { + let mut shift = 0; + for _ in 0..4 { + let d = (sc[is] & 0xF) as i32; + is += 1; + let mut isuml = 0; + for l in 0..16 { + isuml += q8[l] as i32 * (((q2[l] >> shift) & 3) as i32); + } + isum += d * isuml; + let d = (sc[is] & 0xF) as i32; + is += 1; + isuml = 0; + for l in 16..32 { + isuml += q8[l] as i32 * (((q2[l] >> shift) & 3) as i32); + } + isum += d * isuml; + shift += 2; + // adjust the indexing + q8 = &q8[32..]; + } + // adjust the indexing + q2 = &q2[32..]; + } + sumf += dall * isum as f32 - dmin * summs as f32; + } + + sumf + } + + // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L279 + fn from_float(xs: &[f32], ys: &mut [Self]) { + const Q4SCALE: f32 = 15.0; + + for (block, x) in group_for_quantization(xs, ys) { + //calculate scales and mins + let mut mins: [f32; QK_K / 16] = [0.0; QK_K / 16]; + let mut scales: [f32; QK_K / 16] = [0.0; QK_K / 16]; + + for (j, x_scale_slice) in x.chunks(16).enumerate() { + (scales[j], mins[j]) = make_qkx1_quants(3, 5, x_scale_slice); + } + // get max scale and max min and ensure they are >= 0.0 + let max_scale = scales.iter().fold(0.0, |max, &val| val.max(max)); + let max_min = mins.iter().fold(0.0, |max, &val| val.max(max)); + + if max_scale > 0.0 { + let iscale = Q4SCALE / max_scale; + for (j, scale) in scales.iter().enumerate().take(QK_K / 16) { + block.scales[j] = nearest_int(iscale * scale) as u8; + } + block.d = f16::from_f32(max_scale / Q4SCALE); + } else { + for j in 0..QK_K / 16 { + block.scales[j] = 0; + } + block.d = f16::from_f32(0.0); + } + + if max_min > 0.0 { + let iscale = Q4SCALE / max_min; + for (j, scale) in block.scales.iter_mut().enumerate() { + let l = nearest_int(iscale * mins[j]) as u8; + *scale |= l << 4; + } + block.dmin = f16::from_f32(max_min / Q4SCALE); + } else { + block.dmin = f16::from_f32(0.0); + } + + let mut big_l: [u8; QK_K] = [0; QK_K]; + + for j in 0..QK_K / 16 { + let d = block.d.to_f32() * (block.scales[j] & 0xF) as f32; + if d == 0.0 { + continue; + } + let dm = block.dmin.to_f32() * (block.scales[j] >> 4) as f32; + for ii in 0..16 { + let ll = nearest_int((x[16 * j + ii] + dm) / d).clamp(0, 3); + big_l[16 * j + ii] = ll as u8; + } + } + + for j in (0..QK_K).step_by(128) { + for ll in 0..32 { + block.qs[j / 4 + ll] = big_l[j + ll] + | (big_l[j + ll + 32] << 2) + | (big_l[j + ll + 64] << 4) + | (big_l[j + ll + 96] << 6); + } + } + } + } + + fn from_float_imatrix(xs: &[f32], ys: &mut [Self], imatrix_weights: &[f32], n_per_row: usize) { + for (sblk_idx, (block, x)) in group_for_quantization(xs, ys).into_iter().enumerate() { + let mut mins: [f32; QK_K / 16] = [0.0; QK_K / 16]; + let mut scales: [f32; QK_K / 16] = [0.0; QK_K / 16]; + let mut weights: [f32; 16] = [0.0; 16]; + let mut sw: [f32; QK_K / 16] = [0.0; QK_K / 16]; + let mut ls: [u8; QK_K / 16] = [0; QK_K / 16]; + let mut lm: [u8; QK_K / 16] = [0; QK_K / 16]; + + let sum_x2 = x.iter().map(|x| x * x).sum::(); + let sigma2 = sum_x2 / QK_K as f32; + for (j, x_scale_slice) in x.chunks_exact(16).enumerate() { + for (l, (w_elem, x_elem)) in weights.iter_mut().zip(x_scale_slice).enumerate() { + let imatrix_row = sblk_idx % (n_per_row / QK_K); + let imatrix_w = imatrix_weights[imatrix_row * QK_K + 16 * j + l]; + *w_elem = imatrix_w * (sigma2 + x_elem * x_elem).sqrt(); + } + let sumw = weights.iter().sum::(); + sw[j] = sumw; + (scales[j], mins[j]) = + make_qkx3_quants(3, x_scale_slice, Some(&weights), -0.9, 0.05, 36, false); + } + + let d_block = make_qp_quants(QK_K / 16, 15, &scales, &mut ls, &sw); + let m_block = make_qp_quants(QK_K / 16, 15, &mins, &mut lm, &sw); + + block.d = f16::from_f32(d_block); + block.dmin = f16::from_f32(m_block); + + for j in 0..QK_K / 16 { + block.scales[j] = ls[j] | (lm[j] << 4); + } + + let mut big_l: [u8; QK_K] = [0; QK_K]; + + for j in 0..QK_K / 16 { + let d = block.d.to_f32() * (block.scales[j] & 0xF) as f32; + if d == 0.0 { + continue; + } + let dm = block.dmin.to_f32() * (block.scales[j] >> 4) as f32; + for ii in 0..16 { + let ll = nearest_int((x[16 * j + ii] + dm) / d).clamp(0, 3); + big_l[16 * j + ii] = ll as u8; + } + } + + for j in (0..QK_K).step_by(128) { + for ll in 0..32 { + block.qs[j / 4 + ll] = big_l[j + ll] + | (big_l[j + ll + 32] << 2) + | (big_l[j + ll + 64] << 4) + | (big_l[j + ll + 96] << 6); + } + } + } + } + // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L354 + fn to_float(xs: &[Self], ys: &mut [f32]) { + for (block, y) in group_for_dequantization(xs, ys) { + let d = block.d.to_f32(); + let min = block.dmin.to_f32(); + + let mut is = 0; + + for (y_block, qs) in y.chunks_exact_mut(128).zip(block.qs.chunks_exact(32)) { + // Step by 32 over q. + let mut shift = 0; + let mut y_block_index = 0; + for _j in 0..4 { + let sc = block.scales[is]; + is += 1; + let dl = d * (sc & 0xF) as f32; + let ml = min * (sc >> 4) as f32; + for q in &qs[..16] { + let y = dl * ((q >> shift) & 3) as f32 - ml; + y_block[y_block_index] = y; + y_block_index += 1; + } + + let sc = block.scales[is]; + is += 1; + let dl = d * (sc & 0xF) as f32; + let ml = min * (sc >> 4) as f32; + for q in &qs[16..] { + let y = dl * ((q >> shift) & 3) as f32 - ml; + y_block[y_block_index] = y; + y_block_index += 1; + } + + shift += 2; + } + } + } + } +} + +impl GgmlType for BlockQ3K { + const DTYPE: GgmlDType = GgmlDType::Q3K; + const BLCK_SIZE: usize = QK_K; + type VecDotType = BlockQ8K; + + #[allow(unreachable_code)] + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + #[cfg(target_feature = "avx2")] + return super::avx::vec_dot_q3k_q8k(n, xs, ys); + + #[cfg(target_feature = "neon")] + return super::neon::vec_dot_q3k_q8k(n, xs, ys); + + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + debug_assert!( + n.is_multiple_of(QK_K), + "vec_dot_q3k_q8k: {n} is not divisible by {QK_K}" + ); + + const KMASK1: u32 = 0x03030303; + const KMASK2: u32 = 0x0f0f0f0f; + + let mut aux8: [i8; QK_K] = [0; QK_K]; + let mut aux16: [i16; 8] = [0; 8]; + let mut sums: [f32; 8] = [0.0; 8]; + let mut aux32: [i32; 8] = [0; 8]; + + let mut auxs: [u32; 4] = [0; 4]; + + for (x, y) in xs.iter().zip(ys.iter()) { + let mut q3: &[u8] = &x.qs; + let hmask: &[u8] = &x.hmask; + let mut q8: &[i8] = &y.qs; + + aux32.fill(0); + let mut a = &mut aux8[..]; + + let mut m = 1; + //Like the GGML original this is written this way to enable the compiler to vectorize it. + for _ in 0..QK_K / 128 { + a.iter_mut() + .take(32) + .zip(q3) + .for_each(|(a_val, q3_val)| *a_val = (q3_val & 3) as i8); + a.iter_mut() + .take(32) + .zip(hmask) + .for_each(|(a_val, hmask_val)| { + *a_val -= if hmask_val & m != 0 { 0 } else { 4 } + }); + a = &mut a[32..]; + m <<= 1; + + a.iter_mut() + .take(32) + .zip(q3) + .for_each(|(a_val, q3_val)| *a_val = ((q3_val >> 2) & 3) as i8); + a.iter_mut() + .take(32) + .zip(hmask) + .for_each(|(a_val, hmask_val)| { + *a_val -= if hmask_val & m != 0 { 0 } else { 4 } + }); + a = &mut a[32..]; + m <<= 1; + + a.iter_mut() + .take(32) + .zip(q3) + .for_each(|(a_val, q3_val)| *a_val = ((q3_val >> 4) & 3) as i8); + a.iter_mut() + .take(32) + .zip(hmask) + .for_each(|(a_val, hmask_val)| { + *a_val -= if hmask_val & m != 0 { 0 } else { 4 } + }); + a = &mut a[32..]; + m <<= 1; + + a.iter_mut() + .take(32) + .zip(q3) + .for_each(|(a_val, q3_val)| *a_val = ((q3_val >> 6) & 3) as i8); + a.iter_mut() + .take(32) + .zip(hmask) + .for_each(|(a_val, hmask_val)| { + *a_val -= if hmask_val & m != 0 { 0 } else { 4 } + }); + a = &mut a[32..]; + m <<= 1; + q3 = &q3[32..]; + } + + a = &mut aux8[..]; + + LittleEndian::read_u32_into(&x.scales, &mut auxs[0..3]); + + let tmp = auxs[2]; + auxs[2] = ((auxs[0] >> 4) & KMASK2) | (((tmp >> 4) & KMASK1) << 4); + auxs[3] = ((auxs[1] >> 4) & KMASK2) | (((tmp >> 6) & KMASK1) << 4); + auxs[0] = (auxs[0] & KMASK2) | (((tmp) & KMASK1) << 4); + auxs[1] = (auxs[1] & KMASK2) | (((tmp >> 2) & KMASK1) << 4); + + for aux in auxs { + for scale in aux.to_le_bytes() { + let scale = i8::from_be_bytes([scale]); + for l in 0..8 { + aux16[l] = q8[l] as i16 * a[l] as i16; + } + for l in 0..8 { + aux32[l] += (scale as i32 - 32) * aux16[l] as i32; + } + q8 = &q8[8..]; + a = &mut a[8..]; + + for l in 0..8 { + aux16[l] = q8[l] as i16 * a[l] as i16; + } + for l in 0..8 { + aux32[l] += (scale as i32 - 32) * aux16[l] as i32; + } + q8 = &q8[8..]; + a = &mut a[8..]; + } + } + let d = x.d.to_f32() * y.d; + for l in 0..8 { + sums[l] += d * aux32[l] as f32; + } + } + + sums.iter().sum() + } + + fn from_float(xs: &[f32], ys: &mut [Self]) { + for (block, x) in group_for_quantization(xs, ys) { + let mut scales: [f32; QK_K / 16] = [0.0; QK_K / 16]; + for (j, x_scale_slice) in x.chunks_exact(16).enumerate() { + scales[j] = make_q3_quants(x_scale_slice, 4, true); + } + + // Get max scale by absolute value. + let mut max_scale: f32 = 0.0; + for &scale in scales.iter() { + if scale.abs() > max_scale.abs() { + max_scale = scale; + } + } + + block.scales.fill(0); + + if max_scale != 0.0 { + let iscale = -32.0 / max_scale; + for (j, scale) in scales.iter().enumerate() { + let l_val = nearest_int(iscale * scale); + let l_val = l_val.clamp(-32, 31) + 32; + if j < 8 { + block.scales[j] = (l_val & 0xF) as u8; + } else { + block.scales[j - 8] |= ((l_val & 0xF) << 4) as u8; + } + let l_val = l_val >> 4; + block.scales[j % 4 + 8] |= (l_val << (2 * (j / 4))) as u8; + } + block.d = f16::from_f32(1.0 / iscale); + } else { + block.d = f16::from_f32(0.0); + } + + let mut l: [i8; QK_K] = [0; QK_K]; + + for j in 0..QK_K / 16 { + let sc = if j < 8 { + block.scales[j] & 0xF + } else { + block.scales[j - 8] >> 4 + }; + let sc = (sc | (((block.scales[8 + j % 4] >> (2 * (j / 4))) & 3) << 4)) as i8 - 32; + let d = block.d.to_f32() * sc as f32; + if d != 0.0 { + for ii in 0..16 { + let l_val = nearest_int(x[16 * j + ii] / d); + l[16 * j + ii] = (l_val.clamp(-4, 3) + 4) as i8; + } + } + } + + block.hmask.fill(0); + let mut m = 0; + let mut hm = 1; + + for ll in l.iter_mut() { + if *ll > 3 { + block.hmask[m] |= hm; + *ll -= 4; + } + m += 1; + if m == QK_K / 8 { + m = 0; + hm <<= 1; + } + } + + for j in (0..QK_K).step_by(128) { + for l_val in 0..32 { + block.qs[j / 4 + l_val] = (l[j + l_val] + | (l[j + l_val + 32] << 2) + | (l[j + l_val + 64] << 4) + | (l[j + l_val + 96] << 6)) + as u8; + } + } + } + } + + fn from_float_imatrix(xs: &[f32], ys: &mut [Self], imatrix_weights: &[f32], n_per_row: usize) { + for (sblk_idx, (block, x)) in group_for_quantization(xs, ys).into_iter().enumerate() { + let mut scales: [f32; QK_K / 16] = [0.0; QK_K / 16]; + let mut weights: [f32; 16] = [0.0; 16]; + let mut sw: [f32; QK_K / 16] = [0.0; QK_K / 16]; + let mut ls: [i8; QK_K / 16] = [0; QK_K / 16]; + let mut l: [i8; QK_K] = [0; QK_K]; + + let sum_x2 = x.iter().map(|x| x * x).sum::(); + let sigma2 = 2. * sum_x2 / QK_K as f32; + + for (j, x_scale_slice) in x.chunks_exact(16).enumerate() { + for (l_idx, (w_elem, x_elem)) in weights.iter_mut().zip(x_scale_slice).enumerate() { + let imatrix_row = sblk_idx % (n_per_row / QK_K); + let imatrix_w = imatrix_weights[imatrix_row * QK_K + 16 * j + l_idx]; + *w_elem = imatrix_w * (sigma2 + x_elem * x_elem).sqrt(); + } + let sumw = weights.iter().sum::(); + sw[j] = sumw; + scales[j] = unsafe { + make_qx_quants( + 16, + 4, + x_scale_slice.as_ptr(), + l.as_mut_ptr().add(16 * j), + 1, + weights.as_ptr(), + ) + }; + } + + block.scales.fill(0); + let d_block = unsafe { + make_qx_quants( + QK_K / 16, + 32, + scales.as_ptr(), + ls.as_mut_ptr(), + 1, + sw.as_ptr(), + ) + }; + block.d = f16::from_f32(d_block); + for (j, l_val) in ls.iter().enumerate().take(QK_K / 16) { + if j < 8 { + block.scales[j] = (l_val & 0xF) as u8; + } else { + block.scales[j - 8] |= ((l_val & 0xF) << 4) as u8; + } + let l_val = l_val >> 4; + block.scales[j % 4 + 8] |= (l_val << (2 * (j / 4))) as u8; + } + + for j in 0..QK_K / 16 { + let sc = if j < 8 { + block.scales[j] & 0xF + } else { + block.scales[j - 8] >> 4 + }; + let sc = (sc | (((block.scales[8 + j % 4] >> (2 * (j / 4))) & 3) << 4)) as i8 - 32; + let d = block.d.to_f32() * sc as f32; + if d != 0.0 { + for ii in 0..16 { + let l_val = nearest_int(x[16 * j + ii] / d); + l[16 * j + ii] = (l_val.clamp(-4, 3) + 4) as i8; + } + } + } + + block.hmask.fill(0); + let mut m = 0; + let mut hm = 1; + + for ll in l.iter_mut() { + if *ll > 3 { + block.hmask[m] |= hm; + *ll -= 4; + } + m += 1; + if m == QK_K / 8 { + m = 0; + hm <<= 1; + } + } + + for j in (0..QK_K).step_by(128) { + for l_val in 0..32 { + block.qs[j / 4 + l_val] = (l[j + l_val] + | (l[j + l_val + 32] << 2) + | (l[j + l_val + 64] << 4) + | (l[j + l_val + 96] << 6)) + as u8; + } + } + } + } + + // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L533 + fn to_float(xs: &[Self], ys: &mut [f32]) { + const KMASK1: u32 = 0x03030303; + const KMASK2: u32 = 0x0f0f0f0f; + + for (block, y) in group_for_dequantization(xs, ys) { + //Reconstruct the scales + let mut aux = [0; 4]; + LittleEndian::read_u32_into(&block.scales, &mut aux[0..3]); + + let tmp = aux[2]; + aux[2] = ((aux[0] >> 4) & KMASK2) | (((tmp >> 4) & KMASK1) << 4); + aux[3] = ((aux[1] >> 4) & KMASK2) | (((tmp >> 6) & KMASK1) << 4); + aux[0] = (aux[0] & KMASK2) | (((tmp) & KMASK1) << 4); + aux[1] = (aux[1] & KMASK2) | (((tmp >> 2) & KMASK1) << 4); + + //Transfer the scales into an i8 array + let scales: &mut [i8] = + unsafe { std::slice::from_raw_parts_mut(aux.as_mut_ptr() as *mut i8, 16) }; + + let d_all = block.d.to_f32(); + let mut m = 1; + let mut is = 0; + + // Dequantize both 128 long blocks + // 32 qs values per 128 long block + // Each 16 elements get a scale + for (y, qs) in y.chunks_exact_mut(128).zip(block.qs.chunks_exact(32)) { + let mut shift = 0; + for shift_scoped_y in y.chunks_exact_mut(32) { + for (scale_index, scale_scoped_y) in + shift_scoped_y.chunks_exact_mut(16).enumerate() + { + let dl = d_all * (scales[is] as f32 - 32.0); + for (i, inner_y) in scale_scoped_y.iter_mut().enumerate() { + let new_y = dl + * (((qs[i + 16 * scale_index] >> shift) & 3) as i8 + - if (block.hmask[i + 16 * scale_index] & m) == 0 { + 4 + } else { + 0 + }) as f32; + *inner_y = new_y; + } + // 16 block finished => advance scale index + is += 1; + } + // 32 block finished => increase shift and m + shift += 2; + m <<= 1; + } + } + } + } +} + +impl GgmlType for BlockQ4K { + const DTYPE: GgmlDType = GgmlDType::Q4K; + const BLCK_SIZE: usize = QK_K; + type VecDotType = BlockQ8K; + + #[allow(unreachable_code)] + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + #[cfg(target_feature = "avx2")] + return super::avx::vec_dot_q4k_q8k(n, xs, ys); + + #[cfg(target_feature = "neon")] + return super::neon::vec_dot_q4k_q8k(n, xs, ys); + + #[cfg(target_feature = "simd128")] + return super::simd128::vec_dot_q4k_q8k(n, xs, ys); + + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + debug_assert!( + n.is_multiple_of(QK_K), + "vec_dot_q4k_q8k: {n} is not divisible by {QK_K}" + ); + + const KMASK1: u32 = 0x3f3f3f3f; + const KMASK2: u32 = 0x0f0f0f0f; + const KMASK3: u32 = 0x03030303; + + let mut utmp: [u32; 4] = [0; 4]; + let mut scales: [u8; 8] = [0; 8]; + let mut mins: [u8; 8] = [0; 8]; + + let mut aux8: [i8; QK_K] = [0; QK_K]; + let mut aux16: [i16; 8] = [0; 8]; + let mut sums: [f32; 8] = [0.0; 8]; + let mut aux32: [i32; 8] = [0; 8]; + + let mut sumf = 0.0; + for (y, x) in ys.iter().zip(xs.iter()) { + let q4 = &x.qs; + let q8 = &y.qs; + aux32.fill(0); + + let mut a = &mut aux8[..]; + let mut q4 = &q4[..]; + for _ in 0..QK_K / 64 { + for l in 0..32 { + a[l] = (q4[l] & 0xF) as i8; + } + a = &mut a[32..]; + for l in 0..32 { + a[l] = (q4[l] >> 4) as i8; + } + a = &mut a[32..]; + q4 = &q4[32..]; + } + + LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); + + utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); + let uaux = utmp[1] & KMASK1; + utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); + utmp[2] = uaux; + utmp[0] &= KMASK1; + + //extract scales and mins + LittleEndian::write_u32_into(&utmp[0..2], &mut scales); + LittleEndian::write_u32_into(&utmp[2..4], &mut mins); + + let mut sumi = 0; + for j in 0..QK_K / 16 { + sumi += y.bsums[j] as i32 * mins[j / 2] as i32; + } + + let mut a = &mut aux8[..]; + let mut q8 = &q8[..]; + + for scale in scales { + let scale = scale as i32; + for _ in 0..4 { + for l in 0..8 { + aux16[l] = q8[l] as i16 * a[l] as i16; + } + for l in 0..8 { + aux32[l] += scale * aux16[l] as i32; + } + q8 = &q8[8..]; + a = &mut a[8..]; + } + } + let d = x.d.to_f32() * y.d; + for l in 0..8 { + sums[l] += d * aux32[l] as f32; + } + let dmin = x.dmin.to_f32() * y.d; + sumf -= dmin * sumi as f32; + } + sumf + sums.iter().sum::() + } + + fn from_float(xs: &[f32], ys: &mut [Self]) { + for (block, x) in group_for_quantization(xs, ys) { + let mut mins: [f32; QK_K / 32] = [0.0; QK_K / 32]; + let mut scales: [f32; QK_K / 32] = [0.0; QK_K / 32]; + + for (j, x_scale_slice) in x.chunks_exact(32).enumerate() { + (scales[j], mins[j]) = make_qkx1_quants(15, 5, x_scale_slice); + } + + // get max scale and max min and ensure they are >= 0.0 + let max_scale = scales.iter().fold(0.0, |max, &val| val.max(max)); + let max_min = mins.iter().fold(0.0, |max, &val| val.max(max)); + + let inv_scale = if max_scale > 0.0 { + 63.0 / max_scale + } else { + 0.0 + }; + let inv_min = if max_min > 0.0 { 63.0 / max_min } else { 0.0 }; + + for j in 0..QK_K / 32 { + let ls = nearest_int(inv_scale * scales[j]).min(63) as u8; + let lm = nearest_int(inv_min * mins[j]).min(63) as u8; + if j < 4 { + block.scales[j] = ls; + block.scales[j + 4] = lm; + } else { + block.scales[j + 4] = (ls & 0xF) | ((lm & 0xF) << 4); + block.scales[j - 4] |= (ls >> 4) << 6; + block.scales[j] |= (lm >> 4) << 6; + } + } + + block.d = f16::from_f32(max_scale / 63.0); + block.dmin = f16::from_f32(max_min / 63.0); + + let mut l: [u8; QK_K] = [0; QK_K]; + + for j in 0..QK_K / 32 { + let (sc, m) = get_scale_min_k4(j, &block.scales); + let d = block.d.to_f32() * sc as f32; + if d != 0.0 { + let dm = block.dmin.to_f32() * m as f32; + for ii in 0..32 { + let l_val = nearest_int((x[32 * j + ii] + dm) / d); + l[32 * j + ii] = l_val.clamp(0, 15) as u8; + } + } + } + + let q = &mut block.qs; + for j in (0..QK_K).step_by(64) { + for l_val in 0..32 { + let offset_index = (j / 64) * 32 + l_val; + q[offset_index] = l[j + l_val] | (l[j + l_val + 32] << 4); + } + } + } + } + + fn from_float_imatrix(xs: &[f32], ys: &mut [Self], imatrix_weights: &[f32], n_per_row: usize) { + for (sblk_idx, (block, x)) in group_for_quantization(xs, ys).into_iter().enumerate() { + let mut mins: [f32; QK_K / 32] = [0.0; QK_K / 32]; + let mut scales: [f32; QK_K / 32] = [0.0; QK_K / 32]; + let mut weights: [f32; 32] = [0.0; 32]; + let mut sw: [f32; QK_K / 32] = [0.0; QK_K / 32]; + let mut ls: [u8; QK_K / 32] = [0; QK_K / 32]; + let mut lm: [u8; QK_K / 32] = [0; QK_K / 32]; + + let sum_x2 = x.iter().map(|x| x * x).sum::(); + let sigma2 = 2. * sum_x2 / QK_K as f32; + + for (j, x_scale_slice) in x.chunks_exact(32).enumerate() { + for (l, (w_elem, x_elem)) in weights.iter_mut().zip(x_scale_slice).enumerate() { + let imatrix_row = sblk_idx % (n_per_row / QK_K); + let imatrix_w = imatrix_weights[imatrix_row * QK_K + 32 * j + l]; + *w_elem = imatrix_w * (sigma2 + x_elem * x_elem).sqrt(); + } + let sumw = weights.iter().sum::(); + sw[j] = sumw; + (scales[j], mins[j]) = + make_qkx3_quants(15, x_scale_slice, Some(&weights), -0.9, 0.05, 36, false); + } + + let d_block = make_qp_quants(QK_K / 32, 63, &scales, &mut ls, &sw); + let m_block = make_qp_quants(QK_K / 32, 63, &mins, &mut lm, &sw); + for j in 0..QK_K / 32 { + let ls_val = ls[j]; + let lm_val = lm[j]; + if j < 4 { + block.scales[j] = ls_val; + block.scales[j + 4] = lm_val; + } else { + block.scales[j + 4] = (ls_val & 0xF) | ((lm_val & 0xF) << 4); + block.scales[j - 4] |= (ls_val >> 4) << 6; + block.scales[j] |= (lm_val >> 4) << 6; + } + } + + block.d = f16::from_f32(d_block); + block.dmin = f16::from_f32(m_block); + + let mut l: [u8; QK_K] = [0; QK_K]; + for j in 0..QK_K / 32 { + let (sc, m) = get_scale_min_k4(j, &block.scales); + let d = block.d.to_f32() * sc as f32; + if d != 0.0 { + let dm = block.dmin.to_f32() * m as f32; + for ii in 0..32 { + let l_val = nearest_int((x[32 * j + ii] + dm) / d); + l[32 * j + ii] = l_val.clamp(0, 15) as u8; + } + } + } + + let q = &mut block.qs; + for j in (0..QK_K).step_by(64) { + for l_val in 0..32 { + let offset_index = (j / 64) * 32 + l_val; + q[offset_index] = l[j + l_val] | (l[j + l_val + 32] << 4); + } + } + } + } + // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L735 + fn to_float(xs: &[Self], ys: &mut [f32]) { + for (block, y) in group_for_dequantization(xs, ys) { + let d = block.d.to_f32(); + let min = block.dmin.to_f32(); + let q = &block.qs; + let mut is = 0; + let mut ys_index = 0; + + for j in (0..QK_K).step_by(64) { + let q = &q[j / 2..j / 2 + 32]; + let (sc, m) = get_scale_min_k4(is, &block.scales); + let d1 = d * sc as f32; + let m1 = min * m as f32; + let (sc, m) = get_scale_min_k4(is + 1, &block.scales); + let d2 = d * sc as f32; + let m2 = min * m as f32; + for q in q { + y[ys_index] = d1 * (q & 0xF) as f32 - m1; + ys_index += 1; + } + for q in q { + y[ys_index] = d2 * (q >> 4) as f32 - m2; + ys_index += 1; + } + is += 2; + } + } + } +} + +// https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L928 +impl GgmlType for BlockQ5K { + const DTYPE: GgmlDType = GgmlDType::Q5K; + const BLCK_SIZE: usize = QK_K; + type VecDotType = BlockQ8K; + + #[allow(unreachable_code)] + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + #[cfg(target_feature = "avx2")] + return super::avx::vec_dot_q5k_q8k(n, xs, ys); + + #[cfg(target_feature = "neon")] + return super::neon::vec_dot_q5k_q8k(n, xs, ys); + + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + debug_assert!( + n.is_multiple_of(QK_K), + "vec_dot_q5k_q8k: {n} is not divisible by {QK_K}" + ); + + const KMASK1: u32 = 0x3f3f3f3f; + const KMASK2: u32 = 0x0f0f0f0f; + const KMASK3: u32 = 0x03030303; + + let mut utmp: [u32; 4] = [0; 4]; + let mut scales: [u8; 8] = [0; 8]; + let mut mins: [u8; 8] = [0; 8]; + + let mut aux8: [i8; QK_K] = [0; QK_K]; + let mut aux16: [i16; 8] = [0; 8]; + let mut sums: [f32; 8] = [0.0; 8]; + let mut aux32: [i32; 8] = [0; 8]; + + let mut sumf = 0.0; + for (y, x) in ys.iter().zip(xs.iter()) { + let q5 = &x.qs; + let hm = &x.qh; + let q8 = &y.qs; + aux32.fill(0); + + let mut a = &mut aux8[..]; + let mut q5 = &q5[..]; + let mut m = 1u8; + + for _ in 0..QK_K / 64 { + for l in 0..32 { + a[l] = (q5[l] & 0xF) as i8; + a[l] += if hm[l] & m != 0 { 16 } else { 0 }; + } + a = &mut a[32..]; + m <<= 1; + for l in 0..32 { + a[l] = (q5[l] >> 4) as i8; + a[l] += if hm[l] & m != 0 { 16 } else { 0 }; + } + a = &mut a[32..]; + m <<= 1; + q5 = &q5[32..]; + } + + LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); + + utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); + let uaux = utmp[1] & KMASK1; + utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); + utmp[2] = uaux; + utmp[0] &= KMASK1; + + //extract scales and mins + LittleEndian::write_u32_into(&utmp[0..2], &mut scales); + LittleEndian::write_u32_into(&utmp[2..4], &mut mins); + + let mut sumi = 0; + for j in 0..QK_K / 16 { + sumi += y.bsums[j] as i32 * mins[j / 2] as i32; + } + + let mut a = &mut aux8[..]; + let mut q8 = &q8[..]; + + for scale in scales { + let scale = scale as i32; + for _ in 0..4 { + for l in 0..8 { + aux16[l] = q8[l] as i16 * a[l] as i16; + } + for l in 0..8 { + aux32[l] += scale * aux16[l] as i32; + } + q8 = &q8[8..]; + a = &mut a[8..]; + } + } + let d = x.d.to_f32() * y.d; + for l in 0..8 { + sums[l] += d * aux32[l] as f32; + } + let dmin = x.dmin.to_f32() * y.d; + sumf -= dmin * sumi as f32; + } + sumf + sums.iter().sum::() + } + + // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L793 + fn from_float(xs: &[f32], ys: &mut [Self]) { + for (block, x) in group_for_quantization(xs, ys) { + let mut mins: [f32; QK_K / 32] = [0.0; QK_K / 32]; + let mut scales: [f32; QK_K / 32] = [0.0; QK_K / 32]; + + for (j, x_scale_slice) in x.chunks_exact(32).enumerate() { + (scales[j], mins[j]) = make_qkx1_quants(31, 5, x_scale_slice); + } + + // get max scale and max min and ensure they are >= 0.0 + let max_scale = scales.iter().fold(0.0, |max, &val| val.max(max)); + let max_min = mins.iter().fold(0.0, |max, &val| val.max(max)); + + let inv_scale = if max_scale > 0.0 { + 63.0 / max_scale + } else { + 0.0 + }; + let inv_min = if max_min > 0.0 { 63.0 / max_min } else { 0.0 }; + for j in 0..QK_K / 32 { + let ls = nearest_int(inv_scale * scales[j]).min(63) as u8; + let lm = nearest_int(inv_min * mins[j]).min(63) as u8; + if j < 4 { + block.scales[j] = ls; + block.scales[j + 4] = lm; + } else { + block.scales[j + 4] = (ls & 0xF) | ((lm & 0xF) << 4); + block.scales[j - 4] |= (ls >> 4) << 6; + block.scales[j] |= (lm >> 4) << 6; + } + } + block.d = f16::from_f32(max_scale / 63.0); + block.dmin = f16::from_f32(max_min / 63.0); + + let mut l: [u8; QK_K] = [0; QK_K]; + for j in 0..QK_K / 32 { + let (sc, m) = get_scale_min_k4(j, &block.scales); + let d = block.d.to_f32() * sc as f32; + if d == 0.0 { + continue; + } + let dm = block.dmin.to_f32() * m as f32; + for ii in 0..32 { + let ll = nearest_int((x[32 * j + ii] + dm) / d); + l[32 * j + ii] = ll.clamp(0, 31) as u8; + } + } + + let qh = &mut block.qh; + let ql = &mut block.qs; + qh.fill(0); + + let mut m1 = 1; + let mut m2 = 2; + for n in (0..QK_K).step_by(64) { + let offset = (n / 64) * 32; + for j in 0..32 { + let mut l1 = l[n + j]; + if l1 > 15 { + l1 -= 16; + qh[j] |= m1; + } + let mut l2 = l[n + j + 32]; + if l2 > 15 { + l2 -= 16; + qh[j] |= m2; + } + ql[offset + j] = l1 | (l2 << 4); + } + m1 <<= 2; + m2 <<= 2; + } + } + } + + fn from_float_imatrix(xs: &[f32], ys: &mut [Self], imatrix_weights: &[f32], n_per_row: usize) { + for (sblk_idx, (block, x)) in group_for_quantization(xs, ys).into_iter().enumerate() { + let mut mins: [f32; QK_K / 32] = [0.0; QK_K / 32]; + let mut scales: [f32; QK_K / 32] = [0.0; QK_K / 32]; + let mut weights: [f32; 32] = [0.0; 32]; + let mut sw: [f32; QK_K / 32] = [0.0; QK_K / 32]; + let mut ls: [u8; QK_K / 32] = [0; QK_K / 32]; + let mut lm: [u8; QK_K / 32] = [0; QK_K / 32]; + + let sum_x2 = x.iter().map(|x| x * x).sum::(); + let sigma2 = 2. * sum_x2 / QK_K as f32; + + for (j, x_scale_slice) in x.chunks_exact(32).enumerate() { + for (l, (w_elem, x_elem)) in weights.iter_mut().zip(x_scale_slice).enumerate() { + let imatrix_row = sblk_idx % (n_per_row / QK_K); + let imatrix_w = imatrix_weights[imatrix_row * QK_K + 32 * j + l]; + *w_elem = imatrix_w * (sigma2 + x_elem * x_elem).sqrt(); + } + let sumw = weights.iter().sum::(); + sw[j] = sumw; + (scales[j], mins[j]) = + make_qkx3_quants(31, x_scale_slice, Some(&weights), -0.9, 0.05, 36, false); + } + + let d_block = make_qp_quants(QK_K / 32, 63, &scales, &mut ls, &sw); + let m_block = make_qp_quants(QK_K / 32, 63, &mins, &mut lm, &sw); + for j in 0..QK_K / 32 { + let ls_val = ls[j].min(63); + let lm_val = lm[j].min(63); + if j < 4 { + block.scales[j] = ls_val; + block.scales[j + 4] = lm_val; + } else { + block.scales[j + 4] = (ls_val & 0xF) | ((lm_val & 0xF) << 4); + block.scales[j - 4] |= (ls_val >> 4) << 6; + block.scales[j] |= (lm_val >> 4) << 6; + } + } + + block.d = f16::from_f32(d_block); + block.dmin = f16::from_f32(m_block); + + let mut l: [u8; QK_K] = [0; QK_K]; + for j in 0..QK_K / 32 { + let (sc, m) = get_scale_min_k4(j, &block.scales); + let d = block.d.to_f32() * sc as f32; + if d != 0.0 { + let dm = block.dmin.to_f32() * m as f32; + for ii in 0..32 { + let l_val = nearest_int((x[32 * j + ii] + dm) / d); + l[32 * j + ii] = l_val.clamp(0, 31) as u8; + } + } + } + + let qh = &mut block.qh; + let ql = &mut block.qs; + qh.fill(0); + + let mut m1 = 1; + let mut m2 = 2; + for n in (0..QK_K).step_by(64) { + let offset = (n / 64) * 32; + for j in 0..32 { + let mut l1 = l[n + j]; + if l1 > 15 { + l1 -= 16; + qh[j] |= m1; + } + let mut l2 = l[n + j + 32]; + if l2 > 15 { + l2 -= 16; + qh[j] |= m2; + } + ql[offset + j] = l1 | (l2 << 4); + } + m1 <<= 2; + m2 <<= 2; + } + } + } + + // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L928 + fn to_float(xs: &[Self], ys: &mut [f32]) { + for (block, y) in group_for_dequantization(xs, ys) { + let d = block.d.to_f32(); + let min = block.dmin.to_f32(); + let ql = &block.qs; + let qh = &block.qh; + let mut is = 0; + let mut u1 = 1; + let mut u2 = 2; + let mut ys_index = 0; + + for j in (0..QK_K).step_by(64) { + let ql = &ql[j / 2..j / 2 + 32]; + let (sc, m) = get_scale_min_k4(is, &block.scales); + let d1 = d * sc as f32; + let m1 = min * m as f32; + let (sc, m) = get_scale_min_k4(is + 1, &block.scales); + let d2 = d * sc as f32; + let m2 = min * m as f32; + for (ql, qh) in ql.iter().zip(qh) { + let to_add = if qh & u1 != 0 { 16f32 } else { 0f32 }; + y[ys_index] = d1 * ((ql & 0xF) as f32 + to_add) - m1; + ys_index += 1; + } + for (ql, qh) in ql.iter().zip(qh) { + let to_add = if qh & u2 != 0 { 16f32 } else { 0f32 }; + y[ys_index] = d2 * ((ql >> 4) as f32 + to_add) - m2; + ys_index += 1; + } + is += 2; + u1 <<= 2; + u2 <<= 2; + } + } + } +} + +impl GgmlType for BlockQ6K { + const DTYPE: GgmlDType = GgmlDType::Q6K; + const BLCK_SIZE: usize = QK_K; + type VecDotType = BlockQ8K; + + #[allow(unreachable_code)] + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + #[cfg(target_feature = "avx2")] + return super::avx::vec_dot_q6k_q8k(n, xs, ys); + + #[cfg(target_feature = "neon")] + return super::neon::vec_dot_q6k_q8k(n, xs, ys); + + #[cfg(target_feature = "simd128")] + return super::simd128::vec_dot_q6k_q8k(n, xs, ys); + + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + debug_assert!( + n.is_multiple_of(QK_K), + "vec_dot_q6k_q8k: {n} is not divisible by {QK_K}" + ); + + let mut aux8 = [0i8; QK_K]; + let mut aux16 = [0i16; 8]; + let mut sums = [0f32; 8]; + let mut aux32 = [0f32; 8]; + + for (x, y) in xs.iter().zip(ys.iter()) { + let q4 = &x.ql; + let qh = &x.qh; + let q8 = &y.qs; + aux32.fill(0f32); + + for j in (0..QK_K).step_by(128) { + let aux8 = &mut aux8[j..]; + let q4 = &q4[j / 2..]; + let qh = &qh[j / 4..]; + for l in 0..32 { + aux8[l] = (((q4[l] & 0xF) | ((qh[l] & 3) << 4)) as i32 - 32) as i8; + aux8[l + 32] = + (((q4[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) as i32 - 32) as i8; + aux8[l + 64] = (((q4[l] >> 4) | (((qh[l] >> 4) & 3) << 4)) as i32 - 32) as i8; + aux8[l + 96] = + (((q4[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) as i32 - 32) as i8; + } + } + + for (j, &scale) in x.scales.iter().enumerate() { + let scale = scale as f32; + let q8 = &q8[16 * j..]; + let aux8 = &aux8[16 * j..]; + for l in 0..8 { + aux16[l] = q8[l] as i16 * aux8[l] as i16; + } + for l in 0..8 { + aux32[l] += scale * aux16[l] as f32 + } + let q8 = &q8[8..]; + let aux8 = &aux8[8..]; + for l in 0..8 { + aux16[l] = q8[l] as i16 * aux8[l] as i16; + } + for l in 0..8 { + aux32[l] += scale * aux16[l] as f32 + } + } + + let d = x.d.to_f32() * y.d; + for (sum, &a) in sums.iter_mut().zip(aux32.iter()) { + *sum += a * d; + } + } + sums.iter().sum() + } + + fn from_float(xs: &[f32], ys: &mut [Self]) { + debug_assert_eq!( + xs.len(), + ys.len() * Self::BLCK_SIZE, + "quantize_row_q6k: size mismatch {} {} {}", + xs.len(), + ys.len(), + Self::BLCK_SIZE + ); + let mut l = [0i8; QK_K]; + let mut scales = [0f32; QK_K / 16]; + let mut x = xs.as_ptr(); + let l = l.as_mut_ptr(); + unsafe { + for y in ys.iter_mut() { + let mut max_scale = 0f32; + let mut max_abs_scale = 0f32; + for (ib, scale_) in scales.iter_mut().enumerate() { + let scale = + make_qx_quants(16, 32, x.add(16 * ib), l.add(16 * ib), 1, std::ptr::null()); + *scale_ = scale; + let abs_scale = scale.abs(); + if abs_scale > max_abs_scale { + max_abs_scale = abs_scale; + max_scale = scale + } + } + + let iscale = -128f32 / max_scale; + y.d = f16::from_f32(1.0 / iscale); + + for (y_scale, scale) in y.scales.iter_mut().zip(scales.iter()) { + *y_scale = nearest_int(iscale * scale).min(127) as i8 + } + + for (j, &y_scale) in y.scales.iter().enumerate() { + let d = y.d.to_f32() * y_scale as f32; + if d == 0. { + continue; + } + for ii in 0..16 { + let ll = nearest_int(*x.add(16 * j + ii) / d).clamp(-32, 31); + *l.add(16 * j + ii) = (ll + 32) as i8 + } + } + + let mut ql = y.ql.as_mut_ptr(); + let mut qh = y.qh.as_mut_ptr(); + + for j in (0..QK_K).step_by(128) { + for l_idx in 0..32 { + let q1 = *l.add(j + l_idx) & 0xF; + let q2 = *l.add(j + l_idx + 32) & 0xF; + let q3 = *l.add(j + l_idx + 64) & 0xF; + let q4 = *l.add(j + l_idx + 96) & 0xF; + *ql.add(l_idx) = (q1 | (q3 << 4)) as u8; + *ql.add(l_idx + 32) = (q2 | (q4 << 4)) as u8; + *qh.add(l_idx) = ((*l.add(j + l_idx) >> 4) + | ((*l.add(j + l_idx + 32) >> 4) << 2) + | ((*l.add(j + l_idx + 64) >> 4) << 4) + | ((*l.add(j + l_idx + 96) >> 4) << 6)) + as u8; + } + ql = ql.add(64); + qh = qh.add(32); + } + + x = x.add(QK_K) + } + } + } + + fn from_float_imatrix(xs: &[f32], ys: &mut [Self], imatrix_weights: &[f32], n_per_row: usize) { + debug_assert_eq!( + xs.len(), + ys.len() * Self::BLCK_SIZE, + "quantize_row_q6k imatrix: size mismatch {} {} {}", + xs.len(), + ys.len(), + Self::BLCK_SIZE + ); + let mut l = [0i8; QK_K]; + let mut scales = [0f32; QK_K / 16]; + let mut x = xs.as_ptr(); + let imatrix_weights = imatrix_weights.as_ptr(); + let l = l.as_mut_ptr(); + unsafe { + for (sblk_idx, y) in ys.iter_mut().enumerate() { + let mut max_scale = 0f32; + let mut max_abs_scale = 0f32; + for (ib, scale_) in scales.iter_mut().enumerate() { + let imatrix_row = sblk_idx % (n_per_row / QK_K); + let scale = make_qx_quants( + 16, + 32, + x.add(16 * ib), + l.add(16 * ib), + 1, + imatrix_weights.add(QK_K * imatrix_row + 16 * ib), + ); + *scale_ = scale; + let abs_scale = scale.abs(); + if abs_scale > max_abs_scale { + max_abs_scale = abs_scale; + max_scale = scale + } + } + + let iscale = -128f32 / max_scale; + y.d = f16::from_f32(1.0 / iscale); + + for (y_scale, scale) in y.scales.iter_mut().zip(scales.iter()) { + *y_scale = nearest_int(iscale * scale).min(127) as i8 + } + + for (j, &y_scale) in y.scales.iter().enumerate() { + let d = y.d.to_f32() * y_scale as f32; + if d == 0. { + continue; + } + for ii in 0..16 { + let ll = nearest_int(*x.add(16 * j + ii) / d).clamp(-32, 31); + *l.add(16 * j + ii) = (ll + 32) as i8 + } + } + + let mut ql = y.ql.as_mut_ptr(); + let mut qh = y.qh.as_mut_ptr(); + + for j in (0..QK_K).step_by(128) { + for l_idx in 0..32 { + let q1 = *l.add(j + l_idx) & 0xF; + let q2 = *l.add(j + l_idx + 32) & 0xF; + let q3 = *l.add(j + l_idx + 64) & 0xF; + let q4 = *l.add(j + l_idx + 96) & 0xF; + *ql.add(l_idx) = (q1 | (q3 << 4)) as u8; + *ql.add(l_idx + 32) = (q2 | (q4 << 4)) as u8; + *qh.add(l_idx) = ((*l.add(j + l_idx) >> 4) + | ((*l.add(j + l_idx + 32) >> 4) << 2) + | ((*l.add(j + l_idx + 64) >> 4) << 4) + | ((*l.add(j + l_idx + 96) >> 4) << 6)) + as u8; + } + ql = ql.add(64); + qh = qh.add(32); + } + + x = x.add(QK_K) + } + } + } + + // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L1067 + fn to_float(xs: &[Self], ys: &mut [f32]) { + let k = ys.len(); + debug_assert!( + k.is_multiple_of(QK_K), + "dequantize_row_q6k: {k} is not divisible by {QK_K}" + ); + + for (idx_x, x) in xs.iter().enumerate() { + let d = x.d.to_f32(); + let ql = &x.ql; + let qh = &x.qh; + let sc = &x.scales; + for n in (0..QK_K).step_by(128) { + let idx = n / 128; + let ys = &mut ys[idx_x * QK_K + n..]; + let sc = &sc[8 * idx..]; + let ql = &ql[64 * idx..]; + let qh = &qh[32 * idx..]; + for l in 0..32 { + let is = l / 16; + let q1 = ((ql[l] & 0xF) | ((qh[l] & 3) << 4)) as i8 - 32; + let q2 = ((ql[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) as i8 - 32; + let q3 = ((ql[l] >> 4) | (((qh[l] >> 4) & 3) << 4)) as i8 - 32; + let q4 = ((ql[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) as i8 - 32; + ys[l] = d * sc[is] as f32 * q1 as f32; + ys[l + 32] = d * sc[is + 2] as f32 * q2 as f32; + ys[l + 64] = d * sc[is + 4] as f32 * q3 as f32; + ys[l + 96] = d * sc[is + 6] as f32 * q4 as f32; + } + } + } + } +} + +impl GgmlType for BlockQ8K { + const DTYPE: GgmlDType = GgmlDType::Q8K; + const BLCK_SIZE: usize = QK_K; + type VecDotType = BlockQ8K; + + #[allow(unreachable_code)] + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + #[cfg(target_feature = "avx2")] + return super::avx::vec_dot_q8k_q8k(n, xs, ys); + + #[cfg(target_feature = "neon")] + return super::neon::vec_dot_q8k_q8k(n, xs, ys); + + #[cfg(target_feature = "simd128")] + return super::simd128::vec_dot_q8k_q8k(n, xs, ys); + + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + debug_assert!( + n.is_multiple_of(QK_K), + "vec_dot_q8k_q8k: {n} is not divisible by {QK_K}" + ); + // Generic implementation. + let mut sumf = 0f32; + for (xs, ys) in xs.iter().zip(ys.iter()) { + let sum_i = xs + .qs + .iter() + .zip(ys.qs.iter()) + .map(|(&x, &y)| x as i32 * y as i32) + .sum::(); + sumf += sum_i as f32 * xs.d * ys.d + } + sumf + } + + fn from_float(xs: &[f32], ys: &mut [Self]) { + let k = xs.len(); + debug_assert!( + k.is_multiple_of(QK_K), + "quantize_row_q8k: {k} is not divisible by {QK_K}" + ); + for (i, y) in ys.iter_mut().enumerate() { + let mut max = 0f32; + let mut amax = 0f32; + let xs = &xs[i * QK_K..(i + 1) * QK_K]; + for &x in xs.iter() { + if amax < x.abs() { + amax = x.abs(); + max = x; + } + } + if amax == 0f32 { + y.d = 0f32; + y.qs.fill(0) + } else { + let iscale = -128f32 / max; + for (j, q) in y.qs.iter_mut().enumerate() { + // ggml uses nearest_int with bit magic here, maybe we want the same + // but we would have to test and benchmark it. + let v = (iscale * xs[j]).round(); + *q = v.min(127.) as i8 + } + for j in 0..QK_K / 16 { + let mut sum = 0i32; + for ii in 0..16 { + sum += y.qs[j * 16 + ii] as i32 + } + y.bsums[j] = sum as i16 + } + y.d = 1.0 / iscale + } + } + } + + fn to_float(xs: &[Self], ys: &mut [f32]) { + let k = ys.len(); + debug_assert!( + k.is_multiple_of(QK_K), + "dequantize_row_q8k: {k} is not divisible by {QK_K}" + ); + for (i, x) in xs.iter().enumerate() { + for (j, &q) in x.qs.iter().enumerate() { + ys[i * QK_K + j] = x.d * q as f32 + } + } + } +} + +// https://github.com/ggml-org/llama.cpp/blob/aa3ee0eb0b80efca126cedf9bcb4fb5864b46ce3/ggml/src/ggml-cpu/ggml-cpu.c#L1205 +pub fn matmul( + (m, k, n): (usize, usize, usize), + lhs: &[f32], + rhs_t: &[T], + dst: &mut [f32], +) -> Result<()> { + debug_assert_eq!( + T::BLCK_SIZE, + T::VecDotType::BLCK_SIZE, + "Mismatched block sizes" + ); + debug_assert_eq!( + m * k, + lhs.len(), + "unexpected lhs length {} ({m},{k},{n})", + lhs.len() + ); + let k_in_blocks = k.div_ceil(T::BLCK_SIZE); + + // TODO: Pre-allocate this. + let mut lhs_b = vec![T::VecDotType::zeros(); m * k_in_blocks]; + // f32, f16, and bf16 support direct copy + if T::DIRECT_COPY { + T::VecDotType::direct_copy(lhs, &mut lhs_b); + } else { + for row_idx in 0..m { + let lhs_b_mut = &mut lhs_b[row_idx * k_in_blocks..(row_idx + 1) * k_in_blocks]; + let lhs = &lhs[row_idx * k..(row_idx + 1) * k]; + T::VecDotType::from_float(lhs, lhs_b_mut) + } + } + + for row_idx in 0..m { + let lhs_row = &lhs_b[row_idx * k_in_blocks..(row_idx + 1) * k_in_blocks]; + let dst_row = &mut dst[row_idx * n..(row_idx + 1) * n]; + + dst_row + .into_par_iter() + .enumerate() + .with_min_len(128) + .with_max_len(512) + .for_each(|(col_idx, dst)| { + let rhs_col = &rhs_t[col_idx * k_in_blocks..(col_idx + 1) * k_in_blocks]; + *dst = T::vec_dot(k, rhs_col, lhs_row); + }); + } + Ok(()) +} + +pub fn matmul_f16( + mkn: (usize, usize, usize), + lhs: &[f16], + rhs_t: &[T], + dst: &mut [f16], +) -> Result<()> { + let (m, k, n) = mkn; + if m * k != lhs.len() { + crate::bail!("unexpected lhs length {} {mkn:?}", lhs.len()); + } + + let k_in_lhs_blocks = k.div_ceil(T::BLCK_SIZE); + let k_in_rhs_blocks = k.div_ceil(T::VecDotType::BLCK_SIZE); + let mut lhs_b = vec![T::VecDotType::zeros(); m * k_in_lhs_blocks]; + for row_idx in 0..m { + let lhs_b = &mut lhs_b[row_idx * k_in_lhs_blocks..(row_idx + 1) * k_in_lhs_blocks]; + let lhs = &lhs[row_idx * k..(row_idx + 1) * k]; + let lhs_f32: Vec<_> = lhs.iter().map(|&x| x.to_f32()).collect(); + T::VecDotType::from_float(&lhs_f32, lhs_b); + } + let lhs_b = lhs_b.as_slice(); + + for row_idx in 0..m { + let lhs_row = &lhs_b[row_idx * k_in_lhs_blocks..(row_idx + 1) * k_in_lhs_blocks]; + let dst_row = &mut dst[row_idx * n..(row_idx + 1) * n]; + + for (col_idx, dst) in dst_row.iter_mut().enumerate() { + let rhs_col = &rhs_t[col_idx * k_in_rhs_blocks..(col_idx + 1) * k_in_rhs_blocks]; + let value = T::vec_dot(k, rhs_col, lhs_row); + *dst = f16::from_f32(value); + } + } + Ok(()) +} + +impl GgmlType for f32 { + const DTYPE: GgmlDType = GgmlDType::F32; + const BLCK_SIZE: usize = 1; + const DIRECT_COPY: bool = true; + type VecDotType = f32; + + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + debug_assert!(xs.len() >= n, "size mismatch xs {} < {n}", xs.len()); + debug_assert!(ys.len() >= n, "size mismatch ys {} < {n}", ys.len()); + let mut res = 0f32; + unsafe { crate::cpu::vec_dot_f32(xs.as_ptr(), ys.as_ptr(), &mut res, n) }; + res + } + + fn from_float(xs: &[f32], ys: &mut [Self]) { + debug_assert_eq!( + xs.len(), + ys.len(), + "size mismatch xs {} != ys {}", + xs.len(), + ys.len() + ); + ys.copy_from_slice(xs); + } + + fn to_float(xs: &[Self], ys: &mut [f32]) { + debug_assert_eq!( + xs.len(), + ys.len(), + "size mismatch xs {} != ys {}", + xs.len(), + ys.len() + ); + ys.copy_from_slice(xs); + } + + fn direct_copy(xs: &[f32], ys: &mut [Self]) { + Self::from_float(xs, ys) + } +} + +impl GgmlType for f16 { + const DTYPE: GgmlDType = GgmlDType::F16; + const BLCK_SIZE: usize = 1; + const DIRECT_COPY: bool = true; + type VecDotType = f16; + + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + debug_assert!(xs.len() >= n, "size mismatch xs {} < {n}", xs.len()); + debug_assert!(ys.len() >= n, "size mismatch ys {} < {n}", ys.len()); + let mut res = 0f32; + unsafe { crate::cpu::vec_dot_f16(xs.as_ptr(), ys.as_ptr(), &mut res, n) }; + res + } + + fn from_float(xs: &[f32], ys: &mut [Self]) { + debug_assert_eq!( + xs.len(), + ys.len(), + "size mismatch xs {} != ys {}", + xs.len(), + ys.len() + ); + ys.convert_from_f32_slice(xs); + } + + fn to_float(xs: &[Self], ys: &mut [f32]) { + debug_assert_eq!( + xs.len(), + ys.len(), + "size mismatch xs {} != ys {}", + xs.len(), + ys.len() + ); + xs.convert_to_f32_slice(ys); + } + + fn direct_copy(xs: &[f32], ys: &mut [Self]) { + Self::from_float(xs, ys) + } +} + +impl GgmlType for bf16 { + const DTYPE: GgmlDType = GgmlDType::BF16; + const BLCK_SIZE: usize = 1; + const DIRECT_COPY: bool = true; + type VecDotType = bf16; + + fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + Self::vec_dot_unopt(n, xs, ys) + } + + fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { + debug_assert!(xs.len() >= n, "size mismatch xs {} < {n}", xs.len()); + debug_assert!(ys.len() >= n, "size mismatch ys {} < {n}", ys.len()); + let mut res = 0f32; + unsafe { crate::cpu::vec_dot_bf16(xs.as_ptr(), ys.as_ptr(), &mut res, n) }; + res + } + + fn from_float(xs: &[f32], ys: &mut [Self]) { + debug_assert_eq!( + xs.len(), + ys.len(), + "size mismatch xs {} != ys {}", + xs.len(), + ys.len() + ); + ys.convert_from_f32_slice(xs); + } + + fn to_float(xs: &[Self], ys: &mut [f32]) { + debug_assert_eq!( + xs.len(), + ys.len(), + "size mismatch xs {} != ys {}", + xs.len(), + ys.len() + ); + xs.convert_to_f32_slice(ys); + } + + fn direct_copy(xs: &[f32], ys: &mut [Self]) { + Self::from_float(xs, ys) + } +} + +macro_rules! verify_block_size { + ( $block_type:ident ) => { + const _: () = + assert!($block_type::BLCK_SIZE == <$block_type as GgmlType>::VecDotType::BLCK_SIZE); + }; +} + +macro_rules! verify_block_sizes { + ( $( $block_type:ident ),* ) => { + $( + verify_block_size!($block_type); + )* + }; +} + +verify_block_sizes!( + BlockQ4_0, BlockQ4_1, BlockQ5_0, BlockQ5_1, BlockQ8_0, BlockQ8_1, BlockQ2K, BlockQ3K, BlockQ4K, + BlockQ5K, BlockQ6K, BlockQ8K, f32, f16, bf16 +); diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/metal.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/metal.rs new file mode 100644 index 0000000000..ad746ef0e3 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/quantized/metal.rs @@ -0,0 +1,391 @@ +use super::{GgmlDType, QStorage}; +use crate::backend::BackendStorage; +use crate::{DType, MetalDevice, MetalStorage, Result, Shape, D}; +use candle_metal_kernels::metal::Buffer; +use std::sync::Arc; + +pub struct QMetalStorage { + dtype: GgmlDType, + device: MetalDevice, + buffer: Arc, +} + +impl QMetalStorage { + pub fn zeros(device: &MetalDevice, elem_count: usize, dtype: GgmlDType) -> Result { + let size = elem_count * dtype.type_size() / dtype.block_size(); + let buffer = device.allocate_zeros(size)?; + Ok(Self { + buffer, + device: device.clone(), + dtype, + }) + } + + pub fn dtype(&self) -> GgmlDType { + self.dtype + } + + pub fn device(&self) -> &MetalDevice { + &self.device + } + + pub fn buffer(&self) -> &Buffer { + &self.buffer + } + + pub fn dequantize(&self, elem_count: usize) -> Result { + use crate::quantized::k_quants::GgmlType; + + let buffer = self.device.allocate_buffer(self.buffer.length())?; + let blit = self.device.blit_command_encoder()?; + blit.set_label("blit_to_cpu"); + blit.copy_from_buffer(&self.buffer, 0, &buffer, 0, self.buffer.length()); + blit.end_encoding(); + self.device.wait_until_completed()?; + let mut out = vec![0.0; elem_count]; + let block_len = elem_count / self.dtype.block_size(); + match self.dtype { + GgmlDType::F32 => { + let vec: Vec = read_to_vec(&buffer, block_len); + f32::to_float(&vec, &mut out); + } + GgmlDType::F16 => { + let vec: Vec = read_to_vec(&buffer, block_len); + half::f16::to_float(&vec, &mut out); + } + GgmlDType::BF16 => { + let vec: Vec = read_to_vec(&buffer, block_len); + half::bf16::to_float(&vec, &mut out); + } + GgmlDType::Q4_0 => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ4_0::to_float(&vec, &mut out); + } + GgmlDType::Q4_1 => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ4_1::to_float(&vec, &mut out); + } + GgmlDType::Q5_0 => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ5_0::to_float(&vec, &mut out); + } + GgmlDType::Q5_1 => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ5_1::to_float(&vec, &mut out); + } + GgmlDType::Q8_0 => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ8_0::to_float(&vec, &mut out); + } + GgmlDType::Q8_1 => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ8_1::to_float(&vec, &mut out); + } + GgmlDType::Q2K => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ2K::to_float(&vec, &mut out); + } + GgmlDType::Q3K => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ3K::to_float(&vec, &mut out); + } + GgmlDType::Q4K => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ4K::to_float(&vec, &mut out); + } + GgmlDType::Q5K => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ5K::to_float(&vec, &mut out); + } + GgmlDType::Q6K => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ6K::to_float(&vec, &mut out); + } + GgmlDType::Q8K => { + let vec: Vec = read_to_vec(&buffer, block_len); + crate::quantized::BlockQ8K::to_float(&vec, &mut out); + } + } + + let buffer = self.device.new_buffer_with_data(&out)?; + Ok(MetalStorage::new( + buffer, + self.device.clone(), + elem_count, + DType::F32, + )) + } + + pub fn quantize(&mut self, src: &MetalStorage) -> Result<()> { + // Quantization only happens on CPU for now. + let src = src.to_cpu::()?; + let elem_count = src.len(); + let src = crate::Storage::Cpu(crate::CpuStorage::F32(src)); + let mut qcpu_storage = crate::Device::Cpu.qzeros(elem_count, self.dtype)?; + qcpu_storage.quantize(&src)?; + let buffer = self.device.new_buffer_with_data(&qcpu_storage.data()?)?; + self.buffer = buffer; + Ok(()) + } + + pub fn quantize_imatrix( + &mut self, + src: &MetalStorage, + imatrix_weights: &[f32], + n_per_row: usize, + ) -> Result<()> { + // Quantization only happens on CPU for now. + let src = src.to_cpu::()?; + let elem_count = src.len(); + let src = crate::Storage::Cpu(crate::CpuStorage::F32(src)); + let mut qcpu_storage = crate::Device::Cpu.qzeros(elem_count, self.dtype)?; + qcpu_storage.quantize_imatrix(&src, imatrix_weights, n_per_row)?; + let buffer = self.device.new_buffer_with_data(&qcpu_storage.data()?)?; + self.buffer = buffer; + Ok(()) + } + + pub fn quantize_imatrix_onto( + &mut self, + src: &crate::CpuStorage, + imatrix_weights: &[f32], + n_per_row: usize, + ) -> Result<()> { + // Quantization only happens on CPU for now. + let elem_count = src.as_slice::()?.len(); + let mut qcpu_storage = crate::Device::Cpu.qzeros(elem_count, self.dtype)?; + + if let QStorage::Cpu(storage) = &mut qcpu_storage { + storage.from_float_imatrix(src.as_slice::()?, imatrix_weights, n_per_row); + } else { + unreachable!() + } + + let buffer = self.device.new_buffer_with_data(&qcpu_storage.data()?)?; + self.buffer = buffer; + Ok(()) + } + + pub fn quantize_onto(&mut self, src: &crate::CpuStorage) -> Result<()> { + // Quantization only happens on CPU for now. + let elem_count = src.as_slice::()?.len(); + let mut qcpu_storage = crate::Device::Cpu.qzeros(elem_count, self.dtype)?; + + if let QStorage::Cpu(storage) = &mut qcpu_storage { + storage.from_float(src.as_slice::()?); + } else { + unreachable!() + } + + let buffer = self.device.new_buffer_with_data(&qcpu_storage.data()?)?; + self.buffer = buffer; + Ok(()) + } + + pub fn storage_size_in_bytes(&self) -> usize { + self.buffer.length() + } + + fn fwd_mv( + &self, + self_shape: &Shape, + storage: &MetalStorage, + layout: &crate::Layout, + ) -> Result<(MetalStorage, Shape)> { + use crate::MetalError; + + if !layout.is_contiguous() { + crate::bail!("input tensor is not contiguous {layout:?}") + } + let src_shape = layout.shape(); + // self is transposed so n is first then k. + if src_shape.rank() < 2 { + crate::bail!("input tensor has only one dimension {layout:?}") + } + let (n, k) = self_shape.dims2()?; + let mut dst_shape = src_shape.dims().to_vec(); + + // We always use a single batch dimension and stack all the tensors in the batch on the + // second dimension as the implementation in candle-metal-kernels doesn't handle batch + // properly. + let m = match dst_shape.len() { + 3 => dst_shape[0] * dst_shape[1], + 2 => dst_shape[0], + n => crate::bail!("Invalid rank {n} for quantized matmul metal"), + }; + let last_k = dst_shape.pop().unwrap(); + if last_k != k { + crate::bail!("input tensor {layout:?} incompatible with {:?}", self_shape) + } + dst_shape.push(n); + let dst_shape = Shape::from(dst_shape); + let device = storage.device().clone(); + let dst = device.new_buffer(dst_shape.elem_count(), DType::F32, "qmatmul")?; + let encoder = device.command_encoder()?; + // In some cases it would be better to use the mm variant, though it has its drawbacks + // around memory alignment. + for batch_id in 0..m { + candle_metal_kernels::call_quantized_matmul_mv_t( + device.device(), + &encoder, + device.kernels(), + self.dtype.into(), + (1, 1, n, k), + storage.buffer(), + (layout.start_offset() + batch_id * k) * storage.dtype().size_in_bytes(), + &self.buffer, + batch_id * n * DType::F32.size_in_bytes(), + &dst, + ) + .map_err(MetalError::from)?; + } + let dst_storage = crate::MetalStorage::new(dst, device, dst_shape.elem_count(), DType::F32); + Ok((dst_storage, dst_shape)) + } + + pub fn fwd( + &self, + self_shape: &Shape, + storage: &MetalStorage, + layout: &crate::Layout, + ) -> Result<(MetalStorage, Shape)> { + use crate::MetalError; + + if !layout.is_contiguous() { + crate::bail!("input tensor is not contiguous {layout:?}") + } + let src_shape = layout.shape(); + // self is transposed so n is first then k. + if src_shape.rank() < 2 { + crate::bail!("input tensor has only one dimension {layout:?}") + } + let n = self_shape.dim(D::Minus2)?; + let k = self_shape.dim(D::Minus1)?; + let mut dst_shape = src_shape.dims().to_vec(); + + if src_shape.rank() < self_shape.rank() { + crate::bail!( + "input rank ({}) must be >= weight rank ({})", + src_shape.rank(), + self_shape.rank() + ) + } + + if src_shape.dim(D::Minus2)? == 1 { + return self.fwd_mv(self_shape, storage, layout); + } + + let last_k = dst_shape.pop().unwrap(); + if last_k != k { + crate::bail!("input tensor {layout:?} incompatible with {:?}", self_shape) + } + dst_shape.push(n); + let dst_shape = Shape::from(dst_shape); + let device = storage.device().clone(); + let dst = device.new_buffer(dst_shape.elem_count(), DType::F32, "qmatmul")?; + let encoder = device.command_encoder()?; + + assert_eq!(storage.dtype(), DType::F32); + + if self_shape.rank() > 4 { + crate::bail!("weight rank ({}) must be <= 4", self_shape.rank()) + } + let src0_l = crate::Layout::contiguous( + [vec![1; 4 - self_shape.rank()], self_shape.dims().to_vec()].concat(), + ); + let src0_stride = src0_l + .stride() + .iter() + .map(|x| { + (*x as f32 * (self.dtype.type_size() as f32 / self.dtype.block_size() as f32)) + as usize + }) + .collect::>(); + + if src_shape.rank() > 4 { + crate::bail!("weight rank ({}) must be <= 4", src_shape.rank()) + } + let src1_l = crate::Layout::contiguous( + [vec![1; 4 - src_shape.rank()], src_shape.dims().to_vec()].concat(), + ); + + candle_metal_kernels::call_quantized_matmul_mm_t( + device.device(), + &encoder, + device.kernels(), + self.dtype.into(), + src0_l.dims(), + &src0_stride, + &self.buffer, + src1_l.dims(), + &src1_l + .stride() + .iter() + .map(|x| x * DType::F32.size_in_bytes()) + .collect::>(), + storage.buffer(), + src1_l.start_offset() * storage.dtype().size_in_bytes(), + dst_shape.dims(), + 0, + &dst, + ) + .map_err(MetalError::from)?; + + let dst_storage = crate::MetalStorage::new(dst, device, dst_shape.elem_count(), DType::F32); + Ok((dst_storage, dst_shape)) + } + + pub fn data(&self) -> Result> { + let buffer = self.device.allocate_buffer(self.buffer.length())?; + { + let blit = self.device.blit_command_encoder()?; + blit.set_label("blit_to_cpu"); + blit.copy_from_buffer(&self.buffer, 0, &buffer, 0, self.buffer.length()); + blit.end_encoding(); + } + self.device.wait_until_completed()?; + Ok(read_to_vec::(&buffer, self.storage_size_in_bytes())) + } +} + +pub fn load_quantized( + device: &MetalDevice, + data: &[T], +) -> Result { + let buffer = device.new_buffer_with_data(data)?; + let device = device.clone(); + Ok(QStorage::Metal(QMetalStorage { + dtype: T::DTYPE, + device, + buffer, + })) +} + +fn read_to_vec(buffer: &Buffer, n: usize) -> Vec { + let ptr = buffer.contents() as *const T; + assert!(!ptr.is_null()); + let slice = unsafe { std::slice::from_raw_parts(ptr, n) }; + slice.to_vec() +} + +impl From for candle_metal_kernels::GgmlDType { + fn from(value: GgmlDType) -> Self { + match value { + GgmlDType::Q4_0 => candle_metal_kernels::GgmlDType::Q4_0, + GgmlDType::Q4_1 => candle_metal_kernels::GgmlDType::Q4_1, + GgmlDType::Q5_0 => candle_metal_kernels::GgmlDType::Q5_0, + GgmlDType::Q5_1 => candle_metal_kernels::GgmlDType::Q5_1, + GgmlDType::Q8_0 => candle_metal_kernels::GgmlDType::Q8_0, + GgmlDType::Q8_1 => candle_metal_kernels::GgmlDType::Q8_1, + GgmlDType::Q2K => candle_metal_kernels::GgmlDType::Q2K, + GgmlDType::Q3K => candle_metal_kernels::GgmlDType::Q3K, + GgmlDType::Q4K => candle_metal_kernels::GgmlDType::Q4K, + GgmlDType::Q5K => candle_metal_kernels::GgmlDType::Q5K, + GgmlDType::Q6K => candle_metal_kernels::GgmlDType::Q6K, + GgmlDType::Q8K => candle_metal_kernels::GgmlDType::Q8K, + GgmlDType::F16 => candle_metal_kernels::GgmlDType::F16, + GgmlDType::F32 => candle_metal_kernels::GgmlDType::F32, + GgmlDType::BF16 => candle_metal_kernels::GgmlDType::F16, + } + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/mod.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/mod.rs new file mode 100644 index 0000000000..7316d29871 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/quantized/mod.rs @@ -0,0 +1,881 @@ +use crate::{ + backend::BackendStorage, CpuStorage, DType, Device, Result, Shape, Storage, Tensor, D, +}; +use k_quants::*; +use std::borrow::Cow; + +#[cfg(target_feature = "avx2")] +pub mod avx; +mod dummy_cuda; +mod dummy_metal; +pub mod ggml_file; +pub mod gguf_file; +pub mod imatrix_file; +pub mod k_quants; +#[cfg(feature = "metal")] +pub mod metal; +#[cfg(not(feature = "metal"))] +mod metal { + pub use super::dummy_metal::*; +} +#[cfg(feature = "cuda")] +pub mod cuda; +#[cfg(not(feature = "cuda"))] +mod cuda { + pub use super::dummy_cuda::*; +} + +#[cfg(target_feature = "neon")] +pub mod neon; +#[cfg(target_feature = "simd128")] +pub mod simd128; +pub mod utils; +use half::{bf16, f16}; + +pub use k_quants::GgmlType; + +fn as_t_slice(data: Cow<'_, [u8]>) -> &[T] { + let size = std::mem::size_of::(); + assert_eq!( + data.len() % size, + 0, + "Data length must be a multiple of T's size" + ); + let ptr = data.as_ptr(); + assert_eq!( + (ptr as usize) % std::mem::align_of::(), + 0, + "Data pointer must be aligned to T's alignment" + ); + unsafe { std::slice::from_raw_parts(ptr as *const T, data.len() / size) } +} + +pub struct QTensor { + storage: QStorage, + shape: Shape, +} + +impl Device { + fn qzeros(&self, elem_count: usize, dtype: GgmlDType) -> Result { + match self { + Device::Cpu => { + let storage = dtype.cpu_zeros(elem_count); + Ok(QStorage::Cpu(storage)) + } + Device::Metal(metal) => { + let storage = metal::QMetalStorage::zeros(metal, elem_count, dtype)?; + Ok(QStorage::Metal(storage)) + } + Device::Cuda(cuda) => { + let storage = cuda::QCudaStorage::zeros(cuda, elem_count, dtype)?; + Ok(QStorage::Cuda(storage)) + } + } + } +} + +pub enum QStorage { + Cpu(Box), + Metal(metal::QMetalStorage), + Cuda(cuda::QCudaStorage), +} + +impl QStorage { + pub fn from_data(data: Cow<'_, [u8]>, device: &Device, dtype: GgmlDType) -> Result { + match device { + Device::Cpu => Ok(Self::Cpu(dtype.from_data(data))), + Device::Metal(d) => match dtype { + GgmlDType::F32 => metal::load_quantized(d, as_t_slice::(data)), + GgmlDType::F16 => metal::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q4_0 => metal::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q4_1 => metal::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q5_0 => metal::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q5_1 => metal::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q8_0 => metal::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q8_1 => metal::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q2K => metal::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q3K => metal::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q4K => metal::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q5K => metal::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q6K => metal::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q8K => metal::load_quantized(d, as_t_slice::(data)), + GgmlDType::BF16 => metal::load_quantized(d, as_t_slice::(data)), + }, + Device::Cuda(d) => match dtype { + GgmlDType::F32 => cuda::load_quantized(d, as_t_slice::(data)), + GgmlDType::F16 => cuda::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q4_0 => cuda::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q4_1 => cuda::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q5_0 => cuda::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q5_1 => cuda::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q8_0 => cuda::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q8_1 => cuda::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q2K => cuda::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q3K => cuda::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q4K => cuda::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q5K => cuda::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q6K => cuda::load_quantized(d, as_t_slice::(data)), + GgmlDType::Q8K => cuda::load_quantized(d, as_t_slice::(data)), + GgmlDType::BF16 => cuda::load_quantized(d, as_t_slice::(data)), + }, + } + } + + fn block_size(&self) -> usize { + match self { + QStorage::Cpu(storage) => storage.block_size(), + QStorage::Metal(storage) => storage.dtype().block_size(), + QStorage::Cuda(storage) => storage.dtype().block_size(), + } + } + + fn dtype(&self) -> GgmlDType { + match self { + QStorage::Cpu(storage) => storage.dtype(), + QStorage::Metal(storage) => storage.dtype(), + QStorage::Cuda(storage) => storage.dtype(), + } + } + + fn device(&self) -> Device { + match self { + QStorage::Cpu(_storage) => Device::Cpu, + QStorage::Metal(storage) => Device::Metal(storage.device().clone()), + QStorage::Cuda(storage) => Device::Cuda(storage.device().clone()), + } + } + + fn size_in_bytes(&self) -> usize { + match self { + QStorage::Cpu(storage) => storage.storage_size_in_bytes(), + QStorage::Metal(storage) => storage.storage_size_in_bytes(), + QStorage::Cuda(storage) => storage.storage_size_in_bytes(), + } + } + + fn quantize(&mut self, src: &Storage) -> Result<()> { + match (self, src) { + (QStorage::Cpu(storage), Storage::Cpu(src)) => { + storage.from_float(src.as_slice::()?); + } + (QStorage::Metal(storage), Storage::Metal(src)) => storage.quantize(src)?, + (QStorage::Cuda(storage), Storage::Cuda(src)) => storage.quantize(src)?, + _ => crate::bail!("Invalid quantize storage locations do not match"), + } + Ok(()) + } + + fn quantize_imatrix( + &mut self, + src: &Storage, + imatrix_weights: &[f32], + n_per_row: usize, + ) -> Result<()> { + match (self, src) { + (QStorage::Cpu(storage), Storage::Cpu(src)) => { + storage.from_float_imatrix(src.as_slice::()?, imatrix_weights, n_per_row); + } + (QStorage::Metal(storage), Storage::Metal(src)) => { + storage.quantize_imatrix(src, imatrix_weights, n_per_row)? + } + (QStorage::Cuda(storage), Storage::Cuda(src)) => { + storage.quantize_imatrix(src, imatrix_weights, n_per_row)? + } + _ => crate::bail!("Invalid quantize storage locations do not match"), + } + Ok(()) + } + + fn quantize_onto(&mut self, src: &Storage) -> Result<()> { + match (self, src) { + (QStorage::Cpu(storage), Storage::Cpu(src)) => { + storage.from_float(src.as_slice::()?); + } + (QStorage::Metal(storage), Storage::Cpu(src)) => storage.quantize_onto(src)?, + (QStorage::Cuda(storage), Storage::Cpu(src)) => storage.quantize_onto(src)?, + _ => crate::bail!("Invalid quantize source storage locations: not on cpu"), + } + Ok(()) + } + + fn quantize_imatrix_onto( + &mut self, + src: &Storage, + imatrix_weights: &[f32], + n_per_row: usize, + ) -> Result<()> { + match (self, src) { + (QStorage::Cpu(storage), Storage::Cpu(src)) => { + storage.from_float_imatrix(src.as_slice::()?, imatrix_weights, n_per_row); + } + (QStorage::Metal(storage), Storage::Cpu(src)) => { + storage.quantize_imatrix_onto(src, imatrix_weights, n_per_row)? + } + (QStorage::Cuda(storage), Storage::Cpu(src)) => { + storage.quantize_imatrix_onto(src, imatrix_weights, n_per_row)? + } + _ => crate::bail!("Invalid quantize storage locations do not match"), + } + Ok(()) + } + + fn dequantize(&self, elem_count: usize) -> Result { + match self { + QStorage::Cpu(storage) => Ok(Storage::Cpu(storage.dequantize(elem_count)?)), + QStorage::Metal(storage) => Ok(Storage::Metal(storage.dequantize(elem_count)?)), + QStorage::Cuda(storage) => Ok(Storage::Cuda(storage.dequantize(elem_count)?)), + } + } + + fn data(&self) -> Result> { + match self { + QStorage::Cpu(storage) => { + let data_ptr = storage.as_ptr(); + let size_in_bytes = storage.storage_size_in_bytes(); + let data = unsafe { std::slice::from_raw_parts(data_ptr, size_in_bytes) }; + Ok(Cow::from(data)) + } + QStorage::Cuda(storage) => Ok(Cow::from(storage.data()?)), + QStorage::Metal(storage) => Ok(Cow::from(storage.data()?)), + } + } + + pub fn device_ptr(&self) -> Result<*const u8> { + match self { + QStorage::Cuda(storage) => storage.device_ptr(), + QStorage::Metal(_) | QStorage::Cpu(_) => { + crate::bail!("not implemented"); + } + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum GgmlDType { + F32, + F16, + BF16, + Q4_0, + Q4_1, + Q5_0, + Q5_1, + Q8_0, + Q8_1, + Q2K, + Q3K, + Q4K, + Q5K, + Q6K, + Q8K, +} + +impl GgmlDType { + pub(crate) fn from_u32(u: u32) -> Result { + let dtype = match u { + 0 => Self::F32, + 1 => Self::F16, + 2 => Self::Q4_0, + 3 => Self::Q4_1, + 6 => Self::Q5_0, + 7 => Self::Q5_1, + 8 => Self::Q8_0, + 9 => Self::Q8_1, + 10 => Self::Q2K, + 11 => Self::Q3K, + 12 => Self::Q4K, + 13 => Self::Q5K, + 14 => Self::Q6K, + 15 => Self::Q8K, + // https://github.com/ggerganov/ggml/blob/29d87fc6676e7ed0cdfdec0804b06001d9c2bb44/include/ggml.h#L389 + 30 => Self::BF16, + _ => crate::bail!("unknown dtype for tensor {u}"), + }; + Ok(dtype) + } + + pub(crate) fn to_u32(self) -> u32 { + match self { + Self::F32 => 0, + Self::F16 => 1, + Self::Q4_0 => 2, + Self::Q4_1 => 3, + Self::Q5_0 => 6, + Self::Q5_1 => 7, + Self::Q8_0 => 8, + Self::Q8_1 => 9, + Self::Q2K => 10, + Self::Q3K => 11, + Self::Q4K => 12, + Self::Q5K => 13, + Self::Q6K => 14, + Self::Q8K => 15, + // https://github.com/ggerganov/ggml/blob/29d87fc6676e7ed0cdfdec0804b06001d9c2bb44/include/ggml.h#L389 + Self::BF16 => 30, + } + } + + /// The block dtype + pub fn cpu_zeros(&self, elem_count: usize) -> Box { + match self { + Self::F32 => Box::new(vec![f32::zeros(); elem_count]), + Self::F16 => Box::new(vec![f16::zeros(); elem_count]), + Self::Q4_0 => Box::new(vec![BlockQ4_0::zeros(); elem_count / BlockQ4_0::BLCK_SIZE]), + Self::Q4_1 => Box::new(vec![BlockQ4_1::zeros(); elem_count / BlockQ4_1::BLCK_SIZE]), + Self::Q5_0 => Box::new(vec![BlockQ5_0::zeros(); elem_count / BlockQ5_0::BLCK_SIZE]), + Self::Q5_1 => Box::new(vec![BlockQ5_1::zeros(); elem_count / BlockQ5_1::BLCK_SIZE]), + Self::Q8_0 => Box::new(vec![BlockQ8_0::zeros(); elem_count / BlockQ8_0::BLCK_SIZE]), + Self::Q8_1 => Box::new(vec![BlockQ8_1::zeros(); elem_count / BlockQ8_1::BLCK_SIZE]), + Self::Q2K => Box::new(vec![BlockQ2K::zeros(); elem_count / BlockQ2K::BLCK_SIZE]), + Self::Q3K => Box::new(vec![BlockQ3K::zeros(); elem_count / BlockQ3K::BLCK_SIZE]), + Self::Q4K => Box::new(vec![BlockQ4K::zeros(); elem_count / BlockQ4K::BLCK_SIZE]), + Self::Q5K => Box::new(vec![BlockQ5K::zeros(); elem_count / BlockQ5K::BLCK_SIZE]), + Self::Q6K => Box::new(vec![BlockQ6K::zeros(); elem_count / BlockQ6K::BLCK_SIZE]), + Self::Q8K => Box::new(vec![BlockQ8K::zeros(); elem_count / BlockQ8K::BLCK_SIZE]), + Self::BF16 => Box::new(vec![bf16::zeros(); elem_count]), + } + } + + pub fn from_data(&self, data: Cow<'_, [u8]>) -> Box { + match self { + Self::F32 => Box::new(as_t_slice::(data).to_vec()), + Self::F16 => Box::new(as_t_slice::(data).to_vec()), + Self::Q4_0 => Box::new(as_t_slice::(data).to_vec()), + Self::Q4_1 => Box::new(as_t_slice::(data).to_vec()), + Self::Q5_0 => Box::new(as_t_slice::(data).to_vec()), + Self::Q5_1 => Box::new(as_t_slice::(data).to_vec()), + Self::Q8_0 => Box::new(as_t_slice::(data).to_vec()), + Self::Q8_1 => Box::new(as_t_slice::(data).to_vec()), + Self::Q2K => Box::new(as_t_slice::(data).to_vec()), + Self::Q3K => Box::new(as_t_slice::(data).to_vec()), + Self::Q4K => Box::new(as_t_slice::(data).to_vec()), + Self::Q5K => Box::new(as_t_slice::(data).to_vec()), + Self::Q6K => Box::new(as_t_slice::(data).to_vec()), + Self::Q8K => Box::new(as_t_slice::(data).to_vec()), + Self::BF16 => Box::new(as_t_slice::(data).to_vec()), + } + } + + /// The type size for blocks in bytes. + pub fn type_size(&self) -> usize { + use k_quants::*; + match self { + Self::F32 => 4, + Self::F16 | Self::BF16 => 2, + Self::Q4_0 => std::mem::size_of::(), + Self::Q4_1 => std::mem::size_of::(), + Self::Q5_0 => std::mem::size_of::(), + Self::Q5_1 => std::mem::size_of::(), + // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L932 + Self::Q8_0 => std::mem::size_of::(), + Self::Q8_1 => std::mem::size_of::(), + Self::Q2K => std::mem::size_of::(), + Self::Q3K => std::mem::size_of::(), + Self::Q4K => std::mem::size_of::(), + Self::Q5K => std::mem::size_of::(), + Self::Q6K => std::mem::size_of::(), + Self::Q8K => std::mem::size_of::(), + } + } + + /// The block size, i.e. the number of elements stored in each block. + pub fn block_size(&self) -> usize { + match self { + Self::F32 => 1, + Self::F16 | Self::BF16 => 1, + Self::Q4_0 => k_quants::QK4_0, + Self::Q4_1 => k_quants::QK4_1, + Self::Q5_0 => k_quants::QK5_0, + Self::Q5_1 => k_quants::QK5_1, + Self::Q8_0 => k_quants::QK8_0, + Self::Q8_1 => k_quants::QK8_1, + Self::Q2K | Self::Q3K | Self::Q4K | Self::Q5K | Self::Q6K | Self::Q8K => k_quants::QK_K, + } + } +} + +// A version of GgmlType without `vec_dot` so that it can be dyn boxed. +pub trait QuantizedType: Send + Sync { + fn dtype(&self) -> GgmlDType; + fn matmul_t(&self, mkn: (usize, usize, usize), lhs: &[f32], dst: &mut [f32]) -> Result<()>; + fn matmul_t_f16(&self, mkn: (usize, usize, usize), lhs: &[f16], dst: &mut [f16]) -> Result<()>; + fn dequantize(&self, elem_count: usize) -> Result; + fn storage_size_in_bytes(&self) -> usize; + fn as_ptr(&self) -> *const u8; + fn block_size(&self) -> usize; + #[allow(clippy::wrong_self_convention)] + fn from_float(&mut self, xs: &[f32]); + #[allow(clippy::wrong_self_convention)] + fn from_float_imatrix(&mut self, xs: &[f32], imatrix_weights: &[f32], n_per_row: usize); + fn size(&self) -> usize; +} + +impl QuantizedType for Vec { + fn matmul_t(&self, mkn: (usize, usize, usize), lhs: &[f32], dst: &mut [f32]) -> Result<()> { + k_quants::matmul(mkn, lhs, self.as_slice(), dst) + } + fn matmul_t_f16(&self, mkn: (usize, usize, usize), lhs: &[f16], dst: &mut [f16]) -> Result<()> { + k_quants::matmul_f16(mkn, lhs, self.as_slice(), dst) + } + + fn size(&self) -> usize { + self.len() * core::mem::size_of::() + } + + fn from_float(&mut self, xs: &[f32]) { + T::from_float(xs, self) + } + + fn from_float_imatrix(&mut self, xs: &[f32], imatrix_weights: &[f32], n_per_row: usize) { + T::from_float_imatrix(xs, self, imatrix_weights, n_per_row) + } + + fn dtype(&self) -> GgmlDType { + T::DTYPE + } + + fn block_size(&self) -> usize { + T::BLCK_SIZE + } + + fn dequantize(&self, elem_count: usize) -> Result { + let mut ys = vec![0.0f32; elem_count]; + T::to_float(self.as_slice(), &mut ys); + Ok(CpuStorage::F32(ys)) + } + + fn storage_size_in_bytes(&self) -> usize { + self.len() * std::mem::size_of::() + } + + fn as_ptr(&self) -> *const u8 { + self.as_ptr() as *const u8 + } +} + +impl std::fmt::Debug for QTensor { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "QTensor[{:?}; {:?}]", self.shape, self.dtype()) + } +} + +fn check_shape(shape: &Shape, block_size: usize) -> Result<()> { + let dims = shape.dims(); + if dims.is_empty() { + crate::bail!("scalar tensor cannot be quantized {shape:?}") + } + if !dims[dims.len() - 1].is_multiple_of(block_size) { + crate::bail!( + "quantized tensor must have their last dim divisible by block size {shape:?} {}", + block_size + ) + } + Ok(()) +} + +impl QTensor { + pub fn new>(storage: QStorage, shape: S) -> Result { + let shape = shape.into(); + check_shape(&shape, storage.block_size())?; + Ok(Self { storage, shape }) + } + + pub fn quantize(src: &Tensor, dtype: GgmlDType) -> Result { + let shape = src.shape(); + let block_size = dtype.block_size(); + check_shape(shape, block_size)?; + let src = src.to_dtype(crate::DType::F32)?.flatten_all()?; + let elem_count = shape.elem_count(); + if !elem_count.is_multiple_of(block_size) { + crate::bail!( + "tensor size ({shape:?}) is not divisible by block size {}", + block_size + ) + } + let mut storage = src.device().qzeros(elem_count, dtype)?; + storage.quantize(&src.storage())?; + Ok(Self { + storage, + shape: shape.clone(), + }) + } + + pub fn quantize_imatrix( + src: &Tensor, + imatrix_weights: &[f32], + dtype: GgmlDType, + ) -> Result { + // (n_per_row/QK_K-1)*QK_K+(QK_K/32-1)*32+32=n_per_row + // Size of imatrix == last dim of tensor + let n_per_row = src.dim(D::Minus1)?; + if imatrix_weights.len() != n_per_row { + crate::bail!( + "imatrix weights must have the same length {} as the last dim of src {}", + imatrix_weights.len(), + src.dim(D::Minus1)? + ); + } + + let shape = src.shape(); + let block_size = dtype.block_size(); + check_shape(shape, block_size)?; + let src = src.to_dtype(crate::DType::F32)?.flatten_all()?; + let elem_count = shape.elem_count(); + if !elem_count.is_multiple_of(block_size) { + crate::bail!( + "tensor size ({shape:?}) is not divisible by block size {}", + block_size + ); + } + let mut storage = src.device().qzeros(elem_count, dtype)?; + storage.quantize_imatrix(&src.storage(), imatrix_weights, n_per_row)?; + Ok(Self { + storage, + shape: shape.clone(), + }) + } + + /// Quantize `src` (currently on the CPU) to a QTensor on `dev` + pub fn quantize_imatrix_onto( + src: &Tensor, + imatrix_weights: &[f32], + dtype: GgmlDType, + dev: &Device, + ) -> Result { + if !src.device().is_cpu() { + crate::bail!( + "`quantize_onto` expects a `src` to be on the cpu, got {:?}.", + src.device() + ) + } + // (n_per_row/QK_K-1)*QK_K+(QK_K/32-1)*32+32=n_per_row + // Size of imatrix == last dim of tensor + let n_per_row = src.dim(D::Minus1)?; + if imatrix_weights.len() != n_per_row { + crate::bail!( + "imatrix weights must have the same length {} as the last dim of src {}", + imatrix_weights.len(), + src.dim(D::Minus1)? + ); + } + let shape = src.shape(); + let block_size = dtype.block_size(); + check_shape(shape, block_size)?; + let src = src.to_dtype(crate::DType::F32)?.flatten_all()?; + let elem_count = shape.elem_count(); + if !elem_count.is_multiple_of(block_size) { + crate::bail!( + "tensor size ({shape:?}) is not divisible by block size {}", + block_size + ) + } + // storage is on the `dev`, src is on `cpu` + let mut storage = dev.qzeros(elem_count, dtype)?; + storage.quantize_imatrix_onto(&src.storage(), imatrix_weights, n_per_row)?; + Ok(Self { + storage, + shape: shape.clone(), + }) + } + + /// Quantize `src` (currently on the CPU) to a QTensor on `dev` + pub fn quantize_onto(src: &Tensor, dtype: GgmlDType, dev: &Device) -> Result { + if !src.device().is_cpu() { + crate::bail!( + "`quantize_onto` expects a `src` to be on the cpu, got {:?}.", + src.device() + ) + } + let shape = src.shape(); + let block_size = dtype.block_size(); + check_shape(shape, block_size)?; + let src = src.to_dtype(crate::DType::F32)?.flatten_all()?; + let elem_count = shape.elem_count(); + if !elem_count.is_multiple_of(block_size) { + crate::bail!( + "tensor size ({shape:?}) is not divisible by block size {}", + block_size + ) + } + // storage is on the `dev`, src is on `cpu` + let mut storage = dev.qzeros(elem_count, dtype)?; + storage.quantize_onto(&src.storage())?; + Ok(Self { + storage, + shape: shape.clone(), + }) + } + + pub fn dtype(&self) -> GgmlDType { + self.storage.dtype() + } + + pub fn device(&self) -> Device { + self.storage.device() + } + + pub fn rank(&self) -> usize { + self.shape.rank() + } + + pub fn shape(&self) -> &Shape { + &self.shape + } + + pub fn dequantize(&self, device: &Device) -> Result { + let storage = self.storage.dequantize(self.shape.elem_count())?; + let none = crate::op::BackpropOp::none(); + crate::tensor::from_storage(storage, self.shape.clone(), none, false).to_device(device) + } + + pub fn dequantize_f16(&self, device: &Device) -> Result { + // In the CUDA case, we have a specialized kernel as this can be useful for volta + // architectures. https://github.com/huggingface/candle/issues/2136 + match &self.storage { + QStorage::Cuda(s) => { + let s = s.dequantize_f16(self.shape.elem_count())?; + let none = crate::op::BackpropOp::none(); + crate::tensor::from_storage(Storage::Cuda(s), self.shape.clone(), none, false) + .to_device(device) + } + _ => { + let s = self.dequantize(device)?.to_dtype(crate::DType::F16)?; + Ok(s) + } + } + } + + pub fn storage_size_in_bytes(&self) -> usize { + self.storage.size_in_bytes() + } + + pub fn data(&self) -> Result> { + self.storage.data() + } + + pub fn indexed_moe_forward(&self, x: &Tensor, ids: &Tensor) -> Result { + match &self.storage { + QStorage::Cuda(s) => match (&*x.storage(), &*ids.storage()) { + (Storage::Cuda(x_storage), Storage::Cuda(ids_storage)) => { + let (storage, out_shape) = s.indexed_moe_forward( + self.shape(), + x_storage, + x.layout(), + ids_storage, + ids.layout(), + )?; + Ok(crate::tensor::from_storage( + Storage::Cuda(storage), + out_shape, + crate::op::BackpropOp::none(), + false, + )) + } + _ => { + panic!("Non-cuda indexed_moe_forward is not implemented!"); + } + }, + _ => { + panic!("indexed_moe_forward is not implemented in this platform!"); + } + } + } + + pub fn device_ptr(&self) -> Result<*const u8> { + match &self.storage { + QStorage::Cuda(storage) => storage.device_ptr(), + QStorage::Metal(_) | QStorage::Cpu(_) => { + crate::bail!("not implemented"); + } + } + } +} + +#[derive(Clone, Debug)] +pub enum QMatMul { + QTensor(std::sync::Arc), + Tensor(Tensor), + TensorF16(Tensor), +} + +thread_local! { + static DEQUANTIZE_ALL: bool = { + match std::env::var("CANDLE_DEQUANTIZE_ALL") { + Ok(s) => { + !s.is_empty() && s != "0" + }, + Err(_) => false, + } + } +} + +thread_local! { + static DEQUANTIZE_ALL_F16: bool = { + match std::env::var("CANDLE_DEQUANTIZE_ALL_F16") { + Ok(s) => { + !s.is_empty() && s != "0" + }, + Err(_) => false, + } + } +} + +impl QMatMul { + pub fn from_arc(qtensor: std::sync::Arc) -> Result { + let dequantize = match qtensor.dtype() { + GgmlDType::F32 | GgmlDType::F16 | GgmlDType::BF16 => true, + _ => DEQUANTIZE_ALL.with(|b| *b), + }; + let t = if dequantize { + let tensor = qtensor.dequantize(&qtensor.device())?; + Self::Tensor(tensor) + } else if DEQUANTIZE_ALL_F16.with(|b| *b) { + let tensor = qtensor.dequantize_f16(&qtensor.device())?; + Self::TensorF16(tensor) + } else { + Self::QTensor(qtensor) + }; + Ok(t) + } + + pub fn from_qtensor(qtensor: QTensor) -> Result { + Self::from_arc(std::sync::Arc::new(qtensor)) + } + + pub fn dequantize_f16(&self) -> Result { + match self { + Self::QTensor(t) => t.dequantize_f16(&t.device()), + Self::Tensor(t) => t.to_dtype(DType::F16), + Self::TensorF16(t) => Ok(t.clone()), + } + } + + pub fn forward_via_f16(&self, xs: &Tensor) -> Result { + let w = self.dequantize_f16()?; + let in_dtype = xs.dtype(); + let w = match *xs.dims() { + [b1, b2, _, _] => w.broadcast_left((b1, b2))?.t()?, + [bsize, _, _] => w.broadcast_left(bsize)?.t()?, + _ => w.t()?, + }; + xs.to_dtype(DType::F16)?.matmul(&w)?.to_dtype(in_dtype) + } + + pub fn indexed_moe_forward(&self, x: &Tensor, ids: &Tensor) -> Result { + match self { + Self::QTensor(t) => t.indexed_moe_forward(x, ids), + _ => { + panic!("Not implemented!") + } + } + } +} + +impl crate::CustomOp1 for QTensor { + fn name(&self) -> &'static str { + "qmatmul" + } + + fn cpu_fwd( + &self, + storage: &crate::CpuStorage, + layout: &crate::Layout, + ) -> Result<(crate::CpuStorage, Shape)> { + if !layout.is_contiguous() { + crate::bail!("input tensor is not contiguous {layout:?}") + } + let src_shape = layout.shape(); + // self is transposed so n is first then k. + let (n, k) = self.shape.dims2()?; + if src_shape.rank() < 2 { + crate::bail!("input tensor has only one dimension {layout:?}") + } + let mut dst_shape = src_shape.dims().to_vec(); + let last_k = dst_shape.pop().unwrap(); + if last_k != k { + crate::bail!("input tensor {layout:?} incompatible with {:?}", self.shape) + } + dst_shape.push(n); + let dst_shape = Shape::from(dst_shape); + #[allow(clippy::infallible_destructuring_match)] + let self_storage = match &self.storage { + QStorage::Cpu(storage) => storage, + QStorage::Metal(_) | QStorage::Cuda(_) => crate::bail!("Invalid storage"), + }; + match storage.dtype() { + DType::F32 => { + let slice = storage.as_slice::()?; + let slice = + &slice[layout.start_offset()..layout.start_offset() + src_shape.elem_count()]; + let mut dst_storage = vec![0f32; dst_shape.elem_count()]; + self_storage.matmul_t( + (dst_shape.elem_count() / n, k, n), + slice, + &mut dst_storage, + )?; + Ok((crate::CpuStorage::F32(dst_storage), dst_shape)) + } + DType::F16 => { + let slice = storage.as_slice::()?; + let slice = + &slice[layout.start_offset()..layout.start_offset() + src_shape.elem_count()]; + let mut dst_storage = vec![f16::ZERO; dst_shape.elem_count()]; + self_storage.matmul_t_f16( + (dst_shape.elem_count() / n, k, n), + slice, + &mut dst_storage, + )?; + Ok((crate::CpuStorage::F16(dst_storage), dst_shape)) + } + _ => crate::bail!("Expected f32/f16"), + } + } + + fn metal_fwd( + &self, + storage: &crate::MetalStorage, + layout: &crate::Layout, + ) -> Result<(crate::MetalStorage, Shape)> { + let self_storage = match &self.storage { + QStorage::Metal(metal) => metal, + _ => unreachable!("Cannot call metal matmul on non metal QTensor"), + }; + self_storage.fwd(&self.shape, storage, layout) + } + + fn cuda_fwd( + &self, + storage: &crate::CudaStorage, + layout: &crate::Layout, + ) -> Result<(crate::CudaStorage, Shape)> { + let self_storage = match &self.storage { + QStorage::Cuda(cuda) => cuda, + _ => unreachable!("Cannot call cuda matmul on non cuda QTensor"), + }; + self_storage.fwd(&self.shape, storage, layout) + } +} + +impl crate::Module for QMatMul { + fn forward(&self, xs: &Tensor) -> Result { + match self { + Self::QTensor(t) => xs.apply_op1_no_bwd(t.as_ref()), + Self::Tensor(w) => { + let w = match *xs.dims() { + [b1, b2, _, _] => w.broadcast_left((b1, b2))?.t()?, + [bsize, _, _] => w.broadcast_left(bsize)?.t()?, + _ => w.t()?, + }; + xs.matmul(&w) + } + Self::TensorF16(w) => { + let in_dtype = xs.dtype(); + let w = match *xs.dims() { + [b1, b2, _, _] => w.broadcast_left((b1, b2))?.t()?, + [bsize, _, _] => w.broadcast_left(bsize)?.t()?, + _ => w.t()?, + }; + xs.to_dtype(DType::F16)?.matmul(&w)?.to_dtype(in_dtype) + } + } + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/neon.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/neon.rs new file mode 100644 index 0000000000..63196769f5 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/quantized/neon.rs @@ -0,0 +1,615 @@ +use super::k_quants::{ + BlockQ2K, BlockQ3K, BlockQ4K, BlockQ4_0, BlockQ5K, BlockQ6K, BlockQ8K, BlockQ8_0, QK8_0, QK_K, +}; +use byteorder::{ByteOrder, LittleEndian}; + +#[allow(unused_imports)] +#[cfg(target_arch = "arm")] +use core::arch::arm::*; + +#[allow(unused_imports)] +#[cfg(target_arch = "aarch64")] +use core::arch::aarch64::*; + +#[inline(always)] +unsafe fn vdotq_s32(a: int8x16_t, b: int8x16_t) -> int32x4_t { + // TODO: dotprod + let p0 = vmull_s8(vget_low_s8(a), vget_low_s8(b)); + let p1 = vmull_s8(vget_high_s8(a), vget_high_s8(b)); + vaddq_s32(vpaddlq_s16(p0), vpaddlq_s16(p1)) +} + +#[inline(always)] +pub(crate) fn vec_dot_q4_0_q8_0(n: usize, xs: &[BlockQ4_0], ys: &[BlockQ8_0]) -> f32 { + debug_assert!( + n.is_multiple_of(QK8_0), + "vec_dot_q4_0_q8_0: {n} is not divisible by {QK8_0}" + ); + let nb = n / QK8_0; + unsafe { + let mut sumv0 = vdupq_n_f32(0.0f32); + for i in 0..nb { + let x0 = &xs[i]; + let y0 = &ys[i]; + + let m4b = vdupq_n_u8(0x0F); + let s8b = vdupq_n_s8(0x8); + + let v0_0 = vld1q_u8(x0.qs.as_ptr()); + + // 4-bit -> 8-bit + let v0_0l = vreinterpretq_s8_u8(vandq_u8(v0_0, m4b)); + let v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4)); + + // sub 8 + let v0_0ls = vsubq_s8(v0_0l, s8b); + let v0_0hs = vsubq_s8(v0_0h, s8b); + + // load y + let v1_0l = vld1q_s8(y0.qs.as_ptr()); + let v1_0h = vld1q_s8(y0.qs.as_ptr().add(16)); + + let pl0 = vdotq_s32(v0_0ls, v1_0l); + let ph0 = vdotq_s32(v0_0hs, v1_0h); + sumv0 = vmlaq_n_f32( + sumv0, + vcvtq_f32_s32(vaddq_s32(pl0, ph0)), + x0.d.to_f32() * y0.d.to_f32(), + ); + } + vaddvq_f32(sumv0) + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q8_0_q8_0(n: usize, xs: &[BlockQ8_0], ys: &[BlockQ8_0]) -> f32 { + debug_assert!( + n.is_multiple_of(QK8_0), + "vec_dot_q8_0_q8_0: {n} is not divisible by {QK8_0}" + ); + let nb = n / QK8_0; + unsafe { + let mut sumv0 = vdupq_n_f32(0.0f32); + for i in 0..nb { + let x0 = &xs[i]; + let y0 = &ys[i]; + + let x0_0 = vld1q_s8(x0.qs.as_ptr()); + let x0_1 = vld1q_s8(x0.qs.as_ptr().add(16)); + + // load y + let y0_0 = vld1q_s8(y0.qs.as_ptr()); + let y0_1 = vld1q_s8(y0.qs.as_ptr().add(16)); + + let p0 = vdotq_s32(x0_0, y0_0); + let p1 = vdotq_s32(x0_1, y0_1); + + sumv0 = vmlaq_n_f32( + sumv0, + vcvtq_f32_s32(vaddq_s32(p0, p1)), + x0.d.to_f32() * y0.d.to_f32(), + ); + } + vaddvq_f32(sumv0) + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q8k_q8k(n: usize, xs: &[BlockQ8K], ys: &[BlockQ8K]) -> f32 { + debug_assert!( + n.is_multiple_of(QK_K), + "vec_dot_q8k_q8k: {n} is not divisible by {QK_K}" + ); + let mut sumf = 0f32; + for (xs, ys) in xs.iter().zip(ys.iter()) { + unsafe { + let mut sum_i = vdupq_n_s32(0); + let scale = xs.d * ys.d; + let xs = xs.qs.as_ptr(); + let ys = ys.qs.as_ptr(); + for i in (0..QK_K).step_by(16) { + let xs = vld1q_s8(xs.add(i)); + let ys = vld1q_s8(ys.add(i)); + let xy = vdotq_s32(xs, ys); + sum_i = vaddq_s32(sum_i, xy) + } + sumf += vaddvq_s32(sum_i) as f32 * scale + } + } + sumf +} + +#[inline(always)] +pub(crate) fn vec_dot_q6k_q8k(n: usize, xs: &[BlockQ6K], ys: &[BlockQ8K]) -> f32 { + debug_assert!( + n.is_multiple_of(QK_K), + "vec_dot_q6k_q8k: {n} is not divisible by {QK_K}" + ); + let mut sum = 0f32; + unsafe { + let m4b = vdupq_n_u8(0xF); + + let mone = vdupq_n_u8(3); + + for (x, y) in xs.iter().zip(ys.iter()) { + let d_all = x.d.to_f32(); + + let mut q6 = x.ql.as_ptr(); + let mut qh = x.qh.as_ptr(); + let mut q8 = y.qs.as_ptr(); + + let mut scale = x.scales.as_ptr(); + + let q8sums = vld1q_s16_x2(y.bsums.as_ptr()); + let scales = vld1q_s8(scale); + let q6scales = int16x8x2_t( + vmovl_s8(vget_low_s8(scales)), + vmovl_s8(vget_high_s8(scales)), + ); + + let prod = vaddq_s32( + vaddq_s32( + vmull_s16(vget_low_s16(q8sums.0), vget_low_s16(q6scales.0)), + vmull_s16(vget_high_s16(q8sums.0), vget_high_s16(q6scales.0)), + ), + vaddq_s32( + vmull_s16(vget_low_s16(q8sums.1), vget_low_s16(q6scales.1)), + vmull_s16(vget_high_s16(q8sums.1), vget_high_s16(q6scales.1)), + ), + ); + let isum_mins = vaddvq_s32(prod); + + let mut isum = 0i32; + + for _j in 0..QK_K / 128 { + let qhbits = vld1q_u8_x2(qh); + qh = qh.add(32); + let q6bits = vld1q_u8_x4(q6); + q6 = q6.add(64); + let q8bytes = vld1q_s8_x4(q8); + q8 = q8.add(64); + + let q6h_0 = vshlq_n_u8(vandq_u8(mone, qhbits.0), 4); + let q6h_1 = vshlq_n_u8(vandq_u8(mone, qhbits.1), 4); + let shifted = vshrq_n_u8(qhbits.0, 2); + let q6h_2 = vshlq_n_u8(vandq_u8(mone, shifted), 4); + let shifted = vshrq_n_u8(qhbits.1, 2); + let q6h_3 = vshlq_n_u8(vandq_u8(mone, shifted), 4); + + let q6bytes_0 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.0, m4b), q6h_0)); + let q6bytes_1 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.1, m4b), q6h_1)); + let q6bytes_2 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.2, m4b), q6h_2)); + let q6bytes_3 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.3, m4b), q6h_3)); + + let p0 = vdotq_s32(q6bytes_0, q8bytes.0); + let p1 = vdotq_s32(q6bytes_1, q8bytes.1); + let (scale0, scale1) = (*scale as i32, *scale.add(1) as i32); + isum += vaddvq_s32(p0) * scale0 + vaddvq_s32(p1) * scale1; + scale = scale.add(2); + + let p2 = vdotq_s32(q6bytes_2, q8bytes.2); + let p3 = vdotq_s32(q6bytes_3, q8bytes.3); + let (scale0, scale1) = (*scale as i32, *scale.add(1) as i32); + isum += vaddvq_s32(p2) * scale0 + vaddvq_s32(p3) * scale1; + scale = scale.add(2); + + let q8bytes = vld1q_s8_x4(q8); + q8 = q8.add(64); + + let shifted = vshrq_n_u8(qhbits.0, 4); + let q6h_0 = vshlq_n_u8(vandq_u8(mone, shifted), 4); + let shifted = vshrq_n_u8(qhbits.1, 4); + let q6h_1 = vshlq_n_u8(vandq_u8(mone, shifted), 4); + let shifted = vshrq_n_u8(qhbits.0, 6); + let q6h_2 = vshlq_n_u8(vandq_u8(mone, shifted), 4); + let shifted = vshrq_n_u8(qhbits.1, 6); + let q6h_3 = vshlq_n_u8(vandq_u8(mone, shifted), 4); + + let q6bytes_0 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.0, 4), q6h_0)); + let q6bytes_1 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.1, 4), q6h_1)); + let q6bytes_2 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.2, 4), q6h_2)); + let q6bytes_3 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.3, 4), q6h_3)); + + let p0 = vdotq_s32(q6bytes_0, q8bytes.0); + let p1 = vdotq_s32(q6bytes_1, q8bytes.1); + let (scale0, scale1) = (*scale as i32, *scale.add(1) as i32); + isum += vaddvq_s32(p0) * scale0 + vaddvq_s32(p1) * scale1; + scale = scale.add(2); + + let p2 = vdotq_s32(q6bytes_2, q8bytes.2); + let p3 = vdotq_s32(q6bytes_3, q8bytes.3); + let (scale0, scale1) = (*scale as i32, *scale.add(1) as i32); + isum += vaddvq_s32(p2) * scale0 + vaddvq_s32(p3) * scale1; + scale = scale.add(2); + } + sum += d_all * y.d * ((isum - 32 * isum_mins) as f32); + } + } + sum +} + +#[inline(always)] +pub(crate) fn vec_dot_q5k_q8k(n: usize, xs: &[BlockQ5K], ys: &[BlockQ8K]) -> f32 { + debug_assert!( + n.is_multiple_of(QK_K), + "vec_dot_q5k_q8k: {n} is not divisible by {QK_K}" + ); + let mut sumf = 0f32; + let mut utmp = [0u32; 4]; + const KMASK1: u32 = 0x3f3f3f3f; + const KMASK2: u32 = 0x0f0f0f0f; + const KMASK3: u32 = 0x03030303; + + unsafe { + let m4b = vdupq_n_u8(0xF); + let mone = vdupq_n_u8(1); + let mtwo = vdupq_n_u8(2); + + for (x, y) in xs.iter().zip(ys.iter()) { + let d = y.d * x.d.to_f32(); + let dmin = y.d * x.dmin.to_f32(); + + let q8sums = vpaddq_s16( + vld1q_s16(y.bsums.as_ptr()), + vld1q_s16(y.bsums.as_ptr().add(8)), + ); + + LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); + + utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); + let uaux = utmp[1] & KMASK1; + utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); + utmp[2] = uaux; + utmp[0] &= KMASK1; + + let mins8 = vld1_u8((utmp.as_ptr() as *const u8).add(8)); + let mins = vreinterpretq_s16_u16(vmovl_u8(mins8)); + let prod = vaddq_s32( + vmull_s16(vget_low_s16(q8sums), vget_low_s16(mins)), + vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)), + ); + let sumi_mins = vaddvq_s32(prod); + + let mut scales = utmp.as_ptr() as *const u8; + + let mut q5 = x.qs.as_ptr(); + let mut q8 = y.qs.as_ptr(); + + let mut qhbits = vld1q_u8_x2(x.qh.as_ptr()); + + let mut sumi = 0i32; + + for _j in 0..QK_K / 64 { + let q5bits = vld1q_u8_x2(q5); + q5 = q5.add(32); + let q8bytes = vld1q_s8_x4(q8); + q8 = q8.add(64); + + let q5h_0 = vshlq_n_u8(vandq_u8(mone, qhbits.0), 4); + let q5h_1 = vshlq_n_u8(vandq_u8(mone, qhbits.1), 4); + let q5h_2 = vshlq_n_u8(vandq_u8(mtwo, qhbits.0), 3); + let q5h_3 = vshlq_n_u8(vandq_u8(mtwo, qhbits.1), 3); + qhbits.0 = vshrq_n_u8(qhbits.0, 2); + qhbits.1 = vshrq_n_u8(qhbits.1, 2); + + let q5bytes_0 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q5bits.0, m4b), q5h_0)); + let q5bytes_1 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q5bits.1, m4b), q5h_1)); + let q5bytes_2 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.0, 4), q5h_2)); + let q5bytes_3 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.1, 4), q5h_3)); + + let p0 = vdotq_s32(q5bytes_0, q8bytes.0); + let p1 = vdotq_s32(q5bytes_1, q8bytes.1); + sumi += vaddvq_s32(vaddq_s32(p0, p1)) * *scales as i32; + scales = scales.add(1); + + let p2 = vdotq_s32(q5bytes_2, q8bytes.2); + let p3 = vdotq_s32(q5bytes_3, q8bytes.3); + sumi += vaddvq_s32(vaddq_s32(p2, p3)) * *scales as i32; + scales = scales.add(1); + } + sumf += d * sumi as f32 - dmin * sumi_mins as f32; + } + } + sumf +} + +#[inline(always)] +pub(crate) fn vec_dot_q4k_q8k(n: usize, xs: &[BlockQ4K], ys: &[BlockQ8K]) -> f32 { + debug_assert!( + n.is_multiple_of(QK_K), + "vec_dot_q4k_q8k: {n} is not divisible by {QK_K}" + ); + let mut sumf = 0f32; + let mut utmp = [0u32; 4]; + let mut scales = [0u8; 16]; + const KMASK1: u32 = 0x3f3f3f3f; + const KMASK2: u32 = 0x0f0f0f0f; + const KMASK3: u32 = 0x03030303; + + unsafe { + let m4b = vdupq_n_u8(0xF); + + for (x, y) in xs.iter().zip(ys.iter()) { + let d = y.d * x.d.to_f32(); + let dmin = y.d * x.dmin.to_f32(); + + let q8sums = vpaddq_s16( + vld1q_s16(y.bsums.as_ptr()), + vld1q_s16(y.bsums.as_ptr().add(8)), + ); + + LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); + + let mins8 = vld1_u32( + [ + utmp[1] & KMASK1, + ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4), + ] + .as_ptr(), + ); + utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); + utmp[0] &= KMASK1; + + let mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins8))); + let prod = vaddq_s32( + vmull_s16(vget_low_s16(q8sums), vget_low_s16(mins)), + vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)), + ); + sumf -= dmin * vaddvq_s32(prod) as f32; + + LittleEndian::write_u32_into(&utmp, &mut scales); + + let mut q4 = x.qs.as_ptr(); + let mut q8 = y.qs.as_ptr(); + + let mut sumi1 = 0i32; + let mut sumi2 = 0i32; + + for j in 0..QK_K / 64 { + let q4bits = vld1q_u8_x2(q4); + q4 = q4.add(32); + let q8bytes = vld1q_s8_x2(q8); + q8 = q8.add(32); + let q4bytes = int8x16x2_t( + vreinterpretq_s8_u8(vandq_u8(q4bits.0, m4b)), + vreinterpretq_s8_u8(vandq_u8(q4bits.1, m4b)), + ); + let p0 = vdotq_s32(q4bytes.0, q8bytes.0); + let p1 = vdotq_s32(q4bytes.1, q8bytes.1); + sumi1 += vaddvq_s32(vaddq_s32(p0, p1)) * scales[2 * j] as i32; + + let q8bytes = vld1q_s8_x2(q8); + q8 = q8.add(32); + let q4bytes = int8x16x2_t( + vreinterpretq_s8_u8(vshrq_n_u8(q4bits.0, 4)), + vreinterpretq_s8_u8(vshrq_n_u8(q4bits.1, 4)), + ); + let p2 = vdotq_s32(q4bytes.0, q8bytes.0); + let p3 = vdotq_s32(q4bytes.1, q8bytes.1); + sumi2 += vaddvq_s32(vaddq_s32(p2, p3)) * scales[2 * j + 1] as i32; + } + sumf += d * (sumi1 + sumi2) as f32; + } + } + sumf +} + +#[inline(always)] +pub(crate) fn vec_dot_q3k_q8k(n: usize, xs: &[BlockQ3K], ys: &[BlockQ8K]) -> f32 { + debug_assert!( + n.is_multiple_of(QK_K), + "vec_dot_q3k_q8k: {n} is not divisible by {QK_K}" + ); + let mut sumf = 0f32; + let mut utmp = [0u32; 4]; + let mut aux = [0u32; 3]; + const KMASK1: u32 = 0x03030303; + const KMASK2: u32 = 0x0f0f0f0f; + + unsafe { + let m3b = vdupq_n_u8(0x3); + let m0 = vdupq_n_u8(1); + let m1 = vshlq_n_u8(m0, 1); + let m2 = vshlq_n_u8(m0, 2); + let m3 = vshlq_n_u8(m0, 3); + for (x, y) in xs.iter().zip(ys.iter()) { + let d = y.d * x.d.to_f32(); + let mut q3 = x.qs.as_ptr(); + let qh = x.hmask.as_ptr(); + let mut q8 = y.qs.as_ptr(); + + let mut qhbits = vld1q_u8_x2(qh); + + let mut isum = 0i32; + + // Set up scales + LittleEndian::read_u32_into(&x.scales, &mut aux); + + utmp[3] = ((aux[1] >> 4) & KMASK2) | (((aux[2] >> 6) & KMASK1) << 4); + utmp[2] = ((aux[0] >> 4) & KMASK2) | (((aux[2] >> 4) & KMASK1) << 4); + utmp[1] = (aux[1] & KMASK2) | (((aux[2] >> 2) & KMASK1) << 4); + utmp[0] = (aux[0] & KMASK2) | ((aux[2] & KMASK1) << 4); + + let mut scale = utmp.as_mut_ptr() as *mut i8; + for j in 0..16 { + *scale.add(j) -= 32i8 + } + + for j in 0..QK_K / 128 { + let q3bits = vld1q_u8_x2(q3); + q3 = q3.add(32); + let q8bytes_1 = vld1q_s8_x4(q8); + q8 = q8.add(64); + let q8bytes_2 = vld1q_s8_x4(q8); + q8 = q8.add(64); + + let q3h_0 = vshlq_n_u8(vbicq_u8(m0, qhbits.0), 2); + let q3h_1 = vshlq_n_u8(vbicq_u8(m0, qhbits.1), 2); + let q3h_2 = vshlq_n_u8(vbicq_u8(m1, qhbits.0), 1); + let q3h_3 = vshlq_n_u8(vbicq_u8(m1, qhbits.1), 1); + + let q3bytes_0 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(q3bits.0, m3b)), + vreinterpretq_s8_u8(q3h_0), + ); + let q3bytes_1 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(q3bits.1, m3b)), + vreinterpretq_s8_u8(q3h_1), + ); + let q3bytes_2 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.0, 2), m3b)), + vreinterpretq_s8_u8(q3h_2), + ); + let q3bytes_3 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.1, 2), m3b)), + vreinterpretq_s8_u8(q3h_3), + ); + + let p0 = vdotq_s32(q3bytes_0, q8bytes_1.0); + let p1 = vdotq_s32(q3bytes_1, q8bytes_1.1); + let p2 = vdotq_s32(q3bytes_2, q8bytes_1.2); + let p3 = vdotq_s32(q3bytes_3, q8bytes_1.3); + isum += vaddvq_s32(p0) * *scale as i32 + + vaddvq_s32(p1) * *scale.add(1) as i32 + + vaddvq_s32(p2) * *scale.add(2) as i32 + + vaddvq_s32(p3) * *scale.add(3) as i32; + scale = scale.add(4); + + let q3h_0 = vbicq_u8(m2, qhbits.0); + let q3h_1 = vbicq_u8(m2, qhbits.1); + let q3h_2 = vshrq_n_u8(vbicq_u8(m3, qhbits.0), 1); + let q3h_3 = vshrq_n_u8(vbicq_u8(m3, qhbits.1), 1); + + let q3bytes_0 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.0, 4), m3b)), + vreinterpretq_s8_u8(q3h_0), + ); + let q3bytes_1 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.1, 4), m3b)), + vreinterpretq_s8_u8(q3h_1), + ); + let q3bytes_2 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.0, 6), m3b)), + vreinterpretq_s8_u8(q3h_2), + ); + let q3bytes_3 = vsubq_s8( + vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.1, 6), m3b)), + vreinterpretq_s8_u8(q3h_3), + ); + + let p0 = vdotq_s32(q3bytes_0, q8bytes_2.0); + let p1 = vdotq_s32(q3bytes_1, q8bytes_2.1); + let p2 = vdotq_s32(q3bytes_2, q8bytes_2.2); + let p3 = vdotq_s32(q3bytes_3, q8bytes_2.3); + isum += vaddvq_s32(p0) * *scale as i32 + + vaddvq_s32(p1) * *scale.add(1) as i32 + + vaddvq_s32(p2) * *scale.add(2) as i32 + + vaddvq_s32(p3) * *scale.add(3) as i32; + scale = scale.add(4); + + if j == 0 { + qhbits.0 = vshrq_n_u8(qhbits.0, 4); + qhbits.1 = vshrq_n_u8(qhbits.1, 4); + } + } + sumf += d * isum as f32; + } + } + sumf +} + +#[inline(always)] +pub(crate) fn vec_dot_q2k_q8k(n: usize, xs: &[BlockQ2K], ys: &[BlockQ8K]) -> f32 { + debug_assert!( + n.is_multiple_of(QK_K), + "vec_dot_q2k_q8k: {n} is not divisible by {QK_K}" + ); + let mut sumf = 0f32; + let mut aux = [0u8; 16]; + + unsafe { + let m3 = vdupq_n_u8(0x3); + let m4 = vdupq_n_u8(0xF); + + for (x, y) in xs.iter().zip(ys.iter()) { + let d = y.d * x.d.to_f32(); + let dmin = -y.d * x.dmin.to_f32(); + + let mut q2 = x.qs.as_ptr(); + let mut q8 = y.qs.as_ptr(); + let sc = x.scales.as_ptr(); + + let mins_and_scales = vld1q_u8(sc); + let scales = vandq_u8(mins_and_scales, m4); + vst1q_u8(aux.as_mut_ptr(), scales); + + let mins = vshrq_n_u8(mins_and_scales, 4); + let q8sums = vld1q_s16_x2(y.bsums.as_ptr()); + let mins16 = int16x8x2_t( + vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(mins))), + vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(mins))), + ); + let s0 = vaddq_s32( + vmull_s16(vget_low_s16(mins16.0), vget_low_s16(q8sums.0)), + vmull_s16(vget_high_s16(mins16.0), vget_high_s16(q8sums.0)), + ); + let s1 = vaddq_s32( + vmull_s16(vget_low_s16(mins16.1), vget_low_s16(q8sums.1)), + vmull_s16(vget_high_s16(mins16.1), vget_high_s16(q8sums.1)), + ); + sumf += dmin * vaddvq_s32(vaddq_s32(s0, s1)) as f32; + + let mut isum = 0i32; + let mut is = 0usize; + + // TODO: dotprod + for _j in 0..QK_K / 128 { + let q2bits = vld1q_u8_x2(q2); + q2 = q2.add(32); + + let q8bytes = vld1q_s8_x2(q8); + q8 = q8.add(32); + let mut q2bytes = int8x16x2_t( + vreinterpretq_s8_u8(vandq_u8(q2bits.0, m3)), + vreinterpretq_s8_u8(vandq_u8(q2bits.1, m3)), + ); + isum += multiply_accum_with_scale(&aux, is, 0, q2bytes, q8bytes); + + let q8bytes = vld1q_s8_x2(q8); + q8 = q8.add(32); + q2bytes.0 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.0, 2), m3)); + q2bytes.1 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.1, 2), m3)); + isum += multiply_accum_with_scale(&aux, is, 2, q2bytes, q8bytes); + + let q8bytes = vld1q_s8_x2(q8); + q8 = q8.add(32); + q2bytes.0 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.0, 4), m3)); + q2bytes.1 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.1, 4), m3)); + isum += multiply_accum_with_scale(&aux, is, 4, q2bytes, q8bytes); + + let q8bytes = vld1q_s8_x2(q8); + q8 = q8.add(32); + q2bytes.0 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.0, 6), m3)); + q2bytes.1 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.1, 6), m3)); + isum += multiply_accum_with_scale(&aux, is, 6, q2bytes, q8bytes); + + is += 8; + } + sumf += d * isum as f32; + } + } + sumf +} + +#[inline(always)] +unsafe fn multiply_accum_with_scale( + aux: &[u8; 16], + is: usize, + index: usize, + q2bytes: int8x16x2_t, + q8bytes: int8x16x2_t, +) -> i32 { + let p1 = vdotq_s32(q2bytes.0, q8bytes.0); + let p2 = vdotq_s32(q2bytes.1, q8bytes.1); + vaddvq_s32(p1) * aux[is + index] as i32 + vaddvq_s32(p2) * aux[is + 1 + index] as i32 +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/simd128.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/simd128.rs new file mode 100644 index 0000000000..4c02f9919e --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/quantized/simd128.rs @@ -0,0 +1,418 @@ +use super::k_quants::{BlockQ2K, BlockQ4K, BlockQ4_0, BlockQ6K, BlockQ8K, BlockQ8_0, QK8_0, QK_K}; +use byteorder::{ByteOrder, LittleEndian}; +use half::f16; + +use core::arch::wasm32::*; + +#[inline(always)] +pub(crate) fn vec_dot_q4_0_q8_0(n: usize, xs: &[BlockQ4_0], ys: &[BlockQ8_0]) -> f32 { + debug_assert!( + n.is_multiple_of(QK8_0), + "vec_dot_q4_0_q8_0: {n} is not divisible by {QK8_0}" + ); + unsafe { + let mut acc = f32x4_splat(0.0f32); + for (x, y) in xs.iter().zip(ys.iter()) { + let x1234 = v128_load(x.qs.as_ptr() as *const v128); + let x12 = v128_and(x1234, u8x16_splat(0x0F)); + let x12 = i8x16_sub(x12, i8x16_splat(8)); + let x34 = u8x16_shr(x1234, 4); + let x34 = i8x16_sub(x34, i8x16_splat(8)); + + let x1 = i16x8_extend_low_i8x16(x12); + let y1 = i16x8_load_extend_i8x8(y.qs.as_ptr()); + let sum_xy = i32x4_dot_i16x8(x1, y1); + + let x2 = i16x8_extend_high_i8x16(x12); + let y2 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(8)); + let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x2, y2)); + + let x3 = i16x8_extend_low_i8x16(x34); + let y3 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(16)); + let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x3, y3)); + + let x4 = i16x8_extend_high_i8x16(x34); + let y4 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(24)); + let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x4, y4)); + + let sum_xy = f32x4_convert_i32x4(sum_xy); + + // f32x4_relaxed_madd is nightly only. + let d = f32x4_splat(f16::to_f32(x.d) * f16::to_f32(y.d)); + let scaled = f32x4_mul(sum_xy, d); + acc = f32x4_add(acc, scaled) + } + let res = f32x4_extract_lane::<0>(acc) + + f32x4_extract_lane::<1>(acc) + + f32x4_extract_lane::<2>(acc) + + f32x4_extract_lane::<3>(acc); + res + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q8_0_q8_0(n: usize, xs: &[BlockQ8_0], ys: &[BlockQ8_0]) -> f32 { + debug_assert!( + n.is_multiple_of(QK8_0), + "vec_dot_q8_0_q8_0: {n} is not divisible by {QK8_0}" + ); + unsafe { + let mut acc = f32x4_splat(0.0f32); + for (x, y) in xs.iter().zip(ys.iter()) { + let x1 = i16x8_load_extend_i8x8(x.qs.as_ptr()); + let y1 = i16x8_load_extend_i8x8(y.qs.as_ptr()); + let sum_xy = i32x4_dot_i16x8(x1, y1); + + let x2 = i16x8_load_extend_i8x8(x.qs.as_ptr().add(8)); + let y2 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(8)); + let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x2, y2)); + + let x3 = i16x8_load_extend_i8x8(x.qs.as_ptr().add(16)); + let y3 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(16)); + let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x3, y3)); + + let x4 = i16x8_load_extend_i8x8(x.qs.as_ptr().add(24)); + let y4 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(24)); + let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x4, y4)); + + let sum_xy = f32x4_convert_i32x4(sum_xy); + + // f32x4_relaxed_madd is nightly only. + let d = f32x4_splat(f16::to_f32(x.d) * f16::to_f32(y.d)); + let scaled = f32x4_mul(sum_xy, d); + acc = f32x4_add(acc, scaled) + } + let res = f32x4_extract_lane::<0>(acc) + + f32x4_extract_lane::<1>(acc) + + f32x4_extract_lane::<2>(acc) + + f32x4_extract_lane::<3>(acc); + res + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q2k_q8k(n: usize, xs: &[BlockQ2K], ys: &[BlockQ8K]) -> f32 { + debug_assert!( + n.is_multiple_of(QK_K), + "vec_dot_q2k_q8k: {n} is not divisible by {QK_K}" + ); + unsafe { + let mut sumf = f32x4_splat(0f32); + for (x, y) in xs.iter().zip(ys.iter()) { + let mut q2: &[_] = &x.qs; + let mut q8: &[_] = &y.qs; + let sc = &x.scales; + + let mut summs = i32x4_splat(0); + for i in (0..(QK_K / 16)).step_by(4) { + let bsums = i32x4_load_extend_i16x4(y.bsums.as_ptr().add(i)); + let scales = i32x4_shr( + i32x4( + sc[i] as i32, + sc[i + 1] as i32, + sc[i + 2] as i32, + sc[i + 3] as i32, + ), + 4, + ); + summs = i32x4_add(summs, i32x4_mul(bsums, scales)) + } + let summs = f32x4_convert_i32x4(summs); + + let dall = y.d * x.d.to_f32(); + let dmin = y.d * x.dmin.to_f32(); + + let mut isum = i32x4_splat(0); + let mut is = 0; + for _ in 0..(QK_K / 128) { + let mut shift = 0; + for _ in 0..4 { + let d = (sc[is] & 0xF) as i32; + is += 1; + let mut isuml = i16x8_splat(0); + for l in (0..16).step_by(8) { + let q8 = i16x8_load_extend_i8x8(q8.as_ptr().add(l)); + let q2 = i16x8_load_extend_u8x8(q2.as_ptr().add(l)); + let q2 = v128_and(i16x8_shr(q2, shift), i16x8_splat(3)); + isuml = i16x8_add(isuml, i16x8_mul(q2, q8)) + } + let dd = i32x4_splat(d); + isum = i32x4_add(isum, i32x4_mul(i32x4_extend_low_i16x8(isuml), dd)); + isum = i32x4_add(isum, i32x4_mul(i32x4_extend_high_i16x8(isuml), dd)); + let d = (sc[is] & 0xF) as i32; + is += 1; + let mut isuml = i16x8_splat(0); + for l in (16..32).step_by(8) { + let q8 = i16x8_load_extend_i8x8(q8.as_ptr().add(l)); + let q2 = i16x8_load_extend_u8x8(q2.as_ptr().add(l)); + let q2 = v128_and(i16x8_shr(q2, shift), i16x8_splat(3)); + isuml = i16x8_add(isuml, i16x8_mul(q2, q8)) + } + let dd = i32x4_splat(d); + isum = i32x4_add(isum, i32x4_mul(i32x4_extend_low_i16x8(isuml), dd)); + isum = i32x4_add(isum, i32x4_mul(i32x4_extend_high_i16x8(isuml), dd)); + shift += 2; + // adjust the indexing + q8 = &q8[32..]; + } + // adjust the indexing + q2 = &q2[32..]; + } + let isum = f32x4_convert_i32x4(isum); + sumf = f32x4_add( + sumf, + f32x4_sub( + f32x4_mul(isum, f32x4_splat(dall)), + f32x4_mul(summs, f32x4_splat(dmin)), + ), + ); + } + let sumf = f32x4_extract_lane::<0>(sumf) + + f32x4_extract_lane::<1>(sumf) + + f32x4_extract_lane::<2>(sumf) + + f32x4_extract_lane::<3>(sumf); + sumf + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q4k_q8k(n: usize, xs: &[BlockQ4K], ys: &[BlockQ8K]) -> f32 { + debug_assert!( + n.is_multiple_of(QK_K), + "vec_dot_q4k_q8k: {n} is not divisible by {QK_K}" + ); + const KMASK1: u32 = 0x3f3f3f3f; + const KMASK2: u32 = 0x0f0f0f0f; + const KMASK3: u32 = 0x03030303; + + let mut utmp: [u32; 4] = [0; 4]; + let mut scales: [u8; 8] = [0; 8]; + let mut mins: [u8; 8] = [0; 8]; + + let mut aux8: [u8; QK_K] = [0; QK_K]; + let mut sums = f32x4_splat(0f32); + unsafe { + for (y, x) in ys.iter().zip(xs.iter()) { + let q4 = &x.qs; + let q8 = &y.qs; + + for j in 0..QK_K / 64 { + let q4_1 = v128_load(q4.as_ptr().add(32 * j) as *const v128); + let q4_2 = v128_load(q4.as_ptr().add(32 * j + 16) as *const v128); + v128_store( + aux8.as_mut_ptr().add(64 * j) as *mut v128, + v128_and(q4_1, u8x16_splat(0x0F)), + ); + v128_store( + aux8.as_mut_ptr().add(64 * j + 16) as *mut v128, + v128_and(q4_2, u8x16_splat(0x0F)), + ); + v128_store( + aux8.as_mut_ptr().add(64 * j + 32) as *mut v128, + u8x16_shr(q4_1, 4), + ); + v128_store( + aux8.as_mut_ptr().add(64 * j + 48) as *mut v128, + u8x16_shr(q4_2, 4), + ); + } + + LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); + + utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); + let uaux = utmp[1] & KMASK1; + utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); + utmp[2] = uaux; + utmp[0] &= KMASK1; + + //extract scales and mins + LittleEndian::write_u32_into(&utmp[0..2], &mut scales); + LittleEndian::write_u32_into(&utmp[2..4], &mut mins); + + let mut sumi = i32x4_splat(0); + for j in (0..QK_K / 16).step_by(4) { + let bsums = i32x4_load_extend_i16x4(y.bsums.as_ptr().add(j)); + let (m1, m2) = (mins[j / 2] as i32, mins[j / 2 + 1] as i32); + let mins = i32x4(m1, m1, m2, m2); + sumi = i32x4_add(sumi, i32x4_mul(bsums, mins)); + } + + let mut aux32 = i32x4_splat(0i32); + for (scale_i, scale) in scales.iter().enumerate() { + let scale = i32x4_splat(*scale as i32); + for j in 0..4 { + let i = 32 * scale_i + 8 * j; + let q8 = i16x8_load_extend_i8x8(q8.as_ptr().add(i)); + let aux8 = i16x8_load_extend_u8x8(aux8.as_ptr().add(i)); + let aux16 = i16x8_mul(q8, aux8); + aux32 = i32x4_add(aux32, i32x4_mul(scale, i32x4_extend_low_i16x8(aux16))); + aux32 = i32x4_add(aux32, i32x4_mul(scale, i32x4_extend_high_i16x8(aux16))); + } + } + let aux32 = f32x4_convert_i32x4(aux32); + let d = f32x4_splat(x.d.to_f32() * y.d); + sums = f32x4_add(sums, f32x4_mul(aux32, d)); + let dmin = x.dmin.to_f32() * y.d; + let dmin = f32x4_splat(dmin); + let sumi = f32x4_convert_i32x4(sumi); + sums = f32x4_sub(sums, f32x4_mul(sumi, dmin)); + } + let sums = f32x4_extract_lane::<0>(sums) + + f32x4_extract_lane::<1>(sums) + + f32x4_extract_lane::<2>(sums) + + f32x4_extract_lane::<3>(sums); + sums + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q6k_q8k(n: usize, xs: &[BlockQ6K], ys: &[BlockQ8K]) -> f32 { + debug_assert!( + n.is_multiple_of(QK_K), + "vec_dot_q6k_q8k: {n} is not divisible by {QK_K}" + ); + let mut aux8 = [0i8; QK_K]; + unsafe { + let mut sums = f32x4_splat(0f32); + + for (x, y) in xs.iter().zip(ys.iter()) { + let q4 = &x.ql; + let qh = &x.qh; + let q8 = &y.qs; + let mut aux32 = f32x4_splat(0f32); + + for j in (0..QK_K).step_by(128) { + let aux8 = aux8.as_mut_ptr().add(j); + let q4 = &q4.as_ptr().add(j / 2); + let qh = &qh.as_ptr().add(j / 4); + for l in (0..32).step_by(16) { + // aux8[l] = (((q4[l] & 0xF) | ((qh[l] & 3) << 4)) as i32 - 32) as i8; + let a8 = v128_or( + v128_and(v128_load(q4.add(l) as *const v128), u8x16_splat(0xF)), + u8x16_shl( + v128_and(v128_load(qh.add(l) as *const v128), u8x16_splat(3)), + 4, + ), + ); + let a8_low = i16x8_sub(i16x8_extend_low_u8x16(a8), i16x8_splat(32)); + let a8_high = i16x8_sub(i16x8_extend_high_u8x16(a8), i16x8_splat(32)); + v128_store( + aux8.add(l) as *mut v128, + i8x16_narrow_i16x8(a8_low, a8_high), + ); + + // aux8[l + 32] = + // (((q4[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) as i32 - 32) as i8; + let a8 = v128_or( + v128_and(v128_load(q4.add(l + 32) as *const v128), u8x16_splat(0xF)), + u8x16_shl( + v128_and( + u8x16_shr(v128_load(qh.add(l) as *const v128), 2), + u8x16_splat(3), + ), + 4, + ), + ); + let a8_low = i16x8_sub(i16x8_extend_low_u8x16(a8), i16x8_splat(32)); + let a8_high = i16x8_sub(i16x8_extend_high_u8x16(a8), i16x8_splat(32)); + v128_store( + aux8.add(l + 32) as *mut v128, + i8x16_narrow_i16x8(a8_low, a8_high), + ); + + // aux8[l + 64] = (((q4[l] >> 4) | (((qh[l] >> 4) & 3) << 4)) as i32 - 32) as i8; + let a8 = v128_or( + u8x16_shr(v128_load(q4.add(l) as *const v128), 4), + u8x16_shl( + v128_and( + u8x16_shr(v128_load(qh.add(l) as *const v128), 4), + u8x16_splat(3), + ), + 4, + ), + ); + let a8_low = i16x8_sub(i16x8_extend_low_u8x16(a8), i16x8_splat(32)); + let a8_high = i16x8_sub(i16x8_extend_high_u8x16(a8), i16x8_splat(32)); + v128_store( + aux8.add(l + 64) as *mut v128, + i8x16_narrow_i16x8(a8_low, a8_high), + ); + + // aux8[l + 96] = + // (((q4[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) as i32 - 32) as i8; + let a8 = v128_or( + u8x16_shr(v128_load(q4.add(l + 32) as *const v128), 4), + u8x16_shl( + v128_and( + u8x16_shr(v128_load(qh.add(l) as *const v128), 6), + u8x16_splat(3), + ), + 4, + ), + ); + let a8_low = i16x8_sub(i16x8_extend_low_u8x16(a8), i16x8_splat(32)); + let a8_high = i16x8_sub(i16x8_extend_high_u8x16(a8), i16x8_splat(32)); + v128_store( + aux8.add(l + 96) as *mut v128, + i8x16_narrow_i16x8(a8_low, a8_high), + ); + } + } + + for (j, &scale) in x.scales.iter().enumerate() { + let scale = f32x4_splat(scale as f32); + for offset in [0, 8] { + let aux16 = i16x8_mul( + i16x8_load_extend_i8x8(q8.as_ptr().add(16 * j + offset)), + i16x8_load_extend_i8x8(aux8.as_ptr().add(16 * j + offset)), + ); + aux32 = f32x4_add( + aux32, + f32x4_mul(f32x4_convert_i32x4(i32x4_extend_low_i16x8(aux16)), scale), + ); + aux32 = f32x4_add( + aux32, + f32x4_mul(f32x4_convert_i32x4(i32x4_extend_high_i16x8(aux16)), scale), + ); + } + } + + let d = f32x4_splat(x.d.to_f32() * y.d); + sums = f32x4_add(sums, f32x4_mul(aux32, d)); + } + let sums = f32x4_extract_lane::<0>(sums) + + f32x4_extract_lane::<1>(sums) + + f32x4_extract_lane::<2>(sums) + + f32x4_extract_lane::<3>(sums); + sums + } +} + +#[inline(always)] +pub(crate) fn vec_dot_q8k_q8k(n: usize, xs: &[BlockQ8K], ys: &[BlockQ8K]) -> f32 { + debug_assert!( + n.is_multiple_of(QK_K), + "vec_dot_q8k_q8k: {n} is not divisible by {QK_K}" + ); + unsafe { + let mut acc = f32x4_splat(0.0f32); + for (xs, ys) in xs.iter().zip(ys.iter()) { + let x_qs = xs.qs.as_ptr(); + let y_qs = ys.qs.as_ptr(); + let mut sumi = i32x4_splat(0); + for j in (0..QK_K).step_by(8) { + let xs = i16x8_load_extend_i8x8(x_qs.add(j)); + let ys = i16x8_load_extend_i8x8(y_qs.add(j)); + let sum_xy = i32x4_dot_i16x8(xs, ys); + sumi = i32x4_add(sumi, sum_xy) + } + let d = f32x4_splat(xs.d * ys.d); + acc = f32x4_add(acc, f32x4_mul(f32x4_convert_i32x4(sumi), d)) + } + let res = f32x4_extract_lane::<0>(acc) + + f32x4_extract_lane::<1>(acc) + + f32x4_extract_lane::<2>(acc) + + f32x4_extract_lane::<3>(acc); + res + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/utils.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/utils.rs new file mode 100644 index 0000000000..9dd9c0918a --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/quantized/utils.rs @@ -0,0 +1,562 @@ +pub(super) fn nearest_int(v: f32) -> i32 { + v.round() as i32 +} + +/// Validates that the input and output are the right size and returns an iterator which maps each +/// input region `xs` to its corresponding output block in `ys`. Each output region is guaranteed +/// to be `T::BLCK_SIZE` long. +pub(super) fn group_for_quantization<'a, 'b, T: super::k_quants::GgmlType>( + xs: &'b [f32], + ys: &'a mut [T], +) -> Vec<(&'a mut T, &'b [f32])> { + let block_size = T::BLCK_SIZE; + let dtype = T::DTYPE; + + let expected_blocks = xs.len() / block_size; + let actual_blocks = ys.len(); + + // Validate that the input is the right size + debug_assert_eq!( + expected_blocks, + actual_blocks, + "quantize {dtype:?}: expected {expected_blocks} blocks but only {actual_blocks} were provided!"); + + ys.iter_mut().zip(xs.chunks_exact(block_size)).collect() +} + +/// Validates that the input and output are the right size and returns an iterator which maps each +/// input block `xs` to its corresponding output region in `ys`. Each output region is guaranteed +/// to be `T::BLCK_SIZE` long. +pub(super) fn group_for_dequantization<'a, 'b, T: super::k_quants::GgmlType>( + xs: &'a [T], + ys: &'b mut [f32], +) -> Vec<(&'a T, &'b mut [f32])> { + let block_size = T::BLCK_SIZE; + let dtype = T::DTYPE; + + let actual_output_len = ys.len(); + let expected_output_len = xs.len() * block_size; + // Validate that the output is the right size + debug_assert_eq!( + expected_output_len, + actual_output_len, + "dequantize {dtype:?}: ys (len = {actual_output_len}) does not match the expected length of {expected_output_len}!" + ); + + // Zip the blocks and outputs together + xs.iter().zip(ys.chunks_exact_mut(block_size)).collect() +} + +pub(super) fn get_scale_min_k4(j: usize, q: &[u8]) -> (u8, u8) { + if j < 4 { + let d = q[j] & 63; + let m = q[j + 4] & 63; + (d, m) + } else { + let d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4); + let m = (q[j + 4] >> 4) | ((q[j] >> 6) << 4); + (d, m) + } +} + +pub(super) unsafe fn make_qx_quants( + n: usize, + nmax: i32, + x: *const f32, + ls: *mut i8, + rmse_type: i32, + qw: *const f32, +) -> f32 { + let mut max = 0f32; + let mut amax = 0f32; + for i in 0..n { + let x = *x.add(i); + let ax = x.abs(); + if ax > amax { + amax = ax; + max = x; + } + } + if amax == 0. { + // all zero + for i in 0..n { + *ls.add(i) = 0; + } + return 0.; + } + let mut iscale = -(nmax as f32) / max; + if rmse_type == 0 { + for i in 0..n { + let x = *x.add(i); + let l = nearest_int(iscale * x); + *ls.add(i) = (nmax + l.clamp(-nmax, nmax - 1)) as i8; + } + return 1.0 / iscale; + } + let weight_type = rmse_type % 2; + let mut sumlx = 0f32; + let mut suml2 = 0f32; + for i in 0..n { + let x = *x.add(i); + let l = nearest_int(iscale * x); + let l = l.clamp(-nmax, nmax - 1); + *ls.add(i) = (l + nmax) as i8; + let w = if !qw.is_null() { + *qw.add(i) + } else if weight_type == 1 { + x * x + } else { + 1.0 + }; + let l = l as f32; + sumlx += w * x * l; + suml2 += w * l * l; + } + let mut scale = sumlx / suml2; + let mut best = scale * sumlx; + for _itry in 0..3 { + let iscale = 1.0 / scale; + let mut slx = 0f32; + let mut sl2 = 0f32; + let mut changed = false; + for i in 0..n { + let x = *x.add(i); + let l = nearest_int(iscale * x); + let l = l.clamp(-nmax, nmax - 1); + if l + nmax != *ls.add(i) as i32 { + changed = true; + } + let w = if !qw.is_null() { + *qw.add(i) + } else if weight_type == 1 { + x * x + } else { + 1.0 + }; + let l = l as f32; + slx += w * x * l; + sl2 += w * l * l; + } + if !changed || sl2 == 0.0 || slx * slx <= best * sl2 { + break; + } + for i in 0..n { + let x = *x.add(i); + let l = nearest_int(iscale * x); + *ls.add(i) = (nmax + l.clamp(-nmax, nmax - 1)) as i8; + } + sumlx = slx; + suml2 = sl2; + scale = sumlx / suml2; + best = scale * sumlx; + } + for _itry in 0..5 { + let mut n_changed = 0; + for i in 0..n { + let x = *x.add(i); + let w = if !qw.is_null() { + *qw.add(i) + } else if weight_type == 1 { + x * x + } else { + 1.0 + }; + let l = *ls.add(i) as i32 - nmax; + let mut slx = sumlx - w * x * l as f32; + if slx > 0. { + let mut sl2 = suml2 - w * l as f32 * l as f32; + let new_l = nearest_int(x * sl2 / slx); + let new_l = new_l.clamp(-nmax, nmax - 1); + if new_l != l { + slx += w * x * new_l as f32; + sl2 += w * new_l as f32 * new_l as f32; + if sl2 > 0. && slx * slx * suml2 > sumlx * sumlx * sl2 { + *ls.add(i) = (nmax + new_l) as i8; + sumlx = slx; + suml2 = sl2; + scale = sumlx / suml2; + best = scale * sumlx; + n_changed += 1; + } + } + } + } + if n_changed == 0 { + break; + } + } + if rmse_type < 3 { + return scale; + } + for is in -4..4 { + if is == 0 { + continue; + } + iscale = -(nmax as f32 + 0.1f32 * is as f32) / max; + let mut sumlx = 0.; + let mut suml2 = 0.; + for i in 0..n { + let x = *x.add(i); + let l = nearest_int(iscale * x); + let l = l.clamp(-nmax, nmax - 1); + let w = if !qw.is_null() { + *qw.add(i) + } else if weight_type == 1 { + x * x + } else { + 1.0 + }; + let l = l as f32; + sumlx += w * x * l; + suml2 += w * l * l; + } + if suml2 > 0. && sumlx * sumlx > best * suml2 { + for i in 0..n { + let x = *x.add(i); + let l = nearest_int(iscale * x); + *ls.add(i) = (nmax + l.clamp(-nmax, nmax - 1)) as i8; + } + scale = sumlx / suml2; + best = scale * sumlx; + } + } + scale +} + +// https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L224 +pub(super) fn make_qkx1_quants(nmax: i32, ntry: usize, x: &[f32]) -> (f32, f32) { + let n = x.len(); + let mut l = vec![0; n]; + // Get min/max + let min = *x + .iter() + .take(n) + .min_by(|a, b| a.total_cmp(b)) + .unwrap_or(&x[0]); + let max = *x.iter().max_by(|a, b| a.total_cmp(b)).unwrap_or(&x[0]); + + // If min == max, all values are the same => nothing to do here + if max == min { + return (0.0, 0.0); + } + + // Ensure min <= 0.0 + let mut min = min.min(0.); + + // Compute scale and inverse scale + let mut iscale = nmax as f32 / (max - min); + let mut scale = 1.0 / iscale; + + for _ in 0..ntry { + let mut sumlx = 0.0; + let mut suml2 = 0; + let mut did_change = false; + + for (i, value) in x.iter().enumerate().take(n) { + let li = nearest_int(iscale * (value - min)).clamp(0, nmax); + let clamped_li = li as u8; + if clamped_li != l[i] { + l[i] = clamped_li; + did_change = true; + } + sumlx += (value - min) * li as f32; + suml2 += li * li; + } + scale = sumlx / suml2 as f32; + + let sum: f32 = x + .iter() + .take(n) + .zip(l.iter().take(n)) + .map(|(xi, &li)| xi - scale * li as f32) + .sum(); + + min = sum / n as f32; + if min > 0.0 { + min = 0.0; + } + iscale = 1.0 / scale; + if !did_change { + break; + } + } + (scale, -min) +} + +// https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L165 +pub(super) fn make_q3_quants(x: &[f32], nmax: i32, do_rmse: bool) -> f32 { + let n = x.len(); + let mut l = vec![0i8; n]; + + let mut max = 0.0; + let mut amax = 0.0; + for &xi in x.iter().take(n) { + let ax = xi.abs(); + if ax > amax { + amax = ax; + max = xi; + } + } + + if amax == 0.0 { + return 0.0; + } + + let iscale = -(nmax as f32) / max; + if do_rmse { + let mut sumlx = 0.0; + let mut suml2 = 0.0; + for i in 0..n { + let li = (iscale * x[i]).round() as i32; + let li = li.clamp(-nmax, nmax - 1); + l[i] = li as i8; + let w = x[i] * x[i]; + sumlx += w * x[i] * li as f32; + suml2 += w * (li * li) as f32; + } + for _ in 0..5 { + let mut n_changed = 0; + for i in 0..n { + let w = x[i] * x[i]; + let mut slx = sumlx - w * x[i] * l[i] as f32; + if slx > 0.0 { + let mut sl2 = suml2 - w * (l[i] as i32 * l[i] as i32) as f32; + let mut new_l = (x[i] * sl2 / slx).round() as i32; + new_l = new_l.clamp(-nmax, nmax - 1); + if new_l != l[i] as i32 { + slx += w * x[i] * new_l as f32; + sl2 += w * (new_l * new_l) as f32; + if sl2 > 0.0 && slx * slx * suml2 > sumlx * sumlx * sl2 { + l[i] = new_l as i8; + sumlx = slx; + suml2 = sl2; + n_changed += 1; + } + } + } + } + if n_changed == 0 { + break; + } + } + for li in l.iter_mut() { + *li += nmax as i8; + } + return sumlx / suml2; + } + for i in 0..n { + let li = (iscale * x[i]).round() as i32; + l[i] = (li.clamp(-nmax, nmax - 1) + nmax) as i8; + } + 1.0 / iscale +} + +// https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/ggml/src/ggml-quants.c#L744 +/// (scale, min) +pub(super) fn make_qkx3_quants( + nmax: i32, + x: &[f32], + weights: Option<&[f32]>, + rmin: f32, + rdelta: f32, + nstep: usize, + use_mad: bool, +) -> (f32, f32) { + let n = x.len(); + let mut l: [u8; 32] = [0; 32]; + let mut l_aux: [u8; 32] = [0; 32]; + + let mut min_val = x[0]; + let mut max_val = x[0]; + let mut sum_w = match weights { + Some(w) => w[0], + None => x[0] * x[0], + }; + let mut sum_x = sum_w * x[0]; + + for i in 1..n { + if x[i] < min_val { + min_val = x[i]; + } + if x[i] > max_val { + max_val = x[i]; + } + let w = match weights { + Some(w) => w[i], + None => x[i] * x[i], + }; + sum_w += w; + sum_x += w * x[i]; + } + + if min_val > 0.0 { + min_val = 0.0; + } + + if max_val <= min_val { + return (0.0, -min_val); + } + + let mut iscale = nmax as f32 / (max_val - min_val); + let mut scale = 1.0 / iscale; + let mut best_mad = 0.0; + + for i in 0..n { + let l_val = nearest_int(iscale * (x[i] - min_val)).clamp(0, nmax) as u8; + l[i] = l_val; + let diff = scale * (l_val as f32) + min_val - x[i]; + let diff = if use_mad { diff.abs() } else { diff * diff }; + let w = match weights { + Some(w) => w[i], + None => x[i] * x[i], + }; + best_mad += w * diff; + } + + if nstep < 1 { + return (scale, -min_val); + } + + for is in 0..=nstep { + iscale = (rmin + rdelta * is as f32 + nmax as f32) / (max_val - min_val); + let (mut sum_l, mut sum_l2, mut sum_xl) = (0.0, 0.0, 0.0); + + for i in 0..n { + let l_val = nearest_int(iscale * (x[i] - min_val)).clamp(0, nmax) as u8; + l_aux[i] = l_val; + let w = match weights { + Some(w) => w[i], + None => x[i] * x[i], + }; + sum_l += w * l_val as f32; + sum_l2 += w * (l_val as f32).powi(2); + sum_xl += w * l_val as f32 * x[i]; + } + + let d = sum_w * sum_l2 - sum_l * sum_l; + if d > 0.0 { + let mut this_scale = (sum_w * sum_xl - sum_x * sum_l) / d; + let mut this_min = (sum_l2 * sum_x - sum_l * sum_xl) / d; + + if this_min > 0.0 { + this_min = 0.0; + this_scale = sum_xl / sum_l2; + } + + let mut mad = 0.0; + for i in 0..n { + let diff = this_scale * (l_aux[i] as f32) + this_min - x[i]; + let diff = if use_mad { diff.abs() } else { diff * diff }; + let w = match weights { + Some(w) => w[i], + None => x[i] * x[i], + }; + mad += w * diff; + } + + if mad < best_mad { + l.copy_from_slice(&l_aux); + best_mad = mad; + scale = this_scale; + min_val = this_min; + } + } + } + + (scale, -min_val) +} + +// https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/ggml/src/ggml-quants.c#L827 +pub(super) fn make_qp_quants( + n: usize, + nmax: u8, + x: &[f32], + l: &mut [u8], + quant_weights: &[f32], +) -> f32 { + assert_eq!(x.len(), n); + assert_eq!(l.len(), n); + assert_eq!(quant_weights.len(), n); + + let max = x.iter().copied().fold(0.0, f32::max); + if max == 0.0 { + l.iter_mut().for_each(|li| *li = 0); + return 0.0; + } + + let mut iscale = nmax as f32 / max; + for (xi, li) in x.iter().zip(l.iter_mut()) { + *li = nearest_int(iscale * xi) as u8; + } + + let scale = 1.0 / iscale; + let mut best_mse = x + .iter() + .zip(l.iter()) + .zip(quant_weights.iter()) + .map(|((&xi, &li), &w)| { + let diff = xi - scale * li as f32; + w * diff * diff + }) + .sum::(); + + for is in -4..=4 { + if is == 0 { + continue; + } + let iscale_is = (0.1 * is as f32 + nmax as f32) / max; + let scale_is = 1.0 / iscale_is; + + let mse = x + .iter() + .zip(quant_weights.iter()) + .map(|(&xi, &w)| { + let mut li = nearest_int(iscale_is * xi) as u8; + li = li.min(nmax); + let diff = xi - scale_is * li as f32; + w * diff * diff + }) + .sum::(); + + if mse < best_mse { + best_mse = mse; + iscale = iscale_is; + } + } + + let mut sumlx = 0.0; + let mut suml2 = 0.0; + for ((xi, li), &w) in x.iter().zip(l.iter_mut()).zip(quant_weights.iter()) { + let mut li_new = (iscale * xi).round() as u8; + li_new = li_new.min(nmax); + *li = li_new; + sumlx += w * xi * li_new as f32; + suml2 += w * (li_new as f32).powi(2); + } + + for _ in 0..5 { + let mut n_changed = 0; + for ((xi, li), &w) in x.iter().zip(l.iter_mut()).zip(quant_weights.iter()) { + let mut slx = sumlx - w * xi * *li as f32; + let mut sl2 = suml2 - w * (*li as f32).powi(2); + if slx > 0.0 && sl2 > 0.0 { + let new_li = (nearest_int(xi * sl2 / slx) as u8).min(nmax); + if new_li != *li { + slx += w * xi * new_li as f32; + sl2 += w * (new_li as f32).powi(2); + if slx.powi(2) * suml2 > sumlx.powi(2) * sl2 { + *li = new_li; + sumlx = slx; + suml2 = sl2; + n_changed += 1; + } + } + } + } + if n_changed == 0 { + break; + } + } + + sumlx / suml2 +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/safetensors.rs b/v2/dependency-patches/candle-core-0.9.2/src/safetensors.rs new file mode 100644 index 0000000000..bec233b614 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/safetensors.rs @@ -0,0 +1,644 @@ +//! Module to load `safetensor` files into CPU/GPU memory. +//! +//! There are multiple ways to load tensors from safetensor files: +//! - `load` function for loading directly into memory and returning a HashMap of tensors +//! - `MmapedSafetensors` for memory mapping files and avoiding full allocation +//! - `SliceSafetensors` for working with in-memory buffers +//! - `BufferedSafetensors` for owning a buffer of data +//! +//! Tensors can also be serialized to safetensor format using the `save` function or +//! `Tensor::save_safetensors` method. +//! +use crate::op::BackpropOp; +use crate::storage::Storage; +use crate::tensor::from_storage; +use crate::{DType, Device, Error, Result, Tensor, WithDType}; +use safetensors::tensor as st; +use safetensors::tensor::SafeTensors; +use std::borrow::Cow; +use std::collections::HashMap; +use std::path::Path; + +impl From for st::Dtype { + fn from(value: DType) -> Self { + match value { + DType::U8 => st::Dtype::U8, + DType::U32 => st::Dtype::U32, + DType::I16 => st::Dtype::I16, + DType::I32 => st::Dtype::I32, + DType::I64 => st::Dtype::I64, + DType::BF16 => st::Dtype::BF16, + DType::F16 => st::Dtype::F16, + DType::F32 => st::Dtype::F32, + DType::F64 => st::Dtype::F64, + DType::F8E4M3 => st::Dtype::F8_E4M3, + DType::F6E2M3 => st::Dtype::F6_E2M3, + DType::F6E3M2 => st::Dtype::F6_E3M2, + DType::F4 => st::Dtype::F4, + DType::F8E8M0 => st::Dtype::F8_E8M0, + } + } +} + +impl TryFrom for DType { + type Error = Error; + fn try_from(value: st::Dtype) -> Result { + match value { + st::Dtype::U8 => Ok(DType::U8), + st::Dtype::U32 => Ok(DType::U32), + st::Dtype::I16 => Ok(DType::I16), + st::Dtype::I32 => Ok(DType::I32), + st::Dtype::I64 => Ok(DType::I64), + st::Dtype::BF16 => Ok(DType::BF16), + st::Dtype::F16 => Ok(DType::F16), + st::Dtype::F32 => Ok(DType::F32), + st::Dtype::F64 => Ok(DType::F64), + st::Dtype::F8_E4M3 => Ok(DType::F8E4M3), + st::Dtype::F6_E2M3 => Ok(DType::F6E2M3), + st::Dtype::F6_E3M2 => Ok(DType::F6E3M2), + st::Dtype::F4 => Ok(DType::F4), + st::Dtype::F8_E8M0 => Ok(DType::F8E8M0), + dtype => Err(Error::UnsupportedSafeTensorDtype(dtype)), + } + } +} + +impl st::View for Tensor { + fn dtype(&self) -> st::Dtype { + self.dtype().into() + } + fn shape(&self) -> &[usize] { + self.shape().dims() + } + + fn data(&self) -> Cow<'_, [u8]> { + // This copies data from GPU to CPU. + // TODO: Avoid the unwrap here. + Cow::Owned(convert_back(self).unwrap()) + } + + fn data_len(&self) -> usize { + let n: usize = self.shape().elem_count(); + let bytes_per_element = self.dtype().size_in_bytes(); + n * bytes_per_element + } +} + +impl st::View for &Tensor { + fn dtype(&self) -> st::Dtype { + (*self).dtype().into() + } + fn shape(&self) -> &[usize] { + self.dims() + } + + fn data(&self) -> Cow<'_, [u8]> { + // This copies data from GPU to CPU. + // TODO: Avoid the unwrap here. + Cow::Owned(convert_back(self).unwrap()) + } + + fn data_len(&self) -> usize { + let n: usize = self.dims().iter().product(); + let bytes_per_element = (*self).dtype().size_in_bytes(); + n * bytes_per_element + } +} + +impl Tensor { + pub fn save_safetensors>(&self, name: &str, filename: P) -> Result<()> { + let data = [(name, self.clone())]; + Ok(st::serialize_to_file(data, None, filename.as_ref())?) + } +} + +fn convert_slice(data: &[u8], shape: &[usize], device: &Device) -> Result { + let size_in_bytes = T::DTYPE.size_in_bytes(); + let elem_count = data.len() / size_in_bytes; + if (data.as_ptr() as usize).is_multiple_of(size_in_bytes) { + // SAFETY This is safe because we just checked that this + // was correctly aligned. + let data: &[T] = + unsafe { std::slice::from_raw_parts(data.as_ptr() as *const T, elem_count) }; + Tensor::from_slice(data, shape, device) + } else { + // XXX: We need to specify `T` here, otherwise the compiler will infer u8 because of the following cast + // Making this vector too small to fit a full f16/f32/f64 weights, resulting in out-of-bounds access + let mut c: Vec = Vec::with_capacity(elem_count); + // SAFETY: We just created c, so the allocated memory is necessarily + // contiguous and non overlapping with the view's data. + // We're downgrading the `c` pointer from T to u8, which removes alignment + // constraints. + unsafe { + std::ptr::copy_nonoverlapping(data.as_ptr(), c.as_mut_ptr() as *mut u8, data.len()); + c.set_len(elem_count) + } + Tensor::from_slice(&c, shape, device) + } +} + +fn convert_slice_with_cast Result>( + data: &[u8], + shape: &[usize], + device: &Device, + conv: F, +) -> Result { + let size_in_bytes = std::mem::size_of::(); + let elem_count = data.len() / size_in_bytes; + if (data.as_ptr() as usize).is_multiple_of(size_in_bytes) { + // SAFETY This is safe because we just checked that this + // was correctly aligned. + let data: &[T] = + unsafe { std::slice::from_raw_parts(data.as_ptr() as *const T, elem_count) }; + let data = data.iter().map(|t| conv(*t)).collect::>>()?; + Tensor::from_vec(data, shape, device) + } else { + // XXX: We need to specify `T` here, otherwise the compiler will infer u8 because of the following cast + // Making this vector too small to fit a full f16/f32/f64 weights, resulting in out-of-bounds access + let mut c: Vec = Vec::with_capacity(elem_count); + // SAFETY: We just created c, so the allocated memory is necessarily + // contiguous and non overlapping with the view's data. + // We're downgrading the `c` pointer from T to u8, which removes alignment + // constraints. + unsafe { + std::ptr::copy_nonoverlapping(data.as_ptr(), c.as_mut_ptr() as *mut u8, data.len()); + c.set_len(elem_count) + } + let c = c.into_iter().map(conv).collect::>>()?; + Tensor::from_vec(c, shape, device) + } +} + +fn convert_with_cast_ Result>( + view: &st::TensorView<'_>, + device: &Device, + conv: F, +) -> Result { + convert_slice_with_cast::(view.data(), view.shape(), device, conv) +} + +fn convert_(view: &st::TensorView<'_>, device: &Device) -> Result { + convert_slice::(view.data(), view.shape(), device) +} + +fn convert_back_(mut vs: Vec) -> Vec { + let size_in_bytes = T::DTYPE.size_in_bytes(); + let length = vs.len() * size_in_bytes; + let capacity = vs.capacity() * size_in_bytes; + let ptr = vs.as_mut_ptr() as *mut u8; + // Don't run the destructor for Vec + std::mem::forget(vs); + // SAFETY: + // + // Every T is larger than u8, so there is no issue regarding alignment. + // This re-interpret the Vec as a Vec. + unsafe { Vec::from_raw_parts(ptr, length, capacity) } +} + +pub trait Load { + fn load(&self, device: &Device) -> Result; +} + +impl Load for st::TensorView<'_> { + fn load(&self, device: &Device) -> Result { + convert(self, device) + } +} + +impl Tensor { + pub fn from_raw_buffer( + data: &[u8], + dtype: DType, + shape: &[usize], + device: &Device, + ) -> Result { + match dtype { + DType::U8 => convert_slice::(data, shape, device), + DType::U32 => convert_slice::(data, shape, device), + DType::I16 => convert_slice::(data, shape, device), + DType::I32 => convert_slice::(data, shape, device), + DType::I64 => convert_slice::(data, shape, device), + DType::BF16 => convert_slice::(data, shape, device), + DType::F16 => convert_slice::(data, shape, device), + DType::F32 => convert_slice::(data, shape, device), + DType::F64 => convert_slice::(data, shape, device), + DType::F8E4M3 => convert_slice::(data, shape, device), + DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { + // For dummy types, create storage with raw bytes + let storage = match device { + Device::Cpu => { + let cpu_storage = match dtype { + DType::F6E2M3 => crate::cpu_backend::CpuStorage::F6E2M3(data.to_vec()), + DType::F6E3M2 => crate::cpu_backend::CpuStorage::F6E3M2(data.to_vec()), + DType::F4 => crate::cpu_backend::CpuStorage::F4(data.to_vec()), + DType::F8E8M0 => crate::cpu_backend::CpuStorage::F8E8M0(data.to_vec()), + _ => unreachable!(), + }; + Storage::Cpu(cpu_storage) + } + #[cfg(feature = "cuda")] + Device::Cuda(device) => { + let mut slice = unsafe { device.alloc::(data.len())? }; + device.memcpy_htod(data, &mut slice)?; + + let slice = match dtype { + DType::F6E2M3 => crate::cuda_backend::CudaStorageSlice::F6E2M3(slice), + DType::F6E3M2 => crate::cuda_backend::CudaStorageSlice::F6E3M2(slice), + DType::F4 => crate::cuda_backend::CudaStorageSlice::F4(slice), + DType::F8E8M0 => crate::cuda_backend::CudaStorageSlice::F8E8M0(slice), + _ => unreachable!(), + }; + let storage = crate::cuda_backend::CudaStorage { + slice, + device: device.clone(), + }; + Storage::Cuda(storage) + } + #[cfg(not(feature = "cuda"))] + Device::Cuda(_) => { + return Err(Error::Msg("CUDA support not compiled".to_string())); + } + #[cfg(feature = "metal")] + Device::Metal(device) => { + let buffer = device.new_buffer_with_data(data)?; + + let storage = crate::metal_backend::MetalStorage::new( + buffer, + device.clone(), + data.len(), + dtype, + ); + Storage::Metal(storage) + } + #[cfg(not(feature = "metal"))] + Device::Metal(_) => { + return Err(Error::Msg("Metal support not compiled".to_string())); + } + }; + + let op = BackpropOp::none(); + Ok(from_storage(storage, shape, op, false)) + } + } + } +} + +fn convert(view: &st::TensorView<'_>, device: &Device) -> Result { + match view.dtype() { + st::Dtype::U8 => convert_::(view, device), + st::Dtype::U16 => { + let conv = |x| Ok(u32::from(x)); + convert_with_cast_::(view, device, conv) + } + st::Dtype::U32 => convert_::(view, device), + st::Dtype::I16 => convert_::(view, device), + st::Dtype::I32 => convert_::(view, device), + st::Dtype::I64 => convert_::(view, device), + st::Dtype::BF16 => convert_::(view, device), + st::Dtype::F16 => convert_::(view, device), + st::Dtype::F32 => convert_::(view, device), + st::Dtype::F64 => convert_::(view, device), + st::Dtype::F8_E4M3 => convert_::(view, device), + st::Dtype::F6_E2M3 | st::Dtype::F6_E3M2 | st::Dtype::F4 | st::Dtype::F8_E8M0 => { + // For dummy types, we need to handle loading by creating a dummy tensor + // Since these types don't have actual data representation, we'll create + // a tensor that indicates it's a dummy type + convert_dummy(view, device) + } + dtype => Err(Error::UnsupportedSafeTensorDtype(dtype)), + } +} + +fn convert_dummy(view: &st::TensorView<'_>, device: &Device) -> Result { + // For dummy types, we'll create the appropriate storage variant that preserves + // both the raw data and the correct dtype + let (dtype, _dtype_name) = match view.dtype() { + st::Dtype::F6_E2M3 => (DType::F6E2M3, "F6_E2M3 (MX6)"), + st::Dtype::F6_E3M2 => (DType::F6E3M2, "F6_E3M2 (MX6)"), + st::Dtype::F4 => (DType::F4, "F4 (MX4)"), + st::Dtype::F8_E8M0 => (DType::F8E8M0, "F8_E8M0"), + _ => unreachable!("convert_dummy called with non-dummy dtype"), + }; + + // Load the raw bytes + let data = view.data(); + let shape = view.shape(); + + // Create storage with the appropriate dummy type variant + let storage = match device { + Device::Cpu => { + let cpu_storage = match dtype { + DType::F6E2M3 => crate::cpu_backend::CpuStorage::F6E2M3(data.to_vec()), + DType::F6E3M2 => crate::cpu_backend::CpuStorage::F6E3M2(data.to_vec()), + DType::F4 => crate::cpu_backend::CpuStorage::F4(data.to_vec()), + DType::F8E8M0 => crate::cpu_backend::CpuStorage::F8E8M0(data.to_vec()), + _ => unreachable!(), + }; + Storage::Cpu(cpu_storage) + } + #[cfg(feature = "cuda")] + Device::Cuda(device) => { + let mut slice = unsafe { device.alloc::(data.len())? }; + device.memcpy_htod(data, &mut slice)?; + + let slice = match dtype { + DType::F6E2M3 => crate::cuda_backend::CudaStorageSlice::F6E2M3(slice), + DType::F6E3M2 => crate::cuda_backend::CudaStorageSlice::F6E3M2(slice), + DType::F4 => crate::cuda_backend::CudaStorageSlice::F4(slice), + DType::F8E8M0 => crate::cuda_backend::CudaStorageSlice::F8E8M0(slice), + _ => unreachable!(), + }; + let storage = crate::cuda_backend::CudaStorage { + slice, + device: device.clone(), + }; + Storage::Cuda(storage) + } + #[cfg(not(feature = "cuda"))] + Device::Cuda(_) => { + return Err(Error::Msg("CUDA support not compiled".to_string())); + } + #[cfg(feature = "metal")] + Device::Metal(device) => { + let buffer = device.new_buffer_with_data(data)?; + + let storage = + crate::metal_backend::MetalStorage::new(buffer, device.clone(), data.len(), dtype); + Storage::Metal(storage) + } + #[cfg(not(feature = "metal"))] + Device::Metal(_) => { + return Err(Error::Msg("Metal support not compiled".to_string())); + } + }; + + // Create tensor with correct dtype + let op = BackpropOp::none(); + Ok(from_storage(storage, shape, op, false)) +} + +fn convert_back(tensor: &Tensor) -> Result> { + // TODO: This makes an unnecessary copy when the tensor is on the cpu. + let tensor = tensor.flatten_all()?; + match tensor.dtype() { + DType::U8 => Ok(convert_back_::(tensor.to_vec1()?)), + DType::U32 => Ok(convert_back_::(tensor.to_vec1()?)), + DType::I16 => Ok(convert_back_::(tensor.to_vec1()?)), + DType::I32 => Ok(convert_back_::(tensor.to_vec1()?)), + DType::I64 => Ok(convert_back_::(tensor.to_vec1()?)), + DType::F16 => Ok(convert_back_::(tensor.to_vec1()?)), + DType::BF16 => Ok(convert_back_::(tensor.to_vec1()?)), + DType::F32 => Ok(convert_back_::(tensor.to_vec1()?)), + DType::F64 => Ok(convert_back_::(tensor.to_vec1()?)), + DType::F8E4M3 => Ok(convert_back_::(tensor.to_vec1()?)), + DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { + Err(Error::Msg("Internal error: dtype mismatch in storage".to_string()).bt()) + } + } +} + +pub fn load>(filename: P, device: &Device) -> Result> { + let data = std::fs::read(filename.as_ref())?; + load_buffer(&data[..], device) +} + +pub fn load_buffer(data: &[u8], device: &Device) -> Result> { + let st = safetensors::SafeTensors::deserialize(data)?; + st.tensors() + .into_iter() + .map(|(name, view)| Ok((name, view.load(device)?))) + .collect() +} + +pub fn save + Ord + std::fmt::Display, P: AsRef>( + tensors: &HashMap, + filename: P, +) -> Result<()> { + Ok(st::serialize_to_file(tensors, None, filename.as_ref())?) +} + +#[derive(yoke::Yokeable)] +struct SafeTensors_<'a>(SafeTensors<'a>); + +pub struct MmapedSafetensors { + safetensors: Vec, memmap2::Mmap>>, + routing: Option>, +} + +impl MmapedSafetensors { + /// Creates a wrapper around a memory mapped file and deserialize the safetensors header. + /// + /// # Safety + /// + /// The unsafe is inherited from [`memmap2::MmapOptions`]. + pub unsafe fn new>(p: P) -> Result { + let p = p.as_ref(); + let file = std::fs::File::open(p).map_err(|e| Error::from(e).with_path(p))?; + let file = memmap2::MmapOptions::new() + .map(&file) + .map_err(|e| Error::from(e).with_path(p))?; + let safetensors = yoke::Yoke::, memmap2::Mmap>::try_attach_to_cart( + file, + |data: &[u8]| { + let st = safetensors::SafeTensors::deserialize(data) + .map_err(|e| Error::from(e).with_path(p))?; + Ok::<_, Error>(SafeTensors_(st)) + }, + )?; + Ok(Self { + safetensors: vec![safetensors], + routing: None, + }) + } + + /// Creates a wrapper around multiple memory mapped file and deserialize the safetensors headers. + /// + /// If a tensor name appears in multiple files, the last entry is returned. + /// + /// # Safety + /// + /// The unsafe is inherited from [`memmap2::MmapOptions`]. + pub unsafe fn multi>(paths: &[P]) -> Result { + let mut routing = HashMap::new(); + let mut safetensors = vec![]; + for (index, p) in paths.iter().enumerate() { + let p = p.as_ref(); + let file = std::fs::File::open(p).map_err(|e| Error::from(e).with_path(p))?; + let file = memmap2::MmapOptions::new() + .map(&file) + .map_err(|e| Error::from(e).with_path(p))?; + let data = yoke::Yoke::, memmap2::Mmap>::try_attach_to_cart( + file, + |data: &[u8]| { + let st = safetensors::SafeTensors::deserialize(data) + .map_err(|e| Error::from(e).with_path(p))?; + Ok::<_, Error>(SafeTensors_(st)) + }, + )?; + for k in data.get().0.names() { + routing.insert(k.to_string(), index); + } + safetensors.push(data) + } + Ok(Self { + safetensors, + routing: Some(routing), + }) + } + + pub fn load(&self, name: &str, dev: &Device) -> Result { + self.get(name)?.load(dev) + } + + pub fn tensors(&self) -> Vec<(String, st::TensorView<'_>)> { + let mut tensors = vec![]; + for safetensors in self.safetensors.iter() { + tensors.push(safetensors.get().0.tensors()) + } + tensors.into_iter().flatten().collect() + } + + pub fn get(&self, name: &str) -> Result> { + let index = match &self.routing { + None => 0, + Some(routing) => { + let index = routing.get(name).ok_or_else(|| { + Error::CannotFindTensor { + path: name.to_string(), + } + .bt() + })?; + *index + } + }; + Ok(self.safetensors[index].get().0.tensor(name)?) + } +} + +pub struct SliceSafetensors<'a> { + safetensors: SafeTensors<'a>, +} + +impl<'a> SliceSafetensors<'a> { + /// Creates a wrapper around a binary buffer and deserialize the safetensors header. + pub fn new(buffer: &'a [u8]) -> Result { + let safetensors = safetensors::SafeTensors::deserialize(buffer)?; + Ok(Self { safetensors }) + } + + pub fn load(&self, name: &str, dev: &Device) -> Result { + self.safetensors.tensor(name)?.load(dev) + } + + pub fn tensors(&self) -> Vec<(String, st::TensorView<'_>)> { + self.safetensors.tensors() + } + + pub fn get(&self, name: &str) -> Result> { + Ok(self.safetensors.tensor(name)?) + } +} + +pub struct BufferedSafetensors { + safetensors: yoke::Yoke, Vec>, +} + +impl BufferedSafetensors { + /// Creates a wrapper around a binary buffer and deserialize the safetensors header. + pub fn new(buffer: Vec) -> Result { + let safetensors = yoke::Yoke::, Vec>::try_attach_to_cart( + buffer, + |data: &[u8]| { + let st = safetensors::SafeTensors::deserialize(data)?; + Ok::<_, Error>(SafeTensors_(st)) + }, + )?; + Ok(Self { safetensors }) + } + + pub fn load(&self, name: &str, dev: &Device) -> Result { + self.get(name)?.load(dev) + } + + pub fn tensors(&self) -> Vec<(String, st::TensorView<'_>)> { + self.safetensors.get().0.tensors() + } + + pub fn get(&self, name: &str) -> Result> { + Ok(self.safetensors.get().0.tensor(name)?) + } +} + +pub struct MmapedFile { + path: std::path::PathBuf, + inner: memmap2::Mmap, +} + +impl MmapedFile { + /// Creates a wrapper around a memory mapped file from which you can retrieve + /// tensors using [`MmapedFile::deserialize`] + /// + /// # Safety + /// + /// The unsafe is inherited from [`memmap2::MmapOptions`]. + pub unsafe fn new>(p: P) -> Result { + let p = p.as_ref(); + let file = std::fs::File::open(p).map_err(|e| Error::from(e).with_path(p))?; + let inner = memmap2::MmapOptions::new() + .map(&file) + .map_err(|e| Error::from(e).with_path(p))?; + Ok(Self { + inner, + path: p.to_path_buf(), + }) + } + + pub fn deserialize(&self) -> Result> { + let st = safetensors::SafeTensors::deserialize(&self.inner) + .map_err(|e| Error::from(e).with_path(&self.path))?; + Ok(st) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::HashMap; + + #[test] + fn save_single_tensor() { + let t = Tensor::zeros((2, 2), DType::F32, &Device::Cpu).unwrap(); + t.save_safetensors("t", "t.safetensors").unwrap(); + let bytes = std::fs::read("t.safetensors").unwrap(); + assert_eq!(bytes, b"@\0\0\0\0\0\0\0{\"t\":{\"dtype\":\"F32\",\"shape\":[2,2],\"data_offsets\":[0,16]}} \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"); + std::fs::remove_file("t.safetensors").unwrap(); + } + + #[test] + fn save_load_multiple_tensors() { + let t = Tensor::zeros((2, 2), DType::F32, &Device::Cpu).unwrap(); + let u = Tensor::zeros((1, 2), DType::F32, &Device::Cpu).unwrap(); + let map: HashMap<_, _> = [("t", t), ("u", u)].into_iter().collect(); + save(&map, "multi.safetensors").unwrap(); + + let weights = load("multi.safetensors", &Device::Cpu).unwrap(); + assert_eq!(weights.get("t").unwrap().dims(), &[2, 2]); + assert_eq!(weights.get("u").unwrap().dims(), &[1, 2]); + let bytes = std::fs::read("multi.safetensors").unwrap(); + assert_eq!(bytes, b"x\0\0\0\0\0\0\0{\"t\":{\"dtype\":\"F32\",\"shape\":[2,2],\"data_offsets\":[0,16]},\"u\":{\"dtype\":\"F32\",\"shape\":[1,2],\"data_offsets\":[16,24]}} \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"); + std::fs::remove_file("multi.safetensors").unwrap(); + } + + #[test] + fn load_u8() { + let bytes = b"8\0\0\0\0\0\0\0{\"x\":{\"dtype\":\"U8\",\"shape\":[2],\"data_offsets\":[0,2]}} \x01\x03"; + std::fs::write("test_u8.safetensors", bytes).unwrap(); + let weights = load("test_u8.safetensors", &Device::Cpu).unwrap(); + let tensor = weights.get("x").unwrap(); + assert_eq!(tensor.dims(), &[2]); + assert_eq!(tensor.dtype(), DType::U8); + let data: Vec = tensor.to_vec1().unwrap(); + assert_eq!(data, vec![1, 3]); + std::fs::remove_file("test_u8.safetensors").unwrap(); + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/scalar.rs b/v2/dependency-patches/candle-core-0.9.2/src/scalar.rs new file mode 100644 index 0000000000..5c512c03b9 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/scalar.rs @@ -0,0 +1,115 @@ +//! TensorScalar Enum and Trait +//! +use crate::{DType, Result, Tensor, WithDType}; +use float8::F8E4M3 as f8e4m3; +use half::{bf16, f16}; + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum Scalar { + U8(u8), + U32(u32), + I16(i16), + I32(i32), + I64(i64), + BF16(bf16), + F16(f16), + F32(f32), + F64(f64), + F8E4M3(f8e4m3), +} + +impl From for Scalar { + fn from(value: T) -> Self { + value.to_scalar() + } +} + +impl Scalar { + pub fn zero(dtype: DType) -> Self { + match dtype { + DType::U8 => Scalar::U8(0), + DType::U32 => Scalar::U32(0), + DType::I16 => Scalar::I16(0), + DType::I32 => Scalar::I32(0), + DType::I64 => Scalar::I64(0), + DType::BF16 => Scalar::BF16(bf16::ZERO), + DType::F16 => Scalar::F16(f16::ZERO), + DType::F32 => Scalar::F32(0.0), + DType::F64 => Scalar::F64(0.0), + DType::F8E4M3 => Scalar::F8E4M3(f8e4m3::ZERO), + DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { + panic!("Cannot create zero scalar for dummy type {dtype:?}") + } + } + } + + pub fn one(dtype: DType) -> Self { + match dtype { + DType::U8 => Scalar::U8(1), + DType::U32 => Scalar::U32(1), + DType::I16 => Scalar::I16(1), + DType::I32 => Scalar::I32(1), + DType::I64 => Scalar::I64(1), + DType::BF16 => Scalar::BF16(bf16::ONE), + DType::F16 => Scalar::F16(f16::ONE), + DType::F32 => Scalar::F32(1.0), + DType::F64 => Scalar::F64(1.0), + DType::F8E4M3 => Scalar::F8E4M3(f8e4m3::ONE), + DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { + panic!("Cannot create one scalar for dummy type {dtype:?}") + } + } + } + + pub fn dtype(&self) -> DType { + match self { + Scalar::U8(_) => DType::U8, + Scalar::U32(_) => DType::U32, + Scalar::I16(_) => DType::I16, + Scalar::I32(_) => DType::I32, + Scalar::I64(_) => DType::I64, + Scalar::BF16(_) => DType::BF16, + Scalar::F16(_) => DType::F16, + Scalar::F32(_) => DType::F32, + Scalar::F64(_) => DType::F64, + Scalar::F8E4M3(_) => DType::F8E4M3, + } + } + + pub fn to_f64(&self) -> f64 { + match self { + Scalar::U8(v) => *v as f64, + Scalar::U32(v) => *v as f64, + Scalar::I16(v) => *v as f64, + Scalar::I32(v) => *v as f64, + Scalar::I64(v) => *v as f64, + Scalar::BF16(v) => v.to_f64(), + Scalar::F16(v) => v.to_f64(), + Scalar::F32(v) => *v as f64, + Scalar::F64(v) => *v, + Scalar::F8E4M3(v) => v.to_f64(), + } + } +} + +pub enum TensorScalar { + Tensor(Tensor), + Scalar(Tensor), +} + +pub trait TensorOrScalar { + fn to_tensor_scalar(self) -> Result; +} + +impl TensorOrScalar for &Tensor { + fn to_tensor_scalar(self) -> Result { + Ok(TensorScalar::Tensor(self.clone())) + } +} + +impl TensorOrScalar for T { + fn to_tensor_scalar(self) -> Result { + let scalar = Tensor::new(self, &crate::Device::Cpu)?; + Ok(TensorScalar::Scalar(scalar)) + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/shape.rs b/v2/dependency-patches/candle-core-0.9.2/src/shape.rs new file mode 100644 index 0000000000..b9e731266f --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/shape.rs @@ -0,0 +1,634 @@ +//! The shape of a tensor is a tuple with the size of each of its dimensions. +#![allow(clippy::redundant_closure_call)] +use crate::{Error, Result}; + +#[derive(Clone, PartialEq, Eq)] +pub struct Shape(Vec); + +pub const SCALAR: Shape = Shape(vec![]); + +impl std::fmt::Debug for Shape { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", &self.dims()) + } +} + +impl From<&[usize; C]> for Shape { + fn from(dims: &[usize; C]) -> Self { + Self(dims.to_vec()) + } +} + +impl From<&[usize]> for Shape { + fn from(dims: &[usize]) -> Self { + Self(dims.to_vec()) + } +} + +impl From<&Shape> for Shape { + fn from(shape: &Shape) -> Self { + Self(shape.0.to_vec()) + } +} + +impl From<()> for Shape { + fn from(_: ()) -> Self { + Self(vec![]) + } +} + +impl From for Shape { + fn from(d1: usize) -> Self { + Self(vec![d1]) + } +} + +macro_rules! impl_from_tuple { + ($tuple:ty, $($index:tt),+) => { + impl From<$tuple> for Shape { + fn from(d: $tuple) -> Self { + Self(vec![$(d.$index,)+]) + } + } + } +} + +impl_from_tuple!((usize,), 0); +impl_from_tuple!((usize, usize), 0, 1); +impl_from_tuple!((usize, usize, usize), 0, 1, 2); +impl_from_tuple!((usize, usize, usize, usize), 0, 1, 2, 3); +impl_from_tuple!((usize, usize, usize, usize, usize), 0, 1, 2, 3, 4); +impl_from_tuple!((usize, usize, usize, usize, usize, usize), 0, 1, 2, 3, 4, 5); + +impl From> for Shape { + fn from(dims: Vec) -> Self { + Self(dims) + } +} + +macro_rules! extract_dims { + ($fn_name:ident, $cnt:tt, $dims:expr, $out_type:ty) => { + pub fn $fn_name(dims: &[usize]) -> Result<$out_type> { + if dims.len() != $cnt { + Err(Error::UnexpectedNumberOfDims { + expected: $cnt, + got: dims.len(), + shape: Shape::from(dims), + } + .bt()) + } else { + Ok($dims(dims)) + } + } + + impl Shape { + pub fn $fn_name(&self) -> Result<$out_type> { + $fn_name(self.0.as_slice()) + } + } + + impl crate::Tensor { + pub fn $fn_name(&self) -> Result<$out_type> { + self.shape().$fn_name() + } + } + + impl std::convert::TryInto<$out_type> for Shape { + type Error = crate::Error; + fn try_into(self) -> std::result::Result<$out_type, Self::Error> { + self.$fn_name() + } + } + }; +} + +impl Shape { + pub fn from_dims(dims: &[usize]) -> Self { + Self(dims.to_vec()) + } + + /// The rank is the number of dimensions, 0 for a scalar value, 1 for a vector, etc. + pub fn rank(&self) -> usize { + self.0.len() + } + + pub fn into_dims(self) -> Vec { + self.0 + } + + /// The dimensions as a slice of `usize`. + pub fn dims(&self) -> &[usize] { + &self.0 + } + + /// The dimension size for a specified dimension index. + pub fn dim(&self, dim: D) -> Result { + let dim = dim.to_index(self, "dim")?; + Ok(self.dims()[dim]) + } + + /// The total number of elements, this is the product of all dimension sizes. + pub fn elem_count(&self) -> usize { + self.0.iter().product() + } + + /// The strides given in number of elements for a contiguous n-dimensional + /// arrays using this shape. + pub(crate) fn stride_contiguous(&self) -> Vec { + let mut stride: Vec<_> = self + .0 + .iter() + .rev() + .scan(1, |prod, u| { + let prod_pre_mult = *prod; + *prod *= u; + Some(prod_pre_mult) + }) + .collect(); + stride.reverse(); + stride + } + + /// Returns true if the strides are C contiguous (aka row major). + pub fn is_contiguous(&self, stride: &[usize]) -> bool { + if self.0.len() != stride.len() { + return false; + } + let mut acc = 1; + for (&stride, &dim) in stride.iter().zip(self.0.iter()).rev() { + if dim > 1 && stride != acc { + return false; + } + acc *= dim; + } + true + } + + /// Returns true if the strides are Fortran contiguous (aka column major). + pub fn is_fortran_contiguous(&self, stride: &[usize]) -> bool { + if self.0.len() != stride.len() { + return false; + } + let mut acc = 1; + for (&stride, &dim) in stride.iter().zip(self.0.iter()) { + if dim > 1 && stride != acc { + return false; + } + acc *= dim; + } + true + } + + /// Modifies the shape by adding a list of additional dimensions at the end of the existing + /// dimensions. + pub fn extend(mut self, additional_dims: &[usize]) -> Self { + self.0.extend(additional_dims); + self + } + + /// Check whether the two shapes are compatible for broadcast, and if it is the case return the + /// broadcasted shape. This is to be used for binary pointwise ops. + pub fn broadcast_shape_binary_op(&self, rhs: &Self, op: &'static str) -> Result { + let lhs = self; + let lhs_dims = lhs.dims(); + let rhs_dims = rhs.dims(); + let lhs_ndims = lhs_dims.len(); + let rhs_ndims = rhs_dims.len(); + let bcast_ndims = usize::max(lhs_ndims, rhs_ndims); + let mut bcast_dims = vec![0; bcast_ndims]; + for (idx, bcast_value) in bcast_dims.iter_mut().enumerate() { + let rev_idx = bcast_ndims - idx; + let l_value = if lhs_ndims < rev_idx { + 1 + } else { + lhs_dims[lhs_ndims - rev_idx] + }; + let r_value = if rhs_ndims < rev_idx { + 1 + } else { + rhs_dims[rhs_ndims - rev_idx] + }; + *bcast_value = if l_value == r_value { + l_value + } else if l_value == 1 { + r_value + } else if r_value == 1 { + l_value + } else { + Err(Error::ShapeMismatchBinaryOp { + lhs: lhs.clone(), + rhs: rhs.clone(), + op, + } + .bt())? + } + } + Ok(Shape::from(bcast_dims)) + } + + pub(crate) fn broadcast_shape_matmul(&self, rhs: &Self) -> Result<(Shape, Shape)> { + let lhs = self; + let lhs_dims = lhs.dims(); + let rhs_dims = rhs.dims(); + if lhs_dims.len() < 2 || rhs_dims.len() < 2 { + crate::bail!("only 2d matrixes are supported {lhs:?} {rhs:?}") + } + let (m, lhs_k) = (lhs_dims[lhs_dims.len() - 2], lhs_dims[lhs_dims.len() - 1]); + let (rhs_k, n) = (rhs_dims[rhs_dims.len() - 2], rhs_dims[rhs_dims.len() - 1]); + if lhs_k != rhs_k { + crate::bail!("different inner dimensions in broadcast matmul {lhs:?} {rhs:?}") + } + + let lhs_b = Self::from(&lhs_dims[..lhs_dims.len() - 2]); + let rhs_b = Self::from(&rhs_dims[..rhs_dims.len() - 2]); + let bcast = lhs_b.broadcast_shape_binary_op(&rhs_b, "broadcast_matmul")?; + let bcast_dims = bcast.dims(); + + let bcast_lhs = [bcast_dims, &[m, lhs_k]].concat(); + let bcast_rhs = [bcast_dims, &[rhs_k, n]].concat(); + Ok((Shape::from(bcast_lhs), Shape::from(bcast_rhs))) + } +} + +pub trait Dim { + fn to_index(&self, shape: &Shape, op: &'static str) -> Result; + fn to_index_plus_one(&self, shape: &Shape, op: &'static str) -> Result; +} + +impl Dim for usize { + fn to_index(&self, shape: &Shape, op: &'static str) -> Result { + let dim = *self; + if dim >= shape.dims().len() { + Err(Error::DimOutOfRange { + shape: shape.clone(), + dim: dim as i32, + op, + } + .bt())? + } else { + Ok(dim) + } + } + + fn to_index_plus_one(&self, shape: &Shape, op: &'static str) -> Result { + let dim = *self; + if dim > shape.dims().len() { + Err(Error::DimOutOfRange { + shape: shape.clone(), + dim: dim as i32, + op, + } + .bt())? + } else { + Ok(dim) + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub enum D { + Minus1, + Minus2, + Minus(usize), +} + +impl D { + fn out_of_range(&self, shape: &Shape, op: &'static str) -> Error { + let dim = match self { + Self::Minus1 => -1, + Self::Minus2 => -2, + Self::Minus(u) => -(*u as i32), + }; + Error::DimOutOfRange { + shape: shape.clone(), + dim, + op, + } + .bt() + } +} + +impl Dim for D { + fn to_index(&self, shape: &Shape, op: &'static str) -> Result { + let rank = shape.rank(); + match self { + Self::Minus1 if rank >= 1 => Ok(rank - 1), + Self::Minus2 if rank >= 2 => Ok(rank - 2), + Self::Minus(u) if *u > 0 && rank >= *u => Ok(rank - *u), + _ => Err(self.out_of_range(shape, op)), + } + } + + fn to_index_plus_one(&self, shape: &Shape, op: &'static str) -> Result { + let rank = shape.rank(); + match self { + Self::Minus1 => Ok(rank), + Self::Minus2 if rank >= 1 => Ok(rank - 1), + Self::Minus(u) if *u > 0 && rank + 1 >= *u => Ok(rank + 1 - *u), + _ => Err(self.out_of_range(shape, op)), + } + } +} + +pub trait Dims: Sized { + fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result>; + + fn to_indexes(self, shape: &Shape, op: &'static str) -> Result> { + let dims = self.to_indexes_internal(shape, op)?; + for (i, &dim) in dims.iter().enumerate() { + if dims[..i].contains(&dim) { + Err(Error::DuplicateDimIndex { + shape: shape.clone(), + dims: dims.clone(), + op, + } + .bt())? + } + if dim >= shape.rank() { + Err(Error::DimOutOfRange { + shape: shape.clone(), + dim: dim as i32, + op, + } + .bt())? + } + } + Ok(dims) + } +} + +impl Dims for Vec { + fn to_indexes_internal(self, _: &Shape, _: &'static str) -> Result> { + Ok(self) + } +} + +impl Dims for [usize; N] { + fn to_indexes_internal(self, _: &Shape, _: &'static str) -> Result> { + Ok(self.to_vec()) + } +} + +impl Dims for &[usize] { + fn to_indexes_internal(self, _: &Shape, _: &'static str) -> Result> { + Ok(self.to_vec()) + } +} + +impl Dims for () { + fn to_indexes_internal(self, _: &Shape, _: &'static str) -> Result> { + Ok(vec![]) + } +} + +impl Dims for D { + fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { + let dim = self.to_index(shape, op)?; + Ok(vec![dim]) + } +} + +impl Dims for (D,) { + fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { + let dim = self.0.to_index(shape, op)?; + Ok(vec![dim]) + } +} + +impl Dims for (D1, D2) { + fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { + let d0 = self.0.to_index(shape, op)?; + let d1 = self.1.to_index(shape, op)?; + Ok(vec![d0, d1]) + } +} + +impl Dims for (D1, D2, D3) { + fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { + let d0 = self.0.to_index(shape, op)?; + let d1 = self.1.to_index(shape, op)?; + let d2 = self.2.to_index(shape, op)?; + Ok(vec![d0, d1, d2]) + } +} + +impl Dims for (D1, D2, D3, D4) { + fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { + let d0 = self.0.to_index(shape, op)?; + let d1 = self.1.to_index(shape, op)?; + let d2 = self.2.to_index(shape, op)?; + let d3 = self.3.to_index(shape, op)?; + Ok(vec![d0, d1, d2, d3]) + } +} + +impl Dims for (D1, D2, D3, D4, D5) { + fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { + let d0 = self.0.to_index(shape, op)?; + let d1 = self.1.to_index(shape, op)?; + let d2 = self.2.to_index(shape, op)?; + let d3 = self.3.to_index(shape, op)?; + let d4 = self.4.to_index(shape, op)?; + Ok(vec![d0, d1, d2, d3, d4]) + } +} + +impl Dims for (D1, D2, D3, D4, D5, D6) { + fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { + let d0 = self.0.to_index(shape, op)?; + let d1 = self.1.to_index(shape, op)?; + let d2 = self.2.to_index(shape, op)?; + let d3 = self.3.to_index(shape, op)?; + let d4 = self.4.to_index(shape, op)?; + let d5 = self.5.to_index(shape, op)?; + Ok(vec![d0, d1, d2, d3, d4, d5]) + } +} + +extract_dims!(dims0, 0, |_: &[usize]| (), ()); +extract_dims!(dims1, 1, |d: &[usize]| d[0], usize); +extract_dims!(dims2, 2, |d: &[usize]| (d[0], d[1]), (usize, usize)); +extract_dims!( + dims3, + 3, + |d: &[usize]| (d[0], d[1], d[2]), + (usize, usize, usize) +); +extract_dims!( + dims4, + 4, + |d: &[usize]| (d[0], d[1], d[2], d[3]), + (usize, usize, usize, usize) +); +extract_dims!( + dims5, + 5, + |d: &[usize]| (d[0], d[1], d[2], d[3], d[4]), + (usize, usize, usize, usize, usize) +); + +pub trait ShapeWithOneHole { + fn into_shape(self, el_count: usize) -> Result; +} + +impl> ShapeWithOneHole for S { + fn into_shape(self, _el_count: usize) -> Result { + Ok(self.into()) + } +} + +impl ShapeWithOneHole for ((),) { + fn into_shape(self, el_count: usize) -> Result { + Ok(el_count.into()) + } +} + +fn hole_size(el_count: usize, prod_d: usize, s: &dyn std::fmt::Debug) -> Result { + if prod_d == 0 { + crate::bail!("cannot reshape tensor of {el_count} elements to {s:?}") + } + if !el_count.is_multiple_of(prod_d) { + crate::bail!("cannot reshape tensor with {el_count} elements to {s:?}") + } + Ok(el_count / prod_d) +} + +impl ShapeWithOneHole for ((), usize) { + fn into_shape(self, el_count: usize) -> Result { + let ((), d1) = self; + Ok((hole_size(el_count, d1, &self)?, d1).into()) + } +} + +impl ShapeWithOneHole for (usize, ()) { + fn into_shape(self, el_count: usize) -> Result { + let (d1, ()) = self; + Ok((d1, hole_size(el_count, d1, &self)?).into()) + } +} + +impl ShapeWithOneHole for ((), usize, usize) { + fn into_shape(self, el_count: usize) -> Result { + let ((), d1, d2) = self; + Ok((hole_size(el_count, d1 * d2, &self)?, d1, d2).into()) + } +} + +impl ShapeWithOneHole for (usize, (), usize) { + fn into_shape(self, el_count: usize) -> Result { + let (d1, (), d2) = self; + Ok((d1, hole_size(el_count, d1 * d2, &self)?, d2).into()) + } +} + +impl ShapeWithOneHole for (usize, usize, ()) { + fn into_shape(self, el_count: usize) -> Result { + let (d1, d2, ()) = self; + Ok((d1, d2, hole_size(el_count, d1 * d2, &self)?).into()) + } +} + +impl ShapeWithOneHole for ((), usize, usize, usize) { + fn into_shape(self, el_count: usize) -> Result { + let ((), d1, d2, d3) = self; + let d = hole_size(el_count, d1 * d2 * d3, &self)?; + Ok((d, d1, d2, d3).into()) + } +} + +impl ShapeWithOneHole for (usize, (), usize, usize) { + fn into_shape(self, el_count: usize) -> Result { + let (d1, (), d2, d3) = self; + let d = hole_size(el_count, d1 * d2 * d3, &self)?; + Ok((d1, d, d2, d3).into()) + } +} + +impl ShapeWithOneHole for (usize, usize, (), usize) { + fn into_shape(self, el_count: usize) -> Result { + let (d1, d2, (), d3) = self; + let d = hole_size(el_count, d1 * d2 * d3, &self)?; + Ok((d1, d2, d, d3).into()) + } +} + +impl ShapeWithOneHole for (usize, usize, usize, ()) { + fn into_shape(self, el_count: usize) -> Result { + let (d1, d2, d3, ()) = self; + let d = hole_size(el_count, d1 * d2 * d3, &self)?; + Ok((d1, d2, d3, d).into()) + } +} + +impl ShapeWithOneHole for ((), usize, usize, usize, usize) { + fn into_shape(self, el_count: usize) -> Result { + let ((), d1, d2, d3, d4) = self; + let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?; + Ok((d, d1, d2, d3, d4).into()) + } +} + +impl ShapeWithOneHole for (usize, (), usize, usize, usize) { + fn into_shape(self, el_count: usize) -> Result { + let (d1, (), d2, d3, d4) = self; + let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?; + Ok((d1, d, d2, d3, d4).into()) + } +} + +impl ShapeWithOneHole for (usize, usize, (), usize, usize) { + fn into_shape(self, el_count: usize) -> Result { + let (d1, d2, (), d3, d4) = self; + let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?; + Ok((d1, d2, d, d3, d4).into()) + } +} + +impl ShapeWithOneHole for (usize, usize, usize, (), usize) { + fn into_shape(self, el_count: usize) -> Result { + let (d1, d2, d3, (), d4) = self; + let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?; + Ok((d1, d2, d3, d, d4).into()) + } +} + +impl ShapeWithOneHole for (usize, usize, usize, usize, ()) { + fn into_shape(self, el_count: usize) -> Result { + let (d1, d2, d3, d4, ()) = self; + let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?; + Ok((d1, d2, d3, d4, d).into()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn stride() { + let shape = Shape::from(()); + assert_eq!(shape.stride_contiguous(), Vec::::new()); + let shape = Shape::from(42); + assert_eq!(shape.stride_contiguous(), [1]); + let shape = Shape::from((42, 1337)); + assert_eq!(shape.stride_contiguous(), [1337, 1]); + let shape = Shape::from((299, 792, 458)); + assert_eq!(shape.stride_contiguous(), [458 * 792, 458, 1]); + } + + #[test] + fn test_from_tuple() { + let shape = Shape::from((2,)); + assert_eq!(shape.dims(), &[2]); + let shape = Shape::from((2, 3)); + assert_eq!(shape.dims(), &[2, 3]); + let shape = Shape::from((2, 3, 4)); + assert_eq!(shape.dims(), &[2, 3, 4]); + let shape = Shape::from((2, 3, 4, 5)); + assert_eq!(shape.dims(), &[2, 3, 4, 5]); + let shape = Shape::from((2, 3, 4, 5, 6)); + assert_eq!(shape.dims(), &[2, 3, 4, 5, 6]); + let shape = Shape::from((2, 3, 4, 5, 6, 7)); + assert_eq!(shape.dims(), &[2, 3, 4, 5, 6, 7]); + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/sort.rs b/v2/dependency-patches/candle-core-0.9.2/src/sort.rs new file mode 100644 index 0000000000..19d783874b --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/sort.rs @@ -0,0 +1,290 @@ +use crate::{Result, Tensor}; +use rayon::prelude::*; + +#[derive(Debug, Clone, Copy)] +struct ArgSort { + asc: bool, + last_dim: usize, +} + +impl ArgSort { + fn asort(&self, vs: &[T], layout: &crate::Layout) -> Vec { + #[allow(clippy::uninit_vec)] + // Safety: indexes are set later in the parallelized section. + let mut sort_indexes = unsafe { + let el_count = layout.shape().elem_count(); + let mut v = Vec::with_capacity(el_count); + v.set_len(el_count); + v + }; + if self.asc { + sort_indexes + .par_chunks_exact_mut(self.last_dim) + .zip(vs.par_chunks_exact(self.last_dim)) + .for_each(|(indexes, vs)| { + indexes + .iter_mut() + .enumerate() + .for_each(|(i, v)| *v = i as u32); + indexes.sort_by(|&i, &j| { + vs[i as usize] + .partial_cmp(&vs[j as usize]) + .unwrap_or(std::cmp::Ordering::Greater) + }) + }); + } else { + sort_indexes + .par_chunks_exact_mut(self.last_dim) + .zip(vs.par_chunks_exact(self.last_dim)) + .for_each(|(indexes, vs)| { + indexes + .iter_mut() + .enumerate() + .for_each(|(i, v)| *v = i as u32); + indexes.sort_by(|&j, &i| { + vs[i as usize] + .partial_cmp(&vs[j as usize]) + .unwrap_or(std::cmp::Ordering::Greater) + }) + }); + } + sort_indexes + } +} + +#[cfg(feature = "cuda")] +mod cuda { + use super::*; + use crate::cuda_backend::cudarc::driver::{ + CudaSlice, DeviceRepr, LaunchConfig, ValidAsZeroBits, + }; + use crate::cuda_backend::{kernel_name, kernels, CudaStorageSlice as S, WrapErr}; + use crate::{CudaDevice, WithDType}; + + impl crate::cuda_backend::Map1Any for ArgSort { + fn f) -> S>( + &self, + src: &CudaSlice, + dev: &CudaDevice, + layout: &crate::Layout, + _wrap: W, + ) -> Result { + use cudarc::driver::PushKernelArg; + + let slice = match layout.contiguous_offsets() { + None => crate::bail!("input has to be contiguous"), + Some((o1, o2)) => src.slice(o1..o2), + }; + let elem_count = layout.shape().elem_count(); + let dst = unsafe { dev.alloc::(elem_count)? }; + let func = if self.asc { + dev.get_or_load_func(&kernel_name::("asort_asc"), &kernels::SORT)? + } else { + dev.get_or_load_func(&kernel_name::("asort_desc"), &kernels::SORT)? + }; + let ncols = self.last_dim; + let nrows = elem_count / ncols; + let ncols_pad = next_power_of_2(ncols); + // Limit block dim to 1024 threads, which is the maximum on modern CUDA gpus. + let block_dim = ncols_pad.min(1024); + let cfg = LaunchConfig { + grid_dim: (nrows as u32, 1, 1), + block_dim: (block_dim as u32, 1, 1), + shared_mem_bytes: (ncols_pad * std::mem::size_of::()) as u32, + }; + let stream = dev.cuda_stream(); + let mut builder = stream.launch_builder(&func); + let ncols = ncols as i32; + let ncols_pad = ncols_pad as i32; + builder.arg(&slice).arg(&dst).arg(&ncols).arg(&ncols_pad); + unsafe { builder.launch(cfg) }.w()?; + Ok(S::U32(dst)) + } + } +} + +impl crate::CustomOp1 for ArgSort { + fn name(&self) -> &'static str { + "argsort" + } + + fn cpu_fwd( + &self, + storage: &crate::CpuStorage, + layout: &crate::Layout, + ) -> Result<(crate::CpuStorage, crate::Shape)> { + let sort_indexes = match storage { + crate::CpuStorage::U8(vs) => self.asort(vs, layout), + crate::CpuStorage::U32(vs) => self.asort(vs, layout), + crate::CpuStorage::I16(vs) => self.asort(vs, layout), + crate::CpuStorage::I32(vs) => self.asort(vs, layout), + crate::CpuStorage::I64(vs) => self.asort(vs, layout), + crate::CpuStorage::BF16(vs) => self.asort(vs, layout), + crate::CpuStorage::F16(vs) => self.asort(vs, layout), + crate::CpuStorage::F32(vs) => self.asort(vs, layout), + crate::CpuStorage::F64(vs) => self.asort(vs, layout), + crate::CpuStorage::F8E4M3(vs) => self.asort(vs, layout), + // Dummy types don't support sorting + crate::CpuStorage::F6E2M3(_) => { + return Err( + crate::Error::UnsupportedDTypeForOp(crate::DType::F6E2M3, "argsort").bt(), + ) + } + crate::CpuStorage::F6E3M2(_) => { + return Err( + crate::Error::UnsupportedDTypeForOp(crate::DType::F6E3M2, "argsort").bt(), + ) + } + crate::CpuStorage::F4(_) => { + return Err(crate::Error::UnsupportedDTypeForOp(crate::DType::F4, "argsort").bt()) + } + crate::CpuStorage::F8E8M0(_) => { + return Err( + crate::Error::UnsupportedDTypeForOp(crate::DType::F8E8M0, "argsort").bt(), + ) + } + }; + let sort_indexes = crate::CpuStorage::U32(sort_indexes); + Ok((sort_indexes, layout.shape().into())) + } + + #[cfg(feature = "cuda")] + fn cuda_fwd( + &self, + storage: &crate::CudaStorage, + layout: &crate::Layout, + ) -> Result<(crate::CudaStorage, crate::Shape)> { + use crate::backend::BackendStorage; + use crate::cuda_backend::Map1Any; + let dev = storage.device(); + let slice = self.map(&storage.slice, dev, layout)?; + let dst = crate::cuda_backend::CudaStorage { + slice, + device: dev.clone(), + }; + Ok((dst, layout.shape().clone())) + } + + #[cfg(feature = "metal")] + fn metal_fwd( + &self, + storage: &crate::MetalStorage, + layout: &crate::Layout, + ) -> Result<(crate::MetalStorage, crate::Shape)> { + use crate::backend::BackendStorage; + use crate::DType; + + let name = { + if self.asc { + match storage.dtype() { + DType::BF16 => "asort_asc_bf16", + DType::F16 => "asort_asc_f16", + DType::F32 => "asort_asc_f32", + DType::F64 => "asort_asc_f64", + DType::U8 => "asort_asc_u8", + DType::U32 => "asort_asc_u32", + DType::I16 => "asort_asc_i16", + DType::I32 => "asort_asc_i32", + DType::I64 => "asort_asc_i64", + DType::F8E4M3 => crate::bail!("Metal device does not yet support F8E4M3."), + DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { + return Err( + crate::Error::UnsupportedDTypeForOp(storage.dtype(), "argsort").bt(), + ) + } + } + } else { + match storage.dtype() { + DType::BF16 => "asort_desc_bf16", + DType::F16 => "asort_desc_f16", + DType::F32 => "asort_desc_f32", + DType::F64 => "asort_desc_f64", + DType::U8 => "asort_desc_u8", + DType::U32 => "asort_desc_u32", + DType::I16 => "asort_desc_i16", + DType::I32 => "asort_desc_i32", + DType::I64 => "asort_desc_i64", + DType::F8E4M3 => crate::bail!("Metal device does not yet support F8E4M3."), + DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { + return Err( + crate::Error::UnsupportedDTypeForOp(storage.dtype(), "argsort").bt(), + ) + } + } + } + }; + let device = storage.device(); + let kernels = device.kernels(); + let command_encoder = device.command_encoder()?; + let el = layout.shape().elem_count(); + let ncols = self.last_dim; + let nrows = el / ncols; + let src = crate::metal_backend::buffer_o(storage.buffer(), layout, storage.dtype()); + let dst = device.new_buffer(el, DType::U32, "asort")?; + let mut ncols_pad = 1; + while ncols_pad < ncols { + ncols_pad *= 2; + } + candle_metal_kernels::call_arg_sort( + device.metal_device(), + &command_encoder, + kernels, + name, + nrows, + ncols, + ncols_pad, + src, + &dst, + ) + .map_err(crate::Error::wrap)?; + let dst = crate::MetalStorage::new(dst, device.clone(), el, DType::U32); + Ok((dst, layout.shape().clone())) + } +} + +#[allow(unused)] +fn next_power_of_2(x: usize) -> usize { + let mut n = 1; + while n < x { + n *= 2 + } + n +} + +impl Tensor { + /// Returns the indices that sort the tensor along the last dimension. + /// + /// If `asc` is `true`, sorting is in ascending order. Otherwise sorting is performed in + /// descending order. The sort is unstable so there is no guarantees on the final order when it + /// comes to ties. + pub fn arg_sort_last_dim(&self, asc: bool) -> Result { + if !self.is_contiguous() { + return Err(crate::Error::RequiresContiguous { + op: "arg_sort_last_dim", + }); + } + let last_dim = match self.dims().last() { + None => crate::bail!("empty last-dim in arg-sort"), + Some(last_dim) => *last_dim, + }; + // No need for a backward pass for arg sort. + self.apply_op1_no_bwd(&ArgSort { asc, last_dim }) + } + + /// Sorts the tensor along the last dimension, returns the sorted tensor together with the + /// sorted indexes. + /// + /// If `asc` is `true`, sorting is in ascending order. Otherwise sorting is performed in + /// descending order. The sort is unstable so there is no guarantees on the final order when it + /// comes to ties. + pub fn sort_last_dim(&self, asc: bool) -> Result<(Tensor, Tensor)> { + if !self.is_contiguous() { + return Err(crate::Error::RequiresContiguous { + op: "sort_last_dim", + }); + } + let asort = self.arg_sort_last_dim(asc)?; + let sorted = self.gather(&asort, crate::D::Minus1)?; + Ok((sorted, asort)) + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/storage.rs b/v2/dependency-patches/candle-core-0.9.2/src/storage.rs new file mode 100644 index 0000000000..4646dc88ea --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/storage.rs @@ -0,0 +1,849 @@ +use crate::backend::BackendStorage; +use crate::op::{self, CmpOp, ReduceOp}; +use crate::scalar::Scalar; +use crate::{CpuStorage, CudaStorage, DType, Device, Error, Layout, MetalStorage, Result, Shape}; +use crate::{CustomOp1, CustomOp2, CustomOp3, InplaceOp1, InplaceOp2, InplaceOp3}; + +// We do not want to implement Clone on Storage as cloning may fail because of +// out of memory. Instead try_clone should be used. +#[derive(Debug)] +pub enum Storage { + Cpu(CpuStorage), + Cuda(CudaStorage), + Metal(MetalStorage), +} + +impl Storage { + pub fn try_clone(&self, layout: &Layout) -> Result { + match self { + Self::Cpu(storage) => Ok(Self::Cpu(storage.clone())), + Self::Cuda(storage) => { + let storage = storage.try_clone(layout)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = storage.try_clone(layout)?; + Ok(Self::Metal(storage)) + } + } + } + + pub fn device(&self) -> Device { + match self { + Self::Cpu(_) => Device::Cpu, + Self::Cuda(storage) => Device::Cuda(storage.device().clone()), + Self::Metal(storage) => Device::Metal(storage.device().clone()), + } + } + + pub fn dtype(&self) -> DType { + match self { + Self::Cpu(storage) => storage.dtype(), + Self::Cuda(storage) => storage.dtype(), + Self::Metal(storage) => storage.dtype(), + } + } + + pub(crate) fn same_device(&self, rhs: &Self, op: &'static str) -> Result<()> { + let lhs_device = self.device(); + let rhs_device = rhs.device(); + let lhs = lhs_device.location(); + let rhs = rhs_device.location(); + let same_device = if self.device().is_metal() { + // On metal, we require the device to be exactly the same rather than + // having the same location. In cuda this is not necessary as all CudaDevice on the + // same GPU will use the same cuda stream. + lhs_device.same_device(&rhs_device) + } else { + lhs == rhs + }; + if !same_device { + Err(Error::DeviceMismatchBinaryOp { lhs, rhs, op }.bt()) + } else { + Ok(()) + } + } + + pub(crate) fn same_dtype(&self, rhs: &Self, op: &'static str) -> Result<()> { + let lhs = self.dtype(); + let rhs = rhs.dtype(); + if lhs != rhs { + Err(Error::DTypeMismatchBinaryOp { lhs, rhs, op }.bt()) + } else { + Ok(()) + } + } + + pub(crate) fn const_set(&mut self, v: Scalar, l: &Layout) -> Result<()> { + match self { + Storage::Cpu(storage) => storage.const_set(v, l), + Storage::Cuda(storage) => storage.const_set(v, l), + Storage::Metal(storage) => storage.const_set(v, l), + } + } + + pub(crate) fn affine(&self, layout: &Layout, mul: f64, add: f64) -> Result { + match self { + Storage::Cpu(storage) => { + let storage = storage.affine(layout, mul, add)?; + Ok(Self::Cpu(storage)) + } + Self::Cuda(storage) => { + let storage = storage.affine(layout, mul, add)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = storage.affine(layout, mul, add)?; + Ok(Self::Metal(storage)) + } + } + } + + pub(crate) fn powf(&self, layout: &Layout, alpha: f64) -> Result { + match self { + Storage::Cpu(storage) => { + let storage = storage.powf(layout, alpha)?; + Ok(Self::Cpu(storage)) + } + Self::Cuda(storage) => { + let storage = storage.powf(layout, alpha)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = storage.powf(layout, alpha)?; + Ok(Self::Metal(storage)) + } + } + } + + pub(crate) fn elu(&self, layout: &Layout, alpha: f64) -> Result { + match self { + Storage::Cpu(storage) => { + let storage = storage.elu(layout, alpha)?; + Ok(Self::Cpu(storage)) + } + Self::Cuda(storage) => { + let storage = storage.elu(layout, alpha)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = storage.elu(layout, alpha)?; + Ok(Self::Metal(storage)) + } + } + } + + pub(crate) fn cmp( + &self, + op: CmpOp, + rhs: &Self, + lhs_layout: &Layout, + rhs_layout: &Layout, + ) -> Result { + self.same_device(rhs, "cmp")?; + self.same_dtype(rhs, "cmp")?; + match (self, rhs) { + (Storage::Cpu(lhs), Storage::Cpu(rhs)) => { + let storage = lhs.cmp(op, rhs, lhs_layout, rhs_layout)?; + Ok(Self::Cpu(storage)) + } + (Self::Cuda(lhs), Self::Cuda(rhs)) => { + let storage = lhs.cmp(op, rhs, lhs_layout, rhs_layout)?; + Ok(Self::Cuda(storage)) + } + (Self::Metal(lhs), Self::Metal(rhs)) => { + let storage = lhs.cmp(op, rhs, lhs_layout, rhs_layout)?; + Ok(Self::Metal(storage)) + } + (lhs, rhs) => { + // Should not happen because of the same device check above but we're defensive + // anyway. + Err(Error::DeviceMismatchBinaryOp { + lhs: lhs.device().location(), + rhs: rhs.device().location(), + op: "cmp", + } + .bt()) + } + } + } + + pub(crate) fn reduce_op(&self, op: ReduceOp, layout: &Layout, s: &[usize]) -> Result { + match self { + Storage::Cpu(storage) => { + let storage = storage.reduce_op(op, layout, s)?; + Ok(Self::Cpu(storage)) + } + Self::Cuda(storage) => { + let storage = storage.reduce_op(op, layout, s)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = storage.reduce_op(op, layout, s)?; + Ok(Self::Metal(storage)) + } + } + } + + pub(crate) fn to_dtype(&self, layout: &Layout, dtype: DType) -> Result { + match self { + Storage::Cpu(storage) => { + let storage = storage.to_dtype(layout, dtype)?; + Ok(Self::Cpu(storage)) + } + Self::Cuda(storage) => { + let storage = storage.to_dtype(layout, dtype)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = storage.to_dtype(layout, dtype)?; + Ok(Self::Metal(storage)) + } + } + } + + pub(crate) fn apply_op1(&self, l: &Layout, c: &dyn CustomOp1) -> Result<(Self, Shape)> { + match self { + Self::Cpu(storage) => { + let (storage, shape) = c.cpu_fwd(storage, l)?; + Ok((Self::Cpu(storage), shape)) + } + Self::Cuda(storage) => { + let (storage, shape) = c.cuda_fwd(storage, l)?; + Ok((Self::Cuda(storage), shape)) + } + Self::Metal(storage) => { + let (storage, shape) = c.metal_fwd(storage, l)?; + Ok((Self::Metal(storage), shape)) + } + } + } + + pub(crate) fn apply_op2( + &self, + l1: &Layout, + t2: &Self, + l2: &Layout, + c: &dyn CustomOp2, + ) -> Result<(Self, Shape)> { + self.same_device(t2, c.name())?; + match (self, t2) { + (Self::Cpu(s1), Self::Cpu(s2)) => { + let (s, shape) = c.cpu_fwd(s1, l1, s2, l2)?; + Ok((Self::Cpu(s), shape)) + } + (Self::Cuda(s1), Self::Cuda(s2)) => { + let (s, shape) = c.cuda_fwd(s1, l1, s2, l2)?; + Ok((Self::Cuda(s), shape)) + } + (Self::Metal(s1), Self::Metal(s2)) => { + let (s, shape) = c.metal_fwd(s1, l1, s2, l2)?; + Ok((Self::Metal(s), shape)) + } + _ => unreachable!(), + } + } + + pub(crate) fn apply_op3( + &self, + l1: &Layout, + t2: &Self, + l2: &Layout, + t3: &Self, + l3: &Layout, + c: &dyn CustomOp3, + ) -> Result<(Self, Shape)> { + self.same_device(t2, c.name())?; + self.same_device(t3, c.name())?; + match (self, t2, t3) { + (Self::Cpu(s1), Self::Cpu(s2), Self::Cpu(s3)) => { + let (s, shape) = c.cpu_fwd(s1, l1, s2, l2, s3, l3)?; + Ok((Self::Cpu(s), shape)) + } + (Self::Cuda(s1), Self::Cuda(s2), Self::Cuda(s3)) => { + let (s, shape) = c.cuda_fwd(s1, l1, s2, l2, s3, l3)?; + Ok((Self::Cuda(s), shape)) + } + (Self::Metal(s1), Self::Metal(s2), Self::Metal(s3)) => { + let (s, shape) = c.metal_fwd(s1, l1, s2, l2, s3, l3)?; + Ok((Self::Metal(s), shape)) + } + _ => unreachable!(), + } + } + + pub(crate) fn inplace_op1(&mut self, l: &Layout, c: &dyn InplaceOp1) -> Result<()> { + match self { + Self::Cpu(storage) => c.cpu_fwd(storage, l), + Self::Cuda(storage) => c.cuda_fwd(storage, l), + Self::Metal(storage) => c.metal_fwd(storage, l), + } + } + + pub(crate) fn inplace_op2( + &mut self, + l1: &Layout, + t2: &Self, + l2: &Layout, + c: &dyn InplaceOp2, + ) -> Result<()> { + self.same_device(t2, c.name())?; + match (self, t2) { + (Self::Cpu(s1), Self::Cpu(s2)) => c.cpu_fwd(s1, l1, s2, l2), + (Self::Cuda(s1), Self::Cuda(s2)) => c.cuda_fwd(s1, l1, s2, l2), + (Self::Metal(s1), Self::Metal(s2)) => c.metal_fwd(s1, l1, s2, l2), + _ => unreachable!(), + } + } + + pub(crate) fn inplace_op3( + &mut self, + l1: &Layout, + t2: &Self, + l2: &Layout, + t3: &Self, + l3: &Layout, + c: &dyn InplaceOp3, + ) -> Result<()> { + self.same_device(t2, c.name())?; + self.same_device(t3, c.name())?; + match (self, t2, t3) { + (Self::Cpu(s1), Self::Cpu(s2), Self::Cpu(s3)) => c.cpu_fwd(s1, l1, s2, l2, s3, l3), + (Self::Cuda(s1), Self::Cuda(s2), Self::Cuda(s3)) => c.cuda_fwd(s1, l1, s2, l2, s3, l3), + (Self::Metal(s1), Self::Metal(s2), Self::Metal(s3)) => { + c.metal_fwd(s1, l1, s2, l2, s3, l3) + } + _ => unreachable!(), + } + } + + pub(crate) fn unary_impl(&self, layout: &Layout) -> Result { + match self { + Storage::Cpu(storage) => { + let storage = storage.unary_impl::(layout)?; + Ok(Self::Cpu(storage)) + } + Self::Cuda(storage) => { + let storage = storage.unary_impl::(layout)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = storage.unary_impl::(layout)?; + Ok(Self::Metal(storage)) + } + } + } + + pub(crate) fn binary_impl( + &self, + rhs: &Self, + lhs_layout: &Layout, + rhs_layout: &Layout, + ) -> Result { + self.same_device(rhs, B::NAME)?; + self.same_dtype(rhs, B::NAME)?; + match (self, rhs) { + (Storage::Cpu(lhs), Storage::Cpu(rhs)) => { + let storage = lhs.binary_impl::(rhs, lhs_layout, rhs_layout)?; + Ok(Self::Cpu(storage)) + } + (Self::Cuda(lhs), Self::Cuda(rhs)) => { + let storage = lhs.binary_impl::(rhs, lhs_layout, rhs_layout)?; + Ok(Self::Cuda(storage)) + } + (Self::Metal(lhs), Self::Metal(rhs)) => { + let storage = lhs.binary_impl::(rhs, lhs_layout, rhs_layout)?; + Ok(Self::Metal(storage)) + } + (lhs, rhs) => { + // Should not happen because of the same device check above but we're defensive + // anyway. + Err(Error::DeviceMismatchBinaryOp { + lhs: lhs.device().location(), + rhs: rhs.device().location(), + op: B::NAME, + } + .bt()) + } + } + } + + pub(crate) fn conv1d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConv1D, + ) -> Result { + self.same_device(kernel, "conv1d")?; + self.same_dtype(kernel, "conv1d")?; + match (self, &kernel) { + (Storage::Cpu(inp), Storage::Cpu(kernel)) => { + let s = inp.conv1d(l, kernel, kernel_l, params)?; + Ok(Self::Cpu(s)) + } + (Storage::Cuda(inp), Storage::Cuda(kernel)) => { + let s = inp.conv1d(l, kernel, kernel_l, params)?; + Ok(Self::Cuda(s)) + } + (Storage::Metal(inp), Storage::Metal(kernel)) => { + let s = inp.conv1d(l, kernel, kernel_l, params)?; + Ok(Self::Metal(s)) + } + (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { + lhs: lhs.device().location(), + rhs: rhs.device().location(), + op: "conv1d", + } + .bt()), + } + } + + pub(crate) fn conv_transpose1d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConvTranspose1D, + ) -> Result { + self.same_device(kernel, "conv-transpose1d")?; + self.same_dtype(kernel, "conv-transpose1d")?; + match (self, &kernel) { + (Storage::Cpu(inp), Storage::Cpu(kernel)) => { + let s = inp.conv_transpose1d(l, kernel, kernel_l, params)?; + Ok(Self::Cpu(s)) + } + (Storage::Cuda(inp), Storage::Cuda(kernel)) => { + let s = inp.conv_transpose1d(l, kernel, kernel_l, params)?; + Ok(Self::Cuda(s)) + } + (Storage::Metal(inp), Storage::Metal(kernel)) => { + let s = inp.conv_transpose1d(l, kernel, kernel_l, params)?; + Ok(Self::Metal(s)) + } + (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { + lhs: lhs.device().location(), + rhs: rhs.device().location(), + op: "conv-transpose1d", + } + .bt()), + } + } + + pub(crate) fn conv2d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConv2D, + ) -> Result { + self.same_device(kernel, "conv2d")?; + self.same_dtype(kernel, "conv2d")?; + match (self, &kernel) { + (Storage::Cpu(inp), Storage::Cpu(kernel)) => { + let s = inp.conv2d(l, kernel, kernel_l, params)?; + Ok(Self::Cpu(s)) + } + (Storage::Cuda(inp), Storage::Cuda(kernel)) => { + let s = inp.conv2d(l, kernel, kernel_l, params)?; + Ok(Self::Cuda(s)) + } + (Storage::Metal(inp), Storage::Metal(kernel)) => { + let s = inp.conv2d(l, kernel, kernel_l, params)?; + Ok(Self::Metal(s)) + } + (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { + lhs: lhs.device().location(), + rhs: rhs.device().location(), + op: "conv2d", + } + .bt()), + } + } + + pub(crate) fn conv_transpose2d( + &self, + l: &Layout, + kernel: &Self, + kernel_l: &Layout, + params: &crate::conv::ParamsConvTranspose2D, + ) -> Result { + self.same_device(kernel, "conv_transpose2d")?; + self.same_dtype(kernel, "conv_transpose2d")?; + match (self, &kernel) { + (Storage::Cpu(inp), Storage::Cpu(kernel)) => { + let s = inp.conv_transpose2d(l, kernel, kernel_l, params)?; + Ok(Self::Cpu(s)) + } + (Storage::Cuda(inp), Storage::Cuda(kernel)) => { + let s = inp.conv_transpose2d(l, kernel, kernel_l, params)?; + Ok(Self::Cuda(s)) + } + (Storage::Metal(inp), Storage::Metal(kernel)) => { + let s = inp.conv_transpose2d(l, kernel, kernel_l, params)?; + Ok(Self::Metal(s)) + } + (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { + lhs: lhs.device().location(), + rhs: rhs.device().location(), + op: "conv_transpose2d", + } + .bt()), + } + } + + pub(crate) fn avg_pool2d( + &self, + layout: &Layout, + kernel_size: (usize, usize), + stride: (usize, usize), + ) -> Result { + match self { + Storage::Cpu(storage) => { + let storage = storage.avg_pool2d(layout, kernel_size, stride)?; + Ok(Self::Cpu(storage)) + } + Self::Cuda(storage) => { + let storage = storage.avg_pool2d(layout, kernel_size, stride)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = storage.avg_pool2d(layout, kernel_size, stride)?; + Ok(Self::Metal(storage)) + } + } + } + + pub(crate) fn max_pool2d( + &self, + layout: &Layout, + kernel_size: (usize, usize), + stride: (usize, usize), + ) -> Result { + match self { + Storage::Cpu(storage) => { + let storage = storage.max_pool2d(layout, kernel_size, stride)?; + Ok(Self::Cpu(storage)) + } + Self::Cuda(storage) => { + let storage = storage.max_pool2d(layout, kernel_size, stride)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = storage.max_pool2d(layout, kernel_size, stride)?; + Ok(Self::Metal(storage)) + } + } + } + + pub(crate) fn upsample_nearest1d(&self, layout: &Layout, sz: usize) -> Result { + match self { + Storage::Cpu(storage) => { + let storage = storage.upsample_nearest1d(layout, sz)?; + Ok(Self::Cpu(storage)) + } + Self::Cuda(storage) => { + let storage = storage.upsample_nearest1d(layout, sz)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = storage.upsample_nearest1d(layout, sz)?; + Ok(Self::Metal(storage)) + } + } + } + + pub(crate) fn upsample_nearest2d(&self, layout: &Layout, h: usize, w: usize) -> Result { + match self { + Storage::Cpu(storage) => { + let storage = storage.upsample_nearest2d(layout, h, w)?; + Ok(Self::Cpu(storage)) + } + Self::Cuda(storage) => { + let storage = storage.upsample_nearest2d(layout, h, w)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = storage.upsample_nearest2d(layout, h, w)?; + Ok(Self::Metal(storage)) + } + } + } + + pub(crate) fn upsample_bilinear2d( + &self, + layout: &Layout, + h: usize, + w: usize, + align_corners: bool, + scale_h: Option, + scale_w: Option, + ) -> Result { + match self { + Storage::Cpu(storage) => { + let storage = + storage.upsample_bilinear2d(layout, h, w, align_corners, scale_h, scale_w)?; + Ok(Self::Cpu(storage)) + } + Self::Cuda(storage) => { + let storage = + storage.upsample_bilinear2d(layout, h, w, align_corners, scale_h, scale_w)?; + Ok(Self::Cuda(storage)) + } + Self::Metal(storage) => { + let storage = + storage.upsample_bilinear2d(layout, h, w, align_corners, scale_h, scale_w)?; + Ok(Self::Metal(storage)) + } + } + } + + pub(crate) fn where_cond( + &self, + layout: &Layout, + t: &Self, + layout_t: &Layout, + f: &Self, + layout_f: &Layout, + ) -> Result { + self.same_device(t, "where")?; + self.same_device(f, "where")?; + t.same_dtype(f, "where")?; + match (self, t, f) { + (Storage::Cpu(cond), Storage::Cpu(t), Storage::Cpu(f)) => { + let storage = cond.where_cond(layout, t, layout_t, f, layout_f)?; + Ok(Self::Cpu(storage)) + } + (Self::Cuda(cond), Self::Cuda(t), Self::Cuda(f)) => { + let storage = cond.where_cond(layout, t, layout_t, f, layout_f)?; + Ok(Self::Cuda(storage)) + } + (Self::Metal(cond), Self::Metal(t), Self::Metal(f)) => { + let storage = cond.where_cond(layout, t, layout_t, f, layout_f)?; + Ok(Self::Metal(storage)) + } + (_, lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { + lhs: lhs.device().location(), + rhs: rhs.device().location(), + op: "where", + } + .bt()), + } + } + + pub(crate) fn gather( + &self, + l: &Layout, + indexes: &Self, + indexes_l: &Layout, + d: usize, + ) -> Result { + self.same_device(indexes, "index-add")?; + match (self, indexes) { + (Self::Cpu(s), Self::Cpu(indexes)) => { + let storage = s.gather(l, indexes, indexes_l, d)?; + Ok(Self::Cpu(storage)) + } + (Self::Cuda(s), Self::Cuda(indexes)) => { + let storage = s.gather(l, indexes, indexes_l, d)?; + Ok(Self::Cuda(storage)) + } + (Self::Metal(s), Self::Metal(indexes)) => { + let storage = s.gather(l, indexes, indexes_l, d)?; + Ok(Self::Metal(storage)) + } + _ => unreachable!(), + } + } + + pub(crate) fn scatter_set( + &mut self, + l: &Layout, + indexes: &Self, + indexes_l: &Layout, + source: &Self, + source_l: &Layout, + d: usize, + ) -> Result<()> { + self.same_device(indexes, "scatter-set")?; + self.same_device(source, "scatter-set")?; + match (self, indexes, source) { + (Self::Cpu(s), Self::Cpu(indexes), Self::Cpu(source)) => { + s.scatter_set(l, indexes, indexes_l, source, source_l, d)?; + } + (Self::Cuda(s), Self::Cuda(indexes), Self::Cuda(source)) => { + s.scatter_set(l, indexes, indexes_l, source, source_l, d)?; + } + (Self::Metal(s), Self::Metal(indexes), Self::Metal(source)) => { + s.scatter_set(l, indexes, indexes_l, source, source_l, d)?; + } + _ => unreachable!(), + } + Ok(()) + } + + pub(crate) fn scatter_add( + &mut self, + l: &Layout, + indexes: &Self, + indexes_l: &Layout, + source: &Self, + source_l: &Layout, + d: usize, + ) -> Result<()> { + self.same_device(indexes, "scatter-add")?; + self.same_device(source, "scatter-add")?; + match (self, indexes, source) { + (Self::Cpu(s), Self::Cpu(indexes), Self::Cpu(source)) => { + s.scatter_add_set(l, indexes, indexes_l, source, source_l, d)?; + } + (Self::Cuda(s), Self::Cuda(indexes), Self::Cuda(source)) => { + s.scatter_add_set(l, indexes, indexes_l, source, source_l, d)?; + } + (Self::Metal(s), Self::Metal(indexes), Self::Metal(source)) => { + s.scatter_add_set(l, indexes, indexes_l, source, source_l, d)?; + } + _ => unreachable!(), + } + Ok(()) + } + + pub(crate) fn index_add( + &self, + l: &Layout, + indexes: &Self, + indexes_l: &Layout, + source: &Self, + source_l: &Layout, + d: usize, + ) -> Result { + self.same_device(indexes, "index-add")?; + self.same_device(source, "index-add")?; + match (self, indexes, source) { + (Self::Cpu(s), Self::Cpu(indexes), Self::Cpu(source)) => { + let storage = s.index_add(l, indexes, indexes_l, source, source_l, d)?; + Ok(Self::Cpu(storage)) + } + (Self::Cuda(s), Self::Cuda(indexes), Self::Cuda(source)) => { + let storage = s.index_add(l, indexes, indexes_l, source, source_l, d)?; + Ok(Self::Cuda(storage)) + } + (Self::Metal(s), Self::Metal(indexes), Self::Metal(source)) => { + let storage = s.index_add(l, indexes, indexes_l, source, source_l, d)?; + Ok(Self::Metal(storage)) + } + _ => unreachable!(), + } + } + + pub(crate) fn index_select( + &self, + rhs: &Self, + lhs_l: &Layout, + rhs_l: &Layout, + d: usize, + ) -> Result { + self.same_device(rhs, "index-select")?; + match (self, rhs) { + (Self::Cpu(lhs), Self::Cpu(rhs)) => { + let storage = lhs.index_select(rhs, lhs_l, rhs_l, d)?; + Ok(Self::Cpu(storage)) + } + (Self::Cuda(lhs), Self::Cuda(rhs)) => { + let storage = lhs.index_select(rhs, lhs_l, rhs_l, d)?; + Ok(Self::Cuda(storage)) + } + (Self::Metal(lhs), Self::Metal(rhs)) => { + let storage = lhs.index_select(rhs, lhs_l, rhs_l, d)?; + Ok(Self::Metal(storage)) + } + (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { + lhs: lhs.device().location(), + rhs: rhs.device().location(), + op: "index-select", + } + .bt()), + } + } + + pub(crate) fn matmul( + &self, + rhs: &Self, + bmnk: (usize, usize, usize, usize), + lhs_layout: &Layout, + rhs_layout: &Layout, + ) -> Result { + self.same_device(rhs, "matmul")?; + self.same_dtype(rhs, "matmul")?; + match (self, rhs) { + (Self::Cpu(lhs), Self::Cpu(rhs)) => { + let storage = lhs.matmul(rhs, bmnk, lhs_layout, rhs_layout)?; + Ok(Self::Cpu(storage)) + } + (Self::Cuda(lhs), Self::Cuda(rhs)) => { + let storage = lhs.matmul(rhs, bmnk, lhs_layout, rhs_layout)?; + Ok(Self::Cuda(storage)) + } + (Self::Metal(lhs), Self::Metal(rhs)) => { + let storage = lhs.matmul(rhs, bmnk, lhs_layout, rhs_layout)?; + Ok(Self::Metal(storage)) + } + (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { + lhs: lhs.device().location(), + rhs: rhs.device().location(), + op: "matmul", + } + .bt()), + } + } + + // self, the source can be strided whereas dst is contiguous. + pub(crate) fn copy_strided_src( + &self, + dst: &mut Self, + dst_offset: usize, + src_l: &Layout, + ) -> Result<()> { + match (self, dst) { + (Self::Cpu(src), Self::Cpu(dst)) => src.copy_strided_src(dst, dst_offset, src_l), + (Self::Cuda(src), Self::Cuda(dst)) => Ok(src.copy_strided_src(dst, dst_offset, src_l)?), + (Self::Metal(src), Self::Metal(dst)) => { + Ok(src.copy_strided_src(dst, dst_offset, src_l)?) + } + (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { + lhs: lhs.device().location(), + rhs: rhs.device().location(), + op: "copy", + } + .bt()), + } + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn copy2d( + &self, + dst: &mut Self, + d1: usize, + d2: usize, + src_s: usize, + dst_s: usize, + src_o: usize, + dst_o: usize, + ) -> Result<()> { + match (self, dst) { + (Self::Cpu(src), Self::Cpu(dst)) => src.copy2d(dst, d1, d2, src_s, dst_s, src_o, dst_o), + (Self::Cuda(src), Self::Cuda(dst)) => { + Ok(src.copy2d(dst, d1, d2, src_s, dst_s, src_o, dst_o)?) + } + (Self::Metal(src), Self::Metal(dst)) => { + Ok(src.copy2d(dst, d1, d2, src_s, dst_s, src_o, dst_o)?) + } + (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { + lhs: lhs.device().location(), + rhs: rhs.device().location(), + op: "copy2d", + } + .bt()), + } + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/streaming.rs b/v2/dependency-patches/candle-core-0.9.2/src/streaming.rs new file mode 100644 index 0000000000..f4c0a9ff0b --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/streaming.rs @@ -0,0 +1,208 @@ +//! StreamTensror useful for streaming ops. +//! +use crate::{Result, Shape, Tensor}; + +pub trait Dim: crate::shape::Dim + Copy {} +impl Dim for T {} + +/// A stream tensor is used in streaming module. It can either contain an actual tensor or be +/// empty. +#[derive(Clone)] +pub struct StreamTensor(Option); + +impl std::fmt::Debug for StreamTensor { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.0 { + Some(t) => write!(f, "{:?}", t.shape()), + None => write!(f, "Empty"), + } + } +} + +impl std::convert::From> for StreamTensor { + fn from(value: Option) -> Self { + Self(value) + } +} + +impl std::convert::From for StreamTensor { + fn from(value: Tensor) -> Self { + Self(Some(value)) + } +} + +impl std::convert::From<()> for StreamTensor { + fn from(_value: ()) -> Self { + Self(None) + } +} + +impl StreamTensor { + pub fn empty() -> Self { + Self(None) + } + + pub fn from_tensor(tensor: Tensor) -> Self { + Self(Some(tensor)) + } + + pub fn shape(&self) -> Option<&Shape> { + self.0.as_ref().map(|t| t.shape()) + } + + pub fn cat2(&self, rhs: &Self, dim: D) -> Result { + let xs = match (&self.0, &rhs.0) { + (Some(lhs), Some(rhs)) => { + let xs = Tensor::cat(&[lhs, rhs], dim)?; + Some(xs) + } + (Some(xs), None) | (None, Some(xs)) => Some(xs.clone()), + (None, None) => None, + }; + Ok(Self(xs)) + } + + pub fn seq_len(&self, dim: D) -> Result { + match &self.0 { + None => Ok(0), + Some(v) => v.dim(dim), + } + } + + pub fn reset(&mut self) { + self.0 = None + } + + pub fn narrow(&self, dim: D, offset: usize, len: usize) -> Result { + let t = match &self.0 { + None => None, + Some(t) => { + let seq_len = t.dim(dim)?; + if seq_len <= offset { + None + } else { + let t = t.narrow(dim, offset, usize::min(len, seq_len - offset))?; + Some(t) + } + } + }; + Ok(Self(t)) + } + + /// Splits the Streaming Tensor on the time axis `dim` with the first `lhs_len` elements + /// returned in the first output and the remaining in the second output. + pub fn split(&self, dim: D, lhs_len: usize) -> Result<(Self, Self)> { + match &self.0 { + None => Ok((Self::empty(), Self::empty())), + Some(t) => { + let seq_len = t.dim(dim)?; + let lhs_len = usize::min(seq_len, lhs_len); + if lhs_len == 0 { + Ok((Self::empty(), t.clone().into())) + } else { + let lhs = Self::from_tensor(t.narrow(dim, 0, lhs_len)?); + let rhs_len = seq_len - lhs_len; + let rhs = if rhs_len == 0 { + Self::empty() + } else { + Self::from_tensor(t.narrow(dim, lhs_len, rhs_len)?) + }; + Ok((lhs, rhs)) + } + } + } + } + + pub fn as_option(&self) -> Option<&Tensor> { + self.0.as_ref() + } + + pub fn apply(&self, m: &M) -> Result { + match &self.0 { + None => Ok(Self::empty()), + Some(t) => Ok(Self::from_tensor(t.apply(m)?)), + } + } +} + +/// Streaming modules take as input a stream tensor and return a stream tensor. They may perform +/// some internal buffering so that enough data has been received for the module to be able to +/// perform some operations. +pub trait StreamingModule { + // TODO: Should we also have a flush method? + fn step(&mut self, xs: &StreamTensor) -> Result; + fn reset_state(&mut self); +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum BinOp { + Add, + Mul, + Sub, + Div, +} + +#[derive(Debug, Clone)] +pub struct StreamingBinOp { + prev_lhs: StreamTensor, + prev_rhs: StreamTensor, + pub op: BinOp, + pub dim: crate::D, +} + +impl StreamingBinOp { + pub fn new(op: BinOp, dim: crate::D) -> Self { + Self { + prev_lhs: StreamTensor::empty(), + prev_rhs: StreamTensor::empty(), + op, + dim, + } + } + + pub fn reset_state(&mut self) { + self.prev_lhs.reset(); + self.prev_rhs.reset(); + } + + pub fn forward(&self, lhs: &Tensor, rhs: &Tensor) -> Result { + match self.op { + BinOp::Add => Tensor::add(lhs, rhs), + BinOp::Mul => Tensor::mul(lhs, rhs), + BinOp::Sub => Tensor::sub(lhs, rhs), + BinOp::Div => Tensor::div(lhs, rhs), + } + } + + pub fn step(&mut self, lhs: &StreamTensor, rhs: &StreamTensor) -> Result { + let lhs = StreamTensor::cat2(&self.prev_lhs, lhs, self.dim)?; + let rhs = StreamTensor::cat2(&self.prev_rhs, rhs, self.dim)?; + let lhs_len = lhs.seq_len(self.dim)?; + let rhs_len = rhs.seq_len(self.dim)?; + let common_len = usize::min(lhs_len, rhs_len); + let (lhs, prev_lhs) = lhs.split(self.dim, common_len)?; + let (rhs, prev_rhs) = rhs.split(self.dim, common_len)?; + let ys = match (lhs.0, rhs.0) { + (Some(lhs), Some(rhs)) => { + let ys = self.forward(&lhs, &rhs)?; + StreamTensor::from_tensor(ys) + } + (None, None) => StreamTensor::empty(), + (lhs, rhs) => crate::bail!("INTERNAL ERROR inconsistent lhs and rhs {lhs:?} {rhs:?}"), + }; + self.prev_lhs = prev_lhs; + self.prev_rhs = prev_rhs; + Ok(ys) + } +} + +/// Simple wrapper that doesn't do any buffering. +pub struct Map(T); + +impl StreamingModule for Map { + fn reset_state(&mut self) {} + + fn step(&mut self, xs: &StreamTensor) -> Result { + xs.apply(&self.0) + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/strided_index.rs b/v2/dependency-patches/candle-core-0.9.2/src/strided_index.rs new file mode 100644 index 0000000000..a31d406a43 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/strided_index.rs @@ -0,0 +1,94 @@ +use crate::Layout; + +/// An iterator over offset position for items of an N-dimensional arrays stored in a +/// flat buffer using some potential strides. +#[derive(Debug)] +pub struct StridedIndex<'a> { + next_storage_index: Option, + multi_index: Vec, + dims: &'a [usize], + stride: &'a [usize], + remaining: usize, +} + +impl<'a> StridedIndex<'a> { + pub(crate) fn new(dims: &'a [usize], stride: &'a [usize], start_offset: usize) -> Self { + let elem_count: usize = dims.iter().product(); + let next_storage_index = if elem_count == 0 { + None + } else { + // This applies to the scalar case. + Some(start_offset) + }; + StridedIndex { + next_storage_index, + multi_index: vec![0; dims.len()], + dims, + stride, + remaining: elem_count, + } + } + + pub(crate) fn from_layout(l: &'a Layout) -> Self { + Self::new(l.dims(), l.stride(), l.start_offset()) + } +} + +impl Iterator for StridedIndex<'_> { + type Item = usize; + + #[inline] + fn next(&mut self) -> Option { + let storage_index = self.next_storage_index?; + let mut updated = false; + let mut next_storage_index = storage_index; + for ((multi_i, max_i), stride_i) in self + .multi_index + .iter_mut() + .zip(self.dims.iter()) + .zip(self.stride.iter()) + .rev() + { + let next_i = *multi_i + 1; + if next_i < *max_i { + *multi_i = next_i; + updated = true; + next_storage_index += stride_i; + break; + } else { + next_storage_index -= *multi_i * stride_i; + *multi_i = 0 + } + } + self.remaining -= 1; + self.next_storage_index = if updated { + Some(next_storage_index) + } else { + None + }; + Some(storage_index) + } + + #[inline] + fn size_hint(&self) -> (usize, Option) { + (self.remaining, Some(self.remaining)) + } +} + +impl ExactSizeIterator for StridedIndex<'_> { + fn len(&self) -> usize { + self.remaining + } +} + +#[derive(Debug)] +pub enum StridedBlocks<'a> { + SingleBlock { + start_offset: usize, + len: usize, + }, + MultipleBlocks { + block_start_index: StridedIndex<'a>, + block_len: usize, + }, +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/tensor.rs b/v2/dependency-patches/candle-core-0.9.2/src/tensor.rs new file mode 100644 index 0000000000..52b7f79b77 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/tensor.rs @@ -0,0 +1,3116 @@ +//! Tensors are N-dimensional matrixes of elements using a single data type. +#![allow(clippy::redundant_closure_call)] +use crate::backend::{BackendDevice, BackendStorage}; +use crate::op::{BackpropOp, BinaryOp, CmpOp, Op, ReduceOp, UnaryOp}; +use crate::scalar::TensorOrScalar; +use crate::shape::{Dim, Dims, ShapeWithOneHole}; +use crate::{bail, storage::Storage, DType, Device, Error, Layout, Result, Shape}; +use std::sync::{Arc, RwLock}; + +/// Unique identifier for tensors. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +pub struct TensorId(usize); + +impl TensorId { + fn new() -> Self { + // https://users.rust-lang.org/t/idiomatic-rust-way-to-generate-unique-id/33805 + use std::sync::atomic; + static COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::new(1); + Self(COUNTER.fetch_add(1, atomic::Ordering::Relaxed)) + } +} + +pub struct Tensor_ { + id: TensorId, + // As we provide inner mutability on the tensor content, the alternatives are: + // - Using a mutex, this would have the highest cost when retrieving the storage but would + // prevent errors when concurrent access takes place. Mutex would also be subject to + // deadlocks for example using the current code if the same tensor is used twice by a single + // binary op. + // - Using a refcell unsafe cell would have some intermediary cost, borrow checking would be + // verified dynamically, but the resulting tensors would not be send or sync. + // - Using an unsafe cell would have the lowest cost but undefined behavior on concurrent + // accesses. + // Ideally, we would use Arc for tensors on which we don't plan on modifying the data + // and Arc> for tensors where the data could be modified, e.g. variables but + // that's tricky to encode in the current setup. + storage: Arc>, + layout: Layout, + op: BackpropOp, + is_variable: bool, + dtype: DType, + device: Device, +} + +impl AsRef for Tensor { + fn as_ref(&self) -> &Tensor { + self + } +} + +// Tensors are refcounted so that cloning is cheap when building the op graph. +// Storages are also refcounted independently so that its possible to avoid +// copying the storage for operations that only modify the shape or stride. +#[derive(Clone)] +/// The core struct for manipulating tensors. +/// +/// ```rust +/// use candle_core::{Tensor, DType, Device}; +/// +/// let a = Tensor::arange(0f32, 6f32, &Device::Cpu)?.reshape((2, 3))?; +/// let b = Tensor::arange(0f32, 12f32, &Device::Cpu)?.reshape((3, 4))?; +/// +/// let c = a.matmul(&b)?; +/// # Ok::<(), candle_core::Error>(()) +/// ``` +/// +/// Tensors are reference counted with [`Arc`] so cloning them is cheap. +pub struct Tensor(Arc); + +impl std::ops::Deref for Tensor { + type Target = Tensor_; + + fn deref(&self) -> &Self::Target { + self.0.as_ref() + } +} + +macro_rules! unary_op { + ($fn_name:ident, $op_name:ident) => { + pub fn $fn_name(&self) -> Result { + let shape = self.shape(); + if shape.elem_count() == 0 { + return Ok(self.clone()); + } + let storage = self + .storage() + .unary_impl::(self.layout())?; + let op = BackpropOp::new1(self, |s| Op::Unary(s, UnaryOp::$op_name)); + Ok(from_storage(storage, shape.clone(), op, false)) + } + }; +} + +macro_rules! binary_op { + ($fn_name:ident, $op_name:ident) => { + pub fn $fn_name(&self, rhs: &Self) -> Result { + let shape = self.same_shape_binary_op(rhs, stringify!($fn_name))?; + if shape.elem_count() == 0 { + return Ok(self.clone()); + } + let storage = self.storage().binary_impl::( + &*rhs.storage(), + self.layout(), + rhs.layout(), + )?; + let op = BackpropOp::new2(self, rhs, |t1, t2| Op::Binary(t1, t2, BinaryOp::$op_name)); + Ok(from_storage(storage, shape.clone(), op, false)) + } + }; +} + +macro_rules! binary_op_scalar { + ($fn_name:ident, $op_name:ident) => { + pub fn $fn_name(&self, rhs: T) -> Result { + let rhs = match rhs.to_tensor_scalar()? { + crate::scalar::TensorScalar::Tensor(rhs) => rhs, + crate::scalar::TensorScalar::Scalar(rhs) => rhs + .to_dtype(self.dtype())? + .to_device(self.device())? + .broadcast_as(self.shape())?, + }; + let shape = self.same_shape_binary_op(&rhs, stringify!($fn_name))?; + if self.elem_count() == 0 { + return Ok(self.clone()); + } + let storage = self.storage().binary_impl::( + &*rhs.storage(), + self.layout(), + rhs.layout(), + )?; + let op = BackpropOp::new2(self, &rhs, |t1, t2| Op::Binary(t1, t2, BinaryOp::$op_name)); + Ok(from_storage(storage, shape.clone(), op, false)) + } + }; +} + +macro_rules! broadcast_binary_op { + ($fn_name:ident, $inner_fn_name:ident) => { + pub fn $fn_name(&self, rhs: &Self) -> Result { + let lhs = self; + let shape = lhs + .shape() + .broadcast_shape_binary_op(rhs.shape(), stringify!($fn_name))?; + let l_broadcast = shape != *lhs.shape(); + let r_broadcast = shape != *rhs.shape(); + match (l_broadcast, r_broadcast) { + (true, true) => lhs + .broadcast_as(&shape)? + .$inner_fn_name(&rhs.broadcast_as(&shape)?), + (false, true) => lhs.$inner_fn_name(&rhs.broadcast_as(&shape)?), + (true, false) => lhs.broadcast_as(&shape)?.$inner_fn_name(rhs), + (false, false) => lhs.$inner_fn_name(rhs), + } + } + }; +} + +/// Creates a fresh tensor structure based on a storage and a shape, this uses contiguous strides. +pub(crate) fn from_storage>( + storage: Storage, + shape: S, + op: BackpropOp, + is_variable: bool, +) -> Tensor { + let dtype = storage.dtype(); + let device = storage.device(); + let tensor_ = Tensor_ { + id: TensorId::new(), + storage: Arc::new(RwLock::new(storage)), + layout: Layout::contiguous(shape), + op, + is_variable, + dtype, + device, + }; + Tensor(Arc::new(tensor_)) +} + +impl Tensor { + pub(crate) fn ones_impl>( + shape: S, + dtype: DType, + device: &Device, + is_variable: bool, + ) -> Result { + let none = BackpropOp::none(); + let shape = shape.into(); + let mut storage = unsafe { device.alloc_uninit(&shape, dtype)? }; + let layout = Layout::contiguous(shape.clone()); + storage.const_set(crate::scalar::Scalar::one(dtype), &layout)?; + Ok(from_storage(storage, shape, none, is_variable)) + } + + /// Creates a new tensor filled with ones. + /// + /// ```rust + /// use candle_core::{Tensor, DType, Device}; + /// let a = Tensor::ones((2, 3), DType::F32, &Device::Cpu)?; + /// let b = Tensor::from_slice(&[1.0f32, 1.0, 1.0, 1.0, 1.0, 1.0], (2, 3), &Device::Cpu)?; + /// // a == b + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn ones>(shape: S, dtype: DType, device: &Device) -> Result { + Self::ones_impl(shape, dtype, device, false) + } + + pub fn const_set(&self, value: crate::scalar::Scalar) -> Result<()> { + self.storage_mut().const_set(value, self.layout()) + } + + pub fn zero_set(&self) -> Result<()> { + self.const_set(crate::scalar::Scalar::zero(self.dtype())) + } + + pub fn one_set(&self) -> Result<()> { + self.const_set(crate::scalar::Scalar::one(self.dtype())) + } + + /// Creates a new tensor filled with ones with same shape, dtype, and device as the other tensor. + /// + /// ```rust + /// use candle_core::{Tensor, DType, Device}; + /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; + /// let b = a.ones_like()?; + /// // b == a + 1 + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn ones_like(&self) -> Result { + Tensor::ones(self.shape(), self.dtype(), self.device()) + } + + // Do not expose outside of the crate, the `is_variable=true` case should only be accessed from + // the variable module. + pub(crate) fn zeros_impl>( + shape: S, + dtype: DType, + device: &Device, + is_variable: bool, + ) -> Result { + let none = BackpropOp::none(); + let shape = shape.into(); + let storage = device.zeros(&shape, dtype)?; + Ok(from_storage(storage, shape, none, is_variable)) + } + + /// Creates a new tensor filled with zeros. + /// + /// ```rust + /// use candle_core::{Tensor, DType, Device}; + /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; + /// let b = Tensor::from_slice(&[0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0], (2, 3), &Device::Cpu)?; + /// // a == b + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn zeros>(shape: S, dtype: DType, device: &Device) -> Result { + Self::zeros_impl(shape, dtype, device, false) + } + + /// Creates a new tensor filled with zeros with same shape, dtype, and device as the other + /// tensor. + /// + /// ```rust + /// use candle_core::{Tensor, DType, Device}; + /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; + /// let b = a.zeros_like()?; + /// // b is on CPU f32. + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn zeros_like(&self) -> Result { + Tensor::zeros(self.shape(), self.dtype(), self.device()) + } + + // Do not expose outside of the crate, the `is_variable=true` case should only be accessed from + // the variable module. + pub(crate) unsafe fn empty_impl>( + shape: S, + dtype: DType, + device: &Device, + is_variable: bool, + ) -> Result { + let none = BackpropOp::none(); + let shape = shape.into(); + let storage = device.alloc_uninit(&shape, dtype)?; + Ok(from_storage(storage, shape, none, is_variable)) + } + + /// Creates a new tensor filled with uninitialized memory. + /// + /// # Safety + /// This returns uninitialized memory. + /// + /// ```rust + /// use candle_core::{Tensor, DType, Device}; + /// let a = unsafe { Tensor::empty((2, 3), DType::F32, &Device::Cpu)? }; + /// // a == b + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub unsafe fn empty>(shape: S, dtype: DType, device: &Device) -> Result { + Self::empty_impl(shape, dtype, device, false) + } + + /// Creates a new tensor filled with uninitialized memory of the same shape, dtype, and device as the other + /// tensor. + /// + /// # Safety + /// This returns uninitialized memory. + /// + /// ```rust + /// use candle_core::{Tensor, DType, Device}; + /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; + /// let b = unsafe { a.empty_like()? }; + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub unsafe fn empty_like(&self) -> Result { + Tensor::empty(self.shape(), self.dtype(), self.device()) + } + + pub(crate) fn rand_impl, T: crate::FloatDType>( + lo: T, + up: T, + s: S, + device: &Device, + is_variable: bool, + ) -> Result { + let s = s.into(); + let storage = device.rand_uniform(lo, up, &s)?; + let none = BackpropOp::none(); + Ok(from_storage(storage, s, none, is_variable)) + } + + pub(crate) fn rand_f64_impl>( + lo: f64, + up: f64, + s: S, + dtype: DType, + device: &Device, + is_variable: bool, + ) -> Result { + let s = s.into(); + let storage = device.rand_uniform_f64(lo, up, &s, dtype)?; + let none = BackpropOp::none(); + Ok(from_storage(storage, s, none, is_variable)) + } + + /// Creates a new tensor initialized with values sampled uniformly between `lo` and `up`. + pub fn rand, T: crate::FloatDType>( + lo: T, + up: T, + s: S, + device: &Device, + ) -> Result { + Self::rand_impl(lo, up, s, device, false) + } + + pub fn rand_like(&self, lo: f64, up: f64) -> Result { + Tensor::rand_f64_impl(lo, up, self.shape(), self.dtype(), self.device(), false) + } + + pub(crate) fn randn_impl, T: crate::FloatDType>( + mean: T, + std: T, + s: S, + device: &Device, + is_variable: bool, + ) -> Result { + let s = s.into(); + let storage = device.rand_normal(mean, std, &s)?; + let none = BackpropOp::none(); + Ok(from_storage(storage, s, none, is_variable)) + } + + pub(crate) fn randn_f64_impl>( + mean: f64, + std: f64, + s: S, + dtype: DType, + device: &Device, + is_variable: bool, + ) -> Result { + let s = s.into(); + let storage = device.rand_normal_f64(mean, std, &s, dtype)?; + let none = BackpropOp::none(); + Ok(from_storage(storage, s, none, is_variable)) + } + + pub fn randn_like(&self, mean: f64, stdev: f64) -> Result { + Tensor::randn_f64_impl( + mean, + stdev, + self.shape(), + self.dtype(), + self.device(), + false, + ) + } + + /// Creates a new tensor initialized with values sampled from a normal distribution with the + /// specified `mean` and standard deviation `std`. + pub fn randn, T: crate::FloatDType>( + mean: T, + std: T, + s: S, + device: &Device, + ) -> Result { + Self::randn_impl(mean, std, s, device, false) + } + + pub(crate) fn new_impl( + array: A, + shape: Shape, + device: &Device, + is_variable: bool, + ) -> Result { + let n: usize = shape.elem_count(); + let buffer_size: usize = array.shape()?.elem_count(); + if buffer_size != n { + return Err(Error::ShapeMismatch { buffer_size, shape }.bt()); + } + let storage = device.storage(array)?; + let none = BackpropOp::none(); + Ok(from_storage(storage, shape, none, is_variable)) + } + + /// Creates a new tensor on the specified device using the content and shape of the input. + pub fn new(array: A, device: &Device) -> Result { + let shape = array.shape()?; + Self::new_impl(array, shape, device, false) + } + + /// Returns a new tensor with all the elements having the same specified value. + ///```rust + /// use candle_core::{Tensor, Device}; + /// let a = Tensor::full(3.5, (2, 4), &Device::Cpu)?; + /// + /// assert_eq!(a.to_vec2::()?, &[ + /// [3.5, 3.5, 3.5, 3.5], + /// [3.5, 3.5, 3.5, 3.5], + /// ]); + /// # Ok::<(), candle_core::Error>(()) + pub fn full>( + value: D, + shape: S, + device: &Device, + ) -> Result { + let none = BackpropOp::none(); + let shape = shape.into(); + let mut storage = unsafe { device.alloc_uninit(&shape, D::DTYPE)? }; + let layout = Layout::contiguous(shape.clone()); + storage.const_set(value.to_scalar(), &layout)?; + Ok(from_storage(storage, shape, none, false)) + } + + /// Creates a new 1D tensor from an iterator. + ///```rust + /// use candle_core::{Tensor, Device}; + /// let a = Tensor::from_iter( [1.0, 2.0, 3.0, 4.0].into_iter(), &Device::Cpu)?; + /// + /// assert_eq!(a.to_vec1::()?, &[1.0, 2.0, 3.0, 4.0]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn from_iter( + iter: impl IntoIterator, + device: &Device, + ) -> Result { + let data = iter.into_iter().collect::>(); + let len = data.len(); + Self::from_vec_impl(data, len, device, false) + } + + /// Creates a new 1D tensor with values from the interval `[start, end)` taken with a common + /// difference `1` from `start`. + ///```rust + /// use candle_core::{Tensor, Device}; + /// let a = Tensor::arange(2., 5., &Device::Cpu)?; + /// + /// assert_eq!(a.to_vec1::()?, &[2., 3., 4.]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn arange(start: D, end: D, device: &Device) -> Result { + Self::arange_step(start, end, D::one(), device) + } + + /// Creates a new 1D tensor with values from the interval `[start, end)` taken with a common + /// difference `step` from `start`. + ///```rust + /// use candle_core::{Tensor, Device}; + /// let a = Tensor::arange_step(2.0, 4.0, 0.5, &Device::Cpu)?; + /// + /// assert_eq!(a.to_vec1::()?, &[2.0, 2.5, 3.0, 3.5]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn arange_step( + start: D, + end: D, + step: D, + device: &Device, + ) -> Result { + if D::is_zero(&step) { + bail!("step cannot be zero") + } + let mut data = vec![]; + let mut current = start; + if step >= D::zero() { + while current < end { + data.push(current); + current += step; + } + } else { + while current > end { + data.push(current); + current += step; + } + } + let len = data.len(); + Self::from_vec_impl(data, len, device, false) + } + + pub(crate) fn from_vec_impl( + data: Vec, + shape: S, + device: &Device, + is_variable: bool, + ) -> Result { + let shape = shape.into_shape(data.len())?; + let storage = device.storage_owned(data)?; + let none = BackpropOp::none(); + Ok(from_storage(storage, shape, none, is_variable)) + } + + /// Creates a new tensor initialized with values from the input vector. The number of elements + /// in this vector must be the same as the number of elements defined by the shape. + /// If the device is cpu, no data copy is made. + ///```rust + /// use candle_core::{Tensor, Device}; + /// let a = Tensor::from_vec(vec!{1., 2., 3., 4., 5., 6.}, (2, 3), &Device::Cpu)?; + /// + /// assert_eq!(a.to_vec2::()?, &[ + /// [1., 2., 3.], + /// [4., 5., 6.] + /// ]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn from_vec( + data: Vec, + shape: S, + device: &Device, + ) -> Result { + Self::from_vec_impl(data, shape, device, false) + } + + /// Creates a new tensor initialized with values from the input slice. The number of elements + /// in this vector must be the same as the number of elements defined by the shape. + ///```rust + /// use candle_core::{Tensor, Device}; + /// let values = vec![1., 2., 3., 4., 5., 6., 7., 8.]; + /// let a = Tensor::from_slice(&values[1..7], (2, 3), &Device::Cpu)?; + /// + /// assert_eq!(a.to_vec2::()?, &[ + /// [2., 3., 4.], + /// [5., 6., 7.] + /// ]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn from_slice( + array: &[D], + shape: S, + device: &Device, + ) -> Result { + let shape = shape.into_shape(array.len())?; + let storage = device.storage_from_slice(array)?; + let none = BackpropOp::none(); + Ok(from_storage(storage, shape, none, false)) + } + + pub(crate) fn same_shape_binary_op(&self, rhs: &Self, op: &'static str) -> Result<&Shape> { + let lhs = self.shape(); + let rhs = rhs.shape(); + if lhs != rhs { + Err(Error::ShapeMismatchBinaryOp { + lhs: lhs.clone(), + rhs: rhs.clone(), + op, + } + .bt()) + } else { + Ok(lhs) + } + } + + /// Returns true if the computation graph should track this op, that is if it is + /// a variable or if it has some variable as dependencies. + pub fn track_op(&self) -> bool { + self.is_variable || self.op.is_some() + } + + /// Creates a fresh tensor structure based on a storage and a shape. + /// + /// # Note + /// - This uses contiguous strides + /// - Ensure the shape is compatible with the shape of the storage. + pub fn from_storage>( + storage: Storage, + shape: S, + op: BackpropOp, + is_variable: bool, + ) -> Tensor { + from_storage(storage, shape, op, is_variable) + } + + // TODO: Also make an inplace version or a pre-allocated? This could be tricky + // if this can create cycles in the compute graph. + binary_op!(add, Add); + binary_op!(mul, Mul); + binary_op!(sub, Sub); + binary_op!(div, Div); + binary_op_scalar!(maximum, Maximum); + binary_op_scalar!(minimum, Minimum); + broadcast_binary_op!(broadcast_add, add); + broadcast_binary_op!(broadcast_mul, mul); + broadcast_binary_op!(broadcast_sub, sub); + broadcast_binary_op!(broadcast_div, div); + broadcast_binary_op!(broadcast_maximum, maximum); + broadcast_binary_op!(broadcast_minimum, minimum); + broadcast_binary_op!(broadcast_eq, eq); + broadcast_binary_op!(broadcast_ne, ne); + broadcast_binary_op!(broadcast_lt, lt); + broadcast_binary_op!(broadcast_le, le); + broadcast_binary_op!(broadcast_gt, gt); + broadcast_binary_op!(broadcast_ge, ge); + + unary_op!(recip, Recip); + unary_op!(neg, Neg); + unary_op!(exp, Exp); + unary_op!(log, Log); + unary_op!(sin, Sin); + unary_op!(cos, Cos); + unary_op!(tanh, Tanh); + unary_op!(abs, Abs); + unary_op!(sqr, Sqr); + unary_op!(sqrt, Sqrt); + unary_op!(gelu, Gelu); + unary_op!(gelu_erf, GeluErf); + unary_op!(erf, Erf); + unary_op!(relu, Relu); + unary_op!(silu, Silu); + unary_op!(ceil, Ceil); + unary_op!(floor, Floor); + unary_op!(round, Round); + unary_op!(sign, Sign); + + /// Round element of the input tensor to the nearest integer. + /// + /// If the number of decimals is negative, it specifies the number of positions to the left of + /// the decimal point. + pub fn round_to(&self, decimals: i32) -> Result { + let mult = 10f64.powi(decimals); + (self * mult)?.round()? * (1f64 / mult) + } + + /// Retrieves the single scalar value hold in the tensor. If the tensor contains multiple + /// dimensions, an error is returned instead. + pub fn to_scalar(&self) -> Result { + if self.rank() != 0 { + Err(Error::UnexpectedNumberOfDims { + expected: 0, + got: self.rank(), + shape: self.shape().clone(), + } + .bt())? + } + let from_cpu_storage = |cpu_storage: &crate::CpuStorage| { + let data = S::cpu_storage_as_slice(cpu_storage)?; + Ok::<_, Error>(data[self.layout().start_offset()]) + }; + match &*self.storage() { + Storage::Cpu(cpu_storage) => from_cpu_storage(cpu_storage), + Storage::Cuda(storage) => from_cpu_storage(&storage.to_cpu_storage()?), + Storage::Metal(storage) => from_cpu_storage(&storage.to_cpu_storage()?), + } + } + + /// An alias for `to_scalar`. + pub fn to_vec0(&self) -> Result { + self.to_scalar::() + } + + /// Repeat this tensor along the specified dimensions. + pub fn repeat>(&self, shape: S) -> Result { + // Similar to PyTorch, we extend the number of dimensions of self if needed. + let repeats = shape.into(); + let repeats = repeats.dims(); + let mut inp = if self.rank() < repeats.len() { + let shape = [vec![1; repeats.len() - self.rank()], self.dims().to_vec()].concat(); + self.reshape(shape)? + } else { + self.clone() + }; + for (idx, &repeat) in repeats.iter().enumerate() { + if repeat > 1 { + inp = Tensor::cat(&vec![&inp; repeat], idx)? + } + } + Ok(inp) + } + + /// Creates grids of coordinates specified by the 1D inputs. + /// + /// # Arguments + /// + /// * `args` - A slice of 1D tensors. + /// * `xy_indexing` - Whether to use xy indexing or ij indexing. If xy is selected, the + /// first dimension corresponds to the cardinality of the second input and the second + /// dimension corresponds to the cardinality of the first input. If ij is selected, the + /// dimensions are in the same order as the cardinality of the inputs. + /// + /// # Examples + /// + /// ```rust + /// use candle_core::{Tensor, Device, Shape}; + /// let x = Tensor::new(&[1f32, 2., 3.], &Device::Cpu)?; + /// let y = Tensor::new(&[4f32, 5., 6.], &Device::Cpu)?; + /// + /// let grids_xy = Tensor::meshgrid(&[&x, &y], true)?; + /// + /// assert_eq!(grids_xy.len(), 2); + /// assert_eq!(grids_xy[0].dims(), &[3, 3]); + /// + /// assert_eq!(grids_xy[0].to_vec2::()?, &[[1., 2., 3.], [1., 2., 3.], [1., 2., 3.]]); + /// assert_eq!(grids_xy[1].to_vec2::()?, &[[4., 4., 4.], [5., 5., 5.], [6., 6., 6.]]); + /// + /// let grids_ij = Tensor::meshgrid(&[&x, &y], false)?; + /// + /// assert_eq!(grids_ij[0].to_vec2::()?, &[[1., 1., 1.], [2., 2., 2.], [3., 3., 3.]]); + /// assert_eq!(grids_ij[1].to_vec2::()?, &[[4., 5., 6.], [4., 5., 6.], [4., 5., 6.]]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + /// + /// # Errors + /// + /// * Will return `Err` if `args` contains less than 2 tensors. + /// + pub fn meshgrid>(args: &[A], xy_indexing: bool) -> Result> { + if args.len() <= 1 { + Err(Error::OpRequiresAtLeastTwoTensors { op: "meshgrid" }.bt())? + } + let args: Vec<_> = if xy_indexing { + args.iter().rev().collect() + } else { + args.iter().collect() + }; + + let mut shape = Vec::with_capacity(args.len()); + for arg in args.iter() { + shape.push(arg.as_ref().dims1()?) + } + + let mut grids = Vec::with_capacity(args.len()); + for idx in 0..args.len() { + let mut ones = vec![1usize; args.len()]; + ones[idx] = shape[idx]; + let arg = args[idx].as_ref().reshape(ones)?; + let mut repeats = shape.clone(); + repeats[idx] = 1; + let repeated_tensor = arg.repeat(repeats)?; + grids.push(repeated_tensor); + } + if xy_indexing { + grids.reverse(); + } + Ok(grids) + } + + /// This operation multiplies the input tensor by `mul` then adds `add` and return the result. + /// The input values `mul` and `add` are casted to the appropriate type so some rounding might + /// be performed. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let a = Tensor::new(&[[0f32, 1.], [2., 3.]], &Device::Cpu)?; + /// let a = a.affine(4., -2.)?; + /// assert_eq!(a.to_vec2::()?, &[[-2.0, 2.0], [6.0, 10.0]]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn affine(&self, mul: f64, add: f64) -> Result { + if self.elem_count() == 0 { + return Ok(self.clone()); + } + let storage = self.storage().affine(self.layout(), mul, add)?; + let op = BackpropOp::new1(self, |arg| Op::Affine { arg, mul, add }); + Ok(from_storage(storage, self.shape(), op, false)) + } + + /// Applies the Exponential Linear Unit (ELU) function on each element of the input tensor. + pub fn elu(&self, alpha: f64) -> Result { + if self.elem_count() == 0 { + return Ok(self.clone()); + } + let storage = self.storage().elu(self.layout(), alpha)?; + let op = BackpropOp::new1(self, |t| Op::Elu(t, alpha)); + Ok(from_storage(storage, self.shape(), op, false)) + } + + /// Raise the tensor to some float exponent `e`. + pub fn powf(&self, e: f64) -> Result { + if self.elem_count() == 0 { + return Ok(self.clone()); + } + let storage = self.storage().powf(self.layout(), e)?; + let op = BackpropOp::new1(self, |t| Op::Powf(t, e)); + Ok(from_storage(storage, self.shape(), op, false)) + } + + pub(crate) fn check_dim(&self, dim: usize, op: &'static str) -> Result<()> { + if dim >= self.dims().len() { + Err(Error::DimOutOfRange { + shape: self.shape().clone(), + dim: dim as i32, + op, + } + .bt())? + } else { + Ok(()) + } + } + + /// Split a tensor into the specified number of chunks, this may return less chunks than + /// specified. + pub fn chunk(&self, chunks: usize, dim: D) -> Result> { + let dim = dim.to_index(self.shape(), "chunk")?; + let size = self.dim(dim)?; + if size < chunks { + (0..size).map(|i| self.narrow(dim, i, 1)).collect() + } else { + let chunk_size = size / chunks; + let cnt_additional = size % chunks; + let mut tensors = vec![]; + let mut sum_chunk_size = 0; + for i in 0..chunks { + let chunk_size = if i < cnt_additional { + chunk_size + 1 + } else { + chunk_size + }; + let tensor = self.narrow(dim, sum_chunk_size, chunk_size)?; + tensors.push(tensor); + sum_chunk_size += chunk_size + } + Ok(tensors) + } + } + + /// Returns a new tensor that is a narrowed version of the input, the dimension `dim` + /// ranges from `start` to `start + len`. + /// ``` + /// use candle_core::{Tensor, Device}; + /// let a = Tensor::new(&[ + /// [0f32, 1., 2.], + /// [3. , 4., 5.], + /// [6. , 7., 8.] + /// ], &Device::Cpu)?; + /// + /// let b = a.narrow(0, 1, 2)?; + /// assert_eq!(b.shape().dims(), &[2, 3]); + /// assert_eq!(b.to_vec2::()?, &[ + /// [3., 4., 5.], + /// [6., 7., 8.] + /// ]); + /// + /// let c = a.narrow(1, 1, 1)?; + /// assert_eq!(c.shape().dims(), &[3, 1]); + /// assert_eq!(c.to_vec2::()?, &[ + /// [1.], + /// [4.], + /// [7.] + /// ]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn narrow(&self, dim: D, start: usize, len: usize) -> Result { + let dims = self.dims(); + let dim = dim.to_index(self.shape(), "narrow")?; + let err = |msg| { + Err::<(), _>( + Error::NarrowInvalidArgs { + shape: self.shape().clone(), + dim, + start, + len, + msg, + } + .bt(), + ) + }; + if start > dims[dim] { + err("start > dim_len")? + } + if start.saturating_add(len) > dims[dim] { + err("start + len > dim_len")? + } + if start == 0 && dims[dim] == len { + Ok(self.clone()) + } else { + let op = BackpropOp::new1(self, |t| Op::Narrow(t, dim, start, len)); + let layout = self.layout().narrow(dim, start, len)?; + let tensor_ = Tensor_ { + id: TensorId::new(), + storage: self.storage.clone(), + layout, + op, + is_variable: false, + dtype: self.dtype, + device: self.device.clone(), + }; + Ok(Tensor(Arc::new(tensor_))) + } + } + + fn squeeze_dims(self, dims: &[usize]) -> Result { + match dims { + [] => Ok(self), + [i] => self.squeeze(*i), + dims => { + let dims = self + .dims() + .iter() + .enumerate() + .filter_map(|(dim_idx, &v)| { + if dims.contains(&dim_idx) { + None + } else { + Some(v) + } + }) + .collect::>(); + self.reshape(dims) + } + } + } + + fn reduce_impl(&self, dim: D, keepdim: bool, op: ReduceOp) -> Result { + let dim = dim.to_index(self.shape(), op.name())?; + let storage = self.storage().reduce_op(op, self.layout(), &[dim])?; + let mut dims = self.dims().to_vec(); + dims[dim] = 1; + let op = match op { + ReduceOp::Sum | ReduceOp::Min | ReduceOp::Max => { + BackpropOp::new1(self, |arg| Op::Reduce(arg, op, dims.to_vec())) + } + ReduceOp::ArgMin | ReduceOp::ArgMax => BackpropOp::none(), + }; + let res = from_storage(storage, dims, op, false); + if keepdim { + Ok(res) + } else { + res.squeeze_dims(&[dim]) + } + } + + fn sum_impl(&self, sum_dims: D, keepdim: bool) -> Result { + let sum_dims = sum_dims.to_indexes(self.shape(), "sum")?; + let storage = self + .storage() + .reduce_op(ReduceOp::Sum, self.layout(), &sum_dims)?; + let mut dims = self.dims().to_vec(); + for &sum_dim in sum_dims.iter() { + dims[sum_dim] = 1 + } + let op = BackpropOp::new1(self, |a| Op::Reduce(a, ReduceOp::Sum, dims.to_vec())); + let sum = from_storage(storage, dims, op, false); + if keepdim { + Ok(sum) + } else { + sum.squeeze_dims(&sum_dims) + } + } + + /// Roll the tensor input along the given dimension. + /// Elements that are shifted beyond the last position are re-introduced at the first position. + /// + /// ```rust + /// # use candle_core::{Tensor, Device}; + /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; + /// let tensor = tensor.roll(1, 0)?; + /// assert_eq!(tensor.to_vec2::()?, &[[4., 5.], [0., 1.], [2., 3.]]); + /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; + /// let tensor = tensor.roll(-1, 0)?; + /// assert_eq!(tensor.to_vec2::()?, &[[2., 3.], [4., 5.], [0., 1.]]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn roll(&self, shift: i32, dim: D) -> Result + where + D: Dim + Clone, + { + let dim = dim.to_index(self.shape(), "roll")?; + let dim_size = self.dim(dim)?; + let shift = shift.rem_euclid(dim_size as i32) as usize; + if shift == 0 { + Ok(self.clone()) + } else { + let a = self.narrow(dim, 0, dim_size - shift)?; + let b = self.narrow(dim, dim_size - shift, shift)?; + Tensor::cat(&[&b, &a], dim) + } + } + + /// Returns the sum of all elements in the input tensor. The sum is performed over all the + /// input dimensions. + /// + /// The resulting tensor has a shape that is similar to the shape of the input tensor, except + /// that the number of elements for each dimension index in `sum_dims` is 1. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let a = Tensor::new(&[[0f32, 1.], [2., 3.]], &Device::Cpu)?; + /// let s = a.sum_keepdim(0)?; + /// assert_eq!(s.to_vec2::()?, &[[2., 4.]]); + /// let s = a.sum_keepdim(1)?; + /// assert_eq!(s.to_vec2::()?, &[[1.], [5.]]); + /// let s = a.sum_keepdim((0, 1))?; + /// assert_eq!(s.to_vec2::()?, &[[6.]]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn sum_keepdim(&self, sum_dims: D) -> Result { + self.sum_impl(sum_dims, true) + } + + /// Returns the sum of all elements in the input tensor. The sum is performed over all the + /// input dimensions and compared to `sum_keepdim` these dimensions are squeezed rather than + /// kept. + pub fn sum(&self, sum_dims: D) -> Result { + self.sum_impl(sum_dims, false) + } + + /// Returns the mean of all elements in the input tensor. The mean is performed over all the + /// input dimensions. + /// + /// The resulting tensor has a shape that is similar to the shape of the input tensor, except + /// that the number of elements for each dimension index in `mean_dims` is 1. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let a = Tensor::new(&[[0f32, 1.], [2., 3.]], &Device::Cpu)?; + /// let s = a.mean_keepdim(0)?; + /// assert_eq!(s.to_vec2::()?, &[[1., 2.]]); + /// let s = a.mean_keepdim(1)?; + /// assert_eq!(s.to_vec2::()?, &[[0.5], [2.5]]); + /// let s = a.mean_keepdim((0, 1))?; + /// assert_eq!(s.to_vec2::()?, &[[1.5]]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn mean_keepdim(&self, mean_dims: D) -> Result { + let mean_dims = mean_dims.to_indexes(self.shape(), "mean-keepdim")?; + let reduced_dim: usize = mean_dims.iter().map(|i| self.dims()[*i]).product(); + let scale = 1f64 / (reduced_dim as f64); + self.sum_impl(mean_dims, true)? * scale + } + + /// Returns the mean of all elements in the input tensor. The mean is performed over all the + /// input dimensions and compared to `mean_keepdim` these dimensions are squeezed rather than + /// kept. + pub fn mean(&self, mean_dims: D) -> Result { + let mean_dims = mean_dims.to_indexes(self.shape(), "mean")?; + let reduced_dim: usize = mean_dims.iter().map(|i| self.dims()[*i]).product(); + let scale = 1f64 / (reduced_dim as f64); + self.sum_impl(mean_dims, false)? * scale + } + + /// Returns the unbiased variance over the selected dimension. + pub fn var_keepdim(&self, dim: D) -> Result { + let dim = dim.to_index(self.shape(), "var")?; + let mean = self.mean_keepdim(dim)?; + let squares = self.broadcast_sub(&mean)?.sqr()?; + squares.sum_impl(dim, true)? / (self.dim(dim)? - 1) as f64 + } + + /// Returns the unbiased variance over the selected dimension. + pub fn var(&self, dim: D) -> Result { + let dim = dim.to_index(self.shape(), "var")?; + self.var_keepdim(dim)?.squeeze(dim) + } + + /// Gathers the maximum value across the selected dimension. The resulting shape has the same + /// number of dimensions as the original tensor and the select dimension has a single element. + pub fn max_keepdim(&self, dim: D) -> Result { + self.reduce_impl(dim, true, ReduceOp::Max) + } + + /// Similar to `max_keepdim` but the target dimension is squeezed. + pub fn max(&self, dim: D) -> Result { + self.reduce_impl(dim, false, ReduceOp::Max) + } + + /// Gathers the minimum value across the selected dimension. The resulting shape has the same + /// number of dimensions as the original tensor and the select dimension has a single element. + pub fn min_keepdim(&self, dim: D) -> Result { + self.reduce_impl(dim, true, ReduceOp::Min) + } + + /// Similar to `min_keepdim` but the target dimension is squeezed. + pub fn min(&self, dim: D) -> Result { + self.reduce_impl(dim, false, ReduceOp::Min) + } + + pub fn argmax_keepdim(&self, dim: D) -> Result { + self.reduce_impl(dim, true, ReduceOp::ArgMax) + } + + /// Similar to `argmax_keepdim` but the target dimension is squeezed. + pub fn argmax(&self, dim: D) -> Result { + self.reduce_impl(dim, false, ReduceOp::ArgMax) + } + + pub fn argmin_keepdim(&self, dim: D) -> Result { + self.reduce_impl(dim, true, ReduceOp::ArgMin) + } + + /// Similar to `argmin_keepdim` but the target dimension is squeezed. + pub fn argmin(&self, dim: D) -> Result { + self.reduce_impl(dim, false, ReduceOp::ArgMin) + } + + /// Element-wise comparison between two tensors, e.g. equality, greater than, ... The actual + /// comparison operation is specified by the `op` argument. + /// + /// The returned tensor has the same shape as the original tensors and uses `u8` elements. + pub fn cmp(&self, rhs: T, op: CmpOp) -> Result { + let rhs = match rhs.to_tensor_scalar()? { + crate::scalar::TensorScalar::Tensor(rhs) => rhs, + crate::scalar::TensorScalar::Scalar(rhs) => rhs + .to_dtype(self.dtype())? + .to_device(self.device())? + .broadcast_as(self.shape())?, + }; + let shape = self.same_shape_binary_op(&rhs, "cmp")?; + let storage = self + .storage() + .cmp(op, &rhs.storage(), self.layout(), rhs.layout())?; + let op = BackpropOp::new1(self, |a| Op::Cmp(a, op)); + Ok(from_storage(storage, shape.dims(), op, false)) + } + + /// Element-wise equality. + pub fn eq(&self, rhs: T) -> Result { + self.cmp(rhs, CmpOp::Eq) + } + + /// Element-wise non-equality. + pub fn ne(&self, rhs: T) -> Result { + self.cmp(rhs, CmpOp::Ne) + } + + /// Element-wise comparison with lower-than, the returned tensor uses value 1 where `self < + /// rhs` and 0 otherwise. + pub fn lt(&self, rhs: T) -> Result { + self.cmp(rhs, CmpOp::Lt) + } + + /// Element-wise comparison with greater-than, the returned tensor uses value 1 where `self > + /// rhs` and 0 otherwise. + pub fn gt(&self, rhs: T) -> Result { + self.cmp(rhs, CmpOp::Gt) + } + + /// Element-wise comparison with greater-equal, the returned tensor uses value 1 where `self >= + /// rhs` and 0 otherwise. + pub fn ge(&self, rhs: T) -> Result { + self.cmp(rhs, CmpOp::Ge) + } + + /// Element-wise comparison with lower-equal, the returned tensor uses value 1 where `self <= + /// rhs` and 0 otherwise. + pub fn le(&self, rhs: T) -> Result { + self.cmp(rhs, CmpOp::Le) + } + + /// Clamp the tensor values to be between `min` and `max`. + pub fn clamp(&self, min: T1, max: T2) -> Result { + self.maximum(min)?.minimum(max) + } + + /// Interpolate the input tensor to the `target_size` size, taking the value of the nearest element. + /// + /// The input tensor should have three dimensions, `(batch, channels, l)`, the returned + /// tensor also has three dimensions, `(batch, channels, target_size)`. + pub fn interpolate1d(&self, target_size: usize) -> Result { + let (n, c, _l) = self.dims3()?; + let op = BackpropOp::new1(self, |arg| Op::UpsampleNearest1D { arg, target_size }); + let storage = self + .storage() + .upsample_nearest1d(self.layout(), target_size)?; + Ok(from_storage(storage, (n, c, target_size), op, false)) + } + + /// Alias for `interpolate1d`. + pub fn upsample_nearest1d(&self, target_size: usize) -> Result { + self.interpolate1d(target_size) + } + + /// Interpolate the input tensor to the `(target_h, target_w)` size, taking the value of the + /// nearest element. + /// + /// The input tensor should have four dimensions, `(batch, channels, h, w)`, the returned + /// tensor also has four dimensions, `(batch, channels, target_h, target_w)`. + pub fn interpolate2d(&self, target_h: usize, target_w: usize) -> Result { + let (n, c, _h, _w) = self.dims4()?; + let op = BackpropOp::new1(self, |arg| Op::UpsampleNearest2D { + arg, + target_h, + target_w, + }); + let storage = self + .storage() + .upsample_nearest2d(self.layout(), target_h, target_w)?; + Ok(from_storage(storage, (n, c, target_h, target_w), op, false)) + } + + /// Alias for `interpolate2d`. + pub fn upsample_nearest2d(&self, target_h: usize, target_w: usize) -> Result { + self.interpolate2d(target_h, target_w) + } + + /// Bilinear interpolation to resize the input tensor to the specified size. + /// + /// The input tensor should have four dimensions: `(batch, channels, h, w)`. + /// The returned tensor also has four dimensions: `(batch, channels, target_h, target_w)`. + /// + /// # Arguments + /// + /// * `target_h` - Target height + /// * `target_w` - Target width + /// * `align_corners` - If true, corner pixels are aligned. If false (default), + /// pixels are treated as areas (matches PyTorch default behavior). + /// + /// # Example + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// # fn main() -> candle_core::Result<()> { + /// let t = Tensor::arange(0f32, 16f32, &Device::Cpu)?.reshape((1, 1, 4, 4))?; + /// let upsampled = t.upsample_bilinear2d(8, 8, false)?; + /// assert_eq!(upsampled.dims(), &[1, 1, 8, 8]); + /// # Ok(()) + /// # } + /// ``` + pub fn upsample_bilinear2d( + &self, + target_h: usize, + target_w: usize, + align_corners: bool, + ) -> Result { + let (n, c, _h, _w) = self.dims4()?; + let op = BackpropOp::new1(self, |arg| Op::UpsampleBilinear2D { + arg, + target_h, + target_w, + align_corners, + }); + // Pass None for scale factors (size mode) + let storage = self.storage().upsample_bilinear2d( + self.layout(), + target_h, + target_w, + align_corners, + None, + None, + )?; + Ok(from_storage(storage, (n, c, target_h, target_w), op, false)) + } + + /// Bilinear interpolation using scale factors. + /// + /// Similar to `upsample_bilinear2d` but uses scale factors instead of absolute sizes. + /// This matches PyTorch's `interpolate(scale_factor=...)` behavior. + /// + /// # Arguments + /// + /// * `scale_h` - Height scaling factor + /// * `scale_w` - Width scaling factor + /// * `align_corners` - If true, corner pixels are aligned + /// + /// # Example + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// # fn main() -> candle_core::Result<()> { + /// let t = Tensor::arange(0f32, 16f32, &Device::Cpu)?.reshape((1, 1, 4, 4))?; + /// // Scale by 2x in both dimensions + /// let upsampled = t.upsample_bilinear2d_with_scale(2.0, 2.0, false)?; + /// assert_eq!(upsampled.dims(), &[1, 1, 8, 8]); + /// # Ok(()) + /// # } + /// ``` + pub fn upsample_bilinear2d_with_scale( + &self, + scale_h: f64, + scale_w: f64, + align_corners: bool, + ) -> Result { + let (n, c, height_in, width_in) = self.dims4()?; + + // Calculate output size (floor, matching PyTorch) + let height_out = (height_in as f64 * scale_h).floor() as usize; + let width_out = (width_in as f64 * scale_w).floor() as usize; + + // Early return if size unchanged + if height_in == height_out && width_in == width_out { + return Ok(self.clone()); + } + + let op = BackpropOp::new1(self, |arg| Op::UpsampleBilinear2D { + arg, + target_h: height_out, + target_w: width_out, + align_corners, + }); + + // Pass original scale factors (scale_factor mode) + // This ensures PyTorch-compatible scale calculation + let storage = self.storage().upsample_bilinear2d( + self.layout(), + height_out, + width_out, + align_corners, + Some(scale_h), + Some(scale_w), + )?; + Ok(from_storage( + storage, + (n, c, height_out, width_out), + op, + false, + )) + } + + /// 2D average pooling over an input tensor with multiple channels. + /// + /// The input tensor should have four dimensions, `(batch, channels, h, w)`, the returned + /// tensor also has four dimensions, `(batch, channels, h', w')`. The pooling is performed on + /// the two last dimensions using a kernel of size `sz`. The returned element is the average + /// value over the kernel window. + pub fn avg_pool2d(&self, sz: T) -> Result { + let sz = sz.to_usize2(); + self.avg_pool2d_with_stride(sz, sz) + } + + /// Same as `avg_pool2d` but with a `stride` that can be set to a value different from the + /// kernel size. + pub fn avg_pool2d_with_stride( + &self, + kernel_size: T, + stride: T, + ) -> Result { + let kernel_size = kernel_size.to_usize2(); + let stride = stride.to_usize2(); + let (n, c, h, w) = self.dims4()?; + if h < kernel_size.0 || w < kernel_size.1 { + bail!("kernel-size {kernel_size:?} is larger than the input size {h},{w}") + } + // https://pytorch.org/docs/stable/generated/torch.nn.AvgPool2d.html#torch.nn.AvgPool2d + let h_out = (h - kernel_size.0) / stride.0 + 1; + let w_out = (w - kernel_size.1) / stride.1 + 1; + let op = BackpropOp::new1(self, |arg| Op::AvgPool2D { + arg, + kernel_size, + stride, + }); + let storage = self + .storage() + .avg_pool2d(self.layout(), kernel_size, stride)?; + Ok(from_storage(storage, (n, c, h_out, w_out), op, false)) + } + + /// 2D max pooling over an input tensor with multiple channels. + /// + /// The input tensor should have four dimensions, `(batch, channels, h, w)`, the returned + /// tensor also has four dimensions, `(batch, channels, h', w')`. The pooling is performed on + /// the two last dimensions using a kernel of size `sz`, the returned element is the maximum + /// value over the kernel window. + pub fn max_pool2d(&self, sz: T) -> Result { + let sz = sz.to_usize2(); + self.max_pool2d_with_stride(sz, sz) + } + + /// Same as `max_pool2d` but with a `stride` that can be set to a value different from the + /// kernel size. + pub fn max_pool2d_with_stride( + &self, + kernel_size: T, + stride: T, + ) -> Result { + let kernel_size = kernel_size.to_usize2(); + let stride = stride.to_usize2(); + let (n, c, h, w) = self.dims4()?; + if h < kernel_size.0 || w < kernel_size.1 { + bail!("kernel-size {kernel_size:?} is larger than the input size {h},{w}") + } + // https://pytorch.org/docs/stable/generated/torch.nn.MaxPool2d.html#torch.nn.MaxPool2d + let h_out = (h - kernel_size.0) / stride.0 + 1; + let w_out = (w - kernel_size.1) / stride.1 + 1; + let op = BackpropOp::new1(self, |arg| Op::MaxPool2D { + arg, + kernel_size, + stride, + }); + let storage = self + .storage() + .max_pool2d(self.layout(), kernel_size, stride)?; + Ok(from_storage(storage, (n, c, h_out, w_out), op, false)) + } + + /// Computes the dot product of two 1D tensors. + /// + /// - If inputs are 1D vectors (`[n]`), returns their scalar dot product. + /// - Panics if shapes are not compatible + /// - Not supported for integer dtypes + /// + /// # Example (vectors) + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let t1 = Tensor::new(&[1.0, 2.0, 3.0], &Device::Cpu)?; + /// let t2 = Tensor::new(&[4.0, 5.0, 6.0], &Device::Cpu)?; + /// let res = t1.dot(&t2)?; + /// assert_eq!(res.to_scalar::()?, 32.); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn dot(&self, rhs: &Self) -> Result { + if self.dims().len() != 1 || rhs.dims().len() != 1 { + return Err(Error::ShapeMismatchBinaryOp { + lhs: self.shape().clone(), + rhs: rhs.shape().clone(), + op: "dot", + }); + } + + (self * rhs).and_then(|ret| ret.sum_all()) + } + + /// Computes the **Frobenius norm** (L2 norm of all elements) of the tensor. + /// - Output is `sqrt(sum(x^2))`. + /// - Always returns a scalar (`[]` shape). + /// + /// # Example + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let t = Tensor::new(&[[3., 4.], [0., 0.]], &Device::Cpu)?; + /// let norm = t.norm()?; + /// assert_eq!(norm.to_scalar::()?, 5.); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn norm(&self) -> Result { + if self.dtype().is_int() { + bail!("norm not supported for integer dtypes"); + } + + self.sqr().and_then(|x| x.sum_all()).and_then(|x| x.sqrt()) + } + + /// Performs strict matrix-vector multiplication (`[m, n] * [n] = [m]`). + /// + /// - If `self` is a matrix (`[m, n]`) and `rhs` is a vector (`[n]`), returns a vector (`[m]`). + /// - **No broadcasting**: Panics if `self` is not 2D or if `rhs` is not 1D with matching size. + /// + /// # Example + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let mat = Tensor::new(&[[1., 2., 3.], [4., 5., 6.]], &Device::Cpu)?; + /// let vec = Tensor::new(&[1., 1., 1.], &Device::Cpu)?; + /// let res = mat.mv(&vec)?; + /// assert_eq!(res.to_vec1::()?, [6., 15.]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn mv(&self, rhs: &Self) -> Result { + // Strict shape checks + let lhs_dims = self.dims(); + let rhs_dims = rhs.dims(); + if lhs_dims.len() != 2 || rhs_dims.len() != 1 || lhs_dims[1] != rhs_dims[0] { + return Err(Error::ShapeMismatchBinaryOp { + lhs: self.shape().clone(), + rhs: rhs.shape().clone(), + op: "mv", + }); + } + + // Direct matmul after ensuring rhs is column vector + self.matmul(&rhs.unsqueeze(1)?)?.squeeze(1) + } + + /// Returns the matrix-multiplication of the input tensor with the other provided tensor. + /// + /// # Arguments + /// + /// * `self` - A tensor with dimensions `b1, b2, ..., bi, m, k`. + /// * `rhs` - A tensor with dimensions `b1, b2, ..., bi, k, n`. + /// + /// The resulting tensor has dimensions `b1, b2, ..., bi, m, n`. + pub fn matmul(&self, rhs: &Self) -> Result { + let a_dims = self.shape().dims(); + let b_dims = rhs.shape().dims(); + + let dim = a_dims.len(); + + if dim < 2 || b_dims.len() != dim { + Err(Error::ShapeMismatchBinaryOp { + lhs: self.shape().clone(), + rhs: rhs.shape().clone(), + op: "matmul", + } + .bt())? + } + + let m = a_dims[dim - 2]; + let k = a_dims[dim - 1]; + let k2 = b_dims[dim - 2]; + let n = b_dims[dim - 1]; + + let c_shape = Shape::from(&a_dims[..dim - 2]).extend(&[m, n]); + if c_shape.elem_count() == 0 || k == 0 { + return Tensor::zeros(c_shape, self.dtype(), self.device()); + } + let batching: usize = a_dims[..dim - 2].iter().product(); + let batching_b: usize = b_dims[..dim - 2].iter().product(); + if k != k2 || batching != batching_b { + Err(Error::ShapeMismatchBinaryOp { + lhs: self.shape().clone(), + rhs: rhs.shape().clone(), + op: "matmul", + } + .bt())? + } + + let storage = self.storage().matmul( + &rhs.storage(), + (batching, m, n, k), + self.layout(), + rhs.layout(), + )?; + let op = BackpropOp::new2(self, rhs, Op::Matmul); + Ok(from_storage(storage, c_shape, op, false)) + } + + /// Matrix-multiplication with broadcasting support. + /// + /// Compared to `matmul` the two matrixes are allowed to have different dimensions as long as + /// they are compatible for broadcast. E.g. if `self` has shape `(j, 1, n, k)` and `rhs` has + /// shape `(l, k, m)`, the output will have shape `(j, l, n, m)`. + pub fn broadcast_matmul(&self, rhs: &Self) -> Result { + let lhs = self; + let (l_shape, r_shape) = lhs.shape().broadcast_shape_matmul(rhs.shape())?; + let l_broadcast = l_shape != *lhs.shape(); + let r_broadcast = r_shape != *rhs.shape(); + // TODO: Avoid concretising the broadcasted matrixes via contiguous. + match (l_broadcast, r_broadcast) { + (true, true) => lhs + .broadcast_as(&l_shape)? + .contiguous()? + .matmul(&rhs.broadcast_as(&r_shape)?.contiguous()?), + (false, true) => lhs.matmul(&rhs.broadcast_as(&r_shape)?.contiguous()?), + (true, false) => lhs.broadcast_as(&l_shape)?.contiguous()?.matmul(rhs), + (false, false) => lhs.matmul(rhs), + } + } + + /// Returns a tensor with the same shape as the input tensor, the values are taken from + /// `on_true` if the input tensor value is not zero, and `on_false` at the positions where the + /// input tensor is equal to zero. + pub fn where_cond(&self, on_true: &Self, on_false: &Self) -> Result { + let _shap = self.same_shape_binary_op(on_true, "where_cond")?; + let shape = self.same_shape_binary_op(on_false, "where_cond")?; + let storage = self.storage().where_cond( + self.layout(), + &on_true.storage(), + on_true.layout(), + &on_false.storage(), + on_false.layout(), + )?; + let op = BackpropOp::new3(self, on_true, on_false, Op::WhereCond); + Ok(from_storage(storage, shape, op, false)) + } + + /// Returns a tensor with the values from the `self` tensor at the index corresponding to the + /// values hold in the `ids` tensor. + /// + /// # Arguments + /// + /// * `self` - A tensor with dimensions `v, h`. + /// * `ids` - A tensor with dimensions `s` and with integer values between 0 and v (exclusive). + /// + /// The resulting tensor has dimensions `s, h`. `s` is called the sequence length, `v` the + /// vocabulary size, and `h` the hidden size. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let values = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; + /// let ids = Tensor::new(&[2u32, 1u32, 2u32], &Device::Cpu)?; + /// let emb = values.embedding(&ids)?; + /// assert_eq!(emb.to_vec2::()?, &[[4., 5.], [2., 3.], [4., 5.]]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn embedding(&self, ids: &Self) -> Result { + if self.rank() != 2 || ids.rank() != 1 { + Err(Error::ShapeMismatchBinaryOp { + lhs: self.shape().clone(), + rhs: ids.shape().clone(), + op: "embedding", + } + .bt())? + } + self.index_select(ids, 0) + } + + fn scatter_checks(&self, indexes: &Self, source: &Self, dim: usize) -> Result<()> { + let source_dims = source.dims(); + let self_dims = self.dims(); + let mismatch = if source_dims.len() != self_dims.len() { + true + } else { + let mut mismatch = false; + for (i, (&d1, &d2)) in self_dims.iter().zip(source_dims.iter()).enumerate() { + if i != dim && d1 != d2 { + mismatch = true; + break; + } + } + mismatch + }; + if mismatch { + Err(Error::ShapeMismatchBinaryOp { + op: "scatter (self, src)", + lhs: self.shape().clone(), + rhs: source.shape().clone(), + } + .bt())? + } + if indexes.dims() != source.dims() { + Err(Error::ShapeMismatchBinaryOp { + op: "scatter (indexes, src)", + lhs: indexes.shape().clone(), + rhs: source.shape().clone(), + } + .bt())? + } + Ok(()) + } + + pub fn scatter(&self, indexes: &Self, source: &Self, dim: D) -> Result { + let dim = dim.to_index(self.shape(), "scatter")?; + self.scatter_checks(indexes, source, dim)?; + let shape = self.shape(); + let mut storage = unsafe { self.device().alloc_uninit(shape, self.dtype())? }; + self.storage() + .copy_strided_src(&mut storage, 0, self.layout())?; + let layout = Layout::contiguous(shape); + storage.scatter_set( + &layout, + &indexes.storage(), + indexes.layout(), + &source.storage(), + source.layout(), + dim, + )?; + let op = BackpropOp::new3(self, indexes, source, |t1, t2, t3| { + Op::Scatter(t1, t2, t3, dim) + }); + Ok(from_storage(storage, self.shape(), op, false)) + } + + pub fn scatter_set(&self, indexes: &Self, source: &Self, dim: D) -> Result<()> { + if self.same_storage(source) { + crate::bail!("cannot use slice_set when self and src share their storage") + } + let dim = dim.to_index(self.shape(), "scatter-set")?; + self.scatter_checks(indexes, source, dim)?; + self.storage_mut().scatter_set( + self.layout(), + &indexes.storage(), + indexes.layout(), + &source.storage(), + source.layout(), + dim, + )?; + Ok(()) + } + + pub fn scatter_add(&self, indexes: &Self, source: &Self, dim: D) -> Result { + let dim = dim.to_index(self.shape(), "scatter-add")?; + self.scatter_checks(indexes, source, dim)?; + let shape = self.shape(); + let mut storage = unsafe { self.device().alloc_uninit(shape, self.dtype())? }; + self.storage() + .copy_strided_src(&mut storage, 0, self.layout())?; + let layout = Layout::contiguous(shape); + storage.scatter_add( + &layout, + &indexes.storage(), + indexes.layout(), + &source.storage(), + source.layout(), + dim, + )?; + let op = BackpropOp::new3(self, indexes, source, |t1, t2, t3| { + Op::ScatterAdd(t1, t2, t3, dim) + }); + Ok(from_storage(storage, self.shape(), op, false)) + } + + pub fn scatter_add_set(&self, indexes: &Self, source: &Self, dim: D) -> Result<()> { + if self.same_storage(source) { + crate::bail!("cannot use slice_set when self and src share their storage") + } + let dim = dim.to_index(self.shape(), "scatter-add-set")?; + self.scatter_checks(indexes, source, dim)?; + self.storage_mut().scatter_add( + self.layout(), + &indexes.storage(), + indexes.layout(), + &source.storage(), + source.layout(), + dim, + )?; + Ok(()) + } + + /// Embeds the values of the `src` tensor into the `self` tensor on the specified dimension. + pub fn slice_scatter(&self, src: &Self, dim: D, start: usize) -> Result { + let dim = dim.to_index(self.shape(), "slice-scatter")?; + if dim == 0 { + self.slice_scatter0(src, start) + } else { + // TODO: Maybe we want to add a more efficient implementation at some point. + self.transpose(0, dim)? + .slice_scatter0(&src.transpose(0, dim)?, start)? + .transpose(0, dim) + } + } + + /// Embeds the values of the `src` tensor into the `self` tensor on the first dimension. + pub fn slice_scatter0(&self, src: &Self, start: usize) -> Result { + if self.dtype() != src.dtype() { + Err(Error::DTypeMismatchBinaryOp { + lhs: self.dtype(), + rhs: src.dtype(), + op: "slice-scatter", + } + .bt())? + } + if self.device().location() != src.device.location() { + Err(Error::DeviceMismatchBinaryOp { + lhs: self.device().location(), + rhs: src.device().location(), + op: "slice-scatter", + } + .bt())? + } + if self.rank() != src.rank() { + Err(Error::UnexpectedNumberOfDims { + expected: self.rank(), + got: src.rank(), + shape: src.shape().clone(), + } + .bt())? + } + let shape_ok = + self.dims() + .iter() + .zip(src.dims().iter()) + .enumerate() + .all(|(dim_idx, (&d1, &d2))| { + if 0 == dim_idx { + d2 + start <= d1 + } else { + d1 == d2 + } + }); + if !shape_ok { + Err(Error::ShapeMismatchBinaryOp { + op: "slice-scatter (self, src)", + lhs: self.shape().clone(), + rhs: src.shape().clone(), + } + .bt())? + } + let mut storage = unsafe { self.device().alloc_uninit(self.shape(), self.dtype())? }; + self.storage() + .copy_strided_src(&mut storage, 0, self.layout())?; + let offset = start * src.dims()[1..].iter().product::(); + src.storage() + .copy_strided_src(&mut storage, offset, src.layout())?; + let op = BackpropOp::new2(self, src, |t1, t2| Op::SliceScatter0(t1, t2, start)); + Ok(from_storage(storage, self.shape(), op, false)) + } + + /// Accumulate element from `source` at indexes `indexes` and add them to `self`. + pub fn index_add(&self, indexes: &Self, source: &Self, dim: D) -> Result { + let dim = dim.to_index(self.shape(), "index-add")?; + let source_dims = source.dims(); + let self_dims = self.dims(); + let mismatch = if source_dims.len() != self_dims.len() { + true + } else { + let mut mismatch = false; + for (i, (&d1, &d2)) in self_dims.iter().zip(source_dims.iter()).enumerate() { + if i != dim && d1 != d2 { + mismatch = true; + break; + } + } + mismatch + }; + if mismatch { + Err(Error::ShapeMismatchBinaryOp { + op: "index-add (self, source)", + lhs: self.shape().clone(), + rhs: source.shape().clone(), + } + .bt())? + } + // The number of element in indexes must match the dimension on which the add is + // performed on the source tensor (and the index values from `indexes` are taken from + // the target tensor self) + let indexes_len = indexes.dims1()?; + if source_dims[dim] != indexes_len { + Err(Error::ShapeMismatchBinaryOp { + op: "index-add (ids, source))", + lhs: indexes.shape().clone(), + rhs: source.shape().clone(), + } + .bt())? + } + let storage = self.storage().index_add( + self.layout(), + &indexes.storage(), + indexes.layout(), + &source.storage(), + source.layout(), + dim, + )?; + let op = BackpropOp::new3(self, indexes, source, |t1, t2, t3| { + Op::IndexAdd(t1, t2, t3, dim) + }); + Ok(from_storage(storage, self.shape(), op, false)) + } + + /// Gather values across the target dimension. + /// + /// # Arguments + /// + /// * `self` - The input tensor. + /// * `indexes` - The indices of elements to gather, this should have same number of dimensions as `self` + /// and indexes.dims()[d] <= self.dims()[d] for all dimensions d != dim + /// * `dim` - the target dimension. + /// + /// The resulting tensor has the same shape as `indexes` and use values from `self` indexed on + /// dimension `dim` by the values in `indexes`. + pub fn gather(&self, indexes: &Self, dim: D) -> Result { + let dim = dim.to_index(self.shape(), "gather")?; + + let self_dims = self.dims(); + let indexes_dims = indexes.dims(); + let mismatch = if indexes_dims.len() != self_dims.len() { + true + } else { + let mut mismatch = false; + for (i, (&d1, &d2)) in self_dims.iter().zip(indexes_dims.iter()).enumerate() { + if i != dim && d1 < d2 { + mismatch = true; + break; + } + } + mismatch + }; + if mismatch { + Err(Error::ShapeMismatchBinaryOp { + op: "gather", + lhs: self.shape().clone(), + rhs: indexes.shape().clone(), + } + .bt())? + } + let storage = + self.storage() + .gather(self.layout(), &indexes.storage(), indexes.layout(), dim)?; + let op = BackpropOp::new2(self, indexes, |t1, t2| Op::Gather(t1, t2, dim)); + Ok(from_storage(storage, indexes.shape(), op, false)) + } + + /// Select values for the input tensor at the target indexes across the specified dimension. + /// + /// The `indexes` is argument is an int tensor with a single dimension. + /// The output has the same number of dimension as the `self` input. The target dimension of + /// the output has length the length of `indexes` and the values are taken from `self` using + /// the index from `indexes`. Other dimensions have the same number of elements as the input + /// tensor. + pub fn index_select(&self, indexes: &Self, dim: D) -> Result { + let dim = dim.to_index(self.shape(), "index-select")?; + let indexes_len = match indexes.dims() { + [l] => *l, + _ => Err(Error::ShapeMismatchBinaryOp { + lhs: self.shape().clone(), + rhs: indexes.shape().clone(), + op: "index-select", + } + .bt())?, + }; + let storage = self.storage().index_select( + &indexes.storage(), + self.layout(), + indexes.layout(), + dim, + )?; + let mut dims = self.dims().to_vec(); + dims[dim] = indexes_len; + let op = BackpropOp::new2(self, indexes, |t1, t2| Op::IndexSelect(t1, t2, dim)); + Ok(from_storage(storage, dims, op, false)) + } + + /// Returns an iterator over position of the elements in the storage when ranging over the + /// index tuples in lexicographic order. + pub fn strided_index(&self) -> crate::StridedIndex<'_> { + self.layout.strided_index() + } + + /// Similar to `strided_index` but returns the position of the start of each contiguous block + /// as well as the length of the contiguous blocks. For a contiguous tensor, the index iterator + /// will only return the start offset and the size would be the number of elements in the + /// tensor. + pub fn strided_blocks(&self) -> crate::StridedBlocks<'_> { + self.layout.strided_blocks() + } + + /// Returns the data contained in a 1D tensor as a vector of scalar values. + pub fn to_vec1(&self) -> Result> { + if self.rank() != 1 { + Err(Error::UnexpectedNumberOfDims { + expected: 1, + got: self.rank(), + shape: self.shape().clone(), + } + .bt())? + } + let from_cpu_storage = |cpu_storage: &crate::CpuStorage| { + let data = S::cpu_storage_as_slice(cpu_storage)?; + let data = match self.layout.contiguous_offsets() { + Some((o1, o2)) => data[o1..o2].to_vec(), + None => self.strided_index().map(|i| data[i]).collect(), + }; + Ok::, Error>(data) + }; + match &*self.storage() { + Storage::Cpu(storage) => from_cpu_storage(storage), + Storage::Cuda(storage) => from_cpu_storage(&storage.to_cpu_storage()?), + Storage::Metal(storage) => from_cpu_storage(&storage.to_cpu_storage()?), + } + } + + /// Returns the data contained in a 2D tensor as a vector of vector of scalar values. + pub fn to_vec2(&self) -> Result>> { + let (dim1, dim2) = self.dims2()?; + let from_cpu_storage = |cpu_storage: &crate::CpuStorage| { + let data = S::cpu_storage_as_slice(cpu_storage)?; + let mut rows = vec![]; + match self.layout.contiguous_offsets() { + Some((o1, o2)) => { + let data = &data[o1..o2]; + for idx_row in 0..dim1 { + rows.push(data[idx_row * dim2..(idx_row + 1) * dim2].to_vec()) + } + } + None => { + let mut src_index = self.strided_index(); + for _idx_row in 0..dim1 { + let row = (0..dim2).map(|_| data[src_index.next().unwrap()]).collect(); + rows.push(row) + } + assert!(src_index.next().is_none()); + } + } + Ok(rows) + }; + match &*self.storage() { + Storage::Cpu(storage) => from_cpu_storage(storage), + Storage::Cuda(storage) => from_cpu_storage(&storage.to_cpu_storage()?), + Storage::Metal(storage) => from_cpu_storage(&storage.to_cpu_storage()?), + } + } + + /// Returns the data contained in a 3D tensor. + pub fn to_vec3(&self) -> Result>>> { + let (dim1, dim2, dim3) = self.dims3()?; + let from_cpu_storage = |cpu_storage: &crate::CpuStorage| { + let data = S::cpu_storage_as_slice(cpu_storage)?; + let mut top_rows = vec![]; + match self.layout.contiguous_offsets() { + Some((o1, o2)) => { + let data = &data[o1..o2]; + let dim23 = dim2 * dim3; + for idx1 in 0..dim1 { + let data = &data[idx1 * dim23..(idx1 + 1) * dim23]; + let mut rows = vec![]; + for idx2 in 0..dim2 { + rows.push(data[idx2 * dim3..(idx2 + 1) * dim3].to_vec()) + } + top_rows.push(rows); + } + } + None => { + let mut src_index = self.strided_index(); + for _idx in 0..dim1 { + let mut rows = vec![]; + for _jdx in 0..dim2 { + let row = (0..dim3).map(|_| data[src_index.next().unwrap()]).collect(); + rows.push(row) + } + top_rows.push(rows); + } + assert!(src_index.next().is_none()); + } + } + Ok(top_rows) + }; + match &*self.storage() { + Storage::Cpu(storage) => from_cpu_storage(storage), + Storage::Cuda(storage) => from_cpu_storage(&storage.to_cpu_storage()?), + Storage::Metal(storage) => from_cpu_storage(&storage.to_cpu_storage()?), + } + } + + /// The dtype for the elements stored in the input tensor. + pub fn dtype(&self) -> DType { + self.dtype + } + + /// The device on which the input tensor is located. + pub fn device(&self) -> &Device { + &self.device + } + + /// The tensor shape, i.e. dimension sizes on each axis. + pub fn shape(&self) -> &Shape { + self.layout().shape() + } + + /// The dimension size for this tensor on each axis. + pub fn dims(&self) -> &[usize] { + self.shape().dims() + } + + /// The dimension size for a specified dimension index. + pub fn dim(&self, dim: D) -> Result { + let dim = dim.to_index(self.shape(), "dim")?; + Ok(self.dims()[dim]) + } + + /// The layout of the input tensor, this stores both the shape of the tensor as well as the + /// strides and the start offset to apply to the underlying storage. + pub fn layout(&self) -> &Layout { + &self.layout + } + + pub fn stride(&self) -> &[usize] { + self.layout.stride() + } + + /// The number of dimensions for this tensor, 0 for a scalar tensor, 1 for a 1D tensor, etc. + pub fn rank(&self) -> usize { + self.shape().rank() + } + + /// The number of elements stored in this tensor. + pub fn elem_count(&self) -> usize { + self.shape().elem_count() + } + + /// The unique identifier for this tensor. + pub fn id(&self) -> TensorId { + self.id + } + + /// Whether this tensor is a variable or not. A variable is a tensor for which gradient is + /// tracked and on which backpropagation can be performed. + pub fn is_variable(&self) -> bool { + self.is_variable + } + + pub(crate) fn op(&self) -> &Option { + &self.op + } + + /// Computes the max of all the elements in this tensor and returns a tensor holding this + /// scalar with zero dimensions. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; + /// let tensor = tensor.max_all()?; + /// assert_eq!(tensor.to_scalar::()?, 5.); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn max_all(&self) -> Result { + if self.rank() == 0 { + Ok(self.clone()) + } else { + self.flatten_all()?.max(0) + } + } + + /// Computes the min of all the elements in this tensor and returns a tensor holding this + /// scalar with zero dimensions. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; + /// let tensor = tensor.min_all()?; + /// assert_eq!(tensor.to_scalar::()?, 0.); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn min_all(&self) -> Result { + if self.rank() == 0 { + Ok(self.clone()) + } else { + self.flatten_all()?.min(0) + } + } + + /// Computes the sum of all the elements in this tensor and returns a tensor holding this + /// scalar with zero dimensions. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; + /// let tensor = tensor.sum_all()?; + /// assert_eq!(tensor.to_scalar::()?, 15.); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn sum_all(&self) -> Result { + let dims: Vec<_> = (0..self.rank()).collect(); + self.sum(dims) + } + + pub fn mean_all(&self) -> Result { + self.sum_all()? / self.elem_count() as f64 + } + + fn flatten_( + &self, + start_dim: Option, + end_dim: Option, + ) -> Result { + if self.rank() == 0 { + self.reshape(1) + } else { + let start_dim = match start_dim { + None => 0, + Some(dim) => dim.to_index(self.shape(), "flatten")?, + }; + let end_dim = match end_dim { + None => self.rank() - 1, + Some(dim) => dim.to_index(self.shape(), "flatten")?, + }; + if start_dim < end_dim { + let dims = self.dims(); + let mut dst_dims = dims[..start_dim].to_vec(); + dst_dims.push(dims[start_dim..end_dim + 1].iter().product::()); + if end_dim + 1 < dims.len() { + dst_dims.extend(&dims[end_dim + 1..]); + } + self.reshape(dst_dims) + } else { + Ok(self.clone()) + } + } + } + + /// Flattens the input tensor on the dimension indexes from `start_dim` to `end_dim` (both + /// inclusive). + pub fn flatten(&self, start_dim: D1, end_dim: D2) -> Result { + self.flatten_(Some(start_dim), Some(end_dim)) + } + + /// Flattens the input tensor on the dimension indexes from `0` to `end_dim` (inclusive). + pub fn flatten_to(&self, end_dim: D) -> Result { + self.flatten_(None::, Some(end_dim)) + } + + /// Flattens the input tensor on the dimension indexes from `start_dim` (inclusive) to the last + /// dimension. + pub fn flatten_from(&self, start_dim: D) -> Result { + self.flatten_(Some(start_dim), None::) + } + + /// Flattens the input tensor by reshaping it into a one dimension tensor. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; + /// let tensor = tensor.flatten_all()?; + /// assert_eq!(tensor.to_vec1::()?, &[0., 1., 2., 3., 4., 5.]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn flatten_all(&self) -> Result { + self.flatten_(None::, None::) + } + + /// Returns the sub-tensor fixing the index at `i` on the first dimension. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; + /// let t = tensor.get(0)?; + /// assert_eq!(t.to_vec1::()?, &[0., 1.]); + /// let t = tensor.get(1)?; + /// assert_eq!(t.to_vec1::()?, &[2., 3.]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn get(&self, i: usize) -> Result { + let dims = self.dims(); + if dims.is_empty() { + Ok(self.clone()) + } else { + self.narrow(0, i, 1)?.reshape(&dims[1..]) + } + } + + /// Returns the sub-tensor fixing the index at `index` on the dimension `dim`. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; + /// let t = tensor.get_on_dim(1, 0)?; + /// assert_eq!(t.to_vec1::()?, &[0., 2., 4.]); + /// let t = tensor.get_on_dim(1, 1)?; + /// assert_eq!(t.to_vec1::()?, &[1., 3., 5.]); + /// let t = tensor.get_on_dim(0, 1)?; + /// assert_eq!(t.to_vec1::()?, &[2., 3.]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn get_on_dim(&self, dim: D, index: usize) -> Result { + let dim = dim.to_index(self.shape(), "get_on_dim")?; + self.narrow(dim, index, 1)?.squeeze(dim) + } + + /// Returns a tensor that is a transposed version of the input, the two last dimensions of the + /// input are swapped. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; + /// let tensor = tensor.t()?; + /// assert_eq!(tensor.to_vec2::()?, &[[0.0, 2.0, 4.0], [1.0, 3.0, 5.0]]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn t(&self) -> Result { + let rank = self.rank(); + if rank < 2 { + Err(Error::UnexpectedNumberOfDims { + expected: 2, + got: rank, + shape: self.shape().clone(), + } + .bt())? + } + self.transpose(rank - 2, rank - 1) + } + + /// Returns a tensor that is a transposed version of the input, the given dimensions are + /// swapped. + pub fn transpose(&self, dim1: D1, dim2: D2) -> Result { + let dim1 = dim1.to_index(self.shape(), "transpose")?; + let dim2 = dim2.to_index(self.shape(), "transpose")?; + if dim1 == dim2 { + return Ok(self.clone()); + } + let op = BackpropOp::new1(self, |t| Op::Transpose(t, dim1, dim2)); + let tensor_ = Tensor_ { + id: TensorId::new(), + storage: self.storage.clone(), + layout: self.layout.transpose(dim1, dim2)?, + op, + is_variable: false, + dtype: self.dtype, + device: self.device.clone(), + }; + Ok(Tensor(Arc::new(tensor_))) + } + + /// Returns a tensor with the same data as the input where the dimensions have been permuted. + /// dims must be a permutation, i.e. include each dimension index exactly once. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let tensor = Tensor::arange(0u32, 120u32, &Device::Cpu)?.reshape((2, 3, 4, 5))?; + /// assert_eq!(tensor.dims(), &[2, 3, 4, 5]); + /// let tensor = tensor.permute((2, 3, 1, 0))?; + /// assert_eq!(tensor.dims(), &[4, 5, 3, 2]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn permute(&self, dims: D) -> Result { + let dims = dims.to_indexes(self.shape(), "permute")?; + // O(n^2) permutation check but these arrays are small. + let is_permutation = + dims.len() == self.rank() && (0..dims.len()).all(|i| dims.contains(&i)); + if !is_permutation { + bail!( + "dimension mismatch in permute, tensor {:?}, dims: {:?}", + self.dims(), + dims + ) + } + let op = BackpropOp::new1(self, |t| Op::Permute(t, dims.clone())); + let tensor_ = Tensor_ { + id: TensorId::new(), + storage: self.storage.clone(), + layout: self.layout.permute(&dims)?, + op, + is_variable: false, + dtype: self.dtype, + device: self.device.clone(), + }; + Ok(Tensor(Arc::new(tensor_))) + } + + /// Returns true if the data is stored in a C contiguous (aka row major) way. + pub fn is_contiguous(&self) -> bool { + self.layout.is_contiguous() + } + + /// Returns true if the data is stored in a Fortran contiguous (aka column major) way. + pub fn is_fortran_contiguous(&self) -> bool { + self.layout.is_fortran_contiguous() + } + + /// Compared to clone, this copies the actual storage but may fail because of running out of + /// memory. + pub fn copy(&self) -> Result { + let op = BackpropOp::new1(self, Op::Copy); + let tensor_ = Tensor_ { + id: TensorId::new(), + storage: Arc::new(RwLock::new(self.storage().try_clone(self.layout())?)), + layout: self.layout.clone(), + op, + is_variable: false, + dtype: self.dtype, + device: self.device.clone(), + }; + Ok(Tensor(Arc::new(tensor_))) + } + + /// Returns a new tensor detached from the current graph, gradient are not propagated through + /// this new node. The storage of this tensor is shared with the initial tensor. + /// + /// If the tensor is already detached from the computation graph, the same tensor is returned. + pub fn detach(&self) -> Tensor { + if self.op.is_none() && !self.is_variable { + self.clone() + } else { + let tensor_ = Tensor_ { + id: TensorId::new(), + storage: self.storage.clone(), + layout: self.layout.clone(), + op: BackpropOp::none(), + is_variable: false, + dtype: self.dtype, + device: self.device.clone(), + }; + Tensor(Arc::new(tensor_)) + } + } + + /// If the target device is the same as the tensor device, only a shallow copy is performed. + pub fn to_device(&self, device: &Device) -> Result { + if self.device().same_device(device) { + Ok(self.clone()) + } else { + let storage = match (&*self.storage(), device) { + (Storage::Cpu(storage), Device::Cuda(cuda)) => { + Storage::Cuda(cuda.storage_from_cpu_storage(storage)?) + } + (Storage::Cpu(storage), Device::Metal(metal)) => { + Storage::Metal(metal.storage_from_cpu_storage(storage)?) + } + (Storage::Cuda(storage), Device::Cpu) => Storage::Cpu(storage.to_cpu_storage()?), + (Storage::Metal(storage), Device::Cpu) => Storage::Cpu(storage.to_cpu_storage()?), + (Storage::Cuda(storage), Device::Cuda(cuda)) => { + // can't clone storage if it's the same device because of the underlying device ptr + let dst_storage = storage.transfer_to_device(cuda)?; + Storage::Cuda(dst_storage) + } + (Storage::Cpu(storage), Device::Cpu) => Storage::Cpu(storage.clone()), + _ => { + bail!( + "not implemented yet, self.device: {:?}, device: {:?}", + self.device(), + device + ) + } + }; + let op = BackpropOp::new1(self, Op::ToDevice); + let tensor_ = Tensor_ { + id: TensorId::new(), + storage: Arc::new(RwLock::new(storage)), + layout: self.layout.clone(), + op, + is_variable: false, + dtype: self.dtype, + device: device.clone(), + }; + Ok(Tensor(Arc::new(tensor_))) + } + } + + /// Returns a new tensor duplicating data from the original tensor. New dimensions are inserted + /// on the left. + pub fn broadcast_left>(&self, left_shape: S) -> Result { + let left_shape = left_shape.into(); + let mut dims = left_shape.into_dims(); + dims.extend(self.dims()); + self.broadcast_as(dims) + } + + /// Broadcast the input tensor to the target shape. This returns an error if the input shape is + /// not compatible with the target shape. + /// + /// If the input shape is `i_1, i_2, ... i_k`, the target shape has to have `k` dimensions or + /// more and shape `j_1, ..., j_l, t_1, t_2, ..., t_k`. The dimensions `j_1` to `j_l` can have + /// any value, the dimension `t_a` must be equal to `i_a` if `i_a` is different from 1. If + /// `i_a` is equal to 1, any value can be used. + pub fn broadcast_as>(&self, shape: S) -> Result { + let tensor_ = Tensor_ { + id: TensorId::new(), + storage: self.storage.clone(), + layout: self.layout.broadcast_as(shape)?, + op: BackpropOp::new1(self, Op::Broadcast), + is_variable: false, + dtype: self.dtype, + device: self.device.clone(), + }; + Ok(Tensor(Arc::new(tensor_))) + } + + /// An alias for broadcast_as. + pub fn expand>(&self, shape: S) -> Result { + self.broadcast_as(shape) + } + + /// Casts the input tensor to the target `dtype`. + /// + /// ```rust + /// use candle_core::{Tensor, Device}; + /// let tensor = Tensor::new(3.14159265358979f64, &Device::Cpu)?; + /// assert_eq!(tensor.to_scalar::()?, 3.14159265358979); + /// let tensor = tensor.to_dtype(candle_core::DType::F32)?; + /// assert_eq!(tensor.to_scalar::()?, 3.1415927); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn to_dtype(&self, dtype: DType) -> Result { + if self.dtype() == dtype { + Ok(self.clone()) + } else { + let shape = self.shape(); + let storage = self.storage().to_dtype(self.layout(), dtype)?; + let op = BackpropOp::new1(self, Op::ToDType); + Ok(from_storage(storage, shape.clone(), op, false)) + } + } + + /// Returns a tensor that is in row major order. This is the same as the original tensor if it + /// was already contiguous, otherwise a copy is triggered. + pub fn contiguous(&self) -> Result { + if self.is_contiguous() { + Ok(self.clone()) + } else { + let shape = self.shape(); + let mut storage = unsafe { self.device().alloc_uninit(shape, self.dtype())? }; + self.storage() + .copy_strided_src(&mut storage, 0, self.layout())?; + let op = BackpropOp::new1(self, Op::Copy); + Ok(from_storage(storage, shape.clone(), op, false)) + } + } + + /// Returns a tensor that is in row major order. This always makes a copy. + pub fn force_contiguous(&self) -> Result { + let shape = self.shape(); + let mut storage = unsafe { self.device().alloc_uninit(shape, self.dtype())? }; + self.storage() + .copy_strided_src(&mut storage, 0, self.layout())?; + let op = BackpropOp::new1(self, Op::Copy); + Ok(from_storage(storage, shape.clone(), op, false)) + } + + /// Create a variable based on the values currently stored in a tensor. The storage is always + /// copied. + pub(crate) fn make_var(&self) -> Result { + let shape = self.shape().clone(); + let mut storage = unsafe { self.device().alloc_uninit(&shape, self.dtype())? }; + self.storage() + .copy_strided_src(&mut storage, 0, self.layout())?; + Ok(from_storage(storage, shape, BackpropOp::none(), true)) + } + + /// Reshape returns a tensor with the target shape provided that the number of elements of the + /// original tensor is the same. + /// If the input tensor is contiguous, this is a view on the original data. Otherwise this uses + /// a new storage and copies the data over, the returned tensor is always contiguous. + /// + /// The shape can be specified using a tuple of `usize` and at most one `()` in which case + /// the behavior is the same as when using `-1` in PyTorch: this dimension size is adjusted so + /// as to match the number of elements in the tensor. + /// + /// ```rust + /// # use candle_core::{Tensor, DType, Device, D}; + /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; + /// + /// let c = a.reshape((1, 6))?; + /// assert_eq!(c.shape().dims(), &[1, 6]); + /// + /// let c = a.reshape((3, 2))?; + /// assert_eq!(c.shape().dims(), &[3, 2]); + /// + /// let c = a.reshape((2, (), 1))?; + /// assert_eq!(c.shape().dims(), &[2, 3, 1]); + /// + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn reshape(&self, s: S) -> Result { + let shape = s.into_shape(self.elem_count())?; + if shape.elem_count() != self.elem_count() { + return Err(Error::ShapeMismatchBinaryOp { + lhs: self.shape().clone(), + rhs: shape, + op: "reshape", + } + .bt()); + } + let op = BackpropOp::new1(self, Op::Reshape); + if self.is_contiguous() { + let tensor_ = Tensor_ { + id: TensorId::new(), + storage: self.storage.clone(), + layout: Layout::contiguous_with_offset(shape, self.layout.start_offset()), + op, + is_variable: false, + dtype: self.dtype, + device: self.device.clone(), + }; + Ok(Tensor(Arc::new(tensor_))) + } else { + let mut storage = unsafe { self.device().alloc_uninit(&shape, self.dtype())? }; + self.storage() + .copy_strided_src(&mut storage, 0, self.layout())?; + Ok(from_storage(storage, shape, op, false)) + } + } + + /// Creates a new tensor with the specified dimension removed if its size was one. + /// + /// ```rust + /// # use candle_core::{Tensor, DType, Device, D}; + /// let a = Tensor::zeros((2, 3, 1), DType::F32, &Device::Cpu)?; + /// + /// let c = a.squeeze(2)?; + /// assert_eq!(c.shape().dims(), &[2, 3]); + /// + /// let c = a.squeeze(D::Minus1)?; + /// assert_eq!(c.shape().dims(), &[2, 3]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn squeeze(&self, dim: D) -> Result { + // The PyTorch semantics are to return the same tensor if the target dimension + // does not have a size of 1. + let dims = self.dims(); + let dim = dim.to_index(self.shape(), "squeeze")?; + if dims[dim] == 1 { + let mut dims = dims.to_vec(); + let mut strides = self.stride().to_vec(); + dims.remove(dim); + strides.remove(dim); + let tensor_ = Tensor_ { + id: TensorId::new(), + storage: self.storage.clone(), + layout: Layout::new(dims.into(), strides, self.layout.start_offset()), + op: BackpropOp::new1(self, Op::Reshape), + is_variable: false, + dtype: self.dtype, + device: self.device.clone(), + }; + Ok(Tensor(Arc::new(tensor_))) + } else { + Ok(self.clone()) + } + } + + /// Creates a new tensor with a dimension of size one inserted at the specified position. + /// + /// ```rust + /// # use candle_core::{Tensor, DType, Device, D}; + /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; + /// + /// let c = a.unsqueeze(0)?; + /// assert_eq!(c.shape().dims(), &[1, 2, 3]); + /// + /// let c = a.unsqueeze(D::Minus1)?; + /// assert_eq!(c.shape().dims(), &[2, 3, 1]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn unsqueeze(&self, dim: D) -> Result { + let mut dims = self.dims().to_vec(); + let mut strides = self.stride().to_vec(); + let dim = dim.to_index_plus_one(self.shape(), "unsqueeze")?; + // Cannot panic because to_index_plus_one already checks dimensions + dims.insert(dim, 1); + // Any stride would work here, but we pick one so as to maximize the probability to remain + // C contiguous. + let stride = if dim < strides.len() { strides[dim] } else { 1 }; + strides.insert(dim, stride); + let tensor_ = Tensor_ { + id: TensorId::new(), + storage: self.storage.clone(), + layout: Layout::new(dims.into(), strides, self.layout.start_offset()), + op: BackpropOp::new1(self, Op::Reshape), + is_variable: false, + dtype: self.dtype, + device: self.device.clone(), + }; + Ok(Tensor(Arc::new(tensor_))) + } + + /// Stacks two or more tensors along a particular dimension. + /// + /// All tensors must have the same rank, and the output has one additional rank + /// + /// ```rust + /// # use candle_core::{Tensor, DType, Device}; + /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; + /// let b = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; + /// + /// let c = Tensor::stack(&[&a, &b], 0)?; + /// assert_eq!(c.shape().dims(), &[2, 2, 3]); + /// + /// let c = Tensor::stack(&[&a, &b], 2)?; + /// assert_eq!(c.shape().dims(), &[2, 3, 2]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn stack, D: Dim>(args: &[A], dim: D) -> Result { + if args.is_empty() { + Err(Error::OpRequiresAtLeastOneTensor { op: "stack" }.bt())? + } + let dim = dim.to_index_plus_one(args[0].as_ref().shape(), "stack")?; + let args = args + .iter() + .map(|t| t.as_ref().unsqueeze(dim)) + .collect::>>()?; + Self::cat(&args, dim) + } + + /// Pad the input tensor using 0s along dimension `dim`. This adds `left` elements before the + /// input tensor values and `right` elements after. + pub fn pad_with_zeros(&self, dim: D, left: usize, right: usize) -> Result { + if left == 0 && right == 0 { + Ok(self.clone()) + } else if left == 0 { + let dim = dim.to_index(self.shape(), "pad_with_zeros")?; + let mut dims = self.dims().to_vec(); + dims[dim] = right; + let right = Tensor::zeros(dims.as_slice(), self.dtype, self.device())?; + Tensor::cat(&[self, &right], dim) + } else if right == 0 { + let dim = dim.to_index(self.shape(), "pad_with_zeros")?; + let mut dims = self.dims().to_vec(); + dims[dim] = left; + let left = Tensor::zeros(dims.as_slice(), self.dtype, self.device())?; + Tensor::cat(&[&left, self], dim) + } else { + let dim = dim.to_index(self.shape(), "pad_with_zeros")?; + let mut dims = self.dims().to_vec(); + dims[dim] = left; + let left = Tensor::zeros(dims.as_slice(), self.dtype, self.device())?; + dims[dim] = right; + let right = Tensor::zeros(dims.as_slice(), self.dtype, self.device())?; + Tensor::cat(&[&left, self, &right], dim) + } + } + + /// Pad the input tensor using same values along dimension `dim`. This adds `left` elements before the + /// input tensor values and `right` elements after. + pub fn pad_with_same(&self, dim: D, left: usize, right: usize) -> Result { + if left == 0 && right == 0 { + Ok(self.clone()) + } else if self.elem_count() == 0 { + bail!("cannot use pad_with_same on an empty tensor") + } else if left == 0 { + let dim = dim.to_index(self.shape(), "pad_with_same")?; + let r = self.narrow(dim, self.dim(dim)? - 1, 1)?; + let mut v = vec![self]; + for _ in 0..right { + v.push(&r) + } + Tensor::cat(&v, dim) + } else if right == 0 { + let dim = dim.to_index(self.shape(), "pad_with_same")?; + let l = self.narrow(dim, 0, 1)?; + let mut v = vec![]; + for _ in 0..left { + v.push(&l) + } + v.push(self); + Tensor::cat(&v, dim) + } else { + let dim = dim.to_index(self.shape(), "pad_with_same")?; + let l = self.narrow(dim, 0, 1)?; + let r = self.narrow(dim, self.dim(dim)? - 1, 1)?; + let mut v = vec![]; + for _ in 0..left { + v.push(&l) + } + v.push(self); + for _ in 0..right { + v.push(&r) + } + Tensor::cat(&v, dim) + } + } + + /// Run the `forward` method of `m` on `self`. + pub fn apply(&self, m: &M) -> Result { + m.forward(self) + } + + /// Run the `forward` method of `m` on `self`. + pub fn apply_t(&self, m: &M, train: bool) -> Result { + m.forward_t(self, train) + } + + pub(crate) fn storage(&self) -> std::sync::RwLockReadGuard<'_, Storage> { + self.storage.read().unwrap() + } + + pub(crate) fn storage_mut(&self) -> std::sync::RwLockWriteGuard<'_, Storage> { + self.storage.write().unwrap() + } + + // If we extend the visibility of this function to be usable outside of this crate, we should + // make it unsafe. + pub(crate) fn storage_mut_and_layout( + &self, + ) -> (std::sync::RwLockWriteGuard<'_, Storage>, &Layout) { + let storage = self.storage.write().unwrap(); + (storage, &self.layout) + } + + /// The storage used by this tensor, together with the layout to use to access it safely. + pub fn storage_and_layout(&self) -> (std::sync::RwLockReadGuard<'_, Storage>, &Layout) { + let storage = self.storage.read().unwrap(); + (storage, &self.layout) + } + + pub(crate) fn same_storage(&self, rhs: &Self) -> bool { + let lhs: &RwLock = self.storage.as_ref(); + let rhs: &RwLock = rhs.storage.as_ref(); + std::ptr::eq(lhs, rhs) + } + + /// Normalize a 'relative' axis value: positive values are kept, negative + /// values means counting the dimensions from the back. + pub fn normalize_axis(&self, axis: i64) -> Result { + let rank = self.rank() as i64; + if rank <= axis { + bail!("axis {axis} is too large, tensor rank {rank}") + } else if 0 <= axis { + Ok(axis as usize) + } else { + let naxis = rank + axis; + if naxis < 0 { + bail!("axis {axis} is too small, tensor rank {rank}") + } + Ok(naxis as usize) + } + } + + /// Returns a lower triangular matrix of ones of size n by n. + pub fn tril2(n: usize, dtype: DType, device: &Device) -> Result { + let t = Tensor::arange(0u32, n as u32, device)?; + let t1 = t.reshape((1, n))?.broadcast_as((n, n))?; + let t2 = t.reshape((n, 1))?.broadcast_as((n, n))?; + t1.le(&t2)?.to_dtype(dtype) + } + + /// Returns an upper triangular matrix of ones of size n by n. + pub fn triu2(n: usize, dtype: DType, device: &Device) -> Result { + let t = Tensor::arange(0u32, n as u32, device)?; + let t1 = t.reshape((1, n))?.broadcast_as((n, n))?; + let t2 = t.reshape((n, 1))?.broadcast_as((n, n))?; + t1.ge(&t2)?.to_dtype(dtype) + } + + /// Returns a matrix with a diagonal of ones of size n by n. + pub fn eye(n: usize, dtype: DType, device: &Device) -> Result { + let t = Tensor::arange(0u32, n as u32, device)?; + let t1 = t.reshape((1, n))?.broadcast_as((n, n))?; + let t2 = t.reshape((n, 1))?.broadcast_as((n, n))?; + t1.eq(&t2)?.to_dtype(dtype) + } + + /// Returns the cumulative sum of elements of the input tensor summed over the specified + /// dimension. + /// + /// This operation is most efficient when dim is the last dimension of the tensor. + pub fn cumsum(&self, dim: D) -> Result { + let dim = dim.to_index(self.shape(), "cumsum")?; + let rank = self.rank(); + if rank == 0 { + return Ok(self.clone()); + } + let n_axis = self.dim(dim)?; + let triu = Tensor::triu2(n_axis, self.dtype(), self.device())?; + if rank == 1 { + self.unsqueeze(0)?.matmul(&triu)?.squeeze(0) + } else { + let last = rank - 1; + let t = self.transpose(dim, last)?; + let t = t.broadcast_matmul(&triu)?; + t.transpose(dim, last) + } + } + + /// Returns a copy of `self` where the values within `ranges` have been replaced with the + /// content of `src`. + pub fn slice_assign>( + &self, + ranges: &[D], + src: &Tensor, + ) -> Result { + let src_dims = src.dims(); + let self_dims = self.dims(); + if self_dims.len() != src_dims.len() { + bail!( + "slice-assign requires input with the same rank {} <> {}", + self_dims.len(), + src_dims.len() + ) + } + if self_dims.len() != ranges.len() { + bail!( + "slice-assign requires input with the same rank as there are ranges {} <> {}", + self_dims.len(), + ranges.len() + ) + } + let mut src = src.clone(); + let mut mask = Self::ones(src.shape(), DType::U8, src.device())?; + for (i, range) in ranges.iter().enumerate() { + let start_included = match range.start_bound() { + std::ops::Bound::Unbounded => 0, + std::ops::Bound::Included(v) => *v, + std::ops::Bound::Excluded(v) => *v + 1, + }; + let end_excluded = match range.end_bound() { + std::ops::Bound::Unbounded => self_dims[i], + std::ops::Bound::Included(v) => *v + 1, + std::ops::Bound::Excluded(v) => *v, + }; + if end_excluded <= start_included { + bail!("slice-assign: empty range for dim {i}, {start_included} {end_excluded}") + } + if self_dims[i] < end_excluded { + bail!( + "slice-assign: upper bound is out of range for dim {i}, {end_excluded} {}", + self_dims[i] + ) + } + if end_excluded - start_included != src_dims[i] { + bail!( + "slice-assign: the range for dim {i} ({start_included}..{end_excluded}) does not match the size of src {}", src_dims[i] + ) + } + src = src.pad_with_zeros(i, start_included, self_dims[i] - end_excluded)?; + mask = mask.pad_with_zeros(i, start_included, self_dims[i] - end_excluded)? + } + mask.where_cond(/* on_true= */ &src, /* on_false= */ self) + } + + /// Returns log(sum(exp(tensor), dim)). + pub fn log_sum_exp(&self, sum_dims: D) -> Result { + let sum_dims = sum_dims.to_indexes(self.shape(), "log-sum-exp")?; + if sum_dims.is_empty() { + return Ok(self.clone()); + } + let max = sum_dims[1..] + .iter() + .try_fold(self.max_keepdim(sum_dims[0])?, |max, &dim| { + max.max_keepdim(dim) + })?; + let exp = self.broadcast_sub(&max)?.exp()?; + let sum = exp.sum(sum_dims.clone())?; + + sum.log()? + max.squeeze_dims(&sum_dims) + } + + /// Pointwise pow operation. + pub fn pow(&self, rhs: &Tensor) -> Result { + rhs.mul(&self.log()?)?.exp() + } + + /// Broadcasting version of `pow`. + pub fn broadcast_pow(&self, rhs: &Tensor) -> Result { + rhs.broadcast_mul(&self.log()?)?.exp() + } + + /// Returns a new tensor with the order of elements reversed along the specified dimensions. + /// This function makes a copy of the tensor’s data. + /// + /// ```rust + /// # use candle_core::{Tensor, Device}; + /// let t = Tensor::arange(0., 6., &Device::Cpu)?.reshape((2, 3))?; + /// assert_eq!(t.to_vec2::()?, &[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); + /// let t_flipped = t.flip(&[0])?; + /// assert_eq!(t_flipped.to_vec2::()?, &[[3.0, 4.0, 5.0], [0.0, 1.0, 2.0]]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn flip(&self, dims: &[usize]) -> Result { + let mut result = self.clone(); + for &dim in dims.iter() { + let size = result.dim(dim)?; + let indices: Vec = (0..size).rev().map(|x| x as i64).collect(); + let indices_tensor = Tensor::from_vec(indices, (size,), result.device())?; + result = result.index_select(&indices_tensor, dim)?; + } + Ok(result) + } + + /// Returns a view of which contains all slices of size `size` from self tensor in the dimension + /// `dim` and stepped by `step`. + pub fn unfold(&self, dim: D, size: usize, step: usize) -> Result { + // https://github.com/pytorch/pytorch/blob/75b0720a97ac5d82e8a7a1a6ae7c5f7a87d7183d/aten/src/ATen/native/TensorShape.cpp#L3785-L3804 + let mut sizes = self.dims().to_vec(); + let mut strides = self.stride().to_vec(); + + let dim = dim.to_index(self.shape(), "unfold")?; + + let max_len = if self.dims().is_empty() { + 1 + } else { + sizes[dim] + }; + if size > max_len { + bail!( + "unsqueeze: maximum size for tensor at dimension {dim} is {max_len} but size is {size}" + ) + } + sizes.push(size); + strides.push(if self.dims().is_empty() { + 1 + } else { + strides[dim] + }); + + if !self.dims().is_empty() { + sizes[dim] = ((sizes[dim] as f32 - size as f32) / step as f32 + 1.) as usize; + strides[dim] *= step; + } + + let tensor_ = Tensor_ { + id: TensorId::new(), + storage: self.storage.clone(), + layout: Layout::new(sizes.into(), strides, self.layout.start_offset()), + op: BackpropOp::new1(self, Op::Reshape), + is_variable: false, + dtype: self.dtype, + device: self.device.clone(), + }; + Ok(Tensor(Arc::new(tensor_))) + } +} + +macro_rules! bin_trait { + ($trait:ident, $fn1:ident, $mul:expr, $add:expr) => { + impl> std::ops::$trait for Tensor { + type Output = Result; + + fn $fn1(self, rhs: B) -> Self::Output { + Tensor::$fn1(&self, rhs.borrow()) + } + } + + impl> std::ops::$trait for &Tensor { + type Output = Result; + + fn $fn1(self, rhs: B) -> Self::Output { + Tensor::$fn1(&self, rhs.borrow()) + } + } + + impl> std::ops::$trait for Result { + type Output = Result; + + fn $fn1(self, rhs: Tensor) -> Self::Output { + Tensor::$fn1(self?.borrow(), &rhs) + } + } + + impl> std::ops::$trait<&Tensor> for Result { + type Output = Result; + + fn $fn1(self, rhs: &Tensor) -> Self::Output { + Tensor::$fn1(self?.borrow(), rhs) + } + } + + impl> std::ops::$trait> for Tensor { + type Output = Result; + + fn $fn1(self, rhs: Result) -> Self::Output { + Tensor::$fn1(&self, rhs?.borrow()) + } + } + + impl> std::ops::$trait> for &Tensor { + type Output = Result; + + fn $fn1(self, rhs: Result) -> Self::Output { + Tensor::$fn1(&self, rhs?.borrow()) + } + } + + impl std::ops::$trait for Tensor { + type Output = Result; + + fn $fn1(self, rhs: f64) -> Self::Output { + self.affine($mul(rhs), $add(rhs)) + } + } + + impl std::ops::$trait for &Tensor { + type Output = Result; + + fn $fn1(self, rhs: f64) -> Self::Output { + self.affine($mul(rhs), $add(rhs)) + } + } + }; +} + +bin_trait!(Add, add, |_| 1., |v| v); +bin_trait!(Sub, sub, |_| 1., |v: f64| -v); +bin_trait!(Mul, mul, |v| v, |_| 0.); +bin_trait!(Div, div, |v| 1. / v, |_| 0.); + +impl std::ops::Add for f64 { + type Output = Result; + + fn add(self, rhs: Tensor) -> Self::Output { + rhs + self + } +} + +impl std::ops::Add<&Tensor> for f64 { + type Output = Result; + + fn add(self, rhs: &Tensor) -> Self::Output { + rhs + self + } +} + +impl std::ops::Mul for f64 { + type Output = Result; + + fn mul(self, rhs: Tensor) -> Self::Output { + rhs * self + } +} + +impl std::ops::Mul<&Tensor> for f64 { + type Output = Result; + + fn mul(self, rhs: &Tensor) -> Self::Output { + rhs * self + } +} + +impl std::ops::Sub for f64 { + type Output = Result; + + fn sub(self, rhs: Tensor) -> Self::Output { + rhs.affine(-1., self) + } +} + +impl std::ops::Sub<&Tensor> for f64 { + type Output = Result; + + fn sub(self, rhs: &Tensor) -> Self::Output { + rhs.affine(-1., self) + } +} + +impl std::ops::Div for f64 { + type Output = Result; + + #[allow(clippy::suspicious_arithmetic_impl)] + fn div(self, rhs: Tensor) -> Self::Output { + rhs.recip()? * self + } +} + +impl std::ops::Div<&Tensor> for f64 { + type Output = Result; + + #[allow(clippy::suspicious_arithmetic_impl)] + fn div(self, rhs: &Tensor) -> Self::Output { + rhs.recip()? * self + } +} + +impl> From<(Storage, S)> for Tensor { + fn from((storage, shape): (Storage, S)) -> Self { + from_storage(storage, shape, BackpropOp::none(), false) + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/tensor_cat.rs b/v2/dependency-patches/candle-core-0.9.2/src/tensor_cat.rs new file mode 100644 index 0000000000..520b246f5e --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/tensor_cat.rs @@ -0,0 +1,303 @@ +use crate::{shape::Dim, Context, Error, Result, Shape, Tensor}; + +impl Tensor { + /// Concatenates two or more tensors along a particular dimension. + /// + /// All tensors must of the same rank, and the output will have + /// the same rank + /// + /// ```rust + /// # use candle_core::{Tensor, DType, Device}; + /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; + /// let b = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; + /// + /// let c = Tensor::cat(&[&a, &b], 0)?; + /// assert_eq!(c.shape().dims(), &[4, 3]); + /// + /// let c = Tensor::cat(&[&a, &b], 1)?; + /// assert_eq!(c.shape().dims(), &[2, 6]); + /// # Ok::<(), candle_core::Error>(()) + /// ``` + pub fn cat, D: Dim>(args: &[A], dim: D) -> Result { + if args.is_empty() { + Err(Error::OpRequiresAtLeastOneTensor { op: "cat" }.bt())? + } + let arg0 = args[0].as_ref(); + if args.len() == 1 { + return Ok(arg0.clone()); + } + let dim = dim.to_index(arg0.shape(), "cat")?; + for arg in args { + arg.as_ref().check_dim(dim, "cat")?; + } + for (arg_idx, arg) in args.iter().enumerate() { + let arg = arg.as_ref(); + if arg0.rank() != arg.rank() { + Err(Error::UnexpectedNumberOfDims { + expected: arg0.rank(), + got: arg.rank(), + shape: arg.shape().clone(), + } + .bt())? + } + for (dim_idx, (v1, v2)) in arg0 + .shape() + .dims() + .iter() + .zip(arg.shape().dims().iter()) + .enumerate() + { + if dim_idx != dim && v1 != v2 { + Err(Error::ShapeMismatchCat { + dim: dim_idx, + first_shape: arg0.shape().clone(), + n: arg_idx + 1, + nth_shape: arg.shape().clone(), + } + .bt())? + } + } + } + let all_contiguous = args.iter().all(|v| v.as_ref().is_contiguous()); + if all_contiguous { + Self::cat_contiguous(args, dim) + } else if dim == 0 { + Self::cat0(args) + } else { + let args: Vec = args + .iter() + .map(|a| a.as_ref().transpose(0, dim)) + .collect::>>()?; + let cat = Self::cat0(&args)?; + cat.transpose(0, dim) + } + } + + fn cat0>(args: &[A]) -> Result { + if args.is_empty() { + Err(Error::OpRequiresAtLeastOneTensor { op: "cat" }.bt())? + } + let arg0 = args[0].as_ref(); + if args.len() == 1 { + return Ok(arg0.clone()); + } + let rank = arg0.rank(); + let device = arg0.device(); + let dtype = arg0.dtype(); + let first_dims = arg0.shape().dims(); + let mut cat_dims = first_dims.to_vec(); + cat_dims[0] = 0; + let mut offsets = vec![0usize]; + for (arg_idx, arg) in args.iter().enumerate() { + let arg = arg.as_ref(); + if arg.dtype() != dtype { + Err(Error::DTypeMismatchBinaryOp { + lhs: dtype, + rhs: arg.dtype(), + op: "cat", + } + .bt())? + } + if arg.device().location() != device.location() { + Err(Error::DeviceMismatchBinaryOp { + lhs: device.location(), + rhs: arg.device().location(), + op: "cat", + } + .bt())? + } + if rank != arg.rank() { + Err(Error::UnexpectedNumberOfDims { + expected: rank, + got: arg.rank(), + shape: arg.shape().clone(), + } + .bt())? + } + for (dim_idx, (v1, v2)) in arg0 + .shape() + .dims() + .iter() + .zip(arg.shape().dims().iter()) + .enumerate() + { + if dim_idx == 0 { + cat_dims[0] += v2; + } + if dim_idx != 0 && v1 != v2 { + Err(Error::ShapeMismatchCat { + dim: dim_idx, + first_shape: arg0.shape().clone(), + n: arg_idx + 1, + nth_shape: arg.shape().clone(), + } + .bt())? + } + } + let next_offset = offsets.last().context("empty offsets")? + arg.elem_count(); + offsets.push(next_offset); + } + let shape = Shape::from(cat_dims); + let op = crate::op::BackpropOp::new(args, |args| crate::op::Op::Cat(args, 0)); + let mut storage = unsafe { device.alloc_uninit(&shape, dtype)? }; + for (arg, &offset) in args.iter().zip(offsets.iter()) { + let arg = arg.as_ref(); + arg.storage() + .copy_strided_src(&mut storage, offset, arg.layout())?; + } + Ok(crate::tensor::from_storage(storage, shape, op, false)) + } + + fn cat_contiguous>(args: &[A], dim: usize) -> Result { + if args.is_empty() { + Err(Error::OpRequiresAtLeastOneTensor { op: "cat" }.bt())? + } + let arg0 = args[0].as_ref(); + if args.len() == 1 { + return Ok(arg0.clone()); + } + let rank = arg0.rank(); + let device = arg0.device(); + let dtype = arg0.dtype(); + let first_dims = arg0.shape().dims(); + let mut cat_dims = first_dims.to_vec(); + cat_dims[dim] = 0; + for (arg_idx, arg) in args.iter().enumerate() { + let arg = arg.as_ref(); + if arg.dtype() != dtype { + Err(Error::DTypeMismatchBinaryOp { + lhs: dtype, + rhs: arg.dtype(), + op: "cat", + } + .bt())? + } + if arg.device().location() != device.location() { + Err(Error::DeviceMismatchBinaryOp { + lhs: device.location(), + rhs: arg.device().location(), + op: "cat", + } + .bt())? + } + if rank != arg.rank() { + Err(Error::UnexpectedNumberOfDims { + expected: rank, + got: arg.rank(), + shape: arg.shape().clone(), + } + .bt())? + } + for (dim_idx, (v1, v2)) in arg0 + .shape() + .dims() + .iter() + .zip(arg.shape().dims().iter()) + .enumerate() + { + if dim_idx == dim { + cat_dims[dim] += v2; + } + if dim_idx != dim && v1 != v2 { + Err(Error::ShapeMismatchCat { + dim: dim_idx, + first_shape: arg0.shape().clone(), + n: arg_idx + 1, + nth_shape: arg.shape().clone(), + } + .bt())? + } + } + } + let cat_target_dim_len = cat_dims[dim]; + let block_size: usize = cat_dims.iter().skip(1 + dim).product(); + let shape = Shape::from(cat_dims); + let op = crate::op::BackpropOp::new(args, |args| crate::op::Op::Cat(args, dim)); + let mut storage = unsafe { device.alloc_uninit(&shape, dtype)? }; + let mut dst_o = 0; + for arg in args.iter() { + let arg = arg.as_ref(); + let arg_dims = arg.shape().dims(); + let d1: usize = arg_dims.iter().take(dim).product(); + let d2 = block_size * arg_dims[dim]; + let dst_s = block_size * cat_target_dim_len; + let src_o = arg.layout().start_offset(); + arg.storage().copy2d( + &mut storage, + d1, + d2, + /* src_s */ d2, + dst_s, + src_o, + dst_o, + )?; + dst_o += d2; + } + Ok(crate::tensor::from_storage(storage, shape, op, false)) + } + + /// Set the values on `self` using values from `src`. The copy starts at the specified + /// `offset` for the target dimension `dim` on `self`. + /// `self` and `src` must have the same shape except on dimension `dim` where the `self` size + /// has to be greater than or equal to `offset` plus the `src` size. + /// + /// Note that this modifies `self` in place and as such is not compatible with + /// back-propagation. + pub fn slice_set(&self, src: &Self, dim: D, offset: usize) -> Result<()> { + let dim = dim.to_index(self.shape(), "slice-set")?; + if !self.is_contiguous() || !src.is_contiguous() { + Err(Error::RequiresContiguous { op: "slice-set" }.bt())? + } + if self.same_storage(src) { + crate::bail!("cannot use slice_set when self and src share their storage") + } + if self.dtype() != src.dtype() { + Err(Error::DTypeMismatchBinaryOp { + lhs: self.dtype(), + rhs: src.dtype(), + op: "slice-set", + } + .bt())? + } + if self.device().location() != src.device().location() { + Err(Error::DeviceMismatchBinaryOp { + lhs: self.device().location(), + rhs: src.device().location(), + op: "slice-set", + } + .bt())? + } + if self.rank() != src.rank() { + Err(Error::UnexpectedNumberOfDims { + expected: self.rank(), + got: src.rank(), + shape: self.shape().clone(), + } + .bt())? + } + for (dim_idx, (v1, v2)) in self.dims().iter().zip(src.dims().iter()).enumerate() { + if dim_idx == dim && *v2 + offset > *v1 { + crate::bail!("shape mismatch on target dim, dst: {v1}, src: {v2} + {offset}") + } + if dim_idx != dim && v1 != v2 { + crate::bail!("shape mismatch on dim {dim_idx}, {v1} <> {v2}") + } + } + let block_size: usize = src.dims().iter().skip(1 + dim).product(); + let d1: usize = src.dims().iter().take(dim).product(); + let d2 = block_size * src.dims()[dim]; + let dst_o = self.layout().start_offset() + offset * block_size; + let src_o = src.layout().start_offset(); + src.storage().copy2d( + &mut self.storage_mut(), + d1, + d2, + /* src_s */ d2, + /* dst_s */ block_size * self.dims()[dim], + src_o, + dst_o, + )?; + + Ok(()) + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/test_utils.rs b/v2/dependency-patches/candle-core-0.9.2/src/test_utils.rs new file mode 100644 index 0000000000..e331399f4d --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/test_utils.rs @@ -0,0 +1,71 @@ +use crate::{Result, Tensor}; + +#[macro_export] +macro_rules! test_device { + // TODO: Switch to generating the two last arguments automatically once concat_idents is + // stable. https://github.com/rust-lang/rust/issues/29599 + ($fn_name: ident, $test_cpu: ident, $test_cuda: ident, $test_metal: ident) => { + #[test] + fn $test_cpu() -> Result<()> { + $fn_name(&Device::Cpu) + } + + #[cfg(feature = "cuda")] + #[test] + fn $test_cuda() -> Result<()> { + $fn_name(&Device::new_cuda(0)?) + } + + #[cfg(feature = "metal")] + #[test] + fn $test_metal() -> Result<()> { + $fn_name(&Device::new_metal(0)?) + } + }; +} + +pub fn assert_tensor_eq(t1: &Tensor, t2: &Tensor) -> Result<()> { + assert_eq!(t1.shape(), t2.shape()); + // Default U8 may not be large enough to hold the sum (`t.sum_all` defaults to the dtype of `t`) + let eq_tensor = t1.eq(t2)?.to_dtype(crate::DType::U32)?; + let all_equal = eq_tensor.sum_all()?; + assert_eq!(all_equal.to_scalar::()?, eq_tensor.elem_count() as u32); + Ok(()) +} + +pub fn to_vec0_round(t: &Tensor, digits: i32) -> Result { + let b = 10f32.powi(digits); + let t = t.to_vec0::()?; + Ok(f32::round(t * b) / b) +} + +pub fn to_vec1_round(t: &Tensor, digits: i32) -> Result> { + let b = 10f32.powi(digits); + let t = t.to_vec1::()?; + let t = t.iter().map(|t| f32::round(t * b) / b).collect(); + Ok(t) +} + +pub fn to_vec2_round(t: &Tensor, digits: i32) -> Result>> { + let b = 10f32.powi(digits); + let t = t.to_vec2::()?; + let t = t + .iter() + .map(|t| t.iter().map(|t| f32::round(t * b) / b).collect()) + .collect(); + Ok(t) +} + +pub fn to_vec3_round(t: &Tensor, digits: i32) -> Result>>> { + let b = 10f32.powi(digits); + let t = t.to_vec3::()?; + let t = t + .iter() + .map(|t| { + t.iter() + .map(|t| t.iter().map(|t| f32::round(t * b) / b).collect()) + .collect() + }) + .collect(); + Ok(t) +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/utils.rs b/v2/dependency-patches/candle-core-0.9.2/src/utils.rs new file mode 100644 index 0000000000..4a58db8113 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/utils.rs @@ -0,0 +1,45 @@ +//! Useful functions for checking features. +use std::str::FromStr; + +pub fn get_num_threads() -> usize { + // Respond to the same environment variable as rayon. + match std::env::var("RAYON_NUM_THREADS") + .ok() + .and_then(|s| usize::from_str(&s).ok()) + { + Some(x) if x > 0 => x, + Some(_) | None => num_cpus::get(), + } +} + +pub fn has_accelerate() -> bool { + cfg!(feature = "accelerate") +} + +pub fn has_mkl() -> bool { + cfg!(feature = "mkl") +} + +pub fn cuda_is_available() -> bool { + cfg!(feature = "cuda") +} + +pub fn metal_is_available() -> bool { + cfg!(feature = "metal") +} + +pub fn with_avx() -> bool { + cfg!(target_feature = "avx2") +} + +pub fn with_neon() -> bool { + cfg!(target_feature = "neon") +} + +pub fn with_simd128() -> bool { + cfg!(target_feature = "simd128") +} + +pub fn with_f16c() -> bool { + cfg!(target_feature = "f16c") +} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/variable.rs b/v2/dependency-patches/candle-core-0.9.2/src/variable.rs new file mode 100644 index 0000000000..1e4880e59a --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/src/variable.rs @@ -0,0 +1,152 @@ +// Variables are wrappers around tensors that can be modified, they are typically used for holding +// weights and being modified by gradient descent. +// We do not expose a public way to create variables as this would break the invariant that the +// tensor within a variable is actually with `is_variable` set to `true`. +use crate::{DType, Device, Error, Result, Shape, Tensor}; + +/// A variable is a wrapper around a tensor, however variables can have their content modified +/// whereas tensors are immutable. +#[derive(Clone, Debug)] +pub struct Var(Tensor); + +impl std::fmt::Display for Var { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + std::fmt::Display::fmt(&self.0, f) + } +} + +impl std::ops::Deref for Var { + type Target = Tensor; + + fn deref(&self) -> &Self::Target { + self.0.as_ref() + } +} + +impl Var { + pub fn zeros>(shape: S, dtype: DType, device: &Device) -> Result { + let inner = Tensor::zeros_impl(shape, dtype, device, true)?; + Ok(Self(inner)) + } + + pub fn ones>(shape: S, dtype: DType, device: &Device) -> Result { + let inner = Tensor::ones_impl(shape, dtype, device, true)?; + Ok(Self(inner)) + } + + // Convert a tensor to a variable, if the tensor is already a variable then it is returned as is. + pub fn from_tensor(t: &Tensor) -> Result { + if t.is_variable() { + Ok(Self(t.clone())) + } else { + let inner = t.make_var()?; + Ok(Self(inner)) + } + } + + pub fn rand_f64>( + lo: f64, + up: f64, + s: S, + dtype: DType, + device: &Device, + ) -> Result { + let inner = Tensor::rand_f64_impl(lo, up, s, dtype, device, true)?; + Ok(Self(inner)) + } + + pub fn randn_f64>( + mean: f64, + std: f64, + s: S, + dtype: DType, + device: &Device, + ) -> Result { + let inner = Tensor::randn_f64_impl(mean, std, s, dtype, device, true)?; + Ok(Self(inner)) + } + + pub fn rand, T: crate::FloatDType>( + lo: T, + up: T, + s: S, + device: &Device, + ) -> Result { + let inner = Tensor::rand_impl(lo, up, s, device, true)?; + Ok(Self(inner)) + } + + pub fn randn, T: crate::FloatDType>( + mean: T, + std: T, + s: S, + device: &Device, + ) -> Result { + let inner = Tensor::randn_impl(mean, std, s, device, true)?; + Ok(Self(inner)) + } + + /// Creates a new tensor on the specified device using the content and shape of the input. + /// This is similar to `new` but the resulting tensor is a variable. + pub fn new(array: A, device: &Device) -> Result { + let shape = array.shape()?; + let inner = Tensor::new_impl(array, shape, device, true)?; + Ok(Self(inner)) + } + + pub fn from_vec, D: crate::WithDType>( + data: Vec, + shape: S, + device: &Device, + ) -> Result { + let inner = Tensor::from_vec_impl(data, shape, device, true)?; + Ok(Self(inner)) + } + + pub fn from_slice, D: crate::WithDType>( + array: &[D], + shape: S, + device: &Device, + ) -> Result { + let inner = Tensor::new_impl(array, shape.into(), device, true)?; + Ok(Self(inner)) + } + + pub fn as_detached_tensor(&self) -> Tensor { + self.0.detach() + } + + pub fn as_tensor(&self) -> &Tensor { + &self.0 + } + + /// Consumes this `Var` and return the underlying tensor. + pub fn into_inner(self) -> Tensor { + self.0 + } + + /// Sets the content of the inner tensor, this does not require a mutable reference as inner + /// mutability is used. + pub fn set(&self, src: &Tensor) -> Result<()> { + if self.same_storage(src) { + let msg = "cannot set a variable to a tensor that is derived from its value"; + Err(Error::CannotSetVar { msg }.bt())? + } + let (mut dst, layout) = self.storage_mut_and_layout(); + if !layout.is_contiguous() { + let msg = "cannot set a non-contiguous variable"; + Err(Error::CannotSetVar { msg }.bt())? + } + let (src, src_l) = src.storage_and_layout(); + if layout.shape() != src_l.shape() { + Err(Error::ShapeMismatchBinaryOp { + lhs: layout.shape().clone(), + rhs: src_l.shape().clone(), + op: "set", + } + .bt())? + } + src.copy_strided_src(&mut dst, layout.start_offset(), src_l)?; + Ok(()) + } +} diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/bilinear_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/bilinear_tests.rs new file mode 100644 index 0000000000..3716df6ba4 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/tests/bilinear_tests.rs @@ -0,0 +1,525 @@ +use candle_core::{test_device, Device, IndexOp, Result, Tensor}; + +// ============================================================================ +// PyTorch Exact Comparison Tests +// ============================================================================ +// These tests compare against exact PyTorch outputs to ensure correctness + +/* Test corresponds to PyTorch: +import torch +import torch.nn.functional as F +input = torch.arange(16, dtype=torch.float32).reshape(1, 1, 4, 4) +output = F.interpolate(input, size=(8, 8), mode='bilinear', align_corners=False) +*/ +fn bilinear_pytorch_2x_upscale(dev: &Device) -> Result<()> { + let input = Tensor::arange(0f32, 16f32, dev)?.reshape((1, 1, 4, 4))?; + let output = input.upsample_bilinear2d(8, 8, false)?; + + // PyTorch expected output (verified from PyTorch 2.10.0) + let expected = Tensor::new( + &[ + 0.0000f32, 0.2500, 0.7500, 1.2500, 1.7500, 2.2500, 2.7500, 3.0000, 1.0000, 1.2500, + 1.7500, 2.2500, 2.7500, 3.2500, 3.7500, 4.0000, 3.0000, 3.2500, 3.7500, 4.2500, 4.7500, + 5.2500, 5.7500, 6.0000, 5.0000, 5.2500, 5.7500, 6.2500, 6.7500, 7.2500, 7.7500, 8.0000, + 7.0000, 7.2500, 7.7500, 8.2500, 8.7500, 9.2500, 9.7500, 10.0000, 9.0000, 9.2500, + 9.7500, 10.2500, 10.7500, 11.2500, 11.7500, 12.0000, 11.0000, 11.2500, 11.7500, + 12.2500, 12.7500, 13.2500, 13.7500, 14.0000, 12.0000, 12.2500, 12.7500, 13.2500, + 13.7500, 14.2500, 14.7500, 15.0000, + ], + dev, + )? + .reshape((1, 1, 8, 8))?; + + let diff = (&output - &expected)?.abs()?.flatten_all()?.max(0)?; + let max_diff = diff.to_vec0::()?; + + assert!( + max_diff < 1e-4, + "Max difference {} exceeds threshold 1e-4", + max_diff + ); + Ok(()) +} + +/* Test corresponds to PyTorch: +import torch +import torch.nn.functional as F +input = torch.arange(64, dtype=torch.float32).reshape(1, 1, 8, 8) +output = F.interpolate(input, size=(4, 4), mode='bilinear', align_corners=False) +*/ +fn bilinear_pytorch_downscale(dev: &Device) -> Result<()> { + let input = Tensor::arange(0f32, 64f32, dev)?.reshape((1, 1, 8, 8))?; + let output = input.upsample_bilinear2d(4, 4, false)?; + + // PyTorch expected output + let expected = Tensor::new( + &[ + 4.5f32, 6.5, 8.5, 10.5, 20.5, 22.5, 24.5, 26.5, 36.5, 38.5, 40.5, 42.5, 52.5, 54.5, + 56.5, 58.5, + ], + dev, + )? + .reshape((1, 1, 4, 4))?; + + let diff = (&output - &expected)?.abs()?.flatten_all()?.max(0)?; + let max_diff = diff.to_vec0::()?; + + assert!( + max_diff < 1e-4, + "Max difference {} exceeds threshold 1e-4", + max_diff + ); + Ok(()) +} + +/* Test corresponds to PyTorch: +import torch +import torch.nn.functional as F +torch.manual_seed(42) +input = torch.randn(1, 2, 4, 4, dtype=torch.float32) +output = F.interpolate(input, size=(8, 8), mode='bilinear', align_corners=False) +*/ +fn bilinear_pytorch_multi_channel(dev: &Device) -> Result<()> { + // Using fixed seed data from PyTorch (seed=42) + let input = Tensor::new( + &[ + // Channel 0 + 1.9269f32, 1.4873, 0.9007, -2.1055, 0.6784, -1.2345, -0.0431, -1.6047, -0.7521, 1.6487, + -0.3925, -1.4036, -0.7279, -0.5594, -0.7688, 0.7624, // Channel 1 + 1.6423f32, -0.1596, -0.4974, 0.4396, -0.7581, 1.0783, 0.8008, 1.6806, 1.2791, 1.2964, + 0.6105, 1.3347, -0.2316, 0.0418, -0.2516, 0.8599, + ], + dev, + )? + .reshape((1, 2, 4, 4))?; + + let output = input.upsample_bilinear2d(8, 8, false)?; + + assert_eq!(output.dims(), &[1, 2, 8, 8]); + + // Verify output is finite and in reasonable range + let output_vec = output.flatten_all()?.to_vec1::()?; + for &val in &output_vec { + assert!(val.is_finite(), "Output contains non-finite value"); + } + + // Check first row of channel 0 from PyTorch output + let output_ch0_row0 = output.i((0, 0, 0, ..))?.to_vec1::()?; + let expected_ch0_row0 = [ + 1.9269f32, 1.8170, 1.5972, 1.3406, 1.0474, 0.1492, -1.3540, -2.1055, + ]; + + for (i, (&out, &exp)) in output_ch0_row0 + .iter() + .zip(expected_ch0_row0.iter()) + .enumerate() + { + let diff = (out - exp).abs(); + assert!( + diff < 1e-3, + "Channel 0, row 0, index {} differs: got {}, expected {}, diff {}", + i, + out, + exp, + diff + ); + } + + // Check first row of channel 1 from PyTorch output + let output_ch1_row0 = output.i((0, 1, 0, ..))?.to_vec1::()?; + let expected_ch1_row0 = [ + 1.6423f32, 1.1918, 0.2909, -0.2440, -0.4129, -0.2632, 0.2053, 0.4396, + ]; + + for (i, (&out, &exp)) in output_ch1_row0 + .iter() + .zip(expected_ch1_row0.iter()) + .enumerate() + { + let diff = (out - exp).abs(); + assert!( + diff < 1e-3, + "Channel 1, row 0, index {} differs: got {}, expected {}, diff {}", + i, + out, + exp, + diff + ); + } + + Ok(()) +} + +/* Test corresponds to PyTorch: +import torch +import torch.nn.functional as F +input = torch.tensor([[[[1.0, 2.0], [3.0, 4.0]]]], dtype=torch.float32) +output = F.interpolate(input, size=(4, 4), mode='bilinear', align_corners=True) +*/ +fn bilinear_pytorch_align_corners_true(dev: &Device) -> Result<()> { + let input = Tensor::from_vec(vec![1.0f32, 2.0, 3.0, 4.0], (1, 1, 2, 2), dev)?; + let output = input.upsample_bilinear2d(4, 4, true)?; + + // PyTorch expected output with align_corners=True + let expected = Tensor::new( + &[ + 1.0f32, 1.3333, 1.6667, 2.0, 1.6667, 2.0, 2.3333, 2.6667, 2.3333, 2.6667, 3.0, 3.3333, + 3.0, 3.3333, 3.6667, 4.0, + ], + dev, + )? + .reshape((1, 1, 4, 4))?; + + let diff = (&output - &expected)?.abs()?.flatten_all()?.max(0)?; + let max_diff = diff.to_vec0::()?; + + assert!( + max_diff < 1e-3, + "Max difference {} exceeds threshold 1e-3", + max_diff + ); + + // Verify corners are exactly preserved with align_corners=True + let output_vec = output.flatten_all()?.to_vec1::()?; + assert!( + (output_vec[0] - 1.0).abs() < 1e-5, + "Top-left corner not preserved" + ); + assert!( + (output_vec[3] - 2.0).abs() < 1e-5, + "Top-right corner not preserved" + ); + assert!( + (output_vec[12] - 3.0).abs() < 1e-5, + "Bottom-left corner not preserved" + ); + assert!( + (output_vec[15] - 4.0).abs() < 1e-5, + "Bottom-right corner not preserved" + ); + + Ok(()) +} + +/* Test corresponds to PyTorch: +import torch +import torch.nn.functional as F +input = torch.arange(16, dtype=torch.float32).reshape(1, 1, 4, 4) +output = F.interpolate(input, scale_factor=2.0, mode='bilinear', align_corners=False) +*/ +fn bilinear_pytorch_scale_factor(dev: &Device) -> Result<()> { + let input = Tensor::arange(0f32, 16f32, dev)?.reshape((1, 1, 4, 4))?; + let output_scale = input.upsample_bilinear2d_with_scale(2.0, 2.0, false)?; + let output_size = input.upsample_bilinear2d(8, 8, false)?; + + // scale_factor=2.0 should produce identical results to size=(8, 8) + let diff = (&output_scale - &output_size)? + .abs()? + .flatten_all()? + .max(0)?; + let max_diff = diff.to_vec0::()?; + + assert!( + max_diff < 1e-6, + "scale_factor and size methods differ by {}", + max_diff + ); + + Ok(()) +} + +/* Test corresponds to PyTorch: +import torch +import torch.nn.functional as F +input = torch.arange(24, dtype=torch.float32).reshape(1, 1, 4, 6) +output = F.interpolate(input, size=(8, 12), mode='bilinear', align_corners=False) +*/ +fn bilinear_pytorch_non_square_exact(dev: &Device) -> Result<()> { + let input = Tensor::arange(0f32, 24f32, dev)?.reshape((1, 1, 4, 6))?; + let output = input.upsample_bilinear2d(8, 12, false)?; + + // PyTorch expected output (verified from PyTorch 2.10.0) + #[rustfmt::skip] + let expected = Tensor::new( + &[ + 0.0f32, 0.25, 0.75, 1.25, 1.75, 2.25, 2.75, 3.25, 3.75, 4.25, 4.75, 5.0, + 1.5, 1.75, 2.25, 2.75, 3.25, 3.75, 4.25, 4.75, 5.25, 5.75, 6.25, 6.5, + 4.5, 4.75, 5.25, 5.75, 6.25, 6.75, 7.25, 7.75, 8.25, 8.75, 9.25, 9.5, + 7.5, 7.75, 8.25, 8.75, 9.25, 9.75, 10.25, 10.75, 11.25, 11.75, 12.25, 12.5, + 10.5, 10.75, 11.25, 11.75, 12.25, 12.75, 13.25, 13.75, 14.25, 14.75, 15.25, 15.5, + 13.5, 13.75, 14.25, 14.75, 15.25, 15.75, 16.25, 16.75, 17.25, 17.75, 18.25, 18.5, + 16.5, 16.75, 17.25, 17.75, 18.25, 18.75, 19.25, 19.75, 20.25, 20.75, 21.25, 21.5, + 18.0, 18.25, 18.75, 19.25, 19.75, 20.25, 20.75, 21.25, 21.75, 22.25, 22.75, 23.0, + ], + dev, + )? + .reshape((1, 1, 8, 12))?; + + let diff = (&output - &expected)?.abs()?.flatten_all()?.max(0)?; + let max_diff = diff.to_vec0::()?; + + assert!( + max_diff < 1e-4, + "Max difference {} exceeds threshold 1e-4", + max_diff + ); + Ok(()) +} + +/* Test corresponds to PyTorch: +import torch +import torch.nn.functional as F +input = torch.tensor([[[[5.0]]]], dtype=torch.float32) +output = F.interpolate(input, size=(3, 3), mode='bilinear', align_corners=False) +*/ +fn bilinear_pytorch_tiny_1x1_to_3x3(dev: &Device) -> Result<()> { + let input = Tensor::new(&[5.0f32], dev)?.reshape((1, 1, 1, 1))?; + let output = input.upsample_bilinear2d(3, 3, false)?; + + // PyTorch expected output: all values should be 5.0 + let expected = Tensor::new(&[5.0f32, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0], dev)? + .reshape((1, 1, 3, 3))?; + + let diff = (&output - &expected)?.abs()?.flatten_all()?.max(0)?; + let max_diff = diff.to_vec0::()?; + + assert!( + max_diff < 1e-6, + "Max difference {} exceeds threshold 1e-6", + max_diff + ); + Ok(()) +} + +/* Test corresponds to PyTorch: +import torch +import torch.nn.functional as F +input = torch.tensor([[[[2.0, 8.0]]]], dtype=torch.float32) +output = F.interpolate(input, size=(3, 6), mode='bilinear', align_corners=False) +*/ +fn bilinear_pytorch_tiny_1x2_to_3x6(dev: &Device) -> Result<()> { + let input = Tensor::new(&[2.0f32, 8.0], dev)?.reshape((1, 1, 1, 2))?; + let output = input.upsample_bilinear2d(3, 6, false)?; + + // PyTorch expected output + #[rustfmt::skip] + let expected = Tensor::new( + &[ + 2.0f32, 2.0, 4.0, 6.0, 8.0, 8.0, + 2.0, 2.0, 4.0, 6.0, 8.0, 8.0, + 2.0, 2.0, 4.0, 6.0, 8.0, 8.0, + ], + dev, + )? + .reshape((1, 1, 3, 6))?; + + let diff = (&output - &expected)?.abs()?.flatten_all()?.max(0)?; + let max_diff = diff.to_vec0::()?; + + assert!( + max_diff < 1e-6, + "Max difference {} exceeds threshold 1e-6", + max_diff + ); + Ok(()) +} + +/* Test corresponds to PyTorch: +import torch +import torch.nn.functional as F +torch.manual_seed(123) +input = torch.randn(1, 1, 64, 64, dtype=torch.float32) +output = F.interpolate(input, size=(128, 128), mode='bilinear', align_corners=False) +*/ +fn bilinear_pytorch_large_64x64_to_128x128(dev: &Device) -> Result<()> { + // Test large tensor for numerical stability + // We'll just verify dimensions and that output is finite + use candle_core::DType; + + let input = Tensor::randn(0f32, 1f32, (1, 1, 64, 64), dev)?; + let output = input.upsample_bilinear2d(128, 128, false)?; + + assert_eq!(output.dims(), &[1, 1, 128, 128]); + assert_eq!(output.dtype(), DType::F32); + + // Verify all values are finite + let output_vec = output.flatten_all()?.to_vec1::()?; + for &val in &output_vec { + assert!( + val.is_finite(), + "Large tensor output contains non-finite value" + ); + } + + // Verify output is in reasonable range (should be similar to input range) + let min_val = output_vec.iter().copied().fold(f32::INFINITY, f32::min); + let max_val = output_vec.iter().copied().fold(f32::NEG_INFINITY, f32::max); + + assert!( + min_val > -10.0 && max_val < 10.0, + "Large tensor output values out of expected range: min={}, max={}", + min_val, + max_val + ); + + Ok(()) +} + +// ============================================================================ +// Dimension and Shape Tests (Consolidated) +// ============================================================================ +// These tests verify correct output dimensions for various input configurations + +fn bilinear_output_dimensions(dev: &Device) -> Result<()> { + // Test 1: Non-square dimensions + let t1 = Tensor::arange(0f32, 32f32, dev)?.reshape((1, 1, 4, 8))?; + let out1 = t1.upsample_bilinear2d(6, 12, false)?; + assert_eq!(out1.dims(), &[1, 1, 6, 12], "Non-square upscale failed"); + + // Test 2: Batch processing + let t2 = Tensor::arange(0f32, 192f32, dev)?.reshape((4, 3, 4, 4))?; + let out2 = t2.upsample_bilinear2d(8, 8, false)?; + assert_eq!(out2.dims(), &[4, 3, 8, 8], "Batch processing failed"); + + // Test 3: Asymmetric scale factors + let t3 = Tensor::arange(0f32, 16f32, dev)?.reshape((1, 1, 4, 4))?; + let out3 = t3.upsample_bilinear2d_with_scale(2.0, 3.0, false)?; + assert_eq!(out3.dims(), &[1, 1, 8, 12], "Asymmetric scale failed"); + + // Test 4: Fractional scale factors + let t4 = Tensor::arange(0f32, 16f32, dev)?.reshape((1, 1, 4, 4))?; + let out4 = t4.upsample_bilinear2d_with_scale(1.5, 1.5, false)?; + assert_eq!(out4.dims(), &[1, 1, 6, 6], "Fractional scale failed"); + + // Test 5: Single pixel output + let t5 = Tensor::arange(0f32, 16f32, dev)?.reshape((1, 1, 4, 4))?; + let out5 = t5.upsample_bilinear2d(1, 1, false)?; + assert_eq!(out5.dims(), &[1, 1, 1, 1], "Single pixel output failed"); + let val = out5.flatten_all()?.to_vec1::()?[0]; + assert!(val.is_finite(), "Single pixel value is not finite"); + + // Test 6: Large scale factor + let t6 = Tensor::arange(0f32, 4f32, dev)?.reshape((1, 1, 2, 2))?; + let out6 = t6.upsample_bilinear2d_with_scale(5.0, 5.0, false)?; + assert_eq!(out6.dims(), &[1, 1, 10, 10], "Large scale factor failed"); + + Ok(()) +} + +// ============================================================================ +// Special Behavior Tests +// ============================================================================ + +fn bilinear_identity(dev: &Device) -> Result<()> { + // Test that upsampling to the same size returns an identical tensor + let t = Tensor::arange(0f32, 16f32, dev)?.reshape((1, 1, 4, 4))?; + let output = t.upsample_bilinear2d(4, 4, false)?; + + let diff = (&t - &output)?.abs()?.flatten_all()?.max(0)?; + assert!(diff.to_vec0::()? < 1e-6); + Ok(()) +} + +fn bilinear_align_corners_difference(dev: &Device) -> Result<()> { + // Test that align_corners parameter produces different results + let t = Tensor::arange(0f32, 16f32, dev)?.reshape((1, 1, 4, 4))?; + + let output_false = t.upsample_bilinear2d(8, 8, false)?; + let output_true = t.upsample_bilinear2d(8, 8, true)?; + + // Results should be different between align_corners modes + let diff = (&output_false - &output_true)?.abs()?.sum_all()?; + assert!(diff.to_vec0::()? > 0.1); + Ok(()) +} + +// ============================================================================ +// Test Device Macros +// ============================================================================ + +// PyTorch exact comparison tests +test_device!( + bilinear_pytorch_2x_upscale, + bilinear_pytorch_2x_upscale_cpu, + bilinear_pytorch_2x_upscale_gpu, + bilinear_pytorch_2x_upscale_metal +); + +test_device!( + bilinear_pytorch_downscale, + bilinear_pytorch_downscale_cpu, + bilinear_pytorch_downscale_gpu, + bilinear_pytorch_downscale_metal +); + +test_device!( + bilinear_pytorch_multi_channel, + bilinear_pytorch_multi_channel_cpu, + bilinear_pytorch_multi_channel_gpu, + bilinear_pytorch_multi_channel_metal +); + +test_device!( + bilinear_pytorch_align_corners_true, + bilinear_pytorch_align_corners_true_cpu, + bilinear_pytorch_align_corners_true_gpu, + bilinear_pytorch_align_corners_true_metal +); + +test_device!( + bilinear_pytorch_scale_factor, + bilinear_pytorch_scale_factor_cpu, + bilinear_pytorch_scale_factor_gpu, + bilinear_pytorch_scale_factor_metal +); + +test_device!( + bilinear_pytorch_non_square_exact, + bilinear_pytorch_non_square_exact_cpu, + bilinear_pytorch_non_square_exact_gpu, + bilinear_pytorch_non_square_exact_metal +); + +test_device!( + bilinear_pytorch_tiny_1x1_to_3x3, + bilinear_pytorch_tiny_1x1_to_3x3_cpu, + bilinear_pytorch_tiny_1x1_to_3x3_gpu, + bilinear_pytorch_tiny_1x1_to_3x3_metal +); + +test_device!( + bilinear_pytorch_tiny_1x2_to_3x6, + bilinear_pytorch_tiny_1x2_to_3x6_cpu, + bilinear_pytorch_tiny_1x2_to_3x6_gpu, + bilinear_pytorch_tiny_1x2_to_3x6_metal +); + +test_device!( + bilinear_pytorch_large_64x64_to_128x128, + bilinear_pytorch_large_64x64_to_128x128_cpu, + bilinear_pytorch_large_64x64_to_128x128_gpu, + bilinear_pytorch_large_64x64_to_128x128_metal +); + +// Dimension tests (consolidated) +test_device!( + bilinear_output_dimensions, + bilinear_output_dimensions_cpu, + bilinear_output_dimensions_gpu, + bilinear_output_dimensions_metal +); + +// Special behavior tests +test_device!( + bilinear_identity, + bilinear_identity_cpu, + bilinear_identity_gpu, + bilinear_identity_metal +); + +test_device!( + bilinear_align_corners_difference, + bilinear_align_corners_difference_cpu, + bilinear_align_corners_difference_gpu, + bilinear_align_corners_difference_metal +); diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/conv_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/conv_tests.rs new file mode 100644 index 0000000000..e492bebab8 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/tests/conv_tests.rs @@ -0,0 +1,894 @@ +use anyhow::Result; +use candle_core::{test_device, test_utils, Device, IndexOp, Tensor}; + +/* This test is based on the following script. +import torch +torch.manual_seed(4242) + +t = torch.randn((1, 4, 5)) +w = torch.randn((2, 4, 3)) +print(t.flatten()) +print(w.flatten()) +res = torch.nn.functional.conv1d(t, w) +print(res.flatten()) +res = torch.nn.functional.conv1d(t, w, padding=1) +print(res.flatten()) + +w_t = w.transpose(0, 1) +res = torch.nn.functional.conv_transpose1d(t, w_t) +print(res.shape) +print(res) +res = torch.nn.functional.conv_transpose1d(t, w_t, groups=2) +print(res.shape) +print(res) +*/ +fn conv1d(dev: &Device) -> Result<()> { + let t = Tensor::new( + &[ + 0.4056f32, -0.8689, -0.0773, -1.5630, 1.2279, -0.9287, -1.7030, 0.1370, 0.1866, 0.4145, + 1.8025, -0.1536, 2.2013, -0.6836, 0.2477, 1.3127, -0.6957, 0.3278, -1.0124, 0.5599, + ], + dev, + )? + .reshape((1, 4, 5))?; + let w = Tensor::new( + &[ + -0.8404f32, -0.3490, 0.0130, 1.3123, 0.1763, -1.9249, 1.4270, 0.9421, 0.8670, -0.7181, + -1.1111, 0.8869, -1.2429, 1.8357, 1.6052, -1.3844, 0.3951, -1.2036, 0.6686, 1.6261, + -0.6451, -0.0840, -1.4247, 0.5512, + ], + dev, + )? + .reshape((2, 4, 3))?; + let res = t.conv1d(&w, 0, 1, 1, 1)?; + assert_eq!(res.dims(), [1, 2, 3]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [2.6357, -1.3336, 4.1393, -1.1784, 3.5675, 0.5069] + ); + let res = t.conv1d(&w, /*padding*/ 1, 1, 1, 1)?; + assert_eq!(res.dims(), [1, 2, 5]); + // Same as pytorch default padding: use zeros. + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [2.4509, 2.6357, -1.3336, 4.1393, 0.5657, 1.8091, -1.1784, 3.5675, 0.5069, 3.3352] + ); + let res = { + let t = Tensor::cat(&[&t.zeros_like()?, &t, &t.zeros_like()?], 0)?; + t.conv1d(&w, /*padding*/ 1, 1, 1, 1)? + }; + assert_eq!(res.dims(), [3, 2, 5]); + // Same as pytorch default padding: use zeros. + assert_eq!( + test_utils::to_vec1_round(&res.i(0)?.flatten_all()?, 4)?, + [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.] + ); + assert_eq!( + test_utils::to_vec1_round(&res.i(1)?.flatten_all()?, 4)?, + [2.4509, 2.6357, -1.3336, 4.1393, 0.5657, 1.8091, -1.1784, 3.5675, 0.5069, 3.3352] + ); + + let w = w.transpose(0, 1)?; + // The CPU kernels applied in the contiguous and non contiguous cases are different. + for w in [w.clone(), w.contiguous()?] { + let res = t.conv_transpose1d(&w, 0, 0, 1, 1, 1)?; + assert_eq!(res.dims(), [1, 2, 7]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [ + 0.0699, -1.2899, 8.3018, 5.5873, 2.4572, -2.6143, -0.0706, 1.8765, 4.8318, 1.1538, + 4.7076, -5.9745, -0.8276, 1.621 + ], + ); + let res = t.conv_transpose1d(&w, 0, 0, 1, 1, 2)?; + assert_eq!(res.dims(), [1, 4, 7]); + assert_eq!( + test_utils::to_vec2_round(&res.squeeze(0)?, 4)?, + [ + [-1.5596, -1.8099, 2.0407, 4.8764, -0.1743, -0.735, -0.7819], + [0.7816, 3.8152, -0.5926, 2.2515, -5.1844, -0.3157, 1.4721], + [1.6295, 0.52, 6.2611, 0.7109, 2.6315, -1.8793, 0.7113], + [1.0949, 1.0166, 1.7464, 2.4561, -0.79, -0.5119, 0.1488] + ] + ); + } + Ok(()) +} + +fn conv1d_small(dev: &Device) -> Result<()> { + let t = Tensor::new(&[0.4056f32, -0.8689, -0.0773, -1.5630], dev)?.reshape((1, 1, 4))?; + let w = Tensor::new(&[1f32, 0., 0.], dev)?.reshape((1, 1, 3))?; + let res = t.conv1d(&w, 0, 1, 1, 1)?; + assert_eq!(res.dims(), [1, 1, 2]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [0.4056, -0.8689] + ); + let res = t.conv1d(&w, /*padding*/ 1, 1, 1, 1)?; + assert_eq!(res.dims(), [1, 1, 4]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [0.0, 0.4056, -0.8689, -0.0773], + ); + Ok(()) +} + +/* This test is based on the following script. +import torch +torch.manual_seed(4242) + +t = torch.randn((1, 4, 5, 5)) +w = torch.randn((2, 4, 3, 3)) +print(t.flatten()) +print(w.flatten()) +res = torch.nn.functional.conv2d(t, w) +print(res.flatten()) + +w_t = w.transpose(0, 1) +res = torch.nn.functional.conv_transpose2d(t, w_t) +print(res.shape) +print(res) + +res = torch.nn.functional.conv2d(t, w, dilation=2) +print(res.shape) +print(res[0]) + +res = torch.nn.functional.conv_transpose2d(t, w_t, dilation=2) +print(res.shape) +print(res) +*/ +fn conv2d(dev: &Device) -> Result<()> { + let t = Tensor::new( + &[ + 0.4056f32, -0.8689, -0.0773, -1.5630, -2.8012, -1.5059, 0.3972, 1.0852, 0.4997, 3.0616, + 1.6541, 0.0964, -0.8338, -1.6523, -0.8323, -0.1699, 0.0823, 0.3526, 0.6843, 0.2395, + 1.2279, -0.9287, -1.7030, 0.1370, 0.6047, 0.3770, -0.6266, 0.3529, 2.2013, -0.6836, + 0.2477, 1.3127, -0.2260, 0.2622, -1.2974, -0.8140, -0.8404, -0.3490, 0.0130, 1.3123, + 1.7569, -0.3956, -1.8255, 0.1727, -0.3538, 2.6941, 1.0529, 0.4219, -0.2071, 1.1586, + 0.4717, 0.3865, -0.5690, -0.5010, -0.1310, 0.7796, 0.6630, -0.2021, 2.6090, 0.2049, + 0.6466, -0.5042, -0.0603, -1.6538, -1.2429, 1.8357, 1.6052, -1.3844, 0.3323, -1.3712, + 0.9634, -0.4799, -0.6451, -0.0840, -1.4247, 0.5512, -0.1747, -0.5509, -0.3742, 0.3790, + -0.4431, -0.4720, -0.7890, 0.2620, 0.7875, 0.5377, -0.6779, -0.8088, 1.9098, 1.2006, + -0.8, -0.4983, 1.5480, 0.8265, -0.1025, 0.5138, 0.5748, 0.3821, -0.4607, 0.0085, + ], + dev, + )?; + let w = Tensor::new( + &[ + -0.9325f32, 0.6451, -0.8537, 0.2378, 0.8764, -0.1832, 0.2987, -0.6488, -0.2273, + -2.4184, -0.1192, -0.4821, -0.5079, -0.5766, -2.4729, 1.6734, 0.4558, 0.2851, 1.1514, + -0.9013, 1.0662, -0.1817, -0.0259, 0.1709, 0.5367, 0.7513, 0.8086, -2.2586, -0.5027, + 0.9141, -1.3086, -1.3343, -1.5669, -0.1657, 0.7958, 0.1432, 0.3896, -0.4501, 0.1667, + 0.0714, -0.0952, 1.2970, -0.1674, -0.3178, 1.0677, 0.3060, 0.7080, 0.1914, 1.1679, + -0.3602, 1.9265, -1.8626, -0.5112, -0.0982, 0.2621, 0.6565, 0.5908, 1.0089, -0.1646, + 1.8032, -0.6286, 0.2016, -0.3370, 1.2555, 0.8009, -0.6488, -0.4652, -1.5685, 1.5860, + 0.5583, 0.4623, 0.6026, + ], + dev, + )?; + let t = t.reshape((1, 4, 5, 5))?; + let w = w.reshape((2, 4, 3, 3))?; + let res = t.conv2d(&w, 0, 1, 1, 1)?; + assert_eq!(res.dims(), [1, 2, 3, 3]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [ + -4.2812, 2.0923, 5.2187, 7.5184, 0.752, -14.9426, 10.0087, 4.391, 0.2918, 1.6715, + 10.389, 3.6023, -4.2808, 0.2672, 5.3646, -5.2023, -2.1955, -9.4075 + ] + ); + let res = { + let t = Tensor::cat(&[&t.zeros_like()?, &t, &t.zeros_like()?], 0)?; + t.conv2d(&w, 0, 1, 1, 1)? + }; + assert_eq!(res.dims(), [3, 2, 3, 3]); + assert_eq!( + test_utils::to_vec1_round(&res.i(0)?.flatten_all()?, 4)?, + [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.] + ); + assert_eq!( + test_utils::to_vec1_round(&res.i(1)?.flatten_all()?, 4)?, + [ + -4.2812, 2.0923, 5.2187, 7.5184, 0.752, -14.9426, 10.0087, 4.391, 0.2918, 1.6715, + 10.389, 3.6023, -4.2808, 0.2672, 5.3646, -5.2023, -2.1955, -9.4075 + ] + ); + + let res = t.conv_transpose2d(&w.transpose(0, 1)?, 0, 0, 1, 1)?; + + assert_eq!(res.dims(), [1, 2, 7, 7]); + assert_eq!( + test_utils::to_vec3_round(&res.i(0)?, 4)?, + [ + [ + [-1.9918, 2.6797, -0.4599, -1.6037, 1.4131, -2.4012, 2.9277], + [1.8016, -3.5361, 1.0757, 3.5395, -8.2168, -3.2023, 0.5375], + [0.8243, 1.8675, 7.8929, -4.0746, -6.4415, 5.1139, 1.6889], + [0.2722, 8.9679, 3.3477, 1.8514, -4.2896, -3.8228, -7.5632], + [-8.5412, -5.8142, -7.1587, -1.6095, 0.4651, 0.2748, -2.0985], + [2.0833, -0.6482, -12.1692, -4.1284, -2.9765, -0.0656, -4.5114], + [5.307, 2.6957, 2.3087, 1.0478, 0.7808, -1.1519, -0.9579] + ], + [ + [1.089, 0.1872, -0.6408, -0.9897, 0.8503, 1.1019, -0.9211], + [-0.1741, -0.2915, 4.2472, 1.9417, 1.65, 0.6303, -4.7131], + [1.6555, 2.4026, -2.9293, 2.9953, 0.5328, 3.5873, -0.9621], + [-1.4289, -3.2787, 4.1747, -6.0341, -4.6341, -5.7945, 4.142], + [7.5973, 6.4431, 5.9872, 2.1639, -8.6566, 3.3143, -3.4059], + [-0.8775, -3.048, 11.6543, 0.6442, 2.3218, -0.4765, 1.1516], + [-5.5423, -2.5188, 1.0754, -0.0563, -2.9386, -1.1504, 1.0171] + ] + ] + ); + + // Dilations. + let res = t.conv2d(&w, 0, 1, 2, 1)?; + assert_eq!(res.dims(), [1, 2, 1, 1]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [2.45, -2.3504], + ); + + // Transpose and dilations. + let res = t.conv_transpose2d(&w.transpose(0, 1)?, 0, 0, 1, 2)?; + assert_eq!(res.dims(), [1, 2, 9, 9]); + assert_eq!( + test_utils::to_vec3_round(&res.i(0)?, 4)?, + [ + [ + [-1.9918, 3.1652, -0.6778, -4.3442, 4.4351, 0.6652, -3.0124, -0.6031, 2.9277], + [2.7036, -1.7156, -0.3969, 1.0516, 1.6381, -2.8886, -0.205, 2.4682, -1.0499], + [-0.9459, 3.1631, 3.707, -4.8369, -8.5166, -1.4496, -2.7559, -3.2698, 1.4376], + [-0.2157, 3.7786, -2.0252, -4.2633, 3.6731, -1.5142, 5.9391, -0.2622, -0.141], + [-6.8121, -3.1744, 1.5945, 3.0637, -9.6088, 1.4446, 2.9489, -3.0082, -7.3822], + [0.2371, 3.3303, 0.3861, 2.2646, -4.6784, 4.1235, -0.0109, 0.3176, -0.03], + [-2.5339, -2.9564, -3.4518, -4.4594, -9.1873, -1.9709, -0.4676, 0.51, -3.5024], + [4.007, 0.3067, -2.2954, 1.1105, -0.1992, 1.6372, -2.9268, 0.2807, -1.2787], + [5.307, 1.1317, 1.3518, 0.9049, 3.8116, -0.4075, -0.8874, -0.2241, -0.9579] + ], + [ + [1.089, -0.6483, 0.0726, -0.4752, -1.3283, 1.7103, 1.0703, 0.1076, -0.9211], + [-0.8629, 0.1376, 0.3202, 2.0955, 0.9696, 2.8988, -1.0012, 1.5049, -0.1278], + [1.9286, -1.5255, -2.9563, 2.4589, 3.3611, -0.6951, 0.3525, -1.7724, -5.9861], + [1.1226, 2.1561, 3.6417, 4.7546, -0.692, 4.4126, -5.1902, 6.0805, 2.3185], + [1.0111, 0.3604, 0.6432, -3.6605, 7.9517, -9.2955, -5.2988, -3.7803, -2.0642], + [3.3172, -1.7967, -3.6576, -2.0942, 1.3158, 0.112, -1.7405, 2.9167, 0.7957], + [5.1001, 1.8995, -1.8639, 1.1262, 9.9629, 2.683, -3.6319, -1.1607, 0.5856], + [-4.8445, -0.5642, 4.2317, 0.0856, 1.2267, -0.5712, 1.736, 1.0997, 0.6908], + [-5.5423, -1.1831, -1.2176, 0.0843, 0.0446, -0.7545, -2.4798, -0.0827, 1.0171] + ] + ] + ); + + Ok(()) +} + +/* This test is based on the following script. +import torch +torch.manual_seed(4242) + +t = torch.randn((1, 2, 3, 3)) +w = torch.randn((1, 2, 1, 1)) +print(t.flatten()) +print(w.flatten()) +res = torch.nn.functional.conv2d(t, w) +print(res.flatten()) + +w_t = w.transpose(0, 1) +res = torch.nn.functional.conv_transpose2d(t, w_t) +print(res.shape) +print(res.flatten()) + +t_t = w.transpose(0, 1) +res = torch.nn.functional.conv_transpose2d(t_t, w) +print(res.shape) +print(res.flatten()) +*/ +fn conv2d_small(dev: &Device) -> Result<()> { + let t = Tensor::new( + &[ + 0.4056f32, -0.8689, 0.6843, 0.2395, 1.2279, -0.9287, -1.7030, 0.1370, 0.1866, 0.4145, + -0.6266, 0.3529, 2.2013, -0.6836, 0.2477, 1.3127, -0.6957, 0.3278, + ], + dev, + )?; + let w = Tensor::new(&[-0.9259f32, 1.3017], dev)?; + let t = t.reshape((1, 2, 3, 3))?; + let w = w.reshape((1, 2, 1, 1))?; + let res = t.conv2d(&w, 0, 1, 1, 1)?; + assert_eq!(res.dims(), [1, 1, 3, 3]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [0.164, -0.0111, -0.1742, 2.6437, -2.0268, 1.1823, 3.2855, -1.0324, 0.2539] + ); + let res = t.conv2d(&w, 2, 1, 1, 1)?; + assert_eq!(res.dims(), [1, 1, 7, 7]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [ + 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.1640, + -0.0111, -0.1742, 0.0, 0.0, 0.0, 0.0, 2.6437, -2.0268, 1.1823, 0.0, 0.0, 0.0, 0.0, + 3.2855, -1.0324, 0.2539, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, + 0.0, 0.0, 0.0, 0.0 + ] + ); + + let res = t.conv_transpose2d(&w.transpose(0, 1)?, 0, 0, 1, 1)?; + assert_eq!(res.dims(), [1, 1, 3, 3]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [0.164, -0.0111, -0.1742, 2.6437, -2.0268, 1.1823, 3.2855, -1.0324, 0.2539], + ); + let res = t.transpose(0, 1)?.conv_transpose2d(&w, 0, 0, 1, 1)?; + assert_eq!(res.dims(), [2, 2, 3, 3]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [ + -0.3755, 0.8045, -0.6336, -0.2218, -1.1369, 0.8599, 1.5768, -0.1268, -0.1728, 0.528, + -1.131, 0.8908, 0.3118, 1.5984, -1.2089, -2.2168, 0.1783, 0.2429, -0.3838, 0.5802, + -0.3268, -2.0382, 0.6329, -0.2293, -1.2154, 0.6441, -0.3035, 0.5396, -0.8156, 0.4594, + 2.8654, -0.8898, 0.3224, 1.7087, -0.9056, 0.4267 + ] + ); + Ok(()) +} + +fn conv2d_smaller(dev: &Device) -> Result<()> { + let t = Tensor::new( + &[ + 0.4056f32, -0.8689, 0.6843, 0.2395, 1.2279, -0.9287, -1.7030, 0.1370, 0.1866, + ], + dev, + )?; + let w = Tensor::new(&[1f32, 1., 1., 1., 1., 1., 1., 1., 1.], dev)?; + let t = t.reshape((1, 1, 3, 3))?; + let w = w.reshape((1, 1, 3, 3))?; + let res = t.conv2d(&w, 0, 1, 1, 1)?; + assert_eq!(res.dims(), [1, 1, 1, 1]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [-0.6197] + ); + Ok(()) +} + +/* This test is based on the following script. +import torch +torch.manual_seed(4242) + +t = torch.randn((1, 2, 4, 2)) +w = torch.randn((1, 2, 1, 1)) +print(t.flatten()) +print(w.flatten()) +res = torch.nn.functional.conv2d(t, w) +print(res.flatten()) +*/ +fn conv2d_non_square(dev: &Device) -> Result<()> { + let t = Tensor::new( + &[ + 0.4056f32, -0.8689, -0.0773, -1.5630, -2.8012, -1.5059, 0.3972, 1.0852, 0.4997, 3.0616, + 1.6541, 0.0964, -0.8338, -1.6523, -0.8323, -0.1699, + ], + dev, + )?; + let w = Tensor::new(&[-1.1351f32, 1.3841], dev)?; + let t = t.reshape((1, 2, 4, 2))?; + let w = w.reshape((1, 2, 1, 1))?; + let res = t.conv2d(&w, 0, 1, 1, 1)?; + assert_eq!(res.dims(), [1, 1, 4, 2]); + assert_eq!( + test_utils::to_vec1_round(&res.flatten_all()?, 4)?, + [0.2312, 5.2238, 2.3772, 1.9076, 2.0256, -0.5776, -1.6028, -1.467] + ); + Ok(()) +} + +/* +import torch +torch.manual_seed(4242) + +t = torch.randn((1, 4, 5, 5), requires_grad=True) +w = torch.randn((2, 4, 3, 3), requires_grad=True) +print(t.flatten()) +print(w.flatten()) +res = torch.nn.functional.conv2d(t, w) +print(res.flatten()) +loss = (res ** 2).sum() +print(loss) +loss.backward() +print(t.grad.shape) +print(t.grad.flatten()) +print(w.grad.shape) +print(w.grad.flatten()) + +t.grad.zero_() +w.grad.zero_() +res = torch.nn.functional.conv2d(t, w, stride=2) +print(res.flatten()) +loss = (res ** 2).sum() +print(loss) +loss.backward() +print(t.grad.shape) +print(t.grad[0]) +print(w.grad.shape) +print(w.grad[0]) +*/ +fn conv2d_grad(dev: &Device) -> Result<()> { + // conv-transposes are not implemented for metal + use candle_core::Var; + let t = Var::from_slice( + &[ + 0.4056f32, -0.8689, -0.0773, -1.5630, -2.8012, -1.5059, 0.3972, 1.0852, 0.4997, 3.0616, + 1.6541, 0.0964, -0.8338, -1.6523, -0.8323, -0.1699, 0.0823, 0.3526, 0.6843, 0.2395, + 1.2279, -0.9287, -1.7030, 0.1370, 0.6047, 0.3770, -0.6266, 0.3529, 2.2013, -0.6836, + 0.2477, 1.3127, -0.2260, 0.2622, -1.2974, -0.8140, -0.8404, -0.3490, 0.0130, 1.3123, + 1.7569, -0.3956, -1.8255, 0.1727, -0.3538, 2.6941, 1.0529, 0.4219, -0.2071, 1.1586, + 0.4717, 0.3865, -0.5690, -0.5010, -0.1310, 0.7796, 0.6630, -0.2021, 2.6090, 0.2049, + 0.6466, -0.5042, -0.0603, -1.6538, -1.2429, 1.8357, 1.6052, -1.3844, 0.3323, -1.3712, + 0.9634, -0.4799, -0.6451, -0.0840, -1.4247, 0.5512, -0.1747, -0.5509, -0.3742, 0.3790, + -0.4431, -0.4720, -0.7890, 0.2620, 0.7875, 0.5377, -0.6779, -0.8088, 1.9098, 1.2006, + -0.8, -0.4983, 1.5480, 0.8265, -0.1025, 0.5138, 0.5748, 0.3821, -0.4607, 0.0085, + ], + (1, 4, 5, 5), + dev, + )?; + let w = Var::from_slice( + &[ + -0.9325f32, 0.6451, -0.8537, 0.2378, 0.8764, -0.1832, 0.2987, -0.6488, -0.2273, + -2.4184, -0.1192, -0.4821, -0.5079, -0.5766, -2.4729, 1.6734, 0.4558, 0.2851, 1.1514, + -0.9013, 1.0662, -0.1817, -0.0259, 0.1709, 0.5367, 0.7513, 0.8086, -2.2586, -0.5027, + 0.9141, -1.3086, -1.3343, -1.5669, -0.1657, 0.7958, 0.1432, 0.3896, -0.4501, 0.1667, + 0.0714, -0.0952, 1.2970, -0.1674, -0.3178, 1.0677, 0.3060, 0.7080, 0.1914, 1.1679, + -0.3602, 1.9265, -1.8626, -0.5112, -0.0982, 0.2621, 0.6565, 0.5908, 1.0089, -0.1646, + 1.8032, -0.6286, 0.2016, -0.3370, 1.2555, 0.8009, -0.6488, -0.4652, -1.5685, 1.5860, + 0.5583, 0.4623, 0.6026, + ], + (2, 4, 3, 3), + dev, + )?; + let res = t.conv2d(&w, 0, 1, 1, 1)?; + let loss = res.sqr()?.sum_all()?; + assert_eq!(test_utils::to_vec0_round(&loss, 2)?, 741.12f32); + let grads = loss.backward()?; + let grad_t = grads.get(&t).unwrap(); + let grad_w = grads.get(&w).unwrap(); + assert_eq!(grad_t.dims(), [1, 4, 5, 5]); + assert_eq!(grad_w.dims(), [2, 4, 3, 3]); + assert_eq!( + test_utils::to_vec1_round(&grad_t.flatten_all()?, 2)?, + [ + 9.29, -2.84, -5.71, 3.38, -7.71, -19.15, 7.02, 29.1, 9.34, 34.73, -22.87, 24.35, + -39.88, -14.01, 21.08, 9.94, 13.63, -34.68, 11.21, -6.26, 7.72, -6.32, -16.64, -1.08, + -20.22, 21.73, -0.37, -4.06, 5.82, -3.65, -30.73, 14.55, 87.7, 31.6, 4.53, -89.78, + -75.37, -57.43, -7.56, 92.96, 18.79, -4.63, -159.75, -42.47, -47.26, 52.88, 37.32, + 49.0, 12.82, 2.01, -8.98, 20.18, 16.62, 12.06, 15.38, 20.0, 2.57, -15.22, 72.62, + -10.75, 2.25, -31.2, 3.75, -0.2, 9.76, -0.68, 5.21, -40.44, -22.59, -61.61, 17.28, + 20.41, 37.55, 5.23, 6.81, 23.54, 23.62, -9.99, -9.13, 4.87, -35.06, -26.1, 63.48, + 25.81, -39.21, -70.68, -46.96, 2.33, 41.81, 82.42, -28.63, -11.78, -35.33, -10.28, + -28.57, -9.13, 7.21, -9.05, -9.62, -11.25 + ] + ); + assert_eq!( + test_utils::to_vec1_round(&grad_w.flatten_all()?, 2)?, + [ + -28.92, -22.88, -141.23, 73.35, 61.07, 47.81, -20.0, -73.71, -41.82, -13.59, 21.5, + 28.72, 28.57, -46.85, -90.19, 143.61, 16.68, 7.43, 18.88, -90.81, -20.29, 54.79, 82.63, + 22.94, 77.81, -16.39, -13.2, 9.34, -40.39, -26.62, 5.33, -60.91, 9.09, -59.37, 7.08, + 58.64, 5.55, 20.52, 2.5, -17.25, -6.8, 22.21, 30.15, -7.52, -37.46, 5.67, 22.58, 9.03, + 47.05, 17.61, 37.31, -98.13, -14.61, -4.8, -6.36, 44.69, 23.34, 8.37, -13.52, 80.05, + -34.24, -16.36, -12.31, 1.92, -33.62, -14.1, -49.23, -7.39, 11.5, -9.98, 9.66, 29.6 + ] + ); + + // Same as before but with stride. + let res = t.conv2d(&w, 0, 2, 1, 1)?; + let loss = res.sqr()?.sum_all()?; + assert_eq!(test_utils::to_vec0_round(&loss, 2)?, 277.16f32); + let grads = loss.backward()?; + let grad_t = grads.get(&t).unwrap(); + let grad_w = grads.get(&w).unwrap(); + assert_eq!(grad_t.dims(), [1, 4, 5, 5]); + assert_eq!(grad_w.dims(), [2, 4, 3, 3]); + assert_eq!( + test_utils::to_vec3_round(&grad_t.i(0)?, 2)?, + [ + [ + [9.29, -7.03, 0.94, 3.49, -7.71], + [-1.8, -7.82, 8.9, 8.46, 7.43], + [-25.84, 22.09, -19.27, -0.22, 1.69], + [4.02, 18.53, -18.37, 2.3, -24.51], + [7.72, -9.68, -12.34, 5.6, -20.22] + ], + [ + [21.73, 3.39, -18.27, 3.86, -3.65], + [8.25, 3.73, 30.73, -8.61, -11.93], + [-72.15, -15.36, -17.53, -12.32, -1.61], + [-22.32, -7.79, -91.82, 6.44, -37.69], + [52.88, 14.44, 42.75, 9.88, 2.01] + ], + [ + [-8.98, 9.91, 6.75, -4.68, 15.38], + [4.93, -0.33, 9.94, -1.46, 14.78], + [13.62, -30.63, 3.96, -3.58, -4.48], + [-14.13, 1.19, -34.43, 3.08, -33.83], + [17.28, 12.94, 31.83, -3.35, 6.81] + ], + [ + [23.54, 6.98, -24.52, 0.52, 4.87], + [9.65, 6.18, 1.71, -25.23, -4.93], + [-54.99, -23.66, 3.19, -3.73, 18.58], + [-21.35, -10.39, -39.88, 28.73, -30.76], + [-9.13, 11.12, -14.0, -8.23, -11.25] + ] + ] + ); + assert_eq!( + test_utils::to_vec3_round(&grad_w.i(0)?, 2)?, + [ + [ + [28.34, -7.91, -45.75], + [21.03, 3.86, 29.86], + [0.72, -36.58, -35.28] + ], + [ + [-16.04, 11.53, -16.38], + [29.62, -16.32, -48.35], + [57.5, 28.29, 25.81] + ], + [ + [2.93, -19.6, 1.57], + [27.15, 53.88, -24.64], + [12.74, -22.6, -26.2] + ], + [ + [-0.18, -14.86, -6.82], + [-19.55, -2.72, 45.9], + [-2.54, 36.97, 27.11] + ] + ] + ); + + // Replicate the issue from https://github.com/huggingface/candle/issues/1212 + let res = t.i((.., .., 0..4, 0..4))?.conv2d(&w, 0, 2, 1, 1)?; + let loss = res.sqr()?.sum_all()?; + assert_eq!(test_utils::to_vec0_round(&loss, 2)?, 21.12f32); + let grads = loss.backward()?; + let grad_t = grads.get(&t).unwrap(); + let grad_w = grads.get(&w).unwrap(); + assert_eq!(grad_t.dims(), [1, 4, 5, 5]); + assert_eq!(grad_w.dims(), [2, 4, 3, 3]); + assert_eq!( + test_utils::to_vec3_round(&grad_t.i(0)?, 2)?, + [ + [ + [9.29, -7.03, 7.87, 0.0, 0.0], + [-1.8, -7.82, 5.9, 0.0, 0.0], + [-3.12, 4.49, 5.52, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0] + ], + [ + [21.73, 3.39, 4.77, 0.0, 0.0], + [8.25, 3.73, 27.61, 0.0, 0.0], + [-20.55, -5.61, -2.77, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0] + ], + [ + [-8.98, 9.91, -7.15, 0.0, 0.0], + [4.93, -0.33, 4.56, 0.0, 0.0], + [-6.7, -5.76, -8.05, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0] + ], + [ + [23.54, 6.98, -10.0, 0.0, 0.0], + [9.65, 6.18, 18.72, 0.0, 0.0], + [3.29, -5.27, 0.79, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0], + [0.0, 0.0, 0.0, 0.0, 0.0] + ] + ] + ); + assert_eq!( + test_utils::to_vec3_round(&grad_w.i(0)?, 2)?, + [ + [ + [-3.47, 7.44, 0.66], + [12.89, -3.4, -9.29], + [-14.16, -0.83, 7.14] + ], + [ + [-3.23, 5.37, -3.02], + [-2.12, -11.24, 1.94], + [6.97, 7.2, 2.99] + ], + [ + [-4.04, -3.31, 4.87], + [-6.68, -5.68, 1.73], + [-5.54, 4.32, 0.52] + ], + [[-4.72, 1.5, 4.72], [3.79, 4.04, 6.76], [-4.6, 5.8, 6.93]] + ] + ); + + // Conv Transpose 2d Test + //tested against following python + + // import torch + // torch.manual_seed(4242) + // padding = 4 + // outpadding = 2 + // dilation = 3 + // stride = 3 + // input = torch.randn((1, 4, 7, 5), requires_grad=True) + // kernel = torch.randn((4, 2, 3, 5), requires_grad=True) + // print("input", input.flatten()) + // print("kernel", kernel.flatten()) + // res = torch.nn.functional.conv_transpose2d( + // input, + // kernel, + // stride=stride, + // padding=padding, + // dilation=dilation, + // output_padding=outpadding, + // ) + // res.retain_grad() + // print(res.shape) + // loss = (res**2).sum() + // print(loss) + // loss.backward() + // print(input.grad.shape) + // print("input grad", torch.round(input.grad, decimals=1)) + // print(kernel.grad.shape) + // print("kernel grad", torch.round(kernel.grad.flatten(), decimals=1)) + + let padding = 4; + let outpadding = 2; + let dilation = 3; + let stride = 3; + + let t = Var::from_slice( + &[ + 0.4056_f32, -0.8689, -0.0773, -1.5630, -2.8012, -1.5059, 0.3972, 1.0852, 0.4997, + 3.0616, 1.6541, 0.0964, -0.8338, -1.6523, -0.8323, -0.1699, 0.0823, 0.3526, 0.6843, + 0.2395, 1.2279, -0.9287, -1.7030, 0.1370, 0.6047, 0.3770, -0.6266, 0.3529, 2.2013, + -0.6836, 0.2477, 1.3127, -0.2260, 0.2622, -1.2974, -0.8140, -0.8404, -0.3490, 0.0130, + 1.3123, 1.7569, -0.3956, -1.8255, 0.1727, -0.3538, 2.6941, 1.0529, 0.4219, -0.2071, + 1.1586, 0.4717, 0.3865, -0.5690, -0.5010, -0.1310, 0.7796, 0.6630, -0.2021, 2.6090, + 0.2049, 0.6466, -0.5042, -0.0603, -1.6538, -1.2429, 1.8357, 1.6052, -1.3844, 0.3323, + -1.3712, 0.9634, -0.4799, -0.6451, -0.0840, -1.4247, 0.5512, -0.1747, -0.5509, -0.3742, + 0.3790, -0.4431, -0.4720, -0.7890, 0.2620, 0.5411, -1.1715, -2.4997, 2.3249, -0.8912, + -0.4733, -0.5701, -2.8888, -1.4112, -0.5471, -0.9234, -1.1660, 0.4189, -0.7465, + -0.6473, 0.1402, 0.7875, 0.5377, -0.6779, -0.8088, -0.4864, -0.2312, 0.9279, 0.1264, + 1.5480, 0.8265, -0.1025, 0.5138, -0.2512, 0.1576, 1.2705, 0.3641, -0.9325, 0.6451, + -0.8537, 0.2378, 0.1794, 0.2752, -0.3687, -1.1149, -0.1410, -0.5829, -0.0892, 1.4258, + -2.2789, 0.5270, 0.1825, 1.7007, -0.5263, -0.2954, 0.4440, 0.5537, 0.3492, 0.6186, + 1.6475, 0.2219, + ], + (1, 4, 7, 5), + dev, + )?; + + #[rustfmt::skip] + let w = Var::from_slice( + &[ + -1.1744_f32, 0.3266, 2.5893, 1.0142, 0.1763, 0.7752, 0.6604, 0.2029, -0.2145, 0.7234, + -0.3441, -1.5400, -0.6333, 0.6613, 0.2083, 0.6230, -1.7002, 0.3393, 0.4049, 1.0762, + 0.2723, 1.4181, 0.0029, -0.2122, 1.7668, 1.4168, 0.3320, -0.2719, 0.7932, -0.7204, + 0.4447, 0.1211, 0.5908, 1.0089, -0.1646, 1.8033, -0.6286, 0.2016, -0.3370, 1.2555, + 0.8009, -0.6488, -0.4652, -1.5685, 1.5860, 0.5583, 0.4623, 0.6026, 0.8828, 2.4990, + 0.6811, -0.3369, 1.3320, 1.7669, -1.1067, 1.2958, -0.9415, -0.9655, -0.4462, 0.7181, + 0.5181, -1.1658, -1.8467, -0.7763, 1.2769, 0.8651, 0.9890, 1.5092, 0.7207, -0.8481, + 0.7417, 0.3375, -1.2685, 1.4572, 1.0915, 0.1093, -0.8550, -0.5831, -0.6309, -0.2509, + 0.5220, -0.0914, 0.7900, 0.1096, 0.3258, 0.2723, -1.0942, -0.3393, -0.1653, 0.5732, + -0.8014, 1.8194, -1.9023, 0.2127, 1.8636, -0.8979, 0.1927, -0.2778, 0.3105, 0.0071, + -1.1823, 0.2476, -0.7178, -1.3821, 1.0769, -0.4376, -0.9967, -0.1227, 1.6197, -1.0604, + 0.1372, 0.8141, -0.6163, 0.7304, -0.8285, 2.0636, -0.7176, 0.2495, -0.2581, -0.4478, + ], + (4, 2, 3, 5), + dev, + )?; + let res = t.conv_transpose2d(&w, padding, outpadding, stride, dilation)?; + let loss = res.sqr()?.sum_all()?; + assert_eq!(test_utils::to_vec0_round(&loss, 0)?, 2904.0); + let grads = loss.backward()?; + + let grad_t = grads.get(&t).unwrap(); + let grad_w = grads.get(&w).unwrap(); + assert_eq!(grad_t.dims(), [1, 4, 7, 5]); + assert_eq!(grad_w.dims(), [4, 2, 3, 5]); + + assert_eq!( + test_utils::to_vec1_round(&grad_w.flatten_all()?, 1)?, + [ + // torch gets 89.1 + -89.0, -135.3, 136.7, 102.0, -53.4, 117.9, 118.6, -43.9, -218.0, -58.5, -114.3, -150.0, + -15.6, 172.1, 66.3, -64.3, -27.9, -19.8, 31.7, 62.1, 5.5, 92.6, 28.2, -29.6, 55.9, + 52.7, -72.7, -119.8, 53.8, -25.5, 128.8, 19.3, 68.0, 190.9, -64.1, -86.2, -111.2, + 106.6, -67.7, 37.8, 115.9, 50.4, -77.7, -54.9, 22.3, -4.6, 89.8, 61.7, 122.4, 192.6, + -27.8, -104.6, 57.0, 166.4, 27.1, 6.1, 18.7, -93.2, 31.5, 168.2, -3.7, -99.5, -55.5, + -10.8, 17.5, 20.8, 16.9, 43.8, 42.0, -89.2, 18.8, -9.6, -84.1, 212.6, 19.7, -50.0, + -52.0, -40.0, -166.6, -73.2, -10.8, -73.3, 31.5, -23.4, -79.3, -27.0, -84.4, -42.9, + -20.3, 51.8, -16.7, 76.3, -120.5, -65.8, 96.5, -10.7, -45.9, -88.1, 65.4, -7.0, -1.5, + 92.8, -25.1, -114.2, -5.8, -14.8, -51.2, -20.7, 54.2, -79.8, 47.7, -29.2, -8.8, 53.5, + -28.4, 85.0, -18.3, 107.0, 28.3, -71.8 + ] + ); + + assert_eq!( + test_utils::to_vec3_round(&grad_t.i(0)?, 1)?, + [ + [ + [32.3, -41.6, -24.0, 14.1, 17.6], + [-11.8, 72.5, 87.6, 46.4, 61.5], + [115.0, 108.5, -48.6, -63.4, -50.0], + [51.3, 5.4, 31.3, 91.1, -30.9], + [52.7, 92.8, -68.0, -47.0, 83.0], + // pytorch gets -107.1 + [-10.2, -107.0, -5.4, 213.1, -31.4], + [-2.4, 65.1, 9.2, -146.2, -24.2] + ], + [ + [-72.6, -63.9, -61.9, 45.3, 33.0], + [79.3, -0.5, -26.2, 78.2, 42.7], + [90.9, 141.6, 40.1, -62.7, 37.0], + [32.8, 198.2, -0.8, -31.1, 27.3], + // torch gets 48.0 + [34.5, 34.9, -47.9, 127.6, -12.3], + [-61.4, -3.2, -2.9, -10.9, -16.6], + [74.6, 60.1, -68.9, 34.5, -50.4] + ], + [ + [37.5, -56.9, -43.6, -13.5, -9.9], + [40.0, 97.3, 28.6, 14.2, -30.1], + [-22.3, -126.3, -68.8, -8.2, 26.1], + [-32.9, 37.3, 108.5, -54.8, 29.6], + [34.9, -176.9, -125.0, -28.3, -13.9], + [-54.9, 142.6, 62.1, -80.4, -65.6], + [7.4, -91.1, -67.6, 35.0, 39.7] + ], + [ + [-57.2, -40.9, -10.1, 32.6, 29.4], + [18.7, -18.0, 29.5, -1.2, 59.2], + [-14.0, -74.4, 19.8, -117.0, 58.2], + [-21.8, 163.5, -71.1, -99.0, 80.9], + [-58.9, -10.9, 93.8, -139.6, 98.0], + // torch gets 54.5 + [-54.4, 135.3, 6.0, -79.1, 134.6], + [27.5, -76.0, 43.4, -2.8, -7.8] + ] + ] + ); + + // Test the same, but then with the following properties, t & w are unmodified. + let padding = 1; + let outpadding = 1; + let dilation = 1; + let stride = 2; + + let res = t.conv_transpose2d(&w, padding, outpadding, stride, dilation)?; + let loss = res.sqr()?.sum_all()?; + assert_eq!(test_utils::to_vec0_round(&loss, 0)?, 3627.0); // torch gives 3626.8560 + + let grads = loss.backward()?; + + let grad_t = grads.get(&t).unwrap(); + let grad_w = grads.get(&w).unwrap(); + assert_eq!(grad_t.dims(), [1, 4, 7, 5]); + assert_eq!(grad_w.dims(), [4, 2, 3, 5]); + + #[rustfmt::skip] + assert_eq!( + test_utils::to_vec3_round(&grad_t.i(0)?, 1)?, + [ + [ + [ 13.2, -40.7, -9.7, -47.3, -82.7], + [ -98.2, 9.7, 57.7, -6.2, 180.7], + [ 100.2, 24.1, 3.7, -100.5, -48.1], + [ -0.3, 13.5, -2.9, 80.0, -49.8], + [ 47.2, -25.6, -74.4, 61.2, -18.4], + [ 4.6, -69.5, 27.9, 66.5, -88.1], + // 4th column on next row; torch is 4.2 + [ -12.0, 79.2, -40.0, 4.1, -97.1], + ], + [ + [ -42.2, -36.5, -51.1, 7.5, 32.3], + [ 74.1, -44.6, -68.8, 19.5, 7.7], + [ 137.1, 54.2, 153.8, -58.0, 45.5], + [ 24.4, -56.8, 9.7, -41.0, -14.5], + [ -3.7, 72.6, 8.3, 134.8, 40.5], + [ 43.2, -56.9, -47.5, -89.4, -95.4], + [ 68.2, 108.1, -80.0, 57.0, -121.1] + ], + [ + [ 31.1, -11.4, -34.8, 33.1, -44.2], + [ 29.4, -31.6, -40.2, 13.7, 13.1], + [ -0.8, -83.8, -7.8, -17.3, 78.2], + [ 12.0, -118.7, 137.5, -76.7, 50.8], + [ -28.7, -114.2, -3.7, -96.3, -13.8], + [ -31.8, 28.5, -14.3, 4.6, 13.4], + [ 28.0, -0.2, -38.9, -29.7, -59.0] + ], + [ + [ -16.8, 38.5, 15.5, 26.6, 48.9], + [ 14.5, 49.6, -24.8, 65.6, 61.7], + [ 22.1, -64.7, -4.3, -51.0, 36.3], + [ 31.0, -88.9, 47.1, -123.5, -3.8], + [ -14.8, -39.8, 128.2, -110.3, 42.6], + // 1st column on next row; torch is -7.2 + [ -7.1, 95.3, -21.3, -58.7, -13.9], + [ 26.9, 21.3, 16.1, 70.3, 32.1] + ] + ] + ); + + #[rustfmt::skip] + assert_eq!( + test_utils::to_vec1_round(&grad_w.flatten_all()?, 1)?, + [ + // 2nd value; torch gets -3.2, 3rd value; torch gets 221.8 + -2.460e+01, -3.100e+00, 2.219e+02, 7.400e+00, 5.620e+01, + 7.420e+01, 7.830e+01, 8.900e+00, 1.050e+01, 2.810e+01, + 5.100e+00, -1.046e+02, -1.572e+02, 8.710e+01, -9.840e+01, + -4.230e+01, -1.898e+02, 1.860e+01, -3.570e+01, 9.810e+01, + 4.680e+01, 1.182e+02, 4.020e+01, -1.900e+00, 1.508e+02, + 1.094e+02, 1.018e+02, -4.620e+01, 1.591e+02, -2.320e+01, + // 5th value; torch gets 7.1 + -8.450e+01, -4.600e+00, 6.330e+01, 1.123e+02, -7.000e+00, + 1.101e+02, -6.620e+01, 2.090e+01, -5.120e+01, 8.990e+01, + 9.050e+01, -6.990e+01, 6.800e+01, -9.250e+01, 1.380e+02, + 4.720e+01, 4.710e+01, 6.210e+01, 8.870e+01, 2.098e+02, + 3.870e+01, -1.390e+01, 6.270e+01, 1.484e+02, -9.920e+01, + -4.200e+01, -1.505e+02, -1.480e+01, -2.620e+01, 8.220e+01, + -3.350e+01, -2.260e+01, -1.198e+02, -5.080e+01, 1.259e+02, + 5.600e+01, 9.270e+01, 1.209e+02, 6.590e+01, -8.330e+01, + 7.000e+00, -2.600e+01, -1.133e+02, 3.870e+01, 4.020e+01, + -6.300e+00, -8.710e+01, -5.150e+01, -8.510e+01, 2.000e-01, + 3.640e+01, -6.100e+00, 6.590e+01, -2.700e+00, 6.550e+01, + // 4th value; torch gets 3.8 + 5.300e+00, -6.760e+01, -4.270e+01, -3.900e+00, 2.880e+01, + 5.260e+01, 6.170e+01, -1.203e+02, -1.610e+01, 7.740e+01, + -1.008e+02, -1.070e+01, -9.900e+00, 3.300e+00, -2.620e+01, + -4.440e+01, 2.580e+01, -6.920e+01, -4.220e+01, 1.108e+02, + 1.240e+01, -3.440e+01, -2.800e+00, 7.880e+01, -6.690e+01, + 1.480e+01, 2.310e+01, -4.260e+01, -1.500e+00, -4.760e+01, + 5.350e+01, -2.260e+01, 8.000e-01, -3.840e+01, -2.500e+00 + ] + ); + + Ok(()) +} + +test_device!(conv1d, conv1d_cpu, conv1d_gpu, conv1d_metal); +test_device!( + conv1d_small, + conv1d_small_cpu, + conv1d_small_gpu, + conv1d_small_metal +); +test_device!(conv2d, conv2d_cpu, conv2d_gpu, conv2d_metal); +test_device!( + conv2d_non_square, + conv2d_non_square_cpu, + conv2d_non_square_gpu, + conv2d_non_square_metal +); +test_device!( + conv2d_small, + conv2d_small_cpu, + conv2d_small_gpu, + conv2d_small_metal +); +test_device!( + conv2d_smaller, + conv2d_smaller_cpu, + conv2d_smaller_gpu, + conv2d_smaller_metal +); +test_device!( + conv2d_grad, + conv2d_grad_cpu, + conv2d_grad_gpu, + conv2_grad_metal +); diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/custom_op_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/custom_op_tests.rs new file mode 100644 index 0000000000..cea9b90cac --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/tests/custom_op_tests.rs @@ -0,0 +1,182 @@ +use candle_core::backend::BackendStorage; +use candle_core::cpu_backend; +use candle_core::test_utils::to_vec1_round; +use candle_core::{CpuStorage, CustomOp1, DType, Device, Error, Layout, Result, Shape, Tensor}; + +fn fwd(v: T, alpha: f64) -> T { + if v.is_sign_positive() { + v + } else { + let alpha = T::from(alpha).unwrap_or(T::nan()); + (v.exp() - T::one()) * alpha + } +} + +struct Elu { + alpha: f64, +} + +impl CustomOp1 for Elu { + fn name(&self) -> &'static str { + "elu" + } + + fn cpu_fwd(&self, s: &CpuStorage, l: &Layout) -> Result<(CpuStorage, Shape)> { + let storage = candle_core::map_dtype!( + "elu", + s, + |s| cpu_backend::unary_map(s, l, |v| fwd(v, self.alpha)), + (F8E4M3, BF16, F16, F32, F64) + ); + Ok((storage, l.shape().clone())) + } +} + +#[test] +fn custom_op1_no_backward() -> Result<()> { + let cpu = &Device::Cpu; + let t = Tensor::arange(0u32, 12u32, cpu)?.to_dtype(DType::F32)?; + let t = (t - 5.)?; + let elu_t = t.apply_op1_no_bwd(&Elu { alpha: 1. })?; + assert_eq!( + to_vec1_round(&elu_t, 4)?, + &[-0.9933, -0.9817, -0.9502, -0.8647, -0.6321, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0] + ); + Ok(()) +} + +// Define a similar struct as Elu but with backward support. +fn bwd(v: T, alpha: f64) -> T { + if v.is_sign_positive() { + T::one() + } else { + let alpha = T::from(alpha).unwrap_or(T::nan()); + v.exp() * alpha + } +} + +struct EluBackward { + alpha: f64, +} + +impl CustomOp1 for EluBackward { + fn name(&self) -> &'static str { + "elu-bwd" + } + + fn cpu_fwd(&self, s: &CpuStorage, l: &Layout) -> Result<(CpuStorage, Shape)> { + let storage = candle_core::map_dtype!( + "elu-bwd", + s, + |s| cpu_backend::unary_map(s, l, |v| bwd(v, self.alpha)), + (F8E4M3, BF16, F16, F32, F64) + ); + Ok((storage, l.shape().clone())) + } +} + +struct EluWithBackward(Elu); + +impl EluWithBackward { + fn new(alpha: f64) -> Self { + Self(Elu { alpha }) + } +} + +impl CustomOp1 for EluWithBackward { + fn name(&self) -> &'static str { + "elu" + } + + fn cpu_fwd(&self, s: &CpuStorage, l: &Layout) -> Result<(CpuStorage, Shape)> { + self.0.cpu_fwd(s, l) + } + + fn bwd(&self, arg: &Tensor, _res: &Tensor, grad_res: &Tensor) -> Result> { + let alpha = self.0.alpha; + let bwd = arg.apply_op1(EluBackward { alpha })?; + Ok(Some(grad_res.mul(&bwd)?)) + } +} + +#[test] +fn custom_op1_with_backward() -> Result<()> { + let cpu = &Device::Cpu; + let t = candle_core::Var::new(&[-2f32, 0f32, 2f32], cpu)?; + let elu_t = t.apply_op1(EluWithBackward::new(2.))?; + assert_eq!(to_vec1_round(&elu_t, 4)?, &[-1.7293, 0.0, 2.0]); + + let grads = elu_t.backward()?; + let grad_x = grads.get(&t).unwrap(); + assert_eq!(to_vec1_round(grad_x, 4)?, [0.2707, 1.0, 1.0]); + + Ok(()) +} + +impl candle_core::InplaceOp1 for Elu { + fn name(&self) -> &'static str { + "elu" + } + + fn cpu_fwd(&self, s: &mut CpuStorage, _l: &Layout) -> Result<()> { + let alpha = self.alpha; + match s { + CpuStorage::F8E4M3(s) => s.iter_mut().for_each(|v| *v = fwd(*v, alpha)), + CpuStorage::BF16(s) => s.iter_mut().for_each(|v| *v = fwd(*v, alpha)), + CpuStorage::F16(s) => s.iter_mut().for_each(|v| *v = fwd(*v, alpha)), + CpuStorage::F32(s) => s.iter_mut().for_each(|v| *v = fwd(*v, alpha)), + CpuStorage::F64(s) => s.iter_mut().for_each(|v| *v = fwd(*v, alpha)), + _ => candle_core::bail!("unsupported dtype for inplace elu"), + } + Ok(()) + } +} + +#[test] +fn inplace_op1() -> Result<()> { + let cpu = &Device::Cpu; + let t = Tensor::arange(0u32, 12u32, cpu)?.to_dtype(DType::F32)?; + let t = (t - 5.)?; + t.inplace_op1(&Elu { alpha: 1. })?; + assert_eq!( + to_vec1_round(&t, 4)?, + &[-0.9933, -0.9817, -0.9502, -0.8647, -0.6321, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0] + ); + Ok(()) +} + +#[cfg(all(feature = "ug", any(feature = "cuda", feature = "metal")))] +#[allow(clippy::approx_constant)] +#[test] +fn ug_op() -> Result<()> { + let kernel = { + use candle_ug::lang::op; + + let layout = candle_ug::Layout::from_shape(&[12]); + let ptr = op::Arg::ptr(candle_ug::DType::F32); + let src = op::load(ptr.id(), layout.clone(), candle_ug::DType::F32)?; + let src = op::unary(op::UnaryOp::Exp, src)?; + let st = op::store(ptr.id(), layout, src)?; + let kernel = op::Kernel::new("exp".to_string(), vec![ptr], vec![st]); + let opts: candle_ug::lower_op::Opts = Default::default(); + kernel.lower(&opts)? + }; + let device = if candle_core::utils::cuda_is_available() { + Device::new_cuda(0)? + } else if candle_core::utils::metal_is_available() { + Device::new_metal(0)? + } else { + candle_core::bail!("metal/cuda is mandatory for this test") + }; + let op = candle_core::UgIOp1::new("test", kernel, &device)?; + let t = Tensor::arange(0u32, 12u32, &device)?.to_dtype(DType::F32)?; + t.inplace_op1(&op)?; + assert_eq!( + to_vec1_round(&t, 2)?, + &[ + 1.0, 2.72, 7.39, 20.09, 54.6, 148.41, 403.43, 1096.63, 2980.96, 8103.08, 22026.47, + 59874.13 + ] + ); + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/display_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/display_tests.rs new file mode 100644 index 0000000000..8ab8bb6ab7 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/tests/display_tests.rs @@ -0,0 +1,84 @@ +use anyhow::Result; +use candle_core::{DType, Device::Cpu, Tensor}; + +#[test] +fn display_scalar() -> Result<()> { + let t = Tensor::new(1234u32, &Cpu)?; + let s = format!("{t}"); + assert_eq!(&s, "[1234]\nTensor[[], u32]"); + let t = t.to_dtype(DType::F32)?.neg()?; + let s = format!("{}", (&t / 10.0)?); + assert_eq!(&s, "[-123.4000]\nTensor[[], f32]"); + let s = format!("{}", (&t / 1e8)?); + assert_eq!(&s, "[-1.2340e-5]\nTensor[[], f32]"); + let s = format!("{}", (&t * 1e8)?); + assert_eq!(&s, "[-1.2340e11]\nTensor[[], f32]"); + let s = format!("{}", (&t * 0.)?); + assert_eq!(&s, "[0.]\nTensor[[], f32]"); + Ok(()) +} + +#[test] +fn display_vector() -> Result<()> { + let t = Tensor::new::<&[u32; 0]>(&[], &Cpu)?; + let s = format!("{t}"); + assert_eq!(&s, "[]\nTensor[[0], u32]"); + let t = Tensor::new(&[0.1234567, 1.0, -1.2, 4.1, f64::NAN], &Cpu)?; + let s = format!("{t}"); + assert_eq!( + &s, + "[ 0.1235, 1.0000, -1.2000, 4.1000, NaN]\nTensor[[5], f64]" + ); + let t = (Tensor::ones(50, DType::F32, &Cpu)? * 42.)?; + let s = format!("\n{t}"); + let expected = r#" +[42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., + 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., + 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., + 42., 42.] +Tensor[[50], f32]"#; + assert_eq!(&s, expected); + let t = (Tensor::ones(11000, DType::F32, &Cpu)? * 42.)?; + let s = format!("{t}"); + assert_eq!( + &s, + "[42., 42., 42., ..., 42., 42., 42.]\nTensor[[11000], f32]" + ); + Ok(()) +} + +#[test] +fn display_multi_dim() -> Result<()> { + let t = (Tensor::ones((200, 100), DType::F32, &Cpu)? * 42.)?; + let s = format!("\n{t}"); + let expected = r#" +[[42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + ... + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.]] +Tensor[[200, 100], f32]"#; + assert_eq!(&s, expected); + let t = t.reshape(&[2, 1, 1, 100, 100])?; + let t = format!("\n{t}"); + let expected = r#" +[[[[[42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + ... + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.]]]], + [[[[42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + ... + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.]]]]] +Tensor[[2, 1, 1, 100, 100], f32]"#; + assert_eq!(&t, expected); + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/fortran_tensor_3d.pth b/v2/dependency-patches/candle-core-0.9.2/tests/fortran_tensor_3d.pth new file mode 100644 index 0000000000000000000000000000000000000000..bd50b03d9cd74135fa6f53e2bf39da2f7ac0879d GIT binary patch literal 1486 zcmbVMON-M`6uxcRw3C_f{b*;vl}H8cq_1fM4rq~Vu3&5tikp&2uAN|L<|elT3Zfes z`~xoi8LnNr)ZgHra3gq5ax?akg5!b1*QDoseErTjwWv}cq+BK|u|vw_We|$co{j~Z zMnO1kxqGfH?E3L5-|QF23yAmmx15YFK)0-sEj;IlAasuEF|D;O?2<9xF#&?}#<-dQ*_3!wA}nJZmn5c(c}+fVNvudj z=N{boHv9_DnCqRZ5+lR86Bb6WEM~uRZtLQCw=`7nI)k}hL5W)3v?BD(;LAu`VHkz$&w_)rOI+dOSykWlHOd3LLTKWc|6b;pN%gI@N0_} N Result<()> { + let x = Var::new(&[3f32, 1., 4.], device)?; + let x = x.as_tensor(); + let y = (((x * x)? + x * 5f64)? + 4f64)?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!(x.to_vec1::()?, [3., 1., 4.]); + // y = x^2 + 5.x + 4 + assert_eq!(y.to_vec1::()?, [28., 10., 40.]); + // dy/dx = 2.x + 5 + assert_eq!(grad_x.to_vec1::()?, [11., 7., 13.]); + Ok(()) +} + +fn sum_grad(device: &Device) -> Result<()> { + let x = Var::new(&[3f32, 1., 4.], device)?; + let x = x.as_tensor(); + let y = (x.sqr()?.sum_keepdim(0)? * 2.)?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!(y.to_vec1::()?, [52.]); + // y = 2.x^2 so dy/dx = 4.x + assert_eq!(grad_x.to_vec1::()?, &[12., 4., 16.]); + + // Same test as before but squeezing on the last dimension. + let y = (x.sqr()?.sum_keepdim(0)? * 2.)?.squeeze(0)?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!(y.to_scalar::()?, 52.); + // y = 2.x^2 so dy/dx = 4.x + assert_eq!(grad_x.to_vec1::()?, &[12., 4., 16.]); + Ok(()) +} + +fn matmul_grad(device: &Device) -> Result<()> { + let data: Vec<_> = (0..12).map(|i| i as f32).collect(); + let x = Var::from_slice(&data, (2, 2, 3), device)?; + let data: Vec<_> = (0..12).map(|i| i as f32).collect(); + let y = Var::from_slice(&data, (2, 3, 2), device)?; + let c = x.matmul(&y)?; + let grads = c.backward()?; + let grad_x = grads.get(&x).context("no grad for x")?; + let grad_y = grads.get(&y).context("no grad for y")?; + assert_eq!(grad_x.shape(), &Shape::from((2, 2, 3))); + assert_eq!(grad_y.shape(), &Shape::from((2, 3, 2))); + assert_eq!( + &*grad_x.to_vec3::()?, + &[ + [[1., 5., 9.], [1., 5., 9.]], + [[13., 17., 21.], [13., 17., 21.]] + ] + ); + assert_eq!( + &*grad_y.to_vec3::()?, + &[ + [[3., 3.], [5., 5.], [7., 7.]], + [[15., 15.], [17., 17.], [19., 19.]] + ] + ); + Ok(()) +} + +// The simplest gradient descent, using scalar variable. +fn grad_descent(device: &Device) -> Result<()> { + let x = Var::new(0f32, device)?; + let learning_rate = 0.1; + for _step in 0..100 { + let xt = x.as_tensor(); + let c = ((xt - 4.2)? * (xt - 4.2)?)?; + let grads = c.backward()?; + let x_grad = grads.get(&x).context("no grad for x")?; + x.set(&(xt - x_grad * learning_rate)?)? + } + assert_eq!(x.to_scalar::()?, 4.199999); + Ok(()) +} + +fn unary_grad(device: &Device) -> Result<()> { + let x = Var::new(&[3f32, 1., 4., 0.15], device)?; + let x = x.as_tensor(); + let y = (x.log()? + 1.)?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!( + test_utils::to_vec1_round(&y, 4)?, + [2.0986, 1.0, 2.3863, -0.8971] + ); + assert_eq!( + test_utils::to_vec1_round(grad_x, 4)?, + [0.3333, 1.0, 0.25, 6.6667] + ); + let y = x.exp()?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!( + test_utils::to_vec1_round(&y, 4)?, + [20.0855, 2.7183, 54.5982, 1.1618] + ); + assert_eq!( + test_utils::to_vec1_round(grad_x, 4)?, + [20.0855, 2.7183, 54.5982, 1.1618] + ); + let y = x.exp()?.sqr()?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!( + test_utils::to_vec1_round(&y, 3)?, + [403.429, 7.389, 2980.958, 1.35] + ); + // exp(x)^2 = exp(2*x) + assert_eq!( + test_utils::to_vec1_round(grad_x, 2)?, + [806.86, 14.78, 5961.92, 2.7] + ); + let y = x.sin()?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!( + test_utils::to_vec1_round(&y, 4)?, + [0.1411, 0.8415, -0.7568, 0.1494], + ); + assert_eq!( + test_utils::to_vec1_round(grad_x, 4)?, + [-0.99, 0.5403, -0.6536, 0.9888], + ); + let y = x.cos()?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!( + test_utils::to_vec1_round(&y, 4)?, + [-0.99, 0.5403, -0.6536, 0.9888], + ); + assert_eq!( + test_utils::to_vec1_round(grad_x, 4)?, + [-0.1411, -0.8415, 0.7568, -0.1494], + ); + let y = x.sqr()?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!(y.to_vec1::()?, [9.0, 1.0, 16.0, 0.0225]); + assert_eq!(grad_x.to_vec1::()?, [6.0, 2.0, 8.0, 0.3]); + let y = x.sqr()?.sqrt()?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!(y.to_vec1::()?, [3.0, 1.0, 4.0, 0.15]); + assert_eq!(test_utils::to_vec1_round(grad_x, 4)?, [1.0, 1.0, 1.0, 1.0]); + let y = x.neg()?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!(y.to_vec1::()?, [-3.0, -1.0, -4.0, -0.15]); + assert_eq!(grad_x.to_vec1::()?, [-1.0, -1.0, -1.0, -1.0]); + let y = x.affine(0.2, 1.)?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!(y.to_vec1::()?, [1.6, 1.2, 1.8, 1.03]); + assert_eq!(grad_x.to_vec1::()?, [0.2, 0.2, 0.2, 0.2]); + let y = Tensor::new(1f32, device)?.broadcast_div(x)?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!( + test_utils::to_vec1_round(&y, 4)?, + [0.3333, 1.0, 0.25, 6.6667] + ); + assert_eq!( + grad_x.to_vec1::()?, + [-0.11111111, -1.0, -0.0625, -44.444443], + ); + let y = x.broadcast_div(&Tensor::new(0.5f32, device)?)?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!(y.to_vec1::()?, [6., 2., 8., 0.3]); + assert_eq!(grad_x.to_vec1::()?, [2., 2., 2., 2.]); + + let x = Var::new(&[3f32, 1., 4., 0.15], device)?; + let y = x.powf(2.5)?; + let grads = y.backward()?; + let grad_x = grads.get(&x).context("no grad for x")?; + assert_eq!(test_utils::to_vec1_round(&y, 2)?, [15.59, 1.0, 32.0, 0.01]); + assert_eq!( + test_utils::to_vec1_round(grad_x, 2)?, + [12.99, 2.5, 20.0, 0.15] + ); + + let y = x.tanh()?; + let grads = y.backward()?; + let grad_x = grads.get(&x).context("no grad for x")?; + assert_eq!(test_utils::to_vec1_round(&y, 2)?, [1.0, 0.76, 1.0, 0.15]); + assert_eq!( + test_utils::to_vec1_round(grad_x, 2)?, + [0.01, 0.42, 0.0, 0.98], + ); + + // testing compared to pytorch nn.GELU(approximate = 'tanh') + let y = x.gelu()?; + let grads = y.backward()?; + let grad_x = grads.get(&x).context("no grad for x")?; + assert_eq!( + test_utils::to_vec1_round(&y, 4)?, + [2.9964, 0.8412, 3.9999, 0.0839] + ); + assert_eq!( + test_utils::to_vec1_round(grad_x, 4)?, + [1.0116, 1.0830, 1.0003, 0.6188], + ); + + // Testing compared to pytorch torch.erf + // + // import torch + // x = torch.tensor([3.0, 1.0, 4.0, 0.15], requires_grad=True) + // y = x.erf() + // print(y) + // loss = y.sum() + // loss.backward() + // print(x.grad) + let y = x.erf()?; + let grads = y.backward()?; + let grad_x = grads.get(&x).context("no grad for x")?; + assert_eq!(test_utils::to_vec1_round(&y, 4)?, [1.0, 0.8427, 1.0, 0.168]); + assert_eq!( + test_utils::to_vec1_round(grad_x, 4)?, + [0.0001, 0.4151, 0.0, 1.1033], + ); + + // Testing compared to pytorch nn.GELU(approximate = 'none') + // + // import torch + // import torch.nn.functional as F + // x = torch.tensor([3.0, 1.0, 4.0, 0.15], requires_grad=True) + // y = F.gelu(x, approximate='none') + // print(y) + // loss = y.sum() + // loss.backward() + // print(x.grad) + let y = x.gelu_erf()?; + let grads = y.backward()?; + let grad_x = grads.get(&x).context("no grad for x")?; + assert_eq!( + test_utils::to_vec1_round(&y, 4)?, + [2.9960, 0.8413, 3.9999, 0.0839] + ); + assert_eq!( + test_utils::to_vec1_round(grad_x, 4)?, + [1.0119, 1.0833, 1.0005, 0.6188], + ); + + // Testing compared to pytorch elu + // + // import torch + // import torch.nn.functional as F + // x = torch.tensor([-1.0, 0.0, -2.0, 3.0], requires_grad=True) + // y = F.elu(x, alpha=2.0) + // print(y) + // loss = y.min + // loss = y.sum() + // loss.backward() + // print(x.grad) + let elu_x = Var::new(&[-1.0f32, 0., -2., 3.], device)?; + let y = elu_x.elu(2.)?; + let grads = y.backward()?; + let grad_x = grads.get(&elu_x).context("no grad for x")?; + + assert_eq!( + test_utils::to_vec1_round(&y, 4)?, + [-1.2642, 0.0000, -1.7293, 3.0000] + ); + assert_eq!( + test_utils::to_vec1_round(grad_x, 4)?, + [0.7358, 2.0000, 0.2707, 1.0000] + ); + + // testing compared to pytorch nn.Silu() + let y = x.silu()?; + let grads = y.backward()?; + let grad_x = grads.get(&x).context("no grad for x")?; + assert_eq!( + test_utils::to_vec1_round(&y, 4)?, + [2.8577, 0.7311, 3.9281, 0.0806] + ); + assert_eq!( + test_utils::to_vec1_round(grad_x, 4)?, + [1.0881, 0.9277, 1.0527, 0.5747], + ); + + if device.is_cpu() { + let x = Var::new(&[[[1f32, 2., 3.], [4., 5., 6.], [7., 8., 9.]]], device)?; + let y = x.interpolate1d(12)?.reshape(36)?; + + let z = Tensor::new( + &[ + 1_f32, 02., 03., 04., 05., 06., 07., 08., 09., 10., 11., 12., 13., 14., 15., 16., + 17., 18., 19., 20., 21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., + 33., 34., 35., 36., + ], + device, + )?; + + let loss = y.unsqueeze(1)?.transpose(0, 1)?.matmul(&z.unsqueeze(1)?)?; + let grads = loss.backward()?; + let grad_x = grads.get(&x).context("no grad for x")?; + + assert_eq!( + test_utils::to_vec3_round(grad_x, 4)?, + [[[10_f32, 26., 42.], [58., 74., 90.], [106., 122., 138.]]] + ); + } + + // manually checked: see comments + let x = Var::new(&[[[[1f32, 2., 3.], [4., 5., 6.], [7., 8., 9.]]]], device)?; + let y = x.interpolate2d(6, 6)?.reshape(36)?; + + let z = Tensor::new( + &[ + 1_f32, 02., 03., 04., 05., 06., 07., 08., 09., 10., 11., 12., 13., 14., 15., 16., 17., + 18., 19., 20., 21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., 33., 34., + 35., 36., + ], + device, + )?; + // gradient should be + // row 1 + // 1+2+7+8 = 18 + // 3+4+9+10 = 26 + // 5+6+11+12 = 34 + // row 2 + // 13+14+19+20 = 66 + // 15+16+21+22 = 74 + // 17+18+23+24 = 82 + // row 3 + // 25+26+31+32 = 114 + // 27+28+33+34 = 122 + // 29+30+35+36 = 130 + let loss = y.unsqueeze(1)?.transpose(0, 1)?.matmul(&z.unsqueeze(1)?)?; + + let grads = loss.backward()?; + + let grad_x = grads.get(&x).context("no grad for x")?; + assert_eq!( + test_utils::to_vec2_round(&grad_x.flatten(0, 2)?, 4)?, + [[18_f32, 26., 34.], [66., 74., 82.], [114., 122., 130.]] + ); + + // manually checked: see comments + let x = Var::new(&[[[[1f32, 2.], [4., 5.]]]], device)?; + let y = x.interpolate2d(6, 6)?.reshape(36)?; + + let z = Tensor::new( + &[ + 1_f32, 02., 03., 04., 05., 06., 07., 08., 09., 10., 11., 12., 13., 14., 15., 16., 17., + 18., 19., 20., 21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., 33., 34., + 35., 36., + ], + device, + )?; + // gradient should be + // row 1 + // 1+2+3+7+8+9+13+14+15 = 72 + // 4+5+6+10+11+12+16+17+18 = 99 + // row 2 + // 19+20+21+25+26+27+31+32+33 = 234 + // 22+23+24+28+29+30+34+35+36 = 243 + let loss = y.unsqueeze(1)?.transpose(0, 1)?.matmul(&z.unsqueeze(1)?)?; + + let grads = loss.backward()?; + + let grad_x = grads.get(&x).context("no grad for x")?; + assert_eq!( + test_utils::to_vec2_round(&grad_x.flatten(0, 2)?, 4)?, + [[72_f32, 99.], [234., 261.]] + ); + + // manually checked: see comments + let x = Var::new(&[[[[1f32, 2.], [4., 5.]], [[6f32, 7.], [8., 9.]]]], device)?; + + let y = x.interpolate2d(4, 4)?.reshape(32)?; + + #[rustfmt::skip] + let z = Tensor::new( + &[ + 1_f32, 02., 03., 04., + 05., 06., 07., 08., + 09., 10., 11., 12., + 13., 14., 15., 16., + 17., 18., 19., 20., + 21., 22., 23., 24., + 25., 26., 27., 28., + 29., 30., 31., 32. + ], + device, + )?; + // gradient should be + // m1r1 + // 1+2+5+6=14 + // 3+4+7+8=22 + // m1r2 + // 9+10+13+14=46 + // 11+12+15+16=54 + // m2r1 + // 17+18+21+22=78 + // 19+20+23+24=86 + // m2r2 + // 25+26+29+30=110 + // 27+28+31+32=118 + let loss = y.unsqueeze(1)?.transpose(0, 1)?.matmul(&z.unsqueeze(1)?)?; + + let grads = loss.backward()?; + + let grad_x = grads.get(&x).context("no grad for x")?; + + assert_eq!( + test_utils::to_vec3_round(&grad_x.flatten(0, 1)?, 4)?, + [[[14_f32, 22.], [46., 54.]], [[78., 86.], [110., 118.]]] + ); + + // manually checked: see comments + let x = Var::new( + &[[[[1f32, 2.], [4., 5.]]], [[[6f32, 7.], [8., 9.]]]], + device, + )?; + + let y = x.interpolate2d(4, 4)?.reshape(32)?; + + #[rustfmt::skip] + let z = Tensor::new( + &[ + 1_f32, 02., 03., 04., + 05., 06., 07., 08., + 09., 10., 11., 12., + 13., 14., 15., 16., + 17., 18., 19., 20., + 21., 22., 23., 24., + 25., 26., 27., 28., + 29., 30., 31., 32. + ], + device, + )?; + // gradient should be + // m1r1 + // 1+2+5+6=14 + // 3+4+7+8=22 + // m1r2 + // 9+10+13+14=46 + // 11+12+15+16=54 + // m2r1 + // 17+18+21+22=78 + // 19+20+23+24=86 + // m2r2 + // 25+26+29+30=110 + // 27+28+31+32=118 + let loss = y.unsqueeze(1)?.transpose(0, 1)?.matmul(&z.unsqueeze(1)?)?; + + let grads = loss.backward()?; + + let grad_x = grads.get(&x).context("no grad for x")?; + + assert_eq!( + test_utils::to_vec3_round(&grad_x.flatten(0, 1)?, 4)?, + [[[14_f32, 22.], [46., 54.]], [[78., 86.], [110., 118.]]] + ); + Ok(()) +} + +fn binary_grad(device: &Device) -> Result<()> { + let x = Var::new(&[3f32, 1., -4., -1.], device)?; + let x = x.as_tensor(); + // leaky relu + let y = x.maximum(&(x * 0.1)?)?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!(x.to_vec1::()?, [3., 1., -4., -1.]); + assert_eq!(y.to_vec1::()?, [3., 1., -0.4, -0.1]); + assert_eq!(grad_x.to_vec1::()?, [1., 1., 0.1, 0.1]); + + let y = x.minimum(&(x * 0.1)?)?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!(y.to_vec1::()?, [0.3, 0.1, -4., -1.]); + assert_eq!(grad_x.to_vec1::()?, [0.1, 0.1, 1., 1.]); + + // This one is easy to mess up, we want the gradient to be one as it is the identity function. + let y = x.minimum(x)?; + let grads = y.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + assert_eq!(y.to_vec1::()?, [3., 1., -4., -1.]); + assert_eq!(grad_x.to_vec1::()?, [1., 1., 1., 1.]); + + let x_var = Var::new(&[3f32, 1., -4., -1., 5., 9.], device)?; + let x = x_var.as_tensor(); + let y_var = Var::new(&[2f32, 7., 1.], device)?; + let y = y_var.as_tensor(); + + let ss = x + .reshape((2, 3))? + .slice_scatter0(&y.reshape((1, 3))?, 1)? + .sqr()?; + let grads = ss.backward()?; + let grad_x = grads.get(x).context("no grad for x")?; + let grad_y = grads.get(y).context("no grad for y")?; + assert_eq!(ss.to_vec2::()?, [[9., 1., 16.], [4., 49., 1.]]); + assert_eq!(grad_x.to_vec1::()?, [6.0, 2.0, -8.0, 0.0, 0.0, 0.0]); + assert_eq!(grad_y.to_vec1::()?, [4.0, 14.0, 2.0]); + Ok(()) +} + +#[test] +fn test_flip_backprop() -> Result<()> { + let device = &Device::Cpu; + + // Create a tensor (leaf node) that requires gradients + let x = Var::ones((2, 2), DType::F64, device)?; + let weights = Tensor::arange(1.0, 5.0, device)?.reshape((2, 2))?; + + let y = x.matmul(&weights)?; + let expected_y = Tensor::from_vec(vec![4.0, 6.0, 4.0, 6.0], (2, 2), device)?; + candle_core::test_utils::assert_tensor_eq(&y, &expected_y)?; + + let z = y.flip(&[1])?; + let expected_z = Tensor::from_vec(vec![6.0, 4.0, 6.0, 4.0], (2, 2), device)?; + candle_core::test_utils::assert_tensor_eq(&z, &expected_z)?; + + let loss = z.sum_all()?; + + let grad_store = loss.backward()?; + let grad_x = grad_store.get_id(x.id()).unwrap(); + + let flipped_weights = weights.flip(&[1])?; + let dloss_dy = Tensor::ones((2, 2), DType::F64, device)?; + // dloss/dx = dloss/dy @ dy/dx = ones @ weight.flip.T + let expected_grad = dloss_dy.matmul(&flipped_weights.t()?)?; + candle_core::test_utils::assert_tensor_eq(grad_x, &expected_grad)?; + + Ok(()) +} + +test_device!( + simple_grad, + simple_grad_cpu, + simple_grad_gpu, + simple_grad_metal +); +test_device!(sum_grad, sum_grad_cpu, sum_grad_gpu, sum_grad_metal); +test_device!( + matmul_grad, + matmul_grad_cpu, + matmul_grad_gpu, + matmul_grad_metal +); +test_device!( + grad_descent, + grad_descent_cpu, + grad_descent_gpu, + grad_descent_metal +); +test_device!(unary_grad, unary_grad_cpu, unary_grad_gpu, unary_grad_metal); +test_device!( + binary_grad, + binary_grad_cpu, + binary_grad_gpu, + binary_grad_metal +); diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/indexing_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/indexing_tests.rs new file mode 100644 index 0000000000..047205a31f --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/tests/indexing_tests.rs @@ -0,0 +1,122 @@ +use anyhow::Result; +use candle_core::{Device, IndexOp, Tensor}; + +#[test] +fn integer_index() -> Result<()> { + let dev = Device::Cpu; + + let tensor = Tensor::arange(0u32, 2 * 3, &dev)?.reshape((2, 3))?; + let result = tensor.i(1)?; + assert_eq!(result.dims(), &[3]); + assert_eq!(result.to_vec1::()?, &[3, 4, 5]); + + let result = tensor.i((.., 2))?; + assert_eq!(result.dims(), &[2]); + assert_eq!(result.to_vec1::()?, &[2, 5]); + + Ok(()) +} + +#[test] +fn range_index() -> Result<()> { + let dev = Device::Cpu; + // RangeFull + let tensor = Tensor::arange(0u32, 2 * 3, &dev)?.reshape((2, 3))?; + let result = tensor.i(..)?; + assert_eq!(result.dims(), &[2, 3]); + assert_eq!(result.to_vec2::()?, &[[0, 1, 2], [3, 4, 5]]); + + // Range + let tensor = Tensor::arange(0u32, 4 * 3, &dev)?.reshape((4, 3))?; + let result = tensor.i(1..3)?; + assert_eq!(result.dims(), &[2, 3]); + assert_eq!(result.to_vec2::()?, &[[3, 4, 5], [6, 7, 8]]); + + // RangeFrom + let result = tensor.i(2..)?; + assert_eq!(result.dims(), &[2, 3]); + assert_eq!(result.to_vec2::()?, &[[6, 7, 8], [9, 10, 11]]); + + // RangeTo + let result = tensor.i(..2)?; + assert_eq!(result.dims(), &[2, 3]); + assert_eq!(result.to_vec2::()?, &[[0, 1, 2], [3, 4, 5]]); + + // RangeInclusive + let result = tensor.i(1..=2)?; + assert_eq!(result.dims(), &[2, 3]); + assert_eq!(result.to_vec2::()?, &[[3, 4, 5], [6, 7, 8]]); + + // RangeTo + let result = tensor.i(..1)?; + assert_eq!(result.dims(), &[1, 3]); + assert_eq!(result.to_vec2::()?, &[[0, 1, 2]]); + + // RangeToInclusive + let result = tensor.i(..=1)?; + assert_eq!(result.dims(), &[2, 3]); + assert_eq!(result.to_vec2::()?, &[[0, 1, 2], [3, 4, 5]]); + + // Empty range + let result = tensor.i(1..1)?; + assert_eq!(result.dims(), &[0, 3]); + let empty: [[u32; 3]; 0] = []; + assert_eq!(result.to_vec2::()?, &empty); + + // Similar to PyTorch, allow empty ranges when the computed length is negative. + #[allow(clippy::reversed_empty_ranges)] + let result = tensor.i(1..0)?; + assert_eq!(result.dims(), &[0, 3]); + let empty: [[u32; 3]; 0] = []; + assert_eq!(result.to_vec2::()?, &empty); + Ok(()) +} + +#[test] +fn index_3d() -> Result<()> { + let tensor = Tensor::from_iter(0..24u32, &Device::Cpu)?.reshape((2, 3, 4))?; + assert_eq!(tensor.i((0, 0, 0))?.to_scalar::()?, 0); + assert_eq!(tensor.i((1, 0, 0))?.to_scalar::()?, 12); + assert_eq!(tensor.i((0, 1, 0))?.to_scalar::()?, 4); + assert_eq!(tensor.i((0, 1, 3))?.to_scalar::()?, 7); + assert_eq!(tensor.i((0..2, 0, 0))?.to_vec1::()?, &[0, 12]); + assert_eq!( + tensor.i((0..2, .., 0))?.to_vec2::()?, + &[[0, 4, 8], [12, 16, 20]] + ); + assert_eq!( + tensor.i((..2, .., 3))?.to_vec2::()?, + &[[3, 7, 11], [15, 19, 23]] + ); + assert_eq!(tensor.i((1, .., 3))?.to_vec1::()?, &[15, 19, 23]); + Ok(()) +} + +#[test] +fn slice_assign() -> Result<()> { + let dev = Device::Cpu; + + let tensor = Tensor::arange(0u32, 4 * 5, &dev)?.reshape((4, 5))?; + let src = Tensor::arange(0u32, 2 * 3, &dev)?.reshape((3, 2))?; + let out = tensor.slice_assign(&[1..4, 3..5], &src)?; + assert_eq!( + out.to_vec2::()?, + &[ + [0, 1, 2, 3, 4], + [5, 6, 7, 0, 1], + [10, 11, 12, 2, 3], + [15, 16, 17, 4, 5] + ] + ); + let out = tensor.slice_assign(&[0..3, 0..2], &src)?; + assert_eq!( + out.to_vec2::()?, + &[ + [0, 1, 2, 3, 4], + [2, 3, 7, 8, 9], + [4, 5, 12, 13, 14], + [15, 16, 17, 18, 19] + ] + ); + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/layout_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/layout_tests.rs new file mode 100644 index 0000000000..bc67f7defc --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/tests/layout_tests.rs @@ -0,0 +1,152 @@ +use candle::{test_device, Device, IndexOp, Result, Tensor}; +use candle_core as candle; + +fn contiguous(device: &Device) -> Result<()> { + let tensor = Tensor::arange(0u32, 24u32, device)?.reshape((2, 3, 4))?; + assert_eq!( + tensor.to_vec3::()?, + &[ + [[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]], + [[12, 13, 14, 15], [16, 17, 18, 19], [20, 21, 22, 23]] + ] + ); + assert_eq!( + tensor.t()?.contiguous()?.to_vec3::()?, + &[ + [[0, 4, 8], [1, 5, 9], [2, 6, 10], [3, 7, 11]], + [[12, 16, 20], [13, 17, 21], [14, 18, 22], [15, 19, 23]] + ] + ); + assert_eq!( + tensor.transpose(0, 1)?.contiguous()?.to_vec3::()?, + &[ + [[0, 1, 2, 3], [12, 13, 14, 15]], + [[4, 5, 6, 7], [16, 17, 18, 19]], + [[8, 9, 10, 11], [20, 21, 22, 23]] + ] + ); + assert_eq!( + tensor.transpose(0, 1)?.flatten_all()?.to_vec1::()?, + &[0, 1, 2, 3, 12, 13, 14, 15, 4, 5, 6, 7, 16, 17, 18, 19, 8, 9, 10, 11, 20, 21, 22, 23] + ); + assert_eq!( + tensor + .i(1..)? + .transpose(0, 1)? + .contiguous()? + .to_vec3::()?, + &[[[12, 13, 14, 15]], [[16, 17, 18, 19]], [[20, 21, 22, 23]]] + ); + assert_eq!( + tensor.transpose(0, 2)?.contiguous()?.to_vec3::()?, + &[ + [[0, 12], [4, 16], [8, 20]], + [[1, 13], [5, 17], [9, 21]], + [[2, 14], [6, 18], [10, 22]], + [[3, 15], [7, 19], [11, 23]] + ] + ); + Ok(()) +} + +test_device!(contiguous, contiguous_cpu, contiguous_gpu, contiguous_metal); + +#[test] +fn strided_blocks() -> Result<()> { + use candle::Device::Cpu; + let tensor = Tensor::arange(0u32, 24u32, &Cpu)?.reshape((2, 3, 4))?; + match tensor.strided_blocks() { + candle::StridedBlocks::SingleBlock { start_offset, len } => { + assert_eq!(start_offset, 0); + assert_eq!(len, 24); + } + candle::StridedBlocks::MultipleBlocks { .. } => { + panic!("unexpected block structure") + } + }; + let tensor = Tensor::arange(0u32, 26u32, &Cpu)? + .i(2..)? + .reshape((2, 3, 4))?; + match tensor.strided_blocks() { + candle::StridedBlocks::SingleBlock { start_offset, len } => { + assert_eq!(start_offset, 2); + assert_eq!(len, 24); + } + candle::StridedBlocks::MultipleBlocks { .. } => { + panic!("unexpected block structure") + } + }; + let tensor = Tensor::arange(0u32, 24u32, &Cpu)?.reshape((2, 3, 4))?; + let tensor = tensor.i(1)?; + match tensor.strided_blocks() { + candle::StridedBlocks::SingleBlock { start_offset, len } => { + assert_eq!(start_offset, 12); + assert_eq!(len, 12); + } + candle::StridedBlocks::MultipleBlocks { .. } => { + panic!("unexpected block structure") + } + }; + let tensor = Tensor::arange(0u32, 24u32, &Cpu)?.reshape((2, 3, 4))?; + let tensor = tensor.i((.., 1))?.contiguous()?; + match tensor.strided_blocks() { + candle::StridedBlocks::SingleBlock { start_offset, len } => { + assert_eq!(start_offset, 0); + assert_eq!(len, 8); + assert_eq!(tensor.to_vec2::()?, &[[4, 5, 6, 7], [16, 17, 18, 19]]); + } + candle::StridedBlocks::MultipleBlocks { .. } => { + panic!("unexpected block structure") + } + }; + let tensor = Tensor::arange(0u32, 24u32, &Cpu)?.reshape((2, 3, 4))?; + let tensor = tensor.i((.., 1))?; + match tensor.strided_blocks() { + candle::StridedBlocks::SingleBlock { .. } => { + panic!("unexpected block structure") + } + candle::StridedBlocks::MultipleBlocks { + block_len, + block_start_index, + } => { + assert_eq!(block_len, 4); + assert_eq!(block_start_index.collect::>(), &[4, 16]) + } + }; + let tensor = Tensor::arange(0u32, 24u32, &Cpu)?.reshape((2, 3, 4))?; + match tensor.t()?.strided_blocks() { + candle::StridedBlocks::SingleBlock { .. } => { + panic!("unexpected block structure") + } + candle::StridedBlocks::MultipleBlocks { + block_start_index, + block_len, + } => { + assert_eq!(block_len, 1); + assert_eq!( + block_start_index.collect::>(), + &[ + 0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11, 12, 16, 20, 13, 17, 21, 14, 18, 22, 15, + 19, 23 + ] + ) + } + }; + let tensor = Tensor::arange(0u32, 24u32, &Cpu)?.reshape((2, 3, 4))?; + match tensor.transpose(0, 1)?.strided_blocks() { + candle::StridedBlocks::SingleBlock { .. } => { + panic!("unexpected block structure") + } + candle::StridedBlocks::MultipleBlocks { + block_start_index, + block_len, + } => { + assert_eq!(block_len, 4); + assert_eq!( + block_start_index.collect::>(), + &[0, 12, 4, 16, 8, 20] + ) + } + }; + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/matmul_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/matmul_tests.rs new file mode 100644 index 0000000000..aa2189f7a0 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/tests/matmul_tests.rs @@ -0,0 +1,146 @@ +use candle_core::{test_device, DType, Device, IndexOp, Result, Tensor}; + +fn matmul(device: &Device) -> Result<()> { + let data = vec![1.0f32, 2.0, 3.0, 4.0]; + let a = Tensor::from_slice(&data, (2, 2), device)?; + let data = vec![1.0f32, 2.0, 3.0, 4.0]; + let b = Tensor::from_slice(&data, (2, 2), device)?; + + let c = a.matmul(&b)?; + assert_eq!(c.to_vec2::()?, &[[7.0f32, 10.0], [15.0, 22.0]]); + + let data = vec![1.0f32, 2.0]; + let a = Tensor::from_slice(&data, (2, 1), device)?; + let data = vec![3.0f32, 4.0]; + let b = Tensor::from_slice(&data, (1, 2), device)?; + let c = a.matmul(&b)?; + assert_eq!(c.to_vec2::()?, &[&[3.0, 4.0], &[6.0, 8.0]]); + + let data: Vec<_> = (0..6).map(|i| i as f32).collect(); + let a = Tensor::from_slice(&data, (2, 3), device)?; + let data: Vec<_> = (0..6).map(|i| (i + 2) as f32).collect(); + let b = Tensor::from_slice(&data, (3, 2), device)?; + let c = a.matmul(&b)?; + assert_eq!(c.to_vec2::()?, &[&[16., 19.], &[52., 64.]]); + + let data: Vec<_> = (0..12).map(|i| i as f32).collect(); + let a = Tensor::from_slice(&data, (2, 2, 3), device)?; + let data: Vec<_> = (0..12).map(|i| (i + 2) as f32).collect(); + let b = Tensor::from_slice(&data, (2, 3, 2), device)?; + let expected = [[[16., 19.], [52., 64.]], [[214., 235.], [304., 334.]]]; + + let c = a.matmul(&b)?; + assert_eq!(c.to_vec3::()?, &expected); + + // Also perform the matmul on contiguous transposed versions. + let a_tt = a.t()?.contiguous()?.t()?; + assert!(!a_tt.is_contiguous()); + assert_eq!(a.dims(), a_tt.dims()); + assert_eq!(a_tt.stride(), &[6, 1, 2]); + + let b_tt = b.t()?.contiguous()?.t()?; + assert!(!b_tt.is_contiguous()); + assert_eq!(b.dims(), b_tt.dims()); + assert_eq!(b_tt.stride(), &[6, 1, 3]); + + assert_eq!(a_tt.matmul(&b)?.to_vec3::()?, &expected); + assert_eq!(a.matmul(&b_tt)?.to_vec3::()?, &expected); + assert_eq!(a_tt.matmul(&b_tt)?.to_vec3::()?, &expected); + Ok(()) +} + +fn matmul_bf16(device: &Device) -> Result<()> { + if !device.supports_bf16() { + return Ok(()); + } + let data = vec![1.0f32, 2.0, 3.0, 4.0]; + let a = Tensor::from_slice(&data, (2, 2), device)?.to_dtype(DType::BF16)?; + let data = vec![1.0f32, 2.0, 3.0, 4.0]; + let b = Tensor::from_slice(&data, (2, 2), device)?.to_dtype(DType::BF16)?; + + let c = a.matmul(&b)?.to_dtype(DType::F32)?; + assert_eq!(c.to_vec2::()?, &[[7.0f32, 10.0], [15.0, 22.0]]); + Ok(()) +} + +fn broadcast_matmul(device: &Device) -> Result<()> { + let lhs = Tensor::randn(0f32, 1f32, (3, 1, 4, 5), device)?; + let rhs = Tensor::randn(0f32, 1f32, (6, 5, 2), device)?; + let out = lhs.broadcast_matmul(&rhs)?; + assert_eq!(out.dims(), &[3, 6, 4, 2]); + for idx1 in 0..3 { + for idx2 in 0..6 { + let out = out.i((idx1, idx2))?; + let lhs = lhs.i((idx1, 0))?; + let rhs = rhs.i(idx2)?; + let out2 = lhs.matmul(&rhs); + let sum_diff2 = (out - out2)?.sqr()?.sum_all()?; + // With cuda, we see errors of up to ~1e-12. + assert!(sum_diff2.to_vec0::()? < 1e-6) + } + } + Ok(()) +} + +#[test] +fn tensor_dot() -> Result<()> { + let lhs = Tensor::new(&[1., 2., 3.], &Device::Cpu)?; + let rhs = Tensor::new(&[4., 5., 6.], &Device::Cpu)?; + let expected = Tensor::new(32., &Device::Cpu)?; + let dot_ret = lhs.dot(&rhs)?; + candle_core::test_utils::assert_tensor_eq(&dot_ret, &expected)?; + Ok(()) +} + +#[test] +fn tensor_mv() -> Result<()> { + let mat = Tensor::new(&[[1., 2., 3.], [4., 5., 6.]], &Device::Cpu)?; + let vec = Tensor::new(&[1., 1., 1.], &Device::Cpu)?; + let expected = Tensor::new(&[6., 15.], &Device::Cpu)?; + let mv_ret = mat.mv(&vec)?; + candle_core::test_utils::assert_tensor_eq(&mv_ret, &expected)?; + Ok(()) +} + +// https://github.com/huggingface/candle/issues/1948 +fn squeeze_mm(device: &Device) -> Result<()> { + let seq_len = 8_usize; + let a = Tensor::zeros((1, seq_len, 16), DType::F32, device)?; + let x = a.i((.., seq_len - 1, ..))?; + let w = Tensor::zeros((32, 16), DType::F32, device)?.t()?; + let x = x.matmul(&w)?; + assert_eq!(x.dims(), &[1, 32]); + Ok(()) +} + +// https://github.com/huggingface/candle/issues/1992 +fn mm_layout(device: &Device) -> Result<()> { + let a = Tensor::arange(0f32, 16f32, device)?.reshape((1, 1, 4, 4))?; + let b = Tensor::arange(0f32, 8f32, device)?.reshape((1, 1, 4, 2))?; + let mm1 = a.matmul(&b)?; + // Forces the layout to be: + // shape: [1, 1, 4, 2], stride: [8, 2, 2, 1], start_offset: 0 + // This is still a contiguous matrix but matmul checks are only the two last dimensions have + // non 1 sizes but matmul check may be reluctant to handle it. + let b = b.transpose(1, 2)?.force_contiguous()?.transpose(1, 2)?; + let mm2 = a.matmul(&b)?; + let diff = (mm1 - mm2)?.abs()?.sum_all()?.to_vec0::()?; + assert_eq!(diff, 0.); + Ok(()) +} + +test_device!(matmul, matmul_cpu, matmul_gpu, matmul_metal); +test_device!( + matmul_bf16, + matmul_bf16_cpu, + matmul_bf16_gpu, + matmul_bf16_metal +); +test_device!( + broadcast_matmul, + broadcast_matmul_cpu, + broadcast_matmul_gpu, + broadcast_matmul_metal +); +test_device!(squeeze_mm, squeeze_mm_cpu, squeeze_mm_gpu, squeeze_mm_metal); +test_device!(mm_layout, mm_layout_cpu, mm_layout_gpu, mm_layout_metal); diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/npy.py b/v2/dependency-patches/candle-core-0.9.2/tests/npy.py new file mode 100644 index 0000000000..0fd2778a15 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/tests/npy.py @@ -0,0 +1,9 @@ +import numpy as np +x = np.arange(10) + +# Write a npy file. +np.save("test.npy", x) + +# Write multiple values to a npz file. +values = { "x": x, "x_plus_one": x + 1 } +np.savez("test.npz", **values) diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/pool_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/pool_tests.rs new file mode 100644 index 0000000000..1edb7d353b --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/tests/pool_tests.rs @@ -0,0 +1,117 @@ +use candle_core::{test_device, test_utils, Device, IndexOp, Result, Tensor}; + +// https://github.com/huggingface/candle/issues/364 +fn avg_pool2d(dev: &Device) -> Result<()> { + let data: Vec = vec![ + 1., 1., 1., 1., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., + ]; + let t = Tensor::from_vec(data, (1, 1, 4, 4), dev)?; + let pool = t.avg_pool2d(2)?.squeeze(0)?.squeeze(0)?; + assert_eq!(pool.to_vec2::()?, [[0.5f32, 1.], [1., 1.]]); + + let data: Vec = vec![ + 1., 2., 1., 3., 0., 0., 1., 1., 1., 1., 1., 1., 5., 1., 1., 1., + ]; + let t = Tensor::from_vec(data, (1, 1, 2, 8), dev)?; + let pool = t.avg_pool2d(2)?.squeeze(0)?.squeeze(0)?; + assert_eq!(pool.to_vec2::()?, [[5. / 4., 6. / 4., 6. / 4., 1.]]); + Ok(()) +} + +fn max_pool2d(dev: &Device) -> Result<()> { + let data: Vec = vec![ + 1., 2., 1., 3., 0., 0., 1., 1., 1., 1., 1., 1., 5., 1., 1., 1., + ]; + let t = Tensor::from_vec(data, (1, 1, 4, 4), dev)?; + + let pool = t.max_pool2d(2)?.squeeze(0)?.squeeze(0)?; + assert_eq!(pool.to_vec2::()?, [[2f32, 3.], [5., 1.]]); + + let t = t.reshape((1, 1, 2, 8))?; + let pool = t.max_pool2d(2)?.squeeze(0)?.squeeze(0)?; + assert_eq!(pool.to_vec2::()?, [[2.0, 3.0, 5.0, 1.0]]); + Ok(()) +} + +/* This test corresponds to the following PyTorch script. +import torch +torch.manual_seed(4242) + +t = torch.randn((1, 2, 4, 4)) +print(t.flatten()) +res = torch.nn.functional.avg_pool2d(t, 2) +print(res) +*/ +fn avg_pool2d_pytorch(dev: &Device) -> Result<()> { + if dev.is_metal() { + return Ok(()); + } + let t = Tensor::new( + &[ + 0.4056f32, -0.8689, -0.0773, -1.5630, -2.8012, -1.5059, 0.3972, 1.0852, 0.4997, 3.0616, + 1.6541, 0.0964, -0.8338, -1.6523, -0.8323, -0.1699, 0.0823, 0.3526, 0.6843, 0.2395, + 1.2279, -0.9287, -1.7030, 0.1370, 0.6047, 0.3770, -0.6266, 0.3529, 2.2013, -0.6836, + 0.2477, 1.3127, + ], + dev, + )? + .reshape((1, 2, 4, 4))?; + let pool = t.avg_pool2d(2)?.squeeze(0)?; + assert_eq!( + test_utils::to_vec3_round(&pool, 4)?, + [ + [[-1.1926, -0.0395], [0.2688, 0.1871]], + [[0.1835, -0.1606], [0.6249, 0.3217]] + ] + ); + let pool = t.avg_pool2d(3)?.squeeze(0)?; + assert_eq!( + test_utils::to_vec3_round(&pool, 4)?, + [[[0.085]], [[0.0078]]] + ); + + let t = t.reshape((1, 1, 4, 8))?; + let pool = t.avg_pool2d(2)?.squeeze(0)?.squeeze(0)?; + assert_eq!( + test_utils::to_vec2_round(&pool, 4)?, + [ + [0.7745, 0.0276, -1.6983, 0.12], + [0.3542, 0.1625, 0.4542, -0.0014] + ] + ); + Ok(()) +} + +fn upsample_nearest2d(dev: &Device) -> Result<()> { + let t = Tensor::arange(0f32, 6f32, dev)?.reshape((1, 1, 2, 3))?; + let upsampled = t.upsample_nearest2d(4, 6)?.i(0)?.i(0)?; + assert_eq!( + t.i(0)?.i(0)?.to_vec2::()?, + [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]] + ); + assert_eq!( + upsampled.to_vec2::()?, + [ + [0.0, 0.0, 1.0, 1.0, 2.0, 2.0], + [0.0, 0.0, 1.0, 1.0, 2.0, 2.0], + [3.0, 3.0, 4.0, 4.0, 5.0, 5.0], + [3.0, 3.0, 4.0, 4.0, 5.0, 5.0] + ] + ); + Ok(()) +} + +test_device!(avg_pool2d, avg_pool2d_cpu, avg_pool2d_gpu, avg_pool2d_metal); +test_device!( + avg_pool2d_pytorch, + avg_pool2d_pytorch_cpu, + avg_pool2d_pytorch_gpu, + avg_pool2d_pytorch_metal +); +test_device!(max_pool2d, max_pool2d_cpu, max_pool2d_gpu, max_pool2d_metal); +test_device!( + upsample_nearest2d, + upsample_nearest2d_cpu, + upsample_nearest2d_gpu, + upsample_nearest2d_metal +); diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/pth.py b/v2/dependency-patches/candle-core-0.9.2/tests/pth.py new file mode 100644 index 0000000000..5c787c2036 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/tests/pth.py @@ -0,0 +1,37 @@ +import torch +from collections import OrderedDict + +# Write a trivial tensor to a pt file +a= torch.tensor([[1,2,3,4], [5,6,7,8]]) +o = OrderedDict() +o["test"] = a + +# Write a trivial tensor to a pt file +torch.save(o, "test.pt") + +############################################################################################################ +# Write a trivial tensor to a pt file with a key +torch.save({"model_state_dict": o}, "test_with_key.pt") + +############################################################################################################ +# Create a tensor with fortran contiguous memory layout +import numpy as np + +# Step 1: Create a 3D NumPy array with Fortran order using a range of numbers +# For example, creating a 2x3x4 array +array_fortran = np.asfortranarray(np.arange(1, 2*3*4 + 1).reshape(2, 3, 4)) + +# Verify the memory order +print("Is Fortran contiguous (F order):", array_fortran.flags['F_CONTIGUOUS']) # Should be True +print("Is C contiguous (C order):", array_fortran.flags['C_CONTIGUOUS']) # Should be False + +# Step 2: Convert the NumPy array to a PyTorch tensor +tensor_fortran = torch.from_numpy(array_fortran) + +# Verify the tensor layout +print("Tensor stride:", tensor_fortran.stride()) # Stride will reflect the Fortran memory layout + +# Step 3: Save the PyTorch tensor to a .pth file +torch.save({"tensor_fortran": tensor_fortran}, 'fortran_tensor_3d.pth') + +print("3D Tensor saved with Fortran layout.") diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/pth_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/pth_tests.rs new file mode 100644 index 0000000000..7ea3d1420e --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/tests/pth_tests.rs @@ -0,0 +1,31 @@ +/// Regression test for pth files not loading on Windows. +#[test] +fn test_pth() { + let tensors = candle_core::pickle::PthTensors::new("tests/test.pt", None).unwrap(); + tensors.get("test").unwrap().unwrap(); +} + +#[test] +fn test_pth_with_key() { + let tensors = + candle_core::pickle::PthTensors::new("tests/test_with_key.pt", Some("model_state_dict")) + .unwrap(); + tensors.get("test").unwrap().unwrap(); +} + +#[test] +fn test_pth_fortran_contiguous() { + let tensors = + candle_core::pickle::PthTensors::new("tests/fortran_tensor_3d.pth", None).unwrap(); + let tensor = tensors.get("tensor_fortran").unwrap().unwrap(); + + assert_eq!(tensor.dims3().unwrap(), (2, 3, 4)); + + assert_eq!( + tensor.to_vec3::().unwrap(), + [ + [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]], + [[13, 14, 15, 16], [17, 18, 19, 20], [21, 22, 23, 24]] + ] + ); +} diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/quantized_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/quantized_tests.rs new file mode 100644 index 0000000000..c69034a579 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/tests/quantized_tests.rs @@ -0,0 +1,1402 @@ +use candle_core::{ + bail, + quantized::{self, GgmlDType}, + test_device, + test_utils::to_vec2_round, + DType, Device, IndexOp, Module, Result, Tensor, Var, +}; +use quantized::{k_quants, GgmlType}; +use rand::prelude::*; + +const GGML_TEST_SIZE: usize = 32 * 128; + +const GGML_MAX_QUANTIZATION_TOTAL_ERROR: f32 = 0.002; +const GGML_MAX_QUANTIZATION_TOTAL_ERROR_2BITS: f32 = 0.0075; +const GGML_MAX_QUANTIZATION_TOTAL_ERROR_3BITS: f32 = 0.0040; +const GGML_MAX_DOT_PRODUCT_ERROR: f32 = 0.02; + +fn test_matmul( + device: &Device, + (b, m, n, k): (usize, usize, usize, usize), + dtype: GgmlDType, +) -> Result<()> { + if (device.is_cuda() || device.is_metal()) + && (dtype == GgmlDType::Q8_1 || dtype == GgmlDType::Q8K) + { + return Ok(()); + } + + let lhs = (0..(m * k)) + .map(|v| v as f32 / (m * k) as f32) + .collect::>(); + let rhs = (0..(k * n)) + .map(|v| v as f32 / (n * k) as f32) + .collect::>(); + + let lhs = Tensor::from_slice(&lhs, (m, k), device)?; + let rhs = Tensor::from_slice(&rhs, (k, n), device)?; + let mm = lhs.matmul(&rhs)?; + let qtensor = quantized::QTensor::quantize(&rhs.t()?, dtype)?; + let matmul = quantized::QMatMul::from_qtensor(qtensor)?; + let res = matmul.forward(&lhs)?; + + let error: f32 = ((&mm - &res)?.abs()? / &mm.abs()?)? + .sum_all()? + .to_scalar()?; + let error = error / (b * m * n) as f32; + assert!( + error <= 0.02, + "Error {error} is too big. \nExpected:\n {mm} \nFound:\n {res}\n for {dtype:?}" + ); + + Ok(()) +} + +#[cfg(feature = "metal")] +#[test] +fn test_matmul_mm() -> Result<()> { + let dtype = GgmlDType::Q8_0; + let device = Device::new_metal(0)?; + + let m = 32; + let n = 32; + let k = 32; + let lhs = (0..(m * k)) + .map(|v| v as f32 / (m * k) as f32) + .collect::>(); + let rhs = (0..(k * n)) + .map(|v| v as f32 / (n * k) as f32) + .collect::>(); + + let lhs = Tensor::from_slice(&lhs, (m, k), &device)?; + let rhs = Tensor::from_slice(&rhs, (1, 1, k, n), &device)?.repeat((5, 20, 1, 1))?; + let mm = lhs.broadcast_matmul(&rhs)?; + let qtensor = quantized::QTensor::quantize(&lhs.t()?, dtype)?; + let matmul = quantized::QMatMul::from_qtensor(qtensor)?; + let res = matmul.forward(&rhs)?; + + let error: f32 = ((&mm - &res)?.abs()? / &mm.abs()?)? + .sum_all()? + .to_scalar()?; + + let error = error / res.elem_count() as f32; + assert!( + error <= 0.001, + "Error {error} is too big. \nExpected:\n {mm} \nFound:\n {res}\n for {dtype:?}" + ); + + Ok(()) +} + +fn quantized_matmul(device: &Device) -> Result<()> { + let (m, k, n) = (3, 64, 4); + let lhs_s = (0..(m * k)).map(|v| v as f32).collect::>(); + let lhs = Tensor::from_slice(&lhs_s, (m, k), device)?; + let mut dst = vec![42.; 3 * 4]; + let mut rhs_t = vec![k_quants::BlockQ4_0::zeros(); 8]; + let rhs = (0..(k * n)).map(|v| v as f32).collect::>(); + k_quants::BlockQ4_0::from_float(&rhs, &mut rhs_t); + k_quants::matmul((m, k, n), &lhs_s, &rhs_t, &mut dst)?; + assert_eq!( + dst.iter().map(|x| x.round()).collect::>(), + &[ + 85120.0, 214562.0, 345455.0, 474748.0, 213475.0, 604465.0, 1000686.0, 1388317.0, + 341876.0, 994283.0, 1655709.0, 2301518.0 + ] + ); + let tensor_rhs = Tensor::from_slice(&rhs, (n, k), device)?.t()?; + let mm = lhs.matmul(&tensor_rhs)?; + assert_eq!( + mm.to_vec2::()?, + &[ + [85344.0, 214368.0, 343392.0, 472416.0], + [214368.0, 605536.0, 996704.0, 1387872.0], + [343392.0, 996704.0, 1650016.0, 2303328.0] + ] + ); + + let qtensor = quantized::QTensor::quantize(&tensor_rhs.t()?, GgmlDType::Q4_0)?; + let matmul = quantized::QMatMul::from_qtensor(qtensor)?; + let res = matmul.forward(&lhs)?; + match device { + Device::Metal(_) => assert_eq!( + to_vec2_round(&res, 0)?, + &[ + [84946.0, 214126.0, 344757.0, 473798.0], + [213458.0, 604350.0, 1000469.0, 1387990.0], + [341970.0, 994574.0, 1656181.0, 2302182.0] + ] + ), + Device::Cuda(_) => assert_eq!( + to_vec2_round(&res, 0)?, + &[ + [84866.0, 214045.0, 344676.0, 473707.0], + [213425.0, 604313.0, 1000431.0, 1387960.0], + [342030.0, 994630.0, 1656248.0, 2302250.0] + ] + ), + Device::Cpu => assert_eq!( + to_vec2_round(&res, 0)?, + &[ + [85120.0, 214562.0, 345455.0, 474748.0], + [213475.0, 604465.0, 1000686.0, 1388317.0], + [341876.0, 994283.0, 1655709.0, 2301518.0] + ] + ), + } + test_matmul(device, (1, 3, 4, 256), GgmlDType::Q4_0)?; + Ok(()) +} + +fn quantized_matmul_neg(device: &Device) -> Result<()> { + let (m, k, n) = (3, 64, 4); + let lhs_s = (0..(m * k)) + .map(|v| v as f32 - (m * k) as f32 / 2.0) + .collect::>(); + let lhs = Tensor::from_slice(&lhs_s, (m, k), device)?; + let mut dst = vec![42.; 3 * 4]; + let mut rhs_t = vec![k_quants::BlockQ4_0::zeros(); 8]; + let rhs = (0..k * n) + .map(|v| v as f32 - (k * n) as f32 / 3.0) + .collect::>(); + let tensor_rhs = Tensor::from_slice(&rhs, (n, k), device)?.t()?; + k_quants::BlockQ4_0::from_float(&rhs, &mut rhs_t); + k_quants::matmul((m, k, n), &lhs_s, &rhs_t, &mut dst)?; + assert_eq!( + dst.iter().map(|x| x.round()).collect::>(), + &[ + 243524.0, -19596.0, -285051.0, -549815.0, 23777.0, 21651.0, 19398.0, 18367.0, + -196472.0, 63012.0, 324585.0, 587902.0 + ] + ); + let mm = lhs.matmul(&tensor_rhs)?; + assert_eq!( + to_vec2_round(&mm, 0)?, + &[ + [244064.0, -20128.0, -284320.0, -548512.0], + [23563.0, 21515.0, 19467.0, 17419.0], + [-196939.0, 63157.0, 323253.0, 583349.0] + ] + ); + + let qtensor = quantized::QTensor::quantize(&tensor_rhs.t()?, GgmlDType::Q4_0)?; + let matmul = quantized::QMatMul::from_qtensor(qtensor)?; + let res = matmul.forward(&lhs)?; + match device { + Device::Metal(_) => assert_eq!( + to_vec2_round(&res, 0)?, + &[ + [243659.0, -19716.0, -285444.0, -550439.0], + [23779.0, 21653.0, 19404.0, 18349.0], + [-196101.0, 63021.0, 324252.0, 587137.0] + ] + ), + Device::Cuda(_) => assert_eq!( + to_vec2_round(&res, 0)?, + &[ + [243740.0, -19762.0, -285476.0, -550498.0], + [23774.0, 21645.0, 19395.0, 18364.0], + [-196045.0, 63030.0, 324120.0, 587079.0] + ] + ), + Device::Cpu => assert_eq!( + to_vec2_round(&res, 0)?, + &[ + [243524.0, -19596.0, -285051.0, -549815.0], + [23777.0, 21651.0, 19398.0, 18367.0], + [-196472.0, 63012.0, 324585.0, 587902.0] + ] + ), + } + let lhs2 = Tensor::stack(&[&lhs, &lhs], 0)?; + let res2 = matmul.forward(&lhs2)?; + let res2 = res2.i(1)?; + let diff = (&res - res2)?.abs()?.mean_all()?.to_vec0::()? / res.elem_count() as f32; + if device.is_cuda() { + assert!(diff < 0.1); + } else { + assert!(diff < 0.96); + } + Ok(()) +} + +fn qmm_batch(dev: &Device) -> Result<()> { + let (lhs, rhs, _mm) = get_random_tensors(2, 256, 6, dev)?; + let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q2K)?; + let rhs = quantized::QMatMul::from_qtensor(rhs)?; + let mm = rhs.forward(&lhs)?; + assert_eq!(mm.shape().dims(), [2, 6]); + let lhs2 = Tensor::cat(&[&lhs, &lhs], 0)?; + let mm2 = rhs.forward(&lhs2)?; + assert_eq!(mm2.shape().dims(), [4, 6]); + let diff2 = (mm2.i(2..)? - &mm)?.abs()?.sum_all()?.to_vec0::()?; + assert_eq!(diff2, 0.0); + let lhs3 = Tensor::cat(&[&lhs2, &lhs], 0)?; + let mm3 = rhs.forward(&lhs3)?; + assert_eq!(mm3.shape().dims(), [6, 6]); + let diff3 = (mm3.i(2..4)? - &mm)?.abs()?.sum_all()?.to_vec0::()?; + assert_eq!(diff3, 0.0); + let diff3 = (mm3.i(4..)? - &mm)?.abs()?.sum_all()?.to_vec0::()?; + assert_eq!(diff3, 0.0); + let lhs4 = Tensor::cat(&[&lhs3, &lhs3], 0)?; + let mm4 = rhs.forward(&lhs4)?; + assert_eq!(mm4.shape().dims(), [12, 6]); + let diff4 = (mm4.i(..6)? - &mm3)?.abs()?.sum_all()?.to_vec0::()?; + if dev.is_cuda() { + // We use a different kernel for sizes from 1 to 8 on cuda which explains + // the difference here. + assert!(0. < diff4 && diff4 < 1e-4) + } else { + assert_eq!(diff4, 0.0) + }; + let diff4 = (mm4.i(6..)? - &mm4.i(..6)?)? + .abs()? + .sum_all()? + .to_vec0::()?; + assert_eq!(diff4, 0.0); + Ok(()) +} + +test_device!(quantized_matmul, qmm_cpu, qmm_cuda, qmm_metal); +test_device!(quantized_matmul_neg, qmm_n_cpu, qmm_n_cuda, qmm_n_metal); +test_device!(qmm_batch, qmm_b_cpu, qmm_b_cuda, qmm_b_metal); + +fn quantize_q4_0(device: &Device) -> Result<()> { + let src = (0..32 * 4).map(|v| v as f32).collect::>(); + + let src = Tensor::from_slice(&src, (32 * 4,), device)?; + let quant = quantized::QTensor::quantize(&src, GgmlDType::Q4_0)?; + let dst = quant.dequantize(device)?; + let dst_f16 = quant.dequantize_f16(device)?; + let diff = (dst.to_dtype(DType::F16)? - dst_f16)? + .to_dtype(DType::F32)? + .abs()? + .sum_all()? + .to_vec0::()?; + assert_eq!(diff, 0.); + assert_eq!( + dst.to_vec1::()?, + &[ + -0.0, -0.0, 3.875, 3.875, 3.875, 3.875, 7.75, 7.75, 7.75, 7.75, 11.625, 11.625, 11.625, + 11.625, 15.5, 15.5, 15.5, 15.5, 19.375, 19.375, 19.375, 19.375, 23.25, 23.25, 23.25, + 23.25, 27.125, 27.125, 27.125, 27.125, 31.0, 31.0, 31.5, 31.5, 31.5, 31.5, 39.375, + 39.375, 39.375, 39.375, 39.375, 39.375, 39.375, 39.375, 47.25, 47.25, 47.25, 47.25, + 47.25, 47.25, 47.25, 47.25, 55.125, 55.125, 55.125, 55.125, 55.125, 55.125, 55.125, + 55.125, 63.0, 63.0, 63.0, 63.0, 59.375, 59.375, 71.25, 71.25, 71.25, 71.25, 71.25, + 71.25, 71.25, 71.25, 71.25, 71.25, 71.25, 71.25, 83.125, 83.125, 83.125, 83.125, + 83.125, 83.125, 83.125, 83.125, 83.125, 83.125, 83.125, 83.125, 95.0, 95.0, 95.0, 95.0, + 95.0, 95.0, 95.25, 95.25, 95.25, 95.25, 95.25, 95.25, 95.25, 95.25, 111.125, 111.125, + 111.125, 111.125, 111.125, 111.125, 111.125, 111.125, 111.125, 111.125, 111.125, + 111.125, 111.125, 111.125, 111.125, 111.125, 127.0, 127.0, 127.0, 127.0, 127.0, 127.0, + 127.0, 127.0 + ] + ); + ggml_quantization_error_test(GgmlDType::Q4_0, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; + Ok(()) +} + +fn quantize_q4_1(device: &Device) -> Result<()> { + let src = (0..32 * 4).map(|v| v as f32).collect::>(); + let src = Tensor::from_slice(&src, (32 * 4,), device)?; + let quant = quantized::QTensor::quantize(&src, GgmlDType::Q4_1)?; + let dst = quant.dequantize(device)?; + let dst_f16 = quant.dequantize_f16(device)?; + let diff = (dst.to_dtype(DType::F16)? - dst_f16)? + .to_dtype(DType::F32)? + .abs()? + .sum_all()? + .to_vec0::()?; + assert_eq!(diff, 0.); + assert_eq!( + round_vector(&dst.to_vec1::()?), + &[ + 0.0, 0.0, 2.066, 2.066, 4.133, 4.133, 6.199, 6.199, 8.266, 8.266, 10.332, 10.332, + 12.398, 12.398, 14.465, 14.465, 16.531, 16.531, 18.598, 18.598, 20.664, 20.664, 22.73, + 22.73, 24.797, 24.797, 26.863, 26.863, 28.93, 28.93, 30.996, 30.996, 32.0, 32.0, + 34.066, 34.066, 36.133, 36.133, 38.199, 38.199, 40.266, 40.266, 42.332, 42.332, 44.398, + 44.398, 46.465, 46.465, 48.531, 48.531, 50.598, 50.598, 52.664, 52.664, 54.73, 54.73, + 56.797, 56.797, 58.863, 58.863, 60.93, 60.93, 62.996, 62.996, 64.0, 64.0, 66.066, + 66.066, 68.133, 68.133, 70.199, 70.199, 72.266, 72.266, 74.332, 74.332, 76.398, 76.398, + 78.465, 78.465, 80.531, 80.531, 82.598, 82.598, 84.664, 84.664, 86.73, 86.73, 88.797, + 88.797, 90.863, 90.863, 92.93, 92.93, 94.996, 94.996, 96.0, 96.0, 98.066, 98.066, + 100.133, 100.133, 102.199, 102.199, 104.266, 104.266, 106.332, 106.332, 108.398, + 108.398, 110.465, 110.465, 112.531, 112.531, 114.598, 114.598, 116.664, 116.664, + 118.73, 118.73, 120.797, 120.797, 122.863, 122.863, 124.93, 124.93, 126.996, 126.996 + ] + ); + ggml_quantization_error_test(GgmlDType::Q4_1, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; + Ok(()) +} + +fn quantize_q5_0(device: &Device) -> Result<()> { + let src = (0..32 * 4).map(|v| v as f32).collect::>(); + let src = Tensor::from_slice(&src, (32 * 4,), device)?; + let quant = quantized::QTensor::quantize(&src, GgmlDType::Q5_0)?; + let dst = quant.dequantize(device)?; + let dst_f16 = quant.dequantize_f16(device)?; + let diff = (dst.to_dtype(DType::F16)? - dst_f16)? + .to_dtype(DType::F32)? + .abs()? + .sum_all()? + .to_vec0::()?; + assert_eq!(diff, 0.); + assert_eq!( + round_vector(&dst.to_vec1::()?), + &[ + -0.0, 1.938, 1.938, 3.875, 3.875, 5.813, 5.813, 7.75, 7.75, 9.688, 9.688, 11.625, + 11.625, 13.563, 13.563, 15.5, 15.5, 17.438, 17.438, 19.375, 19.375, 21.313, 21.313, + 23.25, 23.25, 25.188, 25.188, 27.125, 27.125, 29.063, 29.063, 31.0, 31.5, 31.5, 35.438, + 35.438, 35.438, 35.438, 39.375, 39.375, 39.375, 39.375, 43.313, 43.313, 43.313, 43.313, + 47.25, 47.25, 47.25, 47.25, 51.188, 51.188, 51.188, 51.188, 55.125, 55.125, 55.125, + 55.125, 59.063, 59.063, 59.063, 59.063, 63.0, 63.0, 65.313, 65.313, 65.313, 65.313, + 65.313, 71.25, 71.25, 71.25, 71.25, 71.25, 71.25, 77.188, 77.188, 77.188, 77.188, + 77.188, 77.188, 83.125, 83.125, 83.125, 83.125, 83.125, 83.125, 89.063, 89.063, 89.063, + 89.063, 89.063, 89.063, 95.0, 95.0, 95.0, 95.25, 95.25, 95.25, 95.25, 103.188, 103.188, + 103.188, 103.188, 103.188, 103.188, 103.188, 103.188, 111.125, 111.125, 111.125, + 111.125, 111.125, 111.125, 111.125, 111.125, 119.063, 119.063, 119.063, 119.063, + 119.063, 119.063, 119.063, 119.063, 127.0, 127.0, 127.0, 127.0 + ] + ); + ggml_quantization_error_test(GgmlDType::Q5_0, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; + Ok(()) +} + +fn quantize_q5_1(device: &Device) -> Result<()> { + let src = (0..32 * 4).map(|v| v as f32).collect::>(); + let src = Tensor::from_slice(&src, (32 * 4,), device)?; + let quant = quantized::QTensor::quantize(&src, GgmlDType::Q5_1)?; + let dst = quant.dequantize(device)?; + let dst_f16 = quant.dequantize_f16(device)?; + let diff = (dst.to_dtype(DType::F16)? - dst_f16)? + .to_dtype(DType::F32)? + .abs()? + .sum_all()? + .to_vec0::()?; + assert_eq!(diff, 0.); + assert_eq!( + round_vector(&dst.to_vec1::()?), + &[ + 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, + 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, + 30.0, 31.0, 32.0, 33.0, 34.0, 35.0, 36.0, 37.0, 38.0, 39.0, 40.0, 41.0, 42.0, 43.0, + 44.0, 45.0, 46.0, 47.0, 48.0, 49.0, 50.0, 51.0, 52.0, 53.0, 54.0, 55.0, 56.0, 57.0, + 58.0, 59.0, 60.0, 61.0, 62.0, 63.0, 64.0, 65.0, 66.0, 67.0, 68.0, 69.0, 70.0, 71.0, + 72.0, 73.0, 74.0, 75.0, 76.0, 77.0, 78.0, 79.0, 80.0, 81.0, 82.0, 83.0, 84.0, 85.0, + 86.0, 87.0, 88.0, 89.0, 90.0, 91.0, 92.0, 93.0, 94.0, 95.0, 96.0, 97.0, 98.0, 99.0, + 100.0, 101.0, 102.0, 103.0, 104.0, 105.0, 106.0, 107.0, 108.0, 109.0, 110.0, 111.0, + 112.0, 113.0, 114.0, 115.0, 116.0, 117.0, 118.0, 119.0, 120.0, 121.0, 122.0, 123.0, + 124.0, 125.0, 126.0, 127.0 + ] + ); + ggml_quantization_error_test(GgmlDType::Q5_1, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; + Ok(()) +} + +fn get_test_vector2(bound: f32, size: usize, device: &Device) -> Result { + assert!( + size.is_multiple_of(crate::quantized::k_quants::QK_K), + "size must be a multiple of {}", + crate::quantized::k_quants::QK_K + ); + + let src = (0..size) + .map(|v| (v as f32 - size as f32 / 2.) * bound / (size as f32 / 2.)) + .collect::>(); + assert_eq!([src[0], src[size / 2]], [-bound, 0.0]); + Tensor::from_vec(src, (size,), device) +} + +/// Round a vector +fn round_vector(values: &[f32]) -> Vec { + values + .iter() + .map(|x| (1000. * x).round() / 1000.) + .collect::>() +} + +fn compare_with_error(values: &[f32], expected: &[f32], tolerance: f32) { + for (i, (value, expected_value)) in values.iter().zip(expected.iter()).enumerate() { + let difference = (value - expected_value).abs(); + + assert!( + difference < tolerance, + "Error at index {i}: value = {value}, expected = {expected_value}. Difference = {difference} exceeds tolerance = {tolerance}." + ); + } +} + +/// Creates a vector similar to the ones used in GGML unit tests: +/// https://github.com/ggerganov/llama.cpp/blob/master/tests/test-quantize-fns.cpp#L26-L30 +fn create_ggml_like_vector(offset: f32) -> Vec { + (0..GGML_TEST_SIZE) + .map(|i| 0.1 + 2.0 * (i as f32 + offset).cos()) + .collect() +} + +/// Calculates the root mean square error between two vectors +fn calculate_rmse(a: &[f32], b: &[f32]) -> f32 { + assert_eq!(a.len(), b.len()); + let sum = a + .iter() + .zip(b) + .map(|(a, b)| (a - b).powi(2)) + .sum::() + .sqrt(); + sum / a.len() as f32 +} + +/// Similar to the GGML quantization unit test: +/// https://github.com/ggerganov/llama.cpp/blob/master/tests/test-quantize-fns.cpp#L43-L50 +fn ggml_quantization_error_test(dtype: GgmlDType, device: &Device, max_error: f32) -> Result<()> { + let src = create_ggml_like_vector(0.0); + let src = Tensor::from_slice(&src, (GGML_TEST_SIZE,), device)?; + let quant = quantized::QTensor::quantize(&src, dtype)?; + let dst = quant.dequantize(device)?; + let dst_f16 = quant.dequantize_f16(device)?; + let diff = (dst.to_dtype(DType::F16)? - dst_f16)? + .to_dtype(DType::F32)? + .abs()? + .sum_all()? + .to_vec0::()?; + assert_eq!(diff, 0.); + let error = calculate_rmse(&src.to_vec1::()?, &dst.to_vec1::()?); + if error > max_error { + bail!( + "Quantization error {} exceeds max error {}", + error, + max_error + ); + } + Ok(()) +} + +#[test] +fn imatrix_quantize_q6k() -> Result<()> { + let cpu = &Device::Cpu; + + let mut row_counts = 0f64; + let mut ncall = 0f64; + let mut values = Tensor::zeros((768,), DType::F32, cpu)?; + + for _ in 0..10 { + let lhs = Var::from_tensor(&Tensor::randn(0f32, 1f32, (1024, 512), cpu)?)?; + let rhs = Var::from_tensor(&Tensor::randn(0f32, 1f32, (512, 768), cpu)?)?; + let res = lhs.matmul(&rhs)?; + + // https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/examples/imatrix/imatrix.cpp#L180-L186 + values = (values + res.sqr()?.sum(0)?)?; + row_counts += res.dim(0)? as f64; + ncall += 1.; + } + + // https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/examples/imatrix/imatrix.cpp#L275 + let out = ((values / row_counts)? * ncall)?; + let imatrix = out.to_vec1::()?; + + let xs = Tensor::randn(0f32, 1f32, (1024, 768), cpu)?; + + let quant1 = quantized::QTensor::quantize(&xs, GgmlDType::Q6K)?; + let quant2 = quantized::QTensor::quantize_imatrix(&xs, &imatrix, GgmlDType::Q6K)?; + + let dequant1 = quant1.dequantize(cpu)?; + let dequant2 = quant2.dequantize(cpu)?; + + let err1 = (dequant1 - &xs)?.abs()?.mean_all()?.to_scalar::()?; + let err2 = (dequant2 - &xs)?.abs()?.mean_all()?.to_scalar::()?; + assert!(err2 < err1, "err2 {err2} > err1 {err1}"); + + Ok(()) +} + +#[test] +fn imatrix_quantize_q5k() -> Result<()> { + let cpu = &Device::Cpu; + + let mut row_counts = 0f64; + let mut ncall = 0f64; + let mut values = Tensor::zeros((768,), DType::F32, cpu)?; + + for _ in 0..10 { + let lhs = Var::from_tensor(&Tensor::randn(0f32, 1f32, (1024, 512), cpu)?)?; + let rhs = Var::from_tensor(&Tensor::randn(0f32, 1f32, (512, 768), cpu)?)?; + let res = lhs.matmul(&rhs)?; + + // https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/examples/imatrix/imatrix.cpp#L180-L186 + values = (values + res.sqr()?.sum(0)?)?; + row_counts += res.dim(0)? as f64; + ncall += 1.; + } + + // https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/examples/imatrix/imatrix.cpp#L275 + let out = ((values / row_counts)? * ncall)?; + let imatrix = out.to_vec1::()?; + + let xs = Tensor::randn(0f32, 1f32, (1024, 768), cpu)?; + + let quant1 = quantized::QTensor::quantize(&xs, GgmlDType::Q5K)?; + let quant2 = quantized::QTensor::quantize_imatrix(&xs, &imatrix, GgmlDType::Q5K)?; + + let dequant1 = quant1.dequantize(cpu)?; + let dequant2 = quant2.dequantize(cpu)?; + + let err1 = (dequant1 - &xs)?.abs()?.mean_all()?.to_scalar::()?; + let err2 = (dequant2 - &xs)?.abs()?.mean_all()?.to_scalar::()?; + assert!(err2 < err1, "err2 {err2} > err1 {err1}"); + + Ok(()) +} + +#[test] +fn imatrix_quantize_q4k() -> Result<()> { + // let data = + // quantized::imatrix_file::load_imatrix("../Llama-3.2-3B-Instruct.imatrix").unwrap(); + // for (name, weights) in &data { + // println!("{name}, {} elems", weights.len()); + // } + // dbg!(&data["blk.0.attn_q.weight"].len()); + + let cpu = &Device::Cpu; + + let mut row_counts = 0f64; + let mut ncall = 0f64; + let mut values = Tensor::zeros((768,), DType::F32, cpu)?; + + for _ in 0..10 { + let lhs = Var::from_tensor(&Tensor::randn(0f32, 1f32, (1024, 512), cpu)?)?; + let rhs = Var::from_tensor(&Tensor::randn(0f32, 1f32, (512, 768), cpu)?)?; + let res = lhs.matmul(&rhs)?; + + // https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/examples/imatrix/imatrix.cpp#L180-L186 + values = (values + res.sqr()?.sum(0)?)?; + row_counts += res.dim(0)? as f64; + ncall += 1.; + } + + // https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/examples/imatrix/imatrix.cpp#L275 + let out = ((values / row_counts)? * ncall)?; + let imatrix = out.to_vec1::()?; + + let xs = Tensor::randn(0f32, 1f32, (1024, 768), cpu)?; + + let quant1 = quantized::QTensor::quantize(&xs, GgmlDType::Q4K)?; + let quant2 = quantized::QTensor::quantize_imatrix(&xs, &imatrix, GgmlDType::Q4K)?; + + let dequant1 = quant1.dequantize(cpu)?; + let dequant2 = quant2.dequantize(cpu)?; + + let err1 = (dequant1 - &xs)?.abs()?.mean_all()?.to_scalar::()?; + let err2 = (dequant2 - &xs)?.abs()?.mean_all()?.to_scalar::()?; + assert!(err2 < err1, "err2 {err2} > err1 {err1}"); + + Ok(()) +} + +#[test] +fn imatrix_quantize_q3k() -> Result<()> { + let cpu = &Device::Cpu; + + let mut row_counts = 0f64; + let mut ncall = 0f64; + let mut values = Tensor::zeros((768,), DType::F32, cpu)?; + + for _ in 0..10 { + let lhs = Var::from_tensor(&Tensor::randn(0f32, 1f32, (1024, 512), cpu)?)?; + let rhs = Var::from_tensor(&Tensor::randn(0f32, 1f32, (512, 768), cpu)?)?; + let res = lhs.matmul(&rhs)?; + + // https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/examples/imatrix/imatrix.cpp#L180-L186 + values = (values + res.sqr()?.sum(0)?)?; + row_counts += res.dim(0)? as f64; + ncall += 1.; + } + + // https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/examples/imatrix/imatrix.cpp#L275 + let out = ((values / row_counts)? * ncall)?; + let imatrix = out.to_vec1::()?; + + let xs = Tensor::randn(0f32, 1f32, (1024, 768), cpu)?; + + let quant1 = quantized::QTensor::quantize(&xs, GgmlDType::Q3K)?; + let quant2 = quantized::QTensor::quantize_imatrix(&xs, &imatrix, GgmlDType::Q3K)?; + + let dequant1 = quant1.dequantize(cpu)?; + let dequant2 = quant2.dequantize(cpu)?; + + let err1 = (dequant1 - &xs)?.abs()?.mean_all()?.to_scalar::()?; + let err2 = (dequant2 - &xs)?.abs()?.mean_all()?.to_scalar::()?; + assert!(err2 < err1, "err2 {err2} > err1 {err1}"); + + Ok(()) +} + +#[test] +fn imatrix_quantize_q2k() -> Result<()> { + let cpu = &Device::Cpu; + + let mut row_counts = 0f64; + let mut ncall = 0f64; + let mut values = Tensor::zeros((768,), DType::F32, cpu)?; + + for _ in 0..10 { + let lhs = Var::from_tensor(&Tensor::randn(0f32, 1f32, (1024, 512), cpu)?)?; + let rhs = Var::from_tensor(&Tensor::randn(0f32, 1f32, (512, 768), cpu)?)?; + let res = lhs.matmul(&rhs)?; + + // https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/examples/imatrix/imatrix.cpp#L180-L186 + values = (values + res.sqr()?.sum(0)?)?; + row_counts += res.dim(0)? as f64; + ncall += 1.; + } + + // https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/examples/imatrix/imatrix.cpp#L275 + let out = ((values / row_counts)? * ncall)?; + let imatrix = out.to_vec1::()?; + + let xs = Tensor::randn(0f32, 1f32, (1024, 768), cpu)?; + + let quant1 = quantized::QTensor::quantize(&xs, GgmlDType::Q2K)?; + let quant2 = quantized::QTensor::quantize_imatrix(&xs, &imatrix, GgmlDType::Q2K)?; + + let dequant1 = quant1.dequantize(cpu)?; + let dequant2 = quant2.dequantize(cpu)?; + + let err1 = (dequant1 - &xs)?.abs()?.mean_all()?.to_scalar::()?; + let err2 = (dequant2 - &xs)?.abs()?.mean_all()?.to_scalar::()?; + assert!(err2 < err1, "err2 {err2} > err1 {err1}"); + + Ok(()) +} + +fn quantize_q2k(device: &Device) -> Result<()> { + let dtype = GgmlDType::Q2K; + + let src = get_test_vector2(0.5, 1024, device)?; + let quant = quantized::QTensor::quantize(&src, dtype)?; + let dst = quant.dequantize(device)?; + let dst_f16 = quant.dequantize_f16(device)?; + let diff = (dst.to_dtype(DType::F16)? - dst_f16)? + .to_dtype(DType::F32)? + .abs()? + .sum_all()? + .to_vec0::()?; + assert_eq!(diff, 0.); + + let src = src.to_vec1::()?; + let dst = dst.to_vec1::()?; + compare_with_error(dst.as_slice(), src.as_slice(), 0.1); + + // Test some specific values + assert_eq!( + [src[0], src[128], src[256], src[512], src[800], src[1023]], + [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] + ); + let dst = round_vector(&dst); + assert_eq!( + [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], + [-0.499, -0.366, -0.249, 0.0, 0.295, 0.492] + ); + + let src_big = get_test_vector2(128.0, 1024, device)?; + let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; + let dst_big = quant_big.dequantize(device)?; + let dst_big_f16 = quant_big.dequantize_f16(device)?; + let diff = (dst_big.to_dtype(DType::F16)? - dst_big_f16)? + .to_dtype(DType::F32)? + .abs()? + .sum_all()? + .to_vec0::()?; + assert_eq!(diff, 0.); + + let src_big = src_big.to_vec1::()?; + let dst_big = dst_big.to_vec1::()?; + compare_with_error(dst_big.as_slice(), src_big.as_slice(), 6.0); + + ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR_2BITS)?; + Ok(()) +} + +fn quantize_q3k(device: &Device) -> Result<()> { + let dtype = GgmlDType::Q3K; + let src = get_test_vector2(0.5, 1024, device)?; + let quant = quantized::QTensor::quantize(&src, dtype)?; + let dst = quant.dequantize(device)?; + let dst_f16 = quant.dequantize_f16(device)?; + let diff = (dst.to_dtype(DType::F16)? - dst_f16)? + .to_dtype(DType::F32)? + .abs()? + .sum_all()? + .to_vec0::()?; + assert_eq!(diff, 0.); + + let src = src.to_vec1::()?; + let dst = dst.to_vec1::()?; + compare_with_error(dst.as_slice(), src.as_slice(), 0.03); + + // Test some specific values + assert_eq!( + [src[0], src[128], src[256], src[512], src[800], src[1023]], + [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] + ); + let dst = round_vector(&dst); + assert_eq!( + [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], + [-0.493, -0.37, -0.243, -0.0, 0.292, 0.492] + ); + + let src_big = get_test_vector2(128.0, 1024, device)?; + let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; + let dst_big = quant_big.dequantize(device)?; + let dst_big_f16 = quant_big.dequantize_f16(device)?; + let diff = (dst_big.to_dtype(DType::F16)? - dst_big_f16)? + .to_dtype(DType::F32)? + .abs()? + .sum_all()? + .to_vec0::()?; + assert_eq!(diff, 0.); + + let src_big = src_big.to_vec1::()?; + let dst_big = dst_big.to_vec1::()?; + compare_with_error(dst_big.as_slice(), src_big.as_slice(), 3.5); + + ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR_3BITS)?; + Ok(()) +} + +fn quantize_q4k(device: &Device) -> Result<()> { + let dtype = GgmlDType::Q4K; + let src = get_test_vector2(0.5, 1024, device)?; + let quant = quantized::QTensor::quantize(&src, dtype)?; + let dst = quant.dequantize(device)?; + let dst_f16 = quant.dequantize_f16(device)?; + let diff = (dst.to_dtype(DType::F16)? - dst_f16)? + .to_dtype(DType::F32)? + .abs()? + .sum_all()? + .to_vec0::()?; + assert_eq!(diff, 0.); + + let src = src.to_vec1::()?; + let dst = dst.to_vec1::()?; + compare_with_error(dst.as_slice(), src.as_slice(), 0.017); + + // Test some specific values + assert_eq!( + [src[0], src[128], src[256], src[512], src[800], src[1023]], + [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] + ); + let dst = round_vector(&dst); + assert_eq!( + [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], + [-0.5, -0.373, -0.25, 0.0, 0.288, 0.498] + ); + + let src_big = get_test_vector2(128.0, 1024, device)?; + let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; + let dst_big = quant_big.dequantize(device)?; + let dst_big_f16 = quant_big.dequantize_f16(device)?; + let diff = (dst_big.to_dtype(DType::F16)? - dst_big_f16)? + .to_dtype(DType::F32)? + .abs()? + .sum_all()? + .to_vec0::()?; + assert_eq!(diff, 0.); + + let src_big = src_big.to_vec1::()?; + let dst_big = dst_big.to_vec1::()?; + compare_with_error(dst_big.as_slice(), src_big.as_slice(), 4.5); + + ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; + Ok(()) +} + +fn quantize_q5k(device: &Device) -> Result<()> { + let dtype = GgmlDType::Q5K; + let src = get_test_vector2(0.5, 1024, device)?; + let quant = quantized::QTensor::quantize(&src, dtype)?; + let dst = quant.dequantize(device)?; + let dst_f16 = quant.dequantize_f16(device)?; + let diff = (dst.to_dtype(DType::F16)? - dst_f16)? + .to_dtype(DType::F32)? + .abs()? + .sum_all()? + .to_vec0::()?; + assert_eq!(diff, 0.); + + let src = src.to_vec1::()?; + let dst = dst.to_vec1::()?; + compare_with_error(dst.as_slice(), src.as_slice(), 0.009); + + // Test some specific values + assert_eq!( + [src[0], src[128], src[256], src[512], src[800], src[1023]], + [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] + ); + let dst = round_vector(&dst); + assert_eq!( + [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], + [-0.5, -0.373, -0.25, 0.0, 0.279, 0.499] + ); + + let src_big = get_test_vector2(128.0, 1024, device)?; + let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; + let dst_big = quant_big.dequantize(device)?; + let dst_big_f16 = quant_big.dequantize_f16(device)?; + let diff = (dst_big.to_dtype(DType::F16)? - dst_big_f16)? + .to_dtype(DType::F32)? + .abs()? + .sum_all()? + .to_vec0::()?; + assert_eq!(diff, 0.); + + let src_big = src_big.to_vec1::()?; + let dst_big = dst_big.to_vec1::()?; + compare_with_error(dst_big.as_slice(), src_big.as_slice(), 2.5); + + ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; + Ok(()) +} + +fn quantize_q6k(device: &Device) -> Result<()> { + let dtype = GgmlDType::Q6K; + let src = get_test_vector2(0.5, 1024, device)?; + let quant = quantized::QTensor::quantize(&src, dtype)?; + let dst = quant.dequantize(device)?; + let dst_f16 = quant.dequantize_f16(device)?; + let diff = (dst.to_dtype(DType::F16)? - dst_f16)? + .to_dtype(DType::F32)? + .abs()? + .sum_all()? + .to_vec0::()?; + assert_eq!(diff, 0.); + + let src = src.to_vec1::()?; + let dst = dst.to_vec1::()?; + compare_with_error(dst.as_slice(), src.as_slice(), 0.008); + + // Test some specific values + assert_eq!( + [src[0], src[128], src[256], src[512], src[800], src[1023]], + [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] + ); + let dst = round_vector(&dst); + assert_eq!( + [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], + [-0.497, -0.372, -0.25, -0.0, 0.284, 0.5] + ); + + let src_big = get_test_vector2(128.0, 1024, device)?; + let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; + let dst_big = quant_big.dequantize(device)?; + let dst_big_f16 = quant_big.dequantize_f16(device)?; + let diff = (dst_big.to_dtype(DType::F16)? - dst_big_f16)? + .to_dtype(DType::F32)? + .abs()? + .sum_all()? + .to_vec0::()?; + assert_eq!(diff, 0.); + + let src_big = src_big.to_vec1::()?; + let dst_big = dst_big.to_vec1::()?; + compare_with_error(dst_big.as_slice(), src_big.as_slice(), 2.0); + + ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; + Ok(()) +} + +fn quantize_q8k(device: &Device) -> Result<()> { + let dtype = GgmlDType::Q8K; + let src = get_test_vector2(0.5, 1024, device)?; + let quant = quantized::QTensor::quantize(&src, dtype)?; + let dst = quant.dequantize(device)?; + let dst_f16 = quant.dequantize_f16(device)?; + let diff = (dst.to_dtype(DType::F16)? - dst_f16)? + .to_dtype(DType::F32)? + .abs()? + .sum_all()? + .to_vec0::()?; + assert_eq!(diff, 0.); + + let src = src.to_vec1::()?; + let dst = dst.to_vec1::()?; + compare_with_error(dst.as_slice(), src.as_slice(), 0.008); + + // Test some specific values + assert_eq!( + [src[0], src[128], src[256], src[512], src[800], src[1023]], + [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] + ); + let dst = round_vector(&dst); + assert_eq!( + [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], + [-0.5, -0.375, -0.25, -0.0, 0.281, 0.499] + ); + + let src_big = get_test_vector2(128.0, 1024, device)?; + let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; + let dst_big = quant_big.dequantize(device)?; + let dst_big_f16 = quant_big.dequantize_f16(device)?; + let diff = (dst_big.to_dtype(DType::F16)? - dst_big_f16)? + .to_dtype(DType::F32)? + .abs()? + .sum_all()? + .to_vec0::()?; + assert_eq!(diff, 0.); + + let src_big = src_big.to_vec1::()?; + let dst_big = dst_big.to_vec1::()?; + compare_with_error(dst_big.as_slice(), src_big.as_slice(), 0.6); + + ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; + Ok(()) +} + +test_device!( + quantize_q4_0, + quantize_q4_0_cpu, + quantize_q4_0_cuda, + quantize_q4_0_metal +); +test_device!( + quantize_q4_1, + quantize_q4_1_cpu, + quantize_q4_1_cuda, + quantize_q4_1_metal +); +test_device!( + quantize_q5_0, + quantize_q5_0_cpu, + quantize_q5_0_cuda, + quantize_q5_0_metal +); +test_device!( + quantize_q5_1, + quantize_q5_1_cpu, + quantize_q5_1_cuda, + quantize_q5_1_metal +); +test_device!( + quantize_q2k, + quantize_q2k_cpu, + quantize_q2k_cuda, + quantize_q2k_metal +); +test_device!( + quantize_q3k, + quantize_q3k_cpu, + quantize_q3k_cuda, + quantize_q3k_metal +); +test_device!( + quantize_q4k, + quantize_q4k_cpu, + quantize_q4k_cuda, + quantize_q4k_metal +); +test_device!( + quantize_q5k, + quantize_q5k_cpu, + quantize_q5k_cuda, + quantize_q5k_metal +); +test_device!( + quantize_q6k, + quantize_q6k_cpu, + quantize_q6k_cuda, + quantize_q6k_metal +); +test_device!( + quantize_q8k, + quantize_q8k_cpu, + quantize_q8k_cuda, + quantize_q8k_metal +); + +/// Very simple dot product implementation +fn vec_dot_reference(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b).map(|(a, b)| a * b).sum() +} + +/// Returns the error achieved by the GGML matmul unit test. +fn ggml_reference_matmul_error(dtype: GgmlDType) -> Result { + let err = match dtype { + GgmlDType::F32 => 0.000000, + GgmlDType::F16 => 0.000010, + GgmlDType::BF16 => 0.000200, + GgmlDType::Q2K => 0.004086, + GgmlDType::Q3K => 0.016148, + GgmlDType::Q4K => 0.002425, + GgmlDType::Q5K => 0.000740, + GgmlDType::Q6K => 0.000952, + GgmlDType::Q4_0 => 0.001143, + GgmlDType::Q4_1 => 0.008, + GgmlDType::Q5_0 => 0.001353, + GgmlDType::Q5_1 => 0.00149, + GgmlDType::Q8_0 => 0.000092, + GgmlDType::Q8_1 => 0.000092, + + // Not from the ggml repo. + GgmlDType::Q8K => 0.00065, + }; + Ok(err) +} + +/// Similar to the GGML matmul unit test: +/// https://github.com/ggerganov/llama.cpp/blob/master/tests/test-quantize-fns.cpp#L76-L91 +fn ggml_matmul_error_test() -> Result<()> { + let a = create_ggml_like_vector(0.0); + let b = create_ggml_like_vector(1.0); + ggml_matmul_error_test_::(a.as_slice(), b.as_slice(), 1.0)?; + // Another example that is more likely to trigger the overflow reported in #1526 + let a = (0..GGML_TEST_SIZE) + .map(|i| i as f32 / GGML_TEST_SIZE as f32) + .collect::>(); + let b = (0..GGML_TEST_SIZE) + .map(|i| i as f32 / GGML_TEST_SIZE as f32) + .collect::>(); + ggml_matmul_error_test_::(a.as_slice(), b.as_slice(), 2.0)?; + Ok(()) +} + +fn ggml_matmul_error_test_(a: &[f32], b: &[f32], err_m: f32) -> Result<()> { + let length = a.len(); + + let mut a_quant = vec![T::zeros(); length / T::BLCK_SIZE]; + let mut b_quant = vec![T::VecDotType::zeros(); length / T::VecDotType::BLCK_SIZE]; + T::from_float(a, &mut a_quant); + T::VecDotType::from_float(b, &mut b_quant); + + let result = T::vec_dot(length, &a_quant, &b_quant); + let result_unopt = T::vec_dot_unopt(length, &a_quant, &b_quant); + + if (result - result_unopt).abs() / length as f32 > 1e-6 { + bail!( + "the opt and unopt vec-dot returned different values, opt: {result} vs unopt: {result_unopt}" + ) + } + + let mut dst = vec![0.0f32; 1]; + crate::k_quants::matmul((1, length, 1), b, &a_quant, &mut dst)?; + let result_matmul = dst[0]; + + if (result_matmul - result).abs() / length as f32 > 1e-6 { + bail!( + "calling matmul vs calling vec-dot directly returned different values, matmul: {result_matmul} vs vec-dot: {result}" + ) + } + + let reference_result = vec_dot_reference(a, b); + + let verify_result = |result: f32, source: &str| { + let error = (result - reference_result).abs() / length as f32; + let ggml_error = ggml_reference_matmul_error(T::DTYPE)? * err_m; + if !error.is_finite() || error > GGML_MAX_DOT_PRODUCT_ERROR { + bail!("Dot product with dtype {:?} error {error} exceeds max error {GGML_MAX_DOT_PRODUCT_ERROR}. Source: {source}", T::DTYPE); + } + // We diverge slightly due to different rounding behavior / f16 to f32 conversions in GGML + // => we use a slightly higher error threshold + const ERROR_LENIENCY: f32 = 0.00001; + if error - ERROR_LENIENCY > ggml_error { + bail!( + "Dot product with dtype {:?} error {error} exceeds ggml reference error {ggml_error}. Source: {source}", + T::DTYPE, + ); + } + Ok(()) + }; + + verify_result(result, "vec-dot")?; + verify_result(result_matmul, "matmul")?; + Ok(()) +} + +#[test] +fn quantized_mm() -> Result<()> { + ggml_matmul_error_test::()?; + ggml_matmul_error_test::()?; + //ggml_matmul_error_test::()?; TODO: Fails on ubuntu and windows. Check CpuBF16 impl + ggml_matmul_error_test::()?; + ggml_matmul_error_test::()?; + ggml_matmul_error_test::()?; + ggml_matmul_error_test::()?; + ggml_matmul_error_test::()?; + ggml_matmul_error_test::()?; + Ok(()) +} + +/// generates random tensors of size `m x k` and `n x k` and calculates their expected matrix multiplication result. +fn get_random_tensors( + m: usize, + k: usize, + n: usize, + device: &Device, +) -> Result<(Tensor, Tensor, Tensor)> { + let mut rng = StdRng::seed_from_u64(314159265358979); + + let lhs = (0..m * k) + .map(|_| rng.random::() - 0.5) + .collect::>(); + let rhs = (0..n * k) + .map(|_| rng.random::() - 0.5) + .collect::>(); + + let lhs = Tensor::from_vec(lhs, (m, k), device)?; + let rhs = Tensor::from_vec(rhs, (n, k), device)?; + + let mm = lhs.matmul(&rhs.t()?)?; + Ok((lhs, rhs, mm)) +} + +#[macro_export] +macro_rules! quantized_matmul { + // TODO: Switch to generating the two last arguments automatically once concat_idents is + // stable. https://github.com/rust-lang/rust/issues/29599 + ($fn_name: ident, $fn_name_cpu: ident, $fn_name_cuda: ident, $fn_name_metal: ident, $dtype: expr) => { + fn $fn_name(device: &Device) -> Result<()> { + test_matmul(device, (1, 3, 4, 256), $dtype)?; + Ok(()) + } + + test_device!($fn_name, $fn_name_cpu, $fn_name_cuda, $fn_name_metal); + }; +} + +quantized_matmul!( + quantized_matmul_q4_0_bis, + quantized_matmul_q4_0_cpu, + quantized_matmul_q4_0_cuda, + quantized_matmul_q4_0_metal, + GgmlDType::Q4_0 +); +quantized_matmul!( + quantized_matmul_q4_1_bis, + quantized_matmul_q4_1_cpu, + quantized_matmul_q4_1_cuda, + quantized_matmul_q4_1_metal, + GgmlDType::Q4_1 +); +quantized_matmul!( + quantized_matmul_q5_0_bis, + quantized_matmul_q5_0_cpu, + quantized_matmul_q5_0_cuda, + quantized_matmul_q5_0_metal, + GgmlDType::Q5_0 +); +quantized_matmul!( + quantized_matmul_q5_1_bis, + quantized_matmul_q5_1_cpu, + quantized_matmul_q5_1_cuda, + quantized_matmul_q5_1_metal, + GgmlDType::Q5_1 +); +quantized_matmul!( + quantized_matmul_q8_0_bis, + quantized_matmul_q8_0_cpu, + quantized_matmul_q8_0_cuda, + quantized_matmul_q8_0_metal, + GgmlDType::Q8_0 +); +quantized_matmul!( + quantized_matmul_q8_1_bis, + quantized_matmul_q8_1_cpu, + quantized_matmul_q8_1_cuda, + quantized_matmul_q8_1_metal, + GgmlDType::Q8_1 +); +quantized_matmul!( + quantized_matmul_q2k_bis, + quantized_matmul_q2k_cpu, + quantized_matmul_q2k_cuda, + quantized_matmul_q2k_metal, + GgmlDType::Q2K +); +quantized_matmul!( + quantized_matmul_q3k_bis, + quantized_matmul_q3k_cpu, + quantized_matmul_q3k_cuda, + quantized_matmul_q3k_metal, + GgmlDType::Q3K +); +quantized_matmul!( + quantized_matmul_q4k_bis, + quantized_matmul_q4k_cpu, + quantized_matmul_q4k_cuda, + quantized_matmul_q4k_metal, + GgmlDType::Q4K +); +quantized_matmul!( + quantized_matmul_q5k_bis, + quantized_matmul_q5k_cpu, + quantized_matmul_q5k_cuda, + quantized_matmul_q5k_metal, + GgmlDType::Q5K +); +quantized_matmul!( + quantized_matmul_q6k_bis, + quantized_matmul_q6k_cpu, + quantized_matmul_q6k_cuda, + quantized_matmul_q6k_metal, + GgmlDType::Q6K +); +// Not implemented on metal +quantized_matmul!( + quantized_matmul_q8k_bis, + quantized_matmul_q8k_cpu, + quantized_matmul_q8k_cuda, + quantized_matmul_q8k_metal, + GgmlDType::Q8K +); + +#[test] +fn quantized_matmul_q2k() -> Result<()> { + use k_quants::BlockQ2K; + + let cpu = &Device::Cpu; + let (m, k, n) = (11, 512, 21); + let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); + + let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q2K)?; + let rhs = quantized::QMatMul::from_qtensor(rhs)?; + let mm = rhs.forward(&lhs)?; + + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [0.916, 0.422, 0.215, 1.668]); + + ggml_matmul_error_test::()?; + + Ok(()) +} + +#[test] +fn quantized_matmul_q3k() -> Result<()> { + use k_quants::BlockQ3K; + + let cpu = &Device::Cpu; + let (m, k, n) = (11, 512, 21); + let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); + + let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q3K)?; + let rhs = quantized::QMatMul::from_qtensor(rhs)?; + let mm = rhs.forward(&lhs)?; + + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [1.029, 1.418, -0.314, 1.495]); + + ggml_matmul_error_test::()?; + + Ok(()) +} + +#[test] +fn quantized_matmul_q4k() -> Result<()> { + use k_quants::BlockQ4K; + + let cpu = &Device::Cpu; + let (m, k, n) = (11, 512, 21); + let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); + + let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q4K)?; + let rhs = quantized::QMatMul::from_qtensor(rhs)?; + let mm = rhs.forward(&lhs)?; + + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [1.125, 1.435, -0.201, 1.589]); + + ggml_matmul_error_test::()?; + + Ok(()) +} + +#[test] +fn quantized_matmul_q5k() -> Result<()> { + use k_quants::BlockQ5K; + + let cpu = &Device::Cpu; + let (m, k, n) = (11, 512, 21); + let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); + + let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q5K)?; + let rhs = quantized::QMatMul::from_qtensor(rhs)?; + let mm = rhs.forward(&lhs)?; + + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [1.192, 1.491, -0.18, 1.743]); + + //Expected: 0.000740408897 + ggml_matmul_error_test::()?; + + Ok(()) +} + +#[test] +fn quantized_matmul_q6k() -> Result<()> { + use k_quants::BlockQ6K; + + let cpu = &Device::Cpu; + let (m, k, n) = (11, 512, 21); + let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); + + let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q6K)?; + let rhs = quantized::QMatMul::from_qtensor(rhs)?; + let mm = rhs.forward(&lhs)?; + + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [1.324, 1.49, -0.164, 1.741]); + + ggml_matmul_error_test::()?; + Ok(()) +} + +#[test] +fn quantized_matmul_q8k() -> Result<()> { + use k_quants::BlockQ8K; + + let cpu = &Device::Cpu; + let (m, k, n) = (11, 512, 21); + let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); + + let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q8K)?; + let rhs = quantized::QMatMul::from_qtensor(rhs)?; + let mm = rhs.forward(&lhs)?; + + assert_eq!(mm.dims(), [m, n]); + let dst = mm.flatten_all()?.to_vec1::()?; + let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); + assert_eq!(dst, [1.266, 1.504, -0.204, 1.7]); + + ggml_matmul_error_test::()?; + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/serialization_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/serialization_tests.rs new file mode 100644 index 0000000000..f81350e6e1 --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/tests/serialization_tests.rs @@ -0,0 +1,71 @@ +use candle_core::{DType, Result, Tensor}; + +struct TmpFile(std::path::PathBuf); + +impl TmpFile { + fn create(base: &str) -> TmpFile { + let filename = std::env::temp_dir().join(format!( + "candle-{}-{}-{:?}", + base, + std::process::id(), + std::thread::current().id(), + )); + TmpFile(filename) + } +} + +impl std::convert::AsRef for TmpFile { + fn as_ref(&self) -> &std::path::Path { + self.0.as_path() + } +} + +impl Drop for TmpFile { + fn drop(&mut self) { + std::fs::remove_file(&self.0).unwrap() + } +} + +#[test] +fn npy() -> Result<()> { + let npy = Tensor::read_npy("tests/test.npy")?; + assert_eq!( + npy.to_dtype(DType::U8)?.to_vec1::()?, + [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] + ); + Ok(()) +} + +#[test] +fn npz() -> Result<()> { + let npz = Tensor::read_npz("tests/test.npz")?; + assert_eq!(npz.len(), 2); + assert_eq!(npz[0].0, "x"); + assert_eq!(npz[1].0, "x_plus_one"); + assert_eq!( + npz[1].1.to_dtype(DType::U8)?.to_vec1::()?, + [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + ); + Ok(()) +} + +#[test] +fn safetensors() -> Result<()> { + use candle_core::safetensors::Load; + + let tmp_file = TmpFile::create("st"); + let t = Tensor::arange(0f32, 24f32, &candle_core::Device::Cpu)?; + t.save_safetensors("t", &tmp_file)?; + // Load from file. + let st = candle_core::safetensors::load(&tmp_file, &candle_core::Device::Cpu)?; + let t2 = st.get("t").unwrap(); + let diff = (&t - t2)?.abs()?.sum_all()?.to_vec0::()?; + assert_eq!(diff, 0f32); + // Load from bytes. + let bytes = std::fs::read(tmp_file)?; + let st = candle_core::safetensors::SliceSafetensors::new(&bytes)?; + let t2 = st.get("t").unwrap().load(&candle_core::Device::Cpu); + let diff = (&t - t2)?.abs()?.sum_all()?.to_vec0::()?; + assert_eq!(diff, 0f32); + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/tensor_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/tensor_tests.rs new file mode 100644 index 0000000000..5422353b0e --- /dev/null +++ b/v2/dependency-patches/candle-core-0.9.2/tests/tensor_tests.rs @@ -0,0 +1,2062 @@ +use candle_core::{test_device, test_utils, DType, Device, IndexOp, Result, Tensor, D}; +use float8::F8E4M3; + +fn zeros(device: &Device) -> Result<()> { + let tensor = Tensor::zeros((5, 2), DType::F32, device)?; + let (dim1, dim2) = tensor.dims2()?; + assert_eq!(dim1, 5); + assert_eq!(dim2, 2); + Ok(()) +} + +fn ones(device: &Device) -> Result<()> { + assert_eq!( + Tensor::ones((2, 3), DType::U8, device)?.to_vec2::()?, + [[1, 1, 1], [1, 1, 1]], + ); + assert_eq!( + Tensor::ones((2, 3), DType::U32, device)?.to_vec2::()?, + [[1, 1, 1], [1, 1, 1]], + ); + assert_eq!( + Tensor::ones((2, 3), DType::I64, device)?.to_vec2::()?, + [[1, 1, 1], [1, 1, 1]], + ); + assert_eq!( + Tensor::ones((2, 3), DType::F32, device)?.to_vec2::()?, + [[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]], + ); + if !device.is_metal() { + assert_eq!( + Tensor::ones((2, 3), DType::F64, device)?.to_vec2::()?, + [[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]], + ); + } + assert_eq!( + Tensor::ones((2, 3), DType::F16, device)?.to_vec2::()?, + [ + [ + half::f16::from_f32(1.0), + half::f16::from_f32(1.0), + half::f16::from_f32(1.0) + ], + [ + half::f16::from_f32(1.0), + half::f16::from_f32(1.0), + half::f16::from_f32(1.0) + ] + ], + ); + assert_eq!( + Tensor::ones((2, 3), DType::BF16, device)?.to_vec2::()?, + [ + [ + half::bf16::from_f32(1.0), + half::bf16::from_f32(1.0), + half::bf16::from_f32(1.0) + ], + [ + half::bf16::from_f32(1.0), + half::bf16::from_f32(1.0), + half::bf16::from_f32(1.0) + ] + ], + ); + + if !device.is_metal() { + assert_eq!( + Tensor::ones((2, 3), DType::F8E4M3, device)?.to_vec2::()?, + [ + [ + F8E4M3::from_f32(1.), + F8E4M3::from_f32(1.), + F8E4M3::from_f32(1.) + ], + [ + F8E4M3::from_f32(1.), + F8E4M3::from_f32(1.), + F8E4M3::from_f32(1.) + ] + ], + ); + } + Ok(()) +} + +fn full(device: &Device) -> Result<()> { + let tensor = Tensor::zeros((3, 4), DType::U32, device)?; + tensor.const_set(42u32.into())?; + assert_eq!( + tensor.to_vec2::()?, + [[42, 42, 42, 42], [42, 42, 42, 42], [42, 42, 42, 42]] + ); + tensor.i((.., 2))?.const_set(1337u32.into())?; + assert_eq!( + tensor.to_vec2::()?, + [[42, 42, 1337, 42], [42, 42, 1337, 42], [42, 42, 1337, 42]] + ); + tensor.i((2, ..))?.const_set(1u32.into())?; + assert_eq!( + tensor.to_vec2::()?, + [[42, 42, 1337, 42], [42, 42, 1337, 42], [1, 1, 1, 1]] + ); + Ok(()) +} + +fn const_set(device: &Device) -> Result<()> { + assert_eq!( + Tensor::full(42u32, (2, 3), device)?.to_vec2::()?, + [[42, 42, 42], [42, 42, 42]], + ); + Ok(()) +} + +fn arange(device: &Device) -> Result<()> { + assert_eq!( + Tensor::arange(0u8, 5u8, device)?.to_vec1::()?, + [0, 1, 2, 3, 4], + ); + assert_eq!( + Tensor::arange_step(0u8, 5u8, 2, device)?.to_vec1::()?, + [0, 2, 4], + ); + assert_eq!( + Tensor::arange_step(0u8, 5u8, 3, device)?.to_vec1::()?, + [0, 3], + ); + assert_eq!( + Tensor::arange_step(5i64, 0i64, -1, device)?.to_vec1::()?, + [5, 4, 3, 2, 1], + ); + + if !device.is_metal() { + assert_eq!( + Tensor::arange_step( + F8E4M3::from_f32(0.), + F8E4M3::from_f32(5.), + F8E4M3::from_f32(2.), + device + )? + .to_vec1::()?, + [ + F8E4M3::from_f32(0.), + F8E4M3::from_f32(2.), + F8E4M3::from_f32(4.), + ], + ); + } + + Ok(()) +} + +fn add_mul(device: &Device) -> Result<()> { + let tensor = Tensor::new(&[3f32, 1., 4.], device)?; + let dim1 = tensor.dims1()?; + assert_eq!(dim1, 3); + let content: Vec = tensor.to_vec1()?; + assert_eq!(content, [3., 1., 4.]); + let tensor = Tensor::add(&tensor, &tensor)?; + let content: Vec = tensor.to_vec1()?; + assert_eq!(content, [6., 2., 8.]); + let tensor = Tensor::mul(&tensor, &tensor)?; + let content: Vec = tensor.to_vec1()?; + assert_eq!(content, [36., 4., 64.]); + Ok(()) +} + +fn tensor_2d(device: &Device) -> Result<()> { + let data = &[[3f32, 1., 4., 1., 5.], [2., 1., 7., 8., 2.]]; + let tensor = Tensor::new(data, device)?; + let dims = tensor.dims2()?; + assert_eq!(dims, (2, 5)); + let content: Vec> = tensor.to_vec2()?; + assert_eq!(content, data); + Ok(()) +} + +fn clamp(device: &Device) -> Result<()> { + let data = &[[3f32, 1., 4., 1., 5.], [2., 1., 7., 8., 2.]]; + let tensor = Tensor::new(data, device)?; + let tensor = tensor.clamp(1.5, 6.2)?; + assert_eq!( + tensor.to_vec2::()?, + [[3.0, 1.5, 4.0, 1.5, 5.0], [2.0, 1.5, 6.2, 6.2, 2.0]], + ); + Ok(()) +} + +fn asort(device: &Device) -> Result<()> { + let data = &[[3f32, 1., 4., 1.1, 5.], [2.1, 1., 7., 8., 2.]]; + let tensor = Tensor::new(data, device)?; + let indexes = tensor.arg_sort_last_dim(true)?; + assert_eq!( + indexes.to_vec2::()?, + [[1, 3, 0, 2, 4], [1, 4, 0, 2, 3]], + ); + let indexes = tensor.arg_sort_last_dim(false)?; + assert_eq!( + indexes.to_vec2::()?, + [[4, 2, 0, 3, 1], [3, 2, 0, 4, 1]], + ); + let (sorted, indexes) = tensor.sort_last_dim(true)?; + assert_eq!( + indexes.to_vec2::()?, + [[1, 3, 0, 2, 4], [1, 4, 0, 2, 3]], + ); + assert_eq!( + sorted.to_vec2::()?, + [[1.0, 1.1, 3.0, 4.0, 5.0], [1.0, 2.0, 2.1, 7.0, 8.0]] + ); + let (sorted, indexes) = tensor.sort_last_dim(false)?; + assert_eq!( + indexes.to_vec2::()?, + [[4, 2, 0, 3, 1], [3, 2, 0, 4, 1]], + ); + assert_eq!( + sorted.to_vec2::()?, + [[5.0, 4.0, 3.0, 1.1, 1.0], [8.0, 7.0, 2.1, 2.0, 1.0]] + ); + Ok(()) +} + +/// Test sorting a large tensor that exceeds 1024 elements. +fn asort_big(device: &Device) -> Result<()> { + // Skip on metal for now + if device.is_metal() { + return Ok(()); + } + const SIZE: usize = 2000; + let data: Vec = (0..SIZE).map(|x| (SIZE - x) as f32).collect(); + let tensor = Tensor::new(data.as_slice(), device)?; + + let indexes = tensor.arg_sort_last_dim(true)?; + let expected_indexes: Vec = (0..SIZE).rev().map(|x| x as u32).collect(); + assert_eq!(indexes.to_vec1::()?, expected_indexes); + + let indexes = tensor.arg_sort_last_dim(false)?; + let expected_indexes: Vec = (0..SIZE).map(|x| x as u32).collect(); + assert_eq!(indexes.to_vec1::()?, expected_indexes); + Ok(()) +} + +fn unary_op(device: &Device) -> Result<()> { + let data = &[[-3f32, 1., 4., -0.1, 0.5], [2.7, -1.8, -0.28, 1.8, 2.8]]; + let tensor = Tensor::new(data, device)?; + assert_eq!( + test_utils::to_vec2_round(&tensor.gelu()?, 4)?, + [ + [-0.0036, 0.8412, 3.9999, -0.046, 0.3457], + [2.6911, -0.0647, -0.1091, 1.7353, 2.7933] + ] + ); + let t_f16 = tensor.to_dtype(DType::F16)?.gelu()?.to_dtype(DType::F32)?; + let max_diff = (tensor.gelu()? - t_f16)?.flatten_all()?.max(0)?; + assert!(max_diff.to_vec0::()? < 5e-3); + assert_eq!( + test_utils::to_vec2_round(&tensor.gelu_erf()?, 4)?, + [ + [-0.004, 0.8413, 3.9999, -0.046, 0.3457], + [2.6906, -0.0647, -0.1091, 1.7353, 2.7928] + ] + ); + assert_eq!( + test_utils::to_vec2_round(&tensor.erf()?, 4)?, + [ + [-1.0, 0.8427, 1.0, -0.1125, 0.5205], + [0.9999, -0.9891, -0.3079, 0.9891, 0.9999] + ] + ); + assert_eq!( + test_utils::to_vec2_round(&tensor.silu()?, 4)?, + [ + [-0.1423, 0.7311, 3.9281, -0.0475, 0.3112], + [2.53, -0.2553, -0.1205, 1.5447, 2.6395] + ] + ); + assert_eq!( + test_utils::to_vec2_round(&tensor.ceil()?, 4)?, + [[-3.0, 1.0, 4.0, -0.0, 1.0], [3.0, -1.0, -0.0, 2.0, 3.0]] + ); + assert_eq!( + test_utils::to_vec2_round(&tensor.floor()?, 4)?, + [[-3.0, 1.0, 4.0, -1.0, 0.0], [2.0, -2.0, -1.0, 1.0, 2.0]] + ); + assert_eq!( + test_utils::to_vec2_round(&tensor.round()?, 4)?, + [[-3.0, 1.0, 4.0, -0.0, 1.0], [3.0, -2.0, -0.0, 2.0, 3.0]] + ); + let tensor = Tensor::new(&[2997.9246, 314.15926f32], device)?; + assert_eq!( + test_utils::to_vec1_round(&tensor.round_to(2)?, 4)?, + [2997.92, 314.16] + ); + assert_eq!( + test_utils::to_vec1_round(&tensor.round_to(-2)?, 4)?, + [3000.0, 300.] + ); + let tensor = Tensor::new( + &[-1.01f32, -0.9, -0.1, 0.0, -0.0, 0.1, 0.9, 1.0, 1.1], + device, + )?; + assert_eq!( + tensor.sign()?.to_vec1::()?, + [-1., -1., -1., 0., 0., 1., 1., 1., 1.] + ); + let tensor = Tensor::new(&[-1.0f32, 0., -2., 3.], device)?; + let y = tensor.elu(2.)?; + assert_eq!( + test_utils::to_vec1_round(&y, 4)?, + [-1.2642, 0.0000, -1.7293, 3.0000] + ); + // This test failed on metal prior to the following PR: + // https://github.com/huggingface/candle/pull/2490 + let y = tensor.reshape((2, 2))?.t()?.elu(2.)?.flatten_all()?; + assert_eq!( + test_utils::to_vec1_round(&y, 4)?, + [-1.2642, -1.7293, 0.0000, 3.0000] + ); + Ok(()) +} + +fn binary_op(device: &Device) -> Result<()> { + let data = &[[3f32, 1., 4., 1., 5.], [2., 1., 7., 8., 2.]]; + let tensor1 = Tensor::new(data, device)?; + let data2 = &[[5f32, 5., 5., 5., 5.], [2., 1., 7., 8., 2.]]; + let tensor2 = Tensor::new(data2, device)?; + let tensor = (&tensor1 + (&tensor1 * &tensor1)? / (&tensor1 + &tensor2))?; + let dims = tensor.dims2()?; + assert_eq!(dims, (2, 5)); + let content: Vec> = tensor.to_vec2()?; + assert_eq!(content[0], [4.125, 1.1666666, 5.7777777, 1.1666666, 7.5]); + assert_eq!(content[1], [3.0, 1.5, 10.5, 12.0, 3.0]); + #[allow(clippy::eq_op)] + let tensor = (&tensor - &tensor)?; + let content: Vec> = tensor.to_vec2()?; + assert_eq!(content[0], [0., 0., 0., 0., 0.]); + + let min = tensor1.minimum(&(&tensor2 * 0.5)?)?; + let max = tensor1.maximum(&(&tensor2 * 0.5)?)?; + assert_eq!( + min.to_vec2::()?, + [[2.5, 1.0, 2.5, 1.0, 2.5], [1.0, 0.5, 3.5, 4.0, 1.0]], + ); + assert_eq!( + max.to_vec2::()?, + [[3.0, 2.5, 4.0, 2.5, 5.0], [2.0, 1.0, 7.0, 8.0, 2.0]] + ); + Ok(()) +} + +fn ternary_op(device: &Device) -> Result<()> { + let data = &[[0u8, 1, 0, 1, 0], [1, 1, 1, 0, 0]]; + let ids = Tensor::new(data, device)?; + let data = &[[0f32, 1., 2., 3., 4.], [5., 6., 7., 8., 9.]]; + let a = Tensor::new(data, device)?; + let data = &[[10f32, 11., 12., 13., 14.], [15., 16., 17., 18., 19.]]; + let b = Tensor::new(data, device)?; + let tensor = ids.where_cond(&a, &b)?; + let dims = tensor.dims(); + assert_eq!(dims, [2, 5]); + let result: Vec = tensor.flatten_all()?.to_vec1()?; + assert_eq!(result, [10., 1., 12., 3., 14., 5., 6., 7., 18., 19.]); + Ok(()) +} + +fn transpose(device: &Device) -> Result<()> { + let data = &[[3f32, 1., 4., 1., 5.], [2., 1., 7., 8., 2.]]; + let tensor = Tensor::new(data, device)?.t()?; + let dims = tensor.dims2()?; + assert_eq!(dims, (5, 2)); + assert_eq!( + tensor.to_vec2::()?, + &[[3f32, 2.], [1., 1.], [4., 7.], [1., 8.], [5., 2.]] + ); + assert_eq!(tensor.t()?.to_vec2::()?, data); + assert_eq!(tensor.contiguous()?.t()?.to_vec2::()?, data); + assert_eq!(((tensor + 1.)?.t()? - 1.)?.to_vec2::()?, data); + Ok(()) +} + +fn var(device: &Device) -> Result<()> { + // Values taken from https://pytorch.org/docs/stable/generated/torch.var.html + let data = &[ + [0.2035f32, 1.2959, 1.8101, -0.4644], + [1.5027, -0.3270, 0.5905, 0.6538], + [-1.5745, 1.3330, -0.5596, -0.6548], + [0.1264, -0.5080, 1.6420, 0.1992], + ]; + let tensor = Tensor::new(data, device)?; + assert_eq!( + test_utils::to_vec2_round(&tensor.var_keepdim(1)?, 4)?, + &[[1.0631], [0.559], [1.4893], [0.8258]] + ); + Ok(()) +} + +fn sum(device: &Device) -> Result<()> { + let data = &[[[3u32, 1, 4], [1, 5, 9]], [[2, 1, 7], [8, 2, 8]]]; + let tensor = Tensor::new(data, device)?; + assert_eq!( + tensor.sum_keepdim(2)?.to_vec3::()?, + &[[[8], [15]], [[10], [18]]] + ); + assert_eq!( + tensor.sum_keepdim(0)?.to_vec3::()?, + &[[[5, 2, 11], [9, 7, 17]]], + ); + assert_eq!(tensor.sum_keepdim((0, 2, 1))?.to_vec3::()?, &[[[51]]],); + assert_eq!( + tensor.t()?.sum_keepdim(1)?.t()?.to_vec3::()?, + &[[[8], [15]], [[10], [18]]] + ); + assert_eq!( + tensor.sum_keepdim((2, 1))?.to_vec3::()?, + &[[[8 + 15]], [[10 + 18]]] + ); + let data: Vec = (0..4000u32).collect(); + let tensor = Tensor::new(data.as_slice(), device)?; + assert_eq!(tensor.sum_keepdim(0)?.to_vec1::()?, &[7998000]); + let tensor = tensor.reshape((2000, 2))?; + assert_eq!(tensor.sum_keepdim((0, 1))?.to_vec2::()?, &[[7998000]]); + assert_eq!( + tensor.sum_keepdim(0)?.sum_keepdim(1)?.to_vec2::()?, + &[[7998000]] + ); + assert_eq!( + tensor.sum_keepdim(1)?.sum_keepdim(0)?.to_vec2::()?, + &[[7998000]] + ); + assert_eq!( + tensor.sum_keepdim(0)?.to_vec2::()?, + &[[3998000, 4000000]] + ); + + // Make the tensor non contiguous. + let tensor = tensor.t()?.contiguous()?.t()?; + assert_eq!(tensor.sum_keepdim((0, 1))?.to_vec2::()?, &[[7998000]]); + assert_eq!( + tensor.sum_keepdim(0)?.sum_keepdim(1)?.to_vec2::()?, + &[[7998000]] + ); + assert_eq!( + tensor.sum_keepdim(1)?.sum_keepdim(0)?.to_vec2::()?, + &[[7998000]] + ); + assert_eq!( + tensor.sum_keepdim(0)?.to_vec2::()?, + &[[3998000, 4000000]] + ); + + let t1 = tensor.reshape((200, 5, 4))?; + let t2 = t1.transpose(0, 2)?.contiguous()?.transpose(0, 2)?; + for tensor in [t1, t2] { + assert_eq!( + tensor.sum_keepdim((0, 1, 2))?.to_vec3::()?, + &[[[7998000]]] + ); + assert_eq!( + tensor + .sum_keepdim(0)? + .sum_keepdim(2)? + .sum_keepdim(1)? + .to_vec3::()?, + &[[[7998000]]] + ); + assert_eq!( + tensor + .sum_keepdim(0)? + .sum_keepdim((1, 2))? + .to_vec3::()?, + &[[[7998000]]] + ); + assert_eq!( + tensor + .sum_keepdim(1)? + .sum_keepdim((0, 2))? + .to_vec3::()?, + &[[[7998000]]] + ); + assert_eq!( + tensor.sum_keepdim(0)?.to_vec3::()?, + &[[ + [398000, 398200, 398400, 398600], + [398800, 399000, 399200, 399400], + [399600, 399800, 400000, 400200], + [400400, 400600, 400800, 401000], + [401200, 401400, 401600, 401800] + ]] + ); + } + Ok(()) +} + +fn min(device: &Device) -> Result<()> { + let data = &[[[3u32, 1, 4], [1, 5, 9]], [[2, 1, 7], [8, 2, 8]]]; + let tensor = Tensor::new(data, device)?; + assert_eq!( + tensor.min_keepdim(2)?.to_vec3::()?, + &[[[1], [1]], [[1], [2]]] + ); + assert_eq!( + tensor.min_keepdim(0)?.to_vec3::()?, + &[[[2, 1, 4], [1, 2, 8]]], + ); + let data: Vec = (200..4000u32).collect(); + let tensor = Tensor::new(data.as_slice(), device)?; + assert_eq!(tensor.min_keepdim(0)?.to_vec1::()?, &[200]); + let tensor = tensor.reshape((1900, 2))?; + assert_eq!( + tensor.min_keepdim(0)?.min_keepdim(1)?.to_vec2::()?, + &[[200]] + ); + assert_eq!( + tensor.min_keepdim(1)?.min_keepdim(0)?.to_vec2::()?, + &[[200]] + ); + assert_eq!(tensor.min_keepdim(0)?.to_vec2::()?, &[[200, 201]]); + + // Make the tensor non contiguous. + let tensor = tensor.t()?.contiguous()?.t()?; + assert_eq!( + tensor.min_keepdim(0)?.min_keepdim(1)?.to_vec2::()?, + &[[200]] + ); + assert_eq!( + tensor.min_keepdim(1)?.min_keepdim(0)?.to_vec2::()?, + &[[200]] + ); + assert_eq!(tensor.min_keepdim(0)?.to_vec2::()?, &[[200, 201]]); + + let t1 = tensor.reshape((190, 5, 4))?; + let t2 = t1.transpose(0, 2)?.contiguous()?.transpose(0, 2)?; + for tensor in [t1, t2] { + assert_eq!( + tensor + .min_keepdim(0)? + .min_keepdim(2)? + .min_keepdim(1)? + .to_vec3::()?, + &[[[200]]] + ); + assert_eq!( + tensor.min_keepdim(0)?.to_vec3::()?, + &[[ + [200, 201, 202, 203], + [204, 205, 206, 207], + [208, 209, 210, 211], + [212, 213, 214, 215], + [216, 217, 218, 219] + ]] + ); + } + Ok(()) +} + +fn max(device: &Device) -> Result<()> { + let data = &[[[3u32, 1, 4], [1, 5, 9]], [[2, 1, 7], [8, 2, 8]]]; + let tensor = Tensor::new(data, device)?; + assert_eq!( + tensor.max_keepdim(2)?.to_vec3::()?, + &[[[4], [9]], [[7], [8]]] + ); + assert_eq!( + tensor.max_keepdim(0)?.to_vec3::()?, + &[[[3, 1, 7], [8, 5, 9]]], + ); + let data: Vec = (200..4000u32).collect(); + let tensor = Tensor::new(data.as_slice(), device)?; + assert_eq!(tensor.max_keepdim(0)?.to_vec1::()?, &[3999]); + let tensor = tensor.reshape((1900, 2))?; + assert_eq!( + tensor.max_keepdim(0)?.max_keepdim(1)?.to_vec2::()?, + &[[3999]] + ); + assert_eq!( + tensor.max_keepdim(1)?.max_keepdim(0)?.to_vec2::()?, + &[[3999]] + ); + assert_eq!(tensor.max_keepdim(0)?.to_vec2::()?, &[[3998, 3999]]); + + // Make the tensor non contiguous. + let tensor = tensor.t()?.contiguous()?.t()?; + assert_eq!( + tensor.max_keepdim(0)?.max_keepdim(1)?.to_vec2::()?, + &[[3999]] + ); + assert_eq!( + tensor.max_keepdim(1)?.max_keepdim(0)?.to_vec2::()?, + &[[3999]] + ); + assert_eq!(tensor.max_keepdim(0)?.to_vec2::()?, &[[3998, 3999]]); + + let t1 = tensor.reshape((190, 5, 4))?; + let t2 = t1.transpose(0, 2)?.contiguous()?.transpose(0, 2)?; + for tensor in [t1, t2] { + assert_eq!( + tensor + .max_keepdim(0)? + .max_keepdim(2)? + .max_keepdim(1)? + .to_vec3::()?, + &[[[3999]]] + ); + assert_eq!( + tensor.max_keepdim(0)?.to_vec3::()?, + &[[ + [3980, 3981, 3982, 3983], + [3984, 3985, 3986, 3987], + [3988, 3989, 3990, 3991], + [3992, 3993, 3994, 3995], + [3996, 3997, 3998, 3999] + ]] + ); + } + Ok(()) +} + +fn argmin(device: &Device) -> Result<()> { + let data = &[[[3u32, 1, 4], [1, 5, 9]], [[2, 1, 7], [8, 2, 8]]]; + let tensor = Tensor::new(data, device)?; + assert_eq!( + tensor.argmin_keepdim(2)?.to_vec3::()?, + &[[[1], [0]], [[1], [1]]] + ); + assert_eq!( + tensor.argmin_keepdim(0)?.to_vec3::()?, + &[[[1, 0, 0], [0, 1, 1]]], + ); + let data: Vec = (200..4000u32).collect(); + let tensor = Tensor::new(data.as_slice(), device)?; + assert_eq!(tensor.argmin_keepdim(0)?.to_vec1::()?, &[0]); + let tensor = tensor.reshape((1900, 2))?; + assert_eq!( + tensor + .argmin_keepdim(0)? + .argmin_keepdim(1)? + .to_vec2::()?, + &[[0]] + ); + assert_eq!( + tensor + .argmin_keepdim(1)? + .argmin_keepdim(0)? + .to_vec2::()?, + &[[0]] + ); + assert_eq!(tensor.argmin_keepdim(0)?.to_vec2::()?, &[[0, 0]]); + + // Make the tensor non contiguous. + let tensor = tensor.t()?.contiguous()?.t()?; + assert_eq!( + tensor + .argmin_keepdim(0)? + .argmin_keepdim(1)? + .to_vec2::()?, + &[[0]] + ); + assert_eq!( + tensor + .argmin_keepdim(1)? + .argmin_keepdim(0)? + .to_vec2::()?, + &[[0]] + ); + assert_eq!(tensor.argmin_keepdim(0)?.to_vec2::()?, &[[0, 0]]); + + let t1 = tensor.reshape((190, 5, 4))?; + let t2 = t1.transpose(0, 2)?.contiguous()?.transpose(0, 2)?; + for tensor in [t1, t2] { + assert_eq!( + tensor + .argmin_keepdim(0)? + .argmin_keepdim(2)? + .argmin_keepdim(1)? + .to_vec3::()?, + &[[[0]]] + ); + assert_eq!( + tensor.argmin_keepdim(0)?.to_vec3::()?, + &[[ + [0, 0, 0, 0], + [0, 0, 0, 0], + [0, 0, 0, 0], + [0, 0, 0, 0], + [0, 0, 0, 0], + ]] + ); + } + Ok(()) +} + +fn argmax(device: &Device) -> Result<()> { + let data = &[[[3u32, 1, 4], [1, 5, 9]], [[2, 1, 7], [8, 2, 8]]]; + let tensor = Tensor::new(data, device)?; + assert_eq!( + tensor.argmax_keepdim(2)?.to_vec3::()?, + &[[[2], [2]], [[2], [0]]] + ); + assert_eq!( + tensor.argmax_keepdim(0)?.to_vec3::()?, + &[[[0, 0, 1], [1, 0, 0]]], + ); + let data: Vec = (200..4000u32).collect(); + let tensor = Tensor::new(data.as_slice(), device)?; + assert_eq!(tensor.argmax_keepdim(0)?.to_vec1::()?, &[3799]); + let tensor = tensor.reshape((1900, 2))?; + assert_eq!( + tensor + .argmax_keepdim(0)? + .argmax_keepdim(1)? + .to_vec2::()?, + &[[0]] + ); + assert_eq!( + tensor + .argmax_keepdim(1)? + .argmax_keepdim(0)? + .to_vec2::()?, + &[[0]] + ); + assert_eq!(tensor.argmax_keepdim(0)?.to_vec2::()?, &[[1899, 1899]]); + + // Make the tensor non contiguous. + let tensor = tensor.t()?.contiguous()?.t()?; + assert_eq!( + tensor + .argmax_keepdim(0)? + .argmax_keepdim(1)? + .to_vec2::()?, + &[[0]] + ); + assert_eq!( + tensor + .argmax_keepdim(1)? + .argmax_keepdim(0)? + .to_vec2::()?, + &[[0]] + ); + assert_eq!(tensor.argmax_keepdim(0)?.to_vec2::()?, &[[1899, 1899]]); + + let t1 = tensor.reshape((190, 5, 4))?; + let t2 = t1.transpose(0, 2)?.contiguous()?.transpose(0, 2)?; + for tensor in [t1, t2] { + assert_eq!( + tensor + .argmax_keepdim(0)? + .argmax_keepdim(2)? + .argmax_keepdim(1)? + .to_vec3::()?, + &[[[0]]] + ); + assert_eq!( + tensor.argmax_keepdim(0)?.to_vec3::()?, + &[[ + [189, 189, 189, 189], + [189, 189, 189, 189], + [189, 189, 189, 189], + [189, 189, 189, 189], + [189, 189, 189, 189], + ]] + ); + } + Ok(()) +} + +fn narrow(device: &Device) -> Result<()> { + let data = &[[[3f32, 1., 4.], [1., 5., 9.]], [[2., 1., 7.], [8., 2., 8.]]]; + let tensor = Tensor::new(data, device)?; + assert_eq!( + tensor.narrow(2, 1, 2)?.to_vec3::()?, + &[[[1.0, 4.0], [5.0, 9.0]], [[1.0, 7.0], [2.0, 8.0]]], + ); + assert_eq!( + tensor.narrow(1, 1, 1)?.to_vec3::()?, + &[[[1.0, 5.0, 9.0]], [[8.0, 2.0, 8.0]]], + ); + assert_eq!( + tensor.narrow(0, 0, 1)?.to_vec3::()?, + &[[[3.0, 1.0, 4.0], [1.0, 5.0, 9.0]]], + ); + assert_eq!( + tensor.narrow(0, 1, 1)?.to_vec3::()?, + &[[[2.0, 1.0, 7.0], [8.0, 2.0, 8.0]]], + ); + // The following has been checked against PyTorch via: + // import torch + // t = torch.tensor([[[3., 1., 4.], [1., 5., 9.]], [[2., 1., 7.], [8., 2., 8.]]]) + // t.transpose(-1, -2).narrow(1, 1, 2) + assert_eq!( + tensor.t()?.narrow(1, 1, 2)?.to_vec3::()?, + &[[[1.0, 5.0], [4.0, 9.0]], [[1.0, 2.0], [7.0, 8.0]]], + ); + Ok(()) +} + +fn broadcast(device: &Device) -> Result<()> { + let data = &[3f32, 1., 4.]; + let tensor = Tensor::new(data, device)?; + assert_eq!( + tensor.broadcast_left((3, 1))?.to_vec3::()?, + &[[[3.0, 1.0, 4.0]], [[3.0, 1.0, 4.0]], [[3.0, 1.0, 4.0]]] + ); + Ok(()) +} + +fn slice_set(device: &Device) -> Result<()> { + let (b, h, max_t, d) = (2, 4, 7, 3); + let cache = Tensor::zeros((b, h, max_t, d), DType::F32, device)?; + let tensor = Tensor::randn(0f32, 1f32, (b, h, 4, d), device)?; + cache.slice_set(&tensor, 2, 0)?; + let cache_t = cache.narrow(2, 0, 4)?; + let diff = (cache_t - &tensor)?.abs()?.sum_all()?.to_vec0::()?; + assert_eq!(diff, 0.); + cache.slice_set(&tensor, 2, 1)?; + let cache_t = cache.narrow(2, 1, 4)?; + let diff = (cache_t - &tensor)?.abs()?.sum_all()?.to_vec0::()?; + assert_eq!(diff, 0.); + let ones = Tensor::ones((b, h, 1, d), DType::F32, device)?; + cache.slice_set(&ones, 2, 6)?; + let diff = cache.narrow(2, 5, 1)?.abs()?.sum_all()?.to_vec0::()?; + assert_eq!(diff, 0.); + let diff = (cache.narrow(2, 6, 1)? - 1.)? + .abs()? + .sum_all()? + .to_vec0::()?; + assert_eq!(diff, 0.); + // This used to create a deadlock rather than returning an actual error. + assert!(cache.slice_set(&cache, 0, 0).is_err()); + Ok(()) +} + +fn cat(device: &Device) -> Result<()> { + // 1D + let t1 = Tensor::new(&[3f32, 1., 4.], device)?; + let t2 = Tensor::new(&[1f32, 5., 9., 2.], device)?; + let t3 = Tensor::new(&[6f32, 5., 3., 5., 8., 9.], device)?; + assert_eq!(Tensor::cat(&[&t1], 0)?.to_vec1::()?, [3f32, 1., 4.],); + assert_eq!( + Tensor::cat(&[&t1, &t2], 0)?.to_vec1::()?, + [3f32, 1., 4., 1., 5., 9., 2.], + ); + assert_eq!( + Tensor::cat(&[&t1, &t2, &t3], 0)?.to_vec1::()?, + [3f32, 1., 4., 1., 5., 9., 2., 6., 5., 3., 5., 8., 9.], + ); + + // 2D + let data = &[[3f32, 1., 4., 1., 5.], [2., 7., 1., 8., 2.]]; + let t1 = Tensor::new(data, device)?; + let data2 = &[[5f32, 5., 5., 5., 5.], [2., 7., 1., 8., 2.]]; + let t2 = Tensor::new(data2, device)?; + assert_eq!( + Tensor::cat(&[&t1, &t2], 0)?.to_vec2::()?, + [ + [3.0, 1.0, 4.0, 1.0, 5.0], + [2.0, 7.0, 1.0, 8.0, 2.0], + [5.0, 5.0, 5.0, 5.0, 5.0], + [2.0, 7.0, 1.0, 8.0, 2.0] + ] + ); + // PyTorch equivalent: + // import torch + // t1 = torch.tensor([[3, 1, 4, 1, 5], [2, 7, 1, 8, 2]]) + // t2 = torch.tensor([[5]*5, [2, 7, 1, 8, 2]]) + // torch.cat([t1.t(), t2.t()], dim=1).t() + assert_eq!( + Tensor::cat(&[&t1.t()?, &t2.t()?], 1)? + .t()? + .to_vec2::()?, + [ + [3.0, 1.0, 4.0, 1.0, 5.0], + [2.0, 7.0, 1.0, 8.0, 2.0], + [5.0, 5.0, 5.0, 5.0, 5.0], + [2.0, 7.0, 1.0, 8.0, 2.0] + ] + ); + assert_eq!( + Tensor::cat(&[&t1, &t2], 1)?.to_vec2::()?, + [ + [3.0, 1.0, 4.0, 1.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0], + [2.0, 7.0, 1.0, 8.0, 2.0, 2.0, 7.0, 1.0, 8.0, 2.0] + ] + ); + + // 3D + let t1 = Tensor::arange(0, 48i64, device)?.reshape((2, 6, 4))?; + let t2 = Tensor::arange(100, 124i64, device)?.reshape((2, 3, 4))?; + let t3 = Tensor::arange(10000, 10032i64, device)?.reshape((2, 4, 4))?; + + let t_cat = Tensor::cat(&[&t1, &t2, &t3], 1)?; + + let t1 = t1.t()?.contiguous()?.t()?; + let t2 = t2.t()?.contiguous()?.t()?; + let t3 = t3.t()?.contiguous()?.t()?; + let t_cat2 = Tensor::cat(&[&t1, &t2, &t3], 1)?; + + let diff = t_cat.eq(&t_cat2)?.to_dtype(DType::F32)?.sum_all()?; + assert_eq!(diff.to_vec0::()?, 104.0); + assert_eq!(t_cat.i((0, 0, 0))?.to_vec0::()?, 0); + assert_eq!(t_cat.i((0, 4, 0))?.to_vec0::()?, 16); + assert_eq!(t_cat.i((0, 5, 0))?.to_vec0::()?, 20); + assert_eq!(t_cat.i((1, 5, 0))?.to_vec0::()?, 44); + assert_eq!(t_cat.i((0, 6, 0))?.to_vec0::()?, 100); + assert_eq!(t_cat.i((1, 6, 0))?.to_vec0::()?, 112); + assert_eq!(t_cat.i((0, 6, 1))?.to_vec0::()?, 101); + assert_eq!(t_cat.i((0, 7, 1))?.to_vec0::()?, 105); + assert_eq!(t_cat.i((0, 12, 1))?.to_vec0::()?, 10013); + assert_eq!(t_cat.i((1, 12, 3))?.to_vec0::()?, 10031); + Ok(()) +} + +fn embeddings(device: &Device) -> Result<()> { + let ids = Tensor::new(&[0u32, 2u32, 1u32], device)?; + let t = Tensor::new(&[[0f32, 1f32], [2f32, 3f32], [4f32, 5f32]], device)?; + let hs = t.embedding(&ids)?; + assert_eq!(hs.to_vec2::()?, &[[0.0, 1.0], [4.0, 5.0], [2.0, 3.0]]); + let hs = t.index_select(&ids, 0)?; + assert_eq!(hs.to_vec2::()?, &[[0.0, 1.0], [4.0, 5.0], [2.0, 3.0]]); + let hs = t.index_select(&ids.to_dtype(DType::I64)?, 0)?; + assert_eq!(hs.to_vec2::()?, &[[0.0, 1.0], [4.0, 5.0], [2.0, 3.0]]); + let ids = Tensor::new(&[u32::MAX, 2u32, u32::MAX], device)?; + let hs = t.index_select(&ids, 0)?; + assert_eq!(hs.to_vec2::()?, &[[0.0, 0.0], [4.0, 5.0], [0.0, 0.0]]); + Ok(()) +} + +#[test] +fn index_select_fail() -> Result<()> { + // Check that an error is properly reported on out of bounds. + let ids = Tensor::new(&[4u32, 2u32, 1u32], &Device::Cpu)?; + let t = Tensor::new(&[[0f32, 1f32], [2f32, 3f32], [4f32, 5f32]], &Device::Cpu)?; + let hs = t.index_select(&ids, 0); + assert!(hs.is_err()); + Ok(()) +} + +// The test below triggers an unwinding panic as there is a panic within the +// #[cfg(feature = "cuda")] +// #[test] +// #[should_panic] +// fn index_select_fail_gpu() { +// // Check that a panic happens for out of bounds in cuda +// if let Ok(device) = Device::new_cuda(0) { +// if let Ok(ids) = Tensor::new(&[4u32, 2u32, 1u32], &device) { +// if let Ok(t) = Tensor::new(&[[0f32, 1f32], [2f32, 3f32], [4f32, 5f32]], &device) { +// let _ = t.index_select(&ids, 0); +// } +// } +// } +// } + +fn cmp(device: &Device) -> Result<()> { + let t1 = Tensor::new(&[[0f32, 1f32], [2f32, 3f32], [4f32, 5f32]], device)?; + let t2 = Tensor::new(&[[1f32, 0f32], [3f32, 3f32], [4f32, 7f32]], device)?; + assert_eq!(t1.eq(&t2)?.to_vec2::()?, &[[0, 0], [0, 1], [1, 0]]); + assert_eq!(t1.ne(&t2)?.to_vec2::()?, &[[1, 1], [1, 0], [0, 1]]); + assert_eq!(t1.le(&t2)?.to_vec2::()?, &[[1, 0], [1, 1], [1, 1]]); + assert_eq!(t1.lt(&t2)?.to_vec2::()?, &[[1, 0], [1, 0], [0, 1]]); + assert_eq!(t1.gt(&t2)?.to_vec2::()?, &[[0, 1], [0, 0], [0, 0]]); + assert_eq!(t1.ge(&t2)?.to_vec2::()?, &[[0, 1], [0, 1], [1, 0]]); + Ok(()) +} + +fn index_select(device: &Device) -> Result<()> { + let ids = Tensor::new(&[0u32, 2u32, 1u32], device)?; + let t = Tensor::arange(0f32, 12f32, device)?.reshape((4, 3))?; + assert_eq!( + t.to_vec2::()?, + &[ + [0.0, 1.0, 2.0], + [3.0, 4.0, 5.0], + [6.0, 7.0, 8.0], + [9.0, 10.0, 11.0] + ] + ); + for dtype in [DType::U8, DType::U32, DType::I64] { + let ids = ids.to_dtype(dtype)?; + let hs = t.index_select(&ids, 1)?; + assert_eq!( + hs.to_vec2::()?, + &[ + [0.0, 2.0, 1.0], + [3.0, 5.0, 4.0], + [6.0, 8.0, 7.0], + [9.0, 11.0, 10.0] + ] + ); + let hs = t.index_select(&ids, 0)?; + assert_eq!( + hs.to_vec2::()?, + &[[0.0, 1.0, 2.0], [6.0, 7.0, 8.0], [3.0, 4.0, 5.0]] + ); + // Prior to https://github.com/huggingface/candle/pull/1022 + // There would be a bug where the last values in the result tensor would be set to 0. + let ids = Tensor::new(&[0u32, 2u32, 1u32, 0u32, 2u32, 1u32], device)?; + let hs = t.index_select(&ids, 0)?; + assert_eq!( + hs.to_vec2::()?, + &[ + [0.0, 1.0, 2.0], + [6.0, 7.0, 8.0], + [3.0, 4.0, 5.0], + [0.0, 1.0, 2.0], + [6.0, 7.0, 8.0], + [3.0, 4.0, 5.0], + ] + ); + + // Test when selecting dim > 0 with ids size different from elem count of + // target dim in source/input. + let ids = Tensor::new(&[1u32, 0u32, 1u32], device)?; + let t = Tensor::arange(1f32, 5f32, device)?.reshape((2, 2))?; + assert_eq!(t.to_vec2::()?, &[[1.0, 2.0], [3.0, 4.0]]); + let hs = t.index_select(&ids, 1)?; + assert_eq!(hs.to_vec2::()?, &[[2.0, 1.0, 2.0], [4.0, 3.0, 4.0]]); + } + + Ok(()) +} + +fn index_add(device: &Device) -> Result<()> { + let ids = Tensor::new(&[0u32, 1u32, 1u32], device)?; + let t = Tensor::arange(0f32, 12f32, device)?.reshape((4, 3))?; + assert_eq!( + t.to_vec2::()?, + &[ + [0.0, 1.0, 2.0], + [3.0, 4.0, 5.0], + [6.0, 7.0, 8.0], + [9.0, 10.0, 11.0] + ] + ); + let init = Tensor::ones((4, 2), DType::F32, device)?; + let hs = init.index_add(&ids, &t, 1)?; + assert_eq!( + hs.to_vec2::()?, + &[[1.0, 4.0], [4.0, 10.0], [7.0, 16.0], [10.0, 22.0]], + ); + let init = Tensor::zeros((4, 2), DType::F32, device)?; + let ids = Tensor::new(&[1u32, 0u32, 0u32], device)?; + let hs = init.index_add(&ids, &t, 1)?; + assert_eq!( + hs.to_vec2::()?, + &[[3.0, 0.0], [9.0, 3.0], [15.0, 6.0], [21.0, 9.0]], + ); + + let init = Tensor::zeros((6, 3), DType::F32, device)?; + let ids = Tensor::new(&[5u32, 0u32, 1u32, 0u32], device)?; + let hs = init.index_add(&ids, &t, 0)?; + assert_eq!( + hs.to_vec2::()?, + &[ + [12.0, 14.0, 16.0], + [6.0, 7.0, 8.0], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + [0.0, 1.0, 2.0] + ] + ); + Ok(()) +} + +fn slice_scatter(device: &Device) -> Result<()> { + let t = Tensor::arange(0f32, 12f32, device)?.reshape((4, 3))?; + assert_eq!( + t.to_vec2::()?, + &[ + [0.0, 1.0, 2.0], + [3.0, 4.0, 5.0], + [6.0, 7.0, 8.0], + [9.0, 10.0, 11.0] + ] + ); + let src = Tensor::arange(100f32, 106f32, device)?.reshape((2, 3))?; + assert_eq!( + t.slice_scatter0(&src, 0)?.to_vec2::()?, + &[ + [100.0, 101.0, 102.0], + [103.0, 104.0, 105.0], + [6.0, 7.0, 8.0], + [9.0, 10.0, 11.0] + ] + ); + assert_eq!( + t.slice_scatter0(&src, 1)?.to_vec2::()?, + &[ + [0.0, 1.0, 2.0], + [100.0, 101.0, 102.0], + [103.0, 104.0, 105.0], + [9.0, 10.0, 11.0] + ] + ); + assert_eq!( + t.slice_scatter0(&src, 2)?.to_vec2::()?, + &[ + [0.0, 1.0, 2.0], + [3.0, 4.0, 5.0], + [100.0, 101.0, 102.0], + [103.0, 104.0, 105.0], + ] + ); + Ok(()) +} + +fn scatter(device: &Device) -> Result<()> { + let t = Tensor::arange(0f32, 12f32, device)?.reshape((4, 3))?; + assert_eq!( + t.to_vec2::()?, + &[ + [0.0, 1.0, 2.0], + [3.0, 4.0, 5.0], + [6.0, 7.0, 8.0], + [9.0, 10.0, 11.0] + ] + ); + let ids = Tensor::new(&[[0u32, 1, 2], [3, 4, 0], [3, 3, 1], [2, 0, 4]], device)?; + let init = Tensor::ones((4, 5), DType::F32, device)?; + let hs = init.scatter_add(&ids, &t, 1)?; + assert_eq!( + hs.to_vec2::()?, + &[ + [1.0, 2.0, 3.0, 1.0, 1.0], + [6.0, 1.0, 1.0, 4.0, 5.0], + [1.0, 9.0, 1.0, 14.0, 1.0], + [11.0, 1.0, 10.0, 1.0, 12.0] + ] + ); + + let hs = init.scatter(&ids, &t, 1)?; + assert_eq!( + hs.to_vec2::()?, + &[ + [0.0, 1.0, 2.0, 1.0, 1.0], + [5.0, 1.0, 1.0, 3.0, 4.0], + [1.0, 8.0, 1.0, 7.0, 1.0], + [10.0, 1.0, 9.0, 1.0, 11.0] + ] + ); + + let init = Tensor::ones((6, 3), DType::F32, device)?; + let hs = init.scatter_add(&ids, &t, 0)?; + assert_eq!( + hs.to_vec2::()?, + &[ + [1.0, 11.0, 6.0], + [1.0, 2.0, 9.0], + [10.0, 1.0, 3.0], + [10.0, 8.0, 1.0], + [1.0, 5.0, 12.0], + [1.0, 1.0, 1.0] + ] + ); + let hs = init.scatter(&ids, &t, 0)?; + assert_eq!( + hs.to_vec2::()?, + &[ + [0.0, 10.0, 5.0], + [1.0, 1.0, 8.0], + [9.0, 1.0, 2.0], + [6.0, 7.0, 1.0], + [1.0, 4.0, 11.0], + [1.0, 1.0, 1.0] + ] + ); + + let hs = { + let ids = Tensor::new( + &[ + [0u32, u32::MAX, 2], + [3, 4, u32::MAX], + [3, 3, 1], + [u32::MAX, u32::MAX, 4], + ], + device, + )?; + init.scatter(&ids, &t, 0)? + }; + assert_eq!( + hs.to_vec2::()?, + &[ + [0.0, 1.0, 1.0], + [1.0, 1.0, 8.0], + [1.0, 1.0, 2.0], + [6.0, 7.0, 1.0], + [1.0, 4.0, 11.0], + [1.0, 1.0, 1.0] + ] + ); + + init.scatter_set(&ids, &t, 0)?; + assert_eq!( + init.to_vec2::()?, + &[ + [0.0, 10.0, 5.0], + [1.0, 1.0, 8.0], + [9.0, 1.0, 2.0], + [6.0, 7.0, 1.0], + [1.0, 4.0, 11.0], + [1.0, 1.0, 1.0] + ] + ); + + Ok(()) +} + +fn gather(device: &Device) -> Result<()> { + let ids = Tensor::new(&[[0u32], [2u32], [1u32], [0u32]], device)?; + let t = Tensor::arange(0f32, 12f32, device)?.reshape((4, 3))?; + assert_eq!( + t.to_vec2::()?, + &[ + [0.0, 1.0, 2.0], + [3.0, 4.0, 5.0], + [6.0, 7.0, 8.0], + [9.0, 10.0, 11.0] + ] + ); + let hs = t.gather(&ids, 1)?; + assert_eq!(hs.to_vec2::()?, &[[0.0], [5.0], [7.0], [9.0]]); + let ids = Tensor::new( + &[[0u32, 0u32], [2u32, 0u32], [1u32, 1u32], [0u32, 2u32]], + device, + )?; + let hs = t.gather(&ids, 1)?; + assert_eq!( + hs.to_vec2::()?, + &[[0.0, 0.0], [5.0, 3.0], [7.0, 7.0], [9.0, 11.0]] + ); + let ids = Tensor::new(&[[0u32, 2u32, 0u32]], device)?; + let hs = t.gather(&ids, 0)?; + assert_eq!(hs.to_vec2::()?, &[[0.0, 7.0, 2.0]]); + let ids = Tensor::new(&[[0u32, 2u32, 0u32], [0u32, 1u32, 1u32]], device)?; + let hs = t.gather(&ids, 0)?; + assert_eq!(hs.to_vec2::()?, &[[0.0, 7.0, 2.0], [0.0, 4.0, 5.0]]); + + let hs = { + let ids = Tensor::new( + &[ + [0u32, 0u32], + [2u32, u32::MAX], + [u32::MAX, 1u32], + [0u32, 2u32], + ], + device, + )?; + t.gather(&ids, 1)? + }; + assert_eq!( + hs.to_vec2::()?, + &[[0.0, 0.0], [5.0, 0.0], [0.0, 7.0], [9.0, 11.0]] + ); + + // Random data + + // Dim: 0 + let t = Tensor::new( + &[ + [ + [108_f32, -47., 16., -56., -83., -130., 210.], + [253., 95., 151., 228., -210., -123., -127.], + [-9., -217., 2., -78., 163., 245., -204.], + [-246., 79., -238., 88., -226., -184., 171.], + [8., -48., -153., 234., -34., 166., -153.], + [124., 0., -10., -61., -242., -15., -238.], + ], + [ + [12., -64., -199., 244., -240., 156., -128.], + [173., -57., 4., -198., 233., -110., 238.], + [95., 82., 0., 240., 53., -211., 209.], + [-122., 167., -212., 227., -144., 61., 118.], + [-63., -146., 200., 244., 168., -167., 116.], + [-125., -147., 110., -253., -178., -250., -18.], + ], + [ + [57., 86., -50., 56., 92., 205., -78.], + [-137., -156., -18., 248., -61., -239., 14.], + [-248., -30., -50., -70., -251., 250., -83.], + [-221., 67., 72., 59., -24., -154., 232.], + [-144., -23., -74., 5., 93., 171., 205.], + [46., -77., -38., -226., 246., 161., -17.], + ], + [ + [-153., -231., -236., 161., 126., 2., -22.], + [-229., -41., 209., 164., 234., 160., 57.], + [223., 254., -186., -162., -46., -160., -102.], + [65., 30., 213., -253., 59., 224., -154.], + [-82., -203., -177., 17., 31., -256., -246.], + [176., -135., -65., 54., -56., 210., 76.], + ], + [ + [-10., -245., 168., 124., -14., -33., -178.], + [25., -43., -39., 132., -89., 169., 179.], + [187., -215., 32., -133., 87., -7., -168.], + [-224., -215., -5., -230., -58., -162., 128.], + [158., -137., -122., -100., -202., -83., 136.], + [30., -185., -144., 250., 209., -40., 127.], + ], + [ + [-196., 108., -245., 122., 146., -228., 62.], + [-1., -66., 160., 137., 13., -172., -21.], + [244., 199., -164., 28., 119., -175., 198.], + [-62., 253., -162., 195., -95., -230., -211.], + [123., -72., -26., -107., -139., 64., 245.], + [11., -126., -182., 108., -12., 184., -127.], + ], + [ + [-159., 126., 176., 161., 73., -111., -138.], + [-187., 214., -217., -33., -223., -201., -212.], + [-61., -120., -166., -172., -95., 53., 196.], + [-33., 86., 134., -152., 154., -53., 74.], + [186., -28., -154., -174., 141., -109., 217.], + [82., 35., 252., 145., 181., 74., -87.], + ], + ], + device, + )?; + + let ids = Tensor::new( + &[ + [ + [6_u32, 6, 4, 3, 4, 4, 6], + [3, 3, 2, 4, 4, 4, 6], + [3, 3, 0, 2, 4, 6, 4], + [2, 5, 1, 2, 6, 6, 1], + [2, 1, 6, 5, 3, 2, 3], + [6, 1, 0, 1, 0, 2, 6], + ], + [ + [4, 6, 4, 3, 3, 3, 2], + [4, 3, 2, 4, 4, 4, 6], + [2, 3, 0, 2, 4, 6, 4], + [6, 5, 1, 2, 6, 6, 1], + [4, 1, 6, 5, 3, 2, 3], + [1, 1, 0, 1, 0, 2, 6], + ], + [ + [3, 6, 4, 3, 3, 3, 2], + [2, 3, 2, 4, 4, 4, 6], + [4, 3, 0, 2, 4, 6, 4], + [0, 5, 1, 2, 6, 6, 1], + [6, 1, 6, 5, 3, 2, 3], + [4, 1, 0, 1, 0, 2, 6], + ], + [ + [0, 6, 4, 3, 3, 3, 2], + [5, 3, 2, 4, 4, 4, 6], + [0, 3, 0, 2, 4, 6, 4], + [3, 5, 1, 2, 6, 6, 1], + [0, 1, 6, 5, 3, 2, 3], + [3, 1, 0, 1, 0, 2, 6], + ], + ], + device, + )?; + + let hs = t.gather(&ids, 0)?; + assert_eq!( + hs.to_vec3::()?, + &[ + [ + [-159_f32, 126., 168., 161., -14., -33., -138.], + [-229., -41., -18., 132., -89., 169., -212.], + [223., 254., 2., -70., 87., 53., -168.], + [-221., 253., -212., 59., 154., -53., 118.], + [-144., -146., -154., -107., 31., 171., -246.], + [82., -147., -10., -253., -242., 161., -87.] + ], + [ + [-10., 126., 168., 161., 126., 2., -78.], + [25., -41., -18., 132., -89., 169., -212.], + [-248., 254., 2., -70., 87., 53., -168.], + [-33., 253., -212., 59., 154., -53., 118.], + [158., -146., -154., -107., 31., 171., -246.], + [-125., -147., -10., -253., -242., 161., -87.] + ], + [ + [-153., 126., 168., 161., 126., 2., -78.], + [-137., -41., -18., 132., -89., 169., -212.], + [187., 254., 2., -70., 87., 53., -168.], + [-246., 253., -212., 59., 154., -53., 118.], + [186., -146., -154., -107., 31., 171., -246.], + [30., -147., -10., -253., -242., 161., -87.] + ], + [ + [108., 126., 168., 161., 126., 2., -78.], + [-1., -41., -18., 132., -89., 169., -212.], + [-9., 254., 2., -70., 87., 53., -168.], + [65., 253., -212., 59., 154., -53., 118.], + [8., -146., -154., -107., 31., 171., -246.], + [176., -147., -10., -253., -242., 161., -87.] + ] + ] + ); + + // Dim: 1 + let t = Tensor::new( + &[ + [ + [-117_f32, -175., 69., -163.], + [200., 242., -21., -67.], + [179., 150., -126., -75.], + [-118., 38., -138., -13.], + [-221., 136., -185., 180.], + [58., 182., -204., -149.], + ], + [ + [3., -148., -58., -154.], + [-43., 45., -108., 4.], + [-69., -249., -71., -21.], + [80., 110., -152., -235.], + [-88., 7., 92., -250.], + [-186., 207., -242., 98.], + ], + [ + [238., 19., 64., -242.], + [-150., -97., 218., 58.], + [111., -233., 204., -212.], + [-242., -232., 83., 42.], + [153., 62., -251., 219.], + [-117., 36., -119., 10.], + ], + [ + [215., 159., -169., -27.], + [-83., 101., -88., 169.], + [-205., 93., 225., -64.], + [-162., 240., 214., 23.], + [-112., 6., 21., 245.], + [-38., 113., 93., 215.], + ], + [ + [91., -188., -148., 101.], + [74., 203., -35., 55.], + [-116., -130., -153., -96.], + [58., 22., -45., -194.], + [-221., -134., 73., 159.], + [-203., -254., 31., 235.], + ], + [ + [105., -53., 61., 186.], + [-195., 234., 75., -1.], + [51., 139., 160., -108.], + [-173., -167., 161., 19.], + [83., -246., 156., -222.], + [109., 39., -149., 137.], + ], + ], + device, + )?; + + let ids = Tensor::new( + &[ + [[4_u32, 4, 4, 2]], + [[0, 4, 4, 3]], + [[1, 5, 3, 4]], + [[0, 3, 3, 2]], + [[1, 1, 5, 2]], + [[1, 4, 5, 4]], + ], + device, + )?; + + let hs = t.gather(&ids, 1)?; + assert_eq!( + hs.to_vec3::()?, + &[ + [[-221., 136., -185., -75.]], + [[3., 7., 92., -235.]], + [[-150., 36., 83., 219.]], + [[215., 240., 214., -64.]], + [[74., 203., 31., -96.]], + [[-195., -246., -149., -222.]] + ] + ); + + // Dim: 2 + let t = Tensor::new( + &[ + [[-162_f32, 202.], [-126., -39.], [35., -65.], [1., 80.]], + [[37., 248.], [-191., 89.], [117., -40.], [-217., 220.]], + ], + device, + )?; + + let ids = Tensor::new(&[[[1_u32], [0], [1], [1]], [[0], [1], [0], [1]]], device)?; + + let hs = t.gather(&ids, 2)?; + assert_eq!( + hs.to_vec3::()?, + &[ + [[202.], [-126.], [-65.], [80.]], + [[37.], [89.], [117.], [220.]] + ] + ); + + let t = Tensor::new( + &[ + [[-21_f32, -197.], [194., 122.]], + [[255., -106.], [-191., 250.]], + [[33., -117.], [43., 10.]], + [[-130., 238.], [-217., -92.]], + ], + device, + )?; + + let ids = Tensor::new( + &[ + [[0_u32, 1], [1, 0]], + [[1, 0], [0, 1]], + [[0, 1], [0, 1]], + [[1, 0], [1, 0]], + ], + device, + )?; + + let hs = t.gather(&ids, 2)?; + assert_eq!( + hs.to_vec3::()?, + &[ + [[-21., -197.], [122., 194.]], + [[-106., 255.], [-191., 250.]], + [[33., -117.], [43., 10.]], + [[238., -130.], [-92., -217.]] + ] + ); + + Ok(()) +} + +fn broadcasting(device: &Device) -> Result<()> { + let t1 = Tensor::arange(0f32, 24f32, device)?.reshape((4, 2, 3))?; + let t2 = Tensor::new(&[100f32, 200f32], device)?; + let s = t1.broadcast_add(&t2.reshape((2, 1))?)?; + assert_eq!( + s.to_vec3::()?, + &[ + [[100.0, 101.0, 102.0], [203.0, 204.0, 205.0]], + [[106.0, 107.0, 108.0], [209.0, 210.0, 211.0]], + [[112.0, 113.0, 114.0], [215.0, 216.0, 217.0]], + [[118.0, 119.0, 120.0], [221.0, 222.0, 223.0]] + ] + ); + let s = t1.t()?.broadcast_add(&t2)?; + assert_eq!( + s.to_vec3::()?, + &[ + [[100.0, 203.0], [101.0, 204.0], [102.0, 205.0]], + [[106.0, 209.0], [107.0, 210.0], [108.0, 211.0]], + [[112.0, 215.0], [113.0, 216.0], [114.0, 217.0]], + [[118.0, 221.0], [119.0, 222.0], [120.0, 223.0]] + ] + ); + let s = t1.broadcast_sub(&t2.reshape((2, 1))?)?; + assert_eq!( + s.to_vec3::()?, + &[ + [[-100.0, -99.0, -98.0], [-197.0, -196.0, -195.0]], + [[-94.0, -93.0, -92.0], [-191.0, -190.0, -189.0]], + [[-88.0, -87.0, -86.0], [-185.0, -184.0, -183.0]], + [[-82.0, -81.0, -80.0], [-179.0, -178.0, -177.0]] + ] + ); + let s = t1.t()?.broadcast_sub(&t2)?; + assert_eq!( + s.to_vec3::()?, + &[ + [[-100.0, -197.0], [-99.0, -196.0], [-98.0, -195.0]], + [[-94.0, -191.0], [-93.0, -190.0], [-92.0, -189.0]], + [[-88.0, -185.0], [-87.0, -184.0], [-86.0, -183.0]], + [[-82.0, -179.0], [-81.0, -178.0], [-80.0, -177.0]] + ] + ); + // Test a narrowed version as this uses a layout start_offset. + let t1 = t1.i(2..)?; + let s = t1.broadcast_add(&t2.reshape((2, 1))?)?; + assert_eq!( + s.to_vec3::()?, + &[ + [[112.0, 113.0, 114.0], [215.0, 216.0, 217.0]], + [[118.0, 119.0, 120.0], [221.0, 222.0, 223.0]] + ] + ); + let s = t1.t()?.broadcast_add(&t2)?; + assert_eq!( + s.to_vec3::()?, + &[ + [[112.0, 215.0], [113.0, 216.0], [114.0, 217.0]], + [[118.0, 221.0], [119.0, 222.0], [120.0, 223.0]] + ] + ); + let s = t1.broadcast_sub(&t2.reshape((2, 1))?)?; + assert_eq!( + s.to_vec3::()?, + &[ + [[-88.0, -87.0, -86.0], [-185.0, -184.0, -183.0]], + [[-82.0, -81.0, -80.0], [-179.0, -178.0, -177.0]] + ] + ); + let s = t1.t()?.broadcast_sub(&t2)?; + assert_eq!( + s.to_vec3::()?, + &[ + [[-88.0, -185.0], [-87.0, -184.0], [-86.0, -183.0]], + [[-82.0, -179.0], [-81.0, -178.0], [-80.0, -177.0]] + ] + ); + let t3 = Tensor::new(1f32, device)?.broadcast_div(&t2)?; + let s = t1.broadcast_mul(&t2.reshape((2, 1))?)?; + let s_div = t1.broadcast_div(&t3.reshape((2, 1))?)?; + assert_eq!( + s.to_vec3::()?, + &[ + [[1200.0, 1300.0, 1400.0], [3000.0, 3200.0, 3400.0]], + [[1800.0, 1900.0, 2000.0], [4200.0, 4400.0, 4600.0]] + ] + ); + assert_eq!(s.to_vec3::()?, s_div.to_vec3::()?,); + let s = t1.t()?.broadcast_mul(&t2)?; + let s_div = t1.t()?.broadcast_div(&t3)?; + assert_eq!( + s.to_vec3::()?, + &[ + [[1200.0, 3000.0], [1300.0, 3200.0], [1400.0, 3400.0]], + [[1800.0, 4200.0], [1900.0, 4400.0], [2000.0, 4600.0]] + ] + ); + assert_eq!(s.to_vec3::()?, s_div.to_vec3::()?,); + Ok(()) +} + +fn randn(device: &Device) -> Result<()> { + let tensor = Tensor::randn(0f32, 1f32, (5, 3), device)?; + assert_eq!(tensor.dims(), [5, 3]); + // Check that the seed gets updated by checking that + // a new series of numbers is generated each time + let tensor2 = Tensor::randn(0f32, 1f32, (5, 3), device)?; + assert_ne!(tensor.to_vec2::()?, tensor2.to_vec2::()?); + let tensor = Tensor::rand(0f32, 1f32, (5, 3), device)?; + assert_eq!(tensor.dims(), [5, 3]); + // Check that the seed gets updated by checking that + // a new series of numbers is generated each time + let tensor2 = Tensor::rand(0f32, 1f32, (5, 3), device)?; + assert_ne!(tensor.to_vec2::()?, tensor2.to_vec2::()?); + // We do not expect deterministic elements at any index. + // There once was a bug that had a deterministic zero element in evenly sized tensors. + const N: usize = 2; + let v = (0..100) + .map(|_| Tensor::randn(0f32, 1f32, N, device).and_then(|t| t.to_vec1::())) + .collect::>>()?; + assert!( + (0..N).all(|i| v.windows(2).any(|pair| pair[0][i] != pair[1][i])), + "There are deterministic values in the randn tensors" + ); + let v = (0..100) + .map(|_| Tensor::rand(0f32, 1f32, N, device).and_then(|t| t.to_vec1::())) + .collect::>>()?; + assert!( + (0..N).all(|i| v.windows(2).any(|pair| pair[0][i] != pair[1][i])), + "There are deterministic values in the rand tensors" + ); + Ok(()) +} + +fn zero_dim(device: &Device) -> Result<()> { + let t = Tensor::zeros((4, 0, 1), DType::F32, device)?; + assert_eq!(t.dims3()?, (4, 0, 1)); + let t2 = Tensor::zeros((4, 3, 1), DType::F32, device)?; + let t_cat = Tensor::cat(&[&t, &t2], 1)?; + assert_eq!(t_cat.dims3()?, (4, 3, 1)); + let t_cat = Tensor::cat(&[&t, &t], 1)?; + assert_eq!(t_cat.dims3()?, (4, 0, 1)); + let t_unary = t.sqrt()?; + assert_eq!(t_unary.dims3()?, (4, 0, 1)); + let t_plus = (&t + 1.)?; + assert_eq!(t_plus.dims3()?, (4, 0, 1)); + let t_mm = t2.matmul(&t.t()?)?; + assert_eq!(t_mm.dims3()?, (4, 3, 0)); + let t_mm = t.matmul(&t2.t()?)?; + assert_eq!(t_mm.dims3()?, (4, 0, 3)); + let t_mm = t.t()?.matmul(&t)?; + assert_eq!(t_mm.dims3()?, (4, 1, 1)); + Ok(()) +} + +test_device!(zeros, zeros_cpu, zeros_gpu, zeros_metal); +test_device!(ones, ones_cpu, ones_gpu, ones_metal); +test_device!(full, full_cpu, full_gpu, full_metal); +test_device!(const_set, cs_cpu, cs_gpu, cs_metal); +test_device!(arange, arange_cpu, arange_gpu, arange_metal); +test_device!(add_mul, add_mul_cpu, add_mul_gpu, add_mul_metal); +test_device!(tensor_2d, tensor_2d_cpu, tensor_2d_gpu, tensor_2d_metal); +test_device!(narrow, narrow_cpu, narrow_gpu, narrow_metal); +test_device!(broadcast, broadcast_cpu, broadcast_gpu, broadcast_metal); +test_device!(slice_set, ss_cpu, ss_gpu, ss_metal); +test_device!(cat, cat_cpu, cat_gpu, cat_metal); +test_device!(sum, sum_cpu, sum_gpu, sum_metal); +test_device!(min, min_cpu, min_gpu, min_metal); +test_device!(max, max_cpu, max_gpu, max_metal); +test_device!(argmax, argmax_cpu, argmax_gpu, argmax_metal); +test_device!(argmin, argmin_cpu, argmin_gpu, argmin_metal); +test_device!(transpose, transpose_cpu, transpose_gpu, transpose_metal); +test_device!(unary_op, unary_op_cpu, unary_op_gpu, unary_op_metal); +test_device!(binary_op, binary_op_cpu, binary_op_gpu, binary_op_metal); +test_device!(ternary_op, ternary_op_cpu, ternary_op_gpu, ternary_op_metal); +test_device!(embeddings, embeddings_cpu, embeddings_gpu, embeddings_metal); +test_device!(cmp, cmp_cpu, cmp_gpu, cmp_metal); +test_device!( + broadcasting, + broadcasting_cpu, + broadcasting_gpu, + broadcasting_metal +); +test_device!( + index_select, + index_select_cpu, + index_select_gpu, + index_select_metal +); +test_device!(index_add, index_add_cpu, index_add_gpu, index_add_metal); +test_device!(gather, gather_cpu, gather_gpu, gather_metal); +test_device!(scatter, scatter_cpu, scatter_gpu, scatter_metal); +test_device!( + slice_scatter, + slice_scatter_cpu, + slice_scatter_gpu, + slice_scatter_metal +); +test_device!(randn, randn_cpu, randn_gpu, randn_metal); +test_device!(clamp, clamp_cpu, clamp_gpu, clamp_metal); +test_device!(asort, asort_cpu, asort_gpu, asort_metal); +test_device!(asort_big, asort_big_cpu, asort_big_gpu, asort_big_metal); +test_device!(var, var_cpu, var_gpu, var_metal); +test_device!(zero_dim, zero_dim_cpu, zero_dim_gpu, zero_dim_metal); + +fn tensor_send_sync(device: &Device) -> Result<()> { + let tensor = Tensor::new(vec![1.0f32, 2.0, 3.0], device)?; + + for _ in 0..10 { + let tensor = tensor.clone(); + std::thread::spawn(move || { + let new = tensor.add(&tensor).unwrap(); + let result: Vec = new.to_vec1().unwrap(); + assert_eq!(result, vec![2.0f32, 4.0, 6.0]); + }); + } + let result: Vec = tensor.to_vec1().unwrap(); + assert_eq!(result, vec![1.0f32, 2.0, 3.0]); + + let tensor = Tensor::new(vec![1.0f32, 2.0, 3.0], device)?; + tensor.device().synchronize().unwrap(); + + let new = std::thread::spawn(move || { + let new = tensor.add(&tensor).unwrap(); + new.device().synchronize().unwrap(); + new + }) + .join() + .unwrap(); + let result: Vec = new.to_vec1().unwrap(); + assert_eq!(result, vec![2.0f32, 4.0, 6.0]); + + Ok(()) +} +test_device!( + tensor_send_sync, + tensor_send_sync_cpu, + tensor_send_sync_gpu, + tensor_send_sync_metal +); + +// There was originally a bug on the CPU implementation for randn +// https://github.com/huggingface/candle/issues/381 +#[test] +fn randn_hasneg() -> Result<()> { + let t = Tensor::randn(0f32, 1f32, 200, &Device::Cpu)?.to_vec1::()?; + if t.iter().all(|&v| v >= 0.) { + candle_core::bail!("all values in tensors are non-negative") + } + Ok(()) +} + +#[test] +fn pad_with_same() -> Result<()> { + let t = Tensor::arange(1f32, 5f32, &Device::Cpu)?.reshape((2, 2))?; + let t0 = t.pad_with_same(0, 1, 2)?; + assert_eq!( + t0.to_vec2::()?, + [[1.0, 2.0], [1.0, 2.0], [3.0, 4.0], [3.0, 4.0], [3.0, 4.0]] + ); + let t1 = t.pad_with_same(1, 1, 2)?; + assert_eq!( + t1.to_vec2::()?, + [[1.0, 1.0, 2.0, 2.0, 2.0], [3.0, 3.0, 4.0, 4.0, 4.0]] + ); + Ok(()) +} + +#[test] +fn i64_abs() -> Result<()> { + let t = Tensor::new(&[-42i64, 1337], &Device::Cpu)?; + let t = t.abs()?; + assert_eq!(t.to_vec1::()?, [42, 1337]); + Ok(()) +} + +#[test] +fn tril_triu_eye() -> Result<()> { + let t = Tensor::tril2(4, DType::F32, &Device::Cpu)?; + assert_eq!( + t.to_vec2::()?, + [ + [1.0, 0.0, 0.0, 0.0], + [1.0, 1.0, 0.0, 0.0], + [1.0, 1.0, 1.0, 0.0], + [1.0, 1.0, 1.0, 1.0] + ], + ); + let t = Tensor::triu2(4, DType::F32, &Device::Cpu)?; + assert_eq!( + t.to_vec2::()?, + [ + [1.0, 1.0, 1.0, 1.0], + [0.0, 1.0, 1.0, 1.0], + [0.0, 0.0, 1.0, 1.0], + [0.0, 0.0, 0.0, 1.0] + ] + ); + let t = Tensor::eye(4, DType::F32, &Device::Cpu)?; + assert_eq!( + t.to_vec2::()?, + [ + [1.0, 0.0, 0.0, 0.0], + [0.0, 1.0, 0.0, 0.0], + [0.0, 0.0, 1.0, 0.0], + [0.0, 0.0, 0.0, 1.0] + ] + ); + Ok(()) +} + +#[test] +fn cumsum() -> Result<()> { + let t = &[3f32, 1., 4., 1., 5.]; + let t = Tensor::new(t, &Device::Cpu)?; + assert_eq!(t.cumsum(0)?.to_vec1::()?, [3., 4., 8., 9., 14.]); + let t = t.unsqueeze(1)?; + assert_eq!( + t.cumsum(0)?.to_vec2::()?, + [[3.0], [4.0], [8.0], [9.0], [14.0]] + ); + assert_eq!( + t.cumsum(1)?.to_vec2::()?, + [[3.0], [1.0], [4.0], [1.0], [5.0]] + ); + let t = &[[3f32, 1., 4., 1., 5.], [2., 1., 7., 8., 2.]]; + let t = Tensor::new(t, &Device::Cpu)?; + assert_eq!( + t.cumsum(1)?.to_vec2::()?, + [[3.0, 4.0, 8.0, 9.0, 14.0], [2.0, 3.0, 10.0, 18.0, 20.0]], + ); + assert_eq!( + t.cumsum(0)?.to_vec2::()?, + [[3.0, 1.0, 4.0, 1.0, 5.0], [5.0, 2.0, 11.0, 9.0, 7.0]] + ); + Ok(()) +} + +/// A helper function for floating point comparison. Both a and b must be 1D Tensor and contains the same amount of data. +/// Assertion passes if the difference of all pairs of a and b is smaller than epsilon. +fn assert_close(a: &Tensor, b: &Tensor, epsilon: f64) -> Result<()> { + let a_vec: Vec = a.to_vec1()?; + let b_vec: Vec = b.to_vec1()?; + + assert_eq!(a_vec.len(), b_vec.len()); + for (a, b) in a_vec.iter().zip(b_vec.iter()) { + assert!((a - b).abs() < epsilon); + } + Ok(()) +} + +#[test] +fn log_sum_exp() -> Result<()> { + let input = Tensor::new( + &[ + [[1f64, 2., 3.], [4., 5., 6.]], + [[-1000.0, -999.0, -1001.0], [1000.0, 999.0, 1001.0]], + ], + &Device::Cpu, + )?; + + let output = input.log_sum_exp(D::Minus1)?; + // The expectations obtained from pytorch. + let expected = Tensor::new(&[[3.4076, 6.4076], [-998.5924, 1001.4076]], &Device::Cpu)?; + assert_eq!(output.dims(), expected.dims()); + assert_close(&output.flatten_all()?, &expected.flatten_all()?, 0.00001)?; + + assert_eq!( + input.log_sum_exp((0, 1))?.to_vec1::()?, + [1000.0, 999.0, 1001.0] + ); + assert_eq!( + input.log_sum_exp(())?.to_vec3::()?, + input.to_vec3::()? + ); + + Ok(()) +} + +#[test] +fn pow() -> Result<()> { + let lhs = Tensor::new(&[[1f32, 2., 3.], [4., 5., 6.]], &Device::Cpu)?; + let rhs = (&lhs - 2.)?; + let res = lhs.pow(&rhs)?; + assert_eq!( + test_utils::to_vec2_round(&res, 3)?, + [[1.0, 1.0, 3.0], [16.0, 125.0, 1296.0]] + ); + Ok(()) +} + +#[test] +fn test_flip_1d() -> Result<()> { + // 1D: [0, 1, 2, 3, 4] + let t = Tensor::arange(0.0, 5.0, &Device::Cpu)?.reshape((5,))?; + let flipped = t.flip(&[0])?; + // Expected: [4, 3, 2, 1, 0] + let expected = Tensor::from_vec(vec![4.0, 3.0, 2.0, 1.0, 0.0], (5,), &Device::Cpu)?; + candle_core::test_utils::assert_tensor_eq(&flipped, &expected)?; + Ok(()) +} + +#[test] +fn test_flip_2d() -> Result<()> { + // 2D: + // [[0, 1, 2], + // [3, 4, 5]] + let t = Tensor::arange(0.0, 6.0, &Device::Cpu)?.reshape((2, 3))?; + let flipped = t.flip(&[0, 1])?; + // Expected: + // [[5, 4, 3], + // [2, 1, 0]] + let expected = Tensor::from_vec(vec![5.0, 4.0, 3.0, 2.0, 1.0, 0.0], (2, 3), &Device::Cpu)?; + candle_core::test_utils::assert_tensor_eq(&flipped, &expected)?; + Ok(()) +} + +#[test] +fn test_flip_3d_channels() -> Result<()> { + // 3D: + // [[[0,1,2], + // [3,4,5]], + // + // [[6,7,8], + // [9,10,11]]] + let t = Tensor::arange(0.0, 12.0, &Device::Cpu)?.reshape((2, 2, 3))?; + let flipped = t.flip(&[2])?; + // Expected: + // [[[2,1,0], + // [5,4,3]], + // + // [[8,7,6], + // [11,10,9]]] + let expected = Tensor::from_vec( + vec![2.0, 1.0, 0.0, 5.0, 4.0, 3.0, 8.0, 7.0, 6.0, 11.0, 10.0, 9.0], + (2, 2, 3), + &Device::Cpu, + )?; + candle_core::test_utils::assert_tensor_eq(&flipped, &expected)?; + Ok(()) +} + +#[test] +fn tensor_new() -> Result<()> { + let t1 = Tensor::new(vec![1f32, 2.0, 3.0], &Device::Cpu)?; + assert_eq!(t1.to_vec1::()?, [1.0, 2.0, 3.0]); + let t2 = Tensor::new(vec![vec![1f32, 2., 3.], vec![4., 5., 6.]], &Device::Cpu)?; + assert_eq!(t2.to_vec2::()?, [[1., 2., 3.], [4., 5., 6.]]); + let t3 = Tensor::new( + vec![ + vec![vec![1f32, 2., 3.], vec![4., 5., 6.]], + vec![vec![3f32, 1., 4.], vec![1., 5., 9.]], + ], + &Device::Cpu, + )?; + assert_eq!( + t3.to_vec3::()?, + [ + [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], + [[3.0, 1.0, 4.0], [1.0, 5.0, 9.0]] + ] + ); + Ok(()) +} + +#[test] +fn tensor_norm() -> Result<()> { + let t = Tensor::new(&[[3., 4.], [0., 0.]], &Device::Cpu)?; + let norm = t.norm()?; + assert_eq!(norm.to_scalar::()?, 5.); + Ok(()) +} + +#[cfg(feature = "cuda")] +#[test] +fn transfers_cuda_to_device() -> Result<()> { + use rand::seq::SliceRandom; + + let devices = cudarc::driver::safe::CudaContext::device_count() + .map_err(candle_core::cuda::CudaError::from)?; + if devices < 2 { + return Ok(()); + } + let first = Device::new_cuda(0)?; + + let mut data: Vec = (0..262144).collect(); + let mut rng = rand::rng(); + data.shuffle(&mut rng); + + let t1 = Tensor::from_vec(data, (512, 512), &first)?; + let second = Device::new_cuda(1)?; + let t2 = t1.to_device(&second)?; + + assert_ne!( + t1.device().as_cuda_device()?.id(), + t2.device().as_cuda_device()?.id() + ); + Ok(()) +} + +#[cfg(feature = "cuda")] +#[test] +fn allocates_twice_when_transferring_to_same_device() -> Result<()> { + use std::{ops::Deref, sync::RwLockReadGuard}; + + use candle_core::Storage; + use rand::seq::SliceRandom; + + let first = Device::new_cuda(0)?; + let second = Device::new_cuda(0)?; + + let mut data: Vec = (0..262144).collect(); + let mut rng = rand::rng(); + data.shuffle(&mut rng); + + let t1 = Tensor::from_vec(data, (512, 512), &first)?; + let t2 = t1.to_device(&second)?; + + let (storage1, _) = t1.storage_and_layout(); + let (storage2, _) = t2.storage_and_layout(); + let extract = |s: RwLockReadGuard<'_, Storage>| match &s.deref() { + Storage::Cuda(c) => { + use cudarc::driver::DevicePtr; + let slice = c.as_cuda_slice::().unwrap(); + let ptr = slice.device_ptr(slice.stream()).0; + ptr + } + _ => unimplemented!(), + }; + let id1 = extract(storage1); + let id2 = extract(storage2); + assert_ne!(id1, id2); + Ok(()) +} diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/test.npy b/v2/dependency-patches/candle-core-0.9.2/tests/test.npy new file mode 100644 index 0000000000000000000000000000000000000000..a3ff5af9540510ac6518b5a2478bc1a40f992371 GIT binary patch literal 208 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= wXCxM+0{I$-20EHL3bhL411<(AV1&|4P?{M^vp{K9D9r|?*`YKCl;(ue0P!XpoB#j- literal 0 HcmV?d00001 diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/test.npz b/v2/dependency-patches/candle-core-0.9.2/tests/test.npz new file mode 100644 index 0000000000000000000000000000000000000000..b6683caab0e0e9e66cb5ca649aab4980a1e650df GIT binary patch literal 668 zcmWIWW@Zs#fB;1X8xBpr3qTGCvoeSRzLK#d@ni)#7KxtMe%?72}p)?1S=7iD#D8ZokVdgfgc5ZyS5W1OUF4@IgfaSj3!+p!yZC;|&vLW_h60tGIXYp+tbR@}8WmEgpM zO8o`Bapb}U_$P4S5AY|DxUgf#!FKBdBaPM{^PXqkJlnY0LP(O(mFS~w#7M$MzRTR< z=}BmrTj)(S-V2@=g(2}+5G@kTjR$rRn5(#DL_r#w{fSQL9lLubi?Fp-^@ z94>>866e`Y|{Pa$jw1Xu^ zrRNx70k_WS4cx+wb1GjPK?hw%dsEs^hBod<4~{Qif95DiyLR*qw3Cf~_?D3fMzYN9 ztFwa^VZc~OfcnG1uN{%oLe^7v(W8H=jugv0xSqGFbKN~i0U=}u0!bi1b%=I$q+P)! zs0)!V96JB>^~;lo{263@=RMAm`r(Yk317Bl3cU5P3=H2t&R;?Yko63^$jBImV5w0; z;=m1qcP?1R34F^`|6dDL)%SE=n`qfpi3(kQ`+4)n zfOiOK<3=i3?);)L`uu%4O8-F>T3q}_3HOKJrO>sHs@S6R-4gZ{-|YIcBc%!~D&8vr zH~Cz}JynrK6?RL=X^y;pKQcsFZQ}Y+30Wu-S|m&nlEnez+N;E>iM#fu4OV8_A{%=Ua&@QUk(LBL$$hm%--75a)d;S;$UxT1w^n387>AHq2KYo>>4lqPFX(eg-Z1eK!Yr}cbF5km4 zMqWW11%Nh;xri}WZ5bp#%(Vi@y9JO`dS!(CFxCqQCOo-FBa=WGBHyXa0>t+Ph|6ba t(q?lj%e+;)Gsl?ec^02%3viYRAMz53|Hu{qk4@UZrG#B*5WJt>{S7%wAvgd4 literal 0 HcmV?d00001 diff --git a/v2/dependency-patches/compat-tests/Cargo.lock b/v2/dependency-patches/compat-tests/Cargo.lock new file mode 100644 index 0000000000..b47c10c170 --- /dev/null +++ b/v2/dependency-patches/compat-tests/Cargo.lock @@ -0,0 +1,887 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "bytemuck" +version = "1.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a1f896587b6f2c069c73d2f0913e2d590c3990285cd2f0b6aa02b786b4c679c" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "candle-core" +version = "0.4.1" +dependencies = [ + "byteorder", + "gemm", + "half", + "memmap2", + "num-traits", + "num_cpus", + "rand", + "rand_distr", + "rayon", + "safetensors", + "thiserror", + "yoke", + "zip", +] + +[[package]] +name = "cfg-if" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" + +[[package]] +name = "cfg_aliases" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" + +[[package]] +name = "crc32fast" +version = "1.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + +[[package]] +name = "dyn-stack" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e53799688f5632f364f8fb387488dd05db9fe45db7011be066fc20e7027f8b" +dependencies = [ + "bytemuck", + "reborrow", +] + +[[package]] +name = "either" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" + +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "equivalent" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" + +[[package]] +name = "gemm" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ab24cc62135b40090e31a76a9b2766a501979f3070fa27f689c27ec04377d32" +dependencies = [ + "dyn-stack", + "gemm-c32", + "gemm-c64", + "gemm-common", + "gemm-f16", + "gemm-f32", + "gemm-f64", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-c32" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c030d0b983d1e34a546b86e08f600c11696fde16199f971cd46c12e67512c0" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-c64" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbb5f2e79fefb9693d18e1066a557b4546cd334b226beadc68b11a8f9431852a" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-common" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2e7ea062c987abcd8db95db917b4ffb4ecdfd0668471d8dc54734fdff2354e8" +dependencies = [ + "bytemuck", + "dyn-stack", + "half", + "num-complex", + "num-traits", + "once_cell", + "paste", + "pulp", + "raw-cpuid", + "rayon", + "seq-macro", + "sysctl", +] + +[[package]] +name = "gemm-f16" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ca4c06b9b11952071d317604acb332e924e817bd891bec8dfb494168c7cedd4" +dependencies = [ + "dyn-stack", + "gemm-common", + "gemm-f32", + "half", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "rayon", + "seq-macro", +] + +[[package]] +name = "gemm-f32" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9a69f51aaefbd9cf12d18faf273d3e982d9d711f60775645ed5c8047b4ae113" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-f64" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa397a48544fadf0b81ec8741e5c0fba0043008113f71f2034def1935645d2b0" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "getrandom" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" +dependencies = [ + "cfg-if", + "libc", + "r-efi", +] + +[[package]] +name = "half" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872" +dependencies = [ + "bytemuck", + "cfg-if", + "crunchy", + "num-traits", + "rand", + "rand_distr", +] + +[[package]] +name = "hashbrown" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "hermit-abi" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" + +[[package]] +name = "indexmap" +version = "2.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "libc" +version = "0.2.191" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" + +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", + "stable_deref_trait", +] + +[[package]] +name = "nix" +version = "0.31.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" +dependencies = [ + "bitflags 2.13.2", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "bytemuck", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "once_cell" +version = "1.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "ppv-lite86" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "pulp" +version = "0.18.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0a01a0dc67cf4558d279f0c25b0962bd08fc6dec0137699eae304103e882fe6" +dependencies = [ + "bytemuck", + "libm", + "num-complex", + "reborrow", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + +[[package]] +name = "rand" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom 0.2.17", +] + +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand", +] + +[[package]] +name = "raw-cpuid" +version = "10.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "reborrow" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" + +[[package]] +name = "rustix" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" +dependencies = [ + "bitflags 2.13.2", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "ruview-candle04-zip9-compat-test" +version = "0.1.0" +dependencies = [ + "candle-core", + "half", + "tempfile", +] + +[[package]] +name = "safetensors" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44560c11236a6130a46ce36c836a62936dc81ebf8c36a37947423571be0e55b6" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "serde" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.229" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "serde_json" +version = "1.0.154" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "synstructure" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", +] + +[[package]] +name = "sysctl" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea" +dependencies = [ + "bitflags 2.13.2", + "byteorder", + "enum-as-inner", + "libc", + "thiserror", + "walkdir", +] + +[[package]] +name = "tempfile" +version = "3.27.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" +dependencies = [ + "fastrand", + "getrandom 0.4.3", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "thiserror" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "typed-path" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54ec2755506471f54baf5e4bb5499d8e510cc1caf271270e778a736bd2d7336c" + +[[package]] +name = "unicode-ident" +version = "1.0.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.1+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure 0.13.2", +] + +[[package]] +name = "zerocopy" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.8.63" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "zerofrom" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.9", + "synstructure 0.14.0", +] + +[[package]] +name = "zip" +version = "9.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e" +dependencies = [ + "crc32fast", + "indexmap", + "memchr", + "nix", + "typed-path", +] + +[[package]] +name = "zmij" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/v2/dependency-patches/compat-tests/Cargo.toml b/v2/dependency-patches/compat-tests/Cargo.toml new file mode 100644 index 0000000000..2b4b879f50 --- /dev/null +++ b/v2/dependency-patches/compat-tests/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "ruview-candle04-zip9-compat-test" +version = "0.1.0" +edition = "2021" +publish = false + +# Isolated because the legacy optional backend predates half's rand0.9 +# migration; the main workspace's burn backend requires half2.7. +[workspace] +[dependencies] +candle-core = { path = "../candle-core-0.4.1", default-features = false } +half = "=2.3.1" +tempfile = "3" diff --git a/v2/dependency-patches/compat-tests/src/lib.rs b/v2/dependency-patches/compat-tests/src/lib.rs new file mode 100644 index 0000000000..e9386bad6a --- /dev/null +++ b/v2/dependency-patches/compat-tests/src/lib.rs @@ -0,0 +1,14 @@ +// ZIP9 compatibility must preserve actual tensor names and values, not just compile. +use candle_core::{Device, Tensor}; +#[test] +fn npz_roundtrip_retains_unicode_and_nested_tensor_names() -> candle_core::Result<()> { + let directory = tempfile::tempdir()?; + let path = directory.path().join("tensors.npz"); + let tensor = Tensor::new(&[1f32, -2.5, 3.25], &Device::Cpu)?; + Tensor::write_npz(&[("nested/room-α", &tensor)], &path)?; + let restored = Tensor::read_npz(&path)?; + assert_eq!(restored.len(), 1); + assert_eq!(restored[0].0, "nested/room-α"); + assert_eq!(restored[0].1.to_vec1::()?, vec![1., -2.5, 3.25]); + Ok(()) +} diff --git a/v2/dependency-patches/compatibility.patch b/v2/dependency-patches/compatibility.patch new file mode 100644 index 0000000000..2ca57831b3 --- /dev/null +++ b/v2/dependency-patches/compatibility.patch @@ -0,0 +1,185 @@ +--- upstream/candle-core-0.4.1/Cargo.toml ++++ patched/candle-core-0.4.1/Cargo.toml +@@ -104,7 +104,7 @@ + features = ["derive"] + + [dependencies.zip] +-version = "0.6.6" ++version = "9.0.1" + default-features = false + + [dev-dependencies.anyhow] +--- upstream/candle-core-0.4.1/src/npy.rs ++++ patched/candle-core-0.4.1/src/npy.rs +@@ -261,8 +261,8 @@ + for i in 0..zip.len() { + let mut reader = zip.by_index(i)?; + let name = { +- let name = reader.name(); +- name.strip_suffix(NPY_SUFFIX).unwrap_or(name).to_owned() ++ let name = reader.name()?; ++ name.strip_suffix(NPY_SUFFIX).unwrap_or(&name).to_owned() + }; + let header = read_header(&mut reader)?; + let header = Header::parse(&header)?; +@@ -331,7 +331,7 @@ + ) -> Result<()> { + let mut zip = zip::ZipWriter::new(File::create(path.as_ref())?); + let options = +- zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Stored); ++ zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + + for (name, tensor) in ts.iter() { + zip.start_file(format!("{}.npy", name.as_ref()), options)?; +@@ -358,8 +358,8 @@ + for i in 0..zip.len() { + let file = zip.by_index(i)?; + let name = { +- let name = file.name(); +- name.strip_suffix(NPY_SUFFIX).unwrap_or(name).to_owned() ++ let name = file.name()?; ++ name.strip_suffix(NPY_SUFFIX).unwrap_or(&name).to_owned() + }; + index_per_name.insert(name, i); + } +--- upstream/candle-core-0.4.1/src/pickle.rs ++++ patched/candle-core-0.4.1/src/pickle.rs +@@ -653,8 +653,8 @@ + let mut zip = zip::ZipArchive::new(zip_reader)?; + let zip_file_names = zip + .file_names() +- .map(|f| f.to_string()) +- .collect::>(); ++ .map(|name| name.map(|name| name.into_owned())) ++ .collect::>>()?; + + let mut tensor_infos = vec![]; + for file_name in zip_file_names.iter() { +--- upstream/candle-core-0.9.2/Cargo.toml ++++ patched/candle-core-0.9.2/Cargo.toml +@@ -242,7 +242,7 @@ + features = ["derive"] + + [dependencies.zip] +-version = "7.2.0" ++version = "9.0.1" + default-features = false + + [dev-dependencies.anyhow] +--- upstream/candle-core-0.9.2/src/npy.rs ++++ patched/candle-core-0.9.2/src/npy.rs +@@ -288,8 +288,8 @@ + for i in 0..zip.len() { + let mut reader = zip.by_index(i)?; + let name = { +- let name = reader.name(); +- name.strip_suffix(NPY_SUFFIX).unwrap_or(name).to_owned() ++ let name = reader.name()?; ++ name.strip_suffix(NPY_SUFFIX).unwrap_or(&name).to_owned() + }; + let header = read_header(&mut reader)?; + let header = Header::parse(&header)?; +@@ -358,7 +358,7 @@ + ) -> Result<()> { + let mut zip = zip::ZipWriter::new(File::create(path.as_ref())?); + let options: zip::write::FileOptions<()> = +- zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Stored); ++ zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + + for (name, tensor) in ts.iter() { + zip.start_file(format!("{}.npy", name.as_ref()), options)?; +@@ -385,8 +385,8 @@ + for i in 0..zip.len() { + let file = zip.by_index(i)?; + let name = { +- let name = file.name(); +- name.strip_suffix(NPY_SUFFIX).unwrap_or(name).to_owned() ++ let name = file.name()?; ++ name.strip_suffix(NPY_SUFFIX).unwrap_or(&name).to_owned() + }; + index_per_name.insert(name, i); + } +--- upstream/candle-core-0.9.2/src/pickle.rs ++++ patched/candle-core-0.9.2/src/pickle.rs +@@ -677,8 +677,8 @@ + let mut zip = zip::ZipArchive::new(zip_reader)?; + let zip_file_names = zip + .file_names() +- .map(|f| f.to_string()) +- .collect::>(); ++ .map(|name| name.map(|name| name.into_owned())) ++ .collect::>>()?; + + let mut tensor_infos = vec![]; + for file_name in zip_file_names.iter() { +--- upstream/tch-0.24.0/Cargo.toml ++++ patched/tch-0.24.0/Cargo.toml +@@ -240,7 +240,7 @@ + version = "0.24.0" + + [dependencies.zip] +-version = "0.6" ++version = "9.0.1" + + [dev-dependencies.anyhow] + version = "^1.0.60" +--- upstream/tch-0.24.0/src/tensor/npy.rs ++++ patched/tch-0.24.0/src/tensor/npy.rs +@@ -188,8 +188,8 @@ + for i in 0..zip.len() { + let mut reader = zip.by_index(i).unwrap(); + let name = { +- let name = reader.name(); +- name.strip_suffix(NPY_SUFFIX).unwrap_or(name).to_owned() ++ let name = reader.name()?; ++ name.strip_suffix(NPY_SUFFIX).unwrap_or(&name).to_owned() + }; + let header = read_header(&mut reader)?; + let header = Header::parse(&header)?; +@@ -236,7 +236,7 @@ + ) -> Result<(), TchError> { + let mut zip = zip::ZipWriter::new(File::create(path.as_ref())?); + let options = +- zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Stored); ++ zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + + for (name, tensor) in ts.iter() { + zip.start_file(format!("{}.npy", name.as_ref()), options)?; +--- upstream/torch-sys-0.24.0/Cargo.toml ++++ patched/torch-sys-0.24.0/Cargo.toml +@@ -76,4 +76,4 @@ + optional = true + + [build-dependencies.zip] +-version = "0.6" ++version = "9.0.1" +--- upstream/torch-sys-0.24.0/build.rs ++++ patched/torch-sys-0.24.0/build.rs +@@ -121,25 +121,8 @@ + let file = fs::File::open(&filename)?; + let buf = io::BufReader::new(file); + let mut archive = zip::ZipArchive::new(buf)?; +- for i in 0..archive.len() { +- let mut file = archive.by_index(i)?; +- let outpath = outpath.as_ref().join(file.mangled_name()); +- if !file.name().ends_with('/') { +- println!( +- "File {} extracted to \"{}\" ({} bytes)", +- i, +- outpath.as_path().display(), +- file.size() +- ); +- if let Some(p) = outpath.parent() { +- if !p.exists() { +- fs::create_dir_all(p)?; +- } +- } +- let mut outfile = fs::File::create(&outpath)?; +- io::copy(&mut file, &mut outfile)?; +- } +- } ++ // ZIP 9's checked extraction rejects path/symlink escapes and unsafe permissions. ++ archive.extract(outpath.as_ref())?; + + // This is if we're unzipping a python wheel. + if outpath.as_ref().join("torch").exists() { diff --git a/v2/dependency-patches/patches.json b/v2/dependency-patches/patches.json new file mode 100644 index 0000000000..ac5fcd607b --- /dev/null +++ b/v2/dependency-patches/patches.json @@ -0,0 +1,1877 @@ +{ + "schema": 1, + "packages": [ + { + "package": "candle-core-0.4.1", + "archiveSha256": "6f1b20174c1707e20f4cb364a355b449803c03e9b0c9193324623cf9787a4e00", + "changedFiles": [ + "Cargo.toml", + "src/npy.rs", + "src/pickle.rs" + ], + "files": [ + { + "path": "Cargo.toml", + "upstreamSha256": "73aa7197c515b16c9391b15beaf7cec96ae2d9023b1fa445ae39d8f3e7c835f1", + "sha256": "8da30d82317883eaba530173eba8fa663eba366e05f60728156ca79c3b38409f" + }, + { + "path": "Cargo.toml.orig", + "upstreamSha256": "f63b2a021badc93735c8fbce9922d08cb171f1dea8cc6cc47c0c0e89165e427f", + "sha256": "f63b2a021badc93735c8fbce9922d08cb171f1dea8cc6cc47c0c0e89165e427f" + }, + { + "path": "LICENSE", + "upstreamSha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4", + "sha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4" + }, + { + "path": "README.md", + "upstreamSha256": "9b00007b1d568125e67d6f2ff5c40acd8227dec4c62681cbc87775768a125200", + "sha256": "9b00007b1d568125e67d6f2ff5c40acd8227dec4c62681cbc87775768a125200" + }, + { + "path": "benches/bench_main.rs", + "upstreamSha256": "74b8b99df438f178eaa64bdff5c94eefbc67d2bedac347e3978feb32430db22f", + "sha256": "74b8b99df438f178eaa64bdff5c94eefbc67d2bedac347e3978feb32430db22f" + }, + { + "path": "benches/benchmarks/affine.rs", + "upstreamSha256": "2bd50867a662b354c63531fa2cf33dbc62cdc5a27006fe2defae655076d19e7a", + "sha256": "2bd50867a662b354c63531fa2cf33dbc62cdc5a27006fe2defae655076d19e7a" + }, + { + "path": "benches/benchmarks/matmul.rs", + "upstreamSha256": "268cd905c3e1a2d76b87bdee52a57adcae8f60accb61faea28492c10533bd2a5", + "sha256": "268cd905c3e1a2d76b87bdee52a57adcae8f60accb61faea28492c10533bd2a5" + }, + { + "path": "benches/benchmarks/mod.rs", + "upstreamSha256": "6e4538e0bd00bc1c90aad32485ad71b9944b47c1bffc6f9d85ac2a1977a1f852", + "sha256": "6e4538e0bd00bc1c90aad32485ad71b9944b47c1bffc6f9d85ac2a1977a1f852" + }, + { + "path": "benches/benchmarks/random.rs", + "upstreamSha256": "d19e9712cabd73fd2f9b630b8e0dbdd227b987282b70b86026e6789b985a193b", + "sha256": "d19e9712cabd73fd2f9b630b8e0dbdd227b987282b70b86026e6789b985a193b" + }, + { + "path": "benches/benchmarks/where_cond.rs", + "upstreamSha256": "9da8cedbf0c0cd5026f3b75b5c8bf99a3620216ff8c1de5b9866083decf1bbf7", + "sha256": "9da8cedbf0c0cd5026f3b75b5c8bf99a3620216ff8c1de5b9866083decf1bbf7" + }, + { + "path": "examples/basics.rs", + "upstreamSha256": "834936869ca3542647c1c0feaafce578708cda17381b8010fb9ce2c1b78ef838", + "sha256": "834936869ca3542647c1c0feaafce578708cda17381b8010fb9ce2c1b78ef838" + }, + { + "path": "examples/cuda_basics.rs", + "upstreamSha256": "ef49742e6d60eefc67369bb2ad7148afa6af3fcb8a90614d6988163cc2894505", + "sha256": "ef49742e6d60eefc67369bb2ad7148afa6af3fcb8a90614d6988163cc2894505" + }, + { + "path": "examples/cuda_sum_benchmark.rs", + "upstreamSha256": "8c1ccac409d77695793186588274f8b08c2c66337f0751485634a0329a55a9b7", + "sha256": "8c1ccac409d77695793186588274f8b08c2c66337f0751485634a0329a55a9b7" + }, + { + "path": "examples/tensor-tools.rs", + "upstreamSha256": "7a1d7b40bf6f8bd6ef288357a23fe5030f1907629cd046e5746a1a516dc15f67", + "sha256": "7a1d7b40bf6f8bd6ef288357a23fe5030f1907629cd046e5746a1a516dc15f67" + }, + { + "path": "src/accelerate.rs", + "upstreamSha256": "9be99929bb6553d9baf6b6a5a56fa66bfdbee122f76c4792a00e09ac8a0d638a", + "sha256": "9be99929bb6553d9baf6b6a5a56fa66bfdbee122f76c4792a00e09ac8a0d638a" + }, + { + "path": "src/backend.rs", + "upstreamSha256": "4ea8f8411ee365aadcf07c3dfa3e5b7b9120d888e638103a783b9b0c51b83572", + "sha256": "4ea8f8411ee365aadcf07c3dfa3e5b7b9120d888e638103a783b9b0c51b83572" + }, + { + "path": "src/backprop.rs", + "upstreamSha256": "e40a3a24153e8b3c589447a90e1598d7a2f3c41e0eec6851c54259ccf1bb7615", + "sha256": "e40a3a24153e8b3c589447a90e1598d7a2f3c41e0eec6851c54259ccf1bb7615" + }, + { + "path": "src/conv.rs", + "upstreamSha256": "973c9176dad4060b7ec749c080a9ce4098c05266bff585b0ac2df28c1050379e", + "sha256": "973c9176dad4060b7ec749c080a9ce4098c05266bff585b0ac2df28c1050379e" + }, + { + "path": "src/convert.rs", + "upstreamSha256": "859ba5f6c8a9ae69dd1f5442904cd86202d98c24d96d55c9c1cfe04ff37b50a0", + "sha256": "859ba5f6c8a9ae69dd1f5442904cd86202d98c24d96d55c9c1cfe04ff37b50a0" + }, + { + "path": "src/cpu/avx.rs", + "upstreamSha256": "6a30f8f1c6a7c6b103ce7d675d84f550a622369839b115399f65d72a504cbd10", + "sha256": "6a30f8f1c6a7c6b103ce7d675d84f550a622369839b115399f65d72a504cbd10" + }, + { + "path": "src/cpu/erf.rs", + "upstreamSha256": "d71df728dca01fd96e9fffef7778149ac1ac776d65434d0087f9ae9bb266224c", + "sha256": "d71df728dca01fd96e9fffef7778149ac1ac776d65434d0087f9ae9bb266224c" + }, + { + "path": "src/cpu/kernels.rs", + "upstreamSha256": "ac7cbf3b316c2ff0b532d3c74a2b1e086bbf074899ffebeaa9d97bf337577c6a", + "sha256": "ac7cbf3b316c2ff0b532d3c74a2b1e086bbf074899ffebeaa9d97bf337577c6a" + }, + { + "path": "src/cpu/mod.rs", + "upstreamSha256": "ce333c5327a3250d3efd04dac983f27341480c13822f1a63e24832f928bc932b", + "sha256": "ce333c5327a3250d3efd04dac983f27341480c13822f1a63e24832f928bc932b" + }, + { + "path": "src/cpu/neon.rs", + "upstreamSha256": "c180dffb8a342c2f026d8fdcc235c7d9e8f0aac8a5f6f03de4c2f4de8dda609c", + "sha256": "c180dffb8a342c2f026d8fdcc235c7d9e8f0aac8a5f6f03de4c2f4de8dda609c" + }, + { + "path": "src/cpu/simd128.rs", + "upstreamSha256": "8ed8cff924086325c4d4af5a12746a7ce4a5a05f18cdeaa28bc0ff9c64060906", + "sha256": "8ed8cff924086325c4d4af5a12746a7ce4a5a05f18cdeaa28bc0ff9c64060906" + }, + { + "path": "src/cpu_backend.rs", + "upstreamSha256": "c136829a246ef9088e84926c73b8721327473d893756f7bd7da2dd549b88ad7f", + "sha256": "c136829a246ef9088e84926c73b8721327473d893756f7bd7da2dd549b88ad7f" + }, + { + "path": "src/cuda_backend.rs", + "upstreamSha256": "f765d6c163aa9c1072a46b9a92ebdc83a5bcf580f76f0e3bbdd68450fa1f5be6", + "sha256": "f765d6c163aa9c1072a46b9a92ebdc83a5bcf580f76f0e3bbdd68450fa1f5be6" + }, + { + "path": "src/cudnn.rs", + "upstreamSha256": "d4381e8361653afc0dc9f9ec55c1881e36ec53504f9dbf3d76e0c9834ef4ec00", + "sha256": "d4381e8361653afc0dc9f9ec55c1881e36ec53504f9dbf3d76e0c9834ef4ec00" + }, + { + "path": "src/device.rs", + "upstreamSha256": "5fa4b7ff27f477d8b74687bed8904feee6c8bc12757cbf653c35c7a5e093dab3", + "sha256": "5fa4b7ff27f477d8b74687bed8904feee6c8bc12757cbf653c35c7a5e093dab3" + }, + { + "path": "src/display.rs", + "upstreamSha256": "ed33ff8bce426676a73df05713f14c2139f6312fa953a34ee8793f4ac56d16e0", + "sha256": "ed33ff8bce426676a73df05713f14c2139f6312fa953a34ee8793f4ac56d16e0" + }, + { + "path": "src/dtype.rs", + "upstreamSha256": "349c6efecf22cd3f1160cd0d38758400d6835c881481a2e88af716e9550c547c", + "sha256": "349c6efecf22cd3f1160cd0d38758400d6835c881481a2e88af716e9550c547c" + }, + { + "path": "src/dummy_cuda_backend.rs", + "upstreamSha256": "e9df849b9597e770954e251e318f7f784fe103b8ccfbf4fcfd0a07b63e78e4ed", + "sha256": "e9df849b9597e770954e251e318f7f784fe103b8ccfbf4fcfd0a07b63e78e4ed" + }, + { + "path": "src/dummy_metal_backend.rs", + "upstreamSha256": "c7b8f2574ec053142b1331ee993f54a6ab12f324fa4adf55f72e89d7569763c6", + "sha256": "c7b8f2574ec053142b1331ee993f54a6ab12f324fa4adf55f72e89d7569763c6" + }, + { + "path": "src/error.rs", + "upstreamSha256": "4423c8adff65933639b571ddae12acacd94d74eea0e93747dec917ec0b32464a", + "sha256": "4423c8adff65933639b571ddae12acacd94d74eea0e93747dec917ec0b32464a" + }, + { + "path": "src/indexer.rs", + "upstreamSha256": "1636ab33e7197eb11eff58f94b6bf6c19919680659a1c0265c9245ed95217779", + "sha256": "1636ab33e7197eb11eff58f94b6bf6c19919680659a1c0265c9245ed95217779" + }, + { + "path": "src/layout.rs", + "upstreamSha256": "b5de2481ae06d8e40b02a53fae66a3bc45e7f61de531cadc77d3b04cc796de4a", + "sha256": "b5de2481ae06d8e40b02a53fae66a3bc45e7f61de531cadc77d3b04cc796de4a" + }, + { + "path": "src/lib.rs", + "upstreamSha256": "b657d353d87d9b20cf65877260084fa4db4600fa9a2db910998ba440805e1923", + "sha256": "b657d353d87d9b20cf65877260084fa4db4600fa9a2db910998ba440805e1923" + }, + { + "path": "src/metal_backend.rs", + "upstreamSha256": "b2a6adfccaf0a7f88cf9b64c4cd67414be755a1fbf15e90d20facfdf15c297bc", + "sha256": "b2a6adfccaf0a7f88cf9b64c4cd67414be755a1fbf15e90d20facfdf15c297bc" + }, + { + "path": "src/mkl.rs", + "upstreamSha256": "c80564bd6139736e0e28336ba3130f386da963d62238eb45f2f1a15c506c2254", + "sha256": "c80564bd6139736e0e28336ba3130f386da963d62238eb45f2f1a15c506c2254" + }, + { + "path": "src/npy.rs", + "upstreamSha256": "b1c0cba48be9752eefe4ee940b37f477ea15650bc2e3257f2c98dd9a7c9fa2fb", + "sha256": "ae2eb805486716804a17003a2d0075c9db164968aaa2014cd4e0b6fbf5c45f19" + }, + { + "path": "src/op.rs", + "upstreamSha256": "170fb16bf46153a5c7fda99405a32b5b84a901a9fd6a8dcb5609c2230b842c03", + "sha256": "170fb16bf46153a5c7fda99405a32b5b84a901a9fd6a8dcb5609c2230b842c03" + }, + { + "path": "src/pickle.rs", + "upstreamSha256": "0ae3f2cb79983853437156fefa7a23ec9e42610d1e68fcb5e6b55b783b9ba84c", + "sha256": "70503eeb3f83ef0fce94989ebe39c062fc6087382bbe890acd18c820a4c7b73a" + }, + { + "path": "src/quantized/avx.rs", + "upstreamSha256": "24caf149e175e12ac4b55f94074f051518c94d85d3b6e749581e1241b15b5bf5", + "sha256": "24caf149e175e12ac4b55f94074f051518c94d85d3b6e749581e1241b15b5bf5" + }, + { + "path": "src/quantized/cuda.rs", + "upstreamSha256": "7083e8ee864ea7841c43732ed0a9019aab7824d859803d84e9c6e7a9ef93b197", + "sha256": "7083e8ee864ea7841c43732ed0a9019aab7824d859803d84e9c6e7a9ef93b197" + }, + { + "path": "src/quantized/dummy_cuda.rs", + "upstreamSha256": "b3d3fc2d28cb380fcb76703ad8ff47432456491b8ad97e65fbd19657ac90f3ad", + "sha256": "b3d3fc2d28cb380fcb76703ad8ff47432456491b8ad97e65fbd19657ac90f3ad" + }, + { + "path": "src/quantized/dummy_metal.rs", + "upstreamSha256": "fcdbac00595c9035874d28c4539aee6143629ee4bd8ca128ebe3614e87eb4bca", + "sha256": "fcdbac00595c9035874d28c4539aee6143629ee4bd8ca128ebe3614e87eb4bca" + }, + { + "path": "src/quantized/ggml_file.rs", + "upstreamSha256": "63153bc7242b8af7a24fa6e89fcbd634122d789aa35f8b631791a8e1308bc9d7", + "sha256": "63153bc7242b8af7a24fa6e89fcbd634122d789aa35f8b631791a8e1308bc9d7" + }, + { + "path": "src/quantized/gguf_file.rs", + "upstreamSha256": "4328832e0470f18d7cd0783ac9c94e8fbecce828111f706082f8837015625b37", + "sha256": "4328832e0470f18d7cd0783ac9c94e8fbecce828111f706082f8837015625b37" + }, + { + "path": "src/quantized/k_quants.rs", + "upstreamSha256": "1c5abc34ffdfb3aba16905b3c79b7f71242e2b2a927b67be39923b636ab81f30", + "sha256": "1c5abc34ffdfb3aba16905b3c79b7f71242e2b2a927b67be39923b636ab81f30" + }, + { + "path": "src/quantized/metal.rs", + "upstreamSha256": "f75fcb39ee6b13e0d6f99661b5e29e9df7b823ac601dfd70560bc1ec47c60272", + "sha256": "f75fcb39ee6b13e0d6f99661b5e29e9df7b823ac601dfd70560bc1ec47c60272" + }, + { + "path": "src/quantized/mod.rs", + "upstreamSha256": "9f4542e55e4e2297fdbcd3f46072f450a70b3851cba413425bfdb82ed96e9707", + "sha256": "9f4542e55e4e2297fdbcd3f46072f450a70b3851cba413425bfdb82ed96e9707" + }, + { + "path": "src/quantized/neon.rs", + "upstreamSha256": "e76ef3a2ae350cffe35764845481e6fbb1553da3cd34b500e7e6ecc127a41c70", + "sha256": "e76ef3a2ae350cffe35764845481e6fbb1553da3cd34b500e7e6ecc127a41c70" + }, + { + "path": "src/quantized/simd128.rs", + "upstreamSha256": "6afe4c9677961ddc5cbeaaeb3aa48a8af200134cea8d39e43c239e0df5d1d3d5", + "sha256": "6afe4c9677961ddc5cbeaaeb3aa48a8af200134cea8d39e43c239e0df5d1d3d5" + }, + { + "path": "src/quantized/utils.rs", + "upstreamSha256": "622e915fbcc13b03d827284fc802076b2955c53744ff6d7a9a881a4509d2f0c5", + "sha256": "622e915fbcc13b03d827284fc802076b2955c53744ff6d7a9a881a4509d2f0c5" + }, + { + "path": "src/safetensors.rs", + "upstreamSha256": "71a4bbfbf5b0ba5445df8c030b825dce30b3cf96de1f1e5c1e68c87aad2e046b", + "sha256": "71a4bbfbf5b0ba5445df8c030b825dce30b3cf96de1f1e5c1e68c87aad2e046b" + }, + { + "path": "src/scalar.rs", + "upstreamSha256": "26158f228b848a3b8fc61c86b37903175835c39a00986a6304b79dca31e9a3e9", + "sha256": "26158f228b848a3b8fc61c86b37903175835c39a00986a6304b79dca31e9a3e9" + }, + { + "path": "src/shape.rs", + "upstreamSha256": "76679ea3209dd844dc58f79f6686b1339bb59fa2558e9179d4f2e7f000973e8b", + "sha256": "76679ea3209dd844dc58f79f6686b1339bb59fa2558e9179d4f2e7f000973e8b" + }, + { + "path": "src/storage.rs", + "upstreamSha256": "b9916e7360569ab2807317eecd302e26e5f1e0bbca1f0e25f8cb2958c0fe21fc", + "sha256": "b9916e7360569ab2807317eecd302e26e5f1e0bbca1f0e25f8cb2958c0fe21fc" + }, + { + "path": "src/strided_index.rs", + "upstreamSha256": "86e85bdfc9d38c1e7d69935015079b4a1b6b546e5f444297e9f127a98241cfa4", + "sha256": "86e85bdfc9d38c1e7d69935015079b4a1b6b546e5f444297e9f127a98241cfa4" + }, + { + "path": "src/tensor.rs", + "upstreamSha256": "282929063ff6299e9a0feffc456c48a9a0869cd941a9de3e5d84a3550d1879ec", + "sha256": "282929063ff6299e9a0feffc456c48a9a0869cd941a9de3e5d84a3550d1879ec" + }, + { + "path": "src/test_utils.rs", + "upstreamSha256": "1f4483bb02ab7fdd90828b947d4d5029bbbe400387438dceea9ea6171841929d", + "sha256": "1f4483bb02ab7fdd90828b947d4d5029bbbe400387438dceea9ea6171841929d" + }, + { + "path": "src/utils.rs", + "upstreamSha256": "ba1c0946bcc179d4b7a2f7c74cefa0a212187272d9c6e245af20a51e73cdfa8a", + "sha256": "ba1c0946bcc179d4b7a2f7c74cefa0a212187272d9c6e245af20a51e73cdfa8a" + }, + { + "path": "src/variable.rs", + "upstreamSha256": "4872d2dff978932f73b03bd6721cf85d2b4944f3745f049699baadfb7ad9e9af", + "sha256": "4872d2dff978932f73b03bd6721cf85d2b4944f3745f049699baadfb7ad9e9af" + }, + { + "path": "tests/conv_tests.rs", + "upstreamSha256": "d286576393be58478de44d1ea78419d8fb0e6c7c6c9042a38043d706065e8fb2", + "sha256": "d286576393be58478de44d1ea78419d8fb0e6c7c6c9042a38043d706065e8fb2" + }, + { + "path": "tests/custom_op_tests.rs", + "upstreamSha256": "1b9db66297b8c19536206027e382f808d499e639a453a58e05cf1b9a54f32879", + "sha256": "1b9db66297b8c19536206027e382f808d499e639a453a58e05cf1b9a54f32879" + }, + { + "path": "tests/display_tests.rs", + "upstreamSha256": "cf59d2ee9954a639db5c43ade847d4629503b706282419fae8c6481790c41e9b", + "sha256": "cf59d2ee9954a639db5c43ade847d4629503b706282419fae8c6481790c41e9b" + }, + { + "path": "tests/fortran_tensor_3d.pth", + "upstreamSha256": "f5178bcd53eb6cb54c9a5e825492c717bf4e7bbadb9f0f3833b6c5c6764f7a38", + "sha256": "f5178bcd53eb6cb54c9a5e825492c717bf4e7bbadb9f0f3833b6c5c6764f7a38" + }, + { + "path": "tests/grad_tests.rs", + "upstreamSha256": "91a17f47c667766551f90175d01b79595dcc3c8fcde43e809ce48a9fbf65ebde", + "sha256": "91a17f47c667766551f90175d01b79595dcc3c8fcde43e809ce48a9fbf65ebde" + }, + { + "path": "tests/indexing_tests.rs", + "upstreamSha256": "3d9e855b36a3e0b8683777b3d13c6d34859773f47e5ff309d612cbfcafdd6be4", + "sha256": "3d9e855b36a3e0b8683777b3d13c6d34859773f47e5ff309d612cbfcafdd6be4" + }, + { + "path": "tests/layout_tests.rs", + "upstreamSha256": "7fb5e266dae460593c076067e733a010666328b9353d42e62cff395df41f9073", + "sha256": "7fb5e266dae460593c076067e733a010666328b9353d42e62cff395df41f9073" + }, + { + "path": "tests/npy.py", + "upstreamSha256": "e2547b01e13fc3807991286645dba5eea7c5b3d76e821be2cfa7822be7a2f818", + "sha256": "e2547b01e13fc3807991286645dba5eea7c5b3d76e821be2cfa7822be7a2f818" + }, + { + "path": "tests/pool_tests.rs", + "upstreamSha256": "61947694711e0fa576606cddd50df7099705de7741ce8395dba316189ae21129", + "sha256": "61947694711e0fa576606cddd50df7099705de7741ce8395dba316189ae21129" + }, + { + "path": "tests/pth.py", + "upstreamSha256": "7ef7b4686691b33173f376f22daa121e92c91bec172994afe3deebd53e77f0b7", + "sha256": "7ef7b4686691b33173f376f22daa121e92c91bec172994afe3deebd53e77f0b7" + }, + { + "path": "tests/pth_tests.rs", + "upstreamSha256": "08496d5434bfb2382497f1e5d7f83ebe149bd4030b06882912d457d579d011a8", + "sha256": "08496d5434bfb2382497f1e5d7f83ebe149bd4030b06882912d457d579d011a8" + }, + { + "path": "tests/quantized_tests.rs", + "upstreamSha256": "c4208384d84b3bab5398f05bf99ab02fdcbcf54ccc43a801a07deada1e1269b3", + "sha256": "c4208384d84b3bab5398f05bf99ab02fdcbcf54ccc43a801a07deada1e1269b3" + }, + { + "path": "tests/serialization_tests.rs", + "upstreamSha256": "623c851de37f9d6c5c8fc64da257cf1a3359274851d25b1637abc1d6fe29d48c", + "sha256": "623c851de37f9d6c5c8fc64da257cf1a3359274851d25b1637abc1d6fe29d48c" + }, + { + "path": "tests/tensor_tests.rs", + "upstreamSha256": "9f19a2046372d16316a8452db504241ba6145720362504bf3b48456c17307dab", + "sha256": "9f19a2046372d16316a8452db504241ba6145720362504bf3b48456c17307dab" + }, + { + "path": "tests/test.npy", + "upstreamSha256": "f314ddddf18999d4890f4fec8bb6539fc62b6f41861fa85e1c04c98a03097dda", + "sha256": "f314ddddf18999d4890f4fec8bb6539fc62b6f41861fa85e1c04c98a03097dda" + }, + { + "path": "tests/test.npz", + "upstreamSha256": "999497e0c309302915e4db2b16118ce097c68aac66c08058fac8db3d58c7809a", + "sha256": "999497e0c309302915e4db2b16118ce097c68aac66c08058fac8db3d58c7809a" + }, + { + "path": "tests/test.pt", + "upstreamSha256": "2bc164fa96f78ef8fcb8eab5fdae9b991efe946b9b63071fdf1769dba5d3fe25", + "sha256": "2bc164fa96f78ef8fcb8eab5fdae9b991efe946b9b63071fdf1769dba5d3fe25" + }, + { + "path": "tests/test_with_key.pt", + "upstreamSha256": "2775d95f17d2630e2534e3ae09462624955031407a67f0f3b52115c49601b747", + "sha256": "2775d95f17d2630e2534e3ae09462624955031407a67f0f3b52115c49601b747" + } + ] + }, + { + "package": "candle-core-0.9.2", + "archiveSha256": "c15b675b80d994b2eadb20a4bbe434eabeb454eac3ee5e2b4cf6f147ee9be091", + "changedFiles": [ + "Cargo.toml", + "src/npy.rs", + "src/pickle.rs" + ], + "files": [ + { + "path": "Cargo.toml", + "upstreamSha256": "fbb9960d0496ea8cb11aead7548a70af655af410d5e3f7f094882dd11738f4f9", + "sha256": "d22d8e98c28b136040aac1d306599911635da862a2b3ac2f895aa7b19d92cff3" + }, + { + "path": "Cargo.toml.orig", + "upstreamSha256": "b7d9510ca0933845e9f1acad1b5504aff030e34a6877cf8900bf6bd70ee0484d", + "sha256": "b7d9510ca0933845e9f1acad1b5504aff030e34a6877cf8900bf6bd70ee0484d" + }, + { + "path": "LICENSE", + "upstreamSha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4", + "sha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4" + }, + { + "path": "README.md", + "upstreamSha256": "9b00007b1d568125e67d6f2ff5c40acd8227dec4c62681cbc87775768a125200", + "sha256": "9b00007b1d568125e67d6f2ff5c40acd8227dec4c62681cbc87775768a125200" + }, + { + "path": "benches/bench_main.rs", + "upstreamSha256": "0228bb14b13ee0cd2d7e907d8e66017fa291d81d39316c183f4bf5aed0657518", + "sha256": "0228bb14b13ee0cd2d7e907d8e66017fa291d81d39316c183f4bf5aed0657518" + }, + { + "path": "benches/benchmarks/affine.rs", + "upstreamSha256": "260d6aeb9faf50808ac278dd93664dd24769bcd5c53333490d742cdff298844c", + "sha256": "260d6aeb9faf50808ac278dd93664dd24769bcd5c53333490d742cdff298844c" + }, + { + "path": "benches/benchmarks/binary.rs", + "upstreamSha256": "86335f2ca24005c215d590797cfde35c6a3bba21978b0cf184ac906c6f255e40", + "sha256": "86335f2ca24005c215d590797cfde35c6a3bba21978b0cf184ac906c6f255e40" + }, + { + "path": "benches/benchmarks/broadcast.rs", + "upstreamSha256": "9da8fde24dcbed559dfd44e7ec6607762ed2a7f80857a057f619838a4c723618", + "sha256": "9da8fde24dcbed559dfd44e7ec6607762ed2a7f80857a057f619838a4c723618" + }, + { + "path": "benches/benchmarks/conv_transpose2d.rs", + "upstreamSha256": "f57723066b6fc173d2957181e2e54a06505213303f82577b10d482983048981a", + "sha256": "f57723066b6fc173d2957181e2e54a06505213303f82577b10d482983048981a" + }, + { + "path": "benches/benchmarks/copy.rs", + "upstreamSha256": "406c04fa73a664a785dd7d9c6a4ad4d09b4c0c0b52072b8e429566485fd61938", + "sha256": "406c04fa73a664a785dd7d9c6a4ad4d09b4c0c0b52072b8e429566485fd61938" + }, + { + "path": "benches/benchmarks/matmul.rs", + "upstreamSha256": "8ceb3d0e312a82770f85414459009d0385043eb1fa384f8dd34476b84cf42bbb", + "sha256": "8ceb3d0e312a82770f85414459009d0385043eb1fa384f8dd34476b84cf42bbb" + }, + { + "path": "benches/benchmarks/mod.rs", + "upstreamSha256": "28ed8ad6b0e938364c62d64456dd98bd84faca4b038b7f3661ebe013555833ac", + "sha256": "28ed8ad6b0e938364c62d64456dd98bd84faca4b038b7f3661ebe013555833ac" + }, + { + "path": "benches/benchmarks/qmatmul.rs", + "upstreamSha256": "cd71f24bc0dbdd716f36fec4658a444cde64fdeacca1cdf63bbf1501ede12e40", + "sha256": "cd71f24bc0dbdd716f36fec4658a444cde64fdeacca1cdf63bbf1501ede12e40" + }, + { + "path": "benches/benchmarks/random.rs", + "upstreamSha256": "0f68d50a3da9f1096ab00c6a1141b58454b335604d6c2ff3d39af830a47b9138", + "sha256": "0f68d50a3da9f1096ab00c6a1141b58454b335604d6c2ff3d39af830a47b9138" + }, + { + "path": "benches/benchmarks/reduce.rs", + "upstreamSha256": "8a0ce9e9f10ca9d210db3c424325dd082ccc08fd288080f0b579ab1c7faa8fc6", + "sha256": "8a0ce9e9f10ca9d210db3c424325dd082ccc08fd288080f0b579ab1c7faa8fc6" + }, + { + "path": "benches/benchmarks/unary.rs", + "upstreamSha256": "40153b0fe906af2235e0a356719a208d82b5910853df93957f8e18bcb61bcef1", + "sha256": "40153b0fe906af2235e0a356719a208d82b5910853df93957f8e18bcb61bcef1" + }, + { + "path": "benches/benchmarks/where_cond.rs", + "upstreamSha256": "41be1e61510cfdfbdc74cbb523ec8756fad12eb53a3b6add8f032999e98370aa", + "sha256": "41be1e61510cfdfbdc74cbb523ec8756fad12eb53a3b6add8f032999e98370aa" + }, + { + "path": "examples/basics.rs", + "upstreamSha256": "6c85b5c338290f8d2eeeb548209fc4f4d4052fa4bfd2ea07846731cd9c36d88d", + "sha256": "6c85b5c338290f8d2eeeb548209fc4f4d4052fa4bfd2ea07846731cd9c36d88d" + }, + { + "path": "examples/cuda_basics.rs", + "upstreamSha256": "ec4aa80f89acf913ca74828fdf19ceb24808406794e98caa3f524dd23ce66cd4", + "sha256": "ec4aa80f89acf913ca74828fdf19ceb24808406794e98caa3f524dd23ce66cd4" + }, + { + "path": "examples/cuda_sum_benchmark.rs", + "upstreamSha256": "f53810234d3b90cbef0cc23c7ac1ad8141a9a93d1e86f7c358f5d32fcc5c76cc", + "sha256": "f53810234d3b90cbef0cc23c7ac1ad8141a9a93d1e86f7c358f5d32fcc5c76cc" + }, + { + "path": "examples/metal_basics.rs", + "upstreamSha256": "4f2cb274449cf1b6326b1ed2211f1076d55e621fcc81cdd501eca72e7203351e", + "sha256": "4f2cb274449cf1b6326b1ed2211f1076d55e621fcc81cdd501eca72e7203351e" + }, + { + "path": "src/accelerate.rs", + "upstreamSha256": "9be99929bb6553d9baf6b6a5a56fa66bfdbee122f76c4792a00e09ac8a0d638a", + "sha256": "9be99929bb6553d9baf6b6a5a56fa66bfdbee122f76c4792a00e09ac8a0d638a" + }, + { + "path": "src/backend.rs", + "upstreamSha256": "822a96a62ef7d300141b7d439807482bd40bded3f592e9e72c1e108f43ff63cc", + "sha256": "822a96a62ef7d300141b7d439807482bd40bded3f592e9e72c1e108f43ff63cc" + }, + { + "path": "src/backprop.rs", + "upstreamSha256": "4f370f420aaea68bf23976f807c9a47a97c36d46f17f18ef82132d43a9847a52", + "sha256": "4f370f420aaea68bf23976f807c9a47a97c36d46f17f18ef82132d43a9847a52" + }, + { + "path": "src/conv.rs", + "upstreamSha256": "31b8870d37dad9d2962985657121e85bc350830555d20054700921ea689a0d9f", + "sha256": "31b8870d37dad9d2962985657121e85bc350830555d20054700921ea689a0d9f" + }, + { + "path": "src/convert.rs", + "upstreamSha256": "664562f7fc98008b1cbf4815aacf79311141a40d9a003f86ecdb8cebcf6abbae", + "sha256": "664562f7fc98008b1cbf4815aacf79311141a40d9a003f86ecdb8cebcf6abbae" + }, + { + "path": "src/cpu/avx.rs", + "upstreamSha256": "d06f480b9ec3553568c0ef4b32e30d9f0dc7371f6db94473b42724f2b2519d45", + "sha256": "d06f480b9ec3553568c0ef4b32e30d9f0dc7371f6db94473b42724f2b2519d45" + }, + { + "path": "src/cpu/erf.rs", + "upstreamSha256": "e9d7e063b6e1b3af2b785b4b26ea038e512608a380edbb0d65d0253f0f70dbdc", + "sha256": "e9d7e063b6e1b3af2b785b4b26ea038e512608a380edbb0d65d0253f0f70dbdc" + }, + { + "path": "src/cpu/kernels.rs", + "upstreamSha256": "ad187fd28c7b491ae982e5e9c714331b89c91056c370f6a7417cc79dfda7aab1", + "sha256": "ad187fd28c7b491ae982e5e9c714331b89c91056c370f6a7417cc79dfda7aab1" + }, + { + "path": "src/cpu/mod.rs", + "upstreamSha256": "667e12b8807e5b73cb3b84f4078029e1060fc2b4147122584c24dc0c083c3738", + "sha256": "667e12b8807e5b73cb3b84f4078029e1060fc2b4147122584c24dc0c083c3738" + }, + { + "path": "src/cpu/neon.rs", + "upstreamSha256": "c180dffb8a342c2f026d8fdcc235c7d9e8f0aac8a5f6f03de4c2f4de8dda609c", + "sha256": "c180dffb8a342c2f026d8fdcc235c7d9e8f0aac8a5f6f03de4c2f4de8dda609c" + }, + { + "path": "src/cpu/simd128.rs", + "upstreamSha256": "8ed8cff924086325c4d4af5a12746a7ce4a5a05f18cdeaa28bc0ff9c64060906", + "sha256": "8ed8cff924086325c4d4af5a12746a7ce4a5a05f18cdeaa28bc0ff9c64060906" + }, + { + "path": "src/cpu_backend/conv2d.rs", + "upstreamSha256": "367f0c7e9380215879e6c95cffabbd9df76206d96716f8c8ccfdded2ac646bf1", + "sha256": "367f0c7e9380215879e6c95cffabbd9df76206d96716f8c8ccfdded2ac646bf1" + }, + { + "path": "src/cpu_backend/mod.rs", + "upstreamSha256": "93512c3d7c4ea7b1a0b74300eea658f0afb7d51850cec928a995ebeeacde66b5", + "sha256": "93512c3d7c4ea7b1a0b74300eea658f0afb7d51850cec928a995ebeeacde66b5" + }, + { + "path": "src/cpu_backend/utils.rs", + "upstreamSha256": "85b05a7f3d6934a115f72f1caf977d7bbdd680715939fb3bdfbbf24af5ac15b1", + "sha256": "85b05a7f3d6934a115f72f1caf977d7bbdd680715939fb3bdfbbf24af5ac15b1" + }, + { + "path": "src/cuda_backend/cudnn.rs", + "upstreamSha256": "3e3237559a6148567a369effe6e9616bbc12a6b67c13c09b2d52e6f6382f04b8", + "sha256": "3e3237559a6148567a369effe6e9616bbc12a6b67c13c09b2d52e6f6382f04b8" + }, + { + "path": "src/cuda_backend/device.rs", + "upstreamSha256": "40f4bb9512f9001a983802129b6cfee16381a24b7876f13ba02cd49b6dfe1925", + "sha256": "40f4bb9512f9001a983802129b6cfee16381a24b7876f13ba02cd49b6dfe1925" + }, + { + "path": "src/cuda_backend/error.rs", + "upstreamSha256": "b60bce29f50ce640fb1932009bd447f0ff7fc0e293900befb05a9d8df0e43413", + "sha256": "b60bce29f50ce640fb1932009bd447f0ff7fc0e293900befb05a9d8df0e43413" + }, + { + "path": "src/cuda_backend/mod.rs", + "upstreamSha256": "bc2bd5fe3702bfccb49d64af07dc4197013d6617c720643cd2a14b718f4134e2", + "sha256": "bc2bd5fe3702bfccb49d64af07dc4197013d6617c720643cd2a14b718f4134e2" + }, + { + "path": "src/cuda_backend/utils.rs", + "upstreamSha256": "24ceb7c9e50389a911e9aaf4d4f01e92ca3f2a06743adfa72981ba870a0574d9", + "sha256": "24ceb7c9e50389a911e9aaf4d4f01e92ca3f2a06743adfa72981ba870a0574d9" + }, + { + "path": "src/custom_op.rs", + "upstreamSha256": "c0a4953dad882f3a392070fa6bbabeaa582d2576d50bf349e12a97d680161ff1", + "sha256": "c0a4953dad882f3a392070fa6bbabeaa582d2576d50bf349e12a97d680161ff1" + }, + { + "path": "src/device.rs", + "upstreamSha256": "0929b7e72e83dea86a40f43f55a61eb2ca0dcb2d1c3c2d6e698bd18cf801754b", + "sha256": "0929b7e72e83dea86a40f43f55a61eb2ca0dcb2d1c3c2d6e698bd18cf801754b" + }, + { + "path": "src/display.rs", + "upstreamSha256": "17d38758a849310137f69069a0806224b9fdb682c0b44196b6d5885327515211", + "sha256": "17d38758a849310137f69069a0806224b9fdb682c0b44196b6d5885327515211" + }, + { + "path": "src/dtype.rs", + "upstreamSha256": "7aa09f4fb008d8ad5c2bbc83cfcd4bbcaab8f5e351d5f4ba6b93e7a1f1f2fe76", + "sha256": "7aa09f4fb008d8ad5c2bbc83cfcd4bbcaab8f5e351d5f4ba6b93e7a1f1f2fe76" + }, + { + "path": "src/dummy_cuda_backend.rs", + "upstreamSha256": "842836efb0c1ca8097d1a5c11140207f185b7f877603e98091afdf7d494fee47", + "sha256": "842836efb0c1ca8097d1a5c11140207f185b7f877603e98091afdf7d494fee47" + }, + { + "path": "src/dummy_dtype.rs", + "upstreamSha256": "50bcae6eeba464291a8d85020e967056b42cc7199658999de7159d0c4e8190e0", + "sha256": "50bcae6eeba464291a8d85020e967056b42cc7199658999de7159d0c4e8190e0" + }, + { + "path": "src/dummy_metal_backend.rs", + "upstreamSha256": "e59c40e1aea0210adac21316af7b7e9b838d8357da9492045f530a139d35acd7", + "sha256": "e59c40e1aea0210adac21316af7b7e9b838d8357da9492045f530a139d35acd7" + }, + { + "path": "src/error.rs", + "upstreamSha256": "e6605678f4093e0867e73a5579aa8d046d13d65bd609d5851edf42d44051927d", + "sha256": "e6605678f4093e0867e73a5579aa8d046d13d65bd609d5851edf42d44051927d" + }, + { + "path": "src/indexer.rs", + "upstreamSha256": "2df921d3e8284540a34c197b692b384ed5d6a25047d3685535f01c227718a3e0", + "sha256": "2df921d3e8284540a34c197b692b384ed5d6a25047d3685535f01c227718a3e0" + }, + { + "path": "src/layout.rs", + "upstreamSha256": "1bf7fca1aa8759cfe4663a63ed9cce5dcc7ae8da84fc5a2e043697a279980db7", + "sha256": "1bf7fca1aa8759cfe4663a63ed9cce5dcc7ae8da84fc5a2e043697a279980db7" + }, + { + "path": "src/lib.rs", + "upstreamSha256": "a7d035c1a1153b3acdee0c9378f5d2fbd14eb2e16e9e3f2fb27db687030640ab", + "sha256": "a7d035c1a1153b3acdee0c9378f5d2fbd14eb2e16e9e3f2fb27db687030640ab" + }, + { + "path": "src/metal_backend/device.rs", + "upstreamSha256": "3ca33c5b6eec88fa8893f14726bd60ed259c60f42b7646464b375f8c9ff2434f", + "sha256": "3ca33c5b6eec88fa8893f14726bd60ed259c60f42b7646464b375f8c9ff2434f" + }, + { + "path": "src/metal_backend/mod.rs", + "upstreamSha256": "16bc99ee096c581eaa83ce6cf9d007674d4eb6bb5b6e30560ccb62a3daac07a1", + "sha256": "16bc99ee096c581eaa83ce6cf9d007674d4eb6bb5b6e30560ccb62a3daac07a1" + }, + { + "path": "src/mkl.rs", + "upstreamSha256": "c80564bd6139736e0e28336ba3130f386da963d62238eb45f2f1a15c506c2254", + "sha256": "c80564bd6139736e0e28336ba3130f386da963d62238eb45f2f1a15c506c2254" + }, + { + "path": "src/npy.rs", + "upstreamSha256": "e53194fd98b92c5c2bc4948cc39bcd59e63b3b477d550c2a8e4bc9c66db67bca", + "sha256": "f638519dc260eb834cdbce9db7f6b3e570bf7589bf40bf6dca055f35b4ceb86e" + }, + { + "path": "src/op.rs", + "upstreamSha256": "8787d8e941217e500b7f653aa4252c41a0e15459c31f7373d0dc896fce4c4521", + "sha256": "8787d8e941217e500b7f653aa4252c41a0e15459c31f7373d0dc896fce4c4521" + }, + { + "path": "src/pickle.rs", + "upstreamSha256": "7fcb931fb616c35c7817e5461cc6173d7c6635339223c360a6c22fb077c07d63", + "sha256": "3b98dee96ce63c91da0644b2d63040e924b92bb17a49a3f15ade096ce2d49a91" + }, + { + "path": "src/quantized/avx.rs", + "upstreamSha256": "69cb5408571eb64812cb6386f0e24a7ff39fdc16482d44ee9f31fb40e536665c", + "sha256": "69cb5408571eb64812cb6386f0e24a7ff39fdc16482d44ee9f31fb40e536665c" + }, + { + "path": "src/quantized/cuda.rs", + "upstreamSha256": "a085e61faf3d383fe332775673d691b585cabcb12701351a75839d9ebb8196c5", + "sha256": "a085e61faf3d383fe332775673d691b585cabcb12701351a75839d9ebb8196c5" + }, + { + "path": "src/quantized/dummy_cuda.rs", + "upstreamSha256": "534ff012566fea238b247715c278352fa1860202ea9c8db32636469769f6dd97", + "sha256": "534ff012566fea238b247715c278352fa1860202ea9c8db32636469769f6dd97" + }, + { + "path": "src/quantized/dummy_metal.rs", + "upstreamSha256": "a9c727095f509046d1a50d131eee1529f9aa97ab306ceb1a44a1e827ea10109d", + "sha256": "a9c727095f509046d1a50d131eee1529f9aa97ab306ceb1a44a1e827ea10109d" + }, + { + "path": "src/quantized/ggml_file.rs", + "upstreamSha256": "dfc48f822e4393bc836efcf782f4a6bb36168ac9d641324a054f13ee862aadac", + "sha256": "dfc48f822e4393bc836efcf782f4a6bb36168ac9d641324a054f13ee862aadac" + }, + { + "path": "src/quantized/gguf_file.rs", + "upstreamSha256": "a263c9076e3e233457cc3d7780e3240c6448c7b5b8c517e480353691be5ff186", + "sha256": "a263c9076e3e233457cc3d7780e3240c6448c7b5b8c517e480353691be5ff186" + }, + { + "path": "src/quantized/imatrix_file.rs", + "upstreamSha256": "4ad710e64a0c9d3e5c2be1f6ca0a760035babb66c5fb91774acd1d7a30288d66", + "sha256": "4ad710e64a0c9d3e5c2be1f6ca0a760035babb66c5fb91774acd1d7a30288d66" + }, + { + "path": "src/quantized/k_quants.rs", + "upstreamSha256": "736a4b74c26c8c3be2917ec4c4ca88cf87e892c6b3278f5a466726bdf84f21b1", + "sha256": "736a4b74c26c8c3be2917ec4c4ca88cf87e892c6b3278f5a466726bdf84f21b1" + }, + { + "path": "src/quantized/metal.rs", + "upstreamSha256": "cacf5336009a8f83e13fa32a3f1c5e28ab5a5d52fc91d9ec4ddc265a1bcc7eb4", + "sha256": "cacf5336009a8f83e13fa32a3f1c5e28ab5a5d52fc91d9ec4ddc265a1bcc7eb4" + }, + { + "path": "src/quantized/mod.rs", + "upstreamSha256": "82fe93c2b9c0625c9b7147a60e977acaff841a7e4e139f631e6c06fc5eb0e8a0", + "sha256": "82fe93c2b9c0625c9b7147a60e977acaff841a7e4e139f631e6c06fc5eb0e8a0" + }, + { + "path": "src/quantized/neon.rs", + "upstreamSha256": "2a46cd5cf8f296c3673f61380bac49fd98232472ae5db60dc7babc788fdd0be7", + "sha256": "2a46cd5cf8f296c3673f61380bac49fd98232472ae5db60dc7babc788fdd0be7" + }, + { + "path": "src/quantized/simd128.rs", + "upstreamSha256": "5c8000c6dea9803de8befde33b15970c1b1dcbf83b914c6261a3e9c2b658c1da", + "sha256": "5c8000c6dea9803de8befde33b15970c1b1dcbf83b914c6261a3e9c2b658c1da" + }, + { + "path": "src/quantized/utils.rs", + "upstreamSha256": "33a4ff45fafb4baecbd6b95f68fd373d447283be19e38e63f1f4da3434f3a47b", + "sha256": "33a4ff45fafb4baecbd6b95f68fd373d447283be19e38e63f1f4da3434f3a47b" + }, + { + "path": "src/safetensors.rs", + "upstreamSha256": "1259250e0899d842d34e0a2495c414fb63efdee0d78e74092a47648c5380bfb3", + "sha256": "1259250e0899d842d34e0a2495c414fb63efdee0d78e74092a47648c5380bfb3" + }, + { + "path": "src/scalar.rs", + "upstreamSha256": "d98ca484e6fd0580f2b8bb2c3c43283671496042aa9d58115829052532d6200c", + "sha256": "d98ca484e6fd0580f2b8bb2c3c43283671496042aa9d58115829052532d6200c" + }, + { + "path": "src/shape.rs", + "upstreamSha256": "0fb3b21b63f348efcc18222cecbc3b5ea510d14c50f728c9471a30e24901ba74", + "sha256": "0fb3b21b63f348efcc18222cecbc3b5ea510d14c50f728c9471a30e24901ba74" + }, + { + "path": "src/sort.rs", + "upstreamSha256": "4c1263ab7b6f80a1acd05813a4afd4ef2aa98d44378458829dfc403bccfdbb92", + "sha256": "4c1263ab7b6f80a1acd05813a4afd4ef2aa98d44378458829dfc403bccfdbb92" + }, + { + "path": "src/storage.rs", + "upstreamSha256": "6241271357ee152f0244e47053f40de811832fca89e8aece6420bbc1b5c485c4", + "sha256": "6241271357ee152f0244e47053f40de811832fca89e8aece6420bbc1b5c485c4" + }, + { + "path": "src/streaming.rs", + "upstreamSha256": "abd0de65d894b617e6e3fb0c0787f220582bcb1f8eb9341c33bb74aaa23bf165", + "sha256": "abd0de65d894b617e6e3fb0c0787f220582bcb1f8eb9341c33bb74aaa23bf165" + }, + { + "path": "src/strided_index.rs", + "upstreamSha256": "b4884a6ad8d6fcb043b69b47b8203a035f40616057677802df32c4ee3799445b", + "sha256": "b4884a6ad8d6fcb043b69b47b8203a035f40616057677802df32c4ee3799445b" + }, + { + "path": "src/tensor.rs", + "upstreamSha256": "25b95a8deecffa87d55325d9ae258b9552778a7eb24a653b7c1438340aa21487", + "sha256": "25b95a8deecffa87d55325d9ae258b9552778a7eb24a653b7c1438340aa21487" + }, + { + "path": "src/tensor_cat.rs", + "upstreamSha256": "8c97ffbac8ca2fef78d5d1440e05df7915e6b1c2ae34ef891a36e6a6e989f497", + "sha256": "8c97ffbac8ca2fef78d5d1440e05df7915e6b1c2ae34ef891a36e6a6e989f497" + }, + { + "path": "src/test_utils.rs", + "upstreamSha256": "b3eec41e2864344b9e0a0eec4f0be70f8273617b99684d41315809530327debb", + "sha256": "b3eec41e2864344b9e0a0eec4f0be70f8273617b99684d41315809530327debb" + }, + { + "path": "src/utils.rs", + "upstreamSha256": "bdfc884fb12a14eb2b5a0dfafebd7951596ac9361f97c1cd93c4284507eec9e6", + "sha256": "bdfc884fb12a14eb2b5a0dfafebd7951596ac9361f97c1cd93c4284507eec9e6" + }, + { + "path": "src/variable.rs", + "upstreamSha256": "b9b10b5d2d4a81d1ab920f9337e556a634bae7e484a0b04ea855f9a903fc87ec", + "sha256": "b9b10b5d2d4a81d1ab920f9337e556a634bae7e484a0b04ea855f9a903fc87ec" + }, + { + "path": "tests/bilinear_tests.rs", + "upstreamSha256": "6844aa75cbe436d753391e3839619adca77a1f83d6ae78957261181fabbefe9a", + "sha256": "6844aa75cbe436d753391e3839619adca77a1f83d6ae78957261181fabbefe9a" + }, + { + "path": "tests/conv_tests.rs", + "upstreamSha256": "f303e81c201c3b450d95b3b437d128ee4cea4344c3aa31f3d2f09b81d3e9ad8d", + "sha256": "f303e81c201c3b450d95b3b437d128ee4cea4344c3aa31f3d2f09b81d3e9ad8d" + }, + { + "path": "tests/custom_op_tests.rs", + "upstreamSha256": "fa13ae08edf71d1c15a0a7509652367ee53a5960b722d7ee503494455fa4173d", + "sha256": "fa13ae08edf71d1c15a0a7509652367ee53a5960b722d7ee503494455fa4173d" + }, + { + "path": "tests/display_tests.rs", + "upstreamSha256": "cf59d2ee9954a639db5c43ade847d4629503b706282419fae8c6481790c41e9b", + "sha256": "cf59d2ee9954a639db5c43ade847d4629503b706282419fae8c6481790c41e9b" + }, + { + "path": "tests/fortran_tensor_3d.pth", + "upstreamSha256": "f5178bcd53eb6cb54c9a5e825492c717bf4e7bbadb9f0f3833b6c5c6764f7a38", + "sha256": "f5178bcd53eb6cb54c9a5e825492c717bf4e7bbadb9f0f3833b6c5c6764f7a38" + }, + { + "path": "tests/grad_tests.rs", + "upstreamSha256": "f7fc99b411d75c914532236ee1d5ba9f783dc6a4165d40e9145d4110fe7a21c8", + "sha256": "f7fc99b411d75c914532236ee1d5ba9f783dc6a4165d40e9145d4110fe7a21c8" + }, + { + "path": "tests/indexing_tests.rs", + "upstreamSha256": "3d9e855b36a3e0b8683777b3d13c6d34859773f47e5ff309d612cbfcafdd6be4", + "sha256": "3d9e855b36a3e0b8683777b3d13c6d34859773f47e5ff309d612cbfcafdd6be4" + }, + { + "path": "tests/layout_tests.rs", + "upstreamSha256": "25e6347e2c647838331254ad21d437f6ab59bc26dd98110c696d365f05daa18e", + "sha256": "25e6347e2c647838331254ad21d437f6ab59bc26dd98110c696d365f05daa18e" + }, + { + "path": "tests/matmul_tests.rs", + "upstreamSha256": "7499f96b1d2343ab919076f57f6907f85391af5c2e55d057673fb24ea59a061e", + "sha256": "7499f96b1d2343ab919076f57f6907f85391af5c2e55d057673fb24ea59a061e" + }, + { + "path": "tests/npy.py", + "upstreamSha256": "e2547b01e13fc3807991286645dba5eea7c5b3d76e821be2cfa7822be7a2f818", + "sha256": "e2547b01e13fc3807991286645dba5eea7c5b3d76e821be2cfa7822be7a2f818" + }, + { + "path": "tests/pool_tests.rs", + "upstreamSha256": "b574c67ce8486578d55b3f27070f83bf1c1137173b9d3762d5cd2e7d25c9f8b3", + "sha256": "b574c67ce8486578d55b3f27070f83bf1c1137173b9d3762d5cd2e7d25c9f8b3" + }, + { + "path": "tests/pth.py", + "upstreamSha256": "7ef7b4686691b33173f376f22daa121e92c91bec172994afe3deebd53e77f0b7", + "sha256": "7ef7b4686691b33173f376f22daa121e92c91bec172994afe3deebd53e77f0b7" + }, + { + "path": "tests/pth_tests.rs", + "upstreamSha256": "32440ce8dcb64ddf469e7cf122f4d19aa0ae484650c20d127d147fb99052fa35", + "sha256": "32440ce8dcb64ddf469e7cf122f4d19aa0ae484650c20d127d147fb99052fa35" + }, + { + "path": "tests/quantized_tests.rs", + "upstreamSha256": "908ffb8070f75f123edcf7a19586e9c6172bd749b670a1fb53f3bfd523bceb34", + "sha256": "908ffb8070f75f123edcf7a19586e9c6172bd749b670a1fb53f3bfd523bceb34" + }, + { + "path": "tests/serialization_tests.rs", + "upstreamSha256": "98db2da64586ec61234767421a792e712f2c072fe2382b1ff589c771b03b7d57", + "sha256": "98db2da64586ec61234767421a792e712f2c072fe2382b1ff589c771b03b7d57" + }, + { + "path": "tests/tensor_tests.rs", + "upstreamSha256": "8ce0f9b1406a9c30a1d2600dc88887875fec89b3c87e4eea21c971535b4bc261", + "sha256": "8ce0f9b1406a9c30a1d2600dc88887875fec89b3c87e4eea21c971535b4bc261" + }, + { + "path": "tests/test.npy", + "upstreamSha256": "f314ddddf18999d4890f4fec8bb6539fc62b6f41861fa85e1c04c98a03097dda", + "sha256": "f314ddddf18999d4890f4fec8bb6539fc62b6f41861fa85e1c04c98a03097dda" + }, + { + "path": "tests/test.npz", + "upstreamSha256": "999497e0c309302915e4db2b16118ce097c68aac66c08058fac8db3d58c7809a", + "sha256": "999497e0c309302915e4db2b16118ce097c68aac66c08058fac8db3d58c7809a" + }, + { + "path": "tests/test.pt", + "upstreamSha256": "2bc164fa96f78ef8fcb8eab5fdae9b991efe946b9b63071fdf1769dba5d3fe25", + "sha256": "2bc164fa96f78ef8fcb8eab5fdae9b991efe946b9b63071fdf1769dba5d3fe25" + }, + { + "path": "tests/test_with_key.pt", + "upstreamSha256": "2775d95f17d2630e2534e3ae09462624955031407a67f0f3b52115c49601b747", + "sha256": "2775d95f17d2630e2534e3ae09462624955031407a67f0f3b52115c49601b747" + } + ] + }, + { + "package": "tch-0.24.0", + "archiveSha256": "0d3f84a069d8ba16dbf720b61e8bf131d90ffb8e958a664eae8e4993c5c2fa6f", + "changedFiles": [ + "Cargo.toml", + "src/tensor/npy.rs" + ], + "files": [ + { + "path": ".cargo/config.toml", + "upstreamSha256": "c5d69e9af7086ce85ae357f712437ede39c9c95a4348af8eec5931d2e16df9fa", + "sha256": "c5d69e9af7086ce85ae357f712437ede39c9c95a4348af8eec5931d2e16df9fa" + }, + { + "path": ".github/workflows/rust-ci.yml", + "upstreamSha256": "813732a2cbae7f5d4b1e471e3c3ac4a76a88c3aac6e70503dc866104aa806516", + "sha256": "813732a2cbae7f5d4b1e471e3c3ac4a76a88c3aac6e70503dc866104aa806516" + }, + { + "path": ".gitignore", + "upstreamSha256": "12b8a657f57a9113bfa5d6334d53a75883b18a4b5268cb0f0c27c10390acc68c", + "sha256": "12b8a657f57a9113bfa5d6334d53a75883b18a4b5268cb0f0c27c10390acc68c" + }, + { + "path": ".python-version", + "upstreamSha256": "7b55f8e67b5623c4bef3fa691288da9437d79d3aba156de48d481db32ac7d16d", + "sha256": "7b55f8e67b5623c4bef3fa691288da9437d79d3aba156de48d481db32ac7d16d" + }, + { + "path": "CHANGELOG.md", + "upstreamSha256": "d58a09e672d3e0f9019fd241037a646c969242af316a7633475a664b98fad9e0", + "sha256": "d58a09e672d3e0f9019fd241037a646c969242af316a7633475a664b98fad9e0" + }, + { + "path": "Cargo.toml", + "upstreamSha256": "adbef029a0a2df62d5b355d1261a989cb94c58fabb0adfabbf71f7daa12c56a2", + "sha256": "e2abaea0e6fc4018ebe60e48d416dfbfd3f5f0ef42cc6cfe98cba1e7b800c47f" + }, + { + "path": "Cargo.toml.orig", + "upstreamSha256": "510cefeb66e0aeb030fa23912084d3e468d547f863a65b45631980b7ecb1842d", + "sha256": "510cefeb66e0aeb030fa23912084d3e468d547f863a65b45631980b7ecb1842d" + }, + { + "path": "LICENSE-APACHE", + "upstreamSha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4", + "sha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4" + }, + { + "path": "LICENSE-MIT", + "upstreamSha256": "23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3", + "sha256": "23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3" + }, + { + "path": "Makefile", + "upstreamSha256": "7697845148dd4145ffa3945b4a0ec4f5ec640707742def245caff108ef4826b9", + "sha256": "7697845148dd4145ffa3945b4a0ec4f5ec640707742def245caff108ef4826b9" + }, + { + "path": "README.md", + "upstreamSha256": "7de992bd0004550851085eca2c326d3caa68d768fec8fdf614893cb6c9ddfe84", + "sha256": "7de992bd0004550851085eca2c326d3caa68d768fec8fdf614893cb6c9ddfe84" + }, + { + "path": "build.rs", + "upstreamSha256": "c3485073dba6a1b483b781e9699852f4fcfbe7410a5502521b2d4e985fa79250", + "sha256": "c3485073dba6a1b483b781e9699852f4fcfbe7410a5502521b2d4e985fa79250" + }, + { + "path": "clippy.toml", + "upstreamSha256": "7e0750989c182ad8742564a5f3862174984c140e373cf904f8b0c2079dfe2f85", + "sha256": "7e0750989c182ad8742564a5f3862174984c140e373cf904f8b0c2079dfe2f85" + }, + { + "path": "dune-project", + "upstreamSha256": "4d3262e21b391caa29d21ee71b946c3d7c213680169b6e953feee12aca082666", + "sha256": "4d3262e21b391caa29d21ee71b946c3d7c213680169b6e953feee12aca082666" + }, + { + "path": "examples/basics.rs", + "upstreamSha256": "5059e17e83429de3e04f2937f6053032b8ac2e876b2b1d1e5230d952d06dff95", + "sha256": "5059e17e83429de3e04f2937f6053032b8ac2e876b2b1d1e5230d952d06dff95" + }, + { + "path": "examples/char-rnn/README.md", + "upstreamSha256": "e5c0121e55a3d0c71615329c096de82d177281bc97baafab3f30dcf733aa9763", + "sha256": "e5c0121e55a3d0c71615329c096de82d177281bc97baafab3f30dcf733aa9763" + }, + { + "path": "examples/char-rnn/main.rs", + "upstreamSha256": "cfb75afc5557d11a9ba595eff5e34dbfe3c5d9fbf690349e3b51d886af3df40d", + "sha256": "cfb75afc5557d11a9ba595eff5e34dbfe3c5d9fbf690349e3b51d886af3df40d" + }, + { + "path": "examples/cifar/main.rs", + "upstreamSha256": "a4ee823bb1d8e6faac1a4e38221e849b5dc702f9fb6d9b1f3b8ebeccb6bb3991", + "sha256": "a4ee823bb1d8e6faac1a4e38221e849b5dc702f9fb6d9b1f3b8ebeccb6bb3991" + }, + { + "path": "examples/custom-optimizer/README.md", + "upstreamSha256": "d05debefba2b5ba3600cc5496f669b3a916effe43aaf0b95c3420b1551f123c6", + "sha256": "d05debefba2b5ba3600cc5496f669b3a916effe43aaf0b95c3420b1551f123c6" + }, + { + "path": "examples/custom-optimizer/main.rs", + "upstreamSha256": "4927a02258f46a1b24e62eac1e44591aa6efa8ace7d291b167f7b29416e68646", + "sha256": "4927a02258f46a1b24e62eac1e44591aa6efa8ace7d291b167f7b29416e68646" + }, + { + "path": "examples/custom-optimizer/sparse_adam.rs", + "upstreamSha256": "fc00de4d91983408f1a263a9a0295043e5df7c53cabbca4a4ea06f646eaa97f5", + "sha256": "fc00de4d91983408f1a263a9a0295043e5df7c53cabbca4a4ea06f646eaa97f5" + }, + { + "path": "examples/gan/main.rs", + "upstreamSha256": "5cf6987046c7f1a32c8ea61528438311e1f8a6003efaf038e95ee87d5ffc96c2", + "sha256": "5cf6987046c7f1a32c8ea61528438311e1f8a6003efaf038e95ee87d5ffc96c2" + }, + { + "path": "examples/jit/README.md", + "upstreamSha256": "5e3dfe6c37d99face010555ddbee4aeaf86326ffda33834bde6bb823a6673fbe", + "sha256": "5e3dfe6c37d99face010555ddbee4aeaf86326ffda33834bde6bb823a6673fbe" + }, + { + "path": "examples/jit/main.rs", + "upstreamSha256": "f95f8c7f8e17d2d48f546723c9dcd347504f7e0af56d88dd8968d3be3e479e1d", + "sha256": "f95f8c7f8e17d2d48f546723c9dcd347504f7e0af56d88dd8968d3be3e479e1d" + }, + { + "path": "examples/jit/resnet.py", + "upstreamSha256": "d6e57912e48abce72ef364b781d87bb0b0d367f800cffe4a84b7597f908d78cf", + "sha256": "d6e57912e48abce72ef364b781d87bb0b0d367f800cffe4a84b7597f908d78cf" + }, + { + "path": "examples/jit-quantized/README.md", + "upstreamSha256": "f051686f643f3df136667a69a418f9d3d9455d66307b3ffc62c307c17feca810", + "sha256": "f051686f643f3df136667a69a418f9d3d9455d66307b3ffc62c307c17feca810" + }, + { + "path": "examples/jit-quantized/main.rs", + "upstreamSha256": "39f0e44f0fe0261a57b720e8dcaf270b16c0cc8e97bb407951f538cff9b0b4d7", + "sha256": "39f0e44f0fe0261a57b720e8dcaf270b16c0cc8e97bb407951f538cff9b0b4d7" + }, + { + "path": "examples/jit-quantized/resnet.py", + "upstreamSha256": "b05fecadefb77685721373bf421e4753b5c6037a2e48364859f9c96f71a50cfc", + "sha256": "b05fecadefb77685721373bf421e4753b5c6037a2e48364859f9c96f71a50cfc" + }, + { + "path": "examples/jit-trace/README.md", + "upstreamSha256": "fc0de268c396414553f3c828fc607769e441751bf48d9a63058e2a4b13028c9b", + "sha256": "fc0de268c396414553f3c828fc607769e441751bf48d9a63058e2a4b13028c9b" + }, + { + "path": "examples/jit-trace/main.rs", + "upstreamSha256": "dc4ebf024747929f4f69eead423ff02ecdb96996fa21d11566fac048e7cf88d2", + "sha256": "dc4ebf024747929f4f69eead423ff02ecdb96996fa21d11566fac048e7cf88d2" + }, + { + "path": "examples/jit-trace/test.py", + "upstreamSha256": "a9f5c1940e886695b8eeb17344ae7c1beb0a008e9a638fd5206928bc4ca0b272", + "sha256": "a9f5c1940e886695b8eeb17344ae7c1beb0a008e9a638fd5206928bc4ca0b272" + }, + { + "path": "examples/jit-train/README.md", + "upstreamSha256": "12c99ed805300fe8d94bb8ff25a8cb85b4d97a623546f5281057e43328945832", + "sha256": "12c99ed805300fe8d94bb8ff25a8cb85b4d97a623546f5281057e43328945832" + }, + { + "path": "examples/jit-train/main.rs", + "upstreamSha256": "aaa2677aa66d63836d2115fa2e1d1db656503997b95fb6af58baf204e1ab6278", + "sha256": "aaa2677aa66d63836d2115fa2e1d1db656503997b95fb6af58baf204e1ab6278" + }, + { + "path": "examples/jit-train/resnet.py", + "upstreamSha256": "4c1fba01663d8cf36fc18b98ba478049bfc945a26e7fd9e89369ea7db297a6f7", + "sha256": "4c1fba01663d8cf36fc18b98ba478049bfc945a26e7fd9e89369ea7db297a6f7" + }, + { + "path": "examples/llama/convert_checkpoint.py", + "upstreamSha256": "16406a2e91c95ba5ee36690fe89cfc09020f011b0b6c2ee5e8e9bc8f177869ff", + "sha256": "16406a2e91c95ba5ee36690fe89cfc09020f011b0b6c2ee5e8e9bc8f177869ff" + }, + { + "path": "examples/llama/main.rs", + "upstreamSha256": "bc295bfe11070eb5f9d17e9fd077b4ea99b6388018526a1bf2fc3fd71052b418", + "sha256": "bc295bfe11070eb5f9d17e9fd077b4ea99b6388018526a1bf2fc3fd71052b418" + }, + { + "path": "examples/llama/sentencepiece.rs", + "upstreamSha256": "0df84fe132840454f6c20ef5db0679409f4081d294a8bdcd027637f2fd7dc6d9", + "sha256": "0df84fe132840454f6c20ef5db0679409f4081d294a8bdcd027637f2fd7dc6d9" + }, + { + "path": "examples/memory_test.rs", + "upstreamSha256": "5a7c82419512ebc1e484155a94b0ad0b254f53aaf6d3b2b5f04e41d886a10219", + "sha256": "5a7c82419512ebc1e484155a94b0ad0b254f53aaf6d3b2b5f04e41d886a10219" + }, + { + "path": "examples/min-gpt/README.md", + "upstreamSha256": "6324404f9fc300b1b4050d8f10cf891ccd92ab19b021ead52d713f40768bbaa8", + "sha256": "6324404f9fc300b1b4050d8f10cf891ccd92ab19b021ead52d713f40768bbaa8" + }, + { + "path": "examples/min-gpt/main.rs", + "upstreamSha256": "f493826895f9f804e56c766c1f5b80a1f300b17a21e9f6c9df049a39ea686fa4", + "sha256": "f493826895f9f804e56c766c1f5b80a1f300b17a21e9f6c9df049a39ea686fa4" + }, + { + "path": "examples/mnist/README.md", + "upstreamSha256": "c0e42d7b5df9a263ae99e90ce90a1aefe67d3b10d6d406e251460265106ad8f7", + "sha256": "c0e42d7b5df9a263ae99e90ce90a1aefe67d3b10d6d406e251460265106ad8f7" + }, + { + "path": "examples/mnist/main.rs", + "upstreamSha256": "0aecfe7aca1920ec90901f87474612bff5aa72a6877793cfb25f5b2f08de1b6e", + "sha256": "0aecfe7aca1920ec90901f87474612bff5aa72a6877793cfb25f5b2f08de1b6e" + }, + { + "path": "examples/mnist/mnist_conv.rs", + "upstreamSha256": "6a8e430eb80f87662136c56d60df916b9df025818f86cb5e2cf69e6eaefc7004", + "sha256": "6a8e430eb80f87662136c56d60df916b9df025818f86cb5e2cf69e6eaefc7004" + }, + { + "path": "examples/mnist/mnist_linear.rs", + "upstreamSha256": "afa61ca3c1f9bfa914a0ef54ebdfbfa3c5a2a70c60895cb0f34600a097eeb45c", + "sha256": "afa61ca3c1f9bfa914a0ef54ebdfbfa3c5a2a70c60895cb0f34600a097eeb45c" + }, + { + "path": "examples/mnist/mnist_nn.rs", + "upstreamSha256": "93c169a72cd9aefe7a728ace6aa82653859637444d2bcbeb2681ff5633960c97", + "sha256": "93c169a72cd9aefe7a728ace6aa82653859637444d2bcbeb2681ff5633960c97" + }, + { + "path": "examples/neural-style-transfer/README.md", + "upstreamSha256": "e7e7c64e392cbb75ddae3712e74a0dc5139a106020b6d560be5a609941b91463", + "sha256": "e7e7c64e392cbb75ddae3712e74a0dc5139a106020b6d560be5a609941b91463" + }, + { + "path": "examples/neural-style-transfer/main.rs", + "upstreamSha256": "20df81430d7282e02f3db986e110c093635697759ba0ab148b01e7d17b21d89c", + "sha256": "20df81430d7282e02f3db986e110c093635697759ba0ab148b01e7d17b21d89c" + }, + { + "path": "examples/pretrained-models/main.rs", + "upstreamSha256": "e440848e0a1b8917ec3e4fd684377a83df73a38acc6f8beead8cec32568fc3bb", + "sha256": "e440848e0a1b8917ec3e4fd684377a83df73a38acc6f8beead8cec32568fc3bb" + }, + { + "path": "examples/reinforcement-learning/README.md", + "upstreamSha256": "fa06d2e419c0e0b9f19d8945fff4855e3494ecca5cb468e81aa01faa62671950", + "sha256": "fa06d2e419c0e0b9f19d8945fff4855e3494ecca5cb468e81aa01faa62671950" + }, + { + "path": "examples/reinforcement-learning/a2c.rs", + "upstreamSha256": "b009bbdff77a7fda8970886e571f15215549230cbb3581527b5dd1a754c5c01e", + "sha256": "b009bbdff77a7fda8970886e571f15215549230cbb3581527b5dd1a754c5c01e" + }, + { + "path": "examples/reinforcement-learning/atari_wrappers.py", + "upstreamSha256": "6d8c6fb8f9777096c991948fa10cdbf71ffce72c56cb9a8da05ed8f89fdfda96", + "sha256": "6d8c6fb8f9777096c991948fa10cdbf71ffce72c56cb9a8da05ed8f89fdfda96" + }, + { + "path": "examples/reinforcement-learning/ddpg.rs", + "upstreamSha256": "9868b522a46facddd3cb1ad85adb9f802e139c2d6f1529a44c09d15f880d7f96", + "sha256": "9868b522a46facddd3cb1ad85adb9f802e139c2d6f1529a44c09d15f880d7f96" + }, + { + "path": "examples/reinforcement-learning/gym_env.rs", + "upstreamSha256": "07ae70893e77e48b3eee0835b590ddc90bcee4e58e33981d94fb272cbb3ed087", + "sha256": "07ae70893e77e48b3eee0835b590ddc90bcee4e58e33981d94fb272cbb3ed087" + }, + { + "path": "examples/reinforcement-learning/main.rs", + "upstreamSha256": "a8bb5e2b12a4c784899ab84ab2c3812457277b96a7d6d58dd15a068d0da56bd3", + "sha256": "a8bb5e2b12a4c784899ab84ab2c3812457277b96a7d6d58dd15a068d0da56bd3" + }, + { + "path": "examples/reinforcement-learning/policy_gradient.rs", + "upstreamSha256": "85d66e592aab4f433efd9e534b73a82c0160d0422ee318bb43176240e8b7ca29", + "sha256": "85d66e592aab4f433efd9e534b73a82c0160d0422ee318bb43176240e8b7ca29" + }, + { + "path": "examples/reinforcement-learning/ppo.rs", + "upstreamSha256": "cfcf5bac4e000e8e891455c3cc08af3b31a29c386596c5351d2a0e7cbd62f715", + "sha256": "cfcf5bac4e000e8e891455c3cc08af3b31a29c386596c5351d2a0e7cbd62f715" + }, + { + "path": "examples/reinforcement-learning/vec_gym_env.rs", + "upstreamSha256": "c9e8e9502d8f40636969fa6b87cb7150467e06134c7de6c95e4e6681618bc01c", + "sha256": "c9e8e9502d8f40636969fa6b87cb7150467e06134c7de6c95e4e6681618bc01c" + }, + { + "path": "examples/stable-diffusion/README.md", + "upstreamSha256": "8320c7f0f009dcae5d4e73ca8d6e06e013ec8c1ca1c5c1e72908b4e117791db3", + "sha256": "8320c7f0f009dcae5d4e73ca8d6e06e013ec8c1ca1c5c1e72908b4e117791db3" + }, + { + "path": "examples/stable-diffusion/main.rs", + "upstreamSha256": "6cc1de257c7721e944ed38ae6eec27de55450d5443bb0726003ff184117dd78e", + "sha256": "6cc1de257c7721e944ed38ae6eec27de55450d5443bb0726003ff184117dd78e" + }, + { + "path": "examples/tensor-tools.rs", + "upstreamSha256": "3667cb780fb9d978733703a57438f311ffcc6b527e06ea6e940de7d2080e8452", + "sha256": "3667cb780fb9d978733703a57438f311ffcc6b527e06ea6e940de7d2080e8452" + }, + { + "path": "examples/transfer-learning/README.md", + "upstreamSha256": "55d10f559ba7ad67c09f2e1785d8a0d7581c367544561571d21f33cbff89efd9", + "sha256": "55d10f559ba7ad67c09f2e1785d8a0d7581c367544561571d21f33cbff89efd9" + }, + { + "path": "examples/transfer-learning/main.rs", + "upstreamSha256": "15611d564628ab805b85bd16fcf36bbc2fd319e66ff0600063f7d452eb3991f7", + "sha256": "15611d564628ab805b85bd16fcf36bbc2fd319e66ff0600063f7d452eb3991f7" + }, + { + "path": "examples/translation/dataset.rs", + "upstreamSha256": "71dc84d0624442607c4da514e1d3d6bf4ce68115fb05a9a9ddc6f57e08e27021", + "sha256": "71dc84d0624442607c4da514e1d3d6bf4ce68115fb05a9a9ddc6f57e08e27021" + }, + { + "path": "examples/translation/lang.rs", + "upstreamSha256": "afd22986d5685bff5c676b52f8df174500cb0571b67ca9fa86fcd9a02a08d84c", + "sha256": "afd22986d5685bff5c676b52f8df174500cb0571b67ca9fa86fcd9a02a08d84c" + }, + { + "path": "examples/translation/main.rs", + "upstreamSha256": "218f2f42e308f4e82ada9ac593b5b5e0e1ad81f2d8a31f2ca2122268acbdc5d4", + "sha256": "218f2f42e308f4e82ada9ac593b5b5e0e1ad81f2d8a31f2ca2122268acbdc5d4" + }, + { + "path": "examples/vae/main.rs", + "upstreamSha256": "c2918c72fac34a963dfa0af5f95e80c388504a1f2c88880bb3e2edf2554e2c57", + "sha256": "c2918c72fac34a963dfa0af5f95e80c388504a1f2c88880bb3e2edf2554e2c57" + }, + { + "path": "examples/yolo/coco_classes.rs", + "upstreamSha256": "ea234f9868307a4f40ce6db5b9de801173143ee18b76ec19c0839192cb0f6b70", + "sha256": "ea234f9868307a4f40ce6db5b9de801173143ee18b76ec19c0839192cb0f6b70" + }, + { + "path": "examples/yolo/darknet.rs", + "upstreamSha256": "4625b95c621dea03ff14025b835c751a8ba983a34ec72faafb3a40c2de56146b", + "sha256": "4625b95c621dea03ff14025b835c751a8ba983a34ec72faafb3a40c2de56146b" + }, + { + "path": "examples/yolo/main.rs", + "upstreamSha256": "c3c8ca9439f1301e0b4c51d90b0c3bcae6d187edf56f3e7e129a716ad86dbf4b", + "sha256": "c3c8ca9439f1301e0b4c51d90b0c3bcae6d187edf56f3e7e129a716ad86dbf4b" + }, + { + "path": "examples/yolo/yolo-v3.cfg", + "upstreamSha256": "d9ec1f4e2dc3d9644429a2bcd7046ca5441938a742899b6ba917e076f6a6ed9b", + "sha256": "d9ec1f4e2dc3d9644429a2bcd7046ca5441938a742899b6ba917e076f6a6ed9b" + }, + { + "path": "gen/.ocamlformat", + "upstreamSha256": "ade33328770bc5abae551708c82b1dbd5692bcc34f620194cc0e4da0f129a799", + "sha256": "ade33328770bc5abae551708c82b1dbd5692bcc34f620194cc0e4da0f129a799" + }, + { + "path": "gen/dune", + "upstreamSha256": "bdb0b14b2925f3dfc8a3773d68107d6b7152adbeebcdb7089684291cd555ea97", + "sha256": "bdb0b14b2925f3dfc8a3773d68107d6b7152adbeebcdb7089684291cd555ea97" + }, + { + "path": "gen/gen.ml", + "upstreamSha256": "9b6cedf6f67922aa181175be8d2fea30a38ae3a61de63a6ab5be786b4ced2261", + "sha256": "9b6cedf6f67922aa181175be8d2fea30a38ae3a61de63a6ab5be786b4ced2261" + }, + { + "path": "gen/gen.mli", + "upstreamSha256": "473ecfe5ccae7c03cd11744977d1c5829a89027a1608b28378226bf4746afbd6", + "sha256": "473ecfe5ccae7c03cd11744977d1c5829a89027a1608b28378226bf4746afbd6" + }, + { + "path": "main.py", + "upstreamSha256": "683aef4a3ee7db81e8324b2bf7e865f1033a196509e46e0d56e0a1eb345da3a9", + "sha256": "683aef4a3ee7db81e8324b2bf7e865f1033a196509e46e0d56e0a1eb345da3a9" + }, + { + "path": "pyproject.toml", + "upstreamSha256": "55524021b952764e06e4f99ef6d6266df2b5a0a9bb34bb04ae48e47d5d4b1182", + "sha256": "55524021b952764e06e4f99ef6d6266df2b5a0a9bb34bb04ae48e47d5d4b1182" + }, + { + "path": "rustfmt.toml", + "upstreamSha256": "976b4ed388544f55349256eb91d07e16f4bcce42ce325d09bf090cc003ec72aa", + "sha256": "976b4ed388544f55349256eb91d07e16f4bcce42ce325d09bf090cc003ec72aa" + }, + { + "path": "src/data.rs", + "upstreamSha256": "442dda726016da0e48a8aba2d37ba8724c76564e70e5a2527ef0e4637605cedf", + "sha256": "442dda726016da0e48a8aba2d37ba8724c76564e70e5a2527ef0e4637605cedf" + }, + { + "path": "src/error.rs", + "upstreamSha256": "176156fe1a1c60f46ec4d1eace32d23454917cde98e3ebd6c3e7509e5f9855bb", + "sha256": "176156fe1a1c60f46ec4d1eace32d23454917cde98e3ebd6c3e7509e5f9855bb" + }, + { + "path": "src/lib.rs", + "upstreamSha256": "af659efd91873381e49a86cc2c13f030851363f24479ee498c6f157e3a45b4dd", + "sha256": "af659efd91873381e49a86cc2c13f030851363f24479ee498c6f157e3a45b4dd" + }, + { + "path": "src/nn/batch_norm.rs", + "upstreamSha256": "b98a8cc4f570a00c1c6d06700ceb413bfe186b9f56b5ba0a9b97d0bb24a5213d", + "sha256": "b98a8cc4f570a00c1c6d06700ceb413bfe186b9f56b5ba0a9b97d0bb24a5213d" + }, + { + "path": "src/nn/conv.rs", + "upstreamSha256": "4de67d22462918cb67eee394eeaec3659dbd47ea8934304a3272c94b4ecd9c39", + "sha256": "4de67d22462918cb67eee394eeaec3659dbd47ea8934304a3272c94b4ecd9c39" + }, + { + "path": "src/nn/conv_transpose.rs", + "upstreamSha256": "d881e48332ff94ae5a751d939cae64f7da1728b974010da8cc04995b214bb24c", + "sha256": "d881e48332ff94ae5a751d939cae64f7da1728b974010da8cc04995b214bb24c" + }, + { + "path": "src/nn/func.rs", + "upstreamSha256": "0353023499b3424d5b270c240c169d03050a26189a00b89f8ebd0cacfa142558", + "sha256": "0353023499b3424d5b270c240c169d03050a26189a00b89f8ebd0cacfa142558" + }, + { + "path": "src/nn/group_norm.rs", + "upstreamSha256": "5361fd0be596bb29baf44a6803b8be0bcbc5a3e946d580e029d83528ace04777", + "sha256": "5361fd0be596bb29baf44a6803b8be0bcbc5a3e946d580e029d83528ace04777" + }, + { + "path": "src/nn/init.rs", + "upstreamSha256": "8568346e1f460d0d15be56988608a44cd8d153537ecd9d055544881b286aaa57", + "sha256": "8568346e1f460d0d15be56988608a44cd8d153537ecd9d055544881b286aaa57" + }, + { + "path": "src/nn/layer_norm.rs", + "upstreamSha256": "00165a19464fa8f2917725bdd494ba7e73ff5107f58e67dd9e5eaa644ed395ab", + "sha256": "00165a19464fa8f2917725bdd494ba7e73ff5107f58e67dd9e5eaa644ed395ab" + }, + { + "path": "src/nn/linear.rs", + "upstreamSha256": "1195ebadc53485ae45abd68f9c817274330fcfa12d501d81fe8944654e66bc78", + "sha256": "1195ebadc53485ae45abd68f9c817274330fcfa12d501d81fe8944654e66bc78" + }, + { + "path": "src/nn/mod.rs", + "upstreamSha256": "f64e448ed6fe04391178347f4f8eb44283994cdd933babcbd4b2a1f3dc033c30", + "sha256": "f64e448ed6fe04391178347f4f8eb44283994cdd933babcbd4b2a1f3dc033c30" + }, + { + "path": "src/nn/module.rs", + "upstreamSha256": "f2d8a75740c0ab58362f5177708bf1e838c8a77dcacb8ffd0ff241b2069020c4", + "sha256": "f2d8a75740c0ab58362f5177708bf1e838c8a77dcacb8ffd0ff241b2069020c4" + }, + { + "path": "src/nn/optimizer.rs", + "upstreamSha256": "de0149e1e30e3ff1616ee48a84971f413e0f21baa6f89e4f24303f7b822c4353", + "sha256": "de0149e1e30e3ff1616ee48a84971f413e0f21baa6f89e4f24303f7b822c4353" + }, + { + "path": "src/nn/rnn.rs", + "upstreamSha256": "4d3351230b3678ff1a49158d6d88b0c407f30468dd4c46c6dece343c38d46353", + "sha256": "4d3351230b3678ff1a49158d6d88b0c407f30468dd4c46c6dece343c38d46353" + }, + { + "path": "src/nn/sequential.rs", + "upstreamSha256": "a1a7db1e8addeffa920ca44183d8304158dd71b62ac697de1bbf2875f71ae180", + "sha256": "a1a7db1e8addeffa920ca44183d8304158dd71b62ac697de1bbf2875f71ae180" + }, + { + "path": "src/nn/sparse.rs", + "upstreamSha256": "a20f93d1f004f29aaa8022cf0f4ead95c71d622eb1f4429c7ebf52f3f93c71f5", + "sha256": "a20f93d1f004f29aaa8022cf0f4ead95c71d622eb1f4429c7ebf52f3f93c71f5" + }, + { + "path": "src/nn/var_store.rs", + "upstreamSha256": "aaf0c8ba59a7852ae93c02b5c65fe3972bca279937e6fd6f20a358aa455ab2c7", + "sha256": "aaf0c8ba59a7852ae93c02b5c65fe3972bca279937e6fd6f20a358aa455ab2c7" + }, + { + "path": "src/tensor/convert.rs", + "upstreamSha256": "32933952a4f072c08499a23a17e7c781f3643fe6a5ad691855c735f85844aae3", + "sha256": "32933952a4f072c08499a23a17e7c781f3643fe6a5ad691855c735f85844aae3" + }, + { + "path": "src/tensor/display.rs", + "upstreamSha256": "1b25afd950ec9c85a68007f9699986dcd4696a198a0c03ae0b47c2b1da6c9b77", + "sha256": "1b25afd950ec9c85a68007f9699986dcd4696a198a0c03ae0b47c2b1da6c9b77" + }, + { + "path": "src/tensor/index.rs", + "upstreamSha256": "b7d86aea010c98a0c14a5c1899bcba0100039b7c992d1ac52f8b57ac56116173", + "sha256": "b7d86aea010c98a0c14a5c1899bcba0100039b7c992d1ac52f8b57ac56116173" + }, + { + "path": "src/tensor/iter.rs", + "upstreamSha256": "d069c87a72eef94643421a791865cc9d2313ac027809fac16ed601b00f266b6a", + "sha256": "d069c87a72eef94643421a791865cc9d2313ac027809fac16ed601b00f266b6a" + }, + { + "path": "src/tensor/mod.rs", + "upstreamSha256": "c6023b05b03cfe8d806bf2a7d36bb4a377092383eff07fe49218e68fda6aaf91", + "sha256": "c6023b05b03cfe8d806bf2a7d36bb4a377092383eff07fe49218e68fda6aaf91" + }, + { + "path": "src/tensor/npy.rs", + "upstreamSha256": "6f9c3c4dd4a57f6ba7c464e7159c83048a7a287641166dc44f7011b9b40e5728", + "sha256": "11320e9c6d0621874f6e9814063cf793e9a25e24b215ed9761c4e7f39d2e787c" + }, + { + "path": "src/tensor/ops.rs", + "upstreamSha256": "0b653a7c9e135a5b98ca41ffa612d5ded438b8b08d99c1568077c6bb32a1abaa", + "sha256": "0b653a7c9e135a5b98ca41ffa612d5ded438b8b08d99c1568077c6bb32a1abaa" + }, + { + "path": "src/tensor/safetensors.rs", + "upstreamSha256": "68526ff9e266ce26ab543a37e8e221cdd88596a55f263f90d2cd9144789fd63d", + "sha256": "68526ff9e266ce26ab543a37e8e221cdd88596a55f263f90d2cd9144789fd63d" + }, + { + "path": "src/vision/alexnet.rs", + "upstreamSha256": "8d56ae9d6bdc4546969f115fba41e3fb8d86ac1bc441151648830c60bdb88182", + "sha256": "8d56ae9d6bdc4546969f115fba41e3fb8d86ac1bc441151648830c60bdb88182" + }, + { + "path": "src/vision/cifar.rs", + "upstreamSha256": "6f36e04700a6ce5638cedee14d3303433682f97cbc4ee7ae3c618b91749bdadc", + "sha256": "6f36e04700a6ce5638cedee14d3303433682f97cbc4ee7ae3c618b91749bdadc" + }, + { + "path": "src/vision/convmixer.rs", + "upstreamSha256": "4440d6bb206528ecaf790168e3f867f75c9bb8432bba61bf75872c25e3ce0648", + "sha256": "4440d6bb206528ecaf790168e3f867f75c9bb8432bba61bf75872c25e3ce0648" + }, + { + "path": "src/vision/dataset.rs", + "upstreamSha256": "71bfc17de04742e9e45f6552a098674008e3044cd1d23e9fd3016891d958d83d", + "sha256": "71bfc17de04742e9e45f6552a098674008e3044cd1d23e9fd3016891d958d83d" + }, + { + "path": "src/vision/densenet.rs", + "upstreamSha256": "f8c427602f2d8a202f0da8fe0274b3e2ac1cad890874a96ac6adc4734b3a2941", + "sha256": "f8c427602f2d8a202f0da8fe0274b3e2ac1cad890874a96ac6adc4734b3a2941" + }, + { + "path": "src/vision/dinov2.rs", + "upstreamSha256": "a5f28c0e4f2b06175f69209522b273e9b38c55fd757f6258b60ecda0cebf5e9d", + "sha256": "a5f28c0e4f2b06175f69209522b273e9b38c55fd757f6258b60ecda0cebf5e9d" + }, + { + "path": "src/vision/efficientnet.rs", + "upstreamSha256": "acf386a6e4d34918b159c23dfb93b2ac6f4534d52e692eb8768060b51e59a736", + "sha256": "acf386a6e4d34918b159c23dfb93b2ac6f4534d52e692eb8768060b51e59a736" + }, + { + "path": "src/vision/export_dinov2.py", + "upstreamSha256": "b3a8011f7dbb73acf5b3f1d9517cde392343df29528caf786151164419c09758", + "sha256": "b3a8011f7dbb73acf5b3f1d9517cde392343df29528caf786151164419c09758" + }, + { + "path": "src/vision/export_model.py", + "upstreamSha256": "81aeaf9851b7dc1975e60a554a5f6b87a371444425eb2461a80e35202c76b213", + "sha256": "81aeaf9851b7dc1975e60a554a5f6b87a371444425eb2461a80e35202c76b213" + }, + { + "path": "src/vision/image.rs", + "upstreamSha256": "13db66d8de4845d235322af48c02f80e46b441c0a06be0b94e13da56e43433f7", + "sha256": "13db66d8de4845d235322af48c02f80e46b441c0a06be0b94e13da56e43433f7" + }, + { + "path": "src/vision/imagenet.rs", + "upstreamSha256": "afd4332f031b911513023bbab02f6b143dbf0e1b113ccfdfbf2c2fd03436c33d", + "sha256": "afd4332f031b911513023bbab02f6b143dbf0e1b113ccfdfbf2c2fd03436c33d" + }, + { + "path": "src/vision/inception.rs", + "upstreamSha256": "a2486d88d91adcbacfe583e12facb4f1a78b1eb15a5a4848bce5dd5f165f2bfd", + "sha256": "a2486d88d91adcbacfe583e12facb4f1a78b1eb15a5a4848bce5dd5f165f2bfd" + }, + { + "path": "src/vision/mnist.rs", + "upstreamSha256": "74dfd3ea86e9913a977087a52ad511952df1f3c0354712bd7f5f8be1763becf7", + "sha256": "74dfd3ea86e9913a977087a52ad511952df1f3c0354712bd7f5f8be1763becf7" + }, + { + "path": "src/vision/mobilenet.rs", + "upstreamSha256": "14bd40b7642181f6ddaf548f60d76e4ca78cff6256b4996119bbc9c149c72cec", + "sha256": "14bd40b7642181f6ddaf548f60d76e4ca78cff6256b4996119bbc9c149c72cec" + }, + { + "path": "src/vision/mod.rs", + "upstreamSha256": "32cf8ebd17678f80673dac2f2a9718b164e9ef1d0b1a6f6f8d9dd3b98d99449d", + "sha256": "32cf8ebd17678f80673dac2f2a9718b164e9ef1d0b1a6f6f8d9dd3b98d99449d" + }, + { + "path": "src/vision/resnet.rs", + "upstreamSha256": "dca15a6d5cfbe7f37e942c173087fdaeb19a347bfca2718ce45721ef244d2433", + "sha256": "dca15a6d5cfbe7f37e942c173087fdaeb19a347bfca2718ce45721ef244d2433" + }, + { + "path": "src/vision/rust_image.rs", + "upstreamSha256": "2b13844118406c0de90c44d0e0d989a53481a5f919d328269366c7debfdea46a", + "sha256": "2b13844118406c0de90c44d0e0d989a53481a5f919d328269366c7debfdea46a" + }, + { + "path": "src/vision/squeezenet.rs", + "upstreamSha256": "406e5036ab0688b5279bbc149f89c6032c722d9bf5db51d07d58fd8b89b7b547", + "sha256": "406e5036ab0688b5279bbc149f89c6032c722d9bf5db51d07d58fd8b89b7b547" + }, + { + "path": "src/vision/vgg.rs", + "upstreamSha256": "4f35d9d27df98eb1879394e8451b29a99c01a46c94f28b00079884d8b1fb4fb5", + "sha256": "4f35d9d27df98eb1879394e8451b29a99c01a46c94f28b00079884d8b1fb4fb5" + }, + { + "path": "src/wrappers/device.rs", + "upstreamSha256": "c3294ab621b92affb11323257fd3fc21877a9404c0826143b4800e5bfaf42142", + "sha256": "c3294ab621b92affb11323257fd3fc21877a9404c0826143b4800e5bfaf42142" + }, + { + "path": "src/wrappers/image.rs", + "upstreamSha256": "368abc92ebb07db9fdc89d4b068e81e8b48fa07bb6a52ff9d82e1220bad7275d", + "sha256": "368abc92ebb07db9fdc89d4b068e81e8b48fa07bb6a52ff9d82e1220bad7275d" + }, + { + "path": "src/wrappers/jit.rs", + "upstreamSha256": "f3cf318bb0574235b455577cfaa4fcbb89f213d4889f990813b5fe65ec9b7567", + "sha256": "f3cf318bb0574235b455577cfaa4fcbb89f213d4889f990813b5fe65ec9b7567" + }, + { + "path": "src/wrappers/kind.rs", + "upstreamSha256": "93d2a7f93983863ef95f3064a51997604a57244474cbe6362bd9a881b6c71fb4", + "sha256": "93d2a7f93983863ef95f3064a51997604a57244474cbe6362bd9a881b6c71fb4" + }, + { + "path": "src/wrappers/layout.rs", + "upstreamSha256": "b6ca7feb4408dcb0b78fc16679013f0f6764d843e89504e231d8c308754b0b06", + "sha256": "b6ca7feb4408dcb0b78fc16679013f0f6764d843e89504e231d8c308754b0b06" + }, + { + "path": "src/wrappers/mod.rs", + "upstreamSha256": "6a94ed3c44fb1fb1db729cfe938f41e2cf47a7fa64e4d5245d7c1a168835e513", + "sha256": "6a94ed3c44fb1fb1db729cfe938f41e2cf47a7fa64e4d5245d7c1a168835e513" + }, + { + "path": "src/wrappers/optimizer.rs", + "upstreamSha256": "c04cf4f4113a340d2e1ac7f67230eed51d7aff1139027ead196f665ec467ceba", + "sha256": "c04cf4f4113a340d2e1ac7f67230eed51d7aff1139027ead196f665ec467ceba" + }, + { + "path": "src/wrappers/python.rs", + "upstreamSha256": "85e660baa0735954d4003fad47d24a8066451131c56303bf9fee88025eb44bd3", + "sha256": "85e660baa0735954d4003fad47d24a8066451131c56303bf9fee88025eb44bd3" + }, + { + "path": "src/wrappers/scalar.rs", + "upstreamSha256": "cf532532527c50cc6fba9688b0a5d8addb191c248bd251fcca57c468647f3be2", + "sha256": "cf532532527c50cc6fba9688b0a5d8addb191c248bd251fcca57c468647f3be2" + }, + { + "path": "src/wrappers/stream.rs", + "upstreamSha256": "9af1abd309cd2c5b2b0d1e6a42505f983d081e5edbebb82c327b2b48dd1c2ae1", + "sha256": "9af1abd309cd2c5b2b0d1e6a42505f983d081e5edbebb82c327b2b48dd1c2ae1" + }, + { + "path": "src/wrappers/tensor.rs", + "upstreamSha256": "965eea22e487b9cc801c6c96730dd297c9f18d22c2fc8104bc698e5d4eceeb60", + "sha256": "965eea22e487b9cc801c6c96730dd297c9f18d22c2fc8104bc698e5d4eceeb60" + }, + { + "path": "src/wrappers/tensor_fallible_generated.rs", + "upstreamSha256": "ecb190d97a1c7004403e60b3774ad45747f9d5207d69d336c087ee39e73a75fc", + "sha256": "ecb190d97a1c7004403e60b3774ad45747f9d5207d69d336c087ee39e73a75fc" + }, + { + "path": "src/wrappers/tensor_generated.rs", + "upstreamSha256": "bd97c25060b73d68bf3c18de04cbea04420df20713b764b085ad989a174bcdbd", + "sha256": "bd97c25060b73d68bf3c18de04cbea04420df20713b764b085ad989a174bcdbd" + }, + { + "path": "src/wrappers/utils.rs", + "upstreamSha256": "121da922d4856db46f5fb2029bc1c11901ab49a4c4e0570c86eddf82eaf7a2bd", + "sha256": "121da922d4856db46f5fb2029bc1c11901ab49a4c4e0570c86eddf82eaf7a2bd" + }, + { + "path": "tests/autocast.rs", + "upstreamSha256": "42f8866420500d6cf9df5162efcd31abc6f6fb6f23988446efce24b75b93b179", + "sha256": "42f8866420500d6cf9df5162efcd31abc6f6fb6f23988446efce24b75b93b179" + }, + { + "path": "tests/create_jit_models.py", + "upstreamSha256": "493e5de4979bc309866d2ca4826f5220612dded6c560db2317cfc5a11092fcd3", + "sha256": "493e5de4979bc309866d2ca4826f5220612dded6c560db2317cfc5a11092fcd3" + }, + { + "path": "tests/data_tests.rs", + "upstreamSha256": "0b0381105f737084b25e114cbb54c39ceb83c23caa4d22c9ed3dfda9d18c7430", + "sha256": "0b0381105f737084b25e114cbb54c39ceb83c23caa4d22c9ed3dfda9d18c7430" + }, + { + "path": "tests/device_tests.rs", + "upstreamSha256": "75cb156293b0b0b8e029957e509bef8226f7d34409821410605a01c9e33f4e10", + "sha256": "75cb156293b0b0b8e029957e509bef8226f7d34409821410605a01c9e33f4e10" + }, + { + "path": "tests/display_tests.rs", + "upstreamSha256": "397c8a9c034c22b7609cfa8d7673a22a8c2c1c389e24af4bc1d61503639e56fc", + "sha256": "397c8a9c034c22b7609cfa8d7673a22a8c2c1c389e24af4bc1d61503639e56fc" + }, + { + "path": "tests/foo.pt", + "upstreamSha256": "1f951cccc226891f6aff1135dca62ba44c40a36683ec1bbc4011405c4eb4aa58", + "sha256": "1f951cccc226891f6aff1135dca62ba44c40a36683ec1bbc4011405c4eb4aa58" + }, + { + "path": "tests/foo1.pt", + "upstreamSha256": "e2d0f81eeb4f85715d994d6b29fd5543f492d49f28d46ad2ed7e755c0eb07970", + "sha256": "e2d0f81eeb4f85715d994d6b29fd5543f492d49f28d46ad2ed7e755c0eb07970" + }, + { + "path": "tests/foo2.pt", + "upstreamSha256": "7054526195c7750e723530868695c4bab77322834a3444a0fef1173d12536286", + "sha256": "7054526195c7750e723530868695c4bab77322834a3444a0fef1173d12536286" + }, + { + "path": "tests/foo3.pt", + "upstreamSha256": "f9e41c74d9447a5e519e3634239134ee7894b8579daf5d1bfe52625cd20b14ab", + "sha256": "f9e41c74d9447a5e519e3634239134ee7894b8579daf5d1bfe52625cd20b14ab" + }, + { + "path": "tests/foo4.pt", + "upstreamSha256": "96098cabdd9c11975d0326cfb1f4a65d98bad10e96b4f930a94be562a01808d2", + "sha256": "96098cabdd9c11975d0326cfb1f4a65d98bad10e96b4f930a94be562a01808d2" + }, + { + "path": "tests/foo5.pt", + "upstreamSha256": "0955c843a9d227ad995190153d4790439ef76c1123cc0a561e49a5f82af0fdf6", + "sha256": "0955c843a9d227ad995190153d4790439ef76c1123cc0a561e49a5f82af0fdf6" + }, + { + "path": "tests/foo6.pt", + "upstreamSha256": "6db11ded23a342b83b117483dab248edb22cbf2d58df3e1953edc599e581e339", + "sha256": "6db11ded23a342b83b117483dab248edb22cbf2d58df3e1953edc599e581e339" + }, + { + "path": "tests/foo7.pt", + "upstreamSha256": "9700f83cc5cdda151d455d36241cac2a78c6ee565790e1332252a7742df6e332", + "sha256": "9700f83cc5cdda151d455d36241cac2a78c6ee565790e1332252a7742df6e332" + }, + { + "path": "tests/foo8.pt", + "upstreamSha256": "75ad1bef7fd6a8dce6310d0ff6baec481f8f28a8974e1531885183ad415c2b03", + "sha256": "75ad1bef7fd6a8dce6310d0ff6baec481f8f28a8974e1531885183ad415c2b03" + }, + { + "path": "tests/jit_tests.rs", + "upstreamSha256": "99318c64f8f63d05796b823f04836ff67e1b0552128b85ac0cb05799dfc82f5a", + "sha256": "99318c64f8f63d05796b823f04836ff67e1b0552128b85ac0cb05799dfc82f5a" + }, + { + "path": "tests/linear/bs.npy", + "upstreamSha256": "1362db3d95c18b4b5a48bd7a9e2c9ec64a293bb13d2869a9b7a78579f448984e", + "sha256": "1362db3d95c18b4b5a48bd7a9e2c9ec64a293bb13d2869a9b7a78579f448984e" + }, + { + "path": "tests/linear/gen-test-data.py", + "upstreamSha256": "defc2eaf96a98e41e96e5ef12d679fbe1539b7a6279600b331d14438f876dd8e", + "sha256": "defc2eaf96a98e41e96e5ef12d679fbe1539b7a6279600b331d14438f876dd8e" + }, + { + "path": "tests/linear/in.npy", + "upstreamSha256": "a30a0baf260237fcb7e31d9759e1cbfd81857baa41fe5e9b6924d9a9f0ffe29e", + "sha256": "a30a0baf260237fcb7e31d9759e1cbfd81857baa41fe5e9b6924d9a9f0ffe29e" + }, + { + "path": "tests/linear/out.npy", + "upstreamSha256": "0f4b8a9fb298b824565611fabc2e96cbc95c8f15823efe4d05ff4c98c9f5055a", + "sha256": "0f4b8a9fb298b824565611fabc2e96cbc95c8f15823efe4d05ff4c98c9f5055a" + }, + { + "path": "tests/linear/ws.npy", + "upstreamSha256": "3aa3fe5195a384602ea414b135360984b2f632698d1a1b5be65dc1667dc9b654", + "sha256": "3aa3fe5195a384602ea414b135360984b2f632698d1a1b5be65dc1667dc9b654" + }, + { + "path": "tests/nn_tests.rs", + "upstreamSha256": "381c7c7fed56e64eaed0011a810683a6e407ac68906c3c2446020b7190f7b93f", + "sha256": "381c7c7fed56e64eaed0011a810683a6e407ac68906c3c2446020b7190f7b93f" + }, + { + "path": "tests/serialization_tests.rs", + "upstreamSha256": "f4ff52054041f6bfe757755fe0589f14935969e16cb2250dffe20be7d84084e6", + "sha256": "f4ff52054041f6bfe757755fe0589f14935969e16cb2250dffe20be7d84084e6" + }, + { + "path": "tests/tensor_indexing.rs", + "upstreamSha256": "b841fd6555b7f05c214cd2cfc74703f07e89d95e61f47aa29164cdfd1cbd0d44", + "sha256": "b841fd6555b7f05c214cd2cfc74703f07e89d95e61f47aa29164cdfd1cbd0d44" + }, + { + "path": "tests/tensor_tests.rs", + "upstreamSha256": "9ab0cc35b177f518d127458968d823df7b0c010bb5fc83245a1a1a2f272627af", + "sha256": "9ab0cc35b177f518d127458968d823df7b0c010bb5fc83245a1a1a2f272627af" + }, + { + "path": "tests/test_utils.rs", + "upstreamSha256": "728c5a88016b7f563acfb58215df2c1be64d17e4c33ce9ff585e7a0ada5d9e20", + "sha256": "728c5a88016b7f563acfb58215df2c1be64d17e4c33ce9ff585e7a0ada5d9e20" + }, + { + "path": "tests/var_store.rs", + "upstreamSha256": "35bc81d5c57e5021062e8f1559dfe930c8042382f3feebccf3e49f23a9002733", + "sha256": "35bc81d5c57e5021062e8f1559dfe930c8042382f3feebccf3e49f23a9002733" + }, + { + "path": "tests/vision_tests.rs", + "upstreamSha256": "b060ec32a2022bbb56240a46a02b7d3c59cb7e7d993ce5be174b00abd2e96267", + "sha256": "b060ec32a2022bbb56240a46a02b7d3c59cb7e7d993ce5be174b00abd2e96267" + }, + { + "path": "uv.lock", + "upstreamSha256": "00db4fd5ee16bd783595b34dc75ae6f10076f41019311cdbc421f182d628f216", + "sha256": "00db4fd5ee16bd783595b34dc75ae6f10076f41019311cdbc421f182d628f216" + } + ] + }, + { + "package": "torch-sys-0.24.0", + "archiveSha256": "f4ba78777379cf09aaa79708c63e477cf0f95e021d04360c6821f1a9f56173f7", + "changedFiles": [ + "Cargo.toml", + "build.rs" + ], + "files": [ + { + "path": "Cargo.toml", + "upstreamSha256": "90b61bb60089252c0444199ae8716bce42ed8438d96354794813c1b93f2b938e", + "sha256": "a1ba42411b026533f14ba4345c6b0b796dee6d651a345a92aaaa6352326d3703" + }, + { + "path": "Cargo.toml.orig", + "upstreamSha256": "178be14ef5df386b02ef91c6e8dfdb25acdf98c33474fd0feb512cf831e01a41", + "sha256": "178be14ef5df386b02ef91c6e8dfdb25acdf98c33474fd0feb512cf831e01a41" + }, + { + "path": "build.rs", + "upstreamSha256": "3b4de10540ab27d2a85e0226100c2a50bff9fe69a5e83b213a8398f6ed9ee8a8", + "sha256": "3d92b1ad6895ca15022c0b7fc84927d0c70810de8c72a6447d5a431b49095212" + }, + { + "path": "libtch/CMakeLists.txt", + "upstreamSha256": "5cbec8dde02e45be4789454d42d35fe7365785e6ed2fdca6499bd844ef4137a1", + "sha256": "5cbec8dde02e45be4789454d42d35fe7365785e6ed2fdca6499bd844ef4137a1" + }, + { + "path": "libtch/stb_image.h", + "upstreamSha256": "599c64066c333d9b125d65f3394de642a42cf4888c3626535dcf62fcfaee1227", + "sha256": "599c64066c333d9b125d65f3394de642a42cf4888c3626535dcf62fcfaee1227" + }, + { + "path": "libtch/stb_image_resize.h", + "upstreamSha256": "5ef7cd32cfbd9de3f87606711fdf8543958302cafc95905f9397d4bee3ad67cb", + "sha256": "5ef7cd32cfbd9de3f87606711fdf8543958302cafc95905f9397d4bee3ad67cb" + }, + { + "path": "libtch/stb_image_write.h", + "upstreamSha256": "dd0ec918c4cc4477a24a2fd268b8a55c3da6767ba50f4e7a9142b9d8e2592487", + "sha256": "dd0ec918c4cc4477a24a2fd268b8a55c3da6767ba50f4e7a9142b9d8e2592487" + }, + { + "path": "libtch/torch_api.cpp", + "upstreamSha256": "3b416a34ff5a6dda9ec6e88ef8e6e8735228e2501ec3c2a01919d93217eaccea", + "sha256": "3b416a34ff5a6dda9ec6e88ef8e6e8735228e2501ec3c2a01919d93217eaccea" + }, + { + "path": "libtch/torch_api.h", + "upstreamSha256": "b5120b143c396a0ed0c1ca19f3cad1dbe2b605351e2ab5eba2b536e17131bb97", + "sha256": "b5120b143c396a0ed0c1ca19f3cad1dbe2b605351e2ab5eba2b536e17131bb97" + }, + { + "path": "libtch/torch_api_generated.cpp", + "upstreamSha256": "029a25dbc9429277ba03c1b516ee4ce78671e8c620597beaf602647fccb13208", + "sha256": "029a25dbc9429277ba03c1b516ee4ce78671e8c620597beaf602647fccb13208" + }, + { + "path": "libtch/torch_api_generated.h", + "upstreamSha256": "0049a691ffb0fe5822385046367af9f9076fe9814c11da2dccbe052ef1e01af6", + "sha256": "0049a691ffb0fe5822385046367af9f9076fe9814c11da2dccbe052ef1e01af6" + }, + { + "path": "libtch/torch_python.cpp", + "upstreamSha256": "fdfd9bb88687fb47e7066a9358af97292a7fdb3c6222c62f6a8b1f554b613b2f", + "sha256": "fdfd9bb88687fb47e7066a9358af97292a7fdb3c6222c62f6a8b1f554b613b2f" + }, + { + "path": "libtch/torch_python.h", + "upstreamSha256": "da330a4c846a15fddc4ab3142ecaa535273724393bad963aa19cf622c5237e2f", + "sha256": "da330a4c846a15fddc4ab3142ecaa535273724393bad963aa19cf622c5237e2f" + }, + { + "path": "src/c_generated.rs", + "upstreamSha256": "8c8581ec7338e28c74fd605dfa07d3fffebbc6a81730904b223d7c5361a4846f", + "sha256": "8c8581ec7338e28c74fd605dfa07d3fffebbc6a81730904b223d7c5361a4846f" + }, + { + "path": "src/cuda.rs", + "upstreamSha256": "5041c0ab989d13aa3a800c4b3070c74d7e587b7b884e66f431a83e5dfcbd699e", + "sha256": "5041c0ab989d13aa3a800c4b3070c74d7e587b7b884e66f431a83e5dfcbd699e" + }, + { + "path": "src/io.rs", + "upstreamSha256": "fcf37f38ce57acddfa4141fd392dcd5a45a106aa4e76344cfd3902909759bcf6", + "sha256": "fcf37f38ce57acddfa4141fd392dcd5a45a106aa4e76344cfd3902909759bcf6" + }, + { + "path": "src/lib.rs", + "upstreamSha256": "3b01047fc37f8f8de98fed728f0492286da49ac346c3ea1457d97597f1ca8ecf", + "sha256": "3b01047fc37f8f8de98fed728f0492286da49ac346c3ea1457d97597f1ca8ecf" + }, + { + "path": "src/python.rs", + "upstreamSha256": "1f11411d6f1f8664f85a48d10957dfbf79667af2ccb9bd548f749cd597df5f8c", + "sha256": "1f11411d6f1f8664f85a48d10957dfbf79667af2ccb9bd548f749cd597df5f8c" + }, + { + "path": "src/traits.rs", + "upstreamSha256": "c986a2595e1ee98545579e9b46f14e0059aab4d7c02554c9d53c47d74af15ae0", + "sha256": "c986a2595e1ee98545579e9b46f14e0059aab4d7c02554c9d53c47d74af15ae0" + } + ] + } + ] +} diff --git a/v2/dependency-patches/tch-0.24.0/.cargo/config.toml b/v2/dependency-patches/tch-0.24.0/.cargo/config.toml new file mode 100644 index 0000000000..e3033cae22 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/.cargo/config.toml @@ -0,0 +1,2 @@ +[target.x86_64-unknown-linux-gnu] +rustflags = ["-C", "link-arg=-Wl,--allow-shlib-undefined", "-C", "link-arg=-fno-lto"] diff --git a/v2/dependency-patches/tch-0.24.0/.github/workflows/rust-ci.yml b/v2/dependency-patches/tch-0.24.0/.github/workflows/rust-ci.yml new file mode 100644 index 0000000000..74e75bf4fa --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/.github/workflows/rust-ci.yml @@ -0,0 +1,74 @@ +on: [push, pull_request] + +name: Continuous integration + +jobs: + check: + name: Check + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + rust: [stable, nightly] + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.rust }} + override: true + - uses: actions-rs/cargo@v1 + with: + command: check + args: --features download-libtorch + + test: + name: Test Suite + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + rust: [stable, nightly] + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: ${{ matrix.rust }} + override: true + - uses: actions-rs/cargo@v1 + with: + command: test + args: --features download-libtorch + + fmt: + name: Rustfmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - run: rustup component add rustfmt + - uses: actions-rs/cargo@v1 + with: + command: fmt + args: --all -- --check + + clippy: + name: Clippy + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: actions-rs/toolchain@v1 + with: + profile: minimal + toolchain: stable + override: true + - run: rustup component add clippy + - uses: actions-rs/cargo@v1 + with: + command: clippy + args: --examples --tests --all-features --all -- -D warnings diff --git a/v2/dependency-patches/tch-0.24.0/.gitignore b/v2/dependency-patches/tch-0.24.0/.gitignore new file mode 100644 index 0000000000..d201b4d7d5 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/.gitignore @@ -0,0 +1,19 @@ +target/ +_build/ +data/ +.idea/ +gen/.merlin +**/*.rs.bk +*.swp +*.swo +Cargo.lock +__pycache__ +*~ +.*~ +.vscode/ + +*.ot +*.safetensors +*.so +*.dylib +llama-tokenizer.json diff --git a/v2/dependency-patches/tch-0.24.0/.python-version b/v2/dependency-patches/tch-0.24.0/.python-version new file mode 100644 index 0000000000..e4fba21835 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/v2/dependency-patches/tch-0.24.0/CHANGELOG.md b/v2/dependency-patches/tch-0.24.0/CHANGELOG.md new file mode 100644 index 0000000000..0e3a534f5d --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/CHANGELOG.md @@ -0,0 +1,143 @@ +# Changelog +This documents the main changes to the `tch` crate. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## Unreleased +### Changed + +## v0.22.0 +### Changed +- PyTorch v2.9 support + +## v0.21.0 +### Changed +- PyTorch v2.8 support + +## v0.20.0 +### Changed +- PyTorch v2.7 support + +## v0.19.0 +### Changed +- PyTorch v2.6 support + +## v0.18.1 +### Changed +- PyTorch v2.5.1 support + +## v0.18.0 +### Changed +- PyTorch v2.5 support + +## v0.17.0 +### Changed +- PyTorch v2.4 support + +## v0.16.0 +### Changed +- PyTorch v2.3 support + +## v0.15.0 +### Changed +- PyTorch v2.2 support + +## v0.14.0 +### Changed +- PyTorch v2.1 support + [803](https://github.com/LaurentMazare/tch-rs/pull/803). +- Add a `pyo3-tch` crate for interacting with Python via PyO3 + [730](https://github.com/LaurentMazare/tch-rs/pull/730). +- Expose the cuda fuser enabled flag, + [728](https://github.com/LaurentMazare/tch-rs/pull/728). +- Improved the safetensor error wrapping, + [720](https://github.com/LaurentMazare/tch-rs/pull/720). + +## v0.13.0 - 2023-05-18 +### Added +- Support static linking in the build script, + [712](https://github.com/LaurentMazare/tch-rs/pull/712). +- Make the libtorch download opt-in rather than a default behavior. The libtorch + library download can still be triggered by enabling the `download-libtorch` + feature, [707](https://github.com/LaurentMazare/tch-rs/pull/707). +- Rename the `of_...` conversion functions to `from_...` so as to be closer to + the Rust best practices, + [706](https://github.com/LaurentMazare/tch-rs/pull/706). This is a breaking + change and will require modifying calls such as `of_slice` to be `from_slice` + instead. +- Expose some functions so that Python extensions that operates on PyTorch + tensors can be written with `tch`, + [704](https://github.com/LaurentMazare/tch-rs/pull/704). +- Rework the torch-sys build script making it easier to leverage a Python + PyTorch install as a source for libtorch, + [703](https://github.com/LaurentMazare/tch-rs/pull/703). + +## v0.12.0 - 2023-05-10 +### Changed +- EfficientNet models have been reworked, pre-trained models used `safetensors` + weight by default, [679](https://github.com/LaurentMazare/tch-rs/pull/679). +- None can be used for nullable scalar types, + [680](https://github.com/LaurentMazare/tch-rs/pull/680). +- Automated conversion of list arguments: all the generated functions that take + as input a slice of int or float can now be used directly with int values or + fixed length arrays [682](https://github.com/LaurentMazare/tch-rs/pull/682). +- Replace the `From` traits with some `TryFrom` versions, + [683](https://github.com/LaurentMazare/tch-rs/pull/683). This is a breaking + change, note that also the old version would flatten the tensor if needed to + reduce the number of dimensions, this has to be done explicitely with the new + version. + +## v0.11.0 - 2023-03-20 +### Added +- Adapt to C++ PyTorch library (`libtorch`) version `v2.0.0`. +### Changed +- Update the `half` dependency to version `2`, [646](https://github.com/LaurentMazare/tch-rs/pull/646). + +## v0.10.3 - 2023-02-23 +### Added +- Add some helper functions in a utils module to check the available devices and versions. + +## v0.10.2 - 2023-02-19 +### Added +- Add the `eps` and `amsgrad` options to Adam, [600](https://github.com/LaurentMazare/tch-rs/pull/600). +### Changed +- Fix loading of `VarStore` when using `Mps` devices, [623](https://github.com/LaurentMazare/tch-rs/pull/623). +- Use `ureq` instead of `curl` to reduce compile times, [620](https://github.com/LaurentMazare/tch-rs/pull/620). +- Fix the handling of dicts in TorchScript, [597](https://github.com/LaurentMazare/tch-rs/issues/597). + +## v0.10.1 - 2022-12-12 +### Changed +- Default `vs.load` to use the Python weight format when the file extension is `.pt` or `.bin`. + +## v0.10.0 - 2022-12-12 +### Added +- Expose functions for setting manual seeds for CUDA devices, [#500](https://github.com/LaurentMazare/tch-rs/pull/500). +- Expose functions for triggering manual sync, [#500](https://github.com/LaurentMazare/tch-rs/pull/500). +- Add some functions to load Python weight files. +### Changed +- Extending the Kaiming initialization, [#573](https://github.com/LaurentMazare/tch-rs/pull/573). + +## v0.9.0 - 2022-11-05 +### Changed +- Adapt to C++ PyTorch library (`libtorch`) version `v1.13.0`. + +## v0.8.0 - 2022-07-04 +### Changed +- Adapt to C++ PyTorch library (`libtorch`) version `v1.12.0`. + +## v0.7.2 - 2022-05-16 +### Changed +- Adapt to C++ PyTorch library (`libtorch`) version `v1.11.0`. + +## v0.6.1 - 2021-10-25 +### Changed +- Adapt to C++ PyTorch library (`libtorch`) version `v1.10.0`. + +## v0.5.0 - 2021-06-25 +### Changed +- Adapt to C++ PyTorch library (`libtorch`) version `v1.9.0`. + +## v0.4.1 - 2021-05-08 +### Changed +- Adapt to C++ PyTorch library (`libtorch`) version `v1.8.1`. diff --git a/v2/dependency-patches/tch-0.24.0/Cargo.toml b/v2/dependency-patches/tch-0.24.0/Cargo.toml new file mode 100644 index 0000000000..c7c18fd6b3 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/Cargo.toml @@ -0,0 +1,246 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +name = "tch" +version = "0.24.0" +authors = ["Laurent Mazare "] +build = "build.rs" +exclude = ["examples/stable-diffusion/media/*"] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Rust wrappers for the PyTorch C++ api (libtorch)." +readme = "README.md" +keywords = [ + "pytorch", + "deep-learning", + "machine-learning", +] +categories = ["science"] +license = "MIT/Apache-2.0" +repository = "https://github.com/LaurentMazare/tch-rs" + +[package.metadata.docs.rs] +features = ["doc-only"] + +[features] +cuda-tests = [] +doc-only = ["torch-sys/doc-only"] +download-libtorch = ["torch-sys/download-libtorch"] +python-extension = ["torch-sys/python-extension"] +rl-python = ["cpython"] + +[lib] +name = "tch" +path = "src/lib.rs" + +[[example]] +name = "basics" +path = "examples/basics.rs" + +[[example]] +name = "char-rnn" +path = "examples/char-rnn/main.rs" + +[[example]] +name = "cifar" +path = "examples/cifar/main.rs" + +[[example]] +name = "custom-optimizer" +path = "examples/custom-optimizer/main.rs" + +[[example]] +name = "gan" +path = "examples/gan/main.rs" + +[[example]] +name = "jit" +path = "examples/jit/main.rs" + +[[example]] +name = "jit-quantized" +path = "examples/jit-quantized/main.rs" + +[[example]] +name = "jit-trace" +path = "examples/jit-trace/main.rs" + +[[example]] +name = "jit-train" +path = "examples/jit-train/main.rs" + +[[example]] +name = "llama" +path = "examples/llama/main.rs" +required-features = [ + "regex", + "clap", + "serde_json", + "memmap2", +] + +[[example]] +name = "memory_test" +path = "examples/memory_test.rs" + +[[example]] +name = "min-gpt" +path = "examples/min-gpt/main.rs" + +[[example]] +name = "mnist" +path = "examples/mnist/main.rs" + +[[example]] +name = "neural-style-transfer" +path = "examples/neural-style-transfer/main.rs" + +[[example]] +name = "pretrained-models" +path = "examples/pretrained-models/main.rs" + +[[example]] +name = "reinforcement-learning" +path = "examples/reinforcement-learning/main.rs" +required-features = ["rl-python"] + +[[example]] +name = "stable-diffusion" +path = "examples/stable-diffusion/main.rs" +required-features = ["regex"] + +[[example]] +name = "tensor-tools" +path = "examples/tensor-tools.rs" + +[[example]] +name = "transfer-learning" +path = "examples/transfer-learning/main.rs" + +[[example]] +name = "translation" +path = "examples/translation/main.rs" + +[[example]] +name = "vae" +path = "examples/vae/main.rs" + +[[example]] +name = "yolo" +path = "examples/yolo/main.rs" + +[[test]] +name = "autocast" +path = "tests/autocast.rs" + +[[test]] +name = "data_tests" +path = "tests/data_tests.rs" + +[[test]] +name = "device_tests" +path = "tests/device_tests.rs" + +[[test]] +name = "display_tests" +path = "tests/display_tests.rs" + +[[test]] +name = "jit_tests" +path = "tests/jit_tests.rs" + +[[test]] +name = "nn_tests" +path = "tests/nn_tests.rs" + +[[test]] +name = "serialization_tests" +path = "tests/serialization_tests.rs" + +[[test]] +name = "tensor_indexing" +path = "tests/tensor_indexing.rs" + +[[test]] +name = "tensor_tests" +path = "tests/tensor_tests.rs" + +[[test]] +name = "test_utils" +path = "tests/test_utils.rs" + +[[test]] +name = "var_store" +path = "tests/var_store.rs" + +[[test]] +name = "vision_tests" +path = "tests/vision_tests.rs" + +[dependencies.clap] +version = "4.2.4" +features = ["derive"] +optional = true + +[dependencies.cpython] +version = "0.7.1" +optional = true + +[dependencies.half] +version = "2" + +[dependencies.image] +version = "0.24.5" +optional = true + +[dependencies.lazy_static] +version = "1.3.0" + +[dependencies.libc] +version = "0.2.0" + +[dependencies.memmap2] +version = "0.6.1" +optional = true + +[dependencies.ndarray] +version = "0.16.1" + +[dependencies.rand] +version = "0.8" + +[dependencies.regex] +version = "1.6.0" +optional = true + +[dependencies.safetensors] +version = "0.3.0" + +[dependencies.serde_json] +version = "1.0.96" +optional = true + +[dependencies.thiserror] +version = "1" + +[dependencies.torch-sys] +version = "0.24.0" + +[dependencies.zip] +version = "9.0.1" + +[dev-dependencies.anyhow] +version = "^1.0.60" diff --git a/v2/dependency-patches/tch-0.24.0/Cargo.toml.orig b/v2/dependency-patches/tch-0.24.0/Cargo.toml.orig new file mode 100644 index 0000000000..e39ee700fd --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/Cargo.toml.orig @@ -0,0 +1,67 @@ +[package] +name = "tch" +version = "0.24.0" +authors = ["Laurent Mazare "] +edition = "2021" +build = "build.rs" + +description = "Rust wrappers for the PyTorch C++ api (libtorch)." +repository = "https://github.com/LaurentMazare/tch-rs" +keywords = ["pytorch", "deep-learning", "machine-learning"] +categories = ["science"] +license = "MIT/Apache-2.0" +readme = "README.md" + +exclude = [ + "examples/stable-diffusion/media/*", +] + +[dependencies] +lazy_static = "1.3.0" +libc = "0.2.0" +ndarray = "0.16.1" +rand = "0.8" +thiserror = "1" +torch-sys = { version = "0.24.0", path = "torch-sys" } +zip = "0.6" +half = "2" +safetensors = "0.3.0" + +cpython = { version = "0.7.1", optional = true } +regex = { version = "1.6.0", optional = true } +image = { version = "0.24.5", optional = true } +clap = { version = "4.2.4", features = ["derive"], optional = true } +serde_json = { version = "1.0.96", optional = true } +memmap2 = { version = "0.6.1", optional = true } + +[dev-dependencies] +anyhow = "^1.0.60" + +[workspace] +members = [ + "torch-sys", + "pyo3-tch", + "examples/python-extension", +] + +[features] +download-libtorch = ["torch-sys/download-libtorch"] +python-extension = ["torch-sys/python-extension"] +rl-python = ["cpython"] +doc-only = ["torch-sys/doc-only"] +cuda-tests = [] + +[package.metadata.docs.rs] +features = [ "doc-only" ] + +[[example]] +name = "reinforcement-learning" +required-features = ["rl-python"] + +[[example]] +name = "stable-diffusion" +required-features = ["regex"] + +[[example]] +name = "llama" +required-features = ["regex", "clap", "serde_json", "memmap2"] diff --git a/v2/dependency-patches/tch-0.24.0/LICENSE-APACHE b/v2/dependency-patches/tch-0.24.0/LICENSE-APACHE new file mode 100644 index 0000000000..261eeb9e9f --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/v2/dependency-patches/tch-0.24.0/LICENSE-MIT b/v2/dependency-patches/tch-0.24.0/LICENSE-MIT new file mode 100644 index 0000000000..31aa79387f --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/LICENSE-MIT @@ -0,0 +1,23 @@ +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/v2/dependency-patches/tch-0.24.0/Makefile b/v2/dependency-patches/tch-0.24.0/Makefile new file mode 100644 index 0000000000..0c06cffb8a --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/Makefile @@ -0,0 +1,13 @@ +test: .FORCE + cargo test + +clean: .FORCE + cargo clean + +gen: .FORCE + dune exec gen/gen.exe + rustfmt src/wrappers/tensor_fallible_generated.rs + rustfmt src/wrappers/tensor_generated.rs + rustfmt torch-sys/src/c_generated.rs + +.FORCE: diff --git a/v2/dependency-patches/tch-0.24.0/README.md b/v2/dependency-patches/tch-0.24.0/README.md new file mode 100644 index 0000000000..777afa57d5 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/README.md @@ -0,0 +1,372 @@ +# tch-rs +Rust bindings for the C++ api of PyTorch. The goal of the `tch` crate is to +provide some thin wrappers around the C++ PyTorch api (a.k.a. libtorch). It +aims at staying as close as possible to the original C++ api. More idiomatic +rust bindings could then be developed on top of this. The +[documentation](https://docs.rs/tch/) can be found on docs.rs. + +[![Build Status](https://github.com/LaurentMazare/tch-rs/workflows/Continuous%20integration/badge.svg)](https://github.com/LaurentMazare/tch-rs/actions) +[![Latest version](https://img.shields.io/crates/v/tch.svg)](https://crates.io/crates/tch) +[![Documentation](https://docs.rs/tch/badge.svg)](https://docs.rs/tch) +[![Dependency Status](https://deps.rs/repo/github/LaurentMazare/tch-rs/status.svg)](https://deps.rs/repo/github/LaurentMazare/tch-rs) +![License](https://img.shields.io/crates/l/tch.svg) +[changelog](https://github.com/LaurentMazare/tch-rs/blob/main/CHANGELOG.md) + + +The code generation part for the C api on top of libtorch comes from +[ocaml-torch](https://github.com/LaurentMazare/ocaml-torch). + +## Getting Started + +This crate requires the C++ PyTorch library (libtorch) in version *v2.11.0* to be available on +your system. You can either: + +- Use the system-wide libtorch installation (default). +- Install libtorch manually and let the build script know about it via the `LIBTORCH` environment variable. +- Use a Python PyTorch install, to do this set `LIBTORCH_USE_PYTORCH=1`. +- When a system-wide libtorch can't be found and `LIBTORCH` is not set, the + build script can download a pre-built binary version of libtorch by using + the `download-libtorch` feature. By default a CPU version is used. The + `TORCH_CUDA_VERSION` environment variable can be set to `cu117` in order to + get a pre-built binary using CUDA 11.7. + +### System-wide Libtorch + +On linux platforms, the build script will look for a system-wide libtorch +library in `/usr/lib/libtorch.so`. + +### Python PyTorch Install + +If the `LIBTORCH_USE_PYTORCH` environment variable is set, the active python +interpreter is called to retrieve information about the torch python package. +This version is then linked against. + +### Libtorch Manual Install + +- Get `libtorch` from the +[PyTorch website download section](https://pytorch.org/get-started/locally/) and extract +the content of the zip file. +- For Linux and macOS users, add the following to your `.bashrc` or equivalent, where `/path/to/libtorch` +is the path to the directory that was created when unzipping the file. +```bash +export LIBTORCH=/path/to/libtorch +``` +The header files location can also be specified separately from the shared library via +the following: +```bash +# LIBTORCH_INCLUDE must contain `include` directory. +export LIBTORCH_INCLUDE=/path/to/libtorch/ +# LIBTORCH_LIB must contain `lib` directory. +export LIBTORCH_LIB=/path/to/libtorch/ +``` +- For Windows users, assuming that `X:\path\to\libtorch` is the unzipped libtorch directory. + - Navigate to Control Panel -> View advanced system settings -> Environment variables. + - Create the `LIBTORCH` variable and set it to `X:\path\to\libtorch`. + - Append `X:\path\to\libtorch\lib` to the `Path` variable. + + If you prefer to temporarily set environment variables, in PowerShell you can run +```powershell +$Env:LIBTORCH = "X:\path\to\libtorch" +$Env:Path += ";X:\path\to\libtorch\lib" +``` +- You should now be able to run some examples, e.g. `cargo run --example basics`. + +### Windows Specific Notes + +As per [the pytorch docs](https://pytorch.org/cppdocs/installing.html) the Windows debug and release builds are not ABI-compatible. This could lead to some segfaults if the incorrect version of libtorch is used. + +It is recommended to use the MSVC Rust toolchain (e.g. by installing `stable-x86_64-pc-windows-msvc` via rustup) rather than a MinGW based one as PyTorch has compatibilities issues with MinGW. + +### Static Linking + +When setting environment variable `LIBTORCH_STATIC=1`, `libtorch` is statically +linked rather than using the dynamic libraries. The pre-compiled artifacts don't +seem to include `libtorch.a` by default so this would have to be compiled +manually, e.g. via the following: + +```bash +git clone -b v2.11.0 --recurse-submodule https://github.com/pytorch/pytorch.git pytorch-static --depth 1 +cd pytorch-static +USE_CUDA=OFF BUILD_SHARED_LIBS=OFF python setup.py build +# export LIBTORCH to point at the build directory in pytorch-static. +``` + +## Examples + +### Basic Tensor Operations + +This crate provides a tensor type which wraps PyTorch tensors. Here is a minimal +example of how to perform some tensor operations. + +```rust +use tch::Tensor; + +fn main() { + let t = Tensor::from_slice(&[3, 1, 4, 1, 5]); + let t = t * 2; + t.print(); +} +``` + +### Training a Model via Gradient Descent + +PyTorch provides automatic differentiation for most tensor operations +it supports. This is commonly used to train models using gradient +descent. The optimization is performed over variables which are created +via a `nn::VarStore` by defining their shapes and initializations. + +In the example below `my_module` uses two variables `x1` and `x2` +which initial values are 0. The forward pass applied to tensor `xs` +returns `xs * x1 + exp(xs) * x2`. + +Once the model has been generated, a `nn::Sgd` optimizer is created. +Then on each step of the training loop: + +- The forward pass is applied to a mini-batch of data. +- A loss is computed as the mean square error between the model output and the mini-batch ground truth. +- Finally an optimization step is performed: gradients are computed and variables from the `VarStore` are modified accordingly. + + +```rust +use tch::nn::{Module, OptimizerConfig}; +use tch::{kind, nn, Device, Tensor}; + +fn my_module(p: nn::Path, dim: i64) -> impl nn::Module { + let x1 = p.zeros("x1", &[dim]); + let x2 = p.zeros("x2", &[dim]); + nn::func(move |xs| xs * &x1 + xs.exp() * &x2) +} + +fn gradient_descent() { + let vs = nn::VarStore::new(Device::Cpu); + let my_module = my_module(vs.root(), 7); + let mut opt = nn::Sgd::default().build(&vs, 1e-2).unwrap(); + for _idx in 1..50 { + // Dummy mini-batches made of zeros. + let xs = Tensor::zeros(&[7], kind::FLOAT_CPU); + let ys = Tensor::zeros(&[7], kind::FLOAT_CPU); + let loss = (my_module.forward(&xs) - ys).pow_tensor_scalar(2).sum(kind::Kind::Float); + opt.backward_step(&loss); + } +} +``` + +### Writing a Simple Neural Network + +The `nn` api can be used to create neural network architectures, e.g. the following code defines +a simple model with one hidden layer and trains it on the MNIST dataset using the Adam optimizer. + +```rust +use anyhow::Result; +use tch::{nn, nn::Module, nn::OptimizerConfig, Device}; + +const IMAGE_DIM: i64 = 784; +const HIDDEN_NODES: i64 = 128; +const LABELS: i64 = 10; + +fn net(vs: &nn::Path) -> impl Module { + nn::seq() + .add(nn::linear( + vs / "layer1", + IMAGE_DIM, + HIDDEN_NODES, + Default::default(), + )) + .add_fn(|xs| xs.relu()) + .add(nn::linear(vs, HIDDEN_NODES, LABELS, Default::default())) +} + +pub fn run() -> Result<()> { + let m = tch::vision::mnist::load_dir("data")?; + let vs = nn::VarStore::new(Device::Cpu); + let net = net(&vs.root()); + let mut opt = nn::Adam::default().build(&vs, 1e-3)?; + for epoch in 1..200 { + let loss = net + .forward(&m.train_images) + .cross_entropy_for_logits(&m.train_labels); + opt.backward_step(&loss); + let test_accuracy = net + .forward(&m.test_images) + .accuracy_for_logits(&m.test_labels); + println!( + "epoch: {:4} train loss: {:8.5} test acc: {:5.2}%", + epoch, + f64::from(&loss), + 100. * f64::from(&test_accuracy), + ); + } + Ok(()) +} +``` + +More details on the training loop can be found in the +[detailed tutorial](https://github.com/LaurentMazare/tch-rs/tree/master/examples/mnist). + +### Using some Pre-Trained Model + +The [pretrained-models example](https://github.com/LaurentMazare/tch-rs/tree/master/examples/pretrained-models/main.rs) +illustrates how to use some pre-trained computer vision model on an image. +The weights - which have been extracted from the PyTorch implementation - can be +downloaded here [resnet18.ot](https://github.com/LaurentMazare/tch-rs/releases/download/mw/resnet18.ot) +and here [resnet34.ot](https://github.com/LaurentMazare/tch-rs/releases/download/mw/resnet34.ot). + +The example can then be run via the following command: +```bash +cargo run --example pretrained-models -- resnet18.ot tiger.jpg +``` +This should print the top 5 imagenet categories for the image. The code for this example is pretty simple. + +```rust + // First the image is loaded and resized to 224x224. + let image = imagenet::load_image_and_resize(image_file)?; + + // A variable store is created to hold the model parameters. + let vs = tch::nn::VarStore::new(tch::Device::Cpu); + + // Then the model is built on this variable store, and the weights are loaded. + let resnet18 = tch::vision::resnet::resnet18(vs.root(), imagenet::CLASS_COUNT); + vs.load(weight_file)?; + + // Apply the forward pass of the model to get the logits and convert them + // to probabilities via a softmax. + let output = resnet18 + .forward_t(&image.unsqueeze(0), /*train=*/ false) + .softmax(-1); + + // Finally print the top 5 categories and their associated probabilities. + for (probability, class) in imagenet::top(&output, 5).iter() { + println!("{:50} {:5.2}%", class, 100.0 * probability) + } +``` +### Importing Pre-Trained Weights from PyTorch Using SafeTensors + +`safetensors` is a new simple format by HuggingFace for storing tensors. It does not rely on Python's `pickle` module, and therefore the tensors are not bound to the specific classes and the exact directory structure used when the model is saved. It is also zero-copy, which means that reading the file will require no more memory than the original file. + +For more information on `safetensors`, please check out https://github.com/huggingface/safetensors + +#### Installing `safetensors` + +You can install `safetensors` via the pip manager: + +``` +pip install safetensors +``` + +#### Exporting weights in PyTorch + +```python +import torchvision +from safetensors import torch as stt + +model = torchvision.models.resnet18(pretrained=True) +stt.save_file(model.state_dict(), 'resnet18.safetensors') +``` + +*Note: the filename of the export must be named with a `.safetensors` suffix for it to be properly decoded by `tch`.* + +#### Importing weights in `tch` + +```rust +use anyhow::Result; +use tch::{ + Device, + Kind, + nn::VarStore, + vision::{ + imagenet, + resnet::resnet18, + } +}; + +fn main() -> Result<()> { + // Create the model and load the pre-trained weights + let mut vs = VarStore::new(Device::cuda_if_available()); + let model = resnet18(&vs.root(), 1000); + vs.load("resnet18.safetensors")?; + + // Load the image file and resize it to the usual imagenet dimension of 224x224. + let image = imagenet::load_image_and_resize224("dog.jpg")? + .to_device(vs.device()); + + // Apply the forward pass of the model to get the logits + let output = image + .unsqueeze(0) + .apply_t(&model, false) + .softmax(-1, Kind::Float); + + // Print the top 5 categories for this image. + for (probability, class) in imagenet::top(&output, 5).iter() { + println!("{:50} {:5.2}%", class, 100.0 * probability) + } + + Ok(()) +} +``` + +Further examples include: +* A simplified version of + [char-rnn](https://github.com/LaurentMazare/tch-rs/blob/master/examples/char-rnn) + illustrating character level language modeling using Recurrent Neural Networks. +* [Neural style transfer](https://github.com/LaurentMazare/tch-rs/blob/master/examples/neural-style-transfer) + uses a pre-trained VGG-16 model to compose an image in the style of another image (pre-trained weights: + [vgg16.ot](https://github.com/LaurentMazare/tch-rs/releases/download/mw/vgg16.ot)). +* Some [ResNet examples on CIFAR-10](https://github.com/LaurentMazare/tch-rs/tree/master/examples/cifar). +* A [tutorial](https://github.com/LaurentMazare/tch-rs/tree/master/examples/jit) + showing how to deploy/run some Python trained models using + [TorchScript JIT](https://pytorch.org/docs/stable/jit.html). +* Some [Reinforcement Learning](https://github.com/LaurentMazare/tch-rs/blob/master/examples/reinforcement-learning) + examples using the [OpenAI Gym](https://github.com/openai/gym) environment. This includes a policy gradient + example as well as an A2C implementation that can run on Atari games. +* A [Transfer Learning Tutorial](https://github.com/LaurentMazare/tch-rs/blob/master/examples/transfer-learning) + shows how to finetune a pre-trained ResNet model on a very small dataset. +* A [simplified version of GPT](https://github.com/LaurentMazare/tch-rs/blob/master/examples/min-gpt) + similar to minGPT. +* A [Stable Diffusion](https://github.com/LaurentMazare/diffusers-rs) + implementation following the lines of hugginface's diffusers library. + +External material: +* A [tutorial](http://vegapit.com/article/how-to-use-torch-in-rust-with-tch-rs) showing how to use Torch to compute option prices and greeks. +* [tchrs-opencv-webcam-inference](https://github.com/metobom/tchrs-opencv-webcam-inference) uses `tch-rs` and `opencv` to run inference + on a webcam feed for some Python trained model based on mobilenet v3. + +## FAQ + +### What are the best practices for Python to Rust model translations? + +See some details in [this thread](https://github.com/LaurentMazare/tch-rs/issues/549#issuecomment-1296840898). + +### How to get this to work on a M1/M2 mac? + +Check this [issue](https://github.com/LaurentMazare/tch-rs/issues/488). + +### Compilation is slow, torch-sys seems to be rebuilt every time cargo gets run. +See this [issue](https://github.com/LaurentMazare/tch-rs/issues/596), this could +be caused by rust-analyzer not knowing about the proper environment variables +like `LIBTORCH` and `LD_LIBRARY_PATH`. + +### Using Rust/tch code from Python. +It is possible to call Rust/tch code from Python via PyO3, +[tch-ext](https://github.com/LaurentMazare/tch-ext) provides an example of such +a Python extension. + +### Error loading shared libraries. + +If you get an error about not finding some shared libraries when running the generated binaries +(e.g. +` error while loading shared libraries: libtorch_cpu.so: cannot open shared object file: No such file or directory`). +You can try adding the following to your `.bashrc` where `/path/to/libtorch` is the path to your +libtorch install. +``` +# For Linux +export LD_LIBRARY_PATH=/path/to/libtorch/lib:$LD_LIBRARY_PATH +# For macOS +export DYLD_LIBRARY_PATH=/path/to/libtorch/lib:$DYLD_LIBRARY_PATH +``` + +## License +`tch-rs` is distributed under the terms of both the MIT license +and the Apache license (version 2.0), at your option. + +See [LICENSE-APACHE](LICENSE-APACHE), [LICENSE-MIT](LICENSE-MIT) for more +details. diff --git a/v2/dependency-patches/tch-0.24.0/build.rs b/v2/dependency-patches/tch-0.24.0/build.rs new file mode 100644 index 0000000000..181d58e23b --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/build.rs @@ -0,0 +1,13 @@ +fn main() { + let os = std::env::var("CARGO_CFG_TARGET_OS").expect("Unable to get TARGET_OS"); + match os.as_str() { + "linux" | "windows" => { + if let Some(lib_path) = std::env::var_os("DEP_TCH_LIBTORCH_LIB") { + println!("cargo:rustc-link-arg=-Wl,-rpath={}", lib_path.to_string_lossy()); + } + println!("cargo:rustc-link-arg=-Wl,--no-as-needed"); + println!("cargo:rustc-link-arg=-ltorch"); + } + _ => {} + } +} diff --git a/v2/dependency-patches/tch-0.24.0/clippy.toml b/v2/dependency-patches/tch-0.24.0/clippy.toml new file mode 100644 index 0000000000..0d4e02f0b0 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/clippy.toml @@ -0,0 +1 @@ +too-many-arguments-threshold = 20 diff --git a/v2/dependency-patches/tch-0.24.0/dune-project b/v2/dependency-patches/tch-0.24.0/dune-project new file mode 100644 index 0000000000..a26d6e2739 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/dune-project @@ -0,0 +1 @@ +(lang dune 1.6) diff --git a/v2/dependency-patches/tch-0.24.0/examples/basics.rs b/v2/dependency-patches/tch-0.24.0/examples/basics.rs new file mode 100644 index 0000000000..469ce73f0c --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/basics.rs @@ -0,0 +1,36 @@ +use tch::{kind, Tensor}; + +fn grad_example() { + let mut x = Tensor::from(2.0).set_requires_grad(true); + let y = &x * &x + &x + 36; + println!("{}", y.double_value(&[])); + x.zero_grad(); + y.backward(); + let mut dy_over_dx = x.grad(); + dy_over_dx.print(); + let t2 = dy_over_dx.clamp_(1.23, 1.24); + dy_over_dx.print(); + t2.print(); + println!("Grad {}", dy_over_dx.double_value(&[])); +} + +fn main() { + println!("Cuda available: {}", tch::Cuda::is_available()); + println!("Cudnn available: {}", tch::Cuda::cudnn_is_available()); + let device = tch::Device::cuda_if_available(); + let t = Tensor::from_slice(&[3, 1, 4, 1, 5]).to(device); + t.print(); + let t = Tensor::randn([5, 4], kind::FLOAT_CPU); + t.print(); + (&t + 1.5).print(); + (&t + 2.5).print(); + let mut t = Tensor::from_slice(&[1.1f32, 2.1, 3.1]); + t += 42; + t.print(); + println!("{:?} {}", t.size(), t.double_value(&[1])); + grad_example(); + println!("has_mps: {}", tch::utils::has_mps()); + println!("has_vulkan: {}", tch::utils::has_vulkan()); + println!("version_cudnn: {}", tch::utils::version_cudnn()); + println!("version_cudart: {}", tch::utils::version_cudart()); +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/char-rnn/README.md b/v2/dependency-patches/tch-0.24.0/examples/char-rnn/README.md new file mode 100644 index 0000000000..764e4b06c7 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/char-rnn/README.md @@ -0,0 +1,43 @@ +This example implements a character-level language model heavily inspired +by [char-rnn](https://github.com/karpathy/char-rnn). +- For training, the model takes as input a text file and learns to predict the + next character following a given sequence. +- For generation, the model uses a fixed seed, returns the next character + distribution from which a character is randomly sampled and this is iterated + to generate a text. + +At the end of each training epoch, some sample text is generated and printed. + +Any text file can be used as an input, as long as it's large enough for training. +A typical example would be the +[tiny Shakespeare dataset](https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt). +The training text file should be stored in `data/input.txt`. + +Compiling and running the example is done via the following command lines: +```bash +cargo run --example char-rnn +``` + +Here is an example of generated data when training on the Shakespeare dataset after only 5 epochs. +``` +CAMILLO: +Nurse! Wast thou, to eat to be proud it. +My lords, he's gone; hers' falsehsof with a temper, +And all these silk of this issuel, or puck +And the new-vangance is danlike to take and turn; +Let him be through my brother and your arries +It sets do so, a knowledge itself have done! + +HENRY BOLINGBROKE: +Come, come, when, sir, along is war, +Horsein your downrial his worshiph, fair sprenched +feignaconal take for thee, dispatch'd with living +Ingracious enching you. Then, great duke +Can klecked found my propter heart, and fees? + +NORTHUMBERLAND: +Have youths of face, I know, that living sons +Through us within the house of the dear inn: +Before thy fire and loving eyes,-- +``` + diff --git a/v2/dependency-patches/tch-0.24.0/examples/char-rnn/main.rs b/v2/dependency-patches/tch-0.24.0/examples/char-rnn/main.rs new file mode 100644 index 0000000000..d9020421a2 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/char-rnn/main.rs @@ -0,0 +1,68 @@ +/* This example uses the tinyshakespeare dataset which can be downloaded at: + https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt + + It has been heavily inspired by https://github.com/karpathy/char-rnn +*/ + +use anyhow::Result; +use tch::data::TextData; +use tch::nn::{Linear, Module, OptimizerConfig, LSTM, RNN}; +use tch::{nn, Device, Kind, Tensor}; + +const LEARNING_RATE: f64 = 0.01; +const HIDDEN_SIZE: i64 = 256; +const SEQ_LEN: i64 = 180; +const BATCH_SIZE: i64 = 256; +const EPOCHS: i64 = 100; +const SAMPLING_LEN: i64 = 1024; + +/// Generates some sample string using the LSTM + Linear model. +fn sample(data: &TextData, lstm: &LSTM, linear: &Linear, device: Device) -> String { + let labels = data.labels(); + let mut state = lstm.zero_state(1); + let mut last_label = 0i64; + let mut result = String::new(); + for _index in 0..SAMPLING_LEN { + let input = Tensor::zeros([1, labels], (Kind::Float, device)); + let _ = input.narrow(1, last_label, 1).fill_(1.0); + state = lstm.step(&input, &state); + let sampled_y = linear + .forward(&state.h()) + .squeeze_dim(0) + .softmax(-1, Kind::Float) + .multinomial(1, false); + last_label = i64::try_from(sampled_y).unwrap(); + result.push(data.label_to_char(last_label)) + } + result +} + +pub fn main() -> Result<()> { + let device = Device::cuda_if_available(); + let vs = nn::VarStore::new(device); + let data = TextData::new("data/input.txt")?; + let labels = data.labels(); + println!("Dataset loaded, {labels} labels."); + let lstm = nn::lstm(vs.root(), labels, HIDDEN_SIZE, Default::default()); + let linear = nn::linear(vs.root(), HIDDEN_SIZE, labels, Default::default()); + let mut opt = nn::Adam::default().build(&vs, LEARNING_RATE)?; + for epoch in 1..(1 + EPOCHS) { + let mut sum_loss = 0.; + let mut cnt_loss = 0.; + for batch in data.iter_shuffle(SEQ_LEN + 1, BATCH_SIZE) { + let xs_onehot = batch.narrow(1, 0, SEQ_LEN).onehot(labels); + let ys = batch.narrow(1, 1, SEQ_LEN).to_kind(Kind::Int64); + let (lstm_out, _) = lstm.seq(&xs_onehot.to_device(device)); + let logits = linear.forward(&lstm_out); + let loss = logits + .view([BATCH_SIZE * SEQ_LEN, labels]) + .cross_entropy_for_logits(&ys.to_device(device).view([BATCH_SIZE * SEQ_LEN])); + opt.backward_step_clip(&loss, 0.5); + sum_loss += f64::try_from(loss)?; + cnt_loss += 1.0; + } + println!("Epoch: {} loss: {:5.3}", epoch, sum_loss / cnt_loss); + println!("Sample: {}", sample(&data, &lstm, &linear, device)); + } + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/cifar/main.rs b/v2/dependency-patches/tch-0.24.0/examples/cifar/main.rs new file mode 100644 index 0000000000..0e39c44fdd --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/cifar/main.rs @@ -0,0 +1,72 @@ +/* Training various models on the CIFAR-10 dataset. + + The dataset can be downloaded from https://www.cs.toronto.edu/~kriz/cifar.html, files + should be placed in the data/ directory. + + The resnet model reaches 95.4% accuracy. +*/ + +use anyhow::Result; +use tch::nn::{FuncT, ModuleT, OptimizerConfig, SequentialT}; +use tch::{nn, Device}; + +fn conv_bn(vs: &nn::Path, c_in: i64, c_out: i64) -> SequentialT { + let conv2d_cfg = nn::ConvConfig { padding: 1, bias: false, ..Default::default() }; + nn::seq_t() + .add(nn::conv2d(vs, c_in, c_out, 3, conv2d_cfg)) + .add(nn::batch_norm2d(vs, c_out, Default::default())) + .add_fn(|x| x.relu()) +} + +fn layer<'a>(vs: &nn::Path, c_in: i64, c_out: i64) -> FuncT<'a> { + let pre = conv_bn(&vs.sub("pre"), c_in, c_out); + let block1 = conv_bn(&vs.sub("b1"), c_out, c_out); + let block2 = conv_bn(&vs.sub("b2"), c_out, c_out); + nn::func_t(move |xs, train| { + let pre = xs.apply_t(&pre, train).max_pool2d_default(2); + let ys = pre.apply_t(&block1, train).apply_t(&block2, train); + pre + ys + }) +} + +fn fast_resnet(vs: &nn::Path) -> SequentialT { + nn::seq_t() + .add(conv_bn(&vs.sub("pre"), 3, 64)) + .add(layer(&vs.sub("layer1"), 64, 128)) + .add(conv_bn(&vs.sub("inter"), 128, 256)) + .add_fn(|x| x.max_pool2d_default(2)) + .add(layer(&vs.sub("layer2"), 256, 512)) + .add_fn(|x| x.max_pool2d_default(4).flat_view()) + .add(nn::linear(vs.sub("linear"), 512, 10, Default::default())) + .add_fn(|x| x * 0.125) +} + +fn learning_rate(epoch: i64) -> f64 { + if epoch < 50 { + 0.1 + } else if epoch < 100 { + 0.01 + } else { + 0.001 + } +} + +pub fn main() -> Result<()> { + let m = tch::vision::cifar::load_dir("data")?; + let vs = nn::VarStore::new(Device::cuda_if_available()); + let net = fast_resnet(&vs.root()); + let mut opt = + nn::Sgd { momentum: 0.9, dampening: 0., wd: 5e-4, nesterov: true }.build(&vs, 0.)?; + for epoch in 1..150 { + opt.set_lr(learning_rate(epoch)); + for (bimages, blabels) in m.train_iter(64).shuffle().to_device(vs.device()) { + let bimages = tch::vision::dataset::augmentation(&bimages, true, 4, 8); + let loss = net.forward_t(&bimages, true).cross_entropy_for_logits(&blabels); + opt.backward_step(&loss); + } + let test_accuracy = + net.batch_accuracy_for_logits(&m.test_images, &m.test_labels, vs.device(), 512); + println!("epoch: {:4} test acc: {:5.2}%", epoch, 100. * test_accuracy,); + } + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/README.md b/v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/README.md new file mode 100644 index 0000000000..aff46c67b3 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/README.md @@ -0,0 +1,21 @@ +## Custom Optimizer - Sparse Adam optimizer + +This example implements an Adam optimizer for sparse and dense gradients. This +is useful for large embedding matrices, where the gradient is very sparse. +Instead of updating the whole embedding, just a small portion of the matrix is +updated and therefore a significant speed-up in training time is gained. + +For the dense update step it uses the `addcdiv_` (fraction and then add assign +in-place) and `addcmul_` (multiply and then add assign in-place) functions and +for the sparse part `index_select` and `index_add` to reduce the necessary +work. The sparse gradient update subroutine is only faster for sparse matrices. +There is a `force_sparse` parameter, which enforces that the sparse subroutine +is used, even for dense gradients. This is only for testing purposes, because +the problem is actual of dense nature. + +As a toy example, a MNIST classification problem is solved with help of the +Adam implementation. For further information take a look at the [python +implementation](https://github.com/pytorch/pytorch/blob/master/torch/optim/sparse_adam.py) +and at [Adam: A Method for Stochastic +Optimization](https://arxiv.org/abs/1412.6980). The Adam optimizer should reach +97% in about 170 epochs. diff --git a/v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/main.rs b/v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/main.rs new file mode 100644 index 0000000000..8e40d27cc4 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/main.rs @@ -0,0 +1,50 @@ +// This should rearch 97% accuracy. + +mod sparse_adam; + +use anyhow::Result; +use tch::{nn, nn::Module, Device}; + +const IMAGE_DIM: i64 = 784; +const HIDDEN_NODES: i64 = 128; +const LABELS: i64 = 10; + +fn net(vs: &nn::Path) -> impl Module { + nn::seq() + .add(nn::linear(vs / "layer1", IMAGE_DIM, HIDDEN_NODES, Default::default())) + .add_fn(|xs| xs.relu()) + .add(nn::linear(vs, HIDDEN_NODES, LABELS, Default::default())) +} + +pub fn run() -> Result<()> { + let m = tch::vision::mnist::load_dir("data")?; + let vs = nn::VarStore::new(Device::Cpu); + let net = net(&vs.root()); + + // force a sparse update step (in order to test on dense problem) + let force_sparse = false; + // create a custom optimizer with learning rate `0.005`, beta_1 `0.9` and beta2 `0.999` + let mut opt = sparse_adam::SparseAdam::new(&vs, 5e-3, 0.9, 0.999, 1e-8, force_sparse); + + for epoch in 1..200 { + let loss = net.forward(&m.train_images).cross_entropy_for_logits(&m.train_labels); + + // call custom optimizer + opt.zero_grad(); + loss.backward(); + opt.step(); + + let test_accuracy = net.forward(&m.test_images).accuracy_for_logits(&m.test_labels); + println!( + "epoch: {:4} train loss: {:8.5} test acc: {:5.2}%", + epoch, + f64::try_from(&loss)?, + 100. * f64::try_from(&test_accuracy)?, + ); + } + Ok(()) +} + +fn main() { + run().unwrap(); +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/sparse_adam.rs b/v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/sparse_adam.rs new file mode 100644 index 0000000000..3e5a88f474 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/sparse_adam.rs @@ -0,0 +1,147 @@ +use std::sync::{Arc, Mutex}; +use tch::nn::{VarStore, Variables}; +use tch::{no_grad, Device, Kind, Tensor}; + +/// Buffer of first/second order moment for the Adam optimizer +struct Buffer { + pub first_moment: Tensor, + pub second_moment: Tensor, + idx: usize, +} + +impl Buffer { + pub fn new(size: &[i64]) -> Buffer { + Buffer { + first_moment: Tensor::zeros(size, (Kind::Float, Device::Cpu)), + second_moment: Tensor::zeros(size, (Kind::Float, Device::Cpu)), + idx: 1, + } + } + + // Return and increment the timestep + pub fn inc(&mut self) -> usize { + let old_val = self.idx; + self.idx += 1; + + old_val + } +} + +/// Sparse-Adam optimizer supporting sparse and dense gradients +/// +/// This demonstrates how a custom optimizer can be implemented. It handles a shared copy of the +/// variables store `vars` and updates it by traversing the computation graph. +pub struct SparseAdam { + lr: f64, + beta1: f64, + beta2: f64, + eps: f64, + force_sparse: bool, + + vars: Arc>, + buffers: Vec, +} + +impl SparseAdam { + pub fn new( + vs: &VarStore, + lr: f64, + beta1: f64, + beta2: f64, + eps: f64, + force_sparse: bool, + ) -> SparseAdam { + let vars = vs.variables_.clone(); + + // create buffers for every trainable variable + let buffers = vars + .lock() + .unwrap() + .trainable_variables + .iter() + .map(|x| Buffer::new(&x.tensor.size())) + .collect(); + + SparseAdam { lr, beta1, beta2, eps, force_sparse, vars, buffers } + } + + /// Ensure that the gradient update is not part of the autograd routine + pub fn step(&mut self) { + no_grad(|| self._step()); + } + + pub fn _step(&mut self) { + let mut vars = self.vars.lock().unwrap(); + + // iterate through all trainable variables + for (var, buffer) in vars.trainable_variables.iter_mut().zip(&mut self.buffers) { + let mut grad = var.tensor.grad(); + + // calculate both bias correction values + let buffer_idx = buffer.inc(); + let bias_correction1 = 1.0 - self.beta1.powf(buffer_idx as f64); + let bias_correction2 = 1.0 - self.beta2.powf(buffer_idx as f64); + + // check whether the gradient is sparse + if grad.is_sparse() || self.force_sparse { + // convert matrix to sparse matrix if necessary + if !grad.is_sparse() { + grad = grad.to_sparse_sparse_dim(1); + } + + // deduplicate coordinates in the sparse matrix + let grad = grad.coalesce(); + // get indices and values of the sparse gradient + let indices = grad.indices().squeeze(); + let values = grad.values(); + + // for SGD we would do: + //tensor.index_add_(0, &indices, &(-self.lr * &values)); + + // update both moments + // old = b*old + (1-b) * new <==> old += (1-b) * (new - old) + let update_first_moment = + (1.0 - self.beta1) * (&values - buffer.first_moment.index_select(0, &indices)); + let update_second_moment = (1.0 - self.beta2) + * (&values * &values - buffer.second_moment.index_select(0, &indices)); + + let _ = buffer.first_moment.index_add_(0, &indices, &update_first_moment); + let _ = buffer.second_moment.index_add_(0, &indices, &update_second_moment); + + // first part of update step -lr * m_t / (1-b_1^t) + let part1 = + buffer.first_moment.index_select(0, &indices) * (-self.lr / bias_correction1); + // second part of update step sqrt(v_t / (1-b_2^t)) + eps + let part2 = (buffer.second_moment.index_select(0, &indices) / bias_correction2) + .sqrt() + + self.eps; + + let _ = var.tensor.index_add_(0, &indices, &(part1 / part2)); + } else { + // update first moment + buffer.first_moment *= self.beta1; + buffer.first_moment += (1.0 - self.beta1) * &grad; + // update second raw moment + buffer.second_moment *= self.beta2; + let scaled_grad = grad * (1.0 - self.beta2).sqrt(); + let _ = buffer.second_moment.addcmul_(&scaled_grad, &scaled_grad); + + // first part of update step -lr * m_t / (1-b_1^t) + let part1 = &buffer.first_moment * (-self.lr / bias_correction1); + // second part of update step sqrt(v_t / (1-b_2^t)) + eps + let part2 = (&buffer.second_moment / bias_correction2).sqrt() + self.eps; + + // calculate fraction and update parameters + let _ = var.tensor.addcdiv_(&part1, &part2); + } + } + } + + // zero the gradient of all trainable variables + pub fn zero_grad(&mut self) { + let mut vars = self.vars.lock().unwrap(); + for var in &mut vars.trainable_variables { + var.tensor.zero_grad(); + } + } +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/gan/main.rs b/v2/dependency-patches/tch-0.24.0/examples/gan/main.rs new file mode 100644 index 0000000000..c0f14f6bd7 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/gan/main.rs @@ -0,0 +1,148 @@ +// Realtivistic DCGAN. +// https://github.com/AlexiaJM/RelativisticGAN +// +// TODO: override the initializations if this does not converge well. +use anyhow::{bail, Result}; +use tch::{kind, nn, nn::OptimizerConfig, Device, Kind, Tensor}; + +const IMG_SIZE: i64 = 64; +const LATENT_DIM: i64 = 128; +const BATCH_SIZE: i64 = 32; +const LEARNING_RATE: f64 = 1e-4; +const BATCHES: i64 = 100000000; + +fn tr2d(p: nn::Path, c_in: i64, c_out: i64, padding: i64, stride: i64) -> nn::ConvTranspose2D { + let cfg = nn::ConvTransposeConfig { stride, padding, bias: false, ..Default::default() }; + nn::conv_transpose2d(p, c_in, c_out, 4, cfg) +} + +fn conv2d(p: nn::Path, c_in: i64, c_out: i64, padding: i64, stride: i64) -> nn::Conv2D { + let cfg = nn::ConvConfig { stride, padding, bias: false, ..Default::default() }; + nn::conv2d(p, c_in, c_out, 4, cfg) +} + +fn generator(p: nn::Path) -> impl nn::ModuleT { + nn::seq_t() + .add(tr2d(&p / "tr1", LATENT_DIM, 1024, 0, 1)) + .add(nn::batch_norm2d(&p / "bn1", 1024, Default::default())) + .add_fn(|xs| xs.relu()) + .add(tr2d(&p / "tr2", 1024, 512, 1, 2)) + .add(nn::batch_norm2d(&p / "bn2", 512, Default::default())) + .add_fn(|xs| xs.relu()) + .add(tr2d(&p / "tr3", 512, 256, 1, 2)) + .add(nn::batch_norm2d(&p / "bn3", 256, Default::default())) + .add_fn(|xs| xs.relu()) + .add(tr2d(&p / "tr4", 256, 128, 1, 2)) + .add(nn::batch_norm2d(&p / "bn4", 128, Default::default())) + .add_fn(|xs| xs.relu()) + .add(tr2d(&p / "tr5", 128, 3, 1, 2)) + .add_fn(|xs| xs.tanh()) +} + +fn leaky_relu(xs: &Tensor) -> Tensor { + xs.maximum(&(xs * 0.2)) +} + +fn discriminator(p: nn::Path) -> impl nn::ModuleT { + nn::seq_t() + .add(conv2d(&p / "conv1", 3, 128, 1, 2)) + .add_fn(leaky_relu) + .add(conv2d(&p / "conv2", 128, 256, 1, 2)) + .add(nn::batch_norm2d(&p / "bn2", 256, Default::default())) + .add_fn(leaky_relu) + .add(conv2d(&p / "conv3", 256, 512, 1, 2)) + .add(nn::batch_norm2d(&p / "bn3", 512, Default::default())) + .add_fn(leaky_relu) + .add(conv2d(&p / "conv4", 512, 1024, 1, 2)) + .add(nn::batch_norm2d(&p / "bn4", 1024, Default::default())) + .add_fn(leaky_relu) + .add(conv2d(&p / "conv5", 1024, 1, 0, 1)) +} + +fn mse_loss(x: &Tensor, y: &Tensor) -> Tensor { + let diff = x - y; + (&diff * &diff).mean(Kind::Float) +} + +// Generate a 2D matrix of images from a tensor with multiple images. +fn image_matrix(imgs: &Tensor, sz: i64) -> Result { + let imgs = ((imgs + 1.) * 127.5).clamp(0., 255.).to_kind(Kind::Uint8); + let mut ys: Vec = vec![]; + for i in 0..sz { + ys.push(Tensor::cat(&(0..sz).map(|j| imgs.narrow(0, 4 * i + j, 1)).collect::>(), 2)) + } + Ok(Tensor::cat(&ys, 3).squeeze_dim(0)) +} + +pub fn main() -> Result<()> { + let device = Device::cuda_if_available(); + let args: Vec<_> = std::env::args().collect(); + let image_dir = match args.as_slice() { + [_, d] => d.to_owned(), + _ => bail!("usage: main image-dataset-dir"), + }; + let images = tch::vision::image::load_dir(image_dir, IMG_SIZE, IMG_SIZE)?; + println!("loaded dataset: {images:?}"); + let train_size = images.size()[0]; + + let random_batch_images = || { + let index = Tensor::randint(train_size, [BATCH_SIZE], kind::INT64_CPU); + images.index_select(0, &index).to_device(device).to_kind(Kind::Float) / 127.5 - 1. + }; + let rand_latent = || { + (Tensor::rand([BATCH_SIZE, LATENT_DIM, 1, 1], kind::FLOAT_CPU) * 2.0 - 1.0) + .to_device(device) + }; + + let mut generator_vs = nn::VarStore::new(device); + let generator = generator(generator_vs.root()); + let mut opt_g = nn::adam(0.5, 0.999, 0.).build(&generator_vs, LEARNING_RATE)?; + + let mut discriminator_vs = nn::VarStore::new(device); + let discriminator = discriminator(discriminator_vs.root()); + let mut opt_d = nn::adam(0.5, 0.999, 0.).build(&discriminator_vs, LEARNING_RATE)?; + + let fixed_noise = rand_latent(); + + for index in 0..BATCHES { + discriminator_vs.unfreeze(); + generator_vs.freeze(); + let discriminator_loss = { + let batch_images = random_batch_images(); + let y_pred = batch_images.apply_t(&discriminator, true); + let y_pred_fake = rand_latent() + .apply_t(&generator, true) + .copy() + .detach() + .apply_t(&discriminator, true); + mse_loss(&y_pred, &(y_pred_fake.mean(Kind::Float) + 1)) + + mse_loss(&y_pred_fake, &(y_pred.mean(Kind::Float) - 1)) + }; + opt_d.backward_step(&discriminator_loss); + + discriminator_vs.freeze(); + generator_vs.unfreeze(); + + let generator_loss = { + let batch_images = random_batch_images(); + let y_pred = batch_images.apply_t(&discriminator, true); + let y_pred_fake = rand_latent().apply_t(&generator, true).apply_t(&discriminator, true); + mse_loss(&y_pred, &(y_pred_fake.mean(Kind::Float) - 1)) + + mse_loss(&y_pred_fake, &(y_pred.mean(Kind::Float) + 1)) + }; + opt_g.backward_step(&generator_loss); + + if index % 1000 == 0 { + let imgs = fixed_noise + .apply_t(&generator, true) + .view([-1, 3, IMG_SIZE, IMG_SIZE]) + .to_device(Device::Cpu); + tch::vision::image::save(&image_matrix(&imgs, 4)?, format!("relout{index}.png"))? + } + if index % 100 == 0 { + println!("{index}") + }; + } + + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit-quantized/README.md b/v2/dependency-patches/tch-0.24.0/examples/jit-quantized/README.md new file mode 100644 index 0000000000..181b1eceba --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/jit-quantized/README.md @@ -0,0 +1,40 @@ +# Loading and Running a Quantized PyTorch Model in Rust + +This builds upon the [main JIT tutorial](https://github.com/LaurentMazare/tch-rs/blob/master/examples/jit/README.md) +to load and run quantized PyTorch models. + +After a model has been successfully trained, they must often be deployed to +resource-constrained devices (such as cheap cloud computing instances or mobile +phones). A common technique used to speed up inference on devices lacking a GPU +is to quantize the model's weight matrices and perform some or all of its operations +using INT8 tensors (rather than FP32). + +## Quantizing a Python PyTorch Model +The details of quantizing a PyTorch model are described in the PyTorch [documentation](https://pytorch.org/docs/stable/quantization.html). In this example we simply export a model available within torchvision. +```python +import torch +import torchvision + +model = torchvision.models.quantization.resnet18( + pretrained=True, quantize=True +) +model.eval() +example = torch.rand(1, 3, 224, 224) +traced_script_module = torch.jit.trace(model, example) +traced_script_module.save("resnet18_int8.pt") +``` + +## Loading a Quantized Torch Script Model from Rust +The process for loading a quantized Torch Script model is almost identical to that +outlined in the [main jit tutorial](https://github.com/LaurentMazare/tch-rs/blob/master/examples/jit/README.md). +The only difference is that we must specify a quantization engine (or backend) to +use during inference. The two engines currently supported by PyTorch are [FBGEMM](https://github.com/pytorch/FBGEMM) +(for inference on x86 architectures) and [QNNPACK](https://github.com/pytorch/QNNPACK) +(for inference on ARM architectures). + +This can be done using the following: +```rust +tch::QEngine::FBGEMM.set()?, +// or +tch::QEngine::QNNPACK.set()?, +``` diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit-quantized/main.rs b/v2/dependency-patches/tch-0.24.0/examples/jit-quantized/main.rs new file mode 100644 index 0000000000..02c349f215 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/jit-quantized/main.rs @@ -0,0 +1,51 @@ +// This example illustrates how to use a quantized PyTorch model trained and +// exported using the Python JIT API. +// See https://pytorch.org/tutorials/advanced/cpp_export.html and +// https://pytorch.org/docs/stable/quantization.html for more details. +use anyhow::{bail, Result}; +use std::time::SystemTime; +use tch::vision::imagenet; + +const NRUNS: i32 = 10; + +pub fn main() -> Result<()> { + let args: Vec<_> = std::env::args().collect(); + let (model_file, image_file, qengine) = match args.as_slice() { + [_, m, i, q] => (m.to_owned(), i.to_owned(), Some(q.to_owned())), + [_, m, i] => (m.to_owned(), i.to_owned(), None), + _ => bail!("usage: main model.pt image.jpg [fbgemm | qnnpack]"), + }; + + // Set quantization engine + match qengine { + None => (), + Some(qengine) => match qengine.as_str() { + "fbgemm" => tch::QEngine::FBGEMM.set()?, + "qnnpack" => tch::QEngine::QNNPACK.set()?, + _ => bail!("qengine should be one of 'fbgemm' or 'qnnpack' or ommitted"), + }, + }; + + // Load the image file and resize it to the usual imagenet dimension of 224x224. + let image = imagenet::load_image_and_resize224(image_file)?; + + // Load the Python saved module. + let model = tch::CModule::load(model_file)?; + + // Measure the average inference time. + let now = SystemTime::now(); + for _ in 1..NRUNS { + let _output = image.unsqueeze(0).apply(&model); + } + println!("Mean Inference Time: {} ms", now.elapsed().unwrap().as_millis() / NRUNS as u128); + + // Apply the forward pass of the model to get the logits. + let output = image.unsqueeze(0).apply(&model).softmax(-1, tch::Kind::Float); + + // Print the top 5 categories for this image. + println!("Top 5 Predictions:"); + for (probability, class) in imagenet::top(&output, 5).iter() { + println!("{:50} {:5.2}%", class, 100.0 * probability) + } + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit-quantized/resnet.py b/v2/dependency-patches/tch-0.24.0/examples/jit-quantized/resnet.py new file mode 100644 index 0000000000..4b3f79ba67 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/jit-quantized/resnet.py @@ -0,0 +1,22 @@ +import torch +import torchvision + + +def save_resnet18_script_module( + filename: str, pretrained: bool, quantize: bool +) -> None: + """Save quantizable resnet18 as traced script module.""" + model = torchvision.models.quantization.resnet18( + pretrained=pretrained, quantize=quantize + ) + model.eval() + example = torch.rand(1, 3, 224, 224) + traced_script_module = torch.jit.trace(model, example) + traced_script_module.save(filename) + + +# Save fp32 model +save_resnet18_script_module("resnet18_fp32.pt", True, False) + +# Save int8 model +save_resnet18_script_module("resnet18_int8.pt", True, True) diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit-trace/README.md b/v2/dependency-patches/tch-0.24.0/examples/jit-trace/README.md new file mode 100644 index 0000000000..1dd829dc63 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/jit-trace/README.md @@ -0,0 +1,27 @@ +To run this example, you will have to unzip the [MNIST data +files](http://yann.lecun.com/exdb/mnist/) in `data/`. + +# Saving a trained model by tracing + +This tutorial illustrates how to serialize a model trained in Rust as a TorchScript program. + +## Training and serializing a TorchScript program in Rust. + +By running the following command, you can train a model on the MNIST dataset and serialize it as a TorchScript program in Rust. + +```bash +cargo run --example jit-trace +``` + +It will save the model as `model.pt`. + +## Loading the model in Python + +You can load and run the saved model in Python. +`test.py` is an example Python script to load the model from `model.pt` and run it on the MNIST dataset. + +You can run the script by the following command. +Please ensure that `torch` and `torchvision` are installed in your environment before running. +```bash +python ./examples/jit-trace/test.py +``` diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit-trace/main.rs b/v2/dependency-patches/tch-0.24.0/examples/jit-trace/main.rs new file mode 100644 index 0000000000..d27dddf113 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/jit-trace/main.rs @@ -0,0 +1,63 @@ +use anyhow::Result; +use tch::{kind::FLOAT_CUDA, nn, nn::ModuleT, nn::OptimizerConfig, CModule, Device, Tensor}; + +#[derive(Debug)] +struct Net { + conv1: nn::Conv2D, + conv2: nn::Conv2D, + fc1: nn::Linear, + fc2: nn::Linear, +} + +impl Net { + fn new(vs: &nn::Path) -> Net { + let conv1 = nn::conv2d(vs, 1, 32, 5, Default::default()); + let conv2 = nn::conv2d(vs, 32, 64, 5, Default::default()); + let fc1 = nn::linear(vs, 1024, 1024, Default::default()); + let fc2 = nn::linear(vs, 1024, 10, Default::default()); + Net { conv1, conv2, fc1, fc2 } + } +} + +impl nn::ModuleT for Net { + fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor { + xs.view([-1, 1, 28, 28]) + .apply(&self.conv1) + .max_pool2d_default(2) + .apply(&self.conv2) + .max_pool2d_default(2) + .view([-1, 1024]) + .apply(&self.fc1) + .relu() + .dropout(0.5, train) + .apply(&self.fc2) + } +} + +fn main() -> Result<()> { + let m = tch::vision::mnist::load_dir("data")?; + let mut vs = nn::VarStore::new(Device::cuda_if_available()); + let net = Net::new(&vs.root()); + let mut opt = nn::Adam::default().build(&vs, 1e-4)?; + for epoch in 1..10 { + for (bimages, blabels) in m.train_iter(256).shuffle().to_device(vs.device()) { + let loss = net.forward_t(&bimages, true).cross_entropy_for_logits(&blabels); + opt.backward_step(&loss); + } + let test_accuracy = + net.batch_accuracy_for_logits(&m.test_images, &m.test_labels, vs.device(), 1024); + println!("epoch: {:4} test acc: {:5.2}%", epoch, 100. * test_accuracy,); + } + + vs.freeze(); + let mut closure = |input: &[Tensor]| vec![net.forward_t(&input[0], false)]; + let model = CModule::create_by_tracing( + "MyModule", + "forward", + &[Tensor::zeros([784], FLOAT_CUDA)], + &mut closure, + )?; + model.save("model.pt")?; + + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit-trace/test.py b/v2/dependency-patches/tch-0.24.0/examples/jit-trace/test.py new file mode 100644 index 0000000000..497187deee --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/jit-trace/test.py @@ -0,0 +1,45 @@ +# The following code is partially copied from the official example. +# https://github.com/pytorch/examples/blob/ca1bd9167f7216e087532160fc5b98643d53f87e/mnist/main.py + +import torch +from torchvision import datasets, transforms +import torch.nn.functional as F + + +def test(model, device, test_loader): + model.eval() + test_loss = 0 + correct = 0 + with torch.no_grad(): + for data, target in test_loader: + data, target = data.to(device), target.to(device) + output = model(data) + test_loss += F.nll_loss( + output, target, reduction="sum" + ).item() # sum up batch loss + pred = output.argmax( + dim=1, keepdim=True + ) # get the index of the max log-probability + correct += pred.eq(target.view_as(pred)).sum().item() + + test_loss /= len(test_loader.dataset) + + print( + "\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n".format( + test_loss, + correct, + len(test_loader.dataset), + 100.0 * correct / len(test_loader.dataset), + ) + ) + + +model = torch.jit.load("model.pt") +device = torch.device("cuda") +test_kwargs = {"batch_size": 100} +transform = transforms.Compose( + [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))] +) +dataset = datasets.MNIST("./data", train=False, download=True, transform=transform) +test_loader = torch.utils.data.DataLoader(dataset, **test_kwargs) +test(model, device, test_loader) diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit-train/README.md b/v2/dependency-patches/tch-0.24.0/examples/jit-train/README.md new file mode 100644 index 0000000000..a07f9ea14a --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/jit-train/README.md @@ -0,0 +1,136 @@ +To run this example, you will have to unzip the [MNIST data +files](http://yann.lecun.com/exdb/mnist/) in `data/`. + +# Loading and Training a PyTorch Model in Rust + +In this tutorial this is illustrated using a demo model that has not been pre-trained. +We start by defining and serializing the model using the Python api. +The resulting model file is later loaded from Rust and trained on the MNIST dataset. + +## Converting a Python PyTorch Model to Torch Script + +There are various ways to create the Torch Script as detailed +in the original [tutorial](https://pytorch.org/tutorials/advanced/cpp_export.html). + +Here we will use scripting. The following Python script runs the +defined model on some random image and creates the Torch Script +file based on this evaluation. + +```python +import torch +from torch.nn import Module + + +class DemoModule(Module): + def __init__(self): + super().__init__() + self.batch_norm = torch.nn.BatchNorm2d(1) + self.conv1 = torch.nn.Conv2d(1, 8, kernel_size=(5, 5), padding=(2, 2)) + self.conv2 = torch.nn.Conv2d(8, 16, kernel_size=(5, 5), padding=(2, 2)) + self.flatten = torch.nn.Flatten() + self.dropout = torch.nn.Dropout() + self.linear1 = torch.nn.Linear(16 * 28 * 28, 100) + self.linear2 = torch.nn.Linear(100, 10) + + def forward(self, x): + x = self.batch_norm(x) + x = self.conv1(x) + x = self.conv2(x) + x = self.flatten(x) + x = self.dropout(x) + x = self.linear1(x) + return self.linear2(x) + + +traced_script_module = torch.jit.script(DemoModule()) +traced_script_module.save("model.pt") +``` + +The last line creates the `model.pt` Torch Script file which includes both the model +architecture and the weight values. + +## Loading the Torch Script Model from Rust + +The `model.pt` file can then be loaded and trained from Rust. + +```rust +fn train_and_save_model(dataset: &Dataset, device: Device) -> Result<()> { + let vs = VarStore::new(device); + + let mut trainable = TrainableCModule::load("model.pt", vs.root())?; + trainable.set_train(); + let initial_acc = trainable.batch_accuracy_for_logits( + &dataset.test_images, + &dataset.test_labels, + vs.device(), + 1024, + ); + println!("Initial accuracy: {:5.2}%", 100. * initial_acc); + + let mut opt = Adam::default().build(&vs, 1e-4)?; + for epoch in 1..20 { + for (images, labels) in dataset + .train_iter(128) + .shuffle() + .to_device(vs.device()) + .take(50) + { + let loss = trainable + .forward_t(&images, true) + .cross_entropy_for_logits(&labels); + opt.backward_step(&loss); + } + let test_accuracy = trainable.batch_accuracy_for_logits( + &dataset.test_images, + &dataset.test_labels, + vs.device(), + 1024, + ); + println!("epoch: {:4} test acc: {:5.2}%", epoch, 100. * test_accuracy,); + } + + trainable.save("trained_model.pt")?; + Ok(()) +} +``` + +After the model was trained, the updated Torch Script is saved to file `trained_model.pt`. +This can be used for executing the module later on either via Python or +using Rust as per the following. + +```rust +fn load_trained_and_test_acc(dataset: &Dataset, device: Device) -> Result<()> { + let mut module = CModule::load_on_device("trained_model.pt", device)?; + module.set_eval(); + let accuracy = + module.batch_accuracy_for_logits(&dataset.test_images, &dataset.test_labels, device, 1024); + println!("Updated accuracy: {:5.2}%", 100. * accuracy); + Ok(()) +} +``` + +Below is the typical output that this example would produce. + +``` +Initial accuracy: 13.11% +epoch: 1 test acc: 87.11% +epoch: 2 test acc: 90.16% +epoch: 3 test acc: 90.31% +epoch: 4 test acc: 90.02% +epoch: 5 test acc: 90.66% +epoch: 6 test acc: 91.00% +epoch: 7 test acc: 91.48% +epoch: 8 test acc: 91.10% +epoch: 9 test acc: 91.13% +epoch: 10 test acc: 91.35% +epoch: 11 test acc: 91.04% +epoch: 12 test acc: 91.43% +epoch: 13 test acc: 91.34% +epoch: 14 test acc: 91.05% +epoch: 15 test acc: 91.23% +epoch: 16 test acc: 91.30% +epoch: 17 test acc: 91.45% +epoch: 18 test acc: 91.70% +epoch: 19 test acc: 91.44% +Updated accuracy: 91.93% +``` diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit-train/main.rs b/v2/dependency-patches/tch-0.24.0/examples/jit-train/main.rs new file mode 100644 index 0000000000..8e7f9e3761 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/jit-train/main.rs @@ -0,0 +1,57 @@ +use anyhow::Result; +use tch::nn::{Adam, ModuleT, OptimizerConfig, VarStore}; +use tch::vision::dataset::Dataset; +use tch::TrainableCModule; +use tch::{CModule, Device}; + +fn train_and_save_model(dataset: &Dataset, device: Device) -> Result<()> { + let vs = VarStore::new(device); + let mut trainable = TrainableCModule::load("model.pt", vs.root())?; + trainable.set_train(); + let initial_acc = trainable.batch_accuracy_for_logits( + &dataset.test_images, + &dataset.test_labels, + vs.device(), + 1024, + ); + println!("Initial accuracy: {:5.2}%", 100. * initial_acc); + + let mut opt = Adam::default().build(&vs, 1e-4)?; + for epoch in 1..20 { + for (images, labels) in dataset.train_iter(128).shuffle().to_device(vs.device()).take(50) { + let loss = trainable.forward_t(&images, true).cross_entropy_for_logits(&labels); + opt.backward_step(&loss); + } + let test_accuracy = trainable.batch_accuracy_for_logits( + &dataset.test_images, + &dataset.test_labels, + vs.device(), + 1024, + ); + println!("epoch: {:4} test acc: {:5.2}%", epoch, 100. * test_accuracy,); + } + trainable.save("trained_model.pt")?; + Ok(()) +} + +fn load_trained_and_test_acc(dataset: &Dataset, device: Device) -> Result<()> { + let mut module = CModule::load_on_device("trained_model.pt", device)?; + module.set_eval(); + let accuracy = + module.batch_accuracy_for_logits(&dataset.test_images, &dataset.test_labels, device, 1024); + println!("Updated accuracy: {:5.2}%", 100. * accuracy); + Ok(()) +} + +fn main() -> Result<()> { + let dataset = tch::vision::mnist::load_dir("data")?; + let dataset = Dataset { + train_images: dataset.train_images.view([-1, 1, 28, 28]), + test_images: dataset.test_images.view([-1, 1, 28, 28]), + ..dataset + }; + let device = Device::Cpu; + train_and_save_model(&dataset, device)?; + load_trained_and_test_acc(&dataset, device)?; + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit-train/resnet.py b/v2/dependency-patches/tch-0.24.0/examples/jit-train/resnet.py new file mode 100644 index 0000000000..fe389d8127 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/jit-train/resnet.py @@ -0,0 +1,27 @@ +import torch +from torch.nn import Module + + +class DemoModule(Module): + def __init__(self): + super().__init__() + self.batch_norm = torch.nn.BatchNorm2d(1) + self.conv1 = torch.nn.Conv2d(1, 8, kernel_size=(5, 5), padding=(2, 2)) + self.conv2 = torch.nn.Conv2d(8, 16, kernel_size=(5, 5), padding=(2, 2)) + self.flatten = torch.nn.Flatten() + self.dropout = torch.nn.Dropout() + self.linear1 = torch.nn.Linear(16 * 28 * 28, 100) + self.linear2 = torch.nn.Linear(100, 10) + + def forward(self, x): + x = self.batch_norm(x) + x = self.conv1(x) + x = self.conv2(x) + x = self.flatten(x) + x = self.dropout(x) + x = self.linear1(x) + return self.linear2(x) + + +traced_script_module = torch.jit.script(DemoModule()) +traced_script_module.save("model.pt") diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit/README.md b/v2/dependency-patches/tch-0.24.0/examples/jit/README.md new file mode 100644 index 0000000000..9e2ea5bf36 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/jit/README.md @@ -0,0 +1,116 @@ +# Loading and Running a PyTorch Model in Rust + +This tutorial follows the steps of the +[Loading a PyTorch Model in C++ tutorial](https://pytorch.org/tutorials/advanced/cpp_export.html). + +PyTorch models are commonly written and trained in Python. The trained model can then be +serialized in a [Torch Script](https://pytorch.org/docs/stable/jit.html) file. +The Torch Script file contains a description of the model architecture as well as +trained weights. This file can be loaded from Rust to run inference for the saved +model. + +In this tutorial this is illustrated using a ResNet-18 model that has been trained on the +ImageNet dataset. We start by loading and serializing the model using the Python api. +The resulting model file is later loaded from Rust and run on some given image. + +## Converting a Python PyTorch Model to Torch Script + +There are various ways to create the Torch Script as detailed +in the original [tutorial](https://pytorch.org/tutorials/advanced/cpp_export.html). + +Here we will use tracing. The following python script runs the +pre-trained ResNet-18 model on some random image and uses tracing to create +the Torch Script file based on this evaluation. + +```python +import torch +import torchvision + +model = torchvision.models.resnet18(pretrained=True) +model.eval() +example = torch.rand(1, 3, 224, 224) +traced_script_module = torch.jit.trace(model, example) +traced_script_module.save("model.pt") +``` + +Note that `model.eval()` is useful to ensure that the saved model is +in testing mode rather than in training mode. This has an impact on the +batch-norm layers. + +The last line creates the `model.pt` Torch Script file which includes both the model +architecture and the trained weight values. + +## Loading the Torch Script Model from Rust + +The `model.pt` file can then be loaded and executed from Rust. + +```rust +pub fn main() -> anyhow::Result<()> { + let args: Vec<_> = std::env::args().collect(); + let (model_file, image_file) = match args.as_slice() { + [_, m, i] => (m.to_owned(), i.to_owned()), + _ => bail!("usage: main model.pt image.jpg"), + }; + let image = imagenet::load_image_and_resize(image_file)?; + let model = tch::CModule::load(model_file)?; + let output = model.forward_ts(&[image.unsqueeze(0)])?.softmax(-1); + for (probability, class) in imagenet::top(&output, 5).iter() { + println!("{:50} {:5.2}%", class, 100.0 * probability) + } + Ok(()) +} +``` + +Let us have a closer look at what this code is doing. The first couple lines +extract the model and image filenames from command line arguments +Then the image is loaded, resized to 224x224, and converted to a tensor +using ImageNet normalization. + +```rust + let image = imagenet::load_image_and_resize(image_file)?; +``` + +The exported model is loaded. + +```rust + let model = tch::CModule::load(model_file)?; +``` + +Now we can run the model on the image tensor. This returns the logits for each +of the ImageNet 1000 classes. A softmax is applied to get the associated +probabilities. + +```rust + let output = model.forward_ts(&[image.unsqueeze(0)])?.softmax(-1); +``` + +Alternatively, one can write the following instead as `tch::CModule` can be +used as any other module via apply when there is only a single input. +```rust + let output = image.unsqueeze(0).apply(&model).softmax(-1); +``` + +And finally we print the 5 classes with the highest probabilities. +```rust + for (probability, class) in imagenet::top(&output, 5).iter() { + println!("{:50} {:5.2}%", class, 100.0 * probability) + } +``` + +Cargo can be used to run this example. +```bash +cargo run --example jit model.pt image.jpg +``` + +This results in the Rust code printing the top 5 predicted labels +as well as the associated probabilities. + +``` +tiger, Panthera tigris 96.33% +tiger cat 3.56% +zebra 0.09% +jaguar, panther, Panthera onca, Felis onca 0.01% +tabby, tabby cat 0.01% +``` + +![tiger](https://upload.wikimedia.org/wikipedia/commons/thumb/3/3b/Royal_Bengal_Tiger_at_Kanha_National_Park.jpg/800px-Royal_Bengal_Tiger_at_Kanha_National_Park.jpg) diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit/main.rs b/v2/dependency-patches/tch-0.24.0/examples/jit/main.rs new file mode 100644 index 0000000000..fb8e2e79d3 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/jit/main.rs @@ -0,0 +1,27 @@ +// This example illustrates how to use a PyTorch model trained and exported using the +// Python JIT API. +// See https://pytorch.org/tutorials/advanced/cpp_export.html for more details. +use anyhow::{bail, Result}; +use tch::vision::imagenet; + +pub fn main() -> Result<()> { + let args: Vec<_> = std::env::args().collect(); + let (model_file, image_file) = match args.as_slice() { + [_, m, i] => (m.to_owned(), i.to_owned()), + _ => bail!("usage: main model.pt image.jpg"), + }; + // Load the image file and resize it to the usual imagenet dimension of 224x224. + let image = imagenet::load_image_and_resize224(image_file)?; + + // Load the Python saved module. + let model = tch::CModule::load(model_file)?; + + // Apply the forward pass of the model to get the logits. + let output = image.unsqueeze(0).apply(&model).softmax(-1, tch::Kind::Float); + + // Print the top 5 categories for this image. + for (probability, class) in imagenet::top(&output, 5).iter() { + println!("{:50} {:5.2}%", class, 100.0 * probability) + } + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit/resnet.py b/v2/dependency-patches/tch-0.24.0/examples/jit/resnet.py new file mode 100644 index 0000000000..eaaa834f80 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/jit/resnet.py @@ -0,0 +1,8 @@ +import torch +import torchvision + +model = torchvision.models.resnet18(pretrained=True) +model.eval() +example = torch.rand(1, 3, 224, 224) +traced_script_module = torch.jit.trace(model, example) +traced_script_module.save("model.pt") diff --git a/v2/dependency-patches/tch-0.24.0/examples/llama/convert_checkpoint.py b/v2/dependency-patches/tch-0.24.0/examples/llama/convert_checkpoint.py new file mode 100644 index 0000000000..423034304f --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/llama/convert_checkpoint.py @@ -0,0 +1,63 @@ +# Adapted from https://github.com/Lightning-AI/lit-llama/blob/main/scripts/convert_checkpoint.py +import sys +import torch +from typing import Dict +from pathlib import Path +from safetensors.torch import save_file + +def convert_state_dict(state_dict: Dict[str, torch.Tensor], dtype: torch.dtype = torch.float16) -> Dict[str, torch.Tensor]: + print("start conv") + + def get_and_remove(key): + v = state_dict[key].to(dtype) + del state_dict[key] + return v + + converted = {} + converted["transformer.wte.weight"] = get_and_remove("tok_embeddings.weight") + converted["lm_head.weight"] = get_and_remove("output.weight") + converted["transformer.ln_f.scale"] = get_and_remove("norm.weight") + + for layer_idx in sorted(set([k.split(".")[1] for k in state_dict if k.startswith("layers")])): + print(layer_idx) + + # attention + # the wq, wk, wv from the FB model are stacked in our model as c_attn + converted[f"transformer.h.{layer_idx}.attn.c_attn.weight"] = torch.cat( + ( + get_and_remove(f"layers.{layer_idx}.attention.wq.weight"), + get_and_remove(f"layers.{layer_idx}.attention.wk.weight"), + get_and_remove(f"layers.{layer_idx}.attention.wv.weight"), + ) + ) + converted[f"transformer.h.{layer_idx}.attn.c_proj.weight"] = get_and_remove( + f"layers.{layer_idx}.attention.wo.weight" + ) + # mlp + converted[f"transformer.h.{layer_idx}.mlp.c_fc1.weight"] = get_and_remove( + f"layers.{layer_idx}.feed_forward.w1.weight" + ) + converted[f"transformer.h.{layer_idx}.mlp.c_proj.weight"] = get_and_remove( + f"layers.{layer_idx}.feed_forward.w2.weight" + ) + converted[f"transformer.h.{layer_idx}.mlp.c_fc2.weight"] = get_and_remove( + f"layers.{layer_idx}.feed_forward.w3.weight" + ) + # rms norm + converted[f"transformer.h.{layer_idx}.rms_1.scale"] = get_and_remove(f"layers.{layer_idx}.attention_norm.weight") + converted[f"transformer.h.{layer_idx}.rms_2.scale"] = get_and_remove(f"layers.{layer_idx}.ffn_norm.weight") + return converted + +def convert_weights(llama_ckpt, *, output_st: Path = Path("llama.safetensors"), dtype: str = "float16") -> None: + dt = getattr(torch, dtype, None) + if not isinstance(dt, torch.dtype): + raise ValueError(f"{dtype} is not a valid dtype.") + checkpoint = torch.load(llama_ckpt, map_location="cpu") + converted = convert_state_dict(checkpoint, dtype=dt) + del checkpoint + save_file(converted, output_st) + +if __name__ == "__main__": + if len(sys.argv) != 2: + raise ValueError(f"usage: convert_checkpoint.py ..../LLaMA/7B/consolidated.00.pth") + convert_weights(sys.argv[1]) diff --git a/v2/dependency-patches/tch-0.24.0/examples/llama/main.rs b/v2/dependency-patches/tch-0.24.0/examples/llama/main.rs new file mode 100644 index 0000000000..3303859c06 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/llama/main.rs @@ -0,0 +1,397 @@ +// An implementation of LLaMA https://github.com/facebookresearch/llama +// This only contains the inference part. +// +// This is based on nanoGPT in a similar way to: +// https://github.com/Lightning-AI/lit-llama/blob/main/lit_llama/model.py +// +// The tokenizer config can be retrieved from: +// https://huggingface.co/hf-internal-testing/llama-tokenizer/blob/main/tokenizer.json +// +// In order to convert the llama weights to a .safetensors file, run: +// python examples/llama/convert_checkpoint.py ..../LLaMA/7B/consolidated.00.pth +use anyhow::Result; +use clap::Parser; +use tch::nn::{self, Module}; +use tch::{Device, Kind, Tensor}; + +mod sentencepiece; +use sentencepiece::Tokenizer; + +const CONTEXT_SIZE: usize = 512; +const START_PROMPT: &str = r" +EDWARD: +I wonder how our princely father 'scaped, +Or whether he be 'scaped away or no +From Clifford's and Northumberland's pursuit: +Had he been ta'en, we should have heard the news; +Had he been slain, we should have heard the news; +Or had he 'scaped, methinks we should have heard +The happy tidings of his good escape. +How fares my brother? why is he so sad? + +RICHARD: +I cannot joy, until I be resolved +Where our right valiant father is become. +I saw him in the battle range about; +And watch'd him how he singled Clifford forth. +Methought he bore him in the thickest troop +As doth a lion in a herd of neat; +Or as a bear, encompass'd round with dogs, +Who having pinch'd a few and made them cry, +The rest stand all aloof, and bark at him. +So fared our father with his enemies; +So fled his enemies my warlike father: +Methinks, 'tis prize enough to be his son. +See how the morning opes her golden gates, +And takes her farewell of the glorious sun! +How well resembles it the prime of youth, +Trimm'd like a younker prancing to his love! + +EDWARD: +Dazzle mine eyes, or do I see three suns? + +RICHARD: +Three glorious suns, each one a perfect sun; +Not separated with the racking clouds, +But sever'd in a pale clear-shining sky. +See, see! they join, embrace, and seem to kiss, +As if they vow'd some league inviolable: +Now are they but one lamp, one light, one sun. +In this the heaven figures some event. + +EDWARD: +'Tis wondrous strange, the like yet never heard of. +I think it cites us, brother, to the field, +That we, the sons of brave Plantagenet, +Each one already blazing by our meeds, +Should notwithstanding join our lights together +And over-shine the earth as this the world. +Whate'er it bodes, henceforward will I bear +Upon my target three fair-shining suns. +"; + +#[allow(dead_code)] +struct Config { + block_size: usize, + vocab_size: usize, + n_layer: usize, + n_head: usize, + n_embd: usize, +} + +#[allow(dead_code)] +impl Config { + fn config_7b() -> Self { + Self { block_size: 4096, vocab_size: 32000, n_layer: 32, n_head: 32, n_embd: 4096 } + } + + fn config_13b() -> Self { + Self { block_size: 4096, vocab_size: 32000, n_layer: 40, n_head: 40, n_embd: 5120 } + } + + fn config_30b() -> Self { + Self { block_size: 4096, vocab_size: 32000, n_layer: 60, n_head: 52, n_embd: 6656 } + } + + fn config_65b() -> Self { + Self { block_size: 4096, vocab_size: 32000, n_layer: 80, n_head: 64, n_embd: 8192 } + } +} + +#[derive(Debug)] +struct RmsNorm { + scale: Tensor, + size: i64, +} + +impl RmsNorm { + fn new(vs: nn::Path, size: i64) -> Self { + let scale = vs.zeros("scale", &[size]); + Self { scale, size } + } +} + +impl Module for RmsNorm { + fn forward(&self, xs: &Tensor) -> Tensor { + let norm_xs = (xs * xs).mean_dim(-1, true, Kind::Float); + let xs_normed = xs * (norm_xs + 1e-5).rsqrt(); + let scale = self.scale.reshape([1, 1, self.size]); + scale * xs_normed + } +} + +#[derive(Debug)] +struct Mlp { + c_fc1: nn::Linear, + c_fc2: nn::Linear, + c_proj: nn::Linear, +} + +impl Mlp { + fn new(vs: nn::Path, n_embd: i64) -> Self { + let n_hidden = 8 * n_embd / 3; + let n_hidden = (n_hidden - 1) / 256 * 256 + 256; + let c = nn::LinearConfig { bias: false, ..Default::default() }; + let c_fc1 = nn::linear(&vs / "c_fc1", n_embd, n_hidden, c); + let c_fc2 = nn::linear(&vs / "c_fc2", n_embd, n_hidden, c); + let c_proj = nn::linear(&vs / "c_proj", n_hidden, n_embd, c); + Self { c_fc1, c_fc2, c_proj } + } +} + +impl Module for Mlp { + fn forward(&self, xs: &Tensor) -> Tensor { + let xs = xs.apply(&self.c_fc1).silu() * xs.apply(&self.c_fc2); + xs.apply(&self.c_proj) + } +} + +#[derive(Debug)] +struct CausalSelfAttention { + c_attn: nn::Linear, + c_proj: nn::Linear, + n_head: i64, + n_embd: i64, + device: Device, +} + +impl CausalSelfAttention { + fn new(vs: nn::Path, n_head: i64, n_embd: i64) -> Self { + let c = nn::LinearConfig { bias: false, ..Default::default() }; + let c_attn = nn::linear(&vs / "c_attn", n_embd, 3 * n_embd, c); + let c_proj = nn::linear(&vs / "c_proj", n_embd, n_embd, c); + Self { c_attn, c_proj, n_head, n_embd, device: vs.device() } + } + + fn apply_rotary_emb(&self, x: &Tensor, freqs_cis: &Tensor) -> Tensor { + let mut dims = x.size(); + let v = dims.pop().unwrap(); + dims.push(v / 2); + dims.push(2); + let x = x.reshape(&dims); + let re_x = x.slice(-1, 0, 1, 1); + let im_x = x.slice(-1, 1, 2, 1); + let re_f = freqs_cis.slice(-1, 0, 1, 1); + let im_f = freqs_cis.slice(-1, 1, 2, 1); + let re = &re_x * &re_f - &im_x * &im_f; + let im = &re_x * &im_f + &im_x * &re_f; + let rope = Tensor::cat(&[&re, &im], -1); + // TODO: Add the flatten op. + let mut dims = rope.size(); + let v1 = dims.pop().unwrap(); + let v2 = dims.pop().unwrap(); + dims.push(v1 * v2); + rope.reshape(&dims) + } + + fn forward(&self, x: &Tensor, freqs_cis: &Tensor) -> Tensor { + let (b, t, c) = x.size3().unwrap(); + let kind = x.kind(); + let qkv = self.c_attn.forward(x); + let n_embd = self.n_embd; + let q = qkv.slice(2, 0, n_embd, 1); + let k = qkv.slice(2, n_embd, 2 * n_embd, 1); + let v = qkv.slice(2, 2 * n_embd, 3 * n_embd, 1); + let target_dim = [b, t, self.n_head, c / self.n_head]; + let k = k.reshape(target_dim).transpose(1, 2); + let q = q.reshape(target_dim).transpose(1, 2); + let v = v.reshape(target_dim).transpose(1, 2); + let q = self.apply_rotary_emb(&q, freqs_cis); + let k = self.apply_rotary_emb(&k, freqs_cis); + let k_shape = k.size(); + let att: Tensor = q.matmul(&k.transpose(-2, -1)) / (*k_shape.last().unwrap() as f64).sqrt(); + let mask = Tensor::ones([t, t], (kind, self.device)).tril(0).reshape([1, 1, t, t]); + let att = att.masked_fill(&mask.eq(0.), f64::NEG_INFINITY); + let y = att.softmax(-1, kind).matmul(&v); + let y = y.transpose(1, 2).reshape([b, t, c]); + self.c_proj.forward(&y) + } +} + +#[derive(Debug)] +struct Block { + rms_1: RmsNorm, + attn: CausalSelfAttention, + rms_2: RmsNorm, + mlp: Mlp, +} + +impl Block { + fn new(vs: nn::Path, config: &Config) -> Self { + let rms_1 = RmsNorm::new(&vs / "rms_1", config.n_embd as i64); + let attn = + CausalSelfAttention::new(&vs / "attn", config.n_head as i64, config.n_embd as i64); + let rms_2 = RmsNorm::new(&vs / "rms_2", config.n_embd as i64); + let mlp = Mlp::new(&vs / "mlp", config.n_embd as i64); + Self { rms_1, attn, rms_2, mlp } + } + + fn forward(&self, x: &Tensor, freqs_cis: &Tensor) -> Tensor { + let x = self.attn.forward(&self.rms_1.forward(x), freqs_cis) + x; + self.mlp.forward(&self.rms_2.forward(&x)) + x + } +} + +#[derive(Debug)] +struct Llama { + wte: nn::Embedding, + blocks: Vec, + ln_f: RmsNorm, + lm_head: nn::Linear, +} + +impl Llama { + fn new(vs: nn::Path, config: &Config) -> Self { + let c = nn::LinearConfig { bias: false, ..Default::default() }; + let lm_head = + nn::linear(&vs / "lm_head", config.n_embd as i64, config.vocab_size as i64, c); + let wte = nn::embedding( + &vs / "transformer" / "wte", + config.vocab_size as i64, + config.n_embd as i64, + Default::default(), + ); + let blocks = (0..config.n_layer) + .map(|i| Block::new(&vs / "transformer" / "h" / i, config)) + .collect::>(); + let ln_f = RmsNorm::new(&vs / "transformer" / "ln_f", config.n_embd as i64); + Self { wte, blocks, ln_f, lm_head } + } + + fn forward(&self, x: &Tensor, freqs_cis: &Tensor) -> Tensor { + let (_, t) = x.size2().unwrap(); + let mut x = self.wte.forward(x); + for block in self.blocks.iter() { + x = block.forward(&x, freqs_cis); + } + let x = self.ln_f.forward(&x); + let x = x.slice(1, t - 1, t, 1); + self.lm_head.forward(&x) + } +} + +fn precompute_freqs_cis(config: &Config) -> Tensor { + let seq_len = CONTEXT_SIZE; + let n_elem = config.n_embd / config.n_head; + let theta: Vec<_> = + (0..n_elem).step_by(2).map(|i| 1f32 / 10000f32.powf(i as f32 / n_elem as f32)).collect(); + let arange: Vec<_> = (0..seq_len).map(|c| c as f32).collect(); + let theta = Tensor::from_slice(&theta); + let arange = Tensor::from_slice(&arange); + let idx_theta = arange.outer(&theta); + let shape = [1, 1, seq_len as i64, n_elem as i64 / 2, 1]; + let idx_theta_cos = idx_theta.cos().reshape(shape); + let idx_theta_sin = idx_theta.sin().reshape(shape); + Tensor::cat(&[&idx_theta_cos, &idx_theta_sin], -1) +} + +fn llama(vs: nn::Path, args: Args) -> impl Module { + let config = Config::config_7b(); + let freqs_cis = precompute_freqs_cis(&config).to_device(vs.device()); + let llama = Llama::new(vs, &config); + nn::func(move |xs| { + let logits = llama.forward(xs, &freqs_cis); + (logits / args.temperature).softmax(-1, Kind::Float) + }) +} + +#[derive(Debug, Clone, Copy, clap::ValueEnum)] +enum CompKind { + Float, + Half, + #[clap(name = "bfloat16")] + BFloat16, + #[clap(name = "qint8")] + QInt8, +} + +impl CompKind { + fn to_kind(self) -> Kind { + match self { + Self::Float => Kind::Float, + Self::Half => Kind::Half, + Self::BFloat16 => Kind::BFloat16, + Self::QInt8 => Kind::QInt8, + } + } +} + +#[derive(clap::Parser, Debug, Clone, Copy)] +#[command(author, version, about, long_about = None)] +struct Args { + /// Run on CPU rather than on GPU. + #[arg(long)] + cpu: bool, + + /// Use this type of float for computations, float/half/bfloat16/... + #[arg(long, value_enum, default_value = "float")] + kind: CompKind, + + /// The temperature used to generate samples. + #[arg(long, default_value_t = 1.0)] + temperature: f64, + + /// The length of the sample to generate (in tokens). + #[arg(long, default_value_t = 100)] + sample_len: usize, +} + +fn main() -> Result<()> { + let _no_grad = tch::no_grad_guard(); + let args = Args::parse(); + let tokenizer = Tokenizer::from_file("llama-tokenizer.json")?; + let mut tokens = tokenizer.encode(START_PROMPT)?; + let mut new_tokens = vec![]; + let start_build = std::time::Instant::now(); + let device = if args.cpu { + Device::Cpu + } else if tch::utils::has_mps() { + Device::Mps + } else { + Device::Cuda(0) + }; + let mut vs = nn::VarStore::new(device); + let llama = llama(vs.root(), args); + // When loading the weights, assume that they are in the float16 + // format. + vs.set_kind(Kind::Half); + println!("generated the model in {:?}", start_build.elapsed()); + + let start_load = std::time::Instant::now(); + // Instead of the standard `VarStore::load`, we use a mmap version that + // should improve memory efficiency. + { + let file = std::fs::File::open("llama.safetensors")?; + let content = unsafe { memmap2::MmapOptions::new().map(&file)? }; + let safetensors = safetensors::SafeTensors::deserialize(&content)?; + + let mut variables = vs.variables_.lock().unwrap(); + for (name, var) in variables.named_variables.iter_mut() { + let view = safetensors.tensor(name)?; + let size: Vec = view.shape().iter().map(|&x| x as i64).collect(); + let kind: Kind = view.dtype().try_into()?; + // Using from_blob here instead of from_data_size avoids some unnecessary copy. + let src_tensor = + unsafe { Tensor::from_blob(view.data().as_ptr(), &size, &[], kind, Device::Cpu) }; + var.f_copy_(&src_tensor)?; + } + } + println!("loaded weights in {:?}", start_load.elapsed()); + + vs.set_kind(args.kind.to_kind()); + println!("switched to {:?}", args.kind); + + for index in 0..args.sample_len { + let ctxt: Vec<_> = + tokens[tokens.len().saturating_sub(CONTEXT_SIZE)..].iter().map(|c| *c as i64).collect(); + let ctxt = Tensor::from_slice(&ctxt).reshape([1, -1]); + let logits = llama.forward(&ctxt); + let sampled_y = logits.get(0).get(0).multinomial(1, true); + let next_token = i64::try_from(&sampled_y)? as usize; + tokens.push(next_token); + new_tokens.push(next_token); + println!("{} token: {} '{}'", index + 1, next_token, tokenizer.decode(&[next_token])); + } + println!("----\n{}\n----", tokenizer.decode(&new_tokens)); + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/llama/sentencepiece.rs b/v2/dependency-patches/tch-0.24.0/examples/llama/sentencepiece.rs new file mode 100644 index 0000000000..3eb6f0e55b --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/llama/sentencepiece.rs @@ -0,0 +1,149 @@ +// A very naive sentencepiece encoder/decoder, this only supports the BPE model and not the unigram +// one. +use anyhow::{bail, Context, Result}; +use std::collections::{HashMap, HashSet}; +pub struct Tokenizer { + encoder: HashMap, usize>, + // TODO: Maybe use a vec instead of a hashmap? + decoder: HashMap, + bpe_ranks: HashMap<(Vec, Vec), usize>, +} + +const DELIM: char = '▁'; + +impl Tokenizer { + pub fn from_file>(path: P) -> Result { + let reader = std::io::BufReader::new(std::fs::File::open(path)?); + let config: serde_json::Value = serde_json::from_reader(reader)?; + let model = config.get("model").context("no model key")?; + let type_ = + model.get("type").context("no model.type key")?.as_str().context("not a string")?; + if type_ != "BPE" { + bail!(format!("model type is not BPE: {type_}")) + } + let vocab = model + .get("vocab") + .context("no model.vocab key")? + .as_object() + .context("model.vocab not an object")?; + let single_chars: HashSet = vocab + .iter() + .filter_map(|(key, _)| { + let b = key.as_bytes(); + if b.len() == 1 { + Some(b[0]) + } else { + None + } + }) + .collect(); + let encoder = vocab + .iter() + .rev() + .map(|(key, value)| { + let key = key + .strip_prefix("<0x") + .and_then(|s| s.strip_suffix('>')) + .and_then(|s| u8::from_str_radix(s, 16).ok()) + .and_then(|s| if single_chars.contains(&s) { None } else { Some(s) }) + .map_or_else(|| key.as_bytes().to_vec(), |s| vec![s]); + value.as_i64().context("not an int").map(|v| (key, v as usize)) + }) + .collect::>>()?; + let bpe_ranks = model + .get("merges") + .context("no model.merges key")? + .as_array() + .context("model.merges not an array")? + .iter() + .enumerate() + .map(|(i, value)| { + let value = value.as_str().context("not a string")?; + match value.split_once(' ') { + Some((v1, v2)) => { + let key = (v1.as_bytes().to_vec(), v2.as_bytes().to_vec()); + Ok((key, i)) + } + None => bail!(format!("no space in merge '{value}'")), + } + }) + .collect::>>()?; + let decoder = encoder + .iter() + .map(|(k, v)| (*v, String::from_utf8_lossy(k).replace(DELIM, " "))) + .collect(); + Ok(Self { encoder, decoder, bpe_ranks }) + } + + fn get_pairs(word: &[Vec]) -> HashSet<(Vec, Vec)> { + let mut pairs = HashSet::new(); + for (i, v) in word.iter().enumerate() { + if i > 0 { + pairs.insert((word[i - 1].clone(), v.clone())); + } + } + pairs + } + + fn bpe(&self, s: &str) -> Vec { + let mut buffer = [0u8; 4]; + let mut word: Vec> = vec![]; + for c in s.chars() { + let buffer = c.encode_utf8(&mut buffer); + word.push(buffer.as_bytes().to_vec()) + } + if word.is_empty() { + return Vec::new(); + } + while word.len() > 1 { + let mut current_min = None; + let pairs = Self::get_pairs(&word); + for p in pairs.iter() { + match self.bpe_ranks.get(p) { + None => {} + Some(v) => { + let should_replace = match current_min { + None => true, + Some((current_min, _)) => v < current_min, + }; + if should_replace { + current_min = Some((v, p)) + } + } + } + } + let (first, second) = match current_min { + None => break, + Some((_v, (first, second))) => (first, second), + }; + let mut new_word = vec![]; + let mut index = 0; + while index < word.len() { + let w = &word[index]; + if index + 1 < word.len() && w == first && &word[index + 1] == second { + let mut first_and_second = first.clone(); + first_and_second.extend_from_slice(second); + new_word.push(first_and_second); + index += 2 + } else { + new_word.push(w.clone()); + index += 1 + } + } + word = new_word + } + word.iter().filter_map(|x| self.encoder.get(x)).copied().collect() + } + + // Run bpe on the whole string, very very inefficient but should be good enough for + // prompts. The original string should first be split on whitespace/... + pub fn encode(&self, s: &str) -> Result> { + let mut buffer = [0u8; 4]; + let s = format!("{DELIM}{}", s.replace(' ', DELIM.encode_utf8(&mut buffer))); + Ok(self.bpe(&s)) + } + + pub fn decode(&self, tokens: &[usize]) -> String { + tokens.iter().map(|token| self.decoder[token].as_str()).collect() + } +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/memory_test.rs b/v2/dependency-patches/tch-0.24.0/examples/memory_test.rs new file mode 100644 index 0000000000..f2bd0cae8b --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/memory_test.rs @@ -0,0 +1,19 @@ +// This example is useful to check for memory leaks. +// A large number of tensors are created either on the cpu or on the gpu and one +// can monitor the main memory usage or the gpu memory at the same time. +use tch::Tensor; + +fn main() { + let a: Vec = std::env::args().collect(); + let device = match a.iter().map(|x| x.as_str()).collect::>().as_slice() { + [_] => tch::Device::Cpu, + [_, "cpu"] => tch::Device::Cpu, + [_, "gpu"] => tch::Device::Cuda(0), + _ => panic!("usage: main cpu|gpu"), + }; + let slice = vec![0; 1_000_000]; + for i in 1..1_000_000 { + let t = Tensor::from_slice(&slice).to_device(device); + println!("{} {:?}", i, t.size()) + } +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/min-gpt/README.md b/v2/dependency-patches/tch-0.24.0/examples/min-gpt/README.md new file mode 100644 index 0000000000..33e94f2647 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/min-gpt/README.md @@ -0,0 +1,78 @@ +## minGPT-rs + +This example implements a character-level language model using +a simplified version of [minGPT](https://github.com/karpathy/minGPT). +All the model details as well as the training loop can be found +in `main.rs`. + +Besides the model being different, this example is in line with the LSTM +version [char-rnn](https://github.com/LaurentMazare/tch-rs/tree/master/examples/char-rnn). +The same comments apply: +- For training, the model takes as input a text file and learns to predict the + next character following a given sequence. +- For generation, the model uses a fixed seed, returns the next character + distribution from which a character is randomly sampled and this is iterated + to generate a text. + +At the end of each training epoch, some sample text is generated and printed. + +Any text file can be used as an input, as long as it's large enough for training. +A typical example would be the +[tiny Shakespeare dataset](https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt). +The training text file should be stored in `data/input.txt`. + +Compiling and running the example is done via the following command lines: +```bash +cargo run --example min-gpt +``` + +Here is an example of generated data when training on the Shakespeare dataset at +the end of the first epoch. + +``` +LORD: +My nown of none, by God's mother, +No more of this country, nor than I +Who have strengthen lamented to make it stay +there; and yet it will no more but abide. + +AUTOLYCUS: +Very true, and but a month old. + +DORCAS: +Bless me from marrying a usurer! + +AUTOLYCUS: +Here's the midwife's name to't, one Mistress +Overdone's own house, for here be made. + +EXETER: +The doubt is that which calls so coldly? + +GLOUCESTER: +I grant ye. + +LADY ANNE: +Dost grant me, hedgehog? then, God grant me too +Thou mayst be damned for that wicked deed! +O, he was gentle, mild, and virtuous! + +GLOUCESTER: +The fitter for the King of heaven, that hath him. + +LADY ANNE: +He is in heaven, where thou shalt never come. + +GLOUCESTER: +Let him thank me, or I am sure yet; +But send me France and my true Jove's hands. + +WARWICK: +And I choose Clarence only for protector. + +KING HENRY VI: +Warwick and Clarence give me both your hands: +Now join your hands together, and your father's death +And hardening of my brother's son +It reep her to crave the hire, here my mother +``` diff --git a/v2/dependency-patches/tch-0.24.0/examples/min-gpt/main.rs b/v2/dependency-patches/tch-0.24.0/examples/min-gpt/main.rs new file mode 100644 index 0000000000..66ddd985b2 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/min-gpt/main.rs @@ -0,0 +1,212 @@ +/* This example uses the tinyshakespeare dataset which can be downloaded at: + https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt + + This is mostly a rust port of https://github.com/karpathy/minGPT +*/ + +use anyhow::{bail, Result}; +use tch::data::TextData; +use tch::nn::{ModuleT, OptimizerConfig}; +use tch::{nn, Device, IndexOp, Kind, Tensor}; + +const LEARNING_RATE: f64 = 0.0003; +const BLOCK_SIZE: i64 = 128; +const BATCH_SIZE: i64 = 64; +const EPOCHS: i64 = 100; +const SAMPLING_LEN: i64 = 4096; + +#[derive(Debug, Copy, Clone)] +struct Config { + vocab_size: i64, + n_embd: i64, + n_head: i64, + n_layer: i64, + block_size: i64, + attn_pdrop: f64, + resid_pdrop: f64, + embd_pdrop: f64, +} + +// Weight decay only applies to the weight matrixes in the linear layers +const NO_WEIGHT_DECAY_GROUP: usize = 0; +const WEIGHT_DECAY_GROUP: usize = 1; + +// Custom linear layer so that different groups can be used for weight +// and biases. +#[derive(Debug)] +struct Linear { + pub ws: Tensor, + pub bs: Tensor, +} + +impl nn::Module for Linear { + fn forward(&self, xs: &Tensor) -> Tensor { + xs.matmul(&self.ws.tr()) + &self.bs + } +} + +fn linear(vs: nn::Path, in_dim: i64, out_dim: i64) -> Linear { + let wd = vs.set_group(WEIGHT_DECAY_GROUP); + let no_wd = vs.set_group(NO_WEIGHT_DECAY_GROUP); + Linear { + ws: wd.randn("weight", &[out_dim, in_dim], 0.0, 0.02), + bs: no_wd.zeros("bias", &[out_dim]), + } +} + +fn linear_no_bias(vs: nn::Path, in_dim: i64, out_dim: i64) -> Linear { + let wd = vs.set_group(WEIGHT_DECAY_GROUP); + let no_wd = vs.set_group(NO_WEIGHT_DECAY_GROUP); + Linear { + ws: wd.randn("weight", &[out_dim, in_dim], 0.0, 0.02), + bs: no_wd.zeros_no_train("bias", &[out_dim]), + } +} + +fn causal_self_attention(p: &nn::Path, cfg: Config) -> impl ModuleT { + let key = linear(p / "key", cfg.n_embd, cfg.n_embd); + let query = linear(p / "query", cfg.n_embd, cfg.n_embd); + let value = linear(p / "value", cfg.n_embd, cfg.n_embd); + let proj = linear(p / "proj", cfg.n_embd, cfg.n_embd); + let mask_init = + Tensor::ones([cfg.block_size, cfg.block_size], (Kind::Float, p.device())).tril(0); + let mask_init = mask_init.view([1, 1, cfg.block_size, cfg.block_size]); + // let mask = p.var_copy("mask", &mask_init); + let mask = mask_init; + nn::func_t(move |xs, train| { + let (sz_b, sz_t, sz_c) = xs.size3().unwrap(); + let sizes = [sz_b, sz_t, cfg.n_head, sz_c / cfg.n_head]; + let k = xs.apply(&key).view(sizes).transpose(1, 2); + let q = xs.apply(&query).view(sizes).transpose(1, 2); + let v = xs.apply(&value).view(sizes).transpose(1, 2); + let att = q.matmul(&k.transpose(-2, -1)) * (1.0 / f64::sqrt(sizes[3] as f64)); + let att = att.masked_fill(&mask.i((.., .., ..sz_t, ..sz_t)).eq(0.), f64::NEG_INFINITY); + let att = att.softmax(-1, Kind::Float).dropout(cfg.attn_pdrop, train); + let ys = att.matmul(&v).transpose(1, 2).contiguous().view([sz_b, sz_t, sz_c]); + ys.apply(&proj).dropout(cfg.resid_pdrop, train) + }) +} + +fn block(p: &nn::Path, cfg: Config) -> impl ModuleT { + let ln1 = nn::layer_norm(p / "ln1", vec![cfg.n_embd], Default::default()); + let ln2 = nn::layer_norm(p / "ln2", vec![cfg.n_embd], Default::default()); + let attn = causal_self_attention(p, cfg); + let lin1 = linear(p / "lin1", cfg.n_embd, 4 * cfg.n_embd); + let lin2 = linear(p / "lin2", 4 * cfg.n_embd, cfg.n_embd); + nn::func_t(move |xs, train| { + let xs = xs + xs.apply(&ln1).apply_t(&attn, train); + let ys = + xs.apply(&ln2).apply(&lin1).gelu("none").apply(&lin2).dropout(cfg.resid_pdrop, train); + xs + ys + }) +} + +fn gpt(p: nn::Path, cfg: Config) -> impl ModuleT { + let p = &p.set_group(NO_WEIGHT_DECAY_GROUP); + let tok_emb = nn::embedding(p / "tok_emb", cfg.vocab_size, cfg.n_embd, Default::default()); + let pos_emb = p.zeros("pos_emb", &[1, cfg.block_size, cfg.n_embd]); + let ln_f = nn::layer_norm(p / "ln_f", vec![cfg.n_embd], Default::default()); + let head = linear_no_bias(p / "head", cfg.n_embd, cfg.vocab_size); + let mut blocks = nn::seq_t(); + for block_idx in 0..cfg.n_layer { + blocks = blocks.add(block(&(p / block_idx), cfg)); + } + nn::func_t(move |xs, train| { + let (_sz_b, sz_t) = xs.size2().unwrap(); + let tok_emb = xs.apply(&tok_emb); + let pos_emb = pos_emb.i((.., ..sz_t, ..)); + (tok_emb + pos_emb) + .dropout(cfg.embd_pdrop, train) + .apply_t(&blocks, train) + .apply(&ln_f) + .apply(&head) + }) +} + +/// Generates some sample string using the GPT model. +fn sample(data: &TextData, gpt: &impl ModuleT, input: Tensor) -> String { + let mut input = input; + let mut result = String::new(); + for _index in 0..SAMPLING_LEN { + let logits = input.apply_t(gpt, false).i((0, -1, ..)); + let sampled_y = logits.softmax(-1, Kind::Float).multinomial(1, true); + let last_label = i64::try_from(&sampled_y).unwrap(); + result.push(data.label_to_char(last_label)); + input = Tensor::cat(&[input, sampled_y.view([1, 1])], 1).narrow(1, 1, BLOCK_SIZE); + } + result +} + +pub fn main() -> Result<()> { + let device = Device::cuda_if_available(); + let mut vs = nn::VarStore::new(device); + let data = TextData::new("data/input.txt")?; + let labels = data.labels(); + println!("Dataset loaded, {labels} labels."); + let cfg = Config { + vocab_size: labels, + n_embd: 512, + n_head: 8, + n_layer: 8, + block_size: BLOCK_SIZE, + attn_pdrop: 0.1, + resid_pdrop: 0.1, + embd_pdrop: 0.1, + }; + let gpt = gpt(vs.root() / "gpt", cfg); + let args: Vec<_> = std::env::args().collect(); + if args.len() < 2 { + bail!("usage: main (train|predict weights.ot seqstart)") + } + match args[1].as_str() { + "train" => { + let mut opt = nn::AdamW::default().build(&vs, LEARNING_RATE)?; + opt.set_weight_decay_group(NO_WEIGHT_DECAY_GROUP, 0.0); + opt.set_weight_decay_group(WEIGHT_DECAY_GROUP, 0.1); + let mut idx = 0; + for epoch in 1..(1 + EPOCHS) { + let mut sum_loss = 0.; + let mut cnt_loss = 0.; + for batch in data.iter_shuffle(BLOCK_SIZE + 1, BATCH_SIZE) { + let xs = batch.narrow(1, 0, BLOCK_SIZE).to_kind(Kind::Int64).to_device(device); + let ys = batch.narrow(1, 1, BLOCK_SIZE).to_kind(Kind::Int64).to_device(device); + let logits = xs.apply_t(&gpt, true); + let loss = logits + .view([BATCH_SIZE * BLOCK_SIZE, labels]) + .cross_entropy_for_logits(&ys.view([BATCH_SIZE * BLOCK_SIZE])); + opt.backward_step_clip(&loss, 0.5); + sum_loss += f64::try_from(loss)?; + cnt_loss += 1.0; + idx += 1; + if idx % 10000 == 0 { + println!("Epoch: {} loss: {:5.3}", epoch, sum_loss / cnt_loss); + let input = Tensor::zeros([1, BLOCK_SIZE], (Kind::Int64, device)); + println!("Sample: {}", sample(&data, &gpt, input)); + if let Err(err) = vs.save(format!("gpt{idx}.ot")) { + println!("error while saving {err}"); + } + sum_loss = 0.; + cnt_loss = 0.; + } + } + } + } + "predict" => { + vs.load(args[2].as_str())?; + let seqstart = args[3].as_str(); + let input = Tensor::zeros([1, BLOCK_SIZE], (Kind::Int64, device)); + for (idx, c) in seqstart.chars().rev().enumerate() { + let idx = idx as i64; + if idx >= BLOCK_SIZE { + break; + } + let _filled = + input.i((0, BLOCK_SIZE - 1 - idx)).fill_(data.char_to_label(c)? as i64); + } + println!("Sample: {}", sample(&data, &gpt, input)); + } + _ => bail!("usage: main (train|predict weights.ot seqstart)"), + }; + + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/mnist/README.md b/v2/dependency-patches/tch-0.24.0/examples/mnist/README.md new file mode 100644 index 0000000000..b9e7d5409c --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/mnist/README.md @@ -0,0 +1,100 @@ +The examples in this directory have been adapted from the [TensorFlow +tutorials](https://www.tensorflow.org/versions/r0.7/tutorials/mnist/pros/index.html). +To execute these examples, you will have to unzip the [MNIST data +files](http://yann.lecun.com/exdb/mnist/) in `data/`. + +## Linear Classifier + +The code can be found in `mnist_linear.ml`. + +We first load the MNIST data. This is done using the MNIST helper module, +labels are returned as a vector of integer. Train images and labels are used +when training the model. Test images and labels are used to estimate the +validation error. + +```rust +let m = vision::mnist::load_dir("data").unwrap(); +``` + +After that two tensors are initialized to hold the weights and biases for the +linear classifier. `requires_grad` is used when creating the tensors to inform +torch that we will compute some gradients with respect to these tensors. + +```rust +let mut ws = Tensor::zeros(&[IMAGE_DIM, LABELS], kind::FLOAT_CPU).set_requires_grad(true); +let mut bs = Tensor::zeros(&[LABELS], kind::FLOAT_CPU).set_requires_grad(true); +``` + +Using these the model is defined as multiplying an input by the weight matrix +and adding the bias. A softmax function is used to transform the output into a +probability distribution. + +```rust + let logits = m.train_images.mm(&ws) + &bs; +``` + +We use gradient descent to minimize cross-entropy with respect to variables +`ws` and `bs` and iterate this a couple hundred times. + +Rather than using an optimizer we perform the gradient descent updates manually. +This is only to illustrate how gradients can be computed and used. Other examples +such as `mnist_nn.rs` or `mnist_conv.rs` use an Adam optimizer. + +```rust + for epoch in 1..200 { + let logits = m.train_images.mm(&ws) + &bs; + let loss = logits.log_softmax(-1).nll_loss(&m.train_labels); + ws.zero_grad(); + bs.zero_grad(); + loss.backward(); + no_grad(|| { + ws += ws.grad() * (-1); + bs += bs.grad() * (-1); + }); +``` + +Running this code should build a model that has ~92% accuracy. + +## A Simple Neural-Network + +The code can be found in `mnist_nn.rs`, accuracy should reach ~96%. + +## Convolutional Neural-Network + +The code can be found in `mnist_conv.rs`, accuracy should reach ~99%. + +When building models with multiple weights and bias parameters we use +a variable store to keep track of these variables and let the optimizer +know about them. The variable store is created in the first line of the +following snippet, then the model is built using this variable store +and finally we create an optimizer that will performs gradient descent +over the parameters that have been added to the variable store when +creating the network. + +```rust + let vs = nn::VarStore::new(Device::cuda_if_available()); + let net = Net::new(&vs.root()); + let opt = nn::Optimizer::adam(&vs, 1e-4, Default::default()); +``` + +Note that this will automatically run on a GPU when available. +For a convolutional model we cannot run on all the training images in +a single step as this would requires lots of memory. Instead we +run on mini-batches. An iterator makes it easy to loop over all +the training images shuffled and grouped by mini-batches. +This is done in the main training loop: + +```rust + for epoch in 1..100 { + for (bimages, blabels) in m.train_iter(256).shuffle().to_device(vs.device()) { + let loss = net + .forward_t(&bimages, true) + .cross_entropy_for_logits(&blabels); + opt.backward_step(&loss); + } + let test_accuracy = + net.batch_accuracy_for_logits(&m.test_images, &m.test_labels, vs.device(), 1024); + println!("epoch: {:4} test acc: {:5.2}%", epoch, 100. * test_accuracy,); + } +``` + diff --git a/v2/dependency-patches/tch-0.24.0/examples/mnist/main.rs b/v2/dependency-patches/tch-0.24.0/examples/mnist/main.rs new file mode 100644 index 0000000000..88e0483991 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/mnist/main.rs @@ -0,0 +1,25 @@ +/* Some very simple models trained on the MNIST dataset. + The 4 following dataset files can be downloaded from http://yann.lecun.com/exdb/mnist/ + These files should be extracted in the 'data' directory. + train-images-idx3-ubyte.gz + train-labels-idx1-ubyte.gz + t10k-images-idx3-ubyte.gz + t10k-labels-idx1-ubyte.gz +*/ + +use anyhow::Result; + +mod mnist_conv; +mod mnist_linear; +mod mnist_nn; + +fn main() -> Result<()> { + let args: Vec = std::env::args().collect(); + let model = if args.len() < 2 { None } else { Some(args[1].as_str()) }; + match model { + None => mnist_nn::run(), + Some("linear") => mnist_linear::run(), + Some("conv") => mnist_conv::run(), + Some(_) => mnist_nn::run(), + } +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_conv.rs b/v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_conv.rs new file mode 100644 index 0000000000..f4b366946a --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_conv.rs @@ -0,0 +1,54 @@ +// CNN model. This should rearch 99.1% accuracy. + +use anyhow::Result; +use tch::{nn, nn::ModuleT, nn::OptimizerConfig, Device, Tensor}; + +#[derive(Debug)] +struct Net { + conv1: nn::Conv2D, + conv2: nn::Conv2D, + fc1: nn::Linear, + fc2: nn::Linear, +} + +impl Net { + fn new(vs: &nn::Path) -> Net { + let conv1 = nn::conv2d(vs, 1, 32, 5, Default::default()); + let conv2 = nn::conv2d(vs, 32, 64, 5, Default::default()); + let fc1 = nn::linear(vs, 1024, 1024, Default::default()); + let fc2 = nn::linear(vs, 1024, 10, Default::default()); + Net { conv1, conv2, fc1, fc2 } + } +} + +impl nn::ModuleT for Net { + fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor { + xs.view([-1, 1, 28, 28]) + .apply(&self.conv1) + .max_pool2d_default(2) + .apply(&self.conv2) + .max_pool2d_default(2) + .view([-1, 1024]) + .apply(&self.fc1) + .relu() + .dropout(0.5, train) + .apply(&self.fc2) + } +} + +pub fn run() -> Result<()> { + let m = tch::vision::mnist::load_dir("data")?; + let vs = nn::VarStore::new(Device::cuda_if_available()); + let net = Net::new(&vs.root()); + let mut opt = nn::Adam::default().build(&vs, 1e-4)?; + for epoch in 1..100 { + for (bimages, blabels) in m.train_iter(256).shuffle().to_device(vs.device()) { + let loss = net.forward_t(&bimages, true).cross_entropy_for_logits(&blabels); + opt.backward_step(&loss); + } + let test_accuracy = + net.batch_accuracy_for_logits(&m.test_images, &m.test_labels, vs.device(), 1024); + println!("epoch: {:4} test acc: {:5.2}%", epoch, 100. * test_accuracy,); + } + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_linear.rs b/v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_linear.rs new file mode 100644 index 0000000000..cd21117c34 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_linear.rs @@ -0,0 +1,42 @@ +// This should rearch 91.5% accuracy. + +use anyhow::Result; +use tch::{kind, no_grad, vision, Kind, Tensor}; + +const IMAGE_DIM: i64 = 784; +const LABELS: i64 = 10; + +pub fn run() -> Result<()> { + let m = vision::mnist::load_dir("data")?; + println!("train-images: {:?}", m.train_images.size()); + println!("train-labels: {:?}", m.train_labels.size()); + println!("test-images: {:?}", m.test_images.size()); + println!("test-labels: {:?}", m.test_labels.size()); + let mut ws = Tensor::zeros([IMAGE_DIM, LABELS], kind::FLOAT_CPU).set_requires_grad(true); + let mut bs = Tensor::zeros([LABELS], kind::FLOAT_CPU).set_requires_grad(true); + for epoch in 1..200 { + let logits = m.train_images.mm(&ws) + &bs; + let loss = logits.log_softmax(-1, Kind::Float).nll_loss(&m.train_labels); + ws.zero_grad(); + bs.zero_grad(); + loss.backward(); + no_grad(|| { + ws += ws.grad() * (-1); + bs += bs.grad() * (-1); + }); + let test_logits = m.test_images.mm(&ws) + &bs; + let test_accuracy = test_logits + .argmax(Some(-1), false) + .eq_tensor(&m.test_labels) + .to_kind(Kind::Float) + .mean(Kind::Float) + .double_value(&[]); + println!( + "epoch: {:4} train loss: {:8.5} test acc: {:5.2}%", + epoch, + loss.double_value(&[]), + 100. * test_accuracy + ); + } + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_nn.rs b/v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_nn.rs new file mode 100644 index 0000000000..0c799633a3 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_nn.rs @@ -0,0 +1,34 @@ +// This should rearch 97% accuracy. + +use anyhow::Result; +use tch::{nn, nn::Module, nn::OptimizerConfig, Device}; + +const IMAGE_DIM: i64 = 784; +const HIDDEN_NODES: i64 = 128; +const LABELS: i64 = 10; + +fn net(vs: &nn::Path) -> impl Module { + nn::seq() + .add(nn::linear(vs / "layer1", IMAGE_DIM, HIDDEN_NODES, Default::default())) + .add_fn(|xs| xs.relu()) + .add(nn::linear(vs, HIDDEN_NODES, LABELS, Default::default())) +} + +pub fn run() -> Result<()> { + let m = tch::vision::mnist::load_dir("data")?; + let vs = nn::VarStore::new(Device::Cpu); + let net = net(&vs.root()); + let mut opt = nn::Adam::default().build(&vs, 1e-3)?; + for epoch in 1..200 { + let loss = net.forward(&m.train_images).cross_entropy_for_logits(&m.train_labels); + opt.backward_step(&loss); + let test_accuracy = net.forward(&m.test_images).accuracy_for_logits(&m.test_labels); + println!( + "epoch: {:4} train loss: {:8.5} test acc: {:5.2}%", + epoch, + f64::try_from(&loss)?, + 100. * f64::try_from(&test_accuracy)?, + ); + } + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/neural-style-transfer/README.md b/v2/dependency-patches/tch-0.24.0/examples/neural-style-transfer/README.md new file mode 100644 index 0000000000..2f6a8e0010 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/neural-style-transfer/README.md @@ -0,0 +1,162 @@ +## Neural Style Transfer in Rust + +This example shows how to implement +[Neural Style Transfer](https://arxiv.org/abs/1508.06576) using PyTorch and Rust, +it is very close to the +[Neural Transfer using PyTorch tutorial](https://pytorch.org/tutorials/advanced/neural_style_tutorial.html) +which contains far more details. + +The Neural Style Transfer algorithm takes as input a content image and a style image and +produces some new images looking similar to the content image but using the style +from the second image. + +### Running the Code +We use a pre-trained VGG-16 convolutional network, the weights can be found in +[vgg16.ot](https://github.com/LaurentMazare/tch-rs/releases/download/mw/vgg16.ot). +Running the example can be done with the following command, this will create +some new images in the current directory. + +```bash +cargo run --example neural-style-transfer style.jpg content.jpg vgg16.ot +``` + +Here is some example content and output images using [the Starry Night](https://en.wikipedia.org/wiki/The_Starry_Night) +as a style image. + +![20 Fenchurch Street](https://raw.githubusercontent.com/LaurentMazare/ocaml-torch/master/examples/neural_transfer/20fen.jpg) +![Starry 20 Fen](https://raw.githubusercontent.com/LaurentMazare/ocaml-torch/master/examples/neural_transfer/20fen-starry.jpg) + +([original image source](https://commons.wikimedia.org/wiki/File:Walkie-Talkie_-_Sept_2015.jpg)) + +### Neural Style Transfer + +The Neural Style Transfer algorithm works by gradient descent starting from an +initial image -- we use the content image as a starting point but could also +use random noise. + +This gradient descent aims at minimizing a loss composed of two parts: the +style loss evaluates how much our current image is similar to the style image +in terms of style; the content loss focuses on whether our current +image has a content close to the reference content image. + +In order to define these distances we use a pre-trained convolutional neural +network. This network is run on the current image as well as on the style and +content images. Intuitively the features extracted by the lower layers are +more about style and the features extracted by the upper layers about content. + +The content loss is computed using the mean-square error on the extracted +features for some upper layers. This error helps ensuring that the extracted +features happen at the same on-screen places between the content and the +current images. + +For the style loss we do not really care about spatial location of the style +features. So rather than the mean-square error we use some Gram matrix to +define a loss that is location independent. + +```rust +fn gram_matrix(m: &Tensor) -> Tensor { + let (a, b, c, d) = m.size4().unwrap(); + let m = m.view(&[a * b, c * d]); + let g = m.matmul(&m.tr()); + g / (a * b * c * d) +} + +fn style_loss(m1: &Tensor, m2: &Tensor) -> Tensor { + gram_matrix(m1).mse_loss(&gram_matrix(m2), 1) +} +``` + +### Loading the Model and Images + +We start by creating a *device* which can be either a CPU or a GPU depending on +what is available. + +```rust + let device = Device::cuda_if_available(); +``` + +Then we create a variable store on this device, this variable store will be +used to hold the pre-trained variables for the VGG-16 model. Using this +variable store we create the model and load the weights from the weight file. +Finally we *freeze* the variable store ensuring that the model weights are not +modified when running the optimizer later in the process. + +```rust + let mut net_vs = tch::nn::VarStore::new(device); + let net = vgg::vgg16(&net_vs.root(), imagenet::CLASS_COUNT); + net_vs.load(weights_file)?; + net_vs.freeze(); +``` + +Next we load the style and content image, and move them to the device. Note +that the `unsqueeze` method is used to add a batch-dimension of size 1 so that +the model can be run on these images. + +```rust + let style_img = imagenet::load_image(style_img)? + .unsqueeze(0) + .to_device(device); + let content_img = imagenet::load_image(content_img)? + .unsqueeze(0) + .to_device(device); +``` + +### Running the Model on the Content and Style Images + +We now run the model on the style and content images. These calls returns some +vector containing the extracted features for each of the model layers. +```rust + let style_layers = net.forward_all_t(&style_img, false, Some(max_layer)); + let content_layers = net.forward_all_t(&content_img, false, Some(max_layer)); +``` + +As we will use gradient descent to optimize an image, we create a second +variable store to hold this image as a variable. The initial value for this +variable is obtained by copying the content image. + +```rust + let vs = nn::VarStore::new(device); + let input_var = vs.root().var_copy("img", &content_img); +``` + +### Gradient Descent Loop + +We use Adam for optimization: creating an optimizer requires to pass it the +variable store that we want to optimize on. + +```rust + let opt = nn::Adam::default().build(&vs, LEARNING_RATE)?; +``` + +In the gradient descent loop the style and content losses are computed, +summed together, and run an Adam optimization step is performed. +We also regularly print the current loss value and write the current image +to a file. + +```rust + for step_idx in 1..(1 + TOTAL_STEPS) { + // ... compute style_loss and content_loss ... + let loss = style_loss * STYLE_WEIGHT + content_loss; + opt.backward_step(&loss); + if step_idx % 1000 == 0 { + println!("{} {}", step_idx, f64::from(loss)); + imagenet::save_image(&input_var, &format!("out{}.jpg", step_idx))?; + } + } +``` + +In order to compute the losses, the model is evaluated on the current image. +The style loss is then extracted from the style layers and the content loss from +the content layers. + +```rust + let input_layers = net.forward_all_t(&input_var, false, Some(max_layer)); + let style_loss: Tensor = STYLE_INDEXES + .iter() + .map(|&i| style_loss(&input_layers[i], &style_layers[i])) + .sum(); + let content_loss: Tensor = CONTENT_INDEXES + .iter() + .map(|&i| input_layers[i].mse_loss(&content_layers[i], 1)) + .sum(); +``` diff --git a/v2/dependency-patches/tch-0.24.0/examples/neural-style-transfer/main.rs b/v2/dependency-patches/tch-0.24.0/examples/neural-style-transfer/main.rs new file mode 100644 index 0000000000..0951e51243 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/neural-style-transfer/main.rs @@ -0,0 +1,72 @@ +// This is inspired by the Neural Style tutorial from PyTorch.org +// https://pytorch.org/tutorials/advanced/neural_style_tutorial.html +// The pre-trained weights for the VGG16 model can be downloaded from: +// https://github.com/LaurentMazare/tch-rs/releases/download/mw/vgg16.ot +use anyhow::{bail, Result}; +use tch::vision::{imagenet, vgg}; +use tch::{nn, nn::OptimizerConfig, Device, Tensor}; + +const STYLE_WEIGHT: f64 = 1e6; +const LEARNING_RATE: f64 = 1e-1; +const TOTAL_STEPS: i64 = 3000; +const STYLE_INDEXES: [usize; 5] = [0, 2, 5, 7, 10]; +const CONTENT_INDEXES: [usize; 1] = [7]; + +fn gram_matrix(m: &Tensor) -> Tensor { + let (a, b, c, d) = m.size4().unwrap(); + let m = m.view([a * b, c * d]); + let g = m.matmul(&m.tr()); + g / (a * b * c * d) +} + +fn style_loss(m1: &Tensor, m2: &Tensor) -> Tensor { + gram_matrix(m1).mse_loss(&gram_matrix(m2), tch::Reduction::Mean) +} + +pub fn main() -> Result<()> { + let device = Device::cuda_if_available(); + let args: Vec<_> = std::env::args().collect(); + let (style_img, content_img, weights) = match args.as_slice() { + [_, s, c, w] => (s.to_owned(), c.to_owned(), w.to_owned()), + _ => bail!("usage: main style.jpg content.jpg vgg16.ot"), + }; + + let mut net_vs = tch::nn::VarStore::new(device); + let net = vgg::vgg16(&net_vs.root(), imagenet::CLASS_COUNT); + net_vs.load(&weights).unwrap_or_else(|_| panic!("Could not load weights file {}", &weights)); + net_vs.freeze(); + + let style_img = imagenet::load_image(&style_img) + .unwrap_or_else(|_| panic!("Could not load the style file {}", &style_img)) + .unsqueeze(0) + .to_device(device); + let content_img = imagenet::load_image(&content_img) + .unwrap_or_else(|_| panic!("Could not load the content file {}", &content_img)) + .unsqueeze(0) + .to_device(device); + let max_layer = STYLE_INDEXES.iter().max().unwrap() + 1; + let style_layers = net.forward_all_t(&style_img, false, Some(max_layer)); + let content_layers = net.forward_all_t(&content_img, false, Some(max_layer)); + + let vs = nn::VarStore::new(device); + let input_var = vs.root().var_copy("img", &content_img); + let mut opt = nn::Adam::default().build(&vs, LEARNING_RATE)?; + + for step_idx in 1..(1 + TOTAL_STEPS) { + let input_layers = net.forward_all_t(&input_var, false, Some(max_layer)); + let style_loss: Tensor = + STYLE_INDEXES.iter().map(|&i| style_loss(&input_layers[i], &style_layers[i])).sum(); + let content_loss: Tensor = CONTENT_INDEXES + .iter() + .map(|&i| input_layers[i].mse_loss(&content_layers[i], tch::Reduction::Mean)) + .sum(); + let loss = style_loss * STYLE_WEIGHT + content_loss; + opt.backward_step(&loss); + if step_idx % 1000 == 0 { + println!("{} {}", step_idx, f64::try_from(loss)?); + imagenet::save_image(&input_var, format!("out{step_idx}.jpg"))?; + } + } + + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/pretrained-models/main.rs b/v2/dependency-patches/tch-0.24.0/examples/pretrained-models/main.rs new file mode 100644 index 0000000000..3984bfda36 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/pretrained-models/main.rs @@ -0,0 +1,81 @@ +// This example illustrates how to use pre-trained vision models. +// model to get the imagenet label for some image. +// +// The pre-trained weight files containing the pre-trained weights can be found here: +// https://github.com/LaurentMazare/tch-rs/releases/download/mw/resnet18.ot +// https://github.com/LaurentMazare/tch-rs/releases/download/mw/resnet34.ot +// https://github.com/LaurentMazare/tch-rs/releases/download/mw/densenet121.ot +// https://github.com/LaurentMazare/tch-rs/releases/download/mw/vgg13.ot +// https://github.com/LaurentMazare/tch-rs/releases/download/mw/vgg16.ot +// https://github.com/LaurentMazare/tch-rs/releases/download/mw/vgg19.ot +// https://github.com/LaurentMazare/tch-rs/releases/download/mw/squeezenet1_0.ot +// https://github.com/LaurentMazare/tch-rs/releases/download/mw/squeezenet1_1.ot +// https://github.com/LaurentMazare/tch-rs/releases/download/mw/alexnet.ot +// https://github.com/LaurentMazare/tch-rs/releases/download/mw/inception-v3.ot +// https://github.com/LaurentMazare/tch-rs/releases/download/mw/mobilenet-v2.ot +// https://github.com/LaurentMazare/ocaml-torch/releases/download/v0.1-unstable/efficientnet-b0.safetensors +// https://github.com/LaurentMazare/ocaml-torch/releases/download/v0.1-unstable/efficientnet-b1.safetensors +// https://github.com/LaurentMazare/ocaml-torch/releases/download/v0.1-unstable/efficientnet-b2.safetensors +// https://github.com/LaurentMazare/ocaml-torch/releases/download/v0.1-unstable/efficientnet-b3.safetensors +// https://github.com/LaurentMazare/ocaml-torch/releases/download/v0.1-unstable/efficientnet-b4.safetensors +// https://github.com/LaurentMazare/ocaml-torch/releases/download/v0.1-unstable/convmixer1536_20.ot +// https://github.com/LaurentMazare/ocaml-torch/releases/download/v0.1-unstable/convmixer1024_20.ot +// In order to obtain the dinov2 weights, e.g. dinov2_vits14.safetensors, run the +// src/vision/export_dinov2.py +use anyhow::{bail, Context, Result}; +use tch::nn::ModuleT; +use tch::vision::{ + alexnet, convmixer, densenet, dinov2, efficientnet, imagenet, inception, mobilenet, resnet, + squeezenet, vgg, +}; + +pub fn main() -> Result<()> { + let args: Vec<_> = std::env::args().collect(); + let (weights, image) = match args.as_slice() { + [_, w, i] => (std::path::Path::new(w), i.to_owned()), + _ => bail!("usage: main resnet18.ot image.jpg"), + }; + // Load the image file and resize it to the usual imagenet dimension of 224x224. + let image = imagenet::load_image_and_resize224(image)?; + + // Create the model and load the weights from the file. + let mut vs = tch::nn::VarStore::new(tch::Device::Cpu); + let net: Box = + match weights.file_stem().context("no stem")?.to_str().context("invalid stem")? { + "resnet18" => Box::new(resnet::resnet18(&vs.root(), imagenet::CLASS_COUNT)), + "resnet34" => Box::new(resnet::resnet34(&vs.root(), imagenet::CLASS_COUNT)), + "densenet121" => Box::new(densenet::densenet121(&vs.root(), imagenet::CLASS_COUNT)), + "vgg13" => Box::new(vgg::vgg13(&vs.root(), imagenet::CLASS_COUNT)), + "vgg16" => Box::new(vgg::vgg16(&vs.root(), imagenet::CLASS_COUNT)), + "vgg19" => Box::new(vgg::vgg19(&vs.root(), imagenet::CLASS_COUNT)), + "squeezenet1_0" => Box::new(squeezenet::v1_0(&vs.root(), imagenet::CLASS_COUNT)), + "squeezenet1_1" => Box::new(squeezenet::v1_1(&vs.root(), imagenet::CLASS_COUNT)), + "alexnet" => Box::new(alexnet::alexnet(&vs.root(), imagenet::CLASS_COUNT)), + "inception-v3" => Box::new(inception::v3(&vs.root(), imagenet::CLASS_COUNT)), + "mobilenet-v2" => Box::new(mobilenet::v2(&vs.root(), imagenet::CLASS_COUNT)), + "efficientnet-b0" => Box::new(efficientnet::b0(&vs.root(), imagenet::CLASS_COUNT)), + // Maybe the higher resolution models should be handled differently. + "efficientnet-b1" => Box::new(efficientnet::b1(&vs.root(), imagenet::CLASS_COUNT)), + "efficientnet-b2" => Box::new(efficientnet::b2(&vs.root(), imagenet::CLASS_COUNT)), + "efficientnet-b3" => Box::new(efficientnet::b3(&vs.root(), imagenet::CLASS_COUNT)), + "efficientnet-b4" => Box::new(efficientnet::b4(&vs.root(), imagenet::CLASS_COUNT)), + "efficientnet-b5" => Box::new(efficientnet::b5(&vs.root(), imagenet::CLASS_COUNT)), + "efficientnet-b6" => Box::new(efficientnet::b6(&vs.root(), imagenet::CLASS_COUNT)), + "efficientnet-b7" => Box::new(efficientnet::b7(&vs.root(), imagenet::CLASS_COUNT)), + "convmixer1536_20" => Box::new(convmixer::c1536_20(&vs.root(), imagenet::CLASS_COUNT)), + "convmixer1024_20" => Box::new(convmixer::c1024_20(&vs.root(), imagenet::CLASS_COUNT)), + "dinov2_vits14" => Box::new(dinov2::vit_small(&vs.root())), + s => bail!("unknown model for {s}, use a weight file named e.g. resnet18.ot"), + }; + vs.load(weights)?; + + // Apply the forward pass of the model to get the logits. + let output = + net.forward_t(&image.unsqueeze(0), /* train= */ false).softmax(-1, tch::Kind::Float); // Convert to probability. + + // Print the top 5 categories for this image. + for (probability, class) in imagenet::top(&output, 5).iter() { + println!("{:50} {:5.2}%", class, 100.0 * probability) + } + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/README.md b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/README.md new file mode 100644 index 0000000000..7fe1c1735e --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/README.md @@ -0,0 +1,36 @@ +# Reinforcement Learning Examples + +These examples illustrate how to implement a couple reinforcement learning +algorithms to play Atari games. +This uses the [OpenAI Gym](https://github.com/openai/gym) through its Python +api so the gym Python package has to be installed. +The Python library is executed in Rust using +[rust-cpython](https://github.com/dgrunwald/rust-cpython). + +The policy gradient example uses the `CartPole-v0` environment. It can be +run using the following command. + +```bash +cargo run --example reinforcement-learning --features=python pg +``` + +The A2C example can use any Atari game supported in the OpenAI gym. +Training can be launched by running: + +```bash +cargo run --example reinforcement-learning --features=python a2c +``` + +This produces some weight files in the current directory. + +If you get some Python import error when using conda, this can +be related to `rust-cpython` not being able to use the Python +runtime as a shared library. `rust-cpython` uses +`Py_ENABLE_SHARED` to determine the link mode which does not +seem to work properly with conda as reported in this +[issue](https://github.com/conda/conda-build/issues/2738). + +## A2C Agent Playing Breakout +[![Breakout](https://img.youtube.com/vi/Zk6j7fC1C6M/0.jpg)](https://www.youtube.com/watch?v=Zk6j7fC1C6M) +## A2C Agent Playing SpaceInvaders +[![SpaceInvaders](https://img.youtube.com/vi/p16n4w3aE8k/0.jpg)](https://www.youtube.com/watch?v=p16n4w3aE8k) diff --git a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/a2c.rs b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/a2c.rs new file mode 100644 index 0000000000..a5b8ef533f --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/a2c.rs @@ -0,0 +1,171 @@ +/* Advantage Actor Critic (A2C) model. + A2C is a synchronous variant of Asynchronous the Advantage Actor Critic (A3C) + model introduced by DeepMind in https://arxiv.org/abs/1602.01783 + + See https://blog.openai.com/baselines-acktr-a2c/ for a reference + python implementation. +*/ +use super::vec_gym_env::VecGymEnv; +use tch::kind::{FLOAT_CPU, INT64_CPU}; +use tch::{nn, nn::OptimizerConfig, Kind::Float, Tensor}; + +const ENV_NAME: &str = "SpaceInvadersNoFrameskip-v4"; +const NPROCS: i64 = 16; +const NSTEPS: i64 = 5; +const NSTACK: i64 = 4; +const UPDATES: i64 = 1000000; + +type Model = Box (Tensor, Tensor)>; + +fn model(p: &nn::Path, nact: i64) -> Model { + let stride = |s| nn::ConvConfig { stride: s, ..Default::default() }; + let seq = nn::seq() + .add(nn::conv2d(p / "c1", NSTACK, 32, 8, stride(4))) + .add_fn(|xs| xs.relu()) + .add(nn::conv2d(p / "c2", 32, 64, 4, stride(2))) + .add_fn(|xs| xs.relu()) + .add(nn::conv2d(p / "c3", 64, 64, 3, stride(1))) + .add_fn(|xs| xs.relu().flat_view()) + .add(nn::linear(p / "l1", 3136, 512, Default::default())) + .add_fn(|xs| xs.relu()); + let critic = nn::linear(p / "cl", 512, 1, Default::default()); + let actor = nn::linear(p / "al", 512, nact, Default::default()); + let device = p.device(); + Box::new(move |xs: &Tensor| { + let xs = xs.to_device(device).apply(&seq); + (xs.apply(&critic), xs.apply(&actor)) + }) +} + +#[derive(Debug)] +struct FrameStack { + data: Tensor, + nprocs: i64, + nstack: i64, +} + +impl FrameStack { + fn new(nprocs: i64, nstack: i64) -> FrameStack { + FrameStack { data: Tensor::zeros([nprocs, nstack, 84, 84], FLOAT_CPU), nprocs, nstack } + } + + fn update<'a>(&'a mut self, img: &Tensor, masks: Option<&Tensor>) -> &'a Tensor { + if let Some(masks) = masks { + self.data *= masks.view([self.nprocs, 1, 1, 1]) + }; + let slice = |i| self.data.narrow(1, i, 1); + for i in 1..self.nstack { + slice(i - 1).copy_(&slice(i)) + } + slice(self.nstack - 1).copy_(img); + &self.data + } +} + +/// Trains an agent using A2C. +pub fn train() -> cpython::PyResult<()> { + let env = VecGymEnv::new(ENV_NAME, None, NPROCS)?; + println!("action space: {}", env.action_space()); + println!("observation space: {:?}", env.observation_space()); + + let device = tch::Device::cuda_if_available(); + let vs = nn::VarStore::new(device); + let model = model(&vs.root(), env.action_space()); + let mut opt = nn::Adam::default().build(&vs, 1e-4).unwrap(); + + let mut sum_rewards = Tensor::zeros([NPROCS], FLOAT_CPU); + let mut total_rewards = 0f64; + let mut total_episodes = 0f64; + + let mut frame_stack = FrameStack::new(NPROCS, NSTACK); + let _ = frame_stack.update(&env.reset()?, None); + let s_states = Tensor::zeros([NSTEPS + 1, NPROCS, NSTACK, 84, 84], FLOAT_CPU); + for update_index in 0..UPDATES { + s_states.get(0).copy_(&s_states.get(-1)); + let s_values = Tensor::zeros([NSTEPS, NPROCS], FLOAT_CPU); + let s_rewards = Tensor::zeros([NSTEPS, NPROCS], FLOAT_CPU); + let s_actions = Tensor::zeros([NSTEPS, NPROCS], INT64_CPU); + let s_masks = Tensor::zeros([NSTEPS, NPROCS], FLOAT_CPU); + for s in 0..NSTEPS { + let (critic, actor) = tch::no_grad(|| model(&s_states.get(s))); + let probs = actor.softmax(-1, Float); + let actions = probs.multinomial(1, true).squeeze_dim(-1); + let step = env.step(Vec::::try_from(&actions).unwrap())?; + + sum_rewards += &step.reward; + total_rewards += f64::try_from((&sum_rewards * &step.is_done).sum(Float)).unwrap(); + total_episodes += f64::try_from(step.is_done.sum(Float)).unwrap(); + + let masks = Tensor::from(1f32) - step.is_done; + sum_rewards *= &masks; + let obs = frame_stack.update(&step.obs, Some(&masks)); + s_actions.get(s).copy_(&actions); + s_values.get(s).copy_(&critic.squeeze_dim(-1)); + s_states.get(s + 1).copy_(obs); + s_rewards.get(s).copy_(&step.reward); + s_masks.get(s).copy_(&masks); + } + let s_returns = { + let r = Tensor::zeros([NSTEPS + 1, NPROCS], FLOAT_CPU); + let critic = tch::no_grad(|| model(&s_states.get(-1)).0); + r.get(-1).copy_(&critic.view([NPROCS])); + for s in (0..NSTEPS).rev() { + let r_s = s_rewards.get(s) + r.get(s + 1) * s_masks.get(s) * 0.99; + r.get(s).copy_(&r_s); + } + r + }; + let (critic, actor) = + model(&s_states.narrow(0, 0, NSTEPS).view([NSTEPS * NPROCS, NSTACK, 84, 84])); + let critic = critic.view([NSTEPS, NPROCS]); + let actor = actor.view([NSTEPS, NPROCS, -1]); + let log_probs = actor.log_softmax(-1, Float); + let probs = actor.softmax(-1, Float); + let action_log_probs = { + let index = s_actions.unsqueeze(-1).to_device(device); + log_probs.gather(2, &index, false).squeeze_dim(-1) + }; + let dist_entropy = (-log_probs * probs).sum_dim_intlist(-1, false, Float).mean(Float); + let advantages = s_returns.narrow(0, 0, NSTEPS).to_device(device) - critic; + let value_loss = (&advantages * &advantages).mean(Float); + let action_loss = (-advantages.detach() * action_log_probs).mean(Float); + let loss = value_loss * 0.5 + action_loss - dist_entropy * 0.01; + opt.backward_step_clip(&loss, 0.5); + if update_index > 0 && update_index % 500 == 0 { + println!("{} {:.0} {}", update_index, total_episodes, total_rewards / total_episodes); + total_rewards = 0.; + total_episodes = 0.; + } + if update_index > 0 && update_index % 10000 == 0 { + if let Err(err) = vs.save(format!("a2c{update_index}.ot")) { + println!("error while saving {err}") + } + } + } + Ok(()) +} + +/// Samples some trajectory for a trained agent, the observation frames are stored in /dev/shm. +pub fn sample>(weight_file: T) -> cpython::PyResult<()> { + let env = VecGymEnv::new(ENV_NAME, Some("/dev/shm"), 1)?; + println!("action space: {}", env.action_space()); + println!("observation space: {:?}", env.observation_space()); + + let mut vs = nn::VarStore::new(tch::Device::Cpu); + let model = model(&vs.root(), env.action_space()); + vs.load(weight_file).unwrap(); + + let mut frame_stack = FrameStack::new(1, NSTACK); + let mut obs = frame_stack.update(&env.reset()?, None); + + for _index in 0..5000 { + let (_critic, actor) = tch::no_grad(|| model(obs)); + let probs = actor.softmax(-1, Float); + let actions = probs.multinomial(1, true).squeeze_dim(-1); + let step = env.step(Vec::::try_from(&actions).unwrap())?; + + let masks = Tensor::from(1f32) - step.is_done; + obs = frame_stack.update(&step.obs, Some(&masks)); + } + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/atari_wrappers.py b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/atari_wrappers.py new file mode 100644 index 0000000000..ad1d2704e2 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/atari_wrappers.py @@ -0,0 +1,308 @@ +import gym +import numpy as np +from collections import deque +from PIL import Image +from multiprocessing import Process, Pipe + +# atari_wrappers.py +class NoopResetEnv(gym.Wrapper): + def __init__(self, env, noop_max=30): + """Sample initial states by taking random number of no-ops on reset. + No-op is assumed to be action 0. + """ + gym.Wrapper.__init__(self, env) + self.noop_max = noop_max + self.override_num_noops = None + assert env.unwrapped.get_action_meanings()[0] == 'NOOP' + + def reset(self): + """ Do no-op action for a number of steps in [1, noop_max].""" + self.env.reset() + if self.override_num_noops is not None: + noops = self.override_num_noops + else: + noops = self.unwrapped.np_random.integers(1, self.noop_max + 1) #pylint: disable=E1101 + assert noops > 0 + obs = None + for _ in range(noops): + obs, _, done, _ = self.env.step(0) + if done: + obs = self.env.reset() + return obs + +class FireResetEnv(gym.Wrapper): + def __init__(self, env): + """Take action on reset for environments that are fixed until firing.""" + gym.Wrapper.__init__(self, env) + assert env.unwrapped.get_action_meanings()[1] == 'FIRE' + assert len(env.unwrapped.get_action_meanings()) >= 3 + + def reset(self): + self.env.reset() + obs, _, done, _ = self.env.step(1) + if done: + self.env.reset() + obs, _, done, _ = self.env.step(2) + if done: + self.env.reset() + return obs + +class ImageSaver(gym.Wrapper): + def __init__(self, env, img_path, rank): + gym.Wrapper.__init__(self, env) + self._cnt = 0 + self._img_path = img_path + self._rank = rank + + def step(self, action): + step_result = self.env.step(action) + obs, _, _, _ = step_result + img = Image.fromarray(obs, 'RGB') + img.save('%s/out%d-%05d.png' % (self._img_path, self._rank, self._cnt)) + self._cnt += 1 + return step_result + +class EpisodicLifeEnv(gym.Wrapper): + def __init__(self, env): + """Make end-of-life == end-of-episode, but only reset on true game over. + Done by DeepMind for the DQN and co. since it helps value estimation. + """ + gym.Wrapper.__init__(self, env) + self.lives = 0 + self.was_real_done = True + + def step(self, action): + obs, reward, done, info = self.env.step(action) + self.was_real_done = done + # check current lives, make loss of life terminal, + # then update lives to handle bonus lives + lives = self.env.unwrapped.ale.lives() + if lives < self.lives and lives > 0: + # for Qbert somtimes we stay in lives == 0 condtion for a few frames + # so its important to keep lives > 0, so that we only reset once + # the environment advertises done. + done = True + self.lives = lives + return obs, reward, done, info + + def reset(self): + """Reset only when lives are exhausted. + This way all states are still reachable even though lives are episodic, + and the learner need not know about any of this behind-the-scenes. + """ + if self.was_real_done: + obs = self.env.reset() + else: + # no-op step to advance from terminal/lost life state + obs, _, _, _ = self.env.step(0) + self.lives = self.env.unwrapped.ale.lives() + return obs + +class MaxAndSkipEnv(gym.Wrapper): + def __init__(self, env, skip=4): + """Return only every `skip`-th frame""" + gym.Wrapper.__init__(self, env) + # most recent raw observations (for max pooling across time steps) + self._obs_buffer = deque(maxlen=2) + self._skip = skip + + def step(self, action): + """Repeat action, sum reward, and max over last observations.""" + total_reward = 0.0 + done = None + for _ in range(self._skip): + obs, reward, done, info = self.env.step(action) + self._obs_buffer.append(obs) + total_reward += reward + if done: + break + max_frame = np.max(np.stack(self._obs_buffer), axis=0) + + return max_frame, total_reward, done, info + + def reset(self): + """Clear past frame buffer and init. to first obs. from inner env.""" + self._obs_buffer.clear() + obs = self.env.reset() + self._obs_buffer.append(obs) + return obs + +class ClipRewardEnv(gym.RewardWrapper): + def reward(self, reward): + """Bin reward to {+1, 0, -1} by its sign.""" + return np.sign(reward) + +class WarpFrame(gym.ObservationWrapper): + def __init__(self, env): + """Warp frames to 84x84 as done in the Nature paper and later work.""" + gym.ObservationWrapper.__init__(self, env) + self.res = 84 + self.observation_space = gym.spaces.Box(low=0, high=255, shape=(self.res, self.res, 1), dtype='uint8') + + def observation(self, obs): + frame = np.dot(obs.astype('float32'), np.array([0.299, 0.587, 0.114], 'float32')) + frame = np.array(Image.fromarray(frame).resize((self.res, self.res), + resample=Image.BILINEAR), dtype=np.uint8) + return frame.reshape((self.res, self.res, 1)) + +class FrameStack(gym.Wrapper): + def __init__(self, env, k): + """Buffer observations and stack across channels (last axis).""" + gym.Wrapper.__init__(self, env) + self.k = k + self.frames = deque([], maxlen=k) + shp = env.observation_space.shape + assert shp[2] == 1 # can only stack 1-channel frames + self.observation_space = gym.spaces.Box(low=0, high=255, shape=(shp[0], shp[1], k), dtype='uint8') + + def reset(self): + """Clear buffer and re-fill by duplicating the first observation.""" + ob = self.env.reset() + for _ in range(self.k): self.frames.append(ob) + return self.observation() + + def step(self, action): + ob, reward, done, info = self.env.step(action) + self.frames.append(ob) + return self.observation(), reward, done, info + + def observation(self): + assert len(self.frames) == self.k + return np.concatenate(self.frames, axis=2) + +def wrap_deepmind(env, episode_life=True, clip_rewards=True): + """Configure environment for DeepMind-style Atari. + + Note: this does not include frame stacking!""" + assert 'NoFrameskip' in env.spec.id # required for DeepMind-style skip + if episode_life: + env = EpisodicLifeEnv(env) + env = NoopResetEnv(env, noop_max=30) + env = MaxAndSkipEnv(env, skip=4) + if 'FIRE' in env.unwrapped.get_action_meanings(): + env = FireResetEnv(env) + env = WarpFrame(env) + if clip_rewards: + env = ClipRewardEnv(env) + return env + +# envs.py +def make_env(env_id, img_dir, seed, rank): + def _thunk(): + env = gym.make(env_id) + env.seed(seed + rank) + if img_dir is not None: + env = ImageSaver(env, img_dir, rank) + env = wrap_deepmind(env) + env = WrapPyTorch(env) + return env + + return _thunk + +class WrapPyTorch(gym.ObservationWrapper): + def __init__(self, env=None): + super(WrapPyTorch, self).__init__(env) + self.observation_space = gym.spaces.Box(0.0, 1.0, [1, 84, 84], dtype='float32') + + def observation(self, observation): + return observation.transpose(2, 0, 1) + +# vecenv.py +class VecEnv(object): + """ + Vectorized environment base class + """ + def step(self, vac): + """ + Apply sequence of actions to sequence of environments + actions -> (observations, rewards, news) + + where 'news' is a boolean vector indicating whether each element is new. + """ + raise NotImplementedError + def reset(self): + """ + Reset all environments + """ + raise NotImplementedError + def close(self): + pass + +# subproc_vec_env.py +def worker(remote, env_fn_wrapper): + env = env_fn_wrapper.x() + while True: + cmd, data = remote.recv() + if cmd == 'step': + ob, reward, done, info = env.step(data) + if done: + ob = env.reset() + remote.send((ob, reward, done, info)) + elif cmd == 'reset': + ob = env.reset() + remote.send(ob) + elif cmd == 'close': + remote.close() + break + elif cmd == 'get_spaces': + remote.send((env.action_space, env.observation_space)) + else: + raise NotImplementedError + +class CloudpickleWrapper(object): + """ + Uses cloudpickle to serialize contents (otherwise multiprocessing tries to use pickle) + """ + def __init__(self, x): + self.x = x + def __getstate__(self): + import cloudpickle + return cloudpickle.dumps(self.x) + def __setstate__(self, ob): + import pickle + self.x = pickle.loads(ob) + +class SubprocVecEnv(VecEnv): + def __init__(self, env_fns): + """ + envs: list of gym environments to run in subprocesses + """ + nenvs = len(env_fns) + self.remotes, self.work_remotes = zip(*[Pipe() for _ in range(nenvs)]) + self.ps = [Process(target=worker, args=(work_remote, CloudpickleWrapper(env_fn))) + for (work_remote, env_fn) in zip(self.work_remotes, env_fns)] + for p in self.ps: + p.start() + + self.remotes[0].send(('get_spaces', None)) + self.action_space, self.observation_space = self.remotes[0].recv() + + + def step(self, actions): + for remote, action in zip(self.remotes, actions): + remote.send(('step', action)) + results = [remote.recv() for remote in self.remotes] + obs, rews, dones, infos = zip(*results) + return np.stack(obs), np.stack(rews), np.stack(dones), infos + + def reset(self): + for remote in self.remotes: + remote.send(('reset', None)) + return np.stack([remote.recv() for remote in self.remotes]) + + def close(self): + for remote in self.remotes: + remote.send(('close', None)) + for p in self.ps: + p.join() + + @property + def num_envs(self): + return len(self.remotes) + +# Create the environment. +def make(env_name, img_dir, num_processes): + envs = SubprocVecEnv([ + make_env(env_name, img_dir, 1337, i) for i in range(num_processes) + ]) + return envs diff --git a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/ddpg.rs b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/ddpg.rs new file mode 100644 index 0000000000..743411bde7 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/ddpg.rs @@ -0,0 +1,343 @@ +/* Deep Deterministic Policy Gradient. + + Continuous control with deep reinforcement learning, Lillicrap et al. 2015 + https://arxiv.org/abs/1509.02971 + + See https://spinningup.openai.com/en/latest/algorithms/ddpg.html for a + reference python implementation. +*/ +use super::gym_env::GymEnv; +use tch::{ + kind::{FLOAT_CPU, INT64_CPU}, + nn, + nn::OptimizerConfig, + Device, + Kind::Float, + Tensor, +}; + +// The impact of the q value of the next state on the current state's q value. +const GAMMA: f64 = 0.99; +// The weight for updating the target networks. +const TAU: f64 = 0.005; +// The capacity of the replay buffer used for sampling training data. +const REPLAY_BUFFER_CAPACITY: usize = 100_000; +// The training batch size for each training iteration. +const TRAINING_BATCH_SIZE: usize = 100; +// The total number of episodes. +const MAX_EPISODES: usize = 100; +// The maximum length of an episode. +const EPISODE_LENGTH: usize = 200; +// The number of training iterations after one episode finishes. +const TRAINING_ITERATIONS: usize = 200; + +// Ornstein-Uhlenbeck process parameters. +const MU: f64 = 0.0; +const THETA: f64 = 0.15; +const SIGMA: f64 = 0.1; + +const ACTOR_LEARNING_RATE: f64 = 1e-4; +const CRITIC_LEARNING_RATE: f64 = 1e-3; + +struct OuNoise { + mu: f64, + theta: f64, + sigma: f64, + state: Tensor, +} + +impl OuNoise { + fn new(mu: f64, theta: f64, sigma: f64, num_actions: usize) -> Self { + let state = Tensor::ones([num_actions as _], FLOAT_CPU); + Self { mu, theta, sigma, state } + } + + fn sample(&mut self) -> &Tensor { + let dx = self.theta * (self.mu - &self.state) + + self.sigma * Tensor::randn(self.state.size(), FLOAT_CPU); + self.state += dx; + &self.state + } +} + +struct ReplayBuffer { + obs: Tensor, + next_obs: Tensor, + rewards: Tensor, + actions: Tensor, + capacity: usize, + len: usize, + i: usize, +} + +impl ReplayBuffer { + fn new(capacity: usize, num_obs: usize, num_actions: usize) -> Self { + Self { + obs: Tensor::zeros([capacity as _, num_obs as _], FLOAT_CPU), + next_obs: Tensor::zeros([capacity as _, num_obs as _], FLOAT_CPU), + rewards: Tensor::zeros([capacity as _, 1], FLOAT_CPU), + actions: Tensor::zeros([capacity as _, num_actions as _], FLOAT_CPU), + capacity, + len: 0, + i: 0, + } + } + + fn push(&mut self, obs: &Tensor, actions: &Tensor, reward: &Tensor, next_obs: &Tensor) { + let i = self.i % self.capacity; + self.obs.get(i as _).copy_(obs); + self.rewards.get(i as _).copy_(reward); + self.actions.get(i as _).copy_(actions); + self.next_obs.get(i as _).copy_(next_obs); + self.i += 1; + if self.len < self.capacity { + self.len += 1; + } + } + + fn random_batch(&self, batch_size: usize) -> Option<(Tensor, Tensor, Tensor, Tensor)> { + if self.len < 3 { + return None; + } + + let batch_size = batch_size.min(self.len - 1); + let batch_indexes = Tensor::randint((self.len - 2) as _, [batch_size as _], INT64_CPU); + + let states = self.obs.index_select(0, &batch_indexes); + let next_states = self.next_obs.index_select(0, &batch_indexes); + let actions = self.actions.index_select(0, &batch_indexes); + let rewards = self.rewards.index_select(0, &batch_indexes); + + Some((states, actions, rewards, next_states)) + } +} + +struct Actor { + var_store: nn::VarStore, + network: nn::Sequential, + device: Device, + num_obs: usize, + num_actions: usize, + opt: nn::Optimizer, + learning_rate: f64, +} + +impl Clone for Actor { + fn clone(&self) -> Self { + let mut new = Self::new(self.num_obs, self.num_actions, self.learning_rate); + new.var_store.copy(&self.var_store).unwrap(); + new + } +} + +impl Actor { + fn new(num_obs: usize, num_actions: usize, learning_rate: f64) -> Self { + let var_store = nn::VarStore::new(tch::Device::Cpu); + let opt = nn::Adam::default().build(&var_store, learning_rate).unwrap(); + let p = &var_store.root(); + Self { + network: nn::seq() + .add(nn::linear(p / "al1", num_obs as _, 400, Default::default())) + .add_fn(|xs| xs.relu()) + .add(nn::linear(p / "al2", 400, 300, Default::default())) + .add_fn(|xs| xs.relu()) + .add(nn::linear(p / "al3", 300, num_actions as _, Default::default())) + .add_fn(|xs| xs.tanh()), + device: p.device(), + num_obs, + num_actions, + var_store, + opt, + learning_rate, + } + } + + fn forward(&self, obs: &Tensor) -> Tensor { + obs.to_device(self.device).apply(&self.network) + } +} + +struct Critic { + var_store: nn::VarStore, + network: nn::Sequential, + device: Device, + num_obs: usize, + num_actions: usize, + opt: nn::Optimizer, + learning_rate: f64, +} + +impl Clone for Critic { + fn clone(&self) -> Self { + let mut new = Self::new(self.num_obs, self.num_actions, self.learning_rate); + new.var_store.copy(&self.var_store).unwrap(); + new + } +} + +impl Critic { + fn new(num_obs: usize, num_actions: usize, learning_rate: f64) -> Self { + let var_store = nn::VarStore::new(tch::Device::Cpu); + let opt = nn::Adam::default().build(&var_store, 1e-3).unwrap(); + let p = &var_store.root(); + Self { + network: nn::seq() + .add(nn::linear(p / "cl1", (num_obs + num_actions) as _, 400, Default::default())) + .add_fn(|xs| xs.relu()) + .add(nn::linear(p / "cl2", 400, 300, Default::default())) + .add_fn(|xs| xs.relu()) + .add(nn::linear(p / "cl3", 300, 1, Default::default())), + device: p.device(), + var_store, + num_obs, + num_actions, + opt, + learning_rate, + } + } + + fn forward(&self, obs: &Tensor, actions: &Tensor) -> Tensor { + let xs = Tensor::cat(&[actions.copy(), obs.copy()], 1); + xs.to_device(self.device).apply(&self.network) + } +} + +fn track(dest: &mut nn::VarStore, src: &nn::VarStore, tau: f64) { + tch::no_grad(|| { + for (dest, src) in + dest.trainable_variables().iter_mut().zip(src.trainable_variables().iter()) + { + dest.copy_(&(tau * src + (1.0 - tau) * &*dest)); + } + }) +} + +struct Agent { + actor: Actor, + actor_target: Actor, + + critic: Critic, + critic_target: Critic, + + replay_buffer: ReplayBuffer, + + ou_noise: OuNoise, + + train: bool, + + gamma: f64, + tau: f64, +} + +impl Agent { + fn new( + actor: Actor, + critic: Critic, + ou_noise: OuNoise, + replay_buffer_capacity: usize, + train: bool, + gamma: f64, + tau: f64, + ) -> Self { + let actor_target = actor.clone(); + let critic_target = critic.clone(); + let replay_buffer = + ReplayBuffer::new(replay_buffer_capacity, actor.num_obs, actor.num_actions); + Self { + actor, + actor_target, + critic, + critic_target, + replay_buffer, + ou_noise, + train, + gamma, + tau, + } + } + + fn actions(&mut self, obs: &Tensor) -> Tensor { + let mut actions = tch::no_grad(|| self.actor.forward(obs)); + if self.train { + actions += self.ou_noise.sample(); + } + actions + } + + fn remember(&mut self, obs: &Tensor, actions: &Tensor, reward: &Tensor, next_obs: &Tensor) { + self.replay_buffer.push(obs, actions, reward, next_obs); + } + + fn train(&mut self, batch_size: usize) { + let (states, actions, rewards, next_states) = + match self.replay_buffer.random_batch(batch_size) { + Some(v) => v, + _ => return, // We don't have enough samples for training yet. + }; + + let mut q_target = + self.critic_target.forward(&next_states, &self.actor_target.forward(&next_states)); + q_target = rewards + (self.gamma * q_target).detach(); + + let q = self.critic.forward(&states, &actions); + + let diff = q_target - q; + let critic_loss = (&diff * &diff).mean(Float); + + self.critic.opt.zero_grad(); + critic_loss.backward(); + self.critic.opt.step(); + + let actor_loss = -self.critic.forward(&states, &self.actor.forward(&states)).mean(Float); + + self.actor.opt.zero_grad(); + actor_loss.backward(); + self.actor.opt.step(); + + track(&mut self.critic_target.var_store, &self.critic.var_store, self.tau); + track(&mut self.actor_target.var_store, &self.actor.var_store, self.tau); + } +} + +pub fn run() -> cpython::PyResult<()> { + let env = GymEnv::new("Pendulum-v1")?; + println!("action space: {}", env.action_space()); + println!("observation space: {:?}", env.observation_space()); + + let num_obs = env.observation_space().iter().product::() as usize; + let num_actions = env.action_space() as usize; + + let actor = Actor::new(num_obs, num_actions, ACTOR_LEARNING_RATE); + let critic = Critic::new(num_obs, num_actions, CRITIC_LEARNING_RATE); + let ou_noise = OuNoise::new(MU, THETA, SIGMA, num_actions); + let mut agent = Agent::new(actor, critic, ou_noise, REPLAY_BUFFER_CAPACITY, true, GAMMA, TAU); + + for episode in 0..MAX_EPISODES { + let mut obs = env.reset()?; + + let mut total_reward = 0.0; + for _ in 0..EPISODE_LENGTH { + let mut actions = 2.0 * f64::try_from(agent.actions(&obs)).unwrap(); + actions = actions.clamp(-2.0, 2.0); + + let action_vec = vec![actions]; + let step = env.step(&action_vec)?; + total_reward += step.reward; + + agent.remember(&obs, &actions.into(), &step.reward.into(), &step.obs); + + if step.is_done { + break; + } + obs = step.obs; + } + + println!("episode {episode} with total reward of {total_reward}"); + + for _ in 0..TRAINING_ITERATIONS { + agent.train(TRAINING_BATCH_SIZE); + } + } + + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/gym_env.rs b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/gym_env.rs new file mode 100644 index 0000000000..f11c13304c --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/gym_env.rs @@ -0,0 +1,78 @@ +//! Wrappers around the Python API of the OpenAI gym. +use cpython::{NoArgs, ObjectProtocol, PyObject, PyResult, Python, ToPyObject}; +use tch::Tensor; + +/// The return value for a step. +#[derive(Debug)] +pub struct Step { + pub obs: Tensor, + pub action: A, + pub reward: f64, + pub is_done: bool, +} + +impl Step { + /// Returns a copy of this step changing the observation tensor. + pub fn copy_with_obs(&self, obs: &Tensor) -> Step { + Step { obs: obs.copy(), action: self.action, reward: self.reward, is_done: self.is_done } + } +} + +/// An OpenAI Gym session. +pub struct GymEnv { + env: PyObject, + action_space: i64, + observation_space: Vec, +} + +impl GymEnv { + /// Creates a new session of the specified OpenAI Gym environment. + pub fn new(name: &str) -> PyResult { + let gil = Python::acquire_gil(); + let py = gil.python(); + let gym = py.import("gym")?; + let env = gym.call(py, "make", (name,), None)?; + let _ = env.call_method(py, "seed", (42,), None)?; + let action_space = env.getattr(py, "action_space")?; + let action_space = if let Ok(val) = action_space.getattr(py, "n") { + val.extract(py)? + } else { + let action_space: Vec = action_space.getattr(py, "shape")?.extract(py)?; + action_space[0] + }; + let observation_space = env.getattr(py, "observation_space")?; + let observation_space = observation_space.getattr(py, "shape")?.extract(py)?; + Ok(GymEnv { env, action_space, observation_space }) + } + + /// Resets the environment, returning the observation tensor. + pub fn reset(&self) -> PyResult { + let gil = Python::acquire_gil(); + let py = gil.python(); + let obs = self.env.call_method(py, "reset", NoArgs, None)?; + Ok(Tensor::from_slice(&obs.extract::>(py)?)) + } + + /// Applies an environment step using the specified action. + pub fn step(&self, action: A) -> PyResult> { + let gil = Python::acquire_gil(); + let py = gil.python(); + let step = self.env.call_method(py, "step", (action,), None)?; + Ok(Step { + obs: Tensor::from_slice(&step.get_item(py, 0)?.extract::>(py)?), + reward: step.get_item(py, 1)?.extract(py)?, + is_done: step.get_item(py, 2)?.extract(py)?, + action, + }) + } + + /// Returns the number of allowed actions for this environment. + pub fn action_space(&self) -> i64 { + self.action_space + } + + /// Returns the shape of the observation tensors. + pub fn observation_space(&self) -> &[i64] { + &self.observation_space + } +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/main.rs b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/main.rs new file mode 100644 index 0000000000..724d804f95 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/main.rs @@ -0,0 +1,20 @@ +mod a2c; +mod ddpg; +mod gym_env; +mod policy_gradient; +mod ppo; +mod vec_gym_env; + +fn main() -> cpython::PyResult<()> { + let a: Vec = std::env::args().collect(); + match a.iter().map(|x| x.as_str()).collect::>().as_slice() { + [_, "a2c"] => a2c::train()?, + [_, "a2c-sample", weight_file] => a2c::sample(weight_file)?, + [_, "pg"] => policy_gradient::run()?, + [_, "ppo"] => ppo::train()?, + [_, "ppo-sample", weight_file] => ppo::sample(weight_file)?, + [_, "ddpg"] => ddpg::run()?, + _ => println!("usage: main pg|a2c|a2c-sample|ppo|ppo-sample|ddpg"), + } + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/policy_gradient.rs b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/policy_gradient.rs new file mode 100644 index 0000000000..81945ce8e3 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/policy_gradient.rs @@ -0,0 +1,84 @@ +// Policy gradient example. +// This uses OpenAI Gym environment through rust-cpython. +// +// This is adapted from OpenAI Spinning Up series: +// https://spinningup.openai.com/en/latest/spinningup/rl_intro3.html +// A TensorFlow Python implementation can be found here: +// https://github.com/openai/spinningup/blob/master/spinup/examples/pg_math/2_rtg_pg.py + +use super::gym_env::{GymEnv, Step}; +use tch::{nn, nn::OptimizerConfig, Kind::Float, Tensor}; + +fn model(p: &nn::Path, input_shape: &[i64], nact: i64) -> impl nn::Module { + let nin = input_shape.iter().product::(); + nn::seq() + .add(nn::linear(p / "lin1", nin, 32, Default::default())) + .add_fn(|xs| xs.tanh()) + .add(nn::linear(p / "lin2", 32, nact, Default::default())) +} + +fn accumulate_rewards(steps: &[Step]) -> Vec { + let mut rewards: Vec = steps.iter().map(|s| s.reward).collect(); + let mut acc_reward = 0f64; + for (i, reward) in rewards.iter_mut().enumerate().rev() { + if steps[i].is_done { + acc_reward = 0.0; + } + acc_reward += *reward; + *reward = acc_reward; + } + rewards +} + +/// Trains an agent using the policy gradient algorithm. +pub fn run() -> cpython::PyResult<()> { + let env = GymEnv::new("CartPole-v0")?; + println!("action space: {:?}", env.action_space()); + println!("observation space: {:?}", env.observation_space()); + + let vs = nn::VarStore::new(tch::Device::Cpu); + let model = model(&vs.root(), env.observation_space(), env.action_space()); + let mut opt = nn::Adam::default().build(&vs, 1e-2).unwrap(); + + for epoch_idx in 0..50 { + let mut obs = env.reset()?; + let mut steps: Vec> = vec![]; + // Perform some rollouts with the current model. + loop { + let action = tch::no_grad(|| { + obs.unsqueeze(0).apply(&model).softmax(1, Float).multinomial(1, true) + }); + let action = i64::try_from(action).unwrap(); + let step = env.step(action)?; + steps.push(step.copy_with_obs(&obs)); + obs = if step.is_done { env.reset()? } else { step.obs }; + if step.is_done && steps.len() > 5000 { + break; + } + } + let sum_r: f64 = steps.iter().map(|s| s.reward).sum(); + let episodes: i64 = steps.iter().map(|s| s.is_done as i64).sum(); + println!( + "epoch: {:<3} episodes: {:<5} avg reward per episode: {:.2}", + epoch_idx, + episodes, + sum_r / episodes as f64 + ); + + // Train the model via policy gradient on the rollout data. + let batch_size = steps.len() as i64; + let actions: Vec = steps.iter().map(|s| s.action).collect(); + let actions = Tensor::from_slice(&actions).unsqueeze(1); + let rewards = accumulate_rewards(&steps); + let rewards = Tensor::from_slice(&rewards).to_kind(Float); + let action_mask = + Tensor::zeros([batch_size, 2], tch::kind::FLOAT_CPU).scatter_value(1, &actions, 1.0); + let obs: Vec = steps.into_iter().map(|s| s.obs).collect(); + let logits = Tensor::stack(&obs, 0).apply(&model); + let log_probs = + (action_mask * logits.log_softmax(1, Float)).sum_dim_intlist(1, false, Float); + let loss = -(rewards * log_probs).mean(Float); + opt.backward_step(&loss) + } + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/ppo.rs b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/ppo.rs new file mode 100644 index 0000000000..d1b462eda6 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/ppo.rs @@ -0,0 +1,181 @@ +/* Proximal Policy Optimization (PPO) model. + + Proximal Policy Optimization Algorithms, Schulman et al. 2017 + https://arxiv.org/abs/1707.06347 + + See https://spinningup.openai.com/en/latest/algorithms/ppo.html for a + reference python implementation. +*/ +use super::vec_gym_env::VecGymEnv; +use tch::kind::{FLOAT_CPU, INT64_CPU}; +use tch::{nn, nn::OptimizerConfig, Kind, Tensor}; + +const ENV_NAME: &str = "SpaceInvadersNoFrameskip-v4"; +const NPROCS: i64 = 8; +const NSTEPS: i64 = 256; +const NSTACK: i64 = 4; +const UPDATES: i64 = 1000000; +const OPTIM_BATCHSIZE: i64 = 64; +const OPTIM_EPOCHS: i64 = 4; + +type Model = Box (Tensor, Tensor)>; + +fn model(p: &nn::Path, nact: i64) -> Model { + let stride = |s| nn::ConvConfig { stride: s, ..Default::default() }; + let seq = nn::seq() + .add(nn::conv2d(p / "c1", NSTACK, 32, 8, stride(4))) + .add_fn(|xs| xs.relu()) + .add(nn::conv2d(p / "c2", 32, 64, 4, stride(2))) + .add_fn(|xs| xs.relu()) + .add(nn::conv2d(p / "c3", 64, 64, 3, stride(1))) + .add_fn(|xs| xs.relu().flat_view()) + .add(nn::linear(p / "l1", 3136, 512, Default::default())) + .add_fn(|xs| xs.relu()); + let critic = nn::linear(p / "cl", 512, 1, Default::default()); + let actor = nn::linear(p / "al", 512, nact, Default::default()); + let device = p.device(); + Box::new(move |xs: &Tensor| { + let xs = xs.to_device(device).apply(&seq); + (xs.apply(&critic), xs.apply(&actor)) + }) +} + +#[derive(Debug)] +struct FrameStack { + data: Tensor, + nprocs: i64, + nstack: i64, +} + +impl FrameStack { + fn new(nprocs: i64, nstack: i64) -> FrameStack { + FrameStack { data: Tensor::zeros([nprocs, nstack, 84, 84], FLOAT_CPU), nprocs, nstack } + } + + fn update<'a>(&'a mut self, img: &Tensor, masks: Option<&Tensor>) -> &'a Tensor { + if let Some(masks) = masks { + self.data *= masks.view([self.nprocs, 1, 1, 1]) + }; + let slice = |i| self.data.narrow(1, i, 1); + for i in 1..self.nstack { + slice(i - 1).copy_(&slice(i)) + } + slice(self.nstack - 1).copy_(img); + &self.data + } +} + +pub fn train() -> cpython::PyResult<()> { + let env = VecGymEnv::new(ENV_NAME, None, NPROCS)?; + println!("action space: {}", env.action_space()); + println!("observation space: {:?}", env.observation_space()); + + let device = tch::Device::cuda_if_available(); + let vs = nn::VarStore::new(device); + let model = model(&vs.root(), env.action_space()); + let mut opt = nn::Adam::default().build(&vs, 1e-4).unwrap(); + + let mut sum_rewards = Tensor::zeros([NPROCS], FLOAT_CPU); + let mut total_rewards = 0f64; + let mut total_episodes = 0f64; + + let mut frame_stack = FrameStack::new(NPROCS, NSTACK); + let _ = frame_stack.update(&env.reset()?, None); + let s_states = Tensor::zeros([NSTEPS + 1, NPROCS, NSTACK, 84, 84], FLOAT_CPU); + + let train_size = NSTEPS * NPROCS; + for update_index in 0..UPDATES { + s_states.get(0).copy_(&s_states.get(-1)); + let s_values = Tensor::zeros([NSTEPS, NPROCS], FLOAT_CPU); + let s_rewards = Tensor::zeros([NSTEPS, NPROCS], FLOAT_CPU); + let s_actions = Tensor::zeros([NSTEPS, NPROCS], INT64_CPU); + let s_masks = Tensor::zeros([NSTEPS, NPROCS], FLOAT_CPU); + for s in 0..NSTEPS { + let (critic, actor) = tch::no_grad(|| model(&s_states.get(s))); + let probs = actor.softmax(-1, Kind::Float); + let actions = probs.multinomial(1, true).squeeze_dim(-1); + let step = env.step(Vec::::try_from(&actions).unwrap())?; + + sum_rewards += &step.reward; + total_rewards += + f64::try_from((&sum_rewards * &step.is_done).sum(Kind::Float)).unwrap(); + total_episodes += f64::try_from(step.is_done.sum(Kind::Float)).unwrap(); + + let masks = Tensor::from(1f32) - step.is_done; + sum_rewards *= &masks; + let obs = frame_stack.update(&step.obs, Some(&masks)); + s_actions.get(s).copy_(&actions); + s_values.get(s).copy_(&critic.squeeze_dim(-1)); + s_states.get(s + 1).copy_(obs); + s_rewards.get(s).copy_(&step.reward); + s_masks.get(s).copy_(&masks); + } + let states = s_states.narrow(0, 0, NSTEPS).view([train_size, NSTACK, 84, 84]); + let returns = { + let r = Tensor::zeros([NSTEPS + 1, NPROCS], FLOAT_CPU); + let critic = tch::no_grad(|| model(&s_states.get(-1)).0); + r.get(-1).copy_(&critic.view([NPROCS])); + for s in (0..NSTEPS).rev() { + let r_s = s_rewards.get(s) + r.get(s + 1) * s_masks.get(s) * 0.99; + r.get(s).copy_(&r_s); + } + r.narrow(0, 0, NSTEPS).view([train_size, 1]) + }; + let actions = s_actions.view([train_size]); + for _index in 0..OPTIM_EPOCHS { + let batch_indexes = Tensor::randint(train_size, [OPTIM_BATCHSIZE], INT64_CPU); + let states = states.index_select(0, &batch_indexes); + let actions = actions.index_select(0, &batch_indexes); + let returns = returns.index_select(0, &batch_indexes); + let (critic, actor) = model(&states); + let log_probs = actor.log_softmax(-1, Kind::Float); + let probs = actor.softmax(-1, Kind::Float); + let action_log_probs = { + let index = actions.unsqueeze(-1).to_device(device); + log_probs.gather(-1, &index, false).squeeze_dim(-1) + }; + let dist_entropy = + (-log_probs * probs).sum_dim_intlist(-1, false, Kind::Float).mean(Kind::Float); + let advantages = returns.to_device(device) - critic; + let value_loss = (&advantages * &advantages).mean(Kind::Float); + let action_loss = (-advantages.detach() * action_log_probs).mean(Kind::Float); + let loss = value_loss * 0.5 + action_loss - dist_entropy * 0.01; + opt.backward_step_clip(&loss, 0.5); + } + if update_index > 0 && update_index % 25 == 0 { + println!("{} {:.0} {}", update_index, total_episodes, total_rewards / total_episodes); + total_rewards = 0.; + total_episodes = 0.; + } + if update_index > 0 && update_index % 1000 == 0 { + if let Err(err) = vs.save(format!("trpo{update_index}.ot")) { + println!("error while saving {err}") + } + } + } + Ok(()) +} + +pub fn sample>(weight_file: T) -> cpython::PyResult<()> { + let env = VecGymEnv::new(ENV_NAME, Some("/dev/shm"), 1)?; + println!("action space: {}", env.action_space()); + println!("observation space: {:?}", env.observation_space()); + + let mut vs = nn::VarStore::new(tch::Device::Cpu); + let model = model(&vs.root(), env.action_space()); + vs.load(weight_file).unwrap(); + + let mut frame_stack = FrameStack::new(1, NSTACK); + let mut obs = frame_stack.update(&env.reset()?, None); + + for _index in 0..5000 { + let (_critic, actor) = tch::no_grad(|| model(obs)); + let probs = actor.softmax(-1, Kind::Float); + let actions = probs.multinomial(1, true).squeeze_dim(-1); + let step = env.step(Vec::::try_from(&actions).unwrap())?; + + let masks = Tensor::from(1f32) - step.is_done; + obs = frame_stack.update(&step.obs, Some(&masks)); + } + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/vec_gym_env.rs b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/vec_gym_env.rs new file mode 100644 index 0000000000..e13d378740 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/vec_gym_env.rs @@ -0,0 +1,66 @@ +// Vectorized version of the gym environment. +use cpython::{buffer::PyBuffer, NoArgs, ObjectProtocol, PyObject, PyResult, Python}; +use tch::Tensor; + +#[derive(Debug)] +pub struct Step { + pub obs: Tensor, + pub reward: Tensor, + pub is_done: Tensor, +} + +pub struct VecGymEnv { + env: PyObject, + action_space: i64, + observation_space: Vec, +} + +impl VecGymEnv { + pub fn new(name: &str, img_dir: Option<&str>, nprocesses: i64) -> PyResult { + let gil = Python::acquire_gil(); + let py = gil.python(); + let sys = py.import("sys")?; + let path = sys.get(py, "path")?; + let _ = path.call_method(py, "append", ("examples/reinforcement-learning",), None)?; + let gym = py.import("atari_wrappers")?; + let env = gym.call(py, "make", (name, img_dir, nprocesses), None)?; + let action_space = env.getattr(py, "action_space")?; + let action_space = action_space.getattr(py, "n")?.extract(py)?; + let observation_space = env.getattr(py, "observation_space")?; + let observation_space: Vec = observation_space.getattr(py, "shape")?.extract(py)?; + let observation_space = + [vec![nprocesses].as_slice(), observation_space.as_slice()].concat(); + Ok(VecGymEnv { env, action_space, observation_space }) + } + + pub fn reset(&self) -> PyResult { + let gil = Python::acquire_gil(); + let py = gil.python(); + let obs = self.env.call_method(py, "reset", NoArgs, None)?; + let obs = obs.call_method(py, "flatten", NoArgs, None)?; + let obs = Tensor::from_slice(&obs.extract::>(py)?); + Ok(obs.view_(&self.observation_space)) + } + + pub fn step(&self, action: Vec) -> PyResult { + let gil = Python::acquire_gil(); + let py = gil.python(); + let step = self.env.call_method(py, "step", (action,), None)?; + let obs = step.get_item(py, 0)?.call_method(py, "flatten", NoArgs, None)?; + let obs_buffer = PyBuffer::get(py, &obs)?; + let obs_vec: Vec = obs_buffer.to_vec(py)?; + let obs = + Tensor::from_slice(&obs_vec).view_(&self.observation_space).to_kind(tch::Kind::Float); + let reward = Tensor::from_slice(&step.get_item(py, 1)?.extract::>(py)?); + let is_done = Tensor::from_slice(&step.get_item(py, 2)?.extract::>(py)?); + Ok(Step { obs, reward, is_done }) + } + + pub fn action_space(&self) -> i64 { + self.action_space + } + + pub fn observation_space(&self) -> &[i64] { + &self.observation_space + } +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/stable-diffusion/README.md b/v2/dependency-patches/tch-0.24.0/examples/stable-diffusion/README.md new file mode 100644 index 0000000000..af9f4d8463 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/stable-diffusion/README.md @@ -0,0 +1,11 @@ +![rusty robot holding a torch](media/robot13.jpg) + +_A rusty robot holding a fire torch_, generated by stable diffusion using Rust and libtorch. + + +__IMPORTANT NOTE__ + +This example has now been moved in its own repo +[diffusers-rs](https://github.com/LaurentMazare/diffusers-rs). +The new repo includes new functionalities like inpainting, pre-packaged weights +so that it's not necessary to run the conversion steps below anymore, etc. diff --git a/v2/dependency-patches/tch-0.24.0/examples/stable-diffusion/main.rs b/v2/dependency-patches/tch-0.24.0/examples/stable-diffusion/main.rs new file mode 100644 index 0000000000..47ab967c10 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/stable-diffusion/main.rs @@ -0,0 +1,2512 @@ +// Stable Diffusion implementation inspired: +// - Huggingface's amazing diffuser Python api: https://huggingface.co/blog/annotated-diffusion +// - Huggingface's (also amazing) blog post: https://huggingface.co/blog/annotated-diffusion +// - The "Grokking Stable Diffusion" notebook by Jonathan Whitaker. +// https://colab.research.google.com/drive/1dlgggNa5Mz8sEAGU0wFCHhGLFooW_pf1?usp=sharing +// +// In order to run this, first download the following and extract the file in data/ +// +// mkdir -p data && cd data +// wget https://github.com/openai/CLIP/raw/main/clip/bpe_simple_vocab_16e6.txt.gz +// gunzip bpe_simple_vocab_16e6.txt.gz +// +// Download and convert the weights: +// +// 1. Clip Encoding Weights +// wget https://huggingface.co/openai/clip-vit-large-patch14/resolve/main/pytorch_model.bin +// From python, extract the weights and save them as a .npz file. +// import numpy as np +// import torch +// model = torch.load("./pytorch_model.bin") +// np.savez("./pytorch_model.npz", **{k: v.numpy() for k, v in model.items() if "text_model" in k}) +// +// Then use tensor_tools to convert this to a .ot file that tch can use. +// cargo run --release --example tensor-tools cp ./data/pytorch_model.npz ./data/pytorch_model.ot +// +// 2. VAE and Unet Weights +// https://huggingface.co/CompVis/stable-diffusion-v1-4/blob/main/vae/diffusion_pytorch_model.bin +// https://huggingface.co/CompVis/stable-diffusion-v1-4/blob/main/unet/diffusion_pytorch_model.bin +// +// import numpy as np +// import torch +// model = torch.load("./vae.bin") +// np.savez("./vae.npz", **{k: v.numpy() for k, v in model.items()}) +// model = torch.load("./unet.bin") +// np.savez("./unet.npz", **{k: v.numpy() for k, v in model.items()}) +// +// cargo run --release --example tensor-tools cp ./data/vae.npz ./data/vae.ot +// cargo run --release --example tensor-tools cp ./data/unet.npz ./data/unet.ot +// TODO: fix tensor_tools so that it works properly there. +// TODO: Split this file, probably in a way similar to huggingface/diffusers. +use std::collections::{HashMap, HashSet}; +use std::io::BufRead; +use tch::{kind, nn, nn::Module, Device, Kind, Tensor}; + +// The config details can be found in the "text_config" section of this json file: +// https://huggingface.co/openai/clip-vit-large-patch14/blob/main/config.json +// "hidden_act": "quick_gelu" +const VOCAB_SIZE: i64 = 49408; +const EMBED_DIM: i64 = 768; // a.k.a. config.hidden_size +const INTERMEDIATE_SIZE: i64 = 3072; +const MAX_POSITION_EMBEDDINGS: usize = 77; +const NUM_HIDDEN_LAYERS: i64 = 12; +const NUM_ATTENTION_HEADS: i64 = 12; + +const HEIGHT: i64 = 512; +const WIDTH: i64 = 512; +const GUIDANCE_SCALE: f64 = 7.5; + +const BYTES_TO_UNICODE: [(u8, char); 256] = [ + (33, '!'), + (34, '"'), + (35, '#'), + (36, '$'), + (37, '%'), + (38, '&'), + (39, '\''), + (40, '('), + (41, ')'), + (42, '*'), + (43, '+'), + (44, ','), + (45, '-'), + (46, '.'), + (47, '/'), + (48, '0'), + (49, '1'), + (50, '2'), + (51, '3'), + (52, '4'), + (53, '5'), + (54, '6'), + (55, '7'), + (56, '8'), + (57, '9'), + (58, ':'), + (59, ';'), + (60, '<'), + (61, '='), + (62, '>'), + (63, '?'), + (64, '@'), + (65, 'A'), + (66, 'B'), + (67, 'C'), + (68, 'D'), + (69, 'E'), + (70, 'F'), + (71, 'G'), + (72, 'H'), + (73, 'I'), + (74, 'J'), + (75, 'K'), + (76, 'L'), + (77, 'M'), + (78, 'N'), + (79, 'O'), + (80, 'P'), + (81, 'Q'), + (82, 'R'), + (83, 'S'), + (84, 'T'), + (85, 'U'), + (86, 'V'), + (87, 'W'), + (88, 'X'), + (89, 'Y'), + (90, 'Z'), + (91, '['), + (92, '\\'), + (93, ']'), + (94, '^'), + (95, '_'), + (96, '`'), + (97, 'a'), + (98, 'b'), + (99, 'c'), + (100, 'd'), + (101, 'e'), + (102, 'f'), + (103, 'g'), + (104, 'h'), + (105, 'i'), + (106, 'j'), + (107, 'k'), + (108, 'l'), + (109, 'm'), + (110, 'n'), + (111, 'o'), + (112, 'p'), + (113, 'q'), + (114, 'r'), + (115, 's'), + (116, 't'), + (117, 'u'), + (118, 'v'), + (119, 'w'), + (120, 'x'), + (121, 'y'), + (122, 'z'), + (123, '{'), + (124, '|'), + (125, '}'), + (126, '~'), + (161, '¡'), + (162, '¢'), + (163, '£'), + (164, '¤'), + (165, '¥'), + (166, '¦'), + (167, '§'), + (168, '¨'), + (169, '©'), + (170, 'ª'), + (171, '«'), + (172, '¬'), + (174, '®'), + (175, '¯'), + (176, '°'), + (177, '±'), + (178, '²'), + (179, '³'), + (180, '´'), + (181, 'µ'), + (182, '¶'), + (183, '·'), + (184, '¸'), + (185, '¹'), + (186, 'º'), + (187, '»'), + (188, '¼'), + (189, '½'), + (190, '¾'), + (191, '¿'), + (192, 'À'), + (193, 'Á'), + (194, 'Â'), + (195, 'Ã'), + (196, 'Ä'), + (197, 'Å'), + (198, 'Æ'), + (199, 'Ç'), + (200, 'È'), + (201, 'É'), + (202, 'Ê'), + (203, 'Ë'), + (204, 'Ì'), + (205, 'Í'), + (206, 'Î'), + (207, 'Ï'), + (208, 'Ð'), + (209, 'Ñ'), + (210, 'Ò'), + (211, 'Ó'), + (212, 'Ô'), + (213, 'Õ'), + (214, 'Ö'), + (215, '×'), + (216, 'Ø'), + (217, 'Ù'), + (218, 'Ú'), + (219, 'Û'), + (220, 'Ü'), + (221, 'Ý'), + (222, 'Þ'), + (223, 'ß'), + (224, 'à'), + (225, 'á'), + (226, 'â'), + (227, 'ã'), + (228, 'ä'), + (229, 'å'), + (230, 'æ'), + (231, 'ç'), + (232, 'è'), + (233, 'é'), + (234, 'ê'), + (235, 'ë'), + (236, 'ì'), + (237, 'í'), + (238, 'î'), + (239, 'ï'), + (240, 'ð'), + (241, 'ñ'), + (242, 'ò'), + (243, 'ó'), + (244, 'ô'), + (245, 'õ'), + (246, 'ö'), + (247, '÷'), + (248, 'ø'), + (249, 'ù'), + (250, 'ú'), + (251, 'û'), + (252, 'ü'), + (253, 'ý'), + (254, 'þ'), + (255, 'ÿ'), + (0, 'Ā'), + (1, 'ā'), + (2, 'Ă'), + (3, 'ă'), + (4, 'Ą'), + (5, 'ą'), + (6, 'Ć'), + (7, 'ć'), + (8, 'Ĉ'), + (9, 'ĉ'), + (10, 'Ċ'), + (11, 'ċ'), + (12, 'Č'), + (13, 'č'), + (14, 'Ď'), + (15, 'ď'), + (16, 'Đ'), + (17, 'đ'), + (18, 'Ē'), + (19, 'ē'), + (20, 'Ĕ'), + (21, 'ĕ'), + (22, 'Ė'), + (23, 'ė'), + (24, 'Ę'), + (25, 'ę'), + (26, 'Ě'), + (27, 'ě'), + (28, 'Ĝ'), + (29, 'ĝ'), + (30, 'Ğ'), + (31, 'ğ'), + (32, 'Ġ'), + (127, 'ġ'), + (128, 'Ģ'), + (129, 'ģ'), + (130, 'Ĥ'), + (131, 'ĥ'), + (132, 'Ħ'), + (133, 'ħ'), + (134, 'Ĩ'), + (135, 'ĩ'), + (136, 'Ī'), + (137, 'ī'), + (138, 'Ĭ'), + (139, 'ĭ'), + (140, 'Į'), + (141, 'į'), + (142, 'İ'), + (143, 'ı'), + (144, 'IJ'), + (145, 'ij'), + (146, 'Ĵ'), + (147, 'ĵ'), + (148, 'Ķ'), + (149, 'ķ'), + (150, 'ĸ'), + (151, 'Ĺ'), + (152, 'ĺ'), + (153, 'Ļ'), + (154, 'ļ'), + (155, 'Ľ'), + (156, 'ľ'), + (157, 'Ŀ'), + (158, 'ŀ'), + (159, 'Ł'), + (160, 'ł'), + (173, 'Ń'), +]; + +const PAT: &str = + r"<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+"; + +// This is mostly a Rust rewrite of the original Python CLIP code. +// https://github.com/openai/CLIP/blob/main/clip/simple_tokenizer.py +struct Tokenizer { + re: regex::Regex, + encoder: HashMap, + decoder: HashMap, + bpe_ranks: HashMap<(String, String), usize>, + start_of_text_token: usize, + end_of_text_token: usize, +} + +impl Tokenizer { + fn create>(bpe_path: T) -> anyhow::Result { + let bpe_file = std::fs::File::open(bpe_path)?; + let bpe_lines: Result, _> = std::io::BufReader::new(bpe_file).lines().collect(); + let bpe_lines = bpe_lines?; + let bpe_lines: Result, _> = bpe_lines[1..49152 - 256 - 2 + 1] + .iter() + .map(|line| { + let vs: Vec<_> = line.split_whitespace().collect(); + if vs.len() != 2 { + anyhow::bail!("expected two items got {} '{}'", vs.len(), line) + } + Ok((vs[0].to_string(), vs[1].to_string())) + }) + .collect(); + let bpe_lines = bpe_lines?; + let mut vocab: Vec = Vec::new(); + for (_index, elem) in BYTES_TO_UNICODE { + vocab.push(elem.into()) + } + for (_index, elem) in BYTES_TO_UNICODE { + vocab.push(format!("{elem}")); + } + for elem in bpe_lines.iter() { + vocab.push(format!("{}{}", elem.0, elem.1)) + } + let start_of_text_token = vocab.len(); + vocab.push("<|startoftext|>".to_string()); + let end_of_text_token = vocab.len(); + vocab.push("<|endoftext|>".to_string()); + let encoder: HashMap<_, _> = vocab.into_iter().enumerate().map(|(i, v)| (v, i)).collect(); + let decoder: HashMap<_, _> = encoder.iter().map(|(k, v)| (*v, k.clone())).collect(); + let bpe_ranks: HashMap<_, _> = + bpe_lines.into_iter().enumerate().map(|(i, v)| (v, i)).collect(); + let re = regex::Regex::new(PAT)?; + let tokenizer = + Tokenizer { encoder, re, bpe_ranks, decoder, start_of_text_token, end_of_text_token }; + Ok(tokenizer) + } + + fn get_pairs(word: &[String]) -> HashSet<(String, String)> { + let mut pairs = HashSet::new(); + for (i, v) in word.iter().enumerate() { + if i > 0 { + pairs.insert((word[i - 1].clone(), v.clone())); + } + } + pairs + } + + fn bpe(&self, token: &str) -> Vec { + let mut word: Vec = token.chars().map(|x| x.to_string()).collect(); + if word.is_empty() { + return Vec::new(); + } + let last_index = word.len() - 1; + word[last_index] = format!("{}", word[last_index]); + while word.len() > 1 { + let mut current_min = None; + let pairs = Self::get_pairs(&word); + for p in pairs.iter() { + match self.bpe_ranks.get(p) { + None => {} + Some(v) => { + let should_replace = match current_min { + None => true, + Some((current_min, _)) => v < current_min, + }; + if should_replace { + current_min = Some((v, p)) + } + } + } + } + let (first, second) = match current_min { + None => break, + Some((_v, (first, second))) => (first, second), + }; + let mut new_word = vec![]; + let mut index = 0; + while index < word.len() { + let w = &word[index]; + if index + 1 < word.len() && w == first && &word[index + 1] == second { + new_word.push(format!("{first}{second}")); + index += 2 + } else { + new_word.push(w.clone()); + index += 1 + } + } + word = new_word + } + word.iter().map(|x| *self.encoder.get(x).unwrap()).collect() + } + + fn encode(&self, s: &str, pad_size_to: Option) -> anyhow::Result> { + let s = s.to_lowercase(); + let mut bpe_tokens: Vec = vec![self.start_of_text_token]; + for token in self.re.captures_iter(&s) { + let token = token.get(0).unwrap().as_str(); + bpe_tokens.extend(self.bpe(token)) + } + match pad_size_to { + None => bpe_tokens.push(self.end_of_text_token), + Some(pad_size_to) => { + bpe_tokens.resize_with( + std::cmp::min(bpe_tokens.len(), pad_size_to - 1), + Default::default, + ); + while bpe_tokens.len() < pad_size_to { + bpe_tokens.push(self.end_of_text_token) + } + } + } + Ok(bpe_tokens) + } + + fn decode(&self, tokens: &[usize]) -> String { + let s: String = tokens.iter().map(|token| self.decoder[token].as_str()).collect(); + s.replace("", " ") + } +} + +// CLIP Text Model +// https://github.com/huggingface/transformers/blob/674f750a57431222fa2832503a108df3badf1564/src/transformers/models/clip/modeling_clip.py +#[derive(Debug)] +struct ClipTextEmbeddings { + token_embedding: nn::Embedding, + position_embedding: nn::Embedding, + position_ids: Tensor, +} + +impl ClipTextEmbeddings { + fn new(vs: nn::Path) -> Self { + let token_embedding = + nn::embedding(&vs / "token_embedding", VOCAB_SIZE, EMBED_DIM, Default::default()); + let position_embedding = nn::embedding( + &vs / "position_embedding", + MAX_POSITION_EMBEDDINGS as i64, + EMBED_DIM, + Default::default(), + ); + let position_ids = + Tensor::arange(MAX_POSITION_EMBEDDINGS as i64, (Kind::Int64, vs.device())) + .expand([1, -1], false); + ClipTextEmbeddings { token_embedding, position_embedding, position_ids } + } +} + +impl Module for ClipTextEmbeddings { + fn forward(&self, xs: &Tensor) -> Tensor { + let token_embedding = self.token_embedding.forward(xs); + let position_embedding = self.position_embedding.forward(&self.position_ids); + token_embedding + position_embedding + } +} + +fn quick_gelu(xs: &Tensor) -> Tensor { + xs * (xs * 1.702).sigmoid() +} + +#[derive(Debug)] +struct ClipAttention { + k_proj: nn::Linear, + v_proj: nn::Linear, + q_proj: nn::Linear, + out_proj: nn::Linear, + head_dim: i64, + scale: f64, +} + +impl ClipAttention { + fn new(vs: nn::Path) -> Self { + let k_proj = nn::linear(&vs / "k_proj", EMBED_DIM, EMBED_DIM, Default::default()); + let v_proj = nn::linear(&vs / "v_proj", EMBED_DIM, EMBED_DIM, Default::default()); + let q_proj = nn::linear(&vs / "q_proj", EMBED_DIM, EMBED_DIM, Default::default()); + let out_proj = nn::linear(&vs / "out_proj", EMBED_DIM, EMBED_DIM, Default::default()); + let head_dim = EMBED_DIM / NUM_ATTENTION_HEADS; + let scale = (head_dim as f64).powf(-0.5); + ClipAttention { k_proj, v_proj, q_proj, out_proj, head_dim, scale } + } + + fn shape(&self, xs: &Tensor, seq_len: i64, bsz: i64) -> Tensor { + xs.view((bsz, seq_len, NUM_ATTENTION_HEADS, self.head_dim)).transpose(1, 2).contiguous() + } + + fn forward(&self, xs: &Tensor, causal_attention_mask: &Tensor) -> Tensor { + let (bsz, tgt_len, embed_dim) = xs.size3().unwrap(); + let query_states = xs.apply(&self.q_proj) * self.scale; + let proj_shape = (bsz * NUM_ATTENTION_HEADS, -1, self.head_dim); + let query_states = self.shape(&query_states, tgt_len, bsz).view(proj_shape); + let key_states = self.shape(&xs.apply(&self.k_proj), -1, bsz).view(proj_shape); + let value_states = self.shape(&xs.apply(&self.v_proj), -1, bsz).view(proj_shape); + let attn_weights = query_states.bmm(&key_states.transpose(1, 2)); + + let src_len = key_states.size()[1]; + let attn_weights = + attn_weights.view((bsz, NUM_ATTENTION_HEADS, tgt_len, src_len)) + causal_attention_mask; + let attn_weights = attn_weights.view((bsz * NUM_ATTENTION_HEADS, tgt_len, src_len)); + let attn_weights = attn_weights.softmax(-1, Kind::Float); + + let attn_output = attn_weights.bmm(&value_states); + attn_output + .view((bsz, NUM_ATTENTION_HEADS, tgt_len, self.head_dim)) + .transpose(1, 2) + .reshape([bsz, tgt_len, embed_dim]) + .apply(&self.out_proj) + } +} + +#[derive(Debug)] +struct ClipMlp { + fc1: nn::Linear, + fc2: nn::Linear, +} + +impl ClipMlp { + fn new(vs: nn::Path) -> Self { + let fc1 = nn::linear(&vs / "fc1", EMBED_DIM, INTERMEDIATE_SIZE, Default::default()); + let fc2 = nn::linear(&vs / "fc2", INTERMEDIATE_SIZE, EMBED_DIM, Default::default()); + ClipMlp { fc1, fc2 } + } +} + +impl Module for ClipMlp { + fn forward(&self, xs: &Tensor) -> Tensor { + let xs = xs.apply(&self.fc1); + quick_gelu(&xs).apply(&self.fc2) + } +} + +#[derive(Debug)] +struct ClipEncoderLayer { + self_attn: ClipAttention, + layer_norm1: nn::LayerNorm, + mlp: ClipMlp, + layer_norm2: nn::LayerNorm, +} + +impl ClipEncoderLayer { + fn new(vs: nn::Path) -> Self { + let self_attn = ClipAttention::new(&vs / "self_attn"); + let layer_norm1 = nn::layer_norm(&vs / "layer_norm1", vec![EMBED_DIM], Default::default()); + let mlp = ClipMlp::new(&vs / "mlp"); + let layer_norm2 = nn::layer_norm(&vs / "layer_norm2", vec![EMBED_DIM], Default::default()); + ClipEncoderLayer { self_attn, layer_norm1, mlp, layer_norm2 } + } + + fn forward(&self, xs: &Tensor, causal_attention_mask: &Tensor) -> Tensor { + let residual = xs; + let xs = self.layer_norm1.forward(xs); + let xs = self.self_attn.forward(&xs, causal_attention_mask); + let xs = xs + residual; + + let residual = &xs; + let xs = self.layer_norm2.forward(&xs); + let xs = self.mlp.forward(&xs); + xs + residual + } +} + +#[derive(Debug)] +struct ClipEncoder { + layers: Vec, +} + +impl ClipEncoder { + fn new(vs: nn::Path) -> Self { + let vs = &vs / "layers"; + let mut layers: Vec = Vec::new(); + for index in 0..NUM_HIDDEN_LAYERS { + let layer = ClipEncoderLayer::new(&vs / index); + layers.push(layer) + } + ClipEncoder { layers } + } + + fn forward(&self, xs: &Tensor, causal_attention_mask: &Tensor) -> Tensor { + let mut xs = xs.shallow_clone(); + for layer in self.layers.iter() { + xs = layer.forward(&xs, causal_attention_mask) + } + xs + } +} + +#[derive(Debug)] +struct ClipTextTransformer { + embeddings: ClipTextEmbeddings, + encoder: ClipEncoder, + final_layer_norm: nn::LayerNorm, +} + +impl ClipTextTransformer { + fn new(vs: nn::Path) -> Self { + let vs = &vs / "text_model"; + let embeddings = ClipTextEmbeddings::new(&vs / "embeddings"); + let encoder = ClipEncoder::new(&vs / "encoder"); + let final_layer_norm = + nn::layer_norm(&vs / "final_layer_norm", vec![EMBED_DIM], Default::default()); + ClipTextTransformer { embeddings, encoder, final_layer_norm } + } + + // https://github.com/huggingface/transformers/blob/674f750a57431222fa2832503a108df3badf1564/src/transformers/models/clip/modeling_clip.py#L678 + fn build_causal_attention_mask(bsz: i64, seq_len: i64, device: Device) -> Tensor { + let mut mask = Tensor::ones([bsz, seq_len, seq_len], (Kind::Float, device)); + mask.fill_(f32::MIN as f64).triu_(1).unsqueeze(1) + } +} + +impl Module for ClipTextTransformer { + fn forward(&self, xs: &Tensor) -> Tensor { + let (bsz, seq_len) = xs.size2().unwrap(); + let xs = self.embeddings.forward(xs); + let causal_attention_mask = Self::build_causal_attention_mask(bsz, seq_len, xs.device()); + let xs = self.encoder.forward(&xs, &causal_attention_mask); + xs.apply(&self.final_layer_norm) + } +} + +#[derive(Debug)] +struct GeGlu { + proj: nn::Linear, +} + +impl GeGlu { + fn new(vs: nn::Path, dim_in: i64, dim_out: i64) -> Self { + let proj = nn::linear(&vs / "proj", dim_in, dim_out * 2, Default::default()); + Self { proj } + } +} + +impl Module for GeGlu { + fn forward(&self, xs: &Tensor) -> Tensor { + let hidden_states_and_gate = xs.apply(&self.proj).chunk(2, -1); + &hidden_states_and_gate[0] * hidden_states_and_gate[1].gelu("none") + } +} + +#[derive(Debug)] +struct FeedForward { + project_in: GeGlu, + linear: nn::Linear, +} + +impl FeedForward { + // The glu parameter in the python code is unused? + // https://github.com/huggingface/diffusers/blob/d3d22ce5a894becb951eec03e663951b28d45135/src/diffusers/models/attention.py#L347 + fn new(vs: nn::Path, dim: i64, dim_out: Option, mult: i64) -> Self { + let inner_dim = dim * mult; + let dim_out = dim_out.unwrap_or(dim); + let vs = &vs / "net"; + let project_in = GeGlu::new(&vs / 0, dim, inner_dim); + let linear = nn::linear(&vs / 2, inner_dim, dim_out, Default::default()); + Self { project_in, linear } + } +} + +impl Module for FeedForward { + fn forward(&self, xs: &Tensor) -> Tensor { + xs.apply(&self.project_in).apply(&self.linear) + } +} + +#[derive(Debug)] +struct CrossAttention { + to_q: nn::Linear, + to_k: nn::Linear, + to_v: nn::Linear, + to_out: nn::Linear, + heads: i64, + scale: f64, +} + +impl CrossAttention { + // Defaults should be heads = 8, dim_head = 64, context_dim = None + fn new( + vs: nn::Path, + query_dim: i64, + context_dim: Option, + heads: i64, + dim_head: i64, + ) -> Self { + let no_bias = nn::LinearConfig { bias: false, ..Default::default() }; + let inner_dim = dim_head * heads; + let context_dim = context_dim.unwrap_or(query_dim); + let scale = 1.0 / f64::sqrt(dim_head as f64); + let to_q = nn::linear(&vs / "to_q", query_dim, inner_dim, no_bias); + let to_k = nn::linear(&vs / "to_k", context_dim, inner_dim, no_bias); + let to_v = nn::linear(&vs / "to_v", context_dim, inner_dim, no_bias); + let to_out = nn::linear(&vs / "to_out" / 0, inner_dim, query_dim, Default::default()); + Self { to_q, to_k, to_v, to_out, heads, scale } + } + + fn reshape_heads_to_batch_dim(&self, xs: &Tensor) -> Tensor { + let (batch_size, seq_len, dim) = xs.size3().unwrap(); + xs.reshape([batch_size, seq_len, self.heads, dim / self.heads]) + .permute([0, 2, 1, 3]) + .reshape([batch_size * self.heads, seq_len, dim / self.heads]) + } + + fn reshape_batch_dim_to_heads(&self, xs: &Tensor) -> Tensor { + let (batch_size, seq_len, dim) = xs.size3().unwrap(); + xs.reshape([batch_size / self.heads, self.heads, seq_len, dim]) + .permute([0, 2, 1, 3]) + .reshape([batch_size / self.heads, seq_len, dim * self.heads]) + } + + fn attention(&self, query: &Tensor, key: &Tensor, value: &Tensor) -> Tensor { + let xs = query + .matmul(&(key.transpose(-1, -2) * self.scale)) + .softmax(-1, Kind::Float) + .matmul(value); + self.reshape_batch_dim_to_heads(&xs) + } + + fn forward(&self, xs: &Tensor, context: Option<&Tensor>) -> Tensor { + let query = xs.apply(&self.to_q); + let context = context.unwrap_or(xs); + let key = context.apply(&self.to_k); + let value = context.apply(&self.to_v); + let query = self.reshape_heads_to_batch_dim(&query); + let key = self.reshape_heads_to_batch_dim(&key); + let value = self.reshape_heads_to_batch_dim(&value); + self.attention(&query, &key, &value).apply(&self.to_out) + } +} + +#[derive(Debug)] +struct BasicTransformerBlock { + attn1: CrossAttention, + ff: FeedForward, + attn2: CrossAttention, + norm1: nn::LayerNorm, + norm2: nn::LayerNorm, + norm3: nn::LayerNorm, +} + +impl BasicTransformerBlock { + fn new(vs: nn::Path, dim: i64, n_heads: i64, d_head: i64, context_dim: Option) -> Self { + let attn1 = CrossAttention::new(&vs / "attn1", dim, None, n_heads, d_head); + let ff = FeedForward::new(&vs / "ff", dim, None, 4); + let attn2 = CrossAttention::new(&vs / "attn2", dim, context_dim, n_heads, d_head); + let norm1 = nn::layer_norm(&vs / "norm1", vec![dim], Default::default()); + let norm2 = nn::layer_norm(&vs / "norm2", vec![dim], Default::default()); + let norm3 = nn::layer_norm(&vs / "norm3", vec![dim], Default::default()); + Self { attn1, ff, attn2, norm1, norm2, norm3 } + } + + fn forward(&self, xs: &Tensor, context: Option<&Tensor>) -> Tensor { + let xs = self.attn1.forward(&xs.apply(&self.norm1), None) + xs; + let xs = self.attn2.forward(&xs.apply(&self.norm2), context) + xs; + xs.apply(&self.norm3).apply(&self.ff) + xs + } +} + +#[derive(Debug, Clone, Copy)] +struct SpatialTransformerConfig { + depth: i64, + num_groups: i64, + context_dim: Option, +} + +impl Default for SpatialTransformerConfig { + fn default() -> Self { + Self { depth: 1, num_groups: 32, context_dim: None } + } +} + +#[derive(Debug)] +struct SpatialTransformer { + norm: nn::GroupNorm, + proj_in: nn::Conv2D, + transformer_blocks: Vec, + proj_out: nn::Conv2D, + #[allow(dead_code)] + config: SpatialTransformerConfig, +} + +impl SpatialTransformer { + fn new( + vs: nn::Path, + in_channels: i64, + n_heads: i64, + d_head: i64, + config: SpatialTransformerConfig, + ) -> Self { + let inner_dim = n_heads * d_head; + let group_cfg = nn::GroupNormConfig { eps: 1e-6, affine: true, ..Default::default() }; + let norm = nn::group_norm(&vs / "norm", config.num_groups, in_channels, group_cfg); + let conv_cfg = nn::ConvConfig { stride: 1, padding: 0, ..Default::default() }; + let proj_in = nn::conv2d(&vs / "proj_in", in_channels, inner_dim, 1, conv_cfg); + let mut transformer_blocks = vec![]; + let vs_tb = &vs / "transformer_blocks"; + for index in 0..config.depth { + let tb = BasicTransformerBlock::new( + &vs_tb / index, + inner_dim, + n_heads, + d_head, + config.context_dim, + ); + transformer_blocks.push(tb) + } + let proj_out = nn::conv2d(&vs / "proj_out", inner_dim, in_channels, 1, conv_cfg); + Self { norm, proj_in, transformer_blocks, proj_out, config } + } + + fn forward(&self, xs: &Tensor, context: Option<&Tensor>) -> Tensor { + let (batch, _channel, height, weight) = xs.size4().unwrap(); + let residual = xs; + let xs = xs.apply(&self.norm).apply(&self.proj_in); + let inner_dim = xs.size()[1]; + let mut xs = xs.permute([0, 2, 3, 1]).view((batch, height * weight, inner_dim)); + for block in self.transformer_blocks.iter() { + xs = block.forward(&xs, context) + } + let xs = + xs.view((batch, height, weight, inner_dim)).permute([0, 3, 1, 2]).apply(&self.proj_out); + xs + residual + } +} + +#[derive(Debug, Clone, Copy)] +struct AttentionBlockConfig { + num_head_channels: Option, + num_groups: i64, + rescale_output_factor: f64, + eps: f64, +} + +impl Default for AttentionBlockConfig { + fn default() -> Self { + Self { num_head_channels: None, num_groups: 32, rescale_output_factor: 1., eps: 1e-5 } + } +} + +#[derive(Debug)] +struct AttentionBlock { + group_norm: nn::GroupNorm, + query: nn::Linear, + key: nn::Linear, + value: nn::Linear, + proj_attn: nn::Linear, + channels: i64, + num_heads: i64, + config: AttentionBlockConfig, +} + +impl AttentionBlock { + fn new(vs: nn::Path, channels: i64, config: AttentionBlockConfig) -> Self { + let num_head_channels = config.num_head_channels.unwrap_or(channels); + let num_heads = channels / num_head_channels; + let group_cfg = nn::GroupNormConfig { eps: config.eps, affine: true, ..Default::default() }; + let group_norm = nn::group_norm(&vs / "group_norm", config.num_groups, channels, group_cfg); + let query = nn::linear(&vs / "query", channels, channels, Default::default()); + let key = nn::linear(&vs / "key", channels, channels, Default::default()); + let value = nn::linear(&vs / "value", channels, channels, Default::default()); + let proj_attn = nn::linear(&vs / "proj_attn", channels, channels, Default::default()); + Self { group_norm, query, key, value, proj_attn, channels, num_heads, config } + } + + fn transpose_for_scores(&self, xs: Tensor) -> Tensor { + let (batch, t, _h_times_d) = xs.size3().unwrap(); + xs.view((batch, t, self.num_heads, -1)).permute([0, 2, 1, 3]) + } +} + +impl Module for AttentionBlock { + fn forward(&self, xs: &Tensor) -> Tensor { + let residual = xs; + let (batch, channel, height, width) = xs.size4().unwrap(); + let xs = xs.apply(&self.group_norm).view((batch, channel, height * width)).transpose(1, 2); + + let query_proj = xs.apply(&self.query); + let key_proj = xs.apply(&self.key); + let value_proj = xs.apply(&self.value); + + let query_states = self.transpose_for_scores(query_proj); + let key_states = self.transpose_for_scores(key_proj); + let value_states = self.transpose_for_scores(value_proj); + + let scale = f64::powf((self.channels as f64) / (self.num_heads as f64), -0.25); + let attention_scores = + (query_states * scale).matmul(&(key_states.transpose(-1, -2) * scale)); + let attention_probs = attention_scores.softmax(-1, Kind::Float); + + let xs = attention_probs.matmul(&value_states); + let xs = xs.permute([0, 2, 1, 3]).contiguous(); + let mut new_xs_shape = xs.size(); + new_xs_shape.pop(); + new_xs_shape.pop(); + new_xs_shape.push(self.channels); + + let xs = xs + .view(new_xs_shape.as_slice()) + .apply(&self.proj_attn) + .transpose(-1, -2) + .view((batch, channel, height, width)); + (xs + residual) / self.config.rescale_output_factor + } +} + +#[derive(Debug)] +struct Downsample2D { + conv: Option, + padding: i64, +} + +impl Downsample2D { + fn new( + vs: nn::Path, + in_channels: i64, + use_conv: bool, + out_channels: i64, + padding: i64, + ) -> Self { + let conv = if use_conv { + let config = nn::ConvConfig { stride: 2, padding, ..Default::default() }; + let conv = nn::conv2d(&vs / "conv", in_channels, out_channels, 3, config); + Some(conv) + } else { + None + }; + Downsample2D { conv, padding } + } +} + +impl Module for Downsample2D { + fn forward(&self, xs: &Tensor) -> Tensor { + match &self.conv { + None => xs.avg_pool2d([2, 2], [2, 2], [0, 0], false, true, None), + Some(conv) => { + if self.padding == 0 { + xs.pad([0, 1, 0, 1], "constant", Some(0.)).apply(conv) + } else { + xs.apply(conv) + } + } + } + } +} + +// This does not support the conv-transpose mode. +#[derive(Debug)] +struct Upsample2D { + conv: nn::Conv2D, +} + +impl Upsample2D { + fn new(vs: nn::Path, in_channels: i64, out_channels: i64) -> Self { + let config = nn::ConvConfig { padding: 1, ..Default::default() }; + let conv = nn::conv2d(&vs / "conv", in_channels, out_channels, 3, config); + Self { conv } + } +} + +impl Upsample2D { + fn forward(&self, xs: &Tensor, size: Option<(i64, i64)>) -> Tensor { + let xs = match size { + None => { + // The following does not work and it's tricky to pass no fixed + // dimensions so hack our way around this. + // xs.upsample_nearest2d(&[], Some(2.), Some(2.) + let (_bsize, _channels, h, w) = xs.size4().unwrap(); + xs.upsample_nearest2d([2 * h, 2 * w], Some(2.), Some(2.)) + } + Some((h, w)) => xs.upsample_nearest2d([h, w], None, None), + }; + xs.apply(&self.conv) + } +} + +#[derive(Debug, Clone, Copy)] +struct ResnetBlock2DConfig { + out_channels: Option, + temb_channels: Option, + groups: i64, + groups_out: Option, + eps: f64, + use_in_shortcut: Option, + // non_linearity: silu + output_scale_factor: f64, +} + +impl Default for ResnetBlock2DConfig { + fn default() -> Self { + Self { + out_channels: None, + temb_channels: Some(512), + groups: 32, + groups_out: None, + eps: 1e-6, + use_in_shortcut: None, + output_scale_factor: 1., + } + } +} + +#[derive(Debug)] +struct ResnetBlock2D { + norm1: nn::GroupNorm, + conv1: nn::Conv2D, + norm2: nn::GroupNorm, + conv2: nn::Conv2D, + time_emb_proj: Option, + conv_shortcut: Option, + config: ResnetBlock2DConfig, +} + +impl ResnetBlock2D { + fn new(vs: nn::Path, in_channels: i64, config: ResnetBlock2DConfig) -> Self { + let out_channels = config.out_channels.unwrap_or(in_channels); + let conv_cfg = nn::ConvConfig { stride: 1, padding: 1, ..Default::default() }; + let group_cfg = nn::GroupNormConfig { eps: config.eps, affine: true, ..Default::default() }; + let norm1 = nn::group_norm(&vs / "norm1", config.groups, in_channels, group_cfg); + let conv1 = nn::conv2d(&vs / "conv1", in_channels, out_channels, 3, conv_cfg); + let groups_out = config.groups_out.unwrap_or(config.groups); + let norm2 = nn::group_norm(&vs / "norm2", groups_out, out_channels, group_cfg); + let conv2 = nn::conv2d(&vs / "conv2", out_channels, out_channels, 3, conv_cfg); + let use_in_shortcut = config.use_in_shortcut.unwrap_or(in_channels != out_channels); + let conv_shortcut = if use_in_shortcut { + let conv_cfg = nn::ConvConfig { stride: 1, padding: 0, ..Default::default() }; + Some(nn::conv2d(&vs / "conv_shortcut", in_channels, out_channels, 1, conv_cfg)) + } else { + None + }; + let time_emb_proj = config.temb_channels.map(|temb_channels| { + nn::linear(&vs / "time_emb_proj", temb_channels, out_channels, Default::default()) + }); + Self { norm1, conv1, norm2, conv2, time_emb_proj, config, conv_shortcut } + } + + fn forward(&self, xs: &Tensor, temb: Option<&Tensor>) -> Tensor { + let shortcut_xs = match &self.conv_shortcut { + Some(conv_shortcut) => xs.apply(conv_shortcut), + None => xs.shallow_clone(), + }; + let xs = xs.apply(&self.norm1).silu().apply(&self.conv1); + let xs = match (temb, &self.time_emb_proj) { + (Some(temb), Some(time_emb_proj)) => { + temb.silu().apply(time_emb_proj).unsqueeze(-1).unsqueeze(-1) + xs + } + _ => xs, + }; + let xs = xs.apply(&self.norm2).silu().apply(&self.conv2); + (shortcut_xs + xs) / self.config.output_scale_factor + } +} + +#[derive(Debug, Clone, Copy)] +struct DownEncoderBlock2DConfig { + num_layers: i64, + resnet_eps: f64, + resnet_groups: i64, + output_scale_factor: f64, + add_downsample: bool, + downsample_padding: i64, +} + +impl Default for DownEncoderBlock2DConfig { + fn default() -> Self { + Self { + num_layers: 1, + resnet_eps: 1e-6, + resnet_groups: 32, + output_scale_factor: 1., + add_downsample: true, + downsample_padding: 1, + } + } +} + +#[derive(Debug)] +struct DownEncoderBlock2D { + resnets: Vec, + downsampler: Option, + #[allow(dead_code)] + config: DownEncoderBlock2DConfig, +} + +impl DownEncoderBlock2D { + fn new( + vs: nn::Path, + in_channels: i64, + out_channels: i64, + config: DownEncoderBlock2DConfig, + ) -> Self { + let resnets: Vec<_> = { + let vs = &vs / "resnets"; + let conv_cfg = ResnetBlock2DConfig { + eps: config.resnet_eps, + out_channels: Some(out_channels), + groups: config.resnet_groups, + output_scale_factor: config.output_scale_factor, + temb_channels: None, + ..Default::default() + }; + (0..(config.num_layers)) + .map(|i| { + let in_channels = if i == 0 { in_channels } else { out_channels }; + ResnetBlock2D::new(&vs / i, in_channels, conv_cfg) + }) + .collect() + }; + let downsampler = if config.add_downsample { + let downsample = Downsample2D::new( + &(&vs / "downsamplers") / 0, + out_channels, + true, + out_channels, + config.downsample_padding, + ); + Some(downsample) + } else { + None + }; + Self { resnets, downsampler, config } + } +} + +impl Module for DownEncoderBlock2D { + fn forward(&self, xs: &Tensor) -> Tensor { + let mut xs = xs.shallow_clone(); + for resnet in self.resnets.iter() { + xs = resnet.forward(&xs, None) + } + match &self.downsampler { + Some(downsampler) => xs.apply(downsampler), + None => xs, + } + } +} + +#[derive(Debug, Clone, Copy)] +struct UpDecoderBlock2DConfig { + num_layers: i64, + resnet_eps: f64, + resnet_groups: i64, + output_scale_factor: f64, + add_upsample: bool, +} + +impl Default for UpDecoderBlock2DConfig { + fn default() -> Self { + Self { + num_layers: 1, + resnet_eps: 1e-6, + resnet_groups: 32, + output_scale_factor: 1., + add_upsample: true, + } + } +} + +#[derive(Debug)] +struct UpDecoderBlock2D { + resnets: Vec, + upsampler: Option, + #[allow(dead_code)] + config: UpDecoderBlock2DConfig, +} + +impl UpDecoderBlock2D { + fn new( + vs: nn::Path, + in_channels: i64, + out_channels: i64, + config: UpDecoderBlock2DConfig, + ) -> Self { + let resnets: Vec<_> = { + let vs = &vs / "resnets"; + let conv_cfg = ResnetBlock2DConfig { + out_channels: Some(out_channels), + eps: config.resnet_eps, + groups: config.resnet_groups, + output_scale_factor: config.output_scale_factor, + temb_channels: None, + ..Default::default() + }; + (0..(config.num_layers)) + .map(|i| { + let in_channels = if i == 0 { in_channels } else { out_channels }; + ResnetBlock2D::new(&vs / i, in_channels, conv_cfg) + }) + .collect() + }; + let upsampler = if config.add_upsample { + let upsample = Upsample2D::new(&vs / "upsamplers" / 0, out_channels, out_channels); + Some(upsample) + } else { + None + }; + Self { resnets, upsampler, config } + } +} + +impl Module for UpDecoderBlock2D { + fn forward(&self, xs: &Tensor) -> Tensor { + let mut xs = xs.shallow_clone(); + for resnet in self.resnets.iter() { + xs = resnet.forward(&xs, None) + } + match &self.upsampler { + Some(upsampler) => upsampler.forward(&xs, None), + None => xs, + } + } +} + +#[derive(Debug, Clone, Copy)] +struct UNetMidBlock2DConfig { + num_layers: i64, + resnet_eps: f64, + resnet_groups: Option, + attn_num_head_channels: Option, + // attention_type "default" + output_scale_factor: f64, +} + +impl Default for UNetMidBlock2DConfig { + fn default() -> Self { + Self { + num_layers: 1, + resnet_eps: 1e-6, + resnet_groups: Some(32), + attn_num_head_channels: Some(1), + output_scale_factor: 1., + } + } +} + +#[derive(Debug)] +struct UNetMidBlock2D { + resnet: ResnetBlock2D, + attn_resnets: Vec<(AttentionBlock, ResnetBlock2D)>, + #[allow(dead_code)] + config: UNetMidBlock2DConfig, +} + +impl UNetMidBlock2D { + fn new( + vs: nn::Path, + in_channels: i64, + temb_channels: Option, + config: UNetMidBlock2DConfig, + ) -> Self { + let vs_resnets = &vs / "resnets"; + let vs_attns = &vs / "attentions"; + let resnet_groups = config.resnet_groups.unwrap_or_else(|| i64::min(in_channels / 4, 32)); + let resnet_cfg = ResnetBlock2DConfig { + eps: config.resnet_eps, + groups: resnet_groups, + output_scale_factor: config.output_scale_factor, + temb_channels, + ..Default::default() + }; + let resnet = ResnetBlock2D::new(&vs_resnets / "0", in_channels, resnet_cfg); + let attn_cfg = AttentionBlockConfig { + num_head_channels: config.attn_num_head_channels, + num_groups: resnet_groups, + rescale_output_factor: config.output_scale_factor, + eps: config.resnet_eps, + }; + let mut attn_resnets = vec![]; + for index in 0..config.num_layers { + let attn = AttentionBlock::new(&vs_attns / index, in_channels, attn_cfg); + let resnet = ResnetBlock2D::new(&vs_resnets / (index + 1), in_channels, resnet_cfg); + attn_resnets.push((attn, resnet)) + } + Self { resnet, attn_resnets, config } + } + + fn forward(&self, xs: &Tensor, temb: Option<&Tensor>) -> Tensor { + let mut xs = self.resnet.forward(xs, temb); + for (attn, resnet) in self.attn_resnets.iter() { + xs = resnet.forward(&xs.apply(attn), temb) + } + xs + } +} + +#[derive(Debug, Clone, Copy)] +struct UNetMidBlock2DCrossAttnConfig { + num_layers: i64, + resnet_eps: f64, + resnet_groups: Option, + attn_num_head_channels: i64, + // attention_type "default" + output_scale_factor: f64, + cross_attn_dim: i64, +} + +impl Default for UNetMidBlock2DCrossAttnConfig { + fn default() -> Self { + Self { + num_layers: 1, + resnet_eps: 1e-6, + resnet_groups: Some(32), + attn_num_head_channels: 1, + output_scale_factor: 1., + cross_attn_dim: 1280, + } + } +} + +#[derive(Debug)] +struct UNetMidBlock2DCrossAttn { + resnet: ResnetBlock2D, + attn_resnets: Vec<(SpatialTransformer, ResnetBlock2D)>, + #[allow(dead_code)] + config: UNetMidBlock2DCrossAttnConfig, +} + +impl UNetMidBlock2DCrossAttn { + fn new( + vs: nn::Path, + in_channels: i64, + temb_channels: Option, + config: UNetMidBlock2DCrossAttnConfig, + ) -> Self { + let vs_resnets = &vs / "resnets"; + let vs_attns = &vs / "attentions"; + let resnet_groups = config.resnet_groups.unwrap_or_else(|| i64::min(in_channels / 4, 32)); + let resnet_cfg = ResnetBlock2DConfig { + eps: config.resnet_eps, + groups: resnet_groups, + output_scale_factor: config.output_scale_factor, + temb_channels, + ..Default::default() + }; + let resnet = ResnetBlock2D::new(&vs_resnets / "0", in_channels, resnet_cfg); + let n_heads = config.attn_num_head_channels; + let attn_cfg = SpatialTransformerConfig { + depth: 1, + num_groups: resnet_groups, + context_dim: Some(config.cross_attn_dim), + }; + let mut attn_resnets = vec![]; + for index in 0..config.num_layers { + let attn = SpatialTransformer::new( + &vs_attns / index, + in_channels, + n_heads, + in_channels / n_heads, + attn_cfg, + ); + let resnet = ResnetBlock2D::new(&vs_resnets / (index + 1), in_channels, resnet_cfg); + attn_resnets.push((attn, resnet)) + } + Self { resnet, attn_resnets, config } + } + + fn forward( + &self, + xs: &Tensor, + temb: Option<&Tensor>, + encoder_hidden_states: Option<&Tensor>, + ) -> Tensor { + let mut xs = self.resnet.forward(xs, temb); + for (attn, resnet) in self.attn_resnets.iter() { + xs = resnet.forward(&attn.forward(&xs, encoder_hidden_states), temb) + } + xs + } +} + +#[derive(Debug, Clone)] +struct EncoderConfig { + // down_block_types: DownEncoderBlock2D + block_out_channels: Vec, + layers_per_block: i64, + norm_num_groups: i64, + double_z: bool, +} + +impl Default for EncoderConfig { + fn default() -> Self { + Self { + block_out_channels: vec![64], + layers_per_block: 2, + norm_num_groups: 32, + double_z: true, + } + } +} + +#[derive(Debug)] +struct Encoder { + conv_in: nn::Conv2D, + down_blocks: Vec, + mid_block: UNetMidBlock2D, + conv_norm_out: nn::GroupNorm, + conv_out: nn::Conv2D, + #[allow(dead_code)] + config: EncoderConfig, +} + +impl Encoder { + fn new(vs: nn::Path, in_channels: i64, out_channels: i64, config: EncoderConfig) -> Self { + let conv_cfg = nn::ConvConfig { stride: 1, padding: 1, ..Default::default() }; + let conv_in = + nn::conv2d(&vs / "conv_in", in_channels, config.block_out_channels[0], 3, conv_cfg); + let mut down_blocks = vec![]; + let vs_down_blocks = &vs / "down_blocks"; + for index in 0..config.block_out_channels.len() { + let out_channels = config.block_out_channels[index]; + let in_channels = if index > 0 { + config.block_out_channels[index - 1] + } else { + config.block_out_channels[0] + }; + let is_final = index + 1 == config.block_out_channels.len(); + let cfg = DownEncoderBlock2DConfig { + num_layers: config.layers_per_block, + resnet_eps: 1e-6, + resnet_groups: config.norm_num_groups, + add_downsample: !is_final, + downsample_padding: 0, + ..Default::default() + }; + let down_block = + DownEncoderBlock2D::new(&vs_down_blocks / index, in_channels, out_channels, cfg); + down_blocks.push(down_block) + } + let last_block_out_channels = *config.block_out_channels.last().unwrap(); + let mid_cfg = UNetMidBlock2DConfig { + resnet_eps: 1e-6, + output_scale_factor: 1., + attn_num_head_channels: None, + resnet_groups: Some(config.norm_num_groups), + ..Default::default() + }; + let mid_block = + UNetMidBlock2D::new(&vs / "mid_block", last_block_out_channels, None, mid_cfg); + let group_cfg = nn::GroupNormConfig { eps: 1e-6, ..Default::default() }; + let conv_norm_out = nn::group_norm( + &vs / "conv_norm_out", + config.norm_num_groups, + last_block_out_channels, + group_cfg, + ); + let conv_out_channels = if config.double_z { 2 * out_channels } else { out_channels }; + let conv_cfg = nn::ConvConfig { padding: 1, ..Default::default() }; + let conv_out = + nn::conv2d(&vs / "conv_out", last_block_out_channels, conv_out_channels, 3, conv_cfg); + Self { conv_in, down_blocks, mid_block, conv_norm_out, conv_out, config } + } +} + +impl Module for Encoder { + fn forward(&self, xs: &Tensor) -> Tensor { + let mut xs = xs.apply(&self.conv_in); + for down_block in self.down_blocks.iter() { + xs = xs.apply(down_block) + } + self.mid_block.forward(&xs, None).apply(&self.conv_norm_out).silu().apply(&self.conv_out) + } +} + +#[derive(Debug, Clone)] +struct DecoderConfig { + // up_block_types: UpDecoderBlock2D + block_out_channels: Vec, + layers_per_block: i64, + norm_num_groups: i64, +} + +impl Default for DecoderConfig { + fn default() -> Self { + Self { block_out_channels: vec![64], layers_per_block: 2, norm_num_groups: 32 } + } +} + +#[derive(Debug)] +struct Decoder { + conv_in: nn::Conv2D, + up_blocks: Vec, + mid_block: UNetMidBlock2D, + conv_norm_out: nn::GroupNorm, + conv_out: nn::Conv2D, + #[allow(dead_code)] + config: DecoderConfig, +} + +impl Decoder { + fn new(vs: nn::Path, in_channels: i64, out_channels: i64, config: DecoderConfig) -> Self { + let n_block_out_channels = config.block_out_channels.len(); + let last_block_out_channels = *config.block_out_channels.last().unwrap(); + let conv_cfg = nn::ConvConfig { stride: 1, padding: 1, ..Default::default() }; + let conv_in = + nn::conv2d(&vs / "conv_in", in_channels, last_block_out_channels, 3, conv_cfg); + let mid_cfg = UNetMidBlock2DConfig { + resnet_eps: 1e-6, + output_scale_factor: 1., + attn_num_head_channels: None, + resnet_groups: Some(config.norm_num_groups), + ..Default::default() + }; + let mid_block = + UNetMidBlock2D::new(&vs / "mid_block", last_block_out_channels, None, mid_cfg); + let mut up_blocks = vec![]; + let vs_up_blocks = &vs / "up_blocks"; + let reversed_block_out_channels: Vec<_> = + config.block_out_channels.iter().copied().rev().collect(); + for index in 0..n_block_out_channels { + let out_channels = reversed_block_out_channels[index]; + let in_channels = if index > 0 { + reversed_block_out_channels[index - 1] + } else { + reversed_block_out_channels[0] + }; + let is_final = index + 1 == n_block_out_channels; + let cfg = UpDecoderBlock2DConfig { + num_layers: config.layers_per_block + 1, + resnet_eps: 1e-6, + resnet_groups: config.norm_num_groups, + add_upsample: !is_final, + ..Default::default() + }; + let up_block = + UpDecoderBlock2D::new(&vs_up_blocks / index, in_channels, out_channels, cfg); + up_blocks.push(up_block) + } + let group_cfg = nn::GroupNormConfig { eps: 1e-6, ..Default::default() }; + let conv_norm_out = nn::group_norm( + &vs / "conv_norm_out", + config.norm_num_groups, + config.block_out_channels[0], + group_cfg, + ); + let conv_cfg = nn::ConvConfig { padding: 1, ..Default::default() }; + let conv_out = + nn::conv2d(&vs / "conv_out", config.block_out_channels[0], out_channels, 3, conv_cfg); + Self { conv_in, up_blocks, mid_block, conv_norm_out, conv_out, config } + } +} + +impl Module for Decoder { + fn forward(&self, xs: &Tensor) -> Tensor { + let mut xs = self.mid_block.forward(&xs.apply(&self.conv_in), None); + for up_block in self.up_blocks.iter() { + xs = xs.apply(up_block) + } + xs.apply(&self.conv_norm_out).silu().apply(&self.conv_out) + } +} + +#[derive(Debug, Clone)] +struct AutoEncoderKLConfig { + block_out_channels: Vec, + layers_per_block: i64, + latent_channels: i64, + norm_num_groups: i64, +} + +impl Default for AutoEncoderKLConfig { + fn default() -> Self { + Self { + block_out_channels: vec![64], + layers_per_block: 1, + latent_channels: 4, + norm_num_groups: 32, + } + } +} + +// https://github.com/huggingface/diffusers/blob/970e30606c2944e3286f56e8eb6d3dc6d1eb85f7/src/diffusers/models/vae.py#L485 +// This implementation is specific to the config used in stable-diffusion-v1-4 +// https://huggingface.co/CompVis/stable-diffusion-v1-4/blob/main/vae/config.json +#[derive(Debug)] +struct AutoEncoderKL { + encoder: Encoder, + decoder: Decoder, + quant_conv: nn::Conv2D, + post_quant_conv: nn::Conv2D, + #[allow(dead_code)] + config: AutoEncoderKLConfig, +} + +impl AutoEncoderKL { + fn new(vs: nn::Path, in_channels: i64, out_channels: i64, config: AutoEncoderKLConfig) -> Self { + let latent_channels = config.latent_channels; + let encoder_cfg = EncoderConfig { + block_out_channels: config.block_out_channels.clone(), + layers_per_block: config.layers_per_block, + norm_num_groups: config.norm_num_groups, + double_z: true, + }; + let encoder = Encoder::new(&vs / "encoder", in_channels, latent_channels, encoder_cfg); + let decoder_cfg = DecoderConfig { + block_out_channels: config.block_out_channels.clone(), + layers_per_block: config.layers_per_block, + norm_num_groups: config.norm_num_groups, + }; + let decoder = Decoder::new(&vs / "decoder", latent_channels, out_channels, decoder_cfg); + let conv_cfg = Default::default(); + let quant_conv = + nn::conv2d(&vs / "quant_conv", 2 * latent_channels, 2 * latent_channels, 1, conv_cfg); + let post_quant_conv = + nn::conv2d(&vs / "post_quant_conv", latent_channels, latent_channels, 1, conv_cfg); + Self { encoder, decoder, quant_conv, post_quant_conv, config } + } + + // Returns the parameters of the latent distribution. + #[allow(dead_code)] + fn encode(&self, xs: &Tensor) -> Tensor { + xs.apply(&self.encoder).apply(&self.quant_conv) + } + + /// Takes as input some sampled values. + fn decode(&self, xs: &Tensor) -> Tensor { + xs.apply(&self.post_quant_conv).apply(&self.decoder) + } +} + +#[derive(Debug, Clone, Copy)] +struct DownBlock2DConfig { + num_layers: i64, + resnet_eps: f64, + // resnet_time_scale_shift: "default" + // resnet_act_fn: "swish" + resnet_groups: i64, + output_scale_factor: f64, + add_downsample: bool, + downsample_padding: i64, +} + +impl Default for DownBlock2DConfig { + fn default() -> Self { + Self { + num_layers: 1, + resnet_eps: 1e-6, + resnet_groups: 32, + output_scale_factor: 1., + add_downsample: true, + downsample_padding: 1, + } + } +} + +#[derive(Debug)] +struct DownBlock2D { + resnets: Vec, + downsampler: Option, + #[allow(dead_code)] + config: DownBlock2DConfig, +} + +impl DownBlock2D { + fn new( + vs: nn::Path, + in_channels: i64, + out_channels: i64, + temb_channels: Option, + config: DownBlock2DConfig, + ) -> Self { + let vs_resnets = &vs / "resnets"; + let resnet_cfg = ResnetBlock2DConfig { + out_channels: Some(out_channels), + eps: config.resnet_eps, + output_scale_factor: config.output_scale_factor, + temb_channels, + ..Default::default() + }; + let resnets = (0..config.num_layers) + .map(|i| { + let in_channels = if i == 0 { in_channels } else { out_channels }; + ResnetBlock2D::new(&vs_resnets / i, in_channels, resnet_cfg) + }) + .collect(); + let downsampler = if config.add_downsample { + let downsampler = Downsample2D::new( + &vs / "downsamplers" / 0, + out_channels, + true, + out_channels, + config.downsample_padding, + ); + Some(downsampler) + } else { + None + }; + Self { resnets, downsampler, config } + } + + fn forward(&self, xs: &Tensor, temb: Option<&Tensor>) -> (Tensor, Vec) { + let mut xs = xs.shallow_clone(); + let mut output_states = vec![]; + for resnet in self.resnets.iter() { + xs = resnet.forward(&xs, temb); + output_states.push(xs.shallow_clone()); + } + let xs = match &self.downsampler { + Some(downsampler) => { + let xs = xs.apply(downsampler); + output_states.push(xs.shallow_clone()); + xs + } + None => xs, + }; + (xs, output_states) + } +} + +#[derive(Debug, Clone, Copy)] +struct CrossAttnDownBlock2DConfig { + downblock: DownBlock2DConfig, + attn_num_head_channels: i64, + cross_attention_dim: i64, + // attention_type: "default" +} + +impl Default for CrossAttnDownBlock2DConfig { + fn default() -> Self { + Self { downblock: Default::default(), attn_num_head_channels: 1, cross_attention_dim: 1280 } + } +} + +#[derive(Debug)] +struct CrossAttnDownBlock2D { + downblock: DownBlock2D, + attentions: Vec, + #[allow(dead_code)] + config: CrossAttnDownBlock2DConfig, +} + +impl CrossAttnDownBlock2D { + fn new( + vs: nn::Path, + in_channels: i64, + out_channels: i64, + temb_channels: Option, + config: CrossAttnDownBlock2DConfig, + ) -> Self { + let downblock = DownBlock2D::new( + vs.clone(), + in_channels, + out_channels, + temb_channels, + config.downblock, + ); + let n_heads = config.attn_num_head_channels; + let cfg = SpatialTransformerConfig { + depth: 1, + context_dim: Some(config.cross_attention_dim), + num_groups: config.downblock.resnet_groups, + }; + let vs_attn = &vs / "attentions"; + let attentions = (0..config.downblock.num_layers) + .map(|i| { + SpatialTransformer::new( + &vs_attn / i, + out_channels, + n_heads, + out_channels / n_heads, + cfg, + ) + }) + .collect(); + Self { downblock, attentions, config } + } + + fn forward( + &self, + xs: &Tensor, + temb: Option<&Tensor>, + encoder_hidden_states: Option<&Tensor>, + ) -> (Tensor, Vec) { + let mut output_states = vec![]; + let mut xs = xs.shallow_clone(); + for (resnet, attn) in self.downblock.resnets.iter().zip(self.attentions.iter()) { + xs = resnet.forward(&xs, temb); + xs = attn.forward(&xs, encoder_hidden_states); + output_states.push(xs.shallow_clone()); + } + let xs = match &self.downblock.downsampler { + Some(downsampler) => { + let xs = xs.apply(downsampler); + output_states.push(xs.shallow_clone()); + xs + } + None => xs, + }; + (xs, output_states) + } +} + +#[derive(Debug, Clone, Copy)] +struct UpBlock2DConfig { + num_layers: i64, + resnet_eps: f64, + // resnet_time_scale_shift: "default" + // resnet_act_fn: "swish" + resnet_groups: i64, + output_scale_factor: f64, + add_upsample: bool, +} + +impl Default for UpBlock2DConfig { + fn default() -> Self { + Self { + num_layers: 1, + resnet_eps: 1e-6, + resnet_groups: 32, + output_scale_factor: 1., + add_upsample: true, + } + } +} + +#[derive(Debug)] +struct UpBlock2D { + resnets: Vec, + upsampler: Option, + #[allow(dead_code)] + config: UpBlock2DConfig, +} + +impl UpBlock2D { + fn new( + vs: nn::Path, + in_channels: i64, + prev_output_channels: i64, + out_channels: i64, + temb_channels: Option, + config: UpBlock2DConfig, + ) -> Self { + let vs_resnets = &vs / "resnets"; + let resnet_cfg = ResnetBlock2DConfig { + out_channels: Some(out_channels), + temb_channels, + eps: config.resnet_eps, + output_scale_factor: config.output_scale_factor, + ..Default::default() + }; + let resnets = (0..config.num_layers) + .map(|i| { + let res_skip_channels = + if i == config.num_layers - 1 { in_channels } else { out_channels }; + let resnet_in_channels = if i == 0 { prev_output_channels } else { out_channels }; + let in_channels = resnet_in_channels + res_skip_channels; + ResnetBlock2D::new(&vs_resnets / i, in_channels, resnet_cfg) + }) + .collect(); + let upsampler = if config.add_upsample { + let upsampler = Upsample2D::new(&vs / "upsamplers" / 0, out_channels, out_channels); + Some(upsampler) + } else { + None + }; + Self { resnets, upsampler, config } + } + + fn forward( + &self, + xs: &Tensor, + res_xs: &[Tensor], + temb: Option<&Tensor>, + upsample_size: Option<(i64, i64)>, + ) -> Tensor { + let mut xs = xs.shallow_clone(); + for (index, resnet) in self.resnets.iter().enumerate() { + xs = Tensor::cat(&[&xs, &res_xs[res_xs.len() - index - 1]], 1); + xs = resnet.forward(&xs, temb); + } + match &self.upsampler { + Some(upsampler) => upsampler.forward(&xs, upsample_size), + None => xs, + } + } +} + +#[derive(Debug, Clone, Copy)] +struct CrossAttnUpBlock2DConfig { + upblock: UpBlock2DConfig, + attn_num_head_channels: i64, + cross_attention_dim: i64, + // attention_type: "default" +} + +impl Default for CrossAttnUpBlock2DConfig { + fn default() -> Self { + Self { upblock: Default::default(), attn_num_head_channels: 1, cross_attention_dim: 1280 } + } +} + +#[derive(Debug)] +struct CrossAttnUpBlock2D { + upblock: UpBlock2D, + attentions: Vec, + #[allow(dead_code)] + config: CrossAttnUpBlock2DConfig, +} + +impl CrossAttnUpBlock2D { + fn new( + vs: nn::Path, + in_channels: i64, + prev_output_channels: i64, + out_channels: i64, + temb_channels: Option, + config: CrossAttnUpBlock2DConfig, + ) -> Self { + let upblock = UpBlock2D::new( + vs.clone(), + in_channels, + prev_output_channels, + out_channels, + temb_channels, + config.upblock, + ); + let n_heads = config.attn_num_head_channels; + let cfg = SpatialTransformerConfig { + depth: 1, + context_dim: Some(config.cross_attention_dim), + num_groups: config.upblock.resnet_groups, + }; + let vs_attn = &vs / "attentions"; + let attentions = (0..config.upblock.num_layers) + .map(|i| { + SpatialTransformer::new( + &vs_attn / i, + out_channels, + n_heads, + out_channels / n_heads, + cfg, + ) + }) + .collect(); + Self { upblock, attentions, config } + } + + fn forward( + &self, + xs: &Tensor, + res_xs: &[Tensor], + temb: Option<&Tensor>, + upsample_size: Option<(i64, i64)>, + encoder_hidden_states: Option<&Tensor>, + ) -> Tensor { + let mut xs = xs.shallow_clone(); + for (index, resnet) in self.upblock.resnets.iter().enumerate() { + xs = Tensor::cat(&[&xs, &res_xs[res_xs.len() - index - 1]], 1); + xs = resnet.forward(&xs, temb); + xs = self.attentions[index].forward(&xs, encoder_hidden_states); + } + match &self.upblock.upsampler { + Some(upsampler) => upsampler.forward(&xs, upsample_size), + None => xs, + } + } +} + +#[derive(Debug)] +struct Timesteps { + num_channels: i64, + flip_sin_to_cos: bool, + downscale_freq_shift: f64, + device: Device, +} + +impl Timesteps { + fn new( + num_channels: i64, + flip_sin_to_cos: bool, + downscale_freq_shift: f64, + device: Device, + ) -> Self { + Self { num_channels, flip_sin_to_cos, downscale_freq_shift, device } + } +} + +impl Module for Timesteps { + fn forward(&self, xs: &Tensor) -> Tensor { + let half_dim = self.num_channels / 2; + let exponent = Tensor::arange(half_dim, (Kind::Float, self.device)) * -f64::ln(10000.); + let exponent = exponent / (half_dim as f64 - self.downscale_freq_shift); + let emb = exponent.exp(); + // emb = timesteps[:, None].float() * emb[None, :] + let emb = xs.unsqueeze(-1) * emb.unsqueeze(0); + let emb = if self.flip_sin_to_cos { + Tensor::cat(&[emb.cos(), emb.sin()], -1) + } else { + Tensor::cat(&[emb.sin(), emb.cos()], -1) + }; + if self.num_channels % 2 == 1 { + emb.pad([0, 1, 0, 0], "constant", None) + } else { + emb + } + } +} + +#[derive(Debug)] +struct TimestepEmbedding { + linear_1: nn::Linear, + linear_2: nn::Linear, +} + +impl TimestepEmbedding { + // act_fn: "silu" + fn new(vs: nn::Path, channel: i64, time_embed_dim: i64) -> Self { + let linear_cfg = Default::default(); + let linear_1 = nn::linear(&vs / "linear_1", channel, time_embed_dim, linear_cfg); + let linear_2 = nn::linear(&vs / "linear_2", time_embed_dim, time_embed_dim, linear_cfg); + Self { linear_1, linear_2 } + } +} + +impl Module for TimestepEmbedding { + fn forward(&self, xs: &Tensor) -> Tensor { + xs.apply(&self.linear_1).silu().apply(&self.linear_2) + } +} + +#[derive(Debug, Clone, Copy)] +struct BlockConfig { + out_channels: i64, + use_cross_attn: bool, +} + +#[derive(Debug, Clone)] +struct UNet2DConditionModelConfig { + center_input_sample: bool, + flip_sin_to_cos: bool, + freq_shift: f64, + blocks: Vec, + layers_per_block: i64, + downsample_padding: i64, + mid_block_scale_factor: f64, + norm_num_groups: i64, + norm_eps: f64, + cross_attention_dim: i64, + attention_head_dim: i64, +} + +impl Default for UNet2DConditionModelConfig { + fn default() -> Self { + Self { + center_input_sample: false, + flip_sin_to_cos: true, + freq_shift: 0., + blocks: vec![ + BlockConfig { out_channels: 320, use_cross_attn: true }, + BlockConfig { out_channels: 640, use_cross_attn: true }, + BlockConfig { out_channels: 1280, use_cross_attn: true }, + BlockConfig { out_channels: 1280, use_cross_attn: false }, + ], + layers_per_block: 2, + downsample_padding: 1, + mid_block_scale_factor: 1., + norm_num_groups: 32, + norm_eps: 1e-5, + cross_attention_dim: 1280, + attention_head_dim: 8, + } + } +} + +#[derive(Debug)] +enum UNetDownBlock { + Basic(DownBlock2D), + CrossAttn(CrossAttnDownBlock2D), +} + +#[derive(Debug)] +enum UNetUpBlock { + Basic(UpBlock2D), + CrossAttn(CrossAttnUpBlock2D), +} + +#[derive(Debug)] +struct UNet2DConditionModel { + conv_in: nn::Conv2D, + time_proj: Timesteps, + time_embedding: TimestepEmbedding, + down_blocks: Vec, + mid_block: UNetMidBlock2DCrossAttn, + up_blocks: Vec, + conv_norm_out: nn::GroupNorm, + conv_out: nn::Conv2D, + config: UNet2DConditionModelConfig, +} + +impl UNet2DConditionModel { + fn new( + vs: nn::Path, + in_channels: i64, + out_channels: i64, + config: UNet2DConditionModelConfig, + ) -> Self { + let n_blocks = config.blocks.len(); + let b_channels = config.blocks[0].out_channels; + let bl_channels = config.blocks.last().unwrap().out_channels; + let time_embed_dim = b_channels * 4; + let conv_cfg = nn::ConvConfig { stride: 1, padding: 1, ..Default::default() }; + let conv_in = nn::conv2d(&vs / "conv_in", in_channels, b_channels, 3, conv_cfg); + + let time_proj = + Timesteps::new(b_channels, config.flip_sin_to_cos, config.freq_shift, vs.device()); + let time_embedding = + TimestepEmbedding::new(&vs / "time_embedding", b_channels, time_embed_dim); + + let vs_db = &vs / "down_blocks"; + let down_blocks = (0..n_blocks) + .map(|i| { + let BlockConfig { out_channels, use_cross_attn } = config.blocks[i]; + let in_channels = + if i > 0 { config.blocks[i - 1].out_channels } else { b_channels }; + let db_cfg = DownBlock2DConfig { + num_layers: config.layers_per_block, + resnet_eps: config.norm_eps, + resnet_groups: config.norm_num_groups, + add_downsample: i < n_blocks - 1, + downsample_padding: config.downsample_padding, + ..Default::default() + }; + if use_cross_attn { + let config = CrossAttnDownBlock2DConfig { + downblock: db_cfg, + attn_num_head_channels: config.attention_head_dim, + cross_attention_dim: config.cross_attention_dim, + }; + let block = CrossAttnDownBlock2D::new( + &vs_db / i, + in_channels, + out_channels, + Some(time_embed_dim), + config, + ); + UNetDownBlock::CrossAttn(block) + } else { + let block = DownBlock2D::new( + &vs_db / i, + in_channels, + out_channels, + Some(time_embed_dim), + db_cfg, + ); + UNetDownBlock::Basic(block) + } + }) + .collect(); + + let mid_cfg = UNetMidBlock2DCrossAttnConfig { + resnet_eps: config.norm_eps, + output_scale_factor: config.mid_block_scale_factor, + cross_attn_dim: config.cross_attention_dim, + attn_num_head_channels: config.attention_head_dim, + resnet_groups: Some(config.norm_num_groups), + ..Default::default() + }; + let mid_block = UNetMidBlock2DCrossAttn::new( + &vs / "mid_block", + bl_channels, + Some(time_embed_dim), + mid_cfg, + ); + + let vs_ub = &vs / "up_blocks"; + let up_blocks = (0..n_blocks) + .map(|i| { + let BlockConfig { out_channels, use_cross_attn } = config.blocks[n_blocks - 1 - i]; + let prev_out_channels = + if i > 0 { config.blocks[n_blocks - i].out_channels } else { bl_channels }; + let in_channels = { + let index = if i == n_blocks - 1 { 0 } else { n_blocks - i - 2 }; + config.blocks[index].out_channels + }; + let ub_cfg = UpBlock2DConfig { + num_layers: config.layers_per_block + 1, + resnet_eps: config.norm_eps, + resnet_groups: config.norm_num_groups, + add_upsample: i < n_blocks - 1, + ..Default::default() + }; + if use_cross_attn { + let config = CrossAttnUpBlock2DConfig { + upblock: ub_cfg, + attn_num_head_channels: config.attention_head_dim, + cross_attention_dim: config.cross_attention_dim, + }; + let block = CrossAttnUpBlock2D::new( + &vs_ub / i, + in_channels, + prev_out_channels, + out_channels, + Some(time_embed_dim), + config, + ); + UNetUpBlock::CrossAttn(block) + } else { + let block = UpBlock2D::new( + &vs_ub / i, + in_channels, + prev_out_channels, + out_channels, + Some(time_embed_dim), + ub_cfg, + ); + UNetUpBlock::Basic(block) + } + }) + .collect(); + + let group_cfg = nn::GroupNormConfig { eps: config.norm_eps, ..Default::default() }; + let conv_norm_out = + nn::group_norm(&vs / "conv_norm_out", config.norm_num_groups, b_channels, group_cfg); + let conv_out = nn::conv2d(&vs / "conv_out", b_channels, out_channels, 3, conv_cfg); + Self { + conv_in, + time_proj, + time_embedding, + down_blocks, + mid_block, + up_blocks, + conv_norm_out, + conv_out, + config, + } + } +} + +impl UNet2DConditionModel { + fn forward(&self, xs: &Tensor, timestep: f64, encoder_hidden_states: &Tensor) -> Tensor { + let (bsize, _channels, height, width) = xs.size4().unwrap(); + let device = xs.device(); + let n_blocks = self.config.blocks.len(); + let num_upsamplers = n_blocks - 1; + let default_overall_up_factor = 2i64.pow(num_upsamplers as u32); + let forward_upsample_size = + height % default_overall_up_factor != 0 || width % default_overall_up_factor != 0; + // 0. center input if necessary + let xs = if self.config.center_input_sample { xs * 2.0 - 1.0 } else { xs.shallow_clone() }; + // 1. time + let emb = (Tensor::ones([bsize], (Kind::Float, device)) * timestep) + .apply(&self.time_proj) + .apply(&self.time_embedding); + // 2. pre-process + let xs = xs.apply(&self.conv_in); + // 3. down + let mut down_block_res_xs = vec![xs.shallow_clone()]; + let mut xs = xs; + for down_block in self.down_blocks.iter() { + let (_xs, res_xs) = match down_block { + UNetDownBlock::Basic(b) => b.forward(&xs, Some(&emb)), + UNetDownBlock::CrossAttn(b) => { + b.forward(&xs, Some(&emb), Some(encoder_hidden_states)) + } + }; + down_block_res_xs.extend(res_xs); + xs = _xs; + } + // 4. mid + let xs = self.mid_block.forward(&xs, Some(&emb), Some(encoder_hidden_states)); + // 5. up + let mut xs = xs; + let mut upsample_size = None; + for (i, up_block) in self.up_blocks.iter().enumerate() { + let n_resnets = match up_block { + UNetUpBlock::Basic(b) => b.resnets.len(), + UNetUpBlock::CrossAttn(b) => b.upblock.resnets.len(), + }; + let res_xs = down_block_res_xs.split_off(down_block_res_xs.len() - n_resnets); + if i < n_blocks - 1 && forward_upsample_size { + let (_, _, h, w) = down_block_res_xs.last().unwrap().size4().unwrap(); + upsample_size = Some((h, w)) + } + xs = match up_block { + UNetUpBlock::Basic(b) => b.forward(&xs, &res_xs, Some(&emb), upsample_size), + UNetUpBlock::CrossAttn(b) => { + b.forward(&xs, &res_xs, Some(&emb), upsample_size, Some(encoder_hidden_states)) + } + }; + } + // 6. post-process + xs.apply(&self.conv_norm_out).silu().apply(&self.conv_out) + } +} + +// TODO: LMSDiscreteScheduler +// https://github.com/huggingface/diffusers/blob/32bf4fdc4386809c870528cb261028baae012d27/src/diffusers/schedulers/scheduling_lms_discrete.py#L47 + +fn build_clip_transformer(device: Device) -> anyhow::Result { + let mut vs = nn::VarStore::new(device); + let text_model = ClipTextTransformer::new(vs.root()); + vs.load("data/pytorch_model.ot")?; + Ok(text_model) +} + +fn build_vae(device: Device) -> anyhow::Result { + let mut vs_ae = nn::VarStore::new(device); + // https://huggingface.co/CompVis/stable-diffusion-v1-4/blob/main/vae/config.json + let autoencoder_cfg = AutoEncoderKLConfig { + block_out_channels: vec![128, 256, 512, 512], + layers_per_block: 2, + latent_channels: 4, + norm_num_groups: 32, + }; + let autoencoder = AutoEncoderKL::new(vs_ae.root(), 3, 3, autoencoder_cfg); + vs_ae.load("data/vae.ot")?; + Ok(autoencoder) +} + +fn build_unet(device: Device) -> anyhow::Result { + let mut vs_unet = nn::VarStore::new(device); + // https://huggingface.co/CompVis/stable-diffusion-v1-4/blob/main/unet/config.json + let unet_cfg = UNet2DConditionModelConfig { + attention_head_dim: 8, + blocks: vec![ + BlockConfig { out_channels: 320, use_cross_attn: true }, + BlockConfig { out_channels: 640, use_cross_attn: true }, + BlockConfig { out_channels: 1280, use_cross_attn: true }, + BlockConfig { out_channels: 1280, use_cross_attn: false }, + ], + center_input_sample: false, + cross_attention_dim: 768, + downsample_padding: 1, + flip_sin_to_cos: true, + freq_shift: 0., + layers_per_block: 2, + mid_block_scale_factor: 1., + norm_eps: 1e-5, + norm_num_groups: 32, + }; + let unet = UNet2DConditionModel::new(vs_unet.root(), 4, 4, unet_cfg); + vs_unet.load("data/unet.ot")?; + Ok(unet) +} + +#[derive(Debug, Clone, Copy)] +enum BetaSchedule { + #[allow(dead_code)] + Linear, + ScaledLinear, +} + +#[derive(Debug, Clone, Copy)] +struct DDIMSchedulerConfig { + beta_start: f64, + beta_end: f64, + beta_schedule: BetaSchedule, + eta: f64, +} + +impl Default for DDIMSchedulerConfig { + fn default() -> Self { + Self { + beta_start: 0.00085f64, + beta_end: 0.012f64, + beta_schedule: BetaSchedule::ScaledLinear, + eta: 0., + } + } +} + +#[derive(Debug, Clone)] +struct DDIMScheduler { + timesteps: Vec, + alphas_cumprod: Vec, + step_ratio: usize, + #[allow(dead_code)] + config: DDIMSchedulerConfig, +} + +// clip_sample: False, set_alpha_to_one: False +impl DDIMScheduler { + fn new(inference_steps: usize, train_timesteps: usize, config: DDIMSchedulerConfig) -> Self { + let step_ratio = train_timesteps / inference_steps; + // TODO: Remove this hack which aimed at matching the behavior of diffusers==0.2.4 + let timesteps = (0..(inference_steps + 1)).map(|s| s * step_ratio).rev().collect(); + let betas = match config.beta_schedule { + BetaSchedule::ScaledLinear => Tensor::linspace( + config.beta_start.sqrt(), + config.beta_end.sqrt(), + train_timesteps as i64, + kind::FLOAT_CPU, + ) + .square(), + BetaSchedule::Linear => Tensor::linspace( + config.beta_start, + config.beta_end, + train_timesteps as i64, + kind::FLOAT_CPU, + ), + }; + let alphas: Tensor = 1.0 - betas; + let alphas_cumprod = Vec::::try_from(alphas.cumprod(0, Kind::Double)).unwrap(); + Self { alphas_cumprod, timesteps, step_ratio, config } + } + + // https://github.com/huggingface/diffusers/blob/6e099e2c8ce4c4f5c7318e970a8c093dc5c7046e/src/diffusers/schedulers/scheduling_ddim.py#L195 + fn step(&self, model_output: &Tensor, timestep: usize, sample: &Tensor) -> Tensor { + let prev_timestep = timestep.saturating_sub(self.step_ratio); + + let alpha_prod_t = self.alphas_cumprod[timestep]; + let alpha_prod_t_prev = self.alphas_cumprod[prev_timestep]; + let beta_prod_t = 1. - alpha_prod_t; + let beta_prod_t_prev = 1. - alpha_prod_t_prev; + + let pred_original_sample = + (sample - beta_prod_t.sqrt() * model_output) / alpha_prod_t.sqrt(); + + let variance = (beta_prod_t_prev / beta_prod_t) * (1. - alpha_prod_t / alpha_prod_t_prev); + let std_dev_t = self.config.eta * variance.sqrt(); + + let pred_sample_direction = + (1. - alpha_prod_t_prev - std_dev_t * std_dev_t).sqrt() * model_output; + let prev_sample = alpha_prod_t_prev.sqrt() * pred_original_sample + pred_sample_direction; + if self.config.eta > 0. { + &prev_sample + Tensor::randn_like(&prev_sample) * std_dev_t + } else { + prev_sample + } + } +} + +fn main() -> anyhow::Result<()> { + println!("Cuda available: {}", tch::Cuda::is_available()); + println!("Cudnn available: {}", tch::Cuda::cudnn_is_available()); + // TODO: Switch to using claps to allow more flags? + let mut prompt = "A rusty robot holding a fire torch in its hand".to_string(); + let mut device = Device::cuda_if_available(); + for arg in std::env::args().skip(1) { + if arg.as_str() == "cpu" { + device = Device::Cpu; + } else { + prompt = arg; + } + } + let n_steps = 30; + let scheduler = DDIMScheduler::new(n_steps, 1000, Default::default()); + + let tokenizer = Tokenizer::create("data/bpe_simple_vocab_16e6.txt")?; + let tokens = tokenizer.encode(&prompt, Some(MAX_POSITION_EMBEDDINGS))?; + let str = tokenizer.decode(&tokens); + println!("Str: {str}"); + let tokens: Vec = tokens.iter().map(|x| *x as i64).collect(); + let tokens = Tensor::from_slice(&tokens).view((1, -1)).to(device); + let uncond_tokens = tokenizer.encode("", Some(MAX_POSITION_EMBEDDINGS))?; + let uncond_tokens: Vec = uncond_tokens.iter().map(|x| *x as i64).collect(); + let uncond_tokens = Tensor::from_slice(&uncond_tokens).view((1, -1)).to(device); + println!("Tokens: {tokens:?}"); + + let no_grad_guard = tch::no_grad_guard(); + println!("Building the Clip transformer."); + let text_model = build_clip_transformer(device)?; + let text_embeddings = text_model.forward(&tokens); + let uncond_embeddings = text_model.forward(&uncond_tokens); + let text_embeddings = Tensor::cat(&[uncond_embeddings, text_embeddings], 0); + println!("Text embeddings: {text_embeddings:?}"); + + println!("Building the autoencoder."); + let vae = build_vae(device)?; + println!("Building the unet."); + let unet = build_unet(device)?; + + let bsize = 1; + // DETERMINISTIC SEEDING + tch::manual_seed(32); + let mut latents = Tensor::randn([bsize, 4, HEIGHT / 8, WIDTH / 8], (Kind::Float, device)); + + for (timestep_index, ×tep) in scheduler.timesteps.iter().enumerate() { + println!("Timestep {timestep_index} {timestep} {latents:?}"); + let latent_model_input = Tensor::cat(&[&latents, &latents], 0); + let noise_pred = unet.forward(&latent_model_input, timestep as f64, &text_embeddings); + let noise_pred = noise_pred.chunk(2, 0); + let (noise_pred_uncond, noise_pred_text) = (&noise_pred[0], &noise_pred[1]); + let noise_pred = noise_pred_uncond + (noise_pred_text - noise_pred_uncond) * GUIDANCE_SCALE; + latents = scheduler.step(&noise_pred, timestep, &latents); + + let image = vae.decode(&(&latents / 0.18215)); + let image = (image / 2 + 0.5).clamp(0., 1.).to_device(Device::Cpu); + let image = (image * 255.).to_kind(Kind::Uint8); + tch::vision::image::save(&image, format!("sd_{timestep_index}.png"))? + } + + drop(no_grad_guard); + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/tensor-tools.rs b/v2/dependency-patches/tch-0.24.0/examples/tensor-tools.rs new file mode 100644 index 0000000000..3c6c6ffed3 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/tensor-tools.rs @@ -0,0 +1,79 @@ +// A small tensor tool utility. +// +// - List the content of some npy/npz/ot file. +// tensor-tools ls a.npy b.npz c.ot +// +// - Convert a npz file to an ot file. +// tensor-tools cp src.npz dst.ot +// Or the other way around. +// tensor-tools cp src.ot dst.npz + +use anyhow::{bail, ensure, Result}; + +pub fn main() -> Result<()> { + let args: Vec<_> = std::env::args().collect(); + ensure!(args.len() >= 2, "usage: {} (ls|cp) ...", args[0]); + match args[1].as_str() { + "ls" => { + for filename in args.iter().skip(2) { + if filename.ends_with(".npy") { + let tensor = tch::Tensor::read_npy(filename)?; + println!("{filename}: {tensor:?}"); + } else if filename.ends_with(".npz") { + let tensors = tch::Tensor::read_npz(filename)?; + for (name, tensor) in tensors.iter() { + println!("{filename}: {name} {tensor:?}") + } + } else if filename.ends_with(".safetensors") { + let tensors = tch::Tensor::read_safetensors(filename)?; + for (name, tensor) in tensors.iter() { + println!("{filename}: {name} {tensor:?}") + } + } else if filename.ends_with(".ot") { + let tensors = tch::Tensor::load_multi(filename)?; + for (name, tensor) in tensors.iter() { + println!("{filename}: {name} {tensor:?}") + } + } else if filename.ends_with(".bin") || filename.ends_with(".zip") { + let tensors = tch::Tensor::loadz_multi(filename)?; + for (name, tensor) in tensors.iter() { + println!("{filename}: {name} {tensor:?}") + } + } else { + bail!("unhandled file {}", filename); + } + } + } + "cp" => { + ensure!(args.len() == 4, "usage: {} cp src.ot dst.npz", args[0]); + let src_filename = &args[2]; + let dst_filename = &args[3]; + let tensors = if src_filename.ends_with(".npz") { + tch::Tensor::read_npz(src_filename)? + } else if src_filename.ends_with(".safetensors") { + tch::Tensor::read_safetensors(src_filename)? + } else if src_filename.ends_with(".ot") { + tch::Tensor::load_multi(src_filename)? + } else if src_filename.ends_with(".bin") || src_filename.ends_with(".zip") { + tch::Tensor::loadz_multi(src_filename)? + } else { + bail!("unhandled file {}", src_filename) + }; + for (name, tensor) in tensors.iter() { + println!("{src_filename}: {name} {tensor:?}") + } + if dst_filename.ends_with(".npz") { + tch::Tensor::write_npz(&tensors, dst_filename)? + } else if dst_filename.ends_with(".safetensors") { + tch::Tensor::write_safetensors(&tensors, dst_filename)? + } else if dst_filename.ends_with(".ot") { + tch::Tensor::save_multi(&tensors, dst_filename)? + } else { + bail!("unhandled file {}", dst_filename) + }; + } + _ => bail!("usage: {} (ls|cp) ...", args[0]), + } + + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/transfer-learning/README.md b/v2/dependency-patches/tch-0.24.0/examples/transfer-learning/README.md new file mode 100644 index 0000000000..1861dfcacb --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/transfer-learning/README.md @@ -0,0 +1,141 @@ +# Transfer Learning Tutorial + +This tutorial follows the lines of the +[PyTorch Transfer Learning Tutorial](https://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html). + +We will use transfer learning to leverage a pretrained ResNet model on a small dataset. +This dataset is made of images of ants and bees that we want to classify, +there are roughly 240 training images and 150 validation images each of them of size +224x224. The dataset is so small that training even the simplest convolutional neural +network on it would be very difficult. + +Instead the original tutorial proposes two alternatives to train the classifier. + +- *Finetuning the pretrained model.* We start from a ResNet-18 model pretrained on ImageNet +1000 categories, replace the last layer by a binary classifier and train the resulting model +as usual. +- *Using a pretrained model as feature extractor.* The pretrained model weights are frozen and +we run this model and store the outputs of the last layer before the final classifier. +We then train a binary classifier on the resulting features. + +We will focus on the second alternative but first we need to get the code +building and running and we also have to download the dataset and pretrained +weights. + +## Installation Instructions +Run the following commands to download the latest tch-rs version +and run the tests, this installs the CPU version of libtorch if necessary. + +```bash +git clone https://github.com/LaurentMazare/tch-rs.git +cd tch-rs +cargo test +``` + +The ants and bees dataset can be downloaded [here](https://download.pytorch.org/tutorial/hymenoptera_data.zip). +You can download the weights for a ResNet-18 network pretrained on ImageNet, +[resnet18.ot](https://github.com/LaurentMazare/tch-rs/releases/download/untagged-eb220e5c19f9bb250bd1/resnet18.ot). + +Once this is done and the dataset has been extracted we can build and run the code with: +```bash +cargo run --example transfer-learning resnet18.ot hymenoptera_data +``` + +## Loading the Data + +Let us now have a look at the code from `main.rs`. +The dataset is loaded via some helper functions. + +```rust +let dataset = imagenet::load_from_dir(dataset_dir)?; +println!("{:?}", dataset); +``` + +The `println!` macro prints the dimensions of the tensors that have +been created. For training the tensor has shape `211x3x224x224`, this +corresponds to 211 images of height and width both 224 with 3 channels +(PyTorch uses the NCHW ordering for image data). The testing image +tensor has dimensions `127` so there are 127 images with the +same size as used in training. + + +## Using a Pretrained ResNet as Feature Extractor + +The pixel data from the dataset is converted to features by running +a pre-trained ResNet model. + +```rust +let mut vs = tch::nn::VarStore::new(tch::Device::Cpu); +let net = resnet::resnet18_no_final_layer(&vs.root()); +vs.load(weights)?; + +let train_images = tch::no_grad(|| dataset.train_images.apply_t(&net, false)); +let test_images = tch::no_grad(|| dataset.test_images.apply_t(&net, false)); +``` + +This snippet performs the following steps: +- A variable store `vs` is created to hold the network weights. +- A ResNet-18 model is created using this variable store. At this point the + model weights are randomly initialized. +- `vs.load(weights)` loads the weights stored in a given file and copy their values + to some tensors. Tensors are named in the serialized file in a way that matches + the names we used when creating the ResNet model. +- Finally for each tensor of the training and testing datasets, `apply` + performs a forward pass on the model and returns the resulting tensor. In this + case the result is a vector of 512 values per sample. +The `no_grad` closure informs PyTorch that there is no need to keep a graph of the +forward pass as we do not plan on asking for gradients. + +## Training a Linear Layer on top of the Extracted Features + +Now that we have precomputed the output of the ResNet model on our training and +testing images we will train a linear binary classifier to recognize ants vs bees. + +We start by defining a model, for this we need a variable store to hold the +trainable variables. + +```rust +let vs = tch::nn::VarStore::new(tch::Device::Cpu); +let linear = nn::linear(vs.root(), 512, dataset.labels, Default::default()); +``` + +We will use stochastic gradient descent to minimize the cross-entropy loss +on the classification task. To do this we create a `sgd` optimizer and then +iterate on the training dataset. After each epoch the accuracy is computed +on the testing set and printed. + +```rust +let sgd = nn::Sgd::default().build(&vs, 1e-3)?; +for epoch_idx in 1..1001 { + let predicted = train_images.apply(&linear); + let loss = predicted.cross_entropy_for_logits(&dataset.train_labels); + sgd.backward_step(&loss); + + let test_accuracy = test_images + .apply(&linear) + .accuracy_for_logits(&dataset.test_labels); + println!("{} {:.2}%", epoch_idx, 100. * f64::from(test_accuracy)); +} +``` + +On each training step the model output is computed through a forward pass. The +cross-entropy loss is then evaluated on the resulting logits using the training labels. +The backward pass then evaluates gradients for the trainable variables of our +model and these variables are updated by the optimizer. +```rust + let predicted = train_images.apply(&linear); + let loss = predicted.cross_entropy_for_logits(&dataset.train_labels); + sgd.backward_step(&loss); +``` + +After each epoch the accuracy is evaluated on the testing set and printed out. +```rust + let test_accuracy = test_images + .apply(&linear) + .accuracy_for_logits(&dataset.test_labels); + println!("{} {:.2}%", epoch_idx, 100. * f64::from(test_accuracy)); +``` + +This should result in a `94.5%` accuracy on the testing set. +The whole code for this example can be found in [main.rs](main.rs). + diff --git a/v2/dependency-patches/tch-0.24.0/examples/transfer-learning/main.rs b/v2/dependency-patches/tch-0.24.0/examples/transfer-learning/main.rs new file mode 100644 index 0000000000..bb773e50e5 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/transfer-learning/main.rs @@ -0,0 +1,42 @@ +// This example illustrates how to use transfer learning to fine tune a pre-trained +// imagenet model on another dataset. +// +// The pre-trained weight files containing the pre-trained weights can be found here: +// https://github.com/LaurentMazare/tch-rs/releases/download/untagged-eb220e5c19f9bb250bd1/resnet18.ot +use anyhow::{bail, Result}; +use tch::nn::{self, OptimizerConfig}; +use tch::vision::{imagenet, resnet}; + +pub fn main() -> Result<()> { + let args: Vec<_> = std::env::args().collect(); + let (weights, dataset_dir) = match args.as_slice() { + [_, w, d] => (std::path::Path::new(w), d.to_owned()), + _ => bail!("usage: main resnet18.ot dataset-path"), + }; + // Load the dataset and resize it to the usual imagenet dimension of 224x224. + let dataset = imagenet::load_from_dir(dataset_dir)?; + println!("{dataset:?}"); + + // Create the model and load the weights from the file. + let mut vs = tch::nn::VarStore::new(tch::Device::Cpu); + let net = resnet::resnet18_no_final_layer(&vs.root()); + vs.load(weights)?; + + // Pre-compute the final activations. + let train_images = tch::no_grad(|| dataset.train_images.apply_t(&net, false)); + let test_images = tch::no_grad(|| dataset.test_images.apply_t(&net, false)); + + let vs = nn::VarStore::new(tch::Device::Cpu); + let linear = nn::linear(vs.root(), 512, dataset.labels, Default::default()); + let mut sgd = nn::Sgd::default().build(&vs, 1e-3)?; + + for epoch_idx in 1..1001 { + let predicted = train_images.apply(&linear); + let loss = predicted.cross_entropy_for_logits(&dataset.train_labels); + sgd.backward_step(&loss); + + let test_accuracy = test_images.apply(&linear).accuracy_for_logits(&dataset.test_labels); + println!("{} {:.2}%", epoch_idx, 100. * f64::try_from(test_accuracy)?); + } + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/translation/dataset.rs b/v2/dependency-patches/tch-0.24.0/examples/translation/dataset.rs new file mode 100644 index 0000000000..cf2f61e135 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/translation/dataset.rs @@ -0,0 +1,115 @@ +use super::lang; +use anyhow::{bail, Result}; +use std::fs::File; +use std::io::{BufRead, BufReader}; + +const PREFIXES: [&str; 12] = [ + "i am ", + "i m ", + "you are ", + "you re", + "he is ", + "he s ", + "she is ", + "she s ", + "we are ", + "we re ", + "they are ", + "they re ", +]; + +#[derive(Debug)] +pub struct Dataset { + input_lang: lang::Lang, + output_lang: lang::Lang, + pairs: Vec<(String, String)>, +} + +fn normalize(s: &str) -> String { + s.to_lowercase() + .chars() + .map(|c| match c { + '!' => " !".to_string(), + '.' => " .".to_string(), + '?' => " ?".to_string(), + _ => { + if c.is_alphanumeric() { + c.to_string() + } else { + " ".to_string() + } + } + }) + .collect() +} + +fn to_indexes(s: &str, lang: &lang::Lang) -> Vec { + let mut res = s.split_whitespace().filter_map(|x| lang.get_index(x)).collect::>(); + res.push(lang.eos_token()); + res +} + +fn filter_prefix(s: &str) -> bool { + PREFIXES.iter().any(|prefix| s.starts_with(prefix)) +} + +fn read_pairs(ilang: &str, olang: &str, max_length: usize) -> Result> { + let file = File::open(format!("data/{ilang}-{olang}.txt"))?; + let mut res: Vec<(String, String)> = vec![]; + for line in BufReader::new(file).lines() { + let line = line?; + match line.split('\t').collect::>().as_slice() { + [lhs, rhs] => { + let lhs = normalize(lhs); + let rhs = normalize(rhs); + if lhs.split_whitespace().count() < max_length + && rhs.split_whitespace().count() < max_length + && (filter_prefix(&lhs) || filter_prefix(&rhs)) + { + res.push((lhs, rhs)) + } + } + _ => bail!("a line does not contain a single tab {}", line), + } + } + Ok(res) +} + +impl Dataset { + pub fn new(ilang: &str, olang: &str, max_length: usize) -> Result { + let pairs = read_pairs(ilang, olang, max_length)?; + let mut input_lang = lang::Lang::new(ilang); + let mut output_lang = lang::Lang::new(olang); + for (lhs, rhs) in pairs.iter() { + input_lang.add_sentence(lhs); + output_lang.add_sentence(rhs); + } + let dataset = Dataset { input_lang, output_lang, pairs }; + Ok(dataset) + } + + pub fn input_lang(&self) -> &lang::Lang { + &self.input_lang + } + + pub fn output_lang(&self) -> &lang::Lang { + &self.output_lang + } + + pub fn reverse(self) -> Self { + Dataset { + input_lang: self.output_lang, + output_lang: self.input_lang, + pairs: self.pairs.into_iter().map(|(x, y)| (y, x)).collect(), + } + } + + pub fn pairs(&self) -> Vec<(Vec, Vec)> { + self.pairs + .iter() + .map(|(lhs, rhs)| { + (to_indexes(lhs, &self.input_lang), to_indexes(rhs, &self.output_lang)) + }) + .collect::>() + } +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/translation/lang.rs b/v2/dependency-patches/tch-0.24.0/examples/translation/lang.rs new file mode 100644 index 0000000000..b85371d605 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/translation/lang.rs @@ -0,0 +1,73 @@ +use std::collections::HashMap; + +const SOS_TOKEN: &str = "SOS"; +const EOS_TOKEN: &str = "EOS"; + +#[derive(Debug)] +pub struct Lang { + name: String, + word_to_index_and_count: HashMap, + index_to_word: HashMap, +} + +impl Lang { + fn add_word(&mut self, word: &str) { + if !word.is_empty() { + match self.word_to_index_and_count.get_mut(word) { + None => { + let length = self.word_to_index_and_count.len(); + self.word_to_index_and_count.insert(word.to_string(), (length, 1)); + self.index_to_word.insert(length, word.to_owned()); + } + Some((_, cnt)) => { + *cnt += 1; + } + } + } + } + + pub fn new(name: &str) -> Lang { + let mut lang = Lang { + name: name.to_string(), + word_to_index_and_count: HashMap::new(), + index_to_word: HashMap::new(), + }; + lang.add_word(SOS_TOKEN); + lang.add_word(EOS_TOKEN); + lang + } + + pub fn add_sentence(&mut self, sentence: &str) { + for word in sentence.split_whitespace() { + self.add_word(word); + } + } + + pub fn len(&self) -> usize { + self.index_to_word.len() + } + + pub fn sos_token(&self) -> usize { + self.word_to_index_and_count.get(SOS_TOKEN).unwrap().0 + } + + pub fn eos_token(&self) -> usize { + self.word_to_index_and_count.get(EOS_TOKEN).unwrap().0 + } + + pub fn name(&self) -> &str { + &self.name + } + + pub fn get_index(&self, word: &str) -> Option { + self.word_to_index_and_count.get(word).map(|x| x.0) + } + + pub fn seq_to_string(&self, seq: &[usize]) -> String { + let mut res: Vec = vec![]; + for s in seq.iter() { + res.push(self.index_to_word.get(s).unwrap().to_string()); + } + res.join(" ") + } +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/translation/main.rs b/v2/dependency-patches/tch-0.24.0/examples/translation/main.rs new file mode 100644 index 0000000000..504fe7061a --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/translation/main.rs @@ -0,0 +1,224 @@ +/* Translation with a Sequence to Sequence Model and Attention. + + This follows the line of the PyTorch tutorial: + https://pytorch.org/tutorials/intermediate/seq2seq_translation_tutorial.html + And trains a Sequence to Sequence (seq2seq) model using attention to + perform translation between French and English. + + The dataset can be downloaded from the following link: + https://download.pytorch.org/tutorial/data.zip + The eng-fra.txt file should be moved in the data directory. +*/ +use anyhow::Result; +use rand::prelude::*; +use tch::nn::{GRUState, Module, OptimizerConfig, RNN}; +use tch::{nn, Device, Kind, Tensor}; + +mod dataset; +use dataset::Dataset; +mod lang; +use lang::Lang; + +const MAX_LENGTH: usize = 10; +const LEARNING_RATE: f64 = 0.001; +const SAMPLES: usize = 100_000; +const HIDDEN_SIZE: usize = 256; + +struct Encoder { + embedding: nn::Embedding, + gru: nn::GRU, +} + +impl Encoder { + fn new(vs: nn::Path, in_dim: usize, hidden_dim: usize) -> Self { + let in_dim = in_dim as i64; + let hidden_dim = hidden_dim as i64; + let gru = nn::gru(&vs, hidden_dim, hidden_dim, Default::default()); + let embedding = nn::embedding(&vs, in_dim, hidden_dim, Default::default()); + Encoder { embedding, gru } + } + + fn forward(&self, xs: &Tensor, state: &GRUState) -> (Tensor, GRUState) { + let xs = self.embedding.forward(xs).view([1, -1]); + let state = self.gru.step(&xs, state); + (state.value().squeeze_dim(1), state) + } +} + +struct Decoder { + device: Device, + embedding: nn::Embedding, + gru: nn::GRU, + attn: nn::Linear, + attn_combine: nn::Linear, + linear: nn::Linear, +} + +impl Decoder { + fn new(vs: nn::Path, hidden_dim: usize, out_dim: usize) -> Self { + let hidden_dim = hidden_dim as i64; + let out_dim = out_dim as i64; + Decoder { + device: vs.device(), + embedding: nn::embedding(&vs, out_dim, hidden_dim, Default::default()), + gru: nn::gru(&vs, hidden_dim, hidden_dim, Default::default()), + attn: nn::linear(&vs, hidden_dim * 2, MAX_LENGTH as i64, Default::default()), + attn_combine: nn::linear(&vs, hidden_dim * 2, hidden_dim, Default::default()), + linear: nn::linear(&vs, hidden_dim, out_dim, Default::default()), + } + } + + fn forward( + &self, + xs: &Tensor, + state: &GRUState, + enc_outputs: &Tensor, + is_training: bool, + ) -> (Tensor, GRUState) { + let xs = self.embedding.forward(xs).dropout(0.1, is_training).view([1, -1]); + let attn_weights = + Tensor::cat(&[&xs, &state.value().squeeze_dim(1)], 1).apply(&self.attn).unsqueeze(0); + let (sz1, sz2, sz3) = enc_outputs.size3().unwrap(); + let enc_outputs = if sz2 == MAX_LENGTH as i64 { + enc_outputs.shallow_clone() + } else { + let shape = [sz1, MAX_LENGTH as i64 - sz2, sz3]; + let zeros = Tensor::zeros(shape, (Kind::Float, self.device)); + Tensor::cat(&[enc_outputs, &zeros], 1) + }; + let attn_applied = attn_weights.bmm(&enc_outputs).squeeze_dim(1); + let xs = Tensor::cat(&[&xs, &attn_applied], 1).apply(&self.attn_combine).relu(); + let state = self.gru.step(&xs, state); + (self.linear.forward(&state.value()).log_softmax(-1, Kind::Float).squeeze_dim(1), state) + } +} + +struct Model { + encoder: Encoder, + decoder: Decoder, + decoder_start: Tensor, + decoder_eos: usize, + device: Device, +} + +impl Model { + fn new(vs: nn::Path, ilang: &Lang, olang: &Lang, hidden_dim: usize) -> Self { + Model { + encoder: Encoder::new(&vs / "enc", ilang.len(), hidden_dim), + decoder: Decoder::new(&vs / "dec", hidden_dim, olang.len()), + decoder_start: Tensor::from_slice(&[olang.sos_token() as i64]).to_device(vs.device()), + decoder_eos: olang.eos_token(), + device: vs.device(), + } + } + + #[allow(clippy::assign_op_pattern)] + fn train_loss(&self, input_: &[usize], target: &[usize], rng: &mut ThreadRng) -> Tensor { + let mut state = self.encoder.gru.zero_state(1); + let mut enc_outputs = vec![]; + for &s in input_.iter() { + let s = Tensor::from_slice(&[s as i64]).to_device(self.device); + let (out, state_) = self.encoder.forward(&s, &state); + enc_outputs.push(out); + state = state_; + } + let enc_outputs = Tensor::stack(&enc_outputs, 1); + let use_teacher_forcing: bool = rng.gen(); + let mut loss = Tensor::from(0f32).to_device(self.device); + let mut prev = self.decoder_start.shallow_clone(); + for &s in target.iter() { + let (output, state_) = self.decoder.forward(&prev, &state, &enc_outputs, true); + state = state_; + let target_tensor = Tensor::from_slice(&[s as i64]).to_device(self.device); + loss = loss + output.nll_loss(&target_tensor); + let (_, output) = output.topk(1, -1, true, true); + if self.decoder_eos == i64::try_from(&output).unwrap() as usize { + break; + } + prev = if use_teacher_forcing { target_tensor } else { output }; + } + loss + } + + fn predict(&self, input_: &[usize]) -> Vec { + let mut state = self.encoder.gru.zero_state(1); + let mut enc_outputs = vec![]; + for &s in input_.iter() { + let s = Tensor::from_slice(&[s as i64]).to_device(self.device); + let (out, state_) = self.encoder.forward(&s, &state); + enc_outputs.push(out); + state = state_; + } + let enc_outputs = Tensor::stack(&enc_outputs, 1); + let mut prev = self.decoder_start.shallow_clone(); + let mut output_seq: Vec = vec![]; + for _i in 0..MAX_LENGTH { + let (output, state_) = self.decoder.forward(&prev, &state, &enc_outputs, true); + let (_, output) = output.topk(1, -1, true, true); + let output_ = i64::try_from(&output).unwrap() as usize; + output_seq.push(output_); + if self.decoder_eos == output_ { + break; + } + state = state_; + prev = output; + } + output_seq + } +} + +struct LossStats { + total_loss: f64, + samples: usize, +} + +impl LossStats { + fn new() -> LossStats { + LossStats { total_loss: 0., samples: 0 } + } + + fn update(&mut self, loss: f64) { + self.total_loss += loss; + self.samples += 1; + } + + fn avg_and_reset(&mut self) -> f64 { + let avg = self.total_loss / self.samples as f64; + self.total_loss = 0.; + self.samples = 0; + avg + } +} + +pub fn main() -> Result<()> { + let dataset = Dataset::new("eng", "fra", MAX_LENGTH)?.reverse(); + let ilang = dataset.input_lang(); + let olang = dataset.output_lang(); + let pairs = dataset.pairs(); + println!("Input: {} {} words.", ilang.name(), ilang.len()); + println!("Output: {} {} words.", olang.name(), olang.len()); + println!("Pairs: {}.", pairs.len()); + let mut rng = thread_rng(); + let device = Device::cuda_if_available(); + let vs = nn::VarStore::new(device); + let model = Model::new(vs.root(), ilang, olang, HIDDEN_SIZE); + let mut opt = nn::Adam::default().build(&vs, LEARNING_RATE)?; + let mut loss_stats = LossStats::new(); + for idx in 1..=SAMPLES { + let (input_, target) = pairs.choose(&mut rng).unwrap(); + let loss = model.train_loss(input_, target, &mut rng); + opt.backward_step(&loss); + loss_stats.update(f64::try_from(loss)? / target.len() as f64); + if idx % 1000 == 0 { + println!("{} {}", idx, loss_stats.avg_and_reset()); + for _pred_index in 1..5 { + let (input_, target) = pairs.choose(&mut rng).unwrap(); + let predict = model.predict(input_); + println!("in: {}", ilang.seq_to_string(input_)); + println!("tgt: {}", olang.seq_to_string(target)); + println!("out: {}", olang.seq_to_string(&predict)); + } + } + } + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/vae/main.rs b/v2/dependency-patches/tch-0.24.0/examples/vae/main.rs new file mode 100644 index 0000000000..96b7562799 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/vae/main.rs @@ -0,0 +1,95 @@ +/* Variational Auto-Encoder on MNIST. + The implementation is based on: + https://github.com/pytorch/examples/blob/master/vae/main.py + + The 4 following dataset files can be downloaded from http://yann.lecun.com/exdb/mnist/ + These files should be extracted in the 'data' directory. + train-images-idx3-ubyte.gz + train-labels-idx1-ubyte.gz + t10k-images-idx3-ubyte.gz + t10k-labels-idx1-ubyte.gz +*/ + +use anyhow::Result; +use tch::{nn, nn::Module, nn::OptimizerConfig, Kind, Reduction, Tensor}; + +struct Vae { + fc1: nn::Linear, + fc21: nn::Linear, + fc22: nn::Linear, + fc3: nn::Linear, + fc4: nn::Linear, +} + +impl Vae { + fn new(vs: &nn::Path) -> Self { + Vae { + fc1: nn::linear(vs / "fc1", 784, 400, Default::default()), + fc21: nn::linear(vs / "fc21", 400, 20, Default::default()), + fc22: nn::linear(vs / "fc22", 400, 20, Default::default()), + fc3: nn::linear(vs / "fc3", 20, 400, Default::default()), + fc4: nn::linear(vs / "fc4", 400, 784, Default::default()), + } + } + + fn encode(&self, xs: &Tensor) -> (Tensor, Tensor) { + let h1 = xs.apply(&self.fc1).relu(); + (self.fc21.forward(&h1), self.fc22.forward(&h1)) + } + + fn decode(&self, zs: &Tensor) -> Tensor { + zs.apply(&self.fc3).relu().apply(&self.fc4).sigmoid() + } + + fn forward(&self, xs: &Tensor) -> (Tensor, Tensor, Tensor) { + let (mu, logvar) = self.encode(&xs.view([-1, 784])); + let std = (&logvar * 0.5).exp(); + let eps = std.randn_like(); + (self.decode(&(&mu + eps * std)), mu, logvar) + } +} + +// Reconstruction + KL divergence losses summed over all elements and batch dimension. +fn loss(recon_x: &Tensor, x: &Tensor, mu: &Tensor, logvar: &Tensor) -> Tensor { + let bce = recon_x.binary_cross_entropy::(&x.view([-1, 784]), None, Reduction::Sum); + // See Appendix B from VAE paper: + // Kingma and Welling. Auto-Encoding Variational Bayes. ICLR, 2014 + // https://arxiv.org/abs/1312.6114 + // 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2) + let kld = -0.5 * (1i64 + logvar - mu.pow_tensor_scalar(2) - logvar.exp()).sum(Kind::Float); + bce + kld +} + +// Generate a 2D matrix of images from a tensor with multiple images. +fn image_matrix(imgs: &Tensor, sz: i64) -> Result { + let imgs = (imgs * 256.).clamp(0., 255.).to_kind(Kind::Uint8); + let mut ys: Vec = vec![]; + for i in 0..sz { + ys.push(Tensor::cat(&(0..sz).map(|j| imgs.narrow(0, 4 * i + j, 1)).collect::>(), 2)) + } + Ok(Tensor::cat(&ys, 3).squeeze_dim(0)) +} + +pub fn main() -> Result<()> { + let device = tch::Device::cuda_if_available(); + let m = tch::vision::mnist::load_dir("data")?; + let vs = nn::VarStore::new(device); + let vae = Vae::new(&vs.root()); + let mut opt = nn::Adam::default().build(&vs, 1e-3)?; + for epoch in 1..21 { + let mut train_loss = 0f64; + let mut samples = 0f64; + for (bimages, _) in m.train_iter(128).shuffle().to_device(vs.device()) { + let (recon_batch, mu, logvar) = vae.forward(&bimages); + let loss = loss(&recon_batch, &bimages, &mu, &logvar); + opt.backward_step(&loss); + train_loss += f64::try_from(&loss)?; + samples += bimages.size()[0] as f64; + } + println!("Epoch: {}, loss: {}", epoch, train_loss / samples); + let s = Tensor::randn([64, 20], tch::kind::FLOAT_CPU).to(device); + let s = vae.decode(&s).to(tch::Device::Cpu).view([64, 1, 28, 28]); + tch::vision::image::save(&image_matrix(&s, 8)?, format!("s_{epoch}.png"))? + } + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/yolo/coco_classes.rs b/v2/dependency-patches/tch-0.24.0/examples/yolo/coco_classes.rs new file mode 100644 index 0000000000..0075352492 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/yolo/coco_classes.rs @@ -0,0 +1,82 @@ +pub const NAMES: [&str; 80] = [ + "person", + "bicycle", + "car", + "motorbike", + "aeroplane", + "bus", + "train", + "truck", + "boat", + "traffic light", + "fire hydrant", + "stop sign", + "parking meter", + "bench", + "bird", + "cat", + "dog", + "horse", + "sheep", + "cow", + "elephant", + "bear", + "zebra", + "giraffe", + "backpack", + "umbrella", + "handbag", + "tie", + "suitcase", + "frisbee", + "skis", + "snowboard", + "sports ball", + "kite", + "baseball bat", + "baseball glove", + "skateboard", + "surfboard", + "tennis racket", + "bottle", + "wine glass", + "cup", + "fork", + "knife", + "spoon", + "bowl", + "banana", + "apple", + "sandwich", + "orange", + "broccoli", + "carrot", + "hot dog", + "pizza", + "donut", + "cake", + "chair", + "sofa", + "pottedplant", + "bed", + "diningtable", + "toilet", + "tvmonitor", + "laptop", + "mouse", + "remote", + "keyboard", + "cell phone", + "microwave", + "oven", + "toaster", + "sink", + "refrigerator", + "book", + "clock", + "vase", + "scissors", + "teddy bear", + "hair drier", + "toothbrush", +]; diff --git a/v2/dependency-patches/tch-0.24.0/examples/yolo/darknet.rs b/v2/dependency-patches/tch-0.24.0/examples/yolo/darknet.rs new file mode 100644 index 0000000000..283b1cadf3 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/yolo/darknet.rs @@ -0,0 +1,286 @@ +use anyhow::{bail, ensure, Result}; +use std::collections::BTreeMap; +use std::fs::File; +use std::io::{BufRead, BufReader}; +use std::path::Path; +use tch::{ + nn, + nn::{FuncT, ModuleT}, + Tensor, +}; + +#[derive(Debug)] +struct Block { + block_type: String, + parameters: BTreeMap, +} + +impl Block { + fn get(&self, key: &str) -> Result<&str> { + match self.parameters.get(key) { + None => bail!("cannot find {} in {}", key, self.block_type), + Some(value) => Ok(value), + } + } +} + +#[derive(Debug)] +pub struct Darknet { + blocks: Vec, + parameters: BTreeMap, +} + +impl Darknet { + fn get(&self, key: &str) -> Result<&str> { + match self.parameters.get(key) { + None => bail!("cannot find {} in net parameters", key), + Some(value) => Ok(value), + } + } +} + +struct Accumulator { + block_type: Option, + parameters: BTreeMap, + net: Darknet, +} + +impl Accumulator { + fn new() -> Accumulator { + Accumulator { + block_type: None, + parameters: BTreeMap::new(), + net: Darknet { blocks: vec![], parameters: BTreeMap::new() }, + } + } + + fn finish_block(&mut self) { + match &self.block_type { + None => (), + Some(block_type) => { + if block_type == "net" { + self.net.parameters = self.parameters.clone(); + } else { + let block = Block { + block_type: block_type.to_string(), + parameters: self.parameters.clone(), + }; + self.net.blocks.push(block); + } + self.parameters.clear(); + } + } + self.block_type = None; + } +} + +pub fn parse_config>(path: T) -> Result { + let file = File::open(path.as_ref())?; + let mut acc = Accumulator::new(); + for line in BufReader::new(file).lines() { + let line = line?; + if line.is_empty() || line.starts_with('#') { + continue; + } + let line = line.trim(); + if line.starts_with('[') { + ensure!(line.ends_with(']'), "line does not end with ']' {}", line); + let line = &line[1..line.len() - 1]; + acc.finish_block(); + acc.block_type = Some(line.to_string()); + } else { + let key_value: Vec<&str> = line.splitn(2, '=').collect(); + ensure!(key_value.len() == 2, "missing equal {}", line); + let prev = acc + .parameters + .insert(key_value[0].trim().to_owned(), key_value[1].trim().to_owned()); + ensure!(prev.is_none(), "multiple value for key {}", line); + } + } + acc.finish_block(); + Ok(acc.net) +} + +enum Bl { + Layer(Box), + Route(Vec), + Shortcut(usize), + Yolo(i64, Vec<(i64, i64)>), +} + +fn conv(vs: nn::Path, index: usize, p: i64, b: &Block) -> Result<(i64, Bl)> { + let activation = b.get("activation")?; + let filters = b.get("filters")?.parse::()?; + let pad = b.get("pad")?.parse::()?; + let size = b.get("size")?.parse::()?; + let stride = b.get("stride")?.parse::()?; + let pad = if pad != 0 { (size - 1) / 2 } else { 0 }; + let (bn, bias) = match b.parameters.get("batch_normalize") { + Some(p) if p.parse::()? != 0 => { + let vs = &vs / format!("batch_norm_{index}"); + let bn = nn::batch_norm2d(vs, filters, Default::default()); + (Some(bn), false) + } + Some(_) | None => (None, true), + }; + let conv_cfg = nn::ConvConfig { stride, padding: pad, bias, ..Default::default() }; + let vs = &vs / format!("conv_{index}"); + let conv = nn::conv2d(vs, p, filters, size, conv_cfg); + let leaky = match activation { + "leaky" => true, + "linear" => false, + otherwise => bail!("unsupported activation {}", otherwise), + }; + let func = nn::func_t(move |xs, train| { + let xs = xs.apply(&conv); + let xs = match &bn { + Some(bn) => xs.apply_t(bn, train), + None => xs, + }; + if leaky { + xs.maximum(&(&xs * 0.1)) + } else { + xs + } + }); + Ok((filters, Bl::Layer(Box::new(func)))) +} + +fn upsample(prev_channels: i64) -> Result<(i64, Bl)> { + let layer = nn::func_t(|xs, _is_training| { + let (_n, _c, h, w) = xs.size4().unwrap(); + xs.upsample_nearest2d([2 * h, 2 * w], 2.0, 2.0) + }); + Ok((prev_channels, Bl::Layer(Box::new(layer)))) +} + +fn int_list_of_string(s: &str) -> Result> { + let res: Result, _> = s.split(',').map(|xs| xs.trim().parse::()).collect(); + Ok(res?) +} + +fn usize_of_index(index: usize, i: i64) -> usize { + if i >= 0 { + i as usize + } else { + (index as i64 + i) as usize + } +} + +fn route(index: usize, p: &[(i64, Bl)], block: &Block) -> Result<(i64, Bl)> { + let layers = int_list_of_string(block.get("layers")?)?; + let layers: Vec = layers.into_iter().map(|l| usize_of_index(index, l)).collect(); + let channels = layers.iter().map(|&l| p[l].0).sum(); + Ok((channels, Bl::Route(layers))) +} + +fn shortcut(index: usize, p: i64, block: &Block) -> Result<(i64, Bl)> { + let from = block.get("from")?.parse::()?; + Ok((p, Bl::Shortcut(usize_of_index(index, from)))) +} + +fn yolo(p: i64, block: &Block) -> Result<(i64, Bl)> { + let classes = block.get("classes")?.parse::()?; + let flat = int_list_of_string(block.get("anchors")?)?; + ensure!(flat.len() % 2 == 0, "even number of anchors"); + let anchors: Vec<_> = (0..(flat.len() / 2)).map(|i| (flat[2 * i], flat[2 * i + 1])).collect(); + let mask = int_list_of_string(block.get("mask")?)?; + let anchors = mask.into_iter().map(|i| anchors[i as usize]).collect(); + Ok((p, Bl::Yolo(classes, anchors))) +} + +// Apply f to a slice of tensor xs and replace xs values with f output. +fn slice_apply_and_set(xs: &mut Tensor, start: i64, len: i64, f: F) +where + F: FnOnce(&Tensor) -> Tensor, +{ + let mut slice = xs.narrow(2, start, len); + let src = f(&slice); + slice.copy_(&src) +} + +fn detect(xs: &Tensor, image_height: i64, classes: i64, anchors: &[(i64, i64)]) -> Tensor { + let (bsize, _channels, height, _width) = xs.size4().unwrap(); + let stride = image_height / height; + let grid_size = image_height / stride; + let bbox_attrs = 5 + classes; + let nanchors = anchors.len() as i64; + let mut xs = xs + .view((bsize, bbox_attrs * nanchors, grid_size * grid_size)) + .transpose(1, 2) + .contiguous() + .view((bsize, grid_size * grid_size * nanchors, bbox_attrs)); + let grid = Tensor::arange(grid_size, tch::kind::FLOAT_CPU); + let a = grid.repeat([grid_size, 1]); + let b = a.tr().contiguous(); + let x_offset = a.view((-1, 1)); + let y_offset = b.view((-1, 1)); + let xy_offset = + Tensor::cat(&[x_offset, y_offset], 1).repeat([1, nanchors]).view((-1, 2)).unsqueeze(0); + let anchors: Vec = anchors + .iter() + .flat_map(|&(x, y)| vec![x as f32 / stride as f32, y as f32 / stride as f32].into_iter()) + .collect(); + let anchors = + Tensor::from_slice(&anchors).view((-1, 2)).repeat([grid_size * grid_size, 1]).unsqueeze(0); + slice_apply_and_set(&mut xs, 0, 2, |xs| xs.sigmoid() + xy_offset); + slice_apply_and_set(&mut xs, 4, 1 + classes, Tensor::sigmoid); + slice_apply_and_set(&mut xs, 2, 2, |xs| xs.exp() * anchors); + slice_apply_and_set(&mut xs, 0, 4, |xs| xs * stride); + xs +} + +impl Darknet { + pub fn height(&self) -> Result { + let image_height = self.get("height")?.parse::()?; + Ok(image_height) + } + + pub fn width(&self) -> Result { + let image_width = self.get("width")?.parse::()?; + Ok(image_width) + } + + pub fn build_model(&self, vs: &nn::Path) -> Result> { + let mut blocks: Vec<(i64, Bl)> = vec![]; + let mut prev_channels: i64 = 3; + for (index, block) in self.blocks.iter().enumerate() { + let channels_and_bl = match block.block_type.as_str() { + "convolutional" => conv(vs / index, index, prev_channels, block)?, + "upsample" => upsample(prev_channels)?, + "shortcut" => shortcut(index, prev_channels, block)?, + "route" => route(index, &blocks, block)?, + "yolo" => yolo(prev_channels, block)?, + otherwise => bail!("unsupported block type {}", otherwise), + }; + prev_channels = channels_and_bl.0; + blocks.push(channels_and_bl); + } + let image_height = self.height()?; + let func = nn::func_t(move |xs, train| { + let mut prev_ys: Vec = vec![]; + let mut detections: Vec = vec![]; + for (_, b) in blocks.iter() { + let ys = match b { + Bl::Layer(l) => { + let xs = prev_ys.last().unwrap_or(xs); + l.forward_t(xs, train) + } + Bl::Route(layers) => { + let layers: Vec<_> = layers.iter().map(|&i| &prev_ys[i]).collect(); + Tensor::cat(&layers, 1) + } + Bl::Shortcut(from) => prev_ys.last().unwrap() + prev_ys.get(*from).unwrap(), + Bl::Yolo(classes, anchors) => { + let xs = prev_ys.last().unwrap_or(xs); + detections.push(detect(xs, image_height, *classes, anchors)); + Tensor::default() + } + }; + prev_ys.push(ys); + } + Tensor::cat(&detections, 1) + }); + Ok(func) + } +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/yolo/main.rs b/v2/dependency-patches/tch-0.24.0/examples/yolo/main.rs new file mode 100644 index 0000000000..a84281f19d --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/yolo/main.rs @@ -0,0 +1,153 @@ +// The pre-trained weights can be downloaded here: +// https://github.com/LaurentMazare/ocaml-torch/releases/download/v0.1-unstable/yolo-v3.ot +// +// These weights have been generated by using the Python version at: +// https://github.com/ayooshkathuria/YOLO_v3_tutorial_from_scratch +// The weights are exported in npz format by adding the following code in detect.py after the +// model.load_weights(...) call. +// +// ```python +// def remove_prefix(text, prefix): +// return text[text.startswith(prefix) and len(prefix):] +// nps = {} +// for k, v in model.state_dict().items(): +// k = remove_prefix(k, 'module_list.') +// nps[k] = v.detach().numpy() +// np.savez('yolo-v3.ot', **nps) +// ``` +// +// Then tch-rs tensor-tools example can be used to convert the .npz weights +// to the requested .ot file. +// cargo run --example tensor-tools cp yolo-v3.ot.npz yolo-v3.ot + +mod coco_classes; +mod darknet; + +use anyhow::{ensure, Result}; +use tch::nn::ModuleT; +use tch::vision::image; +use tch::Tensor; +const CONFIG_NAME: &str = "examples/yolo/yolo-v3.cfg"; +const CONFIDENCE_THRESHOLD: f64 = 0.5; +const NMS_THRESHOLD: f64 = 0.4; + +#[derive(Debug, Clone, Copy)] +struct Bbox { + xmin: f64, + ymin: f64, + xmax: f64, + ymax: f64, + confidence: f64, +} + +// Intersection over union of two bounding boxes. +fn iou(b1: &Bbox, b2: &Bbox) -> f64 { + let b1_area = (b1.xmax - b1.xmin + 1.) * (b1.ymax - b1.ymin + 1.); + let b2_area = (b2.xmax - b2.xmin + 1.) * (b2.ymax - b2.ymin + 1.); + let i_xmin = b1.xmin.max(b2.xmin); + let i_xmax = b1.xmax.min(b2.xmax); + let i_ymin = b1.ymin.max(b2.ymin); + let i_ymax = b1.ymax.min(b2.ymax); + let i_area = (i_xmax - i_xmin + 1.).max(0.) * (i_ymax - i_ymin + 1.).max(0.); + i_area / (b1_area + b2_area - i_area) +} + +// Assumes x1 <= x2 and y1 <= y2 +pub fn draw_rect(t: &mut Tensor, x1: i64, x2: i64, y1: i64, y2: i64) { + let color = Tensor::from_slice(&[0., 0., 1.]).view([3, 1, 1]); + t.narrow(2, x1, x2 - x1).narrow(1, y1, y2 - y1).copy_(&color) +} + +pub fn report(pred: &Tensor, img: &Tensor, w: i64, h: i64) -> Result { + let (npreds, pred_size) = pred.size2()?; + let nclasses = (pred_size - 5) as usize; + // The bounding boxes grouped by (maximum) class index. + let mut bboxes: Vec> = (0..nclasses).map(|_| vec![]).collect(); + // Extract the bounding boxes for which confidence is above the threshold. + for index in 0..npreds { + let pred = Vec::::try_from(pred.get(index))?; + let confidence = pred[4]; + if confidence > CONFIDENCE_THRESHOLD { + let mut class_index = 0; + for i in 0..nclasses { + if pred[5 + i] > pred[5 + class_index] { + class_index = i + } + } + if pred[class_index + 5] > 0. { + let bbox = Bbox { + xmin: pred[0] - pred[2] / 2., + ymin: pred[1] - pred[3] / 2., + xmax: pred[0] + pred[2] / 2., + ymax: pred[1] + pred[3] / 2., + confidence, + }; + bboxes[class_index].push(bbox) + } + } + } + // Perform non-maximum suppression. + for bboxes_for_class in bboxes.iter_mut() { + bboxes_for_class.sort_by(|b1, b2| b2.confidence.partial_cmp(&b1.confidence).unwrap()); + let mut current_index = 0; + for index in 0..bboxes_for_class.len() { + let mut drop = false; + for prev_index in 0..current_index { + let iou = iou(&bboxes_for_class[prev_index], &bboxes_for_class[index]); + if iou > NMS_THRESHOLD { + drop = true; + break; + } + } + if !drop { + bboxes_for_class.swap(current_index, index); + current_index += 1; + } + } + bboxes_for_class.truncate(current_index); + } + // Annotate the original image and print boxes information. + let (_, initial_h, initial_w) = img.size3()?; + let mut img = img.to_kind(tch::Kind::Float) / 255.; + let w_ratio = initial_w as f64 / w as f64; + let h_ratio = initial_h as f64 / h as f64; + for (class_index, bboxes_for_class) in bboxes.iter().enumerate() { + for b in bboxes_for_class.iter() { + println!("{}: {:?}", coco_classes::NAMES[class_index], b); + let xmin = ((b.xmin * w_ratio) as i64).clamp(0, initial_w - 1); + let ymin = ((b.ymin * h_ratio) as i64).clamp(0, initial_h - 1); + let xmax = ((b.xmax * w_ratio) as i64).clamp(0, initial_w - 1); + let ymax = ((b.ymax * h_ratio) as i64).clamp(0, initial_h - 1); + draw_rect(&mut img, xmin, xmax, ymin, ymax.min(ymin + 2)); + draw_rect(&mut img, xmin, xmax, ymin.max(ymax - 2), ymax); + draw_rect(&mut img, xmin, xmax.min(xmin + 2), ymin, ymax); + draw_rect(&mut img, xmin.max(xmax - 2), xmax, ymin, ymax); + } + } + Ok((img * 255.).to_kind(tch::Kind::Uint8)) +} + +pub fn main() -> Result<()> { + let args: Vec<_> = std::env::args().collect(); + ensure!(args.len() >= 3, "usage: main yolo-v3.ot img.jpg ..."); + + // Create the model and load the weights from the file. + let mut vs = tch::nn::VarStore::new(tch::Device::Cpu); + let darknet = darknet::parse_config(CONFIG_NAME)?; + let model = darknet.build_model(&vs.root())?; + vs.load(&args[1])?; + + for (index, image) in args.iter().skip(2).enumerate() { + // Load the image file and resize it. + let original_image = image::load(image)?; + let net_width = darknet.width()?; + let net_height = darknet.height()?; + let image = image::resize(&original_image, net_width, net_height)?; + let image = image.unsqueeze(0).to_kind(tch::Kind::Float) / 255.; + let predictions = model.forward_t(&image, false).squeeze(); + let image = report(&predictions, &original_image, net_width, net_height)?; + image::save(&image, format!("output-{index:05}.jpg"))?; + println!("Converted {index}"); + } + Ok(()) +} diff --git a/v2/dependency-patches/tch-0.24.0/examples/yolo/yolo-v3.cfg b/v2/dependency-patches/tch-0.24.0/examples/yolo/yolo-v3.cfg new file mode 100644 index 0000000000..6772dddbf8 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/examples/yolo/yolo-v3.cfg @@ -0,0 +1,790 @@ +[net] +# Testing +batch=1 +subdivisions=1 +# Training +# batch=64 +# subdivisions=16 +width= 416 + +height = 416 +channels=3 +momentum=0.9 +decay=0.0005 +angle=0 +saturation = 1.5 +exposure = 1.5 +hue=.1 + +learning_rate=0.001 +burn_in=1000 +max_batches = 500200 +policy=steps +steps=400000,450000 +scales=.1,.1 + +[convolutional] +batch_normalize=1 +filters=32 +size=3 +stride=1 +pad=1 +activation=leaky + +# Downsample + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=2 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=32 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=64 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +# Downsample + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=2 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=64 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +# Downsample + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=2 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +# Downsample + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=2 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +# Downsample + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=2 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=1024 +size=3 +stride=1 +pad=1 +activation=leaky + +[shortcut] +from=-3 +activation=linear + +###################### + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=512 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=1024 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=linear + + +[yolo] +mask = 6,7,8 +anchors = 10,13, 16,30, 33,23, 30,61, 62,45, 59,119, 116,90, 156,198, 373,326 +classes=80 +num=9 +jitter=.3 +ignore_thresh = .5 +truth_thresh = 1 +random=1 + + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[upsample] +stride=2 + +[route] +layers = -1, 61 + + + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=256 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=512 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=linear + + +[yolo] +mask = 3,4,5 +anchors = 10,13, 16,30, 33,23, 30,61, 62,45, 59,119, 116,90, 156,198, 373,326 +classes=80 +num=9 +jitter=.3 +ignore_thresh = .5 +truth_thresh = 1 +random=1 + + + +[route] +layers = -4 + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[upsample] +stride=2 + +[route] +layers = -1, 36 + + + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +batch_normalize=1 +filters=128 +size=1 +stride=1 +pad=1 +activation=leaky + +[convolutional] +batch_normalize=1 +size=3 +stride=1 +pad=1 +filters=256 +activation=leaky + +[convolutional] +size=1 +stride=1 +pad=1 +filters=255 +activation=linear + + +[yolo] +mask = 0,1,2 +anchors = 10,13, 16,30, 33,23, 30,61, 62,45, 59,119, 116,90, 156,198, 373,326 +classes=80 +num=9 +jitter=.3 +ignore_thresh = .5 +truth_thresh = 1 +random=1 + diff --git a/v2/dependency-patches/tch-0.24.0/gen/.ocamlformat b/v2/dependency-patches/tch-0.24.0/gen/.ocamlformat new file mode 100644 index 0000000000..6d3685839c --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/gen/.ocamlformat @@ -0,0 +1,2 @@ +profile = janestreet +let-binding-spacing = compact diff --git a/v2/dependency-patches/tch-0.24.0/gen/dune b/v2/dependency-patches/tch-0.24.0/gen/dune new file mode 100644 index 0000000000..b5b2c527be --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/gen/dune @@ -0,0 +1,3 @@ +(executables + (names gen) + (libraries base stdio yaml)) diff --git a/v2/dependency-patches/tch-0.24.0/gen/gen.ml b/v2/dependency-patches/tch-0.24.0/gen/gen.ml new file mode 100644 index 0000000000..55ec8a93fc --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/gen/gen.ml @@ -0,0 +1,892 @@ +(* Automatically generate the C++ -> C -> rust bindings. + This takes as input the Descriptions.yaml file that gets generated when + building PyTorch from source. + + Run with: dune exec gen/gen.exe +*) +open Base +open Stdio + +let excluded_functions = + Set.of_list + (module String) + [ "multi_margin_loss" + ; "multi_margin_loss_out" + ; "log_softmax_backward_data" + ; "softmax_backward_data" + ; "clone" + ; "copy" + ; "copy_out" + ; "copy_" + ; "conv_transpose2d_backward_out" + ; "conv_transpose3d_backward_out" + ; "slow_conv_transpose2d_backward_out" + ; "slow_conv_transpose3d_backward_out" + ; "slow_conv3d_backward_out" + ; "normal" + ; "_cufft_set_plan_cache_max_size" + ; "_cufft_clear_plan_cache" + ; "backward" + ; "_amp_non_finite_check_and_unscale_" + ; "_cummin_helper" + ; "_cummax_helper" + ; "retain_grad" + ; "_validate_sparse_coo_tensor_args" + ; "_sparse_semi_structured_addmm" + ; "_backward" + ; "size" + ; "stride" + ; "_assert_async" + ; "gradient" + ; "linalg_vector_norm" + ; "linalg_vector_norm_out" + ; "linalg_matrix_norm" + ; "linalg_matrix_norm_out" + ; "linalg__powsum" + (* Deactivate normal_out, bernoulli_out as these result in some + ambiguous function calls. *) + ; "normal_out" + ; "bernoulli_out" + ; "nested_tensor" + ; "arange_out" + ] + +let no_tensor_options = + Set.of_list + (module String) + [ "zeros_like" + ; "empty_like" + ; "full_like" + ; "ones_like" + ; "rand_like" + ; "randint_like" + ; "randn_like" + ] + +let prefixed_functions = + Set.of_list + (module String) + [ "add" + ; "add_" + ; "div" + ; "div_" + ; "mul" + ; "mul_" + ; "sub" + ; "sub_" + ; "nll_loss" + ; "to_mkldnn" + ] + +(* By default, scalar argument that have a default value are not available on + the Rust side, this is to preserve the Rust api simplicity assuming that + these scalars arguments are not often overriden. + Adding function name [foo] in [with_optional_scalar_args] results in having + explicit scalar arguments even if a default is present. *) +let with_optional_scalar_args = Set.of_list (module String) [ "arange"; "baddbmm" ] + +let excluded_prefixes = + [ "_thnn_" + ; "_th_" + ; "thnn_" + ; "th_" + ; "_foreach" + ; "_amp_foreach" + ; "_nested_tensor" + ; "_fused_adam" + ; "_fused_adagrad" + ; "sym_" + ; "_fused_sgd" + ; "fbgemm_" + ] + +let excluded_suffixes = [ "_forward"; "_forward_out" ] +let yaml_error yaml ~msg = Printf.failwithf "%s, %s" msg (Yaml.to_string_exn yaml) () + +let extract_bool = function + | `Bool b -> b + | `String "true" -> true + | `String "false" -> false + | yaml -> yaml_error yaml ~msg:"expected bool" + +let extract_list = function + | `A l -> l + | yaml -> yaml_error yaml ~msg:"expected list" + +let extract_map = function + | `O map -> Map.of_alist_exn (module String) map + | yaml -> yaml_error yaml ~msg:"expected map" + +let extract_string = function + | `String s -> s + (* The yaml spec for torch uses n which is converted to a bool. *) + | `Bool b -> if b then "y" else "n" + | `Float f -> Float.to_string f + | yaml -> yaml_error yaml ~msg:"expected string" + +module Func = struct + type arg_type = + | Bool + | Int64 + | Int64Option + | Double + | DoubleOption + | Tensor + | TensorOption (* Tensor.t option *) + | IntList + | IntListOption + | DoubleList + | TensorOptList + | TensorList + | TensorOptions (* Tensor kind and device *) + | Scalar + | ScalarType + | ScalarTypeOption + | Device + | String + | Layout + | LayoutOption + + type arg = + { arg_name : string + ; arg_type : arg_type + ; default_value : string option + } + + type t = + { name : string + ; operator_name : string + ; overload_name : string + ; args : arg list + ; returns : [ `fixed of int | `dynamic | `bool | `int64_t | `double | `nothing ] + ; (* number of tensors that are returned *) + kind : [ `function_ | `method_ ] + } + + let arg_type_of_string str ~is_nullable = + match String.lowercase str with + | "bool" -> Some Bool + | "int64_t" -> Some (if is_nullable then Int64Option else Int64) + | "double" -> Some (if is_nullable then DoubleOption else Double) + | "at::tensor" -> Some (if is_nullable then TensorOption else Tensor) + | "at::tensoroptions" -> Some TensorOptions + | "at::intarrayref" -> Some (if is_nullable then IntListOption else IntList) + | "at::arrayref" -> Some DoubleList + | "const c10::list<::std::optional> &" + | "const c10::list> &" -> Some TensorOptList + | "const at::itensorlistref &" | "at::tensorlist" -> Some TensorList + | "at::device" -> Some Device + | "const at::scalar &" | "at::scalar" -> Some Scalar + | "at::scalartype" -> if is_nullable then Some ScalarTypeOption else Some ScalarType + | "c10::string_view" -> Some String + | "at::layout" -> Some (if is_nullable then LayoutOption else Layout) + | _ -> None + + let c_typed_args_list t = + List.map t.args ~f:(fun { arg_name; arg_type; _ } -> + match arg_type with + | IntList | IntListOption -> + Printf.sprintf "int64_t *%s_data, int %s_len" arg_name arg_name + | DoubleList -> Printf.sprintf "double *%s_data, int %s_len" arg_name arg_name + | TensorOptList | TensorList -> + Printf.sprintf "tensor *%s_data, int %s_len" arg_name arg_name + | TensorOptions -> Printf.sprintf "int %s_kind, int %s_device" arg_name arg_name + | String -> Printf.sprintf "char* %s_ptr, int %s_len" arg_name arg_name + | Int64Option -> Printf.sprintf "int64_t %s_v, uint8_t %s_null" arg_name arg_name + | DoubleOption -> Printf.sprintf "double %s_v, uint8_t %s_null" arg_name arg_name + | otherwise -> + let simple_type_cstring = + match otherwise with + | Bool -> "int" + | Int64 -> "int64_t" + | Double -> "double" + | Tensor -> "tensor" + | TensorOption -> "tensor" + | ScalarType -> "int" + | ScalarTypeOption -> "int" + | Device -> "int" + | Scalar -> "scalar" + | Layout | LayoutOption -> "int8_t" + | Int64Option + | DoubleOption + | String + | IntList + | IntListOption + | DoubleList + | TensorOptList + | TensorList + | TensorOptions -> assert false + in + Printf.sprintf "%s %s" simple_type_cstring arg_name) + |> String.concat ~sep:", " + + let c_args_list args = + List.map args ~f:(fun { arg_name; arg_type; _ } -> + match arg_type with + | Scalar | Tensor -> "*" ^ arg_name + | Layout -> Printf.sprintf "static_cast(%s)" arg_name + | LayoutOption -> + Printf.sprintf + "(%s == -1 ? c10::nullopt : \ + c10::optional(static_cast(%s)))" + arg_name + arg_name + | TensorOption -> Printf.sprintf "(%s ? ::std::optional(*%s) : ::std::nullopt)" arg_name arg_name + | Bool -> "(bool)" ^ arg_name + | IntList -> Printf.sprintf "torch::IntArrayRef(%s_data, %s_len)" arg_name arg_name + | IntListOption -> + Printf.sprintf + "%s_data == nullptr ? c10::nullopt : \ + c10::optional(torch::IntArrayRef(%s_data, %s_len))" + arg_name + arg_name + arg_name + | DoubleList -> + Printf.sprintf "at::ArrayRef(%s_data, %s_len)" arg_name arg_name + | String -> Printf.sprintf "std::string(%s_ptr, %s_len)" arg_name arg_name + | TensorOptList -> + Printf.sprintf "of_carray_tensor_opt(%s_data, %s_len)" arg_name arg_name + | TensorList -> Printf.sprintf "of_carray_tensor(%s_data, %s_len)" arg_name arg_name + | TensorOptions -> + Printf.sprintf + "at::device(device_of_int(%s_device)).dtype(at::ScalarType(%s_kind))" + arg_name + arg_name + | Int64Option -> + Printf.sprintf + "%s_null ? c10::nullopt : c10::optional(%s_v)" + arg_name + arg_name + | DoubleOption -> + Printf.sprintf + "%s_null ? c10::nullopt : c10::optional(%s_v)" + arg_name + arg_name + | ScalarType -> Printf.sprintf "at::ScalarType(%s)" arg_name + | ScalarTypeOption -> + Printf.sprintf + "%s < 0 ? c10::nullopt : c10::optional(at::ScalarType(%s))" + arg_name + arg_name + | Device -> Printf.sprintf "device_of_int(%s)" arg_name + | _ -> arg_name) + |> String.concat ~sep:", " + + let c_call t = + match t.kind with + | `function_ -> Printf.sprintf "torch::%s(%s)" t.name (c_args_list t.args) + | `method_ -> + (match t.args with + | head :: tail -> + Printf.sprintf "%s->%s(%s)" head.arg_name t.name (c_args_list tail) + | [] -> + Printf.failwithf "Method calls should have at least one argument %s" t.name ()) + + let replace_map = + Map.of_alist_exn + (module String) + [ "t", "tr" + ; "where", "where_" + ; "view", "view_" + ; "unsafe", "unsafe_" + ; "to_device", "to_device_" + ] + + let rust_name name = + let name = + Map.find replace_map name + |> Option.value ~default:name + |> String.lowercase + |> String.substr_replace_all ~pattern:"__" ~with_:"_" + in + if String.is_prefix name ~prefix:"_" then "internal" ^ name else name + + let operator_name t = + match String.lowercase t.operator_name with + | "scatter_reduce" -> + (* scatter_reduce is both an operator name and also obtained from the + scatter operator when using the reduce overload. *) + "_scatter_reduce" + | "scatter_reduce_" -> "_scatter_reduce_" + | other -> other + + let c_rust_args_list t = + List.map t.args ~f:(fun arg -> + let an = arg.arg_name in + let single_param = Printf.sprintf "%s_: %s" an in + match arg.arg_type with + | Bool -> single_param "c_int" + | Layout | LayoutOption -> single_param "i8" + | Int64 -> single_param "i64" + | Double -> single_param "f64" + | Tensor -> single_param "*mut C_tensor" + | TensorOption -> single_param "*mut C_tensor" + | Scalar -> single_param "*mut C_scalar" + | ScalarType | ScalarTypeOption -> single_param "c_int" + | Device -> single_param "c_int" + | String -> Printf.sprintf "%s_ptr: *const u8, %s_len: c_int" an an + | IntList | IntListOption -> + Printf.sprintf "%s_data: *const i64, %s_len: c_int" an an + | DoubleList -> Printf.sprintf "%s_data: *const f64, %s_len: c_int" an an + | TensorOptList -> + Printf.sprintf "%s_data: *const *mut C_tensor, %s_len: c_int" an an + | TensorList -> Printf.sprintf "%s_data: *const *mut C_tensor, %s_len: c_int" an an + | Int64Option -> Printf.sprintf "%s_v: i64, %s_null: i8" an an + | DoubleOption -> Printf.sprintf "%s_v: f64, %s_null: i8" an an + | TensorOptions -> Printf.sprintf "%s_kind: c_int, %s_device: c_int" an an) + |> String.concat ~sep:", " + + let self_name = "self" + let input_name = "input" + + let self_tensor arg = + match arg.arg_type with + | Tensor -> String.( = ) arg.arg_name self_name + | _ -> false + + let input_tensor arg = + match arg.arg_type with + | Tensor -> String.( = ) arg.arg_name input_name + | _ -> false + + let type_parameters t = + let needs_scalar_parameter = + List.exists t.args ~f:(fun arg -> + match arg.arg_type with + | Scalar -> true + | _ -> false) + in + let needs_type_parameter = + List.exists t.args ~f:(fun arg -> + match arg.arg_type with + | TensorOptList | TensorList | TensorOption -> true + | _ -> false) + in + let type_parameter = if needs_type_parameter then [ "T: Borrow" ] else [] in + let scalar_parameter = if needs_scalar_parameter then [ "S: Into" ] else [] in + let parameters = type_parameter @ scalar_parameter in + match parameters with + | [] -> "" + | p -> "<" ^ String.concat p ~sep:", " ^ ">" + + let rust_args_list t = + match List.partition_tf t.args ~f:self_tensor with + | [ self ], args_list -> Some self, args_list + | _, _ -> + (match List.partition_tf t.args ~f:input_tensor with + | [ self ], args_list -> Some self, args_list + | _, _ -> None, t.args) + + let rust_typed_args_list t = + let to_string args = + List.map args ~f:(fun arg -> + let rust_arg_type = + match arg.arg_type with + | Bool -> "bool" + | Layout -> "Layout" + | LayoutOption -> "Option" + | Int64 -> + if String.( = ) arg.arg_name "reduction" then "crate::Reduction" else "i64" + | Double -> "f64" + | Tensor -> "&Tensor" + | TensorOption -> "Option" + | IntList -> "impl IntList" + | IntListOption -> "impl IntListOption" + | DoubleList -> "impl DoubleList" + | TensorOptList -> "&[Option]" + | TensorList -> "&[T]" + | String -> "&str" + | TensorOptions -> "(Kind, Device)" + | Int64Option -> "impl Into>" + | DoubleOption -> "impl Into>" + | Scalar -> "S" + | ScalarType -> "Kind" + | ScalarTypeOption -> "impl Into>" + | Device -> "Device" + in + Printf.sprintf "%s: %s" (rust_name arg.arg_name) rust_arg_type) + |> String.concat ~sep:", " + in + let self_arg = + if String.is_suffix t.name ~suffix:"_" || String.( = ) t.name "set_data" + then "&mut self" + else "&self" + in + match List.partition_tf t.args ~f:self_tensor with + | [ self ], args_list -> + Some self.arg_name, Printf.sprintf "%s, %s" self_arg (to_string args_list) + | _, _ -> + (match List.partition_tf t.args ~f:input_tensor with + | [ self ], args_list -> + Some self.arg_name, Printf.sprintf "%s, %s" self_arg (to_string args_list) + | _, _ -> None, to_string t.args) + + let rust_return_type t ~fallible = + let returns = + match t.returns with + | `nothing -> None + | `fixed 1 -> Some "Tensor" + | `fixed v -> + List.init v ~f:(fun _ -> "Tensor") + |> String.concat ~sep:", " + |> Printf.sprintf "(%s)" + |> Option.some + | `dynamic -> Some "Vec" + | `bool -> Some "bool" + | `int64_t -> Some "i64" + | `double -> Some "f64" + in + match returns with + | None -> if fallible then Printf.sprintf " -> Result<(), TchError>" else "" + | Some returns -> + if fallible + then Printf.sprintf " -> Result<%s, TchError>" returns + else Printf.sprintf " -> %s" returns + + let rust_binding_args t ~self = + List.map t.args ~f:(fun arg -> + let name = + if Option.value_map self ~default:false ~f:(String.( = ) arg.arg_name) + then "self" + else rust_name arg.arg_name + in + match arg.arg_type with + | Tensor -> Printf.sprintf "%s.c_tensor" name + | Scalar -> Printf.sprintf "%s.into().c_scalar" name + | Bool -> Printf.sprintf "if %s { 1 } else { 0 }" name + | ScalarType -> Printf.sprintf "%s.c_int()" name + | ScalarTypeOption -> Printf.sprintf "%s.into().map_or(-1, |s| s.c_int())" name + | Device -> Printf.sprintf "%s.c_int()" name + | TensorOptions -> Printf.sprintf "%s.0.c_int(), %s.1.c_int()" name name + | Int64Option -> Printf.sprintf "%s.unwrap_or(0i64), %s.is_none() as i8" name name + | DoubleOption -> + Printf.sprintf "%s.unwrap_or(std::f64::NAN), %s.is_none() as i8" name name + | String -> Printf.sprintf "%s.as_ptr(), %s.len() as i32" name name + | IntList | IntListOption | DoubleList -> + Printf.sprintf "%s.as_ptr(), %s.len_i32()" name name + | TensorOptList -> + Printf.sprintf "ptr_list_opt(%s).as_ptr(), %s.len() as i32" name name + | TensorList -> Printf.sprintf "ptr_list(%s).as_ptr(), %s.len() as i32" name name + | TensorOption -> + Printf.sprintf + "%s.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor)" + name + | Int64 when String.( = ) name "reduction" -> "reduction.to_int()" + | Layout -> Printf.sprintf "%s.to_i8()" name + | LayoutOption -> Printf.sprintf "%s.map_or(-1, |s| s.to_i8())" name + | _ -> name) + |> String.concat ~sep:",\n " +end + +exception Not_a_simple_arg + +let read_yaml filename = + let funcs = + (* Split the file to avoid Yaml.of_string_exn segfaulting. *) + In_channel.with_file filename ~f:In_channel.input_lines + |> List.group ~break:(fun _ l -> String.length l > 0 && Char.( = ) l.[0] '-') + |> List.concat_map ~f:(fun lines -> + Yaml.of_string_exn (String.concat lines ~sep:"\n") |> extract_list) + in + printf "Read %s, got %d functions.\n%!" filename (List.length funcs); + List.filter_map funcs ~f:(fun yaml -> + let map = extract_map yaml in + let name = Map.find_exn map "name" |> extract_string in + let operator_name = Map.find_exn map "operator_name" |> extract_string in + let overload_name = Map.find_exn map "overload_name" |> extract_string in + let deprecated = Map.find_exn map "deprecated" |> extract_bool in + let method_of = + Map.find_exn map "method_of" |> extract_list |> List.map ~f:extract_string + in + let arguments = Map.find_exn map "arguments" |> extract_list in + let returns = + let is_tensor returns = + let returns = extract_map returns in + let return_type = Map.find_exn returns "dynamic_type" |> extract_string in + String.( = ) return_type "at::Tensor" + in + let returns = Map.find_exn map "returns" |> extract_list in + if List.is_empty returns + then Some `nothing + else if List.for_all returns ~f:is_tensor + then Some (`fixed (List.length returns)) + else ( + match returns with + | [ returns ] -> + let return_type = + Map.find_exn (extract_map returns) "dynamic_type" |> extract_string + in + (match return_type with + | "bool" -> Some `bool + | "int64_t" -> Some `int64_t + | "double" -> Some `double + | "at::TensorList" | "dynamic_type: const c10::List> &" + -> Some `dynamic + | _ -> None) + | [] | _ :: _ :: _ -> None) + in + let kind = + if List.exists method_of ~f:(String.( = ) "namespace") + then Some `function_ + else if List.exists method_of ~f:(String.( = ) "Tensor") + then Some `method_ + else None + in + if (not deprecated) + && (not + (List.exists excluded_prefixes ~f:(fun prefix -> + String.is_prefix name ~prefix))) + && (not + (List.exists excluded_suffixes ~f:(fun suffix -> + String.is_suffix name ~suffix))) + && not (Set.mem excluded_functions name) + then + Option.both returns kind + |> Option.bind ~f:(fun (returns, kind) -> + try + let args ~with_optional_scalar_args = + List.filter_map arguments ~f:(fun arg -> + let arg = extract_map arg in + let arg_name = Map.find_exn arg "name" |> extract_string in + let arg_type = Map.find_exn arg "dynamic_type" |> extract_string in + let is_nullable = + Map.find arg "is_nullable" + |> Option.value_map ~default:false ~f:extract_bool + in + let default_value = + Map.find arg "default" |> Option.map ~f:extract_string + in + match Func.arg_type_of_string arg_type ~is_nullable with + | Some Scalar when Option.is_some default_value && not is_nullable -> + if with_optional_scalar_args + then Some { Func.arg_name; arg_type = Scalar; default_value } + else None + | Some TensorOptions + when Option.is_some default_value && Set.mem no_tensor_options name -> + None + | Some arg_type -> + let arg_name = + match arg_name, arg_type with + | "self", Scalar -> "self_scalar" + | _, _ -> arg_name + in + Some { Func.arg_name; arg_type; default_value } + | None -> + if Option.is_some default_value then None else raise Not_a_simple_arg) + in + let args = + args ~with_optional_scalar_args:(Set.mem with_optional_scalar_args name) + in + Some [ { Func.name; operator_name; overload_name; args; returns; kind } ] + with + | Not_a_simple_arg -> None) + else None) + +let p out_channel s = + Printf.ksprintf + (fun line -> + Out_channel.output_string out_channel line; + Out_channel.output_char out_channel '\n') + s + +let write_cpp funcs filename = + Out_channel.with_file (filename ^ ".cpp") ~f:(fun out_cpp -> + Out_channel.with_file (filename ^ ".h") ~f:(fun out_h -> + let pc s = p out_cpp s in + let ph s = p out_h s in + pc "// THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT BY HAND!"; + pc "#include \"%s.h\"" (Stdlib.Filename.basename filename); + pc ""; + ph "// THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT BY HAND!"; + ph "#include \"torch_api.h\""; + ph ""; + ph "extern \"C\" {"; + Map.iteri funcs ~f:(fun ~key:exported_name ~data:func -> + let c_typed_args_list = Func.c_typed_args_list func in + match func.returns with + | `nothing -> + pc "void atg_%s(%s) {" exported_name c_typed_args_list; + pc " PROTECT("; + pc " %s;" (Func.c_call func); + pc " )"; + pc "}"; + pc ""; + ph "void atg_%s(%s);" exported_name c_typed_args_list + | `fixed ntensors -> + pc "void atg_%s(tensor *out__, %s) {" exported_name c_typed_args_list; + pc " PROTECT("; + pc " auto outputs__ = %s;" (Func.c_call func); + if ntensors = 1 + then pc " out__[0] = new torch::Tensor(outputs__);" + else + for i = 0 to ntensors - 1 do + pc " out__[%d] = new torch::Tensor(std::get<%d>(outputs__));" i i + done; + pc " )"; + pc "}"; + pc ""; + ph "void atg_%s(tensor *, %s);" exported_name c_typed_args_list + | `dynamic -> + pc "tensor *atg_%s(%s) {" exported_name c_typed_args_list; + pc " PROTECT("; + pc " auto outputs__ = %s;" (Func.c_call func); + (* the returned type is a C++ vector of tensors *) + pc " int sz = outputs__.size();"; + pc + " torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * \ + sizeof(torch::Tensor*));"; + pc " for (int i = 0; i < sz; ++i)"; + pc " out__[i] = new torch::Tensor(outputs__[i]);"; + pc " out__[sz] = nullptr;"; + pc " return out__;"; + pc " )"; + pc " return nullptr;"; + pc "}"; + pc ""; + ph "tensor *atg_%s(%s);" exported_name c_typed_args_list + | (`bool | `int64_t | `double) as returns -> + let c_type = + match returns with + | `bool -> "int" + | `int64_t -> "int64_t" + | `double -> "double" + in + pc "%s atg_%s(%s) {" c_type exported_name c_typed_args_list; + pc " PROTECT("; + pc " return %s;" (Func.c_call func); + pc " )"; + pc " return 0;"; + pc "}"; + pc ""; + ph "%s atg_%s(%s);" c_type exported_name c_typed_args_list); + ph "}")) + +let write_fallible_wrapper funcs filename = + Out_channel.with_file filename ~f:(fun out_ml -> + let pm s = p out_ml s in + pm "/* THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT BY HAND! */"; + pm "#![allow(clippy::all)]"; + pm "use torch_sys::*;"; + pm "use torch_sys::c_generated::*;"; + pm "use crate::{Device, Kind, Scalar, TchError, Tensor, Layout};"; + pm "use std::convert::Into;"; + pm "use std::borrow::Borrow;"; + pm ""; + pm "fn ptr_list_opt>(l: &[Option]) -> Vec<*mut C_tensor> {"; + pm + " l.iter().map(|x| x.as_ref().map_or(std::ptr::null_mut(), |x| \ + x.borrow().c_tensor)).collect()"; + pm "}"; + pm ""; + pm "fn ptr_list>(l: &[T]) -> Vec<*mut C_tensor> {"; + pm " l.iter().map(|x| x.borrow().c_tensor).collect()"; + pm "}"; + pm ""; + pm "impl Tensor {"; + Map.iteri funcs ~f:(fun ~key:exported_name ~data:(func : Func.t) -> + let rust_name = Func.rust_name exported_name in + let self, rust_args_list = Func.rust_typed_args_list func in + pm ""; + pm " pub fn f_%s%s(" rust_name (Func.type_parameters func); + pm " %s" rust_args_list; + pm " )%s {" (Func.rust_return_type func ~fallible:true); + List.iter func.args ~f:(fun arg -> + match arg.arg_type with + | DoubleOption | Int64Option -> + let arg_name = Func.rust_name arg.arg_name in + pm " let %s = %s.into();" arg_name arg_name + | _ -> ()); + match func.returns with + | `dynamic -> + pm " let c_tensors = unsafe_torch_err!("; + pm " atg_%s(" exported_name; + pm " %s));" (Func.rust_binding_args func ~self); + pm " let mut r__ = vec![];"; + pm " let mut i = 0;"; + pm " loop {"; + pm " let c__ = unsafe{*c_tensors.add(i)};"; + pm " if c__.is_null() { break }"; + pm " r__.push(Tensor {c_tensor: c__});"; + pm " i += 1;"; + pm " }"; + pm " unsafe{libc::free(c_tensors as *mut libc::c_void)}"; + pm " Ok(r__)"; + pm " }" + | `nothing -> + pm " unsafe_torch_err!("; + pm " atg_%s(" exported_name; + pm " %s" (Func.rust_binding_args func ~self); + pm " ));"; + pm " Ok(())"; + pm " }" + | `fixed ntensors -> + pm " let mut c_tensors = [std::ptr::null_mut(); %d];" ntensors; + pm " unsafe_torch_err!("; + pm " atg_%s(c_tensors.as_mut_ptr()," exported_name; + pm " %s" (Func.rust_binding_args func ~self); + pm " ));"; + let returns = + if ntensors = 1 + then "Tensor { c_tensor: c_tensors[0] }" + else + List.init ntensors ~f:(Printf.sprintf "Tensor { c_tensor: c_tensors[%d] }") + |> String.concat ~sep:", " + |> Printf.sprintf "(%s)" + in + pm " Ok(%s)" returns; + pm " }" + | (`bool | `int64_t | `double) as returns -> + let is_bool = + match returns with + | `bool -> true + | `int64_t | `double -> false + in + pm " let return_;"; + pm " unsafe_torch_err!("; + pm " return_ = atg_%s(" exported_name; + pm " %s" (Func.rust_binding_args func ~self); + pm " ));"; + let return_ = if is_bool then "return_ != 0" else "return_" in + pm " Ok(%s)" return_; + pm " }"); + pm "}") + +let write_wrapper funcs filename = + Out_channel.with_file filename ~f:(fun out_ml -> + let pm s = p out_ml s in + pm "/* THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT BY HAND! */"; + pm "#![allow(clippy::all)]"; + pm "use crate::{Device, Kind, Scalar, Tensor, Layout};"; + pm "use std::convert::Into;"; + pm "use std::borrow::Borrow;"; + pm "use torch_sys::*;"; + pm ""; + pm "impl Tensor {"; + Map.iteri funcs ~f:(fun ~key:exported_name ~data:(func : Func.t) -> + let rust_name = Func.rust_name exported_name in + let rust_name, fallible_rust_name = + if Set.mem prefixed_functions func.name + then "g_" ^ rust_name, "f_" ^ rust_name + else rust_name, "f_" ^ rust_name + in + pm ""; + pm " pub fn %s%s(" rust_name (Func.type_parameters func); + let _self, rust_args_list = Func.rust_typed_args_list func in + pm " %s" rust_args_list; + pm " )%s {" (Func.rust_return_type func ~fallible:false); + let self, rust_args_list = Func.rust_args_list func in + let self = if Option.is_some self then "self." else "Tensor::" in + let rust_args_list = + List.map rust_args_list ~f:(fun arg -> Func.rust_name arg.Func.arg_name) + |> String.concat ~sep:", " + in + pm " %s%s(%s).unwrap()" self fallible_rust_name rust_args_list; + pm " }"); + pm "}") + +let write_ffi funcs filename = + Out_channel.with_file filename ~f:(fun out_ml -> + let pm s = p out_ml s in + pm "/* THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT BY HAND! */"; + pm "#[allow(clippy::all)]"; + pm "use crate::{C_scalar, C_tensor};"; + pm "use libc::c_int;"; + pm ""; + pm "extern \"C\" {"; + Map.iteri funcs ~f:(fun ~key:exported_name ~data:func -> + match func.Func.returns with + | `nothing -> pm " pub fn atg_%s(%s);" exported_name (Func.c_rust_args_list func) + | `fixed _ -> + pm + " pub fn atg_%s(out__: *mut *mut C_tensor, %s);" + exported_name + (Func.c_rust_args_list func) + | `dynamic -> + pm + " pub fn atg_%s(%s) -> *mut *mut C_tensor;" + exported_name + (Func.c_rust_args_list func) + | (`bool | `int64_t | `double) as returns -> + let rust_type = + match returns with + | `bool -> "c_int" + | `int64_t -> "i64" + | `double -> "f64" + in + pm + " pub fn atg_%s(%s) -> %s;" + exported_name + (Func.c_rust_args_list func) + rust_type); + pm "}") + +let methods = + let c name args = + { Func.name + ; operator_name = name + ; overload_name = "" + ; args + ; returns = `fixed 1 + ; kind = `method_ + } + in + let ca arg_name arg_type = { Func.arg_name; arg_type; default_value = None } in + [ c "grad" [ ca "self" Tensor ] + ; c "set_requires_grad" [ ca "self" Tensor; ca "r" Bool ] + ; c "toType" [ ca "self" Tensor; ca "scalar_type" ScalarType ] + ; c "to" [ ca "self" Tensor; ca "device" Device ] + ] + +let run + ~yaml_filename + ~cpp_filename + ~ffi_filename + ~wrapper_filename + ~fallible_wrapper_filename + = + let funcs = read_yaml yaml_filename |> List.concat in + let funcs = methods @ funcs in + printf "Generating code for %d functions.\n%!" (List.length funcs); + (* Generate some unique names for overloaded functions. *) + let funcs = + List.map funcs ~f:(fun func -> Func.operator_name func, func) + |> Map.of_alist_multi (module String) + |> Map.to_alist + |> List.concat_map ~f:(fun (name, funcs) -> + match funcs with + | [] -> assert false + | [ func ] -> [ name, func ] + | funcs -> + let has_empty_overload = + List.exists funcs ~f:(fun (func : Func.t) -> + String.is_empty func.overload_name) + in + List.sort funcs ~compare:(fun (f1 : Func.t) (f2 : Func.t) -> + match Int.compare (String.length f1.name) (String.length f2.name) with + | 0 -> Int.compare (List.length f1.args) (List.length f2.args) + | cmp -> cmp) + |> List.mapi ~f:(fun index (func : Func.t) -> + let operator_name = Func.operator_name func in + let overload_name = String.lowercase func.overload_name in + let name = + if String.is_empty overload_name || (index = 0 && not has_empty_overload) + then operator_name + else if String.is_suffix operator_name ~suffix:"_" + then operator_name ^ overload_name ^ "_" + else operator_name ^ "_" ^ overload_name + in + name, func)) + |> Map.of_alist_exn (module String) + in + write_cpp funcs cpp_filename; + write_ffi funcs ffi_filename; + write_wrapper funcs wrapper_filename; + write_fallible_wrapper funcs fallible_wrapper_filename + +let () = + run + ~yaml_filename:"third_party/pytorch/Declarations-v2.11.0.yaml" + ~cpp_filename:"torch-sys/libtch/torch_api_generated" + ~ffi_filename:"torch-sys/src/c_generated.rs" + ~wrapper_filename:"src/wrappers/tensor_generated.rs" + ~fallible_wrapper_filename:"src/wrappers/tensor_fallible_generated.rs" diff --git a/v2/dependency-patches/tch-0.24.0/gen/gen.mli b/v2/dependency-patches/tch-0.24.0/gen/gen.mli new file mode 100644 index 0000000000..6d56210951 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/gen/gen.mli @@ -0,0 +1 @@ +(* Intentionally left blank. *) diff --git a/v2/dependency-patches/tch-0.24.0/main.py b/v2/dependency-patches/tch-0.24.0/main.py new file mode 100644 index 0000000000..8bbdbfe955 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/main.py @@ -0,0 +1,6 @@ +def main(): + print("Hello from tch-rs!") + + +if __name__ == "__main__": + main() diff --git a/v2/dependency-patches/tch-0.24.0/pyproject.toml b/v2/dependency-patches/tch-0.24.0/pyproject.toml new file mode 100644 index 0000000000..a8e15e75a2 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/pyproject.toml @@ -0,0 +1,9 @@ +[project] +name = "tch-rs" +version = "0.1.0" +description = "Rust bindings for PyTorch" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "torch==2.11.0", +] diff --git a/v2/dependency-patches/tch-0.24.0/rustfmt.toml b/v2/dependency-patches/tch-0.24.0/rustfmt.toml new file mode 100644 index 0000000000..8e8f481d13 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/rustfmt.toml @@ -0,0 +1,2 @@ +use_small_heuristics = "Max" +edition = "2021" diff --git a/v2/dependency-patches/tch-0.24.0/src/data.rs b/v2/dependency-patches/tch-0.24.0/src/data.rs new file mode 100644 index 0000000000..97256e8018 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/data.rs @@ -0,0 +1,203 @@ +//! Dataset iterators. +use crate::{kind, kind::Kind, Device, IndexOp, TchError, Tensor}; +use std::collections::HashMap; + +/// An iterator over a pair of tensors which have the same first dimension +/// size. +/// The typical use case is to iterate over batches. Each batch is a pair +/// containing a (potentially random) slice of each of the two input +/// tensors. +#[derive(Debug)] +pub struct Iter2 { + xs: Tensor, + ys: Tensor, + batch_index: i64, + batch_size: i64, + total_size: i64, + device: Device, + return_smaller_last_batch: bool, +} + +impl Iter2 { + /// Returns a new iterator. + /// + /// This takes as input two tensors which first dimension must match. The + /// returned iterator can be used to range over mini-batches of data of + /// specified size. + /// An error is returned if `xs` and `ys` have different first dimension + /// sizes. + /// + /// # Arguments + /// + /// * `xs` - the features to be used by the model. + /// * `ys` - the targets that the model attempts to predict. + /// * `batch_size` - the size of batches to be returned. + pub fn f_new(xs: &Tensor, ys: &Tensor, batch_size: i64) -> Result { + let total_size = xs.size()[0]; + if ys.size()[0] != total_size { + return Err(TchError::Shape(format!( + "different dimension for the two inputs {xs:?} {ys:?}" + ))); + } + Ok(Iter2 { + xs: xs.shallow_clone(), + ys: ys.shallow_clone(), + batch_index: 0, + batch_size, + total_size, + device: Device::Cpu, + return_smaller_last_batch: false, + }) + } + + /// Returns a new iterator. + /// + /// This takes as input two tensors which first dimension must match. The + /// returned iterator can be used to range over mini-batches of data of + /// specified size. + /// Panics if `xs` and `ys` have different first dimension sizes. + /// + /// # Arguments + /// + /// * `xs` - the features to be used by the model. + /// * `ys` - the targets that the model attempts to predict. + /// * `batch_size` - the size of batches to be returned. + pub fn new(xs: &Tensor, ys: &Tensor, batch_size: i64) -> Iter2 { + Iter2::f_new(xs, ys, batch_size).unwrap() + } + + /// Shuffles the dataset. + /// + /// The iterator would still run over the whole dataset but the order in + /// which elements are grouped in mini-batches is randomized. + pub fn shuffle(&mut self) -> &mut Iter2 { + let index = Tensor::randperm(self.total_size, (Kind::Int64, self.device)); + self.xs = self.xs.index_select(0, &index); + self.ys = self.ys.index_select(0, &index); + self + } + + /// Transfers the mini-batches to a specified device. + #[allow(clippy::wrong_self_convention)] + pub fn to_device(&mut self, device: Device) -> &mut Iter2 { + self.device = device; + self + } + + /// When set, returns the last batch even if smaller than the batch size. + pub fn return_smaller_last_batch(&mut self) -> &mut Iter2 { + self.return_smaller_last_batch = true; + self + } +} + +impl Iterator for Iter2 { + type Item = (Tensor, Tensor); + + fn next(&mut self) -> Option { + let start = self.batch_index * self.batch_size; + let size = std::cmp::min(self.batch_size, self.total_size - start); + if size <= 0 || (!self.return_smaller_last_batch && size < self.batch_size) { + None + } else { + self.batch_index += 1; + Some(( + self.xs.i(start..start + size).to_device(self.device), + self.ys.i(start..start + size).to_device(self.device), + )) + } + } +} + +/// Text data holder. +#[derive(Debug)] +pub struct TextData { + data: Tensor, + char_for_label: Vec, + label_for_char: HashMap, +} + +/// Text data iterator. +#[derive(Debug)] +pub struct TextDataIter { + data: Tensor, + seq_len: i64, + batch_index: i64, + batch_size: i64, + indexes: Tensor, + indexes_len: i64, +} + +impl TextData { + /// Creates a text dataset from a file. + pub fn new>(filename: P) -> Result { + let mut buffer = std::fs::read(&filename).map_err(|err| { + std::io::Error::new(err.kind(), format!("{:?} {err}", filename.as_ref())) + })?; + + let mut label_for_char = HashMap::::new(); + let mut char_for_label = Vec::::new(); + for c in buffer.iter_mut() { + *c = *label_for_char.entry(*c).or_insert_with(|| { + let label = char_for_label.len() as u8; + char_for_label.push(*c as char); + label + }) + } + + Ok(TextData { data: Tensor::from_slice(&buffer), char_for_label, label_for_char }) + } + + /// Returns the number of different characters/labels used by the dataset. + pub fn labels(&self) -> i64 { + self.char_for_label.len() as i64 + } + + /// Returns a shallow copy of the data. + pub fn data(&self) -> Tensor { + self.data.shallow_clone() + } + + pub fn label_to_char(&self, label: i64) -> char { + self.char_for_label[label as usize] + } + + pub fn char_to_label(&self, c: char) -> Result { + match self.label_for_char.get(&(c as u8)) { + None => Err(TchError::Convert(format!("cannot find char {c}"))), + Some(v) => Ok(*v), + } + } + + /// Returns a batch iterator over the dataset. + /// Each sample is made of seq_len characters. + pub fn iter_shuffle(&self, seq_len: i64, batch_size: i64) -> TextDataIter { + let indexes_len = self.data.size()[0] - seq_len + 1; + TextDataIter { + data: self.data.shallow_clone(), + seq_len, + batch_index: 0, + batch_size, + indexes: Tensor::randperm(indexes_len, kind::INT64_CPU), + indexes_len, + } + } +} + +impl Iterator for TextDataIter { + type Item = Tensor; + + fn next(&mut self) -> Option { + let start = self.batch_index * self.batch_size; + let size = std::cmp::min(self.batch_size, self.indexes_len - start); + if size < self.batch_size { + None + } else { + self.batch_index += 1; + let indexes = Vec::::try_from(&self.indexes.i(start..start + size)).unwrap(); + let batch: Vec<_> = indexes.iter().map(|&i| self.data.i(i..i + self.seq_len)).collect(); + let batch: Vec<_> = batch.iter().collect(); + Some(Tensor::stack(&batch, 0)) + } + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/error.rs b/v2/dependency-patches/tch-0.24.0/src/error.rs new file mode 100644 index 0000000000..1f7cc3e3d9 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/error.rs @@ -0,0 +1,74 @@ +use std::ffi::NulError; +use std::io; +use std::num::ParseIntError; + +use thiserror::Error; +use zip::result::ZipError; + +/// Main library error type. +#[derive(Error, Debug)] +pub enum TchError { + /// Conversion error. + #[error("conversion error: {0}")] + Convert(String), + + /// Invalid file format. + #[error("invalid file format: {0}")] + FileFormat(String), + + /// Missing tensor with name. + #[error("cannot find the tensor named {0} in {1}")] + TensorNameNotFound(String, String), + + /// I/O error. + #[error(transparent)] + Io(#[from] io::Error), + + /// Tensor kind error. + #[error("tensor kind error: {0}")] + Kind(String), + + /// Missing image. + #[error("no image found in {0}")] + MissingImage(String), + + /// Null pointer. + #[error(transparent)] + Nul(#[from] NulError), + + /// Integer parse error. + #[error(transparent)] + ParseInt(#[from] ParseIntError), + + /// Invalid shape. + #[error("invalid shape: {0}")] + Shape(String), + + /// Unknown kind + #[error("unknown kind: {0}")] + UnknownKind(libc::c_int), + + /// Errors returned by the Torch C++ API. + #[error("Internal torch error: {0}")] + Torch(String), + + /// Zip file format error. + #[error(transparent)] + Zip(#[from] ZipError), + + #[error(transparent)] + NdArray(#[from] ndarray::ShapeError), + + /// Errors returned by the safetensors library. + #[error("safetensors error {path}: {err}")] + SafeTensorError { path: String, err: safetensors::SafeTensorError }, +} + +impl TchError { + pub fn path_context(&self, path_name: &str) -> Self { + match self { + TchError::Torch(error) => TchError::Torch(format!("{path_name}: {error}")), + _ => unimplemented!(), + } + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/lib.rs b/v2/dependency-patches/tch-0.24.0/src/lib.rs new file mode 100644 index 0000000000..83caa31849 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/lib.rs @@ -0,0 +1,32 @@ +#[macro_use] +extern crate lazy_static; + +pub mod data; + +mod error; +pub use error::TchError; +pub type Result = std::result::Result; + +pub(crate) mod wrappers; +pub use wrappers::device::{Cuda, Device}; +pub use wrappers::jit::{self, CModule, IValue, TrainableCModule}; +pub use wrappers::kind::{self, Kind}; +pub use wrappers::layout::Layout; +pub use wrappers::optimizer::COptimizer; +#[cfg(feature = "python-extension")] +pub use wrappers::python; +pub use wrappers::scalar::Scalar; +pub use wrappers::utils; +pub use wrappers::{ + get_num_interop_threads, get_num_threads, manual_seed, set_num_interop_threads, + set_num_threads, QEngine, +}; + +mod tensor; +pub use tensor::{ + autocast, display, index, no_grad, no_grad_guard, with_grad, IndexOp, NewAxis, NoGradGuard, + Reduction, Shape, Tensor, TensorIndexer, +}; + +pub mod nn; +pub mod vision; diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/batch_norm.rs b/v2/dependency-patches/tch-0.24.0/src/nn/batch_norm.rs new file mode 100644 index 0000000000..8a8d2ceec6 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/nn/batch_norm.rs @@ -0,0 +1,132 @@ +//! A batch-normalization layer. +use crate::Tensor; +use std::borrow::Borrow; + +/// Batch-normalization config. +#[derive(Debug, Clone, Copy)] +pub struct BatchNormConfig { + pub cudnn_enabled: bool, + pub eps: f64, + pub momentum: f64, + pub affine: bool, + pub ws_init: super::Init, + pub bs_init: super::Init, +} + +impl Default for BatchNormConfig { + fn default() -> Self { + BatchNormConfig { + cudnn_enabled: true, + eps: 1e-5, + momentum: 0.1, + affine: true, + ws_init: super::Init::Uniform { lo: 0., up: 1. }, + bs_init: super::Init::Const(0.), + } + } +} + +/// A batch-normalization layer. +#[derive(Debug)] +pub struct BatchNorm { + config: BatchNormConfig, + pub running_mean: Tensor, + pub running_var: Tensor, + pub ws: Option, + pub bs: Option, + pub nd: usize, +} + +fn batch_norm<'a, T: Borrow>>( + vs: T, + nd: usize, + out_dim: i64, + config: BatchNormConfig, +) -> BatchNorm { + let vs = vs.borrow(); + let (ws, bs) = if config.affine { + let ws = vs.var("weight", &[out_dim], config.ws_init); + let bs = vs.var("bias", &[out_dim], config.bs_init); + (Some(ws), Some(bs)) + } else { + (None, None) + }; + BatchNorm { + config, + running_mean: vs.zeros_no_train("running_mean", &[out_dim]), + running_var: vs.ones_no_train("running_var", &[out_dim]), + ws, + bs, + nd, + } +} + +/// Applies Batch Normalization over a three dimension input. +/// +/// The input shape is assumed to be (N, C, L). Normalization +/// is performed over the first batch dimension N. +pub fn batch_norm1d<'a, T: Borrow>>( + vs: T, + out_dim: i64, + config: BatchNormConfig, +) -> BatchNorm { + batch_norm(vs, 1, out_dim, config) +} + +/// Applies Batch Normalization over a four dimension input. +/// +/// The input shape is assumed to be (N, C, H, W). Normalization +/// is performed over the first batch dimension N. +pub fn batch_norm2d<'a, T: Borrow>>( + vs: T, + out_dim: i64, + config: BatchNormConfig, +) -> BatchNorm { + batch_norm(vs, 2, out_dim, config) +} + +/// Applies Batch Normalization over a five dimension input. +/// +/// The input shape is assumed to be (N, C, D, H, W). Normalization +/// is performed over the first batch dimension N. +pub fn batch_norm3d<'a, T: Borrow>>( + vs: T, + out_dim: i64, + config: BatchNormConfig, +) -> BatchNorm { + batch_norm(vs, 3, out_dim, config) +} + +impl super::module::ModuleT for BatchNorm { + fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor { + let dim = xs.dim(); + if self.nd == 1 && dim != 2 && dim != 3 { + panic!( + "as nd={}, expected an input tensor with 2 or 3 dims, got {} ({:?})", + self.nd, + dim, + xs.size() + ) + } + if self.nd > 1 && xs.dim() != self.nd + 2 { + panic!( + "as nd={}, expected an input tensor with {} dims, got {} ({:?})", + self.nd, + self.nd + 2, + dim, + xs.size() + ) + }; + Tensor::batch_norm( + xs, + self.ws.as_ref(), + self.bs.as_ref(), + Some(&self.running_mean), + Some(&self.running_var), + train, + self.config.momentum, + self.config.eps, + self.config.cudnn_enabled, + ) + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/conv.rs b/v2/dependency-patches/tch-0.24.0/src/nn/conv.rs new file mode 100644 index 0000000000..6c99248fd6 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/nn/conv.rs @@ -0,0 +1,242 @@ +//! N-dimensional convolution layers. +use super::Path; +use crate::{TchError, Tensor}; +use std::borrow::Borrow; + +/// How padding is performed by convolution operations +/// on the edge of the input tensor. +#[derive(Debug, Clone, Copy)] +pub enum PaddingMode { + Zeros, + Reflect, + Replicate, + Circular, +} + +impl PaddingMode { + fn to_string(self) -> &'static str { + // This has to match the internal representation used on the C++ + // side. + match self { + // The default value when using constant is zero. + PaddingMode::Zeros => "constant", + PaddingMode::Reflect => "reflect", + PaddingMode::Replicate => "replicate", + PaddingMode::Circular => "circular", + } + } + + pub fn f_pad( + self, + xs: &Tensor, + reversed_padding_repeated_twice: &[i64], + ) -> Result { + xs.f_pad(reversed_padding_repeated_twice, self.to_string(), None) + } + + pub fn pad(self, xs: &Tensor, reversed_padding_repeated_twice: &[i64]) -> Tensor { + xs.pad(reversed_padding_repeated_twice, self.to_string(), None) + } +} + +/// Generic convolution config. +#[allow(clippy::upper_case_acronyms)] +#[derive(Debug, Clone, Copy)] +pub struct ConvConfigND { + pub stride: ND, + pub padding: ND, + pub dilation: ND, + pub groups: i64, + pub bias: bool, + pub ws_init: super::Init, + pub bs_init: super::Init, + pub padding_mode: PaddingMode, +} + +/// Convolution config using the same parameters on all dimensions. +pub type ConvConfig = ConvConfigND; + +impl Default for ConvConfig { + fn default() -> Self { + ConvConfig { + stride: 1, + padding: 0, + dilation: 1, + groups: 1, + bias: true, + ws_init: super::init::DEFAULT_KAIMING_UNIFORM, + bs_init: super::Init::Const(0.), + padding_mode: PaddingMode::Zeros, + } + } +} + +impl Default for ConvConfigND<[i64; 2]> { + fn default() -> Self { + ConvConfigND::<[i64; 2]> { + stride: [1, 1], + padding: [0, 0], + dilation: [1, 1], + groups: 1, + bias: true, + ws_init: super::init::DEFAULT_KAIMING_UNIFORM, + bs_init: super::Init::Const(0.), + padding_mode: PaddingMode::Zeros, + } + } +} + +/// The default convolution config without bias. +pub fn no_bias() -> ConvConfig { + ConvConfig { bias: false, ..Default::default() } +} + +// Use const generics when they have landed in stable rust. +/// A N-dimensional convolution layer. +#[derive(Debug)] +pub struct Conv { + pub ws: Tensor, + pub bs: Option, + reversed_padding_repeated_twice: Vec, + config: ConvConfigND, +} + +/// One dimension convolution layer. +pub type Conv1D = Conv<[i64; 1]>; + +/// Two dimensions convolution layer. +pub type Conv2D = Conv<[i64; 2]>; + +/// Three dimensions convolution layer. +pub type Conv3D = Conv<[i64; 3]>; + +/// Creates a new convolution layer for any number of dimensions. +pub fn conv<'a, ND: std::convert::AsRef<[i64]>, T: Borrow>>( + vs: T, + in_dim: i64, + out_dim: i64, + ksizes: ND, + config: ConvConfigND, +) -> Conv { + let vs = vs.borrow(); + let bs = if config.bias { Some(vs.var("bias", &[out_dim], config.bs_init)) } else { None }; + let mut weight_size = vec![out_dim, in_dim / config.groups]; + weight_size.extend(ksizes.as_ref().iter()); + let ws = vs.var("weight", weight_size.as_slice(), config.ws_init); + let mut reversed_padding_repeated_twice = vec![]; + for &v in config.padding.as_ref().iter().rev() { + reversed_padding_repeated_twice.push(v) + } + for &v in config.padding.as_ref().iter().rev() { + reversed_padding_repeated_twice.push(v) + } + Conv { ws, bs, config, reversed_padding_repeated_twice } +} + +trait Create: std::convert::AsRef<[i64]> + std::marker::Sized { + fn make_array(i: i64) -> Self; + + fn conv<'a, T: Borrow>>( + vs: T, + in_dim: i64, + out_dim: i64, + ksize: i64, + config: ConvConfig, + ) -> Conv { + let config = ConvConfigND:: { + stride: Self::make_array(config.stride), + padding: Self::make_array(config.padding), + dilation: Self::make_array(config.dilation), + groups: config.groups, + bias: config.bias, + ws_init: config.ws_init, + bs_init: config.bs_init, + padding_mode: config.padding_mode, + }; + conv(vs, in_dim, out_dim, Self::make_array(ksize), config) + } +} + +impl Create for [i64; 1] { + fn make_array(i: i64) -> Self { + [i] + } +} + +impl Create for [i64; 2] { + fn make_array(i: i64) -> Self { + [i, i] + } +} + +impl Create for [i64; 3] { + fn make_array(i: i64) -> Self { + [i, i, i] + } +} + +/// Creates a new one dimension convolution layer. +pub fn conv1d<'a, T: Borrow>>(vs: T, i: i64, o: i64, k: i64, c: ConvConfig) -> Conv1D { + <[i64; 1]>::conv(vs, i, o, k, c) +} + +/// Creates a new two dimension convolution layer. +pub fn conv2d<'a, T: Borrow>>(vs: T, i: i64, o: i64, k: i64, c: ConvConfig) -> Conv2D { + <[i64; 2]>::conv(vs, i, o, k, c) +} + +/// Creates a new three dimension convolution layer. +pub fn conv3d<'a, T: Borrow>>(vs: T, i: i64, o: i64, k: i64, c: ConvConfig) -> Conv3D { + <[i64; 3]>::conv(vs, i, o, k, c) +} + +impl super::module::Module for Conv1D { + fn forward(&self, xs: &Tensor) -> Tensor { + let (xs, padding) = match self.config.padding_mode { + PaddingMode::Zeros => (xs.shallow_clone(), self.config.padding), + p => (p.pad(xs, &self.reversed_padding_repeated_twice), [0]), + }; + xs.conv1d( + &self.ws, + self.bs.as_ref(), + self.config.stride, + padding, + self.config.dilation, + self.config.groups, + ) + } +} + +impl super::module::Module for Conv2D { + fn forward(&self, xs: &Tensor) -> Tensor { + let (xs, padding) = match self.config.padding_mode { + PaddingMode::Zeros => (xs.shallow_clone(), self.config.padding), + p => (p.pad(xs, &self.reversed_padding_repeated_twice), [0, 0]), + }; + xs.conv2d( + &self.ws, + self.bs.as_ref(), + self.config.stride, + padding, + self.config.dilation, + self.config.groups, + ) + } +} + +impl super::module::Module for Conv3D { + fn forward(&self, xs: &Tensor) -> Tensor { + let (xs, padding) = match self.config.padding_mode { + PaddingMode::Zeros => (xs.shallow_clone(), self.config.padding), + p => (p.pad(xs, &self.reversed_padding_repeated_twice), [0, 0, 0]), + }; + xs.conv3d( + &self.ws, + self.bs.as_ref(), + self.config.stride, + padding, + self.config.dilation, + self.config.groups, + ) + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/conv_transpose.rs b/v2/dependency-patches/tch-0.24.0/src/nn/conv_transpose.rs new file mode 100644 index 0000000000..f183eaecfa --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/nn/conv_transpose.rs @@ -0,0 +1,189 @@ +//! A two dimension transposed convolution layer. +use super::Path; +use crate::Tensor; +use std::borrow::Borrow; + +/// A generic transposed convolution configuration. +#[allow(clippy::upper_case_acronyms)] +#[derive(Debug, Clone, Copy)] +pub struct ConvTransposeConfigND { + pub stride: ND, + pub padding: ND, + pub output_padding: ND, + pub groups: i64, + pub bias: bool, + pub dilation: ND, + pub ws_init: super::Init, + pub bs_init: super::Init, +} + +/// A transposed convolution configuration using the same values on each dimension. +pub type ConvTransposeConfig = ConvTransposeConfigND; + +impl Default for ConvTransposeConfig { + fn default() -> Self { + ConvTransposeConfigND { + stride: 1, + padding: 0, + output_padding: 0, + dilation: 1, + groups: 1, + bias: true, + ws_init: super::init::DEFAULT_KAIMING_UNIFORM, + bs_init: super::Init::Const(0.), + } + } +} + +/// A generic transposed convolution layer. +#[allow(clippy::upper_case_acronyms)] +#[derive(Debug)] +pub struct ConvTransposeND { + pub ws: Tensor, + pub bs: Option, + config: ConvTransposeConfigND, +} + +/// A one dimension transposed convolution layer. +pub type ConvTranspose1D = ConvTransposeND<[i64; 1]>; + +/// A two dimension transposed convolution layer. +pub type ConvTranspose2D = ConvTransposeND<[i64; 2]>; + +/// A three dimension transposed convolution layer. +pub type ConvTranspose3D = ConvTransposeND<[i64; 3]>; + +fn conv_transpose<'a, ND: std::convert::AsRef<[i64]>, T: Borrow>>( + vs: T, + in_dim: i64, + out_dim: i64, + ksizes: ND, + config: ConvTransposeConfigND, +) -> ConvTransposeND { + let vs = vs.borrow(); + let bs = if config.bias { Some(vs.var("bias", &[out_dim], config.bs_init)) } else { None }; + let mut weight_size = vec![in_dim, out_dim / config.groups]; + weight_size.extend(ksizes.as_ref().iter()); + let ws = vs.var("weight", weight_size.as_slice(), config.ws_init); + ConvTransposeND { ws, bs, config } +} + +trait Create: std::convert::AsRef<[i64]> + std::marker::Sized { + fn make_array(i: i64) -> Self; + + fn conv_transpose<'a, T: Borrow>>( + vs: T, + in_dim: i64, + out_dim: i64, + ksize: i64, + config: ConvTransposeConfig, + ) -> ConvTransposeND { + let config = ConvTransposeConfigND:: { + stride: Self::make_array(config.stride), + padding: Self::make_array(config.padding), + output_padding: Self::make_array(config.output_padding), + dilation: Self::make_array(config.dilation), + groups: config.groups, + bias: config.bias, + ws_init: config.ws_init, + bs_init: config.bs_init, + }; + conv_transpose(vs, in_dim, out_dim, Self::make_array(ksize), config) + } +} + +impl Create for [i64; 1] { + fn make_array(i: i64) -> Self { + [i] + } +} + +impl Create for [i64; 2] { + fn make_array(i: i64) -> Self { + [i, i] + } +} + +impl Create for [i64; 3] { + fn make_array(i: i64) -> Self { + [i, i, i] + } +} + +/// Creates a one dimension transposed convolution layer. +pub fn conv_transpose1d<'a, T: Borrow>>( + vs: T, + i: i64, + o: i64, + k: i64, + c: ConvTransposeConfig, +) -> ConvTranspose1D { + <[i64; 1]>::conv_transpose(vs, i, o, k, c) +} + +/// Creates a two dimension transposed convolution layer. +pub fn conv_transpose2d<'a, T: Borrow>>( + vs: T, + i: i64, + o: i64, + k: i64, + c: ConvTransposeConfig, +) -> ConvTranspose2D { + <[i64; 2]>::conv_transpose(vs, i, o, k, c) +} + +/// Creates a three dimension transposed convolution layer. +pub fn conv_transpose3d<'a, T: Borrow>>( + vs: T, + i: i64, + o: i64, + k: i64, + c: ConvTransposeConfig, +) -> ConvTranspose3D { + <[i64; 3]>::conv_transpose(vs, i, o, k, c) +} + +impl super::module::Module for ConvTranspose1D { + fn forward(&self, xs: &Tensor) -> Tensor { + Tensor::conv_transpose1d( + xs, + &self.ws, + self.bs.as_ref(), + self.config.stride, + self.config.padding, + self.config.output_padding, + self.config.groups, + self.config.dilation, + ) + } +} + +impl super::module::Module for ConvTranspose2D { + fn forward(&self, xs: &Tensor) -> Tensor { + Tensor::conv_transpose2d( + xs, + &self.ws, + self.bs.as_ref(), + self.config.stride, + self.config.padding, + self.config.output_padding, + self.config.groups, + self.config.dilation, + ) + } +} + +impl super::module::Module for ConvTranspose3D { + fn forward(&self, xs: &Tensor) -> Tensor { + Tensor::conv_transpose3d( + xs, + &self.ws, + self.bs.as_ref(), + self.config.stride, + self.config.padding, + self.config.output_padding, + self.config.groups, + self.config.dilation, + ) + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/func.rs b/v2/dependency-patches/tch-0.24.0/src/nn/func.rs new file mode 100644 index 0000000000..123be59c0c --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/nn/func.rs @@ -0,0 +1,51 @@ +//! Layers defined by closures. +use crate::Tensor; + +/// A layer defined by a simple closure. +pub struct Func<'a> { + f: Box Tensor + Send>, +} + +impl std::fmt::Debug for Func<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "func") + } +} + +pub fn func<'a, F>(f: F) -> Func<'a> +where + F: 'a + Fn(&Tensor) -> Tensor + Send, +{ + Func { f: Box::new(f) } +} + +impl super::module::Module for Func<'_> { + fn forward(&self, xs: &Tensor) -> Tensor { + (*self.f)(xs) + } +} + +/// A layer defined by a closure with an additional training parameter. +#[allow(clippy::type_complexity)] +pub struct FuncT<'a> { + f: Box Tensor + Send>, +} + +impl std::fmt::Debug for FuncT<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "funcT") + } +} + +pub fn func_t<'a, F>(f: F) -> FuncT<'a> +where + F: 'a + Fn(&Tensor, bool) -> Tensor + Send, +{ + FuncT { f: Box::new(f) } +} + +impl super::module::ModuleT for FuncT<'_> { + fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor { + (*self.f)(xs, train) + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/group_norm.rs b/v2/dependency-patches/tch-0.24.0/src/nn/group_norm.rs new file mode 100644 index 0000000000..f02af14ee9 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/nn/group_norm.rs @@ -0,0 +1,66 @@ +//! A group-normalization layer. +//! Group Normalization +use crate::Tensor; +use std::borrow::Borrow; + +/// Group-normalization config. +#[derive(Debug, Clone, Copy)] +pub struct GroupNormConfig { + pub cudnn_enabled: bool, + pub eps: f64, + pub affine: bool, + pub ws_init: super::Init, + pub bs_init: super::Init, +} + +impl Default for GroupNormConfig { + fn default() -> Self { + GroupNormConfig { + cudnn_enabled: true, + eps: 1e-5, + affine: true, + ws_init: super::Init::Const(1.), + bs_init: super::Init::Const(0.), + } + } +} + +/// A group-normalization layer. +#[derive(Debug)] +pub struct GroupNorm { + config: GroupNormConfig, + pub ws: Option, + pub bs: Option, + pub num_groups: i64, + pub num_channels: i64, +} + +pub fn group_norm<'a, T: Borrow>>( + vs: T, + num_groups: i64, + num_channels: i64, + config: GroupNormConfig, +) -> GroupNorm { + let vs = vs.borrow(); + let (ws, bs) = if config.affine { + let ws = vs.var("weight", &[num_channels], config.ws_init); + let bs = vs.var("bias", &[num_channels], config.bs_init); + (Some(ws), Some(bs)) + } else { + (None, None) + }; + GroupNorm { config, ws, bs, num_groups, num_channels } +} + +impl super::module::Module for GroupNorm { + fn forward(&self, xs: &Tensor) -> Tensor { + Tensor::group_norm( + xs, + self.num_groups, + self.ws.as_ref(), + self.bs.as_ref(), + self.config.eps, + self.config.cudnn_enabled, + ) + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/init.rs b/v2/dependency-patches/tch-0.24.0/src/nn/init.rs new file mode 100644 index 0000000000..89c43f9da9 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/nn/init.rs @@ -0,0 +1,212 @@ +//! Variable initialization. +use crate::{Device, Kind, TchError, Tensor}; + +/// Number of features as input or output of a layer. +/// In Kaiming initialization, choosing `FanIn` preserves +/// the magnitude of the variance of the weights in the +/// forward pass, choosing `FanOut` preserves this +/// magnitude in the backward pass. +#[derive(Debug, Copy, Clone)] +pub enum FanInOut { + FanIn, + FanOut, +} + +impl FanInOut { + /// Compute the fan-in or fan-out value for a weight tensor of + /// the specified dimensions. + /// + pub fn for_weight_dims(&self, dims: &[i64]) -> i64 { + let receptive_field_size: i64 = dims.iter().skip(2).product(); + match &self { + FanInOut::FanIn => { + if dims.len() < 2 { + 1 + } else { + dims[1] * receptive_field_size + } + } + FanInOut::FanOut => { + if dims.is_empty() { + 1 + } else { + dims[0] * receptive_field_size + } + } + } + } +} + +#[derive(Debug, Copy, Clone)] +pub enum NormalOrUniform { + Normal, + Uniform, +} + +/// The non-linear function that follows this layer. ReLU is the +/// recommended value. +#[derive(Debug, Copy, Clone)] +pub enum NonLinearity { + ReLU, + Linear, + Sigmoid, + Tanh, + SELU, + ExplicitGain(f64), +} + +impl NonLinearity { + pub fn gain(&self) -> f64 { + match *self { + NonLinearity::ReLU => 2f64.sqrt(), + NonLinearity::Tanh => 5. / 3., + NonLinearity::Linear | NonLinearity::Sigmoid => 1., + NonLinearity::SELU => 0.75, + NonLinearity::ExplicitGain(g) => g, + } + } +} + +/// Variable initializations. +#[derive(Debug, Copy, Clone)] +pub enum Init { + /// Constant value. + Const(f64), + + /// Random normal with some mean and standard deviation. + Randn { mean: f64, stdev: f64 }, + + /// Uniform initialization between some lower and upper bounds. + Uniform { lo: f64, up: f64 }, + + /// Kaiming uniform initialization. + /// See "Delving deep into rectifiers: Surpassing human-level performance on ImageNet classification" + /// He, K. et al. (2015). This uses a uniform distribution. + Kaiming { dist: NormalOrUniform, fan: FanInOut, non_linearity: NonLinearity }, + + /// Orthogonal initialization + Orthogonal { gain: f64 }, +} + +pub const DEFAULT_KAIMING_UNIFORM: Init = Init::Kaiming { + dist: NormalOrUniform::Uniform, + fan: FanInOut::FanIn, + non_linearity: NonLinearity::ReLU, +}; + +pub const DEFAULT_KAIMING_NORMAL: Init = Init::Kaiming { + dist: NormalOrUniform::Normal, + fan: FanInOut::FanIn, + non_linearity: NonLinearity::ReLU, +}; + +/// Creates a new float tensor with the specified shape, device, and initialization. +pub fn f_init(i: Init, dims: &[i64], device: Device, kind: Kind) -> Result { + match i { + Init::Const(cst) => { + // Optimize the case for which a single C++ code can be done. + if cst == 0. { + Tensor::f_zeros(dims, (kind, device)) + } else if (cst - 1.).abs() <= f64::EPSILON { + Tensor::f_ones(dims, (kind, device)) + } else { + Tensor::f_ones(dims, (kind, device)).map(|t| t * cst) + } + } + Init::Uniform { lo, up } => Tensor::f_zeros(dims, (kind, device))?.f_uniform_(lo, up), + Init::Randn { mean, stdev } => { + if mean == 0. && (stdev - 1.).abs() <= f64::EPSILON { + Tensor::f_randn(dims, (kind, device)) + } else { + Tensor::f_randn(dims, (kind, device)).map(|t| t * stdev + mean) + } + } + Init::Kaiming { dist, fan, non_linearity } => { + let fan = fan.for_weight_dims(dims); + let gain = non_linearity.gain(); + let std = gain / (fan as f64).sqrt(); + match dist { + NormalOrUniform::Uniform => { + let bound = 3f64.sqrt() * std; + Tensor::f_zeros(dims, (kind, device))?.f_uniform_(-bound, bound) + } + NormalOrUniform::Normal => { + let randn = Tensor::f_randn(dims, (kind, device))?; + Ok(randn * std) + } + } + } + Init::Orthogonal { gain } => { + if dims.len() < 2 { + return Err(TchError::Shape( + "Only tensors with 2 or more dimensions are supported".to_string(), + )); + } + let rows = dims[0]; + let cols: i64 = dims.iter().skip(1).product(); + + let mut flattened = + Tensor::f_empty([rows, cols], (kind, device))?.f_normal_(0.0, 1.0)?; + let flattened = if rows < cols { flattened.f_t_()? } else { flattened }; + + let (mut q, r) = Tensor::f_linalg_qr(&flattened, "reduced")?; + let d = r.f_diag(0)?; + let ph = d.f_sign()?; + q *= ph; + + let mut q = if rows < cols { q.f_t_()? } else { q }; + crate::no_grad(|| q *= gain); + + q.f_contiguous() + } + } +} + +/// Creates a new float tensor with the specified shape, device, and initialization. +pub fn init(i: Init, dims: &[i64], device: Device) -> Tensor { + f_init(i, dims, device, Kind::Float).unwrap() +} + +impl Init { + /// Re-initializes an existing tensor with the specified initialization + pub fn set(self, tensor: &mut Tensor) { + match self { + Init::Const(cst) => { + let _ = tensor.fill_(cst); + } + Init::Uniform { lo, up } => { + let _ = tensor.uniform_(lo, up); + } + Init::Kaiming { dist, fan, non_linearity } => { + let fan = fan.for_weight_dims(&tensor.size()); + let gain = non_linearity.gain(); + let std = gain / (fan as f64).sqrt(); + match dist { + NormalOrUniform::Uniform => { + let bound = 3f64.sqrt() * std; + let _ = tensor.uniform_(-bound, bound); + } + NormalOrUniform::Normal => { + tensor.copy_(&(tensor.randn_like() * std)); + } + } + } + Init::Randn { mean, stdev } => { + tensor.copy_(&(tensor.randn_like() * stdev + mean)); + } + Init::Orthogonal { gain } => { + let q = + f_init(Init::Orthogonal { gain }, &tensor.size(), tensor.device(), Kind::Float) + .unwrap(); + crate::no_grad(|| tensor.view_as(&q).copy_(&q)); + } + } + } +} + +impl Tensor { + /// Re-initializes the tensor using the specified initialization. + pub fn init(&mut self, i: Init) { + i.set(self) + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/layer_norm.rs b/v2/dependency-patches/tch-0.24.0/src/nn/layer_norm.rs new file mode 100644 index 0000000000..b522f872a6 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/nn/layer_norm.rs @@ -0,0 +1,65 @@ +//! A layer-normalization layer. +use crate::Tensor; +use std::borrow::Borrow; + +/// Layer-normalization config. +#[derive(Debug, Clone, Copy)] +pub struct LayerNormConfig { + pub cudnn_enabled: bool, + pub eps: f64, + pub elementwise_affine: bool, + pub ws_init: super::Init, + pub bs_init: super::Init, +} + +impl Default for LayerNormConfig { + fn default() -> Self { + LayerNormConfig { + cudnn_enabled: true, + eps: 1e-5, + elementwise_affine: true, + ws_init: super::Init::Const(1.), + bs_init: super::Init::Const(0.), + } + } +} + +/// A layer-normalization layer. +#[derive(Debug)] +pub struct LayerNorm { + config: LayerNormConfig, + pub ws: Option, + pub bs: Option, + pub normalized_shape: Vec, +} + +pub fn layer_norm<'a, T: Borrow>>( + vs: T, + normalized_shape: Vec, + config: LayerNormConfig, +) -> LayerNorm { + let vs = vs.borrow(); + + let (ws, bs) = if config.elementwise_affine { + let ws = vs.var("weight", normalized_shape.as_slice(), config.ws_init); + let bs = vs.var("bias", normalized_shape.as_slice(), config.bs_init); + (Some(ws), Some(bs)) + } else { + (None, None) + }; + + LayerNorm { config, ws, bs, normalized_shape } +} + +impl super::module::Module for LayerNorm { + fn forward(&self, xs: &Tensor) -> Tensor { + Tensor::layer_norm( + xs, + self.normalized_shape.as_slice(), + self.ws.as_ref(), + self.bs.as_ref(), + self.config.eps, + self.config.cudnn_enabled, + ) + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/linear.rs b/v2/dependency-patches/tch-0.24.0/src/nn/linear.rs new file mode 100644 index 0000000000..e5d385ba51 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/nn/linear.rs @@ -0,0 +1,74 @@ +//! A linear fully-connected layer. +use crate::Tensor; +use std::borrow::Borrow; + +/// Configuration for a linear layer. +#[derive(Debug, Clone, Copy)] +pub struct LinearConfig { + pub ws_init: super::Init, + pub bs_init: Option, + pub bias: bool, +} + +impl Default for LinearConfig { + fn default() -> Self { + LinearConfig { ws_init: super::init::DEFAULT_KAIMING_UNIFORM, bs_init: None, bias: true } + } +} + +/// A linear fully-connected layer. +#[derive(Debug)] +pub struct Linear { + pub ws: Tensor, + pub bs: Option, +} + +/// Creates a new linear layer. +pub fn linear<'a, T: Borrow>>( + vs: T, + in_dim: i64, + out_dim: i64, + c: LinearConfig, +) -> Linear { + let vs = vs.borrow(); + let bs = if c.bias { + let bs_init = c.bs_init.unwrap_or_else(|| { + let bound = 1.0 / (in_dim as f64).sqrt(); + super::Init::Uniform { lo: -bound, up: bound } + }); + Some(vs.var("bias", &[out_dim], bs_init)) + } else { + None + }; + + Linear { ws: vs.var("weight", &[out_dim, in_dim], c.ws_init), bs } +} + +impl super::module::Module for Linear { + fn forward(&self, xs: &Tensor) -> Tensor { + xs.linear(&self.ws, self.bs.as_ref()) + } +} + +#[test] +fn matches_pytorch() { + use crate::nn::Module; + + let input = Tensor::read_npy("tests/linear/in.npy").unwrap(); + let expected_output = Tensor::read_npy("tests/linear/out.npy").unwrap(); + let ws = Tensor::read_npy("tests/linear/ws.npy").unwrap(); + let bs = Some(Tensor::read_npy("tests/linear/bs.npy").unwrap()); + + let original_output = + if let Some(bias) = &bs { input.matmul(&ws.tr()) + bias } else { input.matmul(&ws.tr()) }; + + let linear = Linear { ws, bs }; + let output = linear.forward(&input); + + let delta_output: f32 = (&output - &expected_output).norm().try_into().unwrap(); + let delta_original: f32 = (&original_output - &expected_output).norm().try_into().unwrap(); + + // The `matmul()` implementation is close, but `linear()` is at least as close or closer. + assert!(output.allclose(&expected_output, 1e-5, 1e-8, false)); + assert!(delta_output <= delta_original); +} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/mod.rs b/v2/dependency-patches/tch-0.24.0/src/nn/mod.rs new file mode 100644 index 0000000000..9343462d7e --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/nn/mod.rs @@ -0,0 +1,69 @@ +//! A small neural-network library based on Torch. +//! +//! This library tries to stay as close as possible to the original +//! Python and C++ implementations. +pub mod init; +pub use init::{f_init, init, Init}; + +mod var_store; +pub use var_store::{Path, VarStore, Variables}; + +mod module; +pub use module::{Module, ModuleT}; + +mod linear; +pub use linear::*; + +mod conv; +pub use conv::*; + +mod conv_transpose; +pub use conv_transpose::*; + +mod batch_norm; +pub use batch_norm::*; + +mod group_norm; +pub use group_norm::*; + +mod layer_norm; +pub use layer_norm::*; + +mod sparse; +pub use sparse::*; + +mod rnn; +pub use rnn::*; + +mod func; +pub use func::*; + +mod sequential; +pub use sequential::*; + +mod optimizer; +pub use optimizer::{ + adam, adamw, rms_prop, sgd, Adam, AdamW, Optimizer, OptimizerConfig, RmsProp, Sgd, +}; + +/// An identity layer. This just propagates its tensor input as output. +#[derive(Debug)] +pub struct Id(); + +impl ModuleT for Id { + fn forward_t(&self, xs: &crate::Tensor, _train: bool) -> crate::Tensor { + xs.shallow_clone() + } +} + +impl Module for crate::CModule { + fn forward(&self, xs: &crate::Tensor) -> crate::Tensor { + self.forward_ts(&[xs]).unwrap() + } +} + +impl ModuleT for crate::TrainableCModule { + fn forward_t(&self, xs: &crate::Tensor, _train: bool) -> crate::Tensor { + self.inner.forward_ts(&[xs]).unwrap() + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/module.rs b/v2/dependency-patches/tch-0.24.0/src/nn/module.rs new file mode 100644 index 0000000000..6301cbfacf --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/nn/module.rs @@ -0,0 +1,67 @@ +//! Basic module traits defining the forward pass. +use crate::{data::Iter2, Device, Tensor}; + +/// The simplest module trait, defining a forward function. +pub trait Module: std::fmt::Debug + Send { + fn forward(&self, xs: &Tensor) -> Tensor; +} + +/// Module trait with an additional train parameter. +/// +/// The train parameter is commonly used to have different behavior between training +/// and evaluation, e.g. when using dropout or batch-normalization. +pub trait ModuleT: std::fmt::Debug + Send { + fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor; + + fn batch_accuracy_for_logits( + &self, + xs: &Tensor, + ys: &Tensor, + d: Device, + batch_size: i64, + ) -> f64 { + let _no_grad = crate::no_grad_guard(); + let mut sum_accuracy = 0f64; + let mut sample_count = 0f64; + for (xs, ys) in Iter2::new(xs, ys, batch_size).return_smaller_last_batch() { + let acc = self.forward_t(&xs.to_device(d), false).accuracy_for_logits(&ys.to_device(d)); + let size = xs.size()[0] as f64; + sum_accuracy += f64::try_from(&acc).unwrap() * size; + sample_count += size; + } + sum_accuracy / sample_count + } +} + +impl ModuleT for T +where + T: Module, +{ + fn forward_t(&self, xs: &Tensor, _train: bool) -> Tensor { + self.forward(xs) + } +} + +impl Tensor { + pub fn apply(&self, m: &M) -> Tensor { + m.forward(self) + } + + pub fn apply_t(&self, m: &M, train: bool) -> Tensor { + m.forward_t(self, train) + } + + pub fn apply_opt(&self, m: &Option) -> Tensor { + match m { + Some(m) => m.forward(self), + None => self.shallow_clone(), + } + } + + pub fn apply_opt_t(&self, m: &Option, train: bool) -> Tensor { + match m { + Some(m) => m.forward_t(self, train), + None => self.shallow_clone(), + } + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/optimizer.rs b/v2/dependency-patches/tch-0.24.0/src/nn/optimizer.rs new file mode 100644 index 0000000000..a4ceb5d28d --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/nn/optimizer.rs @@ -0,0 +1,324 @@ +//! Optimizers to be used for gradient-descent based training. +use super::var_store::{VarStore, Variables}; +use crate::wrappers::optimizer::COptimizer; +use crate::{TchError, Tensor}; +use std::sync::{Arc, Mutex}; + +/// An optimizer to run gradient descent. +#[derive(Debug)] +pub struct Optimizer { + opt: COptimizer, + variables: Arc>, + variables_in_optimizer: usize, +} + +/// Optimizer configurations. These configs can be used to build optimizer. +pub trait OptimizerConfig +where + Self: std::marker::Sized, +{ + fn build_copt(&self, lr: f64) -> Result; + + /// Builds an optimizer with the specified learning rate handling variables stored in `vs`. + fn build(self, vs: &VarStore, lr: f64) -> Result { + let mut opt = self.build_copt(lr)?; + let v = vs.variables_.lock().unwrap(); + for var in &v.trainable_variables { + opt.add_parameters(&var.tensor, var.group)?; + } + Ok(Optimizer { + opt, + variables: vs.variables_.clone(), + variables_in_optimizer: v.trainable_variables.len(), + }) + } +} + +/// Parameters for the SGD optimizer. +#[derive(Debug, Copy, Clone)] +pub struct Sgd { + pub momentum: f64, + pub dampening: f64, + pub wd: f64, + pub nesterov: bool, +} + +impl Default for Sgd { + fn default() -> Self { + Sgd { momentum: 0., dampening: 0., wd: 0., nesterov: false } + } +} + +/// Creates the configuration for a Stochastic Gradient Descent (SGD) optimizer. +pub fn sgd(momentum: f64, dampening: f64, wd: f64, nesterov: bool) -> Sgd { + Sgd { momentum, dampening, wd, nesterov } +} + +impl OptimizerConfig for Sgd { + fn build_copt(&self, lr: f64) -> Result { + COptimizer::sgd(lr, self.momentum, self.dampening, self.wd, self.nesterov) + } +} + +/// Parameters for the Adam optimizer. +#[derive(Debug, Copy, Clone)] +pub struct Adam { + pub beta1: f64, + pub beta2: f64, + pub wd: f64, + pub eps: f64, + pub amsgrad: bool, +} + +impl Default for Adam { + fn default() -> Self { + Adam { beta1: 0.9, beta2: 0.999, wd: 0., eps: 1e-8, amsgrad: false } + } +} + +/// Creates the configuration for the Adam optimizer. +pub fn adam(beta1: f64, beta2: f64, wd: f64) -> Adam { + Adam { beta1, beta2, wd, eps: 1e-8, amsgrad: false } +} + +impl Adam { + pub fn beta1(mut self, b: f64) -> Self { + self.beta1 = b; + self + } + + pub fn beta2(mut self, b: f64) -> Self { + self.beta2 = b; + self + } + + pub fn wd(mut self, w: f64) -> Self { + self.wd = w; + self + } + + pub fn eps(mut self, e: f64) -> Self { + self.eps = e; + self + } + + pub fn amsgrad(mut self, a: bool) -> Self { + self.amsgrad = a; + self + } +} + +impl OptimizerConfig for Adam { + fn build_copt(&self, lr: f64) -> Result { + COptimizer::adam(lr, self.beta1, self.beta2, self.wd, self.eps, self.amsgrad) + } +} + +/// Parameters for the AdamW optimizer. +#[derive(Debug, Copy, Clone)] +pub struct AdamW { + pub beta1: f64, + pub beta2: f64, + pub wd: f64, + pub eps: f64, + pub amsgrad: bool, +} + +impl Default for AdamW { + fn default() -> Self { + AdamW { beta1: 0.9, beta2: 0.999, wd: 0.01, eps: 1e-8, amsgrad: false } + } +} + +/// Creates the configuration for the AdamW optimizer. +pub fn adamw(beta1: f64, beta2: f64, wd: f64) -> AdamW { + AdamW { beta1, beta2, wd, eps: 1e-8, amsgrad: false } +} + +impl AdamW { + pub fn beta1(mut self, b: f64) -> Self { + self.beta1 = b; + self + } + + pub fn beta2(mut self, b: f64) -> Self { + self.beta2 = b; + self + } + + pub fn wd(mut self, w: f64) -> Self { + self.wd = w; + self + } + + pub fn eps(mut self, e: f64) -> Self { + self.eps = e; + self + } + + pub fn amsgrad(mut self, a: bool) -> Self { + self.amsgrad = a; + self + } +} + +impl OptimizerConfig for AdamW { + fn build_copt(&self, lr: f64) -> Result { + COptimizer::adamw(lr, self.beta1, self.beta2, self.wd, self.eps, self.amsgrad) + } +} + +/// Parameters for the RmsProp optimizer. +#[derive(Debug, Copy, Clone)] +pub struct RmsProp { + pub alpha: f64, + pub eps: f64, + pub wd: f64, + pub momentum: f64, + pub centered: bool, +} + +impl Default for RmsProp { + fn default() -> Self { + RmsProp { alpha: 0.99, eps: 1e-8, wd: 0., momentum: 0., centered: false } + } +} + +/// Creates the configuration for the RmsProp optimizer. +pub fn rms_prop(alpha: f64, eps: f64, wd: f64, momentum: f64, centered: bool) -> RmsProp { + RmsProp { alpha, eps, wd, momentum, centered } +} + +impl OptimizerConfig for RmsProp { + fn build_copt(&self, lr: f64) -> Result { + COptimizer::rms_prop(lr, self.alpha, self.eps, self.wd, self.momentum, self.centered) + } +} + +impl Optimizer { + fn add_missing_variables(&mut self) { + let v = self.variables.lock().unwrap(); + if v.trainable_variables.len() > self.variables_in_optimizer { + for var in &v.trainable_variables[self.variables_in_optimizer..] { + self.opt.add_parameters(&var.tensor, var.group).unwrap(); + } + self.variables_in_optimizer = v.trainable_variables.len(); + } + } + + /// Zeroes the gradient for the tensors tracked by this optimizer. + pub fn zero_grad(&mut self) { + self.add_missing_variables(); + self.opt.zero_grad().unwrap() + } + + /// Clips gradient value at some specified maximum value. + pub fn clip_grad_value(&self, max: f64) { + let v = self.variables.lock().unwrap(); + for var in v.trainable_variables.iter() { + let mut grad = var.tensor.grad(); + if grad.defined() { + let _t = grad.clamp_(-max, max); + } + } + } + + /// Clips gradient L2 norm over all trainable parameters. + /// + /// The norm is computed over all gradients together, as if they were + /// concatenated into a single vector. + pub fn clip_grad_norm(&self, max: f64) { + crate::no_grad(|| { + let v = self.variables.lock().unwrap(); + let mut norms = vec![]; + for var in v.trainable_variables.iter() { + let grad = var.tensor.grad(); + if grad.defined() { + norms.push(grad.norm()); + } + } + let total_norm = f64::try_from(Tensor::stack(&norms, 0).norm()).unwrap(); + let clip_coef = max / (total_norm + 1e-6); + if clip_coef < 1.0 { + for var in v.trainable_variables.iter() { + let mut grad = var.tensor.grad(); + if grad.defined() { + let _t = grad.g_mul_scalar_(clip_coef); + } + } + } + }) + } + + /// Performs an optimization step, updating the tracked tensors based on their gradients. + pub fn step(&mut self) { + self.add_missing_variables(); + self.opt.step().unwrap() + } + + /// Applies a backward step pass, update the gradients, and performs an optimization step. + pub fn backward_step(&mut self, loss: &Tensor) { + self.add_missing_variables(); + self.opt.zero_grad().unwrap(); + loss.backward(); + self.opt.step().unwrap() + } + + /// Applies a backward step pass, update the gradients, and performs an optimization step. + /// + /// The gradients are clipped based on `max` before being applied. + pub fn backward_step_clip(&mut self, loss: &Tensor, max: f64) { + self.add_missing_variables(); + self.opt.zero_grad().unwrap(); + loss.backward(); + self.clip_grad_value(max); + self.opt.step().unwrap() + } + + /// Applies a backward step pass, update the gradients, and performs an optimization step. + /// + /// The gradients L2 norm is clipped based on `max`. + pub fn backward_step_clip_norm(&mut self, loss: &Tensor, max: f64) { + self.add_missing_variables(); + self.opt.zero_grad().unwrap(); + loss.backward(); + self.clip_grad_norm(max); + self.opt.step().unwrap() + } + + /// Sets the optimizer learning rate. + pub fn set_lr(&mut self, lr: f64) { + self.opt.set_learning_rate(lr).unwrap() + } + + /// Sets the optimizer momentum. + pub fn set_momentum(&mut self, m: f64) { + self.opt.set_momentum(m).unwrap() + } + + /// Sets the optimizer learning rate for a parameter group. + pub fn set_lr_group(&mut self, group: usize, lr: f64) { + self.opt.set_learning_rate_group(group, lr).unwrap() + } + + /// Sets the optimizer momentum. + pub fn set_momentum_group(&mut self, group: usize, m: f64) { + self.opt.set_momentum_group(group, m).unwrap() + } + + /// Returns all the trainable variables for this optimizer. + pub fn trainable_variables(&self) -> Vec { + let variables = self.variables.lock().unwrap(); + variables.trainable_variables.iter().map(|v| v.tensor.shallow_clone()).collect() + } + + /// Sets the optimizer weight decay. + pub fn set_weight_decay(&mut self, weight_decay: f64) { + self.opt.set_weight_decay(weight_decay).unwrap() + } + + /// Sets the optimizer weight decay. + pub fn set_weight_decay_group(&mut self, group: usize, weight_decay: f64) { + self.opt.set_weight_decay_group(group, weight_decay).unwrap() + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/rnn.rs b/v2/dependency-patches/tch-0.24.0/src/nn/rnn.rs new file mode 100644 index 0000000000..f74fd7f016 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/nn/rnn.rs @@ -0,0 +1,285 @@ +//! Recurrent Neural Networks +use crate::{Device, Tensor}; +use std::borrow::Borrow; + +/// Trait for Recurrent Neural Networks. +#[allow(clippy::upper_case_acronyms)] +pub trait RNN { + type State; + + /// A zero state from which the recurrent network is usually initialized. + fn zero_state(&self, batch_dim: i64) -> Self::State; + + /// Applies a single step of the recurrent network. + /// + /// The input should have dimensions [batch_size, features]. + fn step(&self, input: &Tensor, state: &Self::State) -> Self::State; + + /// Applies multiple steps of the recurrent network. + /// + /// The input should have dimensions [batch_size, seq_len, features]. + /// The initial state is the result of applying zero_state. + fn seq(&self, input: &Tensor) -> (Tensor, Self::State) { + let batch_dim = input.size()[0]; + let state = self.zero_state(batch_dim); + self.seq_init(input, &state) + } + + /// Applies multiple steps of the recurrent network. + /// + /// The input should have dimensions [batch_size, seq_len, features]. + fn seq_init(&self, input: &Tensor, state: &Self::State) -> (Tensor, Self::State); +} + +/// The state for a LSTM network, this contains two tensors. +#[allow(clippy::upper_case_acronyms)] +#[derive(Debug)] +pub struct LSTMState(pub (Tensor, Tensor)); + +impl LSTMState { + /// The hidden state vector, which is also the output of the LSTM. + pub fn h(&self) -> Tensor { + (self.0).0.shallow_clone() + } + + /// The cell state vector. + pub fn c(&self) -> Tensor { + (self.0).1.shallow_clone() + } +} + +// The GRU and LSTM layers share the same config. +/// Configuration for the GRU and LSTM layers. +#[allow(clippy::upper_case_acronyms)] +#[derive(Debug, Clone, Copy)] +pub struct RNNConfig { + pub has_biases: bool, + pub num_layers: i64, + pub dropout: f64, + pub train: bool, + pub bidirectional: bool, + pub batch_first: bool, + pub w_ih_init: super::Init, + pub w_hh_init: super::Init, + pub b_ih_init: Option, + pub b_hh_init: Option, +} + +impl Default for RNNConfig { + fn default() -> Self { + RNNConfig { + has_biases: true, + num_layers: 1, + dropout: 0., + train: true, + bidirectional: false, + batch_first: true, + w_ih_init: super::init::DEFAULT_KAIMING_UNIFORM, + w_hh_init: super::init::DEFAULT_KAIMING_UNIFORM, + b_ih_init: Some(super::Init::Const(0.)), + b_hh_init: Some(super::Init::Const(0.)), + } + } +} + +fn rnn_weights<'a, T: Borrow>>( + vs: T, + in_dim: i64, + hidden_dim: i64, + gate_dim: i64, + num_directions: i64, + c: RNNConfig, +) -> Vec { + let vs = vs.borrow(); + let mut flat_weights = vec![]; + for layer_idx in 0..c.num_layers { + for direction_idx in 0..num_directions { + let in_dim = if layer_idx == 0 { in_dim } else { hidden_dim * num_directions }; + let suffix = if direction_idx == 1 { "_reverse" } else { "" }; + let w_ih = vs.var( + &format!("weight_ih_l{layer_idx}{suffix}"), + &[gate_dim, in_dim], + c.w_ih_init, + ); + let w_hh = vs.var( + &format!("weight_hh_l{layer_idx}{suffix}"), + &[gate_dim, hidden_dim], + c.w_hh_init, + ); + flat_weights.push(w_ih); + flat_weights.push(w_hh); + if c.has_biases { + let b_ih = vs.var( + &format!("bias_ih_l{layer_idx}{suffix}"), + &[gate_dim], + c.b_ih_init.unwrap(), + ); + let b_hh = vs.var( + &format!("bias_hh_l{layer_idx}{suffix}"), + &[gate_dim], + c.b_hh_init.unwrap(), + ); + flat_weights.push(b_ih); + flat_weights.push(b_hh); + } + } + } + flat_weights +} + +/// A Long Short-Term Memory (LSTM) layer. +/// +/// +#[allow(clippy::upper_case_acronyms)] +#[derive(Debug)] +pub struct LSTM { + flat_weights: Vec, + hidden_dim: i64, + config: RNNConfig, + device: Device, +} + +/// Creates a LSTM layer. +pub fn lstm<'a, T: Borrow>>( + vs: T, + in_dim: i64, + hidden_dim: i64, + c: RNNConfig, +) -> LSTM { + let vs = vs.borrow(); + let num_directions = if c.bidirectional { 2 } else { 1 }; + let gate_dim = 4 * hidden_dim; + let flat_weights = rnn_weights(vs, in_dim, hidden_dim, gate_dim, num_directions, c); + + if vs.device().is_cuda() && crate::Cuda::cudnn_is_available() { + let _ = Tensor::internal_cudnn_rnn_flatten_weight( + &flat_weights, + 4, + in_dim, + 2, /* 2 for LSTM see rnn.cpp in pytorch */ + hidden_dim, + 0, /* disables projections */ + c.num_layers, + c.batch_first, + c.bidirectional, + ); + } + LSTM { flat_weights, hidden_dim, config: c, device: vs.device() } +} + +impl RNN for LSTM { + type State = LSTMState; + + fn zero_state(&self, batch_dim: i64) -> LSTMState { + let num_directions = if self.config.bidirectional { 2 } else { 1 }; + let layer_dim = self.config.num_layers * num_directions; + let shape = [layer_dim, batch_dim, self.hidden_dim]; + let zeros = Tensor::zeros(shape, (self.flat_weights[0].kind(), self.device)); + LSTMState((zeros.shallow_clone(), zeros.shallow_clone())) + } + + fn step(&self, input: &Tensor, in_state: &LSTMState) -> LSTMState { + let input = input.unsqueeze(1); + let (_output, state) = self.seq_init(&input, in_state); + state + } + + fn seq_init(&self, input: &Tensor, in_state: &LSTMState) -> (Tensor, LSTMState) { + let LSTMState((h, c)) = in_state; + let flat_weights = self.flat_weights.iter().collect::>(); + let (output, h, c) = input.lstm( + &[h, c], + &flat_weights, + self.config.has_biases, + self.config.num_layers, + self.config.dropout, + self.config.train, + self.config.bidirectional, + self.config.batch_first, + ); + (output, LSTMState((h, c))) + } +} + +/// A GRU state, this contains a single tensor. +#[allow(clippy::upper_case_acronyms)] +#[derive(Debug)] +pub struct GRUState(pub Tensor); + +impl GRUState { + pub fn value(&self) -> Tensor { + self.0.shallow_clone() + } +} + +/// A Gated Recurrent Unit (GRU) layer. +/// +/// +#[allow(clippy::upper_case_acronyms)] +#[derive(Debug)] +pub struct GRU { + flat_weights: Vec, + hidden_dim: i64, + config: RNNConfig, + device: Device, +} + +/// Creates a new GRU layer. +pub fn gru<'a, T: Borrow>>( + vs: T, + in_dim: i64, + hidden_dim: i64, + c: RNNConfig, +) -> GRU { + let vs = vs.borrow(); + let num_directions = if c.bidirectional { 2 } else { 1 }; + let gate_dim = 3 * hidden_dim; + let flat_weights = rnn_weights(vs, in_dim, hidden_dim, gate_dim, num_directions, c); + + if vs.device().is_cuda() && crate::Cuda::cudnn_is_available() { + let _ = Tensor::internal_cudnn_rnn_flatten_weight( + &flat_weights, + 4, + in_dim, + 3, /* 3 for GRU see rnn.cpp in pytorch */ + hidden_dim, + 0, /* disables projections */ + c.num_layers, + c.batch_first, + c.bidirectional, + ); + } + GRU { flat_weights, hidden_dim, config: c, device: vs.device() } +} + +impl RNN for GRU { + type State = GRUState; + + fn zero_state(&self, batch_dim: i64) -> GRUState { + let num_directions = if self.config.bidirectional { 2 } else { 1 }; + let layer_dim = self.config.num_layers * num_directions; + let shape = [layer_dim, batch_dim, self.hidden_dim]; + GRUState(Tensor::zeros(shape, (self.flat_weights[0].kind(), self.device))) + } + + fn step(&self, input: &Tensor, in_state: &GRUState) -> GRUState { + let input = input.unsqueeze(1); + let (_output, state) = self.seq_init(&input, in_state); + state + } + + fn seq_init(&self, input: &Tensor, in_state: &GRUState) -> (Tensor, GRUState) { + let GRUState(h) = in_state; + let (output, h) = input.gru( + h, + &self.flat_weights, + self.config.has_biases, + self.config.num_layers, + self.config.dropout, + self.config.train, + self.config.bidirectional, + self.config.batch_first, + ); + (output, GRUState(h)) + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/sequential.rs b/v2/dependency-patches/tch-0.24.0/src/nn/sequential.rs new file mode 100644 index 0000000000..ea1be81a15 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/nn/sequential.rs @@ -0,0 +1,149 @@ +//! A sequential layer used to chain multiple layers and closures. +use super::{Module, ModuleT}; +use crate::Tensor; + +/// A sequential layer combining multiple other layers. +#[derive(Debug)] +pub struct Sequential { + layers: Vec>, +} + +/// Creates a new empty sequential layer. +pub fn seq() -> Sequential { + Sequential { layers: vec![] } +} + +impl Sequential { + /// The number of sub-layers embedded in this layer. + pub fn len(&self) -> i64 { + self.layers.len() as i64 + } + + /// Returns true if this layer does not have any sub-layer. + pub fn is_empty(&self) -> bool { + self.layers.is_empty() + } +} + +impl Module for Sequential { + fn forward(&self, xs: &Tensor) -> Tensor { + if self.layers.is_empty() { + xs.shallow_clone() + } else { + let xs = self.layers[0].forward(xs); + self.layers.iter().skip(1).fold(xs, |xs, layer| layer.forward(&xs)) + } + } +} + +impl Sequential { + /// Appends a layer after all the current layers. + #[allow(clippy::should_implement_trait)] + pub fn add(mut self, layer: M) -> Self { + self.layers.push(Box::new(layer)); + self + } + + /// Appends a closure after all the current layers. + pub fn add_fn(self, f: F) -> Self + where + F: 'static + Fn(&Tensor) -> Tensor + Send, + { + self.add(super::func(f)) + } + + /// Applies the forward pass and returns the output for each layer. + pub fn forward_all(&self, xs: &Tensor, n: Option) -> Vec { + if self.layers.is_empty() { + vec![xs.shallow_clone()] + } else { + let n = n.unwrap_or(self.layers.len()); + let xs = self.layers[0].forward(xs); + let mut vec = vec![]; + let out = self.layers.iter().take(n).skip(1).fold(xs, |xs, layer| { + let out = layer.forward(&xs); + vec.push(xs); + out + }); + vec.push(out); + vec + } + } +} + +/// A sequential layer combining new layers with support for a training mode. +#[derive(Debug)] +pub struct SequentialT { + layers: Vec>, +} + +/// Creates a new empty sequential layer. +pub fn seq_t() -> SequentialT { + SequentialT { layers: vec![] } +} + +impl SequentialT { + /// The number of sub-layers embedded in this layer. + pub fn len(&self) -> i64 { + self.layers.len() as i64 + } + + /// Returns true if this layer does not have any sub-layer. + pub fn is_empty(&self) -> bool { + self.layers.is_empty() + } +} + +impl ModuleT for SequentialT { + fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor { + if self.layers.is_empty() { + xs.shallow_clone() + } else { + let xs = self.layers[0].forward_t(xs, train); + self.layers.iter().skip(1).fold(xs, |xs, layer| layer.forward_t(&xs, train)) + } + } +} + +impl SequentialT { + /// Appends a layer after all the current layers. + #[allow(clippy::should_implement_trait)] + pub fn add(mut self, layer: M) -> Self { + self.layers.push(Box::new(layer)); + self + } + + /// Appends a closure after all the current layers. + pub fn add_fn(self, f: F) -> Self + where + F: 'static + Fn(&Tensor) -> Tensor + Send, + { + self.add(super::func(f)) + } + + /// Appends a closure after all the current layers. + pub fn add_fn_t(self, f: F) -> Self + where + F: 'static + Fn(&Tensor, bool) -> Tensor + Send, + { + self.add(super::func_t(f)) + } + + /// Applies the forward pass and returns the output for each layer. + pub fn forward_all_t(&self, xs: &Tensor, train: bool, n: Option) -> Vec { + if self.layers.is_empty() { + vec![xs.shallow_clone()] + } else { + let n = n.unwrap_or(self.layers.len()); + let xs = self.layers[0].forward_t(xs, train); + let mut vec = vec![]; + let out = self.layers.iter().take(n).skip(1).fold(xs, |xs, layer| { + let out = layer.forward_t(&xs, train); + vec.push(xs); + out + }); + vec.push(out); + vec + } + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/sparse.rs b/v2/dependency-patches/tch-0.24.0/src/nn/sparse.rs new file mode 100644 index 0000000000..6a6eb08fc8 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/nn/sparse.rs @@ -0,0 +1,55 @@ +//! Sparse Layers +use crate::Tensor; +use std::borrow::Borrow; + +/// Configuration option for an embedding layer. +#[derive(Debug, Clone, Copy)] +pub struct EmbeddingConfig { + pub sparse: bool, + pub scale_grad_by_freq: bool, + pub ws_init: super::Init, + pub padding_idx: i64, +} + +impl Default for EmbeddingConfig { + fn default() -> Self { + EmbeddingConfig { + sparse: false, + scale_grad_by_freq: false, + ws_init: super::Init::Randn { mean: 0., stdev: 1. }, + padding_idx: -1, + } + } +} + +/// An embedding layer. +/// +/// An embedding layer acts as a simple lookup table that stores embeddings. +/// This is commonly used to store word embeddings. +#[derive(Debug)] +pub struct Embedding { + pub ws: Tensor, + config: EmbeddingConfig, +} + +pub fn embedding<'a, T: Borrow>>( + vs: T, + num_embeddings: i64, + embedding_dim: i64, + config: EmbeddingConfig, +) -> Embedding { + let vs = vs.borrow(); + Embedding { ws: vs.var("weight", &[num_embeddings, embedding_dim], config.ws_init), config } +} + +impl super::module::Module for Embedding { + fn forward(&self, xs: &Tensor) -> Tensor { + Tensor::embedding( + &self.ws, + xs, + self.config.padding_idx, + self.config.scale_grad_by_freq, + self.config.sparse, + ) + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/var_store.rs b/v2/dependency-patches/tch-0.24.0/src/nn/var_store.rs new file mode 100644 index 0000000000..0f51bfdef7 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/nn/var_store.rs @@ -0,0 +1,935 @@ +//! Variable stores. +use super::Init; +use crate::tensor::Tensor; +use crate::wrappers::stream::ReadSeekAdapter; +use crate::{Device, Kind, TchError}; +use std::collections::hash_map::Entry::{Occupied, Vacant}; +use std::collections::HashMap; +use std::io::{Read, Seek}; +use std::ops::Div; +use std::sync::{Arc, Mutex, MutexGuard}; + +/// The separator is used to separate path elements in the tensor names. +const SEP: char = '.'; + +#[derive(Debug)] +pub struct Var { + pub tensor: Tensor, + pub group: usize, +} + +// When the variable store is frozen, the trainable_variables vector +// still contains the same tensors however these tensors are set not +// to require gradients. +#[derive(Debug)] +pub struct Variables { + pub named_variables: HashMap, + pub trainable_variables: Vec, +} + +/// A VarStore is used to store variables used by one or multiple layers. +/// It specifies a single device where all variables are stored. +#[derive(Debug)] +pub struct VarStore { + pub variables_: Arc>, + device: Device, + kind: Kind, +} + +/// A variable store with an associated path for variables naming. +#[derive(Debug, Clone)] +pub struct Path<'a> { + path: Vec, + group: usize, + var_store: &'a VarStore, +} + +/// An Entry holds an entry corresponding to a given name in Path. +#[derive(Debug)] +pub struct Entry<'a> { + name: &'a str, + variables: MutexGuard<'a, Variables>, + // This field holds the mutex lock + path: &'a Path<'a>, +} + +impl VarStore { + /// Creates a new var-store located on the specified device. + pub fn new(device: Device) -> VarStore { + let variables = + Variables { named_variables: HashMap::new(), trainable_variables: Vec::new() }; + VarStore { variables_: Arc::new(Mutex::new(variables)), device, kind: Kind::Float } + } + + pub fn merge(var_stores: Vec<(VarStore, Option<&str>)>) -> Result { + let mut new_var_store = VarStore::new(Device::Cpu); + + if var_stores.is_empty() { + Ok(new_var_store) + } else { + let mut new_variables = + Variables { named_variables: HashMap::new(), trainable_variables: Vec::new() }; + let device = var_stores[0].0.device(); + + for (var_store, prefix) in var_stores { + if var_store.device() != device { + return Err(TchError::Torch(format!( + "All VarStores must be on the same device, got {:?} and {:?}", + device, + var_store.device() + ))); + } + for (var_name, var) in var_store.variables() { + let new_var_name = format!("{}{}", prefix.unwrap_or(""), var_name); + match new_variables.named_variables.entry(new_var_name) { + Occupied(v) => { + return Err(TchError::Torch(format!( + "Duplicate variable name found: {}. Provide a unique prefix to allow merge operation", + v.key(), + ))); + } + Vacant(v) => { + v.insert(var); + } + } + } + for trainable_var in + var_store.variables_.lock().unwrap().trainable_variables.drain(..) + { + new_variables.trainable_variables.push(trainable_var); + } + } + new_var_store.variables_ = Arc::new(Mutex::new(new_variables)); + new_var_store.device = device; + + Ok(new_var_store) + } + } + + /// Gets the device for this var-store. + pub fn device(&self) -> Device { + self.device + } + + /// Gets the default kind of new variables + pub fn kind(&self) -> Kind { + self.kind + } + + /// Returns the number of tensors currently stored on this var-store. + pub fn len(&self) -> usize { + let variables = self.variables_.lock().unwrap(); + variables.named_variables.len() + } + + /// Returns true if no tensors are currently stored on this var-store. + pub fn is_empty(&self) -> bool { + let variables = self.variables_.lock().unwrap(); + variables.named_variables.is_empty() + } + + /// Returns all the trainable variables for this var-store. + pub fn trainable_variables(&self) -> Vec { + let variables = self.variables_.lock().unwrap(); + variables.trainable_variables.iter().map(|v| v.tensor.shallow_clone()).collect() + } + + /// Returns all variables along with their names. + pub fn variables(&self) -> HashMap { + let variables = self.variables_.lock().unwrap(); + variables + .named_variables + .iter() + .map(|(name, v)| (name.clone(), v.shallow_clone())) + .collect() + } + + /// Gets the root path for this variable store. + /// + /// Variables are named and organized using paths. This function returns + /// the top level path for the var store and can be combined with '/' + /// to create sub-paths. + pub fn root(&self) -> Path<'_> { + Path { path: vec![], group: 0, var_store: self } + } + + /// Saves the var-store variable values to a file. + /// + /// Weight values for all the tensors currently stored in the + /// var-store are saved in the given file. + /// + /// If the given path ends with the suffix `.safetensors`, the file will + /// be saved in safetensors format. Otherwise, libtorch C++ module format + /// will be used. Note that saving in pickle format (`.pt` extension) is + /// not supported by the C++ API of Torch. + pub fn save>(&self, path: T) -> Result<(), TchError> { + let variables = self.variables_.lock().unwrap(); + let named_tensors = variables.named_variables.iter().collect::>(); + match path.as_ref().extension().and_then(|x| x.to_str()) { + Some("safetensors") => Tensor::write_safetensors(named_tensors.as_slice(), path), + Some(_) | None => Tensor::save_multi(named_tensors.as_slice(), path), + } + } + + /// Saves the var-store variable values to a stream. + /// + /// Weight values for all the tensors currently stored in the + /// var-store gets saved in the given stream. + pub fn save_to_stream(&self, stream: W) -> Result<(), TchError> { + let variables = self.variables_.lock().unwrap(); + let named_tensors = variables.named_variables.iter().collect::>(); + Tensor::save_multi_to_stream(named_tensors.as_slice(), stream) + } + + fn named_tensors>( + &self, + path: T, + ) -> Result, TchError> { + let named_tensors = match path.as_ref().extension().and_then(|x| x.to_str()) { + Some("bin") | Some("pt") => Tensor::loadz_multi_with_device(&path, self.device), + Some("safetensors") => Tensor::read_safetensors(path), + Some(_) | None => Tensor::load_multi_with_device(&path, self.device), + }; + Ok(named_tensors?.into_iter().collect()) + } + + /// Copies the data from source tensor to destination + /// + /// Updates the precision of the destination to match the source + fn copy_data_with_precision_update(src: &Tensor, dst: &mut Tensor) -> Result<(), TchError> { + dst.set_data(&dst.to_kind(src.kind())); + dst.f_copy_(src) + } + + fn load_internal>(&mut self, path: T) -> Result<(), TchError> { + let named_tensors = self.named_tensors(&path)?; + let mut variables = self.variables_.lock().unwrap(); + for (name, var) in variables.named_variables.iter_mut() { + match named_tensors.get(name) { + Some(src) => crate::no_grad(|| { + Self::copy_data_with_precision_update(src, var) + .map_err(|e| e.path_context(name)) + })?, + None => { + return Err(TchError::TensorNameNotFound( + name.to_string(), + path.as_ref().to_string_lossy().into_owned(), + )); + } + } + } + Ok(()) + } + + /// Loads the var-store variable values from a file. + /// + /// Weight values for all the tensors currently stored in the + /// var-store are loaded from the given file. Note that the set of + /// variables stored in the var-store is not changed, only the values + /// for these tensors are modified. + /// + /// The format of the file is deduced from the file extension: + /// - `.safetensors`: The file is assumed to be in safetensors format. + /// - `.bin` or `.pt`: The file is assumed to be in pickle format. + /// - Otherwise, the file is assumed to be in libtorch C++ module format. + pub fn load>(&mut self, path: T) -> Result<(), TchError> { + if self.device != Device::Mps { + self.load_internal(path) + } else { + // Current workaround to allow loading in MPS device. + // On new libtorch releases check if direct loading becomes possible and revert + // See (https://github.com/LaurentMazare/tch-rs/issues/609#issuecomment-1427071598). + self.set_device(Device::Cpu); + let or_error = self.load_internal(path); + // Be cautious not to early exit so as to ensure that the device is set back to Mps + // even on errors. + self.set_device(Device::Mps); + or_error + } + } + + /// Loads the var-store variable values from a stream. + /// + /// Weight values for all the tensors currently stored in the + /// var-store gets loaded from the given stream. Note that the set of + /// variables stored in the var-store is not changed, only the values + /// for these tensors are modified. + pub fn load_from_stream(&mut self, stream: S) -> Result<(), TchError> { + let adapter = ReadSeekAdapter::new(stream); + let named_tensors = Tensor::load_multi_from_stream_with_device(adapter, self.device)?; + let named_tensors: HashMap<_, _> = named_tensors.into_iter().collect(); + let mut variables = self.variables_.lock().unwrap(); + for (name, var) in variables.named_variables.iter_mut() { + match named_tensors.get(name) { + Some(src) => crate::no_grad(|| { + Self::copy_data_with_precision_update(src, var) + .map_err(|e| e.path_context(name)) + })?, + None => { + return Err(TchError::TensorNameNotFound( + name.to_string(), + "source stream".to_string(), + )); + } + } + } + Ok(()) + } + + /// Loads the var-store variable values from a file if it exists. + /// + /// Weight values for the tensors currently stored in the var-store and the given file get + /// loaded from the given file. If a variable in the var store is not present in the given file, + /// it is skipped and its values are not updated. This method should be used if pre-trained + /// weight for only parts of the model are available. + /// Note that the set of variables stored in the var-store is not changed, only the values + /// for these tensors are modified. + /// + /// Returns a String Vector containing the names of missing variables. + pub fn load_partial>( + &mut self, + path: T, + ) -> Result, TchError> { + let named_tensors = self.named_tensors(&path)?; + let mut variables = self.variables_.lock().unwrap(); + let mut missing_variables = Vec::new(); + for (name, var) in variables.named_variables.iter_mut() { + match named_tensors.get(name) { + Some(src) => crate::no_grad(|| { + Self::copy_data_with_precision_update(src, var) + .map_err(|e| e.path_context(name)) + })?, + None => { + missing_variables.push(name.to_owned()); + } + } + } + Ok(missing_variables) + } + + /// Freezes a var store. + /// + /// Gradients for the variables in this store are not tracked + /// anymore. + pub fn freeze(&mut self) { + let variables = self.variables_.lock().unwrap(); + for variable in variables.trainable_variables.iter() { + let _v = variable.tensor.set_requires_grad(false); + } + } + + /// Unfreezes a var store. + /// + /// Gradients for the variables in this store are tracked again. + pub fn unfreeze(&mut self) { + let variables = self.variables_.lock().unwrap(); + for variable in variables.trainable_variables.iter() { + let _v = variable.tensor.set_requires_grad(true); + } + } + + /// Casts all variables in a var store to the target kind and sets the default kind + /// for new variables. + /// + /// For floating-point conversion, methods `half`, `bfloat16`, `float` and `double` + /// should be preferred as they ensure only float-like variables will be converted + /// to the target type. + pub fn set_kind(&mut self, kind: Kind) { + self.root().set_kind(kind); + self.kind = kind; + } + + /// Casts all float-like variable of a var store to half-precision (Half kind). + pub fn half(&mut self) { + self.root().half(); + } + + /// Casts all float-like variable of a var store to bfloat16-precision (BFloat16 kind). + pub fn bfloat16(&mut self) { + self.root().bfloat16(); + } + + /// Casts all float-like variable of a var store to single-precision (Float kind). + pub fn float(&mut self) { + self.root().float(); + } + + /// Casts all float-like variable of a var store to single-precision (Double kind). + pub fn double(&mut self) { + self.root().double(); + } + + /// Migrates a var store and all its tensor to a target device. + pub fn set_device(&mut self, device: Device) { + let mut variables = self.variables_.lock().unwrap(); + for (_, variable) in variables.named_variables.iter_mut() { + variable.set_data(&variable.to_device(device)); + } + self.device = device + } + + /// Copies variable values from a source var store to this var store. + /// + /// All the variables in this var store have to exist with the same + /// name in the source var store, otherwise an error is returned. + pub fn copy(&mut self, src: &VarStore) -> Result<(), TchError> { + let mut variables = self.variables_.lock().unwrap(); + let src_variables = src.variables_.lock().unwrap(); + let device = self.device; + for name in variables.named_variables.keys() { + if !src_variables.named_variables.contains_key(name) { + return Err(TchError::TensorNameNotFound( + name.to_string(), + "src var-store".to_string(), + )); + } + } + for (name, var) in variables.named_variables.iter_mut() { + let src_var = src_variables.named_variables.get(name).unwrap(); + crate::no_grad(|| var.f_copy_(&src_var.to_device(device)))?; + } + Ok(()) + } +} + +impl<'a> Path<'a> { + /// Get the components of the path. + pub fn components(&self) -> impl Iterator { + self.path.iter().map(String::as_str) + } + + /// Gets a sub-path of the given path. + pub fn sub(&self, s: T) -> Path<'a> { + let s = s.to_string(); + if s.chars().any(|x| x == SEP) { + panic!("sub name cannot contain {SEP} {s}"); + } + let mut path = self.path.clone(); + path.push(s); + Path { path, group: self.group, var_store: self.var_store } + } + + pub fn set_group(&self, group: usize) -> Path<'a> { + Path { path: self.path.clone(), group, var_store: self.var_store } + } + + /// Gets the device where the var-store variables are stored. + pub fn device(&self) -> Device { + self.var_store.device + } + + /// Gets the default kind of new variables + pub fn kind(&self) -> Kind { + self.var_store.kind + } + + pub fn path(&self, name: &str) -> String { + if name.chars().any(|x| x == SEP) { + panic!("variable name cannot contain {SEP} {name}"); + } + if self.path.is_empty() { + name.to_string() + } else { + format!("{}{}{}", self.path.join(&SEP.to_string()), SEP, name) + } + } + + /// Casts all variables in a var store sub-path to the target kind . + /// + /// Only the variable in the path sub-tree are cast to the target kind: + /// other var store variables are unaffected. For floating-point conversion, methods + /// `half`, `bfloat16`, `float` and `double` should be preferred as they ensure only + /// float-like variables will be converted to the target type. + pub fn set_kind(&mut self, kind: Kind) { + let path_root = self.path.join(SEP.to_string().as_str()); + let mut variables = self.var_store.variables_.lock().unwrap(); + for (variable_name, variable) in variables.named_variables.iter_mut() { + if variable_name.starts_with(&path_root) { + variable.set_data(&variable.to_kind(kind)); + } + } + } + + /// Casts all float-like variables in a var store sub-path to the target kind . + /// + /// Only the float-like variable in the path sub-tree are cast to the target kind: + /// other var store variables are unaffected + fn set_float_kind(&mut self, kind: Kind) { + let path_root = self.path.join(SEP.to_string().as_str()); + let mut variables = self.var_store.variables_.lock().unwrap(); + for (variable_name, variable) in variables.named_variables.iter_mut() { + if variable_name.starts_with(&path_root) & variable.is_floating_point() { + variable.set_data(&variable.to_kind(kind)); + } + } + } + + /// Casts all float-like variables in a var store sub-path to half-precision (Half kind). + /// + /// Only the variable in the path sub-tree are cast to half-precision: + /// other var store variables are unaffected + pub fn half(&mut self) { + self.set_float_kind(Kind::Half); + } + + /// Casts all float-like variables in a var store sub-path to bfloat16-precision (BFloat16 kind). + /// + /// Only the variable in the path sub-tree are cast to bfloat16-precision: + /// other var store variables are unaffected + pub fn bfloat16(&mut self) { + self.set_float_kind(Kind::BFloat16); + } + + /// Casts all float-like variables in a var store sub-path to single-precision (Float kind). + /// + /// Only the variable in the path sub-tree are cast to single-precision: + /// other var store variables are unaffected + pub fn float(&mut self) { + self.set_float_kind(Kind::Float); + } + + /// Casts all float-like variables in a var store sub-path to double-precision (Double kind). + /// + /// Only the variable in the path sub-tree are cast to double-precision: + /// other var store variables are unaffected + pub fn double(&mut self) { + self.set_float_kind(Kind::Double); + } + + pub fn add(&self, name: &str, tensor: Tensor, trainable: bool) -> Tensor { + let path = self.path(name); + let mut variables = self.var_store.variables_.lock().unwrap(); + let path = if variables.named_variables.contains_key(&path) { + format!("{}__{}", path, variables.named_variables.len()) + } else { + path + }; + let tensor = if trainable { tensor.set_requires_grad(true) } else { tensor }; + if trainable { + let var = Var { tensor: tensor.shallow_clone(), group: self.group }; + variables.trainable_variables.push(var); + }; + variables.named_variables.insert(path, tensor.shallow_clone()); + tensor + } + + fn get_or_add_with_lock( + &self, + name: &str, + tensor: Tensor, + trainable: bool, + mut variables: MutexGuard, + ) -> Tensor { + let path = self.path(name); + if let Some(var) = variables.named_variables.get(&path) { + return var.shallow_clone(); + } + + let tensor = if trainable { tensor.set_requires_grad(true) } else { tensor }; + if trainable { + let var = Var { tensor: tensor.shallow_clone(), group: self.group }; + variables.trainable_variables.push(var); + } + variables.named_variables.insert(path, tensor.shallow_clone()); + tensor + } + + /// Creates a new variable initialized with zeros. + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable will not be trainable so + /// gradients will not be tracked. + /// The variable uses a float tensor initialized with zeros. + pub fn f_zeros_no_train(&self, name: &str, dims: &[i64]) -> Result { + let z = Tensor::f_zeros(dims, (Kind::Float, self.device()))?; + Ok(self.add(name, z, false)) + } + + /// Creates a new variable initialized with ones. + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable will not be trainable so + /// gradients will not be tracked. + /// The variable uses a float tensor initialized with ones. + pub fn f_ones_no_train(&self, name: &str, dims: &[i64]) -> Result { + let o = Tensor::f_ones(dims, (Kind::Float, self.device()))?; + Ok(self.add(name, o, false)) + } + + /// Creates a new variable. + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable is trainable, its gradient + /// will be tracked. + /// The variable uses a float tensor initialized as per the + /// related argument. + pub fn f_var(&self, name: &str, dims: &[i64], init: Init) -> Result { + let v = super::f_init(init, dims, self.device(), self.kind())?; + Ok(self.add(name, v, true)) + } + + /// Creates a new variable initialized with zeros. + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable is trainable, its gradient + /// will be tracked. + /// The variable uses a float tensor initialized with zeros. + pub fn f_zeros(&self, name: &str, dims: &[i64]) -> Result { + self.f_var(name, dims, Init::Const(0.)) + } + + /// Creates a new variable initialized with ones. + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable is trainable, its gradient + /// will be tracked. + /// The variable uses a float tensor initialized with ones. + pub fn f_ones(&self, name: &str, dims: &[i64]) -> Result { + self.f_var(name, dims, Init::Const(1.)) + } + + /// Creates a new variable initialized randomly with normal distribution. + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable is trainable, its gradient + /// will be tracked. + /// The variable uses a float tensor initialized randomly using a + /// standard normal distribution. + pub fn f_randn_standard(&self, name: &str, dims: &[i64]) -> Result { + let init = Init::Randn { mean: 0., stdev: 1. }; + self.f_var(name, dims, init) + } + + /// Creates a new variable initialized randomly with normal distribution. + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable is trainable, its gradient + /// will be tracked. + /// The variable uses a float tensor initialized randomly using a + /// normal distribution with the specified mean and standard deviation. + pub fn f_randn( + &self, + name: &str, + dims: &[i64], + mean: f64, + stdev: f64, + ) -> Result { + self.f_var(name, dims, Init::Randn { mean, stdev }) + } + + /// Creates a new variable initialized randomly with uniform distribution. + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable is trainable, its gradient + /// will be tracked. + /// The variable uses a float tensor initialized randomly using a + /// uniform distribution between the specified bounds. + pub fn f_uniform( + &self, + name: &str, + dims: &[i64], + lo: f64, + up: f64, + ) -> Result { + self.f_var(name, dims, Init::Uniform { lo, up }) + } + + /// Creates a new variable initialized randomly with kaiming uniform. + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable is trainable, its gradient + /// will be tracked. + /// The variable uses a float tensor initialized randomly using a + /// uniform distribution which bounds follow Kaiming initialization. + pub fn f_kaiming_uniform(&self, name: &str, dims: &[i64]) -> Result { + self.f_var(name, dims, super::init::DEFAULT_KAIMING_UNIFORM) + } + + /// Creates a new variable initialized randomly with kaiming normal. + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable is trainable, its gradient + /// will be tracked. + /// The variable uses a float tensor initialized randomly using a + /// normal distribution which stdev follow Kaiming initialization. + pub fn f_kaiming_normal(&self, name: &str, dims: &[i64]) -> Result { + self.f_var(name, dims, super::init::DEFAULT_KAIMING_NORMAL) + } + + /// Creates a new variable initialized randomly with an orthogonal matrix + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable is trainable, its gradient + /// will be tracked. + /// The variable uses a float tensor initialized randomly with an orthogonal + /// matrix as described in *Exact solutions to the nonlinear dynamics + /// of learning in deep linear neural networks* - Saxe, A. et. al. (2013). + /// The input tensor must have at least 2 dimensions, and for tensors + /// with more than 2 dimensions the trailing dimensions are flattened. + pub fn f_orthogonal(&self, name: &str, dims: &[i64], gain: f64) -> Result { + self.f_var(name, dims, Init::Orthogonal { gain }) + } + + /// Creates a new variable initialized by copying an existing tensor. + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable is trainable, its gradient + /// will be tracked. + /// The variable uses a float tensor initialized by copying some + /// given tensor. + pub fn f_var_copy(&self, name: &str, t: &Tensor) -> Result { + let mut v = self.f_zeros(name, &t.size())?; + crate::no_grad(|| v.f_copy_(t))?; + Ok(v) + } + + /// Creates a new variable initialized with zeros. + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable will not be trainable so + /// gradients will not be tracked. + /// The variable uses a float tensor initialized with zeros. + pub fn zeros_no_train(&self, name: &str, dims: &[i64]) -> Tensor { + self.f_zeros_no_train(name, dims).unwrap() + } + + /// Creates a new variable initialized with ones. + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable will not be trainable so + /// gradients will not be tracked. + /// The variable uses a float tensor initialized with ones. + pub fn ones_no_train(&self, name: &str, dims: &[i64]) -> Tensor { + self.f_ones_no_train(name, dims).unwrap() + } + + /// Creates a new variable. + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable is trainable, its gradient + /// will be tracked. + /// The variable uses a float tensor initialized as per the + /// related argument. + pub fn var(&self, name: &str, dims: &[i64], init: Init) -> Tensor { + self.f_var(name, dims, init).unwrap() + } + + /// Creates a new variable initialized with zeros. + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable is trainable, its gradient + /// will be tracked. + /// The variable uses a float tensor initialized with zeros. + pub fn zeros(&self, name: &str, dims: &[i64]) -> Tensor { + self.f_zeros(name, dims).unwrap() + } + + /// Creates a new variable initialized with ones. + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable is trainable, its gradient + /// will be tracked. + /// The variable uses a float tensor initialized with ones. + pub fn ones(&self, name: &str, dims: &[i64]) -> Tensor { + self.f_ones(name, dims).unwrap() + } + + /// Creates a new variable initialized randomly with normal distribution. + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable is trainable, its gradient + /// will be tracked. + /// The variable uses a float tensor initialized randomly using a + /// standard normal distribution. + pub fn randn_standard(&self, name: &str, dims: &[i64]) -> Tensor { + self.f_randn_standard(name, dims).unwrap() + } + + /// Creates a new variable initialized randomly with normal distribution. + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable is trainable, its gradient + /// will be tracked. + /// The variable uses a float tensor initialized randomly using a + /// normal distribution with the specified mean and standard deviation. + pub fn randn(&self, name: &str, dims: &[i64], mean: f64, stdev: f64) -> Tensor { + self.f_randn(name, dims, mean, stdev).unwrap() + } + + /// Creates a new variable initialized randomly with uniform distribution. + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable is trainable, its gradient + /// will be tracked. + /// The variable uses a float tensor initialized randomly using a + /// uniform distribution between the specified bounds. + pub fn uniform(&self, name: &str, dims: &[i64], lo: f64, up: f64) -> Tensor { + self.f_uniform(name, dims, lo, up).unwrap() + } + + /// Creates a new variable initialized randomly with kaiming uniform. + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable is trainable, its gradient + /// will be tracked. + /// The variable uses a float tensor initialized randomly using a + /// uniform distribution which bounds follow Kaiming initialization. + pub fn kaiming_uniform(&self, name: &str, dims: &[i64]) -> Tensor { + self.f_kaiming_uniform(name, dims).unwrap() + } + + /// Creates a new variable initialized randomly with kaiming normal. + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable is trainable, its gradient + /// will be tracked. + /// The variable uses a float tensor initialized randomly using a + /// normal distribution which stdev follow Kaiming initialization. + pub fn kaiming_normal(&self, name: &str, dims: &[i64]) -> Tensor { + self.f_kaiming_normal(name, dims).unwrap() + } + + /// Creates a new variable initialized randomly with an orthogonal matrix + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable is trainable, its gradient + /// will be tracked. + /// The variable uses a float tensor initialized randomly with an orthogonal + /// matrix as described in *Exact solutions to the nonlinear dynamics + /// of learning in deep linear neural networks* - Saxe, A. et. al. (2013). + /// The input tensor must have at least 2 dimensions, and for tensors + /// with more than 2 dimensions the trailing dimensions are flattened. + pub fn orthogonal(&self, name: &str, dims: &[i64], gain: f64) -> Tensor { + self.f_orthogonal(name, dims, gain).unwrap() + } + + /// Creates a new variable initialized by copying an existing tensor. + /// + /// The new variable is named according to the name parameter and + /// has the specified shape. The variable is trainable, its gradient + /// will be tracked. + /// The variable uses a float tensor initialized by copying some + /// given tensor. + pub fn var_copy(&self, name: &str, t: &Tensor) -> Tensor { + self.f_var_copy(name, t).unwrap() + } + + /// Gets the tensor corresponding to a given name if present. + pub fn get(&self, name: &str) -> Option { + let path = self.path(name); + let variables = self.var_store.variables_.lock().unwrap(); + variables.named_variables.get(&path).map(|v| v.shallow_clone()) + } + + /// Gets the entry corresponding to a given name for in-place manipulation. + pub fn entry<'b>(&'b self, name: &'b str) -> Entry<'b> { + let variables = self.var_store.variables_.lock().unwrap(); + Entry { name, variables, path: self } + } +} + +impl Entry<'_> { + /// Returns the existing entry if, otherwise create a new variable. + /// + /// If this entry name matches the name of a variables stored in the + /// var store, the corresponding tensor is returned. Otherwise a new + /// variable is added to the var-store with the entry name and is + /// initialized according to the init parameter. + pub fn or_var(self, dims: &[i64], init: Init) -> Tensor { + let v = super::init(init, dims, self.path.device()); + self.path.get_or_add_with_lock(self.name, v, true, self.variables) + } + + /// Returns the existing entry if, otherwise create a new variable. + pub fn or_var_copy(self, tensor: &Tensor) -> Tensor { + let mut v = self.or_zeros(&tensor.size()); + crate::no_grad(|| v.copy_(tensor)); + v + } + + /// Returns the existing entry if, otherwise create a new variable. + pub fn or_kaiming_uniform(self, dims: &[i64]) -> Tensor { + self.or_var(dims, super::init::DEFAULT_KAIMING_NORMAL) + } + + /// Returns the existing entry if, otherwise create a new variable. + pub fn or_kaiming_normal(self, dims: &[i64]) -> Tensor { + self.or_var(dims, super::init::DEFAULT_KAIMING_NORMAL) + } + + /// Returns the existing entry if, otherwise create a new variable. + pub fn or_orthogonal(self, dims: &[i64], gain: f64) -> Tensor { + self.or_var(dims, Init::Orthogonal { gain }) + } + + /// Returns the existing entry if, otherwise create a new variable. + pub fn or_ones(self, dims: &[i64]) -> Tensor { + self.or_var(dims, Init::Const(1.)) + } + + /// Returns the existing entry if, otherwise create a new variable. + pub fn or_ones_no_train(self, dims: &[i64]) -> Tensor { + let o = Tensor::ones(dims, (Kind::Float, self.path.device())); + self.path.get_or_add_with_lock(self.name, o, true, self.variables) + } + + /// Returns the existing entry if, otherwise create a new variable. + pub fn or_randn(self, dims: &[i64], mean: f64, stdev: f64) -> Tensor { + self.or_var(dims, Init::Randn { mean, stdev }) + } + + /// Returns the existing entry if, otherwise create a new variable. + pub fn or_randn_standard(self, dims: &[i64]) -> Tensor { + let init = Init::Randn { mean: 0., stdev: 1. }; + self.or_var(dims, init) + } + + /// Returns the existing entry if, otherwise create a new variable. + pub fn or_uniform(self, dims: &[i64], lo: f64, up: f64) -> Tensor { + self.or_var(dims, Init::Uniform { lo, up }) + } + + /// Returns the existing entry if, otherwise create a new variable. + pub fn or_zeros(self, dims: &[i64]) -> Tensor { + self.or_var(dims, Init::Const(0.)) + } + + /// Returns the existing entry if, otherwise create a new variable. + pub fn or_zeros_no_train(self, dims: &[i64]) -> Tensor { + let z = Tensor::zeros(dims, (Kind::Float, self.path.device())); + self.path.get_or_add_with_lock(self.name, z, true, self.variables) + } +} + +impl<'a, T> Div for &mut Path<'a> +where + T: std::string::ToString, +{ + type Output = Path<'a>; + + fn div(self, rhs: T) -> Self::Output { + self.sub(rhs.to_string()) + } +} + +impl<'a, T> Div for &Path<'a> +where + T: std::string::ToString, +{ + type Output = Path<'a>; + + fn div(self, rhs: T) -> Self::Output { + self.sub(rhs.to_string()) + } +} + +impl<'a, T> Div for Path<'a> +where + T: std::string::ToString, +{ + type Output = Path<'a>; + + fn div(self, rhs: T) -> Self::Output { + self.sub(rhs.to_string()) + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/tensor/convert.rs b/v2/dependency-patches/tch-0.24.0/src/tensor/convert.rs new file mode 100644 index 0000000000..a0b8c3101f --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/tensor/convert.rs @@ -0,0 +1,182 @@ +//! Implement conversion traits for tensors +use super::Tensor; +use crate::{kind::Element, TchError}; +use half::{bf16, f16}; +use std::convert::{TryFrom, TryInto}; + +impl TryFrom<&Tensor> for Vec { + type Error = TchError; + fn try_from(tensor: &Tensor) -> Result { + let size = tensor.size(); + if size.len() != 1 { + Err(TchError::Convert(format!( + "Attempting to convert a Tensor with {} dimensions to flat vector", + size.len() + )))?; + } + let numel = size[0] as usize; + let mut vec = vec![T::ZERO; numel]; + tensor.f_to_kind(T::KIND)?.f_copy_data(&mut vec, numel)?; + Ok(vec) + } +} + +impl TryFrom<&Tensor> for Vec> { + type Error = TchError; + fn try_from(tensor: &Tensor) -> Result { + let (s1, s2) = tensor.size2()?; + let s1 = s1 as usize; + let s2 = s2 as usize; + let num_elem = s1 * s2; + // TODO: Try to remove this intermediary copy. + let mut all_elems = vec![T::ZERO; num_elem]; + tensor.f_to_kind(T::KIND)?.f_copy_data(&mut all_elems, num_elem)?; + let out = (0..s1).map(|i1| (0..s2).map(|i2| all_elems[i1 * s2 + i2]).collect()).collect(); + Ok(out) + } +} + +impl TryFrom<&Tensor> for Vec>> { + type Error = TchError; + fn try_from(tensor: &Tensor) -> Result { + let (s1, s2, s3) = tensor.size3()?; + let s1 = s1 as usize; + let s2 = s2 as usize; + let s3 = s3 as usize; + let num_elem = s1 * s2 * s3; + // TODO: Try to remove this intermediary copy. + let mut all_elems = vec![T::ZERO; num_elem]; + tensor.f_to_kind(T::KIND)?.f_copy_data(&mut all_elems, num_elem)?; + let out = (0..s1) + .map(|i1| { + (0..s2) + .map(|i2| (0..s3).map(|i3| all_elems[i1 * s2 * s3 + i2 * s3 + i3]).collect()) + .collect() + }) + .collect(); + Ok(out) + } +} + +impl TryFrom for Vec { + type Error = TchError; + fn try_from(tensor: Tensor) -> Result { + Vec::::try_from(&tensor) + } +} + +impl TryFrom for Vec> { + type Error = TchError; + fn try_from(tensor: Tensor) -> Result { + Vec::>::try_from(&tensor) + } +} + +impl TryFrom for Vec>> { + type Error = TchError; + fn try_from(tensor: Tensor) -> Result { + Vec::>>::try_from(&tensor) + } +} + +macro_rules! from_tensor { + ($typ:ident) => { + impl TryFrom<&Tensor> for $typ { + type Error = TchError; + + fn try_from(tensor: &Tensor) -> Result { + let numel = tensor.numel(); + if numel != 1 { + return Err(TchError::Convert(format!( + "expected exactly one element, got {}", + numel + ))); + } + let mut vec = [$typ::ZERO; 1]; + tensor + .f_to_device(crate::Device::Cpu)? + .f_to_kind($typ::KIND)? + .f_copy_data(&mut vec, numel)?; + Ok(vec[0]) + } + } + + impl TryFrom for $typ { + type Error = TchError; + + fn try_from(tensor: Tensor) -> Result { + $typ::try_from(&tensor) + } + } + }; +} + +from_tensor!(f64); +from_tensor!(f32); +from_tensor!(f16); +from_tensor!(i64); +from_tensor!(i32); +from_tensor!(i16); +from_tensor!(i8); +from_tensor!(u8); +from_tensor!(bool); +from_tensor!(bf16); + +impl TryInto> for &Tensor { + type Error = TchError; + + fn try_into(self) -> Result, Self::Error> { + let num_elem = self.numel(); + let mut vec = vec![T::ZERO; num_elem]; + self.f_to_kind(T::KIND)?.f_copy_data(&mut vec, num_elem)?; + let shape: Vec = self.size().iter().map(|s| *s as usize).collect(); + Ok(ndarray::ArrayD::from_shape_vec(ndarray::IxDyn(&shape), vec)?) + } +} + +impl TryFrom<&ndarray::ArrayBase> for Tensor +where + T: ndarray::Data, + T::Elem: Element, + D: ndarray::Dimension, +{ + type Error = TchError; + + fn try_from(value: &ndarray::ArrayBase) -> Result { + let slice = value + .as_slice() + .ok_or_else(|| TchError::Convert("cannot convert to slice".to_string()))?; + let tn = Self::f_from_slice(slice)?; + let shape: Vec = value.shape().iter().map(|s| *s as i64).collect(); + tn.f_reshape(shape) + } +} + +impl TryFrom> for Tensor +where + T: ndarray::Data, + T::Elem: Element, + D: ndarray::Dimension, +{ + type Error = TchError; + + fn try_from(value: ndarray::ArrayBase) -> Result { + Self::try_from(&value) + } +} + +impl TryFrom<&Vec> for Tensor { + type Error = TchError; + + fn try_from(value: &Vec) -> Result { + Self::f_from_slice(value.as_slice()) + } +} + +impl TryFrom> for Tensor { + type Error = TchError; + + fn try_from(value: Vec) -> Result { + Self::try_from(&value) + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/tensor/display.rs b/v2/dependency-patches/tch-0.24.0/src/tensor/display.rs new file mode 100644 index 0000000000..572d68b0b1 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/tensor/display.rs @@ -0,0 +1,434 @@ +/// Pretty printing of tensors +/// This implementation should be in line with the PyTorch version. +/// https://github.com/pytorch/pytorch/blob/7b419e8513a024e172eae767e24ec1b849976b13/torch/_tensor_str.py +use crate::{Kind, Tensor}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BasicKind { + Float, + Int, + Bool, + Complex, +} + +impl BasicKind { + fn for_tensor(t: &Tensor) -> BasicKind { + match t.f_kind() { + Err(_) => BasicKind::Complex, + Ok(kind) => match kind { + Kind::Int | Kind::Int8 | Kind::Uint8 | Kind::Int16 | Kind::Int64 => BasicKind::Int, + Kind::BFloat16 + | Kind::QInt8 + | Kind::QUInt8 + | Kind::QInt32 + | Kind::Half + | Kind::Float + | Kind::Double + | Kind::Float8e5m2 + | Kind::Float8e4m3fn + | Kind::Float8e5m2fnuz + | Kind::Float8e4m3fnuz => BasicKind::Float, + Kind::Bool => BasicKind::Bool, + Kind::ComplexHalf | Kind::ComplexFloat | Kind::ComplexDouble => BasicKind::Complex, + }, + } + } + + fn _is_floating_point(&self) -> bool { + match self { + BasicKind::Float => true, + BasicKind::Bool | BasicKind::Int | BasicKind::Complex => false, + } + } +} + +impl std::fmt::Debug for Tensor { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + if self.defined() { + match self.f_kind() { + Err(err) => write!(f, "Tensor[{:?}, {:?}]", self.size(), err), + Ok(kind) => { + let (is_int, is_float) = match kind { + Kind::Int | Kind::Int8 | Kind::Uint8 | Kind::Int16 | Kind::Int64 => { + (true, false) + } + Kind::BFloat16 + | Kind::QInt8 + | Kind::QUInt8 + | Kind::QInt32 + | Kind::Half + | Kind::Float + | Kind::Double + | Kind::Float8e5m2 + | Kind::Float8e4m3fn + | Kind::Float8e5m2fnuz + | Kind::Float8e4m3fnuz => (false, true), + Kind::Bool + | Kind::ComplexHalf + | Kind::ComplexFloat + | Kind::ComplexDouble => (false, false), + }; + match (self.size().as_slice(), is_int, is_float) { + ([], true, false) => write!(f, "[{}]", i64::try_from(self).unwrap()), + ([s], true, false) if *s < 10 => { + write!(f, "{:?}", Vec::::try_from(self).unwrap()) + } + ([], false, true) => write!(f, "[{}]", f64::try_from(self).unwrap()), + ([s], false, true) if *s < 10 => { + write!(f, "{:?}", Vec::::try_from(self).unwrap()) + } + _ => write!(f, "Tensor[{:?}, {:?}]", self.size(), kind), + } + } + } + } else { + write!(f, "Tensor[Undefined]") + } + } +} + +/// Options for Tensor pretty printing +pub struct PrinterOptions { + precision: usize, + threshold: usize, + edge_items: usize, + line_width: usize, + sci_mode: Option, +} + +lazy_static! { + static ref PRINT_OPTS: std::sync::Mutex = + std::sync::Mutex::new(Default::default()); +} + +pub fn set_print_options(options: PrinterOptions) { + *PRINT_OPTS.lock().unwrap() = options +} + +pub fn set_print_options_default() { + *PRINT_OPTS.lock().unwrap() = Default::default() +} + +pub fn set_print_options_short() { + *PRINT_OPTS.lock().unwrap() = PrinterOptions { + precision: 2, + threshold: 1000, + edge_items: 2, + line_width: 80, + sci_mode: None, + } +} + +pub fn set_print_options_full() { + *PRINT_OPTS.lock().unwrap() = PrinterOptions { + precision: 4, + threshold: usize::MAX, + edge_items: 3, + line_width: 80, + sci_mode: None, + } +} + +impl Default for PrinterOptions { + fn default() -> Self { + Self { precision: 4, threshold: 1000, edge_items: 3, line_width: 80, sci_mode: None } + } +} + +trait TensorFormatter { + type Elem; + + fn fmt(&self, v: Self::Elem, max_w: usize, f: &mut T) -> std::fmt::Result; + + fn value(tensor: &Tensor) -> Self::Elem; + + fn values(tensor: &Tensor) -> Vec; + + fn max_width(&self, to_display: &Tensor) -> usize { + let mut max_width = 1; + for v in Self::values(to_display) { + let mut fmt_size = FmtSize::new(); + let _res = self.fmt(v, 1, &mut fmt_size); + max_width = usize::max(max_width, fmt_size.final_size()) + } + max_width + } + + fn write_newline_indent(i: usize, f: &mut std::fmt::Formatter) -> std::fmt::Result { + writeln!(f)?; + for _ in 0..i { + write!(f, " ")? + } + Ok(()) + } + + fn fmt_tensor( + &self, + t: &Tensor, + indent: usize, + max_w: usize, + summarize: bool, + po: &PrinterOptions, + f: &mut std::fmt::Formatter, + ) -> std::fmt::Result { + let size = t.size(); + let edge_items = po.edge_items as i64; + write!(f, "[")?; + match size.as_slice() { + [] => self.fmt(Self::value(t), max_w, f)?, + [v] if summarize && *v > 2 * edge_items => { + for v in Self::values(&t.slice(0, None, Some(edge_items), 1)).into_iter() { + self.fmt(v, max_w, f)?; + write!(f, ", ")?; + } + write!(f, "...")?; + for v in Self::values(&t.slice(0, Some(-edge_items), None, 1)).into_iter() { + write!(f, ", ")?; + self.fmt(v, max_w, f)? + } + } + [_] => { + let elements_per_line = usize::max(1, po.line_width / (max_w + 2)); + for (i, v) in Self::values(t).into_iter().enumerate() { + if i > 0 { + if i % elements_per_line == 0 { + write!(f, ",")?; + Self::write_newline_indent(indent, f)? + } else { + write!(f, ", ")?; + } + } + self.fmt(v, max_w, f)? + } + } + _ => { + if summarize && size[0] > 2 * edge_items { + for i in 0..edge_items { + self.fmt_tensor(&t.get(i), indent + 1, max_w, summarize, po, f)?; + write!(f, ",")?; + Self::write_newline_indent(indent, f)? + } + write!(f, "...")?; + Self::write_newline_indent(indent, f)?; + for i in size[0] - edge_items..size[0] { + self.fmt_tensor(&t.get(i), indent + 1, max_w, summarize, po, f)?; + if i + 1 != size[0] { + write!(f, ",")?; + Self::write_newline_indent(indent, f)? + } + } + } else { + for i in 0..size[0] { + self.fmt_tensor(&t.get(i), indent + 1, max_w, summarize, po, f)?; + if i + 1 != size[0] { + write!(f, ",")?; + Self::write_newline_indent(indent, f)? + } + } + } + } + } + write!(f, "]")?; + Ok(()) + } +} + +struct FloatFormatter { + int_mode: bool, + sci_mode: bool, + precision: usize, +} + +struct FmtSize { + current_size: usize, +} + +impl FmtSize { + fn new() -> Self { + Self { current_size: 0 } + } + + fn final_size(self) -> usize { + self.current_size + } +} + +impl std::fmt::Write for FmtSize { + fn write_str(&mut self, s: &str) -> std::fmt::Result { + self.current_size += s.len(); + Ok(()) + } +} + +impl FloatFormatter { + fn new(t: &Tensor, po: &PrinterOptions) -> Self { + let mut int_mode = true; + let mut sci_mode = false; + + let _guard = crate::no_grad_guard(); + let t = t.to_device(crate::Device::Cpu); + + // Rather than containing all values, this should only include + // values that end up being displayed according to [threshold]. + let nonzero_finite_vals = { + let t = t.reshape([-1]); + t.masked_select(&t.isfinite().logical_and(&t.ne(0.))) + }; + + let values = Vec::::try_from(&nonzero_finite_vals).unwrap(); + if nonzero_finite_vals.numel() > 0 { + let nonzero_finite_abs = nonzero_finite_vals.abs(); + let nonzero_finite_min = nonzero_finite_abs.min().double_value(&[]); + let nonzero_finite_max = nonzero_finite_abs.max().double_value(&[]); + + for &value in values.iter() { + if value.ceil() != value { + int_mode = false; + break; + } + } + + sci_mode = nonzero_finite_max / nonzero_finite_min > 1000. + || nonzero_finite_max > 1e8 + || nonzero_finite_min < 1e-4 + } + + match po.sci_mode { + None => {} + Some(v) => sci_mode = v, + } + Self { int_mode, sci_mode, precision: po.precision } + } +} + +impl TensorFormatter for FloatFormatter { + type Elem = f64; + + fn fmt(&self, v: Self::Elem, max_w: usize, f: &mut T) -> std::fmt::Result { + if self.sci_mode { + write!(f, "{v:width$.prec$e}", v = v, width = max_w, prec = self.precision) + } else if self.int_mode { + if v.is_finite() { + write!(f, "{v:width$.0}.", v = v, width = max_w - 1) + } else { + write!(f, "{v:max_w$.0}") + } + } else { + write!(f, "{v:width$.prec$}", v = v, width = max_w, prec = self.precision) + } + } + + fn value(tensor: &Tensor) -> Self::Elem { + tensor.double_value(&[]) + } + + fn values(tensor: &Tensor) -> Vec { + Vec::::try_from(tensor.reshape(-1)).unwrap() + } +} + +struct IntFormatter; + +impl TensorFormatter for IntFormatter { + type Elem = i64; + + fn fmt(&self, v: Self::Elem, max_w: usize, f: &mut T) -> std::fmt::Result { + write!(f, "{v:max_w$}") + } + + fn value(tensor: &Tensor) -> Self::Elem { + tensor.int64_value(&[]) + } + + fn values(tensor: &Tensor) -> Vec { + Vec::::try_from(tensor.reshape(-1)).unwrap() + } +} + +struct BoolFormatter; + +impl TensorFormatter for BoolFormatter { + type Elem = bool; + + fn fmt(&self, v: Self::Elem, max_w: usize, f: &mut T) -> std::fmt::Result { + let v = if v { "true" } else { "false" }; + write!(f, "{v:max_w$}") + } + + fn value(tensor: &Tensor) -> Self::Elem { + tensor.int64_value(&[]) != 0 + } + + fn values(tensor: &Tensor) -> Vec { + Vec::::try_from(tensor.reshape(-1)).unwrap() + } +} + +fn get_summarized_data(t: &Tensor, edge_items: i64) -> Tensor { + let size = t.size(); + if size.is_empty() { + t.shallow_clone() + } else if size.len() == 1 { + if size[0] > 2 * edge_items { + Tensor::cat( + &[t.slice(0, None, Some(edge_items), 1), t.slice(0, Some(-edge_items), None, 1)], + 0, + ) + } else { + t.shallow_clone() + } + } else if size[0] > 2 * edge_items { + let mut vs: Vec<_> = + (0..edge_items).map(|i| get_summarized_data(&t.get(i), edge_items)).collect(); + for i in (size[0] - edge_items)..size[0] { + vs.push(get_summarized_data(&t.get(i), edge_items)) + } + Tensor::stack(&vs, 0) + } else { + let vs: Vec<_> = (0..size[0]).map(|i| get_summarized_data(&t.get(i), edge_items)).collect(); + Tensor::stack(&vs, 0) + } +} + +impl std::fmt::Display for Tensor { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + if self.defined() { + let po = PRINT_OPTS.lock().unwrap(); + let summarize = self.numel() > po.threshold; + let basic_kind = BasicKind::for_tensor(self); + let to_display = if summarize { + get_summarized_data(self, po.edge_items as i64) + } else { + self.shallow_clone() + }; + match basic_kind { + BasicKind::Int => { + let tf = IntFormatter; + let max_w = tf.max_width(&to_display); + tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; + writeln!(f)?; + } + BasicKind::Float => { + let tf = FloatFormatter::new(&to_display, &po); + let max_w = tf.max_width(&to_display); + tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; + writeln!(f)?; + } + BasicKind::Bool => { + let tf = BoolFormatter; + let max_w = tf.max_width(&to_display); + tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; + writeln!(f)?; + } + BasicKind::Complex => {} + }; + let kind = match self.f_kind() { + Ok(kind) => format!("{kind:?}"), + Err(err) => format!("{err:?}"), + }; + write!(f, "Tensor[{:?}, {}]", self.size(), kind) + } else { + write!(f, "Tensor[Undefined]") + } + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/tensor/index.rs b/v2/dependency-patches/tch-0.24.0/src/tensor/index.rs new file mode 100644 index 0000000000..2bed77724d --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/tensor/index.rs @@ -0,0 +1,372 @@ +//! Indexing operations +//! +//! This module defines the `i` indexing operation. This can be used in various +//! scenarios. +//! +//! Using an integer index returns the slice obtained by selecting elements with +//! the specified index. Negative values can be used for the index, and `..` can +//! be used to get all the indexes from a given dimension. +//! +//! ```ignore +//! use crate::tch::{IndexOp, Tensor}; +//! let tensor = Tensor::from_slice(&[1, 2, 3, 4, 5, 6]).view((2, 3)); +//! let t = tensor.i(1); +//! let t = tensor.i((.., -2)); +//! ``` +//! +//! Indexes like `1..`, `..1`, or `1..2`, can be used to narrow a dimension. +//! +//! ```ignore +//! use crate::tch::{IndexOp, Tensor}; +//! let tensor = Tensor::from_slice(&[1, 2, 3, 4, 5, 6]).view((2, 3)); +//! let t = tensor.i((.., 1..)); +//! assert_eq!(t.size(), [2, 2]); +//! assert_eq!(Vec::::from(t.contiguous().view(-1)), [2, 3, 5, 6]); +//! let t = tensor.i((..1, ..)); +//! assert_eq!(t.size(), [1, 3]); +//! assert_eq!(Vec::::from(t.contiguous().view(-1)), [1, 2, 3]); +//! let t = tensor.i((.., 1..2)); +//! assert_eq!(t.size(), [2, 1]); +//! assert_eq!(Vec::::from(t.contiguous().view(-1)), [2, 5]); +//! let t = tensor.i((.., 1..=2)); +//! assert_eq!(t.size(), [2, 2]); +//! assert_eq!(Vec::::from(t.contiguous().view(-1)), [2, 3, 5, 6]); +//! ``` +//! +//! The `NewAxis` index can be used to insert a dimension. +//! +//! ```ignore +//! use crate::tch::{IndexOp, NewAxis, Tensor}; +//! let tensor = Tensor::from_slice(&[1, 2, 3, 4, 5, 6]).view((2, 3)); +//! let t = tensor.i((NewAxis,)); +//! assert_eq!(t.size(), &[1, 2, 3]); +//! let t = tensor.i((.., .., NewAxis)); +//! assert_eq!(t.size(), &[2, 3, 1]); +//! ``` +//! +//! Unlike NumPy, the `i` operation does not support advanced indexing. +//! The result can be different from NumPy with same set of arguments. +//! For example, `tensor.i(..1, vec![0, 3], vec![2, 1, 3])` does narrowing +//! on first dimension, and index selection on second and third dimensions. +//! The analogous NumPy indexing `array[:1, [0, 3], [2, 1, 3]]` throws +//! shape mismatch error due to advanced indexing rule. Another distinction +//! is that `i` guarantees the input and result tensor shares the same +//! underlying storage, while NumPy may copy the tensor in certain scenarios. +use crate::{Result, TchError, Tensor}; +use std::ops::{ + Bound, Range, RangeBounds, RangeFrom, RangeFull, RangeInclusive, RangeTo, RangeToInclusive, +}; + +#[derive(Debug, PartialEq, Eq)] +pub struct NewAxis; + +#[derive(Debug, PartialEq)] +pub enum TensorIndexer { + Select(i64), + Narrow(Bound, Bound), + IndexSelect(Tensor), + InsertNewAxis, +} + +impl From for TensorIndexer { + fn from(_index: NewAxis) -> Self { + TensorIndexer::InsertNewAxis + } +} + +impl From for TensorIndexer { + fn from(index: i64) -> Self { + TensorIndexer::Select(index) + } +} + +impl From<&[i64]> for TensorIndexer { + fn from(index: &[i64]) -> Self { + let tensor = index.into(); + TensorIndexer::IndexSelect(tensor) + } +} + +impl From> for TensorIndexer { + fn from(index: Vec) -> Self { + let tensor = Tensor::from_slice(&index); + TensorIndexer::IndexSelect(tensor) + } +} + +impl From<&Tensor> for TensorIndexer { + fn from(tensor: &Tensor) -> Self { + TensorIndexer::IndexSelect(tensor.shallow_clone()) + } +} + +macro_rules! impl_from_range { + ($range_type:ty) => { + impl From<$range_type> for TensorIndexer { + fn from(range: $range_type) -> Self { + use std::ops::Bound::*; + + let start = match range.start_bound() { + Included(idx) => Included(*idx), + Excluded(idx) => Excluded(*idx), + Unbounded => Unbounded, + }; + + let end = match range.end_bound() { + Included(idx) => Included(*idx), + Excluded(idx) => Excluded(*idx), + Unbounded => Unbounded, + }; + + TensorIndexer::Narrow(start, end) + } + } + }; +} + +impl_from_range!(Range); +impl_from_range!(RangeFrom); +impl_from_range!(RangeFull); +impl_from_range!(RangeInclusive); +impl_from_range!(RangeTo); +impl_from_range!(RangeToInclusive); + +pub trait IndexOp { + fn i(&self, index: T) -> Tensor; + fn f_i(&self, index: T) -> Result; +} + +impl IndexOp for Tensor +where + A: Into, +{ + fn i(&self, index: A) -> Tensor { + self.f_i(index).unwrap() + } + + fn f_i(&self, index: A) -> Result { + self.f_indexer(&[index.into()]) + } +} + +impl IndexOp<(A,)> for Tensor +where + A: Into, +{ + fn i(&self, index: (A,)) -> Tensor { + self.f_i(index).unwrap() + } + + fn f_i(&self, index: (A,)) -> Result { + let idx_a = index.0.into(); + self.f_indexer(&[idx_a]) + } +} + +impl IndexOp<(A, B)> for Tensor +where + A: Into, + B: Into, +{ + fn i(&self, index: (A, B)) -> Tensor { + self.f_i(index).unwrap() + } + + fn f_i(&self, index: (A, B)) -> Result { + let idx_a = index.0.into(); + let idx_b = index.1.into(); + self.f_indexer(&[idx_a, idx_b]) + } +} + +impl IndexOp<(A, B, C)> for Tensor +where + A: Into, + B: Into, + C: Into, +{ + fn i(&self, index: (A, B, C)) -> Tensor { + self.f_i(index).unwrap() + } + + fn f_i(&self, index: (A, B, C)) -> Result { + let idx_a = index.0.into(); + let idx_b = index.1.into(); + let idx_c = index.2.into(); + self.f_indexer(&[idx_a, idx_b, idx_c]) + } +} + +impl IndexOp<(A, B, C, D)> for Tensor +where + A: Into, + B: Into, + C: Into, + D: Into, +{ + fn i(&self, index: (A, B, C, D)) -> Tensor { + self.f_i(index).unwrap() + } + + fn f_i(&self, index: (A, B, C, D)) -> Result { + let idx_a = index.0.into(); + let idx_b = index.1.into(); + let idx_c = index.2.into(); + let idx_d = index.3.into(); + self.f_indexer(&[idx_a, idx_b, idx_c, idx_d]) + } +} + +impl IndexOp<(A, B, C, D, E)> for Tensor +where + A: Into, + B: Into, + C: Into, + D: Into, + E: Into, +{ + fn i(&self, index: (A, B, C, D, E)) -> Tensor { + self.f_i(index).unwrap() + } + + fn f_i(&self, index: (A, B, C, D, E)) -> Result { + let idx_a = index.0.into(); + let idx_b = index.1.into(); + let idx_c = index.2.into(); + let idx_d = index.3.into(); + let idx_e = index.4.into(); + self.f_indexer(&[idx_a, idx_b, idx_c, idx_d, idx_e]) + } +} + +impl IndexOp<(A, B, C, D, E, F)> for Tensor +where + A: Into, + B: Into, + C: Into, + D: Into, + E: Into, + F: Into, +{ + fn i(&self, index: (A, B, C, D, E, F)) -> Tensor { + self.f_i(index).unwrap() + } + + fn f_i(&self, index: (A, B, C, D, E, F)) -> Result { + let idx_a = index.0.into(); + let idx_b = index.1.into(); + let idx_c = index.2.into(); + let idx_d = index.3.into(); + let idx_e = index.4.into(); + let idx_f = index.5.into(); + self.f_indexer(&[idx_a, idx_b, idx_c, idx_d, idx_e, idx_f]) + } +} + +impl IndexOp<(A, B, C, D, E, F, G)> for Tensor +where + A: Into, + B: Into, + C: Into, + D: Into, + E: Into, + F: Into, + G: Into, +{ + fn i(&self, index: (A, B, C, D, E, F, G)) -> Tensor { + self.f_i(index).unwrap() + } + + fn f_i(&self, index: (A, B, C, D, E, F, G)) -> Result { + let idx_a = index.0.into(); + let idx_b = index.1.into(); + let idx_c = index.2.into(); + let idx_d = index.3.into(); + let idx_e = index.4.into(); + let idx_f = index.5.into(); + let idx_g = index.6.into(); + self.f_indexer(&[idx_a, idx_b, idx_c, idx_d, idx_e, idx_f, idx_g]) + } +} + +impl Tensor { + fn f_indexer(&self, index_spec: &[TensorIndexer]) -> Result { + use std::ops::Bound::*; + use TensorIndexer::*; + + // Make sure n. non-newaxis does not exceed n. of dimensions + let n_newaxis = index_spec.iter().filter(|spec| *spec == &InsertNewAxis).count(); + + if index_spec.len() > self.size().len() + n_newaxis { + return Err(TchError::Shape(format!( + "too many indices for tensor of dimension {}", + self.size().len() + ))); + } + + // Make sure tensors conform the format + for spec in index_spec.iter() { + use super::Kind::*; + if let IndexSelect(tensor) = spec { + if tensor.size().len() != 1 { + return Err(TchError::Shape( + "Multi-dimensional tensor is not supported for indexing".to_string(), + )); + } + match tensor.f_kind()? { + Int64 => {} + Int16 => {} + Int8 => {} + Int => {} + _ => { + return Err(TchError::Kind(format!("the kind of tensors used as indices must be one of {Int64:?}, {Int16:?}, {Int8:?}, {Int:?}"))); + } + } + } + } + + // Apply indexing from left to right + let mut curr_tensor = self.shallow_clone(); + let mut curr_idx: i64 = 0; + + for spec in index_spec.iter() { + let (next_tensor, next_idx) = match spec { + InsertNewAxis => (curr_tensor.unsqueeze(curr_idx), curr_idx + 1), + Select(index) => ( + curr_tensor.select(curr_idx, *index), + curr_idx, // not advanced because select() squeezes dimension + ), + Narrow(start, end) => { + if let Some((start, length)) = match (start, end) { + (Unbounded, Unbounded) => None, + (Included(start), Unbounded) => { + let dim_len = curr_tensor.size()[curr_idx as usize]; + Some((*start, dim_len - *start)) + } + (Excluded(start), Unbounded) => { + let dim_len = curr_tensor.size()[curr_idx as usize]; + Some((*start + 1, dim_len - *start - 1)) + } + (Unbounded, Included(end)) => Some((0, *end + 1)), + (Unbounded, Excluded(end)) => Some((0, *end)), + (Included(start), Included(end)) => Some((*start, *end - *start + 1)), + (Included(start), Excluded(end)) => Some((*start, *end - *start)), + (Excluded(start), Included(end)) => Some((*start + 1, *end - *start)), + (Excluded(start), Excluded(end)) => Some((*start + 1, *end - *start - 1)), + } { + (curr_tensor.f_narrow(curr_idx, start, length.max(0))?, curr_idx + 1) + } else { + (curr_tensor, curr_idx + 1) + } + } + IndexSelect(index_tensor) => { + let index_tensor = index_tensor.to_device(curr_tensor.device()); + (curr_tensor.index_select(curr_idx, &index_tensor), curr_idx + 1) + } + }; + curr_tensor = next_tensor; + curr_idx = next_idx; + } + Ok(curr_tensor) + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/tensor/iter.rs b/v2/dependency-patches/tch-0.24.0/src/tensor/iter.rs new file mode 100644 index 0000000000..bf0efdd8bd --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/tensor/iter.rs @@ -0,0 +1,62 @@ +use super::Tensor; +use crate::TchError; + +pub struct Iter { + index: i64, + len: i64, + content: Tensor, + phantom: std::marker::PhantomData, +} + +impl Tensor { + pub fn iter(&self) -> Result, TchError> { + Ok(Iter { + index: 0, + len: self.size1()?, + content: self.shallow_clone(), + phantom: std::marker::PhantomData, + }) + } +} + +impl std::iter::Iterator for Iter { + type Item = i64; + fn next(&mut self) -> Option { + if self.index >= self.len { + return None; + } + let v = self.content.int64_value(&[self.index]); + self.index += 1; + Some(v) + } +} + +impl std::iter::Iterator for Iter { + type Item = f64; + fn next(&mut self) -> Option { + if self.index >= self.len { + return None; + } + let v = self.content.double_value(&[self.index]); + self.index += 1; + Some(v) + } +} + +impl std::iter::Sum for Tensor { + fn sum>(mut iter: I) -> Tensor { + match iter.next() { + None => Tensor::from(0.), + Some(t) => iter.fold(t, |acc, x| x + acc), + } + } +} + +impl<'a> std::iter::Sum<&'a Tensor> for Tensor { + fn sum>(mut iter: I) -> Tensor { + match iter.next() { + None => Tensor::from(0.), + Some(t) => iter.fold(t.shallow_clone(), |acc, x| x + acc), + } + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/tensor/mod.rs b/v2/dependency-patches/tch-0.24.0/src/tensor/mod.rs new file mode 100644 index 0000000000..839e9d18a1 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/tensor/mod.rs @@ -0,0 +1,253 @@ +//! A Torch tensor. +use crate::{Device, Kind, TchError}; + +mod convert; +pub mod display; +pub mod index; +mod iter; +mod npy; +mod ops; +mod safetensors; + +pub use super::wrappers::tensor::{ + autocast, no_grad, no_grad_guard, with_grad, NoGradGuard, Reduction, Tensor, +}; +pub use index::{IndexOp, NewAxis, TensorIndexer}; + +pub trait Shape { + fn to_shape(&self) -> Box<[i64]>; +} + +macro_rules! impl_shape { + ($v:expr) => { + impl Shape for [i64; $v] { + fn to_shape(&self) -> Box<[i64]> { + Box::new(*self) + } + } + }; +} + +impl_shape!(0); +impl_shape!(1); +impl_shape!(2); +impl_shape!(3); +impl_shape!(4); +impl_shape!(5); +impl_shape!(6); + +impl Shape for () { + fn to_shape(&self) -> Box<[i64]> { + Box::new([]) + } +} + +impl Shape for &[i64] { + fn to_shape(&self) -> Box<[i64]> { + (*self).into() + } +} + +impl Shape for i64 { + fn to_shape(&self) -> Box<[i64]> { + Box::new([*self]) + } +} + +impl Shape for usize { + fn to_shape(&self) -> Box<[i64]> { + Box::new([*self as i64]) + } +} + +impl Shape for i32 { + fn to_shape(&self) -> Box<[i64]> { + Box::new([i64::from(*self)]) + } +} + +impl Shape for (i64,) { + fn to_shape(&self) -> Box<[i64]> { + Box::new([self.0]) + } +} + +impl Shape for (i64, i64) { + fn to_shape(&self) -> Box<[i64]> { + Box::new([self.0, self.1]) + } +} + +impl Shape for (i64, i64, i64) { + fn to_shape(&self) -> Box<[i64]> { + Box::new([self.0, self.1, self.2]) + } +} + +impl Shape for (i64, i64, i64, i64) { + fn to_shape(&self) -> Box<[i64]> { + Box::new([self.0, self.1, self.2, self.3]) + } +} + +impl Tensor { + pub fn f_view(&self, s: T) -> Result { + self.f_view_(&s.to_shape()) + } + + pub fn view(&self, s: T) -> Tensor { + self.view_(&s.to_shape()) + } + + pub fn f_zero_pad1d(&self, left: i64, right: i64) -> Result { + if self.dim() != 3 { + return Err(TchError::Shape(format!( + "expected a 3 dimension tensor, got {:?}", + self.size() + ))); + } + self.f_constant_pad_nd([left, right]) + } + + pub fn zero_pad1d(&self, left: i64, right: i64) -> Tensor { + self.f_zero_pad1d(left, right).unwrap() + } + + pub fn f_zero_pad2d( + &self, + left: i64, + right: i64, + top: i64, + bottom: i64, + ) -> Result { + if self.dim() != 4 { + return Err(TchError::Shape(format!( + "expected a 4 dimension tensor, got {:?}", + self.size() + ))); + } + self.f_constant_pad_nd([left, right, top, bottom]) + } + + pub fn zero_pad2d(&self, left: i64, right: i64, top: i64, bottom: i64) -> Tensor { + self.f_zero_pad2d(left, right, top, bottom).unwrap() + } +} + +impl From<&[T]> for Tensor { + fn from(v: &[T]) -> Tensor { + Tensor::from_slice(v) + } +} + +impl From for Tensor { + fn from(v: T) -> Tensor { + Tensor::from_slice(&[v]).view(()) + } +} +impl Tensor { + /// Casts a tensor to a specified kind. + pub fn to_kind(&self, kind: Kind) -> Tensor { + self.totype(kind) + } + + pub fn f_to_kind(&self, kind: Kind) -> Result { + self.f_totype(kind) + } + + pub fn nll_loss(&self, targets: &Tensor) -> Tensor { + self.g_nll_loss::(targets, None, Reduction::Mean, -100) + } +} + +impl Tensor { + /// Computes the cross-entropy loss based on some logits and targets. + pub fn cross_entropy_for_logits(&self, targets: &Tensor) -> Tensor { + self.log_softmax(-1, Kind::Float).nll_loss(targets) + } + + /// Returns the average accuracy for some given logits assuming that + /// targets represent ground-truth. + pub fn accuracy_for_logits(&self, targets: &Tensor) -> Tensor { + self.argmax(-1, false).eq_tensor(targets).to_kind(Kind::Float).mean(Kind::Float) + } + + pub fn random_batch(&self, batch_size: i64) -> Tensor { + let len: i64 = self.size()[0]; + let index = Tensor::randint(len, [batch_size], (Kind::Int64, self.device())); + self.index_select(0, &index) + } + + pub fn random_batch2(t1: &Tensor, t2: &Tensor, batch_size: i64) -> (Tensor, Tensor) { + let len1: i64 = t1.size()[0]; + let len2: i64 = t2.size()[0]; + if len1 != len2 { + panic!("random_batch2: shape mismatch {:?} {:?}", t1.size(), t2.size()) + } + let device1 = t1.device(); + let device2 = t2.device(); + if device1 != device2 { + panic!("random_batch2: device mismatch {device1:?} {device2:?}") + } + let index = Tensor::randint(len1, [batch_size], (Kind::Int64, device1)); + let batch1 = t1.index_select(0, &index); + let batch2 = t2.index_select(0, &index); + (batch1, batch2) + } + + /// Moves a tensor to a specified device. + pub fn to_device(&self, device: Device) -> Tensor { + self.to(device) + } + + pub fn f_to_device(&self, device: Device) -> Result { + self.f_to(device) + } + + pub fn avg_pool2d_default(&self, ksize: i64) -> Tensor { + self.avg_pool2d([ksize, ksize], [ksize, ksize], [0, 0], false, true, 1) + } + + pub fn max_pool2d_default(&self, ksize: i64) -> Tensor { + self.max_pool2d([ksize, ksize], [ksize, ksize], [0, 0], [1, 1], false) + } + + /// Flattens a tensor. + /// + /// This returns a flattened version of the given tensor. The first dimension + /// is preserved as it is assumed to be the mini-batch dimension. + pub fn flat_view(&self) -> Tensor { + self.view((self.size()[0], -1)) + } + + /// Converts a tensor to a one-hot encoded version. + /// + /// If the input has a size [N1, N2, ..., Nk], the returned tensor has a size + /// [N1, ..., Nk, labels]. The returned tensor uses float values. + /// Elements of the input vector are expected to be between 0 and labels-1. + pub fn onehot(&self, labels: i64) -> Tensor { + Tensor::zeros([self.size(), vec![labels]].concat(), (Kind::Float, self.device())) + .scatter_value_(-1, &self.unsqueeze(-1).to_kind(Kind::Int64), 1.0) + } + + /// Copies a tensor to a newly allocated tensor using the same shape and device. + pub fn copy(&self) -> Tensor { + let mut result = self.zeros_like(); + result.copy_(self); + result + } + + /// Copies the data from a two dimensional slice in a tensor object. + pub fn from_slice2(v: &[U]) -> Tensor + where + T: crate::kind::Element, + U: AsRef<[T]>, + { + let inner: Vec = v.iter().map(|v| Tensor::from_slice(v.as_ref())).collect(); + Tensor::stack(&inner, 0) + } + + pub fn to_mkldnn(&self) -> Tensor { + self.g_to_mkldnn(self.kind()) + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/tensor/npy.rs b/v2/dependency-patches/tch-0.24.0/src/tensor/npy.rs new file mode 100644 index 0000000000..855d834cd5 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/tensor/npy.rs @@ -0,0 +1,277 @@ +//! Numpy support for tensors. +//! +//! This module implements the support for reading and writing `.npy` and `.npz` files. The file +//! format spec can be found at: +//! . +use crate::{Kind, TchError, Tensor}; +use std::collections::HashMap; +use std::fs::File; +use std::io::{BufReader, Read, Write}; +use std::path::Path; + +const NPY_MAGIC_STRING: &[u8] = b"\x93NUMPY"; +const NPY_SUFFIX: &str = ".npy"; + +fn read_header(reader: &mut R) -> Result { + let mut magic_string = vec![0u8; NPY_MAGIC_STRING.len()]; + reader.read_exact(&mut magic_string)?; + if magic_string != NPY_MAGIC_STRING { + return Err(TchError::FileFormat("magic string mismatch".to_string())); + } + let mut version = [0u8; 2]; + reader.read_exact(&mut version)?; + let header_len_len = match version[0] { + 1 => 2, + 2 => 4, + otherwise => return Err(TchError::FileFormat(format!("unsupported version {otherwise}"))), + }; + let mut header_len = vec![0u8; header_len_len]; + reader.read_exact(&mut header_len)?; + let header_len = header_len.iter().rev().fold(0_usize, |acc, &v| 256 * acc + v as usize); + let mut header = vec![0u8; header_len]; + reader.read_exact(&mut header)?; + Ok(String::from_utf8_lossy(&header).to_string()) +} + +#[derive(Debug, PartialEq)] +struct Header { + descr: Kind, + fortran_order: bool, + shape: Vec, +} + +impl Header { + fn to_string(&self) -> Result { + let fortran_order = if self.fortran_order { "True" } else { "False" }; + let mut shape = self.shape.iter().map(|x| x.to_string()).collect::>().join(","); + let descr = match self.descr { + Kind::Half => "f2", + Kind::Float => "f4", + Kind::Double => "f8", + Kind::Int => "i4", + Kind::Int64 => "i8", + Kind::Int16 => "i2", + Kind::Int8 => "i1", + Kind::Uint8 => "u1", + descr => return Err(TchError::FileFormat(format!("unsupported kind {descr:?}"))), + }; + if !shape.is_empty() { + shape.push(',') + } + Ok(format!( + "{{'descr': '<{descr}', 'fortran_order': {fortran_order}, 'shape': ({shape}), }}" + )) + } + + // Hacky parser for the npy header, a typical example would be: + // {'descr': ' Result { + let header = + header.trim_matches(|c: char| c == '{' || c == '}' || c == ',' || c.is_whitespace()); + + let mut parts: Vec = vec![]; + let mut start_index = 0usize; + let mut cnt_parenthesis = 0i64; + for (index, c) in header.char_indices() { + match c { + '(' => cnt_parenthesis += 1, + ')' => cnt_parenthesis -= 1, + ',' => { + if cnt_parenthesis == 0 { + parts.push(header[start_index..index].to_owned()); + start_index = index + 1; + } + } + _ => {} + } + } + parts.push(header[start_index..].to_owned()); + let mut part_map: HashMap = HashMap::new(); + for part in parts.iter() { + let part = part.trim(); + if !part.is_empty() { + match part.split(':').collect::>().as_slice() { + [key, value] => { + let key = key.trim_matches(|c: char| c == '\'' || c.is_whitespace()); + let value = value.trim_matches(|c: char| c == '\'' || c.is_whitespace()); + let _ = part_map.insert(key.to_owned(), value.to_owned()); + } + _ => { + return Err(TchError::FileFormat(format!( + "unable to parse header {header}" + ))) + } + } + } + } + let fortran_order = match part_map.get("fortran_order") { + None => false, + Some(fortran_order) => match fortran_order.as_ref() { + "False" => false, + "True" => true, + _ => { + return Err(TchError::FileFormat(format!( + "unknown fortran_order {fortran_order}" + ))) + } + }, + }; + let descr = match part_map.get("descr") { + None => return Err(TchError::FileFormat("no descr in header".to_string())), + Some(descr) => { + if descr.is_empty() { + return Err(TchError::FileFormat("empty descr".to_string())); + } + if descr.starts_with('>') { + return Err(TchError::FileFormat(format!("little-endian descr {descr}"))); + } + // the only supported types in tensor are: + // float64, float32, float16, + // complex64, complex128, + // int64, int32, int16, int8, + // uint8, and bool. + match descr.trim_matches(|c: char| c == '=' || c == '<' || c == '|') { + "e" | "f2" => Kind::Half, + "f" | "f4" => Kind::Float, + "d" | "f8" => Kind::Double, + "i" | "i4" => Kind::Int, + "q" | "i8" => Kind::Int64, + "h" | "i2" => Kind::Int16, + "b" | "i1" => Kind::Int8, + "B" | "u1" => Kind::Uint8, + "?" | "b1" => Kind::Bool, + "F" | "F4" | "c8" => Kind::ComplexFloat, + "D" | "F8" | "c16" => Kind::ComplexDouble, + descr => { + return Err(TchError::FileFormat(format!("unrecognized descr {descr}"))) + } + } + } + }; + let shape = match part_map.get("shape") { + None => return Err(TchError::FileFormat("no shape in header".to_string())), + Some(shape) => { + let shape = shape.trim_matches(|c: char| c == '(' || c == ')' || c == ','); + if shape.is_empty() { + vec![] + } else { + shape + .split(',') + .map(|v| v.trim().parse::()) + .collect::, _>>()? + } + } + }; + Ok(Header { descr, fortran_order, shape }) + } +} + +impl crate::Tensor { + /// Reads a npy file and return the stored tensor. + pub fn read_npy>(path: T) -> Result { + let mut reader = File::open(path.as_ref())?; + let header = read_header(&mut reader)?; + let header = Header::parse(&header)?; + if header.fortran_order { + return Err(TchError::FileFormat("fortran order not supported".to_string())); + } + let mut data: Vec = vec![]; + reader.read_to_end(&mut data)?; + Tensor::f_from_data_size(&data, &header.shape, header.descr) + } + + /// Reads a npz file and returns some named tensors. + pub fn read_npz>(path: T) -> Result, TchError> { + let zip_reader = BufReader::new(File::open(path.as_ref())?); + let mut zip = zip::ZipArchive::new(zip_reader)?; + let mut result = vec![]; + for i in 0..zip.len() { + let mut reader = zip.by_index(i).unwrap(); + let name = { + let name = reader.name()?; + name.strip_suffix(NPY_SUFFIX).unwrap_or(&name).to_owned() + }; + let header = read_header(&mut reader)?; + let header = Header::parse(&header)?; + if header.fortran_order { + return Err(TchError::FileFormat("fortran order not supported".to_string())); + } + let mut data: Vec = vec![]; + reader.read_to_end(&mut data)?; + let tensor = Tensor::f_from_data_size(&data, &header.shape, header.descr)?; + result.push((name, tensor)) + } + Ok(result) + } + + fn write(&self, f: &mut T) -> Result<(), TchError> { + f.write_all(NPY_MAGIC_STRING)?; + f.write_all(&[1u8, 0u8])?; + let kind = self.f_kind()?; + let header = Header { descr: kind, fortran_order: false, shape: self.size() }; + let mut header = header.to_string()?; + let pad = 16 - (NPY_MAGIC_STRING.len() + 5 + header.len()) % 16; + for _ in 0..pad % 16 { + header.push(' ') + } + header.push('\n'); + f.write_all(&[(header.len() % 256) as u8, (header.len() / 256) as u8])?; + f.write_all(header.as_bytes())?; + let numel = self.numel(); + let mut content = vec![0u8; numel * kind.elt_size_in_bytes()]; + self.f_copy_data_u8(&mut content, numel)?; + f.write_all(&content)?; + Ok(()) + } + + /// Writes a tensor in the npy format so that it can be read using python. + pub fn write_npy>(&self, path: T) -> Result<(), TchError> { + let mut f = File::create(path.as_ref())?; + self.write(&mut f) + } + + pub fn write_npz, T: AsRef, P: AsRef>( + ts: &[(S, T)], + path: P, + ) -> Result<(), TchError> { + let mut zip = zip::ZipWriter::new(File::create(path.as_ref())?); + let options = + zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + + for (name, tensor) in ts.iter() { + zip.start_file(format!("{}.npy", name.as_ref()), options)?; + tensor.as_ref().write(&mut zip)? + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::Header; + + #[test] + fn parse() { + let h = "{'descr': '(v: T) -> T { + v +} + +fn neg(t: Tensor) -> Tensor { + t.neg() +} + +fn inv(t: Tensor) -> Tensor { + t.pow_tensor_scalar(-1) +} + +macro_rules! impl_op { + ($trait:ident, $func:ident, $op:ident) => { + impl $trait for Tensor { + type Output = Tensor; + + fn $func(self, rhs: Tensor) -> Self::Output { + self.$op(&rhs) + } + } + + impl $trait<&Tensor> for Tensor { + type Output = Tensor; + + fn $func(self, rhs: &Tensor) -> Self::Output { + self.$op(rhs) + } + } + + impl<'a> $trait<&Tensor> for &'a Tensor { + type Output = Tensor; + + fn $func(self, rhs: &Tensor) -> Self::Output { + self.$op(rhs) + } + } + + impl $trait for &Tensor { + type Output = Tensor; + + fn $func(self, rhs: Tensor) -> Self::Output { + self.$op(&rhs) + } + } + }; +} + +impl Add for &Tensor +where + S: Into, +{ + type Output = Tensor; + + fn add(self, rhs: S) -> Self::Output { + self.g_add_scalar(rhs) + } +} + +impl Add for Tensor +where + S: Into, +{ + type Output = Tensor; + + fn add(self, rhs: S) -> Self::Output { + (&self).add(rhs) + } +} + +impl Sub for &Tensor +where + S: Into, +{ + type Output = Tensor; + + fn sub(self, rhs: S) -> Self::Output { + self.g_sub_scalar(rhs) + } +} + +impl Sub for Tensor +where + S: Into, +{ + type Output = Tensor; + + fn sub(self, rhs: S) -> Self::Output { + (&self).sub(rhs) + } +} + +impl Mul for &Tensor +where + S: Into, +{ + type Output = Tensor; + + fn mul(self, rhs: S) -> Self::Output { + self.g_mul_scalar(rhs) + } +} + +impl Mul for Tensor +where + S: Into, +{ + type Output = Tensor; + + fn mul(self, rhs: S) -> Self::Output { + (&self).mul(rhs) + } +} + +impl Div for &Tensor +where + S: Into, +{ + type Output = Tensor; + + fn div(self, rhs: S) -> Self::Output { + self.g_div_scalar(rhs) + } +} + +impl Div for Tensor +where + S: Into, +{ + type Output = Tensor; + + fn div(self, rhs: S) -> Self::Output { + (&self).div(rhs) + } +} + +macro_rules! impl_op_basic { + /* rev such that rev(op(b, a)) = op(a, b) */ + ($trait:ident, $func:ident, $op:ident, $rev:ident) => { + impl $trait for i32 { + type Output = Tensor; + + fn $func(self, rhs: Tensor) -> Self::Output { + self.$func(&rhs) + } + } + + impl $trait for i64 { + type Output = Tensor; + + fn $func(self, rhs: Tensor) -> Self::Output { + self.$func(&rhs) + } + } + + impl $trait for f32 { + type Output = Tensor; + + fn $func(self, rhs: Tensor) -> Self::Output { + self.$func(&rhs) + } + } + + impl $trait for f64 { + type Output = Tensor; + + fn $func(self, rhs: Tensor) -> Self::Output { + self.$func(&rhs) + } + } + + impl $trait<&Tensor> for i32 { + type Output = Tensor; + + fn $func(self, rhs: &Tensor) -> Self::Output { + $rev(rhs.$op(self as i64)) + } + } + + impl $trait<&Tensor> for i64 { + type Output = Tensor; + + fn $func(self, rhs: &Tensor) -> Self::Output { + $rev(rhs.$op(self)) + } + } + + impl $trait<&Tensor> for f32 { + type Output = Tensor; + + fn $func(self, rhs: &Tensor) -> Self::Output { + $rev(rhs.$op(self as f64)) + } + } + + impl $trait<&Tensor> for f64 { + type Output = Tensor; + + fn $func(self, rhs: &Tensor) -> Self::Output { + $rev(rhs.$op(self)) + } + } + }; +} + +macro_rules! impl_op_assign { + ($trait:ident, $func:ident, $op:ident) => { + impl $trait for Tensor { + fn $func(&mut self, rhs: Tensor) { + let _ = self.$op(&rhs); + } + } + + impl $trait<&Tensor> for Tensor { + fn $func(&mut self, rhs: &Tensor) { + let _ = self.$op(rhs); + } + } + }; +} + +macro_rules! impl_op_assign_basic { + ($trait:ident, $func:ident, $op:ident) => { + impl $trait for Tensor { + fn $func(&mut self, rhs: i32) { + let _ = self.$op(rhs as i64); + } + } + + impl $trait for Tensor { + fn $func(&mut self, rhs: i64) { + let _ = self.$op(rhs); + } + } + + impl $trait for Tensor { + fn $func(&mut self, rhs: f32) { + let _ = self.$op(rhs as f64); + } + } + + impl $trait for Tensor { + fn $func(&mut self, rhs: f64) { + let _ = self.$op(rhs); + } + } + }; +} + +impl_op!(Add, add, g_add); +impl_op_basic!(Add, add, g_add_scalar, id); +impl_op_assign!(AddAssign, add_assign, g_add_); +impl_op_assign_basic!(AddAssign, add_assign, g_add_scalar_); + +impl_op!(Mul, mul, g_mul); +impl_op_basic!(Mul, mul, g_mul_scalar, id); +impl_op_assign!(MulAssign, mul_assign, g_mul_); +impl_op_assign_basic!(MulAssign, mul_assign, g_mul_scalar_); + +impl_op!(Div, div, g_div); +impl_op_basic!(Div, div, g_div_scalar, inv); +impl_op_assign!(DivAssign, div_assign, g_div_); +impl_op_assign_basic!(DivAssign, div_assign, g_div_scalar_); + +impl_op!(Sub, sub, g_sub); +impl_op_basic!(Sub, sub, g_sub_scalar, neg); +impl_op_assign!(SubAssign, sub_assign, g_sub_); +impl_op_assign_basic!(SubAssign, sub_assign, g_sub_scalar_); + +impl Neg for Tensor { + type Output = Tensor; + + fn neg(self) -> Tensor { + self.f_neg().unwrap() + } +} + +impl Neg for &Tensor { + type Output = Tensor; + + fn neg(self) -> Tensor { + self.f_neg().unwrap() + } +} + +impl PartialEq for Tensor { + fn eq(&self, other: &Tensor) -> bool { + if self.size() != other.size() { + return false; + } + match self.f_eq_tensor(other) { + Err(_) => false, + Ok(v) => match v.f_all() { + Err(_) => false, + Ok(v) => match i64::try_from(v) { + Err(_) => false, + Ok(v) => v > 0, + }, + }, + } + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/tensor/safetensors.rs b/v2/dependency-patches/tch-0.24.0/src/tensor/safetensors.rs new file mode 100644 index 0000000000..9ca4984a9a --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/tensor/safetensors.rs @@ -0,0 +1,183 @@ +//! Safetensors support for tensors. +//! +//! This module implements reading and writing tensors in the `.safetensors` format. +//! +use crate::nn::VarStore; +use crate::{Kind, TchError, Tensor}; + +use std::convert::{TryFrom, TryInto}; +use std::path::Path; + +use safetensors::tensor::{Dtype, SafeTensors, TensorView, View}; + +impl TryFrom for Dtype { + type Error = TchError; + fn try_from(kind: Kind) -> Result { + let dtype = match kind { + Kind::Bool => Dtype::BOOL, + Kind::Uint8 => Dtype::U8, + Kind::Int8 => Dtype::I8, + Kind::Int16 => Dtype::I16, + Kind::Int => Dtype::I32, + Kind::Int64 => Dtype::I64, + Kind::BFloat16 => Dtype::BF16, + Kind::Half => Dtype::F16, + Kind::Float => Dtype::F32, + Kind::Double => Dtype::F64, + kind => return Err(TchError::Convert(format!("unsupported kind ({kind:?})"))), + }; + Ok(dtype) + } +} + +impl TryFrom for Kind { + type Error = TchError; + fn try_from(dtype: Dtype) -> Result { + let kind = match dtype { + Dtype::BOOL => Kind::Bool, + Dtype::U8 => Kind::Uint8, + Dtype::I8 => Kind::Int8, + Dtype::I16 => Kind::Int16, + Dtype::I32 => Kind::Int, + Dtype::I64 => Kind::Int64, + Dtype::BF16 => Kind::BFloat16, + Dtype::F16 => Kind::Half, + Dtype::F32 => Kind::Float, + Dtype::F64 => Kind::Double, + dtype => return Err(TchError::Convert(format!("unsupported dtype {dtype:?}"))), + }; + Ok(kind) + } +} + +impl<'a> TryFrom> for Tensor { + type Error = TchError; + fn try_from(view: TensorView<'a>) -> Result { + let size: Vec = view.shape().iter().map(|&x| x as i64).collect(); + let kind: Kind = view.dtype().try_into()?; + Tensor::f_from_data_size(view.data(), &size, kind) + } +} + +struct SafeView<'a> { + tensor: &'a Tensor, + shape: Vec, + dtype: Dtype, +} + +impl<'a> TryFrom<&'a Tensor> for SafeView<'a> { + type Error = TchError; + + fn try_from(tensor: &'a Tensor) -> Result { + if tensor.is_sparse() { + return Err(TchError::Convert("Cannot save sparse tensors".to_string())); + } + + if !tensor.is_contiguous() { + return Err(TchError::Convert("Cannot save non contiguous tensors".to_string())); + } + + let dtype = tensor.kind().try_into()?; + let shape = tensor.size().iter().map(|&x| x as usize).collect(); + Ok(Self { tensor, shape, dtype }) + } +} + +impl View for SafeView<'_> { + fn dtype(&self) -> Dtype { + self.dtype + } + fn shape(&self) -> &[usize] { + &self.shape + } + + fn data(&self) -> std::borrow::Cow<'_, [u8]> { + let mut data = vec![0; self.data_len()]; + let numel = self.tensor.numel(); + self.tensor.f_copy_data_u8(&mut data, numel).unwrap(); + data.into() + } + + fn data_len(&self) -> usize { + self.tensor.numel() * self.tensor.kind().elt_size_in_bytes() + } +} + +fn wrap_err>(path: P, err: safetensors::SafeTensorError) -> TchError { + TchError::SafeTensorError { path: path.as_ref().to_string_lossy().to_string(), err } +} + +impl crate::Tensor { + /// Reads a safetensors file and returns some named tensors. + pub fn read_safetensors>(path: T) -> Result, TchError> { + let file = std::fs::read(&path).map_err(|e| wrap_err(&path, e.into()))?; + let safetensors = SafeTensors::deserialize(&file).map_err(|e| wrap_err(&path, e))?; + safetensors.tensors().into_iter().map(|(name, view)| Ok((name, view.try_into()?))).collect() + } + + /// Writes a tensor in the safetensors format. + pub fn write_safetensors, T: AsRef, P: AsRef>( + tensors: &[(S, T)], + path: P, + ) -> Result<(), TchError> { + let views = tensors + .iter() + .map(|(name, tensor)| { + Ok::<(&str, SafeView), TchError>((name.as_ref(), tensor.as_ref().try_into()?)) + }) + .collect::, _>>()?; + safetensors::tensor::serialize_to_file(views, &None, path.as_ref()) + .map_err(|e| wrap_err(path, e))?; + Ok(()) + } +} + +impl VarStore { + /// Read data from safe tensor file, missing tensors will raise a error. + pub fn read_safetensors>(&self, path: T) -> Result<(), TchError> { + let file = std::fs::read(&path).map_err(|e| wrap_err(&path, e.into()))?; + let safetensors = SafeTensors::deserialize(&file).map_err(|e| wrap_err(&path, e))?; + for (name, tensor) in self.variables_.lock().unwrap().named_variables.iter_mut() { + let view = safetensors.tensor(name).map_err(|e| wrap_err(&path, e))?; + let data: Tensor = view.try_into()?; + tensor.f_copy_(&data)? + } + Ok(()) + } + + pub fn fill_safetensors>(&self, path: P) -> Result<(), TchError> { + for (name, tensor) in Tensor::read_safetensors(path)? { + if let Some(s) = self.variables_.lock().unwrap().named_variables.get_mut(&name) { + s.f_copy_(&tensor)? + } + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use std::convert::TryInto; + + use crate::Kind; + use safetensors::Dtype; + + #[test] + fn parse() { + // From Kind to Dtype + assert_eq!(TryInto::::try_into(Kind::Double).unwrap(), Dtype::F64); + assert_eq!(TryInto::::try_into(Kind::Float).unwrap(), Dtype::F32); + assert_eq!(TryInto::::try_into(Kind::Half).unwrap(), Dtype::F16); + + assert_eq!(TryInto::::try_into(Kind::Int8).unwrap(), Dtype::I8); + assert_eq!(TryInto::::try_into(Kind::Uint8).unwrap(), Dtype::U8); + + // From Dtype to Kind + assert_eq!(TryInto::::try_into(Dtype::F64).unwrap(), Kind::Double); + assert_eq!(TryInto::::try_into(Dtype::F32).unwrap(), Kind::Float); + assert_eq!(TryInto::::try_into(Dtype::F16).unwrap(), Kind::Half); + + assert_eq!(TryInto::::try_into(Dtype::I8).unwrap(), Kind::Int8); + assert_eq!(TryInto::::try_into(Dtype::U8).unwrap(), Kind::Uint8); + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/alexnet.rs b/v2/dependency-patches/tch-0.24.0/src/vision/alexnet.rs new file mode 100644 index 0000000000..6338ef0a60 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/vision/alexnet.rs @@ -0,0 +1,44 @@ +//! AlexNet. +//! +use crate::{nn, nn::Conv2D, nn::ModuleT, Tensor}; + +fn conv2d(p: nn::Path, c_in: i64, c_out: i64, ksize: i64, padding: i64, stride: i64) -> Conv2D { + let conv2d_cfg = nn::ConvConfig { stride, padding, ..Default::default() }; + nn::conv2d(p, c_in, c_out, ksize, conv2d_cfg) +} + +fn max_pool2d(xs: Tensor, ksize: i64, stride: i64) -> Tensor { + xs.max_pool2d([ksize, ksize], [stride, stride], [0, 0], [1, 1], false) +} + +fn features(p: nn::Path) -> impl ModuleT { + nn::seq_t() + .add(conv2d(&p / "0", 3, 64, 11, 2, 4)) + .add_fn(|xs| max_pool2d(xs.relu(), 3, 2)) + .add(conv2d(&p / "3", 64, 192, 5, 1, 2)) + .add_fn(|xs| max_pool2d(xs.relu(), 3, 2)) + .add(conv2d(&p / "6", 192, 384, 3, 1, 1)) + .add_fn(|xs| xs.relu()) + .add(conv2d(&p / "8", 384, 256, 3, 1, 1)) + .add_fn(|xs| xs.relu()) + .add(conv2d(&p / "10", 256, 256, 3, 1, 1)) + .add_fn(|xs| max_pool2d(xs.relu(), 3, 2)) +} + +fn classifier(p: nn::Path, nclasses: i64) -> impl ModuleT { + nn::seq_t() + .add_fn_t(|xs, train| xs.dropout(0.5, train)) + .add(nn::linear(&p / "1", 256 * 6 * 6, 4096, Default::default())) + .add_fn(|xs| xs.relu()) + .add_fn_t(|xs, train| xs.dropout(0.5, train)) + .add(nn::linear(&p / "4", 4096, 4096, Default::default())) + .add_fn(|xs| xs.relu()) + .add(nn::linear(&p / "6", 4096, nclasses, Default::default())) +} + +pub fn alexnet(p: &nn::Path, nclasses: i64) -> impl ModuleT { + nn::seq_t() + .add(features(p / "features")) + .add_fn(|xs| xs.adaptive_avg_pool2d([6, 6]).flat_view()) + .add(classifier(p / "classifier", nclasses)) +} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/cifar.rs b/v2/dependency-patches/tch-0.24.0/src/vision/cifar.rs new file mode 100644 index 0000000000..112345d10c --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/vision/cifar.rs @@ -0,0 +1,64 @@ +//! The CIFAR-10 dataset. +//! +//! The files can be downloaded from the following page: +//! +//! The binary version of the dataset is used. +use super::dataset::Dataset; +use crate::{kind, IndexOp, Kind, Tensor}; +use std::fs::File; +use std::io::{BufReader, Read, Result}; + +const W: i64 = 32; +const H: i64 = 32; +const C: i64 = 3; +const BYTES_PER_IMAGE: i64 = W * H * C + 1; +const SAMPLES_PER_FILE: i64 = 10000; + +fn read_file_(filename: &std::path::Path) -> Result<(Tensor, Tensor)> { + let mut buf_reader = BufReader::new(File::open(filename)?); + let mut data = vec![0u8; (SAMPLES_PER_FILE * BYTES_PER_IMAGE) as usize]; + buf_reader.read_exact(&mut data)?; + let content = Tensor::from_slice(&data); + let images = Tensor::zeros([SAMPLES_PER_FILE, C, H, W], kind::FLOAT_CPU); + let labels = Tensor::zeros([SAMPLES_PER_FILE], kind::INT64_CPU); + for index in 0..SAMPLES_PER_FILE { + let content_offset = BYTES_PER_IMAGE * index; + labels.i(index).copy_(&content.i(content_offset)); + images.i(index).copy_( + &content + .narrow(0, 1 + content_offset, BYTES_PER_IMAGE - 1) + .view((C, H, W)) + .to_kind(Kind::Float), + ); + } + Ok((images.to_kind(Kind::Float) / 255.0, labels)) +} + +fn read_file(filename: &std::path::Path) -> Result<(Tensor, Tensor)> { + read_file_(filename) + .map_err(|err| std::io::Error::new(err.kind(), format!("{filename:?} {err}"))) +} + +pub fn load_dir>(dir: T) -> Result { + let dir = dir.as_ref(); + let (test_images, test_labels) = read_file(&dir.join("test_batch.bin"))?; + let train_images_and_labels = [ + "data_batch_1.bin", + "data_batch_2.bin", + "data_batch_3.bin", + "data_batch_4.bin", + "data_batch_5.bin", + ] + .iter() + .map(|x| read_file(&dir.join(x))) + .collect::>>()?; + let (train_images, train_labels): (Vec<_>, Vec<_>) = + train_images_and_labels.into_iter().unzip(); + Ok(Dataset { + train_images: Tensor::cat(&train_images, 0), + train_labels: Tensor::cat(&train_labels, 0), + test_images, + test_labels, + labels: 10, + }) +} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/convmixer.rs b/v2/dependency-patches/tch-0.24.0/src/vision/convmixer.rs new file mode 100644 index 0000000000..ce0379bbba --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/vision/convmixer.rs @@ -0,0 +1,49 @@ +//! ConvMixer implementation. +//! +//! See "Patches Are All You Need?", submitted to ICLR'21 +//! +use crate::nn; + +fn block(p: nn::Path, dim: i64, kernel_size: i64) -> impl nn::ModuleT { + let conv2d_cfg = nn::ConvConfig { groups: dim, ..Default::default() }; + let p_fn = &(&p / "0") / "fn"; + let conv1 = + crate::vision::efficientnet::conv2d_same(&p_fn / "0", dim, dim, kernel_size, conv2d_cfg); + let bn1 = nn::batch_norm2d(&p_fn / "2", dim, Default::default()); + let conv2 = nn::conv2d(&p / "1", dim, dim, 1, Default::default()); + let bn2 = nn::batch_norm2d(&p / "3", dim, Default::default()); + nn::func_t(move |xs, train| { + let ys = xs.apply(&conv1).gelu("none").apply_t(&bn1, train); + (xs + ys).apply(&conv2).gelu("none").apply_t(&bn2, train) + }) +} + +fn convmixer( + p: &nn::Path, + nclasses: i64, + dim: i64, + depth: i64, + kernel_size: i64, + patch_size: i64, +) -> nn::FuncT<'static> { + let conv2d_cfg = nn::ConvConfig { stride: patch_size, ..Default::default() }; + let conv1 = nn::conv2d(p / "0", 3, dim, patch_size, conv2d_cfg); + let bn1 = nn::batch_norm2d(p / "2", dim, Default::default()); + let blocks: Vec<_> = (0..depth).map(|index| block(p / (3 + index), dim, kernel_size)).collect(); + let fc = nn::linear(p / "25", dim, nclasses, Default::default()); + nn::func_t(move |xs, train| { + let mut xs = xs.apply(&conv1).gelu("none").apply_t(&bn1, train); + for block in blocks.iter() { + xs = xs.apply_t(block, train) + } + xs.adaptive_avg_pool2d([1, 1]).flat_view().apply(&fc) + }) +} + +pub fn c1536_20(p: &nn::Path, nclasses: i64) -> nn::FuncT<'static> { + convmixer(p, nclasses, 1536, 20, 9, 7) +} + +pub fn c1024_20(p: &nn::Path, nclasses: i64) -> nn::FuncT<'static> { + convmixer(p, nclasses, 1024, 20, 9, 14) +} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/dataset.rs b/v2/dependency-patches/tch-0.24.0/src/vision/dataset.rs new file mode 100644 index 0000000000..782401248c --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/vision/dataset.rs @@ -0,0 +1,95 @@ +//! A simple dataset structure shared by various computer vision datasets. +use crate::data::Iter2; +use crate::{IndexOp, Tensor}; +use rand::Rng; + +#[derive(Debug)] +pub struct Dataset { + pub train_images: Tensor, + pub train_labels: Tensor, + pub test_images: Tensor, + pub test_labels: Tensor, + pub labels: i64, +} + +impl Dataset { + pub fn train_iter(&self, batch_size: i64) -> Iter2 { + Iter2::new(&self.train_images, &self.train_labels, batch_size) + } + + pub fn test_iter(&self, batch_size: i64) -> Iter2 { + Iter2::new(&self.test_images, &self.test_labels, batch_size) + } +} + +/// Randomly applies horizontal flips +/// This expects a 4 dimension NCHW tensor and returns a tensor with +/// an identical shape. +pub fn random_flip(t: &Tensor) -> Tensor { + let size = t.size(); + if size.len() != 4 { + panic!("unexpected shape for tensor {t:?}") + } + let output = t.zeros_like(); + for batch_index in 0..size[0] { + let mut output_view = output.i(batch_index); + let t_view = t.i(batch_index); + let src = if rand::random() { t_view } else { t_view.flip([2]) }; + output_view.copy_(&src) + } + output +} + +/// Pad the image using reflections and take some random crops. +/// This expects a 4 dimension NCHW tensor and returns a tensor with +/// an identical shape. +pub fn random_crop(t: &Tensor, pad: i64) -> Tensor { + let size = t.size(); + if size.len() != 4 { + panic!("unexpected shape for tensor {t:?}") + } + let sz_h = size[2]; + let sz_w = size[3]; + let padded = t.reflection_pad2d([pad, pad, pad, pad]); + let output = t.zeros_like(); + for bindex in 0..size[0] { + let mut output_view = output.i(bindex); + let start_w = rand::thread_rng().gen_range(0..2 * pad); + let start_h = rand::thread_rng().gen_range(0..2 * pad); + let src = padded.i((bindex, .., start_h..start_h + sz_h, start_w..start_w + sz_w)); + output_view.copy_(&src) + } + output +} + +/// Applies cutout: randomly remove some square areas in the original images. +/// +pub fn random_cutout(t: &Tensor, sz: i64) -> Tensor { + let size = t.size(); + if size.len() != 4 || sz > size[2] || sz > size[3] { + panic!("unexpected shape for tensor {t:?} {sz}") + } + let mut output = t.zeros_like(); + output.copy_(t); + for bindex in 0..size[0] { + let start_h = rand::thread_rng().gen_range(0..size[2] - sz + 1); + let start_w = rand::thread_rng().gen_range(0..size[3] - sz + 1); + let _output = + output.i((bindex, .., start_h..start_h + sz, start_w..start_w + sz)).fill_(0.0); + } + output +} + +pub fn augmentation(t: &Tensor, flip: bool, crop: i64, cutout: i64) -> Tensor { + let mut t = t.shallow_clone(); + if flip { + t = random_flip(&t); + } + if crop > 0 { + t = random_crop(&t, crop); + } + if cutout > 0 { + t = random_cutout(&t, cutout); + } + t +} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/densenet.rs b/v2/dependency-patches/tch-0.24.0/src/vision/densenet.rs new file mode 100644 index 0000000000..03b95dc3a6 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/vision/densenet.rs @@ -0,0 +1,96 @@ +//! DenseNet implementation. +//! +//! See "Densely Connected Convolutional Networks", Huang et al 2016. +//! +//! +//! Pre-trained weights for the densenet-121 variants can be found here: +//! +use crate::{nn, nn::Conv2D, nn::ModuleT, Tensor}; + +fn conv2d(p: nn::Path, c_in: i64, c_out: i64, ksize: i64, padding: i64, stride: i64) -> Conv2D { + let conv2d_cfg = nn::ConvConfig { stride, padding, bias: false, ..Default::default() }; + nn::conv2d(p, c_in, c_out, ksize, conv2d_cfg) +} + +fn dense_layer(p: nn::Path, c_in: i64, bn_size: i64, growth: i64) -> impl ModuleT { + let c_inter = bn_size * growth; + let bn1 = nn::batch_norm2d(&p / "norm1", c_in, Default::default()); + let conv1 = conv2d(&p / "conv1", c_in, c_inter, 1, 0, 1); + let bn2 = nn::batch_norm2d(&p / "norm2", c_inter, Default::default()); + let conv2 = conv2d(&p / "conv2", c_inter, growth, 3, 1, 1); + nn::func_t(move |xs, train| { + let ys = + xs.apply_t(&bn1, train).relu().apply(&conv1).apply_t(&bn2, train).relu().apply(&conv2); + Tensor::cat(&[xs, &ys], 1) + }) +} + +fn dense_block(p: nn::Path, c_in: i64, bn_size: i64, growth: i64, nlayers: i64) -> impl ModuleT { + let mut seq = nn::seq_t(); + for i in 0..nlayers { + seq = seq.add(dense_layer( + &p / &format!("denselayer{}", 1 + i), + c_in + i * growth, + bn_size, + growth, + )); + } + seq +} + +fn transition(p: nn::Path, c_in: i64, c_out: i64) -> impl ModuleT { + nn::seq_t() + .add(nn::batch_norm2d(&p / "norm", c_in, Default::default())) + .add_fn(|xs| xs.relu()) + .add(conv2d(&p / "conv", c_in, c_out, 1, 0, 1)) + .add_fn(|xs| xs.avg_pool2d_default(2)) +} + +fn densenet( + p: &nn::Path, + c_in: i64, + bn_size: i64, + growth: i64, + block_config: &[i64], + c_out: i64, +) -> impl ModuleT { + let fp = p / "features"; + let mut seq = nn::seq_t() + .add(conv2d(&fp / "conv0", 3, c_in, 7, 3, 2)) + .add(nn::batch_norm2d(&fp / "norm0", c_in, Default::default())) + .add_fn(|xs| xs.relu().max_pool2d([3, 3], [2, 2], [1, 1], [1, 1], false)); + let mut nfeat = c_in; + for (i, &nlayers) in block_config.iter().enumerate() { + seq = seq.add(dense_block( + &fp / &format!("denseblock{}", 1 + i), + nfeat, + bn_size, + growth, + nlayers, + )); + nfeat += nlayers * growth; + if i + 1 != block_config.len() { + seq = seq.add(transition(&fp / &format!("transition{}", 1 + i), nfeat, nfeat / 2)); + nfeat /= 2 + } + } + seq.add(nn::batch_norm2d(&fp / "norm5", nfeat, Default::default())) + .add_fn(|xs| xs.relu().avg_pool2d([7, 7], [1, 1], [0, 0], false, true, 1).flat_view()) + .add(nn::linear(p / "classifier", nfeat, c_out, Default::default())) +} + +pub fn densenet121(p: &nn::Path, nclasses: i64) -> impl ModuleT { + densenet(p, 64, 4, 32, &[6, 12, 24, 16], nclasses) +} + +pub fn densenet161(p: &nn::Path, nclasses: i64) -> impl ModuleT { + densenet(p, 96, 4, 48, &[6, 12, 36, 24], nclasses) +} + +pub fn densenet169(p: &nn::Path, nclasses: i64) -> impl ModuleT { + densenet(p, 64, 4, 32, &[6, 12, 32, 32], nclasses) +} + +pub fn densenet201(p: &nn::Path, nclasses: i64) -> impl ModuleT { + densenet(p, 64, 4, 32, &[6, 12, 48, 32], nclasses) +} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/dinov2.rs b/v2/dependency-patches/tch-0.24.0/src/vision/dinov2.rs new file mode 100644 index 0000000000..c6a00d82f0 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/vision/dinov2.rs @@ -0,0 +1,234 @@ +//! DINOv2: Learning Robust Visual Features without Supervision +//! https://github.com/facebookresearch/dinov2 +//! The weights can be extracted from pre-trained Python models +//! using `python src/vision/export_dinov2.py`. +// TODO: use swiglu. +use crate::{nn, IndexOp, Kind, Tensor}; + +const IMG_SIZE: i64 = 518; +const PATCH_SIZE: i64 = 14; +const NUM_CLASSES: i64 = 1000; + +#[derive(Debug)] +struct Attention { + qkv: nn::Linear, + proj: nn::Linear, + num_heads: i64, + scale: f64, +} + +impl Attention { + fn new(vs: nn::Path, dim: i64, num_heads: i64, qkv_bias: bool, proj_bias: bool) -> Self { + let qkv_config = nn::LinearConfig { bias: qkv_bias, ..Default::default() }; + let proj_config = nn::LinearConfig { bias: proj_bias, ..Default::default() }; + let qkv = nn::linear(&vs / "qkv", dim, dim * 3, qkv_config); + let proj = nn::linear(&vs / "proj", dim, dim, proj_config); + let scale = 1. / ((dim / num_heads) as f64).sqrt(); + Self { qkv, proj, num_heads, scale } + } +} + +impl nn::Module for Attention { + fn forward(&self, xs: &Tensor) -> Tensor { + let (b, n, c) = xs.size3().unwrap(); + let qkv = self + .qkv + .forward(xs) + .reshape([b, n, 3, self.num_heads, c / self.num_heads]) + .permute([2, 0, 3, 1, 4]); + let q = qkv.get(0) * self.scale; + let k = qkv.get(1); + let v = qkv.get(2); + let attn = q.matmul(&k.transpose(-2, -1)).softmax(-1, Kind::Float); + attn.matmul(&v).transpose(1, 2).reshape([b, n, c]).apply(&self.proj) + } +} + +#[derive(Debug)] +struct LayerScale { + gamma: Tensor, +} + +impl LayerScale { + fn new(vs: nn::Path, dim: i64) -> Self { + let gamma = vs.var("gamma", &[dim], nn::Init::Const(0.)); + Self { gamma } + } +} + +impl nn::Module for LayerScale { + fn forward(&self, xs: &Tensor) -> Tensor { + xs * &self.gamma + } +} + +#[derive(Debug)] +struct Mlp { + fc1: nn::Linear, + fc2: nn::Linear, +} + +impl Mlp { + fn new(vs: nn::Path, in_features: i64, hidden_features: i64, bias: bool) -> Self { + let out_features = in_features; + let config = nn::LinearConfig { bias, ..Default::default() }; + let fc1 = nn::linear(&vs / "fc1", in_features, hidden_features, config); + let fc2 = nn::linear(&vs / "fc2", hidden_features, out_features, config); + Self { fc1, fc2 } + } +} + +impl nn::Module for Mlp { + fn forward(&self, xs: &Tensor) -> Tensor { + xs.apply(&self.fc1).gelu("none").apply(&self.fc2) + } +} + +#[derive(Debug)] +struct Block { + norm1: nn::LayerNorm, + attn: Attention, + ls1: LayerScale, + norm2: nn::LayerNorm, + mlp: Mlp, + ls2: LayerScale, +} + +impl Block { + fn new(vs: nn::Path, dim: i64, num_heads: i64) -> Self { + let norm1 = nn::layer_norm(&vs / "norm1", vec![dim], Default::default()); + let attn = Attention::new(&vs / "attn", dim, num_heads, true, true); + let ls1 = LayerScale::new(&vs / "ls1", dim); + let norm2 = nn::layer_norm(&vs / "norm2", vec![dim], Default::default()); + let mlp = Mlp::new(&vs / "mlp", dim, dim * 4, true); + let ls2 = LayerScale::new(&vs / "ls2", dim); + Self { norm1, attn, ls1, norm2, mlp, ls2 } + } +} + +impl nn::Module for Block { + fn forward(&self, xs: &Tensor) -> Tensor { + let xs = xs + xs.apply(&self.norm1).apply(&self.attn).apply(&self.ls1); + &xs + xs.apply(&self.norm2).apply(&self.mlp).apply(&self.ls2) + } +} + +#[derive(Debug)] +struct PatchEmbed { + proj: nn::Conv2D, + patch_size: (i64, i64), + num_patches: i64, +} + +impl PatchEmbed { + fn new(vs: nn::Path, img_size: i64, patch_size: i64, in_chans: i64, embed_dim: i64) -> Self { + let config = nn::ConvConfig { stride: patch_size, ..Default::default() }; + let proj = nn::conv2d(vs / "proj", in_chans, embed_dim, patch_size, config); + let num_patches = (img_size / patch_size) * (img_size / patch_size); + Self { proj, patch_size: (patch_size, patch_size), num_patches } + } +} + +impl nn::Module for PatchEmbed { + fn forward(&self, xs: &Tensor) -> Tensor { + let (_b, _c, h, w) = xs.size4().unwrap(); + let (patch_h, patch_w) = self.patch_size; + if (h % patch_h) != 0 { + panic!("image height {h} is not a multiple of patch height {patch_h}") + } + if (w % patch_w) != 0 { + panic!("image width {w} is not a multiple of patch width {patch_w}") + } + let xs = xs.apply(&self.proj); + let (b, c, h, w) = xs.size4().unwrap(); + // flatten embeddings. + xs.reshape([b, c, h * w]).transpose(1, 2) + } +} + +#[derive(Debug)] +pub struct DinoVisionTransformer { + patch_embed: PatchEmbed, + cls_token: Tensor, + pos_embed: Tensor, + blocks: Vec, + norm: nn::LayerNorm, + head: nn::Linear, +} + +impl DinoVisionTransformer { + pub fn new(vs: &nn::Path, depth: usize, embed_dim: i64, num_heads: i64) -> Self { + let patch_embed = PatchEmbed::new(vs / "patch_embed", IMG_SIZE, PATCH_SIZE, 3, embed_dim); + let cls_token = vs.var("cls_token", &[1, 1, embed_dim], nn::Init::Const(0.)); + let num_tokens = 1; + let pos_embed = vs.var( + "pos_embed", + &[1, patch_embed.num_patches + num_tokens, embed_dim], + nn::Init::Const(0.), + ); + let head = nn::linear(vs / "head", 2 * embed_dim, NUM_CLASSES, Default::default()); + let norm = nn::layer_norm(vs / "norm", vec![embed_dim], Default::default()); + let blocks = + (0..depth).map(|i| Block::new(vs / "blocks" / i, embed_dim, num_heads)).collect(); + Self { patch_embed, cls_token, pos_embed, blocks, norm, head } + } + + pub fn interpolate_pos_encoding(&self, xs: &Tensor, w: i64, h: i64) -> Tensor { + let npatch = xs.size()[1] - 1; + let n = self.pos_embed.size()[1] - 1; + let sqrt_n = (n as f64).sqrt(); + if npatch == n && w == h { + return xs.shallow_clone(); + } + let class_pos_embed = self.pos_embed.i((.., ..1)); + let patch_pos_embed = self.pos_embed.i((.., 1..)); + let dim = *xs.size().last().unwrap(); + let (w0, h0) = ((w / PATCH_SIZE) as f64 + 0.1, (h / PATCH_SIZE) as f64 + 0.1); + let patch_pos_embed = patch_pos_embed + .reshape([1, sqrt_n as i64, sqrt_n as i64, dim]) + .permute([0, 3, 1, 2]) + .upsample_bicubic2d([w0 as i64, h0 as i64], false, w0 / sqrt_n, h0 / sqrt_n); + let patch_pos_embed = patch_pos_embed.permute([0, 2, 3, 1]).reshape([1, -1, dim]); + Tensor::cat(&[class_pos_embed, patch_pos_embed], 1) + } + + pub fn prepare_tokens_with_mask(&self, xs: &Tensor) -> Tensor { + let (b, _nc, w, h) = xs.size4().unwrap(); + let xs = xs.apply(&self.patch_embed); + let xs = Tensor::concat(&[self.cls_token.expand([b, -1, -1], false), xs], 1); + &xs + &self.interpolate_pos_encoding(&xs, w, h) + } + + pub fn extract_features(&self, xs: &Tensor) -> Tensor { + let mut xs = self.prepare_tokens_with_mask(xs); + for blk in self.blocks.iter() { + xs = xs.apply(blk) + } + let xs = xs.apply(&self.norm); + let xs_norm_clstoken = xs.i((.., 0)); + let xs_norm_patchtokens = xs.i((.., 1..)).mean_dim(1, false, None); + Tensor::concat(&[xs_norm_clstoken, xs_norm_patchtokens], -1) + } +} + +impl nn::Module for DinoVisionTransformer { + fn forward(&self, xs: &Tensor) -> Tensor { + self.extract_features(xs).apply(&self.head) + } +} + +pub fn vit_small(vs: &nn::Path) -> DinoVisionTransformer { + DinoVisionTransformer::new(vs, 12, 384, 6) +} + +pub fn vit_base(vs: &nn::Path) -> DinoVisionTransformer { + DinoVisionTransformer::new(vs, 12, 768, 12) +} + +pub fn vit_large(vs: &nn::Path) -> DinoVisionTransformer { + DinoVisionTransformer::new(vs, 24, 1024, 16) +} + +pub fn vit_giant(vs: &nn::Path) -> DinoVisionTransformer { + DinoVisionTransformer::new(vs, 40, 1536, 24) +} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/efficientnet.rs b/v2/dependency-patches/tch-0.24.0/src/vision/efficientnet.rs new file mode 100644 index 0000000000..d38c4a41c6 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/vision/efficientnet.rs @@ -0,0 +1,329 @@ +//! EfficientNet implementation. +use crate::nn::{self, ConvConfig, Module, ModuleT}; +use crate::Tensor; + +const BATCH_NORM_MOMENTUM: f64 = 0.99; +const BATCH_NORM_EPSILON: f64 = 1e-3; + +// Based on the Python version from torchvision. +// https://github.com/pytorch/vision/blob/0d75d9e5516f446c9c0ef93bd4ed9fea13992d06/torchvision/models/efficientnet.py#L47 +#[derive(Debug, Clone, Copy)] +pub struct MBConvConfig { + expand_ratio: f64, + kernel: i64, + stride: i64, + input_channels: i64, + out_channels: i64, + num_layers: usize, +} + +fn make_divisible(v: f64, divisor: i64) -> i64 { + let min_value = divisor; + let new_v = i64::max(min_value, (v + divisor as f64 * 0.5) as i64 / divisor * divisor); + if (new_v as f64) < 0.9 * v { + new_v + divisor + } else { + new_v + } +} + +fn bneck_confs(width_mult: f64, depth_mult: f64) -> Vec { + let bneck_conf = |e, k, s, i, o, n| { + let input_channels = make_divisible(i as f64 * width_mult, 8); + let out_channels = make_divisible(o as f64 * width_mult, 8); + let num_layers = (n as f64 * depth_mult).ceil() as usize; + MBConvConfig { + expand_ratio: e, + kernel: k, + stride: s, + input_channels, + out_channels, + num_layers, + } + }; + vec![ + bneck_conf(1., 3, 1, 32, 16, 1), + bneck_conf(6., 3, 2, 16, 24, 2), + bneck_conf(6., 5, 2, 24, 40, 2), + bneck_conf(6., 3, 2, 40, 80, 3), + bneck_conf(6., 5, 1, 80, 112, 3), + bneck_conf(6., 5, 2, 112, 192, 4), + bneck_conf(6., 3, 1, 192, 320, 1), + ] +} + +impl MBConvConfig { + fn b0() -> Vec { + bneck_confs(1.0, 1.0) + } + fn b1() -> Vec { + bneck_confs(1.0, 1.1) + } + fn b2() -> Vec { + bneck_confs(1.1, 1.2) + } + fn b3() -> Vec { + bneck_confs(1.2, 1.4) + } + fn b4() -> Vec { + bneck_confs(1.4, 1.8) + } + fn b5() -> Vec { + bneck_confs(1.6, 2.2) + } + fn b6() -> Vec { + bneck_confs(1.8, 2.6) + } + fn b7() -> Vec { + bneck_confs(2.0, 3.1) + } +} + +/// Conv2D with same padding. +#[derive(Debug)] +struct Conv2DSame { + conv2d: nn::Conv2D, + s: i64, + k: i64, +} + +impl Conv2DSame { + fn new(vs: nn::Path, i: i64, o: i64, k: i64, stride: i64, groups: i64, b: bool) -> Self { + let conv_config = nn::ConvConfig { stride, groups, bias: b, ..Default::default() }; + let conv2d = nn::conv2d(vs, i, o, k, conv_config); + Self { conv2d, s: stride, k } + } +} + +impl Module for Conv2DSame { + fn forward(&self, xs: &Tensor) -> Tensor { + let s = self.s; + let k = self.k; + let size = xs.size(); + let ih = size[2]; + let iw = size[3]; + let oh = (ih + s - 1) / s; + let ow = (iw + s - 1) / s; + let pad_h = i64::max((oh - 1) * s + k - ih, 0); + let pad_w = i64::max((ow - 1) * s + k - iw, 0); + if pad_h > 0 || pad_w > 0 { + xs.zero_pad2d(pad_w / 2, pad_w - pad_w / 2, pad_h / 2, pad_h - pad_h / 2) + .apply(&self.conv2d) + } else { + xs.apply(&self.conv2d) + } + } +} + +#[derive(Debug)] +struct ConvNormActivation { + conv2d: Conv2DSame, + bn2d: nn::BatchNorm, + activation: bool, +} + +impl ConvNormActivation { + fn new(vs: nn::Path, i: i64, o: i64, k: i64, stride: i64, groups: i64) -> Self { + let conv2d = Conv2DSame::new(&vs / 0, i, o, k, stride, groups, false); + let bn_config = nn::BatchNormConfig { + momentum: 1.0 - BATCH_NORM_MOMENTUM, + eps: BATCH_NORM_EPSILON, + ..Default::default() + }; + let bn2d = nn::batch_norm2d(&vs / 1, o, bn_config); + Self { conv2d, bn2d, activation: true } + } + + fn no_activation(self) -> Self { + Self { activation: false, ..self } + } +} + +impl ModuleT for ConvNormActivation { + fn forward_t(&self, xs: &Tensor, t: bool) -> Tensor { + let xs = xs.apply(&self.conv2d).apply_t(&self.bn2d, t); + if self.activation { + xs.swish() + } else { + xs + } + } +} + +#[derive(Debug)] +struct SqueezeExcitation { + fc1: Conv2DSame, + fc2: Conv2DSame, +} + +impl SqueezeExcitation { + fn new(vs: nn::Path, input_channels: i64, squeeze_channels: i64) -> Self { + let fc1 = Conv2DSame::new(&vs / "fc1", input_channels, squeeze_channels, 1, 1, 1, true); + let fc2 = Conv2DSame::new(&vs / "fc2", squeeze_channels, input_channels, 1, 1, 1, true); + Self { fc1, fc2 } + } +} + +impl ModuleT for SqueezeExcitation { + fn forward_t(&self, xs: &Tensor, t: bool) -> Tensor { + let scale = xs + .adaptive_avg_pool2d([1, 1]) + .apply_t(&self.fc1, t) + .swish() + .apply_t(&self.fc2, t) + .sigmoid(); + scale * xs + } +} + +#[derive(Debug)] +struct MBConv { + expand_cna: Option, + depthwise_cna: ConvNormActivation, + squeeze_excitation: SqueezeExcitation, + project_cna: ConvNormActivation, + config: MBConvConfig, +} + +impl MBConv { + fn new(vs: nn::Path, c: MBConvConfig) -> Self { + let vs = &vs / "block"; + let exp = make_divisible(c.input_channels as f64 * c.expand_ratio, 8); + let expand_cna = if exp != c.input_channels { + Some(ConvNormActivation::new(&vs / 0, c.input_channels, exp, 1, 1, 1)) + } else { + None + }; + let start_index = if expand_cna.is_some() { 1 } else { 0 }; + let depthwise_cna = + ConvNormActivation::new(&vs / start_index, exp, exp, c.kernel, c.stride, exp); + let squeeze_channels = i64::max(1, c.input_channels / 4); + let squeeze_excitation = + SqueezeExcitation::new(&vs / (start_index + 1), exp, squeeze_channels); + let project_cna = + ConvNormActivation::new(&vs / (start_index + 2), exp, c.out_channels, 1, 1, 1) + .no_activation(); + Self { expand_cna, depthwise_cna, squeeze_excitation, project_cna, config: c } + } +} + +impl ModuleT for MBConv { + fn forward_t(&self, xs: &Tensor, t: bool) -> Tensor { + let use_res_connect = + self.config.stride == 1 && self.config.input_channels == self.config.out_channels; + let ys = match &self.expand_cna { + Some(expand_cna) => xs.apply_t(expand_cna, t), + None => xs.shallow_clone(), + }; + let ys = ys + .apply_t(&self.depthwise_cna, t) + .apply_t(&self.squeeze_excitation, t) + .apply_t(&self.project_cna, t); + if use_res_connect { + ys + xs + } else { + ys + } + } +} + +impl Tensor { + fn swish(&self) -> Tensor { + self * self.sigmoid() + } +} + +#[derive(Debug)] +struct EfficientNet { + init_cna: ConvNormActivation, + blocks: Vec, + final_cna: ConvNormActivation, + classifier: nn::Linear, +} + +impl EfficientNet { + fn new(p: &nn::Path, configs: Vec, nclasses: i64) -> Self { + let f_p = p / "features"; + let first_in_c = configs[0].input_channels; + let last_out_c = configs.last().unwrap().out_channels; + let final_out_c = 4 * last_out_c; + let init_cna = ConvNormActivation::new(&f_p / 0, 3, first_in_c, 3, 2, 1); + let nconfigs = configs.len(); + let mut blocks = vec![]; + for (index, cnf) in configs.into_iter().enumerate() { + let f_p = &f_p / (index + 1); + for r_index in 0..cnf.num_layers { + let cnf = if r_index == 0 { + cnf + } else { + MBConvConfig { input_channels: cnf.out_channels, stride: 1, ..cnf } + }; + blocks.push(MBConv::new(&f_p / r_index, cnf)) + } + } + let final_cna = + ConvNormActivation::new(&f_p / (nconfigs + 1), last_out_c, final_out_c, 1, 1, 1); + let classifier = + nn::linear(p / "classifier" / 1, final_out_c, nclasses, Default::default()); + Self { init_cna, blocks, final_cna, classifier } + } +} + +impl ModuleT for EfficientNet { + fn forward_t(&self, xs: &Tensor, t: bool) -> Tensor { + let mut xs = xs.apply_t(&self.init_cna, t); + for block in self.blocks.iter() { + xs = xs.apply_t(block, t) + } + xs.apply_t(&self.final_cna, t) + .adaptive_avg_pool2d([1, 1]) + .squeeze_dim(-1) + .squeeze_dim(-1) + .apply(&self.classifier) + } +} + +pub fn b0(p: &nn::Path, nclasses: i64) -> impl ModuleT { + EfficientNet::new(p, MBConvConfig::b0(), nclasses) +} +pub fn b1(p: &nn::Path, nclasses: i64) -> impl ModuleT { + EfficientNet::new(p, MBConvConfig::b1(), nclasses) +} +pub fn b2(p: &nn::Path, nclasses: i64) -> impl ModuleT { + EfficientNet::new(p, MBConvConfig::b2(), nclasses) +} +pub fn b3(p: &nn::Path, nclasses: i64) -> impl ModuleT { + EfficientNet::new(p, MBConvConfig::b3(), nclasses) +} +pub fn b4(p: &nn::Path, nclasses: i64) -> impl ModuleT { + EfficientNet::new(p, MBConvConfig::b4(), nclasses) +} +pub fn b5(p: &nn::Path, nclasses: i64) -> impl ModuleT { + EfficientNet::new(p, MBConvConfig::b5(), nclasses) +} +pub fn b6(p: &nn::Path, nclasses: i64) -> impl ModuleT { + EfficientNet::new(p, MBConvConfig::b6(), nclasses) +} +pub fn b7(p: &nn::Path, nclasses: i64) -> impl ModuleT { + EfficientNet::new(p, MBConvConfig::b7(), nclasses) +} + +#[allow(clippy::many_single_char_names)] +pub fn conv2d_same(vs: nn::Path, i: i64, o: i64, k: i64, c: ConvConfig) -> impl Module { + let conv2d = nn::conv2d(vs, i, o, k, c); + let s = c.stride; + nn::func(move |xs| { + let size = xs.size(); + let ih = size[2]; + let iw = size[3]; + let oh = (ih + s - 1) / s; + let ow = (iw + s - 1) / s; + let pad_h = i64::max((oh - 1) * s + k - ih, 0); + let pad_w = i64::max((ow - 1) * s + k - iw, 0); + if pad_h > 0 || pad_w > 0 { + xs.zero_pad2d(pad_w / 2, pad_w - pad_w / 2, pad_h / 2, pad_h - pad_h / 2).apply(&conv2d) + } else { + xs.apply(&conv2d) + } + }) +} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/export_dinov2.py b/v2/dependency-patches/tch-0.24.0/src/vision/export_dinov2.py new file mode 100644 index 0000000000..b3cf1815a8 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/vision/export_dinov2.py @@ -0,0 +1,16 @@ +import torch +from safetensors.torch import save_file + +def normalize_key(k): + if k.startswith("backbone."): + k = k[9:] + if k.startswith("linear_head."): + k = k[7:] + return k + +for model_size in ["small", "base", "large", "giant"]: + letter = model_size[0] + dinov2_vits14 = torch.hub.load('facebookresearch/dinov2', f'dinov2_vit{letter}14_lc', layers=1) + weights = dinov2_vits14.state_dict() + weights = {normalize_key(k): v for k, v in weights.items()} + save_file(weights, f"dinov2_vit{letter}14.safetensors") diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/export_model.py b/v2/dependency-patches/tch-0.24.0/src/vision/export_model.py new file mode 100644 index 0000000000..e2fc3580b9 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/vision/export_model.py @@ -0,0 +1,16 @@ +# This script exports pre-trained model weights in the safetensors format. +import numpy as np +import torch +import torchvision +from safetensors import torch as stt + +m = torchvision.models.efficientnet_b0(pretrained=True) +stt.save_file(m.state_dict(), 'efficientnet-b0.safetensors') +m = torchvision.models.efficientnet_b1(pretrained=True) +stt.save_file(m.state_dict(), 'efficientnet-b1.safetensors') +m = torchvision.models.efficientnet_b2(pretrained=True) +stt.save_file(m.state_dict(), 'efficientnet-b2.safetensors') +m = torchvision.models.efficientnet_b3(pretrained=True) +stt.save_file(m.state_dict(), 'efficientnet-b3.safetensors') +m = torchvision.models.efficientnet_b4(pretrained=True) +stt.save_file(m.state_dict(), 'efficientnet-b4.safetensors') diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/image.rs b/v2/dependency-patches/tch-0.24.0/src/vision/image.rs new file mode 100644 index 0000000000..fcc830eb80 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/vision/image.rs @@ -0,0 +1,146 @@ +//! Utility functions to manipulate images. +use crate::wrappers::image::{load_hwc, load_hwc_from_mem, resize_hwc, save_hwc}; +use crate::{Device, TchError, Tensor}; +use std::io; +use std::path::Path; + +pub(crate) fn hwc_to_chw(tensor: &Tensor) -> Tensor { + tensor.permute([2, 0, 1]) +} + +pub(crate) fn chw_to_hwc(tensor: &Tensor) -> Tensor { + tensor.permute([1, 2, 0]) +} + +/// Loads an image from a file. +/// +/// On success returns a tensor of shape [channel, height, width]. +pub fn load>(path: T) -> Result { + let tensor = load_hwc(path)?; + Ok(hwc_to_chw(&tensor)) +} + +/// Loads an image from memory. +/// +/// On success returns a tensor of shape [channel, height, width]. +pub fn load_from_memory(img_data: &[u8]) -> Result { + let tensor = load_hwc_from_mem(img_data)?; + Ok(hwc_to_chw(&tensor)) +} + +/// Saves an image to a file. +/// +/// This expects as input a tensor of shape [channel, height, width]. +/// The image format is based on the filename suffix, supported suffixes +/// are jpg, png, tga, and bmp. +/// The tensor input should be of kind UInt8 with values ranging from +/// 0 to 255. +pub fn save>(t: &Tensor, path: T) -> Result<(), TchError> { + let t = t.to_kind(crate::Kind::Uint8); + match t.size().as_slice() { + [1, _, _, _] => save_hwc(&chw_to_hwc(&t.squeeze_dim(0)).to_device(Device::Cpu), path), + [_, _, _] => save_hwc(&chw_to_hwc(&t).to_device(Device::Cpu), path), + sz => Err(TchError::FileFormat(format!("unexpected size for image tensor {sz:?}"))), + } +} + +/// Resizes an image. +/// +/// This expects as input a tensor of shape [channel, height, width] and returns +/// a tensor of shape [channel, out_h, out_w]. +pub fn resize(t: &Tensor, out_w: i64, out_h: i64) -> Result { + Ok(hwc_to_chw(&resize_hwc(&chw_to_hwc(t), out_w, out_h)?)) +} + +pub fn resize_preserve_aspect_ratio_hwc( + t: &Tensor, + out_w: i64, + out_h: i64, +) -> Result { + let tensor_size = t.size(); + let (w, h) = (tensor_size[0], tensor_size[1]); + if w * out_h == h * out_w { + Ok(hwc_to_chw(&resize_hwc(t, out_w, out_h)?)) + } else { + let (resize_w, resize_h) = { + let ratio_w = out_w as f64 / w as f64; + let ratio_h = out_h as f64 / h as f64; + let ratio = ratio_w.max(ratio_h); + ((ratio * h as f64) as i64, (ratio * w as f64) as i64) + }; + let resize_w = i64::max(resize_w, out_w); + let resize_h = i64::max(resize_h, out_h); + let t = hwc_to_chw(&resize_hwc(t, resize_w, resize_h)?); + let t = if resize_w == out_w { t } else { t.f_narrow(2, (resize_w - out_w) / 2, out_w)? }; + let t = if resize_h == out_h { t } else { t.f_narrow(1, (resize_h - out_h) / 2, out_h)? }; + Ok(t) + } +} + +/// Resize an image, preserve the aspect ratio by taking a center crop. +/// +/// This expects as input a tensor of shape [channel, height, width] and returns +pub fn resize_preserve_aspect_ratio( + t: &Tensor, + out_w: i64, + out_h: i64, +) -> Result { + resize_preserve_aspect_ratio_hwc(&chw_to_hwc(t), out_w, out_h) +} + +/// Loads and resize an image, preserve the aspect ratio by taking a center crop. +pub fn load_and_resize>( + path: T, + out_w: i64, + out_h: i64, +) -> Result { + let tensor = load_hwc(path)?; + resize_preserve_aspect_ratio_hwc(&tensor, out_w, out_h) +} + +/// Loads and resize an image from memory, preserve the aspect ratio by taking a center crop. +pub fn load_and_resize_from_memory( + img_data: &[u8], + out_w: i64, + out_h: i64, +) -> Result { + let tensor = load_hwc_from_mem(img_data)?; + resize_preserve_aspect_ratio_hwc(&tensor, out_w, out_h) +} + +fn visit_dirs(dir: &Path, files: &mut Vec) -> Result<(), TchError> { + if dir.is_dir() { + for entry in std::fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + if path.is_dir() { + visit_dirs(&path, files)?; + } else if entry + .file_name() + .to_str() + .is_some_and(|s| s.ends_with(".png") || s.ends_with(".jpg")) + { + files.push(entry); + } + } + } + Ok(()) +} + +/// Loads all the images in a directory. +pub fn load_dir>(path: T, out_w: i64, out_h: i64) -> Result { + let mut files: Vec = vec![]; + visit_dirs(path.as_ref(), &mut files)?; + if files.is_empty() { + return Err(TchError::Io(io::Error::new( + io::ErrorKind::NotFound, + format!("no image found in {:?}", path.as_ref(),), + ))); + } + let v: Vec<_> = files + .iter() + // Silently discard image errors. + .filter_map(|x| load_and_resize(x.path(), out_w, out_h).ok()) + .collect(); + Ok(Tensor::stack(&v, 0)) +} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/imagenet.rs b/v2/dependency-patches/tch-0.24.0/src/vision/imagenet.rs new file mode 100644 index 0000000000..e4b93d4060 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/vision/imagenet.rs @@ -0,0 +1,1170 @@ +//! Helper functions for ImageNet like datasets. +use super::dataset::Dataset; +use crate::{kind, Device, Kind, TchError, Tensor}; +use std::io; +use std::path::Path; +use std::sync::Mutex; + +lazy_static! { + static ref IMAGENET_MEAN: Mutex = + Mutex::new(Tensor::from_slice(&[0.485f32, 0.456, 0.406]).view((3, 1, 1))); + static ref IMAGENET_STD: Mutex = + Mutex::new(Tensor::from_slice(&[0.229f32, 0.224, 0.225]).view((3, 1, 1))); +} + +pub fn normalize(tensor: &Tensor) -> Result { + let mean = IMAGENET_MEAN.lock().unwrap(); + let std = IMAGENET_STD.lock().unwrap(); + (tensor.to_kind(Kind::Float) / 255.0).f_sub(&mean)?.f_div(&std) +} + +pub fn unnormalize(tensor: &Tensor) -> Result { + let mean = IMAGENET_MEAN.lock().unwrap(); + let std = IMAGENET_STD.lock().unwrap(); + let tensor = (tensor.f_mul(&std)?.f_add(&mean)? * 255.0).clamp(0., 255.0).to_kind(Kind::Uint8); + Ok(tensor) +} + +/// Saves an image to a path. +/// This unapplies the ImageNet normalization. +pub fn save_image>(tensor: &Tensor, path: T) -> Result<(), TchError> { + super::image::save(&unnormalize(&tensor.to_device(Device::Cpu))?, path) +} + +/// Loads an image from a file and applies the ImageNet normalization. +pub fn load_image>(path: T) -> Result { + normalize(&super::image::load(path)?) +} + +/// Loads an image from memory and applies the ImageNet normalization. +pub fn load_image_from_memory(img_data: &[u8]) -> Result { + normalize(&super::image::load_from_memory(img_data)?) +} + +/// Loads an image from a file and resize it to 224x224. +/// This applies the ImageNet normalization. +pub fn load_image_and_resize224>(path: T) -> Result { + normalize(&super::image::load_and_resize(path, 224, 224)?) +} + +/// Loads an image from memory and resize it to 224x224. +/// This applies the ImageNet normalization. +pub fn load_image_and_resize224_from_memory(img_data: &[u8]) -> Result { + normalize(&super::image::load_and_resize_from_memory(img_data, 224, 224)?) +} + +/// Loads an image from a file and resize it to the specified width and height. +/// This applies the ImageNet normalization. +pub fn load_image_and_resize>(path: T, w: i64, h: i64) -> Result { + normalize(&super::image::load_and_resize(path, w, h)?) +} + +/// Loads an image from memory and resize it to the specified width and height. +/// This applies the ImageNet normalization. +pub fn load_image_and_resize_from_memory( + img_data: &[u8], + w: i64, + h: i64, +) -> Result { + normalize(&super::image::load_and_resize_from_memory(img_data, w, h)?) +} + +fn has_image_suffix>(path: T) -> bool { + match path.as_ref().extension() { + None => false, + Some(extension) => match extension.to_str() { + Some("jpg") | Some("jpeg") | Some("png") | Some("JPG") | Some("JPEG") | Some("PNG") => { + true + } + Some(_) | None => false, + }, + } +} + +fn load_images_from_dir(dir: std::path::PathBuf) -> Result { + let mut images: Vec = vec![]; + for image in std::fs::read_dir(&dir)? { + match image { + Err(_) => {} + Ok(image) => { + let image = image.path(); + if !has_image_suffix(&image) { + continue; + } + match load_image_and_resize224(image) { + Err(_) => {} + Ok(image) => images.push(image), + } + } + } + } + if images.is_empty() { + return Err(TchError::Io(io::Error::new( + io::ErrorKind::NotFound, + format!("no image found in {dir:?}"), + ))); + } + Tensor::f_stack(&images, 0) +} + +/// Loads a dataset from a directory. +/// +/// This assumes that the directory contains two subdirectories named train and val. +/// In each of these datasets, there should be a subdirectory per class named +/// in the same way. +/// The ImageNet normalization is applied, image are resized to 224x224. +pub fn load_from_dir>(dir: T) -> Result { + let train_path = dir.as_ref().join("train"); + let valid_path = dir.as_ref().join("val"); + let classes = std::fs::read_dir(&valid_path)? + .filter_map(|d| d.ok().map(|d| d.path())) + .filter(|d| d.is_dir()) + .filter_map(|d| d.file_name().map(|d| d.to_os_string())) + .collect::>(); + println!("classes: {classes:?}"); + let mut train_images: Vec = vec![]; + let mut train_labels: Vec = vec![]; + let mut test_images: Vec = vec![]; + let mut test_labels: Vec = vec![]; + for (label_index, label_dir) in classes.iter().enumerate() { + let label_index = label_index as i64; + let images = load_images_from_dir(train_path.join(label_dir))?; + let nimages = images.size()[0]; + train_images.push(images); + train_labels.push(Tensor::ones([nimages], kind::INT64_CPU) * label_index); + let images = load_images_from_dir(valid_path.join(label_dir))?; + let nimages = images.size()[0]; + test_images.push(images); + test_labels.push(Tensor::ones([nimages], kind::INT64_CPU) * label_index); + } + Ok(Dataset { + train_images: Tensor::f_cat(&train_images, 0)?, + train_labels: Tensor::f_cat(&train_labels, 0)?, + test_images: Tensor::f_cat(&test_images, 0)?, + test_labels: Tensor::f_cat(&test_labels, 0)?, + labels: classes.len() as i64, + }) +} + +pub const CLASS_COUNT: i64 = 1000; + +pub const CLASSES: [&str; 1000] = [ + "tench, Tinca tinca", + "goldfish, Carassius auratus", + "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", + "tiger shark, Galeocerdo cuvieri", + "hammerhead, hammerhead shark", + "electric ray, crampfish, numbfish, torpedo", + "stingray", + "cock", + "hen", + "ostrich, Struthio camelus", + "brambling, Fringilla montifringilla", + "goldfinch, Carduelis carduelis", + "house finch, linnet, Carpodacus mexicanus", + "junco, snowbird", + "indigo bunting, indigo finch, indigo bird, Passerina cyanea", + "robin, American robin, Turdus migratorius", + "bulbul", + "jay", + "magpie", + "chickadee", + "water ouzel, dipper", + "kite", + "bald eagle, American eagle, Haliaeetus leucocephalus", + "vulture", + "great grey owl, great gray owl, Strix nebulosa", + "European fire salamander, Salamandra salamandra", + "common newt, Triturus vulgaris", + "eft", + "spotted salamander, Ambystoma maculatum", + "axolotl, mud puppy, Ambystoma mexicanum", + "bullfrog, Rana catesbeiana", + "tree frog, tree-frog", + "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", + "loggerhead, loggerhead turtle, Caretta caretta", + "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", + "mud turtle", + "terrapin", + "box turtle, box tortoise", + "banded gecko", + "common iguana, iguana, Iguana iguana", + "American chameleon, anole, Anolis carolinensis", + "whiptail, whiptail lizard", + "agama", + "frilled lizard, Chlamydosaurus kingi", + "alligator lizard", + "Gila monster, Heloderma suspectum", + "green lizard, Lacerta viridis", + "African chameleon, Chamaeleo chamaeleon", + "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", + "African crocodile, Nile crocodile, Crocodylus niloticus", + "American alligator, Alligator mississipiensis", + "triceratops", + "thunder snake, worm snake, Carphophis amoenus", + "ringneck snake, ring-necked snake, ring snake", + "hognose snake, puff adder, sand viper", + "green snake, grass snake", + "king snake, kingsnake", + "garter snake, grass snake", + "water snake", + "vine snake", + "night snake, Hypsiglena torquata", + "boa constrictor, Constrictor constrictor", + "rock python, rock snake, Python sebae", + "Indian cobra, Naja naja", + "green mamba", + "sea snake", + "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", + "diamondback, diamondback rattlesnake, Crotalus adamanteus", + "sidewinder, horned rattlesnake, Crotalus cerastes", + "trilobite", + "harvestman, daddy longlegs, Phalangium opilio", + "scorpion", + "black and gold garden spider, Argiope aurantia", + "barn spider, Araneus cavaticus", + "garden spider, Aranea diademata", + "black widow, Latrodectus mactans", + "tarantula", + "wolf spider, hunting spider", + "tick", + "centipede", + "black grouse", + "ptarmigan", + "ruffed grouse, partridge, Bonasa umbellus", + "prairie chicken, prairie grouse, prairie fowl", + "peacock", + "quail", + "partridge", + "African grey, African gray, Psittacus erithacus", + "macaw", + "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", + "lorikeet", + "coucal", + "bee eater", + "hornbill", + "hummingbird", + "jacamar", + "toucan", + "drake", + "red-breasted merganser, Mergus serrator", + "goose", + "black swan, Cygnus atratus", + "tusker", + "echidna, spiny anteater, anteater", + "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", + "wallaby, brush kangaroo", + "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", + "wombat", + "jellyfish", + "sea anemone, anemone", + "brain coral", + "flatworm, platyhelminth", + "nematode, nematode worm, roundworm", + "conch", + "snail", + "slug", + "sea slug, nudibranch", + "chiton, coat-of-mail shell, sea cradle, polyplacophore", + "chambered nautilus, pearly nautilus, nautilus", + "Dungeness crab, Cancer magister", + "rock crab, Cancer irroratus", + "fiddler crab", + "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", + "American lobster, Northern lobster, Maine lobster, Homarus americanus", + "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", + "crayfish, crawfish, crawdad, crawdaddy", + "hermit crab", + "isopod", + "white stork, Ciconia ciconia", + "black stork, Ciconia nigra", + "spoonbill", + "flamingo", + "little blue heron, Egretta caerulea", + "American egret, great white heron, Egretta albus", + "bittern", + "crane", + "limpkin, Aramus pictus", + "European gallinule, Porphyrio porphyrio", + "American coot, marsh hen, mud hen, water hen, Fulica americana", + "bustard", + "ruddy turnstone, Arenaria interpres", + "red-backed sandpiper, dunlin, Erolia alpina", + "redshank, Tringa totanus", + "dowitcher", + "oystercatcher, oyster catcher", + "pelican", + "king penguin, Aptenodytes patagonica", + "albatross, mollymawk", + "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", + "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", + "dugong, Dugong dugon", + "sea lion", + "Chihuahua", + "Japanese spaniel", + "Maltese dog, Maltese terrier, Maltese", + "Pekinese, Pekingese, Peke", + "Shih-Tzu", + "Blenheim spaniel", + "papillon", + "toy terrier", + "Rhodesian ridgeback", + "Afghan hound, Afghan", + "basset, basset hound", + "beagle", + "bloodhound, sleuthhound", + "bluetick", + "black-and-tan coonhound", + "Walker hound, Walker foxhound", + "English foxhound", + "redbone", + "borzoi, Russian wolfhound", + "Irish wolfhound", + "Italian greyhound", + "whippet", + "Ibizan hound, Ibizan Podenco", + "Norwegian elkhound, elkhound", + "otterhound, otter hound", + "Saluki, gazelle hound", + "Scottish deerhound, deerhound", + "Weimaraner", + "Staffordshire bullterrier, Staffordshire bull terrier", + "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", + "Bedlington terrier", + "Border terrier", + "Kerry blue terrier", + "Irish terrier", + "Norfolk terrier", + "Norwich terrier", + "Yorkshire terrier", + "wire-haired fox terrier", + "Lakeland terrier", + "Sealyham terrier, Sealyham", + "Airedale, Airedale terrier", + "cairn, cairn terrier", + "Australian terrier", + "Dandie Dinmont, Dandie Dinmont terrier", + "Boston bull, Boston terrier", + "miniature schnauzer", + "giant schnauzer", + "standard schnauzer", + "Scotch terrier, Scottish terrier, Scottie", + "Tibetan terrier, chrysanthemum dog", + "silky terrier, Sydney silky", + "soft-coated wheaten terrier", + "West Highland white terrier", + "Lhasa, Lhasa apso", + "flat-coated retriever", + "curly-coated retriever", + "golden retriever", + "Labrador retriever", + "Chesapeake Bay retriever", + "German short-haired pointer", + "vizsla, Hungarian pointer", + "English setter", + "Irish setter, red setter", + "Gordon setter", + "Brittany spaniel", + "clumber, clumber spaniel", + "English springer, English springer spaniel", + "Welsh springer spaniel", + "cocker spaniel, English cocker spaniel, cocker", + "Sussex spaniel", + "Irish water spaniel", + "kuvasz", + "schipperke", + "groenendael", + "malinois", + "briard", + "kelpie", + "komondor", + "Old English sheepdog, bobtail", + "Shetland sheepdog, Shetland sheep dog, Shetland", + "collie", + "Border collie", + "Bouvier des Flandres, Bouviers des Flandres", + "Rottweiler", + "German shepherd, German shepherd dog, German police dog, alsatian", + "Doberman, Doberman pinscher", + "miniature pinscher", + "Greater Swiss Mountain dog", + "Bernese mountain dog", + "Appenzeller", + "EntleBucher", + "boxer", + "bull mastiff", + "Tibetan mastiff", + "French bulldog", + "Great Dane", + "Saint Bernard, St Bernard", + "Eskimo dog, husky", + "malamute, malemute, Alaskan malamute", + "Siberian husky", + "dalmatian, coach dog, carriage dog", + "affenpinscher, monkey pinscher, monkey dog", + "basenji", + "pug, pug-dog", + "Leonberg", + "Newfoundland, Newfoundland dog", + "Great Pyrenees", + "Samoyed, Samoyede", + "Pomeranian", + "chow, chow chow", + "keeshond", + "Brabancon griffon", + "Pembroke, Pembroke Welsh corgi", + "Cardigan, Cardigan Welsh corgi", + "toy poodle", + "miniature poodle", + "standard poodle", + "Mexican hairless", + "timber wolf, grey wolf, gray wolf, Canis lupus", + "white wolf, Arctic wolf, Canis lupus tundrarum", + "red wolf, maned wolf, Canis rufus, Canis niger", + "coyote, prairie wolf, brush wolf, Canis latrans", + "dingo, warrigal, warragal, Canis dingo", + "dhole, Cuon alpinus", + "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", + "hyena, hyaena", + "red fox, Vulpes vulpes", + "kit fox, Vulpes macrotis", + "Arctic fox, white fox, Alopex lagopus", + "grey fox, gray fox, Urocyon cinereoargenteus", + "tabby, tabby cat", + "tiger cat", + "Persian cat", + "Siamese cat, Siamese", + "Egyptian cat", + "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", + "lynx, catamount", + "leopard, Panthera pardus", + "snow leopard, ounce, Panthera uncia", + "jaguar, panther, Panthera onca, Felis onca", + "lion, king of beasts, Panthera leo", + "tiger, Panthera tigris", + "cheetah, chetah, Acinonyx jubatus", + "brown bear, bruin, Ursus arctos", + "American black bear, black bear, Ursus americanus, Euarctos americanus", + "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", + "sloth bear, Melursus ursinus, Ursus ursinus", + "mongoose", + "meerkat, mierkat", + "tiger beetle", + "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", + "ground beetle, carabid beetle", + "long-horned beetle, longicorn, longicorn beetle", + "leaf beetle, chrysomelid", + "dung beetle", + "rhinoceros beetle", + "weevil", + "fly", + "bee", + "ant, emmet, pismire", + "grasshopper, hopper", + "cricket", + "walking stick, walkingstick, stick insect", + "cockroach, roach", + "mantis, mantid", + "cicada, cicala", + "leafhopper", + "lacewing, lacewing fly", + "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", + "damselfly", + "admiral", + "ringlet, ringlet butterfly", + "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", + "cabbage butterfly", + "sulphur butterfly, sulfur butterfly", + "lycaenid, lycaenid butterfly", + "starfish, sea star", + "sea urchin", + "sea cucumber, holothurian", + "wood rabbit, cottontail, cottontail rabbit", + "hare", + "Angora, Angora rabbit", + "hamster", + "porcupine, hedgehog", + "fox squirrel, eastern fox squirrel, Sciurus niger", + "marmot", + "beaver", + "guinea pig, Cavia cobaya", + "sorrel", + "zebra", + "hog, pig, grunter, squealer, Sus scrofa", + "wild boar, boar, Sus scrofa", + "warthog", + "hippopotamus, hippo, river horse, Hippopotamus amphibius", + "ox", + "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", + "bison", + "ram, tup", + "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", + "ibex, Capra ibex", + "hartebeest", + "impala, Aepyceros melampus", + "gazelle", + "Arabian camel, dromedary, Camelus dromedarius", + "llama", + "weasel", + "mink", + "polecat, fitch, foulmart, foumart, Mustela putorius", + "black-footed ferret, ferret, Mustela nigripes", + "otter", + "skunk, polecat, wood pussy", + "badger", + "armadillo", + "three-toed sloth, ai, Bradypus tridactylus", + "orangutan, orang, orangutang, Pongo pygmaeus", + "gorilla, Gorilla gorilla", + "chimpanzee, chimp, Pan troglodytes", + "gibbon, Hylobates lar", + "siamang, Hylobates syndactylus, Symphalangus syndactylus", + "guenon, guenon monkey", + "patas, hussar monkey, Erythrocebus patas", + "baboon", + "macaque", + "langur", + "colobus, colobus monkey", + "proboscis monkey, Nasalis larvatus", + "marmoset", + "capuchin, ringtail, Cebus capucinus", + "howler monkey, howler", + "titi, titi monkey", + "spider monkey, Ateles geoffroyi", + "squirrel monkey, Saimiri sciureus", + "Madagascar cat, ring-tailed lemur, Lemur catta", + "indri, indris, Indri indri, Indri brevicaudatus", + "Indian elephant, Elephas maximus", + "African elephant, Loxodonta africana", + "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", + "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", + "barracouta, snoek", + "eel", + "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", + "rock beauty, Holocanthus tricolor", + "anemone fish", + "sturgeon", + "gar, garfish, garpike, billfish, Lepisosteus osseus", + "lionfish", + "puffer, pufferfish, blowfish, globefish", + "abacus", + "abaya", + "academic gown, academic robe, judge's robe", + "accordion, piano accordion, squeeze box", + "acoustic guitar", + "aircraft carrier, carrier, flattop, attack aircraft carrier", + "airliner", + "airship, dirigible", + "altar", + "ambulance", + "amphibian, amphibious vehicle", + "analog clock", + "apiary, bee house", + "apron", + "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", + "assault rifle, assault gun", + "backpack, back pack, knapsack, packsack, rucksack, haversack", + "bakery, bakeshop, bakehouse", + "balance beam, beam", + "balloon", + "ballpoint, ballpoint pen, ballpen, Biro", + "Band Aid", + "banjo", + "bannister, banister, balustrade, balusters, handrail", + "barbell", + "barber chair", + "barbershop", + "barn", + "barometer", + "barrel, cask", + "barrow, garden cart, lawn cart, wheelbarrow", + "baseball", + "basketball", + "bassinet", + "bassoon", + "bathing cap, swimming cap", + "bath towel", + "bathtub, bathing tub, bath, tub", + "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", + "beacon, lighthouse, beacon light, pharos", + "beaker", + "bearskin, busby, shako", + "beer bottle", + "beer glass", + "bell cote, bell cot", + "bib", + "bicycle-built-for-two, tandem bicycle, tandem", + "bikini, two-piece", + "binder, ring-binder", + "binoculars, field glasses, opera glasses", + "birdhouse", + "boathouse", + "bobsled, bobsleigh, bob", + "bolo tie, bolo, bola tie, bola", + "bonnet, poke bonnet", + "bookcase", + "bookshop, bookstore, bookstall", + "bottlecap", + "bow", + "bow tie, bow-tie, bowtie", + "brass, memorial tablet, plaque", + "brassiere, bra, bandeau", + "breakwater, groin, groyne, mole, bulwark, seawall, jetty", + "breastplate, aegis, egis", + "broom", + "bucket, pail", + "buckle", + "bulletproof vest", + "bullet train, bullet", + "butcher shop, meat market", + "cab, hack, taxi, taxicab", + "caldron, cauldron", + "candle, taper, wax light", + "cannon", + "canoe", + "can opener, tin opener", + "cardigan", + "car mirror", + "carousel, carrousel, merry-go-round, roundabout, whirligig", + "carpenter's kit, tool kit", + "carton", + "car wheel", + "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", + "cassette", + "cassette player", + "castle", + "catamaran", + "CD player", + "cello, violoncello", + "cellular telephone, cellular phone, cellphone, cell, mobile phone", + "chain", + "chainlink fence", + "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", + "chain saw, chainsaw", + "chest", + "chiffonier, commode", + "chime, bell, gong", + "china cabinet, china closet", + "Christmas stocking", + "church, church building", + "cinema, movie theater, movie theatre, movie house, picture palace", + "cleaver, meat cleaver, chopper", + "cliff dwelling", + "cloak", + "clog, geta, patten, sabot", + "cocktail shaker", + "coffee mug", + "coffeepot", + "coil, spiral, volute, whorl, helix", + "combination lock", + "computer keyboard, keypad", + "confectionery, confectionary, candy store", + "container ship, containership, container vessel", + "convertible", + "corkscrew, bottle screw", + "cornet, horn, trumpet, trump", + "cowboy boot", + "cowboy hat, ten-gallon hat", + "cradle", + "crane", + "crash helmet", + "crate", + "crib, cot", + "Crock Pot", + "croquet ball", + "crutch", + "cuirass", + "dam, dike, dyke", + "desk", + "desktop computer", + "dial telephone, dial phone", + "diaper, nappy, napkin", + "digital clock", + "digital watch", + "dining table, board", + "dishrag, dishcloth", + "dishwasher, dish washer, dishwashing machine", + "disk brake, disc brake", + "dock, dockage, docking facility", + "dogsled, dog sled, dog sleigh", + "dome", + "doormat, welcome mat", + "drilling platform, offshore rig", + "drum, membranophone, tympan", + "drumstick", + "dumbbell", + "Dutch oven", + "electric fan, blower", + "electric guitar", + "electric locomotive", + "entertainment center", + "envelope", + "espresso maker", + "face powder", + "feather boa, boa", + "file, file cabinet, filing cabinet", + "fireboat", + "fire engine, fire truck", + "fire screen, fireguard", + "flagpole, flagstaff", + "flute, transverse flute", + "folding chair", + "football helmet", + "forklift", + "fountain", + "fountain pen", + "four-poster", + "freight car", + "French horn, horn", + "frying pan, frypan, skillet", + "fur coat", + "garbage truck, dustcart", + "gasmask, respirator, gas helmet", + "gas pump, gasoline pump, petrol pump, island dispenser", + "goblet", + "go-kart", + "golf ball", + "golfcart, golf cart", + "gondola", + "gong, tam-tam", + "gown", + "grand piano, grand", + "greenhouse, nursery, glasshouse", + "grille, radiator grille", + "grocery store, grocery, food market, market", + "guillotine", + "hair slide", + "hair spray", + "half track", + "hammer", + "hamper", + "hand blower, blow dryer, blow drier, hair dryer, hair drier", + "hand-held computer, hand-held microcomputer", + "handkerchief, hankie, hanky, hankey", + "hard disc, hard disk, fixed disk", + "harmonica, mouth organ, harp, mouth harp", + "harp", + "harvester, reaper", + "hatchet", + "holster", + "home theater, home theatre", + "honeycomb", + "hook, claw", + "hoopskirt, crinoline", + "horizontal bar, high bar", + "horse cart, horse-cart", + "hourglass", + "iPod", + "iron, smoothing iron", + "jack-o'-lantern", + "jean, blue jean, denim", + "jeep, landrover", + "jersey, T-shirt, tee shirt", + "jigsaw puzzle", + "jinrikisha, ricksha, rickshaw", + "joystick", + "kimono", + "knee pad", + "knot", + "lab coat, laboratory coat", + "ladle", + "lampshade, lamp shade", + "laptop, laptop computer", + "lawn mower, mower", + "lens cap, lens cover", + "letter opener, paper knife, paperknife", + "library", + "lifeboat", + "lighter, light, igniter, ignitor", + "limousine, limo", + "liner, ocean liner", + "lipstick, lip rouge", + "Loafer", + "lotion", + "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", + "loupe, jeweler's loupe", + "lumbermill, sawmill", + "magnetic compass", + "mailbag, postbag", + "mailbox, letter box", + "maillot", + "maillot, tank suit", + "manhole cover", + "maraca", + "marimba, xylophone", + "mask", + "matchstick", + "maypole", + "maze, labyrinth", + "measuring cup", + "medicine chest, medicine cabinet", + "megalith, megalithic structure", + "microphone, mike", + "microwave, microwave oven", + "military uniform", + "milk can", + "minibus", + "miniskirt, mini", + "minivan", + "missile", + "mitten", + "mixing bowl", + "mobile home, manufactured home", + "Model T", + "modem", + "monastery", + "monitor", + "moped", + "mortar", + "mortarboard", + "mosque", + "mosquito net", + "motor scooter, scooter", + "mountain bike, all-terrain bike, off-roader", + "mountain tent", + "mouse, computer mouse", + "mousetrap", + "moving van", + "muzzle", + "nail", + "neck brace", + "necklace", + "nipple", + "notebook, notebook computer", + "obelisk", + "oboe, hautboy, hautbois", + "ocarina, sweet potato", + "odometer, hodometer, mileometer, milometer", + "oil filter", + "organ, pipe organ", + "oscilloscope, scope, cathode-ray oscilloscope, CRO", + "overskirt", + "oxcart", + "oxygen mask", + "packet", + "paddle, boat paddle", + "paddlewheel, paddle wheel", + "padlock", + "paintbrush", + "pajama, pyjama, pj's, jammies", + "palace", + "panpipe, pandean pipe, syrinx", + "paper towel", + "parachute, chute", + "parallel bars, bars", + "park bench", + "parking meter", + "passenger car, coach, carriage", + "patio, terrace", + "pay-phone, pay-station", + "pedestal, plinth, footstall", + "pencil box, pencil case", + "pencil sharpener", + "perfume, essence", + "Petri dish", + "photocopier", + "pick, plectrum, plectron", + "pickelhaube", + "picket fence, paling", + "pickup, pickup truck", + "pier", + "piggy bank, penny bank", + "pill bottle", + "pillow", + "ping-pong ball", + "pinwheel", + "pirate, pirate ship", + "pitcher, ewer", + "plane, carpenter's plane, woodworking plane", + "planetarium", + "plastic bag", + "plate rack", + "plow, plough", + "plunger, plumber's helper", + "Polaroid camera, Polaroid Land camera", + "pole", + "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", + "poncho", + "pool table, billiard table, snooker table", + "pop bottle, soda bottle", + "pot, flowerpot", + "potter's wheel", + "power drill", + "prayer rug, prayer mat", + "printer", + "prison, prison house", + "projectile, missile", + "projector", + "puck, hockey puck", + "punching bag, punch bag, punching ball, punchball", + "purse", + "quill, quill pen", + "quilt, comforter, comfort, puff", + "racer, race car, racing car", + "racket, racquet", + "radiator", + "radio, wireless", + "radio telescope, radio reflector", + "rain barrel", + "recreational vehicle, RV, R.V.", + "reel", + "reflex camera", + "refrigerator, icebox", + "remote control, remote", + "restaurant, eating house, eating place, eatery", + "revolver, six-gun, six-shooter", + "rifle", + "rocking chair, rocker", + "rotisserie", + "rubber eraser, rubber, pencil eraser", + "rugby ball", + "rule, ruler", + "running shoe", + "safe", + "safety pin", + "saltshaker, salt shaker", + "sandal", + "sarong", + "sax, saxophone", + "scabbard", + "scale, weighing machine", + "school bus", + "schooner", + "scoreboard", + "screen, CRT screen", + "screw", + "screwdriver", + "seat belt, seatbelt", + "sewing machine", + "shield, buckler", + "shoe shop, shoe-shop, shoe store", + "shoji", + "shopping basket", + "shopping cart", + "shovel", + "shower cap", + "shower curtain", + "ski", + "ski mask", + "sleeping bag", + "slide rule, slipstick", + "sliding door", + "slot, one-armed bandit", + "snorkel", + "snowmobile", + "snowplow, snowplough", + "soap dispenser", + "soccer ball", + "sock", + "solar dish, solar collector, solar furnace", + "sombrero", + "soup bowl", + "space bar", + "space heater", + "space shuttle", + "spatula", + "speedboat", + "spider web, spider's web", + "spindle", + "sports car, sport car", + "spotlight, spot", + "stage", + "steam locomotive", + "steel arch bridge", + "steel drum", + "stethoscope", + "stole", + "stone wall", + "stopwatch, stop watch", + "stove", + "strainer", + "streetcar, tram, tramcar, trolley, trolley car", + "stretcher", + "studio couch, day bed", + "stupa, tope", + "submarine, pigboat, sub, U-boat", + "suit, suit of clothes", + "sundial", + "sunglass", + "sunglasses, dark glasses, shades", + "sunscreen, sunblock, sun blocker", + "suspension bridge", + "swab, swob, mop", + "sweatshirt", + "swimming trunks, bathing trunks", + "swing", + "switch, electric switch, electrical switch", + "syringe", + "table lamp", + "tank, army tank, armored combat vehicle, armoured combat vehicle", + "tape player", + "teapot", + "teddy, teddy bear", + "television, television system", + "tennis ball", + "thatch, thatched roof", + "theater curtain, theatre curtain", + "thimble", + "thresher, thrasher, threshing machine", + "throne", + "tile roof", + "toaster", + "tobacco shop, tobacconist shop, tobacconist", + "toilet seat", + "torch", + "totem pole", + "tow truck, tow car, wrecker", + "toyshop", + "tractor", + "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", + "tray", + "trench coat", + "tricycle, trike, velocipede", + "trimaran", + "tripod", + "triumphal arch", + "trolleybus, trolley coach, trackless trolley", + "trombone", + "tub, vat", + "turnstile", + "typewriter keyboard", + "umbrella", + "unicycle, monocycle", + "upright, upright piano", + "vacuum, vacuum cleaner", + "vase", + "vault", + "velvet", + "vending machine", + "vestment", + "viaduct", + "violin, fiddle", + "volleyball", + "waffle iron", + "wall clock", + "wallet, billfold, notecase, pocketbook", + "wardrobe, closet, press", + "warplane, military plane", + "washbasin, handbasin, washbowl, lavabo, wash-hand basin", + "washer, automatic washer, washing machine", + "water bottle", + "water jug", + "water tower", + "whiskey jug", + "whistle", + "wig", + "window screen", + "window shade", + "Windsor tie", + "wine bottle", + "wing", + "wok", + "wooden spoon", + "wool, woolen, woollen", + "worm fence, snake fence, snake-rail fence, Virginia fence", + "wreck", + "yawl", + "yurt", + "web site, website, internet site, site", + "comic book", + "crossword puzzle, crossword", + "street sign", + "traffic light, traffic signal, stoplight", + "book jacket, dust cover, dust jacket, dust wrapper", + "menu", + "plate", + "guacamole", + "consomme", + "hot pot, hotpot", + "trifle", + "ice cream, icecream", + "ice lolly, lolly, lollipop, popsicle", + "French loaf", + "bagel, beigel", + "pretzel", + "cheeseburger", + "hotdog, hot dog, red hot", + "mashed potato", + "head cabbage", + "broccoli", + "cauliflower", + "zucchini, courgette", + "spaghetti squash", + "acorn squash", + "butternut squash", + "cucumber, cuke", + "artichoke, globe artichoke", + "bell pepper", + "cardoon", + "mushroom", + "Granny Smith", + "strawberry", + "orange", + "lemon", + "fig", + "pineapple, ananas", + "banana", + "jackfruit, jak, jack", + "custard apple", + "pomegranate", + "hay", + "carbonara", + "chocolate sauce, chocolate syrup", + "dough", + "meat loaf, meatloaf", + "pizza, pizza pie", + "potpie", + "burrito", + "red wine", + "espresso", + "cup", + "eggnog", + "alp", + "bubble", + "cliff, drop, drop-off", + "coral reef", + "geyser", + "lakeside, lakeshore", + "promontory, headland, head, foreland", + "sandbar, sand bar", + "seashore, coast, seacoast, sea-coast", + "valley, vale", + "volcano", + "ballplayer, baseball player", + "groom, bridegroom", + "scuba diver", + "rapeseed", + "daisy", + "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", + "corn", + "acorn", + "hip, rose hip, rosehip", + "buckeye, horse chestnut, conker", + "coral fungus", + "agaric", + "gyromitra", + "stinkhorn, carrion fungus", + "earthstar", + "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", + "bolete", + "ear, spike, capitulum", + "toilet tissue, toilet paper, bathroom tissue", +]; + +/// Returns the top k classes as well as the associated scores. +pub fn top(tensor: &Tensor, k: i64) -> Vec<(f64, String)> { + let tensor = match tensor.size().as_slice() { + [CLASS_COUNT] => tensor.shallow_clone(), + [1, CLASS_COUNT] => tensor.view((CLASS_COUNT,)), + [1, 1, CLASS_COUNT] => tensor.view((CLASS_COUNT,)), + _ => panic!("unexpected tensor shape {tensor:?}"), + }; + let (values, indexes) = tensor.topk(k, 0, true, true); + let values = Vec::::try_from(values).unwrap(); + let indexes = Vec::::try_from(indexes).unwrap(); + values + .iter() + .zip(indexes.iter()) + .map(|(&value, &index)| (value, CLASSES[index as usize].to_owned())) + .collect() +} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/inception.rs b/v2/dependency-patches/tch-0.24.0/src/vision/inception.rs new file mode 100644 index 0000000000..7bc8738f57 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/vision/inception.rs @@ -0,0 +1,154 @@ +//! InceptionV3. +use crate::{nn, nn::ModuleT, Tensor}; + +fn conv_bn(p: nn::Path, c_in: i64, c_out: i64, ksize: i64, pad: i64, stride: i64) -> impl ModuleT { + let conv2d_cfg = nn::ConvConfig { stride, padding: pad, bias: false, ..Default::default() }; + let bn_cfg = nn::BatchNormConfig { eps: 0.001, ..Default::default() }; + nn::seq_t() + .add(nn::conv2d(&p / "conv", c_in, c_out, ksize, conv2d_cfg)) + .add(nn::batch_norm2d(&p / "bn", c_out, bn_cfg)) + .add_fn(|xs| xs.relu()) +} + +fn conv_bn2(p: nn::Path, c_in: i64, c_out: i64, ksize: [i64; 2], pad: [i64; 2]) -> impl ModuleT { + let conv2d_cfg = + nn::ConvConfigND::<[i64; 2]> { padding: pad, bias: false, ..Default::default() }; + let bn_cfg = nn::BatchNormConfig { eps: 0.001, ..Default::default() }; + nn::seq_t() + .add(nn::conv(&p / "conv", c_in, c_out, ksize, conv2d_cfg)) + .add(nn::batch_norm2d(&p / "bn", c_out, bn_cfg)) + .add_fn(|xs| xs.relu()) +} + +fn max_pool2d(xs: &Tensor, ksize: i64, stride: i64) -> Tensor { + xs.max_pool2d([ksize, ksize], [stride, stride], [0, 0], [1, 1], false) +} + +fn inception_a(p: nn::Path, c_in: i64, c_pool: i64) -> impl ModuleT { + let b1 = conv_bn(&p / "branch1x1", c_in, 64, 1, 0, 1); + let b2_1 = conv_bn(&p / "branch5x5_1", c_in, 48, 1, 0, 1); + let b2_2 = conv_bn(&p / "branch5x5_2", 48, 64, 5, 2, 1); + let b3_1 = conv_bn(&p / "branch3x3dbl_1", c_in, 64, 1, 0, 1); + let b3_2 = conv_bn(&p / "branch3x3dbl_2", 64, 96, 3, 1, 1); + let b3_3 = conv_bn(&p / "branch3x3dbl_3", 96, 96, 3, 1, 1); + let bpool = conv_bn(&p / "branch_pool", c_in, c_pool, 1, 0, 1); + nn::func_t(move |xs, tr| { + let b1 = xs.apply_t(&b1, tr); + let b2 = xs.apply_t(&b2_1, tr).apply_t(&b2_2, tr); + let b3 = xs.apply_t(&b3_1, tr).apply_t(&b3_2, tr).apply_t(&b3_3, tr); + let bpool = xs.avg_pool2d([3, 3], [1, 1], [1, 1], false, true, 9).apply_t(&bpool, tr); + Tensor::cat(&[b1, b2, b3, bpool], 1) + }) +} + +fn inception_b(p: nn::Path, c_in: i64) -> impl ModuleT { + let b1 = conv_bn(&p / "branch3x3", c_in, 384, 3, 0, 2); + let b2_1 = conv_bn(&p / "branch3x3dbl_1", c_in, 64, 1, 0, 1); + let b2_2 = conv_bn(&p / "branch3x3dbl_2", 64, 96, 3, 1, 1); + let b2_3 = conv_bn(&p / "branch3x3dbl_3", 96, 96, 3, 0, 2); + nn::func_t(move |xs, tr| { + let b1 = xs.apply_t(&b1, tr); + let b2 = xs.apply_t(&b2_1, tr).apply_t(&b2_2, tr).apply_t(&b2_3, tr); + let bpool = max_pool2d(xs, 3, 2); + Tensor::cat(&[b1, b2, bpool], 1) + }) +} + +fn inception_c(p: nn::Path, c_in: i64, c7: i64) -> impl ModuleT { + let b1 = conv_bn(&p / "branch1x1", c_in, 192, 1, 0, 1); + + let b2_1 = conv_bn(&p / "branch7x7_1", c_in, c7, 1, 0, 1); + let b2_2 = conv_bn2(&p / "branch7x7_2", c7, c7, [1, 7], [0, 3]); + let b2_3 = conv_bn2(&p / "branch7x7_3", c7, 192, [7, 1], [3, 0]); + + let b3_1 = conv_bn(&p / "branch7x7dbl_1", c_in, c7, 1, 0, 1); + let b3_2 = conv_bn2(&p / "branch7x7dbl_2", c7, c7, [7, 1], [3, 0]); + let b3_3 = conv_bn2(&p / "branch7x7dbl_3", c7, c7, [1, 7], [0, 3]); + let b3_4 = conv_bn2(&p / "branch7x7dbl_4", c7, c7, [7, 1], [3, 0]); + let b3_5 = conv_bn2(&p / "branch7x7dbl_5", c7, 192, [1, 7], [0, 3]); + + let bpool = conv_bn(&p / "branch_pool", c_in, 192, 1, 0, 1); + + nn::func_t(move |xs, tr| { + let b1 = xs.apply_t(&b1, tr); + let b2 = xs.apply_t(&b2_1, tr).apply_t(&b2_2, tr).apply_t(&b2_3, tr); + let b3 = xs + .apply_t(&b3_1, tr) + .apply_t(&b3_2, tr) + .apply_t(&b3_3, tr) + .apply_t(&b3_4, tr) + .apply_t(&b3_5, tr); + let bpool = xs.avg_pool2d([3, 3], [1, 1], [1, 1], false, true, 9).apply_t(&bpool, tr); + Tensor::cat(&[b1, b2, b3, bpool], 1) + }) +} + +fn inception_d(p: nn::Path, c_in: i64) -> impl ModuleT { + let b1_1 = conv_bn(&p / "branch3x3_1", c_in, 192, 1, 0, 1); + let b1_2 = conv_bn(&p / "branch3x3_2", 192, 320, 3, 0, 2); + + let b2_1 = conv_bn(&p / "branch7x7x3_1", c_in, 192, 1, 0, 1); + let b2_2 = conv_bn2(&p / "branch7x7x3_2", 192, 192, [1, 7], [0, 3]); + let b2_3 = conv_bn2(&p / "branch7x7x3_3", 192, 192, [7, 1], [3, 0]); + let b2_4 = conv_bn(&p / "branch7x7x3_4", 192, 192, 3, 0, 2); + + nn::func_t(move |xs, tr| { + let b1 = xs.apply_t(&b1_1, tr).apply_t(&b1_2, tr); + let b2 = xs.apply_t(&b2_1, tr).apply_t(&b2_2, tr).apply_t(&b2_3, tr).apply_t(&b2_4, tr); + let bpool = max_pool2d(xs, 3, 2); + Tensor::cat(&[b1, b2, bpool], 1) + }) +} + +fn inception_e(p: nn::Path, c_in: i64) -> impl ModuleT { + let b1 = conv_bn(&p / "branch1x1", c_in, 320, 1, 0, 1); + + let b2_1 = conv_bn(&p / "branch3x3_1", c_in, 384, 1, 0, 1); + let b2_2a = conv_bn2(&p / "branch3x3_2a", 384, 384, [1, 3], [0, 1]); + let b2_2b = conv_bn2(&p / "branch3x3_2b", 384, 384, [3, 1], [1, 0]); + + let b3_1 = conv_bn(&p / "branch3x3dbl_1", c_in, 448, 1, 0, 1); + let b3_2 = conv_bn(&p / "branch3x3dbl_2", 448, 384, 3, 1, 1); + let b3_3a = conv_bn2(&p / "branch3x3dbl_3a", 384, 384, [1, 3], [0, 1]); + let b3_3b = conv_bn2(&p / "branch3x3dbl_3b", 384, 384, [3, 1], [1, 0]); + + let bpool = conv_bn(&p / "branch_pool", c_in, 192, 1, 0, 1); + + nn::func_t(move |xs, tr| { + let b1 = xs.apply_t(&b1, tr); + + let b2 = xs.apply_t(&b2_1, tr); + let b2 = Tensor::cat(&[b2.apply_t(&b2_2a, tr), b2.apply_t(&b2_2b, tr)], 1); + + let b3 = xs.apply_t(&b3_1, tr).apply_t(&b3_2, tr); + let b3 = Tensor::cat(&[b3.apply_t(&b3_3a, tr), b3.apply_t(&b3_3b, tr)], 1); + + let bpool = xs.avg_pool2d([3, 3], [1, 1], [1, 1], false, true, 9).apply_t(&bpool, tr); + + Tensor::cat(&[b1, b2, b3, bpool], 1) + }) +} + +pub fn v3(p: &nn::Path, nclasses: i64) -> impl ModuleT { + nn::seq_t() + .add(conv_bn(p / "Conv2d_1a_3x3", 3, 32, 3, 0, 2)) + .add(conv_bn(p / "Conv2d_2a_3x3", 32, 32, 3, 0, 1)) + .add(conv_bn(p / "Conv2d_2b_3x3", 32, 64, 3, 1, 1)) + .add_fn(|xs| max_pool2d(&xs.relu(), 3, 2)) + .add(conv_bn(p / "Conv2d_3b_1x1", 64, 80, 1, 0, 1)) + .add(conv_bn(p / "Conv2d_4a_3x3", 80, 192, 3, 0, 1)) + .add_fn(|xs| max_pool2d(&xs.relu(), 3, 2)) + .add(inception_a(p / "Mixed_5b", 192, 32)) + .add(inception_a(p / "Mixed_5c", 256, 64)) + .add(inception_a(p / "Mixed_5d", 288, 64)) + .add(inception_b(p / "Mixed_6a", 288)) + .add(inception_c(p / "Mixed_6b", 768, 128)) + .add(inception_c(p / "Mixed_6c", 768, 160)) + .add(inception_c(p / "Mixed_6d", 768, 160)) + .add(inception_c(p / "Mixed_6e", 768, 192)) + .add(inception_d(p / "Mixed_7a", 768)) + .add(inception_e(p / "Mixed_7b", 1280)) + .add(inception_e(p / "Mixed_7c", 2048)) + .add_fn_t(|xs, train| xs.adaptive_avg_pool2d([1, 1]).dropout(0.5, train).flat_view()) + .add(nn::linear(p / "fc", 2048, nclasses, Default::default())) +} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/mnist.rs b/v2/dependency-patches/tch-0.24.0/src/vision/mnist.rs new file mode 100644 index 0000000000..30e3cc9a94 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/vision/mnist.rs @@ -0,0 +1,69 @@ +//! The MNIST hand-written digit dataset. +//! +//! The files can be obtained from the following link: +//! +use super::dataset::Dataset; +use crate::{Kind, Tensor}; +use std::fs::File; +use std::io::{self, BufReader, Read, Result}; + +fn read_u32(reader: &mut T) -> Result { + let mut b = vec![0u8; 4]; + reader.read_exact(&mut b)?; + let (result, _) = + b.iter().rev().fold((0u64, 1u64), |(s, basis), &x| (s + basis * u64::from(x), basis * 256)); + Ok(result as u32) +} + +fn check_magic_number(reader: &mut T, expected: u32) -> Result<()> { + let magic_number = read_u32(reader)?; + if magic_number != expected { + return Err(io::Error::other(format!( + "incorrect magic number {magic_number} != {expected}" + ))); + } + Ok(()) +} + +fn read_labels_(filename: &std::path::Path) -> Result { + let mut buf_reader = BufReader::new(File::open(filename)?); + check_magic_number(&mut buf_reader, 2049)?; + let samples = read_u32(&mut buf_reader)?; + let mut data = vec![0u8; samples as usize]; + buf_reader.read_exact(&mut data)?; + Ok(Tensor::from_slice(&data).to_kind(Kind::Int64)) +} + +fn read_images_(filename: &std::path::Path) -> Result { + let mut buf_reader = BufReader::new(File::open(filename)?); + check_magic_number(&mut buf_reader, 2051)?; + let samples = read_u32(&mut buf_reader)?; + let rows = read_u32(&mut buf_reader)?; + let cols = read_u32(&mut buf_reader)?; + let data_len = samples * rows * cols; + let mut data = vec![0u8; data_len as usize]; + buf_reader.read_exact(&mut data)?; + let tensor = Tensor::from_slice(&data) + .view((i64::from(samples), i64::from(rows * cols))) + .to_kind(Kind::Float); + Ok(tensor / 255.) +} + +fn read_labels(filename: &std::path::Path) -> Result { + read_labels_(filename) + .map_err(|err| std::io::Error::new(err.kind(), format!("{filename:?} {err}"))) +} + +fn read_images(filename: &std::path::Path) -> Result { + read_images_(filename) + .map_err(|err| std::io::Error::new(err.kind(), format!("{filename:?} {err}"))) +} + +pub fn load_dir>(dir: T) -> Result { + let dir = dir.as_ref(); + let train_images = read_images(&dir.join("train-images-idx3-ubyte"))?; + let train_labels = read_labels(&dir.join("train-labels-idx1-ubyte"))?; + let test_images = read_images(&dir.join("t10k-images-idx3-ubyte"))?; + let test_labels = read_labels(&dir.join("t10k-labels-idx1-ubyte"))?; + Ok(Dataset { train_images, train_labels, test_images, test_labels, labels: 10 }) +} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/mobilenet.rs b/v2/dependency-patches/tch-0.24.0/src/vision/mobilenet.rs new file mode 100644 index 0000000000..31eea884da --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/vision/mobilenet.rs @@ -0,0 +1,83 @@ +//! MobileNet V2 implementation. +//! +use crate::nn::{self, ModuleT}; + +#[allow(clippy::identity_op)] +// Conv2D + BatchNorm2D + ReLU6 +fn cbr(p: nn::Path, c_in: i64, c_out: i64, ks: i64, stride: i64, g: i64) -> impl ModuleT { + let conv2d = nn::ConvConfig { + stride, + padding: (ks - 1) / 2, + groups: g, + bias: false, + ..Default::default() + }; + nn::seq_t() + .add(nn::conv2d(&p / 0, c_in, c_out, ks, conv2d)) + .add(nn::batch_norm2d(&p / 1, c_out, Default::default())) + .add_fn(|xs| xs.relu().clamp_max(6.)) +} + +// Inverted Residual block. +fn inv(p: nn::Path, c_in: i64, c_out: i64, stride: i64, er: i64) -> impl ModuleT { + let c_hidden = er * c_in; + let mut conv = nn::seq_t(); + let mut id = 0; + if er != 1 { + conv = conv.add(cbr(&p / id, c_in, c_hidden, 1, 1, 1)); + id += 1; + } + conv = conv + .add(cbr(&p / id, c_hidden, c_hidden, 3, stride, c_hidden)) + .add(nn::conv2d(&p / (id + 1), c_hidden, c_out, 1, nn::no_bias())) + .add(nn::batch_norm2d(&p / (id + 2), c_out, Default::default())); + nn::func_t(move |xs, train| { + let ys = xs.apply_t(&conv, train); + if stride == 1 && c_in == c_out { + xs + ys + } else { + ys + } + }) +} + +const INVERTED_RESIDUAL_SETTINGS: [(i64, i64, i64, i64); 7] = [ + (1, 16, 1, 1), + (6, 24, 2, 2), + (6, 32, 3, 2), + (6, 64, 4, 2), + (6, 96, 3, 1), + (6, 160, 3, 2), + (6, 320, 1, 1), +]; + +#[allow(clippy::identity_op)] +pub fn v2(p: &nn::Path, nclasses: i64) -> impl ModuleT { + let f_p = p / "features"; + let c_p = p / "classifier"; + let mut c_in = 32; + let mut features = nn::seq_t().add(cbr(&f_p / "0", 3, c_in, 3, 2, 1)); + let mut layer_id = 1; + for &(er, c_out, n, stride) in INVERTED_RESIDUAL_SETTINGS.iter() { + for i in 0..n { + let stride = if i == 0 { stride } else { 1 }; + let f_p = &f_p / layer_id; + features = features.add(inv(&f_p / "conv", c_in, c_out, stride, er)); + c_in = c_out; + layer_id += 1; + } + } + features = features.add(cbr(&f_p / layer_id, c_in, 1280, 1, 1, 1)); + let classifier = nn::seq_t().add_fn_t(|xs, train| xs.dropout(0.2, train)).add(nn::linear( + &c_p / 1, + 1280, + nclasses, + Default::default(), + )); + nn::func_t(move |xs, train| { + xs.apply_t(&features, train) + .mean_dim(Some([2i64].as_slice()), false, crate::Kind::Float) + .mean_dim(Some([2i64].as_slice()), false, crate::Kind::Float) + .apply_t(&classifier, train) + }) +} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/mod.rs b/v2/dependency-patches/tch-0.24.0/src/vision/mod.rs new file mode 100644 index 0000000000..cbfbb436aa --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/vision/mod.rs @@ -0,0 +1,34 @@ +//! The `vision` module groups functions and models related to +//! computer vision. +pub mod dataset; + +pub mod image; + +pub mod mnist; + +pub mod cifar; + +pub mod alexnet; + +pub mod inception; + +pub mod resnet; + +pub mod densenet; + +pub mod vgg; + +pub mod squeezenet; + +pub mod mobilenet; + +pub mod imagenet; + +pub mod efficientnet; + +pub mod convmixer; + +pub mod dinov2; + +#[cfg(feature = "image")] +mod rust_image; diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/resnet.rs b/v2/dependency-patches/tch-0.24.0/src/vision/resnet.rs new file mode 100644 index 0000000000..09c0f28406 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/vision/resnet.rs @@ -0,0 +1,183 @@ +//! ResNet implementation. +//! +//! See "Deep Residual Learning for Image Recognition" He et al. 2015 +//! +use crate::{nn, nn::Conv2D, nn::FuncT, nn::ModuleT}; + +fn conv2d(p: nn::Path, c_in: i64, c_out: i64, ksize: i64, padding: i64, stride: i64) -> Conv2D { + let conv2d_cfg = nn::ConvConfig { stride, padding, bias: false, ..Default::default() }; + nn::conv2d(p, c_in, c_out, ksize, conv2d_cfg) +} + +fn downsample(p: nn::Path, c_in: i64, c_out: i64, stride: i64) -> impl ModuleT { + if stride != 1 || c_in != c_out { + nn::seq_t().add(conv2d(&p / "0", c_in, c_out, 1, 0, stride)).add(nn::batch_norm2d( + &p / "1", + c_out, + Default::default(), + )) + } else { + nn::seq_t() + } +} + +fn basic_block(p: nn::Path, c_in: i64, c_out: i64, stride: i64) -> impl ModuleT { + let conv1 = conv2d(&p / "conv1", c_in, c_out, 3, 1, stride); + let bn1 = nn::batch_norm2d(&p / "bn1", c_out, Default::default()); + let conv2 = conv2d(&p / "conv2", c_out, c_out, 3, 1, 1); + let bn2 = nn::batch_norm2d(&p / "bn2", c_out, Default::default()); + let downsample = downsample(&p / "downsample", c_in, c_out, stride); + nn::func_t(move |xs, train| { + let ys = xs.apply(&conv1).apply_t(&bn1, train).relu().apply(&conv2).apply_t(&bn2, train); + (xs.apply_t(&downsample, train) + ys).relu() + }) +} + +fn basic_layer(p: nn::Path, c_in: i64, c_out: i64, stride: i64, cnt: i64) -> impl ModuleT { + let mut layer = nn::seq_t().add(basic_block(&p / "0", c_in, c_out, stride)); + for block_index in 1..cnt { + layer = layer.add(basic_block(&p / &block_index.to_string(), c_out, c_out, 1)) + } + layer +} + +fn resnet( + p: &nn::Path, + nclasses: Option, + c1: i64, + c2: i64, + c3: i64, + c4: i64, +) -> FuncT<'static> { + let conv1 = conv2d(p / "conv1", 3, 64, 7, 3, 2); + let bn1 = nn::batch_norm2d(p / "bn1", 64, Default::default()); + let layer1 = basic_layer(p / "layer1", 64, 64, 1, c1); + let layer2 = basic_layer(p / "layer2", 64, 128, 2, c2); + let layer3 = basic_layer(p / "layer3", 128, 256, 2, c3); + let layer4 = basic_layer(p / "layer4", 256, 512, 2, c4); + let fc = nclasses.map(|n| nn::linear(p / "fc", 512, n, Default::default())); + nn::func_t(move |xs, train| { + xs.apply(&conv1) + .apply_t(&bn1, train) + .relu() + .max_pool2d([3, 3], [2, 2], [1, 1], [1, 1], false) + .apply_t(&layer1, train) + .apply_t(&layer2, train) + .apply_t(&layer3, train) + .apply_t(&layer4, train) + .adaptive_avg_pool2d([1, 1]) + .flat_view() + .apply_opt(&fc) + }) +} + +/// Creates a ResNet-18 model. +/// +/// Pre-trained weights can be downloaded at the following link: +/// +pub fn resnet18(p: &nn::Path, num_classes: i64) -> FuncT<'static> { + resnet(p, Some(num_classes), 2, 2, 2, 2) +} + +pub fn resnet18_no_final_layer(p: &nn::Path) -> FuncT<'static> { + resnet(p, None, 2, 2, 2, 2) +} + +/// Creates a ResNet-34 model. +/// +/// Pre-trained weights can be downloaded at the following link: +/// +pub fn resnet34(p: &nn::Path, num_classes: i64) -> FuncT<'static> { + resnet(p, Some(num_classes), 3, 4, 6, 3) +} + +pub fn resnet34_no_final_layer(p: &nn::Path) -> FuncT<'static> { + resnet(p, None, 3, 4, 6, 3) +} + +// Bottleneck versions for ResNet 50, 101, and 152. + +fn bottleneck_block(p: nn::Path, c_in: i64, c_out: i64, stride: i64, e: i64) -> impl ModuleT { + let e_dim = e * c_out; + let conv1 = conv2d(&p / "conv1", c_in, c_out, 1, 0, 1); + let bn1 = nn::batch_norm2d(&p / "bn1", c_out, Default::default()); + let conv2 = conv2d(&p / "conv2", c_out, c_out, 3, 1, stride); + let bn2 = nn::batch_norm2d(&p / "bn2", c_out, Default::default()); + let conv3 = conv2d(&p / "conv3", c_out, e_dim, 1, 0, 1); + let bn3 = nn::batch_norm2d(&p / "bn3", e_dim, Default::default()); + let downsample = downsample(&p / "downsample", c_in, e_dim, stride); + nn::func_t(move |xs, train| { + let ys = xs + .apply(&conv1) + .apply_t(&bn1, train) + .relu() + .apply(&conv2) + .apply_t(&bn2, train) + .relu() + .apply(&conv3) + .apply_t(&bn3, train); + (xs.apply_t(&downsample, train) + ys).relu() + }) +} + +fn bottleneck_layer(p: nn::Path, c_in: i64, c_out: i64, stride: i64, cnt: i64) -> impl ModuleT { + let mut layer = nn::seq_t().add(bottleneck_block(&p / "0", c_in, c_out, stride, 4)); + for block_index in 1..cnt { + layer = layer.add(bottleneck_block(&p / &block_index.to_string(), 4 * c_out, c_out, 1, 4)) + } + layer +} + +fn bottleneck_resnet( + p: &nn::Path, + nclasses: Option, + c1: i64, + c2: i64, + c3: i64, + c4: i64, +) -> impl ModuleT { + let conv1 = conv2d(p / "conv1", 3, 64, 7, 3, 2); + let bn1 = nn::batch_norm2d(p / "bn1", 64, Default::default()); + let layer1 = bottleneck_layer(p / "layer1", 64, 64, 1, c1); + let layer2 = bottleneck_layer(p / "layer2", 4 * 64, 128, 2, c2); + let layer3 = bottleneck_layer(p / "layer3", 4 * 128, 256, 2, c3); + let layer4 = bottleneck_layer(p / "layer4", 4 * 256, 512, 2, c4); + let fc = nclasses.map(|n| nn::linear(p / "fc", 4 * 512, n, Default::default())); + nn::func_t(move |xs, train| { + xs.apply(&conv1) + .apply_t(&bn1, train) + .relu() + .max_pool2d([3, 3], [2, 2], [1, 1], [1, 1], false) + .apply_t(&layer1, train) + .apply_t(&layer2, train) + .apply_t(&layer3, train) + .apply_t(&layer4, train) + .adaptive_avg_pool2d([1, 1]) + .flat_view() + .apply_opt(&fc) + }) +} + +pub fn resnet50(p: &nn::Path, num_classes: i64) -> impl ModuleT { + bottleneck_resnet(p, Some(num_classes), 3, 4, 6, 3) +} + +pub fn resnet50_no_final_layer(p: &nn::Path) -> impl ModuleT { + bottleneck_resnet(p, None, 3, 4, 6, 3) +} + +pub fn resnet101(p: &nn::Path, num_classes: i64) -> impl ModuleT { + bottleneck_resnet(p, Some(num_classes), 3, 4, 23, 3) +} + +pub fn resnet101_no_final_layer(p: &nn::Path) -> impl ModuleT { + bottleneck_resnet(p, None, 3, 4, 23, 3) +} + +pub fn resnet152(p: &nn::Path, num_classes: i64) -> impl ModuleT { + bottleneck_resnet(p, Some(num_classes), 3, 8, 36, 3) +} + +pub fn resnet150_no_final_layer(p: &nn::Path) -> impl ModuleT { + bottleneck_resnet(p, None, 3, 8, 36, 3) +} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/rust_image.rs b/v2/dependency-patches/tch-0.24.0/src/vision/rust_image.rs new file mode 100644 index 0000000000..0d78b258ad --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/vision/rust_image.rs @@ -0,0 +1,145 @@ +/// Conversion helpers for the image crate. +use std::convert::TryFrom; + +use image::{ + DynamicImage, EncodableLayout, GrayAlphaImage, GrayImage, Rgb32FImage, RgbImage, Rgba32FImage, + RgbaImage, +}; + +use crate::vision::image::{chw_to_hwc, hwc_to_chw}; +use crate::{Kind, TchError, Tensor}; + +impl<'i> TryFrom<&'i DynamicImage> for Tensor { + type Error = TchError; + + fn try_from(image: &'i DynamicImage) -> Result { + match image { + DynamicImage::ImageLuma8(gray) => Tensor::try_from(gray), + DynamicImage::ImageLumaA8(gray_a) => Tensor::try_from(gray_a), + DynamicImage::ImageRgb8(rgb) => Tensor::try_from(rgb), + DynamicImage::ImageRgba8(rgba) => Tensor::try_from(rgba), + DynamicImage::ImageRgb32F(rgb) => Tensor::try_from(rgb), + DynamicImage::ImageRgba32F(rgba) => Tensor::try_from(rgba), + _ => Err(TchError::Convert("unsupported DynamicImage variant".to_string())), + } + } +} + +impl<'i> TryFrom<&'i GrayImage> for Tensor { + type Error = TchError; + + /// `h * w` => `1 * 1 * h * w` + fn try_from(gray: &'i GrayImage) -> Result { + let size = &[gray.height() as i64, gray.width() as i64, 1]; + let tensor = Tensor::f_from_data_size(gray.as_bytes(), size, Kind::Uint8)?; + Ok(hwc_to_chw(&tensor)) + } +} + +impl<'i> TryFrom<&'i GrayAlphaImage> for Tensor { + type Error = TchError; + + /// `2 * h * w` => `1 * 2 * h * w` + fn try_from(gray: &'i GrayAlphaImage) -> Result { + let size = &[gray.height() as i64, gray.width() as i64, 2]; + let tensor = Tensor::f_from_data_size(gray.as_bytes(), size, Kind::Uint8)?; + Ok(hwc_to_chw(&tensor)) + } +} + +impl<'i> TryFrom<&'i RgbImage> for Tensor { + type Error = TchError; + + /// `h * w * 3` => `1 * 3 * h * w` + fn try_from(rgb: &'i RgbImage) -> Result { + let size = &[rgb.height() as i64, rgb.width() as i64, 3]; + let tensor = Tensor::f_from_data_size(rgb.as_raw(), size, Kind::Uint8)?; + Ok(hwc_to_chw(&tensor)) + } +} + +impl<'i> TryFrom<&'i RgbaImage> for Tensor { + type Error = TchError; + + /// `h * w * 4` => `1 * 4 * h * w` + fn try_from(rgb: &'i RgbaImage) -> Result { + let kind = Kind::Uint8; + let size = &[rgb.height() as i64, rgb.width() as i64, 4]; + let tensor = Tensor::f_from_data_size(rgb.as_raw(), size, kind)?; + Ok(hwc_to_chw(&tensor)) + } +} + +impl<'i> TryFrom<&'i Tensor> for RgbImage { + type Error = TchError; + + /// `1 * 3 * h * w` => `h * w * 3` + fn try_from(value: &'i Tensor) -> Result { + let tensor = assert_tensor_as_image(value, Kind::Uint8, 3)?; + let width = tensor.size()[1] as u32; + let height = tensor.size()[0] as u32; + let length = (width * height * 3) as usize; + let mut buffer = vec![0; length]; + tensor.f_copy_data(&mut buffer, length)?; + RgbImage::from_raw(width, height, buffer) + .ok_or_else(|| TchError::Convert("Failed to convert tensor to image".to_string())) + } +} + +impl<'i> TryFrom<&'i Rgb32FImage> for Tensor { + type Error = TchError; + + /// `h * w * 3` => `1 * 3 * h * w` + fn try_from(rgb: &'i Rgb32FImage) -> Result { + let kind = Kind::Float; + let size = &[rgb.height() as i64, rgb.width() as i64, 3]; + let tensor = Tensor::f_from_data_size(rgb.as_bytes(), size, kind)?; + Ok(hwc_to_chw(&tensor)) + } +} + +impl<'i> TryFrom<&'i Rgba32FImage> for Tensor { + type Error = TchError; + + /// `h * w * 4` => `1 * 4 * h * w` + fn try_from(rgb: &'i Rgba32FImage) -> Result { + let kind = Kind::Float; + let size = &[rgb.height() as i64, rgb.width() as i64, 4]; + let tensor = Tensor::f_from_data_size(rgb.as_bytes(), size, kind)?; + Ok(hwc_to_chw(&tensor)) + } +} + +impl<'i> TryFrom<&'i Tensor> for Rgb32FImage { + type Error = TchError; + + /// `1 * 3 * h * w` => `h * w * 3` + fn try_from(value: &'i Tensor) -> Result { + let tensor = assert_tensor_as_image(value, Kind::Float, 3)?; + let width = tensor.size()[1] as u32; + let height = tensor.size()[0] as u32; + let length = (width * height * 3) as usize; + let mut buffer = vec![0.0; length]; + tensor.f_copy_data(&mut buffer, length)?; + Rgb32FImage::from_raw(width, height, buffer) + .ok_or_else(|| TchError::Convert("Failed to convert tensor to image".to_string())) + } +} + +#[inline] +fn assert_tensor_as_image(tensor: &Tensor, except: Kind, channel: i64) -> Result { + let kind = tensor.kind(); + let size = tensor.size(); + if size.len() != 3 { + let msg = format!("Tensor size is `{size:?}`, except rank 3 tensor"); + Err(TchError::Convert(msg)) + } else if size[0] != channel { + let msg = format!("Tensor size is `{size:?}`, except {channel} channels"); + Err(TchError::Convert(msg)) + } else if kind != except { + let msg = format!("Tensor kind is `{kind:?}`, except `{except:?}` tensor"); + Err(TchError::Convert(msg)) + } else { + Ok(chw_to_hwc(tensor)) + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/squeezenet.rs b/v2/dependency-patches/tch-0.24.0/src/vision/squeezenet.rs new file mode 100644 index 0000000000..d2b9ce86e3 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/vision/squeezenet.rs @@ -0,0 +1,68 @@ +//! SqueezeNet implementation. + +use crate::{nn, nn::Module, nn::ModuleT, Tensor}; + +fn max_pool2d(xs: &Tensor) -> Tensor { + xs.max_pool2d([3, 3], [2, 2], [0, 0], [1, 1], true) +} + +fn fire(p: nn::Path, c_in: i64, c_squeeze: i64, c_exp1: i64, c_exp3: i64) -> impl Module { + let cfg3 = nn::ConvConfig { padding: 1, ..Default::default() }; + let squeeze = nn::conv2d(&p / "squeeze", c_in, c_squeeze, 1, Default::default()); + let exp1 = nn::conv2d(&p / "expand1x1", c_squeeze, c_exp1, 1, Default::default()); + let exp3 = nn::conv2d(&p / "expand3x3", c_squeeze, c_exp3, 3, cfg3); + nn::func(move |xs| { + let xs = xs.apply(&squeeze).relu(); + Tensor::cat(&[xs.apply(&exp1).relu(), xs.apply(&exp3).relu()], 1) + }) +} + +fn squeezenet(p: &nn::Path, v1_0: bool, nclasses: i64) -> impl ModuleT { + let f_p = p / "features"; + let c_p = p / "classifier"; + let initial_conv_cfg = nn::ConvConfig { stride: 2, ..Default::default() }; + let final_conv_cfg = nn::ConvConfig { stride: 1, ..Default::default() }; + let features = if v1_0 { + nn::seq_t() + .add(nn::conv2d(&f_p / "0", 3, 96, 7, initial_conv_cfg)) + .add_fn(|xs| xs.relu()) + .add_fn(max_pool2d) + .add(fire(&f_p / "3", 96, 16, 64, 64)) + .add(fire(&f_p / "4", 128, 16, 64, 64)) + .add(fire(&f_p / "5", 128, 32, 128, 128)) + .add_fn(max_pool2d) + .add(fire(&f_p / "7", 256, 32, 128, 128)) + .add(fire(&f_p / "8", 256, 48, 192, 192)) + .add(fire(&f_p / "9", 384, 48, 192, 192)) + .add(fire(&f_p / "10", 384, 64, 256, 256)) + .add_fn(max_pool2d) + .add(fire(&f_p / "12", 512, 64, 256, 256)) + } else { + nn::seq_t() + .add(nn::conv2d(&f_p / "0", 3, 64, 3, initial_conv_cfg)) + .add_fn(|xs| xs.relu()) + .add_fn(max_pool2d) + .add(fire(&f_p / "3", 64, 16, 64, 64)) + .add(fire(&f_p / "4", 128, 16, 64, 64)) + .add_fn(max_pool2d) + .add(fire(&f_p / "6", 128, 32, 128, 128)) + .add(fire(&f_p / "7", 256, 32, 128, 128)) + .add_fn(max_pool2d) + .add(fire(&f_p / "9", 256, 48, 192, 192)) + .add(fire(&f_p / "10", 384, 48, 192, 192)) + .add(fire(&f_p / "11", 384, 64, 256, 256)) + .add(fire(&f_p / "12", 512, 64, 256, 256)) + }; + features + .add_fn_t(|xs, train| xs.dropout(0.5, train)) + .add(nn::conv2d(&c_p / "1", 512, nclasses, 1, final_conv_cfg)) + .add_fn(|xs| xs.relu().adaptive_avg_pool2d([1, 1]).flat_view()) +} + +pub fn v1_0(p: &nn::Path, nclasses: i64) -> impl ModuleT { + squeezenet(p, true, nclasses) +} + +pub fn v1_1(p: &nn::Path, nclasses: i64) -> impl ModuleT { + squeezenet(p, false, nclasses) +} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/vgg.rs b/v2/dependency-patches/tch-0.24.0/src/vision/vgg.rs new file mode 100644 index 0000000000..7adf0bd02b --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/vision/vgg.rs @@ -0,0 +1,100 @@ +//! VGG models. +//! +//! Pre-trained weights for the vgg-13/16/19 models can be found here: +//! +//! +//! +use crate::{nn, nn::Conv2D, nn::SequentialT}; + +// Each list element contains multiple convolutions with some specified number +// of features followed by a single max-pool layer. +fn layers_a() -> Vec> { + vec![vec![64], vec![128], vec![256, 256], vec![512, 512], vec![512, 512]] +} + +fn layers_b() -> Vec> { + vec![vec![64, 64], vec![128, 128], vec![256, 256], vec![512, 512], vec![512, 512]] +} +fn layers_d() -> Vec> { + vec![ + vec![64, 64], + vec![128, 128], + vec![256, 256, 256], + vec![512, 512, 512], + vec![512, 512, 512], + ] +} +fn layers_e() -> Vec> { + vec![ + vec![64, 64], + vec![128, 128], + vec![256, 256, 256, 256], + vec![512, 512, 512, 512], + vec![512, 512, 512, 512], + ] +} + +fn conv2d(p: nn::Path, c_in: i64, c_out: i64) -> Conv2D { + let conv2d_cfg = nn::ConvConfig { stride: 1, padding: 1, ..Default::default() }; + nn::conv2d(p, c_in, c_out, 3, conv2d_cfg) +} + +fn vgg(p: &nn::Path, cfg: Vec>, nclasses: i64, batch_norm: bool) -> SequentialT { + let c = p / "classifier"; + let mut seq = nn::seq_t(); + let f = p / "features"; + let mut c_in = 3; + for channels in cfg.into_iter() { + for &c_out in channels.iter() { + let l = seq.len(); + seq = seq.add(conv2d(&f / &l.to_string(), c_in, c_out)); + if batch_norm { + let l = seq.len(); + seq = seq.add(nn::batch_norm2d(&f / &l.to_string(), c_out, Default::default())); + }; + seq = seq.add_fn(|xs| xs.relu()); + c_in = c_out; + } + seq = seq.add_fn(|xs| xs.max_pool2d_default(2)); + } + seq.add_fn(|xs| xs.flat_view()) + .add(nn::linear(&c / "0", 512 * 7 * 7, 4096, Default::default())) + .add_fn(|xs| xs.relu()) + .add_fn_t(|xs, train| xs.dropout(0.5, train)) + .add(nn::linear(&c / "3", 4096, 4096, Default::default())) + .add_fn(|xs| xs.relu()) + .add_fn_t(|xs, train| xs.dropout(0.5, train)) + .add(nn::linear(&c / "6", 4096, nclasses, Default::default())) +} + +pub fn vgg11(p: &nn::Path, nclasses: i64) -> SequentialT { + vgg(p, layers_a(), nclasses, false) +} + +pub fn vgg11_bn(p: &nn::Path, nclasses: i64) -> SequentialT { + vgg(p, layers_a(), nclasses, true) +} + +pub fn vgg13(p: &nn::Path, nclasses: i64) -> SequentialT { + vgg(p, layers_b(), nclasses, false) +} + +pub fn vgg13_bn(p: &nn::Path, nclasses: i64) -> SequentialT { + vgg(p, layers_b(), nclasses, true) +} + +pub fn vgg16(p: &nn::Path, nclasses: i64) -> SequentialT { + vgg(p, layers_d(), nclasses, false) +} + +pub fn vgg16_bn(p: &nn::Path, nclasses: i64) -> SequentialT { + vgg(p, layers_d(), nclasses, true) +} + +pub fn vgg19(p: &nn::Path, nclasses: i64) -> SequentialT { + vgg(p, layers_e(), nclasses, false) +} + +pub fn vgg19_bn(p: &nn::Path, nclasses: i64) -> SequentialT { + vgg(p, layers_e(), nclasses, true) +} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/device.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/device.rs new file mode 100644 index 0000000000..9dadd192e7 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/wrappers/device.rs @@ -0,0 +1,120 @@ +//! Devices on which tensor computations are run. + +/// A torch device. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub enum Device { + /// The main CPU device. + Cpu, + /// The main GPU device. + Cuda(usize), + /// The main MPS device. + Mps, + /// The main Vulkan device. + Vulkan, +} + +/// Cuda related helper functions. +pub enum Cuda {} +impl Cuda { + /// Returns the number of CUDA devices available. + pub fn device_count() -> i64 { + let res = unsafe_torch!(torch_sys::cuda::atc_cuda_device_count()); + i64::from(res) + } + + /// Returns true if at least one CUDA device is available. + pub fn is_available() -> bool { + unsafe_torch!(torch_sys::cuda::atc_cuda_is_available()) != 0 + } + + /// Returns true if CUDA is available, and CuDNN is available. + pub fn cudnn_is_available() -> bool { + unsafe_torch!(torch_sys::cuda::atc_cudnn_is_available()) != 0 + } + + /// Sets the seed for the current GPU. + /// + /// # Arguments + /// + /// * `seed` - An unsigned 64bit int to be used as seed. + pub fn manual_seed(seed: u64) { + unsafe_torch!(torch_sys::cuda::atc_manual_seed(seed)); + } + + /// Sets the seed for all available GPUs. + /// + /// # Arguments + /// + /// * `seed` - An unsigned 64bit int to be used as seed. + pub fn manual_seed_all(seed: u64) { + unsafe_torch!(torch_sys::cuda::atc_manual_seed_all(seed)); + } + + /// Waits for all kernels in all streams on a CUDA device to complete. + /// + /// # Arguments + /// + /// * `device_index` - A signed 64bit int to indice which device to wait for. + pub fn synchronize(device_index: i64) { + unsafe_torch!(torch_sys::cuda::atc_synchronize(device_index)); + } + + /// Returns true if cudnn is enabled by the user. + /// + /// This does not indicate whether cudnn is actually usable. + pub fn user_enabled_cudnn() -> bool { + unsafe_torch!(torch_sys::cuda::atc_user_enabled_cudnn()) != 0 + } + + /// Enable or disable cudnn. + pub fn set_user_enabled_cudnn(b: bool) { + unsafe_torch!(torch_sys::cuda::atc_set_user_enabled_cudnn(i32::from(b))) + } + + /// Sets cudnn benchmark mode. + /// + /// When set cudnn will try to optimize the generators durning + /// the first network runs and then use the optimized architecture + /// in the following runs. This can result in significant performance + /// improvements. + pub fn cudnn_set_benchmark(b: bool) { + unsafe_torch!(torch_sys::cuda::atc_set_benchmark_cudnn(i32::from(b))) + } +} + +impl Device { + pub(super) fn c_int(self) -> libc::c_int { + match self { + Device::Cpu => -1, + Device::Cuda(device_index) => device_index as libc::c_int, + Device::Mps => -2, + Device::Vulkan => -3, + } + } + + pub(super) fn from_c_int(v: libc::c_int) -> Self { + match v { + -1 => Device::Cpu, + -2 => Device::Mps, + -3 => Device::Vulkan, + index if index >= 0 => Device::Cuda(index as usize), + _ => panic!("unexpected device {v}"), + } + } + + /// Returns a GPU device if available, else default to CPU. + pub fn cuda_if_available() -> Device { + if Cuda::is_available() { + Device::Cuda(0) + } else { + Device::Cpu + } + } + + pub fn is_cuda(self) -> bool { + match self { + Device::Cuda(_) => true, + Device::Cpu | Device::Mps | Device::Vulkan => false, + } + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/image.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/image.rs new file mode 100644 index 0000000000..bc4a7b186e --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/wrappers/image.rs @@ -0,0 +1,35 @@ +use super::tensor::Tensor; +use super::utils::path_to_cstring; +use crate::TchError; +use libc::c_int; +use std::path::Path; + +/// On success returns a tensor of shape [width, height, channels]. +pub fn load_hwc>(path: T) -> Result { + let path = path_to_cstring(path)?; + let c_tensor = unsafe_torch_err!(torch_sys::at_load_image(path.as_ptr())); + Ok(Tensor { c_tensor }) +} + +/// On success returns a tensor of shape [width, height, channels]. +pub fn load_hwc_from_mem(data: &[u8]) -> Result { + let c_tensor = + unsafe_torch_err!(torch_sys::at_load_image_from_memory(data.as_ptr(), data.len())); + Ok(Tensor { c_tensor }) +} + +/// Expects a tensor of shape [width, height, channels]. +pub fn save_hwc>(t: &Tensor, path: T) -> Result<(), TchError> { + let path = path_to_cstring(path)?; + let _ = unsafe_torch_err!(torch_sys::at_save_image(t.c_tensor, path.as_ptr())); + Ok(()) +} + +/// Expects a tensor of shape [width, height, channels]. +/// On success returns a tensor of shape [width, height, channels]. +pub fn resize_hwc(t: &Tensor, out_w: i64, out_h: i64) -> Result { + let c_tensor = unsafe_torch_err!({ + torch_sys::at_resize_image(t.c_tensor, out_w as c_int, out_h as c_int) + }); + Ok(Tensor { c_tensor }) +} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/jit.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/jit.rs new file mode 100644 index 0000000000..f3c316a837 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/wrappers/jit.rs @@ -0,0 +1,879 @@ +//! JIT interface to run model trained/saved using PyTorch Python API. +use super::utils::{path_to_cstring, ptr_to_string}; +use super::{device::Device, kind::Kind}; +use crate::{nn::Path, TchError, Tensor}; +use libc::{c_int, c_void}; +use std::borrow::Borrow; +use std::convert::TryFrom; +use torch_sys::*; + +/// Argument and output values for JIT models. These represent arbitrary values, +/// e.g. tensors, atomic values, pairs of values, etc. +#[derive(Debug, PartialEq)] +#[non_exhaustive] +pub enum IValue { + None, + Tensor(crate::Tensor), + Double(f64), + Int(i64), + Bool(bool), + Tuple(Vec), + IntList(Vec), + DoubleList(Vec), + BoolList(Vec), + String(String), + StringList(Vec), + TensorList(Vec), + GenericList(Vec), + // We use a vec to represent dictionaries as f64 does not implement + // Eq or Hash out of the box in rust. TODO: improve this? + GenericDict(Vec<(IValue, IValue)>), + Object(Object), + Device(Device), +} + +impl IValue { + fn type_str(self) -> &'static str { + match self { + IValue::None => "None", + IValue::Tensor(_) => "Tensor", + IValue::Double(_) => "Double", + IValue::Int(_) => "Int", + IValue::Bool(_) => "Bool", + IValue::Tuple(_) => "Tuple", + IValue::IntList(_) => "IntList", + IValue::DoubleList(_) => "DoubleList", + IValue::BoolList(_) => "BoolList", + IValue::String(_) => "String", + IValue::StringList(_) => "StringList", + IValue::TensorList(_) => "TensorList", + IValue::GenericList(_) => "GenericList", + IValue::GenericDict(_) => "GenericDict", + IValue::Object(_) => "Object", + IValue::Device(_) => "Device", + } + } +} + +impl From<()> for IValue { + fn from((): ()) -> Self { + IValue::None + } +} + +impl, T2: Into> From<(T1, T2)> for IValue { + fn from((p1, p2): (T1, T2)) -> Self { + IValue::Tuple(vec![p1.into(), p2.into()]) + } +} + +impl, T2: Into, T3: Into> From<(T1, T2, T3)> for IValue { + fn from((p1, p2, p3): (T1, T2, T3)) -> Self { + IValue::Tuple(vec![p1.into(), p2.into(), p3.into()]) + } +} + +impl, T2: Into, T3: Into, T4: Into> From<(T1, T2, T3, T4)> + for IValue +{ + fn from((p1, p2, p3, p4): (T1, T2, T3, T4)) -> Self { + IValue::Tuple(vec![p1.into(), p2.into(), p3.into(), p4.into()]) + } +} + +impl TryFrom for (T1, T2) +where + T1: TryFrom, + TchError: From, + T2: TryFrom, + TchError: From, +{ + type Error = TchError; + fn try_from(value: IValue) -> Result { + match value { + IValue::GenericList(mut vec) | IValue::Tuple(mut vec) => { + if vec.len() == 2 { + let t2 = T2::try_from(vec.pop().unwrap())?; + let t1 = T1::try_from(vec.pop().unwrap())?; + Ok((t1, t2)) + } else { + Err(TchError::Kind(format!( + "unable to unpack ivalue, expected a tuple of len 2 got {}", + vec.len() + ))) + } + } + _ => Err(TchError::Kind(format!( + "unable to unpack ivalue, expected a tuple got {}", + value.type_str() + ))), + } + } +} + +impl TryFrom for (T1, T2, T3) +where + T1: TryFrom, + TchError: From, + T2: TryFrom, + TchError: From, + T3: TryFrom, + TchError: From, +{ + type Error = TchError; + fn try_from(value: IValue) -> Result { + match value { + IValue::GenericList(mut vec) | IValue::Tuple(mut vec) => { + if vec.len() == 3 { + let t3 = T3::try_from(vec.pop().unwrap())?; + let t2 = T2::try_from(vec.pop().unwrap())?; + let t1 = T1::try_from(vec.pop().unwrap())?; + Ok((t1, t2, t3)) + } else { + Err(TchError::Kind(format!( + "unable to unpack ivalue, expected a tuple of len 3 got {}", + vec.len() + ))) + } + } + _ => Err(TchError::Kind(format!( + "unable to unpack ivalue, expected a tuple got {}", + value.type_str() + ))), + } + } +} + +impl TryFrom for (T1, T2, T3, T4) +where + T1: TryFrom, + TchError: From, + T2: TryFrom, + TchError: From, + T3: TryFrom, + TchError: From, + T4: TryFrom, + TchError: From, +{ + type Error = TchError; + fn try_from(value: IValue) -> Result { + match value { + IValue::GenericList(mut vec) | IValue::Tuple(mut vec) => { + if vec.len() == 4 { + let t4 = T4::try_from(vec.pop().unwrap())?; + let t3 = T3::try_from(vec.pop().unwrap())?; + let t2 = T2::try_from(vec.pop().unwrap())?; + let t1 = T1::try_from(vec.pop().unwrap())?; + Ok((t1, t2, t3, t4)) + } else { + Err(TchError::Kind(format!( + "unable to unpack ivalue, expected a tuple of len 4 got {}", + vec.len() + ))) + } + } + _ => Err(TchError::Kind(format!( + "unable to unpack ivalue, expected a tuple got {}", + value.type_str() + ))), + } + } +} + +macro_rules! impl_from { + ($type_:ty, $cons:ident) => { + impl From<$type_> for IValue { + fn from(v: $type_) -> Self { + IValue::$cons(v) + } + } + + impl TryFrom for $type_ { + type Error = TchError; + fn try_from(value: IValue) -> Result<$type_, TchError> { + match value { + IValue::$cons(t) => Ok(t), + _ => Err(TchError::Kind(format!( + "unable to unpack ivalue, expected {} got {}", + std::stringify!($cons), + value.type_str() + ))), + } + } + } + + // A generic trait for Option would seem nicer but because + // of E0119, this is currently hard to do. + // See https://github.com/rust-lang/rust/issues/50133 + impl TryFrom for Option<$type_> { + type Error = TchError; + fn try_from(value: IValue) -> Result { + match value { + IValue::None => Ok(None), + IValue::$cons(t) => Ok(Some(t)), + _ => Err(TchError::Kind(format!( + "unable to unpack ivalue, expected {} or None got {}", + std::stringify!($cons), + value.type_str() + ))), + } + } + } + }; +} + +impl_from!(i64, Int); +impl_from!(f64, Double); +impl_from!(bool, Bool); +impl_from!(String, String); +impl_from!(Tensor, Tensor); +impl_from!(Vec, IntList); +impl_from!(Vec, DoubleList); +impl_from!(Vec, BoolList); +impl_from!(Vec, StringList); +impl_from!(Vec, TensorList); +impl_from!(Vec, GenericList); +impl_from!(Vec<(IValue, IValue)>, GenericDict); +impl_from!(Object, Object); +impl_from!(Device, Device); + +impl From<&str> for IValue { + fn from(s: &str) -> Self { + IValue::String(s.to_string()) + } +} + +impl IValue { + #![allow(unused_unsafe)] + pub(super) fn to_c(&self) -> Result<*mut CIValue, TchError> { + let c = unsafe_torch_err!(match self { + IValue::Tensor(tensor) => ati_tensor(tensor.c_tensor), + IValue::Int(i) => ati_int(*i), + IValue::None => ati_none(), + IValue::Double(f) => ati_double(*f), + IValue::Bool(b) => ati_bool(i32::from(*b)), + IValue::Tuple(v) => { + let v = v.iter().map(Self::to_c).collect::, TchError>>()?; + let tuple = ati_tuple(v.as_ptr(), v.len() as c_int); + for x in v { + ati_free(x); + } + + tuple + } + IValue::GenericList(v) => { + let v = v.iter().map(Self::to_c).collect::, TchError>>()?; + let list = ati_generic_list(v.as_ptr(), v.len() as c_int); + for x in v { + ati_free(x); + } + list + } + IValue::IntList(v) => ati_int_list(v.as_ptr(), v.len() as c_int), + IValue::DoubleList(v) => ati_double_list(v.as_ptr(), v.len() as c_int), + IValue::BoolList(v) => { + let v: Vec = v.iter().map(|&b| libc::c_char::from(b)).collect(); + ati_bool_list(v.as_ptr(), v.len() as c_int) + } + IValue::TensorList(v) => { + let v = v.iter().map(|t| t.c_tensor).collect::>(); + ati_tensor_list(v.as_ptr(), v.len() as c_int) + } + IValue::String(string) => { + let c_str = std::ffi::CString::new(string.as_str())?; + ati_string(c_str.as_ptr()) + } + IValue::StringList(strings) => { + let mut v = vec![]; + for s in strings { + v.push(std::ffi::CString::new(s.as_str())?); + } + let v_ptr: Vec<_> = v.iter().map(|s| s.as_ptr()).collect(); + ati_string_list(v_ptr.as_ptr(), v.len() as c_int) + } + IValue::GenericDict(dict) => { + let v = dict + .iter() + .flat_map(|(k, v)| vec![Self::to_c(k), Self::to_c(v)]) + .collect::, TchError>>()?; + let dict = ati_generic_dict(v.as_ptr(), dict.len() as c_int); + for x in v { + ati_free(x); + } + dict + } + IValue::Object(Object { c_ivalue }) => { + // Clone the object if necessary before passing the pointer to the C++ side. + unsafe_torch_err!(ati_clone(*c_ivalue)) + } + IValue::Device(device) => { + ati_device(device.c_int()) + } + }); + Ok(c) + } + + // This consumes the pointer and frees the associated memory (unless it is an Object). + pub(super) fn from_c(c_ivalue: *mut CIValue) -> Result { + let mut free = true; + let tag = unsafe_torch_err!(ati_tag(c_ivalue)); + let v = match tag { + 0 => IValue::None, + 1 => { + let c_tensor = unsafe_torch_err!(ati_to_tensor(c_ivalue)); + IValue::Tensor(crate::Tensor { c_tensor }) + } + 2 => IValue::Double(unsafe_torch_err!(ati_to_double(c_ivalue))), + 3 => IValue::Int(unsafe_torch_err!(ati_to_int(c_ivalue))), + 4 => { + let b = unsafe_torch_err!(ati_to_bool(c_ivalue)); + if b < 0 { + return Err(TchError::Kind(format!("unexpected bool value {b}"))); + } + IValue::Bool(b != 0) + } + 5 => { + let len = unsafe_torch_err!(ati_tuple_length(c_ivalue)); + let mut c_ivalues: Vec<_> = + (0..len).map(|_| std::ptr::null_mut::()).collect(); + unsafe_torch_err!(ati_to_tuple(c_ivalue, c_ivalues.as_mut_ptr(), len)); + let vec: Result, _> = + c_ivalues.iter().map(|&c_ivalue| Self::from_c(c_ivalue)).collect(); + IValue::Tuple(vec?) + } + 6 => { + let len = unsafe_torch_err!(ati_length(c_ivalue)); + let mut c_array = vec![0i64; len as usize]; + unsafe_torch_err!(ati_to_int_list(c_ivalue, c_array.as_mut_ptr(), len)); + IValue::IntList(c_array) + } + 7 => { + let len = unsafe_torch_err!(ati_length(c_ivalue)); + let mut c_array = vec![0f64; len as usize]; + unsafe_torch_err!(ati_to_double_list(c_ivalue, c_array.as_mut_ptr(), len)); + IValue::DoubleList(c_array) + } + 8 => { + let len = unsafe_torch_err!(ati_length(c_ivalue)); + let mut c_array = vec![0_i8; len as usize]; + let c_array_ptr = c_array.as_mut_ptr() as *mut libc::c_char; + unsafe_torch_err!(ati_to_bool_list(c_ivalue, c_array_ptr, len)); + IValue::BoolList(c_array.iter().map(|&x| x != 0).collect()) + } + 9 => { + let ptr = unsafe_torch_err!(ati_to_string(c_ivalue)); + let string = match unsafe { ptr_to_string(ptr) } { + None => return Err(TchError::Kind("nullptr representation".to_string())), + Some(s) => s, + }; + IValue::String(string) + } + 10 => { + let len = unsafe_torch_err!(ati_length(c_ivalue)); + let mut c_tensors: Vec<_> = + (0..len).map(|_| std::ptr::null_mut::()).collect(); + unsafe_torch_err!(ati_to_tensor_list(c_ivalue, c_tensors.as_mut_ptr(), len)); + let vec: Vec<_> = c_tensors.iter().map(|&c_tensor| Tensor { c_tensor }).collect(); + IValue::TensorList(vec) + } + 12 => { + let len = unsafe_torch_err!(ati_length(c_ivalue)); + let mut c_ivalues: Vec<_> = + (0..len).map(|_| std::ptr::null_mut::()).collect(); + unsafe_torch_err!(ati_to_generic_list(c_ivalue, c_ivalues.as_mut_ptr(), len)); + let vec: Result, _> = + c_ivalues.iter().map(|&c_ivalue| Self::from_c(c_ivalue)).collect(); + IValue::GenericList(vec?) + } + 13 => { + let len = unsafe_torch_err!(ati_length(c_ivalue)); + let mut c_ivalues: Vec<_> = + (0..2 * len).map(|_| std::ptr::null_mut::()).collect(); + unsafe_torch_err!(ati_to_generic_dict(c_ivalue, c_ivalues.as_mut_ptr(), len)); + let mut res: Vec<(IValue, IValue)> = vec![]; + for i in 0..(len as usize) { + let key = Self::from_c(c_ivalues[2 * i])?; + let value = Self::from_c(c_ivalues[2 * i + 1])?; + res.push((key, value)) + } + IValue::GenericDict(res) + } + 14 => { + free = false; + IValue::Object(Object { c_ivalue }) + } + _ => return Err(TchError::Kind(format!("unhandled tag {tag}"))), + }; + if free { + unsafe_torch_err!(ati_free(c_ivalue)); + } + Ok(v) + } +} + +/// A jit PyTorch module. +/// +/// These modules can be created via the +/// [TorchScript python api](https://pytorch.org/docs/stable/jit.html). +#[derive(Debug)] +pub struct CModule { + pub(super) c_module: *mut CModule_, +} + +unsafe impl Send for CModule {} + +unsafe impl Sync for CModule {} + +impl Drop for CModule { + fn drop(&mut self) { + unsafe_torch!(atm_free(self.c_module)) + } +} + +impl CModule { + /// Loads a PyTorch saved JIT model from a file. + pub fn load>(path: T) -> Result { + let path = path_to_cstring(path)?; + let c_module = unsafe_torch_err!(atm_load(path.as_ptr())); + Ok(CModule { c_module }) + } + + /// Loads a PyTorch saved JIT model from a file onto the given device. + /// + /// This function loads the model directly on the specified device, + /// which means it also allows loading a GPU model on the CPU without having a CUDA enabled GPU. + pub fn load_on_device>( + path: T, + device: Device, + ) -> Result { + let path = path_to_cstring(path)?; + let c_module = unsafe_torch_err!(atm_load_on_device(path.as_ptr(), device.c_int())); + Ok(CModule { c_module }) + } + + /// Loads a PyTorch saved JIT model from a read instance. + pub fn load_data(f: &mut T) -> Result { + let mut buffer = Vec::new(); + f.read_to_end(&mut buffer)?; + let buffer_ptr = buffer.as_ptr() as *const libc::c_char; + let c_module = unsafe_torch_err!(atm_load_str(buffer_ptr, buffer.len())); + Ok(CModule { c_module }) + } + + /// Loads a PyTorch saved JIT model from a read instance. + /// + /// This function loads the model directly on the specified device, + /// which means it also allows loading a GPU model on the CPU without having a CUDA enabled GPU. + pub fn load_data_on_device( + f: &mut T, + device: Device, + ) -> Result { + let mut buffer = Vec::new(); + f.read_to_end(&mut buffer)?; + let buffer_ptr = buffer.as_ptr() as *const libc::c_char; + let c_module = + unsafe_torch_err!(atm_load_str_on_device(buffer_ptr, buffer.len(), device.c_int())); + Ok(CModule { c_module }) + } + + /// Performs the forward pass for a model on some specified tensor inputs. This is equivalent + /// to calling method_ts with the 'forward' method name, and returns a single tensor. + pub fn forward_ts>(&self, ts: &[T]) -> Result { + let ts: Vec<_> = ts.iter().map(|x| x.borrow().c_tensor).collect(); + let c_tensor = + unsafe_torch_err!(atm_forward(self.c_module, ts.as_ptr(), ts.len() as c_int)); + Ok(Tensor { c_tensor }) + } + + /// Performs the forward pass for a model on some specified ivalue inputs. This is equivalent + /// to calling method_is with the 'forward' method name, and returns an arbitrary ivalue. + pub fn forward_is>(&self, ts: &[T]) -> Result { + let ts = ts.iter().map(|x| x.borrow().to_c()).collect::, TchError>>()?; + let c_ivalue = + unsafe_torch_err!(atm_forward_(self.c_module, ts.as_ptr(), ts.len() as c_int)); + for x in ts { + unsafe { ati_free(x) } + } + IValue::from_c(c_ivalue) + } + + /// Runs a specified entry point for a model on some given tensor inputs. + pub fn method_ts>( + &self, + method_name: &str, + ts: &[T], + ) -> Result { + let ts: Vec<_> = ts.iter().map(|x| x.borrow().c_tensor).collect(); + let method_name = std::ffi::CString::new(method_name)?; + let c_tensor = unsafe_torch_err!(atm_method( + self.c_module, + method_name.as_ptr(), + ts.as_ptr(), + ts.len() as c_int + )); + Ok(Tensor { c_tensor }) + } + + /// Runs a specified entry point for a model on some given ivalue inputs. + pub fn method_is>( + &self, + method_name: &str, + ts: &[T], + ) -> Result { + let ts = ts.iter().map(|x| x.borrow().to_c()).collect::, TchError>>()?; + let method_name = std::ffi::CString::new(method_name)?; + let c_ivalue = unsafe_torch_err!(atm_method_( + self.c_module, + method_name.as_ptr(), + ts.as_ptr(), + ts.len() as c_int + )); + for x in ts { + unsafe { ati_free(x) } + } + IValue::from_c(c_ivalue) + } + + /// Create a specified custom JIT class object with the given class name, eg: `__torch__.foo.Bar` + pub fn create_class_is>( + &self, + clz_name: &str, + ts: &[T], + ) -> Result { + let ts = ts.iter().map(|x| x.borrow().to_c()).collect::, TchError>>()?; + let clz_name = std::ffi::CString::new(clz_name)?; + let c_ivalue = unsafe_torch_err!(atm_create_class_( + self.c_module, + clz_name.as_ptr(), + ts.as_ptr(), + ts.len() as c_int + )); + for x in ts { + unsafe { ati_free(x) } + } + IValue::from_c(c_ivalue) + } + + /// Switches the module to evaluation mode. + pub fn f_set_eval(&mut self) -> Result<(), TchError> { + unsafe_torch_err!(atm_eval(self.c_module)); + Ok(()) + } + + /// Switches the module to evaluation mode. + pub fn set_eval(&mut self) { + self.f_set_eval().unwrap(); + } + + /// Switches the module to training mode. + pub fn f_set_train(&mut self) -> Result<(), TchError> { + unsafe_torch_err!(atm_train(self.c_module)); + Ok(()) + } + + /// Switches the module to training mode. + pub fn set_train(&mut self) { + self.f_set_train().unwrap(); + } + + /// Moves the module to a different device and converts the kind. + pub fn to(&mut self, device: Device, kind: Kind, non_blocking: bool) { + unsafe_torch!(atm_to(self.c_module, device.c_int(), kind.c_int(), non_blocking)); + } + + /// Saves a module to a given path. + pub fn save>(&self, path: T) -> Result<(), TchError> { + let path = path_to_cstring(path)?; + unsafe_torch_err!(atm_save(self.c_module, path.as_ptr())); + Ok(()) + } + + /// Loads some named tensors from a module + pub fn named_parameters(&self) -> Result, TchError> { + let mut v: Vec<(String, Tensor)> = vec![]; + unsafe_torch_err!(atm_named_parameters( + self.c_module, + &mut v as *mut _ as *mut c_void, + super::tensor::add_callback + )); + Ok(v) + } + + /// Create a new module by tracing the application of the specified function on + /// the given inputs. + pub fn create_by_tracing( + modl_name: &str, + fn_name: &str, + inputs: &[Tensor], + closure: &mut F, + ) -> Result + where + F: FnMut(&[Tensor]) -> Vec, + { + let modl_name = std::ffi::CString::new(modl_name)?; + let fn_name = std::ffi::CString::new(fn_name)?; + let c_inputs = inputs.iter().map(|tensor| tensor.c_tensor).collect::>(); + let c_module = unsafe_torch_err!(atm_create_for_tracing( + modl_name.as_ptr(), + c_inputs.as_ptr(), + c_inputs.len() as c_int + )); + let outputs = closure(inputs); + let c_outputs = outputs.iter().map(|tensor| tensor.c_tensor).collect::>(); + unsafe_torch_err!(atm_end_tracing( + c_module, + fn_name.as_ptr(), + c_outputs.as_ptr(), + c_outputs.len() as c_int, + )); + Ok(CModule { c_module }) + } +} + +/// The trainable version of a jit PyTorch module. +/// +/// These modules can be created via the +/// [TorchScript python api](https://pytorch.org/docs/stable/jit.html). +#[derive(Debug)] +pub struct TrainableCModule { + pub(crate) inner: CModule, +} + +impl TrainableCModule { + /// Loads a PyTorch saved JIT module from a file. + /// + /// This function also adds the tensors from the JIT module to the VarStore path + /// passed as argument so that the module can be trained. + pub fn load>(module_path: T, path: Path) -> Result { + let inner = CModule::load_on_device(module_path, path.device())?; + for (name, tensor) in inner.named_parameters()? { + let requires_grad = tensor.requires_grad(); + let _t = path.add(&name.replace('.', "_"), tensor, requires_grad); + } + Ok(TrainableCModule { inner }) + } + + /// Loads a PyTorch saved JIT model from a read instance. + /// + /// This function also adds the tensors from the JIT module to the VarStore path + /// passed as argument so that the module can be trained. + pub fn load_data(data: &mut T, path: Path) -> Result { + let inner = CModule::load_data_on_device(data, path.device())?; + for (name, tensor) in inner.named_parameters()? { + let requires_grad = tensor.requires_grad(); + let _t = path.add(&name.replace('.', "_"), tensor, requires_grad); + } + Ok(TrainableCModule { inner }) + } + + pub fn save>(&self, module_path: T) -> Result<(), TchError> { + self.inner.save(module_path) + } + + /// Switches the module to training mode. + pub fn f_set_train(&mut self) -> Result<(), TchError> { + self.inner.f_set_train() + } + + /// Switches the module to training mode. + pub fn set_train(&mut self) { + self.inner.set_train() + } + + /// Switches the module to evaluation mode. + pub fn f_set_eval(&mut self) -> Result<(), TchError> { + self.inner.f_set_eval() + } + + /// Switches the module to evaluation mode. + pub fn set_eval(&mut self) { + self.inner.set_eval() + } + + /// Performs the forward pass for a model on some specified tensor inputs. + pub fn forward_ts>(&self, ts: &[T]) -> Result { + self.inner.forward_ts(ts) + } + + /// Performs the forward pass for a model on some specified ivalue inputs. + pub fn forward_is>(&self, ts: &[T]) -> Result { + self.inner.forward_is(ts) + } + + /// Runs a specified entry point for a model on some given tensor inputs. + pub fn method_ts>( + &self, + method_name: &str, + ts: &[T], + ) -> Result { + self.inner.method_ts(method_name, ts) + } + + /// Runs a specified entry point for a model on some given ivalue inputs. + pub fn method_is>( + &self, + method_name: &str, + ts: &[T], + ) -> Result { + self.inner.method_is(method_name, ts) + } +} + +/// Returns whether profiling mode is set or not. +pub fn f_get_profiling_mode() -> Result { + Ok(unsafe_torch_err!(atm_get_profiling_mode()) != 0) +} + +/// Returns whether profiling mode is set or not. +pub fn get_profiling_mode() -> bool { + f_get_profiling_mode().unwrap() +} + +/// Activates or deactivates the profiling mode. +pub fn f_set_profiling_mode(b: bool) -> Result<(), TchError> { + unsafe_torch_err!(atm_set_profiling_mode(b as c_int)); + Ok(()) +} + +/// Activates or deactivates the profiling mode. +pub fn set_profiling_mode(b: bool) { + f_set_profiling_mode(b).unwrap() +} + +pub fn f_fuser_cuda_set_enabled(enabled: bool) -> Result<(), TchError> { + unsafe_torch_err!(atm_fuser_cuda_set_enabled(enabled)); + Ok(()) +} + +pub fn fuser_cuda_set_enabled(enabled: bool) { + f_fuser_cuda_set_enabled(enabled).unwrap() +} + +pub fn f_fuser_cuda_is_enabled() -> Result { + let b = unsafe_torch_err!(atm_fuser_cuda_is_enabled()); + Ok(b) +} + +pub fn fuser_cuda_is_enabled() -> bool { + f_fuser_cuda_is_enabled().unwrap() +} + +pub fn f_set_tensor_expr_fuser_enabled(b: bool) -> Result<(), TchError> { + unsafe_torch_err!(atm_set_tensor_expr_fuser_enabled(b as c_int)); + Ok(()) +} + +pub fn set_tensor_expr_fuser_enabled(b: bool) { + f_set_tensor_expr_fuser_enabled(b).unwrap() +} + +pub fn f_get_tensor_expr_fuser_enabled() -> Result { + Ok(unsafe_torch_err!(atm_get_tensor_expr_fuser_enabled())) +} + +pub fn get_tensor_expr_fuser_enabled() -> bool { + f_get_tensor_expr_fuser_enabled().unwrap() +} + +/// Enables or disables the graph executor optimizer for the current thread. +/// +/// # Arguments +/// +/// * `b` - A boolean that if true enables the graph executor optimizer for the current thread. +/// +/// This function returns an error if it is not possible to enable or disable the graph executor optimizer. +pub fn f_set_graph_executor_optimize(b: bool) -> Result<(), TchError> { + unsafe_torch_err!(at_set_graph_executor_optimize(b)); + Ok(()) +} + +/// Enables or disables the graph executor optimizer for the current thread. +/// +/// # Arguments +/// +/// * `b` - A boolean that if true enables the graph executor optimizer for the current thread. +/// +/// This panics if it is not possible to enable or disable the graph executor optimizer. +pub fn set_graph_executor_optimize(b: bool) { + f_set_graph_executor_optimize(b).unwrap(); +} + +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Debug, PartialEq)] +pub struct Object { + c_ivalue: *mut CIValue, +} + +impl Object { + /// Applies the specified method to the object. The method takes as argument an arbitrary + /// number of ivalues and returns an ivalue. + pub fn method_is>( + &self, + method_name: &str, + ts: &[T], + ) -> Result { + let ts = ts.iter().map(|x| x.borrow().to_c()).collect::, TchError>>()?; + let method_name = std::ffi::CString::new(method_name)?; + let c_ivalue = unsafe_torch_err!(ati_object_method_( + self.c_ivalue, + method_name.as_ptr(), + ts.as_ptr(), + ts.len() as c_int + )); + for x in ts { + unsafe { ati_free(x) } + } + IValue::from_c(c_ivalue) + } + + /// Retrieves the specified attribute from an object as an ivalue. + pub fn getattr(&self, attr_name: &str) -> Result { + let property_name = std::ffi::CString::new(attr_name)?; + let c_ivalue = + unsafe_torch_err!(ati_object_getattr_(self.c_ivalue, property_name.as_ptr())); + if c_ivalue.is_null() { + return Err(TchError::Torch(format!( + "Object.getattr(\"{attr_name}\") returned CIValue nullptr" + ))); + } + IValue::from_c(c_ivalue) + } +} + +impl Drop for Object { + fn drop(&mut self) { + unsafe_torch!(ati_free(self.c_ivalue)) + } +} + +#[cfg(test)] +mod tests { + use super::IValue; + use std::f64::consts; + + fn round_trip>(t: T) { + let ivalue: IValue = t.into(); + let ivalue2 = IValue::from_c(ivalue.to_c().unwrap()).unwrap(); + assert_eq!(ivalue, ivalue2); + } + #[test] + fn ivalue_round_trip() { + round_trip(()); + round_trip(true); + round_trip(false); + round_trip(-1); + round_trip(42); + round_trip(15); + round_trip("".to_string()); + round_trip("foobar".to_string()); + round_trip((42, consts::PI)); + round_trip(vec![42, 1337]); + round_trip(vec![consts::E, consts::PI, 299792458.00001]); + round_trip((vec![true, false, true, true], vec![consts::E, consts::PI, 299792458.00001])); + round_trip(vec![IValue::from(42), IValue::from("foobar")]); + round_trip(vec![ + (IValue::from(42), IValue::from("foobar")), + (IValue::from("foo"), IValue::from("bar")), + ]); + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/kind.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/kind.rs new file mode 100644 index 0000000000..390c623fec --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/wrappers/kind.rs @@ -0,0 +1,176 @@ +//! The different kind of elements supported in Torch. + +use half; + +/// The different kind of elements that a Tensor can hold. +#[allow(clippy::upper_case_acronyms)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub enum Kind { + Uint8, + Int8, + Int16, + Int, + Int64, + Half, + Float, + Double, + ComplexHalf, + ComplexFloat, + ComplexDouble, + Bool, + QInt8, + QUInt8, + QInt32, + BFloat16, + Float8e5m2, + Float8e4m3fn, + Float8e5m2fnuz, + Float8e4m3fnuz, +} + +impl Kind { + pub(super) fn c_int(self) -> libc::c_int { + // These values should be in sync with include/c10/core/ScalarType.h + // https://github.com/pytorch/pytorch/blob/a8d6afb511a69687bbb2b7e88a3cf67917e1697e/c10/core/ScalarType.h#L57 + match self { + Kind::Uint8 => 0, + Kind::Int8 => 1, + Kind::Int16 => 2, + Kind::Int => 3, + Kind::Int64 => 4, + Kind::Half => 5, + Kind::Float => 6, + Kind::Double => 7, + Kind::ComplexHalf => 8, + Kind::ComplexFloat => 9, + Kind::ComplexDouble => 10, + Kind::Bool => 11, + Kind::QInt8 => 12, + Kind::QUInt8 => 13, + Kind::QInt32 => 14, + Kind::BFloat16 => 15, + Kind::Float8e5m2 => 23, + Kind::Float8e4m3fn => 24, + Kind::Float8e5m2fnuz => 25, + Kind::Float8e4m3fnuz => 26, + } + } + + pub(super) fn from_c_int(v: libc::c_int) -> Result { + match v { + 0 => Ok(Kind::Uint8), + 1 => Ok(Kind::Int8), + 2 => Ok(Kind::Int16), + 3 => Ok(Kind::Int), + 4 => Ok(Kind::Int64), + 5 => Ok(Kind::Half), + 6 => Ok(Kind::Float), + 7 => Ok(Kind::Double), + 8 => Ok(Kind::ComplexHalf), + 9 => Ok(Kind::ComplexFloat), + 10 => Ok(Kind::ComplexDouble), + 11 => Ok(Kind::Bool), + 12 => Ok(Kind::QInt8), + 13 => Ok(Kind::QUInt8), + 14 => Ok(Kind::QInt32), + 15 => Ok(Kind::BFloat16), + 23 => Ok(Kind::Float8e5m2), + 24 => Ok(Kind::Float8e4m3fn), + 25 => Ok(Kind::Float8e5m2fnuz), + 26 => Ok(Kind::Float8e4m3fnuz), + _ => Err(crate::TchError::UnknownKind(v)), + } + } + + pub fn elt_size_in_bytes(self) -> usize { + match self { + Kind::Uint8 => 1, + Kind::Int8 => 1, + Kind::Int16 => 2, + Kind::Int => 4, + Kind::Int64 => 8, + Kind::Half => 2, + Kind::Float => 4, + Kind::Double => 8, + Kind::ComplexHalf => 4, + Kind::ComplexFloat => 8, + Kind::ComplexDouble => 16, + Kind::Bool => 1, + Kind::QInt8 => 1, + Kind::QUInt8 => 1, + Kind::QInt32 => 4, + Kind::BFloat16 => 2, + Kind::Float8e5m2 => 1, + Kind::Float8e4m3fn => 1, + Kind::Float8e5m2fnuz => 1, + Kind::Float8e4m3fnuz => 1, + } + } +} + +pub const FLOAT_CPU: (Kind, crate::Device) = (Kind::Float, crate::Device::Cpu); +pub const DOUBLE_CPU: (Kind, crate::Device) = (Kind::Double, crate::Device::Cpu); +pub const INT64_CPU: (Kind, crate::Device) = (Kind::Int64, crate::Device::Cpu); + +pub const FLOAT_CUDA: (Kind, crate::Device) = (Kind::Float, crate::Device::Cuda(0)); +pub const DOUBLE_CUDA: (Kind, crate::Device) = (Kind::Double, crate::Device::Cuda(0)); +pub const INT64_CUDA: (Kind, crate::Device) = (Kind::Int64, crate::Device::Cuda(0)); + +/// Kinds for tensor elements +/// +/// # Safety +/// The specified Kind must be for a type that has the same length as Self. +pub unsafe trait Element: Clone { + const KIND: Kind; + const ZERO: Self; +} + +unsafe impl Element for u8 { + const KIND: Kind = Kind::Uint8; + const ZERO: Self = 0; +} + +unsafe impl Element for i8 { + const KIND: Kind = Kind::Int8; + const ZERO: Self = 0; +} + +unsafe impl Element for i16 { + const KIND: Kind = Kind::Int16; + const ZERO: Self = 0; +} + +unsafe impl Element for i32 { + const KIND: Kind = Kind::Int; + const ZERO: Self = 0; +} + +unsafe impl Element for i64 { + const KIND: Kind = Kind::Int64; + const ZERO: Self = 0; +} + +unsafe impl Element for half::f16 { + const KIND: Kind = Kind::Half; + const ZERO: Self = half::f16::ZERO; +} + +unsafe impl Element for half::bf16 { + const KIND: Kind = Kind::Half; + const ZERO: Self = half::bf16::ZERO; +} + +unsafe impl Element for f32 { + const KIND: Kind = Kind::Float; + const ZERO: Self = 0.; +} + +unsafe impl Element for f64 { + const KIND: Kind = Kind::Double; + const ZERO: Self = 0.; +} + +unsafe impl Element for bool { + const KIND: Kind = Kind::Bool; + const ZERO: Self = false; +} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/layout.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/layout.rs new file mode 100644 index 0000000000..5c26aded2a --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/wrappers/layout.rs @@ -0,0 +1,29 @@ +/// A tensor layout. + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub enum Layout { + Strided, + Sparse, + SparseCsr, + Mkldnn, + SparseCsc, + SparseBsr, + SparseBsc, + NumOptions, +} + +impl Layout { + // This should be kept in sync with include/c10/core/Layout.h + pub fn to_i8(&self) -> i8 { + match self { + Self::Strided => 0, + Self::Sparse => 1, + Self::SparseCsr => 2, + Self::Mkldnn => 3, + Self::SparseCsc => 4, + Self::SparseBsr => 5, + Self::SparseBsc => 6, + Self::NumOptions => 7, + } + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/mod.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/mod.rs new file mode 100644 index 0000000000..a3b69b9b9f --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/wrappers/mod.rs @@ -0,0 +1,21 @@ +#[macro_use] +pub mod utils; + +pub use utils::{ + get_num_interop_threads, get_num_threads, manual_seed, set_num_interop_threads, + set_num_threads, QEngine, +}; + +pub(crate) mod device; +pub(crate) mod image; +pub mod jit; +pub mod kind; +pub(crate) mod layout; +pub(crate) mod optimizer; +#[cfg(feature = "python-extension")] +pub mod python; +pub(crate) mod scalar; +pub(crate) mod stream; +pub(crate) mod tensor; +pub(crate) mod tensor_fallible_generated; +pub(crate) mod tensor_generated; diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/optimizer.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/optimizer.rs new file mode 100644 index 0000000000..c55d04fc4b --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/wrappers/optimizer.rs @@ -0,0 +1,129 @@ +use super::tensor::Tensor; +use crate::TchError; + +pub struct COptimizer { + c_optimizer: *mut torch_sys::C_optimizer, +} + +unsafe impl Send for COptimizer {} + +impl std::fmt::Debug for COptimizer { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + write!(f, "optimizer") + } +} + +impl COptimizer { + pub fn adam( + lr: f64, + beta1: f64, + beta2: f64, + wd: f64, + eps: f64, + amsgrad: bool, + ) -> Result { + let c_optimizer = + unsafe_torch_err!(torch_sys::ato_adam(lr, beta1, beta2, wd, eps, amsgrad)); + Ok(COptimizer { c_optimizer }) + } + + pub fn adamw( + lr: f64, + beta1: f64, + beta2: f64, + wd: f64, + eps: f64, + amsgrad: bool, + ) -> Result { + let c_optimizer = + unsafe_torch_err!(torch_sys::ato_adamw(lr, beta1, beta2, wd, eps, amsgrad)); + Ok(COptimizer { c_optimizer }) + } + + // Maybe we should use the builder pattern to provide default values for these ? + pub fn rms_prop( + lr: f64, + alpha: f64, + eps: f64, + wd: f64, + momentum: f64, + centered: bool, + ) -> Result { + let centered = i32::from(centered); + let c_optimizer = + unsafe_torch_err!(torch_sys::ato_rms_prop(lr, alpha, eps, wd, momentum, centered)); + Ok(COptimizer { c_optimizer }) + } + + pub fn sgd( + lr: f64, + momentum: f64, + dampening: f64, + wd: f64, + nesterov: bool, + ) -> Result { + let nesterov = i32::from(nesterov); + let c_optimizer = + unsafe_torch_err!(torch_sys::ato_sgd(lr, momentum, dampening, wd, nesterov)); + Ok(COptimizer { c_optimizer }) + } + + pub fn add_parameters(&mut self, t: &Tensor, group: usize) -> Result<(), TchError> { + unsafe_torch_err!(torch_sys::ato_add_parameters(self.c_optimizer, t.c_tensor, group)); + Ok(()) + } + + pub fn set_learning_rate(&mut self, lr: f64) -> Result<(), TchError> { + unsafe_torch_err!(torch_sys::ato_set_learning_rate(self.c_optimizer, lr)); + Ok(()) + } + + pub fn set_learning_rate_group(&mut self, group: usize, lr: f64) -> Result<(), TchError> { + unsafe_torch_err!(torch_sys::ato_set_learning_rate_group(self.c_optimizer, group, lr)); + Ok(()) + } + + pub fn set_momentum(&mut self, m: f64) -> Result<(), TchError> { + unsafe_torch_err!(torch_sys::ato_set_momentum(self.c_optimizer, m)); + Ok(()) + } + + pub fn set_momentum_group(&mut self, group: usize, m: f64) -> Result<(), TchError> { + unsafe_torch_err!(torch_sys::ato_set_momentum_group(self.c_optimizer, group, m)); + Ok(()) + } + + pub fn set_weight_decay(&mut self, weight_decay: f64) -> Result<(), TchError> { + unsafe_torch_err!(torch_sys::ato_set_weight_decay(self.c_optimizer, weight_decay)); + Ok(()) + } + + pub fn set_weight_decay_group( + &mut self, + group: usize, + weight_decay: f64, + ) -> Result<(), TchError> { + unsafe_torch_err!(torch_sys::ato_set_weight_decay_group( + self.c_optimizer, + group, + weight_decay + )); + Ok(()) + } + + pub fn zero_grad(&self) -> Result<(), TchError> { + unsafe_torch_err!(torch_sys::ato_zero_grad(self.c_optimizer)); + Ok(()) + } + + pub fn step(&self) -> Result<(), TchError> { + unsafe_torch_err!(torch_sys::ato_step(self.c_optimizer)); + Ok(()) + } +} + +impl Drop for COptimizer { + fn drop(&mut self) { + unsafe { torch_sys::ato_free(self.c_optimizer) } + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/python.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/python.rs new file mode 100644 index 0000000000..136625d63c --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/wrappers/python.rs @@ -0,0 +1,34 @@ +use crate::{TchError, Tensor}; +use torch_sys::python::{self, C_pyobject}; + +pub type CPyObject = C_pyobject; + +/// Check whether an object is a wrapped tensor or not. +/// +/// # Safety +/// Undefined behavior if the given pointer is not a valid PyObject. +pub unsafe fn pyobject_check(pyobject: *mut CPyObject) -> Result { + let v = unsafe_torch_err!(python::thp_variable_check(pyobject)); + Ok(v) +} + +impl Tensor { + /// Wrap a tensor in a Python object. + pub fn pyobject_wrap(&self) -> Result<*mut CPyObject, TchError> { + let v = unsafe_torch_err!(python::thp_variable_wrap(self.c_tensor)); + Ok(v) + } + + /// Unwrap a tensor stored in a Python object. This returns `Ok(None)` if + /// the object is not a wrapped tensor. + /// + /// # Safety + /// Undefined behavior if the given pointer is not a valid PyObject. + pub unsafe fn pyobject_unpack(pyobject: *mut CPyObject) -> Result, TchError> { + if !pyobject_check(pyobject)? { + return Ok(None); + } + let v = unsafe_torch_err!(python::thp_variable_unpack(pyobject)); + Ok(Some(Tensor::from_ptr(v))) + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/scalar.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/scalar.rs new file mode 100644 index 0000000000..3ae275919b --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/wrappers/scalar.rs @@ -0,0 +1,111 @@ +//! Scalar elements. + +use crate::TchError; + +/// A single scalar value. +pub struct Scalar { + pub(super) c_scalar: *mut torch_sys::C_scalar, +} + +impl Scalar { + /// Creates an integer scalar. + pub fn int(v: i64) -> Scalar { + let c_scalar = unsafe_torch!(torch_sys::ats_int(v)); + Scalar { c_scalar } + } + + /// Creates a float scalar scalar. + pub fn float(v: f64) -> Scalar { + let c_scalar = unsafe_torch!(torch_sys::ats_float(v)); + Scalar { c_scalar } + } + + /// Returns an integer value. + pub fn to_int(&self) -> Result { + let i = unsafe_torch_err!(torch_sys::ats_to_int(self.c_scalar)); + Ok(i) + } + + /// Returns a float value. + pub fn to_float(&self) -> Result { + let f = unsafe_torch_err!(torch_sys::ats_to_float(self.c_scalar)); + Ok(f) + } + + /// Returns a string representation of the scalar. + pub fn to_string(&self) -> Result { + let s = unsafe_torch_err!({ + super::utils::ptr_to_string(torch_sys::ats_to_string(self.c_scalar)) + }); + match s { + None => Err(TchError::Kind("nullptr representation".to_string())), + Some(s) => Ok(s), + } + } +} + +impl std::fmt::Debug for Scalar { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self.to_string() { + Err(_) => write!(f, "err"), + Ok(s) => write!(f, "scalar<{s}>"), + } + } +} + +impl Drop for Scalar { + fn drop(&mut self) { + unsafe_torch!(torch_sys::ats_free(self.c_scalar)) + } +} + +impl From for Scalar { + fn from(v: i64) -> Scalar { + Scalar::int(v) + } +} + +impl From for Scalar { + fn from(v: f64) -> Scalar { + Scalar::float(v) + } +} + +impl From for i64 { + fn from(s: Scalar) -> i64 { + Self::from(&s) + } +} + +impl From for f64 { + fn from(s: Scalar) -> f64 { + Self::from(&s) + } +} + +impl From<&Scalar> for i64 { + fn from(s: &Scalar) -> i64 { + s.to_int().unwrap() + } +} + +impl From<&Scalar> for f64 { + fn from(s: &Scalar) -> f64 { + s.to_float().unwrap() + } +} + +#[cfg(test)] +mod tests { + use super::Scalar; + #[test] + fn scalar() { + let pi = Scalar::float(std::f64::consts::PI); + assert_eq!(i64::from(&pi), 3); + assert_eq!(f64::from(&pi), std::f64::consts::PI); + let leet = Scalar::int(1337); + assert_eq!(i64::from(&leet), 1337); + assert_eq!(f64::from(&leet), 1337.); + assert_eq!(&format!("{pi:?}"), "scalar<3.14159>"); + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/stream.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/stream.rs new file mode 100644 index 0000000000..4468d328eb --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/wrappers/stream.rs @@ -0,0 +1,26 @@ +use std::io::{Read, Result, Seek, SeekFrom}; +use torch_sys::io::ReadStream; + +pub struct ReadSeekAdapter { + inner: T, +} + +impl ReadSeekAdapter { + pub fn new(inner: T) -> Self { + Self { inner } + } +} + +impl Read for ReadSeekAdapter { + fn read(&mut self, buf: &mut [u8]) -> Result { + self.inner.read(buf) + } +} + +impl Seek for ReadSeekAdapter { + fn seek(&mut self, pos: SeekFrom) -> Result { + self.inner.seek(pos) + } +} + +impl ReadStream for ReadSeekAdapter {} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/tensor.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/tensor.rs new file mode 100644 index 0000000000..3d5cf7fc74 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/wrappers/tensor.rs @@ -0,0 +1,883 @@ +use super::stream::ReadSeekAdapter; +use super::utils::{path_to_cstring, ptr_to_string}; +use super::{ + device::{Cuda, Device}, + kind, + kind::Kind, +}; +use crate::TchError; +use libc::{c_char, c_int, c_void}; +use std::borrow::Borrow; +use std::io::{Read, Seek, Write}; +use std::path::Path; +use torch_sys::io::ReadStream; +use torch_sys::*; + +/// A tensor object. +#[must_use] +pub struct Tensor { + pub(super) c_tensor: *mut C_tensor, +} + +unsafe impl Send for Tensor {} + +pub extern "C" fn add_callback(data: *mut c_void, name: *const c_char, c_tensor: *mut C_tensor) { + let name = unsafe { std::ffi::CStr::from_ptr(name).to_str().unwrap() }; + let name = name.replace('|', "."); + let v: &mut Vec<(String, Tensor)> = unsafe { &mut *(data as *mut Vec<(String, Tensor)>) }; + v.push((name, Tensor { c_tensor })) +} + +impl Tensor { + /// Creates a new tensor. + pub fn new() -> Tensor { + let c_tensor = unsafe_torch!(at_new_tensor()); + Tensor { c_tensor } + } + + /// Creates a new tensor from the pointer to an existing C++ tensor. + /// + /// # Safety + /// + /// The caller must ensures that the pointer outlives the Rust + /// object. + pub unsafe fn from_ptr(c_tensor: *mut C_tensor) -> Self { + Self { c_tensor } + } + + /// Creates a new tensor from the pointer to an existing C++ tensor. + /// + /// # Safety + /// + /// A shallow copy of the pointer is made so there is no need for + /// this pointer to remain valid for the whole lifetime of the Rust + /// object. + pub unsafe fn clone_from_ptr(c_tensor: *mut C_tensor) -> Self { + let c_tensor = at_shallow_clone(c_tensor); + crate::wrappers::utils::read_and_clean_error().unwrap(); + Self { c_tensor } + } + + /// Returns a pointer to the underlying C++ tensor. + /// + /// The caller must ensures that the Rust tensor object outlives + /// this pointer. + pub fn as_ptr(&self) -> *const C_tensor { + self.c_tensor + } + + /// Returns a mutable pointer to the underlying C++ tensor. + /// + /// The caller must ensures that the Rust tensor object outlives + /// this pointer. + pub fn as_mut_ptr(&mut self) -> *mut C_tensor { + self.c_tensor + } + + /// Returns the number of dimension of the tensor. + pub fn dim(&self) -> usize { + unsafe_torch!(at_dim(self.c_tensor)) + } + + /// Returns the shape of the input tensor. + pub fn size(&self) -> Vec { + let dim = unsafe_torch!(at_dim(self.c_tensor)); + let mut sz = vec![0i64; dim]; + unsafe_torch!(at_shape(self.c_tensor, sz.as_mut_ptr())); + sz + } + + /// Returns the tensor size for single dimension tensors. + pub fn size1(&self) -> Result { + match self.size().as_slice() { + &[s0] => Ok(s0), + size => Err(TchError::Shape(format!("expected one dim, got {size:?}"))), + } + } + + /// Returns the tensor sizes for two dimension tensors. + pub fn size2(&self) -> Result<(i64, i64), TchError> { + match self.size().as_slice() { + &[s0, s1] => Ok((s0, s1)), + size => Err(TchError::Shape(format!("expected two dims, got {size:?}"))), + } + } + + /// Returns the tensor sizes for three dimension tensors. + pub fn size3(&self) -> Result<(i64, i64, i64), TchError> { + match self.size().as_slice() { + &[s0, s1, s2] => Ok((s0, s1, s2)), + size => Err(TchError::Shape(format!("expected three dims, got {size:?}"))), + } + } + + /// Returns the tensor sizes for four dimension tensors. + pub fn size4(&self) -> Result<(i64, i64, i64, i64), TchError> { + match self.size().as_slice() { + &[s0, s1, s2, s3] => Ok((s0, s1, s2, s3)), + size => Err(TchError::Shape(format!("expected four dims, got {size:?}"))), + } + } + + /// Returns the tensor sizes for five dimension tensors. + pub fn size5(&self) -> Result<(i64, i64, i64, i64, i64), TchError> { + match self.size().as_slice() { + &[s0, s1, s2, s3, s4] => Ok((s0, s1, s2, s3, s4)), + size => Err(TchError::Shape(format!("expected five dims, got {size:?}"))), + } + } + + /// Returns the tensor sizes for six dimension tensors. + pub fn size6(&self) -> Result<(i64, i64, i64, i64, i64, i64), TchError> { + match self.size().as_slice() { + &[s0, s1, s2, s3, s4, s5] => Ok((s0, s1, s2, s3, s4, s5)), + size => Err(TchError::Shape(format!("expected six dims, got {size:?}"))), + } + } + + /// Returns the stride of the input tensor. + pub fn stride(&self) -> Vec { + let dim = unsafe_torch!(at_dim(self.c_tensor)); + let mut sz = vec![0i64; dim]; + unsafe_torch!(at_stride(self.c_tensor, sz.as_mut_ptr())); + sz + } + + /// Returns the tensor strides for single dimension tensors. + pub fn stride1(&self) -> Result { + match self.stride().as_slice() { + &[s0] => Ok(s0), + size => Err(TchError::Shape(format!("expected one dim, got {size:?}"))), + } + } + + /// Returns the tensor strides for two dimension tensors. + pub fn stride2(&self) -> Result<(i64, i64), TchError> { + match self.stride().as_slice() { + &[s0, s1] => Ok((s0, s1)), + size => Err(TchError::Shape(format!("expected two dims, got {size:?}"))), + } + } + + /// Returns the tensor strides for three dimension tensors. + pub fn stride3(&self) -> Result<(i64, i64, i64), TchError> { + match self.stride().as_slice() { + &[s0, s1, s2] => Ok((s0, s1, s2)), + size => Err(TchError::Shape(format!("expected three dims, got {size:?}"))), + } + } + + /// Returns the tensor strides for four dimension tensors. + pub fn stride4(&self) -> Result<(i64, i64, i64, i64), TchError> { + match self.stride().as_slice() { + &[s0, s1, s2, s3] => Ok((s0, s1, s2, s3)), + size => Err(TchError::Shape(format!("expected four dims, got {size:?}"))), + } + } + + /// Returns the tensor strides for five dimension tensors. + pub fn stride5(&self) -> Result<(i64, i64, i64, i64, i64), TchError> { + match self.stride().as_slice() { + &[s0, s1, s2, s3, s4] => Ok((s0, s1, s2, s3, s4)), + size => Err(TchError::Shape(format!("expected five dims, got {size:?}"))), + } + } + + /// Returns the tensor strides for six dimension tensors. + pub fn stride6(&self) -> Result<(i64, i64, i64, i64, i64, i64), TchError> { + match self.stride().as_slice() { + &[s0, s1, s2, s3, s4, s5] => Ok((s0, s1, s2, s3, s4, s5)), + size => Err(TchError::Shape(format!("expected six dims, got {size:?}"))), + } + } + + /// Returns the kind of elements stored in the input tensor. Returns + /// an error on undefined tensors and unsupported data types. + pub fn f_kind(&self) -> Result { + let kind = unsafe_torch!(at_scalar_type(self.c_tensor)); + Kind::from_c_int(kind) + } + + /// Returns the kind of elements stored in the input tensor. Panics + /// an error on undefined tensors and unsupported data types. + pub fn kind(&self) -> Kind { + self.f_kind().unwrap() + } + + /// Returns the device on which the input tensor is located. + pub fn device(&self) -> Device { + let device = unsafe_torch!(at_device(self.c_tensor)); + Device::from_c_int(device) + } + + /// Prints the input tensor. + /// + /// Caution: this uses the C++ printer which prints the whole tensor even if + /// it is very large. + pub fn print(&self) { + unsafe_torch!(at_print(self.c_tensor)) + } + + /// Returns a double value on tensors holding a single element. An error is + /// returned otherwise. + pub fn f_double_value(&self, idx: &[i64]) -> Result { + Ok(unsafe_torch_err!({ + at_double_value_at_indexes(self.c_tensor, idx.as_ptr(), idx.len() as i32) + })) + } + + /// Returns an int value on tensors holding a single element. An error is + /// returned otherwise. + pub fn f_int64_value(&self, idx: &[i64]) -> Result { + Ok(unsafe_torch_err!({ + at_int64_value_at_indexes(self.c_tensor, idx.as_ptr(), idx.len() as i32) + })) + } + + /// Returns a double value on tensors holding a single element. Panics otherwise. + pub fn double_value(&self, idx: &[i64]) -> f64 { + self.f_double_value(idx).unwrap() + } + + /// Returns an int value on tensors holding a single element. Panics otherwise. + pub fn int64_value(&self, idx: &[i64]) -> i64 { + self.f_int64_value(idx).unwrap() + } + + /// Returns true if gradient are currently tracked for this tensor. + pub fn requires_grad(&self) -> bool { + unsafe_torch!(at_requires_grad(self.c_tensor)) != 0 + } + + /// Returns the address of the first element of this tensor. + pub fn data_ptr(&self) -> *mut c_void { + unsafe_torch!(at_data_ptr(self.c_tensor)) + } + + /// Returns true if the tensor is defined. + pub fn defined(&self) -> bool { + unsafe_torch!(at_defined(self.c_tensor) != 0) + } + + /// Returns true if the tensor is compatible with MKL-DNN (oneDNN). + pub fn is_mkldnn(&self) -> bool { + unsafe_torch!(at_is_mkldnn(self.c_tensor) != 0) + } + + /// Returns true if the tensor is sparse. + pub fn is_sparse(&self) -> bool { + unsafe_torch!(at_is_sparse(self.c_tensor) != 0) + } + + // Returns true if the tensor if contiguous + pub fn is_contiguous(&self) -> bool { + unsafe_torch!(at_is_contiguous(self.c_tensor) != 0) + } + + /// Zeroes the gradient tensor attached to this tensor if defined. + pub fn zero_grad(&mut self) { + let mut grad = self.grad(); + if grad.defined() { + let _ = grad.detach_().zero_(); + } + } + + /// Runs the backward pass, populating the gradient tensors for tensors + /// which gradients are tracked. + /// + /// Gradients tracking can be turned on via `set_requires_grad`. + pub fn f_backward(&self) -> Result<(), TchError> { + unsafe_torch_err!(at_backward(self.c_tensor, 0, 0)); + Ok(()) + } + + /// Runs the backward pass, populating the gradient tensors for tensors + /// which gradients are tracked. + /// + /// Gradients tracking can be turned on via `set_requires_grad`. + /// Panics if the C++ api returns an exception. + pub fn backward(&self) { + self.f_backward().unwrap() + } + + pub fn f_run_backward( + tensors: &[T1], + inputs: &[T2], + keep_graph: bool, + create_graph: bool, + ) -> Result, TchError> + where + T1: Borrow, + T2: Borrow, + { + let mut outputs = vec![std::ptr::null_mut(); inputs.len()]; + let tensors: Vec<_> = tensors.iter().map(|x| x.borrow().c_tensor).collect(); + let inputs: Vec<_> = inputs.iter().map(|x| x.borrow().c_tensor).collect(); + unsafe_torch_err!(at_run_backward( + tensors.as_ptr(), + tensors.len() as c_int, + inputs.as_ptr(), + inputs.len() as c_int, + outputs.as_mut_ptr(), + keep_graph as c_int, + create_graph as c_int, + )); + Ok(outputs.into_iter().map(|c_tensor| Tensor { c_tensor }).collect()) + } + + pub fn run_backward( + tensors: &[T1], + inputs: &[T2], + keep_graph: bool, + create_graph: bool, + ) -> Vec + where + T1: Borrow, + T2: Borrow, + { + Tensor::f_run_backward(tensors, inputs, keep_graph, create_graph).unwrap() + } + + /// Copies `numel` elements from `self` to `dst`. + pub fn f_copy_data_u8(&self, dst: &mut [u8], numel: usize) -> Result<(), TchError> { + let elt_size_in_bytes = self.f_kind()?.elt_size_in_bytes(); + if dst.len() < numel * elt_size_in_bytes { + return Err(TchError::Shape(format!("slice len < {numel}"))); + } + unsafe_torch_err!(at_copy_data( + self.c_tensor, + dst.as_mut_ptr() as *const c_void, + numel, + elt_size_in_bytes, + )); + Ok(()) + } + + /// Unscale tensor while checking for infinities. + /// + /// `found_inf` is a singleton tensor that is used to record the + /// presence of infinite values. `inv_scale` is a scalar containing + /// the inverse scaling factor. This method is only available + /// for CUDA tensors. + pub fn f_internal_amp_non_finite_check_and_unscale( + &mut self, + found_inf: &mut Tensor, + inv_scale: &Tensor, + ) -> Result<(), TchError> { + unsafe_torch_err!(at__amp_non_finite_check_and_unscale( + self.c_tensor, + found_inf.c_tensor, + inv_scale.c_tensor + )); + + Ok(()) + } + + /// Unscale tensor while checking for infinities. + /// + /// `found_inf` is a singleton tensor that is used to record the + /// presence of infinite values. `inv_scale` is a scalar containing + /// the inverse scaling factor. This method is only available + /// for CUDA tensors. + pub fn internal_amp_non_finite_check_and_unscale( + &mut self, + found_inf: &mut Tensor, + inv_scale: &Tensor, + ) { + self.f_internal_amp_non_finite_check_and_unscale(found_inf, inv_scale).unwrap() + } + + /// Copies `numel` elements from `self` to `dst`. + pub fn copy_data_u8(&self, dst: &mut [u8], numel: usize) { + self.f_copy_data_u8(dst, numel).unwrap() + } + + /// Copies `numel` elements from `self` to `dst`. + pub fn f_copy_data( + &self, + dst: &mut [T], + numel: usize, + ) -> Result<(), TchError> { + if T::KIND != self.f_kind()? { + return Err(TchError::Kind(format!( + "incoherent elt kind, {:?} != {:?}", + self.f_kind(), + T::KIND + ))); + } + if dst.len() < numel { + return Err(TchError::Shape(format!("slice len < {numel}"))); + } + unsafe_torch_err!(at_copy_data( + self.c_tensor, + dst.as_mut_ptr() as *const c_void, + numel, + T::KIND.elt_size_in_bytes(), + )); + Ok(()) + } + + /// Copies `numel` elements from `self` to `dst`. + pub fn copy_data(&self, dst: &mut [T], numel: usize) { + self.f_copy_data(dst, numel).unwrap() + } + + /// Returns the total number of elements stored in a tensor. + pub fn numel(&self) -> usize { + self.size().iter().product::() as usize + } + + // This is similar to vec_... but faster as it directly blits the data. + /// Converts a slice to a tensor. + pub fn f_from_slice(data: &[T]) -> Result { + let data_len = data.len(); + let data = data.as_ptr() as *const c_void; + let c_tensor = unsafe_torch_err!(at_tensor_of_data( + data, + [data_len as i64].as_ptr(), + 1, + T::KIND.elt_size_in_bytes(), + T::KIND.c_int(), + )); + Ok(Tensor { c_tensor }) + } + + /// Converts a slice to a tensor. + pub fn from_slice(data: &[T]) -> Tensor { + Self::f_from_slice(data).unwrap() + } + + /// Converts some byte data to a tensor with some specified kind and shape. + pub fn f_from_data_size(data: &[u8], size: &[i64], kind: Kind) -> Result { + let data = data.as_ptr() as *const c_void; + let elt_size_in_bytes = kind.elt_size_in_bytes(); + let c_tensor = unsafe_torch_err!(at_tensor_of_data( + data, + size.as_ptr(), + size.len(), + elt_size_in_bytes, + kind.c_int(), + )); + Ok(Tensor { c_tensor }) + } + + /// Creates a tensor from data that is assumed to be initialized. + /// Resize operations are not allowed on this tensor without copying the data first. + /// An empty strides slice will result in using the default strides. + /// # Safety + /// Behavior is undefined if `data` points to invalid data. + pub unsafe fn f_from_blob( + data: *const u8, + size: &[i64], + strides: &[i64], + kind: Kind, + device: Device, + ) -> Result { + let data = data as *const c_void; + #[allow(unused_unsafe)] + let c_tensor = unsafe_torch_err!(at_tensor_of_blob( + data, + size.as_ptr(), + size.len(), + strides.as_ptr(), + strides.len(), + kind.c_int(), + device.c_int() + )); + Ok(Tensor { c_tensor }) + } + + /// Creates a tensor from data that is assumed to be initialized. + /// Resize operations are not allowed on this tensor without copying the data first. + /// An empty strides slice will result in using the default strides. + /// # Safety + /// Behavior is undefined if `data` points to invalid data. + pub unsafe fn from_blob( + data: *const u8, + size: &[i64], + strides: &[i64], + kind: Kind, + device: Device, + ) -> Tensor { + Self::f_from_blob(data, size, strides, kind, device).unwrap() + } + + /// Converts some byte data to a tensor with some specified kind and shape. + pub fn from_data_size(data: &[u8], size: &[i64], kind: Kind) -> Tensor { + Self::f_from_data_size(data, size, kind).unwrap() + } + + /// Returns a new tensor that share storage with the input tensor. + pub fn shallow_clone(&self) -> Tensor { + let c_tensor = unsafe_torch!(at_shallow_clone(self.c_tensor)); + Tensor { c_tensor } + } + + /// Gets the sub-tensor at the given index. + pub fn f_get(&self, index: i64) -> Result { + let c_tensor = unsafe_torch_err!(at_get(self.c_tensor, index as c_int)); + Ok(Tensor { c_tensor }) + } + + /// Gets the sub-tensor at the given index. + pub fn get(&self, index: i64) -> Tensor { + self.f_get(index).unwrap() + } + + /// Copies values from the argument tensor to the input tensor. + pub fn f_copy_(&mut self, src: &Tensor) -> Result<(), TchError> { + unsafe_torch_err!(at_copy_(self.c_tensor, src.c_tensor)); + Ok(()) + } + + /// Copies values from the argument tensor to the input tensor. + pub fn copy_(&mut self, src: &Tensor) { + self.f_copy_(src).unwrap() + } + + /// Loads a tensor from a file. + /// + /// The file format is the same as the one used by the PyTorch C++ API. + pub fn load>(path: T) -> Result { + let path = path_to_cstring(path)?; + let c_tensor = unsafe_torch_err!(at_load(path.as_ptr())); + Ok(Tensor { c_tensor }) + } + + /// Loads a tensor from a stream. + /// + /// The file format is the same as the one used by the PyTorch C++ API. + pub fn load_from_stream(stream: T) -> Result { + let adapter = ReadSeekAdapter::new(stream); + let boxed_stream: Box> = Box::new(Box::new(adapter)); + let c_tensor = + unsafe_torch_err!(at_load_from_stream(Box::into_raw(boxed_stream) as *mut c_void,)); + Ok(Tensor { c_tensor }) + } + + /// Saves a tensor to a file. + /// + /// The file format is the same as the one used by the PyTorch C++ API. + pub fn save>(&self, path: T) -> Result<(), TchError> { + let path = path_to_cstring(path)?; + unsafe_torch_err!(at_save(self.c_tensor, path.as_ptr())); + Ok(()) + } + + /// Saves a tensor to a stream. + /// + /// The file format is the same as the one used by the PyTorch C++ API. + pub fn save_to_stream(&self, stream: W) -> Result<(), TchError> { + let boxed_stream: Box> = Box::new(Box::new(stream)); + unsafe_torch_err!(at_save_to_stream( + self.c_tensor, + Box::into_raw(boxed_stream) as *mut c_void, + )); + Ok(()) + } + + /// Saves some named tensors to a file + /// + /// The file format is the same as the one used by the PyTorch C++ API. + pub fn save_multi, T: AsRef, P: AsRef>( + named_tensors: &[(S, T)], + path: P, + ) -> Result<(), TchError> { + let path = path_to_cstring(path)?; + let c_tensors = named_tensors.iter().map(|nt| nt.1.as_ref().c_tensor).collect::>(); + let names = named_tensors + .iter() + .map(|nt| nt.0.as_ref().replace('.', "|").into_bytes()) + .map(std::ffi::CString::new) + .collect::, _>>()?; + let name_ptrs = names.iter().map(|n| n.as_ptr()).collect::>(); + unsafe_torch_err!(at_save_multi( + c_tensors.as_ptr(), + name_ptrs.as_ptr(), + names.len() as i32, + path.as_ptr(), + )); + Ok(()) + } + + /// Saves some named tensors to a stream + /// + /// The file format is the same as the one used by the PyTorch C++ API. + pub fn save_multi_to_stream, T: AsRef, W: Write>( + named_tensors: &[(S, T)], + stream: W, + ) -> Result<(), TchError> { + let boxed_stream: Box> = Box::new(Box::new(stream)); + let c_tensors = named_tensors.iter().map(|nt| nt.1.as_ref().c_tensor).collect::>(); + let names = named_tensors + .iter() + .map(|nt| nt.0.as_ref().replace('.', "|").into_bytes()) + .map(std::ffi::CString::new) + .collect::, _>>()?; + let name_ptrs = names.iter().map(|n| n.as_ptr()).collect::>(); + unsafe_torch_err!(at_save_multi_to_stream( + c_tensors.as_ptr(), + name_ptrs.as_ptr(), + names.len() as i32, + Box::into_raw(boxed_stream) as *mut c_void, + )); + Ok(()) + } + + /// Loads some named tensors from a file + /// + /// The file format is the same as the one used for modules in the PyTorch C++ API. + /// It commonly uses the .ot extension. + pub fn load_multi>(path: T) -> Result, TchError> { + let path = path_to_cstring(path)?; + let mut v: Vec<(String, Tensor)> = vec![]; + unsafe_torch_err!(at_load_callback( + path.as_ptr(), + &mut v as *mut _ as *mut c_void, + add_callback + )); + Ok(v) + } + + /// Loads some named tensors from a file to a given device + /// + /// The file format is the same as the one used for modules in the PyTorch C++ API. + /// It commonly uses the .ot extension. + pub fn load_multi_with_device>( + path: T, + device: Device, + ) -> Result, TchError> { + let path = path_to_cstring(path)?; + let mut v: Vec<(String, Tensor)> = vec![]; + unsafe_torch_err!(at_load_callback_with_device( + path.as_ptr(), + &mut v as *mut _ as *mut c_void, + add_callback, + device.c_int(), + )); + Ok(v) + } + + /// Loads some named tensors from a zip file + /// + /// The expected file format is a zip archive containing a data.pkl file describing + /// the embedded tensors. These are commonly used with the .bin extension to export + /// PyTorch models and weights using the Python api. + pub fn loadz_multi>(path: T) -> Result, TchError> { + let path = path_to_cstring(path)?; + let mut v: Vec<(String, Tensor)> = vec![]; + unsafe_torch_err!(at_loadz_callback( + path.as_ptr(), + &mut v as *mut _ as *mut c_void, + add_callback + )); + Ok(v) + } + + /// Loads some named tensors from a zip file to a given device + /// + /// The expected file format is a zip archive containing a data.pkl file describing + /// the embedded tensors. These are commonly used with the .bin extension to export + /// PyTorch models and weights using the Python api. + pub fn loadz_multi_with_device>( + path: T, + device: Device, + ) -> Result, TchError> { + let path = path_to_cstring(path)?; + let mut v: Vec<(String, Tensor)> = vec![]; + unsafe_torch_err!(at_loadz_callback_with_device( + path.as_ptr(), + &mut v as *mut _ as *mut c_void, + add_callback, + device.c_int(), + )); + Ok(v) + } + + /// Loads some named tensors from a stream + /// + /// The file format is the same as the one used by the PyTorch C++ API. + pub fn load_multi_from_stream( + stream: T, + ) -> Result, TchError> { + let adapter = ReadSeekAdapter::new(stream); + let boxed_stream: Box> = Box::new(Box::new(adapter)); + let mut v: Vec<(String, Tensor)> = vec![]; + unsafe_torch_err!(at_load_from_stream_callback( + Box::into_raw(boxed_stream) as *mut c_void, + &mut v as *mut _ as *mut c_void, + add_callback, + false, + 0, + )); + Ok(v) + } + + /// Loads some named tensors from a stream to a given device + /// + /// The file format is the same as the one used by the PyTorch C++ API. + pub fn load_multi_from_stream_with_device( + stream: T, + device: Device, + ) -> Result, TchError> { + let adapter = ReadSeekAdapter::new(stream); + let boxed_stream: Box> = Box::new(Box::new(adapter)); + let mut v: Vec<(String, Tensor)> = vec![]; + unsafe_torch_err!(at_load_from_stream_callback( + Box::into_raw(boxed_stream) as *mut c_void, + &mut v as *mut _ as *mut c_void, + add_callback, + true, + device.c_int(), + )); + Ok(v) + } + + /// Returns a string representation for the tensor. + /// + /// The representation will contain all the tensor element hence may be huge for + /// large tensors. + pub fn to_string(&self, lw: i64) -> Result { + let s = + unsafe_torch_err!(ptr_to_string(torch_sys::at_to_string(self.c_tensor, lw as c_int))); + match s { + None => Err(TchError::Kind("nullptr representation".to_string())), + Some(s) => Ok(s), + } + } +} + +impl Default for Tensor { + fn default() -> Self { + Self::new() + } +} + +impl Drop for Tensor { + fn drop(&mut self) { + unsafe_torch!(at_free(self.c_tensor)) + } +} + +fn autocast_clear_cache() { + unsafe_torch!(at_autocast_clear_cache()) +} + +fn autocast_decrement_nesting() -> isize { + unsafe_torch!(at_autocast_decrement_nesting() as isize) +} + +fn autocast_increment_nesting() -> isize { + unsafe_torch!(at_autocast_increment_nesting() as isize) +} + +fn autocast_is_enabled() -> bool { + unsafe_torch!(at_autocast_is_enabled() != 0) +} + +fn autocast_set_enabled(b: bool) -> bool { + unsafe_torch!(at_autocast_set_enabled(i32::from(b)) != 0) +} + +/// Runs a closure in mixed precision. +pub fn autocast(enabled: bool, f: F) -> T +where + F: FnOnce() -> T, +{ + if !Cuda::is_available() { + return f(); + } + + // Check whether we are using CUDA. + let prev = autocast_is_enabled(); + autocast_set_enabled(enabled); + autocast_increment_nesting(); + + let result = f(); + + if autocast_decrement_nesting() == 0 { + autocast_clear_cache(); + } + autocast_set_enabled(prev); + + result +} + +fn grad_set_enabled(b: bool) -> bool { + unsafe_torch!(at_grad_set_enabled(i32::from(b)) != 0) +} + +/// Runs a closure without keeping track of gradients. +pub fn no_grad(f: F) -> T +where + F: FnOnce() -> T, +{ + let prev = grad_set_enabled(false); + let result = f(); + let _false = grad_set_enabled(prev); + result +} + +/// Runs a closure explicitly keeping track of gradients, this could be +/// run within a no_grad closure for example. +pub fn with_grad(f: F) -> T +where + F: FnOnce() -> T, +{ + let prev = grad_set_enabled(true); + let result = f(); + let _false = grad_set_enabled(prev); + result +} + +/// A RAII guard that prevents gradient tracking until deallocated. +pub struct NoGradGuard { + enabled: bool, +} + +/// Disables gradient tracking, this will be enabled back when the +/// returned value gets deallocated. +/// Note that it is important to bind this to a name like `_guard` +/// and not to `_` as the latter would immediately drop the guard. +/// See +/// for more details. +pub fn no_grad_guard() -> NoGradGuard { + NoGradGuard { enabled: grad_set_enabled(false) } +} + +impl std::convert::AsRef for Tensor { + fn as_ref(&self) -> &Self { + self + } +} + +impl Drop for NoGradGuard { + fn drop(&mut self) { + let _enabled = grad_set_enabled(self.enabled); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub enum Reduction { + /// Do not reduce. + None, + /// Mean of losses. + Mean, + /// Sum of losses. + Sum, + /// Escape hatch in case new options become available. + Other(i64), +} + +impl Reduction { + // This has to stay in sync with + // pytorch/aten/src/ATen/core/Reduction.h + pub fn to_int(self) -> i64 { + match self { + Reduction::None => 0, + Reduction::Mean => 1, + Reduction::Sum => 2, + Reduction::Other(i) => i, + } + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/tensor_fallible_generated.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/tensor_fallible_generated.rs new file mode 100644 index 0000000000..2aff06e73e --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/wrappers/tensor_fallible_generated.rs @@ -0,0 +1,38995 @@ +/* THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT BY HAND! */ +#![allow(clippy::all)] +use crate::{Device, Kind, Layout, Scalar, TchError, Tensor}; +use std::borrow::Borrow; +use std::convert::Into; +use torch_sys::c_generated::*; +use torch_sys::*; + +fn ptr_list_opt>(l: &[Option]) -> Vec<*mut C_tensor> { + l.iter().map(|x| x.as_ref().map_or(std::ptr::null_mut(), |x| x.borrow().c_tensor)).collect() +} + +fn ptr_list>(l: &[T]) -> Vec<*mut C_tensor> { + l.iter().map(|x| x.borrow().c_tensor).collect() +} + +impl Tensor { + pub fn f_internal_and_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___and__( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_and_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___and__tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_iand_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___iand__( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_iand_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___iand__tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_ilshift_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___ilshift__( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_ilshift_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___ilshift__tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_ior_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___ior__( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_ior_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___ior__tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_irshift_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___irshift__( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_irshift_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___irshift__tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_ixor_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___ixor__( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_ixor_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___ixor__tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_lshift_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___lshift__( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_lshift_scalar_out_>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___lshift__scalar_out_( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_lshift_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___lshift__tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_lshift_tensor_out_( + &self, + out: &Tensor, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___lshift__tensor_out_( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_or_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___or__(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_or_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___or__tensor_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_rshift_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___rshift__( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_rshift_scalar_out_>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___rshift__scalar_out_( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_rshift_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___rshift__tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_rshift_tensor_out_( + &self, + out: &Tensor, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___rshift__tensor_out_( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_xor_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___xor__( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_xor_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg___xor__tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_adaptive_avg_pool2d( + &self, + output_size: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__adaptive_avg_pool2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_adaptive_avg_pool2d_backward( + &self, + grad_output: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__adaptive_avg_pool2d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_adaptive_avg_pool2d_backward_out( + &self, + out: &Tensor, + grad_output: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__adaptive_avg_pool2d_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad_output.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_adaptive_avg_pool2d_out( + &self, + out: &Tensor, + output_size: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__adaptive_avg_pool2d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_adaptive_avg_pool3d( + &self, + output_size: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__adaptive_avg_pool3d( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_adaptive_avg_pool3d_backward( + &self, + grad_output: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__adaptive_avg_pool3d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_adaptive_avg_pool3d_backward_out( + &self, + out: &Tensor, + grad_output: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__adaptive_avg_pool3d_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad_output.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_adaptive_avg_pool3d_out( + &self, + out: &Tensor, + output_size: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__adaptive_avg_pool3d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_add_batch_dim(&self, batch_dim: i64, level: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__add_batch_dim( + c_tensors.as_mut_ptr(), + self.c_tensor, + batch_dim, + level + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_add_relu(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__add_relu(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_add_relu_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__add_relu_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_add_relu_out( + &self, + out: &Tensor, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__add_relu_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_add_relu_scalar>( + &self, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__add_relu_scalar( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_add_relu_scalar_>( + &mut self, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__add_relu_scalar_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_add_relu_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__add_relu_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_addmm_activation( + &self, + mat1: &Tensor, + mat2: &Tensor, + use_gelu: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__addmm_activation( + c_tensors.as_mut_ptr(), + self.c_tensor, + mat1.c_tensor, + mat2.c_tensor, + if use_gelu { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_addmm_activation_out( + &self, + out: &Tensor, + mat1: &Tensor, + mat2: &Tensor, + use_gelu: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__addmm_activation_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mat1.c_tensor, + mat2.c_tensor, + if use_gelu { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_aminmax(&self) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__aminmax(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_aminmax_dim( + &self, + dim: i64, + keepdim: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__aminmax_dim( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + if keepdim { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_aminmax_dim_out( + &self, + out0: &Tensor, + out1: &Tensor, + dim: i64, + keepdim: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__aminmax_dim_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + self.c_tensor, + dim, + if keepdim { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_aminmax_out( + &self, + out0: &Tensor, + out1: &Tensor, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__aminmax_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + self.c_tensor + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_amp_update_scale( + &self, + growth_tracker: &Tensor, + found_inf: &Tensor, + scale_growth_factor: f64, + scale_backoff_factor: f64, + growth_interval: i64, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__amp_update_scale( + c_tensors.as_mut_ptr(), + self.c_tensor, + growth_tracker.c_tensor, + found_inf.c_tensor, + scale_growth_factor, + scale_backoff_factor, + growth_interval + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_amp_update_scale_( + &mut self, + growth_tracker: &Tensor, + found_inf: &Tensor, + scale_growth_factor: f64, + scale_backoff_factor: f64, + growth_interval: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__amp_update_scale_( + c_tensors.as_mut_ptr(), + self.c_tensor, + growth_tracker.c_tensor, + found_inf.c_tensor, + scale_growth_factor, + scale_backoff_factor, + growth_interval + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_amp_update_scale_out( + &self, + out: &Tensor, + growth_tracker: &Tensor, + found_inf: &Tensor, + scale_growth_factor: f64, + scale_backoff_factor: f64, + growth_interval: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__amp_update_scale_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + growth_tracker.c_tensor, + found_inf.c_tensor, + scale_growth_factor, + scale_backoff_factor, + growth_interval + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_assert_scalar>( + self_scalar: S, + assert_msg: &str, + ) -> Result<(), TchError> { + unsafe_torch_err!(atg__assert_scalar( + self_scalar.into().c_scalar, + assert_msg.as_ptr(), + assert_msg.len() as i32 + )); + Ok(()) + } + + pub fn f_internal_assert_tensor_metadata( + a: &Tensor, + size: impl IntListOption, + stride: impl IntListOption, + dtype: impl Into>, + device: Device, + layout: Option, + ) -> Result<(), TchError> { + unsafe_torch_err!(atg__assert_tensor_metadata( + a.c_tensor, + size.as_ptr(), + size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + dtype.into().map_or(-1, |s| s.c_int()), + device.c_int(), + layout.map_or(-1, |s| s.to_i8()) + )); + Ok(()) + } + + pub fn f_internal_autocast_to_full_precision( + &self, + cuda_enabled: bool, + cpu_enabled: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__autocast_to_full_precision( + c_tensors.as_mut_ptr(), + self.c_tensor, + if cuda_enabled { 1 } else { 0 }, + if cpu_enabled { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_autocast_to_reduced_precision( + &self, + cuda_enabled: bool, + cpu_enabled: bool, + cuda_dtype: Kind, + cpu_dtype: Kind, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__autocast_to_reduced_precision( + c_tensors.as_mut_ptr(), + self.c_tensor, + if cuda_enabled { 1 } else { 0 }, + if cpu_enabled { 1 } else { 0 }, + cuda_dtype.c_int(), + cpu_dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_batch_norm_no_update>( + &self, + weight: Option, + bias: Option, + running_mean: Option, + running_var: Option, + momentum: f64, + eps: f64, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg__batch_norm_no_update( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + momentum, + eps + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_internal_batch_norm_no_update_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + out3: &Tensor, + weight: Option, + bias: Option, + running_mean: Option, + running_var: Option, + momentum: f64, + eps: f64, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg__batch_norm_no_update_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + out2.c_tensor, + out3.c_tensor, + self.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + momentum, + eps + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_internal_batch_norm_with_update>( + &self, + weight: Option, + bias: Option, + running_mean: &Tensor, + running_var: &Tensor, + momentum: f64, + eps: f64, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg__batch_norm_with_update( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_mean.c_tensor, + running_var.c_tensor, + momentum, + eps + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_internal_batch_norm_with_update_functional>( + &self, + weight: Option, + bias: Option, + running_mean: &Tensor, + running_var: &Tensor, + momentum: f64, + eps: f64, + ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 6]; + unsafe_torch_err!(atg__batch_norm_with_update_functional( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_mean.c_tensor, + running_var.c_tensor, + momentum, + eps + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + Tensor { c_tensor: c_tensors[4] }, + Tensor { c_tensor: c_tensors[5] }, + )) + } + + pub fn f_internal_batch_norm_with_update_out>( + &self, + out: &Tensor, + save_mean: &Tensor, + save_invstd: &Tensor, + reserve: &Tensor, + weight: Option, + bias: Option, + running_mean: &Tensor, + running_var: &Tensor, + momentum: f64, + eps: f64, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg__batch_norm_with_update_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + save_mean.c_tensor, + save_invstd.c_tensor, + reserve.c_tensor, + self.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_mean.c_tensor, + running_var.c_tensor, + momentum, + eps + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_internal_cast_byte(&self, non_blocking: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__cast_byte( + c_tensors.as_mut_ptr(), + self.c_tensor, + if non_blocking { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_cast_char(&self, non_blocking: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__cast_char( + c_tensors.as_mut_ptr(), + self.c_tensor, + if non_blocking { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_cast_double(&self, non_blocking: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__cast_double( + c_tensors.as_mut_ptr(), + self.c_tensor, + if non_blocking { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_cast_float(&self, non_blocking: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__cast_float( + c_tensors.as_mut_ptr(), + self.c_tensor, + if non_blocking { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_cast_half(&self, non_blocking: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__cast_half( + c_tensors.as_mut_ptr(), + self.c_tensor, + if non_blocking { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_cast_int(&self, non_blocking: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__cast_int( + c_tensors.as_mut_ptr(), + self.c_tensor, + if non_blocking { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_cast_long(&self, non_blocking: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__cast_long( + c_tensors.as_mut_ptr(), + self.c_tensor, + if non_blocking { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_cast_short(&self, non_blocking: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__cast_short( + c_tensors.as_mut_ptr(), + self.c_tensor, + if non_blocking { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_cdist_backward( + grad: &Tensor, + x1: &Tensor, + x2: &Tensor, + p: f64, + cdist: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__cdist_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + x1.c_tensor, + x2.c_tensor, + p, + cdist.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_cdist_backward_out( + out: &Tensor, + grad: &Tensor, + x1: &Tensor, + x2: &Tensor, + p: f64, + cdist: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__cdist_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad.c_tensor, + x1.c_tensor, + x2.c_tensor, + p, + cdist.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_cholesky_solve_helper( + &self, + a: &Tensor, + upper: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__cholesky_solve_helper( + c_tensors.as_mut_ptr(), + self.c_tensor, + a.c_tensor, + if upper { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_cholesky_solve_helper_out( + &self, + out: &Tensor, + a: &Tensor, + upper: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__cholesky_solve_helper_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + a.c_tensor, + if upper { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_chunk_cat>( + tensors: &[T], + dim: i64, + num_chunks: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__chunk_cat( + c_tensors.as_mut_ptr(), + ptr_list(tensors).as_ptr(), + tensors.len() as i32, + dim, + num_chunks + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_chunk_cat_out>( + out: &Tensor, + tensors: &[T], + dim: i64, + num_chunks: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__chunk_cat_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + ptr_list(tensors).as_ptr(), + tensors.len() as i32, + dim, + num_chunks + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_coalesce(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__coalesce(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_coalesce_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__coalesce_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_coalesced(&self, coalesced: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__coalesced( + c_tensors.as_mut_ptr(), + self.c_tensor, + if coalesced { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_coalesced_(&mut self, coalesced: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__coalesced_( + c_tensors.as_mut_ptr(), + self.c_tensor, + if coalesced { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_coalesced_out( + &self, + out: &Tensor, + coalesced: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__coalesced_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + if coalesced { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_compute_linear_combination( + &self, + coefficients: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__compute_linear_combination( + c_tensors.as_mut_ptr(), + self.c_tensor, + coefficients.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_compute_linear_combination_out( + &self, + out: &Tensor, + coefficients: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__compute_linear_combination_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + coefficients.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_conj(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__conj(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_conj_copy(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__conj_copy(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_conj_copy_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__conj_copy_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_conj_physical(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__conj_physical(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_conj_physical_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__conj_physical_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_conv_depthwise2d>( + &self, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__conv_depthwise2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_conv_depthwise2d_out>( + &self, + out: &Tensor, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__conv_depthwise2d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_convert_indices_from_coo_to_csr( + &self, + size: i64, + out_int32: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__convert_indices_from_coo_to_csr( + c_tensors.as_mut_ptr(), + self.c_tensor, + size, + if out_int32 { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_convert_indices_from_coo_to_csr_out( + &self, + out: &Tensor, + size: i64, + out_int32: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__convert_indices_from_coo_to_csr_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + size, + if out_int32 { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_convert_indices_from_csr_to_coo( + crow_indices: &Tensor, + col_indices: &Tensor, + out_int32: bool, + transpose: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__convert_indices_from_csr_to_coo( + c_tensors.as_mut_ptr(), + crow_indices.c_tensor, + col_indices.c_tensor, + if out_int32 { 1 } else { 0 }, + if transpose { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_convert_indices_from_csr_to_coo_out( + out: &Tensor, + crow_indices: &Tensor, + col_indices: &Tensor, + out_int32: bool, + transpose: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__convert_indices_from_csr_to_coo_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + crow_indices.c_tensor, + col_indices.c_tensor, + if out_int32 { 1 } else { 0 }, + if transpose { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_convert_weight_to_int4pack( + &self, + innerktiles: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__convert_weight_to_int4pack( + c_tensors.as_mut_ptr(), + self.c_tensor, + innerktiles + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_convert_weight_to_int4pack_for_cpu( + &self, + innerktiles: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__convert_weight_to_int4pack_for_cpu( + c_tensors.as_mut_ptr(), + self.c_tensor, + innerktiles + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_convolution>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + transposed: bool, + output_padding: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + cudnn_enabled: bool, + allow_tf32: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__convolution( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if transposed { 1 } else { 0 }, + output_padding.as_ptr(), + output_padding.len_i32(), + groups, + if benchmark { 1 } else { 0 }, + if deterministic { 1 } else { 0 }, + if cudnn_enabled { 1 } else { 0 }, + if allow_tf32 { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_convolution_deprecated>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + transposed: bool, + output_padding: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + cudnn_enabled: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__convolution_deprecated( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if transposed { 1 } else { 0 }, + output_padding.as_ptr(), + output_padding.len_i32(), + groups, + if benchmark { 1 } else { 0 }, + if deterministic { 1 } else { 0 }, + if cudnn_enabled { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_convolution_mode>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: &str, + dilation: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__convolution_mode( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len() as i32, + dilation.as_ptr(), + dilation.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_convolution_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + transposed: bool, + output_padding: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + cudnn_enabled: bool, + allow_tf32: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__convolution_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if transposed { 1 } else { 0 }, + output_padding.as_ptr(), + output_padding.len_i32(), + groups, + if benchmark { 1 } else { 0 }, + if deterministic { 1 } else { 0 }, + if cudnn_enabled { 1 } else { 0 }, + if allow_tf32 { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_copy_from( + &self, + dst: &Tensor, + non_blocking: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__copy_from( + c_tensors.as_mut_ptr(), + self.c_tensor, + dst.c_tensor, + if non_blocking { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_copy_from_and_resize(&self, dst: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__copy_from_and_resize( + c_tensors.as_mut_ptr(), + self.c_tensor, + dst.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_copy_from_and_resize_out( + &self, + out: &Tensor, + dst: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__copy_from_and_resize_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dst.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_copy_from_out( + &self, + out: &Tensor, + dst: &Tensor, + non_blocking: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__copy_from_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dst.c_tensor, + if non_blocking { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_cslt_compress(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__cslt_compress(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_cslt_sparse_mm>( + compressed_a: &Tensor, + dense_b: &Tensor, + bias: Option, + alpha: Option, + out_dtype: impl Into>, + transpose_result: bool, + alg_id: i64, + split_k: i64, + split_k_mode: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__cslt_sparse_mm( + c_tensors.as_mut_ptr(), + compressed_a.c_tensor, + dense_b.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + alpha.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + out_dtype.into().map_or(-1, |s| s.c_int()), + if transpose_result { 1 } else { 0 }, + alg_id, + split_k, + split_k_mode + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_cslt_sparse_mm_search>( + compressed_a: &Tensor, + dense_b: &Tensor, + bias: Option, + alpha: Option, + out_dtype: impl Into>, + transpose_result: bool, + ) -> Result { + let return_; + unsafe_torch_err!( + return_ = atg__cslt_sparse_mm_search( + compressed_a.c_tensor, + dense_b.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + alpha.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + out_dtype.into().map_or(-1, |s| s.c_int()), + if transpose_result { 1 } else { 0 } + ) + ); + Ok(return_) + } + + pub fn f_internal_ctc_loss( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: impl IntList, + target_lengths: impl IntList, + blank: i64, + zero_infinity: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__ctc_loss( + c_tensors.as_mut_ptr(), + log_probs.c_tensor, + targets.c_tensor, + input_lengths.as_ptr(), + input_lengths.len_i32(), + target_lengths.as_ptr(), + target_lengths.len_i32(), + blank, + if zero_infinity { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_ctc_loss_backward( + grad: &Tensor, + log_probs: &Tensor, + targets: &Tensor, + input_lengths: impl IntList, + target_lengths: impl IntList, + neg_log_likelihood: &Tensor, + log_alpha: &Tensor, + blank: i64, + zero_infinity: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__ctc_loss_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + log_probs.c_tensor, + targets.c_tensor, + input_lengths.as_ptr(), + input_lengths.len_i32(), + target_lengths.as_ptr(), + target_lengths.len_i32(), + neg_log_likelihood.c_tensor, + log_alpha.c_tensor, + blank, + if zero_infinity { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_ctc_loss_backward_out( + out: &Tensor, + grad: &Tensor, + log_probs: &Tensor, + targets: &Tensor, + input_lengths: impl IntList, + target_lengths: impl IntList, + neg_log_likelihood: &Tensor, + log_alpha: &Tensor, + blank: i64, + zero_infinity: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__ctc_loss_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad.c_tensor, + log_probs.c_tensor, + targets.c_tensor, + input_lengths.as_ptr(), + input_lengths.len_i32(), + target_lengths.as_ptr(), + target_lengths.len_i32(), + neg_log_likelihood.c_tensor, + log_alpha.c_tensor, + blank, + if zero_infinity { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_ctc_loss_backward_tensor( + grad: &Tensor, + log_probs: &Tensor, + targets: &Tensor, + input_lengths: &Tensor, + target_lengths: &Tensor, + neg_log_likelihood: &Tensor, + log_alpha: &Tensor, + blank: i64, + zero_infinity: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__ctc_loss_backward_tensor( + c_tensors.as_mut_ptr(), + grad.c_tensor, + log_probs.c_tensor, + targets.c_tensor, + input_lengths.c_tensor, + target_lengths.c_tensor, + neg_log_likelihood.c_tensor, + log_alpha.c_tensor, + blank, + if zero_infinity { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_ctc_loss_out( + out0: &Tensor, + out1: &Tensor, + log_probs: &Tensor, + targets: &Tensor, + input_lengths: impl IntList, + target_lengths: impl IntList, + blank: i64, + zero_infinity: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__ctc_loss_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + log_probs.c_tensor, + targets.c_tensor, + input_lengths.as_ptr(), + input_lengths.len_i32(), + target_lengths.as_ptr(), + target_lengths.len_i32(), + blank, + if zero_infinity { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_ctc_loss_tensor( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: &Tensor, + target_lengths: &Tensor, + blank: i64, + zero_infinity: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__ctc_loss_tensor( + c_tensors.as_mut_ptr(), + log_probs.c_tensor, + targets.c_tensor, + input_lengths.c_tensor, + target_lengths.c_tensor, + blank, + if zero_infinity { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_ctc_loss_tensor_out( + out0: &Tensor, + out1: &Tensor, + log_probs: &Tensor, + targets: &Tensor, + input_lengths: &Tensor, + target_lengths: &Tensor, + blank: i64, + zero_infinity: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__ctc_loss_tensor_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + log_probs.c_tensor, + targets.c_tensor, + input_lengths.c_tensor, + target_lengths.c_tensor, + blank, + if zero_infinity { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_cudnn_attention_backward( + grad_out: &Tensor, + query: &Tensor, + key: &Tensor, + value: &Tensor, + out: &Tensor, + logsumexp: &Tensor, + philox_seed: &Tensor, + philox_offset: &Tensor, + attn_bias: &Tensor, + cum_seq_q: &Tensor, + cum_seq_k: &Tensor, + max_q: i64, + max_k: i64, + dropout_p: f64, + is_causal: bool, + scale: impl Into>, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let scale = scale.into(); + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__cudnn_attention_backward( + c_tensors.as_mut_ptr(), + grad_out.c_tensor, + query.c_tensor, + key.c_tensor, + value.c_tensor, + out.c_tensor, + logsumexp.c_tensor, + philox_seed.c_tensor, + philox_offset.c_tensor, + attn_bias.c_tensor, + cum_seq_q.c_tensor, + cum_seq_k.c_tensor, + max_q, + max_k, + dropout_p, + if is_causal { 1 } else { 0 }, + scale.unwrap_or(std::f64::NAN), + scale.is_none() as i8 + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_cudnn_ctc_loss( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: impl IntList, + target_lengths: impl IntList, + blank: i64, + deterministic: bool, + zero_infinity: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__cudnn_ctc_loss( + c_tensors.as_mut_ptr(), + log_probs.c_tensor, + targets.c_tensor, + input_lengths.as_ptr(), + input_lengths.len_i32(), + target_lengths.as_ptr(), + target_lengths.len_i32(), + blank, + if deterministic { 1 } else { 0 }, + if zero_infinity { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_cudnn_ctc_loss_out( + out0: &Tensor, + out1: &Tensor, + log_probs: &Tensor, + targets: &Tensor, + input_lengths: impl IntList, + target_lengths: impl IntList, + blank: i64, + deterministic: bool, + zero_infinity: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__cudnn_ctc_loss_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + log_probs.c_tensor, + targets.c_tensor, + input_lengths.as_ptr(), + input_lengths.len_i32(), + target_lengths.as_ptr(), + target_lengths.len_i32(), + blank, + if deterministic { 1 } else { 0 }, + if zero_infinity { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_cudnn_ctc_loss_tensor( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: &Tensor, + target_lengths: &Tensor, + blank: i64, + deterministic: bool, + zero_infinity: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__cudnn_ctc_loss_tensor( + c_tensors.as_mut_ptr(), + log_probs.c_tensor, + targets.c_tensor, + input_lengths.c_tensor, + target_lengths.c_tensor, + blank, + if deterministic { 1 } else { 0 }, + if zero_infinity { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_cudnn_init_dropout_state( + dropout: f64, + train: bool, + dropout_seed: i64, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__cudnn_init_dropout_state( + c_tensors.as_mut_ptr(), + dropout, + if train { 1 } else { 0 }, + dropout_seed, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_cudnn_init_dropout_state_out( + out: &Tensor, + dropout: f64, + train: bool, + dropout_seed: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__cudnn_init_dropout_state_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + dropout, + if train { 1 } else { 0 }, + dropout_seed + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_cudnn_rnn>( + &self, + weight: &[T], + weight_stride0: i64, + weight_buf: Option, + hx: &Tensor, + cx: Option, + mode: i64, + hidden_size: i64, + proj_size: i64, + num_layers: i64, + batch_first: bool, + dropout: f64, + train: bool, + bidirectional: bool, + batch_sizes: impl IntList, + dropout_state: Option, + ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 5]; + unsafe_torch_err!(atg__cudnn_rnn( + c_tensors.as_mut_ptr(), + self.c_tensor, + ptr_list(weight).as_ptr(), + weight.len() as i32, + weight_stride0, + weight_buf.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + hx.c_tensor, + cx.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + mode, + hidden_size, + proj_size, + num_layers, + if batch_first { 1 } else { 0 }, + dropout, + if train { 1 } else { 0 }, + if bidirectional { 1 } else { 0 }, + batch_sizes.as_ptr(), + batch_sizes.len_i32(), + dropout_state.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + Tensor { c_tensor: c_tensors[4] }, + )) + } + + pub fn f_internal_cudnn_rnn_flatten_weight>( + weight_arr: &[T], + weight_stride0: i64, + input_size: i64, + mode: i64, + hidden_size: i64, + proj_size: i64, + num_layers: i64, + batch_first: bool, + bidirectional: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__cudnn_rnn_flatten_weight( + c_tensors.as_mut_ptr(), + ptr_list(weight_arr).as_ptr(), + weight_arr.len() as i32, + weight_stride0, + input_size, + mode, + hidden_size, + proj_size, + num_layers, + if batch_first { 1 } else { 0 }, + if bidirectional { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_cudnn_rnn_flatten_weight_out>( + out: &Tensor, + weight_arr: &[T], + weight_stride0: i64, + input_size: i64, + mode: i64, + hidden_size: i64, + proj_size: i64, + num_layers: i64, + batch_first: bool, + bidirectional: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__cudnn_rnn_flatten_weight_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + ptr_list(weight_arr).as_ptr(), + weight_arr.len() as i32, + weight_stride0, + input_size, + mode, + hidden_size, + proj_size, + num_layers, + if batch_first { 1 } else { 0 }, + if bidirectional { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_cudnn_rnn_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + out3: &Tensor, + out4: &Tensor, + weight: &[T], + weight_stride0: i64, + weight_buf: Option, + hx: &Tensor, + cx: Option, + mode: i64, + hidden_size: i64, + proj_size: i64, + num_layers: i64, + batch_first: bool, + dropout: f64, + train: bool, + bidirectional: bool, + batch_sizes: impl IntList, + dropout_state: Option, + ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 5]; + unsafe_torch_err!(atg__cudnn_rnn_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + out2.c_tensor, + out3.c_tensor, + out4.c_tensor, + self.c_tensor, + ptr_list(weight).as_ptr(), + weight.len() as i32, + weight_stride0, + weight_buf.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + hx.c_tensor, + cx.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + mode, + hidden_size, + proj_size, + num_layers, + if batch_first { 1 } else { 0 }, + dropout, + if train { 1 } else { 0 }, + if bidirectional { 1 } else { 0 }, + batch_sizes.as_ptr(), + batch_sizes.len_i32(), + dropout_state.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + Tensor { c_tensor: c_tensors[4] }, + )) + } + + pub fn f_internal_debug_has_internal_overlap(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg__debug_has_internal_overlap(self.c_tensor)); + Ok(return_) + } + + pub fn f_internal_dim_arange(like: &Tensor, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__dim_arange(c_tensors.as_mut_ptr(), like.c_tensor, dim)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_dimi(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg__dimi(self.c_tensor)); + Ok(return_) + } + + pub fn f_internal_dimv(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg__dimv(self.c_tensor)); + Ok(return_) + } + + pub fn f_internal_dirichlet_grad( + x: &Tensor, + alpha: &Tensor, + total: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__dirichlet_grad( + c_tensors.as_mut_ptr(), + x.c_tensor, + alpha.c_tensor, + total.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_dirichlet_grad_out( + out: &Tensor, + x: &Tensor, + alpha: &Tensor, + total: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__dirichlet_grad_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + alpha.c_tensor, + total.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_dyn_quant_matmul_4bit( + inp: &Tensor, + packed_weights: &Tensor, + block_size: i64, + in_features: i64, + out_features: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__dyn_quant_matmul_4bit( + c_tensors.as_mut_ptr(), + inp.c_tensor, + packed_weights.c_tensor, + block_size, + in_features, + out_features + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_dyn_quant_pack_4bit_weight>( + weights: &Tensor, + scales_zeros: &Tensor, + bias: Option, + block_size: i64, + in_features: i64, + out_features: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__dyn_quant_pack_4bit_weight( + c_tensors.as_mut_ptr(), + weights.c_tensor, + scales_zeros.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + block_size, + in_features, + out_features + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_efficient_attention_backward>( + grad_out_: &Tensor, + query: &Tensor, + key: &Tensor, + value: &Tensor, + bias: Option, + out: &Tensor, + cu_seqlens_q: Option, + cu_seqlens_k: Option, + max_seqlen_q: i64, + max_seqlen_k: i64, + logsumexp: &Tensor, + dropout_p: f64, + philox_seed: &Tensor, + philox_offset: &Tensor, + custom_mask_type: i64, + bias_requires_grad: bool, + scale: impl Into>, + num_splits_key: impl Into>, + window_size: impl Into>, + shared_storage_dqdkdv: bool, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let scale = scale.into(); + let num_splits_key = num_splits_key.into(); + let window_size = window_size.into(); + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg__efficient_attention_backward( + c_tensors.as_mut_ptr(), + grad_out_.c_tensor, + query.c_tensor, + key.c_tensor, + value.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + out.c_tensor, + cu_seqlens_q.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + cu_seqlens_k.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + max_seqlen_q, + max_seqlen_k, + logsumexp.c_tensor, + dropout_p, + philox_seed.c_tensor, + philox_offset.c_tensor, + custom_mask_type, + if bias_requires_grad { 1 } else { 0 }, + scale.unwrap_or(std::f64::NAN), + scale.is_none() as i8, + num_splits_key.unwrap_or(0i64), + num_splits_key.is_none() as i8, + window_size.unwrap_or(0i64), + window_size.is_none() as i8, + if shared_storage_dqdkdv { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_internal_efficientzerotensor( + size: impl IntList, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__efficientzerotensor( + c_tensors.as_mut_ptr(), + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_efficientzerotensor_out( + out: &Tensor, + size: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__efficientzerotensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_embedding_bag>( + weight: &Tensor, + indices: &Tensor, + offsets: &Tensor, + scale_grad_by_freq: bool, + mode: i64, + sparse: bool, + per_sample_weights: Option, + include_last_offset: bool, + padding_idx: i64, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg__embedding_bag( + c_tensors.as_mut_ptr(), + weight.c_tensor, + indices.c_tensor, + offsets.c_tensor, + if scale_grad_by_freq { 1 } else { 0 }, + mode, + if sparse { 1 } else { 0 }, + per_sample_weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if include_last_offset { 1 } else { 0 }, + padding_idx + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_internal_embedding_bag_backward>( + grad: &Tensor, + indices: &Tensor, + offsets: &Tensor, + offset2bag: &Tensor, + bag_size: &Tensor, + maximum_indices: &Tensor, + num_weights: i64, + scale_grad_by_freq: bool, + mode: i64, + sparse: bool, + per_sample_weights: Option, + padding_idx: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__embedding_bag_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + indices.c_tensor, + offsets.c_tensor, + offset2bag.c_tensor, + bag_size.c_tensor, + maximum_indices.c_tensor, + num_weights, + if scale_grad_by_freq { 1 } else { 0 }, + mode, + if sparse { 1 } else { 0 }, + per_sample_weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + padding_idx + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_embedding_bag_dense_backward>( + grad: &Tensor, + indices: &Tensor, + offset2bag: &Tensor, + bag_size: &Tensor, + maximum_indices: &Tensor, + num_weights: i64, + scale_grad_by_freq: bool, + mode: i64, + per_sample_weights: Option, + padding_idx: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__embedding_bag_dense_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + indices.c_tensor, + offset2bag.c_tensor, + bag_size.c_tensor, + maximum_indices.c_tensor, + num_weights, + if scale_grad_by_freq { 1 } else { 0 }, + mode, + per_sample_weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + padding_idx + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_embedding_bag_dense_backward_out>( + out: &Tensor, + grad: &Tensor, + indices: &Tensor, + offset2bag: &Tensor, + bag_size: &Tensor, + maximum_indices: &Tensor, + num_weights: i64, + scale_grad_by_freq: bool, + mode: i64, + per_sample_weights: Option, + padding_idx: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__embedding_bag_dense_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad.c_tensor, + indices.c_tensor, + offset2bag.c_tensor, + bag_size.c_tensor, + maximum_indices.c_tensor, + num_weights, + if scale_grad_by_freq { 1 } else { 0 }, + mode, + per_sample_weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + padding_idx + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_embedding_bag_forward_only>( + weight: &Tensor, + indices: &Tensor, + offsets: &Tensor, + scale_grad_by_freq: bool, + mode: i64, + sparse: bool, + per_sample_weights: Option, + include_last_offset: bool, + padding_idx: i64, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg__embedding_bag_forward_only( + c_tensors.as_mut_ptr(), + weight.c_tensor, + indices.c_tensor, + offsets.c_tensor, + if scale_grad_by_freq { 1 } else { 0 }, + mode, + if sparse { 1 } else { 0 }, + per_sample_weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if include_last_offset { 1 } else { 0 }, + padding_idx + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_internal_embedding_bag_forward_only_out>( + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + out3: &Tensor, + weight: &Tensor, + indices: &Tensor, + offsets: &Tensor, + scale_grad_by_freq: bool, + mode: i64, + sparse: bool, + per_sample_weights: Option, + include_last_offset: bool, + padding_idx: i64, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg__embedding_bag_forward_only_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + out2.c_tensor, + out3.c_tensor, + weight.c_tensor, + indices.c_tensor, + offsets.c_tensor, + if scale_grad_by_freq { 1 } else { 0 }, + mode, + if sparse { 1 } else { 0 }, + per_sample_weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if include_last_offset { 1 } else { 0 }, + padding_idx + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_internal_embedding_bag_out>( + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + out3: &Tensor, + weight: &Tensor, + indices: &Tensor, + offsets: &Tensor, + scale_grad_by_freq: bool, + mode: i64, + sparse: bool, + per_sample_weights: Option, + include_last_offset: bool, + padding_idx: i64, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg__embedding_bag_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + out2.c_tensor, + out3.c_tensor, + weight.c_tensor, + indices.c_tensor, + offsets.c_tensor, + if scale_grad_by_freq { 1 } else { 0 }, + mode, + if sparse { 1 } else { 0 }, + per_sample_weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if include_last_offset { 1 } else { 0 }, + padding_idx + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_internal_embedding_bag_per_sample_weights_backward( + grad: &Tensor, + weight: &Tensor, + indices: &Tensor, + offsets: &Tensor, + offset2bag: &Tensor, + mode: i64, + padding_idx: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__embedding_bag_per_sample_weights_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + weight.c_tensor, + indices.c_tensor, + offsets.c_tensor, + offset2bag.c_tensor, + mode, + padding_idx + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_embedding_bag_per_sample_weights_backward_out( + out: &Tensor, + grad: &Tensor, + weight: &Tensor, + indices: &Tensor, + offsets: &Tensor, + offset2bag: &Tensor, + mode: i64, + padding_idx: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__embedding_bag_per_sample_weights_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad.c_tensor, + weight.c_tensor, + indices.c_tensor, + offsets.c_tensor, + offset2bag.c_tensor, + mode, + padding_idx + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_embedding_bag_sparse_backward>( + grad: &Tensor, + indices: &Tensor, + offsets: &Tensor, + offset2bag: &Tensor, + bag_size: &Tensor, + num_weights: i64, + scale_grad_by_freq: bool, + mode: i64, + per_sample_weights: Option, + padding_idx: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__embedding_bag_sparse_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + indices.c_tensor, + offsets.c_tensor, + offset2bag.c_tensor, + bag_size.c_tensor, + num_weights, + if scale_grad_by_freq { 1 } else { 0 }, + mode, + per_sample_weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + padding_idx + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_empty_affine_quantized( + size: impl IntList, + options: (Kind, Device), + scale: f64, + zero_point: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__empty_affine_quantized( + c_tensors.as_mut_ptr(), + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int(), + scale, + zero_point + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_empty_affine_quantized_out( + out: &Tensor, + size: impl IntList, + scale: f64, + zero_point: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__empty_affine_quantized_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + size.as_ptr(), + size.len_i32(), + scale, + zero_point + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_empty_per_channel_affine_quantized( + size: impl IntList, + scales: &Tensor, + zero_points: &Tensor, + axis: i64, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__empty_per_channel_affine_quantized( + c_tensors.as_mut_ptr(), + size.as_ptr(), + size.len_i32(), + scales.c_tensor, + zero_points.c_tensor, + axis, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_empty_per_channel_affine_quantized_out( + out: &Tensor, + size: impl IntList, + scales: &Tensor, + zero_points: &Tensor, + axis: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__empty_per_channel_affine_quantized_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + size.as_ptr(), + size.len_i32(), + scales.c_tensor, + zero_points.c_tensor, + axis + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_euclidean_dist(x1: &Tensor, x2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__euclidean_dist(c_tensors.as_mut_ptr(), x1.c_tensor, x2.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_euclidean_dist_out( + out: &Tensor, + x1: &Tensor, + x2: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__euclidean_dist_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x1.c_tensor, + x2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_fake_quantize_learnable_per_channel_affine( + &self, + scale: &Tensor, + zero_point: &Tensor, + axis: i64, + quant_min: i64, + quant_max: i64, + grad_factor: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__fake_quantize_learnable_per_channel_affine( + c_tensors.as_mut_ptr(), + self.c_tensor, + scale.c_tensor, + zero_point.c_tensor, + axis, + quant_min, + quant_max, + grad_factor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_fake_quantize_learnable_per_channel_affine_backward( + &self, + grad: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + axis: i64, + quant_min: i64, + quant_max: i64, + grad_factor: f64, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__fake_quantize_learnable_per_channel_affine_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + self.c_tensor, + scale.c_tensor, + zero_point.c_tensor, + axis, + quant_min, + quant_max, + grad_factor + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_fake_quantize_learnable_per_channel_affine_out( + &self, + out: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + axis: i64, + quant_min: i64, + quant_max: i64, + grad_factor: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__fake_quantize_learnable_per_channel_affine_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + scale.c_tensor, + zero_point.c_tensor, + axis, + quant_min, + quant_max, + grad_factor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_fake_quantize_learnable_per_tensor_affine( + &self, + scale: &Tensor, + zero_point: &Tensor, + quant_min: i64, + quant_max: i64, + grad_factor: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__fake_quantize_learnable_per_tensor_affine( + c_tensors.as_mut_ptr(), + self.c_tensor, + scale.c_tensor, + zero_point.c_tensor, + quant_min, + quant_max, + grad_factor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_fake_quantize_learnable_per_tensor_affine_backward( + &self, + grad: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + quant_min: i64, + quant_max: i64, + grad_factor: f64, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__fake_quantize_learnable_per_tensor_affine_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + self.c_tensor, + scale.c_tensor, + zero_point.c_tensor, + quant_min, + quant_max, + grad_factor + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_fake_quantize_learnable_per_tensor_affine_out( + &self, + out: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + quant_min: i64, + quant_max: i64, + grad_factor: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__fake_quantize_learnable_per_tensor_affine_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + scale.c_tensor, + zero_point.c_tensor, + quant_min, + quant_max, + grad_factor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_fake_quantize_per_tensor_affine_cachemask_tensor_qparams( + &self, + scale: &Tensor, + zero_point: &Tensor, + fake_quant_enabled: &Tensor, + quant_min: i64, + quant_max: i64, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__fake_quantize_per_tensor_affine_cachemask_tensor_qparams( + c_tensors.as_mut_ptr(), + self.c_tensor, + scale.c_tensor, + zero_point.c_tensor, + fake_quant_enabled.c_tensor, + quant_min, + quant_max + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_fake_quantize_per_tensor_affine_cachemask_tensor_qparams_out( + &self, + out0: &Tensor, + out1: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + fake_quant_enabled: &Tensor, + quant_min: i64, + quant_max: i64, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__fake_quantize_per_tensor_affine_cachemask_tensor_qparams_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + self.c_tensor, + scale.c_tensor, + zero_point.c_tensor, + fake_quant_enabled.c_tensor, + quant_min, + quant_max + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_fft_c2c( + &self, + dim: impl IntList, + normalization: i64, + forward: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__fft_c2c( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + normalization, + if forward { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_fft_c2c_out( + &self, + out: &Tensor, + dim: impl IntList, + normalization: i64, + forward: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__fft_c2c_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + normalization, + if forward { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_fft_c2r( + &self, + dim: impl IntList, + normalization: i64, + last_dim_size: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__fft_c2r( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + normalization, + last_dim_size + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_fft_c2r_out( + &self, + out: &Tensor, + dim: impl IntList, + normalization: i64, + last_dim_size: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__fft_c2r_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + normalization, + last_dim_size + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_fft_r2c( + &self, + dim: impl IntList, + normalization: i64, + onesided: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__fft_r2c( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + normalization, + if onesided { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_fft_r2c_out( + &self, + out: &Tensor, + dim: impl IntList, + normalization: i64, + onesided: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__fft_r2c_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + normalization, + if onesided { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_fill_mem_eff_dropout_mask_( + &mut self, + dropout_p: f64, + seed: i64, + offset: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__fill_mem_eff_dropout_mask_( + c_tensors.as_mut_ptr(), + self.c_tensor, + dropout_p, + seed, + offset + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_flash_attention_backward( + grad_out: &Tensor, + query: &Tensor, + key: &Tensor, + value: &Tensor, + out: &Tensor, + logsumexp: &Tensor, + cum_seq_q: &Tensor, + cum_seq_k: &Tensor, + max_q: i64, + max_k: i64, + dropout_p: f64, + is_causal: bool, + rng_state: &Tensor, + unused: &Tensor, + scale: impl Into>, + window_size_left: impl Into>, + window_size_right: impl Into>, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let scale = scale.into(); + let window_size_left = window_size_left.into(); + let window_size_right = window_size_right.into(); + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__flash_attention_backward( + c_tensors.as_mut_ptr(), + grad_out.c_tensor, + query.c_tensor, + key.c_tensor, + value.c_tensor, + out.c_tensor, + logsumexp.c_tensor, + cum_seq_q.c_tensor, + cum_seq_k.c_tensor, + max_q, + max_k, + dropout_p, + if is_causal { 1 } else { 0 }, + rng_state.c_tensor, + unused.c_tensor, + scale.unwrap_or(std::f64::NAN), + scale.is_none() as i8, + window_size_left.unwrap_or(0i64), + window_size_left.is_none() as i8, + window_size_right.unwrap_or(0i64), + window_size_right.is_none() as i8 + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_foobar( + &self, + arg1: bool, + arg2: bool, + arg3: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__foobar( + c_tensors.as_mut_ptr(), + self.c_tensor, + if arg1 { 1 } else { 0 }, + if arg2 { 1 } else { 0 }, + if arg3 { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_foobar_out( + &self, + out: &Tensor, + arg1: bool, + arg2: bool, + arg3: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__foobar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + if arg1 { 1 } else { 0 }, + if arg2 { 1 } else { 0 }, + if arg3 { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_functional_assert_async( + &self, + assert_msg: &str, + dep_token: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__functional_assert_async( + c_tensors.as_mut_ptr(), + self.c_tensor, + assert_msg.as_ptr(), + assert_msg.len() as i32, + dep_token.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_functional_assert_scalar>( + self_scalar: S, + assert_msg: &str, + dep_token: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__functional_assert_scalar( + c_tensors.as_mut_ptr(), + self_scalar.into().c_scalar, + assert_msg.as_ptr(), + assert_msg.len() as i32, + dep_token.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_functional_sym_constrain_range>( + size: S, + min: impl Into>, + max: impl Into>, + dep_token: &Tensor, + ) -> Result { + let min = min.into(); + let max = max.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__functional_sym_constrain_range( + c_tensors.as_mut_ptr(), + size.into().c_scalar, + min.unwrap_or(0i64), + min.is_none() as i8, + max.unwrap_or(0i64), + max.is_none() as i8, + dep_token.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_functional_sym_constrain_range_for_size>( + size: S, + min: impl Into>, + max: impl Into>, + dep_token: &Tensor, + ) -> Result { + let min = min.into(); + let max = max.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__functional_sym_constrain_range_for_size( + c_tensors.as_mut_ptr(), + size.into().c_scalar, + min.unwrap_or(0i64), + min.is_none() as i8, + max.unwrap_or(0i64), + max.is_none() as i8, + dep_token.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_fused_dropout(&self, p: f64) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__fused_dropout(c_tensors.as_mut_ptr(), self.c_tensor, p)); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_fused_dropout_out( + &self, + out0: &Tensor, + out1: &Tensor, + p: f64, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__fused_dropout_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + self.c_tensor, + p + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_fused_moving_avg_obs_fq_helper( + &self, + observer_on: &Tensor, + fake_quant_on: &Tensor, + running_min: &Tensor, + running_max: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + averaging_const: f64, + quant_min: i64, + quant_max: i64, + ch_axis: i64, + per_row_fake_quant: bool, + symmetric_quant: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__fused_moving_avg_obs_fq_helper( + c_tensors.as_mut_ptr(), + self.c_tensor, + observer_on.c_tensor, + fake_quant_on.c_tensor, + running_min.c_tensor, + running_max.c_tensor, + scale.c_tensor, + zero_point.c_tensor, + averaging_const, + quant_min, + quant_max, + ch_axis, + if per_row_fake_quant { 1 } else { 0 }, + if symmetric_quant { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_fused_moving_avg_obs_fq_helper_functional( + &self, + observer_on: &Tensor, + fake_quant_on: &Tensor, + running_min: &Tensor, + running_max: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + averaging_const: f64, + quant_min: i64, + quant_max: i64, + ch_axis: i64, + per_row_fake_quant: bool, + symmetric_quant: bool, + ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 6]; + unsafe_torch_err!(atg__fused_moving_avg_obs_fq_helper_functional( + c_tensors.as_mut_ptr(), + self.c_tensor, + observer_on.c_tensor, + fake_quant_on.c_tensor, + running_min.c_tensor, + running_max.c_tensor, + scale.c_tensor, + zero_point.c_tensor, + averaging_const, + quant_min, + quant_max, + ch_axis, + if per_row_fake_quant { 1 } else { 0 }, + if symmetric_quant { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + Tensor { c_tensor: c_tensors[4] }, + Tensor { c_tensor: c_tensors[5] }, + )) + } + + pub fn f_internal_fused_moving_avg_obs_fq_helper_out( + &self, + out0: &Tensor, + out1: &Tensor, + observer_on: &Tensor, + fake_quant_on: &Tensor, + running_min: &Tensor, + running_max: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + averaging_const: f64, + quant_min: i64, + quant_max: i64, + ch_axis: i64, + per_row_fake_quant: bool, + symmetric_quant: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__fused_moving_avg_obs_fq_helper_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + self.c_tensor, + observer_on.c_tensor, + fake_quant_on.c_tensor, + running_min.c_tensor, + running_max.c_tensor, + scale.c_tensor, + zero_point.c_tensor, + averaging_const, + quant_min, + quant_max, + ch_axis, + if per_row_fake_quant { 1 } else { 0 }, + if symmetric_quant { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_fused_rms_norm>( + &self, + normalized_shape: impl IntList, + weight: Option, + eps: impl Into>, + ) -> Result<(Tensor, Tensor), TchError> { + let eps = eps.into(); + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__fused_rms_norm( + c_tensors.as_mut_ptr(), + self.c_tensor, + normalized_shape.as_ptr(), + normalized_shape.len_i32(), + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + eps.unwrap_or(std::f64::NAN), + eps.is_none() as i8 + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_fused_sdp_choice>( + query: &Tensor, + key: &Tensor, + value: &Tensor, + attn_mask: Option, + dropout_p: f64, + is_causal: bool, + scale: impl Into>, + enable_gqa: bool, + ) -> Result { + let scale = scale.into(); + let return_; + unsafe_torch_err!( + return_ = atg__fused_sdp_choice( + query.c_tensor, + key.c_tensor, + value.c_tensor, + attn_mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + dropout_p, + if is_causal { 1 } else { 0 }, + scale.unwrap_or(std::f64::NAN), + scale.is_none() as i8, + if enable_gqa { 1 } else { 0 } + ) + ); + Ok(return_) + } + + pub fn f_internal_fw_primal(&self, level: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__fw_primal(c_tensors.as_mut_ptr(), self.c_tensor, level)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_fw_primal_copy(&self, level: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__fw_primal_copy(c_tensors.as_mut_ptr(), self.c_tensor, level)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_fw_primal_copy_out( + &self, + out: &Tensor, + level: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__fw_primal_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + level + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_gather_sparse_backward( + &self, + dim: i64, + index: &Tensor, + grad: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__gather_sparse_backward( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + grad.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_grid_sampler_2d_cpu_fallback( + &self, + grid: &Tensor, + interpolation_mode: i64, + padding_mode: i64, + align_corners: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__grid_sampler_2d_cpu_fallback( + c_tensors.as_mut_ptr(), + self.c_tensor, + grid.c_tensor, + interpolation_mode, + padding_mode, + if align_corners { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_grid_sampler_2d_cpu_fallback_backward( + &self, + grad_output: &Tensor, + grid: &Tensor, + interpolation_mode: i64, + padding_mode: i64, + align_corners: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__grid_sampler_2d_cpu_fallback_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + grid.c_tensor, + interpolation_mode, + padding_mode, + if align_corners { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_grid_sampler_2d_cpu_fallback_out( + &self, + out: &Tensor, + grid: &Tensor, + interpolation_mode: i64, + padding_mode: i64, + align_corners: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__grid_sampler_2d_cpu_fallback_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + grid.c_tensor, + interpolation_mode, + padding_mode, + if align_corners { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_grouped_mm>( + &self, + mat2: &Tensor, + offs: Option, + bias: Option, + out_dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__grouped_mm( + c_tensors.as_mut_ptr(), + self.c_tensor, + mat2.c_tensor, + offs.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + out_dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_has_compatible_shallow_copy_type( + &self, + from: &Tensor, + ) -> Result { + let return_; + unsafe_torch_err!( + return_ = atg__has_compatible_shallow_copy_type(self.c_tensor, from.c_tensor) + ); + Ok(return_ != 0) + } + + pub fn f_internal_has_same_storage_numel(&self, other: &Tensor) -> Result { + let return_; + unsafe_torch_err!(return_ = atg__has_same_storage_numel(self.c_tensor, other.c_tensor)); + Ok(return_ != 0) + } + + pub fn f_internal_histogramdd_bin_edges>( + &self, + bins: impl IntList, + range: impl DoubleList, + weight: Option, + density: bool, + ) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg__histogramdd_bin_edges( + self.c_tensor, + bins.as_ptr(), + bins.len_i32(), + range.as_ptr(), + range.len_i32(), + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if density { 1 } else { 0 } + )); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_internal_histogramdd_bin_edges_out>( + &self, + out: &[T], + bins: impl IntList, + range: impl DoubleList, + weight: Option, + density: bool, + ) -> Result<(), TchError> { + unsafe_torch_err!(atg__histogramdd_bin_edges_out( + ptr_list(out).as_ptr(), + out.len() as i32, + self.c_tensor, + bins.as_ptr(), + bins.len_i32(), + range.as_ptr(), + range.len_i32(), + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if density { 1 } else { 0 } + )); + Ok(()) + } + + pub fn f_internal_histogramdd_from_bin_cts>( + &self, + bins: impl IntList, + range: impl DoubleList, + weight: Option, + density: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__histogramdd_from_bin_cts( + c_tensors.as_mut_ptr(), + self.c_tensor, + bins.as_ptr(), + bins.len_i32(), + range.as_ptr(), + range.len_i32(), + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if density { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_histogramdd_from_bin_cts_out>( + &self, + out: &Tensor, + bins: impl IntList, + range: impl DoubleList, + weight: Option, + density: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__histogramdd_from_bin_cts_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + bins.as_ptr(), + bins.len_i32(), + range.as_ptr(), + range.len_i32(), + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if density { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_histogramdd_from_bin_tensors>( + &self, + bins: &[T], + weight: Option, + density: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__histogramdd_from_bin_tensors( + c_tensors.as_mut_ptr(), + self.c_tensor, + ptr_list(bins).as_ptr(), + bins.len() as i32, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if density { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_histogramdd_from_bin_tensors_out>( + &self, + out: &Tensor, + bins: &[T], + weight: Option, + density: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__histogramdd_from_bin_tensors_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + ptr_list(bins).as_ptr(), + bins.len() as i32, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if density { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_index_put_impl>( + &self, + indices: &[Option], + values: &Tensor, + accumulate: bool, + unsafe_: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__index_put_impl( + c_tensors.as_mut_ptr(), + self.c_tensor, + ptr_list_opt(indices).as_ptr(), + indices.len() as i32, + values.c_tensor, + if accumulate { 1 } else { 0 }, + if unsafe_ { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_index_put_impl_>( + &mut self, + indices: &[Option], + values: &Tensor, + accumulate: bool, + unsafe_: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__index_put_impl_( + c_tensors.as_mut_ptr(), + self.c_tensor, + ptr_list_opt(indices).as_ptr(), + indices.len() as i32, + values.c_tensor, + if accumulate { 1 } else { 0 }, + if unsafe_ { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_index_put_impl_out>( + &self, + out: &Tensor, + indices: &[Option], + values: &Tensor, + accumulate: bool, + unsafe_: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__index_put_impl_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + ptr_list_opt(indices).as_ptr(), + indices.len() as i32, + values.c_tensor, + if accumulate { 1 } else { 0 }, + if unsafe_ { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_indices(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__indices(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_indices_copy(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__indices_copy(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_indices_copy_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__indices_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_int_mm(&self, mat2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__int_mm(c_tensors.as_mut_ptr(), self.c_tensor, mat2.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_int_mm_out(&self, out: &Tensor, mat2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__int_mm_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mat2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_is_all_true(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__is_all_true(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_is_any_true(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__is_any_true(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_is_zerotensor(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg__is_zerotensor(self.c_tensor)); + Ok(return_ != 0) + } + + pub fn f_internal_lazy_clone(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__lazy_clone(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_linalg_check_errors( + info: &Tensor, + api_name: &str, + is_matrix: bool, + ) -> Result<(), TchError> { + unsafe_torch_err!(atg__linalg_check_errors( + info.c_tensor, + api_name.as_ptr(), + api_name.len() as i32, + if is_matrix { 1 } else { 0 } + )); + Ok(()) + } + + pub fn f_internal_linalg_det(a: &Tensor) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__linalg_det(c_tensors.as_mut_ptr(), a.c_tensor)); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_linalg_det_result( + result: &Tensor, + lu: &Tensor, + pivots: &Tensor, + a: &Tensor, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__linalg_det_result( + c_tensors.as_mut_ptr(), + result.c_tensor, + lu.c_tensor, + pivots.c_tensor, + a.c_tensor + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_linalg_eigh( + a: &Tensor, + uplo: &str, + compute_v: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__linalg_eigh( + c_tensors.as_mut_ptr(), + a.c_tensor, + uplo.as_ptr(), + uplo.len() as i32, + if compute_v { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_linalg_eigh_eigenvalues( + eigenvalues: &Tensor, + eigenvectors: &Tensor, + a: &Tensor, + uplo: &str, + compute_v: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__linalg_eigh_eigenvalues( + c_tensors.as_mut_ptr(), + eigenvalues.c_tensor, + eigenvectors.c_tensor, + a.c_tensor, + uplo.as_ptr(), + uplo.len() as i32, + if compute_v { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_linalg_eigvals(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__linalg_eigvals(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_linalg_slogdet( + a: &Tensor, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg__linalg_slogdet(c_tensors.as_mut_ptr(), a.c_tensor)); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_internal_linalg_slogdet_sign( + sign: &Tensor, + logabsdet: &Tensor, + lu: &Tensor, + pivots: &Tensor, + a: &Tensor, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg__linalg_slogdet_sign( + c_tensors.as_mut_ptr(), + sign.c_tensor, + logabsdet.c_tensor, + lu.c_tensor, + pivots.c_tensor, + a.c_tensor + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_internal_linalg_solve_ex( + a: &Tensor, + b: &Tensor, + left: bool, + check_errors: bool, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg__linalg_solve_ex( + c_tensors.as_mut_ptr(), + a.c_tensor, + b.c_tensor, + if left { 1 } else { 0 }, + if check_errors { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_internal_linalg_solve_ex_result( + result: &Tensor, + lu: &Tensor, + pivots: &Tensor, + info: &Tensor, + a: &Tensor, + b: &Tensor, + left: bool, + check_errors: bool, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg__linalg_solve_ex_result( + c_tensors.as_mut_ptr(), + result.c_tensor, + lu.c_tensor, + pivots.c_tensor, + info.c_tensor, + a.c_tensor, + b.c_tensor, + if left { 1 } else { 0 }, + if check_errors { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_internal_linalg_svd( + a: &Tensor, + full_matrices: bool, + compute_uv: bool, + driver: &str, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__linalg_svd( + c_tensors.as_mut_ptr(), + a.c_tensor, + if full_matrices { 1 } else { 0 }, + if compute_uv { 1 } else { 0 }, + driver.as_ptr(), + driver.len() as i32 + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_linalg_svd_u( + u: &Tensor, + s: &Tensor, + vh: &Tensor, + a: &Tensor, + full_matrices: bool, + compute_uv: bool, + driver: &str, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__linalg_svd_u( + c_tensors.as_mut_ptr(), + u.c_tensor, + s.c_tensor, + vh.c_tensor, + a.c_tensor, + if full_matrices { 1 } else { 0 }, + if compute_uv { 1 } else { 0 }, + driver.as_ptr(), + driver.len() as i32 + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_log_softmax( + &self, + dim: i64, + half_to_float: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__log_softmax( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + if half_to_float { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_log_softmax_backward_data( + grad_output: &Tensor, + output: &Tensor, + dim: i64, + input_dtype: Kind, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__log_softmax_backward_data( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + output.c_tensor, + dim, + input_dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_log_softmax_backward_data_out( + out: &Tensor, + grad_output: &Tensor, + output: &Tensor, + dim: i64, + input_dtype: Kind, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__log_softmax_backward_data_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad_output.c_tensor, + output.c_tensor, + dim, + input_dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_log_softmax_out( + &self, + out: &Tensor, + dim: i64, + half_to_float: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__log_softmax_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + if half_to_float { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_logcumsumexp(&self, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__logcumsumexp(c_tensors.as_mut_ptr(), self.c_tensor, dim)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_logcumsumexp_out(&self, out: &Tensor, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__logcumsumexp_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_lstm_mps>( + &self, + hx: &[T], + params: &[T], + has_biases: bool, + num_layers: i64, + dropout: f64, + train: bool, + bidirectional: bool, + batch_first: bool, + ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 6]; + unsafe_torch_err!(atg__lstm_mps( + c_tensors.as_mut_ptr(), + self.c_tensor, + ptr_list(hx).as_ptr(), + hx.len() as i32, + ptr_list(params).as_ptr(), + params.len() as i32, + if has_biases { 1 } else { 0 }, + num_layers, + dropout, + if train { 1 } else { 0 }, + if bidirectional { 1 } else { 0 }, + if batch_first { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + Tensor { c_tensor: c_tensors[4] }, + Tensor { c_tensor: c_tensors[5] }, + )) + } + + pub fn f_internal_lstm_mps_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + out3: &Tensor, + out4: &Tensor, + out5: &Tensor, + hx: &[T], + params: &[T], + has_biases: bool, + num_layers: i64, + dropout: f64, + train: bool, + bidirectional: bool, + batch_first: bool, + ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 6]; + unsafe_torch_err!(atg__lstm_mps_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + out2.c_tensor, + out3.c_tensor, + out4.c_tensor, + out5.c_tensor, + self.c_tensor, + ptr_list(hx).as_ptr(), + hx.len() as i32, + ptr_list(params).as_ptr(), + params.len() as i32, + if has_biases { 1 } else { 0 }, + num_layers, + dropout, + if train { 1 } else { 0 }, + if bidirectional { 1 } else { 0 }, + if batch_first { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + Tensor { c_tensor: c_tensors[4] }, + Tensor { c_tensor: c_tensors[5] }, + )) + } + + pub fn f_internal_lu_with_info( + &self, + pivot: bool, + check_errors: bool, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__lu_with_info( + c_tensors.as_mut_ptr(), + self.c_tensor, + if pivot { 1 } else { 0 }, + if check_errors { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_make_dep_token(options: (Kind, Device)) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__make_dep_token( + c_tensors.as_mut_ptr(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_make_dual( + primal: &Tensor, + tangent: &Tensor, + level: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__make_dual( + c_tensors.as_mut_ptr(), + primal.c_tensor, + tangent.c_tensor, + level + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_make_dual_copy( + primal: &Tensor, + tangent: &Tensor, + level: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__make_dual_copy( + c_tensors.as_mut_ptr(), + primal.c_tensor, + tangent.c_tensor, + level + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_make_dual_copy_out( + out: &Tensor, + primal: &Tensor, + tangent: &Tensor, + level: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__make_dual_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + primal.c_tensor, + tangent.c_tensor, + level + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_make_per_channel_quantized_tensor( + &self, + scale: &Tensor, + zero_point: &Tensor, + axis: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__make_per_channel_quantized_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + scale.c_tensor, + zero_point.c_tensor, + axis + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_make_per_channel_quantized_tensor_out( + &self, + out: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + axis: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__make_per_channel_quantized_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + scale.c_tensor, + zero_point.c_tensor, + axis + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_make_per_tensor_quantized_tensor( + &self, + scale: f64, + zero_point: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__make_per_tensor_quantized_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + scale, + zero_point + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_make_per_tensor_quantized_tensor_out( + &self, + out: &Tensor, + scale: f64, + zero_point: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__make_per_tensor_quantized_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + scale, + zero_point + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_masked_scale(&self, mask: &Tensor, scale: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__masked_scale( + c_tensors.as_mut_ptr(), + self.c_tensor, + mask.c_tensor, + scale + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_masked_scale_out( + &self, + out: &Tensor, + mask: &Tensor, + scale: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__masked_scale_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mask.c_tensor, + scale + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_masked_softmax( + &self, + mask: &Tensor, + dim: impl Into>, + mask_type: impl Into>, + ) -> Result { + let dim = dim.into(); + let mask_type = mask_type.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__masked_softmax( + c_tensors.as_mut_ptr(), + self.c_tensor, + mask.c_tensor, + dim.unwrap_or(0i64), + dim.is_none() as i8, + mask_type.unwrap_or(0i64), + mask_type.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_masked_softmax_backward( + grad_output: &Tensor, + output: &Tensor, + mask: &Tensor, + dim: impl Into>, + ) -> Result { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__masked_softmax_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + output.c_tensor, + mask.c_tensor, + dim.unwrap_or(0i64), + dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_masked_softmax_backward_out( + out: &Tensor, + grad_output: &Tensor, + output: &Tensor, + mask: &Tensor, + dim: impl Into>, + ) -> Result { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__masked_softmax_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad_output.c_tensor, + output.c_tensor, + mask.c_tensor, + dim.unwrap_or(0i64), + dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_masked_softmax_out( + &self, + out: &Tensor, + mask: &Tensor, + dim: impl Into>, + mask_type: impl Into>, + ) -> Result { + let dim = dim.into(); + let mask_type = mask_type.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__masked_softmax_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mask.c_tensor, + dim.unwrap_or(0i64), + dim.is_none() as i8, + mask_type.unwrap_or(0i64), + mask_type.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_mixed_dtypes_linear>( + &self, + weight: &Tensor, + scale: &Tensor, + bias: Option, + activation: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__mixed_dtypes_linear( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + scale.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + activation.as_ptr(), + activation.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_mkldnn_reshape(&self, shape: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__mkldnn_reshape( + c_tensors.as_mut_ptr(), + self.c_tensor, + shape.as_ptr(), + shape.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_mkldnn_reshape_out( + &self, + out: &Tensor, + shape: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__mkldnn_reshape_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + shape.as_ptr(), + shape.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_mkldnn_transpose(&self, dim0: i64, dim1: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__mkldnn_transpose(c_tensors.as_mut_ptr(), self.c_tensor, dim0, dim1)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_mkldnn_transpose_( + &mut self, + dim0: i64, + dim1: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__mkldnn_transpose_( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim0, + dim1 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_mkldnn_transpose_out( + &self, + out: &Tensor, + dim0: i64, + dim1: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__mkldnn_transpose_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim0, + dim1 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_mps_convolution>( + &self, + weight: &Tensor, + bias: Option, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__mps_convolution( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + padding.as_ptr(), + padding.len_i32(), + stride.as_ptr(), + stride.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_mps_convolution_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__mps_convolution_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + padding.as_ptr(), + padding.len_i32(), + stride.as_ptr(), + stride.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_mps_convolution_transpose( + &self, + weight: &Tensor, + padding: impl IntList, + output_padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__mps_convolution_transpose( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + padding.as_ptr(), + padding.len_i32(), + output_padding.as_ptr(), + output_padding.len_i32(), + stride.as_ptr(), + stride.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_mps_convolution_transpose_out( + &self, + out: &Tensor, + weight: &Tensor, + padding: impl IntList, + output_padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__mps_convolution_transpose_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + padding.as_ptr(), + padding.len_i32(), + output_padding.as_ptr(), + output_padding.len_i32(), + stride.as_ptr(), + stride.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_native_batch_norm_legit>( + &self, + weight: Option, + bias: Option, + running_mean: &Tensor, + running_var: &Tensor, + training: bool, + momentum: f64, + eps: f64, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__native_batch_norm_legit( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_mean.c_tensor, + running_var.c_tensor, + if training { 1 } else { 0 }, + momentum, + eps + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_native_batch_norm_legit_functional>( + &self, + weight: Option, + bias: Option, + running_mean: &Tensor, + running_var: &Tensor, + training: bool, + momentum: f64, + eps: f64, + ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 5]; + unsafe_torch_err!(atg__native_batch_norm_legit_functional( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_mean.c_tensor, + running_var.c_tensor, + if training { 1 } else { 0 }, + momentum, + eps + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + Tensor { c_tensor: c_tensors[4] }, + )) + } + + pub fn f_internal_native_batch_norm_legit_no_stats>( + &self, + weight: Option, + bias: Option, + training: bool, + momentum: f64, + eps: f64, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__native_batch_norm_legit_no_stats( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if training { 1 } else { 0 }, + momentum, + eps + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_native_batch_norm_legit_no_stats_out>( + &self, + out: &Tensor, + save_mean: &Tensor, + save_invstd: &Tensor, + weight: Option, + bias: Option, + training: bool, + momentum: f64, + eps: f64, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__native_batch_norm_legit_no_stats_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + save_mean.c_tensor, + save_invstd.c_tensor, + self.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if training { 1 } else { 0 }, + momentum, + eps + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_native_batch_norm_legit_no_training>( + &self, + weight: Option, + bias: Option, + running_mean: &Tensor, + running_var: &Tensor, + momentum: f64, + eps: f64, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__native_batch_norm_legit_no_training( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_mean.c_tensor, + running_var.c_tensor, + momentum, + eps + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_native_batch_norm_legit_no_training_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + weight: Option, + bias: Option, + running_mean: &Tensor, + running_var: &Tensor, + momentum: f64, + eps: f64, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__native_batch_norm_legit_no_training_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + out2.c_tensor, + self.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_mean.c_tensor, + running_var.c_tensor, + momentum, + eps + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_native_batch_norm_legit_out>( + &self, + out: &Tensor, + save_mean: &Tensor, + save_invstd: &Tensor, + weight: Option, + bias: Option, + running_mean: &Tensor, + running_var: &Tensor, + training: bool, + momentum: f64, + eps: f64, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__native_batch_norm_legit_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + save_mean.c_tensor, + save_invstd.c_tensor, + self.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_mean.c_tensor, + running_var.c_tensor, + if training { 1 } else { 0 }, + momentum, + eps + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_native_multi_head_attention>( + query: &Tensor, + key: &Tensor, + value: &Tensor, + embed_dim: i64, + num_head: i64, + qkv_weight: &Tensor, + qkv_bias: &Tensor, + proj_weight: &Tensor, + proj_bias: &Tensor, + mask: Option, + need_weights: bool, + average_attn_weights: bool, + mask_type: impl Into>, + ) -> Result<(Tensor, Tensor), TchError> { + let mask_type = mask_type.into(); + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__native_multi_head_attention( + c_tensors.as_mut_ptr(), + query.c_tensor, + key.c_tensor, + value.c_tensor, + embed_dim, + num_head, + qkv_weight.c_tensor, + qkv_bias.c_tensor, + proj_weight.c_tensor, + proj_bias.c_tensor, + mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if need_weights { 1 } else { 0 }, + if average_attn_weights { 1 } else { 0 }, + mask_type.unwrap_or(0i64), + mask_type.is_none() as i8 + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_native_multi_head_attention_out>( + out0: &Tensor, + out1: &Tensor, + query: &Tensor, + key: &Tensor, + value: &Tensor, + embed_dim: i64, + num_head: i64, + qkv_weight: &Tensor, + qkv_bias: &Tensor, + proj_weight: &Tensor, + proj_bias: &Tensor, + mask: Option, + need_weights: bool, + average_attn_weights: bool, + mask_type: impl Into>, + ) -> Result<(Tensor, Tensor), TchError> { + let mask_type = mask_type.into(); + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__native_multi_head_attention_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + query.c_tensor, + key.c_tensor, + value.c_tensor, + embed_dim, + num_head, + qkv_weight.c_tensor, + qkv_bias.c_tensor, + proj_weight.c_tensor, + proj_bias.c_tensor, + mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if need_weights { 1 } else { 0 }, + if average_attn_weights { 1 } else { 0 }, + mask_type.unwrap_or(0i64), + mask_type.is_none() as i8 + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_neg_view(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__neg_view(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_neg_view_copy(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__neg_view_copy(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_neg_view_copy_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__neg_view_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nested_compute_contiguous_strides_offsets( + nested_size: &Tensor, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__nested_compute_contiguous_strides_offsets( + c_tensors.as_mut_ptr(), + nested_size.c_tensor + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_nested_from_padded( + padded: &Tensor, + cpu_nested_shape_example: &Tensor, + fuse_transform_0213: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nested_from_padded( + c_tensors.as_mut_ptr(), + padded.c_tensor, + cpu_nested_shape_example.c_tensor, + if fuse_transform_0213 { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nested_from_padded_and_nested_example( + padded: &Tensor, + nt_example: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nested_from_padded_and_nested_example( + c_tensors.as_mut_ptr(), + padded.c_tensor, + nt_example.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nested_from_padded_and_nested_example_out( + out: &Tensor, + padded: &Tensor, + nt_example: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nested_from_padded_and_nested_example_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + padded.c_tensor, + nt_example.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nested_from_padded_out( + out: &Tensor, + padded: &Tensor, + cpu_nested_shape_example: &Tensor, + fuse_transform_0213: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nested_from_padded_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + padded.c_tensor, + cpu_nested_shape_example.c_tensor, + if fuse_transform_0213 { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nested_from_padded_tensor>( + padded: &Tensor, + offsets: &Tensor, + dummy: &Tensor, + ragged_idx: i64, + min_seqlen: Option, + max_seqlen: Option, + sum_s: impl Into>, + ) -> Result { + let sum_s = sum_s.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nested_from_padded_tensor( + c_tensors.as_mut_ptr(), + padded.c_tensor, + offsets.c_tensor, + dummy.c_tensor, + ragged_idx, + min_seqlen.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + max_seqlen.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + sum_s.unwrap_or(0i64), + sum_s.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nested_get_jagged_dummy(any: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nested_get_jagged_dummy(c_tensors.as_mut_ptr(), any.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nested_get_lengths(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nested_get_lengths(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nested_get_max_seqlen(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nested_get_max_seqlen(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nested_get_min_seqlen(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nested_get_min_seqlen(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nested_get_offsets(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nested_get_offsets(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nested_get_ragged_idx(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg__nested_get_ragged_idx(self.c_tensor)); + Ok(return_) + } + + pub fn f_internal_nested_get_values(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nested_get_values(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nested_get_values_copy(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nested_get_values_copy(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nested_get_values_copy_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nested_get_values_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nested_select_backward( + &self, + grad_output: &Tensor, + dim: i64, + index: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nested_select_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + dim, + index + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nested_sum_backward( + &self, + grad: &Tensor, + dim: impl IntListOption, + keepdim: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nested_sum_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nested_view_from_buffer( + &self, + nested_size: &Tensor, + nested_strides: &Tensor, + offsets: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nested_view_from_buffer( + c_tensors.as_mut_ptr(), + self.c_tensor, + nested_size.c_tensor, + nested_strides.c_tensor, + offsets.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nested_view_from_buffer_copy( + &self, + nested_size: &Tensor, + nested_strides: &Tensor, + offsets: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nested_view_from_buffer_copy( + c_tensors.as_mut_ptr(), + self.c_tensor, + nested_size.c_tensor, + nested_strides.c_tensor, + offsets.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nested_view_from_buffer_copy_out( + &self, + out: &Tensor, + nested_size: &Tensor, + nested_strides: &Tensor, + offsets: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nested_view_from_buffer_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + nested_size.c_tensor, + nested_strides.c_tensor, + offsets.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nested_view_from_jagged>( + &self, + offsets: &Tensor, + dummy: &Tensor, + lengths: Option, + ragged_idx: i64, + min_seqlen: Option, + max_seqlen: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nested_view_from_jagged( + c_tensors.as_mut_ptr(), + self.c_tensor, + offsets.c_tensor, + dummy.c_tensor, + lengths.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + ragged_idx, + min_seqlen.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + max_seqlen.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nested_view_from_jagged_copy>( + &self, + offsets: &Tensor, + dummy: &Tensor, + lengths: Option, + ragged_idx: i64, + min_seqlen: Option, + max_seqlen: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nested_view_from_jagged_copy( + c_tensors.as_mut_ptr(), + self.c_tensor, + offsets.c_tensor, + dummy.c_tensor, + lengths.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + ragged_idx, + min_seqlen.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + max_seqlen.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nested_view_from_jagged_copy_out>( + &self, + out: &Tensor, + offsets: &Tensor, + dummy: &Tensor, + lengths: Option, + ragged_idx: i64, + min_seqlen: Option, + max_seqlen: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nested_view_from_jagged_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + offsets.c_tensor, + dummy.c_tensor, + lengths.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + ragged_idx, + min_seqlen.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + max_seqlen.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_new_zeros_with_same_feature_meta( + &self, + other: &Tensor, + self_num_batch_dims: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__new_zeros_with_same_feature_meta( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor, + self_num_batch_dims + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_new_zeros_with_same_feature_meta_out( + &self, + out: &Tensor, + other: &Tensor, + self_num_batch_dims: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__new_zeros_with_same_feature_meta_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor, + self_num_batch_dims + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nnpack_available() -> Result { + let return_; + unsafe_torch_err!(return_ = atg__nnpack_available()); + Ok(return_ != 0) + } + + pub fn f_internal_nnpack_spatial_convolution>( + &self, + weight: &Tensor, + bias: Option, + padding: impl IntList, + stride: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nnpack_spatial_convolution( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + padding.as_ptr(), + padding.len_i32(), + stride.as_ptr(), + stride.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nnpack_spatial_convolution_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + padding: impl IntList, + stride: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__nnpack_spatial_convolution_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + padding.as_ptr(), + padding.len_i32(), + stride.as_ptr(), + stride.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_nnz(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg__nnz(self.c_tensor)); + Ok(return_) + } + + pub fn f_internal_pack_padded_sequence( + &self, + lengths: &Tensor, + batch_first: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__pack_padded_sequence( + c_tensors.as_mut_ptr(), + self.c_tensor, + lengths.c_tensor, + if batch_first { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_pack_padded_sequence_backward( + grad: &Tensor, + input_size: impl IntList, + batch_sizes: &Tensor, + batch_first: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__pack_padded_sequence_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + input_size.as_ptr(), + input_size.len_i32(), + batch_sizes.c_tensor, + if batch_first { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_pack_padded_sequence_out( + &self, + out0: &Tensor, + out1: &Tensor, + lengths: &Tensor, + batch_first: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__pack_padded_sequence_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + self.c_tensor, + lengths.c_tensor, + if batch_first { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_pad_circular(&self, pad: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__pad_circular( + c_tensors.as_mut_ptr(), + self.c_tensor, + pad.as_ptr(), + pad.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_pad_enum( + &self, + pad: impl IntList, + mode: i64, + value: impl Into>, + ) -> Result { + let value = value.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__pad_enum( + c_tensors.as_mut_ptr(), + self.c_tensor, + pad.as_ptr(), + pad.len_i32(), + mode, + value.unwrap_or(std::f64::NAN), + value.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_pad_packed_sequence>( + data: &Tensor, + batch_sizes: &Tensor, + batch_first: bool, + padding_value: S, + total_length: i64, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__pad_packed_sequence( + c_tensors.as_mut_ptr(), + data.c_tensor, + batch_sizes.c_tensor, + if batch_first { 1 } else { 0 }, + padding_value.into().c_scalar, + total_length + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_pdist_backward( + &self, + grad: &Tensor, + p: f64, + pdist: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__pdist_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + self.c_tensor, + p, + pdist.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_pdist_backward_out( + &self, + out: &Tensor, + grad: &Tensor, + p: f64, + pdist: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__pdist_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad.c_tensor, + self.c_tensor, + p, + pdist.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_pin_memory(&self, device: Device) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__pin_memory(c_tensors.as_mut_ptr(), self.c_tensor, device.c_int())); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_pin_memory_out( + &self, + out: &Tensor, + device: Device, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__pin_memory_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + device.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_prelu_kernel(&self, weight: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__prelu_kernel( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_prelu_kernel_backward( + &self, + grad_output: &Tensor, + weight: &Tensor, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__prelu_kernel_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + weight.c_tensor + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_print(s: &str) -> Result<(), TchError> { + unsafe_torch_err!(atg__print(s.as_ptr(), s.len() as i32)); + Ok(()) + } + + pub fn f_internal_propagate_xla_data(&self, output: &Tensor) -> Result<(), TchError> { + unsafe_torch_err!(atg__propagate_xla_data(self.c_tensor, output.c_tensor)); + Ok(()) + } + + pub fn f_internal_remove_batch_dim( + &self, + level: i64, + batch_size: i64, + out_dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__remove_batch_dim( + c_tensors.as_mut_ptr(), + self.c_tensor, + level, + batch_size, + out_dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_reshape_alias( + &self, + size: impl IntList, + stride: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__reshape_alias( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32(), + stride.as_ptr(), + stride.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_reshape_alias_copy( + &self, + size: impl IntList, + stride: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__reshape_alias_copy( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32(), + stride.as_ptr(), + stride.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_reshape_alias_copy_out( + &self, + out: &Tensor, + size: impl IntList, + stride: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__reshape_alias_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + size.as_ptr(), + size.len_i32(), + stride.as_ptr(), + stride.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_reshape_copy(&self, size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__reshape_copy( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_reshape_from_tensor(&self, shape: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__reshape_from_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + shape.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_resize_output( + &self, + size: impl IntList, + device: Device, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__resize_output( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32(), + device.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_resize_output_( + &mut self, + size: impl IntList, + device: Device, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__resize_output_( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32(), + device.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_resize_output_out( + &self, + out: &Tensor, + size: impl IntList, + device: Device, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__resize_output_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + size.as_ptr(), + size.len_i32(), + device.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_rowwise_prune( + weight: &Tensor, + mask: &Tensor, + compressed_indices_dtype: Kind, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__rowwise_prune( + c_tensors.as_mut_ptr(), + weight.c_tensor, + mask.c_tensor, + compressed_indices_dtype.c_int() + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_safe_softmax( + &self, + dim: i64, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__safe_softmax( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sample_dirichlet(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sample_dirichlet(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sample_dirichlet_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sample_dirichlet_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_saturate_weight_to_fp16(weight: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__saturate_weight_to_fp16(c_tensors.as_mut_ptr(), weight.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_scaled_dot_product_attention_math>( + query: &Tensor, + key: &Tensor, + value: &Tensor, + attn_mask: Option, + dropout_p: f64, + is_causal: bool, + dropout_mask: Option, + scale: impl Into>, + enable_gqa: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let scale = scale.into(); + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__scaled_dot_product_attention_math( + c_tensors.as_mut_ptr(), + query.c_tensor, + key.c_tensor, + value.c_tensor, + attn_mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + dropout_p, + if is_causal { 1 } else { 0 }, + dropout_mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + scale.unwrap_or(std::f64::NAN), + scale.is_none() as i8, + if enable_gqa { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_scaled_dot_product_attention_math_for_mps>( + query: &Tensor, + key: &Tensor, + value: &Tensor, + attn_mask: Option, + dropout_p: f64, + is_causal: bool, + dropout_mask: Option, + scale: impl Into>, + ) -> Result<(Tensor, Tensor), TchError> { + let scale = scale.into(); + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__scaled_dot_product_attention_math_for_mps( + c_tensors.as_mut_ptr(), + query.c_tensor, + key.c_tensor, + value.c_tensor, + attn_mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + dropout_p, + if is_causal { 1 } else { 0 }, + dropout_mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + scale.unwrap_or(std::f64::NAN), + scale.is_none() as i8 + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_scaled_dot_product_cudnn_attention_backward( + grad_out: &Tensor, + query: &Tensor, + key: &Tensor, + value: &Tensor, + out: &Tensor, + logsumexp: &Tensor, + philox_seed: &Tensor, + philox_offset: &Tensor, + attn_bias: &Tensor, + cum_seq_q: &Tensor, + cum_seq_k: &Tensor, + max_q: i64, + max_k: i64, + dropout_p: f64, + is_causal: bool, + scale: impl Into>, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let scale = scale.into(); + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__scaled_dot_product_cudnn_attention_backward( + c_tensors.as_mut_ptr(), + grad_out.c_tensor, + query.c_tensor, + key.c_tensor, + value.c_tensor, + out.c_tensor, + logsumexp.c_tensor, + philox_seed.c_tensor, + philox_offset.c_tensor, + attn_bias.c_tensor, + cum_seq_q.c_tensor, + cum_seq_k.c_tensor, + max_q, + max_k, + dropout_p, + if is_causal { 1 } else { 0 }, + scale.unwrap_or(std::f64::NAN), + scale.is_none() as i8 + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_scaled_dot_product_efficient_attention>( + query: &Tensor, + key: &Tensor, + value: &Tensor, + attn_bias: Option, + compute_log_sumexp: bool, + dropout_p: f64, + is_causal: bool, + scale: impl Into>, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let scale = scale.into(); + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg__scaled_dot_product_efficient_attention( + c_tensors.as_mut_ptr(), + query.c_tensor, + key.c_tensor, + value.c_tensor, + attn_bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if compute_log_sumexp { 1 } else { 0 }, + dropout_p, + if is_causal { 1 } else { 0 }, + scale.unwrap_or(std::f64::NAN), + scale.is_none() as i8 + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_internal_scaled_dot_product_flash_attention_backward( + grad_out: &Tensor, + query: &Tensor, + key: &Tensor, + value: &Tensor, + out: &Tensor, + logsumexp: &Tensor, + cum_seq_q: &Tensor, + cum_seq_k: &Tensor, + max_q: i64, + max_k: i64, + dropout_p: f64, + is_causal: bool, + philox_seed: &Tensor, + philox_offset: &Tensor, + scale: impl Into>, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let scale = scale.into(); + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__scaled_dot_product_flash_attention_backward( + c_tensors.as_mut_ptr(), + grad_out.c_tensor, + query.c_tensor, + key.c_tensor, + value.c_tensor, + out.c_tensor, + logsumexp.c_tensor, + cum_seq_q.c_tensor, + cum_seq_k.c_tensor, + max_q, + max_k, + dropout_p, + if is_causal { 1 } else { 0 }, + philox_seed.c_tensor, + philox_offset.c_tensor, + scale.unwrap_or(std::f64::NAN), + scale.is_none() as i8 + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_scaled_dot_product_flash_attention_for_cpu>( + query: &Tensor, + key: &Tensor, + value: &Tensor, + dropout_p: f64, + is_causal: bool, + attn_mask: Option, + scale: impl Into>, + ) -> Result<(Tensor, Tensor), TchError> { + let scale = scale.into(); + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__scaled_dot_product_flash_attention_for_cpu( + c_tensors.as_mut_ptr(), + query.c_tensor, + key.c_tensor, + value.c_tensor, + dropout_p, + if is_causal { 1 } else { 0 }, + attn_mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + scale.unwrap_or(std::f64::NAN), + scale.is_none() as i8 + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_scaled_dot_product_flash_attention_for_cpu_backward>( + grad_out: &Tensor, + query: &Tensor, + key: &Tensor, + value: &Tensor, + out: &Tensor, + logsumexp: &Tensor, + dropout_p: f64, + is_causal: bool, + attn_mask: Option, + scale: impl Into>, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let scale = scale.into(); + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__scaled_dot_product_flash_attention_for_cpu_backward( + c_tensors.as_mut_ptr(), + grad_out.c_tensor, + query.c_tensor, + key.c_tensor, + value.c_tensor, + out.c_tensor, + logsumexp.c_tensor, + dropout_p, + if is_causal { 1 } else { 0 }, + attn_mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + scale.unwrap_or(std::f64::NAN), + scale.is_none() as i8 + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_scaled_grouped_mm>( + &self, + mat2: &Tensor, + scale_a: &Tensor, + scale_b: &Tensor, + offs: Option, + bias: Option, + scale_result: Option, + out_dtype: impl Into>, + use_fast_accum: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__scaled_grouped_mm( + c_tensors.as_mut_ptr(), + self.c_tensor, + mat2.c_tensor, + scale_a.c_tensor, + scale_b.c_tensor, + offs.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + scale_result.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + out_dtype.into().map_or(-1, |s| s.c_int()), + if use_fast_accum { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_scaled_grouped_mm_v2>( + &self, + mat2: &Tensor, + scale_a: &[T], + recipe_a: impl IntList, + swizzle_a: impl IntList, + scale_b: &[T], + recipe_b: impl IntList, + swizzle_b: impl IntList, + offs: Option, + bias: Option, + out_dtype: impl Into>, + contraction_dim: impl IntList, + use_fast_accum: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__scaled_grouped_mm_v2( + c_tensors.as_mut_ptr(), + self.c_tensor, + mat2.c_tensor, + ptr_list(scale_a).as_ptr(), + scale_a.len() as i32, + recipe_a.as_ptr(), + recipe_a.len_i32(), + swizzle_a.as_ptr(), + swizzle_a.len_i32(), + ptr_list(scale_b).as_ptr(), + scale_b.len() as i32, + recipe_b.as_ptr(), + recipe_b.len_i32(), + swizzle_b.as_ptr(), + swizzle_b.len_i32(), + offs.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + out_dtype.into().map_or(-1, |s| s.c_int()), + contraction_dim.as_ptr(), + contraction_dim.len_i32(), + if use_fast_accum { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_scaled_mm>( + &self, + mat2: &Tensor, + scale_a: &Tensor, + scale_b: &Tensor, + bias: Option, + scale_result: Option, + out_dtype: impl Into>, + use_fast_accum: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__scaled_mm( + c_tensors.as_mut_ptr(), + self.c_tensor, + mat2.c_tensor, + scale_a.c_tensor, + scale_b.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + scale_result.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + out_dtype.into().map_or(-1, |s| s.c_int()), + if use_fast_accum { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_scaled_mm_out>( + &self, + out: &Tensor, + mat2: &Tensor, + scale_a: &Tensor, + scale_b: &Tensor, + bias: Option, + scale_result: Option, + out_dtype: impl Into>, + use_fast_accum: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__scaled_mm_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mat2.c_tensor, + scale_a.c_tensor, + scale_b.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + scale_result.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + out_dtype.into().map_or(-1, |s| s.c_int()), + if use_fast_accum { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_scaled_mm_v2>( + &self, + mat2: &Tensor, + scale_a: &[T], + recipe_a: impl IntList, + swizzle_a: impl IntList, + scale_b: &[T], + recipe_b: impl IntList, + swizzle_b: impl IntList, + bias: Option, + out_dtype: impl Into>, + contraction_dim: impl IntList, + use_fast_accum: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__scaled_mm_v2( + c_tensors.as_mut_ptr(), + self.c_tensor, + mat2.c_tensor, + ptr_list(scale_a).as_ptr(), + scale_a.len() as i32, + recipe_a.as_ptr(), + recipe_a.len_i32(), + swizzle_a.as_ptr(), + swizzle_a.len_i32(), + ptr_list(scale_b).as_ptr(), + scale_b.len() as i32, + recipe_b.as_ptr(), + recipe_b.len_i32(), + swizzle_b.as_ptr(), + swizzle_b.len_i32(), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + out_dtype.into().map_or(-1, |s| s.c_int()), + contraction_dim.as_ptr(), + contraction_dim.len_i32(), + if use_fast_accum { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_scaled_mm_v2_out>( + &self, + out: &Tensor, + mat2: &Tensor, + scale_a: &[T], + recipe_a: impl IntList, + swizzle_a: impl IntList, + scale_b: &[T], + recipe_b: impl IntList, + swizzle_b: impl IntList, + bias: Option, + out_dtype: impl Into>, + contraction_dim: impl IntList, + use_fast_accum: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__scaled_mm_v2_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mat2.c_tensor, + ptr_list(scale_a).as_ptr(), + scale_a.len() as i32, + recipe_a.as_ptr(), + recipe_a.len_i32(), + swizzle_a.as_ptr(), + swizzle_a.len_i32(), + ptr_list(scale_b).as_ptr(), + scale_b.len() as i32, + recipe_b.as_ptr(), + recipe_b.len_i32(), + swizzle_b.as_ptr(), + swizzle_b.len_i32(), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + out_dtype.into().map_or(-1, |s| s.c_int()), + contraction_dim.as_ptr(), + contraction_dim.len_i32(), + if use_fast_accum { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_scatter_reduce( + &self, + dim: i64, + index: &Tensor, + src: &Tensor, + reduce: &str, + include_self: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__scatter_reduce( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + src.c_tensor, + reduce.as_ptr(), + reduce.len() as i32, + if include_self { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_scatter_reduce_( + &mut self, + dim: i64, + index: &Tensor, + src: &Tensor, + reduce: &str, + include_self: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__scatter_reduce_( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + src.c_tensor, + reduce.as_ptr(), + reduce.len() as i32, + if include_self { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_scatter_reduce_two_out( + &self, + out: &Tensor, + dim: i64, + index: &Tensor, + src: &Tensor, + reduce: &str, + include_self: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__scatter_reduce_two_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + index.c_tensor, + src.c_tensor, + reduce.as_ptr(), + reduce.len() as i32, + if include_self { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_segment_reduce_backward, S: Into>( + grad: &Tensor, + output: &Tensor, + data: &Tensor, + reduce: &str, + lengths: Option, + offsets: Option, + axis: i64, + initial: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__segment_reduce_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + output.c_tensor, + data.c_tensor, + reduce.as_ptr(), + reduce.len() as i32, + lengths.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + offsets.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + axis, + initial.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_segment_reduce_backward_out, S: Into>( + out: &Tensor, + grad: &Tensor, + output: &Tensor, + data: &Tensor, + reduce: &str, + lengths: Option, + offsets: Option, + axis: i64, + initial: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__segment_reduce_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad.c_tensor, + output.c_tensor, + data.c_tensor, + reduce.as_ptr(), + reduce.len() as i32, + lengths.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + offsets.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + axis, + initial.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_shape_as_tensor(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__shape_as_tensor(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_slow_conv2d_backward( + &self, + grad_input: &Tensor, + grad_weight: &Tensor, + grad_bias: &Tensor, + grad_output: &Tensor, + weight: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__slow_conv2d_backward( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_weight.c_tensor, + grad_bias.c_tensor, + grad_output.c_tensor, + self.c_tensor, + weight.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32() + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_sobol_engine_draw( + quasi: &Tensor, + n: i64, + sobolstate: &Tensor, + dimension: i64, + num_generated: i64, + dtype: impl Into>, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__sobol_engine_draw( + c_tensors.as_mut_ptr(), + quasi.c_tensor, + n, + sobolstate.c_tensor, + dimension, + num_generated, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_sobol_engine_ff_( + &mut self, + n: i64, + sobolstate: &Tensor, + dimension: i64, + num_generated: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sobol_engine_ff_( + c_tensors.as_mut_ptr(), + self.c_tensor, + n, + sobolstate.c_tensor, + dimension, + num_generated + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sobol_engine_initialize_state_( + &mut self, + dimension: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sobol_engine_initialize_state_( + c_tensors.as_mut_ptr(), + self.c_tensor, + dimension + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sobol_engine_scramble_( + &mut self, + ltm: &Tensor, + dimension: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sobol_engine_scramble_( + c_tensors.as_mut_ptr(), + self.c_tensor, + ltm.c_tensor, + dimension + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_softmax(&self, dim: i64, half_to_float: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__softmax( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + if half_to_float { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_softmax_backward_data( + grad_output: &Tensor, + output: &Tensor, + dim: i64, + input_dtype: Kind, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__softmax_backward_data( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + output.c_tensor, + dim, + input_dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_softmax_backward_data_out( + grad_input: &Tensor, + grad_output: &Tensor, + output: &Tensor, + dim: i64, + input_dtype: Kind, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__softmax_backward_data_out( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + output.c_tensor, + dim, + input_dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_softmax_out( + &self, + out: &Tensor, + dim: i64, + half_to_float: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__softmax_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + if half_to_float { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_addmm( + &self, + mat1: &Tensor, + mat2: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_addmm( + c_tensors.as_mut_ptr(), + self.c_tensor, + mat1.c_tensor, + mat2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_addmm_out( + &self, + out: &Tensor, + mat1: &Tensor, + mat2: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_addmm_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mat1.c_tensor, + mat2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_broadcast_to(&self, size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_broadcast_to( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_broadcast_to_copy( + &self, + size: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_broadcast_to_copy( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_broadcast_to_copy_out( + &self, + out: &Tensor, + size: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_broadcast_to_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_bsc_tensor_unsafe( + ccol_indices: &Tensor, + row_indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_bsc_tensor_unsafe( + c_tensors.as_mut_ptr(), + ccol_indices.c_tensor, + row_indices.c_tensor, + values.c_tensor, + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_bsr_tensor_unsafe( + crow_indices: &Tensor, + col_indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_bsr_tensor_unsafe( + c_tensors.as_mut_ptr(), + crow_indices.c_tensor, + col_indices.c_tensor, + values.c_tensor, + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_compressed_tensor_unsafe( + compressed_indices: &Tensor, + plain_indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_compressed_tensor_unsafe( + c_tensors.as_mut_ptr(), + compressed_indices.c_tensor, + plain_indices.c_tensor, + values.c_tensor, + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_compressed_tensor_with_dims( + nnz: i64, + dense_dim: i64, + size: impl IntList, + blocksize: impl IntList, + index_dtype: Kind, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_compressed_tensor_with_dims( + c_tensors.as_mut_ptr(), + nnz, + dense_dim, + size.as_ptr(), + size.len_i32(), + blocksize.as_ptr(), + blocksize.len_i32(), + index_dtype.c_int(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_coo_tensor_unsafe( + indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + is_coalesced: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_coo_tensor_unsafe( + c_tensors.as_mut_ptr(), + indices.c_tensor, + values.c_tensor, + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int(), + if is_coalesced { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_coo_tensor_with_dims( + sparse_dim: i64, + dense_dim: i64, + size: impl IntList, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_coo_tensor_with_dims( + c_tensors.as_mut_ptr(), + sparse_dim, + dense_dim, + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_coo_tensor_with_dims_and_tensors( + sparse_dim: i64, + dense_dim: i64, + size: impl IntList, + indices: &Tensor, + values: &Tensor, + options: (Kind, Device), + is_coalesced: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_coo_tensor_with_dims_and_tensors( + c_tensors.as_mut_ptr(), + sparse_dim, + dense_dim, + size.as_ptr(), + size.len_i32(), + indices.c_tensor, + values.c_tensor, + options.0.c_int(), + options.1.c_int(), + if is_coalesced { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_coo_tensor_with_dims_and_tensors_out( + out: &Tensor, + sparse_dim: i64, + dense_dim: i64, + size: impl IntList, + indices: &Tensor, + values: &Tensor, + is_coalesced: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_coo_tensor_with_dims_and_tensors_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + sparse_dim, + dense_dim, + size.as_ptr(), + size.len_i32(), + indices.c_tensor, + values.c_tensor, + if is_coalesced { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_coo_tensor_with_dims_out( + out: &Tensor, + sparse_dim: i64, + dense_dim: i64, + size: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_coo_tensor_with_dims_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + sparse_dim, + dense_dim, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_csc_tensor_unsafe( + ccol_indices: &Tensor, + row_indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_csc_tensor_unsafe( + c_tensors.as_mut_ptr(), + ccol_indices.c_tensor, + row_indices.c_tensor, + values.c_tensor, + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_csr_prod( + &self, + dim: impl IntList, + keepdim: bool, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_csr_prod( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 }, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_csr_prod_dim_dtype_out( + &self, + out: &Tensor, + dim: impl IntList, + keepdim: bool, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_csr_prod_dim_dtype_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 }, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_csr_sum( + &self, + dim: impl IntList, + keepdim: bool, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_csr_sum( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 }, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_csr_sum_dim_dtype_out( + &self, + out: &Tensor, + dim: impl IntList, + keepdim: bool, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_csr_sum_dim_dtype_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 }, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_csr_tensor_unsafe( + crow_indices: &Tensor, + col_indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_csr_tensor_unsafe( + c_tensors.as_mut_ptr(), + crow_indices.c_tensor, + col_indices.c_tensor, + values.c_tensor, + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_log_softmax( + &self, + dim: i64, + half_to_float: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_log_softmax( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + if half_to_float { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_log_softmax_backward_data( + &self, + grad_output: &Tensor, + output: &Tensor, + dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_log_softmax_backward_data( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + output.c_tensor, + dim, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_log_softmax_backward_data_out( + &self, + out: &Tensor, + grad_output: &Tensor, + output: &Tensor, + dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_log_softmax_backward_data_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad_output.c_tensor, + output.c_tensor, + dim, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_log_softmax_int( + &self, + dim: i64, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_log_softmax_int( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_log_softmax_out( + &self, + out: &Tensor, + dim: i64, + half_to_float: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_log_softmax_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + if half_to_float { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_mask_projection( + &self, + mask: &Tensor, + accumulate_matches: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_mask_projection( + c_tensors.as_mut_ptr(), + self.c_tensor, + mask.c_tensor, + if accumulate_matches { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_mask_projection_out( + &self, + out: &Tensor, + mask: &Tensor, + accumulate_matches: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_mask_projection_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mask.c_tensor, + if accumulate_matches { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_mm(sparse: &Tensor, dense: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_mm(c_tensors.as_mut_ptr(), sparse.c_tensor, dense.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_mm_reduce( + sparse: &Tensor, + dense: &Tensor, + reduce: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_mm_reduce( + c_tensors.as_mut_ptr(), + sparse.c_tensor, + dense.c_tensor, + reduce.as_ptr(), + reduce.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_mm_reduce_impl( + &self, + other: &Tensor, + reduce: &str, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__sparse_mm_reduce_impl( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor, + reduce.as_ptr(), + reduce.len() as i32 + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_sparse_semi_structured_apply( + &self, + thread_masks: &Tensor, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__sparse_semi_structured_apply( + c_tensors.as_mut_ptr(), + self.c_tensor, + thread_masks.c_tensor + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_sparse_semi_structured_apply_dense( + &self, + thread_masks: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_semi_structured_apply_dense( + c_tensors.as_mut_ptr(), + self.c_tensor, + thread_masks.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_semi_structured_linear>( + &self, + weight: &Tensor, + meta: &Tensor, + bias: Option, + activation: &str, + out_dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_semi_structured_linear( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + meta.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + activation.as_ptr(), + activation.len() as i32, + out_dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_semi_structured_mm( + mat1: &Tensor, + mat1_meta: &Tensor, + mat2: &Tensor, + out_dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_semi_structured_mm( + c_tensors.as_mut_ptr(), + mat1.c_tensor, + mat1_meta.c_tensor, + mat2.c_tensor, + out_dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_semi_structured_tile( + &self, + algorithm: &str, + use_cutlass: bool, + ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 5]; + unsafe_torch_err!(atg__sparse_semi_structured_tile( + c_tensors.as_mut_ptr(), + self.c_tensor, + algorithm.as_ptr(), + algorithm.len() as i32, + if use_cutlass { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + Tensor { c_tensor: c_tensors[4] }, + )) + } + + pub fn f_internal_sparse_softmax( + &self, + dim: i64, + half_to_float: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_softmax( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + if half_to_float { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_softmax_backward_data( + &self, + grad_output: &Tensor, + output: &Tensor, + dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_softmax_backward_data( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + output.c_tensor, + dim, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_softmax_backward_data_out( + &self, + out: &Tensor, + grad_output: &Tensor, + output: &Tensor, + dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_softmax_backward_data_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad_output.c_tensor, + output.c_tensor, + dim, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_softmax_int( + &self, + dim: i64, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_softmax_int( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_softmax_out( + &self, + out: &Tensor, + dim: i64, + half_to_float: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_softmax_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + if half_to_float { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_sparse_matmul(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_sparse_matmul( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_sparse_matmul_out( + &self, + out: &Tensor, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_sparse_matmul_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_sum(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_sum(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_sum_backward( + &self, + grad: &Tensor, + dim: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_sum_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_sum_backward_out( + &self, + out: &Tensor, + grad: &Tensor, + dim: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_sum_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_sum_dim(&self, dim: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_sum_dim( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_sum_dim_dtype( + &self, + dim: impl IntList, + dtype: Kind, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_sum_dim_dtype( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_sum_dim_out( + &self, + out: &Tensor, + dim: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_sum_dim_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_sparse_sum_dtype(&self, dtype: Kind) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__sparse_sum_dtype( + c_tensors.as_mut_ptr(), + self.c_tensor, + dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_spdiags( + diagonals: &Tensor, + offsets: &Tensor, + shape: impl IntList, + layout: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__spdiags( + c_tensors.as_mut_ptr(), + diagonals.c_tensor, + offsets.c_tensor, + shape.as_ptr(), + shape.len_i32(), + layout.map_or(-1, |s| s.to_i8()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_spdiags_out( + out: &Tensor, + diagonals: &Tensor, + offsets: &Tensor, + shape: impl IntList, + layout: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__spdiags_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + diagonals.c_tensor, + offsets.c_tensor, + shape.as_ptr(), + shape.len_i32(), + layout.map_or(-1, |s| s.to_i8()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_spsolve(a: &Tensor, b: &Tensor, left: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__spsolve( + c_tensors.as_mut_ptr(), + a.c_tensor, + b.c_tensor, + if left { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_stack>( + tensors: &[T], + dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__stack( + c_tensors.as_mut_ptr(), + ptr_list(tensors).as_ptr(), + tensors.len() as i32, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_stack_out>( + out: &Tensor, + tensors: &[T], + dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__stack_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + ptr_list(tensors).as_ptr(), + tensors.len() as i32, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_standard_gamma(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__standard_gamma(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_standard_gamma_grad(&self, output: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__standard_gamma_grad( + c_tensors.as_mut_ptr(), + self.c_tensor, + output.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_standard_gamma_grad_out( + &self, + out: &Tensor, + output: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__standard_gamma_grad_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + output.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_standard_gamma_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__standard_gamma_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_test_ambiguous_defaults( + dummy: &Tensor, + a: i64, + b: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__test_ambiguous_defaults( + c_tensors.as_mut_ptr(), + dummy.c_tensor, + a, + b + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_test_ambiguous_defaults_b( + dummy: &Tensor, + a: i64, + b: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__test_ambiguous_defaults_b( + c_tensors.as_mut_ptr(), + dummy.c_tensor, + a, + b.as_ptr(), + b.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_test_autograd_multiple_dispatch(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__test_autograd_multiple_dispatch( + c_tensors.as_mut_ptr(), + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_test_autograd_multiple_dispatch_fullcoverage_out( + &self, + out: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__test_autograd_multiple_dispatch_fullcoverage_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_test_autograd_multiple_dispatch_ntonly( + &self, + b: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__test_autograd_multiple_dispatch_ntonly( + c_tensors.as_mut_ptr(), + self.c_tensor, + if b { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_test_autograd_multiple_dispatch_view(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__test_autograd_multiple_dispatch_view( + c_tensors.as_mut_ptr(), + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_test_autograd_multiple_dispatch_view_copy(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__test_autograd_multiple_dispatch_view_copy( + c_tensors.as_mut_ptr(), + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_test_autograd_multiple_dispatch_view_copy_out( + &self, + out: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__test_autograd_multiple_dispatch_view_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_test_check_tensor(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__test_check_tensor(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_test_functorch_fallback(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__test_functorch_fallback( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_test_functorch_fallback_out( + &self, + out: &Tensor, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__test_functorch_fallback_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_test_optional_filled_intlist( + values: &Tensor, + addends: impl IntListOption, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__test_optional_filled_intlist( + c_tensors.as_mut_ptr(), + values.c_tensor, + addends.as_ptr(), + addends.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_test_optional_filled_intlist_out( + out: &Tensor, + values: &Tensor, + addends: impl IntListOption, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__test_optional_filled_intlist_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + values.c_tensor, + addends.as_ptr(), + addends.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_test_optional_floatlist( + values: &Tensor, + addends: impl DoubleList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__test_optional_floatlist( + c_tensors.as_mut_ptr(), + values.c_tensor, + addends.as_ptr(), + addends.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_test_optional_floatlist_out( + out: &Tensor, + values: &Tensor, + addends: impl DoubleList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__test_optional_floatlist_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + values.c_tensor, + addends.as_ptr(), + addends.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_test_optional_intlist( + values: &Tensor, + addends: impl IntListOption, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__test_optional_intlist( + c_tensors.as_mut_ptr(), + values.c_tensor, + addends.as_ptr(), + addends.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_test_optional_intlist_out( + out: &Tensor, + values: &Tensor, + addends: impl IntListOption, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__test_optional_intlist_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + values.c_tensor, + addends.as_ptr(), + addends.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_test_parallel_materialize( + &self, + num_parallel: i64, + skip_first: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__test_parallel_materialize( + c_tensors.as_mut_ptr(), + self.c_tensor, + num_parallel, + if skip_first { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_test_serialization_subcmul( + &self, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__test_serialization_subcmul( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_test_string_default( + dummy: &Tensor, + a: &str, + b: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__test_string_default( + c_tensors.as_mut_ptr(), + dummy.c_tensor, + a.as_ptr(), + a.len() as i32, + b.as_ptr(), + b.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_test_warn_in_autograd(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__test_warn_in_autograd(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_test_warn_in_autograd_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__test_warn_in_autograd_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_to_copy( + &self, + options: (Kind, Device), + non_blocking: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__to_copy( + c_tensors.as_mut_ptr(), + self.c_tensor, + options.0.c_int(), + options.1.c_int(), + if non_blocking { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_to_copy_out( + &self, + out: &Tensor, + non_blocking: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__to_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + if non_blocking { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_to_cpu>(tensors: &[T]) -> Result, TchError> { + let c_tensors = + unsafe_torch_err!(atg__to_cpu(ptr_list(tensors).as_ptr(), tensors.len() as i32)); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_internal_to_dense( + &self, + dtype: impl Into>, + masked_grad: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__to_dense( + c_tensors.as_mut_ptr(), + self.c_tensor, + dtype.into().map_or(-1, |s| s.c_int()), + if masked_grad { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_to_dense_out( + &self, + out: &Tensor, + dtype: impl Into>, + masked_grad: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__to_dense_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dtype.into().map_or(-1, |s| s.c_int()), + if masked_grad { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_to_sparse( + &self, + layout: Option, + blocksize: impl IntListOption, + dense_dim: impl Into>, + ) -> Result { + let dense_dim = dense_dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__to_sparse( + c_tensors.as_mut_ptr(), + self.c_tensor, + layout.map_or(-1, |s| s.to_i8()), + blocksize.as_ptr(), + blocksize.len_i32(), + dense_dim.unwrap_or(0i64), + dense_dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_to_sparse_bsc( + &self, + blocksize: impl IntList, + dense_dim: impl Into>, + ) -> Result { + let dense_dim = dense_dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__to_sparse_bsc( + c_tensors.as_mut_ptr(), + self.c_tensor, + blocksize.as_ptr(), + blocksize.len_i32(), + dense_dim.unwrap_or(0i64), + dense_dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_to_sparse_bsc_out( + &self, + out: &Tensor, + blocksize: impl IntList, + dense_dim: impl Into>, + ) -> Result { + let dense_dim = dense_dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__to_sparse_bsc_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + blocksize.as_ptr(), + blocksize.len_i32(), + dense_dim.unwrap_or(0i64), + dense_dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_to_sparse_bsr( + &self, + blocksize: impl IntList, + dense_dim: impl Into>, + ) -> Result { + let dense_dim = dense_dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__to_sparse_bsr( + c_tensors.as_mut_ptr(), + self.c_tensor, + blocksize.as_ptr(), + blocksize.len_i32(), + dense_dim.unwrap_or(0i64), + dense_dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_to_sparse_bsr_out( + &self, + out: &Tensor, + blocksize: impl IntList, + dense_dim: impl Into>, + ) -> Result { + let dense_dim = dense_dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__to_sparse_bsr_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + blocksize.as_ptr(), + blocksize.len_i32(), + dense_dim.unwrap_or(0i64), + dense_dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_to_sparse_csc( + &self, + dense_dim: impl Into>, + ) -> Result { + let dense_dim = dense_dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__to_sparse_csc( + c_tensors.as_mut_ptr(), + self.c_tensor, + dense_dim.unwrap_or(0i64), + dense_dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_to_sparse_csc_out( + &self, + out: &Tensor, + dense_dim: impl Into>, + ) -> Result { + let dense_dim = dense_dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__to_sparse_csc_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dense_dim.unwrap_or(0i64), + dense_dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_to_sparse_csr( + &self, + dense_dim: impl Into>, + ) -> Result { + let dense_dim = dense_dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__to_sparse_csr( + c_tensors.as_mut_ptr(), + self.c_tensor, + dense_dim.unwrap_or(0i64), + dense_dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_to_sparse_csr_out( + &self, + out: &Tensor, + dense_dim: impl Into>, + ) -> Result { + let dense_dim = dense_dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__to_sparse_csr_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dense_dim.unwrap_or(0i64), + dense_dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_to_sparse_out( + &self, + out: &Tensor, + layout: Option, + blocksize: impl IntListOption, + dense_dim: impl Into>, + ) -> Result { + let dense_dim = dense_dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__to_sparse_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + layout.map_or(-1, |s| s.to_i8()), + blocksize.as_ptr(), + blocksize.len_i32(), + dense_dim.unwrap_or(0i64), + dense_dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_to_sparse_semi_structured( + dense: &Tensor, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__to_sparse_semi_structured(c_tensors.as_mut_ptr(), dense.c_tensor)); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_to_sparse_sparse_dim(&self, sparse_dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__to_sparse_sparse_dim( + c_tensors.as_mut_ptr(), + self.c_tensor, + sparse_dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_to_sparse_sparse_dim_out( + &self, + out: &Tensor, + sparse_dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__to_sparse_sparse_dim_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + sparse_dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_transform_bias_rescale_qkv( + qkv: &Tensor, + qkv_bias: &Tensor, + num_heads: i64, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__transform_bias_rescale_qkv( + c_tensors.as_mut_ptr(), + qkv.c_tensor, + qkv_bias.c_tensor, + num_heads + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_transform_bias_rescale_qkv_out( + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + qkv: &Tensor, + qkv_bias: &Tensor, + num_heads: i64, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__transform_bias_rescale_qkv_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + out2.c_tensor, + qkv.c_tensor, + qkv_bias.c_tensor, + num_heads + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_transformer_encoder_layer_fwd>( + src: &Tensor, + embed_dim: i64, + num_heads: i64, + qkv_weight: &Tensor, + qkv_bias: &Tensor, + proj_weight: &Tensor, + proj_bias: &Tensor, + use_gelu: bool, + norm_first: bool, + eps: f64, + norm_weight_1: &Tensor, + norm_bias_1: &Tensor, + norm_weight_2: &Tensor, + norm_bias_2: &Tensor, + ffn_weight_1: &Tensor, + ffn_bias_1: &Tensor, + ffn_weight_2: &Tensor, + ffn_bias_2: &Tensor, + mask: Option, + mask_type: impl Into>, + ) -> Result { + let mask_type = mask_type.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__transformer_encoder_layer_fwd( + c_tensors.as_mut_ptr(), + src.c_tensor, + embed_dim, + num_heads, + qkv_weight.c_tensor, + qkv_bias.c_tensor, + proj_weight.c_tensor, + proj_bias.c_tensor, + if use_gelu { 1 } else { 0 }, + if norm_first { 1 } else { 0 }, + eps, + norm_weight_1.c_tensor, + norm_bias_1.c_tensor, + norm_weight_2.c_tensor, + norm_bias_2.c_tensor, + ffn_weight_1.c_tensor, + ffn_bias_1.c_tensor, + ffn_weight_2.c_tensor, + ffn_bias_2.c_tensor, + mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + mask_type.unwrap_or(0i64), + mask_type.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_transformer_encoder_layer_fwd_out>( + out: &Tensor, + src: &Tensor, + embed_dim: i64, + num_heads: i64, + qkv_weight: &Tensor, + qkv_bias: &Tensor, + proj_weight: &Tensor, + proj_bias: &Tensor, + use_gelu: bool, + norm_first: bool, + eps: f64, + norm_weight_1: &Tensor, + norm_bias_1: &Tensor, + norm_weight_2: &Tensor, + norm_bias_2: &Tensor, + ffn_weight_1: &Tensor, + ffn_bias_1: &Tensor, + ffn_weight_2: &Tensor, + ffn_bias_2: &Tensor, + mask: Option, + mask_type: impl Into>, + ) -> Result { + let mask_type = mask_type.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__transformer_encoder_layer_fwd_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + src.c_tensor, + embed_dim, + num_heads, + qkv_weight.c_tensor, + qkv_bias.c_tensor, + proj_weight.c_tensor, + proj_bias.c_tensor, + if use_gelu { 1 } else { 0 }, + if norm_first { 1 } else { 0 }, + eps, + norm_weight_1.c_tensor, + norm_bias_1.c_tensor, + norm_weight_2.c_tensor, + norm_bias_2.c_tensor, + ffn_weight_1.c_tensor, + ffn_bias_1.c_tensor, + ffn_weight_2.c_tensor, + ffn_bias_2.c_tensor, + mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + mask_type.unwrap_or(0i64), + mask_type.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_trilinear( + i1: &Tensor, + i2: &Tensor, + i3: &Tensor, + expand1: impl IntList, + expand2: impl IntList, + expand3: impl IntList, + sumdim: impl IntList, + unroll_dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__trilinear( + c_tensors.as_mut_ptr(), + i1.c_tensor, + i2.c_tensor, + i3.c_tensor, + expand1.as_ptr(), + expand1.len_i32(), + expand2.as_ptr(), + expand2.len_i32(), + expand3.as_ptr(), + expand3.len_i32(), + sumdim.as_ptr(), + sumdim.len_i32(), + unroll_dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_trilinear_out( + out: &Tensor, + i1: &Tensor, + i2: &Tensor, + i3: &Tensor, + expand1: impl IntList, + expand2: impl IntList, + expand3: impl IntList, + sumdim: impl IntList, + unroll_dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__trilinear_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + i1.c_tensor, + i2.c_tensor, + i3.c_tensor, + expand1.as_ptr(), + expand1.len_i32(), + expand2.as_ptr(), + expand2.len_i32(), + expand3.as_ptr(), + expand3.len_i32(), + sumdim.as_ptr(), + sumdim.len_i32(), + unroll_dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_triton_multi_head_attention>( + query: &Tensor, + key: &Tensor, + value: &Tensor, + embed_dim: i64, + num_head: i64, + qkv_weight: &Tensor, + qkv_bias: &Tensor, + proj_weight: &Tensor, + proj_bias: &Tensor, + mask: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__triton_multi_head_attention( + c_tensors.as_mut_ptr(), + query.c_tensor, + key.c_tensor, + value.c_tensor, + embed_dim, + num_head, + qkv_weight.c_tensor, + qkv_bias.c_tensor, + proj_weight.c_tensor, + proj_bias.c_tensor, + mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_triton_multi_head_attention_out>( + out: &Tensor, + query: &Tensor, + key: &Tensor, + value: &Tensor, + embed_dim: i64, + num_head: i64, + qkv_weight: &Tensor, + qkv_bias: &Tensor, + proj_weight: &Tensor, + proj_bias: &Tensor, + mask: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__triton_multi_head_attention_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + query.c_tensor, + key.c_tensor, + value.c_tensor, + embed_dim, + num_head, + qkv_weight.c_tensor, + qkv_bias.c_tensor, + proj_weight.c_tensor, + proj_bias.c_tensor, + mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_triton_scaled_dot_attention( + q: &Tensor, + k: &Tensor, + v: &Tensor, + dropout_p: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__triton_scaled_dot_attention( + c_tensors.as_mut_ptr(), + q.c_tensor, + k.c_tensor, + v.c_tensor, + dropout_p + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_triton_scaled_dot_attention_out( + out: &Tensor, + q: &Tensor, + k: &Tensor, + v: &Tensor, + dropout_p: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__triton_scaled_dot_attention_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + q.c_tensor, + k.c_tensor, + v.c_tensor, + dropout_p + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_unique( + &self, + sorted: bool, + return_inverse: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__unique( + c_tensors.as_mut_ptr(), + self.c_tensor, + if sorted { 1 } else { 0 }, + if return_inverse { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_unique2( + &self, + sorted: bool, + return_inverse: bool, + return_counts: bool, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__unique2( + c_tensors.as_mut_ptr(), + self.c_tensor, + if sorted { 1 } else { 0 }, + if return_inverse { 1 } else { 0 }, + if return_counts { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_unique2_out( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + sorted: bool, + return_inverse: bool, + return_counts: bool, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg__unique2_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + out2.c_tensor, + self.c_tensor, + if sorted { 1 } else { 0 }, + if return_inverse { 1 } else { 0 }, + if return_counts { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_internal_unique_out( + &self, + out0: &Tensor, + out1: &Tensor, + sorted: bool, + return_inverse: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__unique_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + self.c_tensor, + if sorted { 1 } else { 0 }, + if return_inverse { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_unpack_dual(dual: &Tensor, level: i64) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__unpack_dual(c_tensors.as_mut_ptr(), dual.c_tensor, level)); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_unsafe_index>( + &self, + indices: &[Option], + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__unsafe_index( + c_tensors.as_mut_ptr(), + self.c_tensor, + ptr_list_opt(indices).as_ptr(), + indices.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_unsafe_index_put>( + &self, + indices: &[Option], + values: &Tensor, + accumulate: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__unsafe_index_put( + c_tensors.as_mut_ptr(), + self.c_tensor, + ptr_list_opt(indices).as_ptr(), + indices.len() as i32, + values.c_tensor, + if accumulate { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_unsafe_masked_index, S: Into>( + &self, + mask: &Tensor, + indices: &[Option], + fill: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__unsafe_masked_index( + c_tensors.as_mut_ptr(), + self.c_tensor, + mask.c_tensor, + ptr_list_opt(indices).as_ptr(), + indices.len() as i32, + fill.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_unsafe_masked_index_put_accumulate>( + &self, + mask: &Tensor, + indices: &[Option], + values: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__unsafe_masked_index_put_accumulate( + c_tensors.as_mut_ptr(), + self.c_tensor, + mask.c_tensor, + ptr_list_opt(indices).as_ptr(), + indices.len() as i32, + values.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_unsafe_view(&self, size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__unsafe_view( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_unsafe_view_out( + &self, + out: &Tensor, + size: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__unsafe_view_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_bicubic2d_aa( + &self, + output_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_bicubic2d_aa( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_bicubic2d_aa_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_bicubic2d_aa_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_bicubic2d_aa_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_bicubic2d_aa_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_bicubic2d_aa_out( + &self, + out: &Tensor, + output_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_bicubic2d_aa_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_bicubic2d_aa_vec( + &self, + output_size: impl IntListOption, + align_corners: bool, + scale_factors: impl DoubleList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_bicubic2d_aa_vec( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + if align_corners { 1 } else { 0 }, + scale_factors.as_ptr(), + scale_factors.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_bilinear2d_aa( + &self, + output_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_bilinear2d_aa( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_bilinear2d_aa_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_bilinear2d_aa_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_bilinear2d_aa_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_bilinear2d_aa_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_bilinear2d_aa_out( + &self, + out: &Tensor, + output_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_bilinear2d_aa_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_bilinear2d_aa_vec( + &self, + output_size: impl IntListOption, + align_corners: bool, + scale_factors: impl DoubleList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_bilinear2d_aa_vec( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + if align_corners { 1 } else { 0 }, + scale_factors.as_ptr(), + scale_factors.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_nearest_exact1d( + &self, + output_size: impl IntList, + scales: impl Into>, + ) -> Result { + let scales = scales.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_nearest_exact1d( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + scales.unwrap_or(std::f64::NAN), + scales.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_nearest_exact1d_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales: impl Into>, + ) -> Result { + let scales = scales.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_nearest_exact1d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + scales.unwrap_or(std::f64::NAN), + scales.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_nearest_exact1d_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales: impl Into>, + ) -> Result { + let scales = scales.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_nearest_exact1d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + scales.unwrap_or(std::f64::NAN), + scales.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_nearest_exact1d_out( + &self, + out: &Tensor, + output_size: impl IntList, + scales: impl Into>, + ) -> Result { + let scales = scales.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_nearest_exact1d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + scales.unwrap_or(std::f64::NAN), + scales.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_nearest_exact1d_vec( + &self, + output_size: impl IntListOption, + scale_factors: impl DoubleList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_nearest_exact1d_vec( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + scale_factors.as_ptr(), + scale_factors.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_nearest_exact2d( + &self, + output_size: impl IntList, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_nearest_exact2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_nearest_exact2d_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_nearest_exact2d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_nearest_exact2d_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_nearest_exact2d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_nearest_exact2d_out( + &self, + out: &Tensor, + output_size: impl IntList, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_nearest_exact2d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_nearest_exact2d_vec( + &self, + output_size: impl IntListOption, + scale_factors: impl DoubleList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_nearest_exact2d_vec( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + scale_factors.as_ptr(), + scale_factors.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_nearest_exact3d( + &self, + output_size: impl IntList, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_d = scales_d.into(); + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_nearest_exact3d( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + scales_d.unwrap_or(std::f64::NAN), + scales_d.is_none() as i8, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_nearest_exact3d_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_d = scales_d.into(); + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_nearest_exact3d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + scales_d.unwrap_or(std::f64::NAN), + scales_d.is_none() as i8, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_nearest_exact3d_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_d = scales_d.into(); + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_nearest_exact3d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + scales_d.unwrap_or(std::f64::NAN), + scales_d.is_none() as i8, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_nearest_exact3d_out( + &self, + out: &Tensor, + output_size: impl IntList, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_d = scales_d.into(); + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_nearest_exact3d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + scales_d.unwrap_or(std::f64::NAN), + scales_d.is_none() as i8, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_upsample_nearest_exact3d_vec( + &self, + output_size: impl IntListOption, + scale_factors: impl DoubleList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__upsample_nearest_exact3d_vec( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + scale_factors.as_ptr(), + scale_factors.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_use_cudnn_ctc_loss( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: impl IntList, + target_lengths: impl IntList, + blank: i64, + ) -> Result { + let return_; + unsafe_torch_err!( + return_ = atg__use_cudnn_ctc_loss( + log_probs.c_tensor, + targets.c_tensor, + input_lengths.as_ptr(), + input_lengths.len_i32(), + target_lengths.as_ptr(), + target_lengths.len_i32(), + blank + ) + ); + Ok(return_ != 0) + } + + pub fn f_internal_use_cudnn_ctc_loss_tensor( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: &Tensor, + target_lengths: &Tensor, + blank: i64, + ) -> Result { + let return_; + unsafe_torch_err!( + return_ = atg__use_cudnn_ctc_loss_tensor( + log_probs.c_tensor, + targets.c_tensor, + input_lengths.c_tensor, + target_lengths.c_tensor, + blank + ) + ); + Ok(return_ != 0) + } + + pub fn f_internal_use_cudnn_rnn_flatten_weight() -> Result { + let return_; + unsafe_torch_err!(return_ = atg__use_cudnn_rnn_flatten_weight()); + Ok(return_ != 0) + } + + pub fn f_internal_use_miopen_ctc_loss( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: impl IntList, + target_lengths: impl IntList, + blank: i64, + ) -> Result { + let return_; + unsafe_torch_err!( + return_ = atg__use_miopen_ctc_loss( + log_probs.c_tensor, + targets.c_tensor, + input_lengths.as_ptr(), + input_lengths.len_i32(), + target_lengths.as_ptr(), + target_lengths.len_i32(), + blank + ) + ); + Ok(return_ != 0) + } + + pub fn f_internal_use_miopen_ctc_loss_tensor( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: &Tensor, + target_lengths: &Tensor, + blank: i64, + ) -> Result { + let return_; + unsafe_torch_err!( + return_ = atg__use_miopen_ctc_loss_tensor( + log_probs.c_tensor, + targets.c_tensor, + input_lengths.c_tensor, + target_lengths.c_tensor, + blank + ) + ); + Ok(return_ != 0) + } + + pub fn f_internal_validate_compressed_sparse_indices( + is_crow: bool, + compressed_idx: &Tensor, + plain_idx: &Tensor, + cdim: i64, + dim: i64, + nnz: i64, + ) -> Result<(), TchError> { + unsafe_torch_err!(atg__validate_compressed_sparse_indices( + if is_crow { 1 } else { 0 }, + compressed_idx.c_tensor, + plain_idx.c_tensor, + cdim, + dim, + nnz + )); + Ok(()) + } + + pub fn f_internal_validate_sparse_bsc_tensor_args( + ccol_indices: &Tensor, + row_indices: &Tensor, + values: &Tensor, + size: impl IntList, + check_pinning: bool, + ) -> Result<(), TchError> { + unsafe_torch_err!(atg__validate_sparse_bsc_tensor_args( + ccol_indices.c_tensor, + row_indices.c_tensor, + values.c_tensor, + size.as_ptr(), + size.len_i32(), + if check_pinning { 1 } else { 0 } + )); + Ok(()) + } + + pub fn f_internal_validate_sparse_bsr_tensor_args( + crow_indices: &Tensor, + col_indices: &Tensor, + values: &Tensor, + size: impl IntList, + check_pinning: bool, + ) -> Result<(), TchError> { + unsafe_torch_err!(atg__validate_sparse_bsr_tensor_args( + crow_indices.c_tensor, + col_indices.c_tensor, + values.c_tensor, + size.as_ptr(), + size.len_i32(), + if check_pinning { 1 } else { 0 } + )); + Ok(()) + } + + pub fn f_internal_validate_sparse_compressed_tensor_args( + compressed_indices: &Tensor, + plain_indices: &Tensor, + values: &Tensor, + size: impl IntList, + layout: Layout, + check_pinning: bool, + ) -> Result<(), TchError> { + unsafe_torch_err!(atg__validate_sparse_compressed_tensor_args( + compressed_indices.c_tensor, + plain_indices.c_tensor, + values.c_tensor, + size.as_ptr(), + size.len_i32(), + layout.to_i8(), + if check_pinning { 1 } else { 0 } + )); + Ok(()) + } + + pub fn f_internal_validate_sparse_csc_tensor_args( + ccol_indices: &Tensor, + row_indices: &Tensor, + values: &Tensor, + size: impl IntList, + check_pinning: bool, + ) -> Result<(), TchError> { + unsafe_torch_err!(atg__validate_sparse_csc_tensor_args( + ccol_indices.c_tensor, + row_indices.c_tensor, + values.c_tensor, + size.as_ptr(), + size.len_i32(), + if check_pinning { 1 } else { 0 } + )); + Ok(()) + } + + pub fn f_internal_validate_sparse_csr_tensor_args( + crow_indices: &Tensor, + col_indices: &Tensor, + values: &Tensor, + size: impl IntList, + check_pinning: bool, + ) -> Result<(), TchError> { + unsafe_torch_err!(atg__validate_sparse_csr_tensor_args( + crow_indices.c_tensor, + col_indices.c_tensor, + values.c_tensor, + size.as_ptr(), + size.len_i32(), + if check_pinning { 1 } else { 0 } + )); + Ok(()) + } + + pub fn f_internal_values(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__values(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_values_copy(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__values_copy(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_values_copy_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__values_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_version(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg__version(self.c_tensor)); + Ok(return_) + } + + pub fn f_internal_weight_int4pack_mm( + &self, + mat2: &Tensor, + qgroupsize: i64, + qscaleandzeros: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__weight_int4pack_mm( + c_tensors.as_mut_ptr(), + self.c_tensor, + mat2.c_tensor, + qgroupsize, + qscaleandzeros.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_weight_int4pack_mm_for_cpu( + &self, + mat2: &Tensor, + qgroupsize: i64, + qscaleandzeros: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__weight_int4pack_mm_for_cpu( + c_tensors.as_mut_ptr(), + self.c_tensor, + mat2.c_tensor, + qgroupsize, + qscaleandzeros.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_weight_int4pack_mm_with_scales_and_zeros( + &self, + mat2: &Tensor, + qgroupsize: i64, + qscale: &Tensor, + qzeros: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__weight_int4pack_mm_with_scales_and_zeros( + c_tensors.as_mut_ptr(), + self.c_tensor, + mat2.c_tensor, + qgroupsize, + qscale.c_tensor, + qzeros.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_weight_int8pack_mm( + &self, + mat2: &Tensor, + scales: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__weight_int8pack_mm( + c_tensors.as_mut_ptr(), + self.c_tensor, + mat2.c_tensor, + scales.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_weight_norm(v: &Tensor, g: &Tensor, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__weight_norm(c_tensors.as_mut_ptr(), v.c_tensor, g.c_tensor, dim)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_weight_norm_differentiable_backward( + grad_w: &Tensor, + saved_v: &Tensor, + saved_g: &Tensor, + saved_norms: &Tensor, + dim: i64, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__weight_norm_differentiable_backward( + c_tensors.as_mut_ptr(), + grad_w.c_tensor, + saved_v.c_tensor, + saved_g.c_tensor, + saved_norms.c_tensor, + dim + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_weight_norm_interface( + v: &Tensor, + g: &Tensor, + dim: i64, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__weight_norm_interface( + c_tensors.as_mut_ptr(), + v.c_tensor, + g.c_tensor, + dim + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_weight_norm_interface_backward( + grad_w: &Tensor, + saved_v: &Tensor, + saved_g: &Tensor, + saved_norms: &Tensor, + dim: i64, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__weight_norm_interface_backward( + c_tensors.as_mut_ptr(), + grad_w.c_tensor, + saved_v.c_tensor, + saved_g.c_tensor, + saved_norms.c_tensor, + dim + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_weight_norm_interface_backward_out( + out0: &Tensor, + out1: &Tensor, + grad_w: &Tensor, + saved_v: &Tensor, + saved_g: &Tensor, + saved_norms: &Tensor, + dim: i64, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__weight_norm_interface_backward_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + grad_w.c_tensor, + saved_v.c_tensor, + saved_g.c_tensor, + saved_norms.c_tensor, + dim + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_weight_norm_interface_out( + out0: &Tensor, + out1: &Tensor, + v: &Tensor, + g: &Tensor, + dim: i64, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg__weight_norm_interface_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + v.c_tensor, + g.c_tensor, + dim + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_internal_wrapped_linear_prepack( + weight: &Tensor, + weight_scale: &Tensor, + weight_zero_point: &Tensor, + bias: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__wrapped_linear_prepack( + c_tensors.as_mut_ptr(), + weight.c_tensor, + weight_scale.c_tensor, + weight_zero_point.c_tensor, + bias.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_internal_wrapped_quantized_linear_prepacked( + &self, + input_scale: &Tensor, + input_zero_point: &Tensor, + packed_weight: &Tensor, + output_scale: &Tensor, + output_zero_point: &Tensor, + out_channel: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg__wrapped_quantized_linear_prepacked( + c_tensors.as_mut_ptr(), + self.c_tensor, + input_scale.c_tensor, + input_zero_point.c_tensor, + packed_weight.c_tensor, + output_scale.c_tensor, + output_zero_point.c_tensor, + out_channel + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_abs(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_abs(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_abs_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_abs_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_abs_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_abs_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_absolute(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_absolute(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_absolute_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_absolute_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_absolute_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_absolute_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_acos(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_acos(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_acos_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_acos_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_acos_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_acos_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_acosh(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_acosh(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_acosh_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_acosh_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_acosh_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_acosh_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_adaptive_avg_pool1d(&self, output_size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_adaptive_avg_pool1d( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_adaptive_avg_pool1d_out( + &self, + out: &Tensor, + output_size: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_adaptive_avg_pool1d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_adaptive_avg_pool2d(&self, output_size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_adaptive_avg_pool2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_adaptive_avg_pool2d_out( + &self, + out: &Tensor, + output_size: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_adaptive_avg_pool2d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_adaptive_avg_pool3d(&self, output_size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_adaptive_avg_pool3d( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_adaptive_avg_pool3d_backward( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_adaptive_avg_pool3d_backward( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_adaptive_avg_pool3d_out( + &self, + out: &Tensor, + output_size: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_adaptive_avg_pool3d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_adaptive_max_pool1d( + &self, + output_size: impl IntList, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_adaptive_max_pool1d( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32() + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_adaptive_max_pool2d( + &self, + output_size: impl IntList, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_adaptive_max_pool2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32() + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_adaptive_max_pool2d_backward( + &self, + grad_output: &Tensor, + indices: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_adaptive_max_pool2d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + indices.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_adaptive_max_pool2d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + indices: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_adaptive_max_pool2d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + indices.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_adaptive_max_pool2d_out( + &self, + out: &Tensor, + indices: &Tensor, + output_size: impl IntList, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_adaptive_max_pool2d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + indices.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32() + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_adaptive_max_pool3d( + &self, + output_size: impl IntList, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_adaptive_max_pool3d( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32() + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_adaptive_max_pool3d_backward( + &self, + grad_output: &Tensor, + indices: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_adaptive_max_pool3d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + indices.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_adaptive_max_pool3d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + indices: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_adaptive_max_pool3d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + indices.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_adaptive_max_pool3d_out( + &self, + out: &Tensor, + indices: &Tensor, + output_size: impl IntList, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_adaptive_max_pool3d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + indices.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32() + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_add(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_add(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_add_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_add_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_add_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_add_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_add_scalar>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_add_scalar( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_add_scalar_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_add_scalar_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_add_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_add_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_addbmm(&self, batch1: &Tensor, batch2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_addbmm( + c_tensors.as_mut_ptr(), + self.c_tensor, + batch1.c_tensor, + batch2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_addbmm_(&mut self, batch1: &Tensor, batch2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_addbmm_( + c_tensors.as_mut_ptr(), + self.c_tensor, + batch1.c_tensor, + batch2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_addbmm_out( + &self, + out: &Tensor, + batch1: &Tensor, + batch2: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_addbmm_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + batch1.c_tensor, + batch2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_addcdiv(&self, tensor1: &Tensor, tensor2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_addcdiv( + c_tensors.as_mut_ptr(), + self.c_tensor, + tensor1.c_tensor, + tensor2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_addcdiv_(&mut self, tensor1: &Tensor, tensor2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_addcdiv_( + c_tensors.as_mut_ptr(), + self.c_tensor, + tensor1.c_tensor, + tensor2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_addcdiv_out( + &self, + out: &Tensor, + tensor1: &Tensor, + tensor2: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_addcdiv_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + tensor1.c_tensor, + tensor2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_addcmul(&self, tensor1: &Tensor, tensor2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_addcmul( + c_tensors.as_mut_ptr(), + self.c_tensor, + tensor1.c_tensor, + tensor2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_addcmul_(&mut self, tensor1: &Tensor, tensor2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_addcmul_( + c_tensors.as_mut_ptr(), + self.c_tensor, + tensor1.c_tensor, + tensor2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_addcmul_out( + &self, + out: &Tensor, + tensor1: &Tensor, + tensor2: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_addcmul_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + tensor1.c_tensor, + tensor2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_addmm(&self, mat1: &Tensor, mat2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_addmm( + c_tensors.as_mut_ptr(), + self.c_tensor, + mat1.c_tensor, + mat2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_addmm_(&mut self, mat1: &Tensor, mat2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_addmm_( + c_tensors.as_mut_ptr(), + self.c_tensor, + mat1.c_tensor, + mat2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_addmm_dtype( + &self, + mat1: &Tensor, + mat2: &Tensor, + out_dtype: Kind, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_addmm_dtype( + c_tensors.as_mut_ptr(), + self.c_tensor, + mat1.c_tensor, + mat2.c_tensor, + out_dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_addmm_dtype_out( + &self, + out: &Tensor, + mat1: &Tensor, + mat2: &Tensor, + out_dtype: Kind, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_addmm_dtype_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mat1.c_tensor, + mat2.c_tensor, + out_dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_addmm_out( + &self, + out: &Tensor, + mat1: &Tensor, + mat2: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_addmm_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mat1.c_tensor, + mat2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_addmv(&self, mat: &Tensor, vec: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_addmv( + c_tensors.as_mut_ptr(), + self.c_tensor, + mat.c_tensor, + vec.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_addmv_(&mut self, mat: &Tensor, vec: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_addmv_( + c_tensors.as_mut_ptr(), + self.c_tensor, + mat.c_tensor, + vec.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_addmv_out( + &self, + out: &Tensor, + mat: &Tensor, + vec: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_addmv_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mat.c_tensor, + vec.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_addr(&self, vec1: &Tensor, vec2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_addr( + c_tensors.as_mut_ptr(), + self.c_tensor, + vec1.c_tensor, + vec2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_addr_(&mut self, vec1: &Tensor, vec2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_addr_( + c_tensors.as_mut_ptr(), + self.c_tensor, + vec1.c_tensor, + vec2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_addr_out( + &self, + out: &Tensor, + vec1: &Tensor, + vec2: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_addr_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + vec1.c_tensor, + vec2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_adjoint(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_adjoint(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_affine_grid_generator( + theta: &Tensor, + size: impl IntList, + align_corners: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_affine_grid_generator( + c_tensors.as_mut_ptr(), + theta.c_tensor, + size.as_ptr(), + size.len_i32(), + if align_corners { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_affine_grid_generator_backward( + grad: &Tensor, + size: impl IntList, + align_corners: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_affine_grid_generator_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + size.as_ptr(), + size.len_i32(), + if align_corners { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_affine_grid_generator_out( + out: &Tensor, + theta: &Tensor, + size: impl IntList, + align_corners: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_affine_grid_generator_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + theta.c_tensor, + size.as_ptr(), + size.len_i32(), + if align_corners { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_alias(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_alias(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_alias_copy(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_alias_copy(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_alias_copy_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_alias_copy_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_align_as(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_align_as(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_align_tensors>(tensors: &[T]) -> Result, TchError> { + let c_tensors = + unsafe_torch_err!(atg_align_tensors(ptr_list(tensors).as_ptr(), tensors.len() as i32)); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_all(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_all(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_all_all_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_all_all_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_all_dim(&self, dim: i64, keepdim: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_all_dim( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_all_dims(&self, dim: impl IntListOption, keepdim: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_all_dims( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_all_dims_out( + &self, + out: &Tensor, + dim: impl IntListOption, + keepdim: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_all_dims_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_all_out(&self, out: &Tensor, dim: i64, keepdim: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_all_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_allclose( + &self, + other: &Tensor, + rtol: f64, + atol: f64, + equal_nan: bool, + ) -> Result { + let return_; + unsafe_torch_err!( + return_ = atg_allclose( + self.c_tensor, + other.c_tensor, + rtol, + atol, + if equal_nan { 1 } else { 0 } + ) + ); + Ok(return_ != 0) + } + + pub fn f_alpha_dropout(&self, p: f64, train: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_alpha_dropout( + c_tensors.as_mut_ptr(), + self.c_tensor, + p, + if train { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_alpha_dropout_(&mut self, p: f64, train: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_alpha_dropout_( + c_tensors.as_mut_ptr(), + self.c_tensor, + p, + if train { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_amax(&self, dim: impl IntList, keepdim: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_amax( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_amax_out( + &self, + out: &Tensor, + dim: impl IntList, + keepdim: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_amax_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_amin(&self, dim: impl IntList, keepdim: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_amin( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_amin_out( + &self, + out: &Tensor, + dim: impl IntList, + keepdim: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_amin_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_aminmax( + &self, + dim: impl Into>, + keepdim: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_aminmax( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.unwrap_or(0i64), + dim.is_none() as i8, + if keepdim { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_aminmax_out( + &self, + min: &Tensor, + max: &Tensor, + dim: impl Into>, + keepdim: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_aminmax_out( + c_tensors.as_mut_ptr(), + min.c_tensor, + max.c_tensor, + self.c_tensor, + dim.unwrap_or(0i64), + dim.is_none() as i8, + if keepdim { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_angle(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_angle(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_angle_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_angle_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_any(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_any(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_any_all_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_any_all_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_any_dim(&self, dim: i64, keepdim: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_any_dim( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_any_dims(&self, dim: impl IntListOption, keepdim: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_any_dims( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_any_dims_out( + &self, + out: &Tensor, + dim: impl IntListOption, + keepdim: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_any_dims_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_any_out(&self, out: &Tensor, dim: i64, keepdim: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_any_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arange>(end: S, options: (Kind, Device)) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arange( + c_tensors.as_mut_ptr(), + end.into().c_scalar, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arange_start>( + start: S, + end: S, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arange_start( + c_tensors.as_mut_ptr(), + start.into().c_scalar, + end.into().c_scalar, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arange_start_step>( + start: S, + end: S, + step: S, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arange_start_step( + c_tensors.as_mut_ptr(), + start.into().c_scalar, + end.into().c_scalar, + step.into().c_scalar, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arccos(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arccos(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arccos_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arccos_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arccos_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arccos_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arccosh(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arccosh(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arccosh_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arccosh_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arccosh_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arccosh_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arcsin(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arcsin(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arcsin_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arcsin_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arcsin_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arcsin_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arcsinh(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arcsinh(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arcsinh_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arcsinh_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arcsinh_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arcsinh_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arctan(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arctan(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arctan2(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arctan2(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arctan2_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arctan2_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arctan2_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arctan2_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arctan_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arctan_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arctan_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arctan_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arctanh(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arctanh(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arctanh_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arctanh_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_arctanh_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_arctanh_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_argmax(&self, dim: impl Into>, keepdim: bool) -> Result { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_argmax( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.unwrap_or(0i64), + dim.is_none() as i8, + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_argmax_out( + &self, + out: &Tensor, + dim: impl Into>, + keepdim: bool, + ) -> Result { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_argmax_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.unwrap_or(0i64), + dim.is_none() as i8, + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_argmin(&self, dim: impl Into>, keepdim: bool) -> Result { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_argmin( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.unwrap_or(0i64), + dim.is_none() as i8, + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_argmin_out( + &self, + out: &Tensor, + dim: impl Into>, + keepdim: bool, + ) -> Result { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_argmin_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.unwrap_or(0i64), + dim.is_none() as i8, + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_argsort(&self, dim: i64, descending: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_argsort( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + if descending { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_argsort_stable( + &self, + stable: bool, + dim: i64, + descending: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_argsort_stable( + c_tensors.as_mut_ptr(), + self.c_tensor, + if stable { 1 } else { 0 }, + dim, + if descending { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_argsort_stable_out( + &self, + out: &Tensor, + stable: bool, + dim: i64, + descending: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_argsort_stable_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + if stable { 1 } else { 0 }, + dim, + if descending { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_argwhere(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_argwhere(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_as_strided( + &self, + size: impl IntList, + stride: impl IntList, + storage_offset: impl Into>, + ) -> Result { + let storage_offset = storage_offset.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_as_strided( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + storage_offset.unwrap_or(0i64), + storage_offset.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_as_strided_( + &mut self, + size: impl IntList, + stride: impl IntList, + storage_offset: impl Into>, + ) -> Result { + let storage_offset = storage_offset.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_as_strided_( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + storage_offset.unwrap_or(0i64), + storage_offset.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_as_strided_copy( + &self, + size: impl IntList, + stride: impl IntList, + storage_offset: impl Into>, + ) -> Result { + let storage_offset = storage_offset.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_as_strided_copy( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + storage_offset.unwrap_or(0i64), + storage_offset.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_as_strided_copy_out( + &self, + out: &Tensor, + size: impl IntList, + stride: impl IntList, + storage_offset: impl Into>, + ) -> Result { + let storage_offset = storage_offset.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_as_strided_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + size.as_ptr(), + size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + storage_offset.unwrap_or(0i64), + storage_offset.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_as_strided_scatter( + &self, + src: &Tensor, + size: impl IntList, + stride: impl IntList, + storage_offset: impl Into>, + ) -> Result { + let storage_offset = storage_offset.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_as_strided_scatter( + c_tensors.as_mut_ptr(), + self.c_tensor, + src.c_tensor, + size.as_ptr(), + size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + storage_offset.unwrap_or(0i64), + storage_offset.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_as_strided_scatter_out( + &self, + out: &Tensor, + src: &Tensor, + size: impl IntList, + stride: impl IntList, + storage_offset: impl Into>, + ) -> Result { + let storage_offset = storage_offset.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_as_strided_scatter_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + src.c_tensor, + size.as_ptr(), + size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + storage_offset.unwrap_or(0i64), + storage_offset.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_asin(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_asin(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_asin_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_asin_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_asin_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_asin_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_asinh(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_asinh(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_asinh_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_asinh_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_asinh_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_asinh_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_atan(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_atan(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_atan2(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_atan2(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_atan2_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_atan2_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_atan2_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_atan2_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_atan_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_atan_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_atan_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_atan_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_atanh(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_atanh(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_atanh_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_atanh_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_atanh_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_atanh_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_atleast_1d(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_atleast_1d(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_atleast_1d_sequence>( + tensors: &[T], + ) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_atleast_1d_sequence( + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_atleast_2d(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_atleast_2d(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_atleast_2d_sequence>( + tensors: &[T], + ) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_atleast_2d_sequence( + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_atleast_3d(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_atleast_3d(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_atleast_3d_sequence>( + tensors: &[T], + ) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_atleast_3d_sequence( + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_avg_pool1d( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ceil_mode: bool, + count_include_pad: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_avg_pool1d( + c_tensors.as_mut_ptr(), + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + if ceil_mode { 1 } else { 0 }, + if count_include_pad { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_avg_pool1d_out( + &self, + out: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ceil_mode: bool, + count_include_pad: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_avg_pool1d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + if ceil_mode { 1 } else { 0 }, + if count_include_pad { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_avg_pool2d( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ceil_mode: bool, + count_include_pad: bool, + divisor_override: impl Into>, + ) -> Result { + let divisor_override = divisor_override.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_avg_pool2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + if ceil_mode { 1 } else { 0 }, + if count_include_pad { 1 } else { 0 }, + divisor_override.unwrap_or(0i64), + divisor_override.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_avg_pool2d_backward( + &self, + grad_output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ceil_mode: bool, + count_include_pad: bool, + divisor_override: impl Into>, + ) -> Result { + let divisor_override = divisor_override.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_avg_pool2d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + if ceil_mode { 1 } else { 0 }, + if count_include_pad { 1 } else { 0 }, + divisor_override.unwrap_or(0i64), + divisor_override.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_avg_pool2d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ceil_mode: bool, + count_include_pad: bool, + divisor_override: impl Into>, + ) -> Result { + let divisor_override = divisor_override.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_avg_pool2d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + if ceil_mode { 1 } else { 0 }, + if count_include_pad { 1 } else { 0 }, + divisor_override.unwrap_or(0i64), + divisor_override.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_avg_pool2d_out( + &self, + out: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ceil_mode: bool, + count_include_pad: bool, + divisor_override: impl Into>, + ) -> Result { + let divisor_override = divisor_override.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_avg_pool2d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + if ceil_mode { 1 } else { 0 }, + if count_include_pad { 1 } else { 0 }, + divisor_override.unwrap_or(0i64), + divisor_override.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_avg_pool3d( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ceil_mode: bool, + count_include_pad: bool, + divisor_override: impl Into>, + ) -> Result { + let divisor_override = divisor_override.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_avg_pool3d( + c_tensors.as_mut_ptr(), + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + if ceil_mode { 1 } else { 0 }, + if count_include_pad { 1 } else { 0 }, + divisor_override.unwrap_or(0i64), + divisor_override.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_avg_pool3d_backward( + &self, + grad_output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ceil_mode: bool, + count_include_pad: bool, + divisor_override: impl Into>, + ) -> Result { + let divisor_override = divisor_override.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_avg_pool3d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + if ceil_mode { 1 } else { 0 }, + if count_include_pad { 1 } else { 0 }, + divisor_override.unwrap_or(0i64), + divisor_override.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_avg_pool3d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ceil_mode: bool, + count_include_pad: bool, + divisor_override: impl Into>, + ) -> Result { + let divisor_override = divisor_override.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_avg_pool3d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + if ceil_mode { 1 } else { 0 }, + if count_include_pad { 1 } else { 0 }, + divisor_override.unwrap_or(0i64), + divisor_override.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_avg_pool3d_out( + &self, + out: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ceil_mode: bool, + count_include_pad: bool, + divisor_override: impl Into>, + ) -> Result { + let divisor_override = divisor_override.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_avg_pool3d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + if ceil_mode { 1 } else { 0 }, + if count_include_pad { 1 } else { 0 }, + divisor_override.unwrap_or(0i64), + divisor_override.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_baddbmm>( + &self, + batch1: &Tensor, + batch2: &Tensor, + beta: S, + alpha: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_baddbmm( + c_tensors.as_mut_ptr(), + self.c_tensor, + batch1.c_tensor, + batch2.c_tensor, + beta.into().c_scalar, + alpha.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_baddbmm_(&mut self, batch1: &Tensor, batch2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_baddbmm_( + c_tensors.as_mut_ptr(), + self.c_tensor, + batch1.c_tensor, + batch2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_baddbmm_dtype>( + &self, + batch1: &Tensor, + batch2: &Tensor, + out_dtype: Kind, + beta: S, + alpha: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_baddbmm_dtype( + c_tensors.as_mut_ptr(), + self.c_tensor, + batch1.c_tensor, + batch2.c_tensor, + out_dtype.c_int(), + beta.into().c_scalar, + alpha.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_baddbmm_dtype_out( + &self, + out: &Tensor, + batch1: &Tensor, + batch2: &Tensor, + out_dtype: Kind, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_baddbmm_dtype_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + batch1.c_tensor, + batch2.c_tensor, + out_dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_baddbmm_out( + &self, + out: &Tensor, + batch1: &Tensor, + batch2: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_baddbmm_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + batch1.c_tensor, + batch2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bartlett_window( + window_length: i64, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bartlett_window( + c_tensors.as_mut_ptr(), + window_length, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bartlett_window_out(out: &Tensor, window_length: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bartlett_window_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + window_length + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bartlett_window_periodic( + window_length: i64, + periodic: bool, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bartlett_window_periodic( + c_tensors.as_mut_ptr(), + window_length, + if periodic { 1 } else { 0 }, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bartlett_window_periodic_out( + out: &Tensor, + window_length: i64, + periodic: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bartlett_window_periodic_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + window_length, + if periodic { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_batch_norm>( + &self, + weight: Option, + bias: Option, + running_mean: Option, + running_var: Option, + training: bool, + momentum: f64, + eps: f64, + cudnn_enabled: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_batch_norm( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if training { 1 } else { 0 }, + momentum, + eps, + if cudnn_enabled { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_batch_norm_backward_elemt>( + &self, + grad_out: &Tensor, + mean: &Tensor, + invstd: &Tensor, + weight: Option, + sum_dy: &Tensor, + sum_dy_xmu: &Tensor, + count: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_batch_norm_backward_elemt( + c_tensors.as_mut_ptr(), + grad_out.c_tensor, + self.c_tensor, + mean.c_tensor, + invstd.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + sum_dy.c_tensor, + sum_dy_xmu.c_tensor, + count.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_batch_norm_backward_elemt_out>( + &self, + out: &Tensor, + grad_out: &Tensor, + mean: &Tensor, + invstd: &Tensor, + weight: Option, + sum_dy: &Tensor, + sum_dy_xmu: &Tensor, + count: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_batch_norm_backward_elemt_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad_out.c_tensor, + self.c_tensor, + mean.c_tensor, + invstd.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + sum_dy.c_tensor, + sum_dy_xmu.c_tensor, + count.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_batch_norm_backward_reduce>( + &self, + grad_out: &Tensor, + mean: &Tensor, + invstd: &Tensor, + weight: Option, + input_g: bool, + weight_g: bool, + bias_g: bool, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg_batch_norm_backward_reduce( + c_tensors.as_mut_ptr(), + grad_out.c_tensor, + self.c_tensor, + mean.c_tensor, + invstd.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if input_g { 1 } else { 0 }, + if weight_g { 1 } else { 0 }, + if bias_g { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_batch_norm_backward_reduce_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + out3: &Tensor, + grad_out: &Tensor, + mean: &Tensor, + invstd: &Tensor, + weight: Option, + input_g: bool, + weight_g: bool, + bias_g: bool, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg_batch_norm_backward_reduce_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + out2.c_tensor, + out3.c_tensor, + grad_out.c_tensor, + self.c_tensor, + mean.c_tensor, + invstd.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if input_g { 1 } else { 0 }, + if weight_g { 1 } else { 0 }, + if bias_g { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_batch_norm_elemt>( + &self, + weight: Option, + bias: Option, + mean: &Tensor, + invstd: &Tensor, + eps: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_batch_norm_elemt( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + mean.c_tensor, + invstd.c_tensor, + eps + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_batch_norm_elemt_out>( + &self, + out: &Tensor, + weight: Option, + bias: Option, + mean: &Tensor, + invstd: &Tensor, + eps: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_batch_norm_elemt_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + mean.c_tensor, + invstd.c_tensor, + eps + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_batch_norm_gather_stats>( + &self, + mean: &Tensor, + invstd: &Tensor, + running_mean: Option, + running_var: Option, + momentum: f64, + eps: f64, + count: i64, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_batch_norm_gather_stats( + c_tensors.as_mut_ptr(), + self.c_tensor, + mean.c_tensor, + invstd.c_tensor, + running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + momentum, + eps, + count + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_batch_norm_gather_stats_out>( + &self, + out0: &Tensor, + out1: &Tensor, + mean: &Tensor, + invstd: &Tensor, + running_mean: Option, + running_var: Option, + momentum: f64, + eps: f64, + count: i64, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_batch_norm_gather_stats_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + self.c_tensor, + mean.c_tensor, + invstd.c_tensor, + running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + momentum, + eps, + count + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_batch_norm_gather_stats_with_counts>( + &self, + mean: &Tensor, + invstd: &Tensor, + running_mean: Option, + running_var: Option, + momentum: f64, + eps: f64, + counts: &Tensor, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_batch_norm_gather_stats_with_counts( + c_tensors.as_mut_ptr(), + self.c_tensor, + mean.c_tensor, + invstd.c_tensor, + running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + momentum, + eps, + counts.c_tensor + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_batch_norm_gather_stats_with_counts_out>( + &self, + out0: &Tensor, + out1: &Tensor, + mean: &Tensor, + invstd: &Tensor, + running_mean: Option, + running_var: Option, + momentum: f64, + eps: f64, + counts: &Tensor, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_batch_norm_gather_stats_with_counts_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + self.c_tensor, + mean.c_tensor, + invstd.c_tensor, + running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + momentum, + eps, + counts.c_tensor + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_batch_norm_stats(&self, eps: f64) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_batch_norm_stats(c_tensors.as_mut_ptr(), self.c_tensor, eps)); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_batch_norm_stats_out( + &self, + out0: &Tensor, + out1: &Tensor, + eps: f64, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_batch_norm_stats_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + self.c_tensor, + eps + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_batch_norm_update_stats>( + &self, + running_mean: Option, + running_var: Option, + momentum: f64, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_batch_norm_update_stats( + c_tensors.as_mut_ptr(), + self.c_tensor, + running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + momentum + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_batch_norm_update_stats_out>( + &self, + out0: &Tensor, + out1: &Tensor, + running_mean: Option, + running_var: Option, + momentum: f64, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_batch_norm_update_stats_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + self.c_tensor, + running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + momentum + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_bernoulli(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bernoulli(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bernoulli_(&mut self, p: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bernoulli_(c_tensors.as_mut_ptr(), self.c_tensor, p.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bernoulli_float_(&mut self, p: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bernoulli_float_(c_tensors.as_mut_ptr(), self.c_tensor, p)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bernoulli_p(&self, p: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bernoulli_p(c_tensors.as_mut_ptr(), self.c_tensor, p)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bernoulli_tensor(&self, p: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bernoulli_tensor(c_tensors.as_mut_ptr(), self.c_tensor, p.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bilinear>( + input1: &Tensor, + input2: &Tensor, + weight: &Tensor, + bias: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bilinear( + c_tensors.as_mut_ptr(), + input1.c_tensor, + input2.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_binary_cross_entropy>( + &self, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_binary_cross_entropy( + c_tensors.as_mut_ptr(), + self.c_tensor, + target.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_binary_cross_entropy_backward>( + &self, + grad_output: &Tensor, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_binary_cross_entropy_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + target.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_binary_cross_entropy_backward_grad_input>( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_binary_cross_entropy_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + target.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_binary_cross_entropy_out>( + &self, + out: &Tensor, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_binary_cross_entropy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + target.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_binary_cross_entropy_with_logits>( + &self, + target: &Tensor, + weight: Option, + pos_weight: Option, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_binary_cross_entropy_with_logits( + c_tensors.as_mut_ptr(), + self.c_tensor, + target.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + pos_weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_binary_cross_entropy_with_logits_out>( + &self, + out: &Tensor, + target: &Tensor, + weight: Option, + pos_weight: Option, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_binary_cross_entropy_with_logits_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + target.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + pos_weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bincount>( + &self, + weights: Option, + minlength: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bincount( + c_tensors.as_mut_ptr(), + self.c_tensor, + weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + minlength + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bincount_out>( + &self, + out: &Tensor, + weights: Option, + minlength: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bincount_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + minlength + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_binomial(count: &Tensor, prob: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_binomial(c_tensors.as_mut_ptr(), count.c_tensor, prob.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_binomial_out(out: &Tensor, count: &Tensor, prob: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_binomial_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + count.c_tensor, + prob.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_and>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_and( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_and_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_and_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_and_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_and_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_and_scalar_tensor>( + self_scalar: S, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_and_scalar_tensor( + c_tensors.as_mut_ptr(), + self_scalar.into().c_scalar, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_and_scalar_tensor_out>( + out: &Tensor, + self_scalar: S, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_and_scalar_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self_scalar.into().c_scalar, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_and_tensor(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_and_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_and_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_and_tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_and_tensor_out( + &self, + out: &Tensor, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_and_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_left_shift(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_left_shift( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_left_shift_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_left_shift_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_left_shift_scalar_tensor>( + self_scalar: S, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_left_shift_scalar_tensor( + c_tensors.as_mut_ptr(), + self_scalar.into().c_scalar, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_left_shift_scalar_tensor_out>( + out: &Tensor, + self_scalar: S, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_left_shift_scalar_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self_scalar.into().c_scalar, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_left_shift_tensor_out( + &self, + out: &Tensor, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_left_shift_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_left_shift_tensor_scalar>( + &self, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_left_shift_tensor_scalar( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_left_shift_tensor_scalar_>( + &mut self, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_left_shift_tensor_scalar_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_left_shift_tensor_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_left_shift_tensor_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_not(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_not(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_not_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_not_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_not_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_not_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_or>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_or( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_or_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_or_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_or_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_or_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_or_scalar_tensor>( + self_scalar: S, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_or_scalar_tensor( + c_tensors.as_mut_ptr(), + self_scalar.into().c_scalar, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_or_scalar_tensor_out>( + out: &Tensor, + self_scalar: S, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_or_scalar_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self_scalar.into().c_scalar, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_or_tensor(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_or_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_or_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_or_tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_or_tensor_out( + &self, + out: &Tensor, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_or_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_right_shift(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_right_shift( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_right_shift_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_right_shift_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_right_shift_scalar_tensor>( + self_scalar: S, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_right_shift_scalar_tensor( + c_tensors.as_mut_ptr(), + self_scalar.into().c_scalar, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_right_shift_scalar_tensor_out>( + out: &Tensor, + self_scalar: S, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_right_shift_scalar_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self_scalar.into().c_scalar, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_right_shift_tensor_out( + &self, + out: &Tensor, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_right_shift_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_right_shift_tensor_scalar>( + &self, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_right_shift_tensor_scalar( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_right_shift_tensor_scalar_>( + &mut self, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_right_shift_tensor_scalar_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_right_shift_tensor_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_right_shift_tensor_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_xor>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_xor( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_xor_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_xor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_xor_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_xor_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_xor_scalar_tensor>( + self_scalar: S, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_xor_scalar_tensor( + c_tensors.as_mut_ptr(), + self_scalar.into().c_scalar, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_xor_scalar_tensor_out>( + out: &Tensor, + self_scalar: S, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_xor_scalar_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self_scalar.into().c_scalar, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_xor_tensor(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_xor_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_xor_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_xor_tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bitwise_xor_tensor_out( + &self, + out: &Tensor, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bitwise_xor_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_blackman_window( + window_length: i64, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_blackman_window( + c_tensors.as_mut_ptr(), + window_length, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_blackman_window_out(out: &Tensor, window_length: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_blackman_window_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + window_length + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_blackman_window_periodic( + window_length: i64, + periodic: bool, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_blackman_window_periodic( + c_tensors.as_mut_ptr(), + window_length, + if periodic { 1 } else { 0 }, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_blackman_window_periodic_out( + out: &Tensor, + window_length: i64, + periodic: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_blackman_window_periodic_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + window_length, + if periodic { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_block_diag>(tensors: &[T]) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_block_diag( + c_tensors.as_mut_ptr(), + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_block_diag_out>( + out: &Tensor, + tensors: &[T], + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_block_diag_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bmm(&self, mat2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bmm(c_tensors.as_mut_ptr(), self.c_tensor, mat2.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bmm_dtype(&self, mat2: &Tensor, out_dtype: Kind) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bmm_dtype( + c_tensors.as_mut_ptr(), + self.c_tensor, + mat2.c_tensor, + out_dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bmm_dtype_out( + &self, + out: &Tensor, + mat2: &Tensor, + out_dtype: Kind, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bmm_dtype_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mat2.c_tensor, + out_dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bmm_out(&self, out: &Tensor, mat2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bmm_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mat2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_broadcast_tensors>(tensors: &[T]) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_broadcast_tensors( + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_broadcast_to(&self, size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_broadcast_to( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bucketize( + &self, + boundaries: &Tensor, + out_int32: bool, + right: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bucketize( + c_tensors.as_mut_ptr(), + self.c_tensor, + boundaries.c_tensor, + if out_int32 { 1 } else { 0 }, + if right { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bucketize_scalar>( + self_scalar: S, + boundaries: &Tensor, + out_int32: bool, + right: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bucketize_scalar( + c_tensors.as_mut_ptr(), + self_scalar.into().c_scalar, + boundaries.c_tensor, + if out_int32 { 1 } else { 0 }, + if right { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bucketize_scalar_out>( + out: &Tensor, + self_scalar: S, + boundaries: &Tensor, + out_int32: bool, + right: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bucketize_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self_scalar.into().c_scalar, + boundaries.c_tensor, + if out_int32 { 1 } else { 0 }, + if right { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_bucketize_tensor_out( + &self, + out: &Tensor, + boundaries: &Tensor, + out_int32: bool, + right: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_bucketize_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + boundaries.c_tensor, + if out_int32 { 1 } else { 0 }, + if right { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_can_cast(from_: Kind, to: Kind) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_can_cast(from_.c_int(), to.c_int())); + Ok(return_ != 0) + } + + pub fn f_cartesian_prod>(tensors: &[T]) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cartesian_prod( + c_tensors.as_mut_ptr(), + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cat>(tensors: &[T], dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cat( + c_tensors.as_mut_ptr(), + ptr_list(tensors).as_ptr(), + tensors.len() as i32, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cat_out>( + out: &Tensor, + tensors: &[T], + dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cat_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + ptr_list(tensors).as_ptr(), + tensors.len() as i32, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cauchy(&self, median: f64, sigma: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cauchy(c_tensors.as_mut_ptr(), self.c_tensor, median, sigma)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cauchy_(&mut self, median: f64, sigma: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cauchy_(c_tensors.as_mut_ptr(), self.c_tensor, median, sigma)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cauchy_out(&self, out: &Tensor, median: f64, sigma: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cauchy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + median, + sigma + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ccol_indices(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ccol_indices(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ccol_indices_copy(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ccol_indices_copy(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ccol_indices_copy_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ccol_indices_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cdist( + x1: &Tensor, + x2: &Tensor, + p: f64, + compute_mode: impl Into>, + ) -> Result { + let compute_mode = compute_mode.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cdist( + c_tensors.as_mut_ptr(), + x1.c_tensor, + x2.c_tensor, + p, + compute_mode.unwrap_or(0i64), + compute_mode.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ceil(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ceil(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ceil_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ceil_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ceil_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ceil_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_celu(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_celu(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_celu_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_celu_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_celu_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_celu_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_chain_matmul>(matrices: &[T]) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_chain_matmul( + c_tensors.as_mut_ptr(), + ptr_list(matrices).as_ptr(), + matrices.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_chain_matmul_out>( + out: &Tensor, + matrices: &[T], + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_chain_matmul_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + ptr_list(matrices).as_ptr(), + matrices.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_chalf(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_chalf(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_channel_shuffle(&self, groups: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_channel_shuffle(c_tensors.as_mut_ptr(), self.c_tensor, groups)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_channel_shuffle_out(&self, out: &Tensor, groups: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_channel_shuffle_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cholesky(&self, upper: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cholesky( + c_tensors.as_mut_ptr(), + self.c_tensor, + if upper { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cholesky_inverse(&self, upper: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cholesky_inverse( + c_tensors.as_mut_ptr(), + self.c_tensor, + if upper { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cholesky_inverse_out(&self, out: &Tensor, upper: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cholesky_inverse_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + if upper { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cholesky_out(&self, out: &Tensor, upper: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cholesky_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + if upper { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cholesky_solve(&self, input2: &Tensor, upper: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cholesky_solve( + c_tensors.as_mut_ptr(), + self.c_tensor, + input2.c_tensor, + if upper { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cholesky_solve_out( + &self, + out: &Tensor, + input2: &Tensor, + upper: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cholesky_solve_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + input2.c_tensor, + if upper { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_choose_qparams_optimized( + &self, + numel: i64, + n_bins: i64, + ratio: f64, + bit_width: i64, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_choose_qparams_optimized( + c_tensors.as_mut_ptr(), + self.c_tensor, + numel, + n_bins, + ratio, + bit_width + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_chunk(&self, chunks: i64, dim: i64) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_chunk(self.c_tensor, chunks, dim)); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_clamp>(&self, min: S, max: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clamp( + c_tensors.as_mut_ptr(), + self.c_tensor, + min.into().c_scalar, + max.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clamp_>(&mut self, min: S, max: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clamp_( + c_tensors.as_mut_ptr(), + self.c_tensor, + min.into().c_scalar, + max.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clamp_max>(&self, max: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clamp_max( + c_tensors.as_mut_ptr(), + self.c_tensor, + max.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clamp_max_>(&mut self, max: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clamp_max_( + c_tensors.as_mut_ptr(), + self.c_tensor, + max.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clamp_max_out>( + &self, + out: &Tensor, + max: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clamp_max_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + max.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clamp_max_tensor(&self, max: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clamp_max_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + max.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clamp_max_tensor_(&mut self, max: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clamp_max_tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + max.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clamp_max_tensor_out(&self, out: &Tensor, max: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clamp_max_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + max.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clamp_min>(&self, min: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clamp_min( + c_tensors.as_mut_ptr(), + self.c_tensor, + min.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clamp_min_>(&mut self, min: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clamp_min_( + c_tensors.as_mut_ptr(), + self.c_tensor, + min.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clamp_min_out>( + &self, + out: &Tensor, + min: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clamp_min_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + min.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clamp_min_tensor(&self, min: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clamp_min_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + min.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clamp_min_tensor_(&mut self, min: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clamp_min_tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + min.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clamp_min_tensor_out(&self, out: &Tensor, min: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clamp_min_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + min.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clamp_out>( + &self, + out: &Tensor, + min: S, + max: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clamp_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + min.into().c_scalar, + max.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clamp_tensor>( + &self, + min: Option, + max: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clamp_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + min.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + max.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clamp_tensor_>( + &mut self, + min: Option, + max: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clamp_tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + min.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + max.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clamp_tensor_out>( + &self, + out: &Tensor, + min: Option, + max: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clamp_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + min.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + max.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clip>(&self, min: S, max: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clip( + c_tensors.as_mut_ptr(), + self.c_tensor, + min.into().c_scalar, + max.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clip_>(&mut self, min: S, max: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clip_( + c_tensors.as_mut_ptr(), + self.c_tensor, + min.into().c_scalar, + max.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clip_out>( + &self, + out: &Tensor, + min: S, + max: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clip_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + min.into().c_scalar, + max.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clip_tensor>( + &self, + min: Option, + max: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clip_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + min.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + max.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clip_tensor_>( + &mut self, + min: Option, + max: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clip_tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + min.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + max.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clip_tensor_out>( + &self, + out: &Tensor, + min: Option, + max: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clip_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + min.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + max.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_clone(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_clone(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_coalesce(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_coalesce(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_col2im( + &self, + output_size: impl IntList, + kernel_size: impl IntList, + dilation: impl IntList, + padding: impl IntList, + stride: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_col2im( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + kernel_size.as_ptr(), + kernel_size.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + padding.as_ptr(), + padding.len_i32(), + stride.as_ptr(), + stride.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_col2im_out( + &self, + out: &Tensor, + output_size: impl IntList, + kernel_size: impl IntList, + dilation: impl IntList, + padding: impl IntList, + stride: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_col2im_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + kernel_size.as_ptr(), + kernel_size.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + padding.as_ptr(), + padding.len_i32(), + stride.as_ptr(), + stride.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_col_indices(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_col_indices(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_col_indices_copy(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_col_indices_copy(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_col_indices_copy_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_col_indices_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_column_stack>(tensors: &[T]) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_column_stack( + c_tensors.as_mut_ptr(), + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_column_stack_out>( + out: &Tensor, + tensors: &[T], + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_column_stack_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_combinations(&self, r: i64, with_replacement: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_combinations( + c_tensors.as_mut_ptr(), + self.c_tensor, + r, + if with_replacement { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_complex(real: &Tensor, imag: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_complex(c_tensors.as_mut_ptr(), real.c_tensor, imag.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_complex_out(out: &Tensor, real: &Tensor, imag: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_complex_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + real.c_tensor, + imag.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_concat>(tensors: &[T], dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_concat( + c_tensors.as_mut_ptr(), + ptr_list(tensors).as_ptr(), + tensors.len() as i32, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_concat_out>( + out: &Tensor, + tensors: &[T], + dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_concat_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + ptr_list(tensors).as_ptr(), + tensors.len() as i32, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_concatenate>(tensors: &[T], dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_concatenate( + c_tensors.as_mut_ptr(), + ptr_list(tensors).as_ptr(), + tensors.len() as i32, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_concatenate_out>( + out: &Tensor, + tensors: &[T], + dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_concatenate_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + ptr_list(tensors).as_ptr(), + tensors.len() as i32, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_conj(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_conj(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_conj_physical(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_conj_physical(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_conj_physical_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_conj_physical_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_conj_physical_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_conj_physical_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_constant_pad_nd(&self, pad: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_constant_pad_nd( + c_tensors.as_mut_ptr(), + self.c_tensor, + pad.as_ptr(), + pad.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_constant_pad_nd_out( + &self, + out: &Tensor, + pad: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_constant_pad_nd_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + pad.as_ptr(), + pad.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_contiguous(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_contiguous(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_conv1d>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_conv1d( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_conv1d_padding>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: &str, + dilation: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_conv1d_padding( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len() as i32, + dilation.as_ptr(), + dilation.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_conv2d>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_conv2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_conv2d_padding>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: &str, + dilation: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_conv2d_padding( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len() as i32, + dilation.as_ptr(), + dilation.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_conv3d>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_conv3d( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_conv3d_padding>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: &str, + dilation: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_conv3d_padding( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len() as i32, + dilation.as_ptr(), + dilation.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_conv_depthwise3d>( + &self, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_conv_depthwise3d( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_conv_depthwise3d_out>( + &self, + out: &Tensor, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_conv_depthwise3d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_conv_tbc(&self, weight: &Tensor, bias: &Tensor, pad: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_conv_tbc( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.c_tensor, + pad + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_conv_tbc_backward( + &self, + input: &Tensor, + weight: &Tensor, + bias: &Tensor, + pad: i64, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_conv_tbc_backward( + c_tensors.as_mut_ptr(), + self.c_tensor, + input.c_tensor, + weight.c_tensor, + bias.c_tensor, + pad + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_conv_tbc_out( + &self, + out: &Tensor, + weight: &Tensor, + bias: &Tensor, + pad: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_conv_tbc_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + bias.c_tensor, + pad + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_conv_transpose1d>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + output_padding: impl IntList, + groups: i64, + dilation: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_conv_transpose1d( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + output_padding.as_ptr(), + output_padding.len_i32(), + groups, + dilation.as_ptr(), + dilation.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_conv_transpose2d>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + output_padding: impl IntList, + groups: i64, + dilation: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_conv_transpose2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + output_padding.as_ptr(), + output_padding.len_i32(), + groups, + dilation.as_ptr(), + dilation.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_conv_transpose3d>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + output_padding: impl IntList, + groups: i64, + dilation: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_conv_transpose3d( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + output_padding.as_ptr(), + output_padding.len_i32(), + groups, + dilation.as_ptr(), + dilation.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_convolution>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + transposed: bool, + output_padding: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_convolution( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if transposed { 1 } else { 0 }, + output_padding.as_ptr(), + output_padding.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_convolution_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + transposed: bool, + output_padding: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_convolution_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if transposed { 1 } else { 0 }, + output_padding.as_ptr(), + output_padding.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_convolution_overrideable>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + transposed: bool, + output_padding: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_convolution_overrideable( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if transposed { 1 } else { 0 }, + output_padding.as_ptr(), + output_padding.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_convolution_overrideable_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + transposed: bool, + output_padding: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_convolution_overrideable_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if transposed { 1 } else { 0 }, + output_padding.as_ptr(), + output_padding.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_copy_sparse_to_sparse( + &self, + src: &Tensor, + non_blocking: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_copy_sparse_to_sparse( + c_tensors.as_mut_ptr(), + self.c_tensor, + src.c_tensor, + if non_blocking { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_copy_sparse_to_sparse_( + &mut self, + src: &Tensor, + non_blocking: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_copy_sparse_to_sparse_( + c_tensors.as_mut_ptr(), + self.c_tensor, + src.c_tensor, + if non_blocking { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_copy_sparse_to_sparse_out( + &self, + out: &Tensor, + src: &Tensor, + non_blocking: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_copy_sparse_to_sparse_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + src.c_tensor, + if non_blocking { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_copysign(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_copysign(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_copysign_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_copysign_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_copysign_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_copysign_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_copysign_scalar>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_copysign_scalar( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_copysign_scalar_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_copysign_scalar_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_copysign_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_copysign_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_corrcoef(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_corrcoef(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cos(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cos(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cos_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cos_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cos_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cos_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cosh(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cosh(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cosh_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cosh_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cosh_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cosh_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cosine_embedding_loss( + input1: &Tensor, + input2: &Tensor, + target: &Tensor, + margin: f64, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cosine_embedding_loss( + c_tensors.as_mut_ptr(), + input1.c_tensor, + input2.c_tensor, + target.c_tensor, + margin, + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cosine_similarity( + x1: &Tensor, + x2: &Tensor, + dim: i64, + eps: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cosine_similarity( + c_tensors.as_mut_ptr(), + x1.c_tensor, + x2.c_tensor, + dim, + eps + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_count_nonzero(&self, dim: impl Into>) -> Result { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_count_nonzero( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.unwrap_or(0i64), + dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_count_nonzero_dim_intlist(&self, dim: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_count_nonzero_dim_intlist( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_count_nonzero_dim_intlist_out( + &self, + out: &Tensor, + dim: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_count_nonzero_dim_intlist_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_count_nonzero_out( + &self, + out: &Tensor, + dim: impl Into>, + ) -> Result { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_count_nonzero_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.unwrap_or(0i64), + dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cov>( + &self, + correction: i64, + fweights: Option, + aweights: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cov( + c_tensors.as_mut_ptr(), + self.c_tensor, + correction, + fweights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + aweights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cross(&self, other: &Tensor, dim: impl Into>) -> Result { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cross( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor, + dim.unwrap_or(0i64), + dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cross_entropy_loss>( + &self, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ignore_index: i64, + label_smoothing: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cross_entropy_loss( + c_tensors.as_mut_ptr(), + self.c_tensor, + target.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + reduction.to_int(), + ignore_index, + label_smoothing + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cross_out( + &self, + out: &Tensor, + other: &Tensor, + dim: impl Into>, + ) -> Result { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cross_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor, + dim.unwrap_or(0i64), + dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_crow_indices(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_crow_indices(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_crow_indices_copy(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_crow_indices_copy(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_crow_indices_copy_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_crow_indices_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ctc_loss( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: impl IntList, + target_lengths: impl IntList, + blank: i64, + reduction: crate::Reduction, + zero_infinity: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ctc_loss( + c_tensors.as_mut_ptr(), + log_probs.c_tensor, + targets.c_tensor, + input_lengths.as_ptr(), + input_lengths.len_i32(), + target_lengths.as_ptr(), + target_lengths.len_i32(), + blank, + reduction.to_int(), + if zero_infinity { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ctc_loss_tensor( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: &Tensor, + target_lengths: &Tensor, + blank: i64, + reduction: crate::Reduction, + zero_infinity: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ctc_loss_tensor( + c_tensors.as_mut_ptr(), + log_probs.c_tensor, + targets.c_tensor, + input_lengths.c_tensor, + target_lengths.c_tensor, + blank, + reduction.to_int(), + if zero_infinity { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cudnn_affine_grid_generator( + theta: &Tensor, + n: i64, + c: i64, + h: i64, + w: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cudnn_affine_grid_generator( + c_tensors.as_mut_ptr(), + theta.c_tensor, + n, + c, + h, + w + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cudnn_affine_grid_generator_backward( + grad: &Tensor, + n: i64, + c: i64, + h: i64, + w: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cudnn_affine_grid_generator_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + n, + c, + h, + w + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cudnn_affine_grid_generator_backward_out( + out: &Tensor, + grad: &Tensor, + n: i64, + c: i64, + h: i64, + w: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cudnn_affine_grid_generator_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad.c_tensor, + n, + c, + h, + w + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cudnn_affine_grid_generator_out( + out: &Tensor, + theta: &Tensor, + n: i64, + c: i64, + h: i64, + w: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cudnn_affine_grid_generator_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + theta.c_tensor, + n, + c, + h, + w + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cudnn_batch_norm>( + &self, + weight: &Tensor, + bias: Option, + running_mean: Option, + running_var: Option, + training: bool, + exponential_average_factor: f64, + epsilon: f64, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg_cudnn_batch_norm( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if training { 1 } else { 0 }, + exponential_average_factor, + epsilon + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_cudnn_batch_norm_backward>( + &self, + grad_output: &Tensor, + weight: &Tensor, + running_mean: Option, + running_var: Option, + save_mean: Option, + save_var: Option, + epsilon: f64, + reservespace: &Tensor, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_cudnn_batch_norm_backward( + c_tensors.as_mut_ptr(), + self.c_tensor, + grad_output.c_tensor, + weight.c_tensor, + running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + save_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + save_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + epsilon, + reservespace.c_tensor + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_cudnn_batch_norm_backward_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + grad_output: &Tensor, + weight: &Tensor, + running_mean: Option, + running_var: Option, + save_mean: Option, + save_var: Option, + epsilon: f64, + reservespace: &Tensor, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_cudnn_batch_norm_backward_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + out2.c_tensor, + self.c_tensor, + grad_output.c_tensor, + weight.c_tensor, + running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + save_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + save_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + epsilon, + reservespace.c_tensor + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_cudnn_batch_norm_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + out3: &Tensor, + weight: &Tensor, + bias: Option, + running_mean: Option, + running_var: Option, + training: bool, + exponential_average_factor: f64, + epsilon: f64, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg_cudnn_batch_norm_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + out2.c_tensor, + out3.c_tensor, + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if training { 1 } else { 0 }, + exponential_average_factor, + epsilon + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_cudnn_convolution( + &self, + weight: &Tensor, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + allow_tf32: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cudnn_convolution( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + padding.as_ptr(), + padding.len_i32(), + stride.as_ptr(), + stride.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups, + if benchmark { 1 } else { 0 }, + if deterministic { 1 } else { 0 }, + if allow_tf32 { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cudnn_convolution_add_relu, S: Into>( + &self, + weight: &Tensor, + z: &Tensor, + alpha: S, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cudnn_convolution_add_relu( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + z.c_tensor, + alpha.into().c_scalar, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cudnn_convolution_add_relu_out, S: Into>( + &self, + out: &Tensor, + weight: &Tensor, + z: &Tensor, + alpha: S, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cudnn_convolution_add_relu_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + z.c_tensor, + alpha.into().c_scalar, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cudnn_convolution_out( + &self, + out: &Tensor, + weight: &Tensor, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + allow_tf32: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cudnn_convolution_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + padding.as_ptr(), + padding.len_i32(), + stride.as_ptr(), + stride.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups, + if benchmark { 1 } else { 0 }, + if deterministic { 1 } else { 0 }, + if allow_tf32 { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cudnn_convolution_relu>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cudnn_convolution_relu( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cudnn_convolution_relu_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cudnn_convolution_relu_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cudnn_convolution_transpose( + &self, + weight: &Tensor, + padding: impl IntList, + output_padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + allow_tf32: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cudnn_convolution_transpose( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + padding.as_ptr(), + padding.len_i32(), + output_padding.as_ptr(), + output_padding.len_i32(), + stride.as_ptr(), + stride.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups, + if benchmark { 1 } else { 0 }, + if deterministic { 1 } else { 0 }, + if allow_tf32 { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cudnn_convolution_transpose_out( + &self, + out: &Tensor, + weight: &Tensor, + padding: impl IntList, + output_padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + allow_tf32: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cudnn_convolution_transpose_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + padding.as_ptr(), + padding.len_i32(), + output_padding.as_ptr(), + output_padding.len_i32(), + stride.as_ptr(), + stride.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups, + if benchmark { 1 } else { 0 }, + if deterministic { 1 } else { 0 }, + if allow_tf32 { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cudnn_grid_sampler(&self, grid: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cudnn_grid_sampler( + c_tensors.as_mut_ptr(), + self.c_tensor, + grid.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cudnn_grid_sampler_backward( + &self, + grid: &Tensor, + grad_output: &Tensor, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_cudnn_grid_sampler_backward( + c_tensors.as_mut_ptr(), + self.c_tensor, + grid.c_tensor, + grad_output.c_tensor + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_cudnn_grid_sampler_backward_out( + &self, + out0: &Tensor, + out1: &Tensor, + grid: &Tensor, + grad_output: &Tensor, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_cudnn_grid_sampler_backward_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + self.c_tensor, + grid.c_tensor, + grad_output.c_tensor + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_cudnn_grid_sampler_out( + &self, + out: &Tensor, + grid: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cudnn_grid_sampler_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + grid.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cudnn_is_acceptable(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_cudnn_is_acceptable(self.c_tensor)); + Ok(return_ != 0) + } + + pub fn f_cummax(&self, dim: i64) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_cummax(c_tensors.as_mut_ptr(), self.c_tensor, dim)); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_cummax_out( + &self, + values: &Tensor, + indices: &Tensor, + dim: i64, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_cummax_out( + c_tensors.as_mut_ptr(), + values.c_tensor, + indices.c_tensor, + self.c_tensor, + dim + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_cummaxmin_backward( + &self, + grad: &Tensor, + indices: &Tensor, + dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cummaxmin_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + self.c_tensor, + indices.c_tensor, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cummin(&self, dim: i64) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_cummin(c_tensors.as_mut_ptr(), self.c_tensor, dim)); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_cummin_out( + &self, + values: &Tensor, + indices: &Tensor, + dim: i64, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_cummin_out( + c_tensors.as_mut_ptr(), + values.c_tensor, + indices.c_tensor, + self.c_tensor, + dim + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_cumprod(&self, dim: i64, dtype: impl Into>) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cumprod( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cumprod_( + &mut self, + dim: i64, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cumprod_( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cumprod_backward( + &self, + grad: &Tensor, + dim: i64, + output: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cumprod_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + self.c_tensor, + dim, + output.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cumprod_out( + &self, + out: &Tensor, + dim: i64, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cumprod_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cumsum(&self, dim: i64, dtype: impl Into>) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cumsum( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cumsum_( + &mut self, + dim: i64, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cumsum_( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cumsum_out( + &self, + out: &Tensor, + dim: i64, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cumsum_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cumulative_trapezoid(y: &Tensor, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cumulative_trapezoid(c_tensors.as_mut_ptr(), y.c_tensor, dim)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_cumulative_trapezoid_x(y: &Tensor, x: &Tensor, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_cumulative_trapezoid_x( + c_tensors.as_mut_ptr(), + y.c_tensor, + x.c_tensor, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_data(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_data(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_deg2rad(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_deg2rad(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_deg2rad_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_deg2rad_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_deg2rad_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_deg2rad_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_dense_dim(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_dense_dim(self.c_tensor)); + Ok(return_) + } + + pub fn f_dequantize(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_dequantize(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_dequantize_self_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_dequantize_self_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_dequantize_tensors>(tensors: &[T]) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_dequantize_tensors( + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_dequantize_tensors_out>( + out: &[T], + tensors: &[T], + ) -> Result<(), TchError> { + unsafe_torch_err!(atg_dequantize_tensors_out( + ptr_list(out).as_ptr(), + out.len() as i32, + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + Ok(()) + } + + pub fn f_det(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_det(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_detach(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_detach(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_detach_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_detach_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_detach_copy(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_detach_copy(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_detach_copy_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_detach_copy_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_diag(&self, diagonal: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_diag(c_tensors.as_mut_ptr(), self.c_tensor, diagonal)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_diag_embed(&self, offset: i64, dim1: i64, dim2: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_diag_embed( + c_tensors.as_mut_ptr(), + self.c_tensor, + offset, + dim1, + dim2 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_diag_embed_out( + &self, + out: &Tensor, + offset: i64, + dim1: i64, + dim2: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_diag_embed_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + offset, + dim1, + dim2 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_diag_out(&self, out: &Tensor, diagonal: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_diag_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + diagonal + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_diagflat(&self, offset: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_diagflat(c_tensors.as_mut_ptr(), self.c_tensor, offset)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_diagonal(&self, offset: i64, dim1: i64, dim2: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_diagonal(c_tensors.as_mut_ptr(), self.c_tensor, offset, dim1, dim2)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_diagonal_backward( + grad_output: &Tensor, + input_sizes: impl IntList, + offset: i64, + dim1: i64, + dim2: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_diagonal_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + input_sizes.as_ptr(), + input_sizes.len_i32(), + offset, + dim1, + dim2 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_diagonal_backward_out( + out: &Tensor, + grad_output: &Tensor, + input_sizes: impl IntList, + offset: i64, + dim1: i64, + dim2: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_diagonal_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad_output.c_tensor, + input_sizes.as_ptr(), + input_sizes.len_i32(), + offset, + dim1, + dim2 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_diagonal_copy(&self, offset: i64, dim1: i64, dim2: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_diagonal_copy( + c_tensors.as_mut_ptr(), + self.c_tensor, + offset, + dim1, + dim2 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_diagonal_copy_out( + &self, + out: &Tensor, + offset: i64, + dim1: i64, + dim2: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_diagonal_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + offset, + dim1, + dim2 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_diagonal_scatter( + &self, + src: &Tensor, + offset: i64, + dim1: i64, + dim2: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_diagonal_scatter( + c_tensors.as_mut_ptr(), + self.c_tensor, + src.c_tensor, + offset, + dim1, + dim2 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_diagonal_scatter_out( + &self, + out: &Tensor, + src: &Tensor, + offset: i64, + dim1: i64, + dim2: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_diagonal_scatter_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + src.c_tensor, + offset, + dim1, + dim2 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_diff>( + &self, + n: i64, + dim: i64, + prepend: Option, + append: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_diff( + c_tensors.as_mut_ptr(), + self.c_tensor, + n, + dim, + prepend.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + append.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_diff_out>( + &self, + out: &Tensor, + n: i64, + dim: i64, + prepend: Option, + append: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_diff_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + n, + dim, + prepend.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + append.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_digamma(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_digamma(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_digamma_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_digamma_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_digamma_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_digamma_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_dist(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_dist(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_dist_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_dist_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_div(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_div(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_div_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_div_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_div_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_div_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_div_out_mode( + &self, + out: &Tensor, + other: &Tensor, + rounding_mode: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_div_out_mode( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor, + rounding_mode.as_ptr(), + rounding_mode.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_div_scalar>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_div_scalar( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_div_scalar_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_div_scalar_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_div_scalar_mode>( + &self, + other: S, + rounding_mode: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_div_scalar_mode( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar, + rounding_mode.as_ptr(), + rounding_mode.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_div_scalar_mode_>( + &mut self, + other: S, + rounding_mode: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_div_scalar_mode_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar, + rounding_mode.as_ptr(), + rounding_mode.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_div_scalar_mode_out>( + &self, + out: &Tensor, + other: S, + rounding_mode: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_div_scalar_mode_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar, + rounding_mode.as_ptr(), + rounding_mode.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_div_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_div_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_div_tensor_mode( + &self, + other: &Tensor, + rounding_mode: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_div_tensor_mode( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor, + rounding_mode.as_ptr(), + rounding_mode.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_div_tensor_mode_( + &mut self, + other: &Tensor, + rounding_mode: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_div_tensor_mode_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor, + rounding_mode.as_ptr(), + rounding_mode.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_divide(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_divide(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_divide_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_divide_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_divide_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_divide_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_divide_out_mode( + &self, + out: &Tensor, + other: &Tensor, + rounding_mode: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_divide_out_mode( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor, + rounding_mode.as_ptr(), + rounding_mode.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_divide_scalar>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_divide_scalar( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_divide_scalar_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_divide_scalar_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_divide_scalar_mode>( + &self, + other: S, + rounding_mode: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_divide_scalar_mode( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar, + rounding_mode.as_ptr(), + rounding_mode.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_divide_scalar_mode_>( + &mut self, + other: S, + rounding_mode: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_divide_scalar_mode_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar, + rounding_mode.as_ptr(), + rounding_mode.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_divide_tensor_mode( + &self, + other: &Tensor, + rounding_mode: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_divide_tensor_mode( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor, + rounding_mode.as_ptr(), + rounding_mode.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_divide_tensor_mode_( + &mut self, + other: &Tensor, + rounding_mode: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_divide_tensor_mode_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor, + rounding_mode.as_ptr(), + rounding_mode.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_dot(&self, tensor: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_dot(c_tensors.as_mut_ptr(), self.c_tensor, tensor.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_dot_out(&self, out: &Tensor, tensor: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_dot_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + tensor.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_dropout(&self, p: f64, train: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_dropout( + c_tensors.as_mut_ptr(), + self.c_tensor, + p, + if train { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_dropout_(&mut self, p: f64, train: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_dropout_( + c_tensors.as_mut_ptr(), + self.c_tensor, + p, + if train { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_dsplit(&self, sections: i64) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_dsplit(self.c_tensor, sections)); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_dsplit_array(&self, indices: impl IntList) -> Result, TchError> { + let c_tensors = + unsafe_torch_err!(atg_dsplit_array(self.c_tensor, indices.as_ptr(), indices.len_i32())); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_dstack>(tensors: &[T]) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_dstack( + c_tensors.as_mut_ptr(), + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_dstack_out>( + out: &Tensor, + tensors: &[T], + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_dstack_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_einsum>( + equation: &str, + tensors: &[T], + path: impl IntListOption, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_einsum( + c_tensors.as_mut_ptr(), + equation.as_ptr(), + equation.len() as i32, + ptr_list(tensors).as_ptr(), + tensors.len() as i32, + path.as_ptr(), + path.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_elu(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_elu(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_elu_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_elu_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_elu_backward>( + grad_output: &Tensor, + alpha: S, + scale: S, + input_scale: S, + is_result: bool, + self_or_result: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_elu_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + alpha.into().c_scalar, + scale.into().c_scalar, + input_scale.into().c_scalar, + if is_result { 1 } else { 0 }, + self_or_result.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_elu_backward_grad_input>( + grad_input: &Tensor, + grad_output: &Tensor, + alpha: S, + scale: S, + input_scale: S, + is_result: bool, + self_or_result: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_elu_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + alpha.into().c_scalar, + scale.into().c_scalar, + input_scale.into().c_scalar, + if is_result { 1 } else { 0 }, + self_or_result.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_elu_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_elu_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_embedding( + weight: &Tensor, + indices: &Tensor, + padding_idx: i64, + scale_grad_by_freq: bool, + sparse: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_embedding( + c_tensors.as_mut_ptr(), + weight.c_tensor, + indices.c_tensor, + padding_idx, + if scale_grad_by_freq { 1 } else { 0 }, + if sparse { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_embedding_backward( + grad: &Tensor, + indices: &Tensor, + num_weights: i64, + padding_idx: i64, + scale_grad_by_freq: bool, + sparse: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_embedding_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + indices.c_tensor, + num_weights, + padding_idx, + if scale_grad_by_freq { 1 } else { 0 }, + if sparse { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_embedding_bag>( + weight: &Tensor, + indices: &Tensor, + offsets: &Tensor, + scale_grad_by_freq: bool, + mode: i64, + sparse: bool, + per_sample_weights: Option, + include_last_offset: bool, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg_embedding_bag( + c_tensors.as_mut_ptr(), + weight.c_tensor, + indices.c_tensor, + offsets.c_tensor, + if scale_grad_by_freq { 1 } else { 0 }, + mode, + if sparse { 1 } else { 0 }, + per_sample_weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if include_last_offset { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_embedding_bag_padding_idx>( + weight: &Tensor, + indices: &Tensor, + offsets: &Tensor, + scale_grad_by_freq: bool, + mode: i64, + sparse: bool, + per_sample_weights: Option, + include_last_offset: bool, + padding_idx: impl Into>, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let padding_idx = padding_idx.into(); + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg_embedding_bag_padding_idx( + c_tensors.as_mut_ptr(), + weight.c_tensor, + indices.c_tensor, + offsets.c_tensor, + if scale_grad_by_freq { 1 } else { 0 }, + mode, + if sparse { 1 } else { 0 }, + per_sample_weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if include_last_offset { 1 } else { 0 }, + padding_idx.unwrap_or(0i64), + padding_idx.is_none() as i8 + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_embedding_dense_backward( + grad_output: &Tensor, + indices: &Tensor, + num_weights: i64, + padding_idx: i64, + scale_grad_by_freq: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_embedding_dense_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + indices.c_tensor, + num_weights, + padding_idx, + if scale_grad_by_freq { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_embedding_dense_backward_out( + out: &Tensor, + grad_output: &Tensor, + indices: &Tensor, + num_weights: i64, + padding_idx: i64, + scale_grad_by_freq: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_embedding_dense_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad_output.c_tensor, + indices.c_tensor, + num_weights, + padding_idx, + if scale_grad_by_freq { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_embedding_out( + out: &Tensor, + weight: &Tensor, + indices: &Tensor, + padding_idx: i64, + scale_grad_by_freq: bool, + sparse: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_embedding_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + weight.c_tensor, + indices.c_tensor, + padding_idx, + if scale_grad_by_freq { 1 } else { 0 }, + if sparse { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_embedding_renorm( + &self, + indices: &Tensor, + max_norm: f64, + norm_type: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_embedding_renorm( + c_tensors.as_mut_ptr(), + self.c_tensor, + indices.c_tensor, + max_norm, + norm_type + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_embedding_renorm_( + &mut self, + indices: &Tensor, + max_norm: f64, + norm_type: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_embedding_renorm_( + c_tensors.as_mut_ptr(), + self.c_tensor, + indices.c_tensor, + max_norm, + norm_type + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_embedding_renorm_out( + &self, + out: &Tensor, + indices: &Tensor, + max_norm: f64, + norm_type: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_embedding_renorm_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + indices.c_tensor, + max_norm, + norm_type + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_embedding_sparse_backward( + grad: &Tensor, + indices: &Tensor, + num_weights: i64, + padding_idx: i64, + scale_grad_by_freq: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_embedding_sparse_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + indices.c_tensor, + num_weights, + padding_idx, + if scale_grad_by_freq { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_empty(size: impl IntList, options: (Kind, Device)) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_empty( + c_tensors.as_mut_ptr(), + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_empty_like(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_empty_like(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_empty_like_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_empty_like_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_empty_out(out: &Tensor, size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_empty_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_empty_permuted( + size: impl IntList, + physical_layout: impl IntList, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_empty_permuted( + c_tensors.as_mut_ptr(), + size.as_ptr(), + size.len_i32(), + physical_layout.as_ptr(), + physical_layout.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_empty_permuted_out( + out: &Tensor, + size: impl IntList, + physical_layout: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_empty_permuted_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + size.as_ptr(), + size.len_i32(), + physical_layout.as_ptr(), + physical_layout.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_empty_quantized( + size: impl IntList, + qtensor: &Tensor, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_empty_quantized( + c_tensors.as_mut_ptr(), + size.as_ptr(), + size.len_i32(), + qtensor.c_tensor, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_empty_quantized_out( + out: &Tensor, + size: impl IntList, + qtensor: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_empty_quantized_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + size.as_ptr(), + size.len_i32(), + qtensor.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_empty_strided( + size: impl IntList, + stride: impl IntList, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_empty_strided( + c_tensors.as_mut_ptr(), + size.as_ptr(), + size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_empty_strided_out( + out: &Tensor, + size: impl IntList, + stride: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_empty_strided_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + size.as_ptr(), + size.len_i32(), + stride.as_ptr(), + stride.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_eq>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_eq(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_eq_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_eq_(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_eq_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_eq_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_eq_tensor(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_eq_tensor(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_eq_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_eq_tensor_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_eq_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_eq_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_equal(&self, other: &Tensor) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_equal(self.c_tensor, other.c_tensor)); + Ok(return_ != 0) + } + + pub fn f_erf(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_erf(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_erf_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_erf_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_erf_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_erf_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_erfc(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_erfc(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_erfc_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_erfc_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_erfc_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_erfc_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_erfinv(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_erfinv(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_erfinv_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_erfinv_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_erfinv_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_erfinv_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_exp(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_exp(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_exp2(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_exp2(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_exp2_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_exp2_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_exp2_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_exp2_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_exp_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_exp_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_exp_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_exp_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_expand(&self, size: impl IntList, implicit: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_expand( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32(), + if implicit { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_expand_as(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_expand_as(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_expand_copy(&self, size: impl IntList, implicit: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_expand_copy( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32(), + if implicit { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_expand_copy_out( + &self, + out: &Tensor, + size: impl IntList, + implicit: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_expand_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + size.as_ptr(), + size.len_i32(), + if implicit { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_expm1(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_expm1(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_expm1_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_expm1_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_expm1_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_expm1_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_exponential(&self, lambd: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_exponential(c_tensors.as_mut_ptr(), self.c_tensor, lambd)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_exponential_(&mut self, lambd: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_exponential_(c_tensors.as_mut_ptr(), self.c_tensor, lambd)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_exponential_out(&self, out: &Tensor, lambd: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_exponential_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + lambd + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_eye(n: i64, options: (Kind, Device)) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_eye(c_tensors.as_mut_ptr(), n, options.0.c_int(), options.1.c_int())); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_eye_m(n: i64, m: i64, options: (Kind, Device)) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_eye_m( + c_tensors.as_mut_ptr(), + n, + m, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_eye_m_out(out: &Tensor, n: i64, m: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_eye_m_out(c_tensors.as_mut_ptr(), out.c_tensor, n, m)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_eye_out(out: &Tensor, n: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_eye_out(c_tensors.as_mut_ptr(), out.c_tensor, n)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fake_quantize_per_channel_affine( + &self, + scale: &Tensor, + zero_point: &Tensor, + axis: i64, + quant_min: i64, + quant_max: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fake_quantize_per_channel_affine( + c_tensors.as_mut_ptr(), + self.c_tensor, + scale.c_tensor, + zero_point.c_tensor, + axis, + quant_min, + quant_max + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fake_quantize_per_channel_affine_cachemask( + &self, + scale: &Tensor, + zero_point: &Tensor, + axis: i64, + quant_min: i64, + quant_max: i64, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_fake_quantize_per_channel_affine_cachemask( + c_tensors.as_mut_ptr(), + self.c_tensor, + scale.c_tensor, + zero_point.c_tensor, + axis, + quant_min, + quant_max + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_fake_quantize_per_channel_affine_cachemask_backward( + grad: &Tensor, + mask: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fake_quantize_per_channel_affine_cachemask_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + mask.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fake_quantize_per_channel_affine_cachemask_out( + &self, + out0: &Tensor, + out1: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + axis: i64, + quant_min: i64, + quant_max: i64, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_fake_quantize_per_channel_affine_cachemask_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + self.c_tensor, + scale.c_tensor, + zero_point.c_tensor, + axis, + quant_min, + quant_max + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_fake_quantize_per_tensor_affine( + &self, + scale: f64, + zero_point: i64, + quant_min: i64, + quant_max: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fake_quantize_per_tensor_affine( + c_tensors.as_mut_ptr(), + self.c_tensor, + scale, + zero_point, + quant_min, + quant_max + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fake_quantize_per_tensor_affine_cachemask( + &self, + scale: f64, + zero_point: i64, + quant_min: i64, + quant_max: i64, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_fake_quantize_per_tensor_affine_cachemask( + c_tensors.as_mut_ptr(), + self.c_tensor, + scale, + zero_point, + quant_min, + quant_max + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_fake_quantize_per_tensor_affine_cachemask_backward( + grad: &Tensor, + mask: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fake_quantize_per_tensor_affine_cachemask_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + mask.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fake_quantize_per_tensor_affine_cachemask_out( + &self, + out0: &Tensor, + out1: &Tensor, + scale: f64, + zero_point: i64, + quant_min: i64, + quant_max: i64, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_fake_quantize_per_tensor_affine_cachemask_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + self.c_tensor, + scale, + zero_point, + quant_min, + quant_max + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_fake_quantize_per_tensor_affine_tensor_qparams( + &self, + scale: &Tensor, + zero_point: &Tensor, + quant_min: i64, + quant_max: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fake_quantize_per_tensor_affine_tensor_qparams( + c_tensors.as_mut_ptr(), + self.c_tensor, + scale.c_tensor, + zero_point.c_tensor, + quant_min, + quant_max + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_feature_alpha_dropout(&self, p: f64, train: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_feature_alpha_dropout( + c_tensors.as_mut_ptr(), + self.c_tensor, + p, + if train { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_feature_alpha_dropout_(&mut self, p: f64, train: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_feature_alpha_dropout_( + c_tensors.as_mut_ptr(), + self.c_tensor, + p, + if train { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_feature_dropout(&self, p: f64, train: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_feature_dropout( + c_tensors.as_mut_ptr(), + self.c_tensor, + p, + if train { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_feature_dropout_(&mut self, p: f64, train: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_feature_dropout_( + c_tensors.as_mut_ptr(), + self.c_tensor, + p, + if train { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_fft( + &self, + n: impl Into>, + dim: i64, + norm: &str, + ) -> Result { + let n = n.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_fft( + c_tensors.as_mut_ptr(), + self.c_tensor, + n.unwrap_or(0i64), + n.is_none() as i8, + dim, + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_fft2( + &self, + s: impl IntListOption, + dim: impl IntList, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_fft2( + c_tensors.as_mut_ptr(), + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_fft2_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntList, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_fft2_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_fft_out( + &self, + out: &Tensor, + n: impl Into>, + dim: i64, + norm: &str, + ) -> Result { + let n = n.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_fft_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + n.unwrap_or(0i64), + n.is_none() as i8, + dim, + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_fftfreq(n: i64, d: f64, options: (Kind, Device)) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_fftfreq( + c_tensors.as_mut_ptr(), + n, + d, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_fftfreq_out(out: &Tensor, n: i64, d: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_fftfreq_out(c_tensors.as_mut_ptr(), out.c_tensor, n, d)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_fftn( + &self, + s: impl IntListOption, + dim: impl IntListOption, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_fftn( + c_tensors.as_mut_ptr(), + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_fftn_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntListOption, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_fftn_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_fftshift(&self, dim: impl IntListOption) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_fftshift( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_hfft( + &self, + n: impl Into>, + dim: i64, + norm: &str, + ) -> Result { + let n = n.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_hfft( + c_tensors.as_mut_ptr(), + self.c_tensor, + n.unwrap_or(0i64), + n.is_none() as i8, + dim, + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_hfft2( + &self, + s: impl IntListOption, + dim: impl IntList, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_hfft2( + c_tensors.as_mut_ptr(), + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_hfft2_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntList, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_hfft2_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_hfft_out( + &self, + out: &Tensor, + n: impl Into>, + dim: i64, + norm: &str, + ) -> Result { + let n = n.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_hfft_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + n.unwrap_or(0i64), + n.is_none() as i8, + dim, + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_hfftn( + &self, + s: impl IntListOption, + dim: impl IntListOption, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_hfftn( + c_tensors.as_mut_ptr(), + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_hfftn_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntListOption, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_hfftn_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_ifft( + &self, + n: impl Into>, + dim: i64, + norm: &str, + ) -> Result { + let n = n.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_ifft( + c_tensors.as_mut_ptr(), + self.c_tensor, + n.unwrap_or(0i64), + n.is_none() as i8, + dim, + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_ifft2( + &self, + s: impl IntListOption, + dim: impl IntList, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_ifft2( + c_tensors.as_mut_ptr(), + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_ifft2_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntList, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_ifft2_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_ifft_out( + &self, + out: &Tensor, + n: impl Into>, + dim: i64, + norm: &str, + ) -> Result { + let n = n.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_ifft_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + n.unwrap_or(0i64), + n.is_none() as i8, + dim, + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_ifftn( + &self, + s: impl IntListOption, + dim: impl IntListOption, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_ifftn( + c_tensors.as_mut_ptr(), + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_ifftn_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntListOption, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_ifftn_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_ifftshift(&self, dim: impl IntListOption) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_ifftshift( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_ihfft( + &self, + n: impl Into>, + dim: i64, + norm: &str, + ) -> Result { + let n = n.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_ihfft( + c_tensors.as_mut_ptr(), + self.c_tensor, + n.unwrap_or(0i64), + n.is_none() as i8, + dim, + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_ihfft2( + &self, + s: impl IntListOption, + dim: impl IntList, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_ihfft2( + c_tensors.as_mut_ptr(), + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_ihfft2_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntList, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_ihfft2_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_ihfft_out( + &self, + out: &Tensor, + n: impl Into>, + dim: i64, + norm: &str, + ) -> Result { + let n = n.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_ihfft_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + n.unwrap_or(0i64), + n.is_none() as i8, + dim, + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_ihfftn( + &self, + s: impl IntListOption, + dim: impl IntListOption, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_ihfftn( + c_tensors.as_mut_ptr(), + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_ihfftn_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntListOption, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_ihfftn_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_irfft( + &self, + n: impl Into>, + dim: i64, + norm: &str, + ) -> Result { + let n = n.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_irfft( + c_tensors.as_mut_ptr(), + self.c_tensor, + n.unwrap_or(0i64), + n.is_none() as i8, + dim, + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_irfft2( + &self, + s: impl IntListOption, + dim: impl IntList, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_irfft2( + c_tensors.as_mut_ptr(), + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_irfft2_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntList, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_irfft2_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_irfft_out( + &self, + out: &Tensor, + n: impl Into>, + dim: i64, + norm: &str, + ) -> Result { + let n = n.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_irfft_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + n.unwrap_or(0i64), + n.is_none() as i8, + dim, + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_irfftn( + &self, + s: impl IntListOption, + dim: impl IntListOption, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_irfftn( + c_tensors.as_mut_ptr(), + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_irfftn_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntListOption, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_irfftn_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_rfft( + &self, + n: impl Into>, + dim: i64, + norm: &str, + ) -> Result { + let n = n.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_rfft( + c_tensors.as_mut_ptr(), + self.c_tensor, + n.unwrap_or(0i64), + n.is_none() as i8, + dim, + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_rfft2( + &self, + s: impl IntListOption, + dim: impl IntList, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_rfft2( + c_tensors.as_mut_ptr(), + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_rfft2_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntList, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_rfft2_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_rfft_out( + &self, + out: &Tensor, + n: impl Into>, + dim: i64, + norm: &str, + ) -> Result { + let n = n.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_rfft_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + n.unwrap_or(0i64), + n.is_none() as i8, + dim, + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_rfftfreq(n: i64, d: f64, options: (Kind, Device)) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_rfftfreq( + c_tensors.as_mut_ptr(), + n, + d, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_rfftfreq_out(out: &Tensor, n: i64, d: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_rfftfreq_out(c_tensors.as_mut_ptr(), out.c_tensor, n, d)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_rfftn( + &self, + s: impl IntListOption, + dim: impl IntListOption, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_rfftn( + c_tensors.as_mut_ptr(), + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fft_rfftn_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntListOption, + norm: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fft_rfftn_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + s.as_ptr(), + s.len_i32(), + dim.as_ptr(), + dim.len_i32(), + norm.as_ptr(), + norm.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fill>(&self, value: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fill(c_tensors.as_mut_ptr(), self.c_tensor, value.into().c_scalar)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fill_>(&mut self, value: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fill_(c_tensors.as_mut_ptr(), self.c_tensor, value.into().c_scalar)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fill_diagonal_>( + &mut self, + fill_value: S, + wrap: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fill_diagonal_( + c_tensors.as_mut_ptr(), + self.c_tensor, + fill_value.into().c_scalar, + if wrap { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fill_scalar_out>( + &self, + out: &Tensor, + value: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fill_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + value.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fill_tensor(&self, value: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fill_tensor(c_tensors.as_mut_ptr(), self.c_tensor, value.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fill_tensor_(&mut self, value: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fill_tensor_(c_tensors.as_mut_ptr(), self.c_tensor, value.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fill_tensor_out(&self, out: &Tensor, value: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fill_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + value.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fix(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fix(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fix_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fix_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fix_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fix_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_flatten(&self, start_dim: i64, end_dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_flatten(c_tensors.as_mut_ptr(), self.c_tensor, start_dim, end_dim)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_flatten_dense_tensors>(tensors: &[T]) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_flatten_dense_tensors( + c_tensors.as_mut_ptr(), + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_flip(&self, dims: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_flip( + c_tensors.as_mut_ptr(), + self.c_tensor, + dims.as_ptr(), + dims.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_flip_out(&self, out: &Tensor, dims: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_flip_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dims.as_ptr(), + dims.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fliplr(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fliplr(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_flipud(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_flipud(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_float_power(&self, exponent: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_float_power( + c_tensors.as_mut_ptr(), + self.c_tensor, + exponent.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_float_power_>(&mut self, exponent: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_float_power_( + c_tensors.as_mut_ptr(), + self.c_tensor, + exponent.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_float_power_scalar>( + self_scalar: S, + exponent: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_float_power_scalar( + c_tensors.as_mut_ptr(), + self_scalar.into().c_scalar, + exponent.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_float_power_scalar_out>( + out: &Tensor, + self_scalar: S, + exponent: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_float_power_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self_scalar.into().c_scalar, + exponent.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_float_power_tensor_(&mut self, exponent: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_float_power_tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + exponent.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_float_power_tensor_scalar>( + &self, + exponent: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_float_power_tensor_scalar( + c_tensors.as_mut_ptr(), + self.c_tensor, + exponent.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_float_power_tensor_scalar_out>( + &self, + out: &Tensor, + exponent: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_float_power_tensor_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + exponent.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_float_power_tensor_tensor_out( + &self, + out: &Tensor, + exponent: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_float_power_tensor_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + exponent.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_floor(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_floor(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_floor_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_floor_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_floor_divide(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_floor_divide(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_floor_divide_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_floor_divide_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_floor_divide_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_floor_divide_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_floor_divide_scalar>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_floor_divide_scalar( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_floor_divide_scalar_>( + &mut self, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_floor_divide_scalar_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_floor_divide_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_floor_divide_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_floor_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_floor_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fmax(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fmax(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fmax_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fmax_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fmin(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fmin(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fmin_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fmin_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fmod>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fmod(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fmod_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fmod_(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fmod_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fmod_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fmod_tensor(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fmod_tensor(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fmod_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fmod_tensor_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fmod_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fmod_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_frac(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_frac(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_frac_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_frac_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_frac_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_frac_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fractional_max_pool2d( + &self, + kernel_size: impl IntList, + output_size: impl IntList, + random_samples: &Tensor, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_fractional_max_pool2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + output_size.as_ptr(), + output_size.len_i32(), + random_samples.c_tensor + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_fractional_max_pool2d_backward( + &self, + grad_output: &Tensor, + kernel_size: impl IntList, + output_size: impl IntList, + indices: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fractional_max_pool2d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + output_size.as_ptr(), + output_size.len_i32(), + indices.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fractional_max_pool2d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + kernel_size: impl IntList, + output_size: impl IntList, + indices: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fractional_max_pool2d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + output_size.as_ptr(), + output_size.len_i32(), + indices.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fractional_max_pool2d_output( + &self, + output: &Tensor, + indices: &Tensor, + kernel_size: impl IntList, + output_size: impl IntList, + random_samples: &Tensor, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_fractional_max_pool2d_output( + c_tensors.as_mut_ptr(), + output.c_tensor, + indices.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + output_size.as_ptr(), + output_size.len_i32(), + random_samples.c_tensor + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_fractional_max_pool3d( + &self, + kernel_size: impl IntList, + output_size: impl IntList, + random_samples: &Tensor, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_fractional_max_pool3d( + c_tensors.as_mut_ptr(), + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + output_size.as_ptr(), + output_size.len_i32(), + random_samples.c_tensor + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_fractional_max_pool3d_backward( + &self, + grad_output: &Tensor, + kernel_size: impl IntList, + output_size: impl IntList, + indices: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fractional_max_pool3d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + output_size.as_ptr(), + output_size.len_i32(), + indices.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fractional_max_pool3d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + kernel_size: impl IntList, + output_size: impl IntList, + indices: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fractional_max_pool3d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + output_size.as_ptr(), + output_size.len_i32(), + indices.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fractional_max_pool3d_output( + &self, + output: &Tensor, + indices: &Tensor, + kernel_size: impl IntList, + output_size: impl IntList, + random_samples: &Tensor, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_fractional_max_pool3d_output( + c_tensors.as_mut_ptr(), + output.c_tensor, + indices.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + output_size.as_ptr(), + output_size.len_i32(), + random_samples.c_tensor + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_frexp(&self) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_frexp(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_frexp_tensor_out( + &self, + mantissa: &Tensor, + exponent: &Tensor, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_frexp_tensor_out( + c_tensors.as_mut_ptr(), + mantissa.c_tensor, + exponent.c_tensor, + self.c_tensor + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_frobenius_norm(&self, dim: impl IntList, keepdim: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_frobenius_norm( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_frobenius_norm_out( + &self, + out: &Tensor, + dim: impl IntList, + keepdim: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_frobenius_norm_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_from_file( + filename: &str, + shared: bool, + size: impl Into>, + options: (Kind, Device), + ) -> Result { + let size = size.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_from_file( + c_tensors.as_mut_ptr(), + filename.as_ptr(), + filename.len() as i32, + if shared { 1 } else { 0 }, + size.unwrap_or(0i64), + size.is_none() as i8, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_from_file_out( + out: &Tensor, + filename: &str, + shared: bool, + size: impl Into>, + ) -> Result { + let size = size.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_from_file_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + filename.as_ptr(), + filename.len() as i32, + if shared { 1 } else { 0 }, + size.unwrap_or(0i64), + size.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_full>( + size: impl IntList, + fill_value: S, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_full( + c_tensors.as_mut_ptr(), + size.as_ptr(), + size.len_i32(), + fill_value.into().c_scalar, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_full_like>(&self, fill_value: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_full_like( + c_tensors.as_mut_ptr(), + self.c_tensor, + fill_value.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_full_like_out>( + &self, + out: &Tensor, + fill_value: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_full_like_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + fill_value.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_full_out>( + out: &Tensor, + size: impl IntList, + fill_value: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_full_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + size.as_ptr(), + size.len_i32(), + fill_value.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_fused_moving_avg_obs_fake_quant( + &self, + observer_on: &Tensor, + fake_quant_on: &Tensor, + running_min: &Tensor, + running_max: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + averaging_const: f64, + quant_min: i64, + quant_max: i64, + ch_axis: i64, + per_row_fake_quant: bool, + symmetric_quant: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_fused_moving_avg_obs_fake_quant( + c_tensors.as_mut_ptr(), + self.c_tensor, + observer_on.c_tensor, + fake_quant_on.c_tensor, + running_min.c_tensor, + running_max.c_tensor, + scale.c_tensor, + zero_point.c_tensor, + averaging_const, + quant_min, + quant_max, + ch_axis, + if per_row_fake_quant { 1 } else { 0 }, + if symmetric_quant { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_gather( + &self, + dim: i64, + index: &Tensor, + sparse_grad: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_gather( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + if sparse_grad { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_gather_backward( + &self, + grad: &Tensor, + dim: i64, + index: &Tensor, + sparse_grad: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_gather_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + self.c_tensor, + dim, + index.c_tensor, + if sparse_grad { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_gather_out( + &self, + out: &Tensor, + dim: i64, + index: &Tensor, + sparse_grad: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_gather_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + index.c_tensor, + if sparse_grad { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_gcd(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_gcd(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_gcd_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_gcd_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_gcd_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_gcd_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ge>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ge(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ge_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ge_(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ge_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ge_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ge_tensor(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ge_tensor(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ge_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ge_tensor_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ge_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ge_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_gelu(&self, approximate: &str) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_gelu( + c_tensors.as_mut_ptr(), + self.c_tensor, + approximate.as_ptr(), + approximate.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_gelu_(&mut self, approximate: &str) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_gelu_( + c_tensors.as_mut_ptr(), + self.c_tensor, + approximate.as_ptr(), + approximate.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_gelu_backward( + &self, + grad_output: &Tensor, + approximate: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_gelu_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + approximate.as_ptr(), + approximate.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_gelu_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + approximate: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_gelu_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + approximate.as_ptr(), + approximate.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_gelu_out(&self, out: &Tensor, approximate: &str) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_gelu_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + approximate.as_ptr(), + approximate.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_geometric(&self, p: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_geometric(c_tensors.as_mut_ptr(), self.c_tensor, p)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_geometric_(&mut self, p: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_geometric_(c_tensors.as_mut_ptr(), self.c_tensor, p)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_geometric_out(&self, out: &Tensor, p: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_geometric_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + p + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_geqrf(&self) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_geqrf(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_geqrf_a(&self, a: &Tensor, tau: &Tensor) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_geqrf_a( + c_tensors.as_mut_ptr(), + a.c_tensor, + tau.c_tensor, + self.c_tensor + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_ger(&self, vec2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ger(c_tensors.as_mut_ptr(), self.c_tensor, vec2.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ger_out(&self, out: &Tensor, vec2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ger_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + vec2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_glu(&self, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_glu(c_tensors.as_mut_ptr(), self.c_tensor, dim)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_glu_backward(&self, grad_output: &Tensor, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_glu_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_glu_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_glu_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_glu_backward_jvp( + grad_x: &Tensor, + grad_glu: &Tensor, + x: &Tensor, + dgrad_glu: &Tensor, + dx: &Tensor, + dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_glu_backward_jvp( + c_tensors.as_mut_ptr(), + grad_x.c_tensor, + grad_glu.c_tensor, + x.c_tensor, + dgrad_glu.c_tensor, + dx.c_tensor, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_glu_backward_jvp_out( + out: &Tensor, + grad_x: &Tensor, + grad_glu: &Tensor, + x: &Tensor, + dgrad_glu: &Tensor, + dx: &Tensor, + dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_glu_backward_jvp_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad_x.c_tensor, + grad_glu.c_tensor, + x.c_tensor, + dgrad_glu.c_tensor, + dx.c_tensor, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_glu_jvp(glu: &Tensor, x: &Tensor, dx: &Tensor, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_glu_jvp( + c_tensors.as_mut_ptr(), + glu.c_tensor, + x.c_tensor, + dx.c_tensor, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_glu_jvp_out( + out: &Tensor, + glu: &Tensor, + x: &Tensor, + dx: &Tensor, + dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_glu_jvp_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + glu.c_tensor, + x.c_tensor, + dx.c_tensor, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_glu_out(&self, out: &Tensor, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_glu_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor, dim)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_grad(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_grad(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_greater>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_greater( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_greater_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_greater_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_greater_equal>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_greater_equal( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_greater_equal_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_greater_equal_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_greater_equal_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_greater_equal_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_greater_equal_tensor(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_greater_equal_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_greater_equal_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_greater_equal_tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_greater_equal_tensor_out( + &self, + out: &Tensor, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_greater_equal_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_greater_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_greater_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_greater_tensor(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_greater_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_greater_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_greater_tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_greater_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_greater_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_grid_sampler( + &self, + grid: &Tensor, + interpolation_mode: i64, + padding_mode: i64, + align_corners: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_grid_sampler( + c_tensors.as_mut_ptr(), + self.c_tensor, + grid.c_tensor, + interpolation_mode, + padding_mode, + if align_corners { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_grid_sampler_2d( + &self, + grid: &Tensor, + interpolation_mode: i64, + padding_mode: i64, + align_corners: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_grid_sampler_2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + grid.c_tensor, + interpolation_mode, + padding_mode, + if align_corners { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_grid_sampler_2d_out( + &self, + out: &Tensor, + grid: &Tensor, + interpolation_mode: i64, + padding_mode: i64, + align_corners: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_grid_sampler_2d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + grid.c_tensor, + interpolation_mode, + padding_mode, + if align_corners { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_grid_sampler_3d( + &self, + grid: &Tensor, + interpolation_mode: i64, + padding_mode: i64, + align_corners: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_grid_sampler_3d( + c_tensors.as_mut_ptr(), + self.c_tensor, + grid.c_tensor, + interpolation_mode, + padding_mode, + if align_corners { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_grid_sampler_3d_out( + &self, + out: &Tensor, + grid: &Tensor, + interpolation_mode: i64, + padding_mode: i64, + align_corners: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_grid_sampler_3d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + grid.c_tensor, + interpolation_mode, + padding_mode, + if align_corners { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_group_norm>( + &self, + num_groups: i64, + weight: Option, + bias: Option, + eps: f64, + cudnn_enabled: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_group_norm( + c_tensors.as_mut_ptr(), + self.c_tensor, + num_groups, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + eps, + if cudnn_enabled { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_gru>( + &self, + hx: &Tensor, + params: &[T], + has_biases: bool, + num_layers: i64, + dropout: f64, + train: bool, + bidirectional: bool, + batch_first: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_gru( + c_tensors.as_mut_ptr(), + self.c_tensor, + hx.c_tensor, + ptr_list(params).as_ptr(), + params.len() as i32, + if has_biases { 1 } else { 0 }, + num_layers, + dropout, + if train { 1 } else { 0 }, + if bidirectional { 1 } else { 0 }, + if batch_first { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_gru_cell>( + &self, + hx: &Tensor, + w_ih: &Tensor, + w_hh: &Tensor, + b_ih: Option, + b_hh: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_gru_cell( + c_tensors.as_mut_ptr(), + self.c_tensor, + hx.c_tensor, + w_ih.c_tensor, + w_hh.c_tensor, + b_ih.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + b_hh.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_gru_data>( + data: &Tensor, + batch_sizes: &Tensor, + hx: &Tensor, + params: &[T], + has_biases: bool, + num_layers: i64, + dropout: f64, + train: bool, + bidirectional: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_gru_data( + c_tensors.as_mut_ptr(), + data.c_tensor, + batch_sizes.c_tensor, + hx.c_tensor, + ptr_list(params).as_ptr(), + params.len() as i32, + if has_biases { 1 } else { 0 }, + num_layers, + dropout, + if train { 1 } else { 0 }, + if bidirectional { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_gt>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_gt(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_gt_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_gt_(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_gt_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_gt_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_gt_tensor(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_gt_tensor(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_gt_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_gt_tensor_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_gt_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_gt_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hamming_window( + window_length: i64, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hamming_window( + c_tensors.as_mut_ptr(), + window_length, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hamming_window_out(out: &Tensor, window_length: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hamming_window_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + window_length + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hamming_window_periodic( + window_length: i64, + periodic: bool, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hamming_window_periodic( + c_tensors.as_mut_ptr(), + window_length, + if periodic { 1 } else { 0 }, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hamming_window_periodic_alpha( + window_length: i64, + periodic: bool, + alpha: f64, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hamming_window_periodic_alpha( + c_tensors.as_mut_ptr(), + window_length, + if periodic { 1 } else { 0 }, + alpha, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hamming_window_periodic_alpha_beta( + window_length: i64, + periodic: bool, + alpha: f64, + beta: f64, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hamming_window_periodic_alpha_beta( + c_tensors.as_mut_ptr(), + window_length, + if periodic { 1 } else { 0 }, + alpha, + beta, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hamming_window_periodic_alpha_beta_out( + out: &Tensor, + window_length: i64, + periodic: bool, + alpha: f64, + beta: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hamming_window_periodic_alpha_beta_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + window_length, + if periodic { 1 } else { 0 }, + alpha, + beta + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hamming_window_periodic_alpha_out( + out: &Tensor, + window_length: i64, + periodic: bool, + alpha: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hamming_window_periodic_alpha_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + window_length, + if periodic { 1 } else { 0 }, + alpha + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hamming_window_periodic_out( + out: &Tensor, + window_length: i64, + periodic: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hamming_window_periodic_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + window_length, + if periodic { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hann_window(window_length: i64, options: (Kind, Device)) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hann_window( + c_tensors.as_mut_ptr(), + window_length, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hann_window_out(out: &Tensor, window_length: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hann_window_out(c_tensors.as_mut_ptr(), out.c_tensor, window_length)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hann_window_periodic( + window_length: i64, + periodic: bool, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hann_window_periodic( + c_tensors.as_mut_ptr(), + window_length, + if periodic { 1 } else { 0 }, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hann_window_periodic_out( + out: &Tensor, + window_length: i64, + periodic: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hann_window_periodic_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + window_length, + if periodic { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hardshrink(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hardshrink(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hardshrink_backward>( + &self, + grad_out: &Tensor, + lambd: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hardshrink_backward( + c_tensors.as_mut_ptr(), + grad_out.c_tensor, + self.c_tensor, + lambd.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hardshrink_backward_grad_input>( + &self, + grad_input: &Tensor, + grad_out: &Tensor, + lambd: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hardshrink_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_out.c_tensor, + self.c_tensor, + lambd.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hardshrink_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hardshrink_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hardsigmoid(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hardsigmoid(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hardsigmoid_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hardsigmoid_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hardsigmoid_backward(&self, grad_output: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hardsigmoid_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hardsigmoid_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hardsigmoid_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hardsigmoid_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hardsigmoid_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hardswish(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hardswish(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hardswish_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hardswish_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hardswish_backward(&self, grad_output: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hardswish_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hardswish_backward_out( + &self, + out: &Tensor, + grad_output: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hardswish_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad_output.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hardswish_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hardswish_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hardtanh(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hardtanh(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hardtanh_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hardtanh_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hardtanh_backward>( + &self, + grad_output: &Tensor, + min_val: S, + max_val: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hardtanh_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + min_val.into().c_scalar, + max_val.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hardtanh_backward_grad_input>( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + min_val: S, + max_val: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hardtanh_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + min_val.into().c_scalar, + max_val.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hardtanh_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hardtanh_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hash_tensor( + &self, + dim: impl IntList, + keepdim: bool, + mode: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hash_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 }, + mode + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hash_tensor_out( + &self, + out: &Tensor, + dim: impl IntList, + keepdim: bool, + mode: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hash_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 }, + mode + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_heaviside(&self, values: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_heaviside(c_tensors.as_mut_ptr(), self.c_tensor, values.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_heaviside_(&mut self, values: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_heaviside_(c_tensors.as_mut_ptr(), self.c_tensor, values.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_heaviside_out(&self, out: &Tensor, values: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_heaviside_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + values.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hinge_embedding_loss( + &self, + target: &Tensor, + margin: f64, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hinge_embedding_loss( + c_tensors.as_mut_ptr(), + self.c_tensor, + target.c_tensor, + margin, + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_histc(&self, bins: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_histc(c_tensors.as_mut_ptr(), self.c_tensor, bins)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_histc_out(&self, out: &Tensor, bins: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_histc_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor, bins)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_histogram>( + &self, + bins: &Tensor, + weight: Option, + density: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_histogram( + c_tensors.as_mut_ptr(), + self.c_tensor, + bins.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if density { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_histogram_bin_ct>( + &self, + bins: i64, + range: impl DoubleList, + weight: Option, + density: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_histogram_bin_ct( + c_tensors.as_mut_ptr(), + self.c_tensor, + bins, + range.as_ptr(), + range.len_i32(), + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if density { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_histogram_bin_ct_out>( + &self, + hist: &Tensor, + bin_edges: &Tensor, + bins: i64, + range: impl DoubleList, + weight: Option, + density: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_histogram_bin_ct_out( + c_tensors.as_mut_ptr(), + hist.c_tensor, + bin_edges.c_tensor, + self.c_tensor, + bins, + range.as_ptr(), + range.len_i32(), + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if density { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_histogram_bins_tensor_out>( + &self, + hist: &Tensor, + bin_edges: &Tensor, + bins: &Tensor, + weight: Option, + density: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_histogram_bins_tensor_out( + c_tensors.as_mut_ptr(), + hist.c_tensor, + bin_edges.c_tensor, + self.c_tensor, + bins.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if density { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_hsplit(&self, sections: i64) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_hsplit(self.c_tensor, sections)); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_hsplit_array(&self, indices: impl IntList) -> Result, TchError> { + let c_tensors = + unsafe_torch_err!(atg_hsplit_array(self.c_tensor, indices.as_ptr(), indices.len_i32())); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_hspmm(mat1: &Tensor, mat2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hspmm(c_tensors.as_mut_ptr(), mat1.c_tensor, mat2.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hspmm_out(out: &Tensor, mat1: &Tensor, mat2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hspmm_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + mat1.c_tensor, + mat2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hstack>(tensors: &[T]) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hstack( + c_tensors.as_mut_ptr(), + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hstack_out>( + out: &Tensor, + tensors: &[T], + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hstack_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_huber_loss( + &self, + target: &Tensor, + reduction: crate::Reduction, + delta: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_huber_loss( + c_tensors.as_mut_ptr(), + self.c_tensor, + target.c_tensor, + reduction.to_int(), + delta + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_huber_loss_backward( + &self, + grad_output: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + delta: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_huber_loss_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + target.c_tensor, + reduction.to_int(), + delta + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_huber_loss_backward_out( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + delta: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_huber_loss_backward_out( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + target.c_tensor, + reduction.to_int(), + delta + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_huber_loss_out( + &self, + out: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + delta: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_huber_loss_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + target.c_tensor, + reduction.to_int(), + delta + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hypot(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hypot(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hypot_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hypot_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_hypot_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_hypot_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_i0(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_i0(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_i0_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_i0_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_i0_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_i0_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_igamma(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_igamma(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_igamma_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_igamma_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_igamma_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_igamma_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_igammac(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_igammac(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_igammac_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_igammac_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_igammac_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_igammac_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_im2col( + &self, + kernel_size: impl IntList, + dilation: impl IntList, + padding: impl IntList, + stride: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_im2col( + c_tensors.as_mut_ptr(), + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + padding.as_ptr(), + padding.len_i32(), + stride.as_ptr(), + stride.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_im2col_out( + &self, + out: &Tensor, + kernel_size: impl IntList, + dilation: impl IntList, + padding: impl IntList, + stride: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_im2col_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + padding.as_ptr(), + padding.len_i32(), + stride.as_ptr(), + stride.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_imag(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_imag(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index>(&self, indices: &[Option]) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index( + c_tensors.as_mut_ptr(), + self.c_tensor, + ptr_list_opt(indices).as_ptr(), + indices.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index_add( + &self, + dim: i64, + index: &Tensor, + source: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index_add( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + source.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index_add_( + &mut self, + dim: i64, + index: &Tensor, + source: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index_add_( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + source.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index_add_out( + &self, + out: &Tensor, + dim: i64, + index: &Tensor, + source: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index_add_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + index.c_tensor, + source.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index_copy( + &self, + dim: i64, + index: &Tensor, + source: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index_copy( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + source.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index_copy_( + &mut self, + dim: i64, + index: &Tensor, + source: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index_copy_( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + source.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index_copy_out( + &self, + out: &Tensor, + dim: i64, + index: &Tensor, + source: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + index.c_tensor, + source.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index_fill>( + &self, + dim: i64, + index: &Tensor, + value: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index_fill( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + value.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index_fill_>( + &mut self, + dim: i64, + index: &Tensor, + value: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index_fill_( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + value.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index_fill_int_scalar_out>( + &self, + out: &Tensor, + dim: i64, + index: &Tensor, + value: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index_fill_int_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + index.c_tensor, + value.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index_fill_int_tensor( + &self, + dim: i64, + index: &Tensor, + value: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index_fill_int_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + value.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index_fill_int_tensor_( + &mut self, + dim: i64, + index: &Tensor, + value: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index_fill_int_tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + value.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index_fill_int_tensor_out( + &self, + out: &Tensor, + dim: i64, + index: &Tensor, + value: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index_fill_int_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + index.c_tensor, + value.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index_put>( + &self, + indices: &[Option], + values: &Tensor, + accumulate: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index_put( + c_tensors.as_mut_ptr(), + self.c_tensor, + ptr_list_opt(indices).as_ptr(), + indices.len() as i32, + values.c_tensor, + if accumulate { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index_put_>( + &mut self, + indices: &[Option], + values: &Tensor, + accumulate: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index_put_( + c_tensors.as_mut_ptr(), + self.c_tensor, + ptr_list_opt(indices).as_ptr(), + indices.len() as i32, + values.c_tensor, + if accumulate { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index_put_out>( + &self, + out: &Tensor, + indices: &[Option], + values: &Tensor, + accumulate: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index_put_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + ptr_list_opt(indices).as_ptr(), + indices.len() as i32, + values.c_tensor, + if accumulate { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index_reduce( + &self, + dim: i64, + index: &Tensor, + source: &Tensor, + reduce: &str, + include_self: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index_reduce( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + source.c_tensor, + reduce.as_ptr(), + reduce.len() as i32, + if include_self { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index_reduce_( + &mut self, + dim: i64, + index: &Tensor, + source: &Tensor, + reduce: &str, + include_self: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index_reduce_( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + source.c_tensor, + reduce.as_ptr(), + reduce.len() as i32, + if include_self { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index_reduce_out( + &self, + out: &Tensor, + dim: i64, + index: &Tensor, + source: &Tensor, + reduce: &str, + include_self: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index_reduce_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + index.c_tensor, + source.c_tensor, + reduce.as_ptr(), + reduce.len() as i32, + if include_self { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index_select(&self, dim: i64, index: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index_select( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index_select_backward( + grad: &Tensor, + self_sizes: impl IntList, + dim: i64, + index: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index_select_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + self_sizes.as_ptr(), + self_sizes.len_i32(), + dim, + index.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index_select_out( + &self, + out: &Tensor, + dim: i64, + index: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index_select_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + index.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_index_tensor_out>( + &self, + out: &Tensor, + indices: &[Option], + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_index_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + ptr_list_opt(indices).as_ptr(), + indices.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_indices(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_indices(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_indices_copy(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_indices_copy(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_indices_copy_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_indices_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_infinitely_differentiable_gelu_backward( + &self, + grad: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_infinitely_differentiable_gelu_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_inner(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_inner(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_inner_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_inner_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_instance_norm>( + &self, + weight: Option, + bias: Option, + running_mean: Option, + running_var: Option, + use_input_stats: bool, + momentum: f64, + eps: f64, + cudnn_enabled: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_instance_norm( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if use_input_stats { 1 } else { 0 }, + momentum, + eps, + if cudnn_enabled { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_int_repr(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_int_repr(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_int_repr_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_int_repr_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_inverse(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_inverse(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_inverse_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_inverse_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_is_coalesced(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_is_coalesced(self.c_tensor)); + Ok(return_ != 0) + } + + pub fn f_is_complex(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_is_complex(self.c_tensor)); + Ok(return_ != 0) + } + + pub fn f_is_conj(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_is_conj(self.c_tensor)); + Ok(return_ != 0) + } + + pub fn f_is_distributed(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_is_distributed(self.c_tensor)); + Ok(return_ != 0) + } + + pub fn f_is_floating_point(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_is_floating_point(self.c_tensor)); + Ok(return_ != 0) + } + + pub fn f_is_inference(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_is_inference(self.c_tensor)); + Ok(return_ != 0) + } + + pub fn f_is_leaf(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_is_leaf(self.c_tensor)); + Ok(return_ != 0) + } + + pub fn f_is_neg(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_is_neg(self.c_tensor)); + Ok(return_ != 0) + } + + pub fn f_is_nonzero(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_is_nonzero(self.c_tensor)); + Ok(return_ != 0) + } + + pub fn f_is_pinned(&self, device: Device) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_is_pinned(self.c_tensor, device.c_int())); + Ok(return_ != 0) + } + + pub fn f_is_same_size(&self, other: &Tensor) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_is_same_size(self.c_tensor, other.c_tensor)); + Ok(return_ != 0) + } + + pub fn f_is_set_to(&self, tensor: &Tensor) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_is_set_to(self.c_tensor, tensor.c_tensor)); + Ok(return_ != 0) + } + + pub fn f_is_signed(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_is_signed(self.c_tensor)); + Ok(return_ != 0) + } + + pub fn f_is_vulkan_available() -> Result { + let return_; + unsafe_torch_err!(return_ = atg_is_vulkan_available()); + Ok(return_ != 0) + } + + pub fn f_isclose( + &self, + other: &Tensor, + rtol: f64, + atol: f64, + equal_nan: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_isclose( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor, + rtol, + atol, + if equal_nan { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_isfinite(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_isfinite(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_isin( + elements: &Tensor, + test_elements: &Tensor, + assume_unique: bool, + invert: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_isin( + c_tensors.as_mut_ptr(), + elements.c_tensor, + test_elements.c_tensor, + if assume_unique { 1 } else { 0 }, + if invert { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_isin_scalar_tensor>( + element: S, + test_elements: &Tensor, + assume_unique: bool, + invert: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_isin_scalar_tensor( + c_tensors.as_mut_ptr(), + element.into().c_scalar, + test_elements.c_tensor, + if assume_unique { 1 } else { 0 }, + if invert { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_isin_scalar_tensor_out>( + out: &Tensor, + element: S, + test_elements: &Tensor, + assume_unique: bool, + invert: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_isin_scalar_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + element.into().c_scalar, + test_elements.c_tensor, + if assume_unique { 1 } else { 0 }, + if invert { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_isin_tensor_scalar>( + elements: &Tensor, + test_element: S, + assume_unique: bool, + invert: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_isin_tensor_scalar( + c_tensors.as_mut_ptr(), + elements.c_tensor, + test_element.into().c_scalar, + if assume_unique { 1 } else { 0 }, + if invert { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_isin_tensor_scalar_out>( + out: &Tensor, + elements: &Tensor, + test_element: S, + assume_unique: bool, + invert: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_isin_tensor_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + elements.c_tensor, + test_element.into().c_scalar, + if assume_unique { 1 } else { 0 }, + if invert { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_isin_tensor_tensor_out( + out: &Tensor, + elements: &Tensor, + test_elements: &Tensor, + assume_unique: bool, + invert: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_isin_tensor_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + elements.c_tensor, + test_elements.c_tensor, + if assume_unique { 1 } else { 0 }, + if invert { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_isinf(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_isinf(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_isinf_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_isinf_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_isnan(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_isnan(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_isnan_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_isnan_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_isneginf(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_isneginf(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_isneginf_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_isneginf_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_isposinf(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_isposinf(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_isposinf_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_isposinf_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_isreal(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_isreal(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_istft>( + &self, + n_fft: i64, + hop_length: impl Into>, + win_length: impl Into>, + window: Option, + center: bool, + normalized: bool, + onesided: bool, + length: impl Into>, + return_complex: bool, + ) -> Result { + let hop_length = hop_length.into(); + let win_length = win_length.into(); + let length = length.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_istft( + c_tensors.as_mut_ptr(), + self.c_tensor, + n_fft, + hop_length.unwrap_or(0i64), + hop_length.is_none() as i8, + win_length.unwrap_or(0i64), + win_length.is_none() as i8, + window.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if center { 1 } else { 0 }, + if normalized { 1 } else { 0 }, + if onesided { 1 } else { 0 }, + length.unwrap_or(0i64), + length.is_none() as i8, + if return_complex { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_kaiser_window( + window_length: i64, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_kaiser_window( + c_tensors.as_mut_ptr(), + window_length, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_kaiser_window_beta( + window_length: i64, + periodic: bool, + beta: f64, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_kaiser_window_beta( + c_tensors.as_mut_ptr(), + window_length, + if periodic { 1 } else { 0 }, + beta, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_kaiser_window_beta_out( + out: &Tensor, + window_length: i64, + periodic: bool, + beta: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_kaiser_window_beta_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + window_length, + if periodic { 1 } else { 0 }, + beta + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_kaiser_window_out(out: &Tensor, window_length: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_kaiser_window_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + window_length + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_kaiser_window_periodic( + window_length: i64, + periodic: bool, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_kaiser_window_periodic( + c_tensors.as_mut_ptr(), + window_length, + if periodic { 1 } else { 0 }, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_kaiser_window_periodic_out( + out: &Tensor, + window_length: i64, + periodic: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_kaiser_window_periodic_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + window_length, + if periodic { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_kl_div( + &self, + target: &Tensor, + reduction: crate::Reduction, + log_target: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_kl_div( + c_tensors.as_mut_ptr(), + self.c_tensor, + target.c_tensor, + reduction.to_int(), + if log_target { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_kron(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_kron(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_kron_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_kron_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_kthvalue( + &self, + k: i64, + dim: i64, + keepdim: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_kthvalue( + c_tensors.as_mut_ptr(), + self.c_tensor, + k, + dim, + if keepdim { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_kthvalue_values( + &self, + values: &Tensor, + indices: &Tensor, + k: i64, + dim: i64, + keepdim: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_kthvalue_values( + c_tensors.as_mut_ptr(), + values.c_tensor, + indices.c_tensor, + self.c_tensor, + k, + dim, + if keepdim { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_l1_loss( + &self, + target: &Tensor, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_l1_loss( + c_tensors.as_mut_ptr(), + self.c_tensor, + target.c_tensor, + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_layer_norm>( + &self, + normalized_shape: impl IntList, + weight: Option, + bias: Option, + eps: f64, + cudnn_enable: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_layer_norm( + c_tensors.as_mut_ptr(), + self.c_tensor, + normalized_shape.as_ptr(), + normalized_shape.len_i32(), + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + eps, + if cudnn_enable { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lcm(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lcm(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lcm_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lcm_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lcm_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lcm_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ldexp(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ldexp(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ldexp_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ldexp_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ldexp_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ldexp_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_le>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_le(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_le_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_le_(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_le_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_le_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_le_tensor(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_le_tensor(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_le_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_le_tensor_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_le_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_le_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_leaky_relu(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_leaky_relu(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_leaky_relu_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_leaky_relu_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_leaky_relu_backward>( + &self, + grad_output: &Tensor, + negative_slope: S, + self_is_result: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_leaky_relu_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + negative_slope.into().c_scalar, + if self_is_result { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_leaky_relu_backward_grad_input>( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + negative_slope: S, + self_is_result: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_leaky_relu_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + negative_slope.into().c_scalar, + if self_is_result { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_leaky_relu_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_leaky_relu_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lerp>(&self, end: &Tensor, weight: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lerp( + c_tensors.as_mut_ptr(), + self.c_tensor, + end.c_tensor, + weight.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lerp_>( + &mut self, + end: &Tensor, + weight: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lerp_( + c_tensors.as_mut_ptr(), + self.c_tensor, + end.c_tensor, + weight.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lerp_scalar_out>( + &self, + out: &Tensor, + end: &Tensor, + weight: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lerp_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + end.c_tensor, + weight.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lerp_tensor(&self, end: &Tensor, weight: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lerp_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + end.c_tensor, + weight.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lerp_tensor_(&mut self, end: &Tensor, weight: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lerp_tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + end.c_tensor, + weight.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lerp_tensor_out( + &self, + out: &Tensor, + end: &Tensor, + weight: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lerp_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + end.c_tensor, + weight.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_less>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_less(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_less_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_less_(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_less_equal>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_less_equal( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_less_equal_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_less_equal_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_less_equal_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_less_equal_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_less_equal_tensor(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_less_equal_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_less_equal_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_less_equal_tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_less_equal_tensor_out( + &self, + out: &Tensor, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_less_equal_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_less_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_less_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_less_tensor(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_less_tensor(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_less_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_less_tensor_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_less_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_less_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lgamma(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lgamma(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lgamma_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lgamma_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lgamma_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lgamma_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lift(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lift(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lift_fresh(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lift_fresh(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lift_fresh_copy(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lift_fresh_copy(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lift_fresh_copy_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lift_fresh_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lift_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lift_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_cholesky(&self, upper: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_cholesky( + c_tensors.as_mut_ptr(), + self.c_tensor, + if upper { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_cholesky_ex( + &self, + upper: bool, + check_errors: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_linalg_cholesky_ex( + c_tensors.as_mut_ptr(), + self.c_tensor, + if upper { 1 } else { 0 }, + if check_errors { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_linalg_cholesky_ex_l( + &self, + l: &Tensor, + info: &Tensor, + upper: bool, + check_errors: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_linalg_cholesky_ex_l( + c_tensors.as_mut_ptr(), + l.c_tensor, + info.c_tensor, + self.c_tensor, + if upper { 1 } else { 0 }, + if check_errors { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_linalg_cholesky_out(&self, out: &Tensor, upper: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_cholesky_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + if upper { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_cond>(&self, p: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_cond( + c_tensors.as_mut_ptr(), + self.c_tensor, + p.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_cond_out>( + &self, + out: &Tensor, + p: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_cond_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + p.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_cond_p_str(&self, p: &str) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_cond_p_str( + c_tensors.as_mut_ptr(), + self.c_tensor, + p.as_ptr(), + p.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_cond_p_str_out(&self, out: &Tensor, p: &str) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_cond_p_str_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + p.as_ptr(), + p.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_cross(&self, other: &Tensor, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_cross( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_cross_out( + &self, + out: &Tensor, + other: &Tensor, + dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_cross_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_det(a: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_det(c_tensors.as_mut_ptr(), a.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_det_out(out: &Tensor, a: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_det_out(c_tensors.as_mut_ptr(), out.c_tensor, a.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_diagonal( + a: &Tensor, + offset: i64, + dim1: i64, + dim2: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_diagonal( + c_tensors.as_mut_ptr(), + a.c_tensor, + offset, + dim1, + dim2 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_eig(&self) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_linalg_eig(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_linalg_eig_out( + &self, + eigenvalues: &Tensor, + eigenvectors: &Tensor, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_linalg_eig_out( + c_tensors.as_mut_ptr(), + eigenvalues.c_tensor, + eigenvectors.c_tensor, + self.c_tensor + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_linalg_eigh(&self, uplo: &str) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_linalg_eigh( + c_tensors.as_mut_ptr(), + self.c_tensor, + uplo.as_ptr(), + uplo.len() as i32 + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_linalg_eigh_eigvals( + &self, + eigvals: &Tensor, + eigvecs: &Tensor, + uplo: &str, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_linalg_eigh_eigvals( + c_tensors.as_mut_ptr(), + eigvals.c_tensor, + eigvecs.c_tensor, + self.c_tensor, + uplo.as_ptr(), + uplo.len() as i32 + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_linalg_eigvals(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_eigvals(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_eigvals_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_eigvals_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_eigvalsh(&self, uplo: &str) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_eigvalsh( + c_tensors.as_mut_ptr(), + self.c_tensor, + uplo.as_ptr(), + uplo.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_eigvalsh_out(&self, out: &Tensor, uplo: &str) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_eigvalsh_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + uplo.as_ptr(), + uplo.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_householder_product(&self, tau: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_householder_product( + c_tensors.as_mut_ptr(), + self.c_tensor, + tau.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_householder_product_out( + &self, + out: &Tensor, + tau: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_householder_product_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + tau.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_inv(a: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_inv(c_tensors.as_mut_ptr(), a.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_inv_ex(a: &Tensor, check_errors: bool) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_linalg_inv_ex( + c_tensors.as_mut_ptr(), + a.c_tensor, + if check_errors { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_linalg_inv_ex_inverse( + inverse: &Tensor, + info: &Tensor, + a: &Tensor, + check_errors: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_linalg_inv_ex_inverse( + c_tensors.as_mut_ptr(), + inverse.c_tensor, + info.c_tensor, + a.c_tensor, + if check_errors { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_linalg_inv_out(out: &Tensor, a: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_inv_out(c_tensors.as_mut_ptr(), out.c_tensor, a.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_ldl_factor(&self, hermitian: bool) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_linalg_ldl_factor( + c_tensors.as_mut_ptr(), + self.c_tensor, + if hermitian { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_linalg_ldl_factor_ex( + &self, + hermitian: bool, + check_errors: bool, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_linalg_ldl_factor_ex( + c_tensors.as_mut_ptr(), + self.c_tensor, + if hermitian { 1 } else { 0 }, + if check_errors { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_linalg_ldl_factor_ex_out( + &self, + ld: &Tensor, + pivots: &Tensor, + info: &Tensor, + hermitian: bool, + check_errors: bool, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_linalg_ldl_factor_ex_out( + c_tensors.as_mut_ptr(), + ld.c_tensor, + pivots.c_tensor, + info.c_tensor, + self.c_tensor, + if hermitian { 1 } else { 0 }, + if check_errors { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_linalg_ldl_factor_out( + &self, + ld: &Tensor, + pivots: &Tensor, + hermitian: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_linalg_ldl_factor_out( + c_tensors.as_mut_ptr(), + ld.c_tensor, + pivots.c_tensor, + self.c_tensor, + if hermitian { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_linalg_ldl_solve( + ld: &Tensor, + pivots: &Tensor, + b: &Tensor, + hermitian: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_ldl_solve( + c_tensors.as_mut_ptr(), + ld.c_tensor, + pivots.c_tensor, + b.c_tensor, + if hermitian { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_ldl_solve_out( + out: &Tensor, + ld: &Tensor, + pivots: &Tensor, + b: &Tensor, + hermitian: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_ldl_solve_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + ld.c_tensor, + pivots.c_tensor, + b.c_tensor, + if hermitian { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_lstsq( + &self, + b: &Tensor, + rcond: impl Into>, + driver: &str, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let rcond = rcond.into(); + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg_linalg_lstsq( + c_tensors.as_mut_ptr(), + self.c_tensor, + b.c_tensor, + rcond.unwrap_or(std::f64::NAN), + rcond.is_none() as i8, + driver.as_ptr(), + driver.len() as i32 + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_linalg_lstsq_out( + &self, + solution: &Tensor, + residuals: &Tensor, + rank: &Tensor, + singular_values: &Tensor, + b: &Tensor, + rcond: impl Into>, + driver: &str, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let rcond = rcond.into(); + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg_linalg_lstsq_out( + c_tensors.as_mut_ptr(), + solution.c_tensor, + residuals.c_tensor, + rank.c_tensor, + singular_values.c_tensor, + self.c_tensor, + b.c_tensor, + rcond.unwrap_or(std::f64::NAN), + rcond.is_none() as i8, + driver.as_ptr(), + driver.len() as i32 + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_linalg_lu(a: &Tensor, pivot: bool) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_linalg_lu( + c_tensors.as_mut_ptr(), + a.c_tensor, + if pivot { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_linalg_lu_factor(a: &Tensor, pivot: bool) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_linalg_lu_factor( + c_tensors.as_mut_ptr(), + a.c_tensor, + if pivot { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_linalg_lu_factor_ex( + a: &Tensor, + pivot: bool, + check_errors: bool, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_linalg_lu_factor_ex( + c_tensors.as_mut_ptr(), + a.c_tensor, + if pivot { 1 } else { 0 }, + if check_errors { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_linalg_lu_factor_ex_out( + lu: &Tensor, + pivots: &Tensor, + info: &Tensor, + a: &Tensor, + pivot: bool, + check_errors: bool, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_linalg_lu_factor_ex_out( + c_tensors.as_mut_ptr(), + lu.c_tensor, + pivots.c_tensor, + info.c_tensor, + a.c_tensor, + if pivot { 1 } else { 0 }, + if check_errors { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_linalg_lu_factor_out( + lu: &Tensor, + pivots: &Tensor, + a: &Tensor, + pivot: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_linalg_lu_factor_out( + c_tensors.as_mut_ptr(), + lu.c_tensor, + pivots.c_tensor, + a.c_tensor, + if pivot { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_linalg_lu_out( + p: &Tensor, + l: &Tensor, + u: &Tensor, + a: &Tensor, + pivot: bool, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_linalg_lu_out( + c_tensors.as_mut_ptr(), + p.c_tensor, + l.c_tensor, + u.c_tensor, + a.c_tensor, + if pivot { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_linalg_lu_solve( + lu: &Tensor, + pivots: &Tensor, + b: &Tensor, + left: bool, + adjoint: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_lu_solve( + c_tensors.as_mut_ptr(), + lu.c_tensor, + pivots.c_tensor, + b.c_tensor, + if left { 1 } else { 0 }, + if adjoint { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_lu_solve_out( + out: &Tensor, + lu: &Tensor, + pivots: &Tensor, + b: &Tensor, + left: bool, + adjoint: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_lu_solve_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + lu.c_tensor, + pivots.c_tensor, + b.c_tensor, + if left { 1 } else { 0 }, + if adjoint { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_matmul(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_matmul(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_matmul_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_matmul_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_matrix_exp(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_matrix_exp(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_matrix_exp_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_matrix_exp_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_matrix_power(&self, n: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_matrix_power(c_tensors.as_mut_ptr(), self.c_tensor, n)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_matrix_power_out(&self, out: &Tensor, n: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_matrix_power_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + n + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_matrix_rank(&self, tol: f64, hermitian: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_matrix_rank( + c_tensors.as_mut_ptr(), + self.c_tensor, + tol, + if hermitian { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_matrix_rank_atol_rtol_float( + &self, + atol: impl Into>, + rtol: impl Into>, + hermitian: bool, + ) -> Result { + let atol = atol.into(); + let rtol = rtol.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_matrix_rank_atol_rtol_float( + c_tensors.as_mut_ptr(), + self.c_tensor, + atol.unwrap_or(std::f64::NAN), + atol.is_none() as i8, + rtol.unwrap_or(std::f64::NAN), + rtol.is_none() as i8, + if hermitian { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_matrix_rank_atol_rtol_float_out( + &self, + out: &Tensor, + atol: impl Into>, + rtol: impl Into>, + hermitian: bool, + ) -> Result { + let atol = atol.into(); + let rtol = rtol.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_matrix_rank_atol_rtol_float_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + atol.unwrap_or(std::f64::NAN), + atol.is_none() as i8, + rtol.unwrap_or(std::f64::NAN), + rtol.is_none() as i8, + if hermitian { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_matrix_rank_atol_rtol_tensor>( + &self, + atol: Option, + rtol: Option, + hermitian: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_matrix_rank_atol_rtol_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + atol.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + rtol.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if hermitian { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_matrix_rank_atol_rtol_tensor_out>( + &self, + out: &Tensor, + atol: Option, + rtol: Option, + hermitian: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_matrix_rank_atol_rtol_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + atol.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + rtol.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if hermitian { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_matrix_rank_out( + &self, + out: &Tensor, + tol: f64, + hermitian: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_matrix_rank_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + tol, + if hermitian { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_matrix_rank_out_tol_tensor( + &self, + out: &Tensor, + tol: &Tensor, + hermitian: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_matrix_rank_out_tol_tensor( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + tol.c_tensor, + if hermitian { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_matrix_rank_tol_tensor( + &self, + tol: &Tensor, + hermitian: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_matrix_rank_tol_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + tol.c_tensor, + if hermitian { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_multi_dot>(tensors: &[T]) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_multi_dot( + c_tensors.as_mut_ptr(), + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_multi_dot_out>( + out: &Tensor, + tensors: &[T], + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_multi_dot_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_norm>( + &self, + ord: S, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_norm( + c_tensors.as_mut_ptr(), + self.c_tensor, + ord.into().c_scalar, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 }, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_norm_ord_str( + &self, + ord: &str, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_norm_ord_str( + c_tensors.as_mut_ptr(), + self.c_tensor, + ord.as_ptr(), + ord.len() as i32, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 }, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_norm_ord_str_out( + &self, + out: &Tensor, + ord: &str, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_norm_ord_str_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + ord.as_ptr(), + ord.len() as i32, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 }, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_norm_out>( + &self, + out: &Tensor, + ord: S, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_norm_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + ord.into().c_scalar, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 }, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_pinv(&self, rcond: f64, hermitian: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_pinv( + c_tensors.as_mut_ptr(), + self.c_tensor, + rcond, + if hermitian { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_pinv_atol_rtol_float( + &self, + atol: impl Into>, + rtol: impl Into>, + hermitian: bool, + ) -> Result { + let atol = atol.into(); + let rtol = rtol.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_pinv_atol_rtol_float( + c_tensors.as_mut_ptr(), + self.c_tensor, + atol.unwrap_or(std::f64::NAN), + atol.is_none() as i8, + rtol.unwrap_or(std::f64::NAN), + rtol.is_none() as i8, + if hermitian { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_pinv_atol_rtol_float_out( + &self, + out: &Tensor, + atol: impl Into>, + rtol: impl Into>, + hermitian: bool, + ) -> Result { + let atol = atol.into(); + let rtol = rtol.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_pinv_atol_rtol_float_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + atol.unwrap_or(std::f64::NAN), + atol.is_none() as i8, + rtol.unwrap_or(std::f64::NAN), + rtol.is_none() as i8, + if hermitian { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_pinv_atol_rtol_tensor>( + &self, + atol: Option, + rtol: Option, + hermitian: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_pinv_atol_rtol_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + atol.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + rtol.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if hermitian { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_pinv_atol_rtol_tensor_out>( + &self, + out: &Tensor, + atol: Option, + rtol: Option, + hermitian: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_pinv_atol_rtol_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + atol.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + rtol.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if hermitian { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_pinv_out( + &self, + out: &Tensor, + rcond: f64, + hermitian: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_pinv_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + rcond, + if hermitian { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_pinv_out_rcond_tensor( + &self, + out: &Tensor, + rcond: &Tensor, + hermitian: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_pinv_out_rcond_tensor( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + rcond.c_tensor, + if hermitian { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_pinv_rcond_tensor( + &self, + rcond: &Tensor, + hermitian: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_pinv_rcond_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + rcond.c_tensor, + if hermitian { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_qr(a: &Tensor, mode: &str) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_linalg_qr( + c_tensors.as_mut_ptr(), + a.c_tensor, + mode.as_ptr(), + mode.len() as i32 + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_linalg_qr_out( + q: &Tensor, + r: &Tensor, + a: &Tensor, + mode: &str, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_linalg_qr_out( + c_tensors.as_mut_ptr(), + q.c_tensor, + r.c_tensor, + a.c_tensor, + mode.as_ptr(), + mode.len() as i32 + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_linalg_slogdet(a: &Tensor) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_linalg_slogdet(c_tensors.as_mut_ptr(), a.c_tensor)); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_linalg_slogdet_out( + sign: &Tensor, + logabsdet: &Tensor, + a: &Tensor, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_linalg_slogdet_out( + c_tensors.as_mut_ptr(), + sign.c_tensor, + logabsdet.c_tensor, + a.c_tensor + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_linalg_solve(a: &Tensor, b: &Tensor, left: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_solve( + c_tensors.as_mut_ptr(), + a.c_tensor, + b.c_tensor, + if left { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_solve_ex( + a: &Tensor, + b: &Tensor, + left: bool, + check_errors: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_linalg_solve_ex( + c_tensors.as_mut_ptr(), + a.c_tensor, + b.c_tensor, + if left { 1 } else { 0 }, + if check_errors { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_linalg_solve_ex_out( + result: &Tensor, + info: &Tensor, + a: &Tensor, + b: &Tensor, + left: bool, + check_errors: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_linalg_solve_ex_out( + c_tensors.as_mut_ptr(), + result.c_tensor, + info.c_tensor, + a.c_tensor, + b.c_tensor, + if left { 1 } else { 0 }, + if check_errors { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_linalg_solve_out( + out: &Tensor, + a: &Tensor, + b: &Tensor, + left: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_solve_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + a.c_tensor, + b.c_tensor, + if left { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_solve_triangular( + &self, + b: &Tensor, + upper: bool, + left: bool, + unitriangular: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_solve_triangular( + c_tensors.as_mut_ptr(), + self.c_tensor, + b.c_tensor, + if upper { 1 } else { 0 }, + if left { 1 } else { 0 }, + if unitriangular { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_solve_triangular_out( + &self, + out: &Tensor, + b: &Tensor, + upper: bool, + left: bool, + unitriangular: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_solve_triangular_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + b.c_tensor, + if upper { 1 } else { 0 }, + if left { 1 } else { 0 }, + if unitriangular { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_svd( + a: &Tensor, + full_matrices: bool, + driver: &str, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_linalg_svd( + c_tensors.as_mut_ptr(), + a.c_tensor, + if full_matrices { 1 } else { 0 }, + driver.as_ptr(), + driver.len() as i32 + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_linalg_svd_u( + u: &Tensor, + s: &Tensor, + vh: &Tensor, + a: &Tensor, + full_matrices: bool, + driver: &str, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_linalg_svd_u( + c_tensors.as_mut_ptr(), + u.c_tensor, + s.c_tensor, + vh.c_tensor, + a.c_tensor, + if full_matrices { 1 } else { 0 }, + driver.as_ptr(), + driver.len() as i32 + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_linalg_svdvals(a: &Tensor, driver: &str) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_svdvals( + c_tensors.as_mut_ptr(), + a.c_tensor, + driver.as_ptr(), + driver.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_svdvals_out( + out: &Tensor, + a: &Tensor, + driver: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_svdvals_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + a.c_tensor, + driver.as_ptr(), + driver.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_tensorinv(&self, ind: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_tensorinv(c_tensors.as_mut_ptr(), self.c_tensor, ind)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_tensorinv_out(&self, out: &Tensor, ind: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_tensorinv_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + ind + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_tensorsolve( + &self, + other: &Tensor, + dims: impl IntListOption, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_tensorsolve( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor, + dims.as_ptr(), + dims.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_tensorsolve_out( + &self, + out: &Tensor, + other: &Tensor, + dims: impl IntListOption, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_tensorsolve_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor, + dims.as_ptr(), + dims.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_vander(x: &Tensor, n: impl Into>) -> Result { + let n = n.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_vander( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.unwrap_or(0i64), + n.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_vecdot(x: &Tensor, y: &Tensor, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_vecdot(c_tensors.as_mut_ptr(), x.c_tensor, y.c_tensor, dim)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linalg_vecdot_out( + out: &Tensor, + x: &Tensor, + y: &Tensor, + dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linalg_vecdot_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + y.c_tensor, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linear>( + &self, + weight: &Tensor, + bias: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linear( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linear_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linear_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linspace>( + start: S, + end: S, + steps: i64, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linspace( + c_tensors.as_mut_ptr(), + start.into().c_scalar, + end.into().c_scalar, + steps, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linspace_out>( + out: &Tensor, + start: S, + end: S, + steps: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linspace_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + start.into().c_scalar, + end.into().c_scalar, + steps + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linspace_scalar_tensor>( + start: S, + end: &Tensor, + steps: i64, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linspace_scalar_tensor( + c_tensors.as_mut_ptr(), + start.into().c_scalar, + end.c_tensor, + steps, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linspace_scalar_tensor_out>( + out: &Tensor, + start: S, + end: &Tensor, + steps: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linspace_scalar_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + start.into().c_scalar, + end.c_tensor, + steps + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linspace_tensor_scalar>( + start: &Tensor, + end: S, + steps: i64, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linspace_tensor_scalar( + c_tensors.as_mut_ptr(), + start.c_tensor, + end.into().c_scalar, + steps, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linspace_tensor_scalar_out>( + out: &Tensor, + start: &Tensor, + end: S, + steps: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linspace_tensor_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + start.c_tensor, + end.into().c_scalar, + steps + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linspace_tensor_tensor( + start: &Tensor, + end: &Tensor, + steps: i64, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linspace_tensor_tensor( + c_tensors.as_mut_ptr(), + start.c_tensor, + end.c_tensor, + steps, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_linspace_tensor_tensor_out( + out: &Tensor, + start: &Tensor, + end: &Tensor, + steps: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_linspace_tensor_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + start.c_tensor, + end.c_tensor, + steps + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_log(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_log(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_log10(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_log10(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_log10_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_log10_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_log10_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_log10_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_log1p(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_log1p(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_log1p_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_log1p_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_log1p_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_log1p_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_log2(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_log2(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_log2_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_log2_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_log2_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_log2_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_log_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_log_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_log_normal(&self, mean: f64, std: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_log_normal(c_tensors.as_mut_ptr(), self.c_tensor, mean, std)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_log_normal_(&mut self, mean: f64, std: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_log_normal_(c_tensors.as_mut_ptr(), self.c_tensor, mean, std)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_log_normal_out(&self, out: &Tensor, mean: f64, std: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_log_normal_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mean, + std + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_log_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_log_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_log_sigmoid(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_log_sigmoid(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_log_sigmoid_backward( + &self, + grad_output: &Tensor, + buffer: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_log_sigmoid_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + buffer.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_log_sigmoid_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + buffer: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_log_sigmoid_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + buffer.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_log_sigmoid_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_log_sigmoid_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_log_softmax( + &self, + dim: i64, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_log_softmax( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_log_softmax_int_out( + &self, + out: &Tensor, + dim: i64, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_log_softmax_int_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logaddexp(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logaddexp(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logaddexp2(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logaddexp2(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logaddexp2_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logaddexp2_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logaddexp_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logaddexp_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logcumsumexp(&self, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logcumsumexp(c_tensors.as_mut_ptr(), self.c_tensor, dim)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logcumsumexp_out(&self, out: &Tensor, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logcumsumexp_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logdet(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logdet(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logical_and(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logical_and(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logical_and_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logical_and_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logical_and_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logical_and_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logical_not(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logical_not(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logical_not_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logical_not_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logical_not_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logical_not_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logical_or(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logical_or(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logical_or_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logical_or_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logical_or_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logical_or_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logical_xor(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logical_xor(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logical_xor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logical_xor_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logical_xor_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logical_xor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logit(&self, eps: impl Into>) -> Result { + let eps = eps.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logit( + c_tensors.as_mut_ptr(), + self.c_tensor, + eps.unwrap_or(std::f64::NAN), + eps.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logit_(&mut self, eps: impl Into>) -> Result { + let eps = eps.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logit_( + c_tensors.as_mut_ptr(), + self.c_tensor, + eps.unwrap_or(std::f64::NAN), + eps.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logit_backward( + &self, + grad_output: &Tensor, + eps: impl Into>, + ) -> Result { + let eps = eps.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logit_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + eps.unwrap_or(std::f64::NAN), + eps.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logit_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + eps: impl Into>, + ) -> Result { + let eps = eps.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logit_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + eps.unwrap_or(std::f64::NAN), + eps.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logit_out( + &self, + out: &Tensor, + eps: impl Into>, + ) -> Result { + let eps = eps.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logit_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + eps.unwrap_or(std::f64::NAN), + eps.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logspace>( + start: S, + end: S, + steps: i64, + base: f64, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logspace( + c_tensors.as_mut_ptr(), + start.into().c_scalar, + end.into().c_scalar, + steps, + base, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logspace_out>( + out: &Tensor, + start: S, + end: S, + steps: i64, + base: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logspace_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + start.into().c_scalar, + end.into().c_scalar, + steps, + base + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logspace_scalar_tensor>( + start: S, + end: &Tensor, + steps: i64, + base: f64, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logspace_scalar_tensor( + c_tensors.as_mut_ptr(), + start.into().c_scalar, + end.c_tensor, + steps, + base, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logspace_scalar_tensor_out>( + out: &Tensor, + start: S, + end: &Tensor, + steps: i64, + base: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logspace_scalar_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + start.into().c_scalar, + end.c_tensor, + steps, + base + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logspace_tensor_scalar>( + start: &Tensor, + end: S, + steps: i64, + base: f64, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logspace_tensor_scalar( + c_tensors.as_mut_ptr(), + start.c_tensor, + end.into().c_scalar, + steps, + base, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logspace_tensor_scalar_out>( + out: &Tensor, + start: &Tensor, + end: S, + steps: i64, + base: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logspace_tensor_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + start.c_tensor, + end.into().c_scalar, + steps, + base + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logspace_tensor_tensor( + start: &Tensor, + end: &Tensor, + steps: i64, + base: f64, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logspace_tensor_tensor( + c_tensors.as_mut_ptr(), + start.c_tensor, + end.c_tensor, + steps, + base, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logspace_tensor_tensor_out( + out: &Tensor, + start: &Tensor, + end: &Tensor, + steps: i64, + base: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logspace_tensor_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + start.c_tensor, + end.c_tensor, + steps, + base + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logsumexp(&self, dim: impl IntList, keepdim: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logsumexp( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_logsumexp_out( + &self, + out: &Tensor, + dim: impl IntList, + keepdim: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_logsumexp_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lstm>( + &self, + hx: &[T], + params: &[T], + has_biases: bool, + num_layers: i64, + dropout: f64, + train: bool, + bidirectional: bool, + batch_first: bool, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_lstm( + c_tensors.as_mut_ptr(), + self.c_tensor, + ptr_list(hx).as_ptr(), + hx.len() as i32, + ptr_list(params).as_ptr(), + params.len() as i32, + if has_biases { 1 } else { 0 }, + num_layers, + dropout, + if train { 1 } else { 0 }, + if bidirectional { 1 } else { 0 }, + if batch_first { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_lstm_cell>( + &self, + hx: &[T], + w_ih: &Tensor, + w_hh: &Tensor, + b_ih: Option, + b_hh: Option, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_lstm_cell( + c_tensors.as_mut_ptr(), + self.c_tensor, + ptr_list(hx).as_ptr(), + hx.len() as i32, + w_ih.c_tensor, + w_hh.c_tensor, + b_ih.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + b_hh.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_lstm_data>( + data: &Tensor, + batch_sizes: &Tensor, + hx: &[T], + params: &[T], + has_biases: bool, + num_layers: i64, + dropout: f64, + train: bool, + bidirectional: bool, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_lstm_data( + c_tensors.as_mut_ptr(), + data.c_tensor, + batch_sizes.c_tensor, + ptr_list(hx).as_ptr(), + hx.len() as i32, + ptr_list(params).as_ptr(), + params.len() as i32, + if has_biases { 1 } else { 0 }, + num_layers, + dropout, + if train { 1 } else { 0 }, + if bidirectional { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_lstm_mps_backward>( + &self, + out0: &Tensor, + out1: &[T], + out2: &[T], + grad_y: Option, + grad_hy: Option, + grad_cy: Option, + z_state: &Tensor, + cell_state_fwd: &Tensor, + layersoutputs: &Tensor, + hx: &[T], + params: &[T], + has_biases: bool, + num_layers: i64, + dropout: f64, + train: bool, + bidirectional: bool, + batch_first: bool, + ) -> Result<(), TchError> { + unsafe_torch_err!(atg_lstm_mps_backward( + out0.c_tensor, + ptr_list(out1).as_ptr(), + out1.len() as i32, + ptr_list(out2).as_ptr(), + out2.len() as i32, + grad_y.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + grad_hy.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + grad_cy.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + z_state.c_tensor, + cell_state_fwd.c_tensor, + self.c_tensor, + layersoutputs.c_tensor, + ptr_list(hx).as_ptr(), + hx.len() as i32, + ptr_list(params).as_ptr(), + params.len() as i32, + if has_biases { 1 } else { 0 }, + num_layers, + dropout, + if train { 1 } else { 0 }, + if bidirectional { 1 } else { 0 }, + if batch_first { 1 } else { 0 } + )); + Ok(()) + } + + pub fn f_lt>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lt(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lt_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lt_(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lt_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lt_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lt_tensor(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lt_tensor(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lt_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lt_tensor_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lt_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lt_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lu_solve(&self, lu_data: &Tensor, lu_pivots: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lu_solve( + c_tensors.as_mut_ptr(), + self.c_tensor, + lu_data.c_tensor, + lu_pivots.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lu_solve_out( + &self, + out: &Tensor, + lu_data: &Tensor, + lu_pivots: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_lu_solve_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + lu_data.c_tensor, + lu_pivots.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_lu_unpack( + lu_data: &Tensor, + lu_pivots: &Tensor, + unpack_data: bool, + unpack_pivots: bool, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_lu_unpack( + c_tensors.as_mut_ptr(), + lu_data.c_tensor, + lu_pivots.c_tensor, + if unpack_data { 1 } else { 0 }, + if unpack_pivots { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_lu_unpack_out( + p: &Tensor, + l: &Tensor, + u: &Tensor, + lu_data: &Tensor, + lu_pivots: &Tensor, + unpack_data: bool, + unpack_pivots: bool, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_lu_unpack_out( + c_tensors.as_mut_ptr(), + p.c_tensor, + l.c_tensor, + u.c_tensor, + lu_data.c_tensor, + lu_pivots.c_tensor, + if unpack_data { 1 } else { 0 }, + if unpack_pivots { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_margin_ranking_loss( + input1: &Tensor, + input2: &Tensor, + target: &Tensor, + margin: f64, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_margin_ranking_loss( + c_tensors.as_mut_ptr(), + input1.c_tensor, + input2.c_tensor, + target.c_tensor, + margin, + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_masked_fill>( + &self, + mask: &Tensor, + value: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_masked_fill( + c_tensors.as_mut_ptr(), + self.c_tensor, + mask.c_tensor, + value.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_masked_fill_>( + &mut self, + mask: &Tensor, + value: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_masked_fill_( + c_tensors.as_mut_ptr(), + self.c_tensor, + mask.c_tensor, + value.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_masked_fill_scalar_out>( + &self, + out: &Tensor, + mask: &Tensor, + value: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_masked_fill_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mask.c_tensor, + value.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_masked_fill_tensor(&self, mask: &Tensor, value: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_masked_fill_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + mask.c_tensor, + value.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_masked_fill_tensor_( + &mut self, + mask: &Tensor, + value: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_masked_fill_tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + mask.c_tensor, + value.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_masked_fill_tensor_out( + &self, + out: &Tensor, + mask: &Tensor, + value: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_masked_fill_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mask.c_tensor, + value.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_masked_scatter(&self, mask: &Tensor, source: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_masked_scatter( + c_tensors.as_mut_ptr(), + self.c_tensor, + mask.c_tensor, + source.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_masked_scatter_( + &mut self, + mask: &Tensor, + source: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_masked_scatter_( + c_tensors.as_mut_ptr(), + self.c_tensor, + mask.c_tensor, + source.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_masked_scatter_backward( + grad_output: &Tensor, + mask: &Tensor, + sizes: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_masked_scatter_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + mask.c_tensor, + sizes.as_ptr(), + sizes.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_masked_scatter_out( + &self, + out: &Tensor, + mask: &Tensor, + source: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_masked_scatter_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mask.c_tensor, + source.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_masked_select(&self, mask: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_masked_select(c_tensors.as_mut_ptr(), self.c_tensor, mask.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_masked_select_backward( + &self, + grad: &Tensor, + mask: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_masked_select_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + self.c_tensor, + mask.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_masked_select_out(&self, out: &Tensor, mask: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_masked_select_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mask.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_matmul(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_matmul(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_matmul_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_matmul_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_matrix_exp(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_matrix_exp(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_matrix_exp_backward(&self, grad: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_matrix_exp_backward( + c_tensors.as_mut_ptr(), + self.c_tensor, + grad.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_matrix_h(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_matrix_h(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_matrix_power(&self, n: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_matrix_power(c_tensors.as_mut_ptr(), self.c_tensor, n)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_matrix_power_out(&self, out: &Tensor, n: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_matrix_power_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + n + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_max(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_max(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_max_dim(&self, dim: i64, keepdim: bool) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_max_dim( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + if keepdim { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_max_dim_max( + &self, + max: &Tensor, + max_values: &Tensor, + dim: i64, + keepdim: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_max_dim_max( + c_tensors.as_mut_ptr(), + max.c_tensor, + max_values.c_tensor, + self.c_tensor, + dim, + if keepdim { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_max_other(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_max_other(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_max_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_max_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_max_pool1d( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_max_pool1d( + c_tensors.as_mut_ptr(), + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_max_pool1d_with_indices( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_max_pool1d_with_indices( + c_tensors.as_mut_ptr(), + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_max_pool2d( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_max_pool2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_max_pool2d_backward( + &self, + grad_output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_max_pool2d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_max_pool2d_backward_out( + &self, + out: &Tensor, + grad_output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_max_pool2d_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad_output.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_max_pool2d_with_indices( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_max_pool2d_with_indices( + c_tensors.as_mut_ptr(), + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_max_pool2d_with_indices_backward( + &self, + grad_output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + indices: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_max_pool2d_with_indices_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 }, + indices.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_max_pool2d_with_indices_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + indices: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_max_pool2d_with_indices_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 }, + indices.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_max_pool2d_with_indices_out( + &self, + out: &Tensor, + indices: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_max_pool2d_with_indices_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + indices.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_max_pool3d( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_max_pool3d( + c_tensors.as_mut_ptr(), + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_max_pool3d_with_indices( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_max_pool3d_with_indices( + c_tensors.as_mut_ptr(), + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_max_pool3d_with_indices_backward( + &self, + grad_output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + indices: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_max_pool3d_with_indices_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 }, + indices.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_max_pool3d_with_indices_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + indices: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_max_pool3d_with_indices_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 }, + indices.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_max_pool3d_with_indices_out( + &self, + out: &Tensor, + indices: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_max_pool3d_with_indices_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + indices.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_max_unary_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_max_unary_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_max_unpool2d( + &self, + indices: &Tensor, + output_size: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_max_unpool2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + indices.c_tensor, + output_size.as_ptr(), + output_size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_max_unpool2d_out( + &self, + out: &Tensor, + indices: &Tensor, + output_size: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_max_unpool2d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + indices.c_tensor, + output_size.as_ptr(), + output_size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_max_unpool3d( + &self, + indices: &Tensor, + output_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_max_unpool3d( + c_tensors.as_mut_ptr(), + self.c_tensor, + indices.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_max_unpool3d_out( + &self, + out: &Tensor, + indices: &Tensor, + output_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_max_unpool3d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + indices.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_maximum(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_maximum(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_maximum_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_maximum_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mean(&self, dtype: impl Into>) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mean( + c_tensors.as_mut_ptr(), + self.c_tensor, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mean_dim( + &self, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mean_dim( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 }, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mean_dtype_out( + &self, + out: &Tensor, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mean_dtype_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mean_out( + &self, + out: &Tensor, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mean_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 }, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_median(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_median(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_median_dim(&self, dim: i64, keepdim: bool) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_median_dim( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + if keepdim { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_median_dim_values( + &self, + values: &Tensor, + indices: &Tensor, + dim: i64, + keepdim: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_median_dim_values( + c_tensors.as_mut_ptr(), + values.c_tensor, + indices.c_tensor, + self.c_tensor, + dim, + if keepdim { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_median_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_median_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_meshgrid>(tensors: &[T]) -> Result, TchError> { + let c_tensors = + unsafe_torch_err!(atg_meshgrid(ptr_list(tensors).as_ptr(), tensors.len() as i32)); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_meshgrid_indexing>( + tensors: &[T], + indexing: &str, + ) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_meshgrid_indexing( + ptr_list(tensors).as_ptr(), + tensors.len() as i32, + indexing.as_ptr(), + indexing.len() as i32 + )); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_mh(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mh(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_min(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_min(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_min_dim(&self, dim: i64, keepdim: bool) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_min_dim( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + if keepdim { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_min_dim_min( + &self, + min: &Tensor, + min_indices: &Tensor, + dim: i64, + keepdim: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_min_dim_min( + c_tensors.as_mut_ptr(), + min.c_tensor, + min_indices.c_tensor, + self.c_tensor, + dim, + if keepdim { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_min_other(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_min_other(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_min_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_min_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_min_unary_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_min_unary_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_minimum(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_minimum(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_minimum_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_minimum_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_miopen_batch_norm>( + &self, + weight: &Tensor, + bias: Option, + running_mean: Option, + running_var: Option, + training: bool, + exponential_average_factor: f64, + epsilon: f64, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_miopen_batch_norm( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if training { 1 } else { 0 }, + exponential_average_factor, + epsilon + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_miopen_batch_norm_backward>( + &self, + grad_output: &Tensor, + weight: &Tensor, + running_mean: Option, + running_var: Option, + save_mean: Option, + save_var: Option, + epsilon: f64, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_miopen_batch_norm_backward( + c_tensors.as_mut_ptr(), + self.c_tensor, + grad_output.c_tensor, + weight.c_tensor, + running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + save_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + save_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + epsilon + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_miopen_batch_norm_backward_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + grad_output: &Tensor, + weight: &Tensor, + running_mean: Option, + running_var: Option, + save_mean: Option, + save_var: Option, + epsilon: f64, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_miopen_batch_norm_backward_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + out2.c_tensor, + self.c_tensor, + grad_output.c_tensor, + weight.c_tensor, + running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + save_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + save_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + epsilon + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_miopen_batch_norm_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + weight: &Tensor, + bias: Option, + running_mean: Option, + running_var: Option, + training: bool, + exponential_average_factor: f64, + epsilon: f64, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_miopen_batch_norm_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + out2.c_tensor, + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if training { 1 } else { 0 }, + exponential_average_factor, + epsilon + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_miopen_convolution>( + &self, + weight: &Tensor, + bias: Option, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_miopen_convolution( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + padding.as_ptr(), + padding.len_i32(), + stride.as_ptr(), + stride.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups, + if benchmark { 1 } else { 0 }, + if deterministic { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_miopen_convolution_add_relu, S: Into>( + &self, + weight: &Tensor, + z: &Tensor, + alpha: S, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_miopen_convolution_add_relu( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + z.c_tensor, + alpha.into().c_scalar, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_miopen_convolution_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_miopen_convolution_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + padding.as_ptr(), + padding.len_i32(), + stride.as_ptr(), + stride.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups, + if benchmark { 1 } else { 0 }, + if deterministic { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_miopen_convolution_relu>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_miopen_convolution_relu( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_miopen_convolution_transpose>( + &self, + weight: &Tensor, + bias: Option, + padding: impl IntList, + output_padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_miopen_convolution_transpose( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + padding.as_ptr(), + padding.len_i32(), + output_padding.as_ptr(), + output_padding.len_i32(), + stride.as_ptr(), + stride.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups, + if benchmark { 1 } else { 0 }, + if deterministic { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_miopen_convolution_transpose_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + padding: impl IntList, + output_padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_miopen_convolution_transpose_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + padding.as_ptr(), + padding.len_i32(), + output_padding.as_ptr(), + output_padding.len_i32(), + stride.as_ptr(), + stride.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups, + if benchmark { 1 } else { 0 }, + if deterministic { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_miopen_ctc_loss( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: impl IntList, + target_lengths: impl IntList, + blank: i64, + deterministic: bool, + zero_infinity: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_miopen_ctc_loss( + c_tensors.as_mut_ptr(), + log_probs.c_tensor, + targets.c_tensor, + input_lengths.as_ptr(), + input_lengths.len_i32(), + target_lengths.as_ptr(), + target_lengths.len_i32(), + blank, + if deterministic { 1 } else { 0 }, + if zero_infinity { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_miopen_ctc_loss_out( + out0: &Tensor, + out1: &Tensor, + log_probs: &Tensor, + targets: &Tensor, + input_lengths: impl IntList, + target_lengths: impl IntList, + blank: i64, + deterministic: bool, + zero_infinity: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_miopen_ctc_loss_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + log_probs.c_tensor, + targets.c_tensor, + input_lengths.as_ptr(), + input_lengths.len_i32(), + target_lengths.as_ptr(), + target_lengths.len_i32(), + blank, + if deterministic { 1 } else { 0 }, + if zero_infinity { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_miopen_ctc_loss_tensor( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: &Tensor, + target_lengths: &Tensor, + blank: i64, + deterministic: bool, + zero_infinity: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_miopen_ctc_loss_tensor( + c_tensors.as_mut_ptr(), + log_probs.c_tensor, + targets.c_tensor, + input_lengths.c_tensor, + target_lengths.c_tensor, + blank, + if deterministic { 1 } else { 0 }, + if zero_infinity { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_miopen_depthwise_convolution>( + &self, + weight: &Tensor, + bias: Option, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_miopen_depthwise_convolution( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + padding.as_ptr(), + padding.len_i32(), + stride.as_ptr(), + stride.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups, + if benchmark { 1 } else { 0 }, + if deterministic { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_miopen_depthwise_convolution_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_miopen_depthwise_convolution_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + padding.as_ptr(), + padding.len_i32(), + stride.as_ptr(), + stride.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups, + if benchmark { 1 } else { 0 }, + if deterministic { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_miopen_rnn>( + &self, + weight: &[T], + weight_stride0: i64, + hx: &Tensor, + cx: Option, + mode: i64, + hidden_size: i64, + num_layers: i64, + batch_first: bool, + dropout: f64, + train: bool, + bidirectional: bool, + batch_sizes: impl IntList, + dropout_state: Option, + ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 5]; + unsafe_torch_err!(atg_miopen_rnn( + c_tensors.as_mut_ptr(), + self.c_tensor, + ptr_list(weight).as_ptr(), + weight.len() as i32, + weight_stride0, + hx.c_tensor, + cx.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + mode, + hidden_size, + num_layers, + if batch_first { 1 } else { 0 }, + dropout, + if train { 1 } else { 0 }, + if bidirectional { 1 } else { 0 }, + batch_sizes.as_ptr(), + batch_sizes.len_i32(), + dropout_state.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + Tensor { c_tensor: c_tensors[4] }, + )) + } + + pub fn f_miopen_rnn_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + out3: &Tensor, + out4: &Tensor, + weight: &[T], + weight_stride0: i64, + hx: &Tensor, + cx: Option, + mode: i64, + hidden_size: i64, + num_layers: i64, + batch_first: bool, + dropout: f64, + train: bool, + bidirectional: bool, + batch_sizes: impl IntList, + dropout_state: Option, + ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 5]; + unsafe_torch_err!(atg_miopen_rnn_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + out2.c_tensor, + out3.c_tensor, + out4.c_tensor, + self.c_tensor, + ptr_list(weight).as_ptr(), + weight.len() as i32, + weight_stride0, + hx.c_tensor, + cx.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + mode, + hidden_size, + num_layers, + if batch_first { 1 } else { 0 }, + dropout, + if train { 1 } else { 0 }, + if bidirectional { 1 } else { 0 }, + batch_sizes.as_ptr(), + batch_sizes.len_i32(), + dropout_state.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + Tensor { c_tensor: c_tensors[4] }, + )) + } + + pub fn f_mish(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mish(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mish_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mish_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mish_backward(&self, grad_output: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mish_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mish_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mish_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_adaptive_avg_pool2d( + &self, + output_size: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mkldnn_adaptive_avg_pool2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_adaptive_avg_pool2d_backward( + &self, + grad_output: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mkldnn_adaptive_avg_pool2d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_adaptive_avg_pool2d_backward_out( + &self, + out: &Tensor, + grad_output: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mkldnn_adaptive_avg_pool2d_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad_output.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_adaptive_avg_pool2d_out( + &self, + out: &Tensor, + output_size: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mkldnn_adaptive_avg_pool2d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_convolution>( + &self, + weight: &Tensor, + bias: Option, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mkldnn_convolution( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + padding.as_ptr(), + padding.len_i32(), + stride.as_ptr(), + stride.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_convolution_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mkldnn_convolution_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + padding.as_ptr(), + padding.len_i32(), + stride.as_ptr(), + stride.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_linear>( + &self, + weight: &Tensor, + bias: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mkldnn_linear( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_linear_backward_input( + input_size: impl IntList, + grad_output: &Tensor, + weight: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mkldnn_linear_backward_input( + c_tensors.as_mut_ptr(), + input_size.as_ptr(), + input_size.len_i32(), + grad_output.c_tensor, + weight.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_linear_backward_input_out( + out: &Tensor, + input_size: impl IntList, + grad_output: &Tensor, + weight: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mkldnn_linear_backward_input_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + input_size.as_ptr(), + input_size.len_i32(), + grad_output.c_tensor, + weight.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_linear_backward_weights( + &self, + grad_output: &Tensor, + weight: &Tensor, + bias_defined: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_mkldnn_linear_backward_weights( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + weight.c_tensor, + if bias_defined { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_mkldnn_linear_backward_weights_out( + &self, + out0: &Tensor, + out1: &Tensor, + grad_output: &Tensor, + weight: &Tensor, + bias_defined: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_mkldnn_linear_backward_weights_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + grad_output.c_tensor, + self.c_tensor, + weight.c_tensor, + if bias_defined { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_mkldnn_linear_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mkldnn_linear_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_max_pool2d( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mkldnn_max_pool2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_max_pool2d_backward( + &self, + grad_output: &Tensor, + output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mkldnn_max_pool2d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + output.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_max_pool2d_backward_out( + &self, + out: &Tensor, + grad_output: &Tensor, + output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mkldnn_max_pool2d_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad_output.c_tensor, + output.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_max_pool2d_out( + &self, + out: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mkldnn_max_pool2d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_max_pool3d( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mkldnn_max_pool3d( + c_tensors.as_mut_ptr(), + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_max_pool3d_backward( + &self, + grad_output: &Tensor, + output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mkldnn_max_pool3d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + output.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_max_pool3d_backward_out( + &self, + out: &Tensor, + grad_output: &Tensor, + output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mkldnn_max_pool3d_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad_output.c_tensor, + output.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_max_pool3d_out( + &self, + out: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mkldnn_max_pool3d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_reorder_conv2d_weight( + &self, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + input_size: impl IntListOption, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mkldnn_reorder_conv2d_weight( + c_tensors.as_mut_ptr(), + self.c_tensor, + padding.as_ptr(), + padding.len_i32(), + stride.as_ptr(), + stride.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups, + input_size.as_ptr(), + input_size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_reorder_conv2d_weight_out( + &self, + out: &Tensor, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + input_size: impl IntListOption, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mkldnn_reorder_conv2d_weight_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + padding.as_ptr(), + padding.len_i32(), + stride.as_ptr(), + stride.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups, + input_size.as_ptr(), + input_size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_reorder_conv3d_weight( + &self, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + input_size: impl IntListOption, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mkldnn_reorder_conv3d_weight( + c_tensors.as_mut_ptr(), + self.c_tensor, + padding.as_ptr(), + padding.len_i32(), + stride.as_ptr(), + stride.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups, + input_size.as_ptr(), + input_size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_reorder_conv3d_weight_out( + &self, + out: &Tensor, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + input_size: impl IntListOption, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mkldnn_reorder_conv3d_weight_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + padding.as_ptr(), + padding.len_i32(), + stride.as_ptr(), + stride.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + groups, + input_size.as_ptr(), + input_size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mkldnn_rnn_layer( + &self, + weight0: &Tensor, + weight1: &Tensor, + weight2: &Tensor, + weight3: &Tensor, + hx_: &Tensor, + cx_: &Tensor, + reverse: bool, + batch_sizes: impl IntList, + mode: i64, + hidden_size: i64, + num_layers: i64, + has_biases: bool, + bidirectional: bool, + batch_first: bool, + train: bool, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg_mkldnn_rnn_layer( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight0.c_tensor, + weight1.c_tensor, + weight2.c_tensor, + weight3.c_tensor, + hx_.c_tensor, + cx_.c_tensor, + if reverse { 1 } else { 0 }, + batch_sizes.as_ptr(), + batch_sizes.len_i32(), + mode, + hidden_size, + num_layers, + if has_biases { 1 } else { 0 }, + if bidirectional { 1 } else { 0 }, + if batch_first { 1 } else { 0 }, + if train { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_mkldnn_rnn_layer_backward>( + &self, + weight1: &Tensor, + weight2: &Tensor, + weight3: &Tensor, + weight4: &Tensor, + hx_: &Tensor, + cx_tmp: &Tensor, + output: &Tensor, + hy_: &Tensor, + cy_: &Tensor, + grad_output: Option, + grad_hy: Option, + grad_cy: Option, + reverse: bool, + mode: i64, + hidden_size: i64, + num_layers: i64, + has_biases: bool, + train: bool, + bidirectional: bool, + batch_sizes: impl IntList, + batch_first: bool, + workspace: &Tensor, + ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 7]; + unsafe_torch_err!(atg_mkldnn_rnn_layer_backward( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight1.c_tensor, + weight2.c_tensor, + weight3.c_tensor, + weight4.c_tensor, + hx_.c_tensor, + cx_tmp.c_tensor, + output.c_tensor, + hy_.c_tensor, + cy_.c_tensor, + grad_output.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + grad_hy.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + grad_cy.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if reverse { 1 } else { 0 }, + mode, + hidden_size, + num_layers, + if has_biases { 1 } else { 0 }, + if train { 1 } else { 0 }, + if bidirectional { 1 } else { 0 }, + batch_sizes.as_ptr(), + batch_sizes.len_i32(), + if batch_first { 1 } else { 0 }, + workspace.c_tensor + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + Tensor { c_tensor: c_tensors[4] }, + Tensor { c_tensor: c_tensors[5] }, + Tensor { c_tensor: c_tensors[6] }, + )) + } + + pub fn f_mkldnn_rnn_layer_backward_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + out3: &Tensor, + out4: &Tensor, + out5: &Tensor, + out6: &Tensor, + weight1: &Tensor, + weight2: &Tensor, + weight3: &Tensor, + weight4: &Tensor, + hx_: &Tensor, + cx_tmp: &Tensor, + output: &Tensor, + hy_: &Tensor, + cy_: &Tensor, + grad_output: Option, + grad_hy: Option, + grad_cy: Option, + reverse: bool, + mode: i64, + hidden_size: i64, + num_layers: i64, + has_biases: bool, + train: bool, + bidirectional: bool, + batch_sizes: impl IntList, + batch_first: bool, + workspace: &Tensor, + ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 7]; + unsafe_torch_err!(atg_mkldnn_rnn_layer_backward_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + out2.c_tensor, + out3.c_tensor, + out4.c_tensor, + out5.c_tensor, + out6.c_tensor, + self.c_tensor, + weight1.c_tensor, + weight2.c_tensor, + weight3.c_tensor, + weight4.c_tensor, + hx_.c_tensor, + cx_tmp.c_tensor, + output.c_tensor, + hy_.c_tensor, + cy_.c_tensor, + grad_output.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + grad_hy.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + grad_cy.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if reverse { 1 } else { 0 }, + mode, + hidden_size, + num_layers, + if has_biases { 1 } else { 0 }, + if train { 1 } else { 0 }, + if bidirectional { 1 } else { 0 }, + batch_sizes.as_ptr(), + batch_sizes.len_i32(), + if batch_first { 1 } else { 0 }, + workspace.c_tensor + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + Tensor { c_tensor: c_tensors[4] }, + Tensor { c_tensor: c_tensors[5] }, + Tensor { c_tensor: c_tensors[6] }, + )) + } + + pub fn f_mkldnn_rnn_layer_out( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + out3: &Tensor, + weight0: &Tensor, + weight1: &Tensor, + weight2: &Tensor, + weight3: &Tensor, + hx_: &Tensor, + cx_: &Tensor, + reverse: bool, + batch_sizes: impl IntList, + mode: i64, + hidden_size: i64, + num_layers: i64, + has_biases: bool, + bidirectional: bool, + batch_first: bool, + train: bool, + ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 4]; + unsafe_torch_err!(atg_mkldnn_rnn_layer_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + out2.c_tensor, + out3.c_tensor, + self.c_tensor, + weight0.c_tensor, + weight1.c_tensor, + weight2.c_tensor, + weight3.c_tensor, + hx_.c_tensor, + cx_.c_tensor, + if reverse { 1 } else { 0 }, + batch_sizes.as_ptr(), + batch_sizes.len_i32(), + mode, + hidden_size, + num_layers, + if has_biases { 1 } else { 0 }, + if bidirectional { 1 } else { 0 }, + if batch_first { 1 } else { 0 }, + if train { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + Tensor { c_tensor: c_tensors[3] }, + )) + } + + pub fn f_mm(&self, mat2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mm(c_tensors.as_mut_ptr(), self.c_tensor, mat2.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mm_dtype(&self, mat2: &Tensor, out_dtype: Kind) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mm_dtype( + c_tensors.as_mut_ptr(), + self.c_tensor, + mat2.c_tensor, + out_dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mm_dtype_out( + &self, + out: &Tensor, + mat2: &Tensor, + out_dtype: Kind, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mm_dtype_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mat2.c_tensor, + out_dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mm_out(&self, out: &Tensor, mat2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mm_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mat2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mode(&self, dim: i64, keepdim: bool) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_mode( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + if keepdim { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_mode_values( + &self, + values: &Tensor, + indices: &Tensor, + dim: i64, + keepdim: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_mode_values( + c_tensors.as_mut_ptr(), + values.c_tensor, + indices.c_tensor, + self.c_tensor, + dim, + if keepdim { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_moveaxis( + &self, + source: impl IntList, + destination: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_moveaxis( + c_tensors.as_mut_ptr(), + self.c_tensor, + source.as_ptr(), + source.len_i32(), + destination.as_ptr(), + destination.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_moveaxis_int(&self, source: i64, destination: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_moveaxis_int( + c_tensors.as_mut_ptr(), + self.c_tensor, + source, + destination + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_movedim( + &self, + source: impl IntList, + destination: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_movedim( + c_tensors.as_mut_ptr(), + self.c_tensor, + source.as_ptr(), + source.len_i32(), + destination.as_ptr(), + destination.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_movedim_int(&self, source: i64, destination: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_movedim_int( + c_tensors.as_mut_ptr(), + self.c_tensor, + source, + destination + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mse_loss( + &self, + target: &Tensor, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mse_loss( + c_tensors.as_mut_ptr(), + self.c_tensor, + target.c_tensor, + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mse_loss_backward( + &self, + grad_output: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mse_loss_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + target.c_tensor, + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mse_loss_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mse_loss_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + target.c_tensor, + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mse_loss_out( + &self, + out: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mse_loss_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + target.c_tensor, + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_msort(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_msort(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_msort_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_msort_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mt(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mt(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mul(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mul(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mul_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mul_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mul_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mul_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mul_scalar>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mul_scalar( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mul_scalar_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mul_scalar_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mul_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mul_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_multi_margin_loss_backward, S: Into>( + &self, + grad_output: &Tensor, + target: &Tensor, + p: S, + margin: S, + weight: Option, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_multi_margin_loss_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + target.c_tensor, + p.into().c_scalar, + margin.into().c_scalar, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_multi_margin_loss_backward_grad_input, S: Into>( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + target: &Tensor, + p: S, + margin: S, + weight: Option, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_multi_margin_loss_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + target.c_tensor, + p.into().c_scalar, + margin.into().c_scalar, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_multilabel_margin_loss( + &self, + target: &Tensor, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_multilabel_margin_loss( + c_tensors.as_mut_ptr(), + self.c_tensor, + target.c_tensor, + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_multilabel_margin_loss_backward( + &self, + grad_output: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + is_target: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_multilabel_margin_loss_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + target.c_tensor, + reduction.to_int(), + is_target.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_multilabel_margin_loss_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + is_target: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_multilabel_margin_loss_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + target.c_tensor, + reduction.to_int(), + is_target.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_multilabel_margin_loss_out( + &self, + out: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_multilabel_margin_loss_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + target.c_tensor, + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_multinomial(&self, num_samples: i64, replacement: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_multinomial( + c_tensors.as_mut_ptr(), + self.c_tensor, + num_samples, + if replacement { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_multinomial_out( + &self, + out: &Tensor, + num_samples: i64, + replacement: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_multinomial_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + num_samples, + if replacement { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_multiply(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_multiply(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_multiply_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_multiply_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_multiply_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_multiply_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_multiply_scalar>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_multiply_scalar( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_multiply_scalar_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_multiply_scalar_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mv(&self, vec: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mv(c_tensors.as_mut_ptr(), self.c_tensor, vec.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mv_out(&self, out: &Tensor, vec: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mv_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + vec.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mvlgamma(&self, p: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mvlgamma(c_tensors.as_mut_ptr(), self.c_tensor, p)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mvlgamma_(&mut self, p: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mvlgamma_(c_tensors.as_mut_ptr(), self.c_tensor, p)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_mvlgamma_out(&self, out: &Tensor, p: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_mvlgamma_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor, p)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nan_to_num( + &self, + nan: impl Into>, + posinf: impl Into>, + neginf: impl Into>, + ) -> Result { + let nan = nan.into(); + let posinf = posinf.into(); + let neginf = neginf.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nan_to_num( + c_tensors.as_mut_ptr(), + self.c_tensor, + nan.unwrap_or(std::f64::NAN), + nan.is_none() as i8, + posinf.unwrap_or(std::f64::NAN), + posinf.is_none() as i8, + neginf.unwrap_or(std::f64::NAN), + neginf.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nan_to_num_( + &mut self, + nan: impl Into>, + posinf: impl Into>, + neginf: impl Into>, + ) -> Result { + let nan = nan.into(); + let posinf = posinf.into(); + let neginf = neginf.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nan_to_num_( + c_tensors.as_mut_ptr(), + self.c_tensor, + nan.unwrap_or(std::f64::NAN), + nan.is_none() as i8, + posinf.unwrap_or(std::f64::NAN), + posinf.is_none() as i8, + neginf.unwrap_or(std::f64::NAN), + neginf.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nan_to_num_out( + &self, + out: &Tensor, + nan: impl Into>, + posinf: impl Into>, + neginf: impl Into>, + ) -> Result { + let nan = nan.into(); + let posinf = posinf.into(); + let neginf = neginf.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nan_to_num_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + nan.unwrap_or(std::f64::NAN), + nan.is_none() as i8, + posinf.unwrap_or(std::f64::NAN), + posinf.is_none() as i8, + neginf.unwrap_or(std::f64::NAN), + neginf.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nanmean( + &self, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nanmean( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 }, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nanmean_out( + &self, + out: &Tensor, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nanmean_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 }, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nanmedian(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nanmedian(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nanmedian_dim(&self, dim: i64, keepdim: bool) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_nanmedian_dim( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + if keepdim { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_nanmedian_dim_values( + &self, + values: &Tensor, + indices: &Tensor, + dim: i64, + keepdim: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_nanmedian_dim_values( + c_tensors.as_mut_ptr(), + values.c_tensor, + indices.c_tensor, + self.c_tensor, + dim, + if keepdim { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_nanmedian_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nanmedian_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nanquantile( + &self, + q: &Tensor, + dim: impl Into>, + keepdim: bool, + interpolation: &str, + ) -> Result { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nanquantile( + c_tensors.as_mut_ptr(), + self.c_tensor, + q.c_tensor, + dim.unwrap_or(0i64), + dim.is_none() as i8, + if keepdim { 1 } else { 0 }, + interpolation.as_ptr(), + interpolation.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nanquantile_out( + &self, + out: &Tensor, + q: &Tensor, + dim: impl Into>, + keepdim: bool, + interpolation: &str, + ) -> Result { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nanquantile_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + q.c_tensor, + dim.unwrap_or(0i64), + dim.is_none() as i8, + if keepdim { 1 } else { 0 }, + interpolation.as_ptr(), + interpolation.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nanquantile_scalar( + &self, + q: f64, + dim: impl Into>, + keepdim: bool, + interpolation: &str, + ) -> Result { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nanquantile_scalar( + c_tensors.as_mut_ptr(), + self.c_tensor, + q, + dim.unwrap_or(0i64), + dim.is_none() as i8, + if keepdim { 1 } else { 0 }, + interpolation.as_ptr(), + interpolation.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nanquantile_scalar_out( + &self, + out: &Tensor, + q: f64, + dim: impl Into>, + keepdim: bool, + interpolation: &str, + ) -> Result { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nanquantile_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + q, + dim.unwrap_or(0i64), + dim.is_none() as i8, + if keepdim { 1 } else { 0 }, + interpolation.as_ptr(), + interpolation.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nansum( + &self, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nansum( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 }, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nansum_out( + &self, + out: &Tensor, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nansum_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 }, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_narrow(&self, dim: i64, start: i64, length: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_narrow(c_tensors.as_mut_ptr(), self.c_tensor, dim, start, length)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_narrow_copy(&self, dim: i64, start: i64, length: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_narrow_copy( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + start, + length + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_narrow_copy_out( + &self, + out: &Tensor, + dim: i64, + start: i64, + length: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_narrow_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + start, + length + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_narrow_tensor( + &self, + dim: i64, + start: &Tensor, + length: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_narrow_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + start.c_tensor, + length + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_native_batch_norm>( + &self, + weight: Option, + bias: Option, + running_mean: Option, + running_var: Option, + training: bool, + momentum: f64, + eps: f64, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_native_batch_norm( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if training { 1 } else { 0 }, + momentum, + eps + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_native_batch_norm_out>( + &self, + out: &Tensor, + save_mean: &Tensor, + save_invstd: &Tensor, + weight: Option, + bias: Option, + running_mean: Option, + running_var: Option, + training: bool, + momentum: f64, + eps: f64, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_native_batch_norm_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + save_mean.c_tensor, + save_invstd.c_tensor, + self.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if training { 1 } else { 0 }, + momentum, + eps + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_native_channel_shuffle(&self, groups: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_native_channel_shuffle( + c_tensors.as_mut_ptr(), + self.c_tensor, + groups + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_native_dropout(&self, p: f64, train: bool) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_native_dropout( + c_tensors.as_mut_ptr(), + self.c_tensor, + p, + if train { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_native_dropout_backward( + grad_output: &Tensor, + mask: &Tensor, + scale: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_native_dropout_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + mask.c_tensor, + scale + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_native_dropout_backward_out( + out: &Tensor, + grad_output: &Tensor, + mask: &Tensor, + scale: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_native_dropout_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad_output.c_tensor, + mask.c_tensor, + scale + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_native_dropout_out( + &self, + out0: &Tensor, + out1: &Tensor, + p: f64, + train: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_native_dropout_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + self.c_tensor, + p, + if train { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_native_group_norm>( + &self, + weight: Option, + bias: Option, + n: i64, + c: i64, + hxw: i64, + group: i64, + eps: f64, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_native_group_norm( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + n, + c, + hxw, + group, + eps + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_native_group_norm_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + weight: Option, + bias: Option, + n: i64, + c: i64, + hxw: i64, + group: i64, + eps: f64, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_native_group_norm_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + out2.c_tensor, + self.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + n, + c, + hxw, + group, + eps + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_native_layer_norm>( + &self, + normalized_shape: impl IntList, + weight: Option, + bias: Option, + eps: f64, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_native_layer_norm( + c_tensors.as_mut_ptr(), + self.c_tensor, + normalized_shape.as_ptr(), + normalized_shape.len_i32(), + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + eps + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_native_layer_norm_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + normalized_shape: impl IntList, + weight: Option, + bias: Option, + eps: f64, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_native_layer_norm_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + out2.c_tensor, + self.c_tensor, + normalized_shape.as_ptr(), + normalized_shape.len_i32(), + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + eps + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_native_norm(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_native_norm(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_native_norm_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_native_norm_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_native_norm_scalaropt_dim_dtype>( + &self, + p: S, + dim: impl IntList, + keepdim: bool, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_native_norm_scalaropt_dim_dtype( + c_tensors.as_mut_ptr(), + self.c_tensor, + p.into().c_scalar, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 }, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_native_norm_scalaropt_dim_dtype_out>( + &self, + out: &Tensor, + p: S, + dim: impl IntList, + keepdim: bool, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_native_norm_scalaropt_dim_dtype_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + p.into().c_scalar, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 }, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ne>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ne(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ne_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ne_(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ne_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ne_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ne_tensor(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ne_tensor(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ne_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ne_tensor_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ne_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ne_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_neg(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_neg(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_neg_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_neg_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_neg_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_neg_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_negative(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_negative(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_negative_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_negative_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_negative_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_negative_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nested_to_padded_tensor( + &self, + padding: f64, + output_size: impl IntListOption, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nested_to_padded_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + padding, + output_size.as_ptr(), + output_size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_new_empty( + &self, + size: impl IntList, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_new_empty( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_new_empty_out(&self, out: &Tensor, size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_new_empty_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_new_empty_strided( + &self, + size: impl IntList, + stride: impl IntList, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_new_empty_strided( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_new_empty_strided_out( + &self, + out: &Tensor, + size: impl IntList, + stride: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_new_empty_strided_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + size.as_ptr(), + size.len_i32(), + stride.as_ptr(), + stride.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_new_full>( + &self, + size: impl IntList, + fill_value: S, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_new_full( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32(), + fill_value.into().c_scalar, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_new_full_out>( + &self, + out: &Tensor, + size: impl IntList, + fill_value: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_new_full_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + size.as_ptr(), + size.len_i32(), + fill_value.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_new_ones( + &self, + size: impl IntList, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_new_ones( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_new_ones_out(&self, out: &Tensor, size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_new_ones_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_new_zeros( + &self, + size: impl IntList, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_new_zeros( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_new_zeros_out(&self, out: &Tensor, size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_new_zeros_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nextafter(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nextafter(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nextafter_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nextafter_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nextafter_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nextafter_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nll_loss>( + &self, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ignore_index: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nll_loss( + c_tensors.as_mut_ptr(), + self.c_tensor, + target.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + reduction.to_int(), + ignore_index + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nll_loss2d>( + &self, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ignore_index: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nll_loss2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + target.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + reduction.to_int(), + ignore_index + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nll_loss2d_backward>( + &self, + grad_output: &Tensor, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ignore_index: i64, + total_weight: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nll_loss2d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + target.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + reduction.to_int(), + ignore_index, + total_weight.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nll_loss2d_backward_grad_input>( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ignore_index: i64, + total_weight: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nll_loss2d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + target.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + reduction.to_int(), + ignore_index, + total_weight.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nll_loss2d_out>( + &self, + out: &Tensor, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ignore_index: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nll_loss2d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + target.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + reduction.to_int(), + ignore_index + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nll_loss_backward>( + &self, + grad_output: &Tensor, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ignore_index: i64, + total_weight: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nll_loss_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + target.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + reduction.to_int(), + ignore_index, + total_weight.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nll_loss_backward_grad_input>( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ignore_index: i64, + total_weight: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nll_loss_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + target.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + reduction.to_int(), + ignore_index, + total_weight.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nll_loss_nd>( + &self, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ignore_index: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nll_loss_nd( + c_tensors.as_mut_ptr(), + self.c_tensor, + target.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + reduction.to_int(), + ignore_index + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nll_loss_out>( + &self, + out: &Tensor, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ignore_index: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nll_loss_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + target.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + reduction.to_int(), + ignore_index + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nonzero(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nonzero(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nonzero_numpy(&self) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_nonzero_numpy(self.c_tensor)); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_nonzero_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nonzero_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nonzero_static(&self, size: i64, fill_value: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nonzero_static( + c_tensors.as_mut_ptr(), + self.c_tensor, + size, + fill_value + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nonzero_static_out( + &self, + out: &Tensor, + size: i64, + fill_value: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nonzero_static_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + size, + fill_value + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_norm(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_norm(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_norm_dtype_out>( + &self, + out: &Tensor, + p: S, + dim: impl IntList, + keepdim: bool, + dtype: Kind, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_norm_dtype_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + p.into().c_scalar, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 }, + dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_norm_except_dim(v: &Tensor, pow: i64, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_norm_except_dim(c_tensors.as_mut_ptr(), v.c_tensor, pow, dim)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_norm_out>( + &self, + out: &Tensor, + p: S, + dim: impl IntList, + keepdim: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_norm_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + p.into().c_scalar, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_norm_scalar_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_norm_scalar_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_norm_scalaropt_dim>( + &self, + p: S, + dim: impl IntList, + keepdim: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_norm_scalaropt_dim( + c_tensors.as_mut_ptr(), + self.c_tensor, + p.into().c_scalar, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_norm_scalaropt_dim_dtype>( + &self, + p: S, + dim: impl IntList, + keepdim: bool, + dtype: Kind, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_norm_scalaropt_dim_dtype( + c_tensors.as_mut_ptr(), + self.c_tensor, + p.into().c_scalar, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 }, + dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_norm_scalaropt_dtype>( + &self, + p: S, + dtype: Kind, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_norm_scalaropt_dtype( + c_tensors.as_mut_ptr(), + self.c_tensor, + p.into().c_scalar, + dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_norm_scalaropt_dtype_out>( + &self, + out: &Tensor, + p: S, + dtype: Kind, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_norm_scalaropt_dtype_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + p.into().c_scalar, + dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_normal_(&mut self, mean: f64, std: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_normal_(c_tensors.as_mut_ptr(), self.c_tensor, mean, std)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_normal_functional(&self, mean: f64, std: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_normal_functional(c_tensors.as_mut_ptr(), self.c_tensor, mean, std)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_not_equal>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_not_equal( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_not_equal_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_not_equal_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_not_equal_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_not_equal_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_not_equal_tensor(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_not_equal_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_not_equal_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_not_equal_tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_not_equal_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_not_equal_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nuclear_norm(&self, keepdim: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nuclear_norm( + c_tensors.as_mut_ptr(), + self.c_tensor, + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nuclear_norm_dim(&self, dim: impl IntList, keepdim: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nuclear_norm_dim( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nuclear_norm_dim_out( + &self, + out: &Tensor, + dim: impl IntList, + keepdim: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nuclear_norm_dim_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_nuclear_norm_out(&self, out: &Tensor, keepdim: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_nuclear_norm_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_numpy_t(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_numpy_t(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_one_hot(&self, num_classes: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_one_hot(c_tensors.as_mut_ptr(), self.c_tensor, num_classes)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ones(size: impl IntList, options: (Kind, Device)) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ones( + c_tensors.as_mut_ptr(), + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ones_like(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ones_like(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ones_like_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ones_like_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ones_out(out: &Tensor, size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ones_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_orgqr(&self, input2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_orgqr(c_tensors.as_mut_ptr(), self.c_tensor, input2.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_orgqr_out(&self, out: &Tensor, input2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_orgqr_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + input2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ormqr( + &self, + input2: &Tensor, + input3: &Tensor, + left: bool, + transpose: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ormqr( + c_tensors.as_mut_ptr(), + self.c_tensor, + input2.c_tensor, + input3.c_tensor, + if left { 1 } else { 0 }, + if transpose { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ormqr_out( + &self, + out: &Tensor, + input2: &Tensor, + input3: &Tensor, + left: bool, + transpose: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ormqr_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + input2.c_tensor, + input3.c_tensor, + if left { 1 } else { 0 }, + if transpose { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_outer(&self, vec2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_outer(c_tensors.as_mut_ptr(), self.c_tensor, vec2.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_outer_out(&self, out: &Tensor, vec2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_outer_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + vec2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_output_nr(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_output_nr(self.c_tensor)); + Ok(return_) + } + + pub fn f_pad( + &self, + pad: impl IntList, + mode: &str, + value: impl Into>, + ) -> Result { + let value = value.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_pad( + c_tensors.as_mut_ptr(), + self.c_tensor, + pad.as_ptr(), + pad.len_i32(), + mode.as_ptr(), + mode.len() as i32, + value.unwrap_or(std::f64::NAN), + value.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_pad_sequence>( + sequences: &[T], + batch_first: bool, + padding_value: f64, + padding_side: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_pad_sequence( + c_tensors.as_mut_ptr(), + ptr_list(sequences).as_ptr(), + sequences.len() as i32, + if batch_first { 1 } else { 0 }, + padding_value, + padding_side.as_ptr(), + padding_side.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_pairwise_distance( + x1: &Tensor, + x2: &Tensor, + p: f64, + eps: f64, + keepdim: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_pairwise_distance( + c_tensors.as_mut_ptr(), + x1.c_tensor, + x2.c_tensor, + p, + eps, + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_pdist(&self, p: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_pdist(c_tensors.as_mut_ptr(), self.c_tensor, p)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_permute(&self, dims: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_permute( + c_tensors.as_mut_ptr(), + self.c_tensor, + dims.as_ptr(), + dims.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_permute_copy(&self, dims: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_permute_copy( + c_tensors.as_mut_ptr(), + self.c_tensor, + dims.as_ptr(), + dims.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_permute_copy_out(&self, out: &Tensor, dims: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_permute_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dims.as_ptr(), + dims.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_pin_memory(&self, device: Device) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_pin_memory(c_tensors.as_mut_ptr(), self.c_tensor, device.c_int())); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_pinverse(&self, rcond: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_pinverse(c_tensors.as_mut_ptr(), self.c_tensor, rcond)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_pixel_shuffle(&self, upscale_factor: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_pixel_shuffle(c_tensors.as_mut_ptr(), self.c_tensor, upscale_factor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_pixel_shuffle_out( + &self, + out: &Tensor, + upscale_factor: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_pixel_shuffle_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + upscale_factor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_pixel_unshuffle(&self, downscale_factor: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_pixel_unshuffle( + c_tensors.as_mut_ptr(), + self.c_tensor, + downscale_factor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_pixel_unshuffle_out( + &self, + out: &Tensor, + downscale_factor: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_pixel_unshuffle_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + downscale_factor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_poisson(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_poisson(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_poisson_nll_loss( + &self, + target: &Tensor, + log_input: bool, + full: bool, + eps: f64, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_poisson_nll_loss( + c_tensors.as_mut_ptr(), + self.c_tensor, + target.c_tensor, + if log_input { 1 } else { 0 }, + if full { 1 } else { 0 }, + eps, + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_poisson_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_poisson_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_polar(abs: &Tensor, angle: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_polar(c_tensors.as_mut_ptr(), abs.c_tensor, angle.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_polar_out(out: &Tensor, abs: &Tensor, angle: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_polar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + abs.c_tensor, + angle.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_polygamma(&self, n: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_polygamma(c_tensors.as_mut_ptr(), n, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_polygamma_(&mut self, n: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_polygamma_(c_tensors.as_mut_ptr(), self.c_tensor, n)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_polygamma_out(&self, out: &Tensor, n: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_polygamma_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + n, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_positive(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_positive(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_pow(&self, exponent: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_pow(c_tensors.as_mut_ptr(), self.c_tensor, exponent.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_pow_>(&mut self, exponent: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_pow_( + c_tensors.as_mut_ptr(), + self.c_tensor, + exponent.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_pow_scalar>( + self_scalar: S, + exponent: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_pow_scalar( + c_tensors.as_mut_ptr(), + self_scalar.into().c_scalar, + exponent.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_pow_scalar_out>( + out: &Tensor, + self_scalar: S, + exponent: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_pow_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self_scalar.into().c_scalar, + exponent.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_pow_tensor_(&mut self, exponent: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_pow_tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + exponent.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_pow_tensor_scalar>(&self, exponent: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_pow_tensor_scalar( + c_tensors.as_mut_ptr(), + self.c_tensor, + exponent.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_pow_tensor_scalar_out>( + &self, + out: &Tensor, + exponent: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_pow_tensor_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + exponent.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_pow_tensor_tensor_out( + &self, + out: &Tensor, + exponent: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_pow_tensor_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + exponent.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_prelu(&self, weight: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_prelu(c_tensors.as_mut_ptr(), self.c_tensor, weight.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_prod(&self, dtype: impl Into>) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_prod( + c_tensors.as_mut_ptr(), + self.c_tensor, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_prod_dim_int( + &self, + dim: i64, + keepdim: bool, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_prod_dim_int( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + if keepdim { 1 } else { 0 }, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_prod_int_out( + &self, + out: &Tensor, + dim: i64, + keepdim: bool, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_prod_int_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + if keepdim { 1 } else { 0 }, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_prod_out( + &self, + out: &Tensor, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_prod_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_put( + &self, + index: &Tensor, + source: &Tensor, + accumulate: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_put( + c_tensors.as_mut_ptr(), + self.c_tensor, + index.c_tensor, + source.c_tensor, + if accumulate { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_put_( + &mut self, + index: &Tensor, + source: &Tensor, + accumulate: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_put_( + c_tensors.as_mut_ptr(), + self.c_tensor, + index.c_tensor, + source.c_tensor, + if accumulate { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_put_out( + &self, + out: &Tensor, + index: &Tensor, + source: &Tensor, + accumulate: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_put_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + index.c_tensor, + source.c_tensor, + if accumulate { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_q_per_channel_axis(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_q_per_channel_axis(self.c_tensor)); + Ok(return_) + } + + pub fn f_q_per_channel_scales(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_q_per_channel_scales(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_q_per_channel_scales_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_q_per_channel_scales_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_q_per_channel_zero_points(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_q_per_channel_zero_points(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_q_per_channel_zero_points_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_q_per_channel_zero_points_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_q_scale(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_q_scale(self.c_tensor)); + Ok(return_) + } + + pub fn f_q_zero_point(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_q_zero_point(self.c_tensor)); + Ok(return_) + } + + pub fn f_qr(&self, some: bool) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_qr(c_tensors.as_mut_ptr(), self.c_tensor, if some { 1 } else { 0 })); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_qr_q(&self, q: &Tensor, r: &Tensor, some: bool) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_qr_q( + c_tensors.as_mut_ptr(), + q.c_tensor, + r.c_tensor, + self.c_tensor, + if some { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_quantile( + &self, + q: &Tensor, + dim: impl Into>, + keepdim: bool, + interpolation: &str, + ) -> Result { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantile( + c_tensors.as_mut_ptr(), + self.c_tensor, + q.c_tensor, + dim.unwrap_or(0i64), + dim.is_none() as i8, + if keepdim { 1 } else { 0 }, + interpolation.as_ptr(), + interpolation.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_quantile_out( + &self, + out: &Tensor, + q: &Tensor, + dim: impl Into>, + keepdim: bool, + interpolation: &str, + ) -> Result { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantile_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + q.c_tensor, + dim.unwrap_or(0i64), + dim.is_none() as i8, + if keepdim { 1 } else { 0 }, + interpolation.as_ptr(), + interpolation.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_quantile_scalar( + &self, + q: f64, + dim: impl Into>, + keepdim: bool, + interpolation: &str, + ) -> Result { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantile_scalar( + c_tensors.as_mut_ptr(), + self.c_tensor, + q, + dim.unwrap_or(0i64), + dim.is_none() as i8, + if keepdim { 1 } else { 0 }, + interpolation.as_ptr(), + interpolation.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_quantile_scalar_out( + &self, + out: &Tensor, + q: f64, + dim: impl Into>, + keepdim: bool, + interpolation: &str, + ) -> Result { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantile_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + q, + dim.unwrap_or(0i64), + dim.is_none() as i8, + if keepdim { 1 } else { 0 }, + interpolation.as_ptr(), + interpolation.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_quantize_per_channel( + &self, + scales: &Tensor, + zero_points: &Tensor, + axis: i64, + dtype: Kind, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantize_per_channel( + c_tensors.as_mut_ptr(), + self.c_tensor, + scales.c_tensor, + zero_points.c_tensor, + axis, + dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_quantize_per_channel_out( + &self, + out: &Tensor, + scales: &Tensor, + zero_points: &Tensor, + axis: i64, + dtype: Kind, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantize_per_channel_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + scales.c_tensor, + zero_points.c_tensor, + axis, + dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_quantize_per_tensor( + &self, + scale: f64, + zero_point: i64, + dtype: Kind, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantize_per_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + scale, + zero_point, + dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_quantize_per_tensor_dynamic( + &self, + dtype: Kind, + reduce_range: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantize_per_tensor_dynamic( + c_tensors.as_mut_ptr(), + self.c_tensor, + dtype.c_int(), + if reduce_range { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_quantize_per_tensor_dynamic_out( + &self, + out: &Tensor, + dtype: Kind, + reduce_range: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantize_per_tensor_dynamic_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dtype.c_int(), + if reduce_range { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_quantize_per_tensor_out( + &self, + out: &Tensor, + scale: f64, + zero_point: i64, + dtype: Kind, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantize_per_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + scale, + zero_point, + dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_quantize_per_tensor_tensor_qparams( + &self, + scale: &Tensor, + zero_point: &Tensor, + dtype: Kind, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantize_per_tensor_tensor_qparams( + c_tensors.as_mut_ptr(), + self.c_tensor, + scale.c_tensor, + zero_point.c_tensor, + dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_quantize_per_tensor_tensor_qparams_out( + &self, + out: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + dtype: Kind, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantize_per_tensor_tensor_qparams_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + scale.c_tensor, + zero_point.c_tensor, + dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_quantize_per_tensor_tensors>( + tensors: &[T], + scales: &Tensor, + zero_points: &Tensor, + dtype: Kind, + ) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_quantize_per_tensor_tensors( + ptr_list(tensors).as_ptr(), + tensors.len() as i32, + scales.c_tensor, + zero_points.c_tensor, + dtype.c_int() + )); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_quantize_per_tensor_tensors_out>( + out: &[T], + tensors: &[T], + scales: &Tensor, + zero_points: &Tensor, + dtype: Kind, + ) -> Result<(), TchError> { + unsafe_torch_err!(atg_quantize_per_tensor_tensors_out( + ptr_list(out).as_ptr(), + out.len() as i32, + ptr_list(tensors).as_ptr(), + tensors.len() as i32, + scales.c_tensor, + zero_points.c_tensor, + dtype.c_int() + )); + Ok(()) + } + + pub fn f_quantized_batch_norm>( + &self, + weight: Option, + bias: Option, + mean: &Tensor, + var: &Tensor, + eps: f64, + output_scale: f64, + output_zero_point: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantized_batch_norm( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + mean.c_tensor, + var.c_tensor, + eps, + output_scale, + output_zero_point + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_quantized_batch_norm_out>( + &self, + out: &Tensor, + weight: Option, + bias: Option, + mean: &Tensor, + var: &Tensor, + eps: f64, + output_scale: f64, + output_zero_point: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantized_batch_norm_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + mean.c_tensor, + var.c_tensor, + eps, + output_scale, + output_zero_point + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_quantized_gru_cell>( + &self, + hx: &Tensor, + w_ih: &Tensor, + w_hh: &Tensor, + b_ih: &Tensor, + b_hh: &Tensor, + packed_ih: &Tensor, + packed_hh: &Tensor, + col_offsets_ih: &Tensor, + col_offsets_hh: &Tensor, + scale_ih: S, + scale_hh: S, + zero_point_ih: S, + zero_point_hh: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantized_gru_cell( + c_tensors.as_mut_ptr(), + self.c_tensor, + hx.c_tensor, + w_ih.c_tensor, + w_hh.c_tensor, + b_ih.c_tensor, + b_hh.c_tensor, + packed_ih.c_tensor, + packed_hh.c_tensor, + col_offsets_ih.c_tensor, + col_offsets_hh.c_tensor, + scale_ih.into().c_scalar, + scale_hh.into().c_scalar, + zero_point_ih.into().c_scalar, + zero_point_hh.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_quantized_lstm_cell, S: Into>( + &self, + hx: &[T], + w_ih: &Tensor, + w_hh: &Tensor, + b_ih: &Tensor, + b_hh: &Tensor, + packed_ih: &Tensor, + packed_hh: &Tensor, + col_offsets_ih: &Tensor, + col_offsets_hh: &Tensor, + scale_ih: S, + scale_hh: S, + zero_point_ih: S, + zero_point_hh: S, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_quantized_lstm_cell( + c_tensors.as_mut_ptr(), + self.c_tensor, + ptr_list(hx).as_ptr(), + hx.len() as i32, + w_ih.c_tensor, + w_hh.c_tensor, + b_ih.c_tensor, + b_hh.c_tensor, + packed_ih.c_tensor, + packed_hh.c_tensor, + col_offsets_ih.c_tensor, + col_offsets_hh.c_tensor, + scale_ih.into().c_scalar, + scale_hh.into().c_scalar, + zero_point_ih.into().c_scalar, + zero_point_hh.into().c_scalar + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_quantized_max_pool1d( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantized_max_pool1d( + c_tensors.as_mut_ptr(), + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_quantized_max_pool1d_out( + &self, + out: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantized_max_pool1d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_quantized_max_pool2d( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantized_max_pool2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_quantized_max_pool2d_out( + &self, + out: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantized_max_pool2d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_quantized_max_pool3d( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantized_max_pool3d( + c_tensors.as_mut_ptr(), + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_quantized_max_pool3d_out( + &self, + out: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantized_max_pool3d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32(), + if ceil_mode { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_quantized_rnn_relu_cell>( + &self, + hx: &Tensor, + w_ih: &Tensor, + w_hh: &Tensor, + b_ih: &Tensor, + b_hh: &Tensor, + packed_ih: &Tensor, + packed_hh: &Tensor, + col_offsets_ih: &Tensor, + col_offsets_hh: &Tensor, + scale_ih: S, + scale_hh: S, + zero_point_ih: S, + zero_point_hh: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantized_rnn_relu_cell( + c_tensors.as_mut_ptr(), + self.c_tensor, + hx.c_tensor, + w_ih.c_tensor, + w_hh.c_tensor, + b_ih.c_tensor, + b_hh.c_tensor, + packed_ih.c_tensor, + packed_hh.c_tensor, + col_offsets_ih.c_tensor, + col_offsets_hh.c_tensor, + scale_ih.into().c_scalar, + scale_hh.into().c_scalar, + zero_point_ih.into().c_scalar, + zero_point_hh.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_quantized_rnn_tanh_cell>( + &self, + hx: &Tensor, + w_ih: &Tensor, + w_hh: &Tensor, + b_ih: &Tensor, + b_hh: &Tensor, + packed_ih: &Tensor, + packed_hh: &Tensor, + col_offsets_ih: &Tensor, + col_offsets_hh: &Tensor, + scale_ih: S, + scale_hh: S, + zero_point_ih: S, + zero_point_hh: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_quantized_rnn_tanh_cell( + c_tensors.as_mut_ptr(), + self.c_tensor, + hx.c_tensor, + w_ih.c_tensor, + w_hh.c_tensor, + b_ih.c_tensor, + b_hh.c_tensor, + packed_ih.c_tensor, + packed_hh.c_tensor, + col_offsets_ih.c_tensor, + col_offsets_hh.c_tensor, + scale_ih.into().c_scalar, + scale_hh.into().c_scalar, + zero_point_ih.into().c_scalar, + zero_point_hh.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rad2deg(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rad2deg(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rad2deg_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rad2deg_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rad2deg_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rad2deg_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rand(size: impl IntList, options: (Kind, Device)) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rand( + c_tensors.as_mut_ptr(), + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rand_like(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rand_like(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rand_like_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rand_like_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rand_out(out: &Tensor, size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rand_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_randint( + high: i64, + size: impl IntList, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_randint( + c_tensors.as_mut_ptr(), + high, + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_randint_like(&self, high: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_randint_like(c_tensors.as_mut_ptr(), self.c_tensor, high)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_randint_like_low_dtype(&self, low: i64, high: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_randint_like_low_dtype( + c_tensors.as_mut_ptr(), + self.c_tensor, + low, + high + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_randint_like_low_dtype_out( + &self, + out: &Tensor, + low: i64, + high: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_randint_like_low_dtype_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + low, + high + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_randint_like_out(&self, out: &Tensor, high: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_randint_like_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + high + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_randint_like_tensor(&self, high: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_randint_like_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + high.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_randint_like_tensor_out( + &self, + out: &Tensor, + high: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_randint_like_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + high.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_randint_low( + low: i64, + high: i64, + size: impl IntList, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_randint_low( + c_tensors.as_mut_ptr(), + low, + high, + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_randint_low_out( + out: &Tensor, + low: i64, + high: i64, + size: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_randint_low_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + low, + high, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_randint_out(out: &Tensor, high: i64, size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_randint_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + high, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_randn(size: impl IntList, options: (Kind, Device)) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_randn( + c_tensors.as_mut_ptr(), + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_randn_like(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_randn_like(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_randn_like_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_randn_like_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_randn_out(out: &Tensor, size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_randn_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_random(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_random(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_random_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_random_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_random_from(&self, from: i64, to: impl Into>) -> Result { + let to = to.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_random_from( + c_tensors.as_mut_ptr(), + self.c_tensor, + from, + to.unwrap_or(0i64), + to.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_random_from_( + &mut self, + from: i64, + to: impl Into>, + ) -> Result { + let to = to.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_random_from_( + c_tensors.as_mut_ptr(), + self.c_tensor, + from, + to.unwrap_or(0i64), + to.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_random_from_out( + &self, + out: &Tensor, + from: i64, + to: impl Into>, + ) -> Result { + let to = to.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_random_from_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + from, + to.unwrap_or(0i64), + to.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_random_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_random_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_random_to(&self, to: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_random_to(c_tensors.as_mut_ptr(), self.c_tensor, to)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_random_to_(&mut self, to: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_random_to_(c_tensors.as_mut_ptr(), self.c_tensor, to)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_random_to_out(&self, out: &Tensor, to: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_random_to_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + to + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_randperm(n: i64, options: (Kind, Device)) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_randperm( + c_tensors.as_mut_ptr(), + n, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_randperm_out(out: &Tensor, n: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_randperm_out(c_tensors.as_mut_ptr(), out.c_tensor, n)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_range>( + start: S, + end: S, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_range( + c_tensors.as_mut_ptr(), + start.into().c_scalar, + end.into().c_scalar, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_range_out>( + out: &Tensor, + start: S, + end: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_range_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + start.into().c_scalar, + end.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_range_out_>( + out: &Tensor, + start: S, + end: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_range_out_( + c_tensors.as_mut_ptr(), + out.c_tensor, + start.into().c_scalar, + end.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_range_step>( + start: S, + end: S, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_range_step( + c_tensors.as_mut_ptr(), + start.into().c_scalar, + end.into().c_scalar, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_ravel(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_ravel(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_real(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_real(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_reciprocal(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_reciprocal(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_reciprocal_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_reciprocal_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_reciprocal_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_reciprocal_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_reflection_pad1d(&self, padding: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_reflection_pad1d( + c_tensors.as_mut_ptr(), + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_reflection_pad1d_backward( + &self, + grad_output: &Tensor, + padding: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_reflection_pad1d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_reflection_pad1d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + padding: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_reflection_pad1d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_reflection_pad1d_out( + &self, + out: &Tensor, + padding: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_reflection_pad1d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_reflection_pad2d(&self, padding: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_reflection_pad2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_reflection_pad2d_backward( + &self, + grad_output: &Tensor, + padding: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_reflection_pad2d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_reflection_pad2d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + padding: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_reflection_pad2d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_reflection_pad2d_out( + &self, + out: &Tensor, + padding: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_reflection_pad2d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_reflection_pad3d(&self, padding: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_reflection_pad3d( + c_tensors.as_mut_ptr(), + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_reflection_pad3d_backward( + &self, + grad_output: &Tensor, + padding: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_reflection_pad3d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_reflection_pad3d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + padding: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_reflection_pad3d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_reflection_pad3d_out( + &self, + out: &Tensor, + padding: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_reflection_pad3d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_relu(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_relu(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_relu6(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_relu6(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_relu6_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_relu6_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_relu_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_relu_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_relu_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_relu_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_remainder>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_remainder( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_remainder_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_remainder_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_remainder_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_remainder_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_remainder_scalar_tensor>( + self_scalar: S, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_remainder_scalar_tensor( + c_tensors.as_mut_ptr(), + self_scalar.into().c_scalar, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_remainder_scalar_tensor_out>( + out: &Tensor, + self_scalar: S, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_remainder_scalar_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self_scalar.into().c_scalar, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_remainder_tensor(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_remainder_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_remainder_tensor_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_remainder_tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_remainder_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_remainder_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_renorm>( + &self, + p: S, + dim: i64, + maxnorm: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_renorm( + c_tensors.as_mut_ptr(), + self.c_tensor, + p.into().c_scalar, + dim, + maxnorm.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_renorm_>( + &mut self, + p: S, + dim: i64, + maxnorm: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_renorm_( + c_tensors.as_mut_ptr(), + self.c_tensor, + p.into().c_scalar, + dim, + maxnorm.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_renorm_out>( + &self, + out: &Tensor, + p: S, + dim: i64, + maxnorm: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_renorm_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + p.into().c_scalar, + dim, + maxnorm.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_repeat(&self, repeats: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_repeat( + c_tensors.as_mut_ptr(), + self.c_tensor, + repeats.as_ptr(), + repeats.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_repeat_interleave( + repeats: &Tensor, + output_size: impl Into>, + ) -> Result { + let output_size = output_size.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_repeat_interleave( + c_tensors.as_mut_ptr(), + repeats.c_tensor, + output_size.unwrap_or(0i64), + output_size.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_repeat_interleave_self_int( + &self, + repeats: i64, + dim: impl Into>, + output_size: impl Into>, + ) -> Result { + let dim = dim.into(); + let output_size = output_size.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_repeat_interleave_self_int( + c_tensors.as_mut_ptr(), + self.c_tensor, + repeats, + dim.unwrap_or(0i64), + dim.is_none() as i8, + output_size.unwrap_or(0i64), + output_size.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_repeat_interleave_self_tensor( + &self, + repeats: &Tensor, + dim: impl Into>, + output_size: impl Into>, + ) -> Result { + let dim = dim.into(); + let output_size = output_size.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_repeat_interleave_self_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + repeats.c_tensor, + dim.unwrap_or(0i64), + dim.is_none() as i8, + output_size.unwrap_or(0i64), + output_size.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_repeat_interleave_tensor_out( + out: &Tensor, + repeats: &Tensor, + output_size: impl Into>, + ) -> Result { + let output_size = output_size.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_repeat_interleave_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + repeats.c_tensor, + output_size.unwrap_or(0i64), + output_size.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_repeat_out(&self, out: &Tensor, repeats: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_repeat_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + repeats.as_ptr(), + repeats.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_replication_pad1d(&self, padding: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_replication_pad1d( + c_tensors.as_mut_ptr(), + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_replication_pad1d_backward( + &self, + grad_output: &Tensor, + padding: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_replication_pad1d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_replication_pad1d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + padding: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_replication_pad1d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_replication_pad1d_out( + &self, + out: &Tensor, + padding: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_replication_pad1d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_replication_pad2d(&self, padding: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_replication_pad2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_replication_pad2d_backward( + &self, + grad_output: &Tensor, + padding: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_replication_pad2d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_replication_pad2d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + padding: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_replication_pad2d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_replication_pad2d_out( + &self, + out: &Tensor, + padding: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_replication_pad2d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_replication_pad3d(&self, padding: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_replication_pad3d( + c_tensors.as_mut_ptr(), + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_replication_pad3d_backward( + &self, + grad_output: &Tensor, + padding: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_replication_pad3d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_replication_pad3d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + padding: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_replication_pad3d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_replication_pad3d_out( + &self, + out: &Tensor, + padding: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_replication_pad3d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_requires_grad_(&mut self, requires_grad: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_requires_grad_( + c_tensors.as_mut_ptr(), + self.c_tensor, + if requires_grad { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_reshape(&self, shape: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_reshape( + c_tensors.as_mut_ptr(), + self.c_tensor, + shape.as_ptr(), + shape.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_reshape_as(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_reshape_as(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_resize(&self, size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_resize( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_resize_(&mut self, size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_resize_( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_resize_as(&self, the_template: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_resize_as( + c_tensors.as_mut_ptr(), + self.c_tensor, + the_template.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_resize_as_(&mut self, the_template: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_resize_as_( + c_tensors.as_mut_ptr(), + self.c_tensor, + the_template.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_resize_as_out(&self, out: &Tensor, the_template: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_resize_as_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + the_template.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_resize_as_sparse(&self, the_template: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_resize_as_sparse( + c_tensors.as_mut_ptr(), + self.c_tensor, + the_template.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_resize_as_sparse_(&mut self, the_template: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_resize_as_sparse_( + c_tensors.as_mut_ptr(), + self.c_tensor, + the_template.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_resize_as_sparse_out( + &self, + out: &Tensor, + the_template: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_resize_as_sparse_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + the_template.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_resize_out(&self, out: &Tensor, size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_resize_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_resolve_conj(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_resolve_conj(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_resolve_neg(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_resolve_neg(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_retains_grad(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_retains_grad(self.c_tensor)); + Ok(return_ != 0) + } + + pub fn f_rms_norm>( + &self, + normalized_shape: impl IntList, + weight: Option, + eps: impl Into>, + ) -> Result { + let eps = eps.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rms_norm( + c_tensors.as_mut_ptr(), + self.c_tensor, + normalized_shape.as_ptr(), + normalized_shape.len_i32(), + weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + eps.unwrap_or(std::f64::NAN), + eps.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rnn_relu>( + &self, + hx: &Tensor, + params: &[T], + has_biases: bool, + num_layers: i64, + dropout: f64, + train: bool, + bidirectional: bool, + batch_first: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_rnn_relu( + c_tensors.as_mut_ptr(), + self.c_tensor, + hx.c_tensor, + ptr_list(params).as_ptr(), + params.len() as i32, + if has_biases { 1 } else { 0 }, + num_layers, + dropout, + if train { 1 } else { 0 }, + if bidirectional { 1 } else { 0 }, + if batch_first { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_rnn_relu_cell>( + &self, + hx: &Tensor, + w_ih: &Tensor, + w_hh: &Tensor, + b_ih: Option, + b_hh: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rnn_relu_cell( + c_tensors.as_mut_ptr(), + self.c_tensor, + hx.c_tensor, + w_ih.c_tensor, + w_hh.c_tensor, + b_ih.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + b_hh.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rnn_relu_data>( + data: &Tensor, + batch_sizes: &Tensor, + hx: &Tensor, + params: &[T], + has_biases: bool, + num_layers: i64, + dropout: f64, + train: bool, + bidirectional: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_rnn_relu_data( + c_tensors.as_mut_ptr(), + data.c_tensor, + batch_sizes.c_tensor, + hx.c_tensor, + ptr_list(params).as_ptr(), + params.len() as i32, + if has_biases { 1 } else { 0 }, + num_layers, + dropout, + if train { 1 } else { 0 }, + if bidirectional { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_rnn_tanh>( + &self, + hx: &Tensor, + params: &[T], + has_biases: bool, + num_layers: i64, + dropout: f64, + train: bool, + bidirectional: bool, + batch_first: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_rnn_tanh( + c_tensors.as_mut_ptr(), + self.c_tensor, + hx.c_tensor, + ptr_list(params).as_ptr(), + params.len() as i32, + if has_biases { 1 } else { 0 }, + num_layers, + dropout, + if train { 1 } else { 0 }, + if bidirectional { 1 } else { 0 }, + if batch_first { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_rnn_tanh_cell>( + &self, + hx: &Tensor, + w_ih: &Tensor, + w_hh: &Tensor, + b_ih: Option, + b_hh: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rnn_tanh_cell( + c_tensors.as_mut_ptr(), + self.c_tensor, + hx.c_tensor, + w_ih.c_tensor, + w_hh.c_tensor, + b_ih.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + b_hh.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rnn_tanh_data>( + data: &Tensor, + batch_sizes: &Tensor, + hx: &Tensor, + params: &[T], + has_biases: bool, + num_layers: i64, + dropout: f64, + train: bool, + bidirectional: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_rnn_tanh_data( + c_tensors.as_mut_ptr(), + data.c_tensor, + batch_sizes.c_tensor, + hx.c_tensor, + ptr_list(params).as_ptr(), + params.len() as i32, + if has_biases { 1 } else { 0 }, + num_layers, + dropout, + if train { 1 } else { 0 }, + if bidirectional { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_roll(&self, shifts: impl IntList, dims: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_roll( + c_tensors.as_mut_ptr(), + self.c_tensor, + shifts.as_ptr(), + shifts.len_i32(), + dims.as_ptr(), + dims.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_roll_out( + &self, + out: &Tensor, + shifts: impl IntList, + dims: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_roll_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + shifts.as_ptr(), + shifts.len_i32(), + dims.as_ptr(), + dims.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rot90(&self, k: i64, dims: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rot90( + c_tensors.as_mut_ptr(), + self.c_tensor, + k, + dims.as_ptr(), + dims.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rot90_out( + &self, + out: &Tensor, + k: i64, + dims: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rot90_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + k, + dims.as_ptr(), + dims.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_round(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_round(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_round_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_round_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_round_decimals(&self, decimals: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_round_decimals(c_tensors.as_mut_ptr(), self.c_tensor, decimals)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_round_decimals_(&mut self, decimals: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_round_decimals_(c_tensors.as_mut_ptr(), self.c_tensor, decimals)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_round_decimals_out(&self, out: &Tensor, decimals: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_round_decimals_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + decimals + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_round_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_round_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_row_indices(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_row_indices(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_row_indices_copy(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_row_indices_copy(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_row_indices_copy_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_row_indices_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_row_stack>(tensors: &[T]) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_row_stack( + c_tensors.as_mut_ptr(), + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_row_stack_out>( + out: &Tensor, + tensors: &[T], + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_row_stack_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rrelu(&self, training: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rrelu( + c_tensors.as_mut_ptr(), + self.c_tensor, + if training { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rrelu_(&mut self, training: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rrelu_( + c_tensors.as_mut_ptr(), + self.c_tensor, + if training { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rrelu_with_noise(&self, noise: &Tensor, training: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rrelu_with_noise( + c_tensors.as_mut_ptr(), + self.c_tensor, + noise.c_tensor, + if training { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rrelu_with_noise_( + &mut self, + noise: &Tensor, + training: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rrelu_with_noise_( + c_tensors.as_mut_ptr(), + self.c_tensor, + noise.c_tensor, + if training { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rrelu_with_noise_backward>( + &self, + grad_output: &Tensor, + noise: &Tensor, + lower: S, + upper: S, + training: bool, + self_is_result: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rrelu_with_noise_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + noise.c_tensor, + lower.into().c_scalar, + upper.into().c_scalar, + if training { 1 } else { 0 }, + if self_is_result { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rrelu_with_noise_backward_out>( + &self, + out: &Tensor, + grad_output: &Tensor, + noise: &Tensor, + lower: S, + upper: S, + training: bool, + self_is_result: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rrelu_with_noise_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad_output.c_tensor, + self.c_tensor, + noise.c_tensor, + lower.into().c_scalar, + upper.into().c_scalar, + if training { 1 } else { 0 }, + if self_is_result { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rrelu_with_noise_functional( + &self, + noise: &Tensor, + training: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_rrelu_with_noise_functional( + c_tensors.as_mut_ptr(), + self.c_tensor, + noise.c_tensor, + if training { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_rrelu_with_noise_out( + &self, + out: &Tensor, + noise: &Tensor, + training: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rrelu_with_noise_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + noise.c_tensor, + if training { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rsqrt(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rsqrt(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rsqrt_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rsqrt_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rsqrt_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rsqrt_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rsub(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rsub(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rsub_scalar>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rsub_scalar( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rsub_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rsub_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_rsub_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_rsub_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_scalar_tensor>( + s: S, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_scalar_tensor( + c_tensors.as_mut_ptr(), + s.into().c_scalar, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_scalar_tensor_out>(out: &Tensor, s: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_scalar_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + s.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_scaled_dot_product_attention>( + query: &Tensor, + key: &Tensor, + value: &Tensor, + attn_mask: Option, + dropout_p: f64, + is_causal: bool, + scale: impl Into>, + enable_gqa: bool, + ) -> Result { + let scale = scale.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_scaled_dot_product_attention( + c_tensors.as_mut_ptr(), + query.c_tensor, + key.c_tensor, + value.c_tensor, + attn_mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + dropout_p, + if is_causal { 1 } else { 0 }, + scale.unwrap_or(std::f64::NAN), + scale.is_none() as i8, + if enable_gqa { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_scatter(&self, dim: i64, index: &Tensor, src: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_scatter( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + src.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_scatter_( + &mut self, + dim: i64, + index: &Tensor, + src: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_scatter_( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + src.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_scatter_add( + &self, + dim: i64, + index: &Tensor, + src: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_scatter_add( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + src.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_scatter_add_( + &mut self, + dim: i64, + index: &Tensor, + src: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_scatter_add_( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + src.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_scatter_add_out( + &self, + out: &Tensor, + dim: i64, + index: &Tensor, + src: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_scatter_add_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + index.c_tensor, + src.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_scatter_reduce( + &self, + dim: i64, + index: &Tensor, + src: &Tensor, + reduce: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_scatter_reduce( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + src.c_tensor, + reduce.as_ptr(), + reduce.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_scatter_reduce_( + &mut self, + dim: i64, + index: &Tensor, + src: &Tensor, + reduce: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_scatter_reduce_( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + src.c_tensor, + reduce.as_ptr(), + reduce.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_scatter_reduce_out( + &self, + out: &Tensor, + dim: i64, + index: &Tensor, + src: &Tensor, + reduce: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_scatter_reduce_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + index.c_tensor, + src.c_tensor, + reduce.as_ptr(), + reduce.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_scatter_src_out( + &self, + out: &Tensor, + dim: i64, + index: &Tensor, + src: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_scatter_src_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + index.c_tensor, + src.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_scatter_value>( + &self, + dim: i64, + index: &Tensor, + value: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_scatter_value( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + value.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_scatter_value_>( + &mut self, + dim: i64, + index: &Tensor, + value: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_scatter_value_( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + value.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_scatter_value_out>( + &self, + out: &Tensor, + dim: i64, + index: &Tensor, + value: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_scatter_value_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + index.c_tensor, + value.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_scatter_value_reduce>( + &self, + dim: i64, + index: &Tensor, + value: S, + reduce: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_scatter_value_reduce( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + value.into().c_scalar, + reduce.as_ptr(), + reduce.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_scatter_value_reduce_>( + &mut self, + dim: i64, + index: &Tensor, + value: S, + reduce: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_scatter_value_reduce_( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + index.c_tensor, + value.into().c_scalar, + reduce.as_ptr(), + reduce.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_scatter_value_reduce_out>( + &self, + out: &Tensor, + dim: i64, + index: &Tensor, + value: S, + reduce: &str, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_scatter_value_reduce_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + index.c_tensor, + value.into().c_scalar, + reduce.as_ptr(), + reduce.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_searchsorted>( + &self, + sorted_sequence: &Tensor, + out_int32: bool, + right: bool, + side: &str, + sorter: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_searchsorted( + c_tensors.as_mut_ptr(), + sorted_sequence.c_tensor, + self.c_tensor, + if out_int32 { 1 } else { 0 }, + if right { 1 } else { 0 }, + side.as_ptr(), + side.len() as i32, + sorter.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_searchsorted_scalar, S: Into>( + sorted_sequence: &Tensor, + self_scalar: S, + out_int32: bool, + right: bool, + side: &str, + sorter: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_searchsorted_scalar( + c_tensors.as_mut_ptr(), + sorted_sequence.c_tensor, + self_scalar.into().c_scalar, + if out_int32 { 1 } else { 0 }, + if right { 1 } else { 0 }, + side.as_ptr(), + side.len() as i32, + sorter.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_searchsorted_scalar_out, S: Into>( + out: &Tensor, + sorted_sequence: &Tensor, + self_scalar: S, + out_int32: bool, + right: bool, + side: &str, + sorter: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_searchsorted_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + sorted_sequence.c_tensor, + self_scalar.into().c_scalar, + if out_int32 { 1 } else { 0 }, + if right { 1 } else { 0 }, + side.as_ptr(), + side.len() as i32, + sorter.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_searchsorted_tensor_out>( + &self, + out: &Tensor, + sorted_sequence: &Tensor, + out_int32: bool, + right: bool, + side: &str, + sorter: Option, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_searchsorted_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + sorted_sequence.c_tensor, + self.c_tensor, + if out_int32 { 1 } else { 0 }, + if right { 1 } else { 0 }, + side.as_ptr(), + side.len() as i32, + sorter.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_segment_reduce, S: Into>( + data: &Tensor, + reduce: &str, + lengths: Option, + indices: Option, + offsets: Option, + axis: i64, + unsafe_: bool, + initial: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_segment_reduce( + c_tensors.as_mut_ptr(), + data.c_tensor, + reduce.as_ptr(), + reduce.len() as i32, + lengths.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + indices.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + offsets.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + axis, + if unsafe_ { 1 } else { 0 }, + initial.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_segment_reduce_out, S: Into>( + out: &Tensor, + data: &Tensor, + reduce: &str, + lengths: Option, + indices: Option, + offsets: Option, + axis: i64, + unsafe_: bool, + initial: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_segment_reduce_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + data.c_tensor, + reduce.as_ptr(), + reduce.len() as i32, + lengths.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + indices.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + offsets.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + axis, + if unsafe_ { 1 } else { 0 }, + initial.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_select(&self, dim: i64, index: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_select(c_tensors.as_mut_ptr(), self.c_tensor, dim, index)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_select_backward( + grad_output: &Tensor, + input_sizes: impl IntList, + dim: i64, + index: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_select_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + input_sizes.as_ptr(), + input_sizes.len_i32(), + dim, + index + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_select_backward_out( + out: &Tensor, + grad_output: &Tensor, + input_sizes: impl IntList, + dim: i64, + index: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_select_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad_output.c_tensor, + input_sizes.as_ptr(), + input_sizes.len_i32(), + dim, + index + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_select_copy(&self, dim: i64, index: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_select_copy(c_tensors.as_mut_ptr(), self.c_tensor, dim, index)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_select_copy_int_out( + &self, + out: &Tensor, + dim: i64, + index: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_select_copy_int_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + index + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_select_scatter(&self, src: &Tensor, dim: i64, index: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_select_scatter( + c_tensors.as_mut_ptr(), + self.c_tensor, + src.c_tensor, + dim, + index + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_select_scatter_out( + &self, + out: &Tensor, + src: &Tensor, + dim: i64, + index: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_select_scatter_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + src.c_tensor, + dim, + index + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_selu(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_selu(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_selu_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_selu_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_set(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_set(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_set_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_set_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_set_data(&mut self, new_data: &Tensor) -> Result<(), TchError> { + unsafe_torch_err!(atg_set_data(self.c_tensor, new_data.c_tensor)); + Ok(()) + } + + pub fn f_set_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_set_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_set_requires_grad(&self, r: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_set_requires_grad( + c_tensors.as_mut_ptr(), + self.c_tensor, + if r { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_set_source_tensor(&self, source: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_set_source_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + source.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_set_source_tensor_(&mut self, source: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_set_source_tensor_( + c_tensors.as_mut_ptr(), + self.c_tensor, + source.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_set_source_tensor_out( + &self, + out: &Tensor, + source: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_set_source_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + source.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_set_source_tensor_storage_offset_( + &mut self, + source: &Tensor, + storage_offset: i64, + size: impl IntList, + stride: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_set_source_tensor_storage_offset_( + c_tensors.as_mut_ptr(), + self.c_tensor, + source.c_tensor, + storage_offset, + size.as_ptr(), + size.len_i32(), + stride.as_ptr(), + stride.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sgn(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sgn(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sgn_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sgn_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sgn_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sgn_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sigmoid(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sigmoid(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sigmoid_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sigmoid_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sigmoid_backward(grad_output: &Tensor, output: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sigmoid_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + output.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sigmoid_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sigmoid_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + output.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sigmoid_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sigmoid_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sign(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sign(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sign_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sign_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sign_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sign_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_signbit(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_signbit(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_signbit_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_signbit_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_silu(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_silu(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_silu_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_silu_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_silu_backward(&self, grad_output: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_silu_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_silu_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_silu_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_silu_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_silu_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sin(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sin(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sin_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sin_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sin_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sin_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sinc(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sinc(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sinc_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sinc_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sinc_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sinc_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sinh(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sinh(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sinh_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sinh_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sinh_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sinh_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_slice( + &self, + dim: i64, + start: impl Into>, + end: impl Into>, + step: i64, + ) -> Result { + let start = start.into(); + let end = end.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_slice( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + start.unwrap_or(0i64), + start.is_none() as i8, + end.unwrap_or(0i64), + end.is_none() as i8, + step + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_slice_backward( + grad_output: &Tensor, + input_sizes: impl IntList, + dim: i64, + start: i64, + end: i64, + step: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_slice_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + input_sizes.as_ptr(), + input_sizes.len_i32(), + dim, + start, + end, + step + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_slice_backward_out( + out: &Tensor, + grad_output: &Tensor, + input_sizes: impl IntList, + dim: i64, + start: i64, + end: i64, + step: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_slice_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad_output.c_tensor, + input_sizes.as_ptr(), + input_sizes.len_i32(), + dim, + start, + end, + step + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_slice_copy( + &self, + dim: i64, + start: impl Into>, + end: impl Into>, + step: i64, + ) -> Result { + let start = start.into(); + let end = end.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_slice_copy( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + start.unwrap_or(0i64), + start.is_none() as i8, + end.unwrap_or(0i64), + end.is_none() as i8, + step + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_slice_copy_tensor_out( + &self, + out: &Tensor, + dim: i64, + start: impl Into>, + end: impl Into>, + step: i64, + ) -> Result { + let start = start.into(); + let end = end.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_slice_copy_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + start.unwrap_or(0i64), + start.is_none() as i8, + end.unwrap_or(0i64), + end.is_none() as i8, + step + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_slice_inverse( + &self, + src: &Tensor, + dim: i64, + start: impl Into>, + end: impl Into>, + step: i64, + ) -> Result { + let start = start.into(); + let end = end.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_slice_inverse( + c_tensors.as_mut_ptr(), + self.c_tensor, + src.c_tensor, + dim, + start.unwrap_or(0i64), + start.is_none() as i8, + end.unwrap_or(0i64), + end.is_none() as i8, + step + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_slice_scatter( + &self, + src: &Tensor, + dim: i64, + start: impl Into>, + end: impl Into>, + step: i64, + ) -> Result { + let start = start.into(); + let end = end.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_slice_scatter( + c_tensors.as_mut_ptr(), + self.c_tensor, + src.c_tensor, + dim, + start.unwrap_or(0i64), + start.is_none() as i8, + end.unwrap_or(0i64), + end.is_none() as i8, + step + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_slice_scatter_out( + &self, + out: &Tensor, + src: &Tensor, + dim: i64, + start: impl Into>, + end: impl Into>, + step: i64, + ) -> Result { + let start = start.into(); + let end = end.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_slice_scatter_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + src.c_tensor, + dim, + start.unwrap_or(0i64), + start.is_none() as i8, + end.unwrap_or(0i64), + end.is_none() as i8, + step + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_slogdet(&self) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_slogdet(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_slogdet_out( + &self, + sign: &Tensor, + logabsdet: &Tensor, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_slogdet_out( + c_tensors.as_mut_ptr(), + sign.c_tensor, + logabsdet.c_tensor, + self.c_tensor + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_slow_conv3d>( + &self, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_slow_conv3d( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_slow_conv3d_out>( + &self, + out: &Tensor, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_slow_conv3d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_slow_conv_dilated2d>( + &self, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_slow_conv_dilated2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_slow_conv_dilated2d_out>( + &self, + out: &Tensor, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_slow_conv_dilated2d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_slow_conv_dilated3d>( + &self, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_slow_conv_dilated3d( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_slow_conv_dilated3d_out>( + &self, + out: &Tensor, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_slow_conv_dilated3d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_slow_conv_transpose2d>( + &self, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + output_padding: impl IntList, + dilation: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_slow_conv_transpose2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + output_padding.as_ptr(), + output_padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_slow_conv_transpose2d_out>( + &self, + out: &Tensor, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + output_padding: impl IntList, + dilation: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_slow_conv_transpose2d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + output_padding.as_ptr(), + output_padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_slow_conv_transpose3d>( + &self, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + output_padding: impl IntList, + dilation: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_slow_conv_transpose3d( + c_tensors.as_mut_ptr(), + self.c_tensor, + weight.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + output_padding.as_ptr(), + output_padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_slow_conv_transpose3d_out>( + &self, + out: &Tensor, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + output_padding: impl IntList, + dilation: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_slow_conv_transpose3d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + weight.c_tensor, + kernel_size.as_ptr(), + kernel_size.len_i32(), + bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + stride.as_ptr(), + stride.len_i32(), + padding.as_ptr(), + padding.len_i32(), + output_padding.as_ptr(), + output_padding.len_i32(), + dilation.as_ptr(), + dilation.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_smm(&self, mat2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_smm(c_tensors.as_mut_ptr(), self.c_tensor, mat2.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_smooth_l1_loss( + &self, + target: &Tensor, + reduction: crate::Reduction, + beta: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_smooth_l1_loss( + c_tensors.as_mut_ptr(), + self.c_tensor, + target.c_tensor, + reduction.to_int(), + beta + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_smooth_l1_loss_backward( + &self, + grad_output: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + beta: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_smooth_l1_loss_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + target.c_tensor, + reduction.to_int(), + beta + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_smooth_l1_loss_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + beta: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_smooth_l1_loss_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + target.c_tensor, + reduction.to_int(), + beta + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_smooth_l1_loss_out( + &self, + out: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + beta: f64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_smooth_l1_loss_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + target.c_tensor, + reduction.to_int(), + beta + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_soft_margin_loss( + &self, + target: &Tensor, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_soft_margin_loss( + c_tensors.as_mut_ptr(), + self.c_tensor, + target.c_tensor, + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_soft_margin_loss_backward( + &self, + grad_output: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_soft_margin_loss_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + target.c_tensor, + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_soft_margin_loss_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_soft_margin_loss_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + target.c_tensor, + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_soft_margin_loss_out( + &self, + out: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_soft_margin_loss_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + target.c_tensor, + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_softmax(&self, dim: i64, dtype: impl Into>) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_softmax( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_softmax_int_out( + &self, + out: &Tensor, + dim: i64, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_softmax_int_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_softplus(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_softplus(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_softplus_backward>( + &self, + grad_output: &Tensor, + beta: S, + threshold: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_softplus_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + beta.into().c_scalar, + threshold.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_softplus_backward_grad_input>( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + beta: S, + threshold: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_softplus_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + beta.into().c_scalar, + threshold.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_softplus_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_softplus_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_softshrink(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_softshrink(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_softshrink_backward>( + &self, + grad_output: &Tensor, + lambd: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_softshrink_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + lambd.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_softshrink_backward_grad_input>( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + lambd: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_softshrink_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + lambd.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_softshrink_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_softshrink_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sort(&self, dim: i64, descending: bool) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_sort( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + if descending { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_sort_stable( + &self, + stable: bool, + dim: i64, + descending: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_sort_stable( + c_tensors.as_mut_ptr(), + self.c_tensor, + if stable { 1 } else { 0 }, + dim, + if descending { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_sort_values( + &self, + values: &Tensor, + indices: &Tensor, + dim: i64, + descending: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_sort_values( + c_tensors.as_mut_ptr(), + values.c_tensor, + indices.c_tensor, + self.c_tensor, + dim, + if descending { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_sort_values_stable( + &self, + values: &Tensor, + indices: &Tensor, + stable: bool, + dim: i64, + descending: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_sort_values_stable( + c_tensors.as_mut_ptr(), + values.c_tensor, + indices.c_tensor, + self.c_tensor, + if stable { 1 } else { 0 }, + dim, + if descending { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_sparse_bsc_tensor( + ccol_indices: &Tensor, + row_indices: &Tensor, + values: &Tensor, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_bsc_tensor( + c_tensors.as_mut_ptr(), + ccol_indices.c_tensor, + row_indices.c_tensor, + values.c_tensor, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_bsc_tensor_ccol_row_value_size( + ccol_indices: &Tensor, + row_indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_bsc_tensor_ccol_row_value_size( + c_tensors.as_mut_ptr(), + ccol_indices.c_tensor, + row_indices.c_tensor, + values.c_tensor, + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_bsr_tensor( + crow_indices: &Tensor, + col_indices: &Tensor, + values: &Tensor, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_bsr_tensor( + c_tensors.as_mut_ptr(), + crow_indices.c_tensor, + col_indices.c_tensor, + values.c_tensor, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_bsr_tensor_crow_col_value_size( + crow_indices: &Tensor, + col_indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_bsr_tensor_crow_col_value_size( + c_tensors.as_mut_ptr(), + crow_indices.c_tensor, + col_indices.c_tensor, + values.c_tensor, + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_compressed_tensor( + compressed_indices: &Tensor, + plain_indices: &Tensor, + values: &Tensor, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_compressed_tensor( + c_tensors.as_mut_ptr(), + compressed_indices.c_tensor, + plain_indices.c_tensor, + values.c_tensor, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_compressed_tensor_comp_plain_value_size( + compressed_indices: &Tensor, + plain_indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_compressed_tensor_comp_plain_value_size( + c_tensors.as_mut_ptr(), + compressed_indices.c_tensor, + plain_indices.c_tensor, + values.c_tensor, + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_coo_tensor( + size: impl IntList, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_coo_tensor( + c_tensors.as_mut_ptr(), + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_coo_tensor_indices( + indices: &Tensor, + values: &Tensor, + options: (Kind, Device), + is_coalesced: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_coo_tensor_indices( + c_tensors.as_mut_ptr(), + indices.c_tensor, + values.c_tensor, + options.0.c_int(), + options.1.c_int(), + if is_coalesced { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_coo_tensor_indices_size( + indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + is_coalesced: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_coo_tensor_indices_size( + c_tensors.as_mut_ptr(), + indices.c_tensor, + values.c_tensor, + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int(), + if is_coalesced { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_coo_tensor_size_out( + out: &Tensor, + size: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_coo_tensor_size_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_csc_tensor( + ccol_indices: &Tensor, + row_indices: &Tensor, + values: &Tensor, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_csc_tensor( + c_tensors.as_mut_ptr(), + ccol_indices.c_tensor, + row_indices.c_tensor, + values.c_tensor, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_csc_tensor_ccol_row_value_size( + ccol_indices: &Tensor, + row_indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_csc_tensor_ccol_row_value_size( + c_tensors.as_mut_ptr(), + ccol_indices.c_tensor, + row_indices.c_tensor, + values.c_tensor, + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_csr_tensor( + crow_indices: &Tensor, + col_indices: &Tensor, + values: &Tensor, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_csr_tensor( + c_tensors.as_mut_ptr(), + crow_indices.c_tensor, + col_indices.c_tensor, + values.c_tensor, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_csr_tensor_crow_col_value_size( + crow_indices: &Tensor, + col_indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_csr_tensor_crow_col_value_size( + c_tensors.as_mut_ptr(), + crow_indices.c_tensor, + col_indices.c_tensor, + values.c_tensor, + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_dim(&self) -> Result { + let return_; + unsafe_torch_err!(return_ = atg_sparse_dim(self.c_tensor)); + Ok(return_) + } + + pub fn f_sparse_mask(&self, mask: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_mask(c_tensors.as_mut_ptr(), self.c_tensor, mask.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_mask_out(&self, out: &Tensor, mask: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_mask_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mask.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_resize( + &self, + size: impl IntList, + sparse_dim: i64, + dense_dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_resize( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32(), + sparse_dim, + dense_dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_resize_( + &mut self, + size: impl IntList, + sparse_dim: i64, + dense_dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_resize_( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32(), + sparse_dim, + dense_dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_resize_and_clear( + &self, + size: impl IntList, + sparse_dim: i64, + dense_dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_resize_and_clear( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32(), + sparse_dim, + dense_dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_resize_and_clear_( + &mut self, + size: impl IntList, + sparse_dim: i64, + dense_dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_resize_and_clear_( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32(), + sparse_dim, + dense_dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_resize_and_clear_out( + &self, + out: &Tensor, + size: impl IntList, + sparse_dim: i64, + dense_dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_resize_and_clear_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + size.as_ptr(), + size.len_i32(), + sparse_dim, + dense_dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_resize_out( + &self, + out: &Tensor, + size: impl IntList, + sparse_dim: i64, + dense_dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_resize_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + size.as_ptr(), + size.len_i32(), + sparse_dim, + dense_dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_sampled_addmm(&self, mat1: &Tensor, mat2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_sampled_addmm( + c_tensors.as_mut_ptr(), + self.c_tensor, + mat1.c_tensor, + mat2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sparse_sampled_addmm_out( + &self, + out: &Tensor, + mat1: &Tensor, + mat2: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sparse_sampled_addmm_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mat1.c_tensor, + mat2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_airy_ai(x: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_airy_ai(c_tensors.as_mut_ptr(), x.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_airy_ai_out(out: &Tensor, x: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_airy_ai_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_bessel_j0(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_bessel_j0(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_bessel_j0_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_bessel_j0_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_bessel_j1(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_bessel_j1(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_bessel_j1_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_bessel_j1_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_bessel_y0(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_bessel_y0(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_bessel_y0_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_bessel_y0_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_bessel_y1(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_bessel_y1(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_bessel_y1_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_bessel_y1_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_t(x: &Tensor, n: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_t( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_t_n_scalar>( + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_t_n_scalar( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_t_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_t_n_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_t_out( + out: &Tensor, + x: &Tensor, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_t_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_t_x_scalar>( + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_t_x_scalar( + c_tensors.as_mut_ptr(), + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_t_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_t_x_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_u(x: &Tensor, n: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_u( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_u_n_scalar>( + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_u_n_scalar( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_u_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_u_n_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_u_out( + out: &Tensor, + x: &Tensor, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_u_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_u_x_scalar>( + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_u_x_scalar( + c_tensors.as_mut_ptr(), + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_u_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_u_x_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_v(x: &Tensor, n: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_v( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_v_n_scalar>( + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_v_n_scalar( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_v_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_v_n_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_v_out( + out: &Tensor, + x: &Tensor, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_v_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_v_x_scalar>( + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_v_x_scalar( + c_tensors.as_mut_ptr(), + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_v_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_v_x_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_w(x: &Tensor, n: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_w( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_w_n_scalar>( + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_w_n_scalar( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_w_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_w_n_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_w_out( + out: &Tensor, + x: &Tensor, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_w_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_w_x_scalar>( + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_w_x_scalar( + c_tensors.as_mut_ptr(), + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_chebyshev_polynomial_w_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_chebyshev_polynomial_w_x_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_digamma(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_digamma(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_digamma_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_digamma_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_entr(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_entr(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_entr_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_entr_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_erf(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_erf(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_erf_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_erf_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_erfc(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_erfc(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_erfc_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_erfc_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_erfcx(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_erfcx(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_erfcx_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_erfcx_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_erfinv(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_erfinv(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_erfinv_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_erfinv_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_exp2(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_exp2(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_exp2_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_exp2_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_expit(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_expit(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_expit_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_expit_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_expm1(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_expm1(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_expm1_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_expm1_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_gammainc(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_gammainc( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_gammainc_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_gammainc_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_gammaincc(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_gammaincc( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_gammaincc_out( + &self, + out: &Tensor, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_gammaincc_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_gammaln(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_gammaln(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_gammaln_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_gammaln_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_hermite_polynomial_h(x: &Tensor, n: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_hermite_polynomial_h( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_hermite_polynomial_h_n_scalar>( + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_hermite_polynomial_h_n_scalar( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_hermite_polynomial_h_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_hermite_polynomial_h_n_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_hermite_polynomial_h_out( + out: &Tensor, + x: &Tensor, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_hermite_polynomial_h_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_hermite_polynomial_h_x_scalar>( + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_hermite_polynomial_h_x_scalar( + c_tensors.as_mut_ptr(), + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_hermite_polynomial_h_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_hermite_polynomial_h_x_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_hermite_polynomial_he(x: &Tensor, n: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_hermite_polynomial_he( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_hermite_polynomial_he_n_scalar>( + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_hermite_polynomial_he_n_scalar( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_hermite_polynomial_he_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_hermite_polynomial_he_n_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_hermite_polynomial_he_out( + out: &Tensor, + x: &Tensor, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_hermite_polynomial_he_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_hermite_polynomial_he_x_scalar>( + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_hermite_polynomial_he_x_scalar( + c_tensors.as_mut_ptr(), + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_hermite_polynomial_he_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_hermite_polynomial_he_x_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_i0(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_i0(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_i0_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_i0_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_i0e(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_i0e(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_i0e_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_i0e_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_i1(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_i1(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_i1_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_i1_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_i1e(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_i1e(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_i1e_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_i1e_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_laguerre_polynomial_l(x: &Tensor, n: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_laguerre_polynomial_l( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_laguerre_polynomial_l_n_scalar>( + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_laguerre_polynomial_l_n_scalar( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_laguerre_polynomial_l_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_laguerre_polynomial_l_n_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_laguerre_polynomial_l_out( + out: &Tensor, + x: &Tensor, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_laguerre_polynomial_l_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_laguerre_polynomial_l_x_scalar>( + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_laguerre_polynomial_l_x_scalar( + c_tensors.as_mut_ptr(), + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_laguerre_polynomial_l_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_laguerre_polynomial_l_x_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_legendre_polynomial_p(x: &Tensor, n: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_legendre_polynomial_p( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_legendre_polynomial_p_n_scalar>( + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_legendre_polynomial_p_n_scalar( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_legendre_polynomial_p_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_legendre_polynomial_p_n_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_legendre_polynomial_p_out( + out: &Tensor, + x: &Tensor, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_legendre_polynomial_p_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_legendre_polynomial_p_x_scalar>( + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_legendre_polynomial_p_x_scalar( + c_tensors.as_mut_ptr(), + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_legendre_polynomial_p_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_legendre_polynomial_p_x_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_log1p(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_log1p(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_log1p_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_log1p_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_log_ndtr(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_log_ndtr(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_log_ndtr_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_log_ndtr_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_log_softmax( + &self, + dim: i64, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_log_softmax( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_logit(&self, eps: impl Into>) -> Result { + let eps = eps.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_logit( + c_tensors.as_mut_ptr(), + self.c_tensor, + eps.unwrap_or(std::f64::NAN), + eps.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_logit_out( + &self, + out: &Tensor, + eps: impl Into>, + ) -> Result { + let eps = eps.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_logit_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + eps.unwrap_or(std::f64::NAN), + eps.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_logsumexp( + &self, + dim: impl IntList, + keepdim: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_logsumexp( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_logsumexp_out( + &self, + out: &Tensor, + dim: impl IntList, + keepdim: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_logsumexp_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_modified_bessel_i0(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_modified_bessel_i0(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_modified_bessel_i0_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_modified_bessel_i0_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_modified_bessel_i1(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_modified_bessel_i1(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_modified_bessel_i1_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_modified_bessel_i1_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_modified_bessel_k0(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_modified_bessel_k0(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_modified_bessel_k0_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_modified_bessel_k0_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_modified_bessel_k1(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_modified_bessel_k1(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_modified_bessel_k1_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_modified_bessel_k1_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_multigammaln(&self, p: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_multigammaln(c_tensors.as_mut_ptr(), self.c_tensor, p)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_multigammaln_out(&self, out: &Tensor, p: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_multigammaln_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + p + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_ndtr(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_ndtr(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_ndtr_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_ndtr_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_ndtri(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_ndtri(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_ndtri_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_ndtri_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_polygamma(&self, n: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_polygamma(c_tensors.as_mut_ptr(), n, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_polygamma_out(&self, out: &Tensor, n: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_polygamma_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + n, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_psi(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_psi(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_psi_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_psi_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_round(&self, decimals: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_round(c_tensors.as_mut_ptr(), self.c_tensor, decimals)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_round_out(&self, out: &Tensor, decimals: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_round_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + decimals + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_scaled_modified_bessel_k0(x: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_scaled_modified_bessel_k0( + c_tensors.as_mut_ptr(), + x.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_scaled_modified_bessel_k0_out( + out: &Tensor, + x: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_scaled_modified_bessel_k0_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_scaled_modified_bessel_k1(x: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_scaled_modified_bessel_k1( + c_tensors.as_mut_ptr(), + x.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_scaled_modified_bessel_k1_out( + out: &Tensor, + x: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_scaled_modified_bessel_k1_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_t( + x: &Tensor, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_t( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_t_n_scalar>( + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_t_n_scalar( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_t_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_t_n_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_t_out( + out: &Tensor, + x: &Tensor, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_t_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_t_x_scalar>( + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_t_x_scalar( + c_tensors.as_mut_ptr(), + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_t_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_t_x_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_u( + x: &Tensor, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_u( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_u_n_scalar>( + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_u_n_scalar( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_u_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_u_n_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_u_out( + out: &Tensor, + x: &Tensor, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_u_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_u_x_scalar>( + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_u_x_scalar( + c_tensors.as_mut_ptr(), + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_u_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_u_x_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_v( + x: &Tensor, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_v( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_v_n_scalar>( + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_v_n_scalar( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_v_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_v_n_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_v_out( + out: &Tensor, + x: &Tensor, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_v_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_v_x_scalar>( + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_v_x_scalar( + c_tensors.as_mut_ptr(), + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_v_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_v_x_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_w( + x: &Tensor, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_w( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_w_n_scalar>( + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_w_n_scalar( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_w_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_w_n_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_w_out( + out: &Tensor, + x: &Tensor, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_w_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_w_x_scalar>( + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_w_x_scalar( + c_tensors.as_mut_ptr(), + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_shifted_chebyshev_polynomial_w_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_w_x_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.into().c_scalar, + n.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_sinc(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_sinc(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_sinc_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_sinc_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_softmax( + &self, + dim: i64, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_softmax( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_spherical_bessel_j0(x: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_spherical_bessel_j0(c_tensors.as_mut_ptr(), x.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_spherical_bessel_j0_out(out: &Tensor, x: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_spherical_bessel_j0_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + x.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_xlog1py(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_xlog1py( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_xlog1py_other_scalar>( + &self, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_xlog1py_other_scalar( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_xlog1py_other_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_xlog1py_other_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_xlog1py_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_xlog1py_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_xlog1py_self_scalar>( + self_scalar: S, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_xlog1py_self_scalar( + c_tensors.as_mut_ptr(), + self_scalar.into().c_scalar, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_xlog1py_self_scalar_out>( + out: &Tensor, + self_scalar: S, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_xlog1py_self_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self_scalar.into().c_scalar, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_xlogy(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_xlogy(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_xlogy_other_scalar>( + &self, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_xlogy_other_scalar( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_xlogy_other_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_xlogy_other_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_xlogy_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_xlogy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_xlogy_self_scalar>( + self_scalar: S, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_xlogy_self_scalar( + c_tensors.as_mut_ptr(), + self_scalar.into().c_scalar, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_xlogy_self_scalar_out>( + out: &Tensor, + self_scalar: S, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_xlogy_self_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self_scalar.into().c_scalar, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_zeta(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_zeta(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_zeta_other_scalar>( + &self, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_zeta_other_scalar( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_zeta_other_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_zeta_other_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_zeta_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_zeta_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_zeta_self_scalar>( + self_scalar: S, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_zeta_self_scalar( + c_tensors.as_mut_ptr(), + self_scalar.into().c_scalar, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_special_zeta_self_scalar_out>( + out: &Tensor, + self_scalar: S, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_special_zeta_self_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self_scalar.into().c_scalar, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_split(&self, split_size: i64, dim: i64) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_split(self.c_tensor, split_size, dim)); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_split_copy(&self, split_size: i64, dim: i64) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_split_copy(self.c_tensor, split_size, dim)); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_split_copy_tensor_out>( + &self, + out: &[T], + split_size: i64, + dim: i64, + ) -> Result<(), TchError> { + unsafe_torch_err!(atg_split_copy_tensor_out( + ptr_list(out).as_ptr(), + out.len() as i32, + self.c_tensor, + split_size, + dim + )); + Ok(()) + } + + pub fn f_split_sizes( + &self, + split_size: impl IntList, + dim: i64, + ) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_split_sizes( + self.c_tensor, + split_size.as_ptr(), + split_size.len_i32(), + dim + )); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_split_with_sizes( + &self, + split_sizes: impl IntList, + dim: i64, + ) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_split_with_sizes( + self.c_tensor, + split_sizes.as_ptr(), + split_sizes.len_i32(), + dim + )); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_split_with_sizes_copy( + &self, + split_sizes: impl IntList, + dim: i64, + ) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_split_with_sizes_copy( + self.c_tensor, + split_sizes.as_ptr(), + split_sizes.len_i32(), + dim + )); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_split_with_sizes_copy_out>( + &self, + out: &[T], + split_sizes: impl IntList, + dim: i64, + ) -> Result<(), TchError> { + unsafe_torch_err!(atg_split_with_sizes_copy_out( + ptr_list(out).as_ptr(), + out.len() as i32, + self.c_tensor, + split_sizes.as_ptr(), + split_sizes.len_i32(), + dim + )); + Ok(()) + } + + pub fn f_sqrt(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sqrt(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sqrt_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sqrt_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sqrt_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sqrt_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_square(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_square(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_square_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_square_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_square_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_square_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_squeeze(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_squeeze(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_squeeze_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_squeeze_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_squeeze_copy(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_squeeze_copy(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_squeeze_copy_dim(&self, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_squeeze_copy_dim(c_tensors.as_mut_ptr(), self.c_tensor, dim)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_squeeze_copy_dim_out(&self, out: &Tensor, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_squeeze_copy_dim_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_squeeze_copy_dims(&self, dim: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_squeeze_copy_dims( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_squeeze_copy_dims_out( + &self, + out: &Tensor, + dim: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_squeeze_copy_dims_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_squeeze_copy_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_squeeze_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_squeeze_dim(&self, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_squeeze_dim(c_tensors.as_mut_ptr(), self.c_tensor, dim)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_squeeze_dim_(&mut self, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_squeeze_dim_(c_tensors.as_mut_ptr(), self.c_tensor, dim)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_squeeze_dims(&self, dim: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_squeeze_dims( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_squeeze_dims_(&mut self, dim: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_squeeze_dims_( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sspaddmm(&self, mat1: &Tensor, mat2: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sspaddmm( + c_tensors.as_mut_ptr(), + self.c_tensor, + mat1.c_tensor, + mat2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sspaddmm_out( + &self, + out: &Tensor, + mat1: &Tensor, + mat2: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sspaddmm_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + mat1.c_tensor, + mat2.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_stack>(tensors: &[T], dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_stack( + c_tensors.as_mut_ptr(), + ptr_list(tensors).as_ptr(), + tensors.len() as i32, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_stack_out>( + out: &Tensor, + tensors: &[T], + dim: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_stack_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + ptr_list(tensors).as_ptr(), + tensors.len() as i32, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_std(&self, unbiased: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_std( + c_tensors.as_mut_ptr(), + self.c_tensor, + if unbiased { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_std_correction>( + &self, + dim: impl IntListOption, + correction: S, + keepdim: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_std_correction( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + correction.into().c_scalar, + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_std_correction_out>( + &self, + out: &Tensor, + dim: impl IntListOption, + correction: S, + keepdim: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_std_correction_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + correction.into().c_scalar, + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_std_dim( + &self, + dim: impl IntListOption, + unbiased: bool, + keepdim: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_std_dim( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if unbiased { 1 } else { 0 }, + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_std_mean(&self, unbiased: bool) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_std_mean( + c_tensors.as_mut_ptr(), + self.c_tensor, + if unbiased { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_std_mean_correction>( + &self, + dim: impl IntListOption, + correction: S, + keepdim: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_std_mean_correction( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + correction.into().c_scalar, + if keepdim { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_std_mean_correction_out>( + &self, + out0: &Tensor, + out1: &Tensor, + dim: impl IntListOption, + correction: S, + keepdim: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_std_mean_correction_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + correction.into().c_scalar, + if keepdim { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_std_mean_dim( + &self, + dim: impl IntListOption, + unbiased: bool, + keepdim: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_std_mean_dim( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if unbiased { 1 } else { 0 }, + if keepdim { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_std_out( + &self, + out: &Tensor, + dim: impl IntListOption, + unbiased: bool, + keepdim: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_std_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if unbiased { 1 } else { 0 }, + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_stft>( + &self, + n_fft: i64, + hop_length: impl Into>, + win_length: impl Into>, + window: Option, + normalized: bool, + onesided: bool, + return_complex: bool, + align_to_window: bool, + ) -> Result { + let hop_length = hop_length.into(); + let win_length = win_length.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_stft( + c_tensors.as_mut_ptr(), + self.c_tensor, + n_fft, + hop_length.unwrap_or(0i64), + hop_length.is_none() as i8, + win_length.unwrap_or(0i64), + win_length.is_none() as i8, + window.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if normalized { 1 } else { 0 }, + if onesided { 1 } else { 0 }, + if return_complex { 1 } else { 0 }, + if align_to_window { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_stft_center>( + &self, + n_fft: i64, + hop_length: impl Into>, + win_length: impl Into>, + window: Option, + center: bool, + pad_mode: &str, + normalized: bool, + onesided: bool, + return_complex: bool, + align_to_window: bool, + ) -> Result { + let hop_length = hop_length.into(); + let win_length = win_length.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_stft_center( + c_tensors.as_mut_ptr(), + self.c_tensor, + n_fft, + hop_length.unwrap_or(0i64), + hop_length.is_none() as i8, + win_length.unwrap_or(0i64), + win_length.is_none() as i8, + window.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), + if center { 1 } else { 0 }, + pad_mode.as_ptr(), + pad_mode.len() as i32, + if normalized { 1 } else { 0 }, + if onesided { 1 } else { 0 }, + if return_complex { 1 } else { 0 }, + if align_to_window { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sub(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sub(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sub_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sub_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sub_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sub_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sub_scalar>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sub_scalar( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sub_scalar_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sub_scalar_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sub_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sub_scalar_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_subtract(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_subtract(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_subtract_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_subtract_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_subtract_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_subtract_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_subtract_scalar>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_subtract_scalar( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_subtract_scalar_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_subtract_scalar_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sum(&self, dtype: impl Into>) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sum( + c_tensors.as_mut_ptr(), + self.c_tensor, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sum_dim_intlist( + &self, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sum_dim_intlist( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 }, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sum_intlist_out( + &self, + out: &Tensor, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sum_intlist_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if keepdim { 1 } else { 0 }, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sum_out( + &self, + out: &Tensor, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sum_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_sum_to_size(&self, size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_sum_to_size( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_svd( + &self, + some: bool, + compute_uv: bool, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_svd( + c_tensors.as_mut_ptr(), + self.c_tensor, + if some { 1 } else { 0 }, + if compute_uv { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_svd_u( + &self, + u: &Tensor, + s: &Tensor, + v: &Tensor, + some: bool, + compute_uv: bool, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_svd_u( + c_tensors.as_mut_ptr(), + u.c_tensor, + s.c_tensor, + v.c_tensor, + self.c_tensor, + if some { 1 } else { 0 }, + if compute_uv { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_swapaxes(&self, axis0: i64, axis1: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_swapaxes(c_tensors.as_mut_ptr(), self.c_tensor, axis0, axis1)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_swapaxes_(&mut self, axis0: i64, axis1: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_swapaxes_(c_tensors.as_mut_ptr(), self.c_tensor, axis0, axis1)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_swapdims(&self, dim0: i64, dim1: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_swapdims(c_tensors.as_mut_ptr(), self.c_tensor, dim0, dim1)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_swapdims_(&mut self, dim0: i64, dim1: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_swapdims_(c_tensors.as_mut_ptr(), self.c_tensor, dim0, dim1)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_tr(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_t(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_t_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_t_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_t_copy(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_t_copy(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_t_copy_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_t_copy_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_take(&self, index: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_take(c_tensors.as_mut_ptr(), self.c_tensor, index.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_take_along_dim( + &self, + indices: &Tensor, + dim: impl Into>, + ) -> Result { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_take_along_dim( + c_tensors.as_mut_ptr(), + self.c_tensor, + indices.c_tensor, + dim.unwrap_or(0i64), + dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_take_along_dim_out( + &self, + out: &Tensor, + indices: &Tensor, + dim: impl Into>, + ) -> Result { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_take_along_dim_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + indices.c_tensor, + dim.unwrap_or(0i64), + dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_take_out(&self, out: &Tensor, index: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_take_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + index.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_tan(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_tan(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_tan_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_tan_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_tan_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_tan_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_tanh(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_tanh(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_tanh_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_tanh_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_tanh_backward(grad_output: &Tensor, output: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_tanh_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + output.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_tanh_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_tanh_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + output.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_tanh_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_tanh_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_tensor_split(&self, sections: i64, dim: i64) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_tensor_split(self.c_tensor, sections, dim)); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_tensor_split_indices( + &self, + indices: impl IntList, + dim: i64, + ) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_tensor_split_indices( + self.c_tensor, + indices.as_ptr(), + indices.len_i32(), + dim + )); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_tensor_split_tensor_indices_or_sections( + &self, + tensor_indices_or_sections: &Tensor, + dim: i64, + ) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_tensor_split_tensor_indices_or_sections( + self.c_tensor, + tensor_indices_or_sections.c_tensor, + dim + )); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_tensordot( + &self, + other: &Tensor, + dims_self: impl IntList, + dims_other: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_tensordot( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor, + dims_self.as_ptr(), + dims_self.len_i32(), + dims_other.as_ptr(), + dims_other.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_tensordot_out( + &self, + out: &Tensor, + other: &Tensor, + dims_self: impl IntList, + dims_other: impl IntList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_tensordot_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor, + dims_self.as_ptr(), + dims_self.len_i32(), + dims_other.as_ptr(), + dims_other.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_threshold>(&self, threshold: S, value: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_threshold( + c_tensors.as_mut_ptr(), + self.c_tensor, + threshold.into().c_scalar, + value.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_threshold_>( + &mut self, + threshold: S, + value: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_threshold_( + c_tensors.as_mut_ptr(), + self.c_tensor, + threshold.into().c_scalar, + value.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_threshold_backward>( + &self, + grad_output: &Tensor, + threshold: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_threshold_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + self.c_tensor, + threshold.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_threshold_backward_grad_input>( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + threshold: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_threshold_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + self.c_tensor, + threshold.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_threshold_out>( + &self, + out: &Tensor, + threshold: S, + value: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_threshold_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + threshold.into().c_scalar, + value.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_tile(&self, dims: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_tile( + c_tensors.as_mut_ptr(), + self.c_tensor, + dims.as_ptr(), + dims.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_to(&self, device: Device) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_to(c_tensors.as_mut_ptr(), self.c_tensor, device.c_int())); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_to_dense( + &self, + dtype: impl Into>, + masked_grad: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_to_dense( + c_tensors.as_mut_ptr(), + self.c_tensor, + dtype.into().map_or(-1, |s| s.c_int()), + if masked_grad { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_to_dense_backward( + &self, + grad: &Tensor, + masked_grad: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_to_dense_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + self.c_tensor, + if masked_grad { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_to_device_( + &self, + device: Device, + dtype: Kind, + non_blocking: bool, + copy: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_to_device( + c_tensors.as_mut_ptr(), + self.c_tensor, + device.c_int(), + dtype.c_int(), + if non_blocking { 1 } else { 0 }, + if copy { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_to_dtype( + &self, + dtype: Kind, + non_blocking: bool, + copy: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_to_dtype( + c_tensors.as_mut_ptr(), + self.c_tensor, + dtype.c_int(), + if non_blocking { 1 } else { 0 }, + if copy { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_to_dtype_layout( + &self, + options: (Kind, Device), + non_blocking: bool, + copy: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_to_dtype_layout( + c_tensors.as_mut_ptr(), + self.c_tensor, + options.0.c_int(), + options.1.c_int(), + if non_blocking { 1 } else { 0 }, + if copy { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_to_mkldnn(&self, dtype: impl Into>) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_to_mkldnn( + c_tensors.as_mut_ptr(), + self.c_tensor, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_to_mkldnn_backward(&self, grad: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_to_mkldnn_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_to_mkldnn_out( + &self, + out: &Tensor, + dtype: impl Into>, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_to_mkldnn_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dtype.into().map_or(-1, |s| s.c_int()) + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_to_other( + &self, + other: &Tensor, + non_blocking: bool, + copy: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_to_other( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.c_tensor, + if non_blocking { 1 } else { 0 }, + if copy { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_to_padded_tensor( + &self, + padding: f64, + output_size: impl IntListOption, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_to_padded_tensor( + c_tensors.as_mut_ptr(), + self.c_tensor, + padding, + output_size.as_ptr(), + output_size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_to_padded_tensor_out( + &self, + out: &Tensor, + padding: f64, + output_size: impl IntListOption, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_to_padded_tensor_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + padding, + output_size.as_ptr(), + output_size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_to_sparse( + &self, + layout: Option, + blocksize: impl IntListOption, + dense_dim: impl Into>, + ) -> Result { + let dense_dim = dense_dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_to_sparse( + c_tensors.as_mut_ptr(), + self.c_tensor, + layout.map_or(-1, |s| s.to_i8()), + blocksize.as_ptr(), + blocksize.len_i32(), + dense_dim.unwrap_or(0i64), + dense_dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_to_sparse_bsc( + &self, + blocksize: impl IntList, + dense_dim: impl Into>, + ) -> Result { + let dense_dim = dense_dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_to_sparse_bsc( + c_tensors.as_mut_ptr(), + self.c_tensor, + blocksize.as_ptr(), + blocksize.len_i32(), + dense_dim.unwrap_or(0i64), + dense_dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_to_sparse_bsr( + &self, + blocksize: impl IntList, + dense_dim: impl Into>, + ) -> Result { + let dense_dim = dense_dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_to_sparse_bsr( + c_tensors.as_mut_ptr(), + self.c_tensor, + blocksize.as_ptr(), + blocksize.len_i32(), + dense_dim.unwrap_or(0i64), + dense_dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_to_sparse_csc(&self, dense_dim: impl Into>) -> Result { + let dense_dim = dense_dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_to_sparse_csc( + c_tensors.as_mut_ptr(), + self.c_tensor, + dense_dim.unwrap_or(0i64), + dense_dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_to_sparse_csr(&self, dense_dim: impl Into>) -> Result { + let dense_dim = dense_dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_to_sparse_csr( + c_tensors.as_mut_ptr(), + self.c_tensor, + dense_dim.unwrap_or(0i64), + dense_dim.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_to_sparse_sparse_dim(&self, sparse_dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_to_sparse_sparse_dim( + c_tensors.as_mut_ptr(), + self.c_tensor, + sparse_dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_topk( + &self, + k: i64, + dim: i64, + largest: bool, + sorted: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_topk( + c_tensors.as_mut_ptr(), + self.c_tensor, + k, + dim, + if largest { 1 } else { 0 }, + if sorted { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_topk_values( + &self, + values: &Tensor, + indices: &Tensor, + k: i64, + dim: i64, + largest: bool, + sorted: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_topk_values( + c_tensors.as_mut_ptr(), + values.c_tensor, + indices.c_tensor, + self.c_tensor, + k, + dim, + if largest { 1 } else { 0 }, + if sorted { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_totype(&self, scalar_type: Kind) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_totype(c_tensors.as_mut_ptr(), self.c_tensor, scalar_type.c_int())); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_trace(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_trace(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_trace_backward(grad: &Tensor, sizes: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_trace_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + sizes.as_ptr(), + sizes.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_trace_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_trace_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_transpose(&self, dim0: i64, dim1: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_transpose(c_tensors.as_mut_ptr(), self.c_tensor, dim0, dim1)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_transpose_(&mut self, dim0: i64, dim1: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_transpose_(c_tensors.as_mut_ptr(), self.c_tensor, dim0, dim1)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_transpose_copy(&self, dim0: i64, dim1: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_transpose_copy(c_tensors.as_mut_ptr(), self.c_tensor, dim0, dim1)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_transpose_copy_int_out( + &self, + out: &Tensor, + dim0: i64, + dim1: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_transpose_copy_int_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim0, + dim1 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_trapezoid(y: &Tensor, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_trapezoid(c_tensors.as_mut_ptr(), y.c_tensor, dim)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_trapezoid_x(y: &Tensor, x: &Tensor, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_trapezoid_x(c_tensors.as_mut_ptr(), y.c_tensor, x.c_tensor, dim)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_trapz(y: &Tensor, x: &Tensor, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_trapz(c_tensors.as_mut_ptr(), y.c_tensor, x.c_tensor, dim)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_trapz_dx(y: &Tensor, dx: f64, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_trapz_dx(c_tensors.as_mut_ptr(), y.c_tensor, dx, dim)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_triangular_solve( + &self, + a: &Tensor, + upper: bool, + transpose: bool, + unitriangular: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_triangular_solve( + c_tensors.as_mut_ptr(), + self.c_tensor, + a.c_tensor, + if upper { 1 } else { 0 }, + if transpose { 1 } else { 0 }, + if unitriangular { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_triangular_solve_x( + &self, + x: &Tensor, + m: &Tensor, + a: &Tensor, + upper: bool, + transpose: bool, + unitriangular: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_triangular_solve_x( + c_tensors.as_mut_ptr(), + x.c_tensor, + m.c_tensor, + self.c_tensor, + a.c_tensor, + if upper { 1 } else { 0 }, + if transpose { 1 } else { 0 }, + if unitriangular { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_tril(&self, diagonal: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_tril(c_tensors.as_mut_ptr(), self.c_tensor, diagonal)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_tril_(&mut self, diagonal: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_tril_(c_tensors.as_mut_ptr(), self.c_tensor, diagonal)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_tril_indices( + row: i64, + col: i64, + offset: i64, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_tril_indices( + c_tensors.as_mut_ptr(), + row, + col, + offset, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_tril_indices_out( + out: &Tensor, + row: i64, + col: i64, + offset: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_tril_indices_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + row, + col, + offset + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_tril_out(&self, out: &Tensor, diagonal: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_tril_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + diagonal + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_triplet_margin_loss( + anchor: &Tensor, + positive: &Tensor, + negative: &Tensor, + margin: f64, + p: f64, + eps: f64, + swap: bool, + reduction: crate::Reduction, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_triplet_margin_loss( + c_tensors.as_mut_ptr(), + anchor.c_tensor, + positive.c_tensor, + negative.c_tensor, + margin, + p, + eps, + if swap { 1 } else { 0 }, + reduction.to_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_triu(&self, diagonal: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_triu(c_tensors.as_mut_ptr(), self.c_tensor, diagonal)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_triu_(&mut self, diagonal: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_triu_(c_tensors.as_mut_ptr(), self.c_tensor, diagonal)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_triu_indices( + row: i64, + col: i64, + offset: i64, + options: (Kind, Device), + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_triu_indices( + c_tensors.as_mut_ptr(), + row, + col, + offset, + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_triu_indices_out( + out: &Tensor, + row: i64, + col: i64, + offset: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_triu_indices_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + row, + col, + offset + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_triu_out(&self, out: &Tensor, diagonal: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_triu_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + diagonal + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_true_divide(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_true_divide(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_true_divide_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_true_divide_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_true_divide_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_true_divide_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_true_divide_scalar>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_true_divide_scalar( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_true_divide_scalar_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_true_divide_scalar_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_trunc(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_trunc(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_trunc_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_trunc_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_trunc_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_trunc_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_type_as(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_type_as(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_unbind(&self, dim: i64) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_unbind(self.c_tensor, dim)); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_unbind_copy(&self, dim: i64) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_unbind_copy(self.c_tensor, dim)); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_unbind_copy_int_out>( + &self, + out: &[T], + dim: i64, + ) -> Result<(), TchError> { + unsafe_torch_err!(atg_unbind_copy_int_out( + ptr_list(out).as_ptr(), + out.len() as i32, + self.c_tensor, + dim + )); + Ok(()) + } + + pub fn f_unflatten(&self, dim: i64, sizes: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_unflatten( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + sizes.as_ptr(), + sizes.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_unflatten_dense_tensors>( + flat: &Tensor, + tensors: &[T], + ) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_unflatten_dense_tensors( + flat.c_tensor, + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_unfold(&self, dimension: i64, size: i64, step: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_unfold(c_tensors.as_mut_ptr(), self.c_tensor, dimension, size, step)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_unfold_backward( + grad_in: &Tensor, + input_sizes: impl IntList, + dim: i64, + size: i64, + step: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_unfold_backward( + c_tensors.as_mut_ptr(), + grad_in.c_tensor, + input_sizes.as_ptr(), + input_sizes.len_i32(), + dim, + size, + step + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_unfold_backward_out( + out: &Tensor, + grad_in: &Tensor, + input_sizes: impl IntList, + dim: i64, + size: i64, + step: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_unfold_backward_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + grad_in.c_tensor, + input_sizes.as_ptr(), + input_sizes.len_i32(), + dim, + size, + step + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_unfold_copy(&self, dimension: i64, size: i64, step: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_unfold_copy( + c_tensors.as_mut_ptr(), + self.c_tensor, + dimension, + size, + step + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_unfold_copy_out( + &self, + out: &Tensor, + dimension: i64, + size: i64, + step: i64, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_unfold_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dimension, + size, + step + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_uniform(&self, from: f64, to: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_uniform(c_tensors.as_mut_ptr(), self.c_tensor, from, to)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_uniform_(&mut self, from: f64, to: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_uniform_(c_tensors.as_mut_ptr(), self.c_tensor, from, to)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_uniform_out(&self, out: &Tensor, from: f64, to: f64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_uniform_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + from, + to + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_unique_consecutive( + &self, + return_inverse: bool, + return_counts: bool, + dim: impl Into>, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_unique_consecutive( + c_tensors.as_mut_ptr(), + self.c_tensor, + if return_inverse { 1 } else { 0 }, + if return_counts { 1 } else { 0 }, + dim.unwrap_or(0i64), + dim.is_none() as i8 + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_unique_consecutive_out( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + return_inverse: bool, + return_counts: bool, + dim: impl Into>, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let dim = dim.into(); + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_unique_consecutive_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + out2.c_tensor, + self.c_tensor, + if return_inverse { 1 } else { 0 }, + if return_counts { 1 } else { 0 }, + dim.unwrap_or(0i64), + dim.is_none() as i8 + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_unique_dim( + &self, + dim: i64, + sorted: bool, + return_inverse: bool, + return_counts: bool, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_unique_dim( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + if sorted { 1 } else { 0 }, + if return_inverse { 1 } else { 0 }, + if return_counts { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_unique_dim_consecutive( + &self, + dim: i64, + return_inverse: bool, + return_counts: bool, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_unique_dim_consecutive( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim, + if return_inverse { 1 } else { 0 }, + if return_counts { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_unique_dim_consecutive_out( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + dim: i64, + return_inverse: bool, + return_counts: bool, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_unique_dim_consecutive_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + out2.c_tensor, + self.c_tensor, + dim, + if return_inverse { 1 } else { 0 }, + if return_counts { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_unique_dim_out( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + dim: i64, + sorted: bool, + return_inverse: bool, + return_counts: bool, + ) -> Result<(Tensor, Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 3]; + unsafe_torch_err!(atg_unique_dim_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + out2.c_tensor, + self.c_tensor, + dim, + if sorted { 1 } else { 0 }, + if return_inverse { 1 } else { 0 }, + if return_counts { 1 } else { 0 } + )); + Ok(( + Tensor { c_tensor: c_tensors[0] }, + Tensor { c_tensor: c_tensors[1] }, + Tensor { c_tensor: c_tensors[2] }, + )) + } + + pub fn f_unsafe_chunk(&self, chunks: i64, dim: i64) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_unsafe_chunk(self.c_tensor, chunks, dim)); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_unsafe_split(&self, split_size: i64, dim: i64) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_unsafe_split(self.c_tensor, split_size, dim)); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_unsafe_split_tensor_out>( + &self, + out: &[T], + split_size: i64, + dim: i64, + ) -> Result<(), TchError> { + unsafe_torch_err!(atg_unsafe_split_tensor_out( + ptr_list(out).as_ptr(), + out.len() as i32, + self.c_tensor, + split_size, + dim + )); + Ok(()) + } + + pub fn f_unsafe_split_with_sizes( + &self, + split_sizes: impl IntList, + dim: i64, + ) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_unsafe_split_with_sizes( + self.c_tensor, + split_sizes.as_ptr(), + split_sizes.len_i32(), + dim + )); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_unsafe_split_with_sizes_out>( + &self, + out: &[T], + split_sizes: impl IntList, + dim: i64, + ) -> Result<(), TchError> { + unsafe_torch_err!(atg_unsafe_split_with_sizes_out( + ptr_list(out).as_ptr(), + out.len() as i32, + self.c_tensor, + split_sizes.as_ptr(), + split_sizes.len_i32(), + dim + )); + Ok(()) + } + + pub fn f_unsqueeze(&self, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_unsqueeze(c_tensors.as_mut_ptr(), self.c_tensor, dim)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_unsqueeze_(&mut self, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_unsqueeze_(c_tensors.as_mut_ptr(), self.c_tensor, dim)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_unsqueeze_copy(&self, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_unsqueeze_copy(c_tensors.as_mut_ptr(), self.c_tensor, dim)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_unsqueeze_copy_out(&self, out: &Tensor, dim: i64) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_unsqueeze_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_bicubic2d( + &self, + output_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_bicubic2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_bicubic2d_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_bicubic2d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_bicubic2d_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_bicubic2d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_bicubic2d_out( + &self, + out: &Tensor, + output_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_bicubic2d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_bicubic2d_vec( + &self, + output_size: impl IntListOption, + align_corners: bool, + scale_factors: impl DoubleList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_bicubic2d_vec( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + if align_corners { 1 } else { 0 }, + scale_factors.as_ptr(), + scale_factors.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_bilinear2d( + &self, + output_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_bilinear2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_bilinear2d_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_bilinear2d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_bilinear2d_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_bilinear2d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_bilinear2d_out( + &self, + out: &Tensor, + output_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_bilinear2d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_bilinear2d_vec( + &self, + output_size: impl IntListOption, + align_corners: bool, + scale_factors: impl DoubleList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_bilinear2d_vec( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + if align_corners { 1 } else { 0 }, + scale_factors.as_ptr(), + scale_factors.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_bilinear2d_vec_out( + &self, + out: &Tensor, + output_size: impl IntListOption, + align_corners: bool, + scale_factors: impl DoubleList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_bilinear2d_vec_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + if align_corners { 1 } else { 0 }, + scale_factors.as_ptr(), + scale_factors.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_linear1d( + &self, + output_size: impl IntList, + align_corners: bool, + scales: impl Into>, + ) -> Result { + let scales = scales.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_linear1d( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales.unwrap_or(std::f64::NAN), + scales.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_linear1d_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales: impl Into>, + ) -> Result { + let scales = scales.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_linear1d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales.unwrap_or(std::f64::NAN), + scales.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_linear1d_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales: impl Into>, + ) -> Result { + let scales = scales.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_linear1d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales.unwrap_or(std::f64::NAN), + scales.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_linear1d_out( + &self, + out: &Tensor, + output_size: impl IntList, + align_corners: bool, + scales: impl Into>, + ) -> Result { + let scales = scales.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_linear1d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales.unwrap_or(std::f64::NAN), + scales.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_linear1d_vec( + &self, + output_size: impl IntListOption, + align_corners: bool, + scale_factors: impl DoubleList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_linear1d_vec( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + if align_corners { 1 } else { 0 }, + scale_factors.as_ptr(), + scale_factors.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_nearest1d( + &self, + output_size: impl IntList, + scales: impl Into>, + ) -> Result { + let scales = scales.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_nearest1d( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + scales.unwrap_or(std::f64::NAN), + scales.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_nearest1d_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales: impl Into>, + ) -> Result { + let scales = scales.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_nearest1d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + scales.unwrap_or(std::f64::NAN), + scales.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_nearest1d_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales: impl Into>, + ) -> Result { + let scales = scales.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_nearest1d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + scales.unwrap_or(std::f64::NAN), + scales.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_nearest1d_out( + &self, + out: &Tensor, + output_size: impl IntList, + scales: impl Into>, + ) -> Result { + let scales = scales.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_nearest1d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + scales.unwrap_or(std::f64::NAN), + scales.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_nearest1d_vec( + &self, + output_size: impl IntListOption, + scale_factors: impl DoubleList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_nearest1d_vec( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + scale_factors.as_ptr(), + scale_factors.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_nearest2d( + &self, + output_size: impl IntList, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_nearest2d( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_nearest2d_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_nearest2d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_nearest2d_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_nearest2d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_nearest2d_out( + &self, + out: &Tensor, + output_size: impl IntList, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_nearest2d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_nearest2d_vec( + &self, + output_size: impl IntListOption, + scale_factors: impl DoubleList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_nearest2d_vec( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + scale_factors.as_ptr(), + scale_factors.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_nearest2d_vec_out( + &self, + out: &Tensor, + output_size: impl IntListOption, + scale_factors: impl DoubleList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_nearest2d_vec_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + scale_factors.as_ptr(), + scale_factors.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_nearest3d( + &self, + output_size: impl IntList, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_d = scales_d.into(); + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_nearest3d( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + scales_d.unwrap_or(std::f64::NAN), + scales_d.is_none() as i8, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_nearest3d_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_d = scales_d.into(); + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_nearest3d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + scales_d.unwrap_or(std::f64::NAN), + scales_d.is_none() as i8, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_nearest3d_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_d = scales_d.into(); + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_nearest3d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + scales_d.unwrap_or(std::f64::NAN), + scales_d.is_none() as i8, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_nearest3d_out( + &self, + out: &Tensor, + output_size: impl IntList, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_d = scales_d.into(); + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_nearest3d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + scales_d.unwrap_or(std::f64::NAN), + scales_d.is_none() as i8, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_nearest3d_vec( + &self, + output_size: impl IntListOption, + scale_factors: impl DoubleList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_nearest3d_vec( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + scale_factors.as_ptr(), + scale_factors.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_trilinear3d( + &self, + output_size: impl IntList, + align_corners: bool, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_d = scales_d.into(); + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_trilinear3d( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales_d.unwrap_or(std::f64::NAN), + scales_d.is_none() as i8, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_trilinear3d_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_d = scales_d.into(); + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_trilinear3d_backward( + c_tensors.as_mut_ptr(), + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales_d.unwrap_or(std::f64::NAN), + scales_d.is_none() as i8, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_trilinear3d_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_d = scales_d.into(); + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_trilinear3d_backward_grad_input( + c_tensors.as_mut_ptr(), + grad_input.c_tensor, + grad_output.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + input_size.as_ptr(), + input_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales_d.unwrap_or(std::f64::NAN), + scales_d.is_none() as i8, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_trilinear3d_out( + &self, + out: &Tensor, + output_size: impl IntList, + align_corners: bool, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Result { + let scales_d = scales_d.into(); + let scales_h = scales_h.into(); + let scales_w = scales_w.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_trilinear3d_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + if align_corners { 1 } else { 0 }, + scales_d.unwrap_or(std::f64::NAN), + scales_d.is_none() as i8, + scales_h.unwrap_or(std::f64::NAN), + scales_h.is_none() as i8, + scales_w.unwrap_or(std::f64::NAN), + scales_w.is_none() as i8 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_upsample_trilinear3d_vec( + &self, + output_size: impl IntListOption, + align_corners: bool, + scale_factors: impl DoubleList, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_upsample_trilinear3d_vec( + c_tensors.as_mut_ptr(), + self.c_tensor, + output_size.as_ptr(), + output_size.len_i32(), + if align_corners { 1 } else { 0 }, + scale_factors.as_ptr(), + scale_factors.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_value_selecting_reduction_backward( + grad: &Tensor, + dim: i64, + indices: &Tensor, + sizes: impl IntList, + keepdim: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_value_selecting_reduction_backward( + c_tensors.as_mut_ptr(), + grad.c_tensor, + dim, + indices.c_tensor, + sizes.as_ptr(), + sizes.len_i32(), + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_values(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_values(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_values_copy(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_values_copy(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_values_copy_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_values_copy_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_vander( + x: &Tensor, + n: impl Into>, + increasing: bool, + ) -> Result { + let n = n.into(); + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_vander( + c_tensors.as_mut_ptr(), + x.c_tensor, + n.unwrap_or(0i64), + n.is_none() as i8, + if increasing { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_var(&self, unbiased: bool) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_var( + c_tensors.as_mut_ptr(), + self.c_tensor, + if unbiased { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_var_correction>( + &self, + dim: impl IntListOption, + correction: S, + keepdim: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_var_correction( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + correction.into().c_scalar, + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_var_correction_out>( + &self, + out: &Tensor, + dim: impl IntListOption, + correction: S, + keepdim: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_var_correction_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + correction.into().c_scalar, + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_var_dim( + &self, + dim: impl IntListOption, + unbiased: bool, + keepdim: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_var_dim( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if unbiased { 1 } else { 0 }, + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_var_mean(&self, unbiased: bool) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_var_mean( + c_tensors.as_mut_ptr(), + self.c_tensor, + if unbiased { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_var_mean_correction>( + &self, + dim: impl IntListOption, + correction: S, + keepdim: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_var_mean_correction( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + correction.into().c_scalar, + if keepdim { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_var_mean_correction_out>( + &self, + out0: &Tensor, + out1: &Tensor, + dim: impl IntListOption, + correction: S, + keepdim: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_var_mean_correction_out( + c_tensors.as_mut_ptr(), + out0.c_tensor, + out1.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + correction.into().c_scalar, + if keepdim { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_var_mean_dim( + &self, + dim: impl IntListOption, + unbiased: bool, + keepdim: bool, + ) -> Result<(Tensor, Tensor), TchError> { + let mut c_tensors = [std::ptr::null_mut(); 2]; + unsafe_torch_err!(atg_var_mean_dim( + c_tensors.as_mut_ptr(), + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if unbiased { 1 } else { 0 }, + if keepdim { 1 } else { 0 } + )); + Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) + } + + pub fn f_var_out( + &self, + out: &Tensor, + dim: impl IntListOption, + unbiased: bool, + keepdim: bool, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_var_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dim.as_ptr(), + dim.len_i32(), + if unbiased { 1 } else { 0 }, + if keepdim { 1 } else { 0 } + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_vdot(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_vdot(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_vdot_out(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_vdot_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_view_(&self, size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_view( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_view_as(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_view_as(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_view_as_complex(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_view_as_complex(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_view_as_complex_copy(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_view_as_complex_copy(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_view_as_complex_copy_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_view_as_complex_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_view_as_real(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_view_as_real(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_view_as_real_copy(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_view_as_real_copy(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_view_as_real_copy_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_view_as_real_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_view_copy(&self, size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_view_copy( + c_tensors.as_mut_ptr(), + self.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_view_copy_dtype(&self, dtype: Kind) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_view_copy_dtype( + c_tensors.as_mut_ptr(), + self.c_tensor, + dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_view_copy_dtype_out(&self, out: &Tensor, dtype: Kind) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_view_copy_dtype_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + dtype.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_view_copy_out(&self, out: &Tensor, size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_view_copy_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_view_dtype(&self, dtype: Kind) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_view_dtype(c_tensors.as_mut_ptr(), self.c_tensor, dtype.c_int())); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_vsplit(&self, sections: i64) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_vsplit(self.c_tensor, sections)); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_vsplit_array(&self, indices: impl IntList) -> Result, TchError> { + let c_tensors = + unsafe_torch_err!(atg_vsplit_array(self.c_tensor, indices.as_ptr(), indices.len_i32())); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_vstack>(tensors: &[T]) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_vstack( + c_tensors.as_mut_ptr(), + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_vstack_out>( + out: &Tensor, + tensors: &[T], + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_vstack_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + ptr_list(tensors).as_ptr(), + tensors.len() as i32 + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_where_(condition: &Tensor) -> Result, TchError> { + let c_tensors = unsafe_torch_err!(atg_where(condition.c_tensor)); + let mut r__ = vec![]; + let mut i = 0; + loop { + let c__ = unsafe { *c_tensors.add(i) }; + if c__.is_null() { + break; + } + r__.push(Tensor { c_tensor: c__ }); + i += 1; + } + unsafe { libc::free(c_tensors as *mut libc::c_void) } + Ok(r__) + } + + pub fn f_where_scalar>( + condition: &Tensor, + self_scalar: S, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_where_scalar( + c_tensors.as_mut_ptr(), + condition.c_tensor, + self_scalar.into().c_scalar, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_where_scalarother>( + &self, + condition: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_where_scalarother( + c_tensors.as_mut_ptr(), + condition.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_where_scalarself>( + condition: &Tensor, + self_scalar: S, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_where_scalarself( + c_tensors.as_mut_ptr(), + condition.c_tensor, + self_scalar.into().c_scalar, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_where_self(&self, condition: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_where_self( + c_tensors.as_mut_ptr(), + condition.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_where_self_out( + &self, + out: &Tensor, + condition: &Tensor, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_where_self_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + condition.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_xlogy(&self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_xlogy(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_xlogy_(&mut self, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_xlogy_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_xlogy_outscalar_other>( + &self, + out: &Tensor, + other: S, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_xlogy_outscalar_other( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_xlogy_outscalar_self>( + out: &Tensor, + self_scalar: S, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_xlogy_outscalar_self( + c_tensors.as_mut_ptr(), + out.c_tensor, + self_scalar.into().c_scalar, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_xlogy_outtensor(&self, out: &Tensor, other: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_xlogy_outtensor( + c_tensors.as_mut_ptr(), + out.c_tensor, + self.c_tensor, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_xlogy_scalar_other>(&self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_xlogy_scalar_other( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_xlogy_scalar_other_>(&mut self, other: S) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_xlogy_scalar_other_( + c_tensors.as_mut_ptr(), + self.c_tensor, + other.into().c_scalar + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_xlogy_scalar_self>( + self_scalar: S, + other: &Tensor, + ) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_xlogy_scalar_self( + c_tensors.as_mut_ptr(), + self_scalar.into().c_scalar, + other.c_tensor + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_zero(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_zero(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_zero_(&mut self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_zero_(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_zero_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_zero_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_zeros(size: impl IntList, options: (Kind, Device)) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_zeros( + c_tensors.as_mut_ptr(), + size.as_ptr(), + size.len_i32(), + options.0.c_int(), + options.1.c_int() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_zeros_like(&self) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_zeros_like(c_tensors.as_mut_ptr(), self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_zeros_like_out(&self, out: &Tensor) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_zeros_like_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); + Ok(Tensor { c_tensor: c_tensors[0] }) + } + + pub fn f_zeros_out(out: &Tensor, size: impl IntList) -> Result { + let mut c_tensors = [std::ptr::null_mut(); 1]; + unsafe_torch_err!(atg_zeros_out( + c_tensors.as_mut_ptr(), + out.c_tensor, + size.as_ptr(), + size.len_i32() + )); + Ok(Tensor { c_tensor: c_tensors[0] }) + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/tensor_generated.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/tensor_generated.rs new file mode 100644 index 0000000000..e71463f387 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/wrappers/tensor_generated.rs @@ -0,0 +1,19703 @@ +/* THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT BY HAND! */ +#![allow(clippy::all)] +use crate::{Device, Kind, Layout, Scalar, Tensor}; +use std::borrow::Borrow; +use std::convert::Into; +use torch_sys::*; + +impl Tensor { + pub fn internal_and_>(&mut self, other: S) -> Tensor { + self.f_internal_and_(other).unwrap() + } + + pub fn internal_and_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_internal_and_tensor_(other).unwrap() + } + + pub fn internal_iand_>(&mut self, other: S) -> Tensor { + self.f_internal_iand_(other).unwrap() + } + + pub fn internal_iand_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_internal_iand_tensor_(other).unwrap() + } + + pub fn internal_ilshift_>(&mut self, other: S) -> Tensor { + self.f_internal_ilshift_(other).unwrap() + } + + pub fn internal_ilshift_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_internal_ilshift_tensor_(other).unwrap() + } + + pub fn internal_ior_>(&mut self, other: S) -> Tensor { + self.f_internal_ior_(other).unwrap() + } + + pub fn internal_ior_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_internal_ior_tensor_(other).unwrap() + } + + pub fn internal_irshift_>(&mut self, other: S) -> Tensor { + self.f_internal_irshift_(other).unwrap() + } + + pub fn internal_irshift_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_internal_irshift_tensor_(other).unwrap() + } + + pub fn internal_ixor_>(&mut self, other: S) -> Tensor { + self.f_internal_ixor_(other).unwrap() + } + + pub fn internal_ixor_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_internal_ixor_tensor_(other).unwrap() + } + + pub fn internal_lshift_>(&mut self, other: S) -> Tensor { + self.f_internal_lshift_(other).unwrap() + } + + pub fn internal_lshift_scalar_out_>(&self, out: &Tensor, other: S) -> Tensor { + self.f_internal_lshift_scalar_out_(out, other).unwrap() + } + + pub fn internal_lshift_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_internal_lshift_tensor_(other).unwrap() + } + + pub fn internal_lshift_tensor_out_(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_internal_lshift_tensor_out_(out, other).unwrap() + } + + pub fn internal_or_>(&mut self, other: S) -> Tensor { + self.f_internal_or_(other).unwrap() + } + + pub fn internal_or_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_internal_or_tensor_(other).unwrap() + } + + pub fn internal_rshift_>(&mut self, other: S) -> Tensor { + self.f_internal_rshift_(other).unwrap() + } + + pub fn internal_rshift_scalar_out_>(&self, out: &Tensor, other: S) -> Tensor { + self.f_internal_rshift_scalar_out_(out, other).unwrap() + } + + pub fn internal_rshift_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_internal_rshift_tensor_(other).unwrap() + } + + pub fn internal_rshift_tensor_out_(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_internal_rshift_tensor_out_(out, other).unwrap() + } + + pub fn internal_xor_>(&mut self, other: S) -> Tensor { + self.f_internal_xor_(other).unwrap() + } + + pub fn internal_xor_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_internal_xor_tensor_(other).unwrap() + } + + pub fn internal_adaptive_avg_pool2d(&self, output_size: impl IntList) -> Tensor { + self.f_internal_adaptive_avg_pool2d(output_size).unwrap() + } + + pub fn internal_adaptive_avg_pool2d_backward(&self, grad_output: &Tensor) -> Tensor { + self.f_internal_adaptive_avg_pool2d_backward(grad_output).unwrap() + } + + pub fn internal_adaptive_avg_pool2d_backward_out( + &self, + out: &Tensor, + grad_output: &Tensor, + ) -> Tensor { + self.f_internal_adaptive_avg_pool2d_backward_out(out, grad_output).unwrap() + } + + pub fn internal_adaptive_avg_pool2d_out( + &self, + out: &Tensor, + output_size: impl IntList, + ) -> Tensor { + self.f_internal_adaptive_avg_pool2d_out(out, output_size).unwrap() + } + + pub fn internal_adaptive_avg_pool3d(&self, output_size: impl IntList) -> Tensor { + self.f_internal_adaptive_avg_pool3d(output_size).unwrap() + } + + pub fn internal_adaptive_avg_pool3d_backward(&self, grad_output: &Tensor) -> Tensor { + self.f_internal_adaptive_avg_pool3d_backward(grad_output).unwrap() + } + + pub fn internal_adaptive_avg_pool3d_backward_out( + &self, + out: &Tensor, + grad_output: &Tensor, + ) -> Tensor { + self.f_internal_adaptive_avg_pool3d_backward_out(out, grad_output).unwrap() + } + + pub fn internal_adaptive_avg_pool3d_out( + &self, + out: &Tensor, + output_size: impl IntList, + ) -> Tensor { + self.f_internal_adaptive_avg_pool3d_out(out, output_size).unwrap() + } + + pub fn internal_add_batch_dim(&self, batch_dim: i64, level: i64) -> Tensor { + self.f_internal_add_batch_dim(batch_dim, level).unwrap() + } + + pub fn internal_add_relu(&self, other: &Tensor) -> Tensor { + self.f_internal_add_relu(other).unwrap() + } + + pub fn internal_add_relu_(&mut self, other: &Tensor) -> Tensor { + self.f_internal_add_relu_(other).unwrap() + } + + pub fn internal_add_relu_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_internal_add_relu_out(out, other).unwrap() + } + + pub fn internal_add_relu_scalar>(&self, other: S) -> Tensor { + self.f_internal_add_relu_scalar(other).unwrap() + } + + pub fn internal_add_relu_scalar_>(&mut self, other: S) -> Tensor { + self.f_internal_add_relu_scalar_(other).unwrap() + } + + pub fn internal_add_relu_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_internal_add_relu_scalar_out(out, other).unwrap() + } + + pub fn internal_addmm_activation( + &self, + mat1: &Tensor, + mat2: &Tensor, + use_gelu: bool, + ) -> Tensor { + self.f_internal_addmm_activation(mat1, mat2, use_gelu).unwrap() + } + + pub fn internal_addmm_activation_out( + &self, + out: &Tensor, + mat1: &Tensor, + mat2: &Tensor, + use_gelu: bool, + ) -> Tensor { + self.f_internal_addmm_activation_out(out, mat1, mat2, use_gelu).unwrap() + } + + pub fn internal_aminmax(&self) -> (Tensor, Tensor) { + self.f_internal_aminmax().unwrap() + } + + pub fn internal_aminmax_dim(&self, dim: i64, keepdim: bool) -> (Tensor, Tensor) { + self.f_internal_aminmax_dim(dim, keepdim).unwrap() + } + + pub fn internal_aminmax_dim_out( + &self, + out0: &Tensor, + out1: &Tensor, + dim: i64, + keepdim: bool, + ) -> (Tensor, Tensor) { + self.f_internal_aminmax_dim_out(out0, out1, dim, keepdim).unwrap() + } + + pub fn internal_aminmax_out(&self, out0: &Tensor, out1: &Tensor) -> (Tensor, Tensor) { + self.f_internal_aminmax_out(out0, out1).unwrap() + } + + pub fn internal_amp_update_scale( + &self, + growth_tracker: &Tensor, + found_inf: &Tensor, + scale_growth_factor: f64, + scale_backoff_factor: f64, + growth_interval: i64, + ) -> (Tensor, Tensor) { + self.f_internal_amp_update_scale( + growth_tracker, + found_inf, + scale_growth_factor, + scale_backoff_factor, + growth_interval, + ) + .unwrap() + } + + pub fn internal_amp_update_scale_( + &mut self, + growth_tracker: &Tensor, + found_inf: &Tensor, + scale_growth_factor: f64, + scale_backoff_factor: f64, + growth_interval: i64, + ) -> Tensor { + self.f_internal_amp_update_scale_( + growth_tracker, + found_inf, + scale_growth_factor, + scale_backoff_factor, + growth_interval, + ) + .unwrap() + } + + pub fn internal_amp_update_scale_out( + &self, + out: &Tensor, + growth_tracker: &Tensor, + found_inf: &Tensor, + scale_growth_factor: f64, + scale_backoff_factor: f64, + growth_interval: i64, + ) -> Tensor { + self.f_internal_amp_update_scale_out( + out, + growth_tracker, + found_inf, + scale_growth_factor, + scale_backoff_factor, + growth_interval, + ) + .unwrap() + } + + pub fn internal_assert_scalar>(self_scalar: S, assert_msg: &str) { + Tensor::f_internal_assert_scalar(self_scalar, assert_msg).unwrap() + } + + pub fn internal_assert_tensor_metadata( + a: &Tensor, + size: impl IntListOption, + stride: impl IntListOption, + dtype: impl Into>, + device: Device, + layout: Option, + ) { + Tensor::f_internal_assert_tensor_metadata(a, size, stride, dtype, device, layout).unwrap() + } + + pub fn internal_autocast_to_full_precision( + &self, + cuda_enabled: bool, + cpu_enabled: bool, + ) -> Tensor { + self.f_internal_autocast_to_full_precision(cuda_enabled, cpu_enabled).unwrap() + } + + pub fn internal_autocast_to_reduced_precision( + &self, + cuda_enabled: bool, + cpu_enabled: bool, + cuda_dtype: Kind, + cpu_dtype: Kind, + ) -> Tensor { + self.f_internal_autocast_to_reduced_precision( + cuda_enabled, + cpu_enabled, + cuda_dtype, + cpu_dtype, + ) + .unwrap() + } + + pub fn internal_batch_norm_no_update>( + &self, + weight: Option, + bias: Option, + running_mean: Option, + running_var: Option, + momentum: f64, + eps: f64, + ) -> (Tensor, Tensor, Tensor, Tensor) { + self.f_internal_batch_norm_no_update(weight, bias, running_mean, running_var, momentum, eps) + .unwrap() + } + + pub fn internal_batch_norm_no_update_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + out3: &Tensor, + weight: Option, + bias: Option, + running_mean: Option, + running_var: Option, + momentum: f64, + eps: f64, + ) -> (Tensor, Tensor, Tensor, Tensor) { + self.f_internal_batch_norm_no_update_out( + out0, + out1, + out2, + out3, + weight, + bias, + running_mean, + running_var, + momentum, + eps, + ) + .unwrap() + } + + pub fn internal_batch_norm_with_update>( + &self, + weight: Option, + bias: Option, + running_mean: &Tensor, + running_var: &Tensor, + momentum: f64, + eps: f64, + ) -> (Tensor, Tensor, Tensor, Tensor) { + self.f_internal_batch_norm_with_update( + weight, + bias, + running_mean, + running_var, + momentum, + eps, + ) + .unwrap() + } + + pub fn internal_batch_norm_with_update_functional>( + &self, + weight: Option, + bias: Option, + running_mean: &Tensor, + running_var: &Tensor, + momentum: f64, + eps: f64, + ) -> (Tensor, Tensor, Tensor, Tensor, Tensor, Tensor) { + self.f_internal_batch_norm_with_update_functional( + weight, + bias, + running_mean, + running_var, + momentum, + eps, + ) + .unwrap() + } + + pub fn internal_batch_norm_with_update_out>( + &self, + out: &Tensor, + save_mean: &Tensor, + save_invstd: &Tensor, + reserve: &Tensor, + weight: Option, + bias: Option, + running_mean: &Tensor, + running_var: &Tensor, + momentum: f64, + eps: f64, + ) -> (Tensor, Tensor, Tensor, Tensor) { + self.f_internal_batch_norm_with_update_out( + out, + save_mean, + save_invstd, + reserve, + weight, + bias, + running_mean, + running_var, + momentum, + eps, + ) + .unwrap() + } + + pub fn internal_cast_byte(&self, non_blocking: bool) -> Tensor { + self.f_internal_cast_byte(non_blocking).unwrap() + } + + pub fn internal_cast_char(&self, non_blocking: bool) -> Tensor { + self.f_internal_cast_char(non_blocking).unwrap() + } + + pub fn internal_cast_double(&self, non_blocking: bool) -> Tensor { + self.f_internal_cast_double(non_blocking).unwrap() + } + + pub fn internal_cast_float(&self, non_blocking: bool) -> Tensor { + self.f_internal_cast_float(non_blocking).unwrap() + } + + pub fn internal_cast_half(&self, non_blocking: bool) -> Tensor { + self.f_internal_cast_half(non_blocking).unwrap() + } + + pub fn internal_cast_int(&self, non_blocking: bool) -> Tensor { + self.f_internal_cast_int(non_blocking).unwrap() + } + + pub fn internal_cast_long(&self, non_blocking: bool) -> Tensor { + self.f_internal_cast_long(non_blocking).unwrap() + } + + pub fn internal_cast_short(&self, non_blocking: bool) -> Tensor { + self.f_internal_cast_short(non_blocking).unwrap() + } + + pub fn internal_cdist_backward( + grad: &Tensor, + x1: &Tensor, + x2: &Tensor, + p: f64, + cdist: &Tensor, + ) -> Tensor { + Tensor::f_internal_cdist_backward(grad, x1, x2, p, cdist).unwrap() + } + + pub fn internal_cdist_backward_out( + out: &Tensor, + grad: &Tensor, + x1: &Tensor, + x2: &Tensor, + p: f64, + cdist: &Tensor, + ) -> Tensor { + Tensor::f_internal_cdist_backward_out(out, grad, x1, x2, p, cdist).unwrap() + } + + pub fn internal_cholesky_solve_helper(&self, a: &Tensor, upper: bool) -> Tensor { + self.f_internal_cholesky_solve_helper(a, upper).unwrap() + } + + pub fn internal_cholesky_solve_helper_out( + &self, + out: &Tensor, + a: &Tensor, + upper: bool, + ) -> Tensor { + self.f_internal_cholesky_solve_helper_out(out, a, upper).unwrap() + } + + pub fn internal_chunk_cat>( + tensors: &[T], + dim: i64, + num_chunks: i64, + ) -> Tensor { + Tensor::f_internal_chunk_cat(tensors, dim, num_chunks).unwrap() + } + + pub fn internal_chunk_cat_out>( + out: &Tensor, + tensors: &[T], + dim: i64, + num_chunks: i64, + ) -> Tensor { + Tensor::f_internal_chunk_cat_out(out, tensors, dim, num_chunks).unwrap() + } + + pub fn internal_coalesce(&self) -> Tensor { + self.f_internal_coalesce().unwrap() + } + + pub fn internal_coalesce_out(&self, out: &Tensor) -> Tensor { + self.f_internal_coalesce_out(out).unwrap() + } + + pub fn internal_coalesced(&self, coalesced: bool) -> Tensor { + self.f_internal_coalesced(coalesced).unwrap() + } + + pub fn internal_coalesced_(&mut self, coalesced: bool) -> Tensor { + self.f_internal_coalesced_(coalesced).unwrap() + } + + pub fn internal_coalesced_out(&self, out: &Tensor, coalesced: bool) -> Tensor { + self.f_internal_coalesced_out(out, coalesced).unwrap() + } + + pub fn internal_compute_linear_combination(&self, coefficients: &Tensor) -> Tensor { + self.f_internal_compute_linear_combination(coefficients).unwrap() + } + + pub fn internal_compute_linear_combination_out( + &self, + out: &Tensor, + coefficients: &Tensor, + ) -> Tensor { + self.f_internal_compute_linear_combination_out(out, coefficients).unwrap() + } + + pub fn internal_conj(&self) -> Tensor { + self.f_internal_conj().unwrap() + } + + pub fn internal_conj_copy(&self) -> Tensor { + self.f_internal_conj_copy().unwrap() + } + + pub fn internal_conj_copy_out(&self, out: &Tensor) -> Tensor { + self.f_internal_conj_copy_out(out).unwrap() + } + + pub fn internal_conj_physical(&self) -> Tensor { + self.f_internal_conj_physical().unwrap() + } + + pub fn internal_conj_physical_out(&self, out: &Tensor) -> Tensor { + self.f_internal_conj_physical_out(out).unwrap() + } + + pub fn internal_conv_depthwise2d>( + &self, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ) -> Tensor { + self.f_internal_conv_depthwise2d(weight, kernel_size, bias, stride, padding, dilation) + .unwrap() + } + + pub fn internal_conv_depthwise2d_out>( + &self, + out: &Tensor, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ) -> Tensor { + self.f_internal_conv_depthwise2d_out( + out, + weight, + kernel_size, + bias, + stride, + padding, + dilation, + ) + .unwrap() + } + + pub fn internal_convert_indices_from_coo_to_csr(&self, size: i64, out_int32: bool) -> Tensor { + self.f_internal_convert_indices_from_coo_to_csr(size, out_int32).unwrap() + } + + pub fn internal_convert_indices_from_coo_to_csr_out( + &self, + out: &Tensor, + size: i64, + out_int32: bool, + ) -> Tensor { + self.f_internal_convert_indices_from_coo_to_csr_out(out, size, out_int32).unwrap() + } + + pub fn internal_convert_indices_from_csr_to_coo( + crow_indices: &Tensor, + col_indices: &Tensor, + out_int32: bool, + transpose: bool, + ) -> Tensor { + Tensor::f_internal_convert_indices_from_csr_to_coo( + crow_indices, + col_indices, + out_int32, + transpose, + ) + .unwrap() + } + + pub fn internal_convert_indices_from_csr_to_coo_out( + out: &Tensor, + crow_indices: &Tensor, + col_indices: &Tensor, + out_int32: bool, + transpose: bool, + ) -> Tensor { + Tensor::f_internal_convert_indices_from_csr_to_coo_out( + out, + crow_indices, + col_indices, + out_int32, + transpose, + ) + .unwrap() + } + + pub fn internal_convert_weight_to_int4pack(&self, innerktiles: i64) -> Tensor { + self.f_internal_convert_weight_to_int4pack(innerktiles).unwrap() + } + + pub fn internal_convert_weight_to_int4pack_for_cpu(&self, innerktiles: i64) -> Tensor { + self.f_internal_convert_weight_to_int4pack_for_cpu(innerktiles).unwrap() + } + + pub fn internal_convolution>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + transposed: bool, + output_padding: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + cudnn_enabled: bool, + allow_tf32: bool, + ) -> Tensor { + self.f_internal_convolution( + weight, + bias, + stride, + padding, + dilation, + transposed, + output_padding, + groups, + benchmark, + deterministic, + cudnn_enabled, + allow_tf32, + ) + .unwrap() + } + + pub fn internal_convolution_deprecated>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + transposed: bool, + output_padding: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + cudnn_enabled: bool, + ) -> Tensor { + self.f_internal_convolution_deprecated( + weight, + bias, + stride, + padding, + dilation, + transposed, + output_padding, + groups, + benchmark, + deterministic, + cudnn_enabled, + ) + .unwrap() + } + + pub fn internal_convolution_mode>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: &str, + dilation: impl IntList, + groups: i64, + ) -> Tensor { + self.f_internal_convolution_mode(weight, bias, stride, padding, dilation, groups).unwrap() + } + + pub fn internal_convolution_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + transposed: bool, + output_padding: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + cudnn_enabled: bool, + allow_tf32: bool, + ) -> Tensor { + self.f_internal_convolution_out( + out, + weight, + bias, + stride, + padding, + dilation, + transposed, + output_padding, + groups, + benchmark, + deterministic, + cudnn_enabled, + allow_tf32, + ) + .unwrap() + } + + pub fn internal_copy_from(&self, dst: &Tensor, non_blocking: bool) -> Tensor { + self.f_internal_copy_from(dst, non_blocking).unwrap() + } + + pub fn internal_copy_from_and_resize(&self, dst: &Tensor) -> Tensor { + self.f_internal_copy_from_and_resize(dst).unwrap() + } + + pub fn internal_copy_from_and_resize_out(&self, out: &Tensor, dst: &Tensor) -> Tensor { + self.f_internal_copy_from_and_resize_out(out, dst).unwrap() + } + + pub fn internal_copy_from_out(&self, out: &Tensor, dst: &Tensor, non_blocking: bool) -> Tensor { + self.f_internal_copy_from_out(out, dst, non_blocking).unwrap() + } + + pub fn internal_cslt_compress(&self) -> Tensor { + self.f_internal_cslt_compress().unwrap() + } + + pub fn internal_cslt_sparse_mm>( + compressed_a: &Tensor, + dense_b: &Tensor, + bias: Option, + alpha: Option, + out_dtype: impl Into>, + transpose_result: bool, + alg_id: i64, + split_k: i64, + split_k_mode: i64, + ) -> Tensor { + Tensor::f_internal_cslt_sparse_mm( + compressed_a, + dense_b, + bias, + alpha, + out_dtype, + transpose_result, + alg_id, + split_k, + split_k_mode, + ) + .unwrap() + } + + pub fn internal_cslt_sparse_mm_search>( + compressed_a: &Tensor, + dense_b: &Tensor, + bias: Option, + alpha: Option, + out_dtype: impl Into>, + transpose_result: bool, + ) -> i64 { + Tensor::f_internal_cslt_sparse_mm_search( + compressed_a, + dense_b, + bias, + alpha, + out_dtype, + transpose_result, + ) + .unwrap() + } + + pub fn internal_ctc_loss( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: impl IntList, + target_lengths: impl IntList, + blank: i64, + zero_infinity: bool, + ) -> (Tensor, Tensor) { + Tensor::f_internal_ctc_loss( + log_probs, + targets, + input_lengths, + target_lengths, + blank, + zero_infinity, + ) + .unwrap() + } + + pub fn internal_ctc_loss_backward( + grad: &Tensor, + log_probs: &Tensor, + targets: &Tensor, + input_lengths: impl IntList, + target_lengths: impl IntList, + neg_log_likelihood: &Tensor, + log_alpha: &Tensor, + blank: i64, + zero_infinity: bool, + ) -> Tensor { + Tensor::f_internal_ctc_loss_backward( + grad, + log_probs, + targets, + input_lengths, + target_lengths, + neg_log_likelihood, + log_alpha, + blank, + zero_infinity, + ) + .unwrap() + } + + pub fn internal_ctc_loss_backward_out( + out: &Tensor, + grad: &Tensor, + log_probs: &Tensor, + targets: &Tensor, + input_lengths: impl IntList, + target_lengths: impl IntList, + neg_log_likelihood: &Tensor, + log_alpha: &Tensor, + blank: i64, + zero_infinity: bool, + ) -> Tensor { + Tensor::f_internal_ctc_loss_backward_out( + out, + grad, + log_probs, + targets, + input_lengths, + target_lengths, + neg_log_likelihood, + log_alpha, + blank, + zero_infinity, + ) + .unwrap() + } + + pub fn internal_ctc_loss_backward_tensor( + grad: &Tensor, + log_probs: &Tensor, + targets: &Tensor, + input_lengths: &Tensor, + target_lengths: &Tensor, + neg_log_likelihood: &Tensor, + log_alpha: &Tensor, + blank: i64, + zero_infinity: bool, + ) -> Tensor { + Tensor::f_internal_ctc_loss_backward_tensor( + grad, + log_probs, + targets, + input_lengths, + target_lengths, + neg_log_likelihood, + log_alpha, + blank, + zero_infinity, + ) + .unwrap() + } + + pub fn internal_ctc_loss_out( + out0: &Tensor, + out1: &Tensor, + log_probs: &Tensor, + targets: &Tensor, + input_lengths: impl IntList, + target_lengths: impl IntList, + blank: i64, + zero_infinity: bool, + ) -> (Tensor, Tensor) { + Tensor::f_internal_ctc_loss_out( + out0, + out1, + log_probs, + targets, + input_lengths, + target_lengths, + blank, + zero_infinity, + ) + .unwrap() + } + + pub fn internal_ctc_loss_tensor( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: &Tensor, + target_lengths: &Tensor, + blank: i64, + zero_infinity: bool, + ) -> (Tensor, Tensor) { + Tensor::f_internal_ctc_loss_tensor( + log_probs, + targets, + input_lengths, + target_lengths, + blank, + zero_infinity, + ) + .unwrap() + } + + pub fn internal_ctc_loss_tensor_out( + out0: &Tensor, + out1: &Tensor, + log_probs: &Tensor, + targets: &Tensor, + input_lengths: &Tensor, + target_lengths: &Tensor, + blank: i64, + zero_infinity: bool, + ) -> (Tensor, Tensor) { + Tensor::f_internal_ctc_loss_tensor_out( + out0, + out1, + log_probs, + targets, + input_lengths, + target_lengths, + blank, + zero_infinity, + ) + .unwrap() + } + + pub fn internal_cudnn_attention_backward( + grad_out: &Tensor, + query: &Tensor, + key: &Tensor, + value: &Tensor, + out: &Tensor, + logsumexp: &Tensor, + philox_seed: &Tensor, + philox_offset: &Tensor, + attn_bias: &Tensor, + cum_seq_q: &Tensor, + cum_seq_k: &Tensor, + max_q: i64, + max_k: i64, + dropout_p: f64, + is_causal: bool, + scale: impl Into>, + ) -> (Tensor, Tensor, Tensor) { + Tensor::f_internal_cudnn_attention_backward( + grad_out, + query, + key, + value, + out, + logsumexp, + philox_seed, + philox_offset, + attn_bias, + cum_seq_q, + cum_seq_k, + max_q, + max_k, + dropout_p, + is_causal, + scale, + ) + .unwrap() + } + + pub fn internal_cudnn_ctc_loss( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: impl IntList, + target_lengths: impl IntList, + blank: i64, + deterministic: bool, + zero_infinity: bool, + ) -> (Tensor, Tensor) { + Tensor::f_internal_cudnn_ctc_loss( + log_probs, + targets, + input_lengths, + target_lengths, + blank, + deterministic, + zero_infinity, + ) + .unwrap() + } + + pub fn internal_cudnn_ctc_loss_out( + out0: &Tensor, + out1: &Tensor, + log_probs: &Tensor, + targets: &Tensor, + input_lengths: impl IntList, + target_lengths: impl IntList, + blank: i64, + deterministic: bool, + zero_infinity: bool, + ) -> (Tensor, Tensor) { + Tensor::f_internal_cudnn_ctc_loss_out( + out0, + out1, + log_probs, + targets, + input_lengths, + target_lengths, + blank, + deterministic, + zero_infinity, + ) + .unwrap() + } + + pub fn internal_cudnn_ctc_loss_tensor( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: &Tensor, + target_lengths: &Tensor, + blank: i64, + deterministic: bool, + zero_infinity: bool, + ) -> (Tensor, Tensor) { + Tensor::f_internal_cudnn_ctc_loss_tensor( + log_probs, + targets, + input_lengths, + target_lengths, + blank, + deterministic, + zero_infinity, + ) + .unwrap() + } + + pub fn internal_cudnn_init_dropout_state( + dropout: f64, + train: bool, + dropout_seed: i64, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_internal_cudnn_init_dropout_state(dropout, train, dropout_seed, options).unwrap() + } + + pub fn internal_cudnn_init_dropout_state_out( + out: &Tensor, + dropout: f64, + train: bool, + dropout_seed: i64, + ) -> Tensor { + Tensor::f_internal_cudnn_init_dropout_state_out(out, dropout, train, dropout_seed).unwrap() + } + + pub fn internal_cudnn_rnn>( + &self, + weight: &[T], + weight_stride0: i64, + weight_buf: Option, + hx: &Tensor, + cx: Option, + mode: i64, + hidden_size: i64, + proj_size: i64, + num_layers: i64, + batch_first: bool, + dropout: f64, + train: bool, + bidirectional: bool, + batch_sizes: impl IntList, + dropout_state: Option, + ) -> (Tensor, Tensor, Tensor, Tensor, Tensor) { + self.f_internal_cudnn_rnn( + weight, + weight_stride0, + weight_buf, + hx, + cx, + mode, + hidden_size, + proj_size, + num_layers, + batch_first, + dropout, + train, + bidirectional, + batch_sizes, + dropout_state, + ) + .unwrap() + } + + pub fn internal_cudnn_rnn_flatten_weight>( + weight_arr: &[T], + weight_stride0: i64, + input_size: i64, + mode: i64, + hidden_size: i64, + proj_size: i64, + num_layers: i64, + batch_first: bool, + bidirectional: bool, + ) -> Tensor { + Tensor::f_internal_cudnn_rnn_flatten_weight( + weight_arr, + weight_stride0, + input_size, + mode, + hidden_size, + proj_size, + num_layers, + batch_first, + bidirectional, + ) + .unwrap() + } + + pub fn internal_cudnn_rnn_flatten_weight_out>( + out: &Tensor, + weight_arr: &[T], + weight_stride0: i64, + input_size: i64, + mode: i64, + hidden_size: i64, + proj_size: i64, + num_layers: i64, + batch_first: bool, + bidirectional: bool, + ) -> Tensor { + Tensor::f_internal_cudnn_rnn_flatten_weight_out( + out, + weight_arr, + weight_stride0, + input_size, + mode, + hidden_size, + proj_size, + num_layers, + batch_first, + bidirectional, + ) + .unwrap() + } + + pub fn internal_cudnn_rnn_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + out3: &Tensor, + out4: &Tensor, + weight: &[T], + weight_stride0: i64, + weight_buf: Option, + hx: &Tensor, + cx: Option, + mode: i64, + hidden_size: i64, + proj_size: i64, + num_layers: i64, + batch_first: bool, + dropout: f64, + train: bool, + bidirectional: bool, + batch_sizes: impl IntList, + dropout_state: Option, + ) -> (Tensor, Tensor, Tensor, Tensor, Tensor) { + self.f_internal_cudnn_rnn_out( + out0, + out1, + out2, + out3, + out4, + weight, + weight_stride0, + weight_buf, + hx, + cx, + mode, + hidden_size, + proj_size, + num_layers, + batch_first, + dropout, + train, + bidirectional, + batch_sizes, + dropout_state, + ) + .unwrap() + } + + pub fn internal_debug_has_internal_overlap(&self) -> i64 { + self.f_internal_debug_has_internal_overlap().unwrap() + } + + pub fn internal_dim_arange(like: &Tensor, dim: i64) -> Tensor { + Tensor::f_internal_dim_arange(like, dim).unwrap() + } + + pub fn internal_dimi(&self) -> i64 { + self.f_internal_dimi().unwrap() + } + + pub fn internal_dimv(&self) -> i64 { + self.f_internal_dimv().unwrap() + } + + pub fn internal_dirichlet_grad(x: &Tensor, alpha: &Tensor, total: &Tensor) -> Tensor { + Tensor::f_internal_dirichlet_grad(x, alpha, total).unwrap() + } + + pub fn internal_dirichlet_grad_out( + out: &Tensor, + x: &Tensor, + alpha: &Tensor, + total: &Tensor, + ) -> Tensor { + Tensor::f_internal_dirichlet_grad_out(out, x, alpha, total).unwrap() + } + + pub fn internal_dyn_quant_matmul_4bit( + inp: &Tensor, + packed_weights: &Tensor, + block_size: i64, + in_features: i64, + out_features: i64, + ) -> Tensor { + Tensor::f_internal_dyn_quant_matmul_4bit( + inp, + packed_weights, + block_size, + in_features, + out_features, + ) + .unwrap() + } + + pub fn internal_dyn_quant_pack_4bit_weight>( + weights: &Tensor, + scales_zeros: &Tensor, + bias: Option, + block_size: i64, + in_features: i64, + out_features: i64, + ) -> Tensor { + Tensor::f_internal_dyn_quant_pack_4bit_weight( + weights, + scales_zeros, + bias, + block_size, + in_features, + out_features, + ) + .unwrap() + } + + pub fn internal_efficient_attention_backward>( + grad_out_: &Tensor, + query: &Tensor, + key: &Tensor, + value: &Tensor, + bias: Option, + out: &Tensor, + cu_seqlens_q: Option, + cu_seqlens_k: Option, + max_seqlen_q: i64, + max_seqlen_k: i64, + logsumexp: &Tensor, + dropout_p: f64, + philox_seed: &Tensor, + philox_offset: &Tensor, + custom_mask_type: i64, + bias_requires_grad: bool, + scale: impl Into>, + num_splits_key: impl Into>, + window_size: impl Into>, + shared_storage_dqdkdv: bool, + ) -> (Tensor, Tensor, Tensor, Tensor) { + Tensor::f_internal_efficient_attention_backward( + grad_out_, + query, + key, + value, + bias, + out, + cu_seqlens_q, + cu_seqlens_k, + max_seqlen_q, + max_seqlen_k, + logsumexp, + dropout_p, + philox_seed, + philox_offset, + custom_mask_type, + bias_requires_grad, + scale, + num_splits_key, + window_size, + shared_storage_dqdkdv, + ) + .unwrap() + } + + pub fn internal_efficientzerotensor(size: impl IntList, options: (Kind, Device)) -> Tensor { + Tensor::f_internal_efficientzerotensor(size, options).unwrap() + } + + pub fn internal_efficientzerotensor_out(out: &Tensor, size: impl IntList) -> Tensor { + Tensor::f_internal_efficientzerotensor_out(out, size).unwrap() + } + + pub fn internal_embedding_bag>( + weight: &Tensor, + indices: &Tensor, + offsets: &Tensor, + scale_grad_by_freq: bool, + mode: i64, + sparse: bool, + per_sample_weights: Option, + include_last_offset: bool, + padding_idx: i64, + ) -> (Tensor, Tensor, Tensor, Tensor) { + Tensor::f_internal_embedding_bag( + weight, + indices, + offsets, + scale_grad_by_freq, + mode, + sparse, + per_sample_weights, + include_last_offset, + padding_idx, + ) + .unwrap() + } + + pub fn internal_embedding_bag_backward>( + grad: &Tensor, + indices: &Tensor, + offsets: &Tensor, + offset2bag: &Tensor, + bag_size: &Tensor, + maximum_indices: &Tensor, + num_weights: i64, + scale_grad_by_freq: bool, + mode: i64, + sparse: bool, + per_sample_weights: Option, + padding_idx: i64, + ) -> Tensor { + Tensor::f_internal_embedding_bag_backward( + grad, + indices, + offsets, + offset2bag, + bag_size, + maximum_indices, + num_weights, + scale_grad_by_freq, + mode, + sparse, + per_sample_weights, + padding_idx, + ) + .unwrap() + } + + pub fn internal_embedding_bag_dense_backward>( + grad: &Tensor, + indices: &Tensor, + offset2bag: &Tensor, + bag_size: &Tensor, + maximum_indices: &Tensor, + num_weights: i64, + scale_grad_by_freq: bool, + mode: i64, + per_sample_weights: Option, + padding_idx: i64, + ) -> Tensor { + Tensor::f_internal_embedding_bag_dense_backward( + grad, + indices, + offset2bag, + bag_size, + maximum_indices, + num_weights, + scale_grad_by_freq, + mode, + per_sample_weights, + padding_idx, + ) + .unwrap() + } + + pub fn internal_embedding_bag_dense_backward_out>( + out: &Tensor, + grad: &Tensor, + indices: &Tensor, + offset2bag: &Tensor, + bag_size: &Tensor, + maximum_indices: &Tensor, + num_weights: i64, + scale_grad_by_freq: bool, + mode: i64, + per_sample_weights: Option, + padding_idx: i64, + ) -> Tensor { + Tensor::f_internal_embedding_bag_dense_backward_out( + out, + grad, + indices, + offset2bag, + bag_size, + maximum_indices, + num_weights, + scale_grad_by_freq, + mode, + per_sample_weights, + padding_idx, + ) + .unwrap() + } + + pub fn internal_embedding_bag_forward_only>( + weight: &Tensor, + indices: &Tensor, + offsets: &Tensor, + scale_grad_by_freq: bool, + mode: i64, + sparse: bool, + per_sample_weights: Option, + include_last_offset: bool, + padding_idx: i64, + ) -> (Tensor, Tensor, Tensor, Tensor) { + Tensor::f_internal_embedding_bag_forward_only( + weight, + indices, + offsets, + scale_grad_by_freq, + mode, + sparse, + per_sample_weights, + include_last_offset, + padding_idx, + ) + .unwrap() + } + + pub fn internal_embedding_bag_forward_only_out>( + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + out3: &Tensor, + weight: &Tensor, + indices: &Tensor, + offsets: &Tensor, + scale_grad_by_freq: bool, + mode: i64, + sparse: bool, + per_sample_weights: Option, + include_last_offset: bool, + padding_idx: i64, + ) -> (Tensor, Tensor, Tensor, Tensor) { + Tensor::f_internal_embedding_bag_forward_only_out( + out0, + out1, + out2, + out3, + weight, + indices, + offsets, + scale_grad_by_freq, + mode, + sparse, + per_sample_weights, + include_last_offset, + padding_idx, + ) + .unwrap() + } + + pub fn internal_embedding_bag_out>( + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + out3: &Tensor, + weight: &Tensor, + indices: &Tensor, + offsets: &Tensor, + scale_grad_by_freq: bool, + mode: i64, + sparse: bool, + per_sample_weights: Option, + include_last_offset: bool, + padding_idx: i64, + ) -> (Tensor, Tensor, Tensor, Tensor) { + Tensor::f_internal_embedding_bag_out( + out0, + out1, + out2, + out3, + weight, + indices, + offsets, + scale_grad_by_freq, + mode, + sparse, + per_sample_weights, + include_last_offset, + padding_idx, + ) + .unwrap() + } + + pub fn internal_embedding_bag_per_sample_weights_backward( + grad: &Tensor, + weight: &Tensor, + indices: &Tensor, + offsets: &Tensor, + offset2bag: &Tensor, + mode: i64, + padding_idx: i64, + ) -> Tensor { + Tensor::f_internal_embedding_bag_per_sample_weights_backward( + grad, + weight, + indices, + offsets, + offset2bag, + mode, + padding_idx, + ) + .unwrap() + } + + pub fn internal_embedding_bag_per_sample_weights_backward_out( + out: &Tensor, + grad: &Tensor, + weight: &Tensor, + indices: &Tensor, + offsets: &Tensor, + offset2bag: &Tensor, + mode: i64, + padding_idx: i64, + ) -> Tensor { + Tensor::f_internal_embedding_bag_per_sample_weights_backward_out( + out, + grad, + weight, + indices, + offsets, + offset2bag, + mode, + padding_idx, + ) + .unwrap() + } + + pub fn internal_embedding_bag_sparse_backward>( + grad: &Tensor, + indices: &Tensor, + offsets: &Tensor, + offset2bag: &Tensor, + bag_size: &Tensor, + num_weights: i64, + scale_grad_by_freq: bool, + mode: i64, + per_sample_weights: Option, + padding_idx: i64, + ) -> Tensor { + Tensor::f_internal_embedding_bag_sparse_backward( + grad, + indices, + offsets, + offset2bag, + bag_size, + num_weights, + scale_grad_by_freq, + mode, + per_sample_weights, + padding_idx, + ) + .unwrap() + } + + pub fn internal_empty_affine_quantized( + size: impl IntList, + options: (Kind, Device), + scale: f64, + zero_point: i64, + ) -> Tensor { + Tensor::f_internal_empty_affine_quantized(size, options, scale, zero_point).unwrap() + } + + pub fn internal_empty_affine_quantized_out( + out: &Tensor, + size: impl IntList, + scale: f64, + zero_point: i64, + ) -> Tensor { + Tensor::f_internal_empty_affine_quantized_out(out, size, scale, zero_point).unwrap() + } + + pub fn internal_empty_per_channel_affine_quantized( + size: impl IntList, + scales: &Tensor, + zero_points: &Tensor, + axis: i64, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_internal_empty_per_channel_affine_quantized( + size, + scales, + zero_points, + axis, + options, + ) + .unwrap() + } + + pub fn internal_empty_per_channel_affine_quantized_out( + out: &Tensor, + size: impl IntList, + scales: &Tensor, + zero_points: &Tensor, + axis: i64, + ) -> Tensor { + Tensor::f_internal_empty_per_channel_affine_quantized_out( + out, + size, + scales, + zero_points, + axis, + ) + .unwrap() + } + + pub fn internal_euclidean_dist(x1: &Tensor, x2: &Tensor) -> Tensor { + Tensor::f_internal_euclidean_dist(x1, x2).unwrap() + } + + pub fn internal_euclidean_dist_out(out: &Tensor, x1: &Tensor, x2: &Tensor) -> Tensor { + Tensor::f_internal_euclidean_dist_out(out, x1, x2).unwrap() + } + + pub fn internal_fake_quantize_learnable_per_channel_affine( + &self, + scale: &Tensor, + zero_point: &Tensor, + axis: i64, + quant_min: i64, + quant_max: i64, + grad_factor: f64, + ) -> Tensor { + self.f_internal_fake_quantize_learnable_per_channel_affine( + scale, + zero_point, + axis, + quant_min, + quant_max, + grad_factor, + ) + .unwrap() + } + + pub fn internal_fake_quantize_learnable_per_channel_affine_backward( + &self, + grad: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + axis: i64, + quant_min: i64, + quant_max: i64, + grad_factor: f64, + ) -> (Tensor, Tensor, Tensor) { + self.f_internal_fake_quantize_learnable_per_channel_affine_backward( + grad, + scale, + zero_point, + axis, + quant_min, + quant_max, + grad_factor, + ) + .unwrap() + } + + pub fn internal_fake_quantize_learnable_per_channel_affine_out( + &self, + out: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + axis: i64, + quant_min: i64, + quant_max: i64, + grad_factor: f64, + ) -> Tensor { + self.f_internal_fake_quantize_learnable_per_channel_affine_out( + out, + scale, + zero_point, + axis, + quant_min, + quant_max, + grad_factor, + ) + .unwrap() + } + + pub fn internal_fake_quantize_learnable_per_tensor_affine( + &self, + scale: &Tensor, + zero_point: &Tensor, + quant_min: i64, + quant_max: i64, + grad_factor: f64, + ) -> Tensor { + self.f_internal_fake_quantize_learnable_per_tensor_affine( + scale, + zero_point, + quant_min, + quant_max, + grad_factor, + ) + .unwrap() + } + + pub fn internal_fake_quantize_learnable_per_tensor_affine_backward( + &self, + grad: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + quant_min: i64, + quant_max: i64, + grad_factor: f64, + ) -> (Tensor, Tensor, Tensor) { + self.f_internal_fake_quantize_learnable_per_tensor_affine_backward( + grad, + scale, + zero_point, + quant_min, + quant_max, + grad_factor, + ) + .unwrap() + } + + pub fn internal_fake_quantize_learnable_per_tensor_affine_out( + &self, + out: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + quant_min: i64, + quant_max: i64, + grad_factor: f64, + ) -> Tensor { + self.f_internal_fake_quantize_learnable_per_tensor_affine_out( + out, + scale, + zero_point, + quant_min, + quant_max, + grad_factor, + ) + .unwrap() + } + + pub fn internal_fake_quantize_per_tensor_affine_cachemask_tensor_qparams( + &self, + scale: &Tensor, + zero_point: &Tensor, + fake_quant_enabled: &Tensor, + quant_min: i64, + quant_max: i64, + ) -> (Tensor, Tensor) { + self.f_internal_fake_quantize_per_tensor_affine_cachemask_tensor_qparams( + scale, + zero_point, + fake_quant_enabled, + quant_min, + quant_max, + ) + .unwrap() + } + + pub fn internal_fake_quantize_per_tensor_affine_cachemask_tensor_qparams_out( + &self, + out0: &Tensor, + out1: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + fake_quant_enabled: &Tensor, + quant_min: i64, + quant_max: i64, + ) -> (Tensor, Tensor) { + self.f_internal_fake_quantize_per_tensor_affine_cachemask_tensor_qparams_out( + out0, + out1, + scale, + zero_point, + fake_quant_enabled, + quant_min, + quant_max, + ) + .unwrap() + } + + pub fn internal_fft_c2c(&self, dim: impl IntList, normalization: i64, forward: bool) -> Tensor { + self.f_internal_fft_c2c(dim, normalization, forward).unwrap() + } + + pub fn internal_fft_c2c_out( + &self, + out: &Tensor, + dim: impl IntList, + normalization: i64, + forward: bool, + ) -> Tensor { + self.f_internal_fft_c2c_out(out, dim, normalization, forward).unwrap() + } + + pub fn internal_fft_c2r( + &self, + dim: impl IntList, + normalization: i64, + last_dim_size: i64, + ) -> Tensor { + self.f_internal_fft_c2r(dim, normalization, last_dim_size).unwrap() + } + + pub fn internal_fft_c2r_out( + &self, + out: &Tensor, + dim: impl IntList, + normalization: i64, + last_dim_size: i64, + ) -> Tensor { + self.f_internal_fft_c2r_out(out, dim, normalization, last_dim_size).unwrap() + } + + pub fn internal_fft_r2c( + &self, + dim: impl IntList, + normalization: i64, + onesided: bool, + ) -> Tensor { + self.f_internal_fft_r2c(dim, normalization, onesided).unwrap() + } + + pub fn internal_fft_r2c_out( + &self, + out: &Tensor, + dim: impl IntList, + normalization: i64, + onesided: bool, + ) -> Tensor { + self.f_internal_fft_r2c_out(out, dim, normalization, onesided).unwrap() + } + + pub fn internal_fill_mem_eff_dropout_mask_( + &mut self, + dropout_p: f64, + seed: i64, + offset: i64, + ) -> Tensor { + self.f_internal_fill_mem_eff_dropout_mask_(dropout_p, seed, offset).unwrap() + } + + pub fn internal_flash_attention_backward( + grad_out: &Tensor, + query: &Tensor, + key: &Tensor, + value: &Tensor, + out: &Tensor, + logsumexp: &Tensor, + cum_seq_q: &Tensor, + cum_seq_k: &Tensor, + max_q: i64, + max_k: i64, + dropout_p: f64, + is_causal: bool, + rng_state: &Tensor, + unused: &Tensor, + scale: impl Into>, + window_size_left: impl Into>, + window_size_right: impl Into>, + ) -> (Tensor, Tensor, Tensor) { + Tensor::f_internal_flash_attention_backward( + grad_out, + query, + key, + value, + out, + logsumexp, + cum_seq_q, + cum_seq_k, + max_q, + max_k, + dropout_p, + is_causal, + rng_state, + unused, + scale, + window_size_left, + window_size_right, + ) + .unwrap() + } + + pub fn internal_foobar(&self, arg1: bool, arg2: bool, arg3: bool) -> Tensor { + self.f_internal_foobar(arg1, arg2, arg3).unwrap() + } + + pub fn internal_foobar_out(&self, out: &Tensor, arg1: bool, arg2: bool, arg3: bool) -> Tensor { + self.f_internal_foobar_out(out, arg1, arg2, arg3).unwrap() + } + + pub fn internal_functional_assert_async(&self, assert_msg: &str, dep_token: &Tensor) -> Tensor { + self.f_internal_functional_assert_async(assert_msg, dep_token).unwrap() + } + + pub fn internal_functional_assert_scalar>( + self_scalar: S, + assert_msg: &str, + dep_token: &Tensor, + ) -> Tensor { + Tensor::f_internal_functional_assert_scalar(self_scalar, assert_msg, dep_token).unwrap() + } + + pub fn internal_functional_sym_constrain_range>( + size: S, + min: impl Into>, + max: impl Into>, + dep_token: &Tensor, + ) -> Tensor { + Tensor::f_internal_functional_sym_constrain_range(size, min, max, dep_token).unwrap() + } + + pub fn internal_functional_sym_constrain_range_for_size>( + size: S, + min: impl Into>, + max: impl Into>, + dep_token: &Tensor, + ) -> Tensor { + Tensor::f_internal_functional_sym_constrain_range_for_size(size, min, max, dep_token) + .unwrap() + } + + pub fn internal_fused_dropout(&self, p: f64) -> (Tensor, Tensor) { + self.f_internal_fused_dropout(p).unwrap() + } + + pub fn internal_fused_dropout_out( + &self, + out0: &Tensor, + out1: &Tensor, + p: f64, + ) -> (Tensor, Tensor) { + self.f_internal_fused_dropout_out(out0, out1, p).unwrap() + } + + pub fn internal_fused_moving_avg_obs_fq_helper( + &self, + observer_on: &Tensor, + fake_quant_on: &Tensor, + running_min: &Tensor, + running_max: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + averaging_const: f64, + quant_min: i64, + quant_max: i64, + ch_axis: i64, + per_row_fake_quant: bool, + symmetric_quant: bool, + ) -> (Tensor, Tensor) { + self.f_internal_fused_moving_avg_obs_fq_helper( + observer_on, + fake_quant_on, + running_min, + running_max, + scale, + zero_point, + averaging_const, + quant_min, + quant_max, + ch_axis, + per_row_fake_quant, + symmetric_quant, + ) + .unwrap() + } + + pub fn internal_fused_moving_avg_obs_fq_helper_functional( + &self, + observer_on: &Tensor, + fake_quant_on: &Tensor, + running_min: &Tensor, + running_max: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + averaging_const: f64, + quant_min: i64, + quant_max: i64, + ch_axis: i64, + per_row_fake_quant: bool, + symmetric_quant: bool, + ) -> (Tensor, Tensor, Tensor, Tensor, Tensor, Tensor) { + self.f_internal_fused_moving_avg_obs_fq_helper_functional( + observer_on, + fake_quant_on, + running_min, + running_max, + scale, + zero_point, + averaging_const, + quant_min, + quant_max, + ch_axis, + per_row_fake_quant, + symmetric_quant, + ) + .unwrap() + } + + pub fn internal_fused_moving_avg_obs_fq_helper_out( + &self, + out0: &Tensor, + out1: &Tensor, + observer_on: &Tensor, + fake_quant_on: &Tensor, + running_min: &Tensor, + running_max: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + averaging_const: f64, + quant_min: i64, + quant_max: i64, + ch_axis: i64, + per_row_fake_quant: bool, + symmetric_quant: bool, + ) -> (Tensor, Tensor) { + self.f_internal_fused_moving_avg_obs_fq_helper_out( + out0, + out1, + observer_on, + fake_quant_on, + running_min, + running_max, + scale, + zero_point, + averaging_const, + quant_min, + quant_max, + ch_axis, + per_row_fake_quant, + symmetric_quant, + ) + .unwrap() + } + + pub fn internal_fused_rms_norm>( + &self, + normalized_shape: impl IntList, + weight: Option, + eps: impl Into>, + ) -> (Tensor, Tensor) { + self.f_internal_fused_rms_norm(normalized_shape, weight, eps).unwrap() + } + + pub fn internal_fused_sdp_choice>( + query: &Tensor, + key: &Tensor, + value: &Tensor, + attn_mask: Option, + dropout_p: f64, + is_causal: bool, + scale: impl Into>, + enable_gqa: bool, + ) -> i64 { + Tensor::f_internal_fused_sdp_choice( + query, key, value, attn_mask, dropout_p, is_causal, scale, enable_gqa, + ) + .unwrap() + } + + pub fn internal_fw_primal(&self, level: i64) -> Tensor { + self.f_internal_fw_primal(level).unwrap() + } + + pub fn internal_fw_primal_copy(&self, level: i64) -> Tensor { + self.f_internal_fw_primal_copy(level).unwrap() + } + + pub fn internal_fw_primal_copy_out(&self, out: &Tensor, level: i64) -> Tensor { + self.f_internal_fw_primal_copy_out(out, level).unwrap() + } + + pub fn internal_gather_sparse_backward( + &self, + dim: i64, + index: &Tensor, + grad: &Tensor, + ) -> Tensor { + self.f_internal_gather_sparse_backward(dim, index, grad).unwrap() + } + + pub fn internal_grid_sampler_2d_cpu_fallback( + &self, + grid: &Tensor, + interpolation_mode: i64, + padding_mode: i64, + align_corners: bool, + ) -> Tensor { + self.f_internal_grid_sampler_2d_cpu_fallback( + grid, + interpolation_mode, + padding_mode, + align_corners, + ) + .unwrap() + } + + pub fn internal_grid_sampler_2d_cpu_fallback_backward( + &self, + grad_output: &Tensor, + grid: &Tensor, + interpolation_mode: i64, + padding_mode: i64, + align_corners: bool, + ) -> (Tensor, Tensor) { + self.f_internal_grid_sampler_2d_cpu_fallback_backward( + grad_output, + grid, + interpolation_mode, + padding_mode, + align_corners, + ) + .unwrap() + } + + pub fn internal_grid_sampler_2d_cpu_fallback_out( + &self, + out: &Tensor, + grid: &Tensor, + interpolation_mode: i64, + padding_mode: i64, + align_corners: bool, + ) -> Tensor { + self.f_internal_grid_sampler_2d_cpu_fallback_out( + out, + grid, + interpolation_mode, + padding_mode, + align_corners, + ) + .unwrap() + } + + pub fn internal_grouped_mm>( + &self, + mat2: &Tensor, + offs: Option, + bias: Option, + out_dtype: impl Into>, + ) -> Tensor { + self.f_internal_grouped_mm(mat2, offs, bias, out_dtype).unwrap() + } + + pub fn internal_has_compatible_shallow_copy_type(&self, from: &Tensor) -> bool { + self.f_internal_has_compatible_shallow_copy_type(from).unwrap() + } + + pub fn internal_has_same_storage_numel(&self, other: &Tensor) -> bool { + self.f_internal_has_same_storage_numel(other).unwrap() + } + + pub fn internal_histogramdd_bin_edges>( + &self, + bins: impl IntList, + range: impl DoubleList, + weight: Option, + density: bool, + ) -> Vec { + self.f_internal_histogramdd_bin_edges(bins, range, weight, density).unwrap() + } + + pub fn internal_histogramdd_bin_edges_out>( + &self, + out: &[T], + bins: impl IntList, + range: impl DoubleList, + weight: Option, + density: bool, + ) { + self.f_internal_histogramdd_bin_edges_out(out, bins, range, weight, density).unwrap() + } + + pub fn internal_histogramdd_from_bin_cts>( + &self, + bins: impl IntList, + range: impl DoubleList, + weight: Option, + density: bool, + ) -> Tensor { + self.f_internal_histogramdd_from_bin_cts(bins, range, weight, density).unwrap() + } + + pub fn internal_histogramdd_from_bin_cts_out>( + &self, + out: &Tensor, + bins: impl IntList, + range: impl DoubleList, + weight: Option, + density: bool, + ) -> Tensor { + self.f_internal_histogramdd_from_bin_cts_out(out, bins, range, weight, density).unwrap() + } + + pub fn internal_histogramdd_from_bin_tensors>( + &self, + bins: &[T], + weight: Option, + density: bool, + ) -> Tensor { + self.f_internal_histogramdd_from_bin_tensors(bins, weight, density).unwrap() + } + + pub fn internal_histogramdd_from_bin_tensors_out>( + &self, + out: &Tensor, + bins: &[T], + weight: Option, + density: bool, + ) -> Tensor { + self.f_internal_histogramdd_from_bin_tensors_out(out, bins, weight, density).unwrap() + } + + pub fn internal_index_put_impl>( + &self, + indices: &[Option], + values: &Tensor, + accumulate: bool, + unsafe_: bool, + ) -> Tensor { + self.f_internal_index_put_impl(indices, values, accumulate, unsafe_).unwrap() + } + + pub fn internal_index_put_impl_>( + &mut self, + indices: &[Option], + values: &Tensor, + accumulate: bool, + unsafe_: bool, + ) -> Tensor { + self.f_internal_index_put_impl_(indices, values, accumulate, unsafe_).unwrap() + } + + pub fn internal_index_put_impl_out>( + &self, + out: &Tensor, + indices: &[Option], + values: &Tensor, + accumulate: bool, + unsafe_: bool, + ) -> Tensor { + self.f_internal_index_put_impl_out(out, indices, values, accumulate, unsafe_).unwrap() + } + + pub fn internal_indices(&self) -> Tensor { + self.f_internal_indices().unwrap() + } + + pub fn internal_indices_copy(&self) -> Tensor { + self.f_internal_indices_copy().unwrap() + } + + pub fn internal_indices_copy_out(&self, out: &Tensor) -> Tensor { + self.f_internal_indices_copy_out(out).unwrap() + } + + pub fn internal_int_mm(&self, mat2: &Tensor) -> Tensor { + self.f_internal_int_mm(mat2).unwrap() + } + + pub fn internal_int_mm_out(&self, out: &Tensor, mat2: &Tensor) -> Tensor { + self.f_internal_int_mm_out(out, mat2).unwrap() + } + + pub fn internal_is_all_true(&self) -> Tensor { + self.f_internal_is_all_true().unwrap() + } + + pub fn internal_is_any_true(&self) -> Tensor { + self.f_internal_is_any_true().unwrap() + } + + pub fn internal_is_zerotensor(&self) -> bool { + self.f_internal_is_zerotensor().unwrap() + } + + pub fn internal_lazy_clone(&self) -> Tensor { + self.f_internal_lazy_clone().unwrap() + } + + pub fn internal_linalg_check_errors(info: &Tensor, api_name: &str, is_matrix: bool) { + Tensor::f_internal_linalg_check_errors(info, api_name, is_matrix).unwrap() + } + + pub fn internal_linalg_det(a: &Tensor) -> (Tensor, Tensor, Tensor) { + Tensor::f_internal_linalg_det(a).unwrap() + } + + pub fn internal_linalg_det_result( + result: &Tensor, + lu: &Tensor, + pivots: &Tensor, + a: &Tensor, + ) -> (Tensor, Tensor, Tensor) { + Tensor::f_internal_linalg_det_result(result, lu, pivots, a).unwrap() + } + + pub fn internal_linalg_eigh(a: &Tensor, uplo: &str, compute_v: bool) -> (Tensor, Tensor) { + Tensor::f_internal_linalg_eigh(a, uplo, compute_v).unwrap() + } + + pub fn internal_linalg_eigh_eigenvalues( + eigenvalues: &Tensor, + eigenvectors: &Tensor, + a: &Tensor, + uplo: &str, + compute_v: bool, + ) -> (Tensor, Tensor) { + Tensor::f_internal_linalg_eigh_eigenvalues(eigenvalues, eigenvectors, a, uplo, compute_v) + .unwrap() + } + + pub fn internal_linalg_eigvals(&self) -> Tensor { + self.f_internal_linalg_eigvals().unwrap() + } + + pub fn internal_linalg_slogdet(a: &Tensor) -> (Tensor, Tensor, Tensor, Tensor) { + Tensor::f_internal_linalg_slogdet(a).unwrap() + } + + pub fn internal_linalg_slogdet_sign( + sign: &Tensor, + logabsdet: &Tensor, + lu: &Tensor, + pivots: &Tensor, + a: &Tensor, + ) -> (Tensor, Tensor, Tensor, Tensor) { + Tensor::f_internal_linalg_slogdet_sign(sign, logabsdet, lu, pivots, a).unwrap() + } + + pub fn internal_linalg_solve_ex( + a: &Tensor, + b: &Tensor, + left: bool, + check_errors: bool, + ) -> (Tensor, Tensor, Tensor, Tensor) { + Tensor::f_internal_linalg_solve_ex(a, b, left, check_errors).unwrap() + } + + pub fn internal_linalg_solve_ex_result( + result: &Tensor, + lu: &Tensor, + pivots: &Tensor, + info: &Tensor, + a: &Tensor, + b: &Tensor, + left: bool, + check_errors: bool, + ) -> (Tensor, Tensor, Tensor, Tensor) { + Tensor::f_internal_linalg_solve_ex_result( + result, + lu, + pivots, + info, + a, + b, + left, + check_errors, + ) + .unwrap() + } + + pub fn internal_linalg_svd( + a: &Tensor, + full_matrices: bool, + compute_uv: bool, + driver: &str, + ) -> (Tensor, Tensor, Tensor) { + Tensor::f_internal_linalg_svd(a, full_matrices, compute_uv, driver).unwrap() + } + + pub fn internal_linalg_svd_u( + u: &Tensor, + s: &Tensor, + vh: &Tensor, + a: &Tensor, + full_matrices: bool, + compute_uv: bool, + driver: &str, + ) -> (Tensor, Tensor, Tensor) { + Tensor::f_internal_linalg_svd_u(u, s, vh, a, full_matrices, compute_uv, driver).unwrap() + } + + pub fn internal_log_softmax(&self, dim: i64, half_to_float: bool) -> Tensor { + self.f_internal_log_softmax(dim, half_to_float).unwrap() + } + + pub fn internal_log_softmax_backward_data( + grad_output: &Tensor, + output: &Tensor, + dim: i64, + input_dtype: Kind, + ) -> Tensor { + Tensor::f_internal_log_softmax_backward_data(grad_output, output, dim, input_dtype).unwrap() + } + + pub fn internal_log_softmax_backward_data_out( + out: &Tensor, + grad_output: &Tensor, + output: &Tensor, + dim: i64, + input_dtype: Kind, + ) -> Tensor { + Tensor::f_internal_log_softmax_backward_data_out(out, grad_output, output, dim, input_dtype) + .unwrap() + } + + pub fn internal_log_softmax_out(&self, out: &Tensor, dim: i64, half_to_float: bool) -> Tensor { + self.f_internal_log_softmax_out(out, dim, half_to_float).unwrap() + } + + pub fn internal_logcumsumexp(&self, dim: i64) -> Tensor { + self.f_internal_logcumsumexp(dim).unwrap() + } + + pub fn internal_logcumsumexp_out(&self, out: &Tensor, dim: i64) -> Tensor { + self.f_internal_logcumsumexp_out(out, dim).unwrap() + } + + pub fn internal_lstm_mps>( + &self, + hx: &[T], + params: &[T], + has_biases: bool, + num_layers: i64, + dropout: f64, + train: bool, + bidirectional: bool, + batch_first: bool, + ) -> (Tensor, Tensor, Tensor, Tensor, Tensor, Tensor) { + self.f_internal_lstm_mps( + hx, + params, + has_biases, + num_layers, + dropout, + train, + bidirectional, + batch_first, + ) + .unwrap() + } + + pub fn internal_lstm_mps_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + out3: &Tensor, + out4: &Tensor, + out5: &Tensor, + hx: &[T], + params: &[T], + has_biases: bool, + num_layers: i64, + dropout: f64, + train: bool, + bidirectional: bool, + batch_first: bool, + ) -> (Tensor, Tensor, Tensor, Tensor, Tensor, Tensor) { + self.f_internal_lstm_mps_out( + out0, + out1, + out2, + out3, + out4, + out5, + hx, + params, + has_biases, + num_layers, + dropout, + train, + bidirectional, + batch_first, + ) + .unwrap() + } + + pub fn internal_lu_with_info( + &self, + pivot: bool, + check_errors: bool, + ) -> (Tensor, Tensor, Tensor) { + self.f_internal_lu_with_info(pivot, check_errors).unwrap() + } + + pub fn internal_make_dep_token(options: (Kind, Device)) -> Tensor { + Tensor::f_internal_make_dep_token(options).unwrap() + } + + pub fn internal_make_dual(primal: &Tensor, tangent: &Tensor, level: i64) -> Tensor { + Tensor::f_internal_make_dual(primal, tangent, level).unwrap() + } + + pub fn internal_make_dual_copy(primal: &Tensor, tangent: &Tensor, level: i64) -> Tensor { + Tensor::f_internal_make_dual_copy(primal, tangent, level).unwrap() + } + + pub fn internal_make_dual_copy_out( + out: &Tensor, + primal: &Tensor, + tangent: &Tensor, + level: i64, + ) -> Tensor { + Tensor::f_internal_make_dual_copy_out(out, primal, tangent, level).unwrap() + } + + pub fn internal_make_per_channel_quantized_tensor( + &self, + scale: &Tensor, + zero_point: &Tensor, + axis: i64, + ) -> Tensor { + self.f_internal_make_per_channel_quantized_tensor(scale, zero_point, axis).unwrap() + } + + pub fn internal_make_per_channel_quantized_tensor_out( + &self, + out: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + axis: i64, + ) -> Tensor { + self.f_internal_make_per_channel_quantized_tensor_out(out, scale, zero_point, axis).unwrap() + } + + pub fn internal_make_per_tensor_quantized_tensor(&self, scale: f64, zero_point: i64) -> Tensor { + self.f_internal_make_per_tensor_quantized_tensor(scale, zero_point).unwrap() + } + + pub fn internal_make_per_tensor_quantized_tensor_out( + &self, + out: &Tensor, + scale: f64, + zero_point: i64, + ) -> Tensor { + self.f_internal_make_per_tensor_quantized_tensor_out(out, scale, zero_point).unwrap() + } + + pub fn internal_masked_scale(&self, mask: &Tensor, scale: f64) -> Tensor { + self.f_internal_masked_scale(mask, scale).unwrap() + } + + pub fn internal_masked_scale_out(&self, out: &Tensor, mask: &Tensor, scale: f64) -> Tensor { + self.f_internal_masked_scale_out(out, mask, scale).unwrap() + } + + pub fn internal_masked_softmax( + &self, + mask: &Tensor, + dim: impl Into>, + mask_type: impl Into>, + ) -> Tensor { + self.f_internal_masked_softmax(mask, dim, mask_type).unwrap() + } + + pub fn internal_masked_softmax_backward( + grad_output: &Tensor, + output: &Tensor, + mask: &Tensor, + dim: impl Into>, + ) -> Tensor { + Tensor::f_internal_masked_softmax_backward(grad_output, output, mask, dim).unwrap() + } + + pub fn internal_masked_softmax_backward_out( + out: &Tensor, + grad_output: &Tensor, + output: &Tensor, + mask: &Tensor, + dim: impl Into>, + ) -> Tensor { + Tensor::f_internal_masked_softmax_backward_out(out, grad_output, output, mask, dim).unwrap() + } + + pub fn internal_masked_softmax_out( + &self, + out: &Tensor, + mask: &Tensor, + dim: impl Into>, + mask_type: impl Into>, + ) -> Tensor { + self.f_internal_masked_softmax_out(out, mask, dim, mask_type).unwrap() + } + + pub fn internal_mixed_dtypes_linear>( + &self, + weight: &Tensor, + scale: &Tensor, + bias: Option, + activation: &str, + ) -> Tensor { + self.f_internal_mixed_dtypes_linear(weight, scale, bias, activation).unwrap() + } + + pub fn internal_mkldnn_reshape(&self, shape: impl IntList) -> Tensor { + self.f_internal_mkldnn_reshape(shape).unwrap() + } + + pub fn internal_mkldnn_reshape_out(&self, out: &Tensor, shape: impl IntList) -> Tensor { + self.f_internal_mkldnn_reshape_out(out, shape).unwrap() + } + + pub fn internal_mkldnn_transpose(&self, dim0: i64, dim1: i64) -> Tensor { + self.f_internal_mkldnn_transpose(dim0, dim1).unwrap() + } + + pub fn internal_mkldnn_transpose_(&mut self, dim0: i64, dim1: i64) -> Tensor { + self.f_internal_mkldnn_transpose_(dim0, dim1).unwrap() + } + + pub fn internal_mkldnn_transpose_out(&self, out: &Tensor, dim0: i64, dim1: i64) -> Tensor { + self.f_internal_mkldnn_transpose_out(out, dim0, dim1).unwrap() + } + + pub fn internal_mps_convolution>( + &self, + weight: &Tensor, + bias: Option, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Tensor { + self.f_internal_mps_convolution(weight, bias, padding, stride, dilation, groups).unwrap() + } + + pub fn internal_mps_convolution_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Tensor { + self.f_internal_mps_convolution_out(out, weight, bias, padding, stride, dilation, groups) + .unwrap() + } + + pub fn internal_mps_convolution_transpose( + &self, + weight: &Tensor, + padding: impl IntList, + output_padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Tensor { + self.f_internal_mps_convolution_transpose( + weight, + padding, + output_padding, + stride, + dilation, + groups, + ) + .unwrap() + } + + pub fn internal_mps_convolution_transpose_out( + &self, + out: &Tensor, + weight: &Tensor, + padding: impl IntList, + output_padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Tensor { + self.f_internal_mps_convolution_transpose_out( + out, + weight, + padding, + output_padding, + stride, + dilation, + groups, + ) + .unwrap() + } + + pub fn internal_native_batch_norm_legit>( + &self, + weight: Option, + bias: Option, + running_mean: &Tensor, + running_var: &Tensor, + training: bool, + momentum: f64, + eps: f64, + ) -> (Tensor, Tensor, Tensor) { + self.f_internal_native_batch_norm_legit( + weight, + bias, + running_mean, + running_var, + training, + momentum, + eps, + ) + .unwrap() + } + + pub fn internal_native_batch_norm_legit_functional>( + &self, + weight: Option, + bias: Option, + running_mean: &Tensor, + running_var: &Tensor, + training: bool, + momentum: f64, + eps: f64, + ) -> (Tensor, Tensor, Tensor, Tensor, Tensor) { + self.f_internal_native_batch_norm_legit_functional( + weight, + bias, + running_mean, + running_var, + training, + momentum, + eps, + ) + .unwrap() + } + + pub fn internal_native_batch_norm_legit_no_stats>( + &self, + weight: Option, + bias: Option, + training: bool, + momentum: f64, + eps: f64, + ) -> (Tensor, Tensor, Tensor) { + self.f_internal_native_batch_norm_legit_no_stats(weight, bias, training, momentum, eps) + .unwrap() + } + + pub fn internal_native_batch_norm_legit_no_stats_out>( + &self, + out: &Tensor, + save_mean: &Tensor, + save_invstd: &Tensor, + weight: Option, + bias: Option, + training: bool, + momentum: f64, + eps: f64, + ) -> (Tensor, Tensor, Tensor) { + self.f_internal_native_batch_norm_legit_no_stats_out( + out, + save_mean, + save_invstd, + weight, + bias, + training, + momentum, + eps, + ) + .unwrap() + } + + pub fn internal_native_batch_norm_legit_no_training>( + &self, + weight: Option, + bias: Option, + running_mean: &Tensor, + running_var: &Tensor, + momentum: f64, + eps: f64, + ) -> (Tensor, Tensor, Tensor) { + self.f_internal_native_batch_norm_legit_no_training( + weight, + bias, + running_mean, + running_var, + momentum, + eps, + ) + .unwrap() + } + + pub fn internal_native_batch_norm_legit_no_training_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + weight: Option, + bias: Option, + running_mean: &Tensor, + running_var: &Tensor, + momentum: f64, + eps: f64, + ) -> (Tensor, Tensor, Tensor) { + self.f_internal_native_batch_norm_legit_no_training_out( + out0, + out1, + out2, + weight, + bias, + running_mean, + running_var, + momentum, + eps, + ) + .unwrap() + } + + pub fn internal_native_batch_norm_legit_out>( + &self, + out: &Tensor, + save_mean: &Tensor, + save_invstd: &Tensor, + weight: Option, + bias: Option, + running_mean: &Tensor, + running_var: &Tensor, + training: bool, + momentum: f64, + eps: f64, + ) -> (Tensor, Tensor, Tensor) { + self.f_internal_native_batch_norm_legit_out( + out, + save_mean, + save_invstd, + weight, + bias, + running_mean, + running_var, + training, + momentum, + eps, + ) + .unwrap() + } + + pub fn internal_native_multi_head_attention>( + query: &Tensor, + key: &Tensor, + value: &Tensor, + embed_dim: i64, + num_head: i64, + qkv_weight: &Tensor, + qkv_bias: &Tensor, + proj_weight: &Tensor, + proj_bias: &Tensor, + mask: Option, + need_weights: bool, + average_attn_weights: bool, + mask_type: impl Into>, + ) -> (Tensor, Tensor) { + Tensor::f_internal_native_multi_head_attention( + query, + key, + value, + embed_dim, + num_head, + qkv_weight, + qkv_bias, + proj_weight, + proj_bias, + mask, + need_weights, + average_attn_weights, + mask_type, + ) + .unwrap() + } + + pub fn internal_native_multi_head_attention_out>( + out0: &Tensor, + out1: &Tensor, + query: &Tensor, + key: &Tensor, + value: &Tensor, + embed_dim: i64, + num_head: i64, + qkv_weight: &Tensor, + qkv_bias: &Tensor, + proj_weight: &Tensor, + proj_bias: &Tensor, + mask: Option, + need_weights: bool, + average_attn_weights: bool, + mask_type: impl Into>, + ) -> (Tensor, Tensor) { + Tensor::f_internal_native_multi_head_attention_out( + out0, + out1, + query, + key, + value, + embed_dim, + num_head, + qkv_weight, + qkv_bias, + proj_weight, + proj_bias, + mask, + need_weights, + average_attn_weights, + mask_type, + ) + .unwrap() + } + + pub fn internal_neg_view(&self) -> Tensor { + self.f_internal_neg_view().unwrap() + } + + pub fn internal_neg_view_copy(&self) -> Tensor { + self.f_internal_neg_view_copy().unwrap() + } + + pub fn internal_neg_view_copy_out(&self, out: &Tensor) -> Tensor { + self.f_internal_neg_view_copy_out(out).unwrap() + } + + pub fn internal_nested_compute_contiguous_strides_offsets( + nested_size: &Tensor, + ) -> (Tensor, Tensor) { + Tensor::f_internal_nested_compute_contiguous_strides_offsets(nested_size).unwrap() + } + + pub fn internal_nested_from_padded( + padded: &Tensor, + cpu_nested_shape_example: &Tensor, + fuse_transform_0213: bool, + ) -> Tensor { + Tensor::f_internal_nested_from_padded(padded, cpu_nested_shape_example, fuse_transform_0213) + .unwrap() + } + + pub fn internal_nested_from_padded_and_nested_example( + padded: &Tensor, + nt_example: &Tensor, + ) -> Tensor { + Tensor::f_internal_nested_from_padded_and_nested_example(padded, nt_example).unwrap() + } + + pub fn internal_nested_from_padded_and_nested_example_out( + out: &Tensor, + padded: &Tensor, + nt_example: &Tensor, + ) -> Tensor { + Tensor::f_internal_nested_from_padded_and_nested_example_out(out, padded, nt_example) + .unwrap() + } + + pub fn internal_nested_from_padded_out( + out: &Tensor, + padded: &Tensor, + cpu_nested_shape_example: &Tensor, + fuse_transform_0213: bool, + ) -> Tensor { + Tensor::f_internal_nested_from_padded_out( + out, + padded, + cpu_nested_shape_example, + fuse_transform_0213, + ) + .unwrap() + } + + pub fn internal_nested_from_padded_tensor>( + padded: &Tensor, + offsets: &Tensor, + dummy: &Tensor, + ragged_idx: i64, + min_seqlen: Option, + max_seqlen: Option, + sum_s: impl Into>, + ) -> Tensor { + Tensor::f_internal_nested_from_padded_tensor( + padded, offsets, dummy, ragged_idx, min_seqlen, max_seqlen, sum_s, + ) + .unwrap() + } + + pub fn internal_nested_get_jagged_dummy(any: &Tensor) -> Tensor { + Tensor::f_internal_nested_get_jagged_dummy(any).unwrap() + } + + pub fn internal_nested_get_lengths(&self) -> Tensor { + self.f_internal_nested_get_lengths().unwrap() + } + + pub fn internal_nested_get_max_seqlen(&self) -> Tensor { + self.f_internal_nested_get_max_seqlen().unwrap() + } + + pub fn internal_nested_get_min_seqlen(&self) -> Tensor { + self.f_internal_nested_get_min_seqlen().unwrap() + } + + pub fn internal_nested_get_offsets(&self) -> Tensor { + self.f_internal_nested_get_offsets().unwrap() + } + + pub fn internal_nested_get_ragged_idx(&self) -> i64 { + self.f_internal_nested_get_ragged_idx().unwrap() + } + + pub fn internal_nested_get_values(&self) -> Tensor { + self.f_internal_nested_get_values().unwrap() + } + + pub fn internal_nested_get_values_copy(&self) -> Tensor { + self.f_internal_nested_get_values_copy().unwrap() + } + + pub fn internal_nested_get_values_copy_out(&self, out: &Tensor) -> Tensor { + self.f_internal_nested_get_values_copy_out(out).unwrap() + } + + pub fn internal_nested_select_backward( + &self, + grad_output: &Tensor, + dim: i64, + index: i64, + ) -> Tensor { + self.f_internal_nested_select_backward(grad_output, dim, index).unwrap() + } + + pub fn internal_nested_sum_backward( + &self, + grad: &Tensor, + dim: impl IntListOption, + keepdim: bool, + ) -> Tensor { + self.f_internal_nested_sum_backward(grad, dim, keepdim).unwrap() + } + + pub fn internal_nested_view_from_buffer( + &self, + nested_size: &Tensor, + nested_strides: &Tensor, + offsets: &Tensor, + ) -> Tensor { + self.f_internal_nested_view_from_buffer(nested_size, nested_strides, offsets).unwrap() + } + + pub fn internal_nested_view_from_buffer_copy( + &self, + nested_size: &Tensor, + nested_strides: &Tensor, + offsets: &Tensor, + ) -> Tensor { + self.f_internal_nested_view_from_buffer_copy(nested_size, nested_strides, offsets).unwrap() + } + + pub fn internal_nested_view_from_buffer_copy_out( + &self, + out: &Tensor, + nested_size: &Tensor, + nested_strides: &Tensor, + offsets: &Tensor, + ) -> Tensor { + self.f_internal_nested_view_from_buffer_copy_out(out, nested_size, nested_strides, offsets) + .unwrap() + } + + pub fn internal_nested_view_from_jagged>( + &self, + offsets: &Tensor, + dummy: &Tensor, + lengths: Option, + ragged_idx: i64, + min_seqlen: Option, + max_seqlen: Option, + ) -> Tensor { + self.f_internal_nested_view_from_jagged( + offsets, dummy, lengths, ragged_idx, min_seqlen, max_seqlen, + ) + .unwrap() + } + + pub fn internal_nested_view_from_jagged_copy>( + &self, + offsets: &Tensor, + dummy: &Tensor, + lengths: Option, + ragged_idx: i64, + min_seqlen: Option, + max_seqlen: Option, + ) -> Tensor { + self.f_internal_nested_view_from_jagged_copy( + offsets, dummy, lengths, ragged_idx, min_seqlen, max_seqlen, + ) + .unwrap() + } + + pub fn internal_nested_view_from_jagged_copy_out>( + &self, + out: &Tensor, + offsets: &Tensor, + dummy: &Tensor, + lengths: Option, + ragged_idx: i64, + min_seqlen: Option, + max_seqlen: Option, + ) -> Tensor { + self.f_internal_nested_view_from_jagged_copy_out( + out, offsets, dummy, lengths, ragged_idx, min_seqlen, max_seqlen, + ) + .unwrap() + } + + pub fn internal_new_zeros_with_same_feature_meta( + &self, + other: &Tensor, + self_num_batch_dims: i64, + ) -> Tensor { + self.f_internal_new_zeros_with_same_feature_meta(other, self_num_batch_dims).unwrap() + } + + pub fn internal_new_zeros_with_same_feature_meta_out( + &self, + out: &Tensor, + other: &Tensor, + self_num_batch_dims: i64, + ) -> Tensor { + self.f_internal_new_zeros_with_same_feature_meta_out(out, other, self_num_batch_dims) + .unwrap() + } + + pub fn internal_nnpack_available() -> bool { + Tensor::f_internal_nnpack_available().unwrap() + } + + pub fn internal_nnpack_spatial_convolution>( + &self, + weight: &Tensor, + bias: Option, + padding: impl IntList, + stride: impl IntList, + ) -> Tensor { + self.f_internal_nnpack_spatial_convolution(weight, bias, padding, stride).unwrap() + } + + pub fn internal_nnpack_spatial_convolution_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + padding: impl IntList, + stride: impl IntList, + ) -> Tensor { + self.f_internal_nnpack_spatial_convolution_out(out, weight, bias, padding, stride).unwrap() + } + + pub fn internal_nnz(&self) -> i64 { + self.f_internal_nnz().unwrap() + } + + pub fn internal_pack_padded_sequence( + &self, + lengths: &Tensor, + batch_first: bool, + ) -> (Tensor, Tensor) { + self.f_internal_pack_padded_sequence(lengths, batch_first).unwrap() + } + + pub fn internal_pack_padded_sequence_backward( + grad: &Tensor, + input_size: impl IntList, + batch_sizes: &Tensor, + batch_first: bool, + ) -> Tensor { + Tensor::f_internal_pack_padded_sequence_backward(grad, input_size, batch_sizes, batch_first) + .unwrap() + } + + pub fn internal_pack_padded_sequence_out( + &self, + out0: &Tensor, + out1: &Tensor, + lengths: &Tensor, + batch_first: bool, + ) -> (Tensor, Tensor) { + self.f_internal_pack_padded_sequence_out(out0, out1, lengths, batch_first).unwrap() + } + + pub fn internal_pad_circular(&self, pad: impl IntList) -> Tensor { + self.f_internal_pad_circular(pad).unwrap() + } + + pub fn internal_pad_enum( + &self, + pad: impl IntList, + mode: i64, + value: impl Into>, + ) -> Tensor { + self.f_internal_pad_enum(pad, mode, value).unwrap() + } + + pub fn internal_pad_packed_sequence>( + data: &Tensor, + batch_sizes: &Tensor, + batch_first: bool, + padding_value: S, + total_length: i64, + ) -> (Tensor, Tensor) { + Tensor::f_internal_pad_packed_sequence( + data, + batch_sizes, + batch_first, + padding_value, + total_length, + ) + .unwrap() + } + + pub fn internal_pdist_backward(&self, grad: &Tensor, p: f64, pdist: &Tensor) -> Tensor { + self.f_internal_pdist_backward(grad, p, pdist).unwrap() + } + + pub fn internal_pdist_backward_out( + &self, + out: &Tensor, + grad: &Tensor, + p: f64, + pdist: &Tensor, + ) -> Tensor { + self.f_internal_pdist_backward_out(out, grad, p, pdist).unwrap() + } + + pub fn internal_pin_memory(&self, device: Device) -> Tensor { + self.f_internal_pin_memory(device).unwrap() + } + + pub fn internal_pin_memory_out(&self, out: &Tensor, device: Device) -> Tensor { + self.f_internal_pin_memory_out(out, device).unwrap() + } + + pub fn internal_prelu_kernel(&self, weight: &Tensor) -> Tensor { + self.f_internal_prelu_kernel(weight).unwrap() + } + + pub fn internal_prelu_kernel_backward( + &self, + grad_output: &Tensor, + weight: &Tensor, + ) -> (Tensor, Tensor) { + self.f_internal_prelu_kernel_backward(grad_output, weight).unwrap() + } + + pub fn internal_print(s: &str) { + Tensor::f_internal_print(s).unwrap() + } + + pub fn internal_propagate_xla_data(&self, output: &Tensor) { + self.f_internal_propagate_xla_data(output).unwrap() + } + + pub fn internal_remove_batch_dim(&self, level: i64, batch_size: i64, out_dim: i64) -> Tensor { + self.f_internal_remove_batch_dim(level, batch_size, out_dim).unwrap() + } + + pub fn internal_reshape_alias(&self, size: impl IntList, stride: impl IntList) -> Tensor { + self.f_internal_reshape_alias(size, stride).unwrap() + } + + pub fn internal_reshape_alias_copy(&self, size: impl IntList, stride: impl IntList) -> Tensor { + self.f_internal_reshape_alias_copy(size, stride).unwrap() + } + + pub fn internal_reshape_alias_copy_out( + &self, + out: &Tensor, + size: impl IntList, + stride: impl IntList, + ) -> Tensor { + self.f_internal_reshape_alias_copy_out(out, size, stride).unwrap() + } + + pub fn internal_reshape_copy(&self, size: impl IntList) -> Tensor { + self.f_internal_reshape_copy(size).unwrap() + } + + pub fn internal_reshape_from_tensor(&self, shape: &Tensor) -> Tensor { + self.f_internal_reshape_from_tensor(shape).unwrap() + } + + pub fn internal_resize_output(&self, size: impl IntList, device: Device) -> Tensor { + self.f_internal_resize_output(size, device).unwrap() + } + + pub fn internal_resize_output_(&mut self, size: impl IntList, device: Device) -> Tensor { + self.f_internal_resize_output_(size, device).unwrap() + } + + pub fn internal_resize_output_out( + &self, + out: &Tensor, + size: impl IntList, + device: Device, + ) -> Tensor { + self.f_internal_resize_output_out(out, size, device).unwrap() + } + + pub fn internal_rowwise_prune( + weight: &Tensor, + mask: &Tensor, + compressed_indices_dtype: Kind, + ) -> (Tensor, Tensor) { + Tensor::f_internal_rowwise_prune(weight, mask, compressed_indices_dtype).unwrap() + } + + pub fn internal_safe_softmax(&self, dim: i64, dtype: impl Into>) -> Tensor { + self.f_internal_safe_softmax(dim, dtype).unwrap() + } + + pub fn internal_sample_dirichlet(&self) -> Tensor { + self.f_internal_sample_dirichlet().unwrap() + } + + pub fn internal_sample_dirichlet_out(&self, out: &Tensor) -> Tensor { + self.f_internal_sample_dirichlet_out(out).unwrap() + } + + pub fn internal_saturate_weight_to_fp16(weight: &Tensor) -> Tensor { + Tensor::f_internal_saturate_weight_to_fp16(weight).unwrap() + } + + pub fn internal_scaled_dot_product_attention_math>( + query: &Tensor, + key: &Tensor, + value: &Tensor, + attn_mask: Option, + dropout_p: f64, + is_causal: bool, + dropout_mask: Option, + scale: impl Into>, + enable_gqa: bool, + ) -> (Tensor, Tensor) { + Tensor::f_internal_scaled_dot_product_attention_math( + query, + key, + value, + attn_mask, + dropout_p, + is_causal, + dropout_mask, + scale, + enable_gqa, + ) + .unwrap() + } + + pub fn internal_scaled_dot_product_attention_math_for_mps>( + query: &Tensor, + key: &Tensor, + value: &Tensor, + attn_mask: Option, + dropout_p: f64, + is_causal: bool, + dropout_mask: Option, + scale: impl Into>, + ) -> (Tensor, Tensor) { + Tensor::f_internal_scaled_dot_product_attention_math_for_mps( + query, + key, + value, + attn_mask, + dropout_p, + is_causal, + dropout_mask, + scale, + ) + .unwrap() + } + + pub fn internal_scaled_dot_product_cudnn_attention_backward( + grad_out: &Tensor, + query: &Tensor, + key: &Tensor, + value: &Tensor, + out: &Tensor, + logsumexp: &Tensor, + philox_seed: &Tensor, + philox_offset: &Tensor, + attn_bias: &Tensor, + cum_seq_q: &Tensor, + cum_seq_k: &Tensor, + max_q: i64, + max_k: i64, + dropout_p: f64, + is_causal: bool, + scale: impl Into>, + ) -> (Tensor, Tensor, Tensor) { + Tensor::f_internal_scaled_dot_product_cudnn_attention_backward( + grad_out, + query, + key, + value, + out, + logsumexp, + philox_seed, + philox_offset, + attn_bias, + cum_seq_q, + cum_seq_k, + max_q, + max_k, + dropout_p, + is_causal, + scale, + ) + .unwrap() + } + + pub fn internal_scaled_dot_product_efficient_attention>( + query: &Tensor, + key: &Tensor, + value: &Tensor, + attn_bias: Option, + compute_log_sumexp: bool, + dropout_p: f64, + is_causal: bool, + scale: impl Into>, + ) -> (Tensor, Tensor, Tensor, Tensor) { + Tensor::f_internal_scaled_dot_product_efficient_attention( + query, + key, + value, + attn_bias, + compute_log_sumexp, + dropout_p, + is_causal, + scale, + ) + .unwrap() + } + + pub fn internal_scaled_dot_product_flash_attention_backward( + grad_out: &Tensor, + query: &Tensor, + key: &Tensor, + value: &Tensor, + out: &Tensor, + logsumexp: &Tensor, + cum_seq_q: &Tensor, + cum_seq_k: &Tensor, + max_q: i64, + max_k: i64, + dropout_p: f64, + is_causal: bool, + philox_seed: &Tensor, + philox_offset: &Tensor, + scale: impl Into>, + ) -> (Tensor, Tensor, Tensor) { + Tensor::f_internal_scaled_dot_product_flash_attention_backward( + grad_out, + query, + key, + value, + out, + logsumexp, + cum_seq_q, + cum_seq_k, + max_q, + max_k, + dropout_p, + is_causal, + philox_seed, + philox_offset, + scale, + ) + .unwrap() + } + + pub fn internal_scaled_dot_product_flash_attention_for_cpu>( + query: &Tensor, + key: &Tensor, + value: &Tensor, + dropout_p: f64, + is_causal: bool, + attn_mask: Option, + scale: impl Into>, + ) -> (Tensor, Tensor) { + Tensor::f_internal_scaled_dot_product_flash_attention_for_cpu( + query, key, value, dropout_p, is_causal, attn_mask, scale, + ) + .unwrap() + } + + pub fn internal_scaled_dot_product_flash_attention_for_cpu_backward>( + grad_out: &Tensor, + query: &Tensor, + key: &Tensor, + value: &Tensor, + out: &Tensor, + logsumexp: &Tensor, + dropout_p: f64, + is_causal: bool, + attn_mask: Option, + scale: impl Into>, + ) -> (Tensor, Tensor, Tensor) { + Tensor::f_internal_scaled_dot_product_flash_attention_for_cpu_backward( + grad_out, query, key, value, out, logsumexp, dropout_p, is_causal, attn_mask, scale, + ) + .unwrap() + } + + pub fn internal_scaled_grouped_mm>( + &self, + mat2: &Tensor, + scale_a: &Tensor, + scale_b: &Tensor, + offs: Option, + bias: Option, + scale_result: Option, + out_dtype: impl Into>, + use_fast_accum: bool, + ) -> Tensor { + self.f_internal_scaled_grouped_mm( + mat2, + scale_a, + scale_b, + offs, + bias, + scale_result, + out_dtype, + use_fast_accum, + ) + .unwrap() + } + + pub fn internal_scaled_grouped_mm_v2>( + &self, + mat2: &Tensor, + scale_a: &[T], + recipe_a: impl IntList, + swizzle_a: impl IntList, + scale_b: &[T], + recipe_b: impl IntList, + swizzle_b: impl IntList, + offs: Option, + bias: Option, + out_dtype: impl Into>, + contraction_dim: impl IntList, + use_fast_accum: bool, + ) -> Tensor { + self.f_internal_scaled_grouped_mm_v2( + mat2, + scale_a, + recipe_a, + swizzle_a, + scale_b, + recipe_b, + swizzle_b, + offs, + bias, + out_dtype, + contraction_dim, + use_fast_accum, + ) + .unwrap() + } + + pub fn internal_scaled_mm>( + &self, + mat2: &Tensor, + scale_a: &Tensor, + scale_b: &Tensor, + bias: Option, + scale_result: Option, + out_dtype: impl Into>, + use_fast_accum: bool, + ) -> Tensor { + self.f_internal_scaled_mm( + mat2, + scale_a, + scale_b, + bias, + scale_result, + out_dtype, + use_fast_accum, + ) + .unwrap() + } + + pub fn internal_scaled_mm_out>( + &self, + out: &Tensor, + mat2: &Tensor, + scale_a: &Tensor, + scale_b: &Tensor, + bias: Option, + scale_result: Option, + out_dtype: impl Into>, + use_fast_accum: bool, + ) -> Tensor { + self.f_internal_scaled_mm_out( + out, + mat2, + scale_a, + scale_b, + bias, + scale_result, + out_dtype, + use_fast_accum, + ) + .unwrap() + } + + pub fn internal_scaled_mm_v2>( + &self, + mat2: &Tensor, + scale_a: &[T], + recipe_a: impl IntList, + swizzle_a: impl IntList, + scale_b: &[T], + recipe_b: impl IntList, + swizzle_b: impl IntList, + bias: Option, + out_dtype: impl Into>, + contraction_dim: impl IntList, + use_fast_accum: bool, + ) -> Tensor { + self.f_internal_scaled_mm_v2( + mat2, + scale_a, + recipe_a, + swizzle_a, + scale_b, + recipe_b, + swizzle_b, + bias, + out_dtype, + contraction_dim, + use_fast_accum, + ) + .unwrap() + } + + pub fn internal_scaled_mm_v2_out>( + &self, + out: &Tensor, + mat2: &Tensor, + scale_a: &[T], + recipe_a: impl IntList, + swizzle_a: impl IntList, + scale_b: &[T], + recipe_b: impl IntList, + swizzle_b: impl IntList, + bias: Option, + out_dtype: impl Into>, + contraction_dim: impl IntList, + use_fast_accum: bool, + ) -> Tensor { + self.f_internal_scaled_mm_v2_out( + out, + mat2, + scale_a, + recipe_a, + swizzle_a, + scale_b, + recipe_b, + swizzle_b, + bias, + out_dtype, + contraction_dim, + use_fast_accum, + ) + .unwrap() + } + + pub fn internal_scatter_reduce( + &self, + dim: i64, + index: &Tensor, + src: &Tensor, + reduce: &str, + include_self: bool, + ) -> Tensor { + self.f_internal_scatter_reduce(dim, index, src, reduce, include_self).unwrap() + } + + pub fn internal_scatter_reduce_( + &mut self, + dim: i64, + index: &Tensor, + src: &Tensor, + reduce: &str, + include_self: bool, + ) -> Tensor { + self.f_internal_scatter_reduce_(dim, index, src, reduce, include_self).unwrap() + } + + pub fn internal_scatter_reduce_two_out( + &self, + out: &Tensor, + dim: i64, + index: &Tensor, + src: &Tensor, + reduce: &str, + include_self: bool, + ) -> Tensor { + self.f_internal_scatter_reduce_two_out(out, dim, index, src, reduce, include_self).unwrap() + } + + pub fn internal_segment_reduce_backward, S: Into>( + grad: &Tensor, + output: &Tensor, + data: &Tensor, + reduce: &str, + lengths: Option, + offsets: Option, + axis: i64, + initial: S, + ) -> Tensor { + Tensor::f_internal_segment_reduce_backward( + grad, output, data, reduce, lengths, offsets, axis, initial, + ) + .unwrap() + } + + pub fn internal_segment_reduce_backward_out, S: Into>( + out: &Tensor, + grad: &Tensor, + output: &Tensor, + data: &Tensor, + reduce: &str, + lengths: Option, + offsets: Option, + axis: i64, + initial: S, + ) -> Tensor { + Tensor::f_internal_segment_reduce_backward_out( + out, grad, output, data, reduce, lengths, offsets, axis, initial, + ) + .unwrap() + } + + pub fn internal_shape_as_tensor(&self) -> Tensor { + self.f_internal_shape_as_tensor().unwrap() + } + + pub fn internal_slow_conv2d_backward( + &self, + grad_input: &Tensor, + grad_weight: &Tensor, + grad_bias: &Tensor, + grad_output: &Tensor, + weight: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ) -> (Tensor, Tensor, Tensor) { + self.f_internal_slow_conv2d_backward( + grad_input, + grad_weight, + grad_bias, + grad_output, + weight, + kernel_size, + stride, + padding, + ) + .unwrap() + } + + pub fn internal_sobol_engine_draw( + quasi: &Tensor, + n: i64, + sobolstate: &Tensor, + dimension: i64, + num_generated: i64, + dtype: impl Into>, + ) -> (Tensor, Tensor) { + Tensor::f_internal_sobol_engine_draw(quasi, n, sobolstate, dimension, num_generated, dtype) + .unwrap() + } + + pub fn internal_sobol_engine_ff_( + &mut self, + n: i64, + sobolstate: &Tensor, + dimension: i64, + num_generated: i64, + ) -> Tensor { + self.f_internal_sobol_engine_ff_(n, sobolstate, dimension, num_generated).unwrap() + } + + pub fn internal_sobol_engine_initialize_state_(&mut self, dimension: i64) -> Tensor { + self.f_internal_sobol_engine_initialize_state_(dimension).unwrap() + } + + pub fn internal_sobol_engine_scramble_(&mut self, ltm: &Tensor, dimension: i64) -> Tensor { + self.f_internal_sobol_engine_scramble_(ltm, dimension).unwrap() + } + + pub fn internal_softmax(&self, dim: i64, half_to_float: bool) -> Tensor { + self.f_internal_softmax(dim, half_to_float).unwrap() + } + + pub fn internal_softmax_backward_data( + grad_output: &Tensor, + output: &Tensor, + dim: i64, + input_dtype: Kind, + ) -> Tensor { + Tensor::f_internal_softmax_backward_data(grad_output, output, dim, input_dtype).unwrap() + } + + pub fn internal_softmax_backward_data_out( + grad_input: &Tensor, + grad_output: &Tensor, + output: &Tensor, + dim: i64, + input_dtype: Kind, + ) -> Tensor { + Tensor::f_internal_softmax_backward_data_out( + grad_input, + grad_output, + output, + dim, + input_dtype, + ) + .unwrap() + } + + pub fn internal_softmax_out(&self, out: &Tensor, dim: i64, half_to_float: bool) -> Tensor { + self.f_internal_softmax_out(out, dim, half_to_float).unwrap() + } + + pub fn internal_sparse_addmm(&self, mat1: &Tensor, mat2: &Tensor) -> Tensor { + self.f_internal_sparse_addmm(mat1, mat2).unwrap() + } + + pub fn internal_sparse_addmm_out(&self, out: &Tensor, mat1: &Tensor, mat2: &Tensor) -> Tensor { + self.f_internal_sparse_addmm_out(out, mat1, mat2).unwrap() + } + + pub fn internal_sparse_broadcast_to(&self, size: impl IntList) -> Tensor { + self.f_internal_sparse_broadcast_to(size).unwrap() + } + + pub fn internal_sparse_broadcast_to_copy(&self, size: impl IntList) -> Tensor { + self.f_internal_sparse_broadcast_to_copy(size).unwrap() + } + + pub fn internal_sparse_broadcast_to_copy_out( + &self, + out: &Tensor, + size: impl IntList, + ) -> Tensor { + self.f_internal_sparse_broadcast_to_copy_out(out, size).unwrap() + } + + pub fn internal_sparse_bsc_tensor_unsafe( + ccol_indices: &Tensor, + row_indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_internal_sparse_bsc_tensor_unsafe( + ccol_indices, + row_indices, + values, + size, + options, + ) + .unwrap() + } + + pub fn internal_sparse_bsr_tensor_unsafe( + crow_indices: &Tensor, + col_indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_internal_sparse_bsr_tensor_unsafe( + crow_indices, + col_indices, + values, + size, + options, + ) + .unwrap() + } + + pub fn internal_sparse_compressed_tensor_unsafe( + compressed_indices: &Tensor, + plain_indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_internal_sparse_compressed_tensor_unsafe( + compressed_indices, + plain_indices, + values, + size, + options, + ) + .unwrap() + } + + pub fn internal_sparse_compressed_tensor_with_dims( + nnz: i64, + dense_dim: i64, + size: impl IntList, + blocksize: impl IntList, + index_dtype: Kind, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_internal_sparse_compressed_tensor_with_dims( + nnz, + dense_dim, + size, + blocksize, + index_dtype, + options, + ) + .unwrap() + } + + pub fn internal_sparse_coo_tensor_unsafe( + indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + is_coalesced: bool, + ) -> Tensor { + Tensor::f_internal_sparse_coo_tensor_unsafe(indices, values, size, options, is_coalesced) + .unwrap() + } + + pub fn internal_sparse_coo_tensor_with_dims( + sparse_dim: i64, + dense_dim: i64, + size: impl IntList, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_internal_sparse_coo_tensor_with_dims(sparse_dim, dense_dim, size, options) + .unwrap() + } + + pub fn internal_sparse_coo_tensor_with_dims_and_tensors( + sparse_dim: i64, + dense_dim: i64, + size: impl IntList, + indices: &Tensor, + values: &Tensor, + options: (Kind, Device), + is_coalesced: bool, + ) -> Tensor { + Tensor::f_internal_sparse_coo_tensor_with_dims_and_tensors( + sparse_dim, + dense_dim, + size, + indices, + values, + options, + is_coalesced, + ) + .unwrap() + } + + pub fn internal_sparse_coo_tensor_with_dims_and_tensors_out( + out: &Tensor, + sparse_dim: i64, + dense_dim: i64, + size: impl IntList, + indices: &Tensor, + values: &Tensor, + is_coalesced: bool, + ) -> Tensor { + Tensor::f_internal_sparse_coo_tensor_with_dims_and_tensors_out( + out, + sparse_dim, + dense_dim, + size, + indices, + values, + is_coalesced, + ) + .unwrap() + } + + pub fn internal_sparse_coo_tensor_with_dims_out( + out: &Tensor, + sparse_dim: i64, + dense_dim: i64, + size: impl IntList, + ) -> Tensor { + Tensor::f_internal_sparse_coo_tensor_with_dims_out(out, sparse_dim, dense_dim, size) + .unwrap() + } + + pub fn internal_sparse_csc_tensor_unsafe( + ccol_indices: &Tensor, + row_indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_internal_sparse_csc_tensor_unsafe( + ccol_indices, + row_indices, + values, + size, + options, + ) + .unwrap() + } + + pub fn internal_sparse_csr_prod( + &self, + dim: impl IntList, + keepdim: bool, + dtype: impl Into>, + ) -> Tensor { + self.f_internal_sparse_csr_prod(dim, keepdim, dtype).unwrap() + } + + pub fn internal_sparse_csr_prod_dim_dtype_out( + &self, + out: &Tensor, + dim: impl IntList, + keepdim: bool, + dtype: impl Into>, + ) -> Tensor { + self.f_internal_sparse_csr_prod_dim_dtype_out(out, dim, keepdim, dtype).unwrap() + } + + pub fn internal_sparse_csr_sum( + &self, + dim: impl IntList, + keepdim: bool, + dtype: impl Into>, + ) -> Tensor { + self.f_internal_sparse_csr_sum(dim, keepdim, dtype).unwrap() + } + + pub fn internal_sparse_csr_sum_dim_dtype_out( + &self, + out: &Tensor, + dim: impl IntList, + keepdim: bool, + dtype: impl Into>, + ) -> Tensor { + self.f_internal_sparse_csr_sum_dim_dtype_out(out, dim, keepdim, dtype).unwrap() + } + + pub fn internal_sparse_csr_tensor_unsafe( + crow_indices: &Tensor, + col_indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_internal_sparse_csr_tensor_unsafe( + crow_indices, + col_indices, + values, + size, + options, + ) + .unwrap() + } + + pub fn internal_sparse_log_softmax(&self, dim: i64, half_to_float: bool) -> Tensor { + self.f_internal_sparse_log_softmax(dim, half_to_float).unwrap() + } + + pub fn internal_sparse_log_softmax_backward_data( + &self, + grad_output: &Tensor, + output: &Tensor, + dim: i64, + ) -> Tensor { + self.f_internal_sparse_log_softmax_backward_data(grad_output, output, dim).unwrap() + } + + pub fn internal_sparse_log_softmax_backward_data_out( + &self, + out: &Tensor, + grad_output: &Tensor, + output: &Tensor, + dim: i64, + ) -> Tensor { + self.f_internal_sparse_log_softmax_backward_data_out(out, grad_output, output, dim).unwrap() + } + + pub fn internal_sparse_log_softmax_int( + &self, + dim: i64, + dtype: impl Into>, + ) -> Tensor { + self.f_internal_sparse_log_softmax_int(dim, dtype).unwrap() + } + + pub fn internal_sparse_log_softmax_out( + &self, + out: &Tensor, + dim: i64, + half_to_float: bool, + ) -> Tensor { + self.f_internal_sparse_log_softmax_out(out, dim, half_to_float).unwrap() + } + + pub fn internal_sparse_mask_projection( + &self, + mask: &Tensor, + accumulate_matches: bool, + ) -> Tensor { + self.f_internal_sparse_mask_projection(mask, accumulate_matches).unwrap() + } + + pub fn internal_sparse_mask_projection_out( + &self, + out: &Tensor, + mask: &Tensor, + accumulate_matches: bool, + ) -> Tensor { + self.f_internal_sparse_mask_projection_out(out, mask, accumulate_matches).unwrap() + } + + pub fn internal_sparse_mm(sparse: &Tensor, dense: &Tensor) -> Tensor { + Tensor::f_internal_sparse_mm(sparse, dense).unwrap() + } + + pub fn internal_sparse_mm_reduce(sparse: &Tensor, dense: &Tensor, reduce: &str) -> Tensor { + Tensor::f_internal_sparse_mm_reduce(sparse, dense, reduce).unwrap() + } + + pub fn internal_sparse_mm_reduce_impl(&self, other: &Tensor, reduce: &str) -> (Tensor, Tensor) { + self.f_internal_sparse_mm_reduce_impl(other, reduce).unwrap() + } + + pub fn internal_sparse_semi_structured_apply(&self, thread_masks: &Tensor) -> (Tensor, Tensor) { + self.f_internal_sparse_semi_structured_apply(thread_masks).unwrap() + } + + pub fn internal_sparse_semi_structured_apply_dense(&self, thread_masks: &Tensor) -> Tensor { + self.f_internal_sparse_semi_structured_apply_dense(thread_masks).unwrap() + } + + pub fn internal_sparse_semi_structured_linear>( + &self, + weight: &Tensor, + meta: &Tensor, + bias: Option, + activation: &str, + out_dtype: impl Into>, + ) -> Tensor { + self.f_internal_sparse_semi_structured_linear(weight, meta, bias, activation, out_dtype) + .unwrap() + } + + pub fn internal_sparse_semi_structured_mm( + mat1: &Tensor, + mat1_meta: &Tensor, + mat2: &Tensor, + out_dtype: impl Into>, + ) -> Tensor { + Tensor::f_internal_sparse_semi_structured_mm(mat1, mat1_meta, mat2, out_dtype).unwrap() + } + + pub fn internal_sparse_semi_structured_tile( + &self, + algorithm: &str, + use_cutlass: bool, + ) -> (Tensor, Tensor, Tensor, Tensor, Tensor) { + self.f_internal_sparse_semi_structured_tile(algorithm, use_cutlass).unwrap() + } + + pub fn internal_sparse_softmax(&self, dim: i64, half_to_float: bool) -> Tensor { + self.f_internal_sparse_softmax(dim, half_to_float).unwrap() + } + + pub fn internal_sparse_softmax_backward_data( + &self, + grad_output: &Tensor, + output: &Tensor, + dim: i64, + ) -> Tensor { + self.f_internal_sparse_softmax_backward_data(grad_output, output, dim).unwrap() + } + + pub fn internal_sparse_softmax_backward_data_out( + &self, + out: &Tensor, + grad_output: &Tensor, + output: &Tensor, + dim: i64, + ) -> Tensor { + self.f_internal_sparse_softmax_backward_data_out(out, grad_output, output, dim).unwrap() + } + + pub fn internal_sparse_softmax_int(&self, dim: i64, dtype: impl Into>) -> Tensor { + self.f_internal_sparse_softmax_int(dim, dtype).unwrap() + } + + pub fn internal_sparse_softmax_out( + &self, + out: &Tensor, + dim: i64, + half_to_float: bool, + ) -> Tensor { + self.f_internal_sparse_softmax_out(out, dim, half_to_float).unwrap() + } + + pub fn internal_sparse_sparse_matmul(&self, other: &Tensor) -> Tensor { + self.f_internal_sparse_sparse_matmul(other).unwrap() + } + + pub fn internal_sparse_sparse_matmul_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_internal_sparse_sparse_matmul_out(out, other).unwrap() + } + + pub fn internal_sparse_sum(&self) -> Tensor { + self.f_internal_sparse_sum().unwrap() + } + + pub fn internal_sparse_sum_backward(&self, grad: &Tensor, dim: impl IntList) -> Tensor { + self.f_internal_sparse_sum_backward(grad, dim).unwrap() + } + + pub fn internal_sparse_sum_backward_out( + &self, + out: &Tensor, + grad: &Tensor, + dim: impl IntList, + ) -> Tensor { + self.f_internal_sparse_sum_backward_out(out, grad, dim).unwrap() + } + + pub fn internal_sparse_sum_dim(&self, dim: impl IntList) -> Tensor { + self.f_internal_sparse_sum_dim(dim).unwrap() + } + + pub fn internal_sparse_sum_dim_dtype(&self, dim: impl IntList, dtype: Kind) -> Tensor { + self.f_internal_sparse_sum_dim_dtype(dim, dtype).unwrap() + } + + pub fn internal_sparse_sum_dim_out(&self, out: &Tensor, dim: impl IntList) -> Tensor { + self.f_internal_sparse_sum_dim_out(out, dim).unwrap() + } + + pub fn internal_sparse_sum_dtype(&self, dtype: Kind) -> Tensor { + self.f_internal_sparse_sum_dtype(dtype).unwrap() + } + + pub fn internal_spdiags( + diagonals: &Tensor, + offsets: &Tensor, + shape: impl IntList, + layout: Option, + ) -> Tensor { + Tensor::f_internal_spdiags(diagonals, offsets, shape, layout).unwrap() + } + + pub fn internal_spdiags_out( + out: &Tensor, + diagonals: &Tensor, + offsets: &Tensor, + shape: impl IntList, + layout: Option, + ) -> Tensor { + Tensor::f_internal_spdiags_out(out, diagonals, offsets, shape, layout).unwrap() + } + + pub fn internal_spsolve(a: &Tensor, b: &Tensor, left: bool) -> Tensor { + Tensor::f_internal_spsolve(a, b, left).unwrap() + } + + pub fn internal_stack>(tensors: &[T], dim: i64) -> Tensor { + Tensor::f_internal_stack(tensors, dim).unwrap() + } + + pub fn internal_stack_out>(out: &Tensor, tensors: &[T], dim: i64) -> Tensor { + Tensor::f_internal_stack_out(out, tensors, dim).unwrap() + } + + pub fn internal_standard_gamma(&self) -> Tensor { + self.f_internal_standard_gamma().unwrap() + } + + pub fn internal_standard_gamma_grad(&self, output: &Tensor) -> Tensor { + self.f_internal_standard_gamma_grad(output).unwrap() + } + + pub fn internal_standard_gamma_grad_out(&self, out: &Tensor, output: &Tensor) -> Tensor { + self.f_internal_standard_gamma_grad_out(out, output).unwrap() + } + + pub fn internal_standard_gamma_out(&self, out: &Tensor) -> Tensor { + self.f_internal_standard_gamma_out(out).unwrap() + } + + pub fn internal_test_ambiguous_defaults(dummy: &Tensor, a: i64, b: i64) -> Tensor { + Tensor::f_internal_test_ambiguous_defaults(dummy, a, b).unwrap() + } + + pub fn internal_test_ambiguous_defaults_b(dummy: &Tensor, a: i64, b: &str) -> Tensor { + Tensor::f_internal_test_ambiguous_defaults_b(dummy, a, b).unwrap() + } + + pub fn internal_test_autograd_multiple_dispatch(&self) -> Tensor { + self.f_internal_test_autograd_multiple_dispatch().unwrap() + } + + pub fn internal_test_autograd_multiple_dispatch_fullcoverage_out( + &self, + out: &Tensor, + ) -> Tensor { + self.f_internal_test_autograd_multiple_dispatch_fullcoverage_out(out).unwrap() + } + + pub fn internal_test_autograd_multiple_dispatch_ntonly(&self, b: bool) -> Tensor { + self.f_internal_test_autograd_multiple_dispatch_ntonly(b).unwrap() + } + + pub fn internal_test_autograd_multiple_dispatch_view(&self) -> Tensor { + self.f_internal_test_autograd_multiple_dispatch_view().unwrap() + } + + pub fn internal_test_autograd_multiple_dispatch_view_copy(&self) -> Tensor { + self.f_internal_test_autograd_multiple_dispatch_view_copy().unwrap() + } + + pub fn internal_test_autograd_multiple_dispatch_view_copy_out(&self, out: &Tensor) -> Tensor { + self.f_internal_test_autograd_multiple_dispatch_view_copy_out(out).unwrap() + } + + pub fn internal_test_check_tensor(&self) -> Tensor { + self.f_internal_test_check_tensor().unwrap() + } + + pub fn internal_test_functorch_fallback(&self, other: &Tensor) -> Tensor { + self.f_internal_test_functorch_fallback(other).unwrap() + } + + pub fn internal_test_functorch_fallback_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_internal_test_functorch_fallback_out(out, other).unwrap() + } + + pub fn internal_test_optional_filled_intlist( + values: &Tensor, + addends: impl IntListOption, + ) -> Tensor { + Tensor::f_internal_test_optional_filled_intlist(values, addends).unwrap() + } + + pub fn internal_test_optional_filled_intlist_out( + out: &Tensor, + values: &Tensor, + addends: impl IntListOption, + ) -> Tensor { + Tensor::f_internal_test_optional_filled_intlist_out(out, values, addends).unwrap() + } + + pub fn internal_test_optional_floatlist(values: &Tensor, addends: impl DoubleList) -> Tensor { + Tensor::f_internal_test_optional_floatlist(values, addends).unwrap() + } + + pub fn internal_test_optional_floatlist_out( + out: &Tensor, + values: &Tensor, + addends: impl DoubleList, + ) -> Tensor { + Tensor::f_internal_test_optional_floatlist_out(out, values, addends).unwrap() + } + + pub fn internal_test_optional_intlist(values: &Tensor, addends: impl IntListOption) -> Tensor { + Tensor::f_internal_test_optional_intlist(values, addends).unwrap() + } + + pub fn internal_test_optional_intlist_out( + out: &Tensor, + values: &Tensor, + addends: impl IntListOption, + ) -> Tensor { + Tensor::f_internal_test_optional_intlist_out(out, values, addends).unwrap() + } + + pub fn internal_test_parallel_materialize( + &self, + num_parallel: i64, + skip_first: bool, + ) -> Tensor { + self.f_internal_test_parallel_materialize(num_parallel, skip_first).unwrap() + } + + pub fn internal_test_serialization_subcmul(&self, other: &Tensor) -> Tensor { + self.f_internal_test_serialization_subcmul(other).unwrap() + } + + pub fn internal_test_string_default(dummy: &Tensor, a: &str, b: &str) -> Tensor { + Tensor::f_internal_test_string_default(dummy, a, b).unwrap() + } + + pub fn internal_test_warn_in_autograd(&self) -> Tensor { + self.f_internal_test_warn_in_autograd().unwrap() + } + + pub fn internal_test_warn_in_autograd_out(&self, out: &Tensor) -> Tensor { + self.f_internal_test_warn_in_autograd_out(out).unwrap() + } + + pub fn internal_to_copy(&self, options: (Kind, Device), non_blocking: bool) -> Tensor { + self.f_internal_to_copy(options, non_blocking).unwrap() + } + + pub fn internal_to_copy_out(&self, out: &Tensor, non_blocking: bool) -> Tensor { + self.f_internal_to_copy_out(out, non_blocking).unwrap() + } + + pub fn internal_to_cpu>(tensors: &[T]) -> Vec { + Tensor::f_internal_to_cpu(tensors).unwrap() + } + + pub fn internal_to_dense(&self, dtype: impl Into>, masked_grad: bool) -> Tensor { + self.f_internal_to_dense(dtype, masked_grad).unwrap() + } + + pub fn internal_to_dense_out( + &self, + out: &Tensor, + dtype: impl Into>, + masked_grad: bool, + ) -> Tensor { + self.f_internal_to_dense_out(out, dtype, masked_grad).unwrap() + } + + pub fn internal_to_sparse( + &self, + layout: Option, + blocksize: impl IntListOption, + dense_dim: impl Into>, + ) -> Tensor { + self.f_internal_to_sparse(layout, blocksize, dense_dim).unwrap() + } + + pub fn internal_to_sparse_bsc( + &self, + blocksize: impl IntList, + dense_dim: impl Into>, + ) -> Tensor { + self.f_internal_to_sparse_bsc(blocksize, dense_dim).unwrap() + } + + pub fn internal_to_sparse_bsc_out( + &self, + out: &Tensor, + blocksize: impl IntList, + dense_dim: impl Into>, + ) -> Tensor { + self.f_internal_to_sparse_bsc_out(out, blocksize, dense_dim).unwrap() + } + + pub fn internal_to_sparse_bsr( + &self, + blocksize: impl IntList, + dense_dim: impl Into>, + ) -> Tensor { + self.f_internal_to_sparse_bsr(blocksize, dense_dim).unwrap() + } + + pub fn internal_to_sparse_bsr_out( + &self, + out: &Tensor, + blocksize: impl IntList, + dense_dim: impl Into>, + ) -> Tensor { + self.f_internal_to_sparse_bsr_out(out, blocksize, dense_dim).unwrap() + } + + pub fn internal_to_sparse_csc(&self, dense_dim: impl Into>) -> Tensor { + self.f_internal_to_sparse_csc(dense_dim).unwrap() + } + + pub fn internal_to_sparse_csc_out( + &self, + out: &Tensor, + dense_dim: impl Into>, + ) -> Tensor { + self.f_internal_to_sparse_csc_out(out, dense_dim).unwrap() + } + + pub fn internal_to_sparse_csr(&self, dense_dim: impl Into>) -> Tensor { + self.f_internal_to_sparse_csr(dense_dim).unwrap() + } + + pub fn internal_to_sparse_csr_out( + &self, + out: &Tensor, + dense_dim: impl Into>, + ) -> Tensor { + self.f_internal_to_sparse_csr_out(out, dense_dim).unwrap() + } + + pub fn internal_to_sparse_out( + &self, + out: &Tensor, + layout: Option, + blocksize: impl IntListOption, + dense_dim: impl Into>, + ) -> Tensor { + self.f_internal_to_sparse_out(out, layout, blocksize, dense_dim).unwrap() + } + + pub fn internal_to_sparse_semi_structured(dense: &Tensor) -> (Tensor, Tensor) { + Tensor::f_internal_to_sparse_semi_structured(dense).unwrap() + } + + pub fn internal_to_sparse_sparse_dim(&self, sparse_dim: i64) -> Tensor { + self.f_internal_to_sparse_sparse_dim(sparse_dim).unwrap() + } + + pub fn internal_to_sparse_sparse_dim_out(&self, out: &Tensor, sparse_dim: i64) -> Tensor { + self.f_internal_to_sparse_sparse_dim_out(out, sparse_dim).unwrap() + } + + pub fn internal_transform_bias_rescale_qkv( + qkv: &Tensor, + qkv_bias: &Tensor, + num_heads: i64, + ) -> (Tensor, Tensor, Tensor) { + Tensor::f_internal_transform_bias_rescale_qkv(qkv, qkv_bias, num_heads).unwrap() + } + + pub fn internal_transform_bias_rescale_qkv_out( + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + qkv: &Tensor, + qkv_bias: &Tensor, + num_heads: i64, + ) -> (Tensor, Tensor, Tensor) { + Tensor::f_internal_transform_bias_rescale_qkv_out( + out0, out1, out2, qkv, qkv_bias, num_heads, + ) + .unwrap() + } + + pub fn internal_transformer_encoder_layer_fwd>( + src: &Tensor, + embed_dim: i64, + num_heads: i64, + qkv_weight: &Tensor, + qkv_bias: &Tensor, + proj_weight: &Tensor, + proj_bias: &Tensor, + use_gelu: bool, + norm_first: bool, + eps: f64, + norm_weight_1: &Tensor, + norm_bias_1: &Tensor, + norm_weight_2: &Tensor, + norm_bias_2: &Tensor, + ffn_weight_1: &Tensor, + ffn_bias_1: &Tensor, + ffn_weight_2: &Tensor, + ffn_bias_2: &Tensor, + mask: Option, + mask_type: impl Into>, + ) -> Tensor { + Tensor::f_internal_transformer_encoder_layer_fwd( + src, + embed_dim, + num_heads, + qkv_weight, + qkv_bias, + proj_weight, + proj_bias, + use_gelu, + norm_first, + eps, + norm_weight_1, + norm_bias_1, + norm_weight_2, + norm_bias_2, + ffn_weight_1, + ffn_bias_1, + ffn_weight_2, + ffn_bias_2, + mask, + mask_type, + ) + .unwrap() + } + + pub fn internal_transformer_encoder_layer_fwd_out>( + out: &Tensor, + src: &Tensor, + embed_dim: i64, + num_heads: i64, + qkv_weight: &Tensor, + qkv_bias: &Tensor, + proj_weight: &Tensor, + proj_bias: &Tensor, + use_gelu: bool, + norm_first: bool, + eps: f64, + norm_weight_1: &Tensor, + norm_bias_1: &Tensor, + norm_weight_2: &Tensor, + norm_bias_2: &Tensor, + ffn_weight_1: &Tensor, + ffn_bias_1: &Tensor, + ffn_weight_2: &Tensor, + ffn_bias_2: &Tensor, + mask: Option, + mask_type: impl Into>, + ) -> Tensor { + Tensor::f_internal_transformer_encoder_layer_fwd_out( + out, + src, + embed_dim, + num_heads, + qkv_weight, + qkv_bias, + proj_weight, + proj_bias, + use_gelu, + norm_first, + eps, + norm_weight_1, + norm_bias_1, + norm_weight_2, + norm_bias_2, + ffn_weight_1, + ffn_bias_1, + ffn_weight_2, + ffn_bias_2, + mask, + mask_type, + ) + .unwrap() + } + + pub fn internal_trilinear( + i1: &Tensor, + i2: &Tensor, + i3: &Tensor, + expand1: impl IntList, + expand2: impl IntList, + expand3: impl IntList, + sumdim: impl IntList, + unroll_dim: i64, + ) -> Tensor { + Tensor::f_internal_trilinear(i1, i2, i3, expand1, expand2, expand3, sumdim, unroll_dim) + .unwrap() + } + + pub fn internal_trilinear_out( + out: &Tensor, + i1: &Tensor, + i2: &Tensor, + i3: &Tensor, + expand1: impl IntList, + expand2: impl IntList, + expand3: impl IntList, + sumdim: impl IntList, + unroll_dim: i64, + ) -> Tensor { + Tensor::f_internal_trilinear_out( + out, i1, i2, i3, expand1, expand2, expand3, sumdim, unroll_dim, + ) + .unwrap() + } + + pub fn internal_triton_multi_head_attention>( + query: &Tensor, + key: &Tensor, + value: &Tensor, + embed_dim: i64, + num_head: i64, + qkv_weight: &Tensor, + qkv_bias: &Tensor, + proj_weight: &Tensor, + proj_bias: &Tensor, + mask: Option, + ) -> Tensor { + Tensor::f_internal_triton_multi_head_attention( + query, + key, + value, + embed_dim, + num_head, + qkv_weight, + qkv_bias, + proj_weight, + proj_bias, + mask, + ) + .unwrap() + } + + pub fn internal_triton_multi_head_attention_out>( + out: &Tensor, + query: &Tensor, + key: &Tensor, + value: &Tensor, + embed_dim: i64, + num_head: i64, + qkv_weight: &Tensor, + qkv_bias: &Tensor, + proj_weight: &Tensor, + proj_bias: &Tensor, + mask: Option, + ) -> Tensor { + Tensor::f_internal_triton_multi_head_attention_out( + out, + query, + key, + value, + embed_dim, + num_head, + qkv_weight, + qkv_bias, + proj_weight, + proj_bias, + mask, + ) + .unwrap() + } + + pub fn internal_triton_scaled_dot_attention( + q: &Tensor, + k: &Tensor, + v: &Tensor, + dropout_p: f64, + ) -> Tensor { + Tensor::f_internal_triton_scaled_dot_attention(q, k, v, dropout_p).unwrap() + } + + pub fn internal_triton_scaled_dot_attention_out( + out: &Tensor, + q: &Tensor, + k: &Tensor, + v: &Tensor, + dropout_p: f64, + ) -> Tensor { + Tensor::f_internal_triton_scaled_dot_attention_out(out, q, k, v, dropout_p).unwrap() + } + + pub fn internal_unique(&self, sorted: bool, return_inverse: bool) -> (Tensor, Tensor) { + self.f_internal_unique(sorted, return_inverse).unwrap() + } + + pub fn internal_unique2( + &self, + sorted: bool, + return_inverse: bool, + return_counts: bool, + ) -> (Tensor, Tensor, Tensor) { + self.f_internal_unique2(sorted, return_inverse, return_counts).unwrap() + } + + pub fn internal_unique2_out( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + sorted: bool, + return_inverse: bool, + return_counts: bool, + ) -> (Tensor, Tensor, Tensor) { + self.f_internal_unique2_out(out0, out1, out2, sorted, return_inverse, return_counts) + .unwrap() + } + + pub fn internal_unique_out( + &self, + out0: &Tensor, + out1: &Tensor, + sorted: bool, + return_inverse: bool, + ) -> (Tensor, Tensor) { + self.f_internal_unique_out(out0, out1, sorted, return_inverse).unwrap() + } + + pub fn internal_unpack_dual(dual: &Tensor, level: i64) -> (Tensor, Tensor) { + Tensor::f_internal_unpack_dual(dual, level).unwrap() + } + + pub fn internal_unsafe_index>(&self, indices: &[Option]) -> Tensor { + self.f_internal_unsafe_index(indices).unwrap() + } + + pub fn internal_unsafe_index_put>( + &self, + indices: &[Option], + values: &Tensor, + accumulate: bool, + ) -> Tensor { + self.f_internal_unsafe_index_put(indices, values, accumulate).unwrap() + } + + pub fn internal_unsafe_masked_index, S: Into>( + &self, + mask: &Tensor, + indices: &[Option], + fill: S, + ) -> Tensor { + self.f_internal_unsafe_masked_index(mask, indices, fill).unwrap() + } + + pub fn internal_unsafe_masked_index_put_accumulate>( + &self, + mask: &Tensor, + indices: &[Option], + values: &Tensor, + ) -> Tensor { + self.f_internal_unsafe_masked_index_put_accumulate(mask, indices, values).unwrap() + } + + pub fn internal_unsafe_view(&self, size: impl IntList) -> Tensor { + self.f_internal_unsafe_view(size).unwrap() + } + + pub fn internal_unsafe_view_out(&self, out: &Tensor, size: impl IntList) -> Tensor { + self.f_internal_unsafe_view_out(out, size).unwrap() + } + + pub fn internal_upsample_bicubic2d_aa( + &self, + output_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + self.f_internal_upsample_bicubic2d_aa(output_size, align_corners, scales_h, scales_w) + .unwrap() + } + + pub fn internal_upsample_bicubic2d_aa_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + Tensor::f_internal_upsample_bicubic2d_aa_backward( + grad_output, + output_size, + input_size, + align_corners, + scales_h, + scales_w, + ) + .unwrap() + } + + pub fn internal_upsample_bicubic2d_aa_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + Tensor::f_internal_upsample_bicubic2d_aa_backward_grad_input( + grad_input, + grad_output, + output_size, + input_size, + align_corners, + scales_h, + scales_w, + ) + .unwrap() + } + + pub fn internal_upsample_bicubic2d_aa_out( + &self, + out: &Tensor, + output_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + self.f_internal_upsample_bicubic2d_aa_out( + out, + output_size, + align_corners, + scales_h, + scales_w, + ) + .unwrap() + } + + pub fn internal_upsample_bicubic2d_aa_vec( + &self, + output_size: impl IntListOption, + align_corners: bool, + scale_factors: impl DoubleList, + ) -> Tensor { + self.f_internal_upsample_bicubic2d_aa_vec(output_size, align_corners, scale_factors) + .unwrap() + } + + pub fn internal_upsample_bilinear2d_aa( + &self, + output_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + self.f_internal_upsample_bilinear2d_aa(output_size, align_corners, scales_h, scales_w) + .unwrap() + } + + pub fn internal_upsample_bilinear2d_aa_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + Tensor::f_internal_upsample_bilinear2d_aa_backward( + grad_output, + output_size, + input_size, + align_corners, + scales_h, + scales_w, + ) + .unwrap() + } + + pub fn internal_upsample_bilinear2d_aa_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + Tensor::f_internal_upsample_bilinear2d_aa_backward_grad_input( + grad_input, + grad_output, + output_size, + input_size, + align_corners, + scales_h, + scales_w, + ) + .unwrap() + } + + pub fn internal_upsample_bilinear2d_aa_out( + &self, + out: &Tensor, + output_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + self.f_internal_upsample_bilinear2d_aa_out( + out, + output_size, + align_corners, + scales_h, + scales_w, + ) + .unwrap() + } + + pub fn internal_upsample_bilinear2d_aa_vec( + &self, + output_size: impl IntListOption, + align_corners: bool, + scale_factors: impl DoubleList, + ) -> Tensor { + self.f_internal_upsample_bilinear2d_aa_vec(output_size, align_corners, scale_factors) + .unwrap() + } + + pub fn internal_upsample_nearest_exact1d( + &self, + output_size: impl IntList, + scales: impl Into>, + ) -> Tensor { + self.f_internal_upsample_nearest_exact1d(output_size, scales).unwrap() + } + + pub fn internal_upsample_nearest_exact1d_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales: impl Into>, + ) -> Tensor { + Tensor::f_internal_upsample_nearest_exact1d_backward( + grad_output, + output_size, + input_size, + scales, + ) + .unwrap() + } + + pub fn internal_upsample_nearest_exact1d_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales: impl Into>, + ) -> Tensor { + Tensor::f_internal_upsample_nearest_exact1d_backward_grad_input( + grad_input, + grad_output, + output_size, + input_size, + scales, + ) + .unwrap() + } + + pub fn internal_upsample_nearest_exact1d_out( + &self, + out: &Tensor, + output_size: impl IntList, + scales: impl Into>, + ) -> Tensor { + self.f_internal_upsample_nearest_exact1d_out(out, output_size, scales).unwrap() + } + + pub fn internal_upsample_nearest_exact1d_vec( + &self, + output_size: impl IntListOption, + scale_factors: impl DoubleList, + ) -> Tensor { + self.f_internal_upsample_nearest_exact1d_vec(output_size, scale_factors).unwrap() + } + + pub fn internal_upsample_nearest_exact2d( + &self, + output_size: impl IntList, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + self.f_internal_upsample_nearest_exact2d(output_size, scales_h, scales_w).unwrap() + } + + pub fn internal_upsample_nearest_exact2d_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + Tensor::f_internal_upsample_nearest_exact2d_backward( + grad_output, + output_size, + input_size, + scales_h, + scales_w, + ) + .unwrap() + } + + pub fn internal_upsample_nearest_exact2d_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + Tensor::f_internal_upsample_nearest_exact2d_backward_grad_input( + grad_input, + grad_output, + output_size, + input_size, + scales_h, + scales_w, + ) + .unwrap() + } + + pub fn internal_upsample_nearest_exact2d_out( + &self, + out: &Tensor, + output_size: impl IntList, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + self.f_internal_upsample_nearest_exact2d_out(out, output_size, scales_h, scales_w).unwrap() + } + + pub fn internal_upsample_nearest_exact2d_vec( + &self, + output_size: impl IntListOption, + scale_factors: impl DoubleList, + ) -> Tensor { + self.f_internal_upsample_nearest_exact2d_vec(output_size, scale_factors).unwrap() + } + + pub fn internal_upsample_nearest_exact3d( + &self, + output_size: impl IntList, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + self.f_internal_upsample_nearest_exact3d(output_size, scales_d, scales_h, scales_w).unwrap() + } + + pub fn internal_upsample_nearest_exact3d_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + Tensor::f_internal_upsample_nearest_exact3d_backward( + grad_output, + output_size, + input_size, + scales_d, + scales_h, + scales_w, + ) + .unwrap() + } + + pub fn internal_upsample_nearest_exact3d_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + Tensor::f_internal_upsample_nearest_exact3d_backward_grad_input( + grad_input, + grad_output, + output_size, + input_size, + scales_d, + scales_h, + scales_w, + ) + .unwrap() + } + + pub fn internal_upsample_nearest_exact3d_out( + &self, + out: &Tensor, + output_size: impl IntList, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + self.f_internal_upsample_nearest_exact3d_out(out, output_size, scales_d, scales_h, scales_w) + .unwrap() + } + + pub fn internal_upsample_nearest_exact3d_vec( + &self, + output_size: impl IntListOption, + scale_factors: impl DoubleList, + ) -> Tensor { + self.f_internal_upsample_nearest_exact3d_vec(output_size, scale_factors).unwrap() + } + + pub fn internal_use_cudnn_ctc_loss( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: impl IntList, + target_lengths: impl IntList, + blank: i64, + ) -> bool { + Tensor::f_internal_use_cudnn_ctc_loss( + log_probs, + targets, + input_lengths, + target_lengths, + blank, + ) + .unwrap() + } + + pub fn internal_use_cudnn_ctc_loss_tensor( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: &Tensor, + target_lengths: &Tensor, + blank: i64, + ) -> bool { + Tensor::f_internal_use_cudnn_ctc_loss_tensor( + log_probs, + targets, + input_lengths, + target_lengths, + blank, + ) + .unwrap() + } + + pub fn internal_use_cudnn_rnn_flatten_weight() -> bool { + Tensor::f_internal_use_cudnn_rnn_flatten_weight().unwrap() + } + + pub fn internal_use_miopen_ctc_loss( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: impl IntList, + target_lengths: impl IntList, + blank: i64, + ) -> bool { + Tensor::f_internal_use_miopen_ctc_loss( + log_probs, + targets, + input_lengths, + target_lengths, + blank, + ) + .unwrap() + } + + pub fn internal_use_miopen_ctc_loss_tensor( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: &Tensor, + target_lengths: &Tensor, + blank: i64, + ) -> bool { + Tensor::f_internal_use_miopen_ctc_loss_tensor( + log_probs, + targets, + input_lengths, + target_lengths, + blank, + ) + .unwrap() + } + + pub fn internal_validate_compressed_sparse_indices( + is_crow: bool, + compressed_idx: &Tensor, + plain_idx: &Tensor, + cdim: i64, + dim: i64, + nnz: i64, + ) { + Tensor::f_internal_validate_compressed_sparse_indices( + is_crow, + compressed_idx, + plain_idx, + cdim, + dim, + nnz, + ) + .unwrap() + } + + pub fn internal_validate_sparse_bsc_tensor_args( + ccol_indices: &Tensor, + row_indices: &Tensor, + values: &Tensor, + size: impl IntList, + check_pinning: bool, + ) { + Tensor::f_internal_validate_sparse_bsc_tensor_args( + ccol_indices, + row_indices, + values, + size, + check_pinning, + ) + .unwrap() + } + + pub fn internal_validate_sparse_bsr_tensor_args( + crow_indices: &Tensor, + col_indices: &Tensor, + values: &Tensor, + size: impl IntList, + check_pinning: bool, + ) { + Tensor::f_internal_validate_sparse_bsr_tensor_args( + crow_indices, + col_indices, + values, + size, + check_pinning, + ) + .unwrap() + } + + pub fn internal_validate_sparse_compressed_tensor_args( + compressed_indices: &Tensor, + plain_indices: &Tensor, + values: &Tensor, + size: impl IntList, + layout: Layout, + check_pinning: bool, + ) { + Tensor::f_internal_validate_sparse_compressed_tensor_args( + compressed_indices, + plain_indices, + values, + size, + layout, + check_pinning, + ) + .unwrap() + } + + pub fn internal_validate_sparse_csc_tensor_args( + ccol_indices: &Tensor, + row_indices: &Tensor, + values: &Tensor, + size: impl IntList, + check_pinning: bool, + ) { + Tensor::f_internal_validate_sparse_csc_tensor_args( + ccol_indices, + row_indices, + values, + size, + check_pinning, + ) + .unwrap() + } + + pub fn internal_validate_sparse_csr_tensor_args( + crow_indices: &Tensor, + col_indices: &Tensor, + values: &Tensor, + size: impl IntList, + check_pinning: bool, + ) { + Tensor::f_internal_validate_sparse_csr_tensor_args( + crow_indices, + col_indices, + values, + size, + check_pinning, + ) + .unwrap() + } + + pub fn internal_values(&self) -> Tensor { + self.f_internal_values().unwrap() + } + + pub fn internal_values_copy(&self) -> Tensor { + self.f_internal_values_copy().unwrap() + } + + pub fn internal_values_copy_out(&self, out: &Tensor) -> Tensor { + self.f_internal_values_copy_out(out).unwrap() + } + + pub fn internal_version(&self) -> i64 { + self.f_internal_version().unwrap() + } + + pub fn internal_weight_int4pack_mm( + &self, + mat2: &Tensor, + qgroupsize: i64, + qscaleandzeros: &Tensor, + ) -> Tensor { + self.f_internal_weight_int4pack_mm(mat2, qgroupsize, qscaleandzeros).unwrap() + } + + pub fn internal_weight_int4pack_mm_for_cpu( + &self, + mat2: &Tensor, + qgroupsize: i64, + qscaleandzeros: &Tensor, + ) -> Tensor { + self.f_internal_weight_int4pack_mm_for_cpu(mat2, qgroupsize, qscaleandzeros).unwrap() + } + + pub fn internal_weight_int4pack_mm_with_scales_and_zeros( + &self, + mat2: &Tensor, + qgroupsize: i64, + qscale: &Tensor, + qzeros: &Tensor, + ) -> Tensor { + self.f_internal_weight_int4pack_mm_with_scales_and_zeros(mat2, qgroupsize, qscale, qzeros) + .unwrap() + } + + pub fn internal_weight_int8pack_mm(&self, mat2: &Tensor, scales: &Tensor) -> Tensor { + self.f_internal_weight_int8pack_mm(mat2, scales).unwrap() + } + + pub fn internal_weight_norm(v: &Tensor, g: &Tensor, dim: i64) -> Tensor { + Tensor::f_internal_weight_norm(v, g, dim).unwrap() + } + + pub fn internal_weight_norm_differentiable_backward( + grad_w: &Tensor, + saved_v: &Tensor, + saved_g: &Tensor, + saved_norms: &Tensor, + dim: i64, + ) -> (Tensor, Tensor) { + Tensor::f_internal_weight_norm_differentiable_backward( + grad_w, + saved_v, + saved_g, + saved_norms, + dim, + ) + .unwrap() + } + + pub fn internal_weight_norm_interface(v: &Tensor, g: &Tensor, dim: i64) -> (Tensor, Tensor) { + Tensor::f_internal_weight_norm_interface(v, g, dim).unwrap() + } + + pub fn internal_weight_norm_interface_backward( + grad_w: &Tensor, + saved_v: &Tensor, + saved_g: &Tensor, + saved_norms: &Tensor, + dim: i64, + ) -> (Tensor, Tensor) { + Tensor::f_internal_weight_norm_interface_backward( + grad_w, + saved_v, + saved_g, + saved_norms, + dim, + ) + .unwrap() + } + + pub fn internal_weight_norm_interface_backward_out( + out0: &Tensor, + out1: &Tensor, + grad_w: &Tensor, + saved_v: &Tensor, + saved_g: &Tensor, + saved_norms: &Tensor, + dim: i64, + ) -> (Tensor, Tensor) { + Tensor::f_internal_weight_norm_interface_backward_out( + out0, + out1, + grad_w, + saved_v, + saved_g, + saved_norms, + dim, + ) + .unwrap() + } + + pub fn internal_weight_norm_interface_out( + out0: &Tensor, + out1: &Tensor, + v: &Tensor, + g: &Tensor, + dim: i64, + ) -> (Tensor, Tensor) { + Tensor::f_internal_weight_norm_interface_out(out0, out1, v, g, dim).unwrap() + } + + pub fn internal_wrapped_linear_prepack( + weight: &Tensor, + weight_scale: &Tensor, + weight_zero_point: &Tensor, + bias: &Tensor, + ) -> Tensor { + Tensor::f_internal_wrapped_linear_prepack(weight, weight_scale, weight_zero_point, bias) + .unwrap() + } + + pub fn internal_wrapped_quantized_linear_prepacked( + &self, + input_scale: &Tensor, + input_zero_point: &Tensor, + packed_weight: &Tensor, + output_scale: &Tensor, + output_zero_point: &Tensor, + out_channel: i64, + ) -> Tensor { + self.f_internal_wrapped_quantized_linear_prepacked( + input_scale, + input_zero_point, + packed_weight, + output_scale, + output_zero_point, + out_channel, + ) + .unwrap() + } + + pub fn abs(&self) -> Tensor { + self.f_abs().unwrap() + } + + pub fn abs_(&mut self) -> Tensor { + self.f_abs_().unwrap() + } + + pub fn abs_out(&self, out: &Tensor) -> Tensor { + self.f_abs_out(out).unwrap() + } + + pub fn absolute(&self) -> Tensor { + self.f_absolute().unwrap() + } + + pub fn absolute_(&mut self) -> Tensor { + self.f_absolute_().unwrap() + } + + pub fn absolute_out(&self, out: &Tensor) -> Tensor { + self.f_absolute_out(out).unwrap() + } + + pub fn acos(&self) -> Tensor { + self.f_acos().unwrap() + } + + pub fn acos_(&mut self) -> Tensor { + self.f_acos_().unwrap() + } + + pub fn acos_out(&self, out: &Tensor) -> Tensor { + self.f_acos_out(out).unwrap() + } + + pub fn acosh(&self) -> Tensor { + self.f_acosh().unwrap() + } + + pub fn acosh_(&mut self) -> Tensor { + self.f_acosh_().unwrap() + } + + pub fn acosh_out(&self, out: &Tensor) -> Tensor { + self.f_acosh_out(out).unwrap() + } + + pub fn adaptive_avg_pool1d(&self, output_size: impl IntList) -> Tensor { + self.f_adaptive_avg_pool1d(output_size).unwrap() + } + + pub fn adaptive_avg_pool1d_out(&self, out: &Tensor, output_size: impl IntList) -> Tensor { + self.f_adaptive_avg_pool1d_out(out, output_size).unwrap() + } + + pub fn adaptive_avg_pool2d(&self, output_size: impl IntList) -> Tensor { + self.f_adaptive_avg_pool2d(output_size).unwrap() + } + + pub fn adaptive_avg_pool2d_out(&self, out: &Tensor, output_size: impl IntList) -> Tensor { + self.f_adaptive_avg_pool2d_out(out, output_size).unwrap() + } + + pub fn adaptive_avg_pool3d(&self, output_size: impl IntList) -> Tensor { + self.f_adaptive_avg_pool3d(output_size).unwrap() + } + + pub fn adaptive_avg_pool3d_backward( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + ) -> Tensor { + self.f_adaptive_avg_pool3d_backward(grad_input, grad_output).unwrap() + } + + pub fn adaptive_avg_pool3d_out(&self, out: &Tensor, output_size: impl IntList) -> Tensor { + self.f_adaptive_avg_pool3d_out(out, output_size).unwrap() + } + + pub fn adaptive_max_pool1d(&self, output_size: impl IntList) -> (Tensor, Tensor) { + self.f_adaptive_max_pool1d(output_size).unwrap() + } + + pub fn adaptive_max_pool2d(&self, output_size: impl IntList) -> (Tensor, Tensor) { + self.f_adaptive_max_pool2d(output_size).unwrap() + } + + pub fn adaptive_max_pool2d_backward(&self, grad_output: &Tensor, indices: &Tensor) -> Tensor { + self.f_adaptive_max_pool2d_backward(grad_output, indices).unwrap() + } + + pub fn adaptive_max_pool2d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + indices: &Tensor, + ) -> Tensor { + self.f_adaptive_max_pool2d_backward_grad_input(grad_input, grad_output, indices).unwrap() + } + + pub fn adaptive_max_pool2d_out( + &self, + out: &Tensor, + indices: &Tensor, + output_size: impl IntList, + ) -> (Tensor, Tensor) { + self.f_adaptive_max_pool2d_out(out, indices, output_size).unwrap() + } + + pub fn adaptive_max_pool3d(&self, output_size: impl IntList) -> (Tensor, Tensor) { + self.f_adaptive_max_pool3d(output_size).unwrap() + } + + pub fn adaptive_max_pool3d_backward(&self, grad_output: &Tensor, indices: &Tensor) -> Tensor { + self.f_adaptive_max_pool3d_backward(grad_output, indices).unwrap() + } + + pub fn adaptive_max_pool3d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + indices: &Tensor, + ) -> Tensor { + self.f_adaptive_max_pool3d_backward_grad_input(grad_input, grad_output, indices).unwrap() + } + + pub fn adaptive_max_pool3d_out( + &self, + out: &Tensor, + indices: &Tensor, + output_size: impl IntList, + ) -> (Tensor, Tensor) { + self.f_adaptive_max_pool3d_out(out, indices, output_size).unwrap() + } + + pub fn g_add(&self, other: &Tensor) -> Tensor { + self.f_add(other).unwrap() + } + + pub fn g_add_(&mut self, other: &Tensor) -> Tensor { + self.f_add_(other).unwrap() + } + + pub fn add_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_add_out(out, other).unwrap() + } + + pub fn g_add_scalar>(&self, other: S) -> Tensor { + self.f_add_scalar(other).unwrap() + } + + pub fn g_add_scalar_>(&mut self, other: S) -> Tensor { + self.f_add_scalar_(other).unwrap() + } + + pub fn add_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_add_scalar_out(out, other).unwrap() + } + + pub fn addbmm(&self, batch1: &Tensor, batch2: &Tensor) -> Tensor { + self.f_addbmm(batch1, batch2).unwrap() + } + + pub fn addbmm_(&mut self, batch1: &Tensor, batch2: &Tensor) -> Tensor { + self.f_addbmm_(batch1, batch2).unwrap() + } + + pub fn addbmm_out(&self, out: &Tensor, batch1: &Tensor, batch2: &Tensor) -> Tensor { + self.f_addbmm_out(out, batch1, batch2).unwrap() + } + + pub fn addcdiv(&self, tensor1: &Tensor, tensor2: &Tensor) -> Tensor { + self.f_addcdiv(tensor1, tensor2).unwrap() + } + + pub fn addcdiv_(&mut self, tensor1: &Tensor, tensor2: &Tensor) -> Tensor { + self.f_addcdiv_(tensor1, tensor2).unwrap() + } + + pub fn addcdiv_out(&self, out: &Tensor, tensor1: &Tensor, tensor2: &Tensor) -> Tensor { + self.f_addcdiv_out(out, tensor1, tensor2).unwrap() + } + + pub fn addcmul(&self, tensor1: &Tensor, tensor2: &Tensor) -> Tensor { + self.f_addcmul(tensor1, tensor2).unwrap() + } + + pub fn addcmul_(&mut self, tensor1: &Tensor, tensor2: &Tensor) -> Tensor { + self.f_addcmul_(tensor1, tensor2).unwrap() + } + + pub fn addcmul_out(&self, out: &Tensor, tensor1: &Tensor, tensor2: &Tensor) -> Tensor { + self.f_addcmul_out(out, tensor1, tensor2).unwrap() + } + + pub fn addmm(&self, mat1: &Tensor, mat2: &Tensor) -> Tensor { + self.f_addmm(mat1, mat2).unwrap() + } + + pub fn addmm_(&mut self, mat1: &Tensor, mat2: &Tensor) -> Tensor { + self.f_addmm_(mat1, mat2).unwrap() + } + + pub fn addmm_dtype(&self, mat1: &Tensor, mat2: &Tensor, out_dtype: Kind) -> Tensor { + self.f_addmm_dtype(mat1, mat2, out_dtype).unwrap() + } + + pub fn addmm_dtype_out( + &self, + out: &Tensor, + mat1: &Tensor, + mat2: &Tensor, + out_dtype: Kind, + ) -> Tensor { + self.f_addmm_dtype_out(out, mat1, mat2, out_dtype).unwrap() + } + + pub fn addmm_out(&self, out: &Tensor, mat1: &Tensor, mat2: &Tensor) -> Tensor { + self.f_addmm_out(out, mat1, mat2).unwrap() + } + + pub fn addmv(&self, mat: &Tensor, vec: &Tensor) -> Tensor { + self.f_addmv(mat, vec).unwrap() + } + + pub fn addmv_(&mut self, mat: &Tensor, vec: &Tensor) -> Tensor { + self.f_addmv_(mat, vec).unwrap() + } + + pub fn addmv_out(&self, out: &Tensor, mat: &Tensor, vec: &Tensor) -> Tensor { + self.f_addmv_out(out, mat, vec).unwrap() + } + + pub fn addr(&self, vec1: &Tensor, vec2: &Tensor) -> Tensor { + self.f_addr(vec1, vec2).unwrap() + } + + pub fn addr_(&mut self, vec1: &Tensor, vec2: &Tensor) -> Tensor { + self.f_addr_(vec1, vec2).unwrap() + } + + pub fn addr_out(&self, out: &Tensor, vec1: &Tensor, vec2: &Tensor) -> Tensor { + self.f_addr_out(out, vec1, vec2).unwrap() + } + + pub fn adjoint(&self) -> Tensor { + self.f_adjoint().unwrap() + } + + pub fn affine_grid_generator( + theta: &Tensor, + size: impl IntList, + align_corners: bool, + ) -> Tensor { + Tensor::f_affine_grid_generator(theta, size, align_corners).unwrap() + } + + pub fn affine_grid_generator_backward( + grad: &Tensor, + size: impl IntList, + align_corners: bool, + ) -> Tensor { + Tensor::f_affine_grid_generator_backward(grad, size, align_corners).unwrap() + } + + pub fn affine_grid_generator_out( + out: &Tensor, + theta: &Tensor, + size: impl IntList, + align_corners: bool, + ) -> Tensor { + Tensor::f_affine_grid_generator_out(out, theta, size, align_corners).unwrap() + } + + pub fn alias(&self) -> Tensor { + self.f_alias().unwrap() + } + + pub fn alias_copy(&self) -> Tensor { + self.f_alias_copy().unwrap() + } + + pub fn alias_copy_out(&self, out: &Tensor) -> Tensor { + self.f_alias_copy_out(out).unwrap() + } + + pub fn align_as(&self, other: &Tensor) -> Tensor { + self.f_align_as(other).unwrap() + } + + pub fn align_tensors>(tensors: &[T]) -> Vec { + Tensor::f_align_tensors(tensors).unwrap() + } + + pub fn all(&self) -> Tensor { + self.f_all().unwrap() + } + + pub fn all_all_out(&self, out: &Tensor) -> Tensor { + self.f_all_all_out(out).unwrap() + } + + pub fn all_dim(&self, dim: i64, keepdim: bool) -> Tensor { + self.f_all_dim(dim, keepdim).unwrap() + } + + pub fn all_dims(&self, dim: impl IntListOption, keepdim: bool) -> Tensor { + self.f_all_dims(dim, keepdim).unwrap() + } + + pub fn all_dims_out(&self, out: &Tensor, dim: impl IntListOption, keepdim: bool) -> Tensor { + self.f_all_dims_out(out, dim, keepdim).unwrap() + } + + pub fn all_out(&self, out: &Tensor, dim: i64, keepdim: bool) -> Tensor { + self.f_all_out(out, dim, keepdim).unwrap() + } + + pub fn allclose(&self, other: &Tensor, rtol: f64, atol: f64, equal_nan: bool) -> bool { + self.f_allclose(other, rtol, atol, equal_nan).unwrap() + } + + pub fn alpha_dropout(&self, p: f64, train: bool) -> Tensor { + self.f_alpha_dropout(p, train).unwrap() + } + + pub fn alpha_dropout_(&mut self, p: f64, train: bool) -> Tensor { + self.f_alpha_dropout_(p, train).unwrap() + } + + pub fn amax(&self, dim: impl IntList, keepdim: bool) -> Tensor { + self.f_amax(dim, keepdim).unwrap() + } + + pub fn amax_out(&self, out: &Tensor, dim: impl IntList, keepdim: bool) -> Tensor { + self.f_amax_out(out, dim, keepdim).unwrap() + } + + pub fn amin(&self, dim: impl IntList, keepdim: bool) -> Tensor { + self.f_amin(dim, keepdim).unwrap() + } + + pub fn amin_out(&self, out: &Tensor, dim: impl IntList, keepdim: bool) -> Tensor { + self.f_amin_out(out, dim, keepdim).unwrap() + } + + pub fn aminmax(&self, dim: impl Into>, keepdim: bool) -> (Tensor, Tensor) { + self.f_aminmax(dim, keepdim).unwrap() + } + + pub fn aminmax_out( + &self, + min: &Tensor, + max: &Tensor, + dim: impl Into>, + keepdim: bool, + ) -> (Tensor, Tensor) { + self.f_aminmax_out(min, max, dim, keepdim).unwrap() + } + + pub fn angle(&self) -> Tensor { + self.f_angle().unwrap() + } + + pub fn angle_out(&self, out: &Tensor) -> Tensor { + self.f_angle_out(out).unwrap() + } + + pub fn any(&self) -> Tensor { + self.f_any().unwrap() + } + + pub fn any_all_out(&self, out: &Tensor) -> Tensor { + self.f_any_all_out(out).unwrap() + } + + pub fn any_dim(&self, dim: i64, keepdim: bool) -> Tensor { + self.f_any_dim(dim, keepdim).unwrap() + } + + pub fn any_dims(&self, dim: impl IntListOption, keepdim: bool) -> Tensor { + self.f_any_dims(dim, keepdim).unwrap() + } + + pub fn any_dims_out(&self, out: &Tensor, dim: impl IntListOption, keepdim: bool) -> Tensor { + self.f_any_dims_out(out, dim, keepdim).unwrap() + } + + pub fn any_out(&self, out: &Tensor, dim: i64, keepdim: bool) -> Tensor { + self.f_any_out(out, dim, keepdim).unwrap() + } + + pub fn arange>(end: S, options: (Kind, Device)) -> Tensor { + Tensor::f_arange(end, options).unwrap() + } + + pub fn arange_start>(start: S, end: S, options: (Kind, Device)) -> Tensor { + Tensor::f_arange_start(start, end, options).unwrap() + } + + pub fn arange_start_step>( + start: S, + end: S, + step: S, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_arange_start_step(start, end, step, options).unwrap() + } + + pub fn arccos(&self) -> Tensor { + self.f_arccos().unwrap() + } + + pub fn arccos_(&mut self) -> Tensor { + self.f_arccos_().unwrap() + } + + pub fn arccos_out(&self, out: &Tensor) -> Tensor { + self.f_arccos_out(out).unwrap() + } + + pub fn arccosh(&self) -> Tensor { + self.f_arccosh().unwrap() + } + + pub fn arccosh_(&mut self) -> Tensor { + self.f_arccosh_().unwrap() + } + + pub fn arccosh_out(&self, out: &Tensor) -> Tensor { + self.f_arccosh_out(out).unwrap() + } + + pub fn arcsin(&self) -> Tensor { + self.f_arcsin().unwrap() + } + + pub fn arcsin_(&mut self) -> Tensor { + self.f_arcsin_().unwrap() + } + + pub fn arcsin_out(&self, out: &Tensor) -> Tensor { + self.f_arcsin_out(out).unwrap() + } + + pub fn arcsinh(&self) -> Tensor { + self.f_arcsinh().unwrap() + } + + pub fn arcsinh_(&mut self) -> Tensor { + self.f_arcsinh_().unwrap() + } + + pub fn arcsinh_out(&self, out: &Tensor) -> Tensor { + self.f_arcsinh_out(out).unwrap() + } + + pub fn arctan(&self) -> Tensor { + self.f_arctan().unwrap() + } + + pub fn arctan2(&self, other: &Tensor) -> Tensor { + self.f_arctan2(other).unwrap() + } + + pub fn arctan2_(&mut self, other: &Tensor) -> Tensor { + self.f_arctan2_(other).unwrap() + } + + pub fn arctan2_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_arctan2_out(out, other).unwrap() + } + + pub fn arctan_(&mut self) -> Tensor { + self.f_arctan_().unwrap() + } + + pub fn arctan_out(&self, out: &Tensor) -> Tensor { + self.f_arctan_out(out).unwrap() + } + + pub fn arctanh(&self) -> Tensor { + self.f_arctanh().unwrap() + } + + pub fn arctanh_(&mut self) -> Tensor { + self.f_arctanh_().unwrap() + } + + pub fn arctanh_out(&self, out: &Tensor) -> Tensor { + self.f_arctanh_out(out).unwrap() + } + + pub fn argmax(&self, dim: impl Into>, keepdim: bool) -> Tensor { + self.f_argmax(dim, keepdim).unwrap() + } + + pub fn argmax_out(&self, out: &Tensor, dim: impl Into>, keepdim: bool) -> Tensor { + self.f_argmax_out(out, dim, keepdim).unwrap() + } + + pub fn argmin(&self, dim: impl Into>, keepdim: bool) -> Tensor { + self.f_argmin(dim, keepdim).unwrap() + } + + pub fn argmin_out(&self, out: &Tensor, dim: impl Into>, keepdim: bool) -> Tensor { + self.f_argmin_out(out, dim, keepdim).unwrap() + } + + pub fn argsort(&self, dim: i64, descending: bool) -> Tensor { + self.f_argsort(dim, descending).unwrap() + } + + pub fn argsort_stable(&self, stable: bool, dim: i64, descending: bool) -> Tensor { + self.f_argsort_stable(stable, dim, descending).unwrap() + } + + pub fn argsort_stable_out( + &self, + out: &Tensor, + stable: bool, + dim: i64, + descending: bool, + ) -> Tensor { + self.f_argsort_stable_out(out, stable, dim, descending).unwrap() + } + + pub fn argwhere(&self) -> Tensor { + self.f_argwhere().unwrap() + } + + pub fn as_strided( + &self, + size: impl IntList, + stride: impl IntList, + storage_offset: impl Into>, + ) -> Tensor { + self.f_as_strided(size, stride, storage_offset).unwrap() + } + + pub fn as_strided_( + &mut self, + size: impl IntList, + stride: impl IntList, + storage_offset: impl Into>, + ) -> Tensor { + self.f_as_strided_(size, stride, storage_offset).unwrap() + } + + pub fn as_strided_copy( + &self, + size: impl IntList, + stride: impl IntList, + storage_offset: impl Into>, + ) -> Tensor { + self.f_as_strided_copy(size, stride, storage_offset).unwrap() + } + + pub fn as_strided_copy_out( + &self, + out: &Tensor, + size: impl IntList, + stride: impl IntList, + storage_offset: impl Into>, + ) -> Tensor { + self.f_as_strided_copy_out(out, size, stride, storage_offset).unwrap() + } + + pub fn as_strided_scatter( + &self, + src: &Tensor, + size: impl IntList, + stride: impl IntList, + storage_offset: impl Into>, + ) -> Tensor { + self.f_as_strided_scatter(src, size, stride, storage_offset).unwrap() + } + + pub fn as_strided_scatter_out( + &self, + out: &Tensor, + src: &Tensor, + size: impl IntList, + stride: impl IntList, + storage_offset: impl Into>, + ) -> Tensor { + self.f_as_strided_scatter_out(out, src, size, stride, storage_offset).unwrap() + } + + pub fn asin(&self) -> Tensor { + self.f_asin().unwrap() + } + + pub fn asin_(&mut self) -> Tensor { + self.f_asin_().unwrap() + } + + pub fn asin_out(&self, out: &Tensor) -> Tensor { + self.f_asin_out(out).unwrap() + } + + pub fn asinh(&self) -> Tensor { + self.f_asinh().unwrap() + } + + pub fn asinh_(&mut self) -> Tensor { + self.f_asinh_().unwrap() + } + + pub fn asinh_out(&self, out: &Tensor) -> Tensor { + self.f_asinh_out(out).unwrap() + } + + pub fn atan(&self) -> Tensor { + self.f_atan().unwrap() + } + + pub fn atan2(&self, other: &Tensor) -> Tensor { + self.f_atan2(other).unwrap() + } + + pub fn atan2_(&mut self, other: &Tensor) -> Tensor { + self.f_atan2_(other).unwrap() + } + + pub fn atan2_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_atan2_out(out, other).unwrap() + } + + pub fn atan_(&mut self) -> Tensor { + self.f_atan_().unwrap() + } + + pub fn atan_out(&self, out: &Tensor) -> Tensor { + self.f_atan_out(out).unwrap() + } + + pub fn atanh(&self) -> Tensor { + self.f_atanh().unwrap() + } + + pub fn atanh_(&mut self) -> Tensor { + self.f_atanh_().unwrap() + } + + pub fn atanh_out(&self, out: &Tensor) -> Tensor { + self.f_atanh_out(out).unwrap() + } + + pub fn atleast_1d(&self) -> Tensor { + self.f_atleast_1d().unwrap() + } + + pub fn atleast_1d_sequence>(tensors: &[T]) -> Vec { + Tensor::f_atleast_1d_sequence(tensors).unwrap() + } + + pub fn atleast_2d(&self) -> Tensor { + self.f_atleast_2d().unwrap() + } + + pub fn atleast_2d_sequence>(tensors: &[T]) -> Vec { + Tensor::f_atleast_2d_sequence(tensors).unwrap() + } + + pub fn atleast_3d(&self) -> Tensor { + self.f_atleast_3d().unwrap() + } + + pub fn atleast_3d_sequence>(tensors: &[T]) -> Vec { + Tensor::f_atleast_3d_sequence(tensors).unwrap() + } + + pub fn avg_pool1d( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ceil_mode: bool, + count_include_pad: bool, + ) -> Tensor { + self.f_avg_pool1d(kernel_size, stride, padding, ceil_mode, count_include_pad).unwrap() + } + + pub fn avg_pool1d_out( + &self, + out: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ceil_mode: bool, + count_include_pad: bool, + ) -> Tensor { + self.f_avg_pool1d_out(out, kernel_size, stride, padding, ceil_mode, count_include_pad) + .unwrap() + } + + pub fn avg_pool2d( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ceil_mode: bool, + count_include_pad: bool, + divisor_override: impl Into>, + ) -> Tensor { + self.f_avg_pool2d( + kernel_size, + stride, + padding, + ceil_mode, + count_include_pad, + divisor_override, + ) + .unwrap() + } + + pub fn avg_pool2d_backward( + &self, + grad_output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ceil_mode: bool, + count_include_pad: bool, + divisor_override: impl Into>, + ) -> Tensor { + self.f_avg_pool2d_backward( + grad_output, + kernel_size, + stride, + padding, + ceil_mode, + count_include_pad, + divisor_override, + ) + .unwrap() + } + + pub fn avg_pool2d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ceil_mode: bool, + count_include_pad: bool, + divisor_override: impl Into>, + ) -> Tensor { + self.f_avg_pool2d_backward_grad_input( + grad_input, + grad_output, + kernel_size, + stride, + padding, + ceil_mode, + count_include_pad, + divisor_override, + ) + .unwrap() + } + + pub fn avg_pool2d_out( + &self, + out: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ceil_mode: bool, + count_include_pad: bool, + divisor_override: impl Into>, + ) -> Tensor { + self.f_avg_pool2d_out( + out, + kernel_size, + stride, + padding, + ceil_mode, + count_include_pad, + divisor_override, + ) + .unwrap() + } + + pub fn avg_pool3d( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ceil_mode: bool, + count_include_pad: bool, + divisor_override: impl Into>, + ) -> Tensor { + self.f_avg_pool3d( + kernel_size, + stride, + padding, + ceil_mode, + count_include_pad, + divisor_override, + ) + .unwrap() + } + + pub fn avg_pool3d_backward( + &self, + grad_output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ceil_mode: bool, + count_include_pad: bool, + divisor_override: impl Into>, + ) -> Tensor { + self.f_avg_pool3d_backward( + grad_output, + kernel_size, + stride, + padding, + ceil_mode, + count_include_pad, + divisor_override, + ) + .unwrap() + } + + pub fn avg_pool3d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ceil_mode: bool, + count_include_pad: bool, + divisor_override: impl Into>, + ) -> Tensor { + self.f_avg_pool3d_backward_grad_input( + grad_input, + grad_output, + kernel_size, + stride, + padding, + ceil_mode, + count_include_pad, + divisor_override, + ) + .unwrap() + } + + pub fn avg_pool3d_out( + &self, + out: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ceil_mode: bool, + count_include_pad: bool, + divisor_override: impl Into>, + ) -> Tensor { + self.f_avg_pool3d_out( + out, + kernel_size, + stride, + padding, + ceil_mode, + count_include_pad, + divisor_override, + ) + .unwrap() + } + + pub fn baddbmm>( + &self, + batch1: &Tensor, + batch2: &Tensor, + beta: S, + alpha: S, + ) -> Tensor { + self.f_baddbmm(batch1, batch2, beta, alpha).unwrap() + } + + pub fn baddbmm_(&mut self, batch1: &Tensor, batch2: &Tensor) -> Tensor { + self.f_baddbmm_(batch1, batch2).unwrap() + } + + pub fn baddbmm_dtype>( + &self, + batch1: &Tensor, + batch2: &Tensor, + out_dtype: Kind, + beta: S, + alpha: S, + ) -> Tensor { + self.f_baddbmm_dtype(batch1, batch2, out_dtype, beta, alpha).unwrap() + } + + pub fn baddbmm_dtype_out( + &self, + out: &Tensor, + batch1: &Tensor, + batch2: &Tensor, + out_dtype: Kind, + ) -> Tensor { + self.f_baddbmm_dtype_out(out, batch1, batch2, out_dtype).unwrap() + } + + pub fn baddbmm_out(&self, out: &Tensor, batch1: &Tensor, batch2: &Tensor) -> Tensor { + self.f_baddbmm_out(out, batch1, batch2).unwrap() + } + + pub fn bartlett_window(window_length: i64, options: (Kind, Device)) -> Tensor { + Tensor::f_bartlett_window(window_length, options).unwrap() + } + + pub fn bartlett_window_out(out: &Tensor, window_length: i64) -> Tensor { + Tensor::f_bartlett_window_out(out, window_length).unwrap() + } + + pub fn bartlett_window_periodic( + window_length: i64, + periodic: bool, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_bartlett_window_periodic(window_length, periodic, options).unwrap() + } + + pub fn bartlett_window_periodic_out( + out: &Tensor, + window_length: i64, + periodic: bool, + ) -> Tensor { + Tensor::f_bartlett_window_periodic_out(out, window_length, periodic).unwrap() + } + + pub fn batch_norm>( + &self, + weight: Option, + bias: Option, + running_mean: Option, + running_var: Option, + training: bool, + momentum: f64, + eps: f64, + cudnn_enabled: bool, + ) -> Tensor { + self.f_batch_norm( + weight, + bias, + running_mean, + running_var, + training, + momentum, + eps, + cudnn_enabled, + ) + .unwrap() + } + + pub fn batch_norm_backward_elemt>( + &self, + grad_out: &Tensor, + mean: &Tensor, + invstd: &Tensor, + weight: Option, + sum_dy: &Tensor, + sum_dy_xmu: &Tensor, + count: &Tensor, + ) -> Tensor { + self.f_batch_norm_backward_elemt(grad_out, mean, invstd, weight, sum_dy, sum_dy_xmu, count) + .unwrap() + } + + pub fn batch_norm_backward_elemt_out>( + &self, + out: &Tensor, + grad_out: &Tensor, + mean: &Tensor, + invstd: &Tensor, + weight: Option, + sum_dy: &Tensor, + sum_dy_xmu: &Tensor, + count: &Tensor, + ) -> Tensor { + self.f_batch_norm_backward_elemt_out( + out, grad_out, mean, invstd, weight, sum_dy, sum_dy_xmu, count, + ) + .unwrap() + } + + pub fn batch_norm_backward_reduce>( + &self, + grad_out: &Tensor, + mean: &Tensor, + invstd: &Tensor, + weight: Option, + input_g: bool, + weight_g: bool, + bias_g: bool, + ) -> (Tensor, Tensor, Tensor, Tensor) { + self.f_batch_norm_backward_reduce(grad_out, mean, invstd, weight, input_g, weight_g, bias_g) + .unwrap() + } + + pub fn batch_norm_backward_reduce_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + out3: &Tensor, + grad_out: &Tensor, + mean: &Tensor, + invstd: &Tensor, + weight: Option, + input_g: bool, + weight_g: bool, + bias_g: bool, + ) -> (Tensor, Tensor, Tensor, Tensor) { + self.f_batch_norm_backward_reduce_out( + out0, out1, out2, out3, grad_out, mean, invstd, weight, input_g, weight_g, bias_g, + ) + .unwrap() + } + + pub fn batch_norm_elemt>( + &self, + weight: Option, + bias: Option, + mean: &Tensor, + invstd: &Tensor, + eps: f64, + ) -> Tensor { + self.f_batch_norm_elemt(weight, bias, mean, invstd, eps).unwrap() + } + + pub fn batch_norm_elemt_out>( + &self, + out: &Tensor, + weight: Option, + bias: Option, + mean: &Tensor, + invstd: &Tensor, + eps: f64, + ) -> Tensor { + self.f_batch_norm_elemt_out(out, weight, bias, mean, invstd, eps).unwrap() + } + + pub fn batch_norm_gather_stats>( + &self, + mean: &Tensor, + invstd: &Tensor, + running_mean: Option, + running_var: Option, + momentum: f64, + eps: f64, + count: i64, + ) -> (Tensor, Tensor) { + self.f_batch_norm_gather_stats( + mean, + invstd, + running_mean, + running_var, + momentum, + eps, + count, + ) + .unwrap() + } + + pub fn batch_norm_gather_stats_out>( + &self, + out0: &Tensor, + out1: &Tensor, + mean: &Tensor, + invstd: &Tensor, + running_mean: Option, + running_var: Option, + momentum: f64, + eps: f64, + count: i64, + ) -> (Tensor, Tensor) { + self.f_batch_norm_gather_stats_out( + out0, + out1, + mean, + invstd, + running_mean, + running_var, + momentum, + eps, + count, + ) + .unwrap() + } + + pub fn batch_norm_gather_stats_with_counts>( + &self, + mean: &Tensor, + invstd: &Tensor, + running_mean: Option, + running_var: Option, + momentum: f64, + eps: f64, + counts: &Tensor, + ) -> (Tensor, Tensor) { + self.f_batch_norm_gather_stats_with_counts( + mean, + invstd, + running_mean, + running_var, + momentum, + eps, + counts, + ) + .unwrap() + } + + pub fn batch_norm_gather_stats_with_counts_out>( + &self, + out0: &Tensor, + out1: &Tensor, + mean: &Tensor, + invstd: &Tensor, + running_mean: Option, + running_var: Option, + momentum: f64, + eps: f64, + counts: &Tensor, + ) -> (Tensor, Tensor) { + self.f_batch_norm_gather_stats_with_counts_out( + out0, + out1, + mean, + invstd, + running_mean, + running_var, + momentum, + eps, + counts, + ) + .unwrap() + } + + pub fn batch_norm_stats(&self, eps: f64) -> (Tensor, Tensor) { + self.f_batch_norm_stats(eps).unwrap() + } + + pub fn batch_norm_stats_out(&self, out0: &Tensor, out1: &Tensor, eps: f64) -> (Tensor, Tensor) { + self.f_batch_norm_stats_out(out0, out1, eps).unwrap() + } + + pub fn batch_norm_update_stats>( + &self, + running_mean: Option, + running_var: Option, + momentum: f64, + ) -> (Tensor, Tensor) { + self.f_batch_norm_update_stats(running_mean, running_var, momentum).unwrap() + } + + pub fn batch_norm_update_stats_out>( + &self, + out0: &Tensor, + out1: &Tensor, + running_mean: Option, + running_var: Option, + momentum: f64, + ) -> (Tensor, Tensor) { + self.f_batch_norm_update_stats_out(out0, out1, running_mean, running_var, momentum).unwrap() + } + + pub fn bernoulli(&self) -> Tensor { + self.f_bernoulli().unwrap() + } + + pub fn bernoulli_(&mut self, p: &Tensor) -> Tensor { + self.f_bernoulli_(p).unwrap() + } + + pub fn bernoulli_float_(&mut self, p: f64) -> Tensor { + self.f_bernoulli_float_(p).unwrap() + } + + pub fn bernoulli_p(&self, p: f64) -> Tensor { + self.f_bernoulli_p(p).unwrap() + } + + pub fn bernoulli_tensor(&self, p: &Tensor) -> Tensor { + self.f_bernoulli_tensor(p).unwrap() + } + + pub fn bilinear>( + input1: &Tensor, + input2: &Tensor, + weight: &Tensor, + bias: Option, + ) -> Tensor { + Tensor::f_bilinear(input1, input2, weight, bias).unwrap() + } + + pub fn binary_cross_entropy>( + &self, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ) -> Tensor { + self.f_binary_cross_entropy(target, weight, reduction).unwrap() + } + + pub fn binary_cross_entropy_backward>( + &self, + grad_output: &Tensor, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ) -> Tensor { + self.f_binary_cross_entropy_backward(grad_output, target, weight, reduction).unwrap() + } + + pub fn binary_cross_entropy_backward_grad_input>( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ) -> Tensor { + self.f_binary_cross_entropy_backward_grad_input( + grad_input, + grad_output, + target, + weight, + reduction, + ) + .unwrap() + } + + pub fn binary_cross_entropy_out>( + &self, + out: &Tensor, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ) -> Tensor { + self.f_binary_cross_entropy_out(out, target, weight, reduction).unwrap() + } + + pub fn binary_cross_entropy_with_logits>( + &self, + target: &Tensor, + weight: Option, + pos_weight: Option, + reduction: crate::Reduction, + ) -> Tensor { + self.f_binary_cross_entropy_with_logits(target, weight, pos_weight, reduction).unwrap() + } + + pub fn binary_cross_entropy_with_logits_out>( + &self, + out: &Tensor, + target: &Tensor, + weight: Option, + pos_weight: Option, + reduction: crate::Reduction, + ) -> Tensor { + self.f_binary_cross_entropy_with_logits_out(out, target, weight, pos_weight, reduction) + .unwrap() + } + + pub fn bincount>(&self, weights: Option, minlength: i64) -> Tensor { + self.f_bincount(weights, minlength).unwrap() + } + + pub fn bincount_out>( + &self, + out: &Tensor, + weights: Option, + minlength: i64, + ) -> Tensor { + self.f_bincount_out(out, weights, minlength).unwrap() + } + + pub fn binomial(count: &Tensor, prob: &Tensor) -> Tensor { + Tensor::f_binomial(count, prob).unwrap() + } + + pub fn binomial_out(out: &Tensor, count: &Tensor, prob: &Tensor) -> Tensor { + Tensor::f_binomial_out(out, count, prob).unwrap() + } + + pub fn bitwise_and>(&self, other: S) -> Tensor { + self.f_bitwise_and(other).unwrap() + } + + pub fn bitwise_and_>(&mut self, other: S) -> Tensor { + self.f_bitwise_and_(other).unwrap() + } + + pub fn bitwise_and_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_bitwise_and_scalar_out(out, other).unwrap() + } + + pub fn bitwise_and_scalar_tensor>(self_scalar: S, other: &Tensor) -> Tensor { + Tensor::f_bitwise_and_scalar_tensor(self_scalar, other).unwrap() + } + + pub fn bitwise_and_scalar_tensor_out>( + out: &Tensor, + self_scalar: S, + other: &Tensor, + ) -> Tensor { + Tensor::f_bitwise_and_scalar_tensor_out(out, self_scalar, other).unwrap() + } + + pub fn bitwise_and_tensor(&self, other: &Tensor) -> Tensor { + self.f_bitwise_and_tensor(other).unwrap() + } + + pub fn bitwise_and_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_bitwise_and_tensor_(other).unwrap() + } + + pub fn bitwise_and_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_bitwise_and_tensor_out(out, other).unwrap() + } + + pub fn bitwise_left_shift(&self, other: &Tensor) -> Tensor { + self.f_bitwise_left_shift(other).unwrap() + } + + pub fn bitwise_left_shift_(&mut self, other: &Tensor) -> Tensor { + self.f_bitwise_left_shift_(other).unwrap() + } + + pub fn bitwise_left_shift_scalar_tensor>( + self_scalar: S, + other: &Tensor, + ) -> Tensor { + Tensor::f_bitwise_left_shift_scalar_tensor(self_scalar, other).unwrap() + } + + pub fn bitwise_left_shift_scalar_tensor_out>( + out: &Tensor, + self_scalar: S, + other: &Tensor, + ) -> Tensor { + Tensor::f_bitwise_left_shift_scalar_tensor_out(out, self_scalar, other).unwrap() + } + + pub fn bitwise_left_shift_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_bitwise_left_shift_tensor_out(out, other).unwrap() + } + + pub fn bitwise_left_shift_tensor_scalar>(&self, other: S) -> Tensor { + self.f_bitwise_left_shift_tensor_scalar(other).unwrap() + } + + pub fn bitwise_left_shift_tensor_scalar_>(&mut self, other: S) -> Tensor { + self.f_bitwise_left_shift_tensor_scalar_(other).unwrap() + } + + pub fn bitwise_left_shift_tensor_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Tensor { + self.f_bitwise_left_shift_tensor_scalar_out(out, other).unwrap() + } + + pub fn bitwise_not(&self) -> Tensor { + self.f_bitwise_not().unwrap() + } + + pub fn bitwise_not_(&mut self) -> Tensor { + self.f_bitwise_not_().unwrap() + } + + pub fn bitwise_not_out(&self, out: &Tensor) -> Tensor { + self.f_bitwise_not_out(out).unwrap() + } + + pub fn bitwise_or>(&self, other: S) -> Tensor { + self.f_bitwise_or(other).unwrap() + } + + pub fn bitwise_or_>(&mut self, other: S) -> Tensor { + self.f_bitwise_or_(other).unwrap() + } + + pub fn bitwise_or_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_bitwise_or_scalar_out(out, other).unwrap() + } + + pub fn bitwise_or_scalar_tensor>(self_scalar: S, other: &Tensor) -> Tensor { + Tensor::f_bitwise_or_scalar_tensor(self_scalar, other).unwrap() + } + + pub fn bitwise_or_scalar_tensor_out>( + out: &Tensor, + self_scalar: S, + other: &Tensor, + ) -> Tensor { + Tensor::f_bitwise_or_scalar_tensor_out(out, self_scalar, other).unwrap() + } + + pub fn bitwise_or_tensor(&self, other: &Tensor) -> Tensor { + self.f_bitwise_or_tensor(other).unwrap() + } + + pub fn bitwise_or_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_bitwise_or_tensor_(other).unwrap() + } + + pub fn bitwise_or_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_bitwise_or_tensor_out(out, other).unwrap() + } + + pub fn bitwise_right_shift(&self, other: &Tensor) -> Tensor { + self.f_bitwise_right_shift(other).unwrap() + } + + pub fn bitwise_right_shift_(&mut self, other: &Tensor) -> Tensor { + self.f_bitwise_right_shift_(other).unwrap() + } + + pub fn bitwise_right_shift_scalar_tensor>( + self_scalar: S, + other: &Tensor, + ) -> Tensor { + Tensor::f_bitwise_right_shift_scalar_tensor(self_scalar, other).unwrap() + } + + pub fn bitwise_right_shift_scalar_tensor_out>( + out: &Tensor, + self_scalar: S, + other: &Tensor, + ) -> Tensor { + Tensor::f_bitwise_right_shift_scalar_tensor_out(out, self_scalar, other).unwrap() + } + + pub fn bitwise_right_shift_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_bitwise_right_shift_tensor_out(out, other).unwrap() + } + + pub fn bitwise_right_shift_tensor_scalar>(&self, other: S) -> Tensor { + self.f_bitwise_right_shift_tensor_scalar(other).unwrap() + } + + pub fn bitwise_right_shift_tensor_scalar_>(&mut self, other: S) -> Tensor { + self.f_bitwise_right_shift_tensor_scalar_(other).unwrap() + } + + pub fn bitwise_right_shift_tensor_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Tensor { + self.f_bitwise_right_shift_tensor_scalar_out(out, other).unwrap() + } + + pub fn bitwise_xor>(&self, other: S) -> Tensor { + self.f_bitwise_xor(other).unwrap() + } + + pub fn bitwise_xor_>(&mut self, other: S) -> Tensor { + self.f_bitwise_xor_(other).unwrap() + } + + pub fn bitwise_xor_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_bitwise_xor_scalar_out(out, other).unwrap() + } + + pub fn bitwise_xor_scalar_tensor>(self_scalar: S, other: &Tensor) -> Tensor { + Tensor::f_bitwise_xor_scalar_tensor(self_scalar, other).unwrap() + } + + pub fn bitwise_xor_scalar_tensor_out>( + out: &Tensor, + self_scalar: S, + other: &Tensor, + ) -> Tensor { + Tensor::f_bitwise_xor_scalar_tensor_out(out, self_scalar, other).unwrap() + } + + pub fn bitwise_xor_tensor(&self, other: &Tensor) -> Tensor { + self.f_bitwise_xor_tensor(other).unwrap() + } + + pub fn bitwise_xor_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_bitwise_xor_tensor_(other).unwrap() + } + + pub fn bitwise_xor_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_bitwise_xor_tensor_out(out, other).unwrap() + } + + pub fn blackman_window(window_length: i64, options: (Kind, Device)) -> Tensor { + Tensor::f_blackman_window(window_length, options).unwrap() + } + + pub fn blackman_window_out(out: &Tensor, window_length: i64) -> Tensor { + Tensor::f_blackman_window_out(out, window_length).unwrap() + } + + pub fn blackman_window_periodic( + window_length: i64, + periodic: bool, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_blackman_window_periodic(window_length, periodic, options).unwrap() + } + + pub fn blackman_window_periodic_out( + out: &Tensor, + window_length: i64, + periodic: bool, + ) -> Tensor { + Tensor::f_blackman_window_periodic_out(out, window_length, periodic).unwrap() + } + + pub fn block_diag>(tensors: &[T]) -> Tensor { + Tensor::f_block_diag(tensors).unwrap() + } + + pub fn block_diag_out>(out: &Tensor, tensors: &[T]) -> Tensor { + Tensor::f_block_diag_out(out, tensors).unwrap() + } + + pub fn bmm(&self, mat2: &Tensor) -> Tensor { + self.f_bmm(mat2).unwrap() + } + + pub fn bmm_dtype(&self, mat2: &Tensor, out_dtype: Kind) -> Tensor { + self.f_bmm_dtype(mat2, out_dtype).unwrap() + } + + pub fn bmm_dtype_out(&self, out: &Tensor, mat2: &Tensor, out_dtype: Kind) -> Tensor { + self.f_bmm_dtype_out(out, mat2, out_dtype).unwrap() + } + + pub fn bmm_out(&self, out: &Tensor, mat2: &Tensor) -> Tensor { + self.f_bmm_out(out, mat2).unwrap() + } + + pub fn broadcast_tensors>(tensors: &[T]) -> Vec { + Tensor::f_broadcast_tensors(tensors).unwrap() + } + + pub fn broadcast_to(&self, size: impl IntList) -> Tensor { + self.f_broadcast_to(size).unwrap() + } + + pub fn bucketize(&self, boundaries: &Tensor, out_int32: bool, right: bool) -> Tensor { + self.f_bucketize(boundaries, out_int32, right).unwrap() + } + + pub fn bucketize_scalar>( + self_scalar: S, + boundaries: &Tensor, + out_int32: bool, + right: bool, + ) -> Tensor { + Tensor::f_bucketize_scalar(self_scalar, boundaries, out_int32, right).unwrap() + } + + pub fn bucketize_scalar_out>( + out: &Tensor, + self_scalar: S, + boundaries: &Tensor, + out_int32: bool, + right: bool, + ) -> Tensor { + Tensor::f_bucketize_scalar_out(out, self_scalar, boundaries, out_int32, right).unwrap() + } + + pub fn bucketize_tensor_out( + &self, + out: &Tensor, + boundaries: &Tensor, + out_int32: bool, + right: bool, + ) -> Tensor { + self.f_bucketize_tensor_out(out, boundaries, out_int32, right).unwrap() + } + + pub fn can_cast(from_: Kind, to: Kind) -> bool { + Tensor::f_can_cast(from_, to).unwrap() + } + + pub fn cartesian_prod>(tensors: &[T]) -> Tensor { + Tensor::f_cartesian_prod(tensors).unwrap() + } + + pub fn cat>(tensors: &[T], dim: i64) -> Tensor { + Tensor::f_cat(tensors, dim).unwrap() + } + + pub fn cat_out>(out: &Tensor, tensors: &[T], dim: i64) -> Tensor { + Tensor::f_cat_out(out, tensors, dim).unwrap() + } + + pub fn cauchy(&self, median: f64, sigma: f64) -> Tensor { + self.f_cauchy(median, sigma).unwrap() + } + + pub fn cauchy_(&mut self, median: f64, sigma: f64) -> Tensor { + self.f_cauchy_(median, sigma).unwrap() + } + + pub fn cauchy_out(&self, out: &Tensor, median: f64, sigma: f64) -> Tensor { + self.f_cauchy_out(out, median, sigma).unwrap() + } + + pub fn ccol_indices(&self) -> Tensor { + self.f_ccol_indices().unwrap() + } + + pub fn ccol_indices_copy(&self) -> Tensor { + self.f_ccol_indices_copy().unwrap() + } + + pub fn ccol_indices_copy_out(&self, out: &Tensor) -> Tensor { + self.f_ccol_indices_copy_out(out).unwrap() + } + + pub fn cdist(x1: &Tensor, x2: &Tensor, p: f64, compute_mode: impl Into>) -> Tensor { + Tensor::f_cdist(x1, x2, p, compute_mode).unwrap() + } + + pub fn ceil(&self) -> Tensor { + self.f_ceil().unwrap() + } + + pub fn ceil_(&mut self) -> Tensor { + self.f_ceil_().unwrap() + } + + pub fn ceil_out(&self, out: &Tensor) -> Tensor { + self.f_ceil_out(out).unwrap() + } + + pub fn celu(&self) -> Tensor { + self.f_celu().unwrap() + } + + pub fn celu_(&mut self) -> Tensor { + self.f_celu_().unwrap() + } + + pub fn celu_out(&self, out: &Tensor) -> Tensor { + self.f_celu_out(out).unwrap() + } + + pub fn chain_matmul>(matrices: &[T]) -> Tensor { + Tensor::f_chain_matmul(matrices).unwrap() + } + + pub fn chain_matmul_out>(out: &Tensor, matrices: &[T]) -> Tensor { + Tensor::f_chain_matmul_out(out, matrices).unwrap() + } + + pub fn chalf(&self) -> Tensor { + self.f_chalf().unwrap() + } + + pub fn channel_shuffle(&self, groups: i64) -> Tensor { + self.f_channel_shuffle(groups).unwrap() + } + + pub fn channel_shuffle_out(&self, out: &Tensor, groups: i64) -> Tensor { + self.f_channel_shuffle_out(out, groups).unwrap() + } + + pub fn cholesky(&self, upper: bool) -> Tensor { + self.f_cholesky(upper).unwrap() + } + + pub fn cholesky_inverse(&self, upper: bool) -> Tensor { + self.f_cholesky_inverse(upper).unwrap() + } + + pub fn cholesky_inverse_out(&self, out: &Tensor, upper: bool) -> Tensor { + self.f_cholesky_inverse_out(out, upper).unwrap() + } + + pub fn cholesky_out(&self, out: &Tensor, upper: bool) -> Tensor { + self.f_cholesky_out(out, upper).unwrap() + } + + pub fn cholesky_solve(&self, input2: &Tensor, upper: bool) -> Tensor { + self.f_cholesky_solve(input2, upper).unwrap() + } + + pub fn cholesky_solve_out(&self, out: &Tensor, input2: &Tensor, upper: bool) -> Tensor { + self.f_cholesky_solve_out(out, input2, upper).unwrap() + } + + pub fn choose_qparams_optimized( + &self, + numel: i64, + n_bins: i64, + ratio: f64, + bit_width: i64, + ) -> (Tensor, Tensor) { + self.f_choose_qparams_optimized(numel, n_bins, ratio, bit_width).unwrap() + } + + pub fn chunk(&self, chunks: i64, dim: i64) -> Vec { + self.f_chunk(chunks, dim).unwrap() + } + + pub fn clamp>(&self, min: S, max: S) -> Tensor { + self.f_clamp(min, max).unwrap() + } + + pub fn clamp_>(&mut self, min: S, max: S) -> Tensor { + self.f_clamp_(min, max).unwrap() + } + + pub fn clamp_max>(&self, max: S) -> Tensor { + self.f_clamp_max(max).unwrap() + } + + pub fn clamp_max_>(&mut self, max: S) -> Tensor { + self.f_clamp_max_(max).unwrap() + } + + pub fn clamp_max_out>(&self, out: &Tensor, max: S) -> Tensor { + self.f_clamp_max_out(out, max).unwrap() + } + + pub fn clamp_max_tensor(&self, max: &Tensor) -> Tensor { + self.f_clamp_max_tensor(max).unwrap() + } + + pub fn clamp_max_tensor_(&mut self, max: &Tensor) -> Tensor { + self.f_clamp_max_tensor_(max).unwrap() + } + + pub fn clamp_max_tensor_out(&self, out: &Tensor, max: &Tensor) -> Tensor { + self.f_clamp_max_tensor_out(out, max).unwrap() + } + + pub fn clamp_min>(&self, min: S) -> Tensor { + self.f_clamp_min(min).unwrap() + } + + pub fn clamp_min_>(&mut self, min: S) -> Tensor { + self.f_clamp_min_(min).unwrap() + } + + pub fn clamp_min_out>(&self, out: &Tensor, min: S) -> Tensor { + self.f_clamp_min_out(out, min).unwrap() + } + + pub fn clamp_min_tensor(&self, min: &Tensor) -> Tensor { + self.f_clamp_min_tensor(min).unwrap() + } + + pub fn clamp_min_tensor_(&mut self, min: &Tensor) -> Tensor { + self.f_clamp_min_tensor_(min).unwrap() + } + + pub fn clamp_min_tensor_out(&self, out: &Tensor, min: &Tensor) -> Tensor { + self.f_clamp_min_tensor_out(out, min).unwrap() + } + + pub fn clamp_out>(&self, out: &Tensor, min: S, max: S) -> Tensor { + self.f_clamp_out(out, min, max).unwrap() + } + + pub fn clamp_tensor>(&self, min: Option, max: Option) -> Tensor { + self.f_clamp_tensor(min, max).unwrap() + } + + pub fn clamp_tensor_>(&mut self, min: Option, max: Option) -> Tensor { + self.f_clamp_tensor_(min, max).unwrap() + } + + pub fn clamp_tensor_out>( + &self, + out: &Tensor, + min: Option, + max: Option, + ) -> Tensor { + self.f_clamp_tensor_out(out, min, max).unwrap() + } + + pub fn clip>(&self, min: S, max: S) -> Tensor { + self.f_clip(min, max).unwrap() + } + + pub fn clip_>(&mut self, min: S, max: S) -> Tensor { + self.f_clip_(min, max).unwrap() + } + + pub fn clip_out>(&self, out: &Tensor, min: S, max: S) -> Tensor { + self.f_clip_out(out, min, max).unwrap() + } + + pub fn clip_tensor>(&self, min: Option, max: Option) -> Tensor { + self.f_clip_tensor(min, max).unwrap() + } + + pub fn clip_tensor_>(&mut self, min: Option, max: Option) -> Tensor { + self.f_clip_tensor_(min, max).unwrap() + } + + pub fn clip_tensor_out>( + &self, + out: &Tensor, + min: Option, + max: Option, + ) -> Tensor { + self.f_clip_tensor_out(out, min, max).unwrap() + } + + pub fn clone(&self, out: &Tensor) -> Tensor { + self.f_clone(out).unwrap() + } + + pub fn coalesce(&self) -> Tensor { + self.f_coalesce().unwrap() + } + + pub fn col2im( + &self, + output_size: impl IntList, + kernel_size: impl IntList, + dilation: impl IntList, + padding: impl IntList, + stride: impl IntList, + ) -> Tensor { + self.f_col2im(output_size, kernel_size, dilation, padding, stride).unwrap() + } + + pub fn col2im_out( + &self, + out: &Tensor, + output_size: impl IntList, + kernel_size: impl IntList, + dilation: impl IntList, + padding: impl IntList, + stride: impl IntList, + ) -> Tensor { + self.f_col2im_out(out, output_size, kernel_size, dilation, padding, stride).unwrap() + } + + pub fn col_indices(&self) -> Tensor { + self.f_col_indices().unwrap() + } + + pub fn col_indices_copy(&self) -> Tensor { + self.f_col_indices_copy().unwrap() + } + + pub fn col_indices_copy_out(&self, out: &Tensor) -> Tensor { + self.f_col_indices_copy_out(out).unwrap() + } + + pub fn column_stack>(tensors: &[T]) -> Tensor { + Tensor::f_column_stack(tensors).unwrap() + } + + pub fn column_stack_out>(out: &Tensor, tensors: &[T]) -> Tensor { + Tensor::f_column_stack_out(out, tensors).unwrap() + } + + pub fn combinations(&self, r: i64, with_replacement: bool) -> Tensor { + self.f_combinations(r, with_replacement).unwrap() + } + + pub fn complex(real: &Tensor, imag: &Tensor) -> Tensor { + Tensor::f_complex(real, imag).unwrap() + } + + pub fn complex_out(out: &Tensor, real: &Tensor, imag: &Tensor) -> Tensor { + Tensor::f_complex_out(out, real, imag).unwrap() + } + + pub fn concat>(tensors: &[T], dim: i64) -> Tensor { + Tensor::f_concat(tensors, dim).unwrap() + } + + pub fn concat_out>(out: &Tensor, tensors: &[T], dim: i64) -> Tensor { + Tensor::f_concat_out(out, tensors, dim).unwrap() + } + + pub fn concatenate>(tensors: &[T], dim: i64) -> Tensor { + Tensor::f_concatenate(tensors, dim).unwrap() + } + + pub fn concatenate_out>(out: &Tensor, tensors: &[T], dim: i64) -> Tensor { + Tensor::f_concatenate_out(out, tensors, dim).unwrap() + } + + pub fn conj(&self) -> Tensor { + self.f_conj().unwrap() + } + + pub fn conj_physical(&self) -> Tensor { + self.f_conj_physical().unwrap() + } + + pub fn conj_physical_(&mut self) -> Tensor { + self.f_conj_physical_().unwrap() + } + + pub fn conj_physical_out(&self, out: &Tensor) -> Tensor { + self.f_conj_physical_out(out).unwrap() + } + + pub fn constant_pad_nd(&self, pad: impl IntList) -> Tensor { + self.f_constant_pad_nd(pad).unwrap() + } + + pub fn constant_pad_nd_out(&self, out: &Tensor, pad: impl IntList) -> Tensor { + self.f_constant_pad_nd_out(out, pad).unwrap() + } + + pub fn contiguous(&self) -> Tensor { + self.f_contiguous().unwrap() + } + + pub fn conv1d>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Tensor { + self.f_conv1d(weight, bias, stride, padding, dilation, groups).unwrap() + } + + pub fn conv1d_padding>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: &str, + dilation: impl IntList, + groups: i64, + ) -> Tensor { + self.f_conv1d_padding(weight, bias, stride, padding, dilation, groups).unwrap() + } + + pub fn conv2d>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Tensor { + self.f_conv2d(weight, bias, stride, padding, dilation, groups).unwrap() + } + + pub fn conv2d_padding>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: &str, + dilation: impl IntList, + groups: i64, + ) -> Tensor { + self.f_conv2d_padding(weight, bias, stride, padding, dilation, groups).unwrap() + } + + pub fn conv3d>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Tensor { + self.f_conv3d(weight, bias, stride, padding, dilation, groups).unwrap() + } + + pub fn conv3d_padding>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: &str, + dilation: impl IntList, + groups: i64, + ) -> Tensor { + self.f_conv3d_padding(weight, bias, stride, padding, dilation, groups).unwrap() + } + + pub fn conv_depthwise3d>( + &self, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ) -> Tensor { + self.f_conv_depthwise3d(weight, kernel_size, bias, stride, padding, dilation).unwrap() + } + + pub fn conv_depthwise3d_out>( + &self, + out: &Tensor, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ) -> Tensor { + self.f_conv_depthwise3d_out(out, weight, kernel_size, bias, stride, padding, dilation) + .unwrap() + } + + pub fn conv_tbc(&self, weight: &Tensor, bias: &Tensor, pad: i64) -> Tensor { + self.f_conv_tbc(weight, bias, pad).unwrap() + } + + pub fn conv_tbc_backward( + &self, + input: &Tensor, + weight: &Tensor, + bias: &Tensor, + pad: i64, + ) -> (Tensor, Tensor, Tensor) { + self.f_conv_tbc_backward(input, weight, bias, pad).unwrap() + } + + pub fn conv_tbc_out(&self, out: &Tensor, weight: &Tensor, bias: &Tensor, pad: i64) -> Tensor { + self.f_conv_tbc_out(out, weight, bias, pad).unwrap() + } + + pub fn conv_transpose1d>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + output_padding: impl IntList, + groups: i64, + dilation: impl IntList, + ) -> Tensor { + self.f_conv_transpose1d(weight, bias, stride, padding, output_padding, groups, dilation) + .unwrap() + } + + pub fn conv_transpose2d>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + output_padding: impl IntList, + groups: i64, + dilation: impl IntList, + ) -> Tensor { + self.f_conv_transpose2d(weight, bias, stride, padding, output_padding, groups, dilation) + .unwrap() + } + + pub fn conv_transpose3d>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + output_padding: impl IntList, + groups: i64, + dilation: impl IntList, + ) -> Tensor { + self.f_conv_transpose3d(weight, bias, stride, padding, output_padding, groups, dilation) + .unwrap() + } + + pub fn convolution>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + transposed: bool, + output_padding: impl IntList, + groups: i64, + ) -> Tensor { + self.f_convolution( + weight, + bias, + stride, + padding, + dilation, + transposed, + output_padding, + groups, + ) + .unwrap() + } + + pub fn convolution_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + transposed: bool, + output_padding: impl IntList, + groups: i64, + ) -> Tensor { + self.f_convolution_out( + out, + weight, + bias, + stride, + padding, + dilation, + transposed, + output_padding, + groups, + ) + .unwrap() + } + + pub fn convolution_overrideable>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + transposed: bool, + output_padding: impl IntList, + groups: i64, + ) -> Tensor { + self.f_convolution_overrideable( + weight, + bias, + stride, + padding, + dilation, + transposed, + output_padding, + groups, + ) + .unwrap() + } + + pub fn convolution_overrideable_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + transposed: bool, + output_padding: impl IntList, + groups: i64, + ) -> Tensor { + self.f_convolution_overrideable_out( + out, + weight, + bias, + stride, + padding, + dilation, + transposed, + output_padding, + groups, + ) + .unwrap() + } + + pub fn copy_sparse_to_sparse(&self, src: &Tensor, non_blocking: bool) -> Tensor { + self.f_copy_sparse_to_sparse(src, non_blocking).unwrap() + } + + pub fn copy_sparse_to_sparse_(&mut self, src: &Tensor, non_blocking: bool) -> Tensor { + self.f_copy_sparse_to_sparse_(src, non_blocking).unwrap() + } + + pub fn copy_sparse_to_sparse_out( + &self, + out: &Tensor, + src: &Tensor, + non_blocking: bool, + ) -> Tensor { + self.f_copy_sparse_to_sparse_out(out, src, non_blocking).unwrap() + } + + pub fn copysign(&self, other: &Tensor) -> Tensor { + self.f_copysign(other).unwrap() + } + + pub fn copysign_(&mut self, other: &Tensor) -> Tensor { + self.f_copysign_(other).unwrap() + } + + pub fn copysign_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_copysign_out(out, other).unwrap() + } + + pub fn copysign_scalar>(&self, other: S) -> Tensor { + self.f_copysign_scalar(other).unwrap() + } + + pub fn copysign_scalar_>(&mut self, other: S) -> Tensor { + self.f_copysign_scalar_(other).unwrap() + } + + pub fn copysign_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_copysign_scalar_out(out, other).unwrap() + } + + pub fn corrcoef(&self) -> Tensor { + self.f_corrcoef().unwrap() + } + + pub fn cos(&self) -> Tensor { + self.f_cos().unwrap() + } + + pub fn cos_(&mut self) -> Tensor { + self.f_cos_().unwrap() + } + + pub fn cos_out(&self, out: &Tensor) -> Tensor { + self.f_cos_out(out).unwrap() + } + + pub fn cosh(&self) -> Tensor { + self.f_cosh().unwrap() + } + + pub fn cosh_(&mut self) -> Tensor { + self.f_cosh_().unwrap() + } + + pub fn cosh_out(&self, out: &Tensor) -> Tensor { + self.f_cosh_out(out).unwrap() + } + + pub fn cosine_embedding_loss( + input1: &Tensor, + input2: &Tensor, + target: &Tensor, + margin: f64, + reduction: crate::Reduction, + ) -> Tensor { + Tensor::f_cosine_embedding_loss(input1, input2, target, margin, reduction).unwrap() + } + + pub fn cosine_similarity(x1: &Tensor, x2: &Tensor, dim: i64, eps: f64) -> Tensor { + Tensor::f_cosine_similarity(x1, x2, dim, eps).unwrap() + } + + pub fn count_nonzero(&self, dim: impl Into>) -> Tensor { + self.f_count_nonzero(dim).unwrap() + } + + pub fn count_nonzero_dim_intlist(&self, dim: impl IntList) -> Tensor { + self.f_count_nonzero_dim_intlist(dim).unwrap() + } + + pub fn count_nonzero_dim_intlist_out(&self, out: &Tensor, dim: impl IntList) -> Tensor { + self.f_count_nonzero_dim_intlist_out(out, dim).unwrap() + } + + pub fn count_nonzero_out(&self, out: &Tensor, dim: impl Into>) -> Tensor { + self.f_count_nonzero_out(out, dim).unwrap() + } + + pub fn cov>( + &self, + correction: i64, + fweights: Option, + aweights: Option, + ) -> Tensor { + self.f_cov(correction, fweights, aweights).unwrap() + } + + pub fn cross(&self, other: &Tensor, dim: impl Into>) -> Tensor { + self.f_cross(other, dim).unwrap() + } + + pub fn cross_entropy_loss>( + &self, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ignore_index: i64, + label_smoothing: f64, + ) -> Tensor { + self.f_cross_entropy_loss(target, weight, reduction, ignore_index, label_smoothing).unwrap() + } + + pub fn cross_out(&self, out: &Tensor, other: &Tensor, dim: impl Into>) -> Tensor { + self.f_cross_out(out, other, dim).unwrap() + } + + pub fn crow_indices(&self) -> Tensor { + self.f_crow_indices().unwrap() + } + + pub fn crow_indices_copy(&self) -> Tensor { + self.f_crow_indices_copy().unwrap() + } + + pub fn crow_indices_copy_out(&self, out: &Tensor) -> Tensor { + self.f_crow_indices_copy_out(out).unwrap() + } + + pub fn ctc_loss( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: impl IntList, + target_lengths: impl IntList, + blank: i64, + reduction: crate::Reduction, + zero_infinity: bool, + ) -> Tensor { + Tensor::f_ctc_loss( + log_probs, + targets, + input_lengths, + target_lengths, + blank, + reduction, + zero_infinity, + ) + .unwrap() + } + + pub fn ctc_loss_tensor( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: &Tensor, + target_lengths: &Tensor, + blank: i64, + reduction: crate::Reduction, + zero_infinity: bool, + ) -> Tensor { + Tensor::f_ctc_loss_tensor( + log_probs, + targets, + input_lengths, + target_lengths, + blank, + reduction, + zero_infinity, + ) + .unwrap() + } + + pub fn cudnn_affine_grid_generator(theta: &Tensor, n: i64, c: i64, h: i64, w: i64) -> Tensor { + Tensor::f_cudnn_affine_grid_generator(theta, n, c, h, w).unwrap() + } + + pub fn cudnn_affine_grid_generator_backward( + grad: &Tensor, + n: i64, + c: i64, + h: i64, + w: i64, + ) -> Tensor { + Tensor::f_cudnn_affine_grid_generator_backward(grad, n, c, h, w).unwrap() + } + + pub fn cudnn_affine_grid_generator_backward_out( + out: &Tensor, + grad: &Tensor, + n: i64, + c: i64, + h: i64, + w: i64, + ) -> Tensor { + Tensor::f_cudnn_affine_grid_generator_backward_out(out, grad, n, c, h, w).unwrap() + } + + pub fn cudnn_affine_grid_generator_out( + out: &Tensor, + theta: &Tensor, + n: i64, + c: i64, + h: i64, + w: i64, + ) -> Tensor { + Tensor::f_cudnn_affine_grid_generator_out(out, theta, n, c, h, w).unwrap() + } + + pub fn cudnn_batch_norm>( + &self, + weight: &Tensor, + bias: Option, + running_mean: Option, + running_var: Option, + training: bool, + exponential_average_factor: f64, + epsilon: f64, + ) -> (Tensor, Tensor, Tensor, Tensor) { + self.f_cudnn_batch_norm( + weight, + bias, + running_mean, + running_var, + training, + exponential_average_factor, + epsilon, + ) + .unwrap() + } + + pub fn cudnn_batch_norm_backward>( + &self, + grad_output: &Tensor, + weight: &Tensor, + running_mean: Option, + running_var: Option, + save_mean: Option, + save_var: Option, + epsilon: f64, + reservespace: &Tensor, + ) -> (Tensor, Tensor, Tensor) { + self.f_cudnn_batch_norm_backward( + grad_output, + weight, + running_mean, + running_var, + save_mean, + save_var, + epsilon, + reservespace, + ) + .unwrap() + } + + pub fn cudnn_batch_norm_backward_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + grad_output: &Tensor, + weight: &Tensor, + running_mean: Option, + running_var: Option, + save_mean: Option, + save_var: Option, + epsilon: f64, + reservespace: &Tensor, + ) -> (Tensor, Tensor, Tensor) { + self.f_cudnn_batch_norm_backward_out( + out0, + out1, + out2, + grad_output, + weight, + running_mean, + running_var, + save_mean, + save_var, + epsilon, + reservespace, + ) + .unwrap() + } + + pub fn cudnn_batch_norm_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + out3: &Tensor, + weight: &Tensor, + bias: Option, + running_mean: Option, + running_var: Option, + training: bool, + exponential_average_factor: f64, + epsilon: f64, + ) -> (Tensor, Tensor, Tensor, Tensor) { + self.f_cudnn_batch_norm_out( + out0, + out1, + out2, + out3, + weight, + bias, + running_mean, + running_var, + training, + exponential_average_factor, + epsilon, + ) + .unwrap() + } + + pub fn cudnn_convolution( + &self, + weight: &Tensor, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + allow_tf32: bool, + ) -> Tensor { + self.f_cudnn_convolution( + weight, + padding, + stride, + dilation, + groups, + benchmark, + deterministic, + allow_tf32, + ) + .unwrap() + } + + pub fn cudnn_convolution_add_relu, S: Into>( + &self, + weight: &Tensor, + z: &Tensor, + alpha: S, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Tensor { + self.f_cudnn_convolution_add_relu(weight, z, alpha, bias, stride, padding, dilation, groups) + .unwrap() + } + + pub fn cudnn_convolution_add_relu_out, S: Into>( + &self, + out: &Tensor, + weight: &Tensor, + z: &Tensor, + alpha: S, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Tensor { + self.f_cudnn_convolution_add_relu_out( + out, weight, z, alpha, bias, stride, padding, dilation, groups, + ) + .unwrap() + } + + pub fn cudnn_convolution_out( + &self, + out: &Tensor, + weight: &Tensor, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + allow_tf32: bool, + ) -> Tensor { + self.f_cudnn_convolution_out( + out, + weight, + padding, + stride, + dilation, + groups, + benchmark, + deterministic, + allow_tf32, + ) + .unwrap() + } + + pub fn cudnn_convolution_relu>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Tensor { + self.f_cudnn_convolution_relu(weight, bias, stride, padding, dilation, groups).unwrap() + } + + pub fn cudnn_convolution_relu_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Tensor { + self.f_cudnn_convolution_relu_out(out, weight, bias, stride, padding, dilation, groups) + .unwrap() + } + + pub fn cudnn_convolution_transpose( + &self, + weight: &Tensor, + padding: impl IntList, + output_padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + allow_tf32: bool, + ) -> Tensor { + self.f_cudnn_convolution_transpose( + weight, + padding, + output_padding, + stride, + dilation, + groups, + benchmark, + deterministic, + allow_tf32, + ) + .unwrap() + } + + pub fn cudnn_convolution_transpose_out( + &self, + out: &Tensor, + weight: &Tensor, + padding: impl IntList, + output_padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + allow_tf32: bool, + ) -> Tensor { + self.f_cudnn_convolution_transpose_out( + out, + weight, + padding, + output_padding, + stride, + dilation, + groups, + benchmark, + deterministic, + allow_tf32, + ) + .unwrap() + } + + pub fn cudnn_grid_sampler(&self, grid: &Tensor) -> Tensor { + self.f_cudnn_grid_sampler(grid).unwrap() + } + + pub fn cudnn_grid_sampler_backward( + &self, + grid: &Tensor, + grad_output: &Tensor, + ) -> (Tensor, Tensor) { + self.f_cudnn_grid_sampler_backward(grid, grad_output).unwrap() + } + + pub fn cudnn_grid_sampler_backward_out( + &self, + out0: &Tensor, + out1: &Tensor, + grid: &Tensor, + grad_output: &Tensor, + ) -> (Tensor, Tensor) { + self.f_cudnn_grid_sampler_backward_out(out0, out1, grid, grad_output).unwrap() + } + + pub fn cudnn_grid_sampler_out(&self, out: &Tensor, grid: &Tensor) -> Tensor { + self.f_cudnn_grid_sampler_out(out, grid).unwrap() + } + + pub fn cudnn_is_acceptable(&self) -> bool { + self.f_cudnn_is_acceptable().unwrap() + } + + pub fn cummax(&self, dim: i64) -> (Tensor, Tensor) { + self.f_cummax(dim).unwrap() + } + + pub fn cummax_out(&self, values: &Tensor, indices: &Tensor, dim: i64) -> (Tensor, Tensor) { + self.f_cummax_out(values, indices, dim).unwrap() + } + + pub fn cummaxmin_backward(&self, grad: &Tensor, indices: &Tensor, dim: i64) -> Tensor { + self.f_cummaxmin_backward(grad, indices, dim).unwrap() + } + + pub fn cummin(&self, dim: i64) -> (Tensor, Tensor) { + self.f_cummin(dim).unwrap() + } + + pub fn cummin_out(&self, values: &Tensor, indices: &Tensor, dim: i64) -> (Tensor, Tensor) { + self.f_cummin_out(values, indices, dim).unwrap() + } + + pub fn cumprod(&self, dim: i64, dtype: impl Into>) -> Tensor { + self.f_cumprod(dim, dtype).unwrap() + } + + pub fn cumprod_(&mut self, dim: i64, dtype: impl Into>) -> Tensor { + self.f_cumprod_(dim, dtype).unwrap() + } + + pub fn cumprod_backward(&self, grad: &Tensor, dim: i64, output: &Tensor) -> Tensor { + self.f_cumprod_backward(grad, dim, output).unwrap() + } + + pub fn cumprod_out(&self, out: &Tensor, dim: i64, dtype: impl Into>) -> Tensor { + self.f_cumprod_out(out, dim, dtype).unwrap() + } + + pub fn cumsum(&self, dim: i64, dtype: impl Into>) -> Tensor { + self.f_cumsum(dim, dtype).unwrap() + } + + pub fn cumsum_(&mut self, dim: i64, dtype: impl Into>) -> Tensor { + self.f_cumsum_(dim, dtype).unwrap() + } + + pub fn cumsum_out(&self, out: &Tensor, dim: i64, dtype: impl Into>) -> Tensor { + self.f_cumsum_out(out, dim, dtype).unwrap() + } + + pub fn cumulative_trapezoid(y: &Tensor, dim: i64) -> Tensor { + Tensor::f_cumulative_trapezoid(y, dim).unwrap() + } + + pub fn cumulative_trapezoid_x(y: &Tensor, x: &Tensor, dim: i64) -> Tensor { + Tensor::f_cumulative_trapezoid_x(y, x, dim).unwrap() + } + + pub fn data(&self) -> Tensor { + self.f_data().unwrap() + } + + pub fn deg2rad(&self) -> Tensor { + self.f_deg2rad().unwrap() + } + + pub fn deg2rad_(&mut self) -> Tensor { + self.f_deg2rad_().unwrap() + } + + pub fn deg2rad_out(&self, out: &Tensor) -> Tensor { + self.f_deg2rad_out(out).unwrap() + } + + pub fn dense_dim(&self) -> i64 { + self.f_dense_dim().unwrap() + } + + pub fn dequantize(&self) -> Tensor { + self.f_dequantize().unwrap() + } + + pub fn dequantize_self_out(&self, out: &Tensor) -> Tensor { + self.f_dequantize_self_out(out).unwrap() + } + + pub fn dequantize_tensors>(tensors: &[T]) -> Vec { + Tensor::f_dequantize_tensors(tensors).unwrap() + } + + pub fn dequantize_tensors_out>(out: &[T], tensors: &[T]) { + Tensor::f_dequantize_tensors_out(out, tensors).unwrap() + } + + pub fn det(&self) -> Tensor { + self.f_det().unwrap() + } + + pub fn detach(&self) -> Tensor { + self.f_detach().unwrap() + } + + pub fn detach_(&mut self) -> Tensor { + self.f_detach_().unwrap() + } + + pub fn detach_copy(&self) -> Tensor { + self.f_detach_copy().unwrap() + } + + pub fn detach_copy_out(&self, out: &Tensor) -> Tensor { + self.f_detach_copy_out(out).unwrap() + } + + pub fn diag(&self, diagonal: i64) -> Tensor { + self.f_diag(diagonal).unwrap() + } + + pub fn diag_embed(&self, offset: i64, dim1: i64, dim2: i64) -> Tensor { + self.f_diag_embed(offset, dim1, dim2).unwrap() + } + + pub fn diag_embed_out(&self, out: &Tensor, offset: i64, dim1: i64, dim2: i64) -> Tensor { + self.f_diag_embed_out(out, offset, dim1, dim2).unwrap() + } + + pub fn diag_out(&self, out: &Tensor, diagonal: i64) -> Tensor { + self.f_diag_out(out, diagonal).unwrap() + } + + pub fn diagflat(&self, offset: i64) -> Tensor { + self.f_diagflat(offset).unwrap() + } + + pub fn diagonal(&self, offset: i64, dim1: i64, dim2: i64) -> Tensor { + self.f_diagonal(offset, dim1, dim2).unwrap() + } + + pub fn diagonal_backward( + grad_output: &Tensor, + input_sizes: impl IntList, + offset: i64, + dim1: i64, + dim2: i64, + ) -> Tensor { + Tensor::f_diagonal_backward(grad_output, input_sizes, offset, dim1, dim2).unwrap() + } + + pub fn diagonal_backward_out( + out: &Tensor, + grad_output: &Tensor, + input_sizes: impl IntList, + offset: i64, + dim1: i64, + dim2: i64, + ) -> Tensor { + Tensor::f_diagonal_backward_out(out, grad_output, input_sizes, offset, dim1, dim2).unwrap() + } + + pub fn diagonal_copy(&self, offset: i64, dim1: i64, dim2: i64) -> Tensor { + self.f_diagonal_copy(offset, dim1, dim2).unwrap() + } + + pub fn diagonal_copy_out(&self, out: &Tensor, offset: i64, dim1: i64, dim2: i64) -> Tensor { + self.f_diagonal_copy_out(out, offset, dim1, dim2).unwrap() + } + + pub fn diagonal_scatter(&self, src: &Tensor, offset: i64, dim1: i64, dim2: i64) -> Tensor { + self.f_diagonal_scatter(src, offset, dim1, dim2).unwrap() + } + + pub fn diagonal_scatter_out( + &self, + out: &Tensor, + src: &Tensor, + offset: i64, + dim1: i64, + dim2: i64, + ) -> Tensor { + self.f_diagonal_scatter_out(out, src, offset, dim1, dim2).unwrap() + } + + pub fn diff>( + &self, + n: i64, + dim: i64, + prepend: Option, + append: Option, + ) -> Tensor { + self.f_diff(n, dim, prepend, append).unwrap() + } + + pub fn diff_out>( + &self, + out: &Tensor, + n: i64, + dim: i64, + prepend: Option, + append: Option, + ) -> Tensor { + self.f_diff_out(out, n, dim, prepend, append).unwrap() + } + + pub fn digamma(&self) -> Tensor { + self.f_digamma().unwrap() + } + + pub fn digamma_(&mut self) -> Tensor { + self.f_digamma_().unwrap() + } + + pub fn digamma_out(&self, out: &Tensor) -> Tensor { + self.f_digamma_out(out).unwrap() + } + + pub fn dist(&self, other: &Tensor) -> Tensor { + self.f_dist(other).unwrap() + } + + pub fn dist_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_dist_out(out, other).unwrap() + } + + pub fn g_div(&self, other: &Tensor) -> Tensor { + self.f_div(other).unwrap() + } + + pub fn g_div_(&mut self, other: &Tensor) -> Tensor { + self.f_div_(other).unwrap() + } + + pub fn div_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_div_out(out, other).unwrap() + } + + pub fn div_out_mode(&self, out: &Tensor, other: &Tensor, rounding_mode: &str) -> Tensor { + self.f_div_out_mode(out, other, rounding_mode).unwrap() + } + + pub fn g_div_scalar>(&self, other: S) -> Tensor { + self.f_div_scalar(other).unwrap() + } + + pub fn g_div_scalar_>(&mut self, other: S) -> Tensor { + self.f_div_scalar_(other).unwrap() + } + + pub fn g_div_scalar_mode>(&self, other: S, rounding_mode: &str) -> Tensor { + self.f_div_scalar_mode(other, rounding_mode).unwrap() + } + + pub fn g_div_scalar_mode_>(&mut self, other: S, rounding_mode: &str) -> Tensor { + self.f_div_scalar_mode_(other, rounding_mode).unwrap() + } + + pub fn div_scalar_mode_out>( + &self, + out: &Tensor, + other: S, + rounding_mode: &str, + ) -> Tensor { + self.f_div_scalar_mode_out(out, other, rounding_mode).unwrap() + } + + pub fn div_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_div_scalar_out(out, other).unwrap() + } + + pub fn g_div_tensor_mode(&self, other: &Tensor, rounding_mode: &str) -> Tensor { + self.f_div_tensor_mode(other, rounding_mode).unwrap() + } + + pub fn g_div_tensor_mode_(&mut self, other: &Tensor, rounding_mode: &str) -> Tensor { + self.f_div_tensor_mode_(other, rounding_mode).unwrap() + } + + pub fn divide(&self, other: &Tensor) -> Tensor { + self.f_divide(other).unwrap() + } + + pub fn divide_(&mut self, other: &Tensor) -> Tensor { + self.f_divide_(other).unwrap() + } + + pub fn divide_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_divide_out(out, other).unwrap() + } + + pub fn divide_out_mode(&self, out: &Tensor, other: &Tensor, rounding_mode: &str) -> Tensor { + self.f_divide_out_mode(out, other, rounding_mode).unwrap() + } + + pub fn divide_scalar>(&self, other: S) -> Tensor { + self.f_divide_scalar(other).unwrap() + } + + pub fn divide_scalar_>(&mut self, other: S) -> Tensor { + self.f_divide_scalar_(other).unwrap() + } + + pub fn divide_scalar_mode>(&self, other: S, rounding_mode: &str) -> Tensor { + self.f_divide_scalar_mode(other, rounding_mode).unwrap() + } + + pub fn divide_scalar_mode_>( + &mut self, + other: S, + rounding_mode: &str, + ) -> Tensor { + self.f_divide_scalar_mode_(other, rounding_mode).unwrap() + } + + pub fn divide_tensor_mode(&self, other: &Tensor, rounding_mode: &str) -> Tensor { + self.f_divide_tensor_mode(other, rounding_mode).unwrap() + } + + pub fn divide_tensor_mode_(&mut self, other: &Tensor, rounding_mode: &str) -> Tensor { + self.f_divide_tensor_mode_(other, rounding_mode).unwrap() + } + + pub fn dot(&self, tensor: &Tensor) -> Tensor { + self.f_dot(tensor).unwrap() + } + + pub fn dot_out(&self, out: &Tensor, tensor: &Tensor) -> Tensor { + self.f_dot_out(out, tensor).unwrap() + } + + pub fn dropout(&self, p: f64, train: bool) -> Tensor { + self.f_dropout(p, train).unwrap() + } + + pub fn dropout_(&mut self, p: f64, train: bool) -> Tensor { + self.f_dropout_(p, train).unwrap() + } + + pub fn dsplit(&self, sections: i64) -> Vec { + self.f_dsplit(sections).unwrap() + } + + pub fn dsplit_array(&self, indices: impl IntList) -> Vec { + self.f_dsplit_array(indices).unwrap() + } + + pub fn dstack>(tensors: &[T]) -> Tensor { + Tensor::f_dstack(tensors).unwrap() + } + + pub fn dstack_out>(out: &Tensor, tensors: &[T]) -> Tensor { + Tensor::f_dstack_out(out, tensors).unwrap() + } + + pub fn einsum>( + equation: &str, + tensors: &[T], + path: impl IntListOption, + ) -> Tensor { + Tensor::f_einsum(equation, tensors, path).unwrap() + } + + pub fn elu(&self) -> Tensor { + self.f_elu().unwrap() + } + + pub fn elu_(&mut self) -> Tensor { + self.f_elu_().unwrap() + } + + pub fn elu_backward>( + grad_output: &Tensor, + alpha: S, + scale: S, + input_scale: S, + is_result: bool, + self_or_result: &Tensor, + ) -> Tensor { + Tensor::f_elu_backward(grad_output, alpha, scale, input_scale, is_result, self_or_result) + .unwrap() + } + + pub fn elu_backward_grad_input>( + grad_input: &Tensor, + grad_output: &Tensor, + alpha: S, + scale: S, + input_scale: S, + is_result: bool, + self_or_result: &Tensor, + ) -> Tensor { + Tensor::f_elu_backward_grad_input( + grad_input, + grad_output, + alpha, + scale, + input_scale, + is_result, + self_or_result, + ) + .unwrap() + } + + pub fn elu_out(&self, out: &Tensor) -> Tensor { + self.f_elu_out(out).unwrap() + } + + pub fn embedding( + weight: &Tensor, + indices: &Tensor, + padding_idx: i64, + scale_grad_by_freq: bool, + sparse: bool, + ) -> Tensor { + Tensor::f_embedding(weight, indices, padding_idx, scale_grad_by_freq, sparse).unwrap() + } + + pub fn embedding_backward( + grad: &Tensor, + indices: &Tensor, + num_weights: i64, + padding_idx: i64, + scale_grad_by_freq: bool, + sparse: bool, + ) -> Tensor { + Tensor::f_embedding_backward( + grad, + indices, + num_weights, + padding_idx, + scale_grad_by_freq, + sparse, + ) + .unwrap() + } + + pub fn embedding_bag>( + weight: &Tensor, + indices: &Tensor, + offsets: &Tensor, + scale_grad_by_freq: bool, + mode: i64, + sparse: bool, + per_sample_weights: Option, + include_last_offset: bool, + ) -> (Tensor, Tensor, Tensor, Tensor) { + Tensor::f_embedding_bag( + weight, + indices, + offsets, + scale_grad_by_freq, + mode, + sparse, + per_sample_weights, + include_last_offset, + ) + .unwrap() + } + + pub fn embedding_bag_padding_idx>( + weight: &Tensor, + indices: &Tensor, + offsets: &Tensor, + scale_grad_by_freq: bool, + mode: i64, + sparse: bool, + per_sample_weights: Option, + include_last_offset: bool, + padding_idx: impl Into>, + ) -> (Tensor, Tensor, Tensor, Tensor) { + Tensor::f_embedding_bag_padding_idx( + weight, + indices, + offsets, + scale_grad_by_freq, + mode, + sparse, + per_sample_weights, + include_last_offset, + padding_idx, + ) + .unwrap() + } + + pub fn embedding_dense_backward( + grad_output: &Tensor, + indices: &Tensor, + num_weights: i64, + padding_idx: i64, + scale_grad_by_freq: bool, + ) -> Tensor { + Tensor::f_embedding_dense_backward( + grad_output, + indices, + num_weights, + padding_idx, + scale_grad_by_freq, + ) + .unwrap() + } + + pub fn embedding_dense_backward_out( + out: &Tensor, + grad_output: &Tensor, + indices: &Tensor, + num_weights: i64, + padding_idx: i64, + scale_grad_by_freq: bool, + ) -> Tensor { + Tensor::f_embedding_dense_backward_out( + out, + grad_output, + indices, + num_weights, + padding_idx, + scale_grad_by_freq, + ) + .unwrap() + } + + pub fn embedding_out( + out: &Tensor, + weight: &Tensor, + indices: &Tensor, + padding_idx: i64, + scale_grad_by_freq: bool, + sparse: bool, + ) -> Tensor { + Tensor::f_embedding_out(out, weight, indices, padding_idx, scale_grad_by_freq, sparse) + .unwrap() + } + + pub fn embedding_renorm(&self, indices: &Tensor, max_norm: f64, norm_type: f64) -> Tensor { + self.f_embedding_renorm(indices, max_norm, norm_type).unwrap() + } + + pub fn embedding_renorm_(&mut self, indices: &Tensor, max_norm: f64, norm_type: f64) -> Tensor { + self.f_embedding_renorm_(indices, max_norm, norm_type).unwrap() + } + + pub fn embedding_renorm_out( + &self, + out: &Tensor, + indices: &Tensor, + max_norm: f64, + norm_type: f64, + ) -> Tensor { + self.f_embedding_renorm_out(out, indices, max_norm, norm_type).unwrap() + } + + pub fn embedding_sparse_backward( + grad: &Tensor, + indices: &Tensor, + num_weights: i64, + padding_idx: i64, + scale_grad_by_freq: bool, + ) -> Tensor { + Tensor::f_embedding_sparse_backward( + grad, + indices, + num_weights, + padding_idx, + scale_grad_by_freq, + ) + .unwrap() + } + + pub fn empty(size: impl IntList, options: (Kind, Device)) -> Tensor { + Tensor::f_empty(size, options).unwrap() + } + + pub fn empty_like(&self) -> Tensor { + self.f_empty_like().unwrap() + } + + pub fn empty_like_out(&self, out: &Tensor) -> Tensor { + self.f_empty_like_out(out).unwrap() + } + + pub fn empty_out(out: &Tensor, size: impl IntList) -> Tensor { + Tensor::f_empty_out(out, size).unwrap() + } + + pub fn empty_permuted( + size: impl IntList, + physical_layout: impl IntList, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_empty_permuted(size, physical_layout, options).unwrap() + } + + pub fn empty_permuted_out( + out: &Tensor, + size: impl IntList, + physical_layout: impl IntList, + ) -> Tensor { + Tensor::f_empty_permuted_out(out, size, physical_layout).unwrap() + } + + pub fn empty_quantized( + size: impl IntList, + qtensor: &Tensor, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_empty_quantized(size, qtensor, options).unwrap() + } + + pub fn empty_quantized_out(out: &Tensor, size: impl IntList, qtensor: &Tensor) -> Tensor { + Tensor::f_empty_quantized_out(out, size, qtensor).unwrap() + } + + pub fn empty_strided( + size: impl IntList, + stride: impl IntList, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_empty_strided(size, stride, options).unwrap() + } + + pub fn empty_strided_out(out: &Tensor, size: impl IntList, stride: impl IntList) -> Tensor { + Tensor::f_empty_strided_out(out, size, stride).unwrap() + } + + pub fn eq>(&self, other: S) -> Tensor { + self.f_eq(other).unwrap() + } + + pub fn eq_>(&mut self, other: S) -> Tensor { + self.f_eq_(other).unwrap() + } + + pub fn eq_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_eq_scalar_out(out, other).unwrap() + } + + pub fn eq_tensor(&self, other: &Tensor) -> Tensor { + self.f_eq_tensor(other).unwrap() + } + + pub fn eq_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_eq_tensor_(other).unwrap() + } + + pub fn eq_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_eq_tensor_out(out, other).unwrap() + } + + pub fn equal(&self, other: &Tensor) -> bool { + self.f_equal(other).unwrap() + } + + pub fn erf(&self) -> Tensor { + self.f_erf().unwrap() + } + + pub fn erf_(&mut self) -> Tensor { + self.f_erf_().unwrap() + } + + pub fn erf_out(&self, out: &Tensor) -> Tensor { + self.f_erf_out(out).unwrap() + } + + pub fn erfc(&self) -> Tensor { + self.f_erfc().unwrap() + } + + pub fn erfc_(&mut self) -> Tensor { + self.f_erfc_().unwrap() + } + + pub fn erfc_out(&self, out: &Tensor) -> Tensor { + self.f_erfc_out(out).unwrap() + } + + pub fn erfinv(&self) -> Tensor { + self.f_erfinv().unwrap() + } + + pub fn erfinv_(&mut self) -> Tensor { + self.f_erfinv_().unwrap() + } + + pub fn erfinv_out(&self, out: &Tensor) -> Tensor { + self.f_erfinv_out(out).unwrap() + } + + pub fn exp(&self) -> Tensor { + self.f_exp().unwrap() + } + + pub fn exp2(&self) -> Tensor { + self.f_exp2().unwrap() + } + + pub fn exp2_(&mut self) -> Tensor { + self.f_exp2_().unwrap() + } + + pub fn exp2_out(&self, out: &Tensor) -> Tensor { + self.f_exp2_out(out).unwrap() + } + + pub fn exp_(&mut self) -> Tensor { + self.f_exp_().unwrap() + } + + pub fn exp_out(&self, out: &Tensor) -> Tensor { + self.f_exp_out(out).unwrap() + } + + pub fn expand(&self, size: impl IntList, implicit: bool) -> Tensor { + self.f_expand(size, implicit).unwrap() + } + + pub fn expand_as(&self, other: &Tensor) -> Tensor { + self.f_expand_as(other).unwrap() + } + + pub fn expand_copy(&self, size: impl IntList, implicit: bool) -> Tensor { + self.f_expand_copy(size, implicit).unwrap() + } + + pub fn expand_copy_out(&self, out: &Tensor, size: impl IntList, implicit: bool) -> Tensor { + self.f_expand_copy_out(out, size, implicit).unwrap() + } + + pub fn expm1(&self) -> Tensor { + self.f_expm1().unwrap() + } + + pub fn expm1_(&mut self) -> Tensor { + self.f_expm1_().unwrap() + } + + pub fn expm1_out(&self, out: &Tensor) -> Tensor { + self.f_expm1_out(out).unwrap() + } + + pub fn exponential(&self, lambd: f64) -> Tensor { + self.f_exponential(lambd).unwrap() + } + + pub fn exponential_(&mut self, lambd: f64) -> Tensor { + self.f_exponential_(lambd).unwrap() + } + + pub fn exponential_out(&self, out: &Tensor, lambd: f64) -> Tensor { + self.f_exponential_out(out, lambd).unwrap() + } + + pub fn eye(n: i64, options: (Kind, Device)) -> Tensor { + Tensor::f_eye(n, options).unwrap() + } + + pub fn eye_m(n: i64, m: i64, options: (Kind, Device)) -> Tensor { + Tensor::f_eye_m(n, m, options).unwrap() + } + + pub fn eye_m_out(out: &Tensor, n: i64, m: i64) -> Tensor { + Tensor::f_eye_m_out(out, n, m).unwrap() + } + + pub fn eye_out(out: &Tensor, n: i64) -> Tensor { + Tensor::f_eye_out(out, n).unwrap() + } + + pub fn fake_quantize_per_channel_affine( + &self, + scale: &Tensor, + zero_point: &Tensor, + axis: i64, + quant_min: i64, + quant_max: i64, + ) -> Tensor { + self.f_fake_quantize_per_channel_affine(scale, zero_point, axis, quant_min, quant_max) + .unwrap() + } + + pub fn fake_quantize_per_channel_affine_cachemask( + &self, + scale: &Tensor, + zero_point: &Tensor, + axis: i64, + quant_min: i64, + quant_max: i64, + ) -> (Tensor, Tensor) { + self.f_fake_quantize_per_channel_affine_cachemask( + scale, zero_point, axis, quant_min, quant_max, + ) + .unwrap() + } + + pub fn fake_quantize_per_channel_affine_cachemask_backward( + grad: &Tensor, + mask: &Tensor, + ) -> Tensor { + Tensor::f_fake_quantize_per_channel_affine_cachemask_backward(grad, mask).unwrap() + } + + pub fn fake_quantize_per_channel_affine_cachemask_out( + &self, + out0: &Tensor, + out1: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + axis: i64, + quant_min: i64, + quant_max: i64, + ) -> (Tensor, Tensor) { + self.f_fake_quantize_per_channel_affine_cachemask_out( + out0, out1, scale, zero_point, axis, quant_min, quant_max, + ) + .unwrap() + } + + pub fn fake_quantize_per_tensor_affine( + &self, + scale: f64, + zero_point: i64, + quant_min: i64, + quant_max: i64, + ) -> Tensor { + self.f_fake_quantize_per_tensor_affine(scale, zero_point, quant_min, quant_max).unwrap() + } + + pub fn fake_quantize_per_tensor_affine_cachemask( + &self, + scale: f64, + zero_point: i64, + quant_min: i64, + quant_max: i64, + ) -> (Tensor, Tensor) { + self.f_fake_quantize_per_tensor_affine_cachemask(scale, zero_point, quant_min, quant_max) + .unwrap() + } + + pub fn fake_quantize_per_tensor_affine_cachemask_backward( + grad: &Tensor, + mask: &Tensor, + ) -> Tensor { + Tensor::f_fake_quantize_per_tensor_affine_cachemask_backward(grad, mask).unwrap() + } + + pub fn fake_quantize_per_tensor_affine_cachemask_out( + &self, + out0: &Tensor, + out1: &Tensor, + scale: f64, + zero_point: i64, + quant_min: i64, + quant_max: i64, + ) -> (Tensor, Tensor) { + self.f_fake_quantize_per_tensor_affine_cachemask_out( + out0, out1, scale, zero_point, quant_min, quant_max, + ) + .unwrap() + } + + pub fn fake_quantize_per_tensor_affine_tensor_qparams( + &self, + scale: &Tensor, + zero_point: &Tensor, + quant_min: i64, + quant_max: i64, + ) -> Tensor { + self.f_fake_quantize_per_tensor_affine_tensor_qparams( + scale, zero_point, quant_min, quant_max, + ) + .unwrap() + } + + pub fn feature_alpha_dropout(&self, p: f64, train: bool) -> Tensor { + self.f_feature_alpha_dropout(p, train).unwrap() + } + + pub fn feature_alpha_dropout_(&mut self, p: f64, train: bool) -> Tensor { + self.f_feature_alpha_dropout_(p, train).unwrap() + } + + pub fn feature_dropout(&self, p: f64, train: bool) -> Tensor { + self.f_feature_dropout(p, train).unwrap() + } + + pub fn feature_dropout_(&mut self, p: f64, train: bool) -> Tensor { + self.f_feature_dropout_(p, train).unwrap() + } + + pub fn fft_fft(&self, n: impl Into>, dim: i64, norm: &str) -> Tensor { + self.f_fft_fft(n, dim, norm).unwrap() + } + + pub fn fft_fft2(&self, s: impl IntListOption, dim: impl IntList, norm: &str) -> Tensor { + self.f_fft_fft2(s, dim, norm).unwrap() + } + + pub fn fft_fft2_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntList, + norm: &str, + ) -> Tensor { + self.f_fft_fft2_out(out, s, dim, norm).unwrap() + } + + pub fn fft_fft_out( + &self, + out: &Tensor, + n: impl Into>, + dim: i64, + norm: &str, + ) -> Tensor { + self.f_fft_fft_out(out, n, dim, norm).unwrap() + } + + pub fn fft_fftfreq(n: i64, d: f64, options: (Kind, Device)) -> Tensor { + Tensor::f_fft_fftfreq(n, d, options).unwrap() + } + + pub fn fft_fftfreq_out(out: &Tensor, n: i64, d: f64) -> Tensor { + Tensor::f_fft_fftfreq_out(out, n, d).unwrap() + } + + pub fn fft_fftn(&self, s: impl IntListOption, dim: impl IntListOption, norm: &str) -> Tensor { + self.f_fft_fftn(s, dim, norm).unwrap() + } + + pub fn fft_fftn_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntListOption, + norm: &str, + ) -> Tensor { + self.f_fft_fftn_out(out, s, dim, norm).unwrap() + } + + pub fn fft_fftshift(&self, dim: impl IntListOption) -> Tensor { + self.f_fft_fftshift(dim).unwrap() + } + + pub fn fft_hfft(&self, n: impl Into>, dim: i64, norm: &str) -> Tensor { + self.f_fft_hfft(n, dim, norm).unwrap() + } + + pub fn fft_hfft2(&self, s: impl IntListOption, dim: impl IntList, norm: &str) -> Tensor { + self.f_fft_hfft2(s, dim, norm).unwrap() + } + + pub fn fft_hfft2_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntList, + norm: &str, + ) -> Tensor { + self.f_fft_hfft2_out(out, s, dim, norm).unwrap() + } + + pub fn fft_hfft_out( + &self, + out: &Tensor, + n: impl Into>, + dim: i64, + norm: &str, + ) -> Tensor { + self.f_fft_hfft_out(out, n, dim, norm).unwrap() + } + + pub fn fft_hfftn(&self, s: impl IntListOption, dim: impl IntListOption, norm: &str) -> Tensor { + self.f_fft_hfftn(s, dim, norm).unwrap() + } + + pub fn fft_hfftn_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntListOption, + norm: &str, + ) -> Tensor { + self.f_fft_hfftn_out(out, s, dim, norm).unwrap() + } + + pub fn fft_ifft(&self, n: impl Into>, dim: i64, norm: &str) -> Tensor { + self.f_fft_ifft(n, dim, norm).unwrap() + } + + pub fn fft_ifft2(&self, s: impl IntListOption, dim: impl IntList, norm: &str) -> Tensor { + self.f_fft_ifft2(s, dim, norm).unwrap() + } + + pub fn fft_ifft2_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntList, + norm: &str, + ) -> Tensor { + self.f_fft_ifft2_out(out, s, dim, norm).unwrap() + } + + pub fn fft_ifft_out( + &self, + out: &Tensor, + n: impl Into>, + dim: i64, + norm: &str, + ) -> Tensor { + self.f_fft_ifft_out(out, n, dim, norm).unwrap() + } + + pub fn fft_ifftn(&self, s: impl IntListOption, dim: impl IntListOption, norm: &str) -> Tensor { + self.f_fft_ifftn(s, dim, norm).unwrap() + } + + pub fn fft_ifftn_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntListOption, + norm: &str, + ) -> Tensor { + self.f_fft_ifftn_out(out, s, dim, norm).unwrap() + } + + pub fn fft_ifftshift(&self, dim: impl IntListOption) -> Tensor { + self.f_fft_ifftshift(dim).unwrap() + } + + pub fn fft_ihfft(&self, n: impl Into>, dim: i64, norm: &str) -> Tensor { + self.f_fft_ihfft(n, dim, norm).unwrap() + } + + pub fn fft_ihfft2(&self, s: impl IntListOption, dim: impl IntList, norm: &str) -> Tensor { + self.f_fft_ihfft2(s, dim, norm).unwrap() + } + + pub fn fft_ihfft2_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntList, + norm: &str, + ) -> Tensor { + self.f_fft_ihfft2_out(out, s, dim, norm).unwrap() + } + + pub fn fft_ihfft_out( + &self, + out: &Tensor, + n: impl Into>, + dim: i64, + norm: &str, + ) -> Tensor { + self.f_fft_ihfft_out(out, n, dim, norm).unwrap() + } + + pub fn fft_ihfftn(&self, s: impl IntListOption, dim: impl IntListOption, norm: &str) -> Tensor { + self.f_fft_ihfftn(s, dim, norm).unwrap() + } + + pub fn fft_ihfftn_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntListOption, + norm: &str, + ) -> Tensor { + self.f_fft_ihfftn_out(out, s, dim, norm).unwrap() + } + + pub fn fft_irfft(&self, n: impl Into>, dim: i64, norm: &str) -> Tensor { + self.f_fft_irfft(n, dim, norm).unwrap() + } + + pub fn fft_irfft2(&self, s: impl IntListOption, dim: impl IntList, norm: &str) -> Tensor { + self.f_fft_irfft2(s, dim, norm).unwrap() + } + + pub fn fft_irfft2_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntList, + norm: &str, + ) -> Tensor { + self.f_fft_irfft2_out(out, s, dim, norm).unwrap() + } + + pub fn fft_irfft_out( + &self, + out: &Tensor, + n: impl Into>, + dim: i64, + norm: &str, + ) -> Tensor { + self.f_fft_irfft_out(out, n, dim, norm).unwrap() + } + + pub fn fft_irfftn(&self, s: impl IntListOption, dim: impl IntListOption, norm: &str) -> Tensor { + self.f_fft_irfftn(s, dim, norm).unwrap() + } + + pub fn fft_irfftn_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntListOption, + norm: &str, + ) -> Tensor { + self.f_fft_irfftn_out(out, s, dim, norm).unwrap() + } + + pub fn fft_rfft(&self, n: impl Into>, dim: i64, norm: &str) -> Tensor { + self.f_fft_rfft(n, dim, norm).unwrap() + } + + pub fn fft_rfft2(&self, s: impl IntListOption, dim: impl IntList, norm: &str) -> Tensor { + self.f_fft_rfft2(s, dim, norm).unwrap() + } + + pub fn fft_rfft2_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntList, + norm: &str, + ) -> Tensor { + self.f_fft_rfft2_out(out, s, dim, norm).unwrap() + } + + pub fn fft_rfft_out( + &self, + out: &Tensor, + n: impl Into>, + dim: i64, + norm: &str, + ) -> Tensor { + self.f_fft_rfft_out(out, n, dim, norm).unwrap() + } + + pub fn fft_rfftfreq(n: i64, d: f64, options: (Kind, Device)) -> Tensor { + Tensor::f_fft_rfftfreq(n, d, options).unwrap() + } + + pub fn fft_rfftfreq_out(out: &Tensor, n: i64, d: f64) -> Tensor { + Tensor::f_fft_rfftfreq_out(out, n, d).unwrap() + } + + pub fn fft_rfftn(&self, s: impl IntListOption, dim: impl IntListOption, norm: &str) -> Tensor { + self.f_fft_rfftn(s, dim, norm).unwrap() + } + + pub fn fft_rfftn_out( + &self, + out: &Tensor, + s: impl IntListOption, + dim: impl IntListOption, + norm: &str, + ) -> Tensor { + self.f_fft_rfftn_out(out, s, dim, norm).unwrap() + } + + pub fn fill>(&self, value: S) -> Tensor { + self.f_fill(value).unwrap() + } + + pub fn fill_>(&mut self, value: S) -> Tensor { + self.f_fill_(value).unwrap() + } + + pub fn fill_diagonal_>(&mut self, fill_value: S, wrap: bool) -> Tensor { + self.f_fill_diagonal_(fill_value, wrap).unwrap() + } + + pub fn fill_scalar_out>(&self, out: &Tensor, value: S) -> Tensor { + self.f_fill_scalar_out(out, value).unwrap() + } + + pub fn fill_tensor(&self, value: &Tensor) -> Tensor { + self.f_fill_tensor(value).unwrap() + } + + pub fn fill_tensor_(&mut self, value: &Tensor) -> Tensor { + self.f_fill_tensor_(value).unwrap() + } + + pub fn fill_tensor_out(&self, out: &Tensor, value: &Tensor) -> Tensor { + self.f_fill_tensor_out(out, value).unwrap() + } + + pub fn fix(&self) -> Tensor { + self.f_fix().unwrap() + } + + pub fn fix_(&mut self) -> Tensor { + self.f_fix_().unwrap() + } + + pub fn fix_out(&self, out: &Tensor) -> Tensor { + self.f_fix_out(out).unwrap() + } + + pub fn flatten(&self, start_dim: i64, end_dim: i64) -> Tensor { + self.f_flatten(start_dim, end_dim).unwrap() + } + + pub fn flatten_dense_tensors>(tensors: &[T]) -> Tensor { + Tensor::f_flatten_dense_tensors(tensors).unwrap() + } + + pub fn flip(&self, dims: impl IntList) -> Tensor { + self.f_flip(dims).unwrap() + } + + pub fn flip_out(&self, out: &Tensor, dims: impl IntList) -> Tensor { + self.f_flip_out(out, dims).unwrap() + } + + pub fn fliplr(&self) -> Tensor { + self.f_fliplr().unwrap() + } + + pub fn flipud(&self) -> Tensor { + self.f_flipud().unwrap() + } + + pub fn float_power(&self, exponent: &Tensor) -> Tensor { + self.f_float_power(exponent).unwrap() + } + + pub fn float_power_>(&mut self, exponent: S) -> Tensor { + self.f_float_power_(exponent).unwrap() + } + + pub fn float_power_scalar>(self_scalar: S, exponent: &Tensor) -> Tensor { + Tensor::f_float_power_scalar(self_scalar, exponent).unwrap() + } + + pub fn float_power_scalar_out>( + out: &Tensor, + self_scalar: S, + exponent: &Tensor, + ) -> Tensor { + Tensor::f_float_power_scalar_out(out, self_scalar, exponent).unwrap() + } + + pub fn float_power_tensor_(&mut self, exponent: &Tensor) -> Tensor { + self.f_float_power_tensor_(exponent).unwrap() + } + + pub fn float_power_tensor_scalar>(&self, exponent: S) -> Tensor { + self.f_float_power_tensor_scalar(exponent).unwrap() + } + + pub fn float_power_tensor_scalar_out>( + &self, + out: &Tensor, + exponent: S, + ) -> Tensor { + self.f_float_power_tensor_scalar_out(out, exponent).unwrap() + } + + pub fn float_power_tensor_tensor_out(&self, out: &Tensor, exponent: &Tensor) -> Tensor { + self.f_float_power_tensor_tensor_out(out, exponent).unwrap() + } + + pub fn floor(&self) -> Tensor { + self.f_floor().unwrap() + } + + pub fn floor_(&mut self) -> Tensor { + self.f_floor_().unwrap() + } + + pub fn floor_divide(&self, other: &Tensor) -> Tensor { + self.f_floor_divide(other).unwrap() + } + + pub fn floor_divide_(&mut self, other: &Tensor) -> Tensor { + self.f_floor_divide_(other).unwrap() + } + + pub fn floor_divide_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_floor_divide_out(out, other).unwrap() + } + + pub fn floor_divide_scalar>(&self, other: S) -> Tensor { + self.f_floor_divide_scalar(other).unwrap() + } + + pub fn floor_divide_scalar_>(&mut self, other: S) -> Tensor { + self.f_floor_divide_scalar_(other).unwrap() + } + + pub fn floor_divide_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_floor_divide_scalar_out(out, other).unwrap() + } + + pub fn floor_out(&self, out: &Tensor) -> Tensor { + self.f_floor_out(out).unwrap() + } + + pub fn fmax(&self, other: &Tensor) -> Tensor { + self.f_fmax(other).unwrap() + } + + pub fn fmax_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_fmax_out(out, other).unwrap() + } + + pub fn fmin(&self, other: &Tensor) -> Tensor { + self.f_fmin(other).unwrap() + } + + pub fn fmin_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_fmin_out(out, other).unwrap() + } + + pub fn fmod>(&self, other: S) -> Tensor { + self.f_fmod(other).unwrap() + } + + pub fn fmod_>(&mut self, other: S) -> Tensor { + self.f_fmod_(other).unwrap() + } + + pub fn fmod_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_fmod_scalar_out(out, other).unwrap() + } + + pub fn fmod_tensor(&self, other: &Tensor) -> Tensor { + self.f_fmod_tensor(other).unwrap() + } + + pub fn fmod_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_fmod_tensor_(other).unwrap() + } + + pub fn fmod_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_fmod_tensor_out(out, other).unwrap() + } + + pub fn frac(&self) -> Tensor { + self.f_frac().unwrap() + } + + pub fn frac_(&mut self) -> Tensor { + self.f_frac_().unwrap() + } + + pub fn frac_out(&self, out: &Tensor) -> Tensor { + self.f_frac_out(out).unwrap() + } + + pub fn fractional_max_pool2d( + &self, + kernel_size: impl IntList, + output_size: impl IntList, + random_samples: &Tensor, + ) -> (Tensor, Tensor) { + self.f_fractional_max_pool2d(kernel_size, output_size, random_samples).unwrap() + } + + pub fn fractional_max_pool2d_backward( + &self, + grad_output: &Tensor, + kernel_size: impl IntList, + output_size: impl IntList, + indices: &Tensor, + ) -> Tensor { + self.f_fractional_max_pool2d_backward(grad_output, kernel_size, output_size, indices) + .unwrap() + } + + pub fn fractional_max_pool2d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + kernel_size: impl IntList, + output_size: impl IntList, + indices: &Tensor, + ) -> Tensor { + self.f_fractional_max_pool2d_backward_grad_input( + grad_input, + grad_output, + kernel_size, + output_size, + indices, + ) + .unwrap() + } + + pub fn fractional_max_pool2d_output( + &self, + output: &Tensor, + indices: &Tensor, + kernel_size: impl IntList, + output_size: impl IntList, + random_samples: &Tensor, + ) -> (Tensor, Tensor) { + self.f_fractional_max_pool2d_output( + output, + indices, + kernel_size, + output_size, + random_samples, + ) + .unwrap() + } + + pub fn fractional_max_pool3d( + &self, + kernel_size: impl IntList, + output_size: impl IntList, + random_samples: &Tensor, + ) -> (Tensor, Tensor) { + self.f_fractional_max_pool3d(kernel_size, output_size, random_samples).unwrap() + } + + pub fn fractional_max_pool3d_backward( + &self, + grad_output: &Tensor, + kernel_size: impl IntList, + output_size: impl IntList, + indices: &Tensor, + ) -> Tensor { + self.f_fractional_max_pool3d_backward(grad_output, kernel_size, output_size, indices) + .unwrap() + } + + pub fn fractional_max_pool3d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + kernel_size: impl IntList, + output_size: impl IntList, + indices: &Tensor, + ) -> Tensor { + self.f_fractional_max_pool3d_backward_grad_input( + grad_input, + grad_output, + kernel_size, + output_size, + indices, + ) + .unwrap() + } + + pub fn fractional_max_pool3d_output( + &self, + output: &Tensor, + indices: &Tensor, + kernel_size: impl IntList, + output_size: impl IntList, + random_samples: &Tensor, + ) -> (Tensor, Tensor) { + self.f_fractional_max_pool3d_output( + output, + indices, + kernel_size, + output_size, + random_samples, + ) + .unwrap() + } + + pub fn frexp(&self) -> (Tensor, Tensor) { + self.f_frexp().unwrap() + } + + pub fn frexp_tensor_out(&self, mantissa: &Tensor, exponent: &Tensor) -> (Tensor, Tensor) { + self.f_frexp_tensor_out(mantissa, exponent).unwrap() + } + + pub fn frobenius_norm(&self, dim: impl IntList, keepdim: bool) -> Tensor { + self.f_frobenius_norm(dim, keepdim).unwrap() + } + + pub fn frobenius_norm_out(&self, out: &Tensor, dim: impl IntList, keepdim: bool) -> Tensor { + self.f_frobenius_norm_out(out, dim, keepdim).unwrap() + } + + pub fn from_file( + filename: &str, + shared: bool, + size: impl Into>, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_from_file(filename, shared, size, options).unwrap() + } + + pub fn from_file_out( + out: &Tensor, + filename: &str, + shared: bool, + size: impl Into>, + ) -> Tensor { + Tensor::f_from_file_out(out, filename, shared, size).unwrap() + } + + pub fn full>( + size: impl IntList, + fill_value: S, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_full(size, fill_value, options).unwrap() + } + + pub fn full_like>(&self, fill_value: S) -> Tensor { + self.f_full_like(fill_value).unwrap() + } + + pub fn full_like_out>(&self, out: &Tensor, fill_value: S) -> Tensor { + self.f_full_like_out(out, fill_value).unwrap() + } + + pub fn full_out>(out: &Tensor, size: impl IntList, fill_value: S) -> Tensor { + Tensor::f_full_out(out, size, fill_value).unwrap() + } + + pub fn fused_moving_avg_obs_fake_quant( + &self, + observer_on: &Tensor, + fake_quant_on: &Tensor, + running_min: &Tensor, + running_max: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + averaging_const: f64, + quant_min: i64, + quant_max: i64, + ch_axis: i64, + per_row_fake_quant: bool, + symmetric_quant: bool, + ) -> Tensor { + self.f_fused_moving_avg_obs_fake_quant( + observer_on, + fake_quant_on, + running_min, + running_max, + scale, + zero_point, + averaging_const, + quant_min, + quant_max, + ch_axis, + per_row_fake_quant, + symmetric_quant, + ) + .unwrap() + } + + pub fn gather(&self, dim: i64, index: &Tensor, sparse_grad: bool) -> Tensor { + self.f_gather(dim, index, sparse_grad).unwrap() + } + + pub fn gather_backward( + &self, + grad: &Tensor, + dim: i64, + index: &Tensor, + sparse_grad: bool, + ) -> Tensor { + self.f_gather_backward(grad, dim, index, sparse_grad).unwrap() + } + + pub fn gather_out(&self, out: &Tensor, dim: i64, index: &Tensor, sparse_grad: bool) -> Tensor { + self.f_gather_out(out, dim, index, sparse_grad).unwrap() + } + + pub fn gcd(&self, other: &Tensor) -> Tensor { + self.f_gcd(other).unwrap() + } + + pub fn gcd_(&mut self, other: &Tensor) -> Tensor { + self.f_gcd_(other).unwrap() + } + + pub fn gcd_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_gcd_out(out, other).unwrap() + } + + pub fn ge>(&self, other: S) -> Tensor { + self.f_ge(other).unwrap() + } + + pub fn ge_>(&mut self, other: S) -> Tensor { + self.f_ge_(other).unwrap() + } + + pub fn ge_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_ge_scalar_out(out, other).unwrap() + } + + pub fn ge_tensor(&self, other: &Tensor) -> Tensor { + self.f_ge_tensor(other).unwrap() + } + + pub fn ge_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_ge_tensor_(other).unwrap() + } + + pub fn ge_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_ge_tensor_out(out, other).unwrap() + } + + pub fn gelu(&self, approximate: &str) -> Tensor { + self.f_gelu(approximate).unwrap() + } + + pub fn gelu_(&mut self, approximate: &str) -> Tensor { + self.f_gelu_(approximate).unwrap() + } + + pub fn gelu_backward(&self, grad_output: &Tensor, approximate: &str) -> Tensor { + self.f_gelu_backward(grad_output, approximate).unwrap() + } + + pub fn gelu_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + approximate: &str, + ) -> Tensor { + self.f_gelu_backward_grad_input(grad_input, grad_output, approximate).unwrap() + } + + pub fn gelu_out(&self, out: &Tensor, approximate: &str) -> Tensor { + self.f_gelu_out(out, approximate).unwrap() + } + + pub fn geometric(&self, p: f64) -> Tensor { + self.f_geometric(p).unwrap() + } + + pub fn geometric_(&mut self, p: f64) -> Tensor { + self.f_geometric_(p).unwrap() + } + + pub fn geometric_out(&self, out: &Tensor, p: f64) -> Tensor { + self.f_geometric_out(out, p).unwrap() + } + + pub fn geqrf(&self) -> (Tensor, Tensor) { + self.f_geqrf().unwrap() + } + + pub fn geqrf_a(&self, a: &Tensor, tau: &Tensor) -> (Tensor, Tensor) { + self.f_geqrf_a(a, tau).unwrap() + } + + pub fn ger(&self, vec2: &Tensor) -> Tensor { + self.f_ger(vec2).unwrap() + } + + pub fn ger_out(&self, out: &Tensor, vec2: &Tensor) -> Tensor { + self.f_ger_out(out, vec2).unwrap() + } + + pub fn glu(&self, dim: i64) -> Tensor { + self.f_glu(dim).unwrap() + } + + pub fn glu_backward(&self, grad_output: &Tensor, dim: i64) -> Tensor { + self.f_glu_backward(grad_output, dim).unwrap() + } + + pub fn glu_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + dim: i64, + ) -> Tensor { + self.f_glu_backward_grad_input(grad_input, grad_output, dim).unwrap() + } + + pub fn glu_backward_jvp( + grad_x: &Tensor, + grad_glu: &Tensor, + x: &Tensor, + dgrad_glu: &Tensor, + dx: &Tensor, + dim: i64, + ) -> Tensor { + Tensor::f_glu_backward_jvp(grad_x, grad_glu, x, dgrad_glu, dx, dim).unwrap() + } + + pub fn glu_backward_jvp_out( + out: &Tensor, + grad_x: &Tensor, + grad_glu: &Tensor, + x: &Tensor, + dgrad_glu: &Tensor, + dx: &Tensor, + dim: i64, + ) -> Tensor { + Tensor::f_glu_backward_jvp_out(out, grad_x, grad_glu, x, dgrad_glu, dx, dim).unwrap() + } + + pub fn glu_jvp(glu: &Tensor, x: &Tensor, dx: &Tensor, dim: i64) -> Tensor { + Tensor::f_glu_jvp(glu, x, dx, dim).unwrap() + } + + pub fn glu_jvp_out(out: &Tensor, glu: &Tensor, x: &Tensor, dx: &Tensor, dim: i64) -> Tensor { + Tensor::f_glu_jvp_out(out, glu, x, dx, dim).unwrap() + } + + pub fn glu_out(&self, out: &Tensor, dim: i64) -> Tensor { + self.f_glu_out(out, dim).unwrap() + } + + pub fn grad(&self) -> Tensor { + self.f_grad().unwrap() + } + + pub fn greater>(&self, other: S) -> Tensor { + self.f_greater(other).unwrap() + } + + pub fn greater_>(&mut self, other: S) -> Tensor { + self.f_greater_(other).unwrap() + } + + pub fn greater_equal>(&self, other: S) -> Tensor { + self.f_greater_equal(other).unwrap() + } + + pub fn greater_equal_>(&mut self, other: S) -> Tensor { + self.f_greater_equal_(other).unwrap() + } + + pub fn greater_equal_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_greater_equal_scalar_out(out, other).unwrap() + } + + pub fn greater_equal_tensor(&self, other: &Tensor) -> Tensor { + self.f_greater_equal_tensor(other).unwrap() + } + + pub fn greater_equal_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_greater_equal_tensor_(other).unwrap() + } + + pub fn greater_equal_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_greater_equal_tensor_out(out, other).unwrap() + } + + pub fn greater_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_greater_scalar_out(out, other).unwrap() + } + + pub fn greater_tensor(&self, other: &Tensor) -> Tensor { + self.f_greater_tensor(other).unwrap() + } + + pub fn greater_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_greater_tensor_(other).unwrap() + } + + pub fn greater_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_greater_tensor_out(out, other).unwrap() + } + + pub fn grid_sampler( + &self, + grid: &Tensor, + interpolation_mode: i64, + padding_mode: i64, + align_corners: bool, + ) -> Tensor { + self.f_grid_sampler(grid, interpolation_mode, padding_mode, align_corners).unwrap() + } + + pub fn grid_sampler_2d( + &self, + grid: &Tensor, + interpolation_mode: i64, + padding_mode: i64, + align_corners: bool, + ) -> Tensor { + self.f_grid_sampler_2d(grid, interpolation_mode, padding_mode, align_corners).unwrap() + } + + pub fn grid_sampler_2d_out( + &self, + out: &Tensor, + grid: &Tensor, + interpolation_mode: i64, + padding_mode: i64, + align_corners: bool, + ) -> Tensor { + self.f_grid_sampler_2d_out(out, grid, interpolation_mode, padding_mode, align_corners) + .unwrap() + } + + pub fn grid_sampler_3d( + &self, + grid: &Tensor, + interpolation_mode: i64, + padding_mode: i64, + align_corners: bool, + ) -> Tensor { + self.f_grid_sampler_3d(grid, interpolation_mode, padding_mode, align_corners).unwrap() + } + + pub fn grid_sampler_3d_out( + &self, + out: &Tensor, + grid: &Tensor, + interpolation_mode: i64, + padding_mode: i64, + align_corners: bool, + ) -> Tensor { + self.f_grid_sampler_3d_out(out, grid, interpolation_mode, padding_mode, align_corners) + .unwrap() + } + + pub fn group_norm>( + &self, + num_groups: i64, + weight: Option, + bias: Option, + eps: f64, + cudnn_enabled: bool, + ) -> Tensor { + self.f_group_norm(num_groups, weight, bias, eps, cudnn_enabled).unwrap() + } + + pub fn gru>( + &self, + hx: &Tensor, + params: &[T], + has_biases: bool, + num_layers: i64, + dropout: f64, + train: bool, + bidirectional: bool, + batch_first: bool, + ) -> (Tensor, Tensor) { + self.f_gru(hx, params, has_biases, num_layers, dropout, train, bidirectional, batch_first) + .unwrap() + } + + pub fn gru_cell>( + &self, + hx: &Tensor, + w_ih: &Tensor, + w_hh: &Tensor, + b_ih: Option, + b_hh: Option, + ) -> Tensor { + self.f_gru_cell(hx, w_ih, w_hh, b_ih, b_hh).unwrap() + } + + pub fn gru_data>( + data: &Tensor, + batch_sizes: &Tensor, + hx: &Tensor, + params: &[T], + has_biases: bool, + num_layers: i64, + dropout: f64, + train: bool, + bidirectional: bool, + ) -> (Tensor, Tensor) { + Tensor::f_gru_data( + data, + batch_sizes, + hx, + params, + has_biases, + num_layers, + dropout, + train, + bidirectional, + ) + .unwrap() + } + + pub fn gt>(&self, other: S) -> Tensor { + self.f_gt(other).unwrap() + } + + pub fn gt_>(&mut self, other: S) -> Tensor { + self.f_gt_(other).unwrap() + } + + pub fn gt_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_gt_scalar_out(out, other).unwrap() + } + + pub fn gt_tensor(&self, other: &Tensor) -> Tensor { + self.f_gt_tensor(other).unwrap() + } + + pub fn gt_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_gt_tensor_(other).unwrap() + } + + pub fn gt_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_gt_tensor_out(out, other).unwrap() + } + + pub fn hamming_window(window_length: i64, options: (Kind, Device)) -> Tensor { + Tensor::f_hamming_window(window_length, options).unwrap() + } + + pub fn hamming_window_out(out: &Tensor, window_length: i64) -> Tensor { + Tensor::f_hamming_window_out(out, window_length).unwrap() + } + + pub fn hamming_window_periodic( + window_length: i64, + periodic: bool, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_hamming_window_periodic(window_length, periodic, options).unwrap() + } + + pub fn hamming_window_periodic_alpha( + window_length: i64, + periodic: bool, + alpha: f64, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_hamming_window_periodic_alpha(window_length, periodic, alpha, options).unwrap() + } + + pub fn hamming_window_periodic_alpha_beta( + window_length: i64, + periodic: bool, + alpha: f64, + beta: f64, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_hamming_window_periodic_alpha_beta(window_length, periodic, alpha, beta, options) + .unwrap() + } + + pub fn hamming_window_periodic_alpha_beta_out( + out: &Tensor, + window_length: i64, + periodic: bool, + alpha: f64, + beta: f64, + ) -> Tensor { + Tensor::f_hamming_window_periodic_alpha_beta_out(out, window_length, periodic, alpha, beta) + .unwrap() + } + + pub fn hamming_window_periodic_alpha_out( + out: &Tensor, + window_length: i64, + periodic: bool, + alpha: f64, + ) -> Tensor { + Tensor::f_hamming_window_periodic_alpha_out(out, window_length, periodic, alpha).unwrap() + } + + pub fn hamming_window_periodic_out(out: &Tensor, window_length: i64, periodic: bool) -> Tensor { + Tensor::f_hamming_window_periodic_out(out, window_length, periodic).unwrap() + } + + pub fn hann_window(window_length: i64, options: (Kind, Device)) -> Tensor { + Tensor::f_hann_window(window_length, options).unwrap() + } + + pub fn hann_window_out(out: &Tensor, window_length: i64) -> Tensor { + Tensor::f_hann_window_out(out, window_length).unwrap() + } + + pub fn hann_window_periodic( + window_length: i64, + periodic: bool, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_hann_window_periodic(window_length, periodic, options).unwrap() + } + + pub fn hann_window_periodic_out(out: &Tensor, window_length: i64, periodic: bool) -> Tensor { + Tensor::f_hann_window_periodic_out(out, window_length, periodic).unwrap() + } + + pub fn hardshrink(&self) -> Tensor { + self.f_hardshrink().unwrap() + } + + pub fn hardshrink_backward>(&self, grad_out: &Tensor, lambd: S) -> Tensor { + self.f_hardshrink_backward(grad_out, lambd).unwrap() + } + + pub fn hardshrink_backward_grad_input>( + &self, + grad_input: &Tensor, + grad_out: &Tensor, + lambd: S, + ) -> Tensor { + self.f_hardshrink_backward_grad_input(grad_input, grad_out, lambd).unwrap() + } + + pub fn hardshrink_out(&self, out: &Tensor) -> Tensor { + self.f_hardshrink_out(out).unwrap() + } + + pub fn hardsigmoid(&self) -> Tensor { + self.f_hardsigmoid().unwrap() + } + + pub fn hardsigmoid_(&mut self) -> Tensor { + self.f_hardsigmoid_().unwrap() + } + + pub fn hardsigmoid_backward(&self, grad_output: &Tensor) -> Tensor { + self.f_hardsigmoid_backward(grad_output).unwrap() + } + + pub fn hardsigmoid_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + ) -> Tensor { + self.f_hardsigmoid_backward_grad_input(grad_input, grad_output).unwrap() + } + + pub fn hardsigmoid_out(&self, out: &Tensor) -> Tensor { + self.f_hardsigmoid_out(out).unwrap() + } + + pub fn hardswish(&self) -> Tensor { + self.f_hardswish().unwrap() + } + + pub fn hardswish_(&mut self) -> Tensor { + self.f_hardswish_().unwrap() + } + + pub fn hardswish_backward(&self, grad_output: &Tensor) -> Tensor { + self.f_hardswish_backward(grad_output).unwrap() + } + + pub fn hardswish_backward_out(&self, out: &Tensor, grad_output: &Tensor) -> Tensor { + self.f_hardswish_backward_out(out, grad_output).unwrap() + } + + pub fn hardswish_out(&self, out: &Tensor) -> Tensor { + self.f_hardswish_out(out).unwrap() + } + + pub fn hardtanh(&self) -> Tensor { + self.f_hardtanh().unwrap() + } + + pub fn hardtanh_(&mut self) -> Tensor { + self.f_hardtanh_().unwrap() + } + + pub fn hardtanh_backward>( + &self, + grad_output: &Tensor, + min_val: S, + max_val: S, + ) -> Tensor { + self.f_hardtanh_backward(grad_output, min_val, max_val).unwrap() + } + + pub fn hardtanh_backward_grad_input>( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + min_val: S, + max_val: S, + ) -> Tensor { + self.f_hardtanh_backward_grad_input(grad_input, grad_output, min_val, max_val).unwrap() + } + + pub fn hardtanh_out(&self, out: &Tensor) -> Tensor { + self.f_hardtanh_out(out).unwrap() + } + + pub fn hash_tensor(&self, dim: impl IntList, keepdim: bool, mode: i64) -> Tensor { + self.f_hash_tensor(dim, keepdim, mode).unwrap() + } + + pub fn hash_tensor_out( + &self, + out: &Tensor, + dim: impl IntList, + keepdim: bool, + mode: i64, + ) -> Tensor { + self.f_hash_tensor_out(out, dim, keepdim, mode).unwrap() + } + + pub fn heaviside(&self, values: &Tensor) -> Tensor { + self.f_heaviside(values).unwrap() + } + + pub fn heaviside_(&mut self, values: &Tensor) -> Tensor { + self.f_heaviside_(values).unwrap() + } + + pub fn heaviside_out(&self, out: &Tensor, values: &Tensor) -> Tensor { + self.f_heaviside_out(out, values).unwrap() + } + + pub fn hinge_embedding_loss( + &self, + target: &Tensor, + margin: f64, + reduction: crate::Reduction, + ) -> Tensor { + self.f_hinge_embedding_loss(target, margin, reduction).unwrap() + } + + pub fn histc(&self, bins: i64) -> Tensor { + self.f_histc(bins).unwrap() + } + + pub fn histc_out(&self, out: &Tensor, bins: i64) -> Tensor { + self.f_histc_out(out, bins).unwrap() + } + + pub fn histogram>( + &self, + bins: &Tensor, + weight: Option, + density: bool, + ) -> (Tensor, Tensor) { + self.f_histogram(bins, weight, density).unwrap() + } + + pub fn histogram_bin_ct>( + &self, + bins: i64, + range: impl DoubleList, + weight: Option, + density: bool, + ) -> (Tensor, Tensor) { + self.f_histogram_bin_ct(bins, range, weight, density).unwrap() + } + + pub fn histogram_bin_ct_out>( + &self, + hist: &Tensor, + bin_edges: &Tensor, + bins: i64, + range: impl DoubleList, + weight: Option, + density: bool, + ) -> (Tensor, Tensor) { + self.f_histogram_bin_ct_out(hist, bin_edges, bins, range, weight, density).unwrap() + } + + pub fn histogram_bins_tensor_out>( + &self, + hist: &Tensor, + bin_edges: &Tensor, + bins: &Tensor, + weight: Option, + density: bool, + ) -> (Tensor, Tensor) { + self.f_histogram_bins_tensor_out(hist, bin_edges, bins, weight, density).unwrap() + } + + pub fn hsplit(&self, sections: i64) -> Vec { + self.f_hsplit(sections).unwrap() + } + + pub fn hsplit_array(&self, indices: impl IntList) -> Vec { + self.f_hsplit_array(indices).unwrap() + } + + pub fn hspmm(mat1: &Tensor, mat2: &Tensor) -> Tensor { + Tensor::f_hspmm(mat1, mat2).unwrap() + } + + pub fn hspmm_out(out: &Tensor, mat1: &Tensor, mat2: &Tensor) -> Tensor { + Tensor::f_hspmm_out(out, mat1, mat2).unwrap() + } + + pub fn hstack>(tensors: &[T]) -> Tensor { + Tensor::f_hstack(tensors).unwrap() + } + + pub fn hstack_out>(out: &Tensor, tensors: &[T]) -> Tensor { + Tensor::f_hstack_out(out, tensors).unwrap() + } + + pub fn huber_loss(&self, target: &Tensor, reduction: crate::Reduction, delta: f64) -> Tensor { + self.f_huber_loss(target, reduction, delta).unwrap() + } + + pub fn huber_loss_backward( + &self, + grad_output: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + delta: f64, + ) -> Tensor { + self.f_huber_loss_backward(grad_output, target, reduction, delta).unwrap() + } + + pub fn huber_loss_backward_out( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + delta: f64, + ) -> Tensor { + self.f_huber_loss_backward_out(grad_input, grad_output, target, reduction, delta).unwrap() + } + + pub fn huber_loss_out( + &self, + out: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + delta: f64, + ) -> Tensor { + self.f_huber_loss_out(out, target, reduction, delta).unwrap() + } + + pub fn hypot(&self, other: &Tensor) -> Tensor { + self.f_hypot(other).unwrap() + } + + pub fn hypot_(&mut self, other: &Tensor) -> Tensor { + self.f_hypot_(other).unwrap() + } + + pub fn hypot_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_hypot_out(out, other).unwrap() + } + + pub fn i0(&self) -> Tensor { + self.f_i0().unwrap() + } + + pub fn i0_(&mut self) -> Tensor { + self.f_i0_().unwrap() + } + + pub fn i0_out(&self, out: &Tensor) -> Tensor { + self.f_i0_out(out).unwrap() + } + + pub fn igamma(&self, other: &Tensor) -> Tensor { + self.f_igamma(other).unwrap() + } + + pub fn igamma_(&mut self, other: &Tensor) -> Tensor { + self.f_igamma_(other).unwrap() + } + + pub fn igamma_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_igamma_out(out, other).unwrap() + } + + pub fn igammac(&self, other: &Tensor) -> Tensor { + self.f_igammac(other).unwrap() + } + + pub fn igammac_(&mut self, other: &Tensor) -> Tensor { + self.f_igammac_(other).unwrap() + } + + pub fn igammac_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_igammac_out(out, other).unwrap() + } + + pub fn im2col( + &self, + kernel_size: impl IntList, + dilation: impl IntList, + padding: impl IntList, + stride: impl IntList, + ) -> Tensor { + self.f_im2col(kernel_size, dilation, padding, stride).unwrap() + } + + pub fn im2col_out( + &self, + out: &Tensor, + kernel_size: impl IntList, + dilation: impl IntList, + padding: impl IntList, + stride: impl IntList, + ) -> Tensor { + self.f_im2col_out(out, kernel_size, dilation, padding, stride).unwrap() + } + + pub fn imag(&self) -> Tensor { + self.f_imag().unwrap() + } + + pub fn index>(&self, indices: &[Option]) -> Tensor { + self.f_index(indices).unwrap() + } + + pub fn index_add(&self, dim: i64, index: &Tensor, source: &Tensor) -> Tensor { + self.f_index_add(dim, index, source).unwrap() + } + + pub fn index_add_(&mut self, dim: i64, index: &Tensor, source: &Tensor) -> Tensor { + self.f_index_add_(dim, index, source).unwrap() + } + + pub fn index_add_out(&self, out: &Tensor, dim: i64, index: &Tensor, source: &Tensor) -> Tensor { + self.f_index_add_out(out, dim, index, source).unwrap() + } + + pub fn index_copy(&self, dim: i64, index: &Tensor, source: &Tensor) -> Tensor { + self.f_index_copy(dim, index, source).unwrap() + } + + pub fn index_copy_(&mut self, dim: i64, index: &Tensor, source: &Tensor) -> Tensor { + self.f_index_copy_(dim, index, source).unwrap() + } + + pub fn index_copy_out( + &self, + out: &Tensor, + dim: i64, + index: &Tensor, + source: &Tensor, + ) -> Tensor { + self.f_index_copy_out(out, dim, index, source).unwrap() + } + + pub fn index_fill>(&self, dim: i64, index: &Tensor, value: S) -> Tensor { + self.f_index_fill(dim, index, value).unwrap() + } + + pub fn index_fill_>(&mut self, dim: i64, index: &Tensor, value: S) -> Tensor { + self.f_index_fill_(dim, index, value).unwrap() + } + + pub fn index_fill_int_scalar_out>( + &self, + out: &Tensor, + dim: i64, + index: &Tensor, + value: S, + ) -> Tensor { + self.f_index_fill_int_scalar_out(out, dim, index, value).unwrap() + } + + pub fn index_fill_int_tensor(&self, dim: i64, index: &Tensor, value: &Tensor) -> Tensor { + self.f_index_fill_int_tensor(dim, index, value).unwrap() + } + + pub fn index_fill_int_tensor_(&mut self, dim: i64, index: &Tensor, value: &Tensor) -> Tensor { + self.f_index_fill_int_tensor_(dim, index, value).unwrap() + } + + pub fn index_fill_int_tensor_out( + &self, + out: &Tensor, + dim: i64, + index: &Tensor, + value: &Tensor, + ) -> Tensor { + self.f_index_fill_int_tensor_out(out, dim, index, value).unwrap() + } + + pub fn index_put>( + &self, + indices: &[Option], + values: &Tensor, + accumulate: bool, + ) -> Tensor { + self.f_index_put(indices, values, accumulate).unwrap() + } + + pub fn index_put_>( + &mut self, + indices: &[Option], + values: &Tensor, + accumulate: bool, + ) -> Tensor { + self.f_index_put_(indices, values, accumulate).unwrap() + } + + pub fn index_put_out>( + &self, + out: &Tensor, + indices: &[Option], + values: &Tensor, + accumulate: bool, + ) -> Tensor { + self.f_index_put_out(out, indices, values, accumulate).unwrap() + } + + pub fn index_reduce( + &self, + dim: i64, + index: &Tensor, + source: &Tensor, + reduce: &str, + include_self: bool, + ) -> Tensor { + self.f_index_reduce(dim, index, source, reduce, include_self).unwrap() + } + + pub fn index_reduce_( + &mut self, + dim: i64, + index: &Tensor, + source: &Tensor, + reduce: &str, + include_self: bool, + ) -> Tensor { + self.f_index_reduce_(dim, index, source, reduce, include_self).unwrap() + } + + pub fn index_reduce_out( + &self, + out: &Tensor, + dim: i64, + index: &Tensor, + source: &Tensor, + reduce: &str, + include_self: bool, + ) -> Tensor { + self.f_index_reduce_out(out, dim, index, source, reduce, include_self).unwrap() + } + + pub fn index_select(&self, dim: i64, index: &Tensor) -> Tensor { + self.f_index_select(dim, index).unwrap() + } + + pub fn index_select_backward( + grad: &Tensor, + self_sizes: impl IntList, + dim: i64, + index: &Tensor, + ) -> Tensor { + Tensor::f_index_select_backward(grad, self_sizes, dim, index).unwrap() + } + + pub fn index_select_out(&self, out: &Tensor, dim: i64, index: &Tensor) -> Tensor { + self.f_index_select_out(out, dim, index).unwrap() + } + + pub fn index_tensor_out>( + &self, + out: &Tensor, + indices: &[Option], + ) -> Tensor { + self.f_index_tensor_out(out, indices).unwrap() + } + + pub fn indices(&self) -> Tensor { + self.f_indices().unwrap() + } + + pub fn indices_copy(&self) -> Tensor { + self.f_indices_copy().unwrap() + } + + pub fn indices_copy_out(&self, out: &Tensor) -> Tensor { + self.f_indices_copy_out(out).unwrap() + } + + pub fn infinitely_differentiable_gelu_backward(&self, grad: &Tensor) -> Tensor { + self.f_infinitely_differentiable_gelu_backward(grad).unwrap() + } + + pub fn inner(&self, other: &Tensor) -> Tensor { + self.f_inner(other).unwrap() + } + + pub fn inner_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_inner_out(out, other).unwrap() + } + + pub fn instance_norm>( + &self, + weight: Option, + bias: Option, + running_mean: Option, + running_var: Option, + use_input_stats: bool, + momentum: f64, + eps: f64, + cudnn_enabled: bool, + ) -> Tensor { + self.f_instance_norm( + weight, + bias, + running_mean, + running_var, + use_input_stats, + momentum, + eps, + cudnn_enabled, + ) + .unwrap() + } + + pub fn int_repr(&self) -> Tensor { + self.f_int_repr().unwrap() + } + + pub fn int_repr_out(&self, out: &Tensor) -> Tensor { + self.f_int_repr_out(out).unwrap() + } + + pub fn inverse(&self) -> Tensor { + self.f_inverse().unwrap() + } + + pub fn inverse_out(&self, out: &Tensor) -> Tensor { + self.f_inverse_out(out).unwrap() + } + + pub fn is_coalesced(&self) -> bool { + self.f_is_coalesced().unwrap() + } + + pub fn is_complex(&self) -> bool { + self.f_is_complex().unwrap() + } + + pub fn is_conj(&self) -> bool { + self.f_is_conj().unwrap() + } + + pub fn is_distributed(&self) -> bool { + self.f_is_distributed().unwrap() + } + + pub fn is_floating_point(&self) -> bool { + self.f_is_floating_point().unwrap() + } + + pub fn is_inference(&self) -> bool { + self.f_is_inference().unwrap() + } + + pub fn is_leaf(&self) -> bool { + self.f_is_leaf().unwrap() + } + + pub fn is_neg(&self) -> bool { + self.f_is_neg().unwrap() + } + + pub fn is_nonzero(&self) -> bool { + self.f_is_nonzero().unwrap() + } + + pub fn is_pinned(&self, device: Device) -> bool { + self.f_is_pinned(device).unwrap() + } + + pub fn is_same_size(&self, other: &Tensor) -> bool { + self.f_is_same_size(other).unwrap() + } + + pub fn is_set_to(&self, tensor: &Tensor) -> bool { + self.f_is_set_to(tensor).unwrap() + } + + pub fn is_signed(&self) -> bool { + self.f_is_signed().unwrap() + } + + pub fn is_vulkan_available() -> bool { + Tensor::f_is_vulkan_available().unwrap() + } + + pub fn isclose(&self, other: &Tensor, rtol: f64, atol: f64, equal_nan: bool) -> Tensor { + self.f_isclose(other, rtol, atol, equal_nan).unwrap() + } + + pub fn isfinite(&self) -> Tensor { + self.f_isfinite().unwrap() + } + + pub fn isin( + elements: &Tensor, + test_elements: &Tensor, + assume_unique: bool, + invert: bool, + ) -> Tensor { + Tensor::f_isin(elements, test_elements, assume_unique, invert).unwrap() + } + + pub fn isin_scalar_tensor>( + element: S, + test_elements: &Tensor, + assume_unique: bool, + invert: bool, + ) -> Tensor { + Tensor::f_isin_scalar_tensor(element, test_elements, assume_unique, invert).unwrap() + } + + pub fn isin_scalar_tensor_out>( + out: &Tensor, + element: S, + test_elements: &Tensor, + assume_unique: bool, + invert: bool, + ) -> Tensor { + Tensor::f_isin_scalar_tensor_out(out, element, test_elements, assume_unique, invert) + .unwrap() + } + + pub fn isin_tensor_scalar>( + elements: &Tensor, + test_element: S, + assume_unique: bool, + invert: bool, + ) -> Tensor { + Tensor::f_isin_tensor_scalar(elements, test_element, assume_unique, invert).unwrap() + } + + pub fn isin_tensor_scalar_out>( + out: &Tensor, + elements: &Tensor, + test_element: S, + assume_unique: bool, + invert: bool, + ) -> Tensor { + Tensor::f_isin_tensor_scalar_out(out, elements, test_element, assume_unique, invert) + .unwrap() + } + + pub fn isin_tensor_tensor_out( + out: &Tensor, + elements: &Tensor, + test_elements: &Tensor, + assume_unique: bool, + invert: bool, + ) -> Tensor { + Tensor::f_isin_tensor_tensor_out(out, elements, test_elements, assume_unique, invert) + .unwrap() + } + + pub fn isinf(&self) -> Tensor { + self.f_isinf().unwrap() + } + + pub fn isinf_out(&self, out: &Tensor) -> Tensor { + self.f_isinf_out(out).unwrap() + } + + pub fn isnan(&self) -> Tensor { + self.f_isnan().unwrap() + } + + pub fn isnan_out(&self, out: &Tensor) -> Tensor { + self.f_isnan_out(out).unwrap() + } + + pub fn isneginf(&self) -> Tensor { + self.f_isneginf().unwrap() + } + + pub fn isneginf_out(&self, out: &Tensor) -> Tensor { + self.f_isneginf_out(out).unwrap() + } + + pub fn isposinf(&self) -> Tensor { + self.f_isposinf().unwrap() + } + + pub fn isposinf_out(&self, out: &Tensor) -> Tensor { + self.f_isposinf_out(out).unwrap() + } + + pub fn isreal(&self) -> Tensor { + self.f_isreal().unwrap() + } + + pub fn istft>( + &self, + n_fft: i64, + hop_length: impl Into>, + win_length: impl Into>, + window: Option, + center: bool, + normalized: bool, + onesided: bool, + length: impl Into>, + return_complex: bool, + ) -> Tensor { + self.f_istft( + n_fft, + hop_length, + win_length, + window, + center, + normalized, + onesided, + length, + return_complex, + ) + .unwrap() + } + + pub fn kaiser_window(window_length: i64, options: (Kind, Device)) -> Tensor { + Tensor::f_kaiser_window(window_length, options).unwrap() + } + + pub fn kaiser_window_beta( + window_length: i64, + periodic: bool, + beta: f64, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_kaiser_window_beta(window_length, periodic, beta, options).unwrap() + } + + pub fn kaiser_window_beta_out( + out: &Tensor, + window_length: i64, + periodic: bool, + beta: f64, + ) -> Tensor { + Tensor::f_kaiser_window_beta_out(out, window_length, periodic, beta).unwrap() + } + + pub fn kaiser_window_out(out: &Tensor, window_length: i64) -> Tensor { + Tensor::f_kaiser_window_out(out, window_length).unwrap() + } + + pub fn kaiser_window_periodic( + window_length: i64, + periodic: bool, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_kaiser_window_periodic(window_length, periodic, options).unwrap() + } + + pub fn kaiser_window_periodic_out(out: &Tensor, window_length: i64, periodic: bool) -> Tensor { + Tensor::f_kaiser_window_periodic_out(out, window_length, periodic).unwrap() + } + + pub fn kl_div(&self, target: &Tensor, reduction: crate::Reduction, log_target: bool) -> Tensor { + self.f_kl_div(target, reduction, log_target).unwrap() + } + + pub fn kron(&self, other: &Tensor) -> Tensor { + self.f_kron(other).unwrap() + } + + pub fn kron_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_kron_out(out, other).unwrap() + } + + pub fn kthvalue(&self, k: i64, dim: i64, keepdim: bool) -> (Tensor, Tensor) { + self.f_kthvalue(k, dim, keepdim).unwrap() + } + + pub fn kthvalue_values( + &self, + values: &Tensor, + indices: &Tensor, + k: i64, + dim: i64, + keepdim: bool, + ) -> (Tensor, Tensor) { + self.f_kthvalue_values(values, indices, k, dim, keepdim).unwrap() + } + + pub fn l1_loss(&self, target: &Tensor, reduction: crate::Reduction) -> Tensor { + self.f_l1_loss(target, reduction).unwrap() + } + + pub fn layer_norm>( + &self, + normalized_shape: impl IntList, + weight: Option, + bias: Option, + eps: f64, + cudnn_enable: bool, + ) -> Tensor { + self.f_layer_norm(normalized_shape, weight, bias, eps, cudnn_enable).unwrap() + } + + pub fn lcm(&self, other: &Tensor) -> Tensor { + self.f_lcm(other).unwrap() + } + + pub fn lcm_(&mut self, other: &Tensor) -> Tensor { + self.f_lcm_(other).unwrap() + } + + pub fn lcm_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_lcm_out(out, other).unwrap() + } + + pub fn ldexp(&self, other: &Tensor) -> Tensor { + self.f_ldexp(other).unwrap() + } + + pub fn ldexp_(&mut self, other: &Tensor) -> Tensor { + self.f_ldexp_(other).unwrap() + } + + pub fn ldexp_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_ldexp_out(out, other).unwrap() + } + + pub fn le>(&self, other: S) -> Tensor { + self.f_le(other).unwrap() + } + + pub fn le_>(&mut self, other: S) -> Tensor { + self.f_le_(other).unwrap() + } + + pub fn le_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_le_scalar_out(out, other).unwrap() + } + + pub fn le_tensor(&self, other: &Tensor) -> Tensor { + self.f_le_tensor(other).unwrap() + } + + pub fn le_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_le_tensor_(other).unwrap() + } + + pub fn le_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_le_tensor_out(out, other).unwrap() + } + + pub fn leaky_relu(&self) -> Tensor { + self.f_leaky_relu().unwrap() + } + + pub fn leaky_relu_(&mut self) -> Tensor { + self.f_leaky_relu_().unwrap() + } + + pub fn leaky_relu_backward>( + &self, + grad_output: &Tensor, + negative_slope: S, + self_is_result: bool, + ) -> Tensor { + self.f_leaky_relu_backward(grad_output, negative_slope, self_is_result).unwrap() + } + + pub fn leaky_relu_backward_grad_input>( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + negative_slope: S, + self_is_result: bool, + ) -> Tensor { + self.f_leaky_relu_backward_grad_input( + grad_input, + grad_output, + negative_slope, + self_is_result, + ) + .unwrap() + } + + pub fn leaky_relu_out(&self, out: &Tensor) -> Tensor { + self.f_leaky_relu_out(out).unwrap() + } + + pub fn lerp>(&self, end: &Tensor, weight: S) -> Tensor { + self.f_lerp(end, weight).unwrap() + } + + pub fn lerp_>(&mut self, end: &Tensor, weight: S) -> Tensor { + self.f_lerp_(end, weight).unwrap() + } + + pub fn lerp_scalar_out>( + &self, + out: &Tensor, + end: &Tensor, + weight: S, + ) -> Tensor { + self.f_lerp_scalar_out(out, end, weight).unwrap() + } + + pub fn lerp_tensor(&self, end: &Tensor, weight: &Tensor) -> Tensor { + self.f_lerp_tensor(end, weight).unwrap() + } + + pub fn lerp_tensor_(&mut self, end: &Tensor, weight: &Tensor) -> Tensor { + self.f_lerp_tensor_(end, weight).unwrap() + } + + pub fn lerp_tensor_out(&self, out: &Tensor, end: &Tensor, weight: &Tensor) -> Tensor { + self.f_lerp_tensor_out(out, end, weight).unwrap() + } + + pub fn less>(&self, other: S) -> Tensor { + self.f_less(other).unwrap() + } + + pub fn less_>(&mut self, other: S) -> Tensor { + self.f_less_(other).unwrap() + } + + pub fn less_equal>(&self, other: S) -> Tensor { + self.f_less_equal(other).unwrap() + } + + pub fn less_equal_>(&mut self, other: S) -> Tensor { + self.f_less_equal_(other).unwrap() + } + + pub fn less_equal_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_less_equal_scalar_out(out, other).unwrap() + } + + pub fn less_equal_tensor(&self, other: &Tensor) -> Tensor { + self.f_less_equal_tensor(other).unwrap() + } + + pub fn less_equal_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_less_equal_tensor_(other).unwrap() + } + + pub fn less_equal_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_less_equal_tensor_out(out, other).unwrap() + } + + pub fn less_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_less_scalar_out(out, other).unwrap() + } + + pub fn less_tensor(&self, other: &Tensor) -> Tensor { + self.f_less_tensor(other).unwrap() + } + + pub fn less_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_less_tensor_(other).unwrap() + } + + pub fn less_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_less_tensor_out(out, other).unwrap() + } + + pub fn lgamma(&self) -> Tensor { + self.f_lgamma().unwrap() + } + + pub fn lgamma_(&mut self) -> Tensor { + self.f_lgamma_().unwrap() + } + + pub fn lgamma_out(&self, out: &Tensor) -> Tensor { + self.f_lgamma_out(out).unwrap() + } + + pub fn lift(&self) -> Tensor { + self.f_lift().unwrap() + } + + pub fn lift_fresh(&self) -> Tensor { + self.f_lift_fresh().unwrap() + } + + pub fn lift_fresh_copy(&self) -> Tensor { + self.f_lift_fresh_copy().unwrap() + } + + pub fn lift_fresh_copy_out(&self, out: &Tensor) -> Tensor { + self.f_lift_fresh_copy_out(out).unwrap() + } + + pub fn lift_out(&self, out: &Tensor) -> Tensor { + self.f_lift_out(out).unwrap() + } + + pub fn linalg_cholesky(&self, upper: bool) -> Tensor { + self.f_linalg_cholesky(upper).unwrap() + } + + pub fn linalg_cholesky_ex(&self, upper: bool, check_errors: bool) -> (Tensor, Tensor) { + self.f_linalg_cholesky_ex(upper, check_errors).unwrap() + } + + pub fn linalg_cholesky_ex_l( + &self, + l: &Tensor, + info: &Tensor, + upper: bool, + check_errors: bool, + ) -> (Tensor, Tensor) { + self.f_linalg_cholesky_ex_l(l, info, upper, check_errors).unwrap() + } + + pub fn linalg_cholesky_out(&self, out: &Tensor, upper: bool) -> Tensor { + self.f_linalg_cholesky_out(out, upper).unwrap() + } + + pub fn linalg_cond>(&self, p: S) -> Tensor { + self.f_linalg_cond(p).unwrap() + } + + pub fn linalg_cond_out>(&self, out: &Tensor, p: S) -> Tensor { + self.f_linalg_cond_out(out, p).unwrap() + } + + pub fn linalg_cond_p_str(&self, p: &str) -> Tensor { + self.f_linalg_cond_p_str(p).unwrap() + } + + pub fn linalg_cond_p_str_out(&self, out: &Tensor, p: &str) -> Tensor { + self.f_linalg_cond_p_str_out(out, p).unwrap() + } + + pub fn linalg_cross(&self, other: &Tensor, dim: i64) -> Tensor { + self.f_linalg_cross(other, dim).unwrap() + } + + pub fn linalg_cross_out(&self, out: &Tensor, other: &Tensor, dim: i64) -> Tensor { + self.f_linalg_cross_out(out, other, dim).unwrap() + } + + pub fn linalg_det(a: &Tensor) -> Tensor { + Tensor::f_linalg_det(a).unwrap() + } + + pub fn linalg_det_out(out: &Tensor, a: &Tensor) -> Tensor { + Tensor::f_linalg_det_out(out, a).unwrap() + } + + pub fn linalg_diagonal(a: &Tensor, offset: i64, dim1: i64, dim2: i64) -> Tensor { + Tensor::f_linalg_diagonal(a, offset, dim1, dim2).unwrap() + } + + pub fn linalg_eig(&self) -> (Tensor, Tensor) { + self.f_linalg_eig().unwrap() + } + + pub fn linalg_eig_out(&self, eigenvalues: &Tensor, eigenvectors: &Tensor) -> (Tensor, Tensor) { + self.f_linalg_eig_out(eigenvalues, eigenvectors).unwrap() + } + + pub fn linalg_eigh(&self, uplo: &str) -> (Tensor, Tensor) { + self.f_linalg_eigh(uplo).unwrap() + } + + pub fn linalg_eigh_eigvals( + &self, + eigvals: &Tensor, + eigvecs: &Tensor, + uplo: &str, + ) -> (Tensor, Tensor) { + self.f_linalg_eigh_eigvals(eigvals, eigvecs, uplo).unwrap() + } + + pub fn linalg_eigvals(&self) -> Tensor { + self.f_linalg_eigvals().unwrap() + } + + pub fn linalg_eigvals_out(&self, out: &Tensor) -> Tensor { + self.f_linalg_eigvals_out(out).unwrap() + } + + pub fn linalg_eigvalsh(&self, uplo: &str) -> Tensor { + self.f_linalg_eigvalsh(uplo).unwrap() + } + + pub fn linalg_eigvalsh_out(&self, out: &Tensor, uplo: &str) -> Tensor { + self.f_linalg_eigvalsh_out(out, uplo).unwrap() + } + + pub fn linalg_householder_product(&self, tau: &Tensor) -> Tensor { + self.f_linalg_householder_product(tau).unwrap() + } + + pub fn linalg_householder_product_out(&self, out: &Tensor, tau: &Tensor) -> Tensor { + self.f_linalg_householder_product_out(out, tau).unwrap() + } + + pub fn linalg_inv(a: &Tensor) -> Tensor { + Tensor::f_linalg_inv(a).unwrap() + } + + pub fn linalg_inv_ex(a: &Tensor, check_errors: bool) -> (Tensor, Tensor) { + Tensor::f_linalg_inv_ex(a, check_errors).unwrap() + } + + pub fn linalg_inv_ex_inverse( + inverse: &Tensor, + info: &Tensor, + a: &Tensor, + check_errors: bool, + ) -> (Tensor, Tensor) { + Tensor::f_linalg_inv_ex_inverse(inverse, info, a, check_errors).unwrap() + } + + pub fn linalg_inv_out(out: &Tensor, a: &Tensor) -> Tensor { + Tensor::f_linalg_inv_out(out, a).unwrap() + } + + pub fn linalg_ldl_factor(&self, hermitian: bool) -> (Tensor, Tensor) { + self.f_linalg_ldl_factor(hermitian).unwrap() + } + + pub fn linalg_ldl_factor_ex( + &self, + hermitian: bool, + check_errors: bool, + ) -> (Tensor, Tensor, Tensor) { + self.f_linalg_ldl_factor_ex(hermitian, check_errors).unwrap() + } + + pub fn linalg_ldl_factor_ex_out( + &self, + ld: &Tensor, + pivots: &Tensor, + info: &Tensor, + hermitian: bool, + check_errors: bool, + ) -> (Tensor, Tensor, Tensor) { + self.f_linalg_ldl_factor_ex_out(ld, pivots, info, hermitian, check_errors).unwrap() + } + + pub fn linalg_ldl_factor_out( + &self, + ld: &Tensor, + pivots: &Tensor, + hermitian: bool, + ) -> (Tensor, Tensor) { + self.f_linalg_ldl_factor_out(ld, pivots, hermitian).unwrap() + } + + pub fn linalg_ldl_solve(ld: &Tensor, pivots: &Tensor, b: &Tensor, hermitian: bool) -> Tensor { + Tensor::f_linalg_ldl_solve(ld, pivots, b, hermitian).unwrap() + } + + pub fn linalg_ldl_solve_out( + out: &Tensor, + ld: &Tensor, + pivots: &Tensor, + b: &Tensor, + hermitian: bool, + ) -> Tensor { + Tensor::f_linalg_ldl_solve_out(out, ld, pivots, b, hermitian).unwrap() + } + + pub fn linalg_lstsq( + &self, + b: &Tensor, + rcond: impl Into>, + driver: &str, + ) -> (Tensor, Tensor, Tensor, Tensor) { + self.f_linalg_lstsq(b, rcond, driver).unwrap() + } + + pub fn linalg_lstsq_out( + &self, + solution: &Tensor, + residuals: &Tensor, + rank: &Tensor, + singular_values: &Tensor, + b: &Tensor, + rcond: impl Into>, + driver: &str, + ) -> (Tensor, Tensor, Tensor, Tensor) { + self.f_linalg_lstsq_out(solution, residuals, rank, singular_values, b, rcond, driver) + .unwrap() + } + + pub fn linalg_lu(a: &Tensor, pivot: bool) -> (Tensor, Tensor, Tensor) { + Tensor::f_linalg_lu(a, pivot).unwrap() + } + + pub fn linalg_lu_factor(a: &Tensor, pivot: bool) -> (Tensor, Tensor) { + Tensor::f_linalg_lu_factor(a, pivot).unwrap() + } + + pub fn linalg_lu_factor_ex( + a: &Tensor, + pivot: bool, + check_errors: bool, + ) -> (Tensor, Tensor, Tensor) { + Tensor::f_linalg_lu_factor_ex(a, pivot, check_errors).unwrap() + } + + pub fn linalg_lu_factor_ex_out( + lu: &Tensor, + pivots: &Tensor, + info: &Tensor, + a: &Tensor, + pivot: bool, + check_errors: bool, + ) -> (Tensor, Tensor, Tensor) { + Tensor::f_linalg_lu_factor_ex_out(lu, pivots, info, a, pivot, check_errors).unwrap() + } + + pub fn linalg_lu_factor_out( + lu: &Tensor, + pivots: &Tensor, + a: &Tensor, + pivot: bool, + ) -> (Tensor, Tensor) { + Tensor::f_linalg_lu_factor_out(lu, pivots, a, pivot).unwrap() + } + + pub fn linalg_lu_out( + p: &Tensor, + l: &Tensor, + u: &Tensor, + a: &Tensor, + pivot: bool, + ) -> (Tensor, Tensor, Tensor) { + Tensor::f_linalg_lu_out(p, l, u, a, pivot).unwrap() + } + + pub fn linalg_lu_solve( + lu: &Tensor, + pivots: &Tensor, + b: &Tensor, + left: bool, + adjoint: bool, + ) -> Tensor { + Tensor::f_linalg_lu_solve(lu, pivots, b, left, adjoint).unwrap() + } + + pub fn linalg_lu_solve_out( + out: &Tensor, + lu: &Tensor, + pivots: &Tensor, + b: &Tensor, + left: bool, + adjoint: bool, + ) -> Tensor { + Tensor::f_linalg_lu_solve_out(out, lu, pivots, b, left, adjoint).unwrap() + } + + pub fn linalg_matmul(&self, other: &Tensor) -> Tensor { + self.f_linalg_matmul(other).unwrap() + } + + pub fn linalg_matmul_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_linalg_matmul_out(out, other).unwrap() + } + + pub fn linalg_matrix_exp(&self) -> Tensor { + self.f_linalg_matrix_exp().unwrap() + } + + pub fn linalg_matrix_exp_out(&self, out: &Tensor) -> Tensor { + self.f_linalg_matrix_exp_out(out).unwrap() + } + + pub fn linalg_matrix_power(&self, n: i64) -> Tensor { + self.f_linalg_matrix_power(n).unwrap() + } + + pub fn linalg_matrix_power_out(&self, out: &Tensor, n: i64) -> Tensor { + self.f_linalg_matrix_power_out(out, n).unwrap() + } + + pub fn linalg_matrix_rank(&self, tol: f64, hermitian: bool) -> Tensor { + self.f_linalg_matrix_rank(tol, hermitian).unwrap() + } + + pub fn linalg_matrix_rank_atol_rtol_float( + &self, + atol: impl Into>, + rtol: impl Into>, + hermitian: bool, + ) -> Tensor { + self.f_linalg_matrix_rank_atol_rtol_float(atol, rtol, hermitian).unwrap() + } + + pub fn linalg_matrix_rank_atol_rtol_float_out( + &self, + out: &Tensor, + atol: impl Into>, + rtol: impl Into>, + hermitian: bool, + ) -> Tensor { + self.f_linalg_matrix_rank_atol_rtol_float_out(out, atol, rtol, hermitian).unwrap() + } + + pub fn linalg_matrix_rank_atol_rtol_tensor>( + &self, + atol: Option, + rtol: Option, + hermitian: bool, + ) -> Tensor { + self.f_linalg_matrix_rank_atol_rtol_tensor(atol, rtol, hermitian).unwrap() + } + + pub fn linalg_matrix_rank_atol_rtol_tensor_out>( + &self, + out: &Tensor, + atol: Option, + rtol: Option, + hermitian: bool, + ) -> Tensor { + self.f_linalg_matrix_rank_atol_rtol_tensor_out(out, atol, rtol, hermitian).unwrap() + } + + pub fn linalg_matrix_rank_out(&self, out: &Tensor, tol: f64, hermitian: bool) -> Tensor { + self.f_linalg_matrix_rank_out(out, tol, hermitian).unwrap() + } + + pub fn linalg_matrix_rank_out_tol_tensor( + &self, + out: &Tensor, + tol: &Tensor, + hermitian: bool, + ) -> Tensor { + self.f_linalg_matrix_rank_out_tol_tensor(out, tol, hermitian).unwrap() + } + + pub fn linalg_matrix_rank_tol_tensor(&self, tol: &Tensor, hermitian: bool) -> Tensor { + self.f_linalg_matrix_rank_tol_tensor(tol, hermitian).unwrap() + } + + pub fn linalg_multi_dot>(tensors: &[T]) -> Tensor { + Tensor::f_linalg_multi_dot(tensors).unwrap() + } + + pub fn linalg_multi_dot_out>(out: &Tensor, tensors: &[T]) -> Tensor { + Tensor::f_linalg_multi_dot_out(out, tensors).unwrap() + } + + pub fn linalg_norm>( + &self, + ord: S, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Tensor { + self.f_linalg_norm(ord, dim, keepdim, dtype).unwrap() + } + + pub fn linalg_norm_ord_str( + &self, + ord: &str, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Tensor { + self.f_linalg_norm_ord_str(ord, dim, keepdim, dtype).unwrap() + } + + pub fn linalg_norm_ord_str_out( + &self, + out: &Tensor, + ord: &str, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Tensor { + self.f_linalg_norm_ord_str_out(out, ord, dim, keepdim, dtype).unwrap() + } + + pub fn linalg_norm_out>( + &self, + out: &Tensor, + ord: S, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Tensor { + self.f_linalg_norm_out(out, ord, dim, keepdim, dtype).unwrap() + } + + pub fn linalg_pinv(&self, rcond: f64, hermitian: bool) -> Tensor { + self.f_linalg_pinv(rcond, hermitian).unwrap() + } + + pub fn linalg_pinv_atol_rtol_float( + &self, + atol: impl Into>, + rtol: impl Into>, + hermitian: bool, + ) -> Tensor { + self.f_linalg_pinv_atol_rtol_float(atol, rtol, hermitian).unwrap() + } + + pub fn linalg_pinv_atol_rtol_float_out( + &self, + out: &Tensor, + atol: impl Into>, + rtol: impl Into>, + hermitian: bool, + ) -> Tensor { + self.f_linalg_pinv_atol_rtol_float_out(out, atol, rtol, hermitian).unwrap() + } + + pub fn linalg_pinv_atol_rtol_tensor>( + &self, + atol: Option, + rtol: Option, + hermitian: bool, + ) -> Tensor { + self.f_linalg_pinv_atol_rtol_tensor(atol, rtol, hermitian).unwrap() + } + + pub fn linalg_pinv_atol_rtol_tensor_out>( + &self, + out: &Tensor, + atol: Option, + rtol: Option, + hermitian: bool, + ) -> Tensor { + self.f_linalg_pinv_atol_rtol_tensor_out(out, atol, rtol, hermitian).unwrap() + } + + pub fn linalg_pinv_out(&self, out: &Tensor, rcond: f64, hermitian: bool) -> Tensor { + self.f_linalg_pinv_out(out, rcond, hermitian).unwrap() + } + + pub fn linalg_pinv_out_rcond_tensor( + &self, + out: &Tensor, + rcond: &Tensor, + hermitian: bool, + ) -> Tensor { + self.f_linalg_pinv_out_rcond_tensor(out, rcond, hermitian).unwrap() + } + + pub fn linalg_pinv_rcond_tensor(&self, rcond: &Tensor, hermitian: bool) -> Tensor { + self.f_linalg_pinv_rcond_tensor(rcond, hermitian).unwrap() + } + + pub fn linalg_qr(a: &Tensor, mode: &str) -> (Tensor, Tensor) { + Tensor::f_linalg_qr(a, mode).unwrap() + } + + pub fn linalg_qr_out(q: &Tensor, r: &Tensor, a: &Tensor, mode: &str) -> (Tensor, Tensor) { + Tensor::f_linalg_qr_out(q, r, a, mode).unwrap() + } + + pub fn linalg_slogdet(a: &Tensor) -> (Tensor, Tensor) { + Tensor::f_linalg_slogdet(a).unwrap() + } + + pub fn linalg_slogdet_out(sign: &Tensor, logabsdet: &Tensor, a: &Tensor) -> (Tensor, Tensor) { + Tensor::f_linalg_slogdet_out(sign, logabsdet, a).unwrap() + } + + pub fn linalg_solve(a: &Tensor, b: &Tensor, left: bool) -> Tensor { + Tensor::f_linalg_solve(a, b, left).unwrap() + } + + pub fn linalg_solve_ex( + a: &Tensor, + b: &Tensor, + left: bool, + check_errors: bool, + ) -> (Tensor, Tensor) { + Tensor::f_linalg_solve_ex(a, b, left, check_errors).unwrap() + } + + pub fn linalg_solve_ex_out( + result: &Tensor, + info: &Tensor, + a: &Tensor, + b: &Tensor, + left: bool, + check_errors: bool, + ) -> (Tensor, Tensor) { + Tensor::f_linalg_solve_ex_out(result, info, a, b, left, check_errors).unwrap() + } + + pub fn linalg_solve_out(out: &Tensor, a: &Tensor, b: &Tensor, left: bool) -> Tensor { + Tensor::f_linalg_solve_out(out, a, b, left).unwrap() + } + + pub fn linalg_solve_triangular( + &self, + b: &Tensor, + upper: bool, + left: bool, + unitriangular: bool, + ) -> Tensor { + self.f_linalg_solve_triangular(b, upper, left, unitriangular).unwrap() + } + + pub fn linalg_solve_triangular_out( + &self, + out: &Tensor, + b: &Tensor, + upper: bool, + left: bool, + unitriangular: bool, + ) -> Tensor { + self.f_linalg_solve_triangular_out(out, b, upper, left, unitriangular).unwrap() + } + + pub fn linalg_svd(a: &Tensor, full_matrices: bool, driver: &str) -> (Tensor, Tensor, Tensor) { + Tensor::f_linalg_svd(a, full_matrices, driver).unwrap() + } + + pub fn linalg_svd_u( + u: &Tensor, + s: &Tensor, + vh: &Tensor, + a: &Tensor, + full_matrices: bool, + driver: &str, + ) -> (Tensor, Tensor, Tensor) { + Tensor::f_linalg_svd_u(u, s, vh, a, full_matrices, driver).unwrap() + } + + pub fn linalg_svdvals(a: &Tensor, driver: &str) -> Tensor { + Tensor::f_linalg_svdvals(a, driver).unwrap() + } + + pub fn linalg_svdvals_out(out: &Tensor, a: &Tensor, driver: &str) -> Tensor { + Tensor::f_linalg_svdvals_out(out, a, driver).unwrap() + } + + pub fn linalg_tensorinv(&self, ind: i64) -> Tensor { + self.f_linalg_tensorinv(ind).unwrap() + } + + pub fn linalg_tensorinv_out(&self, out: &Tensor, ind: i64) -> Tensor { + self.f_linalg_tensorinv_out(out, ind).unwrap() + } + + pub fn linalg_tensorsolve(&self, other: &Tensor, dims: impl IntListOption) -> Tensor { + self.f_linalg_tensorsolve(other, dims).unwrap() + } + + pub fn linalg_tensorsolve_out( + &self, + out: &Tensor, + other: &Tensor, + dims: impl IntListOption, + ) -> Tensor { + self.f_linalg_tensorsolve_out(out, other, dims).unwrap() + } + + pub fn linalg_vander(x: &Tensor, n: impl Into>) -> Tensor { + Tensor::f_linalg_vander(x, n).unwrap() + } + + pub fn linalg_vecdot(x: &Tensor, y: &Tensor, dim: i64) -> Tensor { + Tensor::f_linalg_vecdot(x, y, dim).unwrap() + } + + pub fn linalg_vecdot_out(out: &Tensor, x: &Tensor, y: &Tensor, dim: i64) -> Tensor { + Tensor::f_linalg_vecdot_out(out, x, y, dim).unwrap() + } + + pub fn linear>(&self, weight: &Tensor, bias: Option) -> Tensor { + self.f_linear(weight, bias).unwrap() + } + + pub fn linear_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + ) -> Tensor { + self.f_linear_out(out, weight, bias).unwrap() + } + + pub fn linspace>( + start: S, + end: S, + steps: i64, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_linspace(start, end, steps, options).unwrap() + } + + pub fn linspace_out>(out: &Tensor, start: S, end: S, steps: i64) -> Tensor { + Tensor::f_linspace_out(out, start, end, steps).unwrap() + } + + pub fn linspace_scalar_tensor>( + start: S, + end: &Tensor, + steps: i64, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_linspace_scalar_tensor(start, end, steps, options).unwrap() + } + + pub fn linspace_scalar_tensor_out>( + out: &Tensor, + start: S, + end: &Tensor, + steps: i64, + ) -> Tensor { + Tensor::f_linspace_scalar_tensor_out(out, start, end, steps).unwrap() + } + + pub fn linspace_tensor_scalar>( + start: &Tensor, + end: S, + steps: i64, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_linspace_tensor_scalar(start, end, steps, options).unwrap() + } + + pub fn linspace_tensor_scalar_out>( + out: &Tensor, + start: &Tensor, + end: S, + steps: i64, + ) -> Tensor { + Tensor::f_linspace_tensor_scalar_out(out, start, end, steps).unwrap() + } + + pub fn linspace_tensor_tensor( + start: &Tensor, + end: &Tensor, + steps: i64, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_linspace_tensor_tensor(start, end, steps, options).unwrap() + } + + pub fn linspace_tensor_tensor_out( + out: &Tensor, + start: &Tensor, + end: &Tensor, + steps: i64, + ) -> Tensor { + Tensor::f_linspace_tensor_tensor_out(out, start, end, steps).unwrap() + } + + pub fn log(&self) -> Tensor { + self.f_log().unwrap() + } + + pub fn log10(&self) -> Tensor { + self.f_log10().unwrap() + } + + pub fn log10_(&mut self) -> Tensor { + self.f_log10_().unwrap() + } + + pub fn log10_out(&self, out: &Tensor) -> Tensor { + self.f_log10_out(out).unwrap() + } + + pub fn log1p(&self) -> Tensor { + self.f_log1p().unwrap() + } + + pub fn log1p_(&mut self) -> Tensor { + self.f_log1p_().unwrap() + } + + pub fn log1p_out(&self, out: &Tensor) -> Tensor { + self.f_log1p_out(out).unwrap() + } + + pub fn log2(&self) -> Tensor { + self.f_log2().unwrap() + } + + pub fn log2_(&mut self) -> Tensor { + self.f_log2_().unwrap() + } + + pub fn log2_out(&self, out: &Tensor) -> Tensor { + self.f_log2_out(out).unwrap() + } + + pub fn log_(&mut self) -> Tensor { + self.f_log_().unwrap() + } + + pub fn log_normal(&self, mean: f64, std: f64) -> Tensor { + self.f_log_normal(mean, std).unwrap() + } + + pub fn log_normal_(&mut self, mean: f64, std: f64) -> Tensor { + self.f_log_normal_(mean, std).unwrap() + } + + pub fn log_normal_out(&self, out: &Tensor, mean: f64, std: f64) -> Tensor { + self.f_log_normal_out(out, mean, std).unwrap() + } + + pub fn log_out(&self, out: &Tensor) -> Tensor { + self.f_log_out(out).unwrap() + } + + pub fn log_sigmoid(&self) -> Tensor { + self.f_log_sigmoid().unwrap() + } + + pub fn log_sigmoid_backward(&self, grad_output: &Tensor, buffer: &Tensor) -> Tensor { + self.f_log_sigmoid_backward(grad_output, buffer).unwrap() + } + + pub fn log_sigmoid_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + buffer: &Tensor, + ) -> Tensor { + self.f_log_sigmoid_backward_grad_input(grad_input, grad_output, buffer).unwrap() + } + + pub fn log_sigmoid_out(&self, out: &Tensor) -> Tensor { + self.f_log_sigmoid_out(out).unwrap() + } + + pub fn log_softmax(&self, dim: i64, dtype: impl Into>) -> Tensor { + self.f_log_softmax(dim, dtype).unwrap() + } + + pub fn log_softmax_int_out( + &self, + out: &Tensor, + dim: i64, + dtype: impl Into>, + ) -> Tensor { + self.f_log_softmax_int_out(out, dim, dtype).unwrap() + } + + pub fn logaddexp(&self, other: &Tensor) -> Tensor { + self.f_logaddexp(other).unwrap() + } + + pub fn logaddexp2(&self, other: &Tensor) -> Tensor { + self.f_logaddexp2(other).unwrap() + } + + pub fn logaddexp2_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_logaddexp2_out(out, other).unwrap() + } + + pub fn logaddexp_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_logaddexp_out(out, other).unwrap() + } + + pub fn logcumsumexp(&self, dim: i64) -> Tensor { + self.f_logcumsumexp(dim).unwrap() + } + + pub fn logcumsumexp_out(&self, out: &Tensor, dim: i64) -> Tensor { + self.f_logcumsumexp_out(out, dim).unwrap() + } + + pub fn logdet(&self) -> Tensor { + self.f_logdet().unwrap() + } + + pub fn logical_and(&self, other: &Tensor) -> Tensor { + self.f_logical_and(other).unwrap() + } + + pub fn logical_and_(&mut self, other: &Tensor) -> Tensor { + self.f_logical_and_(other).unwrap() + } + + pub fn logical_and_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_logical_and_out(out, other).unwrap() + } + + pub fn logical_not(&self) -> Tensor { + self.f_logical_not().unwrap() + } + + pub fn logical_not_(&mut self) -> Tensor { + self.f_logical_not_().unwrap() + } + + pub fn logical_not_out(&self, out: &Tensor) -> Tensor { + self.f_logical_not_out(out).unwrap() + } + + pub fn logical_or(&self, other: &Tensor) -> Tensor { + self.f_logical_or(other).unwrap() + } + + pub fn logical_or_(&mut self, other: &Tensor) -> Tensor { + self.f_logical_or_(other).unwrap() + } + + pub fn logical_or_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_logical_or_out(out, other).unwrap() + } + + pub fn logical_xor(&self, other: &Tensor) -> Tensor { + self.f_logical_xor(other).unwrap() + } + + pub fn logical_xor_(&mut self, other: &Tensor) -> Tensor { + self.f_logical_xor_(other).unwrap() + } + + pub fn logical_xor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_logical_xor_out(out, other).unwrap() + } + + pub fn logit(&self, eps: impl Into>) -> Tensor { + self.f_logit(eps).unwrap() + } + + pub fn logit_(&mut self, eps: impl Into>) -> Tensor { + self.f_logit_(eps).unwrap() + } + + pub fn logit_backward(&self, grad_output: &Tensor, eps: impl Into>) -> Tensor { + self.f_logit_backward(grad_output, eps).unwrap() + } + + pub fn logit_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + eps: impl Into>, + ) -> Tensor { + self.f_logit_backward_grad_input(grad_input, grad_output, eps).unwrap() + } + + pub fn logit_out(&self, out: &Tensor, eps: impl Into>) -> Tensor { + self.f_logit_out(out, eps).unwrap() + } + + pub fn logspace>( + start: S, + end: S, + steps: i64, + base: f64, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_logspace(start, end, steps, base, options).unwrap() + } + + pub fn logspace_out>( + out: &Tensor, + start: S, + end: S, + steps: i64, + base: f64, + ) -> Tensor { + Tensor::f_logspace_out(out, start, end, steps, base).unwrap() + } + + pub fn logspace_scalar_tensor>( + start: S, + end: &Tensor, + steps: i64, + base: f64, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_logspace_scalar_tensor(start, end, steps, base, options).unwrap() + } + + pub fn logspace_scalar_tensor_out>( + out: &Tensor, + start: S, + end: &Tensor, + steps: i64, + base: f64, + ) -> Tensor { + Tensor::f_logspace_scalar_tensor_out(out, start, end, steps, base).unwrap() + } + + pub fn logspace_tensor_scalar>( + start: &Tensor, + end: S, + steps: i64, + base: f64, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_logspace_tensor_scalar(start, end, steps, base, options).unwrap() + } + + pub fn logspace_tensor_scalar_out>( + out: &Tensor, + start: &Tensor, + end: S, + steps: i64, + base: f64, + ) -> Tensor { + Tensor::f_logspace_tensor_scalar_out(out, start, end, steps, base).unwrap() + } + + pub fn logspace_tensor_tensor( + start: &Tensor, + end: &Tensor, + steps: i64, + base: f64, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_logspace_tensor_tensor(start, end, steps, base, options).unwrap() + } + + pub fn logspace_tensor_tensor_out( + out: &Tensor, + start: &Tensor, + end: &Tensor, + steps: i64, + base: f64, + ) -> Tensor { + Tensor::f_logspace_tensor_tensor_out(out, start, end, steps, base).unwrap() + } + + pub fn logsumexp(&self, dim: impl IntList, keepdim: bool) -> Tensor { + self.f_logsumexp(dim, keepdim).unwrap() + } + + pub fn logsumexp_out(&self, out: &Tensor, dim: impl IntList, keepdim: bool) -> Tensor { + self.f_logsumexp_out(out, dim, keepdim).unwrap() + } + + pub fn lstm>( + &self, + hx: &[T], + params: &[T], + has_biases: bool, + num_layers: i64, + dropout: f64, + train: bool, + bidirectional: bool, + batch_first: bool, + ) -> (Tensor, Tensor, Tensor) { + self.f_lstm(hx, params, has_biases, num_layers, dropout, train, bidirectional, batch_first) + .unwrap() + } + + pub fn lstm_cell>( + &self, + hx: &[T], + w_ih: &Tensor, + w_hh: &Tensor, + b_ih: Option, + b_hh: Option, + ) -> (Tensor, Tensor) { + self.f_lstm_cell(hx, w_ih, w_hh, b_ih, b_hh).unwrap() + } + + pub fn lstm_data>( + data: &Tensor, + batch_sizes: &Tensor, + hx: &[T], + params: &[T], + has_biases: bool, + num_layers: i64, + dropout: f64, + train: bool, + bidirectional: bool, + ) -> (Tensor, Tensor, Tensor) { + Tensor::f_lstm_data( + data, + batch_sizes, + hx, + params, + has_biases, + num_layers, + dropout, + train, + bidirectional, + ) + .unwrap() + } + + pub fn lstm_mps_backward>( + &self, + out0: &Tensor, + out1: &[T], + out2: &[T], + grad_y: Option, + grad_hy: Option, + grad_cy: Option, + z_state: &Tensor, + cell_state_fwd: &Tensor, + layersoutputs: &Tensor, + hx: &[T], + params: &[T], + has_biases: bool, + num_layers: i64, + dropout: f64, + train: bool, + bidirectional: bool, + batch_first: bool, + ) { + self.f_lstm_mps_backward( + out0, + out1, + out2, + grad_y, + grad_hy, + grad_cy, + z_state, + cell_state_fwd, + layersoutputs, + hx, + params, + has_biases, + num_layers, + dropout, + train, + bidirectional, + batch_first, + ) + .unwrap() + } + + pub fn lt>(&self, other: S) -> Tensor { + self.f_lt(other).unwrap() + } + + pub fn lt_>(&mut self, other: S) -> Tensor { + self.f_lt_(other).unwrap() + } + + pub fn lt_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_lt_scalar_out(out, other).unwrap() + } + + pub fn lt_tensor(&self, other: &Tensor) -> Tensor { + self.f_lt_tensor(other).unwrap() + } + + pub fn lt_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_lt_tensor_(other).unwrap() + } + + pub fn lt_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_lt_tensor_out(out, other).unwrap() + } + + pub fn lu_solve(&self, lu_data: &Tensor, lu_pivots: &Tensor) -> Tensor { + self.f_lu_solve(lu_data, lu_pivots).unwrap() + } + + pub fn lu_solve_out(&self, out: &Tensor, lu_data: &Tensor, lu_pivots: &Tensor) -> Tensor { + self.f_lu_solve_out(out, lu_data, lu_pivots).unwrap() + } + + pub fn lu_unpack( + lu_data: &Tensor, + lu_pivots: &Tensor, + unpack_data: bool, + unpack_pivots: bool, + ) -> (Tensor, Tensor, Tensor) { + Tensor::f_lu_unpack(lu_data, lu_pivots, unpack_data, unpack_pivots).unwrap() + } + + pub fn lu_unpack_out( + p: &Tensor, + l: &Tensor, + u: &Tensor, + lu_data: &Tensor, + lu_pivots: &Tensor, + unpack_data: bool, + unpack_pivots: bool, + ) -> (Tensor, Tensor, Tensor) { + Tensor::f_lu_unpack_out(p, l, u, lu_data, lu_pivots, unpack_data, unpack_pivots).unwrap() + } + + pub fn margin_ranking_loss( + input1: &Tensor, + input2: &Tensor, + target: &Tensor, + margin: f64, + reduction: crate::Reduction, + ) -> Tensor { + Tensor::f_margin_ranking_loss(input1, input2, target, margin, reduction).unwrap() + } + + pub fn masked_fill>(&self, mask: &Tensor, value: S) -> Tensor { + self.f_masked_fill(mask, value).unwrap() + } + + pub fn masked_fill_>(&mut self, mask: &Tensor, value: S) -> Tensor { + self.f_masked_fill_(mask, value).unwrap() + } + + pub fn masked_fill_scalar_out>( + &self, + out: &Tensor, + mask: &Tensor, + value: S, + ) -> Tensor { + self.f_masked_fill_scalar_out(out, mask, value).unwrap() + } + + pub fn masked_fill_tensor(&self, mask: &Tensor, value: &Tensor) -> Tensor { + self.f_masked_fill_tensor(mask, value).unwrap() + } + + pub fn masked_fill_tensor_(&mut self, mask: &Tensor, value: &Tensor) -> Tensor { + self.f_masked_fill_tensor_(mask, value).unwrap() + } + + pub fn masked_fill_tensor_out(&self, out: &Tensor, mask: &Tensor, value: &Tensor) -> Tensor { + self.f_masked_fill_tensor_out(out, mask, value).unwrap() + } + + pub fn masked_scatter(&self, mask: &Tensor, source: &Tensor) -> Tensor { + self.f_masked_scatter(mask, source).unwrap() + } + + pub fn masked_scatter_(&mut self, mask: &Tensor, source: &Tensor) -> Tensor { + self.f_masked_scatter_(mask, source).unwrap() + } + + pub fn masked_scatter_backward( + grad_output: &Tensor, + mask: &Tensor, + sizes: impl IntList, + ) -> Tensor { + Tensor::f_masked_scatter_backward(grad_output, mask, sizes).unwrap() + } + + pub fn masked_scatter_out(&self, out: &Tensor, mask: &Tensor, source: &Tensor) -> Tensor { + self.f_masked_scatter_out(out, mask, source).unwrap() + } + + pub fn masked_select(&self, mask: &Tensor) -> Tensor { + self.f_masked_select(mask).unwrap() + } + + pub fn masked_select_backward(&self, grad: &Tensor, mask: &Tensor) -> Tensor { + self.f_masked_select_backward(grad, mask).unwrap() + } + + pub fn masked_select_out(&self, out: &Tensor, mask: &Tensor) -> Tensor { + self.f_masked_select_out(out, mask).unwrap() + } + + pub fn matmul(&self, other: &Tensor) -> Tensor { + self.f_matmul(other).unwrap() + } + + pub fn matmul_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_matmul_out(out, other).unwrap() + } + + pub fn matrix_exp(&self) -> Tensor { + self.f_matrix_exp().unwrap() + } + + pub fn matrix_exp_backward(&self, grad: &Tensor) -> Tensor { + self.f_matrix_exp_backward(grad).unwrap() + } + + pub fn matrix_h(&self) -> Tensor { + self.f_matrix_h().unwrap() + } + + pub fn matrix_power(&self, n: i64) -> Tensor { + self.f_matrix_power(n).unwrap() + } + + pub fn matrix_power_out(&self, out: &Tensor, n: i64) -> Tensor { + self.f_matrix_power_out(out, n).unwrap() + } + + pub fn max(&self) -> Tensor { + self.f_max().unwrap() + } + + pub fn max_dim(&self, dim: i64, keepdim: bool) -> (Tensor, Tensor) { + self.f_max_dim(dim, keepdim).unwrap() + } + + pub fn max_dim_max( + &self, + max: &Tensor, + max_values: &Tensor, + dim: i64, + keepdim: bool, + ) -> (Tensor, Tensor) { + self.f_max_dim_max(max, max_values, dim, keepdim).unwrap() + } + + pub fn max_other(&self, other: &Tensor) -> Tensor { + self.f_max_other(other).unwrap() + } + + pub fn max_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_max_out(out, other).unwrap() + } + + pub fn max_pool1d( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Tensor { + self.f_max_pool1d(kernel_size, stride, padding, dilation, ceil_mode).unwrap() + } + + pub fn max_pool1d_with_indices( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> (Tensor, Tensor) { + self.f_max_pool1d_with_indices(kernel_size, stride, padding, dilation, ceil_mode).unwrap() + } + + pub fn max_pool2d( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Tensor { + self.f_max_pool2d(kernel_size, stride, padding, dilation, ceil_mode).unwrap() + } + + pub fn max_pool2d_backward( + &self, + grad_output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Tensor { + self.f_max_pool2d_backward(grad_output, kernel_size, stride, padding, dilation, ceil_mode) + .unwrap() + } + + pub fn max_pool2d_backward_out( + &self, + out: &Tensor, + grad_output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Tensor { + self.f_max_pool2d_backward_out( + out, + grad_output, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ) + .unwrap() + } + + pub fn max_pool2d_with_indices( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> (Tensor, Tensor) { + self.f_max_pool2d_with_indices(kernel_size, stride, padding, dilation, ceil_mode).unwrap() + } + + pub fn max_pool2d_with_indices_backward( + &self, + grad_output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + indices: &Tensor, + ) -> Tensor { + self.f_max_pool2d_with_indices_backward( + grad_output, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + indices, + ) + .unwrap() + } + + pub fn max_pool2d_with_indices_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + indices: &Tensor, + ) -> Tensor { + self.f_max_pool2d_with_indices_backward_grad_input( + grad_input, + grad_output, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + indices, + ) + .unwrap() + } + + pub fn max_pool2d_with_indices_out( + &self, + out: &Tensor, + indices: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> (Tensor, Tensor) { + self.f_max_pool2d_with_indices_out( + out, + indices, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ) + .unwrap() + } + + pub fn max_pool3d( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Tensor { + self.f_max_pool3d(kernel_size, stride, padding, dilation, ceil_mode).unwrap() + } + + pub fn max_pool3d_with_indices( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> (Tensor, Tensor) { + self.f_max_pool3d_with_indices(kernel_size, stride, padding, dilation, ceil_mode).unwrap() + } + + pub fn max_pool3d_with_indices_backward( + &self, + grad_output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + indices: &Tensor, + ) -> Tensor { + self.f_max_pool3d_with_indices_backward( + grad_output, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + indices, + ) + .unwrap() + } + + pub fn max_pool3d_with_indices_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + indices: &Tensor, + ) -> Tensor { + self.f_max_pool3d_with_indices_backward_grad_input( + grad_input, + grad_output, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + indices, + ) + .unwrap() + } + + pub fn max_pool3d_with_indices_out( + &self, + out: &Tensor, + indices: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> (Tensor, Tensor) { + self.f_max_pool3d_with_indices_out( + out, + indices, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ) + .unwrap() + } + + pub fn max_unary_out(&self, out: &Tensor) -> Tensor { + self.f_max_unary_out(out).unwrap() + } + + pub fn max_unpool2d(&self, indices: &Tensor, output_size: impl IntList) -> Tensor { + self.f_max_unpool2d(indices, output_size).unwrap() + } + + pub fn max_unpool2d_out( + &self, + out: &Tensor, + indices: &Tensor, + output_size: impl IntList, + ) -> Tensor { + self.f_max_unpool2d_out(out, indices, output_size).unwrap() + } + + pub fn max_unpool3d( + &self, + indices: &Tensor, + output_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ) -> Tensor { + self.f_max_unpool3d(indices, output_size, stride, padding).unwrap() + } + + pub fn max_unpool3d_out( + &self, + out: &Tensor, + indices: &Tensor, + output_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + ) -> Tensor { + self.f_max_unpool3d_out(out, indices, output_size, stride, padding).unwrap() + } + + pub fn maximum(&self, other: &Tensor) -> Tensor { + self.f_maximum(other).unwrap() + } + + pub fn maximum_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_maximum_out(out, other).unwrap() + } + + pub fn mean(&self, dtype: impl Into>) -> Tensor { + self.f_mean(dtype).unwrap() + } + + pub fn mean_dim( + &self, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Tensor { + self.f_mean_dim(dim, keepdim, dtype).unwrap() + } + + pub fn mean_dtype_out(&self, out: &Tensor, dtype: impl Into>) -> Tensor { + self.f_mean_dtype_out(out, dtype).unwrap() + } + + pub fn mean_out( + &self, + out: &Tensor, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Tensor { + self.f_mean_out(out, dim, keepdim, dtype).unwrap() + } + + pub fn median(&self) -> Tensor { + self.f_median().unwrap() + } + + pub fn median_dim(&self, dim: i64, keepdim: bool) -> (Tensor, Tensor) { + self.f_median_dim(dim, keepdim).unwrap() + } + + pub fn median_dim_values( + &self, + values: &Tensor, + indices: &Tensor, + dim: i64, + keepdim: bool, + ) -> (Tensor, Tensor) { + self.f_median_dim_values(values, indices, dim, keepdim).unwrap() + } + + pub fn median_out(&self, out: &Tensor) -> Tensor { + self.f_median_out(out).unwrap() + } + + pub fn meshgrid>(tensors: &[T]) -> Vec { + Tensor::f_meshgrid(tensors).unwrap() + } + + pub fn meshgrid_indexing>(tensors: &[T], indexing: &str) -> Vec { + Tensor::f_meshgrid_indexing(tensors, indexing).unwrap() + } + + pub fn mh(&self) -> Tensor { + self.f_mh().unwrap() + } + + pub fn min(&self) -> Tensor { + self.f_min().unwrap() + } + + pub fn min_dim(&self, dim: i64, keepdim: bool) -> (Tensor, Tensor) { + self.f_min_dim(dim, keepdim).unwrap() + } + + pub fn min_dim_min( + &self, + min: &Tensor, + min_indices: &Tensor, + dim: i64, + keepdim: bool, + ) -> (Tensor, Tensor) { + self.f_min_dim_min(min, min_indices, dim, keepdim).unwrap() + } + + pub fn min_other(&self, other: &Tensor) -> Tensor { + self.f_min_other(other).unwrap() + } + + pub fn min_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_min_out(out, other).unwrap() + } + + pub fn min_unary_out(&self, out: &Tensor) -> Tensor { + self.f_min_unary_out(out).unwrap() + } + + pub fn minimum(&self, other: &Tensor) -> Tensor { + self.f_minimum(other).unwrap() + } + + pub fn minimum_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_minimum_out(out, other).unwrap() + } + + pub fn miopen_batch_norm>( + &self, + weight: &Tensor, + bias: Option, + running_mean: Option, + running_var: Option, + training: bool, + exponential_average_factor: f64, + epsilon: f64, + ) -> (Tensor, Tensor, Tensor) { + self.f_miopen_batch_norm( + weight, + bias, + running_mean, + running_var, + training, + exponential_average_factor, + epsilon, + ) + .unwrap() + } + + pub fn miopen_batch_norm_backward>( + &self, + grad_output: &Tensor, + weight: &Tensor, + running_mean: Option, + running_var: Option, + save_mean: Option, + save_var: Option, + epsilon: f64, + ) -> (Tensor, Tensor, Tensor) { + self.f_miopen_batch_norm_backward( + grad_output, + weight, + running_mean, + running_var, + save_mean, + save_var, + epsilon, + ) + .unwrap() + } + + pub fn miopen_batch_norm_backward_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + grad_output: &Tensor, + weight: &Tensor, + running_mean: Option, + running_var: Option, + save_mean: Option, + save_var: Option, + epsilon: f64, + ) -> (Tensor, Tensor, Tensor) { + self.f_miopen_batch_norm_backward_out( + out0, + out1, + out2, + grad_output, + weight, + running_mean, + running_var, + save_mean, + save_var, + epsilon, + ) + .unwrap() + } + + pub fn miopen_batch_norm_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + weight: &Tensor, + bias: Option, + running_mean: Option, + running_var: Option, + training: bool, + exponential_average_factor: f64, + epsilon: f64, + ) -> (Tensor, Tensor, Tensor) { + self.f_miopen_batch_norm_out( + out0, + out1, + out2, + weight, + bias, + running_mean, + running_var, + training, + exponential_average_factor, + epsilon, + ) + .unwrap() + } + + pub fn miopen_convolution>( + &self, + weight: &Tensor, + bias: Option, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + ) -> Tensor { + self.f_miopen_convolution( + weight, + bias, + padding, + stride, + dilation, + groups, + benchmark, + deterministic, + ) + .unwrap() + } + + pub fn miopen_convolution_add_relu, S: Into>( + &self, + weight: &Tensor, + z: &Tensor, + alpha: S, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Tensor { + self.f_miopen_convolution_add_relu( + weight, z, alpha, bias, stride, padding, dilation, groups, + ) + .unwrap() + } + + pub fn miopen_convolution_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + ) -> Tensor { + self.f_miopen_convolution_out( + out, + weight, + bias, + padding, + stride, + dilation, + groups, + benchmark, + deterministic, + ) + .unwrap() + } + + pub fn miopen_convolution_relu>( + &self, + weight: &Tensor, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Tensor { + self.f_miopen_convolution_relu(weight, bias, stride, padding, dilation, groups).unwrap() + } + + pub fn miopen_convolution_transpose>( + &self, + weight: &Tensor, + bias: Option, + padding: impl IntList, + output_padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + ) -> Tensor { + self.f_miopen_convolution_transpose( + weight, + bias, + padding, + output_padding, + stride, + dilation, + groups, + benchmark, + deterministic, + ) + .unwrap() + } + + pub fn miopen_convolution_transpose_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + padding: impl IntList, + output_padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + ) -> Tensor { + self.f_miopen_convolution_transpose_out( + out, + weight, + bias, + padding, + output_padding, + stride, + dilation, + groups, + benchmark, + deterministic, + ) + .unwrap() + } + + pub fn miopen_ctc_loss( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: impl IntList, + target_lengths: impl IntList, + blank: i64, + deterministic: bool, + zero_infinity: bool, + ) -> (Tensor, Tensor) { + Tensor::f_miopen_ctc_loss( + log_probs, + targets, + input_lengths, + target_lengths, + blank, + deterministic, + zero_infinity, + ) + .unwrap() + } + + pub fn miopen_ctc_loss_out( + out0: &Tensor, + out1: &Tensor, + log_probs: &Tensor, + targets: &Tensor, + input_lengths: impl IntList, + target_lengths: impl IntList, + blank: i64, + deterministic: bool, + zero_infinity: bool, + ) -> (Tensor, Tensor) { + Tensor::f_miopen_ctc_loss_out( + out0, + out1, + log_probs, + targets, + input_lengths, + target_lengths, + blank, + deterministic, + zero_infinity, + ) + .unwrap() + } + + pub fn miopen_ctc_loss_tensor( + log_probs: &Tensor, + targets: &Tensor, + input_lengths: &Tensor, + target_lengths: &Tensor, + blank: i64, + deterministic: bool, + zero_infinity: bool, + ) -> (Tensor, Tensor) { + Tensor::f_miopen_ctc_loss_tensor( + log_probs, + targets, + input_lengths, + target_lengths, + blank, + deterministic, + zero_infinity, + ) + .unwrap() + } + + pub fn miopen_depthwise_convolution>( + &self, + weight: &Tensor, + bias: Option, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + ) -> Tensor { + self.f_miopen_depthwise_convolution( + weight, + bias, + padding, + stride, + dilation, + groups, + benchmark, + deterministic, + ) + .unwrap() + } + + pub fn miopen_depthwise_convolution_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + benchmark: bool, + deterministic: bool, + ) -> Tensor { + self.f_miopen_depthwise_convolution_out( + out, + weight, + bias, + padding, + stride, + dilation, + groups, + benchmark, + deterministic, + ) + .unwrap() + } + + pub fn miopen_rnn>( + &self, + weight: &[T], + weight_stride0: i64, + hx: &Tensor, + cx: Option, + mode: i64, + hidden_size: i64, + num_layers: i64, + batch_first: bool, + dropout: f64, + train: bool, + bidirectional: bool, + batch_sizes: impl IntList, + dropout_state: Option, + ) -> (Tensor, Tensor, Tensor, Tensor, Tensor) { + self.f_miopen_rnn( + weight, + weight_stride0, + hx, + cx, + mode, + hidden_size, + num_layers, + batch_first, + dropout, + train, + bidirectional, + batch_sizes, + dropout_state, + ) + .unwrap() + } + + pub fn miopen_rnn_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + out3: &Tensor, + out4: &Tensor, + weight: &[T], + weight_stride0: i64, + hx: &Tensor, + cx: Option, + mode: i64, + hidden_size: i64, + num_layers: i64, + batch_first: bool, + dropout: f64, + train: bool, + bidirectional: bool, + batch_sizes: impl IntList, + dropout_state: Option, + ) -> (Tensor, Tensor, Tensor, Tensor, Tensor) { + self.f_miopen_rnn_out( + out0, + out1, + out2, + out3, + out4, + weight, + weight_stride0, + hx, + cx, + mode, + hidden_size, + num_layers, + batch_first, + dropout, + train, + bidirectional, + batch_sizes, + dropout_state, + ) + .unwrap() + } + + pub fn mish(&self) -> Tensor { + self.f_mish().unwrap() + } + + pub fn mish_(&mut self) -> Tensor { + self.f_mish_().unwrap() + } + + pub fn mish_backward(&self, grad_output: &Tensor) -> Tensor { + self.f_mish_backward(grad_output).unwrap() + } + + pub fn mish_out(&self, out: &Tensor) -> Tensor { + self.f_mish_out(out).unwrap() + } + + pub fn mkldnn_adaptive_avg_pool2d(&self, output_size: impl IntList) -> Tensor { + self.f_mkldnn_adaptive_avg_pool2d(output_size).unwrap() + } + + pub fn mkldnn_adaptive_avg_pool2d_backward(&self, grad_output: &Tensor) -> Tensor { + self.f_mkldnn_adaptive_avg_pool2d_backward(grad_output).unwrap() + } + + pub fn mkldnn_adaptive_avg_pool2d_backward_out( + &self, + out: &Tensor, + grad_output: &Tensor, + ) -> Tensor { + self.f_mkldnn_adaptive_avg_pool2d_backward_out(out, grad_output).unwrap() + } + + pub fn mkldnn_adaptive_avg_pool2d_out( + &self, + out: &Tensor, + output_size: impl IntList, + ) -> Tensor { + self.f_mkldnn_adaptive_avg_pool2d_out(out, output_size).unwrap() + } + + pub fn mkldnn_convolution>( + &self, + weight: &Tensor, + bias: Option, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Tensor { + self.f_mkldnn_convolution(weight, bias, padding, stride, dilation, groups).unwrap() + } + + pub fn mkldnn_convolution_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + ) -> Tensor { + self.f_mkldnn_convolution_out(out, weight, bias, padding, stride, dilation, groups).unwrap() + } + + pub fn mkldnn_linear>(&self, weight: &Tensor, bias: Option) -> Tensor { + self.f_mkldnn_linear(weight, bias).unwrap() + } + + pub fn mkldnn_linear_backward_input( + input_size: impl IntList, + grad_output: &Tensor, + weight: &Tensor, + ) -> Tensor { + Tensor::f_mkldnn_linear_backward_input(input_size, grad_output, weight).unwrap() + } + + pub fn mkldnn_linear_backward_input_out( + out: &Tensor, + input_size: impl IntList, + grad_output: &Tensor, + weight: &Tensor, + ) -> Tensor { + Tensor::f_mkldnn_linear_backward_input_out(out, input_size, grad_output, weight).unwrap() + } + + pub fn mkldnn_linear_backward_weights( + &self, + grad_output: &Tensor, + weight: &Tensor, + bias_defined: bool, + ) -> (Tensor, Tensor) { + self.f_mkldnn_linear_backward_weights(grad_output, weight, bias_defined).unwrap() + } + + pub fn mkldnn_linear_backward_weights_out( + &self, + out0: &Tensor, + out1: &Tensor, + grad_output: &Tensor, + weight: &Tensor, + bias_defined: bool, + ) -> (Tensor, Tensor) { + self.f_mkldnn_linear_backward_weights_out(out0, out1, grad_output, weight, bias_defined) + .unwrap() + } + + pub fn mkldnn_linear_out>( + &self, + out: &Tensor, + weight: &Tensor, + bias: Option, + ) -> Tensor { + self.f_mkldnn_linear_out(out, weight, bias).unwrap() + } + + pub fn mkldnn_max_pool2d( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Tensor { + self.f_mkldnn_max_pool2d(kernel_size, stride, padding, dilation, ceil_mode).unwrap() + } + + pub fn mkldnn_max_pool2d_backward( + &self, + grad_output: &Tensor, + output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Tensor { + self.f_mkldnn_max_pool2d_backward( + grad_output, + output, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ) + .unwrap() + } + + pub fn mkldnn_max_pool2d_backward_out( + &self, + out: &Tensor, + grad_output: &Tensor, + output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Tensor { + self.f_mkldnn_max_pool2d_backward_out( + out, + grad_output, + output, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ) + .unwrap() + } + + pub fn mkldnn_max_pool2d_out( + &self, + out: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Tensor { + self.f_mkldnn_max_pool2d_out(out, kernel_size, stride, padding, dilation, ceil_mode) + .unwrap() + } + + pub fn mkldnn_max_pool3d( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Tensor { + self.f_mkldnn_max_pool3d(kernel_size, stride, padding, dilation, ceil_mode).unwrap() + } + + pub fn mkldnn_max_pool3d_backward( + &self, + grad_output: &Tensor, + output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Tensor { + self.f_mkldnn_max_pool3d_backward( + grad_output, + output, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ) + .unwrap() + } + + pub fn mkldnn_max_pool3d_backward_out( + &self, + out: &Tensor, + grad_output: &Tensor, + output: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Tensor { + self.f_mkldnn_max_pool3d_backward_out( + out, + grad_output, + output, + kernel_size, + stride, + padding, + dilation, + ceil_mode, + ) + .unwrap() + } + + pub fn mkldnn_max_pool3d_out( + &self, + out: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Tensor { + self.f_mkldnn_max_pool3d_out(out, kernel_size, stride, padding, dilation, ceil_mode) + .unwrap() + } + + pub fn mkldnn_reorder_conv2d_weight( + &self, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + input_size: impl IntListOption, + ) -> Tensor { + self.f_mkldnn_reorder_conv2d_weight(padding, stride, dilation, groups, input_size).unwrap() + } + + pub fn mkldnn_reorder_conv2d_weight_out( + &self, + out: &Tensor, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + input_size: impl IntListOption, + ) -> Tensor { + self.f_mkldnn_reorder_conv2d_weight_out(out, padding, stride, dilation, groups, input_size) + .unwrap() + } + + pub fn mkldnn_reorder_conv3d_weight( + &self, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + input_size: impl IntListOption, + ) -> Tensor { + self.f_mkldnn_reorder_conv3d_weight(padding, stride, dilation, groups, input_size).unwrap() + } + + pub fn mkldnn_reorder_conv3d_weight_out( + &self, + out: &Tensor, + padding: impl IntList, + stride: impl IntList, + dilation: impl IntList, + groups: i64, + input_size: impl IntListOption, + ) -> Tensor { + self.f_mkldnn_reorder_conv3d_weight_out(out, padding, stride, dilation, groups, input_size) + .unwrap() + } + + pub fn mkldnn_rnn_layer( + &self, + weight0: &Tensor, + weight1: &Tensor, + weight2: &Tensor, + weight3: &Tensor, + hx_: &Tensor, + cx_: &Tensor, + reverse: bool, + batch_sizes: impl IntList, + mode: i64, + hidden_size: i64, + num_layers: i64, + has_biases: bool, + bidirectional: bool, + batch_first: bool, + train: bool, + ) -> (Tensor, Tensor, Tensor, Tensor) { + self.f_mkldnn_rnn_layer( + weight0, + weight1, + weight2, + weight3, + hx_, + cx_, + reverse, + batch_sizes, + mode, + hidden_size, + num_layers, + has_biases, + bidirectional, + batch_first, + train, + ) + .unwrap() + } + + pub fn mkldnn_rnn_layer_backward>( + &self, + weight1: &Tensor, + weight2: &Tensor, + weight3: &Tensor, + weight4: &Tensor, + hx_: &Tensor, + cx_tmp: &Tensor, + output: &Tensor, + hy_: &Tensor, + cy_: &Tensor, + grad_output: Option, + grad_hy: Option, + grad_cy: Option, + reverse: bool, + mode: i64, + hidden_size: i64, + num_layers: i64, + has_biases: bool, + train: bool, + bidirectional: bool, + batch_sizes: impl IntList, + batch_first: bool, + workspace: &Tensor, + ) -> (Tensor, Tensor, Tensor, Tensor, Tensor, Tensor, Tensor) { + self.f_mkldnn_rnn_layer_backward( + weight1, + weight2, + weight3, + weight4, + hx_, + cx_tmp, + output, + hy_, + cy_, + grad_output, + grad_hy, + grad_cy, + reverse, + mode, + hidden_size, + num_layers, + has_biases, + train, + bidirectional, + batch_sizes, + batch_first, + workspace, + ) + .unwrap() + } + + pub fn mkldnn_rnn_layer_backward_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + out3: &Tensor, + out4: &Tensor, + out5: &Tensor, + out6: &Tensor, + weight1: &Tensor, + weight2: &Tensor, + weight3: &Tensor, + weight4: &Tensor, + hx_: &Tensor, + cx_tmp: &Tensor, + output: &Tensor, + hy_: &Tensor, + cy_: &Tensor, + grad_output: Option, + grad_hy: Option, + grad_cy: Option, + reverse: bool, + mode: i64, + hidden_size: i64, + num_layers: i64, + has_biases: bool, + train: bool, + bidirectional: bool, + batch_sizes: impl IntList, + batch_first: bool, + workspace: &Tensor, + ) -> (Tensor, Tensor, Tensor, Tensor, Tensor, Tensor, Tensor) { + self.f_mkldnn_rnn_layer_backward_out( + out0, + out1, + out2, + out3, + out4, + out5, + out6, + weight1, + weight2, + weight3, + weight4, + hx_, + cx_tmp, + output, + hy_, + cy_, + grad_output, + grad_hy, + grad_cy, + reverse, + mode, + hidden_size, + num_layers, + has_biases, + train, + bidirectional, + batch_sizes, + batch_first, + workspace, + ) + .unwrap() + } + + pub fn mkldnn_rnn_layer_out( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + out3: &Tensor, + weight0: &Tensor, + weight1: &Tensor, + weight2: &Tensor, + weight3: &Tensor, + hx_: &Tensor, + cx_: &Tensor, + reverse: bool, + batch_sizes: impl IntList, + mode: i64, + hidden_size: i64, + num_layers: i64, + has_biases: bool, + bidirectional: bool, + batch_first: bool, + train: bool, + ) -> (Tensor, Tensor, Tensor, Tensor) { + self.f_mkldnn_rnn_layer_out( + out0, + out1, + out2, + out3, + weight0, + weight1, + weight2, + weight3, + hx_, + cx_, + reverse, + batch_sizes, + mode, + hidden_size, + num_layers, + has_biases, + bidirectional, + batch_first, + train, + ) + .unwrap() + } + + pub fn mm(&self, mat2: &Tensor) -> Tensor { + self.f_mm(mat2).unwrap() + } + + pub fn mm_dtype(&self, mat2: &Tensor, out_dtype: Kind) -> Tensor { + self.f_mm_dtype(mat2, out_dtype).unwrap() + } + + pub fn mm_dtype_out(&self, out: &Tensor, mat2: &Tensor, out_dtype: Kind) -> Tensor { + self.f_mm_dtype_out(out, mat2, out_dtype).unwrap() + } + + pub fn mm_out(&self, out: &Tensor, mat2: &Tensor) -> Tensor { + self.f_mm_out(out, mat2).unwrap() + } + + pub fn mode(&self, dim: i64, keepdim: bool) -> (Tensor, Tensor) { + self.f_mode(dim, keepdim).unwrap() + } + + pub fn mode_values( + &self, + values: &Tensor, + indices: &Tensor, + dim: i64, + keepdim: bool, + ) -> (Tensor, Tensor) { + self.f_mode_values(values, indices, dim, keepdim).unwrap() + } + + pub fn moveaxis(&self, source: impl IntList, destination: impl IntList) -> Tensor { + self.f_moveaxis(source, destination).unwrap() + } + + pub fn moveaxis_int(&self, source: i64, destination: i64) -> Tensor { + self.f_moveaxis_int(source, destination).unwrap() + } + + pub fn movedim(&self, source: impl IntList, destination: impl IntList) -> Tensor { + self.f_movedim(source, destination).unwrap() + } + + pub fn movedim_int(&self, source: i64, destination: i64) -> Tensor { + self.f_movedim_int(source, destination).unwrap() + } + + pub fn mse_loss(&self, target: &Tensor, reduction: crate::Reduction) -> Tensor { + self.f_mse_loss(target, reduction).unwrap() + } + + pub fn mse_loss_backward( + &self, + grad_output: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + ) -> Tensor { + self.f_mse_loss_backward(grad_output, target, reduction).unwrap() + } + + pub fn mse_loss_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + ) -> Tensor { + self.f_mse_loss_backward_grad_input(grad_input, grad_output, target, reduction).unwrap() + } + + pub fn mse_loss_out( + &self, + out: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + ) -> Tensor { + self.f_mse_loss_out(out, target, reduction).unwrap() + } + + pub fn msort(&self) -> Tensor { + self.f_msort().unwrap() + } + + pub fn msort_out(&self, out: &Tensor) -> Tensor { + self.f_msort_out(out).unwrap() + } + + pub fn mt(&self) -> Tensor { + self.f_mt().unwrap() + } + + pub fn g_mul(&self, other: &Tensor) -> Tensor { + self.f_mul(other).unwrap() + } + + pub fn g_mul_(&mut self, other: &Tensor) -> Tensor { + self.f_mul_(other).unwrap() + } + + pub fn mul_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_mul_out(out, other).unwrap() + } + + pub fn g_mul_scalar>(&self, other: S) -> Tensor { + self.f_mul_scalar(other).unwrap() + } + + pub fn g_mul_scalar_>(&mut self, other: S) -> Tensor { + self.f_mul_scalar_(other).unwrap() + } + + pub fn mul_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_mul_scalar_out(out, other).unwrap() + } + + pub fn multi_margin_loss_backward, S: Into>( + &self, + grad_output: &Tensor, + target: &Tensor, + p: S, + margin: S, + weight: Option, + reduction: crate::Reduction, + ) -> Tensor { + self.f_multi_margin_loss_backward(grad_output, target, p, margin, weight, reduction) + .unwrap() + } + + pub fn multi_margin_loss_backward_grad_input, S: Into>( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + target: &Tensor, + p: S, + margin: S, + weight: Option, + reduction: crate::Reduction, + ) -> Tensor { + self.f_multi_margin_loss_backward_grad_input( + grad_input, + grad_output, + target, + p, + margin, + weight, + reduction, + ) + .unwrap() + } + + pub fn multilabel_margin_loss(&self, target: &Tensor, reduction: crate::Reduction) -> Tensor { + self.f_multilabel_margin_loss(target, reduction).unwrap() + } + + pub fn multilabel_margin_loss_backward( + &self, + grad_output: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + is_target: &Tensor, + ) -> Tensor { + self.f_multilabel_margin_loss_backward(grad_output, target, reduction, is_target).unwrap() + } + + pub fn multilabel_margin_loss_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + is_target: &Tensor, + ) -> Tensor { + self.f_multilabel_margin_loss_backward_grad_input( + grad_input, + grad_output, + target, + reduction, + is_target, + ) + .unwrap() + } + + pub fn multilabel_margin_loss_out( + &self, + out: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + ) -> Tensor { + self.f_multilabel_margin_loss_out(out, target, reduction).unwrap() + } + + pub fn multinomial(&self, num_samples: i64, replacement: bool) -> Tensor { + self.f_multinomial(num_samples, replacement).unwrap() + } + + pub fn multinomial_out(&self, out: &Tensor, num_samples: i64, replacement: bool) -> Tensor { + self.f_multinomial_out(out, num_samples, replacement).unwrap() + } + + pub fn multiply(&self, other: &Tensor) -> Tensor { + self.f_multiply(other).unwrap() + } + + pub fn multiply_(&mut self, other: &Tensor) -> Tensor { + self.f_multiply_(other).unwrap() + } + + pub fn multiply_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_multiply_out(out, other).unwrap() + } + + pub fn multiply_scalar>(&self, other: S) -> Tensor { + self.f_multiply_scalar(other).unwrap() + } + + pub fn multiply_scalar_>(&mut self, other: S) -> Tensor { + self.f_multiply_scalar_(other).unwrap() + } + + pub fn mv(&self, vec: &Tensor) -> Tensor { + self.f_mv(vec).unwrap() + } + + pub fn mv_out(&self, out: &Tensor, vec: &Tensor) -> Tensor { + self.f_mv_out(out, vec).unwrap() + } + + pub fn mvlgamma(&self, p: i64) -> Tensor { + self.f_mvlgamma(p).unwrap() + } + + pub fn mvlgamma_(&mut self, p: i64) -> Tensor { + self.f_mvlgamma_(p).unwrap() + } + + pub fn mvlgamma_out(&self, out: &Tensor, p: i64) -> Tensor { + self.f_mvlgamma_out(out, p).unwrap() + } + + pub fn nan_to_num( + &self, + nan: impl Into>, + posinf: impl Into>, + neginf: impl Into>, + ) -> Tensor { + self.f_nan_to_num(nan, posinf, neginf).unwrap() + } + + pub fn nan_to_num_( + &mut self, + nan: impl Into>, + posinf: impl Into>, + neginf: impl Into>, + ) -> Tensor { + self.f_nan_to_num_(nan, posinf, neginf).unwrap() + } + + pub fn nan_to_num_out( + &self, + out: &Tensor, + nan: impl Into>, + posinf: impl Into>, + neginf: impl Into>, + ) -> Tensor { + self.f_nan_to_num_out(out, nan, posinf, neginf).unwrap() + } + + pub fn nanmean( + &self, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Tensor { + self.f_nanmean(dim, keepdim, dtype).unwrap() + } + + pub fn nanmean_out( + &self, + out: &Tensor, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Tensor { + self.f_nanmean_out(out, dim, keepdim, dtype).unwrap() + } + + pub fn nanmedian(&self) -> Tensor { + self.f_nanmedian().unwrap() + } + + pub fn nanmedian_dim(&self, dim: i64, keepdim: bool) -> (Tensor, Tensor) { + self.f_nanmedian_dim(dim, keepdim).unwrap() + } + + pub fn nanmedian_dim_values( + &self, + values: &Tensor, + indices: &Tensor, + dim: i64, + keepdim: bool, + ) -> (Tensor, Tensor) { + self.f_nanmedian_dim_values(values, indices, dim, keepdim).unwrap() + } + + pub fn nanmedian_out(&self, out: &Tensor) -> Tensor { + self.f_nanmedian_out(out).unwrap() + } + + pub fn nanquantile( + &self, + q: &Tensor, + dim: impl Into>, + keepdim: bool, + interpolation: &str, + ) -> Tensor { + self.f_nanquantile(q, dim, keepdim, interpolation).unwrap() + } + + pub fn nanquantile_out( + &self, + out: &Tensor, + q: &Tensor, + dim: impl Into>, + keepdim: bool, + interpolation: &str, + ) -> Tensor { + self.f_nanquantile_out(out, q, dim, keepdim, interpolation).unwrap() + } + + pub fn nanquantile_scalar( + &self, + q: f64, + dim: impl Into>, + keepdim: bool, + interpolation: &str, + ) -> Tensor { + self.f_nanquantile_scalar(q, dim, keepdim, interpolation).unwrap() + } + + pub fn nanquantile_scalar_out( + &self, + out: &Tensor, + q: f64, + dim: impl Into>, + keepdim: bool, + interpolation: &str, + ) -> Tensor { + self.f_nanquantile_scalar_out(out, q, dim, keepdim, interpolation).unwrap() + } + + pub fn nansum( + &self, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Tensor { + self.f_nansum(dim, keepdim, dtype).unwrap() + } + + pub fn nansum_out( + &self, + out: &Tensor, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Tensor { + self.f_nansum_out(out, dim, keepdim, dtype).unwrap() + } + + pub fn narrow(&self, dim: i64, start: i64, length: i64) -> Tensor { + self.f_narrow(dim, start, length).unwrap() + } + + pub fn narrow_copy(&self, dim: i64, start: i64, length: i64) -> Tensor { + self.f_narrow_copy(dim, start, length).unwrap() + } + + pub fn narrow_copy_out(&self, out: &Tensor, dim: i64, start: i64, length: i64) -> Tensor { + self.f_narrow_copy_out(out, dim, start, length).unwrap() + } + + pub fn narrow_tensor(&self, dim: i64, start: &Tensor, length: i64) -> Tensor { + self.f_narrow_tensor(dim, start, length).unwrap() + } + + pub fn native_batch_norm>( + &self, + weight: Option, + bias: Option, + running_mean: Option, + running_var: Option, + training: bool, + momentum: f64, + eps: f64, + ) -> (Tensor, Tensor, Tensor) { + self.f_native_batch_norm(weight, bias, running_mean, running_var, training, momentum, eps) + .unwrap() + } + + pub fn native_batch_norm_out>( + &self, + out: &Tensor, + save_mean: &Tensor, + save_invstd: &Tensor, + weight: Option, + bias: Option, + running_mean: Option, + running_var: Option, + training: bool, + momentum: f64, + eps: f64, + ) -> (Tensor, Tensor, Tensor) { + self.f_native_batch_norm_out( + out, + save_mean, + save_invstd, + weight, + bias, + running_mean, + running_var, + training, + momentum, + eps, + ) + .unwrap() + } + + pub fn native_channel_shuffle(&self, groups: i64) -> Tensor { + self.f_native_channel_shuffle(groups).unwrap() + } + + pub fn native_dropout(&self, p: f64, train: bool) -> (Tensor, Tensor) { + self.f_native_dropout(p, train).unwrap() + } + + pub fn native_dropout_backward(grad_output: &Tensor, mask: &Tensor, scale: f64) -> Tensor { + Tensor::f_native_dropout_backward(grad_output, mask, scale).unwrap() + } + + pub fn native_dropout_backward_out( + out: &Tensor, + grad_output: &Tensor, + mask: &Tensor, + scale: f64, + ) -> Tensor { + Tensor::f_native_dropout_backward_out(out, grad_output, mask, scale).unwrap() + } + + pub fn native_dropout_out( + &self, + out0: &Tensor, + out1: &Tensor, + p: f64, + train: bool, + ) -> (Tensor, Tensor) { + self.f_native_dropout_out(out0, out1, p, train).unwrap() + } + + pub fn native_group_norm>( + &self, + weight: Option, + bias: Option, + n: i64, + c: i64, + hxw: i64, + group: i64, + eps: f64, + ) -> (Tensor, Tensor, Tensor) { + self.f_native_group_norm(weight, bias, n, c, hxw, group, eps).unwrap() + } + + pub fn native_group_norm_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + weight: Option, + bias: Option, + n: i64, + c: i64, + hxw: i64, + group: i64, + eps: f64, + ) -> (Tensor, Tensor, Tensor) { + self.f_native_group_norm_out(out0, out1, out2, weight, bias, n, c, hxw, group, eps).unwrap() + } + + pub fn native_layer_norm>( + &self, + normalized_shape: impl IntList, + weight: Option, + bias: Option, + eps: f64, + ) -> (Tensor, Tensor, Tensor) { + self.f_native_layer_norm(normalized_shape, weight, bias, eps).unwrap() + } + + pub fn native_layer_norm_out>( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + normalized_shape: impl IntList, + weight: Option, + bias: Option, + eps: f64, + ) -> (Tensor, Tensor, Tensor) { + self.f_native_layer_norm_out(out0, out1, out2, normalized_shape, weight, bias, eps).unwrap() + } + + pub fn native_norm(&self) -> Tensor { + self.f_native_norm().unwrap() + } + + pub fn native_norm_out(&self, out: &Tensor) -> Tensor { + self.f_native_norm_out(out).unwrap() + } + + pub fn native_norm_scalaropt_dim_dtype>( + &self, + p: S, + dim: impl IntList, + keepdim: bool, + dtype: impl Into>, + ) -> Tensor { + self.f_native_norm_scalaropt_dim_dtype(p, dim, keepdim, dtype).unwrap() + } + + pub fn native_norm_scalaropt_dim_dtype_out>( + &self, + out: &Tensor, + p: S, + dim: impl IntList, + keepdim: bool, + dtype: impl Into>, + ) -> Tensor { + self.f_native_norm_scalaropt_dim_dtype_out(out, p, dim, keepdim, dtype).unwrap() + } + + pub fn ne>(&self, other: S) -> Tensor { + self.f_ne(other).unwrap() + } + + pub fn ne_>(&mut self, other: S) -> Tensor { + self.f_ne_(other).unwrap() + } + + pub fn ne_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_ne_scalar_out(out, other).unwrap() + } + + pub fn ne_tensor(&self, other: &Tensor) -> Tensor { + self.f_ne_tensor(other).unwrap() + } + + pub fn ne_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_ne_tensor_(other).unwrap() + } + + pub fn ne_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_ne_tensor_out(out, other).unwrap() + } + + pub fn neg(&self) -> Tensor { + self.f_neg().unwrap() + } + + pub fn neg_(&mut self) -> Tensor { + self.f_neg_().unwrap() + } + + pub fn neg_out(&self, out: &Tensor) -> Tensor { + self.f_neg_out(out).unwrap() + } + + pub fn negative(&self) -> Tensor { + self.f_negative().unwrap() + } + + pub fn negative_(&mut self) -> Tensor { + self.f_negative_().unwrap() + } + + pub fn negative_out(&self, out: &Tensor) -> Tensor { + self.f_negative_out(out).unwrap() + } + + pub fn nested_to_padded_tensor(&self, padding: f64, output_size: impl IntListOption) -> Tensor { + self.f_nested_to_padded_tensor(padding, output_size).unwrap() + } + + pub fn new_empty(&self, size: impl IntList, options: (Kind, Device)) -> Tensor { + self.f_new_empty(size, options).unwrap() + } + + pub fn new_empty_out(&self, out: &Tensor, size: impl IntList) -> Tensor { + self.f_new_empty_out(out, size).unwrap() + } + + pub fn new_empty_strided( + &self, + size: impl IntList, + stride: impl IntList, + options: (Kind, Device), + ) -> Tensor { + self.f_new_empty_strided(size, stride, options).unwrap() + } + + pub fn new_empty_strided_out( + &self, + out: &Tensor, + size: impl IntList, + stride: impl IntList, + ) -> Tensor { + self.f_new_empty_strided_out(out, size, stride).unwrap() + } + + pub fn new_full>( + &self, + size: impl IntList, + fill_value: S, + options: (Kind, Device), + ) -> Tensor { + self.f_new_full(size, fill_value, options).unwrap() + } + + pub fn new_full_out>( + &self, + out: &Tensor, + size: impl IntList, + fill_value: S, + ) -> Tensor { + self.f_new_full_out(out, size, fill_value).unwrap() + } + + pub fn new_ones(&self, size: impl IntList, options: (Kind, Device)) -> Tensor { + self.f_new_ones(size, options).unwrap() + } + + pub fn new_ones_out(&self, out: &Tensor, size: impl IntList) -> Tensor { + self.f_new_ones_out(out, size).unwrap() + } + + pub fn new_zeros(&self, size: impl IntList, options: (Kind, Device)) -> Tensor { + self.f_new_zeros(size, options).unwrap() + } + + pub fn new_zeros_out(&self, out: &Tensor, size: impl IntList) -> Tensor { + self.f_new_zeros_out(out, size).unwrap() + } + + pub fn nextafter(&self, other: &Tensor) -> Tensor { + self.f_nextafter(other).unwrap() + } + + pub fn nextafter_(&mut self, other: &Tensor) -> Tensor { + self.f_nextafter_(other).unwrap() + } + + pub fn nextafter_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_nextafter_out(out, other).unwrap() + } + + pub fn g_nll_loss>( + &self, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ignore_index: i64, + ) -> Tensor { + self.f_nll_loss(target, weight, reduction, ignore_index).unwrap() + } + + pub fn nll_loss2d>( + &self, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ignore_index: i64, + ) -> Tensor { + self.f_nll_loss2d(target, weight, reduction, ignore_index).unwrap() + } + + pub fn nll_loss2d_backward>( + &self, + grad_output: &Tensor, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ignore_index: i64, + total_weight: &Tensor, + ) -> Tensor { + self.f_nll_loss2d_backward( + grad_output, + target, + weight, + reduction, + ignore_index, + total_weight, + ) + .unwrap() + } + + pub fn nll_loss2d_backward_grad_input>( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ignore_index: i64, + total_weight: &Tensor, + ) -> Tensor { + self.f_nll_loss2d_backward_grad_input( + grad_input, + grad_output, + target, + weight, + reduction, + ignore_index, + total_weight, + ) + .unwrap() + } + + pub fn nll_loss2d_out>( + &self, + out: &Tensor, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ignore_index: i64, + ) -> Tensor { + self.f_nll_loss2d_out(out, target, weight, reduction, ignore_index).unwrap() + } + + pub fn nll_loss_backward>( + &self, + grad_output: &Tensor, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ignore_index: i64, + total_weight: &Tensor, + ) -> Tensor { + self.f_nll_loss_backward(grad_output, target, weight, reduction, ignore_index, total_weight) + .unwrap() + } + + pub fn nll_loss_backward_grad_input>( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ignore_index: i64, + total_weight: &Tensor, + ) -> Tensor { + self.f_nll_loss_backward_grad_input( + grad_input, + grad_output, + target, + weight, + reduction, + ignore_index, + total_weight, + ) + .unwrap() + } + + pub fn nll_loss_nd>( + &self, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ignore_index: i64, + ) -> Tensor { + self.f_nll_loss_nd(target, weight, reduction, ignore_index).unwrap() + } + + pub fn nll_loss_out>( + &self, + out: &Tensor, + target: &Tensor, + weight: Option, + reduction: crate::Reduction, + ignore_index: i64, + ) -> Tensor { + self.f_nll_loss_out(out, target, weight, reduction, ignore_index).unwrap() + } + + pub fn nonzero(&self) -> Tensor { + self.f_nonzero().unwrap() + } + + pub fn nonzero_numpy(&self) -> Vec { + self.f_nonzero_numpy().unwrap() + } + + pub fn nonzero_out(&self, out: &Tensor) -> Tensor { + self.f_nonzero_out(out).unwrap() + } + + pub fn nonzero_static(&self, size: i64, fill_value: i64) -> Tensor { + self.f_nonzero_static(size, fill_value).unwrap() + } + + pub fn nonzero_static_out(&self, out: &Tensor, size: i64, fill_value: i64) -> Tensor { + self.f_nonzero_static_out(out, size, fill_value).unwrap() + } + + pub fn norm(&self) -> Tensor { + self.f_norm().unwrap() + } + + pub fn norm_dtype_out>( + &self, + out: &Tensor, + p: S, + dim: impl IntList, + keepdim: bool, + dtype: Kind, + ) -> Tensor { + self.f_norm_dtype_out(out, p, dim, keepdim, dtype).unwrap() + } + + pub fn norm_except_dim(v: &Tensor, pow: i64, dim: i64) -> Tensor { + Tensor::f_norm_except_dim(v, pow, dim).unwrap() + } + + pub fn norm_out>( + &self, + out: &Tensor, + p: S, + dim: impl IntList, + keepdim: bool, + ) -> Tensor { + self.f_norm_out(out, p, dim, keepdim).unwrap() + } + + pub fn norm_scalar_out(&self, out: &Tensor) -> Tensor { + self.f_norm_scalar_out(out).unwrap() + } + + pub fn norm_scalaropt_dim>( + &self, + p: S, + dim: impl IntList, + keepdim: bool, + ) -> Tensor { + self.f_norm_scalaropt_dim(p, dim, keepdim).unwrap() + } + + pub fn norm_scalaropt_dim_dtype>( + &self, + p: S, + dim: impl IntList, + keepdim: bool, + dtype: Kind, + ) -> Tensor { + self.f_norm_scalaropt_dim_dtype(p, dim, keepdim, dtype).unwrap() + } + + pub fn norm_scalaropt_dtype>(&self, p: S, dtype: Kind) -> Tensor { + self.f_norm_scalaropt_dtype(p, dtype).unwrap() + } + + pub fn norm_scalaropt_dtype_out>( + &self, + out: &Tensor, + p: S, + dtype: Kind, + ) -> Tensor { + self.f_norm_scalaropt_dtype_out(out, p, dtype).unwrap() + } + + pub fn normal_(&mut self, mean: f64, std: f64) -> Tensor { + self.f_normal_(mean, std).unwrap() + } + + pub fn normal_functional(&self, mean: f64, std: f64) -> Tensor { + self.f_normal_functional(mean, std).unwrap() + } + + pub fn not_equal>(&self, other: S) -> Tensor { + self.f_not_equal(other).unwrap() + } + + pub fn not_equal_>(&mut self, other: S) -> Tensor { + self.f_not_equal_(other).unwrap() + } + + pub fn not_equal_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_not_equal_scalar_out(out, other).unwrap() + } + + pub fn not_equal_tensor(&self, other: &Tensor) -> Tensor { + self.f_not_equal_tensor(other).unwrap() + } + + pub fn not_equal_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_not_equal_tensor_(other).unwrap() + } + + pub fn not_equal_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_not_equal_tensor_out(out, other).unwrap() + } + + pub fn nuclear_norm(&self, keepdim: bool) -> Tensor { + self.f_nuclear_norm(keepdim).unwrap() + } + + pub fn nuclear_norm_dim(&self, dim: impl IntList, keepdim: bool) -> Tensor { + self.f_nuclear_norm_dim(dim, keepdim).unwrap() + } + + pub fn nuclear_norm_dim_out(&self, out: &Tensor, dim: impl IntList, keepdim: bool) -> Tensor { + self.f_nuclear_norm_dim_out(out, dim, keepdim).unwrap() + } + + pub fn nuclear_norm_out(&self, out: &Tensor, keepdim: bool) -> Tensor { + self.f_nuclear_norm_out(out, keepdim).unwrap() + } + + pub fn numpy_t(&self) -> Tensor { + self.f_numpy_t().unwrap() + } + + pub fn one_hot(&self, num_classes: i64) -> Tensor { + self.f_one_hot(num_classes).unwrap() + } + + pub fn ones(size: impl IntList, options: (Kind, Device)) -> Tensor { + Tensor::f_ones(size, options).unwrap() + } + + pub fn ones_like(&self) -> Tensor { + self.f_ones_like().unwrap() + } + + pub fn ones_like_out(&self, out: &Tensor) -> Tensor { + self.f_ones_like_out(out).unwrap() + } + + pub fn ones_out(out: &Tensor, size: impl IntList) -> Tensor { + Tensor::f_ones_out(out, size).unwrap() + } + + pub fn orgqr(&self, input2: &Tensor) -> Tensor { + self.f_orgqr(input2).unwrap() + } + + pub fn orgqr_out(&self, out: &Tensor, input2: &Tensor) -> Tensor { + self.f_orgqr_out(out, input2).unwrap() + } + + pub fn ormqr(&self, input2: &Tensor, input3: &Tensor, left: bool, transpose: bool) -> Tensor { + self.f_ormqr(input2, input3, left, transpose).unwrap() + } + + pub fn ormqr_out( + &self, + out: &Tensor, + input2: &Tensor, + input3: &Tensor, + left: bool, + transpose: bool, + ) -> Tensor { + self.f_ormqr_out(out, input2, input3, left, transpose).unwrap() + } + + pub fn outer(&self, vec2: &Tensor) -> Tensor { + self.f_outer(vec2).unwrap() + } + + pub fn outer_out(&self, out: &Tensor, vec2: &Tensor) -> Tensor { + self.f_outer_out(out, vec2).unwrap() + } + + pub fn output_nr(&self) -> i64 { + self.f_output_nr().unwrap() + } + + pub fn pad(&self, pad: impl IntList, mode: &str, value: impl Into>) -> Tensor { + self.f_pad(pad, mode, value).unwrap() + } + + pub fn pad_sequence>( + sequences: &[T], + batch_first: bool, + padding_value: f64, + padding_side: &str, + ) -> Tensor { + Tensor::f_pad_sequence(sequences, batch_first, padding_value, padding_side).unwrap() + } + + pub fn pairwise_distance(x1: &Tensor, x2: &Tensor, p: f64, eps: f64, keepdim: bool) -> Tensor { + Tensor::f_pairwise_distance(x1, x2, p, eps, keepdim).unwrap() + } + + pub fn pdist(&self, p: f64) -> Tensor { + self.f_pdist(p).unwrap() + } + + pub fn permute(&self, dims: impl IntList) -> Tensor { + self.f_permute(dims).unwrap() + } + + pub fn permute_copy(&self, dims: impl IntList) -> Tensor { + self.f_permute_copy(dims).unwrap() + } + + pub fn permute_copy_out(&self, out: &Tensor, dims: impl IntList) -> Tensor { + self.f_permute_copy_out(out, dims).unwrap() + } + + pub fn pin_memory(&self, device: Device) -> Tensor { + self.f_pin_memory(device).unwrap() + } + + pub fn pinverse(&self, rcond: f64) -> Tensor { + self.f_pinverse(rcond).unwrap() + } + + pub fn pixel_shuffle(&self, upscale_factor: i64) -> Tensor { + self.f_pixel_shuffle(upscale_factor).unwrap() + } + + pub fn pixel_shuffle_out(&self, out: &Tensor, upscale_factor: i64) -> Tensor { + self.f_pixel_shuffle_out(out, upscale_factor).unwrap() + } + + pub fn pixel_unshuffle(&self, downscale_factor: i64) -> Tensor { + self.f_pixel_unshuffle(downscale_factor).unwrap() + } + + pub fn pixel_unshuffle_out(&self, out: &Tensor, downscale_factor: i64) -> Tensor { + self.f_pixel_unshuffle_out(out, downscale_factor).unwrap() + } + + pub fn poisson(&self) -> Tensor { + self.f_poisson().unwrap() + } + + pub fn poisson_nll_loss( + &self, + target: &Tensor, + log_input: bool, + full: bool, + eps: f64, + reduction: crate::Reduction, + ) -> Tensor { + self.f_poisson_nll_loss(target, log_input, full, eps, reduction).unwrap() + } + + pub fn poisson_out(&self, out: &Tensor) -> Tensor { + self.f_poisson_out(out).unwrap() + } + + pub fn polar(abs: &Tensor, angle: &Tensor) -> Tensor { + Tensor::f_polar(abs, angle).unwrap() + } + + pub fn polar_out(out: &Tensor, abs: &Tensor, angle: &Tensor) -> Tensor { + Tensor::f_polar_out(out, abs, angle).unwrap() + } + + pub fn polygamma(&self, n: i64) -> Tensor { + self.f_polygamma(n).unwrap() + } + + pub fn polygamma_(&mut self, n: i64) -> Tensor { + self.f_polygamma_(n).unwrap() + } + + pub fn polygamma_out(&self, out: &Tensor, n: i64) -> Tensor { + self.f_polygamma_out(out, n).unwrap() + } + + pub fn positive(&self) -> Tensor { + self.f_positive().unwrap() + } + + pub fn pow(&self, exponent: &Tensor) -> Tensor { + self.f_pow(exponent).unwrap() + } + + pub fn pow_>(&mut self, exponent: S) -> Tensor { + self.f_pow_(exponent).unwrap() + } + + pub fn pow_scalar>(self_scalar: S, exponent: &Tensor) -> Tensor { + Tensor::f_pow_scalar(self_scalar, exponent).unwrap() + } + + pub fn pow_scalar_out>( + out: &Tensor, + self_scalar: S, + exponent: &Tensor, + ) -> Tensor { + Tensor::f_pow_scalar_out(out, self_scalar, exponent).unwrap() + } + + pub fn pow_tensor_(&mut self, exponent: &Tensor) -> Tensor { + self.f_pow_tensor_(exponent).unwrap() + } + + pub fn pow_tensor_scalar>(&self, exponent: S) -> Tensor { + self.f_pow_tensor_scalar(exponent).unwrap() + } + + pub fn pow_tensor_scalar_out>(&self, out: &Tensor, exponent: S) -> Tensor { + self.f_pow_tensor_scalar_out(out, exponent).unwrap() + } + + pub fn pow_tensor_tensor_out(&self, out: &Tensor, exponent: &Tensor) -> Tensor { + self.f_pow_tensor_tensor_out(out, exponent).unwrap() + } + + pub fn prelu(&self, weight: &Tensor) -> Tensor { + self.f_prelu(weight).unwrap() + } + + pub fn prod(&self, dtype: impl Into>) -> Tensor { + self.f_prod(dtype).unwrap() + } + + pub fn prod_dim_int(&self, dim: i64, keepdim: bool, dtype: impl Into>) -> Tensor { + self.f_prod_dim_int(dim, keepdim, dtype).unwrap() + } + + pub fn prod_int_out( + &self, + out: &Tensor, + dim: i64, + keepdim: bool, + dtype: impl Into>, + ) -> Tensor { + self.f_prod_int_out(out, dim, keepdim, dtype).unwrap() + } + + pub fn prod_out(&self, out: &Tensor, dtype: impl Into>) -> Tensor { + self.f_prod_out(out, dtype).unwrap() + } + + pub fn put(&self, index: &Tensor, source: &Tensor, accumulate: bool) -> Tensor { + self.f_put(index, source, accumulate).unwrap() + } + + pub fn put_(&mut self, index: &Tensor, source: &Tensor, accumulate: bool) -> Tensor { + self.f_put_(index, source, accumulate).unwrap() + } + + pub fn put_out( + &self, + out: &Tensor, + index: &Tensor, + source: &Tensor, + accumulate: bool, + ) -> Tensor { + self.f_put_out(out, index, source, accumulate).unwrap() + } + + pub fn q_per_channel_axis(&self) -> i64 { + self.f_q_per_channel_axis().unwrap() + } + + pub fn q_per_channel_scales(&self) -> Tensor { + self.f_q_per_channel_scales().unwrap() + } + + pub fn q_per_channel_scales_out(&self, out: &Tensor) -> Tensor { + self.f_q_per_channel_scales_out(out).unwrap() + } + + pub fn q_per_channel_zero_points(&self) -> Tensor { + self.f_q_per_channel_zero_points().unwrap() + } + + pub fn q_per_channel_zero_points_out(&self, out: &Tensor) -> Tensor { + self.f_q_per_channel_zero_points_out(out).unwrap() + } + + pub fn q_scale(&self) -> f64 { + self.f_q_scale().unwrap() + } + + pub fn q_zero_point(&self) -> i64 { + self.f_q_zero_point().unwrap() + } + + pub fn qr(&self, some: bool) -> (Tensor, Tensor) { + self.f_qr(some).unwrap() + } + + pub fn qr_q(&self, q: &Tensor, r: &Tensor, some: bool) -> (Tensor, Tensor) { + self.f_qr_q(q, r, some).unwrap() + } + + pub fn quantile( + &self, + q: &Tensor, + dim: impl Into>, + keepdim: bool, + interpolation: &str, + ) -> Tensor { + self.f_quantile(q, dim, keepdim, interpolation).unwrap() + } + + pub fn quantile_out( + &self, + out: &Tensor, + q: &Tensor, + dim: impl Into>, + keepdim: bool, + interpolation: &str, + ) -> Tensor { + self.f_quantile_out(out, q, dim, keepdim, interpolation).unwrap() + } + + pub fn quantile_scalar( + &self, + q: f64, + dim: impl Into>, + keepdim: bool, + interpolation: &str, + ) -> Tensor { + self.f_quantile_scalar(q, dim, keepdim, interpolation).unwrap() + } + + pub fn quantile_scalar_out( + &self, + out: &Tensor, + q: f64, + dim: impl Into>, + keepdim: bool, + interpolation: &str, + ) -> Tensor { + self.f_quantile_scalar_out(out, q, dim, keepdim, interpolation).unwrap() + } + + pub fn quantize_per_channel( + &self, + scales: &Tensor, + zero_points: &Tensor, + axis: i64, + dtype: Kind, + ) -> Tensor { + self.f_quantize_per_channel(scales, zero_points, axis, dtype).unwrap() + } + + pub fn quantize_per_channel_out( + &self, + out: &Tensor, + scales: &Tensor, + zero_points: &Tensor, + axis: i64, + dtype: Kind, + ) -> Tensor { + self.f_quantize_per_channel_out(out, scales, zero_points, axis, dtype).unwrap() + } + + pub fn quantize_per_tensor(&self, scale: f64, zero_point: i64, dtype: Kind) -> Tensor { + self.f_quantize_per_tensor(scale, zero_point, dtype).unwrap() + } + + pub fn quantize_per_tensor_dynamic(&self, dtype: Kind, reduce_range: bool) -> Tensor { + self.f_quantize_per_tensor_dynamic(dtype, reduce_range).unwrap() + } + + pub fn quantize_per_tensor_dynamic_out( + &self, + out: &Tensor, + dtype: Kind, + reduce_range: bool, + ) -> Tensor { + self.f_quantize_per_tensor_dynamic_out(out, dtype, reduce_range).unwrap() + } + + pub fn quantize_per_tensor_out( + &self, + out: &Tensor, + scale: f64, + zero_point: i64, + dtype: Kind, + ) -> Tensor { + self.f_quantize_per_tensor_out(out, scale, zero_point, dtype).unwrap() + } + + pub fn quantize_per_tensor_tensor_qparams( + &self, + scale: &Tensor, + zero_point: &Tensor, + dtype: Kind, + ) -> Tensor { + self.f_quantize_per_tensor_tensor_qparams(scale, zero_point, dtype).unwrap() + } + + pub fn quantize_per_tensor_tensor_qparams_out( + &self, + out: &Tensor, + scale: &Tensor, + zero_point: &Tensor, + dtype: Kind, + ) -> Tensor { + self.f_quantize_per_tensor_tensor_qparams_out(out, scale, zero_point, dtype).unwrap() + } + + pub fn quantize_per_tensor_tensors>( + tensors: &[T], + scales: &Tensor, + zero_points: &Tensor, + dtype: Kind, + ) -> Vec { + Tensor::f_quantize_per_tensor_tensors(tensors, scales, zero_points, dtype).unwrap() + } + + pub fn quantize_per_tensor_tensors_out>( + out: &[T], + tensors: &[T], + scales: &Tensor, + zero_points: &Tensor, + dtype: Kind, + ) { + Tensor::f_quantize_per_tensor_tensors_out(out, tensors, scales, zero_points, dtype).unwrap() + } + + pub fn quantized_batch_norm>( + &self, + weight: Option, + bias: Option, + mean: &Tensor, + var: &Tensor, + eps: f64, + output_scale: f64, + output_zero_point: i64, + ) -> Tensor { + self.f_quantized_batch_norm(weight, bias, mean, var, eps, output_scale, output_zero_point) + .unwrap() + } + + pub fn quantized_batch_norm_out>( + &self, + out: &Tensor, + weight: Option, + bias: Option, + mean: &Tensor, + var: &Tensor, + eps: f64, + output_scale: f64, + output_zero_point: i64, + ) -> Tensor { + self.f_quantized_batch_norm_out( + out, + weight, + bias, + mean, + var, + eps, + output_scale, + output_zero_point, + ) + .unwrap() + } + + pub fn quantized_gru_cell>( + &self, + hx: &Tensor, + w_ih: &Tensor, + w_hh: &Tensor, + b_ih: &Tensor, + b_hh: &Tensor, + packed_ih: &Tensor, + packed_hh: &Tensor, + col_offsets_ih: &Tensor, + col_offsets_hh: &Tensor, + scale_ih: S, + scale_hh: S, + zero_point_ih: S, + zero_point_hh: S, + ) -> Tensor { + self.f_quantized_gru_cell( + hx, + w_ih, + w_hh, + b_ih, + b_hh, + packed_ih, + packed_hh, + col_offsets_ih, + col_offsets_hh, + scale_ih, + scale_hh, + zero_point_ih, + zero_point_hh, + ) + .unwrap() + } + + pub fn quantized_lstm_cell, S: Into>( + &self, + hx: &[T], + w_ih: &Tensor, + w_hh: &Tensor, + b_ih: &Tensor, + b_hh: &Tensor, + packed_ih: &Tensor, + packed_hh: &Tensor, + col_offsets_ih: &Tensor, + col_offsets_hh: &Tensor, + scale_ih: S, + scale_hh: S, + zero_point_ih: S, + zero_point_hh: S, + ) -> (Tensor, Tensor) { + self.f_quantized_lstm_cell( + hx, + w_ih, + w_hh, + b_ih, + b_hh, + packed_ih, + packed_hh, + col_offsets_ih, + col_offsets_hh, + scale_ih, + scale_hh, + zero_point_ih, + zero_point_hh, + ) + .unwrap() + } + + pub fn quantized_max_pool1d( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Tensor { + self.f_quantized_max_pool1d(kernel_size, stride, padding, dilation, ceil_mode).unwrap() + } + + pub fn quantized_max_pool1d_out( + &self, + out: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Tensor { + self.f_quantized_max_pool1d_out(out, kernel_size, stride, padding, dilation, ceil_mode) + .unwrap() + } + + pub fn quantized_max_pool2d( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Tensor { + self.f_quantized_max_pool2d(kernel_size, stride, padding, dilation, ceil_mode).unwrap() + } + + pub fn quantized_max_pool2d_out( + &self, + out: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Tensor { + self.f_quantized_max_pool2d_out(out, kernel_size, stride, padding, dilation, ceil_mode) + .unwrap() + } + + pub fn quantized_max_pool3d( + &self, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Tensor { + self.f_quantized_max_pool3d(kernel_size, stride, padding, dilation, ceil_mode).unwrap() + } + + pub fn quantized_max_pool3d_out( + &self, + out: &Tensor, + kernel_size: impl IntList, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ceil_mode: bool, + ) -> Tensor { + self.f_quantized_max_pool3d_out(out, kernel_size, stride, padding, dilation, ceil_mode) + .unwrap() + } + + pub fn quantized_rnn_relu_cell>( + &self, + hx: &Tensor, + w_ih: &Tensor, + w_hh: &Tensor, + b_ih: &Tensor, + b_hh: &Tensor, + packed_ih: &Tensor, + packed_hh: &Tensor, + col_offsets_ih: &Tensor, + col_offsets_hh: &Tensor, + scale_ih: S, + scale_hh: S, + zero_point_ih: S, + zero_point_hh: S, + ) -> Tensor { + self.f_quantized_rnn_relu_cell( + hx, + w_ih, + w_hh, + b_ih, + b_hh, + packed_ih, + packed_hh, + col_offsets_ih, + col_offsets_hh, + scale_ih, + scale_hh, + zero_point_ih, + zero_point_hh, + ) + .unwrap() + } + + pub fn quantized_rnn_tanh_cell>( + &self, + hx: &Tensor, + w_ih: &Tensor, + w_hh: &Tensor, + b_ih: &Tensor, + b_hh: &Tensor, + packed_ih: &Tensor, + packed_hh: &Tensor, + col_offsets_ih: &Tensor, + col_offsets_hh: &Tensor, + scale_ih: S, + scale_hh: S, + zero_point_ih: S, + zero_point_hh: S, + ) -> Tensor { + self.f_quantized_rnn_tanh_cell( + hx, + w_ih, + w_hh, + b_ih, + b_hh, + packed_ih, + packed_hh, + col_offsets_ih, + col_offsets_hh, + scale_ih, + scale_hh, + zero_point_ih, + zero_point_hh, + ) + .unwrap() + } + + pub fn rad2deg(&self) -> Tensor { + self.f_rad2deg().unwrap() + } + + pub fn rad2deg_(&mut self) -> Tensor { + self.f_rad2deg_().unwrap() + } + + pub fn rad2deg_out(&self, out: &Tensor) -> Tensor { + self.f_rad2deg_out(out).unwrap() + } + + pub fn rand(size: impl IntList, options: (Kind, Device)) -> Tensor { + Tensor::f_rand(size, options).unwrap() + } + + pub fn rand_like(&self) -> Tensor { + self.f_rand_like().unwrap() + } + + pub fn rand_like_out(&self, out: &Tensor) -> Tensor { + self.f_rand_like_out(out).unwrap() + } + + pub fn rand_out(out: &Tensor, size: impl IntList) -> Tensor { + Tensor::f_rand_out(out, size).unwrap() + } + + pub fn randint(high: i64, size: impl IntList, options: (Kind, Device)) -> Tensor { + Tensor::f_randint(high, size, options).unwrap() + } + + pub fn randint_like(&self, high: i64) -> Tensor { + self.f_randint_like(high).unwrap() + } + + pub fn randint_like_low_dtype(&self, low: i64, high: i64) -> Tensor { + self.f_randint_like_low_dtype(low, high).unwrap() + } + + pub fn randint_like_low_dtype_out(&self, out: &Tensor, low: i64, high: i64) -> Tensor { + self.f_randint_like_low_dtype_out(out, low, high).unwrap() + } + + pub fn randint_like_out(&self, out: &Tensor, high: i64) -> Tensor { + self.f_randint_like_out(out, high).unwrap() + } + + pub fn randint_like_tensor(&self, high: &Tensor) -> Tensor { + self.f_randint_like_tensor(high).unwrap() + } + + pub fn randint_like_tensor_out(&self, out: &Tensor, high: &Tensor) -> Tensor { + self.f_randint_like_tensor_out(out, high).unwrap() + } + + pub fn randint_low(low: i64, high: i64, size: impl IntList, options: (Kind, Device)) -> Tensor { + Tensor::f_randint_low(low, high, size, options).unwrap() + } + + pub fn randint_low_out(out: &Tensor, low: i64, high: i64, size: impl IntList) -> Tensor { + Tensor::f_randint_low_out(out, low, high, size).unwrap() + } + + pub fn randint_out(out: &Tensor, high: i64, size: impl IntList) -> Tensor { + Tensor::f_randint_out(out, high, size).unwrap() + } + + pub fn randn(size: impl IntList, options: (Kind, Device)) -> Tensor { + Tensor::f_randn(size, options).unwrap() + } + + pub fn randn_like(&self) -> Tensor { + self.f_randn_like().unwrap() + } + + pub fn randn_like_out(&self, out: &Tensor) -> Tensor { + self.f_randn_like_out(out).unwrap() + } + + pub fn randn_out(out: &Tensor, size: impl IntList) -> Tensor { + Tensor::f_randn_out(out, size).unwrap() + } + + pub fn random(&self) -> Tensor { + self.f_random().unwrap() + } + + pub fn random_(&mut self) -> Tensor { + self.f_random_().unwrap() + } + + pub fn random_from(&self, from: i64, to: impl Into>) -> Tensor { + self.f_random_from(from, to).unwrap() + } + + pub fn random_from_(&mut self, from: i64, to: impl Into>) -> Tensor { + self.f_random_from_(from, to).unwrap() + } + + pub fn random_from_out(&self, out: &Tensor, from: i64, to: impl Into>) -> Tensor { + self.f_random_from_out(out, from, to).unwrap() + } + + pub fn random_out(&self, out: &Tensor) -> Tensor { + self.f_random_out(out).unwrap() + } + + pub fn random_to(&self, to: i64) -> Tensor { + self.f_random_to(to).unwrap() + } + + pub fn random_to_(&mut self, to: i64) -> Tensor { + self.f_random_to_(to).unwrap() + } + + pub fn random_to_out(&self, out: &Tensor, to: i64) -> Tensor { + self.f_random_to_out(out, to).unwrap() + } + + pub fn randperm(n: i64, options: (Kind, Device)) -> Tensor { + Tensor::f_randperm(n, options).unwrap() + } + + pub fn randperm_out(out: &Tensor, n: i64) -> Tensor { + Tensor::f_randperm_out(out, n).unwrap() + } + + pub fn range>(start: S, end: S, options: (Kind, Device)) -> Tensor { + Tensor::f_range(start, end, options).unwrap() + } + + pub fn range_out>(out: &Tensor, start: S, end: S) -> Tensor { + Tensor::f_range_out(out, start, end).unwrap() + } + + pub fn range_out_>(out: &Tensor, start: S, end: S) -> Tensor { + Tensor::f_range_out_(out, start, end).unwrap() + } + + pub fn range_step>(start: S, end: S, options: (Kind, Device)) -> Tensor { + Tensor::f_range_step(start, end, options).unwrap() + } + + pub fn ravel(&self) -> Tensor { + self.f_ravel().unwrap() + } + + pub fn real(&self) -> Tensor { + self.f_real().unwrap() + } + + pub fn reciprocal(&self) -> Tensor { + self.f_reciprocal().unwrap() + } + + pub fn reciprocal_(&mut self) -> Tensor { + self.f_reciprocal_().unwrap() + } + + pub fn reciprocal_out(&self, out: &Tensor) -> Tensor { + self.f_reciprocal_out(out).unwrap() + } + + pub fn reflection_pad1d(&self, padding: impl IntList) -> Tensor { + self.f_reflection_pad1d(padding).unwrap() + } + + pub fn reflection_pad1d_backward(&self, grad_output: &Tensor, padding: impl IntList) -> Tensor { + self.f_reflection_pad1d_backward(grad_output, padding).unwrap() + } + + pub fn reflection_pad1d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + padding: impl IntList, + ) -> Tensor { + self.f_reflection_pad1d_backward_grad_input(grad_input, grad_output, padding).unwrap() + } + + pub fn reflection_pad1d_out(&self, out: &Tensor, padding: impl IntList) -> Tensor { + self.f_reflection_pad1d_out(out, padding).unwrap() + } + + pub fn reflection_pad2d(&self, padding: impl IntList) -> Tensor { + self.f_reflection_pad2d(padding).unwrap() + } + + pub fn reflection_pad2d_backward(&self, grad_output: &Tensor, padding: impl IntList) -> Tensor { + self.f_reflection_pad2d_backward(grad_output, padding).unwrap() + } + + pub fn reflection_pad2d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + padding: impl IntList, + ) -> Tensor { + self.f_reflection_pad2d_backward_grad_input(grad_input, grad_output, padding).unwrap() + } + + pub fn reflection_pad2d_out(&self, out: &Tensor, padding: impl IntList) -> Tensor { + self.f_reflection_pad2d_out(out, padding).unwrap() + } + + pub fn reflection_pad3d(&self, padding: impl IntList) -> Tensor { + self.f_reflection_pad3d(padding).unwrap() + } + + pub fn reflection_pad3d_backward(&self, grad_output: &Tensor, padding: impl IntList) -> Tensor { + self.f_reflection_pad3d_backward(grad_output, padding).unwrap() + } + + pub fn reflection_pad3d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + padding: impl IntList, + ) -> Tensor { + self.f_reflection_pad3d_backward_grad_input(grad_input, grad_output, padding).unwrap() + } + + pub fn reflection_pad3d_out(&self, out: &Tensor, padding: impl IntList) -> Tensor { + self.f_reflection_pad3d_out(out, padding).unwrap() + } + + pub fn relu(&self) -> Tensor { + self.f_relu().unwrap() + } + + pub fn relu6(&self) -> Tensor { + self.f_relu6().unwrap() + } + + pub fn relu6_(&mut self) -> Tensor { + self.f_relu6_().unwrap() + } + + pub fn relu_(&mut self) -> Tensor { + self.f_relu_().unwrap() + } + + pub fn relu_out(&self, out: &Tensor) -> Tensor { + self.f_relu_out(out).unwrap() + } + + pub fn remainder>(&self, other: S) -> Tensor { + self.f_remainder(other).unwrap() + } + + pub fn remainder_>(&mut self, other: S) -> Tensor { + self.f_remainder_(other).unwrap() + } + + pub fn remainder_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_remainder_scalar_out(out, other).unwrap() + } + + pub fn remainder_scalar_tensor>(self_scalar: S, other: &Tensor) -> Tensor { + Tensor::f_remainder_scalar_tensor(self_scalar, other).unwrap() + } + + pub fn remainder_scalar_tensor_out>( + out: &Tensor, + self_scalar: S, + other: &Tensor, + ) -> Tensor { + Tensor::f_remainder_scalar_tensor_out(out, self_scalar, other).unwrap() + } + + pub fn remainder_tensor(&self, other: &Tensor) -> Tensor { + self.f_remainder_tensor(other).unwrap() + } + + pub fn remainder_tensor_(&mut self, other: &Tensor) -> Tensor { + self.f_remainder_tensor_(other).unwrap() + } + + pub fn remainder_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_remainder_tensor_out(out, other).unwrap() + } + + pub fn renorm>(&self, p: S, dim: i64, maxnorm: S) -> Tensor { + self.f_renorm(p, dim, maxnorm).unwrap() + } + + pub fn renorm_>(&mut self, p: S, dim: i64, maxnorm: S) -> Tensor { + self.f_renorm_(p, dim, maxnorm).unwrap() + } + + pub fn renorm_out>(&self, out: &Tensor, p: S, dim: i64, maxnorm: S) -> Tensor { + self.f_renorm_out(out, p, dim, maxnorm).unwrap() + } + + pub fn repeat(&self, repeats: impl IntList) -> Tensor { + self.f_repeat(repeats).unwrap() + } + + pub fn repeat_interleave(repeats: &Tensor, output_size: impl Into>) -> Tensor { + Tensor::f_repeat_interleave(repeats, output_size).unwrap() + } + + pub fn repeat_interleave_self_int( + &self, + repeats: i64, + dim: impl Into>, + output_size: impl Into>, + ) -> Tensor { + self.f_repeat_interleave_self_int(repeats, dim, output_size).unwrap() + } + + pub fn repeat_interleave_self_tensor( + &self, + repeats: &Tensor, + dim: impl Into>, + output_size: impl Into>, + ) -> Tensor { + self.f_repeat_interleave_self_tensor(repeats, dim, output_size).unwrap() + } + + pub fn repeat_interleave_tensor_out( + out: &Tensor, + repeats: &Tensor, + output_size: impl Into>, + ) -> Tensor { + Tensor::f_repeat_interleave_tensor_out(out, repeats, output_size).unwrap() + } + + pub fn repeat_out(&self, out: &Tensor, repeats: impl IntList) -> Tensor { + self.f_repeat_out(out, repeats).unwrap() + } + + pub fn replication_pad1d(&self, padding: impl IntList) -> Tensor { + self.f_replication_pad1d(padding).unwrap() + } + + pub fn replication_pad1d_backward( + &self, + grad_output: &Tensor, + padding: impl IntList, + ) -> Tensor { + self.f_replication_pad1d_backward(grad_output, padding).unwrap() + } + + pub fn replication_pad1d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + padding: impl IntList, + ) -> Tensor { + self.f_replication_pad1d_backward_grad_input(grad_input, grad_output, padding).unwrap() + } + + pub fn replication_pad1d_out(&self, out: &Tensor, padding: impl IntList) -> Tensor { + self.f_replication_pad1d_out(out, padding).unwrap() + } + + pub fn replication_pad2d(&self, padding: impl IntList) -> Tensor { + self.f_replication_pad2d(padding).unwrap() + } + + pub fn replication_pad2d_backward( + &self, + grad_output: &Tensor, + padding: impl IntList, + ) -> Tensor { + self.f_replication_pad2d_backward(grad_output, padding).unwrap() + } + + pub fn replication_pad2d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + padding: impl IntList, + ) -> Tensor { + self.f_replication_pad2d_backward_grad_input(grad_input, grad_output, padding).unwrap() + } + + pub fn replication_pad2d_out(&self, out: &Tensor, padding: impl IntList) -> Tensor { + self.f_replication_pad2d_out(out, padding).unwrap() + } + + pub fn replication_pad3d(&self, padding: impl IntList) -> Tensor { + self.f_replication_pad3d(padding).unwrap() + } + + pub fn replication_pad3d_backward( + &self, + grad_output: &Tensor, + padding: impl IntList, + ) -> Tensor { + self.f_replication_pad3d_backward(grad_output, padding).unwrap() + } + + pub fn replication_pad3d_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + padding: impl IntList, + ) -> Tensor { + self.f_replication_pad3d_backward_grad_input(grad_input, grad_output, padding).unwrap() + } + + pub fn replication_pad3d_out(&self, out: &Tensor, padding: impl IntList) -> Tensor { + self.f_replication_pad3d_out(out, padding).unwrap() + } + + pub fn requires_grad_(&mut self, requires_grad: bool) -> Tensor { + self.f_requires_grad_(requires_grad).unwrap() + } + + pub fn reshape(&self, shape: impl IntList) -> Tensor { + self.f_reshape(shape).unwrap() + } + + pub fn reshape_as(&self, other: &Tensor) -> Tensor { + self.f_reshape_as(other).unwrap() + } + + pub fn resize(&self, size: impl IntList) -> Tensor { + self.f_resize(size).unwrap() + } + + pub fn resize_(&mut self, size: impl IntList) -> Tensor { + self.f_resize_(size).unwrap() + } + + pub fn resize_as(&self, the_template: &Tensor) -> Tensor { + self.f_resize_as(the_template).unwrap() + } + + pub fn resize_as_(&mut self, the_template: &Tensor) -> Tensor { + self.f_resize_as_(the_template).unwrap() + } + + pub fn resize_as_out(&self, out: &Tensor, the_template: &Tensor) -> Tensor { + self.f_resize_as_out(out, the_template).unwrap() + } + + pub fn resize_as_sparse(&self, the_template: &Tensor) -> Tensor { + self.f_resize_as_sparse(the_template).unwrap() + } + + pub fn resize_as_sparse_(&mut self, the_template: &Tensor) -> Tensor { + self.f_resize_as_sparse_(the_template).unwrap() + } + + pub fn resize_as_sparse_out(&self, out: &Tensor, the_template: &Tensor) -> Tensor { + self.f_resize_as_sparse_out(out, the_template).unwrap() + } + + pub fn resize_out(&self, out: &Tensor, size: impl IntList) -> Tensor { + self.f_resize_out(out, size).unwrap() + } + + pub fn resolve_conj(&self) -> Tensor { + self.f_resolve_conj().unwrap() + } + + pub fn resolve_neg(&self) -> Tensor { + self.f_resolve_neg().unwrap() + } + + pub fn retains_grad(&self) -> bool { + self.f_retains_grad().unwrap() + } + + pub fn rms_norm>( + &self, + normalized_shape: impl IntList, + weight: Option, + eps: impl Into>, + ) -> Tensor { + self.f_rms_norm(normalized_shape, weight, eps).unwrap() + } + + pub fn rnn_relu>( + &self, + hx: &Tensor, + params: &[T], + has_biases: bool, + num_layers: i64, + dropout: f64, + train: bool, + bidirectional: bool, + batch_first: bool, + ) -> (Tensor, Tensor) { + self.f_rnn_relu( + hx, + params, + has_biases, + num_layers, + dropout, + train, + bidirectional, + batch_first, + ) + .unwrap() + } + + pub fn rnn_relu_cell>( + &self, + hx: &Tensor, + w_ih: &Tensor, + w_hh: &Tensor, + b_ih: Option, + b_hh: Option, + ) -> Tensor { + self.f_rnn_relu_cell(hx, w_ih, w_hh, b_ih, b_hh).unwrap() + } + + pub fn rnn_relu_data>( + data: &Tensor, + batch_sizes: &Tensor, + hx: &Tensor, + params: &[T], + has_biases: bool, + num_layers: i64, + dropout: f64, + train: bool, + bidirectional: bool, + ) -> (Tensor, Tensor) { + Tensor::f_rnn_relu_data( + data, + batch_sizes, + hx, + params, + has_biases, + num_layers, + dropout, + train, + bidirectional, + ) + .unwrap() + } + + pub fn rnn_tanh>( + &self, + hx: &Tensor, + params: &[T], + has_biases: bool, + num_layers: i64, + dropout: f64, + train: bool, + bidirectional: bool, + batch_first: bool, + ) -> (Tensor, Tensor) { + self.f_rnn_tanh( + hx, + params, + has_biases, + num_layers, + dropout, + train, + bidirectional, + batch_first, + ) + .unwrap() + } + + pub fn rnn_tanh_cell>( + &self, + hx: &Tensor, + w_ih: &Tensor, + w_hh: &Tensor, + b_ih: Option, + b_hh: Option, + ) -> Tensor { + self.f_rnn_tanh_cell(hx, w_ih, w_hh, b_ih, b_hh).unwrap() + } + + pub fn rnn_tanh_data>( + data: &Tensor, + batch_sizes: &Tensor, + hx: &Tensor, + params: &[T], + has_biases: bool, + num_layers: i64, + dropout: f64, + train: bool, + bidirectional: bool, + ) -> (Tensor, Tensor) { + Tensor::f_rnn_tanh_data( + data, + batch_sizes, + hx, + params, + has_biases, + num_layers, + dropout, + train, + bidirectional, + ) + .unwrap() + } + + pub fn roll(&self, shifts: impl IntList, dims: impl IntList) -> Tensor { + self.f_roll(shifts, dims).unwrap() + } + + pub fn roll_out(&self, out: &Tensor, shifts: impl IntList, dims: impl IntList) -> Tensor { + self.f_roll_out(out, shifts, dims).unwrap() + } + + pub fn rot90(&self, k: i64, dims: impl IntList) -> Tensor { + self.f_rot90(k, dims).unwrap() + } + + pub fn rot90_out(&self, out: &Tensor, k: i64, dims: impl IntList) -> Tensor { + self.f_rot90_out(out, k, dims).unwrap() + } + + pub fn round(&self) -> Tensor { + self.f_round().unwrap() + } + + pub fn round_(&mut self) -> Tensor { + self.f_round_().unwrap() + } + + pub fn round_decimals(&self, decimals: i64) -> Tensor { + self.f_round_decimals(decimals).unwrap() + } + + pub fn round_decimals_(&mut self, decimals: i64) -> Tensor { + self.f_round_decimals_(decimals).unwrap() + } + + pub fn round_decimals_out(&self, out: &Tensor, decimals: i64) -> Tensor { + self.f_round_decimals_out(out, decimals).unwrap() + } + + pub fn round_out(&self, out: &Tensor) -> Tensor { + self.f_round_out(out).unwrap() + } + + pub fn row_indices(&self) -> Tensor { + self.f_row_indices().unwrap() + } + + pub fn row_indices_copy(&self) -> Tensor { + self.f_row_indices_copy().unwrap() + } + + pub fn row_indices_copy_out(&self, out: &Tensor) -> Tensor { + self.f_row_indices_copy_out(out).unwrap() + } + + pub fn row_stack>(tensors: &[T]) -> Tensor { + Tensor::f_row_stack(tensors).unwrap() + } + + pub fn row_stack_out>(out: &Tensor, tensors: &[T]) -> Tensor { + Tensor::f_row_stack_out(out, tensors).unwrap() + } + + pub fn rrelu(&self, training: bool) -> Tensor { + self.f_rrelu(training).unwrap() + } + + pub fn rrelu_(&mut self, training: bool) -> Tensor { + self.f_rrelu_(training).unwrap() + } + + pub fn rrelu_with_noise(&self, noise: &Tensor, training: bool) -> Tensor { + self.f_rrelu_with_noise(noise, training).unwrap() + } + + pub fn rrelu_with_noise_(&mut self, noise: &Tensor, training: bool) -> Tensor { + self.f_rrelu_with_noise_(noise, training).unwrap() + } + + pub fn rrelu_with_noise_backward>( + &self, + grad_output: &Tensor, + noise: &Tensor, + lower: S, + upper: S, + training: bool, + self_is_result: bool, + ) -> Tensor { + self.f_rrelu_with_noise_backward(grad_output, noise, lower, upper, training, self_is_result) + .unwrap() + } + + pub fn rrelu_with_noise_backward_out>( + &self, + out: &Tensor, + grad_output: &Tensor, + noise: &Tensor, + lower: S, + upper: S, + training: bool, + self_is_result: bool, + ) -> Tensor { + self.f_rrelu_with_noise_backward_out( + out, + grad_output, + noise, + lower, + upper, + training, + self_is_result, + ) + .unwrap() + } + + pub fn rrelu_with_noise_functional(&self, noise: &Tensor, training: bool) -> (Tensor, Tensor) { + self.f_rrelu_with_noise_functional(noise, training).unwrap() + } + + pub fn rrelu_with_noise_out(&self, out: &Tensor, noise: &Tensor, training: bool) -> Tensor { + self.f_rrelu_with_noise_out(out, noise, training).unwrap() + } + + pub fn rsqrt(&self) -> Tensor { + self.f_rsqrt().unwrap() + } + + pub fn rsqrt_(&mut self) -> Tensor { + self.f_rsqrt_().unwrap() + } + + pub fn rsqrt_out(&self, out: &Tensor) -> Tensor { + self.f_rsqrt_out(out).unwrap() + } + + pub fn rsub(&self, other: &Tensor) -> Tensor { + self.f_rsub(other).unwrap() + } + + pub fn rsub_scalar>(&self, other: S) -> Tensor { + self.f_rsub_scalar(other).unwrap() + } + + pub fn rsub_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_rsub_scalar_out(out, other).unwrap() + } + + pub fn rsub_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_rsub_tensor_out(out, other).unwrap() + } + + pub fn scalar_tensor>(s: S, options: (Kind, Device)) -> Tensor { + Tensor::f_scalar_tensor(s, options).unwrap() + } + + pub fn scalar_tensor_out>(out: &Tensor, s: S) -> Tensor { + Tensor::f_scalar_tensor_out(out, s).unwrap() + } + + pub fn scaled_dot_product_attention>( + query: &Tensor, + key: &Tensor, + value: &Tensor, + attn_mask: Option, + dropout_p: f64, + is_causal: bool, + scale: impl Into>, + enable_gqa: bool, + ) -> Tensor { + Tensor::f_scaled_dot_product_attention( + query, key, value, attn_mask, dropout_p, is_causal, scale, enable_gqa, + ) + .unwrap() + } + + pub fn scatter(&self, dim: i64, index: &Tensor, src: &Tensor) -> Tensor { + self.f_scatter(dim, index, src).unwrap() + } + + pub fn scatter_(&mut self, dim: i64, index: &Tensor, src: &Tensor) -> Tensor { + self.f_scatter_(dim, index, src).unwrap() + } + + pub fn scatter_add(&self, dim: i64, index: &Tensor, src: &Tensor) -> Tensor { + self.f_scatter_add(dim, index, src).unwrap() + } + + pub fn scatter_add_(&mut self, dim: i64, index: &Tensor, src: &Tensor) -> Tensor { + self.f_scatter_add_(dim, index, src).unwrap() + } + + pub fn scatter_add_out(&self, out: &Tensor, dim: i64, index: &Tensor, src: &Tensor) -> Tensor { + self.f_scatter_add_out(out, dim, index, src).unwrap() + } + + pub fn scatter_reduce(&self, dim: i64, index: &Tensor, src: &Tensor, reduce: &str) -> Tensor { + self.f_scatter_reduce(dim, index, src, reduce).unwrap() + } + + pub fn scatter_reduce_( + &mut self, + dim: i64, + index: &Tensor, + src: &Tensor, + reduce: &str, + ) -> Tensor { + self.f_scatter_reduce_(dim, index, src, reduce).unwrap() + } + + pub fn scatter_reduce_out( + &self, + out: &Tensor, + dim: i64, + index: &Tensor, + src: &Tensor, + reduce: &str, + ) -> Tensor { + self.f_scatter_reduce_out(out, dim, index, src, reduce).unwrap() + } + + pub fn scatter_src_out(&self, out: &Tensor, dim: i64, index: &Tensor, src: &Tensor) -> Tensor { + self.f_scatter_src_out(out, dim, index, src).unwrap() + } + + pub fn scatter_value>(&self, dim: i64, index: &Tensor, value: S) -> Tensor { + self.f_scatter_value(dim, index, value).unwrap() + } + + pub fn scatter_value_>( + &mut self, + dim: i64, + index: &Tensor, + value: S, + ) -> Tensor { + self.f_scatter_value_(dim, index, value).unwrap() + } + + pub fn scatter_value_out>( + &self, + out: &Tensor, + dim: i64, + index: &Tensor, + value: S, + ) -> Tensor { + self.f_scatter_value_out(out, dim, index, value).unwrap() + } + + pub fn scatter_value_reduce>( + &self, + dim: i64, + index: &Tensor, + value: S, + reduce: &str, + ) -> Tensor { + self.f_scatter_value_reduce(dim, index, value, reduce).unwrap() + } + + pub fn scatter_value_reduce_>( + &mut self, + dim: i64, + index: &Tensor, + value: S, + reduce: &str, + ) -> Tensor { + self.f_scatter_value_reduce_(dim, index, value, reduce).unwrap() + } + + pub fn scatter_value_reduce_out>( + &self, + out: &Tensor, + dim: i64, + index: &Tensor, + value: S, + reduce: &str, + ) -> Tensor { + self.f_scatter_value_reduce_out(out, dim, index, value, reduce).unwrap() + } + + pub fn searchsorted>( + &self, + sorted_sequence: &Tensor, + out_int32: bool, + right: bool, + side: &str, + sorter: Option, + ) -> Tensor { + self.f_searchsorted(sorted_sequence, out_int32, right, side, sorter).unwrap() + } + + pub fn searchsorted_scalar, S: Into>( + sorted_sequence: &Tensor, + self_scalar: S, + out_int32: bool, + right: bool, + side: &str, + sorter: Option, + ) -> Tensor { + Tensor::f_searchsorted_scalar(sorted_sequence, self_scalar, out_int32, right, side, sorter) + .unwrap() + } + + pub fn searchsorted_scalar_out, S: Into>( + out: &Tensor, + sorted_sequence: &Tensor, + self_scalar: S, + out_int32: bool, + right: bool, + side: &str, + sorter: Option, + ) -> Tensor { + Tensor::f_searchsorted_scalar_out( + out, + sorted_sequence, + self_scalar, + out_int32, + right, + side, + sorter, + ) + .unwrap() + } + + pub fn searchsorted_tensor_out>( + &self, + out: &Tensor, + sorted_sequence: &Tensor, + out_int32: bool, + right: bool, + side: &str, + sorter: Option, + ) -> Tensor { + self.f_searchsorted_tensor_out(out, sorted_sequence, out_int32, right, side, sorter) + .unwrap() + } + + pub fn segment_reduce, S: Into>( + data: &Tensor, + reduce: &str, + lengths: Option, + indices: Option, + offsets: Option, + axis: i64, + unsafe_: bool, + initial: S, + ) -> Tensor { + Tensor::f_segment_reduce(data, reduce, lengths, indices, offsets, axis, unsafe_, initial) + .unwrap() + } + + pub fn segment_reduce_out, S: Into>( + out: &Tensor, + data: &Tensor, + reduce: &str, + lengths: Option, + indices: Option, + offsets: Option, + axis: i64, + unsafe_: bool, + initial: S, + ) -> Tensor { + Tensor::f_segment_reduce_out( + out, data, reduce, lengths, indices, offsets, axis, unsafe_, initial, + ) + .unwrap() + } + + pub fn select(&self, dim: i64, index: i64) -> Tensor { + self.f_select(dim, index).unwrap() + } + + pub fn select_backward( + grad_output: &Tensor, + input_sizes: impl IntList, + dim: i64, + index: i64, + ) -> Tensor { + Tensor::f_select_backward(grad_output, input_sizes, dim, index).unwrap() + } + + pub fn select_backward_out( + out: &Tensor, + grad_output: &Tensor, + input_sizes: impl IntList, + dim: i64, + index: i64, + ) -> Tensor { + Tensor::f_select_backward_out(out, grad_output, input_sizes, dim, index).unwrap() + } + + pub fn select_copy(&self, dim: i64, index: i64) -> Tensor { + self.f_select_copy(dim, index).unwrap() + } + + pub fn select_copy_int_out(&self, out: &Tensor, dim: i64, index: i64) -> Tensor { + self.f_select_copy_int_out(out, dim, index).unwrap() + } + + pub fn select_scatter(&self, src: &Tensor, dim: i64, index: i64) -> Tensor { + self.f_select_scatter(src, dim, index).unwrap() + } + + pub fn select_scatter_out(&self, out: &Tensor, src: &Tensor, dim: i64, index: i64) -> Tensor { + self.f_select_scatter_out(out, src, dim, index).unwrap() + } + + pub fn selu(&self) -> Tensor { + self.f_selu().unwrap() + } + + pub fn selu_(&mut self) -> Tensor { + self.f_selu_().unwrap() + } + + pub fn set(&self) -> Tensor { + self.f_set().unwrap() + } + + pub fn set_(&mut self) -> Tensor { + self.f_set_().unwrap() + } + + pub fn set_data(&mut self, new_data: &Tensor) { + self.f_set_data(new_data).unwrap() + } + + pub fn set_out(&self, out: &Tensor) -> Tensor { + self.f_set_out(out).unwrap() + } + + pub fn set_requires_grad(&self, r: bool) -> Tensor { + self.f_set_requires_grad(r).unwrap() + } + + pub fn set_source_tensor(&self, source: &Tensor) -> Tensor { + self.f_set_source_tensor(source).unwrap() + } + + pub fn set_source_tensor_(&mut self, source: &Tensor) -> Tensor { + self.f_set_source_tensor_(source).unwrap() + } + + pub fn set_source_tensor_out(&self, out: &Tensor, source: &Tensor) -> Tensor { + self.f_set_source_tensor_out(out, source).unwrap() + } + + pub fn set_source_tensor_storage_offset_( + &mut self, + source: &Tensor, + storage_offset: i64, + size: impl IntList, + stride: impl IntList, + ) -> Tensor { + self.f_set_source_tensor_storage_offset_(source, storage_offset, size, stride).unwrap() + } + + pub fn sgn(&self) -> Tensor { + self.f_sgn().unwrap() + } + + pub fn sgn_(&mut self) -> Tensor { + self.f_sgn_().unwrap() + } + + pub fn sgn_out(&self, out: &Tensor) -> Tensor { + self.f_sgn_out(out).unwrap() + } + + pub fn sigmoid(&self) -> Tensor { + self.f_sigmoid().unwrap() + } + + pub fn sigmoid_(&mut self) -> Tensor { + self.f_sigmoid_().unwrap() + } + + pub fn sigmoid_backward(grad_output: &Tensor, output: &Tensor) -> Tensor { + Tensor::f_sigmoid_backward(grad_output, output).unwrap() + } + + pub fn sigmoid_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output: &Tensor, + ) -> Tensor { + Tensor::f_sigmoid_backward_grad_input(grad_input, grad_output, output).unwrap() + } + + pub fn sigmoid_out(&self, out: &Tensor) -> Tensor { + self.f_sigmoid_out(out).unwrap() + } + + pub fn sign(&self) -> Tensor { + self.f_sign().unwrap() + } + + pub fn sign_(&mut self) -> Tensor { + self.f_sign_().unwrap() + } + + pub fn sign_out(&self, out: &Tensor) -> Tensor { + self.f_sign_out(out).unwrap() + } + + pub fn signbit(&self) -> Tensor { + self.f_signbit().unwrap() + } + + pub fn signbit_out(&self, out: &Tensor) -> Tensor { + self.f_signbit_out(out).unwrap() + } + + pub fn silu(&self) -> Tensor { + self.f_silu().unwrap() + } + + pub fn silu_(&mut self) -> Tensor { + self.f_silu_().unwrap() + } + + pub fn silu_backward(&self, grad_output: &Tensor) -> Tensor { + self.f_silu_backward(grad_output).unwrap() + } + + pub fn silu_backward_grad_input(&self, grad_input: &Tensor, grad_output: &Tensor) -> Tensor { + self.f_silu_backward_grad_input(grad_input, grad_output).unwrap() + } + + pub fn silu_out(&self, out: &Tensor) -> Tensor { + self.f_silu_out(out).unwrap() + } + + pub fn sin(&self) -> Tensor { + self.f_sin().unwrap() + } + + pub fn sin_(&mut self) -> Tensor { + self.f_sin_().unwrap() + } + + pub fn sin_out(&self, out: &Tensor) -> Tensor { + self.f_sin_out(out).unwrap() + } + + pub fn sinc(&self) -> Tensor { + self.f_sinc().unwrap() + } + + pub fn sinc_(&mut self) -> Tensor { + self.f_sinc_().unwrap() + } + + pub fn sinc_out(&self, out: &Tensor) -> Tensor { + self.f_sinc_out(out).unwrap() + } + + pub fn sinh(&self) -> Tensor { + self.f_sinh().unwrap() + } + + pub fn sinh_(&mut self) -> Tensor { + self.f_sinh_().unwrap() + } + + pub fn sinh_out(&self, out: &Tensor) -> Tensor { + self.f_sinh_out(out).unwrap() + } + + pub fn slice( + &self, + dim: i64, + start: impl Into>, + end: impl Into>, + step: i64, + ) -> Tensor { + self.f_slice(dim, start, end, step).unwrap() + } + + pub fn slice_backward( + grad_output: &Tensor, + input_sizes: impl IntList, + dim: i64, + start: i64, + end: i64, + step: i64, + ) -> Tensor { + Tensor::f_slice_backward(grad_output, input_sizes, dim, start, end, step).unwrap() + } + + pub fn slice_backward_out( + out: &Tensor, + grad_output: &Tensor, + input_sizes: impl IntList, + dim: i64, + start: i64, + end: i64, + step: i64, + ) -> Tensor { + Tensor::f_slice_backward_out(out, grad_output, input_sizes, dim, start, end, step).unwrap() + } + + pub fn slice_copy( + &self, + dim: i64, + start: impl Into>, + end: impl Into>, + step: i64, + ) -> Tensor { + self.f_slice_copy(dim, start, end, step).unwrap() + } + + pub fn slice_copy_tensor_out( + &self, + out: &Tensor, + dim: i64, + start: impl Into>, + end: impl Into>, + step: i64, + ) -> Tensor { + self.f_slice_copy_tensor_out(out, dim, start, end, step).unwrap() + } + + pub fn slice_inverse( + &self, + src: &Tensor, + dim: i64, + start: impl Into>, + end: impl Into>, + step: i64, + ) -> Tensor { + self.f_slice_inverse(src, dim, start, end, step).unwrap() + } + + pub fn slice_scatter( + &self, + src: &Tensor, + dim: i64, + start: impl Into>, + end: impl Into>, + step: i64, + ) -> Tensor { + self.f_slice_scatter(src, dim, start, end, step).unwrap() + } + + pub fn slice_scatter_out( + &self, + out: &Tensor, + src: &Tensor, + dim: i64, + start: impl Into>, + end: impl Into>, + step: i64, + ) -> Tensor { + self.f_slice_scatter_out(out, src, dim, start, end, step).unwrap() + } + + pub fn slogdet(&self) -> (Tensor, Tensor) { + self.f_slogdet().unwrap() + } + + pub fn slogdet_out(&self, sign: &Tensor, logabsdet: &Tensor) -> (Tensor, Tensor) { + self.f_slogdet_out(sign, logabsdet).unwrap() + } + + pub fn slow_conv3d>( + &self, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + ) -> Tensor { + self.f_slow_conv3d(weight, kernel_size, bias, stride, padding).unwrap() + } + + pub fn slow_conv3d_out>( + &self, + out: &Tensor, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + ) -> Tensor { + self.f_slow_conv3d_out(out, weight, kernel_size, bias, stride, padding).unwrap() + } + + pub fn slow_conv_dilated2d>( + &self, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ) -> Tensor { + self.f_slow_conv_dilated2d(weight, kernel_size, bias, stride, padding, dilation).unwrap() + } + + pub fn slow_conv_dilated2d_out>( + &self, + out: &Tensor, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ) -> Tensor { + self.f_slow_conv_dilated2d_out(out, weight, kernel_size, bias, stride, padding, dilation) + .unwrap() + } + + pub fn slow_conv_dilated3d>( + &self, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ) -> Tensor { + self.f_slow_conv_dilated3d(weight, kernel_size, bias, stride, padding, dilation).unwrap() + } + + pub fn slow_conv_dilated3d_out>( + &self, + out: &Tensor, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + dilation: impl IntList, + ) -> Tensor { + self.f_slow_conv_dilated3d_out(out, weight, kernel_size, bias, stride, padding, dilation) + .unwrap() + } + + pub fn slow_conv_transpose2d>( + &self, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + output_padding: impl IntList, + dilation: impl IntList, + ) -> Tensor { + self.f_slow_conv_transpose2d( + weight, + kernel_size, + bias, + stride, + padding, + output_padding, + dilation, + ) + .unwrap() + } + + pub fn slow_conv_transpose2d_out>( + &self, + out: &Tensor, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + output_padding: impl IntList, + dilation: impl IntList, + ) -> Tensor { + self.f_slow_conv_transpose2d_out( + out, + weight, + kernel_size, + bias, + stride, + padding, + output_padding, + dilation, + ) + .unwrap() + } + + pub fn slow_conv_transpose3d>( + &self, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + output_padding: impl IntList, + dilation: impl IntList, + ) -> Tensor { + self.f_slow_conv_transpose3d( + weight, + kernel_size, + bias, + stride, + padding, + output_padding, + dilation, + ) + .unwrap() + } + + pub fn slow_conv_transpose3d_out>( + &self, + out: &Tensor, + weight: &Tensor, + kernel_size: impl IntList, + bias: Option, + stride: impl IntList, + padding: impl IntList, + output_padding: impl IntList, + dilation: impl IntList, + ) -> Tensor { + self.f_slow_conv_transpose3d_out( + out, + weight, + kernel_size, + bias, + stride, + padding, + output_padding, + dilation, + ) + .unwrap() + } + + pub fn smm(&self, mat2: &Tensor) -> Tensor { + self.f_smm(mat2).unwrap() + } + + pub fn smooth_l1_loss( + &self, + target: &Tensor, + reduction: crate::Reduction, + beta: f64, + ) -> Tensor { + self.f_smooth_l1_loss(target, reduction, beta).unwrap() + } + + pub fn smooth_l1_loss_backward( + &self, + grad_output: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + beta: f64, + ) -> Tensor { + self.f_smooth_l1_loss_backward(grad_output, target, reduction, beta).unwrap() + } + + pub fn smooth_l1_loss_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + beta: f64, + ) -> Tensor { + self.f_smooth_l1_loss_backward_grad_input(grad_input, grad_output, target, reduction, beta) + .unwrap() + } + + pub fn smooth_l1_loss_out( + &self, + out: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + beta: f64, + ) -> Tensor { + self.f_smooth_l1_loss_out(out, target, reduction, beta).unwrap() + } + + pub fn soft_margin_loss(&self, target: &Tensor, reduction: crate::Reduction) -> Tensor { + self.f_soft_margin_loss(target, reduction).unwrap() + } + + pub fn soft_margin_loss_backward( + &self, + grad_output: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + ) -> Tensor { + self.f_soft_margin_loss_backward(grad_output, target, reduction).unwrap() + } + + pub fn soft_margin_loss_backward_grad_input( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + ) -> Tensor { + self.f_soft_margin_loss_backward_grad_input(grad_input, grad_output, target, reduction) + .unwrap() + } + + pub fn soft_margin_loss_out( + &self, + out: &Tensor, + target: &Tensor, + reduction: crate::Reduction, + ) -> Tensor { + self.f_soft_margin_loss_out(out, target, reduction).unwrap() + } + + pub fn softmax(&self, dim: i64, dtype: impl Into>) -> Tensor { + self.f_softmax(dim, dtype).unwrap() + } + + pub fn softmax_int_out( + &self, + out: &Tensor, + dim: i64, + dtype: impl Into>, + ) -> Tensor { + self.f_softmax_int_out(out, dim, dtype).unwrap() + } + + pub fn softplus(&self) -> Tensor { + self.f_softplus().unwrap() + } + + pub fn softplus_backward>( + &self, + grad_output: &Tensor, + beta: S, + threshold: S, + ) -> Tensor { + self.f_softplus_backward(grad_output, beta, threshold).unwrap() + } + + pub fn softplus_backward_grad_input>( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + beta: S, + threshold: S, + ) -> Tensor { + self.f_softplus_backward_grad_input(grad_input, grad_output, beta, threshold).unwrap() + } + + pub fn softplus_out(&self, out: &Tensor) -> Tensor { + self.f_softplus_out(out).unwrap() + } + + pub fn softshrink(&self) -> Tensor { + self.f_softshrink().unwrap() + } + + pub fn softshrink_backward>(&self, grad_output: &Tensor, lambd: S) -> Tensor { + self.f_softshrink_backward(grad_output, lambd).unwrap() + } + + pub fn softshrink_backward_grad_input>( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + lambd: S, + ) -> Tensor { + self.f_softshrink_backward_grad_input(grad_input, grad_output, lambd).unwrap() + } + + pub fn softshrink_out(&self, out: &Tensor) -> Tensor { + self.f_softshrink_out(out).unwrap() + } + + pub fn sort(&self, dim: i64, descending: bool) -> (Tensor, Tensor) { + self.f_sort(dim, descending).unwrap() + } + + pub fn sort_stable(&self, stable: bool, dim: i64, descending: bool) -> (Tensor, Tensor) { + self.f_sort_stable(stable, dim, descending).unwrap() + } + + pub fn sort_values( + &self, + values: &Tensor, + indices: &Tensor, + dim: i64, + descending: bool, + ) -> (Tensor, Tensor) { + self.f_sort_values(values, indices, dim, descending).unwrap() + } + + pub fn sort_values_stable( + &self, + values: &Tensor, + indices: &Tensor, + stable: bool, + dim: i64, + descending: bool, + ) -> (Tensor, Tensor) { + self.f_sort_values_stable(values, indices, stable, dim, descending).unwrap() + } + + pub fn sparse_bsc_tensor( + ccol_indices: &Tensor, + row_indices: &Tensor, + values: &Tensor, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_sparse_bsc_tensor(ccol_indices, row_indices, values, options).unwrap() + } + + pub fn sparse_bsc_tensor_ccol_row_value_size( + ccol_indices: &Tensor, + row_indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_sparse_bsc_tensor_ccol_row_value_size( + ccol_indices, + row_indices, + values, + size, + options, + ) + .unwrap() + } + + pub fn sparse_bsr_tensor( + crow_indices: &Tensor, + col_indices: &Tensor, + values: &Tensor, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_sparse_bsr_tensor(crow_indices, col_indices, values, options).unwrap() + } + + pub fn sparse_bsr_tensor_crow_col_value_size( + crow_indices: &Tensor, + col_indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_sparse_bsr_tensor_crow_col_value_size( + crow_indices, + col_indices, + values, + size, + options, + ) + .unwrap() + } + + pub fn sparse_compressed_tensor( + compressed_indices: &Tensor, + plain_indices: &Tensor, + values: &Tensor, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_sparse_compressed_tensor(compressed_indices, plain_indices, values, options) + .unwrap() + } + + pub fn sparse_compressed_tensor_comp_plain_value_size( + compressed_indices: &Tensor, + plain_indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_sparse_compressed_tensor_comp_plain_value_size( + compressed_indices, + plain_indices, + values, + size, + options, + ) + .unwrap() + } + + pub fn sparse_coo_tensor(size: impl IntList, options: (Kind, Device)) -> Tensor { + Tensor::f_sparse_coo_tensor(size, options).unwrap() + } + + pub fn sparse_coo_tensor_indices( + indices: &Tensor, + values: &Tensor, + options: (Kind, Device), + is_coalesced: bool, + ) -> Tensor { + Tensor::f_sparse_coo_tensor_indices(indices, values, options, is_coalesced).unwrap() + } + + pub fn sparse_coo_tensor_indices_size( + indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + is_coalesced: bool, + ) -> Tensor { + Tensor::f_sparse_coo_tensor_indices_size(indices, values, size, options, is_coalesced) + .unwrap() + } + + pub fn sparse_coo_tensor_size_out(out: &Tensor, size: impl IntList) -> Tensor { + Tensor::f_sparse_coo_tensor_size_out(out, size).unwrap() + } + + pub fn sparse_csc_tensor( + ccol_indices: &Tensor, + row_indices: &Tensor, + values: &Tensor, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_sparse_csc_tensor(ccol_indices, row_indices, values, options).unwrap() + } + + pub fn sparse_csc_tensor_ccol_row_value_size( + ccol_indices: &Tensor, + row_indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_sparse_csc_tensor_ccol_row_value_size( + ccol_indices, + row_indices, + values, + size, + options, + ) + .unwrap() + } + + pub fn sparse_csr_tensor( + crow_indices: &Tensor, + col_indices: &Tensor, + values: &Tensor, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_sparse_csr_tensor(crow_indices, col_indices, values, options).unwrap() + } + + pub fn sparse_csr_tensor_crow_col_value_size( + crow_indices: &Tensor, + col_indices: &Tensor, + values: &Tensor, + size: impl IntList, + options: (Kind, Device), + ) -> Tensor { + Tensor::f_sparse_csr_tensor_crow_col_value_size( + crow_indices, + col_indices, + values, + size, + options, + ) + .unwrap() + } + + pub fn sparse_dim(&self) -> i64 { + self.f_sparse_dim().unwrap() + } + + pub fn sparse_mask(&self, mask: &Tensor) -> Tensor { + self.f_sparse_mask(mask).unwrap() + } + + pub fn sparse_mask_out(&self, out: &Tensor, mask: &Tensor) -> Tensor { + self.f_sparse_mask_out(out, mask).unwrap() + } + + pub fn sparse_resize(&self, size: impl IntList, sparse_dim: i64, dense_dim: i64) -> Tensor { + self.f_sparse_resize(size, sparse_dim, dense_dim).unwrap() + } + + pub fn sparse_resize_( + &mut self, + size: impl IntList, + sparse_dim: i64, + dense_dim: i64, + ) -> Tensor { + self.f_sparse_resize_(size, sparse_dim, dense_dim).unwrap() + } + + pub fn sparse_resize_and_clear( + &self, + size: impl IntList, + sparse_dim: i64, + dense_dim: i64, + ) -> Tensor { + self.f_sparse_resize_and_clear(size, sparse_dim, dense_dim).unwrap() + } + + pub fn sparse_resize_and_clear_( + &mut self, + size: impl IntList, + sparse_dim: i64, + dense_dim: i64, + ) -> Tensor { + self.f_sparse_resize_and_clear_(size, sparse_dim, dense_dim).unwrap() + } + + pub fn sparse_resize_and_clear_out( + &self, + out: &Tensor, + size: impl IntList, + sparse_dim: i64, + dense_dim: i64, + ) -> Tensor { + self.f_sparse_resize_and_clear_out(out, size, sparse_dim, dense_dim).unwrap() + } + + pub fn sparse_resize_out( + &self, + out: &Tensor, + size: impl IntList, + sparse_dim: i64, + dense_dim: i64, + ) -> Tensor { + self.f_sparse_resize_out(out, size, sparse_dim, dense_dim).unwrap() + } + + pub fn sparse_sampled_addmm(&self, mat1: &Tensor, mat2: &Tensor) -> Tensor { + self.f_sparse_sampled_addmm(mat1, mat2).unwrap() + } + + pub fn sparse_sampled_addmm_out(&self, out: &Tensor, mat1: &Tensor, mat2: &Tensor) -> Tensor { + self.f_sparse_sampled_addmm_out(out, mat1, mat2).unwrap() + } + + pub fn special_airy_ai(x: &Tensor) -> Tensor { + Tensor::f_special_airy_ai(x).unwrap() + } + + pub fn special_airy_ai_out(out: &Tensor, x: &Tensor) -> Tensor { + Tensor::f_special_airy_ai_out(out, x).unwrap() + } + + pub fn special_bessel_j0(&self) -> Tensor { + self.f_special_bessel_j0().unwrap() + } + + pub fn special_bessel_j0_out(&self, out: &Tensor) -> Tensor { + self.f_special_bessel_j0_out(out).unwrap() + } + + pub fn special_bessel_j1(&self) -> Tensor { + self.f_special_bessel_j1().unwrap() + } + + pub fn special_bessel_j1_out(&self, out: &Tensor) -> Tensor { + self.f_special_bessel_j1_out(out).unwrap() + } + + pub fn special_bessel_y0(&self) -> Tensor { + self.f_special_bessel_y0().unwrap() + } + + pub fn special_bessel_y0_out(&self, out: &Tensor) -> Tensor { + self.f_special_bessel_y0_out(out).unwrap() + } + + pub fn special_bessel_y1(&self) -> Tensor { + self.f_special_bessel_y1().unwrap() + } + + pub fn special_bessel_y1_out(&self, out: &Tensor) -> Tensor { + self.f_special_bessel_y1_out(out).unwrap() + } + + pub fn special_chebyshev_polynomial_t(x: &Tensor, n: &Tensor) -> Tensor { + Tensor::f_special_chebyshev_polynomial_t(x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_t_n_scalar>(x: &Tensor, n: S) -> Tensor { + Tensor::f_special_chebyshev_polynomial_t_n_scalar(x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_t_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Tensor { + Tensor::f_special_chebyshev_polynomial_t_n_scalar_out(out, x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_t_out(out: &Tensor, x: &Tensor, n: &Tensor) -> Tensor { + Tensor::f_special_chebyshev_polynomial_t_out(out, x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_t_x_scalar>(x: S, n: &Tensor) -> Tensor { + Tensor::f_special_chebyshev_polynomial_t_x_scalar(x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_t_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Tensor { + Tensor::f_special_chebyshev_polynomial_t_x_scalar_out(out, x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_u(x: &Tensor, n: &Tensor) -> Tensor { + Tensor::f_special_chebyshev_polynomial_u(x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_u_n_scalar>(x: &Tensor, n: S) -> Tensor { + Tensor::f_special_chebyshev_polynomial_u_n_scalar(x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_u_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Tensor { + Tensor::f_special_chebyshev_polynomial_u_n_scalar_out(out, x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_u_out(out: &Tensor, x: &Tensor, n: &Tensor) -> Tensor { + Tensor::f_special_chebyshev_polynomial_u_out(out, x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_u_x_scalar>(x: S, n: &Tensor) -> Tensor { + Tensor::f_special_chebyshev_polynomial_u_x_scalar(x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_u_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Tensor { + Tensor::f_special_chebyshev_polynomial_u_x_scalar_out(out, x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_v(x: &Tensor, n: &Tensor) -> Tensor { + Tensor::f_special_chebyshev_polynomial_v(x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_v_n_scalar>(x: &Tensor, n: S) -> Tensor { + Tensor::f_special_chebyshev_polynomial_v_n_scalar(x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_v_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Tensor { + Tensor::f_special_chebyshev_polynomial_v_n_scalar_out(out, x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_v_out(out: &Tensor, x: &Tensor, n: &Tensor) -> Tensor { + Tensor::f_special_chebyshev_polynomial_v_out(out, x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_v_x_scalar>(x: S, n: &Tensor) -> Tensor { + Tensor::f_special_chebyshev_polynomial_v_x_scalar(x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_v_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Tensor { + Tensor::f_special_chebyshev_polynomial_v_x_scalar_out(out, x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_w(x: &Tensor, n: &Tensor) -> Tensor { + Tensor::f_special_chebyshev_polynomial_w(x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_w_n_scalar>(x: &Tensor, n: S) -> Tensor { + Tensor::f_special_chebyshev_polynomial_w_n_scalar(x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_w_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Tensor { + Tensor::f_special_chebyshev_polynomial_w_n_scalar_out(out, x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_w_out(out: &Tensor, x: &Tensor, n: &Tensor) -> Tensor { + Tensor::f_special_chebyshev_polynomial_w_out(out, x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_w_x_scalar>(x: S, n: &Tensor) -> Tensor { + Tensor::f_special_chebyshev_polynomial_w_x_scalar(x, n).unwrap() + } + + pub fn special_chebyshev_polynomial_w_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Tensor { + Tensor::f_special_chebyshev_polynomial_w_x_scalar_out(out, x, n).unwrap() + } + + pub fn special_digamma(&self) -> Tensor { + self.f_special_digamma().unwrap() + } + + pub fn special_digamma_out(&self, out: &Tensor) -> Tensor { + self.f_special_digamma_out(out).unwrap() + } + + pub fn special_entr(&self) -> Tensor { + self.f_special_entr().unwrap() + } + + pub fn special_entr_out(&self, out: &Tensor) -> Tensor { + self.f_special_entr_out(out).unwrap() + } + + pub fn special_erf(&self) -> Tensor { + self.f_special_erf().unwrap() + } + + pub fn special_erf_out(&self, out: &Tensor) -> Tensor { + self.f_special_erf_out(out).unwrap() + } + + pub fn special_erfc(&self) -> Tensor { + self.f_special_erfc().unwrap() + } + + pub fn special_erfc_out(&self, out: &Tensor) -> Tensor { + self.f_special_erfc_out(out).unwrap() + } + + pub fn special_erfcx(&self) -> Tensor { + self.f_special_erfcx().unwrap() + } + + pub fn special_erfcx_out(&self, out: &Tensor) -> Tensor { + self.f_special_erfcx_out(out).unwrap() + } + + pub fn special_erfinv(&self) -> Tensor { + self.f_special_erfinv().unwrap() + } + + pub fn special_erfinv_out(&self, out: &Tensor) -> Tensor { + self.f_special_erfinv_out(out).unwrap() + } + + pub fn special_exp2(&self) -> Tensor { + self.f_special_exp2().unwrap() + } + + pub fn special_exp2_out(&self, out: &Tensor) -> Tensor { + self.f_special_exp2_out(out).unwrap() + } + + pub fn special_expit(&self) -> Tensor { + self.f_special_expit().unwrap() + } + + pub fn special_expit_out(&self, out: &Tensor) -> Tensor { + self.f_special_expit_out(out).unwrap() + } + + pub fn special_expm1(&self) -> Tensor { + self.f_special_expm1().unwrap() + } + + pub fn special_expm1_out(&self, out: &Tensor) -> Tensor { + self.f_special_expm1_out(out).unwrap() + } + + pub fn special_gammainc(&self, other: &Tensor) -> Tensor { + self.f_special_gammainc(other).unwrap() + } + + pub fn special_gammainc_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_special_gammainc_out(out, other).unwrap() + } + + pub fn special_gammaincc(&self, other: &Tensor) -> Tensor { + self.f_special_gammaincc(other).unwrap() + } + + pub fn special_gammaincc_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_special_gammaincc_out(out, other).unwrap() + } + + pub fn special_gammaln(&self) -> Tensor { + self.f_special_gammaln().unwrap() + } + + pub fn special_gammaln_out(&self, out: &Tensor) -> Tensor { + self.f_special_gammaln_out(out).unwrap() + } + + pub fn special_hermite_polynomial_h(x: &Tensor, n: &Tensor) -> Tensor { + Tensor::f_special_hermite_polynomial_h(x, n).unwrap() + } + + pub fn special_hermite_polynomial_h_n_scalar>(x: &Tensor, n: S) -> Tensor { + Tensor::f_special_hermite_polynomial_h_n_scalar(x, n).unwrap() + } + + pub fn special_hermite_polynomial_h_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Tensor { + Tensor::f_special_hermite_polynomial_h_n_scalar_out(out, x, n).unwrap() + } + + pub fn special_hermite_polynomial_h_out(out: &Tensor, x: &Tensor, n: &Tensor) -> Tensor { + Tensor::f_special_hermite_polynomial_h_out(out, x, n).unwrap() + } + + pub fn special_hermite_polynomial_h_x_scalar>(x: S, n: &Tensor) -> Tensor { + Tensor::f_special_hermite_polynomial_h_x_scalar(x, n).unwrap() + } + + pub fn special_hermite_polynomial_h_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Tensor { + Tensor::f_special_hermite_polynomial_h_x_scalar_out(out, x, n).unwrap() + } + + pub fn special_hermite_polynomial_he(x: &Tensor, n: &Tensor) -> Tensor { + Tensor::f_special_hermite_polynomial_he(x, n).unwrap() + } + + pub fn special_hermite_polynomial_he_n_scalar>(x: &Tensor, n: S) -> Tensor { + Tensor::f_special_hermite_polynomial_he_n_scalar(x, n).unwrap() + } + + pub fn special_hermite_polynomial_he_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Tensor { + Tensor::f_special_hermite_polynomial_he_n_scalar_out(out, x, n).unwrap() + } + + pub fn special_hermite_polynomial_he_out(out: &Tensor, x: &Tensor, n: &Tensor) -> Tensor { + Tensor::f_special_hermite_polynomial_he_out(out, x, n).unwrap() + } + + pub fn special_hermite_polynomial_he_x_scalar>(x: S, n: &Tensor) -> Tensor { + Tensor::f_special_hermite_polynomial_he_x_scalar(x, n).unwrap() + } + + pub fn special_hermite_polynomial_he_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Tensor { + Tensor::f_special_hermite_polynomial_he_x_scalar_out(out, x, n).unwrap() + } + + pub fn special_i0(&self) -> Tensor { + self.f_special_i0().unwrap() + } + + pub fn special_i0_out(&self, out: &Tensor) -> Tensor { + self.f_special_i0_out(out).unwrap() + } + + pub fn special_i0e(&self) -> Tensor { + self.f_special_i0e().unwrap() + } + + pub fn special_i0e_out(&self, out: &Tensor) -> Tensor { + self.f_special_i0e_out(out).unwrap() + } + + pub fn special_i1(&self) -> Tensor { + self.f_special_i1().unwrap() + } + + pub fn special_i1_out(&self, out: &Tensor) -> Tensor { + self.f_special_i1_out(out).unwrap() + } + + pub fn special_i1e(&self) -> Tensor { + self.f_special_i1e().unwrap() + } + + pub fn special_i1e_out(&self, out: &Tensor) -> Tensor { + self.f_special_i1e_out(out).unwrap() + } + + pub fn special_laguerre_polynomial_l(x: &Tensor, n: &Tensor) -> Tensor { + Tensor::f_special_laguerre_polynomial_l(x, n).unwrap() + } + + pub fn special_laguerre_polynomial_l_n_scalar>(x: &Tensor, n: S) -> Tensor { + Tensor::f_special_laguerre_polynomial_l_n_scalar(x, n).unwrap() + } + + pub fn special_laguerre_polynomial_l_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Tensor { + Tensor::f_special_laguerre_polynomial_l_n_scalar_out(out, x, n).unwrap() + } + + pub fn special_laguerre_polynomial_l_out(out: &Tensor, x: &Tensor, n: &Tensor) -> Tensor { + Tensor::f_special_laguerre_polynomial_l_out(out, x, n).unwrap() + } + + pub fn special_laguerre_polynomial_l_x_scalar>(x: S, n: &Tensor) -> Tensor { + Tensor::f_special_laguerre_polynomial_l_x_scalar(x, n).unwrap() + } + + pub fn special_laguerre_polynomial_l_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Tensor { + Tensor::f_special_laguerre_polynomial_l_x_scalar_out(out, x, n).unwrap() + } + + pub fn special_legendre_polynomial_p(x: &Tensor, n: &Tensor) -> Tensor { + Tensor::f_special_legendre_polynomial_p(x, n).unwrap() + } + + pub fn special_legendre_polynomial_p_n_scalar>(x: &Tensor, n: S) -> Tensor { + Tensor::f_special_legendre_polynomial_p_n_scalar(x, n).unwrap() + } + + pub fn special_legendre_polynomial_p_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Tensor { + Tensor::f_special_legendre_polynomial_p_n_scalar_out(out, x, n).unwrap() + } + + pub fn special_legendre_polynomial_p_out(out: &Tensor, x: &Tensor, n: &Tensor) -> Tensor { + Tensor::f_special_legendre_polynomial_p_out(out, x, n).unwrap() + } + + pub fn special_legendre_polynomial_p_x_scalar>(x: S, n: &Tensor) -> Tensor { + Tensor::f_special_legendre_polynomial_p_x_scalar(x, n).unwrap() + } + + pub fn special_legendre_polynomial_p_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Tensor { + Tensor::f_special_legendre_polynomial_p_x_scalar_out(out, x, n).unwrap() + } + + pub fn special_log1p(&self) -> Tensor { + self.f_special_log1p().unwrap() + } + + pub fn special_log1p_out(&self, out: &Tensor) -> Tensor { + self.f_special_log1p_out(out).unwrap() + } + + pub fn special_log_ndtr(&self) -> Tensor { + self.f_special_log_ndtr().unwrap() + } + + pub fn special_log_ndtr_out(&self, out: &Tensor) -> Tensor { + self.f_special_log_ndtr_out(out).unwrap() + } + + pub fn special_log_softmax(&self, dim: i64, dtype: impl Into>) -> Tensor { + self.f_special_log_softmax(dim, dtype).unwrap() + } + + pub fn special_logit(&self, eps: impl Into>) -> Tensor { + self.f_special_logit(eps).unwrap() + } + + pub fn special_logit_out(&self, out: &Tensor, eps: impl Into>) -> Tensor { + self.f_special_logit_out(out, eps).unwrap() + } + + pub fn special_logsumexp(&self, dim: impl IntList, keepdim: bool) -> Tensor { + self.f_special_logsumexp(dim, keepdim).unwrap() + } + + pub fn special_logsumexp_out(&self, out: &Tensor, dim: impl IntList, keepdim: bool) -> Tensor { + self.f_special_logsumexp_out(out, dim, keepdim).unwrap() + } + + pub fn special_modified_bessel_i0(&self) -> Tensor { + self.f_special_modified_bessel_i0().unwrap() + } + + pub fn special_modified_bessel_i0_out(&self, out: &Tensor) -> Tensor { + self.f_special_modified_bessel_i0_out(out).unwrap() + } + + pub fn special_modified_bessel_i1(&self) -> Tensor { + self.f_special_modified_bessel_i1().unwrap() + } + + pub fn special_modified_bessel_i1_out(&self, out: &Tensor) -> Tensor { + self.f_special_modified_bessel_i1_out(out).unwrap() + } + + pub fn special_modified_bessel_k0(&self) -> Tensor { + self.f_special_modified_bessel_k0().unwrap() + } + + pub fn special_modified_bessel_k0_out(&self, out: &Tensor) -> Tensor { + self.f_special_modified_bessel_k0_out(out).unwrap() + } + + pub fn special_modified_bessel_k1(&self) -> Tensor { + self.f_special_modified_bessel_k1().unwrap() + } + + pub fn special_modified_bessel_k1_out(&self, out: &Tensor) -> Tensor { + self.f_special_modified_bessel_k1_out(out).unwrap() + } + + pub fn special_multigammaln(&self, p: i64) -> Tensor { + self.f_special_multigammaln(p).unwrap() + } + + pub fn special_multigammaln_out(&self, out: &Tensor, p: i64) -> Tensor { + self.f_special_multigammaln_out(out, p).unwrap() + } + + pub fn special_ndtr(&self) -> Tensor { + self.f_special_ndtr().unwrap() + } + + pub fn special_ndtr_out(&self, out: &Tensor) -> Tensor { + self.f_special_ndtr_out(out).unwrap() + } + + pub fn special_ndtri(&self) -> Tensor { + self.f_special_ndtri().unwrap() + } + + pub fn special_ndtri_out(&self, out: &Tensor) -> Tensor { + self.f_special_ndtri_out(out).unwrap() + } + + pub fn special_polygamma(&self, n: i64) -> Tensor { + self.f_special_polygamma(n).unwrap() + } + + pub fn special_polygamma_out(&self, out: &Tensor, n: i64) -> Tensor { + self.f_special_polygamma_out(out, n).unwrap() + } + + pub fn special_psi(&self) -> Tensor { + self.f_special_psi().unwrap() + } + + pub fn special_psi_out(&self, out: &Tensor) -> Tensor { + self.f_special_psi_out(out).unwrap() + } + + pub fn special_round(&self, decimals: i64) -> Tensor { + self.f_special_round(decimals).unwrap() + } + + pub fn special_round_out(&self, out: &Tensor, decimals: i64) -> Tensor { + self.f_special_round_out(out, decimals).unwrap() + } + + pub fn special_scaled_modified_bessel_k0(x: &Tensor) -> Tensor { + Tensor::f_special_scaled_modified_bessel_k0(x).unwrap() + } + + pub fn special_scaled_modified_bessel_k0_out(out: &Tensor, x: &Tensor) -> Tensor { + Tensor::f_special_scaled_modified_bessel_k0_out(out, x).unwrap() + } + + pub fn special_scaled_modified_bessel_k1(x: &Tensor) -> Tensor { + Tensor::f_special_scaled_modified_bessel_k1(x).unwrap() + } + + pub fn special_scaled_modified_bessel_k1_out(out: &Tensor, x: &Tensor) -> Tensor { + Tensor::f_special_scaled_modified_bessel_k1_out(out, x).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_t(x: &Tensor, n: &Tensor) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_t(x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_t_n_scalar>( + x: &Tensor, + n: S, + ) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_t_n_scalar(x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_t_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_t_n_scalar_out(out, x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_t_out( + out: &Tensor, + x: &Tensor, + n: &Tensor, + ) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_t_out(out, x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_t_x_scalar>( + x: S, + n: &Tensor, + ) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_t_x_scalar(x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_t_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_t_x_scalar_out(out, x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_u(x: &Tensor, n: &Tensor) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_u(x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_u_n_scalar>( + x: &Tensor, + n: S, + ) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_u_n_scalar(x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_u_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_u_n_scalar_out(out, x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_u_out( + out: &Tensor, + x: &Tensor, + n: &Tensor, + ) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_u_out(out, x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_u_x_scalar>( + x: S, + n: &Tensor, + ) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_u_x_scalar(x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_u_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_u_x_scalar_out(out, x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_v(x: &Tensor, n: &Tensor) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_v(x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_v_n_scalar>( + x: &Tensor, + n: S, + ) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_v_n_scalar(x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_v_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_v_n_scalar_out(out, x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_v_out( + out: &Tensor, + x: &Tensor, + n: &Tensor, + ) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_v_out(out, x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_v_x_scalar>( + x: S, + n: &Tensor, + ) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_v_x_scalar(x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_v_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_v_x_scalar_out(out, x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_w(x: &Tensor, n: &Tensor) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_w(x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_w_n_scalar>( + x: &Tensor, + n: S, + ) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_w_n_scalar(x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_w_n_scalar_out>( + out: &Tensor, + x: &Tensor, + n: S, + ) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_w_n_scalar_out(out, x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_w_out( + out: &Tensor, + x: &Tensor, + n: &Tensor, + ) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_w_out(out, x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_w_x_scalar>( + x: S, + n: &Tensor, + ) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_w_x_scalar(x, n).unwrap() + } + + pub fn special_shifted_chebyshev_polynomial_w_x_scalar_out>( + out: &Tensor, + x: S, + n: &Tensor, + ) -> Tensor { + Tensor::f_special_shifted_chebyshev_polynomial_w_x_scalar_out(out, x, n).unwrap() + } + + pub fn special_sinc(&self) -> Tensor { + self.f_special_sinc().unwrap() + } + + pub fn special_sinc_out(&self, out: &Tensor) -> Tensor { + self.f_special_sinc_out(out).unwrap() + } + + pub fn special_softmax(&self, dim: i64, dtype: impl Into>) -> Tensor { + self.f_special_softmax(dim, dtype).unwrap() + } + + pub fn special_spherical_bessel_j0(x: &Tensor) -> Tensor { + Tensor::f_special_spherical_bessel_j0(x).unwrap() + } + + pub fn special_spherical_bessel_j0_out(out: &Tensor, x: &Tensor) -> Tensor { + Tensor::f_special_spherical_bessel_j0_out(out, x).unwrap() + } + + pub fn special_xlog1py(&self, other: &Tensor) -> Tensor { + self.f_special_xlog1py(other).unwrap() + } + + pub fn special_xlog1py_other_scalar>(&self, other: S) -> Tensor { + self.f_special_xlog1py_other_scalar(other).unwrap() + } + + pub fn special_xlog1py_other_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Tensor { + self.f_special_xlog1py_other_scalar_out(out, other).unwrap() + } + + pub fn special_xlog1py_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_special_xlog1py_out(out, other).unwrap() + } + + pub fn special_xlog1py_self_scalar>(self_scalar: S, other: &Tensor) -> Tensor { + Tensor::f_special_xlog1py_self_scalar(self_scalar, other).unwrap() + } + + pub fn special_xlog1py_self_scalar_out>( + out: &Tensor, + self_scalar: S, + other: &Tensor, + ) -> Tensor { + Tensor::f_special_xlog1py_self_scalar_out(out, self_scalar, other).unwrap() + } + + pub fn special_xlogy(&self, other: &Tensor) -> Tensor { + self.f_special_xlogy(other).unwrap() + } + + pub fn special_xlogy_other_scalar>(&self, other: S) -> Tensor { + self.f_special_xlogy_other_scalar(other).unwrap() + } + + pub fn special_xlogy_other_scalar_out>( + &self, + out: &Tensor, + other: S, + ) -> Tensor { + self.f_special_xlogy_other_scalar_out(out, other).unwrap() + } + + pub fn special_xlogy_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_special_xlogy_out(out, other).unwrap() + } + + pub fn special_xlogy_self_scalar>(self_scalar: S, other: &Tensor) -> Tensor { + Tensor::f_special_xlogy_self_scalar(self_scalar, other).unwrap() + } + + pub fn special_xlogy_self_scalar_out>( + out: &Tensor, + self_scalar: S, + other: &Tensor, + ) -> Tensor { + Tensor::f_special_xlogy_self_scalar_out(out, self_scalar, other).unwrap() + } + + pub fn special_zeta(&self, other: &Tensor) -> Tensor { + self.f_special_zeta(other).unwrap() + } + + pub fn special_zeta_other_scalar>(&self, other: S) -> Tensor { + self.f_special_zeta_other_scalar(other).unwrap() + } + + pub fn special_zeta_other_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_special_zeta_other_scalar_out(out, other).unwrap() + } + + pub fn special_zeta_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_special_zeta_out(out, other).unwrap() + } + + pub fn special_zeta_self_scalar>(self_scalar: S, other: &Tensor) -> Tensor { + Tensor::f_special_zeta_self_scalar(self_scalar, other).unwrap() + } + + pub fn special_zeta_self_scalar_out>( + out: &Tensor, + self_scalar: S, + other: &Tensor, + ) -> Tensor { + Tensor::f_special_zeta_self_scalar_out(out, self_scalar, other).unwrap() + } + + pub fn split(&self, split_size: i64, dim: i64) -> Vec { + self.f_split(split_size, dim).unwrap() + } + + pub fn split_copy(&self, split_size: i64, dim: i64) -> Vec { + self.f_split_copy(split_size, dim).unwrap() + } + + pub fn split_copy_tensor_out>(&self, out: &[T], split_size: i64, dim: i64) { + self.f_split_copy_tensor_out(out, split_size, dim).unwrap() + } + + pub fn split_sizes(&self, split_size: impl IntList, dim: i64) -> Vec { + self.f_split_sizes(split_size, dim).unwrap() + } + + pub fn split_with_sizes(&self, split_sizes: impl IntList, dim: i64) -> Vec { + self.f_split_with_sizes(split_sizes, dim).unwrap() + } + + pub fn split_with_sizes_copy(&self, split_sizes: impl IntList, dim: i64) -> Vec { + self.f_split_with_sizes_copy(split_sizes, dim).unwrap() + } + + pub fn split_with_sizes_copy_out>( + &self, + out: &[T], + split_sizes: impl IntList, + dim: i64, + ) { + self.f_split_with_sizes_copy_out(out, split_sizes, dim).unwrap() + } + + pub fn sqrt(&self) -> Tensor { + self.f_sqrt().unwrap() + } + + pub fn sqrt_(&mut self) -> Tensor { + self.f_sqrt_().unwrap() + } + + pub fn sqrt_out(&self, out: &Tensor) -> Tensor { + self.f_sqrt_out(out).unwrap() + } + + pub fn square(&self) -> Tensor { + self.f_square().unwrap() + } + + pub fn square_(&mut self) -> Tensor { + self.f_square_().unwrap() + } + + pub fn square_out(&self, out: &Tensor) -> Tensor { + self.f_square_out(out).unwrap() + } + + pub fn squeeze(&self) -> Tensor { + self.f_squeeze().unwrap() + } + + pub fn squeeze_(&mut self) -> Tensor { + self.f_squeeze_().unwrap() + } + + pub fn squeeze_copy(&self) -> Tensor { + self.f_squeeze_copy().unwrap() + } + + pub fn squeeze_copy_dim(&self, dim: i64) -> Tensor { + self.f_squeeze_copy_dim(dim).unwrap() + } + + pub fn squeeze_copy_dim_out(&self, out: &Tensor, dim: i64) -> Tensor { + self.f_squeeze_copy_dim_out(out, dim).unwrap() + } + + pub fn squeeze_copy_dims(&self, dim: impl IntList) -> Tensor { + self.f_squeeze_copy_dims(dim).unwrap() + } + + pub fn squeeze_copy_dims_out(&self, out: &Tensor, dim: impl IntList) -> Tensor { + self.f_squeeze_copy_dims_out(out, dim).unwrap() + } + + pub fn squeeze_copy_out(&self, out: &Tensor) -> Tensor { + self.f_squeeze_copy_out(out).unwrap() + } + + pub fn squeeze_dim(&self, dim: i64) -> Tensor { + self.f_squeeze_dim(dim).unwrap() + } + + pub fn squeeze_dim_(&mut self, dim: i64) -> Tensor { + self.f_squeeze_dim_(dim).unwrap() + } + + pub fn squeeze_dims(&self, dim: impl IntList) -> Tensor { + self.f_squeeze_dims(dim).unwrap() + } + + pub fn squeeze_dims_(&mut self, dim: impl IntList) -> Tensor { + self.f_squeeze_dims_(dim).unwrap() + } + + pub fn sspaddmm(&self, mat1: &Tensor, mat2: &Tensor) -> Tensor { + self.f_sspaddmm(mat1, mat2).unwrap() + } + + pub fn sspaddmm_out(&self, out: &Tensor, mat1: &Tensor, mat2: &Tensor) -> Tensor { + self.f_sspaddmm_out(out, mat1, mat2).unwrap() + } + + pub fn stack>(tensors: &[T], dim: i64) -> Tensor { + Tensor::f_stack(tensors, dim).unwrap() + } + + pub fn stack_out>(out: &Tensor, tensors: &[T], dim: i64) -> Tensor { + Tensor::f_stack_out(out, tensors, dim).unwrap() + } + + pub fn std(&self, unbiased: bool) -> Tensor { + self.f_std(unbiased).unwrap() + } + + pub fn std_correction>( + &self, + dim: impl IntListOption, + correction: S, + keepdim: bool, + ) -> Tensor { + self.f_std_correction(dim, correction, keepdim).unwrap() + } + + pub fn std_correction_out>( + &self, + out: &Tensor, + dim: impl IntListOption, + correction: S, + keepdim: bool, + ) -> Tensor { + self.f_std_correction_out(out, dim, correction, keepdim).unwrap() + } + + pub fn std_dim(&self, dim: impl IntListOption, unbiased: bool, keepdim: bool) -> Tensor { + self.f_std_dim(dim, unbiased, keepdim).unwrap() + } + + pub fn std_mean(&self, unbiased: bool) -> (Tensor, Tensor) { + self.f_std_mean(unbiased).unwrap() + } + + pub fn std_mean_correction>( + &self, + dim: impl IntListOption, + correction: S, + keepdim: bool, + ) -> (Tensor, Tensor) { + self.f_std_mean_correction(dim, correction, keepdim).unwrap() + } + + pub fn std_mean_correction_out>( + &self, + out0: &Tensor, + out1: &Tensor, + dim: impl IntListOption, + correction: S, + keepdim: bool, + ) -> (Tensor, Tensor) { + self.f_std_mean_correction_out(out0, out1, dim, correction, keepdim).unwrap() + } + + pub fn std_mean_dim( + &self, + dim: impl IntListOption, + unbiased: bool, + keepdim: bool, + ) -> (Tensor, Tensor) { + self.f_std_mean_dim(dim, unbiased, keepdim).unwrap() + } + + pub fn std_out( + &self, + out: &Tensor, + dim: impl IntListOption, + unbiased: bool, + keepdim: bool, + ) -> Tensor { + self.f_std_out(out, dim, unbiased, keepdim).unwrap() + } + + pub fn stft>( + &self, + n_fft: i64, + hop_length: impl Into>, + win_length: impl Into>, + window: Option, + normalized: bool, + onesided: bool, + return_complex: bool, + align_to_window: bool, + ) -> Tensor { + self.f_stft( + n_fft, + hop_length, + win_length, + window, + normalized, + onesided, + return_complex, + align_to_window, + ) + .unwrap() + } + + pub fn stft_center>( + &self, + n_fft: i64, + hop_length: impl Into>, + win_length: impl Into>, + window: Option, + center: bool, + pad_mode: &str, + normalized: bool, + onesided: bool, + return_complex: bool, + align_to_window: bool, + ) -> Tensor { + self.f_stft_center( + n_fft, + hop_length, + win_length, + window, + center, + pad_mode, + normalized, + onesided, + return_complex, + align_to_window, + ) + .unwrap() + } + + pub fn g_sub(&self, other: &Tensor) -> Tensor { + self.f_sub(other).unwrap() + } + + pub fn g_sub_(&mut self, other: &Tensor) -> Tensor { + self.f_sub_(other).unwrap() + } + + pub fn sub_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_sub_out(out, other).unwrap() + } + + pub fn g_sub_scalar>(&self, other: S) -> Tensor { + self.f_sub_scalar(other).unwrap() + } + + pub fn g_sub_scalar_>(&mut self, other: S) -> Tensor { + self.f_sub_scalar_(other).unwrap() + } + + pub fn sub_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { + self.f_sub_scalar_out(out, other).unwrap() + } + + pub fn subtract(&self, other: &Tensor) -> Tensor { + self.f_subtract(other).unwrap() + } + + pub fn subtract_(&mut self, other: &Tensor) -> Tensor { + self.f_subtract_(other).unwrap() + } + + pub fn subtract_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_subtract_out(out, other).unwrap() + } + + pub fn subtract_scalar>(&self, other: S) -> Tensor { + self.f_subtract_scalar(other).unwrap() + } + + pub fn subtract_scalar_>(&mut self, other: S) -> Tensor { + self.f_subtract_scalar_(other).unwrap() + } + + pub fn sum(&self, dtype: impl Into>) -> Tensor { + self.f_sum(dtype).unwrap() + } + + pub fn sum_dim_intlist( + &self, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Tensor { + self.f_sum_dim_intlist(dim, keepdim, dtype).unwrap() + } + + pub fn sum_intlist_out( + &self, + out: &Tensor, + dim: impl IntListOption, + keepdim: bool, + dtype: impl Into>, + ) -> Tensor { + self.f_sum_intlist_out(out, dim, keepdim, dtype).unwrap() + } + + pub fn sum_out(&self, out: &Tensor, dtype: impl Into>) -> Tensor { + self.f_sum_out(out, dtype).unwrap() + } + + pub fn sum_to_size(&self, size: impl IntList) -> Tensor { + self.f_sum_to_size(size).unwrap() + } + + pub fn svd(&self, some: bool, compute_uv: bool) -> (Tensor, Tensor, Tensor) { + self.f_svd(some, compute_uv).unwrap() + } + + pub fn svd_u( + &self, + u: &Tensor, + s: &Tensor, + v: &Tensor, + some: bool, + compute_uv: bool, + ) -> (Tensor, Tensor, Tensor) { + self.f_svd_u(u, s, v, some, compute_uv).unwrap() + } + + pub fn swapaxes(&self, axis0: i64, axis1: i64) -> Tensor { + self.f_swapaxes(axis0, axis1).unwrap() + } + + pub fn swapaxes_(&mut self, axis0: i64, axis1: i64) -> Tensor { + self.f_swapaxes_(axis0, axis1).unwrap() + } + + pub fn swapdims(&self, dim0: i64, dim1: i64) -> Tensor { + self.f_swapdims(dim0, dim1).unwrap() + } + + pub fn swapdims_(&mut self, dim0: i64, dim1: i64) -> Tensor { + self.f_swapdims_(dim0, dim1).unwrap() + } + + pub fn tr(&self) -> Tensor { + self.f_tr().unwrap() + } + + pub fn t_(&mut self) -> Tensor { + self.f_t_().unwrap() + } + + pub fn t_copy(&self) -> Tensor { + self.f_t_copy().unwrap() + } + + pub fn t_copy_out(&self, out: &Tensor) -> Tensor { + self.f_t_copy_out(out).unwrap() + } + + pub fn take(&self, index: &Tensor) -> Tensor { + self.f_take(index).unwrap() + } + + pub fn take_along_dim(&self, indices: &Tensor, dim: impl Into>) -> Tensor { + self.f_take_along_dim(indices, dim).unwrap() + } + + pub fn take_along_dim_out( + &self, + out: &Tensor, + indices: &Tensor, + dim: impl Into>, + ) -> Tensor { + self.f_take_along_dim_out(out, indices, dim).unwrap() + } + + pub fn take_out(&self, out: &Tensor, index: &Tensor) -> Tensor { + self.f_take_out(out, index).unwrap() + } + + pub fn tan(&self) -> Tensor { + self.f_tan().unwrap() + } + + pub fn tan_(&mut self) -> Tensor { + self.f_tan_().unwrap() + } + + pub fn tan_out(&self, out: &Tensor) -> Tensor { + self.f_tan_out(out).unwrap() + } + + pub fn tanh(&self) -> Tensor { + self.f_tanh().unwrap() + } + + pub fn tanh_(&mut self) -> Tensor { + self.f_tanh_().unwrap() + } + + pub fn tanh_backward(grad_output: &Tensor, output: &Tensor) -> Tensor { + Tensor::f_tanh_backward(grad_output, output).unwrap() + } + + pub fn tanh_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output: &Tensor, + ) -> Tensor { + Tensor::f_tanh_backward_grad_input(grad_input, grad_output, output).unwrap() + } + + pub fn tanh_out(&self, out: &Tensor) -> Tensor { + self.f_tanh_out(out).unwrap() + } + + pub fn tensor_split(&self, sections: i64, dim: i64) -> Vec { + self.f_tensor_split(sections, dim).unwrap() + } + + pub fn tensor_split_indices(&self, indices: impl IntList, dim: i64) -> Vec { + self.f_tensor_split_indices(indices, dim).unwrap() + } + + pub fn tensor_split_tensor_indices_or_sections( + &self, + tensor_indices_or_sections: &Tensor, + dim: i64, + ) -> Vec { + self.f_tensor_split_tensor_indices_or_sections(tensor_indices_or_sections, dim).unwrap() + } + + pub fn tensordot( + &self, + other: &Tensor, + dims_self: impl IntList, + dims_other: impl IntList, + ) -> Tensor { + self.f_tensordot(other, dims_self, dims_other).unwrap() + } + + pub fn tensordot_out( + &self, + out: &Tensor, + other: &Tensor, + dims_self: impl IntList, + dims_other: impl IntList, + ) -> Tensor { + self.f_tensordot_out(out, other, dims_self, dims_other).unwrap() + } + + pub fn threshold>(&self, threshold: S, value: S) -> Tensor { + self.f_threshold(threshold, value).unwrap() + } + + pub fn threshold_>(&mut self, threshold: S, value: S) -> Tensor { + self.f_threshold_(threshold, value).unwrap() + } + + pub fn threshold_backward>( + &self, + grad_output: &Tensor, + threshold: S, + ) -> Tensor { + self.f_threshold_backward(grad_output, threshold).unwrap() + } + + pub fn threshold_backward_grad_input>( + &self, + grad_input: &Tensor, + grad_output: &Tensor, + threshold: S, + ) -> Tensor { + self.f_threshold_backward_grad_input(grad_input, grad_output, threshold).unwrap() + } + + pub fn threshold_out>(&self, out: &Tensor, threshold: S, value: S) -> Tensor { + self.f_threshold_out(out, threshold, value).unwrap() + } + + pub fn tile(&self, dims: impl IntList) -> Tensor { + self.f_tile(dims).unwrap() + } + + pub fn to(&self, device: Device) -> Tensor { + self.f_to(device).unwrap() + } + + pub fn to_dense(&self, dtype: impl Into>, masked_grad: bool) -> Tensor { + self.f_to_dense(dtype, masked_grad).unwrap() + } + + pub fn to_dense_backward(&self, grad: &Tensor, masked_grad: bool) -> Tensor { + self.f_to_dense_backward(grad, masked_grad).unwrap() + } + + pub fn to_device_( + &self, + device: Device, + dtype: Kind, + non_blocking: bool, + copy: bool, + ) -> Tensor { + self.f_to_device_(device, dtype, non_blocking, copy).unwrap() + } + + pub fn to_dtype(&self, dtype: Kind, non_blocking: bool, copy: bool) -> Tensor { + self.f_to_dtype(dtype, non_blocking, copy).unwrap() + } + + pub fn to_dtype_layout( + &self, + options: (Kind, Device), + non_blocking: bool, + copy: bool, + ) -> Tensor { + self.f_to_dtype_layout(options, non_blocking, copy).unwrap() + } + + pub fn g_to_mkldnn(&self, dtype: impl Into>) -> Tensor { + self.f_to_mkldnn(dtype).unwrap() + } + + pub fn to_mkldnn_backward(&self, grad: &Tensor) -> Tensor { + self.f_to_mkldnn_backward(grad).unwrap() + } + + pub fn to_mkldnn_out(&self, out: &Tensor, dtype: impl Into>) -> Tensor { + self.f_to_mkldnn_out(out, dtype).unwrap() + } + + pub fn to_other(&self, other: &Tensor, non_blocking: bool, copy: bool) -> Tensor { + self.f_to_other(other, non_blocking, copy).unwrap() + } + + pub fn to_padded_tensor(&self, padding: f64, output_size: impl IntListOption) -> Tensor { + self.f_to_padded_tensor(padding, output_size).unwrap() + } + + pub fn to_padded_tensor_out( + &self, + out: &Tensor, + padding: f64, + output_size: impl IntListOption, + ) -> Tensor { + self.f_to_padded_tensor_out(out, padding, output_size).unwrap() + } + + pub fn to_sparse( + &self, + layout: Option, + blocksize: impl IntListOption, + dense_dim: impl Into>, + ) -> Tensor { + self.f_to_sparse(layout, blocksize, dense_dim).unwrap() + } + + pub fn to_sparse_bsc( + &self, + blocksize: impl IntList, + dense_dim: impl Into>, + ) -> Tensor { + self.f_to_sparse_bsc(blocksize, dense_dim).unwrap() + } + + pub fn to_sparse_bsr( + &self, + blocksize: impl IntList, + dense_dim: impl Into>, + ) -> Tensor { + self.f_to_sparse_bsr(blocksize, dense_dim).unwrap() + } + + pub fn to_sparse_csc(&self, dense_dim: impl Into>) -> Tensor { + self.f_to_sparse_csc(dense_dim).unwrap() + } + + pub fn to_sparse_csr(&self, dense_dim: impl Into>) -> Tensor { + self.f_to_sparse_csr(dense_dim).unwrap() + } + + pub fn to_sparse_sparse_dim(&self, sparse_dim: i64) -> Tensor { + self.f_to_sparse_sparse_dim(sparse_dim).unwrap() + } + + pub fn topk(&self, k: i64, dim: i64, largest: bool, sorted: bool) -> (Tensor, Tensor) { + self.f_topk(k, dim, largest, sorted).unwrap() + } + + pub fn topk_values( + &self, + values: &Tensor, + indices: &Tensor, + k: i64, + dim: i64, + largest: bool, + sorted: bool, + ) -> (Tensor, Tensor) { + self.f_topk_values(values, indices, k, dim, largest, sorted).unwrap() + } + + pub fn totype(&self, scalar_type: Kind) -> Tensor { + self.f_totype(scalar_type).unwrap() + } + + pub fn trace(&self) -> Tensor { + self.f_trace().unwrap() + } + + pub fn trace_backward(grad: &Tensor, sizes: impl IntList) -> Tensor { + Tensor::f_trace_backward(grad, sizes).unwrap() + } + + pub fn trace_out(&self, out: &Tensor) -> Tensor { + self.f_trace_out(out).unwrap() + } + + pub fn transpose(&self, dim0: i64, dim1: i64) -> Tensor { + self.f_transpose(dim0, dim1).unwrap() + } + + pub fn transpose_(&mut self, dim0: i64, dim1: i64) -> Tensor { + self.f_transpose_(dim0, dim1).unwrap() + } + + pub fn transpose_copy(&self, dim0: i64, dim1: i64) -> Tensor { + self.f_transpose_copy(dim0, dim1).unwrap() + } + + pub fn transpose_copy_int_out(&self, out: &Tensor, dim0: i64, dim1: i64) -> Tensor { + self.f_transpose_copy_int_out(out, dim0, dim1).unwrap() + } + + pub fn trapezoid(y: &Tensor, dim: i64) -> Tensor { + Tensor::f_trapezoid(y, dim).unwrap() + } + + pub fn trapezoid_x(y: &Tensor, x: &Tensor, dim: i64) -> Tensor { + Tensor::f_trapezoid_x(y, x, dim).unwrap() + } + + pub fn trapz(y: &Tensor, x: &Tensor, dim: i64) -> Tensor { + Tensor::f_trapz(y, x, dim).unwrap() + } + + pub fn trapz_dx(y: &Tensor, dx: f64, dim: i64) -> Tensor { + Tensor::f_trapz_dx(y, dx, dim).unwrap() + } + + pub fn triangular_solve( + &self, + a: &Tensor, + upper: bool, + transpose: bool, + unitriangular: bool, + ) -> (Tensor, Tensor) { + self.f_triangular_solve(a, upper, transpose, unitriangular).unwrap() + } + + pub fn triangular_solve_x( + &self, + x: &Tensor, + m: &Tensor, + a: &Tensor, + upper: bool, + transpose: bool, + unitriangular: bool, + ) -> (Tensor, Tensor) { + self.f_triangular_solve_x(x, m, a, upper, transpose, unitriangular).unwrap() + } + + pub fn tril(&self, diagonal: i64) -> Tensor { + self.f_tril(diagonal).unwrap() + } + + pub fn tril_(&mut self, diagonal: i64) -> Tensor { + self.f_tril_(diagonal).unwrap() + } + + pub fn tril_indices(row: i64, col: i64, offset: i64, options: (Kind, Device)) -> Tensor { + Tensor::f_tril_indices(row, col, offset, options).unwrap() + } + + pub fn tril_indices_out(out: &Tensor, row: i64, col: i64, offset: i64) -> Tensor { + Tensor::f_tril_indices_out(out, row, col, offset).unwrap() + } + + pub fn tril_out(&self, out: &Tensor, diagonal: i64) -> Tensor { + self.f_tril_out(out, diagonal).unwrap() + } + + pub fn triplet_margin_loss( + anchor: &Tensor, + positive: &Tensor, + negative: &Tensor, + margin: f64, + p: f64, + eps: f64, + swap: bool, + reduction: crate::Reduction, + ) -> Tensor { + Tensor::f_triplet_margin_loss(anchor, positive, negative, margin, p, eps, swap, reduction) + .unwrap() + } + + pub fn triu(&self, diagonal: i64) -> Tensor { + self.f_triu(diagonal).unwrap() + } + + pub fn triu_(&mut self, diagonal: i64) -> Tensor { + self.f_triu_(diagonal).unwrap() + } + + pub fn triu_indices(row: i64, col: i64, offset: i64, options: (Kind, Device)) -> Tensor { + Tensor::f_triu_indices(row, col, offset, options).unwrap() + } + + pub fn triu_indices_out(out: &Tensor, row: i64, col: i64, offset: i64) -> Tensor { + Tensor::f_triu_indices_out(out, row, col, offset).unwrap() + } + + pub fn triu_out(&self, out: &Tensor, diagonal: i64) -> Tensor { + self.f_triu_out(out, diagonal).unwrap() + } + + pub fn true_divide(&self, other: &Tensor) -> Tensor { + self.f_true_divide(other).unwrap() + } + + pub fn true_divide_(&mut self, other: &Tensor) -> Tensor { + self.f_true_divide_(other).unwrap() + } + + pub fn true_divide_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_true_divide_out(out, other).unwrap() + } + + pub fn true_divide_scalar>(&self, other: S) -> Tensor { + self.f_true_divide_scalar(other).unwrap() + } + + pub fn true_divide_scalar_>(&mut self, other: S) -> Tensor { + self.f_true_divide_scalar_(other).unwrap() + } + + pub fn trunc(&self) -> Tensor { + self.f_trunc().unwrap() + } + + pub fn trunc_(&mut self) -> Tensor { + self.f_trunc_().unwrap() + } + + pub fn trunc_out(&self, out: &Tensor) -> Tensor { + self.f_trunc_out(out).unwrap() + } + + pub fn type_as(&self, other: &Tensor) -> Tensor { + self.f_type_as(other).unwrap() + } + + pub fn unbind(&self, dim: i64) -> Vec { + self.f_unbind(dim).unwrap() + } + + pub fn unbind_copy(&self, dim: i64) -> Vec { + self.f_unbind_copy(dim).unwrap() + } + + pub fn unbind_copy_int_out>(&self, out: &[T], dim: i64) { + self.f_unbind_copy_int_out(out, dim).unwrap() + } + + pub fn unflatten(&self, dim: i64, sizes: impl IntList) -> Tensor { + self.f_unflatten(dim, sizes).unwrap() + } + + pub fn unflatten_dense_tensors>(flat: &Tensor, tensors: &[T]) -> Vec { + Tensor::f_unflatten_dense_tensors(flat, tensors).unwrap() + } + + pub fn unfold(&self, dimension: i64, size: i64, step: i64) -> Tensor { + self.f_unfold(dimension, size, step).unwrap() + } + + pub fn unfold_backward( + grad_in: &Tensor, + input_sizes: impl IntList, + dim: i64, + size: i64, + step: i64, + ) -> Tensor { + Tensor::f_unfold_backward(grad_in, input_sizes, dim, size, step).unwrap() + } + + pub fn unfold_backward_out( + out: &Tensor, + grad_in: &Tensor, + input_sizes: impl IntList, + dim: i64, + size: i64, + step: i64, + ) -> Tensor { + Tensor::f_unfold_backward_out(out, grad_in, input_sizes, dim, size, step).unwrap() + } + + pub fn unfold_copy(&self, dimension: i64, size: i64, step: i64) -> Tensor { + self.f_unfold_copy(dimension, size, step).unwrap() + } + + pub fn unfold_copy_out(&self, out: &Tensor, dimension: i64, size: i64, step: i64) -> Tensor { + self.f_unfold_copy_out(out, dimension, size, step).unwrap() + } + + pub fn uniform(&self, from: f64, to: f64) -> Tensor { + self.f_uniform(from, to).unwrap() + } + + pub fn uniform_(&mut self, from: f64, to: f64) -> Tensor { + self.f_uniform_(from, to).unwrap() + } + + pub fn uniform_out(&self, out: &Tensor, from: f64, to: f64) -> Tensor { + self.f_uniform_out(out, from, to).unwrap() + } + + pub fn unique_consecutive( + &self, + return_inverse: bool, + return_counts: bool, + dim: impl Into>, + ) -> (Tensor, Tensor, Tensor) { + self.f_unique_consecutive(return_inverse, return_counts, dim).unwrap() + } + + pub fn unique_consecutive_out( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + return_inverse: bool, + return_counts: bool, + dim: impl Into>, + ) -> (Tensor, Tensor, Tensor) { + self.f_unique_consecutive_out(out0, out1, out2, return_inverse, return_counts, dim).unwrap() + } + + pub fn unique_dim( + &self, + dim: i64, + sorted: bool, + return_inverse: bool, + return_counts: bool, + ) -> (Tensor, Tensor, Tensor) { + self.f_unique_dim(dim, sorted, return_inverse, return_counts).unwrap() + } + + pub fn unique_dim_consecutive( + &self, + dim: i64, + return_inverse: bool, + return_counts: bool, + ) -> (Tensor, Tensor, Tensor) { + self.f_unique_dim_consecutive(dim, return_inverse, return_counts).unwrap() + } + + pub fn unique_dim_consecutive_out( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + dim: i64, + return_inverse: bool, + return_counts: bool, + ) -> (Tensor, Tensor, Tensor) { + self.f_unique_dim_consecutive_out(out0, out1, out2, dim, return_inverse, return_counts) + .unwrap() + } + + pub fn unique_dim_out( + &self, + out0: &Tensor, + out1: &Tensor, + out2: &Tensor, + dim: i64, + sorted: bool, + return_inverse: bool, + return_counts: bool, + ) -> (Tensor, Tensor, Tensor) { + self.f_unique_dim_out(out0, out1, out2, dim, sorted, return_inverse, return_counts).unwrap() + } + + pub fn unsafe_chunk(&self, chunks: i64, dim: i64) -> Vec { + self.f_unsafe_chunk(chunks, dim).unwrap() + } + + pub fn unsafe_split(&self, split_size: i64, dim: i64) -> Vec { + self.f_unsafe_split(split_size, dim).unwrap() + } + + pub fn unsafe_split_tensor_out>(&self, out: &[T], split_size: i64, dim: i64) { + self.f_unsafe_split_tensor_out(out, split_size, dim).unwrap() + } + + pub fn unsafe_split_with_sizes(&self, split_sizes: impl IntList, dim: i64) -> Vec { + self.f_unsafe_split_with_sizes(split_sizes, dim).unwrap() + } + + pub fn unsafe_split_with_sizes_out>( + &self, + out: &[T], + split_sizes: impl IntList, + dim: i64, + ) { + self.f_unsafe_split_with_sizes_out(out, split_sizes, dim).unwrap() + } + + pub fn unsqueeze(&self, dim: i64) -> Tensor { + self.f_unsqueeze(dim).unwrap() + } + + pub fn unsqueeze_(&mut self, dim: i64) -> Tensor { + self.f_unsqueeze_(dim).unwrap() + } + + pub fn unsqueeze_copy(&self, dim: i64) -> Tensor { + self.f_unsqueeze_copy(dim).unwrap() + } + + pub fn unsqueeze_copy_out(&self, out: &Tensor, dim: i64) -> Tensor { + self.f_unsqueeze_copy_out(out, dim).unwrap() + } + + pub fn upsample_bicubic2d( + &self, + output_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + self.f_upsample_bicubic2d(output_size, align_corners, scales_h, scales_w).unwrap() + } + + pub fn upsample_bicubic2d_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + Tensor::f_upsample_bicubic2d_backward( + grad_output, + output_size, + input_size, + align_corners, + scales_h, + scales_w, + ) + .unwrap() + } + + pub fn upsample_bicubic2d_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + Tensor::f_upsample_bicubic2d_backward_grad_input( + grad_input, + grad_output, + output_size, + input_size, + align_corners, + scales_h, + scales_w, + ) + .unwrap() + } + + pub fn upsample_bicubic2d_out( + &self, + out: &Tensor, + output_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + self.f_upsample_bicubic2d_out(out, output_size, align_corners, scales_h, scales_w).unwrap() + } + + pub fn upsample_bicubic2d_vec( + &self, + output_size: impl IntListOption, + align_corners: bool, + scale_factors: impl DoubleList, + ) -> Tensor { + self.f_upsample_bicubic2d_vec(output_size, align_corners, scale_factors).unwrap() + } + + pub fn upsample_bilinear2d( + &self, + output_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + self.f_upsample_bilinear2d(output_size, align_corners, scales_h, scales_w).unwrap() + } + + pub fn upsample_bilinear2d_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + Tensor::f_upsample_bilinear2d_backward( + grad_output, + output_size, + input_size, + align_corners, + scales_h, + scales_w, + ) + .unwrap() + } + + pub fn upsample_bilinear2d_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + Tensor::f_upsample_bilinear2d_backward_grad_input( + grad_input, + grad_output, + output_size, + input_size, + align_corners, + scales_h, + scales_w, + ) + .unwrap() + } + + pub fn upsample_bilinear2d_out( + &self, + out: &Tensor, + output_size: impl IntList, + align_corners: bool, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + self.f_upsample_bilinear2d_out(out, output_size, align_corners, scales_h, scales_w).unwrap() + } + + pub fn upsample_bilinear2d_vec( + &self, + output_size: impl IntListOption, + align_corners: bool, + scale_factors: impl DoubleList, + ) -> Tensor { + self.f_upsample_bilinear2d_vec(output_size, align_corners, scale_factors).unwrap() + } + + pub fn upsample_bilinear2d_vec_out( + &self, + out: &Tensor, + output_size: impl IntListOption, + align_corners: bool, + scale_factors: impl DoubleList, + ) -> Tensor { + self.f_upsample_bilinear2d_vec_out(out, output_size, align_corners, scale_factors).unwrap() + } + + pub fn upsample_linear1d( + &self, + output_size: impl IntList, + align_corners: bool, + scales: impl Into>, + ) -> Tensor { + self.f_upsample_linear1d(output_size, align_corners, scales).unwrap() + } + + pub fn upsample_linear1d_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales: impl Into>, + ) -> Tensor { + Tensor::f_upsample_linear1d_backward( + grad_output, + output_size, + input_size, + align_corners, + scales, + ) + .unwrap() + } + + pub fn upsample_linear1d_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales: impl Into>, + ) -> Tensor { + Tensor::f_upsample_linear1d_backward_grad_input( + grad_input, + grad_output, + output_size, + input_size, + align_corners, + scales, + ) + .unwrap() + } + + pub fn upsample_linear1d_out( + &self, + out: &Tensor, + output_size: impl IntList, + align_corners: bool, + scales: impl Into>, + ) -> Tensor { + self.f_upsample_linear1d_out(out, output_size, align_corners, scales).unwrap() + } + + pub fn upsample_linear1d_vec( + &self, + output_size: impl IntListOption, + align_corners: bool, + scale_factors: impl DoubleList, + ) -> Tensor { + self.f_upsample_linear1d_vec(output_size, align_corners, scale_factors).unwrap() + } + + pub fn upsample_nearest1d( + &self, + output_size: impl IntList, + scales: impl Into>, + ) -> Tensor { + self.f_upsample_nearest1d(output_size, scales).unwrap() + } + + pub fn upsample_nearest1d_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales: impl Into>, + ) -> Tensor { + Tensor::f_upsample_nearest1d_backward(grad_output, output_size, input_size, scales).unwrap() + } + + pub fn upsample_nearest1d_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales: impl Into>, + ) -> Tensor { + Tensor::f_upsample_nearest1d_backward_grad_input( + grad_input, + grad_output, + output_size, + input_size, + scales, + ) + .unwrap() + } + + pub fn upsample_nearest1d_out( + &self, + out: &Tensor, + output_size: impl IntList, + scales: impl Into>, + ) -> Tensor { + self.f_upsample_nearest1d_out(out, output_size, scales).unwrap() + } + + pub fn upsample_nearest1d_vec( + &self, + output_size: impl IntListOption, + scale_factors: impl DoubleList, + ) -> Tensor { + self.f_upsample_nearest1d_vec(output_size, scale_factors).unwrap() + } + + pub fn upsample_nearest2d( + &self, + output_size: impl IntList, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + self.f_upsample_nearest2d(output_size, scales_h, scales_w).unwrap() + } + + pub fn upsample_nearest2d_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + Tensor::f_upsample_nearest2d_backward( + grad_output, + output_size, + input_size, + scales_h, + scales_w, + ) + .unwrap() + } + + pub fn upsample_nearest2d_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + Tensor::f_upsample_nearest2d_backward_grad_input( + grad_input, + grad_output, + output_size, + input_size, + scales_h, + scales_w, + ) + .unwrap() + } + + pub fn upsample_nearest2d_out( + &self, + out: &Tensor, + output_size: impl IntList, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + self.f_upsample_nearest2d_out(out, output_size, scales_h, scales_w).unwrap() + } + + pub fn upsample_nearest2d_vec( + &self, + output_size: impl IntListOption, + scale_factors: impl DoubleList, + ) -> Tensor { + self.f_upsample_nearest2d_vec(output_size, scale_factors).unwrap() + } + + pub fn upsample_nearest2d_vec_out( + &self, + out: &Tensor, + output_size: impl IntListOption, + scale_factors: impl DoubleList, + ) -> Tensor { + self.f_upsample_nearest2d_vec_out(out, output_size, scale_factors).unwrap() + } + + pub fn upsample_nearest3d( + &self, + output_size: impl IntList, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + self.f_upsample_nearest3d(output_size, scales_d, scales_h, scales_w).unwrap() + } + + pub fn upsample_nearest3d_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + Tensor::f_upsample_nearest3d_backward( + grad_output, + output_size, + input_size, + scales_d, + scales_h, + scales_w, + ) + .unwrap() + } + + pub fn upsample_nearest3d_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + Tensor::f_upsample_nearest3d_backward_grad_input( + grad_input, + grad_output, + output_size, + input_size, + scales_d, + scales_h, + scales_w, + ) + .unwrap() + } + + pub fn upsample_nearest3d_out( + &self, + out: &Tensor, + output_size: impl IntList, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + self.f_upsample_nearest3d_out(out, output_size, scales_d, scales_h, scales_w).unwrap() + } + + pub fn upsample_nearest3d_vec( + &self, + output_size: impl IntListOption, + scale_factors: impl DoubleList, + ) -> Tensor { + self.f_upsample_nearest3d_vec(output_size, scale_factors).unwrap() + } + + pub fn upsample_trilinear3d( + &self, + output_size: impl IntList, + align_corners: bool, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + self.f_upsample_trilinear3d(output_size, align_corners, scales_d, scales_h, scales_w) + .unwrap() + } + + pub fn upsample_trilinear3d_backward( + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + Tensor::f_upsample_trilinear3d_backward( + grad_output, + output_size, + input_size, + align_corners, + scales_d, + scales_h, + scales_w, + ) + .unwrap() + } + + pub fn upsample_trilinear3d_backward_grad_input( + grad_input: &Tensor, + grad_output: &Tensor, + output_size: impl IntList, + input_size: impl IntList, + align_corners: bool, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + Tensor::f_upsample_trilinear3d_backward_grad_input( + grad_input, + grad_output, + output_size, + input_size, + align_corners, + scales_d, + scales_h, + scales_w, + ) + .unwrap() + } + + pub fn upsample_trilinear3d_out( + &self, + out: &Tensor, + output_size: impl IntList, + align_corners: bool, + scales_d: impl Into>, + scales_h: impl Into>, + scales_w: impl Into>, + ) -> Tensor { + self.f_upsample_trilinear3d_out( + out, + output_size, + align_corners, + scales_d, + scales_h, + scales_w, + ) + .unwrap() + } + + pub fn upsample_trilinear3d_vec( + &self, + output_size: impl IntListOption, + align_corners: bool, + scale_factors: impl DoubleList, + ) -> Tensor { + self.f_upsample_trilinear3d_vec(output_size, align_corners, scale_factors).unwrap() + } + + pub fn value_selecting_reduction_backward( + grad: &Tensor, + dim: i64, + indices: &Tensor, + sizes: impl IntList, + keepdim: bool, + ) -> Tensor { + Tensor::f_value_selecting_reduction_backward(grad, dim, indices, sizes, keepdim).unwrap() + } + + pub fn values(&self) -> Tensor { + self.f_values().unwrap() + } + + pub fn values_copy(&self) -> Tensor { + self.f_values_copy().unwrap() + } + + pub fn values_copy_out(&self, out: &Tensor) -> Tensor { + self.f_values_copy_out(out).unwrap() + } + + pub fn vander(x: &Tensor, n: impl Into>, increasing: bool) -> Tensor { + Tensor::f_vander(x, n, increasing).unwrap() + } + + pub fn var(&self, unbiased: bool) -> Tensor { + self.f_var(unbiased).unwrap() + } + + pub fn var_correction>( + &self, + dim: impl IntListOption, + correction: S, + keepdim: bool, + ) -> Tensor { + self.f_var_correction(dim, correction, keepdim).unwrap() + } + + pub fn var_correction_out>( + &self, + out: &Tensor, + dim: impl IntListOption, + correction: S, + keepdim: bool, + ) -> Tensor { + self.f_var_correction_out(out, dim, correction, keepdim).unwrap() + } + + pub fn var_dim(&self, dim: impl IntListOption, unbiased: bool, keepdim: bool) -> Tensor { + self.f_var_dim(dim, unbiased, keepdim).unwrap() + } + + pub fn var_mean(&self, unbiased: bool) -> (Tensor, Tensor) { + self.f_var_mean(unbiased).unwrap() + } + + pub fn var_mean_correction>( + &self, + dim: impl IntListOption, + correction: S, + keepdim: bool, + ) -> (Tensor, Tensor) { + self.f_var_mean_correction(dim, correction, keepdim).unwrap() + } + + pub fn var_mean_correction_out>( + &self, + out0: &Tensor, + out1: &Tensor, + dim: impl IntListOption, + correction: S, + keepdim: bool, + ) -> (Tensor, Tensor) { + self.f_var_mean_correction_out(out0, out1, dim, correction, keepdim).unwrap() + } + + pub fn var_mean_dim( + &self, + dim: impl IntListOption, + unbiased: bool, + keepdim: bool, + ) -> (Tensor, Tensor) { + self.f_var_mean_dim(dim, unbiased, keepdim).unwrap() + } + + pub fn var_out( + &self, + out: &Tensor, + dim: impl IntListOption, + unbiased: bool, + keepdim: bool, + ) -> Tensor { + self.f_var_out(out, dim, unbiased, keepdim).unwrap() + } + + pub fn vdot(&self, other: &Tensor) -> Tensor { + self.f_vdot(other).unwrap() + } + + pub fn vdot_out(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_vdot_out(out, other).unwrap() + } + + pub fn view_(&self, size: impl IntList) -> Tensor { + self.f_view_(size).unwrap() + } + + pub fn view_as(&self, other: &Tensor) -> Tensor { + self.f_view_as(other).unwrap() + } + + pub fn view_as_complex(&self) -> Tensor { + self.f_view_as_complex().unwrap() + } + + pub fn view_as_complex_copy(&self) -> Tensor { + self.f_view_as_complex_copy().unwrap() + } + + pub fn view_as_complex_copy_out(&self, out: &Tensor) -> Tensor { + self.f_view_as_complex_copy_out(out).unwrap() + } + + pub fn view_as_real(&self) -> Tensor { + self.f_view_as_real().unwrap() + } + + pub fn view_as_real_copy(&self) -> Tensor { + self.f_view_as_real_copy().unwrap() + } + + pub fn view_as_real_copy_out(&self, out: &Tensor) -> Tensor { + self.f_view_as_real_copy_out(out).unwrap() + } + + pub fn view_copy(&self, size: impl IntList) -> Tensor { + self.f_view_copy(size).unwrap() + } + + pub fn view_copy_dtype(&self, dtype: Kind) -> Tensor { + self.f_view_copy_dtype(dtype).unwrap() + } + + pub fn view_copy_dtype_out(&self, out: &Tensor, dtype: Kind) -> Tensor { + self.f_view_copy_dtype_out(out, dtype).unwrap() + } + + pub fn view_copy_out(&self, out: &Tensor, size: impl IntList) -> Tensor { + self.f_view_copy_out(out, size).unwrap() + } + + pub fn view_dtype(&self, dtype: Kind) -> Tensor { + self.f_view_dtype(dtype).unwrap() + } + + pub fn vsplit(&self, sections: i64) -> Vec { + self.f_vsplit(sections).unwrap() + } + + pub fn vsplit_array(&self, indices: impl IntList) -> Vec { + self.f_vsplit_array(indices).unwrap() + } + + pub fn vstack>(tensors: &[T]) -> Tensor { + Tensor::f_vstack(tensors).unwrap() + } + + pub fn vstack_out>(out: &Tensor, tensors: &[T]) -> Tensor { + Tensor::f_vstack_out(out, tensors).unwrap() + } + + pub fn where_(condition: &Tensor) -> Vec { + Tensor::f_where_(condition).unwrap() + } + + pub fn where_scalar>(condition: &Tensor, self_scalar: S, other: S) -> Tensor { + Tensor::f_where_scalar(condition, self_scalar, other).unwrap() + } + + pub fn where_scalarother>(&self, condition: &Tensor, other: S) -> Tensor { + self.f_where_scalarother(condition, other).unwrap() + } + + pub fn where_scalarself>( + condition: &Tensor, + self_scalar: S, + other: &Tensor, + ) -> Tensor { + Tensor::f_where_scalarself(condition, self_scalar, other).unwrap() + } + + pub fn where_self(&self, condition: &Tensor, other: &Tensor) -> Tensor { + self.f_where_self(condition, other).unwrap() + } + + pub fn where_self_out(&self, out: &Tensor, condition: &Tensor, other: &Tensor) -> Tensor { + self.f_where_self_out(out, condition, other).unwrap() + } + + pub fn xlogy(&self, other: &Tensor) -> Tensor { + self.f_xlogy(other).unwrap() + } + + pub fn xlogy_(&mut self, other: &Tensor) -> Tensor { + self.f_xlogy_(other).unwrap() + } + + pub fn xlogy_outscalar_other>(&self, out: &Tensor, other: S) -> Tensor { + self.f_xlogy_outscalar_other(out, other).unwrap() + } + + pub fn xlogy_outscalar_self>( + out: &Tensor, + self_scalar: S, + other: &Tensor, + ) -> Tensor { + Tensor::f_xlogy_outscalar_self(out, self_scalar, other).unwrap() + } + + pub fn xlogy_outtensor(&self, out: &Tensor, other: &Tensor) -> Tensor { + self.f_xlogy_outtensor(out, other).unwrap() + } + + pub fn xlogy_scalar_other>(&self, other: S) -> Tensor { + self.f_xlogy_scalar_other(other).unwrap() + } + + pub fn xlogy_scalar_other_>(&mut self, other: S) -> Tensor { + self.f_xlogy_scalar_other_(other).unwrap() + } + + pub fn xlogy_scalar_self>(self_scalar: S, other: &Tensor) -> Tensor { + Tensor::f_xlogy_scalar_self(self_scalar, other).unwrap() + } + + pub fn zero(&self) -> Tensor { + self.f_zero().unwrap() + } + + pub fn zero_(&mut self) -> Tensor { + self.f_zero_().unwrap() + } + + pub fn zero_out(&self, out: &Tensor) -> Tensor { + self.f_zero_out(out).unwrap() + } + + pub fn zeros(size: impl IntList, options: (Kind, Device)) -> Tensor { + Tensor::f_zeros(size, options).unwrap() + } + + pub fn zeros_like(&self) -> Tensor { + self.f_zeros_like().unwrap() + } + + pub fn zeros_like_out(&self, out: &Tensor) -> Tensor { + self.f_zeros_like_out(out).unwrap() + } + + pub fn zeros_out(out: &Tensor, size: impl IntList) -> Tensor { + Tensor::f_zeros_out(out, size).unwrap() + } +} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/utils.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/utils.rs new file mode 100644 index 0000000000..c96ea0cb2b --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/src/wrappers/utils.rs @@ -0,0 +1,157 @@ +use crate::TchError; +use libc::c_char; +use std::io; + +// This returns None on the null pointer. If not null, the pointer gets +// freed. +pub(super) unsafe fn ptr_to_string(ptr: *mut c_char) -> Option { + if !ptr.is_null() { + let str = std::ffi::CStr::from_ptr(ptr).to_string_lossy().into_owned(); + libc::free(ptr as *mut libc::c_void); + Some(str) + } else { + None + } +} + +pub(super) fn read_and_clean_error() -> Result<(), TchError> { + unsafe { + match ptr_to_string(torch_sys::get_and_reset_last_err()) { + None => Ok(()), + Some(c_error) => Err(TchError::Torch(c_error)), + } + } +} + +macro_rules! unsafe_torch { + ($e:expr) => {{ + let v = unsafe { $e }; + crate::wrappers::utils::read_and_clean_error().unwrap(); + v + }}; +} + +macro_rules! unsafe_torch_err { + ($e:expr) => {{ + let v = unsafe { $e }; + crate::wrappers::utils::read_and_clean_error()?; + v + }}; +} + +// Be cautious when using this function as the returned CString should be stored +// in a variable when using as_ptr. Otherwise dangling pointer issues are likely +// to happen. +pub(super) fn path_to_cstring>( + path: T, +) -> Result { + let path = path.as_ref(); + match path.to_str() { + Some(path) => Ok(std::ffi::CString::new(path)?), + None => Err(TchError::Io(io::Error::other(format!( + "path {path:?} cannot be converted to UTF-8" + )))), + } +} + +/// Sets the random seed used by torch. +pub fn manual_seed(seed: i64) { + unsafe_torch!(torch_sys::at_manual_seed(seed)) +} + +/// Get the number of threads used by torch for inter-op parallelism. +pub fn get_num_interop_threads() -> i32 { + unsafe_torch!(torch_sys::at_get_num_interop_threads()) +} + +/// Get the number of threads used by torch in parallel regions. +pub fn get_num_threads() -> i32 { + unsafe_torch!(torch_sys::at_get_num_threads()) +} + +/// Set the number of threads used by torch for inter-op parallelism. +pub fn set_num_interop_threads(n_threads: i32) { + unsafe_torch!(torch_sys::at_set_num_interop_threads(n_threads)) +} + +/// Set the number of threads used by torch in parallel regions. +pub fn set_num_threads(n_threads: i32) { + unsafe_torch!(torch_sys::at_set_num_threads(n_threads)) +} + +pub fn has_openmp() -> bool { + unsafe_torch!(torch_sys::at_context_has_openmp()) +} + +pub fn has_mkl() -> bool { + unsafe_torch!(torch_sys::at_context_has_mkl()) +} +pub fn has_lapack() -> bool { + unsafe_torch!(torch_sys::at_context_has_lapack()) +} +pub fn has_mkldnn() -> bool { + unsafe_torch!(torch_sys::at_context_has_mkldnn()) +} +pub fn has_magma() -> bool { + unsafe_torch!(torch_sys::at_context_has_magma()) +} +pub fn has_cuda() -> bool { + unsafe_torch!(torch_sys::at_context_has_cuda()) +} +pub fn has_cudart() -> bool { + unsafe_torch!(torch_sys::at_context_has_cudart()) +} +pub fn has_cusolver() -> bool { + unsafe_torch!(torch_sys::at_context_has_cusolver()) +} +pub fn has_hip() -> bool { + unsafe_torch!(torch_sys::at_context_has_hip()) +} +pub fn has_ipu() -> bool { + unsafe_torch!(torch_sys::at_context_has_ipu()) +} +pub fn has_xla() -> bool { + unsafe_torch!(torch_sys::at_context_has_xla()) +} +pub fn has_lazy() -> bool { + unsafe_torch!(torch_sys::at_context_has_lazy()) +} +pub fn has_mps() -> bool { + unsafe_torch!(torch_sys::at_context_has_mps()) +} +pub fn version_cudnn() -> i64 { + unsafe_torch!(torch_sys::at_context_version_cudnn()) +} +pub fn version_cudart() -> i64 { + unsafe_torch!(torch_sys::at_context_version_cudart()) +} + +/// Check whether the vulkan backend is available. None that this +/// backend is not included by default as of PyTorch 2.0.0. +/// https://pytorch.org/tutorials/prototype/vulkan_workflow.html#building-pytorch-with-vulkan-backend +pub fn has_vulkan() -> bool { + crate::Tensor::is_vulkan_available() +} + +/// Quantization engines +#[allow(clippy::upper_case_acronyms)] +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub enum QEngine { + NoQEngine, + FBGEMM, + QNNPACK, +} + +impl QEngine { + fn to_cint(self) -> i32 { + match self { + QEngine::NoQEngine => 0, + QEngine::FBGEMM => 1, + QEngine::QNNPACK => 2, + } + } + pub fn set(self) -> Result<(), TchError> { + unsafe_torch_err!(torch_sys::at_set_qengine(self.to_cint())); + Ok(()) + } +} diff --git a/v2/dependency-patches/tch-0.24.0/tests/autocast.rs b/v2/dependency-patches/tch-0.24.0/tests/autocast.rs new file mode 100644 index 0000000000..f0eb41808c --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/tests/autocast.rs @@ -0,0 +1,22 @@ +#[cfg(test)] +#[cfg(feature = "cuda-tests")] +mod tests { + use tch::{autocast, Device, Kind, Tensor}; + + #[test] + fn autocast_narrows_type() { + let device = Device::Cuda(0); + + let linear = Tensor::rand([10, 10], (Kind::Float, device)); + let input = Tensor::rand([10], (Kind::Float, device)); + + autocast(true, || { + let output1 = autocast(false, || linear.matmul(&input)); + assert_eq!(output1.kind(), Kind::Float); + let output2 = linear.matmul(&output1); + assert_eq!(output2.kind(), Kind::Half); + let output3 = autocast(false, || linear.matmul(&output1)); + assert_eq!(output3.kind(), Kind::Float); + }); + } +} diff --git a/v2/dependency-patches/tch-0.24.0/tests/create_jit_models.py b/v2/dependency-patches/tch-0.24.0/tests/create_jit_models.py new file mode 100644 index 0000000000..376a76ae56 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/tests/create_jit_models.py @@ -0,0 +1,119 @@ +import torch +from typing import Dict, Tuple + +class Foo(torch.jit.ScriptModule): + def __init__(self, v): + super(Foo, self).__init__() + self.register_buffer('value', v) + + @torch.jit.script_method + def forward(self, x, y): + return 2 * x + y + self.value + +foo = Foo(torch.Tensor([42.0])) +foo.save('foo.pt') + +class Foo1(torch.jit.ScriptModule): + def __init__(self): + super(Foo1, self).__init__() + + @torch.jit.script_method + def forward(self, x, y): + return 2 * x + y + +foo = Foo1() +script_foo = torch.jit.script(foo) +script_foo.save('foo1.pt') + +class Foo2(torch.jit.ScriptModule): + def __init__(self): + super(Foo2, self).__init__() + + @torch.jit.script_method + def forward(self, x, y): + return (2 * x + y, x - y) + +foo = Foo2() +traced_foo = torch.jit.trace(foo, (torch.rand(3), torch.rand(3))) +traced_foo.save('foo2.pt') + +class Foo3(torch.jit.ScriptModule): + def __init__(self): + super(Foo3, self).__init__() + + @torch.jit.script_method + def forward(self, x): + result = x[0] + for i in range(x.size(0)): + if i: result = result * x[i] + return result + +foo = Foo3() +foo.save('foo3.pt') + +from typing import Tuple, List + +class Foo4(torch.jit.ScriptModule): + def __init__(self): + super(Foo4, self).__init__() + + @torch.jit.script_method + def forward(self, x: Tuple[float, float, int]): + return x[0] + x[1] * x[2] + +foo = Foo4() +foo.save('foo4.pt') + +class Foo5(torch.jit.ScriptModule): + def __init__(self): + super(Foo5, self).__init__() + + @torch.jit.script_method + def forward(self, xs: List[str]): + return [x[:-1] for x in xs] + +foo = Foo5() +foo.save('foo5.pt') + +@torch.jit.script +class TorchScriptClass: + def __init__(self, x: torch.Tensor): + self.x = x + + def y(self) -> torch.Tensor: + return self.x * 2 + +@torch.jit.script +def foo_6(x: torch.Tensor): + return TorchScriptClass(x) + + +foo_6.save("foo6.pt") + +# https://github.com/LaurentMazare/tch-rs/issues/475 +@torch.jit.script +class InputObject: + def __init__(self, foo, bar): + self.foo = foo + self.bar = bar + +class TorchScriptExample(torch.jit.ScriptModule): + @torch.jit.script_method + def add_them(self, data: InputObject) -> torch.Tensor: + return data.foo + data.bar + + @torch.jit.script_method + def make_input_object(self, foo, bar): + return InputObject(foo, bar) + +foo_7 = TorchScriptExample() +foo_7.save("foo7.pt") + +# https://github.com/LaurentMazare/tch-rs/issues/597 +class DictExample(torch.jit.ScriptModule): + @torch.jit.script_method + def generate(self, batch: Dict[str, torch.Tensor]) -> Tuple[torch.Tensor, torch.Tensor]: + return batch["foo"], batch["bar"] + +foo_8 = DictExample() +foo_8.save("foo8.pt") diff --git a/v2/dependency-patches/tch-0.24.0/tests/data_tests.rs b/v2/dependency-patches/tch-0.24.0/tests/data_tests.rs new file mode 100644 index 0000000000..f56796495d --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/tests/data_tests.rs @@ -0,0 +1,60 @@ +use std::io::Write; +use tch::{data, IndexOp, Tensor}; + +mod test_utils; +use test_utils::*; + +#[test] +fn iter2() { + let bsize: usize = 4; + let vs: Vec = (0..1337).collect(); + let xs = Tensor::from_slice(&vs); + let ys = Tensor::from_slice(&vs.iter().map(|x| x * 2).collect::>()); + for (batch_xs, batch_ys) in data::Iter2::new(&xs, &ys, bsize as i64) { + let xs = vec_i64_from(&batch_xs); + let ys = vec_i64_from(&batch_ys); + assert_eq!(xs.len(), bsize); + assert_eq!(ys.len(), bsize); + for i in 0..bsize { + assert_eq!(ys[i], 2 * xs[i]); + if i > 0 { + assert_eq!(xs[i - 1] + 1, xs[i]) + } + } + } + let mut all_in_order = true; + for (batch_xs, batch_ys) in data::Iter2::new(&xs, &ys, bsize as i64).shuffle() { + let xs = vec_i64_from(&batch_xs); + let ys = vec_i64_from(&batch_ys); + assert_eq!(xs.len(), bsize); + assert_eq!(ys.len(), bsize); + for i in 0..bsize { + assert_eq!(ys[i], 2 * xs[i]); + if i > 0 && xs[i - 1] + 1 != xs[i] { + all_in_order = false + } + } + } + assert!(!all_in_order) +} + +#[test] +fn text() { + let filename = std::env::temp_dir().join(format!("tch-{}.txt", std::process::id())); + { + let mut file = std::fs::File::create(&filename).unwrap(); + file.write_all(b"01234567890123456789").unwrap(); + } + let text_data = data::TextData::new(&filename).unwrap(); + for i in 0..10 { + assert_eq!(text_data.label_to_char(i), i.to_string().chars().next().unwrap()); + } + for xs in text_data.iter_shuffle(2, 5) { + let first_column_plus_one = (xs.i((.., ..1)) + 1).fmod(10); + let second_column = xs.i((.., 1..=1)); + let err: i64 = from( + &(first_column_plus_one - second_column).pow_tensor_scalar(2).sum(tch::Kind::Float), + ); + assert_eq!(err, 0) + } +} diff --git a/v2/dependency-patches/tch-0.24.0/tests/device_tests.rs b/v2/dependency-patches/tch-0.24.0/tests/device_tests.rs new file mode 100644 index 0000000000..213b70ea0d --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/tests/device_tests.rs @@ -0,0 +1,7 @@ +use tch::{Device, Tensor}; + +#[test] +fn tensor_device() { + let t = Tensor::from_slice(&[3, 1, 4]); + assert_eq!(t.device(), Device::Cpu) +} diff --git a/v2/dependency-patches/tch-0.24.0/tests/display_tests.rs b/v2/dependency-patches/tch-0.24.0/tests/display_tests.rs new file mode 100644 index 0000000000..12bd9f0709 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/tests/display_tests.rs @@ -0,0 +1,79 @@ +use tch::{kind, Tensor}; + +#[test] +fn display_scalar() { + let t = Tensor::ones([], kind::INT64_CPU) * (-1234); + let s = format!("{t}"); + assert_eq!(&s, "[-1234]\nTensor[[], Int64]"); + let s = format!("{}", &t / 10.0); + assert_eq!(&s, "[-123.4000]\nTensor[[], Float]"); + let s = format!("{}", &t / 1e8); + assert_eq!(&s, "[-1.2340e-5]\nTensor[[], Float]"); + let s = format!("{}", &t * 1e8); + assert_eq!(&s, "[-1.2340e11]\nTensor[[], Float]"); + let s = format!("{}", &t * 0.); + assert_eq!(&s, "[-0.]\nTensor[[], Float]"); + let s = format!("{}", &t * (-0.)); + assert_eq!(&s, "[0.]\nTensor[[], Float]"); + let s = format!("{}", &t.eq_tensor(&t)); + assert_eq!(&s, "[true]\nTensor[[], Bool]"); + let s = format!("{}", &t.not_equal_tensor(&t)); + assert_eq!(&s, "[false]\nTensor[[], Bool]"); +} + +#[test] +fn display_vector() { + let t = Tensor::from_slice::(&[]); + let s = format!("{t}"); + assert_eq!(&s, "[]\nTensor[[0], Int64]"); + let t = Tensor::from_slice(&[0.1234567, 1.0, -1.2, 4.1, f64::NAN]); + let s = format!("{t}"); + assert_eq!(&s, "[ 0.1235, 1.0000, -1.2000, 4.1000, NaN]\nTensor[[5], Double]"); + let t = Tensor::ones([50], kind::FLOAT_CPU) * 42; + let s = format!("\n{t}"); + let expected = r#" +[42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., + 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., + 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., + 42., 42.] +Tensor[[50], Float]"#; + assert_eq!(&s, expected); + let t = Tensor::ones([11000], kind::FLOAT_CPU) * 42; + let s = format!("{t}"); + assert_eq!(&s, "[42., 42., 42., ..., 42., 42., 42.]\nTensor[[11000], Float]"); +} + +#[test] +fn display_multi_dim() { + let t = Tensor::ones([200, 100], kind::FLOAT_CPU) * 42; + let s = format!("\n{t}"); + let expected = r#" +[[42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + ... + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.]] +Tensor[[200, 100], Float]"#; + assert_eq!(&s, expected); + let t = t.reshape([2, 1, 1, 100, 100]); + let t = format!("\n{t}"); + let expected = r#" +[[[[[42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + ... + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.]]]], + [[[[42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + ... + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.], + [42., 42., 42., ..., 42., 42., 42.]]]]] +Tensor[[2, 1, 1, 100, 100], Float]"#; + assert_eq!(&t, expected); +} diff --git a/v2/dependency-patches/tch-0.24.0/tests/foo.pt b/v2/dependency-patches/tch-0.24.0/tests/foo.pt new file mode 100644 index 0000000000000000000000000000000000000000..ae7b294023efae893057fb496fcafb0907c15743 GIT binary patch literal 2030 zcmWIWW@cev;NW1u09*_b3~BlK`YDMeiTVa^P9h9ZSO5cqhEsqyI|o2A! zF8`vG)S}cBm(1i6u0jsYppu{nP&k(qC1&Pj=A{>Mc0`B)`SF>>@oA+wIq^w}$=T(J zMJe$a`T5y}Tz;iVh1`0ez?D77o$(N;5`;m4D*}u#%or18kO2p7a(+sxK0HwL3M$>4 zByfhZum9e>Lk0r(D!YV(b#@BAlIZNayGrkFKxt9ZZWkH5l5O03Y_~CYy#4*PeBOy8 z9P`iI6~FD{$E(e7WU^p!$w^O_7Tq^>v#rwJ&%gBRen43^@6~JxwjYIxn?w{lXC9XM z*JWlDy!O0ct=gfBKJA+YyQ`l$f86h%`79wLy!or)YnzH`vtBMzwx9Cyh|ycY-$g6+ z1RG8#=PbO=9{S;1VY_p2LGZfVulFKu>%N$=a(6~WxYW8;N2kf>EadyvYZ4iAN+|Bv zj*H%5Yx6+CG|}~f_X%JS*#P4ScQDB_Xk!m1y%b=|NRJ06OLd$9hL#RnCq?I9HV`>J zIc}TT?k&>W8(W_2-7uxJa|`6%P*FC;>>A|U4>T7pOy$umC=Xj-f_w|IsOp6{ii>fGo?cu%ifBwe$CA-_1a50oRXEu3H<}USDX`%hRp(UF;orxGQJt)eWB#Wv58GJ9tN&UzcLBSHEXJD3vA0oDY8o zOktCNr2y`dPLM$un#%HuOA_-+iXp`vKYF@qVA2F<@mzh*aKhy$H-j_S)Uwo~Vqg*D z=46j<9$NAw$Tj8y`RU@k^iU>X&H>>7Z$=OWuM?0j$dM$EV#E!g=dc=q$o}X)L5>?i6vOO*)eOWi zq*zARjvTn$DB7n$wL_ybz?+Rt2dY4hSr@L94a$PiRlraH0U@XWP#>uFV(0;r3=AOY R0B=^XEE@wG14unYEdYpla1{Um literal 0 HcmV?d00001 diff --git a/v2/dependency-patches/tch-0.24.0/tests/foo1.pt b/v2/dependency-patches/tch-0.24.0/tests/foo1.pt new file mode 100644 index 0000000000000000000000000000000000000000..0a6cc41e738bb2548be475e95b311c92f84c3250 GIT binary patch literal 1659 zcmWIWW@cev;NW1u0K5#M3~BlKhWaUqC5d_k**R`bf(%jUpn)klKE5QsC^;iOp34oW zoU4#Qv$0kq0%T%IQDSCZW?p(BV@HG-kRP8}9G_O2lM|nmn4DdnSdQ`EC)DzR8z=lFNEg;@kJZ zFHU+!s~eov$~_>UuJ-+w)J8duBf0GnOWRLPWt_okv*q)V-LZAs8V*0zK6-5f-$dCQ zX|0tZ`&Jv?s?mEGGG&9vO?FV&7rKF_obi(u1>S&6=X{8m;((lSOZ3g$IJw8UveRCsu&i2lp z-*2qXAXCAXHt9rT(T5Y4dj8aY{#>6^zxFZn=}q(Y%I?|s<zzGz#J4GCLdZ_#Wf^k2v=*s5 z?dtMSz3VZ-K`?&#hvShxLN{mo{M1_a@~m0>;l6pk9fk8O7_FRjLSscwD+lbqg!=6JebKalJY5 z?&s1^_*KfxY`^Noeg9|l%=*t$dgg2}wvXKXCF$GFTepw8|F3Rx$x+&*;lX2F{knR` z%HMLHsu5PZ9+->VFF(*V?f)G6#EZ*YZX}+YbmRJYpBW)$`(M@+bVxO@h8@g5t5#gQ zco~D~3TI}W%Nv+}v+n1ul@JNge(-xwqUf~**;yMH&2kQjxbv`@AVf||ID7IDPpfvRS(QR>>OKP{S&^;$iR@v49QQ}v$_z2 zDlE<96_+IDl@vqryCN)=Amn+a literal 0 HcmV?d00001 diff --git a/v2/dependency-patches/tch-0.24.0/tests/foo2.pt b/v2/dependency-patches/tch-0.24.0/tests/foo2.pt new file mode 100644 index 0000000000000000000000000000000000000000..5c611c168db1f38c60a91bb272991c69c04016af GIT binary patch literal 1723 zcmWIWW@cev;NW1u0K5#M3~BlKM*1m+pkwp>E@)1=|*%nBlAQ1 z4Eqlm@U)iKwH|kSwNN;D$?l1~8#jcW@S4q{KACclLBY@M%w1J=LBW3uto5_z@BKe%(hS3%hfGpcX{# zKGFXcOD8rjT-Q{3-HLzSQX}7zC#97WBHFYcZDY&gX8v%cph$7kp5s4_wSN3;51x0T zboG9z^t}x~qK(W>?_ImG?Cramv(mY(uO5ndrF-m7cwt!9O_AxFT8a`rGl2q4x88R3 z0bqd1GU5v~1qLG=fu@&|npBz|4~}9z(t6^``nI=!JxZ*5sYw}u7YD?dHotZTTN=DNZJ~P~P z7SE7BE7I|+%Or#Oe)s~>Wx`2E^Nx8attnJFbY^pl{5Q$%EDxKa?UdIk^mRSl@amRA;xgT*&f#z3EAY-c7B(!^Z1xT{^b? z-!`4PI;UgHuE)+<6a1s;Mex>q>Gyx%Cm-owrt|f##r1RhfAQsMy;Bs|@j7<=OV@&M z-wS^}eX_iK;=hFO;nRyO3a(f`?%-~YGrm9Dg~yTE_tNXnlkDRU?_spMvVbp&&xFCV ziaGAUPL|k1+gUd-t=_=q8^O+OmLMFpfi?HVfp1Bd`e1mQhK@K$yDaO$CBL@@@ivB4u{m`fh@MdGv xfvS*W)`hEOg|c9@JJ1&(AOsZv>SG7-BiTSC0|Q7pz?&7!V`X4v0I7$l1px3$B^3Yw literal 0 HcmV?d00001 diff --git a/v2/dependency-patches/tch-0.24.0/tests/foo3.pt b/v2/dependency-patches/tch-0.24.0/tests/foo3.pt new file mode 100644 index 0000000000000000000000000000000000000000..b82ffbf335ef71b613ff166e5217230f191196c5 GIT binary patch literal 1723 zcmWIWW@cev;NW1u0K5#M3~BlK#`-CVC5d_k**R`bf(%jUpn)klKE5QsC^;iOp34oW zoU4#Qv$0kq0%T%IQDSCZW?p(BV@HG-kRP8}9G_O2lM|nmn4DdnSdkY51AtBw0XhxSZDI_D5T_;Qr=;q`?bj=)baT?hbR)W(k@=0=1f=Wt>KxrF`a8~axI zNhE(;X|i%zX=G!v&~(0EpPuku5`XqQvm83Gdd+`JIZtcQz>Pw%QBjRrM?DN>|s<-;~L|>`bDgJtHWL|Hj7aya$Hw zIYxY8tia%mBaHP@Qj<#4F$#24YMMDBy0@z&6c==X6 zb#>g(b?aM>$Td;sT`eXv-};WgV!8bx?*DC6I|8w2y5OU;)&-ngdH z?@+Cl^TcAS$uC8`gT(DVPnz95i^>1Ol4tV9aV$*JC!3nQ&;PMzYD~Pyj0X`WOI`Xe z{JHzNr+~v9SAu1xMjkFyU2Lo6xmo(;9GeG9lOj9>pUR{QE&p}%k#ASPw{}|&m=Yj))Zyvo&yC$E+u;YOv_pJ+! zdKSypuxEFzJtW2>vV-g2#Lydz0S`2P_)q)ktX-e-wEyc*-ai2sf;K+S3Oul{XikUY6j@P2PZ;K%CU zY{|1z%KY=r|ICd3zv*QD>_4R^Z=Uxm^K4ZIr6sxT6Qp(n)6pF!NLs>Pln61f!qQS+ zaY8=DSPg&eak rTrDe<1*6@8z5oFsr~ptOJBT021|k_4K+*x;tY981Fdc!^L(~EQYi%fc literal 0 HcmV?d00001 diff --git a/v2/dependency-patches/tch-0.24.0/tests/foo4.pt b/v2/dependency-patches/tch-0.24.0/tests/foo4.pt new file mode 100644 index 0000000000000000000000000000000000000000..420a3d2d374e6f5ab8123626bb551227491cfe5e GIT binary patch literal 1595 zcmWIWW@cev;NW1u0K5#M3~BlKCi*FfC5d_k**R`bf(%jUpn)klKE5QsC^;iOp34oW zoU4#Qv$0kq0%T%IQDSCZW?p(BV@HG-kRP8}9G_O2lM|nmn4DdnSdsaYZ=gyL4gfk$1n4wOw}~+rLY$VIpOUH%w_mTI(#=U1(~anEM&^h1 z*=92v^0a;b!_}LkJ+{z5eNS@qy=) znCJD;--J4rt8H7ek#EkCXFMl2UHAFTIr1q|)?waO@hB7IKhK^zA?Gb=X0qZMnGqS>YXmO{-UI4HOhV+A6wd z%46270}h-UeV5I>vgM?Y--dU0&aVE+rxy~lJD~6(YoSu>GlLKE`#$p)Kiig}FW|&? zE=ryE^xljp_jfz5X3dk^6_)(IV!rU zo*F5&tv}k%E6g`-sJJY@qwqm&Ysl+mjxTm+%{K4|cwF3;9FW7vsPywseaiE`RAvna zK`#fRg*?`i!XF5=v^2F;n@yFEVBPh=Q^Tv`%tv*zuk%CipZ_(fW%``Yt11|nIa4|F zE&t8?Y+v*z_@l&*7D>){%YSn}*B8D2=O_DQ)|$VJplIGP{n(`>U=*`4L82LZjum2% zgGF;*aY>^`3=VQ_-jP?lO$oSC2J=46M* zY~*}~i*3vW^54aI>7hWcF@SJ@HzSCGmr2M)0xw7c1)!8r5Uq$Z137nq${z%n4P=6~ zVw520W*`T>0>S_w>ohPzu$zI%Rp_2U4rU<~(>5`~O+$)2bp6Ny#e1Q=yt*Fd8t zz?+Rt2dYAjSr@LB70QCq?m%CFz$8%20|8JUJBYpsRREF>@MdKLiLnA9NIgU?0Ki`9 AtpET3 literal 0 HcmV?d00001 diff --git a/v2/dependency-patches/tch-0.24.0/tests/foo5.pt b/v2/dependency-patches/tch-0.24.0/tests/foo5.pt new file mode 100644 index 0000000000000000000000000000000000000000..83787825f30c42443261a683c9b894796d88028c GIT binary patch literal 1531 zcmWIWW@cev;NW1u0K5#M3~BlKrur#~C5d_k**R`bf(%jUpn)klKE5QsC^;iOp34oW zoU4#Qv$0kq0%T%IQDSCZW?p(BV@HG-kRP8}9G_O2lM|nmn4DdnSd+pkwp>E@)1=|*%nBlAQ1 ze6tT5@U*@ED{6IX!nCvv-U-(4w)D#jG!+Q#H2u0H!BWS*-Y>|k>dI>Os(Xr7p%ucF zX7WboCgv0~ILwJDx<5rzQ)!RG?c7ZJ>i+^ypD?lSb70=<5wmc?i{}y>%MZUjzEp6% z()LaNXB7#QZHztmwM>OS?$RlVr7vbHotV_7{U}p6M3gPzBHO*@XJN&!bY$ZHFHd9s z5Oc~R?WEr1pV47U7u;=l#IQ!uK;^i&(d@?}oo5$ad*|P^ye4wX(OtXQe}qh$TYA~W zM#?7qyF;?X2_4ZIu3*;Vx8ALCsq~A{lC5I|1*?gesOBSJkVY`#3swaN0UW`qmy()P znjQ~M1Ux7K?CW>h?~sGYT(S5s+;?X9#LSrGv07R@M}(J&?`vbPiQ-w7TL&d|b&ZRq z3!mrNU5?*qy5i2ACr_T(6xqGakXI}zw&$`^l;uBUSIgzzyjzafI{ffDm%NaZ9#Iz^ zS$FL6)(APYH=J>v-;@ZZSc3LA>mT|mYuZ`=4_@0p24u|02>4id zdt&d4g4|;fh8m%L>Z?6>mx^8Y+;d22z0*eJ?TbF2dht2w;Gdi*t!d5wl^UiS%M0BL zy2ZZC@porZaKe^buHvqzjNa=tL+2+i?3%*5}a+qv5Gf(NqWj;n+Zn*PO~f*%b(PghD(O`%t2AoG*yMq6c`=zp;3Z86ACeK!=fawxFj*Jq!^MD z*QTAAUdA;lEBe&m4SLD7E&rXLzL0p4tEI#3mI%(`&3tWXw= kb_e@4+KDc>>zq3Q~^jjz?+o~B*qGaAoUQn05i+DP5=M^ literal 0 HcmV?d00001 diff --git a/v2/dependency-patches/tch-0.24.0/tests/foo6.pt b/v2/dependency-patches/tch-0.24.0/tests/foo6.pt new file mode 100644 index 0000000000000000000000000000000000000000..a87986635012200634c08edf013b3e441042cdfc GIT binary patch literal 1659 zcmWIWW@cev;NW1u0K5#M3~BlKX8I|KC5d_k**R`bf(%jUpn)klKE5QsC^;iOo+}_H zF*!9OKPM%%$TvTwG$)m-kU_JtRwDvrd`VGaW?p7qdLd&+X;LARUVt|{hXq68Q!Ahx z2nPUNA_8;?rc=ZioFFbq&QD3zhg+vtQ0eAmhv_7AC*tL|PB@!&*g&A=eO;IBtqHTX zxI0(B+_^O1zE`A3`#zT%YnK*%(s47MCI2EJd2QeHMiGtE)d!v5daTyFwI_9_ zYmUBA_(cW*s~0CNPV?IA(Ror^Wv|*lx!!l~?ls3sdG_mO3G%)be0_V;vI05f<;O~e zWec18r)_BZ)O+%LPEv^K=kvR+sT_!Z`}kG;nV)Zpzi++z{#ro&j1@UPEoMjR5+l_! z`y$(SE6$8!;xc~7Reti!w=(nBCjVBhtYZX)dESSr#CO2(mSw^h<_ZifIKo^nB{iuu zJszAO7#MOVczYjq5NTDrWXi?w{Eaz?S6i;aoHV%Oat?E+k&r6d6%tn_*=JjbAh?&x)diabC1YS zqgBQi+=7xf7DUC`IdV0NMhbZ?UN&u|ZbZ!ega2J-sc19Xt>!7V`XN!SW9Pc<*qmLq z-AwmveSe5^sCHH+p*&M3&r(&zUD}1Nk2Q`_DS{WA7HLw=ddiB zC)2>lz~BIlQSA9yh`|9Cqj|+8iFqZ(kW6huQ4VTg(gf$^Tz$@PU_k;*HIP(=-3>gj zoLrV#RGgWgM|u7=<^qMl#d+zWK+iLPaDX=>h=P}C$Yl&KNCE|*lyDHOh(ZWCtB9d! z>jLt?S}}?wbTg16M*+o*f50fiZU!QYp?d~7n1xVGyTA-L4Jmrj^&SFB<2?6q;eH9Xg1brM1agMDN4-D%gjqJWbB9#1M=fDi{sNub8_O75|gvb z6N^&fGxGDZ3z_^%lM0#j0=(Hd>bIt_L;_WUZ~)MuB0z^?x>Sro8RF37{FGFExNGzZ zD&3sqF7pqQ zw>^1k`}?Qm)+3c-OU+8Zht00r^(@mwE0r=Kb^4ZfCcCdeh-|J3xCw=7BPkW5y@9 zJ>a|clj+32-`42u@yoT#&u#-dD!A(sy&FX^V!tXJ*JVy#H3vch%-e&X*6i%mF2g z9sR)(SHG|6Q}my4hI`W)zT~A!H_!1_hWwV_Sp47+cidvr9cQmT{S;j(e|*`cUdugM zu1~+ayneARZkwL|o3qZZKb2J%UwXj~ikmm$rdps)P_o>?|3ry@od>w8C2ruDEc1pa|wd&l4oIg+HcOLxC z?zsAgjO#^K;nMjzb8qz66r9~)E1WZV|32xQ4L#+|)Wp;2U=eBiM`)vRD?!cDI zzfJF#Pf#v@w`$q>6@UNnon)8W@a3EFkxd@wy-Ox-(=axka%z#HyWN+%N)O|N16yjY zh!-D@)qXFs;Xs?j)3}VEEPe_X6J8hO@PFX0 zZp}Yv_9o`>{;f4xr$P>Uv`=TA^ThaVJR>NPZ0ykModrxJozO&ry`&LhP=h6syyB9? zypm!_aic^;vS?t^1Q!*#`kdjwf{X!#K}7|2n|K(UAU2hy78Pga=earA5n%^%dP8R$ zbAkMMab9{T(2EQp9N^6eqTppJav{bGl0X3{B`icMqLf3(+`cB0B<%n9jFR9 uW?i^iRwxTby91LK2n0a|fcn@${F!VZl7Rsv9pKFh=CLxcGJw=W)B*r!f<@E- literal 0 HcmV?d00001 diff --git a/v2/dependency-patches/tch-0.24.0/tests/foo8.pt b/v2/dependency-patches/tch-0.24.0/tests/foo8.pt new file mode 100644 index 0000000000000000000000000000000000000000..e6bee10da87fbd9e15e8c1abae6798d1b7a1be35 GIT binary patch literal 1595 zcmWIWW@cev;NW1u0K5#M3~BlK7WyfPC5d_k**R`bf(%jUpn)klKE5QsC^;iOp35aO zxx}?1F}ENmm8+0Jv$0kq0%UMWQDSCZW?p(BV@HG-kRP8}9G_O2lM|nmn4DdnSd|5Emxrr=;q`ouOAy>E@({ z=~Q&5gZZI-j#)#^;MsGjPUb6Iz&e#9d zFWP&y9*XAr^{;okY4HDp#ivbX2sadKbKUdsTvhymsmv)&WSv9YGs-F%gb`s+vZIzM|VEoo%r||Q^l1H zGmeDKJ>#2qwLh)TGk%cyQdB5*c7wVZk;O>^Mi+{4lwOk0Ea_f%G z7lp7tTFIhW2OKyT`fk6QF)O89ojw1an7^EeP34977bm-w@^VyB(M zO(i*>x}NKsBwKAIP4cabIGtbReSeS?Gxd-3W-09Cb&NI^*I|Tcl5D>mqDM( zteUgma=U6397N*(>$>jS+{D1F6~G$Ak)=s%DB4`Nj{5Z9P?IP;|Im>zZw03k-)gU1&v?q8C8fu9TvZN z#U+V(CB=~3s!U9bHZW;|b6l=IXE?ARU;trIj>B#h4}%lLs`R!Hzzw{EI~^% z$XsJCkVh}hOAloN+6=-0-i#m$UM3-z1-u{$6o67fL9`+Y3*`I(DtHiJ5s(Siicxf+ zn}Hk$3J3#$tdqc4!EOd3U!i*jIhch|Ok2qeHw`Hk(e)z-6c37i5nz;oT?37p0B<%n w9jFR9W?i^iRwxTby90dz0+T>74+KDc>>&CkQ~^jjz?+o~B*qGaAoUQn0LF3To&W#< literal 0 HcmV?d00001 diff --git a/v2/dependency-patches/tch-0.24.0/tests/jit_tests.rs b/v2/dependency-patches/tch-0.24.0/tests/jit_tests.rs new file mode 100644 index 0000000000..08d3c36e9e --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/tests/jit_tests.rs @@ -0,0 +1,190 @@ +use std::convert::{TryFrom, TryInto}; +use tch::{IValue, Kind, Tensor}; + +mod test_utils; +use test_utils::*; + +#[test] +fn jit() { + let x = Tensor::from_slice(&[3, 1, 4, 1, 5]).to_kind(Kind::Float); + let y = Tensor::from_slice(&[7]).to_kind(Kind::Float); + // The JIT module is created in create_jit_models.py + let mod_ = tch::CModule::load("tests/foo.pt").unwrap(); + let result = mod_.forward_ts(&[&x, &y]).unwrap(); + let expected = x * 2.0 + y + 42.0; + assert_eq!(vec_f64_from(&result), vec_f64_from(&expected)); +} + +#[test] +fn jit_data() { + let x = Tensor::from_slice(&[3, 1, 4, 1, 5]).to_kind(Kind::Float); + let y = Tensor::from_slice(&[7]).to_kind(Kind::Float); + let mut file = std::fs::File::open("tests/foo.pt").unwrap(); + let mod_ = tch::CModule::load_data(&mut file).unwrap(); + let result = mod_.forward_ts(&[&x, &y]).unwrap(); + let expected = x * 2.0 + y + 42.0; + assert_eq!(vec_f64_from(&result), vec_f64_from(&expected)); +} + +#[test] +fn jit1() { + let mod_ = tch::CModule::load("tests/foo1.pt").unwrap(); + let result = mod_.forward_ts(&[Tensor::from(42), Tensor::from(1337)]).unwrap(); + assert_eq!(from::(&result), 1421); + let result = mod_.method_ts("forward", &[Tensor::from(42), Tensor::from(1337)]).unwrap(); + assert_eq!(from::(&result), 1421); +} + +#[test] +fn jit2() { + let mod_ = tch::CModule::load("tests/foo2.pt").unwrap(); + let result = mod_ + .forward_is(&[IValue::from(Tensor::from(42)), IValue::from(Tensor::from(1337))]) + .unwrap(); + let expected1 = Tensor::from(1421); + let expected2 = Tensor::from(-1295); + assert_eq!(result, IValue::from((expected1, expected2))); + // Destructure the tuple, using an option. + let (v1, v2) = <(Tensor, Option)>::try_from(result).unwrap(); + assert_eq!(from::(&v1), 1421); + assert_eq!(from::(&v2.unwrap()), -1295); + let result = mod_ + .method_is("forward", &[IValue::from(Tensor::from(42)), IValue::from(Tensor::from(1337))]) + .unwrap(); + let expected1 = Tensor::from(1421); + let expected2 = Tensor::from(-1295); + assert_eq!(result, IValue::from((expected1, expected2))); + let (v1, v2) = <(Tensor, Tensor)>::try_from(result).unwrap(); + assert_eq!(from::(&v1), 1421); + assert_eq!(from::(&v2), -1295); +} + +#[test] +fn jit3() { + let mod_ = tch::CModule::load("tests/foo3.pt").unwrap(); + let xs = Tensor::from_slice(&[1.0, 2.0, 3.0, 4.0, 5.0]); + let result = mod_.forward_ts(&[xs]).unwrap(); + assert_eq!(from::(&result), 120.0); +} + +#[test] +fn jit4() { + let mod_ = tch::CModule::load("tests/foo4.pt").unwrap(); + let result = mod_.forward_is(&[IValue::from((2.0, 3.0, 4))]).unwrap(); + assert_eq!(result, 14.0.into()); + let v = f64::try_from(result).unwrap(); + assert_eq!(v, 14.0); + let named_parameters = mod_.named_parameters().unwrap(); + assert_eq!(named_parameters, vec![]); +} + +#[test] +fn profiling_mode() { + assert!(tch::jit::get_profiling_mode()); + tch::jit::set_profiling_mode(false); + assert!(!tch::jit::get_profiling_mode()); + tch::jit::set_profiling_mode(true); + assert!(tch::jit::get_profiling_mode()); +} + +#[test] +fn tensor_expr_fuser() { + tch::jit::set_tensor_expr_fuser_enabled(true); + assert!(tch::jit::get_tensor_expr_fuser_enabled()); + tch::jit::set_tensor_expr_fuser_enabled(false); + assert!(!tch::jit::get_tensor_expr_fuser_enabled()); +} + +#[test] +fn jit5() { + let mod_ = tch::CModule::load("tests/foo5.pt").unwrap(); + let result = mod_ + .forward_is(&[IValue::StringList(vec![ + "foo".to_string(), + "bar".to_string(), + "foobar".to_string(), + ])]) + .unwrap(); + assert_eq!( + result, + IValue::from(vec![IValue::from("fo"), IValue::from("ba"), IValue::from("fooba")]) + ); + // Destructuring of ivalue. + let (v1, v2, v3) = <(String, String, String)>::try_from(result).unwrap(); + assert_eq!(v1, "fo"); + assert_eq!(v2, "ba"); + assert_eq!(v3, "fooba"); +} + +#[test] +fn jit6() { + let mod_ = tch::CModule::load("tests/foo6.pt").unwrap(); + let xs = Tensor::from_slice(&[3.0, 4.0, 5.0]); + let result = mod_.forward_is(&[IValue::Tensor(xs)]).unwrap(); + + let obj = tch::jit::Object::try_from(result).unwrap(); + let result = obj.method_is::("y", &[]).unwrap(); + assert_eq!(result, IValue::Tensor(Tensor::from_slice(&[6.0, 8.0, 10.0]))); +} + +#[test] +fn create_traced() { + let mut closure = |inputs: &[Tensor]| { + let v1 = inputs[0].shallow_clone(); + let v2 = inputs[1].shallow_clone(); + vec![v1 + v2] + }; + let modl = tch::CModule::create_by_tracing( + "MyModule", + "MyFn", + &[Tensor::from(0.0), Tensor::from(1.0)], + &mut closure, + ) + .unwrap(); + let filename = std::env::temp_dir().join(format!("tch-modl-{}", std::process::id())); + modl.save(&filename).unwrap(); + let modl = tch::CModule::load(&filename).unwrap(); + let xs = Tensor::from_slice(&[1.0, 2.0, 3.0, 4.0, 5.0]); + let ys = Tensor::from_slice(&[41.0, 1335.0, std::f64::consts::PI - 3., 4.0, 5.0]); + let result = modl.method_ts("MyFn", &[xs, ys]).unwrap(); + assert_eq!( + Vec::::try_from(&result).unwrap(), + [42.0, 1337.0, std::f64::consts::PI, 8.0, 10.0] + ) +} + +// https://github.com/LaurentMazare/tch-rs/issues/475 +#[test] +fn jit_double_free() { + let mod_ = tch::CModule::load("tests/foo7.pt").unwrap(); + let input = mod_.method_is( + "make_input_object", + &[ + &Tensor::from_slice(&[1_f32, 2_f32, 3_f32]).into(), + &Tensor::from_slice(&[4_f32, 5_f32, 6_f32]).into(), + ], + ); + let result = mod_.method_is("add_them", &[&input.unwrap()]); + let result = match result.unwrap() { + IValue::Tensor(tensor) => tensor, + result => panic!("expected a tensor got {result:?}"), + }; + assert_eq!(Vec::::try_from(&result).unwrap(), [5.0, 7.0, 9.0]) +} + +// https://github.com/LaurentMazare/tch-rs/issues/597 +#[test] +fn specialized_dict() { + let mod_ = tch::CModule::load("tests/foo8.pt").unwrap(); + let input = IValue::GenericDict(vec![ + (IValue::String("bar".to_owned()), IValue::Tensor(Tensor::from_slice(&[1_f32, 7_f32]))), + ( + IValue::String("foo".to_owned()), + IValue::Tensor(Tensor::from_slice(&[1_f32, 2_f32, 3_f32])), + ), + ]); + let result = mod_.method_is("generate", &[input]).unwrap(); + let result: (Tensor, Tensor) = result.try_into().unwrap(); + assert_eq!(Vec::::try_from(&result.0).unwrap(), [1.0, 2.0, 3.0]); + assert_eq!(Vec::::try_from(&result.1).unwrap(), [1.0, 7.0]) +} diff --git a/v2/dependency-patches/tch-0.24.0/tests/linear/bs.npy b/v2/dependency-patches/tch-0.24.0/tests/linear/bs.npy new file mode 100644 index 0000000000000000000000000000000000000000..0c41779d5fa65cfc5af4dc343840765e3b8007a0 GIT binary patch literal 144 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlV+l>qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= mXCxM+0{I#yI+{8PwF(pfE{iO8+h=~yY^v5B*<(=HY6AdA10vx7 literal 0 HcmV?d00001 diff --git a/v2/dependency-patches/tch-0.24.0/tests/linear/gen-test-data.py b/v2/dependency-patches/tch-0.24.0/tests/linear/gen-test-data.py new file mode 100644 index 0000000000..1a8b86094b --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/tests/linear/gen-test-data.py @@ -0,0 +1,16 @@ +# This generates some test weights for a linear layer, as well as a sample input/output pair. This +# is used to demonstrate that the implementing a linear layer with `Tensor::matmul()` produces +# outputs that are close to, but not exactly equal to, the outputs produced by `Tensor::linear()`. +# +# The tensor sizes chosen are as small as I could make them while still reproducing this issue. + +import numpy as np +import torch + +linear = torch.nn.Linear(257, 4) +input = torch.rand(2, 257) + +np.save('ws.npy', linear.weight.data) +np.save('bs.npy', linear.bias.data) +np.save('in.npy', input.data) +np.save('out.npy', linear(input).data) diff --git a/v2/dependency-patches/tch-0.24.0/tests/linear/in.npy b/v2/dependency-patches/tch-0.24.0/tests/linear/in.npy new file mode 100644 index 0000000000000000000000000000000000000000..b88392af3cbda25e078d4969c2fe26da6d5a7f13 GIT binary patch literal 2184 zcmbW2`CAS60>)2C9c7X&k;zeUibhFU>U`h#luD8oA}TFfs8m!qa)!esbY*S27)v*X zWVw+jLfIzNJQ>NZatWU|~z zCRfBb$Zcfun55*?7;fc8s7q{)kfX;E8Gb9PJ_Sg7gwt|{1gISQV0}|b-&d2&Z zg=&)z+#KDhZVkYpJuY;ti{%X?V?Nd#h1Rn^w7Fo3+vBHmU9J|(?(Id|+#G1{+6Rv> z4tx`+N53Q&mP@UL*P$QzWupa5ZaE_TWC3ym22yF1g8Ys~G#pK(4l6T@&q>Lsl4qOwP zz?!E+Q0w&^8;U-QjsMo>fFviBD~x$*@+>Ngtm*OfAwo)}9K23Ox4sRyYi&Wx?B7s% zU6051g<|}3JuZG1CajL6@K8`ZeboKAR#Jewg=3k0C7hj=oiJ|y8CUwx48 zTgPP_d!m7{YbXP6&f=zBm(i?mfIS&ysL)%@@Qvx>y2(ovOt7NEHwBd+UyAS|eK5WU zr>N#KFL3~k;!JU`#*{T2hT4+RjDJ0ieSgtqrD+U?e^asgnlD#0c~WW7lkacVBCW3( zb6j3vSVsbfJKOV{!F1+uG0QJK5K`y;qG5bG{2r^}^)ETMR&5cUC(nwb^KAJ@$Ax~{ zS7E*6BSzVl$Mr?)&;df=Z8Vux9%%?+X`rQb}2qRege0LXQ4SVjcTPiYJ5K zyJ*PS-F)p9DTTpd1Ih-hg{1M0v(w-vT%Gg*9&_3;uiBpuox{1iQO(vr97VyWtx)YP zLQCI?%-pQO_TrbwZ{I0=EtfL;R5Q*B#w)2okr-G zDDG-?WNe;0)B7EVU$crO#Y6a=+Mb$Q)y^~QLYNhr$fo#bs7wF~2MuMreF%+b+=6A~ zX|z}5;B4GH=1z%#^SBfi)o3xm&!66j%ebo2pZfmw$glWGScax>W=1fZ27W?w?PS=8 zE~1^4D@wdd#THd8v)&l+M3*H!MFiICB=g3~;k>XxLcNUd*{<~fFC-Iq%)%2@2d+Wl zW8r)-el;4km2@$Q;+hsC_TIUg64y0~uaOUMqk1)`)C93`%@t9jFT`=}NH)AERy0kH z$A7+;vVZX~u6Ur}CO;ciPEF+fRUd_FTa$2|pU6nz#hHU_|M z=_>@k@M77lW;EoT73KvGakcOWLdtqUy5s{+be=(%j~^EsW{Q?oo~-#;D)PI|!8u>T z!UiX-EsNvB@K?~hTrceWC*nv|FDjqev8=@esk2gO!CCCz1M>66J$ND+)jOAgfVHpQ|UJ*{4R(=AW?g#XPF&E3vFYLDLgb=KVUB zI|fZ=!Ix`)6WN+(#8>BTqx^6W(et-p zx*VNE<;ax+!A&TyYZ1rPj_~l^f`eLiln=Dz^{3a69d5#Z)=g%$n>Hmw1#?SpC<;3c zqqeRUW=AGcvOYwp_h_T!ahniJeRuM5M}MfN;5D2(CTcfq8a z<qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= zXCxM+0{I$7ItnJ5ItsN4WCJc6MmxKgg|2o&EC1SQ*d5+iAkS?lG0oNPe(D_C3GbHd F0|1FtDCPhF literal 0 HcmV?d00001 diff --git a/v2/dependency-patches/tch-0.24.0/tests/linear/ws.npy b/v2/dependency-patches/tch-0.24.0/tests/linear/ws.npy new file mode 100644 index 0000000000000000000000000000000000000000..f8c17d3d88b911ea8c918ac61f04fe312722caa9 GIT binary patch literal 4240 zcmbW3_dnN-`^AZ}NvP~uR0zF=_v^aKCKN?RiDZ_dj3T3@ke2SKv^7*(qQdLC+C$oV z*WJ*fTe`~~r9OTCh41<8oFAW$$2pJl*SLvegCb;Q=E~$)#>J<{rddi&EFnQ+X>DSe zkeZf}7LyX4nid!Tf4O(e%=Gv_d3sXJ?D#+4N@8u|;OI2O+9c29|0!j2F`b*9*TZx- zKThzC<}1CU_`fg<9+t6_3R+7*`9nF)?UjYE702)`w1z2gXMIR1Cux+^x zS_{6AacLmGJ`+RP@{hz95>?EvNaXUj)46x@I(lLjgl;n7?6<|O!6ITVT8x*ll*I7|}J5H8w9Lz2)iH_}va0vs_uNL7kkR9Txg8>7l*a4y+nl zD+q0u$l{(US2`qf)sRGAl46cJuD|;Z zj2)G5r@jtfOf|*7kNl$WA$E9D;x1;X*ivf1GJ3yt9(?X@f{x`iu*BGsbyCgfTHndI zpu&R>v2V)u+?0(q zD>8AIqaJ&XQQ%dw_h7`YS*X8xFt&_NLd1=fwJ42G-!($Nw_)u5IvMlssbgpGSpIla ziRJC0!N4F0Kh+$j*&AZ`c8NY}Whdi`i(SI~{h9pV@paVDzC?NofMuF-4 zwnm@H7En+c4-a-uWTo%Q_&lQ*YFZ7VFTW0wdM{u0iwfsYt>#$Q6v9*bjlu%A33%Z5 z7D^tn05klf_-Ifdo=#p3b*pW8jfOR54p&5Q20_7V3@9lZa!S;@2DxiZ&^_oqd0dUd zr#tnz?N%D!Nt%zDI{<6{(WIT*Rrx^l6po(ujFLi!Vb0yXlo>3Cj(=rhX5R!HFvAT~ zuNA}4{3d9+b(XemmZILWFutR8iOzrYhHpPYxgw>AESDMK=RqMD^Jbm2J0lv71Vmx= zgAw9o>n@>bVkjS%pNdN?1a^3#3ZK=|F-~I&kKq%d5W5HJG$*3(Ve3a)PozG^jj$rU^_95C#T3~bu#i&1kg!G(8;IMQ97jg~GGt3Omw)}Br9p}jZX zGEspU`q^Nbl!S7PDL6H@jk>3ra7Tg+w+Bju9WOsog41Z;vb$RdS6NLXJHA0o;c3ds zcVWLQ1I8L3T(Hmz4U|T3aQ;eCOnd}es|{(!-q9SH@|AS2W{ErWkAr1o3FzDDi0AiR zkcMcb(1AA-*grKCH%R2Ucuh9Wy1WvWbzXs)&Z+1*Vg~nV8_yRn`tbdQTC8LCiH^QD zM2o$Kth`}7pU)|QRW3=m$UGc;zU84`Uo~3jQ7nehdFs115ob+GLG2Uj7~U`*EfnQQ z-#8Rimkb5V6$4R!haP$F_HJxmyqWBUV*rWy&=c&3cI{hWuCum~os~rSvvvzDcLVrf zPY!zZOo3k>zZ)-&8OCLu3me0q9v1ht`LayhP4I6o2lL8YR18YNNCh?QsP{(u#x`1p zPMFkFCps8cQ@?;*a2E|(bCd@9|L8-%Vk3E9?QNPryPv3JE5hl<7O_Zg8PrDRaQgb& zkYka=mSuW4>c>;iF+NQn>}T@JAy?_a@)z`}*$B;=8|iI7Kc3vj1d~ThjWF&ODejo$+rg|q|ljm*r2kW;^jBc)M$SUUeKTSzYOEC zGFjL)REy=T&j|r4Hh5uo4wejZVJj^~LGxUH#Opi2F7qn|8Kgqh_|cs5vZL`tzf%yn zwcRCTzZ6SUs-WzM4z9aZ31zvuJTEDUWouVc*?>W8GiL}iJBA}=7_)nQJN)4%%@S5Y zxzPoRZC(sF4ZaDjDXthT?@L`nO35xZ0FGBR)1625_~=pw%XQk2-Xtj*{*qxM%VKI% z&PFW*C#+dpN*~{NU|PB#+AnrPStA=}FA)?!7|{(C3w--M3iB4W!bVkRoccwFea?uq zYNQ{eSM<=z&k`Oq?$7?YkLB(44BO`RBgnZf49YOaUWrzMn@nGs3NZL#3iVtOJJ0eN-6w?Ug=?87&3|3oVlm~4ZBL*L2fWf7b+vuDNi z135c|sQTF_xOOg!dal~=6@?^(x>2~Z!VV?p4mDOR=?1?Yf#Q>HQ;z&{9)!{TF+8JG zm{GqP{6f2E z?`Bi(iv0##JS?$#$b6y3Hj2N*TfxZ{y-?wOCT=T@z@Ht{dG-dtqLpjms3Z+r-@Cvy zu|JMD^9%+W%JA@#Lh$(hO?+@-14!(*!&Zf%_;2hUAvrcyC~hjG(WmY~O6q=C;8a6J zA-SkgvjOr~9}-_Ar*VXV4z5;9g@Q{B!lAM%(6$L7U;O~QV*3v*?PwLkG;{F!$q3FL zrt5qqZ3OBnyYf)mP)y5P3cWA&;!{nb6!1bHzcecG?u)^!H3_gQv{JmhGZZ|A-;?gu zPbFRBJ>VNNn+J?^<*OCbG2U)I2Ta>dHQ__?Qmhizs~v|?uC-uac^wXIPZMA6Hm1Zi zbz@=W+B_O-(Ji)go`k}K27D;c1V4P+L9Th}usU%7IPC8Nd+iWb zD_%t(gbc3t)B^MN{+1l4QDalh}YNcm46Y`3kVZqsjMmKi5Fua02t3uWM| zp~~xY?t$)}eC#>g0x@ahab=MobjqI+tux)wt5X%bj}?$n;23uQF%Pdy)uShO6L?^Y zCWnOrKT)*gb3U3`t+R{2BAyJ9lVekkZP?WSh!Go+iO zh?(h5V5PsGwm)|QrQh-xDqAB|wYCV;UVf$92S!1ZOg62Ena!J z*6$(NUzm(bGOMAe>JMY>`}Dj25!k#X1Gm@2@%_%}_{7i}Cn=@F(>!fl6BZ#Jo-$Ls z|7r$n5BKA1XXDuaK^(qaIGk>{{SXJtZ6_bIT(J2j3bTI3WAlYQ@Nw`eiY_*WJwLM9 zVuT($oSBAWe$av#G19ETwbEi$Cap^G|^xZWYosMhZ0Ly!%{L35Fwd!5wrwtH{|F%N*ze|TTHBqLo z2^*v&p#85+q+kZ((au5&bLuZX+vv)Hep_kh#x9r_na!(Bf4IyzQcC(;-LY@;66%%m zo&u}QAlLg5D2+Hu*8=qE=B#z}G+&MPy)l58b@5mjEYAkV75T;)q}J31I`exkocyfL z?~|gbHEF(!yUHfoX8V~9h}?b$%83>!ldQM(DqSTBIjPfz5!d9!)y zC@UW45`?*nE#drOb(EGnVyN{C*fM=IZfkeIzhw2$>VgK`7^uou7X1yP;Q@+=<4||f zmlt`~gM*|mk3Xu40aMXAGrYBa_4`KWz@_apTunNx0L~ksa^Mg*9M|1y=%j?ay3TtCx%!U+w9t zQ8zSxN7k~_<;?IxIC3MG+O=Asxn&nLdUMBGizd;zZ|b3CgvA#~d@m#AF zj@VQQ`OkIOW{j$+Bp1q-cG+A}JCw`%{o8PQX#)OSHwxp-C!tiTf;}yBcyRDZs+nlS z$)y7s4wSm+&oV&YLto+c#<}8S`vJUkuO|gm<%xez8?XA>3;GnBbIK|uJTR()B6Iuk zvR6`}DIlFr_Z~pgpD6HbsW+dL$)NXZWI_DLozFYT;KCwxtS)&)S0~nr*F5#{$MXK{ zzdanR=XQ~v!35MP&x1P?{{ipi-^r?GB+u+tG=j4X%N~ zRn>GO1USxiBx+6#X1RoM_%lqTy)c0<^$dghpKf^Pau6?B*BkRcBy!WKlVpFfQuMnq z8RaYbh(YQj!9*g%OB>Zed;BiCxiF1QU)O?ks0u%<{Rpc^$)f7j@uZ;Pg>H{a>9g=j zP;G4ktcclQ$WYLFQ8DUI&_wnlxwP2h9nKj++6 z?}WCJXuM=KlN*0LqYlehl2#svijLm6)o=oy)r{rhNL`F6IS$G;ui?TW4;+2aiq<@L Gr2hf`tBwEw literal 0 HcmV?d00001 diff --git a/v2/dependency-patches/tch-0.24.0/tests/nn_tests.rs b/v2/dependency-patches/tch-0.24.0/tests/nn_tests.rs new file mode 100644 index 0000000000..4fe9a27f60 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/tests/nn_tests.rs @@ -0,0 +1,412 @@ +use tch::nn::{group_norm, layer_norm}; +use tch::nn::{Module, OptimizerConfig}; +use tch::{kind, nn, Device, Kind, Reduction, Tensor}; + +mod test_utils; +use test_utils::*; + +#[test] +fn optimizer_test() { + tch::manual_seed(42); + // Create some linear data. + let xs = Tensor::from_slice(&(1..15).collect::>()).to_kind(Kind::Float).view([-1, 1]); + let ys = &xs * 0.42 + 1.337; + + // Fit a linear model (with deterministic initialization) on the data. + let vs = nn::VarStore::new(Device::Cpu); + let mut opt = nn::Sgd::default().build(&vs, 0.).unwrap(); + let cfg = nn::LinearConfig { + ws_init: nn::Init::Const(0.), + bs_init: Some(nn::Init::Const(0.)), + bias: true, + }; + let mut linear = nn::linear(vs.root(), 1, 1, cfg); + + let loss = xs.apply(&linear).mse_loss(&ys, Reduction::Mean); + let initial_loss: f64 = from(&loss); + assert!(initial_loss > 1.0, "{}", "initial loss {initial_loss}"); + + opt.set_lr(1e-2); + // Optimization loop. + for _idx in 1..50 { + let loss = xs.apply(&linear).mse_loss(&ys, Reduction::Mean); + opt.backward_step(&loss); + } + let loss = xs.apply(&linear).mse_loss(&ys, Reduction::Mean); + let final_loss: f64 = from(&loss); + assert!(final_loss < 0.25, "{}", "final loss {final_loss}"); + + // Reset the weights to their initial values. + tch::no_grad(|| { + linear.ws.init(nn::Init::Const(0.)); + linear.bs.as_mut().unwrap().init(nn::Init::Const(0.)); + }); + let initial_loss2: f64 = from(&xs.apply(&linear).mse_loss(&ys, Reduction::Mean)); + assert_eq!(initial_loss, initial_loss2); + + // Set the learning-rate to be very small and check that the loss does not change + // much. + opt.set_lr(1e-10); + for _idx in 1..50 { + let loss = xs.apply(&linear).mse_loss(&ys, Reduction::Mean); + opt.backward_step(&loss); + } + let loss = xs.apply(&linear).mse_loss(&ys, Reduction::Mean); + let final_loss: f64 = from(&loss); + assert!((final_loss - initial_loss) < 1e-5, "{}", "final loss {final_loss}") +} + +fn my_module(p: nn::Path, dim: i64) -> impl nn::Module { + let x1 = p.zeros("x1", &[dim]); + let x2 = p.zeros("x2", &[dim]); + nn::func(move |xs| xs * &x1 + xs.exp() * &x2) +} + +#[test] +fn gradient_descent_test() { + let vs = nn::VarStore::new(Device::Cpu); + let my_module = my_module(vs.root(), 7); + let mut opt = nn::Sgd::default().build(&vs, 1e-2).unwrap(); + for _idx in 1..50 { + let xs = Tensor::zeros([7], kind::FLOAT_CPU); + let ys = Tensor::zeros([7], kind::FLOAT_CPU); + let loss = (my_module.forward(&xs) - ys).pow_tensor_scalar(2).sum(Kind::Float); + opt.backward_step(&loss); + } +} + +#[test] +fn gradient_descent_test_clip_norm() { + let vs = nn::VarStore::new(Device::Cpu); + let my_module = my_module(vs.root(), 7); + let mut opt = nn::Sgd::default().build(&vs, 1e-2).unwrap(); + for _idx in 1..50 { + let xs = Tensor::zeros([7], kind::FLOAT_CPU); + let ys = Tensor::ones([7], kind::FLOAT_CPU); + let loss = (my_module.forward(&xs) - ys).pow_tensor_scalar(2).sum(Kind::Float); + opt.backward_step_clip_norm(&loss, 0.1); + } +} + +fn round4(t: Tensor) -> Vec { + let v = vec_f64_from(&t); + v.iter().map(|x| (10000. * x).round() / 10000.).collect() +} + +#[test] +fn gradient_clip_test() { + let vs = nn::VarStore::new(Device::Cpu); + let mut opt = nn::Sgd::default().build(&vs, 1e-2).unwrap(); + let root = vs.root(); + let var1 = root.ones("v1", &[2]); + let var2 = root.ones("v2", &[1]); + let mut var3 = root.ones("v3", &[2]); + tch::no_grad(|| { + var3 *= -1; + }); + let all = Tensor::cat(&[&var1, &(2 * &var2), &(2 * &var3)], 0); + let loss = all.pow_tensor_scalar(2).sum(Kind::Float); + opt.zero_grad(); + loss.backward(); + let g1 = var1.grad(); + let g2 = var2.grad(); + let g3 = var3.grad(); + assert_eq!(vec_f64_from(&g1), [2.0, 2.0]); + assert_eq!(vec_f64_from(&g2), [8.0]); + assert_eq!(vec_f64_from(&g3), [-8.0, -8.0]); + // Test clipping the gradient by value. + let loss = all.pow_tensor_scalar(2).sum(Kind::Float); + opt.zero_grad(); + loss.backward(); + opt.clip_grad_value(4.0); + let g1 = var1.grad(); + let g2 = var2.grad(); + let g3 = var3.grad(); + assert_eq!(vec_f64_from(&g1), [2.0, 2.0]); + assert_eq!(vec_f64_from(&g2), [4.0]); + assert_eq!(vec_f64_from(&g3), [-4.0, -4.0]); + // Test clipping the gradient norm. + let loss = all.pow_tensor_scalar(2).sum(Kind::Float); + opt.zero_grad(); + loss.backward(); + opt.clip_grad_norm(1.0); + let g1 = var1.grad(); + let g2 = var2.grad(); + let g3 = var3.grad(); + assert_eq!(round4(g1), [0.1414, 0.1414]); + assert_eq!(round4(g2), [0.5657]); + assert_eq!(round4(g3), [-0.5657, -0.5657]); +} + +#[test] +fn bn_test() { + let opts = (tch::Kind::Float, tch::Device::Cpu); + let vs = nn::VarStore::new(tch::Device::Cpu); + let bn = nn::batch_norm1d(vs.root(), 40, Default::default()); + let x = Tensor::randn([10, 40], opts); + let _y = x.apply_t(&bn, true); + assert_eq!(vs.len(), 4); +} + +#[test] +fn bn_test_no_affine() { + let opts = (tch::Kind::Float, tch::Device::Cpu); + let vs = nn::VarStore::new(tch::Device::Cpu); + let bn_cfg = nn::BatchNormConfig { affine: false, ..Default::default() }; + let bn = nn::batch_norm1d(vs.root(), 40, bn_cfg); + let x = Tensor::randn([10, 40], opts); + let _y = x.apply_t(&bn, true); + assert_eq!(vs.len(), 2); +} + +#[test] +fn layer_norm_test() { + let opts = (tch::Kind::Float, tch::Device::Cpu); + let vs = nn::VarStore::new(tch::Device::Cpu); + let ln = layer_norm(vs.root(), vec![5, 10, 10], Default::default()); + let x = Tensor::randn([20, 5, 10, 10], opts); + let _y = x.apply(&ln); +} + +#[test] +fn group_norm_test() { + let opts = (tch::Kind::Float, tch::Device::Cpu); + let vs = nn::VarStore::new(tch::Device::Cpu); + let ln = group_norm(vs.root(), 2, 10, Default::default()); + let x = Tensor::randn([1, 10, 7], opts); + let _y = x.apply(&ln); +} + +#[test] +fn layer_norm_parameters_test() { + tch::manual_seed(42); + // Create some linear data. + let xs = Tensor::from_slice(&[42.0, 42.0, 42.0, 24.0]).to_kind(Kind::Float).view([-1, 2]); + let ys = &xs * 0.42 + 1.337; + + // Fit a layer normalization layer (with deterministic initialization) on the data. + let vs = nn::VarStore::new(Device::Cpu); + let mut opt = nn::Sgd::default().build(&vs, 1.0).unwrap(); + let mut ln = layer_norm(vs.root(), vec![2], Default::default()); + + let loss = xs.apply(&ln).mse_loss(&ys, Reduction::Mean); + let initial_loss: f64 = from(&loss); + assert!(initial_loss > 1.0, "{}", "initial loss {initial_loss}"); + + // Optimization loop. + for _idx in 1..50 { + let loss = xs.apply(&ln).mse_loss(&ys, Reduction::Mean); + opt.backward_step(&loss); + } + let loss = xs.apply(&ln).mse_loss(&ys, Reduction::Mean); + let final_loss: f64 = from(&loss); + assert!(final_loss < 0.25, "{}", "final loss {final_loss:?}"); + + // Reset the weights to their initial values. + tch::no_grad(|| { + if let Some(ws) = &mut ln.ws { + ws.init(nn::Init::Const(1.)); + } + if let Some(bs) = &mut ln.bs { + bs.init(nn::Init::Const(0.)); + } + }); + let initial_loss2: f64 = from(&xs.apply(&ln).mse_loss(&ys, Reduction::Mean)); + assert_eq!(initial_loss, initial_loss2) +} + +fn gru_test(rnn_config: nn::RNNConfig) { + use nn::RNN; + let batch_dim = 5; + let seq_len = 3; + let input_dim = 2; + let output_dim = 4; + let vs = nn::VarStore::new(tch::Device::Cpu); + let gru = nn::gru(vs.root(), input_dim, output_dim, rnn_config); + + let num_directions = if rnn_config.bidirectional { 2 } else { 1 }; + let layer_dim = rnn_config.num_layers * num_directions; + // + // step test + let input = Tensor::randn([batch_dim, input_dim], kind::FLOAT_CPU); + let nn::GRUState(output) = gru.step(&input, &gru.zero_state(batch_dim)); + assert_eq!(output.size(), [layer_dim, batch_dim, output_dim]); + + // seq test + let input = Tensor::randn([batch_dim, seq_len, input_dim], kind::FLOAT_CPU); + let (output, _) = gru.seq(&input); + assert_eq!(output.size(), [batch_dim, seq_len, output_dim * num_directions]); +} + +#[test] +fn gru() { + gru_test(Default::default()); + gru_test(nn::RNNConfig { has_biases: false, ..Default::default() }); + gru_test(nn::RNNConfig { bidirectional: true, ..Default::default() }); + gru_test(nn::RNNConfig { num_layers: 2, ..Default::default() }); + gru_test(nn::RNNConfig { num_layers: 2, bidirectional: true, ..Default::default() }); +} + +fn lstm_test(rnn_config: nn::RNNConfig) { + use nn::RNN; + let batch_dim = 5; + let seq_len = 3; + let input_dim = 2; + let output_dim = 4; + let vs = nn::VarStore::new(tch::Device::Cpu); + let lstm = nn::lstm(vs.root(), input_dim, output_dim, rnn_config); + + let num_directions = if rnn_config.bidirectional { 2 } else { 1 }; + let layer_dim = rnn_config.num_layers * num_directions; + // + // step test + let input = Tensor::randn([batch_dim, input_dim], kind::FLOAT_CPU); + let nn::LSTMState((h, c)) = lstm.step(&input, &lstm.zero_state(batch_dim)); + assert_eq!(h.size(), [layer_dim, batch_dim, output_dim]); + assert_eq!(c.size(), [layer_dim, batch_dim, output_dim]); + + // seq test + let input = Tensor::randn([batch_dim, seq_len, input_dim], kind::FLOAT_CPU); + let (output, _) = lstm.seq(&input); + assert_eq!(output.size(), [batch_dim, seq_len, output_dim * num_directions]); +} + +#[test] +fn lstm() { + lstm_test(Default::default()); + lstm_test(nn::RNNConfig { has_biases: false, ..Default::default() }); + lstm_test(nn::RNNConfig { bidirectional: true, ..Default::default() }); + lstm_test(nn::RNNConfig { num_layers: 2, ..Default::default() }); + lstm_test(nn::RNNConfig { num_layers: 2, bidirectional: true, ..Default::default() }); +} + +fn embedding_test(embedding_config: nn::EmbeddingConfig) { + let batch_dim = 5; + let seq_len = 7; + let input_dim = 10; + let output_dim = 4; + let vs = nn::VarStore::new(tch::Device::Cpu); + let embeddings = nn::embedding(vs.root(), input_dim, output_dim, embedding_config); + + // forward test + let input = Tensor::randint(10, [batch_dim, seq_len], kind::INT64_CPU); + let output = embeddings.forward(&input); + assert_eq!(output.size(), [batch_dim, seq_len, output_dim]); + + // padding test + let padding_idx = if embedding_config.padding_idx < 0 { + input_dim + embedding_config.padding_idx + } else { + embedding_config.padding_idx + }; + let input = Tensor::from_slice(&[padding_idx]); + let output = embeddings.forward(&input); + assert_eq!(output.size(), [1, output_dim]); + assert_eq!(output.get(0), embeddings.ws.get(padding_idx)); +} + +#[test] +fn embedding_default() { + embedding_test(Default::default()); +} + +#[test] +fn embedding_neg_padding() { + embedding_test(nn::EmbeddingConfig { padding_idx: -1, ..Default::default() }); +} + +#[test] +fn embedding_zero_padding() { + embedding_test(nn::EmbeddingConfig { padding_idx: 0, ..Default::default() }); +} + +fn linear_test(linear_config: nn::LinearConfig) { + let batch_dim = 5; + let input_dim = 10; + let output_dim = 4; + let vs = nn::VarStore::new(tch::Device::Cpu); + let linear = nn::linear(vs.root(), input_dim, output_dim, linear_config); + + // forward test + let input = Tensor::randint(10, [batch_dim, input_dim], kind::FLOAT_CPU); + let expected_var_store_size = if linear_config.bias { 2 } else { 1 }; + + let output = linear.forward(&input); + assert_eq!(output.size(), [batch_dim, output_dim]); + + assert_eq!(output.size(), [batch_dim, output_dim]); + assert_eq!(vs.variables().len(), expected_var_store_size); + assert!(vs.variables().contains_key("weight")); + assert_eq!(vs.variables().contains_key("bias"), linear_config.bias); +} + +#[test] +fn linear() { + linear_test(Default::default()); + linear_test(nn::LinearConfig { bias: true, ..Default::default() }); + linear_test(nn::LinearConfig { bias: false, ..Default::default() }); +} + +#[test] +fn pad() { + let xs = Tensor::from_slice(&[1., 2., 3.]); + let padded = nn::PaddingMode::Zeros.pad(&xs, &[1, 1]); + assert_eq!(vec_f32_from(&padded), [0., 1., 2., 3., 0.]); + + let xs = Tensor::from_slice(&[1., 2., 3.]).view([1, 3]); + let padded = nn::PaddingMode::Zeros.pad(&xs, &[1, 1]); + assert_eq!(vec_f32_from(&padded.reshape(-1)), [0., 1., 2., 3., 0.]); + + let xs = Tensor::from_slice(&[1., 2., 3., 4.]).view([1, 2, 2]); + let padded = nn::PaddingMode::Reflect.pad(&xs, &[1, 1, 1, 1]); + assert_eq!( + vec_f32_from(&padded.reshape(-1)), + &[4.0, 3.0, 4.0, 3.0, 2.0, 1.0, 2.0, 1.0, 4.0, 3.0, 4.0, 3.0, 2.0, 1.0, 2.0, 1.0] + ); + let padded = nn::PaddingMode::Reflect.pad(&xs, &[1, 1, 1, 1]); + assert_eq!( + vec_f32_from(&padded.reshape(-1)), + &[4.0, 3.0, 4.0, 3.0, 2.0, 1.0, 2.0, 1.0, 4.0, 3.0, 4.0, 3.0, 2.0, 1.0, 2.0, 1.0] + ); + let padded = nn::PaddingMode::Reflect.pad(&xs, &[1, 1, 1, 1]); + assert_eq!( + vec_f32_from(&padded.reshape(-1)), + &[4.0, 3.0, 4.0, 3.0, 2.0, 1.0, 2.0, 1.0, 4.0, 3.0, 4.0, 3.0, 2.0, 1.0, 2.0, 1.0] + ); +} + +fn apply_conv(xs: &Tensor, padding_mode: nn::PaddingMode) -> Tensor { + let vs = nn::VarStore::new(Device::Cpu); + let conv_cfg = nn::ConvConfig { padding: 1, bias: false, padding_mode, ..Default::default() }; + let mut conv = nn::conv2d(vs.root(), 1, 1, 3, conv_cfg); + tch::no_grad(|| { + _ = conv.ws.fill_(1.); + }); + conv.forward(xs) +} + +#[test] +fn conv() { + let xs = Tensor::from_slice(&[1f32, 2., 3., 4.]).view([1, 1, 2, 2]); // NCHW + + let conved = apply_conv(&xs, nn::PaddingMode::Zeros); + assert_eq!(vec_f32_from(&conved.reshape(-1)), &[10.0, 10.0, 10.0, 10.0]); + + let conved = apply_conv(&xs, nn::PaddingMode::Reflect); + assert_eq!(vec_f32_from(&conved.reshape(-1)), &[27.0, 24.0, 21.0, 18.0]); + + let conved = apply_conv(&xs, nn::PaddingMode::Circular); + assert_eq!(vec_f32_from(&conved.reshape(-1)), &[27.0, 24.0, 21.0, 18.0]); + + let conved = apply_conv(&xs, nn::PaddingMode::Replicate); + assert_eq!(vec_f32_from(&conved.reshape(-1)), &[18.0, 21.0, 24.0, 27.0]); +} + +#[test] +fn seq() { + let s = nn::seq().add_fn(|xs| xs.shallow_clone().relu_()); + let xs = Tensor::from_slice(&[1.0, -1.0, 2.0]); + let ys = xs.apply(&s); + assert_eq!(vec_f32_from(&xs), [1., 0., 2.]); + assert_eq!(vec_f32_from(&ys), [1., 0., 2.]); +} diff --git a/v2/dependency-patches/tch-0.24.0/tests/serialization_tests.rs b/v2/dependency-patches/tch-0.24.0/tests/serialization_tests.rs new file mode 100644 index 0000000000..0f9ac2e1dc --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/tests/serialization_tests.rs @@ -0,0 +1,195 @@ +use tch::{Kind, Tensor}; + +mod test_utils; +use test_utils::*; + +struct TmpFile(std::path::PathBuf); + +impl TmpFile { + fn create(base: &str) -> TmpFile { + let filename = std::env::temp_dir().join(format!( + "tch-{}-{}-{:?}", + base, + std::process::id(), + std::thread::current().id(), + )); + TmpFile(filename) + } +} + +impl std::convert::AsRef for TmpFile { + fn as_ref(&self) -> &std::path::Path { + self.0.as_path() + } +} + +impl Drop for TmpFile { + fn drop(&mut self) { + std::fs::remove_file(&self.0).unwrap() + } +} + +#[test] +fn save_and_load() { + let tmp_file = TmpFile::create("save-and-load"); + let vec = [3.0, 1.0, 4.0, 1.0, 5.0].to_vec(); + let t1 = Tensor::from_slice(&vec); + t1.save(&tmp_file).unwrap(); + let t2 = Tensor::load(&tmp_file).unwrap(); + assert_eq!(vec_f64_from(&t2), vec) +} + +#[test] +fn save_to_stream_and_load() { + let tmp_file = TmpFile::create("write-stream"); + let vec = [3.0, 1.0, 4.0, 1.0, 5.0].to_vec(); + let t1 = Tensor::from_slice(&vec); + t1.save_to_stream(std::fs::File::create(&tmp_file).unwrap()).unwrap(); + let t2 = Tensor::load(&tmp_file).unwrap(); + assert_eq!(vec_f64_from(&t2), vec) +} + +#[test] +fn save_and_load_from_stream() { + let tmp_file = TmpFile::create("read-stream"); + let vec = [3.0, 1.0, 4.0, 1.0, 5.0].to_vec(); + let t1 = Tensor::from_slice(&vec); + t1.save(&tmp_file).unwrap(); + let reader = std::io::BufReader::new(std::fs::File::open(&tmp_file).unwrap()); + let t2 = Tensor::load_from_stream(reader).unwrap(); + assert_eq!(vec_f64_from(&t2), vec) +} + +#[test] +fn save_and_load_multi() { + let tmp_file = TmpFile::create("save-and-load-multi"); + let pi = Tensor::from_slice(&[3.0, 1.0, 4.0, 1.0, 5.0]); + let e = Tensor::from_slice(&[2, 7, 1, 8, 2, 8, 1, 8, 2, 8, 4, 6]); + Tensor::save_multi(&[(&"pi", &pi), (&"e", &e)], &tmp_file).unwrap(); + let named_tensors = Tensor::load_multi(&tmp_file).unwrap(); + assert_eq!(named_tensors.len(), 2); + assert_eq!(named_tensors[0].0, "pi"); + assert_eq!(named_tensors[1].0, "e"); + assert_eq!(from::(&named_tensors[1].1.sum(tch::Kind::Float)), 57); +} + +#[test] +fn save_to_stream_and_load_multi() { + let tmp_file = TmpFile::create("save-to-stream-and-load-multi"); + let pi = Tensor::from_slice(&[3.0, 1.0, 4.0, 1.0, 5.0]); + let e = Tensor::from_slice(&[2, 7, 1, 8, 2, 8, 1, 8, 2, 8, 4, 6]); + Tensor::save_multi_to_stream( + &[(&"pi", &pi), (&"e", &e)], + std::fs::File::create(&tmp_file).unwrap(), + ) + .unwrap(); + let named_tensors = Tensor::load_multi(&tmp_file).unwrap(); + assert_eq!(named_tensors.len(), 2); + assert_eq!(named_tensors[0].0, "pi"); + assert_eq!(named_tensors[1].0, "e"); + assert_eq!(from::(&named_tensors[1].1.sum(tch::Kind::Float)), 57); +} + +#[test] +fn save_and_load_multi_from_stream() { + let tmp_file = TmpFile::create("save-and-load-multi-from-stream"); + let pi = Tensor::from_slice(&[3.0, 1.0, 4.0, 1.0, 5.0]); + let e = Tensor::from_slice(&[2, 7, 1, 8, 2, 8, 1, 8, 2, 8, 4, 6]); + Tensor::save_multi(&[(&"pi", &pi), (&"e", &e)], &tmp_file).unwrap(); + let reader = std::io::BufReader::new(std::fs::File::open(&tmp_file).unwrap()); + let named_tensors = Tensor::load_multi_from_stream(reader).unwrap(); + assert_eq!(named_tensors.len(), 2); + assert_eq!(named_tensors[0].0, "pi"); + assert_eq!(named_tensors[1].0, "e"); + assert_eq!(from::(&named_tensors[1].1.sum(tch::Kind::Float)), 57); +} + +#[test] +fn save_and_load_npz() { + let tmp_file = TmpFile::create("save-and-load-npz"); + let pi = Tensor::from_slice(&[3.0, 1.0, 4.0, 1.0, 5.0]); + let e = Tensor::from_slice(&[2, 7, 1, 8, 2, 8, 1, 8, 2, 8, 4, 6]); + Tensor::write_npz(&[(&"pi", &pi), (&"e", &e)], &tmp_file).unwrap(); + let named_tensors = Tensor::read_npz(&tmp_file).unwrap(); + assert_eq!(named_tensors.len(), 2); + assert_eq!(named_tensors[0].0, "pi"); + assert_eq!(named_tensors[1].0, "e"); + assert_eq!(from::(&named_tensors[1].1.sum(tch::Kind::Float)), 57); +} + +#[test] +fn save_and_load_npz_half() { + let tmp_file = TmpFile::create("save-and-load-npz-half"); + let pi = Tensor::from_slice(&[3.0, 1.0, 4.0, 1.0, 5.0]).to_dtype(Kind::Half, true, false); + let e = + Tensor::from_slice(&[2, 7, 1, 8, 2, 8, 1, 8, 2, 8, 4, 6]).to_dtype(Kind::Half, true, false); + Tensor::write_npz(&[(&"pi", &pi), (&"e", &e)], &tmp_file).unwrap(); + let named_tensors = Tensor::read_npz(&tmp_file).unwrap(); + assert_eq!(named_tensors.len(), 2); + assert_eq!(named_tensors[0].0, "pi"); + assert_eq!(named_tensors[1].0, "e"); + assert_eq!(from::(&named_tensors[1].1.sum(tch::Kind::Float)), 57); +} + +#[test] +fn save_and_load_npz_byte() { + let tmp_file = TmpFile::create("save-and-load-npz-byte"); + let pi = Tensor::from_slice(&[3.0, 1.0, 4.0, 1.0, 5.0]).to_dtype(Kind::Int8, true, false); + let e = + Tensor::from_slice(&[2, 7, 1, 8, 2, 8, 1, 8, 2, 8, 4, 6]).to_dtype(Kind::Int8, true, false); + Tensor::write_npz(&[(&"pi", &pi), (&"e", &e)], &tmp_file).unwrap(); + let named_tensors = Tensor::read_npz(&tmp_file).unwrap(); + assert_eq!(named_tensors.len(), 2); + assert_eq!(named_tensors[0].0, "pi"); + assert_eq!(named_tensors[1].0, "e"); + assert_eq!(from::(&named_tensors[1].1.sum(tch::Kind::Int8)), 57); +} + +#[test] +fn save_and_load_npy() { + let tmp_file = TmpFile::create("save-and-load-npy"); + let pi = Tensor::from_slice(&[3.0, 1.0, 4.0, 1.0, 5.0, 9.0]); + pi.write_npy(&tmp_file).unwrap(); + let pi = Tensor::read_npy(&tmp_file).unwrap(); + assert_eq!(vec_f64_from(&pi), [3.0, 1.0, 4.0, 1.0, 5.0, 9.0]); + let pi = pi.reshape([3, 1, 2]); + pi.write_npy(&tmp_file).unwrap(); + let pi = Tensor::read_npy(&tmp_file).unwrap(); + assert_eq!(pi.size(), [3, 1, 2]); + assert_eq!(vec_f64_from(&pi.flatten(0, -1)), [3.0, 1.0, 4.0, 1.0, 5.0, 9.0]); +} + +#[test] +fn save_and_load_safetensors() { + let tmp_file = TmpFile::create("save-and-load-safetensors"); + let pi = Tensor::from_slice(&[3.0, 1.0, 4.0, 1.0, 5.0]); + let e = Tensor::from_slice(&[2, 7, 1, 8, 2, 8, 1, 8, 2, 8, 4, 6]); + Tensor::write_safetensors(&[(&"pi", &pi), (&"e", &e)], &tmp_file).unwrap(); + let named_tensors = Tensor::read_safetensors(&tmp_file).unwrap(); + assert_eq!(named_tensors.len(), 2); + for (name, tensor) in named_tensors { + match name.as_str() { + "pi" => assert_eq!(from::(&tensor.sum(tch::Kind::Float)), 14), + "e" => assert_eq!(from::(&tensor.sum(tch::Kind::Float)), 57), + _ => panic!("unknow name tensors"), + } + } +} + +#[test] +fn save_and_load_safetensors_half() { + let tmp_file = TmpFile::create("save-and-load-safetensors-half"); + let pi = Tensor::from_slice(&[3.0, 1.0, 4.0, 1.0, 5.0]).to_dtype(Kind::Half, true, false); + let e = + Tensor::from_slice(&[2, 7, 1, 8, 2, 8, 1, 8, 2, 8, 4, 6]).to_dtype(Kind::Half, true, false); + Tensor::write_safetensors(&[(&"pi", &pi), (&"e", &e)], &tmp_file).unwrap(); + let named_tensors = Tensor::read_safetensors(&tmp_file).unwrap(); + assert_eq!(named_tensors.len(), 2); + for (name, tensor) in named_tensors { + match name.as_str() { + "pi" => assert_eq!(from::(&tensor.sum(tch::Kind::Float)), 14), + "e" => assert_eq!(from::(&tensor.sum(tch::Kind::Float)), 57), + _ => panic!("unknow name tensors"), + } + } +} diff --git a/v2/dependency-patches/tch-0.24.0/tests/tensor_indexing.rs b/v2/dependency-patches/tch-0.24.0/tests/tensor_indexing.rs new file mode 100644 index 0000000000..7b21f022bb --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/tests/tensor_indexing.rs @@ -0,0 +1,229 @@ +use tch::{Device, Kind, Tensor}; +use tch::{IndexOp, NewAxis}; + +mod test_utils; +use test_utils::*; + +#[test] +fn integer_index() { + let opt = (Kind::Float, Device::Cpu); + + let tensor = Tensor::arange_start(0, 2 * 3, opt).view([2, 3]); + let result = tensor.i(1); + assert_eq!(result.size(), &[3]); + assert_eq!(vec_i64_from(&result), &[3, 4, 5]); + + let tensor = Tensor::arange_start(0, 2 * 3, opt).view([2, 3]); + let result = tensor.i((.., 2)); + assert_eq!(result.size(), &[2]); + assert_eq!(vec_i64_from(&result), &[2, 5]); + + let result = tensor.i((.., -2)); + assert_eq!(result.size(), &[2]); + assert_eq!(vec_i64_from(&result), &[1, 4]); +} + +#[test] +fn range_index() { + let opt = (Kind::Float, Device::Cpu); + + // Range + let tensor = Tensor::arange_start(0, 4 * 3, opt).view([4, 3]); + let result = tensor.i(1..3); + assert_eq!(result.size(), &[2, 3]); + assert_eq!(vec_i64_from(&result), &[3, 4, 5, 6, 7, 8]); + + // RangeFull + let tensor = Tensor::arange_start(0, 2 * 3, opt).view([2, 3]); + let result = tensor.i(..); + assert_eq!(result.size(), &[2, 3]); + assert_eq!(vec_i64_from(&result), &[0, 1, 2, 3, 4, 5]); + + // RangeFrom + let tensor = Tensor::arange_start(0, 4 * 3, opt).view([4, 3]); + let result = tensor.i(2..); + assert_eq!(result.size(), &[2, 3]); + assert_eq!(vec_i64_from(&result), &[6, 7, 8, 9, 10, 11]); + + // RangeTo + let tensor = Tensor::arange_start(0, 4 * 3, opt).view([4, 3]); + let result = tensor.i(..2); + assert_eq!(result.size(), &[2, 3]); + assert_eq!(vec_i64_from(&result), &[0, 1, 2, 3, 4, 5]); + + // RangeInclusive + let tensor = Tensor::arange_start(0, 4 * 3, opt).view([4, 3]); + let result = tensor.i(1..=2); + assert_eq!(result.size(), &[2, 3]); + assert_eq!(vec_i64_from(&result), &[3, 4, 5, 6, 7, 8]); + + // RangeTo + let tensor = Tensor::arange_start(0, 4 * 3, opt).view([4, 3]); + let result = tensor.i(..1); + assert_eq!(result.size(), &[1, 3]); + assert_eq!(vec_i64_from(&result), &[0, 1, 2]); + + // RangeToInclusive + let tensor = Tensor::arange_start(0, 4 * 3, opt).view([4, 3]); + let result = tensor.i(..=1); + assert_eq!(result.size(), &[2, 3]); + assert_eq!(vec_i64_from(&result), &[0, 1, 2, 3, 4, 5]); + + // Negative Cases + // + // Rust considers both `0..0` and `1..0` to be valid, empty ranges. + let tensor = Tensor::arange_start(0, 4 * 3, opt).view([4, 3]); + let result = tensor.i(0..0); + assert_eq!(result.size(), &[0, 3]); + assert_eq!(vec_i64_from(&result), &[] as &[i64]); + + // Clippy doesn't like explicitly reversed ranges, so let's pretend the bounds were computed. + let (start, end) = (1, 0); + let tensor = Tensor::arange_start(0, 4 * 3, opt).view([4, 3]); + let result = tensor.i(start..end); + assert_eq!(result.size(), &[0, 3]); + assert_eq!(vec_i64_from(&result), &[] as &[i64]); +} + +#[test] +fn slice_index() { + let opt = (Kind::Float, Device::Cpu); + + let tensor = Tensor::arange_start(0, 6 * 2, opt).view([6, 2]); + let index: &[_] = &[1, 3, 5]; + let result = tensor.i(index); + assert_eq!(result.size(), &[3, 2]); + assert_eq!(vec_i64_from(&result), &[2, 3, 6, 7, 10, 11]); + + let tensor = Tensor::arange_start(0, 3 * 4, opt).view([3, 4]); + let index: &[_] = &[3, 0]; + let result = tensor.i((.., index)); + assert_eq!(result.size(), &[3, 2]); + assert_eq!(vec_i64_from(&result), &[3, 0, 7, 4, 11, 8]); +} + +#[test] +fn new_index() { + let opt = (Kind::Float, Device::Cpu); + + let tensor = Tensor::arange_start(0, 2 * 3, opt).view([2, 3]); + let result = tensor.i((NewAxis,)); + assert_eq!(result.size(), &[1, 2, 3]); + assert_eq!(vec_i64_from(&result), &[0, 1, 2, 3, 4, 5]); + + let tensor = Tensor::arange_start(0, 2 * 3, opt).view([2, 3]); + let result = tensor.i((.., NewAxis)); + assert_eq!(result.size(), &[2, 1, 3]); + assert_eq!(vec_i64_from(&result), &[0, 1, 2, 3, 4, 5]); + + let tensor = Tensor::arange_start(0, 2 * 3, opt).view([2, 3]); + let result = tensor.i((.., .., NewAxis)); + assert_eq!(result.size(), &[2, 3, 1]); + assert_eq!(vec_i64_from(&result), &[0, 1, 2, 3, 4, 5]); +} + +#[cfg(target_os = "linux")] +#[test] +fn complex_index() { + let opt = (Kind::Float, Device::Cpu); + + let tensor = Tensor::arange_start(0, 2 * 3 * 5 * 7, opt).view([2, 3, 5, 7]); + let result = tensor.i((1, 1..2, vec![2, 3, 0].as_slice(), NewAxis, 3..)); + assert_eq!(result.size(), &[1, 3, 1, 4]); + assert_eq!( + vec_i64_from(&result), + &[157, 158, 159, 160, 164, 165, 166, 167, 143, 144, 145, 146] + ); +} + +#[test] +fn index_3d() { + let values: Vec = (0..24).collect(); + let tensor = tch::Tensor::from_slice(&values).view((2, 3, 4)); + assert_eq!(vec_i64_from(&tensor.i((0, 0, 0))), &[0]); + assert_eq!(vec_i64_from(&tensor.i((1, 0, 0))), &[12]); + assert_eq!(vec_i64_from(&tensor.i((0..2, 0, 0))), &[0, 12]); +} + +#[test] +fn tensor_index() { + let t = Tensor::arange(6, (Kind::Int64, Device::Cpu)).view((2, 3)); + let rows_select = Tensor::from_slice(&[0i64, 1, 0]); + let column_select = Tensor::from_slice(&[1i64, 2, 2]); + + let selected = t.index(&[Some(rows_select), Some(column_select)]); + assert_eq!(selected.size(), &[3]); + assert_eq!(vec_i64_from(&selected), &[1, 5, 2]); +} + +#[test] +fn tensor_index2() { + let t = Tensor::arange(400, (Kind::Int64, Device::Cpu)).view((2, 2, 10, 10)); + + let selected = t.index(&[ + Some(Tensor::from(0i64)), + Some(Tensor::from(1i64)), + Some(Tensor::from_slice(&[1i64, 2, 3])), + Some(Tensor::from_slice(&[5i64, 6, 7])), + ]); + assert_eq!(selected.size(), &[3]); + // 115 = 0 * 200 + 1 * 100 + 1 * 10 + 5 + // 126 = 0 * 200 + 1 * 100 + 2 * 10 + 6 + // 137 = 0 * 200 + 1 * 100 + 3 * 10 + 7 + assert_eq!(vec_i64_from(&selected), &[115, 126, 137]); +} + +#[test] +fn tensor_multi_index() { + let t = Tensor::arange(6, (Kind::Int64, Device::Cpu)).view((2, 3)); + + let select_1 = Tensor::from_slice(&[0i64, 1, 0]); + let select_2 = Tensor::from_slice(&[1i64, 0, 0]); + let select_final = Tensor::stack(&[select_1, select_2], 0); + assert_eq!(select_final.size(), &[2, 3]); + + let selected = t.index(&[Some(select_final)]); // index only rows + assert_eq!(selected.size(), &[2, 3, 3]); + assert_eq!(vec_i64_from(&selected), &[0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 0, 1, 2]); + // after flattening +} + +#[test] +fn tensor_put() { + let t = Tensor::arange(6, (Kind::Int64, Device::Cpu)).view((2, 3)); + let rows_select = Tensor::from_slice(&[0i64, 1, 0]); + let column_select = Tensor::from_slice(&[1i64, 2, 2]); + let values = Tensor::from_slice(&[10i64, 12, 24]); + + let updated = t.index_put(&[Some(rows_select), Some(column_select)], &values, false); + assert_eq!(vec_i64_from(&updated), &[0i64, 10, 24, 3, 4, 12]); // after flattening +} + +#[test] +fn indexing_doc() { + let tensor = Tensor::from_slice(&[1, 2, 3, 4, 5, 6]).view((2, 3)); + let t = tensor.i(1); + assert_eq!(vec_i64_from(&t), [4, 5, 6]); + let t = tensor.i((.., -2)); + assert_eq!(vec_i64_from(&t), [2, 5]); + + let tensor = Tensor::from_slice(&[1, 2, 3, 4, 5, 6]).view((2, 3)); + let t = tensor.i((.., 1..)); + assert_eq!(t.size(), [2, 2]); + assert_eq!(vec_i64_from(&t.contiguous().view(-1)), [2, 3, 5, 6]); + let t = tensor.i((..1, ..)); + assert_eq!(t.size(), [1, 3]); + assert_eq!(vec_i64_from(&t.contiguous().view(-1)), [1, 2, 3]); + let t = tensor.i((.., 1..2)); + assert_eq!(t.size(), [2, 1]); + assert_eq!(vec_i64_from(&t.contiguous().view(-1)), [2, 5]); + let t = tensor.i((.., 1..=2)); + assert_eq!(t.size(), [2, 2]); + assert_eq!(vec_i64_from(&t.contiguous().view(-1)), [2, 3, 5, 6]); + + let tensor = Tensor::from_slice(&[1, 2, 3, 4, 5, 6]).view((2, 3)); + let t = tensor.i((NewAxis,)); + assert_eq!(t.size(), &[1, 2, 3]); + let t = tensor.i((.., .., NewAxis)); + assert_eq!(t.size(), &[2, 3, 1]); +} diff --git a/v2/dependency-patches/tch-0.24.0/tests/tensor_tests.rs b/v2/dependency-patches/tch-0.24.0/tests/tensor_tests.rs new file mode 100644 index 0000000000..0fdee88431 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/tests/tensor_tests.rs @@ -0,0 +1,508 @@ +#![allow(clippy::unnecessary_fallible_conversions)] +use anyhow::Result; +use half::f16; +use std::convert::{TryFrom, TryInto}; +use std::f32; +use tch::{Device, TchError, Tensor}; + +mod test_utils; +use test_utils::*; + +#[test] +#[cfg(feature = "cuda-tests")] +fn amp_non_finite_check_and_unscale() { + let mut u = Tensor::from_slice(&[10f32, 20f32]).to_device(Device::Cuda(0)); + let mut found_inf = Tensor::from_slice(&[0f32]).to_device(Device::Cuda(0)); + let inv_scale = Tensor::from_slice(&[0.1f32]).to_device(Device::Cuda(0)); + u.internal_amp_non_finite_check_and_unscale(&mut found_inf, &inv_scale); + assert_eq!(vec_f32_from(&u), &[1f32, 2f32]); + assert_eq!(vec_f32_from(&found_inf), [0f32]); + + let mut v = Tensor::from_slice(&[1f32, f32::INFINITY]).to_device(Device::Cuda(0)); + v.internal_amp_non_finite_check_and_unscale(&mut found_inf, &inv_scale); + assert_eq!(vec_f32_from(&v), &[0.1, f32::INFINITY]); + assert_eq!(vec_f32_from(&found_inf), [1f32]); + + u.internal_amp_non_finite_check_and_unscale(&mut found_inf, &inv_scale); + assert_eq!(vec_f32_from(&u), &[0.1, 0.2]); + // found_inf is sticky + assert_eq!(vec_f32_from(&found_inf), [1f32]); +} + +#[test] +fn assign_ops() { + let mut t = Tensor::from_slice(&[3, 1, 4, 1, 5]); + t += 1; + t *= 2; + t -= 1; + assert_eq!(vec_i64_from(&t), [7, 3, 9, 3, 11]); +} + +#[test] +fn constant_ops() { + let mut t = Tensor::from_slice(&[7i64, 3, 9, 3, 11]); + t = -t; + assert_eq!(vec_i64_from(&t), [-7, -3, -9, -3, -11]); + t = 1 - t; + assert_eq!(vec_i64_from(&t), [8, 4, 10, 4, 12]); + t = 2 * t; + assert_eq!(vec_i64_from(&t), [16, 8, 20, 8, 24]); + + let mut t = Tensor::from_slice(&[0.2f64, 0.1]); + t = 2 / t; + assert_eq!(vec_f64_from(&t), [10.0, 20.0]); +} + +#[test] +fn iter() { + let t = Tensor::from_slice(&[7i64, 3, 9, 3, 11]); + let v = t.iter::().unwrap().collect::>(); + assert_eq!(v, [7, 3, 9, 3, 11]); + let t = Tensor::from_slice(&[std::f64::consts::PI, 15.926, 5.3589, 79.0]); + let v = t.iter::().unwrap().collect::>(); + assert_eq!(v, [std::f64::consts::PI, 15.926, 5.3589, 79.0]); +} + +#[test] +fn array_conversion() { + let vec: Vec<_> = (0..6).map(|x| (x * x) as f64).collect(); + let t = Tensor::from_slice(&vec); + assert_eq!(vec_f64_from(&t), [0.0, 1.0, 4.0, 9.0, 16.0, 25.0]); + let t = t.view([3, 2]); + assert_eq!(from::>>(&t), [[0.0, 1.0], [4.0, 9.0], [16.0, 25.0]]); + let t = t.view([2, 3]); + assert_eq!(from::>>(&t), [[0.0, 1.0, 4.0], [9.0, 16.0, 25.0]]) +} + +#[test] +fn binary_ops() { + let t = Tensor::from_slice(&[3.0, 1.0, 4.0, 1.0, 5.0]); + let t = (&t * &t) + &t - 1.5; + assert_eq!(vec_f64_from(&t), [10.5, 0.5, 18.5, 0.5, 28.5]); +} + +#[test] +fn grad() { + let mut x = Tensor::from(2.0).set_requires_grad(true); + let y = &x * &x + &x + 36; + x.zero_grad(); + y.backward(); + let dy_over_dx = x.grad(); + assert_eq!(vec_f64_from(&dy_over_dx), [5.0]); +} + +#[test] +fn grad_grad() { + // Compute a second order derivative using run_backward. + let mut x = Tensor::from(42.0).set_requires_grad(true); + let y = &x * &x * &x + &x + &x * &x; + x.zero_grad(); + let dy_over_dx = Tensor::run_backward(&[y], &[&x], true, true); + assert_eq!(dy_over_dx.len(), 1); + let dy_over_dx = &dy_over_dx[0]; + dy_over_dx.backward(); + let dy_over_dx2 = x.grad(); + assert_eq!(from::(&dy_over_dx2), 254.0); +} + +#[test] +#[should_panic(expected = "one of the input tensor does not use set_requires_grad")] +fn grad_without_requires() { + let x = Tensor::from(2.0); + let y = &x * &x + &x + 36; + let _dy_over_dx = Tensor::run_backward(&[y], &[&x], true, true); +} + +#[test] +fn cat_and_stack() { + let t = Tensor::from_slice(&[13.0, 37.0]); + let t = Tensor::cat(&[&t, &t, &t], 0); + assert_eq!(t.size(), [6]); + assert_eq!(vec_f64_from(&t), [13.0, 37.0, 13.0, 37.0, 13.0, 37.0]); + + let t = Tensor::from_slice(&[13.0, 37.0]); + let t = Tensor::stack(&[&t, &t, &t], 0); + assert_eq!(t.size(), [3, 2]); + assert_eq!(vec_f64_from(&t), [13.0, 37.0, 13.0, 37.0, 13.0, 37.0]); + + let t = Tensor::from_slice(&[13.0, 37.0]); + let t = Tensor::stack(&[&t, &t, &t], 1); + assert_eq!(t.size(), [2, 3]); + assert_eq!(vec_f64_from(&t), [13.0, 13.0, 13.0, 37.0, 37.0, 37.0]); +} + +#[test] +fn onehot() { + let xs = Tensor::from_slice(&[0, 1, 2, 3]); + let onehot = xs.onehot(4); + assert_eq!( + vec_f64_from(&onehot), + vec![1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0] + ); + assert_eq!(onehot.device(), xs.device()); + assert_eq!(onehot.size(), vec![4, 4]) +} + +#[test] +fn fallible() { + // Try to compare two tensors with incompatible dimensions and check that this returns an + // error. + let xs = Tensor::from_slice(&[0, 1, 2, 3]); + let ys = Tensor::from_slice(&[0, 1, 2, 3, 4]); + assert!(xs.f_eq_tensor(&ys).is_err()) +} + +#[test] +fn chunk() { + let xs = Tensor::from_slice(&[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); + let tensors = xs.chunk(3, 0); + assert_eq!(tensors.len(), 3); + assert_eq!(vec_i64_from(&tensors[0]), vec![0, 1, 2, 3]); + assert_eq!(vec_i64_from(&tensors[1]), vec![4, 5, 6, 7]); + assert_eq!(vec_i64_from(&tensors[2]), vec![8, 9]); +} + +#[test] +fn broadcast() { + let xs = Tensor::from_slice(&[4, 5, 3]); + let ys = Tensor::from(42); + let tensors = Tensor::broadcast_tensors(&[xs, ys]); + assert_eq!(tensors.len(), 2); + assert_eq!(vec_i64_from(&tensors[0]), vec![4, 5, 3]); + assert_eq!(vec_i64_from(&tensors[1]), vec![42, 42, 42]); +} + +#[test] +fn eq() { + let t = Tensor::from_slice(&[3, 1, 4, 1, 5]); + let u = &t + 1 - 1; + assert_eq!(t, u); + assert!(t == u); + assert!(t != u - 1); + + let t = Tensor::from_slice(&[std::f64::consts::PI]); + let u = Tensor::from(std::f64::consts::PI); + // The tensor shape is important for equality. + assert!(t != u); + assert!(t.size() != u.size()); + + let u = u.reshape([1]); + assert_eq!(t, u); + assert!(t == u); + assert!(t != u - 1) +} + +#[test] +fn values_at_index() { + let t = Tensor::from(42); + assert_eq!(t.int64_value(&[]), 42); + assert_eq!(t.double_value(&[]), 42.0); + assert!(t.f_int64_value(&[0]).is_err()); + assert!(t.f_double_value(&[0]).is_err()); +} + +#[test] +fn into_ndarray_f64() { + let tensor = Tensor::from_slice(&[1., 2., 3., 4.]); + let nd: ndarray::ArrayD = (&tensor).try_into().unwrap(); + assert_eq!(vec_f64_from(&tensor).as_slice(), nd.as_slice().unwrap()); +} + +#[test] +fn into_ndarray_i64() { + let tensor = Tensor::from_slice(&[1, 2, 3, 4]); + let nd: ndarray::ArrayD = (&tensor).try_into().unwrap(); + assert_eq!(vec_i64_from(&tensor).as_slice(), nd.as_slice().unwrap()); +} + +#[test] +fn from_ndarray_f64() { + let nd = ndarray::arr2(&[[1f64, 2.], [3., 4.]]); + let tensor = Tensor::try_from(nd.clone()).unwrap(); + assert_eq!(vec_f64_from(&tensor).as_slice(), nd.as_slice().unwrap()); +} + +#[test] +fn from_ndarray_i64() { + let nd = ndarray::arr2(&[[1i64, 2], [3, 4]]); + let tensor = Tensor::try_from(nd.clone()).unwrap(); + assert_eq!(vec_i64_from(&tensor).as_slice(), nd.as_slice().unwrap()); +} + +#[test] +fn from_ndarray_bool() { + let nd = ndarray::arr2(&[[true, false], [true, true]]); + let tensor = Tensor::try_from(nd.clone()).unwrap(); + assert_eq!(vec_bool_from(&tensor).as_slice(), nd.as_slice().unwrap()); +} + +#[test] +fn from_primitive() -> Result<()> { + assert_eq!(vec_i32_from(&Tensor::try_from(1_i32)?), vec![1]); + assert_eq!(vec_i64_from(&Tensor::try_from(1_i64)?), vec![1]); + assert_eq!(vec_f16_from(&Tensor::try_from(f16::from_f64(1.0))?), vec![f16::from_f64(1.0)]); + assert_eq!(vec_f32_from(&Tensor::try_from(1_f32)?), vec![1.0]); + assert_eq!(vec_f64_from(&Tensor::try_from(1_f64)?), vec![1.0]); + assert_eq!(vec_bool_from(&Tensor::try_from(true)?), vec![true]); + Ok(()) +} + +#[test] +fn from_vec() -> Result<()> { + assert_eq!(vec_i32_from(&Tensor::try_from(vec![-1_i32, 0, 1])?), vec![-1, 0, 1]); + assert_eq!(vec_i64_from(&Tensor::try_from(vec![-1_i64, 0, 1])?), vec![-1, 0, 1]); + assert_eq!( + from::>(&Tensor::try_from(vec![ + f16::from_f64(-1.0), + f16::from_f64(0.0), + f16::from_f64(1.0) + ])?), + vec![f16::from_f64(-1.0), f16::from_f64(0.0), f16::from_f64(1.0)] + ); + assert_eq!(vec_f32_from(&Tensor::try_from(vec![-1_f32, 0.0, 1.0])?), vec![-1.0, 0.0, 1.0]); + assert_eq!(vec_f64_from(&Tensor::try_from(vec![-1_f64, 0.0, 1.0])?), vec![-1.0, 0.0, 1.0]); + assert_eq!(vec_bool_from(&Tensor::try_from(vec![true, false])?), vec![true, false]); + Ok(()) +} + +#[test] +fn from_slice() -> Result<()> { + assert_eq!(vec_i32_from(&Tensor::try_from(&[-1_i32, 0, 1] as &[_])?), vec![-1, 0, 1]); + assert_eq!(vec_i64_from(&Tensor::try_from(&[-1_i64, 0, 1] as &[_])?), vec![-1, 0, 1]); + assert_eq!( + vec_f16_from(&Tensor::try_from(&[ + f16::from_f64(-1.0), + f16::from_f64(0.0), + f16::from_f64(1.0) + ] as &[_])?), + vec![f16::from_f64(-1.0), f16::from_f64(0.0), f16::from_f64(1.0)] + ); + assert_eq!(vec_f32_from(&Tensor::try_from(&[-1_f32, 0.0, 1.0] as &[_])?), vec![-1.0, 0.0, 1.0]); + assert_eq!(vec_f64_from(&Tensor::try_from(&[-1_f64, 0.0, 1.0] as &[_])?), vec![-1.0, 0.0, 1.0]); + assert_eq!(vec_bool_from(&Tensor::try_from(&[true, false] as &[_])?), vec![true, false]); + Ok(()) +} + +#[test] +fn test_device() { + let x = Tensor::from(1); + assert_eq!(x.device(), Device::Cpu); + let x = Tensor::from(1).to_device(Device::Cpu); + assert_eq!(x.device(), Device::Cpu); + if tch::Cuda::device_count() > 0 { + let x = Tensor::from(1).to_device(Device::Cuda(0)); + assert_eq!(x.device(), Device::Cuda(0)); + let x = Tensor::from(1).to_device(Device::Cuda(0)).to_device(Device::Cpu); + assert_eq!(x.device(), Device::Cpu); + } +} + +#[test] +fn where_() { + let t1 = Tensor::from_slice(&[3, 1, 4, 1, 5, 9]); + let t2 = Tensor::from_slice(&[2, 7, 1, 8, 2, 8]); + let t = t1.where_self(&t1.lt(4), &t2); + assert_eq!(vec_i64_from(&t), [3, 1, 1, 1, 2, 8]); +} + +#[test] +fn bool_tensor() { + let t1 = Tensor::from_slice(&[true, true, false]); + assert_eq!(vec_i64_from(&t1), [1, 1, 0]); + assert_eq!(vec_bool_from(&t1), [true, true, false]); + let t1 = Tensor::from_slice(&[0, 1, 0]).to_kind(tch::Kind::Bool); + let t2 = Tensor::from_slice(&[1, 1, 1]).to_kind(tch::Kind::Bool); + let t1_any = t1.any(); + let t2_any = t2.any(); + let t1_all = t1.all(); + let t2_all = t2.all(); + assert!(from::(&t1_any)); + assert!(!from::(&t1_all)); + assert!(from::(&t2_any)); + assert!(from::(&t2_all)); +} + +#[test] +fn copy_overflow() { + let mut s = [f32::consts::PI]; + let r = Tensor::zeros([1], (tch::Kind::Int64, Device::Cpu)).f_copy_data(&mut s, 1); + assert!(r.is_err()); + + let mut s: [i8; 0] = []; + let r = Tensor::zeros([10000], (tch::Kind::Int8, Device::Cpu)).f_copy_data(&mut s, 10000); + assert!(r.is_err()); +} + +#[test] +fn mkldnn() { + if tch::utils::has_mkldnn() { + let t = Tensor::randn([5, 5, 5], (tch::Kind::Float, Device::Cpu)); + assert!(!t.is_mkldnn()); + assert!(t.to_mkldnn().is_mkldnn()); + } +} + +#[test] +fn sparse() { + let t = Tensor::from_slice(&[1, 2, 3]); + assert!(!t.is_sparse()); +} + +#[test] +fn einsum() { + // Element-wise squaring of a vector. + let t = Tensor::from_slice(&[1.0, 2.0, 3.0]); + let t = Tensor::einsum("i, i -> i", &[&t, &t], None::); + assert_eq!(vec_f64_from(&t), [1.0, 4.0, 9.0]); + // Matrix transpose + let t = Tensor::from_slice(&[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]).reshape([2, 3]); + let t = Tensor::einsum("ij -> ji", &[t], None::); + assert_eq!(vec_f64_from(&t), [1.0, 4.0, 2.0, 5.0, 3.0, 6.0]); + // Sum all elements + let t = Tensor::einsum("ij -> ", &[t], None::); + assert_eq!(vec_f64_from(&t), [21.0]); +} + +#[test] +fn vec2() { + let tensor = Tensor::from_slice(&[1., 2., 3., 4., 5., 6.]).reshape([2, 3]); + assert_eq!(Vec::>::try_from(tensor).unwrap(), [[1., 2., 3.], [4., 5., 6.]]) +} + +#[test] +fn upsample1d() { + let tensor = Tensor::from_slice(&[1., 2., 3., 4., 5., 6.]).reshape([2, 3, 1]); + let up1 = tensor.upsample_linear1d([2], false, 1.); + assert_eq!( + // Exclude the last element because of some numerical instability. + vec_f64_from(&up1)[0..11], + [1.0, 1.0, 2.0, 2.0, 3.0, 3.0, 4.0, 4.0, 5.0, 5.0, 6.0] + ); + let up1 = tensor.upsample_linear1d([2], false, None); + assert_eq!(vec_f64_from(&up1), [1.0, 1.0, 2.0, 2.0, 3.0, 3.0, 4.0, 4.0, 5.0, 5.0, 6.0, 6.0]); +} + +#[test] +fn argmax() { + let tensor = Tensor::from_slice(&[7., 2., 3., 4., 5., 6.]).reshape([2, 3]); + let argmax = tensor.argmax(None, false); + assert_eq!(vec_i64_from(&argmax), [0],); + let argmax = tensor.argmax(0, false); + assert_eq!(vec_i64_from(&argmax), [0, 1, 1],); + let argmax = tensor.argmax(-1, false); + assert_eq!(vec_i64_from(&argmax), [0, 2],); +} + +#[test] +fn strides() { + fn check_stride(t: &Tensor) { + let shape = t.size(); + let ndim = shape.len(); + let mut c = 1; + let mut strides = vec![0i64; ndim]; + strides[ndim - 1] = c; + for i in (1..ndim).rev() { + c *= shape[i]; + strides[i - 1] = c; + } + + assert_eq!(t.stride(), strides); + } + + let tensor = Tensor::zeros([2, 3, 4], tch::kind::FLOAT_CPU); + check_stride(&tensor); + + let tensor: Tensor = Tensor::ones([3, 4, 5, 6, 7, 8], tch::kind::FLOAT_CPU); + check_stride(&tensor); +} + +#[test] +fn nested_tensor() { + let vec: Vec> = vec![vec![1, 2], vec![1, 2], vec![4, 5]]; + let t = Tensor::from_slice2(&vec); + assert_eq!(t.size(), [3, 2]); + assert_eq!(vec_i32_from(&t.view([-1])), [1, 2, 1, 2, 4, 5]); +} + +#[test] +fn quantized() { + let t = Tensor::from_slice(&[-1f32, 0., 1., 2., 120., 0.42]); + let t = t.quantize_per_tensor(0.1, 10, tch::Kind::QUInt8); + let t = t.dequantize(); + assert_eq!(vec_f32_from(&t), [-1f32, 0., 1., 2., 24.5, 0.4]); +} + +#[test] +fn nll_loss() { + let input = Tensor::randn([3, 5], (tch::Kind::Float, Device::Cpu)).set_requires_grad(true); + let target = Tensor::from_slice(&[1i64, 0, 4]); + let output = input.nll_loss(&target); + output.backward(); + + let weights = Tensor::from_slice(&[1f32, 2.0, 2.0, 1.0, 1.0]); + // This used to segfault, see https://github.com/LaurentMazare/tch-rs/issues/366 + let _output = input.g_nll_loss(&target, Some(weights), tch::Reduction::Mean, -100); +} + +#[test] +fn allclose() { + let t = Tensor::from_slice(&[-1f32, 0., 1., 2., 120., 0.42]); + let t = t.quantize_per_tensor(0.1, 10, tch::Kind::QUInt8); + let t = t.dequantize(); + assert!(!t.allclose(&(&t + 0.1), 1e-5, 1e-8, false)); + assert!(t.allclose(&(&t + 1e-9), 1e-5, 1e-8, false)); +} + +#[test] +fn set_data() { + let mut t = Tensor::from_slice(&[-1f32, 0., 1., 2., 120., 0.42]); + t.set_data(&t.to_kind(tch::Kind::BFloat16)); + assert_eq!(t.kind(), tch::Kind::BFloat16); +} + +#[test] +fn convert_vec() { + let t_1d = Tensor::from_slice(&[0, 1, 2, 3, 4, 5]); + let vec: Vec = Vec::try_from(t_1d).unwrap(); + assert_eq!(vec, vec![0, 1, 2, 3, 4, 5]); + + let t_2d = Tensor::from_slice(&[0, 1, 2, 3, 4, 5]).view((2, 3)); + let vec: Result, TchError> = Vec::try_from(t_2d); + assert!(matches!(vec, Err(TchError::Convert(msg)) if msg== + "Attempting to convert a Tensor with 2 dimensions to flat vector")); + + let t_2d = Tensor::from_slice(&[0, 1, 2, 3, 4, 5]).view((2, 3)); + let vec: Vec> = Vec::try_from(t_2d).unwrap(); + assert_eq!(vec, vec![vec![0, 1, 2], vec![3, 4, 5]]); +} + +#[test] +fn convert_ndarray() { + let t_1d = Tensor::from_slice(&[0, 1, 2, 3, 4, 5]); + let array_1d: ndarray::ArrayD = t_1d.as_ref().try_into().unwrap(); + assert_eq!(array_1d.as_slice(), ndarray::array![0, 1, 2, 3, 4, 5].as_slice()); + + let t_2d = Tensor::from_slice(&[0, 1, 2, 3, 4, 5]).view((2, 3)); + let array_2d: ndarray::ArrayD = t_2d.as_ref().try_into().unwrap(); + assert_eq!(array_2d.as_slice(), ndarray::array![[0, 1, 2], [3, 4, 5]].as_slice()); + + let t_3d = Tensor::from_slice(&[0, 1, 2, 3, 4, 5, 6, 7]).view((2, 2, 2)); + let array_3d: ndarray::ArrayD = t_3d.as_ref().try_into().unwrap(); + assert_eq!(array_3d.as_slice(), ndarray::array![[[0, 1], [2, 3]], [[4, 5], [6, 7]]].as_slice()); +} + +#[test] +fn fp8_tensor() { + let float8_types = [ + tch::Kind::Float8e5m2, + tch::Kind::Float8e4m3fn, + tch::Kind::Float8e5m2fnuz, + tch::Kind::Float8e4m3fnuz, + ]; + + for &kind in &float8_types { + let t = Tensor::from_slice(&[1.0f32, 2.0, 3.0, 4.0]).to_kind(kind); + assert_eq!(t.kind(), kind); + assert_eq!(t.kind().elt_size_in_bytes(), 1); + } +} diff --git a/v2/dependency-patches/tch-0.24.0/tests/test_utils.rs b/v2/dependency-patches/tch-0.24.0/tests/test_utils.rs new file mode 100644 index 0000000000..29f7fe2c7b --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/tests/test_utils.rs @@ -0,0 +1,44 @@ +use tch::Tensor; + +pub fn from<'a, T>(t: &'a Tensor) -> T +where + >::Error: std::fmt::Debug, + T: TryFrom<&'a Tensor>, +{ + T::try_from(t).unwrap() +} + +#[allow(dead_code)] +pub fn f64_from(t: &Tensor) -> f64 { + from::(t) +} + +#[allow(dead_code)] +pub fn vec_f64_from(t: &Tensor) -> Vec { + from::>(&t.reshape(-1)) +} + +#[allow(dead_code)] +pub fn vec_f32_from(t: &Tensor) -> Vec { + from::>(&t.reshape(-1)) +} + +#[allow(dead_code)] +pub fn vec_f16_from(t: &Tensor) -> Vec { + from::>(&t.reshape(-1)) +} + +#[allow(dead_code)] +pub fn vec_i64_from(t: &Tensor) -> Vec { + from::>(&t.reshape(-1)) +} + +#[allow(dead_code)] +pub fn vec_i32_from(t: &Tensor) -> Vec { + from::>(&t.reshape(-1)) +} + +#[allow(dead_code)] +pub fn vec_bool_from(t: &Tensor) -> Vec { + from::>(&t.reshape(-1)) +} diff --git a/v2/dependency-patches/tch-0.24.0/tests/var_store.rs b/v2/dependency-patches/tch-0.24.0/tests/var_store.rs new file mode 100644 index 0000000000..bfc4882abf --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/tests/var_store.rs @@ -0,0 +1,634 @@ +use std::fs; +use tch::nn::OptimizerConfig; +use tch::{nn, nn::linear, nn::Init, nn::VarStore, Device, Kind, TchError, Tensor}; + +mod test_utils; +use test_utils::*; + +#[test] +fn path_components() { + let vs = VarStore::new(Device::Cpu); + let root = vs.root(); + let path = root.sub("a"); + let path = path.sub("test"); + assert_eq!(path.components().collect::>(), vec!["a", "test"]); +} + +#[test] +fn var_store_entry() { + let vs = VarStore::new(Device::Cpu); + let root = vs.root(); + + let t1 = root.entry("key").or_zeros(&[3, 1, 4]); + let t2 = root.entry("key").or_zeros(&[1, 5, 9]); + + assert_eq!(t1.size(), &[3, 1, 4]); + assert_eq!(t2.size(), &[3, 1, 4]); +} + +#[test] +fn save_and_load_var_store() { + let filename = + std::env::temp_dir().join(format!("tch-vs-load-complete-{}", std::process::id())); + let add = |vs: &tch::nn::Path| { + let v = vs.sub("a").sub("b").ones("t2", &[3]); + let u = vs.zeros("t1", &[4]); + let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); + let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); + (u, v) + }; + let mut vs1 = VarStore::new(Device::Cpu); + let (mut u1, mut v1) = add(&vs1.root()); + let mut vs2 = VarStore::new(Device::Cpu); + let (u2, v2) = add(&vs2.root()); + tch::no_grad(|| { + u1 += 42.0; + v1 *= 2.0; + }); + assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); + assert_eq!(f64_from(&v1.mean(Kind::Float)), 2.0); + assert_eq!(f64_from(&u2.mean(Kind::Float)), 0.0); + assert_eq!(f64_from(&v2.mean(Kind::Float)), 1.0); + vs1.save(&filename).unwrap(); + vs2.load(&filename).unwrap(); + assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); + assert_eq!(f64_from(&u2.mean(Kind::Float)), 42.0); + assert_eq!(f64_from(&v2.mean(Kind::Float)), 2.0); + fs::remove_file(&filename).unwrap(); + + // Save and reload in half-precision + vs1.half(); + let mut vs2 = VarStore::new(Device::Cpu); + let (u2, v2) = add(&vs2.root()); + vs1.save(&filename).unwrap(); + vs2.load(&filename).unwrap(); + assert_eq!(u2.kind(), Kind::Half); + assert_eq!(v2.kind(), Kind::Half); + assert_eq!(f64_from(&u2.mean(Kind::Half)), 42.0); + assert_eq!(f64_from(&v2.mean(Kind::Half)), 2.0); + fs::remove_file(filename).unwrap(); +} + +#[test] +fn save_to_stream_and_load_var_store() { + let filename = + std::env::temp_dir().join(format!("tch-vs-load-stream-complete-{}", std::process::id())); + let add = |vs: &tch::nn::Path| { + let v = vs.sub("a").sub("b").ones("t2", &[3]); + let u = vs.zeros("t1", &[4]); + let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); + let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); + (u, v) + }; + let mut vs1 = VarStore::new(Device::Cpu); + let mut vs2 = VarStore::new(Device::Cpu); + let (mut u1, mut v1) = add(&vs1.root()); + let (u2, v2) = add(&vs2.root()); + tch::no_grad(|| { + u1 += 42.0; + v1 *= 2.0; + }); + assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); + assert_eq!(f64_from(&v1.mean(Kind::Float)), 2.0); + assert_eq!(f64_from(&u2.mean(Kind::Float)), 0.0); + assert_eq!(f64_from(&v2.mean(Kind::Float)), 1.0); + vs1.save_to_stream(std::fs::File::create(&filename).unwrap()).unwrap(); + vs2.load(&filename).unwrap(); + assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); + assert_eq!(f64_from(&u2.mean(Kind::Float)), 42.0); + assert_eq!(f64_from(&v2.mean(Kind::Float)), 2.0); + fs::remove_file(&filename).unwrap(); + + // Save and reload in half-precision + vs1.half(); + let mut vs2 = VarStore::new(Device::Cpu); + let (u2, v2) = add(&vs2.root()); + vs1.save_to_stream(std::fs::File::create(&filename).unwrap()).unwrap(); + vs2.load(&filename).unwrap(); + assert_eq!(u2.kind(), Kind::Half); + assert_eq!(v2.kind(), Kind::Half); + assert_eq!(f64_from(&u2.mean(Kind::Half)), 42.0); + assert_eq!(f64_from(&v2.mean(Kind::Half)), 2.0); + fs::remove_file(filename).unwrap(); +} + +#[test] +fn save_and_load_from_stream_var_store() { + let filename = + std::env::temp_dir().join(format!("tch-vs-load-stream-complete2-{}", std::process::id())); + let add = |vs: &tch::nn::Path| { + let v = vs.sub("a").sub("b").ones("t2", &[3]); + let u = vs.zeros("t1", &[4]); + let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); + let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); + (u, v) + }; + let mut vs1 = VarStore::new(Device::Cpu); + let mut vs2 = VarStore::new(Device::Cpu); + let (mut u1, mut v1) = add(&vs1.root()); + let (u2, v2) = add(&vs2.root()); + tch::no_grad(|| { + u1 += 42.0; + v1 *= 2.0; + }); + assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); + assert_eq!(f64_from(&v1.mean(Kind::Float)), 2.0); + assert_eq!(f64_from(&u2.mean(Kind::Float)), 0.0); + assert_eq!(f64_from(&v2.mean(Kind::Float)), 1.0); + vs1.save(&filename).unwrap(); + vs2.load_from_stream(std::fs::File::open(&filename).unwrap()).unwrap(); + assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); + assert_eq!(f64_from(&u2.mean(Kind::Float)), 42.0); + assert_eq!(f64_from(&v2.mean(Kind::Float)), 2.0); + fs::remove_file(&filename).unwrap(); + + // Save and reload in half-precision + vs1.half(); + let mut vs2 = VarStore::new(Device::Cpu); + let (u2, v2) = add(&vs2.root()); + vs1.save(&filename).unwrap(); + vs2.load_from_stream(std::fs::File::open(&filename).unwrap()).unwrap(); + assert_eq!(u2.kind(), Kind::Half); + assert_eq!(v2.kind(), Kind::Half); + assert_eq!(f64_from(&u2.mean(Kind::Half)), 42.0); + assert_eq!(f64_from(&v2.mean(Kind::Half)), 2.0); + fs::remove_file(filename).unwrap(); +} + +#[test] +fn save_and_load_partial_var_store() { + let filename = + std::env::temp_dir().join(format!("tch-vs-partial-load-complete-{}", std::process::id())); + let add = |vs: &tch::nn::Path| { + let v = vs.sub("a").sub("b").ones("t2", &[3]); + let u = vs.zeros("t1", &[4]); + let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); + let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); + (u, v) + }; + let mut vs1 = VarStore::new(Device::Cpu); + let mut vs2 = VarStore::new(Device::Cpu); + let (mut u1, mut v1) = add(&vs1.root()); + let (u2, v2) = add(&vs2.root()); + tch::no_grad(|| { + u1 += 42.0; + v1 *= 2.0; + }); + assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); + assert_eq!(f64_from(&v1.mean(Kind::Float)), 2.0); + assert_eq!(f64_from(&u2.mean(Kind::Float)), 0.0); + assert_eq!(f64_from(&v2.mean(Kind::Float)), 1.0); + vs1.save(&filename).unwrap(); + let missing_variables = vs2.load_partial(&filename).unwrap(); + assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); + assert_eq!(f64_from(&u2.mean(Kind::Float)), 42.0); + assert_eq!(f64_from(&v2.mean(Kind::Float)), 2.0); + assert!(missing_variables.is_empty()); + fs::remove_file(&filename).unwrap(); + + // Save and reload in half-precision + vs1.half(); + let mut vs2 = VarStore::new(Device::Cpu); + let (u2, v2) = add(&vs2.root()); + vs1.save(&filename).unwrap(); + let missing_variables = vs2.load_partial(&filename).unwrap(); + assert_eq!(u2.kind(), Kind::Half); + assert_eq!(v2.kind(), Kind::Half); + assert_eq!(f64_from(&u2.mean(Kind::Half)), 42.0); + assert_eq!(f64_from(&v2.mean(Kind::Half)), 2.0); + assert!(missing_variables.is_empty()); + fs::remove_file(filename).unwrap(); +} + +#[test] +#[should_panic] +fn save_and_load_var_store_incomplete_file() { + let filename = + std::env::temp_dir().join(format!("tch-vs-load-incomplete-{}", std::process::id())); + let add = |vs: &tch::nn::Path| { + let u = vs.zeros("t1", &[4]); + let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); + u + }; + let add_partial = |vs: &tch::nn::Path| { + let v = vs.sub("a").sub("b").ones("t2", &[3]); + let u = vs.zeros("t1", &[4]); + let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); + (u, v) + }; + let vs1 = VarStore::new(Device::Cpu); + let mut vs2 = VarStore::new(Device::Cpu); + let mut u1 = add(&vs1.root()); + let (u2, v2) = add_partial(&vs2.root()); + tch::no_grad(|| { + u1 += 42.0; + }); + assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); + assert_eq!(f64_from(&u2.mean(Kind::Float)), 0.0); + assert_eq!(f64_from(&v2.mean(Kind::Float)), 1.0); + vs1.save(&filename).unwrap(); + vs2.load(&filename).unwrap(); + assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); + assert_eq!(f64_from(&u2.mean(Kind::Float)), 42.0); + assert_eq!(f64_from(&v2.mean(Kind::Float)), 1.0); + fs::remove_file(filename).unwrap(); +} + +#[test] +fn save_and_load_partial_var_store_incomplete_file() { + let filename = + std::env::temp_dir().join(format!("tch-vs-partial-load-incomplete-{}", std::process::id())); + let add = |vs: &tch::nn::Path| { + let u = vs.zeros("t1", &[4]); + let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); + u + }; + let add_partial = |vs: &tch::nn::Path| { + let v = vs.sub("a").sub("b").ones("t2", &[3]); + let u = vs.zeros("t1", &[4]); + let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); + (u, v) + }; + let mut vs1 = VarStore::new(Device::Cpu); + let mut vs2 = VarStore::new(Device::Cpu); + let mut u1 = add(&vs1.root()); + let (u2, v2) = add_partial(&vs2.root()); + tch::no_grad(|| { + u1 += 42.0; + }); + assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); + assert_eq!(f64_from(&u2.mean(Kind::Float)), 0.0); + assert_eq!(f64_from(&v2.mean(Kind::Float)), 1.0); + vs1.save(&filename).unwrap(); + let missing_variables = vs2.load_partial(&filename).unwrap(); + assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); + assert_eq!(f64_from(&u2.mean(Kind::Float)), 42.0); + assert_eq!(f64_from(&v2.mean(Kind::Float)), 1.0); + assert_eq!(missing_variables, vec!(String::from("a.b.t2"))); + fs::remove_file(&filename).unwrap(); + + // Save and reload in half-precision + vs1.half(); + let mut vs2 = VarStore::new(Device::Cpu); + let (u2, v2) = add_partial(&vs2.root()); + vs1.save(&filename).unwrap(); + let missing_variables = vs2.load_partial(&filename).unwrap(); + assert_eq!(u2.kind(), Kind::Half); + assert_eq!(v2.kind(), Kind::Float); + assert_eq!(f64_from(&u2.mean(Kind::Half)), 42.0); + assert_eq!(f64_from(&v2.mean(Kind::Float)), 1.0); + assert_eq!(missing_variables, vec!(String::from("a.b.t2"))); + fs::remove_file(&filename).unwrap(); +} + +#[test] +fn init_test() { + tch::manual_seed(42); + let vs = VarStore::new(Device::Cpu); + let zeros = vs.root().zeros("t1", &[3]); + assert_eq!(vec_f64_from(&zeros), [0., 0., 0.]); + let zeros = vs.root().var("t2", &[3], Init::Const(0.)); + assert_eq!(vec_f64_from(&zeros), [0., 0., 0.]); + let ones = vs.root().var("t3", &[3], Init::Const(1.)); + assert_eq!(vec_f64_from(&ones), [1., 1., 1.]); + let ones = vs.root().var("t4", &[3], Init::Const(0.5)); + assert_eq!(vec_f64_from(&ones), [0.5, 0.5, 0.5]); + let forty_two = vs.root().var("t4", &[2], Init::Const(42.)); + assert_eq!(vec_f64_from(&forty_two), [42., 42.]); + let uniform = vs.root().var("t5", &[100], Init::Uniform { lo: 1.0, up: 2.0 }); + let uniform_min = f64_from(&uniform.min()); + let uniform_max = f64_from(&uniform.max()); + assert!(uniform_min >= 1., "{}", "min {uniform_min}"); + assert!(uniform_max <= 2., "{}", "max {uniform_max}"); + let uniform_std = f64_from(&uniform.std(true)); + assert!(uniform_std > 0.15 && uniform_std < 0.35, "{}", "std {uniform_std}"); + let normal = vs.root().var("normal", &[100], Init::Randn { mean: 0., stdev: 0.02 }); + let normal_std = f64_from(&normal.std(true)); + assert!(normal_std <= 0.03, "{}", "std {normal_std}"); + let mut vs2 = VarStore::new(Device::Cpu); + let ones = vs2.root().ones("t1", &[3]); + assert_eq!(vec_f64_from(&ones), [1., 1., 1.]); + vs2.copy(&vs).unwrap(); + assert_eq!(vec_f64_from(&ones), [0., 0., 0.]); + let ortho = vs.root().var("orthogonal", &[100, 100], Init::Orthogonal { gain: 2.0 }); + let ortho_norm = f64_from(&ortho.linalg_norm_ord_str("fro", None::, true, Kind::Float)); + assert!( + f64::abs(ortho_norm - 20.) < 1e-5, + "{}", + "ortho_norm initialization failed {ortho_norm}" + ); + let ortho_shape_fail = + tch::nn::f_init(Init::Orthogonal { gain: 1.0 }, &[10], Device::Cpu, tch::Kind::Float); + assert!(ortho_shape_fail.is_err()); + let kaiming_u = vs.root().var("kaiming_u", &[20, 100], nn::init::DEFAULT_KAIMING_UNIFORM); + assert!(f64::abs(f64_from(&kaiming_u.mean(Kind::Float))) < 5e-3); + // The expected stdev is sqrt(2 / 100) + assert!(f64::abs(f64_from(&kaiming_u.std(true)) - (0.02f64).sqrt()) < 2e-3); + let kaiming_n = vs.root().var("kaiming_n", &[20, 100], nn::init::DEFAULT_KAIMING_NORMAL); + assert!(f64::abs(f64_from(&kaiming_n.mean(Kind::Float))) < 5e-3); + assert!(f64::abs(f64_from(&kaiming_n.std(true)) - (0.02f64).sqrt()) < 3e-3); +} + +fn check_param_group(mut opt: tch::nn::Optimizer, var_foo: Tensor, var_bar: Tensor) { + opt.set_lr(0.1); + opt.set_lr_group(0, 0.); + for _idx in 1..100 { + let loss = (&var_foo + &var_bar).mse_loss(&Tensor::from(0.42f32), tch::Reduction::Mean); + opt.backward_step(&loss); + } + assert_eq!(format!("{:.2}", f64_from(&var_foo)), "0.00"); + assert_eq!(format!("{:.2}", f64_from(&var_bar)), "0.42"); + opt.set_lr_group(0, 0.1); + for _idx in 1..100 { + let loss = (&var_foo + &var_bar).mse_loss(&Tensor::from(0f32), tch::Reduction::Mean); + opt.backward_step(&loss); + } + assert_eq!(format!("{:.2}", f64_from(&var_foo)), "-0.21"); + assert_eq!(format!("{:.2}", f64_from(&var_bar)), "0.21"); + opt.set_lr_group(7, 0.); + for _idx in 1..100 { + let loss = (&var_foo + &var_bar).mse_loss(&Tensor::from(0.22f32), tch::Reduction::Mean); + opt.backward_step(&loss); + } + assert_eq!(format!("{:.2}", f64_from(&var_foo)), "0.01"); + assert_eq!(format!("{:.2}", f64_from(&var_bar)), "0.21"); + // The following sets the learning rate for both groups. + opt.set_lr(0.); + for _idx in 1..100 { + let loss = (&var_foo + &var_bar).mse_loss(&Tensor::from(0.42f32), tch::Reduction::Mean); + opt.backward_step(&loss); + } + assert_eq!(format!("{:.2}", f64_from(&var_foo)), "0.01"); + assert_eq!(format!("{:.2}", f64_from(&var_bar)), "0.21"); + opt.set_lr(0.1); + for _idx in 1..100 { + let loss = (&var_foo + &var_bar).mse_loss(&Tensor::from(0.42f32), tch::Reduction::Mean); + opt.backward_step(&loss); + } + assert_eq!(format!("{:.2}", f64_from(&var_foo)), "0.11"); + assert_eq!(format!("{:.2}", f64_from(&var_bar)), "0.31"); +} + +#[test] +fn param_group() { + tch::manual_seed(42); + let vs = VarStore::new(Device::Cpu); + let opt = tch::nn::Sgd::default().build(&vs, 1.0).unwrap(); + let root = vs.root(); + let var_foo = root.set_group(0).zeros("var_foo", &[]); + let var_bar = root.set_group(7).zeros("var_bar", &[]); + check_param_group(opt, var_foo, var_bar); +} + +#[test] +fn save_and_load_with_group() { + let filename = std::env::temp_dir().join(format!("tch-vs-load-grp-{}", std::process::id())); + let add = |vs: &tch::nn::Path| { + let v = vs.set_group(1).sub("a").sub("b").ones("t2", &[3]); + let u = vs.zeros("t1", &[4]); + let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); + let _w = vs.sub("a").set_group(4).sub("b").sub("ccc").ones("t123", &[3]); + (u, v) + }; + let vs1 = VarStore::new(Device::Cpu); + let mut vs2 = VarStore::new(Device::Cpu); + let (mut u1, mut v1) = add(&vs1.root()); + let (u2, v2) = add(&vs2.root()); + tch::no_grad(|| { + u1 += 42.0; + v1 *= 2.0; + }); + assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); + assert_eq!(f64_from(&v1.mean(Kind::Float)), 2.0); + assert_eq!(f64_from(&u2.mean(Kind::Float)), 0.0); + assert_eq!(f64_from(&v2.mean(Kind::Float)), 1.0); + vs1.save(&filename).unwrap(); + vs2.load(&filename).unwrap(); + assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); + assert_eq!(f64_from(&u2.mean(Kind::Float)), 42.0); + assert_eq!(f64_from(&v2.mean(Kind::Float)), 2.0); + fs::remove_file(filename).unwrap(); +} + +#[test] +fn param_group_weight_decay() { + tch::manual_seed(42); + let vs = VarStore::new(Device::Cpu); + let mut opt = tch::nn::Sgd::default().build(&vs, 0.0).unwrap(); + opt.set_lr(0.1); + let root = vs.root(); + let var_foo = root.set_group(0).zeros("var_foo", &[]); + let var_bar = root.set_group(7).zeros("var_bar", &[]); + for _idx in 1..100 { + let loss = (&var_foo + &var_bar).mse_loss(&Tensor::from(1f32), tch::Reduction::Mean); + opt.backward_step(&loss); + } + assert_eq!(format!("{:.2}", f64_from(&var_foo)), "0.50"); + assert_eq!(format!("{:.2}", f64_from(&var_bar)), "0.50"); + opt.set_weight_decay(0.1); + for _idx in 1..100 { + let loss = (&var_foo + &var_bar).mse_loss(&Tensor::from(1f32), tch::Reduction::Mean); + opt.backward_step(&loss); + } + assert_eq!(format!("{:.2}", f64_from(&var_foo)), "0.49"); + assert_eq!(format!("{:.2}", f64_from(&var_bar)), "0.49"); + opt.set_weight_decay_group(7, 0.); + for _idx in 1..100 { + let loss = (&var_foo + &var_bar).mse_loss(&Tensor::from(1f32), tch::Reduction::Mean); + opt.backward_step(&loss); + } + assert_eq!(format!("{:.2}", f64_from(&var_foo)), "0.30"); + assert_eq!(format!("{:.2}", f64_from(&var_bar)), "0.69"); +} + +#[test] +fn half_precision_conversion_entire_varstore() { + let mut vs = VarStore::new(Device::Cpu); + + let _ = vs.root().var("zeros", &[1], Init::Const(0.)); + let _ = vs.root().var("ones", &[1], Init::Const(1.)); + let _ = vs.root().var("forty_two", &[1], Init::Const(42.)); + + assert_eq!(vs.root().get("zeros").unwrap().kind(), Kind::Float); + assert_eq!(vs.root().get("ones").unwrap().kind(), Kind::Float); + assert_eq!(vs.root().get("forty_two").unwrap().kind(), Kind::Float); + + vs.half(); + + assert_eq!(vs.root().get("zeros").unwrap().kind(), Kind::Half); + assert_eq!(vs.root().get("ones").unwrap().kind(), Kind::Half); + assert_eq!(vs.root().get("forty_two").unwrap().kind(), Kind::Half); + + vs.bfloat16(); + + assert_eq!(vs.root().get("zeros").unwrap().kind(), Kind::BFloat16); + assert_eq!(vs.root().get("ones").unwrap().kind(), Kind::BFloat16); + assert_eq!(vs.root().get("forty_two").unwrap().kind(), Kind::BFloat16); + + vs.double(); + + assert_eq!(vs.root().get("zeros").unwrap().kind(), Kind::Double); + assert_eq!(vs.root().get("ones").unwrap().kind(), Kind::Double); + assert_eq!(vs.root().get("forty_two").unwrap().kind(), Kind::Double); + + vs.float(); + + assert_eq!(vs.root().get("zeros").unwrap().kind(), Kind::Float); + assert_eq!(vs.root().get("ones").unwrap().kind(), Kind::Float); + assert_eq!(vs.root().get("forty_two").unwrap().kind(), Kind::Float); + assert_eq!(format!("{:.2}", f64_from(&vs.root().get("zeros").unwrap())), "0.00"); + assert_eq!(format!("{:.2}", f64_from(&vs.root().get("ones").unwrap())), "1.00"); + assert_eq!(format!("{:.2}", f64_from(&vs.root().get("forty_two").unwrap())), "42.00"); +} + +#[test] +fn path_half_precision_conversion() { + let vs = VarStore::new(Device::Cpu); + + // Define a VarStore with 3 variables. 2 of them are in a sub-path named "convert" and + // will be cast to half-precision. The other variables in the VarStore will be unaffected + let _ = vs.root().sub("ignore").var("zeros", &[1], Init::Const(0.)); + let _ = vs.root().sub("convert").sub("group_1").var("ones", &[1], Init::Const(1.)); + let linear_layer = + linear(vs.root().sub("convert").sub("group_2").sub("linear"), 10, 42, Default::default()); + + assert_eq!(vs.root().sub("ignore").get("zeros").unwrap().kind(), Kind::Float); + assert_eq!(vs.root().sub("convert").sub("group_1").get("ones").unwrap().kind(), Kind::Float); + assert_eq!(linear_layer.ws.kind(), Kind::Float); + assert_eq!(linear_layer.bs.as_ref().unwrap().kind(), Kind::Float); + + vs.root().sub("convert").half(); + + assert_eq!(vs.root().sub("ignore").get("zeros").unwrap().kind(), Kind::Float); + assert_eq!(vs.root().sub("convert").sub("group_1").get("ones").unwrap().kind(), Kind::Half); + assert_eq!(linear_layer.ws.kind(), Kind::Half); + assert_eq!(linear_layer.bs.as_ref().unwrap().kind(), Kind::Half); + + vs.root().sub("convert").float(); + + assert_eq!(vs.root().sub("ignore").get("zeros").unwrap().kind(), Kind::Float); + assert_eq!(vs.root().sub("convert").sub("group_1").get("ones").unwrap().kind(), Kind::Float); + assert_eq!(linear_layer.ws.kind(), Kind::Float); + assert_eq!(linear_layer.bs.as_ref().unwrap().kind(), Kind::Float); +} + +#[test] +fn path_free_type_conversion() { + let mut vs = VarStore::new(Device::Cpu); + + // Define a VarStore with 3 variables. 2 of them are in a sub-path named "convert" and + // will be cast to half-precision. The other variables in the VarStore will be unaffected + let _ = vs.root().sub("ignore").var("zeros", &[1], Init::Const(0.)); + let _ = vs.root().sub("convert").sub("group_1").var("ones", &[1], Init::Const(1.)); + let _ = vs.root().sub("convert").sub("group_2").var("zeros", &[1], Init::Const(0.)); + + assert_eq!(vs.root().sub("ignore").get("zeros").unwrap().kind(), Kind::Float); + assert_eq!(vs.root().sub("convert").sub("group_1").get("ones").unwrap().kind(), Kind::Float); + assert_eq!(vs.root().sub("convert").sub("group_2").get("zeros").unwrap().kind(), Kind::Float); + + // Disable gradient tracking as this would raise an error when converting to a + // non-float type like bool. + vs.freeze(); + vs.root().sub("convert").set_kind(Kind::Bool); + + assert_eq!(vs.root().sub("ignore").get("zeros").unwrap().kind(), Kind::Float); + assert_eq!(vs.root().sub("convert").sub("group_1").get("ones").unwrap().kind(), Kind::Bool); + assert_eq!(vs.root().sub("convert").sub("group_2").get("zeros").unwrap().kind(), Kind::Bool); + + vs.root().sub("convert").set_kind(Kind::Float); + vs.unfreeze(); + + assert_eq!(vs.root().sub("ignore").get("zeros").unwrap().kind(), Kind::Float); + assert_eq!(vs.root().sub("convert").sub("group_1").get("ones").unwrap().kind(), Kind::Float); + assert_eq!(vs.root().sub("convert").sub("group_2").get("zeros").unwrap().kind(), Kind::Float); + + assert_eq!(format!("{:.2}", f64_from(&vs.root().sub("ignore").get("zeros").unwrap())), "0.00"); + assert_eq!( + format!("{:.2}", f64_from(&vs.root().sub("convert").sub("group_1").get("ones").unwrap())), + "1.00" + ); + assert_eq!( + format!("{:.2}", f64_from(&vs.root().sub("convert").sub("group_2").get("zeros").unwrap())), + "0.00" + ); +} + +#[test] +fn device_migration() { + if tch::Cuda::is_available() { + let mut vs = VarStore::new(Device::Cpu); + + let _ = vs.root().var("zeros", &[3], Init::Const(0.)); + let _ = vs.root().var("ones", &[3], Init::Const(1.)); + let linear_layer = linear(vs.root().sub("linear"), 10, 42, Default::default()); + + vs.set_device(Device::Cuda(0)); + + assert_eq!(vs.root().get("zeros").unwrap().device(), Device::Cuda(0)); + assert_eq!(vs.root().get("ones").unwrap().device(), Device::Cuda(0)); + assert_eq!(linear_layer.ws.device(), Device::Cuda(0)); + assert_eq!(linear_layer.bs.as_ref().unwrap().device(), Device::Cuda(0)); + assert_eq!(vs.device(), Device::Cuda(0)); + + vs.set_device(Device::Cpu); + + assert_eq!(vs.root().get("zeros").unwrap().device(), Device::Cpu); + assert_eq!(vs.root().get("ones").unwrap().device(), Device::Cpu); + assert_eq!(linear_layer.ws.device(), Device::Cpu); + assert_eq!(linear_layer.bs.as_ref().unwrap().device(), Device::Cpu); + assert_eq!(vs.device(), Device::Cpu); + } +} + +#[test] +fn merge_var_stores_no_prefixes() { + let vs_1 = VarStore::new(Device::Cpu); + let _ = vs_1.root().entry("key_1").or_zeros(&[3, 1, 4]); + let _ = vs_1.root().entry("key_2").or_zeros(&[1, 5, 9]); + + let vs_2 = VarStore::new(Device::Cpu); + let _ = vs_2.root().entry("key_3").or_zeros(&[2, 4, 4]); + let _ = vs_2.root().entry("key_4").or_zeros(&[5, 2, 3]); + + let merged_vs = VarStore::merge(vec![(vs_1, None), (vs_2, None)]).unwrap(); + assert_eq!(merged_vs.variables().len(), 4); + assert_eq!(merged_vs.trainable_variables().len(), 4); + assert!(merged_vs.variables().contains_key("key_1")); + assert!(merged_vs.variables().contains_key("key_2")); + assert!(merged_vs.variables().contains_key("key_3")); + assert!(merged_vs.variables().contains_key("key_4")); +} + +#[test] +fn merge_var_stores_conflicting_keys() { + let vs_1 = VarStore::new(Device::Cpu); + let _ = vs_1.root().entry("duplicate_key_1").or_zeros(&[3, 1, 4]); + let _ = vs_1.root().entry("key_2").or_zeros(&[1, 5, 9]); + + let vs_2 = VarStore::new(Device::Cpu); + let _ = vs_2.root().entry("duplicate_key_1").or_zeros(&[2, 4, 4]); + let _ = vs_2.root().entry("key_3").or_zeros(&[5, 2, 3]); + + let merged_vs = VarStore::merge(vec![(vs_1, None), (vs_2, None)]); + assert!(matches!(merged_vs, Err(TchError::Torch(t)) if t== + "Duplicate variable name found: duplicate_key_1. Provide a unique prefix to allow merge operation")); +} + +#[test] +fn merge_var_stores_with_prefixes() { + let vs_1 = VarStore::new(Device::Cpu); + let _ = vs_1.root().entry("key_1").or_zeros(&[3, 1, 4]); + let _ = vs_1.root().entry("key_2").or_zeros(&[1, 5, 9]); + + let vs_2 = VarStore::new(Device::Cpu); + let _ = vs_2.root().entry("key_3").or_zeros(&[2, 4, 4]); + let _ = vs_2.root().entry("key_4").or_zeros(&[5, 2, 3]); + + let merged_vs = VarStore::merge(vec![(vs_1, Some("vs_1.")), (vs_2, Some("vs_2."))]).unwrap(); + assert_eq!(merged_vs.variables().len(), 4); + assert_eq!(merged_vs.trainable_variables().len(), 4); + assert!(merged_vs.variables().contains_key("vs_1.key_1")); + assert!(merged_vs.variables().contains_key("vs_1.key_2")); + assert!(merged_vs.variables().contains_key("vs_2.key_3")); + assert!(merged_vs.variables().contains_key("vs_2.key_4")); +} diff --git a/v2/dependency-patches/tch-0.24.0/tests/vision_tests.rs b/v2/dependency-patches/tch-0.24.0/tests/vision_tests.rs new file mode 100644 index 0000000000..ad92e214bd --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/tests/vision_tests.rs @@ -0,0 +1,34 @@ +use tch::{nn, vision, Tensor}; + +#[test] +fn mobilenet() { + let vs = nn::VarStore::new(tch::Device::Cpu); + let net = vision::mobilenet::v2(&vs.root(), 1000); + let img = Tensor::zeros([1, 3, 224, 224], tch::kind::FLOAT_CPU); + let logits = img.apply_t(&net, false); + assert_eq!(logits.size(), [1, 1000]); +} + +#[test] +fn resize() { + // Check that resizing returns a tensor with the appropriate dimensions. + let img = Tensor::zeros([3, 8, 12], tch::kind::FLOAT_CPU); + let resized_img = vision::image::resize(&img, 16, 16).unwrap(); + assert_eq!(resized_img.size(), [3, 16, 16]); + let resized_img = vision::image::resize(&img, 4, 16).unwrap(); + assert_eq!(resized_img.size(), [3, 16, 4]); + let resized_img = vision::image::resize(&img, 32, 8).unwrap(); + assert_eq!(resized_img.size(), [3, 8, 32]); +} + +#[test] +fn resize_preserve_aspect_ratio() { + // Check that resizing returns a tensor with the appropriate dimensions. + let img = Tensor::zeros([3, 8, 12], tch::kind::FLOAT_CPU); + let resized_img = vision::image::resize_preserve_aspect_ratio(&img, 16, 16).unwrap(); + assert_eq!(resized_img.size(), [3, 16, 16]); + let resized_img = vision::image::resize(&img, 4, 16).unwrap(); + assert_eq!(resized_img.size(), [3, 16, 4]); + let resized_img = vision::image::resize(&img, 32, 8).unwrap(); + assert_eq!(resized_img.size(), [3, 8, 32]); +} diff --git a/v2/dependency-patches/tch-0.24.0/uv.lock b/v2/dependency-patches/tch-0.24.0/uv.lock new file mode 100644 index 0000000000..b0a9edbba1 --- /dev/null +++ b/v2/dependency-patches/tch-0.24.0/uv.lock @@ -0,0 +1,433 @@ +version = 1 +revision = 3 +requires-python = ">=3.12" + +[[package]] +name = "cuda-bindings" +version = "13.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-pathfinder" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/52/c8/b2589d68acf7e3d63e2be330b84bc25712e97ed799affbca7edd7eae25d6/cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e865447abfb83d6a98ad5130ed3c70b1fc295ae3eeee39fd07b4ddb0671b6788", size = 5722404, upload-time = "2026-03-11T00:12:44.041Z" }, + { url = "https://files.pythonhosted.org/packages/1f/92/f899f7bbb5617bb65ec52a6eac1e9a1447a86b916c4194f8a5001b8cde0c/cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46d8776a55d6d5da9dd6e9858fba2efcda2abe6743871dee47dd06eb8cb6d955", size = 6320619, upload-time = "2026-03-11T00:12:45.939Z" }, + { url = "https://files.pythonhosted.org/packages/df/93/eef988860a3ca985f82c4f3174fc0cdd94e07331ba9a92e8e064c260337f/cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6629ca2df6f795b784752409bcaedbd22a7a651b74b56a165ebc0c9dcbd504d0", size = 5614610, upload-time = "2026-03-11T00:12:50.337Z" }, + { url = "https://files.pythonhosted.org/packages/18/23/6db3aba46864aee357ab2415135b3fe3da7e9f1fa0221fa2a86a5968099c/cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dca0da053d3b4cc4869eff49c61c03f3c5dbaa0bcd712317a358d5b8f3f385d", size = 6149914, upload-time = "2026-03-11T00:12:52.374Z" }, + { url = "https://files.pythonhosted.org/packages/c0/87/87a014f045b77c6de5c8527b0757fe644417b184e5367db977236a141602/cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6464b30f46692d6c7f65d4a0e0450d81dd29de3afc1bb515653973d01c2cd6e", size = 5685673, upload-time = "2026-03-11T00:12:56.371Z" }, + { url = "https://files.pythonhosted.org/packages/ee/5e/c0fe77a73aaefd3fff25ffaccaac69c5a63eafdf8b9a4c476626ef0ac703/cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4af9f3e1be603fa12d5ad6cfca7844c9d230befa9792b5abdf7dd79979c3626", size = 6191386, upload-time = "2026-03-11T00:12:58.965Z" }, + { url = "https://files.pythonhosted.org/packages/5f/58/ed2c3b39c8dd5f96aa7a4abef0d47a73932c7a988e30f5fa428f00ed0da1/cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df850a1ff8ce1b3385257b08e47b70e959932f5f432d0a4e46a355962b4e4771", size = 5507469, upload-time = "2026-03-11T00:13:04.063Z" }, + { url = "https://files.pythonhosted.org/packages/1f/01/0c941b112ceeb21439b05895eace78ca1aa2eaaf695c8521a068fd9b4c00/cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8a16384c6494e5485f39314b0b4afb04bee48d49edb16d5d8593fd35bbd231b", size = 6059693, upload-time = "2026-03-11T00:13:06.003Z" }, +] + +[[package]] +name = "cuda-pathfinder" +version = "1.3.3" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0b/02/4dbe7568a42e46582248942f54dc64ad094769532adbe21e525e4edf7bc4/cuda_pathfinder-1.3.3-py3-none-any.whl", hash = "sha256:9984b664e404f7c134954a771be8775dfd6180ea1e1aef4a5a37d4be05d9bbb1", size = 27154, upload-time = "2025-12-04T22:35:08.996Z" }, +] + +[[package]] +name = "cuda-toolkit" +version = "13.0.2" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/57/b2/453099f5f3b698d7d0eab38916aac44c7f76229f451709e2eb9db6615dcd/cuda_toolkit-13.0.2-py2.py3-none-any.whl", hash = "sha256:b198824cf2f54003f50d64ada3a0f184b42ca0846c1c94192fa269ecd97a66eb", size = 2364, upload-time = "2025-12-19T23:24:07.328Z" }, +] + +[package.optional-dependencies] +cublas = [ + { name = "nvidia-cublas", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +cudart = [ + { name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +cufft = [ + { name = "nvidia-cufft", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +cufile = [ + { name = "nvidia-cufile", marker = "sys_platform == 'linux'" }, +] +cupti = [ + { name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +curand = [ + { name = "nvidia-curand", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +cusolver = [ + { name = "nvidia-cusolver", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +cusparse = [ + { name = "nvidia-cusparse", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +nvjitlink = [ + { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +nvrtc = [ + { name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] +nvtx = [ + { name = "nvidia-nvtx", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, +] + +[[package]] +name = "filelock" +version = "3.20.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1d/65/ce7f1b70157833bf3cb851b556a37d4547ceafc158aa9b34b36782f23696/filelock-3.20.3.tar.gz", hash = "sha256:18c57ee915c7ec61cff0ecf7f0f869936c7c30191bb0cf406f1341778d0834e1", size = 19485, upload-time = "2026-01-09T17:55:05.421Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b5/36/7fb70f04bf00bc646cd5bb45aa9eddb15e19437a28b8fb2b4a5249fac770/filelock-3.20.3-py3-none-any.whl", hash = "sha256:4b0dda527ee31078689fc205ec4f1c1bf7d56cf88b6dc9426c4f230e46c2dce1", size = 16701, upload-time = "2026-01-09T17:55:04.334Z" }, +] + +[[package]] +name = "fsspec" +version = "2025.10.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/24/7f/2747c0d332b9acfa75dc84447a066fdf812b5a6b8d30472b74d309bfe8cb/fsspec-2025.10.0.tar.gz", hash = "sha256:b6789427626f068f9a83ca4e8a3cc050850b6c0f71f99ddb4f542b8266a26a59", size = 309285, upload-time = "2025-10-30T14:58:44.036Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/02/a6b21098b1d5d6249b7c5ab69dde30108a71e4e819d4a9778f1de1d5b70d/fsspec-2025.10.0-py3-none-any.whl", hash = "sha256:7c7712353ae7d875407f97715f0e1ffcc21e33d5b24556cb1e090ae9409ec61d", size = 200966, upload-time = "2025-10-30T14:58:42.53Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, + { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, + { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, + { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, + { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, + { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, + { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, + { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, + { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, + { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, + { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, + { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, + { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, + { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, + { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, + { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, + { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, + { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, + { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, + { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, + { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, + { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, + { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, + { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, + { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, + { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, + { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, + { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, + { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, + { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, + { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, + { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, + { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, + { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, + { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, + { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, + { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, + { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, + { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, + { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, + { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, + { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, + { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, + { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, + { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, + { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, + { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, + { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, + { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, + { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, + { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, +] + +[[package]] +name = "mpmath" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, +] + +[[package]] +name = "networkx" +version = "3.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6c/4f/ccdb8ad3a38e583f214547fd2f7ff1fc160c43a75af88e6aec213404b96a/networkx-3.5.tar.gz", hash = "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037", size = 2471065, upload-time = "2025-05-29T11:35:07.804Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl", hash = "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec", size = 2034406, upload-time = "2025-05-29T11:35:04.961Z" }, +] + +[[package]] +name = "nvidia-cublas" +version = "13.1.0.3" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/a5/fce49e2ae977e0ccc084e5adafceb4f0ac0c8333cb6863501618a7277f67/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:c86fc7f7ae36d7528288c5d88098edcb7b02c633d262e7ddbb86b0ad91be5df2", size = 542851226, upload-time = "2025-10-09T08:59:04.818Z" }, + { url = "https://files.pythonhosted.org/packages/e7/44/423ac00af4dd95a5aeb27207e2c0d9b7118702149bf4704c3ddb55bb7429/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:ee8722c1f0145ab246bccb9e452153b5e0515fd094c3678df50b2a0888b8b171", size = 423133236, upload-time = "2025-10-09T08:59:32.536Z" }, +] + +[[package]] +name = "nvidia-cuda-cupti" +version = "13.0.85" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/2a/80353b103fc20ce05ef51e928daed4b6015db4aaa9162ed0997090fe2250/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151", size = 10310827, upload-time = "2025-09-04T08:26:42.012Z" }, + { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597, upload-time = "2025-09-04T08:26:51.312Z" }, +] + +[[package]] +name = "nvidia-cuda-nvrtc" +version = "13.0.88" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575", size = 90215200, upload-time = "2025-09-04T08:28:44.204Z" }, + { url = "https://files.pythonhosted.org/packages/b7/dc/6bb80850e0b7edd6588d560758f17e0550893a1feaf436807d64d2da040f/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b", size = 43015449, upload-time = "2025-09-04T08:28:20.239Z" }, +] + +[[package]] +name = "nvidia-cuda-runtime" +version = "13.0.96" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/87/4f/17d7b9b8e285199c58ce28e31b5c5bbaa4d8271af06a89b6405258245de2/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55", size = 2261060, upload-time = "2025-10-09T08:55:15.78Z" }, + { url = "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548", size = 2243632, upload-time = "2025-10-09T08:55:36.117Z" }, +] + +[[package]] +name = "nvidia-cudnn-cu13" +version = "9.19.0.56" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f1/84/26025437c1e6b61a707442184fa0c03d083b661adf3a3eecfd6d21677740/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:6ed29ffaee1176c612daf442e4dd6cfeb6a0caa43ddcbeb59da94953030b1be4", size = 433781201, upload-time = "2026-02-03T20:40:53.805Z" }, + { url = "https://files.pythonhosted.org/packages/a3/22/0b4b932655d17a6da1b92fa92ab12844b053bb2ac2475e179ba6f043da1e/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:d20e1734305e9d68889a96e3f35094d733ff1f83932ebe462753973e53a572bf", size = 366066321, upload-time = "2026-02-03T20:44:52.837Z" }, +] + +[[package]] +name = "nvidia-cufft" +version = "12.0.0.61" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, + { url = "https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3", size = 214085489, upload-time = "2025-09-04T08:31:56.044Z" }, +] + +[[package]] +name = "nvidia-cufile" +version = "1.15.1.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44", size = 1223672, upload-time = "2025-09-04T08:32:22.779Z" }, + { url = "https://files.pythonhosted.org/packages/ab/73/cc4a14c9813a8a0d509417cf5f4bdaba76e924d58beb9864f5a7baceefbf/nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1", size = 1136992, upload-time = "2025-09-04T08:32:14.119Z" }, +] + +[[package]] +name = "nvidia-curand" +version = "10.4.0.35" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/72/7c2ae24fb6b63a32e6ae5d241cc65263ea18d08802aaae087d9f013335a2/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a", size = 61962106, upload-time = "2025-08-04T10:21:41.128Z" }, + { url = "https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc", size = 59544258, upload-time = "2025-08-04T10:22:03.992Z" }, +] + +[[package]] +name = "nvidia-cusolver" +version = "12.0.4.66" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-cublas" }, + { name = "nvidia-cusparse" }, + { name = "nvidia-nvjitlink" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, + { url = "https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112", size = 200941980, upload-time = "2025-09-04T08:33:22.767Z" }, +] + +[[package]] +name = "nvidia-cusparse" +version = "12.6.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nvidia-nvjitlink" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, + { url = "https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b", size = 145942937, upload-time = "2025-09-04T08:33:58.029Z" }, +] + +[[package]] +name = "nvidia-cusparselt-cu13" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/46/10/8dcd1175260706a2fc92a16a52e306b71d4c1ea0b0cc4a9484183399818a/nvidia_cusparselt_cu13-0.8.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:400c6ed1cf6780fc6efedd64ec9f1345871767e6a1a0a552a1ea0578117ea77c", size = 220791277, upload-time = "2025-08-13T19:22:40.982Z" }, + { url = "https://files.pythonhosted.org/packages/fd/53/43b0d71f4e702fa9733f8b4571fdca50a8813f1e450b656c239beff12315/nvidia_cusparselt_cu13-0.8.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:25e30a8a7323935d4ad0340b95a0b69926eee755767e8e0b1cf8dd85b197d3fd", size = 169884119, upload-time = "2025-08-13T19:23:41.967Z" }, +] + +[[package]] +name = "nvidia-nccl-cu13" +version = "2.28.9" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/55/1920646a2e43ffd4fc958536b276197ed740e9e0c54105b4bb3521591fc7/nvidia_nccl_cu13-2.28.9-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:01c873ba1626b54caa12272ed228dc5b2781545e0ae8ba3f432a8ef1c6d78643", size = 196561677, upload-time = "2025-11-18T05:49:03.45Z" }, + { url = "https://files.pythonhosted.org/packages/b0/b4/878fefaad5b2bcc6fcf8d474a25e3e3774bc5133e4b58adff4d0bca238bc/nvidia_nccl_cu13-2.28.9-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:e4553a30f34195f3fa1da02a6da3d6337d28f2003943aa0a3d247bbc25fefc42", size = 196493177, upload-time = "2025-11-18T05:49:17.677Z" }, +] + +[[package]] +name = "nvidia-nvjitlink" +version = "13.0.88" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/7a/123e033aaff487c77107195fa5a2b8686795ca537935a24efae476c41f05/nvidia_nvjitlink-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:13a74f429e23b921c1109976abefacc69835f2f433ebd323d3946e11d804e47b", size = 40713933, upload-time = "2025-09-04T08:35:43.553Z" }, + { url = "https://files.pythonhosted.org/packages/ab/2c/93c5250e64df4f894f1cbb397c6fd71f79813f9fd79d7cd61de3f97b3c2d/nvidia_nvjitlink-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e931536ccc7d467a98ba1d8b89ff7fa7f1fa3b13f2b0069118cd7f47bff07d0c", size = 38768748, upload-time = "2025-09-04T08:35:20.008Z" }, +] + +[[package]] +name = "nvidia-nvshmem-cu13" +version = "3.4.5" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/0f/05cc9c720236dcd2db9c1ab97fff629e96821be2e63103569da0c9b72f19/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9", size = 60215947, upload-time = "2025-09-06T00:32:20.022Z" }, + { url = "https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80", size = 60412546, upload-time = "2025-09-06T00:32:41.564Z" }, +] + +[[package]] +name = "nvidia-nvtx" +version = "13.0.85" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4", size = 148047, upload-time = "2025-09-04T08:29:01.761Z" }, + { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, +] + +[[package]] +name = "setuptools" +version = "80.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887f014ea83c24ae194cfcd12b9e0fda66b957c69d1fca/setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c", size = 1319958, upload-time = "2025-05-27T00:56:51.443Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" }, +] + +[[package]] +name = "sympy" +version = "1.14.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mpmath" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, +] + +[[package]] +name = "tch-rs" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "torch" }, +] + +[package.metadata] +requires-dist = [{ name = "torch", specifier = "==2.11.0" }] + +[[package]] +name = "torch" +version = "2.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cuda-bindings", marker = "sys_platform == 'linux'" }, + { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx" }, + { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, + { name = "setuptools" }, + { name = "sympy" }, + { name = "triton", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions" }, +] +wheels = [ + { url = "https://files.pythonhosted.org/packages/6f/8b/69e3008d78e5cee2b30183340cc425081b78afc5eff3d080daab0adda9aa/torch-2.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4b5866312ee6e52ea625cd211dcb97d6a2cdc1131a5f15cc0d87eec948f6dd34", size = 80606338, upload-time = "2026-03-23T18:11:34.781Z" }, + { url = "https://files.pythonhosted.org/packages/13/16/42e5915ebe4868caa6bac83a8ed59db57f12e9a61b7d749d584776ed53d5/torch-2.11.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:f99924682ef0aa6a4ab3b1b76f40dc6e273fca09f367d15a524266db100a723f", size = 419731115, upload-time = "2026-03-23T18:11:06.944Z" }, + { url = "https://files.pythonhosted.org/packages/1a/c9/82638ef24d7877510f83baf821f5619a61b45568ce21c0a87a91576510aa/torch-2.11.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:0f68f4ac6d95d12e896c3b7a912b5871619542ec54d3649cf48cc1edd4dd2756", size = 530712279, upload-time = "2026-03-23T18:10:31.481Z" }, + { url = "https://files.pythonhosted.org/packages/1c/ff/6756f1c7ee302f6d202120e0f4f05b432b839908f9071157302cedfc5232/torch-2.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:fbf39280699d1b869f55eac536deceaa1b60bd6788ba74f399cc67e60a5fab10", size = 114556047, upload-time = "2026-03-23T18:10:55.931Z" }, + { url = "https://files.pythonhosted.org/packages/87/89/5ea6722763acee56b045435fb84258db7375c48165ec8be7880ab2b281c5/torch-2.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1e6debd97ccd3205bbb37eb806a9d8219e1139d15419982c09e23ef7d4369d18", size = 80606801, upload-time = "2026-03-23T18:10:18.649Z" }, + { url = "https://files.pythonhosted.org/packages/32/d1/8ed2173589cbfe744ed54e5a73efc107c0085ba5777ee93a5f4c1ab90553/torch-2.11.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:63a68fa59de8f87acc7e85a5478bb2dddbb3392b7593ec3e78827c793c4b73fd", size = 419732382, upload-time = "2026-03-23T18:08:30.835Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e1/b73f7c575a4b8f87a5928f50a1e35416b5e27295d8be9397d5293e7e8d4c/torch-2.11.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:cc89b9b173d9adfab59fd227f0ab5e5516d9a52b658ae41d64e59d2e55a418db", size = 530711509, upload-time = "2026-03-23T18:08:47.213Z" }, + { url = "https://files.pythonhosted.org/packages/66/82/3e3fcdd388fbe54e29fd3f991f36846ff4ac90b0d0181e9c8f7236565f82/torch-2.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:4dda3b3f52d121063a731ddb835f010dc137b920d7fec2778e52f60d8e4bf0cd", size = 114555842, upload-time = "2026-03-23T18:09:52.111Z" }, + { url = "https://files.pythonhosted.org/packages/db/38/8ac78069621b8c2b4979c2f96dc8409ef5e9c4189f6aac629189a78677ca/torch-2.11.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8b394322f49af4362d4f80e424bcaca7efcd049619af03a4cf4501520bdf0fb4", size = 80959574, upload-time = "2026-03-23T18:10:14.214Z" }, + { url = "https://files.pythonhosted.org/packages/6d/6c/56bfb37073e7136e6dd86bfc6af7339946dd684e0ecf2155ac0eee687ae1/torch-2.11.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:2658f34ce7e2dabf4ec73b45e2ca68aedad7a5be87ea756ad656eaf32bf1e1ea", size = 419732324, upload-time = "2026-03-23T18:09:36.604Z" }, + { url = "https://files.pythonhosted.org/packages/07/f4/1b666b6d61d3394cca306ea543ed03a64aad0a201b6cd159f1d41010aeb1/torch-2.11.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:98bb213c3084cfe176302949bdc360074b18a9da7ab59ef2edc9d9f742504778", size = 530596026, upload-time = "2026-03-23T18:09:20.842Z" }, + { url = "https://files.pythonhosted.org/packages/48/6b/30d1459fa7e4b67e9e3fe1685ca1d8bb4ce7c62ef436c3a615963c6c866c/torch-2.11.0-cp313-cp313t-win_amd64.whl", hash = "sha256:a97b94bbf62992949b4730c6cd2cc9aee7b335921ee8dc207d930f2ed09ae2db", size = 114793702, upload-time = "2026-03-23T18:09:47.304Z" }, + { url = "https://files.pythonhosted.org/packages/26/0d/8603382f61abd0db35841148ddc1ffd607bf3100b11c6e1dab6d2fc44e72/torch-2.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:01018087326984a33b64e04c8cb5c2795f9120e0d775ada1f6638840227b04d7", size = 80573442, upload-time = "2026-03-23T18:09:10.117Z" }, + { url = "https://files.pythonhosted.org/packages/c7/86/7cd7c66cb9cec6be330fff36db5bd0eef386d80c031b581ec81be1d4b26c/torch-2.11.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:2bb3cc54bd0dea126b0060bb1ec9de0f9c7f7342d93d436646516b0330cd5be7", size = 419749385, upload-time = "2026-03-23T18:07:33.77Z" }, + { url = "https://files.pythonhosted.org/packages/47/e8/b98ca2d39b2e0e4730c0ee52537e488e7008025bc77ca89552ff91021f7c/torch-2.11.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:4dc8b3809469b6c30b411bb8c4cad3828efd26236153d9beb6a3ec500f211a60", size = 530716756, upload-time = "2026-03-23T18:07:50.02Z" }, + { url = "https://files.pythonhosted.org/packages/78/88/d4a4cda8362f8a30d1ed428564878c3cafb0d87971fbd3947d4c84552095/torch-2.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:2b4e811728bd0cc58fb2b0948fe939a1ee2bf1422f6025be2fca4c7bd9d79718", size = 114552300, upload-time = "2026-03-23T18:09:05.617Z" }, + { url = "https://files.pythonhosted.org/packages/bf/46/4419098ed6d801750f26567b478fc185c3432e11e2cad712bc6b4c2ab0d0/torch-2.11.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8245477871c3700d4370352ffec94b103cfcb737229445cf9946cddb7b2ca7cd", size = 80959460, upload-time = "2026-03-23T18:09:00.818Z" }, + { url = "https://files.pythonhosted.org/packages/fd/66/54a56a4a6ceaffb567231994a9745821d3af922a854ed33b0b3a278e0a99/torch-2.11.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:ab9a8482f475f9ba20e12db84b0e55e2f58784bdca43a854a6ccd3fd4b9f75e6", size = 419735835, upload-time = "2026-03-23T18:07:18.974Z" }, + { url = "https://files.pythonhosted.org/packages/b1/e7/0b6665f533aa9e337662dc190425abc0af1fe3234088f4454c52393ded61/torch-2.11.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:563ed3d25542d7e7bbc5b235ccfacfeb97fb470c7fee257eae599adb8005c8a2", size = 530613405, upload-time = "2026-03-23T18:08:07.014Z" }, + { url = "https://files.pythonhosted.org/packages/cf/bf/c8d12a2c86dbfd7f40fb2f56fbf5a505ccf2d9ce131eb559dfc7c51e1a04/torch-2.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b2a43985ff5ef6ddd923bbcf99943e5f58059805787c5c9a2622bf05ca2965b0", size = 114792991, upload-time = "2026-03-23T18:08:19.216Z" }, +] + +[[package]] +name = "triton" +version = "3.6.0" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/17/5d/08201db32823bdf77a0e2b9039540080b2e5c23a20706ddba942924ebcd6/triton-3.6.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:374f52c11a711fd062b4bfbb201fd9ac0a5febd28a96fb41b4a0f51dde3157f4", size = 176128243, upload-time = "2026-01-20T16:16:07.857Z" }, + { url = "https://files.pythonhosted.org/packages/ab/a8/cdf8b3e4c98132f965f88c2313a4b493266832ad47fb52f23d14d4f86bb5/triton-3.6.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74caf5e34b66d9f3a429af689c1c7128daba1d8208df60e81106b115c00d6fca", size = 188266850, upload-time = "2026-01-20T16:00:43.041Z" }, + { url = "https://files.pythonhosted.org/packages/3c/12/34d71b350e89a204c2c7777a9bba0dcf2f19a5bfdd70b57c4dbc5ffd7154/triton-3.6.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:448e02fe6dc898e9e5aa89cf0ee5c371e99df5aa5e8ad976a80b93334f3494fd", size = 176133521, upload-time = "2026-01-20T16:16:13.321Z" }, + { url = "https://files.pythonhosted.org/packages/f9/0b/37d991d8c130ce81a8728ae3c25b6e60935838e9be1b58791f5997b24a54/triton-3.6.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10c7f76c6e72d2ef08df639e3d0d30729112f47a56b0c81672edc05ee5116ac9", size = 188289450, upload-time = "2026-01-20T16:00:49.136Z" }, + { url = "https://files.pythonhosted.org/packages/ce/4e/41b0c8033b503fd3cfcd12392cdd256945026a91ff02452bef40ec34bee7/triton-3.6.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1722e172d34e32abc3eb7711d0025bb69d7959ebea84e3b7f7a341cd7ed694d6", size = 176276087, upload-time = "2026-01-20T16:16:18.989Z" }, + { url = "https://files.pythonhosted.org/packages/35/f8/9c66bfc55361ec6d0e4040a0337fb5924ceb23de4648b8a81ae9d33b2b38/triton-3.6.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d002e07d7180fd65e622134fbd980c9a3d4211fb85224b56a0a0efbd422ab72f", size = 188400296, upload-time = "2026-01-20T16:00:56.042Z" }, + { url = "https://files.pythonhosted.org/packages/49/55/5ecf0dcaa0f2fbbd4420f7ef227ee3cb172e91e5fede9d0ecaddc43363b4/triton-3.6.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef5523241e7d1abca00f1d240949eebdd7c673b005edbbce0aca95b8191f1d43", size = 176138577, upload-time = "2026-01-20T16:16:25.426Z" }, + { url = "https://files.pythonhosted.org/packages/df/3d/9e7eee57b37c80cec63322c0231bb6da3cfe535a91d7a4d64896fcb89357/triton-3.6.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a17a5d5985f0ac494ed8a8e54568f092f7057ef60e1b0fa09d3fd1512064e803", size = 188273063, upload-time = "2026-01-20T16:01:07.278Z" }, + { url = "https://files.pythonhosted.org/packages/48/db/56ee649cab5eaff4757541325aca81f52d02d4a7cd3506776cad2451e060/triton-3.6.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b3a97e8ed304dfa9bd23bb41ca04cdf6b2e617d5e782a8653d616037a5d537d", size = 176274804, upload-time = "2026-01-20T16:16:31.528Z" }, + { url = "https://files.pythonhosted.org/packages/f6/56/6113c23ff46c00aae423333eb58b3e60bdfe9179d542781955a5e1514cb3/triton-3.6.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46bd1c1af4b6704e554cad2eeb3b0a6513a980d470ccfa63189737340c7746a7", size = 188397994, upload-time = "2026-01-20T16:01:14.236Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +] diff --git a/v2/dependency-patches/torch-sys-0.24.0/Cargo.toml b/v2/dependency-patches/torch-sys-0.24.0/Cargo.toml new file mode 100644 index 0000000000..248d98a17a --- /dev/null +++ b/v2/dependency-patches/torch-sys-0.24.0/Cargo.toml @@ -0,0 +1,79 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +name = "torch-sys" +version = "0.24.0" +authors = ["Laurent Mazare "] +build = "build.rs" +links = "tch" +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Low-level FFI bindings for the PyTorch C++ api (libtorch)." +readme = false +keywords = [ + "pytorch", + "ffi", + "deep-learning", + "machine-learning", +] +categories = [ + "external-ffi-bindings", + "science", +] +license = "MIT/Apache-2.0" +repository = "https://github.com/LaurentMazare/tch-rs" + +[package.metadata.docs.rs] +features = ["doc-only"] + +[features] +doc-only = [] +download-libtorch = [ + "ureq", + "serde", + "serde_json", +] +python-extension = [] + +[lib] +name = "torch_sys" +path = "src/lib.rs" + +[dependencies.libc] +version = "0.2.0" + +[build-dependencies.anyhow] +version = "^1.0.60" + +[build-dependencies.cc] +version = "1.2" + +[build-dependencies.serde] +version = "1.0" +features = ["derive"] +optional = true + +[build-dependencies.serde_json] +version = "1.0" +optional = true + +[build-dependencies.ureq] +version = "2.6" +features = ["json"] +optional = true + +[build-dependencies.zip] +version = "9.0.1" diff --git a/v2/dependency-patches/torch-sys-0.24.0/Cargo.toml.orig b/v2/dependency-patches/torch-sys-0.24.0/Cargo.toml.orig new file mode 100644 index 0000000000..db835c217c --- /dev/null +++ b/v2/dependency-patches/torch-sys-0.24.0/Cargo.toml.orig @@ -0,0 +1,32 @@ +[package] +name = "torch-sys" +version = "0.24.0" +authors = ["Laurent Mazare "] +edition = "2021" +build = "build.rs" +links = "tch" + +description = "Low-level FFI bindings for the PyTorch C++ api (libtorch)." +repository = "https://github.com/LaurentMazare/tch-rs" +keywords = ["pytorch", "ffi", "deep-learning", "machine-learning"] +categories = ["external-ffi-bindings", "science"] +license = "MIT/Apache-2.0" + +[dependencies] +libc = "0.2.0" + +[build-dependencies] +anyhow = "^1.0.60" +cc = "1.2" +ureq = { version = "2.6", optional = true, features = ["json"] } +serde_json = { version = "1.0", optional = true } +serde = { version = "1.0", optional = true, features = ["derive"] } +zip = "0.6" + +[features] +download-libtorch = ["ureq", "serde", "serde_json"] +doc-only = [] +python-extension = [] + +[package.metadata.docs.rs] +features = [ "doc-only" ] diff --git a/v2/dependency-patches/torch-sys-0.24.0/build.rs b/v2/dependency-patches/torch-sys-0.24.0/build.rs new file mode 100644 index 0000000000..9cb2e59859 --- /dev/null +++ b/v2/dependency-patches/torch-sys-0.24.0/build.rs @@ -0,0 +1,493 @@ +// The LIBTORCH environment variable can be used to specify the directory +// where libtorch has been installed. +// When not specified this script downloads the cpu version for libtorch +// and extracts it in OUT_DIR. +// +// On Linux, the TORCH_CUDA_VERSION environment variable can be used, +// like 9.0, 90, or cu90 to specify the version of CUDA to use for libtorch. + +use anyhow::{Context, Result}; +use std::path::{Path, PathBuf}; +use std::{env, fs, io}; + +const TORCH_VERSION: &str = "2.11.0"; +const PYTHON_PRINT_PYTORCH_DETAILS: &str = r" +import torch +from torch.utils import cpp_extension +print('LIBTORCH_VERSION:', torch.__version__.split('+')[0]) +print('LIBTORCH_CXX11:', torch._C._GLIBCXX_USE_CXX11_ABI) +for include_path in cpp_extension.include_paths(): + print('LIBTORCH_INCLUDE:', include_path) +for library_path in cpp_extension.library_paths(): + print('LIBTORCH_LIB:', library_path) +"; + +const PYTHON_PRINT_INCLUDE_PATH: &str = r" +import sysconfig +print('PYTHON_INCLUDE:', sysconfig.get_path('include')) +"; + +const NO_DOWNLOAD_ERROR_MESSAGE: &str = r" +Cannot find a libtorch install, you can either: +- Install libtorch manually and set the LIBTORCH environment variable to appropriate path. +- Use a system wide install in /usr/lib/libtorch.so. +- Use a Python environment with PyTorch installed by setting LIBTORCH_USE_PYTORCH=1 + +See the readme for more details: +https://github.com/LaurentMazare/tch-rs/blob/main/README.md +"; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum LinkType { + Dynamic, + Static, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Os { + Linux, + Macos, + Windows, +} + +#[allow(dead_code)] +#[derive(Debug, Clone)] +struct SystemInfo { + os: Os, + python_interpreter: PathBuf, + cxx11_abi: String, + libtorch_include_dirs: Vec, + libtorch_lib_dir: PathBuf, + link_type: LinkType, +} + +#[cfg(feature = "ureq")] +fn download>(source_url: &str, target_file: P) -> anyhow::Result<()> { + let f = fs::File::create(&target_file)?; + let mut writer = io::BufWriter::new(f); + let response = ureq::get(source_url).call()?; + let response_code = response.status(); + if response_code != 200 { + anyhow::bail!("Unexpected response code {} for {}", response_code, source_url) + } + let mut reader = response.into_reader(); + std::io::copy(&mut reader, &mut writer)?; + Ok(()) +} + +#[cfg(not(feature = "ureq"))] +fn download>(_source_url: &str, _target_file: P) -> anyhow::Result<()> { + anyhow::bail!("cannot use download without the ureq feature") +} + +#[cfg(not(feature = "download-libtorch"))] +fn get_pypi_wheel_url_for_aarch64_macosx() -> anyhow::Result { + anyhow::bail!("cannot get pypi wheel url without the ureq feature") +} + +#[cfg(feature = "download-libtorch")] +#[derive(serde::Deserialize, Debug)] +struct PyPiPackageUrl { + url: String, + filename: String, +} +#[cfg(feature = "download-libtorch")] +#[derive(serde::Deserialize, Debug)] +struct PyPiPackage { + urls: Vec, +} +#[cfg(feature = "download-libtorch")] +fn get_pypi_wheel_url_for_aarch64_macosx() -> anyhow::Result { + let pypi_url = format!("https://pypi.org/pypi/torch/{TORCH_VERSION}/json"); + let response = ureq::get(pypi_url.as_str()).call()?; + let response_code = response.status(); + if response_code != 200 { + anyhow::bail!("Unexpected response code {} for {}", response_code, pypi_url) + } + let pypi_package: PyPiPackage = response.into_json()?; + let urls = pypi_package.urls; + let expected_filename = format!("torch-{TORCH_VERSION}-cp311-none-macosx_11_0_arm64.whl"); + let url = urls.iter().find_map(|pypi_url: &PyPiPackageUrl| { + if pypi_url.filename == expected_filename { + Some(pypi_url.url.clone()) + } else { + None + } + }); + url.context("Failed to find arm64 macosx wheel from pypi") +} + +fn extract>(filename: P, outpath: P) -> anyhow::Result<()> { + let file = fs::File::open(&filename)?; + let buf = io::BufReader::new(file); + let mut archive = zip::ZipArchive::new(buf)?; + // ZIP 9's checked extraction rejects path/symlink escapes and unsafe permissions. + archive.extract(outpath.as_ref())?; + + // This is if we're unzipping a python wheel. + if outpath.as_ref().join("torch").exists() { + fs::rename(outpath.as_ref().join("torch"), outpath.as_ref().join("libtorch"))?; + } + Ok(()) +} + +fn env_var_rerun(name: &str) -> Result { + println!("cargo:rerun-if-env-changed={name}"); + env::var(name) +} + +fn version_check(version: &str) -> Result<()> { + if env_var_rerun("LIBTORCH_BYPASS_VERSION_CHECK").is_ok() { + return Ok(()); + } + let version = version.trim(); + // Typical version number is 2.11.0+cpu or 2.11.0+cu129 + let version = match version.split_once('+') { + None => version, + Some((version, _)) => version, + }; + if version != TORCH_VERSION { + anyhow::bail!("this tch version expects PyTorch {TORCH_VERSION}, got {version}, this check can be bypassed by setting the LIBTORCH_BYPASS_VERSION_CHECK environment variable") + } + Ok(()) +} + +impl SystemInfo { + fn new() -> Result { + let os = match env::var("CARGO_CFG_TARGET_OS").expect("Unable to get TARGET_OS").as_str() { + "linux" => Os::Linux, + "windows" => Os::Windows, + "macos" => Os::Macos, + os => anyhow::bail!("unsupported TARGET_OS '{os}'"), + }; + // Locate the currently active Python binary, similar to: + // https://github.com/PyO3/maturin/blob/243b8ec91d07113f97a6fe74d9b2dcb88086e0eb/src/target.rs#L547 + let python_interpreter = match os { + Os::Windows => PathBuf::from("python.exe"), + Os::Linux | Os::Macos => { + if env::var_os("VIRTUAL_ENV").is_some() { + PathBuf::from("python") + } else { + PathBuf::from("python3") + } + } + }; + let mut libtorch_include_dirs = vec![]; + if cfg!(feature = "python-extension") { + let output = std::process::Command::new(&python_interpreter) + .arg("-c") + .arg(PYTHON_PRINT_INCLUDE_PATH) + .output() + .with_context(|| format!("error running {python_interpreter:?}"))?; + for line in String::from_utf8_lossy(&output.stdout).lines() { + if let Some(path) = line.strip_prefix("PYTHON_INCLUDE: ") { + libtorch_include_dirs.push(PathBuf::from(path)) + } + } + } + let mut libtorch_lib_dir = None; + let cxx11_abi = if env_var_rerun("LIBTORCH_USE_PYTORCH").is_ok() { + let output = std::process::Command::new(&python_interpreter) + .arg("-c") + .arg(PYTHON_PRINT_PYTORCH_DETAILS) + .output() + .with_context(|| format!("error running {python_interpreter:?}"))?; + let mut cxx11_abi = None; + for line in String::from_utf8_lossy(&output.stdout).lines() { + if let Some(version) = line.strip_prefix("LIBTORCH_VERSION: ") { + version_check(version)? + } + match line.strip_prefix("LIBTORCH_CXX11: ") { + Some("True") => cxx11_abi = Some("1".to_owned()), + Some("False") => cxx11_abi = Some("0".to_owned()), + _ => {} + } + if let Some(path) = line.strip_prefix("LIBTORCH_INCLUDE: ") { + libtorch_include_dirs.push(PathBuf::from(path)) + } + if let Some(path) = line.strip_prefix("LIBTORCH_LIB: ") { + libtorch_lib_dir = Some(PathBuf::from(path)) + } + } + match cxx11_abi { + Some(cxx11_abi) => cxx11_abi, + None => anyhow::bail!("no cxx11 abi returned by python {output:?}"), + } + } else { + let libtorch = Self::prepare_libtorch_dir(os)?; + let includes = env_var_rerun("LIBTORCH_INCLUDE") + .map(PathBuf::from) + .unwrap_or_else(|_| libtorch.clone()); + let lib = env_var_rerun("LIBTORCH_LIB") + .map(PathBuf::from) + .unwrap_or_else(|_| libtorch.clone()); + let mut version_file = libtorch; + version_file.push("build-version"); + if version_file.exists() { + if let Ok(version) = std::fs::read_to_string(&version_file) { + version_check(&version)? + } + } + libtorch_include_dirs.push(includes.join("include")); + libtorch_include_dirs.push(includes.join("include/torch/csrc/api/include")); + libtorch_lib_dir = Some(lib.join("lib")); + env_var_rerun("LIBTORCH_CXX11_ABI").unwrap_or_else(|_| "1".to_owned()) + }; + let libtorch_lib_dir = libtorch_lib_dir.expect("no libtorch lib dir found"); + let link_type = match env_var_rerun("LIBTORCH_STATIC").as_deref() { + Err(_) | Ok("0") | Ok("false") | Ok("FALSE") => LinkType::Dynamic, + Ok(_) => LinkType::Static, + }; + Ok(Self { + os, + python_interpreter, + cxx11_abi, + libtorch_include_dirs, + libtorch_lib_dir, + link_type, + }) + } + + fn check_system_location(os: Os) -> Option { + match os { + Os::Linux => Path::new("/usr/lib/libtorch.so").exists().then(|| PathBuf::from("/usr")), + _ => None, + } + } + + fn prepare_libtorch_dir(os: Os) -> Result { + if let Ok(libtorch) = env_var_rerun("LIBTORCH") { + Ok(PathBuf::from(libtorch)) + } else if let Some(pathbuf) = Self::check_system_location(os) { + Ok(pathbuf) + } else { + if !cfg!(feature = "download-libtorch") { + anyhow::bail!(NO_DOWNLOAD_ERROR_MESSAGE) + } + + let device = match env_var_rerun("TORCH_CUDA_VERSION") { + Ok(cuda_env) => match os { + Os::Linux | Os::Windows => cuda_env + .trim() + .to_lowercase() + .trim_start_matches("cu") + .split('.') + .take(2) + .fold("cu".to_owned(), |mut acc, curr| { + acc += curr; + acc + }), + os => anyhow::bail!( + "CUDA was specified with `TORCH_CUDA_VERSION`, but pre-built \ + binaries with CUDA are only available for Linux and Windows, not: {os:?}.", + ), + }, + Err(_) => "cpu".to_owned(), + }; + + let libtorch_dir = + PathBuf::from(env::var("OUT_DIR").context("OUT_DIR variable not set")?) + .join("libtorch"); + if !libtorch_dir.exists() { + fs::create_dir(&libtorch_dir).unwrap_or_default(); + let libtorch_url = match os { + Os::Linux => format!( + "https://download.pytorch.org/libtorch/{}/libtorch-shared-with-deps-{}{}.zip", + device, TORCH_VERSION, match device.as_ref() { + "cpu" => "%2Bcpu", + "cu118" => "%2Bcu118", + "cu121" => "%2Bcu121", + "cu124" => "%2Bcu124", + "cu126" => "%2Bcu126", + "cu128" => "%2Bcu128", + _ => anyhow::bail!("unsupported device {device}, TORCH_CUDA_VERSION may be set incorrectly?"), + } + ), + Os::Macos => { + if env::var("CARGO_CFG_TARGET_ARCH") == Ok(String::from("aarch64")) { + get_pypi_wheel_url_for_aarch64_macosx().expect( + "Failed to retrieve torch from pypi. Pre-built version of libtorch for apple silicon are not available. + You can install torch manually following the indications from https://github.com/LaurentMazare/tch-rs/issues/629 + pip3 install torch=={TORCH_VERSION} + Then update the following environment variables: + export LIBTORCH=$(python3 -c 'import torch; from pathlib import Path; print(Path(torch.__file__).parent)') + export DYLD_LIBRARY_PATH=${{LIBTORCH}}/lib + ") + } else { + format!("https://download.pytorch.org/libtorch/cpu/libtorch-macos-x86_64-{TORCH_VERSION}.zip") + } + }, + Os::Windows => format!( + "https://download.pytorch.org/libtorch/{}/libtorch-win-shared-with-deps-{}{}.zip", + device, TORCH_VERSION, match device.as_ref() { + "cpu" => "%2Bcpu", + "cu118" => "%2Bcu118", + "cu121" => "%2Bcu121", + "cu124" => "%2Bcu124", + "cu126" => "%2Bcu126", + "cu128" => "%2Bcu128", + _ => "" + }), + }; + + let filename = libtorch_dir.join(format!("v{TORCH_VERSION}.zip")); + download(&libtorch_url, &filename)?; + extract(&filename, &libtorch_dir)?; + } + Ok(libtorch_dir.join("libtorch")) + } + } + + fn make(&self) { + println!("cargo:rerun-if-changed=libtch/torch_python.cpp"); + println!("cargo:rerun-if-changed=libtch/torch_python.h"); + println!("cargo:rerun-if-changed=libtch/torch_api_generated.cpp"); + println!("cargo:rerun-if-changed=libtch/torch_api_generated.h"); + println!("cargo:rerun-if-changed=libtch/torch_api.cpp"); + println!("cargo:rerun-if-changed=libtch/torch_api.h"); + println!("cargo:rerun-if-changed=libtch/stb_image_write.h"); + println!("cargo:rerun-if-changed=libtch/stb_image_resize.h"); + println!("cargo:rerun-if-changed=libtch/stb_image.h"); + let mut c_files = vec!["libtch/torch_api.cpp", "libtch/torch_api_generated.cpp"]; + if cfg!(feature = "python-extension") { + c_files.push("libtch/torch_python.cpp") + } + + match self.os { + Os::Linux | Os::Macos => { + // Pass the libtorch lib dir to crates that use torch-sys. This will be available + // as DEP_TORCH_SYS_LIBTORCH_LIB, see: + // https://doc.rust-lang.org/cargo/reference/build-scripts.html#the-links-manifest-key + println!("cargo:libtorch_lib={}", self.libtorch_lib_dir.display()); + cc::Build::new() + .cpp(true) + .pic(true) + .warnings(false) + .includes(&self.libtorch_include_dirs) + .flag(format!("-Wl,-rpath={}", self.libtorch_lib_dir.display())) + .flag("-std=c++17") + .flag(format!("-D_GLIBCXX_USE_CXX11_ABI={}", self.cxx11_abi)) + .flag("-DGLOG_USE_GLOG_EXPORT") + .files(&c_files) + .compile("tch"); + } + Os::Windows => { + // TODO: Pass "/link" "LIBPATH:{}" to cl.exe in order to emulate rpath. + // Not yet supported by cc=rs. + // https://github.com/alexcrichton/cc-rs/issues/323 + cc::Build::new() + .cpp(true) + .pic(true) + .warnings(false) + .includes(&self.libtorch_include_dirs) + .flag("/std:c++17") + .flag("/p:DefineConstants=GLOG_USE_GLOG_EXPORT") + .files(&c_files) + .compile("tch"); + } + }; + } + + fn link(&self, lib_name: &str) { + match self.link_type { + LinkType::Dynamic => println!("cargo:rustc-link-lib={lib_name}"), + LinkType::Static => { + // TODO: whole-archive might only be necessary for libtorch_cpu? + println!("cargo:rustc-link-lib=static:+whole-archive,-bundle={lib_name}") + } + } + } +} + +fn main() -> anyhow::Result<()> { + if !cfg!(feature = "doc-only") { + let system_info = SystemInfo::new()?; + // use_cuda is a hacky way to detect whether cuda is available and + // if it's the case link to it by explicitly depending on a symbol + // from the torch_cuda library. + // It would be better to use -Wl,--no-as-needed but there is no way + // to specify arbitrary linker flags at the moment. + // + // Once https://github.com/rust-lang/cargo/pull/8441 is available + // we should switch to using rustc-link-arg instead e.g. with the + // following flags: + // -Wl,--no-as-needed -Wl,--copy-dt-needed-entries -ltorch + // + // This will be available starting from cargo 1.50 but will be a nightly + // only option to start with. + // https://github.com/rust-lang/cargo/blob/master/CHANGELOG.md + // + // Update: The above doesn't seem to propagate to the downstream binaries + // so doesn't really help, the comment has been kept though to keep track + // if this issue. + // TODO: Try out the as-needed native link modifier when it lands. + // https://github.com/rust-lang/rust/issues/99424 + // + // Update: it seems that the dummy dependency is not necessary anymore, so just + // removing it and keeping this comment around for legacy. + let si_lib = &system_info.libtorch_lib_dir; + let use_cuda = + si_lib.join("libtorch_cuda.so").exists() || si_lib.join("torch_cuda.dll").exists(); + let use_cuda_cu = si_lib.join("libtorch_cuda_cu.so").exists() + || si_lib.join("torch_cuda_cu.dll").exists(); + let use_cuda_cpp = si_lib.join("libtorch_cuda_cpp.so").exists() + || si_lib.join("torch_cuda_cpp.dll").exists(); + let use_hip = + si_lib.join("libtorch_hip.so").exists() || si_lib.join("torch_hip.dll").exists(); + println!("cargo:rustc-link-search=native={}", si_lib.display()); + + system_info.make(); + + println!("cargo:rustc-link-lib=static=tch"); + if use_cuda { + system_info.link("torch_cuda") + } + if use_cuda_cu { + system_info.link("torch_cuda_cu") + } + if use_cuda_cpp { + system_info.link("torch_cuda_cpp") + } + if use_hip { + system_info.link("torch_hip") + } + if cfg!(feature = "python-extension") { + system_info.link("torch_python") + } + if system_info.link_type == LinkType::Static { + // TODO: this has only be tried out on the cpu version. Check that it works + // with cuda too and maybe just try linking all available files? + system_info.link("asmjit"); + system_info.link("clog"); + system_info.link("cpuinfo"); + system_info.link("dnnl"); + system_info.link("dnnl_graph"); + system_info.link("fbgemm"); + system_info.link("gloo"); + system_info.link("kineto"); + system_info.link("nnpack"); + system_info.link("onnx"); + system_info.link("onnx_proto"); + system_info.link("protobuf"); + system_info.link("pthreadpool"); + system_info.link("pytorch_qnnpack"); + system_info.link("sleef"); + system_info.link("tensorpipe"); + system_info.link("tensorpipe_uv"); + system_info.link("XNNPACK"); + } + system_info.link("torch_cpu"); + system_info.link("torch"); + system_info.link("c10"); + if use_hip { + system_info.link("c10_hip"); + } + + let target = env::var("TARGET").context("TARGET variable not set")?; + + if !target.contains("msvc") && !target.contains("apple") { + println!("cargo:rustc-link-lib=gomp"); + } + } + Ok(()) +} diff --git a/v2/dependency-patches/torch-sys-0.24.0/libtch/CMakeLists.txt b/v2/dependency-patches/torch-sys-0.24.0/libtch/CMakeLists.txt new file mode 100644 index 0000000000..bba57bbf18 --- /dev/null +++ b/v2/dependency-patches/torch-sys-0.24.0/libtch/CMakeLists.txt @@ -0,0 +1,9 @@ +cmake_minimum_required(VERSION 3.5 FATAL_ERROR) +project(tch) + +find_package(Torch REQUIRED) + +add_library(tch STATIC torch_api.cpp torch_api_generated.cpp) +target_link_libraries(tch "${TORCH_LIBRARIES}") +set_property(TARGET tch PROPERTY CXX_STANDARD 11) +install(TARGETS tch DESTINATION .) diff --git a/v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image.h b/v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image.h new file mode 100644 index 0000000000..8b710609e3 --- /dev/null +++ b/v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image.h @@ -0,0 +1,7530 @@ +/* stb_image - v2.21 - public domain image loader - http://nothings.org/stb + no warranty implied; use at your own risk + + Do this: + #define STB_IMAGE_IMPLEMENTATION + before you include this file in *one* C or C++ file to create the implementation. + + // i.e. it should look like this: + #include ... + #include ... + #include ... + #define STB_IMAGE_IMPLEMENTATION + #include "stb_image.h" + + You can #define STBI_ASSERT(x) before the #include to avoid using assert.h. + And #define STBI_MALLOC, STBI_REALLOC, and STBI_FREE to avoid using malloc,realloc,free + + + QUICK NOTES: + Primarily of interest to game developers and other people who can + avoid problematic images and only need the trivial interface + + JPEG baseline & progressive (12 bpc/arithmetic not supported, same as stock IJG lib) + PNG 1/2/4/8/16-bit-per-channel + + TGA (not sure what subset, if a subset) + BMP non-1bpp, non-RLE + PSD (composited view only, no extra channels, 8/16 bit-per-channel) + + GIF (*comp always reports as 4-channel) + HDR (radiance rgbE format) + PIC (Softimage PIC) + PNM (PPM and PGM binary only) + + Animated GIF still needs a proper API, but here's one way to do it: + http://gist.github.com/urraka/685d9a6340b26b830d49 + + - decode from memory or through FILE (define STBI_NO_STDIO to remove code) + - decode from arbitrary I/O callbacks + - SIMD acceleration on x86/x64 (SSE2) and ARM (NEON) + + Full documentation under "DOCUMENTATION" below. + + +LICENSE + + See end of file for license information. + +RECENT REVISION HISTORY: + + 2.21 (2019-02-25) fix typo in comment + 2.20 (2019-02-07) support utf8 filenames in Windows; fix warnings and platform ifdefs + 2.19 (2018-02-11) fix warning + 2.18 (2018-01-30) fix warnings + 2.17 (2018-01-29) bugfix, 1-bit BMP, 16-bitness query, fix warnings + 2.16 (2017-07-23) all functions have 16-bit variants; optimizations; bugfixes + 2.15 (2017-03-18) fix png-1,2,4; all Imagenet JPGs; no runtime SSE detection on GCC + 2.14 (2017-03-03) remove deprecated STBI_JPEG_OLD; fixes for Imagenet JPGs + 2.13 (2016-12-04) experimental 16-bit API, only for PNG so far; fixes + 2.12 (2016-04-02) fix typo in 2.11 PSD fix that caused crashes + 2.11 (2016-04-02) 16-bit PNGS; enable SSE2 in non-gcc x64 + RGB-format JPEG; remove white matting in PSD; + allocate large structures on the stack; + correct channel count for PNG & BMP + 2.10 (2016-01-22) avoid warning introduced in 2.09 + 2.09 (2016-01-16) 16-bit TGA; comments in PNM files; STBI_REALLOC_SIZED + + See end of file for full revision history. + + + ============================ Contributors ========================= + + Image formats Extensions, features + Sean Barrett (jpeg, png, bmp) Jetro Lauha (stbi_info) + Nicolas Schulz (hdr, psd) Martin "SpartanJ" Golini (stbi_info) + Jonathan Dummer (tga) James "moose2000" Brown (iPhone PNG) + Jean-Marc Lienher (gif) Ben "Disch" Wenger (io callbacks) + Tom Seddon (pic) Omar Cornut (1/2/4-bit PNG) + Thatcher Ulrich (psd) Nicolas Guillemot (vertical flip) + Ken Miller (pgm, ppm) Richard Mitton (16-bit PSD) + github:urraka (animated gif) Junggon Kim (PNM comments) + Christopher Forseth (animated gif) Daniel Gibson (16-bit TGA) + socks-the-fox (16-bit PNG) + Jeremy Sawicki (handle all ImageNet JPGs) + Optimizations & bugfixes Mikhail Morozov (1-bit BMP) + Fabian "ryg" Giesen Anael Seghezzi (is-16-bit query) + Arseny Kapoulkine + John-Mark Allen + Carmelo J Fdez-Aguera + + Bug & warning fixes + Marc LeBlanc David Woo Guillaume George Martins Mozeiko + Christpher Lloyd Jerry Jansson Joseph Thomson Phil Jordan + Dave Moore Roy Eltham Hayaki Saito Nathan Reed + Won Chun Luke Graham Johan Duparc Nick Verigakis + the Horde3D community Thomas Ruf Ronny Chevalier github:rlyeh + Janez Zemva John Bartholomew Michal Cichon github:romigrou + Jonathan Blow Ken Hamada Tero Hanninen github:svdijk + Laurent Gomila Cort Stratton Sergio Gonzalez github:snagar + Aruelien Pocheville Thibault Reuille Cass Everitt github:Zelex + Ryamond Barbiero Paul Du Bois Engin Manap github:grim210 + Aldo Culquicondor Philipp Wiesemann Dale Weiler github:sammyhw + Oriol Ferrer Mesia Josh Tobin Matthew Gregan github:phprus + Julian Raschke Gregory Mullen Baldur Karlsson github:poppolopoppo + Christian Floisand Kevin Schmidt JR Smith github:darealshinji + Blazej Dariusz Roszkowski github:Michaelangel007 +*/ + +#ifndef STBI_INCLUDE_STB_IMAGE_H +#define STBI_INCLUDE_STB_IMAGE_H + +// DOCUMENTATION +// +// Limitations: +// - no 12-bit-per-channel JPEG +// - no JPEGs with arithmetic coding +// - GIF always returns *comp=4 +// +// Basic usage (see HDR discussion below for HDR usage): +// int x,y,n; +// unsigned char *data = stbi_load(filename, &x, &y, &n, 0); +// // ... process data if not NULL ... +// // ... x = width, y = height, n = # 8-bit components per pixel ... +// // ... replace '0' with '1'..'4' to force that many components per pixel +// // ... but 'n' will always be the number that it would have been if you said 0 +// stbi_image_free(data) +// +// Standard parameters: +// int *x -- outputs image width in pixels +// int *y -- outputs image height in pixels +// int *channels_in_file -- outputs # of image components in image file +// int desired_channels -- if non-zero, # of image components requested in result +// +// The return value from an image loader is an 'unsigned char *' which points +// to the pixel data, or NULL on an allocation failure or if the image is +// corrupt or invalid. The pixel data consists of *y scanlines of *x pixels, +// with each pixel consisting of N interleaved 8-bit components; the first +// pixel pointed to is top-left-most in the image. There is no padding between +// image scanlines or between pixels, regardless of format. The number of +// components N is 'desired_channels' if desired_channels is non-zero, or +// *channels_in_file otherwise. If desired_channels is non-zero, +// *channels_in_file has the number of components that _would_ have been +// output otherwise. E.g. if you set desired_channels to 4, you will always +// get RGBA output, but you can check *channels_in_file to see if it's trivially +// opaque because e.g. there were only 3 channels in the source image. +// +// An output image with N components has the following components interleaved +// in this order in each pixel: +// +// N=#comp components +// 1 grey +// 2 grey, alpha +// 3 red, green, blue +// 4 red, green, blue, alpha +// +// If image loading fails for any reason, the return value will be NULL, +// and *x, *y, *channels_in_file will be unchanged. The function +// stbi_failure_reason() can be queried for an extremely brief, end-user +// unfriendly explanation of why the load failed. Define STBI_NO_FAILURE_STRINGS +// to avoid compiling these strings at all, and STBI_FAILURE_USERMSG to get slightly +// more user-friendly ones. +// +// Paletted PNG, BMP, GIF, and PIC images are automatically depalettized. +// +// =========================================================================== +// +// UNICODE: +// +// If compiling for Windows and you wish to use Unicode filenames, compile +// with +// #define STBI_WINDOWS_UTF8 +// and pass utf8-encoded filenames. Call stbi_convert_wchar_to_utf8 to convert +// Windows wchar_t filenames to utf8. +// +// =========================================================================== +// +// Philosophy +// +// stb libraries are designed with the following priorities: +// +// 1. easy to use +// 2. easy to maintain +// 3. good performance +// +// Sometimes I let "good performance" creep up in priority over "easy to maintain", +// and for best performance I may provide less-easy-to-use APIs that give higher +// performance, in addition to the easy-to-use ones. Nevertheless, it's important +// to keep in mind that from the standpoint of you, a client of this library, +// all you care about is #1 and #3, and stb libraries DO NOT emphasize #3 above all. +// +// Some secondary priorities arise directly from the first two, some of which +// provide more explicit reasons why performance can't be emphasized. +// +// - Portable ("ease of use") +// - Small source code footprint ("easy to maintain") +// - No dependencies ("ease of use") +// +// =========================================================================== +// +// I/O callbacks +// +// I/O callbacks allow you to read from arbitrary sources, like packaged +// files or some other source. Data read from callbacks are processed +// through a small internal buffer (currently 128 bytes) to try to reduce +// overhead. +// +// The three functions you must define are "read" (reads some bytes of data), +// "skip" (skips some bytes of data), "eof" (reports if the stream is at the end). +// +// =========================================================================== +// +// SIMD support +// +// The JPEG decoder will try to automatically use SIMD kernels on x86 when +// supported by the compiler. For ARM Neon support, you must explicitly +// request it. +// +// (The old do-it-yourself SIMD API is no longer supported in the current +// code.) +// +// On x86, SSE2 will automatically be used when available based on a run-time +// test; if not, the generic C versions are used as a fall-back. On ARM targets, +// the typical path is to have separate builds for NEON and non-NEON devices +// (at least this is true for iOS and Android). Therefore, the NEON support is +// toggled by a build flag: define STBI_NEON to get NEON loops. +// +// If for some reason you do not want to use any of SIMD code, or if +// you have issues compiling it, you can disable it entirely by +// defining STBI_NO_SIMD. +// +// =========================================================================== +// +// HDR image support (disable by defining STBI_NO_HDR) +// +// stb_image supports loading HDR images in general, and currently the Radiance +// .HDR file format specifically. You can still load any file through the existing +// interface; if you attempt to load an HDR file, it will be automatically remapped +// to LDR, assuming gamma 2.2 and an arbitrary scale factor defaulting to 1; +// both of these constants can be reconfigured through this interface: +// +// stbi_hdr_to_ldr_gamma(2.2f); +// stbi_hdr_to_ldr_scale(1.0f); +// +// (note, do not use _inverse_ constants; stbi_image will invert them +// appropriately). +// +// Additionally, there is a new, parallel interface for loading files as +// (linear) floats to preserve the full dynamic range: +// +// float *data = stbi_loadf(filename, &x, &y, &n, 0); +// +// If you load LDR images through this interface, those images will +// be promoted to floating point values, run through the inverse of +// constants corresponding to the above: +// +// stbi_ldr_to_hdr_scale(1.0f); +// stbi_ldr_to_hdr_gamma(2.2f); +// +// Finally, given a filename (or an open file or memory block--see header +// file for details) containing image data, you can query for the "most +// appropriate" interface to use (that is, whether the image is HDR or +// not), using: +// +// stbi_is_hdr(char *filename); +// +// =========================================================================== +// +// iPhone PNG support: +// +// By default we convert iphone-formatted PNGs back to RGB, even though +// they are internally encoded differently. You can disable this conversion +// by calling stbi_convert_iphone_png_to_rgb(0), in which case +// you will always just get the native iphone "format" through (which +// is BGR stored in RGB). +// +// Call stbi_set_unpremultiply_on_load(1) as well to force a divide per +// pixel to remove any premultiplied alpha *only* if the image file explicitly +// says there's premultiplied data (currently only happens in iPhone images, +// and only if iPhone convert-to-rgb processing is on). +// +// =========================================================================== +// +// ADDITIONAL CONFIGURATION +// +// - You can suppress implementation of any of the decoders to reduce +// your code footprint by #defining one or more of the following +// symbols before creating the implementation. +// +// STBI_NO_JPEG +// STBI_NO_PNG +// STBI_NO_BMP +// STBI_NO_PSD +// STBI_NO_TGA +// STBI_NO_GIF +// STBI_NO_HDR +// STBI_NO_PIC +// STBI_NO_PNM (.ppm and .pgm) +// +// - You can request *only* certain decoders and suppress all other ones +// (this will be more forward-compatible, as addition of new decoders +// doesn't require you to disable them explicitly): +// +// STBI_ONLY_JPEG +// STBI_ONLY_PNG +// STBI_ONLY_BMP +// STBI_ONLY_PSD +// STBI_ONLY_TGA +// STBI_ONLY_GIF +// STBI_ONLY_HDR +// STBI_ONLY_PIC +// STBI_ONLY_PNM (.ppm and .pgm) +// +// - If you use STBI_NO_PNG (or _ONLY_ without PNG), and you still +// want the zlib decoder to be available, #define STBI_SUPPORT_ZLIB +// + + +#ifndef STBI_NO_STDIO +#include +#endif // STBI_NO_STDIO + +#define STBI_VERSION 1 + +enum +{ + STBI_default = 0, // only used for desired_channels + + STBI_grey = 1, + STBI_grey_alpha = 2, + STBI_rgb = 3, + STBI_rgb_alpha = 4 +}; + +#include +typedef unsigned char stbi_uc; +typedef unsigned short stbi_us; + +#ifdef __cplusplus +extern "C" { +#endif + +#ifdef STB_IMAGE_STATIC +#define STBIDEF static +#else +#define STBIDEF extern +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// PRIMARY API - works on images of any type +// + +// +// load image by filename, open file, or memory buffer +// + +typedef struct +{ + int (*read) (void *user,char *data,int size); // fill 'data' with 'size' bytes. return number of bytes actually read + void (*skip) (void *user,int n); // skip the next 'n' bytes, or 'unget' the last -n bytes if negative + int (*eof) (void *user); // returns nonzero if we are at end of file/data +} stbi_io_callbacks; + +//////////////////////////////////// +// +// 8-bits-per-channel interface +// + +STBIDEF stbi_uc *stbi_load_from_memory (stbi_uc const *buffer, int len , int *x, int *y, int *channels_in_file, int desired_channels); +STBIDEF stbi_uc *stbi_load_from_callbacks(stbi_io_callbacks const *clbk , void *user, int *x, int *y, int *channels_in_file, int desired_channels); + +#ifndef STBI_NO_STDIO +STBIDEF stbi_uc *stbi_load (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels); +STBIDEF stbi_uc *stbi_load_from_file (FILE *f, int *x, int *y, int *channels_in_file, int desired_channels); +// for stbi_load_from_file, file pointer is left pointing immediately after image +#endif + +#ifndef STBI_NO_GIF +STBIDEF stbi_uc *stbi_load_gif_from_memory(stbi_uc const *buffer, int len, int **delays, int *x, int *y, int *z, int *comp, int req_comp); +#endif + +#ifdef STBI_WINDOWS_UTF8 +STBIDEF int stbi_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input); +#endif + +//////////////////////////////////// +// +// 16-bits-per-channel interface +// + +STBIDEF stbi_us *stbi_load_16_from_memory (stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels); +STBIDEF stbi_us *stbi_load_16_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels); + +#ifndef STBI_NO_STDIO +STBIDEF stbi_us *stbi_load_16 (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels); +STBIDEF stbi_us *stbi_load_from_file_16(FILE *f, int *x, int *y, int *channels_in_file, int desired_channels); +#endif + +//////////////////////////////////// +// +// float-per-channel interface +// +#ifndef STBI_NO_LINEAR + STBIDEF float *stbi_loadf_from_memory (stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels); + STBIDEF float *stbi_loadf_from_callbacks (stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels); + + #ifndef STBI_NO_STDIO + STBIDEF float *stbi_loadf (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels); + STBIDEF float *stbi_loadf_from_file (FILE *f, int *x, int *y, int *channels_in_file, int desired_channels); + #endif +#endif + +#ifndef STBI_NO_HDR + STBIDEF void stbi_hdr_to_ldr_gamma(float gamma); + STBIDEF void stbi_hdr_to_ldr_scale(float scale); +#endif // STBI_NO_HDR + +#ifndef STBI_NO_LINEAR + STBIDEF void stbi_ldr_to_hdr_gamma(float gamma); + STBIDEF void stbi_ldr_to_hdr_scale(float scale); +#endif // STBI_NO_LINEAR + +// stbi_is_hdr is always defined, but always returns false if STBI_NO_HDR +STBIDEF int stbi_is_hdr_from_callbacks(stbi_io_callbacks const *clbk, void *user); +STBIDEF int stbi_is_hdr_from_memory(stbi_uc const *buffer, int len); +#ifndef STBI_NO_STDIO +STBIDEF int stbi_is_hdr (char const *filename); +STBIDEF int stbi_is_hdr_from_file(FILE *f); +#endif // STBI_NO_STDIO + + +// get a VERY brief reason for failure +// NOT THREADSAFE +STBIDEF const char *stbi_failure_reason (void); + +// free the loaded image -- this is just free() +STBIDEF void stbi_image_free (void *retval_from_stbi_load); + +// get image dimensions & components without fully decoding +STBIDEF int stbi_info_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp); +STBIDEF int stbi_info_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp); +STBIDEF int stbi_is_16_bit_from_memory(stbi_uc const *buffer, int len); +STBIDEF int stbi_is_16_bit_from_callbacks(stbi_io_callbacks const *clbk, void *user); + +#ifndef STBI_NO_STDIO +STBIDEF int stbi_info (char const *filename, int *x, int *y, int *comp); +STBIDEF int stbi_info_from_file (FILE *f, int *x, int *y, int *comp); +STBIDEF int stbi_is_16_bit (char const *filename); +STBIDEF int stbi_is_16_bit_from_file(FILE *f); +#endif + + + +// for image formats that explicitly notate that they have premultiplied alpha, +// we just return the colors as stored in the file. set this flag to force +// unpremultiplication. results are undefined if the unpremultiply overflow. +STBIDEF void stbi_set_unpremultiply_on_load(int flag_true_if_should_unpremultiply); + +// indicate whether we should process iphone images back to canonical format, +// or just pass them through "as-is" +STBIDEF void stbi_convert_iphone_png_to_rgb(int flag_true_if_should_convert); + +// flip the image vertically, so the first pixel in the output array is the bottom left +STBIDEF void stbi_set_flip_vertically_on_load(int flag_true_if_should_flip); + +// ZLIB client - used by PNG, available for other purposes + +STBIDEF char *stbi_zlib_decode_malloc_guesssize(const char *buffer, int len, int initial_size, int *outlen); +STBIDEF char *stbi_zlib_decode_malloc_guesssize_headerflag(const char *buffer, int len, int initial_size, int *outlen, int parse_header); +STBIDEF char *stbi_zlib_decode_malloc(const char *buffer, int len, int *outlen); +STBIDEF int stbi_zlib_decode_buffer(char *obuffer, int olen, const char *ibuffer, int ilen); + +STBIDEF char *stbi_zlib_decode_noheader_malloc(const char *buffer, int len, int *outlen); +STBIDEF int stbi_zlib_decode_noheader_buffer(char *obuffer, int olen, const char *ibuffer, int ilen); + + +#ifdef __cplusplus +} +#endif + +// +// +//// end header file ///////////////////////////////////////////////////// +#endif // STBI_INCLUDE_STB_IMAGE_H + +#ifdef STB_IMAGE_IMPLEMENTATION + +#if defined(STBI_ONLY_JPEG) || defined(STBI_ONLY_PNG) || defined(STBI_ONLY_BMP) \ + || defined(STBI_ONLY_TGA) || defined(STBI_ONLY_GIF) || defined(STBI_ONLY_PSD) \ + || defined(STBI_ONLY_HDR) || defined(STBI_ONLY_PIC) || defined(STBI_ONLY_PNM) \ + || defined(STBI_ONLY_ZLIB) + #ifndef STBI_ONLY_JPEG + #define STBI_NO_JPEG + #endif + #ifndef STBI_ONLY_PNG + #define STBI_NO_PNG + #endif + #ifndef STBI_ONLY_BMP + #define STBI_NO_BMP + #endif + #ifndef STBI_ONLY_PSD + #define STBI_NO_PSD + #endif + #ifndef STBI_ONLY_TGA + #define STBI_NO_TGA + #endif + #ifndef STBI_ONLY_GIF + #define STBI_NO_GIF + #endif + #ifndef STBI_ONLY_HDR + #define STBI_NO_HDR + #endif + #ifndef STBI_ONLY_PIC + #define STBI_NO_PIC + #endif + #ifndef STBI_ONLY_PNM + #define STBI_NO_PNM + #endif +#endif + +#if defined(STBI_NO_PNG) && !defined(STBI_SUPPORT_ZLIB) && !defined(STBI_NO_ZLIB) +#define STBI_NO_ZLIB +#endif + + +#include +#include // ptrdiff_t on osx +#include +#include +#include + +#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR) +#include // ldexp, pow +#endif + +#ifndef STBI_NO_STDIO +#include +#endif + +#ifndef STBI_ASSERT +#include +#define STBI_ASSERT(x) assert(x) +#endif + +#ifdef __cplusplus +#define STBI_EXTERN extern "C" +#else +#define STBI_EXTERN extern +#endif + + +#ifndef _MSC_VER + #ifdef __cplusplus + #define stbi_inline inline + #else + #define stbi_inline + #endif +#else + #define stbi_inline __forceinline +#endif + + +#ifdef _MSC_VER +typedef unsigned short stbi__uint16; +typedef signed short stbi__int16; +typedef unsigned int stbi__uint32; +typedef signed int stbi__int32; +#else +#include +typedef uint16_t stbi__uint16; +typedef int16_t stbi__int16; +typedef uint32_t stbi__uint32; +typedef int32_t stbi__int32; +#endif + +// should produce compiler error if size is wrong +typedef unsigned char validate_uint32[sizeof(stbi__uint32)==4 ? 1 : -1]; + +#ifdef _MSC_VER +#define STBI_NOTUSED(v) (void)(v) +#else +#define STBI_NOTUSED(v) (void)sizeof(v) +#endif + +#ifdef _MSC_VER +#define STBI_HAS_LROTL +#endif + +#ifdef STBI_HAS_LROTL + #define stbi_lrot(x,y) _lrotl(x,y) +#else + #define stbi_lrot(x,y) (((x) << (y)) | ((x) >> (32 - (y)))) +#endif + +#if defined(STBI_MALLOC) && defined(STBI_FREE) && (defined(STBI_REALLOC) || defined(STBI_REALLOC_SIZED)) +// ok +#elif !defined(STBI_MALLOC) && !defined(STBI_FREE) && !defined(STBI_REALLOC) && !defined(STBI_REALLOC_SIZED) +// ok +#else +#error "Must define all or none of STBI_MALLOC, STBI_FREE, and STBI_REALLOC (or STBI_REALLOC_SIZED)." +#endif + +#ifndef STBI_MALLOC +#define STBI_MALLOC(sz) malloc(sz) +#define STBI_REALLOC(p,newsz) realloc(p,newsz) +#define STBI_FREE(p) free(p) +#endif + +#ifndef STBI_REALLOC_SIZED +#define STBI_REALLOC_SIZED(p,oldsz,newsz) STBI_REALLOC(p,newsz) +#endif + +// x86/x64 detection +#if defined(__x86_64__) || defined(_M_X64) +#define STBI__X64_TARGET +#elif defined(__i386) || defined(_M_IX86) +#define STBI__X86_TARGET +#endif + +#if defined(__GNUC__) && defined(STBI__X86_TARGET) && !defined(__SSE2__) && !defined(STBI_NO_SIMD) +// gcc doesn't support sse2 intrinsics unless you compile with -msse2, +// which in turn means it gets to use SSE2 everywhere. This is unfortunate, +// but previous attempts to provide the SSE2 functions with runtime +// detection caused numerous issues. The way architecture extensions are +// exposed in GCC/Clang is, sadly, not really suited for one-file libs. +// New behavior: if compiled with -msse2, we use SSE2 without any +// detection; if not, we don't use it at all. +#define STBI_NO_SIMD +#endif + +#if defined(__MINGW32__) && defined(STBI__X86_TARGET) && !defined(STBI_MINGW_ENABLE_SSE2) && !defined(STBI_NO_SIMD) +// Note that __MINGW32__ doesn't actually mean 32-bit, so we have to avoid STBI__X64_TARGET +// +// 32-bit MinGW wants ESP to be 16-byte aligned, but this is not in the +// Windows ABI and VC++ as well as Windows DLLs don't maintain that invariant. +// As a result, enabling SSE2 on 32-bit MinGW is dangerous when not +// simultaneously enabling "-mstackrealign". +// +// See https://github.com/nothings/stb/issues/81 for more information. +// +// So default to no SSE2 on 32-bit MinGW. If you've read this far and added +// -mstackrealign to your build settings, feel free to #define STBI_MINGW_ENABLE_SSE2. +#define STBI_NO_SIMD +#endif + +#if !defined(STBI_NO_SIMD) && (defined(STBI__X86_TARGET) || defined(STBI__X64_TARGET)) +#define STBI_SSE2 +#include + +#ifdef _MSC_VER + +#if _MSC_VER >= 1400 // not VC6 +#include // __cpuid +static int stbi__cpuid3(void) +{ + int info[4]; + __cpuid(info,1); + return info[3]; +} +#else +static int stbi__cpuid3(void) +{ + int res; + __asm { + mov eax,1 + cpuid + mov res,edx + } + return res; +} +#endif + +#define STBI_SIMD_ALIGN(type, name) __declspec(align(16)) type name + +#if !defined(STBI_NO_JPEG) && defined(STBI_SSE2) +static int stbi__sse2_available(void) +{ + int info3 = stbi__cpuid3(); + return ((info3 >> 26) & 1) != 0; +} +#endif + +#else // assume GCC-style if not VC++ +#define STBI_SIMD_ALIGN(type, name) type name __attribute__((aligned(16))) + +#if !defined(STBI_NO_JPEG) && defined(STBI_SSE2) +static int stbi__sse2_available(void) +{ + // If we're even attempting to compile this on GCC/Clang, that means + // -msse2 is on, which means the compiler is allowed to use SSE2 + // instructions at will, and so are we. + return 1; +} +#endif + +#endif +#endif + +// ARM NEON +#if defined(STBI_NO_SIMD) && defined(STBI_NEON) +#undef STBI_NEON +#endif + +#ifdef STBI_NEON +#include +// assume GCC or Clang on ARM targets +#define STBI_SIMD_ALIGN(type, name) type name __attribute__((aligned(16))) +#endif + +#ifndef STBI_SIMD_ALIGN +#define STBI_SIMD_ALIGN(type, name) type name +#endif + +/////////////////////////////////////////////// +// +// stbi__context struct and start_xxx functions + +// stbi__context structure is our basic context used by all images, so it +// contains all the IO context, plus some basic image information +typedef struct +{ + stbi__uint32 img_x, img_y; + int img_n, img_out_n; + + stbi_io_callbacks io; + void *io_user_data; + + int read_from_callbacks; + int buflen; + stbi_uc buffer_start[128]; + + stbi_uc *img_buffer, *img_buffer_end; + stbi_uc *img_buffer_original, *img_buffer_original_end; +} stbi__context; + + +static void stbi__refill_buffer(stbi__context *s); + +// initialize a memory-decode context +static void stbi__start_mem(stbi__context *s, stbi_uc const *buffer, int len) +{ + s->io.read = NULL; + s->read_from_callbacks = 0; + s->img_buffer = s->img_buffer_original = (stbi_uc *) buffer; + s->img_buffer_end = s->img_buffer_original_end = (stbi_uc *) buffer+len; +} + +// initialize a callback-based context +static void stbi__start_callbacks(stbi__context *s, stbi_io_callbacks *c, void *user) +{ + s->io = *c; + s->io_user_data = user; + s->buflen = sizeof(s->buffer_start); + s->read_from_callbacks = 1; + s->img_buffer_original = s->buffer_start; + stbi__refill_buffer(s); + s->img_buffer_original_end = s->img_buffer_end; +} + +#ifndef STBI_NO_STDIO + +static int stbi__stdio_read(void *user, char *data, int size) +{ + return (int) fread(data,1,size,(FILE*) user); +} + +static void stbi__stdio_skip(void *user, int n) +{ + fseek((FILE*) user, n, SEEK_CUR); +} + +static int stbi__stdio_eof(void *user) +{ + return feof((FILE*) user); +} + +static stbi_io_callbacks stbi__stdio_callbacks = +{ + stbi__stdio_read, + stbi__stdio_skip, + stbi__stdio_eof, +}; + +static void stbi__start_file(stbi__context *s, FILE *f) +{ + stbi__start_callbacks(s, &stbi__stdio_callbacks, (void *) f); +} + +//static void stop_file(stbi__context *s) { } + +#endif // !STBI_NO_STDIO + +static void stbi__rewind(stbi__context *s) +{ + // conceptually rewind SHOULD rewind to the beginning of the stream, + // but we just rewind to the beginning of the initial buffer, because + // we only use it after doing 'test', which only ever looks at at most 92 bytes + s->img_buffer = s->img_buffer_original; + s->img_buffer_end = s->img_buffer_original_end; +} + +enum +{ + STBI_ORDER_RGB, + STBI_ORDER_BGR +}; + +typedef struct +{ + int bits_per_channel; + int num_channels; + int channel_order; +} stbi__result_info; + +#ifndef STBI_NO_JPEG +static int stbi__jpeg_test(stbi__context *s); +static void *stbi__jpeg_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__jpeg_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_PNG +static int stbi__png_test(stbi__context *s); +static void *stbi__png_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__png_info(stbi__context *s, int *x, int *y, int *comp); +static int stbi__png_is16(stbi__context *s); +#endif + +#ifndef STBI_NO_BMP +static int stbi__bmp_test(stbi__context *s); +static void *stbi__bmp_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__bmp_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_TGA +static int stbi__tga_test(stbi__context *s); +static void *stbi__tga_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__tga_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_PSD +static int stbi__psd_test(stbi__context *s); +static void *stbi__psd_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc); +static int stbi__psd_info(stbi__context *s, int *x, int *y, int *comp); +static int stbi__psd_is16(stbi__context *s); +#endif + +#ifndef STBI_NO_HDR +static int stbi__hdr_test(stbi__context *s); +static float *stbi__hdr_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__hdr_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_PIC +static int stbi__pic_test(stbi__context *s); +static void *stbi__pic_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__pic_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_GIF +static int stbi__gif_test(stbi__context *s); +static void *stbi__gif_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static void *stbi__load_gif_main(stbi__context *s, int **delays, int *x, int *y, int *z, int *comp, int req_comp); +static int stbi__gif_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +#ifndef STBI_NO_PNM +static int stbi__pnm_test(stbi__context *s); +static void *stbi__pnm_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); +static int stbi__pnm_info(stbi__context *s, int *x, int *y, int *comp); +#endif + +// this is not threadsafe +static const char *stbi__g_failure_reason; + +STBIDEF const char *stbi_failure_reason(void) +{ + return stbi__g_failure_reason; +} + +static int stbi__err(const char *str) +{ + stbi__g_failure_reason = str; + return 0; +} + +static void *stbi__malloc(size_t size) +{ + return STBI_MALLOC(size); +} + +// stb_image uses ints pervasively, including for offset calculations. +// therefore the largest decoded image size we can support with the +// current code, even on 64-bit targets, is INT_MAX. this is not a +// significant limitation for the intended use case. +// +// we do, however, need to make sure our size calculations don't +// overflow. hence a few helper functions for size calculations that +// multiply integers together, making sure that they're non-negative +// and no overflow occurs. + +// return 1 if the sum is valid, 0 on overflow. +// negative terms are considered invalid. +static int stbi__addsizes_valid(int a, int b) +{ + if (b < 0) return 0; + // now 0 <= b <= INT_MAX, hence also + // 0 <= INT_MAX - b <= INTMAX. + // And "a + b <= INT_MAX" (which might overflow) is the + // same as a <= INT_MAX - b (no overflow) + return a <= INT_MAX - b; +} + +// returns 1 if the product is valid, 0 on overflow. +// negative factors are considered invalid. +static int stbi__mul2sizes_valid(int a, int b) +{ + if (a < 0 || b < 0) return 0; + if (b == 0) return 1; // mul-by-0 is always safe + // portable way to check for no overflows in a*b + return a <= INT_MAX/b; +} + +// returns 1 if "a*b + add" has no negative terms/factors and doesn't overflow +static int stbi__mad2sizes_valid(int a, int b, int add) +{ + return stbi__mul2sizes_valid(a, b) && stbi__addsizes_valid(a*b, add); +} + +// returns 1 if "a*b*c + add" has no negative terms/factors and doesn't overflow +static int stbi__mad3sizes_valid(int a, int b, int c, int add) +{ + return stbi__mul2sizes_valid(a, b) && stbi__mul2sizes_valid(a*b, c) && + stbi__addsizes_valid(a*b*c, add); +} + +// returns 1 if "a*b*c*d + add" has no negative terms/factors and doesn't overflow +#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR) +static int stbi__mad4sizes_valid(int a, int b, int c, int d, int add) +{ + return stbi__mul2sizes_valid(a, b) && stbi__mul2sizes_valid(a*b, c) && + stbi__mul2sizes_valid(a*b*c, d) && stbi__addsizes_valid(a*b*c*d, add); +} +#endif + +// mallocs with size overflow checking +static void *stbi__malloc_mad2(int a, int b, int add) +{ + if (!stbi__mad2sizes_valid(a, b, add)) return NULL; + return stbi__malloc(a*b + add); +} + +static void *stbi__malloc_mad3(int a, int b, int c, int add) +{ + if (!stbi__mad3sizes_valid(a, b, c, add)) return NULL; + return stbi__malloc(a*b*c + add); +} + +#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR) +static void *stbi__malloc_mad4(int a, int b, int c, int d, int add) +{ + if (!stbi__mad4sizes_valid(a, b, c, d, add)) return NULL; + return stbi__malloc(a*b*c*d + add); +} +#endif + +// stbi__err - error +// stbi__errpf - error returning pointer to float +// stbi__errpuc - error returning pointer to unsigned char + +#ifdef STBI_NO_FAILURE_STRINGS + #define stbi__err(x,y) 0 +#elif defined(STBI_FAILURE_USERMSG) + #define stbi__err(x,y) stbi__err(y) +#else + #define stbi__err(x,y) stbi__err(x) +#endif + +#define stbi__errpf(x,y) ((float *)(size_t) (stbi__err(x,y)?NULL:NULL)) +#define stbi__errpuc(x,y) ((unsigned char *)(size_t) (stbi__err(x,y)?NULL:NULL)) + +STBIDEF void stbi_image_free(void *retval_from_stbi_load) +{ + STBI_FREE(retval_from_stbi_load); +} + +#ifndef STBI_NO_LINEAR +static float *stbi__ldr_to_hdr(stbi_uc *data, int x, int y, int comp); +#endif + +#ifndef STBI_NO_HDR +static stbi_uc *stbi__hdr_to_ldr(float *data, int x, int y, int comp); +#endif + +static int stbi__vertically_flip_on_load = 0; + +STBIDEF void stbi_set_flip_vertically_on_load(int flag_true_if_should_flip) +{ + stbi__vertically_flip_on_load = flag_true_if_should_flip; +} + +static void *stbi__load_main(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc) +{ + memset(ri, 0, sizeof(*ri)); // make sure it's initialized if we add new fields + ri->bits_per_channel = 8; // default is 8 so most paths don't have to be changed + ri->channel_order = STBI_ORDER_RGB; // all current input & output are this, but this is here so we can add BGR order + ri->num_channels = 0; + + #ifndef STBI_NO_JPEG + if (stbi__jpeg_test(s)) return stbi__jpeg_load(s,x,y,comp,req_comp, ri); + #endif + #ifndef STBI_NO_PNG + if (stbi__png_test(s)) return stbi__png_load(s,x,y,comp,req_comp, ri); + #endif + #ifndef STBI_NO_BMP + if (stbi__bmp_test(s)) return stbi__bmp_load(s,x,y,comp,req_comp, ri); + #endif + #ifndef STBI_NO_GIF + if (stbi__gif_test(s)) return stbi__gif_load(s,x,y,comp,req_comp, ri); + #endif + #ifndef STBI_NO_PSD + if (stbi__psd_test(s)) return stbi__psd_load(s,x,y,comp,req_comp, ri, bpc); + #endif + #ifndef STBI_NO_PIC + if (stbi__pic_test(s)) return stbi__pic_load(s,x,y,comp,req_comp, ri); + #endif + #ifndef STBI_NO_PNM + if (stbi__pnm_test(s)) return stbi__pnm_load(s,x,y,comp,req_comp, ri); + #endif + + #ifndef STBI_NO_HDR + if (stbi__hdr_test(s)) { + float *hdr = stbi__hdr_load(s, x,y,comp,req_comp, ri); + return stbi__hdr_to_ldr(hdr, *x, *y, req_comp ? req_comp : *comp); + } + #endif + + #ifndef STBI_NO_TGA + // test tga last because it's a crappy test! + if (stbi__tga_test(s)) + return stbi__tga_load(s,x,y,comp,req_comp, ri); + #endif + + return stbi__errpuc("unknown image type", "Image not of any known type, or corrupt"); +} + +static stbi_uc *stbi__convert_16_to_8(stbi__uint16 *orig, int w, int h, int channels) +{ + int i; + int img_len = w * h * channels; + stbi_uc *reduced; + + reduced = (stbi_uc *) stbi__malloc(img_len); + if (reduced == NULL) return stbi__errpuc("outofmem", "Out of memory"); + + for (i = 0; i < img_len; ++i) + reduced[i] = (stbi_uc)((orig[i] >> 8) & 0xFF); // top half of each byte is sufficient approx of 16->8 bit scaling + + STBI_FREE(orig); + return reduced; +} + +static stbi__uint16 *stbi__convert_8_to_16(stbi_uc *orig, int w, int h, int channels) +{ + int i; + int img_len = w * h * channels; + stbi__uint16 *enlarged; + + enlarged = (stbi__uint16 *) stbi__malloc(img_len*2); + if (enlarged == NULL) return (stbi__uint16 *) stbi__errpuc("outofmem", "Out of memory"); + + for (i = 0; i < img_len; ++i) + enlarged[i] = (stbi__uint16)((orig[i] << 8) + orig[i]); // replicate to high and low byte, maps 0->0, 255->0xffff + + STBI_FREE(orig); + return enlarged; +} + +static void stbi__vertical_flip(void *image, int w, int h, int bytes_per_pixel) +{ + int row; + size_t bytes_per_row = (size_t)w * bytes_per_pixel; + stbi_uc temp[2048]; + stbi_uc *bytes = (stbi_uc *)image; + + for (row = 0; row < (h>>1); row++) { + stbi_uc *row0 = bytes + row*bytes_per_row; + stbi_uc *row1 = bytes + (h - row - 1)*bytes_per_row; + // swap row0 with row1 + size_t bytes_left = bytes_per_row; + while (bytes_left) { + size_t bytes_copy = (bytes_left < sizeof(temp)) ? bytes_left : sizeof(temp); + memcpy(temp, row0, bytes_copy); + memcpy(row0, row1, bytes_copy); + memcpy(row1, temp, bytes_copy); + row0 += bytes_copy; + row1 += bytes_copy; + bytes_left -= bytes_copy; + } + } +} + +#ifndef STBI_NO_GIF +static void stbi__vertical_flip_slices(void *image, int w, int h, int z, int bytes_per_pixel) +{ + int slice; + int slice_size = w * h * bytes_per_pixel; + + stbi_uc *bytes = (stbi_uc *)image; + for (slice = 0; slice < z; ++slice) { + stbi__vertical_flip(bytes, w, h, bytes_per_pixel); + bytes += slice_size; + } +} +#endif + +static unsigned char *stbi__load_and_postprocess_8bit(stbi__context *s, int *x, int *y, int *comp, int req_comp) +{ + stbi__result_info ri; + void *result = stbi__load_main(s, x, y, comp, req_comp, &ri, 8); + + if (result == NULL) + return NULL; + + if (ri.bits_per_channel != 8) { + STBI_ASSERT(ri.bits_per_channel == 16); + result = stbi__convert_16_to_8((stbi__uint16 *) result, *x, *y, req_comp == 0 ? *comp : req_comp); + ri.bits_per_channel = 8; + } + + // @TODO: move stbi__convert_format to here + + if (stbi__vertically_flip_on_load) { + int channels = req_comp ? req_comp : *comp; + stbi__vertical_flip(result, *x, *y, channels * sizeof(stbi_uc)); + } + + return (unsigned char *) result; +} + +static stbi__uint16 *stbi__load_and_postprocess_16bit(stbi__context *s, int *x, int *y, int *comp, int req_comp) +{ + stbi__result_info ri; + void *result = stbi__load_main(s, x, y, comp, req_comp, &ri, 16); + + if (result == NULL) + return NULL; + + if (ri.bits_per_channel != 16) { + STBI_ASSERT(ri.bits_per_channel == 8); + result = stbi__convert_8_to_16((stbi_uc *) result, *x, *y, req_comp == 0 ? *comp : req_comp); + ri.bits_per_channel = 16; + } + + // @TODO: move stbi__convert_format16 to here + // @TODO: special case RGB-to-Y (and RGBA-to-YA) for 8-bit-to-16-bit case to keep more precision + + if (stbi__vertically_flip_on_load) { + int channels = req_comp ? req_comp : *comp; + stbi__vertical_flip(result, *x, *y, channels * sizeof(stbi__uint16)); + } + + return (stbi__uint16 *) result; +} + +#if !defined(STBI_NO_HDR) && !defined(STBI_NO_LINEAR) +static void stbi__float_postprocess(float *result, int *x, int *y, int *comp, int req_comp) +{ + if (stbi__vertically_flip_on_load && result != NULL) { + int channels = req_comp ? req_comp : *comp; + stbi__vertical_flip(result, *x, *y, channels * sizeof(float)); + } +} +#endif + +#ifndef STBI_NO_STDIO + +#if defined(_MSC_VER) && defined(STBI_WINDOWS_UTF8) +STBI_EXTERN __declspec(dllimport) int __stdcall MultiByteToWideChar(unsigned int cp, unsigned long flags, const char *str, int cbmb, wchar_t *widestr, int cchwide); +STBI_EXTERN __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int cp, unsigned long flags, const wchar_t *widestr, int cchwide, char *str, int cbmb, const char *defchar, int *used_default); +#endif + +#if defined(_MSC_VER) && defined(STBI_WINDOWS_UTF8) +STBIDEF int stbi_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input) +{ + return WideCharToMultiByte(65001 /* UTF8 */, 0, input, -1, buffer, bufferlen, NULL, NULL); +} +#endif + +static FILE *stbi__fopen(char const *filename, char const *mode) +{ + FILE *f; +#if defined(_MSC_VER) && defined(STBI_WINDOWS_UTF8) + wchar_t wMode[64]; + wchar_t wFilename[1024]; + if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, filename, -1, wFilename, sizeof(wFilename))) + return 0; + + if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, mode, -1, wMode, sizeof(wMode))) + return 0; + +#if _MSC_VER >= 1400 + if (0 != _wfopen_s(&f, wFilename, wMode)) + f = 0; +#else + f = _wfopen(wFilename, wMode); +#endif + +#elif defined(_MSC_VER) && _MSC_VER >= 1400 + if (0 != fopen_s(&f, filename, mode)) + f=0; +#else + f = fopen(filename, mode); +#endif + return f; +} + + +STBIDEF stbi_uc *stbi_load(char const *filename, int *x, int *y, int *comp, int req_comp) +{ + FILE *f = stbi__fopen(filename, "rb"); + unsigned char *result; + if (!f) return stbi__errpuc("can't fopen", "Unable to open file"); + result = stbi_load_from_file(f,x,y,comp,req_comp); + fclose(f); + return result; +} + +STBIDEF stbi_uc *stbi_load_from_file(FILE *f, int *x, int *y, int *comp, int req_comp) +{ + unsigned char *result; + stbi__context s; + stbi__start_file(&s,f); + result = stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp); + if (result) { + // need to 'unget' all the characters in the IO buffer + fseek(f, - (int) (s.img_buffer_end - s.img_buffer), SEEK_CUR); + } + return result; +} + +STBIDEF stbi__uint16 *stbi_load_from_file_16(FILE *f, int *x, int *y, int *comp, int req_comp) +{ + stbi__uint16 *result; + stbi__context s; + stbi__start_file(&s,f); + result = stbi__load_and_postprocess_16bit(&s,x,y,comp,req_comp); + if (result) { + // need to 'unget' all the characters in the IO buffer + fseek(f, - (int) (s.img_buffer_end - s.img_buffer), SEEK_CUR); + } + return result; +} + +STBIDEF stbi_us *stbi_load_16(char const *filename, int *x, int *y, int *comp, int req_comp) +{ + FILE *f = stbi__fopen(filename, "rb"); + stbi__uint16 *result; + if (!f) return (stbi_us *) stbi__errpuc("can't fopen", "Unable to open file"); + result = stbi_load_from_file_16(f,x,y,comp,req_comp); + fclose(f); + return result; +} + + +#endif //!STBI_NO_STDIO + +STBIDEF stbi_us *stbi_load_16_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__load_and_postprocess_16bit(&s,x,y,channels_in_file,desired_channels); +} + +STBIDEF stbi_us *stbi_load_16_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *)clbk, user); + return stbi__load_and_postprocess_16bit(&s,x,y,channels_in_file,desired_channels); +} + +STBIDEF stbi_uc *stbi_load_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp); +} + +STBIDEF stbi_uc *stbi_load_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user); + return stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp); +} + +#ifndef STBI_NO_GIF +STBIDEF stbi_uc *stbi_load_gif_from_memory(stbi_uc const *buffer, int len, int **delays, int *x, int *y, int *z, int *comp, int req_comp) +{ + unsigned char *result; + stbi__context s; + stbi__start_mem(&s,buffer,len); + + result = (unsigned char*) stbi__load_gif_main(&s, delays, x, y, z, comp, req_comp); + if (stbi__vertically_flip_on_load) { + stbi__vertical_flip_slices( result, *x, *y, *z, *comp ); + } + + return result; +} +#endif + +#ifndef STBI_NO_LINEAR +static float *stbi__loadf_main(stbi__context *s, int *x, int *y, int *comp, int req_comp) +{ + unsigned char *data; + #ifndef STBI_NO_HDR + if (stbi__hdr_test(s)) { + stbi__result_info ri; + float *hdr_data = stbi__hdr_load(s,x,y,comp,req_comp, &ri); + if (hdr_data) + stbi__float_postprocess(hdr_data,x,y,comp,req_comp); + return hdr_data; + } + #endif + data = stbi__load_and_postprocess_8bit(s, x, y, comp, req_comp); + if (data) + return stbi__ldr_to_hdr(data, *x, *y, req_comp ? req_comp : *comp); + return stbi__errpf("unknown image type", "Image not of any known type, or corrupt"); +} + +STBIDEF float *stbi_loadf_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__loadf_main(&s,x,y,comp,req_comp); +} + +STBIDEF float *stbi_loadf_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user); + return stbi__loadf_main(&s,x,y,comp,req_comp); +} + +#ifndef STBI_NO_STDIO +STBIDEF float *stbi_loadf(char const *filename, int *x, int *y, int *comp, int req_comp) +{ + float *result; + FILE *f = stbi__fopen(filename, "rb"); + if (!f) return stbi__errpf("can't fopen", "Unable to open file"); + result = stbi_loadf_from_file(f,x,y,comp,req_comp); + fclose(f); + return result; +} + +STBIDEF float *stbi_loadf_from_file(FILE *f, int *x, int *y, int *comp, int req_comp) +{ + stbi__context s; + stbi__start_file(&s,f); + return stbi__loadf_main(&s,x,y,comp,req_comp); +} +#endif // !STBI_NO_STDIO + +#endif // !STBI_NO_LINEAR + +// these is-hdr-or-not is defined independent of whether STBI_NO_LINEAR is +// defined, for API simplicity; if STBI_NO_LINEAR is defined, it always +// reports false! + +STBIDEF int stbi_is_hdr_from_memory(stbi_uc const *buffer, int len) +{ + #ifndef STBI_NO_HDR + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__hdr_test(&s); + #else + STBI_NOTUSED(buffer); + STBI_NOTUSED(len); + return 0; + #endif +} + +#ifndef STBI_NO_STDIO +STBIDEF int stbi_is_hdr (char const *filename) +{ + FILE *f = stbi__fopen(filename, "rb"); + int result=0; + if (f) { + result = stbi_is_hdr_from_file(f); + fclose(f); + } + return result; +} + +STBIDEF int stbi_is_hdr_from_file(FILE *f) +{ + #ifndef STBI_NO_HDR + long pos = ftell(f); + int res; + stbi__context s; + stbi__start_file(&s,f); + res = stbi__hdr_test(&s); + fseek(f, pos, SEEK_SET); + return res; + #else + STBI_NOTUSED(f); + return 0; + #endif +} +#endif // !STBI_NO_STDIO + +STBIDEF int stbi_is_hdr_from_callbacks(stbi_io_callbacks const *clbk, void *user) +{ + #ifndef STBI_NO_HDR + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user); + return stbi__hdr_test(&s); + #else + STBI_NOTUSED(clbk); + STBI_NOTUSED(user); + return 0; + #endif +} + +#ifndef STBI_NO_LINEAR +static float stbi__l2h_gamma=2.2f, stbi__l2h_scale=1.0f; + +STBIDEF void stbi_ldr_to_hdr_gamma(float gamma) { stbi__l2h_gamma = gamma; } +STBIDEF void stbi_ldr_to_hdr_scale(float scale) { stbi__l2h_scale = scale; } +#endif + +static float stbi__h2l_gamma_i=1.0f/2.2f, stbi__h2l_scale_i=1.0f; + +STBIDEF void stbi_hdr_to_ldr_gamma(float gamma) { stbi__h2l_gamma_i = 1/gamma; } +STBIDEF void stbi_hdr_to_ldr_scale(float scale) { stbi__h2l_scale_i = 1/scale; } + + +////////////////////////////////////////////////////////////////////////////// +// +// Common code used by all image loaders +// + +enum +{ + STBI__SCAN_load=0, + STBI__SCAN_type, + STBI__SCAN_header +}; + +static void stbi__refill_buffer(stbi__context *s) +{ + int n = (s->io.read)(s->io_user_data,(char*)s->buffer_start,s->buflen); + if (n == 0) { + // at end of file, treat same as if from memory, but need to handle case + // where s->img_buffer isn't pointing to safe memory, e.g. 0-byte file + s->read_from_callbacks = 0; + s->img_buffer = s->buffer_start; + s->img_buffer_end = s->buffer_start+1; + *s->img_buffer = 0; + } else { + s->img_buffer = s->buffer_start; + s->img_buffer_end = s->buffer_start + n; + } +} + +stbi_inline static stbi_uc stbi__get8(stbi__context *s) +{ + if (s->img_buffer < s->img_buffer_end) + return *s->img_buffer++; + if (s->read_from_callbacks) { + stbi__refill_buffer(s); + return *s->img_buffer++; + } + return 0; +} + +stbi_inline static int stbi__at_eof(stbi__context *s) +{ + if (s->io.read) { + if (!(s->io.eof)(s->io_user_data)) return 0; + // if feof() is true, check if buffer = end + // special case: we've only got the special 0 character at the end + if (s->read_from_callbacks == 0) return 1; + } + + return s->img_buffer >= s->img_buffer_end; +} + +static void stbi__skip(stbi__context *s, int n) +{ + if (n < 0) { + s->img_buffer = s->img_buffer_end; + return; + } + if (s->io.read) { + int blen = (int) (s->img_buffer_end - s->img_buffer); + if (blen < n) { + s->img_buffer = s->img_buffer_end; + (s->io.skip)(s->io_user_data, n - blen); + return; + } + } + s->img_buffer += n; +} + +static int stbi__getn(stbi__context *s, stbi_uc *buffer, int n) +{ + if (s->io.read) { + int blen = (int) (s->img_buffer_end - s->img_buffer); + if (blen < n) { + int res, count; + + memcpy(buffer, s->img_buffer, blen); + + count = (s->io.read)(s->io_user_data, (char*) buffer + blen, n - blen); + res = (count == (n-blen)); + s->img_buffer = s->img_buffer_end; + return res; + } + } + + if (s->img_buffer+n <= s->img_buffer_end) { + memcpy(buffer, s->img_buffer, n); + s->img_buffer += n; + return 1; + } else + return 0; +} + +static int stbi__get16be(stbi__context *s) +{ + int z = stbi__get8(s); + return (z << 8) + stbi__get8(s); +} + +static stbi__uint32 stbi__get32be(stbi__context *s) +{ + stbi__uint32 z = stbi__get16be(s); + return (z << 16) + stbi__get16be(s); +} + +#if defined(STBI_NO_BMP) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) +// nothing +#else +static int stbi__get16le(stbi__context *s) +{ + int z = stbi__get8(s); + return z + (stbi__get8(s) << 8); +} +#endif + +#ifndef STBI_NO_BMP +static stbi__uint32 stbi__get32le(stbi__context *s) +{ + stbi__uint32 z = stbi__get16le(s); + return z + (stbi__get16le(s) << 16); +} +#endif + +#define STBI__BYTECAST(x) ((stbi_uc) ((x) & 255)) // truncate int to byte without warnings + + +////////////////////////////////////////////////////////////////////////////// +// +// generic converter from built-in img_n to req_comp +// individual types do this automatically as much as possible (e.g. jpeg +// does all cases internally since it needs to colorspace convert anyway, +// and it never has alpha, so very few cases ). png can automatically +// interleave an alpha=255 channel, but falls back to this for other cases +// +// assume data buffer is malloced, so malloc a new one and free that one +// only failure mode is malloc failing + +static stbi_uc stbi__compute_y(int r, int g, int b) +{ + return (stbi_uc) (((r*77) + (g*150) + (29*b)) >> 8); +} + +static unsigned char *stbi__convert_format(unsigned char *data, int img_n, int req_comp, unsigned int x, unsigned int y) +{ + int i,j; + unsigned char *good; + + if (req_comp == img_n) return data; + STBI_ASSERT(req_comp >= 1 && req_comp <= 4); + + good = (unsigned char *) stbi__malloc_mad3(req_comp, x, y, 0); + if (good == NULL) { + STBI_FREE(data); + return stbi__errpuc("outofmem", "Out of memory"); + } + + for (j=0; j < (int) y; ++j) { + unsigned char *src = data + j * x * img_n ; + unsigned char *dest = good + j * x * req_comp; + + #define STBI__COMBO(a,b) ((a)*8+(b)) + #define STBI__CASE(a,b) case STBI__COMBO(a,b): for(i=x-1; i >= 0; --i, src += a, dest += b) + // convert source image with img_n components to one with req_comp components; + // avoid switch per pixel, so use switch per scanline and massive macros + switch (STBI__COMBO(img_n, req_comp)) { + STBI__CASE(1,2) { dest[0]=src[0]; dest[1]=255; } break; + STBI__CASE(1,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; + STBI__CASE(1,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=255; } break; + STBI__CASE(2,1) { dest[0]=src[0]; } break; + STBI__CASE(2,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; + STBI__CASE(2,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=src[1]; } break; + STBI__CASE(3,4) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2];dest[3]=255; } break; + STBI__CASE(3,1) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); } break; + STBI__CASE(3,2) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); dest[1] = 255; } break; + STBI__CASE(4,1) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); } break; + STBI__CASE(4,2) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); dest[1] = src[3]; } break; + STBI__CASE(4,3) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2]; } break; + default: STBI_ASSERT(0); + } + #undef STBI__CASE + } + + STBI_FREE(data); + return good; +} + +static stbi__uint16 stbi__compute_y_16(int r, int g, int b) +{ + return (stbi__uint16) (((r*77) + (g*150) + (29*b)) >> 8); +} + +static stbi__uint16 *stbi__convert_format16(stbi__uint16 *data, int img_n, int req_comp, unsigned int x, unsigned int y) +{ + int i,j; + stbi__uint16 *good; + + if (req_comp == img_n) return data; + STBI_ASSERT(req_comp >= 1 && req_comp <= 4); + + good = (stbi__uint16 *) stbi__malloc(req_comp * x * y * 2); + if (good == NULL) { + STBI_FREE(data); + return (stbi__uint16 *) stbi__errpuc("outofmem", "Out of memory"); + } + + for (j=0; j < (int) y; ++j) { + stbi__uint16 *src = data + j * x * img_n ; + stbi__uint16 *dest = good + j * x * req_comp; + + #define STBI__COMBO(a,b) ((a)*8+(b)) + #define STBI__CASE(a,b) case STBI__COMBO(a,b): for(i=x-1; i >= 0; --i, src += a, dest += b) + // convert source image with img_n components to one with req_comp components; + // avoid switch per pixel, so use switch per scanline and massive macros + switch (STBI__COMBO(img_n, req_comp)) { + STBI__CASE(1,2) { dest[0]=src[0]; dest[1]=0xffff; } break; + STBI__CASE(1,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; + STBI__CASE(1,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=0xffff; } break; + STBI__CASE(2,1) { dest[0]=src[0]; } break; + STBI__CASE(2,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; + STBI__CASE(2,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=src[1]; } break; + STBI__CASE(3,4) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2];dest[3]=0xffff; } break; + STBI__CASE(3,1) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); } break; + STBI__CASE(3,2) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); dest[1] = 0xffff; } break; + STBI__CASE(4,1) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); } break; + STBI__CASE(4,2) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); dest[1] = src[3]; } break; + STBI__CASE(4,3) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2]; } break; + default: STBI_ASSERT(0); + } + #undef STBI__CASE + } + + STBI_FREE(data); + return good; +} + +#ifndef STBI_NO_LINEAR +static float *stbi__ldr_to_hdr(stbi_uc *data, int x, int y, int comp) +{ + int i,k,n; + float *output; + if (!data) return NULL; + output = (float *) stbi__malloc_mad4(x, y, comp, sizeof(float), 0); + if (output == NULL) { STBI_FREE(data); return stbi__errpf("outofmem", "Out of memory"); } + // compute number of non-alpha components + if (comp & 1) n = comp; else n = comp-1; + for (i=0; i < x*y; ++i) { + for (k=0; k < n; ++k) { + output[i*comp + k] = (float) (pow(data[i*comp+k]/255.0f, stbi__l2h_gamma) * stbi__l2h_scale); + } + } + if (n < comp) { + for (i=0; i < x*y; ++i) { + output[i*comp + n] = data[i*comp + n]/255.0f; + } + } + STBI_FREE(data); + return output; +} +#endif + +#ifndef STBI_NO_HDR +#define stbi__float2int(x) ((int) (x)) +static stbi_uc *stbi__hdr_to_ldr(float *data, int x, int y, int comp) +{ + int i,k,n; + stbi_uc *output; + if (!data) return NULL; + output = (stbi_uc *) stbi__malloc_mad3(x, y, comp, 0); + if (output == NULL) { STBI_FREE(data); return stbi__errpuc("outofmem", "Out of memory"); } + // compute number of non-alpha components + if (comp & 1) n = comp; else n = comp-1; + for (i=0; i < x*y; ++i) { + for (k=0; k < n; ++k) { + float z = (float) pow(data[i*comp+k]*stbi__h2l_scale_i, stbi__h2l_gamma_i) * 255 + 0.5f; + if (z < 0) z = 0; + if (z > 255) z = 255; + output[i*comp + k] = (stbi_uc) stbi__float2int(z); + } + if (k < comp) { + float z = data[i*comp+k] * 255 + 0.5f; + if (z < 0) z = 0; + if (z > 255) z = 255; + output[i*comp + k] = (stbi_uc) stbi__float2int(z); + } + } + STBI_FREE(data); + return output; +} +#endif + +////////////////////////////////////////////////////////////////////////////// +// +// "baseline" JPEG/JFIF decoder +// +// simple implementation +// - doesn't support delayed output of y-dimension +// - simple interface (only one output format: 8-bit interleaved RGB) +// - doesn't try to recover corrupt jpegs +// - doesn't allow partial loading, loading multiple at once +// - still fast on x86 (copying globals into locals doesn't help x86) +// - allocates lots of intermediate memory (full size of all components) +// - non-interleaved case requires this anyway +// - allows good upsampling (see next) +// high-quality +// - upsampled channels are bilinearly interpolated, even across blocks +// - quality integer IDCT derived from IJG's 'slow' +// performance +// - fast huffman; reasonable integer IDCT +// - some SIMD kernels for common paths on targets with SSE2/NEON +// - uses a lot of intermediate memory, could cache poorly + +#ifndef STBI_NO_JPEG + +// huffman decoding acceleration +#define FAST_BITS 9 // larger handles more cases; smaller stomps less cache + +typedef struct +{ + stbi_uc fast[1 << FAST_BITS]; + // weirdly, repacking this into AoS is a 10% speed loss, instead of a win + stbi__uint16 code[256]; + stbi_uc values[256]; + stbi_uc size[257]; + unsigned int maxcode[18]; + int delta[17]; // old 'firstsymbol' - old 'firstcode' +} stbi__huffman; + +typedef struct +{ + stbi__context *s; + stbi__huffman huff_dc[4]; + stbi__huffman huff_ac[4]; + stbi__uint16 dequant[4][64]; + stbi__int16 fast_ac[4][1 << FAST_BITS]; + +// sizes for components, interleaved MCUs + int img_h_max, img_v_max; + int img_mcu_x, img_mcu_y; + int img_mcu_w, img_mcu_h; + +// definition of jpeg image component + struct + { + int id; + int h,v; + int tq; + int hd,ha; + int dc_pred; + + int x,y,w2,h2; + stbi_uc *data; + void *raw_data, *raw_coeff; + stbi_uc *linebuf; + short *coeff; // progressive only + int coeff_w, coeff_h; // number of 8x8 coefficient blocks + } img_comp[4]; + + stbi__uint32 code_buffer; // jpeg entropy-coded buffer + int code_bits; // number of valid bits + unsigned char marker; // marker seen while filling entropy buffer + int nomore; // flag if we saw a marker so must stop + + int progressive; + int spec_start; + int spec_end; + int succ_high; + int succ_low; + int eob_run; + int jfif; + int app14_color_transform; // Adobe APP14 tag + int rgb; + + int scan_n, order[4]; + int restart_interval, todo; + +// kernels + void (*idct_block_kernel)(stbi_uc *out, int out_stride, short data[64]); + void (*YCbCr_to_RGB_kernel)(stbi_uc *out, const stbi_uc *y, const stbi_uc *pcb, const stbi_uc *pcr, int count, int step); + stbi_uc *(*resample_row_hv_2_kernel)(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs); +} stbi__jpeg; + +static int stbi__build_huffman(stbi__huffman *h, int *count) +{ + int i,j,k=0; + unsigned int code; + // build size list for each symbol (from JPEG spec) + for (i=0; i < 16; ++i) + for (j=0; j < count[i]; ++j) + h->size[k++] = (stbi_uc) (i+1); + h->size[k] = 0; + + // compute actual symbols (from jpeg spec) + code = 0; + k = 0; + for(j=1; j <= 16; ++j) { + // compute delta to add to code to compute symbol id + h->delta[j] = k - code; + if (h->size[k] == j) { + while (h->size[k] == j) + h->code[k++] = (stbi__uint16) (code++); + if (code-1 >= (1u << j)) return stbi__err("bad code lengths","Corrupt JPEG"); + } + // compute largest code + 1 for this size, preshifted as needed later + h->maxcode[j] = code << (16-j); + code <<= 1; + } + h->maxcode[j] = 0xffffffff; + + // build non-spec acceleration table; 255 is flag for not-accelerated + memset(h->fast, 255, 1 << FAST_BITS); + for (i=0; i < k; ++i) { + int s = h->size[i]; + if (s <= FAST_BITS) { + int c = h->code[i] << (FAST_BITS-s); + int m = 1 << (FAST_BITS-s); + for (j=0; j < m; ++j) { + h->fast[c+j] = (stbi_uc) i; + } + } + } + return 1; +} + +// build a table that decodes both magnitude and value of small ACs in +// one go. +static void stbi__build_fast_ac(stbi__int16 *fast_ac, stbi__huffman *h) +{ + int i; + for (i=0; i < (1 << FAST_BITS); ++i) { + stbi_uc fast = h->fast[i]; + fast_ac[i] = 0; + if (fast < 255) { + int rs = h->values[fast]; + int run = (rs >> 4) & 15; + int magbits = rs & 15; + int len = h->size[fast]; + + if (magbits && len + magbits <= FAST_BITS) { + // magnitude code followed by receive_extend code + int k = ((i << len) & ((1 << FAST_BITS) - 1)) >> (FAST_BITS - magbits); + int m = 1 << (magbits - 1); + if (k < m) k += (~0U << magbits) + 1; + // if the result is small enough, we can fit it in fast_ac table + if (k >= -128 && k <= 127) + fast_ac[i] = (stbi__int16) ((k * 256) + (run * 16) + (len + magbits)); + } + } + } +} + +static void stbi__grow_buffer_unsafe(stbi__jpeg *j) +{ + do { + unsigned int b = j->nomore ? 0 : stbi__get8(j->s); + if (b == 0xff) { + int c = stbi__get8(j->s); + while (c == 0xff) c = stbi__get8(j->s); // consume fill bytes + if (c != 0) { + j->marker = (unsigned char) c; + j->nomore = 1; + return; + } + } + j->code_buffer |= b << (24 - j->code_bits); + j->code_bits += 8; + } while (j->code_bits <= 24); +} + +// (1 << n) - 1 +static const stbi__uint32 stbi__bmask[17]={0,1,3,7,15,31,63,127,255,511,1023,2047,4095,8191,16383,32767,65535}; + +// decode a jpeg huffman value from the bitstream +stbi_inline static int stbi__jpeg_huff_decode(stbi__jpeg *j, stbi__huffman *h) +{ + unsigned int temp; + int c,k; + + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + + // look at the top FAST_BITS and determine what symbol ID it is, + // if the code is <= FAST_BITS + c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1); + k = h->fast[c]; + if (k < 255) { + int s = h->size[k]; + if (s > j->code_bits) + return -1; + j->code_buffer <<= s; + j->code_bits -= s; + return h->values[k]; + } + + // naive test is to shift the code_buffer down so k bits are + // valid, then test against maxcode. To speed this up, we've + // preshifted maxcode left so that it has (16-k) 0s at the + // end; in other words, regardless of the number of bits, it + // wants to be compared against something shifted to have 16; + // that way we don't need to shift inside the loop. + temp = j->code_buffer >> 16; + for (k=FAST_BITS+1 ; ; ++k) + if (temp < h->maxcode[k]) + break; + if (k == 17) { + // error! code not found + j->code_bits -= 16; + return -1; + } + + if (k > j->code_bits) + return -1; + + // convert the huffman code to the symbol id + c = ((j->code_buffer >> (32 - k)) & stbi__bmask[k]) + h->delta[k]; + STBI_ASSERT((((j->code_buffer) >> (32 - h->size[c])) & stbi__bmask[h->size[c]]) == h->code[c]); + + // convert the id to a symbol + j->code_bits -= k; + j->code_buffer <<= k; + return h->values[c]; +} + +// bias[n] = (-1<code_bits < n) stbi__grow_buffer_unsafe(j); + + sgn = (stbi__int32)j->code_buffer >> 31; // sign bit is always in MSB + k = stbi_lrot(j->code_buffer, n); + STBI_ASSERT(n >= 0 && n < (int) (sizeof(stbi__bmask)/sizeof(*stbi__bmask))); + j->code_buffer = k & ~stbi__bmask[n]; + k &= stbi__bmask[n]; + j->code_bits -= n; + return k + (stbi__jbias[n] & ~sgn); +} + +// get some unsigned bits +stbi_inline static int stbi__jpeg_get_bits(stbi__jpeg *j, int n) +{ + unsigned int k; + if (j->code_bits < n) stbi__grow_buffer_unsafe(j); + k = stbi_lrot(j->code_buffer, n); + j->code_buffer = k & ~stbi__bmask[n]; + k &= stbi__bmask[n]; + j->code_bits -= n; + return k; +} + +stbi_inline static int stbi__jpeg_get_bit(stbi__jpeg *j) +{ + unsigned int k; + if (j->code_bits < 1) stbi__grow_buffer_unsafe(j); + k = j->code_buffer; + j->code_buffer <<= 1; + --j->code_bits; + return k & 0x80000000; +} + +// given a value that's at position X in the zigzag stream, +// where does it appear in the 8x8 matrix coded as row-major? +static const stbi_uc stbi__jpeg_dezigzag[64+15] = +{ + 0, 1, 8, 16, 9, 2, 3, 10, + 17, 24, 32, 25, 18, 11, 4, 5, + 12, 19, 26, 33, 40, 48, 41, 34, + 27, 20, 13, 6, 7, 14, 21, 28, + 35, 42, 49, 56, 57, 50, 43, 36, + 29, 22, 15, 23, 30, 37, 44, 51, + 58, 59, 52, 45, 38, 31, 39, 46, + 53, 60, 61, 54, 47, 55, 62, 63, + // let corrupt input sample past end + 63, 63, 63, 63, 63, 63, 63, 63, + 63, 63, 63, 63, 63, 63, 63 +}; + +// decode one 64-entry block-- +static int stbi__jpeg_decode_block(stbi__jpeg *j, short data[64], stbi__huffman *hdc, stbi__huffman *hac, stbi__int16 *fac, int b, stbi__uint16 *dequant) +{ + int diff,dc,k; + int t; + + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + t = stbi__jpeg_huff_decode(j, hdc); + if (t < 0) return stbi__err("bad huffman code","Corrupt JPEG"); + + // 0 all the ac values now so we can do it 32-bits at a time + memset(data,0,64*sizeof(data[0])); + + diff = t ? stbi__extend_receive(j, t) : 0; + dc = j->img_comp[b].dc_pred + diff; + j->img_comp[b].dc_pred = dc; + data[0] = (short) (dc * dequant[0]); + + // decode AC components, see JPEG spec + k = 1; + do { + unsigned int zig; + int c,r,s; + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1); + r = fac[c]; + if (r) { // fast-AC path + k += (r >> 4) & 15; // run + s = r & 15; // combined length + j->code_buffer <<= s; + j->code_bits -= s; + // decode into unzigzag'd location + zig = stbi__jpeg_dezigzag[k++]; + data[zig] = (short) ((r >> 8) * dequant[zig]); + } else { + int rs = stbi__jpeg_huff_decode(j, hac); + if (rs < 0) return stbi__err("bad huffman code","Corrupt JPEG"); + s = rs & 15; + r = rs >> 4; + if (s == 0) { + if (rs != 0xf0) break; // end block + k += 16; + } else { + k += r; + // decode into unzigzag'd location + zig = stbi__jpeg_dezigzag[k++]; + data[zig] = (short) (stbi__extend_receive(j,s) * dequant[zig]); + } + } + } while (k < 64); + return 1; +} + +static int stbi__jpeg_decode_block_prog_dc(stbi__jpeg *j, short data[64], stbi__huffman *hdc, int b) +{ + int diff,dc; + int t; + if (j->spec_end != 0) return stbi__err("can't merge dc and ac", "Corrupt JPEG"); + + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + + if (j->succ_high == 0) { + // first scan for DC coefficient, must be first + memset(data,0,64*sizeof(data[0])); // 0 all the ac values now + t = stbi__jpeg_huff_decode(j, hdc); + diff = t ? stbi__extend_receive(j, t) : 0; + + dc = j->img_comp[b].dc_pred + diff; + j->img_comp[b].dc_pred = dc; + data[0] = (short) (dc << j->succ_low); + } else { + // refinement scan for DC coefficient + if (stbi__jpeg_get_bit(j)) + data[0] += (short) (1 << j->succ_low); + } + return 1; +} + +// @OPTIMIZE: store non-zigzagged during the decode passes, +// and only de-zigzag when dequantizing +static int stbi__jpeg_decode_block_prog_ac(stbi__jpeg *j, short data[64], stbi__huffman *hac, stbi__int16 *fac) +{ + int k; + if (j->spec_start == 0) return stbi__err("can't merge dc and ac", "Corrupt JPEG"); + + if (j->succ_high == 0) { + int shift = j->succ_low; + + if (j->eob_run) { + --j->eob_run; + return 1; + } + + k = j->spec_start; + do { + unsigned int zig; + int c,r,s; + if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); + c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1); + r = fac[c]; + if (r) { // fast-AC path + k += (r >> 4) & 15; // run + s = r & 15; // combined length + j->code_buffer <<= s; + j->code_bits -= s; + zig = stbi__jpeg_dezigzag[k++]; + data[zig] = (short) ((r >> 8) << shift); + } else { + int rs = stbi__jpeg_huff_decode(j, hac); + if (rs < 0) return stbi__err("bad huffman code","Corrupt JPEG"); + s = rs & 15; + r = rs >> 4; + if (s == 0) { + if (r < 15) { + j->eob_run = (1 << r); + if (r) + j->eob_run += stbi__jpeg_get_bits(j, r); + --j->eob_run; + break; + } + k += 16; + } else { + k += r; + zig = stbi__jpeg_dezigzag[k++]; + data[zig] = (short) (stbi__extend_receive(j,s) << shift); + } + } + } while (k <= j->spec_end); + } else { + // refinement scan for these AC coefficients + + short bit = (short) (1 << j->succ_low); + + if (j->eob_run) { + --j->eob_run; + for (k = j->spec_start; k <= j->spec_end; ++k) { + short *p = &data[stbi__jpeg_dezigzag[k]]; + if (*p != 0) + if (stbi__jpeg_get_bit(j)) + if ((*p & bit)==0) { + if (*p > 0) + *p += bit; + else + *p -= bit; + } + } + } else { + k = j->spec_start; + do { + int r,s; + int rs = stbi__jpeg_huff_decode(j, hac); // @OPTIMIZE see if we can use the fast path here, advance-by-r is so slow, eh + if (rs < 0) return stbi__err("bad huffman code","Corrupt JPEG"); + s = rs & 15; + r = rs >> 4; + if (s == 0) { + if (r < 15) { + j->eob_run = (1 << r) - 1; + if (r) + j->eob_run += stbi__jpeg_get_bits(j, r); + r = 64; // force end of block + } else { + // r=15 s=0 should write 16 0s, so we just do + // a run of 15 0s and then write s (which is 0), + // so we don't have to do anything special here + } + } else { + if (s != 1) return stbi__err("bad huffman code", "Corrupt JPEG"); + // sign bit + if (stbi__jpeg_get_bit(j)) + s = bit; + else + s = -bit; + } + + // advance by r + while (k <= j->spec_end) { + short *p = &data[stbi__jpeg_dezigzag[k++]]; + if (*p != 0) { + if (stbi__jpeg_get_bit(j)) + if ((*p & bit)==0) { + if (*p > 0) + *p += bit; + else + *p -= bit; + } + } else { + if (r == 0) { + *p = (short) s; + break; + } + --r; + } + } + } while (k <= j->spec_end); + } + } + return 1; +} + +// take a -128..127 value and stbi__clamp it and convert to 0..255 +stbi_inline static stbi_uc stbi__clamp(int x) +{ + // trick to use a single test to catch both cases + if ((unsigned int) x > 255) { + if (x < 0) return 0; + if (x > 255) return 255; + } + return (stbi_uc) x; +} + +#define stbi__f2f(x) ((int) (((x) * 4096 + 0.5))) +#define stbi__fsh(x) ((x) * 4096) + +// derived from jidctint -- DCT_ISLOW +#define STBI__IDCT_1D(s0,s1,s2,s3,s4,s5,s6,s7) \ + int t0,t1,t2,t3,p1,p2,p3,p4,p5,x0,x1,x2,x3; \ + p2 = s2; \ + p3 = s6; \ + p1 = (p2+p3) * stbi__f2f(0.5411961f); \ + t2 = p1 + p3*stbi__f2f(-1.847759065f); \ + t3 = p1 + p2*stbi__f2f( 0.765366865f); \ + p2 = s0; \ + p3 = s4; \ + t0 = stbi__fsh(p2+p3); \ + t1 = stbi__fsh(p2-p3); \ + x0 = t0+t3; \ + x3 = t0-t3; \ + x1 = t1+t2; \ + x2 = t1-t2; \ + t0 = s7; \ + t1 = s5; \ + t2 = s3; \ + t3 = s1; \ + p3 = t0+t2; \ + p4 = t1+t3; \ + p1 = t0+t3; \ + p2 = t1+t2; \ + p5 = (p3+p4)*stbi__f2f( 1.175875602f); \ + t0 = t0*stbi__f2f( 0.298631336f); \ + t1 = t1*stbi__f2f( 2.053119869f); \ + t2 = t2*stbi__f2f( 3.072711026f); \ + t3 = t3*stbi__f2f( 1.501321110f); \ + p1 = p5 + p1*stbi__f2f(-0.899976223f); \ + p2 = p5 + p2*stbi__f2f(-2.562915447f); \ + p3 = p3*stbi__f2f(-1.961570560f); \ + p4 = p4*stbi__f2f(-0.390180644f); \ + t3 += p1+p4; \ + t2 += p2+p3; \ + t1 += p2+p4; \ + t0 += p1+p3; + +static void stbi__idct_block(stbi_uc *out, int out_stride, short data[64]) +{ + int i,val[64],*v=val; + stbi_uc *o; + short *d = data; + + // columns + for (i=0; i < 8; ++i,++d, ++v) { + // if all zeroes, shortcut -- this avoids dequantizing 0s and IDCTing + if (d[ 8]==0 && d[16]==0 && d[24]==0 && d[32]==0 + && d[40]==0 && d[48]==0 && d[56]==0) { + // no shortcut 0 seconds + // (1|2|3|4|5|6|7)==0 0 seconds + // all separate -0.047 seconds + // 1 && 2|3 && 4|5 && 6|7: -0.047 seconds + int dcterm = d[0]*4; + v[0] = v[8] = v[16] = v[24] = v[32] = v[40] = v[48] = v[56] = dcterm; + } else { + STBI__IDCT_1D(d[ 0],d[ 8],d[16],d[24],d[32],d[40],d[48],d[56]) + // constants scaled things up by 1<<12; let's bring them back + // down, but keep 2 extra bits of precision + x0 += 512; x1 += 512; x2 += 512; x3 += 512; + v[ 0] = (x0+t3) >> 10; + v[56] = (x0-t3) >> 10; + v[ 8] = (x1+t2) >> 10; + v[48] = (x1-t2) >> 10; + v[16] = (x2+t1) >> 10; + v[40] = (x2-t1) >> 10; + v[24] = (x3+t0) >> 10; + v[32] = (x3-t0) >> 10; + } + } + + for (i=0, v=val, o=out; i < 8; ++i,v+=8,o+=out_stride) { + // no fast case since the first 1D IDCT spread components out + STBI__IDCT_1D(v[0],v[1],v[2],v[3],v[4],v[5],v[6],v[7]) + // constants scaled things up by 1<<12, plus we had 1<<2 from first + // loop, plus horizontal and vertical each scale by sqrt(8) so together + // we've got an extra 1<<3, so 1<<17 total we need to remove. + // so we want to round that, which means adding 0.5 * 1<<17, + // aka 65536. Also, we'll end up with -128 to 127 that we want + // to encode as 0..255 by adding 128, so we'll add that before the shift + x0 += 65536 + (128<<17); + x1 += 65536 + (128<<17); + x2 += 65536 + (128<<17); + x3 += 65536 + (128<<17); + // tried computing the shifts into temps, or'ing the temps to see + // if any were out of range, but that was slower + o[0] = stbi__clamp((x0+t3) >> 17); + o[7] = stbi__clamp((x0-t3) >> 17); + o[1] = stbi__clamp((x1+t2) >> 17); + o[6] = stbi__clamp((x1-t2) >> 17); + o[2] = stbi__clamp((x2+t1) >> 17); + o[5] = stbi__clamp((x2-t1) >> 17); + o[3] = stbi__clamp((x3+t0) >> 17); + o[4] = stbi__clamp((x3-t0) >> 17); + } +} + +#ifdef STBI_SSE2 +// sse2 integer IDCT. not the fastest possible implementation but it +// produces bit-identical results to the generic C version so it's +// fully "transparent". +static void stbi__idct_simd(stbi_uc *out, int out_stride, short data[64]) +{ + // This is constructed to match our regular (generic) integer IDCT exactly. + __m128i row0, row1, row2, row3, row4, row5, row6, row7; + __m128i tmp; + + // dot product constant: even elems=x, odd elems=y + #define dct_const(x,y) _mm_setr_epi16((x),(y),(x),(y),(x),(y),(x),(y)) + + // out(0) = c0[even]*x + c0[odd]*y (c0, x, y 16-bit, out 32-bit) + // out(1) = c1[even]*x + c1[odd]*y + #define dct_rot(out0,out1, x,y,c0,c1) \ + __m128i c0##lo = _mm_unpacklo_epi16((x),(y)); \ + __m128i c0##hi = _mm_unpackhi_epi16((x),(y)); \ + __m128i out0##_l = _mm_madd_epi16(c0##lo, c0); \ + __m128i out0##_h = _mm_madd_epi16(c0##hi, c0); \ + __m128i out1##_l = _mm_madd_epi16(c0##lo, c1); \ + __m128i out1##_h = _mm_madd_epi16(c0##hi, c1) + + // out = in << 12 (in 16-bit, out 32-bit) + #define dct_widen(out, in) \ + __m128i out##_l = _mm_srai_epi32(_mm_unpacklo_epi16(_mm_setzero_si128(), (in)), 4); \ + __m128i out##_h = _mm_srai_epi32(_mm_unpackhi_epi16(_mm_setzero_si128(), (in)), 4) + + // wide add + #define dct_wadd(out, a, b) \ + __m128i out##_l = _mm_add_epi32(a##_l, b##_l); \ + __m128i out##_h = _mm_add_epi32(a##_h, b##_h) + + // wide sub + #define dct_wsub(out, a, b) \ + __m128i out##_l = _mm_sub_epi32(a##_l, b##_l); \ + __m128i out##_h = _mm_sub_epi32(a##_h, b##_h) + + // butterfly a/b, add bias, then shift by "s" and pack + #define dct_bfly32o(out0, out1, a,b,bias,s) \ + { \ + __m128i abiased_l = _mm_add_epi32(a##_l, bias); \ + __m128i abiased_h = _mm_add_epi32(a##_h, bias); \ + dct_wadd(sum, abiased, b); \ + dct_wsub(dif, abiased, b); \ + out0 = _mm_packs_epi32(_mm_srai_epi32(sum_l, s), _mm_srai_epi32(sum_h, s)); \ + out1 = _mm_packs_epi32(_mm_srai_epi32(dif_l, s), _mm_srai_epi32(dif_h, s)); \ + } + + // 8-bit interleave step (for transposes) + #define dct_interleave8(a, b) \ + tmp = a; \ + a = _mm_unpacklo_epi8(a, b); \ + b = _mm_unpackhi_epi8(tmp, b) + + // 16-bit interleave step (for transposes) + #define dct_interleave16(a, b) \ + tmp = a; \ + a = _mm_unpacklo_epi16(a, b); \ + b = _mm_unpackhi_epi16(tmp, b) + + #define dct_pass(bias,shift) \ + { \ + /* even part */ \ + dct_rot(t2e,t3e, row2,row6, rot0_0,rot0_1); \ + __m128i sum04 = _mm_add_epi16(row0, row4); \ + __m128i dif04 = _mm_sub_epi16(row0, row4); \ + dct_widen(t0e, sum04); \ + dct_widen(t1e, dif04); \ + dct_wadd(x0, t0e, t3e); \ + dct_wsub(x3, t0e, t3e); \ + dct_wadd(x1, t1e, t2e); \ + dct_wsub(x2, t1e, t2e); \ + /* odd part */ \ + dct_rot(y0o,y2o, row7,row3, rot2_0,rot2_1); \ + dct_rot(y1o,y3o, row5,row1, rot3_0,rot3_1); \ + __m128i sum17 = _mm_add_epi16(row1, row7); \ + __m128i sum35 = _mm_add_epi16(row3, row5); \ + dct_rot(y4o,y5o, sum17,sum35, rot1_0,rot1_1); \ + dct_wadd(x4, y0o, y4o); \ + dct_wadd(x5, y1o, y5o); \ + dct_wadd(x6, y2o, y5o); \ + dct_wadd(x7, y3o, y4o); \ + dct_bfly32o(row0,row7, x0,x7,bias,shift); \ + dct_bfly32o(row1,row6, x1,x6,bias,shift); \ + dct_bfly32o(row2,row5, x2,x5,bias,shift); \ + dct_bfly32o(row3,row4, x3,x4,bias,shift); \ + } + + __m128i rot0_0 = dct_const(stbi__f2f(0.5411961f), stbi__f2f(0.5411961f) + stbi__f2f(-1.847759065f)); + __m128i rot0_1 = dct_const(stbi__f2f(0.5411961f) + stbi__f2f( 0.765366865f), stbi__f2f(0.5411961f)); + __m128i rot1_0 = dct_const(stbi__f2f(1.175875602f) + stbi__f2f(-0.899976223f), stbi__f2f(1.175875602f)); + __m128i rot1_1 = dct_const(stbi__f2f(1.175875602f), stbi__f2f(1.175875602f) + stbi__f2f(-2.562915447f)); + __m128i rot2_0 = dct_const(stbi__f2f(-1.961570560f) + stbi__f2f( 0.298631336f), stbi__f2f(-1.961570560f)); + __m128i rot2_1 = dct_const(stbi__f2f(-1.961570560f), stbi__f2f(-1.961570560f) + stbi__f2f( 3.072711026f)); + __m128i rot3_0 = dct_const(stbi__f2f(-0.390180644f) + stbi__f2f( 2.053119869f), stbi__f2f(-0.390180644f)); + __m128i rot3_1 = dct_const(stbi__f2f(-0.390180644f), stbi__f2f(-0.390180644f) + stbi__f2f( 1.501321110f)); + + // rounding biases in column/row passes, see stbi__idct_block for explanation. + __m128i bias_0 = _mm_set1_epi32(512); + __m128i bias_1 = _mm_set1_epi32(65536 + (128<<17)); + + // load + row0 = _mm_load_si128((const __m128i *) (data + 0*8)); + row1 = _mm_load_si128((const __m128i *) (data + 1*8)); + row2 = _mm_load_si128((const __m128i *) (data + 2*8)); + row3 = _mm_load_si128((const __m128i *) (data + 3*8)); + row4 = _mm_load_si128((const __m128i *) (data + 4*8)); + row5 = _mm_load_si128((const __m128i *) (data + 5*8)); + row6 = _mm_load_si128((const __m128i *) (data + 6*8)); + row7 = _mm_load_si128((const __m128i *) (data + 7*8)); + + // column pass + dct_pass(bias_0, 10); + + { + // 16bit 8x8 transpose pass 1 + dct_interleave16(row0, row4); + dct_interleave16(row1, row5); + dct_interleave16(row2, row6); + dct_interleave16(row3, row7); + + // transpose pass 2 + dct_interleave16(row0, row2); + dct_interleave16(row1, row3); + dct_interleave16(row4, row6); + dct_interleave16(row5, row7); + + // transpose pass 3 + dct_interleave16(row0, row1); + dct_interleave16(row2, row3); + dct_interleave16(row4, row5); + dct_interleave16(row6, row7); + } + + // row pass + dct_pass(bias_1, 17); + + { + // pack + __m128i p0 = _mm_packus_epi16(row0, row1); // a0a1a2a3...a7b0b1b2b3...b7 + __m128i p1 = _mm_packus_epi16(row2, row3); + __m128i p2 = _mm_packus_epi16(row4, row5); + __m128i p3 = _mm_packus_epi16(row6, row7); + + // 8bit 8x8 transpose pass 1 + dct_interleave8(p0, p2); // a0e0a1e1... + dct_interleave8(p1, p3); // c0g0c1g1... + + // transpose pass 2 + dct_interleave8(p0, p1); // a0c0e0g0... + dct_interleave8(p2, p3); // b0d0f0h0... + + // transpose pass 3 + dct_interleave8(p0, p2); // a0b0c0d0... + dct_interleave8(p1, p3); // a4b4c4d4... + + // store + _mm_storel_epi64((__m128i *) out, p0); out += out_stride; + _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p0, 0x4e)); out += out_stride; + _mm_storel_epi64((__m128i *) out, p2); out += out_stride; + _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p2, 0x4e)); out += out_stride; + _mm_storel_epi64((__m128i *) out, p1); out += out_stride; + _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p1, 0x4e)); out += out_stride; + _mm_storel_epi64((__m128i *) out, p3); out += out_stride; + _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p3, 0x4e)); + } + +#undef dct_const +#undef dct_rot +#undef dct_widen +#undef dct_wadd +#undef dct_wsub +#undef dct_bfly32o +#undef dct_interleave8 +#undef dct_interleave16 +#undef dct_pass +} + +#endif // STBI_SSE2 + +#ifdef STBI_NEON + +// NEON integer IDCT. should produce bit-identical +// results to the generic C version. +static void stbi__idct_simd(stbi_uc *out, int out_stride, short data[64]) +{ + int16x8_t row0, row1, row2, row3, row4, row5, row6, row7; + + int16x4_t rot0_0 = vdup_n_s16(stbi__f2f(0.5411961f)); + int16x4_t rot0_1 = vdup_n_s16(stbi__f2f(-1.847759065f)); + int16x4_t rot0_2 = vdup_n_s16(stbi__f2f( 0.765366865f)); + int16x4_t rot1_0 = vdup_n_s16(stbi__f2f( 1.175875602f)); + int16x4_t rot1_1 = vdup_n_s16(stbi__f2f(-0.899976223f)); + int16x4_t rot1_2 = vdup_n_s16(stbi__f2f(-2.562915447f)); + int16x4_t rot2_0 = vdup_n_s16(stbi__f2f(-1.961570560f)); + int16x4_t rot2_1 = vdup_n_s16(stbi__f2f(-0.390180644f)); + int16x4_t rot3_0 = vdup_n_s16(stbi__f2f( 0.298631336f)); + int16x4_t rot3_1 = vdup_n_s16(stbi__f2f( 2.053119869f)); + int16x4_t rot3_2 = vdup_n_s16(stbi__f2f( 3.072711026f)); + int16x4_t rot3_3 = vdup_n_s16(stbi__f2f( 1.501321110f)); + +#define dct_long_mul(out, inq, coeff) \ + int32x4_t out##_l = vmull_s16(vget_low_s16(inq), coeff); \ + int32x4_t out##_h = vmull_s16(vget_high_s16(inq), coeff) + +#define dct_long_mac(out, acc, inq, coeff) \ + int32x4_t out##_l = vmlal_s16(acc##_l, vget_low_s16(inq), coeff); \ + int32x4_t out##_h = vmlal_s16(acc##_h, vget_high_s16(inq), coeff) + +#define dct_widen(out, inq) \ + int32x4_t out##_l = vshll_n_s16(vget_low_s16(inq), 12); \ + int32x4_t out##_h = vshll_n_s16(vget_high_s16(inq), 12) + +// wide add +#define dct_wadd(out, a, b) \ + int32x4_t out##_l = vaddq_s32(a##_l, b##_l); \ + int32x4_t out##_h = vaddq_s32(a##_h, b##_h) + +// wide sub +#define dct_wsub(out, a, b) \ + int32x4_t out##_l = vsubq_s32(a##_l, b##_l); \ + int32x4_t out##_h = vsubq_s32(a##_h, b##_h) + +// butterfly a/b, then shift using "shiftop" by "s" and pack +#define dct_bfly32o(out0,out1, a,b,shiftop,s) \ + { \ + dct_wadd(sum, a, b); \ + dct_wsub(dif, a, b); \ + out0 = vcombine_s16(shiftop(sum_l, s), shiftop(sum_h, s)); \ + out1 = vcombine_s16(shiftop(dif_l, s), shiftop(dif_h, s)); \ + } + +#define dct_pass(shiftop, shift) \ + { \ + /* even part */ \ + int16x8_t sum26 = vaddq_s16(row2, row6); \ + dct_long_mul(p1e, sum26, rot0_0); \ + dct_long_mac(t2e, p1e, row6, rot0_1); \ + dct_long_mac(t3e, p1e, row2, rot0_2); \ + int16x8_t sum04 = vaddq_s16(row0, row4); \ + int16x8_t dif04 = vsubq_s16(row0, row4); \ + dct_widen(t0e, sum04); \ + dct_widen(t1e, dif04); \ + dct_wadd(x0, t0e, t3e); \ + dct_wsub(x3, t0e, t3e); \ + dct_wadd(x1, t1e, t2e); \ + dct_wsub(x2, t1e, t2e); \ + /* odd part */ \ + int16x8_t sum15 = vaddq_s16(row1, row5); \ + int16x8_t sum17 = vaddq_s16(row1, row7); \ + int16x8_t sum35 = vaddq_s16(row3, row5); \ + int16x8_t sum37 = vaddq_s16(row3, row7); \ + int16x8_t sumodd = vaddq_s16(sum17, sum35); \ + dct_long_mul(p5o, sumodd, rot1_0); \ + dct_long_mac(p1o, p5o, sum17, rot1_1); \ + dct_long_mac(p2o, p5o, sum35, rot1_2); \ + dct_long_mul(p3o, sum37, rot2_0); \ + dct_long_mul(p4o, sum15, rot2_1); \ + dct_wadd(sump13o, p1o, p3o); \ + dct_wadd(sump24o, p2o, p4o); \ + dct_wadd(sump23o, p2o, p3o); \ + dct_wadd(sump14o, p1o, p4o); \ + dct_long_mac(x4, sump13o, row7, rot3_0); \ + dct_long_mac(x5, sump24o, row5, rot3_1); \ + dct_long_mac(x6, sump23o, row3, rot3_2); \ + dct_long_mac(x7, sump14o, row1, rot3_3); \ + dct_bfly32o(row0,row7, x0,x7,shiftop,shift); \ + dct_bfly32o(row1,row6, x1,x6,shiftop,shift); \ + dct_bfly32o(row2,row5, x2,x5,shiftop,shift); \ + dct_bfly32o(row3,row4, x3,x4,shiftop,shift); \ + } + + // load + row0 = vld1q_s16(data + 0*8); + row1 = vld1q_s16(data + 1*8); + row2 = vld1q_s16(data + 2*8); + row3 = vld1q_s16(data + 3*8); + row4 = vld1q_s16(data + 4*8); + row5 = vld1q_s16(data + 5*8); + row6 = vld1q_s16(data + 6*8); + row7 = vld1q_s16(data + 7*8); + + // add DC bias + row0 = vaddq_s16(row0, vsetq_lane_s16(1024, vdupq_n_s16(0), 0)); + + // column pass + dct_pass(vrshrn_n_s32, 10); + + // 16bit 8x8 transpose + { +// these three map to a single VTRN.16, VTRN.32, and VSWP, respectively. +// whether compilers actually get this is another story, sadly. +#define dct_trn16(x, y) { int16x8x2_t t = vtrnq_s16(x, y); x = t.val[0]; y = t.val[1]; } +#define dct_trn32(x, y) { int32x4x2_t t = vtrnq_s32(vreinterpretq_s32_s16(x), vreinterpretq_s32_s16(y)); x = vreinterpretq_s16_s32(t.val[0]); y = vreinterpretq_s16_s32(t.val[1]); } +#define dct_trn64(x, y) { int16x8_t x0 = x; int16x8_t y0 = y; x = vcombine_s16(vget_low_s16(x0), vget_low_s16(y0)); y = vcombine_s16(vget_high_s16(x0), vget_high_s16(y0)); } + + // pass 1 + dct_trn16(row0, row1); // a0b0a2b2a4b4a6b6 + dct_trn16(row2, row3); + dct_trn16(row4, row5); + dct_trn16(row6, row7); + + // pass 2 + dct_trn32(row0, row2); // a0b0c0d0a4b4c4d4 + dct_trn32(row1, row3); + dct_trn32(row4, row6); + dct_trn32(row5, row7); + + // pass 3 + dct_trn64(row0, row4); // a0b0c0d0e0f0g0h0 + dct_trn64(row1, row5); + dct_trn64(row2, row6); + dct_trn64(row3, row7); + +#undef dct_trn16 +#undef dct_trn32 +#undef dct_trn64 + } + + // row pass + // vrshrn_n_s32 only supports shifts up to 16, we need + // 17. so do a non-rounding shift of 16 first then follow + // up with a rounding shift by 1. + dct_pass(vshrn_n_s32, 16); + + { + // pack and round + uint8x8_t p0 = vqrshrun_n_s16(row0, 1); + uint8x8_t p1 = vqrshrun_n_s16(row1, 1); + uint8x8_t p2 = vqrshrun_n_s16(row2, 1); + uint8x8_t p3 = vqrshrun_n_s16(row3, 1); + uint8x8_t p4 = vqrshrun_n_s16(row4, 1); + uint8x8_t p5 = vqrshrun_n_s16(row5, 1); + uint8x8_t p6 = vqrshrun_n_s16(row6, 1); + uint8x8_t p7 = vqrshrun_n_s16(row7, 1); + + // again, these can translate into one instruction, but often don't. +#define dct_trn8_8(x, y) { uint8x8x2_t t = vtrn_u8(x, y); x = t.val[0]; y = t.val[1]; } +#define dct_trn8_16(x, y) { uint16x4x2_t t = vtrn_u16(vreinterpret_u16_u8(x), vreinterpret_u16_u8(y)); x = vreinterpret_u8_u16(t.val[0]); y = vreinterpret_u8_u16(t.val[1]); } +#define dct_trn8_32(x, y) { uint32x2x2_t t = vtrn_u32(vreinterpret_u32_u8(x), vreinterpret_u32_u8(y)); x = vreinterpret_u8_u32(t.val[0]); y = vreinterpret_u8_u32(t.val[1]); } + + // sadly can't use interleaved stores here since we only write + // 8 bytes to each scan line! + + // 8x8 8-bit transpose pass 1 + dct_trn8_8(p0, p1); + dct_trn8_8(p2, p3); + dct_trn8_8(p4, p5); + dct_trn8_8(p6, p7); + + // pass 2 + dct_trn8_16(p0, p2); + dct_trn8_16(p1, p3); + dct_trn8_16(p4, p6); + dct_trn8_16(p5, p7); + + // pass 3 + dct_trn8_32(p0, p4); + dct_trn8_32(p1, p5); + dct_trn8_32(p2, p6); + dct_trn8_32(p3, p7); + + // store + vst1_u8(out, p0); out += out_stride; + vst1_u8(out, p1); out += out_stride; + vst1_u8(out, p2); out += out_stride; + vst1_u8(out, p3); out += out_stride; + vst1_u8(out, p4); out += out_stride; + vst1_u8(out, p5); out += out_stride; + vst1_u8(out, p6); out += out_stride; + vst1_u8(out, p7); + +#undef dct_trn8_8 +#undef dct_trn8_16 +#undef dct_trn8_32 + } + +#undef dct_long_mul +#undef dct_long_mac +#undef dct_widen +#undef dct_wadd +#undef dct_wsub +#undef dct_bfly32o +#undef dct_pass +} + +#endif // STBI_NEON + +#define STBI__MARKER_none 0xff +// if there's a pending marker from the entropy stream, return that +// otherwise, fetch from the stream and get a marker. if there's no +// marker, return 0xff, which is never a valid marker value +static stbi_uc stbi__get_marker(stbi__jpeg *j) +{ + stbi_uc x; + if (j->marker != STBI__MARKER_none) { x = j->marker; j->marker = STBI__MARKER_none; return x; } + x = stbi__get8(j->s); + if (x != 0xff) return STBI__MARKER_none; + while (x == 0xff) + x = stbi__get8(j->s); // consume repeated 0xff fill bytes + return x; +} + +// in each scan, we'll have scan_n components, and the order +// of the components is specified by order[] +#define STBI__RESTART(x) ((x) >= 0xd0 && (x) <= 0xd7) + +// after a restart interval, stbi__jpeg_reset the entropy decoder and +// the dc prediction +static void stbi__jpeg_reset(stbi__jpeg *j) +{ + j->code_bits = 0; + j->code_buffer = 0; + j->nomore = 0; + j->img_comp[0].dc_pred = j->img_comp[1].dc_pred = j->img_comp[2].dc_pred = j->img_comp[3].dc_pred = 0; + j->marker = STBI__MARKER_none; + j->todo = j->restart_interval ? j->restart_interval : 0x7fffffff; + j->eob_run = 0; + // no more than 1<<31 MCUs if no restart_interal? that's plenty safe, + // since we don't even allow 1<<30 pixels +} + +static int stbi__parse_entropy_coded_data(stbi__jpeg *z) +{ + stbi__jpeg_reset(z); + if (!z->progressive) { + if (z->scan_n == 1) { + int i,j; + STBI_SIMD_ALIGN(short, data[64]); + int n = z->order[0]; + // non-interleaved data, we just need to process one block at a time, + // in trivial scanline order + // number of blocks to do just depends on how many actual "pixels" this + // component has, independent of interleaved MCU blocking and such + int w = (z->img_comp[n].x+7) >> 3; + int h = (z->img_comp[n].y+7) >> 3; + for (j=0; j < h; ++j) { + for (i=0; i < w; ++i) { + int ha = z->img_comp[n].ha; + if (!stbi__jpeg_decode_block(z, data, z->huff_dc+z->img_comp[n].hd, z->huff_ac+ha, z->fast_ac[ha], n, z->dequant[z->img_comp[n].tq])) return 0; + z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*j*8+i*8, z->img_comp[n].w2, data); + // every data block is an MCU, so countdown the restart interval + if (--z->todo <= 0) { + if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); + // if it's NOT a restart, then just bail, so we get corrupt data + // rather than no data + if (!STBI__RESTART(z->marker)) return 1; + stbi__jpeg_reset(z); + } + } + } + return 1; + } else { // interleaved + int i,j,k,x,y; + STBI_SIMD_ALIGN(short, data[64]); + for (j=0; j < z->img_mcu_y; ++j) { + for (i=0; i < z->img_mcu_x; ++i) { + // scan an interleaved mcu... process scan_n components in order + for (k=0; k < z->scan_n; ++k) { + int n = z->order[k]; + // scan out an mcu's worth of this component; that's just determined + // by the basic H and V specified for the component + for (y=0; y < z->img_comp[n].v; ++y) { + for (x=0; x < z->img_comp[n].h; ++x) { + int x2 = (i*z->img_comp[n].h + x)*8; + int y2 = (j*z->img_comp[n].v + y)*8; + int ha = z->img_comp[n].ha; + if (!stbi__jpeg_decode_block(z, data, z->huff_dc+z->img_comp[n].hd, z->huff_ac+ha, z->fast_ac[ha], n, z->dequant[z->img_comp[n].tq])) return 0; + z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*y2+x2, z->img_comp[n].w2, data); + } + } + } + // after all interleaved components, that's an interleaved MCU, + // so now count down the restart interval + if (--z->todo <= 0) { + if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); + if (!STBI__RESTART(z->marker)) return 1; + stbi__jpeg_reset(z); + } + } + } + return 1; + } + } else { + if (z->scan_n == 1) { + int i,j; + int n = z->order[0]; + // non-interleaved data, we just need to process one block at a time, + // in trivial scanline order + // number of blocks to do just depends on how many actual "pixels" this + // component has, independent of interleaved MCU blocking and such + int w = (z->img_comp[n].x+7) >> 3; + int h = (z->img_comp[n].y+7) >> 3; + for (j=0; j < h; ++j) { + for (i=0; i < w; ++i) { + short *data = z->img_comp[n].coeff + 64 * (i + j * z->img_comp[n].coeff_w); + if (z->spec_start == 0) { + if (!stbi__jpeg_decode_block_prog_dc(z, data, &z->huff_dc[z->img_comp[n].hd], n)) + return 0; + } else { + int ha = z->img_comp[n].ha; + if (!stbi__jpeg_decode_block_prog_ac(z, data, &z->huff_ac[ha], z->fast_ac[ha])) + return 0; + } + // every data block is an MCU, so countdown the restart interval + if (--z->todo <= 0) { + if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); + if (!STBI__RESTART(z->marker)) return 1; + stbi__jpeg_reset(z); + } + } + } + return 1; + } else { // interleaved + int i,j,k,x,y; + for (j=0; j < z->img_mcu_y; ++j) { + for (i=0; i < z->img_mcu_x; ++i) { + // scan an interleaved mcu... process scan_n components in order + for (k=0; k < z->scan_n; ++k) { + int n = z->order[k]; + // scan out an mcu's worth of this component; that's just determined + // by the basic H and V specified for the component + for (y=0; y < z->img_comp[n].v; ++y) { + for (x=0; x < z->img_comp[n].h; ++x) { + int x2 = (i*z->img_comp[n].h + x); + int y2 = (j*z->img_comp[n].v + y); + short *data = z->img_comp[n].coeff + 64 * (x2 + y2 * z->img_comp[n].coeff_w); + if (!stbi__jpeg_decode_block_prog_dc(z, data, &z->huff_dc[z->img_comp[n].hd], n)) + return 0; + } + } + } + // after all interleaved components, that's an interleaved MCU, + // so now count down the restart interval + if (--z->todo <= 0) { + if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); + if (!STBI__RESTART(z->marker)) return 1; + stbi__jpeg_reset(z); + } + } + } + return 1; + } + } +} + +static void stbi__jpeg_dequantize(short *data, stbi__uint16 *dequant) +{ + int i; + for (i=0; i < 64; ++i) + data[i] *= dequant[i]; +} + +static void stbi__jpeg_finish(stbi__jpeg *z) +{ + if (z->progressive) { + // dequantize and idct the data + int i,j,n; + for (n=0; n < z->s->img_n; ++n) { + int w = (z->img_comp[n].x+7) >> 3; + int h = (z->img_comp[n].y+7) >> 3; + for (j=0; j < h; ++j) { + for (i=0; i < w; ++i) { + short *data = z->img_comp[n].coeff + 64 * (i + j * z->img_comp[n].coeff_w); + stbi__jpeg_dequantize(data, z->dequant[z->img_comp[n].tq]); + z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*j*8+i*8, z->img_comp[n].w2, data); + } + } + } + } +} + +static int stbi__process_marker(stbi__jpeg *z, int m) +{ + int L; + switch (m) { + case STBI__MARKER_none: // no marker found + return stbi__err("expected marker","Corrupt JPEG"); + + case 0xDD: // DRI - specify restart interval + if (stbi__get16be(z->s) != 4) return stbi__err("bad DRI len","Corrupt JPEG"); + z->restart_interval = stbi__get16be(z->s); + return 1; + + case 0xDB: // DQT - define quantization table + L = stbi__get16be(z->s)-2; + while (L > 0) { + int q = stbi__get8(z->s); + int p = q >> 4, sixteen = (p != 0); + int t = q & 15,i; + if (p != 0 && p != 1) return stbi__err("bad DQT type","Corrupt JPEG"); + if (t > 3) return stbi__err("bad DQT table","Corrupt JPEG"); + + for (i=0; i < 64; ++i) + z->dequant[t][stbi__jpeg_dezigzag[i]] = (stbi__uint16)(sixteen ? stbi__get16be(z->s) : stbi__get8(z->s)); + L -= (sixteen ? 129 : 65); + } + return L==0; + + case 0xC4: // DHT - define huffman table + L = stbi__get16be(z->s)-2; + while (L > 0) { + stbi_uc *v; + int sizes[16],i,n=0; + int q = stbi__get8(z->s); + int tc = q >> 4; + int th = q & 15; + if (tc > 1 || th > 3) return stbi__err("bad DHT header","Corrupt JPEG"); + for (i=0; i < 16; ++i) { + sizes[i] = stbi__get8(z->s); + n += sizes[i]; + } + L -= 17; + if (tc == 0) { + if (!stbi__build_huffman(z->huff_dc+th, sizes)) return 0; + v = z->huff_dc[th].values; + } else { + if (!stbi__build_huffman(z->huff_ac+th, sizes)) return 0; + v = z->huff_ac[th].values; + } + for (i=0; i < n; ++i) + v[i] = stbi__get8(z->s); + if (tc != 0) + stbi__build_fast_ac(z->fast_ac[th], z->huff_ac + th); + L -= n; + } + return L==0; + } + + // check for comment block or APP blocks + if ((m >= 0xE0 && m <= 0xEF) || m == 0xFE) { + L = stbi__get16be(z->s); + if (L < 2) { + if (m == 0xFE) + return stbi__err("bad COM len","Corrupt JPEG"); + else + return stbi__err("bad APP len","Corrupt JPEG"); + } + L -= 2; + + if (m == 0xE0 && L >= 5) { // JFIF APP0 segment + static const unsigned char tag[5] = {'J','F','I','F','\0'}; + int ok = 1; + int i; + for (i=0; i < 5; ++i) + if (stbi__get8(z->s) != tag[i]) + ok = 0; + L -= 5; + if (ok) + z->jfif = 1; + } else if (m == 0xEE && L >= 12) { // Adobe APP14 segment + static const unsigned char tag[6] = {'A','d','o','b','e','\0'}; + int ok = 1; + int i; + for (i=0; i < 6; ++i) + if (stbi__get8(z->s) != tag[i]) + ok = 0; + L -= 6; + if (ok) { + stbi__get8(z->s); // version + stbi__get16be(z->s); // flags0 + stbi__get16be(z->s); // flags1 + z->app14_color_transform = stbi__get8(z->s); // color transform + L -= 6; + } + } + + stbi__skip(z->s, L); + return 1; + } + + return stbi__err("unknown marker","Corrupt JPEG"); +} + +// after we see SOS +static int stbi__process_scan_header(stbi__jpeg *z) +{ + int i; + int Ls = stbi__get16be(z->s); + z->scan_n = stbi__get8(z->s); + if (z->scan_n < 1 || z->scan_n > 4 || z->scan_n > (int) z->s->img_n) return stbi__err("bad SOS component count","Corrupt JPEG"); + if (Ls != 6+2*z->scan_n) return stbi__err("bad SOS len","Corrupt JPEG"); + for (i=0; i < z->scan_n; ++i) { + int id = stbi__get8(z->s), which; + int q = stbi__get8(z->s); + for (which = 0; which < z->s->img_n; ++which) + if (z->img_comp[which].id == id) + break; + if (which == z->s->img_n) return 0; // no match + z->img_comp[which].hd = q >> 4; if (z->img_comp[which].hd > 3) return stbi__err("bad DC huff","Corrupt JPEG"); + z->img_comp[which].ha = q & 15; if (z->img_comp[which].ha > 3) return stbi__err("bad AC huff","Corrupt JPEG"); + z->order[i] = which; + } + + { + int aa; + z->spec_start = stbi__get8(z->s); + z->spec_end = stbi__get8(z->s); // should be 63, but might be 0 + aa = stbi__get8(z->s); + z->succ_high = (aa >> 4); + z->succ_low = (aa & 15); + if (z->progressive) { + if (z->spec_start > 63 || z->spec_end > 63 || z->spec_start > z->spec_end || z->succ_high > 13 || z->succ_low > 13) + return stbi__err("bad SOS", "Corrupt JPEG"); + } else { + if (z->spec_start != 0) return stbi__err("bad SOS","Corrupt JPEG"); + if (z->succ_high != 0 || z->succ_low != 0) return stbi__err("bad SOS","Corrupt JPEG"); + z->spec_end = 63; + } + } + + return 1; +} + +static int stbi__free_jpeg_components(stbi__jpeg *z, int ncomp, int why) +{ + int i; + for (i=0; i < ncomp; ++i) { + if (z->img_comp[i].raw_data) { + STBI_FREE(z->img_comp[i].raw_data); + z->img_comp[i].raw_data = NULL; + z->img_comp[i].data = NULL; + } + if (z->img_comp[i].raw_coeff) { + STBI_FREE(z->img_comp[i].raw_coeff); + z->img_comp[i].raw_coeff = 0; + z->img_comp[i].coeff = 0; + } + if (z->img_comp[i].linebuf) { + STBI_FREE(z->img_comp[i].linebuf); + z->img_comp[i].linebuf = NULL; + } + } + return why; +} + +static int stbi__process_frame_header(stbi__jpeg *z, int scan) +{ + stbi__context *s = z->s; + int Lf,p,i,q, h_max=1,v_max=1,c; + Lf = stbi__get16be(s); if (Lf < 11) return stbi__err("bad SOF len","Corrupt JPEG"); // JPEG + p = stbi__get8(s); if (p != 8) return stbi__err("only 8-bit","JPEG format not supported: 8-bit only"); // JPEG baseline + s->img_y = stbi__get16be(s); if (s->img_y == 0) return stbi__err("no header height", "JPEG format not supported: delayed height"); // Legal, but we don't handle it--but neither does IJG + s->img_x = stbi__get16be(s); if (s->img_x == 0) return stbi__err("0 width","Corrupt JPEG"); // JPEG requires + c = stbi__get8(s); + if (c != 3 && c != 1 && c != 4) return stbi__err("bad component count","Corrupt JPEG"); + s->img_n = c; + for (i=0; i < c; ++i) { + z->img_comp[i].data = NULL; + z->img_comp[i].linebuf = NULL; + } + + if (Lf != 8+3*s->img_n) return stbi__err("bad SOF len","Corrupt JPEG"); + + z->rgb = 0; + for (i=0; i < s->img_n; ++i) { + static const unsigned char rgb[3] = { 'R', 'G', 'B' }; + z->img_comp[i].id = stbi__get8(s); + if (s->img_n == 3 && z->img_comp[i].id == rgb[i]) + ++z->rgb; + q = stbi__get8(s); + z->img_comp[i].h = (q >> 4); if (!z->img_comp[i].h || z->img_comp[i].h > 4) return stbi__err("bad H","Corrupt JPEG"); + z->img_comp[i].v = q & 15; if (!z->img_comp[i].v || z->img_comp[i].v > 4) return stbi__err("bad V","Corrupt JPEG"); + z->img_comp[i].tq = stbi__get8(s); if (z->img_comp[i].tq > 3) return stbi__err("bad TQ","Corrupt JPEG"); + } + + if (scan != STBI__SCAN_load) return 1; + + if (!stbi__mad3sizes_valid(s->img_x, s->img_y, s->img_n, 0)) return stbi__err("too large", "Image too large to decode"); + + for (i=0; i < s->img_n; ++i) { + if (z->img_comp[i].h > h_max) h_max = z->img_comp[i].h; + if (z->img_comp[i].v > v_max) v_max = z->img_comp[i].v; + } + + // compute interleaved mcu info + z->img_h_max = h_max; + z->img_v_max = v_max; + z->img_mcu_w = h_max * 8; + z->img_mcu_h = v_max * 8; + // these sizes can't be more than 17 bits + z->img_mcu_x = (s->img_x + z->img_mcu_w-1) / z->img_mcu_w; + z->img_mcu_y = (s->img_y + z->img_mcu_h-1) / z->img_mcu_h; + + for (i=0; i < s->img_n; ++i) { + // number of effective pixels (e.g. for non-interleaved MCU) + z->img_comp[i].x = (s->img_x * z->img_comp[i].h + h_max-1) / h_max; + z->img_comp[i].y = (s->img_y * z->img_comp[i].v + v_max-1) / v_max; + // to simplify generation, we'll allocate enough memory to decode + // the bogus oversized data from using interleaved MCUs and their + // big blocks (e.g. a 16x16 iMCU on an image of width 33); we won't + // discard the extra data until colorspace conversion + // + // img_mcu_x, img_mcu_y: <=17 bits; comp[i].h and .v are <=4 (checked earlier) + // so these muls can't overflow with 32-bit ints (which we require) + z->img_comp[i].w2 = z->img_mcu_x * z->img_comp[i].h * 8; + z->img_comp[i].h2 = z->img_mcu_y * z->img_comp[i].v * 8; + z->img_comp[i].coeff = 0; + z->img_comp[i].raw_coeff = 0; + z->img_comp[i].linebuf = NULL; + z->img_comp[i].raw_data = stbi__malloc_mad2(z->img_comp[i].w2, z->img_comp[i].h2, 15); + if (z->img_comp[i].raw_data == NULL) + return stbi__free_jpeg_components(z, i+1, stbi__err("outofmem", "Out of memory")); + // align blocks for idct using mmx/sse + z->img_comp[i].data = (stbi_uc*) (((size_t) z->img_comp[i].raw_data + 15) & ~15); + if (z->progressive) { + // w2, h2 are multiples of 8 (see above) + z->img_comp[i].coeff_w = z->img_comp[i].w2 / 8; + z->img_comp[i].coeff_h = z->img_comp[i].h2 / 8; + z->img_comp[i].raw_coeff = stbi__malloc_mad3(z->img_comp[i].w2, z->img_comp[i].h2, sizeof(short), 15); + if (z->img_comp[i].raw_coeff == NULL) + return stbi__free_jpeg_components(z, i+1, stbi__err("outofmem", "Out of memory")); + z->img_comp[i].coeff = (short*) (((size_t) z->img_comp[i].raw_coeff + 15) & ~15); + } + } + + return 1; +} + +// use comparisons since in some cases we handle more than one case (e.g. SOF) +#define stbi__DNL(x) ((x) == 0xdc) +#define stbi__SOI(x) ((x) == 0xd8) +#define stbi__EOI(x) ((x) == 0xd9) +#define stbi__SOF(x) ((x) == 0xc0 || (x) == 0xc1 || (x) == 0xc2) +#define stbi__SOS(x) ((x) == 0xda) + +#define stbi__SOF_progressive(x) ((x) == 0xc2) + +static int stbi__decode_jpeg_header(stbi__jpeg *z, int scan) +{ + int m; + z->jfif = 0; + z->app14_color_transform = -1; // valid values are 0,1,2 + z->marker = STBI__MARKER_none; // initialize cached marker to empty + m = stbi__get_marker(z); + if (!stbi__SOI(m)) return stbi__err("no SOI","Corrupt JPEG"); + if (scan == STBI__SCAN_type) return 1; + m = stbi__get_marker(z); + while (!stbi__SOF(m)) { + if (!stbi__process_marker(z,m)) return 0; + m = stbi__get_marker(z); + while (m == STBI__MARKER_none) { + // some files have extra padding after their blocks, so ok, we'll scan + if (stbi__at_eof(z->s)) return stbi__err("no SOF", "Corrupt JPEG"); + m = stbi__get_marker(z); + } + } + z->progressive = stbi__SOF_progressive(m); + if (!stbi__process_frame_header(z, scan)) return 0; + return 1; +} + +// decode image to YCbCr format +static int stbi__decode_jpeg_image(stbi__jpeg *j) +{ + int m; + for (m = 0; m < 4; m++) { + j->img_comp[m].raw_data = NULL; + j->img_comp[m].raw_coeff = NULL; + } + j->restart_interval = 0; + if (!stbi__decode_jpeg_header(j, STBI__SCAN_load)) return 0; + m = stbi__get_marker(j); + while (!stbi__EOI(m)) { + if (stbi__SOS(m)) { + if (!stbi__process_scan_header(j)) return 0; + if (!stbi__parse_entropy_coded_data(j)) return 0; + if (j->marker == STBI__MARKER_none ) { + // handle 0s at the end of image data from IP Kamera 9060 + while (!stbi__at_eof(j->s)) { + int x = stbi__get8(j->s); + if (x == 255) { + j->marker = stbi__get8(j->s); + break; + } + } + // if we reach eof without hitting a marker, stbi__get_marker() below will fail and we'll eventually return 0 + } + } else if (stbi__DNL(m)) { + int Ld = stbi__get16be(j->s); + stbi__uint32 NL = stbi__get16be(j->s); + if (Ld != 4) return stbi__err("bad DNL len", "Corrupt JPEG"); + if (NL != j->s->img_y) return stbi__err("bad DNL height", "Corrupt JPEG"); + } else { + if (!stbi__process_marker(j, m)) return 0; + } + m = stbi__get_marker(j); + } + if (j->progressive) + stbi__jpeg_finish(j); + return 1; +} + +// static jfif-centered resampling (across block boundaries) + +typedef stbi_uc *(*resample_row_func)(stbi_uc *out, stbi_uc *in0, stbi_uc *in1, + int w, int hs); + +#define stbi__div4(x) ((stbi_uc) ((x) >> 2)) + +static stbi_uc *resample_row_1(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + STBI_NOTUSED(out); + STBI_NOTUSED(in_far); + STBI_NOTUSED(w); + STBI_NOTUSED(hs); + return in_near; +} + +static stbi_uc* stbi__resample_row_v_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // need to generate two samples vertically for every one in input + int i; + STBI_NOTUSED(hs); + for (i=0; i < w; ++i) + out[i] = stbi__div4(3*in_near[i] + in_far[i] + 2); + return out; +} + +static stbi_uc* stbi__resample_row_h_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // need to generate two samples horizontally for every one in input + int i; + stbi_uc *input = in_near; + + if (w == 1) { + // if only one sample, can't do any interpolation + out[0] = out[1] = input[0]; + return out; + } + + out[0] = input[0]; + out[1] = stbi__div4(input[0]*3 + input[1] + 2); + for (i=1; i < w-1; ++i) { + int n = 3*input[i]+2; + out[i*2+0] = stbi__div4(n+input[i-1]); + out[i*2+1] = stbi__div4(n+input[i+1]); + } + out[i*2+0] = stbi__div4(input[w-2]*3 + input[w-1] + 2); + out[i*2+1] = input[w-1]; + + STBI_NOTUSED(in_far); + STBI_NOTUSED(hs); + + return out; +} + +#define stbi__div16(x) ((stbi_uc) ((x) >> 4)) + +static stbi_uc *stbi__resample_row_hv_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // need to generate 2x2 samples for every one in input + int i,t0,t1; + if (w == 1) { + out[0] = out[1] = stbi__div4(3*in_near[0] + in_far[0] + 2); + return out; + } + + t1 = 3*in_near[0] + in_far[0]; + out[0] = stbi__div4(t1+2); + for (i=1; i < w; ++i) { + t0 = t1; + t1 = 3*in_near[i]+in_far[i]; + out[i*2-1] = stbi__div16(3*t0 + t1 + 8); + out[i*2 ] = stbi__div16(3*t1 + t0 + 8); + } + out[w*2-1] = stbi__div4(t1+2); + + STBI_NOTUSED(hs); + + return out; +} + +#if defined(STBI_SSE2) || defined(STBI_NEON) +static stbi_uc *stbi__resample_row_hv_2_simd(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // need to generate 2x2 samples for every one in input + int i=0,t0,t1; + + if (w == 1) { + out[0] = out[1] = stbi__div4(3*in_near[0] + in_far[0] + 2); + return out; + } + + t1 = 3*in_near[0] + in_far[0]; + // process groups of 8 pixels for as long as we can. + // note we can't handle the last pixel in a row in this loop + // because we need to handle the filter boundary conditions. + for (; i < ((w-1) & ~7); i += 8) { +#if defined(STBI_SSE2) + // load and perform the vertical filtering pass + // this uses 3*x + y = 4*x + (y - x) + __m128i zero = _mm_setzero_si128(); + __m128i farb = _mm_loadl_epi64((__m128i *) (in_far + i)); + __m128i nearb = _mm_loadl_epi64((__m128i *) (in_near + i)); + __m128i farw = _mm_unpacklo_epi8(farb, zero); + __m128i nearw = _mm_unpacklo_epi8(nearb, zero); + __m128i diff = _mm_sub_epi16(farw, nearw); + __m128i nears = _mm_slli_epi16(nearw, 2); + __m128i curr = _mm_add_epi16(nears, diff); // current row + + // horizontal filter works the same based on shifted vers of current + // row. "prev" is current row shifted right by 1 pixel; we need to + // insert the previous pixel value (from t1). + // "next" is current row shifted left by 1 pixel, with first pixel + // of next block of 8 pixels added in. + __m128i prv0 = _mm_slli_si128(curr, 2); + __m128i nxt0 = _mm_srli_si128(curr, 2); + __m128i prev = _mm_insert_epi16(prv0, t1, 0); + __m128i next = _mm_insert_epi16(nxt0, 3*in_near[i+8] + in_far[i+8], 7); + + // horizontal filter, polyphase implementation since it's convenient: + // even pixels = 3*cur + prev = cur*4 + (prev - cur) + // odd pixels = 3*cur + next = cur*4 + (next - cur) + // note the shared term. + __m128i bias = _mm_set1_epi16(8); + __m128i curs = _mm_slli_epi16(curr, 2); + __m128i prvd = _mm_sub_epi16(prev, curr); + __m128i nxtd = _mm_sub_epi16(next, curr); + __m128i curb = _mm_add_epi16(curs, bias); + __m128i even = _mm_add_epi16(prvd, curb); + __m128i odd = _mm_add_epi16(nxtd, curb); + + // interleave even and odd pixels, then undo scaling. + __m128i int0 = _mm_unpacklo_epi16(even, odd); + __m128i int1 = _mm_unpackhi_epi16(even, odd); + __m128i de0 = _mm_srli_epi16(int0, 4); + __m128i de1 = _mm_srli_epi16(int1, 4); + + // pack and write output + __m128i outv = _mm_packus_epi16(de0, de1); + _mm_storeu_si128((__m128i *) (out + i*2), outv); +#elif defined(STBI_NEON) + // load and perform the vertical filtering pass + // this uses 3*x + y = 4*x + (y - x) + uint8x8_t farb = vld1_u8(in_far + i); + uint8x8_t nearb = vld1_u8(in_near + i); + int16x8_t diff = vreinterpretq_s16_u16(vsubl_u8(farb, nearb)); + int16x8_t nears = vreinterpretq_s16_u16(vshll_n_u8(nearb, 2)); + int16x8_t curr = vaddq_s16(nears, diff); // current row + + // horizontal filter works the same based on shifted vers of current + // row. "prev" is current row shifted right by 1 pixel; we need to + // insert the previous pixel value (from t1). + // "next" is current row shifted left by 1 pixel, with first pixel + // of next block of 8 pixels added in. + int16x8_t prv0 = vextq_s16(curr, curr, 7); + int16x8_t nxt0 = vextq_s16(curr, curr, 1); + int16x8_t prev = vsetq_lane_s16(t1, prv0, 0); + int16x8_t next = vsetq_lane_s16(3*in_near[i+8] + in_far[i+8], nxt0, 7); + + // horizontal filter, polyphase implementation since it's convenient: + // even pixels = 3*cur + prev = cur*4 + (prev - cur) + // odd pixels = 3*cur + next = cur*4 + (next - cur) + // note the shared term. + int16x8_t curs = vshlq_n_s16(curr, 2); + int16x8_t prvd = vsubq_s16(prev, curr); + int16x8_t nxtd = vsubq_s16(next, curr); + int16x8_t even = vaddq_s16(curs, prvd); + int16x8_t odd = vaddq_s16(curs, nxtd); + + // undo scaling and round, then store with even/odd phases interleaved + uint8x8x2_t o; + o.val[0] = vqrshrun_n_s16(even, 4); + o.val[1] = vqrshrun_n_s16(odd, 4); + vst2_u8(out + i*2, o); +#endif + + // "previous" value for next iter + t1 = 3*in_near[i+7] + in_far[i+7]; + } + + t0 = t1; + t1 = 3*in_near[i] + in_far[i]; + out[i*2] = stbi__div16(3*t1 + t0 + 8); + + for (++i; i < w; ++i) { + t0 = t1; + t1 = 3*in_near[i]+in_far[i]; + out[i*2-1] = stbi__div16(3*t0 + t1 + 8); + out[i*2 ] = stbi__div16(3*t1 + t0 + 8); + } + out[w*2-1] = stbi__div4(t1+2); + + STBI_NOTUSED(hs); + + return out; +} +#endif + +static stbi_uc *stbi__resample_row_generic(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) +{ + // resample with nearest-neighbor + int i,j; + STBI_NOTUSED(in_far); + for (i=0; i < w; ++i) + for (j=0; j < hs; ++j) + out[i*hs+j] = in_near[i]; + return out; +} + +// this is a reduced-precision calculation of YCbCr-to-RGB introduced +// to make sure the code produces the same results in both SIMD and scalar +#define stbi__float2fixed(x) (((int) ((x) * 4096.0f + 0.5f)) << 8) +static void stbi__YCbCr_to_RGB_row(stbi_uc *out, const stbi_uc *y, const stbi_uc *pcb, const stbi_uc *pcr, int count, int step) +{ + int i; + for (i=0; i < count; ++i) { + int y_fixed = (y[i] << 20) + (1<<19); // rounding + int r,g,b; + int cr = pcr[i] - 128; + int cb = pcb[i] - 128; + r = y_fixed + cr* stbi__float2fixed(1.40200f); + g = y_fixed + (cr*-stbi__float2fixed(0.71414f)) + ((cb*-stbi__float2fixed(0.34414f)) & 0xffff0000); + b = y_fixed + cb* stbi__float2fixed(1.77200f); + r >>= 20; + g >>= 20; + b >>= 20; + if ((unsigned) r > 255) { if (r < 0) r = 0; else r = 255; } + if ((unsigned) g > 255) { if (g < 0) g = 0; else g = 255; } + if ((unsigned) b > 255) { if (b < 0) b = 0; else b = 255; } + out[0] = (stbi_uc)r; + out[1] = (stbi_uc)g; + out[2] = (stbi_uc)b; + out[3] = 255; + out += step; + } +} + +#if defined(STBI_SSE2) || defined(STBI_NEON) +static void stbi__YCbCr_to_RGB_simd(stbi_uc *out, stbi_uc const *y, stbi_uc const *pcb, stbi_uc const *pcr, int count, int step) +{ + int i = 0; + +#ifdef STBI_SSE2 + // step == 3 is pretty ugly on the final interleave, and i'm not convinced + // it's useful in practice (you wouldn't use it for textures, for example). + // so just accelerate step == 4 case. + if (step == 4) { + // this is a fairly straightforward implementation and not super-optimized. + __m128i signflip = _mm_set1_epi8(-0x80); + __m128i cr_const0 = _mm_set1_epi16( (short) ( 1.40200f*4096.0f+0.5f)); + __m128i cr_const1 = _mm_set1_epi16( - (short) ( 0.71414f*4096.0f+0.5f)); + __m128i cb_const0 = _mm_set1_epi16( - (short) ( 0.34414f*4096.0f+0.5f)); + __m128i cb_const1 = _mm_set1_epi16( (short) ( 1.77200f*4096.0f+0.5f)); + __m128i y_bias = _mm_set1_epi8((char) (unsigned char) 128); + __m128i xw = _mm_set1_epi16(255); // alpha channel + + for (; i+7 < count; i += 8) { + // load + __m128i y_bytes = _mm_loadl_epi64((__m128i *) (y+i)); + __m128i cr_bytes = _mm_loadl_epi64((__m128i *) (pcr+i)); + __m128i cb_bytes = _mm_loadl_epi64((__m128i *) (pcb+i)); + __m128i cr_biased = _mm_xor_si128(cr_bytes, signflip); // -128 + __m128i cb_biased = _mm_xor_si128(cb_bytes, signflip); // -128 + + // unpack to short (and left-shift cr, cb by 8) + __m128i yw = _mm_unpacklo_epi8(y_bias, y_bytes); + __m128i crw = _mm_unpacklo_epi8(_mm_setzero_si128(), cr_biased); + __m128i cbw = _mm_unpacklo_epi8(_mm_setzero_si128(), cb_biased); + + // color transform + __m128i yws = _mm_srli_epi16(yw, 4); + __m128i cr0 = _mm_mulhi_epi16(cr_const0, crw); + __m128i cb0 = _mm_mulhi_epi16(cb_const0, cbw); + __m128i cb1 = _mm_mulhi_epi16(cbw, cb_const1); + __m128i cr1 = _mm_mulhi_epi16(crw, cr_const1); + __m128i rws = _mm_add_epi16(cr0, yws); + __m128i gwt = _mm_add_epi16(cb0, yws); + __m128i bws = _mm_add_epi16(yws, cb1); + __m128i gws = _mm_add_epi16(gwt, cr1); + + // descale + __m128i rw = _mm_srai_epi16(rws, 4); + __m128i bw = _mm_srai_epi16(bws, 4); + __m128i gw = _mm_srai_epi16(gws, 4); + + // back to byte, set up for transpose + __m128i brb = _mm_packus_epi16(rw, bw); + __m128i gxb = _mm_packus_epi16(gw, xw); + + // transpose to interleave channels + __m128i t0 = _mm_unpacklo_epi8(brb, gxb); + __m128i t1 = _mm_unpackhi_epi8(brb, gxb); + __m128i o0 = _mm_unpacklo_epi16(t0, t1); + __m128i o1 = _mm_unpackhi_epi16(t0, t1); + + // store + _mm_storeu_si128((__m128i *) (out + 0), o0); + _mm_storeu_si128((__m128i *) (out + 16), o1); + out += 32; + } + } +#endif + +#ifdef STBI_NEON + // in this version, step=3 support would be easy to add. but is there demand? + if (step == 4) { + // this is a fairly straightforward implementation and not super-optimized. + uint8x8_t signflip = vdup_n_u8(0x80); + int16x8_t cr_const0 = vdupq_n_s16( (short) ( 1.40200f*4096.0f+0.5f)); + int16x8_t cr_const1 = vdupq_n_s16( - (short) ( 0.71414f*4096.0f+0.5f)); + int16x8_t cb_const0 = vdupq_n_s16( - (short) ( 0.34414f*4096.0f+0.5f)); + int16x8_t cb_const1 = vdupq_n_s16( (short) ( 1.77200f*4096.0f+0.5f)); + + for (; i+7 < count; i += 8) { + // load + uint8x8_t y_bytes = vld1_u8(y + i); + uint8x8_t cr_bytes = vld1_u8(pcr + i); + uint8x8_t cb_bytes = vld1_u8(pcb + i); + int8x8_t cr_biased = vreinterpret_s8_u8(vsub_u8(cr_bytes, signflip)); + int8x8_t cb_biased = vreinterpret_s8_u8(vsub_u8(cb_bytes, signflip)); + + // expand to s16 + int16x8_t yws = vreinterpretq_s16_u16(vshll_n_u8(y_bytes, 4)); + int16x8_t crw = vshll_n_s8(cr_biased, 7); + int16x8_t cbw = vshll_n_s8(cb_biased, 7); + + // color transform + int16x8_t cr0 = vqdmulhq_s16(crw, cr_const0); + int16x8_t cb0 = vqdmulhq_s16(cbw, cb_const0); + int16x8_t cr1 = vqdmulhq_s16(crw, cr_const1); + int16x8_t cb1 = vqdmulhq_s16(cbw, cb_const1); + int16x8_t rws = vaddq_s16(yws, cr0); + int16x8_t gws = vaddq_s16(vaddq_s16(yws, cb0), cr1); + int16x8_t bws = vaddq_s16(yws, cb1); + + // undo scaling, round, convert to byte + uint8x8x4_t o; + o.val[0] = vqrshrun_n_s16(rws, 4); + o.val[1] = vqrshrun_n_s16(gws, 4); + o.val[2] = vqrshrun_n_s16(bws, 4); + o.val[3] = vdup_n_u8(255); + + // store, interleaving r/g/b/a + vst4_u8(out, o); + out += 8*4; + } + } +#endif + + for (; i < count; ++i) { + int y_fixed = (y[i] << 20) + (1<<19); // rounding + int r,g,b; + int cr = pcr[i] - 128; + int cb = pcb[i] - 128; + r = y_fixed + cr* stbi__float2fixed(1.40200f); + g = y_fixed + cr*-stbi__float2fixed(0.71414f) + ((cb*-stbi__float2fixed(0.34414f)) & 0xffff0000); + b = y_fixed + cb* stbi__float2fixed(1.77200f); + r >>= 20; + g >>= 20; + b >>= 20; + if ((unsigned) r > 255) { if (r < 0) r = 0; else r = 255; } + if ((unsigned) g > 255) { if (g < 0) g = 0; else g = 255; } + if ((unsigned) b > 255) { if (b < 0) b = 0; else b = 255; } + out[0] = (stbi_uc)r; + out[1] = (stbi_uc)g; + out[2] = (stbi_uc)b; + out[3] = 255; + out += step; + } +} +#endif + +// set up the kernels +static void stbi__setup_jpeg(stbi__jpeg *j) +{ + j->idct_block_kernel = stbi__idct_block; + j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_row; + j->resample_row_hv_2_kernel = stbi__resample_row_hv_2; + +#ifdef STBI_SSE2 + if (stbi__sse2_available()) { + j->idct_block_kernel = stbi__idct_simd; + j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_simd; + j->resample_row_hv_2_kernel = stbi__resample_row_hv_2_simd; + } +#endif + +#ifdef STBI_NEON + j->idct_block_kernel = stbi__idct_simd; + j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_simd; + j->resample_row_hv_2_kernel = stbi__resample_row_hv_2_simd; +#endif +} + +// clean up the temporary component buffers +static void stbi__cleanup_jpeg(stbi__jpeg *j) +{ + stbi__free_jpeg_components(j, j->s->img_n, 0); +} + +typedef struct +{ + resample_row_func resample; + stbi_uc *line0,*line1; + int hs,vs; // expansion factor in each axis + int w_lores; // horizontal pixels pre-expansion + int ystep; // how far through vertical expansion we are + int ypos; // which pre-expansion row we're on +} stbi__resample; + +// fast 0..255 * 0..255 => 0..255 rounded multiplication +static stbi_uc stbi__blinn_8x8(stbi_uc x, stbi_uc y) +{ + unsigned int t = x*y + 128; + return (stbi_uc) ((t + (t >>8)) >> 8); +} + +static stbi_uc *load_jpeg_image(stbi__jpeg *z, int *out_x, int *out_y, int *comp, int req_comp) +{ + int n, decode_n, is_rgb; + z->s->img_n = 0; // make stbi__cleanup_jpeg safe + + // validate req_comp + if (req_comp < 0 || req_comp > 4) return stbi__errpuc("bad req_comp", "Internal error"); + + // load a jpeg image from whichever source, but leave in YCbCr format + if (!stbi__decode_jpeg_image(z)) { stbi__cleanup_jpeg(z); return NULL; } + + // determine actual number of components to generate + n = req_comp ? req_comp : z->s->img_n >= 3 ? 3 : 1; + + is_rgb = z->s->img_n == 3 && (z->rgb == 3 || (z->app14_color_transform == 0 && !z->jfif)); + + if (z->s->img_n == 3 && n < 3 && !is_rgb) + decode_n = 1; + else + decode_n = z->s->img_n; + + // resample and color-convert + { + int k; + unsigned int i,j; + stbi_uc *output; + stbi_uc *coutput[4]; + + stbi__resample res_comp[4]; + + for (k=0; k < decode_n; ++k) { + stbi__resample *r = &res_comp[k]; + + // allocate line buffer big enough for upsampling off the edges + // with upsample factor of 4 + z->img_comp[k].linebuf = (stbi_uc *) stbi__malloc(z->s->img_x + 3); + if (!z->img_comp[k].linebuf) { stbi__cleanup_jpeg(z); return stbi__errpuc("outofmem", "Out of memory"); } + + r->hs = z->img_h_max / z->img_comp[k].h; + r->vs = z->img_v_max / z->img_comp[k].v; + r->ystep = r->vs >> 1; + r->w_lores = (z->s->img_x + r->hs-1) / r->hs; + r->ypos = 0; + r->line0 = r->line1 = z->img_comp[k].data; + + if (r->hs == 1 && r->vs == 1) r->resample = resample_row_1; + else if (r->hs == 1 && r->vs == 2) r->resample = stbi__resample_row_v_2; + else if (r->hs == 2 && r->vs == 1) r->resample = stbi__resample_row_h_2; + else if (r->hs == 2 && r->vs == 2) r->resample = z->resample_row_hv_2_kernel; + else r->resample = stbi__resample_row_generic; + } + + // can't error after this so, this is safe + output = (stbi_uc *) stbi__malloc_mad3(n, z->s->img_x, z->s->img_y, 1); + if (!output) { stbi__cleanup_jpeg(z); return stbi__errpuc("outofmem", "Out of memory"); } + + // now go ahead and resample + for (j=0; j < z->s->img_y; ++j) { + stbi_uc *out = output + n * z->s->img_x * j; + for (k=0; k < decode_n; ++k) { + stbi__resample *r = &res_comp[k]; + int y_bot = r->ystep >= (r->vs >> 1); + coutput[k] = r->resample(z->img_comp[k].linebuf, + y_bot ? r->line1 : r->line0, + y_bot ? r->line0 : r->line1, + r->w_lores, r->hs); + if (++r->ystep >= r->vs) { + r->ystep = 0; + r->line0 = r->line1; + if (++r->ypos < z->img_comp[k].y) + r->line1 += z->img_comp[k].w2; + } + } + if (n >= 3) { + stbi_uc *y = coutput[0]; + if (z->s->img_n == 3) { + if (is_rgb) { + for (i=0; i < z->s->img_x; ++i) { + out[0] = y[i]; + out[1] = coutput[1][i]; + out[2] = coutput[2][i]; + out[3] = 255; + out += n; + } + } else { + z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n); + } + } else if (z->s->img_n == 4) { + if (z->app14_color_transform == 0) { // CMYK + for (i=0; i < z->s->img_x; ++i) { + stbi_uc m = coutput[3][i]; + out[0] = stbi__blinn_8x8(coutput[0][i], m); + out[1] = stbi__blinn_8x8(coutput[1][i], m); + out[2] = stbi__blinn_8x8(coutput[2][i], m); + out[3] = 255; + out += n; + } + } else if (z->app14_color_transform == 2) { // YCCK + z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n); + for (i=0; i < z->s->img_x; ++i) { + stbi_uc m = coutput[3][i]; + out[0] = stbi__blinn_8x8(255 - out[0], m); + out[1] = stbi__blinn_8x8(255 - out[1], m); + out[2] = stbi__blinn_8x8(255 - out[2], m); + out += n; + } + } else { // YCbCr + alpha? Ignore the fourth channel for now + z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n); + } + } else + for (i=0; i < z->s->img_x; ++i) { + out[0] = out[1] = out[2] = y[i]; + out[3] = 255; // not used if n==3 + out += n; + } + } else { + if (is_rgb) { + if (n == 1) + for (i=0; i < z->s->img_x; ++i) + *out++ = stbi__compute_y(coutput[0][i], coutput[1][i], coutput[2][i]); + else { + for (i=0; i < z->s->img_x; ++i, out += 2) { + out[0] = stbi__compute_y(coutput[0][i], coutput[1][i], coutput[2][i]); + out[1] = 255; + } + } + } else if (z->s->img_n == 4 && z->app14_color_transform == 0) { + for (i=0; i < z->s->img_x; ++i) { + stbi_uc m = coutput[3][i]; + stbi_uc r = stbi__blinn_8x8(coutput[0][i], m); + stbi_uc g = stbi__blinn_8x8(coutput[1][i], m); + stbi_uc b = stbi__blinn_8x8(coutput[2][i], m); + out[0] = stbi__compute_y(r, g, b); + out[1] = 255; + out += n; + } + } else if (z->s->img_n == 4 && z->app14_color_transform == 2) { + for (i=0; i < z->s->img_x; ++i) { + out[0] = stbi__blinn_8x8(255 - coutput[0][i], coutput[3][i]); + out[1] = 255; + out += n; + } + } else { + stbi_uc *y = coutput[0]; + if (n == 1) + for (i=0; i < z->s->img_x; ++i) out[i] = y[i]; + else + for (i=0; i < z->s->img_x; ++i) { *out++ = y[i]; *out++ = 255; } + } + } + } + stbi__cleanup_jpeg(z); + *out_x = z->s->img_x; + *out_y = z->s->img_y; + if (comp) *comp = z->s->img_n >= 3 ? 3 : 1; // report original components, not output + return output; + } +} + +static void *stbi__jpeg_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + unsigned char* result; + stbi__jpeg* j = (stbi__jpeg*) stbi__malloc(sizeof(stbi__jpeg)); + STBI_NOTUSED(ri); + j->s = s; + stbi__setup_jpeg(j); + result = load_jpeg_image(j, x,y,comp,req_comp); + STBI_FREE(j); + return result; +} + +static int stbi__jpeg_test(stbi__context *s) +{ + int r; + stbi__jpeg* j = (stbi__jpeg*)stbi__malloc(sizeof(stbi__jpeg)); + j->s = s; + stbi__setup_jpeg(j); + r = stbi__decode_jpeg_header(j, STBI__SCAN_type); + stbi__rewind(s); + STBI_FREE(j); + return r; +} + +static int stbi__jpeg_info_raw(stbi__jpeg *j, int *x, int *y, int *comp) +{ + if (!stbi__decode_jpeg_header(j, STBI__SCAN_header)) { + stbi__rewind( j->s ); + return 0; + } + if (x) *x = j->s->img_x; + if (y) *y = j->s->img_y; + if (comp) *comp = j->s->img_n >= 3 ? 3 : 1; + return 1; +} + +static int stbi__jpeg_info(stbi__context *s, int *x, int *y, int *comp) +{ + int result; + stbi__jpeg* j = (stbi__jpeg*) (stbi__malloc(sizeof(stbi__jpeg))); + j->s = s; + result = stbi__jpeg_info_raw(j, x, y, comp); + STBI_FREE(j); + return result; +} +#endif + +// public domain zlib decode v0.2 Sean Barrett 2006-11-18 +// simple implementation +// - all input must be provided in an upfront buffer +// - all output is written to a single output buffer (can malloc/realloc) +// performance +// - fast huffman + +#ifndef STBI_NO_ZLIB + +// fast-way is faster to check than jpeg huffman, but slow way is slower +#define STBI__ZFAST_BITS 9 // accelerate all cases in default tables +#define STBI__ZFAST_MASK ((1 << STBI__ZFAST_BITS) - 1) + +// zlib-style huffman encoding +// (jpegs packs from left, zlib from right, so can't share code) +typedef struct +{ + stbi__uint16 fast[1 << STBI__ZFAST_BITS]; + stbi__uint16 firstcode[16]; + int maxcode[17]; + stbi__uint16 firstsymbol[16]; + stbi_uc size[288]; + stbi__uint16 value[288]; +} stbi__zhuffman; + +stbi_inline static int stbi__bitreverse16(int n) +{ + n = ((n & 0xAAAA) >> 1) | ((n & 0x5555) << 1); + n = ((n & 0xCCCC) >> 2) | ((n & 0x3333) << 2); + n = ((n & 0xF0F0) >> 4) | ((n & 0x0F0F) << 4); + n = ((n & 0xFF00) >> 8) | ((n & 0x00FF) << 8); + return n; +} + +stbi_inline static int stbi__bit_reverse(int v, int bits) +{ + STBI_ASSERT(bits <= 16); + // to bit reverse n bits, reverse 16 and shift + // e.g. 11 bits, bit reverse and shift away 5 + return stbi__bitreverse16(v) >> (16-bits); +} + +static int stbi__zbuild_huffman(stbi__zhuffman *z, const stbi_uc *sizelist, int num) +{ + int i,k=0; + int code, next_code[16], sizes[17]; + + // DEFLATE spec for generating codes + memset(sizes, 0, sizeof(sizes)); + memset(z->fast, 0, sizeof(z->fast)); + for (i=0; i < num; ++i) + ++sizes[sizelist[i]]; + sizes[0] = 0; + for (i=1; i < 16; ++i) + if (sizes[i] > (1 << i)) + return stbi__err("bad sizes", "Corrupt PNG"); + code = 0; + for (i=1; i < 16; ++i) { + next_code[i] = code; + z->firstcode[i] = (stbi__uint16) code; + z->firstsymbol[i] = (stbi__uint16) k; + code = (code + sizes[i]); + if (sizes[i]) + if (code-1 >= (1 << i)) return stbi__err("bad codelengths","Corrupt PNG"); + z->maxcode[i] = code << (16-i); // preshift for inner loop + code <<= 1; + k += sizes[i]; + } + z->maxcode[16] = 0x10000; // sentinel + for (i=0; i < num; ++i) { + int s = sizelist[i]; + if (s) { + int c = next_code[s] - z->firstcode[s] + z->firstsymbol[s]; + stbi__uint16 fastv = (stbi__uint16) ((s << 9) | i); + z->size [c] = (stbi_uc ) s; + z->value[c] = (stbi__uint16) i; + if (s <= STBI__ZFAST_BITS) { + int j = stbi__bit_reverse(next_code[s],s); + while (j < (1 << STBI__ZFAST_BITS)) { + z->fast[j] = fastv; + j += (1 << s); + } + } + ++next_code[s]; + } + } + return 1; +} + +// zlib-from-memory implementation for PNG reading +// because PNG allows splitting the zlib stream arbitrarily, +// and it's annoying structurally to have PNG call ZLIB call PNG, +// we require PNG read all the IDATs and combine them into a single +// memory buffer + +typedef struct +{ + stbi_uc *zbuffer, *zbuffer_end; + int num_bits; + stbi__uint32 code_buffer; + + char *zout; + char *zout_start; + char *zout_end; + int z_expandable; + + stbi__zhuffman z_length, z_distance; +} stbi__zbuf; + +stbi_inline static stbi_uc stbi__zget8(stbi__zbuf *z) +{ + if (z->zbuffer >= z->zbuffer_end) return 0; + return *z->zbuffer++; +} + +static void stbi__fill_bits(stbi__zbuf *z) +{ + do { + STBI_ASSERT(z->code_buffer < (1U << z->num_bits)); + z->code_buffer |= (unsigned int) stbi__zget8(z) << z->num_bits; + z->num_bits += 8; + } while (z->num_bits <= 24); +} + +stbi_inline static unsigned int stbi__zreceive(stbi__zbuf *z, int n) +{ + unsigned int k; + if (z->num_bits < n) stbi__fill_bits(z); + k = z->code_buffer & ((1 << n) - 1); + z->code_buffer >>= n; + z->num_bits -= n; + return k; +} + +static int stbi__zhuffman_decode_slowpath(stbi__zbuf *a, stbi__zhuffman *z) +{ + int b,s,k; + // not resolved by fast table, so compute it the slow way + // use jpeg approach, which requires MSbits at top + k = stbi__bit_reverse(a->code_buffer, 16); + for (s=STBI__ZFAST_BITS+1; ; ++s) + if (k < z->maxcode[s]) + break; + if (s == 16) return -1; // invalid code! + // code size is s, so: + b = (k >> (16-s)) - z->firstcode[s] + z->firstsymbol[s]; + STBI_ASSERT(z->size[b] == s); + a->code_buffer >>= s; + a->num_bits -= s; + return z->value[b]; +} + +stbi_inline static int stbi__zhuffman_decode(stbi__zbuf *a, stbi__zhuffman *z) +{ + int b,s; + if (a->num_bits < 16) stbi__fill_bits(a); + b = z->fast[a->code_buffer & STBI__ZFAST_MASK]; + if (b) { + s = b >> 9; + a->code_buffer >>= s; + a->num_bits -= s; + return b & 511; + } + return stbi__zhuffman_decode_slowpath(a, z); +} + +static int stbi__zexpand(stbi__zbuf *z, char *zout, int n) // need to make room for n bytes +{ + char *q; + int cur, limit, old_limit; + z->zout = zout; + if (!z->z_expandable) return stbi__err("output buffer limit","Corrupt PNG"); + cur = (int) (z->zout - z->zout_start); + limit = old_limit = (int) (z->zout_end - z->zout_start); + while (cur + n > limit) + limit *= 2; + q = (char *) STBI_REALLOC_SIZED(z->zout_start, old_limit, limit); + STBI_NOTUSED(old_limit); + if (q == NULL) return stbi__err("outofmem", "Out of memory"); + z->zout_start = q; + z->zout = q + cur; + z->zout_end = q + limit; + return 1; +} + +static const int stbi__zlength_base[31] = { + 3,4,5,6,7,8,9,10,11,13, + 15,17,19,23,27,31,35,43,51,59, + 67,83,99,115,131,163,195,227,258,0,0 }; + +static const int stbi__zlength_extra[31]= +{ 0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0,0,0 }; + +static const int stbi__zdist_base[32] = { 1,2,3,4,5,7,9,13,17,25,33,49,65,97,129,193, +257,385,513,769,1025,1537,2049,3073,4097,6145,8193,12289,16385,24577,0,0}; + +static const int stbi__zdist_extra[32] = +{ 0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13}; + +static int stbi__parse_huffman_block(stbi__zbuf *a) +{ + char *zout = a->zout; + for(;;) { + int z = stbi__zhuffman_decode(a, &a->z_length); + if (z < 256) { + if (z < 0) return stbi__err("bad huffman code","Corrupt PNG"); // error in huffman codes + if (zout >= a->zout_end) { + if (!stbi__zexpand(a, zout, 1)) return 0; + zout = a->zout; + } + *zout++ = (char) z; + } else { + stbi_uc *p; + int len,dist; + if (z == 256) { + a->zout = zout; + return 1; + } + z -= 257; + len = stbi__zlength_base[z]; + if (stbi__zlength_extra[z]) len += stbi__zreceive(a, stbi__zlength_extra[z]); + z = stbi__zhuffman_decode(a, &a->z_distance); + if (z < 0) return stbi__err("bad huffman code","Corrupt PNG"); + dist = stbi__zdist_base[z]; + if (stbi__zdist_extra[z]) dist += stbi__zreceive(a, stbi__zdist_extra[z]); + if (zout - a->zout_start < dist) return stbi__err("bad dist","Corrupt PNG"); + if (zout + len > a->zout_end) { + if (!stbi__zexpand(a, zout, len)) return 0; + zout = a->zout; + } + p = (stbi_uc *) (zout - dist); + if (dist == 1) { // run of one byte; common in images. + stbi_uc v = *p; + if (len) { do *zout++ = v; while (--len); } + } else { + if (len) { do *zout++ = *p++; while (--len); } + } + } + } +} + +static int stbi__compute_huffman_codes(stbi__zbuf *a) +{ + static const stbi_uc length_dezigzag[19] = { 16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15 }; + stbi__zhuffman z_codelength; + stbi_uc lencodes[286+32+137];//padding for maximum single op + stbi_uc codelength_sizes[19]; + int i,n; + + int hlit = stbi__zreceive(a,5) + 257; + int hdist = stbi__zreceive(a,5) + 1; + int hclen = stbi__zreceive(a,4) + 4; + int ntot = hlit + hdist; + + memset(codelength_sizes, 0, sizeof(codelength_sizes)); + for (i=0; i < hclen; ++i) { + int s = stbi__zreceive(a,3); + codelength_sizes[length_dezigzag[i]] = (stbi_uc) s; + } + if (!stbi__zbuild_huffman(&z_codelength, codelength_sizes, 19)) return 0; + + n = 0; + while (n < ntot) { + int c = stbi__zhuffman_decode(a, &z_codelength); + if (c < 0 || c >= 19) return stbi__err("bad codelengths", "Corrupt PNG"); + if (c < 16) + lencodes[n++] = (stbi_uc) c; + else { + stbi_uc fill = 0; + if (c == 16) { + c = stbi__zreceive(a,2)+3; + if (n == 0) return stbi__err("bad codelengths", "Corrupt PNG"); + fill = lencodes[n-1]; + } else if (c == 17) + c = stbi__zreceive(a,3)+3; + else { + STBI_ASSERT(c == 18); + c = stbi__zreceive(a,7)+11; + } + if (ntot - n < c) return stbi__err("bad codelengths", "Corrupt PNG"); + memset(lencodes+n, fill, c); + n += c; + } + } + if (n != ntot) return stbi__err("bad codelengths","Corrupt PNG"); + if (!stbi__zbuild_huffman(&a->z_length, lencodes, hlit)) return 0; + if (!stbi__zbuild_huffman(&a->z_distance, lencodes+hlit, hdist)) return 0; + return 1; +} + +static int stbi__parse_uncompressed_block(stbi__zbuf *a) +{ + stbi_uc header[4]; + int len,nlen,k; + if (a->num_bits & 7) + stbi__zreceive(a, a->num_bits & 7); // discard + // drain the bit-packed data into header + k = 0; + while (a->num_bits > 0) { + header[k++] = (stbi_uc) (a->code_buffer & 255); // suppress MSVC run-time check + a->code_buffer >>= 8; + a->num_bits -= 8; + } + STBI_ASSERT(a->num_bits == 0); + // now fill header the normal way + while (k < 4) + header[k++] = stbi__zget8(a); + len = header[1] * 256 + header[0]; + nlen = header[3] * 256 + header[2]; + if (nlen != (len ^ 0xffff)) return stbi__err("zlib corrupt","Corrupt PNG"); + if (a->zbuffer + len > a->zbuffer_end) return stbi__err("read past buffer","Corrupt PNG"); + if (a->zout + len > a->zout_end) + if (!stbi__zexpand(a, a->zout, len)) return 0; + memcpy(a->zout, a->zbuffer, len); + a->zbuffer += len; + a->zout += len; + return 1; +} + +static int stbi__parse_zlib_header(stbi__zbuf *a) +{ + int cmf = stbi__zget8(a); + int cm = cmf & 15; + /* int cinfo = cmf >> 4; */ + int flg = stbi__zget8(a); + if ((cmf*256+flg) % 31 != 0) return stbi__err("bad zlib header","Corrupt PNG"); // zlib spec + if (flg & 32) return stbi__err("no preset dict","Corrupt PNG"); // preset dictionary not allowed in png + if (cm != 8) return stbi__err("bad compression","Corrupt PNG"); // DEFLATE required for png + // window = 1 << (8 + cinfo)... but who cares, we fully buffer output + return 1; +} + +static const stbi_uc stbi__zdefault_length[288] = +{ + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, + 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, + 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8 +}; +static const stbi_uc stbi__zdefault_distance[32] = +{ + 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 +}; +/* +Init algorithm: +{ + int i; // use <= to match clearly with spec + for (i=0; i <= 143; ++i) stbi__zdefault_length[i] = 8; + for ( ; i <= 255; ++i) stbi__zdefault_length[i] = 9; + for ( ; i <= 279; ++i) stbi__zdefault_length[i] = 7; + for ( ; i <= 287; ++i) stbi__zdefault_length[i] = 8; + + for (i=0; i <= 31; ++i) stbi__zdefault_distance[i] = 5; +} +*/ + +static int stbi__parse_zlib(stbi__zbuf *a, int parse_header) +{ + int final, type; + if (parse_header) + if (!stbi__parse_zlib_header(a)) return 0; + a->num_bits = 0; + a->code_buffer = 0; + do { + final = stbi__zreceive(a,1); + type = stbi__zreceive(a,2); + if (type == 0) { + if (!stbi__parse_uncompressed_block(a)) return 0; + } else if (type == 3) { + return 0; + } else { + if (type == 1) { + // use fixed code lengths + if (!stbi__zbuild_huffman(&a->z_length , stbi__zdefault_length , 288)) return 0; + if (!stbi__zbuild_huffman(&a->z_distance, stbi__zdefault_distance, 32)) return 0; + } else { + if (!stbi__compute_huffman_codes(a)) return 0; + } + if (!stbi__parse_huffman_block(a)) return 0; + } + } while (!final); + return 1; +} + +static int stbi__do_zlib(stbi__zbuf *a, char *obuf, int olen, int exp, int parse_header) +{ + a->zout_start = obuf; + a->zout = obuf; + a->zout_end = obuf + olen; + a->z_expandable = exp; + + return stbi__parse_zlib(a, parse_header); +} + +STBIDEF char *stbi_zlib_decode_malloc_guesssize(const char *buffer, int len, int initial_size, int *outlen) +{ + stbi__zbuf a; + char *p = (char *) stbi__malloc(initial_size); + if (p == NULL) return NULL; + a.zbuffer = (stbi_uc *) buffer; + a.zbuffer_end = (stbi_uc *) buffer + len; + if (stbi__do_zlib(&a, p, initial_size, 1, 1)) { + if (outlen) *outlen = (int) (a.zout - a.zout_start); + return a.zout_start; + } else { + STBI_FREE(a.zout_start); + return NULL; + } +} + +STBIDEF char *stbi_zlib_decode_malloc(char const *buffer, int len, int *outlen) +{ + return stbi_zlib_decode_malloc_guesssize(buffer, len, 16384, outlen); +} + +STBIDEF char *stbi_zlib_decode_malloc_guesssize_headerflag(const char *buffer, int len, int initial_size, int *outlen, int parse_header) +{ + stbi__zbuf a; + char *p = (char *) stbi__malloc(initial_size); + if (p == NULL) return NULL; + a.zbuffer = (stbi_uc *) buffer; + a.zbuffer_end = (stbi_uc *) buffer + len; + if (stbi__do_zlib(&a, p, initial_size, 1, parse_header)) { + if (outlen) *outlen = (int) (a.zout - a.zout_start); + return a.zout_start; + } else { + STBI_FREE(a.zout_start); + return NULL; + } +} + +STBIDEF int stbi_zlib_decode_buffer(char *obuffer, int olen, char const *ibuffer, int ilen) +{ + stbi__zbuf a; + a.zbuffer = (stbi_uc *) ibuffer; + a.zbuffer_end = (stbi_uc *) ibuffer + ilen; + if (stbi__do_zlib(&a, obuffer, olen, 0, 1)) + return (int) (a.zout - a.zout_start); + else + return -1; +} + +STBIDEF char *stbi_zlib_decode_noheader_malloc(char const *buffer, int len, int *outlen) +{ + stbi__zbuf a; + char *p = (char *) stbi__malloc(16384); + if (p == NULL) return NULL; + a.zbuffer = (stbi_uc *) buffer; + a.zbuffer_end = (stbi_uc *) buffer+len; + if (stbi__do_zlib(&a, p, 16384, 1, 0)) { + if (outlen) *outlen = (int) (a.zout - a.zout_start); + return a.zout_start; + } else { + STBI_FREE(a.zout_start); + return NULL; + } +} + +STBIDEF int stbi_zlib_decode_noheader_buffer(char *obuffer, int olen, const char *ibuffer, int ilen) +{ + stbi__zbuf a; + a.zbuffer = (stbi_uc *) ibuffer; + a.zbuffer_end = (stbi_uc *) ibuffer + ilen; + if (stbi__do_zlib(&a, obuffer, olen, 0, 0)) + return (int) (a.zout - a.zout_start); + else + return -1; +} +#endif + +// public domain "baseline" PNG decoder v0.10 Sean Barrett 2006-11-18 +// simple implementation +// - only 8-bit samples +// - no CRC checking +// - allocates lots of intermediate memory +// - avoids problem of streaming data between subsystems +// - avoids explicit window management +// performance +// - uses stb_zlib, a PD zlib implementation with fast huffman decoding + +#ifndef STBI_NO_PNG +typedef struct +{ + stbi__uint32 length; + stbi__uint32 type; +} stbi__pngchunk; + +static stbi__pngchunk stbi__get_chunk_header(stbi__context *s) +{ + stbi__pngchunk c; + c.length = stbi__get32be(s); + c.type = stbi__get32be(s); + return c; +} + +static int stbi__check_png_header(stbi__context *s) +{ + static const stbi_uc png_sig[8] = { 137,80,78,71,13,10,26,10 }; + int i; + for (i=0; i < 8; ++i) + if (stbi__get8(s) != png_sig[i]) return stbi__err("bad png sig","Not a PNG"); + return 1; +} + +typedef struct +{ + stbi__context *s; + stbi_uc *idata, *expanded, *out; + int depth; +} stbi__png; + + +enum { + STBI__F_none=0, + STBI__F_sub=1, + STBI__F_up=2, + STBI__F_avg=3, + STBI__F_paeth=4, + // synthetic filters used for first scanline to avoid needing a dummy row of 0s + STBI__F_avg_first, + STBI__F_paeth_first +}; + +static stbi_uc first_row_filter[5] = +{ + STBI__F_none, + STBI__F_sub, + STBI__F_none, + STBI__F_avg_first, + STBI__F_paeth_first +}; + +static int stbi__paeth(int a, int b, int c) +{ + int p = a + b - c; + int pa = abs(p-a); + int pb = abs(p-b); + int pc = abs(p-c); + if (pa <= pb && pa <= pc) return a; + if (pb <= pc) return b; + return c; +} + +static const stbi_uc stbi__depth_scale_table[9] = { 0, 0xff, 0x55, 0, 0x11, 0,0,0, 0x01 }; + +// create the png data from post-deflated data +static int stbi__create_png_image_raw(stbi__png *a, stbi_uc *raw, stbi__uint32 raw_len, int out_n, stbi__uint32 x, stbi__uint32 y, int depth, int color) +{ + int bytes = (depth == 16? 2 : 1); + stbi__context *s = a->s; + stbi__uint32 i,j,stride = x*out_n*bytes; + stbi__uint32 img_len, img_width_bytes; + int k; + int img_n = s->img_n; // copy it into a local for later + + int output_bytes = out_n*bytes; + int filter_bytes = img_n*bytes; + int width = x; + + STBI_ASSERT(out_n == s->img_n || out_n == s->img_n+1); + a->out = (stbi_uc *) stbi__malloc_mad3(x, y, output_bytes, 0); // extra bytes to write off the end into + if (!a->out) return stbi__err("outofmem", "Out of memory"); + + if (!stbi__mad3sizes_valid(img_n, x, depth, 7)) return stbi__err("too large", "Corrupt PNG"); + img_width_bytes = (((img_n * x * depth) + 7) >> 3); + img_len = (img_width_bytes + 1) * y; + + // we used to check for exact match between raw_len and img_len on non-interlaced PNGs, + // but issue #276 reported a PNG in the wild that had extra data at the end (all zeros), + // so just check for raw_len < img_len always. + if (raw_len < img_len) return stbi__err("not enough pixels","Corrupt PNG"); + + for (j=0; j < y; ++j) { + stbi_uc *cur = a->out + stride*j; + stbi_uc *prior; + int filter = *raw++; + + if (filter > 4) + return stbi__err("invalid filter","Corrupt PNG"); + + if (depth < 8) { + STBI_ASSERT(img_width_bytes <= x); + cur += x*out_n - img_width_bytes; // store output to the rightmost img_len bytes, so we can decode in place + filter_bytes = 1; + width = img_width_bytes; + } + prior = cur - stride; // bugfix: need to compute this after 'cur +=' computation above + + // if first row, use special filter that doesn't sample previous row + if (j == 0) filter = first_row_filter[filter]; + + // handle first byte explicitly + for (k=0; k < filter_bytes; ++k) { + switch (filter) { + case STBI__F_none : cur[k] = raw[k]; break; + case STBI__F_sub : cur[k] = raw[k]; break; + case STBI__F_up : cur[k] = STBI__BYTECAST(raw[k] + prior[k]); break; + case STBI__F_avg : cur[k] = STBI__BYTECAST(raw[k] + (prior[k]>>1)); break; + case STBI__F_paeth : cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(0,prior[k],0)); break; + case STBI__F_avg_first : cur[k] = raw[k]; break; + case STBI__F_paeth_first: cur[k] = raw[k]; break; + } + } + + if (depth == 8) { + if (img_n != out_n) + cur[img_n] = 255; // first pixel + raw += img_n; + cur += out_n; + prior += out_n; + } else if (depth == 16) { + if (img_n != out_n) { + cur[filter_bytes] = 255; // first pixel top byte + cur[filter_bytes+1] = 255; // first pixel bottom byte + } + raw += filter_bytes; + cur += output_bytes; + prior += output_bytes; + } else { + raw += 1; + cur += 1; + prior += 1; + } + + // this is a little gross, so that we don't switch per-pixel or per-component + if (depth < 8 || img_n == out_n) { + int nk = (width - 1)*filter_bytes; + #define STBI__CASE(f) \ + case f: \ + for (k=0; k < nk; ++k) + switch (filter) { + // "none" filter turns into a memcpy here; make that explicit. + case STBI__F_none: memcpy(cur, raw, nk); break; + STBI__CASE(STBI__F_sub) { cur[k] = STBI__BYTECAST(raw[k] + cur[k-filter_bytes]); } break; + STBI__CASE(STBI__F_up) { cur[k] = STBI__BYTECAST(raw[k] + prior[k]); } break; + STBI__CASE(STBI__F_avg) { cur[k] = STBI__BYTECAST(raw[k] + ((prior[k] + cur[k-filter_bytes])>>1)); } break; + STBI__CASE(STBI__F_paeth) { cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k-filter_bytes],prior[k],prior[k-filter_bytes])); } break; + STBI__CASE(STBI__F_avg_first) { cur[k] = STBI__BYTECAST(raw[k] + (cur[k-filter_bytes] >> 1)); } break; + STBI__CASE(STBI__F_paeth_first) { cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k-filter_bytes],0,0)); } break; + } + #undef STBI__CASE + raw += nk; + } else { + STBI_ASSERT(img_n+1 == out_n); + #define STBI__CASE(f) \ + case f: \ + for (i=x-1; i >= 1; --i, cur[filter_bytes]=255,raw+=filter_bytes,cur+=output_bytes,prior+=output_bytes) \ + for (k=0; k < filter_bytes; ++k) + switch (filter) { + STBI__CASE(STBI__F_none) { cur[k] = raw[k]; } break; + STBI__CASE(STBI__F_sub) { cur[k] = STBI__BYTECAST(raw[k] + cur[k- output_bytes]); } break; + STBI__CASE(STBI__F_up) { cur[k] = STBI__BYTECAST(raw[k] + prior[k]); } break; + STBI__CASE(STBI__F_avg) { cur[k] = STBI__BYTECAST(raw[k] + ((prior[k] + cur[k- output_bytes])>>1)); } break; + STBI__CASE(STBI__F_paeth) { cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k- output_bytes],prior[k],prior[k- output_bytes])); } break; + STBI__CASE(STBI__F_avg_first) { cur[k] = STBI__BYTECAST(raw[k] + (cur[k- output_bytes] >> 1)); } break; + STBI__CASE(STBI__F_paeth_first) { cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k- output_bytes],0,0)); } break; + } + #undef STBI__CASE + + // the loop above sets the high byte of the pixels' alpha, but for + // 16 bit png files we also need the low byte set. we'll do that here. + if (depth == 16) { + cur = a->out + stride*j; // start at the beginning of the row again + for (i=0; i < x; ++i,cur+=output_bytes) { + cur[filter_bytes+1] = 255; + } + } + } + } + + // we make a separate pass to expand bits to pixels; for performance, + // this could run two scanlines behind the above code, so it won't + // intefere with filtering but will still be in the cache. + if (depth < 8) { + for (j=0; j < y; ++j) { + stbi_uc *cur = a->out + stride*j; + stbi_uc *in = a->out + stride*j + x*out_n - img_width_bytes; + // unpack 1/2/4-bit into a 8-bit buffer. allows us to keep the common 8-bit path optimal at minimal cost for 1/2/4-bit + // png guarante byte alignment, if width is not multiple of 8/4/2 we'll decode dummy trailing data that will be skipped in the later loop + stbi_uc scale = (color == 0) ? stbi__depth_scale_table[depth] : 1; // scale grayscale values to 0..255 range + + // note that the final byte might overshoot and write more data than desired. + // we can allocate enough data that this never writes out of memory, but it + // could also overwrite the next scanline. can it overwrite non-empty data + // on the next scanline? yes, consider 1-pixel-wide scanlines with 1-bit-per-pixel. + // so we need to explicitly clamp the final ones + + if (depth == 4) { + for (k=x*img_n; k >= 2; k-=2, ++in) { + *cur++ = scale * ((*in >> 4) ); + *cur++ = scale * ((*in ) & 0x0f); + } + if (k > 0) *cur++ = scale * ((*in >> 4) ); + } else if (depth == 2) { + for (k=x*img_n; k >= 4; k-=4, ++in) { + *cur++ = scale * ((*in >> 6) ); + *cur++ = scale * ((*in >> 4) & 0x03); + *cur++ = scale * ((*in >> 2) & 0x03); + *cur++ = scale * ((*in ) & 0x03); + } + if (k > 0) *cur++ = scale * ((*in >> 6) ); + if (k > 1) *cur++ = scale * ((*in >> 4) & 0x03); + if (k > 2) *cur++ = scale * ((*in >> 2) & 0x03); + } else if (depth == 1) { + for (k=x*img_n; k >= 8; k-=8, ++in) { + *cur++ = scale * ((*in >> 7) ); + *cur++ = scale * ((*in >> 6) & 0x01); + *cur++ = scale * ((*in >> 5) & 0x01); + *cur++ = scale * ((*in >> 4) & 0x01); + *cur++ = scale * ((*in >> 3) & 0x01); + *cur++ = scale * ((*in >> 2) & 0x01); + *cur++ = scale * ((*in >> 1) & 0x01); + *cur++ = scale * ((*in ) & 0x01); + } + if (k > 0) *cur++ = scale * ((*in >> 7) ); + if (k > 1) *cur++ = scale * ((*in >> 6) & 0x01); + if (k > 2) *cur++ = scale * ((*in >> 5) & 0x01); + if (k > 3) *cur++ = scale * ((*in >> 4) & 0x01); + if (k > 4) *cur++ = scale * ((*in >> 3) & 0x01); + if (k > 5) *cur++ = scale * ((*in >> 2) & 0x01); + if (k > 6) *cur++ = scale * ((*in >> 1) & 0x01); + } + if (img_n != out_n) { + int q; + // insert alpha = 255 + cur = a->out + stride*j; + if (img_n == 1) { + for (q=x-1; q >= 0; --q) { + cur[q*2+1] = 255; + cur[q*2+0] = cur[q]; + } + } else { + STBI_ASSERT(img_n == 3); + for (q=x-1; q >= 0; --q) { + cur[q*4+3] = 255; + cur[q*4+2] = cur[q*3+2]; + cur[q*4+1] = cur[q*3+1]; + cur[q*4+0] = cur[q*3+0]; + } + } + } + } + } else if (depth == 16) { + // force the image data from big-endian to platform-native. + // this is done in a separate pass due to the decoding relying + // on the data being untouched, but could probably be done + // per-line during decode if care is taken. + stbi_uc *cur = a->out; + stbi__uint16 *cur16 = (stbi__uint16*)cur; + + for(i=0; i < x*y*out_n; ++i,cur16++,cur+=2) { + *cur16 = (cur[0] << 8) | cur[1]; + } + } + + return 1; +} + +static int stbi__create_png_image(stbi__png *a, stbi_uc *image_data, stbi__uint32 image_data_len, int out_n, int depth, int color, int interlaced) +{ + int bytes = (depth == 16 ? 2 : 1); + int out_bytes = out_n * bytes; + stbi_uc *final; + int p; + if (!interlaced) + return stbi__create_png_image_raw(a, image_data, image_data_len, out_n, a->s->img_x, a->s->img_y, depth, color); + + // de-interlacing + final = (stbi_uc *) stbi__malloc_mad3(a->s->img_x, a->s->img_y, out_bytes, 0); + for (p=0; p < 7; ++p) { + int xorig[] = { 0,4,0,2,0,1,0 }; + int yorig[] = { 0,0,4,0,2,0,1 }; + int xspc[] = { 8,8,4,4,2,2,1 }; + int yspc[] = { 8,8,8,4,4,2,2 }; + int i,j,x,y; + // pass1_x[4] = 0, pass1_x[5] = 1, pass1_x[12] = 1 + x = (a->s->img_x - xorig[p] + xspc[p]-1) / xspc[p]; + y = (a->s->img_y - yorig[p] + yspc[p]-1) / yspc[p]; + if (x && y) { + stbi__uint32 img_len = ((((a->s->img_n * x * depth) + 7) >> 3) + 1) * y; + if (!stbi__create_png_image_raw(a, image_data, image_data_len, out_n, x, y, depth, color)) { + STBI_FREE(final); + return 0; + } + for (j=0; j < y; ++j) { + for (i=0; i < x; ++i) { + int out_y = j*yspc[p]+yorig[p]; + int out_x = i*xspc[p]+xorig[p]; + memcpy(final + out_y*a->s->img_x*out_bytes + out_x*out_bytes, + a->out + (j*x+i)*out_bytes, out_bytes); + } + } + STBI_FREE(a->out); + image_data += img_len; + image_data_len -= img_len; + } + } + a->out = final; + + return 1; +} + +static int stbi__compute_transparency(stbi__png *z, stbi_uc tc[3], int out_n) +{ + stbi__context *s = z->s; + stbi__uint32 i, pixel_count = s->img_x * s->img_y; + stbi_uc *p = z->out; + + // compute color-based transparency, assuming we've + // already got 255 as the alpha value in the output + STBI_ASSERT(out_n == 2 || out_n == 4); + + if (out_n == 2) { + for (i=0; i < pixel_count; ++i) { + p[1] = (p[0] == tc[0] ? 0 : 255); + p += 2; + } + } else { + for (i=0; i < pixel_count; ++i) { + if (p[0] == tc[0] && p[1] == tc[1] && p[2] == tc[2]) + p[3] = 0; + p += 4; + } + } + return 1; +} + +static int stbi__compute_transparency16(stbi__png *z, stbi__uint16 tc[3], int out_n) +{ + stbi__context *s = z->s; + stbi__uint32 i, pixel_count = s->img_x * s->img_y; + stbi__uint16 *p = (stbi__uint16*) z->out; + + // compute color-based transparency, assuming we've + // already got 65535 as the alpha value in the output + STBI_ASSERT(out_n == 2 || out_n == 4); + + if (out_n == 2) { + for (i = 0; i < pixel_count; ++i) { + p[1] = (p[0] == tc[0] ? 0 : 65535); + p += 2; + } + } else { + for (i = 0; i < pixel_count; ++i) { + if (p[0] == tc[0] && p[1] == tc[1] && p[2] == tc[2]) + p[3] = 0; + p += 4; + } + } + return 1; +} + +static int stbi__expand_png_palette(stbi__png *a, stbi_uc *palette, int len, int pal_img_n) +{ + stbi__uint32 i, pixel_count = a->s->img_x * a->s->img_y; + stbi_uc *p, *temp_out, *orig = a->out; + + p = (stbi_uc *) stbi__malloc_mad2(pixel_count, pal_img_n, 0); + if (p == NULL) return stbi__err("outofmem", "Out of memory"); + + // between here and free(out) below, exitting would leak + temp_out = p; + + if (pal_img_n == 3) { + for (i=0; i < pixel_count; ++i) { + int n = orig[i]*4; + p[0] = palette[n ]; + p[1] = palette[n+1]; + p[2] = palette[n+2]; + p += 3; + } + } else { + for (i=0; i < pixel_count; ++i) { + int n = orig[i]*4; + p[0] = palette[n ]; + p[1] = palette[n+1]; + p[2] = palette[n+2]; + p[3] = palette[n+3]; + p += 4; + } + } + STBI_FREE(a->out); + a->out = temp_out; + + STBI_NOTUSED(len); + + return 1; +} + +static int stbi__unpremultiply_on_load = 0; +static int stbi__de_iphone_flag = 0; + +STBIDEF void stbi_set_unpremultiply_on_load(int flag_true_if_should_unpremultiply) +{ + stbi__unpremultiply_on_load = flag_true_if_should_unpremultiply; +} + +STBIDEF void stbi_convert_iphone_png_to_rgb(int flag_true_if_should_convert) +{ + stbi__de_iphone_flag = flag_true_if_should_convert; +} + +static void stbi__de_iphone(stbi__png *z) +{ + stbi__context *s = z->s; + stbi__uint32 i, pixel_count = s->img_x * s->img_y; + stbi_uc *p = z->out; + + if (s->img_out_n == 3) { // convert bgr to rgb + for (i=0; i < pixel_count; ++i) { + stbi_uc t = p[0]; + p[0] = p[2]; + p[2] = t; + p += 3; + } + } else { + STBI_ASSERT(s->img_out_n == 4); + if (stbi__unpremultiply_on_load) { + // convert bgr to rgb and unpremultiply + for (i=0; i < pixel_count; ++i) { + stbi_uc a = p[3]; + stbi_uc t = p[0]; + if (a) { + stbi_uc half = a / 2; + p[0] = (p[2] * 255 + half) / a; + p[1] = (p[1] * 255 + half) / a; + p[2] = ( t * 255 + half) / a; + } else { + p[0] = p[2]; + p[2] = t; + } + p += 4; + } + } else { + // convert bgr to rgb + for (i=0; i < pixel_count; ++i) { + stbi_uc t = p[0]; + p[0] = p[2]; + p[2] = t; + p += 4; + } + } + } +} + +#define STBI__PNG_TYPE(a,b,c,d) (((unsigned) (a) << 24) + ((unsigned) (b) << 16) + ((unsigned) (c) << 8) + (unsigned) (d)) + +static int stbi__parse_png_file(stbi__png *z, int scan, int req_comp) +{ + stbi_uc palette[1024], pal_img_n=0; + stbi_uc has_trans=0, tc[3]={0}; + stbi__uint16 tc16[3]; + stbi__uint32 ioff=0, idata_limit=0, i, pal_len=0; + int first=1,k,interlace=0, color=0, is_iphone=0; + stbi__context *s = z->s; + + z->expanded = NULL; + z->idata = NULL; + z->out = NULL; + + if (!stbi__check_png_header(s)) return 0; + + if (scan == STBI__SCAN_type) return 1; + + for (;;) { + stbi__pngchunk c = stbi__get_chunk_header(s); + switch (c.type) { + case STBI__PNG_TYPE('C','g','B','I'): + is_iphone = 1; + stbi__skip(s, c.length); + break; + case STBI__PNG_TYPE('I','H','D','R'): { + int comp,filter; + if (!first) return stbi__err("multiple IHDR","Corrupt PNG"); + first = 0; + if (c.length != 13) return stbi__err("bad IHDR len","Corrupt PNG"); + s->img_x = stbi__get32be(s); if (s->img_x > (1 << 24)) return stbi__err("too large","Very large image (corrupt?)"); + s->img_y = stbi__get32be(s); if (s->img_y > (1 << 24)) return stbi__err("too large","Very large image (corrupt?)"); + z->depth = stbi__get8(s); if (z->depth != 1 && z->depth != 2 && z->depth != 4 && z->depth != 8 && z->depth != 16) return stbi__err("1/2/4/8/16-bit only","PNG not supported: 1/2/4/8/16-bit only"); + color = stbi__get8(s); if (color > 6) return stbi__err("bad ctype","Corrupt PNG"); + if (color == 3 && z->depth == 16) return stbi__err("bad ctype","Corrupt PNG"); + if (color == 3) pal_img_n = 3; else if (color & 1) return stbi__err("bad ctype","Corrupt PNG"); + comp = stbi__get8(s); if (comp) return stbi__err("bad comp method","Corrupt PNG"); + filter= stbi__get8(s); if (filter) return stbi__err("bad filter method","Corrupt PNG"); + interlace = stbi__get8(s); if (interlace>1) return stbi__err("bad interlace method","Corrupt PNG"); + if (!s->img_x || !s->img_y) return stbi__err("0-pixel image","Corrupt PNG"); + if (!pal_img_n) { + s->img_n = (color & 2 ? 3 : 1) + (color & 4 ? 1 : 0); + if ((1 << 30) / s->img_x / s->img_n < s->img_y) return stbi__err("too large", "Image too large to decode"); + if (scan == STBI__SCAN_header) return 1; + } else { + // if paletted, then pal_n is our final components, and + // img_n is # components to decompress/filter. + s->img_n = 1; + if ((1 << 30) / s->img_x / 4 < s->img_y) return stbi__err("too large","Corrupt PNG"); + // if SCAN_header, have to scan to see if we have a tRNS + } + break; + } + + case STBI__PNG_TYPE('P','L','T','E'): { + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if (c.length > 256*3) return stbi__err("invalid PLTE","Corrupt PNG"); + pal_len = c.length / 3; + if (pal_len * 3 != c.length) return stbi__err("invalid PLTE","Corrupt PNG"); + for (i=0; i < pal_len; ++i) { + palette[i*4+0] = stbi__get8(s); + palette[i*4+1] = stbi__get8(s); + palette[i*4+2] = stbi__get8(s); + palette[i*4+3] = 255; + } + break; + } + + case STBI__PNG_TYPE('t','R','N','S'): { + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if (z->idata) return stbi__err("tRNS after IDAT","Corrupt PNG"); + if (pal_img_n) { + if (scan == STBI__SCAN_header) { s->img_n = 4; return 1; } + if (pal_len == 0) return stbi__err("tRNS before PLTE","Corrupt PNG"); + if (c.length > pal_len) return stbi__err("bad tRNS len","Corrupt PNG"); + pal_img_n = 4; + for (i=0; i < c.length; ++i) + palette[i*4+3] = stbi__get8(s); + } else { + if (!(s->img_n & 1)) return stbi__err("tRNS with alpha","Corrupt PNG"); + if (c.length != (stbi__uint32) s->img_n*2) return stbi__err("bad tRNS len","Corrupt PNG"); + has_trans = 1; + if (z->depth == 16) { + for (k = 0; k < s->img_n; ++k) tc16[k] = (stbi__uint16)stbi__get16be(s); // copy the values as-is + } else { + for (k = 0; k < s->img_n; ++k) tc[k] = (stbi_uc)(stbi__get16be(s) & 255) * stbi__depth_scale_table[z->depth]; // non 8-bit images will be larger + } + } + break; + } + + case STBI__PNG_TYPE('I','D','A','T'): { + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if (pal_img_n && !pal_len) return stbi__err("no PLTE","Corrupt PNG"); + if (scan == STBI__SCAN_header) { s->img_n = pal_img_n; return 1; } + if ((int)(ioff + c.length) < (int)ioff) return 0; + if (ioff + c.length > idata_limit) { + stbi__uint32 idata_limit_old = idata_limit; + stbi_uc *p; + if (idata_limit == 0) idata_limit = c.length > 4096 ? c.length : 4096; + while (ioff + c.length > idata_limit) + idata_limit *= 2; + STBI_NOTUSED(idata_limit_old); + p = (stbi_uc *) STBI_REALLOC_SIZED(z->idata, idata_limit_old, idata_limit); if (p == NULL) return stbi__err("outofmem", "Out of memory"); + z->idata = p; + } + if (!stbi__getn(s, z->idata+ioff,c.length)) return stbi__err("outofdata","Corrupt PNG"); + ioff += c.length; + break; + } + + case STBI__PNG_TYPE('I','E','N','D'): { + stbi__uint32 raw_len, bpl; + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if (scan != STBI__SCAN_load) return 1; + if (z->idata == NULL) return stbi__err("no IDAT","Corrupt PNG"); + // initial guess for decoded data size to avoid unnecessary reallocs + bpl = (s->img_x * z->depth + 7) / 8; // bytes per line, per component + raw_len = bpl * s->img_y * s->img_n /* pixels */ + s->img_y /* filter mode per row */; + z->expanded = (stbi_uc *) stbi_zlib_decode_malloc_guesssize_headerflag((char *) z->idata, ioff, raw_len, (int *) &raw_len, !is_iphone); + if (z->expanded == NULL) return 0; // zlib should set error + STBI_FREE(z->idata); z->idata = NULL; + if ((req_comp == s->img_n+1 && req_comp != 3 && !pal_img_n) || has_trans) + s->img_out_n = s->img_n+1; + else + s->img_out_n = s->img_n; + if (!stbi__create_png_image(z, z->expanded, raw_len, s->img_out_n, z->depth, color, interlace)) return 0; + if (has_trans) { + if (z->depth == 16) { + if (!stbi__compute_transparency16(z, tc16, s->img_out_n)) return 0; + } else { + if (!stbi__compute_transparency(z, tc, s->img_out_n)) return 0; + } + } + if (is_iphone && stbi__de_iphone_flag && s->img_out_n > 2) + stbi__de_iphone(z); + if (pal_img_n) { + // pal_img_n == 3 or 4 + s->img_n = pal_img_n; // record the actual colors we had + s->img_out_n = pal_img_n; + if (req_comp >= 3) s->img_out_n = req_comp; + if (!stbi__expand_png_palette(z, palette, pal_len, s->img_out_n)) + return 0; + } else if (has_trans) { + // non-paletted image with tRNS -> source image has (constant) alpha + ++s->img_n; + } + STBI_FREE(z->expanded); z->expanded = NULL; + return 1; + } + + default: + // if critical, fail + if (first) return stbi__err("first not IHDR", "Corrupt PNG"); + if ((c.type & (1 << 29)) == 0) { + #ifndef STBI_NO_FAILURE_STRINGS + // not threadsafe + static char invalid_chunk[] = "XXXX PNG chunk not known"; + invalid_chunk[0] = STBI__BYTECAST(c.type >> 24); + invalid_chunk[1] = STBI__BYTECAST(c.type >> 16); + invalid_chunk[2] = STBI__BYTECAST(c.type >> 8); + invalid_chunk[3] = STBI__BYTECAST(c.type >> 0); + #endif + return stbi__err(invalid_chunk, "PNG not supported: unknown PNG chunk type"); + } + stbi__skip(s, c.length); + break; + } + // end of PNG chunk, read and skip CRC + stbi__get32be(s); + } +} + +static void *stbi__do_png(stbi__png *p, int *x, int *y, int *n, int req_comp, stbi__result_info *ri) +{ + void *result=NULL; + if (req_comp < 0 || req_comp > 4) return stbi__errpuc("bad req_comp", "Internal error"); + if (stbi__parse_png_file(p, STBI__SCAN_load, req_comp)) { + if (p->depth < 8) + ri->bits_per_channel = 8; + else + ri->bits_per_channel = p->depth; + result = p->out; + p->out = NULL; + if (req_comp && req_comp != p->s->img_out_n) { + if (ri->bits_per_channel == 8) + result = stbi__convert_format((unsigned char *) result, p->s->img_out_n, req_comp, p->s->img_x, p->s->img_y); + else + result = stbi__convert_format16((stbi__uint16 *) result, p->s->img_out_n, req_comp, p->s->img_x, p->s->img_y); + p->s->img_out_n = req_comp; + if (result == NULL) return result; + } + *x = p->s->img_x; + *y = p->s->img_y; + if (n) *n = p->s->img_n; + } + STBI_FREE(p->out); p->out = NULL; + STBI_FREE(p->expanded); p->expanded = NULL; + STBI_FREE(p->idata); p->idata = NULL; + + return result; +} + +static void *stbi__png_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + stbi__png p; + p.s = s; + return stbi__do_png(&p, x,y,comp,req_comp, ri); +} + +static int stbi__png_test(stbi__context *s) +{ + int r; + r = stbi__check_png_header(s); + stbi__rewind(s); + return r; +} + +static int stbi__png_info_raw(stbi__png *p, int *x, int *y, int *comp) +{ + if (!stbi__parse_png_file(p, STBI__SCAN_header, 0)) { + stbi__rewind( p->s ); + return 0; + } + if (x) *x = p->s->img_x; + if (y) *y = p->s->img_y; + if (comp) *comp = p->s->img_n; + return 1; +} + +static int stbi__png_info(stbi__context *s, int *x, int *y, int *comp) +{ + stbi__png p; + p.s = s; + return stbi__png_info_raw(&p, x, y, comp); +} + +static int stbi__png_is16(stbi__context *s) +{ + stbi__png p; + p.s = s; + if (!stbi__png_info_raw(&p, NULL, NULL, NULL)) + return 0; + if (p.depth != 16) { + stbi__rewind(p.s); + return 0; + } + return 1; +} +#endif + +// Microsoft/Windows BMP image + +#ifndef STBI_NO_BMP +static int stbi__bmp_test_raw(stbi__context *s) +{ + int r; + int sz; + if (stbi__get8(s) != 'B') return 0; + if (stbi__get8(s) != 'M') return 0; + stbi__get32le(s); // discard filesize + stbi__get16le(s); // discard reserved + stbi__get16le(s); // discard reserved + stbi__get32le(s); // discard data offset + sz = stbi__get32le(s); + r = (sz == 12 || sz == 40 || sz == 56 || sz == 108 || sz == 124); + return r; +} + +static int stbi__bmp_test(stbi__context *s) +{ + int r = stbi__bmp_test_raw(s); + stbi__rewind(s); + return r; +} + + +// returns 0..31 for the highest set bit +static int stbi__high_bit(unsigned int z) +{ + int n=0; + if (z == 0) return -1; + if (z >= 0x10000) { n += 16; z >>= 16; } + if (z >= 0x00100) { n += 8; z >>= 8; } + if (z >= 0x00010) { n += 4; z >>= 4; } + if (z >= 0x00004) { n += 2; z >>= 2; } + if (z >= 0x00002) { n += 1; z >>= 1; } + return n; +} + +static int stbi__bitcount(unsigned int a) +{ + a = (a & 0x55555555) + ((a >> 1) & 0x55555555); // max 2 + a = (a & 0x33333333) + ((a >> 2) & 0x33333333); // max 4 + a = (a + (a >> 4)) & 0x0f0f0f0f; // max 8 per 4, now 8 bits + a = (a + (a >> 8)); // max 16 per 8 bits + a = (a + (a >> 16)); // max 32 per 8 bits + return a & 0xff; +} + +// extract an arbitrarily-aligned N-bit value (N=bits) +// from v, and then make it 8-bits long and fractionally +// extend it to full full range. +static int stbi__shiftsigned(unsigned int v, int shift, int bits) +{ + static unsigned int mul_table[9] = { + 0, + 0xff/*0b11111111*/, 0x55/*0b01010101*/, 0x49/*0b01001001*/, 0x11/*0b00010001*/, + 0x21/*0b00100001*/, 0x41/*0b01000001*/, 0x81/*0b10000001*/, 0x01/*0b00000001*/, + }; + static unsigned int shift_table[9] = { + 0, 0,0,1,0,2,4,6,0, + }; + if (shift < 0) + v <<= -shift; + else + v >>= shift; + STBI_ASSERT(v >= 0 && v < 256); + v >>= (8-bits); + STBI_ASSERT(bits >= 0 && bits <= 8); + return (int) ((unsigned) v * mul_table[bits]) >> shift_table[bits]; +} + +typedef struct +{ + int bpp, offset, hsz; + unsigned int mr,mg,mb,ma, all_a; +} stbi__bmp_data; + +static void *stbi__bmp_parse_header(stbi__context *s, stbi__bmp_data *info) +{ + int hsz; + if (stbi__get8(s) != 'B' || stbi__get8(s) != 'M') return stbi__errpuc("not BMP", "Corrupt BMP"); + stbi__get32le(s); // discard filesize + stbi__get16le(s); // discard reserved + stbi__get16le(s); // discard reserved + info->offset = stbi__get32le(s); + info->hsz = hsz = stbi__get32le(s); + info->mr = info->mg = info->mb = info->ma = 0; + + if (hsz != 12 && hsz != 40 && hsz != 56 && hsz != 108 && hsz != 124) return stbi__errpuc("unknown BMP", "BMP type not supported: unknown"); + if (hsz == 12) { + s->img_x = stbi__get16le(s); + s->img_y = stbi__get16le(s); + } else { + s->img_x = stbi__get32le(s); + s->img_y = stbi__get32le(s); + } + if (stbi__get16le(s) != 1) return stbi__errpuc("bad BMP", "bad BMP"); + info->bpp = stbi__get16le(s); + if (hsz != 12) { + int compress = stbi__get32le(s); + if (compress == 1 || compress == 2) return stbi__errpuc("BMP RLE", "BMP type not supported: RLE"); + stbi__get32le(s); // discard sizeof + stbi__get32le(s); // discard hres + stbi__get32le(s); // discard vres + stbi__get32le(s); // discard colorsused + stbi__get32le(s); // discard max important + if (hsz == 40 || hsz == 56) { + if (hsz == 56) { + stbi__get32le(s); + stbi__get32le(s); + stbi__get32le(s); + stbi__get32le(s); + } + if (info->bpp == 16 || info->bpp == 32) { + if (compress == 0) { + if (info->bpp == 32) { + info->mr = 0xffu << 16; + info->mg = 0xffu << 8; + info->mb = 0xffu << 0; + info->ma = 0xffu << 24; + info->all_a = 0; // if all_a is 0 at end, then we loaded alpha channel but it was all 0 + } else { + info->mr = 31u << 10; + info->mg = 31u << 5; + info->mb = 31u << 0; + } + } else if (compress == 3) { + info->mr = stbi__get32le(s); + info->mg = stbi__get32le(s); + info->mb = stbi__get32le(s); + // not documented, but generated by photoshop and handled by mspaint + if (info->mr == info->mg && info->mg == info->mb) { + // ?!?!? + return stbi__errpuc("bad BMP", "bad BMP"); + } + } else + return stbi__errpuc("bad BMP", "bad BMP"); + } + } else { + int i; + if (hsz != 108 && hsz != 124) + return stbi__errpuc("bad BMP", "bad BMP"); + info->mr = stbi__get32le(s); + info->mg = stbi__get32le(s); + info->mb = stbi__get32le(s); + info->ma = stbi__get32le(s); + stbi__get32le(s); // discard color space + for (i=0; i < 12; ++i) + stbi__get32le(s); // discard color space parameters + if (hsz == 124) { + stbi__get32le(s); // discard rendering intent + stbi__get32le(s); // discard offset of profile data + stbi__get32le(s); // discard size of profile data + stbi__get32le(s); // discard reserved + } + } + } + return (void *) 1; +} + + +static void *stbi__bmp_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + stbi_uc *out; + unsigned int mr=0,mg=0,mb=0,ma=0, all_a; + stbi_uc pal[256][4]; + int psize=0,i,j,width; + int flip_vertically, pad, target; + stbi__bmp_data info; + STBI_NOTUSED(ri); + + info.all_a = 255; + if (stbi__bmp_parse_header(s, &info) == NULL) + return NULL; // error code already set + + flip_vertically = ((int) s->img_y) > 0; + s->img_y = abs((int) s->img_y); + + mr = info.mr; + mg = info.mg; + mb = info.mb; + ma = info.ma; + all_a = info.all_a; + + if (info.hsz == 12) { + if (info.bpp < 24) + psize = (info.offset - 14 - 24) / 3; + } else { + if (info.bpp < 16) + psize = (info.offset - 14 - info.hsz) >> 2; + } + + s->img_n = ma ? 4 : 3; + if (req_comp && req_comp >= 3) // we can directly decode 3 or 4 + target = req_comp; + else + target = s->img_n; // if they want monochrome, we'll post-convert + + // sanity-check size + if (!stbi__mad3sizes_valid(target, s->img_x, s->img_y, 0)) + return stbi__errpuc("too large", "Corrupt BMP"); + + out = (stbi_uc *) stbi__malloc_mad3(target, s->img_x, s->img_y, 0); + if (!out) return stbi__errpuc("outofmem", "Out of memory"); + if (info.bpp < 16) { + int z=0; + if (psize == 0 || psize > 256) { STBI_FREE(out); return stbi__errpuc("invalid", "Corrupt BMP"); } + for (i=0; i < psize; ++i) { + pal[i][2] = stbi__get8(s); + pal[i][1] = stbi__get8(s); + pal[i][0] = stbi__get8(s); + if (info.hsz != 12) stbi__get8(s); + pal[i][3] = 255; + } + stbi__skip(s, info.offset - 14 - info.hsz - psize * (info.hsz == 12 ? 3 : 4)); + if (info.bpp == 1) width = (s->img_x + 7) >> 3; + else if (info.bpp == 4) width = (s->img_x + 1) >> 1; + else if (info.bpp == 8) width = s->img_x; + else { STBI_FREE(out); return stbi__errpuc("bad bpp", "Corrupt BMP"); } + pad = (-width)&3; + if (info.bpp == 1) { + for (j=0; j < (int) s->img_y; ++j) { + int bit_offset = 7, v = stbi__get8(s); + for (i=0; i < (int) s->img_x; ++i) { + int color = (v>>bit_offset)&0x1; + out[z++] = pal[color][0]; + out[z++] = pal[color][1]; + out[z++] = pal[color][2]; + if (target == 4) out[z++] = 255; + if (i+1 == (int) s->img_x) break; + if((--bit_offset) < 0) { + bit_offset = 7; + v = stbi__get8(s); + } + } + stbi__skip(s, pad); + } + } else { + for (j=0; j < (int) s->img_y; ++j) { + for (i=0; i < (int) s->img_x; i += 2) { + int v=stbi__get8(s),v2=0; + if (info.bpp == 4) { + v2 = v & 15; + v >>= 4; + } + out[z++] = pal[v][0]; + out[z++] = pal[v][1]; + out[z++] = pal[v][2]; + if (target == 4) out[z++] = 255; + if (i+1 == (int) s->img_x) break; + v = (info.bpp == 8) ? stbi__get8(s) : v2; + out[z++] = pal[v][0]; + out[z++] = pal[v][1]; + out[z++] = pal[v][2]; + if (target == 4) out[z++] = 255; + } + stbi__skip(s, pad); + } + } + } else { + int rshift=0,gshift=0,bshift=0,ashift=0,rcount=0,gcount=0,bcount=0,acount=0; + int z = 0; + int easy=0; + stbi__skip(s, info.offset - 14 - info.hsz); + if (info.bpp == 24) width = 3 * s->img_x; + else if (info.bpp == 16) width = 2*s->img_x; + else /* bpp = 32 and pad = 0 */ width=0; + pad = (-width) & 3; + if (info.bpp == 24) { + easy = 1; + } else if (info.bpp == 32) { + if (mb == 0xff && mg == 0xff00 && mr == 0x00ff0000 && ma == 0xff000000) + easy = 2; + } + if (!easy) { + if (!mr || !mg || !mb) { STBI_FREE(out); return stbi__errpuc("bad masks", "Corrupt BMP"); } + // right shift amt to put high bit in position #7 + rshift = stbi__high_bit(mr)-7; rcount = stbi__bitcount(mr); + gshift = stbi__high_bit(mg)-7; gcount = stbi__bitcount(mg); + bshift = stbi__high_bit(mb)-7; bcount = stbi__bitcount(mb); + ashift = stbi__high_bit(ma)-7; acount = stbi__bitcount(ma); + } + for (j=0; j < (int) s->img_y; ++j) { + if (easy) { + for (i=0; i < (int) s->img_x; ++i) { + unsigned char a; + out[z+2] = stbi__get8(s); + out[z+1] = stbi__get8(s); + out[z+0] = stbi__get8(s); + z += 3; + a = (easy == 2 ? stbi__get8(s) : 255); + all_a |= a; + if (target == 4) out[z++] = a; + } + } else { + int bpp = info.bpp; + for (i=0; i < (int) s->img_x; ++i) { + stbi__uint32 v = (bpp == 16 ? (stbi__uint32) stbi__get16le(s) : stbi__get32le(s)); + unsigned int a; + out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mr, rshift, rcount)); + out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mg, gshift, gcount)); + out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mb, bshift, bcount)); + a = (ma ? stbi__shiftsigned(v & ma, ashift, acount) : 255); + all_a |= a; + if (target == 4) out[z++] = STBI__BYTECAST(a); + } + } + stbi__skip(s, pad); + } + } + + // if alpha channel is all 0s, replace with all 255s + if (target == 4 && all_a == 0) + for (i=4*s->img_x*s->img_y-1; i >= 0; i -= 4) + out[i] = 255; + + if (flip_vertically) { + stbi_uc t; + for (j=0; j < (int) s->img_y>>1; ++j) { + stbi_uc *p1 = out + j *s->img_x*target; + stbi_uc *p2 = out + (s->img_y-1-j)*s->img_x*target; + for (i=0; i < (int) s->img_x*target; ++i) { + t = p1[i]; p1[i] = p2[i]; p2[i] = t; + } + } + } + + if (req_comp && req_comp != target) { + out = stbi__convert_format(out, target, req_comp, s->img_x, s->img_y); + if (out == NULL) return out; // stbi__convert_format frees input on failure + } + + *x = s->img_x; + *y = s->img_y; + if (comp) *comp = s->img_n; + return out; +} +#endif + +// Targa Truevision - TGA +// by Jonathan Dummer +#ifndef STBI_NO_TGA +// returns STBI_rgb or whatever, 0 on error +static int stbi__tga_get_comp(int bits_per_pixel, int is_grey, int* is_rgb16) +{ + // only RGB or RGBA (incl. 16bit) or grey allowed + if (is_rgb16) *is_rgb16 = 0; + switch(bits_per_pixel) { + case 8: return STBI_grey; + case 16: if(is_grey) return STBI_grey_alpha; + // fallthrough + case 15: if(is_rgb16) *is_rgb16 = 1; + return STBI_rgb; + case 24: // fallthrough + case 32: return bits_per_pixel/8; + default: return 0; + } +} + +static int stbi__tga_info(stbi__context *s, int *x, int *y, int *comp) +{ + int tga_w, tga_h, tga_comp, tga_image_type, tga_bits_per_pixel, tga_colormap_bpp; + int sz, tga_colormap_type; + stbi__get8(s); // discard Offset + tga_colormap_type = stbi__get8(s); // colormap type + if( tga_colormap_type > 1 ) { + stbi__rewind(s); + return 0; // only RGB or indexed allowed + } + tga_image_type = stbi__get8(s); // image type + if ( tga_colormap_type == 1 ) { // colormapped (paletted) image + if (tga_image_type != 1 && tga_image_type != 9) { + stbi__rewind(s); + return 0; + } + stbi__skip(s,4); // skip index of first colormap entry and number of entries + sz = stbi__get8(s); // check bits per palette color entry + if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) { + stbi__rewind(s); + return 0; + } + stbi__skip(s,4); // skip image x and y origin + tga_colormap_bpp = sz; + } else { // "normal" image w/o colormap - only RGB or grey allowed, +/- RLE + if ( (tga_image_type != 2) && (tga_image_type != 3) && (tga_image_type != 10) && (tga_image_type != 11) ) { + stbi__rewind(s); + return 0; // only RGB or grey allowed, +/- RLE + } + stbi__skip(s,9); // skip colormap specification and image x/y origin + tga_colormap_bpp = 0; + } + tga_w = stbi__get16le(s); + if( tga_w < 1 ) { + stbi__rewind(s); + return 0; // test width + } + tga_h = stbi__get16le(s); + if( tga_h < 1 ) { + stbi__rewind(s); + return 0; // test height + } + tga_bits_per_pixel = stbi__get8(s); // bits per pixel + stbi__get8(s); // ignore alpha bits + if (tga_colormap_bpp != 0) { + if((tga_bits_per_pixel != 8) && (tga_bits_per_pixel != 16)) { + // when using a colormap, tga_bits_per_pixel is the size of the indexes + // I don't think anything but 8 or 16bit indexes makes sense + stbi__rewind(s); + return 0; + } + tga_comp = stbi__tga_get_comp(tga_colormap_bpp, 0, NULL); + } else { + tga_comp = stbi__tga_get_comp(tga_bits_per_pixel, (tga_image_type == 3) || (tga_image_type == 11), NULL); + } + if(!tga_comp) { + stbi__rewind(s); + return 0; + } + if (x) *x = tga_w; + if (y) *y = tga_h; + if (comp) *comp = tga_comp; + return 1; // seems to have passed everything +} + +static int stbi__tga_test(stbi__context *s) +{ + int res = 0; + int sz, tga_color_type; + stbi__get8(s); // discard Offset + tga_color_type = stbi__get8(s); // color type + if ( tga_color_type > 1 ) goto errorEnd; // only RGB or indexed allowed + sz = stbi__get8(s); // image type + if ( tga_color_type == 1 ) { // colormapped (paletted) image + if (sz != 1 && sz != 9) goto errorEnd; // colortype 1 demands image type 1 or 9 + stbi__skip(s,4); // skip index of first colormap entry and number of entries + sz = stbi__get8(s); // check bits per palette color entry + if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) goto errorEnd; + stbi__skip(s,4); // skip image x and y origin + } else { // "normal" image w/o colormap + if ( (sz != 2) && (sz != 3) && (sz != 10) && (sz != 11) ) goto errorEnd; // only RGB or grey allowed, +/- RLE + stbi__skip(s,9); // skip colormap specification and image x/y origin + } + if ( stbi__get16le(s) < 1 ) goto errorEnd; // test width + if ( stbi__get16le(s) < 1 ) goto errorEnd; // test height + sz = stbi__get8(s); // bits per pixel + if ( (tga_color_type == 1) && (sz != 8) && (sz != 16) ) goto errorEnd; // for colormapped images, bpp is size of an index + if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) goto errorEnd; + + res = 1; // if we got this far, everything's good and we can return 1 instead of 0 + +errorEnd: + stbi__rewind(s); + return res; +} + +// read 16bit value and convert to 24bit RGB +static void stbi__tga_read_rgb16(stbi__context *s, stbi_uc* out) +{ + stbi__uint16 px = (stbi__uint16)stbi__get16le(s); + stbi__uint16 fiveBitMask = 31; + // we have 3 channels with 5bits each + int r = (px >> 10) & fiveBitMask; + int g = (px >> 5) & fiveBitMask; + int b = px & fiveBitMask; + // Note that this saves the data in RGB(A) order, so it doesn't need to be swapped later + out[0] = (stbi_uc)((r * 255)/31); + out[1] = (stbi_uc)((g * 255)/31); + out[2] = (stbi_uc)((b * 255)/31); + + // some people claim that the most significant bit might be used for alpha + // (possibly if an alpha-bit is set in the "image descriptor byte") + // but that only made 16bit test images completely translucent.. + // so let's treat all 15 and 16bit TGAs as RGB with no alpha. +} + +static void *stbi__tga_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + // read in the TGA header stuff + int tga_offset = stbi__get8(s); + int tga_indexed = stbi__get8(s); + int tga_image_type = stbi__get8(s); + int tga_is_RLE = 0; + int tga_palette_start = stbi__get16le(s); + int tga_palette_len = stbi__get16le(s); + int tga_palette_bits = stbi__get8(s); + int tga_x_origin = stbi__get16le(s); + int tga_y_origin = stbi__get16le(s); + int tga_width = stbi__get16le(s); + int tga_height = stbi__get16le(s); + int tga_bits_per_pixel = stbi__get8(s); + int tga_comp, tga_rgb16=0; + int tga_inverted = stbi__get8(s); + // int tga_alpha_bits = tga_inverted & 15; // the 4 lowest bits - unused (useless?) + // image data + unsigned char *tga_data; + unsigned char *tga_palette = NULL; + int i, j; + unsigned char raw_data[4] = {0}; + int RLE_count = 0; + int RLE_repeating = 0; + int read_next_pixel = 1; + STBI_NOTUSED(ri); + + // do a tiny bit of precessing + if ( tga_image_type >= 8 ) + { + tga_image_type -= 8; + tga_is_RLE = 1; + } + tga_inverted = 1 - ((tga_inverted >> 5) & 1); + + // If I'm paletted, then I'll use the number of bits from the palette + if ( tga_indexed ) tga_comp = stbi__tga_get_comp(tga_palette_bits, 0, &tga_rgb16); + else tga_comp = stbi__tga_get_comp(tga_bits_per_pixel, (tga_image_type == 3), &tga_rgb16); + + if(!tga_comp) // shouldn't really happen, stbi__tga_test() should have ensured basic consistency + return stbi__errpuc("bad format", "Can't find out TGA pixelformat"); + + // tga info + *x = tga_width; + *y = tga_height; + if (comp) *comp = tga_comp; + + if (!stbi__mad3sizes_valid(tga_width, tga_height, tga_comp, 0)) + return stbi__errpuc("too large", "Corrupt TGA"); + + tga_data = (unsigned char*)stbi__malloc_mad3(tga_width, tga_height, tga_comp, 0); + if (!tga_data) return stbi__errpuc("outofmem", "Out of memory"); + + // skip to the data's starting position (offset usually = 0) + stbi__skip(s, tga_offset ); + + if ( !tga_indexed && !tga_is_RLE && !tga_rgb16 ) { + for (i=0; i < tga_height; ++i) { + int row = tga_inverted ? tga_height -i - 1 : i; + stbi_uc *tga_row = tga_data + row*tga_width*tga_comp; + stbi__getn(s, tga_row, tga_width * tga_comp); + } + } else { + // do I need to load a palette? + if ( tga_indexed) + { + // any data to skip? (offset usually = 0) + stbi__skip(s, tga_palette_start ); + // load the palette + tga_palette = (unsigned char*)stbi__malloc_mad2(tga_palette_len, tga_comp, 0); + if (!tga_palette) { + STBI_FREE(tga_data); + return stbi__errpuc("outofmem", "Out of memory"); + } + if (tga_rgb16) { + stbi_uc *pal_entry = tga_palette; + STBI_ASSERT(tga_comp == STBI_rgb); + for (i=0; i < tga_palette_len; ++i) { + stbi__tga_read_rgb16(s, pal_entry); + pal_entry += tga_comp; + } + } else if (!stbi__getn(s, tga_palette, tga_palette_len * tga_comp)) { + STBI_FREE(tga_data); + STBI_FREE(tga_palette); + return stbi__errpuc("bad palette", "Corrupt TGA"); + } + } + // load the data + for (i=0; i < tga_width * tga_height; ++i) + { + // if I'm in RLE mode, do I need to get a RLE stbi__pngchunk? + if ( tga_is_RLE ) + { + if ( RLE_count == 0 ) + { + // yep, get the next byte as a RLE command + int RLE_cmd = stbi__get8(s); + RLE_count = 1 + (RLE_cmd & 127); + RLE_repeating = RLE_cmd >> 7; + read_next_pixel = 1; + } else if ( !RLE_repeating ) + { + read_next_pixel = 1; + } + } else + { + read_next_pixel = 1; + } + // OK, if I need to read a pixel, do it now + if ( read_next_pixel ) + { + // load however much data we did have + if ( tga_indexed ) + { + // read in index, then perform the lookup + int pal_idx = (tga_bits_per_pixel == 8) ? stbi__get8(s) : stbi__get16le(s); + if ( pal_idx >= tga_palette_len ) { + // invalid index + pal_idx = 0; + } + pal_idx *= tga_comp; + for (j = 0; j < tga_comp; ++j) { + raw_data[j] = tga_palette[pal_idx+j]; + } + } else if(tga_rgb16) { + STBI_ASSERT(tga_comp == STBI_rgb); + stbi__tga_read_rgb16(s, raw_data); + } else { + // read in the data raw + for (j = 0; j < tga_comp; ++j) { + raw_data[j] = stbi__get8(s); + } + } + // clear the reading flag for the next pixel + read_next_pixel = 0; + } // end of reading a pixel + + // copy data + for (j = 0; j < tga_comp; ++j) + tga_data[i*tga_comp+j] = raw_data[j]; + + // in case we're in RLE mode, keep counting down + --RLE_count; + } + // do I need to invert the image? + if ( tga_inverted ) + { + for (j = 0; j*2 < tga_height; ++j) + { + int index1 = j * tga_width * tga_comp; + int index2 = (tga_height - 1 - j) * tga_width * tga_comp; + for (i = tga_width * tga_comp; i > 0; --i) + { + unsigned char temp = tga_data[index1]; + tga_data[index1] = tga_data[index2]; + tga_data[index2] = temp; + ++index1; + ++index2; + } + } + } + // clear my palette, if I had one + if ( tga_palette != NULL ) + { + STBI_FREE( tga_palette ); + } + } + + // swap RGB - if the source data was RGB16, it already is in the right order + if (tga_comp >= 3 && !tga_rgb16) + { + unsigned char* tga_pixel = tga_data; + for (i=0; i < tga_width * tga_height; ++i) + { + unsigned char temp = tga_pixel[0]; + tga_pixel[0] = tga_pixel[2]; + tga_pixel[2] = temp; + tga_pixel += tga_comp; + } + } + + // convert to target component count + if (req_comp && req_comp != tga_comp) + tga_data = stbi__convert_format(tga_data, tga_comp, req_comp, tga_width, tga_height); + + // the things I do to get rid of an error message, and yet keep + // Microsoft's C compilers happy... [8^( + tga_palette_start = tga_palette_len = tga_palette_bits = + tga_x_origin = tga_y_origin = 0; + // OK, done + return tga_data; +} +#endif + +// ************************************************************************************************* +// Photoshop PSD loader -- PD by Thatcher Ulrich, integration by Nicolas Schulz, tweaked by STB + +#ifndef STBI_NO_PSD +static int stbi__psd_test(stbi__context *s) +{ + int r = (stbi__get32be(s) == 0x38425053); + stbi__rewind(s); + return r; +} + +static int stbi__psd_decode_rle(stbi__context *s, stbi_uc *p, int pixelCount) +{ + int count, nleft, len; + + count = 0; + while ((nleft = pixelCount - count) > 0) { + len = stbi__get8(s); + if (len == 128) { + // No-op. + } else if (len < 128) { + // Copy next len+1 bytes literally. + len++; + if (len > nleft) return 0; // corrupt data + count += len; + while (len) { + *p = stbi__get8(s); + p += 4; + len--; + } + } else if (len > 128) { + stbi_uc val; + // Next -len+1 bytes in the dest are replicated from next source byte. + // (Interpret len as a negative 8-bit int.) + len = 257 - len; + if (len > nleft) return 0; // corrupt data + val = stbi__get8(s); + count += len; + while (len) { + *p = val; + p += 4; + len--; + } + } + } + + return 1; +} + +static void *stbi__psd_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc) +{ + int pixelCount; + int channelCount, compression; + int channel, i; + int bitdepth; + int w,h; + stbi_uc *out; + STBI_NOTUSED(ri); + + // Check identifier + if (stbi__get32be(s) != 0x38425053) // "8BPS" + return stbi__errpuc("not PSD", "Corrupt PSD image"); + + // Check file type version. + if (stbi__get16be(s) != 1) + return stbi__errpuc("wrong version", "Unsupported version of PSD image"); + + // Skip 6 reserved bytes. + stbi__skip(s, 6 ); + + // Read the number of channels (R, G, B, A, etc). + channelCount = stbi__get16be(s); + if (channelCount < 0 || channelCount > 16) + return stbi__errpuc("wrong channel count", "Unsupported number of channels in PSD image"); + + // Read the rows and columns of the image. + h = stbi__get32be(s); + w = stbi__get32be(s); + + // Make sure the depth is 8 bits. + bitdepth = stbi__get16be(s); + if (bitdepth != 8 && bitdepth != 16) + return stbi__errpuc("unsupported bit depth", "PSD bit depth is not 8 or 16 bit"); + + // Make sure the color mode is RGB. + // Valid options are: + // 0: Bitmap + // 1: Grayscale + // 2: Indexed color + // 3: RGB color + // 4: CMYK color + // 7: Multichannel + // 8: Duotone + // 9: Lab color + if (stbi__get16be(s) != 3) + return stbi__errpuc("wrong color format", "PSD is not in RGB color format"); + + // Skip the Mode Data. (It's the palette for indexed color; other info for other modes.) + stbi__skip(s,stbi__get32be(s) ); + + // Skip the image resources. (resolution, pen tool paths, etc) + stbi__skip(s, stbi__get32be(s) ); + + // Skip the reserved data. + stbi__skip(s, stbi__get32be(s) ); + + // Find out if the data is compressed. + // Known values: + // 0: no compression + // 1: RLE compressed + compression = stbi__get16be(s); + if (compression > 1) + return stbi__errpuc("bad compression", "PSD has an unknown compression format"); + + // Check size + if (!stbi__mad3sizes_valid(4, w, h, 0)) + return stbi__errpuc("too large", "Corrupt PSD"); + + // Create the destination image. + + if (!compression && bitdepth == 16 && bpc == 16) { + out = (stbi_uc *) stbi__malloc_mad3(8, w, h, 0); + ri->bits_per_channel = 16; + } else + out = (stbi_uc *) stbi__malloc(4 * w*h); + + if (!out) return stbi__errpuc("outofmem", "Out of memory"); + pixelCount = w*h; + + // Initialize the data to zero. + //memset( out, 0, pixelCount * 4 ); + + // Finally, the image data. + if (compression) { + // RLE as used by .PSD and .TIFF + // Loop until you get the number of unpacked bytes you are expecting: + // Read the next source byte into n. + // If n is between 0 and 127 inclusive, copy the next n+1 bytes literally. + // Else if n is between -127 and -1 inclusive, copy the next byte -n+1 times. + // Else if n is 128, noop. + // Endloop + + // The RLE-compressed data is preceded by a 2-byte data count for each row in the data, + // which we're going to just skip. + stbi__skip(s, h * channelCount * 2 ); + + // Read the RLE data by channel. + for (channel = 0; channel < 4; channel++) { + stbi_uc *p; + + p = out+channel; + if (channel >= channelCount) { + // Fill this channel with default data. + for (i = 0; i < pixelCount; i++, p += 4) + *p = (channel == 3 ? 255 : 0); + } else { + // Read the RLE data. + if (!stbi__psd_decode_rle(s, p, pixelCount)) { + STBI_FREE(out); + return stbi__errpuc("corrupt", "bad RLE data"); + } + } + } + + } else { + // We're at the raw image data. It's each channel in order (Red, Green, Blue, Alpha, ...) + // where each channel consists of an 8-bit (or 16-bit) value for each pixel in the image. + + // Read the data by channel. + for (channel = 0; channel < 4; channel++) { + if (channel >= channelCount) { + // Fill this channel with default data. + if (bitdepth == 16 && bpc == 16) { + stbi__uint16 *q = ((stbi__uint16 *) out) + channel; + stbi__uint16 val = channel == 3 ? 65535 : 0; + for (i = 0; i < pixelCount; i++, q += 4) + *q = val; + } else { + stbi_uc *p = out+channel; + stbi_uc val = channel == 3 ? 255 : 0; + for (i = 0; i < pixelCount; i++, p += 4) + *p = val; + } + } else { + if (ri->bits_per_channel == 16) { // output bpc + stbi__uint16 *q = ((stbi__uint16 *) out) + channel; + for (i = 0; i < pixelCount; i++, q += 4) + *q = (stbi__uint16) stbi__get16be(s); + } else { + stbi_uc *p = out+channel; + if (bitdepth == 16) { // input bpc + for (i = 0; i < pixelCount; i++, p += 4) + *p = (stbi_uc) (stbi__get16be(s) >> 8); + } else { + for (i = 0; i < pixelCount; i++, p += 4) + *p = stbi__get8(s); + } + } + } + } + } + + // remove weird white matte from PSD + if (channelCount >= 4) { + if (ri->bits_per_channel == 16) { + for (i=0; i < w*h; ++i) { + stbi__uint16 *pixel = (stbi__uint16 *) out + 4*i; + if (pixel[3] != 0 && pixel[3] != 65535) { + float a = pixel[3] / 65535.0f; + float ra = 1.0f / a; + float inv_a = 65535.0f * (1 - ra); + pixel[0] = (stbi__uint16) (pixel[0]*ra + inv_a); + pixel[1] = (stbi__uint16) (pixel[1]*ra + inv_a); + pixel[2] = (stbi__uint16) (pixel[2]*ra + inv_a); + } + } + } else { + for (i=0; i < w*h; ++i) { + unsigned char *pixel = out + 4*i; + if (pixel[3] != 0 && pixel[3] != 255) { + float a = pixel[3] / 255.0f; + float ra = 1.0f / a; + float inv_a = 255.0f * (1 - ra); + pixel[0] = (unsigned char) (pixel[0]*ra + inv_a); + pixel[1] = (unsigned char) (pixel[1]*ra + inv_a); + pixel[2] = (unsigned char) (pixel[2]*ra + inv_a); + } + } + } + } + + // convert to desired output format + if (req_comp && req_comp != 4) { + if (ri->bits_per_channel == 16) + out = (stbi_uc *) stbi__convert_format16((stbi__uint16 *) out, 4, req_comp, w, h); + else + out = stbi__convert_format(out, 4, req_comp, w, h); + if (out == NULL) return out; // stbi__convert_format frees input on failure + } + + if (comp) *comp = 4; + *y = h; + *x = w; + + return out; +} +#endif + +// ************************************************************************************************* +// Softimage PIC loader +// by Tom Seddon +// +// See http://softimage.wiki.softimage.com/index.php/INFO:_PIC_file_format +// See http://ozviz.wasp.uwa.edu.au/~pbourke/dataformats/softimagepic/ + +#ifndef STBI_NO_PIC +static int stbi__pic_is4(stbi__context *s,const char *str) +{ + int i; + for (i=0; i<4; ++i) + if (stbi__get8(s) != (stbi_uc)str[i]) + return 0; + + return 1; +} + +static int stbi__pic_test_core(stbi__context *s) +{ + int i; + + if (!stbi__pic_is4(s,"\x53\x80\xF6\x34")) + return 0; + + for(i=0;i<84;++i) + stbi__get8(s); + + if (!stbi__pic_is4(s,"PICT")) + return 0; + + return 1; +} + +typedef struct +{ + stbi_uc size,type,channel; +} stbi__pic_packet; + +static stbi_uc *stbi__readval(stbi__context *s, int channel, stbi_uc *dest) +{ + int mask=0x80, i; + + for (i=0; i<4; ++i, mask>>=1) { + if (channel & mask) { + if (stbi__at_eof(s)) return stbi__errpuc("bad file","PIC file too short"); + dest[i]=stbi__get8(s); + } + } + + return dest; +} + +static void stbi__copyval(int channel,stbi_uc *dest,const stbi_uc *src) +{ + int mask=0x80,i; + + for (i=0;i<4; ++i, mask>>=1) + if (channel&mask) + dest[i]=src[i]; +} + +static stbi_uc *stbi__pic_load_core(stbi__context *s,int width,int height,int *comp, stbi_uc *result) +{ + int act_comp=0,num_packets=0,y,chained; + stbi__pic_packet packets[10]; + + // this will (should...) cater for even some bizarre stuff like having data + // for the same channel in multiple packets. + do { + stbi__pic_packet *packet; + + if (num_packets==sizeof(packets)/sizeof(packets[0])) + return stbi__errpuc("bad format","too many packets"); + + packet = &packets[num_packets++]; + + chained = stbi__get8(s); + packet->size = stbi__get8(s); + packet->type = stbi__get8(s); + packet->channel = stbi__get8(s); + + act_comp |= packet->channel; + + if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (reading packets)"); + if (packet->size != 8) return stbi__errpuc("bad format","packet isn't 8bpp"); + } while (chained); + + *comp = (act_comp & 0x10 ? 4 : 3); // has alpha channel? + + for(y=0; ytype) { + default: + return stbi__errpuc("bad format","packet has bad compression type"); + + case 0: {//uncompressed + int x; + + for(x=0;xchannel,dest)) + return 0; + break; + } + + case 1://Pure RLE + { + int left=width, i; + + while (left>0) { + stbi_uc count,value[4]; + + count=stbi__get8(s); + if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (pure read count)"); + + if (count > left) + count = (stbi_uc) left; + + if (!stbi__readval(s,packet->channel,value)) return 0; + + for(i=0; ichannel,dest,value); + left -= count; + } + } + break; + + case 2: {//Mixed RLE + int left=width; + while (left>0) { + int count = stbi__get8(s), i; + if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (mixed read count)"); + + if (count >= 128) { // Repeated + stbi_uc value[4]; + + if (count==128) + count = stbi__get16be(s); + else + count -= 127; + if (count > left) + return stbi__errpuc("bad file","scanline overrun"); + + if (!stbi__readval(s,packet->channel,value)) + return 0; + + for(i=0;ichannel,dest,value); + } else { // Raw + ++count; + if (count>left) return stbi__errpuc("bad file","scanline overrun"); + + for(i=0;ichannel,dest)) + return 0; + } + left-=count; + } + break; + } + } + } + } + + return result; +} + +static void *stbi__pic_load(stbi__context *s,int *px,int *py,int *comp,int req_comp, stbi__result_info *ri) +{ + stbi_uc *result; + int i, x,y, internal_comp; + STBI_NOTUSED(ri); + + if (!comp) comp = &internal_comp; + + for (i=0; i<92; ++i) + stbi__get8(s); + + x = stbi__get16be(s); + y = stbi__get16be(s); + if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (pic header)"); + if (!stbi__mad3sizes_valid(x, y, 4, 0)) return stbi__errpuc("too large", "PIC image too large to decode"); + + stbi__get32be(s); //skip `ratio' + stbi__get16be(s); //skip `fields' + stbi__get16be(s); //skip `pad' + + // intermediate buffer is RGBA + result = (stbi_uc *) stbi__malloc_mad3(x, y, 4, 0); + memset(result, 0xff, x*y*4); + + if (!stbi__pic_load_core(s,x,y,comp, result)) { + STBI_FREE(result); + result=0; + } + *px = x; + *py = y; + if (req_comp == 0) req_comp = *comp; + result=stbi__convert_format(result,4,req_comp,x,y); + + return result; +} + +static int stbi__pic_test(stbi__context *s) +{ + int r = stbi__pic_test_core(s); + stbi__rewind(s); + return r; +} +#endif + +// ************************************************************************************************* +// GIF loader -- public domain by Jean-Marc Lienher -- simplified/shrunk by stb + +#ifndef STBI_NO_GIF +typedef struct +{ + stbi__int16 prefix; + stbi_uc first; + stbi_uc suffix; +} stbi__gif_lzw; + +typedef struct +{ + int w,h; + stbi_uc *out; // output buffer (always 4 components) + stbi_uc *background; // The current "background" as far as a gif is concerned + stbi_uc *history; + int flags, bgindex, ratio, transparent, eflags; + stbi_uc pal[256][4]; + stbi_uc lpal[256][4]; + stbi__gif_lzw codes[8192]; + stbi_uc *color_table; + int parse, step; + int lflags; + int start_x, start_y; + int max_x, max_y; + int cur_x, cur_y; + int line_size; + int delay; +} stbi__gif; + +static int stbi__gif_test_raw(stbi__context *s) +{ + int sz; + if (stbi__get8(s) != 'G' || stbi__get8(s) != 'I' || stbi__get8(s) != 'F' || stbi__get8(s) != '8') return 0; + sz = stbi__get8(s); + if (sz != '9' && sz != '7') return 0; + if (stbi__get8(s) != 'a') return 0; + return 1; +} + +static int stbi__gif_test(stbi__context *s) +{ + int r = stbi__gif_test_raw(s); + stbi__rewind(s); + return r; +} + +static void stbi__gif_parse_colortable(stbi__context *s, stbi_uc pal[256][4], int num_entries, int transp) +{ + int i; + for (i=0; i < num_entries; ++i) { + pal[i][2] = stbi__get8(s); + pal[i][1] = stbi__get8(s); + pal[i][0] = stbi__get8(s); + pal[i][3] = transp == i ? 0 : 255; + } +} + +static int stbi__gif_header(stbi__context *s, stbi__gif *g, int *comp, int is_info) +{ + stbi_uc version; + if (stbi__get8(s) != 'G' || stbi__get8(s) != 'I' || stbi__get8(s) != 'F' || stbi__get8(s) != '8') + return stbi__err("not GIF", "Corrupt GIF"); + + version = stbi__get8(s); + if (version != '7' && version != '9') return stbi__err("not GIF", "Corrupt GIF"); + if (stbi__get8(s) != 'a') return stbi__err("not GIF", "Corrupt GIF"); + + stbi__g_failure_reason = ""; + g->w = stbi__get16le(s); + g->h = stbi__get16le(s); + g->flags = stbi__get8(s); + g->bgindex = stbi__get8(s); + g->ratio = stbi__get8(s); + g->transparent = -1; + + if (comp != 0) *comp = 4; // can't actually tell whether it's 3 or 4 until we parse the comments + + if (is_info) return 1; + + if (g->flags & 0x80) + stbi__gif_parse_colortable(s,g->pal, 2 << (g->flags & 7), -1); + + return 1; +} + +static int stbi__gif_info_raw(stbi__context *s, int *x, int *y, int *comp) +{ + stbi__gif* g = (stbi__gif*) stbi__malloc(sizeof(stbi__gif)); + if (!stbi__gif_header(s, g, comp, 1)) { + STBI_FREE(g); + stbi__rewind( s ); + return 0; + } + if (x) *x = g->w; + if (y) *y = g->h; + STBI_FREE(g); + return 1; +} + +static void stbi__out_gif_code(stbi__gif *g, stbi__uint16 code) +{ + stbi_uc *p, *c; + int idx; + + // recurse to decode the prefixes, since the linked-list is backwards, + // and working backwards through an interleaved image would be nasty + if (g->codes[code].prefix >= 0) + stbi__out_gif_code(g, g->codes[code].prefix); + + if (g->cur_y >= g->max_y) return; + + idx = g->cur_x + g->cur_y; + p = &g->out[idx]; + g->history[idx / 4] = 1; + + c = &g->color_table[g->codes[code].suffix * 4]; + if (c[3] > 128) { // don't render transparent pixels; + p[0] = c[2]; + p[1] = c[1]; + p[2] = c[0]; + p[3] = c[3]; + } + g->cur_x += 4; + + if (g->cur_x >= g->max_x) { + g->cur_x = g->start_x; + g->cur_y += g->step; + + while (g->cur_y >= g->max_y && g->parse > 0) { + g->step = (1 << g->parse) * g->line_size; + g->cur_y = g->start_y + (g->step >> 1); + --g->parse; + } + } +} + +static stbi_uc *stbi__process_gif_raster(stbi__context *s, stbi__gif *g) +{ + stbi_uc lzw_cs; + stbi__int32 len, init_code; + stbi__uint32 first; + stbi__int32 codesize, codemask, avail, oldcode, bits, valid_bits, clear; + stbi__gif_lzw *p; + + lzw_cs = stbi__get8(s); + if (lzw_cs > 12) return NULL; + clear = 1 << lzw_cs; + first = 1; + codesize = lzw_cs + 1; + codemask = (1 << codesize) - 1; + bits = 0; + valid_bits = 0; + for (init_code = 0; init_code < clear; init_code++) { + g->codes[init_code].prefix = -1; + g->codes[init_code].first = (stbi_uc) init_code; + g->codes[init_code].suffix = (stbi_uc) init_code; + } + + // support no starting clear code + avail = clear+2; + oldcode = -1; + + len = 0; + for(;;) { + if (valid_bits < codesize) { + if (len == 0) { + len = stbi__get8(s); // start new block + if (len == 0) + return g->out; + } + --len; + bits |= (stbi__int32) stbi__get8(s) << valid_bits; + valid_bits += 8; + } else { + stbi__int32 code = bits & codemask; + bits >>= codesize; + valid_bits -= codesize; + // @OPTIMIZE: is there some way we can accelerate the non-clear path? + if (code == clear) { // clear code + codesize = lzw_cs + 1; + codemask = (1 << codesize) - 1; + avail = clear + 2; + oldcode = -1; + first = 0; + } else if (code == clear + 1) { // end of stream code + stbi__skip(s, len); + while ((len = stbi__get8(s)) > 0) + stbi__skip(s,len); + return g->out; + } else if (code <= avail) { + if (first) { + return stbi__errpuc("no clear code", "Corrupt GIF"); + } + + if (oldcode >= 0) { + p = &g->codes[avail++]; + if (avail > 8192) { + return stbi__errpuc("too many codes", "Corrupt GIF"); + } + + p->prefix = (stbi__int16) oldcode; + p->first = g->codes[oldcode].first; + p->suffix = (code == avail) ? p->first : g->codes[code].first; + } else if (code == avail) + return stbi__errpuc("illegal code in raster", "Corrupt GIF"); + + stbi__out_gif_code(g, (stbi__uint16) code); + + if ((avail & codemask) == 0 && avail <= 0x0FFF) { + codesize++; + codemask = (1 << codesize) - 1; + } + + oldcode = code; + } else { + return stbi__errpuc("illegal code in raster", "Corrupt GIF"); + } + } + } +} + +// this function is designed to support animated gifs, although stb_image doesn't support it +// two back is the image from two frames ago, used for a very specific disposal format +static stbi_uc *stbi__gif_load_next(stbi__context *s, stbi__gif *g, int *comp, int req_comp, stbi_uc *two_back) +{ + int dispose; + int first_frame; + int pi; + int pcount; + STBI_NOTUSED(req_comp); + + // on first frame, any non-written pixels get the background colour (non-transparent) + first_frame = 0; + if (g->out == 0) { + if (!stbi__gif_header(s, g, comp,0)) return 0; // stbi__g_failure_reason set by stbi__gif_header + g->out = (stbi_uc *) stbi__malloc(4 * g->w * g->h); + g->background = (stbi_uc *) stbi__malloc(4 * g->w * g->h); + g->history = (stbi_uc *) stbi__malloc(g->w * g->h); + if (g->out == 0) return stbi__errpuc("outofmem", "Out of memory"); + + // image is treated as "transparent" at the start - ie, nothing overwrites the current background; + // background colour is only used for pixels that are not rendered first frame, after that "background" + // color refers to the color that was there the previous frame. + memset( g->out, 0x00, 4 * g->w * g->h ); + memset( g->background, 0x00, 4 * g->w * g->h ); // state of the background (starts transparent) + memset( g->history, 0x00, g->w * g->h ); // pixels that were affected previous frame + first_frame = 1; + } else { + // second frame - how do we dispoase of the previous one? + dispose = (g->eflags & 0x1C) >> 2; + pcount = g->w * g->h; + + if ((dispose == 3) && (two_back == 0)) { + dispose = 2; // if I don't have an image to revert back to, default to the old background + } + + if (dispose == 3) { // use previous graphic + for (pi = 0; pi < pcount; ++pi) { + if (g->history[pi]) { + memcpy( &g->out[pi * 4], &two_back[pi * 4], 4 ); + } + } + } else if (dispose == 2) { + // restore what was changed last frame to background before that frame; + for (pi = 0; pi < pcount; ++pi) { + if (g->history[pi]) { + memcpy( &g->out[pi * 4], &g->background[pi * 4], 4 ); + } + } + } else { + // This is a non-disposal case eithe way, so just + // leave the pixels as is, and they will become the new background + // 1: do not dispose + // 0: not specified. + } + + // background is what out is after the undoing of the previou frame; + memcpy( g->background, g->out, 4 * g->w * g->h ); + } + + // clear my history; + memset( g->history, 0x00, g->w * g->h ); // pixels that were affected previous frame + + for (;;) { + int tag = stbi__get8(s); + switch (tag) { + case 0x2C: /* Image Descriptor */ + { + stbi__int32 x, y, w, h; + stbi_uc *o; + + x = stbi__get16le(s); + y = stbi__get16le(s); + w = stbi__get16le(s); + h = stbi__get16le(s); + if (((x + w) > (g->w)) || ((y + h) > (g->h))) + return stbi__errpuc("bad Image Descriptor", "Corrupt GIF"); + + g->line_size = g->w * 4; + g->start_x = x * 4; + g->start_y = y * g->line_size; + g->max_x = g->start_x + w * 4; + g->max_y = g->start_y + h * g->line_size; + g->cur_x = g->start_x; + g->cur_y = g->start_y; + + g->lflags = stbi__get8(s); + + if (g->lflags & 0x40) { + g->step = 8 * g->line_size; // first interlaced spacing + g->parse = 3; + } else { + g->step = g->line_size; + g->parse = 0; + } + + if (g->lflags & 0x80) { + stbi__gif_parse_colortable(s,g->lpal, 2 << (g->lflags & 7), g->eflags & 0x01 ? g->transparent : -1); + g->color_table = (stbi_uc *) g->lpal; + } else if (g->flags & 0x80) { + g->color_table = (stbi_uc *) g->pal; + } else + return stbi__errpuc("missing color table", "Corrupt GIF"); + + o = stbi__process_gif_raster(s, g); + if (o == NULL) return NULL; + + // if this was the first frame, + pcount = g->w * g->h; + if (first_frame && (g->bgindex > 0)) { + // if first frame, any pixel not drawn to gets the background color + for (pi = 0; pi < pcount; ++pi) { + if (g->history[pi] == 0) { + g->pal[g->bgindex][3] = 255; // just in case it was made transparent, undo that; It will be reset next frame if need be; + memcpy( &g->out[pi * 4], &g->pal[g->bgindex], 4 ); + } + } + } + + return o; + } + + case 0x21: // Comment Extension. + { + int len; + int ext = stbi__get8(s); + if (ext == 0xF9) { // Graphic Control Extension. + len = stbi__get8(s); + if (len == 4) { + g->eflags = stbi__get8(s); + g->delay = 10 * stbi__get16le(s); // delay - 1/100th of a second, saving as 1/1000ths. + + // unset old transparent + if (g->transparent >= 0) { + g->pal[g->transparent][3] = 255; + } + if (g->eflags & 0x01) { + g->transparent = stbi__get8(s); + if (g->transparent >= 0) { + g->pal[g->transparent][3] = 0; + } + } else { + // don't need transparent + stbi__skip(s, 1); + g->transparent = -1; + } + } else { + stbi__skip(s, len); + break; + } + } + while ((len = stbi__get8(s)) != 0) { + stbi__skip(s, len); + } + break; + } + + case 0x3B: // gif stream termination code + return (stbi_uc *) s; // using '1' causes warning on some compilers + + default: + return stbi__errpuc("unknown code", "Corrupt GIF"); + } + } +} + +static void *stbi__load_gif_main(stbi__context *s, int **delays, int *x, int *y, int *z, int *comp, int req_comp) +{ + if (stbi__gif_test(s)) { + int layers = 0; + stbi_uc *u = 0; + stbi_uc *out = 0; + stbi_uc *two_back = 0; + stbi__gif g; + int stride; + memset(&g, 0, sizeof(g)); + if (delays) { + *delays = 0; + } + + do { + u = stbi__gif_load_next(s, &g, comp, req_comp, two_back); + if (u == (stbi_uc *) s) u = 0; // end of animated gif marker + + if (u) { + *x = g.w; + *y = g.h; + ++layers; + stride = g.w * g.h * 4; + + if (out) { + out = (stbi_uc*) STBI_REALLOC( out, layers * stride ); + if (delays) { + *delays = (int*) STBI_REALLOC( *delays, sizeof(int) * layers ); + } + } else { + out = (stbi_uc*)stbi__malloc( layers * stride ); + if (delays) { + *delays = (int*) stbi__malloc( layers * sizeof(int) ); + } + } + memcpy( out + ((layers - 1) * stride), u, stride ); + if (layers >= 2) { + two_back = out - 2 * stride; + } + + if (delays) { + (*delays)[layers - 1U] = g.delay; + } + } + } while (u != 0); + + // free temp buffer; + STBI_FREE(g.out); + STBI_FREE(g.history); + STBI_FREE(g.background); + + // do the final conversion after loading everything; + if (req_comp && req_comp != 4) + out = stbi__convert_format(out, 4, req_comp, layers * g.w, g.h); + + *z = layers; + return out; + } else { + return stbi__errpuc("not GIF", "Image was not as a gif type."); + } +} + +static void *stbi__gif_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + stbi_uc *u = 0; + stbi__gif g; + memset(&g, 0, sizeof(g)); + STBI_NOTUSED(ri); + + u = stbi__gif_load_next(s, &g, comp, req_comp, 0); + if (u == (stbi_uc *) s) u = 0; // end of animated gif marker + if (u) { + *x = g.w; + *y = g.h; + + // moved conversion to after successful load so that the same + // can be done for multiple frames. + if (req_comp && req_comp != 4) + u = stbi__convert_format(u, 4, req_comp, g.w, g.h); + } + + // free buffers needed for multiple frame loading; + STBI_FREE(g.history); + STBI_FREE(g.background); + + return u; +} + +static int stbi__gif_info(stbi__context *s, int *x, int *y, int *comp) +{ + return stbi__gif_info_raw(s,x,y,comp); +} +#endif + +// ************************************************************************************************* +// Radiance RGBE HDR loader +// originally by Nicolas Schulz +#ifndef STBI_NO_HDR +static int stbi__hdr_test_core(stbi__context *s, const char *signature) +{ + int i; + for (i=0; signature[i]; ++i) + if (stbi__get8(s) != signature[i]) + return 0; + stbi__rewind(s); + return 1; +} + +static int stbi__hdr_test(stbi__context* s) +{ + int r = stbi__hdr_test_core(s, "#?RADIANCE\n"); + stbi__rewind(s); + if(!r) { + r = stbi__hdr_test_core(s, "#?RGBE\n"); + stbi__rewind(s); + } + return r; +} + +#define STBI__HDR_BUFLEN 1024 +static char *stbi__hdr_gettoken(stbi__context *z, char *buffer) +{ + int len=0; + char c = '\0'; + + c = (char) stbi__get8(z); + + while (!stbi__at_eof(z) && c != '\n') { + buffer[len++] = c; + if (len == STBI__HDR_BUFLEN-1) { + // flush to end of line + while (!stbi__at_eof(z) && stbi__get8(z) != '\n') + ; + break; + } + c = (char) stbi__get8(z); + } + + buffer[len] = 0; + return buffer; +} + +static void stbi__hdr_convert(float *output, stbi_uc *input, int req_comp) +{ + if ( input[3] != 0 ) { + float f1; + // Exponent + f1 = (float) ldexp(1.0f, input[3] - (int)(128 + 8)); + if (req_comp <= 2) + output[0] = (input[0] + input[1] + input[2]) * f1 / 3; + else { + output[0] = input[0] * f1; + output[1] = input[1] * f1; + output[2] = input[2] * f1; + } + if (req_comp == 2) output[1] = 1; + if (req_comp == 4) output[3] = 1; + } else { + switch (req_comp) { + case 4: output[3] = 1; /* fallthrough */ + case 3: output[0] = output[1] = output[2] = 0; + break; + case 2: output[1] = 1; /* fallthrough */ + case 1: output[0] = 0; + break; + } + } +} + +static float *stbi__hdr_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + char buffer[STBI__HDR_BUFLEN]; + char *token; + int valid = 0; + int width, height; + stbi_uc *scanline; + float *hdr_data; + int len; + unsigned char count, value; + int i, j, k, c1,c2, z; + const char *headerToken; + STBI_NOTUSED(ri); + + // Check identifier + headerToken = stbi__hdr_gettoken(s,buffer); + if (strcmp(headerToken, "#?RADIANCE") != 0 && strcmp(headerToken, "#?RGBE") != 0) + return stbi__errpf("not HDR", "Corrupt HDR image"); + + // Parse header + for(;;) { + token = stbi__hdr_gettoken(s,buffer); + if (token[0] == 0) break; + if (strcmp(token, "FORMAT=32-bit_rle_rgbe") == 0) valid = 1; + } + + if (!valid) return stbi__errpf("unsupported format", "Unsupported HDR format"); + + // Parse width and height + // can't use sscanf() if we're not using stdio! + token = stbi__hdr_gettoken(s,buffer); + if (strncmp(token, "-Y ", 3)) return stbi__errpf("unsupported data layout", "Unsupported HDR format"); + token += 3; + height = (int) strtol(token, &token, 10); + while (*token == ' ') ++token; + if (strncmp(token, "+X ", 3)) return stbi__errpf("unsupported data layout", "Unsupported HDR format"); + token += 3; + width = (int) strtol(token, NULL, 10); + + *x = width; + *y = height; + + if (comp) *comp = 3; + if (req_comp == 0) req_comp = 3; + + if (!stbi__mad4sizes_valid(width, height, req_comp, sizeof(float), 0)) + return stbi__errpf("too large", "HDR image is too large"); + + // Read data + hdr_data = (float *) stbi__malloc_mad4(width, height, req_comp, sizeof(float), 0); + if (!hdr_data) + return stbi__errpf("outofmem", "Out of memory"); + + // Load image data + // image data is stored as some number of sca + if ( width < 8 || width >= 32768) { + // Read flat data + for (j=0; j < height; ++j) { + for (i=0; i < width; ++i) { + stbi_uc rgbe[4]; + main_decode_loop: + stbi__getn(s, rgbe, 4); + stbi__hdr_convert(hdr_data + j * width * req_comp + i * req_comp, rgbe, req_comp); + } + } + } else { + // Read RLE-encoded data + scanline = NULL; + + for (j = 0; j < height; ++j) { + c1 = stbi__get8(s); + c2 = stbi__get8(s); + len = stbi__get8(s); + if (c1 != 2 || c2 != 2 || (len & 0x80)) { + // not run-length encoded, so we have to actually use THIS data as a decoded + // pixel (note this can't be a valid pixel--one of RGB must be >= 128) + stbi_uc rgbe[4]; + rgbe[0] = (stbi_uc) c1; + rgbe[1] = (stbi_uc) c2; + rgbe[2] = (stbi_uc) len; + rgbe[3] = (stbi_uc) stbi__get8(s); + stbi__hdr_convert(hdr_data, rgbe, req_comp); + i = 1; + j = 0; + STBI_FREE(scanline); + goto main_decode_loop; // yes, this makes no sense + } + len <<= 8; + len |= stbi__get8(s); + if (len != width) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf("invalid decoded scanline length", "corrupt HDR"); } + if (scanline == NULL) { + scanline = (stbi_uc *) stbi__malloc_mad2(width, 4, 0); + if (!scanline) { + STBI_FREE(hdr_data); + return stbi__errpf("outofmem", "Out of memory"); + } + } + + for (k = 0; k < 4; ++k) { + int nleft; + i = 0; + while ((nleft = width - i) > 0) { + count = stbi__get8(s); + if (count > 128) { + // Run + value = stbi__get8(s); + count -= 128; + if (count > nleft) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf("corrupt", "bad RLE data in HDR"); } + for (z = 0; z < count; ++z) + scanline[i++ * 4 + k] = value; + } else { + // Dump + if (count > nleft) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf("corrupt", "bad RLE data in HDR"); } + for (z = 0; z < count; ++z) + scanline[i++ * 4 + k] = stbi__get8(s); + } + } + } + for (i=0; i < width; ++i) + stbi__hdr_convert(hdr_data+(j*width + i)*req_comp, scanline + i*4, req_comp); + } + if (scanline) + STBI_FREE(scanline); + } + + return hdr_data; +} + +static int stbi__hdr_info(stbi__context *s, int *x, int *y, int *comp) +{ + char buffer[STBI__HDR_BUFLEN]; + char *token; + int valid = 0; + int dummy; + + if (!x) x = &dummy; + if (!y) y = &dummy; + if (!comp) comp = &dummy; + + if (stbi__hdr_test(s) == 0) { + stbi__rewind( s ); + return 0; + } + + for(;;) { + token = stbi__hdr_gettoken(s,buffer); + if (token[0] == 0) break; + if (strcmp(token, "FORMAT=32-bit_rle_rgbe") == 0) valid = 1; + } + + if (!valid) { + stbi__rewind( s ); + return 0; + } + token = stbi__hdr_gettoken(s,buffer); + if (strncmp(token, "-Y ", 3)) { + stbi__rewind( s ); + return 0; + } + token += 3; + *y = (int) strtol(token, &token, 10); + while (*token == ' ') ++token; + if (strncmp(token, "+X ", 3)) { + stbi__rewind( s ); + return 0; + } + token += 3; + *x = (int) strtol(token, NULL, 10); + *comp = 3; + return 1; +} +#endif // STBI_NO_HDR + +#ifndef STBI_NO_BMP +static int stbi__bmp_info(stbi__context *s, int *x, int *y, int *comp) +{ + void *p; + stbi__bmp_data info; + + info.all_a = 255; + p = stbi__bmp_parse_header(s, &info); + stbi__rewind( s ); + if (p == NULL) + return 0; + if (x) *x = s->img_x; + if (y) *y = s->img_y; + if (comp) *comp = info.ma ? 4 : 3; + return 1; +} +#endif + +#ifndef STBI_NO_PSD +static int stbi__psd_info(stbi__context *s, int *x, int *y, int *comp) +{ + int channelCount, dummy, depth; + if (!x) x = &dummy; + if (!y) y = &dummy; + if (!comp) comp = &dummy; + if (stbi__get32be(s) != 0x38425053) { + stbi__rewind( s ); + return 0; + } + if (stbi__get16be(s) != 1) { + stbi__rewind( s ); + return 0; + } + stbi__skip(s, 6); + channelCount = stbi__get16be(s); + if (channelCount < 0 || channelCount > 16) { + stbi__rewind( s ); + return 0; + } + *y = stbi__get32be(s); + *x = stbi__get32be(s); + depth = stbi__get16be(s); + if (depth != 8 && depth != 16) { + stbi__rewind( s ); + return 0; + } + if (stbi__get16be(s) != 3) { + stbi__rewind( s ); + return 0; + } + *comp = 4; + return 1; +} + +static int stbi__psd_is16(stbi__context *s) +{ + int channelCount, depth; + if (stbi__get32be(s) != 0x38425053) { + stbi__rewind( s ); + return 0; + } + if (stbi__get16be(s) != 1) { + stbi__rewind( s ); + return 0; + } + stbi__skip(s, 6); + channelCount = stbi__get16be(s); + if (channelCount < 0 || channelCount > 16) { + stbi__rewind( s ); + return 0; + } + (void) stbi__get32be(s); + (void) stbi__get32be(s); + depth = stbi__get16be(s); + if (depth != 16) { + stbi__rewind( s ); + return 0; + } + return 1; +} +#endif + +#ifndef STBI_NO_PIC +static int stbi__pic_info(stbi__context *s, int *x, int *y, int *comp) +{ + int act_comp=0,num_packets=0,chained,dummy; + stbi__pic_packet packets[10]; + + if (!x) x = &dummy; + if (!y) y = &dummy; + if (!comp) comp = &dummy; + + if (!stbi__pic_is4(s,"\x53\x80\xF6\x34")) { + stbi__rewind(s); + return 0; + } + + stbi__skip(s, 88); + + *x = stbi__get16be(s); + *y = stbi__get16be(s); + if (stbi__at_eof(s)) { + stbi__rewind( s); + return 0; + } + if ( (*x) != 0 && (1 << 28) / (*x) < (*y)) { + stbi__rewind( s ); + return 0; + } + + stbi__skip(s, 8); + + do { + stbi__pic_packet *packet; + + if (num_packets==sizeof(packets)/sizeof(packets[0])) + return 0; + + packet = &packets[num_packets++]; + chained = stbi__get8(s); + packet->size = stbi__get8(s); + packet->type = stbi__get8(s); + packet->channel = stbi__get8(s); + act_comp |= packet->channel; + + if (stbi__at_eof(s)) { + stbi__rewind( s ); + return 0; + } + if (packet->size != 8) { + stbi__rewind( s ); + return 0; + } + } while (chained); + + *comp = (act_comp & 0x10 ? 4 : 3); + + return 1; +} +#endif + +// ************************************************************************************************* +// Portable Gray Map and Portable Pixel Map loader +// by Ken Miller +// +// PGM: http://netpbm.sourceforge.net/doc/pgm.html +// PPM: http://netpbm.sourceforge.net/doc/ppm.html +// +// Known limitations: +// Does not support comments in the header section +// Does not support ASCII image data (formats P2 and P3) +// Does not support 16-bit-per-channel + +#ifndef STBI_NO_PNM + +static int stbi__pnm_test(stbi__context *s) +{ + char p, t; + p = (char) stbi__get8(s); + t = (char) stbi__get8(s); + if (p != 'P' || (t != '5' && t != '6')) { + stbi__rewind( s ); + return 0; + } + return 1; +} + +static void *stbi__pnm_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) +{ + stbi_uc *out; + STBI_NOTUSED(ri); + + if (!stbi__pnm_info(s, (int *)&s->img_x, (int *)&s->img_y, (int *)&s->img_n)) + return 0; + + *x = s->img_x; + *y = s->img_y; + if (comp) *comp = s->img_n; + + if (!stbi__mad3sizes_valid(s->img_n, s->img_x, s->img_y, 0)) + return stbi__errpuc("too large", "PNM too large"); + + out = (stbi_uc *) stbi__malloc_mad3(s->img_n, s->img_x, s->img_y, 0); + if (!out) return stbi__errpuc("outofmem", "Out of memory"); + stbi__getn(s, out, s->img_n * s->img_x * s->img_y); + + if (req_comp && req_comp != s->img_n) { + out = stbi__convert_format(out, s->img_n, req_comp, s->img_x, s->img_y); + if (out == NULL) return out; // stbi__convert_format frees input on failure + } + return out; +} + +static int stbi__pnm_isspace(char c) +{ + return c == ' ' || c == '\t' || c == '\n' || c == '\v' || c == '\f' || c == '\r'; +} + +static void stbi__pnm_skip_whitespace(stbi__context *s, char *c) +{ + for (;;) { + while (!stbi__at_eof(s) && stbi__pnm_isspace(*c)) + *c = (char) stbi__get8(s); + + if (stbi__at_eof(s) || *c != '#') + break; + + while (!stbi__at_eof(s) && *c != '\n' && *c != '\r' ) + *c = (char) stbi__get8(s); + } +} + +static int stbi__pnm_isdigit(char c) +{ + return c >= '0' && c <= '9'; +} + +static int stbi__pnm_getinteger(stbi__context *s, char *c) +{ + int value = 0; + + while (!stbi__at_eof(s) && stbi__pnm_isdigit(*c)) { + value = value*10 + (*c - '0'); + *c = (char) stbi__get8(s); + } + + return value; +} + +static int stbi__pnm_info(stbi__context *s, int *x, int *y, int *comp) +{ + int maxv, dummy; + char c, p, t; + + if (!x) x = &dummy; + if (!y) y = &dummy; + if (!comp) comp = &dummy; + + stbi__rewind(s); + + // Get identifier + p = (char) stbi__get8(s); + t = (char) stbi__get8(s); + if (p != 'P' || (t != '5' && t != '6')) { + stbi__rewind(s); + return 0; + } + + *comp = (t == '6') ? 3 : 1; // '5' is 1-component .pgm; '6' is 3-component .ppm + + c = (char) stbi__get8(s); + stbi__pnm_skip_whitespace(s, &c); + + *x = stbi__pnm_getinteger(s, &c); // read width + stbi__pnm_skip_whitespace(s, &c); + + *y = stbi__pnm_getinteger(s, &c); // read height + stbi__pnm_skip_whitespace(s, &c); + + maxv = stbi__pnm_getinteger(s, &c); // read max value + + if (maxv > 255) + return stbi__err("max value > 255", "PPM image not 8-bit"); + else + return 1; +} +#endif + +static int stbi__info_main(stbi__context *s, int *x, int *y, int *comp) +{ + #ifndef STBI_NO_JPEG + if (stbi__jpeg_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_PNG + if (stbi__png_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_GIF + if (stbi__gif_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_BMP + if (stbi__bmp_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_PSD + if (stbi__psd_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_PIC + if (stbi__pic_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_PNM + if (stbi__pnm_info(s, x, y, comp)) return 1; + #endif + + #ifndef STBI_NO_HDR + if (stbi__hdr_info(s, x, y, comp)) return 1; + #endif + + // test tga last because it's a crappy test! + #ifndef STBI_NO_TGA + if (stbi__tga_info(s, x, y, comp)) + return 1; + #endif + return stbi__err("unknown image type", "Image not of any known type, or corrupt"); +} + +static int stbi__is_16_main(stbi__context *s) +{ + #ifndef STBI_NO_PNG + if (stbi__png_is16(s)) return 1; + #endif + + #ifndef STBI_NO_PSD + if (stbi__psd_is16(s)) return 1; + #endif + + return 0; +} + +#ifndef STBI_NO_STDIO +STBIDEF int stbi_info(char const *filename, int *x, int *y, int *comp) +{ + FILE *f = stbi__fopen(filename, "rb"); + int result; + if (!f) return stbi__err("can't fopen", "Unable to open file"); + result = stbi_info_from_file(f, x, y, comp); + fclose(f); + return result; +} + +STBIDEF int stbi_info_from_file(FILE *f, int *x, int *y, int *comp) +{ + int r; + stbi__context s; + long pos = ftell(f); + stbi__start_file(&s, f); + r = stbi__info_main(&s,x,y,comp); + fseek(f,pos,SEEK_SET); + return r; +} + +STBIDEF int stbi_is_16_bit(char const *filename) +{ + FILE *f = stbi__fopen(filename, "rb"); + int result; + if (!f) return stbi__err("can't fopen", "Unable to open file"); + result = stbi_is_16_bit_from_file(f); + fclose(f); + return result; +} + +STBIDEF int stbi_is_16_bit_from_file(FILE *f) +{ + int r; + stbi__context s; + long pos = ftell(f); + stbi__start_file(&s, f); + r = stbi__is_16_main(&s); + fseek(f,pos,SEEK_SET); + return r; +} +#endif // !STBI_NO_STDIO + +STBIDEF int stbi_info_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__info_main(&s,x,y,comp); +} + +STBIDEF int stbi_info_from_callbacks(stbi_io_callbacks const *c, void *user, int *x, int *y, int *comp) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) c, user); + return stbi__info_main(&s,x,y,comp); +} + +STBIDEF int stbi_is_16_bit_from_memory(stbi_uc const *buffer, int len) +{ + stbi__context s; + stbi__start_mem(&s,buffer,len); + return stbi__is_16_main(&s); +} + +STBIDEF int stbi_is_16_bit_from_callbacks(stbi_io_callbacks const *c, void *user) +{ + stbi__context s; + stbi__start_callbacks(&s, (stbi_io_callbacks *) c, user); + return stbi__is_16_main(&s); +} + +#endif // STB_IMAGE_IMPLEMENTATION + +/* + revision history: + 2.20 (2019-02-07) support utf8 filenames in Windows; fix warnings and platform ifdefs + 2.19 (2018-02-11) fix warning + 2.18 (2018-01-30) fix warnings + 2.17 (2018-01-29) change sbti__shiftsigned to avoid clang -O2 bug + 1-bit BMP + *_is_16_bit api + avoid warnings + 2.16 (2017-07-23) all functions have 16-bit variants; + STBI_NO_STDIO works again; + compilation fixes; + fix rounding in unpremultiply; + optimize vertical flip; + disable raw_len validation; + documentation fixes + 2.15 (2017-03-18) fix png-1,2,4 bug; now all Imagenet JPGs decode; + warning fixes; disable run-time SSE detection on gcc; + uniform handling of optional "return" values; + thread-safe initialization of zlib tables + 2.14 (2017-03-03) remove deprecated STBI_JPEG_OLD; fixes for Imagenet JPGs + 2.13 (2016-11-29) add 16-bit API, only supported for PNG right now + 2.12 (2016-04-02) fix typo in 2.11 PSD fix that caused crashes + 2.11 (2016-04-02) allocate large structures on the stack + remove white matting for transparent PSD + fix reported channel count for PNG & BMP + re-enable SSE2 in non-gcc 64-bit + support RGB-formatted JPEG + read 16-bit PNGs (only as 8-bit) + 2.10 (2016-01-22) avoid warning introduced in 2.09 by STBI_REALLOC_SIZED + 2.09 (2016-01-16) allow comments in PNM files + 16-bit-per-pixel TGA (not bit-per-component) + info() for TGA could break due to .hdr handling + info() for BMP to shares code instead of sloppy parse + can use STBI_REALLOC_SIZED if allocator doesn't support realloc + code cleanup + 2.08 (2015-09-13) fix to 2.07 cleanup, reading RGB PSD as RGBA + 2.07 (2015-09-13) fix compiler warnings + partial animated GIF support + limited 16-bpc PSD support + #ifdef unused functions + bug with < 92 byte PIC,PNM,HDR,TGA + 2.06 (2015-04-19) fix bug where PSD returns wrong '*comp' value + 2.05 (2015-04-19) fix bug in progressive JPEG handling, fix warning + 2.04 (2015-04-15) try to re-enable SIMD on MinGW 64-bit + 2.03 (2015-04-12) extra corruption checking (mmozeiko) + stbi_set_flip_vertically_on_load (nguillemot) + fix NEON support; fix mingw support + 2.02 (2015-01-19) fix incorrect assert, fix warning + 2.01 (2015-01-17) fix various warnings; suppress SIMD on gcc 32-bit without -msse2 + 2.00b (2014-12-25) fix STBI_MALLOC in progressive JPEG + 2.00 (2014-12-25) optimize JPG, including x86 SSE2 & NEON SIMD (ryg) + progressive JPEG (stb) + PGM/PPM support (Ken Miller) + STBI_MALLOC,STBI_REALLOC,STBI_FREE + GIF bugfix -- seemingly never worked + STBI_NO_*, STBI_ONLY_* + 1.48 (2014-12-14) fix incorrectly-named assert() + 1.47 (2014-12-14) 1/2/4-bit PNG support, both direct and paletted (Omar Cornut & stb) + optimize PNG (ryg) + fix bug in interlaced PNG with user-specified channel count (stb) + 1.46 (2014-08-26) + fix broken tRNS chunk (colorkey-style transparency) in non-paletted PNG + 1.45 (2014-08-16) + fix MSVC-ARM internal compiler error by wrapping malloc + 1.44 (2014-08-07) + various warning fixes from Ronny Chevalier + 1.43 (2014-07-15) + fix MSVC-only compiler problem in code changed in 1.42 + 1.42 (2014-07-09) + don't define _CRT_SECURE_NO_WARNINGS (affects user code) + fixes to stbi__cleanup_jpeg path + added STBI_ASSERT to avoid requiring assert.h + 1.41 (2014-06-25) + fix search&replace from 1.36 that messed up comments/error messages + 1.40 (2014-06-22) + fix gcc struct-initialization warning + 1.39 (2014-06-15) + fix to TGA optimization when req_comp != number of components in TGA; + fix to GIF loading because BMP wasn't rewinding (whoops, no GIFs in my test suite) + add support for BMP version 5 (more ignored fields) + 1.38 (2014-06-06) + suppress MSVC warnings on integer casts truncating values + fix accidental rename of 'skip' field of I/O + 1.37 (2014-06-04) + remove duplicate typedef + 1.36 (2014-06-03) + convert to header file single-file library + if de-iphone isn't set, load iphone images color-swapped instead of returning NULL + 1.35 (2014-05-27) + various warnings + fix broken STBI_SIMD path + fix bug where stbi_load_from_file no longer left file pointer in correct place + fix broken non-easy path for 32-bit BMP (possibly never used) + TGA optimization by Arseny Kapoulkine + 1.34 (unknown) + use STBI_NOTUSED in stbi__resample_row_generic(), fix one more leak in tga failure case + 1.33 (2011-07-14) + make stbi_is_hdr work in STBI_NO_HDR (as specified), minor compiler-friendly improvements + 1.32 (2011-07-13) + support for "info" function for all supported filetypes (SpartanJ) + 1.31 (2011-06-20) + a few more leak fixes, bug in PNG handling (SpartanJ) + 1.30 (2011-06-11) + added ability to load files via callbacks to accomidate custom input streams (Ben Wenger) + removed deprecated format-specific test/load functions + removed support for installable file formats (stbi_loader) -- would have been broken for IO callbacks anyway + error cases in bmp and tga give messages and don't leak (Raymond Barbiero, grisha) + fix inefficiency in decoding 32-bit BMP (David Woo) + 1.29 (2010-08-16) + various warning fixes from Aurelien Pocheville + 1.28 (2010-08-01) + fix bug in GIF palette transparency (SpartanJ) + 1.27 (2010-08-01) + cast-to-stbi_uc to fix warnings + 1.26 (2010-07-24) + fix bug in file buffering for PNG reported by SpartanJ + 1.25 (2010-07-17) + refix trans_data warning (Won Chun) + 1.24 (2010-07-12) + perf improvements reading from files on platforms with lock-heavy fgetc() + minor perf improvements for jpeg + deprecated type-specific functions so we'll get feedback if they're needed + attempt to fix trans_data warning (Won Chun) + 1.23 fixed bug in iPhone support + 1.22 (2010-07-10) + removed image *writing* support + stbi_info support from Jetro Lauha + GIF support from Jean-Marc Lienher + iPhone PNG-extensions from James Brown + warning-fixes from Nicolas Schulz and Janez Zemva (i.stbi__err. Janez (U+017D)emva) + 1.21 fix use of 'stbi_uc' in header (reported by jon blow) + 1.20 added support for Softimage PIC, by Tom Seddon + 1.19 bug in interlaced PNG corruption check (found by ryg) + 1.18 (2008-08-02) + fix a threading bug (local mutable static) + 1.17 support interlaced PNG + 1.16 major bugfix - stbi__convert_format converted one too many pixels + 1.15 initialize some fields for thread safety + 1.14 fix threadsafe conversion bug + header-file-only version (#define STBI_HEADER_FILE_ONLY before including) + 1.13 threadsafe + 1.12 const qualifiers in the API + 1.11 Support installable IDCT, colorspace conversion routines + 1.10 Fixes for 64-bit (don't use "unsigned long") + optimized upsampling by Fabian "ryg" Giesen + 1.09 Fix format-conversion for PSD code (bad global variables!) + 1.08 Thatcher Ulrich's PSD code integrated by Nicolas Schulz + 1.07 attempt to fix C++ warning/errors again + 1.06 attempt to fix C++ warning/errors again + 1.05 fix TGA loading to return correct *comp and use good luminance calc + 1.04 default float alpha is 1, not 255; use 'void *' for stbi_image_free + 1.03 bugfixes to STBI_NO_STDIO, STBI_NO_HDR + 1.02 support for (subset of) HDR files, float interface for preferred access to them + 1.01 fix bug: possible bug in handling right-side up bmps... not sure + fix bug: the stbi__bmp_load() and stbi__tga_load() functions didn't work at all + 1.00 interface to zlib that skips zlib header + 0.99 correct handling of alpha in palette + 0.98 TGA loader by lonesock; dynamically add loaders (untested) + 0.97 jpeg errors on too large a file; also catch another malloc failure + 0.96 fix detection of invalid v value - particleman@mollyrocket forum + 0.95 during header scan, seek to markers in case of padding + 0.94 STBI_NO_STDIO to disable stdio usage; rename all #defines the same + 0.93 handle jpegtran output; verbose errors + 0.92 read 4,8,16,24,32-bit BMP files of several formats + 0.91 output 24-bit Windows 3.0 BMP files + 0.90 fix a few more warnings; bump version number to approach 1.0 + 0.61 bugfixes due to Marc LeBlanc, Christopher Lloyd + 0.60 fix compiling as c++ + 0.59 fix warnings: merge Dave Moore's -Wall fixes + 0.58 fix bug: zlib uncompressed mode len/nlen was wrong endian + 0.57 fix bug: jpg last huffman symbol before marker was >9 bits but less than 16 available + 0.56 fix bug: zlib uncompressed mode len vs. nlen + 0.55 fix bug: restart_interval not initialized to 0 + 0.54 allow NULL for 'int *comp' + 0.53 fix bug in png 3->4; speedup png decoding + 0.52 png handles req_comp=3,4 directly; minor cleanup; jpeg comments + 0.51 obey req_comp requests, 1-component jpegs return as 1-component, + on 'test' only check type, not whether we support this variant + 0.50 (2006-11-19) + first released version +*/ + + +/* +------------------------------------------------------------------------------ +This software is available under 2 licenses -- choose whichever you prefer. +------------------------------------------------------------------------------ +ALTERNATIVE A - MIT License +Copyright (c) 2017 Sean Barrett +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +------------------------------------------------------------------------------ +ALTERNATIVE B - Public Domain (www.unlicense.org) +This is free and unencumbered software released into the public domain. +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this +software, either in source code form or as a compiled binary, for any purpose, +commercial or non-commercial, and by any means. +In jurisdictions that recognize copyright laws, the author or authors of this +software dedicate any and all copyright interest in the software to the public +domain. We make this dedication for the benefit of the public at large and to +the detriment of our heirs and successors. We intend this dedication to be an +overt act of relinquishment in perpetuity of all present and future rights to +this software under copyright law. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +------------------------------------------------------------------------------ +*/ diff --git a/v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image_resize.h b/v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image_resize.h new file mode 100644 index 0000000000..031ca99dc3 --- /dev/null +++ b/v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image_resize.h @@ -0,0 +1,2627 @@ +/* stb_image_resize - v0.95 - public domain image resizing + by Jorge L Rodriguez (@VinoBS) - 2014 + http://github.com/nothings/stb + + Written with emphasis on usability, portability, and efficiency. (No + SIMD or threads, so it be easily outperformed by libs that use those.) + Only scaling and translation is supported, no rotations or shears. + Easy API downsamples w/Mitchell filter, upsamples w/cubic interpolation. + + COMPILING & LINKING + In one C/C++ file that #includes this file, do this: + #define STB_IMAGE_RESIZE_IMPLEMENTATION + before the #include. That will create the implementation in that file. + + QUICKSTART + stbir_resize_uint8( input_pixels , in_w , in_h , 0, + output_pixels, out_w, out_h, 0, num_channels) + stbir_resize_float(...) + stbir_resize_uint8_srgb( input_pixels , in_w , in_h , 0, + output_pixels, out_w, out_h, 0, + num_channels , alpha_chan , 0) + stbir_resize_uint8_srgb_edgemode( + input_pixels , in_w , in_h , 0, + output_pixels, out_w, out_h, 0, + num_channels , alpha_chan , 0, STBIR_EDGE_CLAMP) + // WRAP/REFLECT/ZERO + + FULL API + See the "header file" section of the source for API documentation. + + ADDITIONAL DOCUMENTATION + + SRGB & FLOATING POINT REPRESENTATION + The sRGB functions presume IEEE floating point. If you do not have + IEEE floating point, define STBIR_NON_IEEE_FLOAT. This will use + a slower implementation. + + MEMORY ALLOCATION + The resize functions here perform a single memory allocation using + malloc. To control the memory allocation, before the #include that + triggers the implementation, do: + + #define STBIR_MALLOC(size,context) ... + #define STBIR_FREE(ptr,context) ... + + Each resize function makes exactly one call to malloc/free, so to use + temp memory, store the temp memory in the context and return that. + + ASSERT + Define STBIR_ASSERT(boolval) to override assert() and not use assert.h + + OPTIMIZATION + Define STBIR_SATURATE_INT to compute clamp values in-range using + integer operations instead of float operations. This may be faster + on some platforms. + + DEFAULT FILTERS + For functions which don't provide explicit control over what filters + to use, you can change the compile-time defaults with + + #define STBIR_DEFAULT_FILTER_UPSAMPLE STBIR_FILTER_something + #define STBIR_DEFAULT_FILTER_DOWNSAMPLE STBIR_FILTER_something + + See stbir_filter in the header-file section for the list of filters. + + NEW FILTERS + A number of 1D filter kernels are used. For a list of + supported filters see the stbir_filter enum. To add a new filter, + write a filter function and add it to stbir__filter_info_table. + + PROGRESS + For interactive use with slow resize operations, you can install + a progress-report callback: + + #define STBIR_PROGRESS_REPORT(val) some_func(val) + + The parameter val is a float which goes from 0 to 1 as progress is made. + + For example: + + static void my_progress_report(float progress); + #define STBIR_PROGRESS_REPORT(val) my_progress_report(val) + + #define STB_IMAGE_RESIZE_IMPLEMENTATION + #include "stb_image_resize.h" + + static void my_progress_report(float progress) + { + printf("Progress: %f%%\n", progress*100); + } + + MAX CHANNELS + If your image has more than 64 channels, define STBIR_MAX_CHANNELS + to the max you'll have. + + ALPHA CHANNEL + Most of the resizing functions provide the ability to control how + the alpha channel of an image is processed. The important things + to know about this: + + 1. The best mathematically-behaved version of alpha to use is + called "premultiplied alpha", in which the other color channels + have had the alpha value multiplied in. If you use premultiplied + alpha, linear filtering (such as image resampling done by this + library, or performed in texture units on GPUs) does the "right + thing". While premultiplied alpha is standard in the movie CGI + industry, it is still uncommon in the videogame/real-time world. + + If you linearly filter non-premultiplied alpha, strange effects + occur. (For example, the 50/50 average of 99% transparent bright green + and 1% transparent black produces 50% transparent dark green when + non-premultiplied, whereas premultiplied it produces 50% + transparent near-black. The former introduces green energy + that doesn't exist in the source image.) + + 2. Artists should not edit premultiplied-alpha images; artists + want non-premultiplied alpha images. Thus, art tools generally output + non-premultiplied alpha images. + + 3. You will get best results in most cases by converting images + to premultiplied alpha before processing them mathematically. + + 4. If you pass the flag STBIR_FLAG_ALPHA_PREMULTIPLIED, the + resizer does not do anything special for the alpha channel; + it is resampled identically to other channels. This produces + the correct results for premultiplied-alpha images, but produces + less-than-ideal results for non-premultiplied-alpha images. + + 5. If you do not pass the flag STBIR_FLAG_ALPHA_PREMULTIPLIED, + then the resizer weights the contribution of input pixels + based on their alpha values, or, equivalently, it multiplies + the alpha value into the color channels, resamples, then divides + by the resultant alpha value. Input pixels which have alpha=0 do + not contribute at all to output pixels unless _all_ of the input + pixels affecting that output pixel have alpha=0, in which case + the result for that pixel is the same as it would be without + STBIR_FLAG_ALPHA_PREMULTIPLIED. However, this is only true for + input images in integer formats. For input images in float format, + input pixels with alpha=0 have no effect, and output pixels + which have alpha=0 will be 0 in all channels. (For float images, + you can manually achieve the same result by adding a tiny epsilon + value to the alpha channel of every image, and then subtracting + or clamping it at the end.) + + 6. You can suppress the behavior described in #5 and make + all-0-alpha pixels have 0 in all channels by #defining + STBIR_NO_ALPHA_EPSILON. + + 7. You can separately control whether the alpha channel is + interpreted as linear or affected by the colorspace. By default + it is linear; you almost never want to apply the colorspace. + (For example, graphics hardware does not apply sRGB conversion + to the alpha channel.) + + CONTRIBUTORS + Jorge L Rodriguez: Implementation + Sean Barrett: API design, optimizations + Aras Pranckevicius: bugfix + Nathan Reed: warning fixes + + REVISIONS + 0.95 (2017-07-23) fixed warnings + 0.94 (2017-03-18) fixed warnings + 0.93 (2017-03-03) fixed bug with certain combinations of heights + 0.92 (2017-01-02) fix integer overflow on large (>2GB) images + 0.91 (2016-04-02) fix warnings; fix handling of subpixel regions + 0.90 (2014-09-17) first released version + + LICENSE + See end of file for license information. + + TODO + Don't decode all of the image data when only processing a partial tile + Don't use full-width decode buffers when only processing a partial tile + When processing wide images, break processing into tiles so data fits in L1 cache + Installable filters? + Resize that respects alpha test coverage + (Reference code: FloatImage::alphaTestCoverage and FloatImage::scaleAlphaToCoverage: + https://code.google.com/p/nvidia-texture-tools/source/browse/trunk/src/nvimage/FloatImage.cpp ) +*/ + +#ifndef STBIR_INCLUDE_STB_IMAGE_RESIZE_H +#define STBIR_INCLUDE_STB_IMAGE_RESIZE_H + +#ifdef _MSC_VER +typedef unsigned char stbir_uint8; +typedef unsigned short stbir_uint16; +typedef unsigned int stbir_uint32; +#else +#include +typedef uint8_t stbir_uint8; +typedef uint16_t stbir_uint16; +typedef uint32_t stbir_uint32; +#endif + +#ifdef STB_IMAGE_RESIZE_STATIC +#define STBIRDEF static +#else +#ifdef __cplusplus +#define STBIRDEF extern "C" +#else +#define STBIRDEF extern +#endif +#endif + + +////////////////////////////////////////////////////////////////////////////// +// +// Easy-to-use API: +// +// * "input pixels" points to an array of image data with 'num_channels' channels (e.g. RGB=3, RGBA=4) +// * input_w is input image width (x-axis), input_h is input image height (y-axis) +// * stride is the offset between successive rows of image data in memory, in bytes. you can +// specify 0 to mean packed continuously in memory +// * alpha channel is treated identically to other channels. +// * colorspace is linear or sRGB as specified by function name +// * returned result is 1 for success or 0 in case of an error. +// #define STBIR_ASSERT() to trigger an assert on parameter validation errors. +// * Memory required grows approximately linearly with input and output size, but with +// discontinuities at input_w == output_w and input_h == output_h. +// * These functions use a "default" resampling filter defined at compile time. To change the filter, +// you can change the compile-time defaults by #defining STBIR_DEFAULT_FILTER_UPSAMPLE +// and STBIR_DEFAULT_FILTER_DOWNSAMPLE, or you can use the medium-complexity API. + +STBIRDEF int stbir_resize_uint8( const unsigned char *input_pixels , int input_w , int input_h , int input_stride_in_bytes, + unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + int num_channels); + +STBIRDEF int stbir_resize_float( const float *input_pixels , int input_w , int input_h , int input_stride_in_bytes, + float *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + int num_channels); + + +// The following functions interpret image data as gamma-corrected sRGB. +// Specify STBIR_ALPHA_CHANNEL_NONE if you have no alpha channel, +// or otherwise provide the index of the alpha channel. Flags value +// of 0 will probably do the right thing if you're not sure what +// the flags mean. + +#define STBIR_ALPHA_CHANNEL_NONE -1 + +// Set this flag if your texture has premultiplied alpha. Otherwise, stbir will +// use alpha-weighted resampling (effectively premultiplying, resampling, +// then unpremultiplying). +#define STBIR_FLAG_ALPHA_PREMULTIPLIED (1 << 0) +// The specified alpha channel should be handled as gamma-corrected value even +// when doing sRGB operations. +#define STBIR_FLAG_ALPHA_USES_COLORSPACE (1 << 1) + +STBIRDEF int stbir_resize_uint8_srgb(const unsigned char *input_pixels , int input_w , int input_h , int input_stride_in_bytes, + unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + int num_channels, int alpha_channel, int flags); + + +typedef enum +{ + STBIR_EDGE_CLAMP = 1, + STBIR_EDGE_REFLECT = 2, + STBIR_EDGE_WRAP = 3, + STBIR_EDGE_ZERO = 4, +} stbir_edge; + +// This function adds the ability to specify how requests to sample off the edge of the image are handled. +STBIRDEF int stbir_resize_uint8_srgb_edgemode(const unsigned char *input_pixels , int input_w , int input_h , int input_stride_in_bytes, + unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + int num_channels, int alpha_channel, int flags, + stbir_edge edge_wrap_mode); + +////////////////////////////////////////////////////////////////////////////// +// +// Medium-complexity API +// +// This extends the easy-to-use API as follows: +// +// * Alpha-channel can be processed separately +// * If alpha_channel is not STBIR_ALPHA_CHANNEL_NONE +// * Alpha channel will not be gamma corrected (unless flags&STBIR_FLAG_GAMMA_CORRECT) +// * Filters will be weighted by alpha channel (unless flags&STBIR_FLAG_ALPHA_PREMULTIPLIED) +// * Filter can be selected explicitly +// * uint16 image type +// * sRGB colorspace available for all types +// * context parameter for passing to STBIR_MALLOC + +typedef enum +{ + STBIR_FILTER_DEFAULT = 0, // use same filter type that easy-to-use API chooses + STBIR_FILTER_BOX = 1, // A trapezoid w/1-pixel wide ramps, same result as box for integer scale ratios + STBIR_FILTER_TRIANGLE = 2, // On upsampling, produces same results as bilinear texture filtering + STBIR_FILTER_CUBICBSPLINE = 3, // The cubic b-spline (aka Mitchell-Netrevalli with B=1,C=0), gaussian-esque + STBIR_FILTER_CATMULLROM = 4, // An interpolating cubic spline + STBIR_FILTER_MITCHELL = 5, // Mitchell-Netrevalli filter with B=1/3, C=1/3 +} stbir_filter; + +typedef enum +{ + STBIR_COLORSPACE_LINEAR, + STBIR_COLORSPACE_SRGB, + + STBIR_MAX_COLORSPACES, +} stbir_colorspace; + +// The following functions are all identical except for the type of the image data + +STBIRDEF int stbir_resize_uint8_generic( const unsigned char *input_pixels , int input_w , int input_h , int input_stride_in_bytes, + unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + int num_channels, int alpha_channel, int flags, + stbir_edge edge_wrap_mode, stbir_filter filter, stbir_colorspace space, + void *alloc_context); + +STBIRDEF int stbir_resize_uint16_generic(const stbir_uint16 *input_pixels , int input_w , int input_h , int input_stride_in_bytes, + stbir_uint16 *output_pixels , int output_w, int output_h, int output_stride_in_bytes, + int num_channels, int alpha_channel, int flags, + stbir_edge edge_wrap_mode, stbir_filter filter, stbir_colorspace space, + void *alloc_context); + +STBIRDEF int stbir_resize_float_generic( const float *input_pixels , int input_w , int input_h , int input_stride_in_bytes, + float *output_pixels , int output_w, int output_h, int output_stride_in_bytes, + int num_channels, int alpha_channel, int flags, + stbir_edge edge_wrap_mode, stbir_filter filter, stbir_colorspace space, + void *alloc_context); + + + +////////////////////////////////////////////////////////////////////////////// +// +// Full-complexity API +// +// This extends the medium API as follows: +// +// * uint32 image type +// * not typesafe +// * separate filter types for each axis +// * separate edge modes for each axis +// * can specify scale explicitly for subpixel correctness +// * can specify image source tile using texture coordinates + +typedef enum +{ + STBIR_TYPE_UINT8 , + STBIR_TYPE_UINT16, + STBIR_TYPE_UINT32, + STBIR_TYPE_FLOAT , + + STBIR_MAX_TYPES +} stbir_datatype; + +STBIRDEF int stbir_resize( const void *input_pixels , int input_w , int input_h , int input_stride_in_bytes, + void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + stbir_datatype datatype, + int num_channels, int alpha_channel, int flags, + stbir_edge edge_mode_horizontal, stbir_edge edge_mode_vertical, + stbir_filter filter_horizontal, stbir_filter filter_vertical, + stbir_colorspace space, void *alloc_context); + +STBIRDEF int stbir_resize_subpixel(const void *input_pixels , int input_w , int input_h , int input_stride_in_bytes, + void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + stbir_datatype datatype, + int num_channels, int alpha_channel, int flags, + stbir_edge edge_mode_horizontal, stbir_edge edge_mode_vertical, + stbir_filter filter_horizontal, stbir_filter filter_vertical, + stbir_colorspace space, void *alloc_context, + float x_scale, float y_scale, + float x_offset, float y_offset); + +STBIRDEF int stbir_resize_region( const void *input_pixels , int input_w , int input_h , int input_stride_in_bytes, + void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + stbir_datatype datatype, + int num_channels, int alpha_channel, int flags, + stbir_edge edge_mode_horizontal, stbir_edge edge_mode_vertical, + stbir_filter filter_horizontal, stbir_filter filter_vertical, + stbir_colorspace space, void *alloc_context, + float s0, float t0, float s1, float t1); +// (s0, t0) & (s1, t1) are the top-left and bottom right corner (uv addressing style: [0, 1]x[0, 1]) of a region of the input image to use. + +// +// +//// end header file ///////////////////////////////////////////////////// +#endif // STBIR_INCLUDE_STB_IMAGE_RESIZE_H + + + + + +#ifdef STB_IMAGE_RESIZE_IMPLEMENTATION + +#ifndef STBIR_ASSERT +#include +#define STBIR_ASSERT(x) assert(x) +#endif + +// For memset +#include + +#include + +#ifndef STBIR_MALLOC +#include +// use comma operator to evaluate c, to avoid "unused parameter" warnings +#define STBIR_MALLOC(size,c) ((void)(c), malloc(size)) +#define STBIR_FREE(ptr,c) ((void)(c), free(ptr)) +#endif + +#ifndef _MSC_VER +#ifdef __cplusplus +#define stbir__inline inline +#else +#define stbir__inline +#endif +#else +#define stbir__inline __forceinline +#endif + + +// should produce compiler error if size is wrong +typedef unsigned char stbir__validate_uint32[sizeof(stbir_uint32) == 4 ? 1 : -1]; + +#ifdef _MSC_VER +#define STBIR__NOTUSED(v) (void)(v) +#else +#define STBIR__NOTUSED(v) (void)sizeof(v) +#endif + +#define STBIR__ARRAY_SIZE(a) (sizeof((a))/sizeof((a)[0])) + +#ifndef STBIR_DEFAULT_FILTER_UPSAMPLE +#define STBIR_DEFAULT_FILTER_UPSAMPLE STBIR_FILTER_CATMULLROM +#endif + +#ifndef STBIR_DEFAULT_FILTER_DOWNSAMPLE +#define STBIR_DEFAULT_FILTER_DOWNSAMPLE STBIR_FILTER_MITCHELL +#endif + +#ifndef STBIR_PROGRESS_REPORT +#define STBIR_PROGRESS_REPORT(float_0_to_1) +#endif + +#ifndef STBIR_MAX_CHANNELS +#define STBIR_MAX_CHANNELS 64 +#endif + +#if STBIR_MAX_CHANNELS > 65536 +#error "Too many channels; STBIR_MAX_CHANNELS must be no more than 65536." +// because we store the indices in 16-bit variables +#endif + +// This value is added to alpha just before premultiplication to avoid +// zeroing out color values. It is equivalent to 2^-80. If you don't want +// that behavior (it may interfere if you have floating point images with +// very small alpha values) then you can define STBIR_NO_ALPHA_EPSILON to +// disable it. +#ifndef STBIR_ALPHA_EPSILON +#define STBIR_ALPHA_EPSILON ((float)1 / (1 << 20) / (1 << 20) / (1 << 20) / (1 << 20)) +#endif + + + +#ifdef _MSC_VER +#define STBIR__UNUSED_PARAM(v) (void)(v) +#else +#define STBIR__UNUSED_PARAM(v) (void)sizeof(v) +#endif + +// must match stbir_datatype +static unsigned char stbir__type_size[] = { + 1, // STBIR_TYPE_UINT8 + 2, // STBIR_TYPE_UINT16 + 4, // STBIR_TYPE_UINT32 + 4, // STBIR_TYPE_FLOAT +}; + +// Kernel function centered at 0 +typedef float (stbir__kernel_fn)(float x, float scale); +typedef float (stbir__support_fn)(float scale); + +typedef struct +{ + stbir__kernel_fn* kernel; + stbir__support_fn* support; +} stbir__filter_info; + +// When upsampling, the contributors are which source pixels contribute. +// When downsampling, the contributors are which destination pixels are contributed to. +typedef struct +{ + int n0; // First contributing pixel + int n1; // Last contributing pixel +} stbir__contributors; + +typedef struct +{ + const void* input_data; + int input_w; + int input_h; + int input_stride_bytes; + + void* output_data; + int output_w; + int output_h; + int output_stride_bytes; + + float s0, t0, s1, t1; + + float horizontal_shift; // Units: output pixels + float vertical_shift; // Units: output pixels + float horizontal_scale; + float vertical_scale; + + int channels; + int alpha_channel; + stbir_uint32 flags; + stbir_datatype type; + stbir_filter horizontal_filter; + stbir_filter vertical_filter; + stbir_edge edge_horizontal; + stbir_edge edge_vertical; + stbir_colorspace colorspace; + + stbir__contributors* horizontal_contributors; + float* horizontal_coefficients; + + stbir__contributors* vertical_contributors; + float* vertical_coefficients; + + int decode_buffer_pixels; + float* decode_buffer; + + float* horizontal_buffer; + + // cache these because ceil/floor are inexplicably showing up in profile + int horizontal_coefficient_width; + int vertical_coefficient_width; + int horizontal_filter_pixel_width; + int vertical_filter_pixel_width; + int horizontal_filter_pixel_margin; + int vertical_filter_pixel_margin; + int horizontal_num_contributors; + int vertical_num_contributors; + + int ring_buffer_length_bytes; // The length of an individual entry in the ring buffer. The total number of ring buffers is stbir__get_filter_pixel_width(filter) + int ring_buffer_num_entries; // Total number of entries in the ring buffer. + int ring_buffer_first_scanline; + int ring_buffer_last_scanline; + int ring_buffer_begin_index; // first_scanline is at this index in the ring buffer + float* ring_buffer; + + float* encode_buffer; // A temporary buffer to store floats so we don't lose precision while we do multiply-adds. + + int horizontal_contributors_size; + int horizontal_coefficients_size; + int vertical_contributors_size; + int vertical_coefficients_size; + int decode_buffer_size; + int horizontal_buffer_size; + int ring_buffer_size; + int encode_buffer_size; +} stbir__info; + + +static const float stbir__max_uint8_as_float = 255.0f; +static const float stbir__max_uint16_as_float = 65535.0f; +static const double stbir__max_uint32_as_float = 4294967295.0; + + +static stbir__inline int stbir__min(int a, int b) +{ + return a < b ? a : b; +} + +static stbir__inline float stbir__saturate(float x) +{ + if (x < 0) + return 0; + + if (x > 1) + return 1; + + return x; +} + +#ifdef STBIR_SATURATE_INT +static stbir__inline stbir_uint8 stbir__saturate8(int x) +{ + if ((unsigned int) x <= 255) + return x; + + if (x < 0) + return 0; + + return 255; +} + +static stbir__inline stbir_uint16 stbir__saturate16(int x) +{ + if ((unsigned int) x <= 65535) + return x; + + if (x < 0) + return 0; + + return 65535; +} +#endif + +static float stbir__srgb_uchar_to_linear_float[256] = { + 0.000000f, 0.000304f, 0.000607f, 0.000911f, 0.001214f, 0.001518f, 0.001821f, 0.002125f, 0.002428f, 0.002732f, 0.003035f, + 0.003347f, 0.003677f, 0.004025f, 0.004391f, 0.004777f, 0.005182f, 0.005605f, 0.006049f, 0.006512f, 0.006995f, 0.007499f, + 0.008023f, 0.008568f, 0.009134f, 0.009721f, 0.010330f, 0.010960f, 0.011612f, 0.012286f, 0.012983f, 0.013702f, 0.014444f, + 0.015209f, 0.015996f, 0.016807f, 0.017642f, 0.018500f, 0.019382f, 0.020289f, 0.021219f, 0.022174f, 0.023153f, 0.024158f, + 0.025187f, 0.026241f, 0.027321f, 0.028426f, 0.029557f, 0.030713f, 0.031896f, 0.033105f, 0.034340f, 0.035601f, 0.036889f, + 0.038204f, 0.039546f, 0.040915f, 0.042311f, 0.043735f, 0.045186f, 0.046665f, 0.048172f, 0.049707f, 0.051269f, 0.052861f, + 0.054480f, 0.056128f, 0.057805f, 0.059511f, 0.061246f, 0.063010f, 0.064803f, 0.066626f, 0.068478f, 0.070360f, 0.072272f, + 0.074214f, 0.076185f, 0.078187f, 0.080220f, 0.082283f, 0.084376f, 0.086500f, 0.088656f, 0.090842f, 0.093059f, 0.095307f, + 0.097587f, 0.099899f, 0.102242f, 0.104616f, 0.107023f, 0.109462f, 0.111932f, 0.114435f, 0.116971f, 0.119538f, 0.122139f, + 0.124772f, 0.127438f, 0.130136f, 0.132868f, 0.135633f, 0.138432f, 0.141263f, 0.144128f, 0.147027f, 0.149960f, 0.152926f, + 0.155926f, 0.158961f, 0.162029f, 0.165132f, 0.168269f, 0.171441f, 0.174647f, 0.177888f, 0.181164f, 0.184475f, 0.187821f, + 0.191202f, 0.194618f, 0.198069f, 0.201556f, 0.205079f, 0.208637f, 0.212231f, 0.215861f, 0.219526f, 0.223228f, 0.226966f, + 0.230740f, 0.234551f, 0.238398f, 0.242281f, 0.246201f, 0.250158f, 0.254152f, 0.258183f, 0.262251f, 0.266356f, 0.270498f, + 0.274677f, 0.278894f, 0.283149f, 0.287441f, 0.291771f, 0.296138f, 0.300544f, 0.304987f, 0.309469f, 0.313989f, 0.318547f, + 0.323143f, 0.327778f, 0.332452f, 0.337164f, 0.341914f, 0.346704f, 0.351533f, 0.356400f, 0.361307f, 0.366253f, 0.371238f, + 0.376262f, 0.381326f, 0.386430f, 0.391573f, 0.396755f, 0.401978f, 0.407240f, 0.412543f, 0.417885f, 0.423268f, 0.428691f, + 0.434154f, 0.439657f, 0.445201f, 0.450786f, 0.456411f, 0.462077f, 0.467784f, 0.473532f, 0.479320f, 0.485150f, 0.491021f, + 0.496933f, 0.502887f, 0.508881f, 0.514918f, 0.520996f, 0.527115f, 0.533276f, 0.539480f, 0.545725f, 0.552011f, 0.558340f, + 0.564712f, 0.571125f, 0.577581f, 0.584078f, 0.590619f, 0.597202f, 0.603827f, 0.610496f, 0.617207f, 0.623960f, 0.630757f, + 0.637597f, 0.644480f, 0.651406f, 0.658375f, 0.665387f, 0.672443f, 0.679543f, 0.686685f, 0.693872f, 0.701102f, 0.708376f, + 0.715694f, 0.723055f, 0.730461f, 0.737911f, 0.745404f, 0.752942f, 0.760525f, 0.768151f, 0.775822f, 0.783538f, 0.791298f, + 0.799103f, 0.806952f, 0.814847f, 0.822786f, 0.830770f, 0.838799f, 0.846873f, 0.854993f, 0.863157f, 0.871367f, 0.879622f, + 0.887923f, 0.896269f, 0.904661f, 0.913099f, 0.921582f, 0.930111f, 0.938686f, 0.947307f, 0.955974f, 0.964686f, 0.973445f, + 0.982251f, 0.991102f, 1.0f +}; + +static float stbir__srgb_to_linear(float f) +{ + if (f <= 0.04045f) + return f / 12.92f; + else + return (float)pow((f + 0.055f) / 1.055f, 2.4f); +} + +static float stbir__linear_to_srgb(float f) +{ + if (f <= 0.0031308f) + return f * 12.92f; + else + return 1.055f * (float)pow(f, 1 / 2.4f) - 0.055f; +} + +#ifndef STBIR_NON_IEEE_FLOAT +// From https://gist.github.com/rygorous/2203834 + +typedef union +{ + stbir_uint32 u; + float f; +} stbir__FP32; + +static const stbir_uint32 fp32_to_srgb8_tab4[104] = { + 0x0073000d, 0x007a000d, 0x0080000d, 0x0087000d, 0x008d000d, 0x0094000d, 0x009a000d, 0x00a1000d, + 0x00a7001a, 0x00b4001a, 0x00c1001a, 0x00ce001a, 0x00da001a, 0x00e7001a, 0x00f4001a, 0x0101001a, + 0x010e0033, 0x01280033, 0x01410033, 0x015b0033, 0x01750033, 0x018f0033, 0x01a80033, 0x01c20033, + 0x01dc0067, 0x020f0067, 0x02430067, 0x02760067, 0x02aa0067, 0x02dd0067, 0x03110067, 0x03440067, + 0x037800ce, 0x03df00ce, 0x044600ce, 0x04ad00ce, 0x051400ce, 0x057b00c5, 0x05dd00bc, 0x063b00b5, + 0x06970158, 0x07420142, 0x07e30130, 0x087b0120, 0x090b0112, 0x09940106, 0x0a1700fc, 0x0a9500f2, + 0x0b0f01cb, 0x0bf401ae, 0x0ccb0195, 0x0d950180, 0x0e56016e, 0x0f0d015e, 0x0fbc0150, 0x10630143, + 0x11070264, 0x1238023e, 0x1357021d, 0x14660201, 0x156601e9, 0x165a01d3, 0x174401c0, 0x182401af, + 0x18fe0331, 0x1a9602fe, 0x1c1502d2, 0x1d7e02ad, 0x1ed4028d, 0x201a0270, 0x21520256, 0x227d0240, + 0x239f0443, 0x25c003fe, 0x27bf03c4, 0x29a10392, 0x2b6a0367, 0x2d1d0341, 0x2ebe031f, 0x304d0300, + 0x31d105b0, 0x34a80555, 0x37520507, 0x39d504c5, 0x3c37048b, 0x3e7c0458, 0x40a8042a, 0x42bd0401, + 0x44c20798, 0x488e071e, 0x4c1c06b6, 0x4f76065d, 0x52a50610, 0x55ac05cc, 0x5892058f, 0x5b590559, + 0x5e0c0a23, 0x631c0980, 0x67db08f6, 0x6c55087f, 0x70940818, 0x74a007bd, 0x787d076c, 0x7c330723, +}; + +static stbir_uint8 stbir__linear_to_srgb_uchar(float in) +{ + static const stbir__FP32 almostone = { 0x3f7fffff }; // 1-eps + static const stbir__FP32 minval = { (127-13) << 23 }; + stbir_uint32 tab,bias,scale,t; + stbir__FP32 f; + + // Clamp to [2^(-13), 1-eps]; these two values map to 0 and 1, respectively. + // The tests are carefully written so that NaNs map to 0, same as in the reference + // implementation. + if (!(in > minval.f)) // written this way to catch NaNs + in = minval.f; + if (in > almostone.f) + in = almostone.f; + + // Do the table lookup and unpack bias, scale + f.f = in; + tab = fp32_to_srgb8_tab4[(f.u - minval.u) >> 20]; + bias = (tab >> 16) << 9; + scale = tab & 0xffff; + + // Grab next-highest mantissa bits and perform linear interpolation + t = (f.u >> 12) & 0xff; + return (unsigned char) ((bias + scale*t) >> 16); +} + +#else +// sRGB transition values, scaled by 1<<28 +static int stbir__srgb_offset_to_linear_scaled[256] = +{ + 0, 40738, 122216, 203693, 285170, 366648, 448125, 529603, + 611080, 692557, 774035, 855852, 942009, 1033024, 1128971, 1229926, + 1335959, 1447142, 1563542, 1685229, 1812268, 1944725, 2082664, 2226148, + 2375238, 2529996, 2690481, 2856753, 3028870, 3206888, 3390865, 3580856, + 3776916, 3979100, 4187460, 4402049, 4622919, 4850123, 5083710, 5323731, + 5570236, 5823273, 6082892, 6349140, 6622065, 6901714, 7188133, 7481369, + 7781466, 8088471, 8402427, 8723380, 9051372, 9386448, 9728650, 10078021, + 10434603, 10798439, 11169569, 11548036, 11933879, 12327139, 12727857, 13136073, + 13551826, 13975156, 14406100, 14844697, 15290987, 15745007, 16206795, 16676389, + 17153826, 17639142, 18132374, 18633560, 19142734, 19659934, 20185196, 20718552, + 21260042, 21809696, 22367554, 22933648, 23508010, 24090680, 24681686, 25281066, + 25888850, 26505076, 27129772, 27762974, 28404716, 29055026, 29713942, 30381490, + 31057708, 31742624, 32436272, 33138682, 33849884, 34569912, 35298800, 36036568, + 36783260, 37538896, 38303512, 39077136, 39859796, 40651528, 41452360, 42262316, + 43081432, 43909732, 44747252, 45594016, 46450052, 47315392, 48190064, 49074096, + 49967516, 50870356, 51782636, 52704392, 53635648, 54576432, 55526772, 56486700, + 57456236, 58435408, 59424248, 60422780, 61431036, 62449032, 63476804, 64514376, + 65561776, 66619028, 67686160, 68763192, 69850160, 70947088, 72053992, 73170912, + 74297864, 75434880, 76581976, 77739184, 78906536, 80084040, 81271736, 82469648, + 83677792, 84896192, 86124888, 87363888, 88613232, 89872928, 91143016, 92423512, + 93714432, 95015816, 96327688, 97650056, 98982952, 100326408, 101680440, 103045072, + 104420320, 105806224, 107202800, 108610064, 110028048, 111456776, 112896264, 114346544, + 115807632, 117279552, 118762328, 120255976, 121760536, 123276016, 124802440, 126339832, + 127888216, 129447616, 131018048, 132599544, 134192112, 135795792, 137410592, 139036528, + 140673648, 142321952, 143981456, 145652208, 147334208, 149027488, 150732064, 152447968, + 154175200, 155913792, 157663776, 159425168, 161197984, 162982240, 164777968, 166585184, + 168403904, 170234160, 172075968, 173929344, 175794320, 177670896, 179559120, 181458992, + 183370528, 185293776, 187228736, 189175424, 191133888, 193104112, 195086128, 197079968, + 199085648, 201103184, 203132592, 205173888, 207227120, 209292272, 211369392, 213458480, + 215559568, 217672656, 219797792, 221934976, 224084240, 226245600, 228419056, 230604656, + 232802400, 235012320, 237234432, 239468736, 241715280, 243974080, 246245120, 248528464, + 250824112, 253132064, 255452368, 257785040, 260130080, 262487520, 264857376, 267239664, +}; + +static stbir_uint8 stbir__linear_to_srgb_uchar(float f) +{ + int x = (int) (f * (1 << 28)); // has headroom so you don't need to clamp + int v = 0; + int i; + + // Refine the guess with a short binary search. + i = v + 128; if (x >= stbir__srgb_offset_to_linear_scaled[i]) v = i; + i = v + 64; if (x >= stbir__srgb_offset_to_linear_scaled[i]) v = i; + i = v + 32; if (x >= stbir__srgb_offset_to_linear_scaled[i]) v = i; + i = v + 16; if (x >= stbir__srgb_offset_to_linear_scaled[i]) v = i; + i = v + 8; if (x >= stbir__srgb_offset_to_linear_scaled[i]) v = i; + i = v + 4; if (x >= stbir__srgb_offset_to_linear_scaled[i]) v = i; + i = v + 2; if (x >= stbir__srgb_offset_to_linear_scaled[i]) v = i; + i = v + 1; if (x >= stbir__srgb_offset_to_linear_scaled[i]) v = i; + + return (stbir_uint8) v; +} +#endif + +static float stbir__filter_trapezoid(float x, float scale) +{ + float halfscale = scale / 2; + float t = 0.5f + halfscale; + STBIR_ASSERT(scale <= 1); + + x = (float)fabs(x); + + if (x >= t) + return 0; + else + { + float r = 0.5f - halfscale; + if (x <= r) + return 1; + else + return (t - x) / scale; + } +} + +static float stbir__support_trapezoid(float scale) +{ + STBIR_ASSERT(scale <= 1); + return 0.5f + scale / 2; +} + +static float stbir__filter_triangle(float x, float s) +{ + STBIR__UNUSED_PARAM(s); + + x = (float)fabs(x); + + if (x <= 1.0f) + return 1 - x; + else + return 0; +} + +static float stbir__filter_cubic(float x, float s) +{ + STBIR__UNUSED_PARAM(s); + + x = (float)fabs(x); + + if (x < 1.0f) + return (4 + x*x*(3*x - 6))/6; + else if (x < 2.0f) + return (8 + x*(-12 + x*(6 - x)))/6; + + return (0.0f); +} + +static float stbir__filter_catmullrom(float x, float s) +{ + STBIR__UNUSED_PARAM(s); + + x = (float)fabs(x); + + if (x < 1.0f) + return 1 - x*x*(2.5f - 1.5f*x); + else if (x < 2.0f) + return 2 - x*(4 + x*(0.5f*x - 2.5f)); + + return (0.0f); +} + +static float stbir__filter_mitchell(float x, float s) +{ + STBIR__UNUSED_PARAM(s); + + x = (float)fabs(x); + + if (x < 1.0f) + return (16 + x*x*(21 * x - 36))/18; + else if (x < 2.0f) + return (32 + x*(-60 + x*(36 - 7*x)))/18; + + return (0.0f); +} + +static float stbir__support_zero(float s) +{ + STBIR__UNUSED_PARAM(s); + return 0; +} + +static float stbir__support_one(float s) +{ + STBIR__UNUSED_PARAM(s); + return 1; +} + +static float stbir__support_two(float s) +{ + STBIR__UNUSED_PARAM(s); + return 2; +} + +static stbir__filter_info stbir__filter_info_table[] = { + { NULL, stbir__support_zero }, + { stbir__filter_trapezoid, stbir__support_trapezoid }, + { stbir__filter_triangle, stbir__support_one }, + { stbir__filter_cubic, stbir__support_two }, + { stbir__filter_catmullrom, stbir__support_two }, + { stbir__filter_mitchell, stbir__support_two }, +}; + +stbir__inline static int stbir__use_upsampling(float ratio) +{ + return ratio > 1; +} + +stbir__inline static int stbir__use_width_upsampling(stbir__info* stbir_info) +{ + return stbir__use_upsampling(stbir_info->horizontal_scale); +} + +stbir__inline static int stbir__use_height_upsampling(stbir__info* stbir_info) +{ + return stbir__use_upsampling(stbir_info->vertical_scale); +} + +// This is the maximum number of input samples that can affect an output sample +// with the given filter +static int stbir__get_filter_pixel_width(stbir_filter filter, float scale) +{ + STBIR_ASSERT(filter != 0); + STBIR_ASSERT(filter < STBIR__ARRAY_SIZE(stbir__filter_info_table)); + + if (stbir__use_upsampling(scale)) + return (int)ceil(stbir__filter_info_table[filter].support(1/scale) * 2); + else + return (int)ceil(stbir__filter_info_table[filter].support(scale) * 2 / scale); +} + +// This is how much to expand buffers to account for filters seeking outside +// the image boundaries. +static int stbir__get_filter_pixel_margin(stbir_filter filter, float scale) +{ + return stbir__get_filter_pixel_width(filter, scale) / 2; +} + +static int stbir__get_coefficient_width(stbir_filter filter, float scale) +{ + if (stbir__use_upsampling(scale)) + return (int)ceil(stbir__filter_info_table[filter].support(1 / scale) * 2); + else + return (int)ceil(stbir__filter_info_table[filter].support(scale) * 2); +} + +static int stbir__get_contributors(float scale, stbir_filter filter, int input_size, int output_size) +{ + if (stbir__use_upsampling(scale)) + return output_size; + else + return (input_size + stbir__get_filter_pixel_margin(filter, scale) * 2); +} + +static int stbir__get_total_horizontal_coefficients(stbir__info* info) +{ + return info->horizontal_num_contributors + * stbir__get_coefficient_width (info->horizontal_filter, info->horizontal_scale); +} + +static int stbir__get_total_vertical_coefficients(stbir__info* info) +{ + return info->vertical_num_contributors + * stbir__get_coefficient_width (info->vertical_filter, info->vertical_scale); +} + +static stbir__contributors* stbir__get_contributor(stbir__contributors* contributors, int n) +{ + return &contributors[n]; +} + +// For perf reasons this code is duplicated in stbir__resample_horizontal_upsample/downsample, +// if you change it here change it there too. +static float* stbir__get_coefficient(float* coefficients, stbir_filter filter, float scale, int n, int c) +{ + int width = stbir__get_coefficient_width(filter, scale); + return &coefficients[width*n + c]; +} + +static int stbir__edge_wrap_slow(stbir_edge edge, int n, int max) +{ + switch (edge) + { + case STBIR_EDGE_ZERO: + return 0; // we'll decode the wrong pixel here, and then overwrite with 0s later + + case STBIR_EDGE_CLAMP: + if (n < 0) + return 0; + + if (n >= max) + return max - 1; + + return n; // NOTREACHED + + case STBIR_EDGE_REFLECT: + { + if (n < 0) + { + if (n < max) + return -n; + else + return max - 1; + } + + if (n >= max) + { + int max2 = max * 2; + if (n >= max2) + return 0; + else + return max2 - n - 1; + } + + return n; // NOTREACHED + } + + case STBIR_EDGE_WRAP: + if (n >= 0) + return (n % max); + else + { + int m = (-n) % max; + + if (m != 0) + m = max - m; + + return (m); + } + // NOTREACHED + + default: + STBIR_ASSERT(!"Unimplemented edge type"); + return 0; + } +} + +stbir__inline static int stbir__edge_wrap(stbir_edge edge, int n, int max) +{ + // avoid per-pixel switch + if (n >= 0 && n < max) + return n; + return stbir__edge_wrap_slow(edge, n, max); +} + +// What input pixels contribute to this output pixel? +static void stbir__calculate_sample_range_upsample(int n, float out_filter_radius, float scale_ratio, float out_shift, int* in_first_pixel, int* in_last_pixel, float* in_center_of_out) +{ + float out_pixel_center = (float)n + 0.5f; + float out_pixel_influence_lowerbound = out_pixel_center - out_filter_radius; + float out_pixel_influence_upperbound = out_pixel_center + out_filter_radius; + + float in_pixel_influence_lowerbound = (out_pixel_influence_lowerbound + out_shift) / scale_ratio; + float in_pixel_influence_upperbound = (out_pixel_influence_upperbound + out_shift) / scale_ratio; + + *in_center_of_out = (out_pixel_center + out_shift) / scale_ratio; + *in_first_pixel = (int)(floor(in_pixel_influence_lowerbound + 0.5)); + *in_last_pixel = (int)(floor(in_pixel_influence_upperbound - 0.5)); +} + +// What output pixels does this input pixel contribute to? +static void stbir__calculate_sample_range_downsample(int n, float in_pixels_radius, float scale_ratio, float out_shift, int* out_first_pixel, int* out_last_pixel, float* out_center_of_in) +{ + float in_pixel_center = (float)n + 0.5f; + float in_pixel_influence_lowerbound = in_pixel_center - in_pixels_radius; + float in_pixel_influence_upperbound = in_pixel_center + in_pixels_radius; + + float out_pixel_influence_lowerbound = in_pixel_influence_lowerbound * scale_ratio - out_shift; + float out_pixel_influence_upperbound = in_pixel_influence_upperbound * scale_ratio - out_shift; + + *out_center_of_in = in_pixel_center * scale_ratio - out_shift; + *out_first_pixel = (int)(floor(out_pixel_influence_lowerbound + 0.5)); + *out_last_pixel = (int)(floor(out_pixel_influence_upperbound - 0.5)); +} + +static void stbir__calculate_coefficients_upsample(stbir_filter filter, float scale, int in_first_pixel, int in_last_pixel, float in_center_of_out, stbir__contributors* contributor, float* coefficient_group) +{ + int i; + float total_filter = 0; + float filter_scale; + + STBIR_ASSERT(in_last_pixel - in_first_pixel <= (int)ceil(stbir__filter_info_table[filter].support(1/scale) * 2)); // Taken directly from stbir__get_coefficient_width() which we can't call because we don't know if we're horizontal or vertical. + + contributor->n0 = in_first_pixel; + contributor->n1 = in_last_pixel; + + STBIR_ASSERT(contributor->n1 >= contributor->n0); + + for (i = 0; i <= in_last_pixel - in_first_pixel; i++) + { + float in_pixel_center = (float)(i + in_first_pixel) + 0.5f; + coefficient_group[i] = stbir__filter_info_table[filter].kernel(in_center_of_out - in_pixel_center, 1 / scale); + + // If the coefficient is zero, skip it. (Don't do the <0 check here, we want the influence of those outside pixels.) + if (i == 0 && !coefficient_group[i]) + { + contributor->n0 = ++in_first_pixel; + i--; + continue; + } + + total_filter += coefficient_group[i]; + } + + STBIR_ASSERT(stbir__filter_info_table[filter].kernel((float)(in_last_pixel + 1) + 0.5f - in_center_of_out, 1/scale) == 0); + + STBIR_ASSERT(total_filter > 0.9); + STBIR_ASSERT(total_filter < 1.1f); // Make sure it's not way off. + + // Make sure the sum of all coefficients is 1. + filter_scale = 1 / total_filter; + + for (i = 0; i <= in_last_pixel - in_first_pixel; i++) + coefficient_group[i] *= filter_scale; + + for (i = in_last_pixel - in_first_pixel; i >= 0; i--) + { + if (coefficient_group[i]) + break; + + // This line has no weight. We can skip it. + contributor->n1 = contributor->n0 + i - 1; + } +} + +static void stbir__calculate_coefficients_downsample(stbir_filter filter, float scale_ratio, int out_first_pixel, int out_last_pixel, float out_center_of_in, stbir__contributors* contributor, float* coefficient_group) +{ + int i; + + STBIR_ASSERT(out_last_pixel - out_first_pixel <= (int)ceil(stbir__filter_info_table[filter].support(scale_ratio) * 2)); // Taken directly from stbir__get_coefficient_width() which we can't call because we don't know if we're horizontal or vertical. + + contributor->n0 = out_first_pixel; + contributor->n1 = out_last_pixel; + + STBIR_ASSERT(contributor->n1 >= contributor->n0); + + for (i = 0; i <= out_last_pixel - out_first_pixel; i++) + { + float out_pixel_center = (float)(i + out_first_pixel) + 0.5f; + float x = out_pixel_center - out_center_of_in; + coefficient_group[i] = stbir__filter_info_table[filter].kernel(x, scale_ratio) * scale_ratio; + } + + STBIR_ASSERT(stbir__filter_info_table[filter].kernel((float)(out_last_pixel + 1) + 0.5f - out_center_of_in, scale_ratio) == 0); + + for (i = out_last_pixel - out_first_pixel; i >= 0; i--) + { + if (coefficient_group[i]) + break; + + // This line has no weight. We can skip it. + contributor->n1 = contributor->n0 + i - 1; + } +} + +static void stbir__normalize_downsample_coefficients(stbir__contributors* contributors, float* coefficients, stbir_filter filter, float scale_ratio, int input_size, int output_size) +{ + int num_contributors = stbir__get_contributors(scale_ratio, filter, input_size, output_size); + int num_coefficients = stbir__get_coefficient_width(filter, scale_ratio); + int i, j; + int skip; + + for (i = 0; i < output_size; i++) + { + float scale; + float total = 0; + + for (j = 0; j < num_contributors; j++) + { + if (i >= contributors[j].n0 && i <= contributors[j].n1) + { + float coefficient = *stbir__get_coefficient(coefficients, filter, scale_ratio, j, i - contributors[j].n0); + total += coefficient; + } + else if (i < contributors[j].n0) + break; + } + + STBIR_ASSERT(total > 0.9f); + STBIR_ASSERT(total < 1.1f); + + scale = 1 / total; + + for (j = 0; j < num_contributors; j++) + { + if (i >= contributors[j].n0 && i <= contributors[j].n1) + *stbir__get_coefficient(coefficients, filter, scale_ratio, j, i - contributors[j].n0) *= scale; + else if (i < contributors[j].n0) + break; + } + } + + // Optimize: Skip zero coefficients and contributions outside of image bounds. + // Do this after normalizing because normalization depends on the n0/n1 values. + for (j = 0; j < num_contributors; j++) + { + int range, max, width; + + skip = 0; + while (*stbir__get_coefficient(coefficients, filter, scale_ratio, j, skip) == 0) + skip++; + + contributors[j].n0 += skip; + + while (contributors[j].n0 < 0) + { + contributors[j].n0++; + skip++; + } + + range = contributors[j].n1 - contributors[j].n0 + 1; + max = stbir__min(num_coefficients, range); + + width = stbir__get_coefficient_width(filter, scale_ratio); + for (i = 0; i < max; i++) + { + if (i + skip >= width) + break; + + *stbir__get_coefficient(coefficients, filter, scale_ratio, j, i) = *stbir__get_coefficient(coefficients, filter, scale_ratio, j, i + skip); + } + + continue; + } + + // Using min to avoid writing into invalid pixels. + for (i = 0; i < num_contributors; i++) + contributors[i].n1 = stbir__min(contributors[i].n1, output_size - 1); +} + +// Each scan line uses the same kernel values so we should calculate the kernel +// values once and then we can use them for every scan line. +static void stbir__calculate_filters(stbir__contributors* contributors, float* coefficients, stbir_filter filter, float scale_ratio, float shift, int input_size, int output_size) +{ + int n; + int total_contributors = stbir__get_contributors(scale_ratio, filter, input_size, output_size); + + if (stbir__use_upsampling(scale_ratio)) + { + float out_pixels_radius = stbir__filter_info_table[filter].support(1 / scale_ratio) * scale_ratio; + + // Looping through out pixels + for (n = 0; n < total_contributors; n++) + { + float in_center_of_out; // Center of the current out pixel in the in pixel space + int in_first_pixel, in_last_pixel; + + stbir__calculate_sample_range_upsample(n, out_pixels_radius, scale_ratio, shift, &in_first_pixel, &in_last_pixel, &in_center_of_out); + + stbir__calculate_coefficients_upsample(filter, scale_ratio, in_first_pixel, in_last_pixel, in_center_of_out, stbir__get_contributor(contributors, n), stbir__get_coefficient(coefficients, filter, scale_ratio, n, 0)); + } + } + else + { + float in_pixels_radius = stbir__filter_info_table[filter].support(scale_ratio) / scale_ratio; + + // Looping through in pixels + for (n = 0; n < total_contributors; n++) + { + float out_center_of_in; // Center of the current out pixel in the in pixel space + int out_first_pixel, out_last_pixel; + int n_adjusted = n - stbir__get_filter_pixel_margin(filter, scale_ratio); + + stbir__calculate_sample_range_downsample(n_adjusted, in_pixels_radius, scale_ratio, shift, &out_first_pixel, &out_last_pixel, &out_center_of_in); + + stbir__calculate_coefficients_downsample(filter, scale_ratio, out_first_pixel, out_last_pixel, out_center_of_in, stbir__get_contributor(contributors, n), stbir__get_coefficient(coefficients, filter, scale_ratio, n, 0)); + } + + stbir__normalize_downsample_coefficients(contributors, coefficients, filter, scale_ratio, input_size, output_size); + } +} + +static float* stbir__get_decode_buffer(stbir__info* stbir_info) +{ + // The 0 index of the decode buffer starts after the margin. This makes + // it okay to use negative indexes on the decode buffer. + return &stbir_info->decode_buffer[stbir_info->horizontal_filter_pixel_margin * stbir_info->channels]; +} + +#define STBIR__DECODE(type, colorspace) ((type) * (STBIR_MAX_COLORSPACES) + (colorspace)) + +static void stbir__decode_scanline(stbir__info* stbir_info, int n) +{ + int c; + int channels = stbir_info->channels; + int alpha_channel = stbir_info->alpha_channel; + int type = stbir_info->type; + int colorspace = stbir_info->colorspace; + int input_w = stbir_info->input_w; + size_t input_stride_bytes = stbir_info->input_stride_bytes; + float* decode_buffer = stbir__get_decode_buffer(stbir_info); + stbir_edge edge_horizontal = stbir_info->edge_horizontal; + stbir_edge edge_vertical = stbir_info->edge_vertical; + size_t in_buffer_row_offset = stbir__edge_wrap(edge_vertical, n, stbir_info->input_h) * input_stride_bytes; + const void* input_data = (char *) stbir_info->input_data + in_buffer_row_offset; + int max_x = input_w + stbir_info->horizontal_filter_pixel_margin; + int decode = STBIR__DECODE(type, colorspace); + + int x = -stbir_info->horizontal_filter_pixel_margin; + + // special handling for STBIR_EDGE_ZERO because it needs to return an item that doesn't appear in the input, + // and we want to avoid paying overhead on every pixel if not STBIR_EDGE_ZERO + if (edge_vertical == STBIR_EDGE_ZERO && (n < 0 || n >= stbir_info->input_h)) + { + for (; x < max_x; x++) + for (c = 0; c < channels; c++) + decode_buffer[x*channels + c] = 0; + return; + } + + switch (decode) + { + case STBIR__DECODE(STBIR_TYPE_UINT8, STBIR_COLORSPACE_LINEAR): + for (; x < max_x; x++) + { + int decode_pixel_index = x * channels; + int input_pixel_index = stbir__edge_wrap(edge_horizontal, x, input_w) * channels; + for (c = 0; c < channels; c++) + decode_buffer[decode_pixel_index + c] = ((float)((const unsigned char*)input_data)[input_pixel_index + c]) / stbir__max_uint8_as_float; + } + break; + + case STBIR__DECODE(STBIR_TYPE_UINT8, STBIR_COLORSPACE_SRGB): + for (; x < max_x; x++) + { + int decode_pixel_index = x * channels; + int input_pixel_index = stbir__edge_wrap(edge_horizontal, x, input_w) * channels; + for (c = 0; c < channels; c++) + decode_buffer[decode_pixel_index + c] = stbir__srgb_uchar_to_linear_float[((const unsigned char*)input_data)[input_pixel_index + c]]; + + if (!(stbir_info->flags&STBIR_FLAG_ALPHA_USES_COLORSPACE)) + decode_buffer[decode_pixel_index + alpha_channel] = ((float)((const unsigned char*)input_data)[input_pixel_index + alpha_channel]) / stbir__max_uint8_as_float; + } + break; + + case STBIR__DECODE(STBIR_TYPE_UINT16, STBIR_COLORSPACE_LINEAR): + for (; x < max_x; x++) + { + int decode_pixel_index = x * channels; + int input_pixel_index = stbir__edge_wrap(edge_horizontal, x, input_w) * channels; + for (c = 0; c < channels; c++) + decode_buffer[decode_pixel_index + c] = ((float)((const unsigned short*)input_data)[input_pixel_index + c]) / stbir__max_uint16_as_float; + } + break; + + case STBIR__DECODE(STBIR_TYPE_UINT16, STBIR_COLORSPACE_SRGB): + for (; x < max_x; x++) + { + int decode_pixel_index = x * channels; + int input_pixel_index = stbir__edge_wrap(edge_horizontal, x, input_w) * channels; + for (c = 0; c < channels; c++) + decode_buffer[decode_pixel_index + c] = stbir__srgb_to_linear(((float)((const unsigned short*)input_data)[input_pixel_index + c]) / stbir__max_uint16_as_float); + + if (!(stbir_info->flags&STBIR_FLAG_ALPHA_USES_COLORSPACE)) + decode_buffer[decode_pixel_index + alpha_channel] = ((float)((const unsigned short*)input_data)[input_pixel_index + alpha_channel]) / stbir__max_uint16_as_float; + } + break; + + case STBIR__DECODE(STBIR_TYPE_UINT32, STBIR_COLORSPACE_LINEAR): + for (; x < max_x; x++) + { + int decode_pixel_index = x * channels; + int input_pixel_index = stbir__edge_wrap(edge_horizontal, x, input_w) * channels; + for (c = 0; c < channels; c++) + decode_buffer[decode_pixel_index + c] = (float)(((double)((const unsigned int*)input_data)[input_pixel_index + c]) / stbir__max_uint32_as_float); + } + break; + + case STBIR__DECODE(STBIR_TYPE_UINT32, STBIR_COLORSPACE_SRGB): + for (; x < max_x; x++) + { + int decode_pixel_index = x * channels; + int input_pixel_index = stbir__edge_wrap(edge_horizontal, x, input_w) * channels; + for (c = 0; c < channels; c++) + decode_buffer[decode_pixel_index + c] = stbir__srgb_to_linear((float)(((double)((const unsigned int*)input_data)[input_pixel_index + c]) / stbir__max_uint32_as_float)); + + if (!(stbir_info->flags&STBIR_FLAG_ALPHA_USES_COLORSPACE)) + decode_buffer[decode_pixel_index + alpha_channel] = (float)(((double)((const unsigned int*)input_data)[input_pixel_index + alpha_channel]) / stbir__max_uint32_as_float); + } + break; + + case STBIR__DECODE(STBIR_TYPE_FLOAT, STBIR_COLORSPACE_LINEAR): + for (; x < max_x; x++) + { + int decode_pixel_index = x * channels; + int input_pixel_index = stbir__edge_wrap(edge_horizontal, x, input_w) * channels; + for (c = 0; c < channels; c++) + decode_buffer[decode_pixel_index + c] = ((const float*)input_data)[input_pixel_index + c]; + } + break; + + case STBIR__DECODE(STBIR_TYPE_FLOAT, STBIR_COLORSPACE_SRGB): + for (; x < max_x; x++) + { + int decode_pixel_index = x * channels; + int input_pixel_index = stbir__edge_wrap(edge_horizontal, x, input_w) * channels; + for (c = 0; c < channels; c++) + decode_buffer[decode_pixel_index + c] = stbir__srgb_to_linear(((const float*)input_data)[input_pixel_index + c]); + + if (!(stbir_info->flags&STBIR_FLAG_ALPHA_USES_COLORSPACE)) + decode_buffer[decode_pixel_index + alpha_channel] = ((const float*)input_data)[input_pixel_index + alpha_channel]; + } + + break; + + default: + STBIR_ASSERT(!"Unknown type/colorspace/channels combination."); + break; + } + + if (!(stbir_info->flags & STBIR_FLAG_ALPHA_PREMULTIPLIED)) + { + for (x = -stbir_info->horizontal_filter_pixel_margin; x < max_x; x++) + { + int decode_pixel_index = x * channels; + + // If the alpha value is 0 it will clobber the color values. Make sure it's not. + float alpha = decode_buffer[decode_pixel_index + alpha_channel]; +#ifndef STBIR_NO_ALPHA_EPSILON + if (stbir_info->type != STBIR_TYPE_FLOAT) { + alpha += STBIR_ALPHA_EPSILON; + decode_buffer[decode_pixel_index + alpha_channel] = alpha; + } +#endif + for (c = 0; c < channels; c++) + { + if (c == alpha_channel) + continue; + + decode_buffer[decode_pixel_index + c] *= alpha; + } + } + } + + if (edge_horizontal == STBIR_EDGE_ZERO) + { + for (x = -stbir_info->horizontal_filter_pixel_margin; x < 0; x++) + { + for (c = 0; c < channels; c++) + decode_buffer[x*channels + c] = 0; + } + for (x = input_w; x < max_x; x++) + { + for (c = 0; c < channels; c++) + decode_buffer[x*channels + c] = 0; + } + } +} + +static float* stbir__get_ring_buffer_entry(float* ring_buffer, int index, int ring_buffer_length) +{ + return &ring_buffer[index * ring_buffer_length]; +} + +static float* stbir__add_empty_ring_buffer_entry(stbir__info* stbir_info, int n) +{ + int ring_buffer_index; + float* ring_buffer; + + stbir_info->ring_buffer_last_scanline = n; + + if (stbir_info->ring_buffer_begin_index < 0) + { + ring_buffer_index = stbir_info->ring_buffer_begin_index = 0; + stbir_info->ring_buffer_first_scanline = n; + } + else + { + ring_buffer_index = (stbir_info->ring_buffer_begin_index + (stbir_info->ring_buffer_last_scanline - stbir_info->ring_buffer_first_scanline)) % stbir_info->ring_buffer_num_entries; + STBIR_ASSERT(ring_buffer_index != stbir_info->ring_buffer_begin_index); + } + + ring_buffer = stbir__get_ring_buffer_entry(stbir_info->ring_buffer, ring_buffer_index, stbir_info->ring_buffer_length_bytes / sizeof(float)); + memset(ring_buffer, 0, stbir_info->ring_buffer_length_bytes); + + return ring_buffer; +} + + +static void stbir__resample_horizontal_upsample(stbir__info* stbir_info, float* output_buffer) +{ + int x, k; + int output_w = stbir_info->output_w; + int channels = stbir_info->channels; + float* decode_buffer = stbir__get_decode_buffer(stbir_info); + stbir__contributors* horizontal_contributors = stbir_info->horizontal_contributors; + float* horizontal_coefficients = stbir_info->horizontal_coefficients; + int coefficient_width = stbir_info->horizontal_coefficient_width; + + for (x = 0; x < output_w; x++) + { + int n0 = horizontal_contributors[x].n0; + int n1 = horizontal_contributors[x].n1; + + int out_pixel_index = x * channels; + int coefficient_group = coefficient_width * x; + int coefficient_counter = 0; + + STBIR_ASSERT(n1 >= n0); + STBIR_ASSERT(n0 >= -stbir_info->horizontal_filter_pixel_margin); + STBIR_ASSERT(n1 >= -stbir_info->horizontal_filter_pixel_margin); + STBIR_ASSERT(n0 < stbir_info->input_w + stbir_info->horizontal_filter_pixel_margin); + STBIR_ASSERT(n1 < stbir_info->input_w + stbir_info->horizontal_filter_pixel_margin); + + switch (channels) { + case 1: + for (k = n0; k <= n1; k++) + { + int in_pixel_index = k * 1; + float coefficient = horizontal_coefficients[coefficient_group + coefficient_counter++]; + STBIR_ASSERT(coefficient != 0); + output_buffer[out_pixel_index + 0] += decode_buffer[in_pixel_index + 0] * coefficient; + } + break; + case 2: + for (k = n0; k <= n1; k++) + { + int in_pixel_index = k * 2; + float coefficient = horizontal_coefficients[coefficient_group + coefficient_counter++]; + STBIR_ASSERT(coefficient != 0); + output_buffer[out_pixel_index + 0] += decode_buffer[in_pixel_index + 0] * coefficient; + output_buffer[out_pixel_index + 1] += decode_buffer[in_pixel_index + 1] * coefficient; + } + break; + case 3: + for (k = n0; k <= n1; k++) + { + int in_pixel_index = k * 3; + float coefficient = horizontal_coefficients[coefficient_group + coefficient_counter++]; + STBIR_ASSERT(coefficient != 0); + output_buffer[out_pixel_index + 0] += decode_buffer[in_pixel_index + 0] * coefficient; + output_buffer[out_pixel_index + 1] += decode_buffer[in_pixel_index + 1] * coefficient; + output_buffer[out_pixel_index + 2] += decode_buffer[in_pixel_index + 2] * coefficient; + } + break; + case 4: + for (k = n0; k <= n1; k++) + { + int in_pixel_index = k * 4; + float coefficient = horizontal_coefficients[coefficient_group + coefficient_counter++]; + STBIR_ASSERT(coefficient != 0); + output_buffer[out_pixel_index + 0] += decode_buffer[in_pixel_index + 0] * coefficient; + output_buffer[out_pixel_index + 1] += decode_buffer[in_pixel_index + 1] * coefficient; + output_buffer[out_pixel_index + 2] += decode_buffer[in_pixel_index + 2] * coefficient; + output_buffer[out_pixel_index + 3] += decode_buffer[in_pixel_index + 3] * coefficient; + } + break; + default: + for (k = n0; k <= n1; k++) + { + int in_pixel_index = k * channels; + float coefficient = horizontal_coefficients[coefficient_group + coefficient_counter++]; + int c; + STBIR_ASSERT(coefficient != 0); + for (c = 0; c < channels; c++) + output_buffer[out_pixel_index + c] += decode_buffer[in_pixel_index + c] * coefficient; + } + break; + } + } +} + +static void stbir__resample_horizontal_downsample(stbir__info* stbir_info, float* output_buffer) +{ + int x, k; + int input_w = stbir_info->input_w; + int channels = stbir_info->channels; + float* decode_buffer = stbir__get_decode_buffer(stbir_info); + stbir__contributors* horizontal_contributors = stbir_info->horizontal_contributors; + float* horizontal_coefficients = stbir_info->horizontal_coefficients; + int coefficient_width = stbir_info->horizontal_coefficient_width; + int filter_pixel_margin = stbir_info->horizontal_filter_pixel_margin; + int max_x = input_w + filter_pixel_margin * 2; + + STBIR_ASSERT(!stbir__use_width_upsampling(stbir_info)); + + switch (channels) { + case 1: + for (x = 0; x < max_x; x++) + { + int n0 = horizontal_contributors[x].n0; + int n1 = horizontal_contributors[x].n1; + + int in_x = x - filter_pixel_margin; + int in_pixel_index = in_x * 1; + int max_n = n1; + int coefficient_group = coefficient_width * x; + + for (k = n0; k <= max_n; k++) + { + int out_pixel_index = k * 1; + float coefficient = horizontal_coefficients[coefficient_group + k - n0]; + STBIR_ASSERT(coefficient != 0); + output_buffer[out_pixel_index + 0] += decode_buffer[in_pixel_index + 0] * coefficient; + } + } + break; + + case 2: + for (x = 0; x < max_x; x++) + { + int n0 = horizontal_contributors[x].n0; + int n1 = horizontal_contributors[x].n1; + + int in_x = x - filter_pixel_margin; + int in_pixel_index = in_x * 2; + int max_n = n1; + int coefficient_group = coefficient_width * x; + + for (k = n0; k <= max_n; k++) + { + int out_pixel_index = k * 2; + float coefficient = horizontal_coefficients[coefficient_group + k - n0]; + STBIR_ASSERT(coefficient != 0); + output_buffer[out_pixel_index + 0] += decode_buffer[in_pixel_index + 0] * coefficient; + output_buffer[out_pixel_index + 1] += decode_buffer[in_pixel_index + 1] * coefficient; + } + } + break; + + case 3: + for (x = 0; x < max_x; x++) + { + int n0 = horizontal_contributors[x].n0; + int n1 = horizontal_contributors[x].n1; + + int in_x = x - filter_pixel_margin; + int in_pixel_index = in_x * 3; + int max_n = n1; + int coefficient_group = coefficient_width * x; + + for (k = n0; k <= max_n; k++) + { + int out_pixel_index = k * 3; + float coefficient = horizontal_coefficients[coefficient_group + k - n0]; + STBIR_ASSERT(coefficient != 0); + output_buffer[out_pixel_index + 0] += decode_buffer[in_pixel_index + 0] * coefficient; + output_buffer[out_pixel_index + 1] += decode_buffer[in_pixel_index + 1] * coefficient; + output_buffer[out_pixel_index + 2] += decode_buffer[in_pixel_index + 2] * coefficient; + } + } + break; + + case 4: + for (x = 0; x < max_x; x++) + { + int n0 = horizontal_contributors[x].n0; + int n1 = horizontal_contributors[x].n1; + + int in_x = x - filter_pixel_margin; + int in_pixel_index = in_x * 4; + int max_n = n1; + int coefficient_group = coefficient_width * x; + + for (k = n0; k <= max_n; k++) + { + int out_pixel_index = k * 4; + float coefficient = horizontal_coefficients[coefficient_group + k - n0]; + STBIR_ASSERT(coefficient != 0); + output_buffer[out_pixel_index + 0] += decode_buffer[in_pixel_index + 0] * coefficient; + output_buffer[out_pixel_index + 1] += decode_buffer[in_pixel_index + 1] * coefficient; + output_buffer[out_pixel_index + 2] += decode_buffer[in_pixel_index + 2] * coefficient; + output_buffer[out_pixel_index + 3] += decode_buffer[in_pixel_index + 3] * coefficient; + } + } + break; + + default: + for (x = 0; x < max_x; x++) + { + int n0 = horizontal_contributors[x].n0; + int n1 = horizontal_contributors[x].n1; + + int in_x = x - filter_pixel_margin; + int in_pixel_index = in_x * channels; + int max_n = n1; + int coefficient_group = coefficient_width * x; + + for (k = n0; k <= max_n; k++) + { + int c; + int out_pixel_index = k * channels; + float coefficient = horizontal_coefficients[coefficient_group + k - n0]; + STBIR_ASSERT(coefficient != 0); + for (c = 0; c < channels; c++) + output_buffer[out_pixel_index + c] += decode_buffer[in_pixel_index + c] * coefficient; + } + } + break; + } +} + +static void stbir__decode_and_resample_upsample(stbir__info* stbir_info, int n) +{ + // Decode the nth scanline from the source image into the decode buffer. + stbir__decode_scanline(stbir_info, n); + + // Now resample it into the ring buffer. + if (stbir__use_width_upsampling(stbir_info)) + stbir__resample_horizontal_upsample(stbir_info, stbir__add_empty_ring_buffer_entry(stbir_info, n)); + else + stbir__resample_horizontal_downsample(stbir_info, stbir__add_empty_ring_buffer_entry(stbir_info, n)); + + // Now it's sitting in the ring buffer ready to be used as source for the vertical sampling. +} + +static void stbir__decode_and_resample_downsample(stbir__info* stbir_info, int n) +{ + // Decode the nth scanline from the source image into the decode buffer. + stbir__decode_scanline(stbir_info, n); + + memset(stbir_info->horizontal_buffer, 0, stbir_info->output_w * stbir_info->channels * sizeof(float)); + + // Now resample it into the horizontal buffer. + if (stbir__use_width_upsampling(stbir_info)) + stbir__resample_horizontal_upsample(stbir_info, stbir_info->horizontal_buffer); + else + stbir__resample_horizontal_downsample(stbir_info, stbir_info->horizontal_buffer); + + // Now it's sitting in the horizontal buffer ready to be distributed into the ring buffers. +} + +// Get the specified scan line from the ring buffer. +static float* stbir__get_ring_buffer_scanline(int get_scanline, float* ring_buffer, int begin_index, int first_scanline, int ring_buffer_num_entries, int ring_buffer_length) +{ + int ring_buffer_index = (begin_index + (get_scanline - first_scanline)) % ring_buffer_num_entries; + return stbir__get_ring_buffer_entry(ring_buffer, ring_buffer_index, ring_buffer_length); +} + + +static void stbir__encode_scanline(stbir__info* stbir_info, int num_pixels, void *output_buffer, float *encode_buffer, int channels, int alpha_channel, int decode) +{ + int x; + int n; + int num_nonalpha; + stbir_uint16 nonalpha[STBIR_MAX_CHANNELS]; + + if (!(stbir_info->flags&STBIR_FLAG_ALPHA_PREMULTIPLIED)) + { + for (x=0; x < num_pixels; ++x) + { + int pixel_index = x*channels; + + float alpha = encode_buffer[pixel_index + alpha_channel]; + float reciprocal_alpha = alpha ? 1.0f / alpha : 0; + + // unrolling this produced a 1% slowdown upscaling a large RGBA linear-space image on my machine - stb + for (n = 0; n < channels; n++) + if (n != alpha_channel) + encode_buffer[pixel_index + n] *= reciprocal_alpha; + + // We added in a small epsilon to prevent the color channel from being deleted with zero alpha. + // Because we only add it for integer types, it will automatically be discarded on integer + // conversion, so we don't need to subtract it back out (which would be problematic for + // numeric precision reasons). + } + } + + // build a table of all channels that need colorspace correction, so + // we don't perform colorspace correction on channels that don't need it. + for (x = 0, num_nonalpha = 0; x < channels; ++x) + { + if (x != alpha_channel || (stbir_info->flags & STBIR_FLAG_ALPHA_USES_COLORSPACE)) + { + nonalpha[num_nonalpha++] = (stbir_uint16)x; + } + } + + #define STBIR__ROUND_INT(f) ((int) ((f)+0.5)) + #define STBIR__ROUND_UINT(f) ((stbir_uint32) ((f)+0.5)) + + #ifdef STBIR__SATURATE_INT + #define STBIR__ENCODE_LINEAR8(f) stbir__saturate8 (STBIR__ROUND_INT((f) * stbir__max_uint8_as_float )) + #define STBIR__ENCODE_LINEAR16(f) stbir__saturate16(STBIR__ROUND_INT((f) * stbir__max_uint16_as_float)) + #else + #define STBIR__ENCODE_LINEAR8(f) (unsigned char ) STBIR__ROUND_INT(stbir__saturate(f) * stbir__max_uint8_as_float ) + #define STBIR__ENCODE_LINEAR16(f) (unsigned short) STBIR__ROUND_INT(stbir__saturate(f) * stbir__max_uint16_as_float) + #endif + + switch (decode) + { + case STBIR__DECODE(STBIR_TYPE_UINT8, STBIR_COLORSPACE_LINEAR): + for (x=0; x < num_pixels; ++x) + { + int pixel_index = x*channels; + + for (n = 0; n < channels; n++) + { + int index = pixel_index + n; + ((unsigned char*)output_buffer)[index] = STBIR__ENCODE_LINEAR8(encode_buffer[index]); + } + } + break; + + case STBIR__DECODE(STBIR_TYPE_UINT8, STBIR_COLORSPACE_SRGB): + for (x=0; x < num_pixels; ++x) + { + int pixel_index = x*channels; + + for (n = 0; n < num_nonalpha; n++) + { + int index = pixel_index + nonalpha[n]; + ((unsigned char*)output_buffer)[index] = stbir__linear_to_srgb_uchar(encode_buffer[index]); + } + + if (!(stbir_info->flags & STBIR_FLAG_ALPHA_USES_COLORSPACE)) + ((unsigned char *)output_buffer)[pixel_index + alpha_channel] = STBIR__ENCODE_LINEAR8(encode_buffer[pixel_index+alpha_channel]); + } + break; + + case STBIR__DECODE(STBIR_TYPE_UINT16, STBIR_COLORSPACE_LINEAR): + for (x=0; x < num_pixels; ++x) + { + int pixel_index = x*channels; + + for (n = 0; n < channels; n++) + { + int index = pixel_index + n; + ((unsigned short*)output_buffer)[index] = STBIR__ENCODE_LINEAR16(encode_buffer[index]); + } + } + break; + + case STBIR__DECODE(STBIR_TYPE_UINT16, STBIR_COLORSPACE_SRGB): + for (x=0; x < num_pixels; ++x) + { + int pixel_index = x*channels; + + for (n = 0; n < num_nonalpha; n++) + { + int index = pixel_index + nonalpha[n]; + ((unsigned short*)output_buffer)[index] = (unsigned short)STBIR__ROUND_INT(stbir__linear_to_srgb(stbir__saturate(encode_buffer[index])) * stbir__max_uint16_as_float); + } + + if (!(stbir_info->flags&STBIR_FLAG_ALPHA_USES_COLORSPACE)) + ((unsigned short*)output_buffer)[pixel_index + alpha_channel] = STBIR__ENCODE_LINEAR16(encode_buffer[pixel_index + alpha_channel]); + } + + break; + + case STBIR__DECODE(STBIR_TYPE_UINT32, STBIR_COLORSPACE_LINEAR): + for (x=0; x < num_pixels; ++x) + { + int pixel_index = x*channels; + + for (n = 0; n < channels; n++) + { + int index = pixel_index + n; + ((unsigned int*)output_buffer)[index] = (unsigned int)STBIR__ROUND_UINT(((double)stbir__saturate(encode_buffer[index])) * stbir__max_uint32_as_float); + } + } + break; + + case STBIR__DECODE(STBIR_TYPE_UINT32, STBIR_COLORSPACE_SRGB): + for (x=0; x < num_pixels; ++x) + { + int pixel_index = x*channels; + + for (n = 0; n < num_nonalpha; n++) + { + int index = pixel_index + nonalpha[n]; + ((unsigned int*)output_buffer)[index] = (unsigned int)STBIR__ROUND_UINT(((double)stbir__linear_to_srgb(stbir__saturate(encode_buffer[index]))) * stbir__max_uint32_as_float); + } + + if (!(stbir_info->flags&STBIR_FLAG_ALPHA_USES_COLORSPACE)) + ((unsigned int*)output_buffer)[pixel_index + alpha_channel] = (unsigned int)STBIR__ROUND_INT(((double)stbir__saturate(encode_buffer[pixel_index + alpha_channel])) * stbir__max_uint32_as_float); + } + break; + + case STBIR__DECODE(STBIR_TYPE_FLOAT, STBIR_COLORSPACE_LINEAR): + for (x=0; x < num_pixels; ++x) + { + int pixel_index = x*channels; + + for (n = 0; n < channels; n++) + { + int index = pixel_index + n; + ((float*)output_buffer)[index] = encode_buffer[index]; + } + } + break; + + case STBIR__DECODE(STBIR_TYPE_FLOAT, STBIR_COLORSPACE_SRGB): + for (x=0; x < num_pixels; ++x) + { + int pixel_index = x*channels; + + for (n = 0; n < num_nonalpha; n++) + { + int index = pixel_index + nonalpha[n]; + ((float*)output_buffer)[index] = stbir__linear_to_srgb(encode_buffer[index]); + } + + if (!(stbir_info->flags&STBIR_FLAG_ALPHA_USES_COLORSPACE)) + ((float*)output_buffer)[pixel_index + alpha_channel] = encode_buffer[pixel_index + alpha_channel]; + } + break; + + default: + STBIR_ASSERT(!"Unknown type/colorspace/channels combination."); + break; + } +} + +static void stbir__resample_vertical_upsample(stbir__info* stbir_info, int n) +{ + int x, k; + int output_w = stbir_info->output_w; + stbir__contributors* vertical_contributors = stbir_info->vertical_contributors; + float* vertical_coefficients = stbir_info->vertical_coefficients; + int channels = stbir_info->channels; + int alpha_channel = stbir_info->alpha_channel; + int type = stbir_info->type; + int colorspace = stbir_info->colorspace; + int ring_buffer_entries = stbir_info->ring_buffer_num_entries; + void* output_data = stbir_info->output_data; + float* encode_buffer = stbir_info->encode_buffer; + int decode = STBIR__DECODE(type, colorspace); + int coefficient_width = stbir_info->vertical_coefficient_width; + int coefficient_counter; + int contributor = n; + + float* ring_buffer = stbir_info->ring_buffer; + int ring_buffer_begin_index = stbir_info->ring_buffer_begin_index; + int ring_buffer_first_scanline = stbir_info->ring_buffer_first_scanline; + int ring_buffer_length = stbir_info->ring_buffer_length_bytes/sizeof(float); + + int n0,n1, output_row_start; + int coefficient_group = coefficient_width * contributor; + + n0 = vertical_contributors[contributor].n0; + n1 = vertical_contributors[contributor].n1; + + output_row_start = n * stbir_info->output_stride_bytes; + + STBIR_ASSERT(stbir__use_height_upsampling(stbir_info)); + + memset(encode_buffer, 0, output_w * sizeof(float) * channels); + + // I tried reblocking this for better cache usage of encode_buffer + // (using x_outer, k, x_inner), but it lost speed. -- stb + + coefficient_counter = 0; + switch (channels) { + case 1: + for (k = n0; k <= n1; k++) + { + int coefficient_index = coefficient_counter++; + float* ring_buffer_entry = stbir__get_ring_buffer_scanline(k, ring_buffer, ring_buffer_begin_index, ring_buffer_first_scanline, ring_buffer_entries, ring_buffer_length); + float coefficient = vertical_coefficients[coefficient_group + coefficient_index]; + for (x = 0; x < output_w; ++x) + { + int in_pixel_index = x * 1; + encode_buffer[in_pixel_index + 0] += ring_buffer_entry[in_pixel_index + 0] * coefficient; + } + } + break; + case 2: + for (k = n0; k <= n1; k++) + { + int coefficient_index = coefficient_counter++; + float* ring_buffer_entry = stbir__get_ring_buffer_scanline(k, ring_buffer, ring_buffer_begin_index, ring_buffer_first_scanline, ring_buffer_entries, ring_buffer_length); + float coefficient = vertical_coefficients[coefficient_group + coefficient_index]; + for (x = 0; x < output_w; ++x) + { + int in_pixel_index = x * 2; + encode_buffer[in_pixel_index + 0] += ring_buffer_entry[in_pixel_index + 0] * coefficient; + encode_buffer[in_pixel_index + 1] += ring_buffer_entry[in_pixel_index + 1] * coefficient; + } + } + break; + case 3: + for (k = n0; k <= n1; k++) + { + int coefficient_index = coefficient_counter++; + float* ring_buffer_entry = stbir__get_ring_buffer_scanline(k, ring_buffer, ring_buffer_begin_index, ring_buffer_first_scanline, ring_buffer_entries, ring_buffer_length); + float coefficient = vertical_coefficients[coefficient_group + coefficient_index]; + for (x = 0; x < output_w; ++x) + { + int in_pixel_index = x * 3; + encode_buffer[in_pixel_index + 0] += ring_buffer_entry[in_pixel_index + 0] * coefficient; + encode_buffer[in_pixel_index + 1] += ring_buffer_entry[in_pixel_index + 1] * coefficient; + encode_buffer[in_pixel_index + 2] += ring_buffer_entry[in_pixel_index + 2] * coefficient; + } + } + break; + case 4: + for (k = n0; k <= n1; k++) + { + int coefficient_index = coefficient_counter++; + float* ring_buffer_entry = stbir__get_ring_buffer_scanline(k, ring_buffer, ring_buffer_begin_index, ring_buffer_first_scanline, ring_buffer_entries, ring_buffer_length); + float coefficient = vertical_coefficients[coefficient_group + coefficient_index]; + for (x = 0; x < output_w; ++x) + { + int in_pixel_index = x * 4; + encode_buffer[in_pixel_index + 0] += ring_buffer_entry[in_pixel_index + 0] * coefficient; + encode_buffer[in_pixel_index + 1] += ring_buffer_entry[in_pixel_index + 1] * coefficient; + encode_buffer[in_pixel_index + 2] += ring_buffer_entry[in_pixel_index + 2] * coefficient; + encode_buffer[in_pixel_index + 3] += ring_buffer_entry[in_pixel_index + 3] * coefficient; + } + } + break; + default: + for (k = n0; k <= n1; k++) + { + int coefficient_index = coefficient_counter++; + float* ring_buffer_entry = stbir__get_ring_buffer_scanline(k, ring_buffer, ring_buffer_begin_index, ring_buffer_first_scanline, ring_buffer_entries, ring_buffer_length); + float coefficient = vertical_coefficients[coefficient_group + coefficient_index]; + for (x = 0; x < output_w; ++x) + { + int in_pixel_index = x * channels; + int c; + for (c = 0; c < channels; c++) + encode_buffer[in_pixel_index + c] += ring_buffer_entry[in_pixel_index + c] * coefficient; + } + } + break; + } + stbir__encode_scanline(stbir_info, output_w, (char *) output_data + output_row_start, encode_buffer, channels, alpha_channel, decode); +} + +static void stbir__resample_vertical_downsample(stbir__info* stbir_info, int n) +{ + int x, k; + int output_w = stbir_info->output_w; + stbir__contributors* vertical_contributors = stbir_info->vertical_contributors; + float* vertical_coefficients = stbir_info->vertical_coefficients; + int channels = stbir_info->channels; + int ring_buffer_entries = stbir_info->ring_buffer_num_entries; + float* horizontal_buffer = stbir_info->horizontal_buffer; + int coefficient_width = stbir_info->vertical_coefficient_width; + int contributor = n + stbir_info->vertical_filter_pixel_margin; + + float* ring_buffer = stbir_info->ring_buffer; + int ring_buffer_begin_index = stbir_info->ring_buffer_begin_index; + int ring_buffer_first_scanline = stbir_info->ring_buffer_first_scanline; + int ring_buffer_length = stbir_info->ring_buffer_length_bytes/sizeof(float); + int n0,n1; + + n0 = vertical_contributors[contributor].n0; + n1 = vertical_contributors[contributor].n1; + + STBIR_ASSERT(!stbir__use_height_upsampling(stbir_info)); + + for (k = n0; k <= n1; k++) + { + int coefficient_index = k - n0; + int coefficient_group = coefficient_width * contributor; + float coefficient = vertical_coefficients[coefficient_group + coefficient_index]; + + float* ring_buffer_entry = stbir__get_ring_buffer_scanline(k, ring_buffer, ring_buffer_begin_index, ring_buffer_first_scanline, ring_buffer_entries, ring_buffer_length); + + switch (channels) { + case 1: + for (x = 0; x < output_w; x++) + { + int in_pixel_index = x * 1; + ring_buffer_entry[in_pixel_index + 0] += horizontal_buffer[in_pixel_index + 0] * coefficient; + } + break; + case 2: + for (x = 0; x < output_w; x++) + { + int in_pixel_index = x * 2; + ring_buffer_entry[in_pixel_index + 0] += horizontal_buffer[in_pixel_index + 0] * coefficient; + ring_buffer_entry[in_pixel_index + 1] += horizontal_buffer[in_pixel_index + 1] * coefficient; + } + break; + case 3: + for (x = 0; x < output_w; x++) + { + int in_pixel_index = x * 3; + ring_buffer_entry[in_pixel_index + 0] += horizontal_buffer[in_pixel_index + 0] * coefficient; + ring_buffer_entry[in_pixel_index + 1] += horizontal_buffer[in_pixel_index + 1] * coefficient; + ring_buffer_entry[in_pixel_index + 2] += horizontal_buffer[in_pixel_index + 2] * coefficient; + } + break; + case 4: + for (x = 0; x < output_w; x++) + { + int in_pixel_index = x * 4; + ring_buffer_entry[in_pixel_index + 0] += horizontal_buffer[in_pixel_index + 0] * coefficient; + ring_buffer_entry[in_pixel_index + 1] += horizontal_buffer[in_pixel_index + 1] * coefficient; + ring_buffer_entry[in_pixel_index + 2] += horizontal_buffer[in_pixel_index + 2] * coefficient; + ring_buffer_entry[in_pixel_index + 3] += horizontal_buffer[in_pixel_index + 3] * coefficient; + } + break; + default: + for (x = 0; x < output_w; x++) + { + int in_pixel_index = x * channels; + + int c; + for (c = 0; c < channels; c++) + ring_buffer_entry[in_pixel_index + c] += horizontal_buffer[in_pixel_index + c] * coefficient; + } + break; + } + } +} + +static void stbir__buffer_loop_upsample(stbir__info* stbir_info) +{ + int y; + float scale_ratio = stbir_info->vertical_scale; + float out_scanlines_radius = stbir__filter_info_table[stbir_info->vertical_filter].support(1/scale_ratio) * scale_ratio; + + STBIR_ASSERT(stbir__use_height_upsampling(stbir_info)); + + for (y = 0; y < stbir_info->output_h; y++) + { + float in_center_of_out = 0; // Center of the current out scanline in the in scanline space + int in_first_scanline = 0, in_last_scanline = 0; + + stbir__calculate_sample_range_upsample(y, out_scanlines_radius, scale_ratio, stbir_info->vertical_shift, &in_first_scanline, &in_last_scanline, &in_center_of_out); + + STBIR_ASSERT(in_last_scanline - in_first_scanline + 1 <= stbir_info->ring_buffer_num_entries); + + if (stbir_info->ring_buffer_begin_index >= 0) + { + // Get rid of whatever we don't need anymore. + while (in_first_scanline > stbir_info->ring_buffer_first_scanline) + { + if (stbir_info->ring_buffer_first_scanline == stbir_info->ring_buffer_last_scanline) + { + // We just popped the last scanline off the ring buffer. + // Reset it to the empty state. + stbir_info->ring_buffer_begin_index = -1; + stbir_info->ring_buffer_first_scanline = 0; + stbir_info->ring_buffer_last_scanline = 0; + break; + } + else + { + stbir_info->ring_buffer_first_scanline++; + stbir_info->ring_buffer_begin_index = (stbir_info->ring_buffer_begin_index + 1) % stbir_info->ring_buffer_num_entries; + } + } + } + + // Load in new ones. + if (stbir_info->ring_buffer_begin_index < 0) + stbir__decode_and_resample_upsample(stbir_info, in_first_scanline); + + while (in_last_scanline > stbir_info->ring_buffer_last_scanline) + stbir__decode_and_resample_upsample(stbir_info, stbir_info->ring_buffer_last_scanline + 1); + + // Now all buffers should be ready to write a row of vertical sampling. + stbir__resample_vertical_upsample(stbir_info, y); + + STBIR_PROGRESS_REPORT((float)y / stbir_info->output_h); + } +} + +static void stbir__empty_ring_buffer(stbir__info* stbir_info, int first_necessary_scanline) +{ + int output_stride_bytes = stbir_info->output_stride_bytes; + int channels = stbir_info->channels; + int alpha_channel = stbir_info->alpha_channel; + int type = stbir_info->type; + int colorspace = stbir_info->colorspace; + int output_w = stbir_info->output_w; + void* output_data = stbir_info->output_data; + int decode = STBIR__DECODE(type, colorspace); + + float* ring_buffer = stbir_info->ring_buffer; + int ring_buffer_length = stbir_info->ring_buffer_length_bytes/sizeof(float); + + if (stbir_info->ring_buffer_begin_index >= 0) + { + // Get rid of whatever we don't need anymore. + while (first_necessary_scanline > stbir_info->ring_buffer_first_scanline) + { + if (stbir_info->ring_buffer_first_scanline >= 0 && stbir_info->ring_buffer_first_scanline < stbir_info->output_h) + { + int output_row_start = stbir_info->ring_buffer_first_scanline * output_stride_bytes; + float* ring_buffer_entry = stbir__get_ring_buffer_entry(ring_buffer, stbir_info->ring_buffer_begin_index, ring_buffer_length); + stbir__encode_scanline(stbir_info, output_w, (char *) output_data + output_row_start, ring_buffer_entry, channels, alpha_channel, decode); + STBIR_PROGRESS_REPORT((float)stbir_info->ring_buffer_first_scanline / stbir_info->output_h); + } + + if (stbir_info->ring_buffer_first_scanline == stbir_info->ring_buffer_last_scanline) + { + // We just popped the last scanline off the ring buffer. + // Reset it to the empty state. + stbir_info->ring_buffer_begin_index = -1; + stbir_info->ring_buffer_first_scanline = 0; + stbir_info->ring_buffer_last_scanline = 0; + break; + } + else + { + stbir_info->ring_buffer_first_scanline++; + stbir_info->ring_buffer_begin_index = (stbir_info->ring_buffer_begin_index + 1) % stbir_info->ring_buffer_num_entries; + } + } + } +} + +static void stbir__buffer_loop_downsample(stbir__info* stbir_info) +{ + int y; + float scale_ratio = stbir_info->vertical_scale; + int output_h = stbir_info->output_h; + float in_pixels_radius = stbir__filter_info_table[stbir_info->vertical_filter].support(scale_ratio) / scale_ratio; + int pixel_margin = stbir_info->vertical_filter_pixel_margin; + int max_y = stbir_info->input_h + pixel_margin; + + STBIR_ASSERT(!stbir__use_height_upsampling(stbir_info)); + + for (y = -pixel_margin; y < max_y; y++) + { + float out_center_of_in; // Center of the current out scanline in the in scanline space + int out_first_scanline, out_last_scanline; + + stbir__calculate_sample_range_downsample(y, in_pixels_radius, scale_ratio, stbir_info->vertical_shift, &out_first_scanline, &out_last_scanline, &out_center_of_in); + + STBIR_ASSERT(out_last_scanline - out_first_scanline + 1 <= stbir_info->ring_buffer_num_entries); + + if (out_last_scanline < 0 || out_first_scanline >= output_h) + continue; + + stbir__empty_ring_buffer(stbir_info, out_first_scanline); + + stbir__decode_and_resample_downsample(stbir_info, y); + + // Load in new ones. + if (stbir_info->ring_buffer_begin_index < 0) + stbir__add_empty_ring_buffer_entry(stbir_info, out_first_scanline); + + while (out_last_scanline > stbir_info->ring_buffer_last_scanline) + stbir__add_empty_ring_buffer_entry(stbir_info, stbir_info->ring_buffer_last_scanline + 1); + + // Now the horizontal buffer is ready to write to all ring buffer rows. + stbir__resample_vertical_downsample(stbir_info, y); + } + + stbir__empty_ring_buffer(stbir_info, stbir_info->output_h); +} + +static void stbir__setup(stbir__info *info, int input_w, int input_h, int output_w, int output_h, int channels) +{ + info->input_w = input_w; + info->input_h = input_h; + info->output_w = output_w; + info->output_h = output_h; + info->channels = channels; +} + +static void stbir__calculate_transform(stbir__info *info, float s0, float t0, float s1, float t1, float *transform) +{ + info->s0 = s0; + info->t0 = t0; + info->s1 = s1; + info->t1 = t1; + + if (transform) + { + info->horizontal_scale = transform[0]; + info->vertical_scale = transform[1]; + info->horizontal_shift = transform[2]; + info->vertical_shift = transform[3]; + } + else + { + info->horizontal_scale = ((float)info->output_w / info->input_w) / (s1 - s0); + info->vertical_scale = ((float)info->output_h / info->input_h) / (t1 - t0); + + info->horizontal_shift = s0 * info->output_w / (s1 - s0); + info->vertical_shift = t0 * info->output_h / (t1 - t0); + } +} + +static void stbir__choose_filter(stbir__info *info, stbir_filter h_filter, stbir_filter v_filter) +{ + if (h_filter == 0) + h_filter = stbir__use_upsampling(info->horizontal_scale) ? STBIR_DEFAULT_FILTER_UPSAMPLE : STBIR_DEFAULT_FILTER_DOWNSAMPLE; + if (v_filter == 0) + v_filter = stbir__use_upsampling(info->vertical_scale) ? STBIR_DEFAULT_FILTER_UPSAMPLE : STBIR_DEFAULT_FILTER_DOWNSAMPLE; + info->horizontal_filter = h_filter; + info->vertical_filter = v_filter; +} + +static stbir_uint32 stbir__calculate_memory(stbir__info *info) +{ + int pixel_margin = stbir__get_filter_pixel_margin(info->horizontal_filter, info->horizontal_scale); + int filter_height = stbir__get_filter_pixel_width(info->vertical_filter, info->vertical_scale); + + info->horizontal_num_contributors = stbir__get_contributors(info->horizontal_scale, info->horizontal_filter, info->input_w, info->output_w); + info->vertical_num_contributors = stbir__get_contributors(info->vertical_scale , info->vertical_filter , info->input_h, info->output_h); + + // One extra entry because floating point precision problems sometimes cause an extra to be necessary. + info->ring_buffer_num_entries = filter_height + 1; + + info->horizontal_contributors_size = info->horizontal_num_contributors * sizeof(stbir__contributors); + info->horizontal_coefficients_size = stbir__get_total_horizontal_coefficients(info) * sizeof(float); + info->vertical_contributors_size = info->vertical_num_contributors * sizeof(stbir__contributors); + info->vertical_coefficients_size = stbir__get_total_vertical_coefficients(info) * sizeof(float); + info->decode_buffer_size = (info->input_w + pixel_margin * 2) * info->channels * sizeof(float); + info->horizontal_buffer_size = info->output_w * info->channels * sizeof(float); + info->ring_buffer_size = info->output_w * info->channels * info->ring_buffer_num_entries * sizeof(float); + info->encode_buffer_size = info->output_w * info->channels * sizeof(float); + + STBIR_ASSERT(info->horizontal_filter != 0); + STBIR_ASSERT(info->horizontal_filter < STBIR__ARRAY_SIZE(stbir__filter_info_table)); // this now happens too late + STBIR_ASSERT(info->vertical_filter != 0); + STBIR_ASSERT(info->vertical_filter < STBIR__ARRAY_SIZE(stbir__filter_info_table)); // this now happens too late + + if (stbir__use_height_upsampling(info)) + // The horizontal buffer is for when we're downsampling the height and we + // can't output the result of sampling the decode buffer directly into the + // ring buffers. + info->horizontal_buffer_size = 0; + else + // The encode buffer is to retain precision in the height upsampling method + // and isn't used when height downsampling. + info->encode_buffer_size = 0; + + return info->horizontal_contributors_size + info->horizontal_coefficients_size + + info->vertical_contributors_size + info->vertical_coefficients_size + + info->decode_buffer_size + info->horizontal_buffer_size + + info->ring_buffer_size + info->encode_buffer_size; +} + +static int stbir__resize_allocated(stbir__info *info, + const void* input_data, int input_stride_in_bytes, + void* output_data, int output_stride_in_bytes, + int alpha_channel, stbir_uint32 flags, stbir_datatype type, + stbir_edge edge_horizontal, stbir_edge edge_vertical, stbir_colorspace colorspace, + void* tempmem, size_t tempmem_size_in_bytes) +{ + size_t memory_required = stbir__calculate_memory(info); + + int width_stride_input = input_stride_in_bytes ? input_stride_in_bytes : info->channels * info->input_w * stbir__type_size[type]; + int width_stride_output = output_stride_in_bytes ? output_stride_in_bytes : info->channels * info->output_w * stbir__type_size[type]; + +#ifdef STBIR_DEBUG_OVERWRITE_TEST +#define OVERWRITE_ARRAY_SIZE 8 + unsigned char overwrite_output_before_pre[OVERWRITE_ARRAY_SIZE]; + unsigned char overwrite_tempmem_before_pre[OVERWRITE_ARRAY_SIZE]; + unsigned char overwrite_output_after_pre[OVERWRITE_ARRAY_SIZE]; + unsigned char overwrite_tempmem_after_pre[OVERWRITE_ARRAY_SIZE]; + + size_t begin_forbidden = width_stride_output * (info->output_h - 1) + info->output_w * info->channels * stbir__type_size[type]; + memcpy(overwrite_output_before_pre, &((unsigned char*)output_data)[-OVERWRITE_ARRAY_SIZE], OVERWRITE_ARRAY_SIZE); + memcpy(overwrite_output_after_pre, &((unsigned char*)output_data)[begin_forbidden], OVERWRITE_ARRAY_SIZE); + memcpy(overwrite_tempmem_before_pre, &((unsigned char*)tempmem)[-OVERWRITE_ARRAY_SIZE], OVERWRITE_ARRAY_SIZE); + memcpy(overwrite_tempmem_after_pre, &((unsigned char*)tempmem)[tempmem_size_in_bytes], OVERWRITE_ARRAY_SIZE); +#endif + + STBIR_ASSERT(info->channels >= 0); + STBIR_ASSERT(info->channels <= STBIR_MAX_CHANNELS); + + if (info->channels < 0 || info->channels > STBIR_MAX_CHANNELS) + return 0; + + STBIR_ASSERT(info->horizontal_filter < STBIR__ARRAY_SIZE(stbir__filter_info_table)); + STBIR_ASSERT(info->vertical_filter < STBIR__ARRAY_SIZE(stbir__filter_info_table)); + + if (info->horizontal_filter >= STBIR__ARRAY_SIZE(stbir__filter_info_table)) + return 0; + if (info->vertical_filter >= STBIR__ARRAY_SIZE(stbir__filter_info_table)) + return 0; + + if (alpha_channel < 0) + flags |= STBIR_FLAG_ALPHA_USES_COLORSPACE | STBIR_FLAG_ALPHA_PREMULTIPLIED; + + if (!(flags&STBIR_FLAG_ALPHA_USES_COLORSPACE) || !(flags&STBIR_FLAG_ALPHA_PREMULTIPLIED)) + STBIR_ASSERT(alpha_channel >= 0 && alpha_channel < info->channels); + + if (alpha_channel >= info->channels) + return 0; + + STBIR_ASSERT(tempmem); + + if (!tempmem) + return 0; + + STBIR_ASSERT(tempmem_size_in_bytes >= memory_required); + + if (tempmem_size_in_bytes < memory_required) + return 0; + + memset(tempmem, 0, tempmem_size_in_bytes); + + info->input_data = input_data; + info->input_stride_bytes = width_stride_input; + + info->output_data = output_data; + info->output_stride_bytes = width_stride_output; + + info->alpha_channel = alpha_channel; + info->flags = flags; + info->type = type; + info->edge_horizontal = edge_horizontal; + info->edge_vertical = edge_vertical; + info->colorspace = colorspace; + + info->horizontal_coefficient_width = stbir__get_coefficient_width (info->horizontal_filter, info->horizontal_scale); + info->vertical_coefficient_width = stbir__get_coefficient_width (info->vertical_filter , info->vertical_scale ); + info->horizontal_filter_pixel_width = stbir__get_filter_pixel_width (info->horizontal_filter, info->horizontal_scale); + info->vertical_filter_pixel_width = stbir__get_filter_pixel_width (info->vertical_filter , info->vertical_scale ); + info->horizontal_filter_pixel_margin = stbir__get_filter_pixel_margin(info->horizontal_filter, info->horizontal_scale); + info->vertical_filter_pixel_margin = stbir__get_filter_pixel_margin(info->vertical_filter , info->vertical_scale ); + + info->ring_buffer_length_bytes = info->output_w * info->channels * sizeof(float); + info->decode_buffer_pixels = info->input_w + info->horizontal_filter_pixel_margin * 2; + +#define STBIR__NEXT_MEMPTR(current, newtype) (newtype*)(((unsigned char*)current) + current##_size) + + info->horizontal_contributors = (stbir__contributors *) tempmem; + info->horizontal_coefficients = STBIR__NEXT_MEMPTR(info->horizontal_contributors, float); + info->vertical_contributors = STBIR__NEXT_MEMPTR(info->horizontal_coefficients, stbir__contributors); + info->vertical_coefficients = STBIR__NEXT_MEMPTR(info->vertical_contributors, float); + info->decode_buffer = STBIR__NEXT_MEMPTR(info->vertical_coefficients, float); + + if (stbir__use_height_upsampling(info)) + { + info->horizontal_buffer = NULL; + info->ring_buffer = STBIR__NEXT_MEMPTR(info->decode_buffer, float); + info->encode_buffer = STBIR__NEXT_MEMPTR(info->ring_buffer, float); + + STBIR_ASSERT((size_t)STBIR__NEXT_MEMPTR(info->encode_buffer, unsigned char) == (size_t)tempmem + tempmem_size_in_bytes); + } + else + { + info->horizontal_buffer = STBIR__NEXT_MEMPTR(info->decode_buffer, float); + info->ring_buffer = STBIR__NEXT_MEMPTR(info->horizontal_buffer, float); + info->encode_buffer = NULL; + + STBIR_ASSERT((size_t)STBIR__NEXT_MEMPTR(info->ring_buffer, unsigned char) == (size_t)tempmem + tempmem_size_in_bytes); + } + +#undef STBIR__NEXT_MEMPTR + + // This signals that the ring buffer is empty + info->ring_buffer_begin_index = -1; + + stbir__calculate_filters(info->horizontal_contributors, info->horizontal_coefficients, info->horizontal_filter, info->horizontal_scale, info->horizontal_shift, info->input_w, info->output_w); + stbir__calculate_filters(info->vertical_contributors, info->vertical_coefficients, info->vertical_filter, info->vertical_scale, info->vertical_shift, info->input_h, info->output_h); + + STBIR_PROGRESS_REPORT(0); + + if (stbir__use_height_upsampling(info)) + stbir__buffer_loop_upsample(info); + else + stbir__buffer_loop_downsample(info); + + STBIR_PROGRESS_REPORT(1); + +#ifdef STBIR_DEBUG_OVERWRITE_TEST + STBIR_ASSERT(memcmp(overwrite_output_before_pre, &((unsigned char*)output_data)[-OVERWRITE_ARRAY_SIZE], OVERWRITE_ARRAY_SIZE) == 0); + STBIR_ASSERT(memcmp(overwrite_output_after_pre, &((unsigned char*)output_data)[begin_forbidden], OVERWRITE_ARRAY_SIZE) == 0); + STBIR_ASSERT(memcmp(overwrite_tempmem_before_pre, &((unsigned char*)tempmem)[-OVERWRITE_ARRAY_SIZE], OVERWRITE_ARRAY_SIZE) == 0); + STBIR_ASSERT(memcmp(overwrite_tempmem_after_pre, &((unsigned char*)tempmem)[tempmem_size_in_bytes], OVERWRITE_ARRAY_SIZE) == 0); +#endif + + return 1; +} + + +static int stbir__resize_arbitrary( + void *alloc_context, + const void* input_data, int input_w, int input_h, int input_stride_in_bytes, + void* output_data, int output_w, int output_h, int output_stride_in_bytes, + float s0, float t0, float s1, float t1, float *transform, + int channels, int alpha_channel, stbir_uint32 flags, stbir_datatype type, + stbir_filter h_filter, stbir_filter v_filter, + stbir_edge edge_horizontal, stbir_edge edge_vertical, stbir_colorspace colorspace) +{ + stbir__info info; + int result; + size_t memory_required; + void* extra_memory; + + stbir__setup(&info, input_w, input_h, output_w, output_h, channels); + stbir__calculate_transform(&info, s0,t0,s1,t1,transform); + stbir__choose_filter(&info, h_filter, v_filter); + memory_required = stbir__calculate_memory(&info); + extra_memory = STBIR_MALLOC(memory_required, alloc_context); + + if (!extra_memory) + return 0; + + result = stbir__resize_allocated(&info, input_data, input_stride_in_bytes, + output_data, output_stride_in_bytes, + alpha_channel, flags, type, + edge_horizontal, edge_vertical, + colorspace, extra_memory, memory_required); + + STBIR_FREE(extra_memory, alloc_context); + + return result; +} + +STBIRDEF int stbir_resize_uint8( const unsigned char *input_pixels , int input_w , int input_h , int input_stride_in_bytes, + unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + int num_channels) +{ + return stbir__resize_arbitrary(NULL, input_pixels, input_w, input_h, input_stride_in_bytes, + output_pixels, output_w, output_h, output_stride_in_bytes, + 0,0,1,1,NULL,num_channels,-1,0, STBIR_TYPE_UINT8, STBIR_FILTER_DEFAULT, STBIR_FILTER_DEFAULT, + STBIR_EDGE_CLAMP, STBIR_EDGE_CLAMP, STBIR_COLORSPACE_LINEAR); +} + +STBIRDEF int stbir_resize_float( const float *input_pixels , int input_w , int input_h , int input_stride_in_bytes, + float *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + int num_channels) +{ + return stbir__resize_arbitrary(NULL, input_pixels, input_w, input_h, input_stride_in_bytes, + output_pixels, output_w, output_h, output_stride_in_bytes, + 0,0,1,1,NULL,num_channels,-1,0, STBIR_TYPE_FLOAT, STBIR_FILTER_DEFAULT, STBIR_FILTER_DEFAULT, + STBIR_EDGE_CLAMP, STBIR_EDGE_CLAMP, STBIR_COLORSPACE_LINEAR); +} + +STBIRDEF int stbir_resize_uint8_srgb(const unsigned char *input_pixels , int input_w , int input_h , int input_stride_in_bytes, + unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + int num_channels, int alpha_channel, int flags) +{ + return stbir__resize_arbitrary(NULL, input_pixels, input_w, input_h, input_stride_in_bytes, + output_pixels, output_w, output_h, output_stride_in_bytes, + 0,0,1,1,NULL,num_channels,alpha_channel,flags, STBIR_TYPE_UINT8, STBIR_FILTER_DEFAULT, STBIR_FILTER_DEFAULT, + STBIR_EDGE_CLAMP, STBIR_EDGE_CLAMP, STBIR_COLORSPACE_SRGB); +} + +STBIRDEF int stbir_resize_uint8_srgb_edgemode(const unsigned char *input_pixels , int input_w , int input_h , int input_stride_in_bytes, + unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + int num_channels, int alpha_channel, int flags, + stbir_edge edge_wrap_mode) +{ + return stbir__resize_arbitrary(NULL, input_pixels, input_w, input_h, input_stride_in_bytes, + output_pixels, output_w, output_h, output_stride_in_bytes, + 0,0,1,1,NULL,num_channels,alpha_channel,flags, STBIR_TYPE_UINT8, STBIR_FILTER_DEFAULT, STBIR_FILTER_DEFAULT, + edge_wrap_mode, edge_wrap_mode, STBIR_COLORSPACE_SRGB); +} + +STBIRDEF int stbir_resize_uint8_generic( const unsigned char *input_pixels , int input_w , int input_h , int input_stride_in_bytes, + unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + int num_channels, int alpha_channel, int flags, + stbir_edge edge_wrap_mode, stbir_filter filter, stbir_colorspace space, + void *alloc_context) +{ + return stbir__resize_arbitrary(alloc_context, input_pixels, input_w, input_h, input_stride_in_bytes, + output_pixels, output_w, output_h, output_stride_in_bytes, + 0,0,1,1,NULL,num_channels,alpha_channel,flags, STBIR_TYPE_UINT8, filter, filter, + edge_wrap_mode, edge_wrap_mode, space); +} + +STBIRDEF int stbir_resize_uint16_generic(const stbir_uint16 *input_pixels , int input_w , int input_h , int input_stride_in_bytes, + stbir_uint16 *output_pixels , int output_w, int output_h, int output_stride_in_bytes, + int num_channels, int alpha_channel, int flags, + stbir_edge edge_wrap_mode, stbir_filter filter, stbir_colorspace space, + void *alloc_context) +{ + return stbir__resize_arbitrary(alloc_context, input_pixels, input_w, input_h, input_stride_in_bytes, + output_pixels, output_w, output_h, output_stride_in_bytes, + 0,0,1,1,NULL,num_channels,alpha_channel,flags, STBIR_TYPE_UINT16, filter, filter, + edge_wrap_mode, edge_wrap_mode, space); +} + + +STBIRDEF int stbir_resize_float_generic( const float *input_pixels , int input_w , int input_h , int input_stride_in_bytes, + float *output_pixels , int output_w, int output_h, int output_stride_in_bytes, + int num_channels, int alpha_channel, int flags, + stbir_edge edge_wrap_mode, stbir_filter filter, stbir_colorspace space, + void *alloc_context) +{ + return stbir__resize_arbitrary(alloc_context, input_pixels, input_w, input_h, input_stride_in_bytes, + output_pixels, output_w, output_h, output_stride_in_bytes, + 0,0,1,1,NULL,num_channels,alpha_channel,flags, STBIR_TYPE_FLOAT, filter, filter, + edge_wrap_mode, edge_wrap_mode, space); +} + + +STBIRDEF int stbir_resize( const void *input_pixels , int input_w , int input_h , int input_stride_in_bytes, + void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + stbir_datatype datatype, + int num_channels, int alpha_channel, int flags, + stbir_edge edge_mode_horizontal, stbir_edge edge_mode_vertical, + stbir_filter filter_horizontal, stbir_filter filter_vertical, + stbir_colorspace space, void *alloc_context) +{ + return stbir__resize_arbitrary(alloc_context, input_pixels, input_w, input_h, input_stride_in_bytes, + output_pixels, output_w, output_h, output_stride_in_bytes, + 0,0,1,1,NULL,num_channels,alpha_channel,flags, datatype, filter_horizontal, filter_vertical, + edge_mode_horizontal, edge_mode_vertical, space); +} + + +STBIRDEF int stbir_resize_subpixel(const void *input_pixels , int input_w , int input_h , int input_stride_in_bytes, + void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + stbir_datatype datatype, + int num_channels, int alpha_channel, int flags, + stbir_edge edge_mode_horizontal, stbir_edge edge_mode_vertical, + stbir_filter filter_horizontal, stbir_filter filter_vertical, + stbir_colorspace space, void *alloc_context, + float x_scale, float y_scale, + float x_offset, float y_offset) +{ + float transform[4]; + transform[0] = x_scale; + transform[1] = y_scale; + transform[2] = x_offset; + transform[3] = y_offset; + return stbir__resize_arbitrary(alloc_context, input_pixels, input_w, input_h, input_stride_in_bytes, + output_pixels, output_w, output_h, output_stride_in_bytes, + 0,0,1,1,transform,num_channels,alpha_channel,flags, datatype, filter_horizontal, filter_vertical, + edge_mode_horizontal, edge_mode_vertical, space); +} + +STBIRDEF int stbir_resize_region( const void *input_pixels , int input_w , int input_h , int input_stride_in_bytes, + void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, + stbir_datatype datatype, + int num_channels, int alpha_channel, int flags, + stbir_edge edge_mode_horizontal, stbir_edge edge_mode_vertical, + stbir_filter filter_horizontal, stbir_filter filter_vertical, + stbir_colorspace space, void *alloc_context, + float s0, float t0, float s1, float t1) +{ + return stbir__resize_arbitrary(alloc_context, input_pixels, input_w, input_h, input_stride_in_bytes, + output_pixels, output_w, output_h, output_stride_in_bytes, + s0,t0,s1,t1,NULL,num_channels,alpha_channel,flags, datatype, filter_horizontal, filter_vertical, + edge_mode_horizontal, edge_mode_vertical, space); +} + +#endif // STB_IMAGE_RESIZE_IMPLEMENTATION + +/* +------------------------------------------------------------------------------ +This software is available under 2 licenses -- choose whichever you prefer. +------------------------------------------------------------------------------ +ALTERNATIVE A - MIT License +Copyright (c) 2017 Sean Barrett +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +------------------------------------------------------------------------------ +ALTERNATIVE B - Public Domain (www.unlicense.org) +This is free and unencumbered software released into the public domain. +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this +software, either in source code form or as a compiled binary, for any purpose, +commercial or non-commercial, and by any means. +In jurisdictions that recognize copyright laws, the author or authors of this +software dedicate any and all copyright interest in the software to the public +domain. We make this dedication for the benefit of the public at large and to +the detriment of our heirs and successors. We intend this dedication to be an +overt act of relinquishment in perpetuity of all present and future rights to +this software under copyright law. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +------------------------------------------------------------------------------ +*/ diff --git a/v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image_write.h b/v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image_write.h new file mode 100644 index 0000000000..6f36938278 --- /dev/null +++ b/v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image_write.h @@ -0,0 +1,1621 @@ +/* stb_image_write - v1.11 - public domain - http://nothings.org/stb/stb_image_write.h + writes out PNG/BMP/TGA/JPEG/HDR images to C stdio - Sean Barrett 2010-2015 + no warranty implied; use at your own risk + + Before #including, + + #define STB_IMAGE_WRITE_IMPLEMENTATION + + in the file that you want to have the implementation. + + Will probably not work correctly with strict-aliasing optimizations. + + If using a modern Microsoft Compiler, non-safe versions of CRT calls may cause + compilation warnings or even errors. To avoid this, also before #including, + + #define STBI_MSC_SECURE_CRT + +ABOUT: + + This header file is a library for writing images to C stdio or a callback. + + The PNG output is not optimal; it is 20-50% larger than the file + written by a decent optimizing implementation; though providing a custom + zlib compress function (see STBIW_ZLIB_COMPRESS) can mitigate that. + This library is designed for source code compactness and simplicity, + not optimal image file size or run-time performance. + +BUILDING: + + You can #define STBIW_ASSERT(x) before the #include to avoid using assert.h. + You can #define STBIW_MALLOC(), STBIW_REALLOC(), and STBIW_FREE() to replace + malloc,realloc,free. + You can #define STBIW_MEMMOVE() to replace memmove() + You can #define STBIW_ZLIB_COMPRESS to use a custom zlib-style compress function + for PNG compression (instead of the builtin one), it must have the following signature: + unsigned char * my_compress(unsigned char *data, int data_len, int *out_len, int quality); + The returned data will be freed with STBIW_FREE() (free() by default), + so it must be heap allocated with STBIW_MALLOC() (malloc() by default), + +UNICODE: + + If compiling for Windows and you wish to use Unicode filenames, compile + with + #define STBIW_WINDOWS_UTF8 + and pass utf8-encoded filenames. Call stbiw_convert_wchar_to_utf8 to convert + Windows wchar_t filenames to utf8. + +USAGE: + + There are five functions, one for each image file format: + + int stbi_write_png(char const *filename, int w, int h, int comp, const void *data, int stride_in_bytes); + int stbi_write_bmp(char const *filename, int w, int h, int comp, const void *data); + int stbi_write_tga(char const *filename, int w, int h, int comp, const void *data); + int stbi_write_jpg(char const *filename, int w, int h, int comp, const void *data, int quality); + int stbi_write_hdr(char const *filename, int w, int h, int comp, const float *data); + + void stbi_flip_vertically_on_write(int flag); // flag is non-zero to flip data vertically + + There are also five equivalent functions that use an arbitrary write function. You are + expected to open/close your file-equivalent before and after calling these: + + int stbi_write_png_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data, int stride_in_bytes); + int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); + int stbi_write_tga_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); + int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const float *data); + int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int quality); + + where the callback is: + void stbi_write_func(void *context, void *data, int size); + + You can configure it with these global variables: + int stbi_write_tga_with_rle; // defaults to true; set to 0 to disable RLE + int stbi_write_png_compression_level; // defaults to 8; set to higher for more compression + int stbi_write_force_png_filter; // defaults to -1; set to 0..5 to force a filter mode + + + You can define STBI_WRITE_NO_STDIO to disable the file variant of these + functions, so the library will not use stdio.h at all. However, this will + also disable HDR writing, because it requires stdio for formatted output. + + Each function returns 0 on failure and non-0 on success. + + The functions create an image file defined by the parameters. The image + is a rectangle of pixels stored from left-to-right, top-to-bottom. + Each pixel contains 'comp' channels of data stored interleaved with 8-bits + per channel, in the following order: 1=Y, 2=YA, 3=RGB, 4=RGBA. (Y is + monochrome color.) The rectangle is 'w' pixels wide and 'h' pixels tall. + The *data pointer points to the first byte of the top-left-most pixel. + For PNG, "stride_in_bytes" is the distance in bytes from the first byte of + a row of pixels to the first byte of the next row of pixels. + + PNG creates output files with the same number of components as the input. + The BMP format expands Y to RGB in the file format and does not + output alpha. + + PNG supports writing rectangles of data even when the bytes storing rows of + data are not consecutive in memory (e.g. sub-rectangles of a larger image), + by supplying the stride between the beginning of adjacent rows. The other + formats do not. (Thus you cannot write a native-format BMP through the BMP + writer, both because it is in BGR order and because it may have padding + at the end of the line.) + + PNG allows you to set the deflate compression level by setting the global + variable 'stbi_write_png_compression_level' (it defaults to 8). + + HDR expects linear float data. Since the format is always 32-bit rgb(e) + data, alpha (if provided) is discarded, and for monochrome data it is + replicated across all three channels. + + TGA supports RLE or non-RLE compressed data. To use non-RLE-compressed + data, set the global variable 'stbi_write_tga_with_rle' to 0. + + JPEG does ignore alpha channels in input data; quality is between 1 and 100. + Higher quality looks better but results in a bigger image. + JPEG baseline (no JPEG progressive). + +CREDITS: + + + Sean Barrett - PNG/BMP/TGA + Baldur Karlsson - HDR + Jean-Sebastien Guay - TGA monochrome + Tim Kelsey - misc enhancements + Alan Hickman - TGA RLE + Emmanuel Julien - initial file IO callback implementation + Jon Olick - original jo_jpeg.cpp code + Daniel Gibson - integrate JPEG, allow external zlib + Aarni Koskela - allow choosing PNG filter + + bugfixes: + github:Chribba + Guillaume Chereau + github:jry2 + github:romigrou + Sergio Gonzalez + Jonas Karlsson + Filip Wasil + Thatcher Ulrich + github:poppolopoppo + Patrick Boettcher + github:xeekworx + Cap Petschulat + Simon Rodriguez + Ivan Tikhonov + github:ignotion + Adam Schackart + +LICENSE + + See end of file for license information. + +*/ + +#ifndef INCLUDE_STB_IMAGE_WRITE_H +#define INCLUDE_STB_IMAGE_WRITE_H + +#include + +// if STB_IMAGE_WRITE_STATIC causes problems, try defining STBIWDEF to 'inline' or 'static inline' +#ifndef STBIWDEF +#ifdef STB_IMAGE_WRITE_STATIC +#define STBIWDEF static +#else +#ifdef __cplusplus +#define STBIWDEF extern "C" +#else +#define STBIWDEF extern +#endif +#endif +#endif + +#ifndef STB_IMAGE_WRITE_STATIC // C++ forbids static forward declarations +extern int stbi_write_tga_with_rle; +extern int stbi_write_png_compression_level; +extern int stbi_write_force_png_filter; +#endif + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_png(char const *filename, int w, int h, int comp, const void *data, int stride_in_bytes); +STBIWDEF int stbi_write_bmp(char const *filename, int w, int h, int comp, const void *data); +STBIWDEF int stbi_write_tga(char const *filename, int w, int h, int comp, const void *data); +STBIWDEF int stbi_write_hdr(char const *filename, int w, int h, int comp, const float *data); +STBIWDEF int stbi_write_jpg(char const *filename, int x, int y, int comp, const void *data, int quality); + +#ifdef STBI_WINDOWS_UTF8 +STBIWDEF int stbiw_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input); +#endif +#endif + +typedef void stbi_write_func(void *context, void *data, int size); + +STBIWDEF int stbi_write_png_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data, int stride_in_bytes); +STBIWDEF int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); +STBIWDEF int stbi_write_tga_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); +STBIWDEF int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const float *data); +STBIWDEF int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int quality); + +STBIWDEF void stbi_flip_vertically_on_write(int flip_boolean); + +#endif//INCLUDE_STB_IMAGE_WRITE_H + +#ifdef STB_IMAGE_WRITE_IMPLEMENTATION + +#ifdef _WIN32 + #ifndef _CRT_SECURE_NO_WARNINGS + #define _CRT_SECURE_NO_WARNINGS + #endif + #ifndef _CRT_NONSTDC_NO_DEPRECATE + #define _CRT_NONSTDC_NO_DEPRECATE + #endif +#endif + +#ifndef STBI_WRITE_NO_STDIO +#include +#endif // STBI_WRITE_NO_STDIO + +#include +#include +#include +#include + +#if defined(STBIW_MALLOC) && defined(STBIW_FREE) && (defined(STBIW_REALLOC) || defined(STBIW_REALLOC_SIZED)) +// ok +#elif !defined(STBIW_MALLOC) && !defined(STBIW_FREE) && !defined(STBIW_REALLOC) && !defined(STBIW_REALLOC_SIZED) +// ok +#else +#error "Must define all or none of STBIW_MALLOC, STBIW_FREE, and STBIW_REALLOC (or STBIW_REALLOC_SIZED)." +#endif + +#ifndef STBIW_MALLOC +#define STBIW_MALLOC(sz) malloc(sz) +#define STBIW_REALLOC(p,newsz) realloc(p,newsz) +#define STBIW_FREE(p) free(p) +#endif + +#ifndef STBIW_REALLOC_SIZED +#define STBIW_REALLOC_SIZED(p,oldsz,newsz) STBIW_REALLOC(p,newsz) +#endif + + +#ifndef STBIW_MEMMOVE +#define STBIW_MEMMOVE(a,b,sz) memmove(a,b,sz) +#endif + + +#ifndef STBIW_ASSERT +#include +#define STBIW_ASSERT(x) assert(x) +#endif + +#define STBIW_UCHAR(x) (unsigned char) ((x) & 0xff) + +#ifdef STB_IMAGE_WRITE_STATIC +static int stbi__flip_vertically_on_write=0; +static int stbi_write_png_compression_level = 8; +static int stbi_write_tga_with_rle = 1; +static int stbi_write_force_png_filter = -1; +#else +int stbi_write_png_compression_level = 8; +int stbi__flip_vertically_on_write=0; +int stbi_write_tga_with_rle = 1; +int stbi_write_force_png_filter = -1; +#endif + +STBIWDEF void stbi_flip_vertically_on_write(int flag) +{ + stbi__flip_vertically_on_write = flag; +} + +typedef struct +{ + stbi_write_func *func; + void *context; +} stbi__write_context; + +// initialize a callback-based context +static void stbi__start_write_callbacks(stbi__write_context *s, stbi_write_func *c, void *context) +{ + s->func = c; + s->context = context; +} + +#ifndef STBI_WRITE_NO_STDIO + +static void stbi__stdio_write(void *context, void *data, int size) +{ + fwrite(data,1,size,(FILE*) context); +} + +#if defined(_MSC_VER) && defined(STBI_WINDOWS_UTF8) +#ifdef __cplusplus +#define STBIW_EXTERN extern "C" +#else +#define STBIW_EXTERN extern +#endif +STBIW_EXTERN __declspec(dllimport) int __stdcall MultiByteToWideChar(unsigned int cp, unsigned long flags, const char *str, int cbmb, wchar_t *widestr, int cchwide); +STBIW_EXTERN __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int cp, unsigned long flags, const wchar_t *widestr, int cchwide, char *str, int cbmb, const char *defchar, int *used_default); + +STBIWDEF int stbiw_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input) +{ + return WideCharToMultiByte(65001 /* UTF8 */, 0, input, -1, buffer, bufferlen, NULL, NULL); +} +#endif + +static FILE *stbiw__fopen(char const *filename, char const *mode) +{ + FILE *f; +#if defined(_MSC_VER) && defined(STBI_WINDOWS_UTF8) + wchar_t wMode[64]; + wchar_t wFilename[1024]; + if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, filename, -1, wFilename, sizeof(wFilename))) + return 0; + + if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, mode, -1, wMode, sizeof(wMode))) + return 0; + +#if _MSC_VER >= 1400 + if (0 != _wfopen_s(&f, wFilename, wMode)) + f = 0; +#else + f = _wfopen(wFilename, wMode); +#endif + +#elif defined(_MSC_VER) && _MSC_VER >= 1400 + if (0 != fopen_s(&f, filename, mode)) + f=0; +#else + f = fopen(filename, mode); +#endif + return f; +} + +static int stbi__start_write_file(stbi__write_context *s, const char *filename) +{ + FILE *f = stbiw__fopen(filename, "wb"); + stbi__start_write_callbacks(s, stbi__stdio_write, (void *) f); + return f != NULL; +} + +static void stbi__end_write_file(stbi__write_context *s) +{ + fclose((FILE *)s->context); +} + +#endif // !STBI_WRITE_NO_STDIO + +typedef unsigned int stbiw_uint32; +typedef int stb_image_write_test[sizeof(stbiw_uint32)==4 ? 1 : -1]; + +static void stbiw__writefv(stbi__write_context *s, const char *fmt, va_list v) +{ + while (*fmt) { + switch (*fmt++) { + case ' ': break; + case '1': { unsigned char x = STBIW_UCHAR(va_arg(v, int)); + s->func(s->context,&x,1); + break; } + case '2': { int x = va_arg(v,int); + unsigned char b[2]; + b[0] = STBIW_UCHAR(x); + b[1] = STBIW_UCHAR(x>>8); + s->func(s->context,b,2); + break; } + case '4': { stbiw_uint32 x = va_arg(v,int); + unsigned char b[4]; + b[0]=STBIW_UCHAR(x); + b[1]=STBIW_UCHAR(x>>8); + b[2]=STBIW_UCHAR(x>>16); + b[3]=STBIW_UCHAR(x>>24); + s->func(s->context,b,4); + break; } + default: + STBIW_ASSERT(0); + return; + } + } +} + +static void stbiw__writef(stbi__write_context *s, const char *fmt, ...) +{ + va_list v; + va_start(v, fmt); + stbiw__writefv(s, fmt, v); + va_end(v); +} + +static void stbiw__putc(stbi__write_context *s, unsigned char c) +{ + s->func(s->context, &c, 1); +} + +static void stbiw__write3(stbi__write_context *s, unsigned char a, unsigned char b, unsigned char c) +{ + unsigned char arr[3]; + arr[0] = a, arr[1] = b, arr[2] = c; + s->func(s->context, arr, 3); +} + +static void stbiw__write_pixel(stbi__write_context *s, int rgb_dir, int comp, int write_alpha, int expand_mono, unsigned char *d) +{ + unsigned char bg[3] = { 255, 0, 255}, px[3]; + int k; + + if (write_alpha < 0) + s->func(s->context, &d[comp - 1], 1); + + switch (comp) { + case 2: // 2 pixels = mono + alpha, alpha is written separately, so same as 1-channel case + case 1: + if (expand_mono) + stbiw__write3(s, d[0], d[0], d[0]); // monochrome bmp + else + s->func(s->context, d, 1); // monochrome TGA + break; + case 4: + if (!write_alpha) { + // composite against pink background + for (k = 0; k < 3; ++k) + px[k] = bg[k] + ((d[k] - bg[k]) * d[3]) / 255; + stbiw__write3(s, px[1 - rgb_dir], px[1], px[1 + rgb_dir]); + break; + } + /* FALLTHROUGH */ + case 3: + stbiw__write3(s, d[1 - rgb_dir], d[1], d[1 + rgb_dir]); + break; + } + if (write_alpha > 0) + s->func(s->context, &d[comp - 1], 1); +} + +static void stbiw__write_pixels(stbi__write_context *s, int rgb_dir, int vdir, int x, int y, int comp, void *data, int write_alpha, int scanline_pad, int expand_mono) +{ + stbiw_uint32 zero = 0; + int i,j, j_end; + + if (y <= 0) + return; + + if (stbi__flip_vertically_on_write) + vdir *= -1; + + if (vdir < 0) + j_end = -1, j = y-1; + else + j_end = y, j = 0; + + for (; j != j_end; j += vdir) { + for (i=0; i < x; ++i) { + unsigned char *d = (unsigned char *) data + (j*x+i)*comp; + stbiw__write_pixel(s, rgb_dir, comp, write_alpha, expand_mono, d); + } + s->func(s->context, &zero, scanline_pad); + } +} + +static int stbiw__outfile(stbi__write_context *s, int rgb_dir, int vdir, int x, int y, int comp, int expand_mono, void *data, int alpha, int pad, const char *fmt, ...) +{ + if (y < 0 || x < 0) { + return 0; + } else { + va_list v; + va_start(v, fmt); + stbiw__writefv(s, fmt, v); + va_end(v); + stbiw__write_pixels(s,rgb_dir,vdir,x,y,comp,data,alpha,pad, expand_mono); + return 1; + } +} + +static int stbi_write_bmp_core(stbi__write_context *s, int x, int y, int comp, const void *data) +{ + int pad = (-x*3) & 3; + return stbiw__outfile(s,-1,-1,x,y,comp,1,(void *) data,0,pad, + "11 4 22 4" "4 44 22 444444", + 'B', 'M', 14+40+(x*3+pad)*y, 0,0, 14+40, // file header + 40, x,y, 1,24, 0,0,0,0,0,0); // bitmap header +} + +STBIWDEF int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data) +{ + stbi__write_context s; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_bmp_core(&s, x, y, comp, data); +} + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_bmp(char const *filename, int x, int y, int comp, const void *data) +{ + stbi__write_context s; + if (stbi__start_write_file(&s,filename)) { + int r = stbi_write_bmp_core(&s, x, y, comp, data); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif //!STBI_WRITE_NO_STDIO + +static int stbi_write_tga_core(stbi__write_context *s, int x, int y, int comp, void *data) +{ + int has_alpha = (comp == 2 || comp == 4); + int colorbytes = has_alpha ? comp-1 : comp; + int format = colorbytes < 2 ? 3 : 2; // 3 color channels (RGB/RGBA) = 2, 1 color channel (Y/YA) = 3 + + if (y < 0 || x < 0) + return 0; + + if (!stbi_write_tga_with_rle) { + return stbiw__outfile(s, -1, -1, x, y, comp, 0, (void *) data, has_alpha, 0, + "111 221 2222 11", 0, 0, format, 0, 0, 0, 0, 0, x, y, (colorbytes + has_alpha) * 8, has_alpha * 8); + } else { + int i,j,k; + int jend, jdir; + + stbiw__writef(s, "111 221 2222 11", 0,0,format+8, 0,0,0, 0,0,x,y, (colorbytes + has_alpha) * 8, has_alpha * 8); + + if (stbi__flip_vertically_on_write) { + j = 0; + jend = y; + jdir = 1; + } else { + j = y-1; + jend = -1; + jdir = -1; + } + for (; j != jend; j += jdir) { + unsigned char *row = (unsigned char *) data + j * x * comp; + int len; + + for (i = 0; i < x; i += len) { + unsigned char *begin = row + i * comp; + int diff = 1; + len = 1; + + if (i < x - 1) { + ++len; + diff = memcmp(begin, row + (i + 1) * comp, comp); + if (diff) { + const unsigned char *prev = begin; + for (k = i + 2; k < x && len < 128; ++k) { + if (memcmp(prev, row + k * comp, comp)) { + prev += comp; + ++len; + } else { + --len; + break; + } + } + } else { + for (k = i + 2; k < x && len < 128; ++k) { + if (!memcmp(begin, row + k * comp, comp)) { + ++len; + } else { + break; + } + } + } + } + + if (diff) { + unsigned char header = STBIW_UCHAR(len - 1); + s->func(s->context, &header, 1); + for (k = 0; k < len; ++k) { + stbiw__write_pixel(s, -1, comp, has_alpha, 0, begin + k * comp); + } + } else { + unsigned char header = STBIW_UCHAR(len - 129); + s->func(s->context, &header, 1); + stbiw__write_pixel(s, -1, comp, has_alpha, 0, begin); + } + } + } + } + return 1; +} + +STBIWDEF int stbi_write_tga_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data) +{ + stbi__write_context s; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_tga_core(&s, x, y, comp, (void *) data); +} + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_tga(char const *filename, int x, int y, int comp, const void *data) +{ + stbi__write_context s; + if (stbi__start_write_file(&s,filename)) { + int r = stbi_write_tga_core(&s, x, y, comp, (void *) data); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif + +// ************************************************************************************************* +// Radiance RGBE HDR writer +// by Baldur Karlsson + +#define stbiw__max(a, b) ((a) > (b) ? (a) : (b)) + +static void stbiw__linear_to_rgbe(unsigned char *rgbe, float *linear) +{ + int exponent; + float maxcomp = stbiw__max(linear[0], stbiw__max(linear[1], linear[2])); + + if (maxcomp < 1e-32f) { + rgbe[0] = rgbe[1] = rgbe[2] = rgbe[3] = 0; + } else { + float normalize = (float) frexp(maxcomp, &exponent) * 256.0f/maxcomp; + + rgbe[0] = (unsigned char)(linear[0] * normalize); + rgbe[1] = (unsigned char)(linear[1] * normalize); + rgbe[2] = (unsigned char)(linear[2] * normalize); + rgbe[3] = (unsigned char)(exponent + 128); + } +} + +static void stbiw__write_run_data(stbi__write_context *s, int length, unsigned char databyte) +{ + unsigned char lengthbyte = STBIW_UCHAR(length+128); + STBIW_ASSERT(length+128 <= 255); + s->func(s->context, &lengthbyte, 1); + s->func(s->context, &databyte, 1); +} + +static void stbiw__write_dump_data(stbi__write_context *s, int length, unsigned char *data) +{ + unsigned char lengthbyte = STBIW_UCHAR(length); + STBIW_ASSERT(length <= 128); // inconsistent with spec but consistent with official code + s->func(s->context, &lengthbyte, 1); + s->func(s->context, data, length); +} + +static void stbiw__write_hdr_scanline(stbi__write_context *s, int width, int ncomp, unsigned char *scratch, float *scanline) +{ + unsigned char scanlineheader[4] = { 2, 2, 0, 0 }; + unsigned char rgbe[4]; + float linear[3]; + int x; + + scanlineheader[2] = (width&0xff00)>>8; + scanlineheader[3] = (width&0x00ff); + + /* skip RLE for images too small or large */ + if (width < 8 || width >= 32768) { + for (x=0; x < width; x++) { + switch (ncomp) { + case 4: /* fallthrough */ + case 3: linear[2] = scanline[x*ncomp + 2]; + linear[1] = scanline[x*ncomp + 1]; + linear[0] = scanline[x*ncomp + 0]; + break; + default: + linear[0] = linear[1] = linear[2] = scanline[x*ncomp + 0]; + break; + } + stbiw__linear_to_rgbe(rgbe, linear); + s->func(s->context, rgbe, 4); + } + } else { + int c,r; + /* encode into scratch buffer */ + for (x=0; x < width; x++) { + switch(ncomp) { + case 4: /* fallthrough */ + case 3: linear[2] = scanline[x*ncomp + 2]; + linear[1] = scanline[x*ncomp + 1]; + linear[0] = scanline[x*ncomp + 0]; + break; + default: + linear[0] = linear[1] = linear[2] = scanline[x*ncomp + 0]; + break; + } + stbiw__linear_to_rgbe(rgbe, linear); + scratch[x + width*0] = rgbe[0]; + scratch[x + width*1] = rgbe[1]; + scratch[x + width*2] = rgbe[2]; + scratch[x + width*3] = rgbe[3]; + } + + s->func(s->context, scanlineheader, 4); + + /* RLE each component separately */ + for (c=0; c < 4; c++) { + unsigned char *comp = &scratch[width*c]; + + x = 0; + while (x < width) { + // find first run + r = x; + while (r+2 < width) { + if (comp[r] == comp[r+1] && comp[r] == comp[r+2]) + break; + ++r; + } + if (r+2 >= width) + r = width; + // dump up to first run + while (x < r) { + int len = r-x; + if (len > 128) len = 128; + stbiw__write_dump_data(s, len, &comp[x]); + x += len; + } + // if there's a run, output it + if (r+2 < width) { // same test as what we break out of in search loop, so only true if we break'd + // find next byte after run + while (r < width && comp[r] == comp[x]) + ++r; + // output run up to r + while (x < r) { + int len = r-x; + if (len > 127) len = 127; + stbiw__write_run_data(s, len, comp[x]); + x += len; + } + } + } + } + } +} + +static int stbi_write_hdr_core(stbi__write_context *s, int x, int y, int comp, float *data) +{ + if (y <= 0 || x <= 0 || data == NULL) + return 0; + else { + // Each component is stored separately. Allocate scratch space for full output scanline. + unsigned char *scratch = (unsigned char *) STBIW_MALLOC(x*4); + int i, len; + char buffer[128]; + char header[] = "#?RADIANCE\n# Written by stb_image_write.h\nFORMAT=32-bit_rle_rgbe\n"; + s->func(s->context, header, sizeof(header)-1); + +#ifdef STBI_MSC_SECURE_CRT + len = sprintf_s(buffer, "EXPOSURE= 1.0000000000000\n\n-Y %d +X %d\n", y, x); +#else + len = snprintf(buffer, 127, "EXPOSURE= 1.0000000000000\n\n-Y %d +X %d\n", y, x); +#endif + s->func(s->context, buffer, len); + + for(i=0; i < y; i++) + stbiw__write_hdr_scanline(s, x, comp, scratch, data + comp*x*(stbi__flip_vertically_on_write ? y-1-i : i)); + STBIW_FREE(scratch); + return 1; + } +} + +STBIWDEF int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const float *data) +{ + stbi__write_context s; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_hdr_core(&s, x, y, comp, (float *) data); +} + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_hdr(char const *filename, int x, int y, int comp, const float *data) +{ + stbi__write_context s; + if (stbi__start_write_file(&s,filename)) { + int r = stbi_write_hdr_core(&s, x, y, comp, (float *) data); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif // STBI_WRITE_NO_STDIO + + +////////////////////////////////////////////////////////////////////////////// +// +// PNG writer +// + +#ifndef STBIW_ZLIB_COMPRESS +// stretchy buffer; stbiw__sbpush() == vector<>::push_back() -- stbiw__sbcount() == vector<>::size() +#define stbiw__sbraw(a) ((int *) (a) - 2) +#define stbiw__sbm(a) stbiw__sbraw(a)[0] +#define stbiw__sbn(a) stbiw__sbraw(a)[1] + +#define stbiw__sbneedgrow(a,n) ((a)==0 || stbiw__sbn(a)+n >= stbiw__sbm(a)) +#define stbiw__sbmaybegrow(a,n) (stbiw__sbneedgrow(a,(n)) ? stbiw__sbgrow(a,n) : 0) +#define stbiw__sbgrow(a,n) stbiw__sbgrowf((void **) &(a), (n), sizeof(*(a))) + +#define stbiw__sbpush(a, v) (stbiw__sbmaybegrow(a,1), (a)[stbiw__sbn(a)++] = (v)) +#define stbiw__sbcount(a) ((a) ? stbiw__sbn(a) : 0) +#define stbiw__sbfree(a) ((a) ? STBIW_FREE(stbiw__sbraw(a)),0 : 0) + +static void *stbiw__sbgrowf(void **arr, int increment, int itemsize) +{ + int m = *arr ? 2*stbiw__sbm(*arr)+increment : increment+1; + void *p = STBIW_REALLOC_SIZED(*arr ? stbiw__sbraw(*arr) : 0, *arr ? (stbiw__sbm(*arr)*itemsize + sizeof(int)*2) : 0, itemsize * m + sizeof(int)*2); + STBIW_ASSERT(p); + if (p) { + if (!*arr) ((int *) p)[1] = 0; + *arr = (void *) ((int *) p + 2); + stbiw__sbm(*arr) = m; + } + return *arr; +} + +static unsigned char *stbiw__zlib_flushf(unsigned char *data, unsigned int *bitbuffer, int *bitcount) +{ + while (*bitcount >= 8) { + stbiw__sbpush(data, STBIW_UCHAR(*bitbuffer)); + *bitbuffer >>= 8; + *bitcount -= 8; + } + return data; +} + +static int stbiw__zlib_bitrev(int code, int codebits) +{ + int res=0; + while (codebits--) { + res = (res << 1) | (code & 1); + code >>= 1; + } + return res; +} + +static unsigned int stbiw__zlib_countm(unsigned char *a, unsigned char *b, int limit) +{ + int i; + for (i=0; i < limit && i < 258; ++i) + if (a[i] != b[i]) break; + return i; +} + +static unsigned int stbiw__zhash(unsigned char *data) +{ + stbiw_uint32 hash = data[0] + (data[1] << 8) + (data[2] << 16); + hash ^= hash << 3; + hash += hash >> 5; + hash ^= hash << 4; + hash += hash >> 17; + hash ^= hash << 25; + hash += hash >> 6; + return hash; +} + +#define stbiw__zlib_flush() (out = stbiw__zlib_flushf(out, &bitbuf, &bitcount)) +#define stbiw__zlib_add(code,codebits) \ + (bitbuf |= (code) << bitcount, bitcount += (codebits), stbiw__zlib_flush()) +#define stbiw__zlib_huffa(b,c) stbiw__zlib_add(stbiw__zlib_bitrev(b,c),c) +// default huffman tables +#define stbiw__zlib_huff1(n) stbiw__zlib_huffa(0x30 + (n), 8) +#define stbiw__zlib_huff2(n) stbiw__zlib_huffa(0x190 + (n)-144, 9) +#define stbiw__zlib_huff3(n) stbiw__zlib_huffa(0 + (n)-256,7) +#define stbiw__zlib_huff4(n) stbiw__zlib_huffa(0xc0 + (n)-280,8) +#define stbiw__zlib_huff(n) ((n) <= 143 ? stbiw__zlib_huff1(n) : (n) <= 255 ? stbiw__zlib_huff2(n) : (n) <= 279 ? stbiw__zlib_huff3(n) : stbiw__zlib_huff4(n)) +#define stbiw__zlib_huffb(n) ((n) <= 143 ? stbiw__zlib_huff1(n) : stbiw__zlib_huff2(n)) + +#define stbiw__ZHASH 16384 + +#endif // STBIW_ZLIB_COMPRESS + +STBIWDEF unsigned char * stbi_zlib_compress(unsigned char *data, int data_len, int *out_len, int quality) +{ +#ifdef STBIW_ZLIB_COMPRESS + // user provided a zlib compress implementation, use that + return STBIW_ZLIB_COMPRESS(data, data_len, out_len, quality); +#else // use builtin + static unsigned short lengthc[] = { 3,4,5,6,7,8,9,10,11,13,15,17,19,23,27,31,35,43,51,59,67,83,99,115,131,163,195,227,258, 259 }; + static unsigned char lengtheb[]= { 0,0,0,0,0,0,0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0 }; + static unsigned short distc[] = { 1,2,3,4,5,7,9,13,17,25,33,49,65,97,129,193,257,385,513,769,1025,1537,2049,3073,4097,6145,8193,12289,16385,24577, 32768 }; + static unsigned char disteb[] = { 0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13 }; + unsigned int bitbuf=0; + int i,j, bitcount=0; + unsigned char *out = NULL; + unsigned char ***hash_table = (unsigned char***) STBIW_MALLOC(stbiw__ZHASH * sizeof(char**)); + if (hash_table == NULL) + return NULL; + if (quality < 5) quality = 5; + + stbiw__sbpush(out, 0x78); // DEFLATE 32K window + stbiw__sbpush(out, 0x5e); // FLEVEL = 1 + stbiw__zlib_add(1,1); // BFINAL = 1 + stbiw__zlib_add(1,2); // BTYPE = 1 -- fixed huffman + + for (i=0; i < stbiw__ZHASH; ++i) + hash_table[i] = NULL; + + i=0; + while (i < data_len-3) { + // hash next 3 bytes of data to be compressed + int h = stbiw__zhash(data+i)&(stbiw__ZHASH-1), best=3; + unsigned char *bestloc = 0; + unsigned char **hlist = hash_table[h]; + int n = stbiw__sbcount(hlist); + for (j=0; j < n; ++j) { + if (hlist[j]-data > i-32768) { // if entry lies within window + int d = stbiw__zlib_countm(hlist[j], data+i, data_len-i); + if (d >= best) best=d,bestloc=hlist[j]; + } + } + // when hash table entry is too long, delete half the entries + if (hash_table[h] && stbiw__sbn(hash_table[h]) == 2*quality) { + STBIW_MEMMOVE(hash_table[h], hash_table[h]+quality, sizeof(hash_table[h][0])*quality); + stbiw__sbn(hash_table[h]) = quality; + } + stbiw__sbpush(hash_table[h],data+i); + + if (bestloc) { + // "lazy matching" - check match at *next* byte, and if it's better, do cur byte as literal + h = stbiw__zhash(data+i+1)&(stbiw__ZHASH-1); + hlist = hash_table[h]; + n = stbiw__sbcount(hlist); + for (j=0; j < n; ++j) { + if (hlist[j]-data > i-32767) { + int e = stbiw__zlib_countm(hlist[j], data+i+1, data_len-i-1); + if (e > best) { // if next match is better, bail on current match + bestloc = NULL; + break; + } + } + } + } + + if (bestloc) { + int d = (int) (data+i - bestloc); // distance back + STBIW_ASSERT(d <= 32767 && best <= 258); + for (j=0; best > lengthc[j+1]-1; ++j); + stbiw__zlib_huff(j+257); + if (lengtheb[j]) stbiw__zlib_add(best - lengthc[j], lengtheb[j]); + for (j=0; d > distc[j+1]-1; ++j); + stbiw__zlib_add(stbiw__zlib_bitrev(j,5),5); + if (disteb[j]) stbiw__zlib_add(d - distc[j], disteb[j]); + i += best; + } else { + stbiw__zlib_huffb(data[i]); + ++i; + } + } + // write out final bytes + for (;i < data_len; ++i) + stbiw__zlib_huffb(data[i]); + stbiw__zlib_huff(256); // end of block + // pad with 0 bits to byte boundary + while (bitcount) + stbiw__zlib_add(0,1); + + for (i=0; i < stbiw__ZHASH; ++i) + (void) stbiw__sbfree(hash_table[i]); + STBIW_FREE(hash_table); + + { + // compute adler32 on input + unsigned int s1=1, s2=0; + int blocklen = (int) (data_len % 5552); + j=0; + while (j < data_len) { + for (i=0; i < blocklen; ++i) s1 += data[j+i], s2 += s1; + s1 %= 65521, s2 %= 65521; + j += blocklen; + blocklen = 5552; + } + stbiw__sbpush(out, STBIW_UCHAR(s2 >> 8)); + stbiw__sbpush(out, STBIW_UCHAR(s2)); + stbiw__sbpush(out, STBIW_UCHAR(s1 >> 8)); + stbiw__sbpush(out, STBIW_UCHAR(s1)); + } + *out_len = stbiw__sbn(out); + // make returned pointer freeable + STBIW_MEMMOVE(stbiw__sbraw(out), out, *out_len); + return (unsigned char *) stbiw__sbraw(out); +#endif // STBIW_ZLIB_COMPRESS +} + +static unsigned int stbiw__crc32(unsigned char *buffer, int len) +{ +#ifdef STBIW_CRC32 + return STBIW_CRC32(buffer, len); +#else + static unsigned int crc_table[256] = + { + 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, + 0x0eDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91, + 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, + 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5, + 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, + 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, + 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F, + 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, + 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433, + 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01, + 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, + 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C, 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65, + 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, + 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, + 0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086, 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F, + 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD, + 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683, + 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, + 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7, + 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, + 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B, + 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79, + 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, + 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D, + 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713, + 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, + 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E, 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777, + 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45, + 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, + 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0, 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9, + 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, + 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D + }; + + unsigned int crc = ~0u; + int i; + for (i=0; i < len; ++i) + crc = (crc >> 8) ^ crc_table[buffer[i] ^ (crc & 0xff)]; + return ~crc; +#endif +} + +#define stbiw__wpng4(o,a,b,c,d) ((o)[0]=STBIW_UCHAR(a),(o)[1]=STBIW_UCHAR(b),(o)[2]=STBIW_UCHAR(c),(o)[3]=STBIW_UCHAR(d),(o)+=4) +#define stbiw__wp32(data,v) stbiw__wpng4(data, (v)>>24,(v)>>16,(v)>>8,(v)); +#define stbiw__wptag(data,s) stbiw__wpng4(data, s[0],s[1],s[2],s[3]) + +static void stbiw__wpcrc(unsigned char **data, int len) +{ + unsigned int crc = stbiw__crc32(*data - len - 4, len+4); + stbiw__wp32(*data, crc); +} + +static unsigned char stbiw__paeth(int a, int b, int c) +{ + int p = a + b - c, pa = abs(p-a), pb = abs(p-b), pc = abs(p-c); + if (pa <= pb && pa <= pc) return STBIW_UCHAR(a); + if (pb <= pc) return STBIW_UCHAR(b); + return STBIW_UCHAR(c); +} + +// @OPTIMIZE: provide an option that always forces left-predict or paeth predict +static void stbiw__encode_png_line(unsigned char *pixels, int stride_bytes, int width, int height, int y, int n, int filter_type, signed char *line_buffer) +{ + static int mapping[] = { 0,1,2,3,4 }; + static int firstmap[] = { 0,1,0,5,6 }; + int *mymap = (y != 0) ? mapping : firstmap; + int i; + int type = mymap[filter_type]; + unsigned char *z = pixels + stride_bytes * (stbi__flip_vertically_on_write ? height-1-y : y); + int signed_stride = stbi__flip_vertically_on_write ? -stride_bytes : stride_bytes; + + if (type==0) { + memcpy(line_buffer, z, width*n); + return; + } + + // first loop isn't optimized since it's just one pixel + for (i = 0; i < n; ++i) { + switch (type) { + case 1: line_buffer[i] = z[i]; break; + case 2: line_buffer[i] = z[i] - z[i-signed_stride]; break; + case 3: line_buffer[i] = z[i] - (z[i-signed_stride]>>1); break; + case 4: line_buffer[i] = (signed char) (z[i] - stbiw__paeth(0,z[i-signed_stride],0)); break; + case 5: line_buffer[i] = z[i]; break; + case 6: line_buffer[i] = z[i]; break; + } + } + switch (type) { + case 1: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - z[i-n]; break; + case 2: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - z[i-signed_stride]; break; + case 3: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - ((z[i-n] + z[i-signed_stride])>>1); break; + case 4: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - stbiw__paeth(z[i-n], z[i-signed_stride], z[i-signed_stride-n]); break; + case 5: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - (z[i-n]>>1); break; + case 6: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - stbiw__paeth(z[i-n], 0,0); break; + } +} + +STBIWDEF unsigned char *stbi_write_png_to_mem(const unsigned char *pixels, int stride_bytes, int x, int y, int n, int *out_len) +{ + int force_filter = stbi_write_force_png_filter; + int ctype[5] = { -1, 0, 4, 2, 6 }; + unsigned char sig[8] = { 137,80,78,71,13,10,26,10 }; + unsigned char *out,*o, *filt, *zlib; + signed char *line_buffer; + int j,zlen; + + if (stride_bytes == 0) + stride_bytes = x * n; + + if (force_filter >= 5) { + force_filter = -1; + } + + filt = (unsigned char *) STBIW_MALLOC((x*n+1) * y); if (!filt) return 0; + line_buffer = (signed char *) STBIW_MALLOC(x * n); if (!line_buffer) { STBIW_FREE(filt); return 0; } + for (j=0; j < y; ++j) { + int filter_type; + if (force_filter > -1) { + filter_type = force_filter; + stbiw__encode_png_line((unsigned char*)(pixels), stride_bytes, x, y, j, n, force_filter, line_buffer); + } else { // Estimate the best filter by running through all of them: + int best_filter = 0, best_filter_val = 0x7fffffff, est, i; + for (filter_type = 0; filter_type < 5; filter_type++) { + stbiw__encode_png_line((unsigned char*)(pixels), stride_bytes, x, y, j, n, filter_type, line_buffer); + + // Estimate the entropy of the line using this filter; the less, the better. + est = 0; + for (i = 0; i < x*n; ++i) { + est += abs((signed char) line_buffer[i]); + } + if (est < best_filter_val) { + best_filter_val = est; + best_filter = filter_type; + } + } + if (filter_type != best_filter) { // If the last iteration already got us the best filter, don't redo it + stbiw__encode_png_line((unsigned char*)(pixels), stride_bytes, x, y, j, n, best_filter, line_buffer); + filter_type = best_filter; + } + } + // when we get here, filter_type contains the filter type, and line_buffer contains the data + filt[j*(x*n+1)] = (unsigned char) filter_type; + STBIW_MEMMOVE(filt+j*(x*n+1)+1, line_buffer, x*n); + } + STBIW_FREE(line_buffer); + zlib = stbi_zlib_compress(filt, y*( x*n+1), &zlen, stbi_write_png_compression_level); + STBIW_FREE(filt); + if (!zlib) return 0; + + // each tag requires 12 bytes of overhead + out = (unsigned char *) STBIW_MALLOC(8 + 12+13 + 12+zlen + 12); + if (!out) return 0; + *out_len = 8 + 12+13 + 12+zlen + 12; + + o=out; + STBIW_MEMMOVE(o,sig,8); o+= 8; + stbiw__wp32(o, 13); // header length + stbiw__wptag(o, "IHDR"); + stbiw__wp32(o, x); + stbiw__wp32(o, y); + *o++ = 8; + *o++ = STBIW_UCHAR(ctype[n]); + *o++ = 0; + *o++ = 0; + *o++ = 0; + stbiw__wpcrc(&o,13); + + stbiw__wp32(o, zlen); + stbiw__wptag(o, "IDAT"); + STBIW_MEMMOVE(o, zlib, zlen); + o += zlen; + STBIW_FREE(zlib); + stbiw__wpcrc(&o, zlen); + + stbiw__wp32(o,0); + stbiw__wptag(o, "IEND"); + stbiw__wpcrc(&o,0); + + STBIW_ASSERT(o == out + *out_len); + + return out; +} + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_png(char const *filename, int x, int y, int comp, const void *data, int stride_bytes) +{ + FILE *f; + int len; + unsigned char *png = stbi_write_png_to_mem((const unsigned char *) data, stride_bytes, x, y, comp, &len); + if (png == NULL) return 0; + + f = stbiw__fopen(filename, "wb"); + if (!f) { STBIW_FREE(png); return 0; } + fwrite(png, 1, len, f); + fclose(f); + STBIW_FREE(png); + return 1; +} +#endif + +STBIWDEF int stbi_write_png_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int stride_bytes) +{ + int len; + unsigned char *png = stbi_write_png_to_mem((const unsigned char *) data, stride_bytes, x, y, comp, &len); + if (png == NULL) return 0; + func(context, png, len); + STBIW_FREE(png); + return 1; +} + + +/* *************************************************************************** + * + * JPEG writer + * + * This is based on Jon Olick's jo_jpeg.cpp: + * public domain Simple, Minimalistic JPEG writer - http://www.jonolick.com/code.html + */ + +static const unsigned char stbiw__jpg_ZigZag[] = { 0,1,5,6,14,15,27,28,2,4,7,13,16,26,29,42,3,8,12,17,25,30,41,43,9,11,18, + 24,31,40,44,53,10,19,23,32,39,45,52,54,20,22,33,38,46,51,55,60,21,34,37,47,50,56,59,61,35,36,48,49,57,58,62,63 }; + +static void stbiw__jpg_writeBits(stbi__write_context *s, int *bitBufP, int *bitCntP, const unsigned short *bs) { + int bitBuf = *bitBufP, bitCnt = *bitCntP; + bitCnt += bs[1]; + bitBuf |= bs[0] << (24 - bitCnt); + while(bitCnt >= 8) { + unsigned char c = (bitBuf >> 16) & 255; + stbiw__putc(s, c); + if(c == 255) { + stbiw__putc(s, 0); + } + bitBuf <<= 8; + bitCnt -= 8; + } + *bitBufP = bitBuf; + *bitCntP = bitCnt; +} + +static void stbiw__jpg_DCT(float *d0p, float *d1p, float *d2p, float *d3p, float *d4p, float *d5p, float *d6p, float *d7p) { + float d0 = *d0p, d1 = *d1p, d2 = *d2p, d3 = *d3p, d4 = *d4p, d5 = *d5p, d6 = *d6p, d7 = *d7p; + float z1, z2, z3, z4, z5, z11, z13; + + float tmp0 = d0 + d7; + float tmp7 = d0 - d7; + float tmp1 = d1 + d6; + float tmp6 = d1 - d6; + float tmp2 = d2 + d5; + float tmp5 = d2 - d5; + float tmp3 = d3 + d4; + float tmp4 = d3 - d4; + + // Even part + float tmp10 = tmp0 + tmp3; // phase 2 + float tmp13 = tmp0 - tmp3; + float tmp11 = tmp1 + tmp2; + float tmp12 = tmp1 - tmp2; + + d0 = tmp10 + tmp11; // phase 3 + d4 = tmp10 - tmp11; + + z1 = (tmp12 + tmp13) * 0.707106781f; // c4 + d2 = tmp13 + z1; // phase 5 + d6 = tmp13 - z1; + + // Odd part + tmp10 = tmp4 + tmp5; // phase 2 + tmp11 = tmp5 + tmp6; + tmp12 = tmp6 + tmp7; + + // The rotator is modified from fig 4-8 to avoid extra negations. + z5 = (tmp10 - tmp12) * 0.382683433f; // c6 + z2 = tmp10 * 0.541196100f + z5; // c2-c6 + z4 = tmp12 * 1.306562965f + z5; // c2+c6 + z3 = tmp11 * 0.707106781f; // c4 + + z11 = tmp7 + z3; // phase 5 + z13 = tmp7 - z3; + + *d5p = z13 + z2; // phase 6 + *d3p = z13 - z2; + *d1p = z11 + z4; + *d7p = z11 - z4; + + *d0p = d0; *d2p = d2; *d4p = d4; *d6p = d6; +} + +static void stbiw__jpg_calcBits(int val, unsigned short bits[2]) { + int tmp1 = val < 0 ? -val : val; + val = val < 0 ? val-1 : val; + bits[1] = 1; + while(tmp1 >>= 1) { + ++bits[1]; + } + bits[0] = val & ((1<0)&&(DU[end0pos]==0); --end0pos) { + } + // end0pos = first element in reverse order !=0 + if(end0pos == 0) { + stbiw__jpg_writeBits(s, bitBuf, bitCnt, EOB); + return DU[0]; + } + for(i = 1; i <= end0pos; ++i) { + int startpos = i; + int nrzeroes; + unsigned short bits[2]; + for (; DU[i]==0 && i<=end0pos; ++i) { + } + nrzeroes = i-startpos; + if ( nrzeroes >= 16 ) { + int lng = nrzeroes>>4; + int nrmarker; + for (nrmarker=1; nrmarker <= lng; ++nrmarker) + stbiw__jpg_writeBits(s, bitBuf, bitCnt, M16zeroes); + nrzeroes &= 15; + } + stbiw__jpg_calcBits(DU[i], bits); + stbiw__jpg_writeBits(s, bitBuf, bitCnt, HTAC[(nrzeroes<<4)+bits[1]]); + stbiw__jpg_writeBits(s, bitBuf, bitCnt, bits); + } + if(end0pos != 63) { + stbiw__jpg_writeBits(s, bitBuf, bitCnt, EOB); + } + return DU[0]; +} + +static int stbi_write_jpg_core(stbi__write_context *s, int width, int height, int comp, const void* data, int quality) { + // Constants that don't pollute global namespace + static const unsigned char std_dc_luminance_nrcodes[] = {0,0,1,5,1,1,1,1,1,1,0,0,0,0,0,0,0}; + static const unsigned char std_dc_luminance_values[] = {0,1,2,3,4,5,6,7,8,9,10,11}; + static const unsigned char std_ac_luminance_nrcodes[] = {0,0,2,1,3,3,2,4,3,5,5,4,4,0,0,1,0x7d}; + static const unsigned char std_ac_luminance_values[] = { + 0x01,0x02,0x03,0x00,0x04,0x11,0x05,0x12,0x21,0x31,0x41,0x06,0x13,0x51,0x61,0x07,0x22,0x71,0x14,0x32,0x81,0x91,0xa1,0x08, + 0x23,0x42,0xb1,0xc1,0x15,0x52,0xd1,0xf0,0x24,0x33,0x62,0x72,0x82,0x09,0x0a,0x16,0x17,0x18,0x19,0x1a,0x25,0x26,0x27,0x28, + 0x29,0x2a,0x34,0x35,0x36,0x37,0x38,0x39,0x3a,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0x4a,0x53,0x54,0x55,0x56,0x57,0x58,0x59, + 0x5a,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0x6a,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7a,0x83,0x84,0x85,0x86,0x87,0x88,0x89, + 0x8a,0x92,0x93,0x94,0x95,0x96,0x97,0x98,0x99,0x9a,0xa2,0xa3,0xa4,0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xb2,0xb3,0xb4,0xb5,0xb6, + 0xb7,0xb8,0xb9,0xba,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xd2,0xd3,0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda,0xe1,0xe2, + 0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xf1,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0xf8,0xf9,0xfa + }; + static const unsigned char std_dc_chrominance_nrcodes[] = {0,0,3,1,1,1,1,1,1,1,1,1,0,0,0,0,0}; + static const unsigned char std_dc_chrominance_values[] = {0,1,2,3,4,5,6,7,8,9,10,11}; + static const unsigned char std_ac_chrominance_nrcodes[] = {0,0,2,1,2,4,4,3,4,7,5,4,4,0,1,2,0x77}; + static const unsigned char std_ac_chrominance_values[] = { + 0x00,0x01,0x02,0x03,0x11,0x04,0x05,0x21,0x31,0x06,0x12,0x41,0x51,0x07,0x61,0x71,0x13,0x22,0x32,0x81,0x08,0x14,0x42,0x91, + 0xa1,0xb1,0xc1,0x09,0x23,0x33,0x52,0xf0,0x15,0x62,0x72,0xd1,0x0a,0x16,0x24,0x34,0xe1,0x25,0xf1,0x17,0x18,0x19,0x1a,0x26, + 0x27,0x28,0x29,0x2a,0x35,0x36,0x37,0x38,0x39,0x3a,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0x4a,0x53,0x54,0x55,0x56,0x57,0x58, + 0x59,0x5a,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0x6a,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7a,0x82,0x83,0x84,0x85,0x86,0x87, + 0x88,0x89,0x8a,0x92,0x93,0x94,0x95,0x96,0x97,0x98,0x99,0x9a,0xa2,0xa3,0xa4,0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xb2,0xb3,0xb4, + 0xb5,0xb6,0xb7,0xb8,0xb9,0xba,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xd2,0xd3,0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda, + 0xe2,0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0xf8,0xf9,0xfa + }; + // Huffman tables + static const unsigned short YDC_HT[256][2] = { {0,2},{2,3},{3,3},{4,3},{5,3},{6,3},{14,4},{30,5},{62,6},{126,7},{254,8},{510,9}}; + static const unsigned short UVDC_HT[256][2] = { {0,2},{1,2},{2,2},{6,3},{14,4},{30,5},{62,6},{126,7},{254,8},{510,9},{1022,10},{2046,11}}; + static const unsigned short YAC_HT[256][2] = { + {10,4},{0,2},{1,2},{4,3},{11,4},{26,5},{120,7},{248,8},{1014,10},{65410,16},{65411,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {12,4},{27,5},{121,7},{502,9},{2038,11},{65412,16},{65413,16},{65414,16},{65415,16},{65416,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {28,5},{249,8},{1015,10},{4084,12},{65417,16},{65418,16},{65419,16},{65420,16},{65421,16},{65422,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {58,6},{503,9},{4085,12},{65423,16},{65424,16},{65425,16},{65426,16},{65427,16},{65428,16},{65429,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {59,6},{1016,10},{65430,16},{65431,16},{65432,16},{65433,16},{65434,16},{65435,16},{65436,16},{65437,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {122,7},{2039,11},{65438,16},{65439,16},{65440,16},{65441,16},{65442,16},{65443,16},{65444,16},{65445,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {123,7},{4086,12},{65446,16},{65447,16},{65448,16},{65449,16},{65450,16},{65451,16},{65452,16},{65453,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {250,8},{4087,12},{65454,16},{65455,16},{65456,16},{65457,16},{65458,16},{65459,16},{65460,16},{65461,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {504,9},{32704,15},{65462,16},{65463,16},{65464,16},{65465,16},{65466,16},{65467,16},{65468,16},{65469,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {505,9},{65470,16},{65471,16},{65472,16},{65473,16},{65474,16},{65475,16},{65476,16},{65477,16},{65478,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {506,9},{65479,16},{65480,16},{65481,16},{65482,16},{65483,16},{65484,16},{65485,16},{65486,16},{65487,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {1017,10},{65488,16},{65489,16},{65490,16},{65491,16},{65492,16},{65493,16},{65494,16},{65495,16},{65496,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {1018,10},{65497,16},{65498,16},{65499,16},{65500,16},{65501,16},{65502,16},{65503,16},{65504,16},{65505,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {2040,11},{65506,16},{65507,16},{65508,16},{65509,16},{65510,16},{65511,16},{65512,16},{65513,16},{65514,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {65515,16},{65516,16},{65517,16},{65518,16},{65519,16},{65520,16},{65521,16},{65522,16},{65523,16},{65524,16},{0,0},{0,0},{0,0},{0,0},{0,0}, + {2041,11},{65525,16},{65526,16},{65527,16},{65528,16},{65529,16},{65530,16},{65531,16},{65532,16},{65533,16},{65534,16},{0,0},{0,0},{0,0},{0,0},{0,0} + }; + static const unsigned short UVAC_HT[256][2] = { + {0,2},{1,2},{4,3},{10,4},{24,5},{25,5},{56,6},{120,7},{500,9},{1014,10},{4084,12},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {11,4},{57,6},{246,8},{501,9},{2038,11},{4085,12},{65416,16},{65417,16},{65418,16},{65419,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {26,5},{247,8},{1015,10},{4086,12},{32706,15},{65420,16},{65421,16},{65422,16},{65423,16},{65424,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {27,5},{248,8},{1016,10},{4087,12},{65425,16},{65426,16},{65427,16},{65428,16},{65429,16},{65430,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {58,6},{502,9},{65431,16},{65432,16},{65433,16},{65434,16},{65435,16},{65436,16},{65437,16},{65438,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {59,6},{1017,10},{65439,16},{65440,16},{65441,16},{65442,16},{65443,16},{65444,16},{65445,16},{65446,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {121,7},{2039,11},{65447,16},{65448,16},{65449,16},{65450,16},{65451,16},{65452,16},{65453,16},{65454,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {122,7},{2040,11},{65455,16},{65456,16},{65457,16},{65458,16},{65459,16},{65460,16},{65461,16},{65462,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {249,8},{65463,16},{65464,16},{65465,16},{65466,16},{65467,16},{65468,16},{65469,16},{65470,16},{65471,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {503,9},{65472,16},{65473,16},{65474,16},{65475,16},{65476,16},{65477,16},{65478,16},{65479,16},{65480,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {504,9},{65481,16},{65482,16},{65483,16},{65484,16},{65485,16},{65486,16},{65487,16},{65488,16},{65489,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {505,9},{65490,16},{65491,16},{65492,16},{65493,16},{65494,16},{65495,16},{65496,16},{65497,16},{65498,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {506,9},{65499,16},{65500,16},{65501,16},{65502,16},{65503,16},{65504,16},{65505,16},{65506,16},{65507,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {2041,11},{65508,16},{65509,16},{65510,16},{65511,16},{65512,16},{65513,16},{65514,16},{65515,16},{65516,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, + {16352,14},{65517,16},{65518,16},{65519,16},{65520,16},{65521,16},{65522,16},{65523,16},{65524,16},{65525,16},{0,0},{0,0},{0,0},{0,0},{0,0}, + {1018,10},{32707,15},{65526,16},{65527,16},{65528,16},{65529,16},{65530,16},{65531,16},{65532,16},{65533,16},{65534,16},{0,0},{0,0},{0,0},{0,0},{0,0} + }; + static const int YQT[] = {16,11,10,16,24,40,51,61,12,12,14,19,26,58,60,55,14,13,16,24,40,57,69,56,14,17,22,29,51,87,80,62,18,22, + 37,56,68,109,103,77,24,35,55,64,81,104,113,92,49,64,78,87,103,121,120,101,72,92,95,98,112,100,103,99}; + static const int UVQT[] = {17,18,24,47,99,99,99,99,18,21,26,66,99,99,99,99,24,26,56,99,99,99,99,99,47,66,99,99,99,99,99,99, + 99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99}; + static const float aasf[] = { 1.0f * 2.828427125f, 1.387039845f * 2.828427125f, 1.306562965f * 2.828427125f, 1.175875602f * 2.828427125f, + 1.0f * 2.828427125f, 0.785694958f * 2.828427125f, 0.541196100f * 2.828427125f, 0.275899379f * 2.828427125f }; + + int row, col, i, k; + float fdtbl_Y[64], fdtbl_UV[64]; + unsigned char YTable[64], UVTable[64]; + + if(!data || !width || !height || comp > 4 || comp < 1) { + return 0; + } + + quality = quality ? quality : 90; + quality = quality < 1 ? 1 : quality > 100 ? 100 : quality; + quality = quality < 50 ? 5000 / quality : 200 - quality * 2; + + for(i = 0; i < 64; ++i) { + int uvti, yti = (YQT[i]*quality+50)/100; + YTable[stbiw__jpg_ZigZag[i]] = (unsigned char) (yti < 1 ? 1 : yti > 255 ? 255 : yti); + uvti = (UVQT[i]*quality+50)/100; + UVTable[stbiw__jpg_ZigZag[i]] = (unsigned char) (uvti < 1 ? 1 : uvti > 255 ? 255 : uvti); + } + + for(row = 0, k = 0; row < 8; ++row) { + for(col = 0; col < 8; ++col, ++k) { + fdtbl_Y[k] = 1 / (YTable [stbiw__jpg_ZigZag[k]] * aasf[row] * aasf[col]); + fdtbl_UV[k] = 1 / (UVTable[stbiw__jpg_ZigZag[k]] * aasf[row] * aasf[col]); + } + } + + // Write Headers + { + static const unsigned char head0[] = { 0xFF,0xD8,0xFF,0xE0,0,0x10,'J','F','I','F',0,1,1,0,0,1,0,1,0,0,0xFF,0xDB,0,0x84,0 }; + static const unsigned char head2[] = { 0xFF,0xDA,0,0xC,3,1,0,2,0x11,3,0x11,0,0x3F,0 }; + const unsigned char head1[] = { 0xFF,0xC0,0,0x11,8,(unsigned char)(height>>8),STBIW_UCHAR(height),(unsigned char)(width>>8),STBIW_UCHAR(width), + 3,1,0x11,0,2,0x11,1,3,0x11,1,0xFF,0xC4,0x01,0xA2,0 }; + s->func(s->context, (void*)head0, sizeof(head0)); + s->func(s->context, (void*)YTable, sizeof(YTable)); + stbiw__putc(s, 1); + s->func(s->context, UVTable, sizeof(UVTable)); + s->func(s->context, (void*)head1, sizeof(head1)); + s->func(s->context, (void*)(std_dc_luminance_nrcodes+1), sizeof(std_dc_luminance_nrcodes)-1); + s->func(s->context, (void*)std_dc_luminance_values, sizeof(std_dc_luminance_values)); + stbiw__putc(s, 0x10); // HTYACinfo + s->func(s->context, (void*)(std_ac_luminance_nrcodes+1), sizeof(std_ac_luminance_nrcodes)-1); + s->func(s->context, (void*)std_ac_luminance_values, sizeof(std_ac_luminance_values)); + stbiw__putc(s, 1); // HTUDCinfo + s->func(s->context, (void*)(std_dc_chrominance_nrcodes+1), sizeof(std_dc_chrominance_nrcodes)-1); + s->func(s->context, (void*)std_dc_chrominance_values, sizeof(std_dc_chrominance_values)); + stbiw__putc(s, 0x11); // HTUACinfo + s->func(s->context, (void*)(std_ac_chrominance_nrcodes+1), sizeof(std_ac_chrominance_nrcodes)-1); + s->func(s->context, (void*)std_ac_chrominance_values, sizeof(std_ac_chrominance_values)); + s->func(s->context, (void*)head2, sizeof(head2)); + } + + // Encode 8x8 macroblocks + { + static const unsigned short fillBits[] = {0x7F, 7}; + const unsigned char *imageData = (const unsigned char *)data; + int DCY=0, DCU=0, DCV=0; + int bitBuf=0, bitCnt=0; + // comp == 2 is grey+alpha (alpha is ignored) + int ofsG = comp > 2 ? 1 : 0, ofsB = comp > 2 ? 2 : 0; + int x, y, pos; + for(y = 0; y < height; y += 8) { + for(x = 0; x < width; x += 8) { + float YDU[64], UDU[64], VDU[64]; + for(row = y, pos = 0; row < y+8; ++row) { + // row >= height => use last input row + int clamped_row = (row < height) ? row : height - 1; + int base_p = (stbi__flip_vertically_on_write ? (height-1-clamped_row) : clamped_row)*width*comp; + for(col = x; col < x+8; ++col, ++pos) { + float r, g, b; + // if col >= width => use pixel from last input column + int p = base_p + ((col < width) ? col : (width-1))*comp; + + r = imageData[p+0]; + g = imageData[p+ofsG]; + b = imageData[p+ofsB]; + YDU[pos]=+0.29900f*r+0.58700f*g+0.11400f*b-128; + UDU[pos]=-0.16874f*r-0.33126f*g+0.50000f*b; + VDU[pos]=+0.50000f*r-0.41869f*g-0.08131f*b; + } + } + + DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, YDU, fdtbl_Y, DCY, YDC_HT, YAC_HT); + DCU = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, UDU, fdtbl_UV, DCU, UVDC_HT, UVAC_HT); + DCV = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, VDU, fdtbl_UV, DCV, UVDC_HT, UVAC_HT); + } + } + + // Do the bit alignment of the EOI marker + stbiw__jpg_writeBits(s, &bitBuf, &bitCnt, fillBits); + } + + // EOI + stbiw__putc(s, 0xFF); + stbiw__putc(s, 0xD9); + + return 1; +} + +STBIWDEF int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int quality) +{ + stbi__write_context s; + stbi__start_write_callbacks(&s, func, context); + return stbi_write_jpg_core(&s, x, y, comp, (void *) data, quality); +} + + +#ifndef STBI_WRITE_NO_STDIO +STBIWDEF int stbi_write_jpg(char const *filename, int x, int y, int comp, const void *data, int quality) +{ + stbi__write_context s; + if (stbi__start_write_file(&s,filename)) { + int r = stbi_write_jpg_core(&s, x, y, comp, data, quality); + stbi__end_write_file(&s); + return r; + } else + return 0; +} +#endif + +#endif // STB_IMAGE_WRITE_IMPLEMENTATION + +/* Revision history + 1.10 (2019-02-07) + support utf8 filenames in Windows; fix warnings and platform ifdefs + 1.09 (2018-02-11) + fix typo in zlib quality API, improve STB_I_W_STATIC in C++ + 1.08 (2018-01-29) + add stbi__flip_vertically_on_write, external zlib, zlib quality, choose PNG filter + 1.07 (2017-07-24) + doc fix + 1.06 (2017-07-23) + writing JPEG (using Jon Olick's code) + 1.05 ??? + 1.04 (2017-03-03) + monochrome BMP expansion + 1.03 ??? + 1.02 (2016-04-02) + avoid allocating large structures on the stack + 1.01 (2016-01-16) + STBIW_REALLOC_SIZED: support allocators with no realloc support + avoid race-condition in crc initialization + minor compile issues + 1.00 (2015-09-14) + installable file IO function + 0.99 (2015-09-13) + warning fixes; TGA rle support + 0.98 (2015-04-08) + added STBIW_MALLOC, STBIW_ASSERT etc + 0.97 (2015-01-18) + fixed HDR asserts, rewrote HDR rle logic + 0.96 (2015-01-17) + add HDR output + fix monochrome BMP + 0.95 (2014-08-17) + add monochrome TGA output + 0.94 (2014-05-31) + rename private functions to avoid conflicts with stb_image.h + 0.93 (2014-05-27) + warning fixes + 0.92 (2010-08-01) + casts to unsigned char to fix warnings + 0.91 (2010-07-17) + first public release + 0.90 first internal release +*/ + +/* +------------------------------------------------------------------------------ +This software is available under 2 licenses -- choose whichever you prefer. +------------------------------------------------------------------------------ +ALTERNATIVE A - MIT License +Copyright (c) 2017 Sean Barrett +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +------------------------------------------------------------------------------ +ALTERNATIVE B - Public Domain (www.unlicense.org) +This is free and unencumbered software released into the public domain. +Anyone is free to copy, modify, publish, use, compile, sell, or distribute this +software, either in source code form or as a compiled binary, for any purpose, +commercial or non-commercial, and by any means. +In jurisdictions that recognize copyright laws, the author or authors of this +software dedicate any and all copyright interest in the software to the public +domain. We make this dedication for the benefit of the public at large and to +the detriment of our heirs and successors. We intend this dedication to be an +overt act of relinquishment in perpetuity of all present and future rights to +this software under copyright law. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +------------------------------------------------------------------------------ +*/ diff --git a/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api.cpp b/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api.cpp new file mode 100644 index 0000000000..b966279eb5 --- /dev/null +++ b/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api.cpp @@ -0,0 +1,1356 @@ +#include "torch_api.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define STB_IMAGE_IMPLEMENTATION +#include "stb_image.h" + +#define STB_IMAGE_WRITE_IMPLEMENTATION +#include "stb_image_write.h" + +#define STB_IMAGE_RESIZE_IMPLEMENTATION +#include "stb_image_resize.h" + +thread_local char *torch_last_err = nullptr; + +char *get_and_reset_last_err() { + char *tmp = torch_last_err; + torch_last_err = nullptr; + return tmp; +} + +void at_manual_seed(int64_t seed) { torch::manual_seed(seed); } + +vector of_carray_tensor(torch::Tensor **vs, int len) { + vector result; + for (int i = 0; i < len; ++i) + result.push_back(*(vs[i])); + return result; +} + +c10::List> of_carray_tensor_opt(torch::Tensor **vs, + int len) { + vector> result; + for (int i = 0; i < len; ++i) { + result.push_back(vs[i] != nullptr ? c10::optional(*(vs[i])) + : c10::nullopt); + } + return c10::List>(result); +} + +at::Device device_of_int(int d) { + if (d == -3) + return at::Device(at::kVulkan); + if (d == -2) + return at::Device(at::kMPS); + if (d < 0) + return at::Device(at::kCPU); + return at::Device(at::kCUDA, /*index=*/d); +} +tensor at_new_tensor() { + PROTECT(return new torch::Tensor();) + return nullptr; +} + +tensor at_tensor_of_blob(void *data, int64_t *dims, size_t ndims, + int64_t *strides, size_t nstrides, int type, + int device) { + PROTECT( + at::TensorOptions blobOptions = at::TensorOptions() + .device(device_of_int(device)) + .dtype(torch::ScalarType(type)); + if (nstrides == 0) { + return new torch::Tensor( + torch::for_blob(data, torch::IntArrayRef(dims, ndims)) + .options(blobOptions) + .make_tensor()); + } else { + return new torch::Tensor(torch::from_blob( + data, torch::IntArrayRef(dims, ndims), + torch::IntArrayRef(strides, nstrides), blobOptions)); + }) + + return nullptr; +} + +tensor at_tensor_of_data(void *vs, int64_t *dims, size_t ndims, + size_t element_size_in_bytes, int type) { + PROTECT( + torch::Tensor tensor = torch::zeros(torch::IntArrayRef(dims, ndims), + torch::ScalarType(type)); + if ((int64_t)element_size_in_bytes != tensor.element_size()) throw std:: + invalid_argument("incoherent element sizes in bytes"); + void *tensor_data = tensor.data_ptr(); + memcpy(tensor_data, vs, tensor.numel() * element_size_in_bytes); + return new torch::Tensor(tensor);) + return nullptr; +} + +void at_copy_data(tensor tensor, void *vs, size_t numel, + size_t elt_size_in_bytes) { + PROTECT( + if ((int64_t)elt_size_in_bytes != tensor->element_size()) throw std:: + invalid_argument("incoherent element sizes in bytes"); + if ((int64_t)numel > tensor->numel()) throw std::invalid_argument( + "target numel is larger than tensor numel"); + if (tensor->device().type() != at::kCPU) { + torch::Tensor tmp_tensor = tensor->to(at::kCPU).contiguous(); + void *tensor_data = tmp_tensor.data_ptr(); + memcpy(vs, tensor_data, numel * elt_size_in_bytes); + } else { + auto tmp_tensor = tensor->contiguous(); + void *tensor_data = tmp_tensor.data_ptr(); + memcpy(vs, tensor_data, numel * elt_size_in_bytes); + }) +} + +tensor at_shallow_clone(tensor t) { + PROTECT(return new torch::Tensor(*t);) + return nullptr; +} + +void *at_data_ptr(tensor t) { + PROTECT(return t->data_ptr();) + return nullptr; +} + +int at_defined(tensor t) { + PROTECT(return t->defined();) + return -1; +} + +int at_is_mkldnn(tensor t) { + PROTECT(return t->is_mkldnn();) + return -1; +} + +int at_is_sparse(tensor t) { + PROTECT(return t->is_sparse();) + return -1; +} + +int at_is_contiguous(tensor t) { + PROTECT(return t->is_contiguous();) + return -1; +} + +size_t at_dim(tensor t) { + PROTECT(return t->dim();) + return -1; +} + +void at_shape(tensor t, int64_t *dims) { + PROTECT(int i = 0; for (int64_t dim : t->sizes()) dims[i++] = dim;) +} + +void at_stride(tensor t, int64_t *dims) { + PROTECT(int i = 0; for (int64_t dim : t->strides()) dims[i++] = dim;) +} + +int at_scalar_type(tensor t) { + PROTECT(return static_cast(t->scalar_type());) + return -1; +} + +void at__amp_non_finite_check_and_unscale(tensor t, tensor found_inf, + tensor inf_scale) { + PROTECT(at::_amp_foreach_non_finite_check_and_unscale_(*t, *found_inf, + *inf_scale);) +} + +void at_autocast_clear_cache() { at::autocast::clear_cache(); } + +int at_autocast_decrement_nesting() { + PROTECT(return at::autocast::decrement_nesting();) + return -1; +} + +int at_autocast_increment_nesting() { + PROTECT(return at::autocast::increment_nesting();) + return -1; +} + +bool at_autocast_is_enabled() { + PROTECT(return at::autocast::is_enabled();) + return -1; +} + +bool at_autocast_set_enabled(bool b) { + PROTECT(bool is_enabled = at::autocast::is_enabled(); + at::autocast::set_enabled(b); return is_enabled;) + return -1; +} + +int at_device(tensor t) { + PROTECT(auto device = t->device(); if (device.type() == at::kCPU) return -1; + if (device.type() == at::kMPS) return -2; + if (device.type() == at::kVulkan) return -3; + if (device.type() == at::kCUDA) return device.index();) + return -2; +} + +void at_backward(tensor t, int keep_graph, int create_graph) { + PROTECT(t->backward({}, keep_graph, create_graph);) +} + +int at_requires_grad(tensor t) { + PROTECT(return t->requires_grad();) + return -1; +} + +int at_grad_set_enabled(int b) { + PROTECT(bool is_enabled = torch::autograd::GradMode::is_enabled(); + torch::autograd::GradMode::set_enabled(b); return is_enabled;) + return -1; +} + +tensor at_get(tensor t, int index) { + PROTECT(return new torch::Tensor((*t)[index]);) + return nullptr; +} + +template +T at_value_at_indexes(tensor t, int64_t *indexes, int indexes_len) { + PROTECT(torch::Tensor tensor = *t; for (int i = 0; i < indexes_len; ++i) { + tensor = tensor[indexes[i]]; + } return tensor.item();) + return T(); +} + +double at_double_value_at_indexes(tensor t, int64_t *indexes, int indexes_len) { + return at_value_at_indexes(t, indexes, indexes_len); +} + +int64_t at_int64_value_at_indexes(tensor t, int64_t *indexes, int indexes_len) { + return at_value_at_indexes(t, indexes, indexes_len); +} + +template +void at_set_value_at_indexes(tensor t, int *indexes, int indexes_len, T v) { + PROTECT(torch::Tensor tensor = *t; for (int i = 0; i < indexes_len; ++i) { + tensor = tensor[indexes[i]]; + } tensor.fill_(v);) +} + +void at_set_double_value_at_indexes(tensor t, int *indexes, int indexes_len, + double v) { + at_set_value_at_indexes(t, indexes, indexes_len, v); +} + +void at_set_int64_value_at_indexes(tensor t, int *indexes, int indexes_len, + int64_t v) { + at_set_value_at_indexes(t, indexes, indexes_len, v); +} + +void at_fill_double(tensor t, double v) { PROTECT(t->fill_(v);) } + +void at_fill_int64(tensor t, int64_t v) { PROTECT(t->fill_(v);) } + +void at_print(tensor t) { + PROTECT(torch::Tensor *tensor = (torch::Tensor *)t; cout << *tensor << endl;) +} + +char *at_to_string(tensor t, int line_size) { + PROTECT(std::ostringstream oss; torch::print(oss, *t, line_size); + return strdup(oss.str().c_str());) + return nullptr; +} + +void at_copy_(tensor dst, tensor src) { PROTECT(dst->copy_(*src);) } + +namespace { +class WriteStreamAdapter { +private: + void *_stream_ptr; + +public: + WriteStreamAdapter(void *stream_ptr) : _stream_ptr(stream_ptr) {} + + ~WriteStreamAdapter() { tch_write_stream_destructor(_stream_ptr); } + + size_t save(const void *data, size_t size) { + size_t out_size = 0; + if (!tch_write_stream_write(_stream_ptr, static_cast(data), + size, &out_size)) { + throw std::ios_base::failure("tch_write_stream_write has returned error"); + } + return out_size; + } +}; + +class ReadStreamAdapter : public caffe2::serialize::ReadAdapterInterface { +private: + void *_stream_ptr; + +public: + ReadStreamAdapter(void *stream_ptr) : _stream_ptr(stream_ptr) {} + + virtual ~ReadStreamAdapter() { tch_read_stream_destructor(_stream_ptr); } + + virtual size_t size() const { + uint64_t current = 0; + if (!tch_read_stream_stream_position(_stream_ptr, ¤t)) { + throw std::ios_base::failure( + "tch_read_stream_stream_position has returned error"); + } + + uint64_t size = 0; + if (!tch_read_stream_seek_end(_stream_ptr, 0, &size)) { + throw std::ios_base::failure( + "tch_read_stream_seek_end has returned error"); + } + + uint64_t dummy = 0; + if (!tch_read_stream_seek_start(_stream_ptr, current, &dummy)) { + throw std::ios_base::failure( + "tch_read_stream_seek_start has returned error"); + } + + return size; + } + + virtual size_t read(uint64_t pos, void *buf, size_t n, + const char *what) const { + uint64_t dummy = 0; + if (!tch_read_stream_seek_start(_stream_ptr, pos, &dummy)) { + throw std::ios_base::failure( + "tch_read_stream_seek_start has returned error"); + } + + size_t size = 0; + if (!tch_read_stream_read(_stream_ptr, static_cast(buf), n, + &size)) { + throw std::ios_base::failure("tch_read_stream_read has returned error"); + } + + return size; + } +}; +} // namespace + +void at_save(tensor t, char *filename) { PROTECT(torch::save(*t, filename);) } + +void at_save_to_stream(tensor t, void *stream_ptr) { + PROTECT(auto adapter = std::shared_ptr( + new WriteStreamAdapter(stream_ptr)); + torch::save(*t, [adapter](const void *data, size_t size) { + return adapter->save(data, size); + });) +} + +void at_save_multi(tensor *tensors, char **tensor_names, int ntensors, + char *filename) { + PROTECT(torch::serialize::OutputArchive archive; + for (int i = 0; i < ntensors; ++i) archive.write( + std::string(tensor_names[i]), *(tensors[i]), /* buffer=*/false); + archive.save_to(filename);) +} + +void at_save_multi_to_stream(tensor *tensors, char **tensor_names, int ntensors, + void *stream_ptr) { + PROTECT(auto adapter = std::shared_ptr( + new WriteStreamAdapter(stream_ptr)); + torch::serialize::OutputArchive archive; + for (int i = 0; i < ntensors; ++i) archive.write( + std::string(tensor_names[i]), *(tensors[i]), /* buffer=*/false); + archive.save_to([adapter](const void *data, size_t size) { + return adapter->save(data, size); + });) +} + +void at_load_multi(tensor *tensors, char **tensor_names, int ntensors, + char *filename) { + PROTECT(torch::serialize::InputArchive archive; + archive.load_from(std::string(filename)); + vector ts(ntensors); + for (int i = 0; i < ntensors; ++i) + archive.read(std::string(tensor_names[i]), ts[i]); + // Only allocate the new tensor now so that if there is an exception + // raised during [read], no memory has to be freed. + for (int i = 0; i < ntensors; ++i) tensors[i] = + new torch::Tensor(ts[i]);) +} + +void at_loadz_callback(char *filename, void *data, + void (*f)(void *, char *, tensor)) { + PROTECT(auto params = torch::jit::_load_parameters(filename); + for (const auto &p : params) { + f(data, (char *)p.first.c_str(), new torch::Tensor(p.second)); + }) +} + +void at_loadz_callback_with_device(char *filename, void *data, + void (*f)(void *, char *, tensor), + int device_id) { + PROTECT(auto params = + torch::jit::_load_parameters(filename, device_of_int(device_id)); + for (const auto &p : params) { + f(data, (char *)p.first.c_str(), new torch::Tensor(p.second)); + }) +} + +void at_load_callback(char *filename, void *data, + void (*f)(void *, char *, tensor)) { + PROTECT(auto module = torch::jit::load(filename); + for (const auto &p : module.named_parameters()) { + auto v = p.value; + f(data, (char *)p.name.c_str(), new torch::Tensor(v)); + }) +} + +void at_load_callback_with_device(char *filename, void *data, + void (*f)(void *, char *, tensor), + int device_id) { + PROTECT(auto module = torch::jit::load(filename, device_of_int(device_id)); + for (const auto &p : module.named_parameters()) { + auto v = p.value; + f(data, (char *)p.name.c_str(), new torch::Tensor(v)); + }) +} + +void at_load_from_stream_callback(void *stream_ptr, void *data, + void (*f)(void *, char *, tensor), + bool enable_device_id, int device_id) { + PROTECT( + auto adapter = std::shared_ptr( + new ReadStreamAdapter(stream_ptr)); + auto module = enable_device_id + ? torch::jit::load(adapter, device_of_int(device_id)) + : torch::jit::load(adapter); + for (const auto &p : module.named_parameters()) { + auto v = p.value; + f(data, (char *)p.name.c_str(), new torch::Tensor(v)); + }) +} + +void at_load_multi_(tensor *tensors, char **tensor_names, int ntensors, + char *filename) { + PROTECT(torch::NoGradGuard no_grad; torch::serialize::InputArchive archive; + archive.load_from(std::string(filename)); + for (int i = 0; i < ntensors; ++i) { + if (tensors[i]->device().type() == at::kCPU) + archive.read(std::string(tensor_names[i]), *(tensors[i])); + else { + torch::Tensor tmp_tensor = + torch::empty_like(*(tensors[i]), at::device(at::kCPU)); + archive.read(std::string(tensor_names[i]), tmp_tensor); + tensors[i]->copy_(tmp_tensor); + } + }) +} + +tensor at_load(char *filename) { + PROTECT(torch::Tensor tensor; torch::load(tensor, filename); + return new torch::Tensor(tensor);) + return nullptr; +} + +tensor at_load_from_stream(void *stream_ptr) { + PROTECT(torch::NoGradGuard no_grad; torch::Tensor tensor; + auto adapter = + std::shared_ptr( + new ReadStreamAdapter(stream_ptr)); + torch::load( + tensor, + [adapter](uint64_t pos, void *buf, size_t nbytes) { + return adapter->read(pos, buf, nbytes, "tensor"); + }, + [adapter]() { return adapter->size(); }); + return new torch::Tensor(tensor);) + return nullptr; +} + +tensor at_load_image(char *filename) { + PROTECT( + int w = -1; int h = -1; int c = -1; + void *data = stbi_load(filename, &w, &h, &c, 3); + if (data == nullptr) throw std::invalid_argument(stbi_failure_reason()); + torch::Tensor tensor = torch::zeros({h, w, 3}, at::ScalarType::Byte); + memcpy(tensor.data_ptr(), data, h * w * 3); free(data); + return new torch::Tensor(tensor);) + return nullptr; +} + +tensor at_load_image_from_memory(unsigned char *img_data, size_t img_size) { + PROTECT( + int w = -1; int h = -1; int c = -1; + void *data = stbi_load_from_memory(img_data, img_size, &w, &h, &c, 3); + if (data == nullptr) throw std::invalid_argument(stbi_failure_reason()); + torch::Tensor tensor = torch::zeros({h, w, 3}, at::ScalarType::Byte); + memcpy(tensor.data_ptr(), data, h * w * 3); free(data); + return new torch::Tensor(tensor);) + return nullptr; +} + +bool ends_with(const char *str, const char *suffix) { + int suffix_len = strlen(suffix); + int str_len = strlen(str); + if (str_len < suffix_len) + return false; + for (int i = 1; i <= suffix_len; ++i) + if (str[str_len - i] != suffix[suffix_len - i]) + return false; + return true; +} + +int at_save_image(tensor tensor, char *filename) { + PROTECT(auto sizes = tensor->sizes(); + if (tensor->device().type() != at::kCPU) throw std::invalid_argument( + "the input tensor has to be on cpu"); + if (sizes.size() != 3) throw std::invalid_argument( + "invalid number of dimensions, should be 3"); + int h = sizes[0]; int w = sizes[1]; int c = sizes[2]; + auto tmp_tensor = tensor->contiguous(); + void *tensor_data = tmp_tensor.data_ptr(); + if (ends_with(filename, ".jpg")) return stbi_write_jpg( + filename, w, h, c, tensor_data, 90); + if (ends_with(filename, ".bmp")) return stbi_write_bmp( + filename, w, h, c, tensor_data); + if (ends_with(filename, ".tga")) return stbi_write_tga( + filename, w, h, c, tensor_data); + return stbi_write_png(filename, w, h, c, tensor_data, 0);) + return -1; +} + +int at_get_num_interop_threads() { + PROTECT(return at::get_num_interop_threads();) + return -1; +} + +int at_get_num_threads() { + PROTECT(return at::get_num_threads();) + return -1; +} + +void at_set_num_interop_threads(int n_threads) { + PROTECT(at::set_num_interop_threads(n_threads);) +} + +void at_set_num_threads(int n_threads) { + PROTECT(at::set_num_threads(n_threads);) +} + +void at_set_qengine(int qengine_id) { + PROTECT( + at::QEngine qengine = at::QEngine::NoQEngine; switch (qengine_id) { + case 0: + break; + case 1: + qengine = at::QEngine::FBGEMM; + break; + case 2: + qengine = at::QEngine::QNNPACK; + break; + } auto qengines = at::globalContext().supportedQEngines(); + if (std::find(qengines.begin(), qengines.end(), qengine) != + qengines.end()) { + at::globalContext().setQEngine(qengine); + } else throw std::invalid_argument("unsupported qengine");) +} + +tensor at_resize_image(tensor tensor, int out_w, int out_h) { + PROTECT( + auto sizes = tensor->sizes(); + if (tensor->device().type() != at::kCPU) throw std::invalid_argument( + "the input tensor has to be on cpu"); + if (sizes.size() != 3) throw std::invalid_argument( + "invalid number of dimensions, should be 3"); + int h = sizes[0]; int w = sizes[1]; int c = sizes[2]; + auto tmp_tensor = tensor->contiguous(); + const unsigned char *tensor_data = (unsigned char *)tmp_tensor.data_ptr(); + torch::Tensor out = torch::zeros({out_h, out_w, c}, at::ScalarType::Byte); + stbir_resize_uint8(tensor_data, w, h, 0, (unsigned char *)out.data_ptr(), + out_w, out_h, 0, c); + return new torch::Tensor(out);) + return nullptr; +} + +void at_free(tensor t) { delete (t); } + +void at_run_backward(tensor *tensors, int ntensors, tensor *inputs, int ninputs, + tensor *outputs, int keep_graph, int create_graph) { + PROTECT( + vector roots; + for (int i = 0; i < ntensors; ++i) + roots.push_back(torch::autograd::impl::gradient_edge(*tensors[i])); + + vector inputs_; for (int i = 0; i < ninputs; ++i) { + if (!inputs[i]->requires_grad()) + throw std::invalid_argument( + "one of the input tensor does not use set_requires_grad"); + inputs_.push_back(torch::autograd::impl::gradient_edge(*inputs[i])); + } + + vector + grads; + for (int i = 0; i < ntensors; ++i) + grads.push_back(torch::ones_like(*tensors[i])); + + auto vl = torch::autograd::Engine::get_default_engine().execute( + roots, grads, keep_graph, create_graph, false, inputs_); + for (int i = 0; i < ninputs; ++i) { + outputs[i] = static_cast(new torch::autograd::Variable(vl[i])); + }) +} + +optimizer ato_adam(double learning_rate, double beta1, double beta2, + double weight_decay, double eps, bool amsgrad) { + PROTECT(auto options = torch::optim::AdamOptions(learning_rate) + .betas(std::tuple(beta1, beta2)) + .weight_decay(weight_decay) + .eps(eps) + .amsgrad(amsgrad); + return new torch::optim::Adam(vector(), options);) + return nullptr; +} + +optimizer ato_adamw(double learning_rate, double beta1, double beta2, + double weight_decay, double eps, bool amsgrad) { + PROTECT(auto options = torch::optim::AdamWOptions(learning_rate) + .betas(std::tuple(beta1, beta2)) + .weight_decay(weight_decay) + .eps(eps) + .amsgrad(amsgrad); + return new torch::optim::AdamW(vector(), options);) + return nullptr; +} + +optimizer ato_rms_prop(double learning_rate, double alpha, double eps, + double weight_decay, double momentum, int centered) { + PROTECT(auto options = torch::optim::RMSpropOptions(learning_rate) + .alpha(alpha) + .eps(eps) + .weight_decay(weight_decay) + .momentum(momentum) + .centered(centered != 0); + return new torch::optim::RMSprop(vector(), options);) + return nullptr; +} + +optimizer ato_sgd(double learning_rate, double momentum, double dampening, + double weight_decay, int nesterov) { + PROTECT(auto options = torch::optim::SGDOptions(learning_rate) + .momentum(momentum) + .dampening(dampening) + .weight_decay(weight_decay) + .nesterov(nesterov); + return new torch::optim::SGD(vector(), options);) + return nullptr; +} + +void ato_add_parameters(optimizer t, tensor tensor, size_t group) { + PROTECT(auto &groups = t->param_groups(); while (groups.size() <= group) { + groups.push_back( + torch::optim::OptimizerParamGroup({}, t->defaults().clone())); + } groups[group] + .params() + .push_back(*tensor);) +} + +template void set_lr(optimizer t, double learning_rate) { + torch::optim::OptimizerOptions *d = &(t->defaults()); + if (auto p = dynamic_cast(d)) { + p->lr(learning_rate); + for (auto ¶m_group : t->param_groups()) { + torch::optim::OptimizerOptions *d = &(param_group.options()); + if (auto p2 = dynamic_cast(d)) { + p2->lr(learning_rate); + } else + throw std::invalid_argument("unexpected param group type"); + } + } +} + +void ato_set_learning_rate(optimizer t, double learning_rate) { + PROTECT(set_lr(t, learning_rate); + set_lr(t, learning_rate); + set_lr(t, learning_rate); + set_lr(t, learning_rate);) +} + +template +void set_lr_group(optimizer t, size_t group, double learning_rate) { + auto ¶m_group = t->param_groups().at(group); + torch::optim::OptimizerOptions *d = &(param_group.options()); + if (auto p = dynamic_cast(d)) { + p->lr(learning_rate); + } +} + +void ato_set_learning_rate_group(optimizer t, size_t group, + double learning_rate) { + PROTECT(set_lr_group(t, group, learning_rate); + set_lr_group(t, group, learning_rate); + set_lr_group(t, group, learning_rate); + set_lr_group(t, group, learning_rate);) +} + +void ato_set_momentum(optimizer t, double momentum) { + PROTECT( + torch::optim::OptimizerOptions *d = &(t->defaults()); + if (auto adam = dynamic_cast(d)) { + auto betas = adam->betas(); + adam->betas(std::tuple(momentum, get<1>(betas))); + for (auto ¶m_group : t->param_groups()) { + torch::optim::OptimizerOptions *d = &(param_group.options()); + if (auto adam2 = dynamic_cast(d)) { + adam2->betas(std::tuple(momentum, get<1>(betas))); + } else + throw std::invalid_argument("unexpected param group type"); + } + } else if (auto adamw = dynamic_cast(d)) { + auto betas = adamw->betas(); + adamw->betas(std::tuple(momentum, get<1>(betas))); + for (auto ¶m_group : t->param_groups()) { + torch::optim::OptimizerOptions *d = &(param_group.options()); + if (auto adamw2 = dynamic_cast(d)) { + adamw2->betas(std::tuple(momentum, get<1>(betas))); + } else + throw std::invalid_argument("unexpected param group type"); + } + } else if (auto rms = dynamic_cast(d)) { + rms->momentum(momentum); + for (auto ¶m_group : t->param_groups()) { + torch::optim::OptimizerOptions *d = &(param_group.options()); + if (auto rms2 = dynamic_cast(d)) { + rms2->momentum(momentum); + } else + throw std::invalid_argument("unexpected param group type"); + } + } else if (auto sgd = dynamic_cast(d)) { + sgd->momentum(momentum); + for (auto ¶m_group : t->param_groups()) { + torch::optim::OptimizerOptions *d = &(param_group.options()); + if (auto sgd2 = dynamic_cast(d)) { + sgd2->momentum(momentum); + } else + throw std::invalid_argument("unexpected param group type"); + } + } else throw std::invalid_argument("unexpected optimizer");) +} + +void ato_set_momentum_group(optimizer t, size_t group, double momentum) { + PROTECT( + auto ¶m_group = t->param_groups().at(group); + torch::optim::OptimizerOptions *d = &(param_group.options()); + + if (auto adam = dynamic_cast(d)) { + auto betas = adam->betas(); + adam->betas(std::tuple(momentum, get<1>(betas))); + } else if (auto adamw = dynamic_cast(d)) { + auto betas = adamw->betas(); + adamw->betas(std::tuple(momentum, get<1>(betas))); + } else if (auto rms = dynamic_cast(d)) { + rms->momentum(momentum); + } if (auto sgd = dynamic_cast(d)) { + sgd->momentum(momentum); + } else throw std::invalid_argument("unexpected optimizer");) +} + +template void set_weight_decay(optimizer t, double weight_decay) { + torch::optim::OptimizerOptions *d = &(t->defaults()); + if (auto p = dynamic_cast(d)) { + p->weight_decay(weight_decay); + for (auto ¶m_group : t->param_groups()) { + torch::optim::OptimizerOptions *d = &(param_group.options()); + if (auto p2 = dynamic_cast(d)) { + p2->weight_decay(weight_decay); + } else + throw std::invalid_argument("unexpected param group type"); + } + } +} + +void ato_set_weight_decay(optimizer t, double weight_decay) { + PROTECT(set_weight_decay(t, weight_decay); + set_weight_decay(t, weight_decay); + set_weight_decay(t, weight_decay); + set_weight_decay(t, weight_decay);) +} + +template +void set_weight_decay_group(optimizer t, size_t group, double weight_decay) { + auto ¶m_group = t->param_groups().at(group); + torch::optim::OptimizerOptions *d = &(param_group.options()); + if (auto p = dynamic_cast(d)) { + p->weight_decay(weight_decay); + } +} + +void ato_set_weight_decay_group(optimizer t, size_t group, + double weight_decay) { + PROTECT( + set_weight_decay_group(t, group, weight_decay); + set_weight_decay_group(t, group, + weight_decay); + set_weight_decay_group(t, group, + weight_decay); + set_weight_decay_group(t, group, weight_decay);) +} + +void ato_zero_grad(optimizer t) { PROTECT(t->zero_grad();) } + +void ato_step(optimizer t) { PROTECT(t->step();) } + +void ato_free(optimizer t) { delete (t); } + +scalar ats_int(int64_t v) { + PROTECT(return new torch::Scalar(v);) + return nullptr; +} + +scalar ats_float(double v) { + PROTECT(return new torch::Scalar(v);) + return nullptr; +} + +int64_t ats_to_int(scalar s) { + PROTECT(return s->toLong();) + return -1; +} + +double ats_to_float(scalar s) { + PROTECT(return s->toDouble();) + return 0.; +} + +char *ats_to_string(scalar s) { + PROTECT(using namespace at; std::ostringstream oss; oss << (*s); + return strdup(oss.str().c_str());) + return nullptr; +} + +void ats_free(scalar s) { delete (s); } + +int atc_cuda_device_count() { + PROTECT(return torch::cuda::device_count();) + return -1; +} + +int atc_cuda_is_available() { + PROTECT(return torch::cuda::is_available();) + return -1; +} + +int atc_cudnn_is_available() { + PROTECT(return torch::cuda::cudnn_is_available();) + return -1; +} + +void atc_manual_seed(uint64_t seed) { + PROTECT(return torch::cuda::manual_seed(seed);) +} + +void atc_manual_seed_all(uint64_t seed) { + PROTECT(return torch::cuda::manual_seed_all(seed);) +} + +void atc_synchronize(int64_t device_index) { + PROTECT(return torch::cuda::synchronize(device_index);) +} + +int atc_user_enabled_cudnn() { + PROTECT(return at::globalContext().userEnabledCuDNN();) + return -1; +} + +void atc_set_user_enabled_cudnn(int b) { + PROTECT(at::globalContext().setUserEnabledCuDNN(b);) +} + +void atc_set_benchmark_cudnn(int b) { + PROTECT(at::globalContext().setBenchmarkCuDNN(b);) +} + +bool at_context_has_openmp() { + PROTECT(return at::globalContext().hasOpenMP();) + return 0; +} + +bool at_context_has_mkl() { + PROTECT(return at::globalContext().hasMKL();) + return 0; +} + +bool at_context_has_lapack() { + PROTECT(return at::globalContext().hasLAPACK();) + return 0; +} + +bool at_context_has_mkldnn() { + PROTECT(return at::globalContext().hasMKLDNN();) + return 0; +} + +bool at_context_has_magma() { + PROTECT(return at::globalContext().hasMAGMA();) + return 0; +} + +bool at_context_has_cuda() { + PROTECT(return at::globalContext().hasCUDA();) + return 0; +} + +bool at_context_has_cudart() { + PROTECT(return at::globalContext().hasCUDART();) + return 0; +} + +bool at_context_has_cudnn() { + PROTECT(return at::globalContext().hasCuDNN();) + return 0; +} + +int64_t at_context_version_cudnn() { + PROTECT(return at::globalContext().versionCuDNN();) + return 0; +} + +int64_t at_context_version_cudart() { + PROTECT(return at::globalContext().versionCUDART();) + return 0; +} + +bool at_context_has_cusolver() { + PROTECT(return at::globalContext().hasCuSOLVER();) + return 0; +} + +bool at_context_has_hip() { + PROTECT(return at::globalContext().hasHIP();) + return 0; +} + +bool at_context_has_ipu() { + PROTECT(return at::globalContext().hasIPU();) + return 0; +} + +bool at_context_has_xla() { + PROTECT(return at::globalContext().hasXLA();) + return 0; +} + +bool at_context_has_lazy() { + PROTECT(return at::globalContext().hasLazy();) + return 0; +} + +bool at_context_has_mps() { + PROTECT(return at::globalContext().hasMPS();) + return 0; +} + +module atm_load(char *filename) { + PROTECT(return new torch::jit::script::Module(torch::jit::load(filename));) + return nullptr; +} + +module atm_load_on_device(char *filename, int device) { + PROTECT(return new torch::jit::script::Module( + torch::jit::load(filename, device_of_int(device)));) + return nullptr; +} + +module atm_load_str(char *data, size_t sz) { + PROTECT(std::istringstream stream(std::string(data, sz)); + return new torch::jit::script::Module(torch::jit::load(stream));) + return nullptr; +} + +module atm_load_str_on_device(char *data, size_t sz, int device) { + PROTECT(std::istringstream stream(std::string(data, sz)); + return new torch::jit::script::Module( + torch::jit::load(stream, device_of_int(device)));) + return nullptr; +} + +tensor atm_forward(module m, tensor *tensors, int ntensors) { + PROTECT(std::vector inputs; + for (int i = 0; i < ntensors; ++i) inputs.push_back(*(tensors[i])); + torch::jit::IValue output = m->forward(std::move(inputs)); + if (!output.isTensor()) throw std::invalid_argument( + "forward did not return a tensor"); + return new torch::Tensor(output.toTensor());) + return nullptr; +} + +ivalue atm_forward_(module m, ivalue *ivalues, int nivalues) { + PROTECT(std::vector inputs; + for (int i = 0; i < nivalues; ++i) inputs.push_back(*(ivalues[i])); + torch::jit::IValue output = m->forward(std::move(inputs)); + return new torch::jit::IValue(output);) + return nullptr; +} + +tensor atm_method(module m, char *method_name, tensor *tensors, int ntensors) { + PROTECT(std::vector inputs; + for (int i = 0; i < ntensors; ++i) inputs.push_back(*(tensors[i])); + torch::jit::IValue output = + m->get_method(method_name)(std::move(inputs)); + if (!output.isTensor()) throw std::invalid_argument( + "method did not return a tensor"); + return new torch::Tensor(output.toTensor());) + return nullptr; +} + +ivalue atm_method_(module m, char *method_name, ivalue *ivalues, int nivalues) { + PROTECT(std::vector inputs; + for (int i = 0; i < nivalues; ++i) inputs.push_back(*(ivalues[i])); + torch::jit::IValue output = + m->get_method(method_name)(std::move(inputs)); + return new torch::jit::IValue(output);) + return nullptr; +} + +ivalue atm_create_class_(module m, char *clz_name, ivalue *ivalues, + int nivalues) { + PROTECT(std::vector inputs; + for (int i = 0; i < nivalues; ++i) inputs.push_back(*(ivalues[i])); + + c10::QualifiedName base(clz_name); + torch::jit::IValue obj = + m->create_class(c10::QualifiedName(clz_name), std::move(inputs)); + return new torch::jit::IValue(obj);) + return nullptr; +} + +void atm_eval(module m) { PROTECT(m->eval();) } + +void atm_train(module m) { PROTECT(m->train();) } + +void atm_free(module m) { delete (m); } + +void atm_save(module m, char *filename) { PROTECT(m->save(filename);) } + +void atm_to(module m, int device, int dtype, bool non_blocking) { + PROTECT(m->to(device_of_int(device), at::ScalarType(dtype), non_blocking);) +} + +int atm_get_profiling_mode() { + PROTECT(return torch::jit::getProfilingMode();) + return 0; +} + +void atm_set_tensor_expr_fuser_enabled(int enabled) { + PROTECT(torch::jit::setTensorExprFuserEnabled((bool)enabled);) +} + +bool atm_get_tensor_expr_fuser_enabled() { + PROTECT(return torch::jit::tensorExprFuserEnabled();) + return false; +} + +void atm_set_profiling_mode(int b) { + PROTECT(torch::jit::getProfilingMode() = (bool)b;) +} + +void atm_fuser_cuda_set_enabled(bool b) { + PROTECT(torch::jit::fuser::cuda::setEnabled(b);) +} + +bool atm_fuser_cuda_is_enabled() { + PROTECT(return torch::jit::fuser::cuda::isEnabled();) + return false; +} + +module atm_create_for_tracing(char *modl_name, tensor *inputs, int ninputs) { + PROTECT(torch::jit::script::Module modl(modl_name); + if (torch::jit::tracer::isTracing()) throw std::invalid_argument( + "cannot nest tracing calls"); + auto state = std::make_shared(); + torch::jit::tracer::setTracingState(state); + auto *_modl_value = state->graph->insertInput(0, "self")->setType( + modl._ivalue()->type()); + for (int i = 0; i < ninputs; ++i) { + auto value = state->graph->addInput(); + value->setType(torch::jit::TensorType::get()); + state->setValue(*inputs[i], value); + } return new torch::jit::script::Module(modl);) + torch::jit::tracer::abandon(); + return nullptr; +} + +void atm_end_tracing(module m, char *fn_name, tensor *outputs, int noutputs) { + PROTECT( + auto state = torch::jit::tracer::getTracingState(); + if (state == nullptr) throw std::invalid_argument("not in tracing mode"); + for (int i = 0; i < noutputs; ++i) { + state->graph->registerOutput(state->getOutput(*outputs[i], i)); + } torch::jit::FixupTraceScopeBlocks(state->graph, m); + torch::jit::NormalizeOps(state->graph); + torch::jit::tracer::setTracingState(nullptr); + auto fn = m->_ivalue()->compilation_unit()->create_function(fn_name, + state->graph); + m->type()->addMethod(fn);) +} + +void atm_named_parameters(module m, void *data, + void (*f)(void *, char *, tensor)) { + PROTECT(for (const auto &p : m->named_parameters()) { + auto v = p.value; + f(data, (char *)p.name.c_str(), new torch::Tensor(v)); + }) +} + +ivalue ati_tensor(tensor t) { + PROTECT(return new torch::jit::IValue(*t);) + return nullptr; +} + +ivalue ati_int(int64_t i) { + PROTECT(return new torch::jit::IValue(i);) + return nullptr; +} + +ivalue ati_double(double d) { + PROTECT(return new torch::jit::IValue(d);) + return nullptr; +} + +ivalue ati_bool(int i) { + PROTECT(return new torch::jit::IValue((bool)i);) + return nullptr; +} + +ivalue ati_string(char *s) { + PROTECT(string str(s); return new torch::jit::IValue(str);) + return nullptr; +} + +ivalue ati_none() { + PROTECT(return new torch::jit::IValue();) + return nullptr; +} + +ivalue ati_tuple(ivalue *is, int nvalues) { + PROTECT(vector vec; + for (int i = 0; i < nvalues; ++i) vec.push_back(*(is[i])); + return new torch::jit::IValue(torch::ivalue::Tuple::create(vec));) + return nullptr; +} + +ivalue ati_generic_list(ivalue *is, int nvalues) { + PROTECT(c10::List vec(c10::AnyType::get()); + for (int i = 0; i < nvalues; ++i) vec.push_back(*(is[i])); + return new torch::jit::IValue(c10::List(vec));) + return nullptr; +} + +using generic_dict = c10::Dict; + +ivalue ati_generic_dict(ivalue *is, int nvalues) { + PROTECT( + bool all_keys_are_str = true; for (int i = 0; i < nvalues; ++i) { + if (!is[2 * i]->isString()) + all_keys_are_str = false; + } bool all_values_are_tensor = true; + for (int i = 0; i < nvalues; ++i) { + if (!is[2 * i + 1]->isTensor()) + all_values_are_tensor = false; + } if (all_keys_are_str && all_values_are_tensor) { + generic_dict dict(c10::StringType::get(), c10::TensorType::get()); + for (int i = 0; i < nvalues; ++i) + dict.insert(is[2 * i]->toString(), is[2 * i + 1]->toTensor()); + return new torch::jit::IValue(dict); + } else if (all_keys_are_str) { + generic_dict dict(c10::StringType::get(), c10::AnyType::get()); + for (int i = 0; i < nvalues; ++i) + dict.insert(is[2 * i]->toString(), *(is[2 * i + 1])); + return new torch::jit::IValue(dict); + } else { + generic_dict dict(c10::AnyType::get(), c10::AnyType::get()); + for (int i = 0; i < nvalues; ++i) + dict.insert(*(is[2 * i]), *(is[2 * i + 1])); + return new torch::jit::IValue(dict); + }) + return nullptr; +} + +ivalue ati_int_list(int64_t *is, int nvalues) { + PROTECT(c10::List vec; + for (int i = 0; i < nvalues; ++i) vec.push_back(is[i]); + return new torch::jit::IValue(vec);) + return nullptr; +} + +ivalue ati_double_list(double *is, int nvalues) { + PROTECT(c10::List vec; + for (int i = 0; i < nvalues; ++i) vec.push_back(is[i]); + return new torch::jit::IValue(vec);) + return nullptr; +} + +ivalue ati_bool_list(char *is, int nvalues) { + PROTECT(c10::List vec; + for (int i = 0; i < nvalues; ++i) vec.push_back(is[i] != 0); + return new torch::jit::IValue(vec);) + return nullptr; +} + +ivalue ati_string_list(char **is, int nvalues) { + PROTECT(c10::List vec; + for (int i = 0; i < nvalues; ++i) vec.push_back(string(is[i])); + return new torch::jit::IValue(vec);) + return nullptr; +} + +ivalue ati_tensor_list(tensor *is, int nvalues) { + PROTECT(c10::List vec; + for (int i = 0; i < nvalues; ++i) vec.push_back(*(is[i])); + return new torch::jit::IValue(vec);) + return nullptr; +} + +ivalue ati_device(int device_idx) { + PROTECT(return new torch::jit::IValue(device_of_int(device_idx));) + return nullptr; +} + +int ati_tag(ivalue i) { + PROTECT( + if (i->isNone()) return 0; else if (i->isTensor()) return 1; + else if (i->isDouble()) return 2; else if (i->isInt()) return 3; + else if (i->isBool()) return 4; else if (i->isTuple()) return 5; + else if (i->isIntList()) return 6; else if (i->isDoubleList()) return 7; + else if (i->isBoolList()) return 8; else if (i->isString()) return 9; + else if (i->isTensorList()) return 10; else if (i->isList()) return 12; + else if (i->isGenericDict()) return 13; else if (i->isObject()) return 14; + throw std::invalid_argument(("unsupported tag " + i->tagKind()).c_str()); + return -1;) + return -1; +} + +int64_t ati_to_int(ivalue i) { + PROTECT(return i->toInt();) + return -1; +} + +double ati_to_double(ivalue i) { + PROTECT(return i->toDouble();) + return 0; +} + +int ati_to_bool(ivalue i) { + PROTECT(return i->toBool();) + return -1; +} + +char *ati_to_string(ivalue i) { + PROTECT(auto str = i->toStringRef(); return strdup(str.c_str());) + return nullptr; +} + +tensor ati_to_tensor(ivalue i) { + PROTECT(return new torch::Tensor(i->toTensor());) + return nullptr; +} + +int ati_length(ivalue i) { + PROTECT(if (i->isTuple()) return i->toTuple()->elements().size(); + else if (i->isIntList()) return i->toIntList().size(); + else if (i->isDoubleList()) return i->toDoubleList().size(); + else if (i->isBoolList()) return i->toBoolList().size(); + else if (i->isString()) return i->toStringRef().size(); + else if (i->isTensorList()) return i->toTensorList().size(); + else if (i->isList()) return i->toList().size(); + else if (i->isGenericDict()) return i->toGenericDict().size(); + throw std::invalid_argument( + ("unsupported tag for length " + i->tagKind()).c_str()); + return -1;) + return -1; +} + +int ati_tuple_length(ivalue i) { + PROTECT(return i->toTuple()->elements().size();) + return -1; +} + +void ati_to_tuple(ivalue i, ivalue *outputs, int noutputs) { + PROTECT(auto vec = i->toTuple()->elements(); if (vec.size() != noutputs) { + throw std::invalid_argument("unexpected tuple size"); + } for (int i = 0; i < noutputs; + ++i) outputs[i] = new torch::jit::IValue(vec[i]);) +} + +void ati_to_generic_list(ivalue i, ivalue *outputs, int noutputs) { + PROTECT(auto vec = i->toList(); if (vec.size() != noutputs) { + throw std::invalid_argument("unexpected list size"); + } for (int i = 0; i < noutputs; ++i) outputs[i] = + new torch::jit::IValue(vec[i]);) +} + +void ati_to_generic_dict(ivalue i, ivalue *outputs, int noutputs) { + PROTECT( + auto dict = i->toGenericDict(); if (dict.size() != noutputs) { + throw std::invalid_argument("unexpected dict size"); + } int k = 0; + for (auto it = dict.begin(); it != dict.end(); ++it) { + outputs[k++] = new torch::jit::IValue(it->key()); + outputs[k++] = new torch::jit::IValue(it->value()); + }) +} + +void ati_to_int_list(ivalue i, int64_t *outputs, int noutputs) { + PROTECT(auto vec = i->toIntList(); if (vec.size() != noutputs) { + throw std::invalid_argument("unexpected list size"); + } for (int i = 0; i < noutputs; ++i) outputs[i] = vec[i];) +} + +void ati_to_double_list(ivalue i, double *outputs, int noutputs) { + PROTECT(auto vec = i->toDoubleList(); if (vec.size() != noutputs) { + throw std::invalid_argument("unexpected list size"); + } for (int i = 0; i < noutputs; ++i) outputs[i] = vec[i];) +} + +void ati_to_bool_list(ivalue i, char *outputs, int noutputs) { + PROTECT(auto vec = i->toBoolList(); if (vec.size() != noutputs) { + throw std::invalid_argument("unexpected list size"); + } for (int i = 0; i < noutputs; ++i) outputs[i] = vec[i];) +} + +void ati_to_tensor_list(ivalue i, tensor *outputs, int noutputs) { + PROTECT(auto vec = i->toTensorList(); if (vec.size() != noutputs) { + throw std::invalid_argument("unexpected list size"); + } for (int i = 0; i < noutputs; ++i) outputs[i] = new torch::Tensor(vec[i]);) +} + +ivalue ati_object_method_(ivalue i, char *method_name, ivalue *ivalues, + int nivalues) { + PROTECT(std::vector inputs; + inputs.push_back(*i); // self parameter + for (int j = 0; j < nivalues; ++j) inputs.push_back(*(ivalues[j])); + torch::jit::IValue output = i->toObjectRef().type()->getMethod( + method_name)(std::move(inputs)); + return new torch::jit::IValue(output);) + return nullptr; +} + +ivalue ati_object_getattr_(ivalue i, char *attr_name) { + PROTECT(torch::jit::IValue output = i->toObjectRef().getAttr(attr_name); + return new torch::jit::IValue(output);) + return nullptr; +} + +ivalue ati_clone(ivalue i) { + PROTECT(return new torch::jit::IValue(*i);) + return nullptr; +} + +void ati_free(ivalue i) { delete (i); } + +void at_set_graph_executor_optimize(bool o) { + torch::jit::setGraphExecutorOptimize(o); +} diff --git a/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api.h b/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api.h new file mode 100644 index 0000000000..5628963889 --- /dev/null +++ b/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api.h @@ -0,0 +1,295 @@ +#ifndef __TORCH_API_H__ +#define __TORCH_API_H__ +#include + +#ifdef __cplusplus +#include +#include +using namespace std; +extern thread_local char *torch_last_err; + +extern "C" { +typedef torch::Tensor *tensor; +typedef torch::Scalar *scalar; +typedef torch::optim::Optimizer *optimizer; +typedef torch::jit::script::Module *module; +typedef torch::jit::IValue *ivalue; +#define PROTECT(x) \ + try { \ + x \ + } catch (const exception& e) { \ + torch_last_err = strdup(e.what()); \ + } +#else +typedef void *tensor; +typedef void *optimizer; +typedef void *scalar; +typedef void *module; +typedef void *ivalue; +#endif + +char *get_and_reset_last_err(); // thread-local +void at_manual_seed(int64_t); +tensor at_new_tensor(); +tensor at_tensor_of_blob(void *data, int64_t *dims, size_t ndims, int64_t *strides, size_t nstrides, int type, int device); +tensor at_tensor_of_data(void *vs, int64_t *dims, size_t ndims, size_t element_size_in_bytes, int type); +void at_copy_data(tensor tensor, void *vs, size_t numel, size_t element_size_in_bytes); +tensor at_shallow_clone(tensor); + +void *at_data_ptr(tensor); +int at_defined(tensor); +int at_is_mkldnn(tensor); +int at_is_sparse(tensor); +int at_is_contiguous(tensor); +int at_device(tensor); +size_t at_dim(tensor); +void at_shape(tensor, int64_t *); +void at_stride(tensor, int64_t *); +int at_scalar_type(tensor); + +void at__amp_non_finite_check_and_unscale(tensor, tensor, tensor); + +void at_autocast_clear_cache(); +int at_autocast_decrement_nesting(); +int at_autocast_increment_nesting(); +bool at_autocast_is_enabled(); +bool at_autocast_set_enabled(bool b); + +void at_backward(tensor, int, int); +int at_requires_grad(tensor); +int at_grad_set_enabled(int); + +tensor at_get(tensor, int index); +void at_fill_double(tensor, double); +void at_fill_int64(tensor, int64_t); + +double at_double_value_at_indexes(tensor, int64_t *indexes, int indexes_len); +int64_t at_int64_value_at_indexes(tensor, int64_t *indexes, int indexes_len); +void at_set_double_value_at_indexes(tensor, int *indexes, int indexes_len, double v); +void at_set_int64_value_at_indexes(tensor, int *indexes, int indexes_len, int64_t v); + +void at_copy_(tensor dst, tensor src); + +void at_print(tensor); +char *at_to_string(tensor, int line_size); +void at_save(tensor, char *filename); +void at_save_to_stream(tensor t, void *stream_ptr); +tensor at_load(char *filename); +tensor at_load_from_stream(void *stream_ptr); +tensor at_load_image(char *filename); +tensor at_load_image_from_memory(unsigned char *img_data, size_t img_size); +int at_save_image(tensor, char *filename); +tensor at_resize_image(tensor, int w, int h); + +void at_save_multi(tensor *tensors, char **tensor_names, int ntensors, char *filename); +void at_save_multi_to_stream(tensor *tensors, char **tensor_names, int ntensors, void *stream_ptr); +/* [at_load_multi] takes as input an array of nullptr for [tensors]. */ +void at_load_multi(tensor *tensors, char **tensor_names, int ntensors, char *filename); +/* [at_load_multi_] takes as input an array of allocation [tensors]. */ +void at_load_multi_(tensor *tensors, char **tensor_names, int ntensors, char *filename); + +void at_loadz_callback(char *filename, void *data, void (*f)(void *, char *, tensor)); +void at_loadz_callback_with_device(char *filename, void *data, void (*f)(void *, char *, tensor), int device_id); +void at_load_callback(char *filename, void *data, void (*f)(void *, char *, tensor)); +void at_load_callback_with_device(char *filename, void *data, void (*f)(void *, char *, tensor), int device_id); +void at_load_from_stream_callback(void *stream_ptr, void *data, void (*f)(void *, char *, tensor), bool enable_device_id, int device_id); + +int at_get_num_interop_threads(); + +int at_get_num_threads(); + +void at_set_num_interop_threads(int n_threads); + +void at_set_num_threads(int n_threads); + +void at_set_qengine(int qengine); + +void at_free(tensor); + +void at_run_backward(tensor *tensors, + int ntensors, + tensor *inputs, + int ninputs, + tensor *outputs, + int keep_graph, + int create_graph); + +optimizer ato_adam(double learning_rate, + double beta1, + double beta2, + double weight_decay, + double eps, + bool amsgrad); +optimizer ato_adamw(double learning_rate, + double beta1, + double beta2, + double weight_decay, + double eps, + bool amsgrad); +optimizer ato_rms_prop(double learning_rate, + double alpha, + double eps, + double weight_decay, + double momentum, + int centered); +optimizer ato_sgd(double learning_rate, + double momentum, + double dampening, + double weight_decay, + int nesterov); +void ato_add_parameters(optimizer, tensor, size_t group); +void ato_set_learning_rate(optimizer, double learning_rate); +void ato_set_momentum(optimizer, double momentum); +void ato_set_learning_rate_group(optimizer, size_t group, double learning_rate); +void ato_set_momentum_group(optimizer, size_t group, double momentum); +void ato_set_weight_decay(optimizer t, double weight_decay); +void ato_set_weight_decay_group(optimizer t, size_t group, double weight_decay); +void ato_zero_grad(optimizer); +void ato_step(optimizer); +void ato_free(optimizer); + +scalar ats_int(int64_t); +scalar ats_float(double); +int64_t ats_to_int(scalar); +double ats_to_float(scalar); +char *ats_to_string(scalar); +void ats_free(scalar); + +bool at_context_has_openmp(); +bool at_context_has_mkl(); +bool at_context_has_lapack(); +bool at_context_has_mkldnn(); +bool at_context_has_magma(); +bool at_context_has_cuda(); +bool at_context_has_cudart(); +bool at_context_has_cudnn(); +int64_t at_context_version_cudnn(); +int64_t at_context_version_cudart(); +bool at_context_has_cusolver(); +bool at_context_has_hip(); +bool at_context_has_ipu(); +bool at_context_has_xla(); +bool at_context_has_lazy(); +bool at_context_has_mps(); + + +/// Returns the number of CUDA devices available. +int atc_cuda_device_count(); + +/// Returns true if at least one CUDA device is available. +int atc_cuda_is_available(); + +/// Returns true if CUDA is available, and CuDNN is available. +int atc_cudnn_is_available(); + +/// Sets the seed for the current GPU. +void atc_manual_seed(uint64_t seed); + +/// Sets the seed for all available GPUs. +void atc_manual_seed_all(uint64_t seed); + +/// Waits for all kernels in all streams on a CUDA device to complete. +void atc_synchronize(int64_t device_index); + + +int atc_user_enabled_cudnn(); +void atc_set_user_enabled_cudnn(int b); +void atc_set_benchmark_cudnn(int b); + +module atm_load(char *); +module atm_load_on_device(char *, int device); +module atm_load_str(char *, size_t sz); +module atm_load_str_on_device(char *, size_t sz, int device); +tensor atm_forward(module, tensor *tensors, int ntensors); +ivalue atm_forward_(module, + ivalue *ivalues, + int nivalues); +tensor atm_method(module, + char *method_name, + tensor *tensors, + int ntensors); +ivalue atm_method_(module, + char *method_name, + ivalue *ivalues, + int nivalues); +ivalue atm_create_class_(module, + char *clz_name, + ivalue *ivalues, + int nivalues); +void atm_eval(module); +void atm_train(module); +void atm_free(module); +void atm_to(module m, int device, int dtype, bool non_blocking); +void atm_save(module m, char*); +int atm_get_profiling_mode(); +void atm_set_profiling_mode(int); +void atm_fuser_cuda_set_enabled(bool); +bool atm_fuser_cuda_is_enabled(); +void atm_named_parameters(module, void *data, void (*f)(void *, char *, tensor)); + +// This function has to be followed by a call to atm_end_tracing. +module atm_create_for_tracing(char *modl_name, tensor *inputs, int ninputs); +void atm_end_tracing(module m, char *fn_name, tensor *outputs, int noutputs); + +ivalue ati_none(); +ivalue ati_tensor(tensor); +ivalue ati_int(int64_t); +ivalue ati_double(double); +ivalue ati_bool(int); +ivalue ati_string(char *); +ivalue ati_tuple(ivalue *, int); +ivalue ati_generic_list(ivalue *, int); +ivalue ati_generic_dict(ivalue *, int); +ivalue ati_int_list(int64_t *, int); +ivalue ati_double_list(double *, int); +ivalue ati_bool_list(char *, int); +ivalue ati_string_list(char **, int); +ivalue ati_tensor_list(tensor *, int); +ivalue ati_device(int); + +tensor ati_to_tensor(ivalue); +int64_t ati_to_int(ivalue); +double ati_to_double(ivalue); +char *ati_to_string(ivalue); +int ati_to_bool(ivalue); +int ati_length(ivalue); +int ati_tuple_length(ivalue); +void ati_to_tuple(ivalue, ivalue *, int); +void ati_to_generic_list(ivalue, ivalue *, int); +void ati_to_generic_dict(ivalue, ivalue *, int); +void ati_to_int_list(ivalue, int64_t *, int); +void ati_to_double_list(ivalue, double *, int); +void ati_to_bool_list(ivalue, char *, int); +void ati_to_tensor_list(ivalue, tensor *, int); + +void atm_set_tensor_expr_fuser_enabled(int); +bool atm_get_tensor_expr_fuser_enabled(); + +int ati_tag(ivalue); + +ivalue ati_object_method_(ivalue i, char *method_name, ivalue *ivalues, int nivalues); +ivalue ati_object_getattr_(ivalue i, char *attr_name); + +ivalue ati_clone(ivalue); +void ati_free(ivalue); + +/// Enables or disables the graph executor optimizer for the current thread. +void at_set_graph_executor_optimize(bool); + +// for internal use +bool tch_write_stream_destructor(void *stream_ptr); +bool tch_write_stream_write(void *stream_ptr, const uint8_t *buf, size_t size, size_t *out_size); +bool tch_read_stream_destructor(void *stream_ptr); +bool tch_read_stream_stream_position(void *stream_ptr, uint64_t *pos); +bool tch_read_stream_seek_start(void *stream_ptr, uint64_t pos, uint64_t *new_pos); +bool tch_read_stream_seek_end(void *stream_ptr, int64_t pos, uint64_t *new_pos); +bool tch_read_stream_read(void *stream_ptr, uint8_t *buf, size_t size, size_t *new_pos); + +#ifdef __cplusplus +}; + +std::vector of_carray_tensor(torch::Tensor **vs, int len); +at::Device device_of_int(int d); +c10::List> of_carray_tensor_opt(torch::Tensor **vs, int len); +#endif +#endif diff --git a/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api_generated.cpp b/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api_generated.cpp new file mode 100644 index 0000000000..acd34c8355 --- /dev/null +++ b/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api_generated.cpp @@ -0,0 +1,19081 @@ +// THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT BY HAND! +#include "torch_api_generated.h" + +void atg___and__(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::__and__(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___and__tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::__and__(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___iand__(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->__iand__(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___iand__tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->__iand__(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___ilshift__(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->__ilshift__(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___ilshift__tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->__ilshift__(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___ior__(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->__ior__(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___ior__tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->__ior__(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___irshift__(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->__irshift__(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___irshift__tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->__irshift__(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___ixor__(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->__ixor__(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___ixor__tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->__ixor__(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___lshift__(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::__lshift__(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___lshift__scalar_out_(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::__lshift___out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___lshift__tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::__lshift__(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___lshift__tensor_out_(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::__lshift___out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___or__(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::__or__(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___or__tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::__or__(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___rshift__(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::__rshift__(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___rshift__scalar_out_(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::__rshift___out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___rshift__tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::__rshift__(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___rshift__tensor_out_(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::__rshift___out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___xor__(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::__xor__(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg___xor__tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::__xor__(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__adaptive_avg_pool2d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len) { + PROTECT( + auto outputs__ = torch::_adaptive_avg_pool2d(*self, torch::IntArrayRef(output_size_data, output_size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__adaptive_avg_pool2d_backward(tensor *out__, tensor grad_output, tensor self) { + PROTECT( + auto outputs__ = torch::_adaptive_avg_pool2d_backward(*grad_output, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__adaptive_avg_pool2d_backward_out(tensor *out__, tensor out, tensor grad_output, tensor self) { + PROTECT( + auto outputs__ = torch::_adaptive_avg_pool2d_backward_out(*out, *grad_output, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__adaptive_avg_pool2d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len) { + PROTECT( + auto outputs__ = torch::_adaptive_avg_pool2d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__adaptive_avg_pool3d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len) { + PROTECT( + auto outputs__ = torch::_adaptive_avg_pool3d(*self, torch::IntArrayRef(output_size_data, output_size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__adaptive_avg_pool3d_backward(tensor *out__, tensor grad_output, tensor self) { + PROTECT( + auto outputs__ = torch::_adaptive_avg_pool3d_backward(*grad_output, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__adaptive_avg_pool3d_backward_out(tensor *out__, tensor out, tensor grad_output, tensor self) { + PROTECT( + auto outputs__ = torch::_adaptive_avg_pool3d_backward_out(*out, *grad_output, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__adaptive_avg_pool3d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len) { + PROTECT( + auto outputs__ = torch::_adaptive_avg_pool3d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__add_batch_dim(tensor *out__, tensor self, int64_t batch_dim, int64_t level) { + PROTECT( + auto outputs__ = torch::_add_batch_dim(*self, batch_dim, level); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__add_relu(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::_add_relu(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__add_relu_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::_add_relu_(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__add_relu_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::_add_relu_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__add_relu_scalar(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::_add_relu(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__add_relu_scalar_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::_add_relu_(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__add_relu_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::_add_relu_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__addmm_activation(tensor *out__, tensor self, tensor mat1, tensor mat2, int use_gelu) { + PROTECT( + auto outputs__ = torch::_addmm_activation(*self, *mat1, *mat2, (bool)use_gelu); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__addmm_activation_out(tensor *out__, tensor out, tensor self, tensor mat1, tensor mat2, int use_gelu) { + PROTECT( + auto outputs__ = torch::_addmm_activation_out(*out, *self, *mat1, *mat2, (bool)use_gelu); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__aminmax(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_aminmax(*self); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__aminmax_dim(tensor *out__, tensor self, int64_t dim, int keepdim) { + PROTECT( + auto outputs__ = torch::_aminmax(*self, dim, (bool)keepdim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__aminmax_dim_out(tensor *out__, tensor out0, tensor out1, tensor self, int64_t dim, int keepdim) { + PROTECT( + auto outputs__ = torch::_aminmax_out(*out0, *out1, *self, dim, (bool)keepdim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__aminmax_out(tensor *out__, tensor out0, tensor out1, tensor self) { + PROTECT( + auto outputs__ = torch::_aminmax_out(*out0, *out1, *self); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__amp_update_scale(tensor *out__, tensor self, tensor growth_tracker, tensor found_inf, double scale_growth_factor, double scale_backoff_factor, int64_t growth_interval) { + PROTECT( + auto outputs__ = torch::_amp_update_scale(*self, *growth_tracker, *found_inf, scale_growth_factor, scale_backoff_factor, growth_interval); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__amp_update_scale_(tensor *out__, tensor self, tensor growth_tracker, tensor found_inf, double scale_growth_factor, double scale_backoff_factor, int64_t growth_interval) { + PROTECT( + auto outputs__ = torch::_amp_update_scale_(*self, *growth_tracker, *found_inf, scale_growth_factor, scale_backoff_factor, growth_interval); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__amp_update_scale_out(tensor *out__, tensor out, tensor self, tensor growth_tracker, tensor found_inf, double scale_growth_factor, double scale_backoff_factor, int64_t growth_interval) { + PROTECT( + auto outputs__ = torch::_amp_update_scale_out(*out, *self, *growth_tracker, *found_inf, scale_growth_factor, scale_backoff_factor, growth_interval); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__assert_scalar(scalar self_scalar, char* assert_msg_ptr, int assert_msg_len) { + PROTECT( + torch::_assert_scalar(*self_scalar, std::string(assert_msg_ptr, assert_msg_len)); + ) +} + +void atg__assert_tensor_metadata(tensor a, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int dtype, int device, int8_t layout) { + PROTECT( + torch::_assert_tensor_metadata(*a, size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(size_data, size_len)), stride_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(stride_data, stride_len)), dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype)), device_of_int(device), (layout == -1 ? c10::nullopt : c10::optional(static_cast(layout)))); + ) +} + +void atg__autocast_to_full_precision(tensor *out__, tensor self, int cuda_enabled, int cpu_enabled) { + PROTECT( + auto outputs__ = self->_autocast_to_full_precision((bool)cuda_enabled, (bool)cpu_enabled); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__autocast_to_reduced_precision(tensor *out__, tensor self, int cuda_enabled, int cpu_enabled, int cuda_dtype, int cpu_dtype) { + PROTECT( + auto outputs__ = self->_autocast_to_reduced_precision((bool)cuda_enabled, (bool)cpu_enabled, at::ScalarType(cuda_dtype), at::ScalarType(cpu_dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__batch_norm_no_update(tensor *out__, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps) { + PROTECT( + auto outputs__ = torch::_batch_norm_no_update(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), momentum, eps); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg__batch_norm_no_update_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor out3, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps) { + PROTECT( + auto outputs__ = torch::_batch_norm_no_update_out(*out0, *out1, *out2, *out3, *input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), momentum, eps); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg__batch_norm_with_update(tensor *out__, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps) { + PROTECT( + auto outputs__ = torch::_batch_norm_with_update(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *running_mean, *running_var, momentum, eps); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg__batch_norm_with_update_functional(tensor *out__, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps) { + PROTECT( + auto outputs__ = torch::_batch_norm_with_update_functional(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *running_mean, *running_var, momentum, eps); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + out__[4] = new torch::Tensor(std::get<4>(outputs__)); + out__[5] = new torch::Tensor(std::get<5>(outputs__)); + ) +} + +void atg__batch_norm_with_update_out(tensor *out__, tensor out, tensor save_mean, tensor save_invstd, tensor reserve, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps) { + PROTECT( + auto outputs__ = torch::_batch_norm_with_update_out(*out, *save_mean, *save_invstd, *reserve, *input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *running_mean, *running_var, momentum, eps); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg__cast_byte(tensor *out__, tensor self, int non_blocking) { + PROTECT( + auto outputs__ = torch::_cast_Byte(*self, (bool)non_blocking); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__cast_char(tensor *out__, tensor self, int non_blocking) { + PROTECT( + auto outputs__ = torch::_cast_Char(*self, (bool)non_blocking); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__cast_double(tensor *out__, tensor self, int non_blocking) { + PROTECT( + auto outputs__ = torch::_cast_Double(*self, (bool)non_blocking); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__cast_float(tensor *out__, tensor self, int non_blocking) { + PROTECT( + auto outputs__ = torch::_cast_Float(*self, (bool)non_blocking); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__cast_half(tensor *out__, tensor self, int non_blocking) { + PROTECT( + auto outputs__ = torch::_cast_Half(*self, (bool)non_blocking); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__cast_int(tensor *out__, tensor self, int non_blocking) { + PROTECT( + auto outputs__ = torch::_cast_Int(*self, (bool)non_blocking); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__cast_long(tensor *out__, tensor self, int non_blocking) { + PROTECT( + auto outputs__ = torch::_cast_Long(*self, (bool)non_blocking); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__cast_short(tensor *out__, tensor self, int non_blocking) { + PROTECT( + auto outputs__ = torch::_cast_Short(*self, (bool)non_blocking); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__cdist_backward(tensor *out__, tensor grad, tensor x1, tensor x2, double p, tensor cdist) { + PROTECT( + auto outputs__ = torch::_cdist_backward(*grad, *x1, *x2, p, *cdist); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__cdist_backward_out(tensor *out__, tensor out, tensor grad, tensor x1, tensor x2, double p, tensor cdist) { + PROTECT( + auto outputs__ = torch::_cdist_backward_out(*out, *grad, *x1, *x2, p, *cdist); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__cholesky_solve_helper(tensor *out__, tensor self, tensor A, int upper) { + PROTECT( + auto outputs__ = torch::_cholesky_solve_helper(*self, *A, (bool)upper); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__cholesky_solve_helper_out(tensor *out__, tensor out, tensor self, tensor A, int upper) { + PROTECT( + auto outputs__ = torch::_cholesky_solve_helper_out(*out, *self, *A, (bool)upper); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__chunk_cat(tensor *out__, tensor *tensors_data, int tensors_len, int64_t dim, int64_t num_chunks) { + PROTECT( + auto outputs__ = torch::_chunk_cat(of_carray_tensor(tensors_data, tensors_len), dim, num_chunks); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__chunk_cat_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len, int64_t dim, int64_t num_chunks) { + PROTECT( + auto outputs__ = torch::_chunk_cat_out(*out, of_carray_tensor(tensors_data, tensors_len), dim, num_chunks); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__coalesce(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_coalesce(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__coalesce_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::_coalesce_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__coalesced(tensor *out__, tensor self, int coalesced) { + PROTECT( + auto outputs__ = torch::_coalesced(*self, (bool)coalesced); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__coalesced_(tensor *out__, tensor self, int coalesced) { + PROTECT( + auto outputs__ = self->_coalesced_((bool)coalesced); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__coalesced_out(tensor *out__, tensor out, tensor self, int coalesced) { + PROTECT( + auto outputs__ = torch::_coalesced_out(*out, *self, (bool)coalesced); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__compute_linear_combination(tensor *out__, tensor input, tensor coefficients) { + PROTECT( + auto outputs__ = torch::_compute_linear_combination(*input, *coefficients); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__compute_linear_combination_out(tensor *out__, tensor out, tensor input, tensor coefficients) { + PROTECT( + auto outputs__ = torch::_compute_linear_combination_out(*out, *input, *coefficients); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__conj(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_conj(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__conj_copy(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_conj_copy(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__conj_copy_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::_conj_copy_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__conj_physical(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_conj_physical(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__conj_physical_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::_conj_physical_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__conv_depthwise2d(tensor *out__, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len) { + PROTECT( + auto outputs__ = torch::_conv_depthwise2d(*self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__conv_depthwise2d_out(tensor *out__, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len) { + PROTECT( + auto outputs__ = torch::_conv_depthwise2d_out(*out, *self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__convert_indices_from_coo_to_csr(tensor *out__, tensor self, int64_t size, int out_int32) { + PROTECT( + auto outputs__ = torch::_convert_indices_from_coo_to_csr(*self, size, (bool)out_int32); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__convert_indices_from_coo_to_csr_out(tensor *out__, tensor out, tensor self, int64_t size, int out_int32) { + PROTECT( + auto outputs__ = torch::_convert_indices_from_coo_to_csr_out(*out, *self, size, (bool)out_int32); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__convert_indices_from_csr_to_coo(tensor *out__, tensor crow_indices, tensor col_indices, int out_int32, int transpose) { + PROTECT( + auto outputs__ = torch::_convert_indices_from_csr_to_coo(*crow_indices, *col_indices, (bool)out_int32, (bool)transpose); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__convert_indices_from_csr_to_coo_out(tensor *out__, tensor out, tensor crow_indices, tensor col_indices, int out_int32, int transpose) { + PROTECT( + auto outputs__ = torch::_convert_indices_from_csr_to_coo_out(*out, *crow_indices, *col_indices, (bool)out_int32, (bool)transpose); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__convert_weight_to_int4pack(tensor *out__, tensor self, int64_t innerKTiles) { + PROTECT( + auto outputs__ = torch::_convert_weight_to_int4pack(*self, innerKTiles); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__convert_weight_to_int4pack_for_cpu(tensor *out__, tensor self, int64_t innerKTiles) { + PROTECT( + auto outputs__ = torch::_convert_weight_to_int4pack_for_cpu(*self, innerKTiles); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__convolution(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups, int benchmark, int deterministic, int cudnn_enabled, int allow_tf32) { + PROTECT( + auto outputs__ = torch::_convolution(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)transposed, torch::IntArrayRef(output_padding_data, output_padding_len), groups, (bool)benchmark, (bool)deterministic, (bool)cudnn_enabled, (bool)allow_tf32); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__convolution_deprecated(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups, int benchmark, int deterministic, int cudnn_enabled) { + PROTECT( + auto outputs__ = torch::_convolution(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)transposed, torch::IntArrayRef(output_padding_data, output_padding_len), groups, (bool)benchmark, (bool)deterministic, (bool)cudnn_enabled); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__convolution_mode(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, char* padding_ptr, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::_convolution_mode(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), std::string(padding_ptr, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__convolution_out(tensor *out__, tensor out, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups, int benchmark, int deterministic, int cudnn_enabled, int allow_tf32) { + PROTECT( + auto outputs__ = torch::_convolution_out(*out, *input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)transposed, torch::IntArrayRef(output_padding_data, output_padding_len), groups, (bool)benchmark, (bool)deterministic, (bool)cudnn_enabled, (bool)allow_tf32); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__copy_from(tensor *out__, tensor self, tensor dst, int non_blocking) { + PROTECT( + auto outputs__ = torch::_copy_from(*self, *dst, (bool)non_blocking); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__copy_from_and_resize(tensor *out__, tensor self, tensor dst) { + PROTECT( + auto outputs__ = torch::_copy_from_and_resize(*self, *dst); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__copy_from_and_resize_out(tensor *out__, tensor out, tensor self, tensor dst) { + PROTECT( + auto outputs__ = torch::_copy_from_and_resize_out(*out, *self, *dst); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__copy_from_out(tensor *out__, tensor out, tensor self, tensor dst, int non_blocking) { + PROTECT( + auto outputs__ = torch::_copy_from_out(*out, *self, *dst, (bool)non_blocking); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__cslt_compress(tensor *out__, tensor input) { + PROTECT( + auto outputs__ = torch::_cslt_compress(*input); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__cslt_sparse_mm(tensor *out__, tensor compressed_A, tensor dense_B, tensor bias, tensor alpha, int out_dtype, int transpose_result, int64_t alg_id, int64_t split_k, int64_t split_k_mode) { + PROTECT( + auto outputs__ = torch::_cslt_sparse_mm(*compressed_A, *dense_B, (bias ? ::std::optional(*bias) : ::std::nullopt), (alpha ? ::std::optional(*alpha) : ::std::nullopt), out_dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(out_dtype)), (bool)transpose_result, alg_id, split_k, split_k_mode); + out__[0] = new torch::Tensor(outputs__); + ) +} + +int64_t atg__cslt_sparse_mm_search(tensor compressed_A, tensor dense_B, tensor bias, tensor alpha, int out_dtype, int transpose_result) { + PROTECT( + return torch::_cslt_sparse_mm_search(*compressed_A, *dense_B, (bias ? ::std::optional(*bias) : ::std::nullopt), (alpha ? ::std::optional(*alpha) : ::std::nullopt), out_dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(out_dtype)), (bool)transpose_result); + ) + return 0; +} + +void atg__ctc_loss(tensor *out__, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int zero_infinity) { + PROTECT( + auto outputs__ = torch::_ctc_loss(*log_probs, *targets, torch::IntArrayRef(input_lengths_data, input_lengths_len), torch::IntArrayRef(target_lengths_data, target_lengths_len), blank, (bool)zero_infinity); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__ctc_loss_backward(tensor *out__, tensor grad, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, tensor neg_log_likelihood, tensor log_alpha, int64_t blank, int zero_infinity) { + PROTECT( + auto outputs__ = torch::_ctc_loss_backward(*grad, *log_probs, *targets, torch::IntArrayRef(input_lengths_data, input_lengths_len), torch::IntArrayRef(target_lengths_data, target_lengths_len), *neg_log_likelihood, *log_alpha, blank, (bool)zero_infinity); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__ctc_loss_backward_out(tensor *out__, tensor out, tensor grad, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, tensor neg_log_likelihood, tensor log_alpha, int64_t blank, int zero_infinity) { + PROTECT( + auto outputs__ = torch::_ctc_loss_backward_out(*out, *grad, *log_probs, *targets, torch::IntArrayRef(input_lengths_data, input_lengths_len), torch::IntArrayRef(target_lengths_data, target_lengths_len), *neg_log_likelihood, *log_alpha, blank, (bool)zero_infinity); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__ctc_loss_backward_tensor(tensor *out__, tensor grad, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, tensor neg_log_likelihood, tensor log_alpha, int64_t blank, int zero_infinity) { + PROTECT( + auto outputs__ = torch::_ctc_loss_backward(*grad, *log_probs, *targets, *input_lengths, *target_lengths, *neg_log_likelihood, *log_alpha, blank, (bool)zero_infinity); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__ctc_loss_out(tensor *out__, tensor out0, tensor out1, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int zero_infinity) { + PROTECT( + auto outputs__ = torch::_ctc_loss_out(*out0, *out1, *log_probs, *targets, torch::IntArrayRef(input_lengths_data, input_lengths_len), torch::IntArrayRef(target_lengths_data, target_lengths_len), blank, (bool)zero_infinity); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__ctc_loss_tensor(tensor *out__, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank, int zero_infinity) { + PROTECT( + auto outputs__ = torch::_ctc_loss(*log_probs, *targets, *input_lengths, *target_lengths, blank, (bool)zero_infinity); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__ctc_loss_tensor_out(tensor *out__, tensor out0, tensor out1, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank, int zero_infinity) { + PROTECT( + auto outputs__ = torch::_ctc_loss_out(*out0, *out1, *log_probs, *targets, *input_lengths, *target_lengths, blank, (bool)zero_infinity); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__cudnn_attention_backward(tensor *out__, tensor grad_out, tensor query, tensor key, tensor value, tensor out, tensor logsumexp, tensor philox_seed, tensor philox_offset, tensor attn_bias, tensor cum_seq_q, tensor cum_seq_k, int64_t max_q, int64_t max_k, double dropout_p, int is_causal, double scale_v, uint8_t scale_null) { + PROTECT( + auto outputs__ = torch::_cudnn_attention_backward(*grad_out, *query, *key, *value, *out, *logsumexp, *philox_seed, *philox_offset, *attn_bias, *cum_seq_q, *cum_seq_k, max_q, max_k, dropout_p, (bool)is_causal, scale_null ? c10::nullopt : c10::optional(scale_v)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__cudnn_ctc_loss(tensor *out__, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int deterministic, int zero_infinity) { + PROTECT( + auto outputs__ = torch::_cudnn_ctc_loss(*log_probs, *targets, torch::IntArrayRef(input_lengths_data, input_lengths_len), torch::IntArrayRef(target_lengths_data, target_lengths_len), blank, (bool)deterministic, (bool)zero_infinity); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__cudnn_ctc_loss_out(tensor *out__, tensor out0, tensor out1, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int deterministic, int zero_infinity) { + PROTECT( + auto outputs__ = torch::_cudnn_ctc_loss_out(*out0, *out1, *log_probs, *targets, torch::IntArrayRef(input_lengths_data, input_lengths_len), torch::IntArrayRef(target_lengths_data, target_lengths_len), blank, (bool)deterministic, (bool)zero_infinity); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__cudnn_ctc_loss_tensor(tensor *out__, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank, int deterministic, int zero_infinity) { + PROTECT( + auto outputs__ = torch::_cudnn_ctc_loss(*log_probs, *targets, *input_lengths, *target_lengths, blank, (bool)deterministic, (bool)zero_infinity); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__cudnn_init_dropout_state(tensor *out__, double dropout, int train, int64_t dropout_seed, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::_cudnn_init_dropout_state(dropout, (bool)train, dropout_seed, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__cudnn_init_dropout_state_out(tensor *out__, tensor out, double dropout, int train, int64_t dropout_seed) { + PROTECT( + auto outputs__ = torch::_cudnn_init_dropout_state_out(*out, dropout, (bool)train, dropout_seed); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__cudnn_rnn(tensor *out__, tensor input, tensor *weight_data, int weight_len, int64_t weight_stride0, tensor weight_buf, tensor hx, tensor cx, int64_t mode, int64_t hidden_size, int64_t proj_size, int64_t num_layers, int batch_first, double dropout, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, tensor dropout_state) { + PROTECT( + auto outputs__ = torch::_cudnn_rnn(*input, of_carray_tensor(weight_data, weight_len), weight_stride0, (weight_buf ? ::std::optional(*weight_buf) : ::std::nullopt), *hx, (cx ? ::std::optional(*cx) : ::std::nullopt), mode, hidden_size, proj_size, num_layers, (bool)batch_first, dropout, (bool)train, (bool)bidirectional, torch::IntArrayRef(batch_sizes_data, batch_sizes_len), (dropout_state ? ::std::optional(*dropout_state) : ::std::nullopt)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + out__[4] = new torch::Tensor(std::get<4>(outputs__)); + ) +} + +void atg__cudnn_rnn_flatten_weight(tensor *out__, tensor *weight_arr_data, int weight_arr_len, int64_t weight_stride0, int64_t input_size, int64_t mode, int64_t hidden_size, int64_t proj_size, int64_t num_layers, int batch_first, int bidirectional) { + PROTECT( + auto outputs__ = torch::_cudnn_rnn_flatten_weight(of_carray_tensor(weight_arr_data, weight_arr_len), weight_stride0, input_size, mode, hidden_size, proj_size, num_layers, (bool)batch_first, (bool)bidirectional); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__cudnn_rnn_flatten_weight_out(tensor *out__, tensor out, tensor *weight_arr_data, int weight_arr_len, int64_t weight_stride0, int64_t input_size, int64_t mode, int64_t hidden_size, int64_t proj_size, int64_t num_layers, int batch_first, int bidirectional) { + PROTECT( + auto outputs__ = torch::_cudnn_rnn_flatten_weight_out(*out, of_carray_tensor(weight_arr_data, weight_arr_len), weight_stride0, input_size, mode, hidden_size, proj_size, num_layers, (bool)batch_first, (bool)bidirectional); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__cudnn_rnn_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor out3, tensor out4, tensor input, tensor *weight_data, int weight_len, int64_t weight_stride0, tensor weight_buf, tensor hx, tensor cx, int64_t mode, int64_t hidden_size, int64_t proj_size, int64_t num_layers, int batch_first, double dropout, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, tensor dropout_state) { + PROTECT( + auto outputs__ = torch::_cudnn_rnn_out(*out0, *out1, *out2, *out3, *out4, *input, of_carray_tensor(weight_data, weight_len), weight_stride0, (weight_buf ? ::std::optional(*weight_buf) : ::std::nullopt), *hx, (cx ? ::std::optional(*cx) : ::std::nullopt), mode, hidden_size, proj_size, num_layers, (bool)batch_first, dropout, (bool)train, (bool)bidirectional, torch::IntArrayRef(batch_sizes_data, batch_sizes_len), (dropout_state ? ::std::optional(*dropout_state) : ::std::nullopt)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + out__[4] = new torch::Tensor(std::get<4>(outputs__)); + ) +} + +int64_t atg__debug_has_internal_overlap(tensor self) { + PROTECT( + return torch::_debug_has_internal_overlap(*self); + ) + return 0; +} + +void atg__dim_arange(tensor *out__, tensor like, int64_t dim) { + PROTECT( + auto outputs__ = torch::_dim_arange(*like, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +int64_t atg__dimi(tensor self) { + PROTECT( + return self->_dimI(); + ) + return 0; +} + +int64_t atg__dimv(tensor self) { + PROTECT( + return self->_dimV(); + ) + return 0; +} + +void atg__dirichlet_grad(tensor *out__, tensor x, tensor alpha, tensor total) { + PROTECT( + auto outputs__ = torch::_dirichlet_grad(*x, *alpha, *total); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__dirichlet_grad_out(tensor *out__, tensor out, tensor x, tensor alpha, tensor total) { + PROTECT( + auto outputs__ = torch::_dirichlet_grad_out(*out, *x, *alpha, *total); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__dyn_quant_matmul_4bit(tensor *out__, tensor inp, tensor packed_weights, int64_t block_size, int64_t in_features, int64_t out_features) { + PROTECT( + auto outputs__ = torch::_dyn_quant_matmul_4bit(*inp, *packed_weights, block_size, in_features, out_features); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__dyn_quant_pack_4bit_weight(tensor *out__, tensor weights, tensor scales_zeros, tensor bias, int64_t block_size, int64_t in_features, int64_t out_features) { + PROTECT( + auto outputs__ = torch::_dyn_quant_pack_4bit_weight(*weights, *scales_zeros, (bias ? ::std::optional(*bias) : ::std::nullopt), block_size, in_features, out_features); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__efficient_attention_backward(tensor *out__, tensor grad_out_, tensor query, tensor key, tensor value, tensor bias, tensor out, tensor cu_seqlens_q, tensor cu_seqlens_k, int64_t max_seqlen_q, int64_t max_seqlen_k, tensor logsumexp, double dropout_p, tensor philox_seed, tensor philox_offset, int64_t custom_mask_type, int bias_requires_grad, double scale_v, uint8_t scale_null, int64_t num_splits_key_v, uint8_t num_splits_key_null, int64_t window_size_v, uint8_t window_size_null, int shared_storage_dqdkdv) { + PROTECT( + auto outputs__ = torch::_efficient_attention_backward(*grad_out_, *query, *key, *value, (bias ? ::std::optional(*bias) : ::std::nullopt), *out, (cu_seqlens_q ? ::std::optional(*cu_seqlens_q) : ::std::nullopt), (cu_seqlens_k ? ::std::optional(*cu_seqlens_k) : ::std::nullopt), max_seqlen_q, max_seqlen_k, *logsumexp, dropout_p, *philox_seed, *philox_offset, custom_mask_type, (bool)bias_requires_grad, scale_null ? c10::nullopt : c10::optional(scale_v), num_splits_key_null ? c10::nullopt : c10::optional(num_splits_key_v), window_size_null ? c10::nullopt : c10::optional(window_size_v), (bool)shared_storage_dqdkdv); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg__efficientzerotensor(tensor *out__, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::_efficientzerotensor(torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__efficientzerotensor_out(tensor *out__, tensor out, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::_efficientzerotensor_out(*out, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__embedding_bag(tensor *out__, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset, int64_t padding_idx) { + PROTECT( + auto outputs__ = torch::_embedding_bag(*weight, *indices, *offsets, (bool)scale_grad_by_freq, mode, (bool)sparse, (per_sample_weights ? ::std::optional(*per_sample_weights) : ::std::nullopt), (bool)include_last_offset, padding_idx); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg__embedding_bag_backward(tensor *out__, tensor grad, tensor indices, tensor offsets, tensor offset2bag, tensor bag_size, tensor maximum_indices, int64_t num_weights, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int64_t padding_idx) { + PROTECT( + auto outputs__ = torch::_embedding_bag_backward(*grad, *indices, *offsets, *offset2bag, *bag_size, *maximum_indices, num_weights, (bool)scale_grad_by_freq, mode, (bool)sparse, (per_sample_weights ? ::std::optional(*per_sample_weights) : ::std::nullopt), padding_idx); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__embedding_bag_dense_backward(tensor *out__, tensor grad, tensor indices, tensor offset2bag, tensor bag_size, tensor maximum_indices, int64_t num_weights, int scale_grad_by_freq, int64_t mode, tensor per_sample_weights, int64_t padding_idx) { + PROTECT( + auto outputs__ = torch::_embedding_bag_dense_backward(*grad, *indices, *offset2bag, *bag_size, *maximum_indices, num_weights, (bool)scale_grad_by_freq, mode, (per_sample_weights ? ::std::optional(*per_sample_weights) : ::std::nullopt), padding_idx); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__embedding_bag_dense_backward_out(tensor *out__, tensor out, tensor grad, tensor indices, tensor offset2bag, tensor bag_size, tensor maximum_indices, int64_t num_weights, int scale_grad_by_freq, int64_t mode, tensor per_sample_weights, int64_t padding_idx) { + PROTECT( + auto outputs__ = torch::_embedding_bag_dense_backward_out(*out, *grad, *indices, *offset2bag, *bag_size, *maximum_indices, num_weights, (bool)scale_grad_by_freq, mode, (per_sample_weights ? ::std::optional(*per_sample_weights) : ::std::nullopt), padding_idx); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__embedding_bag_forward_only(tensor *out__, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset, int64_t padding_idx) { + PROTECT( + auto outputs__ = torch::_embedding_bag_forward_only(*weight, *indices, *offsets, (bool)scale_grad_by_freq, mode, (bool)sparse, (per_sample_weights ? ::std::optional(*per_sample_weights) : ::std::nullopt), (bool)include_last_offset, padding_idx); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg__embedding_bag_forward_only_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor out3, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset, int64_t padding_idx) { + PROTECT( + auto outputs__ = torch::_embedding_bag_forward_only_out(*out0, *out1, *out2, *out3, *weight, *indices, *offsets, (bool)scale_grad_by_freq, mode, (bool)sparse, (per_sample_weights ? ::std::optional(*per_sample_weights) : ::std::nullopt), (bool)include_last_offset, padding_idx); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg__embedding_bag_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor out3, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset, int64_t padding_idx) { + PROTECT( + auto outputs__ = torch::_embedding_bag_out(*out0, *out1, *out2, *out3, *weight, *indices, *offsets, (bool)scale_grad_by_freq, mode, (bool)sparse, (per_sample_weights ? ::std::optional(*per_sample_weights) : ::std::nullopt), (bool)include_last_offset, padding_idx); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg__embedding_bag_per_sample_weights_backward(tensor *out__, tensor grad, tensor weight, tensor indices, tensor offsets, tensor offset2bag, int64_t mode, int64_t padding_idx) { + PROTECT( + auto outputs__ = torch::_embedding_bag_per_sample_weights_backward(*grad, *weight, *indices, *offsets, *offset2bag, mode, padding_idx); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__embedding_bag_per_sample_weights_backward_out(tensor *out__, tensor out, tensor grad, tensor weight, tensor indices, tensor offsets, tensor offset2bag, int64_t mode, int64_t padding_idx) { + PROTECT( + auto outputs__ = torch::_embedding_bag_per_sample_weights_backward_out(*out, *grad, *weight, *indices, *offsets, *offset2bag, mode, padding_idx); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__embedding_bag_sparse_backward(tensor *out__, tensor grad, tensor indices, tensor offsets, tensor offset2bag, tensor bag_size, int64_t num_weights, int scale_grad_by_freq, int64_t mode, tensor per_sample_weights, int64_t padding_idx) { + PROTECT( + auto outputs__ = torch::_embedding_bag_sparse_backward(*grad, *indices, *offsets, *offset2bag, *bag_size, num_weights, (bool)scale_grad_by_freq, mode, (per_sample_weights ? ::std::optional(*per_sample_weights) : ::std::nullopt), padding_idx); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__empty_affine_quantized(tensor *out__, int64_t *size_data, int size_len, int options_kind, int options_device, double scale, int64_t zero_point) { + PROTECT( + auto outputs__ = torch::_empty_affine_quantized(torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind)), scale, zero_point); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__empty_affine_quantized_out(tensor *out__, tensor out, int64_t *size_data, int size_len, double scale, int64_t zero_point) { + PROTECT( + auto outputs__ = torch::_empty_affine_quantized_out(*out, torch::IntArrayRef(size_data, size_len), scale, zero_point); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__empty_per_channel_affine_quantized(tensor *out__, int64_t *size_data, int size_len, tensor scales, tensor zero_points, int64_t axis, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::_empty_per_channel_affine_quantized(torch::IntArrayRef(size_data, size_len), *scales, *zero_points, axis, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__empty_per_channel_affine_quantized_out(tensor *out__, tensor out, int64_t *size_data, int size_len, tensor scales, tensor zero_points, int64_t axis) { + PROTECT( + auto outputs__ = torch::_empty_per_channel_affine_quantized_out(*out, torch::IntArrayRef(size_data, size_len), *scales, *zero_points, axis); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__euclidean_dist(tensor *out__, tensor x1, tensor x2) { + PROTECT( + auto outputs__ = torch::_euclidean_dist(*x1, *x2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__euclidean_dist_out(tensor *out__, tensor out, tensor x1, tensor x2) { + PROTECT( + auto outputs__ = torch::_euclidean_dist_out(*out, *x1, *x2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__fake_quantize_learnable_per_channel_affine(tensor *out__, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max, double grad_factor) { + PROTECT( + auto outputs__ = torch::_fake_quantize_learnable_per_channel_affine(*self, *scale, *zero_point, axis, quant_min, quant_max, grad_factor); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__fake_quantize_learnable_per_channel_affine_backward(tensor *out__, tensor grad, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max, double grad_factor) { + PROTECT( + auto outputs__ = torch::_fake_quantize_learnable_per_channel_affine_backward(*grad, *self, *scale, *zero_point, axis, quant_min, quant_max, grad_factor); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__fake_quantize_learnable_per_channel_affine_out(tensor *out__, tensor out, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max, double grad_factor) { + PROTECT( + auto outputs__ = torch::_fake_quantize_learnable_per_channel_affine_out(*out, *self, *scale, *zero_point, axis, quant_min, quant_max, grad_factor); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__fake_quantize_learnable_per_tensor_affine(tensor *out__, tensor self, tensor scale, tensor zero_point, int64_t quant_min, int64_t quant_max, double grad_factor) { + PROTECT( + auto outputs__ = torch::_fake_quantize_learnable_per_tensor_affine(*self, *scale, *zero_point, quant_min, quant_max, grad_factor); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__fake_quantize_learnable_per_tensor_affine_backward(tensor *out__, tensor grad, tensor self, tensor scale, tensor zero_point, int64_t quant_min, int64_t quant_max, double grad_factor) { + PROTECT( + auto outputs__ = torch::_fake_quantize_learnable_per_tensor_affine_backward(*grad, *self, *scale, *zero_point, quant_min, quant_max, grad_factor); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__fake_quantize_learnable_per_tensor_affine_out(tensor *out__, tensor out, tensor self, tensor scale, tensor zero_point, int64_t quant_min, int64_t quant_max, double grad_factor) { + PROTECT( + auto outputs__ = torch::_fake_quantize_learnable_per_tensor_affine_out(*out, *self, *scale, *zero_point, quant_min, quant_max, grad_factor); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__fake_quantize_per_tensor_affine_cachemask_tensor_qparams(tensor *out__, tensor self, tensor scale, tensor zero_point, tensor fake_quant_enabled, int64_t quant_min, int64_t quant_max) { + PROTECT( + auto outputs__ = torch::_fake_quantize_per_tensor_affine_cachemask_tensor_qparams(*self, *scale, *zero_point, *fake_quant_enabled, quant_min, quant_max); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__fake_quantize_per_tensor_affine_cachemask_tensor_qparams_out(tensor *out__, tensor out0, tensor out1, tensor self, tensor scale, tensor zero_point, tensor fake_quant_enabled, int64_t quant_min, int64_t quant_max) { + PROTECT( + auto outputs__ = torch::_fake_quantize_per_tensor_affine_cachemask_tensor_qparams_out(*out0, *out1, *self, *scale, *zero_point, *fake_quant_enabled, quant_min, quant_max); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__fft_c2c(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int forward) { + PROTECT( + auto outputs__ = torch::_fft_c2c(*self, torch::IntArrayRef(dim_data, dim_len), normalization, (bool)forward); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__fft_c2c_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int forward) { + PROTECT( + auto outputs__ = torch::_fft_c2c_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), normalization, (bool)forward); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__fft_c2r(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int64_t last_dim_size) { + PROTECT( + auto outputs__ = torch::_fft_c2r(*self, torch::IntArrayRef(dim_data, dim_len), normalization, last_dim_size); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__fft_c2r_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int64_t last_dim_size) { + PROTECT( + auto outputs__ = torch::_fft_c2r_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), normalization, last_dim_size); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__fft_r2c(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int onesided) { + PROTECT( + auto outputs__ = torch::_fft_r2c(*self, torch::IntArrayRef(dim_data, dim_len), normalization, (bool)onesided); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__fft_r2c_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int onesided) { + PROTECT( + auto outputs__ = torch::_fft_r2c_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), normalization, (bool)onesided); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__fill_mem_eff_dropout_mask_(tensor *out__, tensor self, double dropout_p, int64_t seed, int64_t offset) { + PROTECT( + auto outputs__ = torch::_fill_mem_eff_dropout_mask_(*self, dropout_p, seed, offset); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__flash_attention_backward(tensor *out__, tensor grad_out, tensor query, tensor key, tensor value, tensor out, tensor logsumexp, tensor cum_seq_q, tensor cum_seq_k, int64_t max_q, int64_t max_k, double dropout_p, int is_causal, tensor rng_state, tensor unused, double scale_v, uint8_t scale_null, int64_t window_size_left_v, uint8_t window_size_left_null, int64_t window_size_right_v, uint8_t window_size_right_null) { + PROTECT( + auto outputs__ = torch::_flash_attention_backward(*grad_out, *query, *key, *value, *out, *logsumexp, *cum_seq_q, *cum_seq_k, max_q, max_k, dropout_p, (bool)is_causal, *rng_state, *unused, scale_null ? c10::nullopt : c10::optional(scale_v), window_size_left_null ? c10::nullopt : c10::optional(window_size_left_v), window_size_right_null ? c10::nullopt : c10::optional(window_size_right_v)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__foobar(tensor *out__, tensor self, int arg1, int arg2, int arg3) { + PROTECT( + auto outputs__ = torch::_foobar(*self, (bool)arg1, (bool)arg2, (bool)arg3); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__foobar_out(tensor *out__, tensor out, tensor self, int arg1, int arg2, int arg3) { + PROTECT( + auto outputs__ = torch::_foobar_out(*out, *self, (bool)arg1, (bool)arg2, (bool)arg3); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__functional_assert_async(tensor *out__, tensor self, char* assert_msg_ptr, int assert_msg_len, tensor dep_token) { + PROTECT( + auto outputs__ = torch::_functional_assert_async(*self, std::string(assert_msg_ptr, assert_msg_len), *dep_token); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__functional_assert_scalar(tensor *out__, scalar self_scalar, char* assert_msg_ptr, int assert_msg_len, tensor dep_token) { + PROTECT( + auto outputs__ = torch::_functional_assert_scalar(*self_scalar, std::string(assert_msg_ptr, assert_msg_len), *dep_token); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__functional_sym_constrain_range(tensor *out__, scalar size, int64_t min_v, uint8_t min_null, int64_t max_v, uint8_t max_null, tensor dep_token) { + PROTECT( + auto outputs__ = torch::_functional_sym_constrain_range(*size, min_null ? c10::nullopt : c10::optional(min_v), max_null ? c10::nullopt : c10::optional(max_v), *dep_token); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__functional_sym_constrain_range_for_size(tensor *out__, scalar size, int64_t min_v, uint8_t min_null, int64_t max_v, uint8_t max_null, tensor dep_token) { + PROTECT( + auto outputs__ = torch::_functional_sym_constrain_range_for_size(*size, min_null ? c10::nullopt : c10::optional(min_v), max_null ? c10::nullopt : c10::optional(max_v), *dep_token); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__fused_dropout(tensor *out__, tensor self, double p) { + PROTECT( + auto outputs__ = torch::_fused_dropout(*self, p); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__fused_dropout_out(tensor *out__, tensor out0, tensor out1, tensor self, double p) { + PROTECT( + auto outputs__ = torch::_fused_dropout_out(*out0, *out1, *self, p); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__fused_moving_avg_obs_fq_helper(tensor *out__, tensor self, tensor observer_on, tensor fake_quant_on, tensor running_min, tensor running_max, tensor scale, tensor zero_point, double averaging_const, int64_t quant_min, int64_t quant_max, int64_t ch_axis, int per_row_fake_quant, int symmetric_quant) { + PROTECT( + auto outputs__ = torch::_fused_moving_avg_obs_fq_helper(*self, *observer_on, *fake_quant_on, *running_min, *running_max, *scale, *zero_point, averaging_const, quant_min, quant_max, ch_axis, (bool)per_row_fake_quant, (bool)symmetric_quant); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__fused_moving_avg_obs_fq_helper_functional(tensor *out__, tensor self, tensor observer_on, tensor fake_quant_on, tensor running_min, tensor running_max, tensor scale, tensor zero_point, double averaging_const, int64_t quant_min, int64_t quant_max, int64_t ch_axis, int per_row_fake_quant, int symmetric_quant) { + PROTECT( + auto outputs__ = torch::_fused_moving_avg_obs_fq_helper_functional(*self, *observer_on, *fake_quant_on, *running_min, *running_max, *scale, *zero_point, averaging_const, quant_min, quant_max, ch_axis, (bool)per_row_fake_quant, (bool)symmetric_quant); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + out__[4] = new torch::Tensor(std::get<4>(outputs__)); + out__[5] = new torch::Tensor(std::get<5>(outputs__)); + ) +} + +void atg__fused_moving_avg_obs_fq_helper_out(tensor *out__, tensor out0, tensor out1, tensor self, tensor observer_on, tensor fake_quant_on, tensor running_min, tensor running_max, tensor scale, tensor zero_point, double averaging_const, int64_t quant_min, int64_t quant_max, int64_t ch_axis, int per_row_fake_quant, int symmetric_quant) { + PROTECT( + auto outputs__ = torch::_fused_moving_avg_obs_fq_helper_out(*out0, *out1, *self, *observer_on, *fake_quant_on, *running_min, *running_max, *scale, *zero_point, averaging_const, quant_min, quant_max, ch_axis, (bool)per_row_fake_quant, (bool)symmetric_quant); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__fused_rms_norm(tensor *out__, tensor input, int64_t *normalized_shape_data, int normalized_shape_len, tensor weight, double eps_v, uint8_t eps_null) { + PROTECT( + auto outputs__ = torch::_fused_rms_norm(*input, torch::IntArrayRef(normalized_shape_data, normalized_shape_len), (weight ? ::std::optional(*weight) : ::std::nullopt), eps_null ? c10::nullopt : c10::optional(eps_v)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +int64_t atg__fused_sdp_choice(tensor query, tensor key, tensor value, tensor attn_mask, double dropout_p, int is_causal, double scale_v, uint8_t scale_null, int enable_gqa) { + PROTECT( + return torch::_fused_sdp_choice(*query, *key, *value, (attn_mask ? ::std::optional(*attn_mask) : ::std::nullopt), dropout_p, (bool)is_causal, scale_null ? c10::nullopt : c10::optional(scale_v), (bool)enable_gqa); + ) + return 0; +} + +void atg__fw_primal(tensor *out__, tensor self, int64_t level) { + PROTECT( + auto outputs__ = self->_fw_primal(level); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__fw_primal_copy(tensor *out__, tensor self, int64_t level) { + PROTECT( + auto outputs__ = torch::_fw_primal_copy(*self, level); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__fw_primal_copy_out(tensor *out__, tensor out, tensor self, int64_t level) { + PROTECT( + auto outputs__ = torch::_fw_primal_copy_out(*out, *self, level); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__gather_sparse_backward(tensor *out__, tensor self, int64_t dim, tensor index, tensor grad) { + PROTECT( + auto outputs__ = torch::_gather_sparse_backward(*self, dim, *index, *grad); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__grid_sampler_2d_cpu_fallback(tensor *out__, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners) { + PROTECT( + auto outputs__ = torch::_grid_sampler_2d_cpu_fallback(*input, *grid, interpolation_mode, padding_mode, (bool)align_corners); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__grid_sampler_2d_cpu_fallback_backward(tensor *out__, tensor grad_output, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners) { + PROTECT( + auto outputs__ = torch::_grid_sampler_2d_cpu_fallback_backward(*grad_output, *input, *grid, interpolation_mode, padding_mode, (bool)align_corners); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__grid_sampler_2d_cpu_fallback_out(tensor *out__, tensor out, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners) { + PROTECT( + auto outputs__ = torch::_grid_sampler_2d_cpu_fallback_out(*out, *input, *grid, interpolation_mode, padding_mode, (bool)align_corners); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__grouped_mm(tensor *out__, tensor self, tensor mat2, tensor offs, tensor bias, int out_dtype) { + PROTECT( + auto outputs__ = torch::_grouped_mm(*self, *mat2, (offs ? ::std::optional(*offs) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), out_dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(out_dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +int atg__has_compatible_shallow_copy_type(tensor self, tensor from) { + PROTECT( + return torch::_has_compatible_shallow_copy_type(*self, *from); + ) + return 0; +} + +int atg__has_same_storage_numel(tensor self, tensor other) { + PROTECT( + return torch::_has_same_storage_numel(*self, *other); + ) + return 0; +} + +tensor *atg__histogramdd_bin_edges(tensor self, int64_t *bins_data, int bins_len, double *range_data, int range_len, tensor weight, int density) { + PROTECT( + auto outputs__ = torch::_histogramdd_bin_edges(*self, torch::IntArrayRef(bins_data, bins_len), at::ArrayRef(range_data, range_len), (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)density); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg__histogramdd_bin_edges_out(tensor *out_data, int out_len, tensor self, int64_t *bins_data, int bins_len, double *range_data, int range_len, tensor weight, int density) { + PROTECT( + torch::_histogramdd_bin_edges_out(of_carray_tensor(out_data, out_len), *self, torch::IntArrayRef(bins_data, bins_len), at::ArrayRef(range_data, range_len), (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)density); + ) +} + +void atg__histogramdd_from_bin_cts(tensor *out__, tensor self, int64_t *bins_data, int bins_len, double *range_data, int range_len, tensor weight, int density) { + PROTECT( + auto outputs__ = torch::_histogramdd_from_bin_cts(*self, torch::IntArrayRef(bins_data, bins_len), at::ArrayRef(range_data, range_len), (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)density); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__histogramdd_from_bin_cts_out(tensor *out__, tensor out, tensor self, int64_t *bins_data, int bins_len, double *range_data, int range_len, tensor weight, int density) { + PROTECT( + auto outputs__ = torch::_histogramdd_from_bin_cts_out(*out, *self, torch::IntArrayRef(bins_data, bins_len), at::ArrayRef(range_data, range_len), (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)density); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__histogramdd_from_bin_tensors(tensor *out__, tensor self, tensor *bins_data, int bins_len, tensor weight, int density) { + PROTECT( + auto outputs__ = torch::_histogramdd_from_bin_tensors(*self, of_carray_tensor(bins_data, bins_len), (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)density); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__histogramdd_from_bin_tensors_out(tensor *out__, tensor out, tensor self, tensor *bins_data, int bins_len, tensor weight, int density) { + PROTECT( + auto outputs__ = torch::_histogramdd_from_bin_tensors_out(*out, *self, of_carray_tensor(bins_data, bins_len), (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)density); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__index_put_impl(tensor *out__, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate, int unsafe) { + PROTECT( + auto outputs__ = torch::_index_put_impl(*self, of_carray_tensor_opt(indices_data, indices_len), *values, (bool)accumulate, (bool)unsafe); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__index_put_impl_(tensor *out__, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate, int unsafe) { + PROTECT( + auto outputs__ = torch::_index_put_impl_(*self, of_carray_tensor_opt(indices_data, indices_len), *values, (bool)accumulate, (bool)unsafe); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__index_put_impl_out(tensor *out__, tensor out, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate, int unsafe) { + PROTECT( + auto outputs__ = torch::_index_put_impl_out(*out, *self, of_carray_tensor_opt(indices_data, indices_len), *values, (bool)accumulate, (bool)unsafe); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__indices(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->_indices(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__indices_copy(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_indices_copy(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__indices_copy_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::_indices_copy_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__int_mm(tensor *out__, tensor self, tensor mat2) { + PROTECT( + auto outputs__ = torch::_int_mm(*self, *mat2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__int_mm_out(tensor *out__, tensor out, tensor self, tensor mat2) { + PROTECT( + auto outputs__ = torch::_int_mm_out(*out, *self, *mat2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__is_all_true(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_is_all_true(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__is_any_true(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_is_any_true(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +int atg__is_zerotensor(tensor self) { + PROTECT( + return torch::_is_zerotensor(*self); + ) + return 0; +} + +void atg__lazy_clone(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_lazy_clone(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__linalg_check_errors(tensor info, char* api_name_ptr, int api_name_len, int is_matrix) { + PROTECT( + torch::_linalg_check_errors(*info, std::string(api_name_ptr, api_name_len), (bool)is_matrix); + ) +} + +void atg__linalg_det(tensor *out__, tensor A) { + PROTECT( + auto outputs__ = torch::_linalg_det(*A); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__linalg_det_result(tensor *out__, tensor result, tensor LU, tensor pivots, tensor A) { + PROTECT( + auto outputs__ = torch::_linalg_det_out(*result, *LU, *pivots, *A); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__linalg_eigh(tensor *out__, tensor A, char* UPLO_ptr, int UPLO_len, int compute_v) { + PROTECT( + auto outputs__ = torch::_linalg_eigh(*A, std::string(UPLO_ptr, UPLO_len), (bool)compute_v); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__linalg_eigh_eigenvalues(tensor *out__, tensor eigenvalues, tensor eigenvectors, tensor A, char* UPLO_ptr, int UPLO_len, int compute_v) { + PROTECT( + auto outputs__ = torch::_linalg_eigh_out(*eigenvalues, *eigenvectors, *A, std::string(UPLO_ptr, UPLO_len), (bool)compute_v); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__linalg_eigvals(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_linalg_eigvals(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__linalg_slogdet(tensor *out__, tensor A) { + PROTECT( + auto outputs__ = torch::_linalg_slogdet(*A); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg__linalg_slogdet_sign(tensor *out__, tensor sign, tensor logabsdet, tensor LU, tensor pivots, tensor A) { + PROTECT( + auto outputs__ = torch::_linalg_slogdet_out(*sign, *logabsdet, *LU, *pivots, *A); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg__linalg_solve_ex(tensor *out__, tensor A, tensor B, int left, int check_errors) { + PROTECT( + auto outputs__ = torch::_linalg_solve_ex(*A, *B, (bool)left, (bool)check_errors); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg__linalg_solve_ex_result(tensor *out__, tensor result, tensor LU, tensor pivots, tensor info, tensor A, tensor B, int left, int check_errors) { + PROTECT( + auto outputs__ = torch::_linalg_solve_ex_out(*result, *LU, *pivots, *info, *A, *B, (bool)left, (bool)check_errors); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg__linalg_svd(tensor *out__, tensor A, int full_matrices, int compute_uv, char* driver_ptr, int driver_len) { + PROTECT( + auto outputs__ = torch::_linalg_svd(*A, (bool)full_matrices, (bool)compute_uv, std::string(driver_ptr, driver_len)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__linalg_svd_u(tensor *out__, tensor U, tensor S, tensor Vh, tensor A, int full_matrices, int compute_uv, char* driver_ptr, int driver_len) { + PROTECT( + auto outputs__ = torch::_linalg_svd_out(*U, *S, *Vh, *A, (bool)full_matrices, (bool)compute_uv, std::string(driver_ptr, driver_len)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__log_softmax(tensor *out__, tensor self, int64_t dim, int half_to_float) { + PROTECT( + auto outputs__ = torch::_log_softmax(*self, dim, (bool)half_to_float); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__log_softmax_backward_data(tensor *out__, tensor grad_output, tensor output, int64_t dim, int input_dtype) { + PROTECT( + auto outputs__ = torch::_log_softmax_backward_data(*grad_output, *output, dim, at::ScalarType(input_dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__log_softmax_backward_data_out(tensor *out__, tensor out, tensor grad_output, tensor output, int64_t dim, int input_dtype) { + PROTECT( + auto outputs__ = torch::_log_softmax_backward_data_out(*out, *grad_output, *output, dim, at::ScalarType(input_dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__log_softmax_out(tensor *out__, tensor out, tensor self, int64_t dim, int half_to_float) { + PROTECT( + auto outputs__ = torch::_log_softmax_out(*out, *self, dim, (bool)half_to_float); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__logcumsumexp(tensor *out__, tensor self, int64_t dim) { + PROTECT( + auto outputs__ = torch::_logcumsumexp(*self, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__logcumsumexp_out(tensor *out__, tensor out, tensor self, int64_t dim) { + PROTECT( + auto outputs__ = torch::_logcumsumexp_out(*out, *self, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__lstm_mps(tensor *out__, tensor input, tensor *hx_data, int hx_len, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first) { + PROTECT( + auto outputs__ = torch::_lstm_mps(*input, of_carray_tensor(hx_data, hx_len), of_carray_tensor(params_data, params_len), (bool)has_biases, num_layers, dropout, (bool)train, (bool)bidirectional, (bool)batch_first); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + out__[4] = new torch::Tensor(std::get<4>(outputs__)); + out__[5] = new torch::Tensor(std::get<5>(outputs__)); + ) +} + +void atg__lstm_mps_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor out3, tensor out4, tensor out5, tensor input, tensor *hx_data, int hx_len, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first) { + PROTECT( + auto outputs__ = torch::_lstm_mps_out(*out0, *out1, *out2, *out3, *out4, *out5, *input, of_carray_tensor(hx_data, hx_len), of_carray_tensor(params_data, params_len), (bool)has_biases, num_layers, dropout, (bool)train, (bool)bidirectional, (bool)batch_first); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + out__[4] = new torch::Tensor(std::get<4>(outputs__)); + out__[5] = new torch::Tensor(std::get<5>(outputs__)); + ) +} + +void atg__lu_with_info(tensor *out__, tensor self, int pivot, int check_errors) { + PROTECT( + auto outputs__ = torch::_lu_with_info(*self, (bool)pivot, (bool)check_errors); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__make_dep_token(tensor *out__, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::_make_dep_token(at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__make_dual(tensor *out__, tensor primal, tensor tangent, int64_t level) { + PROTECT( + auto outputs__ = torch::_make_dual(*primal, *tangent, level); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__make_dual_copy(tensor *out__, tensor primal, tensor tangent, int64_t level) { + PROTECT( + auto outputs__ = torch::_make_dual_copy(*primal, *tangent, level); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__make_dual_copy_out(tensor *out__, tensor out, tensor primal, tensor tangent, int64_t level) { + PROTECT( + auto outputs__ = torch::_make_dual_copy_out(*out, *primal, *tangent, level); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__make_per_channel_quantized_tensor(tensor *out__, tensor self, tensor scale, tensor zero_point, int64_t axis) { + PROTECT( + auto outputs__ = torch::_make_per_channel_quantized_tensor(*self, *scale, *zero_point, axis); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__make_per_channel_quantized_tensor_out(tensor *out__, tensor out, tensor self, tensor scale, tensor zero_point, int64_t axis) { + PROTECT( + auto outputs__ = torch::_make_per_channel_quantized_tensor_out(*out, *self, *scale, *zero_point, axis); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__make_per_tensor_quantized_tensor(tensor *out__, tensor self, double scale, int64_t zero_point) { + PROTECT( + auto outputs__ = torch::_make_per_tensor_quantized_tensor(*self, scale, zero_point); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__make_per_tensor_quantized_tensor_out(tensor *out__, tensor out, tensor self, double scale, int64_t zero_point) { + PROTECT( + auto outputs__ = torch::_make_per_tensor_quantized_tensor_out(*out, *self, scale, zero_point); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__masked_scale(tensor *out__, tensor self, tensor mask, double scale) { + PROTECT( + auto outputs__ = torch::_masked_scale(*self, *mask, scale); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__masked_scale_out(tensor *out__, tensor out, tensor self, tensor mask, double scale) { + PROTECT( + auto outputs__ = torch::_masked_scale_out(*out, *self, *mask, scale); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__masked_softmax(tensor *out__, tensor self, tensor mask, int64_t dim_v, uint8_t dim_null, int64_t mask_type_v, uint8_t mask_type_null) { + PROTECT( + auto outputs__ = torch::_masked_softmax(*self, *mask, dim_null ? c10::nullopt : c10::optional(dim_v), mask_type_null ? c10::nullopt : c10::optional(mask_type_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__masked_softmax_backward(tensor *out__, tensor grad_output, tensor output, tensor mask, int64_t dim_v, uint8_t dim_null) { + PROTECT( + auto outputs__ = torch::_masked_softmax_backward(*grad_output, *output, *mask, dim_null ? c10::nullopt : c10::optional(dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__masked_softmax_backward_out(tensor *out__, tensor out, tensor grad_output, tensor output, tensor mask, int64_t dim_v, uint8_t dim_null) { + PROTECT( + auto outputs__ = torch::_masked_softmax_backward_out(*out, *grad_output, *output, *mask, dim_null ? c10::nullopt : c10::optional(dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__masked_softmax_out(tensor *out__, tensor out, tensor self, tensor mask, int64_t dim_v, uint8_t dim_null, int64_t mask_type_v, uint8_t mask_type_null) { + PROTECT( + auto outputs__ = torch::_masked_softmax_out(*out, *self, *mask, dim_null ? c10::nullopt : c10::optional(dim_v), mask_type_null ? c10::nullopt : c10::optional(mask_type_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__mixed_dtypes_linear(tensor *out__, tensor input, tensor weight, tensor scale, tensor bias, char* activation_ptr, int activation_len) { + PROTECT( + auto outputs__ = torch::_mixed_dtypes_linear(*input, *weight, *scale, (bias ? ::std::optional(*bias) : ::std::nullopt), std::string(activation_ptr, activation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__mkldnn_reshape(tensor *out__, tensor self, int64_t *shape_data, int shape_len) { + PROTECT( + auto outputs__ = torch::_mkldnn_reshape(*self, torch::IntArrayRef(shape_data, shape_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__mkldnn_reshape_out(tensor *out__, tensor out, tensor self, int64_t *shape_data, int shape_len) { + PROTECT( + auto outputs__ = torch::_mkldnn_reshape_out(*out, *self, torch::IntArrayRef(shape_data, shape_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__mkldnn_transpose(tensor *out__, tensor self, int64_t dim0, int64_t dim1) { + PROTECT( + auto outputs__ = torch::_mkldnn_transpose(*self, dim0, dim1); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__mkldnn_transpose_(tensor *out__, tensor self, int64_t dim0, int64_t dim1) { + PROTECT( + auto outputs__ = torch::_mkldnn_transpose_(*self, dim0, dim1); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__mkldnn_transpose_out(tensor *out__, tensor out, tensor self, int64_t dim0, int64_t dim1) { + PROTECT( + auto outputs__ = torch::_mkldnn_transpose_out(*out, *self, dim0, dim1); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__mps_convolution(tensor *out__, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::_mps_convolution(*self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__mps_convolution_out(tensor *out__, tensor out, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::_mps_convolution_out(*out, *self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__mps_convolution_transpose(tensor *out__, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::_mps_convolution_transpose(*self, *weight, torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__mps_convolution_transpose_out(tensor *out__, tensor out, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::_mps_convolution_transpose_out(*out, *self, *weight, torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__native_batch_norm_legit(tensor *out__, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps) { + PROTECT( + auto outputs__ = torch::_native_batch_norm_legit(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *running_mean, *running_var, (bool)training, momentum, eps); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__native_batch_norm_legit_functional(tensor *out__, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps) { + PROTECT( + auto outputs__ = torch::_native_batch_norm_legit_functional(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *running_mean, *running_var, (bool)training, momentum, eps); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + out__[4] = new torch::Tensor(std::get<4>(outputs__)); + ) +} + +void atg__native_batch_norm_legit_no_stats(tensor *out__, tensor input, tensor weight, tensor bias, int training, double momentum, double eps) { + PROTECT( + auto outputs__ = torch::_native_batch_norm_legit(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), (bool)training, momentum, eps); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__native_batch_norm_legit_no_stats_out(tensor *out__, tensor out, tensor save_mean, tensor save_invstd, tensor input, tensor weight, tensor bias, int training, double momentum, double eps) { + PROTECT( + auto outputs__ = torch::_native_batch_norm_legit_out(*out, *save_mean, *save_invstd, *input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), (bool)training, momentum, eps); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__native_batch_norm_legit_no_training(tensor *out__, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps) { + PROTECT( + auto outputs__ = torch::_native_batch_norm_legit_no_training(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *running_mean, *running_var, momentum, eps); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__native_batch_norm_legit_no_training_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps) { + PROTECT( + auto outputs__ = torch::_native_batch_norm_legit_no_training_out(*out0, *out1, *out2, *input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *running_mean, *running_var, momentum, eps); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__native_batch_norm_legit_out(tensor *out__, tensor out, tensor save_mean, tensor save_invstd, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps) { + PROTECT( + auto outputs__ = torch::_native_batch_norm_legit_out(*out, *save_mean, *save_invstd, *input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *running_mean, *running_var, (bool)training, momentum, eps); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__native_multi_head_attention(tensor *out__, tensor query, tensor key, tensor value, int64_t embed_dim, int64_t num_head, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, tensor mask, int need_weights, int average_attn_weights, int64_t mask_type_v, uint8_t mask_type_null) { + PROTECT( + auto outputs__ = torch::_native_multi_head_attention(*query, *key, *value, embed_dim, num_head, *qkv_weight, *qkv_bias, *proj_weight, *proj_bias, (mask ? ::std::optional(*mask) : ::std::nullopt), (bool)need_weights, (bool)average_attn_weights, mask_type_null ? c10::nullopt : c10::optional(mask_type_v)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__native_multi_head_attention_out(tensor *out__, tensor out0, tensor out1, tensor query, tensor key, tensor value, int64_t embed_dim, int64_t num_head, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, tensor mask, int need_weights, int average_attn_weights, int64_t mask_type_v, uint8_t mask_type_null) { + PROTECT( + auto outputs__ = torch::_native_multi_head_attention_out(*out0, *out1, *query, *key, *value, embed_dim, num_head, *qkv_weight, *qkv_bias, *proj_weight, *proj_bias, (mask ? ::std::optional(*mask) : ::std::nullopt), (bool)need_weights, (bool)average_attn_weights, mask_type_null ? c10::nullopt : c10::optional(mask_type_v)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__neg_view(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_neg_view(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__neg_view_copy(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_neg_view_copy(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__neg_view_copy_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::_neg_view_copy_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__nested_compute_contiguous_strides_offsets(tensor *out__, tensor nested_size) { + PROTECT( + auto outputs__ = torch::_nested_compute_contiguous_strides_offsets(*nested_size); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__nested_from_padded(tensor *out__, tensor padded, tensor cpu_nested_shape_example, int fuse_transform_0213) { + PROTECT( + auto outputs__ = torch::_nested_from_padded(*padded, *cpu_nested_shape_example, (bool)fuse_transform_0213); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__nested_from_padded_and_nested_example(tensor *out__, tensor padded, tensor nt_example) { + PROTECT( + auto outputs__ = torch::_nested_from_padded_and_nested_example(*padded, *nt_example); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__nested_from_padded_and_nested_example_out(tensor *out__, tensor out, tensor padded, tensor nt_example) { + PROTECT( + auto outputs__ = torch::_nested_from_padded_and_nested_example_out(*out, *padded, *nt_example); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__nested_from_padded_out(tensor *out__, tensor out, tensor padded, tensor cpu_nested_shape_example, int fuse_transform_0213) { + PROTECT( + auto outputs__ = torch::_nested_from_padded_out(*out, *padded, *cpu_nested_shape_example, (bool)fuse_transform_0213); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__nested_from_padded_tensor(tensor *out__, tensor padded, tensor offsets, tensor dummy, int64_t ragged_idx, tensor min_seqlen, tensor max_seqlen, int64_t sum_S_v, uint8_t sum_S_null) { + PROTECT( + auto outputs__ = torch::_nested_from_padded_tensor(*padded, *offsets, *dummy, ragged_idx, (min_seqlen ? ::std::optional(*min_seqlen) : ::std::nullopt), (max_seqlen ? ::std::optional(*max_seqlen) : ::std::nullopt), sum_S_null ? c10::nullopt : c10::optional(sum_S_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__nested_get_jagged_dummy(tensor *out__, tensor any) { + PROTECT( + auto outputs__ = torch::_nested_get_jagged_dummy(*any); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__nested_get_lengths(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_nested_get_lengths(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__nested_get_max_seqlen(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_nested_get_max_seqlen(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__nested_get_min_seqlen(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_nested_get_min_seqlen(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__nested_get_offsets(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_nested_get_offsets(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +int64_t atg__nested_get_ragged_idx(tensor self) { + PROTECT( + return torch::_nested_get_ragged_idx(*self); + ) + return 0; +} + +void atg__nested_get_values(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_nested_get_values(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__nested_get_values_copy(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_nested_get_values_copy(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__nested_get_values_copy_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::_nested_get_values_copy_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__nested_select_backward(tensor *out__, tensor grad_output, tensor self, int64_t dim, int64_t index) { + PROTECT( + auto outputs__ = torch::_nested_select_backward(*grad_output, *self, dim, index); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__nested_sum_backward(tensor *out__, tensor grad, tensor self, int64_t *dim_data, int dim_len, int keepdim) { + PROTECT( + auto outputs__ = torch::_nested_sum_backward(*grad, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__nested_view_from_buffer(tensor *out__, tensor self, tensor nested_size, tensor nested_strides, tensor offsets) { + PROTECT( + auto outputs__ = torch::_nested_view_from_buffer(*self, *nested_size, *nested_strides, *offsets); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__nested_view_from_buffer_copy(tensor *out__, tensor self, tensor nested_size, tensor nested_strides, tensor offsets) { + PROTECT( + auto outputs__ = torch::_nested_view_from_buffer_copy(*self, *nested_size, *nested_strides, *offsets); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__nested_view_from_buffer_copy_out(tensor *out__, tensor out, tensor self, tensor nested_size, tensor nested_strides, tensor offsets) { + PROTECT( + auto outputs__ = torch::_nested_view_from_buffer_copy_out(*out, *self, *nested_size, *nested_strides, *offsets); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__nested_view_from_jagged(tensor *out__, tensor self, tensor offsets, tensor dummy, tensor lengths, int64_t ragged_idx, tensor min_seqlen, tensor max_seqlen) { + PROTECT( + auto outputs__ = torch::_nested_view_from_jagged(*self, *offsets, *dummy, (lengths ? ::std::optional(*lengths) : ::std::nullopt), ragged_idx, (min_seqlen ? ::std::optional(*min_seqlen) : ::std::nullopt), (max_seqlen ? ::std::optional(*max_seqlen) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__nested_view_from_jagged_copy(tensor *out__, tensor self, tensor offsets, tensor dummy, tensor lengths, int64_t ragged_idx, tensor min_seqlen, tensor max_seqlen) { + PROTECT( + auto outputs__ = torch::_nested_view_from_jagged_copy(*self, *offsets, *dummy, (lengths ? ::std::optional(*lengths) : ::std::nullopt), ragged_idx, (min_seqlen ? ::std::optional(*min_seqlen) : ::std::nullopt), (max_seqlen ? ::std::optional(*max_seqlen) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__nested_view_from_jagged_copy_out(tensor *out__, tensor out, tensor self, tensor offsets, tensor dummy, tensor lengths, int64_t ragged_idx, tensor min_seqlen, tensor max_seqlen) { + PROTECT( + auto outputs__ = torch::_nested_view_from_jagged_copy_out(*out, *self, *offsets, *dummy, (lengths ? ::std::optional(*lengths) : ::std::nullopt), ragged_idx, (min_seqlen ? ::std::optional(*min_seqlen) : ::std::nullopt), (max_seqlen ? ::std::optional(*max_seqlen) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__new_zeros_with_same_feature_meta(tensor *out__, tensor self, tensor other, int64_t self_num_batch_dims) { + PROTECT( + auto outputs__ = torch::_new_zeros_with_same_feature_meta(*self, *other, self_num_batch_dims); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__new_zeros_with_same_feature_meta_out(tensor *out__, tensor out, tensor self, tensor other, int64_t self_num_batch_dims) { + PROTECT( + auto outputs__ = torch::_new_zeros_with_same_feature_meta_out(*out, *self, *other, self_num_batch_dims); + out__[0] = new torch::Tensor(outputs__); + ) +} + +int atg__nnpack_available() { + PROTECT( + return torch::_nnpack_available(); + ) + return 0; +} + +void atg__nnpack_spatial_convolution(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len) { + PROTECT( + auto outputs__ = torch::_nnpack_spatial_convolution(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__nnpack_spatial_convolution_out(tensor *out__, tensor out, tensor input, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len) { + PROTECT( + auto outputs__ = torch::_nnpack_spatial_convolution_out(*out, *input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +int64_t atg__nnz(tensor self) { + PROTECT( + return self->_nnz(); + ) + return 0; +} + +void atg__pack_padded_sequence(tensor *out__, tensor input, tensor lengths, int batch_first) { + PROTECT( + auto outputs__ = torch::_pack_padded_sequence(*input, *lengths, (bool)batch_first); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__pack_padded_sequence_backward(tensor *out__, tensor grad, int64_t *input_size_data, int input_size_len, tensor batch_sizes, int batch_first) { + PROTECT( + auto outputs__ = torch::_pack_padded_sequence_backward(*grad, torch::IntArrayRef(input_size_data, input_size_len), *batch_sizes, (bool)batch_first); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__pack_padded_sequence_out(tensor *out__, tensor out0, tensor out1, tensor input, tensor lengths, int batch_first) { + PROTECT( + auto outputs__ = torch::_pack_padded_sequence_out(*out0, *out1, *input, *lengths, (bool)batch_first); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__pad_circular(tensor *out__, tensor self, int64_t *pad_data, int pad_len) { + PROTECT( + auto outputs__ = torch::_pad_circular(*self, torch::IntArrayRef(pad_data, pad_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__pad_enum(tensor *out__, tensor self, int64_t *pad_data, int pad_len, int64_t mode, double value_v, uint8_t value_null) { + PROTECT( + auto outputs__ = torch::_pad_enum(*self, torch::IntArrayRef(pad_data, pad_len), mode, value_null ? c10::nullopt : c10::optional(value_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__pad_packed_sequence(tensor *out__, tensor data, tensor batch_sizes, int batch_first, scalar padding_value, int64_t total_length) { + PROTECT( + auto outputs__ = torch::_pad_packed_sequence(*data, *batch_sizes, (bool)batch_first, *padding_value, total_length); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__pdist_backward(tensor *out__, tensor grad, tensor self, double p, tensor pdist) { + PROTECT( + auto outputs__ = torch::_pdist_backward(*grad, *self, p, *pdist); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__pdist_backward_out(tensor *out__, tensor out, tensor grad, tensor self, double p, tensor pdist) { + PROTECT( + auto outputs__ = torch::_pdist_backward_out(*out, *grad, *self, p, *pdist); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__pin_memory(tensor *out__, tensor self, int device) { + PROTECT( + auto outputs__ = torch::_pin_memory(*self, device_of_int(device)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__pin_memory_out(tensor *out__, tensor out, tensor self, int device) { + PROTECT( + auto outputs__ = torch::_pin_memory_out(*out, *self, device_of_int(device)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__prelu_kernel(tensor *out__, tensor self, tensor weight) { + PROTECT( + auto outputs__ = torch::_prelu_kernel(*self, *weight); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__prelu_kernel_backward(tensor *out__, tensor grad_output, tensor self, tensor weight) { + PROTECT( + auto outputs__ = torch::_prelu_kernel_backward(*grad_output, *self, *weight); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__print(char* s_ptr, int s_len) { + PROTECT( + torch::_print(std::string(s_ptr, s_len)); + ) +} + +void atg__propagate_xla_data(tensor input, tensor output) { + PROTECT( + torch::_propagate_xla_data(*input, *output); + ) +} + +void atg__remove_batch_dim(tensor *out__, tensor self, int64_t level, int64_t batch_size, int64_t out_dim) { + PROTECT( + auto outputs__ = torch::_remove_batch_dim(*self, level, batch_size, out_dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__reshape_alias(tensor *out__, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len) { + PROTECT( + auto outputs__ = torch::_reshape_alias(*self, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__reshape_alias_copy(tensor *out__, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len) { + PROTECT( + auto outputs__ = torch::_reshape_alias_copy(*self, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__reshape_alias_copy_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len) { + PROTECT( + auto outputs__ = torch::_reshape_alias_copy_out(*out, *self, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__reshape_copy(tensor *out__, tensor self, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::_reshape_copy(*self, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__reshape_from_tensor(tensor *out__, tensor self, tensor shape) { + PROTECT( + auto outputs__ = torch::_reshape_from_tensor(*self, *shape); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__resize_output(tensor *out__, tensor self, int64_t *size_data, int size_len, int device) { + PROTECT( + auto outputs__ = torch::_resize_output(*self, torch::IntArrayRef(size_data, size_len), device_of_int(device)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__resize_output_(tensor *out__, tensor self, int64_t *size_data, int size_len, int device) { + PROTECT( + auto outputs__ = torch::_resize_output_(*self, torch::IntArrayRef(size_data, size_len), device_of_int(device)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__resize_output_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len, int device) { + PROTECT( + auto outputs__ = torch::_resize_output_out(*out, *self, torch::IntArrayRef(size_data, size_len), device_of_int(device)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__rowwise_prune(tensor *out__, tensor weight, tensor mask, int compressed_indices_dtype) { + PROTECT( + auto outputs__ = torch::_rowwise_prune(*weight, *mask, at::ScalarType(compressed_indices_dtype)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__safe_softmax(tensor *out__, tensor self, int64_t dim, int dtype) { + PROTECT( + auto outputs__ = torch::_safe_softmax(*self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sample_dirichlet(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_sample_dirichlet(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sample_dirichlet_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::_sample_dirichlet_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__saturate_weight_to_fp16(tensor *out__, tensor weight) { + PROTECT( + auto outputs__ = torch::_saturate_weight_to_fp16(*weight); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__scaled_dot_product_attention_math(tensor *out__, tensor query, tensor key, tensor value, tensor attn_mask, double dropout_p, int is_causal, tensor dropout_mask, double scale_v, uint8_t scale_null, int enable_gqa) { + PROTECT( + auto outputs__ = torch::_scaled_dot_product_attention_math(*query, *key, *value, (attn_mask ? ::std::optional(*attn_mask) : ::std::nullopt), dropout_p, (bool)is_causal, (dropout_mask ? ::std::optional(*dropout_mask) : ::std::nullopt), scale_null ? c10::nullopt : c10::optional(scale_v), (bool)enable_gqa); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__scaled_dot_product_attention_math_for_mps(tensor *out__, tensor query, tensor key, tensor value, tensor attn_mask, double dropout_p, int is_causal, tensor dropout_mask, double scale_v, uint8_t scale_null) { + PROTECT( + auto outputs__ = torch::_scaled_dot_product_attention_math_for_mps(*query, *key, *value, (attn_mask ? ::std::optional(*attn_mask) : ::std::nullopt), dropout_p, (bool)is_causal, (dropout_mask ? ::std::optional(*dropout_mask) : ::std::nullopt), scale_null ? c10::nullopt : c10::optional(scale_v)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__scaled_dot_product_cudnn_attention_backward(tensor *out__, tensor grad_out, tensor query, tensor key, tensor value, tensor out, tensor logsumexp, tensor philox_seed, tensor philox_offset, tensor attn_bias, tensor cum_seq_q, tensor cum_seq_k, int64_t max_q, int64_t max_k, double dropout_p, int is_causal, double scale_v, uint8_t scale_null) { + PROTECT( + auto outputs__ = torch::_scaled_dot_product_cudnn_attention_backward(*grad_out, *query, *key, *value, *out, *logsumexp, *philox_seed, *philox_offset, *attn_bias, *cum_seq_q, *cum_seq_k, max_q, max_k, dropout_p, (bool)is_causal, scale_null ? c10::nullopt : c10::optional(scale_v)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__scaled_dot_product_efficient_attention(tensor *out__, tensor query, tensor key, tensor value, tensor attn_bias, int compute_log_sumexp, double dropout_p, int is_causal, double scale_v, uint8_t scale_null) { + PROTECT( + auto outputs__ = torch::_scaled_dot_product_efficient_attention(*query, *key, *value, (attn_bias ? ::std::optional(*attn_bias) : ::std::nullopt), (bool)compute_log_sumexp, dropout_p, (bool)is_causal, scale_null ? c10::nullopt : c10::optional(scale_v)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg__scaled_dot_product_flash_attention_backward(tensor *out__, tensor grad_out, tensor query, tensor key, tensor value, tensor out, tensor logsumexp, tensor cum_seq_q, tensor cum_seq_k, int64_t max_q, int64_t max_k, double dropout_p, int is_causal, tensor philox_seed, tensor philox_offset, double scale_v, uint8_t scale_null) { + PROTECT( + auto outputs__ = torch::_scaled_dot_product_flash_attention_backward(*grad_out, *query, *key, *value, *out, *logsumexp, *cum_seq_q, *cum_seq_k, max_q, max_k, dropout_p, (bool)is_causal, *philox_seed, *philox_offset, scale_null ? c10::nullopt : c10::optional(scale_v)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__scaled_dot_product_flash_attention_for_cpu(tensor *out__, tensor query, tensor key, tensor value, double dropout_p, int is_causal, tensor attn_mask, double scale_v, uint8_t scale_null) { + PROTECT( + auto outputs__ = torch::_scaled_dot_product_flash_attention_for_cpu(*query, *key, *value, dropout_p, (bool)is_causal, (attn_mask ? ::std::optional(*attn_mask) : ::std::nullopt), scale_null ? c10::nullopt : c10::optional(scale_v)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__scaled_dot_product_flash_attention_for_cpu_backward(tensor *out__, tensor grad_out, tensor query, tensor key, tensor value, tensor out, tensor logsumexp, double dropout_p, int is_causal, tensor attn_mask, double scale_v, uint8_t scale_null) { + PROTECT( + auto outputs__ = torch::_scaled_dot_product_flash_attention_for_cpu_backward(*grad_out, *query, *key, *value, *out, *logsumexp, dropout_p, (bool)is_causal, (attn_mask ? ::std::optional(*attn_mask) : ::std::nullopt), scale_null ? c10::nullopt : c10::optional(scale_v)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__scaled_grouped_mm(tensor *out__, tensor self, tensor mat2, tensor scale_a, tensor scale_b, tensor offs, tensor bias, tensor scale_result, int out_dtype, int use_fast_accum) { + PROTECT( + auto outputs__ = torch::_scaled_grouped_mm(*self, *mat2, *scale_a, *scale_b, (offs ? ::std::optional(*offs) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), (scale_result ? ::std::optional(*scale_result) : ::std::nullopt), out_dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(out_dtype)), (bool)use_fast_accum); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__scaled_grouped_mm_v2(tensor *out__, tensor self, tensor mat2, tensor *scale_a_data, int scale_a_len, int64_t *recipe_a_data, int recipe_a_len, int64_t *swizzle_a_data, int swizzle_a_len, tensor *scale_b_data, int scale_b_len, int64_t *recipe_b_data, int recipe_b_len, int64_t *swizzle_b_data, int swizzle_b_len, tensor offs, tensor bias, int out_dtype, int64_t *contraction_dim_data, int contraction_dim_len, int use_fast_accum) { + PROTECT( + auto outputs__ = torch::_scaled_grouped_mm_v2(*self, *mat2, of_carray_tensor(scale_a_data, scale_a_len), torch::IntArrayRef(recipe_a_data, recipe_a_len), torch::IntArrayRef(swizzle_a_data, swizzle_a_len), of_carray_tensor(scale_b_data, scale_b_len), torch::IntArrayRef(recipe_b_data, recipe_b_len), torch::IntArrayRef(swizzle_b_data, swizzle_b_len), (offs ? ::std::optional(*offs) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), out_dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(out_dtype)), torch::IntArrayRef(contraction_dim_data, contraction_dim_len), (bool)use_fast_accum); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__scaled_mm(tensor *out__, tensor self, tensor mat2, tensor scale_a, tensor scale_b, tensor bias, tensor scale_result, int out_dtype, int use_fast_accum) { + PROTECT( + auto outputs__ = torch::_scaled_mm(*self, *mat2, *scale_a, *scale_b, (bias ? ::std::optional(*bias) : ::std::nullopt), (scale_result ? ::std::optional(*scale_result) : ::std::nullopt), out_dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(out_dtype)), (bool)use_fast_accum); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__scaled_mm_out(tensor *out__, tensor out, tensor self, tensor mat2, tensor scale_a, tensor scale_b, tensor bias, tensor scale_result, int out_dtype, int use_fast_accum) { + PROTECT( + auto outputs__ = torch::_scaled_mm_out(*out, *self, *mat2, *scale_a, *scale_b, (bias ? ::std::optional(*bias) : ::std::nullopt), (scale_result ? ::std::optional(*scale_result) : ::std::nullopt), out_dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(out_dtype)), (bool)use_fast_accum); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__scaled_mm_v2(tensor *out__, tensor self, tensor mat2, tensor *scale_a_data, int scale_a_len, int64_t *recipe_a_data, int recipe_a_len, int64_t *swizzle_a_data, int swizzle_a_len, tensor *scale_b_data, int scale_b_len, int64_t *recipe_b_data, int recipe_b_len, int64_t *swizzle_b_data, int swizzle_b_len, tensor bias, int out_dtype, int64_t *contraction_dim_data, int contraction_dim_len, int use_fast_accum) { + PROTECT( + auto outputs__ = torch::_scaled_mm_v2(*self, *mat2, of_carray_tensor(scale_a_data, scale_a_len), torch::IntArrayRef(recipe_a_data, recipe_a_len), torch::IntArrayRef(swizzle_a_data, swizzle_a_len), of_carray_tensor(scale_b_data, scale_b_len), torch::IntArrayRef(recipe_b_data, recipe_b_len), torch::IntArrayRef(swizzle_b_data, swizzle_b_len), (bias ? ::std::optional(*bias) : ::std::nullopt), out_dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(out_dtype)), torch::IntArrayRef(contraction_dim_data, contraction_dim_len), (bool)use_fast_accum); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__scaled_mm_v2_out(tensor *out__, tensor out, tensor self, tensor mat2, tensor *scale_a_data, int scale_a_len, int64_t *recipe_a_data, int recipe_a_len, int64_t *swizzle_a_data, int swizzle_a_len, tensor *scale_b_data, int scale_b_len, int64_t *recipe_b_data, int recipe_b_len, int64_t *swizzle_b_data, int swizzle_b_len, tensor bias, int out_dtype, int64_t *contraction_dim_data, int contraction_dim_len, int use_fast_accum) { + PROTECT( + auto outputs__ = torch::_scaled_mm_v2_out(*out, *self, *mat2, of_carray_tensor(scale_a_data, scale_a_len), torch::IntArrayRef(recipe_a_data, recipe_a_len), torch::IntArrayRef(swizzle_a_data, swizzle_a_len), of_carray_tensor(scale_b_data, scale_b_len), torch::IntArrayRef(recipe_b_data, recipe_b_len), torch::IntArrayRef(swizzle_b_data, swizzle_b_len), (bias ? ::std::optional(*bias) : ::std::nullopt), out_dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(out_dtype)), torch::IntArrayRef(contraction_dim_data, contraction_dim_len), (bool)use_fast_accum); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__scatter_reduce(tensor *out__, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len, int include_self) { + PROTECT( + auto outputs__ = torch::scatter_reduce(*self, dim, *index, *src, std::string(reduce_ptr, reduce_len), (bool)include_self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__scatter_reduce_(tensor *out__, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len, int include_self) { + PROTECT( + auto outputs__ = self->scatter_reduce_(dim, *index, *src, std::string(reduce_ptr, reduce_len), (bool)include_self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__scatter_reduce_two_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len, int include_self) { + PROTECT( + auto outputs__ = torch::scatter_reduce_out(*out, *self, dim, *index, *src, std::string(reduce_ptr, reduce_len), (bool)include_self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__segment_reduce_backward(tensor *out__, tensor grad, tensor output, tensor data, char* reduce_ptr, int reduce_len, tensor lengths, tensor offsets, int64_t axis, scalar initial) { + PROTECT( + auto outputs__ = torch::_segment_reduce_backward(*grad, *output, *data, std::string(reduce_ptr, reduce_len), (lengths ? ::std::optional(*lengths) : ::std::nullopt), (offsets ? ::std::optional(*offsets) : ::std::nullopt), axis, *initial); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__segment_reduce_backward_out(tensor *out__, tensor out, tensor grad, tensor output, tensor data, char* reduce_ptr, int reduce_len, tensor lengths, tensor offsets, int64_t axis, scalar initial) { + PROTECT( + auto outputs__ = torch::_segment_reduce_backward_out(*out, *grad, *output, *data, std::string(reduce_ptr, reduce_len), (lengths ? ::std::optional(*lengths) : ::std::nullopt), (offsets ? ::std::optional(*offsets) : ::std::nullopt), axis, *initial); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__shape_as_tensor(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_shape_as_tensor(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__slow_conv2d_backward(tensor *out__, tensor grad_input, tensor grad_weight, tensor grad_bias, tensor grad_output, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::_slow_conv2d_backward_out(*grad_input, *grad_weight, *grad_bias, *grad_output, *self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__sobol_engine_draw(tensor *out__, tensor quasi, int64_t n, tensor sobolstate, int64_t dimension, int64_t num_generated, int dtype) { + PROTECT( + auto outputs__ = torch::_sobol_engine_draw(*quasi, n, *sobolstate, dimension, num_generated, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__sobol_engine_ff_(tensor *out__, tensor self, int64_t n, tensor sobolstate, int64_t dimension, int64_t num_generated) { + PROTECT( + auto outputs__ = torch::_sobol_engine_ff_(*self, n, *sobolstate, dimension, num_generated); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sobol_engine_initialize_state_(tensor *out__, tensor self, int64_t dimension) { + PROTECT( + auto outputs__ = torch::_sobol_engine_initialize_state_(*self, dimension); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sobol_engine_scramble_(tensor *out__, tensor self, tensor ltm, int64_t dimension) { + PROTECT( + auto outputs__ = torch::_sobol_engine_scramble_(*self, *ltm, dimension); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__softmax(tensor *out__, tensor self, int64_t dim, int half_to_float) { + PROTECT( + auto outputs__ = torch::_softmax(*self, dim, (bool)half_to_float); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__softmax_backward_data(tensor *out__, tensor grad_output, tensor output, int64_t dim, int input_dtype) { + PROTECT( + auto outputs__ = torch::_softmax_backward_data(*grad_output, *output, dim, at::ScalarType(input_dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__softmax_backward_data_out(tensor *out__, tensor grad_input, tensor grad_output, tensor output, int64_t dim, int input_dtype) { + PROTECT( + auto outputs__ = torch::_softmax_backward_data_out(*grad_input, *grad_output, *output, dim, at::ScalarType(input_dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__softmax_out(tensor *out__, tensor out, tensor self, int64_t dim, int half_to_float) { + PROTECT( + auto outputs__ = torch::_softmax_out(*out, *self, dim, (bool)half_to_float); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_addmm(tensor *out__, tensor self, tensor mat1, tensor mat2) { + PROTECT( + auto outputs__ = torch::_sparse_addmm(*self, *mat1, *mat2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_addmm_out(tensor *out__, tensor out, tensor self, tensor mat1, tensor mat2) { + PROTECT( + auto outputs__ = torch::_sparse_addmm_out(*out, *self, *mat1, *mat2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_broadcast_to(tensor *out__, tensor self, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::_sparse_broadcast_to(*self, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_broadcast_to_copy(tensor *out__, tensor self, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::_sparse_broadcast_to_copy(*self, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_broadcast_to_copy_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::_sparse_broadcast_to_copy_out(*out, *self, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_bsc_tensor_unsafe(tensor *out__, tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::_sparse_bsc_tensor_unsafe(*ccol_indices, *row_indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_bsr_tensor_unsafe(tensor *out__, tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::_sparse_bsr_tensor_unsafe(*crow_indices, *col_indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_compressed_tensor_unsafe(tensor *out__, tensor compressed_indices, tensor plain_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::_sparse_compressed_tensor_unsafe(*compressed_indices, *plain_indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_compressed_tensor_with_dims(tensor *out__, int64_t nnz, int64_t dense_dim, int64_t *size_data, int size_len, int64_t *blocksize_data, int blocksize_len, int index_dtype, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::_sparse_compressed_tensor_with_dims(nnz, dense_dim, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(blocksize_data, blocksize_len), at::ScalarType(index_dtype), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_coo_tensor_unsafe(tensor *out__, tensor indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device, int is_coalesced) { + PROTECT( + auto outputs__ = torch::_sparse_coo_tensor_unsafe(*indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind)), (bool)is_coalesced); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_coo_tensor_with_dims(tensor *out__, int64_t sparse_dim, int64_t dense_dim, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::_sparse_coo_tensor_with_dims(sparse_dim, dense_dim, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_coo_tensor_with_dims_and_tensors(tensor *out__, int64_t sparse_dim, int64_t dense_dim, int64_t *size_data, int size_len, tensor indices, tensor values, int options_kind, int options_device, int is_coalesced) { + PROTECT( + auto outputs__ = torch::_sparse_coo_tensor_with_dims_and_tensors(sparse_dim, dense_dim, torch::IntArrayRef(size_data, size_len), *indices, *values, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind)), (bool)is_coalesced); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_coo_tensor_with_dims_and_tensors_out(tensor *out__, tensor out, int64_t sparse_dim, int64_t dense_dim, int64_t *size_data, int size_len, tensor indices, tensor values, int is_coalesced) { + PROTECT( + auto outputs__ = torch::_sparse_coo_tensor_with_dims_and_tensors_out(*out, sparse_dim, dense_dim, torch::IntArrayRef(size_data, size_len), *indices, *values, (bool)is_coalesced); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_coo_tensor_with_dims_out(tensor *out__, tensor out, int64_t sparse_dim, int64_t dense_dim, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::_sparse_coo_tensor_with_dims_out(*out, sparse_dim, dense_dim, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_csc_tensor_unsafe(tensor *out__, tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::_sparse_csc_tensor_unsafe(*ccol_indices, *row_indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_csr_prod(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { + PROTECT( + auto outputs__ = torch::_sparse_csr_prod(*self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_csr_prod_dim_dtype_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { + PROTECT( + auto outputs__ = torch::_sparse_csr_prod_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_csr_sum(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { + PROTECT( + auto outputs__ = torch::_sparse_csr_sum(*self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_csr_sum_dim_dtype_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { + PROTECT( + auto outputs__ = torch::_sparse_csr_sum_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_csr_tensor_unsafe(tensor *out__, tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::_sparse_csr_tensor_unsafe(*crow_indices, *col_indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_log_softmax(tensor *out__, tensor self, int64_t dim, int half_to_float) { + PROTECT( + auto outputs__ = torch::_sparse_log_softmax(*self, dim, (bool)half_to_float); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_log_softmax_backward_data(tensor *out__, tensor grad_output, tensor output, int64_t dim, tensor self) { + PROTECT( + auto outputs__ = torch::_sparse_log_softmax_backward_data(*grad_output, *output, dim, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_log_softmax_backward_data_out(tensor *out__, tensor out, tensor grad_output, tensor output, int64_t dim, tensor self) { + PROTECT( + auto outputs__ = torch::_sparse_log_softmax_backward_data_out(*out, *grad_output, *output, dim, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_log_softmax_int(tensor *out__, tensor self, int64_t dim, int dtype) { + PROTECT( + auto outputs__ = torch::_sparse_log_softmax(*self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_log_softmax_out(tensor *out__, tensor out, tensor self, int64_t dim, int half_to_float) { + PROTECT( + auto outputs__ = torch::_sparse_log_softmax_out(*out, *self, dim, (bool)half_to_float); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_mask_projection(tensor *out__, tensor self, tensor mask, int accumulate_matches) { + PROTECT( + auto outputs__ = self->_sparse_mask_projection(*mask, (bool)accumulate_matches); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_mask_projection_out(tensor *out__, tensor out, tensor self, tensor mask, int accumulate_matches) { + PROTECT( + auto outputs__ = torch::_sparse_mask_projection_out(*out, *self, *mask, (bool)accumulate_matches); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_mm(tensor *out__, tensor sparse, tensor dense) { + PROTECT( + auto outputs__ = torch::_sparse_mm(*sparse, *dense); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_mm_reduce(tensor *out__, tensor sparse, tensor dense, char* reduce_ptr, int reduce_len) { + PROTECT( + auto outputs__ = torch::_sparse_mm(*sparse, *dense, std::string(reduce_ptr, reduce_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_mm_reduce_impl(tensor *out__, tensor self, tensor other, char* reduce_ptr, int reduce_len) { + PROTECT( + auto outputs__ = torch::_sparse_mm_reduce_impl(*self, *other, std::string(reduce_ptr, reduce_len)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__sparse_semi_structured_apply(tensor *out__, tensor input, tensor thread_masks) { + PROTECT( + auto outputs__ = torch::_sparse_semi_structured_apply(*input, *thread_masks); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__sparse_semi_structured_apply_dense(tensor *out__, tensor input, tensor thread_masks) { + PROTECT( + auto outputs__ = torch::_sparse_semi_structured_apply_dense(*input, *thread_masks); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_semi_structured_linear(tensor *out__, tensor input, tensor weight, tensor meta, tensor bias, char* activation_ptr, int activation_len, int out_dtype) { + PROTECT( + auto outputs__ = torch::_sparse_semi_structured_linear(*input, *weight, *meta, (bias ? ::std::optional(*bias) : ::std::nullopt), std::string(activation_ptr, activation_len), out_dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(out_dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_semi_structured_mm(tensor *out__, tensor mat1, tensor mat1_meta, tensor mat2, int out_dtype) { + PROTECT( + auto outputs__ = torch::_sparse_semi_structured_mm(*mat1, *mat1_meta, *mat2, out_dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(out_dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_semi_structured_tile(tensor *out__, tensor input, char* algorithm_ptr, int algorithm_len, int use_cutlass) { + PROTECT( + auto outputs__ = torch::_sparse_semi_structured_tile(*input, std::string(algorithm_ptr, algorithm_len), (bool)use_cutlass); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + out__[4] = new torch::Tensor(std::get<4>(outputs__)); + ) +} + +void atg__sparse_softmax(tensor *out__, tensor self, int64_t dim, int half_to_float) { + PROTECT( + auto outputs__ = torch::_sparse_softmax(*self, dim, (bool)half_to_float); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_softmax_backward_data(tensor *out__, tensor grad_output, tensor output, int64_t dim, tensor self) { + PROTECT( + auto outputs__ = torch::_sparse_softmax_backward_data(*grad_output, *output, dim, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_softmax_backward_data_out(tensor *out__, tensor out, tensor grad_output, tensor output, int64_t dim, tensor self) { + PROTECT( + auto outputs__ = torch::_sparse_softmax_backward_data_out(*out, *grad_output, *output, dim, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_softmax_int(tensor *out__, tensor self, int64_t dim, int dtype) { + PROTECT( + auto outputs__ = torch::_sparse_softmax(*self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_softmax_out(tensor *out__, tensor out, tensor self, int64_t dim, int half_to_float) { + PROTECT( + auto outputs__ = torch::_sparse_softmax_out(*out, *self, dim, (bool)half_to_float); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_sparse_matmul(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::_sparse_sparse_matmul(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_sparse_matmul_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::_sparse_sparse_matmul_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_sum(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_sparse_sum(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_sum_backward(tensor *out__, tensor grad, tensor self, int64_t *dim_data, int dim_len) { + PROTECT( + auto outputs__ = torch::_sparse_sum_backward(*grad, *self, torch::IntArrayRef(dim_data, dim_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_sum_backward_out(tensor *out__, tensor out, tensor grad, tensor self, int64_t *dim_data, int dim_len) { + PROTECT( + auto outputs__ = torch::_sparse_sum_backward_out(*out, *grad, *self, torch::IntArrayRef(dim_data, dim_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_sum_dim(tensor *out__, tensor self, int64_t *dim_data, int dim_len) { + PROTECT( + auto outputs__ = torch::_sparse_sum(*self, torch::IntArrayRef(dim_data, dim_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_sum_dim_dtype(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int dtype) { + PROTECT( + auto outputs__ = torch::_sparse_sum(*self, torch::IntArrayRef(dim_data, dim_len), at::ScalarType(dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_sum_dim_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len) { + PROTECT( + auto outputs__ = torch::_sparse_sum_out(*out, *self, torch::IntArrayRef(dim_data, dim_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__sparse_sum_dtype(tensor *out__, tensor self, int dtype) { + PROTECT( + auto outputs__ = torch::_sparse_sum(*self, at::ScalarType(dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__spdiags(tensor *out__, tensor diagonals, tensor offsets, int64_t *shape_data, int shape_len, int8_t layout) { + PROTECT( + auto outputs__ = torch::_spdiags(*diagonals, *offsets, torch::IntArrayRef(shape_data, shape_len), (layout == -1 ? c10::nullopt : c10::optional(static_cast(layout)))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__spdiags_out(tensor *out__, tensor out, tensor diagonals, tensor offsets, int64_t *shape_data, int shape_len, int8_t layout) { + PROTECT( + auto outputs__ = torch::_spdiags_out(*out, *diagonals, *offsets, torch::IntArrayRef(shape_data, shape_len), (layout == -1 ? c10::nullopt : c10::optional(static_cast(layout)))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__spsolve(tensor *out__, tensor A, tensor B, int left) { + PROTECT( + auto outputs__ = torch::_spsolve(*A, *B, (bool)left); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__stack(tensor *out__, tensor *tensors_data, int tensors_len, int64_t dim) { + PROTECT( + auto outputs__ = torch::_stack(of_carray_tensor(tensors_data, tensors_len), dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__stack_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len, int64_t dim) { + PROTECT( + auto outputs__ = torch::_stack_out(*out, of_carray_tensor(tensors_data, tensors_len), dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__standard_gamma(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_standard_gamma(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__standard_gamma_grad(tensor *out__, tensor self, tensor output) { + PROTECT( + auto outputs__ = torch::_standard_gamma_grad(*self, *output); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__standard_gamma_grad_out(tensor *out__, tensor out, tensor self, tensor output) { + PROTECT( + auto outputs__ = torch::_standard_gamma_grad_out(*out, *self, *output); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__standard_gamma_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::_standard_gamma_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__test_ambiguous_defaults(tensor *out__, tensor dummy, int64_t a, int64_t b) { + PROTECT( + auto outputs__ = torch::_test_ambiguous_defaults(*dummy, a, b); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__test_ambiguous_defaults_b(tensor *out__, tensor dummy, int64_t a, char* b_ptr, int b_len) { + PROTECT( + auto outputs__ = torch::_test_ambiguous_defaults(*dummy, a, std::string(b_ptr, b_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__test_autograd_multiple_dispatch(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_test_autograd_multiple_dispatch(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__test_autograd_multiple_dispatch_fullcoverage_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::_test_autograd_multiple_dispatch_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__test_autograd_multiple_dispatch_ntonly(tensor *out__, tensor self, int b) { + PROTECT( + auto outputs__ = torch::_test_autograd_multiple_dispatch(*self, (bool)b); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__test_autograd_multiple_dispatch_view(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_test_autograd_multiple_dispatch_view(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__test_autograd_multiple_dispatch_view_copy(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_test_autograd_multiple_dispatch_view_copy(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__test_autograd_multiple_dispatch_view_copy_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::_test_autograd_multiple_dispatch_view_copy_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__test_check_tensor(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_test_check_tensor(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__test_functorch_fallback(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::_test_functorch_fallback(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__test_functorch_fallback_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::_test_functorch_fallback_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__test_optional_filled_intlist(tensor *out__, tensor values, int64_t *addends_data, int addends_len) { + PROTECT( + auto outputs__ = torch::_test_optional_filled_intlist(*values, addends_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(addends_data, addends_len))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__test_optional_filled_intlist_out(tensor *out__, tensor out, tensor values, int64_t *addends_data, int addends_len) { + PROTECT( + auto outputs__ = torch::_test_optional_filled_intlist_out(*out, *values, addends_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(addends_data, addends_len))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__test_optional_floatlist(tensor *out__, tensor values, double *addends_data, int addends_len) { + PROTECT( + auto outputs__ = torch::_test_optional_floatlist(*values, at::ArrayRef(addends_data, addends_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__test_optional_floatlist_out(tensor *out__, tensor out, tensor values, double *addends_data, int addends_len) { + PROTECT( + auto outputs__ = torch::_test_optional_floatlist_out(*out, *values, at::ArrayRef(addends_data, addends_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__test_optional_intlist(tensor *out__, tensor values, int64_t *addends_data, int addends_len) { + PROTECT( + auto outputs__ = torch::_test_optional_intlist(*values, addends_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(addends_data, addends_len))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__test_optional_intlist_out(tensor *out__, tensor out, tensor values, int64_t *addends_data, int addends_len) { + PROTECT( + auto outputs__ = torch::_test_optional_intlist_out(*out, *values, addends_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(addends_data, addends_len))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__test_parallel_materialize(tensor *out__, tensor self, int64_t num_parallel, int skip_first) { + PROTECT( + auto outputs__ = torch::_test_parallel_materialize(*self, num_parallel, (bool)skip_first); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__test_serialization_subcmul(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::_test_serialization_subcmul(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__test_string_default(tensor *out__, tensor dummy, char* a_ptr, int a_len, char* b_ptr, int b_len) { + PROTECT( + auto outputs__ = torch::_test_string_default(*dummy, std::string(a_ptr, a_len), std::string(b_ptr, b_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__test_warn_in_autograd(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_test_warn_in_autograd(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__test_warn_in_autograd_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::_test_warn_in_autograd_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__to_copy(tensor *out__, tensor self, int options_kind, int options_device, int non_blocking) { + PROTECT( + auto outputs__ = torch::_to_copy(*self, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind)), (bool)non_blocking); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__to_copy_out(tensor *out__, tensor out, tensor self, int non_blocking) { + PROTECT( + auto outputs__ = torch::_to_copy_out(*out, *self, (bool)non_blocking); + out__[0] = new torch::Tensor(outputs__); + ) +} + +tensor *atg__to_cpu(tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::_to_cpu(of_carray_tensor(tensors_data, tensors_len)); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg__to_dense(tensor *out__, tensor self, int dtype, int masked_grad) { + PROTECT( + auto outputs__ = self->_to_dense(dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype)), (bool)masked_grad); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__to_dense_out(tensor *out__, tensor out, tensor self, int dtype, int masked_grad) { + PROTECT( + auto outputs__ = torch::_to_dense_out(*out, *self, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype)), (bool)masked_grad); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__to_sparse(tensor *out__, tensor self, int8_t layout, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null) { + PROTECT( + auto outputs__ = self->_to_sparse((layout == -1 ? c10::nullopt : c10::optional(static_cast(layout))), blocksize_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(blocksize_data, blocksize_len)), dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__to_sparse_bsc(tensor *out__, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null) { + PROTECT( + auto outputs__ = self->_to_sparse_bsc(torch::IntArrayRef(blocksize_data, blocksize_len), dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__to_sparse_bsc_out(tensor *out__, tensor out, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null) { + PROTECT( + auto outputs__ = torch::_to_sparse_bsc_out(*out, *self, torch::IntArrayRef(blocksize_data, blocksize_len), dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__to_sparse_bsr(tensor *out__, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null) { + PROTECT( + auto outputs__ = self->_to_sparse_bsr(torch::IntArrayRef(blocksize_data, blocksize_len), dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__to_sparse_bsr_out(tensor *out__, tensor out, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null) { + PROTECT( + auto outputs__ = torch::_to_sparse_bsr_out(*out, *self, torch::IntArrayRef(blocksize_data, blocksize_len), dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__to_sparse_csc(tensor *out__, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null) { + PROTECT( + auto outputs__ = self->_to_sparse_csc(dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__to_sparse_csc_out(tensor *out__, tensor out, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null) { + PROTECT( + auto outputs__ = torch::_to_sparse_csc_out(*out, *self, dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__to_sparse_csr(tensor *out__, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null) { + PROTECT( + auto outputs__ = self->_to_sparse_csr(dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__to_sparse_csr_out(tensor *out__, tensor out, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null) { + PROTECT( + auto outputs__ = torch::_to_sparse_csr_out(*out, *self, dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__to_sparse_out(tensor *out__, tensor out, tensor self, int8_t layout, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null) { + PROTECT( + auto outputs__ = torch::_to_sparse_out(*out, *self, (layout == -1 ? c10::nullopt : c10::optional(static_cast(layout))), blocksize_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(blocksize_data, blocksize_len)), dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__to_sparse_semi_structured(tensor *out__, tensor dense) { + PROTECT( + auto outputs__ = torch::_to_sparse_semi_structured(*dense); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__to_sparse_sparse_dim(tensor *out__, tensor self, int64_t sparse_dim) { + PROTECT( + auto outputs__ = self->_to_sparse(sparse_dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__to_sparse_sparse_dim_out(tensor *out__, tensor out, tensor self, int64_t sparse_dim) { + PROTECT( + auto outputs__ = torch::_to_sparse_out(*out, *self, sparse_dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__transform_bias_rescale_qkv(tensor *out__, tensor qkv, tensor qkv_bias, int64_t num_heads) { + PROTECT( + auto outputs__ = torch::_transform_bias_rescale_qkv(*qkv, *qkv_bias, num_heads); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__transform_bias_rescale_qkv_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor qkv, tensor qkv_bias, int64_t num_heads) { + PROTECT( + auto outputs__ = torch::_transform_bias_rescale_qkv_out(*out0, *out1, *out2, *qkv, *qkv_bias, num_heads); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__transformer_encoder_layer_fwd(tensor *out__, tensor src, int64_t embed_dim, int64_t num_heads, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, int use_gelu, int norm_first, double eps, tensor norm_weight_1, tensor norm_bias_1, tensor norm_weight_2, tensor norm_bias_2, tensor ffn_weight_1, tensor ffn_bias_1, tensor ffn_weight_2, tensor ffn_bias_2, tensor mask, int64_t mask_type_v, uint8_t mask_type_null) { + PROTECT( + auto outputs__ = torch::_transformer_encoder_layer_fwd(*src, embed_dim, num_heads, *qkv_weight, *qkv_bias, *proj_weight, *proj_bias, (bool)use_gelu, (bool)norm_first, eps, *norm_weight_1, *norm_bias_1, *norm_weight_2, *norm_bias_2, *ffn_weight_1, *ffn_bias_1, *ffn_weight_2, *ffn_bias_2, (mask ? ::std::optional(*mask) : ::std::nullopt), mask_type_null ? c10::nullopt : c10::optional(mask_type_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__transformer_encoder_layer_fwd_out(tensor *out__, tensor out, tensor src, int64_t embed_dim, int64_t num_heads, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, int use_gelu, int norm_first, double eps, tensor norm_weight_1, tensor norm_bias_1, tensor norm_weight_2, tensor norm_bias_2, tensor ffn_weight_1, tensor ffn_bias_1, tensor ffn_weight_2, tensor ffn_bias_2, tensor mask, int64_t mask_type_v, uint8_t mask_type_null) { + PROTECT( + auto outputs__ = torch::_transformer_encoder_layer_fwd_out(*out, *src, embed_dim, num_heads, *qkv_weight, *qkv_bias, *proj_weight, *proj_bias, (bool)use_gelu, (bool)norm_first, eps, *norm_weight_1, *norm_bias_1, *norm_weight_2, *norm_bias_2, *ffn_weight_1, *ffn_bias_1, *ffn_weight_2, *ffn_bias_2, (mask ? ::std::optional(*mask) : ::std::nullopt), mask_type_null ? c10::nullopt : c10::optional(mask_type_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__trilinear(tensor *out__, tensor i1, tensor i2, tensor i3, int64_t *expand1_data, int expand1_len, int64_t *expand2_data, int expand2_len, int64_t *expand3_data, int expand3_len, int64_t *sumdim_data, int sumdim_len, int64_t unroll_dim) { + PROTECT( + auto outputs__ = torch::_trilinear(*i1, *i2, *i3, torch::IntArrayRef(expand1_data, expand1_len), torch::IntArrayRef(expand2_data, expand2_len), torch::IntArrayRef(expand3_data, expand3_len), torch::IntArrayRef(sumdim_data, sumdim_len), unroll_dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__trilinear_out(tensor *out__, tensor out, tensor i1, tensor i2, tensor i3, int64_t *expand1_data, int expand1_len, int64_t *expand2_data, int expand2_len, int64_t *expand3_data, int expand3_len, int64_t *sumdim_data, int sumdim_len, int64_t unroll_dim) { + PROTECT( + auto outputs__ = torch::_trilinear_out(*out, *i1, *i2, *i3, torch::IntArrayRef(expand1_data, expand1_len), torch::IntArrayRef(expand2_data, expand2_len), torch::IntArrayRef(expand3_data, expand3_len), torch::IntArrayRef(sumdim_data, sumdim_len), unroll_dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__triton_multi_head_attention(tensor *out__, tensor query, tensor key, tensor value, int64_t embed_dim, int64_t num_head, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, tensor mask) { + PROTECT( + auto outputs__ = torch::_triton_multi_head_attention(*query, *key, *value, embed_dim, num_head, *qkv_weight, *qkv_bias, *proj_weight, *proj_bias, (mask ? ::std::optional(*mask) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__triton_multi_head_attention_out(tensor *out__, tensor out, tensor query, tensor key, tensor value, int64_t embed_dim, int64_t num_head, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, tensor mask) { + PROTECT( + auto outputs__ = torch::_triton_multi_head_attention_out(*out, *query, *key, *value, embed_dim, num_head, *qkv_weight, *qkv_bias, *proj_weight, *proj_bias, (mask ? ::std::optional(*mask) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__triton_scaled_dot_attention(tensor *out__, tensor q, tensor k, tensor v, double dropout_p) { + PROTECT( + auto outputs__ = torch::_triton_scaled_dot_attention(*q, *k, *v, dropout_p); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__triton_scaled_dot_attention_out(tensor *out__, tensor out, tensor q, tensor k, tensor v, double dropout_p) { + PROTECT( + auto outputs__ = torch::_triton_scaled_dot_attention_out(*out, *q, *k, *v, dropout_p); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__unique(tensor *out__, tensor self, int sorted, int return_inverse) { + PROTECT( + auto outputs__ = torch::_unique(*self, (bool)sorted, (bool)return_inverse); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__unique2(tensor *out__, tensor self, int sorted, int return_inverse, int return_counts) { + PROTECT( + auto outputs__ = torch::_unique2(*self, (bool)sorted, (bool)return_inverse, (bool)return_counts); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__unique2_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor self, int sorted, int return_inverse, int return_counts) { + PROTECT( + auto outputs__ = torch::_unique2_out(*out0, *out1, *out2, *self, (bool)sorted, (bool)return_inverse, (bool)return_counts); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg__unique_out(tensor *out__, tensor out0, tensor out1, tensor self, int sorted, int return_inverse) { + PROTECT( + auto outputs__ = torch::_unique_out(*out0, *out1, *self, (bool)sorted, (bool)return_inverse); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__unpack_dual(tensor *out__, tensor dual, int64_t level) { + PROTECT( + auto outputs__ = torch::_unpack_dual(*dual, level); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__unsafe_index(tensor *out__, tensor self, tensor *indices_data, int indices_len) { + PROTECT( + auto outputs__ = torch::_unsafe_index(*self, of_carray_tensor_opt(indices_data, indices_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__unsafe_index_put(tensor *out__, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate) { + PROTECT( + auto outputs__ = torch::_unsafe_index_put(*self, of_carray_tensor_opt(indices_data, indices_len), *values, (bool)accumulate); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__unsafe_masked_index(tensor *out__, tensor self, tensor mask, tensor *indices_data, int indices_len, scalar fill) { + PROTECT( + auto outputs__ = torch::_unsafe_masked_index(*self, *mask, of_carray_tensor_opt(indices_data, indices_len), *fill); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__unsafe_masked_index_put_accumulate(tensor *out__, tensor self, tensor mask, tensor *indices_data, int indices_len, tensor values) { + PROTECT( + auto outputs__ = torch::_unsafe_masked_index_put_accumulate(*self, *mask, of_carray_tensor_opt(indices_data, indices_len), *values); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__unsafe_view(tensor *out__, tensor self, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::_unsafe_view(*self, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__unsafe_view_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::_unsafe_view_out(*out, *self, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_bicubic2d_aa(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::_upsample_bicubic2d_aa(*self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_bicubic2d_aa_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::_upsample_bicubic2d_aa_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_bicubic2d_aa_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::_upsample_bicubic2d_aa_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_bicubic2d_aa_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::_upsample_bicubic2d_aa_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_bicubic2d_aa_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len) { + PROTECT( + auto outputs__ = torch::_upsample_bicubic2d_aa(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), (bool)align_corners, at::ArrayRef(scale_factors_data, scale_factors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_bilinear2d_aa(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::_upsample_bilinear2d_aa(*self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_bilinear2d_aa_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::_upsample_bilinear2d_aa_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_bilinear2d_aa_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::_upsample_bilinear2d_aa_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_bilinear2d_aa_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::_upsample_bilinear2d_aa_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_bilinear2d_aa_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len) { + PROTECT( + auto outputs__ = torch::_upsample_bilinear2d_aa(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), (bool)align_corners, at::ArrayRef(scale_factors_data, scale_factors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_nearest_exact1d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, double scales_v, uint8_t scales_null) { + PROTECT( + auto outputs__ = torch::_upsample_nearest_exact1d(*self, torch::IntArrayRef(output_size_data, output_size_len), scales_null ? c10::nullopt : c10::optional(scales_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_nearest_exact1d_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_v, uint8_t scales_null) { + PROTECT( + auto outputs__ = torch::_upsample_nearest_exact1d_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_null ? c10::nullopt : c10::optional(scales_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_nearest_exact1d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_v, uint8_t scales_null) { + PROTECT( + auto outputs__ = torch::_upsample_nearest_exact1d_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_null ? c10::nullopt : c10::optional(scales_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_nearest_exact1d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_v, uint8_t scales_null) { + PROTECT( + auto outputs__ = torch::_upsample_nearest_exact1d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), scales_null ? c10::nullopt : c10::optional(scales_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_nearest_exact1d_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len) { + PROTECT( + auto outputs__ = torch::_upsample_nearest_exact1d(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), at::ArrayRef(scale_factors_data, scale_factors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_nearest_exact2d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::_upsample_nearest_exact2d(*self, torch::IntArrayRef(output_size_data, output_size_len), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_nearest_exact2d_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::_upsample_nearest_exact2d_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_nearest_exact2d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::_upsample_nearest_exact2d_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_nearest_exact2d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::_upsample_nearest_exact2d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_nearest_exact2d_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len) { + PROTECT( + auto outputs__ = torch::_upsample_nearest_exact2d(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), at::ArrayRef(scale_factors_data, scale_factors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_nearest_exact3d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::_upsample_nearest_exact3d(*self, torch::IntArrayRef(output_size_data, output_size_len), scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_nearest_exact3d_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::_upsample_nearest_exact3d_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_nearest_exact3d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::_upsample_nearest_exact3d_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_nearest_exact3d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::_upsample_nearest_exact3d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__upsample_nearest_exact3d_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len) { + PROTECT( + auto outputs__ = torch::_upsample_nearest_exact3d(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), at::ArrayRef(scale_factors_data, scale_factors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +int atg__use_cudnn_ctc_loss(tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank) { + PROTECT( + return torch::_use_cudnn_ctc_loss(*log_probs, *targets, torch::IntArrayRef(input_lengths_data, input_lengths_len), torch::IntArrayRef(target_lengths_data, target_lengths_len), blank); + ) + return 0; +} + +int atg__use_cudnn_ctc_loss_tensor(tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank) { + PROTECT( + return torch::_use_cudnn_ctc_loss(*log_probs, *targets, *input_lengths, *target_lengths, blank); + ) + return 0; +} + +int atg__use_cudnn_rnn_flatten_weight() { + PROTECT( + return torch::_use_cudnn_rnn_flatten_weight(); + ) + return 0; +} + +int atg__use_miopen_ctc_loss(tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank) { + PROTECT( + return torch::_use_miopen_ctc_loss(*log_probs, *targets, torch::IntArrayRef(input_lengths_data, input_lengths_len), torch::IntArrayRef(target_lengths_data, target_lengths_len), blank); + ) + return 0; +} + +int atg__use_miopen_ctc_loss_tensor(tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank) { + PROTECT( + return torch::_use_miopen_ctc_loss(*log_probs, *targets, *input_lengths, *target_lengths, blank); + ) + return 0; +} + +void atg__validate_compressed_sparse_indices(int is_crow, tensor compressed_idx, tensor plain_idx, int64_t cdim, int64_t dim, int64_t nnz) { + PROTECT( + torch::_validate_compressed_sparse_indices((bool)is_crow, *compressed_idx, *plain_idx, cdim, dim, nnz); + ) +} + +void atg__validate_sparse_bsc_tensor_args(tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int check_pinning) { + PROTECT( + torch::_validate_sparse_bsc_tensor_args(*ccol_indices, *row_indices, *values, torch::IntArrayRef(size_data, size_len), (bool)check_pinning); + ) +} + +void atg__validate_sparse_bsr_tensor_args(tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int check_pinning) { + PROTECT( + torch::_validate_sparse_bsr_tensor_args(*crow_indices, *col_indices, *values, torch::IntArrayRef(size_data, size_len), (bool)check_pinning); + ) +} + +void atg__validate_sparse_compressed_tensor_args(tensor compressed_indices, tensor plain_indices, tensor values, int64_t *size_data, int size_len, int8_t layout, int check_pinning) { + PROTECT( + torch::_validate_sparse_compressed_tensor_args(*compressed_indices, *plain_indices, *values, torch::IntArrayRef(size_data, size_len), static_cast(layout), (bool)check_pinning); + ) +} + +void atg__validate_sparse_csc_tensor_args(tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int check_pinning) { + PROTECT( + torch::_validate_sparse_csc_tensor_args(*ccol_indices, *row_indices, *values, torch::IntArrayRef(size_data, size_len), (bool)check_pinning); + ) +} + +void atg__validate_sparse_csr_tensor_args(tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int check_pinning) { + PROTECT( + torch::_validate_sparse_csr_tensor_args(*crow_indices, *col_indices, *values, torch::IntArrayRef(size_data, size_len), (bool)check_pinning); + ) +} + +void atg__values(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->_values(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__values_copy(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::_values_copy(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__values_copy_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::_values_copy_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +int64_t atg__version(tensor self) { + PROTECT( + return self->_version(); + ) + return 0; +} + +void atg__weight_int4pack_mm(tensor *out__, tensor self, tensor mat2, int64_t qGroupSize, tensor qScaleAndZeros) { + PROTECT( + auto outputs__ = torch::_weight_int4pack_mm(*self, *mat2, qGroupSize, *qScaleAndZeros); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__weight_int4pack_mm_for_cpu(tensor *out__, tensor self, tensor mat2, int64_t qGroupSize, tensor qScaleAndZeros) { + PROTECT( + auto outputs__ = torch::_weight_int4pack_mm_for_cpu(*self, *mat2, qGroupSize, *qScaleAndZeros); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__weight_int4pack_mm_with_scales_and_zeros(tensor *out__, tensor self, tensor mat2, int64_t qGroupSize, tensor qScale, tensor qZeros) { + PROTECT( + auto outputs__ = torch::_weight_int4pack_mm_with_scales_and_zeros(*self, *mat2, qGroupSize, *qScale, *qZeros); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__weight_int8pack_mm(tensor *out__, tensor self, tensor mat2, tensor scales) { + PROTECT( + auto outputs__ = torch::_weight_int8pack_mm(*self, *mat2, *scales); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__weight_norm(tensor *out__, tensor v, tensor g, int64_t dim) { + PROTECT( + auto outputs__ = torch::_weight_norm(*v, *g, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__weight_norm_differentiable_backward(tensor *out__, tensor grad_w, tensor saved_v, tensor saved_g, tensor saved_norms, int64_t dim) { + PROTECT( + auto outputs__ = torch::_weight_norm_differentiable_backward(*grad_w, *saved_v, *saved_g, *saved_norms, dim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__weight_norm_interface(tensor *out__, tensor v, tensor g, int64_t dim) { + PROTECT( + auto outputs__ = torch::_weight_norm_interface(*v, *g, dim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__weight_norm_interface_backward(tensor *out__, tensor grad_w, tensor saved_v, tensor saved_g, tensor saved_norms, int64_t dim) { + PROTECT( + auto outputs__ = torch::_weight_norm_interface_backward(*grad_w, *saved_v, *saved_g, *saved_norms, dim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__weight_norm_interface_backward_out(tensor *out__, tensor out0, tensor out1, tensor grad_w, tensor saved_v, tensor saved_g, tensor saved_norms, int64_t dim) { + PROTECT( + auto outputs__ = torch::_weight_norm_interface_backward_out(*out0, *out1, *grad_w, *saved_v, *saved_g, *saved_norms, dim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__weight_norm_interface_out(tensor *out__, tensor out0, tensor out1, tensor v, tensor g, int64_t dim) { + PROTECT( + auto outputs__ = torch::_weight_norm_interface_out(*out0, *out1, *v, *g, dim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg__wrapped_linear_prepack(tensor *out__, tensor weight, tensor weight_scale, tensor weight_zero_point, tensor bias) { + PROTECT( + auto outputs__ = torch::_wrapped_linear_prepack(*weight, *weight_scale, *weight_zero_point, *bias); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg__wrapped_quantized_linear_prepacked(tensor *out__, tensor input, tensor input_scale, tensor input_zero_point, tensor packed_weight, tensor output_scale, tensor output_zero_point, int64_t out_channel) { + PROTECT( + auto outputs__ = torch::_wrapped_quantized_linear_prepacked(*input, *input_scale, *input_zero_point, *packed_weight, *output_scale, *output_zero_point, out_channel); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_abs(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::abs(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_abs_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::abs_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_abs_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::abs_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_absolute(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::absolute(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_absolute_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->absolute_(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_absolute_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::absolute_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_acos(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::acos(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_acos_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::acos_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_acos_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::acos_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_acosh(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::acosh(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_acosh_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::acosh_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_acosh_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::acosh_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_adaptive_avg_pool1d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len) { + PROTECT( + auto outputs__ = torch::adaptive_avg_pool1d(*self, torch::IntArrayRef(output_size_data, output_size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_adaptive_avg_pool1d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len) { + PROTECT( + auto outputs__ = torch::adaptive_avg_pool1d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_adaptive_avg_pool2d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len) { + PROTECT( + auto outputs__ = torch::adaptive_avg_pool2d(*self, torch::IntArrayRef(output_size_data, output_size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_adaptive_avg_pool2d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len) { + PROTECT( + auto outputs__ = torch::adaptive_avg_pool2d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_adaptive_avg_pool3d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len) { + PROTECT( + auto outputs__ = torch::adaptive_avg_pool3d(*self, torch::IntArrayRef(output_size_data, output_size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_adaptive_avg_pool3d_backward(tensor *out__, tensor grad_input, tensor grad_output, tensor self) { + PROTECT( + auto outputs__ = torch::adaptive_avg_pool3d_backward_out(*grad_input, *grad_output, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_adaptive_avg_pool3d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len) { + PROTECT( + auto outputs__ = torch::adaptive_avg_pool3d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_adaptive_max_pool1d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len) { + PROTECT( + auto outputs__ = torch::adaptive_max_pool1d(*self, torch::IntArrayRef(output_size_data, output_size_len)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_adaptive_max_pool2d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len) { + PROTECT( + auto outputs__ = torch::adaptive_max_pool2d(*self, torch::IntArrayRef(output_size_data, output_size_len)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_adaptive_max_pool2d_backward(tensor *out__, tensor grad_output, tensor self, tensor indices) { + PROTECT( + auto outputs__ = torch::adaptive_max_pool2d_backward(*grad_output, *self, *indices); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_adaptive_max_pool2d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor indices) { + PROTECT( + auto outputs__ = torch::adaptive_max_pool2d_backward_out(*grad_input, *grad_output, *self, *indices); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_adaptive_max_pool2d_out(tensor *out__, tensor out, tensor indices, tensor self, int64_t *output_size_data, int output_size_len) { + PROTECT( + auto outputs__ = torch::adaptive_max_pool2d_out(*out, *indices, *self, torch::IntArrayRef(output_size_data, output_size_len)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_adaptive_max_pool3d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len) { + PROTECT( + auto outputs__ = torch::adaptive_max_pool3d(*self, torch::IntArrayRef(output_size_data, output_size_len)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_adaptive_max_pool3d_backward(tensor *out__, tensor grad_output, tensor self, tensor indices) { + PROTECT( + auto outputs__ = torch::adaptive_max_pool3d_backward(*grad_output, *self, *indices); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_adaptive_max_pool3d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor indices) { + PROTECT( + auto outputs__ = torch::adaptive_max_pool3d_backward_out(*grad_input, *grad_output, *self, *indices); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_adaptive_max_pool3d_out(tensor *out__, tensor out, tensor indices, tensor self, int64_t *output_size_data, int output_size_len) { + PROTECT( + auto outputs__ = torch::adaptive_max_pool3d_out(*out, *indices, *self, torch::IntArrayRef(output_size_data, output_size_len)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_add(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::add(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_add_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->add_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_add_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::add_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_add_scalar(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::add(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_add_scalar_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->add_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_add_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::add_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_addbmm(tensor *out__, tensor self, tensor batch1, tensor batch2) { + PROTECT( + auto outputs__ = torch::addbmm(*self, *batch1, *batch2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_addbmm_(tensor *out__, tensor self, tensor batch1, tensor batch2) { + PROTECT( + auto outputs__ = self->addbmm_(*batch1, *batch2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_addbmm_out(tensor *out__, tensor out, tensor self, tensor batch1, tensor batch2) { + PROTECT( + auto outputs__ = torch::addbmm_out(*out, *self, *batch1, *batch2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_addcdiv(tensor *out__, tensor self, tensor tensor1, tensor tensor2) { + PROTECT( + auto outputs__ = torch::addcdiv(*self, *tensor1, *tensor2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_addcdiv_(tensor *out__, tensor self, tensor tensor1, tensor tensor2) { + PROTECT( + auto outputs__ = self->addcdiv_(*tensor1, *tensor2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_addcdiv_out(tensor *out__, tensor out, tensor self, tensor tensor1, tensor tensor2) { + PROTECT( + auto outputs__ = torch::addcdiv_out(*out, *self, *tensor1, *tensor2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_addcmul(tensor *out__, tensor self, tensor tensor1, tensor tensor2) { + PROTECT( + auto outputs__ = torch::addcmul(*self, *tensor1, *tensor2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_addcmul_(tensor *out__, tensor self, tensor tensor1, tensor tensor2) { + PROTECT( + auto outputs__ = self->addcmul_(*tensor1, *tensor2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_addcmul_out(tensor *out__, tensor out, tensor self, tensor tensor1, tensor tensor2) { + PROTECT( + auto outputs__ = torch::addcmul_out(*out, *self, *tensor1, *tensor2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_addmm(tensor *out__, tensor self, tensor mat1, tensor mat2) { + PROTECT( + auto outputs__ = torch::addmm(*self, *mat1, *mat2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_addmm_(tensor *out__, tensor self, tensor mat1, tensor mat2) { + PROTECT( + auto outputs__ = self->addmm_(*mat1, *mat2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_addmm_dtype(tensor *out__, tensor self, tensor mat1, tensor mat2, int out_dtype) { + PROTECT( + auto outputs__ = torch::addmm(*self, *mat1, *mat2, at::ScalarType(out_dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_addmm_dtype_out(tensor *out__, tensor out, tensor self, tensor mat1, tensor mat2, int out_dtype) { + PROTECT( + auto outputs__ = torch::addmm_out(*out, *self, *mat1, *mat2, at::ScalarType(out_dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_addmm_out(tensor *out__, tensor out, tensor self, tensor mat1, tensor mat2) { + PROTECT( + auto outputs__ = torch::addmm_out(*out, *self, *mat1, *mat2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_addmv(tensor *out__, tensor self, tensor mat, tensor vec) { + PROTECT( + auto outputs__ = torch::addmv(*self, *mat, *vec); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_addmv_(tensor *out__, tensor self, tensor mat, tensor vec) { + PROTECT( + auto outputs__ = torch::addmv_(*self, *mat, *vec); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_addmv_out(tensor *out__, tensor out, tensor self, tensor mat, tensor vec) { + PROTECT( + auto outputs__ = torch::addmv_out(*out, *self, *mat, *vec); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_addr(tensor *out__, tensor self, tensor vec1, tensor vec2) { + PROTECT( + auto outputs__ = torch::addr(*self, *vec1, *vec2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_addr_(tensor *out__, tensor self, tensor vec1, tensor vec2) { + PROTECT( + auto outputs__ = self->addr_(*vec1, *vec2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_addr_out(tensor *out__, tensor out, tensor self, tensor vec1, tensor vec2) { + PROTECT( + auto outputs__ = torch::addr_out(*out, *self, *vec1, *vec2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_adjoint(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::adjoint(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_affine_grid_generator(tensor *out__, tensor theta, int64_t *size_data, int size_len, int align_corners) { + PROTECT( + auto outputs__ = torch::affine_grid_generator(*theta, torch::IntArrayRef(size_data, size_len), (bool)align_corners); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_affine_grid_generator_backward(tensor *out__, tensor grad, int64_t *size_data, int size_len, int align_corners) { + PROTECT( + auto outputs__ = torch::affine_grid_generator_backward(*grad, torch::IntArrayRef(size_data, size_len), (bool)align_corners); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_affine_grid_generator_out(tensor *out__, tensor out, tensor theta, int64_t *size_data, int size_len, int align_corners) { + PROTECT( + auto outputs__ = torch::affine_grid_generator_out(*out, *theta, torch::IntArrayRef(size_data, size_len), (bool)align_corners); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_alias(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::alias(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_alias_copy(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::alias_copy(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_alias_copy_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::alias_copy_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_align_as(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->align_as(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +tensor *atg_align_tensors(tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::align_tensors(of_carray_tensor(tensors_data, tensors_len)); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg_all(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::all(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_all_all_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::all_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_all_dim(tensor *out__, tensor self, int64_t dim, int keepdim) { + PROTECT( + auto outputs__ = torch::all(*self, dim, (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_all_dims(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim) { + PROTECT( + auto outputs__ = torch::all(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_all_dims_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim) { + PROTECT( + auto outputs__ = torch::all_out(*out, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_all_out(tensor *out__, tensor out, tensor self, int64_t dim, int keepdim) { + PROTECT( + auto outputs__ = torch::all_out(*out, *self, dim, (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +int atg_allclose(tensor self, tensor other, double rtol, double atol, int equal_nan) { + PROTECT( + return torch::allclose(*self, *other, rtol, atol, (bool)equal_nan); + ) + return 0; +} + +void atg_alpha_dropout(tensor *out__, tensor input, double p, int train) { + PROTECT( + auto outputs__ = torch::alpha_dropout(*input, p, (bool)train); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_alpha_dropout_(tensor *out__, tensor self, double p, int train) { + PROTECT( + auto outputs__ = torch::alpha_dropout_(*self, p, (bool)train); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_amax(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim) { + PROTECT( + auto outputs__ = torch::amax(*self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_amax_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim) { + PROTECT( + auto outputs__ = torch::amax_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_amin(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim) { + PROTECT( + auto outputs__ = torch::amin(*self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_amin_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim) { + PROTECT( + auto outputs__ = torch::amin_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_aminmax(tensor *out__, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim) { + PROTECT( + auto outputs__ = torch::aminmax(*self, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_aminmax_out(tensor *out__, tensor min, tensor max, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim) { + PROTECT( + auto outputs__ = torch::aminmax_out(*min, *max, *self, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_angle(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::angle(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_angle_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::angle_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_any(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::any(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_any_all_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::any_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_any_dim(tensor *out__, tensor self, int64_t dim, int keepdim) { + PROTECT( + auto outputs__ = torch::any(*self, dim, (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_any_dims(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim) { + PROTECT( + auto outputs__ = torch::any(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_any_dims_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim) { + PROTECT( + auto outputs__ = torch::any_out(*out, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_any_out(tensor *out__, tensor out, tensor self, int64_t dim, int keepdim) { + PROTECT( + auto outputs__ = torch::any_out(*out, *self, dim, (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arange(tensor *out__, scalar end, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::arange(*end, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arange_start(tensor *out__, scalar start, scalar end, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::arange(*start, *end, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arange_start_step(tensor *out__, scalar start, scalar end, scalar step, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::arange(*start, *end, *step, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arccos(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::arccos(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arccos_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::arccos_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arccos_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::arccos_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arccosh(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::arccosh(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arccosh_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::arccosh_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arccosh_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::arccosh_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arcsin(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::arcsin(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arcsin_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::arcsin_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arcsin_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::arcsin_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arcsinh(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::arcsinh(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arcsinh_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::arcsinh_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arcsinh_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::arcsinh_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arctan(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::arctan(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arctan2(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::arctan2(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arctan2_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->arctan2_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arctan2_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::arctan2_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arctan_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::arctan_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arctan_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::arctan_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arctanh(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::arctanh(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arctanh_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::arctanh_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_arctanh_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::arctanh_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_argmax(tensor *out__, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim) { + PROTECT( + auto outputs__ = torch::argmax(*self, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_argmax_out(tensor *out__, tensor out, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim) { + PROTECT( + auto outputs__ = torch::argmax_out(*out, *self, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_argmin(tensor *out__, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim) { + PROTECT( + auto outputs__ = torch::argmin(*self, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_argmin_out(tensor *out__, tensor out, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim) { + PROTECT( + auto outputs__ = torch::argmin_out(*out, *self, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_argsort(tensor *out__, tensor self, int64_t dim, int descending) { + PROTECT( + auto outputs__ = torch::argsort(*self, dim, (bool)descending); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_argsort_stable(tensor *out__, tensor self, int stable, int64_t dim, int descending) { + PROTECT( + auto outputs__ = torch::argsort(*self, (bool)stable, dim, (bool)descending); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_argsort_stable_out(tensor *out__, tensor out, tensor self, int stable, int64_t dim, int descending) { + PROTECT( + auto outputs__ = torch::argsort_out(*out, *self, (bool)stable, dim, (bool)descending); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_argwhere(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::argwhere(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_as_strided(tensor *out__, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null) { + PROTECT( + auto outputs__ = torch::as_strided(*self, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len), storage_offset_null ? c10::nullopt : c10::optional(storage_offset_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_as_strided_(tensor *out__, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null) { + PROTECT( + auto outputs__ = torch::as_strided_(*self, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len), storage_offset_null ? c10::nullopt : c10::optional(storage_offset_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_as_strided_copy(tensor *out__, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null) { + PROTECT( + auto outputs__ = torch::as_strided_copy(*self, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len), storage_offset_null ? c10::nullopt : c10::optional(storage_offset_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_as_strided_copy_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null) { + PROTECT( + auto outputs__ = torch::as_strided_copy_out(*out, *self, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len), storage_offset_null ? c10::nullopt : c10::optional(storage_offset_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_as_strided_scatter(tensor *out__, tensor self, tensor src, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null) { + PROTECT( + auto outputs__ = torch::as_strided_scatter(*self, *src, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len), storage_offset_null ? c10::nullopt : c10::optional(storage_offset_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_as_strided_scatter_out(tensor *out__, tensor out, tensor self, tensor src, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null) { + PROTECT( + auto outputs__ = torch::as_strided_scatter_out(*out, *self, *src, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len), storage_offset_null ? c10::nullopt : c10::optional(storage_offset_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_asin(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::asin(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_asin_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::asin_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_asin_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::asin_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_asinh(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::asinh(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_asinh_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::asinh_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_asinh_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::asinh_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_atan(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::atan(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_atan2(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::atan2(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_atan2_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->atan2_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_atan2_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::atan2_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_atan_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::atan_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_atan_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::atan_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_atanh(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::atanh(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_atanh_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::atanh_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_atanh_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::atanh_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_atleast_1d(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::atleast_1d(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +tensor *atg_atleast_1d_sequence(tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::atleast_1d(of_carray_tensor(tensors_data, tensors_len)); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg_atleast_2d(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::atleast_2d(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +tensor *atg_atleast_2d_sequence(tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::atleast_2d(of_carray_tensor(tensors_data, tensors_len)); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg_atleast_3d(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::atleast_3d(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +tensor *atg_atleast_3d_sequence(tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::atleast_3d(of_carray_tensor(tensors_data, tensors_len)); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg_avg_pool1d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad) { + PROTECT( + auto outputs__ = torch::avg_pool1d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), (bool)ceil_mode, (bool)count_include_pad); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_avg_pool1d_out(tensor *out__, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad) { + PROTECT( + auto outputs__ = torch::avg_pool1d_out(*out, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), (bool)ceil_mode, (bool)count_include_pad); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_avg_pool2d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null) { + PROTECT( + auto outputs__ = torch::avg_pool2d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), (bool)ceil_mode, (bool)count_include_pad, divisor_override_null ? c10::nullopt : c10::optional(divisor_override_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_avg_pool2d_backward(tensor *out__, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null) { + PROTECT( + auto outputs__ = torch::avg_pool2d_backward(*grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), (bool)ceil_mode, (bool)count_include_pad, divisor_override_null ? c10::nullopt : c10::optional(divisor_override_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_avg_pool2d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null) { + PROTECT( + auto outputs__ = torch::avg_pool2d_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), (bool)ceil_mode, (bool)count_include_pad, divisor_override_null ? c10::nullopt : c10::optional(divisor_override_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_avg_pool2d_out(tensor *out__, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null) { + PROTECT( + auto outputs__ = torch::avg_pool2d_out(*out, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), (bool)ceil_mode, (bool)count_include_pad, divisor_override_null ? c10::nullopt : c10::optional(divisor_override_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_avg_pool3d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null) { + PROTECT( + auto outputs__ = torch::avg_pool3d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), (bool)ceil_mode, (bool)count_include_pad, divisor_override_null ? c10::nullopt : c10::optional(divisor_override_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_avg_pool3d_backward(tensor *out__, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null) { + PROTECT( + auto outputs__ = torch::avg_pool3d_backward(*grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), (bool)ceil_mode, (bool)count_include_pad, divisor_override_null ? c10::nullopt : c10::optional(divisor_override_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_avg_pool3d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null) { + PROTECT( + auto outputs__ = torch::avg_pool3d_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), (bool)ceil_mode, (bool)count_include_pad, divisor_override_null ? c10::nullopt : c10::optional(divisor_override_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_avg_pool3d_out(tensor *out__, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null) { + PROTECT( + auto outputs__ = torch::avg_pool3d_out(*out, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), (bool)ceil_mode, (bool)count_include_pad, divisor_override_null ? c10::nullopt : c10::optional(divisor_override_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_baddbmm(tensor *out__, tensor self, tensor batch1, tensor batch2, scalar beta, scalar alpha) { + PROTECT( + auto outputs__ = torch::baddbmm(*self, *batch1, *batch2, *beta, *alpha); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_baddbmm_(tensor *out__, tensor self, tensor batch1, tensor batch2) { + PROTECT( + auto outputs__ = self->baddbmm_(*batch1, *batch2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_baddbmm_dtype(tensor *out__, tensor self, tensor batch1, tensor batch2, int out_dtype, scalar beta, scalar alpha) { + PROTECT( + auto outputs__ = torch::baddbmm(*self, *batch1, *batch2, at::ScalarType(out_dtype), *beta, *alpha); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_baddbmm_dtype_out(tensor *out__, tensor out, tensor self, tensor batch1, tensor batch2, int out_dtype) { + PROTECT( + auto outputs__ = torch::baddbmm_out(*out, *self, *batch1, *batch2, at::ScalarType(out_dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_baddbmm_out(tensor *out__, tensor out, tensor self, tensor batch1, tensor batch2) { + PROTECT( + auto outputs__ = torch::baddbmm_out(*out, *self, *batch1, *batch2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bartlett_window(tensor *out__, int64_t window_length, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::bartlett_window(window_length, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bartlett_window_out(tensor *out__, tensor out, int64_t window_length) { + PROTECT( + auto outputs__ = torch::bartlett_window_out(*out, window_length); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bartlett_window_periodic(tensor *out__, int64_t window_length, int periodic, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::bartlett_window(window_length, (bool)periodic, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bartlett_window_periodic_out(tensor *out__, tensor out, int64_t window_length, int periodic) { + PROTECT( + auto outputs__ = torch::bartlett_window_out(*out, window_length, (bool)periodic); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_batch_norm(tensor *out__, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps, int cudnn_enabled) { + PROTECT( + auto outputs__ = torch::batch_norm(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (bool)training, momentum, eps, (bool)cudnn_enabled); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_batch_norm_backward_elemt(tensor *out__, tensor grad_out, tensor input, tensor mean, tensor invstd, tensor weight, tensor sum_dy, tensor sum_dy_xmu, tensor count) { + PROTECT( + auto outputs__ = torch::batch_norm_backward_elemt(*grad_out, *input, *mean, *invstd, (weight ? ::std::optional(*weight) : ::std::nullopt), *sum_dy, *sum_dy_xmu, *count); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_batch_norm_backward_elemt_out(tensor *out__, tensor out, tensor grad_out, tensor input, tensor mean, tensor invstd, tensor weight, tensor sum_dy, tensor sum_dy_xmu, tensor count) { + PROTECT( + auto outputs__ = torch::batch_norm_backward_elemt_out(*out, *grad_out, *input, *mean, *invstd, (weight ? ::std::optional(*weight) : ::std::nullopt), *sum_dy, *sum_dy_xmu, *count); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_batch_norm_backward_reduce(tensor *out__, tensor grad_out, tensor input, tensor mean, tensor invstd, tensor weight, int input_g, int weight_g, int bias_g) { + PROTECT( + auto outputs__ = torch::batch_norm_backward_reduce(*grad_out, *input, *mean, *invstd, (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)input_g, (bool)weight_g, (bool)bias_g); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg_batch_norm_backward_reduce_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor out3, tensor grad_out, tensor input, tensor mean, tensor invstd, tensor weight, int input_g, int weight_g, int bias_g) { + PROTECT( + auto outputs__ = torch::batch_norm_backward_reduce_out(*out0, *out1, *out2, *out3, *grad_out, *input, *mean, *invstd, (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)input_g, (bool)weight_g, (bool)bias_g); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg_batch_norm_elemt(tensor *out__, tensor input, tensor weight, tensor bias, tensor mean, tensor invstd, double eps) { + PROTECT( + auto outputs__ = torch::batch_norm_elemt(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *mean, *invstd, eps); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_batch_norm_elemt_out(tensor *out__, tensor out, tensor input, tensor weight, tensor bias, tensor mean, tensor invstd, double eps) { + PROTECT( + auto outputs__ = torch::batch_norm_elemt_out(*out, *input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *mean, *invstd, eps); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_batch_norm_gather_stats(tensor *out__, tensor input, tensor mean, tensor invstd, tensor running_mean, tensor running_var, double momentum, double eps, int64_t count) { + PROTECT( + auto outputs__ = torch::batch_norm_gather_stats(*input, *mean, *invstd, (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), momentum, eps, count); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_batch_norm_gather_stats_out(tensor *out__, tensor out0, tensor out1, tensor input, tensor mean, tensor invstd, tensor running_mean, tensor running_var, double momentum, double eps, int64_t count) { + PROTECT( + auto outputs__ = torch::batch_norm_gather_stats_out(*out0, *out1, *input, *mean, *invstd, (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), momentum, eps, count); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_batch_norm_gather_stats_with_counts(tensor *out__, tensor input, tensor mean, tensor invstd, tensor running_mean, tensor running_var, double momentum, double eps, tensor counts) { + PROTECT( + auto outputs__ = torch::batch_norm_gather_stats_with_counts(*input, *mean, *invstd, (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), momentum, eps, *counts); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_batch_norm_gather_stats_with_counts_out(tensor *out__, tensor out0, tensor out1, tensor input, tensor mean, tensor invstd, tensor running_mean, tensor running_var, double momentum, double eps, tensor counts) { + PROTECT( + auto outputs__ = torch::batch_norm_gather_stats_with_counts_out(*out0, *out1, *input, *mean, *invstd, (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), momentum, eps, *counts); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_batch_norm_stats(tensor *out__, tensor input, double eps) { + PROTECT( + auto outputs__ = torch::batch_norm_stats(*input, eps); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_batch_norm_stats_out(tensor *out__, tensor out0, tensor out1, tensor input, double eps) { + PROTECT( + auto outputs__ = torch::batch_norm_stats_out(*out0, *out1, *input, eps); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_batch_norm_update_stats(tensor *out__, tensor input, tensor running_mean, tensor running_var, double momentum) { + PROTECT( + auto outputs__ = torch::batch_norm_update_stats(*input, (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), momentum); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_batch_norm_update_stats_out(tensor *out__, tensor out0, tensor out1, tensor input, tensor running_mean, tensor running_var, double momentum) { + PROTECT( + auto outputs__ = torch::batch_norm_update_stats_out(*out0, *out1, *input, (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), momentum); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_bernoulli(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::bernoulli(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bernoulli_(tensor *out__, tensor self, tensor p) { + PROTECT( + auto outputs__ = self->bernoulli_(*p); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bernoulli_float_(tensor *out__, tensor self, double p) { + PROTECT( + auto outputs__ = self->bernoulli_(p); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bernoulli_p(tensor *out__, tensor self, double p) { + PROTECT( + auto outputs__ = torch::bernoulli(*self, p); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bernoulli_tensor(tensor *out__, tensor self, tensor p) { + PROTECT( + auto outputs__ = torch::bernoulli(*self, *p); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bilinear(tensor *out__, tensor input1, tensor input2, tensor weight, tensor bias) { + PROTECT( + auto outputs__ = torch::bilinear(*input1, *input2, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_binary_cross_entropy(tensor *out__, tensor self, tensor target, tensor weight, int64_t reduction) { + PROTECT( + auto outputs__ = torch::binary_cross_entropy(*self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_binary_cross_entropy_backward(tensor *out__, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction) { + PROTECT( + auto outputs__ = torch::binary_cross_entropy_backward(*grad_output, *self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_binary_cross_entropy_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction) { + PROTECT( + auto outputs__ = torch::binary_cross_entropy_backward_out(*grad_input, *grad_output, *self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_binary_cross_entropy_out(tensor *out__, tensor out, tensor self, tensor target, tensor weight, int64_t reduction) { + PROTECT( + auto outputs__ = torch::binary_cross_entropy_out(*out, *self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_binary_cross_entropy_with_logits(tensor *out__, tensor self, tensor target, tensor weight, tensor pos_weight, int64_t reduction) { + PROTECT( + auto outputs__ = torch::binary_cross_entropy_with_logits(*self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), (pos_weight ? ::std::optional(*pos_weight) : ::std::nullopt), reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_binary_cross_entropy_with_logits_out(tensor *out__, tensor out, tensor self, tensor target, tensor weight, tensor pos_weight, int64_t reduction) { + PROTECT( + auto outputs__ = torch::binary_cross_entropy_with_logits_out(*out, *self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), (pos_weight ? ::std::optional(*pos_weight) : ::std::nullopt), reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bincount(tensor *out__, tensor self, tensor weights, int64_t minlength) { + PROTECT( + auto outputs__ = torch::bincount(*self, (weights ? ::std::optional(*weights) : ::std::nullopt), minlength); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bincount_out(tensor *out__, tensor out, tensor self, tensor weights, int64_t minlength) { + PROTECT( + auto outputs__ = torch::bincount_out(*out, *self, (weights ? ::std::optional(*weights) : ::std::nullopt), minlength); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_binomial(tensor *out__, tensor count, tensor prob) { + PROTECT( + auto outputs__ = torch::binomial(*count, *prob); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_binomial_out(tensor *out__, tensor out, tensor count, tensor prob) { + PROTECT( + auto outputs__ = torch::binomial_out(*out, *count, *prob); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_and(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::bitwise_and(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_and_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->bitwise_and_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_and_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::bitwise_and_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_and_scalar_tensor(tensor *out__, scalar self_scalar, tensor other) { + PROTECT( + auto outputs__ = torch::bitwise_and(*self_scalar, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_and_scalar_tensor_out(tensor *out__, tensor out, scalar self_scalar, tensor other) { + PROTECT( + auto outputs__ = torch::bitwise_and_out(*out, *self_scalar, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_and_tensor(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::bitwise_and(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_and_tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->bitwise_and_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_and_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::bitwise_and_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_left_shift(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::bitwise_left_shift(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_left_shift_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->bitwise_left_shift_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_left_shift_scalar_tensor(tensor *out__, scalar self_scalar, tensor other) { + PROTECT( + auto outputs__ = torch::bitwise_left_shift(*self_scalar, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_left_shift_scalar_tensor_out(tensor *out__, tensor out, scalar self_scalar, tensor other) { + PROTECT( + auto outputs__ = torch::bitwise_left_shift_out(*out, *self_scalar, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_left_shift_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::bitwise_left_shift_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_left_shift_tensor_scalar(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::bitwise_left_shift(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_left_shift_tensor_scalar_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->bitwise_left_shift_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_left_shift_tensor_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::bitwise_left_shift_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_not(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::bitwise_not(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_not_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->bitwise_not_(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_not_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::bitwise_not_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_or(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::bitwise_or(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_or_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->bitwise_or_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_or_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::bitwise_or_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_or_scalar_tensor(tensor *out__, scalar self_scalar, tensor other) { + PROTECT( + auto outputs__ = torch::bitwise_or(*self_scalar, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_or_scalar_tensor_out(tensor *out__, tensor out, scalar self_scalar, tensor other) { + PROTECT( + auto outputs__ = torch::bitwise_or_out(*out, *self_scalar, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_or_tensor(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::bitwise_or(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_or_tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->bitwise_or_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_or_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::bitwise_or_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_right_shift(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::bitwise_right_shift(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_right_shift_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->bitwise_right_shift_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_right_shift_scalar_tensor(tensor *out__, scalar self_scalar, tensor other) { + PROTECT( + auto outputs__ = torch::bitwise_right_shift(*self_scalar, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_right_shift_scalar_tensor_out(tensor *out__, tensor out, scalar self_scalar, tensor other) { + PROTECT( + auto outputs__ = torch::bitwise_right_shift_out(*out, *self_scalar, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_right_shift_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::bitwise_right_shift_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_right_shift_tensor_scalar(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::bitwise_right_shift(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_right_shift_tensor_scalar_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->bitwise_right_shift_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_right_shift_tensor_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::bitwise_right_shift_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_xor(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::bitwise_xor(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_xor_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->bitwise_xor_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_xor_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::bitwise_xor_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_xor_scalar_tensor(tensor *out__, scalar self_scalar, tensor other) { + PROTECT( + auto outputs__ = torch::bitwise_xor(*self_scalar, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_xor_scalar_tensor_out(tensor *out__, tensor out, scalar self_scalar, tensor other) { + PROTECT( + auto outputs__ = torch::bitwise_xor_out(*out, *self_scalar, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_xor_tensor(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::bitwise_xor(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_xor_tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->bitwise_xor_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bitwise_xor_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::bitwise_xor_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_blackman_window(tensor *out__, int64_t window_length, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::blackman_window(window_length, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_blackman_window_out(tensor *out__, tensor out, int64_t window_length) { + PROTECT( + auto outputs__ = torch::blackman_window_out(*out, window_length); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_blackman_window_periodic(tensor *out__, int64_t window_length, int periodic, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::blackman_window(window_length, (bool)periodic, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_blackman_window_periodic_out(tensor *out__, tensor out, int64_t window_length, int periodic) { + PROTECT( + auto outputs__ = torch::blackman_window_out(*out, window_length, (bool)periodic); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_block_diag(tensor *out__, tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::block_diag(of_carray_tensor(tensors_data, tensors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_block_diag_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::block_diag_out(*out, of_carray_tensor(tensors_data, tensors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bmm(tensor *out__, tensor self, tensor mat2) { + PROTECT( + auto outputs__ = torch::bmm(*self, *mat2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bmm_dtype(tensor *out__, tensor self, tensor mat2, int out_dtype) { + PROTECT( + auto outputs__ = torch::bmm(*self, *mat2, at::ScalarType(out_dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bmm_dtype_out(tensor *out__, tensor out, tensor self, tensor mat2, int out_dtype) { + PROTECT( + auto outputs__ = torch::bmm_out(*out, *self, *mat2, at::ScalarType(out_dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bmm_out(tensor *out__, tensor out, tensor self, tensor mat2) { + PROTECT( + auto outputs__ = torch::bmm_out(*out, *self, *mat2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +tensor *atg_broadcast_tensors(tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::broadcast_tensors(of_carray_tensor(tensors_data, tensors_len)); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg_broadcast_to(tensor *out__, tensor self, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::broadcast_to(*self, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bucketize(tensor *out__, tensor self, tensor boundaries, int out_int32, int right) { + PROTECT( + auto outputs__ = torch::bucketize(*self, *boundaries, (bool)out_int32, (bool)right); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bucketize_scalar(tensor *out__, scalar self_scalar, tensor boundaries, int out_int32, int right) { + PROTECT( + auto outputs__ = torch::bucketize(*self_scalar, *boundaries, (bool)out_int32, (bool)right); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bucketize_scalar_out(tensor *out__, tensor out, scalar self_scalar, tensor boundaries, int out_int32, int right) { + PROTECT( + auto outputs__ = torch::bucketize_out(*out, *self_scalar, *boundaries, (bool)out_int32, (bool)right); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_bucketize_tensor_out(tensor *out__, tensor out, tensor self, tensor boundaries, int out_int32, int right) { + PROTECT( + auto outputs__ = torch::bucketize_out(*out, *self, *boundaries, (bool)out_int32, (bool)right); + out__[0] = new torch::Tensor(outputs__); + ) +} + +int atg_can_cast(int from_, int to) { + PROTECT( + return torch::can_cast(at::ScalarType(from_), at::ScalarType(to)); + ) + return 0; +} + +void atg_cartesian_prod(tensor *out__, tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::cartesian_prod(of_carray_tensor(tensors_data, tensors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cat(tensor *out__, tensor *tensors_data, int tensors_len, int64_t dim) { + PROTECT( + auto outputs__ = torch::cat(of_carray_tensor(tensors_data, tensors_len), dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cat_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len, int64_t dim) { + PROTECT( + auto outputs__ = torch::cat_out(*out, of_carray_tensor(tensors_data, tensors_len), dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cauchy(tensor *out__, tensor self, double median, double sigma) { + PROTECT( + auto outputs__ = torch::cauchy(*self, median, sigma); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cauchy_(tensor *out__, tensor self, double median, double sigma) { + PROTECT( + auto outputs__ = self->cauchy_(median, sigma); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cauchy_out(tensor *out__, tensor out, tensor self, double median, double sigma) { + PROTECT( + auto outputs__ = torch::cauchy_out(*out, *self, median, sigma); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ccol_indices(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->ccol_indices(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ccol_indices_copy(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::ccol_indices_copy(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ccol_indices_copy_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::ccol_indices_copy_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cdist(tensor *out__, tensor x1, tensor x2, double p, int64_t compute_mode_v, uint8_t compute_mode_null) { + PROTECT( + auto outputs__ = torch::cdist(*x1, *x2, p, compute_mode_null ? c10::nullopt : c10::optional(compute_mode_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ceil(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::ceil(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ceil_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::ceil_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ceil_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::ceil_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_celu(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::celu(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_celu_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::celu_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_celu_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::celu_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_chain_matmul(tensor *out__, tensor *matrices_data, int matrices_len) { + PROTECT( + auto outputs__ = torch::chain_matmul(of_carray_tensor(matrices_data, matrices_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_chain_matmul_out(tensor *out__, tensor out, tensor *matrices_data, int matrices_len) { + PROTECT( + auto outputs__ = torch::chain_matmul_out(*out, of_carray_tensor(matrices_data, matrices_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_chalf(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->chalf(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_channel_shuffle(tensor *out__, tensor self, int64_t groups) { + PROTECT( + auto outputs__ = torch::channel_shuffle(*self, groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_channel_shuffle_out(tensor *out__, tensor out, tensor self, int64_t groups) { + PROTECT( + auto outputs__ = torch::channel_shuffle_out(*out, *self, groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cholesky(tensor *out__, tensor self, int upper) { + PROTECT( + auto outputs__ = torch::cholesky(*self, (bool)upper); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cholesky_inverse(tensor *out__, tensor self, int upper) { + PROTECT( + auto outputs__ = torch::cholesky_inverse(*self, (bool)upper); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cholesky_inverse_out(tensor *out__, tensor out, tensor self, int upper) { + PROTECT( + auto outputs__ = torch::cholesky_inverse_out(*out, *self, (bool)upper); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cholesky_out(tensor *out__, tensor out, tensor self, int upper) { + PROTECT( + auto outputs__ = torch::cholesky_out(*out, *self, (bool)upper); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cholesky_solve(tensor *out__, tensor self, tensor input2, int upper) { + PROTECT( + auto outputs__ = torch::cholesky_solve(*self, *input2, (bool)upper); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cholesky_solve_out(tensor *out__, tensor out, tensor self, tensor input2, int upper) { + PROTECT( + auto outputs__ = torch::cholesky_solve_out(*out, *self, *input2, (bool)upper); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_choose_qparams_optimized(tensor *out__, tensor input, int64_t numel, int64_t n_bins, double ratio, int64_t bit_width) { + PROTECT( + auto outputs__ = torch::choose_qparams_optimized(*input, numel, n_bins, ratio, bit_width); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +tensor *atg_chunk(tensor self, int64_t chunks, int64_t dim) { + PROTECT( + auto outputs__ = torch::chunk(*self, chunks, dim); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg_clamp(tensor *out__, tensor self, scalar min, scalar max) { + PROTECT( + auto outputs__ = torch::clamp(*self, *min, *max); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clamp_(tensor *out__, tensor self, scalar min, scalar max) { + PROTECT( + auto outputs__ = torch::clamp_(*self, *min, *max); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clamp_max(tensor *out__, tensor self, scalar max) { + PROTECT( + auto outputs__ = torch::clamp_max(*self, *max); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clamp_max_(tensor *out__, tensor self, scalar max) { + PROTECT( + auto outputs__ = torch::clamp_max_(*self, *max); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clamp_max_out(tensor *out__, tensor out, tensor self, scalar max) { + PROTECT( + auto outputs__ = torch::clamp_max_out(*out, *self, *max); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clamp_max_tensor(tensor *out__, tensor self, tensor max) { + PROTECT( + auto outputs__ = torch::clamp_max(*self, *max); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clamp_max_tensor_(tensor *out__, tensor self, tensor max) { + PROTECT( + auto outputs__ = torch::clamp_max_(*self, *max); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clamp_max_tensor_out(tensor *out__, tensor out, tensor self, tensor max) { + PROTECT( + auto outputs__ = torch::clamp_max_out(*out, *self, *max); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clamp_min(tensor *out__, tensor self, scalar min) { + PROTECT( + auto outputs__ = torch::clamp_min(*self, *min); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clamp_min_(tensor *out__, tensor self, scalar min) { + PROTECT( + auto outputs__ = torch::clamp_min_(*self, *min); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clamp_min_out(tensor *out__, tensor out, tensor self, scalar min) { + PROTECT( + auto outputs__ = torch::clamp_min_out(*out, *self, *min); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clamp_min_tensor(tensor *out__, tensor self, tensor min) { + PROTECT( + auto outputs__ = torch::clamp_min(*self, *min); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clamp_min_tensor_(tensor *out__, tensor self, tensor min) { + PROTECT( + auto outputs__ = torch::clamp_min_(*self, *min); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clamp_min_tensor_out(tensor *out__, tensor out, tensor self, tensor min) { + PROTECT( + auto outputs__ = torch::clamp_min_out(*out, *self, *min); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clamp_out(tensor *out__, tensor out, tensor self, scalar min, scalar max) { + PROTECT( + auto outputs__ = torch::clamp_out(*out, *self, *min, *max); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clamp_tensor(tensor *out__, tensor self, tensor min, tensor max) { + PROTECT( + auto outputs__ = torch::clamp(*self, (min ? ::std::optional(*min) : ::std::nullopt), (max ? ::std::optional(*max) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clamp_tensor_(tensor *out__, tensor self, tensor min, tensor max) { + PROTECT( + auto outputs__ = torch::clamp_(*self, (min ? ::std::optional(*min) : ::std::nullopt), (max ? ::std::optional(*max) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clamp_tensor_out(tensor *out__, tensor out, tensor self, tensor min, tensor max) { + PROTECT( + auto outputs__ = torch::clamp_out(*out, *self, (min ? ::std::optional(*min) : ::std::nullopt), (max ? ::std::optional(*max) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clip(tensor *out__, tensor self, scalar min, scalar max) { + PROTECT( + auto outputs__ = torch::clip(*self, *min, *max); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clip_(tensor *out__, tensor self, scalar min, scalar max) { + PROTECT( + auto outputs__ = torch::clip_(*self, *min, *max); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clip_out(tensor *out__, tensor out, tensor self, scalar min, scalar max) { + PROTECT( + auto outputs__ = torch::clip_out(*out, *self, *min, *max); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clip_tensor(tensor *out__, tensor self, tensor min, tensor max) { + PROTECT( + auto outputs__ = torch::clip(*self, (min ? ::std::optional(*min) : ::std::nullopt), (max ? ::std::optional(*max) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clip_tensor_(tensor *out__, tensor self, tensor min, tensor max) { + PROTECT( + auto outputs__ = torch::clip_(*self, (min ? ::std::optional(*min) : ::std::nullopt), (max ? ::std::optional(*max) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clip_tensor_out(tensor *out__, tensor out, tensor self, tensor min, tensor max) { + PROTECT( + auto outputs__ = torch::clip_out(*out, *self, (min ? ::std::optional(*min) : ::std::nullopt), (max ? ::std::optional(*max) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_clone(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::clone_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_coalesce(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->coalesce(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_col2im(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, int64_t *kernel_size_data, int kernel_size_len, int64_t *dilation_data, int dilation_len, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len) { + PROTECT( + auto outputs__ = torch::col2im(*self, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(dilation_data, dilation_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_col2im_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int64_t *kernel_size_data, int kernel_size_len, int64_t *dilation_data, int dilation_len, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len) { + PROTECT( + auto outputs__ = torch::col2im_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(dilation_data, dilation_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_col_indices(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->col_indices(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_col_indices_copy(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::col_indices_copy(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_col_indices_copy_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::col_indices_copy_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_column_stack(tensor *out__, tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::column_stack(of_carray_tensor(tensors_data, tensors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_column_stack_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::column_stack_out(*out, of_carray_tensor(tensors_data, tensors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_combinations(tensor *out__, tensor self, int64_t r, int with_replacement) { + PROTECT( + auto outputs__ = torch::combinations(*self, r, (bool)with_replacement); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_complex(tensor *out__, tensor real, tensor imag) { + PROTECT( + auto outputs__ = torch::complex(*real, *imag); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_complex_out(tensor *out__, tensor out, tensor real, tensor imag) { + PROTECT( + auto outputs__ = torch::complex_out(*out, *real, *imag); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_concat(tensor *out__, tensor *tensors_data, int tensors_len, int64_t dim) { + PROTECT( + auto outputs__ = torch::concat(of_carray_tensor(tensors_data, tensors_len), dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_concat_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len, int64_t dim) { + PROTECT( + auto outputs__ = torch::concat_out(*out, of_carray_tensor(tensors_data, tensors_len), dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_concatenate(tensor *out__, tensor *tensors_data, int tensors_len, int64_t dim) { + PROTECT( + auto outputs__ = torch::concatenate(of_carray_tensor(tensors_data, tensors_len), dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_concatenate_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len, int64_t dim) { + PROTECT( + auto outputs__ = torch::concatenate_out(*out, of_carray_tensor(tensors_data, tensors_len), dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_conj(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::conj(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_conj_physical(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::conj_physical(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_conj_physical_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::conj_physical_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_conj_physical_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::conj_physical_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_constant_pad_nd(tensor *out__, tensor self, int64_t *pad_data, int pad_len) { + PROTECT( + auto outputs__ = torch::constant_pad_nd(*self, torch::IntArrayRef(pad_data, pad_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_constant_pad_nd_out(tensor *out__, tensor out, tensor self, int64_t *pad_data, int pad_len) { + PROTECT( + auto outputs__ = torch::constant_pad_nd_out(*out, *self, torch::IntArrayRef(pad_data, pad_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_contiguous(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->contiguous(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_conv1d(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::conv1d(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_conv1d_padding(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, char* padding_ptr, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::conv1d(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), std::string(padding_ptr, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_conv2d(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::conv2d(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_conv2d_padding(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, char* padding_ptr, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::conv2d(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), std::string(padding_ptr, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_conv3d(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::conv3d(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_conv3d_padding(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, char* padding_ptr, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::conv3d(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), std::string(padding_ptr, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_conv_depthwise3d(tensor *out__, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len) { + PROTECT( + auto outputs__ = torch::conv_depthwise3d(*self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_conv_depthwise3d_out(tensor *out__, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len) { + PROTECT( + auto outputs__ = torch::conv_depthwise3d_out(*out, *self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_conv_tbc(tensor *out__, tensor self, tensor weight, tensor bias, int64_t pad) { + PROTECT( + auto outputs__ = torch::conv_tbc(*self, *weight, *bias, pad); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_conv_tbc_backward(tensor *out__, tensor self, tensor input, tensor weight, tensor bias, int64_t pad) { + PROTECT( + auto outputs__ = torch::conv_tbc_backward(*self, *input, *weight, *bias, pad); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_conv_tbc_out(tensor *out__, tensor out, tensor self, tensor weight, tensor bias, int64_t pad) { + PROTECT( + auto outputs__ = torch::conv_tbc_out(*out, *self, *weight, *bias, pad); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_conv_transpose1d(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t groups, int64_t *dilation_data, int dilation_len) { + PROTECT( + auto outputs__ = torch::conv_transpose1d(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), groups, torch::IntArrayRef(dilation_data, dilation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_conv_transpose2d(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t groups, int64_t *dilation_data, int dilation_len) { + PROTECT( + auto outputs__ = torch::conv_transpose2d(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), groups, torch::IntArrayRef(dilation_data, dilation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_conv_transpose3d(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t groups, int64_t *dilation_data, int dilation_len) { + PROTECT( + auto outputs__ = torch::conv_transpose3d(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), groups, torch::IntArrayRef(dilation_data, dilation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_convolution(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::convolution(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)transposed, torch::IntArrayRef(output_padding_data, output_padding_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_convolution_out(tensor *out__, tensor out, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::convolution_out(*out, *input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)transposed, torch::IntArrayRef(output_padding_data, output_padding_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_convolution_overrideable(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::convolution_overrideable(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)transposed, torch::IntArrayRef(output_padding_data, output_padding_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_convolution_overrideable_out(tensor *out__, tensor out, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::convolution_overrideable_out(*out, *input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)transposed, torch::IntArrayRef(output_padding_data, output_padding_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_copy_sparse_to_sparse(tensor *out__, tensor self, tensor src, int non_blocking) { + PROTECT( + auto outputs__ = torch::copy_sparse_to_sparse(*self, *src, (bool)non_blocking); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_copy_sparse_to_sparse_(tensor *out__, tensor self, tensor src, int non_blocking) { + PROTECT( + auto outputs__ = torch::copy_sparse_to_sparse_(*self, *src, (bool)non_blocking); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_copy_sparse_to_sparse_out(tensor *out__, tensor out, tensor self, tensor src, int non_blocking) { + PROTECT( + auto outputs__ = torch::copy_sparse_to_sparse_out(*out, *self, *src, (bool)non_blocking); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_copysign(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::copysign(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_copysign_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->copysign_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_copysign_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::copysign_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_copysign_scalar(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::copysign(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_copysign_scalar_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->copysign_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_copysign_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::copysign_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_corrcoef(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::corrcoef(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cos(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::cos(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cos_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::cos_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cos_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::cos_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cosh(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::cosh(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cosh_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::cosh_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cosh_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::cosh_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cosine_embedding_loss(tensor *out__, tensor input1, tensor input2, tensor target, double margin, int64_t reduction) { + PROTECT( + auto outputs__ = torch::cosine_embedding_loss(*input1, *input2, *target, margin, reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cosine_similarity(tensor *out__, tensor x1, tensor x2, int64_t dim, double eps) { + PROTECT( + auto outputs__ = torch::cosine_similarity(*x1, *x2, dim, eps); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_count_nonzero(tensor *out__, tensor self, int64_t dim_v, uint8_t dim_null) { + PROTECT( + auto outputs__ = torch::count_nonzero(*self, dim_null ? c10::nullopt : c10::optional(dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_count_nonzero_dim_intlist(tensor *out__, tensor self, int64_t *dim_data, int dim_len) { + PROTECT( + auto outputs__ = torch::count_nonzero(*self, torch::IntArrayRef(dim_data, dim_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_count_nonzero_dim_intlist_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len) { + PROTECT( + auto outputs__ = torch::count_nonzero_out(*out, *self, torch::IntArrayRef(dim_data, dim_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_count_nonzero_out(tensor *out__, tensor out, tensor self, int64_t dim_v, uint8_t dim_null) { + PROTECT( + auto outputs__ = torch::count_nonzero_out(*out, *self, dim_null ? c10::nullopt : c10::optional(dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cov(tensor *out__, tensor self, int64_t correction, tensor fweights, tensor aweights) { + PROTECT( + auto outputs__ = torch::cov(*self, correction, (fweights ? ::std::optional(*fweights) : ::std::nullopt), (aweights ? ::std::optional(*aweights) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cross(tensor *out__, tensor self, tensor other, int64_t dim_v, uint8_t dim_null) { + PROTECT( + auto outputs__ = torch::cross(*self, *other, dim_null ? c10::nullopt : c10::optional(dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cross_entropy_loss(tensor *out__, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index, double label_smoothing) { + PROTECT( + auto outputs__ = torch::cross_entropy_loss(*self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction, ignore_index, label_smoothing); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cross_out(tensor *out__, tensor out, tensor self, tensor other, int64_t dim_v, uint8_t dim_null) { + PROTECT( + auto outputs__ = torch::cross_out(*out, *self, *other, dim_null ? c10::nullopt : c10::optional(dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_crow_indices(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->crow_indices(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_crow_indices_copy(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::crow_indices_copy(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_crow_indices_copy_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::crow_indices_copy_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ctc_loss(tensor *out__, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int64_t reduction, int zero_infinity) { + PROTECT( + auto outputs__ = torch::ctc_loss(*log_probs, *targets, torch::IntArrayRef(input_lengths_data, input_lengths_len), torch::IntArrayRef(target_lengths_data, target_lengths_len), blank, reduction, (bool)zero_infinity); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ctc_loss_tensor(tensor *out__, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank, int64_t reduction, int zero_infinity) { + PROTECT( + auto outputs__ = torch::ctc_loss(*log_probs, *targets, *input_lengths, *target_lengths, blank, reduction, (bool)zero_infinity); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cudnn_affine_grid_generator(tensor *out__, tensor theta, int64_t n, int64_t C, int64_t H, int64_t W) { + PROTECT( + auto outputs__ = torch::cudnn_affine_grid_generator(*theta, n, C, H, W); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cudnn_affine_grid_generator_backward(tensor *out__, tensor grad, int64_t n, int64_t C, int64_t H, int64_t W) { + PROTECT( + auto outputs__ = torch::cudnn_affine_grid_generator_backward(*grad, n, C, H, W); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cudnn_affine_grid_generator_backward_out(tensor *out__, tensor out, tensor grad, int64_t n, int64_t C, int64_t H, int64_t W) { + PROTECT( + auto outputs__ = torch::cudnn_affine_grid_generator_backward_out(*out, *grad, n, C, H, W); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cudnn_affine_grid_generator_out(tensor *out__, tensor out, tensor theta, int64_t n, int64_t C, int64_t H, int64_t W) { + PROTECT( + auto outputs__ = torch::cudnn_affine_grid_generator_out(*out, *theta, n, C, H, W); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cudnn_batch_norm(tensor *out__, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double exponential_average_factor, double epsilon) { + PROTECT( + auto outputs__ = torch::cudnn_batch_norm(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (bool)training, exponential_average_factor, epsilon); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg_cudnn_batch_norm_backward(tensor *out__, tensor input, tensor grad_output, tensor weight, tensor running_mean, tensor running_var, tensor save_mean, tensor save_var, double epsilon, tensor reserveSpace) { + PROTECT( + auto outputs__ = torch::cudnn_batch_norm_backward(*input, *grad_output, *weight, (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (save_mean ? ::std::optional(*save_mean) : ::std::nullopt), (save_var ? ::std::optional(*save_var) : ::std::nullopt), epsilon, *reserveSpace); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_cudnn_batch_norm_backward_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor input, tensor grad_output, tensor weight, tensor running_mean, tensor running_var, tensor save_mean, tensor save_var, double epsilon, tensor reserveSpace) { + PROTECT( + auto outputs__ = torch::cudnn_batch_norm_backward_out(*out0, *out1, *out2, *input, *grad_output, *weight, (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (save_mean ? ::std::optional(*save_mean) : ::std::nullopt), (save_var ? ::std::optional(*save_var) : ::std::nullopt), epsilon, *reserveSpace); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_cudnn_batch_norm_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor out3, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double exponential_average_factor, double epsilon) { + PROTECT( + auto outputs__ = torch::cudnn_batch_norm_out(*out0, *out1, *out2, *out3, *input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (bool)training, exponential_average_factor, epsilon); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg_cudnn_convolution(tensor *out__, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic, int allow_tf32) { + PROTECT( + auto outputs__ = torch::cudnn_convolution(*self, *weight, torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, (bool)benchmark, (bool)deterministic, (bool)allow_tf32); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cudnn_convolution_add_relu(tensor *out__, tensor self, tensor weight, tensor z, scalar alpha, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::cudnn_convolution_add_relu(*self, *weight, *z, *alpha, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cudnn_convolution_add_relu_out(tensor *out__, tensor out, tensor self, tensor weight, tensor z, scalar alpha, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::cudnn_convolution_add_relu_out(*out, *self, *weight, *z, *alpha, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cudnn_convolution_out(tensor *out__, tensor out, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic, int allow_tf32) { + PROTECT( + auto outputs__ = torch::cudnn_convolution_out(*out, *self, *weight, torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, (bool)benchmark, (bool)deterministic, (bool)allow_tf32); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cudnn_convolution_relu(tensor *out__, tensor self, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::cudnn_convolution_relu(*self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cudnn_convolution_relu_out(tensor *out__, tensor out, tensor self, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::cudnn_convolution_relu_out(*out, *self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cudnn_convolution_transpose(tensor *out__, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic, int allow_tf32) { + PROTECT( + auto outputs__ = torch::cudnn_convolution_transpose(*self, *weight, torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, (bool)benchmark, (bool)deterministic, (bool)allow_tf32); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cudnn_convolution_transpose_out(tensor *out__, tensor out, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic, int allow_tf32) { + PROTECT( + auto outputs__ = torch::cudnn_convolution_transpose_out(*out, *self, *weight, torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, (bool)benchmark, (bool)deterministic, (bool)allow_tf32); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cudnn_grid_sampler(tensor *out__, tensor self, tensor grid) { + PROTECT( + auto outputs__ = torch::cudnn_grid_sampler(*self, *grid); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cudnn_grid_sampler_backward(tensor *out__, tensor self, tensor grid, tensor grad_output) { + PROTECT( + auto outputs__ = torch::cudnn_grid_sampler_backward(*self, *grid, *grad_output); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_cudnn_grid_sampler_backward_out(tensor *out__, tensor out0, tensor out1, tensor self, tensor grid, tensor grad_output) { + PROTECT( + auto outputs__ = torch::cudnn_grid_sampler_backward_out(*out0, *out1, *self, *grid, *grad_output); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_cudnn_grid_sampler_out(tensor *out__, tensor out, tensor self, tensor grid) { + PROTECT( + auto outputs__ = torch::cudnn_grid_sampler_out(*out, *self, *grid); + out__[0] = new torch::Tensor(outputs__); + ) +} + +int atg_cudnn_is_acceptable(tensor self) { + PROTECT( + return torch::cudnn_is_acceptable(*self); + ) + return 0; +} + +void atg_cummax(tensor *out__, tensor self, int64_t dim) { + PROTECT( + auto outputs__ = torch::cummax(*self, dim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_cummax_out(tensor *out__, tensor values, tensor indices, tensor self, int64_t dim) { + PROTECT( + auto outputs__ = torch::cummax_out(*values, *indices, *self, dim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_cummaxmin_backward(tensor *out__, tensor grad, tensor input, tensor indices, int64_t dim) { + PROTECT( + auto outputs__ = torch::cummaxmin_backward(*grad, *input, *indices, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cummin(tensor *out__, tensor self, int64_t dim) { + PROTECT( + auto outputs__ = torch::cummin(*self, dim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_cummin_out(tensor *out__, tensor values, tensor indices, tensor self, int64_t dim) { + PROTECT( + auto outputs__ = torch::cummin_out(*values, *indices, *self, dim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_cumprod(tensor *out__, tensor self, int64_t dim, int dtype) { + PROTECT( + auto outputs__ = torch::cumprod(*self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cumprod_(tensor *out__, tensor self, int64_t dim, int dtype) { + PROTECT( + auto outputs__ = self->cumprod_(dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cumprod_backward(tensor *out__, tensor grad, tensor input, int64_t dim, tensor output) { + PROTECT( + auto outputs__ = torch::cumprod_backward(*grad, *input, dim, *output); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cumprod_out(tensor *out__, tensor out, tensor self, int64_t dim, int dtype) { + PROTECT( + auto outputs__ = torch::cumprod_out(*out, *self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cumsum(tensor *out__, tensor self, int64_t dim, int dtype) { + PROTECT( + auto outputs__ = torch::cumsum(*self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cumsum_(tensor *out__, tensor self, int64_t dim, int dtype) { + PROTECT( + auto outputs__ = self->cumsum_(dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cumsum_out(tensor *out__, tensor out, tensor self, int64_t dim, int dtype) { + PROTECT( + auto outputs__ = torch::cumsum_out(*out, *self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cumulative_trapezoid(tensor *out__, tensor y, int64_t dim) { + PROTECT( + auto outputs__ = torch::cumulative_trapezoid(*y, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_cumulative_trapezoid_x(tensor *out__, tensor y, tensor x, int64_t dim) { + PROTECT( + auto outputs__ = torch::cumulative_trapezoid(*y, *x, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_data(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->data(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_deg2rad(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::deg2rad(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_deg2rad_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::deg2rad_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_deg2rad_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::deg2rad_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +int64_t atg_dense_dim(tensor self) { + PROTECT( + return self->dense_dim(); + ) + return 0; +} + +void atg_dequantize(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::dequantize(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_dequantize_self_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::dequantize_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +tensor *atg_dequantize_tensors(tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::dequantize(of_carray_tensor(tensors_data, tensors_len)); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg_dequantize_tensors_out(tensor *out_data, int out_len, tensor *tensors_data, int tensors_len) { + PROTECT( + torch::dequantize_out(of_carray_tensor(out_data, out_len), of_carray_tensor(tensors_data, tensors_len)); + ) +} + +void atg_det(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::det(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_detach(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::detach(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_detach_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::detach_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_detach_copy(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::detach_copy(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_detach_copy_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::detach_copy_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_diag(tensor *out__, tensor self, int64_t diagonal) { + PROTECT( + auto outputs__ = torch::diag(*self, diagonal); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_diag_embed(tensor *out__, tensor self, int64_t offset, int64_t dim1, int64_t dim2) { + PROTECT( + auto outputs__ = torch::diag_embed(*self, offset, dim1, dim2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_diag_embed_out(tensor *out__, tensor out, tensor self, int64_t offset, int64_t dim1, int64_t dim2) { + PROTECT( + auto outputs__ = torch::diag_embed_out(*out, *self, offset, dim1, dim2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_diag_out(tensor *out__, tensor out, tensor self, int64_t diagonal) { + PROTECT( + auto outputs__ = torch::diag_out(*out, *self, diagonal); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_diagflat(tensor *out__, tensor self, int64_t offset) { + PROTECT( + auto outputs__ = torch::diagflat(*self, offset); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_diagonal(tensor *out__, tensor self, int64_t offset, int64_t dim1, int64_t dim2) { + PROTECT( + auto outputs__ = torch::diagonal(*self, offset, dim1, dim2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_diagonal_backward(tensor *out__, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t offset, int64_t dim1, int64_t dim2) { + PROTECT( + auto outputs__ = torch::diagonal_backward(*grad_output, torch::IntArrayRef(input_sizes_data, input_sizes_len), offset, dim1, dim2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_diagonal_backward_out(tensor *out__, tensor out, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t offset, int64_t dim1, int64_t dim2) { + PROTECT( + auto outputs__ = torch::diagonal_backward_out(*out, *grad_output, torch::IntArrayRef(input_sizes_data, input_sizes_len), offset, dim1, dim2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_diagonal_copy(tensor *out__, tensor self, int64_t offset, int64_t dim1, int64_t dim2) { + PROTECT( + auto outputs__ = torch::diagonal_copy(*self, offset, dim1, dim2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_diagonal_copy_out(tensor *out__, tensor out, tensor self, int64_t offset, int64_t dim1, int64_t dim2) { + PROTECT( + auto outputs__ = torch::diagonal_copy_out(*out, *self, offset, dim1, dim2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_diagonal_scatter(tensor *out__, tensor self, tensor src, int64_t offset, int64_t dim1, int64_t dim2) { + PROTECT( + auto outputs__ = torch::diagonal_scatter(*self, *src, offset, dim1, dim2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_diagonal_scatter_out(tensor *out__, tensor out, tensor self, tensor src, int64_t offset, int64_t dim1, int64_t dim2) { + PROTECT( + auto outputs__ = torch::diagonal_scatter_out(*out, *self, *src, offset, dim1, dim2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_diff(tensor *out__, tensor self, int64_t n, int64_t dim, tensor prepend, tensor append) { + PROTECT( + auto outputs__ = torch::diff(*self, n, dim, (prepend ? ::std::optional(*prepend) : ::std::nullopt), (append ? ::std::optional(*append) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_diff_out(tensor *out__, tensor out, tensor self, int64_t n, int64_t dim, tensor prepend, tensor append) { + PROTECT( + auto outputs__ = torch::diff_out(*out, *self, n, dim, (prepend ? ::std::optional(*prepend) : ::std::nullopt), (append ? ::std::optional(*append) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_digamma(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::digamma(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_digamma_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->digamma_(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_digamma_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::digamma_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_dist(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::dist(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_dist_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::dist_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_div(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::div(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_div_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->div_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_div_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::div_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_div_out_mode(tensor *out__, tensor out, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len) { + PROTECT( + auto outputs__ = torch::div_out(*out, *self, *other, std::string(rounding_mode_ptr, rounding_mode_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_div_scalar(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::div(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_div_scalar_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->div_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_div_scalar_mode(tensor *out__, tensor self, scalar other, char* rounding_mode_ptr, int rounding_mode_len) { + PROTECT( + auto outputs__ = torch::div(*self, *other, std::string(rounding_mode_ptr, rounding_mode_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_div_scalar_mode_(tensor *out__, tensor self, scalar other, char* rounding_mode_ptr, int rounding_mode_len) { + PROTECT( + auto outputs__ = self->div_(*other, std::string(rounding_mode_ptr, rounding_mode_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_div_scalar_mode_out(tensor *out__, tensor out, tensor self, scalar other, char* rounding_mode_ptr, int rounding_mode_len) { + PROTECT( + auto outputs__ = torch::div_out(*out, *self, *other, std::string(rounding_mode_ptr, rounding_mode_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_div_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::div_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_div_tensor_mode(tensor *out__, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len) { + PROTECT( + auto outputs__ = torch::div(*self, *other, std::string(rounding_mode_ptr, rounding_mode_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_div_tensor_mode_(tensor *out__, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len) { + PROTECT( + auto outputs__ = self->div_(*other, std::string(rounding_mode_ptr, rounding_mode_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_divide(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::divide(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_divide_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->divide_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_divide_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::divide_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_divide_out_mode(tensor *out__, tensor out, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len) { + PROTECT( + auto outputs__ = torch::divide_out(*out, *self, *other, std::string(rounding_mode_ptr, rounding_mode_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_divide_scalar(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::divide(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_divide_scalar_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->divide_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_divide_scalar_mode(tensor *out__, tensor self, scalar other, char* rounding_mode_ptr, int rounding_mode_len) { + PROTECT( + auto outputs__ = torch::divide(*self, *other, std::string(rounding_mode_ptr, rounding_mode_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_divide_scalar_mode_(tensor *out__, tensor self, scalar other, char* rounding_mode_ptr, int rounding_mode_len) { + PROTECT( + auto outputs__ = self->divide_(*other, std::string(rounding_mode_ptr, rounding_mode_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_divide_tensor_mode(tensor *out__, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len) { + PROTECT( + auto outputs__ = torch::divide(*self, *other, std::string(rounding_mode_ptr, rounding_mode_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_divide_tensor_mode_(tensor *out__, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len) { + PROTECT( + auto outputs__ = self->divide_(*other, std::string(rounding_mode_ptr, rounding_mode_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_dot(tensor *out__, tensor self, tensor tensor) { + PROTECT( + auto outputs__ = torch::dot(*self, *tensor); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_dot_out(tensor *out__, tensor out, tensor self, tensor tensor) { + PROTECT( + auto outputs__ = torch::dot_out(*out, *self, *tensor); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_dropout(tensor *out__, tensor input, double p, int train) { + PROTECT( + auto outputs__ = torch::dropout(*input, p, (bool)train); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_dropout_(tensor *out__, tensor self, double p, int train) { + PROTECT( + auto outputs__ = torch::dropout_(*self, p, (bool)train); + out__[0] = new torch::Tensor(outputs__); + ) +} + +tensor *atg_dsplit(tensor self, int64_t sections) { + PROTECT( + auto outputs__ = torch::dsplit(*self, sections); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +tensor *atg_dsplit_array(tensor self, int64_t *indices_data, int indices_len) { + PROTECT( + auto outputs__ = torch::dsplit(*self, torch::IntArrayRef(indices_data, indices_len)); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg_dstack(tensor *out__, tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::dstack(of_carray_tensor(tensors_data, tensors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_dstack_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::dstack_out(*out, of_carray_tensor(tensors_data, tensors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_einsum(tensor *out__, char* equation_ptr, int equation_len, tensor *tensors_data, int tensors_len, int64_t *path_data, int path_len) { + PROTECT( + auto outputs__ = torch::einsum(std::string(equation_ptr, equation_len), of_carray_tensor(tensors_data, tensors_len), path_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(path_data, path_len))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_elu(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::elu(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_elu_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::elu_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_elu_backward(tensor *out__, tensor grad_output, scalar alpha, scalar scale, scalar input_scale, int is_result, tensor self_or_result) { + PROTECT( + auto outputs__ = torch::elu_backward(*grad_output, *alpha, *scale, *input_scale, (bool)is_result, *self_or_result); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_elu_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, scalar alpha, scalar scale, scalar input_scale, int is_result, tensor self_or_result) { + PROTECT( + auto outputs__ = torch::elu_backward_out(*grad_input, *grad_output, *alpha, *scale, *input_scale, (bool)is_result, *self_or_result); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_elu_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::elu_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_embedding(tensor *out__, tensor weight, tensor indices, int64_t padding_idx, int scale_grad_by_freq, int sparse) { + PROTECT( + auto outputs__ = torch::embedding(*weight, *indices, padding_idx, (bool)scale_grad_by_freq, (bool)sparse); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_embedding_backward(tensor *out__, tensor grad, tensor indices, int64_t num_weights, int64_t padding_idx, int scale_grad_by_freq, int sparse) { + PROTECT( + auto outputs__ = torch::embedding_backward(*grad, *indices, num_weights, padding_idx, (bool)scale_grad_by_freq, (bool)sparse); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_embedding_bag(tensor *out__, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset) { + PROTECT( + auto outputs__ = torch::embedding_bag(*weight, *indices, *offsets, (bool)scale_grad_by_freq, mode, (bool)sparse, (per_sample_weights ? ::std::optional(*per_sample_weights) : ::std::nullopt), (bool)include_last_offset); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg_embedding_bag_padding_idx(tensor *out__, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset, int64_t padding_idx_v, uint8_t padding_idx_null) { + PROTECT( + auto outputs__ = torch::embedding_bag(*weight, *indices, *offsets, (bool)scale_grad_by_freq, mode, (bool)sparse, (per_sample_weights ? ::std::optional(*per_sample_weights) : ::std::nullopt), (bool)include_last_offset, padding_idx_null ? c10::nullopt : c10::optional(padding_idx_v)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg_embedding_dense_backward(tensor *out__, tensor grad_output, tensor indices, int64_t num_weights, int64_t padding_idx, int scale_grad_by_freq) { + PROTECT( + auto outputs__ = torch::embedding_dense_backward(*grad_output, *indices, num_weights, padding_idx, (bool)scale_grad_by_freq); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_embedding_dense_backward_out(tensor *out__, tensor out, tensor grad_output, tensor indices, int64_t num_weights, int64_t padding_idx, int scale_grad_by_freq) { + PROTECT( + auto outputs__ = torch::embedding_dense_backward_out(*out, *grad_output, *indices, num_weights, padding_idx, (bool)scale_grad_by_freq); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_embedding_out(tensor *out__, tensor out, tensor weight, tensor indices, int64_t padding_idx, int scale_grad_by_freq, int sparse) { + PROTECT( + auto outputs__ = torch::embedding_out(*out, *weight, *indices, padding_idx, (bool)scale_grad_by_freq, (bool)sparse); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_embedding_renorm(tensor *out__, tensor self, tensor indices, double max_norm, double norm_type) { + PROTECT( + auto outputs__ = torch::embedding_renorm(*self, *indices, max_norm, norm_type); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_embedding_renorm_(tensor *out__, tensor self, tensor indices, double max_norm, double norm_type) { + PROTECT( + auto outputs__ = torch::embedding_renorm_(*self, *indices, max_norm, norm_type); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_embedding_renorm_out(tensor *out__, tensor out, tensor self, tensor indices, double max_norm, double norm_type) { + PROTECT( + auto outputs__ = torch::embedding_renorm_out(*out, *self, *indices, max_norm, norm_type); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_embedding_sparse_backward(tensor *out__, tensor grad, tensor indices, int64_t num_weights, int64_t padding_idx, int scale_grad_by_freq) { + PROTECT( + auto outputs__ = torch::embedding_sparse_backward(*grad, *indices, num_weights, padding_idx, (bool)scale_grad_by_freq); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_empty(tensor *out__, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::empty(torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_empty_like(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::empty_like(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_empty_like_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::empty_like_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_empty_out(tensor *out__, tensor out, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::empty_out(*out, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_empty_permuted(tensor *out__, int64_t *size_data, int size_len, int64_t *physical_layout_data, int physical_layout_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::empty_permuted(torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(physical_layout_data, physical_layout_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_empty_permuted_out(tensor *out__, tensor out, int64_t *size_data, int size_len, int64_t *physical_layout_data, int physical_layout_len) { + PROTECT( + auto outputs__ = torch::empty_permuted_out(*out, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(physical_layout_data, physical_layout_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_empty_quantized(tensor *out__, int64_t *size_data, int size_len, tensor qtensor, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::empty_quantized(torch::IntArrayRef(size_data, size_len), *qtensor, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_empty_quantized_out(tensor *out__, tensor out, int64_t *size_data, int size_len, tensor qtensor) { + PROTECT( + auto outputs__ = torch::empty_quantized_out(*out, torch::IntArrayRef(size_data, size_len), *qtensor); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_empty_strided(tensor *out__, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::empty_strided(torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_empty_strided_out(tensor *out__, tensor out, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len) { + PROTECT( + auto outputs__ = torch::empty_strided_out(*out, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_eq(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::eq(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_eq_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->eq_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_eq_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::eq_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_eq_tensor(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::eq(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_eq_tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->eq_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_eq_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::eq_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +int atg_equal(tensor self, tensor other) { + PROTECT( + return torch::equal(*self, *other); + ) + return 0; +} + +void atg_erf(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::erf(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_erf_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::erf_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_erf_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::erf_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_erfc(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::erfc(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_erfc_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::erfc_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_erfc_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::erfc_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_erfinv(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::erfinv(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_erfinv_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->erfinv_(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_erfinv_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::erfinv_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_exp(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::exp(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_exp2(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::exp2(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_exp2_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::exp2_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_exp2_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::exp2_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_exp_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::exp_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_exp_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::exp_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_expand(tensor *out__, tensor self, int64_t *size_data, int size_len, int implicit) { + PROTECT( + auto outputs__ = self->expand(torch::IntArrayRef(size_data, size_len), (bool)implicit); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_expand_as(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->expand_as(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_expand_copy(tensor *out__, tensor self, int64_t *size_data, int size_len, int implicit) { + PROTECT( + auto outputs__ = torch::expand_copy(*self, torch::IntArrayRef(size_data, size_len), (bool)implicit); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_expand_copy_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len, int implicit) { + PROTECT( + auto outputs__ = torch::expand_copy_out(*out, *self, torch::IntArrayRef(size_data, size_len), (bool)implicit); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_expm1(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::expm1(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_expm1_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::expm1_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_expm1_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::expm1_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_exponential(tensor *out__, tensor self, double lambd) { + PROTECT( + auto outputs__ = torch::exponential(*self, lambd); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_exponential_(tensor *out__, tensor self, double lambd) { + PROTECT( + auto outputs__ = self->exponential_(lambd); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_exponential_out(tensor *out__, tensor out, tensor self, double lambd) { + PROTECT( + auto outputs__ = torch::exponential_out(*out, *self, lambd); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_eye(tensor *out__, int64_t n, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::eye(n, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_eye_m(tensor *out__, int64_t n, int64_t m, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::eye(n, m, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_eye_m_out(tensor *out__, tensor out, int64_t n, int64_t m) { + PROTECT( + auto outputs__ = torch::eye_out(*out, n, m); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_eye_out(tensor *out__, tensor out, int64_t n) { + PROTECT( + auto outputs__ = torch::eye_out(*out, n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fake_quantize_per_channel_affine(tensor *out__, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max) { + PROTECT( + auto outputs__ = torch::fake_quantize_per_channel_affine(*self, *scale, *zero_point, axis, quant_min, quant_max); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fake_quantize_per_channel_affine_cachemask(tensor *out__, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max) { + PROTECT( + auto outputs__ = torch::fake_quantize_per_channel_affine_cachemask(*self, *scale, *zero_point, axis, quant_min, quant_max); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_fake_quantize_per_channel_affine_cachemask_backward(tensor *out__, tensor grad, tensor mask) { + PROTECT( + auto outputs__ = torch::fake_quantize_per_channel_affine_cachemask_backward(*grad, *mask); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fake_quantize_per_channel_affine_cachemask_out(tensor *out__, tensor out0, tensor out1, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max) { + PROTECT( + auto outputs__ = torch::fake_quantize_per_channel_affine_cachemask_out(*out0, *out1, *self, *scale, *zero_point, axis, quant_min, quant_max); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_fake_quantize_per_tensor_affine(tensor *out__, tensor self, double scale, int64_t zero_point, int64_t quant_min, int64_t quant_max) { + PROTECT( + auto outputs__ = torch::fake_quantize_per_tensor_affine(*self, scale, zero_point, quant_min, quant_max); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fake_quantize_per_tensor_affine_cachemask(tensor *out__, tensor self, double scale, int64_t zero_point, int64_t quant_min, int64_t quant_max) { + PROTECT( + auto outputs__ = torch::fake_quantize_per_tensor_affine_cachemask(*self, scale, zero_point, quant_min, quant_max); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_fake_quantize_per_tensor_affine_cachemask_backward(tensor *out__, tensor grad, tensor mask) { + PROTECT( + auto outputs__ = torch::fake_quantize_per_tensor_affine_cachemask_backward(*grad, *mask); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fake_quantize_per_tensor_affine_cachemask_out(tensor *out__, tensor out0, tensor out1, tensor self, double scale, int64_t zero_point, int64_t quant_min, int64_t quant_max) { + PROTECT( + auto outputs__ = torch::fake_quantize_per_tensor_affine_cachemask_out(*out0, *out1, *self, scale, zero_point, quant_min, quant_max); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_fake_quantize_per_tensor_affine_tensor_qparams(tensor *out__, tensor self, tensor scale, tensor zero_point, int64_t quant_min, int64_t quant_max) { + PROTECT( + auto outputs__ = torch::fake_quantize_per_tensor_affine(*self, *scale, *zero_point, quant_min, quant_max); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_feature_alpha_dropout(tensor *out__, tensor input, double p, int train) { + PROTECT( + auto outputs__ = torch::feature_alpha_dropout(*input, p, (bool)train); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_feature_alpha_dropout_(tensor *out__, tensor self, double p, int train) { + PROTECT( + auto outputs__ = torch::feature_alpha_dropout_(*self, p, (bool)train); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_feature_dropout(tensor *out__, tensor input, double p, int train) { + PROTECT( + auto outputs__ = torch::feature_dropout(*input, p, (bool)train); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_feature_dropout_(tensor *out__, tensor self, double p, int train) { + PROTECT( + auto outputs__ = torch::feature_dropout_(*self, p, (bool)train); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_fft(tensor *out__, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_fft(*self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_fft2(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_fft2(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_fft2_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_fft2_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_fft_out(tensor *out__, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_fft_out(*out, *self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_fftfreq(tensor *out__, int64_t n, double d, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::fft_fftfreq(n, d, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_fftfreq_out(tensor *out__, tensor out, int64_t n, double d) { + PROTECT( + auto outputs__ = torch::fft_fftfreq_out(*out, n, d); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_fftn(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_fftn(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_fftn_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_fftn_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_fftshift(tensor *out__, tensor self, int64_t *dim_data, int dim_len) { + PROTECT( + auto outputs__ = torch::fft_fftshift(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_hfft(tensor *out__, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_hfft(*self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_hfft2(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_hfft2(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_hfft2_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_hfft2_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_hfft_out(tensor *out__, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_hfft_out(*out, *self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_hfftn(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_hfftn(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_hfftn_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_hfftn_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_ifft(tensor *out__, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_ifft(*self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_ifft2(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_ifft2(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_ifft2_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_ifft2_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_ifft_out(tensor *out__, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_ifft_out(*out, *self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_ifftn(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_ifftn(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_ifftn_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_ifftn_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_ifftshift(tensor *out__, tensor self, int64_t *dim_data, int dim_len) { + PROTECT( + auto outputs__ = torch::fft_ifftshift(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_ihfft(tensor *out__, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_ihfft(*self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_ihfft2(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_ihfft2(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_ihfft2_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_ihfft2_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_ihfft_out(tensor *out__, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_ihfft_out(*out, *self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_ihfftn(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_ihfftn(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_ihfftn_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_ihfftn_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_irfft(tensor *out__, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_irfft(*self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_irfft2(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_irfft2(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_irfft2_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_irfft2_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_irfft_out(tensor *out__, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_irfft_out(*out, *self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_irfftn(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_irfftn(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_irfftn_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_irfftn_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_rfft(tensor *out__, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_rfft(*self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_rfft2(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_rfft2(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_rfft2_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_rfft2_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_rfft_out(tensor *out__, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_rfft_out(*out, *self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_rfftfreq(tensor *out__, int64_t n, double d, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::fft_rfftfreq(n, d, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_rfftfreq_out(tensor *out__, tensor out, int64_t n, double d) { + PROTECT( + auto outputs__ = torch::fft_rfftfreq_out(*out, n, d); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_rfftn(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_rfftn(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fft_rfftn_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { + PROTECT( + auto outputs__ = torch::fft_rfftn_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fill(tensor *out__, tensor self, scalar value) { + PROTECT( + auto outputs__ = torch::fill(*self, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fill_(tensor *out__, tensor self, scalar value) { + PROTECT( + auto outputs__ = torch::fill_(*self, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fill_diagonal_(tensor *out__, tensor self, scalar fill_value, int wrap) { + PROTECT( + auto outputs__ = self->fill_diagonal_(*fill_value, (bool)wrap); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fill_scalar_out(tensor *out__, tensor out, tensor self, scalar value) { + PROTECT( + auto outputs__ = torch::fill_out(*out, *self, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fill_tensor(tensor *out__, tensor self, tensor value) { + PROTECT( + auto outputs__ = torch::fill(*self, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fill_tensor_(tensor *out__, tensor self, tensor value) { + PROTECT( + auto outputs__ = torch::fill_(*self, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fill_tensor_out(tensor *out__, tensor out, tensor self, tensor value) { + PROTECT( + auto outputs__ = torch::fill_out(*out, *self, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fix(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::fix(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fix_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::fix_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fix_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::fix_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_flatten(tensor *out__, tensor self, int64_t start_dim, int64_t end_dim) { + PROTECT( + auto outputs__ = torch::flatten(*self, start_dim, end_dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_flatten_dense_tensors(tensor *out__, tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::flatten_dense_tensors(of_carray_tensor(tensors_data, tensors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_flip(tensor *out__, tensor self, int64_t *dims_data, int dims_len) { + PROTECT( + auto outputs__ = torch::flip(*self, torch::IntArrayRef(dims_data, dims_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_flip_out(tensor *out__, tensor out, tensor self, int64_t *dims_data, int dims_len) { + PROTECT( + auto outputs__ = torch::flip_out(*out, *self, torch::IntArrayRef(dims_data, dims_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fliplr(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::fliplr(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_flipud(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::flipud(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_float_power(tensor *out__, tensor self, tensor exponent) { + PROTECT( + auto outputs__ = torch::float_power(*self, *exponent); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_float_power_(tensor *out__, tensor self, scalar exponent) { + PROTECT( + auto outputs__ = self->float_power_(*exponent); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_float_power_scalar(tensor *out__, scalar self_scalar, tensor exponent) { + PROTECT( + auto outputs__ = torch::float_power(*self_scalar, *exponent); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_float_power_scalar_out(tensor *out__, tensor out, scalar self_scalar, tensor exponent) { + PROTECT( + auto outputs__ = torch::float_power_out(*out, *self_scalar, *exponent); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_float_power_tensor_(tensor *out__, tensor self, tensor exponent) { + PROTECT( + auto outputs__ = self->float_power_(*exponent); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_float_power_tensor_scalar(tensor *out__, tensor self, scalar exponent) { + PROTECT( + auto outputs__ = torch::float_power(*self, *exponent); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_float_power_tensor_scalar_out(tensor *out__, tensor out, tensor self, scalar exponent) { + PROTECT( + auto outputs__ = torch::float_power_out(*out, *self, *exponent); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_float_power_tensor_tensor_out(tensor *out__, tensor out, tensor self, tensor exponent) { + PROTECT( + auto outputs__ = torch::float_power_out(*out, *self, *exponent); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_floor(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::floor(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_floor_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::floor_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_floor_divide(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::floor_divide(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_floor_divide_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->floor_divide_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_floor_divide_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::floor_divide_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_floor_divide_scalar(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::floor_divide(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_floor_divide_scalar_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->floor_divide_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_floor_divide_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::floor_divide_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_floor_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::floor_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fmax(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::fmax(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fmax_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::fmax_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fmin(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::fmin(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fmin_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::fmin_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fmod(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::fmod(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fmod_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->fmod_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fmod_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::fmod_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fmod_tensor(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::fmod(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fmod_tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->fmod_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fmod_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::fmod_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_frac(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::frac(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_frac_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::frac_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_frac_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::frac_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fractional_max_pool2d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor random_samples) { + PROTECT( + auto outputs__ = torch::fractional_max_pool2d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(output_size_data, output_size_len), *random_samples); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_fractional_max_pool2d_backward(tensor *out__, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor indices) { + PROTECT( + auto outputs__ = torch::fractional_max_pool2d_backward(*grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(output_size_data, output_size_len), *indices); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fractional_max_pool2d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor indices) { + PROTECT( + auto outputs__ = torch::fractional_max_pool2d_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(output_size_data, output_size_len), *indices); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fractional_max_pool2d_output(tensor *out__, tensor output, tensor indices, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor random_samples) { + PROTECT( + auto outputs__ = torch::fractional_max_pool2d_out(*output, *indices, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(output_size_data, output_size_len), *random_samples); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_fractional_max_pool3d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor random_samples) { + PROTECT( + auto outputs__ = torch::fractional_max_pool3d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(output_size_data, output_size_len), *random_samples); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_fractional_max_pool3d_backward(tensor *out__, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor indices) { + PROTECT( + auto outputs__ = torch::fractional_max_pool3d_backward(*grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(output_size_data, output_size_len), *indices); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fractional_max_pool3d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor indices) { + PROTECT( + auto outputs__ = torch::fractional_max_pool3d_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(output_size_data, output_size_len), *indices); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fractional_max_pool3d_output(tensor *out__, tensor output, tensor indices, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor random_samples) { + PROTECT( + auto outputs__ = torch::fractional_max_pool3d_out(*output, *indices, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(output_size_data, output_size_len), *random_samples); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_frexp(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::frexp(*self); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_frexp_tensor_out(tensor *out__, tensor mantissa, tensor exponent, tensor self) { + PROTECT( + auto outputs__ = torch::frexp_out(*mantissa, *exponent, *self); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_frobenius_norm(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim) { + PROTECT( + auto outputs__ = torch::frobenius_norm(*self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_frobenius_norm_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim) { + PROTECT( + auto outputs__ = torch::frobenius_norm_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_from_file(tensor *out__, char* filename_ptr, int filename_len, int shared, int64_t size_v, uint8_t size_null, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::from_file(std::string(filename_ptr, filename_len), (bool)shared, size_null ? c10::nullopt : c10::optional(size_v), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_from_file_out(tensor *out__, tensor out, char* filename_ptr, int filename_len, int shared, int64_t size_v, uint8_t size_null) { + PROTECT( + auto outputs__ = torch::from_file_out(*out, std::string(filename_ptr, filename_len), (bool)shared, size_null ? c10::nullopt : c10::optional(size_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_full(tensor *out__, int64_t *size_data, int size_len, scalar fill_value, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::full(torch::IntArrayRef(size_data, size_len), *fill_value, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_full_like(tensor *out__, tensor self, scalar fill_value) { + PROTECT( + auto outputs__ = torch::full_like(*self, *fill_value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_full_like_out(tensor *out__, tensor out, tensor self, scalar fill_value) { + PROTECT( + auto outputs__ = torch::full_like_out(*out, *self, *fill_value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_full_out(tensor *out__, tensor out, int64_t *size_data, int size_len, scalar fill_value) { + PROTECT( + auto outputs__ = torch::full_out(*out, torch::IntArrayRef(size_data, size_len), *fill_value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_fused_moving_avg_obs_fake_quant(tensor *out__, tensor self, tensor observer_on, tensor fake_quant_on, tensor running_min, tensor running_max, tensor scale, tensor zero_point, double averaging_const, int64_t quant_min, int64_t quant_max, int64_t ch_axis, int per_row_fake_quant, int symmetric_quant) { + PROTECT( + auto outputs__ = torch::fused_moving_avg_obs_fake_quant(*self, *observer_on, *fake_quant_on, *running_min, *running_max, *scale, *zero_point, averaging_const, quant_min, quant_max, ch_axis, (bool)per_row_fake_quant, (bool)symmetric_quant); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_gather(tensor *out__, tensor self, int64_t dim, tensor index, int sparse_grad) { + PROTECT( + auto outputs__ = torch::gather(*self, dim, *index, (bool)sparse_grad); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_gather_backward(tensor *out__, tensor grad, tensor self, int64_t dim, tensor index, int sparse_grad) { + PROTECT( + auto outputs__ = torch::gather_backward(*grad, *self, dim, *index, (bool)sparse_grad); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_gather_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, int sparse_grad) { + PROTECT( + auto outputs__ = torch::gather_out(*out, *self, dim, *index, (bool)sparse_grad); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_gcd(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::gcd(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_gcd_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::gcd_(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_gcd_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::gcd_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ge(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::ge(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ge_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->ge_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ge_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::ge_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ge_tensor(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::ge(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ge_tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->ge_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ge_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::ge_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_gelu(tensor *out__, tensor self, char* approximate_ptr, int approximate_len) { + PROTECT( + auto outputs__ = torch::gelu(*self, std::string(approximate_ptr, approximate_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_gelu_(tensor *out__, tensor self, char* approximate_ptr, int approximate_len) { + PROTECT( + auto outputs__ = torch::gelu_(*self, std::string(approximate_ptr, approximate_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_gelu_backward(tensor *out__, tensor grad_output, tensor self, char* approximate_ptr, int approximate_len) { + PROTECT( + auto outputs__ = torch::gelu_backward(*grad_output, *self, std::string(approximate_ptr, approximate_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_gelu_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, char* approximate_ptr, int approximate_len) { + PROTECT( + auto outputs__ = torch::gelu_backward_out(*grad_input, *grad_output, *self, std::string(approximate_ptr, approximate_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_gelu_out(tensor *out__, tensor out, tensor self, char* approximate_ptr, int approximate_len) { + PROTECT( + auto outputs__ = torch::gelu_out(*out, *self, std::string(approximate_ptr, approximate_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_geometric(tensor *out__, tensor self, double p) { + PROTECT( + auto outputs__ = torch::geometric(*self, p); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_geometric_(tensor *out__, tensor self, double p) { + PROTECT( + auto outputs__ = self->geometric_(p); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_geometric_out(tensor *out__, tensor out, tensor self, double p) { + PROTECT( + auto outputs__ = torch::geometric_out(*out, *self, p); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_geqrf(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::geqrf(*self); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_geqrf_a(tensor *out__, tensor a, tensor tau, tensor self) { + PROTECT( + auto outputs__ = torch::geqrf_out(*a, *tau, *self); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_ger(tensor *out__, tensor self, tensor vec2) { + PROTECT( + auto outputs__ = torch::ger(*self, *vec2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ger_out(tensor *out__, tensor out, tensor self, tensor vec2) { + PROTECT( + auto outputs__ = torch::ger_out(*out, *self, *vec2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_glu(tensor *out__, tensor self, int64_t dim) { + PROTECT( + auto outputs__ = torch::glu(*self, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_glu_backward(tensor *out__, tensor grad_output, tensor self, int64_t dim) { + PROTECT( + auto outputs__ = torch::glu_backward(*grad_output, *self, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_glu_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t dim) { + PROTECT( + auto outputs__ = torch::glu_backward_out(*grad_input, *grad_output, *self, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_glu_backward_jvp(tensor *out__, tensor grad_x, tensor grad_glu, tensor x, tensor dgrad_glu, tensor dx, int64_t dim) { + PROTECT( + auto outputs__ = torch::glu_backward_jvp(*grad_x, *grad_glu, *x, *dgrad_glu, *dx, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_glu_backward_jvp_out(tensor *out__, tensor out, tensor grad_x, tensor grad_glu, tensor x, tensor dgrad_glu, tensor dx, int64_t dim) { + PROTECT( + auto outputs__ = torch::glu_backward_jvp_out(*out, *grad_x, *grad_glu, *x, *dgrad_glu, *dx, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_glu_jvp(tensor *out__, tensor glu, tensor x, tensor dx, int64_t dim) { + PROTECT( + auto outputs__ = torch::glu_jvp(*glu, *x, *dx, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_glu_jvp_out(tensor *out__, tensor out, tensor glu, tensor x, tensor dx, int64_t dim) { + PROTECT( + auto outputs__ = torch::glu_jvp_out(*out, *glu, *x, *dx, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_glu_out(tensor *out__, tensor out, tensor self, int64_t dim) { + PROTECT( + auto outputs__ = torch::glu_out(*out, *self, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_grad(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->grad(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_greater(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::greater(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_greater_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->greater_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_greater_equal(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::greater_equal(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_greater_equal_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->greater_equal_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_greater_equal_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::greater_equal_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_greater_equal_tensor(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::greater_equal(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_greater_equal_tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->greater_equal_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_greater_equal_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::greater_equal_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_greater_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::greater_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_greater_tensor(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::greater(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_greater_tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->greater_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_greater_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::greater_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_grid_sampler(tensor *out__, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners) { + PROTECT( + auto outputs__ = torch::grid_sampler(*input, *grid, interpolation_mode, padding_mode, (bool)align_corners); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_grid_sampler_2d(tensor *out__, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners) { + PROTECT( + auto outputs__ = torch::grid_sampler_2d(*input, *grid, interpolation_mode, padding_mode, (bool)align_corners); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_grid_sampler_2d_out(tensor *out__, tensor out, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners) { + PROTECT( + auto outputs__ = torch::grid_sampler_2d_out(*out, *input, *grid, interpolation_mode, padding_mode, (bool)align_corners); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_grid_sampler_3d(tensor *out__, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners) { + PROTECT( + auto outputs__ = torch::grid_sampler_3d(*input, *grid, interpolation_mode, padding_mode, (bool)align_corners); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_grid_sampler_3d_out(tensor *out__, tensor out, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners) { + PROTECT( + auto outputs__ = torch::grid_sampler_3d_out(*out, *input, *grid, interpolation_mode, padding_mode, (bool)align_corners); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_group_norm(tensor *out__, tensor input, int64_t num_groups, tensor weight, tensor bias, double eps, int cudnn_enabled) { + PROTECT( + auto outputs__ = torch::group_norm(*input, num_groups, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), eps, (bool)cudnn_enabled); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_gru(tensor *out__, tensor input, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first) { + PROTECT( + auto outputs__ = torch::gru(*input, *hx, of_carray_tensor(params_data, params_len), (bool)has_biases, num_layers, dropout, (bool)train, (bool)bidirectional, (bool)batch_first); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_gru_cell(tensor *out__, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh) { + PROTECT( + auto outputs__ = torch::gru_cell(*input, *hx, *w_ih, *w_hh, (b_ih ? ::std::optional(*b_ih) : ::std::nullopt), (b_hh ? ::std::optional(*b_hh) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_gru_data(tensor *out__, tensor data, tensor batch_sizes, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional) { + PROTECT( + auto outputs__ = torch::gru(*data, *batch_sizes, *hx, of_carray_tensor(params_data, params_len), (bool)has_biases, num_layers, dropout, (bool)train, (bool)bidirectional); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_gt(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::gt(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_gt_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->gt_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_gt_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::gt_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_gt_tensor(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::gt(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_gt_tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->gt_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_gt_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::gt_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hamming_window(tensor *out__, int64_t window_length, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::hamming_window(window_length, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hamming_window_out(tensor *out__, tensor out, int64_t window_length) { + PROTECT( + auto outputs__ = torch::hamming_window_out(*out, window_length); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hamming_window_periodic(tensor *out__, int64_t window_length, int periodic, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::hamming_window(window_length, (bool)periodic, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hamming_window_periodic_alpha(tensor *out__, int64_t window_length, int periodic, double alpha, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::hamming_window(window_length, (bool)periodic, alpha, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hamming_window_periodic_alpha_beta(tensor *out__, int64_t window_length, int periodic, double alpha, double beta, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::hamming_window(window_length, (bool)periodic, alpha, beta, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hamming_window_periodic_alpha_beta_out(tensor *out__, tensor out, int64_t window_length, int periodic, double alpha, double beta) { + PROTECT( + auto outputs__ = torch::hamming_window_out(*out, window_length, (bool)periodic, alpha, beta); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hamming_window_periodic_alpha_out(tensor *out__, tensor out, int64_t window_length, int periodic, double alpha) { + PROTECT( + auto outputs__ = torch::hamming_window_out(*out, window_length, (bool)periodic, alpha); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hamming_window_periodic_out(tensor *out__, tensor out, int64_t window_length, int periodic) { + PROTECT( + auto outputs__ = torch::hamming_window_out(*out, window_length, (bool)periodic); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hann_window(tensor *out__, int64_t window_length, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::hann_window(window_length, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hann_window_out(tensor *out__, tensor out, int64_t window_length) { + PROTECT( + auto outputs__ = torch::hann_window_out(*out, window_length); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hann_window_periodic(tensor *out__, int64_t window_length, int periodic, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::hann_window(window_length, (bool)periodic, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hann_window_periodic_out(tensor *out__, tensor out, int64_t window_length, int periodic) { + PROTECT( + auto outputs__ = torch::hann_window_out(*out, window_length, (bool)periodic); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hardshrink(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::hardshrink(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hardshrink_backward(tensor *out__, tensor grad_out, tensor self, scalar lambd) { + PROTECT( + auto outputs__ = torch::hardshrink_backward(*grad_out, *self, *lambd); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hardshrink_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_out, tensor self, scalar lambd) { + PROTECT( + auto outputs__ = torch::hardshrink_backward_out(*grad_input, *grad_out, *self, *lambd); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hardshrink_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::hardshrink_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hardsigmoid(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::hardsigmoid(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hardsigmoid_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::hardsigmoid_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hardsigmoid_backward(tensor *out__, tensor grad_output, tensor self) { + PROTECT( + auto outputs__ = torch::hardsigmoid_backward(*grad_output, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hardsigmoid_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self) { + PROTECT( + auto outputs__ = torch::hardsigmoid_backward_out(*grad_input, *grad_output, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hardsigmoid_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::hardsigmoid_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hardswish(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::hardswish(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hardswish_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::hardswish_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hardswish_backward(tensor *out__, tensor grad_output, tensor self) { + PROTECT( + auto outputs__ = torch::hardswish_backward(*grad_output, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hardswish_backward_out(tensor *out__, tensor out, tensor grad_output, tensor self) { + PROTECT( + auto outputs__ = torch::hardswish_backward_out(*out, *grad_output, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hardswish_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::hardswish_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hardtanh(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::hardtanh(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hardtanh_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::hardtanh_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hardtanh_backward(tensor *out__, tensor grad_output, tensor self, scalar min_val, scalar max_val) { + PROTECT( + auto outputs__ = torch::hardtanh_backward(*grad_output, *self, *min_val, *max_val); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hardtanh_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, scalar min_val, scalar max_val) { + PROTECT( + auto outputs__ = torch::hardtanh_backward_out(*grad_input, *grad_output, *self, *min_val, *max_val); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hardtanh_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::hardtanh_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hash_tensor(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim, int64_t mode) { + PROTECT( + auto outputs__ = torch::hash_tensor(*self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim, mode); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hash_tensor_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int64_t mode) { + PROTECT( + auto outputs__ = torch::hash_tensor_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim, mode); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_heaviside(tensor *out__, tensor self, tensor values) { + PROTECT( + auto outputs__ = torch::heaviside(*self, *values); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_heaviside_(tensor *out__, tensor self, tensor values) { + PROTECT( + auto outputs__ = self->heaviside_(*values); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_heaviside_out(tensor *out__, tensor out, tensor self, tensor values) { + PROTECT( + auto outputs__ = torch::heaviside_out(*out, *self, *values); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hinge_embedding_loss(tensor *out__, tensor self, tensor target, double margin, int64_t reduction) { + PROTECT( + auto outputs__ = torch::hinge_embedding_loss(*self, *target, margin, reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_histc(tensor *out__, tensor self, int64_t bins) { + PROTECT( + auto outputs__ = torch::histc(*self, bins); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_histc_out(tensor *out__, tensor out, tensor self, int64_t bins) { + PROTECT( + auto outputs__ = torch::histc_out(*out, *self, bins); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_histogram(tensor *out__, tensor self, tensor bins, tensor weight, int density) { + PROTECT( + auto outputs__ = torch::histogram(*self, *bins, (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)density); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_histogram_bin_ct(tensor *out__, tensor self, int64_t bins, double *range_data, int range_len, tensor weight, int density) { + PROTECT( + auto outputs__ = torch::histogram(*self, bins, at::ArrayRef(range_data, range_len), (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)density); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_histogram_bin_ct_out(tensor *out__, tensor hist, tensor bin_edges, tensor self, int64_t bins, double *range_data, int range_len, tensor weight, int density) { + PROTECT( + auto outputs__ = torch::histogram_out(*hist, *bin_edges, *self, bins, at::ArrayRef(range_data, range_len), (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)density); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_histogram_bins_tensor_out(tensor *out__, tensor hist, tensor bin_edges, tensor self, tensor bins, tensor weight, int density) { + PROTECT( + auto outputs__ = torch::histogram_out(*hist, *bin_edges, *self, *bins, (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)density); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +tensor *atg_hsplit(tensor self, int64_t sections) { + PROTECT( + auto outputs__ = torch::hsplit(*self, sections); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +tensor *atg_hsplit_array(tensor self, int64_t *indices_data, int indices_len) { + PROTECT( + auto outputs__ = torch::hsplit(*self, torch::IntArrayRef(indices_data, indices_len)); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg_hspmm(tensor *out__, tensor mat1, tensor mat2) { + PROTECT( + auto outputs__ = torch::hspmm(*mat1, *mat2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hspmm_out(tensor *out__, tensor out, tensor mat1, tensor mat2) { + PROTECT( + auto outputs__ = torch::hspmm_out(*out, *mat1, *mat2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hstack(tensor *out__, tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::hstack(of_carray_tensor(tensors_data, tensors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hstack_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::hstack_out(*out, of_carray_tensor(tensors_data, tensors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_huber_loss(tensor *out__, tensor self, tensor target, int64_t reduction, double delta) { + PROTECT( + auto outputs__ = torch::huber_loss(*self, *target, reduction, delta); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_huber_loss_backward(tensor *out__, tensor grad_output, tensor self, tensor target, int64_t reduction, double delta) { + PROTECT( + auto outputs__ = torch::huber_loss_backward(*grad_output, *self, *target, reduction, delta); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_huber_loss_backward_out(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor target, int64_t reduction, double delta) { + PROTECT( + auto outputs__ = torch::huber_loss_backward_out(*grad_input, *grad_output, *self, *target, reduction, delta); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_huber_loss_out(tensor *out__, tensor out, tensor self, tensor target, int64_t reduction, double delta) { + PROTECT( + auto outputs__ = torch::huber_loss_out(*out, *self, *target, reduction, delta); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hypot(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::hypot(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hypot_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->hypot_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_hypot_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::hypot_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_i0(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::i0(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_i0_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::i0_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_i0_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::i0_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_igamma(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::igamma(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_igamma_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->igamma_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_igamma_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::igamma_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_igammac(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::igammac(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_igammac_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->igammac_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_igammac_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::igammac_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_im2col(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *dilation_data, int dilation_len, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len) { + PROTECT( + auto outputs__ = torch::im2col(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(dilation_data, dilation_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_im2col_out(tensor *out__, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *dilation_data, int dilation_len, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len) { + PROTECT( + auto outputs__ = torch::im2col_out(*out, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(dilation_data, dilation_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_imag(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::imag(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index(tensor *out__, tensor self, tensor *indices_data, int indices_len) { + PROTECT( + auto outputs__ = torch::index(*self, of_carray_tensor_opt(indices_data, indices_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index_add(tensor *out__, tensor self, int64_t dim, tensor index, tensor source) { + PROTECT( + auto outputs__ = torch::index_add(*self, dim, *index, *source); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index_add_(tensor *out__, tensor self, int64_t dim, tensor index, tensor source) { + PROTECT( + auto outputs__ = self->index_add_(dim, *index, *source); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index_add_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, tensor source) { + PROTECT( + auto outputs__ = torch::index_add_out(*out, *self, dim, *index, *source); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index_copy(tensor *out__, tensor self, int64_t dim, tensor index, tensor source) { + PROTECT( + auto outputs__ = torch::index_copy(*self, dim, *index, *source); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index_copy_(tensor *out__, tensor self, int64_t dim, tensor index, tensor source) { + PROTECT( + auto outputs__ = self->index_copy_(dim, *index, *source); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index_copy_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, tensor source) { + PROTECT( + auto outputs__ = torch::index_copy_out(*out, *self, dim, *index, *source); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index_fill(tensor *out__, tensor self, int64_t dim, tensor index, scalar value) { + PROTECT( + auto outputs__ = torch::index_fill(*self, dim, *index, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index_fill_(tensor *out__, tensor self, int64_t dim, tensor index, scalar value) { + PROTECT( + auto outputs__ = self->index_fill_(dim, *index, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index_fill_int_scalar_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, scalar value) { + PROTECT( + auto outputs__ = torch::index_fill_out(*out, *self, dim, *index, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index_fill_int_tensor(tensor *out__, tensor self, int64_t dim, tensor index, tensor value) { + PROTECT( + auto outputs__ = torch::index_fill(*self, dim, *index, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index_fill_int_tensor_(tensor *out__, tensor self, int64_t dim, tensor index, tensor value) { + PROTECT( + auto outputs__ = self->index_fill_(dim, *index, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index_fill_int_tensor_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, tensor value) { + PROTECT( + auto outputs__ = torch::index_fill_out(*out, *self, dim, *index, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index_put(tensor *out__, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate) { + PROTECT( + auto outputs__ = torch::index_put(*self, of_carray_tensor_opt(indices_data, indices_len), *values, (bool)accumulate); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index_put_(tensor *out__, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate) { + PROTECT( + auto outputs__ = torch::index_put_(*self, of_carray_tensor_opt(indices_data, indices_len), *values, (bool)accumulate); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index_put_out(tensor *out__, tensor out, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate) { + PROTECT( + auto outputs__ = torch::index_put_out(*out, *self, of_carray_tensor_opt(indices_data, indices_len), *values, (bool)accumulate); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index_reduce(tensor *out__, tensor self, int64_t dim, tensor index, tensor source, char* reduce_ptr, int reduce_len, int include_self) { + PROTECT( + auto outputs__ = torch::index_reduce(*self, dim, *index, *source, std::string(reduce_ptr, reduce_len), (bool)include_self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index_reduce_(tensor *out__, tensor self, int64_t dim, tensor index, tensor source, char* reduce_ptr, int reduce_len, int include_self) { + PROTECT( + auto outputs__ = self->index_reduce_(dim, *index, *source, std::string(reduce_ptr, reduce_len), (bool)include_self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index_reduce_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, tensor source, char* reduce_ptr, int reduce_len, int include_self) { + PROTECT( + auto outputs__ = torch::index_reduce_out(*out, *self, dim, *index, *source, std::string(reduce_ptr, reduce_len), (bool)include_self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index_select(tensor *out__, tensor self, int64_t dim, tensor index) { + PROTECT( + auto outputs__ = torch::index_select(*self, dim, *index); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index_select_backward(tensor *out__, tensor grad, int64_t *self_sizes_data, int self_sizes_len, int64_t dim, tensor index) { + PROTECT( + auto outputs__ = torch::index_select_backward(*grad, torch::IntArrayRef(self_sizes_data, self_sizes_len), dim, *index); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index_select_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index) { + PROTECT( + auto outputs__ = torch::index_select_out(*out, *self, dim, *index); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_index_tensor_out(tensor *out__, tensor out, tensor self, tensor *indices_data, int indices_len) { + PROTECT( + auto outputs__ = torch::index_out(*out, *self, of_carray_tensor_opt(indices_data, indices_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_indices(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->indices(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_indices_copy(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::indices_copy(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_indices_copy_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::indices_copy_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_infinitely_differentiable_gelu_backward(tensor *out__, tensor grad, tensor self) { + PROTECT( + auto outputs__ = torch::infinitely_differentiable_gelu_backward(*grad, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_inner(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::inner(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_inner_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::inner_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_instance_norm(tensor *out__, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int use_input_stats, double momentum, double eps, int cudnn_enabled) { + PROTECT( + auto outputs__ = torch::instance_norm(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (bool)use_input_stats, momentum, eps, (bool)cudnn_enabled); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_int_repr(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::int_repr(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_int_repr_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::int_repr_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_inverse(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::inverse(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_inverse_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::inverse_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +int atg_is_coalesced(tensor self) { + PROTECT( + return self->is_coalesced(); + ) + return 0; +} + +int atg_is_complex(tensor self) { + PROTECT( + return torch::is_complex(*self); + ) + return 0; +} + +int atg_is_conj(tensor self) { + PROTECT( + return torch::is_conj(*self); + ) + return 0; +} + +int atg_is_distributed(tensor self) { + PROTECT( + return torch::is_distributed(*self); + ) + return 0; +} + +int atg_is_floating_point(tensor self) { + PROTECT( + return torch::is_floating_point(*self); + ) + return 0; +} + +int atg_is_inference(tensor self) { + PROTECT( + return torch::is_inference(*self); + ) + return 0; +} + +int atg_is_leaf(tensor self) { + PROTECT( + return self->is_leaf(); + ) + return 0; +} + +int atg_is_neg(tensor self) { + PROTECT( + return torch::is_neg(*self); + ) + return 0; +} + +int atg_is_nonzero(tensor self) { + PROTECT( + return torch::is_nonzero(*self); + ) + return 0; +} + +int atg_is_pinned(tensor self, int device) { + PROTECT( + return self->is_pinned(device_of_int(device)); + ) + return 0; +} + +int atg_is_same_size(tensor self, tensor other) { + PROTECT( + return torch::is_same_size(*self, *other); + ) + return 0; +} + +int atg_is_set_to(tensor self, tensor tensor) { + PROTECT( + return self->is_set_to(*tensor); + ) + return 0; +} + +int atg_is_signed(tensor self) { + PROTECT( + return torch::is_signed(*self); + ) + return 0; +} + +int atg_is_vulkan_available() { + PROTECT( + return torch::is_vulkan_available(); + ) + return 0; +} + +void atg_isclose(tensor *out__, tensor self, tensor other, double rtol, double atol, int equal_nan) { + PROTECT( + auto outputs__ = torch::isclose(*self, *other, rtol, atol, (bool)equal_nan); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_isfinite(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::isfinite(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_isin(tensor *out__, tensor elements, tensor test_elements, int assume_unique, int invert) { + PROTECT( + auto outputs__ = torch::isin(*elements, *test_elements, (bool)assume_unique, (bool)invert); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_isin_scalar_tensor(tensor *out__, scalar element, tensor test_elements, int assume_unique, int invert) { + PROTECT( + auto outputs__ = torch::isin(*element, *test_elements, (bool)assume_unique, (bool)invert); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_isin_scalar_tensor_out(tensor *out__, tensor out, scalar element, tensor test_elements, int assume_unique, int invert) { + PROTECT( + auto outputs__ = torch::isin_out(*out, *element, *test_elements, (bool)assume_unique, (bool)invert); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_isin_tensor_scalar(tensor *out__, tensor elements, scalar test_element, int assume_unique, int invert) { + PROTECT( + auto outputs__ = torch::isin(*elements, *test_element, (bool)assume_unique, (bool)invert); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_isin_tensor_scalar_out(tensor *out__, tensor out, tensor elements, scalar test_element, int assume_unique, int invert) { + PROTECT( + auto outputs__ = torch::isin_out(*out, *elements, *test_element, (bool)assume_unique, (bool)invert); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_isin_tensor_tensor_out(tensor *out__, tensor out, tensor elements, tensor test_elements, int assume_unique, int invert) { + PROTECT( + auto outputs__ = torch::isin_out(*out, *elements, *test_elements, (bool)assume_unique, (bool)invert); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_isinf(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::isinf(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_isinf_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::isinf_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_isnan(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::isnan(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_isnan_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::isnan_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_isneginf(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::isneginf(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_isneginf_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::isneginf_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_isposinf(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::isposinf(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_isposinf_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::isposinf_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_isreal(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::isreal(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_istft(tensor *out__, tensor self, int64_t n_fft, int64_t hop_length_v, uint8_t hop_length_null, int64_t win_length_v, uint8_t win_length_null, tensor window, int center, int normalized, int onesided, int64_t length_v, uint8_t length_null, int return_complex) { + PROTECT( + auto outputs__ = torch::istft(*self, n_fft, hop_length_null ? c10::nullopt : c10::optional(hop_length_v), win_length_null ? c10::nullopt : c10::optional(win_length_v), (window ? ::std::optional(*window) : ::std::nullopt), (bool)center, (bool)normalized, (bool)onesided, length_null ? c10::nullopt : c10::optional(length_v), (bool)return_complex); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_kaiser_window(tensor *out__, int64_t window_length, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::kaiser_window(window_length, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_kaiser_window_beta(tensor *out__, int64_t window_length, int periodic, double beta, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::kaiser_window(window_length, (bool)periodic, beta, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_kaiser_window_beta_out(tensor *out__, tensor out, int64_t window_length, int periodic, double beta) { + PROTECT( + auto outputs__ = torch::kaiser_window_out(*out, window_length, (bool)periodic, beta); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_kaiser_window_out(tensor *out__, tensor out, int64_t window_length) { + PROTECT( + auto outputs__ = torch::kaiser_window_out(*out, window_length); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_kaiser_window_periodic(tensor *out__, int64_t window_length, int periodic, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::kaiser_window(window_length, (bool)periodic, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_kaiser_window_periodic_out(tensor *out__, tensor out, int64_t window_length, int periodic) { + PROTECT( + auto outputs__ = torch::kaiser_window_out(*out, window_length, (bool)periodic); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_kl_div(tensor *out__, tensor self, tensor target, int64_t reduction, int log_target) { + PROTECT( + auto outputs__ = torch::kl_div(*self, *target, reduction, (bool)log_target); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_kron(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::kron(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_kron_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::kron_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_kthvalue(tensor *out__, tensor self, int64_t k, int64_t dim, int keepdim) { + PROTECT( + auto outputs__ = torch::kthvalue(*self, k, dim, (bool)keepdim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_kthvalue_values(tensor *out__, tensor values, tensor indices, tensor self, int64_t k, int64_t dim, int keepdim) { + PROTECT( + auto outputs__ = torch::kthvalue_out(*values, *indices, *self, k, dim, (bool)keepdim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_l1_loss(tensor *out__, tensor self, tensor target, int64_t reduction) { + PROTECT( + auto outputs__ = torch::l1_loss(*self, *target, reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_layer_norm(tensor *out__, tensor input, int64_t *normalized_shape_data, int normalized_shape_len, tensor weight, tensor bias, double eps, int cudnn_enable) { + PROTECT( + auto outputs__ = torch::layer_norm(*input, torch::IntArrayRef(normalized_shape_data, normalized_shape_len), (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), eps, (bool)cudnn_enable); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lcm(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::lcm(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lcm_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::lcm_(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lcm_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::lcm_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ldexp(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::ldexp(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ldexp_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::ldexp_(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ldexp_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::ldexp_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_le(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::le(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_le_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->le_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_le_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::le_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_le_tensor(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::le(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_le_tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->le_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_le_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::le_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_leaky_relu(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::leaky_relu(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_leaky_relu_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::leaky_relu_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_leaky_relu_backward(tensor *out__, tensor grad_output, tensor self, scalar negative_slope, int self_is_result) { + PROTECT( + auto outputs__ = torch::leaky_relu_backward(*grad_output, *self, *negative_slope, (bool)self_is_result); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_leaky_relu_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, scalar negative_slope, int self_is_result) { + PROTECT( + auto outputs__ = torch::leaky_relu_backward_out(*grad_input, *grad_output, *self, *negative_slope, (bool)self_is_result); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_leaky_relu_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::leaky_relu_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lerp(tensor *out__, tensor self, tensor end, scalar weight) { + PROTECT( + auto outputs__ = torch::lerp(*self, *end, *weight); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lerp_(tensor *out__, tensor self, tensor end, scalar weight) { + PROTECT( + auto outputs__ = self->lerp_(*end, *weight); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lerp_scalar_out(tensor *out__, tensor out, tensor self, tensor end, scalar weight) { + PROTECT( + auto outputs__ = torch::lerp_out(*out, *self, *end, *weight); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lerp_tensor(tensor *out__, tensor self, tensor end, tensor weight) { + PROTECT( + auto outputs__ = torch::lerp(*self, *end, *weight); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lerp_tensor_(tensor *out__, tensor self, tensor end, tensor weight) { + PROTECT( + auto outputs__ = self->lerp_(*end, *weight); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lerp_tensor_out(tensor *out__, tensor out, tensor self, tensor end, tensor weight) { + PROTECT( + auto outputs__ = torch::lerp_out(*out, *self, *end, *weight); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_less(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::less(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_less_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->less_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_less_equal(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::less_equal(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_less_equal_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->less_equal_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_less_equal_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::less_equal_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_less_equal_tensor(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::less_equal(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_less_equal_tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->less_equal_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_less_equal_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::less_equal_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_less_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::less_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_less_tensor(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::less(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_less_tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->less_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_less_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::less_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lgamma(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::lgamma(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lgamma_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->lgamma_(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lgamma_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::lgamma_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lift(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::lift(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lift_fresh(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::lift_fresh(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lift_fresh_copy(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::lift_fresh_copy(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lift_fresh_copy_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::lift_fresh_copy_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lift_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::lift_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_cholesky(tensor *out__, tensor self, int upper) { + PROTECT( + auto outputs__ = torch::linalg_cholesky(*self, (bool)upper); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_cholesky_ex(tensor *out__, tensor self, int upper, int check_errors) { + PROTECT( + auto outputs__ = torch::linalg_cholesky_ex(*self, (bool)upper, (bool)check_errors); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_linalg_cholesky_ex_l(tensor *out__, tensor L, tensor info, tensor self, int upper, int check_errors) { + PROTECT( + auto outputs__ = torch::linalg_cholesky_ex_out(*L, *info, *self, (bool)upper, (bool)check_errors); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_linalg_cholesky_out(tensor *out__, tensor out, tensor self, int upper) { + PROTECT( + auto outputs__ = torch::linalg_cholesky_out(*out, *self, (bool)upper); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_cond(tensor *out__, tensor self, scalar p) { + PROTECT( + auto outputs__ = torch::linalg_cond(*self, *p); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_cond_out(tensor *out__, tensor out, tensor self, scalar p) { + PROTECT( + auto outputs__ = torch::linalg_cond_out(*out, *self, *p); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_cond_p_str(tensor *out__, tensor self, char* p_ptr, int p_len) { + PROTECT( + auto outputs__ = torch::linalg_cond(*self, std::string(p_ptr, p_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_cond_p_str_out(tensor *out__, tensor out, tensor self, char* p_ptr, int p_len) { + PROTECT( + auto outputs__ = torch::linalg_cond_out(*out, *self, std::string(p_ptr, p_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_cross(tensor *out__, tensor self, tensor other, int64_t dim) { + PROTECT( + auto outputs__ = torch::linalg_cross(*self, *other, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_cross_out(tensor *out__, tensor out, tensor self, tensor other, int64_t dim) { + PROTECT( + auto outputs__ = torch::linalg_cross_out(*out, *self, *other, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_det(tensor *out__, tensor A) { + PROTECT( + auto outputs__ = torch::linalg_det(*A); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_det_out(tensor *out__, tensor out, tensor A) { + PROTECT( + auto outputs__ = torch::linalg_det_out(*out, *A); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_diagonal(tensor *out__, tensor A, int64_t offset, int64_t dim1, int64_t dim2) { + PROTECT( + auto outputs__ = torch::linalg_diagonal(*A, offset, dim1, dim2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_eig(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::linalg_eig(*self); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_linalg_eig_out(tensor *out__, tensor eigenvalues, tensor eigenvectors, tensor self) { + PROTECT( + auto outputs__ = torch::linalg_eig_out(*eigenvalues, *eigenvectors, *self); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_linalg_eigh(tensor *out__, tensor self, char* UPLO_ptr, int UPLO_len) { + PROTECT( + auto outputs__ = torch::linalg_eigh(*self, std::string(UPLO_ptr, UPLO_len)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_linalg_eigh_eigvals(tensor *out__, tensor eigvals, tensor eigvecs, tensor self, char* UPLO_ptr, int UPLO_len) { + PROTECT( + auto outputs__ = torch::linalg_eigh_out(*eigvals, *eigvecs, *self, std::string(UPLO_ptr, UPLO_len)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_linalg_eigvals(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::linalg_eigvals(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_eigvals_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::linalg_eigvals_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_eigvalsh(tensor *out__, tensor self, char* UPLO_ptr, int UPLO_len) { + PROTECT( + auto outputs__ = torch::linalg_eigvalsh(*self, std::string(UPLO_ptr, UPLO_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_eigvalsh_out(tensor *out__, tensor out, tensor self, char* UPLO_ptr, int UPLO_len) { + PROTECT( + auto outputs__ = torch::linalg_eigvalsh_out(*out, *self, std::string(UPLO_ptr, UPLO_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_householder_product(tensor *out__, tensor input, tensor tau) { + PROTECT( + auto outputs__ = torch::linalg_householder_product(*input, *tau); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_householder_product_out(tensor *out__, tensor out, tensor input, tensor tau) { + PROTECT( + auto outputs__ = torch::linalg_householder_product_out(*out, *input, *tau); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_inv(tensor *out__, tensor A) { + PROTECT( + auto outputs__ = torch::linalg_inv(*A); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_inv_ex(tensor *out__, tensor A, int check_errors) { + PROTECT( + auto outputs__ = torch::linalg_inv_ex(*A, (bool)check_errors); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_linalg_inv_ex_inverse(tensor *out__, tensor inverse, tensor info, tensor A, int check_errors) { + PROTECT( + auto outputs__ = torch::linalg_inv_ex_out(*inverse, *info, *A, (bool)check_errors); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_linalg_inv_out(tensor *out__, tensor out, tensor A) { + PROTECT( + auto outputs__ = torch::linalg_inv_out(*out, *A); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_ldl_factor(tensor *out__, tensor self, int hermitian) { + PROTECT( + auto outputs__ = torch::linalg_ldl_factor(*self, (bool)hermitian); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_linalg_ldl_factor_ex(tensor *out__, tensor self, int hermitian, int check_errors) { + PROTECT( + auto outputs__ = torch::linalg_ldl_factor_ex(*self, (bool)hermitian, (bool)check_errors); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_linalg_ldl_factor_ex_out(tensor *out__, tensor LD, tensor pivots, tensor info, tensor self, int hermitian, int check_errors) { + PROTECT( + auto outputs__ = torch::linalg_ldl_factor_ex_out(*LD, *pivots, *info, *self, (bool)hermitian, (bool)check_errors); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_linalg_ldl_factor_out(tensor *out__, tensor LD, tensor pivots, tensor self, int hermitian) { + PROTECT( + auto outputs__ = torch::linalg_ldl_factor_out(*LD, *pivots, *self, (bool)hermitian); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_linalg_ldl_solve(tensor *out__, tensor LD, tensor pivots, tensor B, int hermitian) { + PROTECT( + auto outputs__ = torch::linalg_ldl_solve(*LD, *pivots, *B, (bool)hermitian); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_ldl_solve_out(tensor *out__, tensor out, tensor LD, tensor pivots, tensor B, int hermitian) { + PROTECT( + auto outputs__ = torch::linalg_ldl_solve_out(*out, *LD, *pivots, *B, (bool)hermitian); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_lstsq(tensor *out__, tensor self, tensor b, double rcond_v, uint8_t rcond_null, char* driver_ptr, int driver_len) { + PROTECT( + auto outputs__ = torch::linalg_lstsq(*self, *b, rcond_null ? c10::nullopt : c10::optional(rcond_v), std::string(driver_ptr, driver_len)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg_linalg_lstsq_out(tensor *out__, tensor solution, tensor residuals, tensor rank, tensor singular_values, tensor self, tensor b, double rcond_v, uint8_t rcond_null, char* driver_ptr, int driver_len) { + PROTECT( + auto outputs__ = torch::linalg_lstsq_out(*solution, *residuals, *rank, *singular_values, *self, *b, rcond_null ? c10::nullopt : c10::optional(rcond_v), std::string(driver_ptr, driver_len)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg_linalg_lu(tensor *out__, tensor A, int pivot) { + PROTECT( + auto outputs__ = torch::linalg_lu(*A, (bool)pivot); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_linalg_lu_factor(tensor *out__, tensor A, int pivot) { + PROTECT( + auto outputs__ = torch::linalg_lu_factor(*A, (bool)pivot); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_linalg_lu_factor_ex(tensor *out__, tensor A, int pivot, int check_errors) { + PROTECT( + auto outputs__ = torch::linalg_lu_factor_ex(*A, (bool)pivot, (bool)check_errors); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_linalg_lu_factor_ex_out(tensor *out__, tensor LU, tensor pivots, tensor info, tensor A, int pivot, int check_errors) { + PROTECT( + auto outputs__ = torch::linalg_lu_factor_ex_out(*LU, *pivots, *info, *A, (bool)pivot, (bool)check_errors); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_linalg_lu_factor_out(tensor *out__, tensor LU, tensor pivots, tensor A, int pivot) { + PROTECT( + auto outputs__ = torch::linalg_lu_factor_out(*LU, *pivots, *A, (bool)pivot); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_linalg_lu_out(tensor *out__, tensor P, tensor L, tensor U, tensor A, int pivot) { + PROTECT( + auto outputs__ = torch::linalg_lu_out(*P, *L, *U, *A, (bool)pivot); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_linalg_lu_solve(tensor *out__, tensor LU, tensor pivots, tensor B, int left, int adjoint) { + PROTECT( + auto outputs__ = torch::linalg_lu_solve(*LU, *pivots, *B, (bool)left, (bool)adjoint); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_lu_solve_out(tensor *out__, tensor out, tensor LU, tensor pivots, tensor B, int left, int adjoint) { + PROTECT( + auto outputs__ = torch::linalg_lu_solve_out(*out, *LU, *pivots, *B, (bool)left, (bool)adjoint); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_matmul(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::linalg_matmul(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_matmul_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::linalg_matmul_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_matrix_exp(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::linalg_matrix_exp(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_matrix_exp_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::linalg_matrix_exp_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_matrix_power(tensor *out__, tensor self, int64_t n) { + PROTECT( + auto outputs__ = torch::linalg_matrix_power(*self, n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_matrix_power_out(tensor *out__, tensor out, tensor self, int64_t n) { + PROTECT( + auto outputs__ = torch::linalg_matrix_power_out(*out, *self, n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_matrix_rank(tensor *out__, tensor self, double tol, int hermitian) { + PROTECT( + auto outputs__ = torch::linalg_matrix_rank(*self, tol, (bool)hermitian); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_matrix_rank_atol_rtol_float(tensor *out__, tensor self, double atol_v, uint8_t atol_null, double rtol_v, uint8_t rtol_null, int hermitian) { + PROTECT( + auto outputs__ = torch::linalg_matrix_rank(*self, atol_null ? c10::nullopt : c10::optional(atol_v), rtol_null ? c10::nullopt : c10::optional(rtol_v), (bool)hermitian); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_matrix_rank_atol_rtol_float_out(tensor *out__, tensor out, tensor self, double atol_v, uint8_t atol_null, double rtol_v, uint8_t rtol_null, int hermitian) { + PROTECT( + auto outputs__ = torch::linalg_matrix_rank_out(*out, *self, atol_null ? c10::nullopt : c10::optional(atol_v), rtol_null ? c10::nullopt : c10::optional(rtol_v), (bool)hermitian); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_matrix_rank_atol_rtol_tensor(tensor *out__, tensor input, tensor atol, tensor rtol, int hermitian) { + PROTECT( + auto outputs__ = torch::linalg_matrix_rank(*input, (atol ? ::std::optional(*atol) : ::std::nullopt), (rtol ? ::std::optional(*rtol) : ::std::nullopt), (bool)hermitian); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_matrix_rank_atol_rtol_tensor_out(tensor *out__, tensor out, tensor input, tensor atol, tensor rtol, int hermitian) { + PROTECT( + auto outputs__ = torch::linalg_matrix_rank_out(*out, *input, (atol ? ::std::optional(*atol) : ::std::nullopt), (rtol ? ::std::optional(*rtol) : ::std::nullopt), (bool)hermitian); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_matrix_rank_out(tensor *out__, tensor out, tensor self, double tol, int hermitian) { + PROTECT( + auto outputs__ = torch::linalg_matrix_rank_out(*out, *self, tol, (bool)hermitian); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_matrix_rank_out_tol_tensor(tensor *out__, tensor out, tensor input, tensor tol, int hermitian) { + PROTECT( + auto outputs__ = torch::linalg_matrix_rank_out(*out, *input, *tol, (bool)hermitian); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_matrix_rank_tol_tensor(tensor *out__, tensor input, tensor tol, int hermitian) { + PROTECT( + auto outputs__ = torch::linalg_matrix_rank(*input, *tol, (bool)hermitian); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_multi_dot(tensor *out__, tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::linalg_multi_dot(of_carray_tensor(tensors_data, tensors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_multi_dot_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::linalg_multi_dot_out(*out, of_carray_tensor(tensors_data, tensors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_norm(tensor *out__, tensor self, scalar ord, int64_t *dim_data, int dim_len, int keepdim, int dtype) { + PROTECT( + auto outputs__ = torch::linalg_norm(*self, *ord, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_norm_ord_str(tensor *out__, tensor self, char* ord_ptr, int ord_len, int64_t *dim_data, int dim_len, int keepdim, int dtype) { + PROTECT( + auto outputs__ = torch::linalg_norm(*self, std::string(ord_ptr, ord_len), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_norm_ord_str_out(tensor *out__, tensor out, tensor self, char* ord_ptr, int ord_len, int64_t *dim_data, int dim_len, int keepdim, int dtype) { + PROTECT( + auto outputs__ = torch::linalg_norm_out(*out, *self, std::string(ord_ptr, ord_len), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_norm_out(tensor *out__, tensor out, tensor self, scalar ord, int64_t *dim_data, int dim_len, int keepdim, int dtype) { + PROTECT( + auto outputs__ = torch::linalg_norm_out(*out, *self, *ord, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_pinv(tensor *out__, tensor self, double rcond, int hermitian) { + PROTECT( + auto outputs__ = torch::linalg_pinv(*self, rcond, (bool)hermitian); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_pinv_atol_rtol_float(tensor *out__, tensor self, double atol_v, uint8_t atol_null, double rtol_v, uint8_t rtol_null, int hermitian) { + PROTECT( + auto outputs__ = torch::linalg_pinv(*self, atol_null ? c10::nullopt : c10::optional(atol_v), rtol_null ? c10::nullopt : c10::optional(rtol_v), (bool)hermitian); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_pinv_atol_rtol_float_out(tensor *out__, tensor out, tensor self, double atol_v, uint8_t atol_null, double rtol_v, uint8_t rtol_null, int hermitian) { + PROTECT( + auto outputs__ = torch::linalg_pinv_out(*out, *self, atol_null ? c10::nullopt : c10::optional(atol_v), rtol_null ? c10::nullopt : c10::optional(rtol_v), (bool)hermitian); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_pinv_atol_rtol_tensor(tensor *out__, tensor self, tensor atol, tensor rtol, int hermitian) { + PROTECT( + auto outputs__ = torch::linalg_pinv(*self, (atol ? ::std::optional(*atol) : ::std::nullopt), (rtol ? ::std::optional(*rtol) : ::std::nullopt), (bool)hermitian); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_pinv_atol_rtol_tensor_out(tensor *out__, tensor out, tensor self, tensor atol, tensor rtol, int hermitian) { + PROTECT( + auto outputs__ = torch::linalg_pinv_out(*out, *self, (atol ? ::std::optional(*atol) : ::std::nullopt), (rtol ? ::std::optional(*rtol) : ::std::nullopt), (bool)hermitian); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_pinv_out(tensor *out__, tensor out, tensor self, double rcond, int hermitian) { + PROTECT( + auto outputs__ = torch::linalg_pinv_out(*out, *self, rcond, (bool)hermitian); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_pinv_out_rcond_tensor(tensor *out__, tensor out, tensor self, tensor rcond, int hermitian) { + PROTECT( + auto outputs__ = torch::linalg_pinv_out(*out, *self, *rcond, (bool)hermitian); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_pinv_rcond_tensor(tensor *out__, tensor self, tensor rcond, int hermitian) { + PROTECT( + auto outputs__ = torch::linalg_pinv(*self, *rcond, (bool)hermitian); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_qr(tensor *out__, tensor A, char* mode_ptr, int mode_len) { + PROTECT( + auto outputs__ = torch::linalg_qr(*A, std::string(mode_ptr, mode_len)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_linalg_qr_out(tensor *out__, tensor Q, tensor R, tensor A, char* mode_ptr, int mode_len) { + PROTECT( + auto outputs__ = torch::linalg_qr_out(*Q, *R, *A, std::string(mode_ptr, mode_len)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_linalg_slogdet(tensor *out__, tensor A) { + PROTECT( + auto outputs__ = torch::linalg_slogdet(*A); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_linalg_slogdet_out(tensor *out__, tensor sign, tensor logabsdet, tensor A) { + PROTECT( + auto outputs__ = torch::linalg_slogdet_out(*sign, *logabsdet, *A); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_linalg_solve(tensor *out__, tensor A, tensor B, int left) { + PROTECT( + auto outputs__ = torch::linalg_solve(*A, *B, (bool)left); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_solve_ex(tensor *out__, tensor A, tensor B, int left, int check_errors) { + PROTECT( + auto outputs__ = torch::linalg_solve_ex(*A, *B, (bool)left, (bool)check_errors); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_linalg_solve_ex_out(tensor *out__, tensor result, tensor info, tensor A, tensor B, int left, int check_errors) { + PROTECT( + auto outputs__ = torch::linalg_solve_ex_out(*result, *info, *A, *B, (bool)left, (bool)check_errors); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_linalg_solve_out(tensor *out__, tensor out, tensor A, tensor B, int left) { + PROTECT( + auto outputs__ = torch::linalg_solve_out(*out, *A, *B, (bool)left); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_solve_triangular(tensor *out__, tensor self, tensor B, int upper, int left, int unitriangular) { + PROTECT( + auto outputs__ = torch::linalg_solve_triangular(*self, *B, (bool)upper, (bool)left, (bool)unitriangular); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_solve_triangular_out(tensor *out__, tensor out, tensor self, tensor B, int upper, int left, int unitriangular) { + PROTECT( + auto outputs__ = torch::linalg_solve_triangular_out(*out, *self, *B, (bool)upper, (bool)left, (bool)unitriangular); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_svd(tensor *out__, tensor A, int full_matrices, char* driver_ptr, int driver_len) { + PROTECT( + auto outputs__ = torch::linalg_svd(*A, (bool)full_matrices, std::string(driver_ptr, driver_len)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_linalg_svd_u(tensor *out__, tensor U, tensor S, tensor Vh, tensor A, int full_matrices, char* driver_ptr, int driver_len) { + PROTECT( + auto outputs__ = torch::linalg_svd_out(*U, *S, *Vh, *A, (bool)full_matrices, std::string(driver_ptr, driver_len)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_linalg_svdvals(tensor *out__, tensor A, char* driver_ptr, int driver_len) { + PROTECT( + auto outputs__ = torch::linalg_svdvals(*A, std::string(driver_ptr, driver_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_svdvals_out(tensor *out__, tensor out, tensor A, char* driver_ptr, int driver_len) { + PROTECT( + auto outputs__ = torch::linalg_svdvals_out(*out, *A, std::string(driver_ptr, driver_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_tensorinv(tensor *out__, tensor self, int64_t ind) { + PROTECT( + auto outputs__ = torch::linalg_tensorinv(*self, ind); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_tensorinv_out(tensor *out__, tensor out, tensor self, int64_t ind) { + PROTECT( + auto outputs__ = torch::linalg_tensorinv_out(*out, *self, ind); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_tensorsolve(tensor *out__, tensor self, tensor other, int64_t *dims_data, int dims_len) { + PROTECT( + auto outputs__ = torch::linalg_tensorsolve(*self, *other, dims_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dims_data, dims_len))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_tensorsolve_out(tensor *out__, tensor out, tensor self, tensor other, int64_t *dims_data, int dims_len) { + PROTECT( + auto outputs__ = torch::linalg_tensorsolve_out(*out, *self, *other, dims_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dims_data, dims_len))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_vander(tensor *out__, tensor x, int64_t n_v, uint8_t n_null) { + PROTECT( + auto outputs__ = torch::linalg_vander(*x, n_null ? c10::nullopt : c10::optional(n_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_vecdot(tensor *out__, tensor x, tensor y, int64_t dim) { + PROTECT( + auto outputs__ = torch::linalg_vecdot(*x, *y, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linalg_vecdot_out(tensor *out__, tensor out, tensor x, tensor y, int64_t dim) { + PROTECT( + auto outputs__ = torch::linalg_vecdot_out(*out, *x, *y, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linear(tensor *out__, tensor input, tensor weight, tensor bias) { + PROTECT( + auto outputs__ = torch::linear(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linear_out(tensor *out__, tensor out, tensor input, tensor weight, tensor bias) { + PROTECT( + auto outputs__ = torch::linear_out(*out, *input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linspace(tensor *out__, scalar start, scalar end, int64_t steps, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::linspace(*start, *end, steps, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linspace_out(tensor *out__, tensor out, scalar start, scalar end, int64_t steps) { + PROTECT( + auto outputs__ = torch::linspace_out(*out, *start, *end, steps); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linspace_scalar_tensor(tensor *out__, scalar start, tensor end, int64_t steps, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::linspace(*start, *end, steps, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linspace_scalar_tensor_out(tensor *out__, tensor out, scalar start, tensor end, int64_t steps) { + PROTECT( + auto outputs__ = torch::linspace_out(*out, *start, *end, steps); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linspace_tensor_scalar(tensor *out__, tensor start, scalar end, int64_t steps, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::linspace(*start, *end, steps, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linspace_tensor_scalar_out(tensor *out__, tensor out, tensor start, scalar end, int64_t steps) { + PROTECT( + auto outputs__ = torch::linspace_out(*out, *start, *end, steps); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linspace_tensor_tensor(tensor *out__, tensor start, tensor end, int64_t steps, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::linspace(*start, *end, steps, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_linspace_tensor_tensor_out(tensor *out__, tensor out, tensor start, tensor end, int64_t steps) { + PROTECT( + auto outputs__ = torch::linspace_out(*out, *start, *end, steps); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_log(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::log(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_log10(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::log10(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_log10_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::log10_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_log10_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::log10_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_log1p(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::log1p(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_log1p_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::log1p_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_log1p_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::log1p_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_log2(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::log2(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_log2_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::log2_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_log2_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::log2_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_log_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::log_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_log_normal(tensor *out__, tensor self, double mean, double std) { + PROTECT( + auto outputs__ = torch::log_normal(*self, mean, std); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_log_normal_(tensor *out__, tensor self, double mean, double std) { + PROTECT( + auto outputs__ = self->log_normal_(mean, std); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_log_normal_out(tensor *out__, tensor out, tensor self, double mean, double std) { + PROTECT( + auto outputs__ = torch::log_normal_out(*out, *self, mean, std); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_log_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::log_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_log_sigmoid(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::log_sigmoid(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_log_sigmoid_backward(tensor *out__, tensor grad_output, tensor self, tensor buffer) { + PROTECT( + auto outputs__ = torch::log_sigmoid_backward(*grad_output, *self, *buffer); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_log_sigmoid_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor buffer) { + PROTECT( + auto outputs__ = torch::log_sigmoid_backward_out(*grad_input, *grad_output, *self, *buffer); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_log_sigmoid_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::log_sigmoid_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_log_softmax(tensor *out__, tensor self, int64_t dim, int dtype) { + PROTECT( + auto outputs__ = torch::log_softmax(*self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_log_softmax_int_out(tensor *out__, tensor out, tensor self, int64_t dim, int dtype) { + PROTECT( + auto outputs__ = torch::log_softmax_out(*out, *self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logaddexp(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::logaddexp(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logaddexp2(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::logaddexp2(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logaddexp2_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::logaddexp2_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logaddexp_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::logaddexp_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logcumsumexp(tensor *out__, tensor self, int64_t dim) { + PROTECT( + auto outputs__ = torch::logcumsumexp(*self, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logcumsumexp_out(tensor *out__, tensor out, tensor self, int64_t dim) { + PROTECT( + auto outputs__ = torch::logcumsumexp_out(*out, *self, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logdet(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::logdet(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logical_and(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::logical_and(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logical_and_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->logical_and_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logical_and_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::logical_and_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logical_not(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::logical_not(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logical_not_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->logical_not_(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logical_not_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::logical_not_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logical_or(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::logical_or(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logical_or_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->logical_or_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logical_or_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::logical_or_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logical_xor(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::logical_xor(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logical_xor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->logical_xor_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logical_xor_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::logical_xor_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logit(tensor *out__, tensor self, double eps_v, uint8_t eps_null) { + PROTECT( + auto outputs__ = torch::logit(*self, eps_null ? c10::nullopt : c10::optional(eps_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logit_(tensor *out__, tensor self, double eps_v, uint8_t eps_null) { + PROTECT( + auto outputs__ = torch::logit_(*self, eps_null ? c10::nullopt : c10::optional(eps_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logit_backward(tensor *out__, tensor grad_output, tensor self, double eps_v, uint8_t eps_null) { + PROTECT( + auto outputs__ = torch::logit_backward(*grad_output, *self, eps_null ? c10::nullopt : c10::optional(eps_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logit_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, double eps_v, uint8_t eps_null) { + PROTECT( + auto outputs__ = torch::logit_backward_out(*grad_input, *grad_output, *self, eps_null ? c10::nullopt : c10::optional(eps_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logit_out(tensor *out__, tensor out, tensor self, double eps_v, uint8_t eps_null) { + PROTECT( + auto outputs__ = torch::logit_out(*out, *self, eps_null ? c10::nullopt : c10::optional(eps_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logspace(tensor *out__, scalar start, scalar end, int64_t steps, double base, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::logspace(*start, *end, steps, base, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logspace_out(tensor *out__, tensor out, scalar start, scalar end, int64_t steps, double base) { + PROTECT( + auto outputs__ = torch::logspace_out(*out, *start, *end, steps, base); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logspace_scalar_tensor(tensor *out__, scalar start, tensor end, int64_t steps, double base, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::logspace(*start, *end, steps, base, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logspace_scalar_tensor_out(tensor *out__, tensor out, scalar start, tensor end, int64_t steps, double base) { + PROTECT( + auto outputs__ = torch::logspace_out(*out, *start, *end, steps, base); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logspace_tensor_scalar(tensor *out__, tensor start, scalar end, int64_t steps, double base, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::logspace(*start, *end, steps, base, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logspace_tensor_scalar_out(tensor *out__, tensor out, tensor start, scalar end, int64_t steps, double base) { + PROTECT( + auto outputs__ = torch::logspace_out(*out, *start, *end, steps, base); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logspace_tensor_tensor(tensor *out__, tensor start, tensor end, int64_t steps, double base, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::logspace(*start, *end, steps, base, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logspace_tensor_tensor_out(tensor *out__, tensor out, tensor start, tensor end, int64_t steps, double base) { + PROTECT( + auto outputs__ = torch::logspace_out(*out, *start, *end, steps, base); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logsumexp(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim) { + PROTECT( + auto outputs__ = torch::logsumexp(*self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_logsumexp_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim) { + PROTECT( + auto outputs__ = torch::logsumexp_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lstm(tensor *out__, tensor input, tensor *hx_data, int hx_len, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first) { + PROTECT( + auto outputs__ = torch::lstm(*input, of_carray_tensor(hx_data, hx_len), of_carray_tensor(params_data, params_len), (bool)has_biases, num_layers, dropout, (bool)train, (bool)bidirectional, (bool)batch_first); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_lstm_cell(tensor *out__, tensor input, tensor *hx_data, int hx_len, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh) { + PROTECT( + auto outputs__ = torch::lstm_cell(*input, of_carray_tensor(hx_data, hx_len), *w_ih, *w_hh, (b_ih ? ::std::optional(*b_ih) : ::std::nullopt), (b_hh ? ::std::optional(*b_hh) : ::std::nullopt)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_lstm_data(tensor *out__, tensor data, tensor batch_sizes, tensor *hx_data, int hx_len, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional) { + PROTECT( + auto outputs__ = torch::lstm(*data, *batch_sizes, of_carray_tensor(hx_data, hx_len), of_carray_tensor(params_data, params_len), (bool)has_biases, num_layers, dropout, (bool)train, (bool)bidirectional); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_lstm_mps_backward(tensor out0, tensor *out1_data, int out1_len, tensor *out2_data, int out2_len, tensor grad_y, tensor grad_hy, tensor grad_cy, tensor z_state, tensor cell_state_fwd, tensor input, tensor layersOutputs, tensor *hx_data, int hx_len, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first) { + PROTECT( + torch::lstm_mps_backward_out(*out0, of_carray_tensor(out1_data, out1_len), of_carray_tensor(out2_data, out2_len), (grad_y ? ::std::optional(*grad_y) : ::std::nullopt), (grad_hy ? ::std::optional(*grad_hy) : ::std::nullopt), (grad_cy ? ::std::optional(*grad_cy) : ::std::nullopt), *z_state, *cell_state_fwd, *input, *layersOutputs, of_carray_tensor(hx_data, hx_len), of_carray_tensor(params_data, params_len), (bool)has_biases, num_layers, dropout, (bool)train, (bool)bidirectional, (bool)batch_first); + ) +} + +void atg_lt(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::lt(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lt_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->lt_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lt_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::lt_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lt_tensor(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::lt(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lt_tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->lt_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lt_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::lt_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lu_solve(tensor *out__, tensor self, tensor LU_data, tensor LU_pivots) { + PROTECT( + auto outputs__ = torch::lu_solve(*self, *LU_data, *LU_pivots); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lu_solve_out(tensor *out__, tensor out, tensor self, tensor LU_data, tensor LU_pivots) { + PROTECT( + auto outputs__ = torch::lu_solve_out(*out, *self, *LU_data, *LU_pivots); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_lu_unpack(tensor *out__, tensor LU_data, tensor LU_pivots, int unpack_data, int unpack_pivots) { + PROTECT( + auto outputs__ = torch::lu_unpack(*LU_data, *LU_pivots, (bool)unpack_data, (bool)unpack_pivots); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_lu_unpack_out(tensor *out__, tensor P, tensor L, tensor U, tensor LU_data, tensor LU_pivots, int unpack_data, int unpack_pivots) { + PROTECT( + auto outputs__ = torch::lu_unpack_out(*P, *L, *U, *LU_data, *LU_pivots, (bool)unpack_data, (bool)unpack_pivots); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_margin_ranking_loss(tensor *out__, tensor input1, tensor input2, tensor target, double margin, int64_t reduction) { + PROTECT( + auto outputs__ = torch::margin_ranking_loss(*input1, *input2, *target, margin, reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_masked_fill(tensor *out__, tensor self, tensor mask, scalar value) { + PROTECT( + auto outputs__ = torch::masked_fill(*self, *mask, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_masked_fill_(tensor *out__, tensor self, tensor mask, scalar value) { + PROTECT( + auto outputs__ = self->masked_fill_(*mask, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_masked_fill_scalar_out(tensor *out__, tensor out, tensor self, tensor mask, scalar value) { + PROTECT( + auto outputs__ = torch::masked_fill_out(*out, *self, *mask, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_masked_fill_tensor(tensor *out__, tensor self, tensor mask, tensor value) { + PROTECT( + auto outputs__ = torch::masked_fill(*self, *mask, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_masked_fill_tensor_(tensor *out__, tensor self, tensor mask, tensor value) { + PROTECT( + auto outputs__ = self->masked_fill_(*mask, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_masked_fill_tensor_out(tensor *out__, tensor out, tensor self, tensor mask, tensor value) { + PROTECT( + auto outputs__ = torch::masked_fill_out(*out, *self, *mask, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_masked_scatter(tensor *out__, tensor self, tensor mask, tensor source) { + PROTECT( + auto outputs__ = torch::masked_scatter(*self, *mask, *source); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_masked_scatter_(tensor *out__, tensor self, tensor mask, tensor source) { + PROTECT( + auto outputs__ = self->masked_scatter_(*mask, *source); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_masked_scatter_backward(tensor *out__, tensor grad_output, tensor mask, int64_t *sizes_data, int sizes_len) { + PROTECT( + auto outputs__ = torch::masked_scatter_backward(*grad_output, *mask, torch::IntArrayRef(sizes_data, sizes_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_masked_scatter_out(tensor *out__, tensor out, tensor self, tensor mask, tensor source) { + PROTECT( + auto outputs__ = torch::masked_scatter_out(*out, *self, *mask, *source); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_masked_select(tensor *out__, tensor self, tensor mask) { + PROTECT( + auto outputs__ = torch::masked_select(*self, *mask); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_masked_select_backward(tensor *out__, tensor grad, tensor input, tensor mask) { + PROTECT( + auto outputs__ = torch::masked_select_backward(*grad, *input, *mask); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_masked_select_out(tensor *out__, tensor out, tensor self, tensor mask) { + PROTECT( + auto outputs__ = torch::masked_select_out(*out, *self, *mask); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_matmul(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::matmul(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_matmul_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::matmul_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_matrix_exp(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::matrix_exp(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_matrix_exp_backward(tensor *out__, tensor self, tensor grad) { + PROTECT( + auto outputs__ = torch::matrix_exp_backward(*self, *grad); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_matrix_h(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->matrix_H(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_matrix_power(tensor *out__, tensor self, int64_t n) { + PROTECT( + auto outputs__ = torch::matrix_power(*self, n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_matrix_power_out(tensor *out__, tensor out, tensor self, int64_t n) { + PROTECT( + auto outputs__ = torch::matrix_power_out(*out, *self, n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_max(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::max(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_max_dim(tensor *out__, tensor self, int64_t dim, int keepdim) { + PROTECT( + auto outputs__ = torch::max(*self, dim, (bool)keepdim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_max_dim_max(tensor *out__, tensor max, tensor max_values, tensor self, int64_t dim, int keepdim) { + PROTECT( + auto outputs__ = torch::max_out(*max, *max_values, *self, dim, (bool)keepdim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_max_other(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::max(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_max_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::max_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_max_pool1d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::max_pool1d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_max_pool1d_with_indices(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::max_pool1d_with_indices(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_max_pool2d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::max_pool2d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_max_pool2d_backward(tensor *out__, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::max_pool2d_backward(*grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_max_pool2d_backward_out(tensor *out__, tensor out, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::max_pool2d_backward_out(*out, *grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_max_pool2d_with_indices(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::max_pool2d_with_indices(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_max_pool2d_with_indices_backward(tensor *out__, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode, tensor indices) { + PROTECT( + auto outputs__ = torch::max_pool2d_with_indices_backward(*grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode, *indices); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_max_pool2d_with_indices_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode, tensor indices) { + PROTECT( + auto outputs__ = torch::max_pool2d_with_indices_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode, *indices); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_max_pool2d_with_indices_out(tensor *out__, tensor out, tensor indices, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::max_pool2d_with_indices_out(*out, *indices, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_max_pool3d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::max_pool3d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_max_pool3d_with_indices(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::max_pool3d_with_indices(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_max_pool3d_with_indices_backward(tensor *out__, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode, tensor indices) { + PROTECT( + auto outputs__ = torch::max_pool3d_with_indices_backward(*grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode, *indices); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_max_pool3d_with_indices_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode, tensor indices) { + PROTECT( + auto outputs__ = torch::max_pool3d_with_indices_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode, *indices); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_max_pool3d_with_indices_out(tensor *out__, tensor out, tensor indices, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::max_pool3d_with_indices_out(*out, *indices, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_max_unary_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::max_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_max_unpool2d(tensor *out__, tensor self, tensor indices, int64_t *output_size_data, int output_size_len) { + PROTECT( + auto outputs__ = torch::max_unpool2d(*self, *indices, torch::IntArrayRef(output_size_data, output_size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_max_unpool2d_out(tensor *out__, tensor out, tensor self, tensor indices, int64_t *output_size_data, int output_size_len) { + PROTECT( + auto outputs__ = torch::max_unpool2d_out(*out, *self, *indices, torch::IntArrayRef(output_size_data, output_size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_max_unpool3d(tensor *out__, tensor self, tensor indices, int64_t *output_size_data, int output_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::max_unpool3d(*self, *indices, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_max_unpool3d_out(tensor *out__, tensor out, tensor self, tensor indices, int64_t *output_size_data, int output_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::max_unpool3d_out(*out, *self, *indices, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_maximum(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::maximum(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_maximum_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::maximum_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mean(tensor *out__, tensor self, int dtype) { + PROTECT( + auto outputs__ = torch::mean(*self, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mean_dim(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { + PROTECT( + auto outputs__ = torch::mean(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mean_dtype_out(tensor *out__, tensor out, tensor self, int dtype) { + PROTECT( + auto outputs__ = torch::mean_out(*out, *self, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mean_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { + PROTECT( + auto outputs__ = torch::mean_out(*out, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_median(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::median(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_median_dim(tensor *out__, tensor self, int64_t dim, int keepdim) { + PROTECT( + auto outputs__ = torch::median(*self, dim, (bool)keepdim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_median_dim_values(tensor *out__, tensor values, tensor indices, tensor self, int64_t dim, int keepdim) { + PROTECT( + auto outputs__ = torch::median_out(*values, *indices, *self, dim, (bool)keepdim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_median_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::median_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +tensor *atg_meshgrid(tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::meshgrid(of_carray_tensor(tensors_data, tensors_len)); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +tensor *atg_meshgrid_indexing(tensor *tensors_data, int tensors_len, char* indexing_ptr, int indexing_len) { + PROTECT( + auto outputs__ = torch::meshgrid(of_carray_tensor(tensors_data, tensors_len), std::string(indexing_ptr, indexing_len)); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg_mh(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->mH(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_min(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::min(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_min_dim(tensor *out__, tensor self, int64_t dim, int keepdim) { + PROTECT( + auto outputs__ = torch::min(*self, dim, (bool)keepdim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_min_dim_min(tensor *out__, tensor min, tensor min_indices, tensor self, int64_t dim, int keepdim) { + PROTECT( + auto outputs__ = torch::min_out(*min, *min_indices, *self, dim, (bool)keepdim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_min_other(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::min(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_min_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::min_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_min_unary_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::min_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_minimum(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::minimum(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_minimum_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::minimum_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_miopen_batch_norm(tensor *out__, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double exponential_average_factor, double epsilon) { + PROTECT( + auto outputs__ = torch::miopen_batch_norm(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (bool)training, exponential_average_factor, epsilon); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_miopen_batch_norm_backward(tensor *out__, tensor input, tensor grad_output, tensor weight, tensor running_mean, tensor running_var, tensor save_mean, tensor save_var, double epsilon) { + PROTECT( + auto outputs__ = torch::miopen_batch_norm_backward(*input, *grad_output, *weight, (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (save_mean ? ::std::optional(*save_mean) : ::std::nullopt), (save_var ? ::std::optional(*save_var) : ::std::nullopt), epsilon); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_miopen_batch_norm_backward_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor input, tensor grad_output, tensor weight, tensor running_mean, tensor running_var, tensor save_mean, tensor save_var, double epsilon) { + PROTECT( + auto outputs__ = torch::miopen_batch_norm_backward_out(*out0, *out1, *out2, *input, *grad_output, *weight, (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (save_mean ? ::std::optional(*save_mean) : ::std::nullopt), (save_var ? ::std::optional(*save_var) : ::std::nullopt), epsilon); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_miopen_batch_norm_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double exponential_average_factor, double epsilon) { + PROTECT( + auto outputs__ = torch::miopen_batch_norm_out(*out0, *out1, *out2, *input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (bool)training, exponential_average_factor, epsilon); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_miopen_convolution(tensor *out__, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic) { + PROTECT( + auto outputs__ = torch::miopen_convolution(*self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, (bool)benchmark, (bool)deterministic); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_miopen_convolution_add_relu(tensor *out__, tensor self, tensor weight, tensor z, scalar alpha, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::miopen_convolution_add_relu(*self, *weight, *z, *alpha, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_miopen_convolution_out(tensor *out__, tensor out, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic) { + PROTECT( + auto outputs__ = torch::miopen_convolution_out(*out, *self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, (bool)benchmark, (bool)deterministic); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_miopen_convolution_relu(tensor *out__, tensor self, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::miopen_convolution_relu(*self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_miopen_convolution_transpose(tensor *out__, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic) { + PROTECT( + auto outputs__ = torch::miopen_convolution_transpose(*self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, (bool)benchmark, (bool)deterministic); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_miopen_convolution_transpose_out(tensor *out__, tensor out, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic) { + PROTECT( + auto outputs__ = torch::miopen_convolution_transpose_out(*out, *self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, (bool)benchmark, (bool)deterministic); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_miopen_ctc_loss(tensor *out__, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int deterministic, int zero_infinity) { + PROTECT( + auto outputs__ = torch::miopen_ctc_loss(*log_probs, *targets, torch::IntArrayRef(input_lengths_data, input_lengths_len), torch::IntArrayRef(target_lengths_data, target_lengths_len), blank, (bool)deterministic, (bool)zero_infinity); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_miopen_ctc_loss_out(tensor *out__, tensor out0, tensor out1, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int deterministic, int zero_infinity) { + PROTECT( + auto outputs__ = torch::miopen_ctc_loss_out(*out0, *out1, *log_probs, *targets, torch::IntArrayRef(input_lengths_data, input_lengths_len), torch::IntArrayRef(target_lengths_data, target_lengths_len), blank, (bool)deterministic, (bool)zero_infinity); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_miopen_ctc_loss_tensor(tensor *out__, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank, int deterministic, int zero_infinity) { + PROTECT( + auto outputs__ = torch::miopen_ctc_loss(*log_probs, *targets, *input_lengths, *target_lengths, blank, (bool)deterministic, (bool)zero_infinity); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_miopen_depthwise_convolution(tensor *out__, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic) { + PROTECT( + auto outputs__ = torch::miopen_depthwise_convolution(*self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, (bool)benchmark, (bool)deterministic); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_miopen_depthwise_convolution_out(tensor *out__, tensor out, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic) { + PROTECT( + auto outputs__ = torch::miopen_depthwise_convolution_out(*out, *self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, (bool)benchmark, (bool)deterministic); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_miopen_rnn(tensor *out__, tensor input, tensor *weight_data, int weight_len, int64_t weight_stride0, tensor hx, tensor cx, int64_t mode, int64_t hidden_size, int64_t num_layers, int batch_first, double dropout, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, tensor dropout_state) { + PROTECT( + auto outputs__ = torch::miopen_rnn(*input, of_carray_tensor(weight_data, weight_len), weight_stride0, *hx, (cx ? ::std::optional(*cx) : ::std::nullopt), mode, hidden_size, num_layers, (bool)batch_first, dropout, (bool)train, (bool)bidirectional, torch::IntArrayRef(batch_sizes_data, batch_sizes_len), (dropout_state ? ::std::optional(*dropout_state) : ::std::nullopt)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + out__[4] = new torch::Tensor(std::get<4>(outputs__)); + ) +} + +void atg_miopen_rnn_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor out3, tensor out4, tensor input, tensor *weight_data, int weight_len, int64_t weight_stride0, tensor hx, tensor cx, int64_t mode, int64_t hidden_size, int64_t num_layers, int batch_first, double dropout, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, tensor dropout_state) { + PROTECT( + auto outputs__ = torch::miopen_rnn_out(*out0, *out1, *out2, *out3, *out4, *input, of_carray_tensor(weight_data, weight_len), weight_stride0, *hx, (cx ? ::std::optional(*cx) : ::std::nullopt), mode, hidden_size, num_layers, (bool)batch_first, dropout, (bool)train, (bool)bidirectional, torch::IntArrayRef(batch_sizes_data, batch_sizes_len), (dropout_state ? ::std::optional(*dropout_state) : ::std::nullopt)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + out__[4] = new torch::Tensor(std::get<4>(outputs__)); + ) +} + +void atg_mish(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::mish(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mish_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::mish_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mish_backward(tensor *out__, tensor grad_output, tensor self) { + PROTECT( + auto outputs__ = torch::mish_backward(*grad_output, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mish_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::mish_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_adaptive_avg_pool2d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len) { + PROTECT( + auto outputs__ = torch::mkldnn_adaptive_avg_pool2d(*self, torch::IntArrayRef(output_size_data, output_size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_adaptive_avg_pool2d_backward(tensor *out__, tensor grad_output, tensor self) { + PROTECT( + auto outputs__ = torch::mkldnn_adaptive_avg_pool2d_backward(*grad_output, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_adaptive_avg_pool2d_backward_out(tensor *out__, tensor out, tensor grad_output, tensor self) { + PROTECT( + auto outputs__ = torch::mkldnn_adaptive_avg_pool2d_backward_out(*out, *grad_output, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_adaptive_avg_pool2d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len) { + PROTECT( + auto outputs__ = torch::mkldnn_adaptive_avg_pool2d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_convolution(tensor *out__, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::mkldnn_convolution(*self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_convolution_out(tensor *out__, tensor out, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups) { + PROTECT( + auto outputs__ = torch::mkldnn_convolution_out(*out, *self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_linear(tensor *out__, tensor self, tensor weight, tensor bias) { + PROTECT( + auto outputs__ = torch::mkldnn_linear(*self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_linear_backward_input(tensor *out__, int64_t *input_size_data, int input_size_len, tensor grad_output, tensor weight) { + PROTECT( + auto outputs__ = torch::mkldnn_linear_backward_input(torch::IntArrayRef(input_size_data, input_size_len), *grad_output, *weight); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_linear_backward_input_out(tensor *out__, tensor out, int64_t *input_size_data, int input_size_len, tensor grad_output, tensor weight) { + PROTECT( + auto outputs__ = torch::mkldnn_linear_backward_input_out(*out, torch::IntArrayRef(input_size_data, input_size_len), *grad_output, *weight); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_linear_backward_weights(tensor *out__, tensor grad_output, tensor input, tensor weight, int bias_defined) { + PROTECT( + auto outputs__ = torch::mkldnn_linear_backward_weights(*grad_output, *input, *weight, (bool)bias_defined); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_mkldnn_linear_backward_weights_out(tensor *out__, tensor out0, tensor out1, tensor grad_output, tensor input, tensor weight, int bias_defined) { + PROTECT( + auto outputs__ = torch::mkldnn_linear_backward_weights_out(*out0, *out1, *grad_output, *input, *weight, (bool)bias_defined); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_mkldnn_linear_out(tensor *out__, tensor out, tensor self, tensor weight, tensor bias) { + PROTECT( + auto outputs__ = torch::mkldnn_linear_out(*out, *self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_max_pool2d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::mkldnn_max_pool2d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_max_pool2d_backward(tensor *out__, tensor grad_output, tensor output, tensor input, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::mkldnn_max_pool2d_backward(*grad_output, *output, *input, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_max_pool2d_backward_out(tensor *out__, tensor out, tensor grad_output, tensor output, tensor input, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::mkldnn_max_pool2d_backward_out(*out, *grad_output, *output, *input, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_max_pool2d_out(tensor *out__, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::mkldnn_max_pool2d_out(*out, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_max_pool3d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::mkldnn_max_pool3d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_max_pool3d_backward(tensor *out__, tensor grad_output, tensor output, tensor input, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::mkldnn_max_pool3d_backward(*grad_output, *output, *input, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_max_pool3d_backward_out(tensor *out__, tensor out, tensor grad_output, tensor output, tensor input, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::mkldnn_max_pool3d_backward_out(*out, *grad_output, *output, *input, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_max_pool3d_out(tensor *out__, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::mkldnn_max_pool3d_out(*out, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_reorder_conv2d_weight(tensor *out__, tensor self, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int64_t *input_size_data, int input_size_len) { + PROTECT( + auto outputs__ = torch::mkldnn_reorder_conv2d_weight(*self, torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, input_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(input_size_data, input_size_len))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_reorder_conv2d_weight_out(tensor *out__, tensor out, tensor self, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int64_t *input_size_data, int input_size_len) { + PROTECT( + auto outputs__ = torch::mkldnn_reorder_conv2d_weight_out(*out, *self, torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, input_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(input_size_data, input_size_len))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_reorder_conv3d_weight(tensor *out__, tensor self, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int64_t *input_size_data, int input_size_len) { + PROTECT( + auto outputs__ = torch::mkldnn_reorder_conv3d_weight(*self, torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, input_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(input_size_data, input_size_len))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_reorder_conv3d_weight_out(tensor *out__, tensor out, tensor self, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int64_t *input_size_data, int input_size_len) { + PROTECT( + auto outputs__ = torch::mkldnn_reorder_conv3d_weight_out(*out, *self, torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, input_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(input_size_data, input_size_len))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mkldnn_rnn_layer(tensor *out__, tensor input, tensor weight0, tensor weight1, tensor weight2, tensor weight3, tensor hx_, tensor cx_, int reverse, int64_t *batch_sizes_data, int batch_sizes_len, int64_t mode, int64_t hidden_size, int64_t num_layers, int has_biases, int bidirectional, int batch_first, int train) { + PROTECT( + auto outputs__ = torch::mkldnn_rnn_layer(*input, *weight0, *weight1, *weight2, *weight3, *hx_, *cx_, (bool)reverse, torch::IntArrayRef(batch_sizes_data, batch_sizes_len), mode, hidden_size, num_layers, (bool)has_biases, (bool)bidirectional, (bool)batch_first, (bool)train); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg_mkldnn_rnn_layer_backward(tensor *out__, tensor input, tensor weight1, tensor weight2, tensor weight3, tensor weight4, tensor hx_, tensor cx_tmp, tensor output, tensor hy_, tensor cy_, tensor grad_output, tensor grad_hy, tensor grad_cy, int reverse, int64_t mode, int64_t hidden_size, int64_t num_layers, int has_biases, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, int batch_first, tensor workspace) { + PROTECT( + auto outputs__ = torch::mkldnn_rnn_layer_backward(*input, *weight1, *weight2, *weight3, *weight4, *hx_, *cx_tmp, *output, *hy_, *cy_, (grad_output ? ::std::optional(*grad_output) : ::std::nullopt), (grad_hy ? ::std::optional(*grad_hy) : ::std::nullopt), (grad_cy ? ::std::optional(*grad_cy) : ::std::nullopt), (bool)reverse, mode, hidden_size, num_layers, (bool)has_biases, (bool)train, (bool)bidirectional, torch::IntArrayRef(batch_sizes_data, batch_sizes_len), (bool)batch_first, *workspace); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + out__[4] = new torch::Tensor(std::get<4>(outputs__)); + out__[5] = new torch::Tensor(std::get<5>(outputs__)); + out__[6] = new torch::Tensor(std::get<6>(outputs__)); + ) +} + +void atg_mkldnn_rnn_layer_backward_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor out3, tensor out4, tensor out5, tensor out6, tensor input, tensor weight1, tensor weight2, tensor weight3, tensor weight4, tensor hx_, tensor cx_tmp, tensor output, tensor hy_, tensor cy_, tensor grad_output, tensor grad_hy, tensor grad_cy, int reverse, int64_t mode, int64_t hidden_size, int64_t num_layers, int has_biases, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, int batch_first, tensor workspace) { + PROTECT( + auto outputs__ = torch::mkldnn_rnn_layer_backward_out(*out0, *out1, *out2, *out3, *out4, *out5, *out6, *input, *weight1, *weight2, *weight3, *weight4, *hx_, *cx_tmp, *output, *hy_, *cy_, (grad_output ? ::std::optional(*grad_output) : ::std::nullopt), (grad_hy ? ::std::optional(*grad_hy) : ::std::nullopt), (grad_cy ? ::std::optional(*grad_cy) : ::std::nullopt), (bool)reverse, mode, hidden_size, num_layers, (bool)has_biases, (bool)train, (bool)bidirectional, torch::IntArrayRef(batch_sizes_data, batch_sizes_len), (bool)batch_first, *workspace); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + out__[4] = new torch::Tensor(std::get<4>(outputs__)); + out__[5] = new torch::Tensor(std::get<5>(outputs__)); + out__[6] = new torch::Tensor(std::get<6>(outputs__)); + ) +} + +void atg_mkldnn_rnn_layer_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor out3, tensor input, tensor weight0, tensor weight1, tensor weight2, tensor weight3, tensor hx_, tensor cx_, int reverse, int64_t *batch_sizes_data, int batch_sizes_len, int64_t mode, int64_t hidden_size, int64_t num_layers, int has_biases, int bidirectional, int batch_first, int train) { + PROTECT( + auto outputs__ = torch::mkldnn_rnn_layer_out(*out0, *out1, *out2, *out3, *input, *weight0, *weight1, *weight2, *weight3, *hx_, *cx_, (bool)reverse, torch::IntArrayRef(batch_sizes_data, batch_sizes_len), mode, hidden_size, num_layers, (bool)has_biases, (bool)bidirectional, (bool)batch_first, (bool)train); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + out__[3] = new torch::Tensor(std::get<3>(outputs__)); + ) +} + +void atg_mm(tensor *out__, tensor self, tensor mat2) { + PROTECT( + auto outputs__ = torch::mm(*self, *mat2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mm_dtype(tensor *out__, tensor self, tensor mat2, int out_dtype) { + PROTECT( + auto outputs__ = torch::mm(*self, *mat2, at::ScalarType(out_dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mm_dtype_out(tensor *out__, tensor out, tensor self, tensor mat2, int out_dtype) { + PROTECT( + auto outputs__ = torch::mm_out(*out, *self, *mat2, at::ScalarType(out_dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mm_out(tensor *out__, tensor out, tensor self, tensor mat2) { + PROTECT( + auto outputs__ = torch::mm_out(*out, *self, *mat2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mode(tensor *out__, tensor self, int64_t dim, int keepdim) { + PROTECT( + auto outputs__ = torch::mode(*self, dim, (bool)keepdim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_mode_values(tensor *out__, tensor values, tensor indices, tensor self, int64_t dim, int keepdim) { + PROTECT( + auto outputs__ = torch::mode_out(*values, *indices, *self, dim, (bool)keepdim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_moveaxis(tensor *out__, tensor self, int64_t *source_data, int source_len, int64_t *destination_data, int destination_len) { + PROTECT( + auto outputs__ = torch::moveaxis(*self, torch::IntArrayRef(source_data, source_len), torch::IntArrayRef(destination_data, destination_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_moveaxis_int(tensor *out__, tensor self, int64_t source, int64_t destination) { + PROTECT( + auto outputs__ = torch::moveaxis(*self, source, destination); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_movedim(tensor *out__, tensor self, int64_t *source_data, int source_len, int64_t *destination_data, int destination_len) { + PROTECT( + auto outputs__ = torch::movedim(*self, torch::IntArrayRef(source_data, source_len), torch::IntArrayRef(destination_data, destination_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_movedim_int(tensor *out__, tensor self, int64_t source, int64_t destination) { + PROTECT( + auto outputs__ = torch::movedim(*self, source, destination); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mse_loss(tensor *out__, tensor self, tensor target, int64_t reduction) { + PROTECT( + auto outputs__ = torch::mse_loss(*self, *target, reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mse_loss_backward(tensor *out__, tensor grad_output, tensor self, tensor target, int64_t reduction) { + PROTECT( + auto outputs__ = torch::mse_loss_backward(*grad_output, *self, *target, reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mse_loss_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor target, int64_t reduction) { + PROTECT( + auto outputs__ = torch::mse_loss_backward_out(*grad_input, *grad_output, *self, *target, reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mse_loss_out(tensor *out__, tensor out, tensor self, tensor target, int64_t reduction) { + PROTECT( + auto outputs__ = torch::mse_loss_out(*out, *self, *target, reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_msort(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::msort(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_msort_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::msort_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mt(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->mT(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mul(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::mul(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mul_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->mul_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mul_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::mul_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mul_scalar(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::mul(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mul_scalar_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->mul_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mul_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::mul_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_multi_margin_loss_backward(tensor *out__, tensor grad_output, tensor self, tensor target, scalar p, scalar margin, tensor weight, int64_t reduction) { + PROTECT( + auto outputs__ = torch::multi_margin_loss_backward(*grad_output, *self, *target, *p, *margin, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_multi_margin_loss_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor target, scalar p, scalar margin, tensor weight, int64_t reduction) { + PROTECT( + auto outputs__ = torch::multi_margin_loss_backward_out(*grad_input, *grad_output, *self, *target, *p, *margin, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_multilabel_margin_loss(tensor *out__, tensor self, tensor target, int64_t reduction) { + PROTECT( + auto outputs__ = torch::multilabel_margin_loss(*self, *target, reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_multilabel_margin_loss_backward(tensor *out__, tensor grad_output, tensor self, tensor target, int64_t reduction, tensor is_target) { + PROTECT( + auto outputs__ = torch::multilabel_margin_loss_backward(*grad_output, *self, *target, reduction, *is_target); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_multilabel_margin_loss_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor target, int64_t reduction, tensor is_target) { + PROTECT( + auto outputs__ = torch::multilabel_margin_loss_backward_out(*grad_input, *grad_output, *self, *target, reduction, *is_target); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_multilabel_margin_loss_out(tensor *out__, tensor out, tensor self, tensor target, int64_t reduction) { + PROTECT( + auto outputs__ = torch::multilabel_margin_loss_out(*out, *self, *target, reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_multinomial(tensor *out__, tensor self, int64_t num_samples, int replacement) { + PROTECT( + auto outputs__ = torch::multinomial(*self, num_samples, (bool)replacement); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_multinomial_out(tensor *out__, tensor out, tensor self, int64_t num_samples, int replacement) { + PROTECT( + auto outputs__ = torch::multinomial_out(*out, *self, num_samples, (bool)replacement); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_multiply(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::multiply(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_multiply_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->multiply_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_multiply_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::multiply_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_multiply_scalar(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::multiply(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_multiply_scalar_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->multiply_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mv(tensor *out__, tensor self, tensor vec) { + PROTECT( + auto outputs__ = torch::mv(*self, *vec); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mv_out(tensor *out__, tensor out, tensor self, tensor vec) { + PROTECT( + auto outputs__ = torch::mv_out(*out, *self, *vec); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mvlgamma(tensor *out__, tensor self, int64_t p) { + PROTECT( + auto outputs__ = torch::mvlgamma(*self, p); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mvlgamma_(tensor *out__, tensor self, int64_t p) { + PROTECT( + auto outputs__ = self->mvlgamma_(p); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_mvlgamma_out(tensor *out__, tensor out, tensor self, int64_t p) { + PROTECT( + auto outputs__ = torch::mvlgamma_out(*out, *self, p); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nan_to_num(tensor *out__, tensor self, double nan_v, uint8_t nan_null, double posinf_v, uint8_t posinf_null, double neginf_v, uint8_t neginf_null) { + PROTECT( + auto outputs__ = torch::nan_to_num(*self, nan_null ? c10::nullopt : c10::optional(nan_v), posinf_null ? c10::nullopt : c10::optional(posinf_v), neginf_null ? c10::nullopt : c10::optional(neginf_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nan_to_num_(tensor *out__, tensor self, double nan_v, uint8_t nan_null, double posinf_v, uint8_t posinf_null, double neginf_v, uint8_t neginf_null) { + PROTECT( + auto outputs__ = torch::nan_to_num_(*self, nan_null ? c10::nullopt : c10::optional(nan_v), posinf_null ? c10::nullopt : c10::optional(posinf_v), neginf_null ? c10::nullopt : c10::optional(neginf_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nan_to_num_out(tensor *out__, tensor out, tensor self, double nan_v, uint8_t nan_null, double posinf_v, uint8_t posinf_null, double neginf_v, uint8_t neginf_null) { + PROTECT( + auto outputs__ = torch::nan_to_num_out(*out, *self, nan_null ? c10::nullopt : c10::optional(nan_v), posinf_null ? c10::nullopt : c10::optional(posinf_v), neginf_null ? c10::nullopt : c10::optional(neginf_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nanmean(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { + PROTECT( + auto outputs__ = torch::nanmean(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nanmean_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { + PROTECT( + auto outputs__ = torch::nanmean_out(*out, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nanmedian(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::nanmedian(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nanmedian_dim(tensor *out__, tensor self, int64_t dim, int keepdim) { + PROTECT( + auto outputs__ = torch::nanmedian(*self, dim, (bool)keepdim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_nanmedian_dim_values(tensor *out__, tensor values, tensor indices, tensor self, int64_t dim, int keepdim) { + PROTECT( + auto outputs__ = torch::nanmedian_out(*values, *indices, *self, dim, (bool)keepdim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_nanmedian_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::nanmedian_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nanquantile(tensor *out__, tensor self, tensor q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len) { + PROTECT( + auto outputs__ = torch::nanquantile(*self, *q, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim, std::string(interpolation_ptr, interpolation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nanquantile_out(tensor *out__, tensor out, tensor self, tensor q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len) { + PROTECT( + auto outputs__ = torch::nanquantile_out(*out, *self, *q, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim, std::string(interpolation_ptr, interpolation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nanquantile_scalar(tensor *out__, tensor self, double q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len) { + PROTECT( + auto outputs__ = torch::nanquantile(*self, q, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim, std::string(interpolation_ptr, interpolation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nanquantile_scalar_out(tensor *out__, tensor out, tensor self, double q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len) { + PROTECT( + auto outputs__ = torch::nanquantile_out(*out, *self, q, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim, std::string(interpolation_ptr, interpolation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nansum(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { + PROTECT( + auto outputs__ = torch::nansum(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nansum_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { + PROTECT( + auto outputs__ = torch::nansum_out(*out, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_narrow(tensor *out__, tensor self, int64_t dim, int64_t start, int64_t length) { + PROTECT( + auto outputs__ = torch::narrow(*self, dim, start, length); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_narrow_copy(tensor *out__, tensor self, int64_t dim, int64_t start, int64_t length) { + PROTECT( + auto outputs__ = torch::narrow_copy(*self, dim, start, length); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_narrow_copy_out(tensor *out__, tensor out, tensor self, int64_t dim, int64_t start, int64_t length) { + PROTECT( + auto outputs__ = torch::narrow_copy_out(*out, *self, dim, start, length); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_narrow_tensor(tensor *out__, tensor self, int64_t dim, tensor start, int64_t length) { + PROTECT( + auto outputs__ = torch::narrow(*self, dim, *start, length); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_native_batch_norm(tensor *out__, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps) { + PROTECT( + auto outputs__ = torch::native_batch_norm(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (bool)training, momentum, eps); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_native_batch_norm_out(tensor *out__, tensor out, tensor save_mean, tensor save_invstd, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps) { + PROTECT( + auto outputs__ = torch::native_batch_norm_out(*out, *save_mean, *save_invstd, *input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (bool)training, momentum, eps); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_native_channel_shuffle(tensor *out__, tensor self, int64_t groups) { + PROTECT( + auto outputs__ = torch::native_channel_shuffle(*self, groups); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_native_dropout(tensor *out__, tensor input, double p, int train) { + PROTECT( + auto outputs__ = torch::native_dropout(*input, p, (bool)train); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_native_dropout_backward(tensor *out__, tensor grad_output, tensor mask, double scale) { + PROTECT( + auto outputs__ = torch::native_dropout_backward(*grad_output, *mask, scale); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_native_dropout_backward_out(tensor *out__, tensor out, tensor grad_output, tensor mask, double scale) { + PROTECT( + auto outputs__ = torch::native_dropout_backward_out(*out, *grad_output, *mask, scale); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_native_dropout_out(tensor *out__, tensor out0, tensor out1, tensor input, double p, int train) { + PROTECT( + auto outputs__ = torch::native_dropout_out(*out0, *out1, *input, p, (bool)train); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_native_group_norm(tensor *out__, tensor input, tensor weight, tensor bias, int64_t n, int64_t C, int64_t HxW, int64_t group, double eps) { + PROTECT( + auto outputs__ = torch::native_group_norm(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), n, C, HxW, group, eps); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_native_group_norm_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor input, tensor weight, tensor bias, int64_t n, int64_t C, int64_t HxW, int64_t group, double eps) { + PROTECT( + auto outputs__ = torch::native_group_norm_out(*out0, *out1, *out2, *input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), n, C, HxW, group, eps); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_native_layer_norm(tensor *out__, tensor input, int64_t *normalized_shape_data, int normalized_shape_len, tensor weight, tensor bias, double eps) { + PROTECT( + auto outputs__ = torch::native_layer_norm(*input, torch::IntArrayRef(normalized_shape_data, normalized_shape_len), (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), eps); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_native_layer_norm_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor input, int64_t *normalized_shape_data, int normalized_shape_len, tensor weight, tensor bias, double eps) { + PROTECT( + auto outputs__ = torch::native_layer_norm_out(*out0, *out1, *out2, *input, torch::IntArrayRef(normalized_shape_data, normalized_shape_len), (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), eps); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_native_norm(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::native_norm(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_native_norm_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::native_norm_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_native_norm_scalaropt_dim_dtype(tensor *out__, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim, int dtype) { + PROTECT( + auto outputs__ = torch::native_norm(*self, *p, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_native_norm_scalaropt_dim_dtype_out(tensor *out__, tensor out, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim, int dtype) { + PROTECT( + auto outputs__ = torch::native_norm_out(*out, *self, *p, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ne(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::ne(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ne_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->ne_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ne_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::ne_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ne_tensor(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::ne(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ne_tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->ne_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ne_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::ne_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_neg(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::neg(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_neg_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::neg_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_neg_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::neg_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_negative(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::negative(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_negative_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::negative_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_negative_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::negative_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nested_to_padded_tensor(tensor *out__, tensor self, double padding, int64_t *output_size_data, int output_size_len) { + PROTECT( + auto outputs__ = torch::nested_to_padded_tensor(*self, padding, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_new_empty(tensor *out__, tensor self, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = self->new_empty(torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_new_empty_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::new_empty_out(*out, *self, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_new_empty_strided(tensor *out__, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = self->new_empty_strided(torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_new_empty_strided_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len) { + PROTECT( + auto outputs__ = torch::new_empty_strided_out(*out, *self, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_new_full(tensor *out__, tensor self, int64_t *size_data, int size_len, scalar fill_value, int options_kind, int options_device) { + PROTECT( + auto outputs__ = self->new_full(torch::IntArrayRef(size_data, size_len), *fill_value, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_new_full_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len, scalar fill_value) { + PROTECT( + auto outputs__ = torch::new_full_out(*out, *self, torch::IntArrayRef(size_data, size_len), *fill_value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_new_ones(tensor *out__, tensor self, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = self->new_ones(torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_new_ones_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::new_ones_out(*out, *self, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_new_zeros(tensor *out__, tensor self, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = self->new_zeros(torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_new_zeros_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::new_zeros_out(*out, *self, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nextafter(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::nextafter(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nextafter_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->nextafter_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nextafter_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::nextafter_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nll_loss(tensor *out__, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index) { + PROTECT( + auto outputs__ = torch::nll_loss(*self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction, ignore_index); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nll_loss2d(tensor *out__, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index) { + PROTECT( + auto outputs__ = torch::nll_loss2d(*self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction, ignore_index); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nll_loss2d_backward(tensor *out__, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index, tensor total_weight) { + PROTECT( + auto outputs__ = torch::nll_loss2d_backward(*grad_output, *self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction, ignore_index, *total_weight); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nll_loss2d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index, tensor total_weight) { + PROTECT( + auto outputs__ = torch::nll_loss2d_backward_out(*grad_input, *grad_output, *self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction, ignore_index, *total_weight); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nll_loss2d_out(tensor *out__, tensor out, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index) { + PROTECT( + auto outputs__ = torch::nll_loss2d_out(*out, *self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction, ignore_index); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nll_loss_backward(tensor *out__, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index, tensor total_weight) { + PROTECT( + auto outputs__ = torch::nll_loss_backward(*grad_output, *self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction, ignore_index, *total_weight); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nll_loss_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index, tensor total_weight) { + PROTECT( + auto outputs__ = torch::nll_loss_backward_out(*grad_input, *grad_output, *self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction, ignore_index, *total_weight); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nll_loss_nd(tensor *out__, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index) { + PROTECT( + auto outputs__ = torch::nll_loss_nd(*self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction, ignore_index); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nll_loss_out(tensor *out__, tensor out, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index) { + PROTECT( + auto outputs__ = torch::nll_loss_out(*out, *self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction, ignore_index); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nonzero(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::nonzero(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +tensor *atg_nonzero_numpy(tensor self) { + PROTECT( + auto outputs__ = torch::nonzero_numpy(*self); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg_nonzero_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::nonzero_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nonzero_static(tensor *out__, tensor self, int64_t size, int64_t fill_value) { + PROTECT( + auto outputs__ = torch::nonzero_static(*self, size, fill_value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nonzero_static_out(tensor *out__, tensor out, tensor self, int64_t size, int64_t fill_value) { + PROTECT( + auto outputs__ = torch::nonzero_static_out(*out, *self, size, fill_value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_norm(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::norm(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_norm_dtype_out(tensor *out__, tensor out, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim, int dtype) { + PROTECT( + auto outputs__ = torch::norm_out(*out, *self, *p, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim, at::ScalarType(dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_norm_except_dim(tensor *out__, tensor v, int64_t pow, int64_t dim) { + PROTECT( + auto outputs__ = torch::norm_except_dim(*v, pow, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_norm_out(tensor *out__, tensor out, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim) { + PROTECT( + auto outputs__ = torch::norm_out(*out, *self, *p, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_norm_scalar_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::norm_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_norm_scalaropt_dim(tensor *out__, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim) { + PROTECT( + auto outputs__ = torch::norm(*self, *p, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_norm_scalaropt_dim_dtype(tensor *out__, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim, int dtype) { + PROTECT( + auto outputs__ = torch::norm(*self, *p, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim, at::ScalarType(dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_norm_scalaropt_dtype(tensor *out__, tensor self, scalar p, int dtype) { + PROTECT( + auto outputs__ = torch::norm(*self, *p, at::ScalarType(dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_norm_scalaropt_dtype_out(tensor *out__, tensor out, tensor self, scalar p, int dtype) { + PROTECT( + auto outputs__ = torch::norm_out(*out, *self, *p, at::ScalarType(dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_normal_(tensor *out__, tensor self, double mean, double std) { + PROTECT( + auto outputs__ = self->normal_(mean, std); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_normal_functional(tensor *out__, tensor self, double mean, double std) { + PROTECT( + auto outputs__ = torch::normal_functional(*self, mean, std); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_not_equal(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::not_equal(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_not_equal_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->not_equal_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_not_equal_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::not_equal_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_not_equal_tensor(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::not_equal(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_not_equal_tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->not_equal_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_not_equal_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::not_equal_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nuclear_norm(tensor *out__, tensor self, int keepdim) { + PROTECT( + auto outputs__ = torch::nuclear_norm(*self, (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nuclear_norm_dim(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim) { + PROTECT( + auto outputs__ = torch::nuclear_norm(*self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nuclear_norm_dim_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim) { + PROTECT( + auto outputs__ = torch::nuclear_norm_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_nuclear_norm_out(tensor *out__, tensor out, tensor self, int keepdim) { + PROTECT( + auto outputs__ = torch::nuclear_norm_out(*out, *self, (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_numpy_t(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->numpy_T(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_one_hot(tensor *out__, tensor self, int64_t num_classes) { + PROTECT( + auto outputs__ = torch::one_hot(*self, num_classes); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ones(tensor *out__, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::ones(torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ones_like(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::ones_like(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ones_like_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::ones_like_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ones_out(tensor *out__, tensor out, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::ones_out(*out, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_orgqr(tensor *out__, tensor self, tensor input2) { + PROTECT( + auto outputs__ = torch::orgqr(*self, *input2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_orgqr_out(tensor *out__, tensor out, tensor self, tensor input2) { + PROTECT( + auto outputs__ = torch::orgqr_out(*out, *self, *input2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ormqr(tensor *out__, tensor self, tensor input2, tensor input3, int left, int transpose) { + PROTECT( + auto outputs__ = torch::ormqr(*self, *input2, *input3, (bool)left, (bool)transpose); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ormqr_out(tensor *out__, tensor out, tensor self, tensor input2, tensor input3, int left, int transpose) { + PROTECT( + auto outputs__ = torch::ormqr_out(*out, *self, *input2, *input3, (bool)left, (bool)transpose); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_outer(tensor *out__, tensor self, tensor vec2) { + PROTECT( + auto outputs__ = torch::outer(*self, *vec2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_outer_out(tensor *out__, tensor out, tensor self, tensor vec2) { + PROTECT( + auto outputs__ = torch::outer_out(*out, *self, *vec2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +int64_t atg_output_nr(tensor self) { + PROTECT( + return self->output_nr(); + ) + return 0; +} + +void atg_pad(tensor *out__, tensor self, int64_t *pad_data, int pad_len, char* mode_ptr, int mode_len, double value_v, uint8_t value_null) { + PROTECT( + auto outputs__ = torch::pad(*self, torch::IntArrayRef(pad_data, pad_len), std::string(mode_ptr, mode_len), value_null ? c10::nullopt : c10::optional(value_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_pad_sequence(tensor *out__, tensor *sequences_data, int sequences_len, int batch_first, double padding_value, char* padding_side_ptr, int padding_side_len) { + PROTECT( + auto outputs__ = torch::pad_sequence(of_carray_tensor(sequences_data, sequences_len), (bool)batch_first, padding_value, std::string(padding_side_ptr, padding_side_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_pairwise_distance(tensor *out__, tensor x1, tensor x2, double p, double eps, int keepdim) { + PROTECT( + auto outputs__ = torch::pairwise_distance(*x1, *x2, p, eps, (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_pdist(tensor *out__, tensor self, double p) { + PROTECT( + auto outputs__ = torch::pdist(*self, p); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_permute(tensor *out__, tensor self, int64_t *dims_data, int dims_len) { + PROTECT( + auto outputs__ = torch::permute(*self, torch::IntArrayRef(dims_data, dims_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_permute_copy(tensor *out__, tensor self, int64_t *dims_data, int dims_len) { + PROTECT( + auto outputs__ = torch::permute_copy(*self, torch::IntArrayRef(dims_data, dims_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_permute_copy_out(tensor *out__, tensor out, tensor self, int64_t *dims_data, int dims_len) { + PROTECT( + auto outputs__ = torch::permute_copy_out(*out, *self, torch::IntArrayRef(dims_data, dims_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_pin_memory(tensor *out__, tensor self, int device) { + PROTECT( + auto outputs__ = self->pin_memory(device_of_int(device)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_pinverse(tensor *out__, tensor self, double rcond) { + PROTECT( + auto outputs__ = torch::pinverse(*self, rcond); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_pixel_shuffle(tensor *out__, tensor self, int64_t upscale_factor) { + PROTECT( + auto outputs__ = torch::pixel_shuffle(*self, upscale_factor); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_pixel_shuffle_out(tensor *out__, tensor out, tensor self, int64_t upscale_factor) { + PROTECT( + auto outputs__ = torch::pixel_shuffle_out(*out, *self, upscale_factor); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_pixel_unshuffle(tensor *out__, tensor self, int64_t downscale_factor) { + PROTECT( + auto outputs__ = torch::pixel_unshuffle(*self, downscale_factor); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_pixel_unshuffle_out(tensor *out__, tensor out, tensor self, int64_t downscale_factor) { + PROTECT( + auto outputs__ = torch::pixel_unshuffle_out(*out, *self, downscale_factor); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_poisson(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::poisson(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_poisson_nll_loss(tensor *out__, tensor input, tensor target, int log_input, int full, double eps, int64_t reduction) { + PROTECT( + auto outputs__ = torch::poisson_nll_loss(*input, *target, (bool)log_input, (bool)full, eps, reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_poisson_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::poisson_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_polar(tensor *out__, tensor abs, tensor angle) { + PROTECT( + auto outputs__ = torch::polar(*abs, *angle); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_polar_out(tensor *out__, tensor out, tensor abs, tensor angle) { + PROTECT( + auto outputs__ = torch::polar_out(*out, *abs, *angle); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_polygamma(tensor *out__, int64_t n, tensor self) { + PROTECT( + auto outputs__ = torch::polygamma(n, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_polygamma_(tensor *out__, tensor self, int64_t n) { + PROTECT( + auto outputs__ = self->polygamma_(n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_polygamma_out(tensor *out__, tensor out, int64_t n, tensor self) { + PROTECT( + auto outputs__ = torch::polygamma_out(*out, n, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_positive(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::positive(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_pow(tensor *out__, tensor self, tensor exponent) { + PROTECT( + auto outputs__ = torch::pow(*self, *exponent); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_pow_(tensor *out__, tensor self, scalar exponent) { + PROTECT( + auto outputs__ = self->pow_(*exponent); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_pow_scalar(tensor *out__, scalar self_scalar, tensor exponent) { + PROTECT( + auto outputs__ = torch::pow(*self_scalar, *exponent); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_pow_scalar_out(tensor *out__, tensor out, scalar self_scalar, tensor exponent) { + PROTECT( + auto outputs__ = torch::pow_out(*out, *self_scalar, *exponent); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_pow_tensor_(tensor *out__, tensor self, tensor exponent) { + PROTECT( + auto outputs__ = self->pow_(*exponent); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_pow_tensor_scalar(tensor *out__, tensor self, scalar exponent) { + PROTECT( + auto outputs__ = torch::pow(*self, *exponent); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_pow_tensor_scalar_out(tensor *out__, tensor out, tensor self, scalar exponent) { + PROTECT( + auto outputs__ = torch::pow_out(*out, *self, *exponent); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_pow_tensor_tensor_out(tensor *out__, tensor out, tensor self, tensor exponent) { + PROTECT( + auto outputs__ = torch::pow_out(*out, *self, *exponent); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_prelu(tensor *out__, tensor self, tensor weight) { + PROTECT( + auto outputs__ = torch::prelu(*self, *weight); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_prod(tensor *out__, tensor self, int dtype) { + PROTECT( + auto outputs__ = torch::prod(*self, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_prod_dim_int(tensor *out__, tensor self, int64_t dim, int keepdim, int dtype) { + PROTECT( + auto outputs__ = torch::prod(*self, dim, (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_prod_int_out(tensor *out__, tensor out, tensor self, int64_t dim, int keepdim, int dtype) { + PROTECT( + auto outputs__ = torch::prod_out(*out, *self, dim, (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_prod_out(tensor *out__, tensor out, tensor self, int dtype) { + PROTECT( + auto outputs__ = torch::prod_out(*out, *self, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_put(tensor *out__, tensor self, tensor index, tensor source, int accumulate) { + PROTECT( + auto outputs__ = torch::put(*self, *index, *source, (bool)accumulate); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_put_(tensor *out__, tensor self, tensor index, tensor source, int accumulate) { + PROTECT( + auto outputs__ = self->put_(*index, *source, (bool)accumulate); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_put_out(tensor *out__, tensor out, tensor self, tensor index, tensor source, int accumulate) { + PROTECT( + auto outputs__ = torch::put_out(*out, *self, *index, *source, (bool)accumulate); + out__[0] = new torch::Tensor(outputs__); + ) +} + +int64_t atg_q_per_channel_axis(tensor self) { + PROTECT( + return torch::q_per_channel_axis(*self); + ) + return 0; +} + +void atg_q_per_channel_scales(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::q_per_channel_scales(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_q_per_channel_scales_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::q_per_channel_scales_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_q_per_channel_zero_points(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::q_per_channel_zero_points(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_q_per_channel_zero_points_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::q_per_channel_zero_points_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +double atg_q_scale(tensor self) { + PROTECT( + return torch::q_scale(*self); + ) + return 0; +} + +int64_t atg_q_zero_point(tensor self) { + PROTECT( + return torch::q_zero_point(*self); + ) + return 0; +} + +void atg_qr(tensor *out__, tensor self, int some) { + PROTECT( + auto outputs__ = torch::qr(*self, (bool)some); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_qr_q(tensor *out__, tensor Q, tensor R, tensor self, int some) { + PROTECT( + auto outputs__ = torch::qr_out(*Q, *R, *self, (bool)some); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_quantile(tensor *out__, tensor self, tensor q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len) { + PROTECT( + auto outputs__ = torch::quantile(*self, *q, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim, std::string(interpolation_ptr, interpolation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_quantile_out(tensor *out__, tensor out, tensor self, tensor q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len) { + PROTECT( + auto outputs__ = torch::quantile_out(*out, *self, *q, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim, std::string(interpolation_ptr, interpolation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_quantile_scalar(tensor *out__, tensor self, double q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len) { + PROTECT( + auto outputs__ = torch::quantile(*self, q, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim, std::string(interpolation_ptr, interpolation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_quantile_scalar_out(tensor *out__, tensor out, tensor self, double q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len) { + PROTECT( + auto outputs__ = torch::quantile_out(*out, *self, q, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim, std::string(interpolation_ptr, interpolation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_quantize_per_channel(tensor *out__, tensor self, tensor scales, tensor zero_points, int64_t axis, int dtype) { + PROTECT( + auto outputs__ = torch::quantize_per_channel(*self, *scales, *zero_points, axis, at::ScalarType(dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_quantize_per_channel_out(tensor *out__, tensor out, tensor self, tensor scales, tensor zero_points, int64_t axis, int dtype) { + PROTECT( + auto outputs__ = torch::quantize_per_channel_out(*out, *self, *scales, *zero_points, axis, at::ScalarType(dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_quantize_per_tensor(tensor *out__, tensor self, double scale, int64_t zero_point, int dtype) { + PROTECT( + auto outputs__ = torch::quantize_per_tensor(*self, scale, zero_point, at::ScalarType(dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_quantize_per_tensor_dynamic(tensor *out__, tensor self, int dtype, int reduce_range) { + PROTECT( + auto outputs__ = torch::quantize_per_tensor_dynamic(*self, at::ScalarType(dtype), (bool)reduce_range); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_quantize_per_tensor_dynamic_out(tensor *out__, tensor out, tensor self, int dtype, int reduce_range) { + PROTECT( + auto outputs__ = torch::quantize_per_tensor_dynamic_out(*out, *self, at::ScalarType(dtype), (bool)reduce_range); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_quantize_per_tensor_out(tensor *out__, tensor out, tensor self, double scale, int64_t zero_point, int dtype) { + PROTECT( + auto outputs__ = torch::quantize_per_tensor_out(*out, *self, scale, zero_point, at::ScalarType(dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_quantize_per_tensor_tensor_qparams(tensor *out__, tensor self, tensor scale, tensor zero_point, int dtype) { + PROTECT( + auto outputs__ = torch::quantize_per_tensor(*self, *scale, *zero_point, at::ScalarType(dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_quantize_per_tensor_tensor_qparams_out(tensor *out__, tensor out, tensor self, tensor scale, tensor zero_point, int dtype) { + PROTECT( + auto outputs__ = torch::quantize_per_tensor_out(*out, *self, *scale, *zero_point, at::ScalarType(dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +tensor *atg_quantize_per_tensor_tensors(tensor *tensors_data, int tensors_len, tensor scales, tensor zero_points, int dtype) { + PROTECT( + auto outputs__ = torch::quantize_per_tensor(of_carray_tensor(tensors_data, tensors_len), *scales, *zero_points, at::ScalarType(dtype)); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg_quantize_per_tensor_tensors_out(tensor *out_data, int out_len, tensor *tensors_data, int tensors_len, tensor scales, tensor zero_points, int dtype) { + PROTECT( + torch::quantize_per_tensor_out(of_carray_tensor(out_data, out_len), of_carray_tensor(tensors_data, tensors_len), *scales, *zero_points, at::ScalarType(dtype)); + ) +} + +void atg_quantized_batch_norm(tensor *out__, tensor input, tensor weight, tensor bias, tensor mean, tensor var, double eps, double output_scale, int64_t output_zero_point) { + PROTECT( + auto outputs__ = torch::quantized_batch_norm(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *mean, *var, eps, output_scale, output_zero_point); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_quantized_batch_norm_out(tensor *out__, tensor out, tensor input, tensor weight, tensor bias, tensor mean, tensor var, double eps, double output_scale, int64_t output_zero_point) { + PROTECT( + auto outputs__ = torch::quantized_batch_norm_out(*out, *input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *mean, *var, eps, output_scale, output_zero_point); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_quantized_gru_cell(tensor *out__, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh, tensor packed_ih, tensor packed_hh, tensor col_offsets_ih, tensor col_offsets_hh, scalar scale_ih, scalar scale_hh, scalar zero_point_ih, scalar zero_point_hh) { + PROTECT( + auto outputs__ = torch::quantized_gru_cell(*input, *hx, *w_ih, *w_hh, *b_ih, *b_hh, *packed_ih, *packed_hh, *col_offsets_ih, *col_offsets_hh, *scale_ih, *scale_hh, *zero_point_ih, *zero_point_hh); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_quantized_lstm_cell(tensor *out__, tensor input, tensor *hx_data, int hx_len, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh, tensor packed_ih, tensor packed_hh, tensor col_offsets_ih, tensor col_offsets_hh, scalar scale_ih, scalar scale_hh, scalar zero_point_ih, scalar zero_point_hh) { + PROTECT( + auto outputs__ = torch::quantized_lstm_cell(*input, of_carray_tensor(hx_data, hx_len), *w_ih, *w_hh, *b_ih, *b_hh, *packed_ih, *packed_hh, *col_offsets_ih, *col_offsets_hh, *scale_ih, *scale_hh, *zero_point_ih, *zero_point_hh); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_quantized_max_pool1d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::quantized_max_pool1d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_quantized_max_pool1d_out(tensor *out__, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::quantized_max_pool1d_out(*out, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_quantized_max_pool2d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::quantized_max_pool2d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_quantized_max_pool2d_out(tensor *out__, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::quantized_max_pool2d_out(*out, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_quantized_max_pool3d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::quantized_max_pool3d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_quantized_max_pool3d_out(tensor *out__, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { + PROTECT( + auto outputs__ = torch::quantized_max_pool3d_out(*out, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_quantized_rnn_relu_cell(tensor *out__, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh, tensor packed_ih, tensor packed_hh, tensor col_offsets_ih, tensor col_offsets_hh, scalar scale_ih, scalar scale_hh, scalar zero_point_ih, scalar zero_point_hh) { + PROTECT( + auto outputs__ = torch::quantized_rnn_relu_cell(*input, *hx, *w_ih, *w_hh, *b_ih, *b_hh, *packed_ih, *packed_hh, *col_offsets_ih, *col_offsets_hh, *scale_ih, *scale_hh, *zero_point_ih, *zero_point_hh); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_quantized_rnn_tanh_cell(tensor *out__, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh, tensor packed_ih, tensor packed_hh, tensor col_offsets_ih, tensor col_offsets_hh, scalar scale_ih, scalar scale_hh, scalar zero_point_ih, scalar zero_point_hh) { + PROTECT( + auto outputs__ = torch::quantized_rnn_tanh_cell(*input, *hx, *w_ih, *w_hh, *b_ih, *b_hh, *packed_ih, *packed_hh, *col_offsets_ih, *col_offsets_hh, *scale_ih, *scale_hh, *zero_point_ih, *zero_point_hh); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rad2deg(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::rad2deg(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rad2deg_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::rad2deg_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rad2deg_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::rad2deg_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rand(tensor *out__, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::rand(torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rand_like(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::rand_like(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rand_like_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::rand_like_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rand_out(tensor *out__, tensor out, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::rand_out(*out, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_randint(tensor *out__, int64_t high, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::randint(high, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_randint_like(tensor *out__, tensor self, int64_t high) { + PROTECT( + auto outputs__ = torch::randint_like(*self, high); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_randint_like_low_dtype(tensor *out__, tensor self, int64_t low, int64_t high) { + PROTECT( + auto outputs__ = torch::randint_like(*self, low, high); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_randint_like_low_dtype_out(tensor *out__, tensor out, tensor self, int64_t low, int64_t high) { + PROTECT( + auto outputs__ = torch::randint_like_out(*out, *self, low, high); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_randint_like_out(tensor *out__, tensor out, tensor self, int64_t high) { + PROTECT( + auto outputs__ = torch::randint_like_out(*out, *self, high); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_randint_like_tensor(tensor *out__, tensor self, tensor high) { + PROTECT( + auto outputs__ = torch::randint_like(*self, *high); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_randint_like_tensor_out(tensor *out__, tensor out, tensor self, tensor high) { + PROTECT( + auto outputs__ = torch::randint_like_out(*out, *self, *high); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_randint_low(tensor *out__, int64_t low, int64_t high, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::randint(low, high, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_randint_low_out(tensor *out__, tensor out, int64_t low, int64_t high, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::randint_out(*out, low, high, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_randint_out(tensor *out__, tensor out, int64_t high, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::randint_out(*out, high, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_randn(tensor *out__, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::randn(torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_randn_like(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::randn_like(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_randn_like_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::randn_like_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_randn_out(tensor *out__, tensor out, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::randn_out(*out, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_random(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::random(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_random_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->random_(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_random_from(tensor *out__, tensor self, int64_t from, int64_t to_v, uint8_t to_null) { + PROTECT( + auto outputs__ = torch::random(*self, from, to_null ? c10::nullopt : c10::optional(to_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_random_from_(tensor *out__, tensor self, int64_t from, int64_t to_v, uint8_t to_null) { + PROTECT( + auto outputs__ = self->random_(from, to_null ? c10::nullopt : c10::optional(to_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_random_from_out(tensor *out__, tensor out, tensor self, int64_t from, int64_t to_v, uint8_t to_null) { + PROTECT( + auto outputs__ = torch::random_out(*out, *self, from, to_null ? c10::nullopt : c10::optional(to_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_random_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::random_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_random_to(tensor *out__, tensor self, int64_t to) { + PROTECT( + auto outputs__ = torch::random(*self, to); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_random_to_(tensor *out__, tensor self, int64_t to) { + PROTECT( + auto outputs__ = self->random_(to); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_random_to_out(tensor *out__, tensor out, tensor self, int64_t to) { + PROTECT( + auto outputs__ = torch::random_out(*out, *self, to); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_randperm(tensor *out__, int64_t n, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::randperm(n, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_randperm_out(tensor *out__, tensor out, int64_t n) { + PROTECT( + auto outputs__ = torch::randperm_out(*out, n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_range(tensor *out__, scalar start, scalar end, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::range(*start, *end, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_range_out(tensor *out__, tensor out, scalar start, scalar end) { + PROTECT( + auto outputs__ = torch::range_out(*out, *start, *end); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_range_out_(tensor *out__, tensor out, scalar start, scalar end) { + PROTECT( + auto outputs__ = torch::range_out(*out, *start, *end); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_range_step(tensor *out__, scalar start, scalar end, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::range(*start, *end, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_ravel(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::ravel(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_real(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::real(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_reciprocal(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::reciprocal(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_reciprocal_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::reciprocal_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_reciprocal_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::reciprocal_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_reflection_pad1d(tensor *out__, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::reflection_pad1d(*self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_reflection_pad1d_backward(tensor *out__, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::reflection_pad1d_backward(*grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_reflection_pad1d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::reflection_pad1d_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_reflection_pad1d_out(tensor *out__, tensor out, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::reflection_pad1d_out(*out, *self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_reflection_pad2d(tensor *out__, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::reflection_pad2d(*self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_reflection_pad2d_backward(tensor *out__, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::reflection_pad2d_backward(*grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_reflection_pad2d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::reflection_pad2d_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_reflection_pad2d_out(tensor *out__, tensor out, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::reflection_pad2d_out(*out, *self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_reflection_pad3d(tensor *out__, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::reflection_pad3d(*self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_reflection_pad3d_backward(tensor *out__, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::reflection_pad3d_backward(*grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_reflection_pad3d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::reflection_pad3d_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_reflection_pad3d_out(tensor *out__, tensor out, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::reflection_pad3d_out(*out, *self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_relu(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::relu(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_relu6(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::relu6(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_relu6_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::relu6_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_relu_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::relu_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_relu_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::relu_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_remainder(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::remainder(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_remainder_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->remainder_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_remainder_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::remainder_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_remainder_scalar_tensor(tensor *out__, scalar self_scalar, tensor other) { + PROTECT( + auto outputs__ = torch::remainder(*self_scalar, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_remainder_scalar_tensor_out(tensor *out__, tensor out, scalar self_scalar, tensor other) { + PROTECT( + auto outputs__ = torch::remainder_out(*out, *self_scalar, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_remainder_tensor(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::remainder(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_remainder_tensor_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->remainder_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_remainder_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::remainder_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_renorm(tensor *out__, tensor self, scalar p, int64_t dim, scalar maxnorm) { + PROTECT( + auto outputs__ = torch::renorm(*self, *p, dim, *maxnorm); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_renorm_(tensor *out__, tensor self, scalar p, int64_t dim, scalar maxnorm) { + PROTECT( + auto outputs__ = self->renorm_(*p, dim, *maxnorm); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_renorm_out(tensor *out__, tensor out, tensor self, scalar p, int64_t dim, scalar maxnorm) { + PROTECT( + auto outputs__ = torch::renorm_out(*out, *self, *p, dim, *maxnorm); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_repeat(tensor *out__, tensor self, int64_t *repeats_data, int repeats_len) { + PROTECT( + auto outputs__ = self->repeat(torch::IntArrayRef(repeats_data, repeats_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_repeat_interleave(tensor *out__, tensor repeats, int64_t output_size_v, uint8_t output_size_null) { + PROTECT( + auto outputs__ = torch::repeat_interleave(*repeats, output_size_null ? c10::nullopt : c10::optional(output_size_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_repeat_interleave_self_int(tensor *out__, tensor self, int64_t repeats, int64_t dim_v, uint8_t dim_null, int64_t output_size_v, uint8_t output_size_null) { + PROTECT( + auto outputs__ = torch::repeat_interleave(*self, repeats, dim_null ? c10::nullopt : c10::optional(dim_v), output_size_null ? c10::nullopt : c10::optional(output_size_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_repeat_interleave_self_tensor(tensor *out__, tensor self, tensor repeats, int64_t dim_v, uint8_t dim_null, int64_t output_size_v, uint8_t output_size_null) { + PROTECT( + auto outputs__ = torch::repeat_interleave(*self, *repeats, dim_null ? c10::nullopt : c10::optional(dim_v), output_size_null ? c10::nullopt : c10::optional(output_size_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_repeat_interleave_tensor_out(tensor *out__, tensor out, tensor repeats, int64_t output_size_v, uint8_t output_size_null) { + PROTECT( + auto outputs__ = torch::repeat_interleave_out(*out, *repeats, output_size_null ? c10::nullopt : c10::optional(output_size_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_repeat_out(tensor *out__, tensor out, tensor self, int64_t *repeats_data, int repeats_len) { + PROTECT( + auto outputs__ = torch::repeat_out(*out, *self, torch::IntArrayRef(repeats_data, repeats_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_replication_pad1d(tensor *out__, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::replication_pad1d(*self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_replication_pad1d_backward(tensor *out__, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::replication_pad1d_backward(*grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_replication_pad1d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::replication_pad1d_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_replication_pad1d_out(tensor *out__, tensor out, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::replication_pad1d_out(*out, *self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_replication_pad2d(tensor *out__, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::replication_pad2d(*self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_replication_pad2d_backward(tensor *out__, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::replication_pad2d_backward(*grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_replication_pad2d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::replication_pad2d_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_replication_pad2d_out(tensor *out__, tensor out, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::replication_pad2d_out(*out, *self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_replication_pad3d(tensor *out__, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::replication_pad3d(*self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_replication_pad3d_backward(tensor *out__, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::replication_pad3d_backward(*grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_replication_pad3d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::replication_pad3d_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_replication_pad3d_out(tensor *out__, tensor out, tensor self, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::replication_pad3d_out(*out, *self, torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_requires_grad_(tensor *out__, tensor self, int requires_grad) { + PROTECT( + auto outputs__ = self->requires_grad_((bool)requires_grad); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_reshape(tensor *out__, tensor self, int64_t *shape_data, int shape_len) { + PROTECT( + auto outputs__ = torch::reshape(*self, torch::IntArrayRef(shape_data, shape_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_reshape_as(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->reshape_as(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_resize(tensor *out__, tensor self, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::resize(*self, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_resize_(tensor *out__, tensor self, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = self->resize_(torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_resize_as(tensor *out__, tensor self, tensor the_template) { + PROTECT( + auto outputs__ = torch::resize_as(*self, *the_template); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_resize_as_(tensor *out__, tensor self, tensor the_template) { + PROTECT( + auto outputs__ = torch::resize_as_(*self, *the_template); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_resize_as_out(tensor *out__, tensor out, tensor self, tensor the_template) { + PROTECT( + auto outputs__ = torch::resize_as_out(*out, *self, *the_template); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_resize_as_sparse(tensor *out__, tensor self, tensor the_template) { + PROTECT( + auto outputs__ = torch::resize_as_sparse(*self, *the_template); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_resize_as_sparse_(tensor *out__, tensor self, tensor the_template) { + PROTECT( + auto outputs__ = torch::resize_as_sparse_(*self, *the_template); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_resize_as_sparse_out(tensor *out__, tensor out, tensor self, tensor the_template) { + PROTECT( + auto outputs__ = torch::resize_as_sparse_out(*out, *self, *the_template); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_resize_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::resize_out(*out, *self, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_resolve_conj(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::resolve_conj(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_resolve_neg(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::resolve_neg(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +int atg_retains_grad(tensor self) { + PROTECT( + return self->retains_grad(); + ) + return 0; +} + +void atg_rms_norm(tensor *out__, tensor input, int64_t *normalized_shape_data, int normalized_shape_len, tensor weight, double eps_v, uint8_t eps_null) { + PROTECT( + auto outputs__ = torch::rms_norm(*input, torch::IntArrayRef(normalized_shape_data, normalized_shape_len), (weight ? ::std::optional(*weight) : ::std::nullopt), eps_null ? c10::nullopt : c10::optional(eps_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rnn_relu(tensor *out__, tensor input, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first) { + PROTECT( + auto outputs__ = torch::rnn_relu(*input, *hx, of_carray_tensor(params_data, params_len), (bool)has_biases, num_layers, dropout, (bool)train, (bool)bidirectional, (bool)batch_first); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_rnn_relu_cell(tensor *out__, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh) { + PROTECT( + auto outputs__ = torch::rnn_relu_cell(*input, *hx, *w_ih, *w_hh, (b_ih ? ::std::optional(*b_ih) : ::std::nullopt), (b_hh ? ::std::optional(*b_hh) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rnn_relu_data(tensor *out__, tensor data, tensor batch_sizes, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional) { + PROTECT( + auto outputs__ = torch::rnn_relu(*data, *batch_sizes, *hx, of_carray_tensor(params_data, params_len), (bool)has_biases, num_layers, dropout, (bool)train, (bool)bidirectional); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_rnn_tanh(tensor *out__, tensor input, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first) { + PROTECT( + auto outputs__ = torch::rnn_tanh(*input, *hx, of_carray_tensor(params_data, params_len), (bool)has_biases, num_layers, dropout, (bool)train, (bool)bidirectional, (bool)batch_first); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_rnn_tanh_cell(tensor *out__, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh) { + PROTECT( + auto outputs__ = torch::rnn_tanh_cell(*input, *hx, *w_ih, *w_hh, (b_ih ? ::std::optional(*b_ih) : ::std::nullopt), (b_hh ? ::std::optional(*b_hh) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rnn_tanh_data(tensor *out__, tensor data, tensor batch_sizes, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional) { + PROTECT( + auto outputs__ = torch::rnn_tanh(*data, *batch_sizes, *hx, of_carray_tensor(params_data, params_len), (bool)has_biases, num_layers, dropout, (bool)train, (bool)bidirectional); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_roll(tensor *out__, tensor self, int64_t *shifts_data, int shifts_len, int64_t *dims_data, int dims_len) { + PROTECT( + auto outputs__ = torch::roll(*self, torch::IntArrayRef(shifts_data, shifts_len), torch::IntArrayRef(dims_data, dims_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_roll_out(tensor *out__, tensor out, tensor self, int64_t *shifts_data, int shifts_len, int64_t *dims_data, int dims_len) { + PROTECT( + auto outputs__ = torch::roll_out(*out, *self, torch::IntArrayRef(shifts_data, shifts_len), torch::IntArrayRef(dims_data, dims_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rot90(tensor *out__, tensor self, int64_t k, int64_t *dims_data, int dims_len) { + PROTECT( + auto outputs__ = torch::rot90(*self, k, torch::IntArrayRef(dims_data, dims_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rot90_out(tensor *out__, tensor out, tensor self, int64_t k, int64_t *dims_data, int dims_len) { + PROTECT( + auto outputs__ = torch::rot90_out(*out, *self, k, torch::IntArrayRef(dims_data, dims_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_round(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::round(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_round_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::round_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_round_decimals(tensor *out__, tensor self, int64_t decimals) { + PROTECT( + auto outputs__ = torch::round(*self, decimals); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_round_decimals_(tensor *out__, tensor self, int64_t decimals) { + PROTECT( + auto outputs__ = torch::round_(*self, decimals); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_round_decimals_out(tensor *out__, tensor out, tensor self, int64_t decimals) { + PROTECT( + auto outputs__ = torch::round_out(*out, *self, decimals); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_round_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::round_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_row_indices(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->row_indices(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_row_indices_copy(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::row_indices_copy(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_row_indices_copy_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::row_indices_copy_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_row_stack(tensor *out__, tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::row_stack(of_carray_tensor(tensors_data, tensors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_row_stack_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::row_stack_out(*out, of_carray_tensor(tensors_data, tensors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rrelu(tensor *out__, tensor self, int training) { + PROTECT( + auto outputs__ = torch::rrelu(*self, (bool)training); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rrelu_(tensor *out__, tensor self, int training) { + PROTECT( + auto outputs__ = torch::rrelu_(*self, (bool)training); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rrelu_with_noise(tensor *out__, tensor self, tensor noise, int training) { + PROTECT( + auto outputs__ = torch::rrelu_with_noise(*self, *noise, (bool)training); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rrelu_with_noise_(tensor *out__, tensor self, tensor noise, int training) { + PROTECT( + auto outputs__ = torch::rrelu_with_noise_(*self, *noise, (bool)training); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rrelu_with_noise_backward(tensor *out__, tensor grad_output, tensor self, tensor noise, scalar lower, scalar upper, int training, int self_is_result) { + PROTECT( + auto outputs__ = torch::rrelu_with_noise_backward(*grad_output, *self, *noise, *lower, *upper, (bool)training, (bool)self_is_result); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rrelu_with_noise_backward_out(tensor *out__, tensor out, tensor grad_output, tensor self, tensor noise, scalar lower, scalar upper, int training, int self_is_result) { + PROTECT( + auto outputs__ = torch::rrelu_with_noise_backward_out(*out, *grad_output, *self, *noise, *lower, *upper, (bool)training, (bool)self_is_result); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rrelu_with_noise_functional(tensor *out__, tensor self, tensor noise, int training) { + PROTECT( + auto outputs__ = torch::rrelu_with_noise_functional(*self, *noise, (bool)training); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_rrelu_with_noise_out(tensor *out__, tensor out, tensor self, tensor noise, int training) { + PROTECT( + auto outputs__ = torch::rrelu_with_noise_out(*out, *self, *noise, (bool)training); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rsqrt(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::rsqrt(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rsqrt_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::rsqrt_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rsqrt_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::rsqrt_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rsub(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::rsub(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rsub_scalar(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::rsub(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rsub_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::rsub_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_rsub_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::rsub_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_scalar_tensor(tensor *out__, scalar s, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::scalar_tensor(*s, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_scalar_tensor_out(tensor *out__, tensor out, scalar s) { + PROTECT( + auto outputs__ = torch::scalar_tensor_out(*out, *s); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_scaled_dot_product_attention(tensor *out__, tensor query, tensor key, tensor value, tensor attn_mask, double dropout_p, int is_causal, double scale_v, uint8_t scale_null, int enable_gqa) { + PROTECT( + auto outputs__ = torch::scaled_dot_product_attention(*query, *key, *value, (attn_mask ? ::std::optional(*attn_mask) : ::std::nullopt), dropout_p, (bool)is_causal, scale_null ? c10::nullopt : c10::optional(scale_v), (bool)enable_gqa); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_scatter(tensor *out__, tensor self, int64_t dim, tensor index, tensor src) { + PROTECT( + auto outputs__ = torch::scatter(*self, dim, *index, *src); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_scatter_(tensor *out__, tensor self, int64_t dim, tensor index, tensor src) { + PROTECT( + auto outputs__ = self->scatter_(dim, *index, *src); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_scatter_add(tensor *out__, tensor self, int64_t dim, tensor index, tensor src) { + PROTECT( + auto outputs__ = torch::scatter_add(*self, dim, *index, *src); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_scatter_add_(tensor *out__, tensor self, int64_t dim, tensor index, tensor src) { + PROTECT( + auto outputs__ = self->scatter_add_(dim, *index, *src); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_scatter_add_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, tensor src) { + PROTECT( + auto outputs__ = torch::scatter_add_out(*out, *self, dim, *index, *src); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_scatter_reduce(tensor *out__, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len) { + PROTECT( + auto outputs__ = torch::scatter(*self, dim, *index, *src, std::string(reduce_ptr, reduce_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_scatter_reduce_(tensor *out__, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len) { + PROTECT( + auto outputs__ = self->scatter_(dim, *index, *src, std::string(reduce_ptr, reduce_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_scatter_reduce_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len) { + PROTECT( + auto outputs__ = torch::scatter_out(*out, *self, dim, *index, *src, std::string(reduce_ptr, reduce_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_scatter_src_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, tensor src) { + PROTECT( + auto outputs__ = torch::scatter_out(*out, *self, dim, *index, *src); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_scatter_value(tensor *out__, tensor self, int64_t dim, tensor index, scalar value) { + PROTECT( + auto outputs__ = torch::scatter(*self, dim, *index, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_scatter_value_(tensor *out__, tensor self, int64_t dim, tensor index, scalar value) { + PROTECT( + auto outputs__ = self->scatter_(dim, *index, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_scatter_value_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, scalar value) { + PROTECT( + auto outputs__ = torch::scatter_out(*out, *self, dim, *index, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_scatter_value_reduce(tensor *out__, tensor self, int64_t dim, tensor index, scalar value, char* reduce_ptr, int reduce_len) { + PROTECT( + auto outputs__ = torch::scatter(*self, dim, *index, *value, std::string(reduce_ptr, reduce_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_scatter_value_reduce_(tensor *out__, tensor self, int64_t dim, tensor index, scalar value, char* reduce_ptr, int reduce_len) { + PROTECT( + auto outputs__ = self->scatter_(dim, *index, *value, std::string(reduce_ptr, reduce_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_scatter_value_reduce_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, scalar value, char* reduce_ptr, int reduce_len) { + PROTECT( + auto outputs__ = torch::scatter_out(*out, *self, dim, *index, *value, std::string(reduce_ptr, reduce_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_searchsorted(tensor *out__, tensor sorted_sequence, tensor self, int out_int32, int right, char* side_ptr, int side_len, tensor sorter) { + PROTECT( + auto outputs__ = torch::searchsorted(*sorted_sequence, *self, (bool)out_int32, (bool)right, std::string(side_ptr, side_len), (sorter ? ::std::optional(*sorter) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_searchsorted_scalar(tensor *out__, tensor sorted_sequence, scalar self_scalar, int out_int32, int right, char* side_ptr, int side_len, tensor sorter) { + PROTECT( + auto outputs__ = torch::searchsorted(*sorted_sequence, *self_scalar, (bool)out_int32, (bool)right, std::string(side_ptr, side_len), (sorter ? ::std::optional(*sorter) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_searchsorted_scalar_out(tensor *out__, tensor out, tensor sorted_sequence, scalar self_scalar, int out_int32, int right, char* side_ptr, int side_len, tensor sorter) { + PROTECT( + auto outputs__ = torch::searchsorted_out(*out, *sorted_sequence, *self_scalar, (bool)out_int32, (bool)right, std::string(side_ptr, side_len), (sorter ? ::std::optional(*sorter) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_searchsorted_tensor_out(tensor *out__, tensor out, tensor sorted_sequence, tensor self, int out_int32, int right, char* side_ptr, int side_len, tensor sorter) { + PROTECT( + auto outputs__ = torch::searchsorted_out(*out, *sorted_sequence, *self, (bool)out_int32, (bool)right, std::string(side_ptr, side_len), (sorter ? ::std::optional(*sorter) : ::std::nullopt)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_segment_reduce(tensor *out__, tensor data, char* reduce_ptr, int reduce_len, tensor lengths, tensor indices, tensor offsets, int64_t axis, int unsafe, scalar initial) { + PROTECT( + auto outputs__ = torch::segment_reduce(*data, std::string(reduce_ptr, reduce_len), (lengths ? ::std::optional(*lengths) : ::std::nullopt), (indices ? ::std::optional(*indices) : ::std::nullopt), (offsets ? ::std::optional(*offsets) : ::std::nullopt), axis, (bool)unsafe, *initial); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_segment_reduce_out(tensor *out__, tensor out, tensor data, char* reduce_ptr, int reduce_len, tensor lengths, tensor indices, tensor offsets, int64_t axis, int unsafe, scalar initial) { + PROTECT( + auto outputs__ = torch::segment_reduce_out(*out, *data, std::string(reduce_ptr, reduce_len), (lengths ? ::std::optional(*lengths) : ::std::nullopt), (indices ? ::std::optional(*indices) : ::std::nullopt), (offsets ? ::std::optional(*offsets) : ::std::nullopt), axis, (bool)unsafe, *initial); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_select(tensor *out__, tensor self, int64_t dim, int64_t index) { + PROTECT( + auto outputs__ = torch::select(*self, dim, index); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_select_backward(tensor *out__, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t index) { + PROTECT( + auto outputs__ = torch::select_backward(*grad_output, torch::IntArrayRef(input_sizes_data, input_sizes_len), dim, index); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_select_backward_out(tensor *out__, tensor out, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t index) { + PROTECT( + auto outputs__ = torch::select_backward_out(*out, *grad_output, torch::IntArrayRef(input_sizes_data, input_sizes_len), dim, index); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_select_copy(tensor *out__, tensor self, int64_t dim, int64_t index) { + PROTECT( + auto outputs__ = torch::select_copy(*self, dim, index); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_select_copy_int_out(tensor *out__, tensor out, tensor self, int64_t dim, int64_t index) { + PROTECT( + auto outputs__ = torch::select_copy_out(*out, *self, dim, index); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_select_scatter(tensor *out__, tensor self, tensor src, int64_t dim, int64_t index) { + PROTECT( + auto outputs__ = torch::select_scatter(*self, *src, dim, index); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_select_scatter_out(tensor *out__, tensor out, tensor self, tensor src, int64_t dim, int64_t index) { + PROTECT( + auto outputs__ = torch::select_scatter_out(*out, *self, *src, dim, index); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_selu(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::selu(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_selu_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::selu_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_set(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::set(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_set_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->set_(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_set_data(tensor self, tensor new_data) { + PROTECT( + self->set_data(*new_data); + ) +} + +void atg_set_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::set_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_set_requires_grad(tensor *out__, tensor self, int r) { + PROTECT( + auto outputs__ = self->set_requires_grad((bool)r); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_set_source_tensor(tensor *out__, tensor self, tensor source) { + PROTECT( + auto outputs__ = torch::set(*self, *source); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_set_source_tensor_(tensor *out__, tensor self, tensor source) { + PROTECT( + auto outputs__ = self->set_(*source); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_set_source_tensor_out(tensor *out__, tensor out, tensor self, tensor source) { + PROTECT( + auto outputs__ = torch::set_out(*out, *self, *source); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_set_source_tensor_storage_offset_(tensor *out__, tensor self, tensor source, int64_t storage_offset, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len) { + PROTECT( + auto outputs__ = self->set_(*source, storage_offset, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sgn(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::sgn(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sgn_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->sgn_(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sgn_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::sgn_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sigmoid(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::sigmoid(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sigmoid_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::sigmoid_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sigmoid_backward(tensor *out__, tensor grad_output, tensor output) { + PROTECT( + auto outputs__ = torch::sigmoid_backward(*grad_output, *output); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sigmoid_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor output) { + PROTECT( + auto outputs__ = torch::sigmoid_backward_out(*grad_input, *grad_output, *output); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sigmoid_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::sigmoid_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sign(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::sign(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sign_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->sign_(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sign_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::sign_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_signbit(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::signbit(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_signbit_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::signbit_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_silu(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::silu(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_silu_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::silu_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_silu_backward(tensor *out__, tensor grad_output, tensor self) { + PROTECT( + auto outputs__ = torch::silu_backward(*grad_output, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_silu_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self) { + PROTECT( + auto outputs__ = torch::silu_backward_out(*grad_input, *grad_output, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_silu_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::silu_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sin(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::sin(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sin_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::sin_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sin_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::sin_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sinc(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::sinc(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sinc_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::sinc_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sinc_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::sinc_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sinh(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::sinh(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sinh_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::sinh_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sinh_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::sinh_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_slice(tensor *out__, tensor self, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step) { + PROTECT( + auto outputs__ = torch::slice(*self, dim, start_null ? c10::nullopt : c10::optional(start_v), end_null ? c10::nullopt : c10::optional(end_v), step); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_slice_backward(tensor *out__, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t start, int64_t end, int64_t step) { + PROTECT( + auto outputs__ = torch::slice_backward(*grad_output, torch::IntArrayRef(input_sizes_data, input_sizes_len), dim, start, end, step); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_slice_backward_out(tensor *out__, tensor out, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t start, int64_t end, int64_t step) { + PROTECT( + auto outputs__ = torch::slice_backward_out(*out, *grad_output, torch::IntArrayRef(input_sizes_data, input_sizes_len), dim, start, end, step); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_slice_copy(tensor *out__, tensor self, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step) { + PROTECT( + auto outputs__ = torch::slice_copy(*self, dim, start_null ? c10::nullopt : c10::optional(start_v), end_null ? c10::nullopt : c10::optional(end_v), step); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_slice_copy_tensor_out(tensor *out__, tensor out, tensor self, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step) { + PROTECT( + auto outputs__ = torch::slice_copy_out(*out, *self, dim, start_null ? c10::nullopt : c10::optional(start_v), end_null ? c10::nullopt : c10::optional(end_v), step); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_slice_inverse(tensor *out__, tensor self, tensor src, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step) { + PROTECT( + auto outputs__ = torch::slice_inverse(*self, *src, dim, start_null ? c10::nullopt : c10::optional(start_v), end_null ? c10::nullopt : c10::optional(end_v), step); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_slice_scatter(tensor *out__, tensor self, tensor src, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step) { + PROTECT( + auto outputs__ = torch::slice_scatter(*self, *src, dim, start_null ? c10::nullopt : c10::optional(start_v), end_null ? c10::nullopt : c10::optional(end_v), step); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_slice_scatter_out(tensor *out__, tensor out, tensor self, tensor src, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step) { + PROTECT( + auto outputs__ = torch::slice_scatter_out(*out, *self, *src, dim, start_null ? c10::nullopt : c10::optional(start_v), end_null ? c10::nullopt : c10::optional(end_v), step); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_slogdet(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::slogdet(*self); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_slogdet_out(tensor *out__, tensor sign, tensor logabsdet, tensor self) { + PROTECT( + auto outputs__ = torch::slogdet_out(*sign, *logabsdet, *self); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_slow_conv3d(tensor *out__, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::slow_conv3d(*self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_slow_conv3d_out(tensor *out__, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len) { + PROTECT( + auto outputs__ = torch::slow_conv3d_out(*out, *self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_slow_conv_dilated2d(tensor *out__, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len) { + PROTECT( + auto outputs__ = torch::slow_conv_dilated2d(*self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_slow_conv_dilated2d_out(tensor *out__, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len) { + PROTECT( + auto outputs__ = torch::slow_conv_dilated2d_out(*out, *self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_slow_conv_dilated3d(tensor *out__, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len) { + PROTECT( + auto outputs__ = torch::slow_conv_dilated3d(*self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_slow_conv_dilated3d_out(tensor *out__, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len) { + PROTECT( + auto outputs__ = torch::slow_conv_dilated3d_out(*out, *self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_slow_conv_transpose2d(tensor *out__, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *dilation_data, int dilation_len) { + PROTECT( + auto outputs__ = torch::slow_conv_transpose2d(*self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), torch::IntArrayRef(dilation_data, dilation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_slow_conv_transpose2d_out(tensor *out__, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *dilation_data, int dilation_len) { + PROTECT( + auto outputs__ = torch::slow_conv_transpose2d_out(*out, *self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), torch::IntArrayRef(dilation_data, dilation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_slow_conv_transpose3d(tensor *out__, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *dilation_data, int dilation_len) { + PROTECT( + auto outputs__ = torch::slow_conv_transpose3d(*self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), torch::IntArrayRef(dilation_data, dilation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_slow_conv_transpose3d_out(tensor *out__, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *dilation_data, int dilation_len) { + PROTECT( + auto outputs__ = torch::slow_conv_transpose3d_out(*out, *self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), torch::IntArrayRef(dilation_data, dilation_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_smm(tensor *out__, tensor self, tensor mat2) { + PROTECT( + auto outputs__ = torch::smm(*self, *mat2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_smooth_l1_loss(tensor *out__, tensor self, tensor target, int64_t reduction, double beta) { + PROTECT( + auto outputs__ = torch::smooth_l1_loss(*self, *target, reduction, beta); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_smooth_l1_loss_backward(tensor *out__, tensor grad_output, tensor self, tensor target, int64_t reduction, double beta) { + PROTECT( + auto outputs__ = torch::smooth_l1_loss_backward(*grad_output, *self, *target, reduction, beta); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_smooth_l1_loss_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor target, int64_t reduction, double beta) { + PROTECT( + auto outputs__ = torch::smooth_l1_loss_backward_out(*grad_input, *grad_output, *self, *target, reduction, beta); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_smooth_l1_loss_out(tensor *out__, tensor out, tensor self, tensor target, int64_t reduction, double beta) { + PROTECT( + auto outputs__ = torch::smooth_l1_loss_out(*out, *self, *target, reduction, beta); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_soft_margin_loss(tensor *out__, tensor self, tensor target, int64_t reduction) { + PROTECT( + auto outputs__ = torch::soft_margin_loss(*self, *target, reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_soft_margin_loss_backward(tensor *out__, tensor grad_output, tensor self, tensor target, int64_t reduction) { + PROTECT( + auto outputs__ = torch::soft_margin_loss_backward(*grad_output, *self, *target, reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_soft_margin_loss_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor target, int64_t reduction) { + PROTECT( + auto outputs__ = torch::soft_margin_loss_backward_out(*grad_input, *grad_output, *self, *target, reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_soft_margin_loss_out(tensor *out__, tensor out, tensor self, tensor target, int64_t reduction) { + PROTECT( + auto outputs__ = torch::soft_margin_loss_out(*out, *self, *target, reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_softmax(tensor *out__, tensor self, int64_t dim, int dtype) { + PROTECT( + auto outputs__ = torch::softmax(*self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_softmax_int_out(tensor *out__, tensor out, tensor self, int64_t dim, int dtype) { + PROTECT( + auto outputs__ = torch::softmax_out(*out, *self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_softplus(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::softplus(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_softplus_backward(tensor *out__, tensor grad_output, tensor self, scalar beta, scalar threshold) { + PROTECT( + auto outputs__ = torch::softplus_backward(*grad_output, *self, *beta, *threshold); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_softplus_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, scalar beta, scalar threshold) { + PROTECT( + auto outputs__ = torch::softplus_backward_out(*grad_input, *grad_output, *self, *beta, *threshold); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_softplus_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::softplus_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_softshrink(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::softshrink(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_softshrink_backward(tensor *out__, tensor grad_output, tensor self, scalar lambd) { + PROTECT( + auto outputs__ = torch::softshrink_backward(*grad_output, *self, *lambd); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_softshrink_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, scalar lambd) { + PROTECT( + auto outputs__ = torch::softshrink_backward_out(*grad_input, *grad_output, *self, *lambd); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_softshrink_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::softshrink_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sort(tensor *out__, tensor self, int64_t dim, int descending) { + PROTECT( + auto outputs__ = torch::sort(*self, dim, (bool)descending); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_sort_stable(tensor *out__, tensor self, int stable, int64_t dim, int descending) { + PROTECT( + auto outputs__ = torch::sort(*self, (bool)stable, dim, (bool)descending); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_sort_values(tensor *out__, tensor values, tensor indices, tensor self, int64_t dim, int descending) { + PROTECT( + auto outputs__ = torch::sort_out(*values, *indices, *self, dim, (bool)descending); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_sort_values_stable(tensor *out__, tensor values, tensor indices, tensor self, int stable, int64_t dim, int descending) { + PROTECT( + auto outputs__ = torch::sort_out(*values, *indices, *self, (bool)stable, dim, (bool)descending); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_sparse_bsc_tensor(tensor *out__, tensor ccol_indices, tensor row_indices, tensor values, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::sparse_bsc_tensor(*ccol_indices, *row_indices, *values, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sparse_bsc_tensor_ccol_row_value_size(tensor *out__, tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::sparse_bsc_tensor(*ccol_indices, *row_indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sparse_bsr_tensor(tensor *out__, tensor crow_indices, tensor col_indices, tensor values, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::sparse_bsr_tensor(*crow_indices, *col_indices, *values, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sparse_bsr_tensor_crow_col_value_size(tensor *out__, tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::sparse_bsr_tensor(*crow_indices, *col_indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sparse_compressed_tensor(tensor *out__, tensor compressed_indices, tensor plain_indices, tensor values, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::sparse_compressed_tensor(*compressed_indices, *plain_indices, *values, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sparse_compressed_tensor_comp_plain_value_size(tensor *out__, tensor compressed_indices, tensor plain_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::sparse_compressed_tensor(*compressed_indices, *plain_indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sparse_coo_tensor(tensor *out__, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::sparse_coo_tensor(torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sparse_coo_tensor_indices(tensor *out__, tensor indices, tensor values, int options_kind, int options_device, int is_coalesced) { + PROTECT( + auto outputs__ = torch::sparse_coo_tensor(*indices, *values, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind)), (bool)is_coalesced); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sparse_coo_tensor_indices_size(tensor *out__, tensor indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device, int is_coalesced) { + PROTECT( + auto outputs__ = torch::sparse_coo_tensor(*indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind)), (bool)is_coalesced); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sparse_coo_tensor_size_out(tensor *out__, tensor out, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::sparse_coo_tensor_out(*out, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sparse_csc_tensor(tensor *out__, tensor ccol_indices, tensor row_indices, tensor values, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::sparse_csc_tensor(*ccol_indices, *row_indices, *values, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sparse_csc_tensor_ccol_row_value_size(tensor *out__, tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::sparse_csc_tensor(*ccol_indices, *row_indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sparse_csr_tensor(tensor *out__, tensor crow_indices, tensor col_indices, tensor values, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::sparse_csr_tensor(*crow_indices, *col_indices, *values, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sparse_csr_tensor_crow_col_value_size(tensor *out__, tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::sparse_csr_tensor(*crow_indices, *col_indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +int64_t atg_sparse_dim(tensor self) { + PROTECT( + return self->sparse_dim(); + ) + return 0; +} + +void atg_sparse_mask(tensor *out__, tensor self, tensor mask) { + PROTECT( + auto outputs__ = self->sparse_mask(*mask); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sparse_mask_out(tensor *out__, tensor out, tensor self, tensor mask) { + PROTECT( + auto outputs__ = torch::sparse_mask_out(*out, *self, *mask); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sparse_resize(tensor *out__, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim) { + PROTECT( + auto outputs__ = torch::sparse_resize(*self, torch::IntArrayRef(size_data, size_len), sparse_dim, dense_dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sparse_resize_(tensor *out__, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim) { + PROTECT( + auto outputs__ = self->sparse_resize_(torch::IntArrayRef(size_data, size_len), sparse_dim, dense_dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sparse_resize_and_clear(tensor *out__, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim) { + PROTECT( + auto outputs__ = torch::sparse_resize_and_clear(*self, torch::IntArrayRef(size_data, size_len), sparse_dim, dense_dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sparse_resize_and_clear_(tensor *out__, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim) { + PROTECT( + auto outputs__ = self->sparse_resize_and_clear_(torch::IntArrayRef(size_data, size_len), sparse_dim, dense_dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sparse_resize_and_clear_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim) { + PROTECT( + auto outputs__ = torch::sparse_resize_and_clear_out(*out, *self, torch::IntArrayRef(size_data, size_len), sparse_dim, dense_dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sparse_resize_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim) { + PROTECT( + auto outputs__ = torch::sparse_resize_out(*out, *self, torch::IntArrayRef(size_data, size_len), sparse_dim, dense_dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sparse_sampled_addmm(tensor *out__, tensor self, tensor mat1, tensor mat2) { + PROTECT( + auto outputs__ = torch::sparse_sampled_addmm(*self, *mat1, *mat2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sparse_sampled_addmm_out(tensor *out__, tensor out, tensor self, tensor mat1, tensor mat2) { + PROTECT( + auto outputs__ = torch::sparse_sampled_addmm_out(*out, *self, *mat1, *mat2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_airy_ai(tensor *out__, tensor x) { + PROTECT( + auto outputs__ = torch::special_airy_ai(*x); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_airy_ai_out(tensor *out__, tensor out, tensor x) { + PROTECT( + auto outputs__ = torch::special_airy_ai_out(*out, *x); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_bessel_j0(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_bessel_j0(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_bessel_j0_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_bessel_j0_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_bessel_j1(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_bessel_j1(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_bessel_j1_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_bessel_j1_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_bessel_y0(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_bessel_y0(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_bessel_y0_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_bessel_y0_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_bessel_y1(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_bessel_y1(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_bessel_y1_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_bessel_y1_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_t(tensor *out__, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_t(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_t_n_scalar(tensor *out__, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_t(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_t_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_t_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_t_out(tensor *out__, tensor out, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_t_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_t_x_scalar(tensor *out__, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_t(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_t_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_t_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_u(tensor *out__, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_u(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_u_n_scalar(tensor *out__, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_u(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_u_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_u_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_u_out(tensor *out__, tensor out, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_u_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_u_x_scalar(tensor *out__, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_u(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_u_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_u_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_v(tensor *out__, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_v(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_v_n_scalar(tensor *out__, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_v(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_v_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_v_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_v_out(tensor *out__, tensor out, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_v_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_v_x_scalar(tensor *out__, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_v(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_v_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_v_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_w(tensor *out__, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_w(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_w_n_scalar(tensor *out__, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_w(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_w_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_w_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_w_out(tensor *out__, tensor out, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_w_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_w_x_scalar(tensor *out__, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_w(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_chebyshev_polynomial_w_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_chebyshev_polynomial_w_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_digamma(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_digamma(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_digamma_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_digamma_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_entr(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_entr(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_entr_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_entr_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_erf(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_erf(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_erf_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_erf_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_erfc(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_erfc(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_erfc_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_erfc_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_erfcx(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_erfcx(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_erfcx_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_erfcx_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_erfinv(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_erfinv(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_erfinv_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_erfinv_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_exp2(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_exp2(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_exp2_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_exp2_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_expit(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_expit(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_expit_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_expit_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_expm1(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_expm1(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_expm1_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_expm1_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_gammainc(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::special_gammainc(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_gammainc_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::special_gammainc_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_gammaincc(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::special_gammaincc(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_gammaincc_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::special_gammaincc_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_gammaln(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_gammaln(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_gammaln_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_gammaln_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_hermite_polynomial_h(tensor *out__, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_hermite_polynomial_h(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_hermite_polynomial_h_n_scalar(tensor *out__, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_hermite_polynomial_h(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_hermite_polynomial_h_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_hermite_polynomial_h_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_hermite_polynomial_h_out(tensor *out__, tensor out, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_hermite_polynomial_h_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_hermite_polynomial_h_x_scalar(tensor *out__, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_hermite_polynomial_h(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_hermite_polynomial_h_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_hermite_polynomial_h_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_hermite_polynomial_he(tensor *out__, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_hermite_polynomial_he(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_hermite_polynomial_he_n_scalar(tensor *out__, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_hermite_polynomial_he(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_hermite_polynomial_he_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_hermite_polynomial_he_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_hermite_polynomial_he_out(tensor *out__, tensor out, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_hermite_polynomial_he_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_hermite_polynomial_he_x_scalar(tensor *out__, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_hermite_polynomial_he(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_hermite_polynomial_he_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_hermite_polynomial_he_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_i0(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_i0(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_i0_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_i0_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_i0e(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_i0e(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_i0e_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_i0e_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_i1(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_i1(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_i1_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_i1_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_i1e(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_i1e(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_i1e_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_i1e_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_laguerre_polynomial_l(tensor *out__, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_laguerre_polynomial_l(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_laguerre_polynomial_l_n_scalar(tensor *out__, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_laguerre_polynomial_l(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_laguerre_polynomial_l_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_laguerre_polynomial_l_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_laguerre_polynomial_l_out(tensor *out__, tensor out, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_laguerre_polynomial_l_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_laguerre_polynomial_l_x_scalar(tensor *out__, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_laguerre_polynomial_l(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_laguerre_polynomial_l_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_laguerre_polynomial_l_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_legendre_polynomial_p(tensor *out__, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_legendre_polynomial_p(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_legendre_polynomial_p_n_scalar(tensor *out__, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_legendre_polynomial_p(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_legendre_polynomial_p_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_legendre_polynomial_p_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_legendre_polynomial_p_out(tensor *out__, tensor out, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_legendre_polynomial_p_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_legendre_polynomial_p_x_scalar(tensor *out__, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_legendre_polynomial_p(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_legendre_polynomial_p_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_legendre_polynomial_p_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_log1p(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_log1p(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_log1p_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_log1p_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_log_ndtr(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_log_ndtr(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_log_ndtr_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_log_ndtr_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_log_softmax(tensor *out__, tensor self, int64_t dim, int dtype) { + PROTECT( + auto outputs__ = torch::special_log_softmax(*self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_logit(tensor *out__, tensor self, double eps_v, uint8_t eps_null) { + PROTECT( + auto outputs__ = torch::special_logit(*self, eps_null ? c10::nullopt : c10::optional(eps_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_logit_out(tensor *out__, tensor out, tensor self, double eps_v, uint8_t eps_null) { + PROTECT( + auto outputs__ = torch::special_logit_out(*out, *self, eps_null ? c10::nullopt : c10::optional(eps_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_logsumexp(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim) { + PROTECT( + auto outputs__ = torch::special_logsumexp(*self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_logsumexp_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim) { + PROTECT( + auto outputs__ = torch::special_logsumexp_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_modified_bessel_i0(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_modified_bessel_i0(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_modified_bessel_i0_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_modified_bessel_i0_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_modified_bessel_i1(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_modified_bessel_i1(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_modified_bessel_i1_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_modified_bessel_i1_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_modified_bessel_k0(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_modified_bessel_k0(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_modified_bessel_k0_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_modified_bessel_k0_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_modified_bessel_k1(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_modified_bessel_k1(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_modified_bessel_k1_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_modified_bessel_k1_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_multigammaln(tensor *out__, tensor self, int64_t p) { + PROTECT( + auto outputs__ = torch::special_multigammaln(*self, p); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_multigammaln_out(tensor *out__, tensor out, tensor self, int64_t p) { + PROTECT( + auto outputs__ = torch::special_multigammaln_out(*out, *self, p); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_ndtr(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_ndtr(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_ndtr_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_ndtr_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_ndtri(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_ndtri(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_ndtri_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_ndtri_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_polygamma(tensor *out__, int64_t n, tensor self) { + PROTECT( + auto outputs__ = torch::special_polygamma(n, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_polygamma_out(tensor *out__, tensor out, int64_t n, tensor self) { + PROTECT( + auto outputs__ = torch::special_polygamma_out(*out, n, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_psi(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_psi(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_psi_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_psi_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_round(tensor *out__, tensor self, int64_t decimals) { + PROTECT( + auto outputs__ = torch::special_round(*self, decimals); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_round_out(tensor *out__, tensor out, tensor self, int64_t decimals) { + PROTECT( + auto outputs__ = torch::special_round_out(*out, *self, decimals); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_scaled_modified_bessel_k0(tensor *out__, tensor x) { + PROTECT( + auto outputs__ = torch::special_scaled_modified_bessel_k0(*x); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_scaled_modified_bessel_k0_out(tensor *out__, tensor out, tensor x) { + PROTECT( + auto outputs__ = torch::special_scaled_modified_bessel_k0_out(*out, *x); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_scaled_modified_bessel_k1(tensor *out__, tensor x) { + PROTECT( + auto outputs__ = torch::special_scaled_modified_bessel_k1(*x); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_scaled_modified_bessel_k1_out(tensor *out__, tensor out, tensor x) { + PROTECT( + auto outputs__ = torch::special_scaled_modified_bessel_k1_out(*out, *x); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_t(tensor *out__, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_t(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_t_n_scalar(tensor *out__, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_t(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_t_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_t_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_t_out(tensor *out__, tensor out, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_t_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_t_x_scalar(tensor *out__, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_t(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_t_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_t_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_u(tensor *out__, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_u(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_u_n_scalar(tensor *out__, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_u(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_u_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_u_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_u_out(tensor *out__, tensor out, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_u_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_u_x_scalar(tensor *out__, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_u(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_u_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_u_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_v(tensor *out__, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_v(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_v_n_scalar(tensor *out__, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_v(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_v_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_v_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_v_out(tensor *out__, tensor out, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_v_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_v_x_scalar(tensor *out__, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_v(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_v_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_v_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_w(tensor *out__, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_w(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_w_n_scalar(tensor *out__, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_w(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_w_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_w_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_w_out(tensor *out__, tensor out, tensor x, tensor n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_w_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_w_x_scalar(tensor *out__, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_w(*x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_shifted_chebyshev_polynomial_w_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { + PROTECT( + auto outputs__ = torch::special_shifted_chebyshev_polynomial_w_out(*out, *x, *n); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_sinc(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::special_sinc(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_sinc_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::special_sinc_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_softmax(tensor *out__, tensor self, int64_t dim, int dtype) { + PROTECT( + auto outputs__ = torch::special_softmax(*self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_spherical_bessel_j0(tensor *out__, tensor x) { + PROTECT( + auto outputs__ = torch::special_spherical_bessel_j0(*x); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_spherical_bessel_j0_out(tensor *out__, tensor out, tensor x) { + PROTECT( + auto outputs__ = torch::special_spherical_bessel_j0_out(*out, *x); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_xlog1py(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::special_xlog1py(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_xlog1py_other_scalar(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::special_xlog1py(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_xlog1py_other_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::special_xlog1py_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_xlog1py_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::special_xlog1py_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_xlog1py_self_scalar(tensor *out__, scalar self_scalar, tensor other) { + PROTECT( + auto outputs__ = torch::special_xlog1py(*self_scalar, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_xlog1py_self_scalar_out(tensor *out__, tensor out, scalar self_scalar, tensor other) { + PROTECT( + auto outputs__ = torch::special_xlog1py_out(*out, *self_scalar, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_xlogy(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::special_xlogy(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_xlogy_other_scalar(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::special_xlogy(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_xlogy_other_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::special_xlogy_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_xlogy_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::special_xlogy_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_xlogy_self_scalar(tensor *out__, scalar self_scalar, tensor other) { + PROTECT( + auto outputs__ = torch::special_xlogy(*self_scalar, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_xlogy_self_scalar_out(tensor *out__, tensor out, scalar self_scalar, tensor other) { + PROTECT( + auto outputs__ = torch::special_xlogy_out(*out, *self_scalar, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_zeta(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::special_zeta(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_zeta_other_scalar(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::special_zeta(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_zeta_other_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::special_zeta_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_zeta_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::special_zeta_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_zeta_self_scalar(tensor *out__, scalar self_scalar, tensor other) { + PROTECT( + auto outputs__ = torch::special_zeta(*self_scalar, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_special_zeta_self_scalar_out(tensor *out__, tensor out, scalar self_scalar, tensor other) { + PROTECT( + auto outputs__ = torch::special_zeta_out(*out, *self_scalar, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +tensor *atg_split(tensor self, int64_t split_size, int64_t dim) { + PROTECT( + auto outputs__ = torch::split(*self, split_size, dim); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +tensor *atg_split_copy(tensor self, int64_t split_size, int64_t dim) { + PROTECT( + auto outputs__ = torch::split_copy(*self, split_size, dim); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg_split_copy_tensor_out(tensor *out_data, int out_len, tensor self, int64_t split_size, int64_t dim) { + PROTECT( + torch::split_copy_out(of_carray_tensor(out_data, out_len), *self, split_size, dim); + ) +} + +tensor *atg_split_sizes(tensor self, int64_t *split_size_data, int split_size_len, int64_t dim) { + PROTECT( + auto outputs__ = torch::split(*self, torch::IntArrayRef(split_size_data, split_size_len), dim); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +tensor *atg_split_with_sizes(tensor self, int64_t *split_sizes_data, int split_sizes_len, int64_t dim) { + PROTECT( + auto outputs__ = torch::split_with_sizes(*self, torch::IntArrayRef(split_sizes_data, split_sizes_len), dim); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +tensor *atg_split_with_sizes_copy(tensor self, int64_t *split_sizes_data, int split_sizes_len, int64_t dim) { + PROTECT( + auto outputs__ = torch::split_with_sizes_copy(*self, torch::IntArrayRef(split_sizes_data, split_sizes_len), dim); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg_split_with_sizes_copy_out(tensor *out_data, int out_len, tensor self, int64_t *split_sizes_data, int split_sizes_len, int64_t dim) { + PROTECT( + torch::split_with_sizes_copy_out(of_carray_tensor(out_data, out_len), *self, torch::IntArrayRef(split_sizes_data, split_sizes_len), dim); + ) +} + +void atg_sqrt(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::sqrt(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sqrt_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::sqrt_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sqrt_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::sqrt_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_square(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::square(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_square_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::square_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_square_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::square_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_squeeze(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::squeeze(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_squeeze_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->squeeze_(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_squeeze_copy(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::squeeze_copy(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_squeeze_copy_dim(tensor *out__, tensor self, int64_t dim) { + PROTECT( + auto outputs__ = torch::squeeze_copy(*self, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_squeeze_copy_dim_out(tensor *out__, tensor out, tensor self, int64_t dim) { + PROTECT( + auto outputs__ = torch::squeeze_copy_out(*out, *self, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_squeeze_copy_dims(tensor *out__, tensor self, int64_t *dim_data, int dim_len) { + PROTECT( + auto outputs__ = torch::squeeze_copy(*self, torch::IntArrayRef(dim_data, dim_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_squeeze_copy_dims_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len) { + PROTECT( + auto outputs__ = torch::squeeze_copy_out(*out, *self, torch::IntArrayRef(dim_data, dim_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_squeeze_copy_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::squeeze_copy_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_squeeze_dim(tensor *out__, tensor self, int64_t dim) { + PROTECT( + auto outputs__ = torch::squeeze(*self, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_squeeze_dim_(tensor *out__, tensor self, int64_t dim) { + PROTECT( + auto outputs__ = self->squeeze_(dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_squeeze_dims(tensor *out__, tensor self, int64_t *dim_data, int dim_len) { + PROTECT( + auto outputs__ = torch::squeeze(*self, torch::IntArrayRef(dim_data, dim_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_squeeze_dims_(tensor *out__, tensor self, int64_t *dim_data, int dim_len) { + PROTECT( + auto outputs__ = self->squeeze_(torch::IntArrayRef(dim_data, dim_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sspaddmm(tensor *out__, tensor self, tensor mat1, tensor mat2) { + PROTECT( + auto outputs__ = torch::sspaddmm(*self, *mat1, *mat2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sspaddmm_out(tensor *out__, tensor out, tensor self, tensor mat1, tensor mat2) { + PROTECT( + auto outputs__ = torch::sspaddmm_out(*out, *self, *mat1, *mat2); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_stack(tensor *out__, tensor *tensors_data, int tensors_len, int64_t dim) { + PROTECT( + auto outputs__ = torch::stack(of_carray_tensor(tensors_data, tensors_len), dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_stack_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len, int64_t dim) { + PROTECT( + auto outputs__ = torch::stack_out(*out, of_carray_tensor(tensors_data, tensors_len), dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_std(tensor *out__, tensor self, int unbiased) { + PROTECT( + auto outputs__ = torch::std(*self, (bool)unbiased); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_std_correction(tensor *out__, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim) { + PROTECT( + auto outputs__ = torch::std(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), *correction, (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_std_correction_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim) { + PROTECT( + auto outputs__ = torch::std_out(*out, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), *correction, (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_std_dim(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim) { + PROTECT( + auto outputs__ = torch::std(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)unbiased, (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_std_mean(tensor *out__, tensor self, int unbiased) { + PROTECT( + auto outputs__ = torch::std_mean(*self, (bool)unbiased); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_std_mean_correction(tensor *out__, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim) { + PROTECT( + auto outputs__ = torch::std_mean(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), *correction, (bool)keepdim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_std_mean_correction_out(tensor *out__, tensor out0, tensor out1, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim) { + PROTECT( + auto outputs__ = torch::std_mean_out(*out0, *out1, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), *correction, (bool)keepdim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_std_mean_dim(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim) { + PROTECT( + auto outputs__ = torch::std_mean(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)unbiased, (bool)keepdim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_std_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim) { + PROTECT( + auto outputs__ = torch::std_out(*out, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)unbiased, (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_stft(tensor *out__, tensor self, int64_t n_fft, int64_t hop_length_v, uint8_t hop_length_null, int64_t win_length_v, uint8_t win_length_null, tensor window, int normalized, int onesided, int return_complex, int align_to_window) { + PROTECT( + auto outputs__ = torch::stft(*self, n_fft, hop_length_null ? c10::nullopt : c10::optional(hop_length_v), win_length_null ? c10::nullopt : c10::optional(win_length_v), (window ? ::std::optional(*window) : ::std::nullopt), (bool)normalized, (bool)onesided, (bool)return_complex, (bool)align_to_window); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_stft_center(tensor *out__, tensor self, int64_t n_fft, int64_t hop_length_v, uint8_t hop_length_null, int64_t win_length_v, uint8_t win_length_null, tensor window, int center, char* pad_mode_ptr, int pad_mode_len, int normalized, int onesided, int return_complex, int align_to_window) { + PROTECT( + auto outputs__ = torch::stft(*self, n_fft, hop_length_null ? c10::nullopt : c10::optional(hop_length_v), win_length_null ? c10::nullopt : c10::optional(win_length_v), (window ? ::std::optional(*window) : ::std::nullopt), (bool)center, std::string(pad_mode_ptr, pad_mode_len), (bool)normalized, (bool)onesided, (bool)return_complex, (bool)align_to_window); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sub(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::sub(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sub_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->sub_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sub_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::sub_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sub_scalar(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::sub(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sub_scalar_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->sub_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sub_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::sub_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_subtract(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::subtract(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_subtract_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->subtract_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_subtract_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::subtract_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_subtract_scalar(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::subtract(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_subtract_scalar_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->subtract_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sum(tensor *out__, tensor self, int dtype) { + PROTECT( + auto outputs__ = torch::sum(*self, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sum_dim_intlist(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { + PROTECT( + auto outputs__ = torch::sum(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sum_intlist_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { + PROTECT( + auto outputs__ = torch::sum_out(*out, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sum_out(tensor *out__, tensor out, tensor self, int dtype) { + PROTECT( + auto outputs__ = torch::sum_out(*out, *self, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_sum_to_size(tensor *out__, tensor self, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = self->sum_to_size(torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_svd(tensor *out__, tensor self, int some, int compute_uv) { + PROTECT( + auto outputs__ = torch::svd(*self, (bool)some, (bool)compute_uv); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_svd_u(tensor *out__, tensor U, tensor S, tensor V, tensor self, int some, int compute_uv) { + PROTECT( + auto outputs__ = torch::svd_out(*U, *S, *V, *self, (bool)some, (bool)compute_uv); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_swapaxes(tensor *out__, tensor self, int64_t axis0, int64_t axis1) { + PROTECT( + auto outputs__ = torch::swapaxes(*self, axis0, axis1); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_swapaxes_(tensor *out__, tensor self, int64_t axis0, int64_t axis1) { + PROTECT( + auto outputs__ = self->swapaxes_(axis0, axis1); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_swapdims(tensor *out__, tensor self, int64_t dim0, int64_t dim1) { + PROTECT( + auto outputs__ = torch::swapdims(*self, dim0, dim1); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_swapdims_(tensor *out__, tensor self, int64_t dim0, int64_t dim1) { + PROTECT( + auto outputs__ = self->swapdims_(dim0, dim1); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_t(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::t(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_t_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->t_(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_t_copy(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::t_copy(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_t_copy_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::t_copy_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_take(tensor *out__, tensor self, tensor index) { + PROTECT( + auto outputs__ = torch::take(*self, *index); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_take_along_dim(tensor *out__, tensor self, tensor indices, int64_t dim_v, uint8_t dim_null) { + PROTECT( + auto outputs__ = torch::take_along_dim(*self, *indices, dim_null ? c10::nullopt : c10::optional(dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_take_along_dim_out(tensor *out__, tensor out, tensor self, tensor indices, int64_t dim_v, uint8_t dim_null) { + PROTECT( + auto outputs__ = torch::take_along_dim_out(*out, *self, *indices, dim_null ? c10::nullopt : c10::optional(dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_take_out(tensor *out__, tensor out, tensor self, tensor index) { + PROTECT( + auto outputs__ = torch::take_out(*out, *self, *index); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_tan(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::tan(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_tan_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::tan_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_tan_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::tan_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_tanh(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::tanh(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_tanh_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::tanh_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_tanh_backward(tensor *out__, tensor grad_output, tensor output) { + PROTECT( + auto outputs__ = torch::tanh_backward(*grad_output, *output); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_tanh_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor output) { + PROTECT( + auto outputs__ = torch::tanh_backward_out(*grad_input, *grad_output, *output); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_tanh_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::tanh_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +tensor *atg_tensor_split(tensor self, int64_t sections, int64_t dim) { + PROTECT( + auto outputs__ = torch::tensor_split(*self, sections, dim); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +tensor *atg_tensor_split_indices(tensor self, int64_t *indices_data, int indices_len, int64_t dim) { + PROTECT( + auto outputs__ = torch::tensor_split(*self, torch::IntArrayRef(indices_data, indices_len), dim); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +tensor *atg_tensor_split_tensor_indices_or_sections(tensor self, tensor tensor_indices_or_sections, int64_t dim) { + PROTECT( + auto outputs__ = torch::tensor_split(*self, *tensor_indices_or_sections, dim); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg_tensordot(tensor *out__, tensor self, tensor other, int64_t *dims_self_data, int dims_self_len, int64_t *dims_other_data, int dims_other_len) { + PROTECT( + auto outputs__ = torch::tensordot(*self, *other, torch::IntArrayRef(dims_self_data, dims_self_len), torch::IntArrayRef(dims_other_data, dims_other_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_tensordot_out(tensor *out__, tensor out, tensor self, tensor other, int64_t *dims_self_data, int dims_self_len, int64_t *dims_other_data, int dims_other_len) { + PROTECT( + auto outputs__ = torch::tensordot_out(*out, *self, *other, torch::IntArrayRef(dims_self_data, dims_self_len), torch::IntArrayRef(dims_other_data, dims_other_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_threshold(tensor *out__, tensor self, scalar threshold, scalar value) { + PROTECT( + auto outputs__ = torch::threshold(*self, *threshold, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_threshold_(tensor *out__, tensor self, scalar threshold, scalar value) { + PROTECT( + auto outputs__ = torch::threshold_(*self, *threshold, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_threshold_backward(tensor *out__, tensor grad_output, tensor self, scalar threshold) { + PROTECT( + auto outputs__ = torch::threshold_backward(*grad_output, *self, *threshold); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_threshold_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, scalar threshold) { + PROTECT( + auto outputs__ = torch::threshold_backward_out(*grad_input, *grad_output, *self, *threshold); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_threshold_out(tensor *out__, tensor out, tensor self, scalar threshold, scalar value) { + PROTECT( + auto outputs__ = torch::threshold_out(*out, *self, *threshold, *value); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_tile(tensor *out__, tensor self, int64_t *dims_data, int dims_len) { + PROTECT( + auto outputs__ = torch::tile(*self, torch::IntArrayRef(dims_data, dims_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_to(tensor *out__, tensor self, int device) { + PROTECT( + auto outputs__ = self->to(device_of_int(device)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_to_dense(tensor *out__, tensor self, int dtype, int masked_grad) { + PROTECT( + auto outputs__ = self->to_dense(dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype)), (bool)masked_grad); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_to_dense_backward(tensor *out__, tensor grad, tensor input, int masked_grad) { + PROTECT( + auto outputs__ = torch::to_dense_backward(*grad, *input, (bool)masked_grad); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_to_device(tensor *out__, tensor self, int device, int dtype, int non_blocking, int copy) { + PROTECT( + auto outputs__ = self->to(device_of_int(device), at::ScalarType(dtype), (bool)non_blocking, (bool)copy); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_to_dtype(tensor *out__, tensor self, int dtype, int non_blocking, int copy) { + PROTECT( + auto outputs__ = self->to(at::ScalarType(dtype), (bool)non_blocking, (bool)copy); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_to_dtype_layout(tensor *out__, tensor self, int options_kind, int options_device, int non_blocking, int copy) { + PROTECT( + auto outputs__ = self->to(at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind)), (bool)non_blocking, (bool)copy); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_to_mkldnn(tensor *out__, tensor self, int dtype) { + PROTECT( + auto outputs__ = self->to_mkldnn(dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_to_mkldnn_backward(tensor *out__, tensor grad, tensor input) { + PROTECT( + auto outputs__ = torch::to_mkldnn_backward(*grad, *input); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_to_mkldnn_out(tensor *out__, tensor out, tensor self, int dtype) { + PROTECT( + auto outputs__ = torch::to_mkldnn_out(*out, *self, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_to_other(tensor *out__, tensor self, tensor other, int non_blocking, int copy) { + PROTECT( + auto outputs__ = self->to(*other, (bool)non_blocking, (bool)copy); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_to_padded_tensor(tensor *out__, tensor self, double padding, int64_t *output_size_data, int output_size_len) { + PROTECT( + auto outputs__ = self->to_padded_tensor(padding, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_to_padded_tensor_out(tensor *out__, tensor out, tensor self, double padding, int64_t *output_size_data, int output_size_len) { + PROTECT( + auto outputs__ = torch::to_padded_tensor_out(*out, *self, padding, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_to_sparse(tensor *out__, tensor self, int8_t layout, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null) { + PROTECT( + auto outputs__ = self->to_sparse((layout == -1 ? c10::nullopt : c10::optional(static_cast(layout))), blocksize_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(blocksize_data, blocksize_len)), dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_to_sparse_bsc(tensor *out__, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null) { + PROTECT( + auto outputs__ = self->to_sparse_bsc(torch::IntArrayRef(blocksize_data, blocksize_len), dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_to_sparse_bsr(tensor *out__, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null) { + PROTECT( + auto outputs__ = self->to_sparse_bsr(torch::IntArrayRef(blocksize_data, blocksize_len), dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_to_sparse_csc(tensor *out__, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null) { + PROTECT( + auto outputs__ = self->to_sparse_csc(dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_to_sparse_csr(tensor *out__, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null) { + PROTECT( + auto outputs__ = self->to_sparse_csr(dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_to_sparse_sparse_dim(tensor *out__, tensor self, int64_t sparse_dim) { + PROTECT( + auto outputs__ = self->to_sparse(sparse_dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_topk(tensor *out__, tensor self, int64_t k, int64_t dim, int largest, int sorted) { + PROTECT( + auto outputs__ = torch::topk(*self, k, dim, (bool)largest, (bool)sorted); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_topk_values(tensor *out__, tensor values, tensor indices, tensor self, int64_t k, int64_t dim, int largest, int sorted) { + PROTECT( + auto outputs__ = torch::topk_out(*values, *indices, *self, k, dim, (bool)largest, (bool)sorted); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_totype(tensor *out__, tensor self, int scalar_type) { + PROTECT( + auto outputs__ = self->toType(at::ScalarType(scalar_type)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_trace(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::trace(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_trace_backward(tensor *out__, tensor grad, int64_t *sizes_data, int sizes_len) { + PROTECT( + auto outputs__ = torch::trace_backward(*grad, torch::IntArrayRef(sizes_data, sizes_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_trace_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::trace_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_transpose(tensor *out__, tensor self, int64_t dim0, int64_t dim1) { + PROTECT( + auto outputs__ = torch::transpose(*self, dim0, dim1); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_transpose_(tensor *out__, tensor self, int64_t dim0, int64_t dim1) { + PROTECT( + auto outputs__ = self->transpose_(dim0, dim1); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_transpose_copy(tensor *out__, tensor self, int64_t dim0, int64_t dim1) { + PROTECT( + auto outputs__ = torch::transpose_copy(*self, dim0, dim1); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_transpose_copy_int_out(tensor *out__, tensor out, tensor self, int64_t dim0, int64_t dim1) { + PROTECT( + auto outputs__ = torch::transpose_copy_out(*out, *self, dim0, dim1); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_trapezoid(tensor *out__, tensor y, int64_t dim) { + PROTECT( + auto outputs__ = torch::trapezoid(*y, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_trapezoid_x(tensor *out__, tensor y, tensor x, int64_t dim) { + PROTECT( + auto outputs__ = torch::trapezoid(*y, *x, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_trapz(tensor *out__, tensor y, tensor x, int64_t dim) { + PROTECT( + auto outputs__ = torch::trapz(*y, *x, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_trapz_dx(tensor *out__, tensor y, double dx, int64_t dim) { + PROTECT( + auto outputs__ = torch::trapz(*y, dx, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_triangular_solve(tensor *out__, tensor self, tensor A, int upper, int transpose, int unitriangular) { + PROTECT( + auto outputs__ = torch::triangular_solve(*self, *A, (bool)upper, (bool)transpose, (bool)unitriangular); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_triangular_solve_x(tensor *out__, tensor X, tensor M, tensor self, tensor A, int upper, int transpose, int unitriangular) { + PROTECT( + auto outputs__ = torch::triangular_solve_out(*X, *M, *self, *A, (bool)upper, (bool)transpose, (bool)unitriangular); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_tril(tensor *out__, tensor self, int64_t diagonal) { + PROTECT( + auto outputs__ = torch::tril(*self, diagonal); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_tril_(tensor *out__, tensor self, int64_t diagonal) { + PROTECT( + auto outputs__ = self->tril_(diagonal); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_tril_indices(tensor *out__, int64_t row, int64_t col, int64_t offset, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::tril_indices(row, col, offset, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_tril_indices_out(tensor *out__, tensor out, int64_t row, int64_t col, int64_t offset) { + PROTECT( + auto outputs__ = torch::tril_indices_out(*out, row, col, offset); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_tril_out(tensor *out__, tensor out, tensor self, int64_t diagonal) { + PROTECT( + auto outputs__ = torch::tril_out(*out, *self, diagonal); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_triplet_margin_loss(tensor *out__, tensor anchor, tensor positive, tensor negative, double margin, double p, double eps, int swap, int64_t reduction) { + PROTECT( + auto outputs__ = torch::triplet_margin_loss(*anchor, *positive, *negative, margin, p, eps, (bool)swap, reduction); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_triu(tensor *out__, tensor self, int64_t diagonal) { + PROTECT( + auto outputs__ = torch::triu(*self, diagonal); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_triu_(tensor *out__, tensor self, int64_t diagonal) { + PROTECT( + auto outputs__ = self->triu_(diagonal); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_triu_indices(tensor *out__, int64_t row, int64_t col, int64_t offset, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::triu_indices(row, col, offset, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_triu_indices_out(tensor *out__, tensor out, int64_t row, int64_t col, int64_t offset) { + PROTECT( + auto outputs__ = torch::triu_indices_out(*out, row, col, offset); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_triu_out(tensor *out__, tensor out, tensor self, int64_t diagonal) { + PROTECT( + auto outputs__ = torch::triu_out(*out, *self, diagonal); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_true_divide(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::true_divide(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_true_divide_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->true_divide_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_true_divide_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::true_divide_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_true_divide_scalar(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::true_divide(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_true_divide_scalar_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = self->true_divide_(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_trunc(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::trunc(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_trunc_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::trunc_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_trunc_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::trunc_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_type_as(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->type_as(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +tensor *atg_unbind(tensor self, int64_t dim) { + PROTECT( + auto outputs__ = torch::unbind(*self, dim); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +tensor *atg_unbind_copy(tensor self, int64_t dim) { + PROTECT( + auto outputs__ = torch::unbind_copy(*self, dim); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg_unbind_copy_int_out(tensor *out_data, int out_len, tensor self, int64_t dim) { + PROTECT( + torch::unbind_copy_out(of_carray_tensor(out_data, out_len), *self, dim); + ) +} + +void atg_unflatten(tensor *out__, tensor self, int64_t dim, int64_t *sizes_data, int sizes_len) { + PROTECT( + auto outputs__ = torch::unflatten(*self, dim, torch::IntArrayRef(sizes_data, sizes_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +tensor *atg_unflatten_dense_tensors(tensor flat, tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::unflatten_dense_tensors(*flat, of_carray_tensor(tensors_data, tensors_len)); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg_unfold(tensor *out__, tensor self, int64_t dimension, int64_t size, int64_t step) { + PROTECT( + auto outputs__ = self->unfold(dimension, size, step); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_unfold_backward(tensor *out__, tensor grad_in, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t size, int64_t step) { + PROTECT( + auto outputs__ = torch::unfold_backward(*grad_in, torch::IntArrayRef(input_sizes_data, input_sizes_len), dim, size, step); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_unfold_backward_out(tensor *out__, tensor out, tensor grad_in, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t size, int64_t step) { + PROTECT( + auto outputs__ = torch::unfold_backward_out(*out, *grad_in, torch::IntArrayRef(input_sizes_data, input_sizes_len), dim, size, step); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_unfold_copy(tensor *out__, tensor self, int64_t dimension, int64_t size, int64_t step) { + PROTECT( + auto outputs__ = torch::unfold_copy(*self, dimension, size, step); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_unfold_copy_out(tensor *out__, tensor out, tensor self, int64_t dimension, int64_t size, int64_t step) { + PROTECT( + auto outputs__ = torch::unfold_copy_out(*out, *self, dimension, size, step); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_uniform(tensor *out__, tensor self, double from, double to) { + PROTECT( + auto outputs__ = torch::uniform(*self, from, to); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_uniform_(tensor *out__, tensor self, double from, double to) { + PROTECT( + auto outputs__ = self->uniform_(from, to); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_uniform_out(tensor *out__, tensor out, tensor self, double from, double to) { + PROTECT( + auto outputs__ = torch::uniform_out(*out, *self, from, to); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_unique_consecutive(tensor *out__, tensor self, int return_inverse, int return_counts, int64_t dim_v, uint8_t dim_null) { + PROTECT( + auto outputs__ = torch::unique_consecutive(*self, (bool)return_inverse, (bool)return_counts, dim_null ? c10::nullopt : c10::optional(dim_v)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_unique_consecutive_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor self, int return_inverse, int return_counts, int64_t dim_v, uint8_t dim_null) { + PROTECT( + auto outputs__ = torch::unique_consecutive_out(*out0, *out1, *out2, *self, (bool)return_inverse, (bool)return_counts, dim_null ? c10::nullopt : c10::optional(dim_v)); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_unique_dim(tensor *out__, tensor self, int64_t dim, int sorted, int return_inverse, int return_counts) { + PROTECT( + auto outputs__ = torch::unique_dim(*self, dim, (bool)sorted, (bool)return_inverse, (bool)return_counts); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_unique_dim_consecutive(tensor *out__, tensor self, int64_t dim, int return_inverse, int return_counts) { + PROTECT( + auto outputs__ = torch::unique_dim_consecutive(*self, dim, (bool)return_inverse, (bool)return_counts); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_unique_dim_consecutive_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor self, int64_t dim, int return_inverse, int return_counts) { + PROTECT( + auto outputs__ = torch::unique_dim_consecutive_out(*out0, *out1, *out2, *self, dim, (bool)return_inverse, (bool)return_counts); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +void atg_unique_dim_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor self, int64_t dim, int sorted, int return_inverse, int return_counts) { + PROTECT( + auto outputs__ = torch::unique_dim_out(*out0, *out1, *out2, *self, dim, (bool)sorted, (bool)return_inverse, (bool)return_counts); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + out__[2] = new torch::Tensor(std::get<2>(outputs__)); + ) +} + +tensor *atg_unsafe_chunk(tensor self, int64_t chunks, int64_t dim) { + PROTECT( + auto outputs__ = torch::unsafe_chunk(*self, chunks, dim); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +tensor *atg_unsafe_split(tensor self, int64_t split_size, int64_t dim) { + PROTECT( + auto outputs__ = torch::unsafe_split(*self, split_size, dim); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg_unsafe_split_tensor_out(tensor *out_data, int out_len, tensor self, int64_t split_size, int64_t dim) { + PROTECT( + torch::unsafe_split_out(of_carray_tensor(out_data, out_len), *self, split_size, dim); + ) +} + +tensor *atg_unsafe_split_with_sizes(tensor self, int64_t *split_sizes_data, int split_sizes_len, int64_t dim) { + PROTECT( + auto outputs__ = torch::unsafe_split_with_sizes(*self, torch::IntArrayRef(split_sizes_data, split_sizes_len), dim); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg_unsafe_split_with_sizes_out(tensor *out_data, int out_len, tensor self, int64_t *split_sizes_data, int split_sizes_len, int64_t dim) { + PROTECT( + torch::unsafe_split_with_sizes_out(of_carray_tensor(out_data, out_len), *self, torch::IntArrayRef(split_sizes_data, split_sizes_len), dim); + ) +} + +void atg_unsqueeze(tensor *out__, tensor self, int64_t dim) { + PROTECT( + auto outputs__ = torch::unsqueeze(*self, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_unsqueeze_(tensor *out__, tensor self, int64_t dim) { + PROTECT( + auto outputs__ = self->unsqueeze_(dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_unsqueeze_copy(tensor *out__, tensor self, int64_t dim) { + PROTECT( + auto outputs__ = torch::unsqueeze_copy(*self, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_unsqueeze_copy_out(tensor *out__, tensor out, tensor self, int64_t dim) { + PROTECT( + auto outputs__ = torch::unsqueeze_copy_out(*out, *self, dim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_bicubic2d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::upsample_bicubic2d(*self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_bicubic2d_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::upsample_bicubic2d_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_bicubic2d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::upsample_bicubic2d_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_bicubic2d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::upsample_bicubic2d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_bicubic2d_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len) { + PROTECT( + auto outputs__ = torch::upsample_bicubic2d(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), (bool)align_corners, at::ArrayRef(scale_factors_data, scale_factors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_bilinear2d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::upsample_bilinear2d(*self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_bilinear2d_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::upsample_bilinear2d_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_bilinear2d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::upsample_bilinear2d_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_bilinear2d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::upsample_bilinear2d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_bilinear2d_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len) { + PROTECT( + auto outputs__ = torch::upsample_bilinear2d(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), (bool)align_corners, at::ArrayRef(scale_factors_data, scale_factors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_bilinear2d_vec_out(tensor *out__, tensor out, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len) { + PROTECT( + auto outputs__ = torch::upsample_bilinear2d_out(*out, *input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), (bool)align_corners, at::ArrayRef(scale_factors_data, scale_factors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_linear1d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_v, uint8_t scales_null) { + PROTECT( + auto outputs__ = torch::upsample_linear1d(*self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_null ? c10::nullopt : c10::optional(scales_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_linear1d_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_v, uint8_t scales_null) { + PROTECT( + auto outputs__ = torch::upsample_linear1d_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_null ? c10::nullopt : c10::optional(scales_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_linear1d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_v, uint8_t scales_null) { + PROTECT( + auto outputs__ = torch::upsample_linear1d_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_null ? c10::nullopt : c10::optional(scales_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_linear1d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_v, uint8_t scales_null) { + PROTECT( + auto outputs__ = torch::upsample_linear1d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_null ? c10::nullopt : c10::optional(scales_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_linear1d_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len) { + PROTECT( + auto outputs__ = torch::upsample_linear1d(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), (bool)align_corners, at::ArrayRef(scale_factors_data, scale_factors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_nearest1d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, double scales_v, uint8_t scales_null) { + PROTECT( + auto outputs__ = torch::upsample_nearest1d(*self, torch::IntArrayRef(output_size_data, output_size_len), scales_null ? c10::nullopt : c10::optional(scales_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_nearest1d_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_v, uint8_t scales_null) { + PROTECT( + auto outputs__ = torch::upsample_nearest1d_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_null ? c10::nullopt : c10::optional(scales_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_nearest1d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_v, uint8_t scales_null) { + PROTECT( + auto outputs__ = torch::upsample_nearest1d_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_null ? c10::nullopt : c10::optional(scales_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_nearest1d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_v, uint8_t scales_null) { + PROTECT( + auto outputs__ = torch::upsample_nearest1d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), scales_null ? c10::nullopt : c10::optional(scales_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_nearest1d_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len) { + PROTECT( + auto outputs__ = torch::upsample_nearest1d(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), at::ArrayRef(scale_factors_data, scale_factors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_nearest2d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::upsample_nearest2d(*self, torch::IntArrayRef(output_size_data, output_size_len), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_nearest2d_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::upsample_nearest2d_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_nearest2d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::upsample_nearest2d_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_nearest2d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::upsample_nearest2d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_nearest2d_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len) { + PROTECT( + auto outputs__ = torch::upsample_nearest2d(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), at::ArrayRef(scale_factors_data, scale_factors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_nearest2d_vec_out(tensor *out__, tensor out, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len) { + PROTECT( + auto outputs__ = torch::upsample_nearest2d_out(*out, *input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), at::ArrayRef(scale_factors_data, scale_factors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_nearest3d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::upsample_nearest3d(*self, torch::IntArrayRef(output_size_data, output_size_len), scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_nearest3d_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::upsample_nearest3d_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_nearest3d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::upsample_nearest3d_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_nearest3d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::upsample_nearest3d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_nearest3d_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len) { + PROTECT( + auto outputs__ = torch::upsample_nearest3d(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), at::ArrayRef(scale_factors_data, scale_factors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_trilinear3d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::upsample_trilinear3d(*self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_trilinear3d_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::upsample_trilinear3d_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_trilinear3d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::upsample_trilinear3d_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_trilinear3d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { + PROTECT( + auto outputs__ = torch::upsample_trilinear3d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_upsample_trilinear3d_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len) { + PROTECT( + auto outputs__ = torch::upsample_trilinear3d(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), (bool)align_corners, at::ArrayRef(scale_factors_data, scale_factors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_value_selecting_reduction_backward(tensor *out__, tensor grad, int64_t dim, tensor indices, int64_t *sizes_data, int sizes_len, int keepdim) { + PROTECT( + auto outputs__ = torch::value_selecting_reduction_backward(*grad, dim, *indices, torch::IntArrayRef(sizes_data, sizes_len), (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_values(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = self->values(); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_values_copy(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::values_copy(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_values_copy_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::values_copy_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_vander(tensor *out__, tensor x, int64_t n_v, uint8_t n_null, int increasing) { + PROTECT( + auto outputs__ = torch::vander(*x, n_null ? c10::nullopt : c10::optional(n_v), (bool)increasing); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_var(tensor *out__, tensor self, int unbiased) { + PROTECT( + auto outputs__ = torch::var(*self, (bool)unbiased); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_var_correction(tensor *out__, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim) { + PROTECT( + auto outputs__ = torch::var(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), *correction, (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_var_correction_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim) { + PROTECT( + auto outputs__ = torch::var_out(*out, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), *correction, (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_var_dim(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim) { + PROTECT( + auto outputs__ = torch::var(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)unbiased, (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_var_mean(tensor *out__, tensor self, int unbiased) { + PROTECT( + auto outputs__ = torch::var_mean(*self, (bool)unbiased); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_var_mean_correction(tensor *out__, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim) { + PROTECT( + auto outputs__ = torch::var_mean(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), *correction, (bool)keepdim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_var_mean_correction_out(tensor *out__, tensor out0, tensor out1, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim) { + PROTECT( + auto outputs__ = torch::var_mean_out(*out0, *out1, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), *correction, (bool)keepdim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_var_mean_dim(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim) { + PROTECT( + auto outputs__ = torch::var_mean(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)unbiased, (bool)keepdim); + out__[0] = new torch::Tensor(std::get<0>(outputs__)); + out__[1] = new torch::Tensor(std::get<1>(outputs__)); + ) +} + +void atg_var_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim) { + PROTECT( + auto outputs__ = torch::var_out(*out, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)unbiased, (bool)keepdim); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_vdot(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::vdot(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_vdot_out(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::vdot_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_view(tensor *out__, tensor self, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = self->view(torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_view_as(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = self->view_as(*other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_view_as_complex(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::view_as_complex(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_view_as_complex_copy(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::view_as_complex_copy(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_view_as_complex_copy_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::view_as_complex_copy_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_view_as_real(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::view_as_real(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_view_as_real_copy(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::view_as_real_copy(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_view_as_real_copy_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::view_as_real_copy_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_view_copy(tensor *out__, tensor self, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::view_copy(*self, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_view_copy_dtype(tensor *out__, tensor self, int dtype) { + PROTECT( + auto outputs__ = torch::view_copy(*self, at::ScalarType(dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_view_copy_dtype_out(tensor *out__, tensor out, tensor self, int dtype) { + PROTECT( + auto outputs__ = torch::view_copy_out(*out, *self, at::ScalarType(dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_view_copy_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::view_copy_out(*out, *self, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_view_dtype(tensor *out__, tensor self, int dtype) { + PROTECT( + auto outputs__ = self->view(at::ScalarType(dtype)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +tensor *atg_vsplit(tensor self, int64_t sections) { + PROTECT( + auto outputs__ = torch::vsplit(*self, sections); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +tensor *atg_vsplit_array(tensor self, int64_t *indices_data, int indices_len) { + PROTECT( + auto outputs__ = torch::vsplit(*self, torch::IntArrayRef(indices_data, indices_len)); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg_vstack(tensor *out__, tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::vstack(of_carray_tensor(tensors_data, tensors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_vstack_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len) { + PROTECT( + auto outputs__ = torch::vstack_out(*out, of_carray_tensor(tensors_data, tensors_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + +tensor *atg_where(tensor condition) { + PROTECT( + auto outputs__ = torch::where(*condition); + int sz = outputs__.size(); + torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); + for (int i = 0; i < sz; ++i) + out__[i] = new torch::Tensor(outputs__[i]); + out__[sz] = nullptr; + return out__; + ) + return nullptr; +} + +void atg_where_scalar(tensor *out__, tensor condition, scalar self_scalar, scalar other) { + PROTECT( + auto outputs__ = torch::where(*condition, *self_scalar, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_where_scalarother(tensor *out__, tensor condition, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::where(*condition, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_where_scalarself(tensor *out__, tensor condition, scalar self_scalar, tensor other) { + PROTECT( + auto outputs__ = torch::where(*condition, *self_scalar, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_where_self(tensor *out__, tensor condition, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::where(*condition, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_where_self_out(tensor *out__, tensor out, tensor condition, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::where_out(*out, *condition, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_xlogy(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::xlogy(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_xlogy_(tensor *out__, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::xlogy_(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_xlogy_outscalar_other(tensor *out__, tensor out, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::xlogy_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_xlogy_outscalar_self(tensor *out__, tensor out, scalar self_scalar, tensor other) { + PROTECT( + auto outputs__ = torch::xlogy_out(*out, *self_scalar, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_xlogy_outtensor(tensor *out__, tensor out, tensor self, tensor other) { + PROTECT( + auto outputs__ = torch::xlogy_out(*out, *self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_xlogy_scalar_other(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::xlogy(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_xlogy_scalar_other_(tensor *out__, tensor self, scalar other) { + PROTECT( + auto outputs__ = torch::xlogy_(*self, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_xlogy_scalar_self(tensor *out__, scalar self_scalar, tensor other) { + PROTECT( + auto outputs__ = torch::xlogy(*self_scalar, *other); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_zero(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::zero(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_zero_(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::zero_(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_zero_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::zero_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_zeros(tensor *out__, int64_t *size_data, int size_len, int options_kind, int options_device) { + PROTECT( + auto outputs__ = torch::zeros(torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_zeros_like(tensor *out__, tensor self) { + PROTECT( + auto outputs__ = torch::zeros_like(*self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_zeros_like_out(tensor *out__, tensor out, tensor self) { + PROTECT( + auto outputs__ = torch::zeros_like_out(*out, *self); + out__[0] = new torch::Tensor(outputs__); + ) +} + +void atg_zeros_out(tensor *out__, tensor out, int64_t *size_data, int size_len) { + PROTECT( + auto outputs__ = torch::zeros_out(*out, torch::IntArrayRef(size_data, size_len)); + out__[0] = new torch::Tensor(outputs__); + ) +} + diff --git a/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api_generated.h b/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api_generated.h new file mode 100644 index 0000000000..1c35bbdfe3 --- /dev/null +++ b/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api_generated.h @@ -0,0 +1,2648 @@ +// THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT BY HAND! +#include "torch_api.h" + +extern "C" { +void atg___and__(tensor *, tensor self, scalar other); +void atg___and__tensor_(tensor *, tensor self, tensor other); +void atg___iand__(tensor *, tensor self, scalar other); +void atg___iand__tensor_(tensor *, tensor self, tensor other); +void atg___ilshift__(tensor *, tensor self, scalar other); +void atg___ilshift__tensor_(tensor *, tensor self, tensor other); +void atg___ior__(tensor *, tensor self, scalar other); +void atg___ior__tensor_(tensor *, tensor self, tensor other); +void atg___irshift__(tensor *, tensor self, scalar other); +void atg___irshift__tensor_(tensor *, tensor self, tensor other); +void atg___ixor__(tensor *, tensor self, scalar other); +void atg___ixor__tensor_(tensor *, tensor self, tensor other); +void atg___lshift__(tensor *, tensor self, scalar other); +void atg___lshift__scalar_out_(tensor *, tensor out, tensor self, scalar other); +void atg___lshift__tensor_(tensor *, tensor self, tensor other); +void atg___lshift__tensor_out_(tensor *, tensor out, tensor self, tensor other); +void atg___or__(tensor *, tensor self, scalar other); +void atg___or__tensor_(tensor *, tensor self, tensor other); +void atg___rshift__(tensor *, tensor self, scalar other); +void atg___rshift__scalar_out_(tensor *, tensor out, tensor self, scalar other); +void atg___rshift__tensor_(tensor *, tensor self, tensor other); +void atg___rshift__tensor_out_(tensor *, tensor out, tensor self, tensor other); +void atg___xor__(tensor *, tensor self, scalar other); +void atg___xor__tensor_(tensor *, tensor self, tensor other); +void atg__adaptive_avg_pool2d(tensor *, tensor self, int64_t *output_size_data, int output_size_len); +void atg__adaptive_avg_pool2d_backward(tensor *, tensor grad_output, tensor self); +void atg__adaptive_avg_pool2d_backward_out(tensor *, tensor out, tensor grad_output, tensor self); +void atg__adaptive_avg_pool2d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len); +void atg__adaptive_avg_pool3d(tensor *, tensor self, int64_t *output_size_data, int output_size_len); +void atg__adaptive_avg_pool3d_backward(tensor *, tensor grad_output, tensor self); +void atg__adaptive_avg_pool3d_backward_out(tensor *, tensor out, tensor grad_output, tensor self); +void atg__adaptive_avg_pool3d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len); +void atg__add_batch_dim(tensor *, tensor self, int64_t batch_dim, int64_t level); +void atg__add_relu(tensor *, tensor self, tensor other); +void atg__add_relu_(tensor *, tensor self, tensor other); +void atg__add_relu_out(tensor *, tensor out, tensor self, tensor other); +void atg__add_relu_scalar(tensor *, tensor self, scalar other); +void atg__add_relu_scalar_(tensor *, tensor self, scalar other); +void atg__add_relu_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg__addmm_activation(tensor *, tensor self, tensor mat1, tensor mat2, int use_gelu); +void atg__addmm_activation_out(tensor *, tensor out, tensor self, tensor mat1, tensor mat2, int use_gelu); +void atg__aminmax(tensor *, tensor self); +void atg__aminmax_dim(tensor *, tensor self, int64_t dim, int keepdim); +void atg__aminmax_dim_out(tensor *, tensor out0, tensor out1, tensor self, int64_t dim, int keepdim); +void atg__aminmax_out(tensor *, tensor out0, tensor out1, tensor self); +void atg__amp_update_scale(tensor *, tensor self, tensor growth_tracker, tensor found_inf, double scale_growth_factor, double scale_backoff_factor, int64_t growth_interval); +void atg__amp_update_scale_(tensor *, tensor self, tensor growth_tracker, tensor found_inf, double scale_growth_factor, double scale_backoff_factor, int64_t growth_interval); +void atg__amp_update_scale_out(tensor *, tensor out, tensor self, tensor growth_tracker, tensor found_inf, double scale_growth_factor, double scale_backoff_factor, int64_t growth_interval); +void atg__assert_scalar(scalar self_scalar, char* assert_msg_ptr, int assert_msg_len); +void atg__assert_tensor_metadata(tensor a, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int dtype, int device, int8_t layout); +void atg__autocast_to_full_precision(tensor *, tensor self, int cuda_enabled, int cpu_enabled); +void atg__autocast_to_reduced_precision(tensor *, tensor self, int cuda_enabled, int cpu_enabled, int cuda_dtype, int cpu_dtype); +void atg__batch_norm_no_update(tensor *, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps); +void atg__batch_norm_no_update_out(tensor *, tensor out0, tensor out1, tensor out2, tensor out3, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps); +void atg__batch_norm_with_update(tensor *, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps); +void atg__batch_norm_with_update_functional(tensor *, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps); +void atg__batch_norm_with_update_out(tensor *, tensor out, tensor save_mean, tensor save_invstd, tensor reserve, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps); +void atg__cast_byte(tensor *, tensor self, int non_blocking); +void atg__cast_char(tensor *, tensor self, int non_blocking); +void atg__cast_double(tensor *, tensor self, int non_blocking); +void atg__cast_float(tensor *, tensor self, int non_blocking); +void atg__cast_half(tensor *, tensor self, int non_blocking); +void atg__cast_int(tensor *, tensor self, int non_blocking); +void atg__cast_long(tensor *, tensor self, int non_blocking); +void atg__cast_short(tensor *, tensor self, int non_blocking); +void atg__cdist_backward(tensor *, tensor grad, tensor x1, tensor x2, double p, tensor cdist); +void atg__cdist_backward_out(tensor *, tensor out, tensor grad, tensor x1, tensor x2, double p, tensor cdist); +void atg__cholesky_solve_helper(tensor *, tensor self, tensor A, int upper); +void atg__cholesky_solve_helper_out(tensor *, tensor out, tensor self, tensor A, int upper); +void atg__chunk_cat(tensor *, tensor *tensors_data, int tensors_len, int64_t dim, int64_t num_chunks); +void atg__chunk_cat_out(tensor *, tensor out, tensor *tensors_data, int tensors_len, int64_t dim, int64_t num_chunks); +void atg__coalesce(tensor *, tensor self); +void atg__coalesce_out(tensor *, tensor out, tensor self); +void atg__coalesced(tensor *, tensor self, int coalesced); +void atg__coalesced_(tensor *, tensor self, int coalesced); +void atg__coalesced_out(tensor *, tensor out, tensor self, int coalesced); +void atg__compute_linear_combination(tensor *, tensor input, tensor coefficients); +void atg__compute_linear_combination_out(tensor *, tensor out, tensor input, tensor coefficients); +void atg__conj(tensor *, tensor self); +void atg__conj_copy(tensor *, tensor self); +void atg__conj_copy_out(tensor *, tensor out, tensor self); +void atg__conj_physical(tensor *, tensor self); +void atg__conj_physical_out(tensor *, tensor out, tensor self); +void atg__conv_depthwise2d(tensor *, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len); +void atg__conv_depthwise2d_out(tensor *, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len); +void atg__convert_indices_from_coo_to_csr(tensor *, tensor self, int64_t size, int out_int32); +void atg__convert_indices_from_coo_to_csr_out(tensor *, tensor out, tensor self, int64_t size, int out_int32); +void atg__convert_indices_from_csr_to_coo(tensor *, tensor crow_indices, tensor col_indices, int out_int32, int transpose); +void atg__convert_indices_from_csr_to_coo_out(tensor *, tensor out, tensor crow_indices, tensor col_indices, int out_int32, int transpose); +void atg__convert_weight_to_int4pack(tensor *, tensor self, int64_t innerKTiles); +void atg__convert_weight_to_int4pack_for_cpu(tensor *, tensor self, int64_t innerKTiles); +void atg__convolution(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups, int benchmark, int deterministic, int cudnn_enabled, int allow_tf32); +void atg__convolution_deprecated(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups, int benchmark, int deterministic, int cudnn_enabled); +void atg__convolution_mode(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, char* padding_ptr, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); +void atg__convolution_out(tensor *, tensor out, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups, int benchmark, int deterministic, int cudnn_enabled, int allow_tf32); +void atg__copy_from(tensor *, tensor self, tensor dst, int non_blocking); +void atg__copy_from_and_resize(tensor *, tensor self, tensor dst); +void atg__copy_from_and_resize_out(tensor *, tensor out, tensor self, tensor dst); +void atg__copy_from_out(tensor *, tensor out, tensor self, tensor dst, int non_blocking); +void atg__cslt_compress(tensor *, tensor input); +void atg__cslt_sparse_mm(tensor *, tensor compressed_A, tensor dense_B, tensor bias, tensor alpha, int out_dtype, int transpose_result, int64_t alg_id, int64_t split_k, int64_t split_k_mode); +int64_t atg__cslt_sparse_mm_search(tensor compressed_A, tensor dense_B, tensor bias, tensor alpha, int out_dtype, int transpose_result); +void atg__ctc_loss(tensor *, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int zero_infinity); +void atg__ctc_loss_backward(tensor *, tensor grad, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, tensor neg_log_likelihood, tensor log_alpha, int64_t blank, int zero_infinity); +void atg__ctc_loss_backward_out(tensor *, tensor out, tensor grad, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, tensor neg_log_likelihood, tensor log_alpha, int64_t blank, int zero_infinity); +void atg__ctc_loss_backward_tensor(tensor *, tensor grad, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, tensor neg_log_likelihood, tensor log_alpha, int64_t blank, int zero_infinity); +void atg__ctc_loss_out(tensor *, tensor out0, tensor out1, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int zero_infinity); +void atg__ctc_loss_tensor(tensor *, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank, int zero_infinity); +void atg__ctc_loss_tensor_out(tensor *, tensor out0, tensor out1, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank, int zero_infinity); +void atg__cudnn_attention_backward(tensor *, tensor grad_out, tensor query, tensor key, tensor value, tensor out, tensor logsumexp, tensor philox_seed, tensor philox_offset, tensor attn_bias, tensor cum_seq_q, tensor cum_seq_k, int64_t max_q, int64_t max_k, double dropout_p, int is_causal, double scale_v, uint8_t scale_null); +void atg__cudnn_ctc_loss(tensor *, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int deterministic, int zero_infinity); +void atg__cudnn_ctc_loss_out(tensor *, tensor out0, tensor out1, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int deterministic, int zero_infinity); +void atg__cudnn_ctc_loss_tensor(tensor *, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank, int deterministic, int zero_infinity); +void atg__cudnn_init_dropout_state(tensor *, double dropout, int train, int64_t dropout_seed, int options_kind, int options_device); +void atg__cudnn_init_dropout_state_out(tensor *, tensor out, double dropout, int train, int64_t dropout_seed); +void atg__cudnn_rnn(tensor *, tensor input, tensor *weight_data, int weight_len, int64_t weight_stride0, tensor weight_buf, tensor hx, tensor cx, int64_t mode, int64_t hidden_size, int64_t proj_size, int64_t num_layers, int batch_first, double dropout, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, tensor dropout_state); +void atg__cudnn_rnn_flatten_weight(tensor *, tensor *weight_arr_data, int weight_arr_len, int64_t weight_stride0, int64_t input_size, int64_t mode, int64_t hidden_size, int64_t proj_size, int64_t num_layers, int batch_first, int bidirectional); +void atg__cudnn_rnn_flatten_weight_out(tensor *, tensor out, tensor *weight_arr_data, int weight_arr_len, int64_t weight_stride0, int64_t input_size, int64_t mode, int64_t hidden_size, int64_t proj_size, int64_t num_layers, int batch_first, int bidirectional); +void atg__cudnn_rnn_out(tensor *, tensor out0, tensor out1, tensor out2, tensor out3, tensor out4, tensor input, tensor *weight_data, int weight_len, int64_t weight_stride0, tensor weight_buf, tensor hx, tensor cx, int64_t mode, int64_t hidden_size, int64_t proj_size, int64_t num_layers, int batch_first, double dropout, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, tensor dropout_state); +int64_t atg__debug_has_internal_overlap(tensor self); +void atg__dim_arange(tensor *, tensor like, int64_t dim); +int64_t atg__dimi(tensor self); +int64_t atg__dimv(tensor self); +void atg__dirichlet_grad(tensor *, tensor x, tensor alpha, tensor total); +void atg__dirichlet_grad_out(tensor *, tensor out, tensor x, tensor alpha, tensor total); +void atg__dyn_quant_matmul_4bit(tensor *, tensor inp, tensor packed_weights, int64_t block_size, int64_t in_features, int64_t out_features); +void atg__dyn_quant_pack_4bit_weight(tensor *, tensor weights, tensor scales_zeros, tensor bias, int64_t block_size, int64_t in_features, int64_t out_features); +void atg__efficient_attention_backward(tensor *, tensor grad_out_, tensor query, tensor key, tensor value, tensor bias, tensor out, tensor cu_seqlens_q, tensor cu_seqlens_k, int64_t max_seqlen_q, int64_t max_seqlen_k, tensor logsumexp, double dropout_p, tensor philox_seed, tensor philox_offset, int64_t custom_mask_type, int bias_requires_grad, double scale_v, uint8_t scale_null, int64_t num_splits_key_v, uint8_t num_splits_key_null, int64_t window_size_v, uint8_t window_size_null, int shared_storage_dqdkdv); +void atg__efficientzerotensor(tensor *, int64_t *size_data, int size_len, int options_kind, int options_device); +void atg__efficientzerotensor_out(tensor *, tensor out, int64_t *size_data, int size_len); +void atg__embedding_bag(tensor *, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset, int64_t padding_idx); +void atg__embedding_bag_backward(tensor *, tensor grad, tensor indices, tensor offsets, tensor offset2bag, tensor bag_size, tensor maximum_indices, int64_t num_weights, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int64_t padding_idx); +void atg__embedding_bag_dense_backward(tensor *, tensor grad, tensor indices, tensor offset2bag, tensor bag_size, tensor maximum_indices, int64_t num_weights, int scale_grad_by_freq, int64_t mode, tensor per_sample_weights, int64_t padding_idx); +void atg__embedding_bag_dense_backward_out(tensor *, tensor out, tensor grad, tensor indices, tensor offset2bag, tensor bag_size, tensor maximum_indices, int64_t num_weights, int scale_grad_by_freq, int64_t mode, tensor per_sample_weights, int64_t padding_idx); +void atg__embedding_bag_forward_only(tensor *, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset, int64_t padding_idx); +void atg__embedding_bag_forward_only_out(tensor *, tensor out0, tensor out1, tensor out2, tensor out3, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset, int64_t padding_idx); +void atg__embedding_bag_out(tensor *, tensor out0, tensor out1, tensor out2, tensor out3, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset, int64_t padding_idx); +void atg__embedding_bag_per_sample_weights_backward(tensor *, tensor grad, tensor weight, tensor indices, tensor offsets, tensor offset2bag, int64_t mode, int64_t padding_idx); +void atg__embedding_bag_per_sample_weights_backward_out(tensor *, tensor out, tensor grad, tensor weight, tensor indices, tensor offsets, tensor offset2bag, int64_t mode, int64_t padding_idx); +void atg__embedding_bag_sparse_backward(tensor *, tensor grad, tensor indices, tensor offsets, tensor offset2bag, tensor bag_size, int64_t num_weights, int scale_grad_by_freq, int64_t mode, tensor per_sample_weights, int64_t padding_idx); +void atg__empty_affine_quantized(tensor *, int64_t *size_data, int size_len, int options_kind, int options_device, double scale, int64_t zero_point); +void atg__empty_affine_quantized_out(tensor *, tensor out, int64_t *size_data, int size_len, double scale, int64_t zero_point); +void atg__empty_per_channel_affine_quantized(tensor *, int64_t *size_data, int size_len, tensor scales, tensor zero_points, int64_t axis, int options_kind, int options_device); +void atg__empty_per_channel_affine_quantized_out(tensor *, tensor out, int64_t *size_data, int size_len, tensor scales, tensor zero_points, int64_t axis); +void atg__euclidean_dist(tensor *, tensor x1, tensor x2); +void atg__euclidean_dist_out(tensor *, tensor out, tensor x1, tensor x2); +void atg__fake_quantize_learnable_per_channel_affine(tensor *, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max, double grad_factor); +void atg__fake_quantize_learnable_per_channel_affine_backward(tensor *, tensor grad, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max, double grad_factor); +void atg__fake_quantize_learnable_per_channel_affine_out(tensor *, tensor out, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max, double grad_factor); +void atg__fake_quantize_learnable_per_tensor_affine(tensor *, tensor self, tensor scale, tensor zero_point, int64_t quant_min, int64_t quant_max, double grad_factor); +void atg__fake_quantize_learnable_per_tensor_affine_backward(tensor *, tensor grad, tensor self, tensor scale, tensor zero_point, int64_t quant_min, int64_t quant_max, double grad_factor); +void atg__fake_quantize_learnable_per_tensor_affine_out(tensor *, tensor out, tensor self, tensor scale, tensor zero_point, int64_t quant_min, int64_t quant_max, double grad_factor); +void atg__fake_quantize_per_tensor_affine_cachemask_tensor_qparams(tensor *, tensor self, tensor scale, tensor zero_point, tensor fake_quant_enabled, int64_t quant_min, int64_t quant_max); +void atg__fake_quantize_per_tensor_affine_cachemask_tensor_qparams_out(tensor *, tensor out0, tensor out1, tensor self, tensor scale, tensor zero_point, tensor fake_quant_enabled, int64_t quant_min, int64_t quant_max); +void atg__fft_c2c(tensor *, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int forward); +void atg__fft_c2c_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int forward); +void atg__fft_c2r(tensor *, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int64_t last_dim_size); +void atg__fft_c2r_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int64_t last_dim_size); +void atg__fft_r2c(tensor *, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int onesided); +void atg__fft_r2c_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int onesided); +void atg__fill_mem_eff_dropout_mask_(tensor *, tensor self, double dropout_p, int64_t seed, int64_t offset); +void atg__flash_attention_backward(tensor *, tensor grad_out, tensor query, tensor key, tensor value, tensor out, tensor logsumexp, tensor cum_seq_q, tensor cum_seq_k, int64_t max_q, int64_t max_k, double dropout_p, int is_causal, tensor rng_state, tensor unused, double scale_v, uint8_t scale_null, int64_t window_size_left_v, uint8_t window_size_left_null, int64_t window_size_right_v, uint8_t window_size_right_null); +void atg__foobar(tensor *, tensor self, int arg1, int arg2, int arg3); +void atg__foobar_out(tensor *, tensor out, tensor self, int arg1, int arg2, int arg3); +void atg__functional_assert_async(tensor *, tensor self, char* assert_msg_ptr, int assert_msg_len, tensor dep_token); +void atg__functional_assert_scalar(tensor *, scalar self_scalar, char* assert_msg_ptr, int assert_msg_len, tensor dep_token); +void atg__functional_sym_constrain_range(tensor *, scalar size, int64_t min_v, uint8_t min_null, int64_t max_v, uint8_t max_null, tensor dep_token); +void atg__functional_sym_constrain_range_for_size(tensor *, scalar size, int64_t min_v, uint8_t min_null, int64_t max_v, uint8_t max_null, tensor dep_token); +void atg__fused_dropout(tensor *, tensor self, double p); +void atg__fused_dropout_out(tensor *, tensor out0, tensor out1, tensor self, double p); +void atg__fused_moving_avg_obs_fq_helper(tensor *, tensor self, tensor observer_on, tensor fake_quant_on, tensor running_min, tensor running_max, tensor scale, tensor zero_point, double averaging_const, int64_t quant_min, int64_t quant_max, int64_t ch_axis, int per_row_fake_quant, int symmetric_quant); +void atg__fused_moving_avg_obs_fq_helper_functional(tensor *, tensor self, tensor observer_on, tensor fake_quant_on, tensor running_min, tensor running_max, tensor scale, tensor zero_point, double averaging_const, int64_t quant_min, int64_t quant_max, int64_t ch_axis, int per_row_fake_quant, int symmetric_quant); +void atg__fused_moving_avg_obs_fq_helper_out(tensor *, tensor out0, tensor out1, tensor self, tensor observer_on, tensor fake_quant_on, tensor running_min, tensor running_max, tensor scale, tensor zero_point, double averaging_const, int64_t quant_min, int64_t quant_max, int64_t ch_axis, int per_row_fake_quant, int symmetric_quant); +void atg__fused_rms_norm(tensor *, tensor input, int64_t *normalized_shape_data, int normalized_shape_len, tensor weight, double eps_v, uint8_t eps_null); +int64_t atg__fused_sdp_choice(tensor query, tensor key, tensor value, tensor attn_mask, double dropout_p, int is_causal, double scale_v, uint8_t scale_null, int enable_gqa); +void atg__fw_primal(tensor *, tensor self, int64_t level); +void atg__fw_primal_copy(tensor *, tensor self, int64_t level); +void atg__fw_primal_copy_out(tensor *, tensor out, tensor self, int64_t level); +void atg__gather_sparse_backward(tensor *, tensor self, int64_t dim, tensor index, tensor grad); +void atg__grid_sampler_2d_cpu_fallback(tensor *, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners); +void atg__grid_sampler_2d_cpu_fallback_backward(tensor *, tensor grad_output, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners); +void atg__grid_sampler_2d_cpu_fallback_out(tensor *, tensor out, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners); +void atg__grouped_mm(tensor *, tensor self, tensor mat2, tensor offs, tensor bias, int out_dtype); +int atg__has_compatible_shallow_copy_type(tensor self, tensor from); +int atg__has_same_storage_numel(tensor self, tensor other); +tensor *atg__histogramdd_bin_edges(tensor self, int64_t *bins_data, int bins_len, double *range_data, int range_len, tensor weight, int density); +void atg__histogramdd_bin_edges_out(tensor *out_data, int out_len, tensor self, int64_t *bins_data, int bins_len, double *range_data, int range_len, tensor weight, int density); +void atg__histogramdd_from_bin_cts(tensor *, tensor self, int64_t *bins_data, int bins_len, double *range_data, int range_len, tensor weight, int density); +void atg__histogramdd_from_bin_cts_out(tensor *, tensor out, tensor self, int64_t *bins_data, int bins_len, double *range_data, int range_len, tensor weight, int density); +void atg__histogramdd_from_bin_tensors(tensor *, tensor self, tensor *bins_data, int bins_len, tensor weight, int density); +void atg__histogramdd_from_bin_tensors_out(tensor *, tensor out, tensor self, tensor *bins_data, int bins_len, tensor weight, int density); +void atg__index_put_impl(tensor *, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate, int unsafe); +void atg__index_put_impl_(tensor *, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate, int unsafe); +void atg__index_put_impl_out(tensor *, tensor out, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate, int unsafe); +void atg__indices(tensor *, tensor self); +void atg__indices_copy(tensor *, tensor self); +void atg__indices_copy_out(tensor *, tensor out, tensor self); +void atg__int_mm(tensor *, tensor self, tensor mat2); +void atg__int_mm_out(tensor *, tensor out, tensor self, tensor mat2); +void atg__is_all_true(tensor *, tensor self); +void atg__is_any_true(tensor *, tensor self); +int atg__is_zerotensor(tensor self); +void atg__lazy_clone(tensor *, tensor self); +void atg__linalg_check_errors(tensor info, char* api_name_ptr, int api_name_len, int is_matrix); +void atg__linalg_det(tensor *, tensor A); +void atg__linalg_det_result(tensor *, tensor result, tensor LU, tensor pivots, tensor A); +void atg__linalg_eigh(tensor *, tensor A, char* UPLO_ptr, int UPLO_len, int compute_v); +void atg__linalg_eigh_eigenvalues(tensor *, tensor eigenvalues, tensor eigenvectors, tensor A, char* UPLO_ptr, int UPLO_len, int compute_v); +void atg__linalg_eigvals(tensor *, tensor self); +void atg__linalg_slogdet(tensor *, tensor A); +void atg__linalg_slogdet_sign(tensor *, tensor sign, tensor logabsdet, tensor LU, tensor pivots, tensor A); +void atg__linalg_solve_ex(tensor *, tensor A, tensor B, int left, int check_errors); +void atg__linalg_solve_ex_result(tensor *, tensor result, tensor LU, tensor pivots, tensor info, tensor A, tensor B, int left, int check_errors); +void atg__linalg_svd(tensor *, tensor A, int full_matrices, int compute_uv, char* driver_ptr, int driver_len); +void atg__linalg_svd_u(tensor *, tensor U, tensor S, tensor Vh, tensor A, int full_matrices, int compute_uv, char* driver_ptr, int driver_len); +void atg__log_softmax(tensor *, tensor self, int64_t dim, int half_to_float); +void atg__log_softmax_backward_data(tensor *, tensor grad_output, tensor output, int64_t dim, int input_dtype); +void atg__log_softmax_backward_data_out(tensor *, tensor out, tensor grad_output, tensor output, int64_t dim, int input_dtype); +void atg__log_softmax_out(tensor *, tensor out, tensor self, int64_t dim, int half_to_float); +void atg__logcumsumexp(tensor *, tensor self, int64_t dim); +void atg__logcumsumexp_out(tensor *, tensor out, tensor self, int64_t dim); +void atg__lstm_mps(tensor *, tensor input, tensor *hx_data, int hx_len, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first); +void atg__lstm_mps_out(tensor *, tensor out0, tensor out1, tensor out2, tensor out3, tensor out4, tensor out5, tensor input, tensor *hx_data, int hx_len, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first); +void atg__lu_with_info(tensor *, tensor self, int pivot, int check_errors); +void atg__make_dep_token(tensor *, int options_kind, int options_device); +void atg__make_dual(tensor *, tensor primal, tensor tangent, int64_t level); +void atg__make_dual_copy(tensor *, tensor primal, tensor tangent, int64_t level); +void atg__make_dual_copy_out(tensor *, tensor out, tensor primal, tensor tangent, int64_t level); +void atg__make_per_channel_quantized_tensor(tensor *, tensor self, tensor scale, tensor zero_point, int64_t axis); +void atg__make_per_channel_quantized_tensor_out(tensor *, tensor out, tensor self, tensor scale, tensor zero_point, int64_t axis); +void atg__make_per_tensor_quantized_tensor(tensor *, tensor self, double scale, int64_t zero_point); +void atg__make_per_tensor_quantized_tensor_out(tensor *, tensor out, tensor self, double scale, int64_t zero_point); +void atg__masked_scale(tensor *, tensor self, tensor mask, double scale); +void atg__masked_scale_out(tensor *, tensor out, tensor self, tensor mask, double scale); +void atg__masked_softmax(tensor *, tensor self, tensor mask, int64_t dim_v, uint8_t dim_null, int64_t mask_type_v, uint8_t mask_type_null); +void atg__masked_softmax_backward(tensor *, tensor grad_output, tensor output, tensor mask, int64_t dim_v, uint8_t dim_null); +void atg__masked_softmax_backward_out(tensor *, tensor out, tensor grad_output, tensor output, tensor mask, int64_t dim_v, uint8_t dim_null); +void atg__masked_softmax_out(tensor *, tensor out, tensor self, tensor mask, int64_t dim_v, uint8_t dim_null, int64_t mask_type_v, uint8_t mask_type_null); +void atg__mixed_dtypes_linear(tensor *, tensor input, tensor weight, tensor scale, tensor bias, char* activation_ptr, int activation_len); +void atg__mkldnn_reshape(tensor *, tensor self, int64_t *shape_data, int shape_len); +void atg__mkldnn_reshape_out(tensor *, tensor out, tensor self, int64_t *shape_data, int shape_len); +void atg__mkldnn_transpose(tensor *, tensor self, int64_t dim0, int64_t dim1); +void atg__mkldnn_transpose_(tensor *, tensor self, int64_t dim0, int64_t dim1); +void atg__mkldnn_transpose_out(tensor *, tensor out, tensor self, int64_t dim0, int64_t dim1); +void atg__mps_convolution(tensor *, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups); +void atg__mps_convolution_out(tensor *, tensor out, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups); +void atg__mps_convolution_transpose(tensor *, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups); +void atg__mps_convolution_transpose_out(tensor *, tensor out, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups); +void atg__native_batch_norm_legit(tensor *, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps); +void atg__native_batch_norm_legit_functional(tensor *, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps); +void atg__native_batch_norm_legit_no_stats(tensor *, tensor input, tensor weight, tensor bias, int training, double momentum, double eps); +void atg__native_batch_norm_legit_no_stats_out(tensor *, tensor out, tensor save_mean, tensor save_invstd, tensor input, tensor weight, tensor bias, int training, double momentum, double eps); +void atg__native_batch_norm_legit_no_training(tensor *, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps); +void atg__native_batch_norm_legit_no_training_out(tensor *, tensor out0, tensor out1, tensor out2, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps); +void atg__native_batch_norm_legit_out(tensor *, tensor out, tensor save_mean, tensor save_invstd, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps); +void atg__native_multi_head_attention(tensor *, tensor query, tensor key, tensor value, int64_t embed_dim, int64_t num_head, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, tensor mask, int need_weights, int average_attn_weights, int64_t mask_type_v, uint8_t mask_type_null); +void atg__native_multi_head_attention_out(tensor *, tensor out0, tensor out1, tensor query, tensor key, tensor value, int64_t embed_dim, int64_t num_head, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, tensor mask, int need_weights, int average_attn_weights, int64_t mask_type_v, uint8_t mask_type_null); +void atg__neg_view(tensor *, tensor self); +void atg__neg_view_copy(tensor *, tensor self); +void atg__neg_view_copy_out(tensor *, tensor out, tensor self); +void atg__nested_compute_contiguous_strides_offsets(tensor *, tensor nested_size); +void atg__nested_from_padded(tensor *, tensor padded, tensor cpu_nested_shape_example, int fuse_transform_0213); +void atg__nested_from_padded_and_nested_example(tensor *, tensor padded, tensor nt_example); +void atg__nested_from_padded_and_nested_example_out(tensor *, tensor out, tensor padded, tensor nt_example); +void atg__nested_from_padded_out(tensor *, tensor out, tensor padded, tensor cpu_nested_shape_example, int fuse_transform_0213); +void atg__nested_from_padded_tensor(tensor *, tensor padded, tensor offsets, tensor dummy, int64_t ragged_idx, tensor min_seqlen, tensor max_seqlen, int64_t sum_S_v, uint8_t sum_S_null); +void atg__nested_get_jagged_dummy(tensor *, tensor any); +void atg__nested_get_lengths(tensor *, tensor self); +void atg__nested_get_max_seqlen(tensor *, tensor self); +void atg__nested_get_min_seqlen(tensor *, tensor self); +void atg__nested_get_offsets(tensor *, tensor self); +int64_t atg__nested_get_ragged_idx(tensor self); +void atg__nested_get_values(tensor *, tensor self); +void atg__nested_get_values_copy(tensor *, tensor self); +void atg__nested_get_values_copy_out(tensor *, tensor out, tensor self); +void atg__nested_select_backward(tensor *, tensor grad_output, tensor self, int64_t dim, int64_t index); +void atg__nested_sum_backward(tensor *, tensor grad, tensor self, int64_t *dim_data, int dim_len, int keepdim); +void atg__nested_view_from_buffer(tensor *, tensor self, tensor nested_size, tensor nested_strides, tensor offsets); +void atg__nested_view_from_buffer_copy(tensor *, tensor self, tensor nested_size, tensor nested_strides, tensor offsets); +void atg__nested_view_from_buffer_copy_out(tensor *, tensor out, tensor self, tensor nested_size, tensor nested_strides, tensor offsets); +void atg__nested_view_from_jagged(tensor *, tensor self, tensor offsets, tensor dummy, tensor lengths, int64_t ragged_idx, tensor min_seqlen, tensor max_seqlen); +void atg__nested_view_from_jagged_copy(tensor *, tensor self, tensor offsets, tensor dummy, tensor lengths, int64_t ragged_idx, tensor min_seqlen, tensor max_seqlen); +void atg__nested_view_from_jagged_copy_out(tensor *, tensor out, tensor self, tensor offsets, tensor dummy, tensor lengths, int64_t ragged_idx, tensor min_seqlen, tensor max_seqlen); +void atg__new_zeros_with_same_feature_meta(tensor *, tensor self, tensor other, int64_t self_num_batch_dims); +void atg__new_zeros_with_same_feature_meta_out(tensor *, tensor out, tensor self, tensor other, int64_t self_num_batch_dims); +int atg__nnpack_available(); +void atg__nnpack_spatial_convolution(tensor *, tensor input, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len); +void atg__nnpack_spatial_convolution_out(tensor *, tensor out, tensor input, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len); +int64_t atg__nnz(tensor self); +void atg__pack_padded_sequence(tensor *, tensor input, tensor lengths, int batch_first); +void atg__pack_padded_sequence_backward(tensor *, tensor grad, int64_t *input_size_data, int input_size_len, tensor batch_sizes, int batch_first); +void atg__pack_padded_sequence_out(tensor *, tensor out0, tensor out1, tensor input, tensor lengths, int batch_first); +void atg__pad_circular(tensor *, tensor self, int64_t *pad_data, int pad_len); +void atg__pad_enum(tensor *, tensor self, int64_t *pad_data, int pad_len, int64_t mode, double value_v, uint8_t value_null); +void atg__pad_packed_sequence(tensor *, tensor data, tensor batch_sizes, int batch_first, scalar padding_value, int64_t total_length); +void atg__pdist_backward(tensor *, tensor grad, tensor self, double p, tensor pdist); +void atg__pdist_backward_out(tensor *, tensor out, tensor grad, tensor self, double p, tensor pdist); +void atg__pin_memory(tensor *, tensor self, int device); +void atg__pin_memory_out(tensor *, tensor out, tensor self, int device); +void atg__prelu_kernel(tensor *, tensor self, tensor weight); +void atg__prelu_kernel_backward(tensor *, tensor grad_output, tensor self, tensor weight); +void atg__print(char* s_ptr, int s_len); +void atg__propagate_xla_data(tensor input, tensor output); +void atg__remove_batch_dim(tensor *, tensor self, int64_t level, int64_t batch_size, int64_t out_dim); +void atg__reshape_alias(tensor *, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len); +void atg__reshape_alias_copy(tensor *, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len); +void atg__reshape_alias_copy_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len); +void atg__reshape_copy(tensor *, tensor self, int64_t *size_data, int size_len); +void atg__reshape_from_tensor(tensor *, tensor self, tensor shape); +void atg__resize_output(tensor *, tensor self, int64_t *size_data, int size_len, int device); +void atg__resize_output_(tensor *, tensor self, int64_t *size_data, int size_len, int device); +void atg__resize_output_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len, int device); +void atg__rowwise_prune(tensor *, tensor weight, tensor mask, int compressed_indices_dtype); +void atg__safe_softmax(tensor *, tensor self, int64_t dim, int dtype); +void atg__sample_dirichlet(tensor *, tensor self); +void atg__sample_dirichlet_out(tensor *, tensor out, tensor self); +void atg__saturate_weight_to_fp16(tensor *, tensor weight); +void atg__scaled_dot_product_attention_math(tensor *, tensor query, tensor key, tensor value, tensor attn_mask, double dropout_p, int is_causal, tensor dropout_mask, double scale_v, uint8_t scale_null, int enable_gqa); +void atg__scaled_dot_product_attention_math_for_mps(tensor *, tensor query, tensor key, tensor value, tensor attn_mask, double dropout_p, int is_causal, tensor dropout_mask, double scale_v, uint8_t scale_null); +void atg__scaled_dot_product_cudnn_attention_backward(tensor *, tensor grad_out, tensor query, tensor key, tensor value, tensor out, tensor logsumexp, tensor philox_seed, tensor philox_offset, tensor attn_bias, tensor cum_seq_q, tensor cum_seq_k, int64_t max_q, int64_t max_k, double dropout_p, int is_causal, double scale_v, uint8_t scale_null); +void atg__scaled_dot_product_efficient_attention(tensor *, tensor query, tensor key, tensor value, tensor attn_bias, int compute_log_sumexp, double dropout_p, int is_causal, double scale_v, uint8_t scale_null); +void atg__scaled_dot_product_flash_attention_backward(tensor *, tensor grad_out, tensor query, tensor key, tensor value, tensor out, tensor logsumexp, tensor cum_seq_q, tensor cum_seq_k, int64_t max_q, int64_t max_k, double dropout_p, int is_causal, tensor philox_seed, tensor philox_offset, double scale_v, uint8_t scale_null); +void atg__scaled_dot_product_flash_attention_for_cpu(tensor *, tensor query, tensor key, tensor value, double dropout_p, int is_causal, tensor attn_mask, double scale_v, uint8_t scale_null); +void atg__scaled_dot_product_flash_attention_for_cpu_backward(tensor *, tensor grad_out, tensor query, tensor key, tensor value, tensor out, tensor logsumexp, double dropout_p, int is_causal, tensor attn_mask, double scale_v, uint8_t scale_null); +void atg__scaled_grouped_mm(tensor *, tensor self, tensor mat2, tensor scale_a, tensor scale_b, tensor offs, tensor bias, tensor scale_result, int out_dtype, int use_fast_accum); +void atg__scaled_grouped_mm_v2(tensor *, tensor self, tensor mat2, tensor *scale_a_data, int scale_a_len, int64_t *recipe_a_data, int recipe_a_len, int64_t *swizzle_a_data, int swizzle_a_len, tensor *scale_b_data, int scale_b_len, int64_t *recipe_b_data, int recipe_b_len, int64_t *swizzle_b_data, int swizzle_b_len, tensor offs, tensor bias, int out_dtype, int64_t *contraction_dim_data, int contraction_dim_len, int use_fast_accum); +void atg__scaled_mm(tensor *, tensor self, tensor mat2, tensor scale_a, tensor scale_b, tensor bias, tensor scale_result, int out_dtype, int use_fast_accum); +void atg__scaled_mm_out(tensor *, tensor out, tensor self, tensor mat2, tensor scale_a, tensor scale_b, tensor bias, tensor scale_result, int out_dtype, int use_fast_accum); +void atg__scaled_mm_v2(tensor *, tensor self, tensor mat2, tensor *scale_a_data, int scale_a_len, int64_t *recipe_a_data, int recipe_a_len, int64_t *swizzle_a_data, int swizzle_a_len, tensor *scale_b_data, int scale_b_len, int64_t *recipe_b_data, int recipe_b_len, int64_t *swizzle_b_data, int swizzle_b_len, tensor bias, int out_dtype, int64_t *contraction_dim_data, int contraction_dim_len, int use_fast_accum); +void atg__scaled_mm_v2_out(tensor *, tensor out, tensor self, tensor mat2, tensor *scale_a_data, int scale_a_len, int64_t *recipe_a_data, int recipe_a_len, int64_t *swizzle_a_data, int swizzle_a_len, tensor *scale_b_data, int scale_b_len, int64_t *recipe_b_data, int recipe_b_len, int64_t *swizzle_b_data, int swizzle_b_len, tensor bias, int out_dtype, int64_t *contraction_dim_data, int contraction_dim_len, int use_fast_accum); +void atg__scatter_reduce(tensor *, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len, int include_self); +void atg__scatter_reduce_(tensor *, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len, int include_self); +void atg__scatter_reduce_two_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len, int include_self); +void atg__segment_reduce_backward(tensor *, tensor grad, tensor output, tensor data, char* reduce_ptr, int reduce_len, tensor lengths, tensor offsets, int64_t axis, scalar initial); +void atg__segment_reduce_backward_out(tensor *, tensor out, tensor grad, tensor output, tensor data, char* reduce_ptr, int reduce_len, tensor lengths, tensor offsets, int64_t axis, scalar initial); +void atg__shape_as_tensor(tensor *, tensor self); +void atg__slow_conv2d_backward(tensor *, tensor grad_input, tensor grad_weight, tensor grad_bias, tensor grad_output, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len); +void atg__sobol_engine_draw(tensor *, tensor quasi, int64_t n, tensor sobolstate, int64_t dimension, int64_t num_generated, int dtype); +void atg__sobol_engine_ff_(tensor *, tensor self, int64_t n, tensor sobolstate, int64_t dimension, int64_t num_generated); +void atg__sobol_engine_initialize_state_(tensor *, tensor self, int64_t dimension); +void atg__sobol_engine_scramble_(tensor *, tensor self, tensor ltm, int64_t dimension); +void atg__softmax(tensor *, tensor self, int64_t dim, int half_to_float); +void atg__softmax_backward_data(tensor *, tensor grad_output, tensor output, int64_t dim, int input_dtype); +void atg__softmax_backward_data_out(tensor *, tensor grad_input, tensor grad_output, tensor output, int64_t dim, int input_dtype); +void atg__softmax_out(tensor *, tensor out, tensor self, int64_t dim, int half_to_float); +void atg__sparse_addmm(tensor *, tensor self, tensor mat1, tensor mat2); +void atg__sparse_addmm_out(tensor *, tensor out, tensor self, tensor mat1, tensor mat2); +void atg__sparse_broadcast_to(tensor *, tensor self, int64_t *size_data, int size_len); +void atg__sparse_broadcast_to_copy(tensor *, tensor self, int64_t *size_data, int size_len); +void atg__sparse_broadcast_to_copy_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len); +void atg__sparse_bsc_tensor_unsafe(tensor *, tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device); +void atg__sparse_bsr_tensor_unsafe(tensor *, tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device); +void atg__sparse_compressed_tensor_unsafe(tensor *, tensor compressed_indices, tensor plain_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device); +void atg__sparse_compressed_tensor_with_dims(tensor *, int64_t nnz, int64_t dense_dim, int64_t *size_data, int size_len, int64_t *blocksize_data, int blocksize_len, int index_dtype, int options_kind, int options_device); +void atg__sparse_coo_tensor_unsafe(tensor *, tensor indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device, int is_coalesced); +void atg__sparse_coo_tensor_with_dims(tensor *, int64_t sparse_dim, int64_t dense_dim, int64_t *size_data, int size_len, int options_kind, int options_device); +void atg__sparse_coo_tensor_with_dims_and_tensors(tensor *, int64_t sparse_dim, int64_t dense_dim, int64_t *size_data, int size_len, tensor indices, tensor values, int options_kind, int options_device, int is_coalesced); +void atg__sparse_coo_tensor_with_dims_and_tensors_out(tensor *, tensor out, int64_t sparse_dim, int64_t dense_dim, int64_t *size_data, int size_len, tensor indices, tensor values, int is_coalesced); +void atg__sparse_coo_tensor_with_dims_out(tensor *, tensor out, int64_t sparse_dim, int64_t dense_dim, int64_t *size_data, int size_len); +void atg__sparse_csc_tensor_unsafe(tensor *, tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device); +void atg__sparse_csr_prod(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); +void atg__sparse_csr_prod_dim_dtype_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); +void atg__sparse_csr_sum(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); +void atg__sparse_csr_sum_dim_dtype_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); +void atg__sparse_csr_tensor_unsafe(tensor *, tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device); +void atg__sparse_log_softmax(tensor *, tensor self, int64_t dim, int half_to_float); +void atg__sparse_log_softmax_backward_data(tensor *, tensor grad_output, tensor output, int64_t dim, tensor self); +void atg__sparse_log_softmax_backward_data_out(tensor *, tensor out, tensor grad_output, tensor output, int64_t dim, tensor self); +void atg__sparse_log_softmax_int(tensor *, tensor self, int64_t dim, int dtype); +void atg__sparse_log_softmax_out(tensor *, tensor out, tensor self, int64_t dim, int half_to_float); +void atg__sparse_mask_projection(tensor *, tensor self, tensor mask, int accumulate_matches); +void atg__sparse_mask_projection_out(tensor *, tensor out, tensor self, tensor mask, int accumulate_matches); +void atg__sparse_mm(tensor *, tensor sparse, tensor dense); +void atg__sparse_mm_reduce(tensor *, tensor sparse, tensor dense, char* reduce_ptr, int reduce_len); +void atg__sparse_mm_reduce_impl(tensor *, tensor self, tensor other, char* reduce_ptr, int reduce_len); +void atg__sparse_semi_structured_apply(tensor *, tensor input, tensor thread_masks); +void atg__sparse_semi_structured_apply_dense(tensor *, tensor input, tensor thread_masks); +void atg__sparse_semi_structured_linear(tensor *, tensor input, tensor weight, tensor meta, tensor bias, char* activation_ptr, int activation_len, int out_dtype); +void atg__sparse_semi_structured_mm(tensor *, tensor mat1, tensor mat1_meta, tensor mat2, int out_dtype); +void atg__sparse_semi_structured_tile(tensor *, tensor input, char* algorithm_ptr, int algorithm_len, int use_cutlass); +void atg__sparse_softmax(tensor *, tensor self, int64_t dim, int half_to_float); +void atg__sparse_softmax_backward_data(tensor *, tensor grad_output, tensor output, int64_t dim, tensor self); +void atg__sparse_softmax_backward_data_out(tensor *, tensor out, tensor grad_output, tensor output, int64_t dim, tensor self); +void atg__sparse_softmax_int(tensor *, tensor self, int64_t dim, int dtype); +void atg__sparse_softmax_out(tensor *, tensor out, tensor self, int64_t dim, int half_to_float); +void atg__sparse_sparse_matmul(tensor *, tensor self, tensor other); +void atg__sparse_sparse_matmul_out(tensor *, tensor out, tensor self, tensor other); +void atg__sparse_sum(tensor *, tensor self); +void atg__sparse_sum_backward(tensor *, tensor grad, tensor self, int64_t *dim_data, int dim_len); +void atg__sparse_sum_backward_out(tensor *, tensor out, tensor grad, tensor self, int64_t *dim_data, int dim_len); +void atg__sparse_sum_dim(tensor *, tensor self, int64_t *dim_data, int dim_len); +void atg__sparse_sum_dim_dtype(tensor *, tensor self, int64_t *dim_data, int dim_len, int dtype); +void atg__sparse_sum_dim_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len); +void atg__sparse_sum_dtype(tensor *, tensor self, int dtype); +void atg__spdiags(tensor *, tensor diagonals, tensor offsets, int64_t *shape_data, int shape_len, int8_t layout); +void atg__spdiags_out(tensor *, tensor out, tensor diagonals, tensor offsets, int64_t *shape_data, int shape_len, int8_t layout); +void atg__spsolve(tensor *, tensor A, tensor B, int left); +void atg__stack(tensor *, tensor *tensors_data, int tensors_len, int64_t dim); +void atg__stack_out(tensor *, tensor out, tensor *tensors_data, int tensors_len, int64_t dim); +void atg__standard_gamma(tensor *, tensor self); +void atg__standard_gamma_grad(tensor *, tensor self, tensor output); +void atg__standard_gamma_grad_out(tensor *, tensor out, tensor self, tensor output); +void atg__standard_gamma_out(tensor *, tensor out, tensor self); +void atg__test_ambiguous_defaults(tensor *, tensor dummy, int64_t a, int64_t b); +void atg__test_ambiguous_defaults_b(tensor *, tensor dummy, int64_t a, char* b_ptr, int b_len); +void atg__test_autograd_multiple_dispatch(tensor *, tensor self); +void atg__test_autograd_multiple_dispatch_fullcoverage_out(tensor *, tensor out, tensor self); +void atg__test_autograd_multiple_dispatch_ntonly(tensor *, tensor self, int b); +void atg__test_autograd_multiple_dispatch_view(tensor *, tensor self); +void atg__test_autograd_multiple_dispatch_view_copy(tensor *, tensor self); +void atg__test_autograd_multiple_dispatch_view_copy_out(tensor *, tensor out, tensor self); +void atg__test_check_tensor(tensor *, tensor self); +void atg__test_functorch_fallback(tensor *, tensor self, tensor other); +void atg__test_functorch_fallback_out(tensor *, tensor out, tensor self, tensor other); +void atg__test_optional_filled_intlist(tensor *, tensor values, int64_t *addends_data, int addends_len); +void atg__test_optional_filled_intlist_out(tensor *, tensor out, tensor values, int64_t *addends_data, int addends_len); +void atg__test_optional_floatlist(tensor *, tensor values, double *addends_data, int addends_len); +void atg__test_optional_floatlist_out(tensor *, tensor out, tensor values, double *addends_data, int addends_len); +void atg__test_optional_intlist(tensor *, tensor values, int64_t *addends_data, int addends_len); +void atg__test_optional_intlist_out(tensor *, tensor out, tensor values, int64_t *addends_data, int addends_len); +void atg__test_parallel_materialize(tensor *, tensor self, int64_t num_parallel, int skip_first); +void atg__test_serialization_subcmul(tensor *, tensor self, tensor other); +void atg__test_string_default(tensor *, tensor dummy, char* a_ptr, int a_len, char* b_ptr, int b_len); +void atg__test_warn_in_autograd(tensor *, tensor self); +void atg__test_warn_in_autograd_out(tensor *, tensor out, tensor self); +void atg__to_copy(tensor *, tensor self, int options_kind, int options_device, int non_blocking); +void atg__to_copy_out(tensor *, tensor out, tensor self, int non_blocking); +tensor *atg__to_cpu(tensor *tensors_data, int tensors_len); +void atg__to_dense(tensor *, tensor self, int dtype, int masked_grad); +void atg__to_dense_out(tensor *, tensor out, tensor self, int dtype, int masked_grad); +void atg__to_sparse(tensor *, tensor self, int8_t layout, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null); +void atg__to_sparse_bsc(tensor *, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null); +void atg__to_sparse_bsc_out(tensor *, tensor out, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null); +void atg__to_sparse_bsr(tensor *, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null); +void atg__to_sparse_bsr_out(tensor *, tensor out, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null); +void atg__to_sparse_csc(tensor *, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null); +void atg__to_sparse_csc_out(tensor *, tensor out, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null); +void atg__to_sparse_csr(tensor *, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null); +void atg__to_sparse_csr_out(tensor *, tensor out, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null); +void atg__to_sparse_out(tensor *, tensor out, tensor self, int8_t layout, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null); +void atg__to_sparse_semi_structured(tensor *, tensor dense); +void atg__to_sparse_sparse_dim(tensor *, tensor self, int64_t sparse_dim); +void atg__to_sparse_sparse_dim_out(tensor *, tensor out, tensor self, int64_t sparse_dim); +void atg__transform_bias_rescale_qkv(tensor *, tensor qkv, tensor qkv_bias, int64_t num_heads); +void atg__transform_bias_rescale_qkv_out(tensor *, tensor out0, tensor out1, tensor out2, tensor qkv, tensor qkv_bias, int64_t num_heads); +void atg__transformer_encoder_layer_fwd(tensor *, tensor src, int64_t embed_dim, int64_t num_heads, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, int use_gelu, int norm_first, double eps, tensor norm_weight_1, tensor norm_bias_1, tensor norm_weight_2, tensor norm_bias_2, tensor ffn_weight_1, tensor ffn_bias_1, tensor ffn_weight_2, tensor ffn_bias_2, tensor mask, int64_t mask_type_v, uint8_t mask_type_null); +void atg__transformer_encoder_layer_fwd_out(tensor *, tensor out, tensor src, int64_t embed_dim, int64_t num_heads, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, int use_gelu, int norm_first, double eps, tensor norm_weight_1, tensor norm_bias_1, tensor norm_weight_2, tensor norm_bias_2, tensor ffn_weight_1, tensor ffn_bias_1, tensor ffn_weight_2, tensor ffn_bias_2, tensor mask, int64_t mask_type_v, uint8_t mask_type_null); +void atg__trilinear(tensor *, tensor i1, tensor i2, tensor i3, int64_t *expand1_data, int expand1_len, int64_t *expand2_data, int expand2_len, int64_t *expand3_data, int expand3_len, int64_t *sumdim_data, int sumdim_len, int64_t unroll_dim); +void atg__trilinear_out(tensor *, tensor out, tensor i1, tensor i2, tensor i3, int64_t *expand1_data, int expand1_len, int64_t *expand2_data, int expand2_len, int64_t *expand3_data, int expand3_len, int64_t *sumdim_data, int sumdim_len, int64_t unroll_dim); +void atg__triton_multi_head_attention(tensor *, tensor query, tensor key, tensor value, int64_t embed_dim, int64_t num_head, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, tensor mask); +void atg__triton_multi_head_attention_out(tensor *, tensor out, tensor query, tensor key, tensor value, int64_t embed_dim, int64_t num_head, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, tensor mask); +void atg__triton_scaled_dot_attention(tensor *, tensor q, tensor k, tensor v, double dropout_p); +void atg__triton_scaled_dot_attention_out(tensor *, tensor out, tensor q, tensor k, tensor v, double dropout_p); +void atg__unique(tensor *, tensor self, int sorted, int return_inverse); +void atg__unique2(tensor *, tensor self, int sorted, int return_inverse, int return_counts); +void atg__unique2_out(tensor *, tensor out0, tensor out1, tensor out2, tensor self, int sorted, int return_inverse, int return_counts); +void atg__unique_out(tensor *, tensor out0, tensor out1, tensor self, int sorted, int return_inverse); +void atg__unpack_dual(tensor *, tensor dual, int64_t level); +void atg__unsafe_index(tensor *, tensor self, tensor *indices_data, int indices_len); +void atg__unsafe_index_put(tensor *, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate); +void atg__unsafe_masked_index(tensor *, tensor self, tensor mask, tensor *indices_data, int indices_len, scalar fill); +void atg__unsafe_masked_index_put_accumulate(tensor *, tensor self, tensor mask, tensor *indices_data, int indices_len, tensor values); +void atg__unsafe_view(tensor *, tensor self, int64_t *size_data, int size_len); +void atg__unsafe_view_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len); +void atg__upsample_bicubic2d_aa(tensor *, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg__upsample_bicubic2d_aa_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg__upsample_bicubic2d_aa_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg__upsample_bicubic2d_aa_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg__upsample_bicubic2d_aa_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len); +void atg__upsample_bilinear2d_aa(tensor *, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg__upsample_bilinear2d_aa_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg__upsample_bilinear2d_aa_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg__upsample_bilinear2d_aa_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg__upsample_bilinear2d_aa_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len); +void atg__upsample_nearest_exact1d(tensor *, tensor self, int64_t *output_size_data, int output_size_len, double scales_v, uint8_t scales_null); +void atg__upsample_nearest_exact1d_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_v, uint8_t scales_null); +void atg__upsample_nearest_exact1d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_v, uint8_t scales_null); +void atg__upsample_nearest_exact1d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_v, uint8_t scales_null); +void atg__upsample_nearest_exact1d_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len); +void atg__upsample_nearest_exact2d(tensor *, tensor self, int64_t *output_size_data, int output_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg__upsample_nearest_exact2d_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg__upsample_nearest_exact2d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg__upsample_nearest_exact2d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg__upsample_nearest_exact2d_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len); +void atg__upsample_nearest_exact3d(tensor *, tensor self, int64_t *output_size_data, int output_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg__upsample_nearest_exact3d_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg__upsample_nearest_exact3d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg__upsample_nearest_exact3d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg__upsample_nearest_exact3d_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len); +int atg__use_cudnn_ctc_loss(tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank); +int atg__use_cudnn_ctc_loss_tensor(tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank); +int atg__use_cudnn_rnn_flatten_weight(); +int atg__use_miopen_ctc_loss(tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank); +int atg__use_miopen_ctc_loss_tensor(tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank); +void atg__validate_compressed_sparse_indices(int is_crow, tensor compressed_idx, tensor plain_idx, int64_t cdim, int64_t dim, int64_t nnz); +void atg__validate_sparse_bsc_tensor_args(tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int check_pinning); +void atg__validate_sparse_bsr_tensor_args(tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int check_pinning); +void atg__validate_sparse_compressed_tensor_args(tensor compressed_indices, tensor plain_indices, tensor values, int64_t *size_data, int size_len, int8_t layout, int check_pinning); +void atg__validate_sparse_csc_tensor_args(tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int check_pinning); +void atg__validate_sparse_csr_tensor_args(tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int check_pinning); +void atg__values(tensor *, tensor self); +void atg__values_copy(tensor *, tensor self); +void atg__values_copy_out(tensor *, tensor out, tensor self); +int64_t atg__version(tensor self); +void atg__weight_int4pack_mm(tensor *, tensor self, tensor mat2, int64_t qGroupSize, tensor qScaleAndZeros); +void atg__weight_int4pack_mm_for_cpu(tensor *, tensor self, tensor mat2, int64_t qGroupSize, tensor qScaleAndZeros); +void atg__weight_int4pack_mm_with_scales_and_zeros(tensor *, tensor self, tensor mat2, int64_t qGroupSize, tensor qScale, tensor qZeros); +void atg__weight_int8pack_mm(tensor *, tensor self, tensor mat2, tensor scales); +void atg__weight_norm(tensor *, tensor v, tensor g, int64_t dim); +void atg__weight_norm_differentiable_backward(tensor *, tensor grad_w, tensor saved_v, tensor saved_g, tensor saved_norms, int64_t dim); +void atg__weight_norm_interface(tensor *, tensor v, tensor g, int64_t dim); +void atg__weight_norm_interface_backward(tensor *, tensor grad_w, tensor saved_v, tensor saved_g, tensor saved_norms, int64_t dim); +void atg__weight_norm_interface_backward_out(tensor *, tensor out0, tensor out1, tensor grad_w, tensor saved_v, tensor saved_g, tensor saved_norms, int64_t dim); +void atg__weight_norm_interface_out(tensor *, tensor out0, tensor out1, tensor v, tensor g, int64_t dim); +void atg__wrapped_linear_prepack(tensor *, tensor weight, tensor weight_scale, tensor weight_zero_point, tensor bias); +void atg__wrapped_quantized_linear_prepacked(tensor *, tensor input, tensor input_scale, tensor input_zero_point, tensor packed_weight, tensor output_scale, tensor output_zero_point, int64_t out_channel); +void atg_abs(tensor *, tensor self); +void atg_abs_(tensor *, tensor self); +void atg_abs_out(tensor *, tensor out, tensor self); +void atg_absolute(tensor *, tensor self); +void atg_absolute_(tensor *, tensor self); +void atg_absolute_out(tensor *, tensor out, tensor self); +void atg_acos(tensor *, tensor self); +void atg_acos_(tensor *, tensor self); +void atg_acos_out(tensor *, tensor out, tensor self); +void atg_acosh(tensor *, tensor self); +void atg_acosh_(tensor *, tensor self); +void atg_acosh_out(tensor *, tensor out, tensor self); +void atg_adaptive_avg_pool1d(tensor *, tensor self, int64_t *output_size_data, int output_size_len); +void atg_adaptive_avg_pool1d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len); +void atg_adaptive_avg_pool2d(tensor *, tensor self, int64_t *output_size_data, int output_size_len); +void atg_adaptive_avg_pool2d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len); +void atg_adaptive_avg_pool3d(tensor *, tensor self, int64_t *output_size_data, int output_size_len); +void atg_adaptive_avg_pool3d_backward(tensor *, tensor grad_input, tensor grad_output, tensor self); +void atg_adaptive_avg_pool3d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len); +void atg_adaptive_max_pool1d(tensor *, tensor self, int64_t *output_size_data, int output_size_len); +void atg_adaptive_max_pool2d(tensor *, tensor self, int64_t *output_size_data, int output_size_len); +void atg_adaptive_max_pool2d_backward(tensor *, tensor grad_output, tensor self, tensor indices); +void atg_adaptive_max_pool2d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor indices); +void atg_adaptive_max_pool2d_out(tensor *, tensor out, tensor indices, tensor self, int64_t *output_size_data, int output_size_len); +void atg_adaptive_max_pool3d(tensor *, tensor self, int64_t *output_size_data, int output_size_len); +void atg_adaptive_max_pool3d_backward(tensor *, tensor grad_output, tensor self, tensor indices); +void atg_adaptive_max_pool3d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor indices); +void atg_adaptive_max_pool3d_out(tensor *, tensor out, tensor indices, tensor self, int64_t *output_size_data, int output_size_len); +void atg_add(tensor *, tensor self, tensor other); +void atg_add_(tensor *, tensor self, tensor other); +void atg_add_out(tensor *, tensor out, tensor self, tensor other); +void atg_add_scalar(tensor *, tensor self, scalar other); +void atg_add_scalar_(tensor *, tensor self, scalar other); +void atg_add_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_addbmm(tensor *, tensor self, tensor batch1, tensor batch2); +void atg_addbmm_(tensor *, tensor self, tensor batch1, tensor batch2); +void atg_addbmm_out(tensor *, tensor out, tensor self, tensor batch1, tensor batch2); +void atg_addcdiv(tensor *, tensor self, tensor tensor1, tensor tensor2); +void atg_addcdiv_(tensor *, tensor self, tensor tensor1, tensor tensor2); +void atg_addcdiv_out(tensor *, tensor out, tensor self, tensor tensor1, tensor tensor2); +void atg_addcmul(tensor *, tensor self, tensor tensor1, tensor tensor2); +void atg_addcmul_(tensor *, tensor self, tensor tensor1, tensor tensor2); +void atg_addcmul_out(tensor *, tensor out, tensor self, tensor tensor1, tensor tensor2); +void atg_addmm(tensor *, tensor self, tensor mat1, tensor mat2); +void atg_addmm_(tensor *, tensor self, tensor mat1, tensor mat2); +void atg_addmm_dtype(tensor *, tensor self, tensor mat1, tensor mat2, int out_dtype); +void atg_addmm_dtype_out(tensor *, tensor out, tensor self, tensor mat1, tensor mat2, int out_dtype); +void atg_addmm_out(tensor *, tensor out, tensor self, tensor mat1, tensor mat2); +void atg_addmv(tensor *, tensor self, tensor mat, tensor vec); +void atg_addmv_(tensor *, tensor self, tensor mat, tensor vec); +void atg_addmv_out(tensor *, tensor out, tensor self, tensor mat, tensor vec); +void atg_addr(tensor *, tensor self, tensor vec1, tensor vec2); +void atg_addr_(tensor *, tensor self, tensor vec1, tensor vec2); +void atg_addr_out(tensor *, tensor out, tensor self, tensor vec1, tensor vec2); +void atg_adjoint(tensor *, tensor self); +void atg_affine_grid_generator(tensor *, tensor theta, int64_t *size_data, int size_len, int align_corners); +void atg_affine_grid_generator_backward(tensor *, tensor grad, int64_t *size_data, int size_len, int align_corners); +void atg_affine_grid_generator_out(tensor *, tensor out, tensor theta, int64_t *size_data, int size_len, int align_corners); +void atg_alias(tensor *, tensor self); +void atg_alias_copy(tensor *, tensor self); +void atg_alias_copy_out(tensor *, tensor out, tensor self); +void atg_align_as(tensor *, tensor self, tensor other); +tensor *atg_align_tensors(tensor *tensors_data, int tensors_len); +void atg_all(tensor *, tensor self); +void atg_all_all_out(tensor *, tensor out, tensor self); +void atg_all_dim(tensor *, tensor self, int64_t dim, int keepdim); +void atg_all_dims(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim); +void atg_all_dims_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim); +void atg_all_out(tensor *, tensor out, tensor self, int64_t dim, int keepdim); +int atg_allclose(tensor self, tensor other, double rtol, double atol, int equal_nan); +void atg_alpha_dropout(tensor *, tensor input, double p, int train); +void atg_alpha_dropout_(tensor *, tensor self, double p, int train); +void atg_amax(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim); +void atg_amax_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim); +void atg_amin(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim); +void atg_amin_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim); +void atg_aminmax(tensor *, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim); +void atg_aminmax_out(tensor *, tensor min, tensor max, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim); +void atg_angle(tensor *, tensor self); +void atg_angle_out(tensor *, tensor out, tensor self); +void atg_any(tensor *, tensor self); +void atg_any_all_out(tensor *, tensor out, tensor self); +void atg_any_dim(tensor *, tensor self, int64_t dim, int keepdim); +void atg_any_dims(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim); +void atg_any_dims_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim); +void atg_any_out(tensor *, tensor out, tensor self, int64_t dim, int keepdim); +void atg_arange(tensor *, scalar end, int options_kind, int options_device); +void atg_arange_start(tensor *, scalar start, scalar end, int options_kind, int options_device); +void atg_arange_start_step(tensor *, scalar start, scalar end, scalar step, int options_kind, int options_device); +void atg_arccos(tensor *, tensor self); +void atg_arccos_(tensor *, tensor self); +void atg_arccos_out(tensor *, tensor out, tensor self); +void atg_arccosh(tensor *, tensor self); +void atg_arccosh_(tensor *, tensor self); +void atg_arccosh_out(tensor *, tensor out, tensor self); +void atg_arcsin(tensor *, tensor self); +void atg_arcsin_(tensor *, tensor self); +void atg_arcsin_out(tensor *, tensor out, tensor self); +void atg_arcsinh(tensor *, tensor self); +void atg_arcsinh_(tensor *, tensor self); +void atg_arcsinh_out(tensor *, tensor out, tensor self); +void atg_arctan(tensor *, tensor self); +void atg_arctan2(tensor *, tensor self, tensor other); +void atg_arctan2_(tensor *, tensor self, tensor other); +void atg_arctan2_out(tensor *, tensor out, tensor self, tensor other); +void atg_arctan_(tensor *, tensor self); +void atg_arctan_out(tensor *, tensor out, tensor self); +void atg_arctanh(tensor *, tensor self); +void atg_arctanh_(tensor *, tensor self); +void atg_arctanh_out(tensor *, tensor out, tensor self); +void atg_argmax(tensor *, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim); +void atg_argmax_out(tensor *, tensor out, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim); +void atg_argmin(tensor *, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim); +void atg_argmin_out(tensor *, tensor out, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim); +void atg_argsort(tensor *, tensor self, int64_t dim, int descending); +void atg_argsort_stable(tensor *, tensor self, int stable, int64_t dim, int descending); +void atg_argsort_stable_out(tensor *, tensor out, tensor self, int stable, int64_t dim, int descending); +void atg_argwhere(tensor *, tensor self); +void atg_as_strided(tensor *, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null); +void atg_as_strided_(tensor *, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null); +void atg_as_strided_copy(tensor *, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null); +void atg_as_strided_copy_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null); +void atg_as_strided_scatter(tensor *, tensor self, tensor src, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null); +void atg_as_strided_scatter_out(tensor *, tensor out, tensor self, tensor src, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null); +void atg_asin(tensor *, tensor self); +void atg_asin_(tensor *, tensor self); +void atg_asin_out(tensor *, tensor out, tensor self); +void atg_asinh(tensor *, tensor self); +void atg_asinh_(tensor *, tensor self); +void atg_asinh_out(tensor *, tensor out, tensor self); +void atg_atan(tensor *, tensor self); +void atg_atan2(tensor *, tensor self, tensor other); +void atg_atan2_(tensor *, tensor self, tensor other); +void atg_atan2_out(tensor *, tensor out, tensor self, tensor other); +void atg_atan_(tensor *, tensor self); +void atg_atan_out(tensor *, tensor out, tensor self); +void atg_atanh(tensor *, tensor self); +void atg_atanh_(tensor *, tensor self); +void atg_atanh_out(tensor *, tensor out, tensor self); +void atg_atleast_1d(tensor *, tensor self); +tensor *atg_atleast_1d_sequence(tensor *tensors_data, int tensors_len); +void atg_atleast_2d(tensor *, tensor self); +tensor *atg_atleast_2d_sequence(tensor *tensors_data, int tensors_len); +void atg_atleast_3d(tensor *, tensor self); +tensor *atg_atleast_3d_sequence(tensor *tensors_data, int tensors_len); +void atg_avg_pool1d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad); +void atg_avg_pool1d_out(tensor *, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad); +void atg_avg_pool2d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null); +void atg_avg_pool2d_backward(tensor *, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null); +void atg_avg_pool2d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null); +void atg_avg_pool2d_out(tensor *, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null); +void atg_avg_pool3d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null); +void atg_avg_pool3d_backward(tensor *, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null); +void atg_avg_pool3d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null); +void atg_avg_pool3d_out(tensor *, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null); +void atg_baddbmm(tensor *, tensor self, tensor batch1, tensor batch2, scalar beta, scalar alpha); +void atg_baddbmm_(tensor *, tensor self, tensor batch1, tensor batch2); +void atg_baddbmm_dtype(tensor *, tensor self, tensor batch1, tensor batch2, int out_dtype, scalar beta, scalar alpha); +void atg_baddbmm_dtype_out(tensor *, tensor out, tensor self, tensor batch1, tensor batch2, int out_dtype); +void atg_baddbmm_out(tensor *, tensor out, tensor self, tensor batch1, tensor batch2); +void atg_bartlett_window(tensor *, int64_t window_length, int options_kind, int options_device); +void atg_bartlett_window_out(tensor *, tensor out, int64_t window_length); +void atg_bartlett_window_periodic(tensor *, int64_t window_length, int periodic, int options_kind, int options_device); +void atg_bartlett_window_periodic_out(tensor *, tensor out, int64_t window_length, int periodic); +void atg_batch_norm(tensor *, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps, int cudnn_enabled); +void atg_batch_norm_backward_elemt(tensor *, tensor grad_out, tensor input, tensor mean, tensor invstd, tensor weight, tensor sum_dy, tensor sum_dy_xmu, tensor count); +void atg_batch_norm_backward_elemt_out(tensor *, tensor out, tensor grad_out, tensor input, tensor mean, tensor invstd, tensor weight, tensor sum_dy, tensor sum_dy_xmu, tensor count); +void atg_batch_norm_backward_reduce(tensor *, tensor grad_out, tensor input, tensor mean, tensor invstd, tensor weight, int input_g, int weight_g, int bias_g); +void atg_batch_norm_backward_reduce_out(tensor *, tensor out0, tensor out1, tensor out2, tensor out3, tensor grad_out, tensor input, tensor mean, tensor invstd, tensor weight, int input_g, int weight_g, int bias_g); +void atg_batch_norm_elemt(tensor *, tensor input, tensor weight, tensor bias, tensor mean, tensor invstd, double eps); +void atg_batch_norm_elemt_out(tensor *, tensor out, tensor input, tensor weight, tensor bias, tensor mean, tensor invstd, double eps); +void atg_batch_norm_gather_stats(tensor *, tensor input, tensor mean, tensor invstd, tensor running_mean, tensor running_var, double momentum, double eps, int64_t count); +void atg_batch_norm_gather_stats_out(tensor *, tensor out0, tensor out1, tensor input, tensor mean, tensor invstd, tensor running_mean, tensor running_var, double momentum, double eps, int64_t count); +void atg_batch_norm_gather_stats_with_counts(tensor *, tensor input, tensor mean, tensor invstd, tensor running_mean, tensor running_var, double momentum, double eps, tensor counts); +void atg_batch_norm_gather_stats_with_counts_out(tensor *, tensor out0, tensor out1, tensor input, tensor mean, tensor invstd, tensor running_mean, tensor running_var, double momentum, double eps, tensor counts); +void atg_batch_norm_stats(tensor *, tensor input, double eps); +void atg_batch_norm_stats_out(tensor *, tensor out0, tensor out1, tensor input, double eps); +void atg_batch_norm_update_stats(tensor *, tensor input, tensor running_mean, tensor running_var, double momentum); +void atg_batch_norm_update_stats_out(tensor *, tensor out0, tensor out1, tensor input, tensor running_mean, tensor running_var, double momentum); +void atg_bernoulli(tensor *, tensor self); +void atg_bernoulli_(tensor *, tensor self, tensor p); +void atg_bernoulli_float_(tensor *, tensor self, double p); +void atg_bernoulli_p(tensor *, tensor self, double p); +void atg_bernoulli_tensor(tensor *, tensor self, tensor p); +void atg_bilinear(tensor *, tensor input1, tensor input2, tensor weight, tensor bias); +void atg_binary_cross_entropy(tensor *, tensor self, tensor target, tensor weight, int64_t reduction); +void atg_binary_cross_entropy_backward(tensor *, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction); +void atg_binary_cross_entropy_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction); +void atg_binary_cross_entropy_out(tensor *, tensor out, tensor self, tensor target, tensor weight, int64_t reduction); +void atg_binary_cross_entropy_with_logits(tensor *, tensor self, tensor target, tensor weight, tensor pos_weight, int64_t reduction); +void atg_binary_cross_entropy_with_logits_out(tensor *, tensor out, tensor self, tensor target, tensor weight, tensor pos_weight, int64_t reduction); +void atg_bincount(tensor *, tensor self, tensor weights, int64_t minlength); +void atg_bincount_out(tensor *, tensor out, tensor self, tensor weights, int64_t minlength); +void atg_binomial(tensor *, tensor count, tensor prob); +void atg_binomial_out(tensor *, tensor out, tensor count, tensor prob); +void atg_bitwise_and(tensor *, tensor self, scalar other); +void atg_bitwise_and_(tensor *, tensor self, scalar other); +void atg_bitwise_and_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_bitwise_and_scalar_tensor(tensor *, scalar self_scalar, tensor other); +void atg_bitwise_and_scalar_tensor_out(tensor *, tensor out, scalar self_scalar, tensor other); +void atg_bitwise_and_tensor(tensor *, tensor self, tensor other); +void atg_bitwise_and_tensor_(tensor *, tensor self, tensor other); +void atg_bitwise_and_tensor_out(tensor *, tensor out, tensor self, tensor other); +void atg_bitwise_left_shift(tensor *, tensor self, tensor other); +void atg_bitwise_left_shift_(tensor *, tensor self, tensor other); +void atg_bitwise_left_shift_scalar_tensor(tensor *, scalar self_scalar, tensor other); +void atg_bitwise_left_shift_scalar_tensor_out(tensor *, tensor out, scalar self_scalar, tensor other); +void atg_bitwise_left_shift_tensor_out(tensor *, tensor out, tensor self, tensor other); +void atg_bitwise_left_shift_tensor_scalar(tensor *, tensor self, scalar other); +void atg_bitwise_left_shift_tensor_scalar_(tensor *, tensor self, scalar other); +void atg_bitwise_left_shift_tensor_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_bitwise_not(tensor *, tensor self); +void atg_bitwise_not_(tensor *, tensor self); +void atg_bitwise_not_out(tensor *, tensor out, tensor self); +void atg_bitwise_or(tensor *, tensor self, scalar other); +void atg_bitwise_or_(tensor *, tensor self, scalar other); +void atg_bitwise_or_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_bitwise_or_scalar_tensor(tensor *, scalar self_scalar, tensor other); +void atg_bitwise_or_scalar_tensor_out(tensor *, tensor out, scalar self_scalar, tensor other); +void atg_bitwise_or_tensor(tensor *, tensor self, tensor other); +void atg_bitwise_or_tensor_(tensor *, tensor self, tensor other); +void atg_bitwise_or_tensor_out(tensor *, tensor out, tensor self, tensor other); +void atg_bitwise_right_shift(tensor *, tensor self, tensor other); +void atg_bitwise_right_shift_(tensor *, tensor self, tensor other); +void atg_bitwise_right_shift_scalar_tensor(tensor *, scalar self_scalar, tensor other); +void atg_bitwise_right_shift_scalar_tensor_out(tensor *, tensor out, scalar self_scalar, tensor other); +void atg_bitwise_right_shift_tensor_out(tensor *, tensor out, tensor self, tensor other); +void atg_bitwise_right_shift_tensor_scalar(tensor *, tensor self, scalar other); +void atg_bitwise_right_shift_tensor_scalar_(tensor *, tensor self, scalar other); +void atg_bitwise_right_shift_tensor_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_bitwise_xor(tensor *, tensor self, scalar other); +void atg_bitwise_xor_(tensor *, tensor self, scalar other); +void atg_bitwise_xor_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_bitwise_xor_scalar_tensor(tensor *, scalar self_scalar, tensor other); +void atg_bitwise_xor_scalar_tensor_out(tensor *, tensor out, scalar self_scalar, tensor other); +void atg_bitwise_xor_tensor(tensor *, tensor self, tensor other); +void atg_bitwise_xor_tensor_(tensor *, tensor self, tensor other); +void atg_bitwise_xor_tensor_out(tensor *, tensor out, tensor self, tensor other); +void atg_blackman_window(tensor *, int64_t window_length, int options_kind, int options_device); +void atg_blackman_window_out(tensor *, tensor out, int64_t window_length); +void atg_blackman_window_periodic(tensor *, int64_t window_length, int periodic, int options_kind, int options_device); +void atg_blackman_window_periodic_out(tensor *, tensor out, int64_t window_length, int periodic); +void atg_block_diag(tensor *, tensor *tensors_data, int tensors_len); +void atg_block_diag_out(tensor *, tensor out, tensor *tensors_data, int tensors_len); +void atg_bmm(tensor *, tensor self, tensor mat2); +void atg_bmm_dtype(tensor *, tensor self, tensor mat2, int out_dtype); +void atg_bmm_dtype_out(tensor *, tensor out, tensor self, tensor mat2, int out_dtype); +void atg_bmm_out(tensor *, tensor out, tensor self, tensor mat2); +tensor *atg_broadcast_tensors(tensor *tensors_data, int tensors_len); +void atg_broadcast_to(tensor *, tensor self, int64_t *size_data, int size_len); +void atg_bucketize(tensor *, tensor self, tensor boundaries, int out_int32, int right); +void atg_bucketize_scalar(tensor *, scalar self_scalar, tensor boundaries, int out_int32, int right); +void atg_bucketize_scalar_out(tensor *, tensor out, scalar self_scalar, tensor boundaries, int out_int32, int right); +void atg_bucketize_tensor_out(tensor *, tensor out, tensor self, tensor boundaries, int out_int32, int right); +int atg_can_cast(int from_, int to); +void atg_cartesian_prod(tensor *, tensor *tensors_data, int tensors_len); +void atg_cat(tensor *, tensor *tensors_data, int tensors_len, int64_t dim); +void atg_cat_out(tensor *, tensor out, tensor *tensors_data, int tensors_len, int64_t dim); +void atg_cauchy(tensor *, tensor self, double median, double sigma); +void atg_cauchy_(tensor *, tensor self, double median, double sigma); +void atg_cauchy_out(tensor *, tensor out, tensor self, double median, double sigma); +void atg_ccol_indices(tensor *, tensor self); +void atg_ccol_indices_copy(tensor *, tensor self); +void atg_ccol_indices_copy_out(tensor *, tensor out, tensor self); +void atg_cdist(tensor *, tensor x1, tensor x2, double p, int64_t compute_mode_v, uint8_t compute_mode_null); +void atg_ceil(tensor *, tensor self); +void atg_ceil_(tensor *, tensor self); +void atg_ceil_out(tensor *, tensor out, tensor self); +void atg_celu(tensor *, tensor self); +void atg_celu_(tensor *, tensor self); +void atg_celu_out(tensor *, tensor out, tensor self); +void atg_chain_matmul(tensor *, tensor *matrices_data, int matrices_len); +void atg_chain_matmul_out(tensor *, tensor out, tensor *matrices_data, int matrices_len); +void atg_chalf(tensor *, tensor self); +void atg_channel_shuffle(tensor *, tensor self, int64_t groups); +void atg_channel_shuffle_out(tensor *, tensor out, tensor self, int64_t groups); +void atg_cholesky(tensor *, tensor self, int upper); +void atg_cholesky_inverse(tensor *, tensor self, int upper); +void atg_cholesky_inverse_out(tensor *, tensor out, tensor self, int upper); +void atg_cholesky_out(tensor *, tensor out, tensor self, int upper); +void atg_cholesky_solve(tensor *, tensor self, tensor input2, int upper); +void atg_cholesky_solve_out(tensor *, tensor out, tensor self, tensor input2, int upper); +void atg_choose_qparams_optimized(tensor *, tensor input, int64_t numel, int64_t n_bins, double ratio, int64_t bit_width); +tensor *atg_chunk(tensor self, int64_t chunks, int64_t dim); +void atg_clamp(tensor *, tensor self, scalar min, scalar max); +void atg_clamp_(tensor *, tensor self, scalar min, scalar max); +void atg_clamp_max(tensor *, tensor self, scalar max); +void atg_clamp_max_(tensor *, tensor self, scalar max); +void atg_clamp_max_out(tensor *, tensor out, tensor self, scalar max); +void atg_clamp_max_tensor(tensor *, tensor self, tensor max); +void atg_clamp_max_tensor_(tensor *, tensor self, tensor max); +void atg_clamp_max_tensor_out(tensor *, tensor out, tensor self, tensor max); +void atg_clamp_min(tensor *, tensor self, scalar min); +void atg_clamp_min_(tensor *, tensor self, scalar min); +void atg_clamp_min_out(tensor *, tensor out, tensor self, scalar min); +void atg_clamp_min_tensor(tensor *, tensor self, tensor min); +void atg_clamp_min_tensor_(tensor *, tensor self, tensor min); +void atg_clamp_min_tensor_out(tensor *, tensor out, tensor self, tensor min); +void atg_clamp_out(tensor *, tensor out, tensor self, scalar min, scalar max); +void atg_clamp_tensor(tensor *, tensor self, tensor min, tensor max); +void atg_clamp_tensor_(tensor *, tensor self, tensor min, tensor max); +void atg_clamp_tensor_out(tensor *, tensor out, tensor self, tensor min, tensor max); +void atg_clip(tensor *, tensor self, scalar min, scalar max); +void atg_clip_(tensor *, tensor self, scalar min, scalar max); +void atg_clip_out(tensor *, tensor out, tensor self, scalar min, scalar max); +void atg_clip_tensor(tensor *, tensor self, tensor min, tensor max); +void atg_clip_tensor_(tensor *, tensor self, tensor min, tensor max); +void atg_clip_tensor_out(tensor *, tensor out, tensor self, tensor min, tensor max); +void atg_clone(tensor *, tensor out, tensor self); +void atg_coalesce(tensor *, tensor self); +void atg_col2im(tensor *, tensor self, int64_t *output_size_data, int output_size_len, int64_t *kernel_size_data, int kernel_size_len, int64_t *dilation_data, int dilation_len, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len); +void atg_col2im_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int64_t *kernel_size_data, int kernel_size_len, int64_t *dilation_data, int dilation_len, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len); +void atg_col_indices(tensor *, tensor self); +void atg_col_indices_copy(tensor *, tensor self); +void atg_col_indices_copy_out(tensor *, tensor out, tensor self); +void atg_column_stack(tensor *, tensor *tensors_data, int tensors_len); +void atg_column_stack_out(tensor *, tensor out, tensor *tensors_data, int tensors_len); +void atg_combinations(tensor *, tensor self, int64_t r, int with_replacement); +void atg_complex(tensor *, tensor real, tensor imag); +void atg_complex_out(tensor *, tensor out, tensor real, tensor imag); +void atg_concat(tensor *, tensor *tensors_data, int tensors_len, int64_t dim); +void atg_concat_out(tensor *, tensor out, tensor *tensors_data, int tensors_len, int64_t dim); +void atg_concatenate(tensor *, tensor *tensors_data, int tensors_len, int64_t dim); +void atg_concatenate_out(tensor *, tensor out, tensor *tensors_data, int tensors_len, int64_t dim); +void atg_conj(tensor *, tensor self); +void atg_conj_physical(tensor *, tensor self); +void atg_conj_physical_(tensor *, tensor self); +void atg_conj_physical_out(tensor *, tensor out, tensor self); +void atg_constant_pad_nd(tensor *, tensor self, int64_t *pad_data, int pad_len); +void atg_constant_pad_nd_out(tensor *, tensor out, tensor self, int64_t *pad_data, int pad_len); +void atg_contiguous(tensor *, tensor self); +void atg_conv1d(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); +void atg_conv1d_padding(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, char* padding_ptr, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); +void atg_conv2d(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); +void atg_conv2d_padding(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, char* padding_ptr, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); +void atg_conv3d(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); +void atg_conv3d_padding(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, char* padding_ptr, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); +void atg_conv_depthwise3d(tensor *, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len); +void atg_conv_depthwise3d_out(tensor *, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len); +void atg_conv_tbc(tensor *, tensor self, tensor weight, tensor bias, int64_t pad); +void atg_conv_tbc_backward(tensor *, tensor self, tensor input, tensor weight, tensor bias, int64_t pad); +void atg_conv_tbc_out(tensor *, tensor out, tensor self, tensor weight, tensor bias, int64_t pad); +void atg_conv_transpose1d(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t groups, int64_t *dilation_data, int dilation_len); +void atg_conv_transpose2d(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t groups, int64_t *dilation_data, int dilation_len); +void atg_conv_transpose3d(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t groups, int64_t *dilation_data, int dilation_len); +void atg_convolution(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups); +void atg_convolution_out(tensor *, tensor out, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups); +void atg_convolution_overrideable(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups); +void atg_convolution_overrideable_out(tensor *, tensor out, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups); +void atg_copy_sparse_to_sparse(tensor *, tensor self, tensor src, int non_blocking); +void atg_copy_sparse_to_sparse_(tensor *, tensor self, tensor src, int non_blocking); +void atg_copy_sparse_to_sparse_out(tensor *, tensor out, tensor self, tensor src, int non_blocking); +void atg_copysign(tensor *, tensor self, tensor other); +void atg_copysign_(tensor *, tensor self, tensor other); +void atg_copysign_out(tensor *, tensor out, tensor self, tensor other); +void atg_copysign_scalar(tensor *, tensor self, scalar other); +void atg_copysign_scalar_(tensor *, tensor self, scalar other); +void atg_copysign_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_corrcoef(tensor *, tensor self); +void atg_cos(tensor *, tensor self); +void atg_cos_(tensor *, tensor self); +void atg_cos_out(tensor *, tensor out, tensor self); +void atg_cosh(tensor *, tensor self); +void atg_cosh_(tensor *, tensor self); +void atg_cosh_out(tensor *, tensor out, tensor self); +void atg_cosine_embedding_loss(tensor *, tensor input1, tensor input2, tensor target, double margin, int64_t reduction); +void atg_cosine_similarity(tensor *, tensor x1, tensor x2, int64_t dim, double eps); +void atg_count_nonzero(tensor *, tensor self, int64_t dim_v, uint8_t dim_null); +void atg_count_nonzero_dim_intlist(tensor *, tensor self, int64_t *dim_data, int dim_len); +void atg_count_nonzero_dim_intlist_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len); +void atg_count_nonzero_out(tensor *, tensor out, tensor self, int64_t dim_v, uint8_t dim_null); +void atg_cov(tensor *, tensor self, int64_t correction, tensor fweights, tensor aweights); +void atg_cross(tensor *, tensor self, tensor other, int64_t dim_v, uint8_t dim_null); +void atg_cross_entropy_loss(tensor *, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index, double label_smoothing); +void atg_cross_out(tensor *, tensor out, tensor self, tensor other, int64_t dim_v, uint8_t dim_null); +void atg_crow_indices(tensor *, tensor self); +void atg_crow_indices_copy(tensor *, tensor self); +void atg_crow_indices_copy_out(tensor *, tensor out, tensor self); +void atg_ctc_loss(tensor *, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int64_t reduction, int zero_infinity); +void atg_ctc_loss_tensor(tensor *, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank, int64_t reduction, int zero_infinity); +void atg_cudnn_affine_grid_generator(tensor *, tensor theta, int64_t n, int64_t C, int64_t H, int64_t W); +void atg_cudnn_affine_grid_generator_backward(tensor *, tensor grad, int64_t n, int64_t C, int64_t H, int64_t W); +void atg_cudnn_affine_grid_generator_backward_out(tensor *, tensor out, tensor grad, int64_t n, int64_t C, int64_t H, int64_t W); +void atg_cudnn_affine_grid_generator_out(tensor *, tensor out, tensor theta, int64_t n, int64_t C, int64_t H, int64_t W); +void atg_cudnn_batch_norm(tensor *, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double exponential_average_factor, double epsilon); +void atg_cudnn_batch_norm_backward(tensor *, tensor input, tensor grad_output, tensor weight, tensor running_mean, tensor running_var, tensor save_mean, tensor save_var, double epsilon, tensor reserveSpace); +void atg_cudnn_batch_norm_backward_out(tensor *, tensor out0, tensor out1, tensor out2, tensor input, tensor grad_output, tensor weight, tensor running_mean, tensor running_var, tensor save_mean, tensor save_var, double epsilon, tensor reserveSpace); +void atg_cudnn_batch_norm_out(tensor *, tensor out0, tensor out1, tensor out2, tensor out3, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double exponential_average_factor, double epsilon); +void atg_cudnn_convolution(tensor *, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic, int allow_tf32); +void atg_cudnn_convolution_add_relu(tensor *, tensor self, tensor weight, tensor z, scalar alpha, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); +void atg_cudnn_convolution_add_relu_out(tensor *, tensor out, tensor self, tensor weight, tensor z, scalar alpha, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); +void atg_cudnn_convolution_out(tensor *, tensor out, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic, int allow_tf32); +void atg_cudnn_convolution_relu(tensor *, tensor self, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); +void atg_cudnn_convolution_relu_out(tensor *, tensor out, tensor self, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); +void atg_cudnn_convolution_transpose(tensor *, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic, int allow_tf32); +void atg_cudnn_convolution_transpose_out(tensor *, tensor out, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic, int allow_tf32); +void atg_cudnn_grid_sampler(tensor *, tensor self, tensor grid); +void atg_cudnn_grid_sampler_backward(tensor *, tensor self, tensor grid, tensor grad_output); +void atg_cudnn_grid_sampler_backward_out(tensor *, tensor out0, tensor out1, tensor self, tensor grid, tensor grad_output); +void atg_cudnn_grid_sampler_out(tensor *, tensor out, tensor self, tensor grid); +int atg_cudnn_is_acceptable(tensor self); +void atg_cummax(tensor *, tensor self, int64_t dim); +void atg_cummax_out(tensor *, tensor values, tensor indices, tensor self, int64_t dim); +void atg_cummaxmin_backward(tensor *, tensor grad, tensor input, tensor indices, int64_t dim); +void atg_cummin(tensor *, tensor self, int64_t dim); +void atg_cummin_out(tensor *, tensor values, tensor indices, tensor self, int64_t dim); +void atg_cumprod(tensor *, tensor self, int64_t dim, int dtype); +void atg_cumprod_(tensor *, tensor self, int64_t dim, int dtype); +void atg_cumprod_backward(tensor *, tensor grad, tensor input, int64_t dim, tensor output); +void atg_cumprod_out(tensor *, tensor out, tensor self, int64_t dim, int dtype); +void atg_cumsum(tensor *, tensor self, int64_t dim, int dtype); +void atg_cumsum_(tensor *, tensor self, int64_t dim, int dtype); +void atg_cumsum_out(tensor *, tensor out, tensor self, int64_t dim, int dtype); +void atg_cumulative_trapezoid(tensor *, tensor y, int64_t dim); +void atg_cumulative_trapezoid_x(tensor *, tensor y, tensor x, int64_t dim); +void atg_data(tensor *, tensor self); +void atg_deg2rad(tensor *, tensor self); +void atg_deg2rad_(tensor *, tensor self); +void atg_deg2rad_out(tensor *, tensor out, tensor self); +int64_t atg_dense_dim(tensor self); +void atg_dequantize(tensor *, tensor self); +void atg_dequantize_self_out(tensor *, tensor out, tensor self); +tensor *atg_dequantize_tensors(tensor *tensors_data, int tensors_len); +void atg_dequantize_tensors_out(tensor *out_data, int out_len, tensor *tensors_data, int tensors_len); +void atg_det(tensor *, tensor self); +void atg_detach(tensor *, tensor self); +void atg_detach_(tensor *, tensor self); +void atg_detach_copy(tensor *, tensor self); +void atg_detach_copy_out(tensor *, tensor out, tensor self); +void atg_diag(tensor *, tensor self, int64_t diagonal); +void atg_diag_embed(tensor *, tensor self, int64_t offset, int64_t dim1, int64_t dim2); +void atg_diag_embed_out(tensor *, tensor out, tensor self, int64_t offset, int64_t dim1, int64_t dim2); +void atg_diag_out(tensor *, tensor out, tensor self, int64_t diagonal); +void atg_diagflat(tensor *, tensor self, int64_t offset); +void atg_diagonal(tensor *, tensor self, int64_t offset, int64_t dim1, int64_t dim2); +void atg_diagonal_backward(tensor *, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t offset, int64_t dim1, int64_t dim2); +void atg_diagonal_backward_out(tensor *, tensor out, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t offset, int64_t dim1, int64_t dim2); +void atg_diagonal_copy(tensor *, tensor self, int64_t offset, int64_t dim1, int64_t dim2); +void atg_diagonal_copy_out(tensor *, tensor out, tensor self, int64_t offset, int64_t dim1, int64_t dim2); +void atg_diagonal_scatter(tensor *, tensor self, tensor src, int64_t offset, int64_t dim1, int64_t dim2); +void atg_diagonal_scatter_out(tensor *, tensor out, tensor self, tensor src, int64_t offset, int64_t dim1, int64_t dim2); +void atg_diff(tensor *, tensor self, int64_t n, int64_t dim, tensor prepend, tensor append); +void atg_diff_out(tensor *, tensor out, tensor self, int64_t n, int64_t dim, tensor prepend, tensor append); +void atg_digamma(tensor *, tensor self); +void atg_digamma_(tensor *, tensor self); +void atg_digamma_out(tensor *, tensor out, tensor self); +void atg_dist(tensor *, tensor self, tensor other); +void atg_dist_out(tensor *, tensor out, tensor self, tensor other); +void atg_div(tensor *, tensor self, tensor other); +void atg_div_(tensor *, tensor self, tensor other); +void atg_div_out(tensor *, tensor out, tensor self, tensor other); +void atg_div_out_mode(tensor *, tensor out, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len); +void atg_div_scalar(tensor *, tensor self, scalar other); +void atg_div_scalar_(tensor *, tensor self, scalar other); +void atg_div_scalar_mode(tensor *, tensor self, scalar other, char* rounding_mode_ptr, int rounding_mode_len); +void atg_div_scalar_mode_(tensor *, tensor self, scalar other, char* rounding_mode_ptr, int rounding_mode_len); +void atg_div_scalar_mode_out(tensor *, tensor out, tensor self, scalar other, char* rounding_mode_ptr, int rounding_mode_len); +void atg_div_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_div_tensor_mode(tensor *, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len); +void atg_div_tensor_mode_(tensor *, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len); +void atg_divide(tensor *, tensor self, tensor other); +void atg_divide_(tensor *, tensor self, tensor other); +void atg_divide_out(tensor *, tensor out, tensor self, tensor other); +void atg_divide_out_mode(tensor *, tensor out, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len); +void atg_divide_scalar(tensor *, tensor self, scalar other); +void atg_divide_scalar_(tensor *, tensor self, scalar other); +void atg_divide_scalar_mode(tensor *, tensor self, scalar other, char* rounding_mode_ptr, int rounding_mode_len); +void atg_divide_scalar_mode_(tensor *, tensor self, scalar other, char* rounding_mode_ptr, int rounding_mode_len); +void atg_divide_tensor_mode(tensor *, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len); +void atg_divide_tensor_mode_(tensor *, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len); +void atg_dot(tensor *, tensor self, tensor tensor); +void atg_dot_out(tensor *, tensor out, tensor self, tensor tensor); +void atg_dropout(tensor *, tensor input, double p, int train); +void atg_dropout_(tensor *, tensor self, double p, int train); +tensor *atg_dsplit(tensor self, int64_t sections); +tensor *atg_dsplit_array(tensor self, int64_t *indices_data, int indices_len); +void atg_dstack(tensor *, tensor *tensors_data, int tensors_len); +void atg_dstack_out(tensor *, tensor out, tensor *tensors_data, int tensors_len); +void atg_einsum(tensor *, char* equation_ptr, int equation_len, tensor *tensors_data, int tensors_len, int64_t *path_data, int path_len); +void atg_elu(tensor *, tensor self); +void atg_elu_(tensor *, tensor self); +void atg_elu_backward(tensor *, tensor grad_output, scalar alpha, scalar scale, scalar input_scale, int is_result, tensor self_or_result); +void atg_elu_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, scalar alpha, scalar scale, scalar input_scale, int is_result, tensor self_or_result); +void atg_elu_out(tensor *, tensor out, tensor self); +void atg_embedding(tensor *, tensor weight, tensor indices, int64_t padding_idx, int scale_grad_by_freq, int sparse); +void atg_embedding_backward(tensor *, tensor grad, tensor indices, int64_t num_weights, int64_t padding_idx, int scale_grad_by_freq, int sparse); +void atg_embedding_bag(tensor *, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset); +void atg_embedding_bag_padding_idx(tensor *, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset, int64_t padding_idx_v, uint8_t padding_idx_null); +void atg_embedding_dense_backward(tensor *, tensor grad_output, tensor indices, int64_t num_weights, int64_t padding_idx, int scale_grad_by_freq); +void atg_embedding_dense_backward_out(tensor *, tensor out, tensor grad_output, tensor indices, int64_t num_weights, int64_t padding_idx, int scale_grad_by_freq); +void atg_embedding_out(tensor *, tensor out, tensor weight, tensor indices, int64_t padding_idx, int scale_grad_by_freq, int sparse); +void atg_embedding_renorm(tensor *, tensor self, tensor indices, double max_norm, double norm_type); +void atg_embedding_renorm_(tensor *, tensor self, tensor indices, double max_norm, double norm_type); +void atg_embedding_renorm_out(tensor *, tensor out, tensor self, tensor indices, double max_norm, double norm_type); +void atg_embedding_sparse_backward(tensor *, tensor grad, tensor indices, int64_t num_weights, int64_t padding_idx, int scale_grad_by_freq); +void atg_empty(tensor *, int64_t *size_data, int size_len, int options_kind, int options_device); +void atg_empty_like(tensor *, tensor self); +void atg_empty_like_out(tensor *, tensor out, tensor self); +void atg_empty_out(tensor *, tensor out, int64_t *size_data, int size_len); +void atg_empty_permuted(tensor *, int64_t *size_data, int size_len, int64_t *physical_layout_data, int physical_layout_len, int options_kind, int options_device); +void atg_empty_permuted_out(tensor *, tensor out, int64_t *size_data, int size_len, int64_t *physical_layout_data, int physical_layout_len); +void atg_empty_quantized(tensor *, int64_t *size_data, int size_len, tensor qtensor, int options_kind, int options_device); +void atg_empty_quantized_out(tensor *, tensor out, int64_t *size_data, int size_len, tensor qtensor); +void atg_empty_strided(tensor *, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int options_kind, int options_device); +void atg_empty_strided_out(tensor *, tensor out, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len); +void atg_eq(tensor *, tensor self, scalar other); +void atg_eq_(tensor *, tensor self, scalar other); +void atg_eq_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_eq_tensor(tensor *, tensor self, tensor other); +void atg_eq_tensor_(tensor *, tensor self, tensor other); +void atg_eq_tensor_out(tensor *, tensor out, tensor self, tensor other); +int atg_equal(tensor self, tensor other); +void atg_erf(tensor *, tensor self); +void atg_erf_(tensor *, tensor self); +void atg_erf_out(tensor *, tensor out, tensor self); +void atg_erfc(tensor *, tensor self); +void atg_erfc_(tensor *, tensor self); +void atg_erfc_out(tensor *, tensor out, tensor self); +void atg_erfinv(tensor *, tensor self); +void atg_erfinv_(tensor *, tensor self); +void atg_erfinv_out(tensor *, tensor out, tensor self); +void atg_exp(tensor *, tensor self); +void atg_exp2(tensor *, tensor self); +void atg_exp2_(tensor *, tensor self); +void atg_exp2_out(tensor *, tensor out, tensor self); +void atg_exp_(tensor *, tensor self); +void atg_exp_out(tensor *, tensor out, tensor self); +void atg_expand(tensor *, tensor self, int64_t *size_data, int size_len, int implicit); +void atg_expand_as(tensor *, tensor self, tensor other); +void atg_expand_copy(tensor *, tensor self, int64_t *size_data, int size_len, int implicit); +void atg_expand_copy_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len, int implicit); +void atg_expm1(tensor *, tensor self); +void atg_expm1_(tensor *, tensor self); +void atg_expm1_out(tensor *, tensor out, tensor self); +void atg_exponential(tensor *, tensor self, double lambd); +void atg_exponential_(tensor *, tensor self, double lambd); +void atg_exponential_out(tensor *, tensor out, tensor self, double lambd); +void atg_eye(tensor *, int64_t n, int options_kind, int options_device); +void atg_eye_m(tensor *, int64_t n, int64_t m, int options_kind, int options_device); +void atg_eye_m_out(tensor *, tensor out, int64_t n, int64_t m); +void atg_eye_out(tensor *, tensor out, int64_t n); +void atg_fake_quantize_per_channel_affine(tensor *, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max); +void atg_fake_quantize_per_channel_affine_cachemask(tensor *, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max); +void atg_fake_quantize_per_channel_affine_cachemask_backward(tensor *, tensor grad, tensor mask); +void atg_fake_quantize_per_channel_affine_cachemask_out(tensor *, tensor out0, tensor out1, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max); +void atg_fake_quantize_per_tensor_affine(tensor *, tensor self, double scale, int64_t zero_point, int64_t quant_min, int64_t quant_max); +void atg_fake_quantize_per_tensor_affine_cachemask(tensor *, tensor self, double scale, int64_t zero_point, int64_t quant_min, int64_t quant_max); +void atg_fake_quantize_per_tensor_affine_cachemask_backward(tensor *, tensor grad, tensor mask); +void atg_fake_quantize_per_tensor_affine_cachemask_out(tensor *, tensor out0, tensor out1, tensor self, double scale, int64_t zero_point, int64_t quant_min, int64_t quant_max); +void atg_fake_quantize_per_tensor_affine_tensor_qparams(tensor *, tensor self, tensor scale, tensor zero_point, int64_t quant_min, int64_t quant_max); +void atg_feature_alpha_dropout(tensor *, tensor input, double p, int train); +void atg_feature_alpha_dropout_(tensor *, tensor self, double p, int train); +void atg_feature_dropout(tensor *, tensor input, double p, int train); +void atg_feature_dropout_(tensor *, tensor self, double p, int train); +void atg_fft_fft(tensor *, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); +void atg_fft_fft2(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_fft2_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_fft_out(tensor *, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); +void atg_fft_fftfreq(tensor *, int64_t n, double d, int options_kind, int options_device); +void atg_fft_fftfreq_out(tensor *, tensor out, int64_t n, double d); +void atg_fft_fftn(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_fftn_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_fftshift(tensor *, tensor self, int64_t *dim_data, int dim_len); +void atg_fft_hfft(tensor *, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); +void atg_fft_hfft2(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_hfft2_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_hfft_out(tensor *, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); +void atg_fft_hfftn(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_hfftn_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_ifft(tensor *, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); +void atg_fft_ifft2(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_ifft2_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_ifft_out(tensor *, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); +void atg_fft_ifftn(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_ifftn_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_ifftshift(tensor *, tensor self, int64_t *dim_data, int dim_len); +void atg_fft_ihfft(tensor *, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); +void atg_fft_ihfft2(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_ihfft2_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_ihfft_out(tensor *, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); +void atg_fft_ihfftn(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_ihfftn_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_irfft(tensor *, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); +void atg_fft_irfft2(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_irfft2_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_irfft_out(tensor *, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); +void atg_fft_irfftn(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_irfftn_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_rfft(tensor *, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); +void atg_fft_rfft2(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_rfft2_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_rfft_out(tensor *, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); +void atg_fft_rfftfreq(tensor *, int64_t n, double d, int options_kind, int options_device); +void atg_fft_rfftfreq_out(tensor *, tensor out, int64_t n, double d); +void atg_fft_rfftn(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fft_rfftn_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); +void atg_fill(tensor *, tensor self, scalar value); +void atg_fill_(tensor *, tensor self, scalar value); +void atg_fill_diagonal_(tensor *, tensor self, scalar fill_value, int wrap); +void atg_fill_scalar_out(tensor *, tensor out, tensor self, scalar value); +void atg_fill_tensor(tensor *, tensor self, tensor value); +void atg_fill_tensor_(tensor *, tensor self, tensor value); +void atg_fill_tensor_out(tensor *, tensor out, tensor self, tensor value); +void atg_fix(tensor *, tensor self); +void atg_fix_(tensor *, tensor self); +void atg_fix_out(tensor *, tensor out, tensor self); +void atg_flatten(tensor *, tensor self, int64_t start_dim, int64_t end_dim); +void atg_flatten_dense_tensors(tensor *, tensor *tensors_data, int tensors_len); +void atg_flip(tensor *, tensor self, int64_t *dims_data, int dims_len); +void atg_flip_out(tensor *, tensor out, tensor self, int64_t *dims_data, int dims_len); +void atg_fliplr(tensor *, tensor self); +void atg_flipud(tensor *, tensor self); +void atg_float_power(tensor *, tensor self, tensor exponent); +void atg_float_power_(tensor *, tensor self, scalar exponent); +void atg_float_power_scalar(tensor *, scalar self_scalar, tensor exponent); +void atg_float_power_scalar_out(tensor *, tensor out, scalar self_scalar, tensor exponent); +void atg_float_power_tensor_(tensor *, tensor self, tensor exponent); +void atg_float_power_tensor_scalar(tensor *, tensor self, scalar exponent); +void atg_float_power_tensor_scalar_out(tensor *, tensor out, tensor self, scalar exponent); +void atg_float_power_tensor_tensor_out(tensor *, tensor out, tensor self, tensor exponent); +void atg_floor(tensor *, tensor self); +void atg_floor_(tensor *, tensor self); +void atg_floor_divide(tensor *, tensor self, tensor other); +void atg_floor_divide_(tensor *, tensor self, tensor other); +void atg_floor_divide_out(tensor *, tensor out, tensor self, tensor other); +void atg_floor_divide_scalar(tensor *, tensor self, scalar other); +void atg_floor_divide_scalar_(tensor *, tensor self, scalar other); +void atg_floor_divide_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_floor_out(tensor *, tensor out, tensor self); +void atg_fmax(tensor *, tensor self, tensor other); +void atg_fmax_out(tensor *, tensor out, tensor self, tensor other); +void atg_fmin(tensor *, tensor self, tensor other); +void atg_fmin_out(tensor *, tensor out, tensor self, tensor other); +void atg_fmod(tensor *, tensor self, scalar other); +void atg_fmod_(tensor *, tensor self, scalar other); +void atg_fmod_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_fmod_tensor(tensor *, tensor self, tensor other); +void atg_fmod_tensor_(tensor *, tensor self, tensor other); +void atg_fmod_tensor_out(tensor *, tensor out, tensor self, tensor other); +void atg_frac(tensor *, tensor self); +void atg_frac_(tensor *, tensor self); +void atg_frac_out(tensor *, tensor out, tensor self); +void atg_fractional_max_pool2d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor random_samples); +void atg_fractional_max_pool2d_backward(tensor *, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor indices); +void atg_fractional_max_pool2d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor indices); +void atg_fractional_max_pool2d_output(tensor *, tensor output, tensor indices, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor random_samples); +void atg_fractional_max_pool3d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor random_samples); +void atg_fractional_max_pool3d_backward(tensor *, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor indices); +void atg_fractional_max_pool3d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor indices); +void atg_fractional_max_pool3d_output(tensor *, tensor output, tensor indices, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor random_samples); +void atg_frexp(tensor *, tensor self); +void atg_frexp_tensor_out(tensor *, tensor mantissa, tensor exponent, tensor self); +void atg_frobenius_norm(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim); +void atg_frobenius_norm_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim); +void atg_from_file(tensor *, char* filename_ptr, int filename_len, int shared, int64_t size_v, uint8_t size_null, int options_kind, int options_device); +void atg_from_file_out(tensor *, tensor out, char* filename_ptr, int filename_len, int shared, int64_t size_v, uint8_t size_null); +void atg_full(tensor *, int64_t *size_data, int size_len, scalar fill_value, int options_kind, int options_device); +void atg_full_like(tensor *, tensor self, scalar fill_value); +void atg_full_like_out(tensor *, tensor out, tensor self, scalar fill_value); +void atg_full_out(tensor *, tensor out, int64_t *size_data, int size_len, scalar fill_value); +void atg_fused_moving_avg_obs_fake_quant(tensor *, tensor self, tensor observer_on, tensor fake_quant_on, tensor running_min, tensor running_max, tensor scale, tensor zero_point, double averaging_const, int64_t quant_min, int64_t quant_max, int64_t ch_axis, int per_row_fake_quant, int symmetric_quant); +void atg_gather(tensor *, tensor self, int64_t dim, tensor index, int sparse_grad); +void atg_gather_backward(tensor *, tensor grad, tensor self, int64_t dim, tensor index, int sparse_grad); +void atg_gather_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, int sparse_grad); +void atg_gcd(tensor *, tensor self, tensor other); +void atg_gcd_(tensor *, tensor self, tensor other); +void atg_gcd_out(tensor *, tensor out, tensor self, tensor other); +void atg_ge(tensor *, tensor self, scalar other); +void atg_ge_(tensor *, tensor self, scalar other); +void atg_ge_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_ge_tensor(tensor *, tensor self, tensor other); +void atg_ge_tensor_(tensor *, tensor self, tensor other); +void atg_ge_tensor_out(tensor *, tensor out, tensor self, tensor other); +void atg_gelu(tensor *, tensor self, char* approximate_ptr, int approximate_len); +void atg_gelu_(tensor *, tensor self, char* approximate_ptr, int approximate_len); +void atg_gelu_backward(tensor *, tensor grad_output, tensor self, char* approximate_ptr, int approximate_len); +void atg_gelu_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, char* approximate_ptr, int approximate_len); +void atg_gelu_out(tensor *, tensor out, tensor self, char* approximate_ptr, int approximate_len); +void atg_geometric(tensor *, tensor self, double p); +void atg_geometric_(tensor *, tensor self, double p); +void atg_geometric_out(tensor *, tensor out, tensor self, double p); +void atg_geqrf(tensor *, tensor self); +void atg_geqrf_a(tensor *, tensor a, tensor tau, tensor self); +void atg_ger(tensor *, tensor self, tensor vec2); +void atg_ger_out(tensor *, tensor out, tensor self, tensor vec2); +void atg_glu(tensor *, tensor self, int64_t dim); +void atg_glu_backward(tensor *, tensor grad_output, tensor self, int64_t dim); +void atg_glu_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t dim); +void atg_glu_backward_jvp(tensor *, tensor grad_x, tensor grad_glu, tensor x, tensor dgrad_glu, tensor dx, int64_t dim); +void atg_glu_backward_jvp_out(tensor *, tensor out, tensor grad_x, tensor grad_glu, tensor x, tensor dgrad_glu, tensor dx, int64_t dim); +void atg_glu_jvp(tensor *, tensor glu, tensor x, tensor dx, int64_t dim); +void atg_glu_jvp_out(tensor *, tensor out, tensor glu, tensor x, tensor dx, int64_t dim); +void atg_glu_out(tensor *, tensor out, tensor self, int64_t dim); +void atg_grad(tensor *, tensor self); +void atg_greater(tensor *, tensor self, scalar other); +void atg_greater_(tensor *, tensor self, scalar other); +void atg_greater_equal(tensor *, tensor self, scalar other); +void atg_greater_equal_(tensor *, tensor self, scalar other); +void atg_greater_equal_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_greater_equal_tensor(tensor *, tensor self, tensor other); +void atg_greater_equal_tensor_(tensor *, tensor self, tensor other); +void atg_greater_equal_tensor_out(tensor *, tensor out, tensor self, tensor other); +void atg_greater_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_greater_tensor(tensor *, tensor self, tensor other); +void atg_greater_tensor_(tensor *, tensor self, tensor other); +void atg_greater_tensor_out(tensor *, tensor out, tensor self, tensor other); +void atg_grid_sampler(tensor *, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners); +void atg_grid_sampler_2d(tensor *, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners); +void atg_grid_sampler_2d_out(tensor *, tensor out, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners); +void atg_grid_sampler_3d(tensor *, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners); +void atg_grid_sampler_3d_out(tensor *, tensor out, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners); +void atg_group_norm(tensor *, tensor input, int64_t num_groups, tensor weight, tensor bias, double eps, int cudnn_enabled); +void atg_gru(tensor *, tensor input, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first); +void atg_gru_cell(tensor *, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh); +void atg_gru_data(tensor *, tensor data, tensor batch_sizes, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional); +void atg_gt(tensor *, tensor self, scalar other); +void atg_gt_(tensor *, tensor self, scalar other); +void atg_gt_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_gt_tensor(tensor *, tensor self, tensor other); +void atg_gt_tensor_(tensor *, tensor self, tensor other); +void atg_gt_tensor_out(tensor *, tensor out, tensor self, tensor other); +void atg_hamming_window(tensor *, int64_t window_length, int options_kind, int options_device); +void atg_hamming_window_out(tensor *, tensor out, int64_t window_length); +void atg_hamming_window_periodic(tensor *, int64_t window_length, int periodic, int options_kind, int options_device); +void atg_hamming_window_periodic_alpha(tensor *, int64_t window_length, int periodic, double alpha, int options_kind, int options_device); +void atg_hamming_window_periodic_alpha_beta(tensor *, int64_t window_length, int periodic, double alpha, double beta, int options_kind, int options_device); +void atg_hamming_window_periodic_alpha_beta_out(tensor *, tensor out, int64_t window_length, int periodic, double alpha, double beta); +void atg_hamming_window_periodic_alpha_out(tensor *, tensor out, int64_t window_length, int periodic, double alpha); +void atg_hamming_window_periodic_out(tensor *, tensor out, int64_t window_length, int periodic); +void atg_hann_window(tensor *, int64_t window_length, int options_kind, int options_device); +void atg_hann_window_out(tensor *, tensor out, int64_t window_length); +void atg_hann_window_periodic(tensor *, int64_t window_length, int periodic, int options_kind, int options_device); +void atg_hann_window_periodic_out(tensor *, tensor out, int64_t window_length, int periodic); +void atg_hardshrink(tensor *, tensor self); +void atg_hardshrink_backward(tensor *, tensor grad_out, tensor self, scalar lambd); +void atg_hardshrink_backward_grad_input(tensor *, tensor grad_input, tensor grad_out, tensor self, scalar lambd); +void atg_hardshrink_out(tensor *, tensor out, tensor self); +void atg_hardsigmoid(tensor *, tensor self); +void atg_hardsigmoid_(tensor *, tensor self); +void atg_hardsigmoid_backward(tensor *, tensor grad_output, tensor self); +void atg_hardsigmoid_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self); +void atg_hardsigmoid_out(tensor *, tensor out, tensor self); +void atg_hardswish(tensor *, tensor self); +void atg_hardswish_(tensor *, tensor self); +void atg_hardswish_backward(tensor *, tensor grad_output, tensor self); +void atg_hardswish_backward_out(tensor *, tensor out, tensor grad_output, tensor self); +void atg_hardswish_out(tensor *, tensor out, tensor self); +void atg_hardtanh(tensor *, tensor self); +void atg_hardtanh_(tensor *, tensor self); +void atg_hardtanh_backward(tensor *, tensor grad_output, tensor self, scalar min_val, scalar max_val); +void atg_hardtanh_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, scalar min_val, scalar max_val); +void atg_hardtanh_out(tensor *, tensor out, tensor self); +void atg_hash_tensor(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim, int64_t mode); +void atg_hash_tensor_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int64_t mode); +void atg_heaviside(tensor *, tensor self, tensor values); +void atg_heaviside_(tensor *, tensor self, tensor values); +void atg_heaviside_out(tensor *, tensor out, tensor self, tensor values); +void atg_hinge_embedding_loss(tensor *, tensor self, tensor target, double margin, int64_t reduction); +void atg_histc(tensor *, tensor self, int64_t bins); +void atg_histc_out(tensor *, tensor out, tensor self, int64_t bins); +void atg_histogram(tensor *, tensor self, tensor bins, tensor weight, int density); +void atg_histogram_bin_ct(tensor *, tensor self, int64_t bins, double *range_data, int range_len, tensor weight, int density); +void atg_histogram_bin_ct_out(tensor *, tensor hist, tensor bin_edges, tensor self, int64_t bins, double *range_data, int range_len, tensor weight, int density); +void atg_histogram_bins_tensor_out(tensor *, tensor hist, tensor bin_edges, tensor self, tensor bins, tensor weight, int density); +tensor *atg_hsplit(tensor self, int64_t sections); +tensor *atg_hsplit_array(tensor self, int64_t *indices_data, int indices_len); +void atg_hspmm(tensor *, tensor mat1, tensor mat2); +void atg_hspmm_out(tensor *, tensor out, tensor mat1, tensor mat2); +void atg_hstack(tensor *, tensor *tensors_data, int tensors_len); +void atg_hstack_out(tensor *, tensor out, tensor *tensors_data, int tensors_len); +void atg_huber_loss(tensor *, tensor self, tensor target, int64_t reduction, double delta); +void atg_huber_loss_backward(tensor *, tensor grad_output, tensor self, tensor target, int64_t reduction, double delta); +void atg_huber_loss_backward_out(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor target, int64_t reduction, double delta); +void atg_huber_loss_out(tensor *, tensor out, tensor self, tensor target, int64_t reduction, double delta); +void atg_hypot(tensor *, tensor self, tensor other); +void atg_hypot_(tensor *, tensor self, tensor other); +void atg_hypot_out(tensor *, tensor out, tensor self, tensor other); +void atg_i0(tensor *, tensor self); +void atg_i0_(tensor *, tensor self); +void atg_i0_out(tensor *, tensor out, tensor self); +void atg_igamma(tensor *, tensor self, tensor other); +void atg_igamma_(tensor *, tensor self, tensor other); +void atg_igamma_out(tensor *, tensor out, tensor self, tensor other); +void atg_igammac(tensor *, tensor self, tensor other); +void atg_igammac_(tensor *, tensor self, tensor other); +void atg_igammac_out(tensor *, tensor out, tensor self, tensor other); +void atg_im2col(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *dilation_data, int dilation_len, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len); +void atg_im2col_out(tensor *, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *dilation_data, int dilation_len, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len); +void atg_imag(tensor *, tensor self); +void atg_index(tensor *, tensor self, tensor *indices_data, int indices_len); +void atg_index_add(tensor *, tensor self, int64_t dim, tensor index, tensor source); +void atg_index_add_(tensor *, tensor self, int64_t dim, tensor index, tensor source); +void atg_index_add_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, tensor source); +void atg_index_copy(tensor *, tensor self, int64_t dim, tensor index, tensor source); +void atg_index_copy_(tensor *, tensor self, int64_t dim, tensor index, tensor source); +void atg_index_copy_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, tensor source); +void atg_index_fill(tensor *, tensor self, int64_t dim, tensor index, scalar value); +void atg_index_fill_(tensor *, tensor self, int64_t dim, tensor index, scalar value); +void atg_index_fill_int_scalar_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, scalar value); +void atg_index_fill_int_tensor(tensor *, tensor self, int64_t dim, tensor index, tensor value); +void atg_index_fill_int_tensor_(tensor *, tensor self, int64_t dim, tensor index, tensor value); +void atg_index_fill_int_tensor_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, tensor value); +void atg_index_put(tensor *, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate); +void atg_index_put_(tensor *, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate); +void atg_index_put_out(tensor *, tensor out, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate); +void atg_index_reduce(tensor *, tensor self, int64_t dim, tensor index, tensor source, char* reduce_ptr, int reduce_len, int include_self); +void atg_index_reduce_(tensor *, tensor self, int64_t dim, tensor index, tensor source, char* reduce_ptr, int reduce_len, int include_self); +void atg_index_reduce_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, tensor source, char* reduce_ptr, int reduce_len, int include_self); +void atg_index_select(tensor *, tensor self, int64_t dim, tensor index); +void atg_index_select_backward(tensor *, tensor grad, int64_t *self_sizes_data, int self_sizes_len, int64_t dim, tensor index); +void atg_index_select_out(tensor *, tensor out, tensor self, int64_t dim, tensor index); +void atg_index_tensor_out(tensor *, tensor out, tensor self, tensor *indices_data, int indices_len); +void atg_indices(tensor *, tensor self); +void atg_indices_copy(tensor *, tensor self); +void atg_indices_copy_out(tensor *, tensor out, tensor self); +void atg_infinitely_differentiable_gelu_backward(tensor *, tensor grad, tensor self); +void atg_inner(tensor *, tensor self, tensor other); +void atg_inner_out(tensor *, tensor out, tensor self, tensor other); +void atg_instance_norm(tensor *, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int use_input_stats, double momentum, double eps, int cudnn_enabled); +void atg_int_repr(tensor *, tensor self); +void atg_int_repr_out(tensor *, tensor out, tensor self); +void atg_inverse(tensor *, tensor self); +void atg_inverse_out(tensor *, tensor out, tensor self); +int atg_is_coalesced(tensor self); +int atg_is_complex(tensor self); +int atg_is_conj(tensor self); +int atg_is_distributed(tensor self); +int atg_is_floating_point(tensor self); +int atg_is_inference(tensor self); +int atg_is_leaf(tensor self); +int atg_is_neg(tensor self); +int atg_is_nonzero(tensor self); +int atg_is_pinned(tensor self, int device); +int atg_is_same_size(tensor self, tensor other); +int atg_is_set_to(tensor self, tensor tensor); +int atg_is_signed(tensor self); +int atg_is_vulkan_available(); +void atg_isclose(tensor *, tensor self, tensor other, double rtol, double atol, int equal_nan); +void atg_isfinite(tensor *, tensor self); +void atg_isin(tensor *, tensor elements, tensor test_elements, int assume_unique, int invert); +void atg_isin_scalar_tensor(tensor *, scalar element, tensor test_elements, int assume_unique, int invert); +void atg_isin_scalar_tensor_out(tensor *, tensor out, scalar element, tensor test_elements, int assume_unique, int invert); +void atg_isin_tensor_scalar(tensor *, tensor elements, scalar test_element, int assume_unique, int invert); +void atg_isin_tensor_scalar_out(tensor *, tensor out, tensor elements, scalar test_element, int assume_unique, int invert); +void atg_isin_tensor_tensor_out(tensor *, tensor out, tensor elements, tensor test_elements, int assume_unique, int invert); +void atg_isinf(tensor *, tensor self); +void atg_isinf_out(tensor *, tensor out, tensor self); +void atg_isnan(tensor *, tensor self); +void atg_isnan_out(tensor *, tensor out, tensor self); +void atg_isneginf(tensor *, tensor self); +void atg_isneginf_out(tensor *, tensor out, tensor self); +void atg_isposinf(tensor *, tensor self); +void atg_isposinf_out(tensor *, tensor out, tensor self); +void atg_isreal(tensor *, tensor self); +void atg_istft(tensor *, tensor self, int64_t n_fft, int64_t hop_length_v, uint8_t hop_length_null, int64_t win_length_v, uint8_t win_length_null, tensor window, int center, int normalized, int onesided, int64_t length_v, uint8_t length_null, int return_complex); +void atg_kaiser_window(tensor *, int64_t window_length, int options_kind, int options_device); +void atg_kaiser_window_beta(tensor *, int64_t window_length, int periodic, double beta, int options_kind, int options_device); +void atg_kaiser_window_beta_out(tensor *, tensor out, int64_t window_length, int periodic, double beta); +void atg_kaiser_window_out(tensor *, tensor out, int64_t window_length); +void atg_kaiser_window_periodic(tensor *, int64_t window_length, int periodic, int options_kind, int options_device); +void atg_kaiser_window_periodic_out(tensor *, tensor out, int64_t window_length, int periodic); +void atg_kl_div(tensor *, tensor self, tensor target, int64_t reduction, int log_target); +void atg_kron(tensor *, tensor self, tensor other); +void atg_kron_out(tensor *, tensor out, tensor self, tensor other); +void atg_kthvalue(tensor *, tensor self, int64_t k, int64_t dim, int keepdim); +void atg_kthvalue_values(tensor *, tensor values, tensor indices, tensor self, int64_t k, int64_t dim, int keepdim); +void atg_l1_loss(tensor *, tensor self, tensor target, int64_t reduction); +void atg_layer_norm(tensor *, tensor input, int64_t *normalized_shape_data, int normalized_shape_len, tensor weight, tensor bias, double eps, int cudnn_enable); +void atg_lcm(tensor *, tensor self, tensor other); +void atg_lcm_(tensor *, tensor self, tensor other); +void atg_lcm_out(tensor *, tensor out, tensor self, tensor other); +void atg_ldexp(tensor *, tensor self, tensor other); +void atg_ldexp_(tensor *, tensor self, tensor other); +void atg_ldexp_out(tensor *, tensor out, tensor self, tensor other); +void atg_le(tensor *, tensor self, scalar other); +void atg_le_(tensor *, tensor self, scalar other); +void atg_le_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_le_tensor(tensor *, tensor self, tensor other); +void atg_le_tensor_(tensor *, tensor self, tensor other); +void atg_le_tensor_out(tensor *, tensor out, tensor self, tensor other); +void atg_leaky_relu(tensor *, tensor self); +void atg_leaky_relu_(tensor *, tensor self); +void atg_leaky_relu_backward(tensor *, tensor grad_output, tensor self, scalar negative_slope, int self_is_result); +void atg_leaky_relu_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, scalar negative_slope, int self_is_result); +void atg_leaky_relu_out(tensor *, tensor out, tensor self); +void atg_lerp(tensor *, tensor self, tensor end, scalar weight); +void atg_lerp_(tensor *, tensor self, tensor end, scalar weight); +void atg_lerp_scalar_out(tensor *, tensor out, tensor self, tensor end, scalar weight); +void atg_lerp_tensor(tensor *, tensor self, tensor end, tensor weight); +void atg_lerp_tensor_(tensor *, tensor self, tensor end, tensor weight); +void atg_lerp_tensor_out(tensor *, tensor out, tensor self, tensor end, tensor weight); +void atg_less(tensor *, tensor self, scalar other); +void atg_less_(tensor *, tensor self, scalar other); +void atg_less_equal(tensor *, tensor self, scalar other); +void atg_less_equal_(tensor *, tensor self, scalar other); +void atg_less_equal_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_less_equal_tensor(tensor *, tensor self, tensor other); +void atg_less_equal_tensor_(tensor *, tensor self, tensor other); +void atg_less_equal_tensor_out(tensor *, tensor out, tensor self, tensor other); +void atg_less_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_less_tensor(tensor *, tensor self, tensor other); +void atg_less_tensor_(tensor *, tensor self, tensor other); +void atg_less_tensor_out(tensor *, tensor out, tensor self, tensor other); +void atg_lgamma(tensor *, tensor self); +void atg_lgamma_(tensor *, tensor self); +void atg_lgamma_out(tensor *, tensor out, tensor self); +void atg_lift(tensor *, tensor self); +void atg_lift_fresh(tensor *, tensor self); +void atg_lift_fresh_copy(tensor *, tensor self); +void atg_lift_fresh_copy_out(tensor *, tensor out, tensor self); +void atg_lift_out(tensor *, tensor out, tensor self); +void atg_linalg_cholesky(tensor *, tensor self, int upper); +void atg_linalg_cholesky_ex(tensor *, tensor self, int upper, int check_errors); +void atg_linalg_cholesky_ex_l(tensor *, tensor L, tensor info, tensor self, int upper, int check_errors); +void atg_linalg_cholesky_out(tensor *, tensor out, tensor self, int upper); +void atg_linalg_cond(tensor *, tensor self, scalar p); +void atg_linalg_cond_out(tensor *, tensor out, tensor self, scalar p); +void atg_linalg_cond_p_str(tensor *, tensor self, char* p_ptr, int p_len); +void atg_linalg_cond_p_str_out(tensor *, tensor out, tensor self, char* p_ptr, int p_len); +void atg_linalg_cross(tensor *, tensor self, tensor other, int64_t dim); +void atg_linalg_cross_out(tensor *, tensor out, tensor self, tensor other, int64_t dim); +void atg_linalg_det(tensor *, tensor A); +void atg_linalg_det_out(tensor *, tensor out, tensor A); +void atg_linalg_diagonal(tensor *, tensor A, int64_t offset, int64_t dim1, int64_t dim2); +void atg_linalg_eig(tensor *, tensor self); +void atg_linalg_eig_out(tensor *, tensor eigenvalues, tensor eigenvectors, tensor self); +void atg_linalg_eigh(tensor *, tensor self, char* UPLO_ptr, int UPLO_len); +void atg_linalg_eigh_eigvals(tensor *, tensor eigvals, tensor eigvecs, tensor self, char* UPLO_ptr, int UPLO_len); +void atg_linalg_eigvals(tensor *, tensor self); +void atg_linalg_eigvals_out(tensor *, tensor out, tensor self); +void atg_linalg_eigvalsh(tensor *, tensor self, char* UPLO_ptr, int UPLO_len); +void atg_linalg_eigvalsh_out(tensor *, tensor out, tensor self, char* UPLO_ptr, int UPLO_len); +void atg_linalg_householder_product(tensor *, tensor input, tensor tau); +void atg_linalg_householder_product_out(tensor *, tensor out, tensor input, tensor tau); +void atg_linalg_inv(tensor *, tensor A); +void atg_linalg_inv_ex(tensor *, tensor A, int check_errors); +void atg_linalg_inv_ex_inverse(tensor *, tensor inverse, tensor info, tensor A, int check_errors); +void atg_linalg_inv_out(tensor *, tensor out, tensor A); +void atg_linalg_ldl_factor(tensor *, tensor self, int hermitian); +void atg_linalg_ldl_factor_ex(tensor *, tensor self, int hermitian, int check_errors); +void atg_linalg_ldl_factor_ex_out(tensor *, tensor LD, tensor pivots, tensor info, tensor self, int hermitian, int check_errors); +void atg_linalg_ldl_factor_out(tensor *, tensor LD, tensor pivots, tensor self, int hermitian); +void atg_linalg_ldl_solve(tensor *, tensor LD, tensor pivots, tensor B, int hermitian); +void atg_linalg_ldl_solve_out(tensor *, tensor out, tensor LD, tensor pivots, tensor B, int hermitian); +void atg_linalg_lstsq(tensor *, tensor self, tensor b, double rcond_v, uint8_t rcond_null, char* driver_ptr, int driver_len); +void atg_linalg_lstsq_out(tensor *, tensor solution, tensor residuals, tensor rank, tensor singular_values, tensor self, tensor b, double rcond_v, uint8_t rcond_null, char* driver_ptr, int driver_len); +void atg_linalg_lu(tensor *, tensor A, int pivot); +void atg_linalg_lu_factor(tensor *, tensor A, int pivot); +void atg_linalg_lu_factor_ex(tensor *, tensor A, int pivot, int check_errors); +void atg_linalg_lu_factor_ex_out(tensor *, tensor LU, tensor pivots, tensor info, tensor A, int pivot, int check_errors); +void atg_linalg_lu_factor_out(tensor *, tensor LU, tensor pivots, tensor A, int pivot); +void atg_linalg_lu_out(tensor *, tensor P, tensor L, tensor U, tensor A, int pivot); +void atg_linalg_lu_solve(tensor *, tensor LU, tensor pivots, tensor B, int left, int adjoint); +void atg_linalg_lu_solve_out(tensor *, tensor out, tensor LU, tensor pivots, tensor B, int left, int adjoint); +void atg_linalg_matmul(tensor *, tensor self, tensor other); +void atg_linalg_matmul_out(tensor *, tensor out, tensor self, tensor other); +void atg_linalg_matrix_exp(tensor *, tensor self); +void atg_linalg_matrix_exp_out(tensor *, tensor out, tensor self); +void atg_linalg_matrix_power(tensor *, tensor self, int64_t n); +void atg_linalg_matrix_power_out(tensor *, tensor out, tensor self, int64_t n); +void atg_linalg_matrix_rank(tensor *, tensor self, double tol, int hermitian); +void atg_linalg_matrix_rank_atol_rtol_float(tensor *, tensor self, double atol_v, uint8_t atol_null, double rtol_v, uint8_t rtol_null, int hermitian); +void atg_linalg_matrix_rank_atol_rtol_float_out(tensor *, tensor out, tensor self, double atol_v, uint8_t atol_null, double rtol_v, uint8_t rtol_null, int hermitian); +void atg_linalg_matrix_rank_atol_rtol_tensor(tensor *, tensor input, tensor atol, tensor rtol, int hermitian); +void atg_linalg_matrix_rank_atol_rtol_tensor_out(tensor *, tensor out, tensor input, tensor atol, tensor rtol, int hermitian); +void atg_linalg_matrix_rank_out(tensor *, tensor out, tensor self, double tol, int hermitian); +void atg_linalg_matrix_rank_out_tol_tensor(tensor *, tensor out, tensor input, tensor tol, int hermitian); +void atg_linalg_matrix_rank_tol_tensor(tensor *, tensor input, tensor tol, int hermitian); +void atg_linalg_multi_dot(tensor *, tensor *tensors_data, int tensors_len); +void atg_linalg_multi_dot_out(tensor *, tensor out, tensor *tensors_data, int tensors_len); +void atg_linalg_norm(tensor *, tensor self, scalar ord, int64_t *dim_data, int dim_len, int keepdim, int dtype); +void atg_linalg_norm_ord_str(tensor *, tensor self, char* ord_ptr, int ord_len, int64_t *dim_data, int dim_len, int keepdim, int dtype); +void atg_linalg_norm_ord_str_out(tensor *, tensor out, tensor self, char* ord_ptr, int ord_len, int64_t *dim_data, int dim_len, int keepdim, int dtype); +void atg_linalg_norm_out(tensor *, tensor out, tensor self, scalar ord, int64_t *dim_data, int dim_len, int keepdim, int dtype); +void atg_linalg_pinv(tensor *, tensor self, double rcond, int hermitian); +void atg_linalg_pinv_atol_rtol_float(tensor *, tensor self, double atol_v, uint8_t atol_null, double rtol_v, uint8_t rtol_null, int hermitian); +void atg_linalg_pinv_atol_rtol_float_out(tensor *, tensor out, tensor self, double atol_v, uint8_t atol_null, double rtol_v, uint8_t rtol_null, int hermitian); +void atg_linalg_pinv_atol_rtol_tensor(tensor *, tensor self, tensor atol, tensor rtol, int hermitian); +void atg_linalg_pinv_atol_rtol_tensor_out(tensor *, tensor out, tensor self, tensor atol, tensor rtol, int hermitian); +void atg_linalg_pinv_out(tensor *, tensor out, tensor self, double rcond, int hermitian); +void atg_linalg_pinv_out_rcond_tensor(tensor *, tensor out, tensor self, tensor rcond, int hermitian); +void atg_linalg_pinv_rcond_tensor(tensor *, tensor self, tensor rcond, int hermitian); +void atg_linalg_qr(tensor *, tensor A, char* mode_ptr, int mode_len); +void atg_linalg_qr_out(tensor *, tensor Q, tensor R, tensor A, char* mode_ptr, int mode_len); +void atg_linalg_slogdet(tensor *, tensor A); +void atg_linalg_slogdet_out(tensor *, tensor sign, tensor logabsdet, tensor A); +void atg_linalg_solve(tensor *, tensor A, tensor B, int left); +void atg_linalg_solve_ex(tensor *, tensor A, tensor B, int left, int check_errors); +void atg_linalg_solve_ex_out(tensor *, tensor result, tensor info, tensor A, tensor B, int left, int check_errors); +void atg_linalg_solve_out(tensor *, tensor out, tensor A, tensor B, int left); +void atg_linalg_solve_triangular(tensor *, tensor self, tensor B, int upper, int left, int unitriangular); +void atg_linalg_solve_triangular_out(tensor *, tensor out, tensor self, tensor B, int upper, int left, int unitriangular); +void atg_linalg_svd(tensor *, tensor A, int full_matrices, char* driver_ptr, int driver_len); +void atg_linalg_svd_u(tensor *, tensor U, tensor S, tensor Vh, tensor A, int full_matrices, char* driver_ptr, int driver_len); +void atg_linalg_svdvals(tensor *, tensor A, char* driver_ptr, int driver_len); +void atg_linalg_svdvals_out(tensor *, tensor out, tensor A, char* driver_ptr, int driver_len); +void atg_linalg_tensorinv(tensor *, tensor self, int64_t ind); +void atg_linalg_tensorinv_out(tensor *, tensor out, tensor self, int64_t ind); +void atg_linalg_tensorsolve(tensor *, tensor self, tensor other, int64_t *dims_data, int dims_len); +void atg_linalg_tensorsolve_out(tensor *, tensor out, tensor self, tensor other, int64_t *dims_data, int dims_len); +void atg_linalg_vander(tensor *, tensor x, int64_t n_v, uint8_t n_null); +void atg_linalg_vecdot(tensor *, tensor x, tensor y, int64_t dim); +void atg_linalg_vecdot_out(tensor *, tensor out, tensor x, tensor y, int64_t dim); +void atg_linear(tensor *, tensor input, tensor weight, tensor bias); +void atg_linear_out(tensor *, tensor out, tensor input, tensor weight, tensor bias); +void atg_linspace(tensor *, scalar start, scalar end, int64_t steps, int options_kind, int options_device); +void atg_linspace_out(tensor *, tensor out, scalar start, scalar end, int64_t steps); +void atg_linspace_scalar_tensor(tensor *, scalar start, tensor end, int64_t steps, int options_kind, int options_device); +void atg_linspace_scalar_tensor_out(tensor *, tensor out, scalar start, tensor end, int64_t steps); +void atg_linspace_tensor_scalar(tensor *, tensor start, scalar end, int64_t steps, int options_kind, int options_device); +void atg_linspace_tensor_scalar_out(tensor *, tensor out, tensor start, scalar end, int64_t steps); +void atg_linspace_tensor_tensor(tensor *, tensor start, tensor end, int64_t steps, int options_kind, int options_device); +void atg_linspace_tensor_tensor_out(tensor *, tensor out, tensor start, tensor end, int64_t steps); +void atg_log(tensor *, tensor self); +void atg_log10(tensor *, tensor self); +void atg_log10_(tensor *, tensor self); +void atg_log10_out(tensor *, tensor out, tensor self); +void atg_log1p(tensor *, tensor self); +void atg_log1p_(tensor *, tensor self); +void atg_log1p_out(tensor *, tensor out, tensor self); +void atg_log2(tensor *, tensor self); +void atg_log2_(tensor *, tensor self); +void atg_log2_out(tensor *, tensor out, tensor self); +void atg_log_(tensor *, tensor self); +void atg_log_normal(tensor *, tensor self, double mean, double std); +void atg_log_normal_(tensor *, tensor self, double mean, double std); +void atg_log_normal_out(tensor *, tensor out, tensor self, double mean, double std); +void atg_log_out(tensor *, tensor out, tensor self); +void atg_log_sigmoid(tensor *, tensor self); +void atg_log_sigmoid_backward(tensor *, tensor grad_output, tensor self, tensor buffer); +void atg_log_sigmoid_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor buffer); +void atg_log_sigmoid_out(tensor *, tensor out, tensor self); +void atg_log_softmax(tensor *, tensor self, int64_t dim, int dtype); +void atg_log_softmax_int_out(tensor *, tensor out, tensor self, int64_t dim, int dtype); +void atg_logaddexp(tensor *, tensor self, tensor other); +void atg_logaddexp2(tensor *, tensor self, tensor other); +void atg_logaddexp2_out(tensor *, tensor out, tensor self, tensor other); +void atg_logaddexp_out(tensor *, tensor out, tensor self, tensor other); +void atg_logcumsumexp(tensor *, tensor self, int64_t dim); +void atg_logcumsumexp_out(tensor *, tensor out, tensor self, int64_t dim); +void atg_logdet(tensor *, tensor self); +void atg_logical_and(tensor *, tensor self, tensor other); +void atg_logical_and_(tensor *, tensor self, tensor other); +void atg_logical_and_out(tensor *, tensor out, tensor self, tensor other); +void atg_logical_not(tensor *, tensor self); +void atg_logical_not_(tensor *, tensor self); +void atg_logical_not_out(tensor *, tensor out, tensor self); +void atg_logical_or(tensor *, tensor self, tensor other); +void atg_logical_or_(tensor *, tensor self, tensor other); +void atg_logical_or_out(tensor *, tensor out, tensor self, tensor other); +void atg_logical_xor(tensor *, tensor self, tensor other); +void atg_logical_xor_(tensor *, tensor self, tensor other); +void atg_logical_xor_out(tensor *, tensor out, tensor self, tensor other); +void atg_logit(tensor *, tensor self, double eps_v, uint8_t eps_null); +void atg_logit_(tensor *, tensor self, double eps_v, uint8_t eps_null); +void atg_logit_backward(tensor *, tensor grad_output, tensor self, double eps_v, uint8_t eps_null); +void atg_logit_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, double eps_v, uint8_t eps_null); +void atg_logit_out(tensor *, tensor out, tensor self, double eps_v, uint8_t eps_null); +void atg_logspace(tensor *, scalar start, scalar end, int64_t steps, double base, int options_kind, int options_device); +void atg_logspace_out(tensor *, tensor out, scalar start, scalar end, int64_t steps, double base); +void atg_logspace_scalar_tensor(tensor *, scalar start, tensor end, int64_t steps, double base, int options_kind, int options_device); +void atg_logspace_scalar_tensor_out(tensor *, tensor out, scalar start, tensor end, int64_t steps, double base); +void atg_logspace_tensor_scalar(tensor *, tensor start, scalar end, int64_t steps, double base, int options_kind, int options_device); +void atg_logspace_tensor_scalar_out(tensor *, tensor out, tensor start, scalar end, int64_t steps, double base); +void atg_logspace_tensor_tensor(tensor *, tensor start, tensor end, int64_t steps, double base, int options_kind, int options_device); +void atg_logspace_tensor_tensor_out(tensor *, tensor out, tensor start, tensor end, int64_t steps, double base); +void atg_logsumexp(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim); +void atg_logsumexp_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim); +void atg_lstm(tensor *, tensor input, tensor *hx_data, int hx_len, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first); +void atg_lstm_cell(tensor *, tensor input, tensor *hx_data, int hx_len, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh); +void atg_lstm_data(tensor *, tensor data, tensor batch_sizes, tensor *hx_data, int hx_len, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional); +void atg_lstm_mps_backward(tensor out0, tensor *out1_data, int out1_len, tensor *out2_data, int out2_len, tensor grad_y, tensor grad_hy, tensor grad_cy, tensor z_state, tensor cell_state_fwd, tensor input, tensor layersOutputs, tensor *hx_data, int hx_len, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first); +void atg_lt(tensor *, tensor self, scalar other); +void atg_lt_(tensor *, tensor self, scalar other); +void atg_lt_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_lt_tensor(tensor *, tensor self, tensor other); +void atg_lt_tensor_(tensor *, tensor self, tensor other); +void atg_lt_tensor_out(tensor *, tensor out, tensor self, tensor other); +void atg_lu_solve(tensor *, tensor self, tensor LU_data, tensor LU_pivots); +void atg_lu_solve_out(tensor *, tensor out, tensor self, tensor LU_data, tensor LU_pivots); +void atg_lu_unpack(tensor *, tensor LU_data, tensor LU_pivots, int unpack_data, int unpack_pivots); +void atg_lu_unpack_out(tensor *, tensor P, tensor L, tensor U, tensor LU_data, tensor LU_pivots, int unpack_data, int unpack_pivots); +void atg_margin_ranking_loss(tensor *, tensor input1, tensor input2, tensor target, double margin, int64_t reduction); +void atg_masked_fill(tensor *, tensor self, tensor mask, scalar value); +void atg_masked_fill_(tensor *, tensor self, tensor mask, scalar value); +void atg_masked_fill_scalar_out(tensor *, tensor out, tensor self, tensor mask, scalar value); +void atg_masked_fill_tensor(tensor *, tensor self, tensor mask, tensor value); +void atg_masked_fill_tensor_(tensor *, tensor self, tensor mask, tensor value); +void atg_masked_fill_tensor_out(tensor *, tensor out, tensor self, tensor mask, tensor value); +void atg_masked_scatter(tensor *, tensor self, tensor mask, tensor source); +void atg_masked_scatter_(tensor *, tensor self, tensor mask, tensor source); +void atg_masked_scatter_backward(tensor *, tensor grad_output, tensor mask, int64_t *sizes_data, int sizes_len); +void atg_masked_scatter_out(tensor *, tensor out, tensor self, tensor mask, tensor source); +void atg_masked_select(tensor *, tensor self, tensor mask); +void atg_masked_select_backward(tensor *, tensor grad, tensor input, tensor mask); +void atg_masked_select_out(tensor *, tensor out, tensor self, tensor mask); +void atg_matmul(tensor *, tensor self, tensor other); +void atg_matmul_out(tensor *, tensor out, tensor self, tensor other); +void atg_matrix_exp(tensor *, tensor self); +void atg_matrix_exp_backward(tensor *, tensor self, tensor grad); +void atg_matrix_h(tensor *, tensor self); +void atg_matrix_power(tensor *, tensor self, int64_t n); +void atg_matrix_power_out(tensor *, tensor out, tensor self, int64_t n); +void atg_max(tensor *, tensor self); +void atg_max_dim(tensor *, tensor self, int64_t dim, int keepdim); +void atg_max_dim_max(tensor *, tensor max, tensor max_values, tensor self, int64_t dim, int keepdim); +void atg_max_other(tensor *, tensor self, tensor other); +void atg_max_out(tensor *, tensor out, tensor self, tensor other); +void atg_max_pool1d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_max_pool1d_with_indices(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_max_pool2d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_max_pool2d_backward(tensor *, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_max_pool2d_backward_out(tensor *, tensor out, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_max_pool2d_with_indices(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_max_pool2d_with_indices_backward(tensor *, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode, tensor indices); +void atg_max_pool2d_with_indices_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode, tensor indices); +void atg_max_pool2d_with_indices_out(tensor *, tensor out, tensor indices, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_max_pool3d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_max_pool3d_with_indices(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_max_pool3d_with_indices_backward(tensor *, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode, tensor indices); +void atg_max_pool3d_with_indices_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode, tensor indices); +void atg_max_pool3d_with_indices_out(tensor *, tensor out, tensor indices, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_max_unary_out(tensor *, tensor out, tensor self); +void atg_max_unpool2d(tensor *, tensor self, tensor indices, int64_t *output_size_data, int output_size_len); +void atg_max_unpool2d_out(tensor *, tensor out, tensor self, tensor indices, int64_t *output_size_data, int output_size_len); +void atg_max_unpool3d(tensor *, tensor self, tensor indices, int64_t *output_size_data, int output_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len); +void atg_max_unpool3d_out(tensor *, tensor out, tensor self, tensor indices, int64_t *output_size_data, int output_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len); +void atg_maximum(tensor *, tensor self, tensor other); +void atg_maximum_out(tensor *, tensor out, tensor self, tensor other); +void atg_mean(tensor *, tensor self, int dtype); +void atg_mean_dim(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); +void atg_mean_dtype_out(tensor *, tensor out, tensor self, int dtype); +void atg_mean_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); +void atg_median(tensor *, tensor self); +void atg_median_dim(tensor *, tensor self, int64_t dim, int keepdim); +void atg_median_dim_values(tensor *, tensor values, tensor indices, tensor self, int64_t dim, int keepdim); +void atg_median_out(tensor *, tensor out, tensor self); +tensor *atg_meshgrid(tensor *tensors_data, int tensors_len); +tensor *atg_meshgrid_indexing(tensor *tensors_data, int tensors_len, char* indexing_ptr, int indexing_len); +void atg_mh(tensor *, tensor self); +void atg_min(tensor *, tensor self); +void atg_min_dim(tensor *, tensor self, int64_t dim, int keepdim); +void atg_min_dim_min(tensor *, tensor min, tensor min_indices, tensor self, int64_t dim, int keepdim); +void atg_min_other(tensor *, tensor self, tensor other); +void atg_min_out(tensor *, tensor out, tensor self, tensor other); +void atg_min_unary_out(tensor *, tensor out, tensor self); +void atg_minimum(tensor *, tensor self, tensor other); +void atg_minimum_out(tensor *, tensor out, tensor self, tensor other); +void atg_miopen_batch_norm(tensor *, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double exponential_average_factor, double epsilon); +void atg_miopen_batch_norm_backward(tensor *, tensor input, tensor grad_output, tensor weight, tensor running_mean, tensor running_var, tensor save_mean, tensor save_var, double epsilon); +void atg_miopen_batch_norm_backward_out(tensor *, tensor out0, tensor out1, tensor out2, tensor input, tensor grad_output, tensor weight, tensor running_mean, tensor running_var, tensor save_mean, tensor save_var, double epsilon); +void atg_miopen_batch_norm_out(tensor *, tensor out0, tensor out1, tensor out2, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double exponential_average_factor, double epsilon); +void atg_miopen_convolution(tensor *, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic); +void atg_miopen_convolution_add_relu(tensor *, tensor self, tensor weight, tensor z, scalar alpha, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); +void atg_miopen_convolution_out(tensor *, tensor out, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic); +void atg_miopen_convolution_relu(tensor *, tensor self, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); +void atg_miopen_convolution_transpose(tensor *, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic); +void atg_miopen_convolution_transpose_out(tensor *, tensor out, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic); +void atg_miopen_ctc_loss(tensor *, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int deterministic, int zero_infinity); +void atg_miopen_ctc_loss_out(tensor *, tensor out0, tensor out1, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int deterministic, int zero_infinity); +void atg_miopen_ctc_loss_tensor(tensor *, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank, int deterministic, int zero_infinity); +void atg_miopen_depthwise_convolution(tensor *, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic); +void atg_miopen_depthwise_convolution_out(tensor *, tensor out, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic); +void atg_miopen_rnn(tensor *, tensor input, tensor *weight_data, int weight_len, int64_t weight_stride0, tensor hx, tensor cx, int64_t mode, int64_t hidden_size, int64_t num_layers, int batch_first, double dropout, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, tensor dropout_state); +void atg_miopen_rnn_out(tensor *, tensor out0, tensor out1, tensor out2, tensor out3, tensor out4, tensor input, tensor *weight_data, int weight_len, int64_t weight_stride0, tensor hx, tensor cx, int64_t mode, int64_t hidden_size, int64_t num_layers, int batch_first, double dropout, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, tensor dropout_state); +void atg_mish(tensor *, tensor self); +void atg_mish_(tensor *, tensor self); +void atg_mish_backward(tensor *, tensor grad_output, tensor self); +void atg_mish_out(tensor *, tensor out, tensor self); +void atg_mkldnn_adaptive_avg_pool2d(tensor *, tensor self, int64_t *output_size_data, int output_size_len); +void atg_mkldnn_adaptive_avg_pool2d_backward(tensor *, tensor grad_output, tensor self); +void atg_mkldnn_adaptive_avg_pool2d_backward_out(tensor *, tensor out, tensor grad_output, tensor self); +void atg_mkldnn_adaptive_avg_pool2d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len); +void atg_mkldnn_convolution(tensor *, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups); +void atg_mkldnn_convolution_out(tensor *, tensor out, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups); +void atg_mkldnn_linear(tensor *, tensor self, tensor weight, tensor bias); +void atg_mkldnn_linear_backward_input(tensor *, int64_t *input_size_data, int input_size_len, tensor grad_output, tensor weight); +void atg_mkldnn_linear_backward_input_out(tensor *, tensor out, int64_t *input_size_data, int input_size_len, tensor grad_output, tensor weight); +void atg_mkldnn_linear_backward_weights(tensor *, tensor grad_output, tensor input, tensor weight, int bias_defined); +void atg_mkldnn_linear_backward_weights_out(tensor *, tensor out0, tensor out1, tensor grad_output, tensor input, tensor weight, int bias_defined); +void atg_mkldnn_linear_out(tensor *, tensor out, tensor self, tensor weight, tensor bias); +void atg_mkldnn_max_pool2d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_mkldnn_max_pool2d_backward(tensor *, tensor grad_output, tensor output, tensor input, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_mkldnn_max_pool2d_backward_out(tensor *, tensor out, tensor grad_output, tensor output, tensor input, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_mkldnn_max_pool2d_out(tensor *, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_mkldnn_max_pool3d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_mkldnn_max_pool3d_backward(tensor *, tensor grad_output, tensor output, tensor input, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_mkldnn_max_pool3d_backward_out(tensor *, tensor out, tensor grad_output, tensor output, tensor input, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_mkldnn_max_pool3d_out(tensor *, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_mkldnn_reorder_conv2d_weight(tensor *, tensor self, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int64_t *input_size_data, int input_size_len); +void atg_mkldnn_reorder_conv2d_weight_out(tensor *, tensor out, tensor self, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int64_t *input_size_data, int input_size_len); +void atg_mkldnn_reorder_conv3d_weight(tensor *, tensor self, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int64_t *input_size_data, int input_size_len); +void atg_mkldnn_reorder_conv3d_weight_out(tensor *, tensor out, tensor self, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int64_t *input_size_data, int input_size_len); +void atg_mkldnn_rnn_layer(tensor *, tensor input, tensor weight0, tensor weight1, tensor weight2, tensor weight3, tensor hx_, tensor cx_, int reverse, int64_t *batch_sizes_data, int batch_sizes_len, int64_t mode, int64_t hidden_size, int64_t num_layers, int has_biases, int bidirectional, int batch_first, int train); +void atg_mkldnn_rnn_layer_backward(tensor *, tensor input, tensor weight1, tensor weight2, tensor weight3, tensor weight4, tensor hx_, tensor cx_tmp, tensor output, tensor hy_, tensor cy_, tensor grad_output, tensor grad_hy, tensor grad_cy, int reverse, int64_t mode, int64_t hidden_size, int64_t num_layers, int has_biases, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, int batch_first, tensor workspace); +void atg_mkldnn_rnn_layer_backward_out(tensor *, tensor out0, tensor out1, tensor out2, tensor out3, tensor out4, tensor out5, tensor out6, tensor input, tensor weight1, tensor weight2, tensor weight3, tensor weight4, tensor hx_, tensor cx_tmp, tensor output, tensor hy_, tensor cy_, tensor grad_output, tensor grad_hy, tensor grad_cy, int reverse, int64_t mode, int64_t hidden_size, int64_t num_layers, int has_biases, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, int batch_first, tensor workspace); +void atg_mkldnn_rnn_layer_out(tensor *, tensor out0, tensor out1, tensor out2, tensor out3, tensor input, tensor weight0, tensor weight1, tensor weight2, tensor weight3, tensor hx_, tensor cx_, int reverse, int64_t *batch_sizes_data, int batch_sizes_len, int64_t mode, int64_t hidden_size, int64_t num_layers, int has_biases, int bidirectional, int batch_first, int train); +void atg_mm(tensor *, tensor self, tensor mat2); +void atg_mm_dtype(tensor *, tensor self, tensor mat2, int out_dtype); +void atg_mm_dtype_out(tensor *, tensor out, tensor self, tensor mat2, int out_dtype); +void atg_mm_out(tensor *, tensor out, tensor self, tensor mat2); +void atg_mode(tensor *, tensor self, int64_t dim, int keepdim); +void atg_mode_values(tensor *, tensor values, tensor indices, tensor self, int64_t dim, int keepdim); +void atg_moveaxis(tensor *, tensor self, int64_t *source_data, int source_len, int64_t *destination_data, int destination_len); +void atg_moveaxis_int(tensor *, tensor self, int64_t source, int64_t destination); +void atg_movedim(tensor *, tensor self, int64_t *source_data, int source_len, int64_t *destination_data, int destination_len); +void atg_movedim_int(tensor *, tensor self, int64_t source, int64_t destination); +void atg_mse_loss(tensor *, tensor self, tensor target, int64_t reduction); +void atg_mse_loss_backward(tensor *, tensor grad_output, tensor self, tensor target, int64_t reduction); +void atg_mse_loss_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor target, int64_t reduction); +void atg_mse_loss_out(tensor *, tensor out, tensor self, tensor target, int64_t reduction); +void atg_msort(tensor *, tensor self); +void atg_msort_out(tensor *, tensor out, tensor self); +void atg_mt(tensor *, tensor self); +void atg_mul(tensor *, tensor self, tensor other); +void atg_mul_(tensor *, tensor self, tensor other); +void atg_mul_out(tensor *, tensor out, tensor self, tensor other); +void atg_mul_scalar(tensor *, tensor self, scalar other); +void atg_mul_scalar_(tensor *, tensor self, scalar other); +void atg_mul_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_multi_margin_loss_backward(tensor *, tensor grad_output, tensor self, tensor target, scalar p, scalar margin, tensor weight, int64_t reduction); +void atg_multi_margin_loss_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor target, scalar p, scalar margin, tensor weight, int64_t reduction); +void atg_multilabel_margin_loss(tensor *, tensor self, tensor target, int64_t reduction); +void atg_multilabel_margin_loss_backward(tensor *, tensor grad_output, tensor self, tensor target, int64_t reduction, tensor is_target); +void atg_multilabel_margin_loss_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor target, int64_t reduction, tensor is_target); +void atg_multilabel_margin_loss_out(tensor *, tensor out, tensor self, tensor target, int64_t reduction); +void atg_multinomial(tensor *, tensor self, int64_t num_samples, int replacement); +void atg_multinomial_out(tensor *, tensor out, tensor self, int64_t num_samples, int replacement); +void atg_multiply(tensor *, tensor self, tensor other); +void atg_multiply_(tensor *, tensor self, tensor other); +void atg_multiply_out(tensor *, tensor out, tensor self, tensor other); +void atg_multiply_scalar(tensor *, tensor self, scalar other); +void atg_multiply_scalar_(tensor *, tensor self, scalar other); +void atg_mv(tensor *, tensor self, tensor vec); +void atg_mv_out(tensor *, tensor out, tensor self, tensor vec); +void atg_mvlgamma(tensor *, tensor self, int64_t p); +void atg_mvlgamma_(tensor *, tensor self, int64_t p); +void atg_mvlgamma_out(tensor *, tensor out, tensor self, int64_t p); +void atg_nan_to_num(tensor *, tensor self, double nan_v, uint8_t nan_null, double posinf_v, uint8_t posinf_null, double neginf_v, uint8_t neginf_null); +void atg_nan_to_num_(tensor *, tensor self, double nan_v, uint8_t nan_null, double posinf_v, uint8_t posinf_null, double neginf_v, uint8_t neginf_null); +void atg_nan_to_num_out(tensor *, tensor out, tensor self, double nan_v, uint8_t nan_null, double posinf_v, uint8_t posinf_null, double neginf_v, uint8_t neginf_null); +void atg_nanmean(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); +void atg_nanmean_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); +void atg_nanmedian(tensor *, tensor self); +void atg_nanmedian_dim(tensor *, tensor self, int64_t dim, int keepdim); +void atg_nanmedian_dim_values(tensor *, tensor values, tensor indices, tensor self, int64_t dim, int keepdim); +void atg_nanmedian_out(tensor *, tensor out, tensor self); +void atg_nanquantile(tensor *, tensor self, tensor q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len); +void atg_nanquantile_out(tensor *, tensor out, tensor self, tensor q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len); +void atg_nanquantile_scalar(tensor *, tensor self, double q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len); +void atg_nanquantile_scalar_out(tensor *, tensor out, tensor self, double q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len); +void atg_nansum(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); +void atg_nansum_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); +void atg_narrow(tensor *, tensor self, int64_t dim, int64_t start, int64_t length); +void atg_narrow_copy(tensor *, tensor self, int64_t dim, int64_t start, int64_t length); +void atg_narrow_copy_out(tensor *, tensor out, tensor self, int64_t dim, int64_t start, int64_t length); +void atg_narrow_tensor(tensor *, tensor self, int64_t dim, tensor start, int64_t length); +void atg_native_batch_norm(tensor *, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps); +void atg_native_batch_norm_out(tensor *, tensor out, tensor save_mean, tensor save_invstd, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps); +void atg_native_channel_shuffle(tensor *, tensor self, int64_t groups); +void atg_native_dropout(tensor *, tensor input, double p, int train); +void atg_native_dropout_backward(tensor *, tensor grad_output, tensor mask, double scale); +void atg_native_dropout_backward_out(tensor *, tensor out, tensor grad_output, tensor mask, double scale); +void atg_native_dropout_out(tensor *, tensor out0, tensor out1, tensor input, double p, int train); +void atg_native_group_norm(tensor *, tensor input, tensor weight, tensor bias, int64_t n, int64_t C, int64_t HxW, int64_t group, double eps); +void atg_native_group_norm_out(tensor *, tensor out0, tensor out1, tensor out2, tensor input, tensor weight, tensor bias, int64_t n, int64_t C, int64_t HxW, int64_t group, double eps); +void atg_native_layer_norm(tensor *, tensor input, int64_t *normalized_shape_data, int normalized_shape_len, tensor weight, tensor bias, double eps); +void atg_native_layer_norm_out(tensor *, tensor out0, tensor out1, tensor out2, tensor input, int64_t *normalized_shape_data, int normalized_shape_len, tensor weight, tensor bias, double eps); +void atg_native_norm(tensor *, tensor self); +void atg_native_norm_out(tensor *, tensor out, tensor self); +void atg_native_norm_scalaropt_dim_dtype(tensor *, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim, int dtype); +void atg_native_norm_scalaropt_dim_dtype_out(tensor *, tensor out, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim, int dtype); +void atg_ne(tensor *, tensor self, scalar other); +void atg_ne_(tensor *, tensor self, scalar other); +void atg_ne_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_ne_tensor(tensor *, tensor self, tensor other); +void atg_ne_tensor_(tensor *, tensor self, tensor other); +void atg_ne_tensor_out(tensor *, tensor out, tensor self, tensor other); +void atg_neg(tensor *, tensor self); +void atg_neg_(tensor *, tensor self); +void atg_neg_out(tensor *, tensor out, tensor self); +void atg_negative(tensor *, tensor self); +void atg_negative_(tensor *, tensor self); +void atg_negative_out(tensor *, tensor out, tensor self); +void atg_nested_to_padded_tensor(tensor *, tensor self, double padding, int64_t *output_size_data, int output_size_len); +void atg_new_empty(tensor *, tensor self, int64_t *size_data, int size_len, int options_kind, int options_device); +void atg_new_empty_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len); +void atg_new_empty_strided(tensor *, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int options_kind, int options_device); +void atg_new_empty_strided_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len); +void atg_new_full(tensor *, tensor self, int64_t *size_data, int size_len, scalar fill_value, int options_kind, int options_device); +void atg_new_full_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len, scalar fill_value); +void atg_new_ones(tensor *, tensor self, int64_t *size_data, int size_len, int options_kind, int options_device); +void atg_new_ones_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len); +void atg_new_zeros(tensor *, tensor self, int64_t *size_data, int size_len, int options_kind, int options_device); +void atg_new_zeros_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len); +void atg_nextafter(tensor *, tensor self, tensor other); +void atg_nextafter_(tensor *, tensor self, tensor other); +void atg_nextafter_out(tensor *, tensor out, tensor self, tensor other); +void atg_nll_loss(tensor *, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index); +void atg_nll_loss2d(tensor *, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index); +void atg_nll_loss2d_backward(tensor *, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index, tensor total_weight); +void atg_nll_loss2d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index, tensor total_weight); +void atg_nll_loss2d_out(tensor *, tensor out, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index); +void atg_nll_loss_backward(tensor *, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index, tensor total_weight); +void atg_nll_loss_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index, tensor total_weight); +void atg_nll_loss_nd(tensor *, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index); +void atg_nll_loss_out(tensor *, tensor out, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index); +void atg_nonzero(tensor *, tensor self); +tensor *atg_nonzero_numpy(tensor self); +void atg_nonzero_out(tensor *, tensor out, tensor self); +void atg_nonzero_static(tensor *, tensor self, int64_t size, int64_t fill_value); +void atg_nonzero_static_out(tensor *, tensor out, tensor self, int64_t size, int64_t fill_value); +void atg_norm(tensor *, tensor self); +void atg_norm_dtype_out(tensor *, tensor out, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim, int dtype); +void atg_norm_except_dim(tensor *, tensor v, int64_t pow, int64_t dim); +void atg_norm_out(tensor *, tensor out, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim); +void atg_norm_scalar_out(tensor *, tensor out, tensor self); +void atg_norm_scalaropt_dim(tensor *, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim); +void atg_norm_scalaropt_dim_dtype(tensor *, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim, int dtype); +void atg_norm_scalaropt_dtype(tensor *, tensor self, scalar p, int dtype); +void atg_norm_scalaropt_dtype_out(tensor *, tensor out, tensor self, scalar p, int dtype); +void atg_normal_(tensor *, tensor self, double mean, double std); +void atg_normal_functional(tensor *, tensor self, double mean, double std); +void atg_not_equal(tensor *, tensor self, scalar other); +void atg_not_equal_(tensor *, tensor self, scalar other); +void atg_not_equal_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_not_equal_tensor(tensor *, tensor self, tensor other); +void atg_not_equal_tensor_(tensor *, tensor self, tensor other); +void atg_not_equal_tensor_out(tensor *, tensor out, tensor self, tensor other); +void atg_nuclear_norm(tensor *, tensor self, int keepdim); +void atg_nuclear_norm_dim(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim); +void atg_nuclear_norm_dim_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim); +void atg_nuclear_norm_out(tensor *, tensor out, tensor self, int keepdim); +void atg_numpy_t(tensor *, tensor self); +void atg_one_hot(tensor *, tensor self, int64_t num_classes); +void atg_ones(tensor *, int64_t *size_data, int size_len, int options_kind, int options_device); +void atg_ones_like(tensor *, tensor self); +void atg_ones_like_out(tensor *, tensor out, tensor self); +void atg_ones_out(tensor *, tensor out, int64_t *size_data, int size_len); +void atg_orgqr(tensor *, tensor self, tensor input2); +void atg_orgqr_out(tensor *, tensor out, tensor self, tensor input2); +void atg_ormqr(tensor *, tensor self, tensor input2, tensor input3, int left, int transpose); +void atg_ormqr_out(tensor *, tensor out, tensor self, tensor input2, tensor input3, int left, int transpose); +void atg_outer(tensor *, tensor self, tensor vec2); +void atg_outer_out(tensor *, tensor out, tensor self, tensor vec2); +int64_t atg_output_nr(tensor self); +void atg_pad(tensor *, tensor self, int64_t *pad_data, int pad_len, char* mode_ptr, int mode_len, double value_v, uint8_t value_null); +void atg_pad_sequence(tensor *, tensor *sequences_data, int sequences_len, int batch_first, double padding_value, char* padding_side_ptr, int padding_side_len); +void atg_pairwise_distance(tensor *, tensor x1, tensor x2, double p, double eps, int keepdim); +void atg_pdist(tensor *, tensor self, double p); +void atg_permute(tensor *, tensor self, int64_t *dims_data, int dims_len); +void atg_permute_copy(tensor *, tensor self, int64_t *dims_data, int dims_len); +void atg_permute_copy_out(tensor *, tensor out, tensor self, int64_t *dims_data, int dims_len); +void atg_pin_memory(tensor *, tensor self, int device); +void atg_pinverse(tensor *, tensor self, double rcond); +void atg_pixel_shuffle(tensor *, tensor self, int64_t upscale_factor); +void atg_pixel_shuffle_out(tensor *, tensor out, tensor self, int64_t upscale_factor); +void atg_pixel_unshuffle(tensor *, tensor self, int64_t downscale_factor); +void atg_pixel_unshuffle_out(tensor *, tensor out, tensor self, int64_t downscale_factor); +void atg_poisson(tensor *, tensor self); +void atg_poisson_nll_loss(tensor *, tensor input, tensor target, int log_input, int full, double eps, int64_t reduction); +void atg_poisson_out(tensor *, tensor out, tensor self); +void atg_polar(tensor *, tensor abs, tensor angle); +void atg_polar_out(tensor *, tensor out, tensor abs, tensor angle); +void atg_polygamma(tensor *, int64_t n, tensor self); +void atg_polygamma_(tensor *, tensor self, int64_t n); +void atg_polygamma_out(tensor *, tensor out, int64_t n, tensor self); +void atg_positive(tensor *, tensor self); +void atg_pow(tensor *, tensor self, tensor exponent); +void atg_pow_(tensor *, tensor self, scalar exponent); +void atg_pow_scalar(tensor *, scalar self_scalar, tensor exponent); +void atg_pow_scalar_out(tensor *, tensor out, scalar self_scalar, tensor exponent); +void atg_pow_tensor_(tensor *, tensor self, tensor exponent); +void atg_pow_tensor_scalar(tensor *, tensor self, scalar exponent); +void atg_pow_tensor_scalar_out(tensor *, tensor out, tensor self, scalar exponent); +void atg_pow_tensor_tensor_out(tensor *, tensor out, tensor self, tensor exponent); +void atg_prelu(tensor *, tensor self, tensor weight); +void atg_prod(tensor *, tensor self, int dtype); +void atg_prod_dim_int(tensor *, tensor self, int64_t dim, int keepdim, int dtype); +void atg_prod_int_out(tensor *, tensor out, tensor self, int64_t dim, int keepdim, int dtype); +void atg_prod_out(tensor *, tensor out, tensor self, int dtype); +void atg_put(tensor *, tensor self, tensor index, tensor source, int accumulate); +void atg_put_(tensor *, tensor self, tensor index, tensor source, int accumulate); +void atg_put_out(tensor *, tensor out, tensor self, tensor index, tensor source, int accumulate); +int64_t atg_q_per_channel_axis(tensor self); +void atg_q_per_channel_scales(tensor *, tensor self); +void atg_q_per_channel_scales_out(tensor *, tensor out, tensor self); +void atg_q_per_channel_zero_points(tensor *, tensor self); +void atg_q_per_channel_zero_points_out(tensor *, tensor out, tensor self); +double atg_q_scale(tensor self); +int64_t atg_q_zero_point(tensor self); +void atg_qr(tensor *, tensor self, int some); +void atg_qr_q(tensor *, tensor Q, tensor R, tensor self, int some); +void atg_quantile(tensor *, tensor self, tensor q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len); +void atg_quantile_out(tensor *, tensor out, tensor self, tensor q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len); +void atg_quantile_scalar(tensor *, tensor self, double q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len); +void atg_quantile_scalar_out(tensor *, tensor out, tensor self, double q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len); +void atg_quantize_per_channel(tensor *, tensor self, tensor scales, tensor zero_points, int64_t axis, int dtype); +void atg_quantize_per_channel_out(tensor *, tensor out, tensor self, tensor scales, tensor zero_points, int64_t axis, int dtype); +void atg_quantize_per_tensor(tensor *, tensor self, double scale, int64_t zero_point, int dtype); +void atg_quantize_per_tensor_dynamic(tensor *, tensor self, int dtype, int reduce_range); +void atg_quantize_per_tensor_dynamic_out(tensor *, tensor out, tensor self, int dtype, int reduce_range); +void atg_quantize_per_tensor_out(tensor *, tensor out, tensor self, double scale, int64_t zero_point, int dtype); +void atg_quantize_per_tensor_tensor_qparams(tensor *, tensor self, tensor scale, tensor zero_point, int dtype); +void atg_quantize_per_tensor_tensor_qparams_out(tensor *, tensor out, tensor self, tensor scale, tensor zero_point, int dtype); +tensor *atg_quantize_per_tensor_tensors(tensor *tensors_data, int tensors_len, tensor scales, tensor zero_points, int dtype); +void atg_quantize_per_tensor_tensors_out(tensor *out_data, int out_len, tensor *tensors_data, int tensors_len, tensor scales, tensor zero_points, int dtype); +void atg_quantized_batch_norm(tensor *, tensor input, tensor weight, tensor bias, tensor mean, tensor var, double eps, double output_scale, int64_t output_zero_point); +void atg_quantized_batch_norm_out(tensor *, tensor out, tensor input, tensor weight, tensor bias, tensor mean, tensor var, double eps, double output_scale, int64_t output_zero_point); +void atg_quantized_gru_cell(tensor *, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh, tensor packed_ih, tensor packed_hh, tensor col_offsets_ih, tensor col_offsets_hh, scalar scale_ih, scalar scale_hh, scalar zero_point_ih, scalar zero_point_hh); +void atg_quantized_lstm_cell(tensor *, tensor input, tensor *hx_data, int hx_len, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh, tensor packed_ih, tensor packed_hh, tensor col_offsets_ih, tensor col_offsets_hh, scalar scale_ih, scalar scale_hh, scalar zero_point_ih, scalar zero_point_hh); +void atg_quantized_max_pool1d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_quantized_max_pool1d_out(tensor *, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_quantized_max_pool2d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_quantized_max_pool2d_out(tensor *, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_quantized_max_pool3d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_quantized_max_pool3d_out(tensor *, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); +void atg_quantized_rnn_relu_cell(tensor *, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh, tensor packed_ih, tensor packed_hh, tensor col_offsets_ih, tensor col_offsets_hh, scalar scale_ih, scalar scale_hh, scalar zero_point_ih, scalar zero_point_hh); +void atg_quantized_rnn_tanh_cell(tensor *, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh, tensor packed_ih, tensor packed_hh, tensor col_offsets_ih, tensor col_offsets_hh, scalar scale_ih, scalar scale_hh, scalar zero_point_ih, scalar zero_point_hh); +void atg_rad2deg(tensor *, tensor self); +void atg_rad2deg_(tensor *, tensor self); +void atg_rad2deg_out(tensor *, tensor out, tensor self); +void atg_rand(tensor *, int64_t *size_data, int size_len, int options_kind, int options_device); +void atg_rand_like(tensor *, tensor self); +void atg_rand_like_out(tensor *, tensor out, tensor self); +void atg_rand_out(tensor *, tensor out, int64_t *size_data, int size_len); +void atg_randint(tensor *, int64_t high, int64_t *size_data, int size_len, int options_kind, int options_device); +void atg_randint_like(tensor *, tensor self, int64_t high); +void atg_randint_like_low_dtype(tensor *, tensor self, int64_t low, int64_t high); +void atg_randint_like_low_dtype_out(tensor *, tensor out, tensor self, int64_t low, int64_t high); +void atg_randint_like_out(tensor *, tensor out, tensor self, int64_t high); +void atg_randint_like_tensor(tensor *, tensor self, tensor high); +void atg_randint_like_tensor_out(tensor *, tensor out, tensor self, tensor high); +void atg_randint_low(tensor *, int64_t low, int64_t high, int64_t *size_data, int size_len, int options_kind, int options_device); +void atg_randint_low_out(tensor *, tensor out, int64_t low, int64_t high, int64_t *size_data, int size_len); +void atg_randint_out(tensor *, tensor out, int64_t high, int64_t *size_data, int size_len); +void atg_randn(tensor *, int64_t *size_data, int size_len, int options_kind, int options_device); +void atg_randn_like(tensor *, tensor self); +void atg_randn_like_out(tensor *, tensor out, tensor self); +void atg_randn_out(tensor *, tensor out, int64_t *size_data, int size_len); +void atg_random(tensor *, tensor self); +void atg_random_(tensor *, tensor self); +void atg_random_from(tensor *, tensor self, int64_t from, int64_t to_v, uint8_t to_null); +void atg_random_from_(tensor *, tensor self, int64_t from, int64_t to_v, uint8_t to_null); +void atg_random_from_out(tensor *, tensor out, tensor self, int64_t from, int64_t to_v, uint8_t to_null); +void atg_random_out(tensor *, tensor out, tensor self); +void atg_random_to(tensor *, tensor self, int64_t to); +void atg_random_to_(tensor *, tensor self, int64_t to); +void atg_random_to_out(tensor *, tensor out, tensor self, int64_t to); +void atg_randperm(tensor *, int64_t n, int options_kind, int options_device); +void atg_randperm_out(tensor *, tensor out, int64_t n); +void atg_range(tensor *, scalar start, scalar end, int options_kind, int options_device); +void atg_range_out(tensor *, tensor out, scalar start, scalar end); +void atg_range_out_(tensor *, tensor out, scalar start, scalar end); +void atg_range_step(tensor *, scalar start, scalar end, int options_kind, int options_device); +void atg_ravel(tensor *, tensor self); +void atg_real(tensor *, tensor self); +void atg_reciprocal(tensor *, tensor self); +void atg_reciprocal_(tensor *, tensor self); +void atg_reciprocal_out(tensor *, tensor out, tensor self); +void atg_reflection_pad1d(tensor *, tensor self, int64_t *padding_data, int padding_len); +void atg_reflection_pad1d_backward(tensor *, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); +void atg_reflection_pad1d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); +void atg_reflection_pad1d_out(tensor *, tensor out, tensor self, int64_t *padding_data, int padding_len); +void atg_reflection_pad2d(tensor *, tensor self, int64_t *padding_data, int padding_len); +void atg_reflection_pad2d_backward(tensor *, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); +void atg_reflection_pad2d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); +void atg_reflection_pad2d_out(tensor *, tensor out, tensor self, int64_t *padding_data, int padding_len); +void atg_reflection_pad3d(tensor *, tensor self, int64_t *padding_data, int padding_len); +void atg_reflection_pad3d_backward(tensor *, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); +void atg_reflection_pad3d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); +void atg_reflection_pad3d_out(tensor *, tensor out, tensor self, int64_t *padding_data, int padding_len); +void atg_relu(tensor *, tensor self); +void atg_relu6(tensor *, tensor self); +void atg_relu6_(tensor *, tensor self); +void atg_relu_(tensor *, tensor self); +void atg_relu_out(tensor *, tensor out, tensor self); +void atg_remainder(tensor *, tensor self, scalar other); +void atg_remainder_(tensor *, tensor self, scalar other); +void atg_remainder_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_remainder_scalar_tensor(tensor *, scalar self_scalar, tensor other); +void atg_remainder_scalar_tensor_out(tensor *, tensor out, scalar self_scalar, tensor other); +void atg_remainder_tensor(tensor *, tensor self, tensor other); +void atg_remainder_tensor_(tensor *, tensor self, tensor other); +void atg_remainder_tensor_out(tensor *, tensor out, tensor self, tensor other); +void atg_renorm(tensor *, tensor self, scalar p, int64_t dim, scalar maxnorm); +void atg_renorm_(tensor *, tensor self, scalar p, int64_t dim, scalar maxnorm); +void atg_renorm_out(tensor *, tensor out, tensor self, scalar p, int64_t dim, scalar maxnorm); +void atg_repeat(tensor *, tensor self, int64_t *repeats_data, int repeats_len); +void atg_repeat_interleave(tensor *, tensor repeats, int64_t output_size_v, uint8_t output_size_null); +void atg_repeat_interleave_self_int(tensor *, tensor self, int64_t repeats, int64_t dim_v, uint8_t dim_null, int64_t output_size_v, uint8_t output_size_null); +void atg_repeat_interleave_self_tensor(tensor *, tensor self, tensor repeats, int64_t dim_v, uint8_t dim_null, int64_t output_size_v, uint8_t output_size_null); +void atg_repeat_interleave_tensor_out(tensor *, tensor out, tensor repeats, int64_t output_size_v, uint8_t output_size_null); +void atg_repeat_out(tensor *, tensor out, tensor self, int64_t *repeats_data, int repeats_len); +void atg_replication_pad1d(tensor *, tensor self, int64_t *padding_data, int padding_len); +void atg_replication_pad1d_backward(tensor *, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); +void atg_replication_pad1d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); +void atg_replication_pad1d_out(tensor *, tensor out, tensor self, int64_t *padding_data, int padding_len); +void atg_replication_pad2d(tensor *, tensor self, int64_t *padding_data, int padding_len); +void atg_replication_pad2d_backward(tensor *, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); +void atg_replication_pad2d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); +void atg_replication_pad2d_out(tensor *, tensor out, tensor self, int64_t *padding_data, int padding_len); +void atg_replication_pad3d(tensor *, tensor self, int64_t *padding_data, int padding_len); +void atg_replication_pad3d_backward(tensor *, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); +void atg_replication_pad3d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); +void atg_replication_pad3d_out(tensor *, tensor out, tensor self, int64_t *padding_data, int padding_len); +void atg_requires_grad_(tensor *, tensor self, int requires_grad); +void atg_reshape(tensor *, tensor self, int64_t *shape_data, int shape_len); +void atg_reshape_as(tensor *, tensor self, tensor other); +void atg_resize(tensor *, tensor self, int64_t *size_data, int size_len); +void atg_resize_(tensor *, tensor self, int64_t *size_data, int size_len); +void atg_resize_as(tensor *, tensor self, tensor the_template); +void atg_resize_as_(tensor *, tensor self, tensor the_template); +void atg_resize_as_out(tensor *, tensor out, tensor self, tensor the_template); +void atg_resize_as_sparse(tensor *, tensor self, tensor the_template); +void atg_resize_as_sparse_(tensor *, tensor self, tensor the_template); +void atg_resize_as_sparse_out(tensor *, tensor out, tensor self, tensor the_template); +void atg_resize_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len); +void atg_resolve_conj(tensor *, tensor self); +void atg_resolve_neg(tensor *, tensor self); +int atg_retains_grad(tensor self); +void atg_rms_norm(tensor *, tensor input, int64_t *normalized_shape_data, int normalized_shape_len, tensor weight, double eps_v, uint8_t eps_null); +void atg_rnn_relu(tensor *, tensor input, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first); +void atg_rnn_relu_cell(tensor *, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh); +void atg_rnn_relu_data(tensor *, tensor data, tensor batch_sizes, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional); +void atg_rnn_tanh(tensor *, tensor input, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first); +void atg_rnn_tanh_cell(tensor *, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh); +void atg_rnn_tanh_data(tensor *, tensor data, tensor batch_sizes, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional); +void atg_roll(tensor *, tensor self, int64_t *shifts_data, int shifts_len, int64_t *dims_data, int dims_len); +void atg_roll_out(tensor *, tensor out, tensor self, int64_t *shifts_data, int shifts_len, int64_t *dims_data, int dims_len); +void atg_rot90(tensor *, tensor self, int64_t k, int64_t *dims_data, int dims_len); +void atg_rot90_out(tensor *, tensor out, tensor self, int64_t k, int64_t *dims_data, int dims_len); +void atg_round(tensor *, tensor self); +void atg_round_(tensor *, tensor self); +void atg_round_decimals(tensor *, tensor self, int64_t decimals); +void atg_round_decimals_(tensor *, tensor self, int64_t decimals); +void atg_round_decimals_out(tensor *, tensor out, tensor self, int64_t decimals); +void atg_round_out(tensor *, tensor out, tensor self); +void atg_row_indices(tensor *, tensor self); +void atg_row_indices_copy(tensor *, tensor self); +void atg_row_indices_copy_out(tensor *, tensor out, tensor self); +void atg_row_stack(tensor *, tensor *tensors_data, int tensors_len); +void atg_row_stack_out(tensor *, tensor out, tensor *tensors_data, int tensors_len); +void atg_rrelu(tensor *, tensor self, int training); +void atg_rrelu_(tensor *, tensor self, int training); +void atg_rrelu_with_noise(tensor *, tensor self, tensor noise, int training); +void atg_rrelu_with_noise_(tensor *, tensor self, tensor noise, int training); +void atg_rrelu_with_noise_backward(tensor *, tensor grad_output, tensor self, tensor noise, scalar lower, scalar upper, int training, int self_is_result); +void atg_rrelu_with_noise_backward_out(tensor *, tensor out, tensor grad_output, tensor self, tensor noise, scalar lower, scalar upper, int training, int self_is_result); +void atg_rrelu_with_noise_functional(tensor *, tensor self, tensor noise, int training); +void atg_rrelu_with_noise_out(tensor *, tensor out, tensor self, tensor noise, int training); +void atg_rsqrt(tensor *, tensor self); +void atg_rsqrt_(tensor *, tensor self); +void atg_rsqrt_out(tensor *, tensor out, tensor self); +void atg_rsub(tensor *, tensor self, tensor other); +void atg_rsub_scalar(tensor *, tensor self, scalar other); +void atg_rsub_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_rsub_tensor_out(tensor *, tensor out, tensor self, tensor other); +void atg_scalar_tensor(tensor *, scalar s, int options_kind, int options_device); +void atg_scalar_tensor_out(tensor *, tensor out, scalar s); +void atg_scaled_dot_product_attention(tensor *, tensor query, tensor key, tensor value, tensor attn_mask, double dropout_p, int is_causal, double scale_v, uint8_t scale_null, int enable_gqa); +void atg_scatter(tensor *, tensor self, int64_t dim, tensor index, tensor src); +void atg_scatter_(tensor *, tensor self, int64_t dim, tensor index, tensor src); +void atg_scatter_add(tensor *, tensor self, int64_t dim, tensor index, tensor src); +void atg_scatter_add_(tensor *, tensor self, int64_t dim, tensor index, tensor src); +void atg_scatter_add_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, tensor src); +void atg_scatter_reduce(tensor *, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len); +void atg_scatter_reduce_(tensor *, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len); +void atg_scatter_reduce_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len); +void atg_scatter_src_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, tensor src); +void atg_scatter_value(tensor *, tensor self, int64_t dim, tensor index, scalar value); +void atg_scatter_value_(tensor *, tensor self, int64_t dim, tensor index, scalar value); +void atg_scatter_value_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, scalar value); +void atg_scatter_value_reduce(tensor *, tensor self, int64_t dim, tensor index, scalar value, char* reduce_ptr, int reduce_len); +void atg_scatter_value_reduce_(tensor *, tensor self, int64_t dim, tensor index, scalar value, char* reduce_ptr, int reduce_len); +void atg_scatter_value_reduce_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, scalar value, char* reduce_ptr, int reduce_len); +void atg_searchsorted(tensor *, tensor sorted_sequence, tensor self, int out_int32, int right, char* side_ptr, int side_len, tensor sorter); +void atg_searchsorted_scalar(tensor *, tensor sorted_sequence, scalar self_scalar, int out_int32, int right, char* side_ptr, int side_len, tensor sorter); +void atg_searchsorted_scalar_out(tensor *, tensor out, tensor sorted_sequence, scalar self_scalar, int out_int32, int right, char* side_ptr, int side_len, tensor sorter); +void atg_searchsorted_tensor_out(tensor *, tensor out, tensor sorted_sequence, tensor self, int out_int32, int right, char* side_ptr, int side_len, tensor sorter); +void atg_segment_reduce(tensor *, tensor data, char* reduce_ptr, int reduce_len, tensor lengths, tensor indices, tensor offsets, int64_t axis, int unsafe, scalar initial); +void atg_segment_reduce_out(tensor *, tensor out, tensor data, char* reduce_ptr, int reduce_len, tensor lengths, tensor indices, tensor offsets, int64_t axis, int unsafe, scalar initial); +void atg_select(tensor *, tensor self, int64_t dim, int64_t index); +void atg_select_backward(tensor *, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t index); +void atg_select_backward_out(tensor *, tensor out, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t index); +void atg_select_copy(tensor *, tensor self, int64_t dim, int64_t index); +void atg_select_copy_int_out(tensor *, tensor out, tensor self, int64_t dim, int64_t index); +void atg_select_scatter(tensor *, tensor self, tensor src, int64_t dim, int64_t index); +void atg_select_scatter_out(tensor *, tensor out, tensor self, tensor src, int64_t dim, int64_t index); +void atg_selu(tensor *, tensor self); +void atg_selu_(tensor *, tensor self); +void atg_set(tensor *, tensor self); +void atg_set_(tensor *, tensor self); +void atg_set_data(tensor self, tensor new_data); +void atg_set_out(tensor *, tensor out, tensor self); +void atg_set_requires_grad(tensor *, tensor self, int r); +void atg_set_source_tensor(tensor *, tensor self, tensor source); +void atg_set_source_tensor_(tensor *, tensor self, tensor source); +void atg_set_source_tensor_out(tensor *, tensor out, tensor self, tensor source); +void atg_set_source_tensor_storage_offset_(tensor *, tensor self, tensor source, int64_t storage_offset, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len); +void atg_sgn(tensor *, tensor self); +void atg_sgn_(tensor *, tensor self); +void atg_sgn_out(tensor *, tensor out, tensor self); +void atg_sigmoid(tensor *, tensor self); +void atg_sigmoid_(tensor *, tensor self); +void atg_sigmoid_backward(tensor *, tensor grad_output, tensor output); +void atg_sigmoid_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor output); +void atg_sigmoid_out(tensor *, tensor out, tensor self); +void atg_sign(tensor *, tensor self); +void atg_sign_(tensor *, tensor self); +void atg_sign_out(tensor *, tensor out, tensor self); +void atg_signbit(tensor *, tensor self); +void atg_signbit_out(tensor *, tensor out, tensor self); +void atg_silu(tensor *, tensor self); +void atg_silu_(tensor *, tensor self); +void atg_silu_backward(tensor *, tensor grad_output, tensor self); +void atg_silu_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self); +void atg_silu_out(tensor *, tensor out, tensor self); +void atg_sin(tensor *, tensor self); +void atg_sin_(tensor *, tensor self); +void atg_sin_out(tensor *, tensor out, tensor self); +void atg_sinc(tensor *, tensor self); +void atg_sinc_(tensor *, tensor self); +void atg_sinc_out(tensor *, tensor out, tensor self); +void atg_sinh(tensor *, tensor self); +void atg_sinh_(tensor *, tensor self); +void atg_sinh_out(tensor *, tensor out, tensor self); +void atg_slice(tensor *, tensor self, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step); +void atg_slice_backward(tensor *, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t start, int64_t end, int64_t step); +void atg_slice_backward_out(tensor *, tensor out, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t start, int64_t end, int64_t step); +void atg_slice_copy(tensor *, tensor self, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step); +void atg_slice_copy_tensor_out(tensor *, tensor out, tensor self, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step); +void atg_slice_inverse(tensor *, tensor self, tensor src, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step); +void atg_slice_scatter(tensor *, tensor self, tensor src, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step); +void atg_slice_scatter_out(tensor *, tensor out, tensor self, tensor src, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step); +void atg_slogdet(tensor *, tensor self); +void atg_slogdet_out(tensor *, tensor sign, tensor logabsdet, tensor self); +void atg_slow_conv3d(tensor *, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len); +void atg_slow_conv3d_out(tensor *, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len); +void atg_slow_conv_dilated2d(tensor *, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len); +void atg_slow_conv_dilated2d_out(tensor *, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len); +void atg_slow_conv_dilated3d(tensor *, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len); +void atg_slow_conv_dilated3d_out(tensor *, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len); +void atg_slow_conv_transpose2d(tensor *, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *dilation_data, int dilation_len); +void atg_slow_conv_transpose2d_out(tensor *, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *dilation_data, int dilation_len); +void atg_slow_conv_transpose3d(tensor *, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *dilation_data, int dilation_len); +void atg_slow_conv_transpose3d_out(tensor *, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *dilation_data, int dilation_len); +void atg_smm(tensor *, tensor self, tensor mat2); +void atg_smooth_l1_loss(tensor *, tensor self, tensor target, int64_t reduction, double beta); +void atg_smooth_l1_loss_backward(tensor *, tensor grad_output, tensor self, tensor target, int64_t reduction, double beta); +void atg_smooth_l1_loss_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor target, int64_t reduction, double beta); +void atg_smooth_l1_loss_out(tensor *, tensor out, tensor self, tensor target, int64_t reduction, double beta); +void atg_soft_margin_loss(tensor *, tensor self, tensor target, int64_t reduction); +void atg_soft_margin_loss_backward(tensor *, tensor grad_output, tensor self, tensor target, int64_t reduction); +void atg_soft_margin_loss_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor target, int64_t reduction); +void atg_soft_margin_loss_out(tensor *, tensor out, tensor self, tensor target, int64_t reduction); +void atg_softmax(tensor *, tensor self, int64_t dim, int dtype); +void atg_softmax_int_out(tensor *, tensor out, tensor self, int64_t dim, int dtype); +void atg_softplus(tensor *, tensor self); +void atg_softplus_backward(tensor *, tensor grad_output, tensor self, scalar beta, scalar threshold); +void atg_softplus_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, scalar beta, scalar threshold); +void atg_softplus_out(tensor *, tensor out, tensor self); +void atg_softshrink(tensor *, tensor self); +void atg_softshrink_backward(tensor *, tensor grad_output, tensor self, scalar lambd); +void atg_softshrink_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, scalar lambd); +void atg_softshrink_out(tensor *, tensor out, tensor self); +void atg_sort(tensor *, tensor self, int64_t dim, int descending); +void atg_sort_stable(tensor *, tensor self, int stable, int64_t dim, int descending); +void atg_sort_values(tensor *, tensor values, tensor indices, tensor self, int64_t dim, int descending); +void atg_sort_values_stable(tensor *, tensor values, tensor indices, tensor self, int stable, int64_t dim, int descending); +void atg_sparse_bsc_tensor(tensor *, tensor ccol_indices, tensor row_indices, tensor values, int options_kind, int options_device); +void atg_sparse_bsc_tensor_ccol_row_value_size(tensor *, tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device); +void atg_sparse_bsr_tensor(tensor *, tensor crow_indices, tensor col_indices, tensor values, int options_kind, int options_device); +void atg_sparse_bsr_tensor_crow_col_value_size(tensor *, tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device); +void atg_sparse_compressed_tensor(tensor *, tensor compressed_indices, tensor plain_indices, tensor values, int options_kind, int options_device); +void atg_sparse_compressed_tensor_comp_plain_value_size(tensor *, tensor compressed_indices, tensor plain_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device); +void atg_sparse_coo_tensor(tensor *, int64_t *size_data, int size_len, int options_kind, int options_device); +void atg_sparse_coo_tensor_indices(tensor *, tensor indices, tensor values, int options_kind, int options_device, int is_coalesced); +void atg_sparse_coo_tensor_indices_size(tensor *, tensor indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device, int is_coalesced); +void atg_sparse_coo_tensor_size_out(tensor *, tensor out, int64_t *size_data, int size_len); +void atg_sparse_csc_tensor(tensor *, tensor ccol_indices, tensor row_indices, tensor values, int options_kind, int options_device); +void atg_sparse_csc_tensor_ccol_row_value_size(tensor *, tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device); +void atg_sparse_csr_tensor(tensor *, tensor crow_indices, tensor col_indices, tensor values, int options_kind, int options_device); +void atg_sparse_csr_tensor_crow_col_value_size(tensor *, tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device); +int64_t atg_sparse_dim(tensor self); +void atg_sparse_mask(tensor *, tensor self, tensor mask); +void atg_sparse_mask_out(tensor *, tensor out, tensor self, tensor mask); +void atg_sparse_resize(tensor *, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim); +void atg_sparse_resize_(tensor *, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim); +void atg_sparse_resize_and_clear(tensor *, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim); +void atg_sparse_resize_and_clear_(tensor *, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim); +void atg_sparse_resize_and_clear_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim); +void atg_sparse_resize_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim); +void atg_sparse_sampled_addmm(tensor *, tensor self, tensor mat1, tensor mat2); +void atg_sparse_sampled_addmm_out(tensor *, tensor out, tensor self, tensor mat1, tensor mat2); +void atg_special_airy_ai(tensor *, tensor x); +void atg_special_airy_ai_out(tensor *, tensor out, tensor x); +void atg_special_bessel_j0(tensor *, tensor self); +void atg_special_bessel_j0_out(tensor *, tensor out, tensor self); +void atg_special_bessel_j1(tensor *, tensor self); +void atg_special_bessel_j1_out(tensor *, tensor out, tensor self); +void atg_special_bessel_y0(tensor *, tensor self); +void atg_special_bessel_y0_out(tensor *, tensor out, tensor self); +void atg_special_bessel_y1(tensor *, tensor self); +void atg_special_bessel_y1_out(tensor *, tensor out, tensor self); +void atg_special_chebyshev_polynomial_t(tensor *, tensor x, tensor n); +void atg_special_chebyshev_polynomial_t_n_scalar(tensor *, tensor x, scalar n); +void atg_special_chebyshev_polynomial_t_n_scalar_out(tensor *, tensor out, tensor x, scalar n); +void atg_special_chebyshev_polynomial_t_out(tensor *, tensor out, tensor x, tensor n); +void atg_special_chebyshev_polynomial_t_x_scalar(tensor *, scalar x, tensor n); +void atg_special_chebyshev_polynomial_t_x_scalar_out(tensor *, tensor out, scalar x, tensor n); +void atg_special_chebyshev_polynomial_u(tensor *, tensor x, tensor n); +void atg_special_chebyshev_polynomial_u_n_scalar(tensor *, tensor x, scalar n); +void atg_special_chebyshev_polynomial_u_n_scalar_out(tensor *, tensor out, tensor x, scalar n); +void atg_special_chebyshev_polynomial_u_out(tensor *, tensor out, tensor x, tensor n); +void atg_special_chebyshev_polynomial_u_x_scalar(tensor *, scalar x, tensor n); +void atg_special_chebyshev_polynomial_u_x_scalar_out(tensor *, tensor out, scalar x, tensor n); +void atg_special_chebyshev_polynomial_v(tensor *, tensor x, tensor n); +void atg_special_chebyshev_polynomial_v_n_scalar(tensor *, tensor x, scalar n); +void atg_special_chebyshev_polynomial_v_n_scalar_out(tensor *, tensor out, tensor x, scalar n); +void atg_special_chebyshev_polynomial_v_out(tensor *, tensor out, tensor x, tensor n); +void atg_special_chebyshev_polynomial_v_x_scalar(tensor *, scalar x, tensor n); +void atg_special_chebyshev_polynomial_v_x_scalar_out(tensor *, tensor out, scalar x, tensor n); +void atg_special_chebyshev_polynomial_w(tensor *, tensor x, tensor n); +void atg_special_chebyshev_polynomial_w_n_scalar(tensor *, tensor x, scalar n); +void atg_special_chebyshev_polynomial_w_n_scalar_out(tensor *, tensor out, tensor x, scalar n); +void atg_special_chebyshev_polynomial_w_out(tensor *, tensor out, tensor x, tensor n); +void atg_special_chebyshev_polynomial_w_x_scalar(tensor *, scalar x, tensor n); +void atg_special_chebyshev_polynomial_w_x_scalar_out(tensor *, tensor out, scalar x, tensor n); +void atg_special_digamma(tensor *, tensor self); +void atg_special_digamma_out(tensor *, tensor out, tensor self); +void atg_special_entr(tensor *, tensor self); +void atg_special_entr_out(tensor *, tensor out, tensor self); +void atg_special_erf(tensor *, tensor self); +void atg_special_erf_out(tensor *, tensor out, tensor self); +void atg_special_erfc(tensor *, tensor self); +void atg_special_erfc_out(tensor *, tensor out, tensor self); +void atg_special_erfcx(tensor *, tensor self); +void atg_special_erfcx_out(tensor *, tensor out, tensor self); +void atg_special_erfinv(tensor *, tensor self); +void atg_special_erfinv_out(tensor *, tensor out, tensor self); +void atg_special_exp2(tensor *, tensor self); +void atg_special_exp2_out(tensor *, tensor out, tensor self); +void atg_special_expit(tensor *, tensor self); +void atg_special_expit_out(tensor *, tensor out, tensor self); +void atg_special_expm1(tensor *, tensor self); +void atg_special_expm1_out(tensor *, tensor out, tensor self); +void atg_special_gammainc(tensor *, tensor self, tensor other); +void atg_special_gammainc_out(tensor *, tensor out, tensor self, tensor other); +void atg_special_gammaincc(tensor *, tensor self, tensor other); +void atg_special_gammaincc_out(tensor *, tensor out, tensor self, tensor other); +void atg_special_gammaln(tensor *, tensor self); +void atg_special_gammaln_out(tensor *, tensor out, tensor self); +void atg_special_hermite_polynomial_h(tensor *, tensor x, tensor n); +void atg_special_hermite_polynomial_h_n_scalar(tensor *, tensor x, scalar n); +void atg_special_hermite_polynomial_h_n_scalar_out(tensor *, tensor out, tensor x, scalar n); +void atg_special_hermite_polynomial_h_out(tensor *, tensor out, tensor x, tensor n); +void atg_special_hermite_polynomial_h_x_scalar(tensor *, scalar x, tensor n); +void atg_special_hermite_polynomial_h_x_scalar_out(tensor *, tensor out, scalar x, tensor n); +void atg_special_hermite_polynomial_he(tensor *, tensor x, tensor n); +void atg_special_hermite_polynomial_he_n_scalar(tensor *, tensor x, scalar n); +void atg_special_hermite_polynomial_he_n_scalar_out(tensor *, tensor out, tensor x, scalar n); +void atg_special_hermite_polynomial_he_out(tensor *, tensor out, tensor x, tensor n); +void atg_special_hermite_polynomial_he_x_scalar(tensor *, scalar x, tensor n); +void atg_special_hermite_polynomial_he_x_scalar_out(tensor *, tensor out, scalar x, tensor n); +void atg_special_i0(tensor *, tensor self); +void atg_special_i0_out(tensor *, tensor out, tensor self); +void atg_special_i0e(tensor *, tensor self); +void atg_special_i0e_out(tensor *, tensor out, tensor self); +void atg_special_i1(tensor *, tensor self); +void atg_special_i1_out(tensor *, tensor out, tensor self); +void atg_special_i1e(tensor *, tensor self); +void atg_special_i1e_out(tensor *, tensor out, tensor self); +void atg_special_laguerre_polynomial_l(tensor *, tensor x, tensor n); +void atg_special_laguerre_polynomial_l_n_scalar(tensor *, tensor x, scalar n); +void atg_special_laguerre_polynomial_l_n_scalar_out(tensor *, tensor out, tensor x, scalar n); +void atg_special_laguerre_polynomial_l_out(tensor *, tensor out, tensor x, tensor n); +void atg_special_laguerre_polynomial_l_x_scalar(tensor *, scalar x, tensor n); +void atg_special_laguerre_polynomial_l_x_scalar_out(tensor *, tensor out, scalar x, tensor n); +void atg_special_legendre_polynomial_p(tensor *, tensor x, tensor n); +void atg_special_legendre_polynomial_p_n_scalar(tensor *, tensor x, scalar n); +void atg_special_legendre_polynomial_p_n_scalar_out(tensor *, tensor out, tensor x, scalar n); +void atg_special_legendre_polynomial_p_out(tensor *, tensor out, tensor x, tensor n); +void atg_special_legendre_polynomial_p_x_scalar(tensor *, scalar x, tensor n); +void atg_special_legendre_polynomial_p_x_scalar_out(tensor *, tensor out, scalar x, tensor n); +void atg_special_log1p(tensor *, tensor self); +void atg_special_log1p_out(tensor *, tensor out, tensor self); +void atg_special_log_ndtr(tensor *, tensor self); +void atg_special_log_ndtr_out(tensor *, tensor out, tensor self); +void atg_special_log_softmax(tensor *, tensor self, int64_t dim, int dtype); +void atg_special_logit(tensor *, tensor self, double eps_v, uint8_t eps_null); +void atg_special_logit_out(tensor *, tensor out, tensor self, double eps_v, uint8_t eps_null); +void atg_special_logsumexp(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim); +void atg_special_logsumexp_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim); +void atg_special_modified_bessel_i0(tensor *, tensor self); +void atg_special_modified_bessel_i0_out(tensor *, tensor out, tensor self); +void atg_special_modified_bessel_i1(tensor *, tensor self); +void atg_special_modified_bessel_i1_out(tensor *, tensor out, tensor self); +void atg_special_modified_bessel_k0(tensor *, tensor self); +void atg_special_modified_bessel_k0_out(tensor *, tensor out, tensor self); +void atg_special_modified_bessel_k1(tensor *, tensor self); +void atg_special_modified_bessel_k1_out(tensor *, tensor out, tensor self); +void atg_special_multigammaln(tensor *, tensor self, int64_t p); +void atg_special_multigammaln_out(tensor *, tensor out, tensor self, int64_t p); +void atg_special_ndtr(tensor *, tensor self); +void atg_special_ndtr_out(tensor *, tensor out, tensor self); +void atg_special_ndtri(tensor *, tensor self); +void atg_special_ndtri_out(tensor *, tensor out, tensor self); +void atg_special_polygamma(tensor *, int64_t n, tensor self); +void atg_special_polygamma_out(tensor *, tensor out, int64_t n, tensor self); +void atg_special_psi(tensor *, tensor self); +void atg_special_psi_out(tensor *, tensor out, tensor self); +void atg_special_round(tensor *, tensor self, int64_t decimals); +void atg_special_round_out(tensor *, tensor out, tensor self, int64_t decimals); +void atg_special_scaled_modified_bessel_k0(tensor *, tensor x); +void atg_special_scaled_modified_bessel_k0_out(tensor *, tensor out, tensor x); +void atg_special_scaled_modified_bessel_k1(tensor *, tensor x); +void atg_special_scaled_modified_bessel_k1_out(tensor *, tensor out, tensor x); +void atg_special_shifted_chebyshev_polynomial_t(tensor *, tensor x, tensor n); +void atg_special_shifted_chebyshev_polynomial_t_n_scalar(tensor *, tensor x, scalar n); +void atg_special_shifted_chebyshev_polynomial_t_n_scalar_out(tensor *, tensor out, tensor x, scalar n); +void atg_special_shifted_chebyshev_polynomial_t_out(tensor *, tensor out, tensor x, tensor n); +void atg_special_shifted_chebyshev_polynomial_t_x_scalar(tensor *, scalar x, tensor n); +void atg_special_shifted_chebyshev_polynomial_t_x_scalar_out(tensor *, tensor out, scalar x, tensor n); +void atg_special_shifted_chebyshev_polynomial_u(tensor *, tensor x, tensor n); +void atg_special_shifted_chebyshev_polynomial_u_n_scalar(tensor *, tensor x, scalar n); +void atg_special_shifted_chebyshev_polynomial_u_n_scalar_out(tensor *, tensor out, tensor x, scalar n); +void atg_special_shifted_chebyshev_polynomial_u_out(tensor *, tensor out, tensor x, tensor n); +void atg_special_shifted_chebyshev_polynomial_u_x_scalar(tensor *, scalar x, tensor n); +void atg_special_shifted_chebyshev_polynomial_u_x_scalar_out(tensor *, tensor out, scalar x, tensor n); +void atg_special_shifted_chebyshev_polynomial_v(tensor *, tensor x, tensor n); +void atg_special_shifted_chebyshev_polynomial_v_n_scalar(tensor *, tensor x, scalar n); +void atg_special_shifted_chebyshev_polynomial_v_n_scalar_out(tensor *, tensor out, tensor x, scalar n); +void atg_special_shifted_chebyshev_polynomial_v_out(tensor *, tensor out, tensor x, tensor n); +void atg_special_shifted_chebyshev_polynomial_v_x_scalar(tensor *, scalar x, tensor n); +void atg_special_shifted_chebyshev_polynomial_v_x_scalar_out(tensor *, tensor out, scalar x, tensor n); +void atg_special_shifted_chebyshev_polynomial_w(tensor *, tensor x, tensor n); +void atg_special_shifted_chebyshev_polynomial_w_n_scalar(tensor *, tensor x, scalar n); +void atg_special_shifted_chebyshev_polynomial_w_n_scalar_out(tensor *, tensor out, tensor x, scalar n); +void atg_special_shifted_chebyshev_polynomial_w_out(tensor *, tensor out, tensor x, tensor n); +void atg_special_shifted_chebyshev_polynomial_w_x_scalar(tensor *, scalar x, tensor n); +void atg_special_shifted_chebyshev_polynomial_w_x_scalar_out(tensor *, tensor out, scalar x, tensor n); +void atg_special_sinc(tensor *, tensor self); +void atg_special_sinc_out(tensor *, tensor out, tensor self); +void atg_special_softmax(tensor *, tensor self, int64_t dim, int dtype); +void atg_special_spherical_bessel_j0(tensor *, tensor x); +void atg_special_spherical_bessel_j0_out(tensor *, tensor out, tensor x); +void atg_special_xlog1py(tensor *, tensor self, tensor other); +void atg_special_xlog1py_other_scalar(tensor *, tensor self, scalar other); +void atg_special_xlog1py_other_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_special_xlog1py_out(tensor *, tensor out, tensor self, tensor other); +void atg_special_xlog1py_self_scalar(tensor *, scalar self_scalar, tensor other); +void atg_special_xlog1py_self_scalar_out(tensor *, tensor out, scalar self_scalar, tensor other); +void atg_special_xlogy(tensor *, tensor self, tensor other); +void atg_special_xlogy_other_scalar(tensor *, tensor self, scalar other); +void atg_special_xlogy_other_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_special_xlogy_out(tensor *, tensor out, tensor self, tensor other); +void atg_special_xlogy_self_scalar(tensor *, scalar self_scalar, tensor other); +void atg_special_xlogy_self_scalar_out(tensor *, tensor out, scalar self_scalar, tensor other); +void atg_special_zeta(tensor *, tensor self, tensor other); +void atg_special_zeta_other_scalar(tensor *, tensor self, scalar other); +void atg_special_zeta_other_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_special_zeta_out(tensor *, tensor out, tensor self, tensor other); +void atg_special_zeta_self_scalar(tensor *, scalar self_scalar, tensor other); +void atg_special_zeta_self_scalar_out(tensor *, tensor out, scalar self_scalar, tensor other); +tensor *atg_split(tensor self, int64_t split_size, int64_t dim); +tensor *atg_split_copy(tensor self, int64_t split_size, int64_t dim); +void atg_split_copy_tensor_out(tensor *out_data, int out_len, tensor self, int64_t split_size, int64_t dim); +tensor *atg_split_sizes(tensor self, int64_t *split_size_data, int split_size_len, int64_t dim); +tensor *atg_split_with_sizes(tensor self, int64_t *split_sizes_data, int split_sizes_len, int64_t dim); +tensor *atg_split_with_sizes_copy(tensor self, int64_t *split_sizes_data, int split_sizes_len, int64_t dim); +void atg_split_with_sizes_copy_out(tensor *out_data, int out_len, tensor self, int64_t *split_sizes_data, int split_sizes_len, int64_t dim); +void atg_sqrt(tensor *, tensor self); +void atg_sqrt_(tensor *, tensor self); +void atg_sqrt_out(tensor *, tensor out, tensor self); +void atg_square(tensor *, tensor self); +void atg_square_(tensor *, tensor self); +void atg_square_out(tensor *, tensor out, tensor self); +void atg_squeeze(tensor *, tensor self); +void atg_squeeze_(tensor *, tensor self); +void atg_squeeze_copy(tensor *, tensor self); +void atg_squeeze_copy_dim(tensor *, tensor self, int64_t dim); +void atg_squeeze_copy_dim_out(tensor *, tensor out, tensor self, int64_t dim); +void atg_squeeze_copy_dims(tensor *, tensor self, int64_t *dim_data, int dim_len); +void atg_squeeze_copy_dims_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len); +void atg_squeeze_copy_out(tensor *, tensor out, tensor self); +void atg_squeeze_dim(tensor *, tensor self, int64_t dim); +void atg_squeeze_dim_(tensor *, tensor self, int64_t dim); +void atg_squeeze_dims(tensor *, tensor self, int64_t *dim_data, int dim_len); +void atg_squeeze_dims_(tensor *, tensor self, int64_t *dim_data, int dim_len); +void atg_sspaddmm(tensor *, tensor self, tensor mat1, tensor mat2); +void atg_sspaddmm_out(tensor *, tensor out, tensor self, tensor mat1, tensor mat2); +void atg_stack(tensor *, tensor *tensors_data, int tensors_len, int64_t dim); +void atg_stack_out(tensor *, tensor out, tensor *tensors_data, int tensors_len, int64_t dim); +void atg_std(tensor *, tensor self, int unbiased); +void atg_std_correction(tensor *, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim); +void atg_std_correction_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim); +void atg_std_dim(tensor *, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim); +void atg_std_mean(tensor *, tensor self, int unbiased); +void atg_std_mean_correction(tensor *, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim); +void atg_std_mean_correction_out(tensor *, tensor out0, tensor out1, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim); +void atg_std_mean_dim(tensor *, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim); +void atg_std_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim); +void atg_stft(tensor *, tensor self, int64_t n_fft, int64_t hop_length_v, uint8_t hop_length_null, int64_t win_length_v, uint8_t win_length_null, tensor window, int normalized, int onesided, int return_complex, int align_to_window); +void atg_stft_center(tensor *, tensor self, int64_t n_fft, int64_t hop_length_v, uint8_t hop_length_null, int64_t win_length_v, uint8_t win_length_null, tensor window, int center, char* pad_mode_ptr, int pad_mode_len, int normalized, int onesided, int return_complex, int align_to_window); +void atg_sub(tensor *, tensor self, tensor other); +void atg_sub_(tensor *, tensor self, tensor other); +void atg_sub_out(tensor *, tensor out, tensor self, tensor other); +void atg_sub_scalar(tensor *, tensor self, scalar other); +void atg_sub_scalar_(tensor *, tensor self, scalar other); +void atg_sub_scalar_out(tensor *, tensor out, tensor self, scalar other); +void atg_subtract(tensor *, tensor self, tensor other); +void atg_subtract_(tensor *, tensor self, tensor other); +void atg_subtract_out(tensor *, tensor out, tensor self, tensor other); +void atg_subtract_scalar(tensor *, tensor self, scalar other); +void atg_subtract_scalar_(tensor *, tensor self, scalar other); +void atg_sum(tensor *, tensor self, int dtype); +void atg_sum_dim_intlist(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); +void atg_sum_intlist_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); +void atg_sum_out(tensor *, tensor out, tensor self, int dtype); +void atg_sum_to_size(tensor *, tensor self, int64_t *size_data, int size_len); +void atg_svd(tensor *, tensor self, int some, int compute_uv); +void atg_svd_u(tensor *, tensor U, tensor S, tensor V, tensor self, int some, int compute_uv); +void atg_swapaxes(tensor *, tensor self, int64_t axis0, int64_t axis1); +void atg_swapaxes_(tensor *, tensor self, int64_t axis0, int64_t axis1); +void atg_swapdims(tensor *, tensor self, int64_t dim0, int64_t dim1); +void atg_swapdims_(tensor *, tensor self, int64_t dim0, int64_t dim1); +void atg_t(tensor *, tensor self); +void atg_t_(tensor *, tensor self); +void atg_t_copy(tensor *, tensor self); +void atg_t_copy_out(tensor *, tensor out, tensor self); +void atg_take(tensor *, tensor self, tensor index); +void atg_take_along_dim(tensor *, tensor self, tensor indices, int64_t dim_v, uint8_t dim_null); +void atg_take_along_dim_out(tensor *, tensor out, tensor self, tensor indices, int64_t dim_v, uint8_t dim_null); +void atg_take_out(tensor *, tensor out, tensor self, tensor index); +void atg_tan(tensor *, tensor self); +void atg_tan_(tensor *, tensor self); +void atg_tan_out(tensor *, tensor out, tensor self); +void atg_tanh(tensor *, tensor self); +void atg_tanh_(tensor *, tensor self); +void atg_tanh_backward(tensor *, tensor grad_output, tensor output); +void atg_tanh_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor output); +void atg_tanh_out(tensor *, tensor out, tensor self); +tensor *atg_tensor_split(tensor self, int64_t sections, int64_t dim); +tensor *atg_tensor_split_indices(tensor self, int64_t *indices_data, int indices_len, int64_t dim); +tensor *atg_tensor_split_tensor_indices_or_sections(tensor self, tensor tensor_indices_or_sections, int64_t dim); +void atg_tensordot(tensor *, tensor self, tensor other, int64_t *dims_self_data, int dims_self_len, int64_t *dims_other_data, int dims_other_len); +void atg_tensordot_out(tensor *, tensor out, tensor self, tensor other, int64_t *dims_self_data, int dims_self_len, int64_t *dims_other_data, int dims_other_len); +void atg_threshold(tensor *, tensor self, scalar threshold, scalar value); +void atg_threshold_(tensor *, tensor self, scalar threshold, scalar value); +void atg_threshold_backward(tensor *, tensor grad_output, tensor self, scalar threshold); +void atg_threshold_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, scalar threshold); +void atg_threshold_out(tensor *, tensor out, tensor self, scalar threshold, scalar value); +void atg_tile(tensor *, tensor self, int64_t *dims_data, int dims_len); +void atg_to(tensor *, tensor self, int device); +void atg_to_dense(tensor *, tensor self, int dtype, int masked_grad); +void atg_to_dense_backward(tensor *, tensor grad, tensor input, int masked_grad); +void atg_to_device(tensor *, tensor self, int device, int dtype, int non_blocking, int copy); +void atg_to_dtype(tensor *, tensor self, int dtype, int non_blocking, int copy); +void atg_to_dtype_layout(tensor *, tensor self, int options_kind, int options_device, int non_blocking, int copy); +void atg_to_mkldnn(tensor *, tensor self, int dtype); +void atg_to_mkldnn_backward(tensor *, tensor grad, tensor input); +void atg_to_mkldnn_out(tensor *, tensor out, tensor self, int dtype); +void atg_to_other(tensor *, tensor self, tensor other, int non_blocking, int copy); +void atg_to_padded_tensor(tensor *, tensor self, double padding, int64_t *output_size_data, int output_size_len); +void atg_to_padded_tensor_out(tensor *, tensor out, tensor self, double padding, int64_t *output_size_data, int output_size_len); +void atg_to_sparse(tensor *, tensor self, int8_t layout, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null); +void atg_to_sparse_bsc(tensor *, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null); +void atg_to_sparse_bsr(tensor *, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null); +void atg_to_sparse_csc(tensor *, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null); +void atg_to_sparse_csr(tensor *, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null); +void atg_to_sparse_sparse_dim(tensor *, tensor self, int64_t sparse_dim); +void atg_topk(tensor *, tensor self, int64_t k, int64_t dim, int largest, int sorted); +void atg_topk_values(tensor *, tensor values, tensor indices, tensor self, int64_t k, int64_t dim, int largest, int sorted); +void atg_totype(tensor *, tensor self, int scalar_type); +void atg_trace(tensor *, tensor self); +void atg_trace_backward(tensor *, tensor grad, int64_t *sizes_data, int sizes_len); +void atg_trace_out(tensor *, tensor out, tensor self); +void atg_transpose(tensor *, tensor self, int64_t dim0, int64_t dim1); +void atg_transpose_(tensor *, tensor self, int64_t dim0, int64_t dim1); +void atg_transpose_copy(tensor *, tensor self, int64_t dim0, int64_t dim1); +void atg_transpose_copy_int_out(tensor *, tensor out, tensor self, int64_t dim0, int64_t dim1); +void atg_trapezoid(tensor *, tensor y, int64_t dim); +void atg_trapezoid_x(tensor *, tensor y, tensor x, int64_t dim); +void atg_trapz(tensor *, tensor y, tensor x, int64_t dim); +void atg_trapz_dx(tensor *, tensor y, double dx, int64_t dim); +void atg_triangular_solve(tensor *, tensor self, tensor A, int upper, int transpose, int unitriangular); +void atg_triangular_solve_x(tensor *, tensor X, tensor M, tensor self, tensor A, int upper, int transpose, int unitriangular); +void atg_tril(tensor *, tensor self, int64_t diagonal); +void atg_tril_(tensor *, tensor self, int64_t diagonal); +void atg_tril_indices(tensor *, int64_t row, int64_t col, int64_t offset, int options_kind, int options_device); +void atg_tril_indices_out(tensor *, tensor out, int64_t row, int64_t col, int64_t offset); +void atg_tril_out(tensor *, tensor out, tensor self, int64_t diagonal); +void atg_triplet_margin_loss(tensor *, tensor anchor, tensor positive, tensor negative, double margin, double p, double eps, int swap, int64_t reduction); +void atg_triu(tensor *, tensor self, int64_t diagonal); +void atg_triu_(tensor *, tensor self, int64_t diagonal); +void atg_triu_indices(tensor *, int64_t row, int64_t col, int64_t offset, int options_kind, int options_device); +void atg_triu_indices_out(tensor *, tensor out, int64_t row, int64_t col, int64_t offset); +void atg_triu_out(tensor *, tensor out, tensor self, int64_t diagonal); +void atg_true_divide(tensor *, tensor self, tensor other); +void atg_true_divide_(tensor *, tensor self, tensor other); +void atg_true_divide_out(tensor *, tensor out, tensor self, tensor other); +void atg_true_divide_scalar(tensor *, tensor self, scalar other); +void atg_true_divide_scalar_(tensor *, tensor self, scalar other); +void atg_trunc(tensor *, tensor self); +void atg_trunc_(tensor *, tensor self); +void atg_trunc_out(tensor *, tensor out, tensor self); +void atg_type_as(tensor *, tensor self, tensor other); +tensor *atg_unbind(tensor self, int64_t dim); +tensor *atg_unbind_copy(tensor self, int64_t dim); +void atg_unbind_copy_int_out(tensor *out_data, int out_len, tensor self, int64_t dim); +void atg_unflatten(tensor *, tensor self, int64_t dim, int64_t *sizes_data, int sizes_len); +tensor *atg_unflatten_dense_tensors(tensor flat, tensor *tensors_data, int tensors_len); +void atg_unfold(tensor *, tensor self, int64_t dimension, int64_t size, int64_t step); +void atg_unfold_backward(tensor *, tensor grad_in, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t size, int64_t step); +void atg_unfold_backward_out(tensor *, tensor out, tensor grad_in, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t size, int64_t step); +void atg_unfold_copy(tensor *, tensor self, int64_t dimension, int64_t size, int64_t step); +void atg_unfold_copy_out(tensor *, tensor out, tensor self, int64_t dimension, int64_t size, int64_t step); +void atg_uniform(tensor *, tensor self, double from, double to); +void atg_uniform_(tensor *, tensor self, double from, double to); +void atg_uniform_out(tensor *, tensor out, tensor self, double from, double to); +void atg_unique_consecutive(tensor *, tensor self, int return_inverse, int return_counts, int64_t dim_v, uint8_t dim_null); +void atg_unique_consecutive_out(tensor *, tensor out0, tensor out1, tensor out2, tensor self, int return_inverse, int return_counts, int64_t dim_v, uint8_t dim_null); +void atg_unique_dim(tensor *, tensor self, int64_t dim, int sorted, int return_inverse, int return_counts); +void atg_unique_dim_consecutive(tensor *, tensor self, int64_t dim, int return_inverse, int return_counts); +void atg_unique_dim_consecutive_out(tensor *, tensor out0, tensor out1, tensor out2, tensor self, int64_t dim, int return_inverse, int return_counts); +void atg_unique_dim_out(tensor *, tensor out0, tensor out1, tensor out2, tensor self, int64_t dim, int sorted, int return_inverse, int return_counts); +tensor *atg_unsafe_chunk(tensor self, int64_t chunks, int64_t dim); +tensor *atg_unsafe_split(tensor self, int64_t split_size, int64_t dim); +void atg_unsafe_split_tensor_out(tensor *out_data, int out_len, tensor self, int64_t split_size, int64_t dim); +tensor *atg_unsafe_split_with_sizes(tensor self, int64_t *split_sizes_data, int split_sizes_len, int64_t dim); +void atg_unsafe_split_with_sizes_out(tensor *out_data, int out_len, tensor self, int64_t *split_sizes_data, int split_sizes_len, int64_t dim); +void atg_unsqueeze(tensor *, tensor self, int64_t dim); +void atg_unsqueeze_(tensor *, tensor self, int64_t dim); +void atg_unsqueeze_copy(tensor *, tensor self, int64_t dim); +void atg_unsqueeze_copy_out(tensor *, tensor out, tensor self, int64_t dim); +void atg_upsample_bicubic2d(tensor *, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg_upsample_bicubic2d_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg_upsample_bicubic2d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg_upsample_bicubic2d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg_upsample_bicubic2d_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len); +void atg_upsample_bilinear2d(tensor *, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg_upsample_bilinear2d_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg_upsample_bilinear2d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg_upsample_bilinear2d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg_upsample_bilinear2d_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len); +void atg_upsample_bilinear2d_vec_out(tensor *, tensor out, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len); +void atg_upsample_linear1d(tensor *, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_v, uint8_t scales_null); +void atg_upsample_linear1d_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_v, uint8_t scales_null); +void atg_upsample_linear1d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_v, uint8_t scales_null); +void atg_upsample_linear1d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_v, uint8_t scales_null); +void atg_upsample_linear1d_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len); +void atg_upsample_nearest1d(tensor *, tensor self, int64_t *output_size_data, int output_size_len, double scales_v, uint8_t scales_null); +void atg_upsample_nearest1d_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_v, uint8_t scales_null); +void atg_upsample_nearest1d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_v, uint8_t scales_null); +void atg_upsample_nearest1d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_v, uint8_t scales_null); +void atg_upsample_nearest1d_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len); +void atg_upsample_nearest2d(tensor *, tensor self, int64_t *output_size_data, int output_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg_upsample_nearest2d_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg_upsample_nearest2d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg_upsample_nearest2d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg_upsample_nearest2d_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len); +void atg_upsample_nearest2d_vec_out(tensor *, tensor out, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len); +void atg_upsample_nearest3d(tensor *, tensor self, int64_t *output_size_data, int output_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg_upsample_nearest3d_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg_upsample_nearest3d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg_upsample_nearest3d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg_upsample_nearest3d_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len); +void atg_upsample_trilinear3d(tensor *, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg_upsample_trilinear3d_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg_upsample_trilinear3d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg_upsample_trilinear3d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); +void atg_upsample_trilinear3d_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len); +void atg_value_selecting_reduction_backward(tensor *, tensor grad, int64_t dim, tensor indices, int64_t *sizes_data, int sizes_len, int keepdim); +void atg_values(tensor *, tensor self); +void atg_values_copy(tensor *, tensor self); +void atg_values_copy_out(tensor *, tensor out, tensor self); +void atg_vander(tensor *, tensor x, int64_t n_v, uint8_t n_null, int increasing); +void atg_var(tensor *, tensor self, int unbiased); +void atg_var_correction(tensor *, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim); +void atg_var_correction_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim); +void atg_var_dim(tensor *, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim); +void atg_var_mean(tensor *, tensor self, int unbiased); +void atg_var_mean_correction(tensor *, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim); +void atg_var_mean_correction_out(tensor *, tensor out0, tensor out1, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim); +void atg_var_mean_dim(tensor *, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim); +void atg_var_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim); +void atg_vdot(tensor *, tensor self, tensor other); +void atg_vdot_out(tensor *, tensor out, tensor self, tensor other); +void atg_view(tensor *, tensor self, int64_t *size_data, int size_len); +void atg_view_as(tensor *, tensor self, tensor other); +void atg_view_as_complex(tensor *, tensor self); +void atg_view_as_complex_copy(tensor *, tensor self); +void atg_view_as_complex_copy_out(tensor *, tensor out, tensor self); +void atg_view_as_real(tensor *, tensor self); +void atg_view_as_real_copy(tensor *, tensor self); +void atg_view_as_real_copy_out(tensor *, tensor out, tensor self); +void atg_view_copy(tensor *, tensor self, int64_t *size_data, int size_len); +void atg_view_copy_dtype(tensor *, tensor self, int dtype); +void atg_view_copy_dtype_out(tensor *, tensor out, tensor self, int dtype); +void atg_view_copy_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len); +void atg_view_dtype(tensor *, tensor self, int dtype); +tensor *atg_vsplit(tensor self, int64_t sections); +tensor *atg_vsplit_array(tensor self, int64_t *indices_data, int indices_len); +void atg_vstack(tensor *, tensor *tensors_data, int tensors_len); +void atg_vstack_out(tensor *, tensor out, tensor *tensors_data, int tensors_len); +tensor *atg_where(tensor condition); +void atg_where_scalar(tensor *, tensor condition, scalar self_scalar, scalar other); +void atg_where_scalarother(tensor *, tensor condition, tensor self, scalar other); +void atg_where_scalarself(tensor *, tensor condition, scalar self_scalar, tensor other); +void atg_where_self(tensor *, tensor condition, tensor self, tensor other); +void atg_where_self_out(tensor *, tensor out, tensor condition, tensor self, tensor other); +void atg_xlogy(tensor *, tensor self, tensor other); +void atg_xlogy_(tensor *, tensor self, tensor other); +void atg_xlogy_outscalar_other(tensor *, tensor out, tensor self, scalar other); +void atg_xlogy_outscalar_self(tensor *, tensor out, scalar self_scalar, tensor other); +void atg_xlogy_outtensor(tensor *, tensor out, tensor self, tensor other); +void atg_xlogy_scalar_other(tensor *, tensor self, scalar other); +void atg_xlogy_scalar_other_(tensor *, tensor self, scalar other); +void atg_xlogy_scalar_self(tensor *, scalar self_scalar, tensor other); +void atg_zero(tensor *, tensor self); +void atg_zero_(tensor *, tensor self); +void atg_zero_out(tensor *, tensor out, tensor self); +void atg_zeros(tensor *, int64_t *size_data, int size_len, int options_kind, int options_device); +void atg_zeros_like(tensor *, tensor self); +void atg_zeros_like_out(tensor *, tensor out, tensor self); +void atg_zeros_out(tensor *, tensor out, int64_t *size_data, int size_len); +} diff --git a/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_python.cpp b/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_python.cpp new file mode 100644 index 0000000000..03ce8743e7 --- /dev/null +++ b/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_python.cpp @@ -0,0 +1,22 @@ +#include "torch_python.h" + +bool thp_variable_check(PyObject* pyobj) { + PROTECT( + return THPVariable_Check(pyobj); + ) + return false; +} + +PyObject* thp_variable_wrap(tensor t) { + PROTECT( + return THPVariable_Wrap(*t); + ) + return nullptr; +} + +tensor thp_variable_unpack(PyObject* pyobj) { + PROTECT( + return new torch::Tensor(THPVariable_Unpack(pyobj)); + ) + return nullptr; +} diff --git a/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_python.h b/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_python.h new file mode 100644 index 0000000000..bd5f17eb6d --- /dev/null +++ b/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_python.h @@ -0,0 +1,19 @@ +#ifndef __TORCH_PYTHON_H__ +#define __TORCH_PYTHON_H__ + +#include "torch_api.h" +#include + +#ifdef __cplusplus +extern "C" { +#endif + +bool thp_variable_check(PyObject*); +PyObject* thp_variable_wrap(tensor); +tensor thp_variable_unpack(PyObject*); + +#ifdef __cplusplus +}; +#endif + +#endif diff --git a/v2/dependency-patches/torch-sys-0.24.0/src/c_generated.rs b/v2/dependency-patches/torch-sys-0.24.0/src/c_generated.rs new file mode 100644 index 0000000000..a89dce6b7c --- /dev/null +++ b/v2/dependency-patches/torch-sys-0.24.0/src/c_generated.rs @@ -0,0 +1,16055 @@ +/* THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT BY HAND! */ +#[allow(clippy::all)] +use crate::{C_scalar, C_tensor}; +use libc::c_int; + +extern "C" { + pub fn atg___and__(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg___and__tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg___iand__(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg___iand__tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg___ilshift__(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg___ilshift__tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg___ior__(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg___ior__tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg___irshift__(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg___irshift__tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg___ixor__(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg___ixor__tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg___lshift__(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg___lshift__scalar_out_( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg___lshift__tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg___lshift__tensor_out_( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg___or__(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg___or__tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg___rshift__(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg___rshift__scalar_out_( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg___rshift__tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg___rshift__tensor_out_( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg___xor__(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg___xor__tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg__adaptive_avg_pool2d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + ); + pub fn atg__adaptive_avg_pool2d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg__adaptive_avg_pool2d_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg__adaptive_avg_pool2d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + ); + pub fn atg__adaptive_avg_pool3d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + ); + pub fn atg__adaptive_avg_pool3d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg__adaptive_avg_pool3d_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg__adaptive_avg_pool3d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + ); + pub fn atg__add_batch_dim( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + batch_dim_: i64, + level_: i64, + ); + pub fn atg__add_relu(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg__add_relu_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg__add_relu_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg__add_relu_scalar( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg__add_relu_scalar_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg__add_relu_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg__addmm_activation( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mat1_: *mut C_tensor, + mat2_: *mut C_tensor, + use_gelu_: c_int, + ); + pub fn atg__addmm_activation_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mat1_: *mut C_tensor, + mat2_: *mut C_tensor, + use_gelu_: c_int, + ); + pub fn atg__aminmax(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__aminmax_dim( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + keepdim_: c_int, + ); + pub fn atg__aminmax_dim_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + keepdim_: c_int, + ); + pub fn atg__aminmax_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg__amp_update_scale( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + growth_tracker_: *mut C_tensor, + found_inf_: *mut C_tensor, + scale_growth_factor_: f64, + scale_backoff_factor_: f64, + growth_interval_: i64, + ); + pub fn atg__amp_update_scale_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + growth_tracker_: *mut C_tensor, + found_inf_: *mut C_tensor, + scale_growth_factor_: f64, + scale_backoff_factor_: f64, + growth_interval_: i64, + ); + pub fn atg__amp_update_scale_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + growth_tracker_: *mut C_tensor, + found_inf_: *mut C_tensor, + scale_growth_factor_: f64, + scale_backoff_factor_: f64, + growth_interval_: i64, + ); + pub fn atg__assert_scalar( + self_scalar_: *mut C_scalar, + assert_msg_ptr: *const u8, + assert_msg_len: c_int, + ); + pub fn atg__assert_tensor_metadata( + a_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + dtype_: c_int, + device_: c_int, + layout_: i8, + ); + pub fn atg__autocast_to_full_precision( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + cuda_enabled_: c_int, + cpu_enabled_: c_int, + ); + pub fn atg__autocast_to_reduced_precision( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + cuda_enabled_: c_int, + cpu_enabled_: c_int, + cuda_dtype_: c_int, + cpu_dtype_: c_int, + ); + pub fn atg__batch_norm_no_update( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + momentum_: f64, + eps_: f64, + ); + pub fn atg__batch_norm_no_update_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + out2_: *mut C_tensor, + out3_: *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + momentum_: f64, + eps_: f64, + ); + pub fn atg__batch_norm_with_update( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + momentum_: f64, + eps_: f64, + ); + pub fn atg__batch_norm_with_update_functional( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + momentum_: f64, + eps_: f64, + ); + pub fn atg__batch_norm_with_update_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + save_mean_: *mut C_tensor, + save_invstd_: *mut C_tensor, + reserve_: *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + momentum_: f64, + eps_: f64, + ); + pub fn atg__cast_byte(out__: *mut *mut C_tensor, self_: *mut C_tensor, non_blocking_: c_int); + pub fn atg__cast_char(out__: *mut *mut C_tensor, self_: *mut C_tensor, non_blocking_: c_int); + pub fn atg__cast_double(out__: *mut *mut C_tensor, self_: *mut C_tensor, non_blocking_: c_int); + pub fn atg__cast_float(out__: *mut *mut C_tensor, self_: *mut C_tensor, non_blocking_: c_int); + pub fn atg__cast_half(out__: *mut *mut C_tensor, self_: *mut C_tensor, non_blocking_: c_int); + pub fn atg__cast_int(out__: *mut *mut C_tensor, self_: *mut C_tensor, non_blocking_: c_int); + pub fn atg__cast_long(out__: *mut *mut C_tensor, self_: *mut C_tensor, non_blocking_: c_int); + pub fn atg__cast_short(out__: *mut *mut C_tensor, self_: *mut C_tensor, non_blocking_: c_int); + pub fn atg__cdist_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + x1_: *mut C_tensor, + x2_: *mut C_tensor, + p_: f64, + cdist_: *mut C_tensor, + ); + pub fn atg__cdist_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_: *mut C_tensor, + x1_: *mut C_tensor, + x2_: *mut C_tensor, + p_: f64, + cdist_: *mut C_tensor, + ); + pub fn atg__cholesky_solve_helper( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + A_: *mut C_tensor, + upper_: c_int, + ); + pub fn atg__cholesky_solve_helper_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + A_: *mut C_tensor, + upper_: c_int, + ); + pub fn atg__chunk_cat( + out__: *mut *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + dim_: i64, + num_chunks_: i64, + ); + pub fn atg__chunk_cat_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + dim_: i64, + num_chunks_: i64, + ); + pub fn atg__coalesce(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__coalesce_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg__coalesced(out__: *mut *mut C_tensor, self_: *mut C_tensor, coalesced_: c_int); + pub fn atg__coalesced_(out__: *mut *mut C_tensor, self_: *mut C_tensor, coalesced_: c_int); + pub fn atg__coalesced_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + coalesced_: c_int, + ); + pub fn atg__compute_linear_combination( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + coefficients_: *mut C_tensor, + ); + pub fn atg__compute_linear_combination_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + input_: *mut C_tensor, + coefficients_: *mut C_tensor, + ); + pub fn atg__conj(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__conj_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__conj_copy_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg__conj_physical(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__conj_physical_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg__conv_depthwise2d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ); + pub fn atg__conv_depthwise2d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ); + pub fn atg__convert_indices_from_coo_to_csr( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_: i64, + out_int32_: c_int, + ); + pub fn atg__convert_indices_from_coo_to_csr_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + size_: i64, + out_int32_: c_int, + ); + pub fn atg__convert_indices_from_csr_to_coo( + out__: *mut *mut C_tensor, + crow_indices_: *mut C_tensor, + col_indices_: *mut C_tensor, + out_int32_: c_int, + transpose_: c_int, + ); + pub fn atg__convert_indices_from_csr_to_coo_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + crow_indices_: *mut C_tensor, + col_indices_: *mut C_tensor, + out_int32_: c_int, + transpose_: c_int, + ); + pub fn atg__convert_weight_to_int4pack( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + innerKTiles_: i64, + ); + pub fn atg__convert_weight_to_int4pack_for_cpu( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + innerKTiles_: i64, + ); + pub fn atg__convolution( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + transposed_: c_int, + output_padding_data: *const i64, + output_padding_len: c_int, + groups_: i64, + benchmark_: c_int, + deterministic_: c_int, + cudnn_enabled_: c_int, + allow_tf32_: c_int, + ); + pub fn atg__convolution_deprecated( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + transposed_: c_int, + output_padding_data: *const i64, + output_padding_len: c_int, + groups_: i64, + benchmark_: c_int, + deterministic_: c_int, + cudnn_enabled_: c_int, + ); + pub fn atg__convolution_mode( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_ptr: *const u8, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + ); + pub fn atg__convolution_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + transposed_: c_int, + output_padding_data: *const i64, + output_padding_len: c_int, + groups_: i64, + benchmark_: c_int, + deterministic_: c_int, + cudnn_enabled_: c_int, + allow_tf32_: c_int, + ); + pub fn atg__copy_from( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dst_: *mut C_tensor, + non_blocking_: c_int, + ); + pub fn atg__copy_from_and_resize( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dst_: *mut C_tensor, + ); + pub fn atg__copy_from_and_resize_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dst_: *mut C_tensor, + ); + pub fn atg__copy_from_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dst_: *mut C_tensor, + non_blocking_: c_int, + ); + pub fn atg__cslt_compress(out__: *mut *mut C_tensor, input_: *mut C_tensor); + pub fn atg__cslt_sparse_mm( + out__: *mut *mut C_tensor, + compressed_A_: *mut C_tensor, + dense_B_: *mut C_tensor, + bias_: *mut C_tensor, + alpha_: *mut C_tensor, + out_dtype_: c_int, + transpose_result_: c_int, + alg_id_: i64, + split_k_: i64, + split_k_mode_: i64, + ); + pub fn atg__cslt_sparse_mm_search( + compressed_A_: *mut C_tensor, + dense_B_: *mut C_tensor, + bias_: *mut C_tensor, + alpha_: *mut C_tensor, + out_dtype_: c_int, + transpose_result_: c_int, + ) -> i64; + pub fn atg__ctc_loss( + out__: *mut *mut C_tensor, + log_probs_: *mut C_tensor, + targets_: *mut C_tensor, + input_lengths_data: *const i64, + input_lengths_len: c_int, + target_lengths_data: *const i64, + target_lengths_len: c_int, + blank_: i64, + zero_infinity_: c_int, + ); + pub fn atg__ctc_loss_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + log_probs_: *mut C_tensor, + targets_: *mut C_tensor, + input_lengths_data: *const i64, + input_lengths_len: c_int, + target_lengths_data: *const i64, + target_lengths_len: c_int, + neg_log_likelihood_: *mut C_tensor, + log_alpha_: *mut C_tensor, + blank_: i64, + zero_infinity_: c_int, + ); + pub fn atg__ctc_loss_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_: *mut C_tensor, + log_probs_: *mut C_tensor, + targets_: *mut C_tensor, + input_lengths_data: *const i64, + input_lengths_len: c_int, + target_lengths_data: *const i64, + target_lengths_len: c_int, + neg_log_likelihood_: *mut C_tensor, + log_alpha_: *mut C_tensor, + blank_: i64, + zero_infinity_: c_int, + ); + pub fn atg__ctc_loss_backward_tensor( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + log_probs_: *mut C_tensor, + targets_: *mut C_tensor, + input_lengths_: *mut C_tensor, + target_lengths_: *mut C_tensor, + neg_log_likelihood_: *mut C_tensor, + log_alpha_: *mut C_tensor, + blank_: i64, + zero_infinity_: c_int, + ); + pub fn atg__ctc_loss_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + log_probs_: *mut C_tensor, + targets_: *mut C_tensor, + input_lengths_data: *const i64, + input_lengths_len: c_int, + target_lengths_data: *const i64, + target_lengths_len: c_int, + blank_: i64, + zero_infinity_: c_int, + ); + pub fn atg__ctc_loss_tensor( + out__: *mut *mut C_tensor, + log_probs_: *mut C_tensor, + targets_: *mut C_tensor, + input_lengths_: *mut C_tensor, + target_lengths_: *mut C_tensor, + blank_: i64, + zero_infinity_: c_int, + ); + pub fn atg__ctc_loss_tensor_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + log_probs_: *mut C_tensor, + targets_: *mut C_tensor, + input_lengths_: *mut C_tensor, + target_lengths_: *mut C_tensor, + blank_: i64, + zero_infinity_: c_int, + ); + pub fn atg__cudnn_attention_backward( + out__: *mut *mut C_tensor, + grad_out_: *mut C_tensor, + query_: *mut C_tensor, + key_: *mut C_tensor, + value_: *mut C_tensor, + out_: *mut C_tensor, + logsumexp_: *mut C_tensor, + philox_seed_: *mut C_tensor, + philox_offset_: *mut C_tensor, + attn_bias_: *mut C_tensor, + cum_seq_q_: *mut C_tensor, + cum_seq_k_: *mut C_tensor, + max_q_: i64, + max_k_: i64, + dropout_p_: f64, + is_causal_: c_int, + scale_v: f64, + scale_null: i8, + ); + pub fn atg__cudnn_ctc_loss( + out__: *mut *mut C_tensor, + log_probs_: *mut C_tensor, + targets_: *mut C_tensor, + input_lengths_data: *const i64, + input_lengths_len: c_int, + target_lengths_data: *const i64, + target_lengths_len: c_int, + blank_: i64, + deterministic_: c_int, + zero_infinity_: c_int, + ); + pub fn atg__cudnn_ctc_loss_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + log_probs_: *mut C_tensor, + targets_: *mut C_tensor, + input_lengths_data: *const i64, + input_lengths_len: c_int, + target_lengths_data: *const i64, + target_lengths_len: c_int, + blank_: i64, + deterministic_: c_int, + zero_infinity_: c_int, + ); + pub fn atg__cudnn_ctc_loss_tensor( + out__: *mut *mut C_tensor, + log_probs_: *mut C_tensor, + targets_: *mut C_tensor, + input_lengths_: *mut C_tensor, + target_lengths_: *mut C_tensor, + blank_: i64, + deterministic_: c_int, + zero_infinity_: c_int, + ); + pub fn atg__cudnn_init_dropout_state( + out__: *mut *mut C_tensor, + dropout_: f64, + train_: c_int, + dropout_seed_: i64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg__cudnn_init_dropout_state_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + dropout_: f64, + train_: c_int, + dropout_seed_: i64, + ); + pub fn atg__cudnn_rnn( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_data: *const *mut C_tensor, + weight_len: c_int, + weight_stride0_: i64, + weight_buf_: *mut C_tensor, + hx_: *mut C_tensor, + cx_: *mut C_tensor, + mode_: i64, + hidden_size_: i64, + proj_size_: i64, + num_layers_: i64, + batch_first_: c_int, + dropout_: f64, + train_: c_int, + bidirectional_: c_int, + batch_sizes_data: *const i64, + batch_sizes_len: c_int, + dropout_state_: *mut C_tensor, + ); + pub fn atg__cudnn_rnn_flatten_weight( + out__: *mut *mut C_tensor, + weight_arr_data: *const *mut C_tensor, + weight_arr_len: c_int, + weight_stride0_: i64, + input_size_: i64, + mode_: i64, + hidden_size_: i64, + proj_size_: i64, + num_layers_: i64, + batch_first_: c_int, + bidirectional_: c_int, + ); + pub fn atg__cudnn_rnn_flatten_weight_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + weight_arr_data: *const *mut C_tensor, + weight_arr_len: c_int, + weight_stride0_: i64, + input_size_: i64, + mode_: i64, + hidden_size_: i64, + proj_size_: i64, + num_layers_: i64, + batch_first_: c_int, + bidirectional_: c_int, + ); + pub fn atg__cudnn_rnn_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + out2_: *mut C_tensor, + out3_: *mut C_tensor, + out4_: *mut C_tensor, + input_: *mut C_tensor, + weight_data: *const *mut C_tensor, + weight_len: c_int, + weight_stride0_: i64, + weight_buf_: *mut C_tensor, + hx_: *mut C_tensor, + cx_: *mut C_tensor, + mode_: i64, + hidden_size_: i64, + proj_size_: i64, + num_layers_: i64, + batch_first_: c_int, + dropout_: f64, + train_: c_int, + bidirectional_: c_int, + batch_sizes_data: *const i64, + batch_sizes_len: c_int, + dropout_state_: *mut C_tensor, + ); + pub fn atg__debug_has_internal_overlap(self_: *mut C_tensor) -> i64; + pub fn atg__dim_arange(out__: *mut *mut C_tensor, like_: *mut C_tensor, dim_: i64); + pub fn atg__dimi(self_: *mut C_tensor) -> i64; + pub fn atg__dimv(self_: *mut C_tensor) -> i64; + pub fn atg__dirichlet_grad( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + alpha_: *mut C_tensor, + total_: *mut C_tensor, + ); + pub fn atg__dirichlet_grad_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + alpha_: *mut C_tensor, + total_: *mut C_tensor, + ); + pub fn atg__dyn_quant_matmul_4bit( + out__: *mut *mut C_tensor, + inp_: *mut C_tensor, + packed_weights_: *mut C_tensor, + block_size_: i64, + in_features_: i64, + out_features_: i64, + ); + pub fn atg__dyn_quant_pack_4bit_weight( + out__: *mut *mut C_tensor, + weights_: *mut C_tensor, + scales_zeros_: *mut C_tensor, + bias_: *mut C_tensor, + block_size_: i64, + in_features_: i64, + out_features_: i64, + ); + pub fn atg__efficient_attention_backward( + out__: *mut *mut C_tensor, + grad_out__: *mut C_tensor, + query_: *mut C_tensor, + key_: *mut C_tensor, + value_: *mut C_tensor, + bias_: *mut C_tensor, + out_: *mut C_tensor, + cu_seqlens_q_: *mut C_tensor, + cu_seqlens_k_: *mut C_tensor, + max_seqlen_q_: i64, + max_seqlen_k_: i64, + logsumexp_: *mut C_tensor, + dropout_p_: f64, + philox_seed_: *mut C_tensor, + philox_offset_: *mut C_tensor, + custom_mask_type_: i64, + bias_requires_grad_: c_int, + scale_v: f64, + scale_null: i8, + num_splits_key_v: i64, + num_splits_key_null: i8, + window_size_v: i64, + window_size_null: i8, + shared_storage_dqdkdv_: c_int, + ); + pub fn atg__efficientzerotensor( + out__: *mut *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg__efficientzerotensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg__embedding_bag( + out__: *mut *mut C_tensor, + weight_: *mut C_tensor, + indices_: *mut C_tensor, + offsets_: *mut C_tensor, + scale_grad_by_freq_: c_int, + mode_: i64, + sparse_: c_int, + per_sample_weights_: *mut C_tensor, + include_last_offset_: c_int, + padding_idx_: i64, + ); + pub fn atg__embedding_bag_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + indices_: *mut C_tensor, + offsets_: *mut C_tensor, + offset2bag_: *mut C_tensor, + bag_size_: *mut C_tensor, + maximum_indices_: *mut C_tensor, + num_weights_: i64, + scale_grad_by_freq_: c_int, + mode_: i64, + sparse_: c_int, + per_sample_weights_: *mut C_tensor, + padding_idx_: i64, + ); + pub fn atg__embedding_bag_dense_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + indices_: *mut C_tensor, + offset2bag_: *mut C_tensor, + bag_size_: *mut C_tensor, + maximum_indices_: *mut C_tensor, + num_weights_: i64, + scale_grad_by_freq_: c_int, + mode_: i64, + per_sample_weights_: *mut C_tensor, + padding_idx_: i64, + ); + pub fn atg__embedding_bag_dense_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_: *mut C_tensor, + indices_: *mut C_tensor, + offset2bag_: *mut C_tensor, + bag_size_: *mut C_tensor, + maximum_indices_: *mut C_tensor, + num_weights_: i64, + scale_grad_by_freq_: c_int, + mode_: i64, + per_sample_weights_: *mut C_tensor, + padding_idx_: i64, + ); + pub fn atg__embedding_bag_forward_only( + out__: *mut *mut C_tensor, + weight_: *mut C_tensor, + indices_: *mut C_tensor, + offsets_: *mut C_tensor, + scale_grad_by_freq_: c_int, + mode_: i64, + sparse_: c_int, + per_sample_weights_: *mut C_tensor, + include_last_offset_: c_int, + padding_idx_: i64, + ); + pub fn atg__embedding_bag_forward_only_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + out2_: *mut C_tensor, + out3_: *mut C_tensor, + weight_: *mut C_tensor, + indices_: *mut C_tensor, + offsets_: *mut C_tensor, + scale_grad_by_freq_: c_int, + mode_: i64, + sparse_: c_int, + per_sample_weights_: *mut C_tensor, + include_last_offset_: c_int, + padding_idx_: i64, + ); + pub fn atg__embedding_bag_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + out2_: *mut C_tensor, + out3_: *mut C_tensor, + weight_: *mut C_tensor, + indices_: *mut C_tensor, + offsets_: *mut C_tensor, + scale_grad_by_freq_: c_int, + mode_: i64, + sparse_: c_int, + per_sample_weights_: *mut C_tensor, + include_last_offset_: c_int, + padding_idx_: i64, + ); + pub fn atg__embedding_bag_per_sample_weights_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + weight_: *mut C_tensor, + indices_: *mut C_tensor, + offsets_: *mut C_tensor, + offset2bag_: *mut C_tensor, + mode_: i64, + padding_idx_: i64, + ); + pub fn atg__embedding_bag_per_sample_weights_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_: *mut C_tensor, + weight_: *mut C_tensor, + indices_: *mut C_tensor, + offsets_: *mut C_tensor, + offset2bag_: *mut C_tensor, + mode_: i64, + padding_idx_: i64, + ); + pub fn atg__embedding_bag_sparse_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + indices_: *mut C_tensor, + offsets_: *mut C_tensor, + offset2bag_: *mut C_tensor, + bag_size_: *mut C_tensor, + num_weights_: i64, + scale_grad_by_freq_: c_int, + mode_: i64, + per_sample_weights_: *mut C_tensor, + padding_idx_: i64, + ); + pub fn atg__empty_affine_quantized( + out__: *mut *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + scale_: f64, + zero_point_: i64, + ); + pub fn atg__empty_affine_quantized_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + scale_: f64, + zero_point_: i64, + ); + pub fn atg__empty_per_channel_affine_quantized( + out__: *mut *mut C_tensor, + size_data: *const i64, + size_len: c_int, + scales_: *mut C_tensor, + zero_points_: *mut C_tensor, + axis_: i64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg__empty_per_channel_affine_quantized_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + scales_: *mut C_tensor, + zero_points_: *mut C_tensor, + axis_: i64, + ); + pub fn atg__euclidean_dist(out__: *mut *mut C_tensor, x1_: *mut C_tensor, x2_: *mut C_tensor); + pub fn atg__euclidean_dist_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x1_: *mut C_tensor, + x2_: *mut C_tensor, + ); + pub fn atg__fake_quantize_learnable_per_channel_affine( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + scale_: *mut C_tensor, + zero_point_: *mut C_tensor, + axis_: i64, + quant_min_: i64, + quant_max_: i64, + grad_factor_: f64, + ); + pub fn atg__fake_quantize_learnable_per_channel_affine_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + self_: *mut C_tensor, + scale_: *mut C_tensor, + zero_point_: *mut C_tensor, + axis_: i64, + quant_min_: i64, + quant_max_: i64, + grad_factor_: f64, + ); + pub fn atg__fake_quantize_learnable_per_channel_affine_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + scale_: *mut C_tensor, + zero_point_: *mut C_tensor, + axis_: i64, + quant_min_: i64, + quant_max_: i64, + grad_factor_: f64, + ); + pub fn atg__fake_quantize_learnable_per_tensor_affine( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + scale_: *mut C_tensor, + zero_point_: *mut C_tensor, + quant_min_: i64, + quant_max_: i64, + grad_factor_: f64, + ); + pub fn atg__fake_quantize_learnable_per_tensor_affine_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + self_: *mut C_tensor, + scale_: *mut C_tensor, + zero_point_: *mut C_tensor, + quant_min_: i64, + quant_max_: i64, + grad_factor_: f64, + ); + pub fn atg__fake_quantize_learnable_per_tensor_affine_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + scale_: *mut C_tensor, + zero_point_: *mut C_tensor, + quant_min_: i64, + quant_max_: i64, + grad_factor_: f64, + ); + pub fn atg__fake_quantize_per_tensor_affine_cachemask_tensor_qparams( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + scale_: *mut C_tensor, + zero_point_: *mut C_tensor, + fake_quant_enabled_: *mut C_tensor, + quant_min_: i64, + quant_max_: i64, + ); + pub fn atg__fake_quantize_per_tensor_affine_cachemask_tensor_qparams_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + self_: *mut C_tensor, + scale_: *mut C_tensor, + zero_point_: *mut C_tensor, + fake_quant_enabled_: *mut C_tensor, + quant_min_: i64, + quant_max_: i64, + ); + pub fn atg__fft_c2c( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + normalization_: i64, + forward_: c_int, + ); + pub fn atg__fft_c2c_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + normalization_: i64, + forward_: c_int, + ); + pub fn atg__fft_c2r( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + normalization_: i64, + last_dim_size_: i64, + ); + pub fn atg__fft_c2r_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + normalization_: i64, + last_dim_size_: i64, + ); + pub fn atg__fft_r2c( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + normalization_: i64, + onesided_: c_int, + ); + pub fn atg__fft_r2c_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + normalization_: i64, + onesided_: c_int, + ); + pub fn atg__fill_mem_eff_dropout_mask_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dropout_p_: f64, + seed_: i64, + offset_: i64, + ); + pub fn atg__flash_attention_backward( + out__: *mut *mut C_tensor, + grad_out_: *mut C_tensor, + query_: *mut C_tensor, + key_: *mut C_tensor, + value_: *mut C_tensor, + out_: *mut C_tensor, + logsumexp_: *mut C_tensor, + cum_seq_q_: *mut C_tensor, + cum_seq_k_: *mut C_tensor, + max_q_: i64, + max_k_: i64, + dropout_p_: f64, + is_causal_: c_int, + rng_state_: *mut C_tensor, + unused_: *mut C_tensor, + scale_v: f64, + scale_null: i8, + window_size_left_v: i64, + window_size_left_null: i8, + window_size_right_v: i64, + window_size_right_null: i8, + ); + pub fn atg__foobar( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + arg1_: c_int, + arg2_: c_int, + arg3_: c_int, + ); + pub fn atg__foobar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + arg1_: c_int, + arg2_: c_int, + arg3_: c_int, + ); + pub fn atg__functional_assert_async( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + assert_msg_ptr: *const u8, + assert_msg_len: c_int, + dep_token_: *mut C_tensor, + ); + pub fn atg__functional_assert_scalar( + out__: *mut *mut C_tensor, + self_scalar_: *mut C_scalar, + assert_msg_ptr: *const u8, + assert_msg_len: c_int, + dep_token_: *mut C_tensor, + ); + pub fn atg__functional_sym_constrain_range( + out__: *mut *mut C_tensor, + size_: *mut C_scalar, + min_v: i64, + min_null: i8, + max_v: i64, + max_null: i8, + dep_token_: *mut C_tensor, + ); + pub fn atg__functional_sym_constrain_range_for_size( + out__: *mut *mut C_tensor, + size_: *mut C_scalar, + min_v: i64, + min_null: i8, + max_v: i64, + max_null: i8, + dep_token_: *mut C_tensor, + ); + pub fn atg__fused_dropout(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: f64); + pub fn atg__fused_dropout_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + self_: *mut C_tensor, + p_: f64, + ); + pub fn atg__fused_moving_avg_obs_fq_helper( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + observer_on_: *mut C_tensor, + fake_quant_on_: *mut C_tensor, + running_min_: *mut C_tensor, + running_max_: *mut C_tensor, + scale_: *mut C_tensor, + zero_point_: *mut C_tensor, + averaging_const_: f64, + quant_min_: i64, + quant_max_: i64, + ch_axis_: i64, + per_row_fake_quant_: c_int, + symmetric_quant_: c_int, + ); + pub fn atg__fused_moving_avg_obs_fq_helper_functional( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + observer_on_: *mut C_tensor, + fake_quant_on_: *mut C_tensor, + running_min_: *mut C_tensor, + running_max_: *mut C_tensor, + scale_: *mut C_tensor, + zero_point_: *mut C_tensor, + averaging_const_: f64, + quant_min_: i64, + quant_max_: i64, + ch_axis_: i64, + per_row_fake_quant_: c_int, + symmetric_quant_: c_int, + ); + pub fn atg__fused_moving_avg_obs_fq_helper_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + self_: *mut C_tensor, + observer_on_: *mut C_tensor, + fake_quant_on_: *mut C_tensor, + running_min_: *mut C_tensor, + running_max_: *mut C_tensor, + scale_: *mut C_tensor, + zero_point_: *mut C_tensor, + averaging_const_: f64, + quant_min_: i64, + quant_max_: i64, + ch_axis_: i64, + per_row_fake_quant_: c_int, + symmetric_quant_: c_int, + ); + pub fn atg__fused_rms_norm( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + normalized_shape_data: *const i64, + normalized_shape_len: c_int, + weight_: *mut C_tensor, + eps_v: f64, + eps_null: i8, + ); + pub fn atg__fused_sdp_choice( + query_: *mut C_tensor, + key_: *mut C_tensor, + value_: *mut C_tensor, + attn_mask_: *mut C_tensor, + dropout_p_: f64, + is_causal_: c_int, + scale_v: f64, + scale_null: i8, + enable_gqa_: c_int, + ) -> i64; + pub fn atg__fw_primal(out__: *mut *mut C_tensor, self_: *mut C_tensor, level_: i64); + pub fn atg__fw_primal_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor, level_: i64); + pub fn atg__fw_primal_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + level_: i64, + ); + pub fn atg__gather_sparse_backward( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + grad_: *mut C_tensor, + ); + pub fn atg__grid_sampler_2d_cpu_fallback( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + grid_: *mut C_tensor, + interpolation_mode_: i64, + padding_mode_: i64, + align_corners_: c_int, + ); + pub fn atg__grid_sampler_2d_cpu_fallback_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + input_: *mut C_tensor, + grid_: *mut C_tensor, + interpolation_mode_: i64, + padding_mode_: i64, + align_corners_: c_int, + ); + pub fn atg__grid_sampler_2d_cpu_fallback_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + input_: *mut C_tensor, + grid_: *mut C_tensor, + interpolation_mode_: i64, + padding_mode_: i64, + align_corners_: c_int, + ); + pub fn atg__grouped_mm( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mat2_: *mut C_tensor, + offs_: *mut C_tensor, + bias_: *mut C_tensor, + out_dtype_: c_int, + ); + pub fn atg__has_compatible_shallow_copy_type( + self_: *mut C_tensor, + from_: *mut C_tensor, + ) -> c_int; + pub fn atg__has_same_storage_numel(self_: *mut C_tensor, other_: *mut C_tensor) -> c_int; + pub fn atg__histogramdd_bin_edges( + self_: *mut C_tensor, + bins_data: *const i64, + bins_len: c_int, + range_data: *const f64, + range_len: c_int, + weight_: *mut C_tensor, + density_: c_int, + ) -> *mut *mut C_tensor; + pub fn atg__histogramdd_bin_edges_out( + out_data: *const *mut C_tensor, + out_len: c_int, + self_: *mut C_tensor, + bins_data: *const i64, + bins_len: c_int, + range_data: *const f64, + range_len: c_int, + weight_: *mut C_tensor, + density_: c_int, + ); + pub fn atg__histogramdd_from_bin_cts( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + bins_data: *const i64, + bins_len: c_int, + range_data: *const f64, + range_len: c_int, + weight_: *mut C_tensor, + density_: c_int, + ); + pub fn atg__histogramdd_from_bin_cts_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + bins_data: *const i64, + bins_len: c_int, + range_data: *const f64, + range_len: c_int, + weight_: *mut C_tensor, + density_: c_int, + ); + pub fn atg__histogramdd_from_bin_tensors( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + bins_data: *const *mut C_tensor, + bins_len: c_int, + weight_: *mut C_tensor, + density_: c_int, + ); + pub fn atg__histogramdd_from_bin_tensors_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + bins_data: *const *mut C_tensor, + bins_len: c_int, + weight_: *mut C_tensor, + density_: c_int, + ); + pub fn atg__index_put_impl( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + indices_data: *const *mut C_tensor, + indices_len: c_int, + values_: *mut C_tensor, + accumulate_: c_int, + unsafe_: c_int, + ); + pub fn atg__index_put_impl_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + indices_data: *const *mut C_tensor, + indices_len: c_int, + values_: *mut C_tensor, + accumulate_: c_int, + unsafe_: c_int, + ); + pub fn atg__index_put_impl_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + indices_data: *const *mut C_tensor, + indices_len: c_int, + values_: *mut C_tensor, + accumulate_: c_int, + unsafe_: c_int, + ); + pub fn atg__indices(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__indices_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__indices_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg__int_mm(out__: *mut *mut C_tensor, self_: *mut C_tensor, mat2_: *mut C_tensor); + pub fn atg__int_mm_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mat2_: *mut C_tensor, + ); + pub fn atg__is_all_true(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__is_any_true(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__is_zerotensor(self_: *mut C_tensor) -> c_int; + pub fn atg__lazy_clone(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__linalg_check_errors( + info_: *mut C_tensor, + api_name_ptr: *const u8, + api_name_len: c_int, + is_matrix_: c_int, + ); + pub fn atg__linalg_det(out__: *mut *mut C_tensor, A_: *mut C_tensor); + pub fn atg__linalg_det_result( + out__: *mut *mut C_tensor, + result_: *mut C_tensor, + LU_: *mut C_tensor, + pivots_: *mut C_tensor, + A_: *mut C_tensor, + ); + pub fn atg__linalg_eigh( + out__: *mut *mut C_tensor, + A_: *mut C_tensor, + UPLO_ptr: *const u8, + UPLO_len: c_int, + compute_v_: c_int, + ); + pub fn atg__linalg_eigh_eigenvalues( + out__: *mut *mut C_tensor, + eigenvalues_: *mut C_tensor, + eigenvectors_: *mut C_tensor, + A_: *mut C_tensor, + UPLO_ptr: *const u8, + UPLO_len: c_int, + compute_v_: c_int, + ); + pub fn atg__linalg_eigvals(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__linalg_slogdet(out__: *mut *mut C_tensor, A_: *mut C_tensor); + pub fn atg__linalg_slogdet_sign( + out__: *mut *mut C_tensor, + sign_: *mut C_tensor, + logabsdet_: *mut C_tensor, + LU_: *mut C_tensor, + pivots_: *mut C_tensor, + A_: *mut C_tensor, + ); + pub fn atg__linalg_solve_ex( + out__: *mut *mut C_tensor, + A_: *mut C_tensor, + B_: *mut C_tensor, + left_: c_int, + check_errors_: c_int, + ); + pub fn atg__linalg_solve_ex_result( + out__: *mut *mut C_tensor, + result_: *mut C_tensor, + LU_: *mut C_tensor, + pivots_: *mut C_tensor, + info_: *mut C_tensor, + A_: *mut C_tensor, + B_: *mut C_tensor, + left_: c_int, + check_errors_: c_int, + ); + pub fn atg__linalg_svd( + out__: *mut *mut C_tensor, + A_: *mut C_tensor, + full_matrices_: c_int, + compute_uv_: c_int, + driver_ptr: *const u8, + driver_len: c_int, + ); + pub fn atg__linalg_svd_u( + out__: *mut *mut C_tensor, + U_: *mut C_tensor, + S_: *mut C_tensor, + Vh_: *mut C_tensor, + A_: *mut C_tensor, + full_matrices_: c_int, + compute_uv_: c_int, + driver_ptr: *const u8, + driver_len: c_int, + ); + pub fn atg__log_softmax( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + half_to_float_: c_int, + ); + pub fn atg__log_softmax_backward_data( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + output_: *mut C_tensor, + dim_: i64, + input_dtype_: c_int, + ); + pub fn atg__log_softmax_backward_data_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_output_: *mut C_tensor, + output_: *mut C_tensor, + dim_: i64, + input_dtype_: c_int, + ); + pub fn atg__log_softmax_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + half_to_float_: c_int, + ); + pub fn atg__logcumsumexp(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64); + pub fn atg__logcumsumexp_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + ); + pub fn atg__lstm_mps( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + hx_data: *const *mut C_tensor, + hx_len: c_int, + params_data: *const *mut C_tensor, + params_len: c_int, + has_biases_: c_int, + num_layers_: i64, + dropout_: f64, + train_: c_int, + bidirectional_: c_int, + batch_first_: c_int, + ); + pub fn atg__lstm_mps_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + out2_: *mut C_tensor, + out3_: *mut C_tensor, + out4_: *mut C_tensor, + out5_: *mut C_tensor, + input_: *mut C_tensor, + hx_data: *const *mut C_tensor, + hx_len: c_int, + params_data: *const *mut C_tensor, + params_len: c_int, + has_biases_: c_int, + num_layers_: i64, + dropout_: f64, + train_: c_int, + bidirectional_: c_int, + batch_first_: c_int, + ); + pub fn atg__lu_with_info( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + pivot_: c_int, + check_errors_: c_int, + ); + pub fn atg__make_dep_token( + out__: *mut *mut C_tensor, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg__make_dual( + out__: *mut *mut C_tensor, + primal_: *mut C_tensor, + tangent_: *mut C_tensor, + level_: i64, + ); + pub fn atg__make_dual_copy( + out__: *mut *mut C_tensor, + primal_: *mut C_tensor, + tangent_: *mut C_tensor, + level_: i64, + ); + pub fn atg__make_dual_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + primal_: *mut C_tensor, + tangent_: *mut C_tensor, + level_: i64, + ); + pub fn atg__make_per_channel_quantized_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + scale_: *mut C_tensor, + zero_point_: *mut C_tensor, + axis_: i64, + ); + pub fn atg__make_per_channel_quantized_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + scale_: *mut C_tensor, + zero_point_: *mut C_tensor, + axis_: i64, + ); + pub fn atg__make_per_tensor_quantized_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + scale_: f64, + zero_point_: i64, + ); + pub fn atg__make_per_tensor_quantized_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + scale_: f64, + zero_point_: i64, + ); + pub fn atg__masked_scale( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mask_: *mut C_tensor, + scale_: f64, + ); + pub fn atg__masked_scale_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mask_: *mut C_tensor, + scale_: f64, + ); + pub fn atg__masked_softmax( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mask_: *mut C_tensor, + dim_v: i64, + dim_null: i8, + mask_type_v: i64, + mask_type_null: i8, + ); + pub fn atg__masked_softmax_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + output_: *mut C_tensor, + mask_: *mut C_tensor, + dim_v: i64, + dim_null: i8, + ); + pub fn atg__masked_softmax_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_output_: *mut C_tensor, + output_: *mut C_tensor, + mask_: *mut C_tensor, + dim_v: i64, + dim_null: i8, + ); + pub fn atg__masked_softmax_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mask_: *mut C_tensor, + dim_v: i64, + dim_null: i8, + mask_type_v: i64, + mask_type_null: i8, + ); + pub fn atg__mixed_dtypes_linear( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + scale_: *mut C_tensor, + bias_: *mut C_tensor, + activation_ptr: *const u8, + activation_len: c_int, + ); + pub fn atg__mkldnn_reshape( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + shape_data: *const i64, + shape_len: c_int, + ); + pub fn atg__mkldnn_reshape_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + shape_data: *const i64, + shape_len: c_int, + ); + pub fn atg__mkldnn_transpose( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim0_: i64, + dim1_: i64, + ); + pub fn atg__mkldnn_transpose_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim0_: i64, + dim1_: i64, + ); + pub fn atg__mkldnn_transpose_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim0_: i64, + dim1_: i64, + ); + pub fn atg__mps_convolution( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + ); + pub fn atg__mps_convolution_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + ); + pub fn atg__mps_convolution_transpose( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + output_padding_data: *const i64, + output_padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + ); + pub fn atg__mps_convolution_transpose_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + output_padding_data: *const i64, + output_padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + ); + pub fn atg__native_batch_norm_legit( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + training_: c_int, + momentum_: f64, + eps_: f64, + ); + pub fn atg__native_batch_norm_legit_functional( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + training_: c_int, + momentum_: f64, + eps_: f64, + ); + pub fn atg__native_batch_norm_legit_no_stats( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + training_: c_int, + momentum_: f64, + eps_: f64, + ); + pub fn atg__native_batch_norm_legit_no_stats_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + save_mean_: *mut C_tensor, + save_invstd_: *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + training_: c_int, + momentum_: f64, + eps_: f64, + ); + pub fn atg__native_batch_norm_legit_no_training( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + momentum_: f64, + eps_: f64, + ); + pub fn atg__native_batch_norm_legit_no_training_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + out2_: *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + momentum_: f64, + eps_: f64, + ); + pub fn atg__native_batch_norm_legit_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + save_mean_: *mut C_tensor, + save_invstd_: *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + training_: c_int, + momentum_: f64, + eps_: f64, + ); + pub fn atg__native_multi_head_attention( + out__: *mut *mut C_tensor, + query_: *mut C_tensor, + key_: *mut C_tensor, + value_: *mut C_tensor, + embed_dim_: i64, + num_head_: i64, + qkv_weight_: *mut C_tensor, + qkv_bias_: *mut C_tensor, + proj_weight_: *mut C_tensor, + proj_bias_: *mut C_tensor, + mask_: *mut C_tensor, + need_weights_: c_int, + average_attn_weights_: c_int, + mask_type_v: i64, + mask_type_null: i8, + ); + pub fn atg__native_multi_head_attention_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + query_: *mut C_tensor, + key_: *mut C_tensor, + value_: *mut C_tensor, + embed_dim_: i64, + num_head_: i64, + qkv_weight_: *mut C_tensor, + qkv_bias_: *mut C_tensor, + proj_weight_: *mut C_tensor, + proj_bias_: *mut C_tensor, + mask_: *mut C_tensor, + need_weights_: c_int, + average_attn_weights_: c_int, + mask_type_v: i64, + mask_type_null: i8, + ); + pub fn atg__neg_view(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__neg_view_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__neg_view_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg__nested_compute_contiguous_strides_offsets( + out__: *mut *mut C_tensor, + nested_size_: *mut C_tensor, + ); + pub fn atg__nested_from_padded( + out__: *mut *mut C_tensor, + padded_: *mut C_tensor, + cpu_nested_shape_example_: *mut C_tensor, + fuse_transform_0213_: c_int, + ); + pub fn atg__nested_from_padded_and_nested_example( + out__: *mut *mut C_tensor, + padded_: *mut C_tensor, + nt_example_: *mut C_tensor, + ); + pub fn atg__nested_from_padded_and_nested_example_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + padded_: *mut C_tensor, + nt_example_: *mut C_tensor, + ); + pub fn atg__nested_from_padded_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + padded_: *mut C_tensor, + cpu_nested_shape_example_: *mut C_tensor, + fuse_transform_0213_: c_int, + ); + pub fn atg__nested_from_padded_tensor( + out__: *mut *mut C_tensor, + padded_: *mut C_tensor, + offsets_: *mut C_tensor, + dummy_: *mut C_tensor, + ragged_idx_: i64, + min_seqlen_: *mut C_tensor, + max_seqlen_: *mut C_tensor, + sum_S_v: i64, + sum_S_null: i8, + ); + pub fn atg__nested_get_jagged_dummy(out__: *mut *mut C_tensor, any_: *mut C_tensor); + pub fn atg__nested_get_lengths(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__nested_get_max_seqlen(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__nested_get_min_seqlen(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__nested_get_offsets(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__nested_get_ragged_idx(self_: *mut C_tensor) -> i64; + pub fn atg__nested_get_values(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__nested_get_values_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__nested_get_values_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg__nested_select_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: i64, + ); + pub fn atg__nested_sum_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + ); + pub fn atg__nested_view_from_buffer( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + nested_size_: *mut C_tensor, + nested_strides_: *mut C_tensor, + offsets_: *mut C_tensor, + ); + pub fn atg__nested_view_from_buffer_copy( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + nested_size_: *mut C_tensor, + nested_strides_: *mut C_tensor, + offsets_: *mut C_tensor, + ); + pub fn atg__nested_view_from_buffer_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + nested_size_: *mut C_tensor, + nested_strides_: *mut C_tensor, + offsets_: *mut C_tensor, + ); + pub fn atg__nested_view_from_jagged( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + offsets_: *mut C_tensor, + dummy_: *mut C_tensor, + lengths_: *mut C_tensor, + ragged_idx_: i64, + min_seqlen_: *mut C_tensor, + max_seqlen_: *mut C_tensor, + ); + pub fn atg__nested_view_from_jagged_copy( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + offsets_: *mut C_tensor, + dummy_: *mut C_tensor, + lengths_: *mut C_tensor, + ragged_idx_: i64, + min_seqlen_: *mut C_tensor, + max_seqlen_: *mut C_tensor, + ); + pub fn atg__nested_view_from_jagged_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + offsets_: *mut C_tensor, + dummy_: *mut C_tensor, + lengths_: *mut C_tensor, + ragged_idx_: i64, + min_seqlen_: *mut C_tensor, + max_seqlen_: *mut C_tensor, + ); + pub fn atg__new_zeros_with_same_feature_meta( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + self_num_batch_dims_: i64, + ); + pub fn atg__new_zeros_with_same_feature_meta_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + self_num_batch_dims_: i64, + ); + pub fn atg__nnpack_available() -> c_int; + pub fn atg__nnpack_spatial_convolution( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + ); + pub fn atg__nnpack_spatial_convolution_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + ); + pub fn atg__nnz(self_: *mut C_tensor) -> i64; + pub fn atg__pack_padded_sequence( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + lengths_: *mut C_tensor, + batch_first_: c_int, + ); + pub fn atg__pack_padded_sequence_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + input_size_data: *const i64, + input_size_len: c_int, + batch_sizes_: *mut C_tensor, + batch_first_: c_int, + ); + pub fn atg__pack_padded_sequence_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + input_: *mut C_tensor, + lengths_: *mut C_tensor, + batch_first_: c_int, + ); + pub fn atg__pad_circular( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + pad_data: *const i64, + pad_len: c_int, + ); + pub fn atg__pad_enum( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + pad_data: *const i64, + pad_len: c_int, + mode_: i64, + value_v: f64, + value_null: i8, + ); + pub fn atg__pad_packed_sequence( + out__: *mut *mut C_tensor, + data_: *mut C_tensor, + batch_sizes_: *mut C_tensor, + batch_first_: c_int, + padding_value_: *mut C_scalar, + total_length_: i64, + ); + pub fn atg__pdist_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + self_: *mut C_tensor, + p_: f64, + pdist_: *mut C_tensor, + ); + pub fn atg__pdist_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_: *mut C_tensor, + self_: *mut C_tensor, + p_: f64, + pdist_: *mut C_tensor, + ); + pub fn atg__pin_memory(out__: *mut *mut C_tensor, self_: *mut C_tensor, device_: c_int); + pub fn atg__pin_memory_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + device_: c_int, + ); + pub fn atg__prelu_kernel( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + ); + pub fn atg__prelu_kernel_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + ); + pub fn atg__print(s_ptr: *const u8, s_len: c_int); + pub fn atg__propagate_xla_data(input_: *mut C_tensor, output_: *mut C_tensor); + pub fn atg__remove_batch_dim( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + level_: i64, + batch_size_: i64, + out_dim_: i64, + ); + pub fn atg__reshape_alias( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + ); + pub fn atg__reshape_alias_copy( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + ); + pub fn atg__reshape_alias_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + ); + pub fn atg__reshape_copy( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg__reshape_from_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + shape_: *mut C_tensor, + ); + pub fn atg__resize_output( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + device_: c_int, + ); + pub fn atg__resize_output_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + device_: c_int, + ); + pub fn atg__resize_output_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + device_: c_int, + ); + pub fn atg__rowwise_prune( + out__: *mut *mut C_tensor, + weight_: *mut C_tensor, + mask_: *mut C_tensor, + compressed_indices_dtype_: c_int, + ); + pub fn atg__safe_softmax( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + dtype_: c_int, + ); + pub fn atg__sample_dirichlet(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__sample_dirichlet_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg__saturate_weight_to_fp16(out__: *mut *mut C_tensor, weight_: *mut C_tensor); + pub fn atg__scaled_dot_product_attention_math( + out__: *mut *mut C_tensor, + query_: *mut C_tensor, + key_: *mut C_tensor, + value_: *mut C_tensor, + attn_mask_: *mut C_tensor, + dropout_p_: f64, + is_causal_: c_int, + dropout_mask_: *mut C_tensor, + scale_v: f64, + scale_null: i8, + enable_gqa_: c_int, + ); + pub fn atg__scaled_dot_product_attention_math_for_mps( + out__: *mut *mut C_tensor, + query_: *mut C_tensor, + key_: *mut C_tensor, + value_: *mut C_tensor, + attn_mask_: *mut C_tensor, + dropout_p_: f64, + is_causal_: c_int, + dropout_mask_: *mut C_tensor, + scale_v: f64, + scale_null: i8, + ); + pub fn atg__scaled_dot_product_cudnn_attention_backward( + out__: *mut *mut C_tensor, + grad_out_: *mut C_tensor, + query_: *mut C_tensor, + key_: *mut C_tensor, + value_: *mut C_tensor, + out_: *mut C_tensor, + logsumexp_: *mut C_tensor, + philox_seed_: *mut C_tensor, + philox_offset_: *mut C_tensor, + attn_bias_: *mut C_tensor, + cum_seq_q_: *mut C_tensor, + cum_seq_k_: *mut C_tensor, + max_q_: i64, + max_k_: i64, + dropout_p_: f64, + is_causal_: c_int, + scale_v: f64, + scale_null: i8, + ); + pub fn atg__scaled_dot_product_efficient_attention( + out__: *mut *mut C_tensor, + query_: *mut C_tensor, + key_: *mut C_tensor, + value_: *mut C_tensor, + attn_bias_: *mut C_tensor, + compute_log_sumexp_: c_int, + dropout_p_: f64, + is_causal_: c_int, + scale_v: f64, + scale_null: i8, + ); + pub fn atg__scaled_dot_product_flash_attention_backward( + out__: *mut *mut C_tensor, + grad_out_: *mut C_tensor, + query_: *mut C_tensor, + key_: *mut C_tensor, + value_: *mut C_tensor, + out_: *mut C_tensor, + logsumexp_: *mut C_tensor, + cum_seq_q_: *mut C_tensor, + cum_seq_k_: *mut C_tensor, + max_q_: i64, + max_k_: i64, + dropout_p_: f64, + is_causal_: c_int, + philox_seed_: *mut C_tensor, + philox_offset_: *mut C_tensor, + scale_v: f64, + scale_null: i8, + ); + pub fn atg__scaled_dot_product_flash_attention_for_cpu( + out__: *mut *mut C_tensor, + query_: *mut C_tensor, + key_: *mut C_tensor, + value_: *mut C_tensor, + dropout_p_: f64, + is_causal_: c_int, + attn_mask_: *mut C_tensor, + scale_v: f64, + scale_null: i8, + ); + pub fn atg__scaled_dot_product_flash_attention_for_cpu_backward( + out__: *mut *mut C_tensor, + grad_out_: *mut C_tensor, + query_: *mut C_tensor, + key_: *mut C_tensor, + value_: *mut C_tensor, + out_: *mut C_tensor, + logsumexp_: *mut C_tensor, + dropout_p_: f64, + is_causal_: c_int, + attn_mask_: *mut C_tensor, + scale_v: f64, + scale_null: i8, + ); + pub fn atg__scaled_grouped_mm( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mat2_: *mut C_tensor, + scale_a_: *mut C_tensor, + scale_b_: *mut C_tensor, + offs_: *mut C_tensor, + bias_: *mut C_tensor, + scale_result_: *mut C_tensor, + out_dtype_: c_int, + use_fast_accum_: c_int, + ); + pub fn atg__scaled_grouped_mm_v2( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mat2_: *mut C_tensor, + scale_a_data: *const *mut C_tensor, + scale_a_len: c_int, + recipe_a_data: *const i64, + recipe_a_len: c_int, + swizzle_a_data: *const i64, + swizzle_a_len: c_int, + scale_b_data: *const *mut C_tensor, + scale_b_len: c_int, + recipe_b_data: *const i64, + recipe_b_len: c_int, + swizzle_b_data: *const i64, + swizzle_b_len: c_int, + offs_: *mut C_tensor, + bias_: *mut C_tensor, + out_dtype_: c_int, + contraction_dim_data: *const i64, + contraction_dim_len: c_int, + use_fast_accum_: c_int, + ); + pub fn atg__scaled_mm( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mat2_: *mut C_tensor, + scale_a_: *mut C_tensor, + scale_b_: *mut C_tensor, + bias_: *mut C_tensor, + scale_result_: *mut C_tensor, + out_dtype_: c_int, + use_fast_accum_: c_int, + ); + pub fn atg__scaled_mm_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mat2_: *mut C_tensor, + scale_a_: *mut C_tensor, + scale_b_: *mut C_tensor, + bias_: *mut C_tensor, + scale_result_: *mut C_tensor, + out_dtype_: c_int, + use_fast_accum_: c_int, + ); + pub fn atg__scaled_mm_v2( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mat2_: *mut C_tensor, + scale_a_data: *const *mut C_tensor, + scale_a_len: c_int, + recipe_a_data: *const i64, + recipe_a_len: c_int, + swizzle_a_data: *const i64, + swizzle_a_len: c_int, + scale_b_data: *const *mut C_tensor, + scale_b_len: c_int, + recipe_b_data: *const i64, + recipe_b_len: c_int, + swizzle_b_data: *const i64, + swizzle_b_len: c_int, + bias_: *mut C_tensor, + out_dtype_: c_int, + contraction_dim_data: *const i64, + contraction_dim_len: c_int, + use_fast_accum_: c_int, + ); + pub fn atg__scaled_mm_v2_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mat2_: *mut C_tensor, + scale_a_data: *const *mut C_tensor, + scale_a_len: c_int, + recipe_a_data: *const i64, + recipe_a_len: c_int, + swizzle_a_data: *const i64, + swizzle_a_len: c_int, + scale_b_data: *const *mut C_tensor, + scale_b_len: c_int, + recipe_b_data: *const i64, + recipe_b_len: c_int, + swizzle_b_data: *const i64, + swizzle_b_len: c_int, + bias_: *mut C_tensor, + out_dtype_: c_int, + contraction_dim_data: *const i64, + contraction_dim_len: c_int, + use_fast_accum_: c_int, + ); + pub fn atg__scatter_reduce( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + src_: *mut C_tensor, + reduce_ptr: *const u8, + reduce_len: c_int, + include_self_: c_int, + ); + pub fn atg__scatter_reduce_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + src_: *mut C_tensor, + reduce_ptr: *const u8, + reduce_len: c_int, + include_self_: c_int, + ); + pub fn atg__scatter_reduce_two_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + src_: *mut C_tensor, + reduce_ptr: *const u8, + reduce_len: c_int, + include_self_: c_int, + ); + pub fn atg__segment_reduce_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + output_: *mut C_tensor, + data_: *mut C_tensor, + reduce_ptr: *const u8, + reduce_len: c_int, + lengths_: *mut C_tensor, + offsets_: *mut C_tensor, + axis_: i64, + initial_: *mut C_scalar, + ); + pub fn atg__segment_reduce_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_: *mut C_tensor, + output_: *mut C_tensor, + data_: *mut C_tensor, + reduce_ptr: *const u8, + reduce_len: c_int, + lengths_: *mut C_tensor, + offsets_: *mut C_tensor, + axis_: i64, + initial_: *mut C_scalar, + ); + pub fn atg__shape_as_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__slow_conv2d_backward( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_weight_: *mut C_tensor, + grad_bias_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg__sobol_engine_draw( + out__: *mut *mut C_tensor, + quasi_: *mut C_tensor, + n_: i64, + sobolstate_: *mut C_tensor, + dimension_: i64, + num_generated_: i64, + dtype_: c_int, + ); + pub fn atg__sobol_engine_ff_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + n_: i64, + sobolstate_: *mut C_tensor, + dimension_: i64, + num_generated_: i64, + ); + pub fn atg__sobol_engine_initialize_state_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dimension_: i64, + ); + pub fn atg__sobol_engine_scramble_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + ltm_: *mut C_tensor, + dimension_: i64, + ); + pub fn atg__softmax( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + half_to_float_: c_int, + ); + pub fn atg__softmax_backward_data( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + output_: *mut C_tensor, + dim_: i64, + input_dtype_: c_int, + ); + pub fn atg__softmax_backward_data_out( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + output_: *mut C_tensor, + dim_: i64, + input_dtype_: c_int, + ); + pub fn atg__softmax_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + half_to_float_: c_int, + ); + pub fn atg__sparse_addmm( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mat1_: *mut C_tensor, + mat2_: *mut C_tensor, + ); + pub fn atg__sparse_addmm_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mat1_: *mut C_tensor, + mat2_: *mut C_tensor, + ); + pub fn atg__sparse_broadcast_to( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg__sparse_broadcast_to_copy( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg__sparse_broadcast_to_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg__sparse_bsc_tensor_unsafe( + out__: *mut *mut C_tensor, + ccol_indices_: *mut C_tensor, + row_indices_: *mut C_tensor, + values_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg__sparse_bsr_tensor_unsafe( + out__: *mut *mut C_tensor, + crow_indices_: *mut C_tensor, + col_indices_: *mut C_tensor, + values_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg__sparse_compressed_tensor_unsafe( + out__: *mut *mut C_tensor, + compressed_indices_: *mut C_tensor, + plain_indices_: *mut C_tensor, + values_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg__sparse_compressed_tensor_with_dims( + out__: *mut *mut C_tensor, + nnz_: i64, + dense_dim_: i64, + size_data: *const i64, + size_len: c_int, + blocksize_data: *const i64, + blocksize_len: c_int, + index_dtype_: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg__sparse_coo_tensor_unsafe( + out__: *mut *mut C_tensor, + indices_: *mut C_tensor, + values_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + is_coalesced_: c_int, + ); + pub fn atg__sparse_coo_tensor_with_dims( + out__: *mut *mut C_tensor, + sparse_dim_: i64, + dense_dim_: i64, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg__sparse_coo_tensor_with_dims_and_tensors( + out__: *mut *mut C_tensor, + sparse_dim_: i64, + dense_dim_: i64, + size_data: *const i64, + size_len: c_int, + indices_: *mut C_tensor, + values_: *mut C_tensor, + options_kind: c_int, + options_device: c_int, + is_coalesced_: c_int, + ); + pub fn atg__sparse_coo_tensor_with_dims_and_tensors_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + sparse_dim_: i64, + dense_dim_: i64, + size_data: *const i64, + size_len: c_int, + indices_: *mut C_tensor, + values_: *mut C_tensor, + is_coalesced_: c_int, + ); + pub fn atg__sparse_coo_tensor_with_dims_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + sparse_dim_: i64, + dense_dim_: i64, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg__sparse_csc_tensor_unsafe( + out__: *mut *mut C_tensor, + ccol_indices_: *mut C_tensor, + row_indices_: *mut C_tensor, + values_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg__sparse_csr_prod( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + dtype_: c_int, + ); + pub fn atg__sparse_csr_prod_dim_dtype_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + dtype_: c_int, + ); + pub fn atg__sparse_csr_sum( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + dtype_: c_int, + ); + pub fn atg__sparse_csr_sum_dim_dtype_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + dtype_: c_int, + ); + pub fn atg__sparse_csr_tensor_unsafe( + out__: *mut *mut C_tensor, + crow_indices_: *mut C_tensor, + col_indices_: *mut C_tensor, + values_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg__sparse_log_softmax( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + half_to_float_: c_int, + ); + pub fn atg__sparse_log_softmax_backward_data( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + output_: *mut C_tensor, + dim_: i64, + self_: *mut C_tensor, + ); + pub fn atg__sparse_log_softmax_backward_data_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_output_: *mut C_tensor, + output_: *mut C_tensor, + dim_: i64, + self_: *mut C_tensor, + ); + pub fn atg__sparse_log_softmax_int( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + dtype_: c_int, + ); + pub fn atg__sparse_log_softmax_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + half_to_float_: c_int, + ); + pub fn atg__sparse_mask_projection( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mask_: *mut C_tensor, + accumulate_matches_: c_int, + ); + pub fn atg__sparse_mask_projection_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mask_: *mut C_tensor, + accumulate_matches_: c_int, + ); + pub fn atg__sparse_mm(out__: *mut *mut C_tensor, sparse_: *mut C_tensor, dense_: *mut C_tensor); + pub fn atg__sparse_mm_reduce( + out__: *mut *mut C_tensor, + sparse_: *mut C_tensor, + dense_: *mut C_tensor, + reduce_ptr: *const u8, + reduce_len: c_int, + ); + pub fn atg__sparse_mm_reduce_impl( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + reduce_ptr: *const u8, + reduce_len: c_int, + ); + pub fn atg__sparse_semi_structured_apply( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + thread_masks_: *mut C_tensor, + ); + pub fn atg__sparse_semi_structured_apply_dense( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + thread_masks_: *mut C_tensor, + ); + pub fn atg__sparse_semi_structured_linear( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + meta_: *mut C_tensor, + bias_: *mut C_tensor, + activation_ptr: *const u8, + activation_len: c_int, + out_dtype_: c_int, + ); + pub fn atg__sparse_semi_structured_mm( + out__: *mut *mut C_tensor, + mat1_: *mut C_tensor, + mat1_meta_: *mut C_tensor, + mat2_: *mut C_tensor, + out_dtype_: c_int, + ); + pub fn atg__sparse_semi_structured_tile( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + algorithm_ptr: *const u8, + algorithm_len: c_int, + use_cutlass_: c_int, + ); + pub fn atg__sparse_softmax( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + half_to_float_: c_int, + ); + pub fn atg__sparse_softmax_backward_data( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + output_: *mut C_tensor, + dim_: i64, + self_: *mut C_tensor, + ); + pub fn atg__sparse_softmax_backward_data_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_output_: *mut C_tensor, + output_: *mut C_tensor, + dim_: i64, + self_: *mut C_tensor, + ); + pub fn atg__sparse_softmax_int( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + dtype_: c_int, + ); + pub fn atg__sparse_softmax_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + half_to_float_: c_int, + ); + pub fn atg__sparse_sparse_matmul( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg__sparse_sparse_matmul_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg__sparse_sum(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__sparse_sum_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + ); + pub fn atg__sparse_sum_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + ); + pub fn atg__sparse_sum_dim( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + ); + pub fn atg__sparse_sum_dim_dtype( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + dtype_: c_int, + ); + pub fn atg__sparse_sum_dim_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + ); + pub fn atg__sparse_sum_dtype(out__: *mut *mut C_tensor, self_: *mut C_tensor, dtype_: c_int); + pub fn atg__spdiags( + out__: *mut *mut C_tensor, + diagonals_: *mut C_tensor, + offsets_: *mut C_tensor, + shape_data: *const i64, + shape_len: c_int, + layout_: i8, + ); + pub fn atg__spdiags_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + diagonals_: *mut C_tensor, + offsets_: *mut C_tensor, + shape_data: *const i64, + shape_len: c_int, + layout_: i8, + ); + pub fn atg__spsolve( + out__: *mut *mut C_tensor, + A_: *mut C_tensor, + B_: *mut C_tensor, + left_: c_int, + ); + pub fn atg__stack( + out__: *mut *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + dim_: i64, + ); + pub fn atg__stack_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + dim_: i64, + ); + pub fn atg__standard_gamma(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__standard_gamma_grad( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_: *mut C_tensor, + ); + pub fn atg__standard_gamma_grad_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + output_: *mut C_tensor, + ); + pub fn atg__standard_gamma_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg__test_ambiguous_defaults( + out__: *mut *mut C_tensor, + dummy_: *mut C_tensor, + a_: i64, + b_: i64, + ); + pub fn atg__test_ambiguous_defaults_b( + out__: *mut *mut C_tensor, + dummy_: *mut C_tensor, + a_: i64, + b_ptr: *const u8, + b_len: c_int, + ); + pub fn atg__test_autograd_multiple_dispatch(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__test_autograd_multiple_dispatch_fullcoverage_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg__test_autograd_multiple_dispatch_ntonly( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + b_: c_int, + ); + pub fn atg__test_autograd_multiple_dispatch_view( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg__test_autograd_multiple_dispatch_view_copy( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg__test_autograd_multiple_dispatch_view_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg__test_check_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__test_functorch_fallback( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg__test_functorch_fallback_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg__test_optional_filled_intlist( + out__: *mut *mut C_tensor, + values_: *mut C_tensor, + addends_data: *const i64, + addends_len: c_int, + ); + pub fn atg__test_optional_filled_intlist_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + values_: *mut C_tensor, + addends_data: *const i64, + addends_len: c_int, + ); + pub fn atg__test_optional_floatlist( + out__: *mut *mut C_tensor, + values_: *mut C_tensor, + addends_data: *const f64, + addends_len: c_int, + ); + pub fn atg__test_optional_floatlist_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + values_: *mut C_tensor, + addends_data: *const f64, + addends_len: c_int, + ); + pub fn atg__test_optional_intlist( + out__: *mut *mut C_tensor, + values_: *mut C_tensor, + addends_data: *const i64, + addends_len: c_int, + ); + pub fn atg__test_optional_intlist_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + values_: *mut C_tensor, + addends_data: *const i64, + addends_len: c_int, + ); + pub fn atg__test_parallel_materialize( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + num_parallel_: i64, + skip_first_: c_int, + ); + pub fn atg__test_serialization_subcmul( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg__test_string_default( + out__: *mut *mut C_tensor, + dummy_: *mut C_tensor, + a_ptr: *const u8, + a_len: c_int, + b_ptr: *const u8, + b_len: c_int, + ); + pub fn atg__test_warn_in_autograd(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__test_warn_in_autograd_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg__to_copy( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + options_kind: c_int, + options_device: c_int, + non_blocking_: c_int, + ); + pub fn atg__to_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + non_blocking_: c_int, + ); + pub fn atg__to_cpu( + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ) -> *mut *mut C_tensor; + pub fn atg__to_dense( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dtype_: c_int, + masked_grad_: c_int, + ); + pub fn atg__to_dense_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dtype_: c_int, + masked_grad_: c_int, + ); + pub fn atg__to_sparse( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + layout_: i8, + blocksize_data: *const i64, + blocksize_len: c_int, + dense_dim_v: i64, + dense_dim_null: i8, + ); + pub fn atg__to_sparse_bsc( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + blocksize_data: *const i64, + blocksize_len: c_int, + dense_dim_v: i64, + dense_dim_null: i8, + ); + pub fn atg__to_sparse_bsc_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + blocksize_data: *const i64, + blocksize_len: c_int, + dense_dim_v: i64, + dense_dim_null: i8, + ); + pub fn atg__to_sparse_bsr( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + blocksize_data: *const i64, + blocksize_len: c_int, + dense_dim_v: i64, + dense_dim_null: i8, + ); + pub fn atg__to_sparse_bsr_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + blocksize_data: *const i64, + blocksize_len: c_int, + dense_dim_v: i64, + dense_dim_null: i8, + ); + pub fn atg__to_sparse_csc( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dense_dim_v: i64, + dense_dim_null: i8, + ); + pub fn atg__to_sparse_csc_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dense_dim_v: i64, + dense_dim_null: i8, + ); + pub fn atg__to_sparse_csr( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dense_dim_v: i64, + dense_dim_null: i8, + ); + pub fn atg__to_sparse_csr_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dense_dim_v: i64, + dense_dim_null: i8, + ); + pub fn atg__to_sparse_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + layout_: i8, + blocksize_data: *const i64, + blocksize_len: c_int, + dense_dim_v: i64, + dense_dim_null: i8, + ); + pub fn atg__to_sparse_semi_structured(out__: *mut *mut C_tensor, dense_: *mut C_tensor); + pub fn atg__to_sparse_sparse_dim( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + sparse_dim_: i64, + ); + pub fn atg__to_sparse_sparse_dim_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + sparse_dim_: i64, + ); + pub fn atg__transform_bias_rescale_qkv( + out__: *mut *mut C_tensor, + qkv_: *mut C_tensor, + qkv_bias_: *mut C_tensor, + num_heads_: i64, + ); + pub fn atg__transform_bias_rescale_qkv_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + out2_: *mut C_tensor, + qkv_: *mut C_tensor, + qkv_bias_: *mut C_tensor, + num_heads_: i64, + ); + pub fn atg__transformer_encoder_layer_fwd( + out__: *mut *mut C_tensor, + src_: *mut C_tensor, + embed_dim_: i64, + num_heads_: i64, + qkv_weight_: *mut C_tensor, + qkv_bias_: *mut C_tensor, + proj_weight_: *mut C_tensor, + proj_bias_: *mut C_tensor, + use_gelu_: c_int, + norm_first_: c_int, + eps_: f64, + norm_weight_1_: *mut C_tensor, + norm_bias_1_: *mut C_tensor, + norm_weight_2_: *mut C_tensor, + norm_bias_2_: *mut C_tensor, + ffn_weight_1_: *mut C_tensor, + ffn_bias_1_: *mut C_tensor, + ffn_weight_2_: *mut C_tensor, + ffn_bias_2_: *mut C_tensor, + mask_: *mut C_tensor, + mask_type_v: i64, + mask_type_null: i8, + ); + pub fn atg__transformer_encoder_layer_fwd_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + src_: *mut C_tensor, + embed_dim_: i64, + num_heads_: i64, + qkv_weight_: *mut C_tensor, + qkv_bias_: *mut C_tensor, + proj_weight_: *mut C_tensor, + proj_bias_: *mut C_tensor, + use_gelu_: c_int, + norm_first_: c_int, + eps_: f64, + norm_weight_1_: *mut C_tensor, + norm_bias_1_: *mut C_tensor, + norm_weight_2_: *mut C_tensor, + norm_bias_2_: *mut C_tensor, + ffn_weight_1_: *mut C_tensor, + ffn_bias_1_: *mut C_tensor, + ffn_weight_2_: *mut C_tensor, + ffn_bias_2_: *mut C_tensor, + mask_: *mut C_tensor, + mask_type_v: i64, + mask_type_null: i8, + ); + pub fn atg__trilinear( + out__: *mut *mut C_tensor, + i1_: *mut C_tensor, + i2_: *mut C_tensor, + i3_: *mut C_tensor, + expand1_data: *const i64, + expand1_len: c_int, + expand2_data: *const i64, + expand2_len: c_int, + expand3_data: *const i64, + expand3_len: c_int, + sumdim_data: *const i64, + sumdim_len: c_int, + unroll_dim_: i64, + ); + pub fn atg__trilinear_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + i1_: *mut C_tensor, + i2_: *mut C_tensor, + i3_: *mut C_tensor, + expand1_data: *const i64, + expand1_len: c_int, + expand2_data: *const i64, + expand2_len: c_int, + expand3_data: *const i64, + expand3_len: c_int, + sumdim_data: *const i64, + sumdim_len: c_int, + unroll_dim_: i64, + ); + pub fn atg__triton_multi_head_attention( + out__: *mut *mut C_tensor, + query_: *mut C_tensor, + key_: *mut C_tensor, + value_: *mut C_tensor, + embed_dim_: i64, + num_head_: i64, + qkv_weight_: *mut C_tensor, + qkv_bias_: *mut C_tensor, + proj_weight_: *mut C_tensor, + proj_bias_: *mut C_tensor, + mask_: *mut C_tensor, + ); + pub fn atg__triton_multi_head_attention_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + query_: *mut C_tensor, + key_: *mut C_tensor, + value_: *mut C_tensor, + embed_dim_: i64, + num_head_: i64, + qkv_weight_: *mut C_tensor, + qkv_bias_: *mut C_tensor, + proj_weight_: *mut C_tensor, + proj_bias_: *mut C_tensor, + mask_: *mut C_tensor, + ); + pub fn atg__triton_scaled_dot_attention( + out__: *mut *mut C_tensor, + q_: *mut C_tensor, + k_: *mut C_tensor, + v_: *mut C_tensor, + dropout_p_: f64, + ); + pub fn atg__triton_scaled_dot_attention_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + q_: *mut C_tensor, + k_: *mut C_tensor, + v_: *mut C_tensor, + dropout_p_: f64, + ); + pub fn atg__unique( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + sorted_: c_int, + return_inverse_: c_int, + ); + pub fn atg__unique2( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + sorted_: c_int, + return_inverse_: c_int, + return_counts_: c_int, + ); + pub fn atg__unique2_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + out2_: *mut C_tensor, + self_: *mut C_tensor, + sorted_: c_int, + return_inverse_: c_int, + return_counts_: c_int, + ); + pub fn atg__unique_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + self_: *mut C_tensor, + sorted_: c_int, + return_inverse_: c_int, + ); + pub fn atg__unpack_dual(out__: *mut *mut C_tensor, dual_: *mut C_tensor, level_: i64); + pub fn atg__unsafe_index( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + indices_data: *const *mut C_tensor, + indices_len: c_int, + ); + pub fn atg__unsafe_index_put( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + indices_data: *const *mut C_tensor, + indices_len: c_int, + values_: *mut C_tensor, + accumulate_: c_int, + ); + pub fn atg__unsafe_masked_index( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mask_: *mut C_tensor, + indices_data: *const *mut C_tensor, + indices_len: c_int, + fill_: *mut C_scalar, + ); + pub fn atg__unsafe_masked_index_put_accumulate( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mask_: *mut C_tensor, + indices_data: *const *mut C_tensor, + indices_len: c_int, + values_: *mut C_tensor, + ); + pub fn atg__unsafe_view( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg__unsafe_view_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg__upsample_bicubic2d_aa( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + align_corners_: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg__upsample_bicubic2d_aa_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + align_corners_: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg__upsample_bicubic2d_aa_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + align_corners_: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg__upsample_bicubic2d_aa_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + align_corners_: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg__upsample_bicubic2d_aa_vec( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + align_corners_: c_int, + scale_factors_data: *const f64, + scale_factors_len: c_int, + ); + pub fn atg__upsample_bilinear2d_aa( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + align_corners_: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg__upsample_bilinear2d_aa_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + align_corners_: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg__upsample_bilinear2d_aa_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + align_corners_: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg__upsample_bilinear2d_aa_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + align_corners_: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg__upsample_bilinear2d_aa_vec( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + align_corners_: c_int, + scale_factors_data: *const f64, + scale_factors_len: c_int, + ); + pub fn atg__upsample_nearest_exact1d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + scales_v: f64, + scales_null: i8, + ); + pub fn atg__upsample_nearest_exact1d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + scales_v: f64, + scales_null: i8, + ); + pub fn atg__upsample_nearest_exact1d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + scales_v: f64, + scales_null: i8, + ); + pub fn atg__upsample_nearest_exact1d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + scales_v: f64, + scales_null: i8, + ); + pub fn atg__upsample_nearest_exact1d_vec( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + scale_factors_data: *const f64, + scale_factors_len: c_int, + ); + pub fn atg__upsample_nearest_exact2d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg__upsample_nearest_exact2d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg__upsample_nearest_exact2d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg__upsample_nearest_exact2d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg__upsample_nearest_exact2d_vec( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + scale_factors_data: *const f64, + scale_factors_len: c_int, + ); + pub fn atg__upsample_nearest_exact3d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + scales_d_v: f64, + scales_d_null: i8, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg__upsample_nearest_exact3d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + scales_d_v: f64, + scales_d_null: i8, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg__upsample_nearest_exact3d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + scales_d_v: f64, + scales_d_null: i8, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg__upsample_nearest_exact3d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + scales_d_v: f64, + scales_d_null: i8, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg__upsample_nearest_exact3d_vec( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + scale_factors_data: *const f64, + scale_factors_len: c_int, + ); + pub fn atg__use_cudnn_ctc_loss( + log_probs_: *mut C_tensor, + targets_: *mut C_tensor, + input_lengths_data: *const i64, + input_lengths_len: c_int, + target_lengths_data: *const i64, + target_lengths_len: c_int, + blank_: i64, + ) -> c_int; + pub fn atg__use_cudnn_ctc_loss_tensor( + log_probs_: *mut C_tensor, + targets_: *mut C_tensor, + input_lengths_: *mut C_tensor, + target_lengths_: *mut C_tensor, + blank_: i64, + ) -> c_int; + pub fn atg__use_cudnn_rnn_flatten_weight() -> c_int; + pub fn atg__use_miopen_ctc_loss( + log_probs_: *mut C_tensor, + targets_: *mut C_tensor, + input_lengths_data: *const i64, + input_lengths_len: c_int, + target_lengths_data: *const i64, + target_lengths_len: c_int, + blank_: i64, + ) -> c_int; + pub fn atg__use_miopen_ctc_loss_tensor( + log_probs_: *mut C_tensor, + targets_: *mut C_tensor, + input_lengths_: *mut C_tensor, + target_lengths_: *mut C_tensor, + blank_: i64, + ) -> c_int; + pub fn atg__validate_compressed_sparse_indices( + is_crow_: c_int, + compressed_idx_: *mut C_tensor, + plain_idx_: *mut C_tensor, + cdim_: i64, + dim_: i64, + nnz_: i64, + ); + pub fn atg__validate_sparse_bsc_tensor_args( + ccol_indices_: *mut C_tensor, + row_indices_: *mut C_tensor, + values_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + check_pinning_: c_int, + ); + pub fn atg__validate_sparse_bsr_tensor_args( + crow_indices_: *mut C_tensor, + col_indices_: *mut C_tensor, + values_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + check_pinning_: c_int, + ); + pub fn atg__validate_sparse_compressed_tensor_args( + compressed_indices_: *mut C_tensor, + plain_indices_: *mut C_tensor, + values_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + layout_: i8, + check_pinning_: c_int, + ); + pub fn atg__validate_sparse_csc_tensor_args( + ccol_indices_: *mut C_tensor, + row_indices_: *mut C_tensor, + values_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + check_pinning_: c_int, + ); + pub fn atg__validate_sparse_csr_tensor_args( + crow_indices_: *mut C_tensor, + col_indices_: *mut C_tensor, + values_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + check_pinning_: c_int, + ); + pub fn atg__values(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__values_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg__values_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg__version(self_: *mut C_tensor) -> i64; + pub fn atg__weight_int4pack_mm( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mat2_: *mut C_tensor, + qGroupSize_: i64, + qScaleAndZeros_: *mut C_tensor, + ); + pub fn atg__weight_int4pack_mm_for_cpu( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mat2_: *mut C_tensor, + qGroupSize_: i64, + qScaleAndZeros_: *mut C_tensor, + ); + pub fn atg__weight_int4pack_mm_with_scales_and_zeros( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mat2_: *mut C_tensor, + qGroupSize_: i64, + qScale_: *mut C_tensor, + qZeros_: *mut C_tensor, + ); + pub fn atg__weight_int8pack_mm( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mat2_: *mut C_tensor, + scales_: *mut C_tensor, + ); + pub fn atg__weight_norm( + out__: *mut *mut C_tensor, + v_: *mut C_tensor, + g_: *mut C_tensor, + dim_: i64, + ); + pub fn atg__weight_norm_differentiable_backward( + out__: *mut *mut C_tensor, + grad_w_: *mut C_tensor, + saved_v_: *mut C_tensor, + saved_g_: *mut C_tensor, + saved_norms_: *mut C_tensor, + dim_: i64, + ); + pub fn atg__weight_norm_interface( + out__: *mut *mut C_tensor, + v_: *mut C_tensor, + g_: *mut C_tensor, + dim_: i64, + ); + pub fn atg__weight_norm_interface_backward( + out__: *mut *mut C_tensor, + grad_w_: *mut C_tensor, + saved_v_: *mut C_tensor, + saved_g_: *mut C_tensor, + saved_norms_: *mut C_tensor, + dim_: i64, + ); + pub fn atg__weight_norm_interface_backward_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + grad_w_: *mut C_tensor, + saved_v_: *mut C_tensor, + saved_g_: *mut C_tensor, + saved_norms_: *mut C_tensor, + dim_: i64, + ); + pub fn atg__weight_norm_interface_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + v_: *mut C_tensor, + g_: *mut C_tensor, + dim_: i64, + ); + pub fn atg__wrapped_linear_prepack( + out__: *mut *mut C_tensor, + weight_: *mut C_tensor, + weight_scale_: *mut C_tensor, + weight_zero_point_: *mut C_tensor, + bias_: *mut C_tensor, + ); + pub fn atg__wrapped_quantized_linear_prepacked( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + input_scale_: *mut C_tensor, + input_zero_point_: *mut C_tensor, + packed_weight_: *mut C_tensor, + output_scale_: *mut C_tensor, + output_zero_point_: *mut C_tensor, + out_channel_: i64, + ); + pub fn atg_abs(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_abs_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_abs_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_absolute(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_absolute_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_absolute_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_acos(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_acos_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_acos_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_acosh(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_acosh_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_acosh_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_adaptive_avg_pool1d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + ); + pub fn atg_adaptive_avg_pool1d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + ); + pub fn atg_adaptive_avg_pool2d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + ); + pub fn atg_adaptive_avg_pool2d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + ); + pub fn atg_adaptive_avg_pool3d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + ); + pub fn atg_adaptive_avg_pool3d_backward( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_adaptive_avg_pool3d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + ); + pub fn atg_adaptive_max_pool1d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + ); + pub fn atg_adaptive_max_pool2d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + ); + pub fn atg_adaptive_max_pool2d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + indices_: *mut C_tensor, + ); + pub fn atg_adaptive_max_pool2d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + indices_: *mut C_tensor, + ); + pub fn atg_adaptive_max_pool2d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + indices_: *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + ); + pub fn atg_adaptive_max_pool3d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + ); + pub fn atg_adaptive_max_pool3d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + indices_: *mut C_tensor, + ); + pub fn atg_adaptive_max_pool3d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + indices_: *mut C_tensor, + ); + pub fn atg_adaptive_max_pool3d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + indices_: *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + ); + pub fn atg_add(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_add_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_add_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_add_scalar(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_add_scalar_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_add_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_addbmm( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + batch1_: *mut C_tensor, + batch2_: *mut C_tensor, + ); + pub fn atg_addbmm_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + batch1_: *mut C_tensor, + batch2_: *mut C_tensor, + ); + pub fn atg_addbmm_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + batch1_: *mut C_tensor, + batch2_: *mut C_tensor, + ); + pub fn atg_addcdiv( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + tensor1_: *mut C_tensor, + tensor2_: *mut C_tensor, + ); + pub fn atg_addcdiv_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + tensor1_: *mut C_tensor, + tensor2_: *mut C_tensor, + ); + pub fn atg_addcdiv_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + tensor1_: *mut C_tensor, + tensor2_: *mut C_tensor, + ); + pub fn atg_addcmul( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + tensor1_: *mut C_tensor, + tensor2_: *mut C_tensor, + ); + pub fn atg_addcmul_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + tensor1_: *mut C_tensor, + tensor2_: *mut C_tensor, + ); + pub fn atg_addcmul_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + tensor1_: *mut C_tensor, + tensor2_: *mut C_tensor, + ); + pub fn atg_addmm( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mat1_: *mut C_tensor, + mat2_: *mut C_tensor, + ); + pub fn atg_addmm_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mat1_: *mut C_tensor, + mat2_: *mut C_tensor, + ); + pub fn atg_addmm_dtype( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mat1_: *mut C_tensor, + mat2_: *mut C_tensor, + out_dtype_: c_int, + ); + pub fn atg_addmm_dtype_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mat1_: *mut C_tensor, + mat2_: *mut C_tensor, + out_dtype_: c_int, + ); + pub fn atg_addmm_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mat1_: *mut C_tensor, + mat2_: *mut C_tensor, + ); + pub fn atg_addmv( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mat_: *mut C_tensor, + vec_: *mut C_tensor, + ); + pub fn atg_addmv_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mat_: *mut C_tensor, + vec_: *mut C_tensor, + ); + pub fn atg_addmv_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mat_: *mut C_tensor, + vec_: *mut C_tensor, + ); + pub fn atg_addr( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + vec1_: *mut C_tensor, + vec2_: *mut C_tensor, + ); + pub fn atg_addr_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + vec1_: *mut C_tensor, + vec2_: *mut C_tensor, + ); + pub fn atg_addr_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + vec1_: *mut C_tensor, + vec2_: *mut C_tensor, + ); + pub fn atg_adjoint(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_affine_grid_generator( + out__: *mut *mut C_tensor, + theta_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + align_corners_: c_int, + ); + pub fn atg_affine_grid_generator_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + align_corners_: c_int, + ); + pub fn atg_affine_grid_generator_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + theta_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + align_corners_: c_int, + ); + pub fn atg_alias(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_alias_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_alias_copy_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_align_as(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_align_tensors( + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ) -> *mut *mut C_tensor; + pub fn atg_all(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_all_all_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_all_dim(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, keepdim_: c_int); + pub fn atg_all_dims( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + ); + pub fn atg_all_dims_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + ); + pub fn atg_all_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + keepdim_: c_int, + ); + pub fn atg_allclose( + self_: *mut C_tensor, + other_: *mut C_tensor, + rtol_: f64, + atol_: f64, + equal_nan_: c_int, + ) -> c_int; + pub fn atg_alpha_dropout( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + p_: f64, + train_: c_int, + ); + pub fn atg_alpha_dropout_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + p_: f64, + train_: c_int, + ); + pub fn atg_amax( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + ); + pub fn atg_amax_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + ); + pub fn atg_amin( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + ); + pub fn atg_amin_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + ); + pub fn atg_aminmax( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_v: i64, + dim_null: i8, + keepdim_: c_int, + ); + pub fn atg_aminmax_out( + out__: *mut *mut C_tensor, + min_: *mut C_tensor, + max_: *mut C_tensor, + self_: *mut C_tensor, + dim_v: i64, + dim_null: i8, + keepdim_: c_int, + ); + pub fn atg_angle(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_angle_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_any(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_any_all_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_any_dim(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, keepdim_: c_int); + pub fn atg_any_dims( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + ); + pub fn atg_any_dims_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + ); + pub fn atg_any_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + keepdim_: c_int, + ); + pub fn atg_arange( + out__: *mut *mut C_tensor, + end_: *mut C_scalar, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_arange_start( + out__: *mut *mut C_tensor, + start_: *mut C_scalar, + end_: *mut C_scalar, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_arange_start_step( + out__: *mut *mut C_tensor, + start_: *mut C_scalar, + end_: *mut C_scalar, + step_: *mut C_scalar, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_arccos(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_arccos_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_arccos_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_arccosh(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_arccosh_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_arccosh_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_arcsin(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_arcsin_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_arcsin_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_arcsinh(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_arcsinh_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_arcsinh_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_arctan(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_arctan2(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_arctan2_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_arctan2_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_arctan_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_arctan_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_arctanh(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_arctanh_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_arctanh_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_argmax( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_v: i64, + dim_null: i8, + keepdim_: c_int, + ); + pub fn atg_argmax_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_v: i64, + dim_null: i8, + keepdim_: c_int, + ); + pub fn atg_argmin( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_v: i64, + dim_null: i8, + keepdim_: c_int, + ); + pub fn atg_argmin_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_v: i64, + dim_null: i8, + keepdim_: c_int, + ); + pub fn atg_argsort( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + descending_: c_int, + ); + pub fn atg_argsort_stable( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + stable_: c_int, + dim_: i64, + descending_: c_int, + ); + pub fn atg_argsort_stable_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + stable_: c_int, + dim_: i64, + descending_: c_int, + ); + pub fn atg_argwhere(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_as_strided( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + storage_offset_v: i64, + storage_offset_null: i8, + ); + pub fn atg_as_strided_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + storage_offset_v: i64, + storage_offset_null: i8, + ); + pub fn atg_as_strided_copy( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + storage_offset_v: i64, + storage_offset_null: i8, + ); + pub fn atg_as_strided_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + storage_offset_v: i64, + storage_offset_null: i8, + ); + pub fn atg_as_strided_scatter( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + src_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + storage_offset_v: i64, + storage_offset_null: i8, + ); + pub fn atg_as_strided_scatter_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + src_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + storage_offset_v: i64, + storage_offset_null: i8, + ); + pub fn atg_asin(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_asin_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_asin_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_asinh(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_asinh_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_asinh_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_atan(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_atan2(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_atan2_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_atan2_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_atan_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_atan_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_atanh(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_atanh_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_atanh_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_atleast_1d(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_atleast_1d_sequence( + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ) -> *mut *mut C_tensor; + pub fn atg_atleast_2d(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_atleast_2d_sequence( + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ) -> *mut *mut C_tensor; + pub fn atg_atleast_3d(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_atleast_3d_sequence( + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ) -> *mut *mut C_tensor; + pub fn atg_avg_pool1d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + ceil_mode_: c_int, + count_include_pad_: c_int, + ); + pub fn atg_avg_pool1d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + ceil_mode_: c_int, + count_include_pad_: c_int, + ); + pub fn atg_avg_pool2d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + ceil_mode_: c_int, + count_include_pad_: c_int, + divisor_override_v: i64, + divisor_override_null: i8, + ); + pub fn atg_avg_pool2d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + ceil_mode_: c_int, + count_include_pad_: c_int, + divisor_override_v: i64, + divisor_override_null: i8, + ); + pub fn atg_avg_pool2d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + ceil_mode_: c_int, + count_include_pad_: c_int, + divisor_override_v: i64, + divisor_override_null: i8, + ); + pub fn atg_avg_pool2d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + ceil_mode_: c_int, + count_include_pad_: c_int, + divisor_override_v: i64, + divisor_override_null: i8, + ); + pub fn atg_avg_pool3d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + ceil_mode_: c_int, + count_include_pad_: c_int, + divisor_override_v: i64, + divisor_override_null: i8, + ); + pub fn atg_avg_pool3d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + ceil_mode_: c_int, + count_include_pad_: c_int, + divisor_override_v: i64, + divisor_override_null: i8, + ); + pub fn atg_avg_pool3d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + ceil_mode_: c_int, + count_include_pad_: c_int, + divisor_override_v: i64, + divisor_override_null: i8, + ); + pub fn atg_avg_pool3d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + ceil_mode_: c_int, + count_include_pad_: c_int, + divisor_override_v: i64, + divisor_override_null: i8, + ); + pub fn atg_baddbmm( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + batch1_: *mut C_tensor, + batch2_: *mut C_tensor, + beta_: *mut C_scalar, + alpha_: *mut C_scalar, + ); + pub fn atg_baddbmm_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + batch1_: *mut C_tensor, + batch2_: *mut C_tensor, + ); + pub fn atg_baddbmm_dtype( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + batch1_: *mut C_tensor, + batch2_: *mut C_tensor, + out_dtype_: c_int, + beta_: *mut C_scalar, + alpha_: *mut C_scalar, + ); + pub fn atg_baddbmm_dtype_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + batch1_: *mut C_tensor, + batch2_: *mut C_tensor, + out_dtype_: c_int, + ); + pub fn atg_baddbmm_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + batch1_: *mut C_tensor, + batch2_: *mut C_tensor, + ); + pub fn atg_bartlett_window( + out__: *mut *mut C_tensor, + window_length_: i64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_bartlett_window_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + window_length_: i64, + ); + pub fn atg_bartlett_window_periodic( + out__: *mut *mut C_tensor, + window_length_: i64, + periodic_: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_bartlett_window_periodic_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + window_length_: i64, + periodic_: c_int, + ); + pub fn atg_batch_norm( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + training_: c_int, + momentum_: f64, + eps_: f64, + cudnn_enabled_: c_int, + ); + pub fn atg_batch_norm_backward_elemt( + out__: *mut *mut C_tensor, + grad_out_: *mut C_tensor, + input_: *mut C_tensor, + mean_: *mut C_tensor, + invstd_: *mut C_tensor, + weight_: *mut C_tensor, + sum_dy_: *mut C_tensor, + sum_dy_xmu_: *mut C_tensor, + count_: *mut C_tensor, + ); + pub fn atg_batch_norm_backward_elemt_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_out_: *mut C_tensor, + input_: *mut C_tensor, + mean_: *mut C_tensor, + invstd_: *mut C_tensor, + weight_: *mut C_tensor, + sum_dy_: *mut C_tensor, + sum_dy_xmu_: *mut C_tensor, + count_: *mut C_tensor, + ); + pub fn atg_batch_norm_backward_reduce( + out__: *mut *mut C_tensor, + grad_out_: *mut C_tensor, + input_: *mut C_tensor, + mean_: *mut C_tensor, + invstd_: *mut C_tensor, + weight_: *mut C_tensor, + input_g_: c_int, + weight_g_: c_int, + bias_g_: c_int, + ); + pub fn atg_batch_norm_backward_reduce_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + out2_: *mut C_tensor, + out3_: *mut C_tensor, + grad_out_: *mut C_tensor, + input_: *mut C_tensor, + mean_: *mut C_tensor, + invstd_: *mut C_tensor, + weight_: *mut C_tensor, + input_g_: c_int, + weight_g_: c_int, + bias_g_: c_int, + ); + pub fn atg_batch_norm_elemt( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + mean_: *mut C_tensor, + invstd_: *mut C_tensor, + eps_: f64, + ); + pub fn atg_batch_norm_elemt_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + mean_: *mut C_tensor, + invstd_: *mut C_tensor, + eps_: f64, + ); + pub fn atg_batch_norm_gather_stats( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + mean_: *mut C_tensor, + invstd_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + momentum_: f64, + eps_: f64, + count_: i64, + ); + pub fn atg_batch_norm_gather_stats_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + input_: *mut C_tensor, + mean_: *mut C_tensor, + invstd_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + momentum_: f64, + eps_: f64, + count_: i64, + ); + pub fn atg_batch_norm_gather_stats_with_counts( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + mean_: *mut C_tensor, + invstd_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + momentum_: f64, + eps_: f64, + counts_: *mut C_tensor, + ); + pub fn atg_batch_norm_gather_stats_with_counts_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + input_: *mut C_tensor, + mean_: *mut C_tensor, + invstd_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + momentum_: f64, + eps_: f64, + counts_: *mut C_tensor, + ); + pub fn atg_batch_norm_stats(out__: *mut *mut C_tensor, input_: *mut C_tensor, eps_: f64); + pub fn atg_batch_norm_stats_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + input_: *mut C_tensor, + eps_: f64, + ); + pub fn atg_batch_norm_update_stats( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + momentum_: f64, + ); + pub fn atg_batch_norm_update_stats_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + input_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + momentum_: f64, + ); + pub fn atg_bernoulli(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_bernoulli_(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: *mut C_tensor); + pub fn atg_bernoulli_float_(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: f64); + pub fn atg_bernoulli_p(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: f64); + pub fn atg_bernoulli_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: *mut C_tensor); + pub fn atg_bilinear( + out__: *mut *mut C_tensor, + input1_: *mut C_tensor, + input2_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + ); + pub fn atg_binary_cross_entropy( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + weight_: *mut C_tensor, + reduction_: i64, + ); + pub fn atg_binary_cross_entropy_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + weight_: *mut C_tensor, + reduction_: i64, + ); + pub fn atg_binary_cross_entropy_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + weight_: *mut C_tensor, + reduction_: i64, + ); + pub fn atg_binary_cross_entropy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + weight_: *mut C_tensor, + reduction_: i64, + ); + pub fn atg_binary_cross_entropy_with_logits( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + weight_: *mut C_tensor, + pos_weight_: *mut C_tensor, + reduction_: i64, + ); + pub fn atg_binary_cross_entropy_with_logits_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + weight_: *mut C_tensor, + pos_weight_: *mut C_tensor, + reduction_: i64, + ); + pub fn atg_bincount( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weights_: *mut C_tensor, + minlength_: i64, + ); + pub fn atg_bincount_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + weights_: *mut C_tensor, + minlength_: i64, + ); + pub fn atg_binomial(out__: *mut *mut C_tensor, count_: *mut C_tensor, prob_: *mut C_tensor); + pub fn atg_binomial_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + count_: *mut C_tensor, + prob_: *mut C_tensor, + ); + pub fn atg_bitwise_and(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_bitwise_and_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_bitwise_and_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_bitwise_and_scalar_tensor( + out__: *mut *mut C_tensor, + self_scalar_: *mut C_scalar, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_and_scalar_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_scalar_: *mut C_scalar, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_and_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_and_tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_and_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_left_shift( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_left_shift_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_left_shift_scalar_tensor( + out__: *mut *mut C_tensor, + self_scalar_: *mut C_scalar, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_left_shift_scalar_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_scalar_: *mut C_scalar, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_left_shift_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_left_shift_tensor_scalar( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_bitwise_left_shift_tensor_scalar_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_bitwise_left_shift_tensor_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_bitwise_not(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_bitwise_not_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_bitwise_not_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_bitwise_or(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_bitwise_or_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_bitwise_or_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_bitwise_or_scalar_tensor( + out__: *mut *mut C_tensor, + self_scalar_: *mut C_scalar, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_or_scalar_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_scalar_: *mut C_scalar, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_or_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_or_tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_or_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_right_shift( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_right_shift_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_right_shift_scalar_tensor( + out__: *mut *mut C_tensor, + self_scalar_: *mut C_scalar, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_right_shift_scalar_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_scalar_: *mut C_scalar, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_right_shift_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_right_shift_tensor_scalar( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_bitwise_right_shift_tensor_scalar_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_bitwise_right_shift_tensor_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_bitwise_xor(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_bitwise_xor_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_bitwise_xor_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_bitwise_xor_scalar_tensor( + out__: *mut *mut C_tensor, + self_scalar_: *mut C_scalar, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_xor_scalar_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_scalar_: *mut C_scalar, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_xor_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_xor_tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_bitwise_xor_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_blackman_window( + out__: *mut *mut C_tensor, + window_length_: i64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_blackman_window_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + window_length_: i64, + ); + pub fn atg_blackman_window_periodic( + out__: *mut *mut C_tensor, + window_length_: i64, + periodic_: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_blackman_window_periodic_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + window_length_: i64, + periodic_: c_int, + ); + pub fn atg_block_diag( + out__: *mut *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ); + pub fn atg_block_diag_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ); + pub fn atg_bmm(out__: *mut *mut C_tensor, self_: *mut C_tensor, mat2_: *mut C_tensor); + pub fn atg_bmm_dtype( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mat2_: *mut C_tensor, + out_dtype_: c_int, + ); + pub fn atg_bmm_dtype_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mat2_: *mut C_tensor, + out_dtype_: c_int, + ); + pub fn atg_bmm_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mat2_: *mut C_tensor, + ); + pub fn atg_broadcast_tensors( + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ) -> *mut *mut C_tensor; + pub fn atg_broadcast_to( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg_bucketize( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + boundaries_: *mut C_tensor, + out_int32_: c_int, + right_: c_int, + ); + pub fn atg_bucketize_scalar( + out__: *mut *mut C_tensor, + self_scalar_: *mut C_scalar, + boundaries_: *mut C_tensor, + out_int32_: c_int, + right_: c_int, + ); + pub fn atg_bucketize_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_scalar_: *mut C_scalar, + boundaries_: *mut C_tensor, + out_int32_: c_int, + right_: c_int, + ); + pub fn atg_bucketize_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + boundaries_: *mut C_tensor, + out_int32_: c_int, + right_: c_int, + ); + pub fn atg_can_cast(from__: c_int, to_: c_int) -> c_int; + pub fn atg_cartesian_prod( + out__: *mut *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ); + pub fn atg_cat( + out__: *mut *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + dim_: i64, + ); + pub fn atg_cat_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + dim_: i64, + ); + pub fn atg_cauchy(out__: *mut *mut C_tensor, self_: *mut C_tensor, median_: f64, sigma_: f64); + pub fn atg_cauchy_(out__: *mut *mut C_tensor, self_: *mut C_tensor, median_: f64, sigma_: f64); + pub fn atg_cauchy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + median_: f64, + sigma_: f64, + ); + pub fn atg_ccol_indices(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_ccol_indices_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_ccol_indices_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_cdist( + out__: *mut *mut C_tensor, + x1_: *mut C_tensor, + x2_: *mut C_tensor, + p_: f64, + compute_mode_v: i64, + compute_mode_null: i8, + ); + pub fn atg_ceil(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_ceil_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_ceil_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_celu(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_celu_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_celu_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_chain_matmul( + out__: *mut *mut C_tensor, + matrices_data: *const *mut C_tensor, + matrices_len: c_int, + ); + pub fn atg_chain_matmul_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + matrices_data: *const *mut C_tensor, + matrices_len: c_int, + ); + pub fn atg_chalf(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_channel_shuffle(out__: *mut *mut C_tensor, self_: *mut C_tensor, groups_: i64); + pub fn atg_channel_shuffle_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + groups_: i64, + ); + pub fn atg_cholesky(out__: *mut *mut C_tensor, self_: *mut C_tensor, upper_: c_int); + pub fn atg_cholesky_inverse(out__: *mut *mut C_tensor, self_: *mut C_tensor, upper_: c_int); + pub fn atg_cholesky_inverse_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + upper_: c_int, + ); + pub fn atg_cholesky_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + upper_: c_int, + ); + pub fn atg_cholesky_solve( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + input2_: *mut C_tensor, + upper_: c_int, + ); + pub fn atg_cholesky_solve_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + input2_: *mut C_tensor, + upper_: c_int, + ); + pub fn atg_choose_qparams_optimized( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + numel_: i64, + n_bins_: i64, + ratio_: f64, + bit_width_: i64, + ); + pub fn atg_chunk(self_: *mut C_tensor, chunks_: i64, dim_: i64) -> *mut *mut C_tensor; + pub fn atg_clamp( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + min_: *mut C_scalar, + max_: *mut C_scalar, + ); + pub fn atg_clamp_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + min_: *mut C_scalar, + max_: *mut C_scalar, + ); + pub fn atg_clamp_max(out__: *mut *mut C_tensor, self_: *mut C_tensor, max_: *mut C_scalar); + pub fn atg_clamp_max_(out__: *mut *mut C_tensor, self_: *mut C_tensor, max_: *mut C_scalar); + pub fn atg_clamp_max_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + max_: *mut C_scalar, + ); + pub fn atg_clamp_max_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + max_: *mut C_tensor, + ); + pub fn atg_clamp_max_tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + max_: *mut C_tensor, + ); + pub fn atg_clamp_max_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + max_: *mut C_tensor, + ); + pub fn atg_clamp_min(out__: *mut *mut C_tensor, self_: *mut C_tensor, min_: *mut C_scalar); + pub fn atg_clamp_min_(out__: *mut *mut C_tensor, self_: *mut C_tensor, min_: *mut C_scalar); + pub fn atg_clamp_min_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + min_: *mut C_scalar, + ); + pub fn atg_clamp_min_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + min_: *mut C_tensor, + ); + pub fn atg_clamp_min_tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + min_: *mut C_tensor, + ); + pub fn atg_clamp_min_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + min_: *mut C_tensor, + ); + pub fn atg_clamp_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + min_: *mut C_scalar, + max_: *mut C_scalar, + ); + pub fn atg_clamp_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + min_: *mut C_tensor, + max_: *mut C_tensor, + ); + pub fn atg_clamp_tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + min_: *mut C_tensor, + max_: *mut C_tensor, + ); + pub fn atg_clamp_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + min_: *mut C_tensor, + max_: *mut C_tensor, + ); + pub fn atg_clip( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + min_: *mut C_scalar, + max_: *mut C_scalar, + ); + pub fn atg_clip_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + min_: *mut C_scalar, + max_: *mut C_scalar, + ); + pub fn atg_clip_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + min_: *mut C_scalar, + max_: *mut C_scalar, + ); + pub fn atg_clip_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + min_: *mut C_tensor, + max_: *mut C_tensor, + ); + pub fn atg_clip_tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + min_: *mut C_tensor, + max_: *mut C_tensor, + ); + pub fn atg_clip_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + min_: *mut C_tensor, + max_: *mut C_tensor, + ); + pub fn atg_clone(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_coalesce(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_col2im( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + kernel_size_data: *const i64, + kernel_size_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + padding_data: *const i64, + padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + ); + pub fn atg_col2im_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + kernel_size_data: *const i64, + kernel_size_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + padding_data: *const i64, + padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + ); + pub fn atg_col_indices(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_col_indices_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_col_indices_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_column_stack( + out__: *mut *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ); + pub fn atg_column_stack_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ); + pub fn atg_combinations( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + r_: i64, + with_replacement_: c_int, + ); + pub fn atg_complex(out__: *mut *mut C_tensor, real_: *mut C_tensor, imag_: *mut C_tensor); + pub fn atg_complex_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + real_: *mut C_tensor, + imag_: *mut C_tensor, + ); + pub fn atg_concat( + out__: *mut *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + dim_: i64, + ); + pub fn atg_concat_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + dim_: i64, + ); + pub fn atg_concatenate( + out__: *mut *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + dim_: i64, + ); + pub fn atg_concatenate_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + dim_: i64, + ); + pub fn atg_conj(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_conj_physical(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_conj_physical_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_conj_physical_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_constant_pad_nd( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + pad_data: *const i64, + pad_len: c_int, + ); + pub fn atg_constant_pad_nd_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + pad_data: *const i64, + pad_len: c_int, + ); + pub fn atg_contiguous(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_conv1d( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + ); + pub fn atg_conv1d_padding( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_ptr: *const u8, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + ); + pub fn atg_conv2d( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + ); + pub fn atg_conv2d_padding( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_ptr: *const u8, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + ); + pub fn atg_conv3d( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + ); + pub fn atg_conv3d_padding( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_ptr: *const u8, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + ); + pub fn atg_conv_depthwise3d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ); + pub fn atg_conv_depthwise3d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ); + pub fn atg_conv_tbc( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + pad_: i64, + ); + pub fn atg_conv_tbc_backward( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + pad_: i64, + ); + pub fn atg_conv_tbc_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + pad_: i64, + ); + pub fn atg_conv_transpose1d( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + output_padding_data: *const i64, + output_padding_len: c_int, + groups_: i64, + dilation_data: *const i64, + dilation_len: c_int, + ); + pub fn atg_conv_transpose2d( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + output_padding_data: *const i64, + output_padding_len: c_int, + groups_: i64, + dilation_data: *const i64, + dilation_len: c_int, + ); + pub fn atg_conv_transpose3d( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + output_padding_data: *const i64, + output_padding_len: c_int, + groups_: i64, + dilation_data: *const i64, + dilation_len: c_int, + ); + pub fn atg_convolution( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + transposed_: c_int, + output_padding_data: *const i64, + output_padding_len: c_int, + groups_: i64, + ); + pub fn atg_convolution_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + transposed_: c_int, + output_padding_data: *const i64, + output_padding_len: c_int, + groups_: i64, + ); + pub fn atg_convolution_overrideable( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + transposed_: c_int, + output_padding_data: *const i64, + output_padding_len: c_int, + groups_: i64, + ); + pub fn atg_convolution_overrideable_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + transposed_: c_int, + output_padding_data: *const i64, + output_padding_len: c_int, + groups_: i64, + ); + pub fn atg_copy_sparse_to_sparse( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + src_: *mut C_tensor, + non_blocking_: c_int, + ); + pub fn atg_copy_sparse_to_sparse_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + src_: *mut C_tensor, + non_blocking_: c_int, + ); + pub fn atg_copy_sparse_to_sparse_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + src_: *mut C_tensor, + non_blocking_: c_int, + ); + pub fn atg_copysign(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_copysign_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_copysign_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_copysign_scalar( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_copysign_scalar_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_copysign_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_corrcoef(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_cos(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_cos_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_cos_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_cosh(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_cosh_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_cosh_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_cosine_embedding_loss( + out__: *mut *mut C_tensor, + input1_: *mut C_tensor, + input2_: *mut C_tensor, + target_: *mut C_tensor, + margin_: f64, + reduction_: i64, + ); + pub fn atg_cosine_similarity( + out__: *mut *mut C_tensor, + x1_: *mut C_tensor, + x2_: *mut C_tensor, + dim_: i64, + eps_: f64, + ); + pub fn atg_count_nonzero( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_v: i64, + dim_null: i8, + ); + pub fn atg_count_nonzero_dim_intlist( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + ); + pub fn atg_count_nonzero_dim_intlist_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + ); + pub fn atg_count_nonzero_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_v: i64, + dim_null: i8, + ); + pub fn atg_cov( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + correction_: i64, + fweights_: *mut C_tensor, + aweights_: *mut C_tensor, + ); + pub fn atg_cross( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + dim_v: i64, + dim_null: i8, + ); + pub fn atg_cross_entropy_loss( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + weight_: *mut C_tensor, + reduction_: i64, + ignore_index_: i64, + label_smoothing_: f64, + ); + pub fn atg_cross_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + dim_v: i64, + dim_null: i8, + ); + pub fn atg_crow_indices(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_crow_indices_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_crow_indices_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_ctc_loss( + out__: *mut *mut C_tensor, + log_probs_: *mut C_tensor, + targets_: *mut C_tensor, + input_lengths_data: *const i64, + input_lengths_len: c_int, + target_lengths_data: *const i64, + target_lengths_len: c_int, + blank_: i64, + reduction_: i64, + zero_infinity_: c_int, + ); + pub fn atg_ctc_loss_tensor( + out__: *mut *mut C_tensor, + log_probs_: *mut C_tensor, + targets_: *mut C_tensor, + input_lengths_: *mut C_tensor, + target_lengths_: *mut C_tensor, + blank_: i64, + reduction_: i64, + zero_infinity_: c_int, + ); + pub fn atg_cudnn_affine_grid_generator( + out__: *mut *mut C_tensor, + theta_: *mut C_tensor, + n_: i64, + C_: i64, + H_: i64, + W_: i64, + ); + pub fn atg_cudnn_affine_grid_generator_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + n_: i64, + C_: i64, + H_: i64, + W_: i64, + ); + pub fn atg_cudnn_affine_grid_generator_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_: *mut C_tensor, + n_: i64, + C_: i64, + H_: i64, + W_: i64, + ); + pub fn atg_cudnn_affine_grid_generator_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + theta_: *mut C_tensor, + n_: i64, + C_: i64, + H_: i64, + W_: i64, + ); + pub fn atg_cudnn_batch_norm( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + training_: c_int, + exponential_average_factor_: f64, + epsilon_: f64, + ); + pub fn atg_cudnn_batch_norm_backward( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + grad_output_: *mut C_tensor, + weight_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + save_mean_: *mut C_tensor, + save_var_: *mut C_tensor, + epsilon_: f64, + reserveSpace_: *mut C_tensor, + ); + pub fn atg_cudnn_batch_norm_backward_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + out2_: *mut C_tensor, + input_: *mut C_tensor, + grad_output_: *mut C_tensor, + weight_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + save_mean_: *mut C_tensor, + save_var_: *mut C_tensor, + epsilon_: f64, + reserveSpace_: *mut C_tensor, + ); + pub fn atg_cudnn_batch_norm_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + out2_: *mut C_tensor, + out3_: *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + training_: c_int, + exponential_average_factor_: f64, + epsilon_: f64, + ); + pub fn atg_cudnn_convolution( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + benchmark_: c_int, + deterministic_: c_int, + allow_tf32_: c_int, + ); + pub fn atg_cudnn_convolution_add_relu( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + z_: *mut C_tensor, + alpha_: *mut C_scalar, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + ); + pub fn atg_cudnn_convolution_add_relu_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + z_: *mut C_tensor, + alpha_: *mut C_scalar, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + ); + pub fn atg_cudnn_convolution_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + benchmark_: c_int, + deterministic_: c_int, + allow_tf32_: c_int, + ); + pub fn atg_cudnn_convolution_relu( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + ); + pub fn atg_cudnn_convolution_relu_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + ); + pub fn atg_cudnn_convolution_transpose( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + output_padding_data: *const i64, + output_padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + benchmark_: c_int, + deterministic_: c_int, + allow_tf32_: c_int, + ); + pub fn atg_cudnn_convolution_transpose_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + output_padding_data: *const i64, + output_padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + benchmark_: c_int, + deterministic_: c_int, + allow_tf32_: c_int, + ); + pub fn atg_cudnn_grid_sampler( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + grid_: *mut C_tensor, + ); + pub fn atg_cudnn_grid_sampler_backward( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + grid_: *mut C_tensor, + grad_output_: *mut C_tensor, + ); + pub fn atg_cudnn_grid_sampler_backward_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + self_: *mut C_tensor, + grid_: *mut C_tensor, + grad_output_: *mut C_tensor, + ); + pub fn atg_cudnn_grid_sampler_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + grid_: *mut C_tensor, + ); + pub fn atg_cudnn_is_acceptable(self_: *mut C_tensor) -> c_int; + pub fn atg_cummax(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64); + pub fn atg_cummax_out( + out__: *mut *mut C_tensor, + values_: *mut C_tensor, + indices_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + ); + pub fn atg_cummaxmin_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + input_: *mut C_tensor, + indices_: *mut C_tensor, + dim_: i64, + ); + pub fn atg_cummin(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64); + pub fn atg_cummin_out( + out__: *mut *mut C_tensor, + values_: *mut C_tensor, + indices_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + ); + pub fn atg_cumprod(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, dtype_: c_int); + pub fn atg_cumprod_(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, dtype_: c_int); + pub fn atg_cumprod_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + input_: *mut C_tensor, + dim_: i64, + output_: *mut C_tensor, + ); + pub fn atg_cumprod_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + dtype_: c_int, + ); + pub fn atg_cumsum(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, dtype_: c_int); + pub fn atg_cumsum_(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, dtype_: c_int); + pub fn atg_cumsum_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + dtype_: c_int, + ); + pub fn atg_cumulative_trapezoid(out__: *mut *mut C_tensor, y_: *mut C_tensor, dim_: i64); + pub fn atg_cumulative_trapezoid_x( + out__: *mut *mut C_tensor, + y_: *mut C_tensor, + x_: *mut C_tensor, + dim_: i64, + ); + pub fn atg_data(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_deg2rad(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_deg2rad_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_deg2rad_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_dense_dim(self_: *mut C_tensor) -> i64; + pub fn atg_dequantize(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_dequantize_self_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_dequantize_tensors( + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ) -> *mut *mut C_tensor; + pub fn atg_dequantize_tensors_out( + out_data: *const *mut C_tensor, + out_len: c_int, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ); + pub fn atg_det(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_detach(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_detach_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_detach_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_detach_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_diag(out__: *mut *mut C_tensor, self_: *mut C_tensor, diagonal_: i64); + pub fn atg_diag_embed( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + offset_: i64, + dim1_: i64, + dim2_: i64, + ); + pub fn atg_diag_embed_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + offset_: i64, + dim1_: i64, + dim2_: i64, + ); + pub fn atg_diag_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + diagonal_: i64, + ); + pub fn atg_diagflat(out__: *mut *mut C_tensor, self_: *mut C_tensor, offset_: i64); + pub fn atg_diagonal( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + offset_: i64, + dim1_: i64, + dim2_: i64, + ); + pub fn atg_diagonal_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + input_sizes_data: *const i64, + input_sizes_len: c_int, + offset_: i64, + dim1_: i64, + dim2_: i64, + ); + pub fn atg_diagonal_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_output_: *mut C_tensor, + input_sizes_data: *const i64, + input_sizes_len: c_int, + offset_: i64, + dim1_: i64, + dim2_: i64, + ); + pub fn atg_diagonal_copy( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + offset_: i64, + dim1_: i64, + dim2_: i64, + ); + pub fn atg_diagonal_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + offset_: i64, + dim1_: i64, + dim2_: i64, + ); + pub fn atg_diagonal_scatter( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + src_: *mut C_tensor, + offset_: i64, + dim1_: i64, + dim2_: i64, + ); + pub fn atg_diagonal_scatter_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + src_: *mut C_tensor, + offset_: i64, + dim1_: i64, + dim2_: i64, + ); + pub fn atg_diff( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + n_: i64, + dim_: i64, + prepend_: *mut C_tensor, + append_: *mut C_tensor, + ); + pub fn atg_diff_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + n_: i64, + dim_: i64, + prepend_: *mut C_tensor, + append_: *mut C_tensor, + ); + pub fn atg_digamma(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_digamma_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_digamma_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_dist(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_dist_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_div(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_div_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_div_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_div_out_mode( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + rounding_mode_ptr: *const u8, + rounding_mode_len: c_int, + ); + pub fn atg_div_scalar(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_div_scalar_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_div_scalar_mode( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + rounding_mode_ptr: *const u8, + rounding_mode_len: c_int, + ); + pub fn atg_div_scalar_mode_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + rounding_mode_ptr: *const u8, + rounding_mode_len: c_int, + ); + pub fn atg_div_scalar_mode_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + rounding_mode_ptr: *const u8, + rounding_mode_len: c_int, + ); + pub fn atg_div_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_div_tensor_mode( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + rounding_mode_ptr: *const u8, + rounding_mode_len: c_int, + ); + pub fn atg_div_tensor_mode_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + rounding_mode_ptr: *const u8, + rounding_mode_len: c_int, + ); + pub fn atg_divide(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_divide_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_divide_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_divide_out_mode( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + rounding_mode_ptr: *const u8, + rounding_mode_len: c_int, + ); + pub fn atg_divide_scalar( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_divide_scalar_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_divide_scalar_mode( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + rounding_mode_ptr: *const u8, + rounding_mode_len: c_int, + ); + pub fn atg_divide_scalar_mode_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + rounding_mode_ptr: *const u8, + rounding_mode_len: c_int, + ); + pub fn atg_divide_tensor_mode( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + rounding_mode_ptr: *const u8, + rounding_mode_len: c_int, + ); + pub fn atg_divide_tensor_mode_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + rounding_mode_ptr: *const u8, + rounding_mode_len: c_int, + ); + pub fn atg_dot(out__: *mut *mut C_tensor, self_: *mut C_tensor, tensor_: *mut C_tensor); + pub fn atg_dot_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + tensor_: *mut C_tensor, + ); + pub fn atg_dropout(out__: *mut *mut C_tensor, input_: *mut C_tensor, p_: f64, train_: c_int); + pub fn atg_dropout_(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: f64, train_: c_int); + pub fn atg_dsplit(self_: *mut C_tensor, sections_: i64) -> *mut *mut C_tensor; + pub fn atg_dsplit_array( + self_: *mut C_tensor, + indices_data: *const i64, + indices_len: c_int, + ) -> *mut *mut C_tensor; + pub fn atg_dstack( + out__: *mut *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ); + pub fn atg_dstack_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ); + pub fn atg_einsum( + out__: *mut *mut C_tensor, + equation_ptr: *const u8, + equation_len: c_int, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + path_data: *const i64, + path_len: c_int, + ); + pub fn atg_elu(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_elu_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_elu_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + alpha_: *mut C_scalar, + scale_: *mut C_scalar, + input_scale_: *mut C_scalar, + is_result_: c_int, + self_or_result_: *mut C_tensor, + ); + pub fn atg_elu_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + alpha_: *mut C_scalar, + scale_: *mut C_scalar, + input_scale_: *mut C_scalar, + is_result_: c_int, + self_or_result_: *mut C_tensor, + ); + pub fn atg_elu_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_embedding( + out__: *mut *mut C_tensor, + weight_: *mut C_tensor, + indices_: *mut C_tensor, + padding_idx_: i64, + scale_grad_by_freq_: c_int, + sparse_: c_int, + ); + pub fn atg_embedding_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + indices_: *mut C_tensor, + num_weights_: i64, + padding_idx_: i64, + scale_grad_by_freq_: c_int, + sparse_: c_int, + ); + pub fn atg_embedding_bag( + out__: *mut *mut C_tensor, + weight_: *mut C_tensor, + indices_: *mut C_tensor, + offsets_: *mut C_tensor, + scale_grad_by_freq_: c_int, + mode_: i64, + sparse_: c_int, + per_sample_weights_: *mut C_tensor, + include_last_offset_: c_int, + ); + pub fn atg_embedding_bag_padding_idx( + out__: *mut *mut C_tensor, + weight_: *mut C_tensor, + indices_: *mut C_tensor, + offsets_: *mut C_tensor, + scale_grad_by_freq_: c_int, + mode_: i64, + sparse_: c_int, + per_sample_weights_: *mut C_tensor, + include_last_offset_: c_int, + padding_idx_v: i64, + padding_idx_null: i8, + ); + pub fn atg_embedding_dense_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + indices_: *mut C_tensor, + num_weights_: i64, + padding_idx_: i64, + scale_grad_by_freq_: c_int, + ); + pub fn atg_embedding_dense_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_output_: *mut C_tensor, + indices_: *mut C_tensor, + num_weights_: i64, + padding_idx_: i64, + scale_grad_by_freq_: c_int, + ); + pub fn atg_embedding_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + weight_: *mut C_tensor, + indices_: *mut C_tensor, + padding_idx_: i64, + scale_grad_by_freq_: c_int, + sparse_: c_int, + ); + pub fn atg_embedding_renorm( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + indices_: *mut C_tensor, + max_norm_: f64, + norm_type_: f64, + ); + pub fn atg_embedding_renorm_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + indices_: *mut C_tensor, + max_norm_: f64, + norm_type_: f64, + ); + pub fn atg_embedding_renorm_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + indices_: *mut C_tensor, + max_norm_: f64, + norm_type_: f64, + ); + pub fn atg_embedding_sparse_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + indices_: *mut C_tensor, + num_weights_: i64, + padding_idx_: i64, + scale_grad_by_freq_: c_int, + ); + pub fn atg_empty( + out__: *mut *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_empty_like(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_empty_like_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_empty_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg_empty_permuted( + out__: *mut *mut C_tensor, + size_data: *const i64, + size_len: c_int, + physical_layout_data: *const i64, + physical_layout_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_empty_permuted_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + physical_layout_data: *const i64, + physical_layout_len: c_int, + ); + pub fn atg_empty_quantized( + out__: *mut *mut C_tensor, + size_data: *const i64, + size_len: c_int, + qtensor_: *mut C_tensor, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_empty_quantized_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + qtensor_: *mut C_tensor, + ); + pub fn atg_empty_strided( + out__: *mut *mut C_tensor, + size_data: *const i64, + size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_empty_strided_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + ); + pub fn atg_eq(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_eq_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_eq_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_eq_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_eq_tensor_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_eq_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_equal(self_: *mut C_tensor, other_: *mut C_tensor) -> c_int; + pub fn atg_erf(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_erf_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_erf_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_erfc(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_erfc_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_erfc_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_erfinv(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_erfinv_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_erfinv_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_exp(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_exp2(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_exp2_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_exp2_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_exp_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_exp_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_expand( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + implicit_: c_int, + ); + pub fn atg_expand_as(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_expand_copy( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + implicit_: c_int, + ); + pub fn atg_expand_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + implicit_: c_int, + ); + pub fn atg_expm1(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_expm1_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_expm1_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_exponential(out__: *mut *mut C_tensor, self_: *mut C_tensor, lambd_: f64); + pub fn atg_exponential_(out__: *mut *mut C_tensor, self_: *mut C_tensor, lambd_: f64); + pub fn atg_exponential_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + lambd_: f64, + ); + pub fn atg_eye(out__: *mut *mut C_tensor, n_: i64, options_kind: c_int, options_device: c_int); + pub fn atg_eye_m( + out__: *mut *mut C_tensor, + n_: i64, + m_: i64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_eye_m_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, n_: i64, m_: i64); + pub fn atg_eye_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, n_: i64); + pub fn atg_fake_quantize_per_channel_affine( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + scale_: *mut C_tensor, + zero_point_: *mut C_tensor, + axis_: i64, + quant_min_: i64, + quant_max_: i64, + ); + pub fn atg_fake_quantize_per_channel_affine_cachemask( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + scale_: *mut C_tensor, + zero_point_: *mut C_tensor, + axis_: i64, + quant_min_: i64, + quant_max_: i64, + ); + pub fn atg_fake_quantize_per_channel_affine_cachemask_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + mask_: *mut C_tensor, + ); + pub fn atg_fake_quantize_per_channel_affine_cachemask_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + self_: *mut C_tensor, + scale_: *mut C_tensor, + zero_point_: *mut C_tensor, + axis_: i64, + quant_min_: i64, + quant_max_: i64, + ); + pub fn atg_fake_quantize_per_tensor_affine( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + scale_: f64, + zero_point_: i64, + quant_min_: i64, + quant_max_: i64, + ); + pub fn atg_fake_quantize_per_tensor_affine_cachemask( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + scale_: f64, + zero_point_: i64, + quant_min_: i64, + quant_max_: i64, + ); + pub fn atg_fake_quantize_per_tensor_affine_cachemask_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + mask_: *mut C_tensor, + ); + pub fn atg_fake_quantize_per_tensor_affine_cachemask_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + self_: *mut C_tensor, + scale_: f64, + zero_point_: i64, + quant_min_: i64, + quant_max_: i64, + ); + pub fn atg_fake_quantize_per_tensor_affine_tensor_qparams( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + scale_: *mut C_tensor, + zero_point_: *mut C_tensor, + quant_min_: i64, + quant_max_: i64, + ); + pub fn atg_feature_alpha_dropout( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + p_: f64, + train_: c_int, + ); + pub fn atg_feature_alpha_dropout_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + p_: f64, + train_: c_int, + ); + pub fn atg_feature_dropout( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + p_: f64, + train_: c_int, + ); + pub fn atg_feature_dropout_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + p_: f64, + train_: c_int, + ); + pub fn atg_fft_fft( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + n_v: i64, + n_null: i8, + dim_: i64, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_fft2( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_fft2_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_fft_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + n_v: i64, + n_null: i8, + dim_: i64, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_fftfreq( + out__: *mut *mut C_tensor, + n_: i64, + d_: f64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_fft_fftfreq_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, n_: i64, d_: f64); + pub fn atg_fft_fftn( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_fftn_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_fftshift( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + ); + pub fn atg_fft_hfft( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + n_v: i64, + n_null: i8, + dim_: i64, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_hfft2( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_hfft2_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_hfft_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + n_v: i64, + n_null: i8, + dim_: i64, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_hfftn( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_hfftn_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_ifft( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + n_v: i64, + n_null: i8, + dim_: i64, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_ifft2( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_ifft2_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_ifft_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + n_v: i64, + n_null: i8, + dim_: i64, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_ifftn( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_ifftn_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_ifftshift( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + ); + pub fn atg_fft_ihfft( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + n_v: i64, + n_null: i8, + dim_: i64, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_ihfft2( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_ihfft2_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_ihfft_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + n_v: i64, + n_null: i8, + dim_: i64, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_ihfftn( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_ihfftn_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_irfft( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + n_v: i64, + n_null: i8, + dim_: i64, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_irfft2( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_irfft2_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_irfft_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + n_v: i64, + n_null: i8, + dim_: i64, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_irfftn( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_irfftn_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_rfft( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + n_v: i64, + n_null: i8, + dim_: i64, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_rfft2( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_rfft2_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_rfft_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + n_v: i64, + n_null: i8, + dim_: i64, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_rfftfreq( + out__: *mut *mut C_tensor, + n_: i64, + d_: f64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_fft_rfftfreq_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, n_: i64, d_: f64); + pub fn atg_fft_rfftn( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fft_rfftn_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + s_data: *const i64, + s_len: c_int, + dim_data: *const i64, + dim_len: c_int, + norm_ptr: *const u8, + norm_len: c_int, + ); + pub fn atg_fill(out__: *mut *mut C_tensor, self_: *mut C_tensor, value_: *mut C_scalar); + pub fn atg_fill_(out__: *mut *mut C_tensor, self_: *mut C_tensor, value_: *mut C_scalar); + pub fn atg_fill_diagonal_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + fill_value_: *mut C_scalar, + wrap_: c_int, + ); + pub fn atg_fill_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + value_: *mut C_scalar, + ); + pub fn atg_fill_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor, value_: *mut C_tensor); + pub fn atg_fill_tensor_(out__: *mut *mut C_tensor, self_: *mut C_tensor, value_: *mut C_tensor); + pub fn atg_fill_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + value_: *mut C_tensor, + ); + pub fn atg_fix(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_fix_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_fix_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_flatten( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + start_dim_: i64, + end_dim_: i64, + ); + pub fn atg_flatten_dense_tensors( + out__: *mut *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ); + pub fn atg_flip( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dims_data: *const i64, + dims_len: c_int, + ); + pub fn atg_flip_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dims_data: *const i64, + dims_len: c_int, + ); + pub fn atg_fliplr(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_flipud(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_float_power( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + exponent_: *mut C_tensor, + ); + pub fn atg_float_power_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + exponent_: *mut C_scalar, + ); + pub fn atg_float_power_scalar( + out__: *mut *mut C_tensor, + self_scalar_: *mut C_scalar, + exponent_: *mut C_tensor, + ); + pub fn atg_float_power_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_scalar_: *mut C_scalar, + exponent_: *mut C_tensor, + ); + pub fn atg_float_power_tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + exponent_: *mut C_tensor, + ); + pub fn atg_float_power_tensor_scalar( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + exponent_: *mut C_scalar, + ); + pub fn atg_float_power_tensor_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + exponent_: *mut C_scalar, + ); + pub fn atg_float_power_tensor_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + exponent_: *mut C_tensor, + ); + pub fn atg_floor(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_floor_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_floor_divide(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_floor_divide_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_floor_divide_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_floor_divide_scalar( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_floor_divide_scalar_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_floor_divide_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_floor_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_fmax(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_fmax_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_fmin(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_fmin_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_fmod(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_fmod_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_fmod_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_fmod_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_fmod_tensor_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_fmod_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_frac(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_frac_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_frac_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_fractional_max_pool2d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + output_size_data: *const i64, + output_size_len: c_int, + random_samples_: *mut C_tensor, + ); + pub fn atg_fractional_max_pool2d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + output_size_data: *const i64, + output_size_len: c_int, + indices_: *mut C_tensor, + ); + pub fn atg_fractional_max_pool2d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + output_size_data: *const i64, + output_size_len: c_int, + indices_: *mut C_tensor, + ); + pub fn atg_fractional_max_pool2d_output( + out__: *mut *mut C_tensor, + output_: *mut C_tensor, + indices_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + output_size_data: *const i64, + output_size_len: c_int, + random_samples_: *mut C_tensor, + ); + pub fn atg_fractional_max_pool3d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + output_size_data: *const i64, + output_size_len: c_int, + random_samples_: *mut C_tensor, + ); + pub fn atg_fractional_max_pool3d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + output_size_data: *const i64, + output_size_len: c_int, + indices_: *mut C_tensor, + ); + pub fn atg_fractional_max_pool3d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + output_size_data: *const i64, + output_size_len: c_int, + indices_: *mut C_tensor, + ); + pub fn atg_fractional_max_pool3d_output( + out__: *mut *mut C_tensor, + output_: *mut C_tensor, + indices_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + output_size_data: *const i64, + output_size_len: c_int, + random_samples_: *mut C_tensor, + ); + pub fn atg_frexp(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_frexp_tensor_out( + out__: *mut *mut C_tensor, + mantissa_: *mut C_tensor, + exponent_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_frobenius_norm( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + ); + pub fn atg_frobenius_norm_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + ); + pub fn atg_from_file( + out__: *mut *mut C_tensor, + filename_ptr: *const u8, + filename_len: c_int, + shared_: c_int, + size_v: i64, + size_null: i8, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_from_file_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + filename_ptr: *const u8, + filename_len: c_int, + shared_: c_int, + size_v: i64, + size_null: i8, + ); + pub fn atg_full( + out__: *mut *mut C_tensor, + size_data: *const i64, + size_len: c_int, + fill_value_: *mut C_scalar, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_full_like( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + fill_value_: *mut C_scalar, + ); + pub fn atg_full_like_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + fill_value_: *mut C_scalar, + ); + pub fn atg_full_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + fill_value_: *mut C_scalar, + ); + pub fn atg_fused_moving_avg_obs_fake_quant( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + observer_on_: *mut C_tensor, + fake_quant_on_: *mut C_tensor, + running_min_: *mut C_tensor, + running_max_: *mut C_tensor, + scale_: *mut C_tensor, + zero_point_: *mut C_tensor, + averaging_const_: f64, + quant_min_: i64, + quant_max_: i64, + ch_axis_: i64, + per_row_fake_quant_: c_int, + symmetric_quant_: c_int, + ); + pub fn atg_gather( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + sparse_grad_: c_int, + ); + pub fn atg_gather_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + sparse_grad_: c_int, + ); + pub fn atg_gather_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + sparse_grad_: c_int, + ); + pub fn atg_gcd(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_gcd_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_gcd_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_ge(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_ge_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_ge_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_ge_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_ge_tensor_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_ge_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_gelu( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + approximate_ptr: *const u8, + approximate_len: c_int, + ); + pub fn atg_gelu_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + approximate_ptr: *const u8, + approximate_len: c_int, + ); + pub fn atg_gelu_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + approximate_ptr: *const u8, + approximate_len: c_int, + ); + pub fn atg_gelu_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + approximate_ptr: *const u8, + approximate_len: c_int, + ); + pub fn atg_gelu_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + approximate_ptr: *const u8, + approximate_len: c_int, + ); + pub fn atg_geometric(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: f64); + pub fn atg_geometric_(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: f64); + pub fn atg_geometric_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + p_: f64, + ); + pub fn atg_geqrf(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_geqrf_a( + out__: *mut *mut C_tensor, + a_: *mut C_tensor, + tau_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_ger(out__: *mut *mut C_tensor, self_: *mut C_tensor, vec2_: *mut C_tensor); + pub fn atg_ger_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + vec2_: *mut C_tensor, + ); + pub fn atg_glu(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64); + pub fn atg_glu_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + ); + pub fn atg_glu_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + ); + pub fn atg_glu_backward_jvp( + out__: *mut *mut C_tensor, + grad_x_: *mut C_tensor, + grad_glu_: *mut C_tensor, + x_: *mut C_tensor, + dgrad_glu_: *mut C_tensor, + dx_: *mut C_tensor, + dim_: i64, + ); + pub fn atg_glu_backward_jvp_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_x_: *mut C_tensor, + grad_glu_: *mut C_tensor, + x_: *mut C_tensor, + dgrad_glu_: *mut C_tensor, + dx_: *mut C_tensor, + dim_: i64, + ); + pub fn atg_glu_jvp( + out__: *mut *mut C_tensor, + glu_: *mut C_tensor, + x_: *mut C_tensor, + dx_: *mut C_tensor, + dim_: i64, + ); + pub fn atg_glu_jvp_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + glu_: *mut C_tensor, + x_: *mut C_tensor, + dx_: *mut C_tensor, + dim_: i64, + ); + pub fn atg_glu_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + ); + pub fn atg_grad(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_greater(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_greater_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_greater_equal( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_greater_equal_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_greater_equal_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_greater_equal_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_greater_equal_tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_greater_equal_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_greater_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_greater_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_greater_tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_greater_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_grid_sampler( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + grid_: *mut C_tensor, + interpolation_mode_: i64, + padding_mode_: i64, + align_corners_: c_int, + ); + pub fn atg_grid_sampler_2d( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + grid_: *mut C_tensor, + interpolation_mode_: i64, + padding_mode_: i64, + align_corners_: c_int, + ); + pub fn atg_grid_sampler_2d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + input_: *mut C_tensor, + grid_: *mut C_tensor, + interpolation_mode_: i64, + padding_mode_: i64, + align_corners_: c_int, + ); + pub fn atg_grid_sampler_3d( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + grid_: *mut C_tensor, + interpolation_mode_: i64, + padding_mode_: i64, + align_corners_: c_int, + ); + pub fn atg_grid_sampler_3d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + input_: *mut C_tensor, + grid_: *mut C_tensor, + interpolation_mode_: i64, + padding_mode_: i64, + align_corners_: c_int, + ); + pub fn atg_group_norm( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + num_groups_: i64, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + eps_: f64, + cudnn_enabled_: c_int, + ); + pub fn atg_gru( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + hx_: *mut C_tensor, + params_data: *const *mut C_tensor, + params_len: c_int, + has_biases_: c_int, + num_layers_: i64, + dropout_: f64, + train_: c_int, + bidirectional_: c_int, + batch_first_: c_int, + ); + pub fn atg_gru_cell( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + hx_: *mut C_tensor, + w_ih_: *mut C_tensor, + w_hh_: *mut C_tensor, + b_ih_: *mut C_tensor, + b_hh_: *mut C_tensor, + ); + pub fn atg_gru_data( + out__: *mut *mut C_tensor, + data_: *mut C_tensor, + batch_sizes_: *mut C_tensor, + hx_: *mut C_tensor, + params_data: *const *mut C_tensor, + params_len: c_int, + has_biases_: c_int, + num_layers_: i64, + dropout_: f64, + train_: c_int, + bidirectional_: c_int, + ); + pub fn atg_gt(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_gt_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_gt_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_gt_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_gt_tensor_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_gt_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_hamming_window( + out__: *mut *mut C_tensor, + window_length_: i64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_hamming_window_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + window_length_: i64, + ); + pub fn atg_hamming_window_periodic( + out__: *mut *mut C_tensor, + window_length_: i64, + periodic_: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_hamming_window_periodic_alpha( + out__: *mut *mut C_tensor, + window_length_: i64, + periodic_: c_int, + alpha_: f64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_hamming_window_periodic_alpha_beta( + out__: *mut *mut C_tensor, + window_length_: i64, + periodic_: c_int, + alpha_: f64, + beta_: f64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_hamming_window_periodic_alpha_beta_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + window_length_: i64, + periodic_: c_int, + alpha_: f64, + beta_: f64, + ); + pub fn atg_hamming_window_periodic_alpha_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + window_length_: i64, + periodic_: c_int, + alpha_: f64, + ); + pub fn atg_hamming_window_periodic_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + window_length_: i64, + periodic_: c_int, + ); + pub fn atg_hann_window( + out__: *mut *mut C_tensor, + window_length_: i64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_hann_window_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, window_length_: i64); + pub fn atg_hann_window_periodic( + out__: *mut *mut C_tensor, + window_length_: i64, + periodic_: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_hann_window_periodic_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + window_length_: i64, + periodic_: c_int, + ); + pub fn atg_hardshrink(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_hardshrink_backward( + out__: *mut *mut C_tensor, + grad_out_: *mut C_tensor, + self_: *mut C_tensor, + lambd_: *mut C_scalar, + ); + pub fn atg_hardshrink_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_out_: *mut C_tensor, + self_: *mut C_tensor, + lambd_: *mut C_scalar, + ); + pub fn atg_hardshrink_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_hardsigmoid(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_hardsigmoid_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_hardsigmoid_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_hardsigmoid_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_hardsigmoid_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_hardswish(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_hardswish_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_hardswish_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_hardswish_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_hardswish_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_hardtanh(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_hardtanh_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_hardtanh_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + min_val_: *mut C_scalar, + max_val_: *mut C_scalar, + ); + pub fn atg_hardtanh_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + min_val_: *mut C_scalar, + max_val_: *mut C_scalar, + ); + pub fn atg_hardtanh_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_hash_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + mode_: i64, + ); + pub fn atg_hash_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + mode_: i64, + ); + pub fn atg_heaviside(out__: *mut *mut C_tensor, self_: *mut C_tensor, values_: *mut C_tensor); + pub fn atg_heaviside_(out__: *mut *mut C_tensor, self_: *mut C_tensor, values_: *mut C_tensor); + pub fn atg_heaviside_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + values_: *mut C_tensor, + ); + pub fn atg_hinge_embedding_loss( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + margin_: f64, + reduction_: i64, + ); + pub fn atg_histc(out__: *mut *mut C_tensor, self_: *mut C_tensor, bins_: i64); + pub fn atg_histc_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + bins_: i64, + ); + pub fn atg_histogram( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + bins_: *mut C_tensor, + weight_: *mut C_tensor, + density_: c_int, + ); + pub fn atg_histogram_bin_ct( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + bins_: i64, + range_data: *const f64, + range_len: c_int, + weight_: *mut C_tensor, + density_: c_int, + ); + pub fn atg_histogram_bin_ct_out( + out__: *mut *mut C_tensor, + hist_: *mut C_tensor, + bin_edges_: *mut C_tensor, + self_: *mut C_tensor, + bins_: i64, + range_data: *const f64, + range_len: c_int, + weight_: *mut C_tensor, + density_: c_int, + ); + pub fn atg_histogram_bins_tensor_out( + out__: *mut *mut C_tensor, + hist_: *mut C_tensor, + bin_edges_: *mut C_tensor, + self_: *mut C_tensor, + bins_: *mut C_tensor, + weight_: *mut C_tensor, + density_: c_int, + ); + pub fn atg_hsplit(self_: *mut C_tensor, sections_: i64) -> *mut *mut C_tensor; + pub fn atg_hsplit_array( + self_: *mut C_tensor, + indices_data: *const i64, + indices_len: c_int, + ) -> *mut *mut C_tensor; + pub fn atg_hspmm(out__: *mut *mut C_tensor, mat1_: *mut C_tensor, mat2_: *mut C_tensor); + pub fn atg_hspmm_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + mat1_: *mut C_tensor, + mat2_: *mut C_tensor, + ); + pub fn atg_hstack( + out__: *mut *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ); + pub fn atg_hstack_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ); + pub fn atg_huber_loss( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + reduction_: i64, + delta_: f64, + ); + pub fn atg_huber_loss_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + reduction_: i64, + delta_: f64, + ); + pub fn atg_huber_loss_backward_out( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + reduction_: i64, + delta_: f64, + ); + pub fn atg_huber_loss_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + reduction_: i64, + delta_: f64, + ); + pub fn atg_hypot(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_hypot_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_hypot_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_i0(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_i0_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_i0_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_igamma(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_igamma_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_igamma_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_igammac(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_igammac_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_igammac_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_im2col( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + padding_data: *const i64, + padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + ); + pub fn atg_im2col_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + padding_data: *const i64, + padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + ); + pub fn atg_imag(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_index( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + indices_data: *const *mut C_tensor, + indices_len: c_int, + ); + pub fn atg_index_add( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + source_: *mut C_tensor, + ); + pub fn atg_index_add_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + source_: *mut C_tensor, + ); + pub fn atg_index_add_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + source_: *mut C_tensor, + ); + pub fn atg_index_copy( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + source_: *mut C_tensor, + ); + pub fn atg_index_copy_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + source_: *mut C_tensor, + ); + pub fn atg_index_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + source_: *mut C_tensor, + ); + pub fn atg_index_fill( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + value_: *mut C_scalar, + ); + pub fn atg_index_fill_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + value_: *mut C_scalar, + ); + pub fn atg_index_fill_int_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + value_: *mut C_scalar, + ); + pub fn atg_index_fill_int_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + value_: *mut C_tensor, + ); + pub fn atg_index_fill_int_tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + value_: *mut C_tensor, + ); + pub fn atg_index_fill_int_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + value_: *mut C_tensor, + ); + pub fn atg_index_put( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + indices_data: *const *mut C_tensor, + indices_len: c_int, + values_: *mut C_tensor, + accumulate_: c_int, + ); + pub fn atg_index_put_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + indices_data: *const *mut C_tensor, + indices_len: c_int, + values_: *mut C_tensor, + accumulate_: c_int, + ); + pub fn atg_index_put_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + indices_data: *const *mut C_tensor, + indices_len: c_int, + values_: *mut C_tensor, + accumulate_: c_int, + ); + pub fn atg_index_reduce( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + source_: *mut C_tensor, + reduce_ptr: *const u8, + reduce_len: c_int, + include_self_: c_int, + ); + pub fn atg_index_reduce_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + source_: *mut C_tensor, + reduce_ptr: *const u8, + reduce_len: c_int, + include_self_: c_int, + ); + pub fn atg_index_reduce_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + source_: *mut C_tensor, + reduce_ptr: *const u8, + reduce_len: c_int, + include_self_: c_int, + ); + pub fn atg_index_select( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + ); + pub fn atg_index_select_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + self_sizes_data: *const i64, + self_sizes_len: c_int, + dim_: i64, + index_: *mut C_tensor, + ); + pub fn atg_index_select_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + ); + pub fn atg_index_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + indices_data: *const *mut C_tensor, + indices_len: c_int, + ); + pub fn atg_indices(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_indices_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_indices_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_infinitely_differentiable_gelu_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_inner(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_inner_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_instance_norm( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + use_input_stats_: c_int, + momentum_: f64, + eps_: f64, + cudnn_enabled_: c_int, + ); + pub fn atg_int_repr(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_int_repr_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_inverse(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_inverse_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_is_coalesced(self_: *mut C_tensor) -> c_int; + pub fn atg_is_complex(self_: *mut C_tensor) -> c_int; + pub fn atg_is_conj(self_: *mut C_tensor) -> c_int; + pub fn atg_is_distributed(self_: *mut C_tensor) -> c_int; + pub fn atg_is_floating_point(self_: *mut C_tensor) -> c_int; + pub fn atg_is_inference(self_: *mut C_tensor) -> c_int; + pub fn atg_is_leaf(self_: *mut C_tensor) -> c_int; + pub fn atg_is_neg(self_: *mut C_tensor) -> c_int; + pub fn atg_is_nonzero(self_: *mut C_tensor) -> c_int; + pub fn atg_is_pinned(self_: *mut C_tensor, device_: c_int) -> c_int; + pub fn atg_is_same_size(self_: *mut C_tensor, other_: *mut C_tensor) -> c_int; + pub fn atg_is_set_to(self_: *mut C_tensor, tensor_: *mut C_tensor) -> c_int; + pub fn atg_is_signed(self_: *mut C_tensor) -> c_int; + pub fn atg_is_vulkan_available() -> c_int; + pub fn atg_isclose( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + rtol_: f64, + atol_: f64, + equal_nan_: c_int, + ); + pub fn atg_isfinite(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_isin( + out__: *mut *mut C_tensor, + elements_: *mut C_tensor, + test_elements_: *mut C_tensor, + assume_unique_: c_int, + invert_: c_int, + ); + pub fn atg_isin_scalar_tensor( + out__: *mut *mut C_tensor, + element_: *mut C_scalar, + test_elements_: *mut C_tensor, + assume_unique_: c_int, + invert_: c_int, + ); + pub fn atg_isin_scalar_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + element_: *mut C_scalar, + test_elements_: *mut C_tensor, + assume_unique_: c_int, + invert_: c_int, + ); + pub fn atg_isin_tensor_scalar( + out__: *mut *mut C_tensor, + elements_: *mut C_tensor, + test_element_: *mut C_scalar, + assume_unique_: c_int, + invert_: c_int, + ); + pub fn atg_isin_tensor_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + elements_: *mut C_tensor, + test_element_: *mut C_scalar, + assume_unique_: c_int, + invert_: c_int, + ); + pub fn atg_isin_tensor_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + elements_: *mut C_tensor, + test_elements_: *mut C_tensor, + assume_unique_: c_int, + invert_: c_int, + ); + pub fn atg_isinf(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_isinf_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_isnan(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_isnan_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_isneginf(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_isneginf_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_isposinf(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_isposinf_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_isreal(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_istft( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + n_fft_: i64, + hop_length_v: i64, + hop_length_null: i8, + win_length_v: i64, + win_length_null: i8, + window_: *mut C_tensor, + center_: c_int, + normalized_: c_int, + onesided_: c_int, + length_v: i64, + length_null: i8, + return_complex_: c_int, + ); + pub fn atg_kaiser_window( + out__: *mut *mut C_tensor, + window_length_: i64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_kaiser_window_beta( + out__: *mut *mut C_tensor, + window_length_: i64, + periodic_: c_int, + beta_: f64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_kaiser_window_beta_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + window_length_: i64, + periodic_: c_int, + beta_: f64, + ); + pub fn atg_kaiser_window_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + window_length_: i64, + ); + pub fn atg_kaiser_window_periodic( + out__: *mut *mut C_tensor, + window_length_: i64, + periodic_: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_kaiser_window_periodic_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + window_length_: i64, + periodic_: c_int, + ); + pub fn atg_kl_div( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + reduction_: i64, + log_target_: c_int, + ); + pub fn atg_kron(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_kron_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_kthvalue( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + k_: i64, + dim_: i64, + keepdim_: c_int, + ); + pub fn atg_kthvalue_values( + out__: *mut *mut C_tensor, + values_: *mut C_tensor, + indices_: *mut C_tensor, + self_: *mut C_tensor, + k_: i64, + dim_: i64, + keepdim_: c_int, + ); + pub fn atg_l1_loss( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + reduction_: i64, + ); + pub fn atg_layer_norm( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + normalized_shape_data: *const i64, + normalized_shape_len: c_int, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + eps_: f64, + cudnn_enable_: c_int, + ); + pub fn atg_lcm(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_lcm_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_lcm_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_ldexp(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_ldexp_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_ldexp_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_le(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_le_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_le_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_le_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_le_tensor_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_le_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_leaky_relu(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_leaky_relu_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_leaky_relu_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + negative_slope_: *mut C_scalar, + self_is_result_: c_int, + ); + pub fn atg_leaky_relu_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + negative_slope_: *mut C_scalar, + self_is_result_: c_int, + ); + pub fn atg_leaky_relu_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_lerp( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + end_: *mut C_tensor, + weight_: *mut C_scalar, + ); + pub fn atg_lerp_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + end_: *mut C_tensor, + weight_: *mut C_scalar, + ); + pub fn atg_lerp_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + end_: *mut C_tensor, + weight_: *mut C_scalar, + ); + pub fn atg_lerp_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + end_: *mut C_tensor, + weight_: *mut C_tensor, + ); + pub fn atg_lerp_tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + end_: *mut C_tensor, + weight_: *mut C_tensor, + ); + pub fn atg_lerp_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + end_: *mut C_tensor, + weight_: *mut C_tensor, + ); + pub fn atg_less(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_less_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_less_equal(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_less_equal_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_less_equal_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_less_equal_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_less_equal_tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_less_equal_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_less_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_less_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_less_tensor_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_less_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_lgamma(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_lgamma_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_lgamma_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_lift(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_lift_fresh(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_lift_fresh_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_lift_fresh_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_lift_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_linalg_cholesky(out__: *mut *mut C_tensor, self_: *mut C_tensor, upper_: c_int); + pub fn atg_linalg_cholesky_ex( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + upper_: c_int, + check_errors_: c_int, + ); + pub fn atg_linalg_cholesky_ex_l( + out__: *mut *mut C_tensor, + L_: *mut C_tensor, + info_: *mut C_tensor, + self_: *mut C_tensor, + upper_: c_int, + check_errors_: c_int, + ); + pub fn atg_linalg_cholesky_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + upper_: c_int, + ); + pub fn atg_linalg_cond(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: *mut C_scalar); + pub fn atg_linalg_cond_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + p_: *mut C_scalar, + ); + pub fn atg_linalg_cond_p_str( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + p_ptr: *const u8, + p_len: c_int, + ); + pub fn atg_linalg_cond_p_str_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + p_ptr: *const u8, + p_len: c_int, + ); + pub fn atg_linalg_cross( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + dim_: i64, + ); + pub fn atg_linalg_cross_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + dim_: i64, + ); + pub fn atg_linalg_det(out__: *mut *mut C_tensor, A_: *mut C_tensor); + pub fn atg_linalg_det_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, A_: *mut C_tensor); + pub fn atg_linalg_diagonal( + out__: *mut *mut C_tensor, + A_: *mut C_tensor, + offset_: i64, + dim1_: i64, + dim2_: i64, + ); + pub fn atg_linalg_eig(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_linalg_eig_out( + out__: *mut *mut C_tensor, + eigenvalues_: *mut C_tensor, + eigenvectors_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_linalg_eigh( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + UPLO_ptr: *const u8, + UPLO_len: c_int, + ); + pub fn atg_linalg_eigh_eigvals( + out__: *mut *mut C_tensor, + eigvals_: *mut C_tensor, + eigvecs_: *mut C_tensor, + self_: *mut C_tensor, + UPLO_ptr: *const u8, + UPLO_len: c_int, + ); + pub fn atg_linalg_eigvals(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_linalg_eigvals_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_linalg_eigvalsh( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + UPLO_ptr: *const u8, + UPLO_len: c_int, + ); + pub fn atg_linalg_eigvalsh_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + UPLO_ptr: *const u8, + UPLO_len: c_int, + ); + pub fn atg_linalg_householder_product( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + tau_: *mut C_tensor, + ); + pub fn atg_linalg_householder_product_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + input_: *mut C_tensor, + tau_: *mut C_tensor, + ); + pub fn atg_linalg_inv(out__: *mut *mut C_tensor, A_: *mut C_tensor); + pub fn atg_linalg_inv_ex(out__: *mut *mut C_tensor, A_: *mut C_tensor, check_errors_: c_int); + pub fn atg_linalg_inv_ex_inverse( + out__: *mut *mut C_tensor, + inverse_: *mut C_tensor, + info_: *mut C_tensor, + A_: *mut C_tensor, + check_errors_: c_int, + ); + pub fn atg_linalg_inv_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, A_: *mut C_tensor); + pub fn atg_linalg_ldl_factor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + hermitian_: c_int, + ); + pub fn atg_linalg_ldl_factor_ex( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + hermitian_: c_int, + check_errors_: c_int, + ); + pub fn atg_linalg_ldl_factor_ex_out( + out__: *mut *mut C_tensor, + LD_: *mut C_tensor, + pivots_: *mut C_tensor, + info_: *mut C_tensor, + self_: *mut C_tensor, + hermitian_: c_int, + check_errors_: c_int, + ); + pub fn atg_linalg_ldl_factor_out( + out__: *mut *mut C_tensor, + LD_: *mut C_tensor, + pivots_: *mut C_tensor, + self_: *mut C_tensor, + hermitian_: c_int, + ); + pub fn atg_linalg_ldl_solve( + out__: *mut *mut C_tensor, + LD_: *mut C_tensor, + pivots_: *mut C_tensor, + B_: *mut C_tensor, + hermitian_: c_int, + ); + pub fn atg_linalg_ldl_solve_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + LD_: *mut C_tensor, + pivots_: *mut C_tensor, + B_: *mut C_tensor, + hermitian_: c_int, + ); + pub fn atg_linalg_lstsq( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + b_: *mut C_tensor, + rcond_v: f64, + rcond_null: i8, + driver_ptr: *const u8, + driver_len: c_int, + ); + pub fn atg_linalg_lstsq_out( + out__: *mut *mut C_tensor, + solution_: *mut C_tensor, + residuals_: *mut C_tensor, + rank_: *mut C_tensor, + singular_values_: *mut C_tensor, + self_: *mut C_tensor, + b_: *mut C_tensor, + rcond_v: f64, + rcond_null: i8, + driver_ptr: *const u8, + driver_len: c_int, + ); + pub fn atg_linalg_lu(out__: *mut *mut C_tensor, A_: *mut C_tensor, pivot_: c_int); + pub fn atg_linalg_lu_factor(out__: *mut *mut C_tensor, A_: *mut C_tensor, pivot_: c_int); + pub fn atg_linalg_lu_factor_ex( + out__: *mut *mut C_tensor, + A_: *mut C_tensor, + pivot_: c_int, + check_errors_: c_int, + ); + pub fn atg_linalg_lu_factor_ex_out( + out__: *mut *mut C_tensor, + LU_: *mut C_tensor, + pivots_: *mut C_tensor, + info_: *mut C_tensor, + A_: *mut C_tensor, + pivot_: c_int, + check_errors_: c_int, + ); + pub fn atg_linalg_lu_factor_out( + out__: *mut *mut C_tensor, + LU_: *mut C_tensor, + pivots_: *mut C_tensor, + A_: *mut C_tensor, + pivot_: c_int, + ); + pub fn atg_linalg_lu_out( + out__: *mut *mut C_tensor, + P_: *mut C_tensor, + L_: *mut C_tensor, + U_: *mut C_tensor, + A_: *mut C_tensor, + pivot_: c_int, + ); + pub fn atg_linalg_lu_solve( + out__: *mut *mut C_tensor, + LU_: *mut C_tensor, + pivots_: *mut C_tensor, + B_: *mut C_tensor, + left_: c_int, + adjoint_: c_int, + ); + pub fn atg_linalg_lu_solve_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + LU_: *mut C_tensor, + pivots_: *mut C_tensor, + B_: *mut C_tensor, + left_: c_int, + adjoint_: c_int, + ); + pub fn atg_linalg_matmul( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_linalg_matmul_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_linalg_matrix_exp(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_linalg_matrix_exp_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_linalg_matrix_power(out__: *mut *mut C_tensor, self_: *mut C_tensor, n_: i64); + pub fn atg_linalg_matrix_power_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + n_: i64, + ); + pub fn atg_linalg_matrix_rank( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + tol_: f64, + hermitian_: c_int, + ); + pub fn atg_linalg_matrix_rank_atol_rtol_float( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + atol_v: f64, + atol_null: i8, + rtol_v: f64, + rtol_null: i8, + hermitian_: c_int, + ); + pub fn atg_linalg_matrix_rank_atol_rtol_float_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + atol_v: f64, + atol_null: i8, + rtol_v: f64, + rtol_null: i8, + hermitian_: c_int, + ); + pub fn atg_linalg_matrix_rank_atol_rtol_tensor( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + atol_: *mut C_tensor, + rtol_: *mut C_tensor, + hermitian_: c_int, + ); + pub fn atg_linalg_matrix_rank_atol_rtol_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + input_: *mut C_tensor, + atol_: *mut C_tensor, + rtol_: *mut C_tensor, + hermitian_: c_int, + ); + pub fn atg_linalg_matrix_rank_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + tol_: f64, + hermitian_: c_int, + ); + pub fn atg_linalg_matrix_rank_out_tol_tensor( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + input_: *mut C_tensor, + tol_: *mut C_tensor, + hermitian_: c_int, + ); + pub fn atg_linalg_matrix_rank_tol_tensor( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + tol_: *mut C_tensor, + hermitian_: c_int, + ); + pub fn atg_linalg_multi_dot( + out__: *mut *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ); + pub fn atg_linalg_multi_dot_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ); + pub fn atg_linalg_norm( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + ord_: *mut C_scalar, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + dtype_: c_int, + ); + pub fn atg_linalg_norm_ord_str( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + ord_ptr: *const u8, + ord_len: c_int, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + dtype_: c_int, + ); + pub fn atg_linalg_norm_ord_str_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ord_ptr: *const u8, + ord_len: c_int, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + dtype_: c_int, + ); + pub fn atg_linalg_norm_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ord_: *mut C_scalar, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + dtype_: c_int, + ); + pub fn atg_linalg_pinv( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + rcond_: f64, + hermitian_: c_int, + ); + pub fn atg_linalg_pinv_atol_rtol_float( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + atol_v: f64, + atol_null: i8, + rtol_v: f64, + rtol_null: i8, + hermitian_: c_int, + ); + pub fn atg_linalg_pinv_atol_rtol_float_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + atol_v: f64, + atol_null: i8, + rtol_v: f64, + rtol_null: i8, + hermitian_: c_int, + ); + pub fn atg_linalg_pinv_atol_rtol_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + atol_: *mut C_tensor, + rtol_: *mut C_tensor, + hermitian_: c_int, + ); + pub fn atg_linalg_pinv_atol_rtol_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + atol_: *mut C_tensor, + rtol_: *mut C_tensor, + hermitian_: c_int, + ); + pub fn atg_linalg_pinv_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + rcond_: f64, + hermitian_: c_int, + ); + pub fn atg_linalg_pinv_out_rcond_tensor( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + rcond_: *mut C_tensor, + hermitian_: c_int, + ); + pub fn atg_linalg_pinv_rcond_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + rcond_: *mut C_tensor, + hermitian_: c_int, + ); + pub fn atg_linalg_qr( + out__: *mut *mut C_tensor, + A_: *mut C_tensor, + mode_ptr: *const u8, + mode_len: c_int, + ); + pub fn atg_linalg_qr_out( + out__: *mut *mut C_tensor, + Q_: *mut C_tensor, + R_: *mut C_tensor, + A_: *mut C_tensor, + mode_ptr: *const u8, + mode_len: c_int, + ); + pub fn atg_linalg_slogdet(out__: *mut *mut C_tensor, A_: *mut C_tensor); + pub fn atg_linalg_slogdet_out( + out__: *mut *mut C_tensor, + sign_: *mut C_tensor, + logabsdet_: *mut C_tensor, + A_: *mut C_tensor, + ); + pub fn atg_linalg_solve( + out__: *mut *mut C_tensor, + A_: *mut C_tensor, + B_: *mut C_tensor, + left_: c_int, + ); + pub fn atg_linalg_solve_ex( + out__: *mut *mut C_tensor, + A_: *mut C_tensor, + B_: *mut C_tensor, + left_: c_int, + check_errors_: c_int, + ); + pub fn atg_linalg_solve_ex_out( + out__: *mut *mut C_tensor, + result_: *mut C_tensor, + info_: *mut C_tensor, + A_: *mut C_tensor, + B_: *mut C_tensor, + left_: c_int, + check_errors_: c_int, + ); + pub fn atg_linalg_solve_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + A_: *mut C_tensor, + B_: *mut C_tensor, + left_: c_int, + ); + pub fn atg_linalg_solve_triangular( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + B_: *mut C_tensor, + upper_: c_int, + left_: c_int, + unitriangular_: c_int, + ); + pub fn atg_linalg_solve_triangular_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + B_: *mut C_tensor, + upper_: c_int, + left_: c_int, + unitriangular_: c_int, + ); + pub fn atg_linalg_svd( + out__: *mut *mut C_tensor, + A_: *mut C_tensor, + full_matrices_: c_int, + driver_ptr: *const u8, + driver_len: c_int, + ); + pub fn atg_linalg_svd_u( + out__: *mut *mut C_tensor, + U_: *mut C_tensor, + S_: *mut C_tensor, + Vh_: *mut C_tensor, + A_: *mut C_tensor, + full_matrices_: c_int, + driver_ptr: *const u8, + driver_len: c_int, + ); + pub fn atg_linalg_svdvals( + out__: *mut *mut C_tensor, + A_: *mut C_tensor, + driver_ptr: *const u8, + driver_len: c_int, + ); + pub fn atg_linalg_svdvals_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + A_: *mut C_tensor, + driver_ptr: *const u8, + driver_len: c_int, + ); + pub fn atg_linalg_tensorinv(out__: *mut *mut C_tensor, self_: *mut C_tensor, ind_: i64); + pub fn atg_linalg_tensorinv_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ind_: i64, + ); + pub fn atg_linalg_tensorsolve( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + dims_data: *const i64, + dims_len: c_int, + ); + pub fn atg_linalg_tensorsolve_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + dims_data: *const i64, + dims_len: c_int, + ); + pub fn atg_linalg_vander(out__: *mut *mut C_tensor, x_: *mut C_tensor, n_v: i64, n_null: i8); + pub fn atg_linalg_vecdot( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + y_: *mut C_tensor, + dim_: i64, + ); + pub fn atg_linalg_vecdot_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + y_: *mut C_tensor, + dim_: i64, + ); + pub fn atg_linear( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + ); + pub fn atg_linear_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + ); + pub fn atg_linspace( + out__: *mut *mut C_tensor, + start_: *mut C_scalar, + end_: *mut C_scalar, + steps_: i64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_linspace_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + start_: *mut C_scalar, + end_: *mut C_scalar, + steps_: i64, + ); + pub fn atg_linspace_scalar_tensor( + out__: *mut *mut C_tensor, + start_: *mut C_scalar, + end_: *mut C_tensor, + steps_: i64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_linspace_scalar_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + start_: *mut C_scalar, + end_: *mut C_tensor, + steps_: i64, + ); + pub fn atg_linspace_tensor_scalar( + out__: *mut *mut C_tensor, + start_: *mut C_tensor, + end_: *mut C_scalar, + steps_: i64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_linspace_tensor_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + start_: *mut C_tensor, + end_: *mut C_scalar, + steps_: i64, + ); + pub fn atg_linspace_tensor_tensor( + out__: *mut *mut C_tensor, + start_: *mut C_tensor, + end_: *mut C_tensor, + steps_: i64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_linspace_tensor_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + start_: *mut C_tensor, + end_: *mut C_tensor, + steps_: i64, + ); + pub fn atg_log(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_log10(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_log10_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_log10_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_log1p(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_log1p_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_log1p_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_log2(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_log2_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_log2_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_log_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_log_normal(out__: *mut *mut C_tensor, self_: *mut C_tensor, mean_: f64, std_: f64); + pub fn atg_log_normal_(out__: *mut *mut C_tensor, self_: *mut C_tensor, mean_: f64, std_: f64); + pub fn atg_log_normal_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mean_: f64, + std_: f64, + ); + pub fn atg_log_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_log_sigmoid(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_log_sigmoid_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + buffer_: *mut C_tensor, + ); + pub fn atg_log_sigmoid_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + buffer_: *mut C_tensor, + ); + pub fn atg_log_sigmoid_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_log_softmax( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + dtype_: c_int, + ); + pub fn atg_log_softmax_int_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + dtype_: c_int, + ); + pub fn atg_logaddexp(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_logaddexp2(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_logaddexp2_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_logaddexp_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_logcumsumexp(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64); + pub fn atg_logcumsumexp_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + ); + pub fn atg_logdet(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_logical_and(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_logical_and_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_logical_and_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_logical_not(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_logical_not_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_logical_not_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_logical_or(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_logical_or_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_logical_or_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_logical_xor(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_logical_xor_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_logical_xor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_logit(out__: *mut *mut C_tensor, self_: *mut C_tensor, eps_v: f64, eps_null: i8); + pub fn atg_logit_(out__: *mut *mut C_tensor, self_: *mut C_tensor, eps_v: f64, eps_null: i8); + pub fn atg_logit_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + eps_v: f64, + eps_null: i8, + ); + pub fn atg_logit_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + eps_v: f64, + eps_null: i8, + ); + pub fn atg_logit_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + eps_v: f64, + eps_null: i8, + ); + pub fn atg_logspace( + out__: *mut *mut C_tensor, + start_: *mut C_scalar, + end_: *mut C_scalar, + steps_: i64, + base_: f64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_logspace_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + start_: *mut C_scalar, + end_: *mut C_scalar, + steps_: i64, + base_: f64, + ); + pub fn atg_logspace_scalar_tensor( + out__: *mut *mut C_tensor, + start_: *mut C_scalar, + end_: *mut C_tensor, + steps_: i64, + base_: f64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_logspace_scalar_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + start_: *mut C_scalar, + end_: *mut C_tensor, + steps_: i64, + base_: f64, + ); + pub fn atg_logspace_tensor_scalar( + out__: *mut *mut C_tensor, + start_: *mut C_tensor, + end_: *mut C_scalar, + steps_: i64, + base_: f64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_logspace_tensor_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + start_: *mut C_tensor, + end_: *mut C_scalar, + steps_: i64, + base_: f64, + ); + pub fn atg_logspace_tensor_tensor( + out__: *mut *mut C_tensor, + start_: *mut C_tensor, + end_: *mut C_tensor, + steps_: i64, + base_: f64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_logspace_tensor_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + start_: *mut C_tensor, + end_: *mut C_tensor, + steps_: i64, + base_: f64, + ); + pub fn atg_logsumexp( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + ); + pub fn atg_logsumexp_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + ); + pub fn atg_lstm( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + hx_data: *const *mut C_tensor, + hx_len: c_int, + params_data: *const *mut C_tensor, + params_len: c_int, + has_biases_: c_int, + num_layers_: i64, + dropout_: f64, + train_: c_int, + bidirectional_: c_int, + batch_first_: c_int, + ); + pub fn atg_lstm_cell( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + hx_data: *const *mut C_tensor, + hx_len: c_int, + w_ih_: *mut C_tensor, + w_hh_: *mut C_tensor, + b_ih_: *mut C_tensor, + b_hh_: *mut C_tensor, + ); + pub fn atg_lstm_data( + out__: *mut *mut C_tensor, + data_: *mut C_tensor, + batch_sizes_: *mut C_tensor, + hx_data: *const *mut C_tensor, + hx_len: c_int, + params_data: *const *mut C_tensor, + params_len: c_int, + has_biases_: c_int, + num_layers_: i64, + dropout_: f64, + train_: c_int, + bidirectional_: c_int, + ); + pub fn atg_lstm_mps_backward( + out0_: *mut C_tensor, + out1_data: *const *mut C_tensor, + out1_len: c_int, + out2_data: *const *mut C_tensor, + out2_len: c_int, + grad_y_: *mut C_tensor, + grad_hy_: *mut C_tensor, + grad_cy_: *mut C_tensor, + z_state_: *mut C_tensor, + cell_state_fwd_: *mut C_tensor, + input_: *mut C_tensor, + layersOutputs_: *mut C_tensor, + hx_data: *const *mut C_tensor, + hx_len: c_int, + params_data: *const *mut C_tensor, + params_len: c_int, + has_biases_: c_int, + num_layers_: i64, + dropout_: f64, + train_: c_int, + bidirectional_: c_int, + batch_first_: c_int, + ); + pub fn atg_lt(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_lt_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_lt_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_lt_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_lt_tensor_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_lt_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_lu_solve( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + LU_data_: *mut C_tensor, + LU_pivots_: *mut C_tensor, + ); + pub fn atg_lu_solve_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + LU_data_: *mut C_tensor, + LU_pivots_: *mut C_tensor, + ); + pub fn atg_lu_unpack( + out__: *mut *mut C_tensor, + LU_data_: *mut C_tensor, + LU_pivots_: *mut C_tensor, + unpack_data_: c_int, + unpack_pivots_: c_int, + ); + pub fn atg_lu_unpack_out( + out__: *mut *mut C_tensor, + P_: *mut C_tensor, + L_: *mut C_tensor, + U_: *mut C_tensor, + LU_data_: *mut C_tensor, + LU_pivots_: *mut C_tensor, + unpack_data_: c_int, + unpack_pivots_: c_int, + ); + pub fn atg_margin_ranking_loss( + out__: *mut *mut C_tensor, + input1_: *mut C_tensor, + input2_: *mut C_tensor, + target_: *mut C_tensor, + margin_: f64, + reduction_: i64, + ); + pub fn atg_masked_fill( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mask_: *mut C_tensor, + value_: *mut C_scalar, + ); + pub fn atg_masked_fill_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mask_: *mut C_tensor, + value_: *mut C_scalar, + ); + pub fn atg_masked_fill_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mask_: *mut C_tensor, + value_: *mut C_scalar, + ); + pub fn atg_masked_fill_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mask_: *mut C_tensor, + value_: *mut C_tensor, + ); + pub fn atg_masked_fill_tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mask_: *mut C_tensor, + value_: *mut C_tensor, + ); + pub fn atg_masked_fill_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mask_: *mut C_tensor, + value_: *mut C_tensor, + ); + pub fn atg_masked_scatter( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mask_: *mut C_tensor, + source_: *mut C_tensor, + ); + pub fn atg_masked_scatter_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mask_: *mut C_tensor, + source_: *mut C_tensor, + ); + pub fn atg_masked_scatter_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + mask_: *mut C_tensor, + sizes_data: *const i64, + sizes_len: c_int, + ); + pub fn atg_masked_scatter_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mask_: *mut C_tensor, + source_: *mut C_tensor, + ); + pub fn atg_masked_select(out__: *mut *mut C_tensor, self_: *mut C_tensor, mask_: *mut C_tensor); + pub fn atg_masked_select_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + input_: *mut C_tensor, + mask_: *mut C_tensor, + ); + pub fn atg_masked_select_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mask_: *mut C_tensor, + ); + pub fn atg_matmul(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_matmul_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_matrix_exp(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_matrix_exp_backward( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + grad_: *mut C_tensor, + ); + pub fn atg_matrix_h(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_matrix_power(out__: *mut *mut C_tensor, self_: *mut C_tensor, n_: i64); + pub fn atg_matrix_power_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + n_: i64, + ); + pub fn atg_max(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_max_dim(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, keepdim_: c_int); + pub fn atg_max_dim_max( + out__: *mut *mut C_tensor, + max_: *mut C_tensor, + max_values_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + keepdim_: c_int, + ); + pub fn atg_max_other(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_max_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_max_pool1d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_max_pool1d_with_indices( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_max_pool2d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_max_pool2d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_max_pool2d_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_max_pool2d_with_indices( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_max_pool2d_with_indices_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + indices_: *mut C_tensor, + ); + pub fn atg_max_pool2d_with_indices_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + indices_: *mut C_tensor, + ); + pub fn atg_max_pool2d_with_indices_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + indices_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_max_pool3d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_max_pool3d_with_indices( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_max_pool3d_with_indices_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + indices_: *mut C_tensor, + ); + pub fn atg_max_pool3d_with_indices_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + indices_: *mut C_tensor, + ); + pub fn atg_max_pool3d_with_indices_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + indices_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_max_unary_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_max_unpool2d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + indices_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + ); + pub fn atg_max_unpool2d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + indices_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + ); + pub fn atg_max_unpool3d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + indices_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_max_unpool3d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + indices_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_maximum(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_maximum_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_mean(out__: *mut *mut C_tensor, self_: *mut C_tensor, dtype_: c_int); + pub fn atg_mean_dim( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + dtype_: c_int, + ); + pub fn atg_mean_dtype_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dtype_: c_int, + ); + pub fn atg_mean_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + dtype_: c_int, + ); + pub fn atg_median(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_median_dim( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + keepdim_: c_int, + ); + pub fn atg_median_dim_values( + out__: *mut *mut C_tensor, + values_: *mut C_tensor, + indices_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + keepdim_: c_int, + ); + pub fn atg_median_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_meshgrid( + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ) -> *mut *mut C_tensor; + pub fn atg_meshgrid_indexing( + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + indexing_ptr: *const u8, + indexing_len: c_int, + ) -> *mut *mut C_tensor; + pub fn atg_mh(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_min(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_min_dim(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, keepdim_: c_int); + pub fn atg_min_dim_min( + out__: *mut *mut C_tensor, + min_: *mut C_tensor, + min_indices_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + keepdim_: c_int, + ); + pub fn atg_min_other(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_min_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_min_unary_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_minimum(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_minimum_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_miopen_batch_norm( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + training_: c_int, + exponential_average_factor_: f64, + epsilon_: f64, + ); + pub fn atg_miopen_batch_norm_backward( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + grad_output_: *mut C_tensor, + weight_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + save_mean_: *mut C_tensor, + save_var_: *mut C_tensor, + epsilon_: f64, + ); + pub fn atg_miopen_batch_norm_backward_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + out2_: *mut C_tensor, + input_: *mut C_tensor, + grad_output_: *mut C_tensor, + weight_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + save_mean_: *mut C_tensor, + save_var_: *mut C_tensor, + epsilon_: f64, + ); + pub fn atg_miopen_batch_norm_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + out2_: *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + training_: c_int, + exponential_average_factor_: f64, + epsilon_: f64, + ); + pub fn atg_miopen_convolution( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + benchmark_: c_int, + deterministic_: c_int, + ); + pub fn atg_miopen_convolution_add_relu( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + z_: *mut C_tensor, + alpha_: *mut C_scalar, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + ); + pub fn atg_miopen_convolution_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + benchmark_: c_int, + deterministic_: c_int, + ); + pub fn atg_miopen_convolution_relu( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + ); + pub fn atg_miopen_convolution_transpose( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + output_padding_data: *const i64, + output_padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + benchmark_: c_int, + deterministic_: c_int, + ); + pub fn atg_miopen_convolution_transpose_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + output_padding_data: *const i64, + output_padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + benchmark_: c_int, + deterministic_: c_int, + ); + pub fn atg_miopen_ctc_loss( + out__: *mut *mut C_tensor, + log_probs_: *mut C_tensor, + targets_: *mut C_tensor, + input_lengths_data: *const i64, + input_lengths_len: c_int, + target_lengths_data: *const i64, + target_lengths_len: c_int, + blank_: i64, + deterministic_: c_int, + zero_infinity_: c_int, + ); + pub fn atg_miopen_ctc_loss_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + log_probs_: *mut C_tensor, + targets_: *mut C_tensor, + input_lengths_data: *const i64, + input_lengths_len: c_int, + target_lengths_data: *const i64, + target_lengths_len: c_int, + blank_: i64, + deterministic_: c_int, + zero_infinity_: c_int, + ); + pub fn atg_miopen_ctc_loss_tensor( + out__: *mut *mut C_tensor, + log_probs_: *mut C_tensor, + targets_: *mut C_tensor, + input_lengths_: *mut C_tensor, + target_lengths_: *mut C_tensor, + blank_: i64, + deterministic_: c_int, + zero_infinity_: c_int, + ); + pub fn atg_miopen_depthwise_convolution( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + benchmark_: c_int, + deterministic_: c_int, + ); + pub fn atg_miopen_depthwise_convolution_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + benchmark_: c_int, + deterministic_: c_int, + ); + pub fn atg_miopen_rnn( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_data: *const *mut C_tensor, + weight_len: c_int, + weight_stride0_: i64, + hx_: *mut C_tensor, + cx_: *mut C_tensor, + mode_: i64, + hidden_size_: i64, + num_layers_: i64, + batch_first_: c_int, + dropout_: f64, + train_: c_int, + bidirectional_: c_int, + batch_sizes_data: *const i64, + batch_sizes_len: c_int, + dropout_state_: *mut C_tensor, + ); + pub fn atg_miopen_rnn_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + out2_: *mut C_tensor, + out3_: *mut C_tensor, + out4_: *mut C_tensor, + input_: *mut C_tensor, + weight_data: *const *mut C_tensor, + weight_len: c_int, + weight_stride0_: i64, + hx_: *mut C_tensor, + cx_: *mut C_tensor, + mode_: i64, + hidden_size_: i64, + num_layers_: i64, + batch_first_: c_int, + dropout_: f64, + train_: c_int, + bidirectional_: c_int, + batch_sizes_data: *const i64, + batch_sizes_len: c_int, + dropout_state_: *mut C_tensor, + ); + pub fn atg_mish(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_mish_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_mish_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_mish_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_mkldnn_adaptive_avg_pool2d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + ); + pub fn atg_mkldnn_adaptive_avg_pool2d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_mkldnn_adaptive_avg_pool2d_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_mkldnn_adaptive_avg_pool2d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + ); + pub fn atg_mkldnn_convolution( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + ); + pub fn atg_mkldnn_convolution_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + ); + pub fn atg_mkldnn_linear( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + ); + pub fn atg_mkldnn_linear_backward_input( + out__: *mut *mut C_tensor, + input_size_data: *const i64, + input_size_len: c_int, + grad_output_: *mut C_tensor, + weight_: *mut C_tensor, + ); + pub fn atg_mkldnn_linear_backward_input_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + input_size_data: *const i64, + input_size_len: c_int, + grad_output_: *mut C_tensor, + weight_: *mut C_tensor, + ); + pub fn atg_mkldnn_linear_backward_weights( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_defined_: c_int, + ); + pub fn atg_mkldnn_linear_backward_weights_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + grad_output_: *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_defined_: c_int, + ); + pub fn atg_mkldnn_linear_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + ); + pub fn atg_mkldnn_max_pool2d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_mkldnn_max_pool2d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + output_: *mut C_tensor, + input_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_mkldnn_max_pool2d_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_output_: *mut C_tensor, + output_: *mut C_tensor, + input_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_mkldnn_max_pool2d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_mkldnn_max_pool3d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_mkldnn_max_pool3d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + output_: *mut C_tensor, + input_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_mkldnn_max_pool3d_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_output_: *mut C_tensor, + output_: *mut C_tensor, + input_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_mkldnn_max_pool3d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_mkldnn_reorder_conv2d_weight( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + input_size_data: *const i64, + input_size_len: c_int, + ); + pub fn atg_mkldnn_reorder_conv2d_weight_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + input_size_data: *const i64, + input_size_len: c_int, + ); + pub fn atg_mkldnn_reorder_conv3d_weight( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + input_size_data: *const i64, + input_size_len: c_int, + ); + pub fn atg_mkldnn_reorder_conv3d_weight_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + stride_data: *const i64, + stride_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + groups_: i64, + input_size_data: *const i64, + input_size_len: c_int, + ); + pub fn atg_mkldnn_rnn_layer( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight0_: *mut C_tensor, + weight1_: *mut C_tensor, + weight2_: *mut C_tensor, + weight3_: *mut C_tensor, + hx__: *mut C_tensor, + cx__: *mut C_tensor, + reverse_: c_int, + batch_sizes_data: *const i64, + batch_sizes_len: c_int, + mode_: i64, + hidden_size_: i64, + num_layers_: i64, + has_biases_: c_int, + bidirectional_: c_int, + batch_first_: c_int, + train_: c_int, + ); + pub fn atg_mkldnn_rnn_layer_backward( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight1_: *mut C_tensor, + weight2_: *mut C_tensor, + weight3_: *mut C_tensor, + weight4_: *mut C_tensor, + hx__: *mut C_tensor, + cx_tmp_: *mut C_tensor, + output_: *mut C_tensor, + hy__: *mut C_tensor, + cy__: *mut C_tensor, + grad_output_: *mut C_tensor, + grad_hy_: *mut C_tensor, + grad_cy_: *mut C_tensor, + reverse_: c_int, + mode_: i64, + hidden_size_: i64, + num_layers_: i64, + has_biases_: c_int, + train_: c_int, + bidirectional_: c_int, + batch_sizes_data: *const i64, + batch_sizes_len: c_int, + batch_first_: c_int, + workspace_: *mut C_tensor, + ); + pub fn atg_mkldnn_rnn_layer_backward_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + out2_: *mut C_tensor, + out3_: *mut C_tensor, + out4_: *mut C_tensor, + out5_: *mut C_tensor, + out6_: *mut C_tensor, + input_: *mut C_tensor, + weight1_: *mut C_tensor, + weight2_: *mut C_tensor, + weight3_: *mut C_tensor, + weight4_: *mut C_tensor, + hx__: *mut C_tensor, + cx_tmp_: *mut C_tensor, + output_: *mut C_tensor, + hy__: *mut C_tensor, + cy__: *mut C_tensor, + grad_output_: *mut C_tensor, + grad_hy_: *mut C_tensor, + grad_cy_: *mut C_tensor, + reverse_: c_int, + mode_: i64, + hidden_size_: i64, + num_layers_: i64, + has_biases_: c_int, + train_: c_int, + bidirectional_: c_int, + batch_sizes_data: *const i64, + batch_sizes_len: c_int, + batch_first_: c_int, + workspace_: *mut C_tensor, + ); + pub fn atg_mkldnn_rnn_layer_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + out2_: *mut C_tensor, + out3_: *mut C_tensor, + input_: *mut C_tensor, + weight0_: *mut C_tensor, + weight1_: *mut C_tensor, + weight2_: *mut C_tensor, + weight3_: *mut C_tensor, + hx__: *mut C_tensor, + cx__: *mut C_tensor, + reverse_: c_int, + batch_sizes_data: *const i64, + batch_sizes_len: c_int, + mode_: i64, + hidden_size_: i64, + num_layers_: i64, + has_biases_: c_int, + bidirectional_: c_int, + batch_first_: c_int, + train_: c_int, + ); + pub fn atg_mm(out__: *mut *mut C_tensor, self_: *mut C_tensor, mat2_: *mut C_tensor); + pub fn atg_mm_dtype( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mat2_: *mut C_tensor, + out_dtype_: c_int, + ); + pub fn atg_mm_dtype_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mat2_: *mut C_tensor, + out_dtype_: c_int, + ); + pub fn atg_mm_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mat2_: *mut C_tensor, + ); + pub fn atg_mode(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, keepdim_: c_int); + pub fn atg_mode_values( + out__: *mut *mut C_tensor, + values_: *mut C_tensor, + indices_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + keepdim_: c_int, + ); + pub fn atg_moveaxis( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + source_data: *const i64, + source_len: c_int, + destination_data: *const i64, + destination_len: c_int, + ); + pub fn atg_moveaxis_int( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + source_: i64, + destination_: i64, + ); + pub fn atg_movedim( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + source_data: *const i64, + source_len: c_int, + destination_data: *const i64, + destination_len: c_int, + ); + pub fn atg_movedim_int( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + source_: i64, + destination_: i64, + ); + pub fn atg_mse_loss( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + reduction_: i64, + ); + pub fn atg_mse_loss_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + reduction_: i64, + ); + pub fn atg_mse_loss_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + reduction_: i64, + ); + pub fn atg_mse_loss_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + reduction_: i64, + ); + pub fn atg_msort(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_msort_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_mt(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_mul(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_mul_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_mul_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_mul_scalar(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_mul_scalar_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_mul_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_multi_margin_loss_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + p_: *mut C_scalar, + margin_: *mut C_scalar, + weight_: *mut C_tensor, + reduction_: i64, + ); + pub fn atg_multi_margin_loss_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + p_: *mut C_scalar, + margin_: *mut C_scalar, + weight_: *mut C_tensor, + reduction_: i64, + ); + pub fn atg_multilabel_margin_loss( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + reduction_: i64, + ); + pub fn atg_multilabel_margin_loss_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + reduction_: i64, + is_target_: *mut C_tensor, + ); + pub fn atg_multilabel_margin_loss_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + reduction_: i64, + is_target_: *mut C_tensor, + ); + pub fn atg_multilabel_margin_loss_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + reduction_: i64, + ); + pub fn atg_multinomial( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + num_samples_: i64, + replacement_: c_int, + ); + pub fn atg_multinomial_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + num_samples_: i64, + replacement_: c_int, + ); + pub fn atg_multiply(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_multiply_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_multiply_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_multiply_scalar( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_multiply_scalar_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_mv(out__: *mut *mut C_tensor, self_: *mut C_tensor, vec_: *mut C_tensor); + pub fn atg_mv_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + vec_: *mut C_tensor, + ); + pub fn atg_mvlgamma(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: i64); + pub fn atg_mvlgamma_(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: i64); + pub fn atg_mvlgamma_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + p_: i64, + ); + pub fn atg_nan_to_num( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + nan_v: f64, + nan_null: i8, + posinf_v: f64, + posinf_null: i8, + neginf_v: f64, + neginf_null: i8, + ); + pub fn atg_nan_to_num_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + nan_v: f64, + nan_null: i8, + posinf_v: f64, + posinf_null: i8, + neginf_v: f64, + neginf_null: i8, + ); + pub fn atg_nan_to_num_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + nan_v: f64, + nan_null: i8, + posinf_v: f64, + posinf_null: i8, + neginf_v: f64, + neginf_null: i8, + ); + pub fn atg_nanmean( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + dtype_: c_int, + ); + pub fn atg_nanmean_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + dtype_: c_int, + ); + pub fn atg_nanmedian(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_nanmedian_dim( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + keepdim_: c_int, + ); + pub fn atg_nanmedian_dim_values( + out__: *mut *mut C_tensor, + values_: *mut C_tensor, + indices_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + keepdim_: c_int, + ); + pub fn atg_nanmedian_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_nanquantile( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + q_: *mut C_tensor, + dim_v: i64, + dim_null: i8, + keepdim_: c_int, + interpolation_ptr: *const u8, + interpolation_len: c_int, + ); + pub fn atg_nanquantile_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + q_: *mut C_tensor, + dim_v: i64, + dim_null: i8, + keepdim_: c_int, + interpolation_ptr: *const u8, + interpolation_len: c_int, + ); + pub fn atg_nanquantile_scalar( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + q_: f64, + dim_v: i64, + dim_null: i8, + keepdim_: c_int, + interpolation_ptr: *const u8, + interpolation_len: c_int, + ); + pub fn atg_nanquantile_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + q_: f64, + dim_v: i64, + dim_null: i8, + keepdim_: c_int, + interpolation_ptr: *const u8, + interpolation_len: c_int, + ); + pub fn atg_nansum( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + dtype_: c_int, + ); + pub fn atg_nansum_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + dtype_: c_int, + ); + pub fn atg_narrow( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + start_: i64, + length_: i64, + ); + pub fn atg_narrow_copy( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + start_: i64, + length_: i64, + ); + pub fn atg_narrow_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + start_: i64, + length_: i64, + ); + pub fn atg_narrow_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + start_: *mut C_tensor, + length_: i64, + ); + pub fn atg_native_batch_norm( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + training_: c_int, + momentum_: f64, + eps_: f64, + ); + pub fn atg_native_batch_norm_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + save_mean_: *mut C_tensor, + save_invstd_: *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + running_mean_: *mut C_tensor, + running_var_: *mut C_tensor, + training_: c_int, + momentum_: f64, + eps_: f64, + ); + pub fn atg_native_channel_shuffle( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + groups_: i64, + ); + pub fn atg_native_dropout( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + p_: f64, + train_: c_int, + ); + pub fn atg_native_dropout_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + mask_: *mut C_tensor, + scale_: f64, + ); + pub fn atg_native_dropout_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_output_: *mut C_tensor, + mask_: *mut C_tensor, + scale_: f64, + ); + pub fn atg_native_dropout_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + input_: *mut C_tensor, + p_: f64, + train_: c_int, + ); + pub fn atg_native_group_norm( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + n_: i64, + C_: i64, + HxW_: i64, + group_: i64, + eps_: f64, + ); + pub fn atg_native_group_norm_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + out2_: *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + n_: i64, + C_: i64, + HxW_: i64, + group_: i64, + eps_: f64, + ); + pub fn atg_native_layer_norm( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + normalized_shape_data: *const i64, + normalized_shape_len: c_int, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + eps_: f64, + ); + pub fn atg_native_layer_norm_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + out2_: *mut C_tensor, + input_: *mut C_tensor, + normalized_shape_data: *const i64, + normalized_shape_len: c_int, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + eps_: f64, + ); + pub fn atg_native_norm(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_native_norm_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_native_norm_scalaropt_dim_dtype( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + p_: *mut C_scalar, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + dtype_: c_int, + ); + pub fn atg_native_norm_scalaropt_dim_dtype_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + p_: *mut C_scalar, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + dtype_: c_int, + ); + pub fn atg_ne(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_ne_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_ne_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_ne_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_ne_tensor_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_ne_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_neg(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_neg_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_neg_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_negative(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_negative_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_negative_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_nested_to_padded_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + padding_: f64, + output_size_data: *const i64, + output_size_len: c_int, + ); + pub fn atg_new_empty( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_new_empty_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg_new_empty_strided( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_new_empty_strided_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + ); + pub fn atg_new_full( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + fill_value_: *mut C_scalar, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_new_full_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + fill_value_: *mut C_scalar, + ); + pub fn atg_new_ones( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_new_ones_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg_new_zeros( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_new_zeros_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg_nextafter(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_nextafter_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_nextafter_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_nll_loss( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + weight_: *mut C_tensor, + reduction_: i64, + ignore_index_: i64, + ); + pub fn atg_nll_loss2d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + weight_: *mut C_tensor, + reduction_: i64, + ignore_index_: i64, + ); + pub fn atg_nll_loss2d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + weight_: *mut C_tensor, + reduction_: i64, + ignore_index_: i64, + total_weight_: *mut C_tensor, + ); + pub fn atg_nll_loss2d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + weight_: *mut C_tensor, + reduction_: i64, + ignore_index_: i64, + total_weight_: *mut C_tensor, + ); + pub fn atg_nll_loss2d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + weight_: *mut C_tensor, + reduction_: i64, + ignore_index_: i64, + ); + pub fn atg_nll_loss_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + weight_: *mut C_tensor, + reduction_: i64, + ignore_index_: i64, + total_weight_: *mut C_tensor, + ); + pub fn atg_nll_loss_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + weight_: *mut C_tensor, + reduction_: i64, + ignore_index_: i64, + total_weight_: *mut C_tensor, + ); + pub fn atg_nll_loss_nd( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + weight_: *mut C_tensor, + reduction_: i64, + ignore_index_: i64, + ); + pub fn atg_nll_loss_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + weight_: *mut C_tensor, + reduction_: i64, + ignore_index_: i64, + ); + pub fn atg_nonzero(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_nonzero_numpy(self_: *mut C_tensor) -> *mut *mut C_tensor; + pub fn atg_nonzero_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_nonzero_static( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_: i64, + fill_value_: i64, + ); + pub fn atg_nonzero_static_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + size_: i64, + fill_value_: i64, + ); + pub fn atg_norm(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_norm_dtype_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + p_: *mut C_scalar, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + dtype_: c_int, + ); + pub fn atg_norm_except_dim(out__: *mut *mut C_tensor, v_: *mut C_tensor, pow_: i64, dim_: i64); + pub fn atg_norm_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + p_: *mut C_scalar, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + ); + pub fn atg_norm_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_norm_scalaropt_dim( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + p_: *mut C_scalar, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + ); + pub fn atg_norm_scalaropt_dim_dtype( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + p_: *mut C_scalar, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + dtype_: c_int, + ); + pub fn atg_norm_scalaropt_dtype( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + p_: *mut C_scalar, + dtype_: c_int, + ); + pub fn atg_norm_scalaropt_dtype_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + p_: *mut C_scalar, + dtype_: c_int, + ); + pub fn atg_normal_(out__: *mut *mut C_tensor, self_: *mut C_tensor, mean_: f64, std_: f64); + pub fn atg_normal_functional( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mean_: f64, + std_: f64, + ); + pub fn atg_not_equal(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_not_equal_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_not_equal_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_not_equal_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_not_equal_tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_not_equal_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_nuclear_norm(out__: *mut *mut C_tensor, self_: *mut C_tensor, keepdim_: c_int); + pub fn atg_nuclear_norm_dim( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + ); + pub fn atg_nuclear_norm_dim_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + ); + pub fn atg_nuclear_norm_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + keepdim_: c_int, + ); + pub fn atg_numpy_t(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_one_hot(out__: *mut *mut C_tensor, self_: *mut C_tensor, num_classes_: i64); + pub fn atg_ones( + out__: *mut *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_ones_like(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_ones_like_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_ones_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg_orgqr(out__: *mut *mut C_tensor, self_: *mut C_tensor, input2_: *mut C_tensor); + pub fn atg_orgqr_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + input2_: *mut C_tensor, + ); + pub fn atg_ormqr( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + input2_: *mut C_tensor, + input3_: *mut C_tensor, + left_: c_int, + transpose_: c_int, + ); + pub fn atg_ormqr_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + input2_: *mut C_tensor, + input3_: *mut C_tensor, + left_: c_int, + transpose_: c_int, + ); + pub fn atg_outer(out__: *mut *mut C_tensor, self_: *mut C_tensor, vec2_: *mut C_tensor); + pub fn atg_outer_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + vec2_: *mut C_tensor, + ); + pub fn atg_output_nr(self_: *mut C_tensor) -> i64; + pub fn atg_pad( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + pad_data: *const i64, + pad_len: c_int, + mode_ptr: *const u8, + mode_len: c_int, + value_v: f64, + value_null: i8, + ); + pub fn atg_pad_sequence( + out__: *mut *mut C_tensor, + sequences_data: *const *mut C_tensor, + sequences_len: c_int, + batch_first_: c_int, + padding_value_: f64, + padding_side_ptr: *const u8, + padding_side_len: c_int, + ); + pub fn atg_pairwise_distance( + out__: *mut *mut C_tensor, + x1_: *mut C_tensor, + x2_: *mut C_tensor, + p_: f64, + eps_: f64, + keepdim_: c_int, + ); + pub fn atg_pdist(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: f64); + pub fn atg_permute( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dims_data: *const i64, + dims_len: c_int, + ); + pub fn atg_permute_copy( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dims_data: *const i64, + dims_len: c_int, + ); + pub fn atg_permute_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dims_data: *const i64, + dims_len: c_int, + ); + pub fn atg_pin_memory(out__: *mut *mut C_tensor, self_: *mut C_tensor, device_: c_int); + pub fn atg_pinverse(out__: *mut *mut C_tensor, self_: *mut C_tensor, rcond_: f64); + pub fn atg_pixel_shuffle(out__: *mut *mut C_tensor, self_: *mut C_tensor, upscale_factor_: i64); + pub fn atg_pixel_shuffle_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + upscale_factor_: i64, + ); + pub fn atg_pixel_unshuffle( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + downscale_factor_: i64, + ); + pub fn atg_pixel_unshuffle_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + downscale_factor_: i64, + ); + pub fn atg_poisson(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_poisson_nll_loss( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + target_: *mut C_tensor, + log_input_: c_int, + full_: c_int, + eps_: f64, + reduction_: i64, + ); + pub fn atg_poisson_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_polar(out__: *mut *mut C_tensor, abs_: *mut C_tensor, angle_: *mut C_tensor); + pub fn atg_polar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + abs_: *mut C_tensor, + angle_: *mut C_tensor, + ); + pub fn atg_polygamma(out__: *mut *mut C_tensor, n_: i64, self_: *mut C_tensor); + pub fn atg_polygamma_(out__: *mut *mut C_tensor, self_: *mut C_tensor, n_: i64); + pub fn atg_polygamma_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + n_: i64, + self_: *mut C_tensor, + ); + pub fn atg_positive(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_pow(out__: *mut *mut C_tensor, self_: *mut C_tensor, exponent_: *mut C_tensor); + pub fn atg_pow_(out__: *mut *mut C_tensor, self_: *mut C_tensor, exponent_: *mut C_scalar); + pub fn atg_pow_scalar( + out__: *mut *mut C_tensor, + self_scalar_: *mut C_scalar, + exponent_: *mut C_tensor, + ); + pub fn atg_pow_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_scalar_: *mut C_scalar, + exponent_: *mut C_tensor, + ); + pub fn atg_pow_tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + exponent_: *mut C_tensor, + ); + pub fn atg_pow_tensor_scalar( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + exponent_: *mut C_scalar, + ); + pub fn atg_pow_tensor_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + exponent_: *mut C_scalar, + ); + pub fn atg_pow_tensor_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + exponent_: *mut C_tensor, + ); + pub fn atg_prelu(out__: *mut *mut C_tensor, self_: *mut C_tensor, weight_: *mut C_tensor); + pub fn atg_prod(out__: *mut *mut C_tensor, self_: *mut C_tensor, dtype_: c_int); + pub fn atg_prod_dim_int( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + keepdim_: c_int, + dtype_: c_int, + ); + pub fn atg_prod_int_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + keepdim_: c_int, + dtype_: c_int, + ); + pub fn atg_prod_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dtype_: c_int, + ); + pub fn atg_put( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + index_: *mut C_tensor, + source_: *mut C_tensor, + accumulate_: c_int, + ); + pub fn atg_put_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + index_: *mut C_tensor, + source_: *mut C_tensor, + accumulate_: c_int, + ); + pub fn atg_put_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + index_: *mut C_tensor, + source_: *mut C_tensor, + accumulate_: c_int, + ); + pub fn atg_q_per_channel_axis(self_: *mut C_tensor) -> i64; + pub fn atg_q_per_channel_scales(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_q_per_channel_scales_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_q_per_channel_zero_points(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_q_per_channel_zero_points_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_q_scale(self_: *mut C_tensor) -> f64; + pub fn atg_q_zero_point(self_: *mut C_tensor) -> i64; + pub fn atg_qr(out__: *mut *mut C_tensor, self_: *mut C_tensor, some_: c_int); + pub fn atg_qr_q( + out__: *mut *mut C_tensor, + Q_: *mut C_tensor, + R_: *mut C_tensor, + self_: *mut C_tensor, + some_: c_int, + ); + pub fn atg_quantile( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + q_: *mut C_tensor, + dim_v: i64, + dim_null: i8, + keepdim_: c_int, + interpolation_ptr: *const u8, + interpolation_len: c_int, + ); + pub fn atg_quantile_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + q_: *mut C_tensor, + dim_v: i64, + dim_null: i8, + keepdim_: c_int, + interpolation_ptr: *const u8, + interpolation_len: c_int, + ); + pub fn atg_quantile_scalar( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + q_: f64, + dim_v: i64, + dim_null: i8, + keepdim_: c_int, + interpolation_ptr: *const u8, + interpolation_len: c_int, + ); + pub fn atg_quantile_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + q_: f64, + dim_v: i64, + dim_null: i8, + keepdim_: c_int, + interpolation_ptr: *const u8, + interpolation_len: c_int, + ); + pub fn atg_quantize_per_channel( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + scales_: *mut C_tensor, + zero_points_: *mut C_tensor, + axis_: i64, + dtype_: c_int, + ); + pub fn atg_quantize_per_channel_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + scales_: *mut C_tensor, + zero_points_: *mut C_tensor, + axis_: i64, + dtype_: c_int, + ); + pub fn atg_quantize_per_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + scale_: f64, + zero_point_: i64, + dtype_: c_int, + ); + pub fn atg_quantize_per_tensor_dynamic( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dtype_: c_int, + reduce_range_: c_int, + ); + pub fn atg_quantize_per_tensor_dynamic_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dtype_: c_int, + reduce_range_: c_int, + ); + pub fn atg_quantize_per_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + scale_: f64, + zero_point_: i64, + dtype_: c_int, + ); + pub fn atg_quantize_per_tensor_tensor_qparams( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + scale_: *mut C_tensor, + zero_point_: *mut C_tensor, + dtype_: c_int, + ); + pub fn atg_quantize_per_tensor_tensor_qparams_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + scale_: *mut C_tensor, + zero_point_: *mut C_tensor, + dtype_: c_int, + ); + pub fn atg_quantize_per_tensor_tensors( + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + scales_: *mut C_tensor, + zero_points_: *mut C_tensor, + dtype_: c_int, + ) -> *mut *mut C_tensor; + pub fn atg_quantize_per_tensor_tensors_out( + out_data: *const *mut C_tensor, + out_len: c_int, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + scales_: *mut C_tensor, + zero_points_: *mut C_tensor, + dtype_: c_int, + ); + pub fn atg_quantized_batch_norm( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + mean_: *mut C_tensor, + var_: *mut C_tensor, + eps_: f64, + output_scale_: f64, + output_zero_point_: i64, + ); + pub fn atg_quantized_batch_norm_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + input_: *mut C_tensor, + weight_: *mut C_tensor, + bias_: *mut C_tensor, + mean_: *mut C_tensor, + var_: *mut C_tensor, + eps_: f64, + output_scale_: f64, + output_zero_point_: i64, + ); + pub fn atg_quantized_gru_cell( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + hx_: *mut C_tensor, + w_ih_: *mut C_tensor, + w_hh_: *mut C_tensor, + b_ih_: *mut C_tensor, + b_hh_: *mut C_tensor, + packed_ih_: *mut C_tensor, + packed_hh_: *mut C_tensor, + col_offsets_ih_: *mut C_tensor, + col_offsets_hh_: *mut C_tensor, + scale_ih_: *mut C_scalar, + scale_hh_: *mut C_scalar, + zero_point_ih_: *mut C_scalar, + zero_point_hh_: *mut C_scalar, + ); + pub fn atg_quantized_lstm_cell( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + hx_data: *const *mut C_tensor, + hx_len: c_int, + w_ih_: *mut C_tensor, + w_hh_: *mut C_tensor, + b_ih_: *mut C_tensor, + b_hh_: *mut C_tensor, + packed_ih_: *mut C_tensor, + packed_hh_: *mut C_tensor, + col_offsets_ih_: *mut C_tensor, + col_offsets_hh_: *mut C_tensor, + scale_ih_: *mut C_scalar, + scale_hh_: *mut C_scalar, + zero_point_ih_: *mut C_scalar, + zero_point_hh_: *mut C_scalar, + ); + pub fn atg_quantized_max_pool1d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_quantized_max_pool1d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_quantized_max_pool2d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_quantized_max_pool2d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_quantized_max_pool3d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_quantized_max_pool3d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ceil_mode_: c_int, + ); + pub fn atg_quantized_rnn_relu_cell( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + hx_: *mut C_tensor, + w_ih_: *mut C_tensor, + w_hh_: *mut C_tensor, + b_ih_: *mut C_tensor, + b_hh_: *mut C_tensor, + packed_ih_: *mut C_tensor, + packed_hh_: *mut C_tensor, + col_offsets_ih_: *mut C_tensor, + col_offsets_hh_: *mut C_tensor, + scale_ih_: *mut C_scalar, + scale_hh_: *mut C_scalar, + zero_point_ih_: *mut C_scalar, + zero_point_hh_: *mut C_scalar, + ); + pub fn atg_quantized_rnn_tanh_cell( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + hx_: *mut C_tensor, + w_ih_: *mut C_tensor, + w_hh_: *mut C_tensor, + b_ih_: *mut C_tensor, + b_hh_: *mut C_tensor, + packed_ih_: *mut C_tensor, + packed_hh_: *mut C_tensor, + col_offsets_ih_: *mut C_tensor, + col_offsets_hh_: *mut C_tensor, + scale_ih_: *mut C_scalar, + scale_hh_: *mut C_scalar, + zero_point_ih_: *mut C_scalar, + zero_point_hh_: *mut C_scalar, + ); + pub fn atg_rad2deg(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_rad2deg_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_rad2deg_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_rand( + out__: *mut *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_rand_like(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_rand_like_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_rand_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg_randint( + out__: *mut *mut C_tensor, + high_: i64, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_randint_like(out__: *mut *mut C_tensor, self_: *mut C_tensor, high_: i64); + pub fn atg_randint_like_low_dtype( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + low_: i64, + high_: i64, + ); + pub fn atg_randint_like_low_dtype_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + low_: i64, + high_: i64, + ); + pub fn atg_randint_like_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + high_: i64, + ); + pub fn atg_randint_like_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + high_: *mut C_tensor, + ); + pub fn atg_randint_like_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + high_: *mut C_tensor, + ); + pub fn atg_randint_low( + out__: *mut *mut C_tensor, + low_: i64, + high_: i64, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_randint_low_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + low_: i64, + high_: i64, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg_randint_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + high_: i64, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg_randn( + out__: *mut *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_randn_like(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_randn_like_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_randn_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg_random(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_random_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_random_from( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + from_: i64, + to_v: i64, + to_null: i8, + ); + pub fn atg_random_from_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + from_: i64, + to_v: i64, + to_null: i8, + ); + pub fn atg_random_from_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + from_: i64, + to_v: i64, + to_null: i8, + ); + pub fn atg_random_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_random_to(out__: *mut *mut C_tensor, self_: *mut C_tensor, to_: i64); + pub fn atg_random_to_(out__: *mut *mut C_tensor, self_: *mut C_tensor, to_: i64); + pub fn atg_random_to_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + to_: i64, + ); + pub fn atg_randperm( + out__: *mut *mut C_tensor, + n_: i64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_randperm_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, n_: i64); + pub fn atg_range( + out__: *mut *mut C_tensor, + start_: *mut C_scalar, + end_: *mut C_scalar, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_range_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + start_: *mut C_scalar, + end_: *mut C_scalar, + ); + pub fn atg_range_out_( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + start_: *mut C_scalar, + end_: *mut C_scalar, + ); + pub fn atg_range_step( + out__: *mut *mut C_tensor, + start_: *mut C_scalar, + end_: *mut C_scalar, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_ravel(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_real(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_reciprocal(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_reciprocal_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_reciprocal_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_reflection_pad1d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_reflection_pad1d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_reflection_pad1d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_reflection_pad1d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_reflection_pad2d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_reflection_pad2d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_reflection_pad2d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_reflection_pad2d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_reflection_pad3d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_reflection_pad3d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_reflection_pad3d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_reflection_pad3d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_relu(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_relu6(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_relu6_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_relu_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_relu_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_remainder(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_remainder_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_remainder_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_remainder_scalar_tensor( + out__: *mut *mut C_tensor, + self_scalar_: *mut C_scalar, + other_: *mut C_tensor, + ); + pub fn atg_remainder_scalar_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_scalar_: *mut C_scalar, + other_: *mut C_tensor, + ); + pub fn atg_remainder_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_remainder_tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_remainder_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_renorm( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + p_: *mut C_scalar, + dim_: i64, + maxnorm_: *mut C_scalar, + ); + pub fn atg_renorm_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + p_: *mut C_scalar, + dim_: i64, + maxnorm_: *mut C_scalar, + ); + pub fn atg_renorm_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + p_: *mut C_scalar, + dim_: i64, + maxnorm_: *mut C_scalar, + ); + pub fn atg_repeat( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + repeats_data: *const i64, + repeats_len: c_int, + ); + pub fn atg_repeat_interleave( + out__: *mut *mut C_tensor, + repeats_: *mut C_tensor, + output_size_v: i64, + output_size_null: i8, + ); + pub fn atg_repeat_interleave_self_int( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + repeats_: i64, + dim_v: i64, + dim_null: i8, + output_size_v: i64, + output_size_null: i8, + ); + pub fn atg_repeat_interleave_self_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + repeats_: *mut C_tensor, + dim_v: i64, + dim_null: i8, + output_size_v: i64, + output_size_null: i8, + ); + pub fn atg_repeat_interleave_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + repeats_: *mut C_tensor, + output_size_v: i64, + output_size_null: i8, + ); + pub fn atg_repeat_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + repeats_data: *const i64, + repeats_len: c_int, + ); + pub fn atg_replication_pad1d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_replication_pad1d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_replication_pad1d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_replication_pad1d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_replication_pad2d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_replication_pad2d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_replication_pad2d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_replication_pad2d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_replication_pad3d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_replication_pad3d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_replication_pad3d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_replication_pad3d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_requires_grad_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + requires_grad_: c_int, + ); + pub fn atg_reshape( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + shape_data: *const i64, + shape_len: c_int, + ); + pub fn atg_reshape_as(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_resize( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg_resize_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg_resize_as( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + the_template_: *mut C_tensor, + ); + pub fn atg_resize_as_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + the_template_: *mut C_tensor, + ); + pub fn atg_resize_as_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + the_template_: *mut C_tensor, + ); + pub fn atg_resize_as_sparse( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + the_template_: *mut C_tensor, + ); + pub fn atg_resize_as_sparse_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + the_template_: *mut C_tensor, + ); + pub fn atg_resize_as_sparse_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + the_template_: *mut C_tensor, + ); + pub fn atg_resize_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg_resolve_conj(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_resolve_neg(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_retains_grad(self_: *mut C_tensor) -> c_int; + pub fn atg_rms_norm( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + normalized_shape_data: *const i64, + normalized_shape_len: c_int, + weight_: *mut C_tensor, + eps_v: f64, + eps_null: i8, + ); + pub fn atg_rnn_relu( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + hx_: *mut C_tensor, + params_data: *const *mut C_tensor, + params_len: c_int, + has_biases_: c_int, + num_layers_: i64, + dropout_: f64, + train_: c_int, + bidirectional_: c_int, + batch_first_: c_int, + ); + pub fn atg_rnn_relu_cell( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + hx_: *mut C_tensor, + w_ih_: *mut C_tensor, + w_hh_: *mut C_tensor, + b_ih_: *mut C_tensor, + b_hh_: *mut C_tensor, + ); + pub fn atg_rnn_relu_data( + out__: *mut *mut C_tensor, + data_: *mut C_tensor, + batch_sizes_: *mut C_tensor, + hx_: *mut C_tensor, + params_data: *const *mut C_tensor, + params_len: c_int, + has_biases_: c_int, + num_layers_: i64, + dropout_: f64, + train_: c_int, + bidirectional_: c_int, + ); + pub fn atg_rnn_tanh( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + hx_: *mut C_tensor, + params_data: *const *mut C_tensor, + params_len: c_int, + has_biases_: c_int, + num_layers_: i64, + dropout_: f64, + train_: c_int, + bidirectional_: c_int, + batch_first_: c_int, + ); + pub fn atg_rnn_tanh_cell( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + hx_: *mut C_tensor, + w_ih_: *mut C_tensor, + w_hh_: *mut C_tensor, + b_ih_: *mut C_tensor, + b_hh_: *mut C_tensor, + ); + pub fn atg_rnn_tanh_data( + out__: *mut *mut C_tensor, + data_: *mut C_tensor, + batch_sizes_: *mut C_tensor, + hx_: *mut C_tensor, + params_data: *const *mut C_tensor, + params_len: c_int, + has_biases_: c_int, + num_layers_: i64, + dropout_: f64, + train_: c_int, + bidirectional_: c_int, + ); + pub fn atg_roll( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + shifts_data: *const i64, + shifts_len: c_int, + dims_data: *const i64, + dims_len: c_int, + ); + pub fn atg_roll_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + shifts_data: *const i64, + shifts_len: c_int, + dims_data: *const i64, + dims_len: c_int, + ); + pub fn atg_rot90( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + k_: i64, + dims_data: *const i64, + dims_len: c_int, + ); + pub fn atg_rot90_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + k_: i64, + dims_data: *const i64, + dims_len: c_int, + ); + pub fn atg_round(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_round_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_round_decimals(out__: *mut *mut C_tensor, self_: *mut C_tensor, decimals_: i64); + pub fn atg_round_decimals_(out__: *mut *mut C_tensor, self_: *mut C_tensor, decimals_: i64); + pub fn atg_round_decimals_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + decimals_: i64, + ); + pub fn atg_round_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_row_indices(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_row_indices_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_row_indices_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_row_stack( + out__: *mut *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ); + pub fn atg_row_stack_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ); + pub fn atg_rrelu(out__: *mut *mut C_tensor, self_: *mut C_tensor, training_: c_int); + pub fn atg_rrelu_(out__: *mut *mut C_tensor, self_: *mut C_tensor, training_: c_int); + pub fn atg_rrelu_with_noise( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + noise_: *mut C_tensor, + training_: c_int, + ); + pub fn atg_rrelu_with_noise_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + noise_: *mut C_tensor, + training_: c_int, + ); + pub fn atg_rrelu_with_noise_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + noise_: *mut C_tensor, + lower_: *mut C_scalar, + upper_: *mut C_scalar, + training_: c_int, + self_is_result_: c_int, + ); + pub fn atg_rrelu_with_noise_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + noise_: *mut C_tensor, + lower_: *mut C_scalar, + upper_: *mut C_scalar, + training_: c_int, + self_is_result_: c_int, + ); + pub fn atg_rrelu_with_noise_functional( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + noise_: *mut C_tensor, + training_: c_int, + ); + pub fn atg_rrelu_with_noise_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + noise_: *mut C_tensor, + training_: c_int, + ); + pub fn atg_rsqrt(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_rsqrt_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_rsqrt_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_rsub(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_rsub_scalar(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_rsub_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_rsub_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_scalar_tensor( + out__: *mut *mut C_tensor, + s_: *mut C_scalar, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_scalar_tensor_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, s_: *mut C_scalar); + pub fn atg_scaled_dot_product_attention( + out__: *mut *mut C_tensor, + query_: *mut C_tensor, + key_: *mut C_tensor, + value_: *mut C_tensor, + attn_mask_: *mut C_tensor, + dropout_p_: f64, + is_causal_: c_int, + scale_v: f64, + scale_null: i8, + enable_gqa_: c_int, + ); + pub fn atg_scatter( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + src_: *mut C_tensor, + ); + pub fn atg_scatter_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + src_: *mut C_tensor, + ); + pub fn atg_scatter_add( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + src_: *mut C_tensor, + ); + pub fn atg_scatter_add_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + src_: *mut C_tensor, + ); + pub fn atg_scatter_add_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + src_: *mut C_tensor, + ); + pub fn atg_scatter_reduce( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + src_: *mut C_tensor, + reduce_ptr: *const u8, + reduce_len: c_int, + ); + pub fn atg_scatter_reduce_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + src_: *mut C_tensor, + reduce_ptr: *const u8, + reduce_len: c_int, + ); + pub fn atg_scatter_reduce_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + src_: *mut C_tensor, + reduce_ptr: *const u8, + reduce_len: c_int, + ); + pub fn atg_scatter_src_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + src_: *mut C_tensor, + ); + pub fn atg_scatter_value( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + value_: *mut C_scalar, + ); + pub fn atg_scatter_value_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + value_: *mut C_scalar, + ); + pub fn atg_scatter_value_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + value_: *mut C_scalar, + ); + pub fn atg_scatter_value_reduce( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + value_: *mut C_scalar, + reduce_ptr: *const u8, + reduce_len: c_int, + ); + pub fn atg_scatter_value_reduce_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + value_: *mut C_scalar, + reduce_ptr: *const u8, + reduce_len: c_int, + ); + pub fn atg_scatter_value_reduce_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: *mut C_tensor, + value_: *mut C_scalar, + reduce_ptr: *const u8, + reduce_len: c_int, + ); + pub fn atg_searchsorted( + out__: *mut *mut C_tensor, + sorted_sequence_: *mut C_tensor, + self_: *mut C_tensor, + out_int32_: c_int, + right_: c_int, + side_ptr: *const u8, + side_len: c_int, + sorter_: *mut C_tensor, + ); + pub fn atg_searchsorted_scalar( + out__: *mut *mut C_tensor, + sorted_sequence_: *mut C_tensor, + self_scalar_: *mut C_scalar, + out_int32_: c_int, + right_: c_int, + side_ptr: *const u8, + side_len: c_int, + sorter_: *mut C_tensor, + ); + pub fn atg_searchsorted_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + sorted_sequence_: *mut C_tensor, + self_scalar_: *mut C_scalar, + out_int32_: c_int, + right_: c_int, + side_ptr: *const u8, + side_len: c_int, + sorter_: *mut C_tensor, + ); + pub fn atg_searchsorted_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + sorted_sequence_: *mut C_tensor, + self_: *mut C_tensor, + out_int32_: c_int, + right_: c_int, + side_ptr: *const u8, + side_len: c_int, + sorter_: *mut C_tensor, + ); + pub fn atg_segment_reduce( + out__: *mut *mut C_tensor, + data_: *mut C_tensor, + reduce_ptr: *const u8, + reduce_len: c_int, + lengths_: *mut C_tensor, + indices_: *mut C_tensor, + offsets_: *mut C_tensor, + axis_: i64, + unsafe_: c_int, + initial_: *mut C_scalar, + ); + pub fn atg_segment_reduce_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + data_: *mut C_tensor, + reduce_ptr: *const u8, + reduce_len: c_int, + lengths_: *mut C_tensor, + indices_: *mut C_tensor, + offsets_: *mut C_tensor, + axis_: i64, + unsafe_: c_int, + initial_: *mut C_scalar, + ); + pub fn atg_select(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, index_: i64); + pub fn atg_select_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + input_sizes_data: *const i64, + input_sizes_len: c_int, + dim_: i64, + index_: i64, + ); + pub fn atg_select_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_output_: *mut C_tensor, + input_sizes_data: *const i64, + input_sizes_len: c_int, + dim_: i64, + index_: i64, + ); + pub fn atg_select_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, index_: i64); + pub fn atg_select_copy_int_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + index_: i64, + ); + pub fn atg_select_scatter( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + src_: *mut C_tensor, + dim_: i64, + index_: i64, + ); + pub fn atg_select_scatter_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + src_: *mut C_tensor, + dim_: i64, + index_: i64, + ); + pub fn atg_selu(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_selu_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_set(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_set_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_set_data(self_: *mut C_tensor, new_data_: *mut C_tensor); + pub fn atg_set_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_set_requires_grad(out__: *mut *mut C_tensor, self_: *mut C_tensor, r_: c_int); + pub fn atg_set_source_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + source_: *mut C_tensor, + ); + pub fn atg_set_source_tensor_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + source_: *mut C_tensor, + ); + pub fn atg_set_source_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + source_: *mut C_tensor, + ); + pub fn atg_set_source_tensor_storage_offset_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + source_: *mut C_tensor, + storage_offset_: i64, + size_data: *const i64, + size_len: c_int, + stride_data: *const i64, + stride_len: c_int, + ); + pub fn atg_sgn(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_sgn_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_sgn_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_sigmoid(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_sigmoid_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_sigmoid_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + output_: *mut C_tensor, + ); + pub fn atg_sigmoid_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + output_: *mut C_tensor, + ); + pub fn atg_sigmoid_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_sign(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_sign_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_sign_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_signbit(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_signbit_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_silu(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_silu_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_silu_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_silu_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_silu_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_sin(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_sin_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_sin_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_sinc(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_sinc_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_sinc_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_sinh(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_sinh_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_sinh_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_slice( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + start_v: i64, + start_null: i8, + end_v: i64, + end_null: i8, + step_: i64, + ); + pub fn atg_slice_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + input_sizes_data: *const i64, + input_sizes_len: c_int, + dim_: i64, + start_: i64, + end_: i64, + step_: i64, + ); + pub fn atg_slice_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_output_: *mut C_tensor, + input_sizes_data: *const i64, + input_sizes_len: c_int, + dim_: i64, + start_: i64, + end_: i64, + step_: i64, + ); + pub fn atg_slice_copy( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + start_v: i64, + start_null: i8, + end_v: i64, + end_null: i8, + step_: i64, + ); + pub fn atg_slice_copy_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + start_v: i64, + start_null: i8, + end_v: i64, + end_null: i8, + step_: i64, + ); + pub fn atg_slice_inverse( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + src_: *mut C_tensor, + dim_: i64, + start_v: i64, + start_null: i8, + end_v: i64, + end_null: i8, + step_: i64, + ); + pub fn atg_slice_scatter( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + src_: *mut C_tensor, + dim_: i64, + start_v: i64, + start_null: i8, + end_v: i64, + end_null: i8, + step_: i64, + ); + pub fn atg_slice_scatter_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + src_: *mut C_tensor, + dim_: i64, + start_v: i64, + start_null: i8, + end_v: i64, + end_null: i8, + step_: i64, + ); + pub fn atg_slogdet(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_slogdet_out( + out__: *mut *mut C_tensor, + sign_: *mut C_tensor, + logabsdet_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_slow_conv3d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_slow_conv3d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + ); + pub fn atg_slow_conv_dilated2d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ); + pub fn atg_slow_conv_dilated2d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ); + pub fn atg_slow_conv_dilated3d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ); + pub fn atg_slow_conv_dilated3d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ); + pub fn atg_slow_conv_transpose2d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + output_padding_data: *const i64, + output_padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ); + pub fn atg_slow_conv_transpose2d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + output_padding_data: *const i64, + output_padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ); + pub fn atg_slow_conv_transpose3d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + output_padding_data: *const i64, + output_padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ); + pub fn atg_slow_conv_transpose3d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + weight_: *mut C_tensor, + kernel_size_data: *const i64, + kernel_size_len: c_int, + bias_: *mut C_tensor, + stride_data: *const i64, + stride_len: c_int, + padding_data: *const i64, + padding_len: c_int, + output_padding_data: *const i64, + output_padding_len: c_int, + dilation_data: *const i64, + dilation_len: c_int, + ); + pub fn atg_smm(out__: *mut *mut C_tensor, self_: *mut C_tensor, mat2_: *mut C_tensor); + pub fn atg_smooth_l1_loss( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + reduction_: i64, + beta_: f64, + ); + pub fn atg_smooth_l1_loss_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + reduction_: i64, + beta_: f64, + ); + pub fn atg_smooth_l1_loss_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + reduction_: i64, + beta_: f64, + ); + pub fn atg_smooth_l1_loss_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + reduction_: i64, + beta_: f64, + ); + pub fn atg_soft_margin_loss( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + reduction_: i64, + ); + pub fn atg_soft_margin_loss_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + reduction_: i64, + ); + pub fn atg_soft_margin_loss_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + reduction_: i64, + ); + pub fn atg_soft_margin_loss_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + target_: *mut C_tensor, + reduction_: i64, + ); + pub fn atg_softmax(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, dtype_: c_int); + pub fn atg_softmax_int_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + dtype_: c_int, + ); + pub fn atg_softplus(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_softplus_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + beta_: *mut C_scalar, + threshold_: *mut C_scalar, + ); + pub fn atg_softplus_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + beta_: *mut C_scalar, + threshold_: *mut C_scalar, + ); + pub fn atg_softplus_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_softshrink(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_softshrink_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + lambd_: *mut C_scalar, + ); + pub fn atg_softshrink_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + lambd_: *mut C_scalar, + ); + pub fn atg_softshrink_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_sort(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, descending_: c_int); + pub fn atg_sort_stable( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + stable_: c_int, + dim_: i64, + descending_: c_int, + ); + pub fn atg_sort_values( + out__: *mut *mut C_tensor, + values_: *mut C_tensor, + indices_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + descending_: c_int, + ); + pub fn atg_sort_values_stable( + out__: *mut *mut C_tensor, + values_: *mut C_tensor, + indices_: *mut C_tensor, + self_: *mut C_tensor, + stable_: c_int, + dim_: i64, + descending_: c_int, + ); + pub fn atg_sparse_bsc_tensor( + out__: *mut *mut C_tensor, + ccol_indices_: *mut C_tensor, + row_indices_: *mut C_tensor, + values_: *mut C_tensor, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_sparse_bsc_tensor_ccol_row_value_size( + out__: *mut *mut C_tensor, + ccol_indices_: *mut C_tensor, + row_indices_: *mut C_tensor, + values_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_sparse_bsr_tensor( + out__: *mut *mut C_tensor, + crow_indices_: *mut C_tensor, + col_indices_: *mut C_tensor, + values_: *mut C_tensor, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_sparse_bsr_tensor_crow_col_value_size( + out__: *mut *mut C_tensor, + crow_indices_: *mut C_tensor, + col_indices_: *mut C_tensor, + values_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_sparse_compressed_tensor( + out__: *mut *mut C_tensor, + compressed_indices_: *mut C_tensor, + plain_indices_: *mut C_tensor, + values_: *mut C_tensor, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_sparse_compressed_tensor_comp_plain_value_size( + out__: *mut *mut C_tensor, + compressed_indices_: *mut C_tensor, + plain_indices_: *mut C_tensor, + values_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_sparse_coo_tensor( + out__: *mut *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_sparse_coo_tensor_indices( + out__: *mut *mut C_tensor, + indices_: *mut C_tensor, + values_: *mut C_tensor, + options_kind: c_int, + options_device: c_int, + is_coalesced_: c_int, + ); + pub fn atg_sparse_coo_tensor_indices_size( + out__: *mut *mut C_tensor, + indices_: *mut C_tensor, + values_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + is_coalesced_: c_int, + ); + pub fn atg_sparse_coo_tensor_size_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg_sparse_csc_tensor( + out__: *mut *mut C_tensor, + ccol_indices_: *mut C_tensor, + row_indices_: *mut C_tensor, + values_: *mut C_tensor, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_sparse_csc_tensor_ccol_row_value_size( + out__: *mut *mut C_tensor, + ccol_indices_: *mut C_tensor, + row_indices_: *mut C_tensor, + values_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_sparse_csr_tensor( + out__: *mut *mut C_tensor, + crow_indices_: *mut C_tensor, + col_indices_: *mut C_tensor, + values_: *mut C_tensor, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_sparse_csr_tensor_crow_col_value_size( + out__: *mut *mut C_tensor, + crow_indices_: *mut C_tensor, + col_indices_: *mut C_tensor, + values_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_sparse_dim(self_: *mut C_tensor) -> i64; + pub fn atg_sparse_mask(out__: *mut *mut C_tensor, self_: *mut C_tensor, mask_: *mut C_tensor); + pub fn atg_sparse_mask_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mask_: *mut C_tensor, + ); + pub fn atg_sparse_resize( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + sparse_dim_: i64, + dense_dim_: i64, + ); + pub fn atg_sparse_resize_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + sparse_dim_: i64, + dense_dim_: i64, + ); + pub fn atg_sparse_resize_and_clear( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + sparse_dim_: i64, + dense_dim_: i64, + ); + pub fn atg_sparse_resize_and_clear_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + sparse_dim_: i64, + dense_dim_: i64, + ); + pub fn atg_sparse_resize_and_clear_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + sparse_dim_: i64, + dense_dim_: i64, + ); + pub fn atg_sparse_resize_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + sparse_dim_: i64, + dense_dim_: i64, + ); + pub fn atg_sparse_sampled_addmm( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mat1_: *mut C_tensor, + mat2_: *mut C_tensor, + ); + pub fn atg_sparse_sampled_addmm_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mat1_: *mut C_tensor, + mat2_: *mut C_tensor, + ); + pub fn atg_special_airy_ai(out__: *mut *mut C_tensor, x_: *mut C_tensor); + pub fn atg_special_airy_ai_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + ); + pub fn atg_special_bessel_j0(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_bessel_j0_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_bessel_j1(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_bessel_j1_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_bessel_y0(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_bessel_y0_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_bessel_y1(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_bessel_y1_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_chebyshev_polynomial_t( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_chebyshev_polynomial_t_n_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_chebyshev_polynomial_t_n_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_chebyshev_polynomial_t_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_chebyshev_polynomial_t_x_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_chebyshev_polynomial_t_x_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_chebyshev_polynomial_u( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_chebyshev_polynomial_u_n_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_chebyshev_polynomial_u_n_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_chebyshev_polynomial_u_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_chebyshev_polynomial_u_x_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_chebyshev_polynomial_u_x_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_chebyshev_polynomial_v( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_chebyshev_polynomial_v_n_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_chebyshev_polynomial_v_n_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_chebyshev_polynomial_v_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_chebyshev_polynomial_v_x_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_chebyshev_polynomial_v_x_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_chebyshev_polynomial_w( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_chebyshev_polynomial_w_n_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_chebyshev_polynomial_w_n_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_chebyshev_polynomial_w_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_chebyshev_polynomial_w_x_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_chebyshev_polynomial_w_x_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_digamma(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_digamma_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_entr(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_entr_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_erf(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_erf_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_erfc(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_erfc_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_erfcx(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_erfcx_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_erfinv(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_erfinv_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_exp2(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_exp2_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_expit(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_expit_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_expm1(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_expm1_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_gammainc( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_special_gammainc_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_special_gammaincc( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_special_gammaincc_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_special_gammaln(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_gammaln_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_hermite_polynomial_h( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_hermite_polynomial_h_n_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_hermite_polynomial_h_n_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_hermite_polynomial_h_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_hermite_polynomial_h_x_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_hermite_polynomial_h_x_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_hermite_polynomial_he( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_hermite_polynomial_he_n_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_hermite_polynomial_he_n_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_hermite_polynomial_he_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_hermite_polynomial_he_x_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_hermite_polynomial_he_x_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_i0(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_i0_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_i0e(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_i0e_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_i1(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_i1_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_i1e(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_i1e_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_laguerre_polynomial_l( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_laguerre_polynomial_l_n_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_laguerre_polynomial_l_n_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_laguerre_polynomial_l_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_laguerre_polynomial_l_x_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_laguerre_polynomial_l_x_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_legendre_polynomial_p( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_legendre_polynomial_p_n_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_legendre_polynomial_p_n_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_legendre_polynomial_p_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_legendre_polynomial_p_x_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_legendre_polynomial_p_x_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_log1p(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_log1p_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_log_ndtr(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_log_ndtr_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_log_softmax( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + dtype_: c_int, + ); + pub fn atg_special_logit( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + eps_v: f64, + eps_null: i8, + ); + pub fn atg_special_logit_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + eps_v: f64, + eps_null: i8, + ); + pub fn atg_special_logsumexp( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + ); + pub fn atg_special_logsumexp_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + ); + pub fn atg_special_modified_bessel_i0(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_modified_bessel_i0_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_modified_bessel_i1(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_modified_bessel_i1_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_modified_bessel_k0(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_modified_bessel_k0_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_modified_bessel_k1(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_modified_bessel_k1_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_multigammaln(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: i64); + pub fn atg_special_multigammaln_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + p_: i64, + ); + pub fn atg_special_ndtr(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_ndtr_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_ndtri(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_ndtri_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_polygamma(out__: *mut *mut C_tensor, n_: i64, self_: *mut C_tensor); + pub fn atg_special_polygamma_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + n_: i64, + self_: *mut C_tensor, + ); + pub fn atg_special_psi(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_psi_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_round(out__: *mut *mut C_tensor, self_: *mut C_tensor, decimals_: i64); + pub fn atg_special_round_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + decimals_: i64, + ); + pub fn atg_special_scaled_modified_bessel_k0(out__: *mut *mut C_tensor, x_: *mut C_tensor); + pub fn atg_special_scaled_modified_bessel_k0_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + ); + pub fn atg_special_scaled_modified_bessel_k1(out__: *mut *mut C_tensor, x_: *mut C_tensor); + pub fn atg_special_scaled_modified_bessel_k1_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + ); + pub fn atg_special_shifted_chebyshev_polynomial_t( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_shifted_chebyshev_polynomial_t_n_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_shifted_chebyshev_polynomial_t_n_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_shifted_chebyshev_polynomial_t_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_shifted_chebyshev_polynomial_t_x_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_shifted_chebyshev_polynomial_t_x_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_shifted_chebyshev_polynomial_u( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_shifted_chebyshev_polynomial_u_n_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_shifted_chebyshev_polynomial_u_n_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_shifted_chebyshev_polynomial_u_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_shifted_chebyshev_polynomial_u_x_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_shifted_chebyshev_polynomial_u_x_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_shifted_chebyshev_polynomial_v( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_shifted_chebyshev_polynomial_v_n_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_shifted_chebyshev_polynomial_v_n_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_shifted_chebyshev_polynomial_v_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_shifted_chebyshev_polynomial_v_x_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_shifted_chebyshev_polynomial_v_x_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_shifted_chebyshev_polynomial_w( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_shifted_chebyshev_polynomial_w_n_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_shifted_chebyshev_polynomial_w_n_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_scalar, + ); + pub fn atg_special_shifted_chebyshev_polynomial_w_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + n_: *mut C_tensor, + ); + pub fn atg_special_shifted_chebyshev_polynomial_w_x_scalar( + out__: *mut *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_shifted_chebyshev_polynomial_w_x_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_scalar, + n_: *mut C_tensor, + ); + pub fn atg_special_sinc(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_special_sinc_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_special_softmax( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + dtype_: c_int, + ); + pub fn atg_special_spherical_bessel_j0(out__: *mut *mut C_tensor, x_: *mut C_tensor); + pub fn atg_special_spherical_bessel_j0_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + x_: *mut C_tensor, + ); + pub fn atg_special_xlog1py( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_special_xlog1py_other_scalar( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_special_xlog1py_other_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_special_xlog1py_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_special_xlog1py_self_scalar( + out__: *mut *mut C_tensor, + self_scalar_: *mut C_scalar, + other_: *mut C_tensor, + ); + pub fn atg_special_xlog1py_self_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_scalar_: *mut C_scalar, + other_: *mut C_tensor, + ); + pub fn atg_special_xlogy( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_special_xlogy_other_scalar( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_special_xlogy_other_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_special_xlogy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_special_xlogy_self_scalar( + out__: *mut *mut C_tensor, + self_scalar_: *mut C_scalar, + other_: *mut C_tensor, + ); + pub fn atg_special_xlogy_self_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_scalar_: *mut C_scalar, + other_: *mut C_tensor, + ); + pub fn atg_special_zeta(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_special_zeta_other_scalar( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_special_zeta_other_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_special_zeta_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_special_zeta_self_scalar( + out__: *mut *mut C_tensor, + self_scalar_: *mut C_scalar, + other_: *mut C_tensor, + ); + pub fn atg_special_zeta_self_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_scalar_: *mut C_scalar, + other_: *mut C_tensor, + ); + pub fn atg_split(self_: *mut C_tensor, split_size_: i64, dim_: i64) -> *mut *mut C_tensor; + pub fn atg_split_copy(self_: *mut C_tensor, split_size_: i64, dim_: i64) -> *mut *mut C_tensor; + pub fn atg_split_copy_tensor_out( + out_data: *const *mut C_tensor, + out_len: c_int, + self_: *mut C_tensor, + split_size_: i64, + dim_: i64, + ); + pub fn atg_split_sizes( + self_: *mut C_tensor, + split_size_data: *const i64, + split_size_len: c_int, + dim_: i64, + ) -> *mut *mut C_tensor; + pub fn atg_split_with_sizes( + self_: *mut C_tensor, + split_sizes_data: *const i64, + split_sizes_len: c_int, + dim_: i64, + ) -> *mut *mut C_tensor; + pub fn atg_split_with_sizes_copy( + self_: *mut C_tensor, + split_sizes_data: *const i64, + split_sizes_len: c_int, + dim_: i64, + ) -> *mut *mut C_tensor; + pub fn atg_split_with_sizes_copy_out( + out_data: *const *mut C_tensor, + out_len: c_int, + self_: *mut C_tensor, + split_sizes_data: *const i64, + split_sizes_len: c_int, + dim_: i64, + ); + pub fn atg_sqrt(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_sqrt_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_sqrt_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_square(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_square_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_square_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_squeeze(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_squeeze_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_squeeze_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_squeeze_copy_dim(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64); + pub fn atg_squeeze_copy_dim_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + ); + pub fn atg_squeeze_copy_dims( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + ); + pub fn atg_squeeze_copy_dims_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + ); + pub fn atg_squeeze_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_squeeze_dim(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64); + pub fn atg_squeeze_dim_(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64); + pub fn atg_squeeze_dims( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + ); + pub fn atg_squeeze_dims_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + ); + pub fn atg_sspaddmm( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + mat1_: *mut C_tensor, + mat2_: *mut C_tensor, + ); + pub fn atg_sspaddmm_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + mat1_: *mut C_tensor, + mat2_: *mut C_tensor, + ); + pub fn atg_stack( + out__: *mut *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + dim_: i64, + ); + pub fn atg_stack_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + dim_: i64, + ); + pub fn atg_std(out__: *mut *mut C_tensor, self_: *mut C_tensor, unbiased_: c_int); + pub fn atg_std_correction( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + correction_: *mut C_scalar, + keepdim_: c_int, + ); + pub fn atg_std_correction_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + correction_: *mut C_scalar, + keepdim_: c_int, + ); + pub fn atg_std_dim( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + unbiased_: c_int, + keepdim_: c_int, + ); + pub fn atg_std_mean(out__: *mut *mut C_tensor, self_: *mut C_tensor, unbiased_: c_int); + pub fn atg_std_mean_correction( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + correction_: *mut C_scalar, + keepdim_: c_int, + ); + pub fn atg_std_mean_correction_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + correction_: *mut C_scalar, + keepdim_: c_int, + ); + pub fn atg_std_mean_dim( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + unbiased_: c_int, + keepdim_: c_int, + ); + pub fn atg_std_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + unbiased_: c_int, + keepdim_: c_int, + ); + pub fn atg_stft( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + n_fft_: i64, + hop_length_v: i64, + hop_length_null: i8, + win_length_v: i64, + win_length_null: i8, + window_: *mut C_tensor, + normalized_: c_int, + onesided_: c_int, + return_complex_: c_int, + align_to_window_: c_int, + ); + pub fn atg_stft_center( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + n_fft_: i64, + hop_length_v: i64, + hop_length_null: i8, + win_length_v: i64, + win_length_null: i8, + window_: *mut C_tensor, + center_: c_int, + pad_mode_ptr: *const u8, + pad_mode_len: c_int, + normalized_: c_int, + onesided_: c_int, + return_complex_: c_int, + align_to_window_: c_int, + ); + pub fn atg_sub(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_sub_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_sub_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_sub_scalar(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_sub_scalar_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); + pub fn atg_sub_scalar_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_subtract(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_subtract_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_subtract_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_subtract_scalar( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_subtract_scalar_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_sum(out__: *mut *mut C_tensor, self_: *mut C_tensor, dtype_: c_int); + pub fn atg_sum_dim_intlist( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + dtype_: c_int, + ); + pub fn atg_sum_intlist_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + keepdim_: c_int, + dtype_: c_int, + ); + pub fn atg_sum_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dtype_: c_int, + ); + pub fn atg_sum_to_size( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg_svd( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + some_: c_int, + compute_uv_: c_int, + ); + pub fn atg_svd_u( + out__: *mut *mut C_tensor, + U_: *mut C_tensor, + S_: *mut C_tensor, + V_: *mut C_tensor, + self_: *mut C_tensor, + some_: c_int, + compute_uv_: c_int, + ); + pub fn atg_swapaxes(out__: *mut *mut C_tensor, self_: *mut C_tensor, axis0_: i64, axis1_: i64); + pub fn atg_swapaxes_(out__: *mut *mut C_tensor, self_: *mut C_tensor, axis0_: i64, axis1_: i64); + pub fn atg_swapdims(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim0_: i64, dim1_: i64); + pub fn atg_swapdims_(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim0_: i64, dim1_: i64); + pub fn atg_t(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_t_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_t_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_t_copy_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_take(out__: *mut *mut C_tensor, self_: *mut C_tensor, index_: *mut C_tensor); + pub fn atg_take_along_dim( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + indices_: *mut C_tensor, + dim_v: i64, + dim_null: i8, + ); + pub fn atg_take_along_dim_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + indices_: *mut C_tensor, + dim_v: i64, + dim_null: i8, + ); + pub fn atg_take_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + index_: *mut C_tensor, + ); + pub fn atg_tan(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_tan_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_tan_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_tanh(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_tanh_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_tanh_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + output_: *mut C_tensor, + ); + pub fn atg_tanh_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + output_: *mut C_tensor, + ); + pub fn atg_tanh_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_tensor_split(self_: *mut C_tensor, sections_: i64, dim_: i64) -> *mut *mut C_tensor; + pub fn atg_tensor_split_indices( + self_: *mut C_tensor, + indices_data: *const i64, + indices_len: c_int, + dim_: i64, + ) -> *mut *mut C_tensor; + pub fn atg_tensor_split_tensor_indices_or_sections( + self_: *mut C_tensor, + tensor_indices_or_sections_: *mut C_tensor, + dim_: i64, + ) -> *mut *mut C_tensor; + pub fn atg_tensordot( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + dims_self_data: *const i64, + dims_self_len: c_int, + dims_other_data: *const i64, + dims_other_len: c_int, + ); + pub fn atg_tensordot_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + dims_self_data: *const i64, + dims_self_len: c_int, + dims_other_data: *const i64, + dims_other_len: c_int, + ); + pub fn atg_threshold( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + threshold_: *mut C_scalar, + value_: *mut C_scalar, + ); + pub fn atg_threshold_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + threshold_: *mut C_scalar, + value_: *mut C_scalar, + ); + pub fn atg_threshold_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + threshold_: *mut C_scalar, + ); + pub fn atg_threshold_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + self_: *mut C_tensor, + threshold_: *mut C_scalar, + ); + pub fn atg_threshold_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + threshold_: *mut C_scalar, + value_: *mut C_scalar, + ); + pub fn atg_tile( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dims_data: *const i64, + dims_len: c_int, + ); + pub fn atg_to(out__: *mut *mut C_tensor, self_: *mut C_tensor, device_: c_int); + pub fn atg_to_dense( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dtype_: c_int, + masked_grad_: c_int, + ); + pub fn atg_to_dense_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + input_: *mut C_tensor, + masked_grad_: c_int, + ); + pub fn atg_to_device( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + device_: c_int, + dtype_: c_int, + non_blocking_: c_int, + copy_: c_int, + ); + pub fn atg_to_dtype( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dtype_: c_int, + non_blocking_: c_int, + copy_: c_int, + ); + pub fn atg_to_dtype_layout( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + options_kind: c_int, + options_device: c_int, + non_blocking_: c_int, + copy_: c_int, + ); + pub fn atg_to_mkldnn(out__: *mut *mut C_tensor, self_: *mut C_tensor, dtype_: c_int); + pub fn atg_to_mkldnn_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + input_: *mut C_tensor, + ); + pub fn atg_to_mkldnn_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dtype_: c_int, + ); + pub fn atg_to_other( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + non_blocking_: c_int, + copy_: c_int, + ); + pub fn atg_to_padded_tensor( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + padding_: f64, + output_size_data: *const i64, + output_size_len: c_int, + ); + pub fn atg_to_padded_tensor_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + padding_: f64, + output_size_data: *const i64, + output_size_len: c_int, + ); + pub fn atg_to_sparse( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + layout_: i8, + blocksize_data: *const i64, + blocksize_len: c_int, + dense_dim_v: i64, + dense_dim_null: i8, + ); + pub fn atg_to_sparse_bsc( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + blocksize_data: *const i64, + blocksize_len: c_int, + dense_dim_v: i64, + dense_dim_null: i8, + ); + pub fn atg_to_sparse_bsr( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + blocksize_data: *const i64, + blocksize_len: c_int, + dense_dim_v: i64, + dense_dim_null: i8, + ); + pub fn atg_to_sparse_csc( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dense_dim_v: i64, + dense_dim_null: i8, + ); + pub fn atg_to_sparse_csr( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dense_dim_v: i64, + dense_dim_null: i8, + ); + pub fn atg_to_sparse_sparse_dim( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + sparse_dim_: i64, + ); + pub fn atg_topk( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + k_: i64, + dim_: i64, + largest_: c_int, + sorted_: c_int, + ); + pub fn atg_topk_values( + out__: *mut *mut C_tensor, + values_: *mut C_tensor, + indices_: *mut C_tensor, + self_: *mut C_tensor, + k_: i64, + dim_: i64, + largest_: c_int, + sorted_: c_int, + ); + pub fn atg_totype(out__: *mut *mut C_tensor, self_: *mut C_tensor, scalar_type_: c_int); + pub fn atg_trace(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_trace_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + sizes_data: *const i64, + sizes_len: c_int, + ); + pub fn atg_trace_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_transpose(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim0_: i64, dim1_: i64); + pub fn atg_transpose_(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim0_: i64, dim1_: i64); + pub fn atg_transpose_copy( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim0_: i64, + dim1_: i64, + ); + pub fn atg_transpose_copy_int_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim0_: i64, + dim1_: i64, + ); + pub fn atg_trapezoid(out__: *mut *mut C_tensor, y_: *mut C_tensor, dim_: i64); + pub fn atg_trapezoid_x( + out__: *mut *mut C_tensor, + y_: *mut C_tensor, + x_: *mut C_tensor, + dim_: i64, + ); + pub fn atg_trapz(out__: *mut *mut C_tensor, y_: *mut C_tensor, x_: *mut C_tensor, dim_: i64); + pub fn atg_trapz_dx(out__: *mut *mut C_tensor, y_: *mut C_tensor, dx_: f64, dim_: i64); + pub fn atg_triangular_solve( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + A_: *mut C_tensor, + upper_: c_int, + transpose_: c_int, + unitriangular_: c_int, + ); + pub fn atg_triangular_solve_x( + out__: *mut *mut C_tensor, + X_: *mut C_tensor, + M_: *mut C_tensor, + self_: *mut C_tensor, + A_: *mut C_tensor, + upper_: c_int, + transpose_: c_int, + unitriangular_: c_int, + ); + pub fn atg_tril(out__: *mut *mut C_tensor, self_: *mut C_tensor, diagonal_: i64); + pub fn atg_tril_(out__: *mut *mut C_tensor, self_: *mut C_tensor, diagonal_: i64); + pub fn atg_tril_indices( + out__: *mut *mut C_tensor, + row_: i64, + col_: i64, + offset_: i64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_tril_indices_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + row_: i64, + col_: i64, + offset_: i64, + ); + pub fn atg_tril_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + diagonal_: i64, + ); + pub fn atg_triplet_margin_loss( + out__: *mut *mut C_tensor, + anchor_: *mut C_tensor, + positive_: *mut C_tensor, + negative_: *mut C_tensor, + margin_: f64, + p_: f64, + eps_: f64, + swap_: c_int, + reduction_: i64, + ); + pub fn atg_triu(out__: *mut *mut C_tensor, self_: *mut C_tensor, diagonal_: i64); + pub fn atg_triu_(out__: *mut *mut C_tensor, self_: *mut C_tensor, diagonal_: i64); + pub fn atg_triu_indices( + out__: *mut *mut C_tensor, + row_: i64, + col_: i64, + offset_: i64, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_triu_indices_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + row_: i64, + col_: i64, + offset_: i64, + ); + pub fn atg_triu_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + diagonal_: i64, + ); + pub fn atg_true_divide(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_true_divide_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_true_divide_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_true_divide_scalar( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_true_divide_scalar_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_trunc(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_trunc_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_trunc_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_type_as(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_unbind(self_: *mut C_tensor, dim_: i64) -> *mut *mut C_tensor; + pub fn atg_unbind_copy(self_: *mut C_tensor, dim_: i64) -> *mut *mut C_tensor; + pub fn atg_unbind_copy_int_out( + out_data: *const *mut C_tensor, + out_len: c_int, + self_: *mut C_tensor, + dim_: i64, + ); + pub fn atg_unflatten( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + sizes_data: *const i64, + sizes_len: c_int, + ); + pub fn atg_unflatten_dense_tensors( + flat_: *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ) -> *mut *mut C_tensor; + pub fn atg_unfold( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dimension_: i64, + size_: i64, + step_: i64, + ); + pub fn atg_unfold_backward( + out__: *mut *mut C_tensor, + grad_in_: *mut C_tensor, + input_sizes_data: *const i64, + input_sizes_len: c_int, + dim_: i64, + size_: i64, + step_: i64, + ); + pub fn atg_unfold_backward_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + grad_in_: *mut C_tensor, + input_sizes_data: *const i64, + input_sizes_len: c_int, + dim_: i64, + size_: i64, + step_: i64, + ); + pub fn atg_unfold_copy( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dimension_: i64, + size_: i64, + step_: i64, + ); + pub fn atg_unfold_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dimension_: i64, + size_: i64, + step_: i64, + ); + pub fn atg_uniform(out__: *mut *mut C_tensor, self_: *mut C_tensor, from_: f64, to_: f64); + pub fn atg_uniform_(out__: *mut *mut C_tensor, self_: *mut C_tensor, from_: f64, to_: f64); + pub fn atg_uniform_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + from_: f64, + to_: f64, + ); + pub fn atg_unique_consecutive( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + return_inverse_: c_int, + return_counts_: c_int, + dim_v: i64, + dim_null: i8, + ); + pub fn atg_unique_consecutive_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + out2_: *mut C_tensor, + self_: *mut C_tensor, + return_inverse_: c_int, + return_counts_: c_int, + dim_v: i64, + dim_null: i8, + ); + pub fn atg_unique_dim( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + sorted_: c_int, + return_inverse_: c_int, + return_counts_: c_int, + ); + pub fn atg_unique_dim_consecutive( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + return_inverse_: c_int, + return_counts_: c_int, + ); + pub fn atg_unique_dim_consecutive_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + out2_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + return_inverse_: c_int, + return_counts_: c_int, + ); + pub fn atg_unique_dim_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + out2_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + sorted_: c_int, + return_inverse_: c_int, + return_counts_: c_int, + ); + pub fn atg_unsafe_chunk(self_: *mut C_tensor, chunks_: i64, dim_: i64) -> *mut *mut C_tensor; + pub fn atg_unsafe_split( + self_: *mut C_tensor, + split_size_: i64, + dim_: i64, + ) -> *mut *mut C_tensor; + pub fn atg_unsafe_split_tensor_out( + out_data: *const *mut C_tensor, + out_len: c_int, + self_: *mut C_tensor, + split_size_: i64, + dim_: i64, + ); + pub fn atg_unsafe_split_with_sizes( + self_: *mut C_tensor, + split_sizes_data: *const i64, + split_sizes_len: c_int, + dim_: i64, + ) -> *mut *mut C_tensor; + pub fn atg_unsafe_split_with_sizes_out( + out_data: *const *mut C_tensor, + out_len: c_int, + self_: *mut C_tensor, + split_sizes_data: *const i64, + split_sizes_len: c_int, + dim_: i64, + ); + pub fn atg_unsqueeze(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64); + pub fn atg_unsqueeze_(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64); + pub fn atg_unsqueeze_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64); + pub fn atg_unsqueeze_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_: i64, + ); + pub fn atg_upsample_bicubic2d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + align_corners_: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg_upsample_bicubic2d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + align_corners_: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg_upsample_bicubic2d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + align_corners_: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg_upsample_bicubic2d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + align_corners_: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg_upsample_bicubic2d_vec( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + align_corners_: c_int, + scale_factors_data: *const f64, + scale_factors_len: c_int, + ); + pub fn atg_upsample_bilinear2d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + align_corners_: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg_upsample_bilinear2d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + align_corners_: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg_upsample_bilinear2d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + align_corners_: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg_upsample_bilinear2d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + align_corners_: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg_upsample_bilinear2d_vec( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + align_corners_: c_int, + scale_factors_data: *const f64, + scale_factors_len: c_int, + ); + pub fn atg_upsample_bilinear2d_vec_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + input_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + align_corners_: c_int, + scale_factors_data: *const f64, + scale_factors_len: c_int, + ); + pub fn atg_upsample_linear1d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + align_corners_: c_int, + scales_v: f64, + scales_null: i8, + ); + pub fn atg_upsample_linear1d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + align_corners_: c_int, + scales_v: f64, + scales_null: i8, + ); + pub fn atg_upsample_linear1d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + align_corners_: c_int, + scales_v: f64, + scales_null: i8, + ); + pub fn atg_upsample_linear1d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + align_corners_: c_int, + scales_v: f64, + scales_null: i8, + ); + pub fn atg_upsample_linear1d_vec( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + align_corners_: c_int, + scale_factors_data: *const f64, + scale_factors_len: c_int, + ); + pub fn atg_upsample_nearest1d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + scales_v: f64, + scales_null: i8, + ); + pub fn atg_upsample_nearest1d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + scales_v: f64, + scales_null: i8, + ); + pub fn atg_upsample_nearest1d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + scales_v: f64, + scales_null: i8, + ); + pub fn atg_upsample_nearest1d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + scales_v: f64, + scales_null: i8, + ); + pub fn atg_upsample_nearest1d_vec( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + scale_factors_data: *const f64, + scale_factors_len: c_int, + ); + pub fn atg_upsample_nearest2d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg_upsample_nearest2d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg_upsample_nearest2d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg_upsample_nearest2d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg_upsample_nearest2d_vec( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + scale_factors_data: *const f64, + scale_factors_len: c_int, + ); + pub fn atg_upsample_nearest2d_vec_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + input_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + scale_factors_data: *const f64, + scale_factors_len: c_int, + ); + pub fn atg_upsample_nearest3d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + scales_d_v: f64, + scales_d_null: i8, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg_upsample_nearest3d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + scales_d_v: f64, + scales_d_null: i8, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg_upsample_nearest3d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + scales_d_v: f64, + scales_d_null: i8, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg_upsample_nearest3d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + scales_d_v: f64, + scales_d_null: i8, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg_upsample_nearest3d_vec( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + scale_factors_data: *const f64, + scale_factors_len: c_int, + ); + pub fn atg_upsample_trilinear3d( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + align_corners_: c_int, + scales_d_v: f64, + scales_d_null: i8, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg_upsample_trilinear3d_backward( + out__: *mut *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + align_corners_: c_int, + scales_d_v: f64, + scales_d_null: i8, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg_upsample_trilinear3d_backward_grad_input( + out__: *mut *mut C_tensor, + grad_input_: *mut C_tensor, + grad_output_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + input_size_data: *const i64, + input_size_len: c_int, + align_corners_: c_int, + scales_d_v: f64, + scales_d_null: i8, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg_upsample_trilinear3d_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + align_corners_: c_int, + scales_d_v: f64, + scales_d_null: i8, + scales_h_v: f64, + scales_h_null: i8, + scales_w_v: f64, + scales_w_null: i8, + ); + pub fn atg_upsample_trilinear3d_vec( + out__: *mut *mut C_tensor, + input_: *mut C_tensor, + output_size_data: *const i64, + output_size_len: c_int, + align_corners_: c_int, + scale_factors_data: *const f64, + scale_factors_len: c_int, + ); + pub fn atg_value_selecting_reduction_backward( + out__: *mut *mut C_tensor, + grad_: *mut C_tensor, + dim_: i64, + indices_: *mut C_tensor, + sizes_data: *const i64, + sizes_len: c_int, + keepdim_: c_int, + ); + pub fn atg_values(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_values_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_values_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_vander( + out__: *mut *mut C_tensor, + x_: *mut C_tensor, + n_v: i64, + n_null: i8, + increasing_: c_int, + ); + pub fn atg_var(out__: *mut *mut C_tensor, self_: *mut C_tensor, unbiased_: c_int); + pub fn atg_var_correction( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + correction_: *mut C_scalar, + keepdim_: c_int, + ); + pub fn atg_var_correction_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + correction_: *mut C_scalar, + keepdim_: c_int, + ); + pub fn atg_var_dim( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + unbiased_: c_int, + keepdim_: c_int, + ); + pub fn atg_var_mean(out__: *mut *mut C_tensor, self_: *mut C_tensor, unbiased_: c_int); + pub fn atg_var_mean_correction( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + correction_: *mut C_scalar, + keepdim_: c_int, + ); + pub fn atg_var_mean_correction_out( + out__: *mut *mut C_tensor, + out0_: *mut C_tensor, + out1_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + correction_: *mut C_scalar, + keepdim_: c_int, + ); + pub fn atg_var_mean_dim( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + unbiased_: c_int, + keepdim_: c_int, + ); + pub fn atg_var_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dim_data: *const i64, + dim_len: c_int, + unbiased_: c_int, + keepdim_: c_int, + ); + pub fn atg_vdot(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_vdot_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_view( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg_view_as(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_view_as_complex(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_view_as_complex_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_view_as_complex_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_view_as_real(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_view_as_real_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_view_as_real_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + ); + pub fn atg_view_copy( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg_view_copy_dtype(out__: *mut *mut C_tensor, self_: *mut C_tensor, dtype_: c_int); + pub fn atg_view_copy_dtype_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + dtype_: c_int, + ); + pub fn atg_view_copy_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); + pub fn atg_view_dtype(out__: *mut *mut C_tensor, self_: *mut C_tensor, dtype_: c_int); + pub fn atg_vsplit(self_: *mut C_tensor, sections_: i64) -> *mut *mut C_tensor; + pub fn atg_vsplit_array( + self_: *mut C_tensor, + indices_data: *const i64, + indices_len: c_int, + ) -> *mut *mut C_tensor; + pub fn atg_vstack( + out__: *mut *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ); + pub fn atg_vstack_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + tensors_data: *const *mut C_tensor, + tensors_len: c_int, + ); + pub fn atg_where(condition_: *mut C_tensor) -> *mut *mut C_tensor; + pub fn atg_where_scalar( + out__: *mut *mut C_tensor, + condition_: *mut C_tensor, + self_scalar_: *mut C_scalar, + other_: *mut C_scalar, + ); + pub fn atg_where_scalarother( + out__: *mut *mut C_tensor, + condition_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_where_scalarself( + out__: *mut *mut C_tensor, + condition_: *mut C_tensor, + self_scalar_: *mut C_scalar, + other_: *mut C_tensor, + ); + pub fn atg_where_self( + out__: *mut *mut C_tensor, + condition_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_where_self_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + condition_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_xlogy(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_xlogy_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); + pub fn atg_xlogy_outscalar_other( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_xlogy_outscalar_self( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_scalar_: *mut C_scalar, + other_: *mut C_tensor, + ); + pub fn atg_xlogy_outtensor( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_tensor, + ); + pub fn atg_xlogy_scalar_other( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_xlogy_scalar_other_( + out__: *mut *mut C_tensor, + self_: *mut C_tensor, + other_: *mut C_scalar, + ); + pub fn atg_xlogy_scalar_self( + out__: *mut *mut C_tensor, + self_scalar_: *mut C_scalar, + other_: *mut C_tensor, + ); + pub fn atg_zero(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_zero_(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_zero_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_zeros( + out__: *mut *mut C_tensor, + size_data: *const i64, + size_len: c_int, + options_kind: c_int, + options_device: c_int, + ); + pub fn atg_zeros_like(out__: *mut *mut C_tensor, self_: *mut C_tensor); + pub fn atg_zeros_like_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); + pub fn atg_zeros_out( + out__: *mut *mut C_tensor, + out_: *mut C_tensor, + size_data: *const i64, + size_len: c_int, + ); +} diff --git a/v2/dependency-patches/torch-sys-0.24.0/src/cuda.rs b/v2/dependency-patches/torch-sys-0.24.0/src/cuda.rs new file mode 100644 index 0000000000..7ea4bbeff3 --- /dev/null +++ b/v2/dependency-patches/torch-sys-0.24.0/src/cuda.rs @@ -0,0 +1,30 @@ +use libc::c_int; + +extern "C" { + /// Returns the number of CUDA devices available. + pub fn atc_cuda_device_count() -> c_int; + + /// Returns true if at least one CUDA device is available. + pub fn atc_cuda_is_available() -> c_int; + + /// Returns true if CUDA is available, and CuDNN is available. + pub fn atc_cudnn_is_available() -> c_int; + + /// Sets the seed for the current GPU. + pub fn atc_manual_seed(seed: u64); + + /// Sets the seed for all available GPUs. + pub fn atc_manual_seed_all(seed: u64); + + /// Waits for all kernels in all streams on a CUDA device to complete. + pub fn atc_synchronize(device_index: i64); + + /// Returns true if CUDNN is enabled by the user. + pub fn atc_user_enabled_cudnn() -> c_int; + + /// Enable or disable CUDNN. + pub fn atc_set_user_enabled_cudnn(b: c_int); + + /// Sets CUDNN benchmark mode. + pub fn atc_set_benchmark_cudnn(b: c_int); +} diff --git a/v2/dependency-patches/torch-sys-0.24.0/src/io.rs b/v2/dependency-patches/torch-sys-0.24.0/src/io.rs new file mode 100644 index 0000000000..0e242eb698 --- /dev/null +++ b/v2/dependency-patches/torch-sys-0.24.0/src/io.rs @@ -0,0 +1,111 @@ +use libc::{c_void, size_t}; +use std::io::{Read, Seek, SeekFrom, Write}; + +pub trait ReadStream: Read + Seek {} + +#[no_mangle] +extern "C" fn tch_write_stream_destructor(stream_ptr: *mut c_void) { + unsafe { + let boxed_stream: Box> = Box::from_raw(stream_ptr as *mut Box); + drop(boxed_stream) + } +} + +#[no_mangle] +extern "C" fn tch_write_stream_write( + stream_ptr: *mut c_void, + buf: *const u8, + size: size_t, + out_size: *mut size_t, +) -> bool { + unsafe { + let boxed_stream = stream_ptr as *mut Box; + let buffer = std::slice::from_raw_parts(buf, size); + match (*boxed_stream).write(buffer) { + Ok(x) => { + *out_size = x; + true + } + Err(_) => false, + } + } +} + +#[no_mangle] +extern "C" fn tch_read_stream_destructor(stream_ptr: *mut c_void) { + unsafe { + let boxed_stream: Box> = + Box::from_raw(stream_ptr as *mut Box); + drop(boxed_stream) + } +} + +#[no_mangle] +extern "C" fn tch_read_stream_stream_position(stream_ptr: *mut c_void, current: *mut u64) -> bool { + unsafe { + let boxed_stream = stream_ptr as *mut Box; + match (*boxed_stream).stream_position() { + Ok(ret) => { + *current = ret; + true + } + Err(_) => false, + } + } +} + +#[no_mangle] +extern "C" fn tch_read_stream_seek_start( + stream_ptr: *mut c_void, + pos: u64, + new_pos: *mut u64, +) -> bool { + unsafe { + let boxed_stream = stream_ptr as *mut Box; + match (*boxed_stream).seek(SeekFrom::Start(pos)) { + Ok(ret) => { + *new_pos = ret; + true + } + Err(_) => false, + } + } +} + +#[no_mangle] +extern "C" fn tch_read_stream_seek_end( + stream_ptr: *mut c_void, + pos: i64, + new_pos: *mut u64, +) -> bool { + unsafe { + let boxed_stream = stream_ptr as *mut Box; + match (*boxed_stream).seek(SeekFrom::End(pos)) { + Ok(ret) => { + *new_pos = ret; + true + } + Err(_) => false, + } + } +} + +#[no_mangle] +extern "C" fn tch_read_stream_read( + stream_ptr: *mut c_void, + buf: *mut u8, + size: size_t, + read_size: *mut size_t, +) -> bool { + unsafe { + let boxed_stream = stream_ptr as *mut Box; + let buffer = std::slice::from_raw_parts_mut(buf, size); + match (*boxed_stream).read(buffer) { + Ok(ret) => { + *read_size = ret; + true + } + Err(_) => false, + } + } +} diff --git a/v2/dependency-patches/torch-sys-0.24.0/src/lib.rs b/v2/dependency-patches/torch-sys-0.24.0/src/lib.rs new file mode 100644 index 0000000000..2ae76e1aff --- /dev/null +++ b/v2/dependency-patches/torch-sys-0.24.0/src/lib.rs @@ -0,0 +1,336 @@ +pub mod cuda; +pub mod io; +#[cfg(feature = "python-extension")] +pub mod python; +mod traits; + +use libc::{c_char, c_int, c_uchar, c_void, size_t}; +pub use traits::{DoubleList, IntList, IntListOption}; + +#[repr(C)] +pub struct C_scalar { + _private: [u8; 0], +} + +extern "C" { + pub fn ats_int(v: i64) -> *mut C_scalar; + pub fn ats_float(v: f64) -> *mut C_scalar; + pub fn ats_to_int(arg: *mut C_scalar) -> i64; + pub fn ats_to_float(arg: *mut C_scalar) -> f64; + pub fn ats_to_string(arg: *mut C_scalar) -> *mut c_char; + pub fn ats_free(arg: *mut C_scalar); +} + +#[repr(C)] +pub struct C_tensor { + _private: [u8; 0], +} + +extern "C" { + pub fn at_new_tensor() -> *mut C_tensor; + pub fn at_shallow_clone(arg: *mut C_tensor) -> *mut C_tensor; + pub fn at_copy_(dst: *mut C_tensor, src: *mut C_tensor); + pub fn at_data_ptr(arg: *mut C_tensor) -> *mut c_void; + pub fn at_defined(arg: *mut C_tensor) -> c_int; + pub fn at_is_sparse(arg: *mut C_tensor) -> c_int; + pub fn at_is_mkldnn(arg: *mut C_tensor) -> c_int; + pub fn at_is_contiguous(args: *mut C_tensor) -> c_int; + pub fn at_backward(arg: *mut C_tensor, keep_graph: c_int, create_graph: c_int); + pub fn at_print(arg: *mut C_tensor); + pub fn at_to_string(arg: *mut C_tensor, line_size: c_int) -> *mut c_char; + pub fn at_dim(arg: *mut C_tensor) -> size_t; + pub fn at_get(arg: *mut C_tensor, index: c_int) -> *mut C_tensor; + pub fn at_requires_grad(arg: *mut C_tensor) -> c_int; + pub fn at_shape(arg: *mut C_tensor, sz: *mut i64); + pub fn at_stride(arg: *mut C_tensor, sz: *mut i64); + pub fn at_double_value_at_indexes(arg: *mut C_tensor, idx: *const i64, idx_len: c_int) -> f64; + pub fn at_int64_value_at_indexes(arg: *mut C_tensor, idx: *const i64, idx_len: c_int) -> i64; + pub fn at_get_num_interop_threads() -> c_int; + pub fn at_get_num_threads() -> c_int; + pub fn at_set_num_interop_threads(n_threads: c_int); + pub fn at_set_num_threads(n_threads: c_int); + pub fn at_set_qengine(qengine: c_int); + pub fn at_free(arg: *mut C_tensor); + pub fn at_run_backward( + arg: *const *mut C_tensor, + ntensors: c_int, + inputs: *const *mut C_tensor, + ninputs: c_int, + outputs: *mut *mut C_tensor, + keep_graph: c_int, + create_graph: c_int, + ); + pub fn at_copy_data( + arg: *mut C_tensor, + vs: *const c_void, + numel: size_t, + elt_size_in_bytes: size_t, + ); + pub fn at_scalar_type(arg: *mut C_tensor) -> c_int; + pub fn at__amp_non_finite_check_and_unscale( + t: *mut C_tensor, + found_inf: *mut C_tensor, + inf_scale: *mut C_tensor, + ); + pub fn at_autocast_clear_cache(); + pub fn at_autocast_decrement_nesting() -> c_int; + pub fn at_autocast_increment_nesting() -> c_int; + pub fn at_autocast_is_enabled() -> c_int; + pub fn at_autocast_set_enabled(b: c_int) -> c_int; + pub fn at_device(arg: *mut C_tensor) -> c_int; + pub fn at_tensor_of_data( + vs: *const c_void, + dims: *const i64, + ndims: size_t, + elt_size_in_bytes: size_t, + kind: c_int, + ) -> *mut C_tensor; + pub fn at_tensor_of_blob( + vs: *const c_void, + dims: *const i64, + ndims: size_t, + strides: *const i64, + nstrides: size_t, + kind: c_int, + device: c_int, + ) -> *mut C_tensor; + pub fn at_grad_set_enabled(b: c_int) -> c_int; + pub fn at_save(arg: *mut C_tensor, filename: *const c_char); + pub fn at_save_to_stream(arg: *mut C_tensor, stream_ptr: *mut c_void); + pub fn at_load(filename: *const c_char) -> *mut C_tensor; + pub fn at_load_from_stream(stream_ptr: *mut c_void) -> *mut C_tensor; + pub fn at_save_multi( + args: *const *mut C_tensor, + names: *const *const c_char, + n: c_int, + filename: *const c_char, + ); + pub fn at_save_multi_to_stream( + args: *const *mut C_tensor, + names: *const *const c_char, + n: c_int, + stream_ptr: *mut c_void, + ); + pub fn at_loadz_callback( + filename: *const c_char, + data: *mut c_void, + f: extern "C" fn(*mut c_void, name: *const c_char, t: *mut C_tensor), + ); + pub fn at_loadz_callback_with_device( + filename: *const c_char, + data: *mut c_void, + f: extern "C" fn(*mut c_void, name: *const c_char, t: *mut C_tensor), + device_id: c_int, + ); + pub fn at_load_callback( + filename: *const c_char, + data: *mut c_void, + f: extern "C" fn(*mut c_void, name: *const c_char, t: *mut C_tensor), + ); + pub fn at_load_callback_with_device( + filename: *const c_char, + data: *mut c_void, + f: extern "C" fn(*mut c_void, name: *const c_char, t: *mut C_tensor), + device_id: c_int, + ); + pub fn at_load_from_stream_callback( + stream_ptr: *mut c_void, + data: *mut c_void, + f: extern "C" fn(*mut c_void, name: *const c_char, t: *mut C_tensor), + enable_device_id: bool, + device_id: c_int, + ); + + pub fn at_manual_seed(seed: i64); + pub fn at_set_graph_executor_optimize(b: bool); + pub fn at_context_has_openmp() -> bool; + pub fn at_context_has_mkl() -> bool; + pub fn at_context_has_lapack() -> bool; + pub fn at_context_has_mkldnn() -> bool; + pub fn at_context_has_magma() -> bool; + pub fn at_context_has_cuda() -> bool; + pub fn at_context_has_cudart() -> bool; + pub fn at_context_has_cusolver() -> bool; + pub fn at_context_has_hip() -> bool; + pub fn at_context_has_ipu() -> bool; + pub fn at_context_has_xla() -> bool; + pub fn at_context_has_lazy() -> bool; + pub fn at_context_has_mps() -> bool; + pub fn at_context_version_cudnn() -> i64; + pub fn at_context_version_cudart() -> i64; +} + +pub mod c_generated; + +extern "C" { + pub fn get_and_reset_last_err() -> *mut c_char; +} + +#[repr(C)] +pub struct C_optimizer { + _private: [u8; 0], +} + +extern "C" { + pub fn ato_adam( + lr: f64, + beta1: f64, + beta2: f64, + wd: f64, + eps: f64, + amsgrad: bool, + ) -> *mut C_optimizer; + pub fn ato_adamw( + lr: f64, + beta1: f64, + beta2: f64, + wd: f64, + eps: f64, + amsgrad: bool, + ) -> *mut C_optimizer; + pub fn ato_rms_prop( + lr: f64, + alpha: f64, + eps: f64, + wd: f64, + momentum: f64, + centered: c_int, + ) -> *mut C_optimizer; + pub fn ato_sgd( + lr: f64, + momentum: f64, + dampening: f64, + wd: f64, + nesterov: c_int, + ) -> *mut C_optimizer; + pub fn ato_add_parameters(arg: *mut C_optimizer, ts: *mut C_tensor, group: size_t); + pub fn ato_set_learning_rate(arg: *mut C_optimizer, lr: f64); + pub fn ato_set_learning_rate_group(arg: *mut C_optimizer, group: size_t, lr: f64); + pub fn ato_set_momentum(arg: *mut C_optimizer, momentum: f64); + pub fn ato_set_momentum_group(arg: *mut C_optimizer, group: size_t, momentum: f64); + pub fn ato_set_weight_decay(arg: *mut C_optimizer, weight_decay: f64); + pub fn ato_set_weight_decay_group(arg: *mut C_optimizer, group: size_t, weight_decay: f64); + pub fn ato_zero_grad(arg: *mut C_optimizer); + pub fn ato_step(arg: *mut C_optimizer); + pub fn ato_free(arg: *mut C_optimizer); + pub fn at_save_image(arg: *mut C_tensor, filename: *const c_char) -> c_int; + pub fn at_load_image(filename: *const c_char) -> *mut C_tensor; + pub fn at_load_image_from_memory( + img_data: *const c_uchar, + img_data_len: size_t, + ) -> *mut C_tensor; + pub fn at_resize_image(arg: *mut C_tensor, out_w: c_int, out_h: c_int) -> *mut C_tensor; +} + +#[allow(clippy::upper_case_acronyms)] +#[repr(C)] +pub struct CIValue { + _private: [u8; 0], +} + +#[repr(C)] +pub struct CModule_ { + _private: [u8; 0], +} + +extern "C" { + // Constructors + pub fn ati_none() -> *mut CIValue; + pub fn ati_bool(b: c_int) -> *mut CIValue; + pub fn ati_int(v: i64) -> *mut CIValue; + pub fn ati_double(v: f64) -> *mut CIValue; + pub fn ati_tensor(v: *mut C_tensor) -> *mut CIValue; + pub fn ati_string(s: *const c_char) -> *mut CIValue; + pub fn ati_tuple(v: *const *mut CIValue, n: c_int) -> *mut CIValue; + pub fn ati_generic_list(v: *const *mut CIValue, n: c_int) -> *mut CIValue; + pub fn ati_generic_dict(v: *const *mut CIValue, n: c_int) -> *mut CIValue; + pub fn ati_int_list(v: *const i64, n: c_int) -> *mut CIValue; + pub fn ati_double_list(v: *const f64, n: c_int) -> *mut CIValue; + pub fn ati_bool_list(v: *const c_char, n: c_int) -> *mut CIValue; + pub fn ati_string_list(v: *const *const c_char, n: c_int) -> *mut CIValue; + pub fn ati_tensor_list(v: *const *mut C_tensor, n: c_int) -> *mut CIValue; + pub fn ati_device(device_idx: c_int) -> *mut CIValue; + + // Type query + pub fn ati_tag(arg: *mut CIValue) -> c_int; + + // Getters + pub fn ati_to_int(arg: *mut CIValue) -> i64; + pub fn ati_to_bool(arg: *mut CIValue) -> c_int; + pub fn ati_to_double(arg: *mut CIValue) -> f64; + pub fn ati_to_tensor(arg: *mut CIValue) -> *mut C_tensor; + pub fn ati_length(arg: *mut CIValue) -> c_int; + pub fn ati_tuple_length(arg: *mut CIValue) -> c_int; + pub fn ati_to_tuple(arg: *mut CIValue, outputs: *mut *mut CIValue, n: c_int); + pub fn ati_to_generic_list(arg: *mut CIValue, outputs: *mut *mut CIValue, n: c_int); + pub fn ati_to_generic_dict(arg: *mut CIValue, outputs: *mut *mut CIValue, n: c_int); + pub fn ati_to_int_list(arg: *mut CIValue, outputs: *mut i64, n: c_int); + pub fn ati_to_double_list(arg: *mut CIValue, outputs: *mut f64, n: c_int); + pub fn ati_to_bool_list(arg: *mut CIValue, outputs: *mut c_char, n: c_int); + pub fn ati_to_tensor_list(arg: *mut CIValue, outputs: *mut *mut C_tensor, n: c_int); + pub fn ati_to_string(arg: *mut CIValue) -> *mut c_char; + + pub fn ati_clone(arg: *mut CIValue) -> *mut CIValue; + pub fn ati_free(arg: *mut CIValue); + + pub fn ati_object_method_( + arg: *mut CIValue, + method_name: *const c_char, + args: *const *mut CIValue, + n: c_int, + ) -> *mut CIValue; + + pub fn ati_object_getattr_(arg: *mut CIValue, attr_name: *const c_char) -> *mut CIValue; + + pub fn atm_load(filename: *const c_char) -> *mut CModule_; + pub fn atm_load_on_device(filename: *const c_char, device: c_int) -> *mut CModule_; + pub fn atm_load_str(data: *const c_char, sz: size_t) -> *mut CModule_; + pub fn atm_load_str_on_device(data: *const c_char, sz: size_t, device: c_int) -> *mut CModule_; + pub fn atm_forward(m: *mut CModule_, args: *const *mut C_tensor, n: c_int) -> *mut C_tensor; + pub fn atm_forward_(m: *mut CModule_, args: *const *mut CIValue, n: c_int) -> *mut CIValue; + pub fn atm_method( + m: *mut CModule_, + method_name: *const c_char, + args: *const *mut C_tensor, + n: c_int, + ) -> *mut C_tensor; + pub fn atm_method_( + m: *mut CModule_, + method_name: *const c_char, + args: *const *mut CIValue, + n: c_int, + ) -> *mut CIValue; + pub fn atm_create_class_( + m: *mut CModule_, + clz_name: *const c_char, + args: *const *mut CIValue, + n: c_int, + ) -> *mut CIValue; + pub fn atm_eval(m: *mut CModule_); + pub fn atm_train(m: *mut CModule_); + pub fn atm_free(m: *mut CModule_); + pub fn atm_to(m: *mut CModule_, device: c_int, kind: c_int, non_blocking: bool); + pub fn atm_save(m: *mut CModule_, filename: *const c_char); + pub fn atm_get_profiling_mode() -> c_int; + pub fn atm_set_profiling_mode(profiling_mode: c_int); + pub fn atm_fuser_cuda_set_enabled(enabled: bool); + pub fn atm_fuser_cuda_is_enabled() -> bool; + pub fn atm_named_parameters( + m: *mut CModule_, + data: *mut c_void, + f: extern "C" fn(*mut c_void, name: *const c_char, t: *mut C_tensor), + ); + pub fn atm_create_for_tracing( + modl_name: *const c_char, + inputs: *const *mut C_tensor, + ninputs: c_int, + ) -> *mut CModule_; + pub fn atm_end_tracing( + m: *mut CModule_, + fn_name: *const c_char, + outputs: *const *mut C_tensor, + noutputs: c_int, + ); + pub fn atm_set_tensor_expr_fuser_enabled(enabled: c_int); + pub fn atm_get_tensor_expr_fuser_enabled() -> bool; +} diff --git a/v2/dependency-patches/torch-sys-0.24.0/src/python.rs b/v2/dependency-patches/torch-sys-0.24.0/src/python.rs new file mode 100644 index 0000000000..64c4b8c649 --- /dev/null +++ b/v2/dependency-patches/torch-sys-0.24.0/src/python.rs @@ -0,0 +1,12 @@ +use super::C_tensor; + +#[repr(C)] +pub struct C_pyobject { + _private: [u8; 0], +} + +extern "C" { + pub fn thp_variable_check(obj: *mut C_pyobject) -> bool; + pub fn thp_variable_wrap(var: *mut C_tensor) -> *mut C_pyobject; + pub fn thp_variable_unpack(obj: *mut C_pyobject) -> *mut C_tensor; +} diff --git a/v2/dependency-patches/torch-sys-0.24.0/src/traits.rs b/v2/dependency-patches/torch-sys-0.24.0/src/traits.rs new file mode 100644 index 0000000000..e40190d8d9 --- /dev/null +++ b/v2/dependency-patches/torch-sys-0.24.0/src/traits.rs @@ -0,0 +1,119 @@ +macro_rules! list_trait { + ($trait: ident, $ty: ident) => { + pub trait $trait { + fn as_ptr(&self) -> *const $ty; + fn len_i32(&self) -> i32; + } + + impl $trait for $ty { + fn as_ptr(&self) -> *const $ty { + self + } + fn len_i32(&self) -> i32 { + 1 + } + } + + impl $trait for &[$ty] { + fn as_ptr(&self) -> *const $ty { + (*self).as_ptr() + } + fn len_i32(&self) -> i32 { + self.len() as i32 + } + } + + impl $trait for Vec<$ty> { + fn as_ptr(&self) -> *const $ty { + self.as_slice().as_ptr() + } + fn len_i32(&self) -> i32 { + self.len() as i32 + } + } + + impl $trait for &Vec<$ty> { + fn as_ptr(&self) -> *const $ty { + self.as_slice().as_ptr() + } + fn len_i32(&self) -> i32 { + self.len() as i32 + } + } + + impl $trait for &Box<[$ty]> { + fn as_ptr(&self) -> *const $ty { + (**self).as_ptr() + } + fn len_i32(&self) -> i32 { + self.len() as i32 + } + } + + impl $trait for [$ty] { + fn as_ptr(&self) -> *const $ty { + self.as_ptr() + } + fn len_i32(&self) -> i32 { + self.len() as i32 + } + } + + impl $trait for &[$ty; N] { + fn as_ptr(&self) -> *const $ty { + self.as_slice().as_ptr() + } + fn len_i32(&self) -> i32 { + self.len() as i32 + } + } + + impl $trait for [$ty; N] { + fn as_ptr(&self) -> *const $ty { + self.as_slice().as_ptr() + } + fn len_i32(&self) -> i32 { + self.len() as i32 + } + } + }; +} + +list_trait!(DoubleList, f64); +list_trait!(IntList, i64); + +pub trait IntListOption { + fn as_ptr(&self) -> *const i64; + fn len_i32(&self) -> i32; +} + +trait IntListOption_: IntList {} + +impl IntListOption for T { + fn as_ptr(&self) -> *const i64 { + self.as_ptr() + } + fn len_i32(&self) -> i32 { + self.len_i32() + } +} + +impl IntListOption for Option { + fn as_ptr(&self) -> *const i64 { + match self { + Some(v) => v.as_ptr(), + None => std::ptr::null(), + } + } + + fn len_i32(&self) -> i32 { + self.as_ref().map_or(-1, |v| v.len_i32()) + } +} + +impl IntListOption_ for i64 {} +impl IntListOption_ for [i64] {} +impl IntListOption_ for &[i64] {} +impl IntListOption_ for Vec {} +impl IntListOption_ for &Vec {} +impl IntListOption_ for &Box<[i64]> {} From feba1626b9f3be497d76a6affe18745b036d8502 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Wed, 21 Oct 2026 12:58:03 +0200 Subject: [PATCH 008/100] fix(server): host the existing MAT API without optional ML payload --- v2/Cargo.lock | 1 + v2/crates/wifi-densepose-mat/Cargo.toml | 5 +- .../wifi-densepose-mat/src/api/handlers.rs | 2 +- .../src/detection/pipeline.rs | 6 + .../wifi-densepose-sensing-server/Cargo.toml | 1 + .../wifi-densepose-sensing-server/src/lib.rs | 2 + .../wifi-densepose-sensing-server/src/main.rs | 3 + .../src/mat_surface.rs | 119 ++++++++++++++++++ 8 files changed, 136 insertions(+), 3 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/mat_surface.rs diff --git a/v2/Cargo.lock b/v2/Cargo.lock index ff7e381171..fe46229b1d 100644 --- a/v2/Cargo.lock +++ b/v2/Cargo.lock @@ -13652,6 +13652,7 @@ dependencies = [ "wifi-densepose-engine", "wifi-densepose-geo", "wifi-densepose-hardware", + "wifi-densepose-mat", "wifi-densepose-physics", "wifi-densepose-rufield", "wifi-densepose-signal", diff --git a/v2/crates/wifi-densepose-mat/Cargo.toml b/v2/crates/wifi-densepose-mat/Cargo.toml index 848770f32c..d4377c2e76 100644 --- a/v2/crates/wifi-densepose-mat/Cargo.toml +++ b/v2/crates/wifi-densepose-mat/Cargo.toml @@ -29,8 +29,9 @@ ml = ["dep:wifi-densepose-nn"] # `cfg_attr(feature = "serde", ...)` derives on domain types are actually # active when the API is on (review finding 5: `api = ["dep:serde"]` enabled # the dependency but left every `feature = "serde"` cfg dead). -# The REST surface exposes ML status (`ml_ready`), so `api` implies `ml`. -api = ["ml", "serde", "dep:axum", "dep:futures-util"] +# API-only hosts expose an honest unavailable ML status without bundling ONNX. +# The default feature set still enables ML for existing full MAT deployments. +api = ["serde", "dep:axum", "dep:futures-util"] # Real ESP32 serial CSI ingest. Pulls the native `serialport` crate (libudev on # Linux) only when enabled, so the default/no-default appliance build stays free # of native serial deps. With the feature OFF, the ESP32 serial *parser* still diff --git a/v2/crates/wifi-densepose-mat/src/api/handlers.rs b/v2/crates/wifi-densepose-mat/src/api/handlers.rs index 39f790ee36..2d7c8e3e99 100644 --- a/v2/crates/wifi-densepose-mat/src/api/handlers.rs +++ b/v2/crates/wifi-densepose-mat/src/api/handlers.rs @@ -1048,7 +1048,7 @@ pub async fn pipeline_status( Ok(Json(PipelineStatusResponse { scanning: state.is_scanning(), buffer_duration_secs: 0.0, - ml_enabled: config.enable_ml, + ml_enabled: cfg!(feature = "ml") && config.enable_ml, ml_ready: pipeline.ml_ready(), sample_rate: config.sample_rate, heartbeat_enabled: config.enable_heartbeat, diff --git a/v2/crates/wifi-densepose-mat/src/detection/pipeline.rs b/v2/crates/wifi-densepose-mat/src/detection/pipeline.rs index 0402a5b544..c7a07bd992 100644 --- a/v2/crates/wifi-densepose-mat/src/detection/pipeline.rs +++ b/v2/crates/wifi-densepose-mat/src/detection/pipeline.rs @@ -192,6 +192,12 @@ impl DetectionPipeline { self.ml_pipeline.as_ref().is_none_or(|ml| ml.is_ready()) } + /// API-only packages have no optional ML runtime to initialize. + #[cfg(not(feature = "ml"))] + pub fn ml_ready(&self) -> bool { + false + } + /// Process a scan zone and return detected vital signs. /// /// CSI data must be pushed into the pipeline via [`add_data`] before calling diff --git a/v2/crates/wifi-densepose-sensing-server/Cargo.toml b/v2/crates/wifi-densepose-sensing-server/Cargo.toml index ab2a191001..f91fb00e1c 100644 --- a/v2/crates/wifi-densepose-sensing-server/Cargo.toml +++ b/v2/crates/wifi-densepose-sensing-server/Cargo.toml @@ -54,6 +54,7 @@ clap = { workspace = true } wifi-densepose-aether = { version = "0.3.0", path = "../wifi-densepose-aether" } wifi-densepose-core = { workspace = true, features = ["serde"] } wifi-densepose-physics = { workspace = true } +wifi-densepose-mat = { version = "0.3.2", path = "../wifi-densepose-mat", default-features = false, features = ["std", "api"] } # Multi-BSSID WiFi scanning pipeline (ADR-022 Phase 3) wifi-densepose-wifiscan = { version = "0.3.0", path = "../wifi-densepose-wifiscan" } diff --git a/v2/crates/wifi-densepose-sensing-server/src/lib.rs b/v2/crates/wifi-densepose-sensing-server/src/lib.rs index 3e3a5460c1..ee8583d95e 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/lib.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/lib.rs @@ -24,6 +24,8 @@ pub mod http_bind; pub mod inference; pub mod introspection; pub mod matter; +/// Existing MAT incident API, hosted under the sensing server's security layers. +pub mod mat_surface; pub mod model_format; pub mod mqtt; pub mod path_safety; diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index ef1a4c363e..5d87edd294 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -15307,6 +15307,9 @@ async fn main() { // so a client on :8765 can stream signed RuField FieldEvents alongside // `/ws/sensing`. Merged with its own FieldState (different state type). .merge(rufield_surface::router(field_surface.clone())) + // The mobile app's available MAT workspace uses the existing incident + // API. Keep it inside the same privacy/auth/host and companion layers. + .merge(wifi_densepose_sensing_server::mat_surface::router()) .layer(axum::middleware::from_fn_with_state( privacy_filter::PrivacyFilter::new(args.privacy_mode), privacy_filter::redact_json_responses, diff --git a/v2/crates/wifi-densepose-sensing-server/src/mat_surface.rs b/v2/crates/wifi-densepose-sensing-server/src/mat_surface.rs new file mode 100644 index 0000000000..f731ec499f --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/mat_surface.rs @@ -0,0 +1,119 @@ +//! Host the existing MAT router without creating demo incidents or detections. +//! Incident state is in-memory; CSI ingest must be configured explicitly. + +/// Attach before host, bearer, companion and privacy middleware are applied. +pub fn router() -> axum::Router { + wifi_densepose_mat::api::create_router(wifi_densepose_mat::api::AppState::new()) +} + +#[cfg(test)] +mod tests { + use axum::{ + body::{to_bytes, Body}, + http::{Request, StatusCode}, + }; + use tower::ServiceExt; + + #[tokio::test] + async fn real_mat_router_starts_empty_and_reports_optional_ml_truthfully() { + let app = super::router(); + for path in [ + "/api/v1/mat/events?page=0&page_size=100", + "/api/v1/mat/scan/status", + ] { + let response = app + .clone() + .oneshot(Request::builder().uri(path).body(Body::empty()).unwrap()) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK, "{path}"); + let body = to_bytes(response.into_body(), 128 * 1024).await.unwrap(); + let value: serde_json::Value = serde_json::from_slice(&body).unwrap(); + if path.contains("events") { + assert_eq!(value["events"].as_array().unwrap().len(), 0); + assert_eq!(value["total"], 0); + } else { + assert_eq!(value["scanning"], false); + assert_eq!(value["ml_enabled"], false); + assert_eq!(value["ml_ready"], false); + } + } + } + + #[tokio::test] + async fn mat_reads_mutations_and_stream_remain_behind_host_auth_layers() { + let app = super::router().layer(axum::middleware::from_fn_with_state( + crate::bearer_auth::AuthState::from_token("mat-test-fixture-token"), + crate::bearer_auth::require_bearer, + )); + for (method, path) in [ + ("GET", "/api/v1/mat/events"), + ("POST", "/api/v1/mat/scan/control"), + ("GET", "/ws/mat/stream"), + ] { + let response = app + .clone() + .oneshot( + Request::builder() + .method(method) + .uri(path) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!( + response.status(), + StatusCode::UNAUTHORIZED, + "{method} {path}" + ); + } + let response = app + .oneshot( + Request::builder() + .uri("/api/v1/mat/scan/status") + .header("authorization", "Bearer mat-test-fixture-token") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + } + + #[tokio::test] + async fn real_incident_create_and_read_use_only_explicit_test_state() { + let app = super::router(); + let body = r#"{"event_type":"Earthquake","latitude":0,"longitude":0,"description":"EXPLICIT ISOLATED TEST FIXTURE"}"#; + let response = app + .clone() + .oneshot( + Request::builder() + .method("POST") + .uri("/api/v1/mat/events") + .header("content-type", "application/json") + .body(Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::CREATED); + let response = app + .oneshot( + Request::builder() + .uri("/api/v1/mat/events?page=0&page_size=100") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let bytes = to_bytes(response.into_body(), 128 * 1024).await.unwrap(); + let value: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(value["total"], 1); + assert_eq!(value["events"][0]["survivor_count"], 0); + assert_eq!( + value["events"][0]["description"], + "EXPLICIT ISOLATED TEST FIXTURE" + ); + } +} From 7cc1953991dcd3e7c73c282096f522075e76ab40 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Wed, 21 Oct 2026 13:03:18 +0200 Subject: [PATCH 009/100] fix(mat): share incident routes on the mobile HTTP origin --- scripts/mat_http_smoke.py | 35 +++++++++++++++++++ .../wifi-densepose-sensing-server/src/main.rs | 6 ++-- 2 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 scripts/mat_http_smoke.py diff --git a/scripts/mat_http_smoke.py b/scripts/mat_http_smoke.py new file mode 100644 index 0000000000..cc9d00eef8 --- /dev/null +++ b/scripts/mat_http_smoke.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +"""Read-only smoke against the compiled server's actual HTTP listener.""" +import argparse +import json +import os +import urllib.error +import urllib.request + +parser = argparse.ArgumentParser(description=__doc__) +parser.add_argument('--base-url', required=True) +args = parser.parse_args() +token = os.environ.get('RUVIEW_MAT_SMOKE_TOKEN') +if not token: + raise SystemExit('Set RUVIEW_MAT_SMOKE_TOKEN for this isolated server; never pass credentials in arguments.') + +def request(path, authenticated): + headers = {'Authorization': 'Bearer ' + token} if authenticated else {} + req = urllib.request.Request(args.base_url.rstrip('/') + path, headers=headers) + try: + with urllib.request.urlopen(req, timeout=5) as response: + return response.status, json.load(response) + except urllib.error.HTTPError as error: + return error.code, None + +for path in ['/api/v1/mat/events?page=0&page_size=100', '/api/v1/mat/scan/status', '/ws/mat/stream']: + status, _ = request(path, False) + assert status in (401, 403), (path, 'unauthenticated access was not rejected', status) +status, events = request('/api/v1/mat/events?page=0&page_size=100', True) +assert status == 200 and isinstance(events['events'], list), ('MAT HTTP event router', status) +assert events['page'] == 0 and events['page_size'] == 100 +status, pipeline = request('/api/v1/mat/scan/status', True) +assert status == 200 and isinstance(pipeline['scanning'], bool), ('MAT HTTP pipeline router', status) +print(json.dumps({'result': 'PASS', 'evidence': 'EXECUTED_COMPILED_HTTP', + 'eventCount': len(events['events']), 'mlEnabled': pipeline['ml_enabled'], + 'mlReady': pipeline['ml_ready'], 'unauthenticatedReadsAndStream': 'denied'})) diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index 5d87edd294..72b76899de 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -15307,9 +15307,6 @@ async fn main() { // so a client on :8765 can stream signed RuField FieldEvents alongside // `/ws/sensing`. Merged with its own FieldState (different state type). .merge(rufield_surface::router(field_surface.clone())) - // The mobile app's available MAT workspace uses the existing incident - // API. Keep it inside the same privacy/auth/host and companion layers. - .merge(wifi_densepose_sensing_server::mat_surface::router()) .layer(axum::middleware::from_fn_with_state( privacy_filter::PrivacyFilter::new(args.privacy_mode), privacy_filter::redact_json_responses, @@ -15503,6 +15500,9 @@ async fn main() { // Merged AFTER `.with_state` (so http_app is already `Router<()>` and // can absorb the field router's own `FieldState`). .merge(rufield_surface::router(field_surface.clone())) + // MAT REST and WebSocket share one incident state on the HTTP origin + // used by the app, inside privacy/auth/host and companion protection. + .merge(wifi_densepose_sensing_server::mat_surface::router()) // #2094: strip biometrics from every JSON response in privacy mode // (no-op otherwise). Applied after the merge so it covers every route. .layer(axum::middleware::from_fn_with_state( From f8e3085957400094ec7d792c9f043a372cbf4d3b Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Thu, 22 Oct 2026 10:28:38 +0200 Subject: [PATCH 010/100] refactor: separate workspace dependency remediation from companion PR --- .github/workflows/security-scan.yml | 3 - scripts/verify_zip_parent_patches.py | 23 - v2/Cargo.lock | 593 +- v2/Cargo.toml | 11 +- .../tests/zip9_tensor_roundtrip.rs | 14 - .../tests/zip9_tensor_roundtrip.rs | 15 - v2/dependency-patches/README.md | 21 - .../candle-core-0.4.1/Cargo.toml | 143 - .../candle-core-0.4.1/Cargo.toml.orig | 50 - .../candle-core-0.4.1/LICENSE | 201 - .../candle-core-0.4.1/README.md | 2 - .../candle-core-0.4.1/benches/bench_main.rs | 9 - .../benches/benchmarks/affine.rs | 43 - .../benches/benchmarks/matmul.rs | 44 - .../benches/benchmarks/mod.rs | 66 - .../benches/benchmarks/random.rs | 63 - .../benches/benchmarks/where_cond.rs | 64 - .../candle-core-0.4.1/examples/basics.rs | 17 - .../candle-core-0.4.1/examples/cuda_basics.rs | 36 - .../examples/cuda_sum_benchmark.rs | 54 - .../examples/tensor-tools.rs | 389 - .../candle-core-0.4.1/src/accelerate.rs | 476 - .../candle-core-0.4.1/src/backend.rs | 124 - .../candle-core-0.4.1/src/backprop.rs | 728 - .../candle-core-0.4.1/src/conv.rs | 358 - .../candle-core-0.4.1/src/convert.rs | 145 - .../candle-core-0.4.1/src/cpu/avx.rs | 148 - .../candle-core-0.4.1/src/cpu/erf.rs | 763 - .../candle-core-0.4.1/src/cpu/kernels.rs | 191 - .../candle-core-0.4.1/src/cpu/mod.rs | 180 - .../candle-core-0.4.1/src/cpu/neon.rs | 74 - .../candle-core-0.4.1/src/cpu/simd128.rs | 64 - .../candle-core-0.4.1/src/cpu_backend.rs | 2819 -- .../candle-core-0.4.1/src/cuda_backend.rs | 2249 - .../candle-core-0.4.1/src/cudnn.rs | 123 - .../candle-core-0.4.1/src/device.rs | 323 - .../candle-core-0.4.1/src/display.rs | 495 - .../candle-core-0.4.1/src/dtype.rs | 201 - .../src/dummy_cuda_backend.rs | 211 - .../src/dummy_metal_backend.rs | 223 - .../candle-core-0.4.1/src/error.rs | 265 - .../candle-core-0.4.1/src/indexer.rs | 168 - .../candle-core-0.4.1/src/layout.rs | 271 - .../candle-core-0.4.1/src/lib.rs | 151 - .../candle-core-0.4.1/src/metal_backend.rs | 1699 - .../candle-core-0.4.1/src/mkl.rs | 418 - .../candle-core-0.4.1/src/npy.rs | 450 - .../candle-core-0.4.1/src/op.rs | 1069 - .../candle-core-0.4.1/src/pickle.rs | 817 - .../candle-core-0.4.1/src/quantized/avx.rs | 667 - .../candle-core-0.4.1/src/quantized/cuda.rs | 321 - .../src/quantized/dummy_cuda.rs | 50 - .../src/quantized/dummy_metal.rs | 50 - .../src/quantized/ggml_file.rs | 265 - .../src/quantized/gguf_file.rs | 534 - .../src/quantized/k_quants.rs | 1965 - .../candle-core-0.4.1/src/quantized/metal.rs | 217 - .../candle-core-0.4.1/src/quantized/mod.rs | 491 - .../candle-core-0.4.1/src/quantized/neon.rs | 613 - .../src/quantized/simd128.rs | 419 - .../candle-core-0.4.1/src/quantized/utils.rs | 326 - .../candle-core-0.4.1/src/safetensors.rs | 441 - .../candle-core-0.4.1/src/scalar.rs | 23 - .../candle-core-0.4.1/src/shape.rs | 629 - .../candle-core-0.4.1/src/storage.rs | 700 - .../candle-core-0.4.1/src/strided_index.rs | 82 - .../candle-core-0.4.1/src/tensor.rs | 2765 -- .../candle-core-0.4.1/src/test_utils.rs | 62 - .../candle-core-0.4.1/src/utils.rs | 44 - .../candle-core-0.4.1/src/variable.rs | 147 - .../candle-core-0.4.1/tests/conv_tests.rs | 609 - .../tests/custom_op_tests.rs | 114 - .../candle-core-0.4.1/tests/display_tests.rs | 84 - .../tests/fortran_tensor_3d.pth | Bin 1486 -> 0 bytes .../candle-core-0.4.1/tests/grad_tests.rs | 531 - .../candle-core-0.4.1/tests/indexing_tests.rs | 122 - .../candle-core-0.4.1/tests/layout_tests.rs | 138 - .../candle-core-0.4.1/tests/npy.py | 9 - .../candle-core-0.4.1/tests/pool_tests.rs | 114 - .../candle-core-0.4.1/tests/pth.py | 37 - .../candle-core-0.4.1/tests/pth_tests.rs | 31 - .../tests/quantized_tests.rs | 1002 - .../tests/serialization_tests.rs | 24 - .../candle-core-0.4.1/tests/tensor_tests.rs | 1274 - .../candle-core-0.4.1/tests/test.npy | Bin 208 -> 0 bytes .../candle-core-0.4.1/tests/test.npz | Bin 668 -> 0 bytes .../candle-core-0.4.1/tests/test.pt | Bin 1165 -> 0 bytes .../candle-core-0.4.1/tests/test_with_key.pt | Bin 1338 -> 0 bytes .../candle-core-0.9.2/Cargo.toml | 262 - .../candle-core-0.9.2/Cargo.toml.orig | 70 - .../candle-core-0.9.2/LICENSE | 201 - .../candle-core-0.9.2/README.md | 2 - .../candle-core-0.9.2/benches/bench_main.rs | 17 - .../benches/benchmarks/affine.rs | 46 - .../benches/benchmarks/binary.rs | 57 - .../benches/benchmarks/broadcast.rs | 47 - .../benches/benchmarks/conv_transpose2d.rs | 60 - .../benches/benchmarks/copy.rs | 39 - .../benches/benchmarks/matmul.rs | 70 - .../benches/benchmarks/mod.rs | 77 - .../benches/benchmarks/qmatmul.rs | 73 - .../benches/benchmarks/random.rs | 64 - .../benches/benchmarks/reduce.rs | 159 - .../benches/benchmarks/unary.rs | 98 - .../benches/benchmarks/where_cond.rs | 65 - .../candle-core-0.9.2/examples/basics.rs | 20 - .../candle-core-0.9.2/examples/cuda_basics.rs | 23 - .../examples/cuda_sum_benchmark.rs | 54 - .../examples/metal_basics.rs | 28 - .../candle-core-0.9.2/src/accelerate.rs | 476 - .../candle-core-0.9.2/src/backend.rs | 174 - .../candle-core-0.9.2/src/backprop.rs | 784 - .../candle-core-0.9.2/src/conv.rs | 387 - .../candle-core-0.9.2/src/convert.rs | 166 - .../candle-core-0.9.2/src/cpu/avx.rs | 227 - .../candle-core-0.9.2/src/cpu/erf.rs | 325 - .../candle-core-0.9.2/src/cpu/kernels.rs | 232 - .../candle-core-0.9.2/src/cpu/mod.rs | 242 - .../candle-core-0.9.2/src/cpu/neon.rs | 74 - .../candle-core-0.9.2/src/cpu/simd128.rs | 64 - .../src/cpu_backend/conv2d.rs | 432 - .../candle-core-0.9.2/src/cpu_backend/mod.rs | 3284 -- .../src/cpu_backend/utils.rs | 409 - .../src/cuda_backend/cudnn.rs | 225 - .../src/cuda_backend/device.rs | 710 - .../src/cuda_backend/error.rs | 62 - .../candle-core-0.9.2/src/cuda_backend/mod.rs | 2682 -- .../src/cuda_backend/utils.rs | 194 - .../candle-core-0.9.2/src/custom_op.rs | 489 - .../candle-core-0.9.2/src/device.rs | 502 - .../candle-core-0.9.2/src/display.rs | 558 - .../candle-core-0.9.2/src/dtype.rs | 294 - .../src/dummy_cuda_backend.rs | 317 - .../candle-core-0.9.2/src/dummy_dtype.rs | 268 - .../src/dummy_metal_backend.rs | 280 - .../candle-core-0.9.2/src/error.rs | 394 - .../candle-core-0.9.2/src/indexer.rs | 257 - .../candle-core-0.9.2/src/layout.rs | 278 - .../candle-core-0.9.2/src/lib.rs | 177 - .../src/metal_backend/device.rs | 287 - .../src/metal_backend/mod.rs | 2137 - .../candle-core-0.9.2/src/mkl.rs | 418 - .../candle-core-0.9.2/src/npy.rs | 477 - .../candle-core-0.9.2/src/op.rs | 1148 - .../candle-core-0.9.2/src/pickle.rs | 841 - .../candle-core-0.9.2/src/quantized/avx.rs | 669 - .../candle-core-0.9.2/src/quantized/cuda.rs | 983 - .../src/quantized/dummy_cuda.rs | 95 - .../src/quantized/dummy_metal.rs | 87 - .../src/quantized/ggml_file.rs | 266 - .../src/quantized/gguf_file.rs | 539 - .../src/quantized/imatrix_file.rs | 85 - .../src/quantized/k_quants.rs | 2505 - .../candle-core-0.9.2/src/quantized/metal.rs | 391 - .../candle-core-0.9.2/src/quantized/mod.rs | 881 - .../candle-core-0.9.2/src/quantized/neon.rs | 615 - .../src/quantized/simd128.rs | 418 - .../candle-core-0.9.2/src/quantized/utils.rs | 562 - .../candle-core-0.9.2/src/safetensors.rs | 644 - .../candle-core-0.9.2/src/scalar.rs | 115 - .../candle-core-0.9.2/src/shape.rs | 634 - .../candle-core-0.9.2/src/sort.rs | 290 - .../candle-core-0.9.2/src/storage.rs | 849 - .../candle-core-0.9.2/src/streaming.rs | 208 - .../candle-core-0.9.2/src/strided_index.rs | 94 - .../candle-core-0.9.2/src/tensor.rs | 3116 -- .../candle-core-0.9.2/src/tensor_cat.rs | 303 - .../candle-core-0.9.2/src/test_utils.rs | 71 - .../candle-core-0.9.2/src/utils.rs | 45 - .../candle-core-0.9.2/src/variable.rs | 152 - .../candle-core-0.9.2/tests/bilinear_tests.rs | 525 - .../candle-core-0.9.2/tests/conv_tests.rs | 894 - .../tests/custom_op_tests.rs | 182 - .../candle-core-0.9.2/tests/display_tests.rs | 84 - .../tests/fortran_tensor_3d.pth | Bin 1486 -> 0 bytes .../candle-core-0.9.2/tests/grad_tests.rs | 563 - .../candle-core-0.9.2/tests/indexing_tests.rs | 122 - .../candle-core-0.9.2/tests/layout_tests.rs | 152 - .../candle-core-0.9.2/tests/matmul_tests.rs | 146 - .../candle-core-0.9.2/tests/npy.py | 9 - .../candle-core-0.9.2/tests/pool_tests.rs | 117 - .../candle-core-0.9.2/tests/pth.py | 37 - .../candle-core-0.9.2/tests/pth_tests.rs | 31 - .../tests/quantized_tests.rs | 1402 - .../tests/serialization_tests.rs | 71 - .../candle-core-0.9.2/tests/tensor_tests.rs | 2062 - .../candle-core-0.9.2/tests/test.npy | Bin 208 -> 0 bytes .../candle-core-0.9.2/tests/test.npz | Bin 668 -> 0 bytes .../candle-core-0.9.2/tests/test.pt | Bin 1165 -> 0 bytes .../candle-core-0.9.2/tests/test_with_key.pt | Bin 1338 -> 0 bytes v2/dependency-patches/compat-tests/Cargo.lock | 887 - v2/dependency-patches/compat-tests/Cargo.toml | 13 - v2/dependency-patches/compat-tests/src/lib.rs | 14 - v2/dependency-patches/compatibility.patch | 185 - v2/dependency-patches/patches.json | 1877 - .../tch-0.24.0/.cargo/config.toml | 2 - .../tch-0.24.0/.github/workflows/rust-ci.yml | 74 - v2/dependency-patches/tch-0.24.0/.gitignore | 19 - .../tch-0.24.0/.python-version | 1 - v2/dependency-patches/tch-0.24.0/CHANGELOG.md | 143 - v2/dependency-patches/tch-0.24.0/Cargo.toml | 246 - .../tch-0.24.0/Cargo.toml.orig | 67 - .../tch-0.24.0/LICENSE-APACHE | 201 - v2/dependency-patches/tch-0.24.0/LICENSE-MIT | 23 - v2/dependency-patches/tch-0.24.0/Makefile | 13 - v2/dependency-patches/tch-0.24.0/README.md | 372 - v2/dependency-patches/tch-0.24.0/build.rs | 13 - v2/dependency-patches/tch-0.24.0/clippy.toml | 1 - v2/dependency-patches/tch-0.24.0/dune-project | 1 - .../tch-0.24.0/examples/basics.rs | 36 - .../tch-0.24.0/examples/char-rnn/README.md | 43 - .../tch-0.24.0/examples/char-rnn/main.rs | 68 - .../tch-0.24.0/examples/cifar/main.rs | 72 - .../examples/custom-optimizer/README.md | 21 - .../examples/custom-optimizer/main.rs | 50 - .../examples/custom-optimizer/sparse_adam.rs | 147 - .../tch-0.24.0/examples/gan/main.rs | 148 - .../examples/jit-quantized/README.md | 40 - .../tch-0.24.0/examples/jit-quantized/main.rs | 51 - .../examples/jit-quantized/resnet.py | 22 - .../tch-0.24.0/examples/jit-trace/README.md | 27 - .../tch-0.24.0/examples/jit-trace/main.rs | 63 - .../tch-0.24.0/examples/jit-trace/test.py | 45 - .../tch-0.24.0/examples/jit-train/README.md | 136 - .../tch-0.24.0/examples/jit-train/main.rs | 57 - .../tch-0.24.0/examples/jit-train/resnet.py | 27 - .../tch-0.24.0/examples/jit/README.md | 116 - .../tch-0.24.0/examples/jit/main.rs | 27 - .../tch-0.24.0/examples/jit/resnet.py | 8 - .../examples/llama/convert_checkpoint.py | 63 - .../tch-0.24.0/examples/llama/main.rs | 397 - .../examples/llama/sentencepiece.rs | 149 - .../tch-0.24.0/examples/memory_test.rs | 19 - .../tch-0.24.0/examples/min-gpt/README.md | 78 - .../tch-0.24.0/examples/min-gpt/main.rs | 212 - .../tch-0.24.0/examples/mnist/README.md | 100 - .../tch-0.24.0/examples/mnist/main.rs | 25 - .../tch-0.24.0/examples/mnist/mnist_conv.rs | 54 - .../tch-0.24.0/examples/mnist/mnist_linear.rs | 42 - .../tch-0.24.0/examples/mnist/mnist_nn.rs | 34 - .../examples/neural-style-transfer/README.md | 162 - .../examples/neural-style-transfer/main.rs | 72 - .../examples/pretrained-models/main.rs | 81 - .../examples/reinforcement-learning/README.md | 36 - .../examples/reinforcement-learning/a2c.rs | 171 - .../reinforcement-learning/atari_wrappers.py | 308 - .../examples/reinforcement-learning/ddpg.rs | 343 - .../reinforcement-learning/gym_env.rs | 78 - .../examples/reinforcement-learning/main.rs | 20 - .../reinforcement-learning/policy_gradient.rs | 84 - .../examples/reinforcement-learning/ppo.rs | 181 - .../reinforcement-learning/vec_gym_env.rs | 66 - .../examples/stable-diffusion/README.md | 11 - .../examples/stable-diffusion/main.rs | 2512 - .../tch-0.24.0/examples/tensor-tools.rs | 79 - .../examples/transfer-learning/README.md | 141 - .../examples/transfer-learning/main.rs | 42 - .../examples/translation/dataset.rs | 115 - .../tch-0.24.0/examples/translation/lang.rs | 73 - .../tch-0.24.0/examples/translation/main.rs | 224 - .../tch-0.24.0/examples/vae/main.rs | 95 - .../tch-0.24.0/examples/yolo/coco_classes.rs | 82 - .../tch-0.24.0/examples/yolo/darknet.rs | 286 - .../tch-0.24.0/examples/yolo/main.rs | 153 - .../tch-0.24.0/examples/yolo/yolo-v3.cfg | 790 - .../tch-0.24.0/gen/.ocamlformat | 2 - v2/dependency-patches/tch-0.24.0/gen/dune | 3 - v2/dependency-patches/tch-0.24.0/gen/gen.ml | 892 - v2/dependency-patches/tch-0.24.0/gen/gen.mli | 1 - v2/dependency-patches/tch-0.24.0/main.py | 6 - .../tch-0.24.0/pyproject.toml | 9 - v2/dependency-patches/tch-0.24.0/rustfmt.toml | 2 - v2/dependency-patches/tch-0.24.0/src/data.rs | 203 - v2/dependency-patches/tch-0.24.0/src/error.rs | 74 - v2/dependency-patches/tch-0.24.0/src/lib.rs | 32 - .../tch-0.24.0/src/nn/batch_norm.rs | 132 - .../tch-0.24.0/src/nn/conv.rs | 242 - .../tch-0.24.0/src/nn/conv_transpose.rs | 189 - .../tch-0.24.0/src/nn/func.rs | 51 - .../tch-0.24.0/src/nn/group_norm.rs | 66 - .../tch-0.24.0/src/nn/init.rs | 212 - .../tch-0.24.0/src/nn/layer_norm.rs | 65 - .../tch-0.24.0/src/nn/linear.rs | 74 - .../tch-0.24.0/src/nn/mod.rs | 69 - .../tch-0.24.0/src/nn/module.rs | 67 - .../tch-0.24.0/src/nn/optimizer.rs | 324 - .../tch-0.24.0/src/nn/rnn.rs | 285 - .../tch-0.24.0/src/nn/sequential.rs | 149 - .../tch-0.24.0/src/nn/sparse.rs | 55 - .../tch-0.24.0/src/nn/var_store.rs | 935 - .../tch-0.24.0/src/tensor/convert.rs | 182 - .../tch-0.24.0/src/tensor/display.rs | 434 - .../tch-0.24.0/src/tensor/index.rs | 372 - .../tch-0.24.0/src/tensor/iter.rs | 62 - .../tch-0.24.0/src/tensor/mod.rs | 253 - .../tch-0.24.0/src/tensor/npy.rs | 277 - .../tch-0.24.0/src/tensor/ops.rs | 307 - .../tch-0.24.0/src/tensor/safetensors.rs | 183 - .../tch-0.24.0/src/vision/alexnet.rs | 44 - .../tch-0.24.0/src/vision/cifar.rs | 64 - .../tch-0.24.0/src/vision/convmixer.rs | 49 - .../tch-0.24.0/src/vision/dataset.rs | 95 - .../tch-0.24.0/src/vision/densenet.rs | 96 - .../tch-0.24.0/src/vision/dinov2.rs | 234 - .../tch-0.24.0/src/vision/efficientnet.rs | 329 - .../tch-0.24.0/src/vision/export_dinov2.py | 16 - .../tch-0.24.0/src/vision/export_model.py | 16 - .../tch-0.24.0/src/vision/image.rs | 146 - .../tch-0.24.0/src/vision/imagenet.rs | 1170 - .../tch-0.24.0/src/vision/inception.rs | 154 - .../tch-0.24.0/src/vision/mnist.rs | 69 - .../tch-0.24.0/src/vision/mobilenet.rs | 83 - .../tch-0.24.0/src/vision/mod.rs | 34 - .../tch-0.24.0/src/vision/resnet.rs | 183 - .../tch-0.24.0/src/vision/rust_image.rs | 145 - .../tch-0.24.0/src/vision/squeezenet.rs | 68 - .../tch-0.24.0/src/vision/vgg.rs | 100 - .../tch-0.24.0/src/wrappers/device.rs | 120 - .../tch-0.24.0/src/wrappers/image.rs | 35 - .../tch-0.24.0/src/wrappers/jit.rs | 879 - .../tch-0.24.0/src/wrappers/kind.rs | 176 - .../tch-0.24.0/src/wrappers/layout.rs | 29 - .../tch-0.24.0/src/wrappers/mod.rs | 21 - .../tch-0.24.0/src/wrappers/optimizer.rs | 129 - .../tch-0.24.0/src/wrappers/python.rs | 34 - .../tch-0.24.0/src/wrappers/scalar.rs | 111 - .../tch-0.24.0/src/wrappers/stream.rs | 26 - .../tch-0.24.0/src/wrappers/tensor.rs | 883 - .../src/wrappers/tensor_fallible_generated.rs | 38995 ---------------- .../src/wrappers/tensor_generated.rs | 19703 -------- .../tch-0.24.0/src/wrappers/utils.rs | 157 - .../tch-0.24.0/tests/autocast.rs | 22 - .../tch-0.24.0/tests/create_jit_models.py | 119 - .../tch-0.24.0/tests/data_tests.rs | 60 - .../tch-0.24.0/tests/device_tests.rs | 7 - .../tch-0.24.0/tests/display_tests.rs | 79 - v2/dependency-patches/tch-0.24.0/tests/foo.pt | Bin 2030 -> 0 bytes .../tch-0.24.0/tests/foo1.pt | Bin 1659 -> 0 bytes .../tch-0.24.0/tests/foo2.pt | Bin 1723 -> 0 bytes .../tch-0.24.0/tests/foo3.pt | Bin 1723 -> 0 bytes .../tch-0.24.0/tests/foo4.pt | Bin 1595 -> 0 bytes .../tch-0.24.0/tests/foo5.pt | Bin 1531 -> 0 bytes .../tch-0.24.0/tests/foo6.pt | Bin 1659 -> 0 bytes .../tch-0.24.0/tests/foo7.pt | Bin 1787 -> 0 bytes .../tch-0.24.0/tests/foo8.pt | Bin 1595 -> 0 bytes .../tch-0.24.0/tests/jit_tests.rs | 190 - .../tch-0.24.0/tests/linear/bs.npy | Bin 144 -> 0 bytes .../tch-0.24.0/tests/linear/gen-test-data.py | 16 - .../tch-0.24.0/tests/linear/in.npy | Bin 2184 -> 0 bytes .../tch-0.24.0/tests/linear/out.npy | Bin 160 -> 0 bytes .../tch-0.24.0/tests/linear/ws.npy | Bin 4240 -> 0 bytes .../tch-0.24.0/tests/nn_tests.rs | 412 - .../tch-0.24.0/tests/serialization_tests.rs | 195 - .../tch-0.24.0/tests/tensor_indexing.rs | 229 - .../tch-0.24.0/tests/tensor_tests.rs | 508 - .../tch-0.24.0/tests/test_utils.rs | 44 - .../tch-0.24.0/tests/var_store.rs | 634 - .../tch-0.24.0/tests/vision_tests.rs | 34 - v2/dependency-patches/tch-0.24.0/uv.lock | 433 - .../torch-sys-0.24.0/Cargo.toml | 79 - .../torch-sys-0.24.0/Cargo.toml.orig | 32 - .../torch-sys-0.24.0/build.rs | 493 - .../torch-sys-0.24.0/libtch/CMakeLists.txt | 9 - .../torch-sys-0.24.0/libtch/stb_image.h | 7530 --- .../libtch/stb_image_resize.h | 2627 -- .../torch-sys-0.24.0/libtch/stb_image_write.h | 1621 - .../torch-sys-0.24.0/libtch/torch_api.cpp | 1356 - .../torch-sys-0.24.0/libtch/torch_api.h | 295 - .../libtch/torch_api_generated.cpp | 19081 -------- .../libtch/torch_api_generated.h | 2648 -- .../torch-sys-0.24.0/libtch/torch_python.cpp | 22 - .../torch-sys-0.24.0/libtch/torch_python.h | 19 - .../torch-sys-0.24.0/src/c_generated.rs | 16055 ------- .../torch-sys-0.24.0/src/cuda.rs | 30 - .../torch-sys-0.24.0/src/io.rs | 111 - .../torch-sys-0.24.0/src/lib.rs | 336 - .../torch-sys-0.24.0/src/python.rs | 12 - .../torch-sys-0.24.0/src/traits.rs | 119 - 378 files changed, 240 insertions(+), 213334 deletions(-) delete mode 100644 scripts/verify_zip_parent_patches.py delete mode 100644 v2/crates/cog-person-count/tests/zip9_tensor_roundtrip.rs delete mode 100644 v2/crates/wifi-densepose-nn/tests/zip9_tensor_roundtrip.rs delete mode 100644 v2/dependency-patches/README.md delete mode 100644 v2/dependency-patches/candle-core-0.4.1/Cargo.toml delete mode 100644 v2/dependency-patches/candle-core-0.4.1/Cargo.toml.orig delete mode 100644 v2/dependency-patches/candle-core-0.4.1/LICENSE delete mode 100644 v2/dependency-patches/candle-core-0.4.1/README.md delete mode 100644 v2/dependency-patches/candle-core-0.4.1/benches/bench_main.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/affine.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/matmul.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/mod.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/random.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/where_cond.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/examples/basics.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/examples/cuda_basics.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/examples/cuda_sum_benchmark.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/examples/tensor-tools.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/accelerate.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/backend.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/backprop.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/conv.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/convert.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/cpu/avx.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/cpu/erf.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/cpu/kernels.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/cpu/mod.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/cpu/neon.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/cpu/simd128.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/cpu_backend.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/cuda_backend.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/cudnn.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/device.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/display.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/dtype.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/dummy_cuda_backend.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/dummy_metal_backend.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/error.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/indexer.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/layout.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/lib.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/metal_backend.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/mkl.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/npy.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/op.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/pickle.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/avx.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/cuda.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/dummy_cuda.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/dummy_metal.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/ggml_file.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/gguf_file.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/k_quants.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/metal.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/mod.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/neon.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/simd128.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/quantized/utils.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/safetensors.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/scalar.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/shape.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/storage.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/strided_index.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/tensor.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/test_utils.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/utils.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/src/variable.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/conv_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/custom_op_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/display_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/fortran_tensor_3d.pth delete mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/grad_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/indexing_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/layout_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/npy.py delete mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/pool_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/pth.py delete mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/pth_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/quantized_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/serialization_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/tensor_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/test.npy delete mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/test.npz delete mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/test.pt delete mode 100644 v2/dependency-patches/candle-core-0.4.1/tests/test_with_key.pt delete mode 100644 v2/dependency-patches/candle-core-0.9.2/Cargo.toml delete mode 100644 v2/dependency-patches/candle-core-0.9.2/Cargo.toml.orig delete mode 100644 v2/dependency-patches/candle-core-0.9.2/LICENSE delete mode 100644 v2/dependency-patches/candle-core-0.9.2/README.md delete mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/bench_main.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/affine.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/binary.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/broadcast.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/conv_transpose2d.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/copy.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/matmul.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/mod.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/qmatmul.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/random.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/reduce.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/unary.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/where_cond.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/examples/basics.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/examples/cuda_basics.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/examples/cuda_sum_benchmark.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/examples/metal_basics.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/accelerate.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/backend.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/backprop.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/conv.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/convert.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cpu/avx.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cpu/erf.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cpu/kernels.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cpu/mod.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cpu/neon.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cpu/simd128.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/conv2d.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/mod.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/utils.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/cudnn.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/device.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/error.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/mod.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/utils.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/custom_op.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/device.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/display.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/dtype.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/dummy_cuda_backend.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/dummy_dtype.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/dummy_metal_backend.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/error.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/indexer.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/layout.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/lib.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/metal_backend/device.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/metal_backend/mod.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/mkl.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/npy.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/op.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/pickle.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/avx.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/cuda.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/dummy_cuda.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/dummy_metal.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/ggml_file.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/gguf_file.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/imatrix_file.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/k_quants.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/metal.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/mod.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/neon.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/simd128.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/quantized/utils.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/safetensors.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/scalar.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/shape.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/sort.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/storage.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/streaming.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/strided_index.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/tensor.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/tensor_cat.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/test_utils.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/utils.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/src/variable.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/bilinear_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/conv_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/custom_op_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/display_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/fortran_tensor_3d.pth delete mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/grad_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/indexing_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/layout_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/matmul_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/npy.py delete mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/pool_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/pth.py delete mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/pth_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/quantized_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/serialization_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/tensor_tests.rs delete mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/test.npy delete mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/test.npz delete mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/test.pt delete mode 100644 v2/dependency-patches/candle-core-0.9.2/tests/test_with_key.pt delete mode 100644 v2/dependency-patches/compat-tests/Cargo.lock delete mode 100644 v2/dependency-patches/compat-tests/Cargo.toml delete mode 100644 v2/dependency-patches/compat-tests/src/lib.rs delete mode 100644 v2/dependency-patches/compatibility.patch delete mode 100644 v2/dependency-patches/patches.json delete mode 100644 v2/dependency-patches/tch-0.24.0/.cargo/config.toml delete mode 100644 v2/dependency-patches/tch-0.24.0/.github/workflows/rust-ci.yml delete mode 100644 v2/dependency-patches/tch-0.24.0/.gitignore delete mode 100644 v2/dependency-patches/tch-0.24.0/.python-version delete mode 100644 v2/dependency-patches/tch-0.24.0/CHANGELOG.md delete mode 100644 v2/dependency-patches/tch-0.24.0/Cargo.toml delete mode 100644 v2/dependency-patches/tch-0.24.0/Cargo.toml.orig delete mode 100644 v2/dependency-patches/tch-0.24.0/LICENSE-APACHE delete mode 100644 v2/dependency-patches/tch-0.24.0/LICENSE-MIT delete mode 100644 v2/dependency-patches/tch-0.24.0/Makefile delete mode 100644 v2/dependency-patches/tch-0.24.0/README.md delete mode 100644 v2/dependency-patches/tch-0.24.0/build.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/clippy.toml delete mode 100644 v2/dependency-patches/tch-0.24.0/dune-project delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/basics.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/char-rnn/README.md delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/char-rnn/main.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/cifar/main.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/README.md delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/main.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/sparse_adam.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/gan/main.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit-quantized/README.md delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit-quantized/main.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit-quantized/resnet.py delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit-trace/README.md delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit-trace/main.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit-trace/test.py delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit-train/README.md delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit-train/main.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit-train/resnet.py delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit/README.md delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit/main.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/jit/resnet.py delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/llama/convert_checkpoint.py delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/llama/main.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/llama/sentencepiece.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/memory_test.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/min-gpt/README.md delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/min-gpt/main.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/mnist/README.md delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/mnist/main.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_conv.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_linear.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_nn.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/neural-style-transfer/README.md delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/neural-style-transfer/main.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/pretrained-models/main.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/README.md delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/a2c.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/atari_wrappers.py delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/ddpg.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/gym_env.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/main.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/policy_gradient.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/ppo.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/vec_gym_env.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/stable-diffusion/README.md delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/stable-diffusion/main.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/tensor-tools.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/transfer-learning/README.md delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/transfer-learning/main.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/translation/dataset.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/translation/lang.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/translation/main.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/vae/main.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/yolo/coco_classes.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/yolo/darknet.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/yolo/main.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/examples/yolo/yolo-v3.cfg delete mode 100644 v2/dependency-patches/tch-0.24.0/gen/.ocamlformat delete mode 100644 v2/dependency-patches/tch-0.24.0/gen/dune delete mode 100644 v2/dependency-patches/tch-0.24.0/gen/gen.ml delete mode 100644 v2/dependency-patches/tch-0.24.0/gen/gen.mli delete mode 100644 v2/dependency-patches/tch-0.24.0/main.py delete mode 100644 v2/dependency-patches/tch-0.24.0/pyproject.toml delete mode 100644 v2/dependency-patches/tch-0.24.0/rustfmt.toml delete mode 100644 v2/dependency-patches/tch-0.24.0/src/data.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/error.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/lib.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/batch_norm.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/conv.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/conv_transpose.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/func.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/group_norm.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/init.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/layer_norm.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/linear.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/mod.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/module.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/optimizer.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/rnn.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/sequential.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/sparse.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/nn/var_store.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/tensor/convert.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/tensor/display.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/tensor/index.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/tensor/iter.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/tensor/mod.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/tensor/npy.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/tensor/ops.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/tensor/safetensors.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/alexnet.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/cifar.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/convmixer.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/dataset.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/densenet.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/dinov2.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/efficientnet.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/export_dinov2.py delete mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/export_model.py delete mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/image.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/imagenet.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/inception.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/mnist.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/mobilenet.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/mod.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/resnet.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/rust_image.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/squeezenet.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/vision/vgg.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/device.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/image.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/jit.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/kind.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/layout.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/mod.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/optimizer.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/python.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/scalar.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/stream.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/tensor.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/tensor_fallible_generated.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/tensor_generated.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/src/wrappers/utils.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/autocast.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/create_jit_models.py delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/data_tests.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/device_tests.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/display_tests.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/foo.pt delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/foo1.pt delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/foo2.pt delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/foo3.pt delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/foo4.pt delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/foo5.pt delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/foo6.pt delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/foo7.pt delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/foo8.pt delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/jit_tests.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/linear/bs.npy delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/linear/gen-test-data.py delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/linear/in.npy delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/linear/out.npy delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/linear/ws.npy delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/nn_tests.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/serialization_tests.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/tensor_indexing.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/tensor_tests.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/test_utils.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/var_store.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/tests/vision_tests.rs delete mode 100644 v2/dependency-patches/tch-0.24.0/uv.lock delete mode 100644 v2/dependency-patches/torch-sys-0.24.0/Cargo.toml delete mode 100644 v2/dependency-patches/torch-sys-0.24.0/Cargo.toml.orig delete mode 100644 v2/dependency-patches/torch-sys-0.24.0/build.rs delete mode 100644 v2/dependency-patches/torch-sys-0.24.0/libtch/CMakeLists.txt delete mode 100644 v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image.h delete mode 100644 v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image_resize.h delete mode 100644 v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image_write.h delete mode 100644 v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api.cpp delete mode 100644 v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api.h delete mode 100644 v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api_generated.cpp delete mode 100644 v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api_generated.h delete mode 100644 v2/dependency-patches/torch-sys-0.24.0/libtch/torch_python.cpp delete mode 100644 v2/dependency-patches/torch-sys-0.24.0/libtch/torch_python.h delete mode 100644 v2/dependency-patches/torch-sys-0.24.0/src/c_generated.rs delete mode 100644 v2/dependency-patches/torch-sys-0.24.0/src/cuda.rs delete mode 100644 v2/dependency-patches/torch-sys-0.24.0/src/io.rs delete mode 100644 v2/dependency-patches/torch-sys-0.24.0/src/lib.rs delete mode 100644 v2/dependency-patches/torch-sys-0.24.0/src/python.rs delete mode 100644 v2/dependency-patches/torch-sys-0.24.0/src/traits.rs diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index b6c73e7e31..68375748f2 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -25,9 +25,6 @@ jobs: - name: Checkout code uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 - - name: Verify ZIP parent patch integrity - run: python3 scripts/verify_zip_parent_patches.py - - name: Install cargo-audit run: cargo install cargo-audit --locked --version 0.22.2 diff --git a/scripts/verify_zip_parent_patches.py b/scripts/verify_zip_parent_patches.py deleted file mode 100644 index 7c40015a34..0000000000 --- a/scripts/verify_zip_parent_patches.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env python3 -"""Verify checked-in ZIP parent patch provenance/integrity; no network or writes.""" -import hashlib -import json -from pathlib import Path - -root = Path(__file__).resolve().parent.parent / "v2" / "dependency-patches" -manifest = json.loads((root / "patches.json").read_text()) -for package in manifest["packages"]: - directory = root / package["package"] - expected = {entry["path"]: entry for entry in package["files"]} - actual = {str(p.relative_to(directory)) for p in directory.rglob("*") if p.is_file()} - assert actual == set(expected), "Patched package inventory changed" - changed = [] - for name, entry in expected.items(): - path = directory / name - assert not path.is_symlink() and directory.resolve() in path.resolve().parents - digest = hashlib.sha256(path.read_bytes()).hexdigest() - assert digest == entry["sha256"], "Patched package digest changed: " + name - if digest != entry["upstreamSha256"]: - changed.append(name) - assert sorted(changed) == sorted(package["changedFiles"]) -print("ZIP parent patch inventory and SHA256: PASS") diff --git a/v2/Cargo.lock b/v2/Cargo.lock index fe46229b1d..821e45e2c3 100644 --- a/v2/Cargo.lock +++ b/v2/Cargo.lock @@ -29,13 +29,13 @@ dependencies = [ [[package]] name = "aes" -version = "0.9.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35f0f96ce78e38c3dc6d8948aa8163d06385be74000f3c7a95bf1eef35d3ea32" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ - "cipher 0.5.2", - "cpubits", - "cpufeatures 0.3.0", + "cfg-if", + "cipher", + "cpufeatures 0.2.17", ] [[package]] @@ -234,6 +234,9 @@ name = "arbitrary" version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] [[package]] name = "arrayref" @@ -425,7 +428,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sha1 0.10.6", + "sha1", "sync_wrapper", "tokio", "tokio-tungstenite", @@ -627,7 +630,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2f6c7dbe95a6ed67ad9f18e57daf93a2f034c524b99fd2b76d18fdfeb6660aa" dependencies = [ "hybrid-array", - "zeroize", ] [[package]] @@ -1053,11 +1055,22 @@ dependencies = [ [[package]] name = "bzip2" -version = "0.6.1" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" dependencies = [ - "libbz2-rs-sys", + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", ] [[package]] @@ -1097,6 +1110,8 @@ dependencies = [ [[package]] name = "candle-core" version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f1b20174c1707e20f4cb364a355b449803c03e9b0c9193324623cf9787a4e00" dependencies = [ "byteorder", "gemm 0.17.1", @@ -1110,12 +1125,14 @@ dependencies = [ "safetensors 0.4.5", "thiserror 1.0.69", "yoke 0.7.5", - "zip", + "zip 0.6.6", ] [[package]] name = "candle-core" version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c15b675b80d994b2eadb20a4bbe434eabeb454eac3ee5e2b4cf6f147ee9be091" dependencies = [ "byteorder", "candle-kernels", @@ -1134,7 +1151,7 @@ dependencies = [ "safetensors 0.7.0", "thiserror 2.0.18", "yoke 0.8.1", - "zip", + "zip 7.2.0", ] [[package]] @@ -1305,7 +1322,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ "cfg-if", - "cipher 0.4.4", + "cipher", "cpufeatures 0.2.17", ] @@ -1328,7 +1345,7 @@ checksum = "10cd79432192d1c0f4e1a0fef9527696cc039165d729fb41b3f4f4f354c2dc35" dependencies = [ "aead", "chacha20 0.9.1", - "cipher 0.4.4", + "cipher", "poly1305", "zeroize", ] @@ -1381,20 +1398,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ "crypto-common 0.1.7", - "inout 0.1.4", + "inout", "zeroize", ] -[[package]] -name = "cipher" -version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8cf2a2c93cd704877c0858356ed03480ff301ee950b43f1cbe4573b088bfa6c" -dependencies = [ - "crypto-common 0.2.2", - "inout 0.2.5", -] - [[package]] name = "clap" version = "4.5.60" @@ -1537,7 +1544,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" dependencies = [ "lazy_static", - "windows-sys 0.48.0", + "windows-sys 0.59.0", ] [[package]] @@ -1546,7 +1553,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -1654,18 +1661,15 @@ dependencies = [ [[package]] name = "constant_time_eq" -version = "0.4.2" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" [[package]] name = "constant_time_eq" -version = "0.6.1" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9eb6feb175df65d0a3ccd3e73484e940d0ac664e8034f0519fd49a93db3071d9" -dependencies = [ - "rustversion", -] +checksum = "3d52eff69cd5e647efe296129160853a42795992097e8af39800e1060caeea9b" [[package]] name = "convert_case" @@ -1787,36 +1791,36 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64" -version = "0.123.19" +version = "0.123.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3741b9ad77e89c710586b2fcdd619baf28482d92b70d88cecb2a081dd912ae6" +checksum = "e4353b283b6c76205abc8eab20b41838e47510c9669ed83c27f7bca8cd1c262f" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.123.19" +version = "0.123.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8243e149085051b23faebd352ef7031aeebc5f45e63aaefd63ee7f810c22c7df" +checksum = "61f6d6eb342f6c48d97278c7549c5b91756f03db80bce9cdeddaa536c356c937" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.123.19" +version = "0.123.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c538af3aee13eb6aa8a85d2ef85b4d0aa41623dae8b19f59bbacfd5d6c114920" +checksum = "c746ef625773869efaffda6918a2b73f973c73e419e1de20833f3a1a9d614006" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-bitset" -version = "0.123.19" +version = "0.123.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2292ea3964d7d751031da1a1261d64feb09e89bd4fbebf3c78f43f9c34c20a11" +checksum = "5447cd11203bcb5b71bb91e1e8b8470f350870ca837847bf82ec2e25f55fcd7b" dependencies = [ "serde", "serde_derive", @@ -1824,9 +1828,9 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.123.19" +version = "0.123.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8967a699a4982d02c32246a00f2aa4cee5bccfaaa042608d5e151cb99cba210e" +checksum = "2ac26ac02abded3bb8223f0cf8a0055c6284be55ca9a6c380f78494bdf686adc" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -1851,9 +1855,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.123.19" +version = "0.123.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e258eeeb6903b4e91e314a3667ab2a1e2924738ef2df08bdb50355e38decc749" +checksum = "3c4a9197c03f7b75c228a66ecaf0b7469e0f9b86fe3cf2cf7ebfbcaacfce0a9b" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -1864,24 +1868,24 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.123.19" +version = "0.123.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d36ff0de30460c01dad152ca2097bcabef2f16c659b9ffbca2f67e9bbdb321" +checksum = "3279a717ec8773811c148650b77cb7c554c6898c858cb547d3da477a8bca21b3" [[package]] name = "cranelift-control" -version = "0.123.19" +version = "0.123.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acc880d91877efb6199031bda872726ca3287dc1e18ccf3f61c07bfdab62930b" +checksum = "2e2a7ad0219d6f4a4899e5e93d5a4944a79dd6cffe2f6d6fba4d6b2ac165a154" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.123.19" +version = "0.123.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f7af321c4b6e5be4729e4a5b1ed0fc78fc31695c1ecd3c9b45b91908bf3d0a3" +checksum = "554aaa52367f0396570d031aa0bdedfff4f63794022dc418d0cdcf8f6d321b60" dependencies = [ "cranelift-bitset", "serde", @@ -1890,9 +1894,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.123.19" +version = "0.123.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3463f1b26faac3491008cede8e9cdf7ceb83139143bd92ba86c14b0882aaf1bb" +checksum = "72db97f8a7ea1626ecfcac1adf664cba1f1d68448abbb36b7dbaa0a6b78bc11f" dependencies = [ "cranelift-codegen", "log", @@ -1902,15 +1906,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.123.19" +version = "0.123.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90b76554b329634c4241c5670a96058affca99ac3b33daa14fb221bd68c0038e" +checksum = "17a237c16305f95ff07031be4a5226a4c0753761f40bbad2105c758a33aee692" [[package]] name = "cranelift-native" -version = "0.123.19" +version = "0.123.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36e7c67bcfd30a8160187a6d52b2ec0cbe45698a9d61f12d1292bd24701f7ed2" +checksum = "dd6f62b06973b3e5fd813167b01958b09f634bfa59a8f5db2cd49b930d93aa6f" dependencies = [ "cranelift-codegen", "libc", @@ -1919,9 +1923,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.123.19" +version = "0.123.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ccca83c45ddfe0412b1b2e4fe509d28d1bdbc014e306574a848519741bebc36" +checksum = "fd76efe2edc25d9f062aaf0a16c7528a30c8ba5782afe6296237ed48d8d6127c" [[package]] name = "crc" @@ -2739,12 +2743,6 @@ dependencies = [ "uuid", ] -[[package]] -name = "deflate64" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2" - [[package]] name = "der" version = "0.7.10" @@ -2777,6 +2775,17 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "derive_more" version = "0.99.20" @@ -2840,8 +2849,6 @@ dependencies = [ "block-buffer 0.12.1", "const-oid 0.10.2", "crypto-common 0.2.2", - "ctutils", - "zeroize", ] [[package]] @@ -3326,7 +3333,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3357,13 +3364,13 @@ checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" [[package]] name = "fastbloom" -version = "0.17.0" +version = "0.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef975e30683b2d965054bb0a836f8973857c4ebf6acf274fe46617cd285060d8" +checksum = "4e7f34442dbe69c60fe8eaf58a8cafff81a1f278816d8ab4db255b3bef4ac3c4" dependencies = [ - "foldhash 0.2.0", + "getrandom 0.3.4", "libm", - "portable-atomic", + "rand 0.9.2", "siphasher 1.0.2", ] @@ -4297,9 +4304,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "wasip2", + "wasm-bindgen", ] [[package]] @@ -4309,13 +4318,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi", "rand_core 0.10.1", "wasip2", "wasip3", - "wasm-bindgen", ] [[package]] @@ -4892,7 +4899,7 @@ version = "0.12.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b5f8eb2ad728638ea2c7d47a21db23b7b58a72ed6a38256b8a1849f15fbbdf7" dependencies = [ - "hmac 0.12.1", + "hmac", ] [[package]] @@ -4904,15 +4911,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "hmac" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6303bc9732ae41b04cb554b844a762b4115a61bfaa81e3e83050991eeb56863f" -dependencies = [ - "digest 0.11.3", -] - [[package]] name = "hmac-sha256" version = "1.1.14" @@ -5233,7 +5231,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.10", + "webpki-roots 1.0.7", ] [[package]] @@ -5508,15 +5506,6 @@ dependencies = [ "generic-array 0.14.7", ] -[[package]] -name = "inout" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b5cf4372d022092674a83b8febf5865f7a573cc0c5baec5e49c879b6a20978f" -dependencies = [ - "hybrid-array", -] - [[package]] name = "io-kit-sys" version = "0.4.1" @@ -5560,7 +5549,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5688,68 +5677,19 @@ dependencies = [ "cesu8", "cfg-if", "combine", - "jni-sys 0.3.0", + "jni-sys", "log", "thiserror 1.0.69", "walkdir", "windows-sys 0.45.0", ] -[[package]] -name = "jni" -version = "0.22.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" -dependencies = [ - "cfg-if", - "combine", - "jni-macros", - "jni-sys 0.4.1", - "log", - "simd_cesu8", - "thiserror 2.0.18", - "walkdir", - "windows-link 0.2.1", -] - -[[package]] -name = "jni-macros" -version = "0.22.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" -dependencies = [ - "proc-macro2", - "quote", - "rustc_version", - "simd_cesu8", - "syn 2.0.117", -] - [[package]] name = "jni-sys" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" -[[package]] -name = "jni-sys" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" -dependencies = [ - "jni-sys-macros", -] - -[[package]] -name = "jni-sys-macros" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" -dependencies = [ - "quote", - "syn 2.0.117", -] - [[package]] name = "jobserver" version = "0.1.34" @@ -5917,17 +5857,11 @@ dependencies = [ "once_cell", ] -[[package]] -name = "libbz2-rs-sys" -version = "0.2.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34b357333733e8260735ba5894eb928c02ecc69c78715f01a8019e7fa7f2db4c" - [[package]] name = "libc" -version = "0.2.191" +version = "0.2.182" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" +checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" [[package]] name = "libloading" @@ -6062,15 +5996,6 @@ version = "0.15.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1670343e58806300d87950e3401e820b519b9384281bbabfb15e3636689ffd69" -[[package]] -name = "lzma-rust2" -version = "0.22.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7277c9742229dc25243236d578b6471a798c554512c6f6a04365ec2e87c2a3bb" -dependencies = [ - "sha2 0.11.0", -] - [[package]] name = "mac" version = "0.1.1" @@ -6253,7 +6178,7 @@ dependencies = [ "js-sys", "quinn", "rcgen", - "rustls-platform-verifier 0.6.2", + "rustls-platform-verifier", "serde", "thiserror 2.0.18", "tokio", @@ -6674,6 +6599,7 @@ dependencies = [ "num-complex", "num-traits", "py_literal", + "zip 6.0.0", ] [[package]] @@ -6683,7 +6609,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3f42e7bbe13d351b6bead8286a43aac9534b82bd3cc43e47037f012ebfd62d4" dependencies = [ "bitflags 2.11.0", - "jni-sys 0.3.0", + "jni-sys", "log", "ndk-sys", "num_enum", @@ -6703,7 +6629,7 @@ version = "0.6.0+11769913" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee6cda3051665f1fb8d9e08fc35c96d5a244fb1be711a03b71118828afc9a873" dependencies = [ - "jni-sys 0.3.0", + "jni-sys", ] [[package]] @@ -6747,18 +6673,6 @@ dependencies = [ "libc", ] -[[package]] -name = "nix" -version = "0.31.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" -dependencies = [ - "bitflags 2.11.0", - "cfg-if", - "cfg_aliases", - "libc", -] - [[package]] name = "nodrop" version = "0.1.14" @@ -7347,7 +7261,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06503bb33f294c5f1ba484011e053bfa6ae227074bdb841e9863492dc5960d4b" dependencies = [ "hmac-sha256", - "lzma-rust2 0.15.7", + "lzma-rust2", "ureq 3.2.0", ] @@ -7358,7 +7272,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.45.0", + "windows-sys 0.61.2", ] [[package]] @@ -7469,6 +7383,17 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "password-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +dependencies = [ + "base64ct", + "rand_core 0.6.4", + "subtle", +] + [[package]] name = "paste" version = "1.0.15" @@ -7483,12 +7408,14 @@ checksum = "df94ce210e5bc13cb6651479fa48d14f601d9858cfe0467f43ae157023b938d3" [[package]] name = "pbkdf2" -version = "0.13.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "112d82ceb8c5bf524d9af484d4e4970c9fd5a0cc15ba14ad93dccd28873b0629" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" dependencies = [ - "digest 0.11.3", - "hmac 0.13.0", + "digest 0.10.7", + "hmac", + "password-hash", + "sha2 0.10.9", ] [[package]] @@ -7901,12 +7828,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" -[[package]] -name = "ppmd-rust" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "494b4379484fc92217cc7d9e3a532375b1d01a61889ec8333025dc3b0377af56" - [[package]] name = "ppv-lite86" version = "0.2.21" @@ -8130,7 +8051,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.117", @@ -8167,9 +8088,9 @@ dependencies = [ [[package]] name = "pulley-interpreter" -version = "36.0.19" +version = "36.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c956c7140270b72e3bf03381a51f2eb79c70fd6881fc17cd027fa7cd3f56ae5a" +checksum = "f09f3fdea932cc327d051b2aaccfc82fddcd49997e8579f3d91ae3468a954b94" dependencies = [ "cranelift-bitset", "log", @@ -8179,9 +8100,9 @@ dependencies = [ [[package]] name = "pulley-macros" -version = "36.0.19" +version = "36.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74db420d7ce2bb03eed5b81c1f778bda5771b3fd818e2b496dfc3a2783ceb60e" +checksum = "96da696c8995a3033eb1287ee418895592da41a5d549b8615b5f1e5851318b48" dependencies = [ "proc-macro2", "quote", @@ -8287,21 +8208,20 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.18" +version = "0.11.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc" +checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" dependencies = [ "bytes", "fastbloom", - "getrandom 0.4.1", + "getrandom 0.3.4", "lru-slab", - "rand 0.10.2", - "rand_pcg 0.10.2", + "rand 0.9.2", "ring", "rustc-hash 2.1.1", "rustls", "rustls-pki-types", - "rustls-platform-verifier 0.7.1", + "rustls-platform-verifier", "slab", "thiserror 2.0.18", "tinyvec", @@ -8320,7 +8240,7 @@ dependencies = [ "once_cell", "socket2 0.6.2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -8358,7 +8278,7 @@ dependencies = [ "rand_chacha 0.2.2", "rand_core 0.5.1", "rand_hc", - "rand_pcg 0.2.1", + "rand_pcg", ] [[package]] @@ -8504,15 +8424,6 @@ dependencies = [ "rand_core 0.5.1", ] -[[package]] -name = "rand_pcg" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" -dependencies = [ - "rand_core 0.10.1", -] - [[package]] name = "rand_xorshift" version = "0.4.0" @@ -8806,7 +8717,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 1.0.10", + "webpki-roots 1.0.7", ] [[package]] @@ -8849,7 +8760,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" dependencies = [ - "hmac 0.12.1", + "hmac", "subtle", ] @@ -9193,14 +9104,14 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "rustls" -version = "0.23.46" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "log", "once_cell", @@ -9241,38 +9152,17 @@ checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" dependencies = [ "core-foundation 0.10.1", "core-foundation-sys", - "jni 0.21.1", - "log", - "once_cell", - "rustls", - "rustls-native-certs", - "rustls-platform-verifier-android 0.1.1", - "rustls-webpki", - "security-framework", - "security-framework-sys", - "webpki-root-certs", - "windows-sys 0.59.0", -] - -[[package]] -name = "rustls-platform-verifier" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" -dependencies = [ - "core-foundation 0.10.1", - "core-foundation-sys", - "jni 0.22.4", + "jni", "log", "once_cell", "rustls", "rustls-native-certs", - "rustls-platform-verifier-android 0.2.0", + "rustls-platform-verifier-android", "rustls-webpki", "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -9281,17 +9171,11 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" -[[package]] -name = "rustls-platform-verifier-android" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" - [[package]] name = "rustls-webpki" -version = "0.103.16" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "ring", "rustls-pki-types", @@ -9716,7 +9600,7 @@ dependencies = [ "candle-core 0.9.2", "candle-nn 0.9.2", "criterion", - "hmac 0.12.1", + "hmac", "nalgebra 0.33.2", "ort", "rand 0.8.5", @@ -10242,17 +10126,6 @@ dependencies = [ "digest 0.10.7", ] -[[package]] -name = "sha1" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" -dependencies = [ - "cfg-if", - "cpufeatures 0.3.0", - "digest 0.11.3", -] - [[package]] name = "sha2" version = "0.10.9" @@ -10374,16 +10247,6 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" -[[package]] -name = "simd_cesu8" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" -dependencies = [ - "rustc_version", - "simdutf8", -] - [[package]] name = "simdutf8" version = "0.1.5" @@ -10905,7 +10768,7 @@ dependencies = [ "gdkwayland-sys", "gdkx11-sys", "gtk", - "jni 0.21.1", + "jni", "libc", "log", "ndk", @@ -10977,7 +10840,7 @@ dependencies = [ "gtk", "heck 0.5.0", "http", - "jni 0.21.1", + "jni", "libc", "log", "mime", @@ -11164,7 +11027,7 @@ dependencies = [ "dpi", "gtk", "http", - "jni 0.21.1", + "jni", "objc2", "objc2-ui-kit", "objc2-web-kit", @@ -11187,7 +11050,7 @@ checksum = "e11ea2e6f801d275fdd890d6c9603736012742a1c33b96d0db788c9cdebf7f9e" dependencies = [ "gtk", "http", - "jni 0.21.1", + "jni", "log", "objc2", "objc2-app-kit", @@ -11257,6 +11120,8 @@ dependencies = [ [[package]] name = "tch" version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d3f84a069d8ba16dbf720b61e8bf131d90ffb8e958a664eae8e4993c5c2fa6f" dependencies = [ "half", "lazy_static", @@ -11266,7 +11131,7 @@ dependencies = [ "safetensors 0.3.3", "thiserror 1.0.69", "torch-sys", - "zip", + "zip 0.6.6", ] [[package]] @@ -11279,7 +11144,7 @@ dependencies = [ "getrandom 0.4.1", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -11386,7 +11251,6 @@ checksum = "743bd48c283afc0388f9b8827b976905fb217ad9e647fae3a379a9283c4def2c" dependencies = [ "deranged", "itoa", - "js-sys", "num-conv", "powerfmt", "serde_core", @@ -11754,11 +11618,13 @@ dependencies = [ [[package]] name = "torch-sys" version = "0.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4ba78777379cf09aaa79708c63e477cf0f95e021d04360c6821f1a9f56173f7" dependencies = [ "anyhow", "cc", "libc", - "zip", + "zip 0.6.6", ] [[package]] @@ -11981,7 +11847,7 @@ dependencies = [ "httparse", "log", "rand 0.8.5", - "sha1 0.10.6", + "sha1", "thiserror 1.0.69", "utf-8", ] @@ -12000,7 +11866,7 @@ dependencies = [ "rand 0.9.2", "rustls", "rustls-pki-types", - "sha1 0.10.6", + "sha1", "thiserror 2.0.18", ] @@ -12644,9 +12510,9 @@ dependencies = [ [[package]] name = "wasmtime" -version = "36.0.19" +version = "36.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60e5f244a2c9d40afa9e9a607ee7f048c0648bb1c22cc28b130c5fc71e23d71d" +checksum = "1edc1374519a6d4c3eb99e0598f2ce069e6feb63de65bed7d0240a5f84b58c3b" dependencies = [ "addr2line", "anyhow", @@ -12699,9 +12565,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "36.0.19" +version = "36.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e246cd5107aa406cfa4acf6942e864bd189ae8136bab853365147da131c9ca3" +checksum = "c7e05a9ebefb914d4df7928652e27aa1e89e1ca23c5ab73c647316783f3bcbc2" dependencies = [ "anyhow", "cpp_demangle", @@ -12726,18 +12592,18 @@ dependencies = [ [[package]] name = "wasmtime-internal-asm-macros" -version = "36.0.19" +version = "36.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5380a8106fea8bfedb7208bb11ced9e7488ee44394ca889ede73acd3bdfd9d0b" +checksum = "83fcf4b431fb653aefd2cc3c3a65ea78988acf5a7626cb825c54b5aab5aa1a8e" dependencies = [ "cfg-if", ] [[package]] name = "wasmtime-internal-cache" -version = "36.0.19" +version = "36.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4108e1964b3a02b4b5604ea1a2918a0cbcab512e5de0ccf4c062fc6c06b2eb8e" +checksum = "65b514e39cd914f34a135ce7696373287ccb9611b083cbe244a48c4903380fb5" dependencies = [ "anyhow", "base64 0.22.1", @@ -12755,9 +12621,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-macro" -version = "36.0.19" +version = "36.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8278aa2bb237788ac9e3a70c5bea3ea3de6b4f95fb003df303858941871b8c4e" +checksum = "a92fe9cf6ca766adc68b5ed4474155e88cc97e79d13c63450fe91779de9c41a3" dependencies = [ "anyhow", "proc-macro2", @@ -12770,15 +12636,15 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-util" -version = "36.0.19" +version = "36.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "838c652b6cafd82a432a3a2c7af5bea4c6957d218aafbe50018e0297486ab7cc" +checksum = "36b8553647a6ae550ec3bca45759565204af763c37b1b51d5e45d6c5a26a1e32" [[package]] name = "wasmtime-internal-cranelift" -version = "36.0.19" +version = "36.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7339fdd64a3fde574505adea23244e065d5b1de0e72c8dd87e150f5d4ab63a63" +checksum = "9bd306f478b4a68d94a80e64d1a92871b697188eafe8bfc387f865085c818a04" dependencies = [ "anyhow", "cfg-if", @@ -12803,9 +12669,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-fiber" -version = "36.0.19" +version = "36.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f3afc72d3c72f71d4ebc72e83f479ba8ca24409c41d43e5e0228f0bf82b873e" +checksum = "e9f9bb63081293450d0dfb439f246339e91e34d51c09b1310bdd297b67be5933" dependencies = [ "anyhow", "cc", @@ -12819,9 +12685,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-debug" -version = "36.0.19" +version = "36.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21d33adf2d1eae36df9b6b033a1d549c7553ea607f4f9a81f75d51627725c354" +checksum = "e3c04edbaf79fb10460ee0867a829f831b7afa8d5d8c9d32287d88b3f1aaef79" dependencies = [ "cc", "object", @@ -12831,9 +12697,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "36.0.19" +version = "36.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bde2fcf14cc76084242c6fe792e49778b5a9008c7df28ca4fe21c35c69d5f55" +checksum = "4e2b3b397afafb7e6b3603dd3f867cc626d3bde726bd65ce01443a70fe66e1c0" dependencies = [ "anyhow", "cfg-if", @@ -12843,24 +12709,24 @@ dependencies = [ [[package]] name = "wasmtime-internal-math" -version = "36.0.19" +version = "36.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd982249e933e730859f84c17254c7e95f38c3b0cdf81695bfc6a270e237852" +checksum = "2509a3a45fbdc79eab62680df8f8d5ffe5420c3bf7f0c2083d49110c09c61b34" dependencies = [ "libm", ] [[package]] name = "wasmtime-internal-slab" -version = "36.0.19" +version = "36.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8280b359384cc412bddf0023193eeaffadcb0d7fbbf860f2b7e0580e8806a344" +checksum = "0985eb32142c197f1aa5d50e94498a769f55095e8ed2838eff386c735f9da72f" [[package]] name = "wasmtime-internal-unwinder" -version = "36.0.19" +version = "36.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb43477c41769fec0d031d32cb3236195acfb1e6aa4efdf463e5fd001c23f6cc" +checksum = "6b88fa7f08080eeb4926d6062c2ae8c38c19a2b1a6d0f01c569b827d987f4909" dependencies = [ "anyhow", "cfg-if", @@ -12871,9 +12737,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-versioned-export-macros" -version = "36.0.19" +version = "36.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b33afac17819f2f767ea280b4b87a6c95458ef1ae43da410cfc5b26813ecb78f" +checksum = "bfff26e96f2539205ff7c049e4dff8343a479722bbbae88bc4da54adde12c1fa" dependencies = [ "proc-macro2", "quote", @@ -12882,9 +12748,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-winch" -version = "36.0.19" +version = "36.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1757c8b0626385c93994c7a3c7a23ee5b469061465b601c1fdce71ff61b11c82" +checksum = "176d188460e507807f9f8b7521d7c1dc62b356fa221b4e2ce566e88652294d9d" dependencies = [ "anyhow", "cranelift-codegen", @@ -12899,9 +12765,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-wit-bindgen" -version = "36.0.19" +version = "36.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061dd5bae2e618b32a8ed2acda5953d07dd308ac0e3f7960afc62ccdf6df7c44" +checksum = "7d3c06e67956fabd2fa3b1aa94200779cc0df8100dc45c6a9b80fda67f565697" dependencies = [ "anyhow", "bitflags 2.11.0", @@ -13010,9 +12876,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.10" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb" +checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca" dependencies = [ "rustls-pki-types", ] @@ -13023,14 +12889,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.10", + "webpki-roots 1.0.7", ] [[package]] name = "webpki-roots" -version = "1.0.10" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" dependencies = [ "rustls-pki-types", ] @@ -13364,7 +13230,7 @@ dependencies = [ "flume 0.11.1", "futures", "hex", - "hmac 0.12.1", + "hmac", "libc", "mdns-sd", "regex", @@ -13421,7 +13287,7 @@ dependencies = [ "chrono", "clap", "criterion", - "hmac 0.12.1", + "hmac", "midstreamer-quic", "midstreamer-scheduler", "serde", @@ -13614,7 +13480,7 @@ dependencies = [ "clap", "criterion", "futures-util", - "hmac 0.12.1", + "hmac", "hyper", "hyper-util", "jsonwebtoken", @@ -13804,7 +13670,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.2", ] [[package]] @@ -13815,9 +13681,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winch-codegen" -version = "36.0.19" +version = "36.0.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69940a29fc6b01b0e6b141912b700c17eaf956d58d63bc0c6e733435caf020e4" +checksum = "cde193a101057fce046a3fff773932419b7861f6bb95632d01bca90843c9ce51" dependencies = [ "anyhow", "cranelift-assembler-x64", @@ -14590,7 +14456,7 @@ dependencies = [ "html5ever", "http", "javascriptcore-rs", - "jni 0.21.1", + "jni", "kuchikiki", "libc", "ndk", @@ -14824,37 +14690,55 @@ dependencies = [ [[package]] name = "zip" -version = "9.0.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" dependencies = [ "aes", + "byteorder", "bzip2", - "constant_time_eq 0.6.1", + "constant_time_eq 0.1.5", "crc32fast", - "deflate64", + "crossbeam-utils", "flate2", - "getrandom 0.4.1", - "hmac 0.13.0", - "indexmap 2.13.0", - "lzma-rust2 0.22.0", - "memchr", - "nix 0.31.3", + "hmac", "pbkdf2", - "ppmd-rust", - "sha1 0.11.0", + "sha1", "time", - "typed-path", - "zeroize", + "zstd 0.11.2+zstd.1.5.2", +] + +[[package]] +name = "zip" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" +dependencies = [ + "arbitrary", + "crc32fast", + "flate2", + "indexmap 2.13.0", + "memchr", "zopfli", - "zstd 0.14.3", +] + +[[package]] +name = "zip" +version = "7.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0" +dependencies = [ + "crc32fast", + "indexmap 2.13.0", + "memchr", + "typed-path", ] [[package]] name = "zlib-rs" -version = "0.6.8" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b268e58e7c693d7c271f93ffc4ba3b380412554231c85bf61ca7af91042a4112" +checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" [[package]] name = "zmij" @@ -14876,45 +14760,46 @@ dependencies = [ [[package]] name = "zstd" -version = "0.13.3" +version = "0.11.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" dependencies = [ - "zstd-safe 7.2.4", + "zstd-safe 5.0.2+zstd.1.5.2", ] [[package]] name = "zstd" -version = "0.14.3" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3ae9bf90af6e40197bef767eeefc46ad6397420fe873830bd85e5fbbfb3c70" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" dependencies = [ - "zstd-safe 8.2.0", + "zstd-safe 7.2.4", ] [[package]] name = "zstd-safe" -version = "7.2.4" +version = "5.0.2+zstd.1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" dependencies = [ + "libc", "zstd-sys", ] [[package]] name = "zstd-safe" -version = "8.2.0" +version = "7.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded520f16b29e1a9b771c722221b2b0a0398c2316d9bb3c42b458a5414d7434b" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.1.2+zstd.1.5.7" +version = "2.0.16+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "481ada9c543b2323cbb7bbdbb5249177fbf0188ed959ed2c15e4b22421c84ed3" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" dependencies = [ "cc", "pkg-config", diff --git a/v2/Cargo.toml b/v2/Cargo.toml index 19d505998c..87cd12a0b6 100644 --- a/v2/Cargo.toml +++ b/v2/Cargo.toml @@ -235,8 +235,7 @@ pcap = "1.1" petgraph = "0.6" # Data loading -# Training uses NPY only; keep complex arrays without the unused NPZ/ZIP backend. -ndarray-npy = { version = "0.10", default-features = false, features = ["num-complex-0_4"] } +ndarray-npy = "0.10" walkdir = "2.4" # Hashing (for proof) @@ -319,11 +318,3 @@ strip = false [profile.bench] inherits = "release" debug = true - -# Narrow ZIP9 compatibility patches retain existing model/framework versions. -# Provenance and review scope: dependency-patches/README.md. -[patch.crates-io] -candle-core-04 = { package = "candle-core", path = "dependency-patches/candle-core-0.4.1" } -candle-core-09 = { package = "candle-core", path = "dependency-patches/candle-core-0.9.2" } -tch = { path = "dependency-patches/tch-0.24.0" } -torch-sys = { path = "dependency-patches/torch-sys-0.24.0" } diff --git a/v2/crates/cog-person-count/tests/zip9_tensor_roundtrip.rs b/v2/crates/cog-person-count/tests/zip9_tensor_roundtrip.rs deleted file mode 100644 index e9386bad6a..0000000000 --- a/v2/crates/cog-person-count/tests/zip9_tensor_roundtrip.rs +++ /dev/null @@ -1,14 +0,0 @@ -// ZIP9 compatibility must preserve actual tensor names and values, not just compile. -use candle_core::{Device, Tensor}; -#[test] -fn npz_roundtrip_retains_unicode_and_nested_tensor_names() -> candle_core::Result<()> { - let directory = tempfile::tempdir()?; - let path = directory.path().join("tensors.npz"); - let tensor = Tensor::new(&[1f32, -2.5, 3.25], &Device::Cpu)?; - Tensor::write_npz(&[("nested/room-α", &tensor)], &path)?; - let restored = Tensor::read_npz(&path)?; - assert_eq!(restored.len(), 1); - assert_eq!(restored[0].0, "nested/room-α"); - assert_eq!(restored[0].1.to_vec1::()?, vec![1., -2.5, 3.25]); - Ok(()) -} diff --git a/v2/crates/wifi-densepose-nn/tests/zip9_tensor_roundtrip.rs b/v2/crates/wifi-densepose-nn/tests/zip9_tensor_roundtrip.rs deleted file mode 100644 index 90b6b243cd..0000000000 --- a/v2/crates/wifi-densepose-nn/tests/zip9_tensor_roundtrip.rs +++ /dev/null @@ -1,15 +0,0 @@ -#![cfg(feature = "candle-backend")] -// ZIP9 compatibility must preserve actual tensor names and values, not just compile. -use candle_core::{Device, Tensor}; -#[test] -fn npz_roundtrip_retains_unicode_and_nested_tensor_names() -> candle_core::Result<()> { - let directory = tempfile::tempdir()?; - let path = directory.path().join("tensors.npz"); - let tensor = Tensor::new(&[1f32, -2.5, 3.25], &Device::Cpu)?; - Tensor::write_npz(&[("nested/room-α", &tensor)], &path)?; - let restored = Tensor::read_npz(&path)?; - assert_eq!(restored.len(), 1); - assert_eq!(restored[0].0, "nested/room-α"); - assert_eq!(restored[0].1.to_vec1::()?, vec![1., -2.5, 3.25]); - Ok(()) -} diff --git a/v2/dependency-patches/README.md b/v2/dependency-patches/README.md deleted file mode 100644 index 639a4aad30..0000000000 --- a/v2/dependency-patches/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# ZIP9 compatibility patches - -These are narrow source patches to the existing crates.io Candle0.4.1, Candle0.9.2, tch0.24.0 and torch-sys0.24.0 releases. The model/framework versions, numerical operators and public APIs are retained. Root Cargo patches redirect those exact package versions to these sources; every ZIP edge resolves to patched ZIP9.0.5 in the root lockfile. - -`patches.json` records downloaded `.crate` SHA256, each retained upstream-file hash, patched-file hash and the changed-file list. `compatibility.patch` contains the complete review delta. Sources came from `https://static.crates.io/crates//-.crate`. MIT/Apache licences remain in each directory. Cargo.lock and cargo VCS metadata were omitted; tch's 87MB `third_party` model/PyTorch test snapshot was omitted because it is not a Rust library or build input. Its asset-dependent upstream tests are not represented as passed. These snapshots are not independently published packages. - -## Changes - -- Four normalized Cargo manifests request ZIP9; old manifests are retained as upstream provenance only. -- NPZ name decoding propagates ZIP9's checked UTF8 result; names are borrowed from Cow without truncation. NPZ writers use SimpleFileOptions. -- PTH file-name iteration collects checked results before reading entries. -- The optional libtorch download build script uses ZIP9's checked extraction instead of joining mangled names manually. This is a build-time path, not a default server runtime. -- No advisory ignore, audit gate change, framework-major migration or tensor algorithm change. - -## Verification - -Run `python3 scripts/verify_zip_parent_patches.py`, then root `cargo audit --file v2/Cargo.lock`. Focused tests include the actual current Candle consumer and NPZ Unicode/nested-name/value roundtrip, sensing-server companion/TLS, NPY loader and Wasmtime sandbox tests. `cargo check -p wifi-densepose-train --features tch-backend,tch/doc-only` checks tch and torch-sys Rust/build-script compatibility without downloading or claiming libtorch execution. - -The legacy optional Candle0.4 backend cannot currently compile against the main workspace's half2.7/rand0.9 combination: its unchanged random operators require rand0.8. Pinning half2.3 in the full workspace conflicts with burn's half2.7 requirement, so no such pin is applied. `cargo test --manifest-path v2/dependency-patches/compat-tests/Cargo.toml` exercises the patched Candle0.4 NPZ API in an isolated compatible half2.3 environment. Its own lockfile is retained. This does not claim that the legacy optional backend is qualified in the main workspace. Full PyTorch/CUDA/model numerical qualification remains unexecuted; the default Mac server does not include those optional backends. - -Remove each snapshot when an upstream compatible package release supplies ZIP>=9. Source changes require the same focused format, build and integrity review. The separate Apple helper binary must be rebuilt from the clean source commit and verified against its embedded manifest before distribution. diff --git a/v2/dependency-patches/candle-core-0.4.1/Cargo.toml b/v2/dependency-patches/candle-core-0.4.1/Cargo.toml deleted file mode 100644 index 3b359c646c..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/Cargo.toml +++ /dev/null @@ -1,143 +0,0 @@ -# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO -# -# When uploading crates to the registry Cargo will automatically -# "normalize" Cargo.toml files for maximal compatibility -# with all versions of Cargo and also rewrite `path` dependencies -# to registry (e.g., crates.io) dependencies. -# -# If you are reading this file be aware that the original Cargo.toml -# will likely look very different (and much more reasonable). -# See Cargo.toml.orig for the original contents. - -[package] -edition = "2021" -name = "candle-core" -version = "0.4.1" -description = "Minimalist ML framework." -readme = "README.md" -keywords = [ - "blas", - "tensor", - "machine-learning", -] -categories = ["science"] -license = "MIT OR Apache-2.0" -repository = "https://github.com/huggingface/candle" - -[[bench]] -name = "bench_main" -harness = false - -[dependencies.accelerate-src] -version = "0.3.2" -optional = true - -[dependencies.byteorder] -version = "1.4.3" - -[dependencies.candle-kernels] -version = "0.4.1" -optional = true - -[dependencies.candle-metal-kernels] -version = "0.4.1" -optional = true - -[dependencies.cudarc] -version = "0.10.0" -features = ["f16"] -optional = true - -[dependencies.gemm] -version = "0.17.0" -features = ["wasm-simd128-enable"] - -[dependencies.half] -version = "2.3.1" -features = [ - "num-traits", - "use-intrinsics", - "rand_distr", -] - -[dependencies.intel-mkl-src] -version = "0.8.1" -features = ["mkl-static-lp64-iomp"] -optional = true - -[dependencies.libc] -version = "0.2.147" -optional = true - -[dependencies.memmap2] -version = "0.9.3" -features = ["stable_deref_trait"] - -[dependencies.metal] -version = "0.27.0" -features = ["mps"] -optional = true - -[dependencies.num-traits] -version = "0.2.15" - -[dependencies.num_cpus] -version = "1.15.0" - -[dependencies.rand] -version = "0.8.5" - -[dependencies.rand_distr] -version = "0.4.3" - -[dependencies.rayon] -version = "1.7.0" - -[dependencies.safetensors] -version = "0.4.1" - -[dependencies.thiserror] -version = "1" - -[dependencies.yoke] -version = "0.7.2" -features = ["derive"] - -[dependencies.zip] -version = "9.0.1" -default-features = false - -[dev-dependencies.anyhow] -version = "1" -features = ["backtrace"] - -[dev-dependencies.clap] -version = "4.2.4" -features = ["derive"] - -[dev-dependencies.criterion] -version = "0.5.1" -default-features = false - -[features] -accelerate = [ - "dep:libc", - "dep:accelerate-src", -] -cuda = [ - "cudarc", - "dep:candle-kernels", -] -cudnn = [ - "cuda", - "cudarc/cudnn", -] -default = [] -metal = [ - "dep:metal", - "dep:candle-metal-kernels", -] -mkl = [ - "dep:libc", - "dep:intel-mkl-src", -] diff --git a/v2/dependency-patches/candle-core-0.4.1/Cargo.toml.orig b/v2/dependency-patches/candle-core-0.4.1/Cargo.toml.orig deleted file mode 100644 index 92a04917db..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/Cargo.toml.orig +++ /dev/null @@ -1,50 +0,0 @@ -[package] -name = "candle-core" -version.workspace = true -edition.workspace = true -description.workspace = true -repository.workspace = true -keywords.workspace = true -categories.workspace = true -license.workspace = true -readme = "README.md" - -[dependencies] -accelerate-src = { workspace = true, optional = true } -byteorder = { workspace = true } -candle-kernels = { workspace = true, optional = true } -candle-metal-kernels = { workspace = true, optional = true } -metal = { workspace = true, optional = true} -cudarc = { workspace = true, optional = true } -gemm = { workspace = true } -half = { workspace = true } -intel-mkl-src = { workspace = true, optional = true } -libc = { workspace = true, optional = true } -memmap2 = { workspace = true } -num-traits = { workspace = true } -num_cpus = { workspace = true } -rand = { workspace = true } -rand_distr = { workspace = true } -rayon = { workspace = true } -safetensors = { workspace = true } -thiserror = { workspace = true } -yoke = { workspace = true } -zip = { workspace = true } - -[dev-dependencies] -anyhow = { workspace = true } -clap = { workspace = true } -criterion = { workspace = true } - - -[features] -default = [] -cuda = ["cudarc", "dep:candle-kernels"] -cudnn = ["cuda", "cudarc/cudnn"] -mkl = ["dep:libc", "dep:intel-mkl-src"] -accelerate = ["dep:libc", "dep:accelerate-src"] -metal = ["dep:metal", "dep:candle-metal-kernels"] - -[[bench]] -name = "bench_main" -harness = false diff --git a/v2/dependency-patches/candle-core-0.4.1/LICENSE b/v2/dependency-patches/candle-core-0.4.1/LICENSE deleted file mode 100644 index 261eeb9e9f..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/v2/dependency-patches/candle-core-0.4.1/README.md b/v2/dependency-patches/candle-core-0.4.1/README.md deleted file mode 100644 index 246f5bae61..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# candle -Minimalist ML framework for Rust diff --git a/v2/dependency-patches/candle-core-0.4.1/benches/bench_main.rs b/v2/dependency-patches/candle-core-0.4.1/benches/bench_main.rs deleted file mode 100644 index 162e3f2be1..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/benches/bench_main.rs +++ /dev/null @@ -1,9 +0,0 @@ -mod benchmarks; - -use criterion::criterion_main; -criterion_main!( - benchmarks::affine::benches, - benchmarks::matmul::benches, - benchmarks::random::benches, - benchmarks::where_cond::benches -); diff --git a/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/affine.rs b/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/affine.rs deleted file mode 100644 index eded9f5787..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/affine.rs +++ /dev/null @@ -1,43 +0,0 @@ -use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; -use candle_core::{DType, Device, Tensor}; -use criterion::{black_box, criterion_group, Criterion, Throughput}; -use std::time::Instant; - -fn run(a: &Tensor) { - a.affine(12.34, 56.78).unwrap(); -} - -fn run_affine_benchmark(c: &mut Criterion, device: &Device, dtype: DType, name: &str) { - let b = 1; - let m = 1024; - let k = 1024; - - let tensor = Tensor::zeros((b, m, k), dtype, &device).unwrap(); - - let flops = b * m * k * dtype.size_in_bytes(); - - let mut group = c.benchmark_group(device.bench_name(name)); - group.throughput(Throughput::Bytes(flops as u64)); - group.bench_function("iter", move |b| { - b.iter_custom(|iters| { - let start = Instant::now(); - for _i in 0..iters { - run(black_box(&tensor)); - } - device.sync().unwrap(); - start.elapsed() - }) - }); - group.finish(); -} - -fn criterion_benchmark(c: &mut Criterion) { - let handler = BenchDeviceHandler::new().unwrap(); - for device in handler.devices { - run_affine_benchmark(c, &device, DType::F32, "affine_f32"); - run_affine_benchmark(c, &device, DType::F16, "affine_f16"); - run_affine_benchmark(c, &device, DType::BF16, "affine_bf16"); - } -} - -criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/matmul.rs b/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/matmul.rs deleted file mode 100644 index 9d67e642cd..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/matmul.rs +++ /dev/null @@ -1,44 +0,0 @@ -use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; -use candle_core::{DType, Device, Tensor}; -use criterion::{black_box, criterion_group, Criterion, Throughput}; -use std::time::Instant; - -fn run(a: &Tensor, b: &Tensor) { - a.matmul(&b.t().unwrap()).unwrap(); -} - -fn run_bench(c: &mut Criterion, device: &Device) { - let b = 1; - let m = 1; - let n = 2048; - let k = 2048; - - let dtype = DType::F32; - let lhs = Tensor::zeros((b, m, k), dtype, device).unwrap(); - let rhs = Tensor::zeros((b, n, k), dtype, device).unwrap(); - - let flops = b * m * n * k; - - let mut group = c.benchmark_group(device.bench_name("matmul")); - group.throughput(Throughput::Bytes(flops as u64)); - group.bench_function("iter", move |b| { - b.iter_custom(|iters| { - let start = Instant::now(); - for _i in 0..iters { - run(black_box(&lhs), black_box(&rhs)); - } - device.sync().unwrap(); - start.elapsed() - }) - }); - group.finish(); -} - -fn criterion_benchmark(c: &mut Criterion) { - let handler = BenchDeviceHandler::new().unwrap(); - for device in handler.devices { - run_bench(c, &device); - } -} - -criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/mod.rs b/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/mod.rs deleted file mode 100644 index c45effee20..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/mod.rs +++ /dev/null @@ -1,66 +0,0 @@ -pub(crate) mod affine; -pub(crate) mod matmul; -pub(crate) mod random; -pub(crate) mod where_cond; - -use candle_core::{Device, Result}; - -pub(crate) trait BenchDevice { - fn sync(&self) -> Result<()>; - - fn bench_name>(&self, name: S) -> String; -} - -impl BenchDevice for Device { - fn sync(&self) -> Result<()> { - match self { - Device::Cpu => Ok(()), - Device::Cuda(device) => { - #[cfg(feature = "cuda")] - return Ok(device.synchronize()?); - #[cfg(not(feature = "cuda"))] - panic!("Cuda device without cuda feature enabled: {:?}", device) - } - Device::Metal(device) => { - #[cfg(feature = "metal")] - return Ok(device.wait_until_completed()?); - #[cfg(not(feature = "metal"))] - panic!("Metal device without metal feature enabled: {:?}", device) - } - } - } - - fn bench_name>(&self, name: S) -> String { - match self { - Device::Cpu => { - let cpu_type = if cfg!(feature = "accelerate") { - "accelerate" - } else if cfg!(feature = "mkl") { - "mkl" - } else { - "cpu" - }; - format!("{}_{}", cpu_type, name.into()) - } - Device::Cuda(_) => format!("cuda_{}", name.into()), - Device::Metal(_) => format!("metal_{}", name.into()), - } - } -} - -struct BenchDeviceHandler { - devices: Vec, -} - -impl BenchDeviceHandler { - pub fn new() -> Result { - let mut devices = Vec::new(); - if cfg!(feature = "metal") { - devices.push(Device::new_metal(0)?); - } else if cfg!(feature = "cuda") { - devices.push(Device::new_cuda(0)?); - } - devices.push(Device::Cpu); - Ok(Self { devices }) - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/random.rs b/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/random.rs deleted file mode 100644 index 22c60ef18c..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/random.rs +++ /dev/null @@ -1,63 +0,0 @@ -use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; -use candle_core::{DType, Device, Tensor}; -use criterion::{black_box, criterion_group, Criterion, Throughput}; -use std::time::Instant; - -fn rand_uniform(a: &Tensor) { - a.rand_like(-1.0, 123.0).unwrap(); -} - -fn rand_normal(a: &Tensor) { - a.randn_like(100.0, 15.0).unwrap(); -} - -fn run_random_bench(c: &mut Criterion, device: &Device) { - let b = 1; - - let rows = 2048; - let cols = 2048; - - let dtype = DType::F32; - let tensor = Tensor::zeros((b, rows, cols), dtype, device).unwrap(); - - let flops = b * rows * cols * dtype.size_in_bytes(); - - let mut group = c.benchmark_group(device.bench_name("random_uniform")); - group.throughput(Throughput::Bytes(flops as u64)); - group.bench_function("iter", move |benches| { - benches.iter_custom(|iters| { - let start = Instant::now(); - for _i in 0..iters { - rand_uniform(black_box(&tensor)); - } - device.sync().unwrap(); - start.elapsed() - }) - }); - group.finish(); - - let tensor = Tensor::zeros((b, rows, cols), dtype, device).unwrap(); - - let mut group = c.benchmark_group(device.bench_name("random_normal")); - group.throughput(Throughput::Bytes(flops as u64)); - group.bench_function("iter", move |benches| { - benches.iter_custom(|iters| { - let start = Instant::now(); - for _i in 0..iters { - rand_normal(black_box(&tensor)); - } - device.sync().unwrap(); - start.elapsed() - }) - }); - group.finish(); -} - -fn criterion_benchmark(c: &mut Criterion) { - let handler = BenchDeviceHandler::new().unwrap(); - for device in handler.devices { - run_random_bench(c, &device); - } -} - -criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/where_cond.rs b/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/where_cond.rs deleted file mode 100644 index c517dcf590..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/benches/benchmarks/where_cond.rs +++ /dev/null @@ -1,64 +0,0 @@ -use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; -use candle_core::{DType, Device, Tensor}; -use criterion::{black_box, criterion_group, Criterion, Throughput}; -use std::time::Instant; - -fn run(a: &Tensor, b: &Tensor, c: &Tensor) { - a.where_cond(b, c).unwrap(); -} - -const fn create_cond_arr() -> [u8; N] { - let mut arr = [0u8; N]; - let mut i = 0; - while i < N { - arr[i] = (i % 2) as u8; - i += 1; - } - arr -} - -const B: usize = 1; -const M: usize = 1024; -const K: usize = 1024; -const SIZE: usize = B * M * K; - -const DATA: [u8; SIZE] = create_cond_arr::(); - -fn run_where_cond_benchmark(c: &mut Criterion, device: &Device, dtype: DType, name: &str) { - let tensor = Tensor::from_slice(DATA.as_slice(), (B, M, K), &device).unwrap(); - let on_true = Tensor::ones((B, M, K), dtype, &device).unwrap(); - let on_false = Tensor::zeros((B, M, K), dtype, &device).unwrap(); - - let elements = B * M * K; - // E.g. 2 f32 tensors + 1 u8 tensor - let flops = (2 * elements * dtype.size_in_bytes()) + elements; - - let mut group = c.benchmark_group(device.bench_name(name)); - group.throughput(Throughput::Bytes(flops as u64)); - group.bench_function("iter", move |b| { - b.iter_custom(|iters| { - let start = Instant::now(); - for _i in 0..iters { - run( - black_box(&tensor), - black_box(&on_true), - black_box(&on_false), - ); - } - device.sync().unwrap(); - start.elapsed() - }) - }); - group.finish(); -} - -fn criterion_benchmark(c: &mut Criterion) { - let device = BenchDeviceHandler::new().unwrap(); - for d in device.devices { - run_where_cond_benchmark(c, &d, DType::F32, "where_cond_f32"); - run_where_cond_benchmark(c, &d, DType::BF16, "where_cond_bf16"); - run_where_cond_benchmark(c, &d, DType::F16, "where_cond_f16"); - } -} - -criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.4.1/examples/basics.rs b/v2/dependency-patches/candle-core-0.4.1/examples/basics.rs deleted file mode 100644 index fe15187b5a..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/examples/basics.rs +++ /dev/null @@ -1,17 +0,0 @@ -#[cfg(feature = "mkl")] -extern crate intel_mkl_src; - -#[cfg(feature = "accelerate")] -extern crate accelerate_src; - -use anyhow::Result; -use candle_core::{Device, Tensor}; - -fn main() -> Result<()> { - let a = Tensor::new(&[[0.0f32, 1.0, 2.0], [3.0, 4.0, 5.0]], &Device::Cpu)?; - let b = Tensor::new(&[[88.0f32, 99.0]], &Device::Cpu)?; - let new_a = a.slice_scatter(&b, 1, 2)?; - assert_eq!(a.to_vec2::()?, [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); - assert_eq!(new_a.to_vec2::()?, [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.4.1/examples/cuda_basics.rs b/v2/dependency-patches/candle-core-0.4.1/examples/cuda_basics.rs deleted file mode 100644 index 7929fba601..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/examples/cuda_basics.rs +++ /dev/null @@ -1,36 +0,0 @@ -#[cfg(feature = "accelerate")] -extern crate accelerate_src; - -#[cfg(feature = "mkl")] -extern crate intel_mkl_src; - -use anyhow::Result; -use candle_core::{Device, Module, Tensor}; - -use candle_core::quantized::{QMatMul, QTensor}; - -fn main() -> Result<()> { - let device = Device::new_cuda(0)?; - let q = Tensor::randn(0f32, 1.0, (72, 256), &device)?; - let q_cpu = q.to_device(&Device::Cpu)?; - let q = QTensor::quantize(&q, candle_core::quantized::GgmlDType::Q8K)?; - let q = QMatMul::from_qtensor(q)?; - let x = Tensor::randn(0f32, 1.0, (5, 256), &device)?; - let res_q_cuda = q.forward(&x)?; - println!("{res_q_cuda}"); - - let q_cpu = QTensor::quantize(&q_cpu, candle_core::quantized::GgmlDType::Q8K)?; - let q_cpu_tensor = q_cpu.dequantize(&Device::Cpu)?; - let q_cpu = QMatMul::from_qtensor(q_cpu)?; - let x_cpu = x.to_device(&Device::Cpu)?; - let res_q_cpu = q_cpu.forward(&x_cpu)?; - println!("{res_q_cpu}"); - - let res_mm = x_cpu.matmul(&q_cpu_tensor.t()?)?; - let diff = (res_mm - res_q_cuda.to_device(&Device::Cpu))? - .abs()? - .flatten_all()? - .max(0)?; - println!("{diff}"); - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.4.1/examples/cuda_sum_benchmark.rs b/v2/dependency-patches/candle-core-0.4.1/examples/cuda_sum_benchmark.rs deleted file mode 100644 index d6d182e8fc..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/examples/cuda_sum_benchmark.rs +++ /dev/null @@ -1,54 +0,0 @@ -#[cfg(feature = "mkl")] -extern crate intel_mkl_src; - -#[cfg(feature = "accelerate")] -extern crate accelerate_src; - -use std::str::FromStr; - -use anyhow::Result; -use candle_core::{Device, Tensor}; - -fn cos_sin(n: usize, device: &Device) -> Result { - let thetas: Vec<_> = (0..n).map(|i| (i as f32 / n as f32)).collect(); - let xs: Vec<_> = thetas.iter().map(|t| t.cos().abs()).collect(); - let ys: Vec<_> = thetas.iter().map(|t| t.sin().abs()).collect(); - let xs = Tensor::from_vec(xs, (n, 1), device)?; - let ys = Tensor::from_vec(ys, (1, n), device)?; - let ys = Tensor::cat(&[&ys, &ys, &ys, &ys, &ys, &ys], 1)?; - Ok(xs.matmul(&ys)?) -} - -fn main() -> Result<()> { - let device = Device::new_cuda(0)?; - let args = std::env::args().collect::>(); - let n = if args.len() < 2 { - 2000usize - } else { - usize::from_str(&args[1])? - }; - let xys_cpu = cos_sin(n, &Device::Cpu)?; - let xys = cos_sin(n, &device)?; - println!("{xys_cpu:?} {xys:?}"); - let sum_keepdim_cpu = xys_cpu.sum_keepdim(1)?; - println!("{sum_keepdim_cpu}"); - let sum_keepdim = xys.sum_keepdim(1)?; - println!("{sum_keepdim}"); - let start = std::time::Instant::now(); - let n_iters = 100; - let mut v = 0f32; - for _i in 0..n_iters { - let sum_keepdim = xys.sum_keepdim(1)?; - let sum_keepdim = sum_keepdim.sum_keepdim(0)?; - let sum_keepdim: f32 = sum_keepdim.reshape(&[])?.to_scalar()?; - v += sum_keepdim; - } - let elapsed = start.elapsed(); - if v > 0. { - println!( - "ran {n_iters} iterations, time per iter: {:?} ({v})", - elapsed.div_f64(n_iters as f64) - ); - } - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.4.1/examples/tensor-tools.rs b/v2/dependency-patches/candle-core-0.4.1/examples/tensor-tools.rs deleted file mode 100644 index 1801ac58ad..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/examples/tensor-tools.rs +++ /dev/null @@ -1,389 +0,0 @@ -use candle_core::quantized::{gguf_file, GgmlDType, QTensor}; -use candle_core::{Device, Result}; -use clap::{Parser, Subcommand, ValueEnum}; -use rayon::prelude::*; - -#[derive(ValueEnum, Debug, Clone)] -enum QuantizationMode { - /// The default quantization includes all 2d tensors, except the output tensor which always - /// uses Q6_K. - Llama, -} - -impl QuantizationMode { - fn quantize(&self, name: &str, tensor: QTensor, dtype: GgmlDType) -> Result { - match self { - Self::Llama => { - // Same behavior as the llama.cpp quantization. - let should_quantize = name.ends_with(".weight") && tensor.rank() == 2; - if should_quantize { - let tensor = tensor.dequantize(&Device::Cpu)?; - if name == "output.weight" { - QTensor::quantize(&tensor, GgmlDType::Q6K) - } else { - QTensor::quantize(&tensor, dtype) - } - } else { - Ok(tensor) - } - } - } - } -} - -#[derive(ValueEnum, Debug, Clone)] -enum Quantization { - #[value(name = "q4_0")] - Q4_0, - #[value(name = "q4_1")] - Q4_1, - #[value(name = "q5_0")] - Q5_0, - #[value(name = "q5_1")] - Q5_1, - #[value(name = "q8_0")] - Q8_0, - #[value(name = "q8_1")] - Q8_1, - Q2k, - Q3k, - Q4k, - Q5k, - Q6k, - Q8k, - F16, - F32, -} - -impl Quantization { - fn dtype(&self) -> GgmlDType { - match self { - Quantization::Q4_0 => GgmlDType::Q4_0, - Quantization::Q4_1 => GgmlDType::Q4_1, - Quantization::Q5_0 => GgmlDType::Q5_0, - Quantization::Q5_1 => GgmlDType::Q5_1, - Quantization::Q8_0 => GgmlDType::Q8_0, - Quantization::Q8_1 => GgmlDType::Q8_1, - Quantization::Q2k => GgmlDType::Q2K, - Quantization::Q3k => GgmlDType::Q3K, - Quantization::Q4k => GgmlDType::Q4K, - Quantization::Q5k => GgmlDType::Q5K, - Quantization::Q6k => GgmlDType::Q6K, - Quantization::Q8k => GgmlDType::Q8K, - Quantization::F16 => GgmlDType::F16, - Quantization::F32 => GgmlDType::F32, - } - } -} - -#[derive(ValueEnum, Debug, Clone)] -enum Format { - Safetensors, - Npz, - Ggml, - Gguf, - Pth, - Pickle, -} - -impl Format { - fn infer>(p: P) -> Option { - p.as_ref() - .extension() - .and_then(|e| e.to_str()) - .and_then(|e| match e { - // We don't infer any format for .bin as it can be used for ggml/gguf or pytorch. - "safetensors" | "safetensor" => Some(Self::Safetensors), - "npz" => Some(Self::Npz), - "pth" | "pt" => Some(Self::Pth), - "ggml" => Some(Self::Ggml), - "gguf" => Some(Self::Gguf), - _ => None, - }) - } -} - -#[derive(Subcommand, Debug, Clone)] -enum Command { - Ls { - files: Vec, - - /// The file format to use, if unspecified infer from the file extension. - #[arg(long, value_enum)] - format: Option, - - /// Enable verbose mode. - #[arg(short, long)] - verbose: bool, - }, - - Quantize { - /// The input file(s), in safetensors format. - in_file: Vec, - - /// The output file, in gguf format. - #[arg(long)] - out_file: std::path::PathBuf, - - /// The quantization schema to apply. - #[arg(long, value_enum)] - quantization: Quantization, - - /// Which tensor to quantize. - #[arg(long, value_enum, default_value_t = QuantizationMode::Llama)] - mode: QuantizationMode, - }, - - Dequantize { - /// The input file, in gguf format. - in_file: std::path::PathBuf, - - /// The output file, in safetensors format. - #[arg(long)] - out_file: std::path::PathBuf, - }, -} - -#[derive(Parser, Debug, Clone)] -struct Args { - #[command(subcommand)] - command: Command, -} - -fn run_ls( - file: &std::path::PathBuf, - format: Option, - verbose: bool, - device: &Device, -) -> Result<()> { - let format = match format { - Some(format) => format, - None => match Format::infer(file) { - Some(format) => format, - None => { - println!( - "{file:?}: cannot infer format from file extension, use the --format flag" - ); - return Ok(()); - } - }, - }; - match format { - Format::Npz => { - let tensors = candle_core::npy::NpzTensors::new(file)?; - let mut names = tensors.names(); - names.sort(); - for name in names { - let shape_dtype = match tensors.get_shape_and_dtype(name) { - Ok((shape, dtype)) => format!("[{shape:?}; {dtype:?}]"), - Err(err) => err.to_string(), - }; - println!("{name}: {shape_dtype}") - } - } - Format::Safetensors => { - let tensors = unsafe { candle_core::safetensors::MmapedSafetensors::new(file)? }; - let mut tensors = tensors.tensors(); - tensors.sort_by(|a, b| a.0.cmp(&b.0)); - for (name, view) in tensors.iter() { - let dtype = view.dtype(); - let dtype = match candle_core::DType::try_from(dtype) { - Ok(dtype) => format!("{dtype:?}"), - Err(_) => format!("{dtype:?}"), - }; - let shape = view.shape(); - println!("{name}: [{shape:?}; {dtype}]") - } - } - Format::Pth => { - let mut tensors = candle_core::pickle::read_pth_tensor_info(file, verbose, None)?; - tensors.sort_by(|a, b| a.name.cmp(&b.name)); - for tensor_info in tensors.iter() { - println!( - "{}: [{:?}; {:?}]", - tensor_info.name, - tensor_info.layout.shape(), - tensor_info.dtype, - ); - if verbose { - println!(" {:?}", tensor_info); - } - } - } - Format::Pickle => { - let file = std::fs::File::open(file)?; - let mut reader = std::io::BufReader::new(file); - let mut stack = candle_core::pickle::Stack::empty(); - stack.read_loop(&mut reader)?; - for (i, obj) in stack.stack().iter().enumerate() { - println!("{i} {obj:?}"); - } - } - Format::Ggml => { - let mut file = std::fs::File::open(file)?; - let content = candle_core::quantized::ggml_file::Content::read(&mut file, device)?; - let mut tensors = content.tensors.into_iter().collect::>(); - tensors.sort_by(|a, b| a.0.cmp(&b.0)); - for (name, qtensor) in tensors.iter() { - println!("{name}: [{:?}; {:?}]", qtensor.shape(), qtensor.dtype()); - } - } - Format::Gguf => { - let mut file = std::fs::File::open(file)?; - let content = gguf_file::Content::read(&mut file)?; - if verbose { - let mut metadata = content.metadata.into_iter().collect::>(); - metadata.sort_by(|a, b| a.0.cmp(&b.0)); - println!("metadata entries ({})", metadata.len()); - for (key, value) in metadata.iter() { - println!(" {key}: {value:?}"); - } - } - let mut tensors = content.tensor_infos.into_iter().collect::>(); - tensors.sort_by(|a, b| a.0.cmp(&b.0)); - for (name, info) in tensors.iter() { - println!("{name}: [{:?}; {:?}]", info.shape, info.ggml_dtype); - } - } - } - Ok(()) -} - -fn run_quantize_safetensors( - in_files: &[std::path::PathBuf], - out_file: std::path::PathBuf, - q: Quantization, -) -> Result<()> { - let mut out_file = std::fs::File::create(out_file)?; - let mut tensors = std::collections::HashMap::new(); - for in_file in in_files.iter() { - let in_tensors = candle_core::safetensors::load(in_file, &Device::Cpu)?; - tensors.extend(in_tensors) - } - println!("tensors: {}", tensors.len()); - - let dtype = q.dtype(); - let block_size = dtype.block_size(); - - let qtensors = tensors - .into_par_iter() - .map(|(name, tensor)| { - let should_quantize = tensor.rank() == 2 && tensor.dim(1)? % block_size == 0; - println!(" quantizing {name} {tensor:?} {should_quantize}"); - let tensor = if should_quantize { - QTensor::quantize(&tensor, dtype)? - } else { - QTensor::quantize(&tensor, GgmlDType::F32)? - }; - Ok((name, tensor)) - }) - .collect::>>()?; - let qtensors = qtensors - .iter() - .map(|(k, v)| (k.as_str(), v)) - .collect::>(); - gguf_file::write(&mut out_file, &[], &qtensors)?; - Ok(()) -} - -fn run_dequantize( - in_file: std::path::PathBuf, - out_file: std::path::PathBuf, - device: &Device, -) -> Result<()> { - let mut in_file = std::fs::File::open(in_file)?; - let content = gguf_file::Content::read(&mut in_file)?; - let mut tensors = std::collections::HashMap::new(); - for (tensor_name, _) in content.tensor_infos.iter() { - let tensor = content.tensor(&mut in_file, tensor_name, device)?; - let tensor = tensor.dequantize(device)?; - tensors.insert(tensor_name.to_string(), tensor); - } - candle_core::safetensors::save(&tensors, out_file)?; - Ok(()) -} - -fn run_quantize( - in_files: &[std::path::PathBuf], - out_file: std::path::PathBuf, - q: Quantization, - qmode: QuantizationMode, - device: &Device, -) -> Result<()> { - if in_files.is_empty() { - candle_core::bail!("no specified input files") - } - if let Some(extension) = out_file.extension() { - if extension == "safetensors" { - candle_core::bail!("the generated file cannot use the safetensors extension") - } - } - if let Some(extension) = in_files[0].extension() { - if extension == "safetensors" { - return run_quantize_safetensors(in_files, out_file, q); - } - } - - if in_files.len() != 1 { - candle_core::bail!("only a single in-file can be used when quantizing gguf files") - } - - // Open the out file early so as to fail directly on missing directories etc. - let mut out_file = std::fs::File::create(out_file)?; - let mut in_ = std::fs::File::open(&in_files[0])?; - let content = gguf_file::Content::read(&mut in_)?; - println!("tensors: {}", content.tensor_infos.len()); - - let dtype = q.dtype(); - let qtensors = content - .tensor_infos - .par_iter() - .map(|(name, _)| { - println!(" quantizing {name}"); - let mut in_file = std::fs::File::open(&in_files[0])?; - let tensor = content.tensor(&mut in_file, name, device)?; - let tensor = qmode.quantize(name, tensor, dtype)?; - Ok((name, tensor)) - }) - .collect::>>()?; - let qtensors = qtensors - .iter() - .map(|(k, v)| (k.as_str(), v)) - .collect::>(); - - let metadata = content - .metadata - .iter() - .map(|(k, v)| (k.as_str(), v)) - .collect::>(); - gguf_file::write(&mut out_file, metadata.as_slice(), &qtensors)?; - Ok(()) -} - -fn main() -> anyhow::Result<()> { - let args = Args::parse(); - let device = Device::Cpu; - match args.command { - Command::Ls { - files, - format, - verbose, - } => { - let multiple_files = files.len() > 1; - for file in files.iter() { - if multiple_files { - println!("--- {file:?} ---"); - } - run_ls(file, format.clone(), verbose, &device)? - } - } - Command::Quantize { - in_file, - out_file, - quantization, - mode, - } => run_quantize(&in_file, out_file, quantization, mode, &device)?, - Command::Dequantize { in_file, out_file } => run_dequantize(in_file, out_file, &device)?, - } - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/accelerate.rs b/v2/dependency-patches/candle-core-0.4.1/src/accelerate.rs deleted file mode 100644 index d371d3b33b..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/accelerate.rs +++ /dev/null @@ -1,476 +0,0 @@ -#![allow(dead_code)] -use libc::{c_char, c_double, c_float, c_int, c_long, c_ulong}; - -mod ffi { - use super::*; - extern "C" { - // It would be nice to be able to switch to the NEWLAPACK version of the function but this - // seems to trigger some link error. Available function names can be seen here: - // /Library/Developer/CommandLineTools/SDKs/MacOSX13.3.sdk/System/Library/Frameworks/Accelerate.framework/Versions/A/Accelerate.tbd - #[link_name = "sgemm_"] - pub fn sgemm_ffi( - transa: *const c_char, - transb: *const c_char, - m: *const c_int, - n: *const c_int, - k: *const c_int, - alpha: *const c_float, - a: *const c_float, - lda: *const c_int, - b: *const c_float, - ldb: *const c_int, - beta: *const c_float, - c: *mut c_float, - ldc: *const c_int, - ); - #[link_name = "dgemm_"] - pub fn dgemm_ffi( - transa: *const c_char, - transb: *const c_char, - m: *const c_int, - n: *const c_int, - k: *const c_int, - alpha: *const c_double, - a: *const c_double, - lda: *const c_int, - b: *const c_double, - ldb: *const c_int, - beta: *const c_double, - c: *mut c_double, - ldc: *const c_int, - ); - - pub fn vvexpf(dst: *mut c_float, src: *const c_float, len: *const c_int); - pub fn vvexp(dst: *mut c_double, src: *const c_double, len: *const c_int); - pub fn vvsqrtf(dst: *mut c_float, src: *const c_float, len: *const c_int); - pub fn vvsqrt(dst: *mut c_double, src: *const c_double, len: *const c_int); - pub fn vvsinf(dst: *mut c_float, src: *const c_float, len: *const c_int); - pub fn vvsin(dst: *mut c_double, src: *const c_double, len: *const c_int); - pub fn vvcosf(dst: *mut c_float, src: *const c_float, len: *const c_int); - pub fn vvcos(dst: *mut c_double, src: *const c_double, len: *const c_int); - pub fn vvlogf(dst: *mut c_float, src: *const c_float, len: *const c_int); - pub fn vvlog(dst: *mut c_double, src: *const c_double, len: *const c_int); - pub fn vvtanhf(dst: *mut c_float, src: *const c_float, len: *const c_int); - pub fn vvtanh(dst: *mut c_double, src: *const c_double, len: *const c_int); - - pub fn vDSP_vaddD( - _: *const c_double, - _: c_long, - _: *const c_double, - _: c_long, - _: *mut c_double, - _: c_long, - _: c_ulong, - ); - pub fn vDSP_vadd( - _: *const c_float, - _: c_long, - _: *const c_float, - _: c_long, - _: *mut c_float, - _: c_long, - _: c_ulong, - ); - pub fn vDSP_vsubD( - _: *const c_double, - _: c_long, - _: *const c_double, - _: c_long, - _: *mut c_double, - _: c_long, - _: c_ulong, - ); - pub fn vDSP_vsub( - _: *const c_float, - _: c_long, - _: *const c_float, - _: c_long, - _: *mut c_float, - _: c_long, - _: c_ulong, - ); - pub fn vDSP_vmulD( - _: *const c_double, - _: c_long, - _: *const c_double, - _: c_long, - _: *mut c_double, - _: c_long, - _: c_ulong, - ); - pub fn vDSP_vmul( - _: *const c_float, - _: c_long, - _: *const c_float, - _: c_long, - _: *mut c_float, - _: c_long, - _: c_ulong, - ); - pub fn vDSP_vdivD( - _: *const c_double, - _: c_long, - _: *const c_double, - _: c_long, - _: *mut c_double, - _: c_long, - _: c_ulong, - ); - pub fn vDSP_vdiv( - _: *const c_float, - _: c_long, - _: *const c_float, - _: c_long, - _: *mut c_float, - _: c_long, - _: c_ulong, - ); - pub fn vDSP_vminD( - _: *const c_double, - _: c_long, - _: *const c_double, - _: c_long, - _: *mut c_double, - _: c_long, - _: c_ulong, - ); - pub fn vDSP_vmin( - _: *const c_float, - _: c_long, - _: *const c_float, - _: c_long, - _: *mut c_float, - _: c_long, - _: c_ulong, - ); - pub fn vDSP_vmaxD( - _: *const c_double, - _: c_long, - _: *const c_double, - _: c_long, - _: *mut c_double, - _: c_long, - _: c_ulong, - ); - pub fn vDSP_vmax( - _: *const c_float, - _: c_long, - _: *const c_float, - _: c_long, - _: *mut c_float, - _: c_long, - _: c_ulong, - ); - } -} - -#[allow(clippy::too_many_arguments)] -#[inline] -pub unsafe fn sgemm( - transa: u8, - transb: u8, - m: i32, - n: i32, - k: i32, - alpha: f32, - a: &[f32], - lda: i32, - b: &[f32], - ldb: i32, - beta: f32, - c: &mut [f32], - ldc: i32, -) { - ffi::sgemm_ffi( - &(transa as c_char), - &(transb as c_char), - &m, - &n, - &k, - &alpha, - a.as_ptr(), - &lda, - b.as_ptr(), - &ldb, - &beta, - c.as_mut_ptr(), - &ldc, - ) -} - -#[allow(clippy::too_many_arguments)] -#[inline] -pub unsafe fn dgemm( - transa: u8, - transb: u8, - m: i32, - n: i32, - k: i32, - alpha: f64, - a: &[f64], - lda: i32, - b: &[f64], - ldb: i32, - beta: f64, - c: &mut [f64], - ldc: i32, -) { - ffi::dgemm_ffi( - &(transa as c_char), - &(transb as c_char), - &m, - &n, - &k, - &alpha, - a.as_ptr(), - &lda, - b.as_ptr(), - &ldb, - &beta, - c.as_mut_ptr(), - &ldc, - ) -} - -#[inline] -pub fn vs_exp(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvexpf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} - -#[inline] -pub fn vd_exp(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvexp(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} - -#[inline] -pub fn vs_sqrt(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvsqrtf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} - -#[inline] -pub fn vd_sqrt(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvsqrt(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} - -#[inline] -pub fn vs_sin(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvsinf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} - -#[inline] -pub fn vd_sin(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvsin(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} -#[inline] -pub fn vs_cos(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvcosf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} - -#[inline] -pub fn vd_cos(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvcos(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} -#[inline] -pub fn vs_tanh(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvtanhf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} - -#[inline] -pub fn vd_tanh(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvtanh(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} - -#[inline] -pub fn vs_ln(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvlogf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} - -#[inline] -pub fn vd_ln(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvlog(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} - -#[inline] -pub fn vs_sqr(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - y.iter_mut().zip(a.iter()).for_each(|(y, a)| *y = *a * *a) -} - -#[inline] -pub fn vd_sqr(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - y.iter_mut().zip(a.iter()).for_each(|(y, a)| *y = *a * *a) -} - -#[inline] -pub fn vs_tanh_inplace(y: &mut [f32]) { - unsafe { ffi::vvtanhf(y.as_mut_ptr(), y.as_ptr(), &(y.len() as i32)) } -} - -#[inline] -pub fn vd_tanh_inplace(y: &mut [f64]) { - unsafe { ffi::vvtanh(y.as_mut_ptr(), y.as_ptr(), &(y.len() as i32)) } -} - -#[inline] -pub fn vs_exp_inplace(y: &mut [f32]) { - unsafe { ffi::vvexpf(y.as_mut_ptr(), y.as_ptr(), &(y.len() as i32)) } -} - -#[inline] -pub fn vd_exp_inplace(y: &mut [f64]) { - unsafe { ffi::vvexp(y.as_mut_ptr(), y.as_ptr(), &(y.len() as i32)) } -} - -#[inline] -pub fn vs_gelu(vs: &[f32], ys: &mut [f32]) { - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = (2.0f32 / std::f32::consts::PI).sqrt() * v * (1.0 + 0.044715 * v * v) - } - vs_tanh_inplace(ys); - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = 0.5 * v * (1.0 + *y) - } -} - -#[inline] -pub fn vd_gelu(vs: &[f64], ys: &mut [f64]) { - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = (2.0f64 / std::f64::consts::PI).sqrt() * v * (1.0 + 0.044715 * v * v) - } - vd_tanh_inplace(ys); - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = 0.5 * v * (1.0 + *y) - } -} - -#[inline] -pub fn vs_silu(vs: &[f32], ys: &mut [f32]) { - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = -v - } - vs_exp_inplace(ys); - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = v / (1.0 + *y) - } -} - -#[inline] -pub fn vd_silu(vs: &[f64], ys: &mut [f64]) { - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = -v - } - vd_exp_inplace(ys); - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = v / (1.0 + *y) - } -} - -macro_rules! binary_op { - ($fn_name:ident, $ty:ty, $accelerate_name:ident) => { - #[inline] - pub fn $fn_name(a: &[$ty], b: &[$ty], y: &mut [$ty]) { - let a_len = a.len(); - let b_len = b.len(); - let y_len = y.len(); - if a_len != y_len || b_len != y_len { - panic!( - "{} a,b,y len mismatch {a_len} {b_len} {y_len}", - stringify!($fn_name) - ); - } - unsafe { - // Weird quirk of accelerate, the rhs comes before the lhs. - ffi::$accelerate_name( - b.as_ptr(), - 1, - a.as_ptr(), - 1, - y.as_mut_ptr(), - 1, - a_len as u64, - ) - } - } - }; -} -binary_op!(vs_add, f32, vDSP_vadd); -binary_op!(vd_add, f64, vDSP_vaddD); -binary_op!(vs_sub, f32, vDSP_vsub); -binary_op!(vd_sub, f64, vDSP_vsubD); -binary_op!(vs_mul, f32, vDSP_vmul); -binary_op!(vd_mul, f64, vDSP_vmulD); -binary_op!(vs_div, f32, vDSP_vdiv); -binary_op!(vd_div, f64, vDSP_vdivD); -binary_op!(vs_max, f32, vDSP_vmax); -binary_op!(vd_max, f64, vDSP_vmaxD); -binary_op!(vs_min, f32, vDSP_vmin); -binary_op!(vd_min, f64, vDSP_vminD); diff --git a/v2/dependency-patches/candle-core-0.4.1/src/backend.rs b/v2/dependency-patches/candle-core-0.4.1/src/backend.rs deleted file mode 100644 index 2125af69e7..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/backend.rs +++ /dev/null @@ -1,124 +0,0 @@ -use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; -use crate::{CpuStorage, DType, Layout, Result, Shape}; - -pub trait BackendStorage: Sized { - type Device: BackendDevice; - - fn try_clone(&self, _: &Layout) -> Result; - - fn dtype(&self) -> DType; - - fn device(&self) -> &Self::Device; - - // Maybe this should return a Cow instead so that no copy is done on the cpu case. - fn to_cpu_storage(&self) -> Result; - - fn affine(&self, _: &Layout, _: f64, _: f64) -> Result; - - fn powf(&self, _: &Layout, _: f64) -> Result; - - fn elu(&self, _: &Layout, _: f64) -> Result; - - fn reduce_op(&self, _: ReduceOp, _: &Layout, _: &[usize]) -> Result; - - fn cmp(&self, _: CmpOp, _: &Self, _: &Layout, _: &Layout) -> Result; - - fn to_dtype(&self, _: &Layout, _: DType) -> Result; - - fn unary_impl(&self, _: &Layout) -> Result; - - fn binary_impl(&self, _: &Self, _: &Layout, _: &Layout) -> Result; - - fn where_cond(&self, _: &Layout, _: &Self, _: &Layout, _: &Self, _: &Layout) -> Result; - - fn conv1d( - &self, - _l: &Layout, - _kernel: &Self, - _kernel_l: &Layout, - _params: &crate::conv::ParamsConv1D, - ) -> Result; - - fn conv_transpose1d( - &self, - _l: &Layout, - _kernel: &Self, - _kernel_l: &Layout, - _params: &crate::conv::ParamsConvTranspose1D, - ) -> Result; - - fn conv2d( - &self, - _l: &Layout, - _kernel: &Self, - _kernel_l: &Layout, - _params: &crate::conv::ParamsConv2D, - ) -> Result; - - fn conv_transpose2d( - &self, - _l: &Layout, - _kernel: &Self, - _kernel_l: &Layout, - _params: &crate::conv::ParamsConvTranspose2D, - ) -> Result; - - fn avg_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result; - fn max_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result; - fn upsample_nearest1d(&self, _: &Layout, _: usize) -> Result; - fn upsample_nearest2d(&self, _: &Layout, _: usize, _: usize) -> Result; - - fn gather(&self, _: &Layout, _: &Self, _: &Layout, _: usize) -> Result; - fn scatter_add( - &self, - _: &Layout, - _: &Self, - _: &Layout, - _: &Self, - _: &Layout, - _: usize, - ) -> Result; - fn index_select(&self, _: &Self, _: &Layout, _: &Layout, _: usize) -> Result; - fn index_add( - &self, - _: &Layout, - _: &Self, - _: &Layout, - _: &Self, - _: &Layout, - _: usize, - ) -> Result; - - fn matmul( - &self, - _: &Self, - _: (usize, usize, usize, usize), - _: &Layout, - _: &Layout, - ) -> Result; - - fn copy_strided_src(&self, _: &mut Self, _: usize, _: &Layout) -> Result<()>; -} - -pub trait BackendDevice: Sized + std::fmt::Debug + Clone { - type Storage: BackendStorage; - - // TODO: Make the usize generic and part of a generic DeviceLocation. - fn new(_: usize) -> Result; - - fn location(&self) -> crate::DeviceLocation; - - fn same_device(&self, _: &Self) -> bool; - - fn zeros_impl(&self, _shape: &Shape, _dtype: DType) -> Result; - - fn ones_impl(&self, _shape: &Shape, _dtype: DType) -> Result; - - fn storage_from_cpu_storage(&self, _: &CpuStorage) -> Result; - - fn rand_uniform(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result; - - fn rand_normal(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result; - - fn set_seed(&self, _: u64) -> Result<()>; -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/backprop.rs b/v2/dependency-patches/candle-core-0.4.1/src/backprop.rs deleted file mode 100644 index 2a1db58a5e..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/backprop.rs +++ /dev/null @@ -1,728 +0,0 @@ -use crate::op::{BinaryOp, Op, ReduceOp, UnaryOp}; -use crate::{Error, Result, Tensor, TensorId}; -use std::collections::HashMap; - -// arg has been reduced to node via reduce_dims, expand it back to arg. -// This has to handle keepdims. -fn broadcast_back(arg: &Tensor, node: &Tensor, reduced_dims: &[usize]) -> Result { - if arg.rank() == node.rank() { - // keepdim = true - node.broadcast_as(arg.shape()) - } else { - // keepdim = false - // first expand the reduced dims. - node.reshape(reduced_dims)?.broadcast_as(arg.shape()) - } -} - -thread_local! { - static CANDLE_GRAD_DO_NOT_DETACH: bool = { - match std::env::var("CANDLE_GRAD_DO_NOT_DETACH") { - Ok(s) => { - !s.is_empty() && s != "0" - }, - Err(_) => false, - } - } -} - -impl Tensor { - /// Return all the nodes that lead to this value in a topologically sorted vec, the first - /// elements having dependencies on the latter ones, e.g. the first element if any is the - /// argument. - /// This assumes that the op graph is a DAG. - fn sorted_nodes(&self) -> Vec<&Tensor> { - // The vec of sorted nodes is passed as an owned value rather than a mutable reference - // to get around some lifetime limitations. - fn walk<'a>( - node: &'a Tensor, - nodes: Vec<&'a Tensor>, - already_seen: &mut HashMap, - ) -> (bool, Vec<&'a Tensor>) { - if let Some(&tg) = already_seen.get(&node.id()) { - return (tg, nodes); - } - let mut track_grad = false; - let mut nodes = if node.is_variable() { - // Do not call recursively on the "leaf" nodes. - track_grad = true; - nodes - } else if node.dtype().is_int() { - nodes - } else if let Some(op) = node.op() { - match op { - Op::IndexAdd(t1, t2, t3, _) - | Op::ScatterAdd(t1, t2, t3, _) - | Op::CustomOp3(t1, t2, t3, _) - | Op::WhereCond(t1, t2, t3) => { - let (tg, nodes) = walk(t1, nodes, already_seen); - track_grad |= tg; - let (tg, nodes) = walk(t2, nodes, already_seen); - track_grad |= tg; - let (tg, nodes) = walk(t3, nodes, already_seen); - track_grad |= tg; - nodes - } - Op::Conv1D { - arg: lhs, - kernel: rhs, - .. - } - | Op::ConvTranspose1D { - arg: lhs, - kernel: rhs, - .. - } - | Op::Conv2D { - arg: lhs, - kernel: rhs, - .. - } - | Op::ConvTranspose2D { - arg: lhs, - kernel: rhs, - .. - } - | Op::CustomOp2(lhs, rhs, _) - | Op::Binary(lhs, rhs, _) - | Op::Gather(lhs, rhs, _) - | Op::IndexSelect(lhs, rhs, _) - | Op::Matmul(lhs, rhs) - | Op::SliceScatter0(lhs, rhs, _) => { - let (tg, nodes) = walk(lhs, nodes, already_seen); - track_grad |= tg; - let (tg, nodes) = walk(rhs, nodes, already_seen); - track_grad |= tg; - nodes - } - Op::Cat(args, _) => args.iter().fold(nodes, |nodes, arg| { - let (tg, nodes) = walk(arg, nodes, already_seen); - track_grad |= tg; - nodes - }), - Op::Affine { arg, mul, .. } => { - if *mul == 0. { - nodes - } else { - let (tg, nodes) = walk(arg, nodes, already_seen); - track_grad |= tg; - nodes - } - } - Op::Unary(_node, UnaryOp::Ceil) - | Op::Unary(_node, UnaryOp::Floor) - | Op::Unary(_node, UnaryOp::Round) => nodes, - Op::Reshape(node) - | Op::UpsampleNearest1D { arg: node, .. } - | Op::UpsampleNearest2D { arg: node, .. } - | Op::AvgPool2D { arg: node, .. } - | Op::MaxPool2D { arg: node, .. } - | Op::Copy(node) - | Op::Broadcast(node) - | Op::Cmp(node, _) - | Op::Reduce(node, ReduceOp::Min | ReduceOp::Sum | ReduceOp::Max, _) - | Op::ToDevice(node) - | Op::Transpose(node, _, _) - | Op::Permute(node, _) - | Op::Narrow(node, _, _, _) - | Op::Unary(node, _) - | Op::Elu(node, _) - | Op::Powf(node, _) - | Op::CustomOp1(node, _) => { - let (tg, nodes) = walk(node, nodes, already_seen); - track_grad |= tg; - nodes - } - Op::ToDType(node) => { - if node.dtype().is_float() { - let (tg, nodes) = walk(node, nodes, already_seen); - track_grad |= tg; - nodes - } else { - nodes - } - } - Op::Reduce(_, ReduceOp::ArgMin | ReduceOp::ArgMax, _) => nodes, - } - } else { - nodes - }; - already_seen.insert(node.id(), track_grad); - if track_grad { - nodes.push(node); - } - (track_grad, nodes) - } - let (_tg, mut nodes) = walk(self, vec![], &mut HashMap::new()); - nodes.reverse(); - nodes - } - - pub fn backward(&self) -> Result { - let sorted_nodes = self.sorted_nodes(); - let mut grads = GradStore::new(); - grads.insert(self, self.ones_like()?.contiguous()?); - for node in sorted_nodes.iter() { - if node.is_variable() { - continue; - } - let grad = grads - .remove(node) - .expect("candle internal error - grad not populated"); - // https://github.com/huggingface/candle/issues/1241 - // Ideally, we would make these operations in place where possible to ensure that we - // do not have to allocate too often. Here we just call `.detach` to avoid computing - // the backprop graph of the backprop itself. This would be an issue for second order - // derivatives but these are out of scope at the moment. - let do_not_detach = CANDLE_GRAD_DO_NOT_DETACH.with(|b| *b); - let grad = if do_not_detach { grad } else { grad.detach() }; - if let Some(op) = node.op() { - match op { - Op::Binary(lhs, rhs, BinaryOp::Add) => { - let lhs_sum_grad = grads.or_insert(lhs)?; - *lhs_sum_grad = lhs_sum_grad.add(&grad)?; - let rhs_sum_grad = grads.or_insert(rhs)?; - *rhs_sum_grad = rhs_sum_grad.add(&grad)?; - } - Op::Binary(lhs, rhs, BinaryOp::Sub) => { - let lhs_sum_grad = grads.or_insert(lhs)?; - *lhs_sum_grad = lhs_sum_grad.add(&grad)?; - let rhs_sum_grad = grads.or_insert(rhs)?; - *rhs_sum_grad = rhs_sum_grad.sub(&grad)?; - } - Op::Binary(lhs, rhs, BinaryOp::Mul) => { - let lhs_grad = grad.mul(rhs)?; - let lhs_sum_grad = grads.or_insert(lhs)?; - *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?; - let rhs_grad = grad.mul(lhs)?; - let rhs_sum_grad = grads.or_insert(rhs)?; - *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?; - } - Op::Binary(lhs, rhs, BinaryOp::Div) => { - let lhs_grad = grad.div(rhs)?; - let lhs_sum_grad = grads.or_insert(lhs)?; - *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?; - let rhs_grad = grad.mul(lhs)?.div(&rhs.sqr()?)?; - let rhs_sum_grad = grads.or_insert(rhs)?; - *rhs_sum_grad = rhs_sum_grad.sub(&rhs_grad)?; - } - Op::Binary(lhs, rhs, BinaryOp::Minimum) - | Op::Binary(lhs, rhs, BinaryOp::Maximum) => { - let mask_lhs = node.eq(lhs)?.to_dtype(grad.dtype())?; - let mask_rhs = node.eq(rhs)?.to_dtype(grad.dtype())?; - - // If both masks are 1 one the same point, we want to scale the - // gradient by 0.5 rather than 1. - let lhs_grad = mask_lhs.mul(&grad)?.div(&(&mask_rhs + 1.)?)?; - let lhs_sum_grad = grads.or_insert(lhs)?; - *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?; - - let rhs_grad = mask_rhs.mul(&grad)?.div(&(&mask_lhs + 1.)?)?; - let rhs_sum_grad = grads.or_insert(rhs)?; - *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?; - } - Op::WhereCond(pred, t, f) => { - let zeros = grad.zeros_like()?; - let t_sum_grad = grads.or_insert(t)?; - let t_grad = pred.where_cond(&grad, &zeros)?; - *t_sum_grad = t_sum_grad.add(&t_grad)?; - let f_sum_grad = grads.or_insert(f)?; - let f_grad = pred.where_cond(&zeros, &grad)?; - *f_sum_grad = f_sum_grad.add(&f_grad)?; - } - Op::Conv1D { - arg, - kernel, - padding, - stride, - dilation, - } => { - // The output height for conv_transpose1d is: - // (l_in - 1) * stride - 2 * padding + dilation * (k_size - 1) + out_padding + 1 - let grad_l_in = grad.dim(2)?; - let k_size = kernel.dim(2)?; - let out_size = - (grad_l_in - 1) * stride + dilation * (k_size - 1) + 1 - 2 * padding; - let out_padding = arg.dim(2)? - out_size; - let grad_arg = grad.conv_transpose1d( - kernel, - *padding, - out_padding, - *stride, - *dilation, - /* groups */ 1, - )?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&grad_arg)?; - - let grad_kernel = arg - .transpose(0, 1)? - .conv1d(&grad.transpose(0, 1)?, *padding, *dilation, *stride, 1)? - .transpose(0, 1)?; - let sum_grad = grads.or_insert(kernel)?; - let (_, _, k0) = kernel.dims3()?; - let (_, _, g_k0) = grad_kernel.dims3()?; - let grad_kernel = if g_k0 != k0 { - grad_kernel.narrow(2, 0, k0)? - } else { - grad_kernel - }; - *sum_grad = sum_grad.add(&grad_kernel)?; - } - Op::Conv2D { - arg, - kernel, - padding, - stride, - dilation, - } => { - // The output height for conv_transpose2d is: - // (i_h - 1) * stride - 2 * padding + dilation * (k_h - 1) + out_padding + 1 - let grad_h = grad.dim(2)?; - let k_h = kernel.dim(2)?; - let out_size = - (grad_h - 1) * stride + dilation * (k_h - 1) + 1 - 2 * padding; - let out_padding = arg.dim(2)? - out_size; - let grad_arg = grad.conv_transpose2d( - kernel, - *padding, - out_padding, - *stride, - *dilation, - )?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&grad_arg)?; - - let grad_kernel = arg - .transpose(0, 1)? - .conv2d(&grad.transpose(0, 1)?, *padding, *dilation, *stride, 1)? - .transpose(0, 1)?; - let sum_grad = grads.or_insert(kernel)?; - let (_, _, k0, k1) = kernel.dims4()?; - let (_, _, g_k0, g_k1) = grad_kernel.dims4()?; - let grad_kernel = if g_k0 != k0 || g_k1 != k1 { - grad_kernel.narrow(2, 0, k0)?.narrow(3, 0, k1)? - } else { - grad_kernel - }; - *sum_grad = sum_grad.add(&grad_kernel)?; - } - Op::ConvTranspose1D { .. } => Err(Error::BackwardNotSupported { - op: "conv-transpose1d", - })?, - Op::ConvTranspose2D { .. } => Err(Error::BackwardNotSupported { - op: "conv-transpose2d", - })?, - Op::AvgPool2D { - arg, - kernel_size, - stride, - } => { - if kernel_size != stride { - crate::bail!("backward not supported for avgpool2d if ksize {kernel_size:?} != stride {stride:?}") - } - let (_n, _c, h, w) = arg.dims4()?; - let grad_arg = grad.upsample_nearest2d(h, w)?; - let grad_arg = - (grad_arg * (1f64 / (kernel_size.0 * kernel_size.1) as f64))?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&grad_arg)?; - } - Op::MaxPool2D { - arg, - kernel_size, - stride, - } => { - if kernel_size != stride { - crate::bail!("backward not supported for maxpool2d if ksize {kernel_size:?} != stride {stride:?}") - } - let (_n, _c, h, w) = arg.dims4()?; - // For computing the max-pool gradient, we compute a mask where a 1 means - // that the element is the maximum, then we apply this mask to the - // upsampled gradient (taking into account that multiple max may exist so - // we scale the gradient for this case). - let node_upsampled = node.upsample_nearest2d(h, w)?; - let mask = arg.eq(&node_upsampled)?.to_dtype(arg.dtype())?; - let avg = mask.avg_pool2d_with_stride(*kernel_size, *stride)?; - let grad_arg = ((grad * avg)?.upsample_nearest2d(h, w)? * mask)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&grad_arg)?; - } - Op::UpsampleNearest1D { arg, target_size } => { - let (_n, c, size) = arg.dims3()?; - if target_size % size != 0 { - crate::bail!("backward not supported for non integer upscaling factors") - } - let scale = target_size / size; - - let kernel = Tensor::ones((c, 1, scale), arg.dtype(), arg.device())?; - let conv_sum = grad.conv1d(&kernel, 0, scale, 1, c)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = conv_sum; - } - Op::UpsampleNearest2D { - arg, - target_h, - target_w, - } => { - let (_n, c, h, w) = arg.dims4()?; - if target_h % h != 0 || target_w % w != 0 { - crate::bail!("backward not supported for non integer upscaling factors") - } - let scale_h = target_h / h; - let scale_w = target_w / w; - - if scale_h != scale_w { - crate::bail!("backward not supported for non uniform upscaling factors") - }; - let kernel = - Tensor::ones((c, 1, scale_h, scale_w), arg.dtype(), arg.device())?; - let conv_sum = grad.conv2d(&kernel, 0, scale_h, 1, c)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = conv_sum; - } - Op::SliceScatter0(lhs, rhs, start_rhs) => { - let rhs_sum_grad = grads.or_insert(rhs)?; - let rhs_grad = grad.narrow(0, *start_rhs, rhs.dim(0)?)?; - *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?; - - let lhs_sum_grad = grads.or_insert(lhs)?; - let lhs_grad = grad.slice_scatter0(&rhs.zeros_like()?, *start_rhs)?; - *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)? - } - Op::Gather(arg, indexes, dim) => { - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.scatter_add(indexes, &grad, *dim)?; - } - Op::ScatterAdd(init, indexes, src, dim) => { - let init_sum_grad = grads.or_insert(init)?; - *init_sum_grad = init_sum_grad.add(&grad)?; - - let src_grad = grad.gather(indexes, *dim)?; - let src_sum_grad = grads.or_insert(src)?; - *src_sum_grad = src_sum_grad.add(&src_grad)?; - } - Op::IndexAdd(init, indexes, src, dim) => { - let init_sum_grad = grads.or_insert(init)?; - *init_sum_grad = init_sum_grad.add(&grad)?; - - let src_grad = grad.index_select(indexes, *dim)?; - let src_sum_grad = grads.or_insert(src)?; - *src_sum_grad = src_sum_grad.add(&src_grad)?; - } - Op::IndexSelect(arg, indexes, dim) => { - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.index_add(indexes, &grad, *dim)?; - } - Op::Matmul(lhs, rhs) => { - // Skipping checks, the op went ok, we can skip - // the matmul size checks for now. - - let lhs_grad = grad.matmul(&rhs.t()?)?; - let lhs_sum_grad = grads.or_insert(lhs)?; - *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?; - - let rhs_grad = lhs.t()?.matmul(&grad)?; - let rhs_sum_grad = grads.or_insert(rhs)?; - *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?; - } - Op::Cat(args, dim) => { - let mut start_idx = 0; - for arg in args { - let len = arg.dims()[*dim]; - let arg_grad = grad.narrow(*dim, start_idx, len)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&arg_grad)?; - start_idx += len; - } - } - Op::Broadcast(arg) => { - let arg_dims = arg.dims(); - let node_dims = node.dims(); - // The number of dims that have been inserted on the left. - let left_dims = node_dims.len() - arg_dims.len(); - let mut sum_dims: Vec = (0..left_dims).collect(); - for (dim, (node_dim, arg_dim)) in node_dims[left_dims..] - .iter() - .zip(arg_dims.iter()) - .enumerate() - { - if node_dim != arg_dim { - sum_dims.push(dim + left_dims) - } - } - - let mut arg_grad = grad.sum_keepdim(sum_dims.as_slice())?; - for _i in 0..left_dims { - arg_grad = arg_grad.squeeze(0)? - } - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&arg_grad.broadcast_as(sum_grad.dims())?)?; - } - Op::Reduce(arg, ReduceOp::Sum, reduced_dims) => { - let grad = broadcast_back(arg, &grad, reduced_dims)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&grad)?; - } - Op::Cmp(_args, _) => {} - Op::Reduce(arg, ReduceOp::Max, reduced_dims) => { - let node = broadcast_back(arg, node, reduced_dims)?; - let grad = broadcast_back(arg, &grad, reduced_dims)?; - let grad = node.eq(arg)?.to_dtype(grad.dtype())?.mul(&grad)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&grad.broadcast_as(sum_grad.dims())?)?; - } - Op::Reduce(arg, ReduceOp::Min, reduced_dims) => { - let node = broadcast_back(arg, node, reduced_dims)?; - let grad = broadcast_back(arg, &grad, reduced_dims)?; - let grad = node.eq(arg)?.to_dtype(grad.dtype())?.mul(&grad)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&grad.broadcast_as(sum_grad.dims())?)?; - } - Op::ToDType(arg) => { - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&grad.to_dtype(arg.dtype())?)? - } - Op::Copy(arg) => { - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&grad)? - } - Op::Affine { arg, mul, .. } => { - let arg_grad = grad.affine(*mul, 0.)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&arg_grad)? - } - Op::Unary(arg, UnaryOp::Log) => { - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&(grad / arg)?)? - } - Op::Unary(arg, UnaryOp::Sin) => { - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&(&grad * arg.cos())?)? - } - Op::Unary(arg, UnaryOp::Cos) => { - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.sub(&(&grad * arg.sin())?)? - } - Op::Unary(arg, UnaryOp::Tanh) => { - let sum_grad = grads.or_insert(arg)?; - let minus_dtanh = (node.sqr()? - 1.)?; - *sum_grad = sum_grad.sub(&(&grad * &minus_dtanh)?)? - } - Op::Unary(arg, UnaryOp::Abs) => { - let sum_grad = grads.or_insert(arg)?; - let ones = arg.ones_like()?; - let abs_grad = arg.ge(&arg.zeros_like()?)?.where_cond(&ones, &ones.neg()?); - *sum_grad = sum_grad.add(&(&grad * abs_grad)?)? - } - Op::Unary(arg, UnaryOp::Exp) => { - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&(&grad * *node)?)? - } - Op::Unary(arg, UnaryOp::Neg) => { - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.sub(&grad)? - } - Op::Unary(arg, UnaryOp::Recip) => { - let sum_grad = grads.or_insert(arg)?; - let grad = (grad / arg.sqr()?)?; - *sum_grad = sum_grad.sub(&grad)? - } - &Op::Narrow(ref arg, dim, start_idx, len) => { - let arg_dims = arg.dims(); - let left_pad = if start_idx == 0 { - None - } else { - let mut dims = arg_dims.to_vec(); - dims[dim] = start_idx; - Some(Tensor::zeros(dims, grad.dtype(), grad.device())?) - }; - let right_pad = arg_dims[dim] - start_idx - len; - let right_pad = if right_pad == 0 { - None - } else { - let mut dims = arg_dims.to_vec(); - dims[dim] = right_pad; - Some(Tensor::zeros(dims, grad.dtype(), grad.device())?) - }; - let arg_grad = match (left_pad, right_pad) { - (None, None) => grad, - (Some(l), None) => Tensor::cat(&[&l, &grad], dim)?, - (None, Some(r)) => Tensor::cat(&[&grad, &r], dim)?, - (Some(l), Some(r)) => Tensor::cat(&[&l, &grad, &r], dim)?, - }; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&arg_grad)? - } - Op::Reduce(_, ReduceOp::ArgMin, _) => {} - Op::Reduce(_, ReduceOp::ArgMax, _) => {} - Op::Reshape(arg) => { - let arg_grad = grad.reshape(arg.dims())?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&arg_grad)? - } - Op::Unary(_, UnaryOp::Ceil) => Err(Error::BackwardNotSupported { op: "ceil" })?, - Op::Unary(_, UnaryOp::Floor) => { - Err(Error::BackwardNotSupported { op: "floor" })? - } - Op::Unary(_, UnaryOp::Round) => { - Err(Error::BackwardNotSupported { op: "round" })? - } - Op::Unary(arg, UnaryOp::Gelu) => { - let sum_grad = grads.or_insert(arg)?; - let cube = arg.powf(3.)?; - let tanh = (0.0356774 * &cube + (0.797885 * arg)?)?.tanh()?; - let gelu_grad = (((0.5 * &tanh)? - + (0.0535161 * cube + (0.398942 * arg)?)? * (1. - tanh.powf(2.)?))? - + 0.5)?; - *sum_grad = sum_grad.add(&(&grad * gelu_grad)?)? - } - Op::Unary(arg, UnaryOp::Erf) => { - let sum_grad = grads.or_insert(arg)?; - // d/dx erf(x) = 2/sqrt(pi) * e^(-x^2) - let erf_grad = - (2. / std::f64::consts::PI.sqrt()) * (arg.sqr()?.neg()?).exp()?; - *sum_grad = sum_grad.add(&(&grad * erf_grad)?)? - } - Op::Unary(arg, UnaryOp::GeluErf) => { - let sum_grad = grads.or_insert(arg)?; - // d/dx gelu_erf(x) = 0.5 + 0.398942 e^(-x^2/2) x + 0.5 erf(x/sqrt(2)) - let neg_half_square = (arg.sqr()?.neg()? / 2.)?; - let scaled_exp_arg = (0.398942 * neg_half_square.exp()? * arg)?; - let arg_scaled_sqrt = (arg / 2f64.sqrt())?; - let erf_scaled_sqrt = (0.5 * arg_scaled_sqrt.erf()?)?; - let gelu_erf_grad = (0.5 + scaled_exp_arg + erf_scaled_sqrt)?; - *sum_grad = sum_grad.add(&(&grad * gelu_erf_grad)?)?; - } - Op::Unary(arg, UnaryOp::Relu) => { - let sum_grad = grads.or_insert(arg)?; - let relu_grad = arg.ge(&arg.zeros_like()?)?.to_dtype(arg.dtype())?; - *sum_grad = sum_grad.add(&(&grad * relu_grad)?)? - } - Op::Unary(arg, UnaryOp::Silu) => { - let sum_grad = grads.or_insert(arg)?; - // d/dx silu = sigmoid(x) * (1 + x * (1 - sigmoid(x))) - let sigmoid_arg = (*node / arg)?; - let silu_grad = (&sigmoid_arg * (1. + (arg * (1. - &sigmoid_arg)?)?)?)?; - *sum_grad = sum_grad.add(&(&grad * silu_grad)?)? - } - Op::Elu(arg, alpha) => { - // d/dx elu(x) = 1 for x > 0, alpha * e^x for x <= 0 - let sum_grad = grads.or_insert(arg)?; - let zeros = arg.zeros_like()?; - let positive_mask = arg.gt(&zeros)?.to_dtype(arg.dtype())?; - let negative_mask = arg.le(&zeros)?.to_dtype(arg.dtype())?; - let negative_exp_mask = ((negative_mask * arg.exp())? * *alpha)?; - let combined_mask = (positive_mask + negative_exp_mask)?; - *sum_grad = sum_grad.add(&(grad * combined_mask)?)? - } - Op::Powf(arg, e) => { - let arg_grad = (&(grad * arg.powf(e - 1.)?)? * *e)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&arg_grad)? - } - Op::CustomOp1(arg, c) => { - if let Some(arg_grad) = c.bwd(arg, node, &grad)? { - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&arg_grad)? - } - } - Op::CustomOp2(arg1, arg2, c) => { - let (arg_grad1, arg_grad2) = c.bwd(arg1, arg2, node, &grad)?; - if let Some(arg_grad1) = arg_grad1 { - let sum_grad = grads.or_insert(arg1)?; - *sum_grad = sum_grad.add(&arg_grad1)? - } - if let Some(arg_grad2) = arg_grad2 { - let sum_grad = grads.or_insert(arg2)?; - *sum_grad = sum_grad.add(&arg_grad2)? - } - } - Op::CustomOp3(arg1, arg2, arg3, c) => { - let (arg_grad1, arg_grad2, arg_grad3) = - c.bwd(arg1, arg2, arg3, node, &grad)?; - if let Some(arg_grad1) = arg_grad1 { - let sum_grad = grads.or_insert(arg1)?; - *sum_grad = sum_grad.add(&arg_grad1)? - } - if let Some(arg_grad2) = arg_grad2 { - let sum_grad = grads.or_insert(arg2)?; - *sum_grad = sum_grad.add(&arg_grad2)? - } - if let Some(arg_grad3) = arg_grad3 { - let sum_grad = grads.or_insert(arg3)?; - *sum_grad = sum_grad.add(&arg_grad3)? - } - } - Op::Unary(arg, UnaryOp::Sqr) => { - let arg_grad = arg.mul(&grad)?.affine(2., 0.)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&arg_grad)? - } - Op::Unary(arg, UnaryOp::Sqrt) => { - let arg_grad = grad.div(node)?.affine(0.5, 0.)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&arg_grad)? - } - Op::ToDevice(arg) => { - let sum_grad = grads.or_insert(arg)?; - let arg_grad = grad.to_device(sum_grad.device())?; - *sum_grad = sum_grad.add(&arg_grad)? - } - Op::Transpose(arg, dim1, dim2) => { - let arg_grad = grad.transpose(*dim1, *dim2)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&arg_grad)? - } - Op::Permute(arg, dims) => { - let mut inv_dims = vec![0; dims.len()]; - for (i, &dim_idx) in dims.iter().enumerate() { - inv_dims[dim_idx] = i - } - let arg_grad = grad.permute(inv_dims)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&arg_grad)? - } - }; - } - } - Ok(grads) - } -} - -#[derive(Debug)] -pub struct GradStore(HashMap); - -impl GradStore { - fn new() -> Self { - GradStore(HashMap::new()) - } - - pub fn get_id(&self, id: TensorId) -> Option<&Tensor> { - self.0.get(&id) - } - - pub fn get(&self, tensor: &Tensor) -> Option<&Tensor> { - self.0.get(&tensor.id()) - } - - pub fn remove(&mut self, tensor: &Tensor) -> Option { - self.0.remove(&tensor.id()) - } - - pub fn insert(&mut self, tensor: &Tensor, grad: Tensor) -> Option { - self.0.insert(tensor.id(), grad) - } - - fn or_insert(&mut self, tensor: &Tensor) -> Result<&mut Tensor> { - use std::collections::hash_map::Entry; - let grad = match self.0.entry(tensor.id()) { - Entry::Occupied(entry) => entry.into_mut(), - Entry::Vacant(entry) => { - let grad = tensor.zeros_like()?; - entry.insert(grad) - } - }; - Ok(grad) - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/conv.rs b/v2/dependency-patches/candle-core-0.4.1/src/conv.rs deleted file mode 100644 index 7b3922dd73..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/conv.rs +++ /dev/null @@ -1,358 +0,0 @@ -use crate::{op::BackpropOp, op::Op, Error, Result, Tensor}; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ParamsConv1D { - pub(crate) b_size: usize, - // Maybe we should have a version without l_in as this bit depends on the input and not only on - // the weights. - pub(crate) l_in: usize, - pub(crate) c_out: usize, - pub(crate) c_in: usize, - pub(crate) k_size: usize, - pub(crate) padding: usize, - pub(crate) stride: usize, - pub(crate) dilation: usize, -} - -impl ParamsConv1D { - pub(crate) fn l_out(&self) -> usize { - (self.l_in + 2 * self.padding - self.dilation * (self.k_size - 1) - 1) / self.stride + 1 - } - - pub(crate) fn out_dims(&self) -> Vec { - let l_out = self.l_out(); - vec![self.b_size, self.c_out, l_out] - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ParamsConvTranspose1D { - pub(crate) b_size: usize, - pub(crate) l_in: usize, - pub(crate) c_out: usize, - pub(crate) c_in: usize, - pub(crate) k_size: usize, - pub(crate) padding: usize, - pub(crate) output_padding: usize, - pub(crate) stride: usize, - pub(crate) dilation: usize, -} - -impl ParamsConvTranspose1D { - pub(crate) fn l_out(&self) -> usize { - (self.l_in - 1) * self.stride - 2 * self.padding - + self.dilation * (self.k_size - 1) - + self.output_padding - + 1 - } - - pub(crate) fn out_dims(&self) -> Vec { - let l_out = self.l_out(); - vec![self.b_size, self.c_out, l_out] - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub enum CudnnFwdAlgo { - ImplicitGemm, - ImplicitPrecompGemm, - Gemm, - Direct, - Fft, - FftTiling, - Winograd, - WinogradNonFused, - Count, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ParamsConv2D { - pub(crate) b_size: usize, - pub(crate) i_h: usize, - pub(crate) i_w: usize, - pub(crate) k_h: usize, - pub(crate) k_w: usize, - pub(crate) c_out: usize, - pub(crate) c_in: usize, - pub(crate) padding: usize, - pub(crate) stride: usize, - pub(crate) dilation: usize, - pub cudnn_fwd_algo: Option, -} - -impl ParamsConv2D { - pub(crate) fn out_h(&self) -> usize { - (self.i_h + 2 * self.padding - self.dilation * (self.k_h - 1) - 1) / self.stride + 1 - } - - pub(crate) fn out_w(&self) -> usize { - (self.i_w + 2 * self.padding - self.dilation * (self.k_w - 1) - 1) / self.stride + 1 - } - - pub(crate) fn out_dims(&self) -> Vec { - vec![self.b_size, self.c_out, self.out_h(), self.out_w()] - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ParamsConvTranspose2D { - pub(crate) b_size: usize, - pub(crate) i_h: usize, - pub(crate) i_w: usize, - pub(crate) k_h: usize, - pub(crate) k_w: usize, - pub(crate) c_out: usize, - pub(crate) c_in: usize, - pub(crate) padding: usize, - pub(crate) output_padding: usize, - pub(crate) stride: usize, - pub(crate) dilation: usize, -} - -impl ParamsConvTranspose2D { - pub(crate) fn out_h(&self) -> usize { - (self.i_h - 1) * self.stride + self.dilation * (self.k_h - 1) + self.output_padding + 1 - - 2 * self.padding - } - - pub(crate) fn out_w(&self) -> usize { - (self.i_w - 1) * self.stride + self.dilation * (self.k_w - 1) + self.output_padding + 1 - - 2 * self.padding - } - - pub(crate) fn out_dims(&self) -> Vec { - vec![self.b_size, self.c_out, self.out_h(), self.out_w()] - } -} - -impl Tensor { - fn conv1d_single_group(&self, kernel: &Self, params: &ParamsConv1D) -> Result { - let storage = - self.storage() - .conv1d(self.layout(), &kernel.storage(), kernel.layout(), params)?; - let op = BackpropOp::new2(self, kernel, |arg, kernel| Op::Conv1D { - arg, - kernel, - padding: params.padding, - stride: params.stride, - dilation: params.dilation, - }); - let out_dims = params.out_dims(); - Ok(crate::tensor::from_storage(storage, out_dims, op, false)) - } - - /// Applies a 1D convolution over the input tensor. - pub fn conv1d( - &self, - kernel: &Self, - padding: usize, - stride: usize, - dilation: usize, - groups: usize, - ) -> Result { - let (c_out, c_in_k, k_size) = kernel.dims3()?; - let (b_size, c_in, l_in) = self.dims3()?; - if c_in != c_in_k * groups { - Err(Error::Conv1dInvalidArgs { - inp_shape: self.shape().clone(), - k_shape: kernel.shape().clone(), - padding, - stride, - msg: "the number of in-channels on the input doesn't match the kernel size", - } - .bt())? - } - - let params = ParamsConv1D { - b_size, - l_in, - c_out: c_out / groups, - c_in: c_in / groups, - k_size, - padding, - stride, - dilation, - }; - if groups == 1 { - self.conv1d_single_group(kernel, ¶ms) - } else { - let blocks = self.chunk(groups, 1)?; - let kernel = kernel.chunk(groups, 0)?; - let blocks = blocks - .iter() - .zip(&kernel) - .map(|(block, kernel)| block.conv1d_single_group(kernel, ¶ms)) - .collect::>>()?; - Tensor::cat(&blocks, 1) - } - } - - fn conv_transpose1d_single_group( - &self, - kernel: &Self, - params: &ParamsConvTranspose1D, - ) -> Result { - let storage = self.storage().conv_transpose1d( - self.layout(), - &kernel.storage(), - kernel.layout(), - params, - )?; - let op = BackpropOp::new2(self, kernel, |arg, kernel| Op::ConvTranspose1D { - arg, - kernel, - padding: params.padding, - output_padding: params.output_padding, - stride: params.stride, - dilation: params.dilation, - }); - let out_dims = params.out_dims(); - Ok(crate::tensor::from_storage(storage, out_dims, op, false)) - } - - /// Applies a 1D transposed convolution over the input tensor. - pub fn conv_transpose1d( - &self, - kernel: &Self, - padding: usize, - output_padding: usize, - stride: usize, - dilation: usize, - groups: usize, - ) -> Result { - let (c_in_k, c_out, k_size) = kernel.dims3()?; - let (b_size, c_in, l_in) = self.dims3()?; - if c_in != c_in_k { - crate::bail!("in_channel mismatch between input ({c_in}) and kernel ({c_in_k})") - } - if c_in % groups != 0 { - crate::bail!("in_channel {c_in} is not divisible by the number of groups") - } - let params = ParamsConvTranspose1D { - b_size, - l_in, - k_size, - c_out, - c_in: c_in / groups, - padding, - output_padding, - stride, - dilation, - }; - if groups == 1 { - self.conv_transpose1d_single_group(kernel, ¶ms) - } else { - let blocks = self.chunk(groups, 1)?; - let kernel = kernel.chunk(groups, 0)?; - let blocks = blocks - .iter() - .zip(&kernel) - .map(|(block, kernel)| block.conv_transpose1d_single_group(kernel, ¶ms)) - .collect::>>()?; - Tensor::cat(&blocks, 1) - } - } - - fn conv2d_single_group(&self, kernel: &Self, params: &ParamsConv2D) -> Result { - let storage = - self.storage() - .conv2d(self.layout(), &kernel.storage(), kernel.layout(), params)?; - let op = BackpropOp::new2(self, kernel, |arg, kernel| Op::Conv2D { - arg, - kernel, - padding: params.padding, - stride: params.stride, - dilation: params.dilation, - }); - let out_dims = params.out_dims(); - Ok(crate::tensor::from_storage(storage, out_dims, op, false)) - } - - /// Applies a 2D convolution over the input tensor. - pub fn conv2d( - &self, - kernel: &Self, - padding: usize, - stride: usize, - dilation: usize, - groups: usize, - ) -> Result { - let (b_size, c_in, i_h, i_w) = self.dims4()?; - let (c_out, c_in_k, k_h, k_w) = kernel.dims4()?; - if c_in != c_in_k * groups { - crate::bail!( - "in_channel mismatch between input ({c_in}, groups {groups}) and kernel ({c_in_k})" - ) - } - let params = ParamsConv2D { - b_size, - i_h, - i_w, - k_h, - k_w, - c_out: c_out / groups, - c_in: c_in / groups, - padding, - stride, - dilation, - cudnn_fwd_algo: None, - }; - if groups == 1 { - self.conv2d_single_group(kernel, ¶ms) - } else { - let blocks = self.chunk(groups, 1)?; - let kernel = kernel.chunk(groups, 0)?; - let blocks = blocks - .iter() - .zip(&kernel) - .map(|(block, kernel)| block.conv2d_single_group(kernel, ¶ms)) - .collect::>>()?; - Tensor::cat(&blocks, 1) - } - } - - /// Applies a 2D transposed convolution over the input tensor. - pub fn conv_transpose2d( - &self, - kernel: &Self, - padding: usize, - output_padding: usize, - stride: usize, - dilation: usize, - ) -> Result { - let (b_size, c_in, i_h, i_w) = self.dims4()?; - let (c_in_k, c_out, k_h, k_w) = kernel.dims4()?; - if c_in != c_in_k { - crate::bail!("in_channel mismatch between input ({c_in}) and kernel ({c_in_k})") - } - let params = ParamsConvTranspose2D { - b_size, - i_h, - i_w, - k_h, - k_w, - c_out, - c_in, - padding, - output_padding, - stride, - dilation, - }; - let storage = self.storage().conv_transpose2d( - self.layout(), - &kernel.storage(), - kernel.layout(), - ¶ms, - )?; - let op = BackpropOp::new2(self, kernel, |arg, kernel| Op::ConvTranspose2D { - arg, - kernel, - padding: params.padding, - output_padding: params.output_padding, - stride: params.stride, - dilation: params.dilation, - }); - let out_dims = params.out_dims(); - Ok(crate::tensor::from_storage(storage, out_dims, op, false)) - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/convert.rs b/v2/dependency-patches/candle-core-0.4.1/src/convert.rs deleted file mode 100644 index 5ea5612a7c..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/convert.rs +++ /dev/null @@ -1,145 +0,0 @@ -//! Implement conversion traits for tensors -use crate::{DType, Device, Error, Tensor, WithDType}; -use half::{bf16, f16, slice::HalfFloatSliceExt}; -use std::convert::TryFrom; - -impl TryFrom<&Tensor> for Vec { - type Error = Error; - fn try_from(tensor: &Tensor) -> Result { - tensor.to_vec1::() - } -} - -impl TryFrom<&Tensor> for Vec> { - type Error = Error; - fn try_from(tensor: &Tensor) -> Result { - tensor.to_vec2::() - } -} - -impl TryFrom<&Tensor> for Vec>> { - type Error = Error; - fn try_from(tensor: &Tensor) -> Result { - tensor.to_vec3::() - } -} - -impl TryFrom for Vec { - type Error = Error; - fn try_from(tensor: Tensor) -> Result { - Vec::::try_from(&tensor) - } -} - -impl TryFrom for Vec> { - type Error = Error; - fn try_from(tensor: Tensor) -> Result { - Vec::>::try_from(&tensor) - } -} - -impl TryFrom for Vec>> { - type Error = Error; - fn try_from(tensor: Tensor) -> Result { - Vec::>>::try_from(&tensor) - } -} - -impl TryFrom<&[T]> for Tensor { - type Error = Error; - fn try_from(v: &[T]) -> Result { - Tensor::from_slice(v, v.len(), &Device::Cpu) - } -} - -impl TryFrom> for Tensor { - type Error = Error; - fn try_from(v: Vec) -> Result { - let len = v.len(); - Tensor::from_vec(v, len, &Device::Cpu) - } -} - -macro_rules! from_tensor { - ($typ:ident) => { - impl TryFrom<&Tensor> for $typ { - type Error = Error; - - fn try_from(tensor: &Tensor) -> Result { - tensor.to_scalar::<$typ>() - } - } - - impl TryFrom for $typ { - type Error = Error; - - fn try_from(tensor: Tensor) -> Result { - $typ::try_from(&tensor) - } - } - - impl TryFrom<$typ> for Tensor { - type Error = Error; - - fn try_from(v: $typ) -> Result { - Tensor::new(v, &Device::Cpu) - } - } - }; -} - -from_tensor!(f64); -from_tensor!(f32); -from_tensor!(f16); -from_tensor!(bf16); -from_tensor!(i64); -from_tensor!(u32); -from_tensor!(u8); - -impl Tensor { - pub fn write_bytes(&self, f: &mut W) -> crate::Result<()> { - use byteorder::{LittleEndian, WriteBytesExt}; - - let vs = self.flatten_all()?; - match self.dtype() { - DType::BF16 => { - let vs = vs.to_vec1::()?; - for &v in vs.reinterpret_cast() { - f.write_u16::(v)? - } - } - DType::F16 => { - let vs = vs.to_vec1::()?; - for &v in vs.reinterpret_cast() { - f.write_u16::(v)? - } - } - DType::F32 => { - // TODO: Avoid using a buffer when data is already on the CPU. - for v in vs.to_vec1::()? { - f.write_f32::(v)? - } - } - DType::F64 => { - for v in vs.to_vec1::()? { - f.write_f64::(v)? - } - } - DType::U32 => { - for v in vs.to_vec1::()? { - f.write_u32::(v)? - } - } - DType::I64 => { - for v in vs.to_vec1::()? { - f.write_i64::(v)? - } - } - DType::U8 => { - let vs = vs.to_vec1::()?; - f.write_all(&vs)?; - } - } - Ok(()) - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/cpu/avx.rs b/v2/dependency-patches/candle-core-0.4.1/src/cpu/avx.rs deleted file mode 100644 index 9398a3460a..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/cpu/avx.rs +++ /dev/null @@ -1,148 +0,0 @@ -use super::{Cpu, CpuF16}; -#[cfg(target_arch = "x86")] -use core::arch::x86::*; -#[cfg(target_arch = "x86_64")] -use core::arch::x86_64::*; - -use half::f16; - -pub struct CurrentCpu {} - -const STEP: usize = 32; -const EPR: usize = 8; -const ARR: usize = STEP / EPR; - -impl Cpu for CurrentCpu { - type Unit = __m256; - type Array = [__m256; ARR]; - - const STEP: usize = STEP; - const EPR: usize = EPR; - - fn n() -> usize { - ARR - } - - unsafe fn zero() -> Self::Unit { - _mm256_setzero_ps() - } - - unsafe fn zero_array() -> Self::Array { - [Self::zero(); ARR] - } - - unsafe fn from_f32(v: f32) -> Self::Unit { - _mm256_set1_ps(v) - } - - unsafe fn load(mem_addr: *const f32) -> Self::Unit { - _mm256_loadu_ps(mem_addr) - } - - unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit { - _mm256_add_ps(a, b) - } - - unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit { - _mm256_add_ps(_mm256_mul_ps(b, c), a) - } - - unsafe fn vec_store(mem_addr: *mut f32, a: Self::Unit) { - _mm256_storeu_ps(mem_addr, a); - } - - unsafe fn vec_reduce(mut x: Self::Array, y: *mut f32) { - for i in 0..ARR / 2 { - x[2 * i] = _mm256_add_ps(x[2 * i], x[2 * i + 1]); - } - for i in 0..ARR / 4 { - x[4 * i] = _mm256_add_ps(x[4 * i], x[4 * i + 2]); - } - #[allow(clippy::reversed_empty_ranges)] - for i in 0..ARR / 8 { - x[8 * i] = _mm256_add_ps(x[8 * i], x[8 * i + 4]); - } - let t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]), _mm256_extractf128_ps(x[0], 1)); - let t1 = _mm_hadd_ps(t0, t0); - *y = _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); - } -} - -pub struct CurrentCpuF16 {} -impl CpuF16 for CurrentCpuF16 { - type Unit = __m256; - type Array = [__m256; ARR]; - - const STEP: usize = STEP; - const EPR: usize = EPR; - - fn n() -> usize { - ARR - } - - unsafe fn zero() -> Self::Unit { - _mm256_setzero_ps() - } - - unsafe fn zero_array() -> Self::Array { - [Self::zero(); ARR] - } - - unsafe fn from_f32(v: f32) -> Self::Unit { - _mm256_set1_ps(v) - } - - #[cfg(target_feature = "f16c")] - unsafe fn load(mem_addr: *const f16) -> Self::Unit { - _mm256_cvtph_ps(_mm_loadu_si128(mem_addr as *const __m128i)) - } - - #[cfg(not(target_feature = "f16c"))] - unsafe fn load(mem_addr: *const f16) -> Self::Unit { - let mut tmp = [0.0f32; 8]; - for i in 0..8 { - tmp[i] = (*mem_addr.add(i)).to_f32(); - } - _mm256_loadu_ps(tmp.as_ptr()) - } - - unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit { - _mm256_add_ps(a, b) - } - - unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit { - _mm256_add_ps(_mm256_mul_ps(b, c), a) - } - - #[cfg(target_feature = "f16c")] - unsafe fn vec_store(mem_addr: *mut f16, a: Self::Unit) { - _mm_storeu_si128(mem_addr as *mut __m128i, _mm256_cvtps_ph(a, 0)) - } - - #[cfg(not(target_feature = "f16c"))] - unsafe fn vec_store(mem_addr: *mut f16, a: Self::Unit) { - let mut tmp = [0.0f32; 8]; - _mm256_storeu_ps(tmp.as_mut_ptr(), a); - for i in 0..8 { - *mem_addr.add(i) = f16::from_f32(tmp[i]); - } - } - - unsafe fn vec_reduce(mut x: Self::Array, y: *mut f32) { - let mut offset = ARR >> 1; - for i in 0..offset { - x[i] = _mm256_add_ps(x[i], x[offset + i]); - } - offset >>= 1; - for i in 0..offset { - x[i] = _mm256_add_ps(x[i], x[offset + i]); - } - offset >>= 1; - for i in 0..offset { - x[i] = _mm256_add_ps(x[i], x[offset + i]); - } - let t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]), _mm256_extractf128_ps(x[0], 1)); - let t1 = _mm_hadd_ps(t0, t0); - *y = _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/cpu/erf.rs b/v2/dependency-patches/candle-core-0.4.1/src/cpu/erf.rs deleted file mode 100644 index ca6be53fd4..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/cpu/erf.rs +++ /dev/null @@ -1,763 +0,0 @@ -#![allow(clippy::excessive_precision)] -// Code taken from https://github.com/statrs-dev/statrs -//! Provides the [error](https://en.wikipedia.org/wiki/Error_function) and -//! related functions - -mod evaluate { - //! Provides functions that don't have a numerical solution and must - //! be solved computationally (e.g. evaluation of a polynomial) - - /// evaluates a polynomial at `z` where `coeff` are the coeffecients - /// to a polynomial of order `k` where `k` is the length of `coeff` and the - /// coeffecient - /// to the `k`th power is the `k`th element in coeff. E.g. [3,-1,2] equates to - /// `2z^2 - z + 3` - /// - /// # Remarks - /// - /// Returns 0 for a 0 length coefficient slice - pub fn polynomial(z: f64, coeff: &[f64]) -> f64 { - let n = coeff.len(); - if n == 0 { - return 0.0; - } - - let mut sum = *coeff.last().unwrap(); - for c in coeff[0..n - 1].iter().rev() { - sum = *c + z * sum; - } - sum - } -} -use std::f64; - -/// `erf` calculates the error function at `x`. -pub fn erf(x: f64) -> f64 { - if x.is_nan() { - f64::NAN - } else if x >= 0.0 && x.is_infinite() { - 1.0 - } else if x <= 0.0 && x.is_infinite() { - -1.0 - } else if x == 0. { - 0.0 - } else { - erf_impl(x, false) - } -} - -/// `erf_inv` calculates the inverse error function -/// at `x`. -pub fn erf_inv(x: f64) -> f64 { - if x == 0.0 { - 0.0 - } else if x >= 1.0 { - f64::INFINITY - } else if x <= -1.0 { - f64::NEG_INFINITY - } else if x < 0.0 { - erf_inv_impl(-x, 1.0 + x, -1.0) - } else { - erf_inv_impl(x, 1.0 - x, 1.0) - } -} - -/// `erfc` calculates the complementary error function -/// at `x`. -pub fn erfc(x: f64) -> f64 { - if x.is_nan() { - f64::NAN - } else if x == f64::INFINITY { - 0.0 - } else if x == f64::NEG_INFINITY { - 2.0 - } else { - erf_impl(x, true) - } -} - -/// `erfc_inv` calculates the complementary inverse -/// error function at `x`. -pub fn erfc_inv(x: f64) -> f64 { - if x <= 0.0 { - f64::INFINITY - } else if x >= 2.0 { - f64::NEG_INFINITY - } else if x > 1.0 { - erf_inv_impl(-1.0 + x, 2.0 - x, -1.0) - } else { - erf_inv_impl(1.0 - x, x, 1.0) - } -} - -// ********************************************************** -// ********** Coefficients for erf_impl polynomial ********** -// ********************************************************** - -/// Polynomial coefficients for a numerator of `erf_impl` -/// in the interval [1e-10, 0.5]. -const ERF_IMPL_AN: &[f64] = &[ - 0.00337916709551257388990745, - -0.00073695653048167948530905, - -0.374732337392919607868241, - 0.0817442448733587196071743, - -0.0421089319936548595203468, - 0.0070165709512095756344528, - -0.00495091255982435110337458, - 0.000871646599037922480317225, -]; - -/// Polynomial coefficients for a denominator of `erf_impl` -/// in the interval [1e-10, 0.5] -const ERF_IMPL_AD: &[f64] = &[ - 1.0, - -0.218088218087924645390535, - 0.412542972725442099083918, - -0.0841891147873106755410271, - 0.0655338856400241519690695, - -0.0120019604454941768171266, - 0.00408165558926174048329689, - -0.000615900721557769691924509, -]; - -/// Polynomial coefficients for a numerator in `erf_impl` -/// in the interval [0.5, 0.75]. -const ERF_IMPL_BN: &[f64] = &[ - -0.0361790390718262471360258, - 0.292251883444882683221149, - 0.281447041797604512774415, - 0.125610208862766947294894, - 0.0274135028268930549240776, - 0.00250839672168065762786937, -]; - -/// Polynomial coefficients for a denominator in `erf_impl` -/// in the interval [0.5, 0.75]. -const ERF_IMPL_BD: &[f64] = &[ - 1.0, - 1.8545005897903486499845, - 1.43575803037831418074962, - 0.582827658753036572454135, - 0.124810476932949746447682, - 0.0113724176546353285778481, -]; - -/// Polynomial coefficients for a numerator in `erf_impl` -/// in the interval [0.75, 1.25]. -const ERF_IMPL_CN: &[f64] = &[ - -0.0397876892611136856954425, - 0.153165212467878293257683, - 0.191260295600936245503129, - 0.10276327061989304213645, - 0.029637090615738836726027, - 0.0046093486780275489468812, - 0.000307607820348680180548455, -]; - -/// Polynomial coefficients for a denominator in `erf_impl` -/// in the interval [0.75, 1.25]. -const ERF_IMPL_CD: &[f64] = &[ - 1.0, - 1.95520072987627704987886, - 1.64762317199384860109595, - 0.768238607022126250082483, - 0.209793185936509782784315, - 0.0319569316899913392596356, - 0.00213363160895785378615014, -]; - -/// Polynomial coefficients for a numerator in `erf_impl` -/// in the interval [1.25, 2.25]. -const ERF_IMPL_DN: &[f64] = &[ - -0.0300838560557949717328341, - 0.0538578829844454508530552, - 0.0726211541651914182692959, - 0.0367628469888049348429018, - 0.00964629015572527529605267, - 0.00133453480075291076745275, - 0.778087599782504251917881e-4, -]; - -/// Polynomial coefficients for a denominator in `erf_impl` -/// in the interval [1.25, 2.25]. -const ERF_IMPL_DD: &[f64] = &[ - 1.0, - 1.75967098147167528287343, - 1.32883571437961120556307, - 0.552528596508757581287907, - 0.133793056941332861912279, - 0.0179509645176280768640766, - 0.00104712440019937356634038, - -0.106640381820357337177643e-7, -]; - -/// Polynomial coefficients for a numerator in `erf_impl` -/// in the interval [2.25, 3.5]. -const ERF_IMPL_EN: &[f64] = &[ - -0.0117907570137227847827732, - 0.014262132090538809896674, - 0.0202234435902960820020765, - 0.00930668299990432009042239, - 0.00213357802422065994322516, - 0.00025022987386460102395382, - 0.120534912219588189822126e-4, -]; - -/// Polynomial coefficients for a denominator in `erf_impl` -/// in the interval [2.25, 3.5]. -const ERF_IMPL_ED: &[f64] = &[ - 1.0, - 1.50376225203620482047419, - 0.965397786204462896346934, - 0.339265230476796681555511, - 0.0689740649541569716897427, - 0.00771060262491768307365526, - 0.000371421101531069302990367, -]; - -/// Polynomial coefficients for a numerator in `erf_impl` -/// in the interval [3.5, 5.25]. -const ERF_IMPL_FN: &[f64] = &[ - -0.00546954795538729307482955, - 0.00404190278731707110245394, - 0.0054963369553161170521356, - 0.00212616472603945399437862, - 0.000394984014495083900689956, - 0.365565477064442377259271e-4, - 0.135485897109932323253786e-5, -]; - -/// Polynomial coefficients for a denominator in `erf_impl` -/// in the interval [3.5, 5.25]. -const ERF_IMPL_FD: &[f64] = &[ - 1.0, - 1.21019697773630784832251, - 0.620914668221143886601045, - 0.173038430661142762569515, - 0.0276550813773432047594539, - 0.00240625974424309709745382, - 0.891811817251336577241006e-4, - -0.465528836283382684461025e-11, -]; - -/// Polynomial coefficients for a numerator in `erf_impl` -/// in the interval [5.25, 8]. -const ERF_IMPL_GN: &[f64] = &[ - -0.00270722535905778347999196, - 0.0013187563425029400461378, - 0.00119925933261002333923989, - 0.00027849619811344664248235, - 0.267822988218331849989363e-4, - 0.923043672315028197865066e-6, -]; - -/// Polynomial coefficients for a denominator in `erf_impl` -/// in the interval [5.25, 8]. -const ERF_IMPL_GD: &[f64] = &[ - 1.0, - 0.814632808543141591118279, - 0.268901665856299542168425, - 0.0449877216103041118694989, - 0.00381759663320248459168994, - 0.000131571897888596914350697, - 0.404815359675764138445257e-11, -]; - -/// Polynomial coefficients for a numerator in `erf_impl` -/// in the interval [8, 11.5]. -const ERF_IMPL_HN: &[f64] = &[ - -0.00109946720691742196814323, - 0.000406425442750422675169153, - 0.000274499489416900707787024, - 0.465293770646659383436343e-4, - 0.320955425395767463401993e-5, - 0.778286018145020892261936e-7, -]; - -/// Polynomial coefficients for a denominator in `erf_impl` -/// in the interval [8, 11.5]. -const ERF_IMPL_HD: &[f64] = &[ - 1.0, - 0.588173710611846046373373, - 0.139363331289409746077541, - 0.0166329340417083678763028, - 0.00100023921310234908642639, - 0.24254837521587225125068e-4, -]; - -/// Polynomial coefficients for a numerator in `erf_impl` -/// in the interval [11.5, 17]. -const ERF_IMPL_IN: &[f64] = &[ - -0.00056907993601094962855594, - 0.000169498540373762264416984, - 0.518472354581100890120501e-4, - 0.382819312231928859704678e-5, - 0.824989931281894431781794e-7, -]; - -/// Polynomial coefficients for a denominator in `erf_impl` -/// in the interval [11.5, 17]. -const ERF_IMPL_ID: &[f64] = &[ - 1.0, - 0.339637250051139347430323, - 0.043472647870310663055044, - 0.00248549335224637114641629, - 0.535633305337152900549536e-4, - -0.117490944405459578783846e-12, -]; - -/// Polynomial coefficients for a numerator in `erf_impl` -/// in the interval [17, 24]. -const ERF_IMPL_JN: &[f64] = &[ - -0.000241313599483991337479091, - 0.574224975202501512365975e-4, - 0.115998962927383778460557e-4, - 0.581762134402593739370875e-6, - 0.853971555085673614607418e-8, -]; - -/// Polynomial coefficients for a denominator in `erf_impl` -/// in the interval [17, 24]. -const ERF_IMPL_JD: &[f64] = &[ - 1.0, - 0.233044138299687841018015, - 0.0204186940546440312625597, - 0.000797185647564398289151125, - 0.117019281670172327758019e-4, -]; - -/// Polynomial coefficients for a numerator in `erf_impl` -/// in the interval [24, 38]. -const ERF_IMPL_KN: &[f64] = &[ - -0.000146674699277760365803642, - 0.162666552112280519955647e-4, - 0.269116248509165239294897e-5, - 0.979584479468091935086972e-7, - 0.101994647625723465722285e-8, -]; - -/// Polynomial coefficients for a denominator in `erf_impl` -/// in the interval [24, 38]. -const ERF_IMPL_KD: &[f64] = &[ - 1.0, - 0.165907812944847226546036, - 0.0103361716191505884359634, - 0.000286593026373868366935721, - 0.298401570840900340874568e-5, -]; - -/// Polynomial coefficients for a numerator in `erf_impl` -/// in the interval [38, 60]. -const ERF_IMPL_LN: &[f64] = &[ - -0.583905797629771786720406e-4, - 0.412510325105496173512992e-5, - 0.431790922420250949096906e-6, - 0.993365155590013193345569e-8, - 0.653480510020104699270084e-10, -]; - -/// Polynomial coefficients for a denominator in `erf_impl` -/// in the interval [38, 60]. -const ERF_IMPL_LD: &[f64] = &[ - 1.0, - 0.105077086072039915406159, - 0.00414278428675475620830226, - 0.726338754644523769144108e-4, - 0.477818471047398785369849e-6, -]; - -/// Polynomial coefficients for a numerator in `erf_impl` -/// in the interval [60, 85]. -const ERF_IMPL_MN: &[f64] = &[ - -0.196457797609229579459841e-4, - 0.157243887666800692441195e-5, - 0.543902511192700878690335e-7, - 0.317472492369117710852685e-9, -]; - -/// Polynomial coefficients for a denominator in `erf_impl` -/// in the interval [60, 85]. -const ERF_IMPL_MD: &[f64] = &[ - 1.0, - 0.052803989240957632204885, - 0.000926876069151753290378112, - 0.541011723226630257077328e-5, - 0.535093845803642394908747e-15, -]; - -/// Polynomial coefficients for a numerator in `erf_impl` -/// in the interval [85, 110]. -const ERF_IMPL_NN: &[f64] = &[ - -0.789224703978722689089794e-5, - 0.622088451660986955124162e-6, - 0.145728445676882396797184e-7, - 0.603715505542715364529243e-10, -]; - -/// Polynomial coefficients for a denominator in `erf_impl` -/// in the interval [85, 110]. -const ERF_IMPL_ND: &[f64] = &[ - 1.0, - 0.0375328846356293715248719, - 0.000467919535974625308126054, - 0.193847039275845656900547e-5, -]; - -// ********************************************************** -// ********** Coefficients for erf_inv_impl polynomial ****** -// ********************************************************** - -/// Polynomial coefficients for a numerator of `erf_inv_impl` -/// in the interval [0, 0.5]. -const ERF_INV_IMPL_AN: &[f64] = &[ - -0.000508781949658280665617, - -0.00836874819741736770379, - 0.0334806625409744615033, - -0.0126926147662974029034, - -0.0365637971411762664006, - 0.0219878681111168899165, - 0.00822687874676915743155, - -0.00538772965071242932965, -]; - -/// Polynomial coefficients for a denominator of `erf_inv_impl` -/// in the interval [0, 0.5]. -const ERF_INV_IMPL_AD: &[f64] = &[ - 1.0, - -0.970005043303290640362, - -1.56574558234175846809, - 1.56221558398423026363, - 0.662328840472002992063, - -0.71228902341542847553, - -0.0527396382340099713954, - 0.0795283687341571680018, - -0.00233393759374190016776, - 0.000886216390456424707504, -]; - -/// Polynomial coefficients for a numerator of `erf_inv_impl` -/// in the interval [0.5, 0.75]. -const ERF_INV_IMPL_BN: &[f64] = &[ - -0.202433508355938759655, - 0.105264680699391713268, - 8.37050328343119927838, - 17.6447298408374015486, - -18.8510648058714251895, - -44.6382324441786960818, - 17.445385985570866523, - 21.1294655448340526258, - -3.67192254707729348546, -]; - -/// Polynomial coefficients for a denominator of `erf_inv_impl` -/// in the interval [0.5, 0.75]. -const ERF_INV_IMPL_BD: &[f64] = &[ - 1.0, - 6.24264124854247537712, - 3.9713437953343869095, - -28.6608180499800029974, - -20.1432634680485188801, - 48.5609213108739935468, - 10.8268667355460159008, - -22.6436933413139721736, - 1.72114765761200282724, -]; - -/// Polynomial coefficients for a numerator of `erf_inv_impl` -/// in the interval [0.75, 1] with x less than 3. -const ERF_INV_IMPL_CN: &[f64] = &[ - -0.131102781679951906451, - -0.163794047193317060787, - 0.117030156341995252019, - 0.387079738972604337464, - 0.337785538912035898924, - 0.142869534408157156766, - 0.0290157910005329060432, - 0.00214558995388805277169, - -0.679465575181126350155e-6, - 0.285225331782217055858e-7, - -0.681149956853776992068e-9, -]; - -/// Polynomial coefficients for a denominator of `erf_inv_impl` -/// in the interval [0.75, 1] with x less than 3. -const ERF_INV_IMPL_CD: &[f64] = &[ - 1.0, - 3.46625407242567245975, - 5.38168345707006855425, - 4.77846592945843778382, - 2.59301921623620271374, - 0.848854343457902036425, - 0.152264338295331783612, - 0.01105924229346489121, -]; - -/// Polynomial coefficients for a numerator of `erf_inv_impl` -/// in the interval [0.75, 1] with x between 3 and 6. -const ERF_INV_IMPL_DN: &[f64] = &[ - -0.0350353787183177984712, - -0.00222426529213447927281, - 0.0185573306514231072324, - 0.00950804701325919603619, - 0.00187123492819559223345, - 0.000157544617424960554631, - 0.460469890584317994083e-5, - -0.230404776911882601748e-9, - 0.266339227425782031962e-11, -]; - -/// Polynomial coefficients for a denominator of `erf_inv_impl` -/// in the interval [0.75, 1] with x between 3 and 6. -const ERF_INV_IMPL_DD: &[f64] = &[ - 1.0, - 1.3653349817554063097, - 0.762059164553623404043, - 0.220091105764131249824, - 0.0341589143670947727934, - 0.00263861676657015992959, - 0.764675292302794483503e-4, -]; - -/// Polynomial coefficients for a numerator of `erf_inv_impl` -/// in the interval [0.75, 1] with x between 6 and 18. -const ERF_INV_IMPL_EN: &[f64] = &[ - -0.0167431005076633737133, - -0.00112951438745580278863, - 0.00105628862152492910091, - 0.000209386317487588078668, - 0.149624783758342370182e-4, - 0.449696789927706453732e-6, - 0.462596163522878599135e-8, - -0.281128735628831791805e-13, - 0.99055709973310326855e-16, -]; - -/// Polynomial coefficients for a denominator of `erf_inv_impl` -/// in the interval [0.75, 1] with x between 6 and 18. -const ERF_INV_IMPL_ED: &[f64] = &[ - 1.0, - 0.591429344886417493481, - 0.138151865749083321638, - 0.0160746087093676504695, - 0.000964011807005165528527, - 0.275335474764726041141e-4, - 0.282243172016108031869e-6, -]; - -/// Polynomial coefficients for a numerator of `erf_inv_impl` -/// in the interval [0.75, 1] with x between 18 and 44. -const ERF_INV_IMPL_FN: &[f64] = &[ - -0.0024978212791898131227, - -0.779190719229053954292e-5, - 0.254723037413027451751e-4, - 0.162397777342510920873e-5, - 0.396341011304801168516e-7, - 0.411632831190944208473e-9, - 0.145596286718675035587e-11, - -0.116765012397184275695e-17, -]; - -/// Polynomial coefficients for a denominator of `erf_inv_impl` -/// in the interval [0.75, 1] with x between 18 and 44. -const ERF_INV_IMPL_FD: &[f64] = &[ - 1.0, - 0.207123112214422517181, - 0.0169410838120975906478, - 0.000690538265622684595676, - 0.145007359818232637924e-4, - 0.144437756628144157666e-6, - 0.509761276599778486139e-9, -]; - -/// Polynomial coefficients for a numerator of `erf_inv_impl` -/// in the interval [0.75, 1] with x greater than 44. -const ERF_INV_IMPL_GN: &[f64] = &[ - -0.000539042911019078575891, - -0.28398759004727721098e-6, - 0.899465114892291446442e-6, - 0.229345859265920864296e-7, - 0.225561444863500149219e-9, - 0.947846627503022684216e-12, - 0.135880130108924861008e-14, - -0.348890393399948882918e-21, -]; - -/// Polynomial coefficients for a denominator of `erf_inv_impl` -/// in the interval [0.75, 1] with x greater than 44. -const ERF_INV_IMPL_GD: &[f64] = &[ - 1.0, - 0.0845746234001899436914, - 0.00282092984726264681981, - 0.468292921940894236786e-4, - 0.399968812193862100054e-6, - 0.161809290887904476097e-8, - 0.231558608310259605225e-11, -]; - -/// `erf_impl` computes the error function at `z`. -/// If `inv` is true, `1 - erf` is calculated as opposed to `erf` -fn erf_impl(z: f64, inv: bool) -> f64 { - if z < 0.0 { - if !inv { - return -erf_impl(-z, false); - } - if z < -0.5 { - return 2.0 - erf_impl(-z, true); - } - return 1.0 + erf_impl(-z, false); - } - - let result = if z < 0.5 { - if z < 1e-10 { - z * 1.125 + z * 0.003379167095512573896158903121545171688 - } else { - z * 1.125 - + z * evaluate::polynomial(z, ERF_IMPL_AN) / evaluate::polynomial(z, ERF_IMPL_AD) - } - } else if z < 110.0 { - let (r, b) = if z < 0.75 { - ( - evaluate::polynomial(z - 0.5, ERF_IMPL_BN) - / evaluate::polynomial(z - 0.5, ERF_IMPL_BD), - 0.3440242112, - ) - } else if z < 1.25 { - ( - evaluate::polynomial(z - 0.75, ERF_IMPL_CN) - / evaluate::polynomial(z - 0.75, ERF_IMPL_CD), - 0.419990927, - ) - } else if z < 2.25 { - ( - evaluate::polynomial(z - 1.25, ERF_IMPL_DN) - / evaluate::polynomial(z - 1.25, ERF_IMPL_DD), - 0.4898625016, - ) - } else if z < 3.5 { - ( - evaluate::polynomial(z - 2.25, ERF_IMPL_EN) - / evaluate::polynomial(z - 2.25, ERF_IMPL_ED), - 0.5317370892, - ) - } else if z < 5.25 { - ( - evaluate::polynomial(z - 3.5, ERF_IMPL_FN) - / evaluate::polynomial(z - 3.5, ERF_IMPL_FD), - 0.5489973426, - ) - } else if z < 8.0 { - ( - evaluate::polynomial(z - 5.25, ERF_IMPL_GN) - / evaluate::polynomial(z - 5.25, ERF_IMPL_GD), - 0.5571740866, - ) - } else if z < 11.5 { - ( - evaluate::polynomial(z - 8.0, ERF_IMPL_HN) - / evaluate::polynomial(z - 8.0, ERF_IMPL_HD), - 0.5609807968, - ) - } else if z < 17.0 { - ( - evaluate::polynomial(z - 11.5, ERF_IMPL_IN) - / evaluate::polynomial(z - 11.5, ERF_IMPL_ID), - 0.5626493692, - ) - } else if z < 24.0 { - ( - evaluate::polynomial(z - 17.0, ERF_IMPL_JN) - / evaluate::polynomial(z - 17.0, ERF_IMPL_JD), - 0.5634598136, - ) - } else if z < 38.0 { - ( - evaluate::polynomial(z - 24.0, ERF_IMPL_KN) - / evaluate::polynomial(z - 24.0, ERF_IMPL_KD), - 0.5638477802, - ) - } else if z < 60.0 { - ( - evaluate::polynomial(z - 38.0, ERF_IMPL_LN) - / evaluate::polynomial(z - 38.0, ERF_IMPL_LD), - 0.5640528202, - ) - } else if z < 85.0 { - ( - evaluate::polynomial(z - 60.0, ERF_IMPL_MN) - / evaluate::polynomial(z - 60.0, ERF_IMPL_MD), - 0.5641309023, - ) - } else { - ( - evaluate::polynomial(z - 85.0, ERF_IMPL_NN) - / evaluate::polynomial(z - 85.0, ERF_IMPL_ND), - 0.5641584396, - ) - }; - let g = (-z * z).exp() / z; - g * b + g * r - } else { - 0.0 - }; - - if inv && z >= 0.5 { - result - } else if z >= 0.5 || inv { - 1.0 - result - } else { - result - } -} - -// `erf_inv_impl` computes the inverse error function where -// `p`,`q`, and `s` are the first, second, and third intermediate -// parameters respectively -fn erf_inv_impl(p: f64, q: f64, s: f64) -> f64 { - let result = if p <= 0.5 { - let y = 0.0891314744949340820313; - let g = p * (p + 10.0); - let r = evaluate::polynomial(p, ERF_INV_IMPL_AN) / evaluate::polynomial(p, ERF_INV_IMPL_AD); - g * y + g * r - } else if q >= 0.25 { - let y = 2.249481201171875; - let g = (-2.0 * q.ln()).sqrt(); - let xs = q - 0.25; - let r = - evaluate::polynomial(xs, ERF_INV_IMPL_BN) / evaluate::polynomial(xs, ERF_INV_IMPL_BD); - g / (y + r) - } else { - let x = (-q.ln()).sqrt(); - if x < 3.0 { - let y = 0.807220458984375; - let xs = x - 1.125; - let r = evaluate::polynomial(xs, ERF_INV_IMPL_CN) - / evaluate::polynomial(xs, ERF_INV_IMPL_CD); - y * x + r * x - } else if x < 6.0 { - let y = 0.93995571136474609375; - let xs = x - 3.0; - let r = evaluate::polynomial(xs, ERF_INV_IMPL_DN) - / evaluate::polynomial(xs, ERF_INV_IMPL_DD); - y * x + r * x - } else if x < 18.0 { - let y = 0.98362827301025390625; - let xs = x - 6.0; - let r = evaluate::polynomial(xs, ERF_INV_IMPL_EN) - / evaluate::polynomial(xs, ERF_INV_IMPL_ED); - y * x + r * x - } else if x < 44.0 { - let y = 0.99714565277099609375; - let xs = x - 18.0; - let r = evaluate::polynomial(xs, ERF_INV_IMPL_FN) - / evaluate::polynomial(xs, ERF_INV_IMPL_FD); - y * x + r * x - } else { - let y = 0.99941349029541015625; - let xs = x - 44.0; - let r = evaluate::polynomial(xs, ERF_INV_IMPL_GN) - / evaluate::polynomial(xs, ERF_INV_IMPL_GD); - y * x + r * x - } - }; - s * result -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/cpu/kernels.rs b/v2/dependency-patches/candle-core-0.4.1/src/cpu/kernels.rs deleted file mode 100644 index 527646d62b..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/cpu/kernels.rs +++ /dev/null @@ -1,191 +0,0 @@ -pub trait VecOps: num_traits::NumAssign + Copy { - fn min(self, rhs: Self) -> Self; - fn max(self, rhs: Self) -> Self; - - /// Dot-product of two vectors. - /// - /// # Safety - /// - /// The length of `lhs` and `rhs` have to be at least `len`. `res` has to point to a valid - /// element. - #[inline(always)] - unsafe fn vec_dot(lhs: *const Self, rhs: *const Self, res: *mut Self, len: usize) { - *res = Self::zero(); - for i in 0..len { - *res += *lhs.add(i) * *rhs.add(i) - } - } - - /// Sum of all elements in a vector. - /// - /// # Safety - /// - /// The length of `xs` must be at least `len`. `res` has to point to a valid - /// element. - #[inline(always)] - unsafe fn vec_reduce_sum(xs: *const Self, res: *mut Self, len: usize) { - *res = Self::zero(); - for i in 0..len { - *res += *xs.add(i) - } - } - - /// Maximum element in a non-empty vector. - /// - /// # Safety - /// - /// The length of `xs` must be at least `len` and positive. `res` has to point to a valid - /// element. - #[inline(always)] - unsafe fn vec_reduce_max(xs: *const Self, res: *mut Self, len: usize) { - *res = *xs; - for i in 1..len { - *res = (*res).max(*xs.add(i)) - } - } - - /// Minimum element in a non-empty vector. - /// - /// # Safety - /// - /// The length of `xs` must be at least `len` and positive. `res` has to point to a valid - /// element. - #[inline(always)] - unsafe fn vec_reduce_min(xs: *const Self, res: *mut Self, len: usize) { - *res = *xs; - for i in 1..len { - *res = (*res).min(*xs.add(i)) - } - } -} - -impl VecOps for f32 { - #[inline(always)] - fn min(self, other: Self) -> Self { - Self::min(self, other) - } - - #[inline(always)] - fn max(self, other: Self) -> Self { - Self::max(self, other) - } - - #[inline(always)] - unsafe fn vec_dot(lhs: *const Self, rhs: *const Self, res: *mut Self, len: usize) { - super::vec_dot_f32(lhs, rhs, res, len) - } - - #[inline(always)] - unsafe fn vec_reduce_sum(xs: *const Self, res: *mut Self, len: usize) { - super::vec_sum(xs, res, len) - } -} - -impl VecOps for half::f16 { - #[inline(always)] - fn min(self, other: Self) -> Self { - Self::min(self, other) - } - - #[inline(always)] - fn max(self, other: Self) -> Self { - Self::max(self, other) - } - - #[inline(always)] - unsafe fn vec_dot(lhs: *const Self, rhs: *const Self, res: *mut Self, len: usize) { - let mut res_f32 = 0f32; - super::vec_dot_f16(lhs, rhs, &mut res_f32, len); - *res = half::f16::from_f32(res_f32); - } -} - -impl VecOps for f64 { - #[inline(always)] - fn min(self, other: Self) -> Self { - Self::min(self, other) - } - - #[inline(always)] - fn max(self, other: Self) -> Self { - Self::max(self, other) - } -} -impl VecOps for half::bf16 { - #[inline(always)] - fn min(self, other: Self) -> Self { - Self::min(self, other) - } - - #[inline(always)] - fn max(self, other: Self) -> Self { - Self::max(self, other) - } -} -impl VecOps for u8 { - #[inline(always)] - fn min(self, other: Self) -> Self { - ::min(self, other) - } - - #[inline(always)] - fn max(self, other: Self) -> Self { - ::max(self, other) - } -} -impl VecOps for u32 { - #[inline(always)] - fn min(self, other: Self) -> Self { - ::min(self, other) - } - - #[inline(always)] - fn max(self, other: Self) -> Self { - ::max(self, other) - } -} -impl VecOps for i64 { - #[inline(always)] - fn min(self, other: Self) -> Self { - ::min(self, other) - } - - #[inline(always)] - fn max(self, other: Self) -> Self { - ::max(self, other) - } -} - -#[inline(always)] -pub fn par_for_each(n_threads: usize, func: impl Fn(usize) + Send + Sync) { - if n_threads == 1 { - func(0) - } else { - rayon::scope(|s| { - for thread_idx in 0..n_threads { - let func = &func; - s.spawn(move |_| func(thread_idx)); - } - }) - } -} - -#[inline(always)] -pub fn par_range(lo: usize, up: usize, n_threads: usize, func: impl Fn(usize) + Send + Sync) { - if n_threads == 1 { - for i in lo..up { - func(i) - } - } else { - rayon::scope(|s| { - for thread_idx in 0..n_threads { - let func = &func; - s.spawn(move |_| { - for i in (thread_idx..up).step_by(n_threads) { - func(i) - } - }); - } - }) - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/cpu/mod.rs b/v2/dependency-patches/candle-core-0.4.1/src/cpu/mod.rs deleted file mode 100644 index 50afb30fe6..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/cpu/mod.rs +++ /dev/null @@ -1,180 +0,0 @@ -pub mod erf; -pub mod kernels; - -trait Cpu { - type Unit; - type Array; - const STEP: usize; - const EPR: usize; - - fn n() -> usize; - unsafe fn zero() -> Self::Unit; - unsafe fn zero_array() -> Self::Array; - unsafe fn load(mem_addr: *const f32) -> Self::Unit; - unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit; - unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit; - unsafe fn vec_reduce(x: Self::Array, y: *mut f32); - unsafe fn from_f32(v: f32) -> Self::Unit; - unsafe fn vec_store(mem_addr: *mut f32, a: Self::Unit); -} - -trait CpuF16 { - type Unit; - type Array; - const STEP: usize; - const EPR: usize; - - fn n() -> usize; - unsafe fn zero() -> Self::Unit; - unsafe fn zero_array() -> Self::Array; - unsafe fn load(mem_addr: *const f16) -> Self::Unit; - unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit; - unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit; - unsafe fn vec_reduce(x: Self::Array, y: *mut f32); - unsafe fn from_f32(v: f32) -> Self::Unit; - unsafe fn vec_store(mem_addr: *mut f16, a: Self::Unit); -} -use half::f16; - -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -#[cfg(target_feature = "avx")] -pub mod avx; -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -#[cfg(target_feature = "avx")] -pub use avx::{CurrentCpu, CurrentCpuF16}; - -#[cfg(target_arch = "wasm32")] -#[cfg(target_feature = "simd128")] -pub mod simd128; -#[cfg(target_arch = "wasm32")] -#[cfg(target_feature = "simd128")] -pub use simd128::CurrentCpu; - -#[cfg(any(target_arch = "arm", target_arch = "aarch64"))] -#[cfg(target_feature = "neon")] -pub mod neon; -#[cfg(any(target_arch = "arm", target_arch = "aarch64"))] -#[cfg(target_feature = "neon")] -pub use neon::CurrentCpu; - -#[cfg(any( - target_feature = "neon", - target_feature = "avx", - target_feature = "simd128" -))] -#[inline(always)] -pub(crate) unsafe fn vec_dot_f32(a_row: *const f32, b_row: *const f32, c: *mut f32, k: usize) { - let np = k & !(CurrentCpu::STEP - 1); - - let mut sum = CurrentCpu::zero_array(); - let mut ax = CurrentCpu::zero_array(); - let mut ay = CurrentCpu::zero_array(); - - for i in (0..np).step_by(CurrentCpu::STEP) { - for j in 0..CurrentCpu::n() { - ax[j] = CurrentCpu::load(a_row.add(i + j * CurrentCpu::EPR)); - ay[j] = CurrentCpu::load(b_row.add(i + j * CurrentCpu::EPR)); - - sum[j] = CurrentCpu::vec_fma(sum[j], ax[j], ay[j]); - } - } - - CurrentCpu::vec_reduce(sum, c); - - // leftovers - for i in np..k { - *c += *a_row.add(i) * (*b_row.add(i)); - } -} - -#[cfg(not(any( - target_feature = "neon", - target_feature = "avx", - target_feature = "simd128" -)))] -#[inline(always)] -pub(crate) unsafe fn vec_dot_f32(a_row: *const f32, b_row: *const f32, c: *mut f32, k: usize) { - // leftovers - for i in 0..k { - *c += *a_row.add(i) * (*b_row.add(i)); - } -} - -#[cfg(any( - target_feature = "neon", - target_feature = "avx", - target_feature = "simd128" -))] -#[inline(always)] -pub(crate) unsafe fn vec_sum(row: *const f32, b: *mut f32, k: usize) { - let np = k & !(CurrentCpu::STEP - 1); - - let mut sum = CurrentCpu::zero_array(); - let mut x = CurrentCpu::zero_array(); - - for i in (0..np).step_by(CurrentCpu::STEP) { - for j in 0..CurrentCpu::n() { - x[j] = CurrentCpu::load(row.add(i + j * CurrentCpu::EPR)); - sum[j] = CurrentCpu::vec_add(sum[j], x[j]); - } - } - - CurrentCpu::vec_reduce(sum, b); - - // leftovers - for i in np..k { - *b += *row.add(i) - } -} - -#[cfg(not(any( - target_feature = "neon", - target_feature = "avx", - target_feature = "simd128" -)))] -#[inline(always)] -pub(crate) unsafe fn vec_sum(row: *const f32, b: *mut f32, k: usize) { - *b = 0f32; - for i in 0..k { - *b += *row.add(i) - } -} - -#[cfg(target_feature = "avx")] -#[inline(always)] -pub(crate) unsafe fn vec_dot_f16(a_row: *const f16, b_row: *const f16, c: *mut f32, k: usize) { - let mut sumf = 0.0f32; - let np = k & !(CurrentCpuF16::STEP - 1); - - let mut sum = CurrentCpuF16::zero_array(); - let mut ax = CurrentCpuF16::zero_array(); - let mut ay = CurrentCpuF16::zero_array(); - - for i in (0..np).step_by(CurrentCpuF16::STEP) { - for j in 0..CurrentCpuF16::n() { - ax[j] = CurrentCpuF16::load(a_row.add(i + j * CurrentCpuF16::EPR)); - ay[j] = CurrentCpuF16::load(b_row.add(i + j * CurrentCpuF16::EPR)); - - sum[j] = CurrentCpuF16::vec_fma(sum[j], ax[j], ay[j]); - } - } - - CurrentCpuF16::vec_reduce(sum, &mut sumf); - - // leftovers - for i in np..k { - sumf += (*a_row.add(i)).to_f32() * (*b_row.add(i)).to_f32(); - } - *c = sumf; -} - -#[cfg(not(target_feature = "avx"))] -#[inline(always)] -pub(crate) unsafe fn vec_dot_f16(a_row: *const f16, b_row: *const f16, c: *mut f32, k: usize) { - // leftovers - let mut sum = 0.0; - for i in 0..k { - sum += (*a_row.add(i)).to_f32() * (*b_row.add(i)).to_f32(); - } - *c = sum; -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/cpu/neon.rs b/v2/dependency-patches/candle-core-0.4.1/src/cpu/neon.rs deleted file mode 100644 index 66b8b45e15..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/cpu/neon.rs +++ /dev/null @@ -1,74 +0,0 @@ -use super::Cpu; -#[cfg(target_arch = "arm")] -use core::arch::arm::*; - -#[cfg(target_arch = "aarch64")] -use core::arch::aarch64::*; - -pub struct CurrentCpu {} - -const STEP: usize = 16; -const EPR: usize = 4; -const ARR: usize = STEP / EPR; - -impl CurrentCpu { - #[cfg(target_arch = "aarch64")] - unsafe fn reduce_one(x: float32x4_t) -> f32 { - vaddvq_f32(x) - } - - #[cfg(target_arch = "arm")] - unsafe fn reduce_one(x: float32x4_t) -> f32 { - vgetq_lane_f32(x, 0) + vgetq_lane_f32(x, 1) + vgetq_lane_f32(x, 2) + vgetq_lane_f32(x, 3) - } -} - -impl Cpu for CurrentCpu { - type Unit = float32x4_t; - type Array = [float32x4_t; ARR]; - - const STEP: usize = STEP; - const EPR: usize = EPR; - - fn n() -> usize { - ARR - } - - unsafe fn zero() -> Self::Unit { - vdupq_n_f32(0.0) - } - - unsafe fn from_f32(x: f32) -> Self::Unit { - vdupq_n_f32(x) - } - - unsafe fn zero_array() -> Self::Array { - [Self::zero(); ARR] - } - - unsafe fn load(mem_addr: *const f32) -> Self::Unit { - vld1q_f32(mem_addr) - } - - unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit { - vaddq_f32(a, b) - } - - unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit { - vfmaq_f32(a, b, c) - } - - unsafe fn vec_store(mem_addr: *mut f32, a: Self::Unit) { - vst1q_f32(mem_addr, a); - } - - unsafe fn vec_reduce(mut x: Self::Array, y: *mut f32) { - for i in 0..ARR / 2 { - x[2 * i] = vaddq_f32(x[2 * i], x[2 * i + 1]); - } - for i in 0..ARR / 4 { - x[4 * i] = vaddq_f32(x[4 * i], x[4 * i + 2]); - } - *y = Self::reduce_one(x[0]); - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/cpu/simd128.rs b/v2/dependency-patches/candle-core-0.4.1/src/cpu/simd128.rs deleted file mode 100644 index 64b94588bd..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/cpu/simd128.rs +++ /dev/null @@ -1,64 +0,0 @@ -use super::Cpu; -use core::arch::wasm32::*; - -pub struct CurrentCpu {} - -const STEP: usize = 16; -const EPR: usize = 4; -const ARR: usize = STEP / EPR; - -impl Cpu for CurrentCpu { - type Unit = v128; - type Array = [v128; ARR]; - - const STEP: usize = STEP; - const EPR: usize = EPR; - - fn n() -> usize { - ARR - } - - unsafe fn zero() -> Self::Unit { - f32x4_splat(0.0) - } - - unsafe fn zero_array() -> Self::Array { - [Self::zero(); ARR] - } - - unsafe fn from_f32(v: f32) -> Self::Unit { - f32x4_splat(v) - } - - unsafe fn load(mem_addr: *const f32) -> Self::Unit { - v128_load(mem_addr as *mut v128) - } - - unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit { - f32x4_add(a, b) - } - - unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit { - f32x4_add(f32x4_mul(b, c), a) - } - - unsafe fn vec_store(mem_addr: *mut f32, a: Self::Unit) { - v128_store(mem_addr as *mut v128, a); - } - - unsafe fn vec_reduce(mut x: Self::Array, y: *mut f32) { - for i in 0..ARR / 2 { - x[2 * i] = f32x4_add(x[2 * i], x[2 * i + 1]); - } - for i in 0..ARR / 4 { - x[4 * i] = f32x4_add(x[4 * i], x[4 * i + 2]); - } - for i in 0..ARR / 8 { - x[8 * i] = f32x4_add(x[8 * i], x[8 * i + 4]); - } - *y = f32x4_extract_lane::<0>(x[0]) - + f32x4_extract_lane::<1>(x[0]) - + f32x4_extract_lane::<2>(x[0]) - + f32x4_extract_lane::<3>(x[0]); - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/cpu_backend.rs b/v2/dependency-patches/candle-core-0.4.1/src/cpu_backend.rs deleted file mode 100644 index 8ae39020fd..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/cpu_backend.rs +++ /dev/null @@ -1,2819 +0,0 @@ -use crate::backend::{BackendDevice, BackendStorage}; -use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; -use crate::{DType, Error, IntDType, Layout, Result, Shape, WithDType}; -use half::{bf16, f16}; -use rayon::prelude::*; - -const USE_IM2COL_CONV1D: bool = true; -const USE_IM2COL_CONV2D: bool = true; - -// TODO: Maybe we should not implement [Clone] here and instead have an explicit allocator + -// intercept the oom errors to avoid panicking and provide a proper error. -#[derive(Debug, Clone)] -pub enum CpuStorage { - U8(Vec), - U32(Vec), - I64(Vec), - BF16(Vec), - F16(Vec), - F32(Vec), - F64(Vec), -} - -#[derive(Debug, Clone)] -pub struct CpuDevice; - -pub trait Map1 { - fn f(&self, vs: &[T], layout: &Layout) -> Result>; - - fn map(&self, vs: &CpuStorage, layout: &Layout) -> Result { - match vs { - CpuStorage::U8(vs) => Ok(CpuStorage::U8(self.f(vs, layout)?)), - CpuStorage::U32(vs) => Ok(CpuStorage::U32(self.f(vs, layout)?)), - CpuStorage::I64(vs) => Ok(CpuStorage::I64(self.f(vs, layout)?)), - CpuStorage::BF16(vs) => Ok(CpuStorage::BF16(self.f(vs, layout)?)), - CpuStorage::F16(vs) => Ok(CpuStorage::F16(self.f(vs, layout)?)), - CpuStorage::F32(vs) => Ok(CpuStorage::F32(self.f(vs, layout)?)), - CpuStorage::F64(vs) => Ok(CpuStorage::F64(self.f(vs, layout)?)), - } - } -} - -pub trait Map1Any { - fn f) -> CpuStorage>( - &self, - vs: &[T], - layout: &Layout, - wrap: W, - ) -> Result; - - fn map(&self, vs: &CpuStorage, layout: &Layout) -> Result { - match vs { - CpuStorage::U8(vs) => Ok(self.f(vs, layout, CpuStorage::U8)?), - CpuStorage::U32(vs) => Ok(self.f(vs, layout, CpuStorage::U32)?), - CpuStorage::I64(vs) => Ok(self.f(vs, layout, CpuStorage::I64)?), - CpuStorage::BF16(vs) => Ok(self.f(vs, layout, CpuStorage::BF16)?), - CpuStorage::F16(vs) => Ok(self.f(vs, layout, CpuStorage::F16)?), - CpuStorage::F32(vs) => Ok(self.f(vs, layout, CpuStorage::F32)?), - CpuStorage::F64(vs) => Ok(self.f(vs, layout, CpuStorage::F64)?), - } - } -} - -type C = CpuStorage; -pub trait Map2 { - const OP: &'static str; - fn f(&self, v1: &[T], l1: &Layout, v2: &[T], l2: &Layout) -> Result>; - - fn map( - &self, - v1: &CpuStorage, - l1: &Layout, - v2: &CpuStorage, - l2: &Layout, - ) -> Result { - match (v1, v2) { - (C::U8(v1), C::U8(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), - (C::U32(v1), C::U32(v2)) => Ok(C::U32(self.f(v1, l1, v2, l2)?)), - (C::I64(v1), C::I64(v2)) => Ok(C::I64(self.f(v1, l1, v2, l2)?)), - (C::BF16(v1), C::BF16(v2)) => Ok(C::BF16(self.f(v1, l1, v2, l2)?)), - (C::F16(v1), C::F16(v2)) => Ok(C::F16(self.f(v1, l1, v2, l2)?)), - (C::F32(v1), C::F32(v2)) => Ok(C::F32(self.f(v1, l1, v2, l2)?)), - (C::F64(v1), C::F64(v2)) => Ok(C::F64(self.f(v1, l1, v2, l2)?)), - _ => Err(Error::DTypeMismatchBinaryOp { - lhs: v1.dtype(), - rhs: v2.dtype(), - op: Self::OP, - } - .bt()), - } - } -} - -pub trait Map2U8 { - const OP: &'static str; - fn f(&self, v1: &[T], l1: &Layout, v2: &[T], l2: &Layout) -> Result>; - - fn map( - &self, - v1: &CpuStorage, - l1: &Layout, - v2: &CpuStorage, - l2: &Layout, - ) -> Result { - match (v1, v2) { - (C::U8(v1), C::U8(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), - (C::U32(v1), C::U32(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), - (C::I64(v1), C::I64(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), - (C::BF16(v1), C::BF16(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), - (C::F16(v1), C::F16(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), - (C::F32(v1), C::F32(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), - (C::F64(v1), C::F64(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), - _ => Err(Error::DTypeMismatchBinaryOp { - lhs: v1.dtype(), - rhs: v2.dtype(), - op: Self::OP, - } - .bt()), - } - } -} - -struct Cmp(CmpOp); -impl Map2U8 for Cmp { - const OP: &'static str = "cmp"; - #[inline(always)] - fn f( - &self, - lhs: &[T], - lhs_l: &Layout, - rhs: &[T], - rhs_l: &Layout, - ) -> Result> { - let dst = match self.0 { - CmpOp::Eq => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x == y)), - CmpOp::Ne => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x != y)), - CmpOp::Lt => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x < y)), - CmpOp::Le => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x <= y)), - CmpOp::Gt => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x > y)), - CmpOp::Ge => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x >= y)), - }; - Ok(dst) - } -} - -struct WCond<'a, T: IntDType>(&'a [T], &'a Layout); - -impl<'a, I: IntDType> Map2 for WCond<'a, I> { - const OP: &'static str = "where"; - #[inline(always)] - fn f(&self, t: &[T], t_l: &Layout, f: &[T], f_l: &Layout) -> Result> { - let vs = match ( - self.1.contiguous_offsets(), - t_l.contiguous_offsets(), - f_l.contiguous_offsets(), - ) { - (Some((o1, o2)), Some((o_t1, o_t2)), Some((o_f1, o_f2))) => { - let pred = &self.0[o1..o2]; - let t = &t[o_t1..o_t2]; - let f = &f[o_f1..o_f2]; - pred.iter() - .zip(t.iter().zip(f.iter())) - .map(|(p, (&t, &f))| if p.is_true() { t } else { f }) - .collect::>() - } - _ => self - .1 - .strided_index() - .zip(t_l.strided_index().zip(f_l.strided_index())) - .map(|(i_p, (i_t, i_f))| { - if self.0[i_p].is_true() { - t[i_t] - } else { - f[i_f] - } - }) - .collect::>(), - }; - Ok(vs) - } -} - -struct ReduceIndex { - reduce_dim_index: usize, - use_min: bool, - return_index: bool, -} - -impl ReduceIndex { - // The value gets replaced if f(s[current_acc], s[i]) returns true. - #[inline(always)] - fn fold_impl(&self, src: &[T], src_l: &Layout, f: F, g: G) -> Result> - where - T: Clone + Copy, - U: Clone + Copy, - F: Fn(T, T) -> bool, - G: Fn(T, usize) -> U, - { - let reduce_dim_size = src_l.dims()[self.reduce_dim_index]; - let reduce_dim_stride = src_l.stride()[self.reduce_dim_index]; - let dst_len = src_l.shape().elem_count() / reduce_dim_size; - let mut dst: Vec = Vec::with_capacity(dst_len); - let dst_to_set = dst.spare_capacity_mut(); - let dst_to_set = unsafe { std::mem::transmute::<_, &mut [U]>(dst_to_set) }; - match src_l.contiguous_offsets() { - Some((o1, o2)) => { - let src = &src[o1..o2]; - if reduce_dim_stride == 1 { - for (start_src_i, dst_v) in dst_to_set.iter_mut().enumerate() { - let start_src_i = start_src_i * reduce_dim_size; - let src = &src[start_src_i..start_src_i + reduce_dim_size]; - let mut acc = 0; - let mut val = src[0]; - for (src_i, &s) in src.iter().enumerate() { - if f(val, s) { - acc = src_i; - val = s - } - } - *dst_v = g(val, acc) - } - } else { - for (start_src_i, dst_v) in dst_to_set.iter_mut().enumerate() { - let (p, q) = ( - start_src_i / reduce_dim_stride, - start_src_i % reduce_dim_stride, - ); - // start_src_i = p * reduce_dim_stride + q - let start_src_i = p * reduce_dim_stride * reduce_dim_size + q; - let src = &src[start_src_i..]; - let mut acc = 0; - let mut val = src[0]; - for src_i in 0..reduce_dim_size { - let s = src[src_i * reduce_dim_stride]; - if f(val, s) { - acc = src_i; - val = s - } - } - *dst_v = g(val, acc) - } - } - } - None => { - let l = src_l.narrow(self.reduce_dim_index, 0, 1)?; - for (unstr_index, src_index) in l.strided_index().enumerate() { - let src = &src[src_index..]; - let mut acc = 0; - let mut val = src[0]; - for src_i in 0..reduce_dim_size { - let s = src[src_i * reduce_dim_stride]; - if f(val, s) { - acc = src_i; - val = s - } - } - dst_to_set[unstr_index] = g(val, acc) - } - } - } - unsafe { dst.set_len(dst_len) }; - Ok(dst) - } -} - -impl Map1Any for ReduceIndex { - #[inline(always)] - fn f) -> CpuStorage>( - &self, - src: &[T], - src_l: &Layout, - wrap: W, - ) -> Result { - if src_l.shape().elem_count() == 0 { - Err(Error::EmptyTensor { op: "reduce" }.bt())? - } - let dst = match (self.return_index, self.use_min) { - (false, true) => wrap(self.fold_impl(src, src_l, |x, y| x > y, |v, _i| v)?), - (false, false) => wrap(self.fold_impl(src, src_l, |x, y| x < y, |v, _i| v)?), - (true, true) => { - CpuStorage::U32(self.fold_impl(src, src_l, |x, y| x > y, |_v, i| i as u32)?) - } - (true, false) => { - CpuStorage::U32(self.fold_impl(src, src_l, |x, y| x < y, |_v, i| i as u32)?) - } - }; - Ok(dst) - } -} - -struct ReduceSum<'a> { - dst_shape: &'a Shape, - reduce_dims: &'a [usize], - reduce_dims_and_stride: Vec<(usize, usize)>, -} - -impl<'a> ReduceSum<'a> { - #[inline(always)] - fn fold_impl(&self, src: &[T], src_l: &Layout, start_elt: T) -> Result> - where - T: WithDType, - { - let mut dst = vec![start_elt; self.dst_shape.elem_count()]; - match src_l.contiguous_offsets() { - Some((o1, o2)) => { - let src = &src[o1..o2]; - // Handle the case where we reduce over the last dimensions separately as it is - // fairly common and easy to optimize. This rely on the layout being contiguous! - // reduce_dims is sorted, check if it is ranging from a to n-1. - let reduce_over_last_dims = self - .reduce_dims - .iter() - .rev() - .enumerate() - .all(|(i, &v)| v == src_l.shape().rank() - 1 - i); - if reduce_over_last_dims { - let reduce_sz = self - .reduce_dims_and_stride - .iter() - .map(|(u, _)| u) - .product::(); - for (dst_i, dst_v) in dst.iter_mut().enumerate() { - let src_i = dst_i * reduce_sz; - unsafe { - T::vec_reduce_sum( - src[src_i..src_i + reduce_sz].as_ptr(), - dst_v, - reduce_sz, - ) - }; - } - return Ok(dst); - }; - for (unstr_index, &src) in src.iter().enumerate() { - let mut dst_index = unstr_index; - // Set the reduce_dims indexes to 0. - for &(dim, stride) in self.reduce_dims_and_stride.iter() { - // The compiler is able to optimize the following in a single divmod op. - let (pre, post) = (dst_index / stride, dst_index % stride); - dst_index = (pre / dim) * stride + post; - } - dst[dst_index] += src; - } - } - None => { - for (unstr_index, src_index) in src_l.strided_index().enumerate() { - let mut dst_index = unstr_index; - // Set the reduce_dims indexes to 0. - for &(dim, stride) in self.reduce_dims_and_stride.iter() { - // The compiler is able to optimize the following in a single divmod op. - let (pre, post) = (dst_index / stride, dst_index % stride); - dst_index = (pre / dim) * stride + post; - } - dst[dst_index] += src[src_index]; - } - } - } - Ok(dst) - } -} - -impl<'a> Map1 for ReduceSum<'a> { - #[inline(always)] - fn f(&self, src: &[T], src_l: &Layout) -> Result> { - self.fold_impl(src, src_l, T::zero()) - } -} - -pub fn unary_map U>( - vs: &[T], - layout: &Layout, - mut f: F, -) -> Vec { - match layout.strided_blocks() { - crate::StridedBlocks::SingleBlock { start_offset, len } => vs - [start_offset..start_offset + len] - .iter() - .map(|&v| f(v)) - .collect(), - crate::StridedBlocks::MultipleBlocks { - block_start_index, - block_len, - } => { - let mut result = Vec::with_capacity(layout.shape().elem_count()); - // Specialize the case where block_len is one to avoid the second loop. - if block_len == 1 { - for index in block_start_index { - let v = unsafe { vs.get_unchecked(index) }; - result.push(f(*v)) - } - } else { - for index in block_start_index { - for offset in 0..block_len { - let v = unsafe { vs.get_unchecked(index + offset) }; - result.push(f(*v)) - } - } - } - result - } - } -} - -pub fn unary_map_vec U, FV: FnMut(&[T], &mut [U])>( - vs: &[T], - layout: &Layout, - mut f: F, - mut f_vec: FV, -) -> Vec { - match layout.strided_blocks() { - crate::StridedBlocks::SingleBlock { start_offset, len } => { - let mut ys: Vec = Vec::with_capacity(len); - let ys_to_set = ys.spare_capacity_mut(); - let ys_to_set = unsafe { std::mem::transmute::<_, &mut [U]>(ys_to_set) }; - f_vec(&vs[start_offset..start_offset + len], ys_to_set); - // SAFETY: values are all set by f_vec. - unsafe { ys.set_len(len) }; - ys - } - crate::StridedBlocks::MultipleBlocks { - block_start_index, - block_len, - } => { - let el_count = layout.shape().elem_count(); - // Specialize the case where block_len is one to avoid the second loop. - if block_len == 1 { - let mut result = Vec::with_capacity(el_count); - for index in block_start_index { - let v = unsafe { vs.get_unchecked(index) }; - result.push(f(*v)) - } - result - } else { - let mut ys: Vec = Vec::with_capacity(el_count); - let ys_to_set = ys.spare_capacity_mut(); - let ys_to_set = unsafe { std::mem::transmute::<_, &mut [U]>(ys_to_set) }; - let mut dst_index = 0; - for src_index in block_start_index { - let vs = &vs[src_index..src_index + block_len]; - let ys = &mut ys_to_set[dst_index..dst_index + block_len]; - f_vec(vs, ys); - dst_index += block_len; - } - // SAFETY: values are all set by f_vec. - unsafe { ys.set_len(el_count) }; - ys - } - } - } -} - -// This function maps over two strided index sequences. -pub fn binary_map U>( - lhs_l: &Layout, - rhs_l: &Layout, - lhs: &[T], - rhs: &[T], - mut f: F, -) -> Vec { - match (lhs_l.contiguous_offsets(), rhs_l.contiguous_offsets()) { - (Some((o_l1, o_l2)), Some((o_r1, o_r2))) => lhs[o_l1..o_l2] - .iter() - .zip(rhs[o_r1..o_r2].iter()) - .map(|(&l, &r)| f(l, r)) - .collect(), - (Some((o_l1, o_l2)), None) => { - // TODO: Maybe we want to avoid going through the layout twice. - match rhs_l.offsets_b() { - Some(ob) => { - let mut i_in_block = 0; - let mut i_right_broadcast = 0; - lhs[o_l1..o_l2] - .iter() - .map(|&l| { - let r = unsafe { rhs.get_unchecked(i_in_block + ob.start) }; - i_right_broadcast += 1; - if i_right_broadcast >= ob.right_broadcast { - i_in_block += 1; - i_right_broadcast = 0; - } - if i_in_block >= ob.len { - i_in_block = 0 - } - f(l, *r) - }) - .collect() - } - None => lhs_l - .strided_index() - .zip(rhs_l.strided_index()) - .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) - .collect(), - } - } - (None, Some((o_r1, o_r2))) => { - // TODO: Maybe we want to avoid going through the layout twice. - match lhs_l.offsets_b() { - Some(ob) => { - let mut i_in_block = 0; - let mut i_right_broadcast = 0; - rhs[o_r1..o_r2] - .iter() - .map(|&r| { - let l = unsafe { lhs.get_unchecked(i_in_block + ob.start) }; - i_right_broadcast += 1; - if i_right_broadcast >= ob.right_broadcast { - i_in_block += 1; - i_right_broadcast = 0; - } - if i_in_block >= ob.len { - i_in_block = 0 - } - f(*l, r) - }) - .collect() - } - None => lhs_l - .strided_index() - .zip(rhs_l.strided_index()) - .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) - .collect(), - } - } - _ => lhs_l - .strided_index() - .zip(rhs_l.strided_index()) - .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) - .collect(), - } -} - -// Similar to binary_map but with vectorized variants. -pub fn binary_map_vec T, FV: FnMut(&[T], &[T], &mut [T])>( - lhs_l: &Layout, - rhs_l: &Layout, - lhs: &[T], - rhs: &[T], - mut f: F, - mut f_vec: FV, -) -> Vec { - let el_count = lhs_l.shape().elem_count(); - match (lhs_l.contiguous_offsets(), rhs_l.contiguous_offsets()) { - (Some((o_l1, o_l2)), Some((o_r1, o_r2))) => { - let mut ys: Vec = Vec::with_capacity(el_count); - let ys_to_set = ys.spare_capacity_mut(); - let ys_to_set = unsafe { std::mem::transmute::<_, &mut [T]>(ys_to_set) }; - f_vec(&lhs[o_l1..o_l2], &rhs[o_r1..o_r2], ys_to_set); - // SAFETY: values are all set by f_vec. - unsafe { ys.set_len(el_count) }; - ys - } - (Some((o_l1, o_l2)), None) => match rhs_l.offsets_b() { - Some(ob) if ob.right_broadcast == 1 => { - let rhs = &rhs[ob.start..ob.start + ob.len]; - let mut ys: Vec = Vec::with_capacity(el_count); - let ys_to_set = ys.spare_capacity_mut(); - let ys_to_set = unsafe { std::mem::transmute::<_, &mut [T]>(ys_to_set) }; - let mut dst_i = 0; - for src_i in (o_l1..o_l2).step_by(ob.len) { - f_vec( - &lhs[src_i..src_i + ob.len], - rhs, - &mut ys_to_set[dst_i..dst_i + ob.len], - ); - dst_i += ob.len; - } - // SAFETY: values are all set by f_vec. - unsafe { ys.set_len(el_count) }; - ys - } - Some(ob) => { - let rhs = &rhs[ob.start..ob.start + ob.len]; - let mut ys = lhs[o_l1..o_l2].to_vec(); - for idx_l in 0..ob.left_broadcast { - let start = idx_l * ob.len * ob.right_broadcast; - for (i, &r) in rhs.iter().enumerate() { - let start = start + i * ob.right_broadcast; - for v in ys[start..start + ob.right_broadcast].iter_mut() { - *v = f(*v, r) - } - } - } - ys - } - None => lhs_l - .strided_index() - .zip(rhs_l.strided_index()) - .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) - .collect(), - }, - (None, Some((o_r1, o_r2))) => match lhs_l.offsets_b() { - Some(ob) if ob.right_broadcast == 1 => { - let lhs = &lhs[ob.start..ob.start + ob.len]; - let mut ys: Vec = Vec::with_capacity(el_count); - let ys_to_set = ys.spare_capacity_mut(); - let ys_to_set = unsafe { std::mem::transmute::<_, &mut [T]>(ys_to_set) }; - let mut dst_i = 0; - for src_i in (o_r1..o_r2).step_by(ob.len) { - f_vec( - lhs, - &rhs[src_i..src_i + ob.len], - &mut ys_to_set[dst_i..dst_i + ob.len], - ); - dst_i += ob.len; - } - // SAFETY: values are all set by f_vec. - unsafe { ys.set_len(el_count) }; - ys - } - Some(ob) => { - let lhs = &lhs[ob.start..ob.start + ob.len]; - let mut ys = rhs[o_r1..o_r2].to_vec(); - for idx_l in 0..ob.left_broadcast { - let start = idx_l * ob.len * ob.right_broadcast; - for (i, &l) in lhs.iter().enumerate() { - let start = start + i * ob.right_broadcast; - for v in ys[start..start + ob.right_broadcast].iter_mut() { - *v = f(l, *v) - } - } - } - ys - } - None => lhs_l - .strided_index() - .zip(rhs_l.strided_index()) - .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) - .collect(), - }, - _ => lhs_l - .strided_index() - .zip(rhs_l.strided_index()) - .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) - .collect(), - } -} - -struct Affine(f64, f64); - -impl Map1 for Affine { - fn f(&self, vs: &[T], layout: &Layout) -> Result> { - let mul = T::from_f64(self.0); - let add = T::from_f64(self.1); - Ok(unary_map(vs, layout, |v| v * mul + add)) - } -} - -struct AvgPool2D((usize, usize), (usize, usize)); - -impl Map1 for AvgPool2D { - fn f(&self, src: &[T], layout: &Layout) -> Result> { - // https://pytorch.org/docs/stable/generated/torch.nn.AvgPool2d.html - let (k_h, k_w) = self.0; - let (s_h, s_w) = self.1; - let (b_sz, c, h, w) = layout.shape().dims4()?; - let stride = layout.stride(); - let (stride_h, stride_w) = (stride[2], stride[3]); - let h_out = (h - k_h) / s_h + 1; - let w_out = (w - k_w) / s_w + 1; - let src_index = layout.start_offset(); - let mut dst = vec![T::zero(); b_sz * c * h_out * w_out]; - let scale = 1f64 / (k_h * k_w) as f64; - let scale = T::from_f64(scale); - for b_idx in 0..b_sz { - let dst = &mut dst[b_idx * c * h_out * w_out..]; - let src_index = src_index + b_idx * stride[0]; - for c_idx in 0..c { - let dst = &mut dst[c_idx * h_out * w_out..]; - let src_index = src_index + c_idx * stride[1]; - for h_idx in 0..h_out { - for w_idx in 0..w_out { - let mut sum = T::zero(); - for m in 0..k_h { - for n in 0..k_w { - let m = s_h * h_idx + m; - let n = s_w * w_idx + n; - sum += src[src_index + m * stride_h + n * stride_w] - } - } - dst[h_idx * w_out + w_idx] = sum * scale; - } - } - } - } - Ok(dst) - } -} - -struct MaxPool2D((usize, usize), (usize, usize)); - -impl Map1 for MaxPool2D { - fn f(&self, src: &[T], layout: &Layout) -> Result> { - // https://pytorch.org/docs/stable/generated/torch.nn.MaxPool2d.html - let (k_h, k_w) = self.0; - let (s_h, s_w) = self.1; - let (b_sz, c, h, w) = layout.shape().dims4()?; - let stride = layout.stride(); - let (stride_h, stride_w) = (stride[2], stride[3]); - let h_out = (h - k_h) / s_h + 1; - let w_out = (w - k_w) / s_w + 1; - let src_index = layout.start_offset(); - let mut dst = vec![T::zero(); b_sz * c * h_out * w_out]; - for b_idx in 0..b_sz { - let dst = &mut dst[b_idx * c * h_out * w_out..]; - let src_index = src_index + b_idx * stride[0]; - for c_idx in 0..c { - let dst = &mut dst[c_idx * h_out * w_out..]; - let src_index = src_index + c_idx * stride[1]; - for h_idx in 0..h_out { - for w_idx in 0..w_out { - let mut largest = - src[src_index + s_h * h_idx * stride_h + s_w * w_idx * stride_w]; - for m in 0..k_h { - for n in 0..k_w { - let m = s_h * h_idx + m; - let n = s_w * w_idx + n; - if largest < src[src_index + m * stride_h + n * stride_w] { - largest = src[src_index + m * stride_h + n * stride_w] - } - } - } - dst[h_idx * w_out + w_idx] = largest; - } - } - } - } - Ok(dst) - } -} - -struct UpsampleNearest1D(usize); - -impl Map1 for UpsampleNearest1D { - fn f(&self, src: &[T], layout: &Layout) -> Result> { - // TODO: Specialized implementation for the case 2*sz? - let dst_sz = self.0; - let (b_sz, c, src_sz) = layout.shape().dims3()?; - let stride = layout.stride(); - let stride_sz = stride[2]; - let src_index = layout.start_offset(); - let scale_sz = src_sz as f64 / dst_sz as f64; - let mut dst = vec![T::zero(); b_sz * c * dst_sz]; - let src_idxs = (0..dst_sz) - .map(|idx| usize::min(src_sz - 1, (idx as f64 * scale_sz) as usize)) - .collect::>(); - for b_idx in 0..b_sz { - let dst = &mut dst[b_idx * c * dst_sz..]; - let src_index = src_index + b_idx * stride[0]; - for c_idx in 0..c { - let dst = &mut dst[c_idx * dst_sz..]; - let src_index = src_index + c_idx * stride[1]; - for (idx, src_idx) in src_idxs.iter().enumerate() { - dst[idx] = src[src_index + src_idx * stride_sz] - } - } - } - Ok(dst) - } -} - -struct UpsampleNearest2D(usize, usize); - -impl Map1 for UpsampleNearest2D { - fn f(&self, src: &[T], layout: &Layout) -> Result> { - // TODO: Specialized implementation for the case 2*h, 2*w? - let (dst_h, dst_w) = (self.0, self.1); - let (b_sz, c, src_h, src_w) = layout.shape().dims4()?; - let stride = layout.stride(); - let (stride_h, stride_w) = (stride[2], stride[3]); - let src_index = layout.start_offset(); - let scale_h = src_h as f64 / dst_h as f64; - let scale_w = src_w as f64 / dst_w as f64; - let mut dst = vec![T::zero(); b_sz * c * dst_h * dst_w]; - let src_h_idxs = (0..dst_h) - .map(|h_idx| usize::min(src_h - 1, (h_idx as f64 * scale_h) as usize)) - .collect::>(); - let src_w_idxs = (0..dst_w) - .map(|w_idx| usize::min(src_w - 1, (w_idx as f64 * scale_w) as usize)) - .collect::>(); - for b_idx in 0..b_sz { - let dst = &mut dst[b_idx * c * dst_h * dst_w..]; - let src_index = src_index + b_idx * stride[0]; - for c_idx in 0..c { - let dst = &mut dst[c_idx * dst_h * dst_w..]; - let src_index = src_index + c_idx * stride[1]; - for (h_idx, src_h_idx) in src_h_idxs.iter().enumerate() { - for (w_idx, src_w_idx) in src_w_idxs.iter().enumerate() { - let src_index = src_index + src_h_idx * stride_h + src_w_idx * stride_w; - dst[h_idx * dst_w + w_idx] = src[src_index] - } - } - } - } - Ok(dst) - } -} - -struct Gather<'a, I: IntDType> { - ids: &'a [I], - ids_l: &'a Layout, - dim: usize, -} - -impl<'a, I: IntDType> Map1 for Gather<'a, I> { - fn f(&self, src: &[T], src_l: &Layout) -> Result> { - let ids = match self.ids_l.contiguous_offsets() { - Some((a, b)) => &self.ids[a..b], - None => Err(Error::RequiresContiguous { op: "gather" }.bt())?, - }; - let src = match src_l.contiguous_offsets() { - Some((a, b)) => &src[a..b], - None => Err(Error::RequiresContiguous { op: "gather" }.bt())?, - }; - let dim = self.dim; - let ids_dims = self.ids_l.dims(); - let src_dims = src_l.dims(); - let dst_len: usize = ids_dims.iter().product(); - let dst_left_len: usize = ids_dims[..dim].iter().product(); - let dst_dim_len = ids_dims[dim]; - let dst_right_len: usize = ids_dims[dim + 1..].iter().product(); - - let src_dim_len = src_dims[dim]; - let src_right_len: usize = src_dims[dim + 1..].iter().product(); - - let mut dst = vec![T::zero(); dst_len]; - for left_i in 0..dst_left_len { - let start_src_idx = left_i * src_right_len * src_dim_len; - let start_dst_idx = left_i * dst_right_len * dst_dim_len; - for i in 0..dst_dim_len { - let start_dst_idx = start_dst_idx + i * dst_right_len; - for right_i in 0..dst_right_len { - let dst_idx = start_dst_idx + right_i; - let index = ids[dst_idx].as_usize(); - if index >= src_dim_len { - Err(Error::InvalidIndex { - index, - size: src_dim_len, - op: "gather", - } - .bt())? - } - let src_idx = start_src_idx + index * src_right_len + right_i; - dst[dst_idx] = src[src_idx] - } - } - } - Ok(dst) - } -} - -struct IndexSelect<'a, T: IntDType> { - ids: &'a [T], - ids_l: &'a Layout, - dim: usize, -} - -impl<'a, I: IntDType> Map1 for IndexSelect<'a, I> { - fn f(&self, src: &[T], layout: &Layout) -> Result> { - let src = match layout.contiguous_offsets() { - Some((a, b)) => &src[a..b], - None => Err(Error::RequiresContiguous { op: "index-select" }.bt())?, - }; - let dim = self.dim; - let n_ids = match self.ids_l.dims() { - [n_ids] => *n_ids, - d => Err(Error::UnexpectedNumberOfDims { - expected: 1, - got: d.len(), - shape: self.ids_l.shape().clone(), - } - .bt())?, - }; - let stride_ids = self.ids_l.stride()[0]; - let mut dst_dims = layout.dims().to_vec(); - let src_dim = dst_dims[dim]; - dst_dims[dim] = n_ids; - let dst_len: usize = dst_dims.iter().product(); - let left_len: usize = dst_dims[..dim].iter().product(); - let right_len: usize = dst_dims[dim + 1..].iter().product(); - let mut dst = vec![T::zero(); dst_len]; - for left_i in 0..left_len { - let start_src_idx = left_i * right_len * src_dim; - let start_dst_idx = left_i * right_len * n_ids; - for i in 0..n_ids { - let index = self.ids[self.ids_l.start_offset() + stride_ids * i].as_usize(); - if index >= src_dim { - Err(Error::InvalidIndex { - index, - size: src_dim, - op: "index-select", - } - .bt())? - } - let start_src_idx = start_src_idx + index * right_len; - let start_dst_idx = start_dst_idx + i * right_len; - dst[start_dst_idx..start_dst_idx + right_len] - .copy_from_slice(&src[start_src_idx..start_src_idx + right_len]) - } - } - Ok(dst) - } -} - -struct ScatterAdd<'a, I: IntDType> { - ids: &'a [I], - ids_l: &'a Layout, - dim: usize, -} - -impl<'a, I: IntDType> Map2 for ScatterAdd<'a, I> { - const OP: &'static str = "scatter-add"; - fn f(&self, v1: &[T], l1: &Layout, src: &[T], src_l: &Layout) -> Result> { - let dst_len = l1.shape().elem_count(); - let mut dst = vec![T::zero(); dst_len]; - copy_strided_src_(v1, &mut dst, 0, l1); - let src = match src_l.contiguous_offsets() { - None => Err(Error::RequiresContiguous { op: "scatter-add" }.bt())?, - Some((o1, o2)) => &src[o1..o2], - }; - - let dim = self.dim; - let ids_dims = self.ids_l.dims(); - let dst_dims = l1.dims(); - let dst_dim_len = dst_dims[dim]; - let dst_right_len: usize = dst_dims[dim + 1..].iter().product(); - - let ids_left_len: usize = ids_dims[..dim].iter().product(); - let ids_dim_len = ids_dims[dim]; - let ids_right_len: usize = ids_dims[dim + 1..].iter().product(); - - let ids = match self.ids_l.contiguous_offsets() { - Some((a, b)) => &self.ids[a..b], - None => Err(Error::RequiresContiguous { op: "gather" }.bt())?, - }; - for left_i in 0..ids_left_len { - let start_ids_idx = left_i * ids_right_len * ids_dim_len; - let start_dst_idx = left_i * dst_right_len * dst_dim_len; - for i in 0..ids_dim_len { - let start_ids_idx = start_ids_idx + i * ids_right_len; - for right_i in 0..dst_right_len { - let ids_idx = start_ids_idx + right_i; - let index = ids[ids_idx].as_usize(); - if index >= dst_dim_len { - Err(Error::InvalidIndex { - index, - size: dst_dim_len, - op: "gather", - } - .bt())? - } - let dst_idx = start_dst_idx + index * dst_right_len + right_i; - dst[dst_idx] += src[ids_idx] - } - } - } - - Ok(dst) - } -} - -struct IndexAdd<'a, I: IntDType> { - ids: &'a [I], - dim: usize, -} - -impl<'a, I: IntDType> Map2 for IndexAdd<'a, I> { - const OP: &'static str = "index-add"; - // https://pytorch.org/docs/stable/generated/torch.Tensor.index_add_.html#torch.Tensor.index_add_ - // v1, l1 -> self - fn f(&self, v1: &[T], l1: &Layout, src: &[T], src_l: &Layout) -> Result> { - let dst_len = l1.shape().elem_count(); - let mut dst = vec![T::zero(); dst_len]; - copy_strided_src_(v1, &mut dst, 0, l1); - let src = match src_l.contiguous_offsets() { - None => Err(Error::RequiresContiguous { op: "index-add" }.bt())?, - Some((o1, o2)) => &src[o1..o2], - }; - let dim = self.dim; - let max_idx = l1.dims()[dim]; - let pre_dim = src_l.dims()[..dim].iter().product::(); - let src_dim_sz = src_l.dims()[dim]; - let post_dim = src_l.dims()[dim + 1..].iter().product::(); - if dim == 0 { - for (src_idx, dst_idx) in self.ids.iter().enumerate() { - let dst_idx = dst_idx.as_usize(); - if dst_idx >= max_idx { - Err(Error::InvalidIndex { - index: dst_idx, - op: "index-add", - size: max_idx, - })? - } - let src_idx = src_idx * post_dim; - let dst_idx = dst_idx * post_dim; - let src = &src[src_idx..src_idx + post_dim]; - let dst = &mut dst[dst_idx..dst_idx + post_dim]; - for (d, &s) in dst.iter_mut().zip(src.iter()) { - *d += s - } - } - } else { - for (src_idx, dst_idx) in self.ids.iter().enumerate() { - let dst_idx = dst_idx.as_usize(); - if dst_idx >= max_idx { - Err(Error::InvalidIndex { - index: dst_idx, - op: "index-add", - size: max_idx, - })? - } - for pre_i in 0..pre_dim { - let pre_src_i = (pre_i * src_dim_sz + src_idx) * post_dim; - let pre_dst_i = (pre_i * max_idx + dst_idx) * post_dim; - let src = &src[pre_src_i..pre_src_i + post_dim]; - let dst = &mut dst[pre_dst_i..pre_dst_i + post_dim]; - for (d, &s) in dst.iter_mut().zip(src.iter()) { - *d += s - } - } - } - } - Ok(dst) - } -} - -fn copy_strided_src_(src: &[T], dst: &mut [T], dst_offset: usize, src_l: &Layout) { - match src_l.strided_blocks() { - crate::StridedBlocks::SingleBlock { start_offset, len } => { - let to_copy = (dst.len() - dst_offset).min(len); - dst[dst_offset..dst_offset + to_copy] - .copy_from_slice(&src[start_offset..start_offset + to_copy]) - } - crate::StridedBlocks::MultipleBlocks { - block_start_index, - block_len: 1, - } => { - for (dst_index, src_index) in block_start_index.enumerate() { - let dst_index = dst_index + dst_offset; - if dst_index >= dst.len() { - break; - } - dst[dst_index] = src[src_index] - } - } - crate::StridedBlocks::MultipleBlocks { - block_start_index, - block_len, - } => { - let mut dst_index = dst_offset; - for src_index in block_start_index { - let next_dst_index = dst_index + block_len; - if dst_index >= dst.len() { - break; - } - let to_copy = usize::min(block_len, dst.len() - dst_index); - dst[dst_index..dst_index + to_copy] - .copy_from_slice(&src[src_index..src_index + to_copy]); - dst_index = next_dst_index - } - } - } -} - -struct Conv1D<'a>(&'a crate::conv::ParamsConv1D); - -impl<'a> Map2 for Conv1D<'a> { - const OP: &'static str = "conv1d"; - fn f(&self, inp: &[T], inp_l: &Layout, k: &[T], k_l: &Layout) -> Result> { - let p = self.0; - let inp = &inp[inp_l.start_offset()..]; - let k = &k[k_l.start_offset()..]; - let (inp_s0, inp_s1, inp_s2) = crate::shape::dims3(inp_l.stride())?; - let (k_s0, k_s1, k_s2) = crate::shape::dims3(k_l.stride())?; - let l_out = p.l_out(); - let dst_elems = p.c_out * l_out * p.b_size; - // The output shape is [b_size, c_out, l_out] - let dst = vec![T::zero(); dst_elems]; - - // TODO: Avoid making this copy if `inp` already has the appropriate layout. - let mut inp_cont = vec![T::zero(); p.b_size * p.c_in * p.l_in]; - for b_idx in 0..p.b_size { - for src_l in 0..p.l_in { - for src_c_idx in 0..p.c_in { - let inp_idx = b_idx * inp_s0 + src_c_idx * inp_s1 + src_l * inp_s2; - inp_cont[b_idx * p.l_in * p.c_in + src_l * p.c_in + src_c_idx] = inp[inp_idx] - } - } - } - - for offset in 0..p.k_size { - (0..p.c_out).into_par_iter().for_each(|dst_c_idx| { - let dst_idx = dst_c_idx * l_out; - let k_cont = (0..p.c_in) - .map(|c_in_idx| k[dst_c_idx * k_s0 + c_in_idx * k_s1 + offset * k_s2]) - .collect::>(); - for b_idx in 0..p.b_size { - let dst_idx = dst_idx + b_idx * p.c_out * l_out; - for dst_l in 0..l_out { - let dst_idx = dst_idx + dst_l; - let src_l = p.stride * dst_l + offset * p.dilation; - if src_l < p.padding || src_l >= p.padding + p.l_in { - continue; - } - let src_l = src_l - p.padding; - let inp_cont = &inp_cont[b_idx * p.l_in * p.c_in + src_l * p.c_in..]; - assert!(inp_cont.len() >= p.c_in); - assert!(k_cont.len() >= p.c_in); - let mut d = T::zero(); - unsafe { T::vec_dot(inp_cont.as_ptr(), k_cont.as_ptr(), &mut d, p.c_in) } - let dst_p = dst.as_ptr(); - // Safety: dst_idx are uniques per dst_c_idx which is used to parallelise - // the different tasks so no two threads can try to write at the same - // location. - unsafe { - let ptr = dst_p.add(dst_idx) as *mut T; - *ptr += d - } - } - } - }) - } - Ok(dst) - } -} - -struct Im2Col1D { - l_k: usize, - stride: usize, - dilation: usize, - padding: usize, -} - -impl Im2Col1D { - fn l_out(&self, l: usize) -> usize { - (l + 2 * self.padding - self.dilation * (self.l_k - 1) - 1) / self.stride + 1 - } -} - -impl Map1 for Im2Col1D { - fn f(&self, vs: &[T], layout: &Layout) -> Result> { - let &Self { - l_k, - stride, - dilation, - padding, - } = self; - let (b, c, l) = layout.shape().dims3()?; - let l_out = self.l_out(l); - let src = &vs[layout.start_offset()..]; - let mut dst = vec![T::zero(); b * l_out * c * l_k]; - let (src_s0, src_s1, src_s2) = { - let s = layout.stride(); - (s[0], s[1], s[2]) - }; - // TODO: provide specialized kernels for the common use cases. - // - l_k = 1 - // - padding = 0 - // - stride = 1 - // - dilation = 1 - for b_idx in 0..b { - let src_idx = b_idx * src_s0; - let dst_idx = b_idx * l_out * c * l_k; - for l_idx in 0..l_out { - let dst_idx = dst_idx + l_idx * c * l_k; - for c_idx in 0..c { - let dst_idx = dst_idx + c_idx * l_k; - let src_idx = c_idx * src_s1 + src_idx; - for l_k_idx in 0..l_k { - let src_l = l_idx * stride + l_k_idx * dilation; - if padding != 0 && (src_l < padding || src_l >= l + padding) { - continue; - } - let src_l = src_l - padding; - let src_idx = src_idx + src_l * src_s2; - let dst_idx = dst_idx + l_k_idx; - dst[dst_idx] = src[src_idx] - } - } - } - } - Ok(dst) - } -} - -struct Im2Col { - h_k: usize, - w_k: usize, - stride: usize, - dilation: usize, - padding: usize, -} - -impl Im2Col { - fn hw_out(&self, h: usize, w: usize) -> (usize, usize) { - let h_out = (h + 2 * self.padding - self.dilation * (self.h_k - 1) - 1) / self.stride + 1; - let w_out = (w + 2 * self.padding - self.dilation * (self.w_k - 1) - 1) / self.stride + 1; - (h_out, w_out) - } -} - -impl Map1 for Im2Col { - fn f(&self, vs: &[T], layout: &Layout) -> Result> { - let &Self { - h_k, - w_k, - stride, - dilation, - padding, - } = self; - let (b, c, h, w) = layout.shape().dims4()?; - let (h_out, w_out) = self.hw_out(h, w); - let src = &vs[layout.start_offset()..]; - let mut dst = vec![T::zero(); b * h_out * w_out * c * h_k * w_k]; - let (src_s0, src_s1, src_s2, src_s3) = { - let s = layout.stride(); - (s[0], s[1], s[2], s[3]) - }; - // TODO: provide specialized kernels for the common use cases. - // - h_k = w_k = 1 - // - padding = 0 - // - stride = 1 - // - dilation = 1 - for b_idx in 0..b { - let src_idx = b_idx * src_s0; - let dst_idx = b_idx * h_out * w_out * c * h_k * w_k; - for h_idx in 0..h_out { - let dst_idx = dst_idx + h_idx * w_out * c * h_k * w_k; - for w_idx in 0..w_out { - let dst_idx = dst_idx + w_idx * c * h_k * w_k; - for c_idx in 0..c { - let dst_idx = dst_idx + c_idx * h_k * w_k; - let src_idx = c_idx * src_s1 + src_idx; - for h_k_idx in 0..h_k { - let src_h = h_idx * stride + h_k_idx * dilation; - if padding != 0 && (src_h < padding || src_h >= h + padding) { - continue; - } - let src_h = src_h - padding; - let src_idx = src_idx + src_h * src_s2; - let dst_idx = dst_idx + h_k_idx * w_k; - for w_k_idx in 0..w_k { - let src_w = w_idx * stride + w_k_idx * dilation; - if padding != 0 && (src_w < padding || src_w >= w + padding) { - continue; - } - let src_w = src_w - padding; - let src_idx = src_idx + src_w * src_s3; - let dst_idx = dst_idx + w_k_idx; - dst[dst_idx] = src[src_idx] - } - } - } - } - } - } - Ok(dst) - } -} - -struct ConvTranspose1D<'a>(&'a crate::conv::ParamsConvTranspose1D); - -impl<'a> Map2 for ConvTranspose1D<'a> { - const OP: &'static str = "conv_transpose1d"; - fn f(&self, inp: &[T], inp_l: &Layout, k: &[T], k_l: &Layout) -> Result> { - let p = self.0; - let inp = &inp[inp_l.start_offset()..]; - let k = &k[k_l.start_offset()..]; - let (inp_s0, inp_s1, inp_s2) = crate::shape::dims3(inp_l.stride())?; - let (k_s0, k_s1, k_s2) = crate::shape::dims3(k_l.stride())?; - let l_out = p.l_out(); - - // Output shape: [b_size, c_out, l_out]. - let dst_elems = p.c_out * l_out * p.b_size; - let dst = vec![T::zero(); dst_elems]; - let dst_s0 = p.c_out * l_out; - let dst_s1 = l_out; - let dst_s2 = 1; - - // TODO: Avoid making this copy if `inp` already has the appropriate layout. - let mut inp_cont = vec![T::zero(); p.b_size * p.c_in * p.l_in]; - let cont_s0 = p.l_in * p.c_in; - let cont_s1 = p.c_in; - for b_idx in 0..p.b_size { - for l_idx in 0..p.l_in { - for c_idx in 0..p.c_in { - let src_idx = b_idx * inp_s0 + c_idx * inp_s1 + l_idx * inp_s2; - let dst_idx = b_idx * cont_s0 + l_idx * cont_s1 + c_idx; - inp_cont[dst_idx] = inp[src_idx] - } - } - } - - for k_idx in 0..p.k_size { - (0..p.c_out).into_par_iter().for_each(|dst_c_idx| { - let k_cont = (0..p.c_in) - .map(|c_in_idx| k[c_in_idx * k_s0 + dst_c_idx * k_s1 + k_idx * k_s2]) - .collect::>(); - for b_idx in 0..p.b_size { - for l_idx in 0..p.l_in { - let out_idx = l_idx * p.stride + k_idx * p.dilation; - if out_idx < p.padding { - continue; - } - let out_idx = out_idx - p.padding; - if out_idx < l_out { - let inp_cont = &inp_cont[b_idx * cont_s0 + l_idx * cont_s1..]; - let dst_idx = b_idx * dst_s0 + out_idx * dst_s2 + dst_c_idx * dst_s1; - let mut d = T::zero(); - unsafe { - T::vec_dot(inp_cont.as_ptr(), k_cont.as_ptr(), &mut d, p.c_in) - } - let dst_p = dst.as_ptr(); - // Safety: dst_idx are uniques per dst_c_idx which is used to - // parallelise the different tasks so no two threads can try to - // write at the same location. - unsafe { - let ptr = dst_p.add(dst_idx) as *mut T; - *ptr += d - } - } - } - } - }) - } - Ok(dst) - } -} - -struct Conv2D<'a>(&'a crate::conv::ParamsConv2D); - -impl<'a> Map2 for Conv2D<'a> { - const OP: &'static str = "conv2d"; - fn f(&self, inp: &[T], inp_l: &Layout, k: &[T], k_l: &Layout) -> Result> { - let p = self.0; - let inp = &inp[inp_l.start_offset()..]; - let (inp_s0, inp_s1, inp_s2, inp_s3) = crate::shape::dims4(inp_l.stride())?; - let k = &k[k_l.start_offset()..]; - let (k_s0, k_s1, k_s2, k_s3) = crate::shape::dims4(k_l.stride())?; - let (out_h, out_w) = (p.out_h(), p.out_w()); - - // Output shape: [b_size, c_out, out_h, out_w]. - let dst = vec![T::zero(); p.b_size * p.c_out * out_h * out_w]; - - // TODO: Avoid making this copy if `inp` already has the appropriate layout. - let mut inp_cont = vec![T::zero(); p.b_size * p.c_in * p.i_h * p.i_w]; - let cont_s0 = p.i_h * p.i_w * p.c_in; - let cont_s1 = p.i_w * p.c_in; - let cont_s2 = p.c_in; - for b_idx in 0..p.b_size { - for h_idx in 0..p.i_h { - for w_idx in 0..p.i_w { - for c_idx in 0..p.c_in { - let src_idx = - b_idx * inp_s0 + c_idx * inp_s1 + h_idx * inp_s2 + w_idx * inp_s3; - let dst_idx = b_idx * cont_s0 + h_idx * cont_s1 + w_idx * cont_s2 + c_idx; - inp_cont[dst_idx] = inp[src_idx] - } - } - } - } - - for offset_h in 0..p.k_h { - for offset_w in 0..p.k_w { - (0..p.c_out).into_par_iter().for_each(|dst_c_idx| { - let dst_idx = dst_c_idx * out_w * out_h; - let k_cont = (0..p.c_in) - .map(|c_in_idx| { - k[dst_c_idx * k_s0 - + c_in_idx * k_s1 - + offset_h * k_s2 - + offset_w * k_s3] - }) - .collect::>(); - for b_idx in 0..p.b_size { - let dst_idx = dst_idx + b_idx * p.c_out * out_h * out_w; - for dst_h in 0..out_h { - let dst_idx = dst_idx + dst_h * out_w; - let src_h = p.stride * dst_h + offset_h * p.dilation; - if src_h < p.padding || src_h >= p.i_h + p.padding { - continue; - } - let src_h = src_h - p.padding; - for dst_w in 0..out_w { - let dst_idx = dst_idx + dst_w; - let src_w = p.stride * dst_w + offset_w * p.dilation; - if src_w < p.padding || src_w >= p.i_w + p.padding { - continue; - } - let src_w = src_w - p.padding; - let inp_cont = &inp_cont - [b_idx * cont_s0 + src_h * cont_s1 + src_w * cont_s2..]; - assert!(inp_cont.len() >= p.c_in); - assert!(k_cont.len() >= p.c_in); - let mut d = T::zero(); - unsafe { - T::vec_dot(inp_cont.as_ptr(), k_cont.as_ptr(), &mut d, p.c_in) - } - let dst_p = dst.as_ptr(); - // Safety: dst_idx are uniques per dst_c_idx which is used to parallelise - // the different tasks so no two threads can try to write at the same - // location. - unsafe { - let ptr = dst_p.add(dst_idx) as *mut T; - *ptr += d - } - } - } - } - }); - } - } - - Ok(dst) - } -} - -struct ConvTranspose2D<'a>(&'a crate::conv::ParamsConvTranspose2D); - -impl<'a> Map2 for ConvTranspose2D<'a> { - const OP: &'static str = "conv_transpose2d"; - fn f(&self, inp: &[T], inp_l: &Layout, k: &[T], k_l: &Layout) -> Result> { - let p = self.0; - let inp = &inp[inp_l.start_offset()..]; - let (inp_s0, inp_s1, inp_s2, inp_s3) = crate::shape::dims4(inp_l.stride())?; - let k = &k[k_l.start_offset()..]; - let (k_s0, k_s1, k_s2, k_s3) = crate::shape::dims4(k_l.stride())?; - let (out_h, out_w) = (p.out_h(), p.out_w()); - - // Output shape: [b_size, c_out, out_h, out_w]. - let dst = vec![T::zero(); p.b_size * p.c_out * out_h * out_w]; - let dst_s0 = p.c_out * out_h * out_w; - let dst_s1 = out_h * out_w; - let dst_s2 = out_w; - let dst_s3 = 1; - - // TODO: Avoid making this copy if `inp` already has the appropriate layout. - let mut inp_cont = vec![T::zero(); p.b_size * p.c_in * p.i_h * p.i_w]; - let cont_s0 = p.i_h * p.i_w * p.c_in; - let cont_s1 = p.i_w * p.c_in; - let cont_s2 = p.c_in; - for b_idx in 0..p.b_size { - for h_idx in 0..p.i_h { - for w_idx in 0..p.i_w { - for c_idx in 0..p.c_in { - let src_idx = - b_idx * inp_s0 + c_idx * inp_s1 + h_idx * inp_s2 + w_idx * inp_s3; - let dst_idx = b_idx * cont_s0 + h_idx * cont_s1 + w_idx * cont_s2 + c_idx; - inp_cont[dst_idx] = inp[src_idx] - } - } - } - } - - for k_y in 0..p.k_h { - for k_x in 0..p.k_w { - (0..p.c_out).into_par_iter().for_each(|dst_c_idx| { - let k_cont = (0..p.c_in) - .map(|c_in_idx| { - k[c_in_idx * k_s0 + dst_c_idx * k_s1 + k_y * k_s2 + k_x * k_s3] - }) - .collect::>(); - for b_idx in 0..p.b_size { - for inp_y in 0..p.i_h { - for inp_x in 0..p.i_w { - let out_x = inp_x * p.stride + k_x * p.dilation; - let out_y = inp_y * p.stride + k_y * p.dilation; - if out_x < p.padding || out_y < p.padding { - continue; - } - let out_x = out_x - p.padding; - let out_y = out_y - p.padding; - if out_x < out_w && out_y < out_h { - let inp_cont = &inp_cont - [b_idx * cont_s0 + inp_y * cont_s1 + inp_x * cont_s2..]; - let dst_idx = b_idx * dst_s0 - + out_y * dst_s2 - + out_x * dst_s3 - + dst_c_idx * dst_s1; - let mut d = T::zero(); - unsafe { - T::vec_dot( - inp_cont.as_ptr(), - k_cont.as_ptr(), - &mut d, - p.c_in, - ) - } - let dst_p = dst.as_ptr(); - // Safety: dst_idx are uniques per dst_c_idx which is used to - // parallelise the different tasks so no two threads can try to - // write at the same location. - unsafe { - let ptr = dst_p.add(dst_idx) as *mut T; - *ptr += d - } - } - } - } - } - }) - } - } - Ok(dst) - } -} - -struct MatMul((usize, usize, usize, usize)); - -impl MatMul { - fn striding_error(&self, lhs_l: &Layout, rhs_l: &Layout, msg: &'static str) -> Error { - Error::MatMulUnexpectedStriding(Box::new(crate::error::MatMulUnexpectedStriding { - lhs_l: lhs_l.clone(), - rhs_l: rhs_l.clone(), - bmnk: self.0, - msg, - })) - .bt() - } -} - -impl Map2 for MatMul { - const OP: &'static str = "mat_mul"; - - #[cfg(all(not(feature = "mkl"), not(feature = "accelerate")))] - fn f( - &self, - lhs: &[T], - lhs_l: &Layout, - rhs: &[T], - rhs_l: &Layout, - ) -> Result> { - use gemm::{gemm, Parallelism}; - - match T::DTYPE { - DType::F16 | DType::F32 | DType::F64 => {} - _ => Err(Error::UnsupportedDTypeForOp(T::DTYPE, "matmul").bt())?, - } - - let (b, m, n, k) = self.0; - let lhs = &lhs[lhs_l.start_offset()..]; - let rhs = &rhs[rhs_l.start_offset()..]; - - let lhs_stride = lhs_l.stride(); - let rhs_stride = rhs_l.stride(); - let rank = lhs_stride.len(); - let lhs_cs = lhs_stride[rank - 1]; - let lhs_rs = lhs_stride[rank - 2]; - - let rhs_cs = rhs_stride[rank - 1]; - let rhs_rs = rhs_stride[rank - 2]; - - let a_skip: usize = match lhs_stride[..rank - 2] { - [s1, stride] if s1 == stride * lhs_l.dims()[1] => stride, - [stride] => stride, - [] => m * k, - _ => Err(self.striding_error(lhs_l, rhs_l, "non-contiguous lhs"))?, - }; - let b_skip: usize = match rhs_stride[..rank - 2] { - [s1, stride] if s1 == stride * rhs_l.dims()[1] => stride, - [stride] => stride, - [] => n * k, - _ => Err(self.striding_error(lhs_l, rhs_l, "non-contiguous rhs"))?, - }; - let c_skip: usize = m * n; - - let dst_shape: Shape = (m, n).into(); - let dst_strides = dst_shape.stride_contiguous(); - let dst_rs = dst_strides[0]; - let dst_cs = dst_strides[1]; - - let mut dst = vec![T::zero(); b * m * n]; - let num_threads = crate::utils::get_num_threads(); - let parallelism = if num_threads > 1 { - Parallelism::Rayon(num_threads) - } else { - Parallelism::None - }; - for step in 0..b { - let lhs_p = &lhs[step * a_skip..]; - let rhs_p = &rhs[step * b_skip..]; - let dst_p = &mut dst[step * c_skip..]; - unsafe { - gemm( - /* m: usize = */ m, - /* n: usize = */ n, - /* k: usize = */ k, - /* dst: *mut T = */ dst_p.as_mut_ptr(), - /* dst_cs: isize = */ dst_cs as isize, - /* dst_rs: isize = */ dst_rs as isize, - /* read_dst: bool = */ false, - /* lhs: *const T = */ lhs_p.as_ptr(), - /* lhs_cs: isize = */ lhs_cs as isize, - /* lhs_rs: isize = */ lhs_rs as isize, - /* rhs: *const T = */ rhs_p.as_ptr(), - /* rhs_cs: isize = */ rhs_cs as isize, - /* rhs_rs: isize = */ rhs_rs as isize, - /* alpha: T = */ T::zero(), - /* beta: T = */ T::one(), - /* conj_dst: bool = */ false, - /* conj_lhs: bool = */ false, - /* conj_rhs: bool = */ false, - parallelism, - ) - } - } - Ok(dst) - } - - #[cfg(feature = "accelerate")] - fn f( - &self, - lhs: &[T], - lhs_l: &Layout, - rhs: &[T], - rhs_l: &Layout, - ) -> Result> { - let (b, m, n, k) = self.0; - let lhs = &lhs[lhs_l.start_offset()..]; - let rhs = &rhs[rhs_l.start_offset()..]; - - let lhs_stride = lhs_l.stride(); - let rhs_stride = rhs_l.stride(); - let rank = lhs_stride.len(); - - let a_skip: usize = match lhs_stride[..rank - 2] { - [s1, stride] if s1 == stride * lhs_l.dims()[1] => stride, - [stride] => stride, - [] => m * k, - _ => Err(self.striding_error(lhs_l, rhs_l, "non-contiguous lhs"))?, - }; - let b_skip: usize = match rhs_stride[..rank - 2] { - [s1, stride] if s1 == stride * rhs_l.dims()[1] => stride, - [stride] => stride, - [] => n * k, - _ => Err(self.striding_error(lhs_l, rhs_l, "non-contiguous rhs"))?, - }; - let c_skip: usize = m * n; - - let rhs_m1 = rhs_stride[rhs_stride.len() - 1]; - let rhs_m2 = rhs_stride[rhs_stride.len() - 2]; - let lhs_m1 = lhs_stride[lhs_stride.len() - 1]; - let lhs_m2 = lhs_stride[lhs_stride.len() - 2]; - - let (lda, transa) = if rhs_m1 == 1 && rhs_m2 == n { - (n as i32, b'N') - } else if rhs_m1 == k && rhs_m2 == 1 { - (k as i32, b'T') - } else { - Err(self.striding_error(lhs_l, rhs_l, "non-contiguous rhs"))? - }; - // The b tensor has dims batching, m, k (lhs) - let (ldb, transb) = if lhs_m1 == 1 && lhs_m2 == k { - (k as i32, b'N') - } else if lhs_m1 == m && lhs_m2 == 1 { - (m as i32, b'T') - } else { - Err(self.striding_error(lhs_l, rhs_l, "non-contiguous lhs"))? - }; - - let mut dst = vec![T::zero(); b * m * n]; - match T::DTYPE { - DType::F16 => { - crate::bail!("the accelerate backend does not support f16 matmul") - } - DType::F32 => { - for step in 0..b { - let lhs_p = &lhs[step * a_skip..]; - let rhs_p = &rhs[step * b_skip..]; - let dst_p = &mut dst[step * c_skip..]; - unsafe { - let a = rhs_p.as_ptr() as *const f32; - let b = lhs_p.as_ptr() as *const f32; - let c = dst_p.as_mut_ptr() as *mut f32; - let a = std::slice::from_raw_parts(a, a_skip); - let b = std::slice::from_raw_parts(b, b_skip); - let c = std::slice::from_raw_parts_mut(c, c_skip); - crate::accelerate::sgemm( - transa, transb, /* m= */ n as i32, /* n= */ m as i32, - /* k= */ k as i32, /* alpha= */ 1., /* a= */ a, - /* lda= */ lda, /* b= */ b, /* ldb= */ ldb, - /* beta= */ 0., /* c= */ c, /* ldc= */ n as i32, - ) - } - } - } - DType::F64 => { - for step in 0..b { - let lhs_p = &lhs[step * a_skip..]; - let rhs_p = &rhs[step * b_skip..]; - let dst_p = &mut dst[step * c_skip..]; - unsafe { - let a = rhs_p.as_ptr() as *const f64; - let b = lhs_p.as_ptr() as *const f64; - let c = dst_p.as_mut_ptr() as *mut f64; - let a = std::slice::from_raw_parts(a, a_skip); - let b = std::slice::from_raw_parts(b, b_skip); - let c = std::slice::from_raw_parts_mut(c, c_skip); - crate::accelerate::dgemm( - transa, transb, /* m= */ n as i32, /* n= */ m as i32, - /* k= */ k as i32, /* alpha= */ 1., /* a= */ a, - /* lda= */ lda, /* b= */ b, /* ldb= */ ldb, - /* beta= */ 0., /* c= */ c, /* ldc= */ n as i32, - ) - } - } - } - dtype => Err(Error::UnsupportedDTypeForOp(dtype, "matmul").bt())?, - } - Ok(dst) - } - - #[cfg(feature = "mkl")] - fn f( - &self, - lhs: &[T], - lhs_l: &Layout, - rhs: &[T], - rhs_l: &Layout, - ) -> Result> { - let (b, m, n, k) = self.0; - let lhs = &lhs[lhs_l.start_offset()..]; - let rhs = &rhs[rhs_l.start_offset()..]; - - let lhs_stride = lhs_l.stride(); - let rhs_stride = rhs_l.stride(); - let rank = lhs_stride.len(); - - let a_skip: usize = match lhs_stride[..rank - 2] { - [s1, stride] if s1 == stride * lhs_l.dims()[1] => stride, - [stride] => stride, - [] => m * k, - _ => Err(self.striding_error(lhs_l, rhs_l, "non-contiguous lhs"))?, - }; - let b_skip: usize = match rhs_stride[..rank - 2] { - [s1, stride] if s1 == stride * rhs_l.dims()[1] => stride, - [stride] => stride, - [] => n * k, - _ => Err(self.striding_error(lhs_l, rhs_l, "non-contiguous rhs"))?, - }; - let c_skip: usize = m * n; - - let rhs_m1 = rhs_stride[rhs_stride.len() - 1]; - let rhs_m2 = rhs_stride[rhs_stride.len() - 2]; - let lhs_m1 = lhs_stride[lhs_stride.len() - 1]; - let lhs_m2 = lhs_stride[lhs_stride.len() - 2]; - - let (lda, transa) = if rhs_m1 == 1 && rhs_m2 == n { - (n as i32, b'N') - } else if rhs_m1 == k && rhs_m2 == 1 { - (k as i32, b'T') - } else { - Err(self.striding_error(lhs_l, rhs_l, "non-contiguous rhs"))? - }; - // The b tensor has dims batching, m, k (lhs) - let (ldb, transb) = if lhs_m1 == 1 && lhs_m2 == k { - (k as i32, b'N') - } else if lhs_m1 == m && lhs_m2 == 1 { - (m as i32, b'T') - } else { - Err(self.striding_error(lhs_l, rhs_l, "non-contiguous lhs"))? - }; - - let mut dst = vec![T::zero(); b * m * n]; - match T::DTYPE { - DType::F16 => { - for step in 0..b { - let lhs_p = &lhs[step * a_skip..]; - let rhs_p = &rhs[step * b_skip..]; - let dst_p = &mut dst[step * c_skip..]; - unsafe { - let a = rhs_p.as_ptr() as *const f16; - let b = lhs_p.as_ptr() as *const f16; - let c = dst_p.as_mut_ptr() as *mut f16; - let a = std::slice::from_raw_parts(a, a_skip); - let b = std::slice::from_raw_parts(b, b_skip); - let c = std::slice::from_raw_parts_mut(c, c_skip); - crate::mkl::hgemm( - transa, - transb, - /* m= */ n as i32, - /* n= */ m as i32, - /* k= */ k as i32, - /* alpha= */ f16::ONE, - /* a= */ a, - /* lda= */ lda, - /* b= */ b, - /* ldb= */ ldb, - /* beta= */ f16::ZERO, - /* c= */ c, - /* ldc= */ n as i32, - ) - } - } - } - DType::F32 => { - for step in 0..b { - let lhs_p = &lhs[step * a_skip..]; - let rhs_p = &rhs[step * b_skip..]; - let dst_p = &mut dst[step * c_skip..]; - unsafe { - let a = rhs_p.as_ptr() as *const f32; - let b = lhs_p.as_ptr() as *const f32; - let c = dst_p.as_mut_ptr() as *mut f32; - let a = std::slice::from_raw_parts(a, a_skip); - let b = std::slice::from_raw_parts(b, b_skip); - let c = std::slice::from_raw_parts_mut(c, c_skip); - crate::mkl::sgemm( - transa, transb, /* m= */ n as i32, /* n= */ m as i32, - /* k= */ k as i32, /* alpha= */ 1., /* a= */ a, - /* lda= */ lda, /* b= */ b, /* ldb= */ ldb, - /* beta= */ 0., /* c= */ c, /* ldc= */ n as i32, - ) - } - } - } - DType::F64 => { - for step in 0..b { - let lhs_p = &lhs[step * a_skip..]; - let rhs_p = &rhs[step * b_skip..]; - let dst_p = &mut dst[step * c_skip..]; - unsafe { - let a = rhs_p.as_ptr() as *const f64; - let b = lhs_p.as_ptr() as *const f64; - let c = dst_p.as_mut_ptr() as *mut f64; - let a = std::slice::from_raw_parts(a, a_skip); - let b = std::slice::from_raw_parts(b, b_skip); - let c = std::slice::from_raw_parts_mut(c, c_skip); - crate::mkl::dgemm( - transa, transb, /* m= */ n as i32, /* n= */ m as i32, - /* k= */ k as i32, /* alpha= */ 1., /* a= */ a, - /* lda= */ lda, /* b= */ b, /* ldb= */ ldb, - /* beta= */ 0., /* c= */ c, /* ldc= */ n as i32, - ) - } - } - } - dtype => Err(Error::UnsupportedDTypeForOp(dtype, "matmul").bt())?, - } - Ok(dst) - } -} - -fn elu(v: T, alpha: T) -> T { - if v.is_sign_positive() { - v - } else { - (v.exp() - T::one()) * alpha - } -} - -impl CpuStorage { - pub fn as_slice(&self) -> Result<&[D]> { - D::cpu_storage_as_slice(self) - } - - pub fn concat(storages: &[CpuStorage]) -> Result { - let storage0 = &storages[0]; - let s = match storage0 { - Self::U8(_) => { - let storages = storages - .iter() - .map(|s| match s { - Self::U8(s) => Ok(s.as_slice()), - _ => crate::bail!("dtype mismatch"), - }) - .collect::>>()? - .concat(); - Self::U8(storages) - } - Self::U32(_) => { - let storages = storages - .iter() - .map(|s| match s { - Self::U32(s) => Ok(s.as_slice()), - _ => crate::bail!("dtype mismatch"), - }) - .collect::>>()? - .concat(); - Self::U32(storages) - } - Self::I64(_) => { - let storages = storages - .iter() - .map(|s| match s { - Self::I64(s) => Ok(s.as_slice()), - _ => crate::bail!("dtype mismatch"), - }) - .collect::>>()? - .concat(); - Self::I64(storages) - } - Self::BF16(_) => { - let storages = storages - .iter() - .map(|s| match s { - Self::BF16(s) => Ok(s.as_slice()), - _ => crate::bail!("dtype mismatch"), - }) - .collect::>>()? - .concat(); - Self::BF16(storages) - } - Self::F16(_) => { - let storages = storages - .iter() - .map(|s| match s { - Self::F16(s) => Ok(s.as_slice()), - _ => crate::bail!("dtype mismatch"), - }) - .collect::>>()? - .concat(); - Self::F16(storages) - } - Self::F32(_) => { - let storages = storages - .iter() - .map(|s| match s { - Self::F32(s) => Ok(s.as_slice()), - _ => crate::bail!("dtype mismatch"), - }) - .collect::>>()? - .concat(); - Self::F32(storages) - } - Self::F64(_) => { - let storages = storages - .iter() - .map(|s| match s { - Self::F64(s) => Ok(s.as_slice()), - _ => crate::bail!("dtype mismatch"), - }) - .collect::>>()? - .concat(); - Self::F64(storages) - } - }; - Ok(s) - } -} - -impl BackendStorage for CpuStorage { - type Device = CpuDevice; - - fn dtype(&self) -> DType { - match self { - Self::U8(_) => DType::U8, - Self::U32(_) => DType::U32, - Self::I64(_) => DType::I64, - Self::BF16(_) => DType::BF16, - Self::F16(_) => DType::F16, - Self::F32(_) => DType::F32, - Self::F64(_) => DType::F64, - } - } - - fn to_dtype(&self, layout: &Layout, dtype: DType) -> Result { - // TODO: find a way around the quadratic number of cases below. - match (self, dtype) { - (Self::U8(storage), DType::BF16) => { - let data = unary_map(storage, layout, |v| bf16::from_f32(v as f32)); - Ok(Self::BF16(data)) - } - (Self::U32(storage), DType::BF16) => { - let data = unary_map(storage, layout, |v| bf16::from_f32(v as f32)); - Ok(Self::BF16(data)) - } - (Self::I64(storage), DType::BF16) => { - let data = unary_map(storage, layout, |v| bf16::from_f32(v as f32)); - Ok(Self::BF16(data)) - } - (Self::BF16(storage), DType::BF16) => { - let data = unary_map(storage, layout, |v| v); - Ok(Self::BF16(data)) - } - (Self::F16(storage), DType::BF16) => { - let data = unary_map(storage, layout, |v| bf16::from_f32(v.to_f32())); - Ok(Self::BF16(data)) - } - (Self::F32(storage), DType::BF16) => { - let data = unary_map(storage, layout, bf16::from_f32); - Ok(Self::BF16(data)) - } - (Self::F64(storage), DType::BF16) => { - let data = unary_map(storage, layout, bf16::from_f64); - Ok(Self::BF16(data)) - } - (Self::U8(storage), DType::F16) => { - let data = unary_map(storage, layout, |v| f16::from_f32(v as f32)); - Ok(Self::F16(data)) - } - (Self::U32(storage), DType::F16) => { - let data = unary_map(storage, layout, |v| f16::from_f32(v as f32)); - Ok(Self::F16(data)) - } - (Self::I64(storage), DType::F16) => { - let data = unary_map(storage, layout, |v| f16::from_f32(v as f32)); - Ok(Self::F16(data)) - } - (Self::BF16(storage), DType::F16) => { - let data = unary_map(storage, layout, |v| f16::from_f32(v.to_f32())); - Ok(Self::F16(data)) - } - (Self::F16(storage), DType::F16) => { - let data = unary_map(storage, layout, |v| v); - Ok(Self::F16(data)) - } - (Self::F32(storage), DType::F16) => { - let data = unary_map(storage, layout, f16::from_f32); - Ok(Self::F16(data)) - } - (Self::F64(storage), DType::F16) => { - let data = unary_map(storage, layout, f16::from_f64); - Ok(Self::F16(data)) - } - (Self::U8(storage), DType::F32) => { - let data = unary_map(storage, layout, |v| v as f32); - Ok(Self::F32(data)) - } - (Self::U32(storage), DType::F32) => { - let data = unary_map(storage, layout, |v| v as f32); - Ok(Self::F32(data)) - } - (Self::I64(storage), DType::F32) => { - let data = unary_map(storage, layout, |v| v as f32); - Ok(Self::F32(data)) - } - (Self::BF16(storage), DType::F32) => { - let data = unary_map(storage, layout, |v| v.to_f32()); - Ok(Self::F32(data)) - } - (Self::F16(storage), DType::F32) => { - let data = unary_map(storage, layout, |v| v.to_f32()); - Ok(Self::F32(data)) - } - (Self::F32(storage), DType::F32) => { - let data = unary_map(storage, layout, |v| v); - Ok(Self::F32(data)) - } - (Self::F64(storage), DType::F32) => { - let data = unary_map(storage, layout, |v| v as f32); - Ok(Self::F32(data)) - } - (Self::U8(storage), DType::U8) => { - let data = unary_map(storage, layout, |v| v); - Ok(Self::U8(data)) - } - (Self::BF16(storage), DType::U8) => { - let data = unary_map(storage, layout, |v| v.to_f32() as u8); - Ok(Self::U8(data)) - } - (Self::F16(storage), DType::U8) => { - let data = unary_map(storage, layout, |v| v.to_f32() as u8); - Ok(Self::U8(data)) - } - (Self::F32(storage), DType::U8) => { - let data = unary_map(storage, layout, |v| v as u8); - Ok(Self::U8(data)) - } - (Self::F64(storage), DType::U8) => { - let data = unary_map(storage, layout, |v| v as u8); - Ok(Self::U8(data)) - } - (Self::U32(storage), DType::U8) => { - let data = unary_map(storage, layout, |v| v as u8); - Ok(Self::U8(data)) - } - (Self::I64(storage), DType::U8) => { - let data = unary_map(storage, layout, |v| v as u8); - Ok(Self::U8(data)) - } - (Self::U8(storage), DType::U32) => { - let data = unary_map(storage, layout, |v| v as u32); - Ok(Self::U32(data)) - } - (Self::U32(storage), DType::U32) => { - let data = unary_map(storage, layout, |v| v); - Ok(Self::U32(data)) - } - (Self::I64(storage), DType::U32) => { - let data = unary_map(storage, layout, |v| v as u32); - Ok(Self::U32(data)) - } - (Self::BF16(storage), DType::U32) => { - let data = unary_map(storage, layout, |v| v.to_f32() as u32); - Ok(Self::U32(data)) - } - (Self::F16(storage), DType::U32) => { - let data = unary_map(storage, layout, |v| v.to_f32() as u32); - Ok(Self::U32(data)) - } - (Self::F32(storage), DType::U32) => { - let data = unary_map(storage, layout, |v| v as u32); - Ok(Self::U32(data)) - } - (Self::F64(storage), DType::U32) => { - let data = unary_map(storage, layout, |v| v as u32); - Ok(Self::U32(data)) - } - (Self::U8(storage), DType::I64) => { - let data = unary_map(storage, layout, |v| v as i64); - Ok(Self::I64(data)) - } - (Self::U32(storage), DType::I64) => { - let data = unary_map(storage, layout, |v| v as i64); - Ok(Self::I64(data)) - } - (Self::I64(storage), DType::I64) => { - let data = unary_map(storage, layout, |v| v); - Ok(Self::I64(data)) - } - (Self::BF16(storage), DType::I64) => { - let data = unary_map(storage, layout, |v| v.to_f32() as i64); - Ok(Self::I64(data)) - } - (Self::F16(storage), DType::I64) => { - let data = unary_map(storage, layout, |v| v.to_f32() as i64); - Ok(Self::I64(data)) - } - (Self::F32(storage), DType::I64) => { - let data = unary_map(storage, layout, |v| v as i64); - Ok(Self::I64(data)) - } - (Self::F64(storage), DType::I64) => { - let data = unary_map(storage, layout, |v| v as i64); - Ok(Self::I64(data)) - } - (Self::U8(storage), DType::F64) => { - let data = unary_map(storage, layout, |v| v as f64); - Ok(Self::F64(data)) - } - (Self::U32(storage), DType::F64) => { - let data = unary_map(storage, layout, |v| v as f64); - Ok(Self::F64(data)) - } - (Self::I64(storage), DType::F64) => { - let data = unary_map(storage, layout, |v| v as f64); - Ok(Self::F64(data)) - } - (Self::BF16(storage), DType::F64) => { - let data = unary_map(storage, layout, |v| v.to_f64()); - Ok(Self::F64(data)) - } - (Self::F16(storage), DType::F64) => { - let data = unary_map(storage, layout, |v| v.to_f64()); - Ok(Self::F64(data)) - } - (Self::F32(storage), DType::F64) => { - let data = unary_map(storage, layout, |v| v as f64); - Ok(Self::F64(data)) - } - (Self::F64(storage), DType::F64) => { - let data = unary_map(storage, layout, |v| v); - Ok(Self::F64(data)) - } - } - } - - fn reduce_op(&self, op: ReduceOp, layout: &Layout, reduce_dims: &[usize]) -> Result { - match op { - ReduceOp::Sum => { - let src_dims = layout.dims(); - let mut dst_dims = src_dims.to_vec(); - for &dim in reduce_dims.iter() { - dst_dims[dim] = 1; - } - let dst_shape = Shape::from(dst_dims); - let mut reduce_dims = reduce_dims.to_vec(); - // Sort the reduce_dims as they have to be processed from left to right when converting the - // indexes. - reduce_dims.sort(); - let reduce_dims_and_stride: Vec<_> = reduce_dims - .iter() - .map(|&d| (src_dims[d], src_dims[d + 1..].iter().product::())) - .collect(); - ReduceSum { - dst_shape: &dst_shape, - reduce_dims: &reduce_dims, - reduce_dims_and_stride, - } - .map(self, layout) - } - ReduceOp::Min | ReduceOp::ArgMin | ReduceOp::Max | ReduceOp::ArgMax => { - let reduce_dim_index = match reduce_dims { - [reduce_dim_index] => *reduce_dim_index, - _ => { - let op = match op { - ReduceOp::Min => "min", - ReduceOp::ArgMin => "argmin", - ReduceOp::Max => "max", - ReduceOp::ArgMax => "argmax", - _ => unreachable!(), - }; - let dims = reduce_dims.to_vec(); - Err(Error::OnlySingleDimension { op, dims })? - } - }; - let (use_min, return_index) = match op { - ReduceOp::Min => (true, false), - ReduceOp::ArgMin => (true, true), - ReduceOp::Max => (false, false), - ReduceOp::ArgMax => (false, true), - _ => unreachable!(), - }; - ReduceIndex { - reduce_dim_index, - use_min, - return_index, - } - .map(self, layout) - } - } - } - - fn cmp(&self, op: CmpOp, rhs: &Self, lhs_l: &Layout, rhs_l: &Layout) -> Result { - Cmp(op).map(self, lhs_l, rhs, rhs_l) - } - - fn affine(&self, layout: &Layout, mul: f64, add: f64) -> Result { - Affine(mul, add).map(self, layout) - } - - fn avg_pool2d( - &self, - layout: &Layout, - kernel_size: (usize, usize), - stride: (usize, usize), - ) -> Result { - AvgPool2D(kernel_size, stride).map(self, layout) - } - - fn max_pool2d( - &self, - layout: &Layout, - kernel_size: (usize, usize), - stride: (usize, usize), - ) -> Result { - MaxPool2D(kernel_size, stride).map(self, layout) - } - - fn upsample_nearest1d(&self, layout: &Layout, sz: usize) -> Result { - UpsampleNearest1D(sz).map(self, layout) - } - - fn upsample_nearest2d(&self, layout: &Layout, h: usize, w: usize) -> Result { - UpsampleNearest2D(h, w).map(self, layout) - } - - fn powf(&self, layout: &Layout, e: f64) -> Result { - use num_traits::Float; - // TODO: Have some generic map for functions that apply on num_traits::Float elements. - match self { - Self::BF16(storage) => { - let data = unary_map(storage, layout, |v| v.powf(bf16::from_f64(e))); - Ok(Self::BF16(data)) - } - Self::F16(storage) => { - let data = unary_map(storage, layout, |v| v.powf(f16::from_f64(e))); - Ok(Self::F16(data)) - } - Self::F32(storage) => { - let data = unary_map(storage, layout, |v| v.powf(e as f32)); - Ok(Self::F32(data)) - } - Self::F64(storage) => { - let data = unary_map(storage, layout, |v| v.powf(e)); - Ok(Self::F64(data)) - } - Self::U8(_) => Err(Error::UnsupportedDTypeForOp(DType::U8, "elu").bt()), - Self::U32(_) => Err(Error::UnsupportedDTypeForOp(DType::U32, "elu").bt()), - Self::I64(_) => Err(Error::UnsupportedDTypeForOp(DType::I64, "elu").bt()), - } - } - - fn elu(&self, layout: &Layout, alpha: f64) -> Result { - // TODO: Have some generic map for functions that apply on num_traits::Float elements. - match self { - Self::BF16(storage) => { - let data = unary_map(storage, layout, |v| elu(v, bf16::from_f64(alpha))); - Ok(Self::BF16(data)) - } - Self::F16(storage) => { - let data = unary_map(storage, layout, |v| elu(v, f16::from_f64(alpha))); - Ok(Self::F16(data)) - } - Self::F32(storage) => { - let data = unary_map(storage, layout, |v| elu(v, f32::from_f64(alpha))); - Ok(Self::F32(data)) - } - Self::F64(storage) => { - let data = unary_map(storage, layout, |v| elu(v, alpha)); - Ok(Self::F64(data)) - } - Self::U8(_) => Err(Error::UnsupportedDTypeForOp(DType::U8, "elu").bt()), - Self::U32(_) => Err(Error::UnsupportedDTypeForOp(DType::U32, "elu").bt()), - Self::I64(_) => Err(Error::UnsupportedDTypeForOp(DType::I64, "elu").bt()), - } - } - - fn unary_impl(&self, layout: &Layout) -> Result { - match self { - Self::BF16(storage) => { - if B::BF16_VEC { - let data = unary_map_vec(storage, layout, B::bf16, B::bf16_vec); - Ok(Self::BF16(data)) - } else { - let data = unary_map(storage, layout, B::bf16); - Ok(Self::BF16(data)) - } - } - Self::F16(storage) => { - if B::F16_VEC { - let data = unary_map_vec(storage, layout, B::f16, B::f16_vec); - Ok(Self::F16(data)) - } else { - let data = unary_map(storage, layout, B::f16); - Ok(Self::F16(data)) - } - } - Self::F32(storage) => { - if B::F32_VEC { - let data = unary_map_vec(storage, layout, B::f32, B::f32_vec); - Ok(Self::F32(data)) - } else { - let data = unary_map(storage, layout, B::f32); - Ok(Self::F32(data)) - } - } - Self::F64(storage) => { - if B::F64_VEC { - let data = unary_map_vec(storage, layout, B::f64, B::f64_vec); - Ok(Self::F64(data)) - } else { - let data = unary_map(storage, layout, B::f64); - Ok(Self::F64(data)) - } - } - Self::U8(storage) => { - let data = unary_map(storage, layout, B::u8); - Ok(Self::U8(data)) - } - Self::U32(storage) => { - let data = unary_map(storage, layout, B::u32); - Ok(Self::U32(data)) - } - Self::I64(storage) => { - let data = unary_map(storage, layout, B::i64); - Ok(Self::I64(data)) - } - } - } - - fn binary_impl( - &self, - rhs: &Self, - lhs_l: &Layout, - rhs_l: &Layout, - ) -> Result { - match (self, rhs) { - (Self::BF16(lhs), Self::BF16(rhs)) => { - let data = if B::BF16_VEC { - binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::bf16, B::bf16_vec) - } else { - binary_map(lhs_l, rhs_l, lhs, rhs, B::bf16) - }; - Ok(Self::BF16(data)) - } - (Self::F16(lhs), Self::F16(rhs)) => { - let data = if B::F16_VEC { - binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::f16, B::f16_vec) - } else { - binary_map(lhs_l, rhs_l, lhs, rhs, B::f16) - }; - Ok(Self::F16(data)) - } - (Self::F32(lhs), Self::F32(rhs)) => { - let data = if B::F32_VEC { - binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::f32, B::f32_vec) - } else { - binary_map(lhs_l, rhs_l, lhs, rhs, B::f32) - }; - Ok(Self::F32(data)) - } - (Self::F64(lhs), Self::F64(rhs)) => { - let data = if B::F64_VEC { - binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::f64, B::f64_vec) - } else { - binary_map(lhs_l, rhs_l, lhs, rhs, B::f64) - }; - Ok(Self::F64(data)) - } - (Self::U32(lhs), Self::U32(rhs)) => { - let data = if B::U32_VEC { - binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::u32, B::u32_vec) - } else { - binary_map(lhs_l, rhs_l, lhs, rhs, B::u32) - }; - Ok(Self::U32(data)) - } - (Self::I64(lhs), Self::I64(rhs)) => { - let data = if B::I64_VEC { - binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::i64, B::i64_vec) - } else { - binary_map(lhs_l, rhs_l, lhs, rhs, B::i64) - }; - Ok(Self::I64(data)) - } - (Self::U8(lhs), Self::U8(rhs)) => { - let data = if B::U8_VEC { - binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::u8, B::u8_vec) - } else { - binary_map(lhs_l, rhs_l, lhs, rhs, B::u8) - }; - Ok(Self::U8(data)) - } - _ => { - // This should be covered by the dtype check above. - Err(Error::DTypeMismatchBinaryOp { - lhs: self.dtype(), - rhs: rhs.dtype(), - op: B::NAME, - } - .bt()) - } - } - } - - fn copy_strided_src(&self, dst: &mut Self, dst_offset: usize, src_l: &Layout) -> Result<()> { - match (self, dst) { - (Self::U8(src), Self::U8(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), - (Self::U32(src), Self::U32(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), - (Self::I64(src), Self::I64(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), - (Self::BF16(src), Self::BF16(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), - (Self::F16(src), Self::F16(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), - (Self::F32(src), Self::F32(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), - (Self::F64(src), Self::F64(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), - (_, dst) => { - // This should be covered by the dtype check above. - return Err(Error::DTypeMismatchBinaryOp { - lhs: self.dtype(), - rhs: dst.dtype(), - op: "copy_strided", - } - .bt()); - } - } - Ok(()) - } - - fn where_cond( - &self, - layout: &Layout, - t: &Self, - t_l: &Layout, - f: &Self, - f_l: &Layout, - ) -> Result { - match self { - Self::U8(pred) => WCond(pred, layout).map(t, t_l, f, f_l), - Self::U32(pred) => WCond(pred, layout).map(t, t_l, f, f_l), - Self::I64(pred) => WCond(pred, layout).map(t, t_l, f, f_l), - _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), "where-cond")), - } - } - - fn conv1d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConv1D, - ) -> Result { - if !USE_IM2COL_CONV1D { - return Conv1D(params).map(self, l, kernel, kernel_l); - } - let op = Im2Col1D { - l_k: params.k_size, - padding: params.padding, - stride: params.stride, - dilation: params.dilation, - }; - let col = op.map(self, l)?; - let b = params.b_size; - let n = params.c_out; - let l_out = params.l_out(); - let k = op.l_k * params.c_in; - let m = l_out; - let col_l = Layout::contiguous((b, m, k)); - let res = if kernel_l.is_contiguous() { - let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) - .transpose(1, 2)? - .broadcast_as((b, k, n))?; - col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? - } else { - // Make the kernel contiguous if not already the case. - let mut kernel_c = self.device().zeros_impl(kernel_l.shape(), kernel.dtype())?; - kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?; - let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) - .transpose(1, 2)? - .broadcast_as((b, k, n))?; - col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? - }; - let res_l = Layout::contiguous((b, l_out, params.c_out)).transpose(1, 2)?; - let mut res_t = self.device().zeros_impl(res_l.shape(), res.dtype())?; - res.copy_strided_src(&mut res_t, 0, &res_l)?; - Ok(res_t) - } - - fn conv_transpose1d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConvTranspose1D, - ) -> Result { - ConvTranspose1D(params).map(self, l, kernel, kernel_l) - } - - fn conv2d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConv2D, - ) -> Result { - if !USE_IM2COL_CONV2D { - return Conv2D(params).map(self, l, kernel, kernel_l); - } - let op = Im2Col { - h_k: params.k_h, - w_k: params.k_w, - padding: params.padding, - stride: params.stride, - dilation: params.dilation, - }; - let col = op.map(self, l)?; - let b = params.b_size; - let n = params.c_out; - let (h_out, w_out) = (params.out_h(), params.out_w()); - let k = op.h_k * op.w_k * params.c_in; - let m = h_out * w_out; - let col_l = Layout::contiguous((b, m, k)); - let res = if kernel_l.is_contiguous() { - let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) - .transpose(1, 2)? - .broadcast_as((b, k, n))?; - col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? - } else { - // Make the kernel contiguous if not already the case. - let mut kernel_c = self.device().zeros_impl(kernel_l.shape(), kernel.dtype())?; - kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?; - let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) - .transpose(1, 2)? - .broadcast_as((b, k, n))?; - col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? - }; - let res_l = Layout::contiguous((b, h_out, w_out, params.c_out)) - .transpose(1, 2)? - .transpose(1, 3)?; - let mut res_t = self.device().zeros_impl(res_l.shape(), res.dtype())?; - res.copy_strided_src(&mut res_t, 0, &res_l)?; - Ok(res_t) - } - - fn conv_transpose2d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConvTranspose2D, - ) -> Result { - ConvTranspose2D(params).map(self, l, kernel, kernel_l) - } - - fn index_select(&self, ids: &Self, l: &Layout, ids_l: &Layout, dim: usize) -> Result { - match ids { - Self::U8(ids) => IndexSelect { ids, ids_l, dim }.map(self, l), - Self::U32(ids) => IndexSelect { ids, ids_l, dim }.map(self, l), - Self::I64(ids) => IndexSelect { ids, ids_l, dim }.map(self, l), - _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), "index-select").bt()), - } - } - - fn gather(&self, l: &Layout, ids: &Self, ids_l: &Layout, dim: usize) -> Result { - match ids { - Self::U8(ids) => Gather { ids, ids_l, dim }.map(self, l), - Self::U32(ids) => Gather { ids, ids_l, dim }.map(self, l), - Self::I64(ids) => Gather { ids, ids_l, dim }.map(self, l), - _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), "gather").bt()), - } - } - - fn scatter_add( - &self, - l: &Layout, - ids: &Self, - ids_l: &Layout, - src: &Self, - src_l: &Layout, - dim: usize, - ) -> Result { - match ids { - Self::U8(ids) => ScatterAdd { ids, ids_l, dim }.map(self, l, src, src_l), - Self::U32(ids) => ScatterAdd { ids, ids_l, dim }.map(self, l, src, src_l), - Self::I64(ids) => ScatterAdd { ids, ids_l, dim }.map(self, l, src, src_l), - _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), "scatter-add").bt()), - } - } - - fn index_add( - &self, - l: &Layout, - ids: &Self, - ids_l: &Layout, - src: &Self, - src_l: &Layout, - dim: usize, - ) -> Result { - match ids { - Self::U8(ids) => { - let ids = match ids_l.contiguous_offsets() { - Some((a, b)) => &ids[a..b], - None => Err(Error::RequiresContiguous { op: "index-add" }.bt())?, - }; - IndexAdd { ids, dim }.map(self, l, src, src_l) - } - Self::U32(ids) => { - let ids = match ids_l.contiguous_offsets() { - Some((a, b)) => &ids[a..b], - None => Err(Error::RequiresContiguous { op: "index-add" }.bt())?, - }; - IndexAdd { ids, dim }.map(self, l, src, src_l) - } - Self::I64(ids) => { - let ids = match ids_l.contiguous_offsets() { - Some((a, b)) => &ids[a..b], - None => Err(Error::RequiresContiguous { op: "index-add" }.bt())?, - }; - IndexAdd { ids, dim }.map(self, l, src, src_l) - } - _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), "index-add").bt()), - } - } - - fn matmul( - &self, - rhs: &Self, - bmnk: (usize, usize, usize, usize), - lhs_l: &Layout, - rhs_l: &Layout, - ) -> Result { - MatMul(bmnk).map(self, lhs_l, rhs, rhs_l) - } - - fn device(&self) -> &Self::Device { - &CpuDevice - } - - fn try_clone(&self, _: &Layout) -> Result { - Ok(self.clone()) - } - - fn to_cpu_storage(&self) -> Result { - Ok(self.clone()) - } -} - -impl BackendDevice for CpuDevice { - type Storage = CpuStorage; - - fn location(&self) -> crate::DeviceLocation { - crate::DeviceLocation::Cpu - } - - fn same_device(&self, _: &Self) -> bool { - true - } - - fn storage_from_cpu_storage(&self, s: &CpuStorage) -> Result { - Ok(s.clone()) - } - - fn new(_: usize) -> Result { - Ok(Self) - } - - fn set_seed(&self, _seed: u64) -> Result<()> { - crate::bail!("cannot seed the CPU rng with set_seed") - } - - fn rand_uniform(&self, shape: &Shape, dtype: DType, min: f64, max: f64) -> Result { - use rand::prelude::*; - - let elem_count = shape.elem_count(); - let mut rng = rand::thread_rng(); - match dtype { - DType::U8 | DType::U32 | DType::I64 => { - Err(Error::UnsupportedDTypeForOp(dtype, "rand_uniform").bt()) - } - DType::BF16 => { - let mut data = Vec::with_capacity(elem_count); - let uniform = - rand::distributions::Uniform::new(bf16::from_f64(min), bf16::from_f64(max)); - for _i in 0..elem_count { - data.push(rng.sample::(uniform)) - } - Ok(CpuStorage::BF16(data)) - } - DType::F16 => { - let mut data = Vec::with_capacity(elem_count); - let uniform = - rand::distributions::Uniform::new(f16::from_f64(min), f16::from_f64(max)); - for _i in 0..elem_count { - data.push(rng.sample::(uniform)) - } - Ok(CpuStorage::F16(data)) - } - DType::F32 => { - let mut data = Vec::with_capacity(elem_count); - let uniform = rand::distributions::Uniform::new(min as f32, max as f32); - for _i in 0..elem_count { - data.push(rng.sample::(uniform)) - } - Ok(CpuStorage::F32(data)) - } - DType::F64 => { - let mut data = Vec::with_capacity(elem_count); - let uniform = rand::distributions::Uniform::new(min, max); - for _i in 0..elem_count { - data.push(rng.sample::(uniform)) - } - Ok(CpuStorage::F64(data)) - } - } - } - - fn rand_normal(&self, shape: &Shape, dtype: DType, mean: f64, std: f64) -> Result { - use rand::prelude::*; - - let elem_count = shape.elem_count(); - let mut rng = rand::thread_rng(); - match dtype { - DType::U8 | DType::U32 | DType::I64 => { - Err(Error::UnsupportedDTypeForOp(dtype, "rand_normal").bt()) - } - DType::BF16 => { - let mut data = Vec::with_capacity(elem_count); - let normal = rand_distr::Normal::new(bf16::from_f64(mean), bf16::from_f64(std)) - .map_err(Error::wrap)?; - for _i in 0..elem_count { - data.push(normal.sample(&mut rng)) - } - Ok(CpuStorage::BF16(data)) - } - DType::F16 => { - let mut data = Vec::with_capacity(elem_count); - let normal = rand_distr::Normal::new(f16::from_f64(mean), f16::from_f64(std)) - .map_err(Error::wrap)?; - for _i in 0..elem_count { - data.push(normal.sample(&mut rng)) - } - Ok(CpuStorage::F16(data)) - } - DType::F32 => { - let mut data = Vec::with_capacity(elem_count); - let normal = - rand_distr::Normal::new(mean as f32, std as f32).map_err(Error::wrap)?; - for _i in 0..elem_count { - data.push(normal.sample(&mut rng)) - } - Ok(CpuStorage::F32(data)) - } - DType::F64 => { - let mut data = Vec::with_capacity(elem_count); - let normal = rand_distr::Normal::new(mean, std).map_err(Error::wrap)?; - for _i in 0..elem_count { - data.push(normal.sample(&mut rng)) - } - Ok(CpuStorage::F64(data)) - } - } - } - - fn ones_impl(&self, shape: &Shape, dtype: DType) -> Result { - let elem_count = shape.elem_count(); - let storage = match dtype { - DType::U8 => CpuStorage::U8(vec![1u8; elem_count]), - DType::U32 => CpuStorage::U32(vec![1u32; elem_count]), - DType::I64 => CpuStorage::I64(vec![1i64; elem_count]), - DType::BF16 => CpuStorage::BF16(vec![bf16::ONE; elem_count]), - DType::F16 => CpuStorage::F16(vec![f16::ONE; elem_count]), - DType::F32 => CpuStorage::F32(vec![1f32; elem_count]), - DType::F64 => CpuStorage::F64(vec![1f64; elem_count]), - }; - Ok(storage) - } - - fn zeros_impl(&self, shape: &Shape, dtype: DType) -> Result { - let elem_count = shape.elem_count(); - let storage = match dtype { - DType::U8 => CpuStorage::U8(vec![0u8; elem_count]), - DType::U32 => CpuStorage::U32(vec![0u32; elem_count]), - DType::I64 => CpuStorage::I64(vec![0i64; elem_count]), - DType::BF16 => CpuStorage::BF16(vec![bf16::ZERO; elem_count]), - DType::F16 => CpuStorage::F16(vec![f16::ZERO; elem_count]), - DType::F32 => CpuStorage::F32(vec![0f32; elem_count]), - DType::F64 => CpuStorage::F64(vec![0f64; elem_count]), - }; - Ok(storage) - } -} - -#[macro_export] -macro_rules! map_dtype { - ($name:expr, $storage:ident, $fn:expr, ($($dtypes:ident),+)) => { - match $storage { - $(CpuStorage::$dtypes(__e) => CpuStorage::$dtypes($fn(__e)),)* - s => Err(Error::UnsupportedDTypeForOp(s.dtype(), $name).bt())?, - } - }; -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/cuda_backend.rs b/v2/dependency-patches/candle-core-0.4.1/src/cuda_backend.rs deleted file mode 100644 index b7756fa6a4..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/cuda_backend.rs +++ /dev/null @@ -1,2249 +0,0 @@ -use crate::backend::{BackendDevice, BackendStorage}; -use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; -use crate::{CpuStorage, DType, Layout, Result, Shape, WithDType}; -pub use candle_kernels as kernels; -pub use cudarc; -use cudarc::cublas::{Gemm, GemmConfig, StridedBatchedConfig}; -use cudarc::driver::{ - CudaFunction, CudaSlice, DevicePtr, DeviceRepr, DeviceSlice, LaunchAsync, LaunchConfig, - ValidAsZeroBits, -}; -use half::{bf16, f16}; -use std::sync::{Arc, Mutex}; - -/// cudarc related errors -#[derive(thiserror::Error, Debug)] -pub enum CudaError { - #[error(transparent)] - Cuda(#[from] cudarc::driver::DriverError), - - #[error(transparent)] - Compiler(#[from] cudarc::nvrtc::CompileError), - - #[error(transparent)] - Cublas(#[from] cudarc::cublas::result::CublasError), - - #[error(transparent)] - Curand(#[from] cudarc::curand::result::CurandError), - - #[error("missing kernel '{module_name}'")] - MissingKernel { module_name: String }, - - #[error("unsupported dtype {dtype:?} for {op}")] - UnsupportedDtype { dtype: DType, op: &'static str }, - - #[error("internal error '{0}'")] - InternalError(&'static str), - - #[error("matmul is only supported for contiguous tensors lstride: {lhs_stride:?} rstride: {rhs_stride:?} mnk: {mnk:?}")] - MatMulNonContiguous { - lhs_stride: Vec, - rhs_stride: Vec, - mnk: (usize, usize, usize), - }, - - #[error("{msg}, expected: {expected:?}, got: {got:?}")] - UnexpectedDType { - msg: &'static str, - expected: DType, - got: DType, - }, - - #[error("{cuda} when loading {module_name}")] - Load { - cuda: cudarc::driver::DriverError, - module_name: String, - }, -} - -impl From for crate::Error { - fn from(val: CudaError) -> Self { - crate::Error::Cuda(Box::new(val)).bt() - } -} - -/// Unique identifier for cuda devices. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct DeviceId(usize); - -impl DeviceId { - fn new() -> Self { - // https://users.rust-lang.org/t/idiomatic-rust-way-to-generate-unique-id/33805 - use std::sync::atomic; - static COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::new(1); - Self(COUNTER.fetch_add(1, atomic::Ordering::Relaxed)) - } -} - -struct CudaRng(cudarc::curand::CudaRng); -unsafe impl Send for CudaRng {} - -#[derive(Clone)] -pub struct CudaDevice { - id: DeviceId, - device: Arc, - blas: Arc, - curand: Arc>, -} - -impl std::fmt::Debug for CudaDevice { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "CudaDevice({:?})", self.id) - } -} - -impl std::ops::Deref for CudaDevice { - type Target = Arc; - - fn deref(&self) -> &Self::Target { - &self.device - } -} - -pub trait WrapErr { - fn w(self) -> std::result::Result; -} - -impl> WrapErr for std::result::Result { - fn w(self) -> std::result::Result { - self.map_err(|e| crate::Error::Cuda(Box::new(e.into()))) - } -} - -impl CudaDevice { - pub fn cuda_device(&self) -> Arc { - self.device.clone() - } - - pub fn id(&self) -> DeviceId { - self.id - } - - fn const_impl(&self, v: f64, shape: &Shape, dtype: DType) -> Result { - let elem_count = shape.elem_count(); - let cfg = LaunchConfig::for_num_elems(elem_count as u32); - let slice = match dtype { - DType::U8 => { - // SAFETY: Set later by running the fill kernel. - let data = unsafe { self.alloc::(elem_count) }.w()?; - let func = self.get_or_load_func("fill_u8", kernels::FILL)?; - let params = (&data, v as u8, elem_count); - unsafe { func.launch(cfg, params) }.w()?; - CudaStorageSlice::U8(data) - } - DType::U32 => { - // SAFETY: Set later by running the fill kernel. - let data = unsafe { self.alloc::(elem_count) }.w()?; - let func = self.get_or_load_func("fill_u32", kernels::FILL)?; - let params = (&data, v as u32, elem_count); - unsafe { func.launch(cfg, params) }.w()?; - CudaStorageSlice::U32(data) - } - DType::I64 => { - // SAFETY: Set later by running the fill kernel. - let data = unsafe { self.alloc::(elem_count) }.w()?; - let func = self.get_or_load_func("fill_i64", kernels::FILL)?; - let params = (&data, v as i64, elem_count); - unsafe { func.launch(cfg, params) }.w()?; - CudaStorageSlice::I64(data) - } - DType::BF16 => { - // SAFETY: Set later by running the fill kernel. - let data = unsafe { self.alloc::(elem_count) }.w()?; - let func = self.get_or_load_func("fill_bf16", kernels::FILL)?; - let params = (&data, bf16::from_f64(v), elem_count); - unsafe { func.launch(cfg, params) }.w()?; - CudaStorageSlice::BF16(data) - } - DType::F16 => { - // SAFETY: Set later by running the fill kernel. - let data = unsafe { self.alloc::(elem_count) }.w()?; - let func = self.get_or_load_func("fill_f16", kernels::FILL)?; - let params = (&data, f16::from_f64(v), elem_count); - unsafe { func.launch(cfg, params) }.w()?; - CudaStorageSlice::F16(data) - } - DType::F32 => { - // SAFETY: Set later by running the fill kernel. - let data = unsafe { self.alloc::(elem_count) }.w()?; - let func = self.get_or_load_func("fill_f32", kernels::FILL)?; - let params = (&data, v as f32, elem_count); - unsafe { func.launch(cfg, params) }.w()?; - CudaStorageSlice::F32(data) - } - DType::F64 => { - // SAFETY: Set later by running the fill kernel. - let data = unsafe { self.alloc::(elem_count) }.w()?; - let func = self.get_or_load_func("fill_f64", kernels::FILL)?; - let params = (&data, v, elem_count); - unsafe { func.launch(cfg, params) }.w()?; - CudaStorageSlice::F64(data) - } - }; - Ok(CudaStorage { - slice, - device: self.clone(), - }) - } - - pub fn get_or_load_func(&self, module_name: &str, ptx: &'static str) -> Result { - if !self.has_func(module_name, module_name) { - // Leaking the string here is a bit sad but we need a &'static str and this is only - // done once per kernel name. - let static_module_name = Box::leak(module_name.to_string().into_boxed_str()); - self.load_ptx(ptx.into(), module_name, &[static_module_name]) - .map_err(|cuda| CudaError::Load { - cuda, - module_name: module_name.to_string(), - }) - .w()?; - } - self.get_func(module_name, module_name) - // Clippy recommends this `ok_or` rather than `ok_or_else` so hopefully the compiler is - // able to only build the error value if needed. - .ok_or(CudaError::MissingKernel { - module_name: module_name.to_string(), - }) - .w() - } -} - -impl BackendDevice for CudaDevice { - type Storage = CudaStorage; - - fn new(ordinal: usize) -> Result { - let device = cudarc::driver::CudaDevice::new(ordinal).w()?; - let blas = cudarc::cublas::CudaBlas::new(device.clone()).w()?; - let curand = cudarc::curand::CudaRng::new(299792458, device.clone()).w()?; - Ok(Self { - id: DeviceId::new(), - device, - blas: Arc::new(blas), - curand: Arc::new(Mutex::new(CudaRng(curand))), - }) - } - - fn set_seed(&self, seed: u64) -> Result<()> { - // We do not call set_seed but instead create a new curand object. This ensures that the - // state will be identical and the same random numbers will be generated. - let mut curand = self.curand.lock().unwrap(); - curand.0 = cudarc::curand::CudaRng::new(seed, self.device.clone()).w()?; - Ok(()) - } - - fn location(&self) -> crate::DeviceLocation { - crate::DeviceLocation::Cuda { - gpu_id: self.device.ordinal(), - } - } - - fn same_device(&self, rhs: &Self) -> bool { - self.id == rhs.id - } - - fn zeros_impl(&self, shape: &Shape, dtype: DType) -> Result { - let elem_count = shape.elem_count(); - let slice = match dtype { - DType::U8 => { - let data = self.alloc_zeros::(elem_count).w()?; - CudaStorageSlice::U8(data) - } - DType::U32 => { - let data = self.alloc_zeros::(elem_count).w()?; - CudaStorageSlice::U32(data) - } - DType::I64 => { - let data = self.alloc_zeros::(elem_count).w()?; - CudaStorageSlice::I64(data) - } - DType::BF16 => { - let data = self.alloc_zeros::(elem_count).w()?; - CudaStorageSlice::BF16(data) - } - DType::F16 => { - let data = self.alloc_zeros::(elem_count).w()?; - CudaStorageSlice::F16(data) - } - DType::F32 => { - let data = self.alloc_zeros::(elem_count).w()?; - CudaStorageSlice::F32(data) - } - DType::F64 => { - let data = self.alloc_zeros::(elem_count).w()?; - CudaStorageSlice::F64(data) - } - }; - Ok(CudaStorage { - slice, - device: self.clone(), - }) - } - - fn rand_uniform(&self, shape: &Shape, dtype: DType, lo: f64, up: f64) -> Result { - let elem_count = shape.elem_count(); - let curand = self.curand.lock().unwrap(); - let slice = match dtype { - // TODO: Add support for F16 and BF16 though this is likely to require some upstream - // cudarc changes. - DType::U8 | DType::U32 | DType::I64 | DType::F16 | DType::BF16 => { - Err(CudaError::UnsupportedDtype { - dtype, - op: "rand_uniform", - }) - .w()? - } - DType::F32 => { - let mut data = unsafe { self.alloc::(elem_count) }.w()?; - curand.0.fill_with_uniform(&mut data).w()?; - CudaStorageSlice::F32(data) - } - DType::F64 => { - let mut data = unsafe { self.alloc::(elem_count) }.w()?; - curand.0.fill_with_uniform(&mut data).w()?; - CudaStorageSlice::F64(data) - } - }; - let slice = if lo == 0. && up == 1.0 { - slice - } else { - let layout = Layout::contiguous(shape); - Affine(up - lo, lo).map(&slice, self, &layout)? - }; - Ok(CudaStorage { - slice, - device: self.clone(), - }) - } - - fn rand_normal(&self, shape: &Shape, dtype: DType, mean: f64, std: f64) -> Result { - // TODO: Add support for F16 and BF16 though this is likely to require some upstream - // cudarc changes. - let elem_count = shape.elem_count(); - let curand = self.curand.lock().unwrap(); - // curand can only generate an odd number of values. - // https://github.com/huggingface/candle/issues/734 - let elem_count_round = if elem_count % 2 == 1 { - elem_count + 1 - } else { - elem_count - }; - let slice = match dtype { - DType::U8 | DType::U32 | DType::I64 | DType::F16 | DType::BF16 => { - Err(CudaError::UnsupportedDtype { - dtype, - op: "rand_normal", - }) - .w()? - } - DType::F32 => { - let mut data = unsafe { self.alloc::(elem_count_round) }.w()?; - curand - .0 - .fill_with_normal(&mut data, mean as f32, std as f32) - .w()?; - CudaStorageSlice::F32(data) - } - DType::F64 => { - let mut data = unsafe { self.alloc::(elem_count_round) }.w()?; - curand.0.fill_with_normal(&mut data, mean, std).w()?; - CudaStorageSlice::F64(data) - } - }; - Ok(CudaStorage { - slice, - device: self.clone(), - }) - } - - fn ones_impl(&self, shape: &Shape, dtype: DType) -> Result { - self.const_impl(1., shape, dtype) - } - - fn storage_from_cpu_storage(&self, storage: &CpuStorage) -> Result { - let slice = match storage { - CpuStorage::U8(storage) => { - let data = self.htod_sync_copy(storage).w()?; - CudaStorageSlice::U8(data) - } - CpuStorage::U32(storage) => { - let data = self.htod_sync_copy(storage).w()?; - CudaStorageSlice::U32(data) - } - CpuStorage::I64(storage) => { - let data = self.htod_sync_copy(storage).w()?; - CudaStorageSlice::I64(data) - } - CpuStorage::BF16(storage) => { - let data = self.htod_sync_copy(storage).w()?; - CudaStorageSlice::BF16(data) - } - CpuStorage::F16(storage) => { - let data = self.htod_sync_copy(storage).w()?; - CudaStorageSlice::F16(data) - } - CpuStorage::F32(storage) => { - let data = self.htod_sync_copy(storage).w()?; - CudaStorageSlice::F32(data) - } - CpuStorage::F64(storage) => { - let data = self.htod_sync_copy(storage).w()?; - CudaStorageSlice::F64(data) - } - }; - Ok(CudaStorage { - slice, - device: self.clone(), - }) - } -} - -#[derive(Debug)] -pub enum CudaStorageSlice { - U8(CudaSlice), - U32(CudaSlice), - I64(CudaSlice), - BF16(CudaSlice), - F16(CudaSlice), - F32(CudaSlice), - F64(CudaSlice), -} -type S = CudaStorageSlice; - -pub trait Map1 { - fn f( - &self, - src: &CudaSlice, - dev: &CudaDevice, - layout: &Layout, - ) -> Result>; - - fn map(&self, s: &S, d: &CudaDevice, l: &Layout) -> Result { - let out = match s { - S::U8(s) => S::U8(self.f(s, d, l)?), - S::U32(s) => S::U32(self.f(s, d, l)?), - S::I64(s) => S::I64(self.f(s, d, l)?), - S::BF16(s) => S::BF16(self.f(s, d, l)?), - S::F16(s) => S::F16(self.f(s, d, l)?), - S::F32(s) => S::F32(self.f(s, d, l)?), - S::F64(s) => S::F64(self.f(s, d, l)?), - }; - Ok(out) - } -} - -pub trait Map2 { - fn f( - &self, - src1: &CudaSlice, - layout1: &Layout, - src2: &CudaSlice, - layout2: &Layout, - dev: &CudaDevice, - ) -> Result>; - - fn map(&self, s1: &S, l1: &Layout, s2: &S, l2: &Layout, d: &CudaDevice) -> Result { - let out = match (s1, s2) { - (S::U8(s1), S::U8(s2)) => S::U8(self.f(s1, l1, s2, l2, d)?), - (S::U32(s1), S::U32(s2)) => S::U32(self.f(s1, l1, s2, l2, d)?), - (S::I64(s1), S::I64(s2)) => S::I64(self.f(s1, l1, s2, l2, d)?), - (S::BF16(s1), S::BF16(s2)) => S::BF16(self.f(s1, l1, s2, l2, d)?), - (S::F16(s1), S::F16(s2)) => S::F16(self.f(s1, l1, s2, l2, d)?), - (S::F32(s1), S::F32(s2)) => S::F32(self.f(s1, l1, s2, l2, d)?), - (S::F64(s1), S::F64(s2)) => S::F64(self.f(s1, l1, s2, l2, d)?), - _ => Err(CudaError::InternalError("dtype mismatch in binary op"))?, - }; - Ok(out) - } -} - -pub trait Map2InPlace { - fn f( - &self, - dst: &mut CudaSlice, - dst_shape: &Shape, - src: &CudaSlice, - src_l: &Layout, - dev: &CudaDevice, - ) -> Result<()>; - - fn map( - &self, - dst: &mut S, - dst_s: &Shape, - src: &S, - src_l: &Layout, - d: &CudaDevice, - ) -> Result<()> { - match (dst, src) { - (S::U8(dst), S::U8(src)) => self.f(dst, dst_s, src, src_l, d), - (S::U32(dst), S::U32(src)) => self.f(dst, dst_s, src, src_l, d), - (S::I64(dst), S::I64(src)) => self.f(dst, dst_s, src, src_l, d), - (S::BF16(dst), S::BF16(src)) => self.f(dst, dst_s, src, src_l, d), - (S::F16(dst), S::F16(src)) => self.f(dst, dst_s, src, src_l, d), - (S::F32(dst), S::F32(src)) => self.f(dst, dst_s, src, src_l, d), - (S::F64(dst), S::F64(src)) => self.f(dst, dst_s, src, src_l, d), - _ => Err(CudaError::InternalError("dtype mismatch in binary op"))?, - } - } -} - -pub trait Map1Any { - fn f) -> S>( - &self, - src: &CudaSlice, - dev: &CudaDevice, - layout: &Layout, - wrap: W, - ) -> Result; - - fn map(&self, s: &S, d: &CudaDevice, l: &Layout) -> Result { - let out = match s { - S::U8(s) => self.f(s, d, l, S::U8)?, - S::U32(s) => self.f(s, d, l, S::U32)?, - S::I64(s) => self.f(s, d, l, S::I64)?, - S::BF16(s) => self.f(s, d, l, S::BF16)?, - S::F16(s) => self.f(s, d, l, S::F16)?, - S::F32(s) => self.f(s, d, l, S::F32)?, - S::F64(s) => self.f(s, d, l, S::F64)?, - }; - Ok(out) - } -} - -pub trait Map2Any { - fn f( - &self, - src1: &CudaSlice, - layout1: &Layout, - src2: &CudaSlice, - layout2: &Layout, - dev: &CudaDevice, - ) -> Result; - - fn map(&self, s1: &S, l1: &Layout, s2: &S, l2: &Layout, d: &CudaDevice) -> Result { - let out = match (s1, s2) { - (S::U8(s1), S::U8(s2)) => self.f(s1, l1, s2, l2, d)?, - (S::U32(s1), S::U32(s2)) => self.f(s1, l1, s2, l2, d)?, - (S::I64(s1), S::I64(s2)) => self.f(s1, l1, s2, l2, d)?, - (S::BF16(s1), S::BF16(s2)) => self.f(s1, l1, s2, l2, d)?, - (S::F16(s1), S::F16(s2)) => self.f(s1, l1, s2, l2, d)?, - (S::F32(s1), S::F32(s2)) => self.f(s1, l1, s2, l2, d)?, - (S::F64(s1), S::F64(s2)) => self.f(s1, l1, s2, l2, d)?, - _ => Err(CudaError::InternalError("dtype mismatch in binary op")).w()?, - }; - Ok(out) - } -} - -struct Clone; -impl Map1 for Clone { - fn f( - &self, - s: &CudaSlice, - _: &CudaDevice, - _: &Layout, - ) -> Result> { - s.try_clone().w() - } -} - -pub fn kernel_name(root: &str) -> String { - let dtype = T::DTYPE.as_str(); - format!("{root}_{dtype}") -} - -struct Affine(f64, f64); -impl Map1 for Affine { - fn f( - &self, - src: &CudaSlice, - dev: &CudaDevice, - layout: &Layout, - ) -> Result> { - let shape = layout.shape(); - let dims = shape.dims(); - let el = shape.elem_count(); - let cfg = LaunchConfig::for_num_elems(el as u32); - let ds = dev.htod_copy([dims, layout.stride()].concat()).w()?; - let src = &src.slice(layout.start_offset()..); - let func = dev.get_or_load_func(&kernel_name::("affine"), kernels::AFFINE)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(el) }.w()?; - let params = ( - el, - dims.len(), - &ds, - src, - &out, - T::from_f64(self.0), - T::from_f64(self.1), - ); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()?; - Ok(out) - } -} - -struct Elu(f64); -impl Map1 for Elu { - fn f( - &self, - src: &CudaSlice, - dev: &CudaDevice, - layout: &Layout, - ) -> Result> { - let shape = layout.shape(); - let dims = shape.dims(); - let el = shape.elem_count(); - let cfg = LaunchConfig::for_num_elems(el as u32); - let ds = dev.htod_copy([dims, layout.stride()].concat()).w()?; - let src = &src.slice(layout.start_offset()..); - let func = dev.get_or_load_func(&kernel_name::("uelu"), kernels::UNARY)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(el) }.w()?; - let params = (el, dims.len(), &ds, T::from_f64(self.0), src, &out); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()?; - Ok(out) - } -} - -struct Im2Col1D { - l_k: usize, - stride: usize, - dilation: usize, - padding: usize, -} - -impl Im2Col1D { - fn l_out(&self, l: usize) -> usize { - (l + 2 * self.padding - self.dilation * (self.l_k - 1) - 1) / self.stride + 1 - } -} - -impl Map1 for Im2Col1D { - fn f( - &self, - src: &CudaSlice, - dev: &CudaDevice, - layout: &Layout, - ) -> Result> { - let shape = layout.shape(); - let dims = shape.dims(); - let l_out = self.l_out(dims[2]); - let dst_el = dims[0] * l_out * dims[1] * self.l_k; - let cfg = LaunchConfig::for_num_elems(dst_el as u32); - let ds = dev.htod_copy([dims, layout.stride()].concat()).w()?; - let src = &src.slice(layout.start_offset()..); - let func = dev.get_or_load_func(&kernel_name::("im2col1d"), kernels::CONV)?; - // SAFETY: Set later by running the kernel. - let dst = unsafe { dev.alloc::(dst_el) }.w()?; - let params = ( - dst_el, - l_out, - self.l_k, - self.stride, - self.padding, - self.dilation, - &ds, - src, - &dst, - ); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()?; - Ok(dst) - } -} - -struct Im2Col { - h_k: usize, - w_k: usize, - stride: usize, - dilation: usize, - padding: usize, -} - -impl Im2Col { - fn hw_out(&self, h: usize, w: usize) -> (usize, usize) { - let h_out = (h + 2 * self.padding - self.dilation * (self.h_k - 1) - 1) / self.stride + 1; - let w_out = (w + 2 * self.padding - self.dilation * (self.w_k - 1) - 1) / self.stride + 1; - (h_out, w_out) - } -} - -impl Map1 for Im2Col { - fn f( - &self, - src: &CudaSlice, - dev: &CudaDevice, - layout: &Layout, - ) -> Result> { - let shape = layout.shape(); - let dims = shape.dims(); - let (h_out, w_out) = self.hw_out(dims[2], dims[3]); - let dst_el = dims[0] * h_out * w_out * dims[1] * self.h_k * self.w_k; - let cfg = LaunchConfig::for_num_elems(dst_el as u32); - let ds = dev.htod_copy([dims, layout.stride()].concat()).w()?; - let src = &src.slice(layout.start_offset()..); - let func = dev.get_or_load_func(&kernel_name::("im2col"), kernels::CONV)?; - // SAFETY: Set later by running the kernel. - let dst = unsafe { dev.alloc::(dst_el) }.w()?; - let params = ( - dst_el, - h_out, - w_out, - self.h_k, - self.w_k, - self.stride, - self.padding, - self.dilation, - &ds, - src, - &dst, - ); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()?; - Ok(dst) - } -} - -struct Powf(f64); -impl Map1 for Powf { - fn f( - &self, - src: &CudaSlice, - dev: &CudaDevice, - layout: &Layout, - ) -> Result> { - let shape = layout.shape(); - let dims = shape.dims(); - let el = shape.elem_count(); - let cfg = LaunchConfig::for_num_elems(el as u32); - let ds = dev.htod_copy([dims, layout.stride()].concat()).w()?; - let src = &src.slice(layout.start_offset()..); - let func = dev.get_or_load_func(&kernel_name::("upowf"), kernels::UNARY)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(el) }.w()?; - let params = (el, dims.len(), &ds, T::from_f64(self.0), src, &out); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()?; - Ok(out) - } -} - -struct Sum<'a>(&'a [usize]); -impl<'a> Map1 for Sum<'a> { - fn f( - &self, - src: &CudaSlice, - dev: &CudaDevice, - layout: &Layout, - ) -> Result> { - let shape = layout.shape(); - let src_dims = shape.dims(); - let el = shape.elem_count(); - let mut dst_el = el; - for &sum_dim in self.0.iter() { - dst_el /= src_dims[sum_dim]; - } - let mut sum_dims = self.0.to_vec(); - // Sort the sum_dims as they have to be processed from left to right when converting the - // indexes. - sum_dims.sort(); - let sum_dims_l: Vec = sum_dims.iter().map(|&d| src_dims[d]).collect(); - let sum_dims_s: Vec = sum_dims - .iter() - .map(|&d| src_dims[d + 1..].iter().product::()) - .collect(); - let cfg = LaunchConfig::for_num_elems(el as u32); - let ds = dev - .htod_copy([src_dims, layout.stride(), &sum_dims_l, &sum_dims_s].concat()) - .w()?; - let src = &src.slice(layout.start_offset()..); - let func = dev.get_or_load_func(&kernel_name::("sum"), kernels::REDUCE)?; - let out = dev.alloc_zeros::(dst_el).w()?; - let params = (el, src_dims.len(), sum_dims.len(), &ds, src, &out); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()?; - Ok(out) - } -} - -struct FastReduce<'a>(&'a [usize], ReduceOp); -impl<'a> Map1Any for FastReduce<'a> { - fn f) -> S>( - &self, - src: &CudaSlice, - dev: &CudaDevice, - layout: &Layout, - wrap: W, - ) -> Result { - let src_stride = layout.stride(); - let src_dims = layout.shape().dims(); - let src_el: usize = src_dims.iter().product(); - // Source dims and strides with the sum dims at the end. - let mut dims = vec![]; - let mut stride = vec![]; - let mut dst_el: usize = 1; - for (dim_idx, &d) in src_dims.iter().enumerate() { - if !self.0.contains(&dim_idx) { - dst_el *= d; - dims.push(d); - stride.push(src_stride[dim_idx]); - } - } - for &dim_idx in self.0.iter() { - dims.push(src_dims[dim_idx]); - stride.push(src_stride[dim_idx]); - } - let el_to_sum_per_block = src_el / dst_el; - // The reduction loop requires the shared array to be properly initialized and for - // this we want the number of threads to be a power of two. - let block_dim = usize::min(1024, el_to_sum_per_block).next_power_of_two(); - let cfg = LaunchConfig { - // TODO: Maybe use grid_y if the output is too large? - // TODO: Specialized implementation when reducing on no or all dimensions or when - // reducing only aggregate a small number of elements together. - grid_dim: (dst_el as u32, 1, 1), - block_dim: (block_dim as u32, 1, 1), - shared_mem_bytes: 0, - }; - let ds = dev - .htod_copy([dims.as_slice(), stride.as_slice()].concat()) - .w()?; - let src = &src.slice(layout.start_offset()..); - let (name, check_empty, return_index) = match self.1 { - ReduceOp::Sum => ("fast_sum", false, false), - ReduceOp::Min => ("fast_min", true, false), - ReduceOp::Max => ("fast_max", true, false), - ReduceOp::ArgMin => ("fast_argmin", true, true), - ReduceOp::ArgMax => ("fast_argmax", true, true), - }; - if check_empty && layout.shape().elem_count() == 0 { - Err(crate::Error::EmptyTensor { op: "reduce" }.bt())? - } - let func = dev.get_or_load_func(&kernel_name::(name), kernels::REDUCE)?; - if return_index { - // SAFETY: filled in by the follow up kernel. - let out = unsafe { dev.alloc::(dst_el) }.w()?; - let params = (src_el, el_to_sum_per_block, src_dims.len(), &ds, src, &out); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()?; - Ok(S::U32(out)) - } else { - // SAFETY: filled in by the follow up kernel. - let out = unsafe { dev.alloc::(dst_el) }.w()?; - let params = (src_el, el_to_sum_per_block, src_dims.len(), &ds, src, &out); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()?; - Ok(wrap(out)) - } - } -} - -impl Map1 for U { - fn f( - &self, - src: &CudaSlice, - dev: &CudaDevice, - layout: &Layout, - ) -> Result> { - let shape = layout.shape(); - let dims = shape.dims(); - let el_count = shape.elem_count(); - let cfg = LaunchConfig::for_num_elems(el_count as u32); - let ds = dev.htod_copy([dims, layout.stride()].concat()).w()?; - let src = &src.slice(layout.start_offset()..); - let func = dev.get_or_load_func(&kernel_name::(U::KERNEL), kernels::UNARY)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(el_count) }.w()?; - let params = (el_count, dims.len(), &ds, src, &out); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()?; - Ok(out) - } -} - -struct IndexSelect<'a>(&'a CudaStorage, &'a Layout, usize); -impl<'a> Map1 for IndexSelect<'a> { - fn f( - &self, - src: &CudaSlice, - dev: &CudaDevice, - src_l: &Layout, - ) -> Result> { - let ids_l = &self.1; - let (name, ids) = match &self.0.slice { - CudaStorageSlice::U32(slice) => { - ("is_u32", *slice.slice(ids_l.start_offset()..).device_ptr()) - } - CudaStorageSlice::U8(slice) => { - ("is_u8", *slice.slice(ids_l.start_offset()..).device_ptr()) - } - CudaStorageSlice::I64(slice) => { - ("is_i64", *slice.slice(ids_l.start_offset()..).device_ptr()) - } - _ => Err(CudaError::UnexpectedDType { - msg: "index_select ids should be u8 or u32", - expected: DType::U32, - got: self.0.dtype(), - }) - .w()?, - }; - let ids_shape = ids_l.shape(); - let ids_dims = ids_shape.dims(); - let ds = dev.htod_copy([ids_dims, ids_l.stride()].concat()).w()?; - let src = match src_l.contiguous_offsets() { - Some((o1, o2)) => src.slice(o1..o2), - None => Err(crate::Error::RequiresContiguous { op: "index-select" }.bt())?, - }; - let left_size: usize = src_l.dims()[..self.2].iter().product(); - let right_size: usize = src_l.dims()[self.2 + 1..].iter().product(); - let src_dim_size = src_l.dims()[self.2]; - let ids_dim_size = ids_shape.elem_count(); - let dst_el = ids_shape.elem_count() * left_size * right_size; - let cfg = LaunchConfig::for_num_elems(dst_el as u32); - let func = dev.get_or_load_func(&kernel_name::(name), kernels::INDEXING)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(dst_el) }.w()?; - let params = ( - dst_el, - ids_dims.len(), - &ds, - ids, - &src, - &out, - left_size, - src_dim_size, - ids_dim_size, - right_size, - ); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()?; - Ok(out) - } -} - -struct Gather<'a>(&'a CudaStorage, &'a Layout, usize); -impl<'a> Map1 for Gather<'a> { - fn f( - &self, - src: &CudaSlice, - dev: &CudaDevice, - src_l: &Layout, - ) -> Result> { - let ids = &self.0; - let ids_l = &self.1; - let dim = self.2; - let (ids_o1, ids_o2) = match ids_l.contiguous_offsets() { - Some(o12) => o12, - None => Err(crate::Error::RequiresContiguous { op: "gather" }.bt())?, - }; - let (name, ids) = match &ids.slice { - CudaStorageSlice::U32(slice) => { - ("gather_u32", *slice.slice(ids_o1..ids_o2).device_ptr()) - } - CudaStorageSlice::U8(slice) => ("gather_u8", *slice.slice(ids_o1..ids_o2).device_ptr()), - CudaStorageSlice::I64(slice) => { - ("gather_i64", *slice.slice(ids_o1..ids_o2).device_ptr()) - } - _ => Err(CudaError::UnexpectedDType { - msg: "gather ids should be u8/u32/i64", - expected: DType::U32, - got: ids.dtype(), - })?, - }; - let el = ids_l.shape().elem_count(); - let cfg = LaunchConfig::for_num_elems(el as u32); - let src = match src_l.contiguous_offsets() { - Some((o1, o2)) => src.slice(o1..o2), - None => Err(crate::Error::RequiresContiguous { op: "gather" }.bt())?, - }; - let left_sz: usize = src_l.dims()[..dim].iter().product(); - let right_sz: usize = src_l.dims()[dim + 1..].iter().product(); - let src_dim_sz = src_l.dims()[dim]; - let ids_dim_sz = ids_l.dims()[dim]; - let func = dev.get_or_load_func(&kernel_name::(name), kernels::INDEXING)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(el) }.w()?; - let params = ( - el, ids, &src, &out, left_sz, src_dim_sz, ids_dim_sz, right_sz, - ); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()?; - Ok(out) - } -} - -struct IndexAdd<'a>(&'a CudaStorage, &'a Layout, usize); -impl<'a> Map2InPlace for IndexAdd<'a> { - fn f( - &self, - dst: &mut CudaSlice, - dst_shape: &Shape, - src: &CudaSlice, - src_l: &Layout, - dev: &CudaDevice, - ) -> Result<()> { - let ids = &self.0; - let ids_l = &self.1; - let dim = self.2; - let (ids_o1, ids_o2) = match ids_l.contiguous_offsets() { - Some(o12) => o12, - None => Err(crate::Error::RequiresContiguous { op: "index-add" }.bt())?, - }; - let (name, ids) = match &ids.slice { - CudaStorageSlice::U32(slice) => ("ia_u32", *slice.slice(ids_o1..ids_o2).device_ptr()), - CudaStorageSlice::I64(slice) => ("ia_i64", *slice.slice(ids_o1..ids_o2).device_ptr()), - CudaStorageSlice::U8(slice) => ("ia_u8", *slice.slice(ids_o1..ids_o2).device_ptr()), - _ => Err(CudaError::UnexpectedDType { - msg: "index-add ids should be u8/u32/i64", - expected: DType::U32, - got: ids.dtype(), - })?, - }; - let src = match src_l.contiguous_offsets() { - Some((o1, o2)) => src.slice(o1..o2), - None => Err(crate::Error::RequiresContiguous { op: "index-add" }.bt())?, - }; - let left_sz: usize = src_l.dims()[..dim].iter().product(); - let right_sz: usize = src_l.dims()[dim + 1..].iter().product(); - let src_dim_sz = src_l.dims()[dim]; - let dst_dim_sz = dst_shape.dims()[dim]; - let ids_dim_sz = ids_l.dims()[0]; - let cfg = LaunchConfig::for_num_elems((left_sz * right_sz) as u32); - let func = dev.get_or_load_func(&kernel_name::(name), kernels::INDEXING)?; - // SAFETY: Set later by running the kernel. - let params = ( - ids, ids_dim_sz, &src, dst, left_sz, src_dim_sz, dst_dim_sz, right_sz, - ); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()?; - Ok(()) - } -} - -struct ScatterAdd<'a>(&'a CudaStorage, &'a Layout, usize); -impl<'a> Map2InPlace for ScatterAdd<'a> { - fn f( - &self, - dst: &mut CudaSlice, - dst_shape: &Shape, - src: &CudaSlice, - src_l: &Layout, - dev: &CudaDevice, - ) -> Result<()> { - let ids = &self.0; - let ids_l = &self.1; - let dim = self.2; - let (ids_o1, ids_o2) = match ids_l.contiguous_offsets() { - Some(o12) => o12, - None => Err(crate::Error::RequiresContiguous { op: "scatter-add" }.bt())?, - }; - let (name, ids) = match &ids.slice { - CudaStorageSlice::U32(slice) => ("sa_u32", *slice.slice(ids_o1..ids_o2).device_ptr()), - CudaStorageSlice::I64(slice) => ("sa_i64", *slice.slice(ids_o1..ids_o2).device_ptr()), - CudaStorageSlice::U8(slice) => ("sa_u8", *slice.slice(ids_o1..ids_o2).device_ptr()), - _ => Err(CudaError::UnexpectedDType { - msg: "scatter-add ids should be u8/u32/i64", - expected: DType::U32, - got: ids.dtype(), - })?, - }; - let src = match src_l.contiguous_offsets() { - Some((o1, o2)) => src.slice(o1..o2), - None => Err(crate::Error::RequiresContiguous { op: "scatter-add" }.bt())?, - }; - let left_sz: usize = src_l.dims()[..dim].iter().product(); - let right_sz: usize = src_l.dims()[dim + 1..].iter().product(); - let src_dim_sz = src_l.dims()[dim]; - let dst_dim_sz = dst_shape.dims()[dim]; - let cfg = LaunchConfig::for_num_elems((left_sz * right_sz) as u32); - let func = dev.get_or_load_func(&kernel_name::(name), kernels::INDEXING)?; - // SAFETY: Set later by running the kernel. - let params = (ids, &src, dst, left_sz, src_dim_sz, dst_dim_sz, right_sz); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()?; - Ok(()) - } -} - -struct Conv1D<'a>(&'a crate::conv::ParamsConv1D); -impl<'a> Map2 for Conv1D<'a> { - fn f( - &self, - inp: &CudaSlice, - inp_l: &Layout, - k: &CudaSlice, - k_l: &Layout, - dev: &CudaDevice, - ) -> Result> { - // Kernel shape: (c_out, c_in_k, k_size) - // Input shape: (b_size, c_in, l_in) or (c_in, l_in) - let p = &self.0; - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(k_l.start_offset()..); - let shape = inp_l.shape(); - let dims = shape.dims(); - let el = shape.elem_count(); - let l_out = p.l_out(); - let dst_el = p.c_out * l_out * p.b_size; - let cfg = LaunchConfig::for_num_elems(dst_el as u32); - let func = dev.get_or_load_func(&kernel_name::("conv1d"), kernels::CONV)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(dst_el) }.w()?; - let ds = if dims.len() == 3 { - [dims, inp_l.stride(), k_l.dims(), k_l.stride()].concat() - } else if dims.len() == 2 { - [&[1], dims, &[1], inp_l.stride(), k_l.dims(), k_l.stride()].concat() - } else { - crate::bail!("unexpected input shape for conv1d {dims:?}") - }; - let ds = dev.htod_copy(ds).w()?; - let params = ( - el, l_out, p.stride, p.padding, p.dilation, &ds, inp, k, &out, - ); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()?; - Ok(out) - } -} - -struct Conv2D<'a>(&'a crate::conv::ParamsConv2D); -impl<'a> Map2 for Conv2D<'a> { - fn f( - &self, - inp: &CudaSlice, - inp_l: &Layout, - k: &CudaSlice, - k_l: &Layout, - dev: &CudaDevice, - ) -> Result> { - // Kernel shape: (c_out, c_in_k, h_k, w_k) - // Input shape: (b_size, c_in, h_in, w_in) - let p = &self.0; - let (out_w, out_h) = (p.out_w(), p.out_h()); - let dst_el = p.c_out * out_w * out_h * p.b_size; - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(k_l.start_offset()..); - let shape = inp_l.shape(); - let dims = shape.dims(); - let el = shape.elem_count(); - - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(dst_el) }.w()?; - let cfg = LaunchConfig::for_num_elems(dst_el as u32); - let func = dev.get_or_load_func(&kernel_name::("conv2d"), kernels::CONV)?; - let ds = if dims.len() == 4 { - [dims, inp_l.stride(), k_l.dims(), k_l.stride()].concat() - } else { - crate::bail!("unexpected input shape for conv2d {dims:?}") - }; - let ds = dev.htod_copy(ds).w()?; - let params = ( - el, out_w, out_h, p.stride, p.padding, p.dilation, &ds, inp, k, &out, - ); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()?; - Ok(out) - } -} - -struct ConvTranspose1D<'a>(&'a crate::conv::ParamsConvTranspose1D); -impl<'a> Map2 for ConvTranspose1D<'a> { - fn f( - &self, - inp: &CudaSlice, - inp_l: &Layout, - k: &CudaSlice, - k_l: &Layout, - dev: &CudaDevice, - ) -> Result> { - // Kernel shape: (c_in_k, c_out, l_k) - // Input shape: (b_size, c_in, l_in) - let p = &self.0; - let l_out = p.l_out(); - let dst_el = p.c_out * l_out * p.b_size; - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(k_l.start_offset()..); - let shape = inp_l.shape(); - let dims = shape.dims(); - let el = shape.elem_count(); - - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(dst_el) }.w()?; - let cfg = LaunchConfig::for_num_elems(dst_el as u32); - let func = dev.get_or_load_func(&kernel_name::("conv_transpose1d"), kernels::CONV)?; - let ds = if dims.len() == 3 { - [dims, inp_l.stride(), k_l.dims(), k_l.stride()].concat() - } else { - crate::bail!("unexpected input shape for conv_transpose1d {dims:?}") - }; - let ds = dev.htod_copy(ds).w()?; - let params = ( - el, - l_out, - p.stride, - p.padding, - p.output_padding, - p.dilation, - &ds, - inp, - k, - &out, - ); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()?; - Ok(out) - } -} - -struct ConvTranspose2D<'a>(&'a crate::conv::ParamsConvTranspose2D); -impl<'a> Map2 for ConvTranspose2D<'a> { - fn f( - &self, - inp: &CudaSlice, - inp_l: &Layout, - k: &CudaSlice, - k_l: &Layout, - dev: &CudaDevice, - ) -> Result> { - // Kernel shape: (c_in_k, c_out, h_k, w_k) - // Input shape: (b_size, c_in, h_in, w_in) - let p = &self.0; - let (out_w, out_h) = (p.out_w(), p.out_h()); - let dst_el = p.c_out * out_w * out_h * p.b_size; - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(k_l.start_offset()..); - let shape = inp_l.shape(); - let dims = shape.dims(); - let el = shape.elem_count(); - - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(dst_el) }.w()?; - let cfg = LaunchConfig::for_num_elems(dst_el as u32); - let func = dev.get_or_load_func(&kernel_name::("conv_transpose2d"), kernels::CONV)?; - let ds = if dims.len() == 4 { - [dims, inp_l.stride(), k_l.dims(), k_l.stride()].concat() - } else { - crate::bail!("unexpected input shape for conv_transpose2d {dims:?}") - }; - let ds = dev.htod_copy(ds).w()?; - let params = ( - el, - out_w, - out_h, - p.stride, - p.padding, - p.output_padding, - p.dilation, - &ds, - inp, - k, - &out, - ); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()?; - Ok(out) - } -} - -enum PoolOp { - Max, - Avg, -} - -struct Pool2D { - w_k: usize, - h_k: usize, - w_stride: usize, - h_stride: usize, - op: PoolOp, -} - -impl Map1 for Pool2D { - fn f( - &self, - inp: &CudaSlice, - dev: &CudaDevice, - inp_l: &Layout, - ) -> Result> { - // Input shape: (b_size, c, h, w) - let inp = &inp.slice(inp_l.start_offset()..); - let shape = inp_l.shape(); - let dims = shape.dims(); - let ds = if dims.len() == 4 { - [dims, inp_l.stride()].concat() - } else { - crate::bail!("unexpected input shape for pool {dims:?}") - }; - let el = shape.elem_count(); - let out_w = (dims[2] - self.w_k) / self.w_stride + 1; - let out_h = (dims[3] - self.h_k) / self.h_stride + 1; - let dst_el = out_w * out_h * dims[0] * dims[1]; - let cfg = LaunchConfig::for_num_elems(dst_el as u32); - let kname = match self.op { - PoolOp::Max => "max_pool2d", - PoolOp::Avg => "avg_pool2d", - }; - let func = dev.get_or_load_func(&kernel_name::(kname), kernels::CONV)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(dst_el) }.w()?; - let ds = dev.htod_copy(ds).w()?; - let params = ( - el, - self.w_k, - self.h_k, - self.w_stride, - self.h_stride, - &ds, - inp, - &out, - ); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()?; - Ok(out) - } -} - -struct UpsampleNearest2D(usize, usize); -impl Map1 for UpsampleNearest2D { - fn f( - &self, - inp: &CudaSlice, - dev: &CudaDevice, - inp_l: &Layout, - ) -> Result> { - // Input shape: (b_size, c, h, w) - let inp = &inp.slice(inp_l.start_offset()..); - let shape = inp_l.shape(); - let dims = shape.dims(); - let ds = if dims.len() == 4 { - [dims, inp_l.stride()].concat() - } else { - crate::bail!("unexpected input shape for upsample {dims:?}") - }; - let (out_w, out_h) = (self.0, self.1); - let dst_el = out_w * out_h * dims[0] * dims[1]; - let cfg = LaunchConfig::for_num_elems(dst_el as u32); - let func = dev.get_or_load_func(&kernel_name::("upsample_nearest2d"), kernels::CONV)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(dst_el) }.w()?; - let ds = dev.htod_copy(ds).w()?; - let scale_w = dims[2] as f64 / out_w as f64; - let scale_h = dims[3] as f64 / out_h as f64; - let params = (out_w, out_h, scale_w, scale_h, &ds, inp, &out); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()?; - Ok(out) - } -} - -struct WhereCond<'a>(&'a CudaStorage, &'a Layout); -impl<'a> Map2 for WhereCond<'a> { - fn f( - &self, - t: &CudaSlice, - layout_t: &Layout, - f: &CudaSlice, - layout_f: &Layout, - dev: &CudaDevice, - ) -> Result> { - let ids_l = &self.1; - let (ids, name) = match &self.0.slice { - CudaStorageSlice::U8(slice) => { - let ptr = *slice.slice(ids_l.start_offset()..).device_ptr(); - (ptr, "where_u8") - } - CudaStorageSlice::U32(slice) => { - let ptr = *slice.slice(ids_l.start_offset()..).device_ptr(); - (ptr, "where_u32") - } - CudaStorageSlice::I64(slice) => { - let ptr = *slice.slice(ids_l.start_offset()..).device_ptr(); - (ptr, "where_i64") - } - _ => Err(CudaError::UnexpectedDType { - msg: "where conditions should be u8/u32/i64", - expected: DType::U32, - got: self.0.dtype(), - }) - .w()?, - }; - let shape = ids_l.shape(); - let dims = shape.dims(); - let el = shape.elem_count(); - let cfg = LaunchConfig::for_num_elems(el as u32); - let ds = dev - .htod_copy([dims, ids_l.stride(), layout_t.stride(), layout_f.stride()].concat()) - .w()?; - let t = &t.slice(layout_t.start_offset()..); - let f = &f.slice(layout_f.start_offset()..); - let func = dev.get_or_load_func(&kernel_name::(name), kernels::TERNARY)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(el) }.w()?; - let params = (el, dims.len(), &ds, ids, t, f, &out); - // SAFETY: ffi - unsafe { func.launch(cfg, params) }.w()?; - Ok(out) - } -} - -impl Map2 for U { - fn f( - &self, - lhs: &CudaSlice, - lhs_l: &Layout, - rhs: &CudaSlice, - rhs_l: &Layout, - dev: &CudaDevice, - ) -> Result> { - let shape = lhs_l.shape(); - let dims = shape.dims(); - let elem_count = shape.elem_count(); - let cfg = LaunchConfig::for_num_elems(elem_count as u32); - let dims_and_strides = dev - .htod_copy([dims, lhs_l.stride(), rhs_l.stride()].concat()) - .w()?; - let lhs = &lhs.slice(lhs_l.start_offset()..); - let rhs = &rhs.slice(rhs_l.start_offset()..); - let func = dev.get_or_load_func(&kernel_name::(U::KERNEL), kernels::BINARY)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(elem_count) }.w()?; - let params = (elem_count, dims.len(), &dims_and_strides, lhs, rhs, &out); - // SAFETY: ffi - unsafe { func.launch(cfg, params) }.w()?; - Ok(out) - } -} - -struct Cmp(CmpOp); -impl Map2Any for Cmp { - fn f( - &self, - lhs: &CudaSlice, - lhs_l: &Layout, - rhs: &CudaSlice, - rhs_l: &Layout, - dev: &CudaDevice, - ) -> Result { - let shape = lhs_l.shape(); - let dims = shape.dims(); - let elem_count = shape.elem_count(); - let cfg = LaunchConfig::for_num_elems(elem_count as u32); - let dims_and_strides = dev - .htod_copy([dims, lhs_l.stride(), rhs_l.stride()].concat()) - .w()?; - let lhs = &lhs.slice(lhs_l.start_offset()..); - let rhs = &rhs.slice(rhs_l.start_offset()..); - let name = match self.0 { - CmpOp::Eq => "eq", - CmpOp::Ne => "ne", - CmpOp::Lt => "lt", - CmpOp::Le => "le", - CmpOp::Gt => "gt", - CmpOp::Ge => "ge", - }; - let func = dev.get_or_load_func(&kernel_name::(name), kernels::BINARY)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(elem_count) }.w()?; - let params = (elem_count, dims.len(), &dims_and_strides, lhs, rhs, &out); - // SAFETY: ffi - unsafe { func.launch(cfg, params) }.w()?; - Ok(S::U8(out)) - } -} - -fn slice_src_and_dst<'a, T>( - src: &'a CudaSlice, - src_l: &Layout, - dst: &'a mut CudaSlice, - dst_offset: usize, -) -> ( - cudarc::driver::CudaView<'a, T>, - cudarc::driver::CudaViewMut<'a, T>, -) { - let src_offset = src_l.start_offset(); - let to_copy = dst - .len() - .saturating_sub(dst_offset) - .min(src.len().saturating_sub(src_offset)); - let src = src.slice(src_offset..src_offset + to_copy); - let dst = dst.slice_mut(dst_offset..dst_offset + to_copy); - (src, dst) -} - -#[derive(Debug)] -pub struct CudaStorage { - pub slice: CudaStorageSlice, - pub device: CudaDevice, -} - -pub trait CudaDType: Sized { - fn as_cuda_slice(s: &CudaStorage) -> Result<&CudaSlice>; - fn wrap_cuda_slice(s: CudaSlice, dev: CudaDevice) -> CudaStorage; -} - -macro_rules! cuda_dtype { - ($ty:ty, $dtype:ident) => { - impl CudaDType for $ty { - fn as_cuda_slice(s: &CudaStorage) -> Result<&CudaSlice> { - match &s.slice { - CudaStorageSlice::$dtype(data) => Ok(&data), - _ => Err(crate::Error::UnexpectedDType { - expected: DType::$dtype, - got: s.dtype(), - msg: "unexpected dtype", - } - .bt()), - } - } - - fn wrap_cuda_slice(slice: CudaSlice, device: CudaDevice) -> CudaStorage { - let slice = CudaStorageSlice::$dtype(slice); - CudaStorage { slice, device } - } - } - }; -} -cuda_dtype!(u8, U8); -cuda_dtype!(u32, U32); -cuda_dtype!(i64, I64); -cuda_dtype!(f16, F16); -cuda_dtype!(bf16, BF16); -cuda_dtype!(f32, F32); -cuda_dtype!(f64, F64); - -impl CudaStorage { - pub fn wrap_cuda_slice(slice: CudaSlice, device: CudaDevice) -> CudaStorage { - T::wrap_cuda_slice(slice, device) - } - - pub fn as_cuda_slice(&self) -> Result<&CudaSlice> { - T::as_cuda_slice(self) - } -} - -fn gemm_config( - alpha: T, - beta: T, - (b, m, n, k): (usize, usize, usize, usize), - lhs_l: &Layout, - rhs_l: &Layout, -) -> Result> { - // https://docs.nvidia.com/cuda/cublas/index.html#cublas-t-gemm - use cudarc::cublas::sys::cublasOperation_t; - - let lhs_stride = lhs_l.stride(); - let rhs_stride = rhs_l.stride(); - let rhs_m1 = rhs_stride[rhs_stride.len() - 1]; - let rhs_m2 = rhs_stride[rhs_stride.len() - 2]; - let lhs_m1 = lhs_stride[lhs_stride.len() - 1]; - let lhs_m2 = lhs_stride[lhs_stride.len() - 2]; - // The a tensor has dims batching, k, n (rhs) - let (lda, transa) = if rhs_m1 == 1 && rhs_m2 == n { - (n as i32, cublasOperation_t::CUBLAS_OP_N) - } else if rhs_m1 == k && rhs_m2 == 1 { - (k as i32, cublasOperation_t::CUBLAS_OP_T) - } else { - Err(CudaError::MatMulNonContiguous { - lhs_stride: lhs_stride.to_vec(), - rhs_stride: rhs_stride.to_vec(), - mnk: (m, n, k), - })? - }; - // The b tensor has dims batching, m, k (lhs) - let (ldb, transb) = if lhs_m1 == 1 && lhs_m2 == k { - (k as i32, cublasOperation_t::CUBLAS_OP_N) - } else if lhs_m1 == m && lhs_m2 == 1 { - (m as i32, cublasOperation_t::CUBLAS_OP_T) - } else { - Err(CudaError::MatMulNonContiguous { - lhs_stride: lhs_stride.to_vec(), - rhs_stride: rhs_stride.to_vec(), - mnk: (m, n, k), - })? - }; - // The setup below was copied from: - // https://github.com/lebedov/scikit-cuda/blob/7e7300474286019c917a6c8a4bca59405c64fbce/tests/test_cublas.py#L531 - let gemm = GemmConfig { - alpha, - beta, - m: n as i32, - n: m as i32, - k: k as i32, - lda, - ldb, - ldc: n as i32, - transa, - transb, - }; - - let stride_b: usize = match lhs_stride[..lhs_stride.len() - 2] { - [s1, stride] if s1 == stride * lhs_l.dims()[1] => stride, - [stride] => stride, - [] => m * k, - _ => Err(CudaError::MatMulNonContiguous { - lhs_stride: lhs_stride.to_vec(), - rhs_stride: rhs_stride.to_vec(), - mnk: (m, n, k), - })?, - }; - let stride_a: usize = match rhs_stride[..rhs_stride.len() - 2] { - [s1, stride] if s1 == stride * rhs_l.dims()[1] => stride, - [stride] => stride, - [] => n * k, - _ => Err(CudaError::MatMulNonContiguous { - lhs_stride: lhs_stride.to_vec(), - rhs_stride: rhs_stride.to_vec(), - mnk: (m, n, k), - })?, - }; - - Ok(StridedBatchedConfig { - batch_size: b as i32, - gemm, - stride_a: stride_a as i64, - stride_b: stride_b as i64, - stride_c: (m * n) as i64, - }) -} - -impl BackendStorage for CudaStorage { - type Device = CudaDevice; - - fn try_clone(&self, layout: &Layout) -> Result { - let slice = Clone.map(&self.slice, self.device(), layout)?; - let device = self.device.clone(); - Ok(Self { slice, device }) - } - - fn dtype(&self) -> DType { - match self.slice { - CudaStorageSlice::U8(_) => DType::U8, - CudaStorageSlice::U32(_) => DType::U32, - CudaStorageSlice::I64(_) => DType::I64, - CudaStorageSlice::BF16(_) => DType::BF16, - CudaStorageSlice::F16(_) => DType::F16, - CudaStorageSlice::F32(_) => DType::F32, - CudaStorageSlice::F64(_) => DType::F64, - } - } - - fn device(&self) -> &CudaDevice { - &self.device - } - - fn to_dtype(&self, layout: &Layout, dtype: DType) -> Result { - let shape = layout.shape(); - let dims = shape.dims(); - let el = shape.elem_count(); - let cfg = LaunchConfig::for_num_elems(el as u32); - let dev = self.device(); - let ds = dev.htod_copy([dims, layout.stride()].concat()).w()?; - let start_o = layout.start_offset(); - // This returns an i64 rather than a &i64, this is useful to get around some temporary - // lifetime issue and is safe as long as self.slice does not go out of scope before inp - // is used. - let inp = match &self.slice { - CudaStorageSlice::U8(inp) => *inp.slice(start_o..).device_ptr(), - CudaStorageSlice::U32(inp) => *inp.slice(start_o..).device_ptr(), - CudaStorageSlice::I64(inp) => *inp.slice(start_o..).device_ptr(), - CudaStorageSlice::BF16(inp) => *inp.slice(start_o..).device_ptr(), - CudaStorageSlice::F16(inp) => *inp.slice(start_o..).device_ptr(), - CudaStorageSlice::F32(inp) => *inp.slice(start_o..).device_ptr(), - CudaStorageSlice::F64(inp) => *inp.slice(start_o..).device_ptr(), - }; - let inp = &inp; - - let kernel_name = format!("cast_{}_{}", self.dtype().as_str(), dtype.as_str()); - let func = dev.get_or_load_func(&kernel_name, kernels::CAST)?; - let slice = match dtype { - DType::U8 => { - let out = unsafe { dev.alloc::(el) }.w()?; - let params = (el, dims.len(), &ds, *inp, &out); - unsafe { func.launch(cfg, params) }.w()?; - CudaStorageSlice::U8(out) - } - DType::U32 => { - let out = unsafe { dev.alloc::(el) }.w()?; - let params = (el, dims.len(), &ds, *inp, &out); - unsafe { func.launch(cfg, params) }.w()?; - CudaStorageSlice::U32(out) - } - DType::I64 => { - let out = unsafe { dev.alloc::(el) }.w()?; - let params = (el, dims.len(), &ds, *inp, &out); - unsafe { func.launch(cfg, params) }.w()?; - CudaStorageSlice::I64(out) - } - DType::BF16 => { - let out = unsafe { dev.alloc::(el) }.w()?; - let params = (el, dims.len(), &ds, *inp, &out); - unsafe { func.launch(cfg, params) }.w()?; - CudaStorageSlice::BF16(out) - } - DType::F16 => { - let out = unsafe { dev.alloc::(el) }.w()?; - let params = (el, dims.len(), &ds, *inp, &out); - unsafe { func.launch(cfg, params) }.w()?; - CudaStorageSlice::F16(out) - } - DType::F32 => { - let out = unsafe { dev.alloc::(el) }.w()?; - let params = (el, dims.len(), &ds, *inp, &out); - unsafe { func.launch(cfg, params) }.w()?; - CudaStorageSlice::F32(out) - } - DType::F64 => { - let out = unsafe { dev.alloc::(el) }.w()?; - let params = (el, dims.len(), &ds, *inp, &out); - unsafe { func.launch(cfg, params) }.w()?; - CudaStorageSlice::F64(out) - } - }; - Ok(Self { - slice, - device: dev.clone(), - }) - } - - fn affine(&self, layout: &Layout, mul: f64, add: f64) -> Result { - let device = self.device().clone(); - let slice = Affine(mul, add).map(&self.slice, &device, layout)?; - Ok(Self { slice, device }) - } - - fn powf(&self, layout: &Layout, e: f64) -> Result { - let device = self.device().clone(); - let slice = Powf(e).map(&self.slice, &device, layout)?; - Ok(Self { slice, device }) - } - - fn elu(&self, layout: &Layout, alpha: f64) -> Result { - let device = self.device().clone(); - let slice = Elu(alpha).map(&self.slice, &device, layout)?; - Ok(Self { slice, device }) - } - - fn reduce_op(&self, op: ReduceOp, layout: &Layout, sum_dims: &[usize]) -> Result { - let device = self.device().clone(); - let slice = FastReduce(sum_dims, op).map(&self.slice, &device, layout)?; - Ok(Self { slice, device }) - } - - fn cmp(&self, op: CmpOp, rhs: &Self, lhs_l: &Layout, rhs_l: &Layout) -> Result { - let device = self.device().clone(); - let slice = Cmp(op).map(&self.slice, lhs_l, &rhs.slice, rhs_l, &device)?; - Ok(Self { slice, device }) - } - - fn unary_impl(&self, layout: &Layout) -> Result { - let device = self.device().clone(); - let slice = U::V.map(&self.slice, &device, layout)?; - Ok(Self { slice, device }) - } - - fn binary_impl( - &self, - rhs: &Self, - lhs_l: &Layout, - rhs_l: &Layout, - ) -> Result { - let device = self.device().clone(); - let slice = B::V.map(&self.slice, lhs_l, &rhs.slice, rhs_l, &device)?; - Ok(Self { slice, device }) - } - - fn to_cpu_storage(&self) -> Result { - match &self.slice { - CudaStorageSlice::U8(slice) => { - let dev = slice.device(); - let cpu_storage = dev.dtoh_sync_copy(slice).w()?; - Ok(CpuStorage::U8(cpu_storage)) - } - CudaStorageSlice::U32(slice) => { - let dev = slice.device(); - let cpu_storage = dev.dtoh_sync_copy(slice).w()?; - Ok(CpuStorage::U32(cpu_storage)) - } - CudaStorageSlice::I64(slice) => { - let dev = slice.device(); - let cpu_storage = dev.dtoh_sync_copy(slice).w()?; - Ok(CpuStorage::I64(cpu_storage)) - } - CudaStorageSlice::BF16(slice) => { - let dev = slice.device(); - let cpu_storage = dev.dtoh_sync_copy(slice).w()?; - Ok(CpuStorage::BF16(cpu_storage)) - } - CudaStorageSlice::F16(slice) => { - let dev = slice.device(); - let cpu_storage = dev.dtoh_sync_copy(slice).w()?; - Ok(CpuStorage::F16(cpu_storage)) - } - CudaStorageSlice::F32(slice) => { - let dev = slice.device(); - let cpu_storage = dev.dtoh_sync_copy(slice).w()?; - Ok(CpuStorage::F32(cpu_storage)) - } - CudaStorageSlice::F64(slice) => { - let dev = slice.device(); - let cpu_storage = dev.dtoh_sync_copy(slice).w()?; - Ok(CpuStorage::F64(cpu_storage)) - } - } - } - - fn where_cond( - &self, - layout: &Layout, - t: &Self, - t_l: &Layout, - f: &Self, - f_l: &Layout, - ) -> Result { - let device = self.device().clone(); - let slice = WhereCond(self, layout).map(&t.slice, t_l, &f.slice, f_l, &device)?; - Ok(Self { slice, device }) - } - - fn conv1d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConv1D, - ) -> Result { - const USE_IM2COL_CONV1D: bool = true; - - let device = self.device().clone(); - if !USE_IM2COL_CONV1D { - let slice = Conv1D(params).map(&self.slice, l, &kernel.slice, kernel_l, &device)?; - return Ok(Self { slice, device }); - } - - let col = Im2Col1D { - l_k: params.k_size, - stride: params.stride, - dilation: params.dilation, - padding: params.padding, - } - .map(&self.slice, &device, l)?; - let col = Self { slice: col, device }; - let l_out = params.l_out(); - let b = params.b_size; - let n = params.c_out; - let k = params.k_size * params.c_in; - let m = l_out; - let col_l = Layout::contiguous((b, m, k)); - let res = if kernel_l.is_contiguous() { - let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) - .transpose(1, 2)? - .broadcast_as((b, k, n))?; - col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? - } else { - // Make the kernel contiguous if not already the case. - let mut kernel_c = self.device().zeros_impl(kernel_l.shape(), kernel.dtype())?; - kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?; - let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) - .transpose(1, 2)? - .broadcast_as((b, k, n))?; - col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? - }; - let res_l = Layout::contiguous((b, l_out, n)).transpose(1, 2)?; - let mut res_t = self.device().zeros_impl(res_l.shape(), res.dtype())?; - res.copy_strided_src(&mut res_t, 0, &res_l)?; - Ok(res_t) - } - - fn conv_transpose1d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConvTranspose1D, - ) -> Result { - let device = self.device().clone(); - let slice = - ConvTranspose1D(params).map(&self.slice, l, &kernel.slice, kernel_l, &device)?; - Ok(Self { slice, device }) - } - - #[cfg(not(feature = "cudnn"))] - fn conv2d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConv2D, - ) -> Result { - const USE_IM2COL_CONV2D: bool = true; - - let device = self.device().clone(); - if !USE_IM2COL_CONV2D { - let slice = Conv2D(params).map(&self.slice, l, &kernel.slice, kernel_l, &device)?; - return Ok(Self { slice, device }); - } - - let col = Im2Col { - h_k: params.k_h, - w_k: params.k_w, - stride: params.stride, - dilation: params.dilation, - padding: params.padding, - } - .map(&self.slice, &device, l)?; - let col = Self { slice: col, device }; - let h_out = params.out_h(); - let w_out = params.out_w(); - let b = params.b_size; - let n = params.c_out; - let k = params.k_h * params.k_w * params.c_in; - let m = h_out * w_out; - let col_l = Layout::contiguous((b, m, k)); - let res = if kernel_l.is_contiguous() { - let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) - .transpose(1, 2)? - .broadcast_as((b, k, n))?; - col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? - } else { - // Make the kernel contiguous if not already the case. - let mut kernel_c = self.device().zeros_impl(kernel_l.shape(), kernel.dtype())?; - kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?; - let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) - .transpose(1, 2)? - .broadcast_as((b, k, n))?; - col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? - }; - let res_l = Layout::contiguous((b, h_out, w_out, n)) - .transpose(1, 2)? - .transpose(1, 3)?; - let mut res_t = self.device().zeros_impl(res_l.shape(), res.dtype())?; - res.copy_strided_src(&mut res_t, 0, &res_l)?; - Ok(res_t) - } - - #[cfg(feature = "cudnn")] - fn conv2d( - &self, - inp_l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConv2D, - ) -> Result { - let device = self.device().clone(); - if !kernel_l.is_contiguous() { - let slice = Conv2D(params).map(&self.slice, inp_l, &kernel.slice, kernel_l, &device)?; - return Ok(Self { slice, device }); - } - let (out_w, out_h) = (params.out_w(), params.out_h()); - let dst_el = params.c_out * out_w * out_h * params.b_size; - let slice = match (&self.slice, &kernel.slice) { - (S::U8(inp), S::U8(k)) => { - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(kernel_l.start_offset()..); - let mut out = unsafe { device.alloc::(dst_el) }.w()?; - crate::cudnn::launch_conv2d::(inp, inp_l, k, &mut out, params, &device) - .map_err(crate::Error::wrap)?; - S::U8(out) - } - (S::BF16(inp), S::BF16(k)) => { - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(kernel_l.start_offset()..); - let mut out = unsafe { device.alloc::(dst_el) }.w()?; - crate::cudnn::launch_conv2d::(inp, inp_l, k, &mut out, params, &device) - .map_err(crate::Error::wrap)?; - S::BF16(out) - } - (S::F16(inp), S::F16(k)) => { - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(kernel_l.start_offset()..); - let mut out = unsafe { device.alloc::(dst_el) }.w()?; - crate::cudnn::launch_conv2d::(inp, inp_l, k, &mut out, params, &device) - .map_err(crate::Error::wrap)?; - S::F16(out) - } - (S::F32(inp), S::F32(k)) => { - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(kernel_l.start_offset()..); - let mut out = unsafe { device.alloc::(dst_el) }.w()?; - crate::cudnn::launch_conv2d::(inp, inp_l, k, &mut out, params, &device) - .map_err(crate::Error::wrap)?; - S::F32(out) - } - (S::F64(inp), S::F64(k)) => { - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(kernel_l.start_offset()..); - let mut out = unsafe { device.alloc::(dst_el) }.w()?; - crate::cudnn::launch_conv2d::(inp, inp_l, k, &mut out, params, &device) - .map_err(crate::Error::wrap)?; - S::F64(out) - } - (S::U32(_), S::U32(_)) => Err(CudaError::InternalError("conv2d does not support u32"))?, - (S::I64(_), S::I64(_)) => Err(CudaError::InternalError("conv2d does not support i64"))?, - _ => Err(CudaError::InternalError("dtype mismatch in conv2d"))?, - }; - Ok(Self { slice, device }) - } - - fn conv_transpose2d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConvTranspose2D, - ) -> Result { - let device = self.device().clone(); - let slice = - ConvTranspose2D(params).map(&self.slice, l, &kernel.slice, kernel_l, &device)?; - Ok(Self { slice, device }) - } - - fn avg_pool2d(&self, l: &Layout, k: (usize, usize), stride: (usize, usize)) -> Result { - let device = self.device().clone(); - let slice = Pool2D { - w_k: k.0, - h_k: k.1, - w_stride: stride.0, - h_stride: stride.1, - op: PoolOp::Avg, - } - .map(&self.slice, &device, l)?; - Ok(Self { slice, device }) - } - - fn max_pool2d(&self, l: &Layout, k: (usize, usize), stride: (usize, usize)) -> Result { - let device = self.device().clone(); - let slice = Pool2D { - w_k: k.0, - h_k: k.1, - w_stride: stride.0, - h_stride: stride.1, - op: PoolOp::Max, - } - .map(&self.slice, &device, l)?; - Ok(Self { slice, device }) - } - - fn upsample_nearest1d(&self, _: &Layout, _out_sz: usize) -> Result { - crate::bail!("upsample-nearest1d is not supported on cuda") - } - - fn upsample_nearest2d(&self, l: &Layout, out_w: usize, out_h: usize) -> Result { - let device = self.device().clone(); - let slice = UpsampleNearest2D(out_w, out_h).map(&self.slice, &device, l)?; - Ok(Self { slice, device }) - } - - fn index_select(&self, ids: &Self, l: &Layout, ids_l: &Layout, dim: usize) -> Result { - let device = self.device().clone(); - let slice = IndexSelect(ids, ids_l, dim).map(&self.slice, &device, l)?; - Ok(Self { slice, device }) - } - fn gather(&self, l: &Layout, ids: &Self, ids_l: &Layout, dim: usize) -> Result { - let device = self.device().clone(); - let slice = Gather(ids, ids_l, dim).map(&self.slice, &device, l)?; - Ok(Self { slice, device }) - } - fn scatter_add( - &self, - l: &Layout, - ids: &Self, - ids_l: &Layout, - src: &Self, - src_l: &Layout, - dim: usize, - ) -> Result { - let device = self.device().clone(); - let mut acc = device.zeros_impl(l.shape(), self.dtype())?; - self.copy_strided_src(&mut acc, 0, l)?; - ScatterAdd(ids, ids_l, dim).map(&mut acc.slice, l.shape(), &src.slice, src_l, &device)?; - Ok(acc) - } - fn index_add( - &self, - l: &Layout, - ids: &Self, - ids_l: &Layout, - src: &Self, - src_l: &Layout, - dim: usize, - ) -> Result { - let device = self.device().clone(); - let mut acc = device.zeros_impl(l.shape(), self.dtype())?; - self.copy_strided_src(&mut acc, 0, l)?; - IndexAdd(ids, ids_l, dim).map(&mut acc.slice, l.shape(), &src.slice, src_l, &device)?; - Ok(acc) - } - - fn matmul( - &self, - rhs: &Self, - (b, m, n, k): (usize, usize, usize, usize), - lhs_l: &Layout, - rhs_l: &Layout, - ) -> Result { - let elem_count = b * m * n; - let dev = &self.device; - let slice = match (&self.slice, &rhs.slice) { - (CudaStorageSlice::BF16(lhs), CudaStorageSlice::BF16(rhs)) => { - let lhs = &lhs.slice(lhs_l.start_offset()..); - let rhs = &rhs.slice(rhs_l.start_offset()..); - let cfg = gemm_config(bf16::ONE, bf16::ZERO, (b, m, n, k), lhs_l, rhs_l)?; - let mut out = unsafe { dev.alloc::(elem_count) }.w()?; - unsafe { - self.device - .blas - .gemm_strided_batched(cfg, rhs, lhs, &mut out) - } - .w()?; - CudaStorageSlice::BF16(out) - } - (CudaStorageSlice::F16(lhs), CudaStorageSlice::F16(rhs)) => { - let lhs = &lhs.slice(lhs_l.start_offset()..); - let rhs = &rhs.slice(rhs_l.start_offset()..); - let cfg = gemm_config(f16::ONE, f16::ZERO, (b, m, n, k), lhs_l, rhs_l)?; - let mut out = unsafe { dev.alloc::(elem_count) }.w()?; - unsafe { - self.device - .blas - .gemm_strided_batched(cfg, rhs, lhs, &mut out) - } - .w()?; - CudaStorageSlice::F16(out) - } - (CudaStorageSlice::F32(lhs), CudaStorageSlice::F32(rhs)) => { - let lhs = &lhs.slice(lhs_l.start_offset()..); - let rhs = &rhs.slice(rhs_l.start_offset()..); - let cfg = gemm_config(1., 0., (b, m, n, k), lhs_l, rhs_l)?; - let mut out = unsafe { dev.alloc::(elem_count) }.w()?; - unsafe { - self.device - .blas - .gemm_strided_batched(cfg, rhs, lhs, &mut out) - } - .w()?; - CudaStorageSlice::F32(out) - } - (CudaStorageSlice::F64(lhs), CudaStorageSlice::F64(rhs)) => { - let lhs = &lhs.slice(lhs_l.start_offset()..); - let rhs = &rhs.slice(rhs_l.start_offset()..); - let cfg = gemm_config(1., 0., (b, m, n, k), lhs_l, rhs_l)?; - let mut out = unsafe { dev.alloc::(elem_count) }.w()?; - unsafe { - self.device - .blas - .gemm_strided_batched(cfg, rhs, lhs, &mut out) - } - .w()?; - CudaStorageSlice::F64(out) - } - _ => Err(CudaError::InternalError("dtype mismatch in matmul op"))?, - }; - let device = dev.clone(); - Ok(Self { slice, device }) - } - - fn copy_strided_src(&self, dst: &mut Self, dst_offset: usize, src_l: &Layout) -> Result<()> { - let src_shape = src_l.shape(); - let dims = src_shape.dims(); - let el_count = src_shape.elem_count(); - if el_count == 0 { - return Ok(()); - } - let cfg = LaunchConfig::for_num_elems(el_count as u32); - let dev = &self.device; - let ds = dev.htod_copy([dims, src_l.stride()].concat()).w()?; - match (&self.slice, &mut dst.slice) { - (CudaStorageSlice::BF16(src), CudaStorageSlice::BF16(dst)) => { - let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); - if src_l.is_contiguous() { - dev.dtod_copy(&src, &mut dst).w()? - } else { - let func = dev.get_or_load_func("ucopy_bf16", kernels::UNARY)?; - // SAFETY: Set later by running the kernel. - let params = (el_count, dims.len(), &ds, &src, &mut dst); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()? - } - } - (CudaStorageSlice::F16(src), CudaStorageSlice::F16(dst)) => { - let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); - if src_l.is_contiguous() { - dev.dtod_copy(&src, &mut dst).w()? - } else { - let func = dev.get_or_load_func("ucopy_f16", kernels::UNARY)?; - // SAFETY: Set later by running the kernel. - let params = (el_count, dims.len(), &ds, &src, &mut dst); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()? - } - } - (CudaStorageSlice::F32(src), CudaStorageSlice::F32(dst)) => { - let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); - if src_l.is_contiguous() { - dev.dtod_copy(&src, &mut dst).w()? - } else { - let func = dev.get_or_load_func("ucopy_f32", kernels::UNARY)?; - // SAFETY: Set later by running the kernel. - let params = (el_count, dims.len(), &ds, &src, &mut dst); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()? - } - } - (CudaStorageSlice::U8(src), CudaStorageSlice::U8(dst)) => { - let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); - if src_l.is_contiguous() { - dev.dtod_copy(&src, &mut dst).w()? - } else { - let func = dev.get_or_load_func("ucopy_u8", kernels::UNARY)?; - // SAFETY: Set later by running the kernel. - let params = (el_count, dims.len(), &ds, &src, &mut dst); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()? - } - } - (CudaStorageSlice::U32(src), CudaStorageSlice::U32(dst)) => { - let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); - if src_l.is_contiguous() { - dev.dtod_copy(&src, &mut dst).w()? - } else { - let func = dev.get_or_load_func("ucopy_u32", kernels::UNARY)?; - // SAFETY: Set later by running the kernel. - let params = (el_count, dims.len(), &ds, &src, &mut dst); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()? - } - } - (CudaStorageSlice::I64(src), CudaStorageSlice::I64(dst)) => { - let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); - if src_l.is_contiguous() { - dev.dtod_copy(&src, &mut dst).w()? - } else { - let func = dev.get_or_load_func("ucopy_i64", kernels::UNARY)?; - // SAFETY: Set later by running the kernel. - let params = (el_count, dims.len(), &ds, &src, &mut dst); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()? - } - } - (CudaStorageSlice::F64(src), CudaStorageSlice::F64(dst)) => { - let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); - if src_l.is_contiguous() { - dev.dtod_copy(&src, &mut dst).w()? - } else { - let func = dev.get_or_load_func("ucopy_f64", kernels::UNARY)?; - // SAFETY: Set later by running the kernel. - let params = (el_count, dims.len(), &ds, &src, &mut dst); - // SAFETY: ffi. - unsafe { func.launch(cfg, params) }.w()?; - } - } - _ => Err(CudaError::InternalError( - "dtype mismatch in copy_strided op", - ))?, - } - Ok(()) - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/cudnn.rs b/v2/dependency-patches/candle-core-0.4.1/src/cudnn.rs deleted file mode 100644 index 0c149cd0d0..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/cudnn.rs +++ /dev/null @@ -1,123 +0,0 @@ -use crate::WithDType; -use cudarc; -use cudarc::cudnn::safe::{Conv2dForward, Cudnn}; -use cudarc::driver::{CudaSlice, CudaView, DeviceRepr, ValidAsZeroBits}; -use std::cell::RefCell; -use std::collections::HashMap; -use std::sync::Arc; - -// The cudnn handles are stored per thread here rather than on the CudaDevice as they are neither -// send nor sync. -thread_local! { - static CUDNN: RefCell>> = HashMap::new().into(); -} - -impl From for crate::Error { - fn from(err: cudarc::cudnn::CudnnError) -> Self { - crate::Error::wrap(err) - } -} - -impl From for crate::Error { - fn from(err: cudarc::driver::DriverError) -> Self { - crate::Error::wrap(err) - } -} - -pub(crate) fn launch_conv2d< - T: DeviceRepr + WithDType + ValidAsZeroBits + cudarc::cudnn::CudnnDataType, ->( - src: &CudaView, - src_l: &crate::Layout, - filter: &CudaView, - dst: &mut CudaSlice, - params: &crate::conv::ParamsConv2D, - dev: &crate::cuda_backend::CudaDevice, -) -> crate::Result<()> { - use crate::conv::CudnnFwdAlgo as CandleAlgo; - use cudarc::cudnn::sys::cudnnConvolutionFwdAlgo_t as A; - - let device_id = dev.id(); - let cudnn = CUDNN.with(|cudnn| { - if let Some(cudnn) = cudnn.borrow().get(&device_id) { - return Ok(cudnn.clone()); - } - let c = Cudnn::new(dev.cuda_device()); - if let Ok(c) = &c { - cudnn.borrow_mut().insert(device_id, c.clone()); - } - c - })?; - let conv = cudnn.create_conv2d::( - /* pad */ [params.padding as i32, params.padding as i32], - /* stride */ [params.stride as i32, params.stride as i32], - /* dilation */ [params.dilation as i32, params.dilation as i32], - cudarc::cudnn::sys::cudnnConvolutionMode_t::CUDNN_CROSS_CORRELATION, - )?; - let x_shape = [ - params.b_size as i32, - params.c_in as i32, - params.i_h as i32, - params.i_w as i32, - ]; - // Note that `src` already starts at the proper offset. - let x = if src_l.is_contiguous() { - cudnn.create_4d_tensor( - cudarc::cudnn::sys::cudnnTensorFormat_t::CUDNN_TENSOR_NCHW, - x_shape, - )? - } else { - let s = src_l.stride(); - cudnn.create_4d_tensor_ex( - x_shape, - [s[0] as i32, s[1] as i32, s[2] as i32, s[3] as i32], - )? - }; - let w = cudnn.create_4d_filter( - cudarc::cudnn::sys::cudnnTensorFormat_t::CUDNN_TENSOR_NCHW, - [ - params.c_out as i32, - params.c_in as i32, - params.k_h as i32, - params.k_w as i32, - ], - )?; - let (w_out, h_out) = (params.out_w() as i32, params.out_h() as i32); - let y = cudnn.create_4d_tensor( - cudarc::cudnn::sys::cudnnTensorFormat_t::CUDNN_TENSOR_NCHW, - [params.b_size as i32, params.c_out as i32, h_out, w_out], - )?; - let conv2d = Conv2dForward { - conv: &conv, - x: &x, - w: &w, - y: &y, - }; - let alg = match params.cudnn_fwd_algo { - None => conv2d.pick_algorithm()?, - Some(CandleAlgo::ImplicitGemm) => A::CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM, - Some(CandleAlgo::ImplicitPrecompGemm) => { - A::CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM - } - Some(CandleAlgo::Gemm) => A::CUDNN_CONVOLUTION_FWD_ALGO_GEMM, - Some(CandleAlgo::Direct) => A::CUDNN_CONVOLUTION_FWD_ALGO_DIRECT, - Some(CandleAlgo::Fft) => A::CUDNN_CONVOLUTION_FWD_ALGO_FFT, - Some(CandleAlgo::FftTiling) => A::CUDNN_CONVOLUTION_FWD_ALGO_FFT_TILING, - Some(CandleAlgo::Winograd) => A::CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD, - Some(CandleAlgo::WinogradNonFused) => A::CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED, - Some(CandleAlgo::Count) => A::CUDNN_CONVOLUTION_FWD_ALGO_COUNT, - }; - let workspace_size = conv2d.get_workspace_size(alg)?; - let mut workspace = dev.cuda_device().alloc_zeros::(workspace_size)?; - unsafe { - conv2d.launch::, _, _, _>( - alg, - Some(&mut workspace), - (T::one(), T::zero()), - src, - filter, - dst, - )?; - } - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/device.rs b/v2/dependency-patches/candle-core-0.4.1/src/device.rs deleted file mode 100644 index 1e33021bf2..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/device.rs +++ /dev/null @@ -1,323 +0,0 @@ -use crate::backend::BackendDevice; -use crate::cpu_backend::CpuDevice; -use crate::{CpuStorage, DType, Result, Shape, Storage, WithDType}; - -/// A `DeviceLocation` represents a physical device whereas multiple `Device` -/// can live on the same location (typically for cuda devices). -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub enum DeviceLocation { - Cpu, - Cuda { gpu_id: usize }, - Metal { gpu_id: usize }, -} - -#[derive(Debug, Clone)] -pub enum Device { - Cpu, - Cuda(crate::CudaDevice), - Metal(crate::MetalDevice), -} - -pub trait NdArray { - fn shape(&self) -> Result; - - fn to_cpu_storage(&self) -> CpuStorage; -} - -impl NdArray for S { - fn shape(&self) -> Result { - Ok(Shape::from(())) - } - - fn to_cpu_storage(&self) -> CpuStorage { - S::to_cpu_storage(&[*self]) - } -} - -impl NdArray for &[S; N] { - fn shape(&self) -> Result { - Ok(Shape::from(self.len())) - } - - fn to_cpu_storage(&self) -> CpuStorage { - S::to_cpu_storage(self.as_slice()) - } -} - -impl NdArray for &[S] { - fn shape(&self) -> Result { - Ok(Shape::from(self.len())) - } - - fn to_cpu_storage(&self) -> CpuStorage { - S::to_cpu_storage(self) - } -} - -impl NdArray for &[[S; N]; M] { - fn shape(&self) -> Result { - Ok(Shape::from((M, N))) - } - - fn to_cpu_storage(&self) -> CpuStorage { - S::to_cpu_storage_owned(self.concat()) - } -} - -impl NdArray - for &[[[S; N3]; N2]; N1] -{ - fn shape(&self) -> Result { - Ok(Shape::from((N1, N2, N3))) - } - - fn to_cpu_storage(&self) -> CpuStorage { - let mut vec = Vec::with_capacity(N1 * N2 * N3); - for i1 in 0..N1 { - for i2 in 0..N2 { - vec.extend(self[i1][i2]) - } - } - S::to_cpu_storage_owned(vec) - } -} - -impl NdArray - for &[[[[S; N4]; N3]; N2]; N1] -{ - fn shape(&self) -> Result { - Ok(Shape::from((N1, N2, N3, N4))) - } - - fn to_cpu_storage(&self) -> CpuStorage { - let mut vec = Vec::with_capacity(N1 * N2 * N3 * N4); - for i1 in 0..N1 { - for i2 in 0..N2 { - for i3 in 0..N3 { - vec.extend(self[i1][i2][i3]) - } - } - } - S::to_cpu_storage_owned(vec) - } -} - -impl NdArray for Vec { - fn shape(&self) -> Result { - if self.is_empty() { - crate::bail!("empty array") - } - let shape0 = self[0].shape()?; - let n = self.len(); - for v in self.iter() { - let shape = v.shape()?; - if shape != shape0 { - crate::bail!("two elements have different shapes {shape:?} {shape0:?}") - } - } - Ok(Shape::from([[n].as_slice(), shape0.dims()].concat())) - } - - fn to_cpu_storage(&self) -> CpuStorage { - // This allocates intermediary memory and shouldn't be necessary. - let storages = self.iter().map(|v| v.to_cpu_storage()).collect::>(); - CpuStorage::concat(storages.as_slice()).unwrap() - } -} - -impl Device { - pub fn new_cuda(ordinal: usize) -> Result { - Ok(Self::Cuda(crate::CudaDevice::new(ordinal)?)) - } - - pub fn new_metal(ordinal: usize) -> Result { - Ok(Self::Metal(crate::MetalDevice::new(ordinal)?)) - } - - pub fn set_seed(&self, seed: u64) -> Result<()> { - match self { - Self::Cpu => CpuDevice.set_seed(seed), - Self::Cuda(c) => c.set_seed(seed), - Self::Metal(m) => m.set_seed(seed), - } - } - - pub fn same_device(&self, rhs: &Self) -> bool { - match (self, rhs) { - (Self::Cpu, Self::Cpu) => true, - (Self::Cuda(lhs), Self::Cuda(rhs)) => lhs.same_device(rhs), - (Self::Metal(lhs), Self::Metal(rhs)) => lhs.same_device(rhs), - _ => false, - } - } - - pub fn location(&self) -> DeviceLocation { - match self { - Self::Cpu => DeviceLocation::Cpu, - Self::Cuda(device) => device.location(), - Device::Metal(device) => device.location(), - } - } - - pub fn is_cpu(&self) -> bool { - matches!(self, Self::Cpu) - } - - pub fn is_cuda(&self) -> bool { - matches!(self, Self::Cuda(_)) - } - - pub fn is_metal(&self) -> bool { - matches!(self, Self::Metal(_)) - } - - pub fn cuda_if_available(ordinal: usize) -> Result { - if crate::utils::cuda_is_available() { - Self::new_cuda(ordinal) - } else { - Ok(Self::Cpu) - } - } - - pub(crate) fn rand_uniform_f64( - &self, - lo: f64, - up: f64, - shape: &Shape, - dtype: DType, - ) -> Result { - match self { - Device::Cpu => { - let storage = CpuDevice.rand_uniform(shape, dtype, lo, up)?; - Ok(Storage::Cpu(storage)) - } - Device::Cuda(device) => { - // TODO: Remove the special case if we start supporting generating f16/bf16 directly. - if dtype == DType::F16 || dtype == DType::BF16 { - let storage = device.rand_uniform(shape, DType::F32, lo, up)?; - Storage::Cuda(storage).to_dtype(&crate::Layout::contiguous(shape), dtype) - } else { - let storage = device.rand_uniform(shape, dtype, lo, up)?; - Ok(Storage::Cuda(storage)) - } - } - Device::Metal(device) => { - let storage = device.rand_uniform(shape, dtype, lo, up)?; - Ok(Storage::Metal(storage)) - } - } - } - - pub(crate) fn rand_uniform( - &self, - lo: T, - up: T, - shape: &Shape, - ) -> Result { - self.rand_uniform_f64(lo.to_f64(), up.to_f64(), shape, T::DTYPE) - } - - pub(crate) fn rand_normal_f64( - &self, - mean: f64, - std: f64, - shape: &Shape, - dtype: DType, - ) -> Result { - match self { - Device::Cpu => { - let storage = CpuDevice.rand_normal(shape, dtype, mean, std)?; - Ok(Storage::Cpu(storage)) - } - Device::Cuda(device) => { - // TODO: Remove the special case if we start supporting generating f16/bf16 directly. - if dtype == DType::F16 || dtype == DType::BF16 { - let storage = device.rand_normal(shape, DType::F32, mean, std)?; - Storage::Cuda(storage).to_dtype(&crate::Layout::contiguous(shape), dtype) - } else { - let storage = device.rand_normal(shape, dtype, mean, std)?; - Ok(Storage::Cuda(storage)) - } - } - Device::Metal(device) => { - let storage = device.rand_normal(shape, dtype, mean, std)?; - Ok(Storage::Metal(storage)) - } - } - } - - pub(crate) fn rand_normal( - &self, - mean: T, - std: T, - shape: &Shape, - ) -> Result { - self.rand_normal_f64(mean.to_f64(), std.to_f64(), shape, T::DTYPE) - } - - pub(crate) fn ones(&self, shape: &Shape, dtype: DType) -> Result { - match self { - Device::Cpu => { - let storage = CpuDevice.ones_impl(shape, dtype)?; - Ok(Storage::Cpu(storage)) - } - Device::Cuda(device) => { - let storage = device.ones_impl(shape, dtype)?; - Ok(Storage::Cuda(storage)) - } - Device::Metal(device) => { - let storage = device.ones_impl(shape, dtype)?; - Ok(Storage::Metal(storage)) - } - } - } - - pub(crate) fn zeros(&self, shape: &Shape, dtype: DType) -> Result { - match self { - Device::Cpu => { - let storage = CpuDevice.zeros_impl(shape, dtype)?; - Ok(Storage::Cpu(storage)) - } - Device::Cuda(device) => { - let storage = device.zeros_impl(shape, dtype)?; - Ok(Storage::Cuda(storage)) - } - Device::Metal(device) => { - let storage = device.zeros_impl(shape, dtype)?; - Ok(Storage::Metal(storage)) - } - } - } - - pub(crate) fn storage(&self, array: A) -> Result { - match self { - Device::Cpu => Ok(Storage::Cpu(array.to_cpu_storage())), - Device::Cuda(device) => { - let storage = array.to_cpu_storage(); - let storage = device.storage_from_cpu_storage(&storage)?; - Ok(Storage::Cuda(storage)) - } - Device::Metal(device) => { - let storage = array.to_cpu_storage(); - let storage = device.storage_from_cpu_storage(&storage)?; - Ok(Storage::Metal(storage)) - } - } - } - - pub(crate) fn storage_owned(&self, data: Vec) -> Result { - match self { - Device::Cpu => Ok(Storage::Cpu(S::to_cpu_storage_owned(data))), - Device::Cuda(device) => { - let storage = S::to_cpu_storage_owned(data); - let storage = device.storage_from_cpu_storage(&storage)?; - Ok(Storage::Cuda(storage)) - } - Device::Metal(device) => { - let storage = S::to_cpu_storage_owned(data); - let storage = device.storage_from_cpu_storage(&storage)?; - Ok(Storage::Metal(storage)) - } - } - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/display.rs b/v2/dependency-patches/candle-core-0.4.1/src/display.rs deleted file mode 100644 index 4f5a390e80..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/display.rs +++ /dev/null @@ -1,495 +0,0 @@ -/// Pretty printing of tensors -/// This implementation should be in line with the PyTorch version. -/// https://github.com/pytorch/pytorch/blob/7b419e8513a024e172eae767e24ec1b849976b13/torch/_tensor_str.py -use crate::{DType, Result, Tensor, WithDType}; -use half::{bf16, f16}; - -impl Tensor { - fn fmt_dt( - &self, - f: &mut std::fmt::Formatter, - ) -> std::fmt::Result { - let device_str = match self.device().location() { - crate::DeviceLocation::Cpu => "".to_owned(), - crate::DeviceLocation::Cuda { gpu_id } => { - format!(", cuda:{}", gpu_id) - } - crate::DeviceLocation::Metal { gpu_id } => { - format!(", metal:{}", gpu_id) - } - }; - - write!(f, "Tensor[")?; - match self.dims() { - [] => { - if let Ok(v) = self.to_scalar::() { - write!(f, "{v}")? - } - } - [s] if *s < 10 => { - if let Ok(vs) = self.to_vec1::() { - for (i, v) in vs.iter().enumerate() { - if i > 0 { - write!(f, ", ")?; - } - write!(f, "{v}")?; - } - } - } - dims => { - write!(f, "dims ")?; - for (i, d) in dims.iter().enumerate() { - if i > 0 { - write!(f, ", ")?; - } - write!(f, "{d}")?; - } - } - } - write!(f, "; {}{}]", self.dtype().as_str(), device_str) - } -} - -impl std::fmt::Debug for Tensor { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self.dtype() { - DType::U8 => self.fmt_dt::(f), - DType::U32 => self.fmt_dt::(f), - DType::I64 => self.fmt_dt::(f), - DType::BF16 => self.fmt_dt::(f), - DType::F16 => self.fmt_dt::(f), - DType::F32 => self.fmt_dt::(f), - DType::F64 => self.fmt_dt::(f), - } - } -} - -/// Options for Tensor pretty printing -pub struct PrinterOptions { - precision: usize, - threshold: usize, - edge_items: usize, - line_width: usize, - sci_mode: Option, -} - -static PRINT_OPTS: std::sync::Mutex = - std::sync::Mutex::new(PrinterOptions::const_default()); - -impl PrinterOptions { - // We cannot use the default trait as it's not const. - const fn const_default() -> Self { - Self { - precision: 4, - threshold: 1000, - edge_items: 3, - line_width: 80, - sci_mode: None, - } - } -} - -pub fn set_print_options(options: PrinterOptions) { - *PRINT_OPTS.lock().unwrap() = options -} - -pub fn set_print_options_default() { - *PRINT_OPTS.lock().unwrap() = PrinterOptions::const_default() -} - -pub fn set_print_options_short() { - *PRINT_OPTS.lock().unwrap() = PrinterOptions { - precision: 2, - threshold: 1000, - edge_items: 2, - line_width: 80, - sci_mode: None, - } -} - -pub fn set_print_options_full() { - *PRINT_OPTS.lock().unwrap() = PrinterOptions { - precision: 4, - threshold: usize::MAX, - edge_items: 3, - line_width: 80, - sci_mode: None, - } -} - -struct FmtSize { - current_size: usize, -} - -impl FmtSize { - fn new() -> Self { - Self { current_size: 0 } - } - - fn final_size(self) -> usize { - self.current_size - } -} - -impl std::fmt::Write for FmtSize { - fn write_str(&mut self, s: &str) -> std::fmt::Result { - self.current_size += s.len(); - Ok(()) - } -} - -trait TensorFormatter { - type Elem: WithDType; - - fn fmt(&self, v: Self::Elem, max_w: usize, f: &mut T) -> std::fmt::Result; - - fn max_width(&self, to_display: &Tensor) -> usize { - let mut max_width = 1; - if let Ok(vs) = to_display.flatten_all().and_then(|t| t.to_vec1()) { - for &v in vs.iter() { - let mut fmt_size = FmtSize::new(); - let _res = self.fmt(v, 1, &mut fmt_size); - max_width = usize::max(max_width, fmt_size.final_size()) - } - } - max_width - } - - fn write_newline_indent(i: usize, f: &mut std::fmt::Formatter) -> std::fmt::Result { - writeln!(f)?; - for _ in 0..i { - write!(f, " ")? - } - Ok(()) - } - - fn fmt_tensor( - &self, - t: &Tensor, - indent: usize, - max_w: usize, - summarize: bool, - po: &PrinterOptions, - f: &mut std::fmt::Formatter, - ) -> std::fmt::Result { - let dims = t.dims(); - let edge_items = po.edge_items; - write!(f, "[")?; - match dims { - [] => { - if let Ok(v) = t.to_scalar::() { - self.fmt(v, max_w, f)? - } - } - [v] if summarize && *v > 2 * edge_items => { - if let Ok(vs) = t - .narrow(0, 0, edge_items) - .and_then(|t| t.to_vec1::()) - { - for v in vs.into_iter() { - self.fmt(v, max_w, f)?; - write!(f, ", ")?; - } - } - write!(f, "...")?; - if let Ok(vs) = t - .narrow(0, v - edge_items, edge_items) - .and_then(|t| t.to_vec1::()) - { - for v in vs.into_iter() { - write!(f, ", ")?; - self.fmt(v, max_w, f)?; - } - } - } - [_] => { - let elements_per_line = usize::max(1, po.line_width / (max_w + 2)); - if let Ok(vs) = t.to_vec1::() { - for (i, v) in vs.into_iter().enumerate() { - if i > 0 { - if i % elements_per_line == 0 { - write!(f, ",")?; - Self::write_newline_indent(indent, f)? - } else { - write!(f, ", ")?; - } - } - self.fmt(v, max_w, f)? - } - } - } - _ => { - if summarize && dims[0] > 2 * edge_items { - for i in 0..edge_items { - match t.get(i) { - Ok(t) => self.fmt_tensor(&t, indent + 1, max_w, summarize, po, f)?, - Err(e) => write!(f, "{e:?}")?, - } - write!(f, ",")?; - Self::write_newline_indent(indent, f)? - } - write!(f, "...")?; - Self::write_newline_indent(indent, f)?; - for i in dims[0] - edge_items..dims[0] { - match t.get(i) { - Ok(t) => self.fmt_tensor(&t, indent + 1, max_w, summarize, po, f)?, - Err(e) => write!(f, "{e:?}")?, - } - if i + 1 != dims[0] { - write!(f, ",")?; - Self::write_newline_indent(indent, f)? - } - } - } else { - for i in 0..dims[0] { - match t.get(i) { - Ok(t) => self.fmt_tensor(&t, indent + 1, max_w, summarize, po, f)?, - Err(e) => write!(f, "{e:?}")?, - } - if i + 1 != dims[0] { - write!(f, ",")?; - Self::write_newline_indent(indent, f)? - } - } - } - } - } - write!(f, "]")?; - Ok(()) - } -} - -struct FloatFormatter { - int_mode: bool, - sci_mode: bool, - precision: usize, - _phantom: std::marker::PhantomData, -} - -impl FloatFormatter -where - S: WithDType + num_traits::Float + std::fmt::Display, -{ - fn new(t: &Tensor, po: &PrinterOptions) -> Result { - let mut int_mode = true; - let mut sci_mode = false; - - // Rather than containing all values, this should only include - // values that end up being displayed according to [threshold]. - let values = t - .flatten_all()? - .to_vec1()? - .into_iter() - .filter(|v: &S| v.is_finite() && !v.is_zero()) - .collect::>(); - if !values.is_empty() { - let mut nonzero_finite_min = S::max_value(); - let mut nonzero_finite_max = S::min_value(); - for &v in values.iter() { - let v = v.abs(); - if v < nonzero_finite_min { - nonzero_finite_min = v - } - if v > nonzero_finite_max { - nonzero_finite_max = v - } - } - - for &value in values.iter() { - if value.ceil() != value { - int_mode = false; - break; - } - } - if let Some(v1) = S::from(1000.) { - if let Some(v2) = S::from(1e8) { - if let Some(v3) = S::from(1e-4) { - sci_mode = nonzero_finite_max / nonzero_finite_min > v1 - || nonzero_finite_max > v2 - || nonzero_finite_min < v3 - } - } - } - } - - match po.sci_mode { - None => {} - Some(v) => sci_mode = v, - } - Ok(Self { - int_mode, - sci_mode, - precision: po.precision, - _phantom: std::marker::PhantomData, - }) - } -} - -impl TensorFormatter for FloatFormatter -where - S: WithDType + num_traits::Float + std::fmt::Display + std::fmt::LowerExp, -{ - type Elem = S; - - fn fmt(&self, v: Self::Elem, max_w: usize, f: &mut T) -> std::fmt::Result { - if self.sci_mode { - write!( - f, - "{v:width$.prec$e}", - v = v, - width = max_w, - prec = self.precision - ) - } else if self.int_mode { - if v.is_finite() { - write!(f, "{v:width$.0}.", v = v, width = max_w - 1) - } else { - write!(f, "{v:max_w$.0}") - } - } else { - write!( - f, - "{v:width$.prec$}", - v = v, - width = max_w, - prec = self.precision - ) - } - } -} - -struct IntFormatter { - _phantom: std::marker::PhantomData, -} - -impl IntFormatter { - fn new() -> Self { - Self { - _phantom: std::marker::PhantomData, - } - } -} - -impl TensorFormatter for IntFormatter -where - S: WithDType + std::fmt::Display, -{ - type Elem = S; - - fn fmt(&self, v: Self::Elem, max_w: usize, f: &mut T) -> std::fmt::Result { - write!(f, "{v:max_w$}") - } -} - -fn get_summarized_data(t: &Tensor, edge_items: usize) -> Result { - let dims = t.dims(); - if dims.is_empty() { - Ok(t.clone()) - } else if dims.len() == 1 { - if dims[0] > 2 * edge_items { - Tensor::cat( - &[ - t.narrow(0, 0, edge_items)?, - t.narrow(0, dims[0] - edge_items, edge_items)?, - ], - 0, - ) - } else { - Ok(t.clone()) - } - } else if dims[0] > 2 * edge_items { - let mut vs: Vec<_> = (0..edge_items) - .map(|i| get_summarized_data(&t.get(i)?, edge_items)) - .collect::>>()?; - for i in (dims[0] - edge_items)..dims[0] { - vs.push(get_summarized_data(&t.get(i)?, edge_items)?) - } - Tensor::cat(&vs, 0) - } else { - let vs: Vec<_> = (0..dims[0]) - .map(|i| get_summarized_data(&t.get(i)?, edge_items)) - .collect::>>()?; - Tensor::cat(&vs, 0) - } -} - -impl std::fmt::Display for Tensor { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - let po = PRINT_OPTS.lock().unwrap(); - let summarize = self.elem_count() > po.threshold; - let to_display = if summarize { - match get_summarized_data(self, po.edge_items) { - Ok(v) => v, - Err(err) => return write!(f, "{err:?}"), - } - } else { - self.clone() - }; - match self.dtype() { - DType::U8 => { - let tf: IntFormatter = IntFormatter::new(); - let max_w = tf.max_width(&to_display); - tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; - writeln!(f)?; - } - DType::U32 => { - let tf: IntFormatter = IntFormatter::new(); - let max_w = tf.max_width(&to_display); - tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; - writeln!(f)?; - } - DType::I64 => { - let tf: IntFormatter = IntFormatter::new(); - let max_w = tf.max_width(&to_display); - tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; - writeln!(f)?; - } - DType::BF16 => { - if let Ok(tf) = FloatFormatter::::new(&to_display, &po) { - let max_w = tf.max_width(&to_display); - tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; - writeln!(f)?; - } - } - DType::F16 => { - if let Ok(tf) = FloatFormatter::::new(&to_display, &po) { - let max_w = tf.max_width(&to_display); - tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; - writeln!(f)?; - } - } - DType::F64 => { - if let Ok(tf) = FloatFormatter::::new(&to_display, &po) { - let max_w = tf.max_width(&to_display); - tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; - writeln!(f)?; - } - } - DType::F32 => { - if let Ok(tf) = FloatFormatter::::new(&to_display, &po) { - let max_w = tf.max_width(&to_display); - tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; - writeln!(f)?; - } - } - }; - - let device_str = match self.device().location() { - crate::DeviceLocation::Cpu => "".to_owned(), - crate::DeviceLocation::Cuda { gpu_id } => { - format!(", cuda:{}", gpu_id) - } - crate::DeviceLocation::Metal { gpu_id } => { - format!(", metal:{}", gpu_id) - } - }; - - write!( - f, - "Tensor[{:?}, {}{}]", - self.dims(), - self.dtype().as_str(), - device_str - ) - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/dtype.rs b/v2/dependency-patches/candle-core-0.4.1/src/dtype.rs deleted file mode 100644 index 94ca57d8fd..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/dtype.rs +++ /dev/null @@ -1,201 +0,0 @@ -//! Types for elements that can be stored and manipulated using tensors. -#![allow(clippy::redundant_closure_call)] -use crate::backend::BackendStorage; -use crate::{CpuStorage, Error, Result}; - -/// The different types of elements allowed in tensors. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub enum DType { - // Unsigned 8 bits integer. - U8, - // Unsigned 32 bits integer. - U32, - // Signed 64 bits integer. - I64, - // Brain floating-point using half precision (16 bits). - BF16, - // Floating-point using half precision (16 bits). - F16, - // Floating-point using single precision (32 bits). - F32, - // Floating-point using double precision (64 bits). - F64, -} - -#[derive(Debug, PartialEq, Eq)] -pub struct DTypeParseError; - -impl std::str::FromStr for DType { - type Err = DTypeParseError; - fn from_str(s: &str) -> std::result::Result { - match s { - "u8" => Ok(Self::U8), - "u32" => Ok(Self::U32), - "i64" => Ok(Self::I64), - "bf16" => Ok(Self::BF16), - "f16" => Ok(Self::F16), - "f32" => Ok(Self::F32), - "f64" => Ok(Self::F64), - _ => Err(DTypeParseError), - } - } -} - -impl DType { - /// String representation for dtypes. - pub fn as_str(&self) -> &'static str { - match self { - Self::U8 => "u8", - Self::U32 => "u32", - Self::I64 => "i64", - Self::BF16 => "bf16", - Self::F16 => "f16", - Self::F32 => "f32", - Self::F64 => "f64", - } - } - - /// The size used by each element in bytes, i.e. 1 for `U8`, 4 for `F32`. - pub fn size_in_bytes(&self) -> usize { - match self { - Self::U8 => 1, - Self::U32 => 4, - Self::I64 => 8, - Self::BF16 => 2, - Self::F16 => 2, - Self::F32 => 4, - Self::F64 => 8, - } - } - - pub fn is_int(&self) -> bool { - match self { - Self::U8 | Self::U32 | Self::I64 => true, - Self::BF16 | Self::F16 | Self::F32 | Self::F64 => false, - } - } - - pub fn is_float(&self) -> bool { - match self { - Self::U8 | Self::U32 | Self::I64 => false, - Self::BF16 | Self::F16 | Self::F32 | Self::F64 => true, - } - } -} - -pub trait WithDType: - Sized - + Copy - + num_traits::NumAssign - + std::cmp::PartialOrd - + std::fmt::Display - + 'static - + Send - + Sync - + crate::cpu::kernels::VecOps -{ - const DTYPE: DType; - - fn from_f64(v: f64) -> Self; - fn to_f64(self) -> f64; - fn to_cpu_storage_owned(data: Vec) -> CpuStorage; - - fn to_cpu_storage(data: &[Self]) -> CpuStorage { - Self::to_cpu_storage_owned(data.to_vec()) - } - - fn cpu_storage_as_slice(s: &CpuStorage) -> Result<&[Self]>; - fn cpu_storage_data(s: CpuStorage) -> Result>; -} - -macro_rules! with_dtype { - ($ty:ty, $dtype:ident, $from_f64:expr, $to_f64:expr) => { - impl WithDType for $ty { - const DTYPE: DType = DType::$dtype; - - fn from_f64(v: f64) -> Self { - $from_f64(v) - } - - fn to_f64(self) -> f64 { - $to_f64(self) - } - - fn to_cpu_storage_owned(data: Vec) -> CpuStorage { - CpuStorage::$dtype(data) - } - - fn cpu_storage_data(s: CpuStorage) -> Result> { - match s { - CpuStorage::$dtype(data) => Ok(data), - _ => Err(Error::UnexpectedDType { - expected: DType::$dtype, - got: s.dtype(), - msg: "unexpected dtype", - } - .bt()), - } - } - - fn cpu_storage_as_slice(s: &CpuStorage) -> Result<&[Self]> { - match s { - CpuStorage::$dtype(data) => Ok(data), - _ => Err(Error::UnexpectedDType { - expected: DType::$dtype, - got: s.dtype(), - msg: "unexpected dtype", - } - .bt()), - } - } - } - }; -} -use half::{bf16, f16}; - -with_dtype!(u8, U8, |v: f64| v as u8, |v: u8| v as f64); -with_dtype!(u32, U32, |v: f64| v as u32, |v: u32| v as f64); -with_dtype!(i64, I64, |v: f64| v as i64, |v: i64| v as f64); -with_dtype!(f16, F16, f16::from_f64, f16::to_f64); -with_dtype!(bf16, BF16, bf16::from_f64, bf16::to_f64); -with_dtype!(f32, F32, |v: f64| v as f32, |v: f32| v as f64); -with_dtype!(f64, F64, |v: f64| v, |v: f64| v); - -pub trait IntDType: WithDType { - fn is_true(&self) -> bool; - fn as_usize(&self) -> usize; -} - -impl IntDType for i64 { - fn is_true(&self) -> bool { - *self != 0 - } - fn as_usize(&self) -> usize { - *self as usize - } -} - -impl IntDType for u32 { - fn is_true(&self) -> bool { - *self != 0 - } - fn as_usize(&self) -> usize { - *self as usize - } -} - -impl IntDType for u8 { - fn is_true(&self) -> bool { - *self != 0 - } - fn as_usize(&self) -> usize { - *self as usize - } -} - -pub trait FloatDType: WithDType {} - -impl FloatDType for f16 {} -impl FloatDType for bf16 {} -impl FloatDType for f32 {} -impl FloatDType for f64 {} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/dummy_cuda_backend.rs b/v2/dependency-patches/candle-core-0.4.1/src/dummy_cuda_backend.rs deleted file mode 100644 index 34c5d97f8c..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/dummy_cuda_backend.rs +++ /dev/null @@ -1,211 +0,0 @@ -#![allow(dead_code)] -use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; -use crate::{CpuStorage, DType, Error, Layout, Result, Shape}; - -#[derive(Debug, Clone)] -pub struct CudaDevice; - -#[derive(Debug)] -pub struct CudaStorage; - -macro_rules! fail { - () => { - unimplemented!("cuda support has not been enabled, add `cuda` feature to enable.") - }; -} - -impl crate::backend::BackendStorage for CudaStorage { - type Device = CudaDevice; - - fn try_clone(&self, _: &Layout) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn dtype(&self) -> DType { - fail!() - } - - fn device(&self) -> &Self::Device { - fail!() - } - - fn to_cpu_storage(&self) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn affine(&self, _: &Layout, _: f64, _: f64) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn powf(&self, _: &Layout, _: f64) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn elu(&self, _: &Layout, _: f64) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn reduce_op(&self, _: ReduceOp, _: &Layout, _: &[usize]) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn cmp(&self, _: CmpOp, _: &Self, _: &Layout, _: &Layout) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn to_dtype(&self, _: &Layout, _: DType) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn unary_impl(&self, _: &Layout) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn binary_impl(&self, _: &Self, _: &Layout, _: &Layout) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn where_cond(&self, _: &Layout, _: &Self, _: &Layout, _: &Self, _: &Layout) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn conv1d( - &self, - _: &Layout, - _: &Self, - _: &Layout, - _: &crate::conv::ParamsConv1D, - ) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn conv_transpose1d( - &self, - _: &Layout, - _: &Self, - _: &Layout, - _: &crate::conv::ParamsConvTranspose1D, - ) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn conv2d( - &self, - _: &Layout, - _: &Self, - _: &Layout, - _: &crate::conv::ParamsConv2D, - ) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn conv_transpose2d( - &self, - _l: &Layout, - _kernel: &Self, - _kernel_l: &Layout, - _params: &crate::conv::ParamsConvTranspose2D, - ) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn index_select(&self, _: &Self, _: &Layout, _: &Layout, _: usize) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - fn gather(&self, _: &Layout, _: &Self, _: &Layout, _: usize) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn scatter_add( - &self, - _: &Layout, - _: &Self, - _: &Layout, - _: &Self, - _: &Layout, - _: usize, - ) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn index_add( - &self, - _: &Layout, - _: &Self, - _: &Layout, - _: &Self, - _: &Layout, - _: usize, - ) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn matmul( - &self, - _: &Self, - _: (usize, usize, usize, usize), - _: &Layout, - _: &Layout, - ) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn copy_strided_src(&self, _: &mut Self, _: usize, _: &Layout) -> Result<()> { - Err(Error::NotCompiledWithCudaSupport) - } - - fn avg_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn max_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn upsample_nearest1d(&self, _: &Layout, _: usize) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn upsample_nearest2d(&self, _: &Layout, _: usize, _: usize) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } -} - -impl crate::backend::BackendDevice for CudaDevice { - type Storage = CudaStorage; - fn new(_: usize) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn set_seed(&self, _: u64) -> Result<()> { - Err(Error::NotCompiledWithCudaSupport) - } - - fn location(&self) -> crate::DeviceLocation { - fail!() - } - - fn same_device(&self, _: &Self) -> bool { - fail!() - } - - fn zeros_impl(&self, _shape: &Shape, _dtype: DType) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn ones_impl(&self, _shape: &Shape, _dtype: DType) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn storage_from_cpu_storage(&self, _: &CpuStorage) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn rand_uniform(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn rand_normal(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/dummy_metal_backend.rs b/v2/dependency-patches/candle-core-0.4.1/src/dummy_metal_backend.rs deleted file mode 100644 index e9d923310b..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/dummy_metal_backend.rs +++ /dev/null @@ -1,223 +0,0 @@ -#![allow(dead_code)] -use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; -use crate::{CpuStorage, DType, Error, Layout, Result, Shape}; - -#[derive(Debug, Clone)] -pub struct MetalDevice; - -#[derive(Debug)] -pub struct MetalStorage; - -#[derive(thiserror::Error, Debug)] -pub enum MetalError { - #[error("{0}")] - Message(String), -} - -impl From for MetalError { - fn from(e: String) -> Self { - MetalError::Message(e) - } -} - -macro_rules! fail { - () => { - unimplemented!("metal support has not been enabled, add `metal` feature to enable.") - }; -} - -impl crate::backend::BackendStorage for MetalStorage { - type Device = MetalDevice; - - fn try_clone(&self, _: &Layout) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn dtype(&self) -> DType { - fail!() - } - - fn device(&self) -> &Self::Device { - fail!() - } - - fn to_cpu_storage(&self) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn affine(&self, _: &Layout, _: f64, _: f64) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn powf(&self, _: &Layout, _: f64) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn elu(&self, _: &Layout, _: f64) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn reduce_op(&self, _: ReduceOp, _: &Layout, _: &[usize]) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn cmp(&self, _: CmpOp, _: &Self, _: &Layout, _: &Layout) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn to_dtype(&self, _: &Layout, _: DType) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn unary_impl(&self, _: &Layout) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn binary_impl(&self, _: &Self, _: &Layout, _: &Layout) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn where_cond(&self, _: &Layout, _: &Self, _: &Layout, _: &Self, _: &Layout) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn conv1d( - &self, - _: &Layout, - _: &Self, - _: &Layout, - _: &crate::conv::ParamsConv1D, - ) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn conv_transpose1d( - &self, - _l: &Layout, - _kernel: &Self, - _kernel_l: &Layout, - _params: &crate::conv::ParamsConvTranspose1D, - ) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn conv2d( - &self, - _: &Layout, - _: &Self, - _: &Layout, - _: &crate::conv::ParamsConv2D, - ) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn conv_transpose2d( - &self, - _l: &Layout, - _kernel: &Self, - _kernel_l: &Layout, - _params: &crate::conv::ParamsConvTranspose2D, - ) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn index_select(&self, _: &Self, _: &Layout, _: &Layout, _: usize) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - fn gather(&self, _: &Layout, _: &Self, _: &Layout, _: usize) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn scatter_add( - &self, - _: &Layout, - _: &Self, - _: &Layout, - _: &Self, - _: &Layout, - _: usize, - ) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn index_add( - &self, - _: &Layout, - _: &Self, - _: &Layout, - _: &Self, - _: &Layout, - _: usize, - ) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn matmul( - &self, - _: &Self, - _: (usize, usize, usize, usize), - _: &Layout, - _: &Layout, - ) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn copy_strided_src(&self, _: &mut Self, _: usize, _: &Layout) -> Result<()> { - Err(Error::NotCompiledWithMetalSupport) - } - - fn avg_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn max_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn upsample_nearest1d(&self, _: &Layout, _: usize) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn upsample_nearest2d(&self, _: &Layout, _: usize, _: usize) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } -} - -impl crate::backend::BackendDevice for MetalDevice { - type Storage = MetalStorage; - fn new(_: usize) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn set_seed(&self, _: u64) -> Result<()> { - Err(Error::NotCompiledWithMetalSupport) - } - - fn location(&self) -> crate::DeviceLocation { - fail!() - } - - fn same_device(&self, _: &Self) -> bool { - fail!() - } - - fn zeros_impl(&self, _shape: &Shape, _dtype: DType) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn ones_impl(&self, _shape: &Shape, _dtype: DType) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn storage_from_cpu_storage(&self, _: &CpuStorage) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn rand_uniform(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn rand_normal(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/error.rs b/v2/dependency-patches/candle-core-0.4.1/src/error.rs deleted file mode 100644 index 60ddea1149..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/error.rs +++ /dev/null @@ -1,265 +0,0 @@ -use crate::{DType, DeviceLocation, Layout, MetalError, Shape}; - -#[derive(Debug, Clone)] -pub struct MatMulUnexpectedStriding { - pub lhs_l: Layout, - pub rhs_l: Layout, - pub bmnk: (usize, usize, usize, usize), - pub msg: &'static str, -} - -/// Main library error type. -#[derive(thiserror::Error, Debug)] -pub enum Error { - // === DType Errors === - #[error("{msg}, expected: {expected:?}, got: {got:?}")] - UnexpectedDType { - msg: &'static str, - expected: DType, - got: DType, - }, - - #[error("dtype mismatch in {op}, lhs: {lhs:?}, rhs: {rhs:?}")] - DTypeMismatchBinaryOp { - lhs: DType, - rhs: DType, - op: &'static str, - }, - - #[error("unsupported dtype {0:?} for op {1}")] - UnsupportedDTypeForOp(DType, &'static str), - - // === Dimension Index Errors === - #[error("{op}: dimension index {dim} out of range for shape {shape:?}")] - DimOutOfRange { - shape: Shape, - dim: i32, - op: &'static str, - }, - - #[error("{op}: duplicate dim index {dims:?} for shape {shape:?}")] - DuplicateDimIndex { - shape: Shape, - dims: Vec, - op: &'static str, - }, - - // === Shape Errors === - #[error("unexpected rank, expected: {expected}, got: {got} ({shape:?})")] - UnexpectedNumberOfDims { - expected: usize, - got: usize, - shape: Shape, - }, - - #[error("{msg}, expected: {expected:?}, got: {got:?}")] - UnexpectedShape { - msg: String, - expected: Shape, - got: Shape, - }, - - #[error( - "Shape mismatch, got buffer of size {buffer_size} which is compatible with shape {shape:?}" - )] - ShapeMismatch { buffer_size: usize, shape: Shape }, - - #[error("shape mismatch in {op}, lhs: {lhs:?}, rhs: {rhs:?}")] - ShapeMismatchBinaryOp { - lhs: Shape, - rhs: Shape, - op: &'static str, - }, - - #[error("shape mismatch in cat for dim {dim}, shape for arg 1: {first_shape:?} shape for arg {n}: {nth_shape:?}")] - ShapeMismatchCat { - dim: usize, - first_shape: Shape, - n: usize, - nth_shape: Shape, - }, - - #[error("Cannot divide tensor of shape {shape:?} equally along dim {dim} into {n_parts}")] - ShapeMismatchSplit { - shape: Shape, - dim: usize, - n_parts: usize, - }, - - #[error("{op} can only be performed on a single dimension")] - OnlySingleDimension { op: &'static str, dims: Vec }, - - #[error("empty tensor for {op}")] - EmptyTensor { op: &'static str }, - - // === Device Errors === - #[error("device mismatch in {op}, lhs: {lhs:?}, rhs: {rhs:?}")] - DeviceMismatchBinaryOp { - lhs: DeviceLocation, - rhs: DeviceLocation, - op: &'static str, - }, - - // === Op Specific Errors === - #[error("narrow invalid args {msg}: {shape:?}, dim: {dim}, start: {start}, len:{len}")] - NarrowInvalidArgs { - shape: Shape, - dim: usize, - start: usize, - len: usize, - msg: &'static str, - }, - - #[error("conv1d invalid args {msg}: inp: {inp_shape:?}, k: {k_shape:?}, pad: {padding}, stride: {stride}")] - Conv1dInvalidArgs { - inp_shape: Shape, - k_shape: Shape, - padding: usize, - stride: usize, - msg: &'static str, - }, - - #[error("{op} invalid index {index} with dim size {size}")] - InvalidIndex { - op: &'static str, - index: usize, - size: usize, - }, - - #[error("cannot broadcast {src_shape:?} to {dst_shape:?}")] - BroadcastIncompatibleShapes { src_shape: Shape, dst_shape: Shape }, - - #[error("cannot set variable {msg}")] - CannotSetVar { msg: &'static str }, - - // Box indirection to avoid large variant. - #[error("{0:?}")] - MatMulUnexpectedStriding(Box), - - #[error("{op} only supports contiguous tensors")] - RequiresContiguous { op: &'static str }, - - #[error("{op} expects at least one tensor")] - OpRequiresAtLeastOneTensor { op: &'static str }, - - #[error("{op} expects at least two tensors")] - OpRequiresAtLeastTwoTensors { op: &'static str }, - - #[error("backward is not supported for {op}")] - BackwardNotSupported { op: &'static str }, - - // === Other Errors === - #[error("the candle crate has not been built with cuda support")] - NotCompiledWithCudaSupport, - - #[error("the candle crate has not been built with metal support")] - NotCompiledWithMetalSupport, - - #[error("cannot find tensor {path}")] - CannotFindTensor { path: String }, - - // === Wrapped Errors === - #[error(transparent)] - Cuda(Box), - - #[error("Metal error {0}")] - Metal(#[from] MetalError), - - #[error(transparent)] - TryFromIntError(#[from] core::num::TryFromIntError), - - #[error("npy/npz error {0}")] - Npy(String), - - /// Zip file format error. - #[error(transparent)] - Zip(#[from] zip::result::ZipError), - - /// Integer parse error. - #[error(transparent)] - ParseInt(#[from] std::num::ParseIntError), - - /// I/O error. - #[error(transparent)] - Io(#[from] std::io::Error), - - /// SafeTensor error. - #[error(transparent)] - SafeTensor(#[from] safetensors::SafeTensorError), - - #[error("unsupported safetensor dtype {0:?}")] - UnsupportedSafeTensorDtype(safetensors::Dtype), - - /// Arbitrary errors wrapping. - #[error(transparent)] - Wrapped(Box), - - /// Adding path information to an error. - #[error("path: {path:?} {inner}")] - WithPath { - inner: Box, - path: std::path::PathBuf, - }, - - #[error("{inner}\n{backtrace}")] - WithBacktrace { - inner: Box, - backtrace: Box, - }, - - /// User generated error message, typically created via `bail!`. - #[error("{0}")] - Msg(String), -} - -pub type Result = std::result::Result; - -impl Error { - pub fn wrap(err: impl std::error::Error + Send + Sync + 'static) -> Self { - Self::Wrapped(Box::new(err)).bt() - } - - pub fn msg(err: impl std::error::Error + Send + Sync + 'static) -> Self { - Self::Msg(err.to_string()).bt() - } - - pub fn bt(self) -> Self { - let backtrace = std::backtrace::Backtrace::capture(); - match backtrace.status() { - std::backtrace::BacktraceStatus::Disabled - | std::backtrace::BacktraceStatus::Unsupported => self, - _ => Self::WithBacktrace { - inner: Box::new(self), - backtrace: Box::new(backtrace), - }, - } - } - - pub fn with_path>(self, p: P) -> Self { - Self::WithPath { - inner: Box::new(self), - path: p.as_ref().to_path_buf(), - } - } -} - -#[macro_export] -macro_rules! bail { - ($msg:literal $(,)?) => { - return Err($crate::Error::Msg(format!($msg).into()).bt()) - }; - ($err:expr $(,)?) => { - return Err($crate::Error::Msg(format!($err).into()).bt()) - }; - ($fmt:expr, $($arg:tt)*) => { - return Err($crate::Error::Msg(format!($fmt, $($arg)*).into()).bt()) - }; -} - -pub fn zip(r1: Result, r2: Result) -> Result<(T, U)> { - match (r1, r2) { - (Ok(r1), Ok(r2)) => Ok((r1, r2)), - (Err(e), _) => Err(e), - (_, Err(e)) => Err(e), - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/indexer.rs b/v2/dependency-patches/candle-core-0.4.1/src/indexer.rs deleted file mode 100644 index e3ed41e52b..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/indexer.rs +++ /dev/null @@ -1,168 +0,0 @@ -use crate::{Error, Tensor}; -use std::ops::{ - Bound, Range, RangeBounds, RangeFrom, RangeFull, RangeInclusive, RangeTo, RangeToInclusive, -}; - -impl Tensor { - /// Intended to be use by the trait `.i()` - /// - /// ``` - /// # use candle_core::{Tensor, DType, Device, IndexOp}; - /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; - /// - /// let c = a.i(0..1)?; - /// assert_eq!(c.shape().dims(), &[1, 3]); - /// - /// let c = a.i(0)?; - /// assert_eq!(c.shape().dims(), &[3]); - /// - /// let c = a.i((.., ..2) )?; - /// assert_eq!(c.shape().dims(), &[2, 2]); - /// - /// let c = a.i((.., ..=2))?; - /// assert_eq!(c.shape().dims(), &[2, 3]); - /// - /// # Ok::<(), candle_core::Error>(()) - /// ``` - fn index(&self, indexers: &[TensorIndexer]) -> Result { - let mut x = self.clone(); - let dims = self.shape().dims(); - let mut current_dim = 0; - for (i, indexer) in indexers.iter().enumerate() { - x = match indexer { - TensorIndexer::Select(n) => x.narrow(current_dim, *n, 1)?.squeeze(current_dim)?, - TensorIndexer::Narrow(left_bound, right_bound) => { - let start = match left_bound { - Bound::Included(n) => *n, - Bound::Excluded(n) => *n + 1, - Bound::Unbounded => 0, - }; - let stop = match right_bound { - Bound::Included(n) => *n + 1, - Bound::Excluded(n) => *n, - Bound::Unbounded => dims[i], - }; - let out = x.narrow(current_dim, start, stop.saturating_sub(start))?; - current_dim += 1; - out - } - TensorIndexer::IndexSelect(indexes) => { - if indexes.rank() != 1 { - crate::bail!("multi-dimensional tensor indexing is not supported") - } - let out = x.index_select(&indexes.to_device(x.device())?, current_dim)?; - current_dim += 1; - out - } - TensorIndexer::Err(e) => crate::bail!("indexing error {e:?}"), - }; - } - Ok(x) - } -} - -#[derive(Debug)] -/// Generic structure used to index a slice of the tensor -pub enum TensorIndexer { - /// This selects the elements for which an index has some specific value. - Select(usize), - /// This is a regular slice, purely indexing a chunk of the tensor - Narrow(Bound, Bound), - /// Indexing via a 1d tensor - IndexSelect(Tensor), - Err(Error), -} - -impl From for TensorIndexer { - fn from(index: usize) -> Self { - TensorIndexer::Select(index) - } -} - -impl From<&[u32]> for TensorIndexer { - fn from(index: &[u32]) -> Self { - match Tensor::new(index, &crate::Device::Cpu) { - Ok(tensor) => TensorIndexer::IndexSelect(tensor), - Err(e) => TensorIndexer::Err(e), - } - } -} - -impl From> for TensorIndexer { - fn from(index: Vec) -> Self { - let len = index.len(); - match Tensor::from_vec(index, len, &crate::Device::Cpu) { - Ok(tensor) => TensorIndexer::IndexSelect(tensor), - Err(e) => TensorIndexer::Err(e), - } - } -} - -impl From<&Tensor> for TensorIndexer { - fn from(tensor: &Tensor) -> Self { - TensorIndexer::IndexSelect(tensor.clone()) - } -} - -trait RB: RangeBounds {} -impl RB for Range {} -impl RB for RangeFrom {} -impl RB for RangeFull {} -impl RB for RangeInclusive {} -impl RB for RangeTo {} -impl RB for RangeToInclusive {} - -impl From for TensorIndexer { - fn from(range: T) -> Self { - use std::ops::Bound::*; - let start = match range.start_bound() { - Included(idx) => Included(*idx), - Excluded(idx) => Excluded(*idx), - Unbounded => Unbounded, - }; - let end = match range.end_bound() { - Included(idx) => Included(*idx), - Excluded(idx) => Excluded(*idx), - Unbounded => Unbounded, - }; - TensorIndexer::Narrow(start, end) - } -} - -/// Trait used to implement multiple signatures for ease of use of the slicing -/// of a tensor -pub trait IndexOp { - /// Returns a slicing iterator which are the chunks of data necessary to - /// reconstruct the desired tensor. - fn i(&self, index: T) -> Result; -} - -impl IndexOp for Tensor -where - T: Into, -{ - fn i(&self, index: T) -> Result { - self.index(&[index.into()]) - } -} - -macro_rules! index_op_tuple { - ($($t:ident),+) => { - #[allow(non_snake_case)] - impl<$($t),*> IndexOp<($($t,)*)> for Tensor - where - $($t: Into,)* - { - fn i(&self, ($($t,)*): ($($t,)*)) -> Result { - self.index(&[$($t.into(),)*]) - } - } - }; -} -index_op_tuple!(A); -index_op_tuple!(A, B); -index_op_tuple!(A, B, C); -index_op_tuple!(A, B, C, D); -index_op_tuple!(A, B, C, D, E); -index_op_tuple!(A, B, C, D, E, F); -index_op_tuple!(A, B, C, D, E, F, G); diff --git a/v2/dependency-patches/candle-core-0.4.1/src/layout.rs b/v2/dependency-patches/candle-core-0.4.1/src/layout.rs deleted file mode 100644 index bf346cf270..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/layout.rs +++ /dev/null @@ -1,271 +0,0 @@ -use crate::{Error, Result, Shape}; - -#[derive(Debug, PartialEq, Eq, Clone)] -pub struct Layout { - shape: Shape, - // The strides are given in number of elements and not in bytes. - stride: Vec, - start_offset: usize, -} - -impl Layout { - pub fn new(shape: Shape, stride: Vec, start_offset: usize) -> Self { - Self { - shape, - stride, - start_offset, - } - } - - pub fn contiguous_with_offset>(shape: S, start_offset: usize) -> Self { - let shape = shape.into(); - let stride = shape.stride_contiguous(); - Self { - shape, - stride, - start_offset, - } - } - - pub fn contiguous>(shape: S) -> Self { - Self::contiguous_with_offset(shape, 0) - } - - pub fn dims(&self) -> &[usize] { - self.shape.dims() - } - - pub fn shape(&self) -> &Shape { - &self.shape - } - - pub fn stride(&self) -> &[usize] { - &self.stride - } - - pub fn start_offset(&self) -> usize { - self.start_offset - } - - /// Returns the appropriate start and stop offset if the data is stored in a C - /// contiguous (aka row major) way. - pub fn contiguous_offsets(&self) -> Option<(usize, usize)> { - if self.is_contiguous() { - let start_o = self.start_offset; - Some((start_o, start_o + self.shape.elem_count())) - } else { - None - } - } - - /// Returns true if the data is stored in a C contiguous (aka row major) way. - /// Note that this does not implies that the start offset is 0 or that there are no extra - /// elements at the end of the storage. - pub fn is_contiguous(&self) -> bool { - self.shape.is_contiguous(&self.stride) - } - - /// Returns true if the data is stored in a Fortran contiguous (aka column major) way. - pub fn is_fortran_contiguous(&self) -> bool { - self.shape.is_fortran_contiguous(&self.stride) - } - - pub(crate) fn narrow(&self, dim: usize, start: usize, len: usize) -> Result { - let dims = self.shape().dims(); - if dim >= dims.len() { - Err(Error::DimOutOfRange { - shape: self.shape().clone(), - dim: dim as i32, - op: "narrow", - } - .bt())? - } - if start + len > dims[dim] { - Err(Error::NarrowInvalidArgs { - shape: self.shape.clone(), - dim, - start, - len, - msg: "start + len > dim_len", - } - .bt())? - } - let mut dims = dims.to_vec(); - dims[dim] = len; - Ok(Self { - shape: Shape::from(dims), - stride: self.stride.clone(), - start_offset: self.start_offset + self.stride[dim] * start, - }) - } - - pub(crate) fn transpose(&self, dim1: usize, dim2: usize) -> Result { - let rank = self.shape.rank(); - if rank <= dim1 || rank <= dim2 { - Err(Error::UnexpectedNumberOfDims { - expected: usize::max(dim1, dim2), - got: rank, - shape: self.shape().clone(), - } - .bt())? - } - let mut stride = self.stride().to_vec(); - let mut dims = self.shape().dims().to_vec(); - dims.swap(dim1, dim2); - stride.swap(dim1, dim2); - Ok(Self { - shape: Shape::from(dims), - stride, - start_offset: self.start_offset, - }) - } - - pub(crate) fn permute(&self, idxs: &[usize]) -> Result { - let is_permutation = - idxs.len() == self.shape.rank() && (0..idxs.len()).all(|i| idxs.contains(&i)); - if !is_permutation { - crate::bail!( - "dimension mismatch in permute, tensor {:?}, dims: {:?}", - self.dims(), - idxs - ) - } - let stride = self.stride(); - let dims = self.shape().dims(); - let mut perm_stride = stride.to_vec(); - let mut perm_dims = dims.to_vec(); - for (i, &idx) in idxs.iter().enumerate() { - perm_stride[i] = stride[idx]; - perm_dims[i] = dims[idx]; - } - Ok(Self { - shape: Shape::from(perm_dims), - stride: perm_stride, - start_offset: self.start_offset, - }) - } - - pub fn broadcast_as>(&self, shape: S) -> Result { - let shape = shape.into(); - if shape.rank() < self.shape().rank() { - return Err(Error::BroadcastIncompatibleShapes { - src_shape: self.shape().clone(), - dst_shape: shape, - } - .bt()); - } - let added_dims = shape.rank() - self.shape().rank(); - let mut stride = vec![0; added_dims]; - for (&dst_dim, (&src_dim, &src_stride)) in shape.dims()[added_dims..] - .iter() - .zip(self.dims().iter().zip(self.stride())) - { - let s = if dst_dim == src_dim { - src_stride - } else if src_dim != 1 { - return Err(Error::BroadcastIncompatibleShapes { - src_shape: self.shape().clone(), - dst_shape: shape, - } - .bt()); - } else { - 0 - }; - stride.push(s) - } - Ok(Self { - shape, - stride, - start_offset: self.start_offset, - }) - } - - pub(crate) fn strided_index(&self) -> crate::StridedIndex { - crate::StridedIndex::from_layout(self) - } - - pub(crate) fn strided_blocks(&self) -> crate::StridedBlocks { - let mut block_len = 1; - let mut contiguous_dims = 0; // These are counted from the right. - for (&stride, &dim) in self.stride().iter().zip(self.dims().iter()).rev() { - if stride != block_len { - break; - } - block_len *= dim; - contiguous_dims += 1; - } - let index_dims = self.dims().len() - contiguous_dims; - if index_dims == 0 { - crate::StridedBlocks::SingleBlock { - start_offset: self.start_offset, - len: block_len, - } - } else { - let block_start_index = crate::StridedIndex::new( - &self.dims()[..index_dims], - &self.stride[..index_dims], - self.start_offset, - ); - crate::StridedBlocks::MultipleBlocks { - block_start_index, - block_len, - } - } - } - - // Returns the contiguous offsets with broadcast if applicable. - pub(crate) fn offsets_b(&self) -> Option { - let mut left_broadcast = 1; - let mut right_broadcast = 1; - let strides = self.stride(); - let dims = self.dims(); - let mut start_cont = 0; - let mut end_cont = dims.len(); - for (&s, &d) in strides.iter().zip(dims.iter()) { - if s != 0 { - break; - } - start_cont += 1; - left_broadcast *= d; - } - if start_cont == dims.len() { - return Some(ContiguousOffsetsWithBroadcast { - start: self.start_offset, - len: 1, - left_broadcast, - right_broadcast: 1, - }); - } - for (&s, &d) in strides.iter().zip(dims.iter()).rev() { - if s != 0 { - break; - } - end_cont -= 1; - right_broadcast *= d; - } - // Check that the inner dims are contiguous - let strides = &strides[start_cont..end_cont]; - let dims = &dims[start_cont..end_cont]; - let mut len = 1; - for (&stride, &dim) in strides.iter().zip(dims.iter()).rev() { - if stride != len { - return None; - } - len *= dim; - } - Some(ContiguousOffsetsWithBroadcast { - start: self.start_offset, - len, - left_broadcast, - right_broadcast, - }) - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ContiguousOffsetsWithBroadcast { - pub start: usize, - pub len: usize, - pub left_broadcast: usize, - pub right_broadcast: usize, -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/lib.rs b/v2/dependency-patches/candle-core-0.4.1/src/lib.rs deleted file mode 100644 index fcc17afcd7..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/lib.rs +++ /dev/null @@ -1,151 +0,0 @@ -//! ML framework for Rust -//! -//! ```rust -//! use candle_core::{Tensor, DType, Device}; -//! # use candle_core::Error; -//! # fn main() -> Result<(), Error>{ -//! -//! let a = Tensor::arange(0f32, 6f32, &Device::Cpu)?.reshape((2, 3))?; -//! let b = Tensor::arange(0f32, 12f32, &Device::Cpu)?.reshape((3, 4))?; -//! -//! let c = a.matmul(&b)?; -//! # Ok(())} -//! ``` -//! -//! ## Features -//! -//! - Simple syntax (looks and like PyTorch) -//! - CPU and Cuda backends (and M1 support) -//! - Enable serverless (CPU) small and fast deployments -//! - Model training -//! - Distributed computing (NCCL). -//! - Models out of the box (Llama, Whisper, Falcon, ...) -//! -//! ## FAQ -//! -//! - Why Candle? -//! -//! Candle stems from the need to reduce binary size in order to *enable serverless* -//! possible by making the whole engine smaller than PyTorch very large library volume -//! -//! And simply *removing Python* from production workloads. -//! Python can really add overhead in more complex workflows and the [GIL](https://www.backblaze.com/blog/the-python-gil-past-present-and-future/) is a notorious source of headaches. -//! -//! Rust is cool, and a lot of the HF ecosystem already has Rust crates [safetensors](https://github.com/huggingface/safetensors) and [tokenizers](https://github.com/huggingface/tokenizers) - -#[cfg(feature = "accelerate")] -mod accelerate; -pub mod backend; -pub mod backprop; -mod conv; -mod convert; -pub mod cpu; -pub mod cpu_backend; -#[cfg(feature = "cuda")] -pub mod cuda_backend; -#[cfg(feature = "cudnn")] -pub mod cudnn; -mod device; -pub mod display; -mod dtype; -mod dummy_cuda_backend; -mod dummy_metal_backend; -pub mod error; -mod indexer; -pub mod layout; -#[cfg(feature = "metal")] -pub mod metal_backend; -#[cfg(feature = "mkl")] -mod mkl; -pub mod npy; -mod op; -pub mod pickle; -pub mod quantized; -pub mod safetensors; -pub mod scalar; -pub mod shape; -mod storage; -mod strided_index; -mod tensor; -pub mod test_utils; -pub mod utils; -mod variable; - -pub use cpu_backend::CpuStorage; -pub use device::{Device, DeviceLocation, NdArray}; -pub use dtype::{DType, FloatDType, IntDType, WithDType}; -pub use error::{Error, Result}; -pub use indexer::IndexOp; -pub use layout::Layout; -pub use op::{CustomOp1, CustomOp2, CustomOp3}; -pub use shape::{Shape, D}; -pub use storage::Storage; -pub use strided_index::{StridedBlocks, StridedIndex}; -pub use tensor::{Tensor, TensorId}; -pub use variable::Var; - -#[cfg(feature = "cuda")] -pub use cuda_backend::{CudaDevice, CudaStorage}; - -#[cfg(not(feature = "cuda"))] -pub use dummy_cuda_backend::{CudaDevice, CudaStorage}; - -#[cfg(feature = "metal")] -pub use metal_backend::{MetalDevice, MetalError, MetalStorage}; - -#[cfg(not(feature = "metal"))] -pub use dummy_metal_backend::{MetalDevice, MetalError, MetalStorage}; - -#[cfg(feature = "mkl")] -extern crate intel_mkl_src; - -#[cfg(feature = "accelerate")] -extern crate accelerate_src; - -pub trait ToUsize2 { - fn to_usize2(self) -> (usize, usize); -} - -impl ToUsize2 for usize { - fn to_usize2(self) -> (usize, usize) { - (self, self) - } -} - -impl ToUsize2 for (usize, usize) { - fn to_usize2(self) -> (usize, usize) { - self - } -} - -// A simple trait defining a module with forward method using a single argument. -pub trait Module { - fn forward(&self, xs: &Tensor) -> Result; -} - -impl Result> Module for T { - fn forward(&self, xs: &Tensor) -> Result { - self(xs) - } -} - -impl Module for Option<&M> { - fn forward(&self, xs: &Tensor) -> Result { - match self { - None => Ok(xs.clone()), - Some(m) => m.forward(xs), - } - } -} - -// A trait defining a module with forward method using a single tensor argument and a flag to -// separate the training and evaluation behaviors. -pub trait ModuleT { - fn forward_t(&self, xs: &Tensor, train: bool) -> Result; -} - -impl ModuleT for M { - fn forward_t(&self, xs: &Tensor, _train: bool) -> Result { - self.forward(xs) - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/metal_backend.rs b/v2/dependency-patches/candle-core-0.4.1/src/metal_backend.rs deleted file mode 100644 index 959f0f310c..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/metal_backend.rs +++ /dev/null @@ -1,1699 +0,0 @@ -use crate::backend::{BackendDevice, BackendStorage}; -use crate::conv::{ParamsConv1D, ParamsConv2D, ParamsConvTranspose1D, ParamsConvTranspose2D}; -use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; -use crate::{CpuStorage, DType, Layout, Result, Shape}; -use candle_metal_kernels; -use candle_metal_kernels::Kernels; -use metal; -use metal::{Buffer, CommandBuffer, CommandQueue, MTLResourceOptions, NSUInteger}; -use std::collections::HashMap; -use std::ffi::c_void; -use std::path::Path; -use std::sync::{Arc, Mutex, RwLock, TryLockError}; - -/// Simple way to catch lock error without -/// depending on T -#[derive(thiserror::Error, Debug)] -pub enum LockError { - #[error("{0}")] - Poisoned(String), - #[error("Would block")] - WouldBlock, -} - -impl From> for MetalError { - fn from(value: TryLockError) -> Self { - match value { - TryLockError::Poisoned(p) => MetalError::LockError(LockError::Poisoned(p.to_string())), - TryLockError::WouldBlock => MetalError::LockError(LockError::WouldBlock), - } - } -} - -/// Metal related errors -#[derive(thiserror::Error, Debug)] -pub enum MetalError { - #[error("{0}")] - Message(String), - #[error(transparent)] - KernelError(#[from] candle_metal_kernels::MetalKernelError), - - #[error("matmul is only supported for contiguous tensors lstride: {lhs_stride:?} rstride: {rhs_stride:?} mnk: {mnk:?}")] - MatMulNonContiguous { - lhs_stride: Vec, - rhs_stride: Vec, - mnk: (usize, usize, usize), - }, - #[error("{0:?}")] - LockError(LockError), - #[error("{msg}, expected: {expected:?}, got: {got:?}")] - UnexpectedDType { - msg: &'static str, - expected: DType, - got: DType, - }, -} - -impl From for MetalError { - fn from(e: String) -> Self { - MetalError::Message(e) - } -} - -type AllocatedBuffers = Arc>>>>; - -#[derive(Clone)] -pub struct MetalDevice { - /// Raw metal device: - device: metal::Device, - - /// Single command queue for the entire device. - command_queue: metal::CommandQueue, - /// One command buffer at a time. - /// The scheduler works by allowing multiple - /// [ComputeCommandEncoder](https://developer.apple.com/documentation/metal/mtlcomputecommandencoder?language=objc) - /// on a single command buffer. Using a single command buffer would be fastest on the GPU but - /// prevents overlapping of CPU and GPU commands (because command buffer needs to be committed - /// to start to work). - /// Despite what the documentation says, command buffers are NOT ordered. They are ordered - /// for their START time, but there's no guarantee that command buffer1 will finish before - /// command buffer2 starts (or there are metal bugs there) - command_buffer: Arc>, - /// Keeps track of the current amount of compute command encoders on the current - /// command buffer - /// Arc, RwLock because of the interior mutability. - command_buffer_index: Arc>, - /// The maximum amount of [compute command encoder](https://developer.apple.com/documentation/metal/mtlcomputecommandencoder?language=objc) per [command buffer](https://developer.apple.com/documentation/metal/mtlcommandbuffer?language=objc) - compute_per_buffer: usize, - /// Simple keeper struct to keep track of the already compiled kernels so we can reuse them. - /// Heavily used by [`candle_metal_kernels`] - kernels: Arc, - /// Simple allocator struct. - /// The buffers are stored in size buckets since ML tends to use similar shapes over and over. - /// We store the buffers in [`Arc`] because it's much faster than Obj-c internal ref counting - /// (could be linked to FFI communication overhead). - /// - /// Whenever a buffer has a strong_count==1, we can reuse it, it means it was dropped in the - /// graph calculation, and only we the allocator kept a reference to it, therefore it's free - /// to be reused. However, in order for this to work, we need to guarantee the order of - /// operation, so that this buffer is not being used by another kernel at the same time. - /// Arc is the CPU reference count, it doesn't mean anything on the GPU side of things. - /// - /// Whenever we actually allocate a new buffer, we make a full sweep to cleanup unused buffers - /// (strong_count = 1). - buffers: AllocatedBuffers, - /// Seed for random number generation. - seed: Arc>, -} - -impl std::fmt::Debug for MetalDevice { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "MetalDevice({:?})", self.device.registry_id()) - } -} - -impl std::ops::Deref for MetalDevice { - type Target = metal::DeviceRef; - - fn deref(&self) -> &Self::Target { - &self.device - } -} - -impl MetalDevice { - pub fn id(&self) -> NSUInteger { - self.registry_id() - } - - pub fn metal_device(&self) -> &metal::Device { - &self.device - } - - pub fn command_queue(&self) -> &CommandQueue { - &self.command_queue - } - - pub fn command_buffer(&self) -> Result { - let mut command_buffer_lock = self.command_buffer.try_write().map_err(MetalError::from)?; - let mut command_buffer = command_buffer_lock.to_owned(); - let mut index = self - .command_buffer_index - .try_write() - .map_err(MetalError::from)?; - if *index > self.compute_per_buffer { - command_buffer.commit(); - command_buffer = self.command_queue.new_command_buffer().to_owned(); - *command_buffer_lock = command_buffer.clone(); - *index = 0; - } - *index += 1; - Ok(command_buffer) - } - - pub fn wait_until_completed(&self) -> Result<()> { - let mut command_buffer = self.command_buffer.try_write().map_err(MetalError::from)?; - match command_buffer.status() { - metal::MTLCommandBufferStatus::Committed - | metal::MTLCommandBufferStatus::Scheduled - | metal::MTLCommandBufferStatus::Completed => { - panic!("Already committed"); - } - _ => {} - } - command_buffer.commit(); - command_buffer.wait_until_completed(); - *command_buffer = self.command_queue.new_command_buffer().to_owned(); - Ok(()) - } - - pub fn kernels(&self) -> &Kernels { - &self.kernels - } - - pub fn device(&self) -> &metal::Device { - &self.device - } - - /// Creates a new buffer (not necessarily zeroed). - /// The buffer is [MTLPrivate](https://developer.apple.com/documentation/metal/mtlstoragemode) - /// This means the buffer data cannot be read on the CPU directly. - /// - /// [`name`] is only used to keep track of the resource origin in case of bugs - pub fn new_buffer( - &self, - element_count: usize, - dtype: DType, - name: &str, - ) -> Result> { - let size = (element_count * dtype.size_in_bytes()) as NSUInteger; - self.allocate_buffer(size, MTLResourceOptions::StorageModePrivate, name) - } - - /// Creates a new buffer (not necessarily zeroed). - /// The buffer is [MTLManaged](https://developer.apple.com/documentation/metal/mtlstoragemode) - /// This means the buffer can be read on the CPU but will require manual - /// synchronization when the CPU memory is modified - /// Used as a bridge to gather data back from the GPU - pub fn new_buffer_managed(&self, size: NSUInteger) -> Result> { - self.allocate_buffer(size, MTLResourceOptions::StorageModeManaged, "managed") - } - - /// Creates a new buffer from data. - /// The buffer is [MTLPrivate](https://developer.apple.com/documentation/metal/mtlstoragemode) - /// - /// This method will block the computation because of the - /// lack of lifetime management through the GPU. - /// Internal comment for technical details. - pub fn new_buffer_with_data(&self, data: &[T]) -> Result> { - let size = core::mem::size_of_val(data) as NSUInteger; - let tmp = self.device.new_buffer_with_data( - data.as_ptr() as *const core::ffi::c_void, - size, - metal::MTLResourceOptions::StorageModeManaged, - ); - let real = self.allocate_buffer( - size, - metal::MTLResourceOptions::StorageModePrivate, - "with_data", - )?; - let command_buffer = self.command_buffer()?; - command_buffer.set_label("with_data"); - let blit = command_buffer.new_blit_command_encoder(); - blit.set_label("with_data_blit"); - blit.copy_from_buffer(&tmp, 0, &real, 0, tmp.length()); - blit.end_encoding(); - - // This is necessary, for mmaped safetensors - // Because of the unsafe slice cast we're doing. - // The slice might not live long enough for metal - // To actually fill the GPU buffer. - // Putting this wait forces the GPU buffer to be filled - // with the actual data allowing the CPU storage to do - // deallocate properly. - self.wait_until_completed()?; - Ok(real) - } - - pub fn allocate_zeros(&self, size_in_bytes: usize) -> Result> { - let buffer = self.allocate_buffer( - size_in_bytes as NSUInteger, - MTLResourceOptions::StorageModePrivate, - "allocate_zeros", - )?; - let command_buffer = self.command_buffer()?; - command_buffer.set_label("zeros"); - let blit = command_buffer.new_blit_command_encoder(); - blit.fill_buffer( - &buffer, - metal::NSRange { - location: 0, - length: buffer.length(), - }, - 0, - ); - blit.end_encoding(); - Ok(buffer) - } - - /// The critical allocator algorithm - fn allocate_buffer( - &self, - size: NSUInteger, - option: MTLResourceOptions, - _name: &str, - ) -> Result> { - let mut buffers = self.buffers.try_write().map_err(MetalError::from)?; - let subbuffers = buffers.entry((size, option)).or_insert(vec![]); - - for sub in &mut *subbuffers { - if Arc::strong_count(sub) == 1 { - return Ok(sub.clone()); - } - } - let new_buffer = self.device.new_buffer(size as NSUInteger, option); - let new_buffer = Arc::new(new_buffer); - subbuffers.push(new_buffer.clone()); - for subbuffers in buffers.values_mut() { - let newbuffers = subbuffers - .iter() - .filter(|s| Arc::strong_count(s) > 1) - .map(Arc::clone) - .collect(); - *subbuffers = newbuffers; - } - Ok(new_buffer) - } - - /// Create a metal GPU capture trace on [`path`]. - pub fn capture>(&self, path: P) -> Result<()> { - let capture = metal::CaptureManager::shared(); - let descriptor = metal::CaptureDescriptor::new(); - descriptor.set_destination(metal::MTLCaptureDestination::GpuTraceDocument); - descriptor.set_capture_device(self); - descriptor.set_output_url(path); - - capture - .start_capture(&descriptor) - .map_err(MetalError::from)?; - Ok(()) - } -} - -#[derive(Debug, Clone)] -pub struct MetalStorage { - /// The actual buffer containing the data. - buffer: Arc, - /// a reference to the device owning this buffer - device: MetalDevice, - /// The dtype is kept since buffers are untyped. - dtype: DType, -} - -impl BackendStorage for MetalStorage { - type Device = MetalDevice; - - fn try_clone(&self, _: &Layout) -> Result { - Ok(self.clone()) - } - - fn dtype(&self) -> DType { - self.dtype - } - - fn device(&self) -> &Self::Device { - &self.device - } - - fn to_cpu_storage(&self) -> Result { - match self.dtype { - DType::U8 => Ok(CpuStorage::U8(self.to_cpu()?)), - DType::U32 => Ok(CpuStorage::U32(self.to_cpu()?)), - DType::I64 => Ok(CpuStorage::I64(self.to_cpu()?)), - DType::F16 => Ok(CpuStorage::F16(self.to_cpu()?)), - DType::BF16 => Ok(CpuStorage::BF16(self.to_cpu()?)), - DType::F32 => Ok(CpuStorage::F32(self.to_cpu()?)), - DType::F64 => Ok(CpuStorage::F64(self.to_cpu()?)), - } - } - - fn affine(&self, layout: &Layout, mul: f64, add: f64) -> Result { - let device = self.device().clone(); - - let shape = layout.shape(); - let el = shape.elem_count(); - let dtype = self.dtype; - - let buffer = device.new_buffer(el, self.dtype, "affine")?; - let command_buffer = self.device.command_buffer()?; - if layout.is_contiguous() && layout.start_offset() == 0 { - let name = match self.dtype { - DType::F32 => "affine_f32", - DType::F16 => "affine_f16", - DType::BF16 => "affine_bf16", - dtype => crate::bail!("Metal contiguous affine {dtype:?} not implemented"), - }; - candle_metal_kernels::call_affine( - &device.device, - &command_buffer, - &device.kernels, - name, - el, - &self.buffer, - &buffer, - mul as f32, - add as f32, - ) - .map_err(MetalError::from)?; - } else { - let name = match self.dtype { - DType::F32 => "affine_f32_strided", - DType::F16 => "affine_f16_strided", - DType::BF16 => "affine_bf16_strided", - dtype => crate::bail!("Metal strided affine {dtype:?} not implemented"), - }; - candle_metal_kernels::call_affine_strided( - &device.device, - &command_buffer, - &device.kernels, - name, - layout.dims(), - &self.buffer, - layout.stride(), - layout.start_offset() * dtype.size_in_bytes(), - &buffer, - mul as f32, - add as f32, - ) - .map_err(MetalError::from)?; - } - Ok(Self::new(buffer, device.clone(), dtype)) - } - - fn powf(&self, layout: &Layout, pow: f64) -> Result { - let device = self.device().clone(); - - let shape = layout.shape(); - let el = shape.elem_count(); - let dtype = self.dtype; - - let buffer = device.new_buffer(el, self.dtype, "powf")?; - let command_buffer = self.device.command_buffer()?; - if layout.is_contiguous() && layout.start_offset() == 0 { - let name = match self.dtype { - DType::F32 => "powf_f32", - DType::F16 => "powf_f16", - dtype => crate::bail!("Metal contiguous powf {dtype:?} not implemented"), - }; - candle_metal_kernels::call_powf( - &device.device, - &command_buffer, - &device.kernels, - name, - el, - &self.buffer, - &buffer, - pow as f32, - ) - .map_err(MetalError::from)?; - } else { - let name = match self.dtype { - DType::F32 => "powf_f32_strided", - DType::F16 => "powf_f16_strided", - dtype => crate::bail!("Metal strided powf {dtype:?} not implemented"), - }; - candle_metal_kernels::call_powf_strided( - &device.device, - &command_buffer, - &device.kernels, - name, - layout.dims(), - &self.buffer, - layout.stride(), - layout.start_offset() * dtype.size_in_bytes(), - &buffer, - pow as f32, - ) - .map_err(MetalError::from)?; - } - Ok(Self::new(buffer, device.clone(), dtype)) - } - - fn elu(&self, layout: &Layout, alpha: f64) -> Result { - let device = self.device().clone(); - - let shape = layout.shape(); - let el = shape.elem_count(); - let dtype = self.dtype; - - let buffer = device.new_buffer(el, self.dtype, "elu")?; - let command_buffer = self.device.command_buffer()?; - if layout.is_contiguous() && layout.start_offset() == 0 { - let name = match self.dtype { - DType::F32 => "elu_f32", - DType::F16 => "elu_f16", - dtype => crate::bail!("Metal contiguous elu {dtype:?} not implemented"), - }; - candle_metal_kernels::call_elu( - &device.device, - &command_buffer, - &device.kernels, - name, - el, - &self.buffer, - &buffer, - alpha as f32, - ) - .map_err(MetalError::from)?; - } else { - let name = match self.dtype { - DType::F32 => "elu_f32_strided", - DType::F16 => "elu_f16_strided", - dtype => crate::bail!("Metal strided elu {dtype:?} not implemented"), - }; - candle_metal_kernels::call_elu_strided( - &device.device, - &command_buffer, - &device.kernels, - name, - layout.dims(), - &self.buffer, - layout.stride(), - layout.start_offset() * dtype.size_in_bytes(), - &buffer, - alpha as f32, - ) - .map_err(MetalError::from)?; - } - Ok(Self::new(buffer, device.clone(), dtype)) - } - - fn reduce_op(&self, op: ReduceOp, layout: &Layout, sum_dims: &[usize]) -> Result { - let device = self.device.clone(); - let src_stride = layout.stride(); - let src_dims = layout.shape().dims(); - // Source dims and strides with the sum dims at the end. - let mut dims = vec![]; - let mut stride = vec![]; - let mut dst_el: usize = 1; - for (dim_idx, &d) in src_dims.iter().enumerate() { - if !sum_dims.contains(&dim_idx) { - dst_el *= d; - dims.push(d); - stride.push(src_stride[dim_idx]); - } - } - for &dim_idx in sum_dims.iter() { - dims.push(src_dims[dim_idx]); - stride.push(src_stride[dim_idx]); - } - - // The reduction loop requires the shared array to be properly initialized and for - // this we want the number of threads to be a power of two. - let (name, check_empty, return_index) = match (op, self.dtype) { - (ReduceOp::Sum, DType::F32) => ("fast_sum_f32_strided", false, false), - (ReduceOp::Min, DType::F32) => ("fast_min_f32_strided", true, false), - (ReduceOp::Max, DType::F32) => ("fast_max_f32_strided", true, false), - (ReduceOp::ArgMin, DType::F32) => ("fast_argmin_f32_strided", true, true), - (ReduceOp::ArgMax, DType::F32) => ("fast_argmax_f32_strided", true, true), - (ReduceOp::Sum, DType::U32) => ("fast_sum_u32_strided", false, false), - (ReduceOp::Min, DType::U32) => ("fast_min_u32_strided", true, false), - (ReduceOp::Max, DType::U32) => ("fast_max_u32_strided", true, false), - (ReduceOp::ArgMin, DType::U32) => ("fast_argmin_u32_strided", true, true), - (ReduceOp::ArgMax, DType::U32) => ("fast_argmax_u32_strided", true, true), - (ReduceOp::Sum, DType::F16) => ("fast_sum_f16_strided", false, false), - (ReduceOp::Min, DType::F16) => ("fast_min_f16_strided", true, false), - (ReduceOp::Max, DType::F16) => ("fast_max_f16_strided", true, false), - (ReduceOp::ArgMin, DType::F16) => ("fast_argmin_f16_strided", true, true), - (ReduceOp::ArgMax, DType::F16) => ("fast_argmax_f16_strided", true, true), - (ReduceOp::Sum, DType::BF16) => ("fast_sum_bf16_strided", false, false), - (ReduceOp::Min, DType::BF16) => ("fast_min_bf16_strided", true, false), - (ReduceOp::Max, DType::BF16) => ("fast_max_bf16_strided", true, false), - (ReduceOp::ArgMin, DType::BF16) => ("fast_argmin_bf16_strided", true, true), - (ReduceOp::ArgMax, DType::BF16) => ("fast_argmax_bf16_strided", true, true), - (ReduceOp::Sum, DType::I64) => ("fast_sum_i64_strided", false, false), - (ReduceOp::Min, DType::I64) => ("fast_min_i64_strided", true, false), - (ReduceOp::Max, DType::I64) => ("fast_max_i64_strided", true, false), - (ReduceOp::ArgMin, DType::I64) => ("fast_argmin_i64_strided", true, true), - (ReduceOp::ArgMax, DType::I64) => ("fast_argmax_i64_strided", true, true), - (ReduceOp::Sum, DType::U8) => ("fast_sum_u8_strided", false, false), - (ReduceOp::Min, DType::U8) => ("fast_min_u8_strided", true, false), - (ReduceOp::Max, DType::U8) => ("fast_max_u8_strided", true, false), - (ReduceOp::ArgMin, DType::U8) => ("fast_argmin_u8_strided", true, true), - (ReduceOp::ArgMax, DType::U8) => ("fast_argmax_u8_strided", true, true), - (k, dtype) => crate::bail!("Metal reduce op {k:?} {dtype:?} not implemented"), - }; - if check_empty && layout.shape().elem_count() == 0 { - Err(crate::Error::EmptyTensor { op: "reduce" }.bt())? - } - let dtype = if return_index { DType::U32 } else { self.dtype }; - let buffer = device.new_buffer(dst_el, dtype, "reduce")?; - let command_buffer = self.device.command_buffer()?; - candle_metal_kernels::call_reduce_strided( - &device.device, - &command_buffer, - &device.kernels, - name, - &dims, - &stride, - dst_el, - &self.buffer, - layout.start_offset() * self.dtype.size_in_bytes(), - &buffer, - ) - .map_err(MetalError::from)?; - - Ok(Self::new(buffer, device, dtype)) - } - - fn cmp(&self, op: CmpOp, rhs: &Self, lhs_l: &Layout, rhs_l: &Layout) -> Result { - let name = match op { - CmpOp::Eq => "eq", - CmpOp::Ne => "ne", - CmpOp::Le => "le", - CmpOp::Ge => "ge", - CmpOp::Lt => "lt", - CmpOp::Gt => "gt", - }; - self.binary(name, rhs, lhs_l, rhs_l) - } - - fn to_dtype(&self, layout: &Layout, dtype: DType) -> Result { - let device = self.device(); - let shape = layout.shape(); - let el_count = shape.elem_count(); - let buffer = device.new_buffer(el_count, dtype, "todtype")?; - let command_buffer = device.command_buffer()?; - if layout.is_contiguous() && layout.start_offset() == 0 { - let kernel_name = match (self.dtype, dtype) { - (DType::U32, DType::F32) => "cast_u32_f32", - (DType::U32, DType::U8) => "cast_u32_u8", - (DType::U32, DType::I64) => "cast_u32_i64", - (DType::U32, DType::BF16) => "cast_u32_bf16", - - (DType::U8, DType::U32) => "cast_u8_u32", - (DType::U8, DType::F32) => "cast_u8_f32", - (DType::U8, DType::I64) => "cast_u8_i64", - (DType::U8, DType::BF16) => "cast_u8_bf16", - - (DType::F32, DType::F16) => "cast_f32_f16", - (DType::F32, DType::BF16) => "cast_f32_bf16", - - (DType::I64, DType::F32) => "cast_i64_f32", - - (DType::F16, DType::BF16) => "cast_f16_bf16", - (DType::F16, DType::F32) => "cast_f16_f32", - - (DType::BF16, DType::U8) => "cast_bf16_u8", - (DType::BF16, DType::U32) => "cast_bf16_u32", - (DType::BF16, DType::F16) => "cast_bf16_f16", - (DType::BF16, DType::F32) => "cast_bf16_f32", - - (left, right) => { - crate::bail!("Metal contiguous to_dtype {left:?} {right:?} not implemented") - } - }; - candle_metal_kernels::call_cast_contiguous( - &device.device, - &command_buffer, - &device.kernels, - kernel_name, - el_count, - &self.buffer, - layout.start_offset() * self.dtype.size_in_bytes(), - &buffer, - ) - .map_err(MetalError::from)?; - } else { - let kernel_name = match (self.dtype, dtype) { - (DType::U32, DType::F32) => "cast_u32_f32_strided", - (DType::U32, DType::U8) => "cast_u32_u8_strided", - (DType::U32, DType::I64) => "cast_u32_i64_strided", - (DType::U8, DType::U32) => "cast_u8_u32_strided", - (DType::U8, DType::F32) => "cast_u8_f32_strided", - (DType::U8, DType::I64) => "cast_u8_i64_strided", - (DType::F32, DType::F16) => "cast_f32_f16_strided", - (DType::F16, DType::F32) => "cast_f16_f32_strided", - (DType::I64, DType::F32) => "cast_i64_f32_strided", - (DType::F32, DType::BF16) => "cast_f32_bf16_strided", - (DType::BF16, DType::F32) => "cast_bf16_f32_strided", - (left, right) => { - crate::bail!("Metal strided to_dtype {left:?} {right:?} not implemented") - } - }; - candle_metal_kernels::call_cast_strided( - &device.device, - &command_buffer, - &device.kernels, - kernel_name, - layout.dims(), - &self.buffer, - layout.stride(), - layout.start_offset() * self.dtype.size_in_bytes(), - &buffer, - ) - .map_err(MetalError::from)?; - } - command_buffer.set_label("to_dtype"); - Ok(Self::new(buffer, device.clone(), dtype)) - } - - fn unary_impl(&self, layout: &Layout) -> Result { - let device = self.device(); - let dtype = self.dtype; - let shape = layout.shape(); - let el_count = shape.elem_count(); - let buffer = device.new_buffer(el_count, dtype, B::KERNEL)?; - let command_buffer = device.command_buffer()?; - command_buffer.set_label(B::KERNEL); - if layout.is_contiguous() && layout.start_offset() == 0 { - use candle_metal_kernels::unary::contiguous; - - let kernel_name = match (B::KERNEL, dtype) { - ("ucos", DType::F32) => contiguous::cos::FLOAT, - ("usin", DType::F32) => contiguous::sin::FLOAT, - ("usqr", DType::F32) => contiguous::sqr::FLOAT, - ("usqrt", DType::F32) => contiguous::sqrt::FLOAT, - ("uneg", DType::F32) => contiguous::neg::FLOAT, - ("uexp", DType::F32) => contiguous::exp::FLOAT, - ("ulog", DType::F32) => contiguous::log::FLOAT, - ("ugelu", DType::F32) => contiguous::gelu::FLOAT, - ("ugelu_erf", DType::F32) => contiguous::gelu_erf::FLOAT, - ("uerf", DType::F32) => contiguous::erf::FLOAT, - ("usilu", DType::F32) => contiguous::silu::FLOAT, - ("uabs", DType::F32) => contiguous::abs::FLOAT, - ("uceil", DType::F32) => contiguous::ceil::FLOAT, - ("ufloor", DType::F32) => contiguous::floor::FLOAT, - ("uround", DType::F32) => contiguous::round::FLOAT, - ("urecip", DType::F32) => contiguous::recip::FLOAT, - ("utanh", DType::F32) => contiguous::tanh::FLOAT, - ("urelu", DType::F32) => contiguous::relu::FLOAT, - ("ucos", DType::F16) => contiguous::cos::HALF, - ("usin", DType::F16) => contiguous::sin::HALF, - ("usqr", DType::F16) => contiguous::sqr::HALF, - ("usqrt", DType::F16) => contiguous::sqrt::HALF, - ("uneg", DType::F16) => contiguous::neg::HALF, - ("uexp", DType::F16) => contiguous::exp::HALF, - ("ulog", DType::F16) => contiguous::log::HALF, - ("ugelu", DType::F16) => contiguous::gelu::HALF, - ("ugelu_erf", DType::F16) => contiguous::gelu_erf::HALF, - ("uerf", DType::F16) => contiguous::erf::HALF, - ("usilu", DType::F16) => contiguous::silu::HALF, - ("uabs", DType::F16) => contiguous::abs::HALF, - ("uceil", DType::F16) => contiguous::ceil::HALF, - ("ufloor", DType::F16) => contiguous::floor::HALF, - ("uround", DType::F16) => contiguous::round::HALF, - ("urecip", DType::F16) => contiguous::recip::HALF, - ("utanh", DType::F16) => contiguous::tanh::HALF, - ("urelu", DType::F16) => contiguous::relu::HALF, - (name, dtype) => { - crate::bail!("Metal contiguous unary {name} {dtype:?} not implemented") - } - }; - candle_metal_kernels::call_unary_contiguous( - &device.device, - &command_buffer, - &device.kernels, - kernel_name, - el_count, - &self.buffer, - &buffer, - ) - .map_err(MetalError::from)?; - } else { - use candle_metal_kernels::unary::strided; - let kernel_name = match (B::KERNEL, dtype) { - ("ucos", DType::F32) => strided::cos::FLOAT, - ("usin", DType::F32) => strided::sin::FLOAT, - ("usqr", DType::F32) => strided::sqr::FLOAT, - ("usqrt", DType::F32) => strided::sqrt::FLOAT, - ("uneg", DType::F32) => strided::neg::FLOAT, - ("uexp", DType::F32) => strided::exp::FLOAT, - ("ulog", DType::F32) => strided::log::FLOAT, - ("ugelu", DType::F32) => strided::gelu::FLOAT, - ("ugelu_erf", DType::F32) => strided::gelu_erf::FLOAT, - ("uerf", DType::F32) => strided::erf::FLOAT, - ("usilu", DType::F32) => strided::silu::FLOAT, - ("uabs", DType::F32) => strided::abs::FLOAT, - ("uceil", DType::F32) => strided::ceil::FLOAT, - ("ufloor", DType::F32) => strided::floor::FLOAT, - ("urelu", DType::F32) => strided::relu::FLOAT, - ("uround", DType::F32) => strided::round::FLOAT, - ("ucos", DType::F16) => strided::cos::HALF, - ("usin", DType::F16) => strided::sin::HALF, - ("usqr", DType::F16) => strided::sqr::HALF, - ("usqrt", DType::F16) => strided::sqrt::HALF, - ("uneg", DType::F16) => strided::neg::HALF, - ("uexp", DType::F16) => strided::exp::HALF, - ("ulog", DType::F16) => strided::log::HALF, - ("ugelu", DType::F16) => strided::gelu::HALF, - ("ugelu_erf", DType::F16) => strided::gelu_erf::HALF, - ("uerf", DType::F16) => strided::erf::HALF, - ("usilu", DType::F16) => strided::silu::HALF, - ("uabs", DType::F16) => strided::abs::HALF, - ("uceil", DType::F16) => strided::ceil::HALF, - ("ufloor", DType::F16) => strided::floor::HALF, - ("urelu", DType::F16) => strided::relu::HALF, - ("uround", DType::F16) => strided::round::HALF, - (name, dtype) => { - crate::bail!("Metal strided unary {name} {dtype:?} not implemented") - } - }; - candle_metal_kernels::call_unary_strided( - &device.device, - &command_buffer, - &device.kernels, - kernel_name, - layout.dims(), - &self.buffer, - layout.stride(), - layout.start_offset() * self.dtype.size_in_bytes(), - &buffer, - 0, - ) - .map_err(MetalError::from)?; - } - Ok(Self::new(buffer, device.clone(), dtype)) - } - - fn binary_impl( - &self, - rhs: &Self, - lhs_l: &Layout, - rhs_l: &Layout, - ) -> Result { - self.binary(B::KERNEL, rhs, lhs_l, rhs_l) - } - - fn where_cond( - &self, - layout: &Layout, - t: &Self, - t_l: &Layout, - f: &Self, - f_l: &Layout, - ) -> Result { - let device = self.device.clone(); - let shape = t_l.shape(); - let dims = shape.dims(); - let el = shape.elem_count(); - let dtype = t.dtype; - let buffer = self.device.new_buffer(el, dtype, "where")?; - let command_buffer = self.device.command_buffer()?; - if t.dtype() != f.dtype() { - crate::bail!( - "Invalid where: different dtypes for values {:?} != {:?}", - t.dtype(), - f.dtype() - ); - } - let name = match (self.dtype, t.dtype()) { - (DType::U8, DType::F32) => "where_u8_f32", - (DType::U8, DType::BF16) => "where_u8_bf16", - (DType::U8, DType::F16) => "where_u8_f16", - (DType::U8, DType::I64) => "where_u8_i64", - (DType::U8, DType::U32) => "where_u8_u32", - (DType::U8, DType::U8) => "where_u8_u8", - (left, right) => crate::bail!("Metal where_cond {left:?} {right:?} not implemented"), - }; - candle_metal_kernels::call_where_cond_strided( - &device.device, - &command_buffer, - &device.kernels, - name, - dims, - &self.buffer, - ( - layout.stride(), - layout.start_offset() * self.dtype.size_in_bytes(), - ), - &t.buffer, - (t_l.stride(), t_l.start_offset() * t.dtype.size_in_bytes()), - &f.buffer, - (f_l.stride(), f_l.start_offset() * f.dtype.size_in_bytes()), - &buffer, - ) - .map_err(MetalError::from)?; - Ok(Self::new(buffer, device, dtype)) - } - - fn conv1d( - &self, - layout: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &ParamsConv1D, - ) -> Result { - let device = self.device().clone(); - let shape = layout.shape(); - let dims = shape.dims(); - let strides = layout.stride(); - - let stride = params.stride; - let dilation = params.dilation; - let padding = params.padding; - let k_size = params.k_size; - let l_out = (dims[2] + 2 * padding - dilation * (k_size - 1) - 1) / stride + 1; - let dst_el = dims[0] * l_out * dims[1] * k_size; - let dst = self - .device - .new_buffer(dst_el, self.dtype, "conv1d_im2col")?; - let command_buffer = self.device.command_buffer()?; - let name = match self.dtype { - DType::F32 => "im2col1d_f32", - dtype => crate::bail!("Metal conv1d {dtype:?} not implemented"), - }; - candle_metal_kernels::call_im2col1d_strided( - &self.device.device, - &command_buffer, - &self.device.kernels, - name, - layout.shape().dims(), - strides, - (k_size, stride, padding, dilation), - &self.buffer, - layout.start_offset() * self.dtype.size_in_bytes(), - &dst, - ) - .map_err(MetalError::from)?; - let col = Self { - buffer: dst, - device, - dtype: self.dtype, - }; - let l_out = params.l_out(); - let b = params.b_size; - let n = params.c_out; - let k = params.k_size * params.c_in; - let m = l_out; - let col_l = Layout::contiguous((b, m, k)); - let res = if kernel_l.is_contiguous() { - let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) - .transpose(1, 2)? - .broadcast_as((b, k, n))?; - col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? - } else { - // Make the kernel contiguous if not already the case. - let mut kernel_c = self.device().zeros_impl(kernel_l.shape(), kernel.dtype())?; - kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?; - let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) - .transpose(1, 2)? - .broadcast_as((b, k, n))?; - col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? - }; - let res_l = Layout::contiguous((b, l_out, n)).transpose(1, 2)?; - let mut res_t = self.device().zeros_impl(res_l.shape(), res.dtype())?; - res.copy_strided_src(&mut res_t, 0, &res_l)?; - Ok(res_t) - } - - fn conv_transpose1d( - &self, - _l: &Layout, - _kernel: &Self, - _kernel_l: &Layout, - _params: &ParamsConvTranspose1D, - ) -> Result { - crate::bail!("Metal conv_transpose1d not implemented") - } - - fn conv2d( - &self, - layout: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &ParamsConv2D, - ) -> Result { - let device = self.device().clone(); - let shape = layout.shape(); - let dims = shape.dims(); - - let stride = params.stride; - let dilation = params.dilation; - let padding = params.padding; - let h_k = params.k_h; - let w_k = params.k_w; - let h = dims[2]; - let w = dims[3]; - let h_out = (h + 2 * padding - dilation * (h_k - 1) - 1) / stride + 1; - let w_out = (w + 2 * padding - dilation * (w_k - 1) - 1) / stride + 1; - let dst_el = dims[0] * h_out * w_out * dims[1] * h_k * w_k; - - let dst = self - .device - .new_buffer(dst_el, self.dtype, "conv2d_im2col")?; - let command_buffer = self.device.command_buffer()?; - let name = match self.dtype { - DType::F32 => "im2col_f32", - dtype => crate::bail!("Metal conv2d {dtype:?} not implemented"), - }; - candle_metal_kernels::call_im2col_strided( - &self.device.device, - &command_buffer, - &self.device.kernels, - name, - layout.shape().dims(), - layout.stride(), - (h_k, w_k, stride, padding, dilation), - &self.buffer, - layout.start_offset() * self.dtype.size_in_bytes(), - &dst, - ) - .map_err(MetalError::from)?; - let col = Self { - buffer: dst, - device, - dtype: self.dtype, - }; - let h_out = params.out_h(); - let w_out = params.out_w(); - let b = params.b_size; - let n = params.c_out; - let k = params.k_h * params.k_w * params.c_in; - let m = h_out * w_out; - let col_l = Layout::contiguous((b, m, k)); - let res = if kernel_l.is_contiguous() { - let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) - .transpose(1, 2)? - .broadcast_as((b, k, n))?; - col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? - } else { - // Make the kernel contiguous if not already the case. - let mut kernel_c = self.device().zeros_impl(kernel_l.shape(), kernel.dtype())?; - kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?; - let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) - .transpose(1, 2)? - .broadcast_as((b, k, n))?; - col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? - }; - let res_l = Layout::contiguous((b, h_out, w_out, n)) - .transpose(1, 2)? - .transpose(1, 3)?; - let mut res_t = self.device().zeros_impl(res_l.shape(), res.dtype())?; - res.copy_strided_src(&mut res_t, 0, &res_l)?; - Ok(res_t) - } - - fn conv_transpose2d( - &self, - _l: &Layout, - _kernel: &Self, - _kernel_l: &Layout, - _params: &ParamsConvTranspose2D, - ) -> Result { - crate::bail!("Metal conv_tranpose2d not implemented") - } - - fn avg_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result { - crate::bail!("Metal avg_pool2d not implemented") - } - - fn max_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result { - crate::bail!("Metal max_pool2d not implemented") - } - - fn upsample_nearest1d(&self, _: &Layout, _: usize) -> Result { - crate::bail!("Metal upsample_nearest1d not implemented") - } - - fn upsample_nearest2d(&self, inp_l: &Layout, out_w: usize, out_h: usize) -> Result { - // let inp = &inp.slice(inp_l.start_offset()..); - let shape = inp_l.shape(); - let dims = shape.dims(); - let strides = inp_l.stride(); - if dims.len() != 4 { - crate::bail!("unexpected input shape for upsample {dims:?}") - } - let name = match self.dtype { - DType::F32 => "upsample_nearest2d_f32", - dtype => crate::bail!("Metal upsample_nearest2d {dtype:?} not implemented"), - }; - - let dst_el = out_w * out_h * dims[0] * dims[1]; - let buffer = self - .device - .new_buffer(dst_el, self.dtype, "upsample_nearest2d")?; - let command_buffer = self.device.command_buffer()?; - candle_metal_kernels::call_upsample_nearest_2d( - &self.device.device, - &command_buffer, - &self.device.kernels, - name, - dims, - strides, - out_w, - out_h, - &self.buffer, - inp_l.start_offset() * self.dtype.size_in_bytes(), - &buffer, - ) - .map_err(MetalError::from)?; - Ok(Self::new(buffer, self.device.clone(), self.dtype)) - } - - fn gather(&self, src_l: &Layout, ids: &Self, ids_l: &Layout, dim: usize) -> Result { - let (ids_o1, _) = match ids_l.contiguous_offsets() { - Some(o12) => o12, - None => Err(crate::Error::RequiresContiguous { op: "gather" }.bt())?, - }; - let ids_el = ids_l.dims()[dim]; - let dst_el = ids_l.shape().elem_count(); - let dtype = self.dtype; - let device = self.device(); - let buffer = device.new_buffer(dst_el, dtype, "index_select")?; - let name = match (ids.dtype, self.dtype) { - (DType::U32, DType::F32) => "gather_u32_f32", - (DType::U32, DType::F16) => "gather_u32_f16", - (left, right) => crate::bail!("Metal gather {left:?} {right:?} not implemented"), - }; - let command_buffer = self.device.command_buffer()?; - candle_metal_kernels::call_gather( - &device.device, - &command_buffer, - &self.device.kernels, - name, - src_l.dims(), - ids_el, - dim, - &self.buffer, - src_l.start_offset() * dtype.size_in_bytes(), - &ids.buffer, - ids_o1 * ids.dtype.size_in_bytes(), - &buffer, - ) - .map_err(MetalError::from)?; - Ok(Self::new(buffer, device.clone(), dtype)) - } - - fn scatter_add( - &self, - l: &Layout, - ids: &Self, - ids_l: &Layout, - src: &Self, - src_l: &Layout, - dim: usize, - ) -> Result { - let mut acc = self.device.zeros_impl(l.shape(), self.dtype())?; - self.copy_strided_src(&mut acc, 0, l)?; - let (ids_offset, _) = match ids_l.contiguous_offsets() { - Some(o12) => o12, - None => Err(crate::Error::RequiresContiguous { op: "scatter-add" }.bt())?, - }; - let src_offset = match src_l.contiguous_offsets() { - Some((o1, _)) => o1, - None => Err(crate::Error::RequiresContiguous { op: "scatter-add" }.bt())?, - }; - let name = match (ids.dtype, self.dtype) { - (DType::U32, DType::F32) => "sa_u32_f32", - _ => Err(MetalError::UnexpectedDType { - msg: "scatter-add ids should be u8/u32/i64", - expected: DType::U32, - got: ids.dtype(), - })?, - }; - let command_buffer = self.device.command_buffer()?; - candle_metal_kernels::call_scatter_add( - &self.device.device, - &command_buffer, - &self.device.kernels, - name, - src_l.dims(), - l.dims(), - dim, - &src.buffer, - src_offset * src.dtype.size_in_bytes(), - &ids.buffer, - ids_offset * ids.dtype.size_in_bytes(), - &acc.buffer, - ) - .map_err(MetalError::from)?; - Ok(acc) - } - - fn index_select(&self, ids: &Self, src_l: &Layout, ids_l: &Layout, dim: usize) -> Result { - if !(src_l.is_contiguous() - && src_l.start_offset() == 0 - && ids_l.is_contiguous() - && ids_l.start_offset() == 0) - { - crate::bail!("Metal strided index_select not implemented"); - } - let left_size: usize = src_l.dims()[..dim].iter().product(); - let right_size: usize = src_l.dims()[dim + 1..].iter().product(); - let ids_el = ids_l.shape().elem_count(); - let dst_el = ids_el * left_size * right_size; - let dtype = self.dtype; - let device = self.device(); - let buffer = device.new_buffer(dst_el, dtype, "index_select")?; - let name = match (ids.dtype, self.dtype) { - (DType::U8, DType::BF16) => "is_u8_bf16", - - (DType::U32, DType::F32) => "is_u32_f32", - (DType::U32, DType::F16) => "is_u32_f16", - (DType::U32, DType::BF16) => "is_u32_bf16", - - (left, right) => { - crate::bail!("Metal contiguous index_select {left:?} {right:?} not implemented") - } - }; - let command_buffer = self.device.command_buffer()?; - candle_metal_kernels::call_index_select( - &device.device, - &command_buffer, - &self.device.kernels, - name, - src_l.dims(), - ids_el, - dim, - &self.buffer, - &ids.buffer, - &buffer, - ) - .map_err(MetalError::from)?; - Ok(Self::new(buffer, device.clone(), dtype)) - } - - fn index_add( - &self, - l: &Layout, - ids: &Self, - ids_l: &Layout, - src: &Self, - src_l: &Layout, - dim: usize, - ) -> Result { - let mut acc = self.device.zeros_impl(l.shape(), self.dtype())?; - self.copy_strided_src(&mut acc, 0, l)?; - let (ids_offset, _) = match ids_l.contiguous_offsets() { - Some(o12) => o12, - None => Err(crate::Error::RequiresContiguous { op: "index-add" }.bt())?, - }; - let src_offset = match src_l.contiguous_offsets() { - Some((o1, _)) => o1, - None => Err(crate::Error::RequiresContiguous { op: "index-add" }.bt())?, - }; - let name = match (ids.dtype, self.dtype) { - (DType::U32, DType::F32) => "ia_u32_f32", - _ => Err(MetalError::UnexpectedDType { - msg: "index-add ids should be u32", - expected: DType::U32, - got: ids.dtype(), - })?, - }; - let command_buffer = self.device.command_buffer()?; - candle_metal_kernels::call_index_add( - &self.device.device, - &command_buffer, - &self.device.kernels, - name, - src_l.dims(), - l.dims(), - ids_l.dims(), - dim, - &src.buffer, - src_offset * src.dtype.size_in_bytes(), - &ids.buffer, - ids_offset * ids.dtype.size_in_bytes(), - &acc.buffer, - ) - .map_err(MetalError::from)?; - Ok(acc) - } - fn matmul( - &self, - rhs: &Self, - (b, m, n, k): (usize, usize, usize, usize), - lhs_l: &Layout, - rhs_l: &Layout, - ) -> Result { - let buffer = self.device.new_buffer(b * m * n, self.dtype, "matmul")?; - let name = match self.dtype { - DType::F32 => "sgemm", - DType::F16 => "hgemm", - dtype => { - return Err(MetalError::Message(format!("matmul doesn't support {dtype:?}")).into()) - } - }; - - let command_buffer = self.device.command_buffer()?; - command_buffer.set_label("matmul"); - candle_metal_kernels::call_gemm( - &self.device.device, - &command_buffer, - &self.device.kernels, - name, - (b, m, n, k), - lhs_l.stride(), - lhs_l.start_offset() * self.dtype.size_in_bytes(), - &self.buffer, - rhs_l.stride(), - rhs_l.start_offset() * rhs.dtype.size_in_bytes(), - &rhs.buffer, - &buffer, - ) - .map_err(MetalError::from)?; - Ok(Self::new(buffer, self.device.clone(), self.dtype())) - } - - fn copy_strided_src(&self, dst: &mut Self, dst_offset: usize, src_l: &Layout) -> Result<()> { - let command_buffer = self.device.command_buffer()?; - if src_l.is_contiguous() && self.dtype == dst.dtype() { - command_buffer.set_label("copy_contiguous"); - let blit = command_buffer.new_blit_command_encoder(); - blit.set_label("copy_contiguous"); - let src_offset = (src_l.start_offset() * self.dtype.size_in_bytes()) as NSUInteger; - let length = (src_l.shape().elem_count() * self.dtype.size_in_bytes()) as NSUInteger; - let dst_offset = (dst_offset * dst.dtype().size_in_bytes()) as NSUInteger; - blit.copy_from_buffer(&self.buffer, src_offset, dst.buffer(), dst_offset, length); - blit.end_encoding(); - } else { - let src_shape = src_l.shape(); - let el_count = src_shape.elem_count(); - if el_count == 0 { - return Ok(()); - } - let kernel_name = match self.dtype { - DType::F32 => candle_metal_kernels::unary::strided::copy::FLOAT, - DType::F16 => candle_metal_kernels::unary::strided::copy::HALF, - DType::BF16 => candle_metal_kernels::unary::strided::copy::BFLOAT, - DType::I64 => candle_metal_kernels::unary::strided::copy::I64, - DType::U32 => candle_metal_kernels::unary::strided::copy::U32, - DType::U8 => candle_metal_kernels::unary::strided::copy::U8, - dtype => crate::bail!("Metal copy_strided {dtype:?} not implemented"), - }; - candle_metal_kernels::call_unary_strided( - &self.device.device, - &command_buffer, - &self.device.kernels, - kernel_name, - src_l.dims(), - &self.buffer, - src_l.stride(), - src_l.start_offset() * self.dtype.size_in_bytes(), - &dst.buffer, - dst_offset * dst.dtype.size_in_bytes(), - ) - .map_err(MetalError::from)?; - command_buffer.set_label("copy_strided"); - } - Ok(()) - } -} - -impl MetalStorage { - pub fn new(buffer: Arc, device: MetalDevice, dtype: DType) -> Self { - Self { - buffer, - device, - dtype, - } - } - - pub fn buffer(&self) -> &Buffer { - &self.buffer - } - - pub fn binary( - &self, - op: &'static str, - rhs: &Self, - lhs_l: &Layout, - rhs_l: &Layout, - ) -> Result { - let device = self.device(); - let shape = lhs_l.shape(); - let el_count = shape.elem_count(); - let command_buffer = device.command_buffer()?; - let (buffer, dtype) = if (lhs_l.is_contiguous() && lhs_l.start_offset() == 0) - && (rhs_l.is_contiguous() && rhs_l.start_offset() == 0) - && &op[..1] != "b" - { - use candle_metal_kernels::binary::contiguous; - - let (kernel_name, dtype) = match (op, self.dtype) { - ("add", DType::F32) => (contiguous::add::FLOAT, self.dtype), - ("sub", DType::F32) => (contiguous::sub::FLOAT, self.dtype), - ("mul", DType::F32) => (contiguous::mul::FLOAT, self.dtype), - ("div", DType::F32) => (contiguous::div::FLOAT, self.dtype), - ("eq", DType::F32) => (contiguous::eq::FLOAT, DType::U8), - ("ne", DType::F32) => (contiguous::ne::FLOAT, DType::U8), - ("le", DType::F32) => (contiguous::le::FLOAT, DType::U8), - ("lt", DType::F32) => (contiguous::lt::FLOAT, DType::U8), - ("ge", DType::F32) => (contiguous::ge::FLOAT, DType::U8), - ("gt", DType::F32) => (contiguous::gt::FLOAT, DType::U8), - - ("add", DType::F16) => (contiguous::add::HALF, self.dtype), - ("sub", DType::F16) => (contiguous::sub::HALF, self.dtype), - ("mul", DType::F16) => (contiguous::mul::HALF, self.dtype), - ("div", DType::F16) => (contiguous::div::HALF, self.dtype), - ("eq", DType::F16) => (contiguous::eq::HALF, DType::U8), - ("ne", DType::F16) => (contiguous::ne::HALF, DType::U8), - ("le", DType::F16) => (contiguous::le::HALF, DType::U8), - ("lt", DType::F16) => (contiguous::lt::HALF, DType::U8), - ("ge", DType::F16) => (contiguous::ge::HALF, DType::U8), - ("gt", DType::F16) => (contiguous::gt::HALF, DType::U8), - - ("add", DType::BF16) => (contiguous::add::BFLOAT, self.dtype), - ("sub", DType::BF16) => (contiguous::sub::BFLOAT, self.dtype), - ("mul", DType::BF16) => (contiguous::mul::BFLOAT, self.dtype), - ("div", DType::BF16) => (contiguous::div::BFLOAT, self.dtype), - ("eq", DType::BF16) => (contiguous::eq::BFLOAT, DType::U8), - ("ne", DType::BF16) => (contiguous::ne::BFLOAT, DType::U8), - ("le", DType::BF16) => (contiguous::le::BFLOAT, DType::U8), - ("lt", DType::BF16) => (contiguous::lt::BFLOAT, DType::U8), - ("ge", DType::BF16) => (contiguous::ge::BFLOAT, DType::U8), - ("gt", DType::BF16) => (contiguous::gt::BFLOAT, DType::U8), - - ("add", DType::I64) => (contiguous::add::I64, self.dtype), - ("sub", DType::I64) => (contiguous::sub::I64, self.dtype), - ("mul", DType::I64) => (contiguous::mul::I64, self.dtype), - ("div", DType::I64) => (contiguous::div::I64, self.dtype), - ("eq", DType::I64) => (contiguous::eq::I64, DType::U8), - ("ne", DType::I64) => (contiguous::ne::I64, DType::U8), - ("le", DType::I64) => (contiguous::le::I64, DType::U8), - ("lt", DType::I64) => (contiguous::lt::I64, DType::U8), - ("ge", DType::I64) => (contiguous::ge::I64, DType::U8), - ("gt", DType::I64) => (contiguous::gt::I64, DType::U8), - - ("add", DType::U32) => (contiguous::add::U32, self.dtype), - ("sub", DType::U32) => (contiguous::sub::U32, self.dtype), - ("mul", DType::U32) => (contiguous::mul::U32, self.dtype), - ("div", DType::U32) => (contiguous::div::U32, self.dtype), - ("eq", DType::U32) => (contiguous::eq::U32, DType::U8), - ("ne", DType::U32) => (contiguous::ne::U32, DType::U8), - ("le", DType::U32) => (contiguous::le::U32, DType::U8), - ("lt", DType::U32) => (contiguous::lt::U32, DType::U8), - ("ge", DType::U32) => (contiguous::ge::U32, DType::U8), - ("gt", DType::U32) => (contiguous::gt::U32, DType::U8), - - ("add", DType::U8) => (contiguous::add::U8, self.dtype), - ("sub", DType::U8) => (contiguous::sub::U8, self.dtype), - ("mul", DType::U8) => (contiguous::mul::U8, self.dtype), - ("div", DType::U8) => (contiguous::div::U8, self.dtype), - ("eq", DType::U8) => (contiguous::eq::U8, DType::U8), - ("ne", DType::U8) => (contiguous::ne::U8, DType::U8), - ("le", DType::U8) => (contiguous::le::U8, DType::U8), - ("lt", DType::U8) => (contiguous::lt::U8, DType::U8), - ("ge", DType::U8) => (contiguous::ge::U8, DType::U8), - ("gt", DType::U8) => (contiguous::gt::U8, DType::U8), - - (name, dtype) => { - crate::bail!("Metal contiguous binary {name} {dtype:?} not implemented") - } - }; - let buffer = device.new_buffer(el_count, dtype, op)?; - candle_metal_kernels::call_binary_contiguous( - &device.device, - &command_buffer, - &device.kernels, - kernel_name, - el_count, - &self.buffer, - &rhs.buffer, - &buffer, - ) - .map_err(MetalError::from)?; - (buffer, dtype) - } else { - use candle_metal_kernels::binary::strided; - - let (kernel_name, dtype) = match (op, self.dtype) { - ("badd", DType::F32) => (strided::add::FLOAT, self.dtype), - ("bsub", DType::F32) => (strided::sub::FLOAT, self.dtype), - ("bmul", DType::F32) => (strided::mul::FLOAT, self.dtype), - ("bdiv", DType::F32) => (strided::div::FLOAT, self.dtype), - ("bminimum", DType::F32) => (strided::min::FLOAT, self.dtype), - ("bmaximum", DType::F32) => (strided::max::FLOAT, self.dtype), - ("eq", DType::F32) => (strided::eq::FLOAT, DType::U8), - ("ne", DType::F32) => (strided::ne::FLOAT, DType::U8), - ("le", DType::F32) => (strided::le::FLOAT, DType::U8), - ("lt", DType::F32) => (strided::lt::FLOAT, DType::U8), - ("ge", DType::F32) => (strided::ge::FLOAT, DType::U8), - ("gt", DType::F32) => (strided::gt::FLOAT, DType::U8), - - ("badd", DType::F16) => (strided::add::HALF, self.dtype), - ("bsub", DType::F16) => (strided::sub::HALF, self.dtype), - ("bmul", DType::F16) => (strided::mul::HALF, self.dtype), - ("bdiv", DType::F16) => (strided::div::HALF, self.dtype), - ("bminimum", DType::F16) => (strided::min::HALF, self.dtype), - ("bmaximum", DType::F16) => (strided::max::HALF, self.dtype), - ("eq", DType::F16) => (strided::eq::HALF, DType::U8), - ("ne", DType::F16) => (strided::ne::HALF, DType::U8), - ("le", DType::F16) => (strided::le::HALF, DType::U8), - ("lt", DType::F16) => (strided::lt::HALF, DType::U8), - ("ge", DType::F16) => (strided::ge::HALF, DType::U8), - ("gt", DType::F16) => (strided::gt::HALF, DType::U8), - - ("badd", DType::BF16) => (strided::add::BFLOAT, self.dtype), - ("bsub", DType::BF16) => (strided::sub::BFLOAT, self.dtype), - ("bmul", DType::BF16) => (strided::mul::BFLOAT, self.dtype), - ("bdiv", DType::BF16) => (strided::div::BFLOAT, self.dtype), - ("bminimum", DType::BF16) => (strided::min::BFLOAT, self.dtype), - ("bmaximum", DType::BF16) => (strided::max::BFLOAT, self.dtype), - ("eq", DType::BF16) => (strided::eq::BFLOAT, DType::U8), - ("ne", DType::BF16) => (strided::ne::BFLOAT, DType::U8), - ("le", DType::BF16) => (strided::le::BFLOAT, DType::U8), - ("lt", DType::BF16) => (strided::lt::BFLOAT, DType::U8), - ("ge", DType::BF16) => (strided::ge::BFLOAT, DType::U8), - ("gt", DType::BF16) => (strided::gt::BFLOAT, DType::U8), - - ("badd", DType::I64) => (strided::add::I64, self.dtype), - ("bsub", DType::I64) => (strided::sub::I64, self.dtype), - ("bmul", DType::I64) => (strided::mul::I64, self.dtype), - ("bdiv", DType::I64) => (strided::div::I64, self.dtype), - ("bminimum", DType::I64) => (strided::min::I64, self.dtype), - ("bmaximum", DType::I64) => (strided::max::I64, self.dtype), - ("eq", DType::I64) => (strided::eq::I64, DType::U8), - ("ne", DType::I64) => (strided::ne::I64, DType::U8), - ("le", DType::I64) => (strided::le::I64, DType::U8), - ("lt", DType::I64) => (strided::lt::I64, DType::U8), - ("ge", DType::I64) => (strided::ge::I64, DType::U8), - ("gt", DType::I64) => (strided::gt::I64, DType::U8), - - ("badd", DType::U32) => (strided::add::U32, self.dtype), - ("bsub", DType::U32) => (strided::sub::U32, self.dtype), - ("bmul", DType::U32) => (strided::mul::U32, self.dtype), - ("bdiv", DType::U32) => (strided::div::U32, self.dtype), - ("bminimum", DType::U32) => (strided::min::U32, self.dtype), - ("bmaximum", DType::U32) => (strided::max::U32, self.dtype), - ("eq", DType::U32) => (strided::eq::U32, DType::U8), - ("ne", DType::U32) => (strided::ne::U32, DType::U8), - ("le", DType::U32) => (strided::le::U32, DType::U8), - ("lt", DType::U32) => (strided::lt::U32, DType::U8), - ("ge", DType::U32) => (strided::ge::U32, DType::U8), - ("gt", DType::U32) => (strided::gt::U32, DType::U8), - - ("badd", DType::U8) => (strided::add::U8, self.dtype), - ("bsub", DType::U8) => (strided::sub::U8, self.dtype), - ("bmul", DType::U8) => (strided::mul::U8, self.dtype), - ("bdiv", DType::U8) => (strided::div::U8, self.dtype), - ("bminimum", DType::U8) => (strided::min::U8, self.dtype), - ("bmaximum", DType::U8) => (strided::max::U8, self.dtype), - ("eq", DType::U8) => (strided::eq::U8, DType::U8), - ("ne", DType::U8) => (strided::ne::U8, DType::U8), - ("le", DType::U8) => (strided::le::U8, DType::U8), - ("lt", DType::U8) => (strided::lt::U8, DType::U8), - ("ge", DType::U8) => (strided::ge::U8, DType::U8), - ("gt", DType::U8) => (strided::gt::U8, DType::U8), - - (name, dtype) => { - crate::bail!("Metal strided binary {name} {dtype:?} not implemented") - } - }; - let buffer = device.new_buffer(el_count, dtype, op)?; - candle_metal_kernels::call_binary_strided( - &device.device, - &command_buffer, - &device.kernels, - kernel_name, - lhs_l.dims(), - &self.buffer, - lhs_l.stride(), - lhs_l.start_offset() * self.dtype.size_in_bytes(), - &rhs.buffer, - rhs_l.stride(), - rhs_l.start_offset() * rhs.dtype.size_in_bytes(), - &buffer, - ) - .map_err(MetalError::from)?; - (buffer, dtype) - }; - command_buffer.set_label("binary"); - Ok(Self::new(buffer, device.clone(), dtype)) - } - - pub(crate) fn to_cpu(&self) -> Result> { - let length = self.buffer.length() as usize; - let size = self.dtype.size_in_bytes(); - if length % size != 0 { - crate::bail!( - "The Metal buffer length is not aligned with dtype {:?}", - self.dtype - ); - } - let buffer = self.device.new_buffer_managed(self.buffer.length())?; - { - let command_buffer = self.device.command_buffer()?; - command_buffer.set_label("to_cpu"); - let blit = command_buffer.new_blit_command_encoder(); - blit.set_label("blit_to_cpu"); - blit.copy_from_buffer(&self.buffer, 0, &buffer, 0, self.buffer.length()); - blit.end_encoding(); - } - self.device.wait_until_completed()?; - Ok(read_to_vec(&buffer, length / size)) - } -} - -impl BackendDevice for MetalDevice { - type Storage = MetalStorage; - - fn new(ordinal: usize) -> Result { - let device = metal::Device::all().swap_remove(ordinal); - let command_queue = device.new_command_queue(); - let command_buffer = command_queue.new_command_buffer().to_owned(); - command_buffer.enqueue(); - let command_buffer = Arc::new(RwLock::new(command_buffer)); - let command_buffer_index = Arc::new(RwLock::new(0)); - let kernels = Arc::new(Kernels::new()); - let buffers = Arc::new(RwLock::new(HashMap::new())); - let compute_per_buffer = match std::env::var("CANDLE_METAL_COMPUTE_PER_BUFFER") { - Ok(val) => val.parse()?, - _ => 10, - }; - let seed = Arc::new(Mutex::new(device.new_buffer_with_data( - [299792458].as_ptr() as *const c_void, - 4, - MTLResourceOptions::StorageModeManaged, - ))); - Ok(Self { - device, - command_queue, - command_buffer, - command_buffer_index, - compute_per_buffer, - buffers, - kernels, - seed, - }) - } - - fn location(&self) -> crate::DeviceLocation { - crate::DeviceLocation::Metal { - gpu_id: self.registry_id() as usize, - } - } - - fn same_device(&self, rhs: &Self) -> bool { - self.device.registry_id() == rhs.device.registry_id() - } - - fn zeros_impl(&self, shape: &Shape, dtype: DType) -> Result { - let size = shape.elem_count() * dtype.size_in_bytes(); - let buffer = self.allocate_zeros(size)?; - Ok(MetalStorage::new(buffer, self.clone(), dtype)) - } - - fn ones_impl(&self, shape: &Shape, dtype: DType) -> Result { - // TODO Is there a faster way ? - let cpu_storage = crate::cpu_backend::CpuDevice.ones_impl(shape, dtype)?; - self.storage_from_cpu_storage(&cpu_storage) - } - - fn storage_from_cpu_storage(&self, storage: &CpuStorage) -> Result { - let buffer = match storage { - CpuStorage::U8(storage) => self.new_buffer_with_data(storage), - CpuStorage::U32(storage) => self.new_buffer_with_data(storage), - CpuStorage::I64(storage) => self.new_buffer_with_data(storage), - CpuStorage::BF16(storage) => self.new_buffer_with_data(storage), - CpuStorage::F16(storage) => self.new_buffer_with_data(storage), - CpuStorage::F32(storage) => self.new_buffer_with_data(storage), - CpuStorage::F64(storage) => self.new_buffer_with_data(storage), - }?; - Ok(Self::Storage::new(buffer, self.clone(), storage.dtype())) - } - - fn rand_uniform( - &self, - shape: &Shape, - dtype: DType, - min: f64, - max: f64, - ) -> Result { - let name = match dtype { - DType::F32 => "rand_uniform_f32", - DType::F16 => "rand_uniform_f16", - DType::BF16 => "rand_uniform_bf16", - dtype => crate::bail!("rand_uniform not implemented for {dtype:?}"), - }; - let buffer = self.new_buffer(shape.elem_count(), dtype, "rand_uniform")?; - let command_buffer = self.command_buffer()?; - candle_metal_kernels::call_random_uniform( - &self.device, - &command_buffer, - &self.kernels, - name, - min as f32, - max as f32, - shape.elem_count(), - &self.seed.lock().unwrap(), - &buffer, - ) - .map_err(MetalError::from)?; - - Ok(Self::Storage::new(buffer, self.clone(), dtype)) - } - - fn rand_normal( - &self, - shape: &Shape, - dtype: DType, - mean: f64, - stddev: f64, - ) -> Result { - let name = match dtype { - DType::F32 => "rand_normal_f32", - DType::F16 => "rand_normal_f16", - DType::BF16 => "rand_normal_bf16", - dtype => crate::bail!("rand_uniform not implemented for {dtype:?}"), - }; - let buffer = self.new_buffer(shape.elem_count(), dtype, "rand_normal")?; - let command_buffer = self.command_buffer()?; - candle_metal_kernels::call_random_normal( - &self.device, - &command_buffer, - &self.kernels, - name, - mean as f32, - stddev as f32, - shape.elem_count(), - &self.seed.lock().unwrap(), - &buffer, - ) - .map_err(MetalError::from)?; - - Ok(Self::Storage::new(buffer, self.clone(), dtype)) - } - - fn set_seed(&self, seed: u64) -> Result<()> { - let seed: u32 = seed.try_into().map_err(|_| { - MetalError::Message("Metal seed must be less than or equal to u32::MAX".to_string()) - })?; - - let seed_buffer = self.seed.try_lock().map_err(MetalError::from)?; - let contents = seed_buffer.contents(); - unsafe { - std::ptr::copy([seed].as_ptr(), contents as *mut u32, 4); - } - seed_buffer.did_modify_range(metal::NSRange::new(0, 4)); - - Ok(()) - } -} - -fn read_to_vec(buffer: &Buffer, n: usize) -> Vec { - let ptr = buffer.contents() as *const T; - assert!(!ptr.is_null()); - let slice = unsafe { std::slice::from_raw_parts(ptr, n) }; - slice.to_vec() -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/mkl.rs b/v2/dependency-patches/candle-core-0.4.1/src/mkl.rs deleted file mode 100644 index 359add741c..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/mkl.rs +++ /dev/null @@ -1,418 +0,0 @@ -#![allow(dead_code)] -use libc::{c_char, c_double, c_float, c_int}; - -mod ffi { - use super::*; - extern "C" { - pub fn vsTanh(n: c_int, a: *const c_float, y: *mut c_float); - pub fn vdTanh(n: c_int, a: *const c_double, y: *mut c_double); - pub fn vsExp(n: c_int, a: *const c_float, y: *mut c_float); - pub fn vdExp(n: c_int, a: *const c_double, y: *mut c_double); - pub fn vsLn(n: c_int, a: *const c_float, y: *mut c_float); - pub fn vdLn(n: c_int, a: *const c_double, y: *mut c_double); - pub fn vsSin(n: c_int, a: *const c_float, y: *mut c_float); - pub fn vdSin(n: c_int, a: *const c_double, y: *mut c_double); - pub fn vsCos(n: c_int, a: *const c_float, y: *mut c_float); - pub fn vdCos(n: c_int, a: *const c_double, y: *mut c_double); - pub fn vsSqrt(n: c_int, a: *const c_float, y: *mut c_float); - pub fn vdSqrt(n: c_int, a: *const c_double, y: *mut c_double); - - pub fn vsAdd(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); - pub fn vdAdd(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); - pub fn vsSub(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); - pub fn vdSub(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); - pub fn vsMul(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); - pub fn vdMul(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); - pub fn vsDiv(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); - pub fn vdDiv(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); - pub fn vsFmax(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); - pub fn vdFmax(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); - pub fn vsFmin(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); - pub fn vdFmin(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); - - pub fn sgemm_( - transa: *const c_char, - transb: *const c_char, - m: *const c_int, - n: *const c_int, - k: *const c_int, - alpha: *const c_float, - a: *const c_float, - lda: *const c_int, - b: *const c_float, - ldb: *const c_int, - beta: *const c_float, - c: *mut c_float, - ldc: *const c_int, - ); - pub fn dgemm_( - transa: *const c_char, - transb: *const c_char, - m: *const c_int, - n: *const c_int, - k: *const c_int, - alpha: *const c_double, - a: *const c_double, - lda: *const c_int, - b: *const c_double, - ldb: *const c_int, - beta: *const c_double, - c: *mut c_double, - ldc: *const c_int, - ); - pub fn hgemm_( - transa: *const c_char, - transb: *const c_char, - m: *const c_int, - n: *const c_int, - k: *const c_int, - alpha: *const half::f16, - a: *const half::f16, - lda: *const c_int, - b: *const half::f16, - ldb: *const c_int, - beta: *const half::f16, - c: *mut half::f16, - ldc: *const c_int, - ); - } -} - -#[allow(clippy::too_many_arguments)] -#[inline] -pub unsafe fn sgemm( - transa: u8, - transb: u8, - m: i32, - n: i32, - k: i32, - alpha: f32, - a: &[f32], - lda: i32, - b: &[f32], - ldb: i32, - beta: f32, - c: &mut [f32], - ldc: i32, -) { - ffi::sgemm_( - &(transa as c_char), - &(transb as c_char), - &m, - &n, - &k, - &alpha, - a.as_ptr(), - &lda, - b.as_ptr(), - &ldb, - &beta, - c.as_mut_ptr(), - &ldc, - ) -} - -#[allow(clippy::too_many_arguments)] -#[inline] -pub unsafe fn dgemm( - transa: u8, - transb: u8, - m: i32, - n: i32, - k: i32, - alpha: f64, - a: &[f64], - lda: i32, - b: &[f64], - ldb: i32, - beta: f64, - c: &mut [f64], - ldc: i32, -) { - ffi::dgemm_( - &(transa as c_char), - &(transb as c_char), - &m, - &n, - &k, - &alpha, - a.as_ptr(), - &lda, - b.as_ptr(), - &ldb, - &beta, - c.as_mut_ptr(), - &ldc, - ) -} - -#[allow(clippy::too_many_arguments)] -#[inline] -pub unsafe fn hgemm( - transa: u8, - transb: u8, - m: i32, - n: i32, - k: i32, - alpha: half::f16, - a: &[half::f16], - lda: i32, - b: &[half::f16], - ldb: i32, - beta: half::f16, - c: &mut [half::f16], - ldc: i32, -) { - ffi::hgemm_( - &(transa as c_char), - &(transb as c_char), - &m, - &n, - &k, - &alpha, - a.as_ptr(), - &lda, - b.as_ptr(), - &ldb, - &beta, - c.as_mut_ptr(), - &ldc, - ) -} - -#[inline] -pub fn vs_exp(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vsExp(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vd_exp(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vdExp(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vs_ln(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vsLn(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vd_ln(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vdLn(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vs_sin(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vsSin(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vd_sin(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vdSin(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vs_cos(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vsCos(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vd_cos(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vdCos(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vs_sqrt(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vsSqrt(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vd_sqrt(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vdSqrt(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vs_sqr(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vsMul(a_len as i32, a.as_ptr(), a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vd_sqr(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vdMul(a_len as i32, a.as_ptr(), a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vs_tanh(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vsTanh(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vd_tanh(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vdTanh(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -// The vector functions from mkl can be performed in place by using the same array for input and -// output. -// https://www.intel.com/content/www/us/en/docs/onemkl/developer-reference-c/2023-2/vector-mathematical-functions.html -#[inline] -pub fn vs_tanh_inplace(y: &mut [f32]) { - unsafe { ffi::vsTanh(y.len() as i32, y.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vd_tanh_inplace(y: &mut [f64]) { - unsafe { ffi::vdTanh(y.len() as i32, y.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vs_exp_inplace(y: &mut [f32]) { - unsafe { ffi::vsExp(y.len() as i32, y.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vd_exp_inplace(y: &mut [f64]) { - unsafe { ffi::vdExp(y.len() as i32, y.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vs_gelu(vs: &[f32], ys: &mut [f32]) { - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = (2.0f32 / std::f32::consts::PI).sqrt() * v * (1.0 + 0.044715 * v * v) - } - vs_tanh_inplace(ys); - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = 0.5 * v * (1.0 + *y) - } -} - -#[inline] -pub fn vd_gelu(vs: &[f64], ys: &mut [f64]) { - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = (2.0f64 / std::f64::consts::PI).sqrt() * v * (1.0 + 0.044715 * v * v) - } - vd_tanh_inplace(ys); - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = 0.5 * v * (1.0 + *y) - } -} - -#[inline] -pub fn vs_silu(vs: &[f32], ys: &mut [f32]) { - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = -v - } - vs_exp_inplace(ys); - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = v / (1.0 + *y) - } -} - -#[inline] -pub fn vd_silu(vs: &[f64], ys: &mut [f64]) { - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = -v - } - vd_exp_inplace(ys); - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = v / (1.0 + *y) - } -} - -macro_rules! binary_op { - ($fn_name:ident, $ty:ty, $mkl_name:ident) => { - #[inline] - pub fn $fn_name(a: &[$ty], b: &[$ty], y: &mut [$ty]) { - let a_len = a.len(); - let b_len = b.len(); - let y_len = y.len(); - if a_len != y_len || b_len != y_len { - panic!( - "{} a,b,y len mismatch {a_len} {b_len} {y_len}", - stringify!($fn_name) - ); - } - unsafe { ffi::$mkl_name(a_len as i32, a.as_ptr(), b.as_ptr(), y.as_mut_ptr()) } - } - }; -} -binary_op!(vs_add, f32, vsAdd); -binary_op!(vd_add, f64, vdAdd); -binary_op!(vs_sub, f32, vsSub); -binary_op!(vd_sub, f64, vdSub); -binary_op!(vs_mul, f32, vsMul); -binary_op!(vd_mul, f64, vdMul); -binary_op!(vs_div, f32, vsDiv); -binary_op!(vd_div, f64, vdDiv); -binary_op!(vs_max, f32, vsFmax); -binary_op!(vd_max, f64, vdFmax); -binary_op!(vs_min, f32, vsFmin); -binary_op!(vd_min, f64, vdFmin); diff --git a/v2/dependency-patches/candle-core-0.4.1/src/npy.rs b/v2/dependency-patches/candle-core-0.4.1/src/npy.rs deleted file mode 100644 index b07c48e9fe..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/npy.rs +++ /dev/null @@ -1,450 +0,0 @@ -//! Numpy support for tensors. -//! -//! The spec for the npy format can be found in -//! [npy-format](https://docs.scipy.org/doc/numpy-1.14.2/neps/npy-format.html). -//! The functions from this module can be used to read tensors from npy/npz files -//! or write tensors to these files. A npy file contains a single tensor (unnamed) -//! whereas a npz file can contain multiple named tensors. npz files are also compressed. -//! -//! These two formats are easy to use in Python using the numpy library. -//! -//! ```python -//! import numpy as np -//! x = np.arange(10) -//! -//! # Write a npy file. -//! np.save("test.npy", x) -//! -//! # Read a value from the npy file. -//! x = np.load("test.npy") -//! -//! # Write multiple values to a npz file. -//! values = { "x": x, "x_plus_one": x + 1 } -//! np.savez("test.npz", **values) -//! -//! # Load multiple values from a npz file. -//! values = np.loadz("test.npz") -//! ``` -use crate::{DType, Device, Error, Result, Shape, Tensor}; -use byteorder::{LittleEndian, ReadBytesExt}; -use half::{bf16, f16, slice::HalfFloatSliceExt}; -use std::collections::HashMap; -use std::fs::File; -use std::io::{BufReader, Read, Write}; -use std::path::Path; - -const NPY_MAGIC_STRING: &[u8] = b"\x93NUMPY"; -const NPY_SUFFIX: &str = ".npy"; - -fn read_header(reader: &mut R) -> Result { - let mut magic_string = vec![0u8; NPY_MAGIC_STRING.len()]; - reader.read_exact(&mut magic_string)?; - if magic_string != NPY_MAGIC_STRING { - return Err(Error::Npy("magic string mismatch".to_string())); - } - let mut version = [0u8; 2]; - reader.read_exact(&mut version)?; - let header_len_len = match version[0] { - 1 => 2, - 2 => 4, - otherwise => return Err(Error::Npy(format!("unsupported version {otherwise}"))), - }; - let mut header_len = vec![0u8; header_len_len]; - reader.read_exact(&mut header_len)?; - let header_len = header_len - .iter() - .rev() - .fold(0_usize, |acc, &v| 256 * acc + v as usize); - let mut header = vec![0u8; header_len]; - reader.read_exact(&mut header)?; - Ok(String::from_utf8_lossy(&header).to_string()) -} - -#[derive(Debug, PartialEq)] -struct Header { - descr: DType, - fortran_order: bool, - shape: Vec, -} - -impl Header { - fn shape(&self) -> Shape { - Shape::from(self.shape.as_slice()) - } - - fn to_string(&self) -> Result { - let fortran_order = if self.fortran_order { "True" } else { "False" }; - let mut shape = self - .shape - .iter() - .map(|x| x.to_string()) - .collect::>() - .join(","); - let descr = match self.descr { - DType::BF16 => Err(Error::Npy("bf16 is not supported".into()))?, - DType::F16 => "f2", - DType::F32 => "f4", - DType::F64 => "f8", - DType::I64 => "i8", - DType::U32 => "u4", - DType::U8 => "u1", - }; - if !shape.is_empty() { - shape.push(',') - } - Ok(format!( - "{{'descr': '<{descr}', 'fortran_order': {fortran_order}, 'shape': ({shape}), }}" - )) - } - - // Hacky parser for the npy header, a typical example would be: - // {'descr': ' Result
{ - let header = - header.trim_matches(|c: char| c == '{' || c == '}' || c == ',' || c.is_whitespace()); - - let mut parts: Vec = vec![]; - let mut start_index = 0usize; - let mut cnt_parenthesis = 0i64; - for (index, c) in header.chars().enumerate() { - match c { - '(' => cnt_parenthesis += 1, - ')' => cnt_parenthesis -= 1, - ',' => { - if cnt_parenthesis == 0 { - parts.push(header[start_index..index].to_owned()); - start_index = index + 1; - } - } - _ => {} - } - } - parts.push(header[start_index..].to_owned()); - let mut part_map: HashMap = HashMap::new(); - for part in parts.iter() { - let part = part.trim(); - if !part.is_empty() { - match part.split(':').collect::>().as_slice() { - [key, value] => { - let key = key.trim_matches(|c: char| c == '\'' || c.is_whitespace()); - let value = value.trim_matches(|c: char| c == '\'' || c.is_whitespace()); - let _ = part_map.insert(key.to_owned(), value.to_owned()); - } - _ => return Err(Error::Npy(format!("unable to parse header {header}"))), - } - } - } - let fortran_order = match part_map.get("fortran_order") { - None => false, - Some(fortran_order) => match fortran_order.as_ref() { - "False" => false, - "True" => true, - _ => return Err(Error::Npy(format!("unknown fortran_order {fortran_order}"))), - }, - }; - let descr = match part_map.get("descr") { - None => return Err(Error::Npy("no descr in header".to_string())), - Some(descr) => { - if descr.is_empty() { - return Err(Error::Npy("empty descr".to_string())); - } - if descr.starts_with('>') { - return Err(Error::Npy(format!("little-endian descr {descr}"))); - } - // the only supported types in tensor are: - // float64, float32, float16, - // complex64, complex128, - // int64, int32, int16, int8, - // uint8, and bool. - match descr.trim_matches(|c: char| c == '=' || c == '<' || c == '|') { - "e" | "f2" => DType::F16, - "f" | "f4" => DType::F32, - "d" | "f8" => DType::F64, - // "i" | "i4" => DType::S32, - "q" | "i8" => DType::I64, - // "h" | "i2" => DType::S16, - // "b" | "i1" => DType::S8, - "B" | "u1" => DType::U8, - "I" | "u4" => DType::U32, - "?" | "b1" => DType::U8, - // "F" | "F4" => DType::C64, - // "D" | "F8" => DType::C128, - descr => return Err(Error::Npy(format!("unrecognized descr {descr}"))), - } - } - }; - let shape = match part_map.get("shape") { - None => return Err(Error::Npy("no shape in header".to_string())), - Some(shape) => { - let shape = shape.trim_matches(|c: char| c == '(' || c == ')' || c == ','); - if shape.is_empty() { - vec![] - } else { - shape - .split(',') - .map(|v| v.trim().parse::()) - .collect::, _>>()? - } - } - }; - Ok(Header { - descr, - fortran_order, - shape, - }) - } -} - -impl Tensor { - // TODO: Add the possibility to read directly to a device? - pub(crate) fn from_reader( - shape: Shape, - dtype: DType, - reader: &mut R, - ) -> Result { - let elem_count = shape.elem_count(); - match dtype { - DType::BF16 => { - let mut data_t = vec![bf16::ZERO; elem_count]; - reader.read_u16_into::(data_t.reinterpret_cast_mut())?; - Tensor::from_vec(data_t, shape, &Device::Cpu) - } - DType::F16 => { - let mut data_t = vec![f16::ZERO; elem_count]; - reader.read_u16_into::(data_t.reinterpret_cast_mut())?; - Tensor::from_vec(data_t, shape, &Device::Cpu) - } - DType::F32 => { - let mut data_t = vec![0f32; elem_count]; - reader.read_f32_into::(&mut data_t)?; - Tensor::from_vec(data_t, shape, &Device::Cpu) - } - DType::F64 => { - let mut data_t = vec![0f64; elem_count]; - reader.read_f64_into::(&mut data_t)?; - Tensor::from_vec(data_t, shape, &Device::Cpu) - } - DType::U8 => { - let mut data_t = vec![0u8; elem_count]; - reader.read_exact(&mut data_t)?; - Tensor::from_vec(data_t, shape, &Device::Cpu) - } - DType::U32 => { - let mut data_t = vec![0u32; elem_count]; - reader.read_u32_into::(&mut data_t)?; - Tensor::from_vec(data_t, shape, &Device::Cpu) - } - DType::I64 => { - let mut data_t = vec![0i64; elem_count]; - reader.read_i64_into::(&mut data_t)?; - Tensor::from_vec(data_t, shape, &Device::Cpu) - } - } - } - - /// Reads a npy file and return the stored multi-dimensional array as a tensor. - pub fn read_npy>(path: T) -> Result { - let mut reader = File::open(path.as_ref())?; - let header = read_header(&mut reader)?; - let header = Header::parse(&header)?; - if header.fortran_order { - return Err(Error::Npy("fortran order not supported".to_string())); - } - Self::from_reader(header.shape(), header.descr, &mut reader) - } - - /// Reads a npz file and returns the stored multi-dimensional arrays together with their names. - pub fn read_npz>(path: T) -> Result> { - let zip_reader = BufReader::new(File::open(path.as_ref())?); - let mut zip = zip::ZipArchive::new(zip_reader)?; - let mut result = vec![]; - for i in 0..zip.len() { - let mut reader = zip.by_index(i)?; - let name = { - let name = reader.name()?; - name.strip_suffix(NPY_SUFFIX).unwrap_or(&name).to_owned() - }; - let header = read_header(&mut reader)?; - let header = Header::parse(&header)?; - if header.fortran_order { - return Err(Error::Npy("fortran order not supported".to_string())); - } - let s = Self::from_reader(header.shape(), header.descr, &mut reader)?; - result.push((name, s)) - } - Ok(result) - } - - /// Reads a npz file and returns the stored multi-dimensional arrays for some specified names. - pub fn read_npz_by_name>(path: T, names: &[&str]) -> Result> { - let zip_reader = BufReader::new(File::open(path.as_ref())?); - let mut zip = zip::ZipArchive::new(zip_reader)?; - let mut result = vec![]; - for name in names.iter() { - let mut reader = match zip.by_name(&format!("{name}{NPY_SUFFIX}")) { - Ok(reader) => reader, - Err(_) => Err(Error::Npy(format!( - "no array for {name} in {:?}", - path.as_ref() - )))?, - }; - let header = read_header(&mut reader)?; - let header = Header::parse(&header)?; - if header.fortran_order { - return Err(Error::Npy("fortran order not supported".to_string())); - } - let s = Self::from_reader(header.shape(), header.descr, &mut reader)?; - result.push(s) - } - Ok(result) - } - - fn write(&self, f: &mut T) -> Result<()> { - f.write_all(NPY_MAGIC_STRING)?; - f.write_all(&[1u8, 0u8])?; - let header = Header { - descr: self.dtype(), - fortran_order: false, - shape: self.dims().to_vec(), - }; - let mut header = header.to_string()?; - let pad = 16 - (NPY_MAGIC_STRING.len() + 5 + header.len()) % 16; - for _ in 0..pad % 16 { - header.push(' ') - } - header.push('\n'); - f.write_all(&[(header.len() % 256) as u8, (header.len() / 256) as u8])?; - f.write_all(header.as_bytes())?; - self.write_bytes(f) - } - - /// Writes a multi-dimensional array in the npy format. - pub fn write_npy>(&self, path: T) -> Result<()> { - let mut f = File::create(path.as_ref())?; - self.write(&mut f) - } - - /// Writes multiple multi-dimensional arrays using the npz format. - pub fn write_npz, T: AsRef, P: AsRef>( - ts: &[(S, T)], - path: P, - ) -> Result<()> { - let mut zip = zip::ZipWriter::new(File::create(path.as_ref())?); - let options = - zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); - - for (name, tensor) in ts.iter() { - zip.start_file(format!("{}.npy", name.as_ref()), options)?; - tensor.as_ref().write(&mut zip)? - } - Ok(()) - } -} - -/// Lazy tensor loader. -pub struct NpzTensors { - index_per_name: HashMap, - path: std::path::PathBuf, - // We do not store a zip reader as it needs mutable access to extract data. Instead we - // re-create a zip reader for each tensor. -} - -impl NpzTensors { - pub fn new>(path: T) -> Result { - let path = path.as_ref().to_owned(); - let zip_reader = BufReader::new(File::open(&path)?); - let mut zip = zip::ZipArchive::new(zip_reader)?; - let mut index_per_name = HashMap::new(); - for i in 0..zip.len() { - let file = zip.by_index(i)?; - let name = { - let name = file.name()?; - name.strip_suffix(NPY_SUFFIX).unwrap_or(&name).to_owned() - }; - index_per_name.insert(name, i); - } - Ok(Self { - index_per_name, - path, - }) - } - - pub fn names(&self) -> Vec<&String> { - self.index_per_name.keys().collect() - } - - /// This only returns the shape and dtype for a named tensor. Compared to `get`, this avoids - /// reading the whole tensor data. - pub fn get_shape_and_dtype(&self, name: &str) -> Result<(Shape, DType)> { - let index = match self.index_per_name.get(name) { - None => crate::bail!("cannot find tensor {name}"), - Some(index) => *index, - }; - let zip_reader = BufReader::new(File::open(&self.path)?); - let mut zip = zip::ZipArchive::new(zip_reader)?; - let mut reader = zip.by_index(index)?; - let header = read_header(&mut reader)?; - let header = Header::parse(&header)?; - Ok((header.shape(), header.descr)) - } - - pub fn get(&self, name: &str) -> Result> { - let index = match self.index_per_name.get(name) { - None => return Ok(None), - Some(index) => *index, - }; - // We hope that the file has not changed since first reading it. - let zip_reader = BufReader::new(File::open(&self.path)?); - let mut zip = zip::ZipArchive::new(zip_reader)?; - let mut reader = zip.by_index(index)?; - let header = read_header(&mut reader)?; - let header = Header::parse(&header)?; - if header.fortran_order { - return Err(Error::Npy("fortran order not supported".to_string())); - } - let tensor = Tensor::from_reader(header.shape(), header.descr, &mut reader)?; - Ok(Some(tensor)) - } -} - -#[cfg(test)] -mod tests { - use super::Header; - - #[test] - fn parse() { - let h = "{'descr': ' &'static str { - match self { - Self::ArgMax => "argmax", - Self::ArgMin => "argmin", - Self::Min => "min", - Self::Max => "max", - Self::Sum => "sum", - } - } -} - -// These ops return the same type as their input type. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum BinaryOp { - Add, - Mul, - Sub, - Div, - Maximum, - Minimum, -} - -// Unary ops with no argument -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum UnaryOp { - Exp, - Log, - Sin, - Cos, - Abs, - Neg, - Recip, - Sqr, - Sqrt, - Gelu, - GeluErf, - Erf, - Relu, - Silu, - Tanh, - Floor, - Ceil, - Round, -} - -#[derive(Clone)] -pub enum Op { - Binary(Tensor, Tensor, BinaryOp), - Unary(Tensor, UnaryOp), - Cmp(Tensor, CmpOp), - // The third argument is the reduced shape with `keepdim=true`. - Reduce(Tensor, ReduceOp, Vec), - Matmul(Tensor, Tensor), - Gather(Tensor, Tensor, usize), - ScatterAdd(Tensor, Tensor, Tensor, usize), - IndexSelect(Tensor, Tensor, usize), - IndexAdd(Tensor, Tensor, Tensor, usize), - WhereCond(Tensor, Tensor, Tensor), - - #[allow(dead_code)] - Conv1D { - arg: Tensor, - kernel: Tensor, - padding: usize, - stride: usize, - dilation: usize, - }, - - #[allow(dead_code)] - ConvTranspose1D { - arg: Tensor, - kernel: Tensor, - padding: usize, - output_padding: usize, - stride: usize, - dilation: usize, - }, - - #[allow(dead_code)] - Conv2D { - arg: Tensor, - kernel: Tensor, - padding: usize, - stride: usize, - dilation: usize, - }, - - #[allow(dead_code)] - ConvTranspose2D { - arg: Tensor, - kernel: Tensor, - padding: usize, - output_padding: usize, - stride: usize, - dilation: usize, - }, - - AvgPool2D { - arg: Tensor, - kernel_size: (usize, usize), - stride: (usize, usize), - }, - - MaxPool2D { - arg: Tensor, - kernel_size: (usize, usize), - stride: (usize, usize), - }, - - UpsampleNearest1D { - arg: Tensor, - target_size: usize, - }, - UpsampleNearest2D { - arg: Tensor, - target_h: usize, - target_w: usize, - }, - - Cat(Vec, usize), - - #[allow(dead_code)] // add is currently unused. - Affine { - arg: Tensor, - mul: f64, - add: f64, - }, - ToDType(Tensor), - Copy(Tensor), - Broadcast(Tensor), - Narrow(Tensor, usize, usize, usize), - SliceScatter0(Tensor, Tensor, usize), - Reshape(Tensor), - ToDevice(Tensor), - Transpose(Tensor, usize, usize), - Permute(Tensor, Vec), - Elu(Tensor, f64), - Powf(Tensor, f64), - CustomOp1(Tensor, std::sync::Arc>), - CustomOp2( - Tensor, - Tensor, - std::sync::Arc>, - ), - CustomOp3( - Tensor, - Tensor, - Tensor, - std::sync::Arc>, - ), -} - -/// Unary ops that can be defined in user-land. -pub trait CustomOp1 { - // Box does not support const yet, so use a function to get the name. - fn name(&self) -> &'static str; - - /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn cpu_fwd(&self, storage: &CpuStorage, layout: &Layout) -> Result<(CpuStorage, Shape)>; - - /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn cuda_fwd(&self, _storage: &CudaStorage, _layout: &Layout) -> Result<(CudaStorage, Shape)> { - Err(crate::Error::Cuda( - format!("no cuda implementation for {}", self.name()).into(), - )) - } - - /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn metal_fwd( - &self, - _storage: &MetalStorage, - _layout: &Layout, - ) -> Result<(MetalStorage, Shape)> { - Err(crate::Error::Metal( - format!("no metal implementation for {}", self.name()).into(), - )) - } - - /// This function takes as argument the argument `arg` used in the forward pass, the result - /// produced by the forward operation `res` and the gradient of the result `grad_res`. - /// The function should return the gradient of the argument. - fn bwd(&self, _arg: &Tensor, _res: &Tensor, _grad_res: &Tensor) -> Result> { - Err(crate::Error::BackwardNotSupported { op: self.name() }) - } -} - -pub trait CustomOp2 { - fn name(&self) -> &'static str; - - /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn cpu_fwd( - &self, - s1: &CpuStorage, - l1: &Layout, - s2: &CpuStorage, - l2: &Layout, - ) -> Result<(CpuStorage, Shape)>; - - /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn cuda_fwd( - &self, - _: &CudaStorage, - _: &Layout, - _: &CudaStorage, - _: &Layout, - ) -> Result<(CudaStorage, Shape)> { - Err(crate::Error::Cuda( - format!("no cuda implementation for {}", self.name()).into(), - )) - } - - /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn metal_fwd( - &self, - _: &MetalStorage, - _: &Layout, - _: &MetalStorage, - _: &Layout, - ) -> Result<(MetalStorage, Shape)> { - Err(crate::Error::Metal( - format!("no metal implementation for {}", self.name()).into(), - )) - } - - fn bwd( - &self, - _arg1: &Tensor, - _arg2: &Tensor, - _res: &Tensor, - _grad_res: &Tensor, - ) -> Result<(Option, Option)> { - Err(crate::Error::BackwardNotSupported { op: self.name() }) - } -} - -pub trait CustomOp3 { - fn name(&self) -> &'static str; - - /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn cpu_fwd( - &self, - s1: &CpuStorage, - l1: &Layout, - s2: &CpuStorage, - l2: &Layout, - s3: &CpuStorage, - l3: &Layout, - ) -> Result<(CpuStorage, Shape)>; - - /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn cuda_fwd( - &self, - _: &CudaStorage, - _: &Layout, - _: &CudaStorage, - _: &Layout, - _: &CudaStorage, - _: &Layout, - ) -> Result<(CudaStorage, Shape)> { - Err(crate::Error::Cuda( - format!("no cuda implementation for {}", self.name()).into(), - )) - } - - /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn metal_fwd( - &self, - _: &MetalStorage, - _: &Layout, - _: &MetalStorage, - _: &Layout, - _: &MetalStorage, - _: &Layout, - ) -> Result<(MetalStorage, Shape)> { - Err(crate::Error::Metal( - format!("no metal implementation for {}", self.name()).into(), - )) - } - - fn bwd( - &self, - _arg1: &Tensor, - _arg2: &Tensor, - _arg3: &Tensor, - _res: &Tensor, - _grad_res: &Tensor, - ) -> Result<(Option, Option, Option)> { - Err(crate::Error::BackwardNotSupported { op: self.name() }) - } -} - -pub trait UnaryOpT { - const NAME: &'static str; - const KERNEL: &'static str; - const V: Self; - fn bf16(v1: bf16) -> bf16; - fn f16(v1: f16) -> f16; - fn f32(v1: f32) -> f32; - fn f64(v1: f64) -> f64; - fn u8(v1: u8) -> u8; - fn u32(v1: u32) -> u32; - fn i64(v1: i64) -> i64; - - // There is no very good way to represent optional function in traits so we go for an explicit - // boolean flag to mark the function as existing. - const BF16_VEC: bool = false; - fn bf16_vec(_xs: &[bf16], _ys: &mut [bf16]) {} - const F16_VEC: bool = false; - fn f16_vec(_xs: &[f16], _ys: &mut [f16]) {} - const F32_VEC: bool = false; - fn f32_vec(_xs: &[f32], _ys: &mut [f32]) {} - const F64_VEC: bool = false; - fn f64_vec(_xs: &[f64], _ys: &mut [f64]) {} -} - -pub trait BinaryOpT { - const NAME: &'static str; - const KERNEL: &'static str; - const V: Self; - fn bf16(v1: bf16, v2: bf16) -> bf16; - fn f16(v1: f16, v2: f16) -> f16; - fn f32(v1: f32, v2: f32) -> f32; - fn f64(v1: f64, v2: f64) -> f64; - fn u8(v1: u8, v2: u8) -> u8; - fn u32(v1: u32, v2: u32) -> u32; - fn i64(v1: i64, v2: i64) -> i64; - - const BF16_VEC: bool = false; - fn bf16_vec(_xs1: &[bf16], _xs2: &[bf16], _ys: &mut [bf16]) {} - const F16_VEC: bool = false; - fn f16_vec(_xs1: &[f16], _xs2: &[f16], _ys: &mut [f16]) {} - const F32_VEC: bool = false; - fn f32_vec(_xs1: &[f32], _xs2: &[f32], _ys: &mut [f32]) {} - const F64_VEC: bool = false; - fn f64_vec(_xs1: &[f64], _xs2: &[f64], _ys: &mut [f64]) {} - const U8_VEC: bool = false; - fn u8_vec(_xs1: &[u8], _xs2: &[u8], _ys: &mut [u8]) {} - const U32_VEC: bool = false; - fn u32_vec(_xs1: &[u32], _xs2: &[u32], _ys: &mut [u32]) {} - const I64_VEC: bool = false; - fn i64_vec(_xs1: &[i64], _xs2: &[i64], _ys: &mut [i64]) {} -} - -pub(crate) struct Add; -pub(crate) struct Div; -pub(crate) struct Mul; -pub(crate) struct Sub; -pub(crate) struct Maximum; -pub(crate) struct Minimum; -pub(crate) struct Exp; -pub(crate) struct Log; -pub(crate) struct Sin; -pub(crate) struct Cos; -pub(crate) struct Abs; -pub(crate) struct Neg; -pub(crate) struct Recip; -pub(crate) struct Sqr; -pub(crate) struct Sqrt; -pub(crate) struct Gelu; -pub(crate) struct GeluErf; -pub(crate) struct Erf; -pub(crate) struct Relu; -pub(crate) struct Silu; -pub(crate) struct Tanh; -pub(crate) struct Floor; -pub(crate) struct Ceil; -pub(crate) struct Round; - -macro_rules! bin_op { - ($op:ident, $name: literal, $e: expr, $f32_vec: ident, $f64_vec: ident) => { - impl BinaryOpT for $op { - const NAME: &'static str = $name; - const KERNEL: &'static str = concat!("b", $name); - const V: Self = $op; - #[inline(always)] - fn bf16(v1: bf16, v2: bf16) -> bf16 { - $e(v1, v2) - } - #[inline(always)] - fn f16(v1: f16, v2: f16) -> f16 { - $e(v1, v2) - } - #[inline(always)] - fn f32(v1: f32, v2: f32) -> f32 { - $e(v1, v2) - } - #[inline(always)] - fn f64(v1: f64, v2: f64) -> f64 { - $e(v1, v2) - } - #[inline(always)] - fn u8(v1: u8, v2: u8) -> u8 { - $e(v1, v2) - } - #[inline(always)] - fn u32(v1: u32, v2: u32) -> u32 { - $e(v1, v2) - } - #[inline(always)] - fn i64(v1: i64, v2: i64) -> i64 { - $e(v1, v2) - } - - #[cfg(feature = "mkl")] - const F32_VEC: bool = true; - #[cfg(feature = "mkl")] - const F64_VEC: bool = true; - #[cfg(feature = "mkl")] - #[inline(always)] - fn f32_vec(xs1: &[f32], xs2: &[f32], ys: &mut [f32]) { - crate::mkl::$f32_vec(xs1, xs2, ys) - } - #[cfg(feature = "mkl")] - #[inline(always)] - fn f64_vec(xs1: &[f64], xs2: &[f64], ys: &mut [f64]) { - crate::mkl::$f64_vec(xs1, xs2, ys) - } - - #[cfg(feature = "accelerate")] - const F32_VEC: bool = true; - #[cfg(feature = "accelerate")] - const F64_VEC: bool = true; - #[cfg(feature = "accelerate")] - #[inline(always)] - fn f32_vec(xs1: &[f32], xs2: &[f32], ys: &mut [f32]) { - crate::accelerate::$f32_vec(xs1, xs2, ys) - } - #[cfg(feature = "accelerate")] - #[inline(always)] - fn f64_vec(xs1: &[f64], xs2: &[f64], ys: &mut [f64]) { - crate::accelerate::$f64_vec(xs1, xs2, ys) - } - } - }; -} - -bin_op!(Add, "add", |v1, v2| v1 + v2, vs_add, vd_add); -bin_op!(Sub, "sub", |v1, v2| v1 - v2, vs_sub, vd_sub); -bin_op!(Mul, "mul", |v1, v2| v1 * v2, vs_mul, vd_mul); -bin_op!(Div, "div", |v1, v2| v1 / v2, vs_div, vd_div); -bin_op!( - Minimum, - "minimum", - |v1, v2| if v1 > v2 { v2 } else { v1 }, - vs_min, - vd_min -); -bin_op!( - Maximum, - "maximum", - |v1, v2| if v1 < v2 { v2 } else { v1 }, - vs_max, - vd_max -); - -#[allow(clippy::redundant_closure_call)] -macro_rules! unary_op { - ($op: ident, $name: literal, $a: ident, $e: expr) => { - impl UnaryOpT for $op { - const NAME: &'static str = $name; - const KERNEL: &'static str = concat!("u", $name); - const V: Self = $op; - #[inline(always)] - fn bf16($a: bf16) -> bf16 { - $e - } - #[inline(always)] - fn f16($a: f16) -> f16 { - $e - } - #[inline(always)] - fn f32($a: f32) -> f32 { - $e - } - #[inline(always)] - fn f64($a: f64) -> f64 { - $e - } - #[inline(always)] - fn u8(_: u8) -> u8 { - todo!("no unary function for u8") - } - #[inline(always)] - fn u32(_: u32) -> u32 { - todo!("no unary function for u32") - } - #[inline(always)] - fn i64(_: i64) -> i64 { - todo!("no unary function for i64") - } - } - }; - - ($op: ident, $name: literal, $a: ident, $e: expr, $f32_vec:ident, $f64_vec:ident) => { - impl UnaryOpT for $op { - const NAME: &'static str = $name; - const KERNEL: &'static str = concat!("u", $name); - const V: Self = $op; - #[inline(always)] - fn bf16($a: bf16) -> bf16 { - $e - } - #[inline(always)] - fn f16($a: f16) -> f16 { - $e - } - #[inline(always)] - fn f32($a: f32) -> f32 { - $e - } - #[inline(always)] - fn f64($a: f64) -> f64 { - $e - } - #[inline(always)] - fn u8(_: u8) -> u8 { - todo!("no unary function for u8") - } - #[inline(always)] - fn u32(_: u32) -> u32 { - todo!("no unary function for u32") - } - #[inline(always)] - fn i64(_: i64) -> i64 { - todo!("no unary function for i64") - } - - #[cfg(feature = "mkl")] - const F32_VEC: bool = true; - #[cfg(feature = "mkl")] - const F64_VEC: bool = true; - #[cfg(feature = "mkl")] - #[inline(always)] - fn f32_vec(xs: &[f32], ys: &mut [f32]) { - crate::mkl::$f32_vec(xs, ys) - } - #[cfg(feature = "mkl")] - #[inline(always)] - fn f64_vec(xs: &[f64], ys: &mut [f64]) { - crate::mkl::$f64_vec(xs, ys) - } - - #[cfg(feature = "accelerate")] - const F32_VEC: bool = true; - #[cfg(feature = "accelerate")] - const F64_VEC: bool = true; - #[cfg(feature = "accelerate")] - #[inline(always)] - fn f32_vec(xs: &[f32], ys: &mut [f32]) { - crate::accelerate::$f32_vec(xs, ys) - } - #[cfg(feature = "accelerate")] - #[inline(always)] - fn f64_vec(xs: &[f64], ys: &mut [f64]) { - crate::accelerate::$f64_vec(xs, ys) - } - } - }; -} - -unary_op!(Exp, "exp", v, v.exp(), vs_exp, vd_exp); -unary_op!(Log, "log", v, v.ln(), vs_ln, vd_ln); -unary_op!(Sin, "sin", v, v.sin(), vs_sin, vd_sin); -unary_op!(Cos, "cos", v, v.cos(), vs_cos, vd_cos); -unary_op!(Tanh, "tanh", v, v.tanh(), vs_tanh, vd_tanh); -unary_op!(Neg, "neg", v, -v); -unary_op!(Recip, "recip", v, v.recip()); -unary_op!(Sqr, "sqr", v, v * v, vs_sqr, vd_sqr); -unary_op!(Sqrt, "sqrt", v, v.sqrt(), vs_sqrt, vd_sqrt); - -/// Tanh based approximation of the `gelu` operation -/// GeluErf is the more precise one. -/// -impl UnaryOpT for Gelu { - const NAME: &'static str = "gelu"; - const V: Self = Gelu; - #[inline(always)] - fn bf16(v: bf16) -> bf16 { - bf16::from_f32_const(0.5) - * v - * (bf16::ONE - + bf16::tanh( - (bf16::from_f32_const(2.0) / bf16::PI).sqrt() - * v - * (bf16::ONE + bf16::from_f32_const(0.044715) * v * v), - )) - } - #[inline(always)] - fn f16(v: f16) -> f16 { - f16::from_f32_const(0.5) - * v - * (f16::ONE - + f16::tanh( - (f16::from_f32_const(2.0) / f16::PI).sqrt() - * v - * (f16::ONE + f16::from_f32_const(0.044715) * v * v), - )) - } - #[inline(always)] - fn f32(v: f32) -> f32 { - 0.5 * v - * (1.0 - + f32::tanh((2.0f32 / std::f32::consts::PI).sqrt() * v * (1.0 + 0.044715 * v * v))) - } - #[inline(always)] - fn f64(v: f64) -> f64 { - 0.5 * v - * (1.0 - + f64::tanh((2.0f64 / std::f64::consts::PI).sqrt() * v * (1.0 + 0.044715 * v * v))) - } - #[inline(always)] - fn u8(_: u8) -> u8 { - 0 - } - #[inline(always)] - fn u32(_: u32) -> u32 { - 0 - } - #[inline(always)] - fn i64(_: i64) -> i64 { - 0 - } - const KERNEL: &'static str = "ugelu"; - - #[cfg(feature = "mkl")] - const F32_VEC: bool = true; - - #[cfg(feature = "mkl")] - #[inline(always)] - fn f32_vec(xs: &[f32], ys: &mut [f32]) { - crate::mkl::vs_gelu(xs, ys) - } - - #[cfg(feature = "mkl")] - const F64_VEC: bool = true; - - #[cfg(feature = "mkl")] - #[inline(always)] - fn f64_vec(xs: &[f64], ys: &mut [f64]) { - crate::mkl::vd_gelu(xs, ys) - } - - #[cfg(feature = "accelerate")] - const F32_VEC: bool = true; - - #[cfg(feature = "accelerate")] - #[inline(always)] - fn f32_vec(xs: &[f32], ys: &mut [f32]) { - crate::accelerate::vs_gelu(xs, ys) - } - - #[cfg(feature = "accelerate")] - const F64_VEC: bool = true; - - #[cfg(feature = "accelerate")] - #[inline(always)] - fn f64_vec(xs: &[f64], ys: &mut [f64]) { - crate::accelerate::vd_gelu(xs, ys) - } -} - -/// `erf` operation -/// -impl UnaryOpT for Erf { - const NAME: &'static str = "erf"; - const KERNEL: &'static str = "uerf"; - const V: Self = Erf; - #[inline(always)] - fn bf16(v: bf16) -> bf16 { - bf16::from_f64(Self::f64(v.to_f64())) - } - #[inline(always)] - fn f16(v: f16) -> f16 { - f16::from_f64(Self::f64(v.to_f64())) - } - #[inline(always)] - fn f32(v: f32) -> f32 { - Self::f64(v as f64) as f32 - } - #[inline(always)] - fn f64(v: f64) -> f64 { - crate::cpu::erf::erf(v) - } - #[inline(always)] - fn u8(_: u8) -> u8 { - 0 - } - #[inline(always)] - fn u32(_: u32) -> u32 { - 0 - } - #[inline(always)] - fn i64(_: i64) -> i64 { - 0 - } -} - -/// Silu operation -impl UnaryOpT for Silu { - const NAME: &'static str = "silu"; - const V: Self = Silu; - #[inline(always)] - fn bf16(v: bf16) -> bf16 { - v / (bf16::ONE + (-v).exp()) - } - #[inline(always)] - fn f16(v: f16) -> f16 { - v / (f16::ONE + (-v).exp()) - } - #[inline(always)] - fn f32(v: f32) -> f32 { - v / (1.0 + (-v).exp()) - } - #[inline(always)] - fn f64(v: f64) -> f64 { - v / (1.0 + (-v).exp()) - } - #[inline(always)] - fn u8(_: u8) -> u8 { - 0 - } - #[inline(always)] - fn u32(_: u32) -> u32 { - 0 - } - #[inline(always)] - fn i64(_: i64) -> i64 { - 0 - } - const KERNEL: &'static str = "usilu"; - - #[cfg(feature = "mkl")] - const F32_VEC: bool = true; - - #[cfg(feature = "mkl")] - #[inline(always)] - fn f32_vec(xs: &[f32], ys: &mut [f32]) { - crate::mkl::vs_silu(xs, ys) - } - - #[cfg(feature = "mkl")] - const F64_VEC: bool = true; - - #[cfg(feature = "mkl")] - #[inline(always)] - fn f64_vec(xs: &[f64], ys: &mut [f64]) { - crate::mkl::vd_silu(xs, ys) - } - - #[cfg(feature = "accelerate")] - const F32_VEC: bool = true; - - #[cfg(feature = "accelerate")] - #[inline(always)] - fn f32_vec(xs: &[f32], ys: &mut [f32]) { - crate::accelerate::vs_silu(xs, ys) - } - - #[cfg(feature = "accelerate")] - const F64_VEC: bool = true; - - #[cfg(feature = "accelerate")] - #[inline(always)] - fn f64_vec(xs: &[f64], ys: &mut [f64]) { - crate::accelerate::vd_silu(xs, ys) - } -} - -impl UnaryOpT for Abs { - const NAME: &'static str = "abs"; - const KERNEL: &'static str = "uabs"; - const V: Self = Abs; - #[inline(always)] - fn bf16(v: bf16) -> bf16 { - v.abs() - } - #[inline(always)] - fn f16(v: f16) -> f16 { - v.abs() - } - #[inline(always)] - fn f32(v: f32) -> f32 { - v.abs() - } - #[inline(always)] - fn f64(v: f64) -> f64 { - v.abs() - } - #[inline(always)] - fn u8(v: u8) -> u8 { - v - } - #[inline(always)] - fn u32(v: u32) -> u32 { - v - } - #[inline(always)] - fn i64(v: i64) -> i64 { - v.abs() - } -} - -impl UnaryOpT for Ceil { - const NAME: &'static str = "ceil"; - const KERNEL: &'static str = "uceil"; - const V: Self = Ceil; - #[inline(always)] - fn bf16(v: bf16) -> bf16 { - v.ceil() - } - #[inline(always)] - fn f16(v: f16) -> f16 { - v.ceil() - } - #[inline(always)] - fn f32(v: f32) -> f32 { - v.ceil() - } - #[inline(always)] - fn f64(v: f64) -> f64 { - v.ceil() - } - #[inline(always)] - fn u8(v: u8) -> u8 { - v - } - #[inline(always)] - fn u32(v: u32) -> u32 { - v - } - #[inline(always)] - fn i64(v: i64) -> i64 { - v - } -} - -impl UnaryOpT for Floor { - const NAME: &'static str = "floor"; - const KERNEL: &'static str = "ufloor"; - const V: Self = Floor; - #[inline(always)] - fn bf16(v: bf16) -> bf16 { - v.floor() - } - #[inline(always)] - fn f16(v: f16) -> f16 { - v.floor() - } - #[inline(always)] - fn f32(v: f32) -> f32 { - v.floor() - } - #[inline(always)] - fn f64(v: f64) -> f64 { - v.floor() - } - #[inline(always)] - fn u8(v: u8) -> u8 { - v - } - #[inline(always)] - fn u32(v: u32) -> u32 { - v - } - #[inline(always)] - fn i64(v: i64) -> i64 { - v - } -} - -impl UnaryOpT for Round { - const NAME: &'static str = "round"; - const KERNEL: &'static str = "uround"; - const V: Self = Round; - #[inline(always)] - fn bf16(v: bf16) -> bf16 { - v.round() - } - #[inline(always)] - fn f16(v: f16) -> f16 { - v.round() - } - #[inline(always)] - fn f32(v: f32) -> f32 { - v.round() - } - #[inline(always)] - fn f64(v: f64) -> f64 { - v.round() - } - #[inline(always)] - fn u8(v: u8) -> u8 { - v - } - #[inline(always)] - fn u32(v: u32) -> u32 { - v - } - #[inline(always)] - fn i64(v: i64) -> i64 { - v - } -} - -impl UnaryOpT for GeluErf { - const NAME: &'static str = "gelu_erf"; - const KERNEL: &'static str = "ugelu_erf"; - const V: Self = GeluErf; - #[inline(always)] - fn bf16(v: bf16) -> bf16 { - bf16::from_f64(Self::f64(v.to_f64())) - } - #[inline(always)] - fn f16(v: f16) -> f16 { - f16::from_f64(Self::f64(v.to_f64())) - } - #[inline(always)] - fn f32(v: f32) -> f32 { - Self::f64(v as f64) as f32 - } - #[inline(always)] - fn f64(v: f64) -> f64 { - (crate::cpu::erf::erf(v / 2f64.sqrt()) + 1.) * 0.5 * v - } - #[inline(always)] - fn u8(_: u8) -> u8 { - 0 - } - #[inline(always)] - fn u32(_: u32) -> u32 { - 0 - } - #[inline(always)] - fn i64(_: i64) -> i64 { - 0 - } -} - -impl UnaryOpT for Relu { - const NAME: &'static str = "relu"; - const KERNEL: &'static str = "urelu"; - const V: Self = Relu; - #[inline(always)] - fn bf16(v: bf16) -> bf16 { - v.max(bf16::ZERO) - } - #[inline(always)] - fn f16(v: f16) -> f16 { - v.max(f16::ZERO) - } - #[inline(always)] - fn f32(v: f32) -> f32 { - v.max(0f32) - } - #[inline(always)] - fn f64(v: f64) -> f64 { - v.max(0f64) - } - #[inline(always)] - fn u8(v: u8) -> u8 { - v - } - #[inline(always)] - fn u32(v: u32) -> u32 { - v - } - #[inline(always)] - fn i64(v: i64) -> i64 { - v - } -} - -/// `BackpropOp` is a wrapper around `Option`. The main goal is to ensure that dependencies are -/// properly checked when creating a new value -#[derive(Clone)] -pub struct BackpropOp(Option); - -impl BackpropOp { - pub(crate) fn none() -> Self { - BackpropOp(None) - } - - pub(crate) fn new1(arg: &Tensor, f: impl Fn(Tensor) -> Op) -> Self { - let op = if arg.track_op() { - Some(f(arg.clone())) - } else { - None - }; - Self(op) - } - - pub(crate) fn new2(arg1: &Tensor, arg2: &Tensor, f: impl Fn(Tensor, Tensor) -> Op) -> Self { - let op = if arg1.track_op() || arg2.track_op() { - Some(f(arg1.clone(), arg2.clone())) - } else { - None - }; - Self(op) - } - - pub(crate) fn new3( - arg1: &Tensor, - arg2: &Tensor, - arg3: &Tensor, - f: impl Fn(Tensor, Tensor, Tensor) -> Op, - ) -> Self { - let op = if arg1.track_op() || arg2.track_op() || arg3.track_op() { - Some(f(arg1.clone(), arg2.clone(), arg3.clone())) - } else { - None - }; - Self(op) - } - - pub(crate) fn new>(args: &[A], f: impl Fn(Vec) -> Op) -> Self { - let op = if args.iter().any(|arg| arg.as_ref().track_op()) { - let args: Vec = args.iter().map(|arg| arg.as_ref().clone()).collect(); - Some(f(args)) - } else { - None - }; - Self(op) - } - - pub(crate) fn is_none(&self) -> bool { - self.0.is_none() - } -} - -impl std::ops::Deref for BackpropOp { - type Target = Option; - fn deref(&self) -> &Self::Target { - &self.0 - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/pickle.rs b/v2/dependency-patches/candle-core-0.4.1/src/pickle.rs deleted file mode 100644 index 7a4182e1ea..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/pickle.rs +++ /dev/null @@ -1,817 +0,0 @@ -// Just enough pickle support to be able to read PyTorch checkpoints. -// This hardcodes objects that are required for tensor reading, we may want to make this a bit more -// composable/tensor agnostic at some point. -use crate::{DType, Error as E, Layout, Result, Tensor}; -use byteorder::{LittleEndian, ReadBytesExt}; -use std::collections::HashMap; -use std::io::BufRead; - -const VERBOSE: bool = false; - -// https://docs.juliahub.com/Pickle/LAUNc/0.1.0/opcode/ -#[repr(u8)] -#[derive(Debug, Eq, PartialEq, Clone)] -pub enum OpCode { - // https://github.com/python/cpython/blob/ed25f097160b5cbb0c9a1f9a746d2f1bbc96515a/Lib/pickletools.py#L2123 - Proto = 0x80, - Global = b'c', - BinPut = b'q', - LongBinPut = b'r', - EmptyTuple = b')', - Reduce = b'R', - Mark = b'(', - BinUnicode = b'X', - BinInt = b'J', - Tuple = b't', - BinPersId = b'Q', - BinInt1 = b'K', - BinInt2 = b'M', - Tuple1 = 0x85, - Tuple2 = 0x86, - Tuple3 = 0x87, - NewTrue = 0x88, - NewFalse = 0x89, - None = b'N', - BinGet = b'h', - LongBinGet = b'j', - SetItem = b's', - SetItems = b'u', - EmptyDict = b'}', - Dict = b'd', - Build = b'b', - Stop = b'.', - NewObj = 0x81, - EmptyList = b']', - BinFloat = b'G', - Append = b'a', - Appends = b'e', -} - -// Avoid using FromPrimitive so as not to drag another dependency. -impl TryFrom for OpCode { - type Error = u8; - fn try_from(value: u8) -> std::result::Result { - match value { - 0x80 => Ok(Self::Proto), - b'c' => Ok(Self::Global), - b'q' => Ok(Self::BinPut), - b'r' => Ok(Self::LongBinPut), - b')' => Ok(Self::EmptyTuple), - b'R' => Ok(Self::Reduce), - b'(' => Ok(Self::Mark), - b'X' => Ok(Self::BinUnicode), - b'J' => Ok(Self::BinInt), - b't' => Ok(Self::Tuple), - b'Q' => Ok(Self::BinPersId), - b'K' => Ok(Self::BinInt1), - b'M' => Ok(Self::BinInt2), - b'N' => Ok(Self::None), - 0x85 => Ok(Self::Tuple1), - 0x86 => Ok(Self::Tuple2), - 0x87 => Ok(Self::Tuple3), - 0x88 => Ok(Self::NewTrue), - 0x89 => Ok(Self::NewFalse), - b'h' => Ok(Self::BinGet), - b'j' => Ok(Self::LongBinGet), - b's' => Ok(Self::SetItem), - b'u' => Ok(Self::SetItems), - b'}' => Ok(Self::EmptyDict), - b'd' => Ok(Self::EmptyDict), - b'b' => Ok(Self::Build), - b'.' => Ok(Self::Stop), - 0x81 => Ok(Self::NewObj), - b']' => Ok(Self::EmptyList), - b'G' => Ok(Self::BinFloat), - b'a' => Ok(Self::Append), - b'e' => Ok(Self::Appends), - value => Err(value), - } - } -} - -fn read_to_newline(r: &mut R) -> Result> { - let mut data: Vec = Vec::with_capacity(32); - r.read_until(b'\n', &mut data)?; - data.pop(); - if data.last() == Some(&b'\r') { - data.pop(); - } - Ok(data) -} - -#[derive(Debug, Clone, PartialEq)] -pub enum Object { - Class { - module_name: String, - class_name: String, - }, - Int(i32), - Float(f64), - Unicode(String), - Bool(bool), - None, - Tuple(Vec), - List(Vec), - Mark, - Dict(Vec<(Object, Object)>), - Reduce { - callable: Box, - args: Box, - }, - Build { - callable: Box, - args: Box, - }, - PersistentLoad(Box), -} - -type OResult = std::result::Result; - -impl Object { - pub fn unicode(self) -> OResult { - match self { - Self::Unicode(t) => Ok(t), - _ => Err(self), - } - } - - pub fn reduce(self) -> OResult<(Self, Self)> { - match self { - Self::Reduce { callable, args } => Ok((*callable, *args)), - _ => Err(self), - } - } - - pub fn none(self) -> OResult<()> { - match self { - Self::None => Ok(()), - _ => Err(self), - } - } - - pub fn persistent_load(self) -> OResult { - match self { - Self::PersistentLoad(t) => Ok(*t), - _ => Err(self), - } - } - - pub fn bool(self) -> OResult { - match self { - Self::Bool(t) => Ok(t), - _ => Err(self), - } - } - - pub fn int(self) -> OResult { - match self { - Self::Int(t) => Ok(t), - _ => Err(self), - } - } - - pub fn tuple(self) -> OResult> { - match self { - Self::Tuple(t) => Ok(t), - _ => Err(self), - } - } - - pub fn dict(self) -> OResult> { - match self { - Self::Dict(t) => Ok(t), - _ => Err(self), - } - } - - pub fn class(self) -> OResult<(String, String)> { - match self { - Self::Class { - module_name, - class_name, - } => Ok((module_name, class_name)), - _ => Err(self), - } - } - - pub fn into_tensor_info( - self, - name: Self, - dir_name: &std::path::Path, - ) -> Result> { - let name = match name.unicode() { - Ok(name) => name, - Err(_) => return Ok(None), - }; - let (callable, args) = match self.reduce() { - Ok(callable_args) => callable_args, - _ => return Ok(None), - }; - let (callable, args) = match callable { - Object::Class { - module_name, - class_name, - } if module_name == "torch._tensor" && class_name == "_rebuild_from_type_v2" => { - let mut args = args.tuple()?; - let callable = args.remove(0); - let args = args.remove(1); - (callable, args) - } - Object::Class { - module_name, - class_name, - } if module_name == "torch._utils" && class_name == "_rebuild_parameter" => { - let mut args = args.tuple()?; - args.remove(0).reduce()? - } - _ => (callable, args), - }; - match callable { - Object::Class { - module_name, - class_name, - } if module_name == "torch._utils" && class_name == "_rebuild_tensor_v2" => {} - _ => return Ok(None), - }; - let (layout, dtype, file_path, storage_size) = rebuild_args(args)?; - Ok(Some(TensorInfo { - name, - dtype, - layout, - path: format!("{}/{}", dir_name.to_string_lossy(), file_path), - storage_size, - })) - } -} - -impl TryFrom for String { - type Error = Object; - fn try_from(value: Object) -> std::result::Result { - match value { - Object::Unicode(s) => Ok(s), - other => Err(other), - } - } -} - -impl TryFrom for usize { - type Error = Object; - fn try_from(value: Object) -> std::result::Result { - match value { - Object::Int(s) if s >= 0 => Ok(s as usize), - other => Err(other), - } - } -} - -impl> TryFrom for Vec { - type Error = Object; - fn try_from(value: Object) -> std::result::Result { - match value { - Object::Tuple(values) => { - // This does not return the appropriate value in the error case but instead return - // the object related to the first error. - values - .into_iter() - .map(|v| T::try_from(v)) - .collect::, Self::Error>>() - } - other => Err(other), - } - } -} - -#[derive(Debug)] -pub struct Stack { - stack: Vec, - memo: HashMap, -} - -impl Stack { - pub fn empty() -> Self { - Self { - stack: Vec::with_capacity(512), - memo: HashMap::new(), - } - } - - pub fn stack(&self) -> &[Object] { - self.stack.as_slice() - } - - pub fn read_loop(&mut self, r: &mut R) -> Result<()> { - loop { - if self.read(r)? { - break; - } - } - Ok(()) - } - - pub fn finalize(mut self) -> Result { - self.pop() - } - - fn push(&mut self, obj: Object) { - self.stack.push(obj) - } - - fn pop(&mut self) -> Result { - match self.stack.pop() { - None => crate::bail!("unexpected empty stack"), - Some(obj) => Ok(obj), - } - } - - // https://docs.juliahub.com/Pickle/LAUNc/0.1.0/opcode/#Pickle.OpCodes.BUILD - fn build(&mut self) -> Result<()> { - let args = self.pop()?; - let obj = self.pop()?; - let obj = match (obj, args) { - (Object::Dict(mut obj), Object::Dict(mut args)) => { - obj.append(&mut args); - Object::Dict(obj) - } - (obj, args) => Object::Build { - callable: Box::new(obj), - args: Box::new(args), - }, - }; - self.push(obj); - Ok(()) - } - - fn reduce(&mut self) -> Result<()> { - let args = self.pop()?; - let callable = self.pop()?; - #[allow(clippy::single_match)] - let reduced = match &callable { - Object::Class { - module_name, - class_name, - } => { - if module_name == "collections" - && (class_name == "OrderedDict" || class_name == "defaultdict") - { - // TODO: have a separate ordered dict and a separate default dict. - Some(Object::Dict(vec![])) - } else { - None - } - } - _ => None, - }; - let reduced = reduced.unwrap_or_else(|| Object::Reduce { - callable: Box::new(callable), - args: Box::new(args), - }); - self.push(reduced); - Ok(()) - } - - fn last(&mut self) -> Result<&mut Object> { - match self.stack.last_mut() { - None => crate::bail!("unexpected empty stack"), - Some(obj) => Ok(obj), - } - } - - fn memo_get(&self, id: u32) -> Result { - match self.memo.get(&id) { - None => crate::bail!("missing object in memo {id}"), - Some(obj) => { - // Maybe we should use refcounting rather than doing potential large clones here. - Ok(obj.clone()) - } - } - } - - fn memo_put(&mut self, id: u32) -> Result<()> { - let obj = self.last()?.clone(); - self.memo.insert(id, obj); - Ok(()) - } - - fn persistent_load(&self, id: Object) -> Result { - Ok(Object::PersistentLoad(Box::new(id))) - } - - fn new_obj(&self, class: Object, args: Object) -> Result { - Ok(Object::Reduce { - callable: Box::new(class), - args: Box::new(args), - }) - } - - fn pop_to_marker(&mut self) -> Result> { - let mut mark_idx = None; - for (idx, obj) in self.stack.iter().enumerate().rev() { - if obj == &Object::Mark { - mark_idx = Some(idx); - break; - } - } - match mark_idx { - Some(mark_idx) => { - let objs = self.stack.split_off(mark_idx + 1); - self.stack.pop(); - Ok(objs) - } - None => { - crate::bail!("marker object not found") - } - } - } - - pub fn read(&mut self, r: &mut R) -> Result { - let op_code = match OpCode::try_from(r.read_u8()?) { - Ok(op_code) => op_code, - Err(op_code) => { - crate::bail!("unknown op-code {op_code}") - } - }; - // println!("op: {op_code:?}"); - // println!("{:?}", self.stack); - match op_code { - OpCode::Proto => { - let version = r.read_u8()?; - if VERBOSE { - println!("proto {version}"); - } - } - OpCode::Global => { - let module_name = read_to_newline(r)?; - let class_name = read_to_newline(r)?; - let module_name = String::from_utf8_lossy(&module_name).to_string(); - let class_name = String::from_utf8_lossy(&class_name).to_string(); - self.push(Object::Class { - module_name, - class_name, - }) - } - OpCode::BinInt1 => { - let arg = r.read_u8()?; - self.push(Object::Int(arg as i32)) - } - OpCode::BinInt2 => { - let arg = r.read_u16::()?; - self.push(Object::Int(arg as i32)) - } - OpCode::BinInt => { - let arg = r.read_i32::()?; - self.push(Object::Int(arg)) - } - OpCode::BinFloat => { - // Somehow floats are encoded using BigEndian whereas int types use LittleEndian. - // https://github.com/python/cpython/blob/0c80da4c14d904a367968955544dd6ae58c8101c/Lib/pickletools.py#L855 - // https://github.com/pytorch/pytorch/blob/372d078f361e726bb4ac0884ac334b04c58179ef/torch/_weights_only_unpickler.py#L243 - let arg = r.read_f64::()?; - self.push(Object::Float(arg)) - } - OpCode::BinUnicode => { - let len = r.read_u32::()?; - let mut data = vec![0u8; len as usize]; - r.read_exact(&mut data)?; - let data = String::from_utf8(data).map_err(E::wrap)?; - self.push(Object::Unicode(data)) - } - OpCode::BinPersId => { - let id = self.pop()?; - let obj = self.persistent_load(id)?; - self.push(obj) - } - OpCode::Tuple => { - let objs = self.pop_to_marker()?; - self.push(Object::Tuple(objs)) - } - OpCode::Tuple1 => { - let obj = self.pop()?; - self.push(Object::Tuple(vec![obj])) - } - OpCode::Tuple2 => { - let obj2 = self.pop()?; - let obj1 = self.pop()?; - self.push(Object::Tuple(vec![obj1, obj2])) - } - OpCode::Tuple3 => { - let obj3 = self.pop()?; - let obj2 = self.pop()?; - let obj1 = self.pop()?; - self.push(Object::Tuple(vec![obj1, obj2, obj3])) - } - OpCode::NewTrue => self.push(Object::Bool(true)), - OpCode::NewFalse => self.push(Object::Bool(false)), - OpCode::Append => { - let value = self.pop()?; - let pylist = self.last()?; - if let Object::List(d) = pylist { - d.push(value) - } else { - crate::bail!("expected a list, got {pylist:?}") - } - } - OpCode::Appends => { - let objs = self.pop_to_marker()?; - let pylist = self.last()?; - if let Object::List(d) = pylist { - d.extend(objs) - } else { - crate::bail!("expected a list, got {pylist:?}") - } - } - OpCode::SetItem => { - let value = self.pop()?; - let key = self.pop()?; - let pydict = self.last()?; - if let Object::Dict(d) = pydict { - d.push((key, value)) - } else { - crate::bail!("expected a dict, got {pydict:?}") - } - } - OpCode::SetItems => { - let mut objs = self.pop_to_marker()?; - let pydict = self.last()?; - if let Object::Dict(d) = pydict { - if objs.len() % 2 != 0 { - crate::bail!("setitems: not an even number of objects") - } - while let Some(value) = objs.pop() { - let key = objs.pop().unwrap(); - d.push((key, value)) - } - } else { - crate::bail!("expected a dict, got {pydict:?}") - } - } - OpCode::None => self.push(Object::None), - OpCode::Stop => { - return Ok(true); - } - OpCode::Build => self.build()?, - OpCode::EmptyDict => self.push(Object::Dict(vec![])), - OpCode::Dict => { - let mut objs = self.pop_to_marker()?; - let mut pydict = vec![]; - if objs.len() % 2 != 0 { - crate::bail!("setitems: not an even number of objects") - } - while let Some(value) = objs.pop() { - let key = objs.pop().unwrap(); - pydict.push((key, value)) - } - self.push(Object::Dict(pydict)) - } - OpCode::Mark => self.push(Object::Mark), - OpCode::Reduce => self.reduce()?, - OpCode::EmptyTuple => self.push(Object::Tuple(vec![])), - OpCode::EmptyList => self.push(Object::List(vec![])), - OpCode::BinGet => { - let arg = r.read_u8()?; - let obj = self.memo_get(arg as u32)?; - self.push(obj) - } - OpCode::LongBinGet => { - let arg = r.read_u32::()?; - let obj = self.memo_get(arg)?; - self.push(obj) - } - OpCode::BinPut => { - let arg = r.read_u8()?; - self.memo_put(arg as u32)? - } - OpCode::LongBinPut => { - let arg = r.read_u32::()?; - self.memo_put(arg)? - } - OpCode::NewObj => { - let args = self.pop()?; - let class = self.pop()?; - let obj = self.new_obj(class, args)?; - self.push(obj) - } - } - Ok(false) - } -} - -impl From for E { - fn from(value: Object) -> Self { - E::Msg(format!("conversion error on {value:?}")) - } -} - -// https://github.com/pytorch/pytorch/blob/4eac43d046ded0f0a5a5fa8db03eb40f45bf656e/torch/_utils.py#L198 -// Arguments: storage, storage_offset, size, stride, requires_grad, backward_hooks -fn rebuild_args(args: Object) -> Result<(Layout, DType, String, usize)> { - let mut args = args.tuple()?; - let stride = Vec::::try_from(args.remove(3))?; - let size = Vec::::try_from(args.remove(2))?; - let offset = args.remove(1).int()? as usize; - let storage = args.remove(0).persistent_load()?; - let mut storage = storage.tuple()?; - let storage_size = storage.remove(4).int()? as usize; - let path = storage.remove(2).unicode()?; - let (_module_name, class_name) = storage.remove(1).class()?; - let dtype = match class_name.as_str() { - "FloatStorage" => DType::F32, - "DoubleStorage" => DType::F64, - "HalfStorage" => DType::F16, - "BFloat16Storage" => DType::BF16, - "ByteStorage" => DType::U8, - "LongStorage" => DType::I64, - other => { - crate::bail!("unsupported storage type {other}") - } - }; - let layout = Layout::new(crate::Shape::from(size), stride, offset); - Ok((layout, dtype, path, storage_size)) -} - -#[derive(Debug, Clone)] -pub struct TensorInfo { - pub name: String, - pub dtype: DType, - pub layout: Layout, - pub path: String, - pub storage_size: usize, -} - -/// Read the tensor info from a .pth file. -/// -/// # Arguments -/// * `file` - The path to the .pth file. -/// * `verbose` - Whether to print debug information. -/// * `key` - Optional key to retrieve `state_dict` from the pth file. -pub fn read_pth_tensor_info>( - file: P, - verbose: bool, - key: Option<&str>, -) -> Result> { - let file = std::fs::File::open(file)?; - let zip_reader = std::io::BufReader::new(file); - let mut zip = zip::ZipArchive::new(zip_reader)?; - let zip_file_names = zip - .file_names() - .map(|name| name.map(|name| name.into_owned())) - .collect::>>()?; - - let mut tensor_infos = vec![]; - for file_name in zip_file_names.iter() { - if !file_name.ends_with("data.pkl") { - continue; - } - let dir_name = std::path::PathBuf::from(file_name.strip_suffix(".pkl").unwrap()); - let reader = zip.by_name(file_name)?; - let mut reader = std::io::BufReader::new(reader); - let mut stack = Stack::empty(); - stack.read_loop(&mut reader)?; - let obj = stack.finalize()?; - if VERBOSE || verbose { - println!("{obj:#?}"); - } - - let obj = match obj { - Object::Build { callable, args } => match *callable { - Object::Reduce { callable, args: _ } => match *callable { - Object::Class { - module_name, - class_name, - } if module_name == "__torch__" && class_name == "Module" => *args, - _ => continue, - }, - _ => continue, - }, - obj => obj, - }; - - // If key is provided, then we need to extract the state_dict from the object. - let obj = if let Some(key) = key { - if let Object::Dict(key_values) = obj { - key_values - .into_iter() - .find(|(k, _)| *k == Object::Unicode(key.to_owned())) - .map(|(_, v)| v) - .ok_or_else(|| E::Msg(format!("key {key} not found")))? - } else { - obj - } - } else { - obj - }; - - // If the object is a dict, then we can extract the tensor info from it. - // NOTE: We are assuming that the `obj` is state_dict by this stage. - if let Object::Dict(key_values) = obj { - for (name, value) in key_values.into_iter() { - match value.into_tensor_info(name, &dir_name) { - Ok(Some(tensor_info)) => tensor_infos.push(tensor_info), - Ok(None) => {} - Err(err) => eprintln!("skipping: {err:?}"), - } - } - } - } - Ok(tensor_infos) -} - -/// Lazy tensor loader. -pub struct PthTensors { - tensor_infos: HashMap, - path: std::path::PathBuf, - // We do not store a zip reader as it needs mutable access to extract data. Instead we - // re-create a zip reader for each tensor. -} - -impl PthTensors { - pub fn new>(path: P, key: Option<&str>) -> Result { - let tensor_infos = read_pth_tensor_info(path.as_ref(), false, key)?; - let tensor_infos = tensor_infos - .into_iter() - .map(|ti| (ti.name.to_string(), ti)) - .collect(); - let path = path.as_ref().to_owned(); - Ok(Self { tensor_infos, path }) - } - - pub fn tensor_infos(&self) -> &HashMap { - &self.tensor_infos - } - - pub fn get(&self, name: &str) -> Result> { - use std::io::Read; - let tensor_info = match self.tensor_infos.get(name) { - None => return Ok(None), - Some(tensor_info) => tensor_info, - }; - // We hope that the file has not changed since first reading it. - let zip_reader = std::io::BufReader::new(std::fs::File::open(&self.path)?); - let mut zip = zip::ZipArchive::new(zip_reader)?; - let mut reader = zip.by_name(&tensor_info.path)?; - let is_fortran_contiguous = tensor_info.layout.is_fortran_contiguous(); - let rank = tensor_info.layout.shape().rank(); - - // Reading the data is a bit tricky as it can be strided, for now only support the basic - // case and when the tensor is fortran contiguous. - if !tensor_info.layout.is_contiguous() && !is_fortran_contiguous { - crate::bail!( - "cannot retrieve non-contiguous tensors {:?}", - tensor_info.layout - ) - } - let start_offset = tensor_info.layout.start_offset(); - if start_offset > 0 { - std::io::copy( - &mut reader.by_ref().take(start_offset as u64), - &mut std::io::sink(), - )?; - } - let tensor = Tensor::from_reader( - tensor_info.layout.shape().clone(), - tensor_info.dtype, - &mut reader, - )?; - - if rank > 1 && is_fortran_contiguous { - // Reverse the shape, e.g. Shape(2, 3, 4) -> Shape(4, 3, 2) - let shape_reversed: Vec<_> = tensor_info.layout.dims().iter().rev().cloned().collect(); - let tensor = tensor.reshape(shape_reversed)?; - - // Permute (transpose) the dimensions, e.g. Shape(4, 3, 2) -> Shape(2, 3, 4) - let dim_indeces_reversed: Vec<_> = (0..rank).rev().collect(); - let tensor = tensor.permute(dim_indeces_reversed)?; - Ok(Some(tensor)) - } else { - Ok(Some(tensor)) - } - } -} - -/// Read all the tensors from a PyTorch pth file with a given key. -/// -/// # Arguments -/// * `path` - Path to the pth file. -/// * `key` - Optional key to retrieve `state_dict` from the pth file. Sometimes the pth file -/// contains multiple objects and the state_dict is the one we are interested in. -pub fn read_all_with_key>( - path: P, - key: Option<&str>, -) -> Result> { - let pth = PthTensors::new(path, key)?; - let tensor_names = pth.tensor_infos.keys(); - let mut tensors = Vec::with_capacity(tensor_names.len()); - for name in tensor_names { - if let Some(tensor) = pth.get(name)? { - tensors.push((name.to_string(), tensor)) - } - } - Ok(tensors) -} - -/// Read all the tensors from a PyTorch pth file. -/// -/// # Arguments -/// * `path` - Path to the pth file. -pub fn read_all>(path: P) -> Result> { - read_all_with_key(path, None) -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/avx.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/avx.rs deleted file mode 100644 index 664f7653ee..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/quantized/avx.rs +++ /dev/null @@ -1,667 +0,0 @@ -use super::k_quants::{ - BlockQ2K, BlockQ3K, BlockQ4K, BlockQ4_0, BlockQ5K, BlockQ6K, BlockQ8K, BlockQ8_0, QK8_0, QK_K, -}; -use crate::Result; -use byteorder::{ByteOrder, LittleEndian}; -use half::f16; - -#[cfg(target_arch = "x86")] -use core::arch::x86::*; -#[cfg(target_arch = "x86_64")] -use core::arch::x86_64::*; - -#[inline(always)] -pub(crate) unsafe fn sum_i16_pairs_float(x: __m256i) -> __m256 { - let ones = _mm256_set1_epi16(1); - let summed_pairs = _mm256_madd_epi16(ones, x); - _mm256_cvtepi32_ps(summed_pairs) -} - -#[inline(always)] -pub(crate) unsafe fn mul_sum_us8_pairs_float(ax: __m256i, sy: __m256i) -> __m256 { - let dot = _mm256_maddubs_epi16(ax, sy); - sum_i16_pairs_float(dot) -} - -#[inline(always)] -pub(crate) unsafe fn hsum_float_8(x: __m256) -> f32 { - let res = _mm256_extractf128_ps(x, 1); - let res = _mm_add_ps(res, _mm256_castps256_ps128(x)); - let res = _mm_add_ps(res, _mm_movehl_ps(res, res)); - let res = _mm_add_ss(res, _mm_movehdup_ps(res)); - _mm_cvtss_f32(res) -} - -#[inline(always)] -pub(crate) unsafe fn bytes_from_nibbles_32(rsi: *const u8) -> __m256i { - let tmp = _mm_loadu_si128(rsi as *const __m128i); - let bytes = _mm256_insertf128_si256::<1>(_mm256_castsi128_si256(tmp), _mm_srli_epi16(tmp, 4)); - let low_mask = _mm256_set1_epi8(0xF); - _mm256_and_si256(low_mask, bytes) -} - -#[inline(always)] -pub(crate) unsafe fn mul_sum_i8_pairs_float(x: __m256i, y: __m256i) -> __m256 { - let ax = _mm256_sign_epi8(x, x); - let sy = _mm256_sign_epi8(y, x); - mul_sum_us8_pairs_float(ax, sy) -} - -#[inline(always)] -pub(crate) fn vec_dot_q4_0_q8_0(n: usize, xs: &[BlockQ4_0], ys: &[BlockQ8_0]) -> Result { - let qk = QK8_0; - if n % QK8_0 != 0 { - crate::bail!("vec_dot_q4_0_q8_0: {n} is not divisible by {qk}") - } - unsafe { - let mut acc = _mm256_setzero_ps(); - for (x, y) in xs.iter().zip(ys.iter()) { - let d = _mm256_set1_ps(f16::to_f32(x.d) * f16::to_f32(y.d)); - let bx = bytes_from_nibbles_32(x.qs.as_ptr()); - let off = _mm256_set1_epi8(8); - let bx = _mm256_sub_epi8(bx, off); - let by = _mm256_loadu_si256(y.qs.as_ptr() as *const __m256i); - let q = mul_sum_i8_pairs_float(bx, by); - acc = _mm256_fmadd_ps(d, q, acc); - } - Ok(hsum_float_8(acc)) - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q8_0_q8_0(n: usize, xs: &[BlockQ8_0], ys: &[BlockQ8_0]) -> Result { - let qk = QK8_0; - if n % QK8_0 != 0 { - crate::bail!("vec_dot_q8_0_q8_0: {n} is not divisible by {qk}") - } - unsafe { - let mut acc = _mm256_setzero_ps(); - for (x, y) in xs.iter().zip(ys.iter()) { - let d = _mm256_set1_ps(f16::to_f32(x.d) * f16::to_f32(y.d)); - let bx = _mm256_loadu_si256(x.qs.as_ptr() as *const __m256i); - let by = _mm256_loadu_si256(y.qs.as_ptr() as *const __m256i); - let q = mul_sum_i8_pairs_float(bx, by); - acc = _mm256_fmadd_ps(d, q, acc); - } - Ok(hsum_float_8(acc)) - } -} - -#[inline(always)] -unsafe fn get_scale_shuffle(i: usize) -> __m128i { - const K_SHUFFLE: [u8; 128] = [ - 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, - 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, - 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, - 11, 11, 11, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 13, 13, 13, 13, 13, 13, 13, - 13, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, - ]; - _mm_loadu_si128((K_SHUFFLE.as_ptr() as *const __m128i).add(i)) -} - -#[inline(always)] -unsafe fn get_scale_shuffle_k4(i: usize) -> __m256i { - const K_SHUFFLE: [u8; 256] = [ - 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, - 0, 1, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, - 2, 3, 2, 3, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, - 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, - 6, 7, 6, 7, 6, 7, 6, 7, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, - 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, - 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 12, 13, 12, 13, 12, 13, - 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, - 13, 12, 13, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, - 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, - ]; - _mm256_loadu_si256((K_SHUFFLE.as_ptr() as *const __m256i).add(i)) -} - -#[inline(always)] -unsafe fn get_scale_shuffle_q3k(i: usize) -> __m256i { - const K_SHUFFLE: [u8; 128] = [ - 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, - 2, 3, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, - 6, 7, 6, 7, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 10, 11, 10, 11, 10, 11, 10, 11, - 10, 11, 10, 11, 10, 11, 10, 11, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, - 13, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, - ]; - _mm256_loadu_si256((K_SHUFFLE.as_ptr() as *const __m256i).add(i)) -} - -#[inline(always)] -pub(crate) fn vec_dot_q6k_q8k(n: usize, xs: &[BlockQ6K], ys: &[BlockQ8K]) -> Result { - let qk = QK_K; - if n % qk != 0 { - crate::bail!("vec_dot_q6k_8k: {n} is not divisible by {qk}") - } - - unsafe { - let m4 = _mm256_set1_epi8(0xF); - let m2 = _mm256_set1_epi8(3); - let m32s = _mm256_set1_epi8(32); - let mut acc = _mm256_setzero_ps(); - for (x, y) in xs.iter().zip(ys.iter()) { - let d = y.d * x.d.to_f32(); - let mut q4 = x.ql.as_ptr(); - let mut qh = x.qh.as_ptr(); - let mut q8 = y.qs.as_ptr(); - - let scales = _mm_loadu_si128(x.scales.as_ptr() as *const __m128i); - let mut sumi = _mm256_setzero_si256(); - - for j in 0..QK_K / 128 { - let is = j * 4; - let scale_0 = _mm_shuffle_epi8(scales, get_scale_shuffle(is)); - let scale_1 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 1)); - let scale_2 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 2)); - let scale_3 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 3)); - - let q4bits1 = _mm256_loadu_si256(q4 as *const __m256i); - q4 = q4.add(32); - let q4bits2 = _mm256_loadu_si256(q4 as *const __m256i); - q4 = q4.add(32); - let q4bits_h = _mm256_loadu_si256(qh as *const __m256i); - qh = qh.add(32); - - let q4h_0 = _mm256_slli_epi16(_mm256_and_si256(q4bits_h, m2), 4); - let q4h_1 = - _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bits_h, 2), m2), 4); - let q4h_2 = - _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bits_h, 4), m2), 4); - let q4h_3 = - _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bits_h, 6), m2), 4); - - let q4_0 = _mm256_or_si256(_mm256_and_si256(q4bits1, m4), q4h_0); - let q4_1 = _mm256_or_si256(_mm256_and_si256(q4bits2, m4), q4h_1); - let q4_2 = - _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits1, 4), m4), q4h_2); - let q4_3 = - _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits2, 4), m4), q4h_3); - - let q8_0 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let q8_1 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let q8_2 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let q8_3 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - - let q8s_0 = _mm256_maddubs_epi16(m32s, q8_0); - let q8s_1 = _mm256_maddubs_epi16(m32s, q8_1); - let q8s_2 = _mm256_maddubs_epi16(m32s, q8_2); - let q8s_3 = _mm256_maddubs_epi16(m32s, q8_3); - - let p16_0 = _mm256_maddubs_epi16(q4_0, q8_0); - let p16_1 = _mm256_maddubs_epi16(q4_1, q8_1); - let p16_2 = _mm256_maddubs_epi16(q4_2, q8_2); - let p16_3 = _mm256_maddubs_epi16(q4_3, q8_3); - - let p16_0 = _mm256_sub_epi16(p16_0, q8s_0); - let p16_1 = _mm256_sub_epi16(p16_1, q8s_1); - let p16_2 = _mm256_sub_epi16(p16_2, q8s_2); - let p16_3 = _mm256_sub_epi16(p16_3, q8s_3); - - let p16_0 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_0), p16_0); - let p16_1 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_1), p16_1); - let p16_2 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_2), p16_2); - let p16_3 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_3), p16_3); - - sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1)); - sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_2, p16_3)); - } - acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc); - } - Ok(hsum_float_8(acc)) - } -} - -#[inline(always)] -unsafe fn mm256_set_m128i(a: __m128i, b: __m128i) -> __m256i { - _mm256_insertf128_si256(_mm256_castsi128_si256(b), a, 1) -} - -#[inline(always)] -pub(crate) fn vec_dot_q2k_q8k(n: usize, xs: &[BlockQ2K], ys: &[BlockQ8K]) -> Result { - if n % QK_K != 0 { - crate::bail!("vec_dot_q2k_q8k: {n} is not divisible by {QK_K}") - } - - unsafe { - let m3 = _mm256_set1_epi8(3); - let m4 = _mm_set1_epi8(0xF); - - let mut acc = _mm256_setzero_ps(); - - for (x, y) in xs.iter().zip(ys.iter()) { - let d = y.d * x.d.to_f32(); - let dmin = -y.d * x.dmin.to_f32(); - - let mut q2 = x.qs.as_ptr(); - let mut q8 = y.qs.as_ptr(); - - let mins_and_scales = _mm_loadu_si128(x.scales.as_ptr() as *const __m128i); - let scales8 = _mm_and_si128(mins_and_scales, m4); - let mins8 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4); - let mins = _mm256_cvtepi8_epi16(mins8); - let prod = - _mm256_madd_epi16(mins, _mm256_loadu_si256(y.bsums.as_ptr() as *const __m256i)); - - acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&dmin), _mm256_cvtepi32_ps(prod), acc); - - let all_scales = _mm256_cvtepi8_epi16(scales8); - let l_scales = _mm256_extracti128_si256(all_scales, 0); - let h_scales = _mm256_extracti128_si256(all_scales, 1); - let scales = [ - mm256_set_m128i(l_scales, l_scales), - mm256_set_m128i(h_scales, h_scales), - ]; - - let mut sumi = _mm256_setzero_si256(); - - for scale in scales { - let q2bits = _mm256_loadu_si256(q2 as *const __m256i); - q2 = q2.add(32); - - let q8_0 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let q8_1 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let q8_2 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let q8_3 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - - let q2_0 = _mm256_and_si256(q2bits, m3); - let q2_1 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 2), m3); - let q2_2 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 4), m3); - let q2_3 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 6), m3); - - let p0 = _mm256_maddubs_epi16(q2_0, q8_0); - let p1 = _mm256_maddubs_epi16(q2_1, q8_1); - let p2 = _mm256_maddubs_epi16(q2_2, q8_2); - let p3 = _mm256_maddubs_epi16(q2_3, q8_3); - - let p0 = - _mm256_madd_epi16(_mm256_shuffle_epi8(scale, get_scale_shuffle_q3k(0)), p0); - let p1 = - _mm256_madd_epi16(_mm256_shuffle_epi8(scale, get_scale_shuffle_q3k(1)), p1); - let p2 = - _mm256_madd_epi16(_mm256_shuffle_epi8(scale, get_scale_shuffle_q3k(2)), p2); - let p3 = - _mm256_madd_epi16(_mm256_shuffle_epi8(scale, get_scale_shuffle_q3k(3)), p3); - - let p0 = _mm256_add_epi32(p0, p1); - let p2 = _mm256_add_epi32(p2, p3); - - sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p0, p2)); - } - acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc); - } - - Ok(hsum_float_8(acc)) - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q3k_q8k(n: usize, xs: &[BlockQ3K], ys: &[BlockQ8K]) -> Result { - if n % QK_K != 0 { - crate::bail!("vec_dot_q3k_q8k: {n} is not divisible by {QK_K}") - } - - const KMASK1: u32 = 0x03030303; - const KMASK2: u32 = 0x0f0f0f0f; - - let mut aux = [0u32; 3]; - - unsafe { - let m3 = _mm256_set1_epi8(3); - let mone = _mm256_set1_epi8(1); - let m32 = _mm_set1_epi8(32); - - let mut acc = _mm256_setzero_ps(); - for (x, y) in xs.iter().zip(ys.iter()) { - let d = y.d * x.d.to_f32(); - - let mut q3 = x.qs.as_ptr(); - let mut q8 = y.qs.as_ptr(); - - LittleEndian::read_u32_into(&x.scales, &mut aux); - let scales128 = _mm_set_epi32( - (((aux[1] >> 4) & KMASK2) | (((aux[2] >> 6) & KMASK1) << 4)) as i32, - (((aux[0] >> 4) & KMASK2) | (((aux[2] >> 4) & KMASK1) << 4)) as i32, - ((aux[1] & KMASK2) | (((aux[2] >> 2) & KMASK1) << 4)) as i32, - ((aux[0] & KMASK2) | (((aux[2]) & KMASK1) << 4)) as i32, - ); - let scales128 = _mm_sub_epi8(scales128, m32); - let all_scales = _mm256_cvtepi8_epi16(scales128); - let l_scales = _mm256_extracti128_si256(all_scales, 0); - let h_scales = _mm256_extracti128_si256(all_scales, 1); - let scales = [ - mm256_set_m128i(l_scales, l_scales), - mm256_set_m128i(h_scales, h_scales), - ]; - - // high bit - let hbits = _mm256_loadu_si256(x.hmask.as_ptr() as *const __m256i); - - let mut sumi = _mm256_setzero_si256(); - - for (j, scale) in scales.iter().enumerate() { - // load low 2 bits - let q3bits = _mm256_loadu_si256(q3 as *const __m256i); - q3 = q3.add(32); - - // Prepare low and high bits - // We hardcode the shifts here to avoid loading them into a separate register - let q3l_0 = _mm256_and_si256(q3bits, m3); - let q3h_0 = if j == 0 { - _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 0)), 0) - } else { - _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 4)), 4) - }; - let q3h_0 = _mm256_slli_epi16(q3h_0, 2); - - let q3l_1 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 2), m3); - let q3h_1 = if j == 0 { - _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 1)), 1) - } else { - _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 5)), 5) - }; - let q3h_1 = _mm256_slli_epi16(q3h_1, 2); - - let q3l_2 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 4), m3); - let q3h_2 = if j == 0 { - _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 2)), 2) - } else { - _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 6)), 6) - }; - let q3h_2 = _mm256_slli_epi16(q3h_2, 2); - - let q3l_3 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 6), m3); - let q3h_3 = if j == 0 { - _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 3)), 3) - } else { - _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 7)), 7) - }; - let q3h_3 = _mm256_slli_epi16(q3h_3, 2); - - // load Q8 quants - let q8_0 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let q8_1 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let q8_2 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let q8_3 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - - // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we - // can use _mm256_maddubs_epi16, and then subtract. The high bit part has the 2 - // already subtracted (and so, it is zero if the high bit was not set, and 2 if the - // high bit was set) - let q8s_0 = _mm256_maddubs_epi16(q3h_0, q8_0); - let q8s_1 = _mm256_maddubs_epi16(q3h_1, q8_1); - let q8s_2 = _mm256_maddubs_epi16(q3h_2, q8_2); - let q8s_3 = _mm256_maddubs_epi16(q3h_3, q8_3); - - let p16_0 = _mm256_maddubs_epi16(q3l_0, q8_0); - let p16_1 = _mm256_maddubs_epi16(q3l_1, q8_1); - let p16_2 = _mm256_maddubs_epi16(q3l_2, q8_2); - let p16_3 = _mm256_maddubs_epi16(q3l_3, q8_3); - - let p16_0 = _mm256_sub_epi16(p16_0, q8s_0); - let p16_1 = _mm256_sub_epi16(p16_1, q8s_1); - let p16_2 = _mm256_sub_epi16(p16_2, q8s_2); - let p16_3 = _mm256_sub_epi16(p16_3, q8s_3); - - // multiply with scales - let p16_0 = - _mm256_madd_epi16(_mm256_shuffle_epi8(*scale, get_scale_shuffle_q3k(0)), p16_0); - let p16_1 = - _mm256_madd_epi16(_mm256_shuffle_epi8(*scale, get_scale_shuffle_q3k(1)), p16_1); - let p16_2 = - _mm256_madd_epi16(_mm256_shuffle_epi8(*scale, get_scale_shuffle_q3k(2)), p16_2); - let p16_3 = - _mm256_madd_epi16(_mm256_shuffle_epi8(*scale, get_scale_shuffle_q3k(3)), p16_3); - - // accumulate - let p16_0 = _mm256_add_epi32(p16_0, p16_1); - let p16_2 = _mm256_add_epi32(p16_2, p16_3); - sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_2)); - } - - // multiply with block scale and accumulate - acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc); - } - Ok(hsum_float_8(acc)) - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q4k_q8k(n: usize, xs: &[BlockQ4K], ys: &[BlockQ8K]) -> Result { - if n % QK_K != 0 { - crate::bail!("vec_dot_q4k_q8k: {n} is not divisible by {QK_K}") - } - let mut utmp = [0u32; 4]; - const KMASK1: u32 = 0x3f3f3f3f; - const KMASK2: u32 = 0x0f0f0f0f; - const KMASK3: u32 = 0x03030303; - - unsafe { - let m4 = _mm256_set1_epi8(0xF); - - let mut acc = _mm256_setzero_ps(); - let mut acc_m = _mm_setzero_ps(); - - for (x, y) in xs.iter().zip(ys.iter()) { - let d = y.d * x.d.to_f32(); - let dmin = -y.d * x.dmin.to_f32(); - - LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); - - utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); - let uaux = utmp[1] & KMASK1; - utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); - utmp[2] = uaux; - utmp[0] &= KMASK1; - - let mut q4 = x.qs.as_ptr(); - let mut q8 = y.qs.as_ptr(); - - let mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32( - utmp[3] as i32, - utmp[2] as i32, - utmp[1] as i32, - utmp[0] as i32, - )); - - let q8sums = _mm256_loadu_si256(y.bsums.as_ptr() as *const __m256i); - let q8s = _mm_hadd_epi16( - _mm256_extracti128_si256(q8sums, 0), - _mm256_extracti128_si256(q8sums, 1), - ); - let prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s); - acc_m = _mm_fmadd_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod), acc_m); - - let sc128 = _mm256_extracti128_si256(mins_and_scales, 0); - let scales = mm256_set_m128i(sc128, sc128); - - let mut sumi = _mm256_setzero_si256(); - - for j in 0..QK_K / 64 { - let scale_l = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2 * j)); - let scale_h = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2 * j + 1)); - - let q4bits = _mm256_loadu_si256(q4 as *const __m256i); - q4 = q4.add(32); - let q4l = _mm256_and_si256(q4bits, m4); - let q4h = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), m4); - - let q8l = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let p16l = _mm256_maddubs_epi16(q4l, q8l); - let p16l = _mm256_madd_epi16(scale_l, p16l); - sumi = _mm256_add_epi32(sumi, p16l); - - let q8h = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let p16h = _mm256_maddubs_epi16(q4h, q8h); - let p16h = _mm256_madd_epi16(scale_h, p16h); - sumi = _mm256_add_epi32(sumi, p16h); - } - - let vd = _mm256_set1_ps(d); - acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc); - } - - let acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m)); - let acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m)); - - Ok(hsum_float_8(acc) + _mm_cvtss_f32(acc_m)) - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q5k_q8k(n: usize, xs: &[BlockQ5K], ys: &[BlockQ8K]) -> Result { - if n % QK_K != 0 { - crate::bail!("vec_dot_q5k_q8k: {n} is not divisible by {QK_K}") - } - let mut utmp = [0u32; 4]; - const KMASK1: u32 = 0x3f3f3f3f; - const KMASK2: u32 = 0x0f0f0f0f; - const KMASK3: u32 = 0x03030303; - - unsafe { - let m4 = _mm256_set1_epi8(0xF); - let mzero = _mm_setzero_si128(); - let mone = _mm256_set1_epi8(1); - - let mut acc = _mm256_setzero_ps(); - let mut summs = 0.0; - - for (x, y) in xs.iter().zip(ys.iter()) { - let d = y.d * x.d.to_f32(); - let dmin = -y.d * x.dmin.to_f32(); - - LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); - - utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); - let uaux = utmp[1] & KMASK1; - utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); - utmp[2] = uaux; - utmp[0] &= KMASK1; - - let mut q5 = x.qs.as_ptr(); - let mut q8 = y.qs.as_ptr(); - - let mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32( - utmp[3] as i32, - utmp[2] as i32, - utmp[1] as i32, - utmp[0] as i32, - )); - - let q8sums = _mm256_loadu_si256(y.bsums.as_ptr() as *const __m256i); - let q8s = _mm_hadd_epi16( - _mm256_extracti128_si256(q8sums, 0), - _mm256_extracti128_si256(q8sums, 1), - ); - let prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s); - let hsum = _mm_hadd_epi32(_mm_hadd_epi32(prod, mzero), mzero); - summs += dmin * _mm_extract_epi32(hsum, 0) as f32; - - let sc128 = _mm256_extracti128_si256(mins_and_scales, 0); - let scales = mm256_set_m128i(sc128, sc128); - - let hbits = _mm256_loadu_si256(x.qh.as_ptr() as *const __m256i); - let mut hmask = mone; - - let mut sumi = _mm256_setzero_si256(); - - for j in 0..QK_K / 64 { - let scale_0 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2 * j)); - let scale_1 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2 * j + 1)); - - let q5bits = _mm256_loadu_si256(q5 as *const __m256i); - q5 = q5.add(32); - - //Similar to q3k we hardcode the shifts here to avoid loading them into a separate register - let q5l_0 = _mm256_and_si256(q5bits, m4); - let q5l_0_shift_input = _mm256_and_si256(hbits, hmask); - let q5l_0_right_shift = match j { - 0 => _mm256_srli_epi16(q5l_0_shift_input, 0), - 1 => _mm256_srli_epi16(q5l_0_shift_input, 2), - 2 => _mm256_srli_epi16(q5l_0_shift_input, 4), - 3 => _mm256_srli_epi16(q5l_0_shift_input, 6), - _ => unreachable!(), - }; - let q5h_0 = _mm256_slli_epi16(q5l_0_right_shift, 4); - let q5_0 = _mm256_add_epi8(q5l_0, q5h_0); - hmask = _mm256_slli_epi16(hmask, 1); - - let q5l_1 = _mm256_and_si256(_mm256_srli_epi16(q5bits, 4), m4); - let q5l_1_shift_input = _mm256_and_si256(hbits, hmask); - let q5l_1_right_shift = match j { - 0 => _mm256_srli_epi16(q5l_1_shift_input, 1), - 1 => _mm256_srli_epi16(q5l_1_shift_input, 3), - 2 => _mm256_srli_epi16(q5l_1_shift_input, 5), - 3 => _mm256_srli_epi16(q5l_1_shift_input, 7), - _ => unreachable!(), - }; - - let q5h_1 = _mm256_slli_epi16(q5l_1_right_shift, 4); - let q5_1 = _mm256_add_epi8(q5l_1, q5h_1); - hmask = _mm256_slli_epi16(hmask, 1); - - let q8_0 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let q8_1 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - - let p16_0 = _mm256_maddubs_epi16(q5_0, q8_0); - let p16_1 = _mm256_maddubs_epi16(q5_1, q8_1); - - let p16_0 = _mm256_madd_epi16(scale_0, p16_0); - let p16_1 = _mm256_madd_epi16(scale_1, p16_1); - - sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1)); - } - let vd = _mm256_set1_ps(d); - acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc); - } - Ok(hsum_float_8(acc) + summs) - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q8k_q8k(n: usize, xs: &[BlockQ8K], ys: &[BlockQ8K]) -> Result { - let qk = QK_K; - if n % qk != 0 { - crate::bail!("vec_dot_q8k_8k: {n} is not divisible by {qk}") - } - - unsafe { - let mut acc = _mm256_setzero_ps(); - for (xs, ys) in xs.iter().zip(ys.iter()) { - let mut sumi = _mm256_setzero_si256(); - let x_qs = xs.qs.as_ptr(); - let y_qs = ys.qs.as_ptr(); - for j in (0..QK_K).step_by(32) { - let xs = _mm256_loadu_si256(x_qs.add(j) as *const __m256i); - let ys = _mm256_loadu_si256(y_qs.add(j) as *const __m256i); - - let xs0 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(xs, 0)); - let ys0 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(ys, 0)); - sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(xs0, ys0)); - - let xs1 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(xs, 1)); - let ys1 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(ys, 1)); - sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(xs1, ys1)); - } - let d = _mm256_set1_ps(xs.d * ys.d); - acc = _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(sumi), acc); - } - Ok(hsum_float_8(acc)) - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/cuda.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/cuda.rs deleted file mode 100644 index 84af483d32..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/quantized/cuda.rs +++ /dev/null @@ -1,321 +0,0 @@ -use super::{GgmlDType, QStorage}; -use crate::{backend::BackendDevice, cuda_backend::WrapErr}; -use crate::{CudaDevice, CudaStorage, Result}; - -use cudarc::driver::{CudaSlice, DeviceSlice}; - -pub struct QCudaStorage { - data: CudaSlice, - dtype: GgmlDType, - device: CudaDevice, -} - -pub const WARP_SIZE: usize = 32; -pub const MMQ_X_Q4_0_AMPERE: usize = 4; -pub const MMQ_Y_Q4_0_AMPERE: usize = 32; -pub const NWARPS_Q4_0_AMPERE: usize = 4; -pub const GGML_CUDA_MMV_X: usize = 32; -pub const GGML_CUDA_MMV_Y: usize = 1; - -fn dequantize( - data: &CudaSlice, - dtype: GgmlDType, - elem_count: usize, - dev: &CudaDevice, -) -> Result { - use cudarc::driver::LaunchAsync; - - let (kernel_name, is_k, block_dim) = match dtype { - GgmlDType::Q4_0 => ("dequantize_block_q4_0", false, 32), - GgmlDType::Q4_1 => ("dequantize_block_q4_1", false, 32), - GgmlDType::Q5_0 => ("dequantize_block_q5_0", false, 32), - GgmlDType::Q5_1 => ("dequantize_block_q5_1", false, 32), - GgmlDType::Q8_0 => ("dequantize_block_q8_0", false, 32), - GgmlDType::Q2K => ("dequantize_block_q2_K", true, 64), - GgmlDType::Q3K => ("dequantize_block_q3_K", true, 64), - GgmlDType::Q4K => ("dequantize_block_q4_K", true, 32), - GgmlDType::Q5K => ("dequantize_block_q5_K", true, 64), - GgmlDType::Q6K => ("dequantize_block_q6_K", true, 64), - GgmlDType::Q8K => ("dequantize_block_q8_K", true, 32), - _ => crate::bail!("unsupported dtype for dequantize {dtype:?}"), - }; - let func = dev.get_or_load_func(kernel_name, candle_kernels::QUANTIZED)?; - let dst = dev.alloc_zeros::(elem_count).w()?; - let nb = (elem_count + 255) / 256; - // See e.g. - // https://github.com/ggerganov/llama.cpp/blob/cbbd1efa06f8c09f9dff58ff9d9af509cc4c152b/ggml-cuda.cu#L7270 - let cfg = cudarc::driver::LaunchConfig { - grid_dim: (nb as u32, 1, 1), - block_dim: (block_dim, 1, 1), - shared_mem_bytes: 0, - }; - - if is_k { - let params = (data, &dst); - unsafe { func.launch(cfg, params) }.w()?; - } else { - let nb32 = elem_count / 32; - let params = (data, &dst, nb32 as i32); - unsafe { func.launch(cfg, params) }.w()?; - } - Ok(CudaStorage::wrap_cuda_slice(dst, dev.clone())) -} - -fn dequantize_mut_mal_vec( - data: &CudaSlice, - y: &cudarc::driver::CudaView, - dtype: GgmlDType, - ncols: usize, - nrows: usize, - dev: &CudaDevice, -) -> Result { - use cudarc::driver::LaunchAsync; - - let kernel_name = match dtype { - GgmlDType::Q4_0 => "dequantize_mul_mat_vec_q4_0_cuda", - GgmlDType::Q4_1 => "dequantize_mul_mat_vec_q4_1_cuda", - GgmlDType::Q5_0 => "dequantize_mul_mat_vec_q5_0_cuda", - GgmlDType::Q5_1 => "dequantize_mul_mat_vec_q5_1_cuda", - GgmlDType::Q8_0 => "dequantize_mul_mat_vec_q8_0_cuda", - GgmlDType::Q2K => "dequantize_mul_mat_vec_q2_k", - GgmlDType::Q3K => "dequantize_mul_mat_vec_q3_k", - GgmlDType::Q4K => "dequantize_mul_mat_vec_q4_k", - GgmlDType::Q5K => "dequantize_mul_mat_vec_q5_k", - GgmlDType::Q6K => "dequantize_mul_mat_vec_q6_k", - _ => crate::bail!("unsupported dtype for quantized matmul {dtype:?}"), - }; - let func = dev.get_or_load_func(kernel_name, candle_kernels::QUANTIZED)?; - let dst = dev.alloc_zeros::(nrows).w()?; - let block_num_y = (nrows + GGML_CUDA_MMV_Y - 1) / GGML_CUDA_MMV_Y; - let cfg = cudarc::driver::LaunchConfig { - grid_dim: (block_num_y as u32, 1, 1), - block_dim: (WARP_SIZE as u32, GGML_CUDA_MMV_Y as u32, 1), - shared_mem_bytes: 0, - }; - - let params = (data, y, &dst, ncols as i32, nrows as i32); - unsafe { func.launch(cfg, params) }.w()?; - Ok(CudaStorage::wrap_cuda_slice(dst, dev.clone())) -} - -impl QCudaStorage { - pub fn zeros(device: &CudaDevice, el_count: usize, dtype: GgmlDType) -> Result { - let size_in_bytes = el_count * dtype.type_size() / dtype.block_size(); - let data = device.alloc_zeros::(size_in_bytes).w()?; - Ok(QCudaStorage { - data, - device: device.clone(), - dtype, - }) - } - - pub fn dtype(&self) -> GgmlDType { - self.dtype - } - - pub fn device(&self) -> &CudaDevice { - &self.device - } - - pub fn dequantize(&self, elem_count: usize) -> Result { - let fast_kernel = matches!( - self.dtype, - GgmlDType::Q4_0 - | GgmlDType::Q4_1 - | GgmlDType::Q5_0 - | GgmlDType::Q5_1 - | GgmlDType::Q8_0 - | GgmlDType::Q2K - | GgmlDType::Q3K - | GgmlDType::Q4K - | GgmlDType::Q5K - | GgmlDType::Q6K - | GgmlDType::Q8K - ); - if fast_kernel { - return dequantize(&self.data, self.dtype, elem_count, self.device()); - } - // Run the dequantization on cpu. - use crate::quantized::k_quants::GgmlType; - - let buffer = self.device.dtoh_sync_copy(&self.data).w()?; - let mut out = vec![0.0; elem_count]; - let block_len = elem_count / self.dtype.block_size(); - match self.dtype { - GgmlDType::F32 => { - let slice = - unsafe { std::slice::from_raw_parts(buffer.as_ptr() as *const f32, block_len) }; - out.copy_from_slice(slice) - } - GgmlDType::F16 => { - let vec: Vec = read_to_vec(&buffer, block_len); - half::f16::to_float(&vec, &mut out)?; - } - GgmlDType::Q4_0 => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ4_0::to_float(&vec, &mut out)?; - } - GgmlDType::Q4_1 => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ4_1::to_float(&vec, &mut out)?; - } - GgmlDType::Q5_0 => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ5_0::to_float(&vec, &mut out)?; - } - GgmlDType::Q5_1 => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ5_1::to_float(&vec, &mut out)?; - } - GgmlDType::Q8_0 => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ8_0::to_float(&vec, &mut out)?; - } - GgmlDType::Q8_1 => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ8_1::to_float(&vec, &mut out)?; - } - GgmlDType::Q2K => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ2K::to_float(&vec, &mut out)?; - } - GgmlDType::Q3K => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ3K::to_float(&vec, &mut out)?; - } - GgmlDType::Q4K => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ4K::to_float(&vec, &mut out)?; - } - GgmlDType::Q5K => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ5K::to_float(&vec, &mut out)?; - } - GgmlDType::Q6K => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ6K::to_float(&vec, &mut out)?; - } - GgmlDType::Q8K => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ8K::to_float(&vec, &mut out)?; - } - } - - self.device - .storage_from_cpu_storage(&crate::CpuStorage::F32(out)) - } - - pub fn quantize(&mut self, src: &CudaStorage) -> Result<()> { - // Run the quantization on cpu. - let src = match &src.slice { - crate::cuda_backend::CudaStorageSlice::F32(data) => { - self.device.dtoh_sync_copy(data).w()? - } - _ => crate::bail!("only f32 can be quantized"), - }; - let src_len = src.len(); - let src = crate::Storage::Cpu(crate::CpuStorage::F32(src)); - let mut qcpu_storage = crate::Device::Cpu.qzeros(src_len, self.dtype)?; - qcpu_storage.quantize(&src)?; - let data = qcpu_storage.data()?; - let data = self.device.htod_sync_copy(data.as_ref()).w()?; - self.data = data; - Ok(()) - } - - pub fn storage_size_in_bytes(&self) -> usize { - self.data.len() - } - - pub fn fwd( - &self, - self_shape: &crate::Shape, - storage: &CudaStorage, - layout: &crate::Layout, - ) -> Result<(CudaStorage, crate::Shape)> { - if matches!(layout.shape().dims(), [1, 1, _] | [1, _]) { - self.dequantize_matmul_vec(self_shape, storage, layout) - } else { - self.dequantize_matmul(self_shape, storage, layout) - } - } -} - -impl QCudaStorage { - fn dequantize_matmul_vec( - &self, - self_shape: &crate::Shape, - rhs: &CudaStorage, - rhs_l: &crate::Layout, - ) -> Result<(CudaStorage, crate::Shape)> { - let (nrows, ncols) = self_shape.dims2()?; - let rhs = rhs.as_cuda_slice::()?; - let rhs = match rhs_l.contiguous_offsets() { - Some((o1, o2)) => rhs.slice(o1..o2), - None => Err(crate::Error::RequiresContiguous { op: "dmmv" }.bt())?, - }; - let (with_batch, k) = match rhs_l.shape().dims() { - [1, 1, k] => (true, k), - [1, k] => (false, k), - _ => crate::bail!("unexpected rhs shape in dmmv {:?}", rhs_l.shape()), - }; - if ncols != *k { - crate::bail!("mismatch on matmul dim {self_shape:?} {:?}", rhs_l.shape()) - } - - let out = - dequantize_mut_mal_vec(&self.data, &rhs, self.dtype, ncols, nrows, self.device())?; - let out_shape = if with_batch { - vec![1, 1, nrows] - } else { - vec![1, nrows] - }; - Ok((out, out_shape.into())) - } - - fn dequantize_matmul( - &self, - self_shape: &crate::Shape, - storage: &CudaStorage, - layout: &crate::Layout, - ) -> Result<(CudaStorage, crate::Shape)> { - use crate::backend::BackendStorage; - let (n, k) = self_shape.dims2()?; - let (b, m, k2) = match layout.shape().dims() { - &[b, m, k2] => (b, m, k2), - &[m, k2] => (1, m, k2), - s => crate::bail!("unexpected shape for input {s:?}"), - }; - if k2 != k { - crate::bail!("mismatch on matmul dim {self_shape:?} {:?}", layout.shape()) - } - - let data_f32 = self.dequantize(n * k)?; - let rhs_l = crate::Layout::new((k, n).into(), vec![1, k], 0); - let out = storage.matmul(&data_f32, (b, m, n, k), layout, &rhs_l)?; - let mut out_shape = layout.shape().dims().to_vec(); - out_shape.pop(); - out_shape.push(n); - Ok((out, out_shape.into())) - } -} - -fn read_to_vec(buffer: &[u8], n: usize) -> Vec { - let slice = unsafe { std::slice::from_raw_parts(buffer.as_ptr() as *const T, n) }; - slice.to_vec() -} - -pub fn load_quantized( - device: &CudaDevice, - data: &[T], -) -> Result { - let data = unsafe { - std::slice::from_raw_parts(data.as_ptr() as *const u8, core::mem::size_of_val(data)) - }; - let data = device.htod_sync_copy(data).w()?; - Ok(QStorage::Cuda(QCudaStorage { - data, - device: device.clone(), - dtype: T::DTYPE, - })) -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/dummy_cuda.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/dummy_cuda.rs deleted file mode 100644 index 598c5cd131..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/quantized/dummy_cuda.rs +++ /dev/null @@ -1,50 +0,0 @@ -#![allow(unused)] -use super::GgmlDType; -use crate::{CudaDevice, CudaStorage, Error, Result}; - -pub struct QCudaStorage { - dtype: GgmlDType, - device: CudaDevice, -} - -impl QCudaStorage { - pub fn zeros(_: &CudaDevice, _: usize, _: GgmlDType) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - pub fn dtype(&self) -> GgmlDType { - self.dtype - } - - pub fn device(&self) -> &CudaDevice { - &self.device - } - - pub fn dequantize(&self, _elem_count: usize) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - pub fn quantize(&mut self, _src: &CudaStorage) -> Result<()> { - Err(Error::NotCompiledWithCudaSupport) - } - - pub fn storage_size_in_bytes(&self) -> usize { - 0 - } - - pub fn fwd( - &self, - _self_shape: &crate::Shape, - _storage: &CudaStorage, - _layout: &crate::Layout, - ) -> Result<(CudaStorage, crate::Shape)> { - Err(Error::NotCompiledWithCudaSupport) - } -} - -pub fn load_quantized( - _device: &CudaDevice, - _data: &[T], -) -> Result { - Err(Error::NotCompiledWithCudaSupport) -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/dummy_metal.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/dummy_metal.rs deleted file mode 100644 index 520d0ed49a..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/quantized/dummy_metal.rs +++ /dev/null @@ -1,50 +0,0 @@ -#![allow(unused)] -use super::GgmlDType; -use crate::{Error, MetalDevice, MetalStorage, Result}; - -pub struct QMetalStorage { - dtype: GgmlDType, - device: MetalDevice, -} - -impl QMetalStorage { - pub fn zeros(_: &MetalDevice, _: usize, _: GgmlDType) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - pub fn dtype(&self) -> GgmlDType { - self.dtype - } - - pub fn device(&self) -> &MetalDevice { - &self.device - } - - pub fn dequantize(&self, _elem_count: usize) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - pub fn quantize(&mut self, _src: &MetalStorage) -> Result<()> { - Err(Error::NotCompiledWithMetalSupport) - } - - pub fn storage_size_in_bytes(&self) -> usize { - 0 - } - - pub fn fwd( - &self, - _self_shape: &crate::Shape, - _storage: &MetalStorage, - _layout: &crate::Layout, - ) -> Result<(MetalStorage, crate::Shape)> { - Err(Error::NotCompiledWithMetalSupport) - } -} - -pub fn load_quantized( - _device: &MetalDevice, - _data: &[T], -) -> Result { - Err(Error::NotCompiledWithMetalSupport) -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/ggml_file.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/ggml_file.rs deleted file mode 100644 index 99200bbd06..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/quantized/ggml_file.rs +++ /dev/null @@ -1,265 +0,0 @@ -//! Support for the GGML file format. - -use super::{k_quants, GgmlDType, QStorage}; -use crate::{Device, Result}; -use byteorder::{LittleEndian, ReadBytesExt}; -use std::collections::HashMap; - -// https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/llama.h#L37 -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Magic { - Ggjt, - Ggla, - Ggmf, - Ggml, - Ggsn, -} - -impl TryFrom for Magic { - type Error = crate::Error; - fn try_from(value: u32) -> Result { - let magic = match value { - 0x67676a74 => Self::Ggjt, - 0x67676c61 => Self::Ggla, - 0x67676d66 => Self::Ggmf, - 0x67676d6c => Self::Ggml, - 0x6767736e => Self::Ggsn, - _ => crate::bail!("unknown magic {value:08x}"), - }; - Ok(magic) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum VersionedMagic { - GgmlUnversioned, - GgmfV1, - GgjtV1, - GgjtV2, - GgjtV3, -} - -impl VersionedMagic { - fn read(reader: &mut R) -> Result { - let magic = reader.read_u32::()?; - let magic = Magic::try_from(magic)?; - if magic == Magic::Ggml { - return Ok(Self::GgmlUnversioned); - } - let version = reader.read_u32::()?; - let versioned_magic = match (magic, version) { - (Magic::Ggmf, 1) => Self::GgmfV1, - (Magic::Ggjt, 1) => Self::GgjtV1, - (Magic::Ggjt, 2) => Self::GgjtV2, - (Magic::Ggjt, 3) => Self::GgjtV3, - _ => crate::bail!("ggml: unsupported magic/version {magic:?}/{version}"), - }; - Ok(versioned_magic) - } - - fn align32(&self) -> bool { - match self { - Self::GgmlUnversioned | Self::GgmfV1 => false, - Self::GgjtV1 | Self::GgjtV2 | Self::GgjtV3 => true, - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct HParams { - pub n_vocab: u32, - pub n_embd: u32, - pub n_mult: u32, - pub n_head: u32, - pub n_layer: u32, - pub n_rot: u32, - pub ftype: u32, -} - -impl HParams { - fn read(reader: &mut R) -> Result { - let n_vocab = reader.read_u32::()?; - let n_embd = reader.read_u32::()?; - let n_mult = reader.read_u32::()?; - let n_head = reader.read_u32::()?; - let n_layer = reader.read_u32::()?; - let n_rot = reader.read_u32::()?; - let ftype = reader.read_u32::()?; - Ok(Self { - n_vocab, - n_embd, - n_mult, - n_head, - n_layer, - n_rot, - ftype, - }) - } -} - -#[derive(Debug, Clone, PartialEq)] -pub struct Vocab { - pub token_score_pairs: Vec<(Vec, f32)>, -} - -impl Vocab { - fn read(reader: &mut R, n_vocab: usize) -> Result { - // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/llama.cpp#L556 - let mut token_score_pairs = Vec::with_capacity(n_vocab); - for _index in 0..n_vocab { - let len = reader.read_u32::()? as usize; - let mut word = vec![0u8; len]; - reader.read_exact(&mut word)?; - let score = reader.read_f32::()?; - token_score_pairs.push((word, score)) - } - Ok(Self { token_score_pairs }) - } -} - -fn from_raw_data( - raw_data: &[u8], - size_in_bytes: usize, - dims: Vec, - device: &Device, -) -> Result { - let raw_data_ptr = raw_data.as_ptr(); - let n_blocks = size_in_bytes / std::mem::size_of::(); - let data = unsafe { std::slice::from_raw_parts(raw_data_ptr as *const T, n_blocks) }; - let data: QStorage = match device { - Device::Cpu => QStorage::Cpu(Box::new(data.to_vec())), - Device::Metal(metal) => super::metal::load_quantized(metal, data)?, - Device::Cuda(cuda) => super::cuda::load_quantized(cuda, data)?, - }; - super::QTensor::new(data, dims) -} - -/// Creates a [Tensor] from a raw GGML tensor. -pub fn qtensor_from_ggml( - ggml_dtype: GgmlDType, - raw_data: &[u8], - dims: Vec, - device: &Device, -) -> Result { - let tensor_elems = dims.iter().product::(); - let block_size = ggml_dtype.block_size(); - if tensor_elems % block_size != 0 { - crate::bail!( - "the number of elements {tensor_elems} is not divisible by the block size {block_size}" - ) - } - let size_in_bytes = tensor_elems / block_size * ggml_dtype.type_size(); - - match ggml_dtype { - GgmlDType::F32 => from_raw_data::(raw_data, size_in_bytes, dims, device), - GgmlDType::F16 => from_raw_data::(raw_data, size_in_bytes, dims, device), - GgmlDType::Q4_0 => { - from_raw_data::(raw_data, size_in_bytes, dims, device) - } - GgmlDType::Q4_1 => { - from_raw_data::(raw_data, size_in_bytes, dims, device) - } - GgmlDType::Q5_0 => { - from_raw_data::(raw_data, size_in_bytes, dims, device) - } - GgmlDType::Q5_1 => { - from_raw_data::(raw_data, size_in_bytes, dims, device) - } - GgmlDType::Q8_0 => { - from_raw_data::(raw_data, size_in_bytes, dims, device) - } - GgmlDType::Q2K => { - from_raw_data::(raw_data, size_in_bytes, dims, device) - } - GgmlDType::Q3K => { - from_raw_data::(raw_data, size_in_bytes, dims, device) - } - GgmlDType::Q4K => { - from_raw_data::(raw_data, size_in_bytes, dims, device) - } - GgmlDType::Q5K => { - from_raw_data::(raw_data, size_in_bytes, dims, device) - } - GgmlDType::Q6K => { - from_raw_data::(raw_data, size_in_bytes, dims, device) - } - _ => crate::bail!("quantized type {ggml_dtype:?} is not supported yet"), - } -} - -fn read_one_tensor( - reader: &mut R, - magic: VersionedMagic, - device: &Device, -) -> Result<(String, super::QTensor)> { - let n_dims = reader.read_u32::()?; - let name_len = reader.read_u32::()?; - let ggml_dtype = reader.read_u32::()?; - let ggml_dtype = GgmlDType::from_u32(ggml_dtype)?; - let mut dims = vec![0u32; n_dims as usize]; - reader.read_u32_into::(&mut dims)?; - // The dimensions are stored in reverse order, see for example: - // https://github.com/ggerganov/llama.cpp/blob/b5ffb2849d23afe73647f68eec7b68187af09be6/convert.py#L969 - dims.reverse(); - let mut name = vec![0u8; name_len as usize]; - reader.read_exact(&mut name)?; - let name = String::from_utf8_lossy(&name).into_owned(); - - if magic.align32() { - let pos = reader.stream_position()?; - reader.seek(std::io::SeekFrom::Current(((32 - pos % 32) % 32) as i64))?; - } - let dims = dims.iter().map(|&u| u as usize).collect::>(); - let tensor_elems = dims.iter().product::(); - let size_in_bytes = tensor_elems * ggml_dtype.type_size() / ggml_dtype.block_size(); - // TODO: Mmap version to avoid copying the data around? - let mut raw_data = vec![0u8; size_in_bytes]; - reader.read_exact(&mut raw_data)?; - match qtensor_from_ggml(ggml_dtype, &raw_data, dims, device) { - Ok(tensor) => Ok((name, tensor)), - Err(e) => crate::bail!("Error creating tensor {name}: {e}"), - } -} - -pub struct Content { - pub magic: VersionedMagic, - pub hparams: HParams, - pub vocab: Vocab, - pub tensors: HashMap, - pub device: Device, -} - -impl Content { - pub fn read( - reader: &mut R, - device: &Device, - ) -> Result { - // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/llama.cpp#L505 - let last_position = reader.seek(std::io::SeekFrom::End(0))?; - reader.seek(std::io::SeekFrom::Start(0))?; - let magic = VersionedMagic::read(reader)?; - let hparams = HParams::read(reader)?; - let vocab = Vocab::read(reader, hparams.n_vocab as usize)?; - let mut tensors = HashMap::new(); - - while reader.stream_position()? != last_position { - let (name, tensor) = read_one_tensor(reader, magic, device)?; - tensors.insert(name, tensor); - } - let device = device.clone(); - Ok(Self { - magic, - hparams, - vocab, - tensors, - device, - }) - } - - pub fn remove(&mut self, name: &str) -> Result { - match self.tensors.remove(name) { - None => crate::bail!("cannot find tensor with name '{name}'"), - Some(tensor) => Ok(tensor), - } - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/gguf_file.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/gguf_file.rs deleted file mode 100644 index b729d4a0fd..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/quantized/gguf_file.rs +++ /dev/null @@ -1,534 +0,0 @@ -//! Support for the GGUF file format. -//! -//! Spec: https://github.com/philpax/ggml/blob/gguf-spec/docs/gguf.md - -use super::{GgmlDType, QTensor}; -use crate::{Device, Result}; -use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; -use std::collections::HashMap; - -pub const DEFAULT_ALIGNMENT: u64 = 32; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Magic { - Gguf, -} - -impl TryFrom for Magic { - type Error = crate::Error; - fn try_from(value: u32) -> Result { - let magic = match value { - 0x46554747 | 0x47475546 => Self::Gguf, - _ => crate::bail!("unknown magic 0x{value:08x}"), - }; - Ok(magic) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum VersionedMagic { - GgufV1, - GgufV2, - GgufV3, -} - -impl VersionedMagic { - fn read(reader: &mut R) -> Result { - let magic = reader.read_u32::()?; - let magic = Magic::try_from(magic)?; - let version = reader.read_u32::()?; - let versioned_magic = match (magic, version) { - (Magic::Gguf, 1) => Self::GgufV1, - (Magic::Gguf, 2) => Self::GgufV2, - (Magic::Gguf, 3) => Self::GgufV3, - _ => crate::bail!("gguf: unsupported magic/version {magic:?}/{version}"), - }; - Ok(versioned_magic) - } -} - -#[derive(Debug)] -pub struct TensorInfo { - pub ggml_dtype: GgmlDType, - pub shape: crate::Shape, - pub offset: u64, -} - -impl TensorInfo { - pub fn read( - &self, - reader: &mut R, - tensor_data_offset: u64, - device: &Device, - ) -> Result { - let tensor_elems = self.shape.elem_count(); - let block_size = self.ggml_dtype.block_size(); - if tensor_elems % block_size != 0 { - crate::bail!( - "the number of elements {tensor_elems} is not divisible by the block size {block_size}" - ) - } - let size_in_bytes = tensor_elems / block_size * self.ggml_dtype.type_size(); - let mut raw_data = vec![0u8; size_in_bytes]; - reader.seek(std::io::SeekFrom::Start(tensor_data_offset + self.offset))?; - reader.read_exact(&mut raw_data)?; - super::ggml_file::qtensor_from_ggml( - self.ggml_dtype, - &raw_data, - self.shape.dims().to_vec(), - device, - ) - } -} - -#[derive(Debug)] -pub struct Content { - pub magic: VersionedMagic, - pub metadata: HashMap, - pub tensor_infos: HashMap, - pub tensor_data_offset: u64, -} - -fn read_string(reader: &mut R, magic: &VersionedMagic) -> Result { - let len = match magic { - VersionedMagic::GgufV1 => reader.read_u32::()? as usize, - VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => { - reader.read_u64::()? as usize - } - }; - let mut v = vec![0u8; len]; - reader.read_exact(&mut v)?; - // GGUF strings are supposed to be non-null terminated but in practice this happens. - while let Some(0) = v.last() { - v.pop(); - } - // GGUF strings are utf8 encoded but there are cases that don't seem to be valid. - Ok(String::from_utf8_lossy(&v).into_owned()) -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum ValueType { - // The value is a 8-bit unsigned integer. - U8, - // The value is a 8-bit signed integer. - I8, - // The value is a 16-bit unsigned little-endian integer. - U16, - // The value is a 16-bit signed little-endian integer. - I16, - // The value is a 32-bit unsigned little-endian integer. - U32, - // The value is a 32-bit signed little-endian integer. - I32, - // The value is a 64-bit unsigned little-endian integer. - U64, - // The value is a 64-bit signed little-endian integer. - I64, - // The value is a 32-bit IEEE754 floating point number. - F32, - // The value is a 64-bit IEEE754 floating point number. - F64, - // The value is a boolean. - // 1-byte value where 0 is false and 1 is true. - // Anything else is invalid, and should be treated as either the model being invalid or the reader being buggy. - Bool, - // The value is a UTF-8 non-null-terminated string, with length prepended. - String, - // The value is an array of other values, with the length and type prepended. - /// - // Arrays can be nested, and the length of the array is the number of elements in the array, not the number of bytes. - Array, -} - -#[derive(Debug, Clone)] -pub enum Value { - U8(u8), - I8(i8), - U16(u16), - I16(i16), - U32(u32), - I32(i32), - U64(u64), - I64(i64), - F32(f32), - F64(f64), - Bool(bool), - String(String), - Array(Vec), -} - -impl Value { - pub fn value_type(&self) -> ValueType { - match self { - Self::U8(_) => ValueType::U8, - Self::I8(_) => ValueType::I8, - Self::U16(_) => ValueType::U16, - Self::I16(_) => ValueType::I16, - Self::U32(_) => ValueType::U32, - Self::I32(_) => ValueType::I32, - Self::U64(_) => ValueType::U64, - Self::I64(_) => ValueType::I64, - Self::F32(_) => ValueType::F32, - Self::F64(_) => ValueType::F64, - Self::Bool(_) => ValueType::Bool, - Self::String(_) => ValueType::String, - Self::Array(_) => ValueType::Array, - } - } - - pub fn to_u8(&self) -> Result { - match self { - Self::U8(v) => Ok(*v), - v => crate::bail!("not a u8 {v:?}"), - } - } - - pub fn to_i8(&self) -> Result { - match self { - Self::I8(v) => Ok(*v), - v => crate::bail!("not a i8 {v:?}"), - } - } - - pub fn to_u16(&self) -> Result { - match self { - Self::U16(v) => Ok(*v), - v => crate::bail!("not a u16 {v:?}"), - } - } - - pub fn to_i16(&self) -> Result { - match self { - Self::I16(v) => Ok(*v), - v => crate::bail!("not a i16 {v:?}"), - } - } - - pub fn to_u32(&self) -> Result { - match self { - Self::U32(v) => Ok(*v), - v => crate::bail!("not a u32 {v:?}"), - } - } - - pub fn to_i32(&self) -> Result { - match self { - Self::I32(v) => Ok(*v), - v => crate::bail!("not a i32 {v:?}"), - } - } - - pub fn to_u64(&self) -> Result { - match self { - Self::U64(v) => Ok(*v), - v => crate::bail!("not a u64 {v:?}"), - } - } - - pub fn to_i64(&self) -> Result { - match self { - Self::I64(v) => Ok(*v), - v => crate::bail!("not a i64 {v:?}"), - } - } - - pub fn to_f32(&self) -> Result { - match self { - Self::F32(v) => Ok(*v), - v => crate::bail!("not a f32 {v:?}"), - } - } - - pub fn to_f64(&self) -> Result { - match self { - Self::F64(v) => Ok(*v), - v => crate::bail!("not a f64 {v:?}"), - } - } - - pub fn to_bool(&self) -> Result { - match self { - Self::Bool(v) => Ok(*v), - v => crate::bail!("not a bool {v:?}"), - } - } - - pub fn to_vec(&self) -> Result<&Vec> { - match self { - Self::Array(v) => Ok(v), - v => crate::bail!("not a vec {v:?}"), - } - } - - pub fn to_string(&self) -> Result<&String> { - match self { - Self::String(v) => Ok(v), - v => crate::bail!("not a string {v:?}"), - } - } - - fn read( - reader: &mut R, - value_type: ValueType, - magic: &VersionedMagic, - ) -> Result { - let v = match value_type { - ValueType::U8 => Self::U8(reader.read_u8()?), - ValueType::I8 => Self::I8(reader.read_i8()?), - ValueType::U16 => Self::U16(reader.read_u16::()?), - ValueType::I16 => Self::I16(reader.read_i16::()?), - ValueType::U32 => Self::U32(reader.read_u32::()?), - ValueType::I32 => Self::I32(reader.read_i32::()?), - ValueType::U64 => Self::U64(reader.read_u64::()?), - ValueType::I64 => Self::I64(reader.read_i64::()?), - ValueType::F32 => Self::F32(reader.read_f32::()?), - ValueType::F64 => Self::F64(reader.read_f64::()?), - ValueType::Bool => match reader.read_u8()? { - 0 => Self::Bool(false), - 1 => Self::Bool(true), - b => crate::bail!("unexpected bool value {b}"), - }, - ValueType::String => Self::String(read_string(reader, magic)?), - ValueType::Array => { - let value_type = reader.read_u32::()?; - let value_type = ValueType::from_u32(value_type)?; - let len = match magic { - VersionedMagic::GgufV1 => reader.read_u32::()? as usize, - VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => { - reader.read_u64::()? as usize - } - }; - let mut vs = Vec::with_capacity(len); - for _ in 0..len { - vs.push(Value::read(reader, value_type, magic)?) - } - Self::Array(vs) - } - }; - Ok(v) - } - - fn write(&self, w: &mut W) -> Result<()> { - match self { - &Self::U8(v) => w.write_u8(v)?, - &Self::I8(v) => w.write_i8(v)?, - &Self::U16(v) => w.write_u16::(v)?, - &Self::I16(v) => w.write_i16::(v)?, - &Self::U32(v) => w.write_u32::(v)?, - &Self::I32(v) => w.write_i32::(v)?, - &Self::U64(v) => w.write_u64::(v)?, - &Self::I64(v) => w.write_i64::(v)?, - &Self::F32(v) => w.write_f32::(v)?, - &Self::F64(v) => w.write_f64::(v)?, - &Self::Bool(v) => w.write_u8(u8::from(v))?, - Self::String(v) => write_string(w, v.as_str())?, - Self::Array(v) => { - // The `Value` type does not enforce that all the values in an Array have the same - // type. - let value_type = if v.is_empty() { - // Doesn't matter, the array is empty. - ValueType::U32 - } else { - let value_type: std::collections::HashSet<_> = - v.iter().map(|elem| elem.value_type()).collect(); - if value_type.len() != 1 { - crate::bail!("multiple value-types in the same array {value_type:?}") - } - value_type.into_iter().next().unwrap() - }; - w.write_u32::(value_type.to_u32())?; - w.write_u64::(v.len() as u64)?; - for elem in v.iter() { - elem.write(w)? - } - } - } - Ok(()) - } -} - -impl ValueType { - fn from_u32(v: u32) -> Result { - let v = match v { - 0 => Self::U8, - 1 => Self::I8, - 2 => Self::U16, - 3 => Self::I16, - 4 => Self::U32, - 5 => Self::I32, - 6 => Self::F32, - 7 => Self::Bool, - 8 => Self::String, - 9 => Self::Array, - 10 => Self::U64, - 11 => Self::I64, - 12 => Self::F64, - v => crate::bail!("unrecognized value-type {v:#08x}"), - }; - Ok(v) - } - - fn to_u32(self) -> u32 { - match self { - Self::U8 => 0, - Self::I8 => 1, - Self::U16 => 2, - Self::I16 => 3, - Self::U32 => 4, - Self::I32 => 5, - Self::F32 => 6, - Self::Bool => 7, - Self::String => 8, - Self::Array => 9, - Self::U64 => 10, - Self::I64 => 11, - Self::F64 => 12, - } - } -} - -impl Content { - pub fn read(reader: &mut R) -> Result { - let magic = VersionedMagic::read(reader)?; - - let tensor_count = match magic { - VersionedMagic::GgufV1 => reader.read_u32::()? as usize, - VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => { - reader.read_u64::()? as usize - } - }; - let metadata_kv_count = match magic { - VersionedMagic::GgufV1 => reader.read_u32::()? as usize, - VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => { - reader.read_u64::()? as usize - } - }; - - let mut metadata = HashMap::new(); - for _idx in 0..metadata_kv_count { - let key = read_string(reader, &magic)?; - let value_type = reader.read_u32::()?; - let value_type = ValueType::from_u32(value_type)?; - let value = Value::read(reader, value_type, &magic)?; - metadata.insert(key, value); - } - let mut tensor_infos = HashMap::new(); - for _idx in 0..tensor_count { - let tensor_name = read_string(reader, &magic)?; - let n_dimensions = reader.read_u32::()?; - - let mut dimensions: Vec = match magic { - VersionedMagic::GgufV1 => { - let mut dimensions = vec![0; n_dimensions as usize]; - reader.read_u32_into::(&mut dimensions)?; - dimensions.into_iter().map(|c| c as usize).collect() - } - VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => { - let mut dimensions = vec![0; n_dimensions as usize]; - reader.read_u64_into::(&mut dimensions)?; - dimensions.into_iter().map(|c| c as usize).collect() - } - }; - - dimensions.reverse(); - let ggml_dtype = reader.read_u32::()?; - let ggml_dtype = GgmlDType::from_u32(ggml_dtype)?; - let offset = reader.read_u64::()?; - tensor_infos.insert( - tensor_name, - TensorInfo { - shape: crate::Shape::from(dimensions), - offset, - ggml_dtype, - }, - ); - } - let position = reader.stream_position()?; - let alignment = match metadata.get("general.alignment") { - Some(Value::U8(v)) => *v as u64, - Some(Value::U16(v)) => *v as u64, - Some(Value::U32(v)) => *v as u64, - Some(Value::I8(v)) if *v >= 0 => *v as u64, - Some(Value::I16(v)) if *v >= 0 => *v as u64, - Some(Value::I32(v)) if *v >= 0 => *v as u64, - _ => DEFAULT_ALIGNMENT, - }; - let tensor_data_offset = (position + alignment - 1) / alignment * alignment; - Ok(Self { - magic, - metadata, - tensor_infos, - tensor_data_offset, - }) - } - - pub fn tensor( - &self, - reader: &mut R, - name: &str, - device: &Device, - ) -> Result { - let tensor_info = match self.tensor_infos.get(name) { - Some(tensor_info) => tensor_info, - None => crate::bail!("cannot find tensor info for {name}"), - }; - tensor_info.read(reader, self.tensor_data_offset, device) - } -} - -fn write_string(w: &mut W, str: &str) -> Result<()> { - let bytes = str.as_bytes(); - w.write_u64::(bytes.len() as u64)?; - w.write_all(bytes)?; - Ok(()) -} - -pub fn write( - w: &mut W, - metadata: &[(&str, &Value)], - tensors: &[(&str, &QTensor)], -) -> Result<()> { - w.write_u32::(0x46554747)?; - w.write_u32::(2)?; // version 2. - w.write_u64::(tensors.len() as u64)?; - w.write_u64::(metadata.len() as u64)?; - for (name, value) in metadata.iter() { - write_string(w, name)?; - w.write_u32::(value.value_type().to_u32())?; - value.write(w)?; - } - let mut offset = 0usize; - let mut offsets = Vec::with_capacity(tensors.len()); - for (name, tensor) in tensors.iter() { - write_string(w, name)?; - let dims = tensor.shape().dims(); - w.write_u32::(dims.len() as u32)?; - for &dim in dims.iter().rev() { - w.write_u64::(dim as u64)?; - } - w.write_u32::(tensor.dtype().to_u32())?; - w.write_u64::(offset as u64)?; - offsets.push(offset); - let size_in_bytes = tensor.storage_size_in_bytes(); - let padding = 31 - (31 + size_in_bytes) % 32; - offset += size_in_bytes + padding; - } - let pos = w.stream_position()? as usize; - let padding = 31 - (31 + pos) % 32; - w.write_all(&vec![0u8; padding])?; - let tensor_start_pos = w.stream_position()? as usize; - for (offset, (_name, tensor)) in offsets.iter().zip(tensors.iter()) { - let pos = w.stream_position()? as usize; - if tensor_start_pos + offset != pos { - crate::bail!( - "internal error, unexpected current position {tensor_start_pos} {offset} {pos}" - ) - } - let data = tensor.data()?; - let size_in_bytes = data.len(); - w.write_all(&data)?; - let padding = 31 - (31 + size_in_bytes) % 32; - w.write_all(&vec![0u8; padding])?; - } - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/k_quants.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/k_quants.rs deleted file mode 100644 index 6210ac1e9f..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/quantized/k_quants.rs +++ /dev/null @@ -1,1965 +0,0 @@ -use super::utils::{ - get_scale_min_k4, group_for_dequantization, group_for_quantization, make_q3_quants, - make_qkx1_quants, make_qx_quants, nearest_int, -}; -use super::GgmlDType; -use crate::Result; -use byteorder::{ByteOrder, LittleEndian}; -use half::f16; -use rayon::prelude::*; - -// Default to QK_K 256 rather than 64. -pub const QK_K: usize = 256; -pub const K_SCALE_SIZE: usize = 12; - -pub const QK4_0: usize = 32; -pub const QK4_1: usize = 32; -pub const QK5_0: usize = 32; -pub const QK5_1: usize = 32; -pub const QK8_0: usize = 32; -pub const QK8_1: usize = 32; - -pub trait GgmlType: Sized + Clone + Send + Sync { - const DTYPE: GgmlDType; - const BLCK_SIZE: usize; - type VecDotType: GgmlType; - - // This is only safe for types that include immediate values such as float/int/... - fn zeros() -> Self { - unsafe { std::mem::MaybeUninit::zeroed().assume_init() } - } - fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()>; - fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()>; - - /// Dot product used as a building block for quantized mat-mul. - /// n is the number of elements to be considered. - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result; - - /// Generic implementation of the dot product without simd optimizations. - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result; -} - -#[derive(Debug, Clone, PartialEq)] -#[repr(C)] -pub struct BlockQ4_0 { - pub(crate) d: f16, - pub(crate) qs: [u8; QK4_0 / 2], -} -const _: () = assert!(std::mem::size_of::() == 18); - -#[derive(Debug, Clone, PartialEq)] -#[repr(C)] -pub struct BlockQ4_1 { - pub(crate) d: f16, - pub(crate) m: f16, - pub(crate) qs: [u8; QK4_1 / 2], -} -const _: () = assert!(std::mem::size_of::() == 20); - -#[derive(Debug, Clone, PartialEq)] -#[repr(C)] -pub struct BlockQ5_0 { - pub(crate) d: f16, - pub(crate) qh: [u8; 4], - pub(crate) qs: [u8; QK5_0 / 2], -} -const _: () = assert!(std::mem::size_of::() == 22); - -#[derive(Debug, Clone, PartialEq)] -#[repr(C)] -pub struct BlockQ5_1 { - pub(crate) d: f16, - pub(crate) m: f16, - pub(crate) qh: [u8; 4], - pub(crate) qs: [u8; QK5_1 / 2], -} -const _: () = assert!(std::mem::size_of::() == 24); - -#[derive(Debug, Clone, PartialEq)] -#[repr(C)] -pub struct BlockQ8_0 { - pub(crate) d: f16, - pub(crate) qs: [i8; QK8_0], -} -const _: () = assert!(std::mem::size_of::() == 34); - -#[derive(Debug, Clone, PartialEq)] -#[repr(C)] -pub struct BlockQ8_1 { - pub(crate) d: f16, - pub(crate) s: f16, - pub(crate) qs: [i8; QK8_1], -} -const _: () = assert!(std::mem::size_of::() == 36); - -#[derive(Debug, Clone, PartialEq)] -#[repr(C)] -pub struct BlockQ2K { - pub(crate) scales: [u8; QK_K / 16], - pub(crate) qs: [u8; QK_K / 4], - pub(crate) d: f16, - pub(crate) dmin: f16, -} -const _: () = assert!(QK_K / 16 + QK_K / 4 + 2 * 2 == std::mem::size_of::()); - -#[derive(Debug, Clone, PartialEq)] -#[repr(C)] -pub struct BlockQ3K { - pub(crate) hmask: [u8; QK_K / 8], - pub(crate) qs: [u8; QK_K / 4], - pub(crate) scales: [u8; 12], - pub(crate) d: f16, -} -const _: () = assert!(QK_K / 8 + QK_K / 4 + 12 + 2 == std::mem::size_of::()); - -#[derive(Debug, Clone, PartialEq)] -// https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/k_quants.h#L82 -#[repr(C)] -pub struct BlockQ4K { - pub(crate) d: f16, - pub(crate) dmin: f16, - pub(crate) scales: [u8; K_SCALE_SIZE], - pub(crate) qs: [u8; QK_K / 2], -} -const _: () = assert!(QK_K / 2 + K_SCALE_SIZE + 2 * 2 == std::mem::size_of::()); - -#[derive(Debug, Clone, PartialEq)] -#[repr(C)] -pub struct BlockQ5K { - pub(crate) d: f16, - pub(crate) dmin: f16, - pub(crate) scales: [u8; K_SCALE_SIZE], - pub(crate) qh: [u8; QK_K / 8], - pub(crate) qs: [u8; QK_K / 2], -} -const _: () = - assert!(QK_K / 8 + QK_K / 2 + 2 * 2 + K_SCALE_SIZE == std::mem::size_of::()); - -#[derive(Debug, Clone, PartialEq)] -#[repr(C)] -pub struct BlockQ6K { - pub(crate) ql: [u8; QK_K / 2], - pub(crate) qh: [u8; QK_K / 4], - pub(crate) scales: [i8; QK_K / 16], - pub(crate) d: f16, -} -const _: () = assert!(3 * QK_K / 4 + QK_K / 16 + 2 == std::mem::size_of::()); - -#[derive(Debug, Clone, PartialEq)] -#[repr(C)] -pub struct BlockQ8K { - pub(crate) d: f32, - pub(crate) qs: [i8; QK_K], - pub(crate) bsums: [i16; QK_K / 16], -} -const _: () = assert!(4 + QK_K + QK_K / 16 * 2 == std::mem::size_of::()); - -impl GgmlType for BlockQ4_0 { - const DTYPE: GgmlDType = GgmlDType::Q4_0; - const BLCK_SIZE: usize = QK4_0; - type VecDotType = BlockQ8_0; - - // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1525 - fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { - let k = ys.len(); - let qk = Self::BLCK_SIZE; - if k % qk != 0 { - crate::bail!("dequantize_row_q4_0: {k} is not divisible by {qk}") - } - - let nb = k / qk; - for i in 0..nb { - let d = xs[i].d.to_f32(); - - for j in 0..(qk / 2) { - let x0 = (xs[i].qs[j] & 0x0F) as i16 - 8; - let x1 = (xs[i].qs[j] >> 4) as i16 - 8; - - ys[i * qk + j] = (x0 as f32) * d; - ys[i * qk + j + qk / 2] = (x1 as f32) * d; - } - } - Ok(()) - } - - fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { - // quantize_row_q4_0 - let qk = Self::BLCK_SIZE; - let k = xs.len(); - if k % qk != 0 { - crate::bail!("{k} is not divisible by {}", qk); - }; - let nb = k / qk; - if ys.len() != nb { - crate::bail!("size mismatch {} {} {}", xs.len(), ys.len(), qk,) - } - for (i, ys) in ys.iter_mut().enumerate() { - let mut amax = 0f32; - let mut max = 0f32; - - let xs = &xs[i * qk..(i + 1) * qk]; - for &x in xs.iter() { - if amax < x.abs() { - amax = x.abs(); - max = x; - } - } - let d = max / -8.0; - let id = if d != 0f32 { 1. / d } else { 0. }; - ys.d = f16::from_f32(d); - - for (j, q) in ys.qs.iter_mut().enumerate() { - let x0 = xs[j] * id; - let x1 = xs[qk / 2 + j] * id; - let xi0 = u8::min(15, (x0 + 8.5) as u8); - let xi1 = u8::min(15, (x1 + 8.5) as u8); - *q = xi0 | (xi1 << 4) - } - } - Ok(()) - } - - // https://github.com/ggerganov/llama.cpp/blob/b5ffb2849d23afe73647f68eec7b68187af09be6/ggml.c#L2361C10-L2361C122 - #[allow(unreachable_code)] - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - #[cfg(target_feature = "avx")] - return super::avx::vec_dot_q4_0_q8_0(n, xs, ys); - - #[cfg(target_feature = "neon")] - return super::neon::vec_dot_q4_0_q8_0(n, xs, ys); - - #[cfg(target_feature = "simd128")] - return super::simd128::vec_dot_q4_0_q8_0(n, xs, ys); - - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - let qk = QK8_0; - if n % QK8_0 != 0 { - crate::bail!("vec_dot_q4_0_q8_0: {n} is not divisible by {qk}") - } - // Generic implementation. - let mut sumf = 0f32; - for (xs, ys) in xs.iter().zip(ys.iter()) { - let mut sum_i = 0; - for j in 0..qk / 2 { - let v0 = (xs.qs[j] & 0x0F) as i32 - 8; - let v1 = (xs.qs[j] >> 4) as i32 - 8; - sum_i += v0 * ys.qs[j] as i32 + v1 * ys.qs[j + qk / 2] as i32 - } - sumf += sum_i as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d) - } - Ok(sumf) - } -} - -impl GgmlType for BlockQ4_1 { - const DTYPE: GgmlDType = GgmlDType::Q4_1; - const BLCK_SIZE: usize = QK4_1; - type VecDotType = BlockQ8_1; - - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - // ggml_vec_dot_q4_1_q8_1 - let qk = QK8_1; - if n % qk != 0 { - crate::bail!("vec_dot_q4_1_q8_1: {n} is not divisible by {qk}") - } - let nb = n / qk; - if nb % 2 != 0 { - crate::bail!("vec_dot_q4_1_q8_1: {n}, nb is not divisible by 2") - } - - // Generic implementation. - let mut sumf = 0f32; - - for (xs, ys) in xs.iter().zip(ys.iter()) { - let mut sumi = 0i32; - - for j in 0..qk / 2 { - let v0 = xs.qs[j] as i32 & 0x0F; - let v1 = xs.qs[j] as i32 >> 4; - sumi += (v0 * ys.qs[j] as i32) + (v1 * ys.qs[j + qk / 2] as i32); - } - - sumf += sumi as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d) - + f16::to_f32(xs.m) * f16::to_f32(ys.s) - } - Ok(sumf) - } - - fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { - // quantize_row_q4_1 - let qk = Self::BLCK_SIZE; - if ys.len() * qk != xs.len() { - crate::bail!("size mismatch {} {} {}", xs.len(), ys.len(), qk,) - } - for (i, ys) in ys.iter_mut().enumerate() { - let xs = &xs[i * qk..(i + 1) * qk]; - - let mut min = f32::INFINITY; - let mut max = f32::NEG_INFINITY; - for &x in xs.iter() { - min = f32::min(x, min); - max = f32::max(x, max); - } - let d = (max - min) / ((1 << 4) - 1) as f32; - let id = if d != 0f32 { 1. / d } else { 0. }; - ys.d = f16::from_f32(d); - ys.m = f16::from_f32(min); - - for (j, q) in ys.qs.iter_mut().take(qk / 2).enumerate() { - let x0 = (xs[j] - min) * id; - let x1 = (xs[qk / 2 + j] - min) * id; - - let xi0 = u8::min(15, (x0 + 0.5) as u8); - let xi1 = u8::min(15, (x1 + 0.5) as u8); - - *q = xi0 | (xi1 << 4); - } - } - Ok(()) - } - - // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1545 - fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { - let k = ys.len(); - if k % QK4_1 != 0 { - crate::bail!("dequantize_row_q4_1: {k} is not divisible by {QK4_1}"); - } - - let nb = k / QK4_1; - for i in 0..nb { - let d = xs[i].d.to_f32(); - let m = xs[i].m.to_f32(); - - for j in 0..(QK4_1 / 2) { - let x0 = xs[i].qs[j] & 0x0F; - let x1 = xs[i].qs[j] >> 4; - - ys[i * QK4_1 + j] = (x0 as f32) * d + m; - ys[i * QK4_1 + j + QK4_1 / 2] = (x1 as f32) * d + m; - } - } - Ok(()) - } -} - -impl GgmlType for BlockQ5_0 { - const DTYPE: GgmlDType = GgmlDType::Q5_0; - const BLCK_SIZE: usize = QK5_0; - type VecDotType = BlockQ8_0; - - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - let qk = Self::BLCK_SIZE; - if n % Self::BLCK_SIZE != 0 { - crate::bail!("vec_dot_q5_0_q8_0: {n} is not divisible by {qk}") - } - let nb = n / qk; - if nb % 2 != 0 { - crate::bail!("vec_dot_q5_0_q8_0: {n}, nb is not divisible by 2") - } - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(_n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - // Generic implementation. - let mut sumf = 0f32; - - for (xs, ys) in xs.iter().zip(ys.iter()) { - let qh = LittleEndian::read_u32(&xs.qh); - let mut sumi = 0i32; - - for j in 0..Self::BLCK_SIZE / 2 { - let xh_0 = (((qh & (1u32 << j)) >> j) << 4) as u8; - let xh_1 = ((qh & (1u32 << (j + 16))) >> (j + 12)) as u8; - - let x0 = ((xs.qs[j] & 0x0F) as i32 | xh_0 as i32) - 16; - let x1 = ((xs.qs[j] >> 4) as i32 | xh_1 as i32) - 16; - - sumi += (x0 * ys.qs[j] as i32) + (x1 * ys.qs[j + Self::BLCK_SIZE / 2] as i32); - } - - sumf += sumi as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d) - } - Ok(sumf) - } - - fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { - // quantize_row_q5_0 - let k = xs.len(); - if ys.len() * Self::BLCK_SIZE != k { - crate::bail!("size mismatch {k} {} {}", ys.len(), Self::BLCK_SIZE) - } - for (i, ys) in ys.iter_mut().enumerate() { - let xs = &xs[i * Self::BLCK_SIZE..(i + 1) * Self::BLCK_SIZE]; - - let mut amax = 0f32; - let mut max = 0f32; - for &x in xs.iter() { - if amax < x.abs() { - amax = x.abs(); - max = x; - } - } - let d = max / -16.; - let id = if d != 0f32 { 1. / d } else { 0. }; - ys.d = f16::from_f32(d); - let mut qh = 0u32; - for j in 0..Self::BLCK_SIZE / 2 { - let x0 = xs[j] * id; - let x1 = xs[j + Self::BLCK_SIZE / 2] * id; - let xi0 = ((x0 + 16.5) as i8).min(31) as u8; - let xi1 = ((x1 + 16.5) as i8).min(31) as u8; - ys.qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4); - qh |= ((xi0 as u32 & 0x10) >> 4) << j; - qh |= ((xi1 as u32 & 0x10) >> 4) << (j + Self::BLCK_SIZE / 2); - } - LittleEndian::write_u32(&mut ys.qh, qh) - } - Ok(()) - } - - // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1566 - fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { - let k = ys.len(); - if k % QK5_0 != 0 { - crate::bail!("dequantize_row_q5_0: {k} is not divisible by {QK5_0}"); - } - - let nb = k / QK5_0; - for i in 0..nb { - let d = xs[i].d.to_f32(); - let qh: u32 = LittleEndian::read_u32(&xs[i].qh); - - for j in 0..(QK5_0 / 2) { - let xh_0 = (((qh >> j) << 4) & 0x10) as u8; - let xh_1 = ((qh >> (j + 12)) & 0x10) as u8; - - let x0 = ((xs[i].qs[j] & 0x0F) | xh_0) as i32 - 16; - let x1 = ((xs[i].qs[j] >> 4) | xh_1) as i32 - 16; - - ys[i * QK5_0 + j] = (x0 as f32) * d; - ys[i * QK5_0 + j + QK5_0 / 2] = (x1 as f32) * d; - } - } - Ok(()) - } -} - -impl GgmlType for BlockQ5_1 { - const DTYPE: GgmlDType = GgmlDType::Q5_1; - const BLCK_SIZE: usize = QK5_1; - type VecDotType = BlockQ8_1; - - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - let qk = Self::BLCK_SIZE; - if n % Self::BLCK_SIZE != 0 { - crate::bail!("vec_dot_q5_1_q8_1: {n} is not divisible by {qk}") - } - let nb = n / qk; - if nb % 2 != 0 { - crate::bail!("vec_dot_q5_1_q8_1: {n}, nb is not divisible by 2") - } - - // Generic implementation. - let mut sumf = 0f32; - - for (xs, ys) in xs.iter().zip(ys.iter()) { - let qh = LittleEndian::read_u32(&xs.qh); - let mut sumi = 0i32; - - for j in 0..Self::BLCK_SIZE / 2 { - let xh_0 = ((qh >> j) << 4) & 0x10; - let xh_1 = (qh >> (j + 12)) & 0x10; - - let x0 = (xs.qs[j] as i32 & 0xF) | xh_0 as i32; - let x1 = (xs.qs[j] as i32 >> 4) | xh_1 as i32; - - sumi += (x0 * ys.qs[j] as i32) + (x1 * ys.qs[j + Self::BLCK_SIZE / 2] as i32); - } - - sumf += sumi as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d) - + f16::to_f32(xs.m) * f16::to_f32(ys.s) - } - Ok(sumf) - } - - fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { - // quantize_row_q5_1 - let qk = Self::BLCK_SIZE; - if ys.len() * qk != xs.len() { - crate::bail!("size mismatch {} {} {}", xs.len(), ys.len(), qk,) - } - for (i, ys) in ys.iter_mut().enumerate() { - let xs = &xs[i * qk..(i + 1) * qk]; - - let mut min = f32::INFINITY; - let mut max = f32::NEG_INFINITY; - for &x in xs.iter() { - min = f32::min(x, min); - max = f32::max(x, max); - } - let d = (max - min) / ((1 << 5) - 1) as f32; - let id = if d != 0f32 { 1. / d } else { 0. }; - ys.d = f16::from_f32(d); - ys.m = f16::from_f32(min); - - let mut qh = 0u32; - for (j, q) in ys.qs.iter_mut().take(qk / 2).enumerate() { - let x0 = (xs[j] - min) * id; - let x1 = (xs[qk / 2 + j] - min) * id; - - let xi0 = (x0 + 0.5) as u8; - let xi1 = (x1 + 0.5) as u8; - - *q = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4); - // get the 5-th bit and store it in qh at the right position - qh |= ((xi0 as u32 & 0x10) >> 4) << j; - qh |= ((xi1 as u32 & 0x10) >> 4) << (j + qk / 2); - } - LittleEndian::write_u32(&mut ys.qh, qh); - } - Ok(()) - } - - // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1592 - fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { - let k = ys.len(); - if k % QK5_1 != 0 { - crate::bail!("dequantize_row_q5_1: {k} is not divisible by {QK5_1}"); - } - - let nb = k / QK5_1; - for i in 0..nb { - let d = xs[i].d.to_f32(); - let m = xs[i].m.to_f32(); - let qh: u32 = LittleEndian::read_u32(&xs[i].qh); - - for j in 0..(QK5_1 / 2) { - let xh_0 = (((qh >> j) << 4) & 0x10) as u8; - let xh_1 = ((qh >> (j + 12)) & 0x10) as u8; - - let x0 = (xs[i].qs[j] & 0x0F) | xh_0; - let x1 = (xs[i].qs[j] >> 4) | xh_1; - - ys[i * QK5_1 + j] = (x0 as f32) * d + m; - ys[i * QK5_1 + j + QK5_1 / 2] = (x1 as f32) * d + m; - } - } - Ok(()) - } -} - -impl GgmlType for BlockQ8_0 { - const DTYPE: GgmlDType = GgmlDType::Q8_0; - const BLCK_SIZE: usize = QK8_0; - type VecDotType = BlockQ8_0; - - // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1619 - fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { - let k = ys.len(); - if k % QK8_0 != 0 { - crate::bail!("dequantize_row_q8_0: {k} is not divisible by {QK8_0}"); - } - - let nb = k / QK8_0; - - for i in 0..nb { - let d = xs[i].d.to_f32(); - - for j in 0..QK8_0 { - ys[i * QK8_0 + j] = xs[i].qs[j] as f32 * d; - } - } - Ok(()) - } - - fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { - // quantize_row_q8_0 - let k = xs.len(); - if k % Self::BLCK_SIZE != 0 { - crate::bail!("{k} is not divisible by {}", Self::BLCK_SIZE); - }; - let nb = k / Self::BLCK_SIZE; - if ys.len() != nb { - crate::bail!( - "size mismatch {} {} {}", - xs.len(), - ys.len(), - Self::BLCK_SIZE - ) - } - for (i, ys) in ys.iter_mut().enumerate() { - let mut amax = 0f32; - let xs = &xs[i * Self::BLCK_SIZE..(i + 1) * Self::BLCK_SIZE]; - for &x in xs.iter() { - amax = amax.max(x.abs()) - } - let d = amax / ((1 << 7) - 1) as f32; - let id = if d != 0f32 { 1. / d } else { 0. }; - ys.d = f16::from_f32(d); - for (y, &x) in ys.qs.iter_mut().zip(xs.iter()) { - *y = f32::round(x * id) as i8 - } - } - Ok(()) - } - - #[allow(unreachable_code)] - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - #[cfg(target_feature = "avx")] - return super::avx::vec_dot_q8_0_q8_0(n, xs, ys); - - #[cfg(target_feature = "neon")] - return super::neon::vec_dot_q8_0_q8_0(n, xs, ys); - - #[cfg(target_feature = "simd128")] - return super::simd128::vec_dot_q8_0_q8_0(n, xs, ys); - - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - let qk = QK8_0; - if n % QK8_0 != 0 { - crate::bail!("vec_dot_q8_0_q8_0: {n} is not divisible by {qk}") - } - - // Generic implementation. - let mut sumf = 0f32; - for (xs, ys) in xs.iter().zip(ys.iter()) { - let sum_i = xs - .qs - .iter() - .zip(ys.qs.iter()) - .map(|(&x, &y)| x as i32 * y as i32) - .sum::(); - sumf += sum_i as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d) - } - Ok(sumf) - } -} - -impl GgmlType for BlockQ8_1 { - const DTYPE: GgmlDType = GgmlDType::Q8_1; - const BLCK_SIZE: usize = QK8_1; - type VecDotType = BlockQ8_1; - - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(_n: usize, _xs: &[Self], _ys: &[Self::VecDotType]) -> Result { - unimplemented!("no support for vec-dot on Q8_1") - } - - fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { - // quantize_row_q8_1 - let k = xs.len(); - if ys.len() * Self::BLCK_SIZE != k { - crate::bail!("size mismatch {k} {} {}", ys.len(), Self::BLCK_SIZE) - } - for (i, ys) in ys.iter_mut().enumerate() { - let mut amax = 0f32; - let xs = &xs[i * Self::BLCK_SIZE..(i + 1) * Self::BLCK_SIZE]; - for &x in xs.iter() { - amax = amax.max(x.abs()) - } - let d = amax / ((1 << 7) - 1) as f32; - let id = if d != 0f32 { 1. / d } else { 0. }; - ys.d = f16::from_f32(d); - let mut sum = 0i32; - for j in 0..Self::BLCK_SIZE / 2 { - let v0 = xs[j] * id; - let v1 = xs[j + Self::BLCK_SIZE / 2] * id; - ys.qs[j] = f32::round(v0) as i8; - ys.qs[j + Self::BLCK_SIZE / 2] = f32::round(v1) as i8; - sum += ys.qs[j] as i32 + ys.qs[j + Self::BLCK_SIZE / 2] as i32; - } - ys.s = f16::from_f32(sum as f32) * ys.d; - } - Ok(()) - } - - fn to_float(_xs: &[Self], _ys: &mut [f32]) -> Result<()> { - unimplemented!("no support for vec-dot on Q8_1") - } -} - -impl GgmlType for BlockQ2K { - const DTYPE: GgmlDType = GgmlDType::Q2K; - const BLCK_SIZE: usize = QK_K; - type VecDotType = BlockQ8K; - - #[allow(unreachable_code)] - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - #[cfg(target_feature = "avx")] - return super::avx::vec_dot_q2k_q8k(n, xs, ys); - - #[cfg(target_feature = "neon")] - return super::neon::vec_dot_q2k_q8k(n, xs, ys); - - #[cfg(target_feature = "simd128")] - return super::simd128::vec_dot_q2k_q8k(n, xs, ys); - - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - if n % QK_K != 0 { - crate::bail!("vec_dot_q2k_q8k: {n} is not divisible by {QK_K}") - } - - let mut sumf = 0.0; - for (x, y) in xs.iter().zip(ys.iter()) { - let mut q2: &[_] = &x.qs; - let mut q8: &[_] = &y.qs; - let sc = &x.scales; - - let mut summs = 0; - for (bsum, scale) in y.bsums.iter().zip(sc) { - summs += *bsum as i32 * ((scale >> 4) as i32); - } - - let dall = y.d * x.d.to_f32(); - let dmin = y.d * x.dmin.to_f32(); - - let mut isum = 0; - let mut is = 0; - for _ in 0..(QK_K / 128) { - let mut shift = 0; - for _ in 0..4 { - let d = (sc[is] & 0xF) as i32; - is += 1; - let mut isuml = 0; - for l in 0..16 { - isuml += q8[l] as i32 * (((q2[l] >> shift) & 3) as i32); - } - isum += d * isuml; - let d = (sc[is] & 0xF) as i32; - is += 1; - isuml = 0; - for l in 16..32 { - isuml += q8[l] as i32 * (((q2[l] >> shift) & 3) as i32); - } - isum += d * isuml; - shift += 2; - // adjust the indexing - q8 = &q8[32..]; - } - // adjust the indexing - q2 = &q2[32..]; - } - sumf += dall * isum as f32 - dmin * summs as f32; - } - - Ok(sumf) - } - - // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L279 - fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { - const Q4SCALE: f32 = 15.0; - - for (block, x) in group_for_quantization(xs, ys)? { - //calculate scales and mins - let mut mins: [f32; QK_K / 16] = [0.0; QK_K / 16]; - let mut scales: [f32; QK_K / 16] = [0.0; QK_K / 16]; - - for (j, x_scale_slice) in x.chunks(16).enumerate() { - (scales[j], mins[j]) = make_qkx1_quants(3, 5, x_scale_slice); - } - // get max scale and max min and ensure they are >= 0.0 - let max_scale = scales.iter().fold(0.0, |max, &val| val.max(max)); - let max_min = mins.iter().fold(0.0, |max, &val| val.max(max)); - - if max_scale > 0.0 { - let iscale = Q4SCALE / max_scale; - for (j, scale) in scales.iter().enumerate().take(QK_K / 16) { - block.scales[j] = nearest_int(iscale * scale) as u8; - } - block.d = f16::from_f32(max_scale / Q4SCALE); - } else { - for j in 0..QK_K / 16 { - block.scales[j] = 0; - } - block.d = f16::from_f32(0.0); - } - - if max_min > 0.0 { - let iscale = Q4SCALE / max_min; - for (j, scale) in block.scales.iter_mut().enumerate() { - let l = nearest_int(iscale * mins[j]) as u8; - *scale |= l << 4; - } - block.dmin = f16::from_f32(max_min / Q4SCALE); - } else { - block.dmin = f16::from_f32(0.0); - } - - let mut big_l: [u8; QK_K] = [0; QK_K]; - - for j in 0..QK_K / 16 { - let d = block.d.to_f32() * (block.scales[j] & 0xF) as f32; - if d == 0.0 { - continue; - } - let dm = block.dmin.to_f32() * (block.scales[j] >> 4) as f32; - for ii in 0..16 { - let ll = nearest_int((x[16 * j + ii] + dm) / d).clamp(0, 3); - big_l[16 * j + ii] = ll as u8; - } - } - - for j in (0..QK_K).step_by(128) { - for ll in 0..32 { - block.qs[j / 4 + ll] = big_l[j + ll] - | (big_l[j + ll + 32] << 2) - | (big_l[j + ll + 64] << 4) - | (big_l[j + ll + 96] << 6); - } - } - } - Ok(()) - } - // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L354 - fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { - for (block, y) in group_for_dequantization(xs, ys)? { - let d = block.d.to_f32(); - let min = block.dmin.to_f32(); - - let mut is = 0; - - for (y_block, qs) in y.chunks_exact_mut(128).zip(block.qs.chunks_exact(32)) { - // Step by 32 over q. - let mut shift = 0; - let mut y_block_index = 0; - for _j in 0..4 { - let sc = block.scales[is]; - is += 1; - let dl = d * (sc & 0xF) as f32; - let ml = min * (sc >> 4) as f32; - for q in &qs[..16] { - let y = dl * ((q >> shift) & 3) as f32 - ml; - y_block[y_block_index] = y; - y_block_index += 1; - } - - let sc = block.scales[is]; - is += 1; - let dl = d * (sc & 0xF) as f32; - let ml = min * (sc >> 4) as f32; - for q in &qs[16..] { - let y = dl * ((q >> shift) & 3) as f32 - ml; - y_block[y_block_index] = y; - y_block_index += 1; - } - - shift += 2; - } - } - } - Ok(()) - } -} - -impl GgmlType for BlockQ3K { - const DTYPE: GgmlDType = GgmlDType::Q3K; - const BLCK_SIZE: usize = QK_K; - type VecDotType = BlockQ8K; - - #[allow(unreachable_code)] - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - #[cfg(target_feature = "avx")] - return super::avx::vec_dot_q3k_q8k(n, xs, ys); - - #[cfg(target_feature = "neon")] - return super::neon::vec_dot_q3k_q8k(n, xs, ys); - - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - if n % QK_K != 0 { - crate::bail!("vec_dot_q3k_q8k: {n} is not divisible by {QK_K}") - } - - const KMASK1: u32 = 0x03030303; - const KMASK2: u32 = 0x0f0f0f0f; - - let mut aux8: [i8; QK_K] = [0; QK_K]; - let mut aux16: [i16; 8] = [0; 8]; - let mut sums: [f32; 8] = [0.0; 8]; - let mut aux32: [i32; 8] = [0; 8]; - - let mut auxs: [u32; 4] = [0; 4]; - - for (x, y) in xs.iter().zip(ys.iter()) { - let mut q3: &[u8] = &x.qs; - let hmask: &[u8] = &x.hmask; - let mut q8: &[i8] = &y.qs; - - aux32.fill(0); - let mut a = &mut aux8[..]; - - let mut m = 1; - //Like the GGML original this is written this way to enable the compiler to vectorize it. - for _ in 0..QK_K / 128 { - a.iter_mut() - .take(32) - .zip(q3) - .for_each(|(a_val, q3_val)| *a_val = (q3_val & 3) as i8); - a.iter_mut() - .take(32) - .zip(hmask) - .for_each(|(a_val, hmask_val)| { - *a_val -= if hmask_val & m != 0 { 0 } else { 4 } - }); - a = &mut a[32..]; - m <<= 1; - - a.iter_mut() - .take(32) - .zip(q3) - .for_each(|(a_val, q3_val)| *a_val = ((q3_val >> 2) & 3) as i8); - a.iter_mut() - .take(32) - .zip(hmask) - .for_each(|(a_val, hmask_val)| { - *a_val -= if hmask_val & m != 0 { 0 } else { 4 } - }); - a = &mut a[32..]; - m <<= 1; - - a.iter_mut() - .take(32) - .zip(q3) - .for_each(|(a_val, q3_val)| *a_val = ((q3_val >> 4) & 3) as i8); - a.iter_mut() - .take(32) - .zip(hmask) - .for_each(|(a_val, hmask_val)| { - *a_val -= if hmask_val & m != 0 { 0 } else { 4 } - }); - a = &mut a[32..]; - m <<= 1; - - a.iter_mut() - .take(32) - .zip(q3) - .for_each(|(a_val, q3_val)| *a_val = ((q3_val >> 6) & 3) as i8); - a.iter_mut() - .take(32) - .zip(hmask) - .for_each(|(a_val, hmask_val)| { - *a_val -= if hmask_val & m != 0 { 0 } else { 4 } - }); - a = &mut a[32..]; - m <<= 1; - q3 = &q3[32..]; - } - - a = &mut aux8[..]; - - LittleEndian::read_u32_into(&x.scales, &mut auxs[0..3]); - - let tmp = auxs[2]; - auxs[2] = ((auxs[0] >> 4) & KMASK2) | (((tmp >> 4) & KMASK1) << 4); - auxs[3] = ((auxs[1] >> 4) & KMASK2) | (((tmp >> 6) & KMASK1) << 4); - auxs[0] = (auxs[0] & KMASK2) | (((tmp) & KMASK1) << 4); - auxs[1] = (auxs[1] & KMASK2) | (((tmp >> 2) & KMASK1) << 4); - - for aux in auxs { - for scale in aux.to_le_bytes() { - let scale = i8::from_be_bytes([scale]); - for l in 0..8 { - aux16[l] = q8[l] as i16 * a[l] as i16; - } - for l in 0..8 { - aux32[l] += (scale as i32 - 32) * aux16[l] as i32; - } - q8 = &q8[8..]; - a = &mut a[8..]; - - for l in 0..8 { - aux16[l] = q8[l] as i16 * a[l] as i16; - } - for l in 0..8 { - aux32[l] += (scale as i32 - 32) * aux16[l] as i32; - } - q8 = &q8[8..]; - a = &mut a[8..]; - } - } - let d = x.d.to_f32() * y.d; - for l in 0..8 { - sums[l] += d * aux32[l] as f32; - } - } - - Ok(sums.iter().sum()) - } - - fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { - for (block, x) in group_for_quantization(xs, ys)? { - let mut scales: [f32; QK_K / 16] = [0.0; QK_K / 16]; - for (j, x_scale_slice) in x.chunks_exact(16).enumerate() { - scales[j] = make_q3_quants(x_scale_slice, 4, true); - } - - // Get max scale by absolute value. - let mut max_scale: f32 = 0.0; - for &scale in scales.iter() { - if scale.abs() > max_scale.abs() { - max_scale = scale; - } - } - - block.scales.fill(0); - - if max_scale != 0.0 { - let iscale = -32.0 / max_scale; - for (j, scale) in scales.iter().enumerate() { - let l_val = nearest_int(iscale * scale); - let l_val = l_val.clamp(-32, 31) + 32; - if j < 8 { - block.scales[j] = (l_val & 0xF) as u8; - } else { - block.scales[j - 8] |= ((l_val & 0xF) << 4) as u8; - } - let l_val = l_val >> 4; - block.scales[j % 4 + 8] |= (l_val << (2 * (j / 4))) as u8; - } - block.d = f16::from_f32(1.0 / iscale); - } else { - block.d = f16::from_f32(0.0); - } - - let mut l: [i8; QK_K] = [0; QK_K]; - - for j in 0..QK_K / 16 { - let sc = if j < 8 { - block.scales[j] & 0xF - } else { - block.scales[j - 8] >> 4 - }; - let sc = (sc | (((block.scales[8 + j % 4] >> (2 * (j / 4))) & 3) << 4)) as i8 - 32; - let d = block.d.to_f32() * sc as f32; - if d != 0.0 { - for ii in 0..16 { - let l_val = nearest_int(x[16 * j + ii] / d); - l[16 * j + ii] = (l_val.clamp(-4, 3) + 4) as i8; - } - } - } - - block.hmask.fill(0); - let mut m = 0; - let mut hm = 1; - - for ll in l.iter_mut() { - if *ll > 3 { - block.hmask[m] |= hm; - *ll -= 4; - } - m += 1; - if m == QK_K / 8 { - m = 0; - hm <<= 1; - } - } - - for j in (0..QK_K).step_by(128) { - for l_val in 0..32 { - block.qs[j / 4 + l_val] = (l[j + l_val] - | (l[j + l_val + 32] << 2) - | (l[j + l_val + 64] << 4) - | (l[j + l_val + 96] << 6)) - as u8; - } - } - } - - Ok(()) - } - - // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L533 - fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { - const KMASK1: u32 = 0x03030303; - const KMASK2: u32 = 0x0f0f0f0f; - - for (block, y) in group_for_dequantization(xs, ys)? { - //Reconstruct the scales - let mut aux = [0; 4]; - LittleEndian::read_u32_into(&block.scales, &mut aux[0..3]); - - let tmp = aux[2]; - aux[2] = ((aux[0] >> 4) & KMASK2) | (((tmp >> 4) & KMASK1) << 4); - aux[3] = ((aux[1] >> 4) & KMASK2) | (((tmp >> 6) & KMASK1) << 4); - aux[0] = (aux[0] & KMASK2) | (((tmp) & KMASK1) << 4); - aux[1] = (aux[1] & KMASK2) | (((tmp >> 2) & KMASK1) << 4); - - //Transfer the scales into an i8 array - let scales: &mut [i8] = - unsafe { std::slice::from_raw_parts_mut(aux.as_mut_ptr() as *mut i8, 16) }; - - let d_all = block.d.to_f32(); - let mut m = 1; - let mut is = 0; - - // Dequantize both 128 long blocks - // 32 qs values per 128 long block - // Each 16 elements get a scale - for (y, qs) in y.chunks_exact_mut(128).zip(block.qs.chunks_exact(32)) { - let mut shift = 0; - for shift_scoped_y in y.chunks_exact_mut(32) { - for (scale_index, scale_scoped_y) in - shift_scoped_y.chunks_exact_mut(16).enumerate() - { - let dl = d_all * (scales[is] as f32 - 32.0); - for (i, inner_y) in scale_scoped_y.iter_mut().enumerate() { - let new_y = dl - * (((qs[i + 16 * scale_index] >> shift) & 3) as i8 - - if (block.hmask[i + 16 * scale_index] & m) == 0 { - 4 - } else { - 0 - }) as f32; - *inner_y = new_y; - } - // 16 block finished => advance scale index - is += 1; - } - // 32 block finished => increase shift and m - shift += 2; - m <<= 1; - } - } - } - - Ok(()) - } -} - -impl GgmlType for BlockQ4K { - const DTYPE: GgmlDType = GgmlDType::Q4K; - const BLCK_SIZE: usize = QK_K; - type VecDotType = BlockQ8K; - - #[allow(unreachable_code)] - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - #[cfg(target_feature = "avx")] - return super::avx::vec_dot_q4k_q8k(n, xs, ys); - - #[cfg(target_feature = "neon")] - return super::neon::vec_dot_q4k_q8k(n, xs, ys); - - #[cfg(target_feature = "simd128")] - return super::simd128::vec_dot_q4k_q8k(n, xs, ys); - - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - if n % QK_K != 0 { - crate::bail!("vec_dot_q4k_q8k: {n} is not divisible by {QK_K}") - } - - const KMASK1: u32 = 0x3f3f3f3f; - const KMASK2: u32 = 0x0f0f0f0f; - const KMASK3: u32 = 0x03030303; - - let mut utmp: [u32; 4] = [0; 4]; - let mut scales: [u8; 8] = [0; 8]; - let mut mins: [u8; 8] = [0; 8]; - - let mut aux8: [i8; QK_K] = [0; QK_K]; - let mut aux16: [i16; 8] = [0; 8]; - let mut sums: [f32; 8] = [0.0; 8]; - let mut aux32: [i32; 8] = [0; 8]; - - let mut sumf = 0.0; - for (y, x) in ys.iter().zip(xs.iter()) { - let q4 = &x.qs; - let q8 = &y.qs; - aux32.fill(0); - - let mut a = &mut aux8[..]; - let mut q4 = &q4[..]; - for _ in 0..QK_K / 64 { - for l in 0..32 { - a[l] = (q4[l] & 0xF) as i8; - } - a = &mut a[32..]; - for l in 0..32 { - a[l] = (q4[l] >> 4) as i8; - } - a = &mut a[32..]; - q4 = &q4[32..]; - } - - LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); - - utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); - let uaux = utmp[1] & KMASK1; - utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); - utmp[2] = uaux; - utmp[0] &= KMASK1; - - //extract scales and mins - LittleEndian::write_u32_into(&utmp[0..2], &mut scales); - LittleEndian::write_u32_into(&utmp[2..4], &mut mins); - - let mut sumi = 0; - for j in 0..QK_K / 16 { - sumi += y.bsums[j] as i32 * mins[j / 2] as i32; - } - - let mut a = &mut aux8[..]; - let mut q8 = &q8[..]; - - for scale in scales { - let scale = scale as i32; - for _ in 0..4 { - for l in 0..8 { - aux16[l] = q8[l] as i16 * a[l] as i16; - } - for l in 0..8 { - aux32[l] += scale * aux16[l] as i32; - } - q8 = &q8[8..]; - a = &mut a[8..]; - } - } - let d = x.d.to_f32() * y.d; - for l in 0..8 { - sums[l] += d * aux32[l] as f32; - } - let dmin = x.dmin.to_f32() * y.d; - sumf -= dmin * sumi as f32; - } - Ok(sumf + sums.iter().sum::()) - } - - fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { - for (block, x) in group_for_quantization(xs, ys)? { - let mut mins: [f32; QK_K / 32] = [0.0; QK_K / 32]; - let mut scales: [f32; QK_K / 32] = [0.0; QK_K / 32]; - - for (j, x_scale_slice) in x.chunks_exact(32).enumerate() { - (scales[j], mins[j]) = make_qkx1_quants(15, 5, x_scale_slice); - } - - // get max scale and max min and ensure they are >= 0.0 - let max_scale = scales.iter().fold(0.0, |max, &val| val.max(max)); - let max_min = mins.iter().fold(0.0, |max, &val| val.max(max)); - - let inv_scale = if max_scale > 0.0 { - 63.0 / max_scale - } else { - 0.0 - }; - let inv_min = if max_min > 0.0 { 63.0 / max_min } else { 0.0 }; - - for j in 0..QK_K / 32 { - let ls = nearest_int(inv_scale * scales[j]).min(63) as u8; - let lm = nearest_int(inv_min * mins[j]).min(63) as u8; - if j < 4 { - block.scales[j] = ls; - block.scales[j + 4] = lm; - } else { - block.scales[j + 4] = (ls & 0xF) | ((lm & 0xF) << 4); - block.scales[j - 4] |= (ls >> 4) << 6; - block.scales[j] |= (lm >> 4) << 6; - } - } - - block.d = f16::from_f32(max_scale / 63.0); - block.dmin = f16::from_f32(max_min / 63.0); - - let mut l: [u8; QK_K] = [0; QK_K]; - - for j in 0..QK_K / 32 { - let (sc, m) = get_scale_min_k4(j, &block.scales); - let d = block.d.to_f32() * sc as f32; - if d != 0.0 { - let dm = block.dmin.to_f32() * m as f32; - for ii in 0..32 { - let l_val = nearest_int((x[32 * j + ii] + dm) / d); - l[32 * j + ii] = l_val.clamp(0, 15) as u8; - } - } - } - - let q = &mut block.qs; - for j in (0..QK_K).step_by(64) { - for l_val in 0..32 { - let offset_index = (j / 64) * 32 + l_val; - q[offset_index] = l[j + l_val] | (l[j + l_val + 32] << 4); - } - } - } - Ok(()) - } - // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L735 - fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { - for (block, y) in group_for_dequantization(xs, ys)? { - let d = block.d.to_f32(); - let min = block.dmin.to_f32(); - let q = &block.qs; - let mut is = 0; - let mut ys_index = 0; - - for j in (0..QK_K).step_by(64) { - let q = &q[j / 2..j / 2 + 32]; - let (sc, m) = get_scale_min_k4(is, &block.scales); - let d1 = d * sc as f32; - let m1 = min * m as f32; - let (sc, m) = get_scale_min_k4(is + 1, &block.scales); - let d2 = d * sc as f32; - let m2 = min * m as f32; - for q in q { - y[ys_index] = d1 * (q & 0xF) as f32 - m1; - ys_index += 1; - } - for q in q { - y[ys_index] = d2 * (q >> 4) as f32 - m2; - ys_index += 1; - } - is += 2; - } - } - Ok(()) - } -} - -// https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L928 -impl GgmlType for BlockQ5K { - const DTYPE: GgmlDType = GgmlDType::Q5K; - const BLCK_SIZE: usize = QK_K; - type VecDotType = BlockQ8K; - - #[allow(unreachable_code)] - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - #[cfg(target_feature = "avx")] - return super::avx::vec_dot_q5k_q8k(n, xs, ys); - - #[cfg(target_feature = "neon")] - return super::neon::vec_dot_q5k_q8k(n, xs, ys); - - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - if n % QK_K != 0 { - crate::bail!("vec_dot_q5k_q8k: {n} is not divisible by {QK_K}") - } - - const KMASK1: u32 = 0x3f3f3f3f; - const KMASK2: u32 = 0x0f0f0f0f; - const KMASK3: u32 = 0x03030303; - - let mut utmp: [u32; 4] = [0; 4]; - let mut scales: [u8; 8] = [0; 8]; - let mut mins: [u8; 8] = [0; 8]; - - let mut aux8: [i8; QK_K] = [0; QK_K]; - let mut aux16: [i16; 8] = [0; 8]; - let mut sums: [f32; 8] = [0.0; 8]; - let mut aux32: [i32; 8] = [0; 8]; - - let mut sumf = 0.0; - for (y, x) in ys.iter().zip(xs.iter()) { - let q5 = &x.qs; - let hm = &x.qh; - let q8 = &y.qs; - aux32.fill(0); - - let mut a = &mut aux8[..]; - let mut q5 = &q5[..]; - let mut m = 1u8; - - for _ in 0..QK_K / 64 { - for l in 0..32 { - a[l] = (q5[l] & 0xF) as i8; - a[l] += if hm[l] & m != 0 { 16 } else { 0 }; - } - a = &mut a[32..]; - m <<= 1; - for l in 0..32 { - a[l] = (q5[l] >> 4) as i8; - a[l] += if hm[l] & m != 0 { 16 } else { 0 }; - } - a = &mut a[32..]; - m <<= 1; - q5 = &q5[32..]; - } - - LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); - - utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); - let uaux = utmp[1] & KMASK1; - utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); - utmp[2] = uaux; - utmp[0] &= KMASK1; - - //extract scales and mins - LittleEndian::write_u32_into(&utmp[0..2], &mut scales); - LittleEndian::write_u32_into(&utmp[2..4], &mut mins); - - let mut sumi = 0; - for j in 0..QK_K / 16 { - sumi += y.bsums[j] as i32 * mins[j / 2] as i32; - } - - let mut a = &mut aux8[..]; - let mut q8 = &q8[..]; - - for scale in scales { - let scale = scale as i32; - for _ in 0..4 { - for l in 0..8 { - aux16[l] = q8[l] as i16 * a[l] as i16; - } - for l in 0..8 { - aux32[l] += scale * aux16[l] as i32; - } - q8 = &q8[8..]; - a = &mut a[8..]; - } - } - let d = x.d.to_f32() * y.d; - for l in 0..8 { - sums[l] += d * aux32[l] as f32; - } - let dmin = x.dmin.to_f32() * y.d; - sumf -= dmin * sumi as f32; - } - Ok(sumf + sums.iter().sum::()) - } - - // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L793 - fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { - for (block, x) in group_for_quantization(xs, ys)? { - let mut mins: [f32; QK_K / 32] = [0.0; QK_K / 32]; - let mut scales: [f32; QK_K / 32] = [0.0; QK_K / 32]; - - for (j, x_scale_slice) in x.chunks_exact(32).enumerate() { - (scales[j], mins[j]) = make_qkx1_quants(31, 5, x_scale_slice); - } - - // get max scale and max min and ensure they are >= 0.0 - let max_scale = scales.iter().fold(0.0, |max, &val| val.max(max)); - let max_min = mins.iter().fold(0.0, |max, &val| val.max(max)); - - let inv_scale = if max_scale > 0.0 { - 63.0 / max_scale - } else { - 0.0 - }; - let inv_min = if max_min > 0.0 { 63.0 / max_min } else { 0.0 }; - for j in 0..QK_K / 32 { - let ls = nearest_int(inv_scale * scales[j]).min(63) as u8; - let lm = nearest_int(inv_min * mins[j]).min(63) as u8; - if j < 4 { - block.scales[j] = ls; - block.scales[j + 4] = lm; - } else { - block.scales[j + 4] = (ls & 0xF) | ((lm & 0xF) << 4); - block.scales[j - 4] |= (ls >> 4) << 6; - block.scales[j] |= (lm >> 4) << 6; - } - } - block.d = f16::from_f32(max_scale / 63.0); - block.dmin = f16::from_f32(max_min / 63.0); - - let mut l: [u8; QK_K] = [0; QK_K]; - for j in 0..QK_K / 32 { - let (sc, m) = get_scale_min_k4(j, &block.scales); - let d = block.d.to_f32() * sc as f32; - if d == 0.0 { - continue; - } - let dm = block.dmin.to_f32() * m as f32; - for ii in 0..32 { - let ll = nearest_int((x[32 * j + ii] + dm) / d); - l[32 * j + ii] = ll.clamp(0, 31) as u8; - } - } - - let qh = &mut block.qh; - let ql = &mut block.qs; - qh.fill(0); - - let mut m1 = 1; - let mut m2 = 2; - for n in (0..QK_K).step_by(64) { - let offset = (n / 64) * 32; - for j in 0..32 { - let mut l1 = l[n + j]; - if l1 > 15 { - l1 -= 16; - qh[j] |= m1; - } - let mut l2 = l[n + j + 32]; - if l2 > 15 { - l2 -= 16; - qh[j] |= m2; - } - ql[offset + j] = l1 | (l2 << 4); - } - m1 <<= 2; - m2 <<= 2; - } - } - - Ok(()) - } - - // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L928 - fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { - for (block, y) in group_for_dequantization(xs, ys)? { - let d = block.d.to_f32(); - let min = block.dmin.to_f32(); - let ql = &block.qs; - let qh = &block.qh; - let mut is = 0; - let mut u1 = 1; - let mut u2 = 2; - let mut ys_index = 0; - - for j in (0..QK_K).step_by(64) { - let ql = &ql[j / 2..j / 2 + 32]; - let (sc, m) = get_scale_min_k4(is, &block.scales); - let d1 = d * sc as f32; - let m1 = min * m as f32; - let (sc, m) = get_scale_min_k4(is + 1, &block.scales); - let d2 = d * sc as f32; - let m2 = min * m as f32; - for (ql, qh) in ql.iter().zip(qh) { - let to_add = if qh & u1 != 0 { 16f32 } else { 0f32 }; - y[ys_index] = d1 * ((ql & 0xF) as f32 + to_add) - m1; - ys_index += 1; - } - for (ql, qh) in ql.iter().zip(qh) { - let to_add = if qh & u2 != 0 { 16f32 } else { 0f32 }; - y[ys_index] = d2 * ((ql >> 4) as f32 + to_add) - m2; - ys_index += 1; - } - is += 2; - u1 <<= 2; - u2 <<= 2; - } - } - Ok(()) - } -} - -impl GgmlType for BlockQ6K { - const DTYPE: GgmlDType = GgmlDType::Q6K; - const BLCK_SIZE: usize = QK_K; - type VecDotType = BlockQ8K; - - #[allow(unreachable_code)] - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - #[cfg(target_feature = "avx")] - return super::avx::vec_dot_q6k_q8k(n, xs, ys); - - #[cfg(target_feature = "neon")] - return super::neon::vec_dot_q6k_q8k(n, xs, ys); - - #[cfg(target_feature = "simd128")] - return super::simd128::vec_dot_q6k_q8k(n, xs, ys); - - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - if n % QK_K != 0 { - crate::bail!("vec_dot_q6k_q8k: {n} is not divisible by {QK_K}") - } - - let mut aux8 = [0i8; QK_K]; - let mut aux16 = [0i16; 8]; - let mut sums = [0f32; 8]; - let mut aux32 = [0f32; 8]; - - for (x, y) in xs.iter().zip(ys.iter()) { - let q4 = &x.ql; - let qh = &x.qh; - let q8 = &y.qs; - aux32.fill(0f32); - - for j in (0..QK_K).step_by(128) { - let aux8 = &mut aux8[j..]; - let q4 = &q4[j / 2..]; - let qh = &qh[j / 4..]; - for l in 0..32 { - aux8[l] = (((q4[l] & 0xF) | ((qh[l] & 3) << 4)) as i32 - 32) as i8; - aux8[l + 32] = - (((q4[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) as i32 - 32) as i8; - aux8[l + 64] = (((q4[l] >> 4) | (((qh[l] >> 4) & 3) << 4)) as i32 - 32) as i8; - aux8[l + 96] = - (((q4[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) as i32 - 32) as i8; - } - } - - for (j, &scale) in x.scales.iter().enumerate() { - let scale = scale as f32; - let q8 = &q8[16 * j..]; - let aux8 = &aux8[16 * j..]; - for l in 0..8 { - aux16[l] = q8[l] as i16 * aux8[l] as i16; - } - for l in 0..8 { - aux32[l] += scale * aux16[l] as f32 - } - let q8 = &q8[8..]; - let aux8 = &aux8[8..]; - for l in 0..8 { - aux16[l] = q8[l] as i16 * aux8[l] as i16; - } - for l in 0..8 { - aux32[l] += scale * aux16[l] as f32 - } - } - - let d = x.d.to_f32() * y.d; - for (sum, &a) in sums.iter_mut().zip(aux32.iter()) { - *sum += a * d; - } - } - Ok(sums.iter().sum()) - } - - fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { - if xs.len() != ys.len() * Self::BLCK_SIZE { - crate::bail!( - "quantize_row_q6k: size mismatch {} {} {}", - xs.len(), - ys.len(), - Self::BLCK_SIZE - ) - } - let mut l = [0i8; QK_K]; - let mut scales = [0f32; QK_K / 16]; - let mut x = xs.as_ptr(); - let l = l.as_mut_ptr(); - unsafe { - for y in ys.iter_mut() { - let mut max_scale = 0f32; - let mut max_abs_scale = 0f32; - for (ib, scale_) in scales.iter_mut().enumerate() { - let scale = make_qx_quants(16, 32, x.add(16 * ib), l.add(16 * ib), 1); - *scale_ = scale; - let abs_scale = scale.abs(); - if abs_scale > max_abs_scale { - max_abs_scale = abs_scale; - max_scale = scale - } - } - - let iscale = -128f32 / max_scale; - y.d = f16::from_f32(1.0 / iscale); - - for (y_scale, scale) in y.scales.iter_mut().zip(scales.iter()) { - *y_scale = nearest_int(iscale * scale).min(127) as i8 - } - - for (j, &y_scale) in y.scales.iter().enumerate() { - let d = y.d.to_f32() * y_scale as f32; - if d == 0. { - continue; - } - for ii in 0..16 { - let ll = nearest_int(*x.add(16 * j + ii) / d).clamp(-32, 31); - *l.add(16 * j + ii) = (ll + 32) as i8 - } - } - - let mut ql = y.ql.as_mut_ptr(); - let mut qh = y.qh.as_mut_ptr(); - - for j in (0..QK_K).step_by(128) { - for l_idx in 0..32 { - let q1 = *l.add(j + l_idx) & 0xF; - let q2 = *l.add(j + l_idx + 32) & 0xF; - let q3 = *l.add(j + l_idx + 64) & 0xF; - let q4 = *l.add(j + l_idx + 96) & 0xF; - *ql.add(l_idx) = (q1 | (q3 << 4)) as u8; - *ql.add(l_idx + 32) = (q2 | (q4 << 4)) as u8; - *qh.add(l_idx) = ((*l.add(j + l_idx) >> 4) - | ((*l.add(j + l_idx + 32) >> 4) << 2) - | ((*l.add(j + l_idx + 64) >> 4) << 4) - | ((*l.add(j + l_idx + 96) >> 4) << 6)) - as u8; - } - ql = ql.add(64); - qh = qh.add(32); - } - - x = x.add(QK_K) - } - } - Ok(()) - } - - // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L1067 - fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { - let k = ys.len(); - if k % QK_K != 0 { - crate::bail!("dequantize_row_q6k: {k} is not divisible by {QK_K}") - } - for (idx_x, x) in xs.iter().enumerate() { - let d = x.d.to_f32(); - let ql = &x.ql; - let qh = &x.qh; - let sc = &x.scales; - for n in (0..QK_K).step_by(128) { - let idx = n / 128; - let ys = &mut ys[idx_x * QK_K + n..]; - let sc = &sc[8 * idx..]; - let ql = &ql[64 * idx..]; - let qh = &qh[32 * idx..]; - for l in 0..32 { - let is = l / 16; - let q1 = ((ql[l] & 0xF) | ((qh[l] & 3) << 4)) as i8 - 32; - let q2 = ((ql[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) as i8 - 32; - let q3 = ((ql[l] >> 4) | (((qh[l] >> 4) & 3) << 4)) as i8 - 32; - let q4 = ((ql[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) as i8 - 32; - ys[l] = d * sc[is] as f32 * q1 as f32; - ys[l + 32] = d * sc[is + 2] as f32 * q2 as f32; - ys[l + 64] = d * sc[is + 4] as f32 * q3 as f32; - ys[l + 96] = d * sc[is + 6] as f32 * q4 as f32; - } - } - } - Ok(()) - } -} - -impl GgmlType for BlockQ8K { - const DTYPE: GgmlDType = GgmlDType::Q8K; - const BLCK_SIZE: usize = QK_K; - type VecDotType = BlockQ8K; - - #[allow(unreachable_code)] - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - #[cfg(target_feature = "avx")] - return super::avx::vec_dot_q8k_q8k(n, xs, ys); - - #[cfg(target_feature = "neon")] - return super::neon::vec_dot_q8k_q8k(n, xs, ys); - - #[cfg(target_feature = "simd128")] - return super::simd128::vec_dot_q8k_q8k(n, xs, ys); - - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - let qk = QK_K; - if n % QK_K != 0 { - crate::bail!("vec_dot_q8k_q8k: {n} is not divisible by {qk}") - } - - // Generic implementation. - let mut sumf = 0f32; - for (xs, ys) in xs.iter().zip(ys.iter()) { - let sum_i = xs - .qs - .iter() - .zip(ys.qs.iter()) - .map(|(&x, &y)| x as i32 * y as i32) - .sum::(); - sumf += sum_i as f32 * xs.d * ys.d - } - Ok(sumf) - } - - fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { - let k = xs.len(); - if k % QK_K != 0 { - crate::bail!("quantize_row_q8k: {k} is not divisible by {QK_K}") - } - for (i, y) in ys.iter_mut().enumerate() { - let mut max = 0f32; - let mut amax = 0f32; - let xs = &xs[i * QK_K..(i + 1) * QK_K]; - for &x in xs.iter() { - if amax < x.abs() { - amax = x.abs(); - max = x; - } - } - if amax == 0f32 { - y.d = 0f32; - y.qs.fill(0) - } else { - let iscale = -128f32 / max; - for (j, q) in y.qs.iter_mut().enumerate() { - // ggml uses nearest_int with bit magic here, maybe we want the same - // but we would have to test and benchmark it. - let v = (iscale * xs[j]).round(); - *q = v.min(127.) as i8 - } - for j in 0..QK_K / 16 { - let mut sum = 0i32; - for ii in 0..16 { - sum += y.qs[j * 16 + ii] as i32 - } - y.bsums[j] = sum as i16 - } - y.d = 1.0 / iscale - } - } - Ok(()) - } - - fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { - let k = ys.len(); - if k % QK_K != 0 { - crate::bail!("dequantize_row_q8k: {k} is not divisible by {QK_K}") - } - for (i, x) in xs.iter().enumerate() { - for (j, &q) in x.qs.iter().enumerate() { - ys[i * QK_K + j] = x.d * q as f32 - } - } - Ok(()) - } -} - -// https://github.com/ggerganov/llama.cpp/blob/b5ffb2849d23afe73647f68eec7b68187af09be6/ggml.c#L10605 -pub fn matmul( - mkn: (usize, usize, usize), - lhs: &[f32], - rhs_t: &[T], - dst: &mut [f32], -) -> Result<()> { - let (m, k, n) = mkn; - if m * k != lhs.len() { - crate::bail!("unexpected lhs length {} {mkn:?}", lhs.len()); - } - - let k_in_lhs_blocks = (k + T::BLCK_SIZE - 1) / T::BLCK_SIZE; - let k_in_rhs_blocks = (k + T::VecDotType::BLCK_SIZE - 1) / T::VecDotType::BLCK_SIZE; - // TODO: Do not make this copy if the DotType is f32. - // TODO: Pre-allocate this. - let mut lhs_b = vec![T::VecDotType::zeros(); m * k_in_lhs_blocks]; - for row_idx in 0..m { - let lhs_b = &mut lhs_b[row_idx * k_in_lhs_blocks..(row_idx + 1) * k_in_lhs_blocks]; - let lhs = &lhs[row_idx * k..(row_idx + 1) * k]; - T::VecDotType::from_float(lhs, lhs_b)? - } - let lhs_b = lhs_b.as_slice(); - - for row_idx in 0..m { - let lhs_row = &lhs_b[row_idx * k_in_lhs_blocks..(row_idx + 1) * k_in_lhs_blocks]; - let dst_row = &mut dst[row_idx * n..(row_idx + 1) * n]; - - let result: Result> = dst_row - .into_par_iter() - .enumerate() - .with_min_len(128) - .with_max_len(512) - .map(|(col_idx, dst)| { - let rhs_col = &rhs_t[col_idx * k_in_rhs_blocks..(col_idx + 1) * k_in_rhs_blocks]; - T::vec_dot(k, rhs_col, lhs_row).map(|value| *dst = value) - }) - .collect(); - - result?; - } - Ok(()) -} - -impl GgmlType for f32 { - const DTYPE: GgmlDType = GgmlDType::F32; - const BLCK_SIZE: usize = 1; - type VecDotType = f32; - - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - if xs.len() < n { - crate::bail!("size mismatch {} < {n}", xs.len()) - } - if ys.len() < n { - crate::bail!("size mismatch {} < {n}", ys.len()) - } - let mut res = 0f32; - unsafe { crate::cpu::vec_dot_f32(xs.as_ptr(), ys.as_ptr(), &mut res, n) }; - Ok(res) - } - - fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { - if xs.len() != ys.len() { - crate::bail!("size mismatch {} {}", xs.len(), ys.len()); - } - ys.copy_from_slice(xs); - Ok(()) - } - - fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { - if xs.len() != ys.len() { - crate::bail!("size mismatch {} {}", xs.len(), ys.len()); - } - ys.copy_from_slice(xs); - Ok(()) - } -} - -impl GgmlType for f16 { - const DTYPE: GgmlDType = GgmlDType::F16; - const BLCK_SIZE: usize = 1; - type VecDotType = f16; - - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> Result { - if xs.len() < n { - crate::bail!("size mismatch {} < {n}", xs.len()) - } - if ys.len() < n { - crate::bail!("size mismatch {} < {n}", ys.len()) - } - let mut res = 0f32; - unsafe { crate::cpu::vec_dot_f16(xs.as_ptr(), ys.as_ptr(), &mut res, n) }; - Ok(res) - } - - fn from_float(xs: &[f32], ys: &mut [Self]) -> Result<()> { - if xs.len() != ys.len() { - crate::bail!("size mismatch {} {}", xs.len(), ys.len()); - } - // TODO: vectorize - for (x, y) in xs.iter().zip(ys.iter_mut()) { - *y = f16::from_f32(*x) - } - Ok(()) - } - - fn to_float(xs: &[Self], ys: &mut [f32]) -> Result<()> { - if xs.len() != ys.len() { - crate::bail!("size mismatch {} {}", xs.len(), ys.len()); - } - // TODO: vectorize - for (x, y) in xs.iter().zip(ys.iter_mut()) { - *y = x.to_f32() - } - Ok(()) - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/metal.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/metal.rs deleted file mode 100644 index af1cf36973..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/quantized/metal.rs +++ /dev/null @@ -1,217 +0,0 @@ -use super::{GgmlDType, QStorage}; -use crate::backend::BackendStorage; -use crate::{DType, MetalDevice, MetalStorage, Result, Shape}; -use metal::Buffer; -use std::sync::Arc; - -pub struct QMetalStorage { - dtype: GgmlDType, - device: MetalDevice, - buffer: Arc, -} - -impl QMetalStorage { - pub fn zeros(device: &MetalDevice, elem_count: usize, dtype: GgmlDType) -> Result { - let size = elem_count * dtype.type_size() / dtype.block_size(); - let buffer = device.allocate_zeros(size)?; - Ok(Self { - buffer, - device: device.clone(), - dtype, - }) - } - - pub fn dtype(&self) -> GgmlDType { - self.dtype - } - - pub fn device(&self) -> &MetalDevice { - &self.device - } - - pub fn buffer(&self) -> &Buffer { - &self.buffer - } - - pub fn dequantize(&self, elem_count: usize) -> Result { - use crate::quantized::k_quants::GgmlType; - - let buffer = self.device.new_buffer_managed(self.buffer.length())?; - let command_buffer = self.device.command_buffer()?; - command_buffer.set_label("to_cpu"); - let blit = command_buffer.new_blit_command_encoder(); - blit.set_label("blit_to_cpu"); - blit.copy_from_buffer(&self.buffer, 0, &buffer, 0, self.buffer.length()); - blit.end_encoding(); - self.device.wait_until_completed()?; - let mut out = vec![0.0; elem_count]; - let block_len = elem_count / self.dtype.block_size(); - match self.dtype { - GgmlDType::F32 => { - let vec: Vec = read_to_vec(&buffer, block_len); - f32::to_float(&vec, &mut out)?; - } - GgmlDType::F16 => { - let vec: Vec = read_to_vec(&buffer, block_len); - half::f16::to_float(&vec, &mut out)?; - } - GgmlDType::Q4_0 => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ4_0::to_float(&vec, &mut out)?; - } - GgmlDType::Q4_1 => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ4_1::to_float(&vec, &mut out)?; - } - GgmlDType::Q5_0 => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ5_0::to_float(&vec, &mut out)?; - } - GgmlDType::Q5_1 => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ5_1::to_float(&vec, &mut out)?; - } - GgmlDType::Q8_0 => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ8_0::to_float(&vec, &mut out)?; - } - GgmlDType::Q8_1 => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ8_1::to_float(&vec, &mut out)?; - } - GgmlDType::Q2K => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ2K::to_float(&vec, &mut out)?; - } - GgmlDType::Q3K => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ3K::to_float(&vec, &mut out)?; - } - GgmlDType::Q4K => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ4K::to_float(&vec, &mut out)?; - } - GgmlDType::Q5K => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ5K::to_float(&vec, &mut out)?; - } - GgmlDType::Q6K => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ6K::to_float(&vec, &mut out)?; - } - GgmlDType::Q8K => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ8K::to_float(&vec, &mut out)?; - } - } - - let buffer = self.device.new_buffer_with_data(&out)?; - Ok(MetalStorage::new(buffer, self.device.clone(), DType::F32)) - } - - pub fn quantize(&mut self, src: &MetalStorage) -> Result<()> { - // Quantization only happens on CPU for now. - let src = src.to_cpu::()?; - let elem_count = src.len(); - let src = crate::Storage::Cpu(crate::CpuStorage::F32(src)); - let mut qcpu_storage = crate::Device::Cpu.qzeros(elem_count, self.dtype)?; - qcpu_storage.quantize(&src)?; - let buffer = self.device.new_buffer_with_data(&qcpu_storage.data()?)?; - self.buffer = buffer; - Ok(()) - } - - pub fn storage_size_in_bytes(&self) -> usize { - self.buffer.length() as usize - } - - pub fn fwd( - &self, - self_shape: &Shape, - storage: &MetalStorage, - layout: &crate::Layout, - ) -> Result<(MetalStorage, Shape)> { - use crate::MetalError; - - if !layout.is_contiguous() { - crate::bail!("input tensor is not contiguous {layout:?}") - } - let src_shape = layout.shape(); - // self is transposed so n is first then k. - if src_shape.rank() < 2 { - crate::bail!("input tensor has only one dimension {layout:?}") - } - let (n, k) = self_shape.dims2()?; - let mut dst_shape = src_shape.dims().to_vec(); - - let (b, m) = match dst_shape.len() { - 3 => (dst_shape[0], dst_shape[1]), - 2 => (1, dst_shape[0]), - n => crate::bail!("Invalid rank {n} for quantized matmul metal"), - }; - let last_k = dst_shape.pop().unwrap(); - if last_k != k { - crate::bail!("input tensor {layout:?} incompatible with {:?}", self_shape) - } - dst_shape.push(n); - let dst_shape = Shape::from(dst_shape); - let device = storage.device().clone(); - let dst = device.new_buffer(dst_shape.elem_count(), DType::F32, "qmatmul")?; - let command_buffer = device.command_buffer()?; - candle_metal_kernels::call_quantized_matmul_t( - device.device(), - &command_buffer, - device.kernels(), - self.dtype.into(), - (b, m, n, k), - storage.buffer(), - layout.start_offset() * storage.dtype().size_in_bytes(), - &self.buffer, - &dst, - ) - .map_err(MetalError::from)?; - let dst_storage = crate::MetalStorage::new(dst, device, DType::F32); - Ok((dst_storage, dst_shape)) - } -} - -pub fn load_quantized( - device: &MetalDevice, - data: &[T], -) -> Result { - let buffer = device.new_buffer_with_data(data)?; - let device = device.clone(); - Ok(QStorage::Metal(QMetalStorage { - dtype: T::DTYPE, - device, - buffer, - })) -} - -fn read_to_vec(buffer: &Buffer, n: usize) -> Vec { - let ptr = buffer.contents() as *const T; - assert!(!ptr.is_null()); - let slice = unsafe { std::slice::from_raw_parts(ptr, n) }; - slice.to_vec() -} - -impl From for candle_metal_kernels::GgmlDType { - fn from(value: GgmlDType) -> Self { - match value { - GgmlDType::Q4_0 => candle_metal_kernels::GgmlDType::Q4_0, - GgmlDType::Q4_1 => candle_metal_kernels::GgmlDType::Q4_1, - GgmlDType::Q5_0 => candle_metal_kernels::GgmlDType::Q5_0, - GgmlDType::Q5_1 => candle_metal_kernels::GgmlDType::Q5_1, - GgmlDType::Q8_0 => candle_metal_kernels::GgmlDType::Q8_0, - GgmlDType::Q8_1 => candle_metal_kernels::GgmlDType::Q8_1, - GgmlDType::Q2K => candle_metal_kernels::GgmlDType::Q2K, - GgmlDType::Q3K => candle_metal_kernels::GgmlDType::Q3K, - GgmlDType::Q4K => candle_metal_kernels::GgmlDType::Q4K, - GgmlDType::Q5K => candle_metal_kernels::GgmlDType::Q5K, - GgmlDType::Q6K => candle_metal_kernels::GgmlDType::Q6K, - GgmlDType::Q8K => candle_metal_kernels::GgmlDType::Q8K, - GgmlDType::F16 => candle_metal_kernels::GgmlDType::F16, - GgmlDType::F32 => candle_metal_kernels::GgmlDType::F32, - } - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/mod.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/mod.rs deleted file mode 100644 index f7abcd93d3..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/quantized/mod.rs +++ /dev/null @@ -1,491 +0,0 @@ -use crate::{CpuStorage, Device, Result, Shape, Storage, Tensor}; -use k_quants::*; -use std::borrow::Cow; - -#[cfg(target_feature = "avx")] -pub mod avx; -mod dummy_cuda; -mod dummy_metal; -pub mod ggml_file; -pub mod gguf_file; -pub mod k_quants; -#[cfg(feature = "metal")] -pub mod metal; -#[cfg(not(feature = "metal"))] -mod metal { - pub use super::dummy_metal::*; -} -#[cfg(feature = "cuda")] -pub mod cuda; -#[cfg(not(feature = "cuda"))] -mod cuda { - pub use super::dummy_cuda::*; -} - -#[cfg(target_feature = "neon")] -pub mod neon; -#[cfg(target_feature = "simd128")] -pub mod simd128; -pub mod utils; -use half::f16; - -pub use k_quants::GgmlType; - -pub struct QTensor { - storage: QStorage, - shape: Shape, -} - -impl Device { - fn qzeros(&self, elem_count: usize, dtype: GgmlDType) -> Result { - match self { - Device::Cpu => { - let storage = dtype.cpu_zeros(elem_count); - Ok(QStorage::Cpu(storage)) - } - Device::Metal(metal) => { - let storage = metal::QMetalStorage::zeros(metal, elem_count, dtype)?; - Ok(QStorage::Metal(storage)) - } - Device::Cuda(cuda) => { - let storage = cuda::QCudaStorage::zeros(cuda, elem_count, dtype)?; - Ok(QStorage::Cuda(storage)) - } - } - } -} - -pub enum QStorage { - Cpu(Box), - Metal(metal::QMetalStorage), - Cuda(cuda::QCudaStorage), -} - -impl QStorage { - fn block_size(&self) -> usize { - match self { - QStorage::Cpu(storage) => storage.block_size(), - QStorage::Metal(storage) => storage.dtype().block_size(), - QStorage::Cuda(storage) => storage.dtype().block_size(), - } - } - - fn dtype(&self) -> GgmlDType { - match self { - QStorage::Cpu(storage) => storage.dtype(), - QStorage::Metal(storage) => storage.dtype(), - QStorage::Cuda(storage) => storage.dtype(), - } - } - - fn device(&self) -> Device { - match self { - QStorage::Cpu(_storage) => Device::Cpu, - QStorage::Metal(storage) => Device::Metal(storage.device().clone()), - QStorage::Cuda(storage) => Device::Cuda(storage.device().clone()), - } - } - - fn size_in_bytes(&self) -> usize { - match self { - QStorage::Cpu(storage) => storage.storage_size_in_bytes(), - QStorage::Metal(storage) => storage.storage_size_in_bytes(), - QStorage::Cuda(storage) => storage.storage_size_in_bytes(), - } - } - - fn quantize(&mut self, src: &Storage) -> Result<()> { - match (self, src) { - (QStorage::Cpu(storage), Storage::Cpu(src)) => { - storage.from_float(src.as_slice::()?)?; - } - (QStorage::Metal(storage), Storage::Metal(src)) => storage.quantize(src)?, - (QStorage::Cuda(storage), Storage::Cuda(src)) => storage.quantize(src)?, - _ => crate::bail!("Invalid dequantize storage locations do not match"), - } - Ok(()) - } - - fn dequantize(&self, elem_count: usize) -> Result { - match self { - QStorage::Cpu(storage) => Ok(Storage::Cpu(storage.dequantize(elem_count)?)), - QStorage::Metal(storage) => Ok(Storage::Metal(storage.dequantize(elem_count)?)), - QStorage::Cuda(storage) => Ok(Storage::Cuda(storage.dequantize(elem_count)?)), - } - } - - fn data(&self) -> Result> { - match self { - QStorage::Cpu(storage) => { - let data_ptr = storage.as_ptr(); - let size_in_bytes = storage.storage_size_in_bytes(); - let data = unsafe { std::slice::from_raw_parts(data_ptr, size_in_bytes) }; - Ok(Cow::from(data)) - } - QStorage::Metal(_) | QStorage::Cuda(_) => { - crate::bail!("not implemented"); - } - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum GgmlDType { - F32, - F16, - Q4_0, - Q4_1, - Q5_0, - Q5_1, - Q8_0, - Q8_1, - Q2K, - Q3K, - Q4K, - Q5K, - Q6K, - Q8K, -} - -impl GgmlDType { - pub(crate) fn from_u32(u: u32) -> Result { - let dtype = match u { - 0 => Self::F32, - 1 => Self::F16, - 2 => Self::Q4_0, - 3 => Self::Q4_1, - 6 => Self::Q5_0, - 7 => Self::Q5_1, - 8 => Self::Q8_0, - 9 => Self::Q8_1, - 10 => Self::Q2K, - 11 => Self::Q3K, - 12 => Self::Q4K, - 13 => Self::Q5K, - 14 => Self::Q6K, - 15 => Self::Q8K, - _ => crate::bail!("unknown dtype for tensor {u}"), - }; - Ok(dtype) - } - - pub(crate) fn to_u32(self) -> u32 { - match self { - Self::F32 => 0, - Self::F16 => 1, - Self::Q4_0 => 2, - Self::Q4_1 => 3, - Self::Q5_0 => 6, - Self::Q5_1 => 7, - Self::Q8_0 => 8, - Self::Q8_1 => 9, - Self::Q2K => 10, - Self::Q3K => 11, - Self::Q4K => 12, - Self::Q5K => 13, - Self::Q6K => 14, - Self::Q8K => 15, - } - } - - /// The block dtype - pub fn cpu_zeros(&self, elem_count: usize) -> Box { - match self { - Self::F32 => Box::new(vec![f32::zeros(); elem_count]), - Self::F16 => Box::new(vec![f16::zeros(); elem_count]), - Self::Q4_0 => Box::new(vec![BlockQ4_0::zeros(); elem_count / BlockQ4_0::BLCK_SIZE]), - Self::Q4_1 => Box::new(vec![BlockQ4_1::zeros(); elem_count / BlockQ4_1::BLCK_SIZE]), - Self::Q5_0 => Box::new(vec![BlockQ5_0::zeros(); elem_count / BlockQ5_0::BLCK_SIZE]), - Self::Q5_1 => Box::new(vec![BlockQ5_1::zeros(); elem_count / BlockQ5_1::BLCK_SIZE]), - Self::Q8_0 => Box::new(vec![BlockQ8_0::zeros(); elem_count / BlockQ8_0::BLCK_SIZE]), - Self::Q8_1 => Box::new(vec![BlockQ8_1::zeros(); elem_count / BlockQ8_1::BLCK_SIZE]), - Self::Q2K => Box::new(vec![BlockQ2K::zeros(); elem_count / BlockQ2K::BLCK_SIZE]), - Self::Q3K => Box::new(vec![BlockQ3K::zeros(); elem_count / BlockQ3K::BLCK_SIZE]), - Self::Q4K => Box::new(vec![BlockQ4K::zeros(); elem_count / BlockQ4K::BLCK_SIZE]), - Self::Q5K => Box::new(vec![BlockQ5K::zeros(); elem_count / BlockQ5K::BLCK_SIZE]), - Self::Q6K => Box::new(vec![BlockQ6K::zeros(); elem_count / BlockQ6K::BLCK_SIZE]), - Self::Q8K => Box::new(vec![BlockQ8K::zeros(); elem_count / BlockQ8K::BLCK_SIZE]), - } - } - /// The type size for blocks in bytes. - pub fn type_size(&self) -> usize { - use k_quants::*; - match self { - Self::F32 => 4, - Self::F16 => 2, - Self::Q4_0 => std::mem::size_of::(), - Self::Q4_1 => std::mem::size_of::(), - Self::Q5_0 => std::mem::size_of::(), - Self::Q5_1 => std::mem::size_of::(), - // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L932 - Self::Q8_0 => std::mem::size_of::(), - Self::Q8_1 => std::mem::size_of::(), - Self::Q2K => std::mem::size_of::(), - Self::Q3K => std::mem::size_of::(), - Self::Q4K => std::mem::size_of::(), - Self::Q5K => std::mem::size_of::(), - Self::Q6K => std::mem::size_of::(), - Self::Q8K => std::mem::size_of::(), - } - } - - /// The block size, i.e. the number of elements stored in each block. - pub fn block_size(&self) -> usize { - match self { - Self::F32 => 1, - Self::F16 => 1, - Self::Q4_0 => k_quants::QK4_0, - Self::Q4_1 => k_quants::QK4_1, - Self::Q5_0 => k_quants::QK5_0, - Self::Q5_1 => k_quants::QK5_1, - Self::Q8_0 => k_quants::QK8_0, - Self::Q8_1 => k_quants::QK8_1, - Self::Q2K | Self::Q3K | Self::Q4K | Self::Q5K | Self::Q6K | Self::Q8K => k_quants::QK_K, - } - } -} - -// A version of GgmlType without `vec_dot` so that it can be dyn boxed. -pub trait QuantizedType: Send + Sync { - fn dtype(&self) -> GgmlDType; - fn matmul_t(&self, mkn: (usize, usize, usize), lhs: &[f32], dst: &mut [f32]) -> Result<()>; - fn dequantize(&self, elem_count: usize) -> Result; - fn storage_size_in_bytes(&self) -> usize; - fn as_ptr(&self) -> *const u8; - fn block_size(&self) -> usize; - #[allow(clippy::wrong_self_convention)] - fn from_float(&mut self, xs: &[f32]) -> Result<()>; - fn size(&self) -> usize; -} - -impl QuantizedType for Vec { - fn matmul_t(&self, mkn: (usize, usize, usize), lhs: &[f32], dst: &mut [f32]) -> Result<()> { - k_quants::matmul(mkn, lhs, self.as_slice(), dst) - } - - fn size(&self) -> usize { - self.len() * core::mem::size_of::() - } - - fn from_float(&mut self, xs: &[f32]) -> Result<()> { - T::from_float(xs, self) - } - - fn dtype(&self) -> GgmlDType { - T::DTYPE - } - - fn block_size(&self) -> usize { - T::BLCK_SIZE - } - - fn dequantize(&self, elem_count: usize) -> Result { - let mut ys = vec![0.0f32; elem_count]; - T::to_float(self.as_slice(), &mut ys)?; - Ok(CpuStorage::F32(ys)) - } - - fn storage_size_in_bytes(&self) -> usize { - self.len() * std::mem::size_of::() - } - - fn as_ptr(&self) -> *const u8 { - self.as_ptr() as *const u8 - } -} - -impl std::fmt::Debug for QTensor { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "QTensor[{:?}; {:?}]", self.shape, self.dtype()) - } -} - -fn check_shape(shape: &Shape, block_size: usize) -> Result<()> { - let dims = shape.dims(); - if dims.is_empty() { - crate::bail!("scalar tensor cannot be quantized {shape:?}") - } - if dims[dims.len() - 1] % block_size != 0 { - crate::bail!( - "quantized tensor must have their last dim divisible by block size {shape:?} {}", - block_size - ) - } - Ok(()) -} - -impl QTensor { - pub fn new>(storage: QStorage, shape: S) -> Result { - let shape = shape.into(); - check_shape(&shape, storage.block_size())?; - Ok(Self { storage, shape }) - } - - pub fn quantize(src: &Tensor, dtype: GgmlDType) -> Result { - let shape = src.shape(); - let block_size = dtype.block_size(); - check_shape(shape, block_size)?; - let src = src.to_dtype(crate::DType::F32)?.flatten_all()?; - let elem_count = shape.elem_count(); - if elem_count % block_size != 0 { - crate::bail!( - "tensor size ({shape:?}) is not divisible by block size {}", - block_size - ) - } - let mut storage = src.device().qzeros(elem_count, dtype)?; - storage.quantize(&src.storage())?; - Ok(Self { - storage, - shape: shape.clone(), - }) - } - - pub fn dtype(&self) -> GgmlDType { - self.storage.dtype() - } - - pub fn device(&self) -> Device { - self.storage.device() - } - - pub fn rank(&self) -> usize { - self.shape.rank() - } - - pub fn shape(&self) -> &Shape { - &self.shape - } - - pub fn dequantize(&self, device: &Device) -> Result { - let storage = self.storage.dequantize(self.shape.elem_count())?; - let none = crate::op::BackpropOp::none(); - let is_variable = false; - crate::tensor::from_storage(storage, self.shape.clone(), none, is_variable) - .to_device(device) - } - - pub fn storage_size_in_bytes(&self) -> usize { - self.storage.size_in_bytes() - } - - pub fn data(&self) -> Result> { - self.storage.data() - } -} - -#[derive(Clone, Debug)] -pub enum QMatMul { - QTensor(std::sync::Arc), - Tensor(Tensor), -} - -thread_local! { - static DEQUANTIZE_ALL: bool = { - match std::env::var("CANDLE_DEQUANTIZE_ALL") { - Ok(s) => { - !s.is_empty() && s != "0" - }, - Err(_) => false, - } - } -} - -impl QMatMul { - pub fn from_arc(qtensor: std::sync::Arc) -> Result { - let dequantize = match qtensor.dtype() { - GgmlDType::F32 | GgmlDType::F16 => true, - _ => DEQUANTIZE_ALL.with(|b| *b), - }; - let t = if dequantize { - let tensor = qtensor.dequantize(&Device::Cpu)?; - Self::Tensor(tensor) - } else { - Self::QTensor(qtensor) - }; - Ok(t) - } - - pub fn from_qtensor(qtensor: QTensor) -> Result { - Self::from_arc(std::sync::Arc::new(qtensor)) - } -} - -impl crate::CustomOp1 for QTensor { - fn name(&self) -> &'static str { - "qmatmul" - } - - fn cpu_fwd( - &self, - storage: &crate::CpuStorage, - layout: &crate::Layout, - ) -> Result<(crate::CpuStorage, Shape)> { - if !layout.is_contiguous() { - crate::bail!("input tensor is not contiguous {layout:?}") - } - let src_shape = layout.shape(); - // self is transposed so n is first then k. - let (n, k) = self.shape.dims2()?; - if src_shape.rank() < 2 { - crate::bail!("input tensor has only one dimension {layout:?}") - } - let mut dst_shape = src_shape.dims().to_vec(); - let last_k = dst_shape.pop().unwrap(); - if last_k != k { - crate::bail!("input tensor {layout:?} incompatible with {:?}", self.shape) - } - dst_shape.push(n); - let dst_shape = Shape::from(dst_shape); - #[allow(clippy::infallible_destructuring_match)] - let self_storage = match &self.storage { - QStorage::Cpu(storage) => storage, - QStorage::Metal(_) | QStorage::Cuda(_) => crate::bail!("Invalid storage"), - }; - let slice = storage.as_slice::()?; - let slice = &slice[layout.start_offset()..layout.start_offset() + src_shape.elem_count()]; - let mut dst_storage = vec![0f32; dst_shape.elem_count()]; - self_storage.matmul_t((dst_shape.elem_count() / n, k, n), slice, &mut dst_storage)?; - Ok((crate::CpuStorage::F32(dst_storage), dst_shape)) - } - - fn metal_fwd( - &self, - storage: &crate::MetalStorage, - layout: &crate::Layout, - ) -> Result<(crate::MetalStorage, Shape)> { - let self_storage = match &self.storage { - QStorage::Metal(metal) => metal, - _ => unreachable!("Cannot call metal matmul on non metal QTensor"), - }; - self_storage.fwd(&self.shape, storage, layout) - } - - fn cuda_fwd( - &self, - storage: &crate::CudaStorage, - layout: &crate::Layout, - ) -> Result<(crate::CudaStorage, Shape)> { - let self_storage = match &self.storage { - QStorage::Cuda(cuda) => cuda, - _ => unreachable!("Cannot call cuda matmul on non cuda QTensor"), - }; - self_storage.fwd(&self.shape, storage, layout) - } -} - -impl crate::Module for QMatMul { - fn forward(&self, xs: &Tensor) -> Result { - match self { - Self::QTensor(t) => xs.apply_op1_no_bwd(t.as_ref()), - Self::Tensor(w) => { - let w = match *xs.dims() { - [b1, b2, _, _] => w.broadcast_left((b1, b2))?.t()?, - [bsize, _, _] => w.broadcast_left(bsize)?.t()?, - _ => w.t()?, - }; - xs.matmul(&w) - } - } - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/neon.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/neon.rs deleted file mode 100644 index c4d5d6f41a..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/quantized/neon.rs +++ /dev/null @@ -1,613 +0,0 @@ -use super::k_quants::{ - BlockQ2K, BlockQ3K, BlockQ4K, BlockQ4_0, BlockQ5K, BlockQ6K, BlockQ8K, BlockQ8_0, QK8_0, QK_K, -}; -use crate::Result; -use byteorder::{ByteOrder, LittleEndian}; - -#[allow(unused_imports)] -#[cfg(target_arch = "arm")] -use core::arch::arm::*; - -#[allow(unused_imports)] -#[cfg(target_arch = "aarch64")] -use core::arch::aarch64::*; - -#[inline(always)] -unsafe fn vdotq_s32(a: int8x16_t, b: int8x16_t) -> int32x4_t { - // TODO: dotprod - let p0 = vmull_s8(vget_low_s8(a), vget_low_s8(b)); - let p1 = vmull_s8(vget_high_s8(a), vget_high_s8(b)); - vaddq_s32(vpaddlq_s16(p0), vpaddlq_s16(p1)) -} - -#[inline(always)] -pub(crate) fn vec_dot_q4_0_q8_0(n: usize, xs: &[BlockQ4_0], ys: &[BlockQ8_0]) -> Result { - let qk = QK8_0; - let nb = n / qk; - if n % QK8_0 != 0 { - crate::bail!("vec_dot_q4_0_q8_0: {n} is not divisible by {qk}") - } - - unsafe { - let mut sumv0 = vdupq_n_f32(0.0f32); - for i in 0..nb { - let x0 = &xs[i]; - let y0 = &ys[i]; - - let m4b = vdupq_n_u8(0x0F); - let s8b = vdupq_n_s8(0x8); - - let v0_0 = vld1q_u8(x0.qs.as_ptr()); - - // 4-bit -> 8-bit - let v0_0l = vreinterpretq_s8_u8(vandq_u8(v0_0, m4b)); - let v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4)); - - // sub 8 - let v0_0ls = vsubq_s8(v0_0l, s8b); - let v0_0hs = vsubq_s8(v0_0h, s8b); - - // load y - let v1_0l = vld1q_s8(y0.qs.as_ptr()); - let v1_0h = vld1q_s8(y0.qs.as_ptr().add(16)); - - let pl0 = vdotq_s32(v0_0ls, v1_0l); - let ph0 = vdotq_s32(v0_0hs, v1_0h); - sumv0 = vmlaq_n_f32( - sumv0, - vcvtq_f32_s32(vaddq_s32(pl0, ph0)), - x0.d.to_f32() * y0.d.to_f32(), - ); - } - Ok(vaddvq_f32(sumv0)) - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q8_0_q8_0(n: usize, xs: &[BlockQ8_0], ys: &[BlockQ8_0]) -> Result { - let qk = QK8_0; - if n % QK8_0 != 0 { - crate::bail!("vec_dot_q8_0_q8_0: {n} is not divisible by {qk}") - } - let nb = n / QK8_0; - unsafe { - let mut sumv0 = vdupq_n_f32(0.0f32); - for i in 0..nb { - let x0 = &xs[i]; - let y0 = &ys[i]; - - let x0_0 = vld1q_s8(x0.qs.as_ptr()); - let x0_1 = vld1q_s8(x0.qs.as_ptr().add(16)); - - // load y - let y0_0 = vld1q_s8(y0.qs.as_ptr()); - let y0_1 = vld1q_s8(y0.qs.as_ptr().add(16)); - - let p0 = vdotq_s32(x0_0, y0_0); - let p1 = vdotq_s32(x0_1, y0_1); - - sumv0 = vmlaq_n_f32( - sumv0, - vcvtq_f32_s32(vaddq_s32(p0, p1)), - x0.d.to_f32() * y0.d.to_f32(), - ); - } - Ok(vaddvq_f32(sumv0)) - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q8k_q8k(n: usize, xs: &[BlockQ8K], ys: &[BlockQ8K]) -> Result { - let qk = QK_K; - if n % QK_K != 0 { - crate::bail!("vec_dot_q8k_q8k: {n} is not divisible by {qk}") - } - - let mut sumf = 0f32; - for (xs, ys) in xs.iter().zip(ys.iter()) { - unsafe { - let mut sum_i = vdupq_n_s32(0); - let scale = xs.d * ys.d; - let xs = xs.qs.as_ptr(); - let ys = ys.qs.as_ptr(); - for i in (0..QK_K).step_by(16) { - let xs = vld1q_s8(xs.add(i)); - let ys = vld1q_s8(ys.add(i)); - let xy = vdotq_s32(xs, ys); - sum_i = vaddq_s32(sum_i, xy) - } - sumf += vaddvq_s32(sum_i) as f32 * scale - } - } - Ok(sumf) -} - -#[inline(always)] -pub(crate) fn vec_dot_q6k_q8k(n: usize, xs: &[BlockQ6K], ys: &[BlockQ8K]) -> Result { - if n % QK_K != 0 { - crate::bail!("vec_dot_q6k_q8k: {n} is not divisible by {QK_K}") - } - let mut sum = 0f32; - unsafe { - let m4b = vdupq_n_u8(0xF); - - let mone = vdupq_n_u8(3); - - for (x, y) in xs.iter().zip(ys.iter()) { - let d_all = x.d.to_f32(); - - let mut q6 = x.ql.as_ptr(); - let mut qh = x.qh.as_ptr(); - let mut q8 = y.qs.as_ptr(); - - let mut scale = x.scales.as_ptr(); - - let q8sums = vld1q_s16_x2(y.bsums.as_ptr()); - let scales = vld1q_s8(scale); - let q6scales = int16x8x2_t( - vmovl_s8(vget_low_s8(scales)), - vmovl_s8(vget_high_s8(scales)), - ); - - let prod = vaddq_s32( - vaddq_s32( - vmull_s16(vget_low_s16(q8sums.0), vget_low_s16(q6scales.0)), - vmull_s16(vget_high_s16(q8sums.0), vget_high_s16(q6scales.0)), - ), - vaddq_s32( - vmull_s16(vget_low_s16(q8sums.1), vget_low_s16(q6scales.1)), - vmull_s16(vget_high_s16(q8sums.1), vget_high_s16(q6scales.1)), - ), - ); - let isum_mins = vaddvq_s32(prod); - - let mut isum = 0i32; - - for _j in 0..QK_K / 128 { - let qhbits = vld1q_u8_x2(qh); - qh = qh.add(32); - let q6bits = vld1q_u8_x4(q6); - q6 = q6.add(64); - let q8bytes = vld1q_s8_x4(q8); - q8 = q8.add(64); - - let q6h_0 = vshlq_n_u8(vandq_u8(mone, qhbits.0), 4); - let q6h_1 = vshlq_n_u8(vandq_u8(mone, qhbits.1), 4); - let shifted = vshrq_n_u8(qhbits.0, 2); - let q6h_2 = vshlq_n_u8(vandq_u8(mone, shifted), 4); - let shifted = vshrq_n_u8(qhbits.1, 2); - let q6h_3 = vshlq_n_u8(vandq_u8(mone, shifted), 4); - - let q6bytes_0 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.0, m4b), q6h_0)); - let q6bytes_1 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.1, m4b), q6h_1)); - let q6bytes_2 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.2, m4b), q6h_2)); - let q6bytes_3 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.3, m4b), q6h_3)); - - let p0 = vdotq_s32(q6bytes_0, q8bytes.0); - let p1 = vdotq_s32(q6bytes_1, q8bytes.1); - let (scale0, scale1) = (*scale as i32, *scale.add(1) as i32); - isum += vaddvq_s32(p0) * scale0 + vaddvq_s32(p1) * scale1; - scale = scale.add(2); - - let p2 = vdotq_s32(q6bytes_2, q8bytes.2); - let p3 = vdotq_s32(q6bytes_3, q8bytes.3); - let (scale0, scale1) = (*scale as i32, *scale.add(1) as i32); - isum += vaddvq_s32(p2) * scale0 + vaddvq_s32(p3) * scale1; - scale = scale.add(2); - - let q8bytes = vld1q_s8_x4(q8); - q8 = q8.add(64); - - let shifted = vshrq_n_u8(qhbits.0, 4); - let q6h_0 = vshlq_n_u8(vandq_u8(mone, shifted), 4); - let shifted = vshrq_n_u8(qhbits.1, 4); - let q6h_1 = vshlq_n_u8(vandq_u8(mone, shifted), 4); - let shifted = vshrq_n_u8(qhbits.0, 6); - let q6h_2 = vshlq_n_u8(vandq_u8(mone, shifted), 4); - let shifted = vshrq_n_u8(qhbits.1, 6); - let q6h_3 = vshlq_n_u8(vandq_u8(mone, shifted), 4); - - let q6bytes_0 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.0, 4), q6h_0)); - let q6bytes_1 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.1, 4), q6h_1)); - let q6bytes_2 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.2, 4), q6h_2)); - let q6bytes_3 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.3, 4), q6h_3)); - - let p0 = vdotq_s32(q6bytes_0, q8bytes.0); - let p1 = vdotq_s32(q6bytes_1, q8bytes.1); - let (scale0, scale1) = (*scale as i32, *scale.add(1) as i32); - isum += vaddvq_s32(p0) * scale0 + vaddvq_s32(p1) * scale1; - scale = scale.add(2); - - let p2 = vdotq_s32(q6bytes_2, q8bytes.2); - let p3 = vdotq_s32(q6bytes_3, q8bytes.3); - let (scale0, scale1) = (*scale as i32, *scale.add(1) as i32); - isum += vaddvq_s32(p2) * scale0 + vaddvq_s32(p3) * scale1; - scale = scale.add(2); - } - sum += d_all * y.d * ((isum - 32 * isum_mins) as f32); - } - } - Ok(sum) -} - -#[inline(always)] -pub(crate) fn vec_dot_q5k_q8k(n: usize, xs: &[BlockQ5K], ys: &[BlockQ8K]) -> Result { - if n % QK_K != 0 { - crate::bail!("vec_dot_q5k_q8k: {n} is not divisible by {QK_K}") - } - let mut sumf = 0f32; - let mut utmp = [0u32; 4]; - const KMASK1: u32 = 0x3f3f3f3f; - const KMASK2: u32 = 0x0f0f0f0f; - const KMASK3: u32 = 0x03030303; - - unsafe { - let m4b = vdupq_n_u8(0xF); - let mone = vdupq_n_u8(1); - let mtwo = vdupq_n_u8(2); - - for (x, y) in xs.iter().zip(ys.iter()) { - let d = y.d * x.d.to_f32(); - let dmin = y.d * x.dmin.to_f32(); - - let q8sums = vpaddq_s16( - vld1q_s16(y.bsums.as_ptr()), - vld1q_s16(y.bsums.as_ptr().add(8)), - ); - - LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); - - utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); - let uaux = utmp[1] & KMASK1; - utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); - utmp[2] = uaux; - utmp[0] &= KMASK1; - - let mins8 = vld1_u8((utmp.as_ptr() as *const u8).add(8)); - let mins = vreinterpretq_s16_u16(vmovl_u8(mins8)); - let prod = vaddq_s32( - vmull_s16(vget_low_s16(q8sums), vget_low_s16(mins)), - vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)), - ); - let sumi_mins = vaddvq_s32(prod); - - let mut scales = utmp.as_ptr() as *const u8; - - let mut q5 = x.qs.as_ptr(); - let mut q8 = y.qs.as_ptr(); - - let mut qhbits = vld1q_u8_x2(x.qh.as_ptr()); - - let mut sumi = 0i32; - - for _j in 0..QK_K / 64 { - let q5bits = vld1q_u8_x2(q5); - q5 = q5.add(32); - let q8bytes = vld1q_s8_x4(q8); - q8 = q8.add(64); - - let q5h_0 = vshlq_n_u8(vandq_u8(mone, qhbits.0), 4); - let q5h_1 = vshlq_n_u8(vandq_u8(mone, qhbits.1), 4); - let q5h_2 = vshlq_n_u8(vandq_u8(mtwo, qhbits.0), 3); - let q5h_3 = vshlq_n_u8(vandq_u8(mtwo, qhbits.1), 3); - qhbits.0 = vshrq_n_u8(qhbits.0, 2); - qhbits.1 = vshrq_n_u8(qhbits.1, 2); - - let q5bytes_0 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q5bits.0, m4b), q5h_0)); - let q5bytes_1 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q5bits.1, m4b), q5h_1)); - let q5bytes_2 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.0, 4), q5h_2)); - let q5bytes_3 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.1, 4), q5h_3)); - - let p0 = vdotq_s32(q5bytes_0, q8bytes.0); - let p1 = vdotq_s32(q5bytes_1, q8bytes.1); - sumi += vaddvq_s32(vaddq_s32(p0, p1)) * *scales as i32; - scales = scales.add(1); - - let p2 = vdotq_s32(q5bytes_2, q8bytes.2); - let p3 = vdotq_s32(q5bytes_3, q8bytes.3); - sumi += vaddvq_s32(vaddq_s32(p2, p3)) * *scales as i32; - scales = scales.add(1); - } - sumf += d * sumi as f32 - dmin * sumi_mins as f32; - } - } - Ok(sumf) -} - -#[inline(always)] -pub(crate) fn vec_dot_q4k_q8k(n: usize, xs: &[BlockQ4K], ys: &[BlockQ8K]) -> Result { - if n % QK_K != 0 { - crate::bail!("vec_dot_q4k_q8k: {n} is not divisible by {QK_K}") - } - let mut sumf = 0f32; - let mut utmp = [0u32; 4]; - let mut scales = [0u8; 16]; - const KMASK1: u32 = 0x3f3f3f3f; - const KMASK2: u32 = 0x0f0f0f0f; - const KMASK3: u32 = 0x03030303; - - unsafe { - let m4b = vdupq_n_u8(0xF); - - for (x, y) in xs.iter().zip(ys.iter()) { - let d = y.d * x.d.to_f32(); - let dmin = y.d * x.dmin.to_f32(); - - let q8sums = vpaddq_s16( - vld1q_s16(y.bsums.as_ptr()), - vld1q_s16(y.bsums.as_ptr().add(8)), - ); - - LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); - - let mins8 = vld1_u32( - [ - utmp[1] & KMASK1, - ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4), - ] - .as_ptr(), - ); - utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); - utmp[0] &= KMASK1; - - let mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins8))); - let prod = vaddq_s32( - vmull_s16(vget_low_s16(q8sums), vget_low_s16(mins)), - vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)), - ); - sumf -= dmin * vaddvq_s32(prod) as f32; - - LittleEndian::write_u32_into(&utmp, &mut scales); - - let mut q4 = x.qs.as_ptr(); - let mut q8 = y.qs.as_ptr(); - - let mut sumi1 = 0i32; - let mut sumi2 = 0i32; - - for j in 0..QK_K / 64 { - let q4bits = vld1q_u8_x2(q4); - q4 = q4.add(32); - let q8bytes = vld1q_s8_x2(q8); - q8 = q8.add(32); - let q4bytes = int8x16x2_t( - vreinterpretq_s8_u8(vandq_u8(q4bits.0, m4b)), - vreinterpretq_s8_u8(vandq_u8(q4bits.1, m4b)), - ); - let p0 = vdotq_s32(q4bytes.0, q8bytes.0); - let p1 = vdotq_s32(q4bytes.1, q8bytes.1); - sumi1 += vaddvq_s32(vaddq_s32(p0, p1)) * scales[2 * j] as i32; - - let q8bytes = vld1q_s8_x2(q8); - q8 = q8.add(32); - let q4bytes = int8x16x2_t( - vreinterpretq_s8_u8(vshrq_n_u8(q4bits.0, 4)), - vreinterpretq_s8_u8(vshrq_n_u8(q4bits.1, 4)), - ); - let p2 = vdotq_s32(q4bytes.0, q8bytes.0); - let p3 = vdotq_s32(q4bytes.1, q8bytes.1); - sumi2 += vaddvq_s32(vaddq_s32(p2, p3)) * scales[2 * j + 1] as i32; - } - sumf += d * (sumi1 + sumi2) as f32; - } - } - Ok(sumf) -} - -#[inline(always)] -pub(crate) fn vec_dot_q3k_q8k(n: usize, xs: &[BlockQ3K], ys: &[BlockQ8K]) -> Result { - if n % QK_K != 0 { - crate::bail!("vec_dot_q3k_q8k: {n} is not divisible by {QK_K}") - } - let mut sumf = 0f32; - let mut utmp = [0u32; 4]; - let mut aux = [0u32; 3]; - const KMASK1: u32 = 0x03030303; - const KMASK2: u32 = 0x0f0f0f0f; - - unsafe { - let m3b = vdupq_n_u8(0x3); - let m0 = vdupq_n_u8(1); - let m1 = vshlq_n_u8(m0, 1); - let m2 = vshlq_n_u8(m0, 2); - let m3 = vshlq_n_u8(m0, 3); - for (x, y) in xs.iter().zip(ys.iter()) { - let d = y.d * x.d.to_f32(); - let mut q3 = x.qs.as_ptr(); - let qh = x.hmask.as_ptr(); - let mut q8 = y.qs.as_ptr(); - - let mut qhbits = vld1q_u8_x2(qh); - - let mut isum = 0i32; - - // Set up scales - LittleEndian::read_u32_into(&x.scales, &mut aux); - - utmp[3] = ((aux[1] >> 4) & KMASK2) | (((aux[2] >> 6) & KMASK1) << 4); - utmp[2] = ((aux[0] >> 4) & KMASK2) | (((aux[2] >> 4) & KMASK1) << 4); - utmp[1] = (aux[1] & KMASK2) | (((aux[2] >> 2) & KMASK1) << 4); - utmp[0] = (aux[0] & KMASK2) | ((aux[2] & KMASK1) << 4); - - let mut scale = utmp.as_mut_ptr() as *mut i8; - for j in 0..16 { - *scale.add(j) -= 32i8 - } - - for j in 0..QK_K / 128 { - let q3bits = vld1q_u8_x2(q3); - q3 = q3.add(32); - let q8bytes_1 = vld1q_s8_x4(q8); - q8 = q8.add(64); - let q8bytes_2 = vld1q_s8_x4(q8); - q8 = q8.add(64); - - let q3h_0 = vshlq_n_u8(vbicq_u8(m0, qhbits.0), 2); - let q3h_1 = vshlq_n_u8(vbicq_u8(m0, qhbits.1), 2); - let q3h_2 = vshlq_n_u8(vbicq_u8(m1, qhbits.0), 1); - let q3h_3 = vshlq_n_u8(vbicq_u8(m1, qhbits.1), 1); - - let q3bytes_0 = vsubq_s8( - vreinterpretq_s8_u8(vandq_u8(q3bits.0, m3b)), - vreinterpretq_s8_u8(q3h_0), - ); - let q3bytes_1 = vsubq_s8( - vreinterpretq_s8_u8(vandq_u8(q3bits.1, m3b)), - vreinterpretq_s8_u8(q3h_1), - ); - let q3bytes_2 = vsubq_s8( - vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.0, 2), m3b)), - vreinterpretq_s8_u8(q3h_2), - ); - let q3bytes_3 = vsubq_s8( - vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.1, 2), m3b)), - vreinterpretq_s8_u8(q3h_3), - ); - - let p0 = vdotq_s32(q3bytes_0, q8bytes_1.0); - let p1 = vdotq_s32(q3bytes_1, q8bytes_1.1); - let p2 = vdotq_s32(q3bytes_2, q8bytes_1.2); - let p3 = vdotq_s32(q3bytes_3, q8bytes_1.3); - isum += vaddvq_s32(p0) * *scale as i32 - + vaddvq_s32(p1) * *scale.add(1) as i32 - + vaddvq_s32(p2) * *scale.add(2) as i32 - + vaddvq_s32(p3) * *scale.add(3) as i32; - scale = scale.add(4); - - let q3h_0 = vbicq_u8(m2, qhbits.0); - let q3h_1 = vbicq_u8(m2, qhbits.1); - let q3h_2 = vshrq_n_u8(vbicq_u8(m3, qhbits.0), 1); - let q3h_3 = vshrq_n_u8(vbicq_u8(m3, qhbits.1), 1); - - let q3bytes_0 = vsubq_s8( - vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.0, 4), m3b)), - vreinterpretq_s8_u8(q3h_0), - ); - let q3bytes_1 = vsubq_s8( - vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.1, 4), m3b)), - vreinterpretq_s8_u8(q3h_1), - ); - let q3bytes_2 = vsubq_s8( - vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.0, 6), m3b)), - vreinterpretq_s8_u8(q3h_2), - ); - let q3bytes_3 = vsubq_s8( - vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.1, 6), m3b)), - vreinterpretq_s8_u8(q3h_3), - ); - - let p0 = vdotq_s32(q3bytes_0, q8bytes_2.0); - let p1 = vdotq_s32(q3bytes_1, q8bytes_2.1); - let p2 = vdotq_s32(q3bytes_2, q8bytes_2.2); - let p3 = vdotq_s32(q3bytes_3, q8bytes_2.3); - isum += vaddvq_s32(p0) * *scale as i32 - + vaddvq_s32(p1) * *scale.add(1) as i32 - + vaddvq_s32(p2) * *scale.add(2) as i32 - + vaddvq_s32(p3) * *scale.add(3) as i32; - scale = scale.add(4); - - if j == 0 { - qhbits.0 = vshrq_n_u8(qhbits.0, 4); - qhbits.1 = vshrq_n_u8(qhbits.1, 4); - } - } - sumf += d * isum as f32; - } - } - Ok(sumf) -} - -#[inline(always)] -pub(crate) fn vec_dot_q2k_q8k(n: usize, xs: &[BlockQ2K], ys: &[BlockQ8K]) -> Result { - if n % QK_K != 0 { - crate::bail!("vec_dot_q2k_q8k: {n} is not divisible by {QK_K}") - } - let mut sumf = 0f32; - let mut aux = [0u8; 16]; - - unsafe { - let m3 = vdupq_n_u8(0x3); - let m4 = vdupq_n_u8(0xF); - - for (x, y) in xs.iter().zip(ys.iter()) { - let d = y.d * x.d.to_f32(); - let dmin = -y.d * x.dmin.to_f32(); - - let mut q2 = x.qs.as_ptr(); - let mut q8 = y.qs.as_ptr(); - let sc = x.scales.as_ptr(); - - let mins_and_scales = vld1q_u8(sc); - let scales = vandq_u8(mins_and_scales, m4); - vst1q_u8(aux.as_mut_ptr(), scales); - - let mins = vshrq_n_u8(mins_and_scales, 4); - let q8sums = vld1q_s16_x2(y.bsums.as_ptr()); - let mins16 = int16x8x2_t( - vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(mins))), - vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(mins))), - ); - let s0 = vaddq_s32( - vmull_s16(vget_low_s16(mins16.0), vget_low_s16(q8sums.0)), - vmull_s16(vget_high_s16(mins16.0), vget_high_s16(q8sums.0)), - ); - let s1 = vaddq_s32( - vmull_s16(vget_low_s16(mins16.1), vget_low_s16(q8sums.1)), - vmull_s16(vget_high_s16(mins16.1), vget_high_s16(q8sums.1)), - ); - sumf += dmin * vaddvq_s32(vaddq_s32(s0, s1)) as f32; - - let mut isum = 0i32; - let mut is = 0usize; - - // TODO: dotprod - for _j in 0..QK_K / 128 { - let q2bits = vld1q_u8_x2(q2); - q2 = q2.add(32); - - let q8bytes = vld1q_s8_x2(q8); - q8 = q8.add(32); - let mut q2bytes = int8x16x2_t( - vreinterpretq_s8_u8(vandq_u8(q2bits.0, m3)), - vreinterpretq_s8_u8(vandq_u8(q2bits.1, m3)), - ); - isum += multiply_accum_with_scale(&aux, is, 0, q2bytes, q8bytes); - - let q8bytes = vld1q_s8_x2(q8); - q8 = q8.add(32); - q2bytes.0 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.0, 2), m3)); - q2bytes.1 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.1, 2), m3)); - isum += multiply_accum_with_scale(&aux, is, 2, q2bytes, q8bytes); - - let q8bytes = vld1q_s8_x2(q8); - q8 = q8.add(32); - q2bytes.0 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.0, 4), m3)); - q2bytes.1 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.1, 4), m3)); - isum += multiply_accum_with_scale(&aux, is, 4, q2bytes, q8bytes); - - let q8bytes = vld1q_s8_x2(q8); - q8 = q8.add(32); - q2bytes.0 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.0, 6), m3)); - q2bytes.1 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.1, 6), m3)); - isum += multiply_accum_with_scale(&aux, is, 6, q2bytes, q8bytes); - - is += 8; - } - sumf += d * isum as f32; - } - } - Ok(sumf) -} - -#[inline(always)] -unsafe fn multiply_accum_with_scale( - aux: &[u8; 16], - is: usize, - index: usize, - q2bytes: int8x16x2_t, - q8bytes: int8x16x2_t, -) -> i32 { - let p1 = vdotq_s32(q2bytes.0, q8bytes.0); - let p2 = vdotq_s32(q2bytes.1, q8bytes.1); - vaddvq_s32(p1) * aux[is + index] as i32 + vaddvq_s32(p2) * aux[is + 1 + index] as i32 -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/simd128.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/simd128.rs deleted file mode 100644 index 1c8c0f2068..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/quantized/simd128.rs +++ /dev/null @@ -1,419 +0,0 @@ -use super::k_quants::{BlockQ2K, BlockQ4K, BlockQ4_0, BlockQ6K, BlockQ8K, BlockQ8_0, QK8_0, QK_K}; -use crate::Result; -use byteorder::{ByteOrder, LittleEndian}; -use half::f16; - -use core::arch::wasm32::*; - -#[inline(always)] -pub(crate) fn vec_dot_q4_0_q8_0(n: usize, xs: &[BlockQ4_0], ys: &[BlockQ8_0]) -> Result { - let qk = QK8_0; - if n % QK8_0 != 0 { - crate::bail!("vec_dot_q4_0_q8_0: {n} is not divisible by {qk}") - } - unsafe { - let mut acc = f32x4_splat(0.0f32); - for (x, y) in xs.iter().zip(ys.iter()) { - let x1234 = v128_load(x.qs.as_ptr() as *const v128); - let x12 = v128_and(x1234, u8x16_splat(0x0F)); - let x12 = i8x16_sub(x12, i8x16_splat(8)); - let x34 = u8x16_shr(x1234, 4); - let x34 = i8x16_sub(x34, i8x16_splat(8)); - - let x1 = i16x8_extend_low_i8x16(x12); - let y1 = i16x8_load_extend_i8x8(y.qs.as_ptr()); - let sum_xy = i32x4_dot_i16x8(x1, y1); - - let x2 = i16x8_extend_high_i8x16(x12); - let y2 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(8)); - let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x2, y2)); - - let x3 = i16x8_extend_low_i8x16(x34); - let y3 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(16)); - let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x3, y3)); - - let x4 = i16x8_extend_high_i8x16(x34); - let y4 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(24)); - let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x4, y4)); - - let sum_xy = f32x4_convert_i32x4(sum_xy); - - // f32x4_relaxed_madd is nightly only. - let d = f32x4_splat(f16::to_f32(x.d) * f16::to_f32(y.d)); - let scaled = f32x4_mul(sum_xy, d); - acc = f32x4_add(acc, scaled) - } - let res = f32x4_extract_lane::<0>(acc) - + f32x4_extract_lane::<1>(acc) - + f32x4_extract_lane::<2>(acc) - + f32x4_extract_lane::<3>(acc); - Ok(res) - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q8_0_q8_0(n: usize, xs: &[BlockQ8_0], ys: &[BlockQ8_0]) -> Result { - let qk = QK8_0; - if n % QK8_0 != 0 { - crate::bail!("vec_dot_q8_0_q8_0: {n} is not divisible by {qk}") - } - unsafe { - let mut acc = f32x4_splat(0.0f32); - for (x, y) in xs.iter().zip(ys.iter()) { - let x1 = i16x8_load_extend_i8x8(x.qs.as_ptr()); - let y1 = i16x8_load_extend_i8x8(y.qs.as_ptr()); - let sum_xy = i32x4_dot_i16x8(x1, y1); - - let x2 = i16x8_load_extend_i8x8(x.qs.as_ptr().add(8)); - let y2 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(8)); - let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x2, y2)); - - let x3 = i16x8_load_extend_i8x8(x.qs.as_ptr().add(16)); - let y3 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(16)); - let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x3, y3)); - - let x4 = i16x8_load_extend_i8x8(x.qs.as_ptr().add(24)); - let y4 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(24)); - let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x4, y4)); - - let sum_xy = f32x4_convert_i32x4(sum_xy); - - // f32x4_relaxed_madd is nightly only. - let d = f32x4_splat(f16::to_f32(x.d) * f16::to_f32(y.d)); - let scaled = f32x4_mul(sum_xy, d); - acc = f32x4_add(acc, scaled) - } - let res = f32x4_extract_lane::<0>(acc) - + f32x4_extract_lane::<1>(acc) - + f32x4_extract_lane::<2>(acc) - + f32x4_extract_lane::<3>(acc); - Ok(res) - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q2k_q8k(n: usize, xs: &[BlockQ2K], ys: &[BlockQ8K]) -> Result { - if n % QK_K != 0 { - crate::bail!("vec_dot_q2k_q8k: {n} is not divisible by {QK_K}") - } - unsafe { - let mut sumf = f32x4_splat(0f32); - for (x, y) in xs.iter().zip(ys.iter()) { - let mut q2: &[_] = &x.qs; - let mut q8: &[_] = &y.qs; - let sc = &x.scales; - - let mut summs = i32x4_splat(0); - for i in (0..(QK_K / 16)).step_by(4) { - let bsums = i32x4_load_extend_i16x4(y.bsums.as_ptr().add(i)); - let scales = i32x4_shr( - i32x4( - sc[i] as i32, - sc[i + 1] as i32, - sc[i + 2] as i32, - sc[i + 3] as i32, - ), - 4, - ); - summs = i32x4_add(summs, i32x4_mul(bsums, scales)) - } - let summs = f32x4_convert_i32x4(summs); - - let dall = y.d * x.d.to_f32(); - let dmin = y.d * x.dmin.to_f32(); - - let mut isum = i32x4_splat(0); - let mut is = 0; - for _ in 0..(QK_K / 128) { - let mut shift = 0; - for _ in 0..4 { - let d = (sc[is] & 0xF) as i32; - is += 1; - let mut isuml = i16x8_splat(0); - for l in (0..16).step_by(8) { - let q8 = i16x8_load_extend_i8x8(q8.as_ptr().add(l)); - let q2 = i16x8_load_extend_u8x8(q2.as_ptr().add(l)); - let q2 = v128_and(i16x8_shr(q2, shift), i16x8_splat(3)); - isuml = i16x8_add(isuml, i16x8_mul(q2, q8)) - } - let dd = i32x4_splat(d); - isum = i32x4_add(isum, i32x4_mul(i32x4_extend_low_i16x8(isuml), dd)); - isum = i32x4_add(isum, i32x4_mul(i32x4_extend_high_i16x8(isuml), dd)); - let d = (sc[is] & 0xF) as i32; - is += 1; - let mut isuml = i16x8_splat(0); - for l in (16..32).step_by(8) { - let q8 = i16x8_load_extend_i8x8(q8.as_ptr().add(l)); - let q2 = i16x8_load_extend_u8x8(q2.as_ptr().add(l)); - let q2 = v128_and(i16x8_shr(q2, shift), i16x8_splat(3)); - isuml = i16x8_add(isuml, i16x8_mul(q2, q8)) - } - let dd = i32x4_splat(d); - isum = i32x4_add(isum, i32x4_mul(i32x4_extend_low_i16x8(isuml), dd)); - isum = i32x4_add(isum, i32x4_mul(i32x4_extend_high_i16x8(isuml), dd)); - shift += 2; - // adjust the indexing - q8 = &q8[32..]; - } - // adjust the indexing - q2 = &q2[32..]; - } - let isum = f32x4_convert_i32x4(isum); - sumf = f32x4_add( - sumf, - f32x4_sub( - f32x4_mul(isum, f32x4_splat(dall)), - f32x4_mul(summs, f32x4_splat(dmin)), - ), - ); - } - let sumf = f32x4_extract_lane::<0>(sumf) - + f32x4_extract_lane::<1>(sumf) - + f32x4_extract_lane::<2>(sumf) - + f32x4_extract_lane::<3>(sumf); - Ok(sumf) - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q4k_q8k(n: usize, xs: &[BlockQ4K], ys: &[BlockQ8K]) -> Result { - if n % QK_K != 0 { - crate::bail!("vec_dot_q4k_q8k: {n} is not divisible by {QK_K}") - } - - const KMASK1: u32 = 0x3f3f3f3f; - const KMASK2: u32 = 0x0f0f0f0f; - const KMASK3: u32 = 0x03030303; - - let mut utmp: [u32; 4] = [0; 4]; - let mut scales: [u8; 8] = [0; 8]; - let mut mins: [u8; 8] = [0; 8]; - - let mut aux8: [u8; QK_K] = [0; QK_K]; - let mut sums = f32x4_splat(0f32); - unsafe { - for (y, x) in ys.iter().zip(xs.iter()) { - let q4 = &x.qs; - let q8 = &y.qs; - - for j in 0..QK_K / 64 { - let q4_1 = v128_load(q4.as_ptr().add(32 * j) as *const v128); - let q4_2 = v128_load(q4.as_ptr().add(32 * j + 16) as *const v128); - v128_store( - aux8.as_mut_ptr().add(64 * j) as *mut v128, - v128_and(q4_1, u8x16_splat(0x0F)), - ); - v128_store( - aux8.as_mut_ptr().add(64 * j + 16) as *mut v128, - v128_and(q4_2, u8x16_splat(0x0F)), - ); - v128_store( - aux8.as_mut_ptr().add(64 * j + 32) as *mut v128, - u8x16_shr(q4_1, 4), - ); - v128_store( - aux8.as_mut_ptr().add(64 * j + 48) as *mut v128, - u8x16_shr(q4_2, 4), - ); - } - - LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); - - utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); - let uaux = utmp[1] & KMASK1; - utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); - utmp[2] = uaux; - utmp[0] &= KMASK1; - - //extract scales and mins - LittleEndian::write_u32_into(&utmp[0..2], &mut scales); - LittleEndian::write_u32_into(&utmp[2..4], &mut mins); - - let mut sumi = i32x4_splat(0); - for j in (0..QK_K / 16).step_by(4) { - let bsums = i32x4_load_extend_i16x4(y.bsums.as_ptr().add(j)); - let (m1, m2) = (mins[j / 2] as i32, mins[j / 2 + 1] as i32); - let mins = i32x4(m1, m1, m2, m2); - sumi = i32x4_add(sumi, i32x4_mul(bsums, mins)); - } - - let mut aux32 = i32x4_splat(0i32); - for (scale_i, scale) in scales.iter().enumerate() { - let scale = i32x4_splat(*scale as i32); - for j in 0..4 { - let i = 32 * scale_i + 8 * j; - let q8 = i16x8_load_extend_i8x8(q8.as_ptr().add(i)); - let aux8 = i16x8_load_extend_u8x8(aux8.as_ptr().add(i)); - let aux16 = i16x8_mul(q8, aux8); - aux32 = i32x4_add(aux32, i32x4_mul(scale, i32x4_extend_low_i16x8(aux16))); - aux32 = i32x4_add(aux32, i32x4_mul(scale, i32x4_extend_high_i16x8(aux16))); - } - } - let aux32 = f32x4_convert_i32x4(aux32); - let d = f32x4_splat(x.d.to_f32() * y.d); - sums = f32x4_add(sums, f32x4_mul(aux32, d)); - let dmin = x.dmin.to_f32() * y.d; - let dmin = f32x4_splat(dmin); - let sumi = f32x4_convert_i32x4(sumi); - sums = f32x4_sub(sums, f32x4_mul(sumi, dmin)); - } - let sums = f32x4_extract_lane::<0>(sums) - + f32x4_extract_lane::<1>(sums) - + f32x4_extract_lane::<2>(sums) - + f32x4_extract_lane::<3>(sums); - Ok(sums) - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q6k_q8k(n: usize, xs: &[BlockQ6K], ys: &[BlockQ8K]) -> Result { - if n % QK_K != 0 { - crate::bail!("vec_dot_q6k_q8k: {n} is not divisible by {QK_K}") - } - - let mut aux8 = [0i8; QK_K]; - unsafe { - let mut sums = f32x4_splat(0f32); - - for (x, y) in xs.iter().zip(ys.iter()) { - let q4 = &x.ql; - let qh = &x.qh; - let q8 = &y.qs; - let mut aux32 = f32x4_splat(0f32); - - for j in (0..QK_K).step_by(128) { - let aux8 = aux8.as_mut_ptr().add(j); - let q4 = &q4.as_ptr().add(j / 2); - let qh = &qh.as_ptr().add(j / 4); - for l in (0..32).step_by(16) { - // aux8[l] = (((q4[l] & 0xF) | ((qh[l] & 3) << 4)) as i32 - 32) as i8; - let a8 = v128_or( - v128_and(v128_load(q4.add(l) as *const v128), u8x16_splat(0xF)), - u8x16_shl( - v128_and(v128_load(qh.add(l) as *const v128), u8x16_splat(3)), - 4, - ), - ); - let a8_low = i16x8_sub(i16x8_extend_low_u8x16(a8), i16x8_splat(32)); - let a8_high = i16x8_sub(i16x8_extend_high_u8x16(a8), i16x8_splat(32)); - v128_store( - aux8.add(l) as *mut v128, - i8x16_narrow_i16x8(a8_low, a8_high), - ); - - // aux8[l + 32] = - // (((q4[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) as i32 - 32) as i8; - let a8 = v128_or( - v128_and(v128_load(q4.add(l + 32) as *const v128), u8x16_splat(0xF)), - u8x16_shl( - v128_and( - u8x16_shr(v128_load(qh.add(l) as *const v128), 2), - u8x16_splat(3), - ), - 4, - ), - ); - let a8_low = i16x8_sub(i16x8_extend_low_u8x16(a8), i16x8_splat(32)); - let a8_high = i16x8_sub(i16x8_extend_high_u8x16(a8), i16x8_splat(32)); - v128_store( - aux8.add(l + 32) as *mut v128, - i8x16_narrow_i16x8(a8_low, a8_high), - ); - - // aux8[l + 64] = (((q4[l] >> 4) | (((qh[l] >> 4) & 3) << 4)) as i32 - 32) as i8; - let a8 = v128_or( - u8x16_shr(v128_load(q4.add(l) as *const v128), 4), - u8x16_shl( - v128_and( - u8x16_shr(v128_load(qh.add(l) as *const v128), 4), - u8x16_splat(3), - ), - 4, - ), - ); - let a8_low = i16x8_sub(i16x8_extend_low_u8x16(a8), i16x8_splat(32)); - let a8_high = i16x8_sub(i16x8_extend_high_u8x16(a8), i16x8_splat(32)); - v128_store( - aux8.add(l + 64) as *mut v128, - i8x16_narrow_i16x8(a8_low, a8_high), - ); - - // aux8[l + 96] = - // (((q4[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) as i32 - 32) as i8; - let a8 = v128_or( - u8x16_shr(v128_load(q4.add(l + 32) as *const v128), 4), - u8x16_shl( - v128_and( - u8x16_shr(v128_load(qh.add(l) as *const v128), 6), - u8x16_splat(3), - ), - 4, - ), - ); - let a8_low = i16x8_sub(i16x8_extend_low_u8x16(a8), i16x8_splat(32)); - let a8_high = i16x8_sub(i16x8_extend_high_u8x16(a8), i16x8_splat(32)); - v128_store( - aux8.add(l + 96) as *mut v128, - i8x16_narrow_i16x8(a8_low, a8_high), - ); - } - } - - for (j, &scale) in x.scales.iter().enumerate() { - let scale = f32x4_splat(scale as f32); - for offset in [0, 8] { - let aux16 = i16x8_mul( - i16x8_load_extend_i8x8(q8.as_ptr().add(16 * j + offset)), - i16x8_load_extend_i8x8(aux8.as_ptr().add(16 * j + offset)), - ); - aux32 = f32x4_add( - aux32, - f32x4_mul(f32x4_convert_i32x4(i32x4_extend_low_i16x8(aux16)), scale), - ); - aux32 = f32x4_add( - aux32, - f32x4_mul(f32x4_convert_i32x4(i32x4_extend_high_i16x8(aux16)), scale), - ); - } - } - - let d = f32x4_splat(x.d.to_f32() * y.d); - sums = f32x4_add(sums, f32x4_mul(aux32, d)); - } - let sums = f32x4_extract_lane::<0>(sums) - + f32x4_extract_lane::<1>(sums) - + f32x4_extract_lane::<2>(sums) - + f32x4_extract_lane::<3>(sums); - Ok(sums) - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q8k_q8k(n: usize, xs: &[BlockQ8K], ys: &[BlockQ8K]) -> Result { - let qk = QK_K; - if n % QK_K != 0 { - crate::bail!("vec_dot_q8k_q8k: {n} is not divisible by {qk}") - } - - unsafe { - let mut acc = f32x4_splat(0.0f32); - for (xs, ys) in xs.iter().zip(ys.iter()) { - let x_qs = xs.qs.as_ptr(); - let y_qs = ys.qs.as_ptr(); - let mut sumi = i32x4_splat(0); - for j in (0..QK_K).step_by(8) { - let xs = i16x8_load_extend_i8x8(x_qs.add(j)); - let ys = i16x8_load_extend_i8x8(y_qs.add(j)); - let sum_xy = i32x4_dot_i16x8(xs, ys); - sumi = i32x4_add(sumi, sum_xy) - } - let d = f32x4_splat(xs.d * ys.d); - acc = f32x4_add(acc, f32x4_mul(f32x4_convert_i32x4(sumi), d)) - } - let res = f32x4_extract_lane::<0>(acc) - + f32x4_extract_lane::<1>(acc) - + f32x4_extract_lane::<2>(acc) - + f32x4_extract_lane::<3>(acc); - Ok(res) - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/quantized/utils.rs b/v2/dependency-patches/candle-core-0.4.1/src/quantized/utils.rs deleted file mode 100644 index fa6eff51d3..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/quantized/utils.rs +++ /dev/null @@ -1,326 +0,0 @@ -use crate::Result; - -pub(super) fn nearest_int(v: f32) -> i32 { - v.round() as i32 -} - -/// Validates that the input and output are the right size and returns an iterator which maps each -/// input region `xs` to its corresponding output block in `ys`. Each output region is guaranteed -/// to be `T::BLCK_SIZE` long. -pub(super) fn group_for_quantization<'a, 'b, T: super::k_quants::GgmlType>( - xs: &'b [f32], - ys: &'a mut [T], -) -> Result> { - let block_size = T::BLCK_SIZE; - let dtype = T::DTYPE; - - let expected_blocks = xs.len() / block_size; - let actual_blocks = ys.len(); - - // Validate that the input is the right size - if expected_blocks != actual_blocks { - crate::bail!("quantize {dtype:?}: expected {expected_blocks} blocks but only {actual_blocks} were provided!") - } - - Ok(ys.iter_mut().zip(xs.chunks_exact(block_size)).collect()) -} - -/// Validates that the input and output are the right size and returns an iterator which maps each -/// input block `xs` to its corresponding output region in `ys`. Each output region is guaranteed -/// to be `T::BLCK_SIZE` long. -pub(super) fn group_for_dequantization<'a, 'b, T: super::k_quants::GgmlType>( - xs: &'a [T], - ys: &'b mut [f32], -) -> Result> { - let block_size = T::BLCK_SIZE; - let dtype = T::DTYPE; - - let actual_output_len = ys.len(); - let expected_output_len = xs.len() * block_size; - // Validate that the output is the right size - if expected_output_len != actual_output_len { - crate::bail!("dequantize {dtype:?}: ys (len = {actual_output_len}) does not match the expected length of {expected_output_len}!") - } - - // Zip the blocks and outputs together - Ok(xs.iter().zip(ys.chunks_exact_mut(block_size)).collect()) -} - -pub(super) fn get_scale_min_k4(j: usize, q: &[u8]) -> (u8, u8) { - if j < 4 { - let d = q[j] & 63; - let m = q[j + 4] & 63; - (d, m) - } else { - let d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4); - let m = (q[j + 4] >> 4) | ((q[j] >> 6) << 4); - (d, m) - } -} - -pub(super) unsafe fn make_qx_quants( - n: usize, - nmax: i32, - x: *const f32, - ls: *mut i8, - rmse_type: i32, -) -> f32 { - let mut max = 0f32; - let mut amax = 0f32; - for i in 0..n { - let x = *x.add(i); - let ax = x.abs(); - if ax > amax { - amax = ax; - max = x; - } - } - if amax == 0. { - // all zero - for i in 0..n { - *ls.add(i) = 0; - } - return 0.; - } - let mut iscale = -(nmax as f32) / max; - if rmse_type == 0 { - for i in 0..n { - let x = *x.add(i); - let l = nearest_int(iscale * x); - *ls.add(i) = (nmax + l.clamp(-nmax, nmax - 1)) as i8; - } - return 1.0 / iscale; - } - let weight_type = rmse_type % 2; - let mut sumlx = 0f32; - let mut suml2 = 0f32; - for i in 0..n { - let x = *x.add(i); - let l = nearest_int(iscale * x); - let l = l.clamp(-nmax, nmax - 1); - *ls.add(i) = (l + nmax) as i8; - let w = if weight_type == 1 { x * x } else { 1.0 }; - let l = l as f32; - sumlx += w * x * l; - suml2 += w * l * l; - } - let mut scale = sumlx / suml2; - let mut best = scale * sumlx; - for _itry in 0..3 { - let iscale = 1.0 / scale; - let mut slx = 0f32; - let mut sl2 = 0f32; - let mut changed = false; - for i in 0..n { - let x = *x.add(i); - let l = nearest_int(iscale * x); - let l = l.clamp(-nmax, nmax - 1); - if l + nmax != *ls.add(i) as i32 { - changed = true; - } - let w = if weight_type == 1 { x * x } else { 1f32 }; - let l = l as f32; - slx += w * x * l; - sl2 += w * l * l; - } - if !changed || sl2 == 0.0 || slx * slx <= best * sl2 { - break; - } - for i in 0..n { - let x = *x.add(i); - let l = nearest_int(iscale * x); - *ls.add(i) = (nmax + l.clamp(-nmax, nmax - 1)) as i8; - } - sumlx = slx; - suml2 = sl2; - scale = sumlx / suml2; - best = scale * sumlx; - } - for _itry in 0..5 { - let mut n_changed = 0; - for i in 0..n { - let x = *x.add(i); - let w = if weight_type == 1 { x * x } else { 1. }; - let l = *ls.add(i) as i32 - nmax; - let mut slx = sumlx - w * x * l as f32; - if slx > 0. { - let mut sl2 = suml2 - w * l as f32 * l as f32; - let new_l = nearest_int(x * sl2 / slx); - let new_l = new_l.clamp(-nmax, nmax - 1); - if new_l != l { - slx += w * x * new_l as f32; - sl2 += w * new_l as f32 * new_l as f32; - if sl2 > 0. && slx * slx * suml2 > sumlx * sumlx * sl2 { - *ls.add(i) = (nmax + new_l) as i8; - sumlx = slx; - suml2 = sl2; - scale = sumlx / suml2; - best = scale * sumlx; - n_changed += 1; - } - } - } - } - if n_changed == 0 { - break; - } - } - if rmse_type < 3 { - return scale; - } - for is in -4..4 { - if is == 0 { - continue; - } - iscale = -(nmax as f32 + 0.1f32 * is as f32) / max; - let mut sumlx = 0.; - let mut suml2 = 0.; - for i in 0..n { - let x = *x.add(i); - let l = nearest_int(iscale * x); - let l = l.clamp(-nmax, nmax - 1); - let w = if weight_type == 1 { x * x } else { 1. }; - let l = l as f32; - sumlx += w * x * l; - suml2 += w * l * l; - } - if suml2 > 0. && sumlx * sumlx > best * suml2 { - for i in 0..n { - let x = *x.add(i); - let l = nearest_int(iscale * x); - *ls.add(i) = (nmax + l.clamp(-nmax, nmax - 1)) as i8; - } - scale = sumlx / suml2; - best = scale * sumlx; - } - } - scale -} - -// https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L224 -pub(super) fn make_qkx1_quants(nmax: i32, ntry: usize, x: &[f32]) -> (f32, f32) { - let n = x.len(); - let mut l = vec![0; n]; - // Get min/max - let min = *x - .iter() - .take(n) - .min_by(|a, b| a.total_cmp(b)) - .unwrap_or(&x[0]); - let max = *x.iter().max_by(|a, b| a.total_cmp(b)).unwrap_or(&x[0]); - - // If min == max, all values are the same => nothing to do here - if max == min { - return (0.0, 0.0); - } - - // Ensure min <= 0.0 - let mut min = min.min(0.); - - // Compute scale and inverse scale - let mut iscale = nmax as f32 / (max - min); - let mut scale = 1.0 / iscale; - - for _ in 0..ntry { - let mut sumlx = 0.0; - let mut suml2 = 0; - let mut did_change = false; - - for (i, value) in x.iter().enumerate().take(n) { - let li = nearest_int(iscale * (value - min)).clamp(0, nmax); - let clamped_li = li as u8; - if clamped_li != l[i] { - l[i] = clamped_li; - did_change = true; - } - sumlx += (value - min) * li as f32; - suml2 += li * li; - } - scale = sumlx / suml2 as f32; - - let sum: f32 = x - .iter() - .take(n) - .zip(l.iter().take(n)) - .map(|(xi, &li)| xi - scale * li as f32) - .sum(); - - min = sum / n as f32; - if min > 0.0 { - min = 0.0; - } - iscale = 1.0 / scale; - if !did_change { - break; - } - } - (scale, -min) -} - -// https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L165 -pub(super) fn make_q3_quants(x: &[f32], nmax: i32, do_rmse: bool) -> f32 { - let n = x.len(); - let mut l = vec![0i8; n]; - - let mut max = 0.0; - let mut amax = 0.0; - for &xi in x.iter().take(n) { - let ax = xi.abs(); - if ax > amax { - amax = ax; - max = xi; - } - } - - if amax == 0.0 { - return 0.0; - } - - let iscale = -(nmax as f32) / max; - if do_rmse { - let mut sumlx = 0.0; - let mut suml2 = 0.0; - for i in 0..n { - let li = (iscale * x[i]).round() as i32; - let li = li.clamp(-nmax, nmax - 1); - l[i] = li as i8; - let w = x[i] * x[i]; - sumlx += w * x[i] * li as f32; - suml2 += w * (li * li) as f32; - } - for _ in 0..5 { - let mut n_changed = 0; - for i in 0..n { - let w = x[i] * x[i]; - let mut slx = sumlx - w * x[i] * l[i] as f32; - if slx > 0.0 { - let mut sl2 = suml2 - w * (l[i] as i32 * l[i] as i32) as f32; - let mut new_l = (x[i] * sl2 / slx).round() as i32; - new_l = new_l.clamp(-nmax, nmax - 1); - if new_l != l[i] as i32 { - slx += w * x[i] * new_l as f32; - sl2 += w * (new_l * new_l) as f32; - if sl2 > 0.0 && slx * slx * suml2 > sumlx * sumlx * sl2 { - l[i] = new_l as i8; - sumlx = slx; - suml2 = sl2; - n_changed += 1; - } - } - } - } - if n_changed == 0 { - break; - } - } - for li in l.iter_mut() { - *li += nmax as i8; - } - return sumlx / suml2; - } - for i in 0..n { - let li = (iscale * x[i]).round() as i32; - l[i] = (li.clamp(-nmax, nmax - 1) + nmax) as i8; - } - 1.0 / iscale -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/safetensors.rs b/v2/dependency-patches/candle-core-0.4.1/src/safetensors.rs deleted file mode 100644 index 02d54e6895..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/safetensors.rs +++ /dev/null @@ -1,441 +0,0 @@ -use crate::{DType, Device, Error, Result, Tensor, WithDType}; -use safetensors::tensor as st; -use safetensors::tensor::SafeTensors; -use std::borrow::Cow; -use std::collections::HashMap; -use std::path::Path; - -impl From for st::Dtype { - fn from(value: DType) -> Self { - match value { - DType::U8 => st::Dtype::U8, - DType::U32 => st::Dtype::U32, - DType::I64 => st::Dtype::I64, - DType::BF16 => st::Dtype::BF16, - DType::F16 => st::Dtype::F16, - DType::F32 => st::Dtype::F32, - DType::F64 => st::Dtype::F64, - } - } -} - -impl TryFrom for DType { - type Error = Error; - fn try_from(value: st::Dtype) -> Result { - match value { - st::Dtype::U8 => Ok(DType::U8), - st::Dtype::U32 => Ok(DType::U32), - st::Dtype::I64 => Ok(DType::I64), - st::Dtype::BF16 => Ok(DType::BF16), - st::Dtype::F16 => Ok(DType::F16), - st::Dtype::F32 => Ok(DType::F32), - st::Dtype::F64 => Ok(DType::F64), - dtype => Err(Error::UnsupportedSafeTensorDtype(dtype)), - } - } -} - -impl st::View for Tensor { - fn dtype(&self) -> st::Dtype { - self.dtype().into() - } - fn shape(&self) -> &[usize] { - self.shape().dims() - } - - fn data(&self) -> Cow<[u8]> { - // This copies data from GPU to CPU. - // TODO: Avoid the unwrap here. - Cow::Owned(convert_back(self).unwrap()) - } - - fn data_len(&self) -> usize { - let n: usize = self.shape().elem_count(); - let bytes_per_element = self.dtype().size_in_bytes(); - n * bytes_per_element - } -} - -impl st::View for &Tensor { - fn dtype(&self) -> st::Dtype { - (*self).dtype().into() - } - fn shape(&self) -> &[usize] { - self.dims() - } - - fn data(&self) -> Cow<[u8]> { - // This copies data from GPU to CPU. - // TODO: Avoid the unwrap here. - Cow::Owned(convert_back(self).unwrap()) - } - - fn data_len(&self) -> usize { - let n: usize = self.dims().iter().product(); - let bytes_per_element = (*self).dtype().size_in_bytes(); - n * bytes_per_element - } -} - -impl Tensor { - pub fn save_safetensors>(&self, name: &str, filename: P) -> Result<()> { - let data = [(name, self.clone())]; - Ok(st::serialize_to_file(data, &None, filename.as_ref())?) - } -} - -fn convert_slice(data: &[u8], shape: &[usize], device: &Device) -> Result { - let size_in_bytes = T::DTYPE.size_in_bytes(); - let elem_count = data.len() / size_in_bytes; - if (data.as_ptr() as usize) % size_in_bytes == 0 { - // SAFETY This is safe because we just checked that this - // was correctly aligned. - let data: &[T] = - unsafe { std::slice::from_raw_parts(data.as_ptr() as *const T, elem_count) }; - Tensor::from_slice(data, shape, device) - } else { - // XXX: We need to specify `T` here, otherwise the compiler will infer u8 because of the following cast - // Making this vector too small to fit a full f16/f32/f64 weights, resulting in out-of-bounds access - let mut c: Vec = Vec::with_capacity(elem_count); - // SAFETY: We just created c, so the allocated memory is necessarily - // contiguous and non overlapping with the view's data. - // We're downgrading the `c` pointer from T to u8, which removes alignment - // constraints. - unsafe { - std::ptr::copy_nonoverlapping(data.as_ptr(), c.as_mut_ptr() as *mut u8, data.len()); - c.set_len(elem_count) - } - Tensor::from_slice(&c, shape, device) - } -} - -fn convert_slice_with_cast Result>( - data: &[u8], - shape: &[usize], - device: &Device, - conv: F, -) -> Result { - let size_in_bytes = std::mem::size_of::(); - let elem_count = data.len() / size_in_bytes; - if (data.as_ptr() as usize) % size_in_bytes == 0 { - // SAFETY This is safe because we just checked that this - // was correctly aligned. - let data: &[T] = - unsafe { std::slice::from_raw_parts(data.as_ptr() as *const T, elem_count) }; - let data = data.iter().map(|t| conv(*t)).collect::>>()?; - Tensor::from_vec(data, shape, device) - } else { - // XXX: We need to specify `T` here, otherwise the compiler will infer u8 because of the following cast - // Making this vector too small to fit a full f16/f32/f64 weights, resulting in out-of-bounds access - let mut c: Vec = Vec::with_capacity(elem_count); - // SAFETY: We just created c, so the allocated memory is necessarily - // contiguous and non overlapping with the view's data. - // We're downgrading the `c` pointer from T to u8, which removes alignment - // constraints. - unsafe { - std::ptr::copy_nonoverlapping(data.as_ptr(), c.as_mut_ptr() as *mut u8, data.len()); - c.set_len(elem_count) - } - let c = c.into_iter().map(conv).collect::>>()?; - Tensor::from_vec(c, shape, device) - } -} - -fn convert_with_cast_ Result>( - view: &st::TensorView<'_>, - device: &Device, - conv: F, -) -> Result { - convert_slice_with_cast::(view.data(), view.shape(), device, conv) -} - -fn convert_(view: &st::TensorView<'_>, device: &Device) -> Result { - convert_slice::(view.data(), view.shape(), device) -} - -fn convert_back_(mut vs: Vec) -> Vec { - let size_in_bytes = T::DTYPE.size_in_bytes(); - let length = vs.len() * size_in_bytes; - let capacity = vs.capacity() * size_in_bytes; - let ptr = vs.as_mut_ptr() as *mut u8; - // Don't run the destructor for Vec - std::mem::forget(vs); - // SAFETY: - // - // Every T is larger than u8, so there is no issue regarding alignment. - // This re-interpret the Vec as a Vec. - unsafe { Vec::from_raw_parts(ptr, length, capacity) } -} - -pub trait Load { - fn load(&self, device: &Device) -> Result; -} - -impl<'a> Load for st::TensorView<'a> { - fn load(&self, device: &Device) -> Result { - convert(self, device) - } -} - -impl Tensor { - pub fn from_raw_buffer( - data: &[u8], - dtype: DType, - shape: &[usize], - device: &Device, - ) -> Result { - match dtype { - DType::U8 => convert_slice::(data, shape, device), - DType::U32 => convert_slice::(data, shape, device), - DType::I64 => convert_slice::(data, shape, device), - DType::BF16 => convert_slice::(data, shape, device), - DType::F16 => convert_slice::(data, shape, device), - DType::F32 => convert_slice::(data, shape, device), - DType::F64 => convert_slice::(data, shape, device), - } - } -} - -fn convert(view: &st::TensorView<'_>, device: &Device) -> Result { - match view.dtype() { - st::Dtype::U8 => convert_::(view, device), - st::Dtype::U16 => { - let conv = |x| Ok(u32::from(x)); - convert_with_cast_::(view, device, conv) - } - st::Dtype::U32 => convert_::(view, device), - st::Dtype::I32 => { - let conv = |x| Ok(i64::from(x)); - convert_with_cast_::(view, device, conv) - } - st::Dtype::I64 => convert_::(view, device), - st::Dtype::BF16 => convert_::(view, device), - st::Dtype::F16 => convert_::(view, device), - st::Dtype::F32 => convert_::(view, device), - st::Dtype::F64 => convert_::(view, device), - dtype => Err(Error::UnsupportedSafeTensorDtype(dtype)), - } -} - -fn convert_back(tensor: &Tensor) -> Result> { - // TODO: This makes an unnecessary copy when the tensor is on the cpu. - let tensor = tensor.flatten_all()?; - match tensor.dtype() { - DType::U8 => Ok(convert_back_::(tensor.to_vec1()?)), - DType::U32 => Ok(convert_back_::(tensor.to_vec1()?)), - DType::I64 => Ok(convert_back_::(tensor.to_vec1()?)), - DType::F16 => Ok(convert_back_::(tensor.to_vec1()?)), - DType::BF16 => Ok(convert_back_::(tensor.to_vec1()?)), - DType::F32 => Ok(convert_back_::(tensor.to_vec1()?)), - DType::F64 => Ok(convert_back_::(tensor.to_vec1()?)), - } -} - -pub fn load>(filename: P, device: &Device) -> Result> { - let data = std::fs::read(filename.as_ref())?; - load_buffer(&data[..], device) -} - -pub fn load_buffer(data: &[u8], device: &Device) -> Result> { - let st = safetensors::SafeTensors::deserialize(data)?; - st.tensors() - .into_iter() - .map(|(name, view)| Ok((name, view.load(device)?))) - .collect() -} - -pub fn save + Ord + std::fmt::Display, P: AsRef>( - tensors: &HashMap, - filename: P, -) -> Result<()> { - Ok(st::serialize_to_file(tensors, &None, filename.as_ref())?) -} - -#[derive(yoke::Yokeable)] -struct SafeTensors_<'a>(SafeTensors<'a>); - -pub struct MmapedSafetensors { - safetensors: Vec, memmap2::Mmap>>, - routing: Option>, -} - -impl MmapedSafetensors { - /// Creates a wrapper around a memory mapped file and deserialize the safetensors header. - /// - /// # Safety - /// - /// The unsafe is inherited from [`memmap2::MmapOptions`]. - pub unsafe fn new>(p: P) -> Result { - let p = p.as_ref(); - let file = std::fs::File::open(p).map_err(|e| Error::from(e).with_path(p))?; - let file = memmap2::MmapOptions::new() - .map(&file) - .map_err(|e| Error::from(e).with_path(p))?; - let safetensors = yoke::Yoke::, memmap2::Mmap>::try_attach_to_cart( - file, - |data: &[u8]| { - let st = safetensors::SafeTensors::deserialize(data) - .map_err(|e| Error::from(e).with_path(p))?; - Ok::<_, Error>(SafeTensors_(st)) - }, - )?; - Ok(Self { - safetensors: vec![safetensors], - routing: None, - }) - } - - /// Creates a wrapper around multiple memory mapped file and deserialize the safetensors headers. - /// - /// If a tensor name appears in multiple files, the last entry is returned. - /// - /// # Safety - /// - /// The unsafe is inherited from [`memmap2::MmapOptions`]. - pub unsafe fn multi>(paths: &[P]) -> Result { - let mut routing = HashMap::new(); - let mut safetensors = vec![]; - for (index, p) in paths.iter().enumerate() { - let p = p.as_ref(); - let file = std::fs::File::open(p).map_err(|e| Error::from(e).with_path(p))?; - let file = memmap2::MmapOptions::new() - .map(&file) - .map_err(|e| Error::from(e).with_path(p))?; - let data = yoke::Yoke::, memmap2::Mmap>::try_attach_to_cart( - file, - |data: &[u8]| { - let st = safetensors::SafeTensors::deserialize(data) - .map_err(|e| Error::from(e).with_path(p))?; - Ok::<_, Error>(SafeTensors_(st)) - }, - )?; - for k in data.get().0.names() { - routing.insert(k.to_string(), index); - } - safetensors.push(data) - } - Ok(Self { - safetensors, - routing: Some(routing), - }) - } - - pub fn load(&self, name: &str, dev: &Device) -> Result { - self.get(name)?.load(dev) - } - - pub fn tensors(&self) -> Vec<(String, st::TensorView<'_>)> { - let mut tensors = vec![]; - for safetensors in self.safetensors.iter() { - tensors.push(safetensors.get().0.tensors()) - } - tensors.into_iter().flatten().collect() - } - - pub fn get(&self, name: &str) -> Result> { - let index = match &self.routing { - None => 0, - Some(routing) => { - let index = routing.get(name).ok_or_else(|| { - Error::CannotFindTensor { - path: name.to_string(), - } - .bt() - })?; - *index - } - }; - Ok(self.safetensors[index].get().0.tensor(name)?) - } -} - -pub struct BufferedSafetensors { - safetensors: yoke::Yoke, Vec>, -} - -impl BufferedSafetensors { - /// Creates a wrapper around a binary buffer and deserialize the safetensors header. - pub fn new(buffer: Vec) -> Result { - let safetensors = yoke::Yoke::, Vec>::try_attach_to_cart( - buffer, - |data: &[u8]| { - let st = safetensors::SafeTensors::deserialize(data)?; - Ok::<_, Error>(SafeTensors_(st)) - }, - )?; - Ok(Self { safetensors }) - } - - pub fn load(&self, name: &str, dev: &Device) -> Result { - self.get(name)?.load(dev) - } - - pub fn tensors(&self) -> Vec<(String, st::TensorView<'_>)> { - self.safetensors.get().0.tensors() - } - - pub fn get(&self, name: &str) -> Result> { - Ok(self.safetensors.get().0.tensor(name)?) - } -} - -pub struct MmapedFile { - path: std::path::PathBuf, - inner: memmap2::Mmap, -} - -impl MmapedFile { - /// Creates a wrapper around a memory mapped file from which you can retrieve - /// tensors using [`MmapedFile::deserialize`] - /// - /// # Safety - /// - /// The unsafe is inherited from [`memmap2::MmapOptions`]. - pub unsafe fn new>(p: P) -> Result { - let p = p.as_ref(); - let file = std::fs::File::open(p).map_err(|e| Error::from(e).with_path(p))?; - let inner = memmap2::MmapOptions::new() - .map(&file) - .map_err(|e| Error::from(e).with_path(p))?; - Ok(Self { - inner, - path: p.to_path_buf(), - }) - } - - pub fn deserialize(&self) -> Result> { - let st = safetensors::SafeTensors::deserialize(&self.inner) - .map_err(|e| Error::from(e).with_path(&self.path))?; - Ok(st) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::HashMap; - - #[test] - fn save_single_tensor() { - let t = Tensor::zeros((2, 2), DType::F32, &Device::Cpu).unwrap(); - t.save_safetensors("t", "t.safetensors").unwrap(); - let bytes = std::fs::read("t.safetensors").unwrap(); - assert_eq!(bytes, b"@\0\0\0\0\0\0\0{\"t\":{\"dtype\":\"F32\",\"shape\":[2,2],\"data_offsets\":[0,16]}} \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"); - std::fs::remove_file("t.safetensors").unwrap(); - } - - #[test] - fn save_load_multiple_tensors() { - let t = Tensor::zeros((2, 2), DType::F32, &Device::Cpu).unwrap(); - let u = Tensor::zeros((1, 2), DType::F32, &Device::Cpu).unwrap(); - let map: HashMap<_, _> = [("t", t), ("u", u)].into_iter().collect(); - save(&map, "multi.safetensors").unwrap(); - - let weights = load("multi.safetensors", &Device::Cpu).unwrap(); - assert_eq!(weights.get("t").unwrap().dims(), &[2, 2]); - assert_eq!(weights.get("u").unwrap().dims(), &[1, 2]); - let bytes = std::fs::read("multi.safetensors").unwrap(); - assert_eq!(bytes, b"x\0\0\0\0\0\0\0{\"t\":{\"dtype\":\"F32\",\"shape\":[2,2],\"data_offsets\":[0,16]},\"u\":{\"dtype\":\"F32\",\"shape\":[1,2],\"data_offsets\":[16,24]}} \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"); - std::fs::remove_file("multi.safetensors").unwrap(); - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/scalar.rs b/v2/dependency-patches/candle-core-0.4.1/src/scalar.rs deleted file mode 100644 index 43e1f4c8c5..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/scalar.rs +++ /dev/null @@ -1,23 +0,0 @@ -use crate::{Result, Tensor, WithDType}; - -pub enum TensorScalar { - Tensor(Tensor), - Scalar(Tensor), -} - -pub trait TensorOrScalar { - fn to_tensor_scalar(self) -> Result; -} - -impl TensorOrScalar for &Tensor { - fn to_tensor_scalar(self) -> Result { - Ok(TensorScalar::Tensor(self.clone())) - } -} - -impl TensorOrScalar for T { - fn to_tensor_scalar(self) -> Result { - let scalar = Tensor::new(self, &crate::Device::Cpu)?; - Ok(TensorScalar::Scalar(scalar)) - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/shape.rs b/v2/dependency-patches/candle-core-0.4.1/src/shape.rs deleted file mode 100644 index 32ebb23f24..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/shape.rs +++ /dev/null @@ -1,629 +0,0 @@ -//! The shape of a tensor is a tuple with the size of each of its dimensions. -#![allow(clippy::redundant_closure_call)] -use crate::{Error, Result}; - -#[derive(Clone, PartialEq, Eq)] -pub struct Shape(Vec); - -pub const SCALAR: Shape = Shape(vec![]); - -impl std::fmt::Debug for Shape { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", &self.dims()) - } -} - -impl From<&[usize; C]> for Shape { - fn from(dims: &[usize; C]) -> Self { - Self(dims.to_vec()) - } -} - -impl From<&[usize]> for Shape { - fn from(dims: &[usize]) -> Self { - Self(dims.to_vec()) - } -} - -impl From<&Shape> for Shape { - fn from(shape: &Shape) -> Self { - Self(shape.0.to_vec()) - } -} - -impl From<()> for Shape { - fn from(_: ()) -> Self { - Self(vec![]) - } -} - -impl From for Shape { - fn from(d1: usize) -> Self { - Self(vec![d1]) - } -} - -impl From<(usize,)> for Shape { - fn from(d1: (usize,)) -> Self { - Self(vec![d1.0]) - } -} - -impl From<(usize, usize)> for Shape { - fn from(d12: (usize, usize)) -> Self { - Self(vec![d12.0, d12.1]) - } -} - -impl From<(usize, usize, usize)> for Shape { - fn from(d123: (usize, usize, usize)) -> Self { - Self(vec![d123.0, d123.1, d123.2]) - } -} - -impl From<(usize, usize, usize, usize)> for Shape { - fn from(d1234: (usize, usize, usize, usize)) -> Self { - Self(vec![d1234.0, d1234.1, d1234.2, d1234.3]) - } -} - -impl From<(usize, usize, usize, usize, usize)> for Shape { - fn from(d12345: (usize, usize, usize, usize, usize)) -> Self { - Self(vec![d12345.0, d12345.1, d12345.2, d12345.3, d12345.4]) - } -} - -impl From<(usize, usize, usize, usize, usize, usize)> for Shape { - fn from(d123456: (usize, usize, usize, usize, usize, usize)) -> Self { - Self(vec![ - d123456.0, d123456.1, d123456.2, d123456.3, d123456.4, d123456.5, - ]) - } -} - -impl From> for Shape { - fn from(dims: Vec) -> Self { - Self(dims) - } -} - -macro_rules! extract_dims { - ($fn_name:ident, $cnt:tt, $dims:expr, $out_type:ty) => { - pub fn $fn_name(dims: &[usize]) -> Result<$out_type> { - if dims.len() != $cnt { - Err(Error::UnexpectedNumberOfDims { - expected: $cnt, - got: dims.len(), - shape: Shape::from(dims), - } - .bt()) - } else { - Ok($dims(dims)) - } - } - - impl Shape { - pub fn $fn_name(&self) -> Result<$out_type> { - $fn_name(self.0.as_slice()) - } - } - - impl crate::Tensor { - pub fn $fn_name(&self) -> Result<$out_type> { - self.shape().$fn_name() - } - } - - impl std::convert::TryInto<$out_type> for Shape { - type Error = crate::Error; - fn try_into(self) -> std::result::Result<$out_type, Self::Error> { - self.$fn_name() - } - } - }; -} - -impl Shape { - pub fn from_dims(dims: &[usize]) -> Self { - Self(dims.to_vec()) - } - - /// The rank is the number of dimensions, 0 for a scalar value, 1 for a vector, etc. - pub fn rank(&self) -> usize { - self.0.len() - } - - pub fn into_dims(self) -> Vec { - self.0 - } - - /// The dimensions as a slice of `usize`. - pub fn dims(&self) -> &[usize] { - &self.0 - } - - /// The total number of elements, this is the product of all dimension sizes. - pub fn elem_count(&self) -> usize { - self.0.iter().product() - } - - /// The strides given in number of elements for a contiguous n-dimensional - /// arrays using this shape. - pub(crate) fn stride_contiguous(&self) -> Vec { - let mut stride: Vec<_> = self - .0 - .iter() - .rev() - .scan(1, |prod, u| { - let prod_pre_mult = *prod; - *prod *= u; - Some(prod_pre_mult) - }) - .collect(); - stride.reverse(); - stride - } - - /// Returns true if the strides are C contiguous (aka row major). - pub fn is_contiguous(&self, stride: &[usize]) -> bool { - if self.0.len() != stride.len() { - return false; - } - let mut acc = 1; - for (&stride, &dim) in stride.iter().zip(self.0.iter()).rev() { - if stride != acc { - return false; - } - acc *= dim; - } - true - } - - /// Returns true if the strides are Fortran contiguous (aka column major). - pub fn is_fortran_contiguous(&self, stride: &[usize]) -> bool { - if self.0.len() != stride.len() { - return false; - } - let mut acc = 1; - for (&stride, &dim) in stride.iter().zip(self.0.iter()) { - if stride != acc { - return false; - } - acc *= dim; - } - true - } - - /// Modifies the shape by adding a list of additional dimensions at the end of the existing - /// dimensions. - pub fn extend(mut self, additional_dims: &[usize]) -> Self { - self.0.extend(additional_dims); - self - } - - /// Check whether the two shapes are compatible for broadcast, and if it is the case return the - /// broadcasted shape. This is to be used for binary pointwise ops. - pub fn broadcast_shape_binary_op(&self, rhs: &Self, op: &'static str) -> Result { - let lhs = self; - let lhs_dims = lhs.dims(); - let rhs_dims = rhs.dims(); - let lhs_ndims = lhs_dims.len(); - let rhs_ndims = rhs_dims.len(); - let bcast_ndims = usize::max(lhs_ndims, rhs_ndims); - let mut bcast_dims = vec![0; bcast_ndims]; - for (idx, bcast_value) in bcast_dims.iter_mut().enumerate() { - let rev_idx = bcast_ndims - idx; - let l_value = if lhs_ndims < rev_idx { - 1 - } else { - lhs_dims[lhs_ndims - rev_idx] - }; - let r_value = if rhs_ndims < rev_idx { - 1 - } else { - rhs_dims[rhs_ndims - rev_idx] - }; - *bcast_value = if l_value == r_value { - l_value - } else if l_value == 1 { - r_value - } else if r_value == 1 { - l_value - } else { - Err(Error::ShapeMismatchBinaryOp { - lhs: lhs.clone(), - rhs: rhs.clone(), - op, - } - .bt())? - } - } - Ok(Shape::from(bcast_dims)) - } - - pub(crate) fn broadcast_shape_matmul(&self, rhs: &Self) -> Result<(Shape, Shape)> { - let lhs = self; - let lhs_dims = lhs.dims(); - let rhs_dims = rhs.dims(); - if lhs_dims.len() < 2 || rhs_dims.len() < 2 { - crate::bail!("only 2d matrixes are supported {lhs:?} {rhs:?}") - } - let (m, lhs_k) = (lhs_dims[lhs_dims.len() - 2], lhs_dims[lhs_dims.len() - 1]); - let (rhs_k, n) = (rhs_dims[rhs_dims.len() - 2], rhs_dims[rhs_dims.len() - 1]); - if lhs_k != rhs_k { - crate::bail!("different inner dimensions in broadcast matmul {lhs:?} {rhs:?}") - } - - let lhs_b = Self::from(&lhs_dims[..lhs_dims.len() - 2]); - let rhs_b = Self::from(&rhs_dims[..rhs_dims.len() - 2]); - let bcast = lhs_b.broadcast_shape_binary_op(&rhs_b, "broadcast_matmul")?; - let bcast_dims = bcast.dims(); - - let bcast_lhs = [bcast_dims, &[m, lhs_k]].concat(); - let bcast_rhs = [bcast_dims, &[rhs_k, n]].concat(); - Ok((Shape::from(bcast_lhs), Shape::from(bcast_rhs))) - } -} - -pub trait Dim { - fn to_index(&self, shape: &Shape, op: &'static str) -> Result; - fn to_index_plus_one(&self, shape: &Shape, op: &'static str) -> Result; -} - -impl Dim for usize { - fn to_index(&self, shape: &Shape, op: &'static str) -> Result { - let dim = *self; - if dim >= shape.dims().len() { - Err(Error::DimOutOfRange { - shape: shape.clone(), - dim: dim as i32, - op, - } - .bt())? - } else { - Ok(dim) - } - } - - fn to_index_plus_one(&self, shape: &Shape, op: &'static str) -> Result { - let dim = *self; - if dim > shape.dims().len() { - Err(Error::DimOutOfRange { - shape: shape.clone(), - dim: dim as i32, - op, - } - .bt())? - } else { - Ok(dim) - } - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub enum D { - Minus1, - Minus2, -} - -impl D { - fn out_of_range(&self, shape: &Shape, op: &'static str) -> Error { - let dim = match self { - Self::Minus1 => -1, - Self::Minus2 => -2, - }; - Error::DimOutOfRange { - shape: shape.clone(), - dim, - op, - } - .bt() - } -} - -impl Dim for D { - fn to_index(&self, shape: &Shape, op: &'static str) -> Result { - let rank = shape.rank(); - match self { - Self::Minus1 if rank >= 1 => Ok(rank - 1), - Self::Minus2 if rank >= 2 => Ok(rank - 2), - _ => Err(self.out_of_range(shape, op)), - } - } - - fn to_index_plus_one(&self, shape: &Shape, op: &'static str) -> Result { - let rank = shape.rank(); - match self { - Self::Minus1 => Ok(rank), - Self::Minus2 if rank >= 1 => Ok(rank - 1), - _ => Err(self.out_of_range(shape, op)), - } - } -} - -pub trait Dims: Sized { - fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result>; - - fn to_indexes(self, shape: &Shape, op: &'static str) -> Result> { - let dims = self.to_indexes_internal(shape, op)?; - for (i, &dim) in dims.iter().enumerate() { - if dims[..i].contains(&dim) { - Err(Error::DuplicateDimIndex { - shape: shape.clone(), - dims: dims.clone(), - op, - } - .bt())? - } - if dim >= shape.rank() { - Err(Error::DimOutOfRange { - shape: shape.clone(), - dim: dim as i32, - op, - } - .bt())? - } - } - Ok(dims) - } -} - -impl Dims for Vec { - fn to_indexes_internal(self, _: &Shape, _: &'static str) -> Result> { - Ok(self) - } -} - -impl Dims for [usize; N] { - fn to_indexes_internal(self, _: &Shape, _: &'static str) -> Result> { - Ok(self.to_vec()) - } -} - -impl Dims for &[usize] { - fn to_indexes_internal(self, _: &Shape, _: &'static str) -> Result> { - Ok(self.to_vec()) - } -} - -impl Dims for () { - fn to_indexes_internal(self, _: &Shape, _: &'static str) -> Result> { - Ok(vec![]) - } -} - -impl Dims for D { - fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { - let dim = self.to_index(shape, op)?; - Ok(vec![dim]) - } -} - -impl Dims for (D,) { - fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { - let dim = self.0.to_index(shape, op)?; - Ok(vec![dim]) - } -} - -impl Dims for (D1, D2) { - fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { - let d0 = self.0.to_index(shape, op)?; - let d1 = self.1.to_index(shape, op)?; - Ok(vec![d0, d1]) - } -} - -impl Dims for (D1, D2, D3) { - fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { - let d0 = self.0.to_index(shape, op)?; - let d1 = self.1.to_index(shape, op)?; - let d2 = self.2.to_index(shape, op)?; - Ok(vec![d0, d1, d2]) - } -} - -impl Dims for (D1, D2, D3, D4) { - fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { - let d0 = self.0.to_index(shape, op)?; - let d1 = self.1.to_index(shape, op)?; - let d2 = self.2.to_index(shape, op)?; - let d3 = self.3.to_index(shape, op)?; - Ok(vec![d0, d1, d2, d3]) - } -} - -impl Dims for (D1, D2, D3, D4, D5) { - fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { - let d0 = self.0.to_index(shape, op)?; - let d1 = self.1.to_index(shape, op)?; - let d2 = self.2.to_index(shape, op)?; - let d3 = self.3.to_index(shape, op)?; - let d4 = self.4.to_index(shape, op)?; - Ok(vec![d0, d1, d2, d3, d4]) - } -} - -impl Dims for (D1, D2, D3, D4, D5, D6) { - fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { - let d0 = self.0.to_index(shape, op)?; - let d1 = self.1.to_index(shape, op)?; - let d2 = self.2.to_index(shape, op)?; - let d3 = self.3.to_index(shape, op)?; - let d4 = self.4.to_index(shape, op)?; - let d5 = self.5.to_index(shape, op)?; - Ok(vec![d0, d1, d2, d3, d4, d5]) - } -} - -extract_dims!(dims0, 0, |_: &[usize]| (), ()); -extract_dims!(dims1, 1, |d: &[usize]| d[0], usize); -extract_dims!(dims2, 2, |d: &[usize]| (d[0], d[1]), (usize, usize)); -extract_dims!( - dims3, - 3, - |d: &[usize]| (d[0], d[1], d[2]), - (usize, usize, usize) -); -extract_dims!( - dims4, - 4, - |d: &[usize]| (d[0], d[1], d[2], d[3]), - (usize, usize, usize, usize) -); -extract_dims!( - dims5, - 5, - |d: &[usize]| (d[0], d[1], d[2], d[3], d[4]), - (usize, usize, usize, usize, usize) -); - -pub trait ShapeWithOneHole { - fn into_shape(self, el_count: usize) -> Result; -} - -impl> ShapeWithOneHole for S { - fn into_shape(self, _el_count: usize) -> Result { - Ok(self.into()) - } -} - -impl ShapeWithOneHole for ((),) { - fn into_shape(self, el_count: usize) -> Result { - Ok(el_count.into()) - } -} - -fn hole_size(el_count: usize, prod_d: usize, s: &dyn std::fmt::Debug) -> Result { - if prod_d == 0 { - crate::bail!("cannot reshape tensor of {el_count} elements to {s:?}") - } - if el_count % prod_d != 0 { - crate::bail!("cannot reshape tensor with {el_count} elements to {s:?}") - } - Ok(el_count / prod_d) -} - -impl ShapeWithOneHole for ((), usize) { - fn into_shape(self, el_count: usize) -> Result { - let ((), d1) = self; - Ok((hole_size(el_count, d1, &self)?, d1).into()) - } -} - -impl ShapeWithOneHole for (usize, ()) { - fn into_shape(self, el_count: usize) -> Result { - let (d1, ()) = self; - Ok((d1, hole_size(el_count, d1, &self)?).into()) - } -} - -impl ShapeWithOneHole for ((), usize, usize) { - fn into_shape(self, el_count: usize) -> Result { - let ((), d1, d2) = self; - Ok((hole_size(el_count, d1 * d2, &self)?, d1, d2).into()) - } -} - -impl ShapeWithOneHole for (usize, (), usize) { - fn into_shape(self, el_count: usize) -> Result { - let (d1, (), d2) = self; - Ok((d1, hole_size(el_count, d1 * d2, &self)?, d2).into()) - } -} - -impl ShapeWithOneHole for (usize, usize, ()) { - fn into_shape(self, el_count: usize) -> Result { - let (d1, d2, ()) = self; - Ok((d1, d2, hole_size(el_count, d1 * d2, &self)?).into()) - } -} - -impl ShapeWithOneHole for ((), usize, usize, usize) { - fn into_shape(self, el_count: usize) -> Result { - let ((), d1, d2, d3) = self; - let d = hole_size(el_count, d1 * d2 * d3, &self)?; - Ok((d, d1, d2, d3).into()) - } -} - -impl ShapeWithOneHole for (usize, (), usize, usize) { - fn into_shape(self, el_count: usize) -> Result { - let (d1, (), d2, d3) = self; - let d = hole_size(el_count, d1 * d2 * d3, &self)?; - Ok((d1, d, d2, d3).into()) - } -} - -impl ShapeWithOneHole for (usize, usize, (), usize) { - fn into_shape(self, el_count: usize) -> Result { - let (d1, d2, (), d3) = self; - let d = hole_size(el_count, d1 * d2 * d3, &self)?; - Ok((d1, d2, d, d3).into()) - } -} - -impl ShapeWithOneHole for (usize, usize, usize, ()) { - fn into_shape(self, el_count: usize) -> Result { - let (d1, d2, d3, ()) = self; - let d = hole_size(el_count, d1 * d2 * d3, &self)?; - Ok((d1, d2, d3, d).into()) - } -} - -impl ShapeWithOneHole for ((), usize, usize, usize, usize) { - fn into_shape(self, el_count: usize) -> Result { - let ((), d1, d2, d3, d4) = self; - let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?; - Ok((d, d1, d2, d3, d4).into()) - } -} - -impl ShapeWithOneHole for (usize, (), usize, usize, usize) { - fn into_shape(self, el_count: usize) -> Result { - let (d1, (), d2, d3, d4) = self; - let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?; - Ok((d1, d, d2, d3, d4).into()) - } -} - -impl ShapeWithOneHole for (usize, usize, (), usize, usize) { - fn into_shape(self, el_count: usize) -> Result { - let (d1, d2, (), d3, d4) = self; - let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?; - Ok((d1, d2, d, d3, d4).into()) - } -} - -impl ShapeWithOneHole for (usize, usize, usize, (), usize) { - fn into_shape(self, el_count: usize) -> Result { - let (d1, d2, d3, (), d4) = self; - let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?; - Ok((d1, d2, d3, d, d4).into()) - } -} - -impl ShapeWithOneHole for (usize, usize, usize, usize, ()) { - fn into_shape(self, el_count: usize) -> Result { - let (d1, d2, d3, d4, ()) = self; - let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?; - Ok((d1, d2, d3, d4, d).into()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn stride() { - let shape = Shape::from(()); - assert_eq!(shape.stride_contiguous(), Vec::::new()); - let shape = Shape::from(42); - assert_eq!(shape.stride_contiguous(), [1]); - let shape = Shape::from((42, 1337)); - assert_eq!(shape.stride_contiguous(), [1337, 1]); - let shape = Shape::from((299, 792, 458)); - assert_eq!(shape.stride_contiguous(), [458 * 792, 458, 1]); - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/storage.rs b/v2/dependency-patches/candle-core-0.4.1/src/storage.rs deleted file mode 100644 index 9e1a2c1dee..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/storage.rs +++ /dev/null @@ -1,700 +0,0 @@ -use crate::backend::BackendStorage; -use crate::op::{self, CmpOp, CustomOp1, CustomOp2, CustomOp3, ReduceOp}; -use crate::{CpuStorage, CudaStorage, DType, Device, Error, Layout, MetalStorage, Result, Shape}; - -// We do not want to implement Clone on Storage as cloning may fail because of -// out of memory. Instead try_clone should be used. -#[derive(Debug)] -pub enum Storage { - Cpu(CpuStorage), - Cuda(CudaStorage), - Metal(MetalStorage), -} - -impl Storage { - pub fn try_clone(&self, layout: &Layout) -> Result { - match self { - Self::Cpu(storage) => Ok(Self::Cpu(storage.clone())), - Self::Cuda(storage) => { - let storage = storage.try_clone(layout)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = storage.try_clone(layout)?; - Ok(Self::Metal(storage)) - } - } - } - - pub fn device(&self) -> Device { - match self { - Self::Cpu(_) => Device::Cpu, - Self::Cuda(storage) => Device::Cuda(storage.device().clone()), - Self::Metal(storage) => Device::Metal(storage.device().clone()), - } - } - - pub fn dtype(&self) -> DType { - match self { - Self::Cpu(storage) => storage.dtype(), - Self::Cuda(storage) => storage.dtype(), - Self::Metal(storage) => storage.dtype(), - } - } - - pub(crate) fn same_device(&self, rhs: &Self, op: &'static str) -> Result<()> { - let lhs = self.device().location(); - let rhs = rhs.device().location(); - if lhs != rhs { - Err(Error::DeviceMismatchBinaryOp { lhs, rhs, op }.bt()) - } else { - Ok(()) - } - } - - pub(crate) fn same_dtype(&self, rhs: &Self, op: &'static str) -> Result<()> { - let lhs = self.dtype(); - let rhs = rhs.dtype(); - if lhs != rhs { - Err(Error::DTypeMismatchBinaryOp { lhs, rhs, op }.bt()) - } else { - Ok(()) - } - } - - pub(crate) fn affine(&self, layout: &Layout, mul: f64, add: f64) -> Result { - match self { - Storage::Cpu(storage) => { - let storage = storage.affine(layout, mul, add)?; - Ok(Self::Cpu(storage)) - } - Self::Cuda(storage) => { - let storage = storage.affine(layout, mul, add)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = storage.affine(layout, mul, add)?; - Ok(Self::Metal(storage)) - } - } - } - - pub(crate) fn powf(&self, layout: &Layout, alpha: f64) -> Result { - match self { - Storage::Cpu(storage) => { - let storage = storage.powf(layout, alpha)?; - Ok(Self::Cpu(storage)) - } - Self::Cuda(storage) => { - let storage = storage.powf(layout, alpha)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = storage.powf(layout, alpha)?; - Ok(Self::Metal(storage)) - } - } - } - - pub(crate) fn elu(&self, layout: &Layout, alpha: f64) -> Result { - match self { - Storage::Cpu(storage) => { - let storage = storage.elu(layout, alpha)?; - Ok(Self::Cpu(storage)) - } - Self::Cuda(storage) => { - let storage = storage.elu(layout, alpha)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = storage.elu(layout, alpha)?; - Ok(Self::Metal(storage)) - } - } - } - - pub(crate) fn cmp( - &self, - op: CmpOp, - rhs: &Self, - lhs_layout: &Layout, - rhs_layout: &Layout, - ) -> Result { - self.same_device(rhs, "cmp")?; - self.same_dtype(rhs, "cmp")?; - match (self, rhs) { - (Storage::Cpu(lhs), Storage::Cpu(rhs)) => { - let storage = lhs.cmp(op, rhs, lhs_layout, rhs_layout)?; - Ok(Self::Cpu(storage)) - } - (Self::Cuda(lhs), Self::Cuda(rhs)) => { - let storage = lhs.cmp(op, rhs, lhs_layout, rhs_layout)?; - Ok(Self::Cuda(storage)) - } - (Self::Metal(lhs), Self::Metal(rhs)) => { - let storage = lhs.cmp(op, rhs, lhs_layout, rhs_layout)?; - Ok(Self::Metal(storage)) - } - (lhs, rhs) => { - // Should not happen because of the same device check above but we're defensive - // anyway. - Err(Error::DeviceMismatchBinaryOp { - lhs: lhs.device().location(), - rhs: rhs.device().location(), - op: "cmp", - } - .bt()) - } - } - } - - pub(crate) fn reduce_op(&self, op: ReduceOp, layout: &Layout, s: &[usize]) -> Result { - match self { - Storage::Cpu(storage) => { - let storage = storage.reduce_op(op, layout, s)?; - Ok(Self::Cpu(storage)) - } - Self::Cuda(storage) => { - let storage = storage.reduce_op(op, layout, s)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = storage.reduce_op(op, layout, s)?; - Ok(Self::Metal(storage)) - } - } - } - - pub(crate) fn to_dtype(&self, layout: &Layout, dtype: DType) -> Result { - match self { - Storage::Cpu(storage) => { - let storage = storage.to_dtype(layout, dtype)?; - Ok(Self::Cpu(storage)) - } - Self::Cuda(storage) => { - let storage = storage.to_dtype(layout, dtype)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = storage.to_dtype(layout, dtype)?; - Ok(Self::Metal(storage)) - } - } - } - - pub(crate) fn apply_op1(&self, l: &Layout, c: &dyn CustomOp1) -> Result<(Self, Shape)> { - match self { - Self::Cpu(storage) => { - let (storage, shape) = c.cpu_fwd(storage, l)?; - Ok((Self::Cpu(storage), shape)) - } - Self::Cuda(storage) => { - let (storage, shape) = c.cuda_fwd(storage, l)?; - Ok((Self::Cuda(storage), shape)) - } - Self::Metal(storage) => { - let (storage, shape) = c.metal_fwd(storage, l)?; - Ok((Self::Metal(storage), shape)) - } - } - } - - pub(crate) fn apply_op2( - &self, - l1: &Layout, - t2: &Self, - l2: &Layout, - c: &dyn CustomOp2, - ) -> Result<(Self, Shape)> { - self.same_device(t2, c.name())?; - match (self, t2) { - (Self::Cpu(s1), Self::Cpu(s2)) => { - let (s, shape) = c.cpu_fwd(s1, l1, s2, l2)?; - Ok((Self::Cpu(s), shape)) - } - (Self::Cuda(s1), Self::Cuda(s2)) => { - let (s, shape) = c.cuda_fwd(s1, l1, s2, l2)?; - Ok((Self::Cuda(s), shape)) - } - (Self::Metal(s1), Self::Metal(s2)) => { - let (s, shape) = c.metal_fwd(s1, l1, s2, l2)?; - Ok((Self::Metal(s), shape)) - } - _ => unreachable!(), - } - } - - pub(crate) fn apply_op3( - &self, - l1: &Layout, - t2: &Self, - l2: &Layout, - t3: &Self, - l3: &Layout, - c: &dyn CustomOp3, - ) -> Result<(Self, Shape)> { - self.same_device(t2, c.name())?; - self.same_device(t3, c.name())?; - match (self, t2, t3) { - (Self::Cpu(s1), Self::Cpu(s2), Self::Cpu(s3)) => { - let (s, shape) = c.cpu_fwd(s1, l1, s2, l2, s3, l3)?; - Ok((Self::Cpu(s), shape)) - } - (Self::Cuda(s1), Self::Cuda(s2), Self::Cuda(s3)) => { - let (s, shape) = c.cuda_fwd(s1, l1, s2, l2, s3, l3)?; - Ok((Self::Cuda(s), shape)) - } - (Self::Metal(s1), Self::Metal(s2), Self::Metal(s3)) => { - let (s, shape) = c.metal_fwd(s1, l1, s2, l2, s3, l3)?; - Ok((Self::Metal(s), shape)) - } - _ => unreachable!(), - } - } - - pub(crate) fn unary_impl(&self, layout: &Layout) -> Result { - match self { - Storage::Cpu(storage) => { - let storage = storage.unary_impl::(layout)?; - Ok(Self::Cpu(storage)) - } - Self::Cuda(storage) => { - let storage = storage.unary_impl::(layout)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = storage.unary_impl::(layout)?; - Ok(Self::Metal(storage)) - } - } - } - - pub(crate) fn binary_impl( - &self, - rhs: &Self, - lhs_layout: &Layout, - rhs_layout: &Layout, - ) -> Result { - self.same_device(rhs, B::NAME)?; - self.same_dtype(rhs, B::NAME)?; - match (self, rhs) { - (Storage::Cpu(lhs), Storage::Cpu(rhs)) => { - let storage = lhs.binary_impl::(rhs, lhs_layout, rhs_layout)?; - Ok(Self::Cpu(storage)) - } - (Self::Cuda(lhs), Self::Cuda(rhs)) => { - let storage = lhs.binary_impl::(rhs, lhs_layout, rhs_layout)?; - Ok(Self::Cuda(storage)) - } - (Self::Metal(lhs), Self::Metal(rhs)) => { - let storage = lhs.binary_impl::(rhs, lhs_layout, rhs_layout)?; - Ok(Self::Metal(storage)) - } - (lhs, rhs) => { - // Should not happen because of the same device check above but we're defensive - // anyway. - Err(Error::DeviceMismatchBinaryOp { - lhs: lhs.device().location(), - rhs: rhs.device().location(), - op: B::NAME, - } - .bt()) - } - } - } - - pub(crate) fn conv1d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConv1D, - ) -> Result { - self.same_device(kernel, "conv1d")?; - self.same_dtype(kernel, "conv1d")?; - match (self, &kernel) { - (Storage::Cpu(inp), Storage::Cpu(kernel)) => { - let s = inp.conv1d(l, kernel, kernel_l, params)?; - Ok(Self::Cpu(s)) - } - (Storage::Cuda(inp), Storage::Cuda(kernel)) => { - let s = inp.conv1d(l, kernel, kernel_l, params)?; - Ok(Self::Cuda(s)) - } - (Storage::Metal(inp), Storage::Metal(kernel)) => { - let s = inp.conv1d(l, kernel, kernel_l, params)?; - Ok(Self::Metal(s)) - } - (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { - lhs: lhs.device().location(), - rhs: rhs.device().location(), - op: "conv1d", - } - .bt()), - } - } - - pub(crate) fn conv_transpose1d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConvTranspose1D, - ) -> Result { - self.same_device(kernel, "conv-transpose1d")?; - self.same_dtype(kernel, "conv-transpose1d")?; - match (self, &kernel) { - (Storage::Cpu(inp), Storage::Cpu(kernel)) => { - let s = inp.conv_transpose1d(l, kernel, kernel_l, params)?; - Ok(Self::Cpu(s)) - } - (Storage::Cuda(inp), Storage::Cuda(kernel)) => { - let s = inp.conv_transpose1d(l, kernel, kernel_l, params)?; - Ok(Self::Cuda(s)) - } - (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { - lhs: lhs.device().location(), - rhs: rhs.device().location(), - op: "conv-transpose1d", - } - .bt()), - } - } - - pub(crate) fn conv2d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConv2D, - ) -> Result { - self.same_device(kernel, "conv2d")?; - self.same_dtype(kernel, "conv2d")?; - match (self, &kernel) { - (Storage::Cpu(inp), Storage::Cpu(kernel)) => { - let s = inp.conv2d(l, kernel, kernel_l, params)?; - Ok(Self::Cpu(s)) - } - (Storage::Cuda(inp), Storage::Cuda(kernel)) => { - let s = inp.conv2d(l, kernel, kernel_l, params)?; - Ok(Self::Cuda(s)) - } - (Storage::Metal(inp), Storage::Metal(kernel)) => { - let s = inp.conv2d(l, kernel, kernel_l, params)?; - Ok(Self::Metal(s)) - } - (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { - lhs: lhs.device().location(), - rhs: rhs.device().location(), - op: "conv2d", - } - .bt()), - } - } - - pub(crate) fn conv_transpose2d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConvTranspose2D, - ) -> Result { - self.same_device(kernel, "conv_transpose2d")?; - self.same_dtype(kernel, "conv_transpose2d")?; - match (self, &kernel) { - (Storage::Cpu(inp), Storage::Cpu(kernel)) => { - let s = inp.conv_transpose2d(l, kernel, kernel_l, params)?; - Ok(Self::Cpu(s)) - } - (Storage::Cuda(inp), Storage::Cuda(kernel)) => { - let s = inp.conv_transpose2d(l, kernel, kernel_l, params)?; - Ok(Self::Cuda(s)) - } - (Storage::Metal(inp), Storage::Metal(kernel)) => { - let s = inp.conv_transpose2d(l, kernel, kernel_l, params)?; - Ok(Self::Metal(s)) - } - (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { - lhs: lhs.device().location(), - rhs: rhs.device().location(), - op: "conv_transpose2d", - } - .bt()), - } - } - - pub(crate) fn avg_pool2d( - &self, - layout: &Layout, - kernel_size: (usize, usize), - stride: (usize, usize), - ) -> Result { - match self { - Storage::Cpu(storage) => { - let storage = storage.avg_pool2d(layout, kernel_size, stride)?; - Ok(Self::Cpu(storage)) - } - Self::Cuda(storage) => { - let storage = storage.avg_pool2d(layout, kernel_size, stride)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = storage.avg_pool2d(layout, kernel_size, stride)?; - Ok(Self::Metal(storage)) - } - } - } - - pub(crate) fn max_pool2d( - &self, - layout: &Layout, - kernel_size: (usize, usize), - stride: (usize, usize), - ) -> Result { - match self { - Storage::Cpu(storage) => { - let storage = storage.max_pool2d(layout, kernel_size, stride)?; - Ok(Self::Cpu(storage)) - } - Self::Cuda(storage) => { - let storage = storage.max_pool2d(layout, kernel_size, stride)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = storage.max_pool2d(layout, kernel_size, stride)?; - Ok(Self::Metal(storage)) - } - } - } - - pub(crate) fn upsample_nearest1d(&self, layout: &Layout, sz: usize) -> Result { - match self { - Storage::Cpu(storage) => { - let storage = storage.upsample_nearest1d(layout, sz)?; - Ok(Self::Cpu(storage)) - } - Self::Cuda(storage) => { - let storage = storage.upsample_nearest1d(layout, sz)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = storage.upsample_nearest1d(layout, sz)?; - Ok(Self::Metal(storage)) - } - } - } - - pub(crate) fn upsample_nearest2d(&self, layout: &Layout, h: usize, w: usize) -> Result { - match self { - Storage::Cpu(storage) => { - let storage = storage.upsample_nearest2d(layout, h, w)?; - Ok(Self::Cpu(storage)) - } - Self::Cuda(storage) => { - let storage = storage.upsample_nearest2d(layout, h, w)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = storage.upsample_nearest2d(layout, h, w)?; - Ok(Self::Metal(storage)) - } - } - } - - pub(crate) fn where_cond( - &self, - layout: &Layout, - t: &Self, - layout_t: &Layout, - f: &Self, - layout_f: &Layout, - ) -> Result { - self.same_device(t, "where")?; - self.same_device(f, "where")?; - t.same_dtype(f, "where")?; - match (self, t, f) { - (Storage::Cpu(cond), Storage::Cpu(t), Storage::Cpu(f)) => { - let storage = cond.where_cond(layout, t, layout_t, f, layout_f)?; - Ok(Self::Cpu(storage)) - } - (Self::Cuda(cond), Self::Cuda(t), Self::Cuda(f)) => { - let storage = cond.where_cond(layout, t, layout_t, f, layout_f)?; - Ok(Self::Cuda(storage)) - } - (Self::Metal(cond), Self::Metal(t), Self::Metal(f)) => { - let storage = cond.where_cond(layout, t, layout_t, f, layout_f)?; - Ok(Self::Metal(storage)) - } - (_, lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { - lhs: lhs.device().location(), - rhs: rhs.device().location(), - op: "where", - } - .bt()), - } - } - - pub(crate) fn gather( - &self, - l: &Layout, - indexes: &Self, - indexes_l: &Layout, - d: usize, - ) -> Result { - self.same_device(indexes, "index-add")?; - match (self, indexes) { - (Self::Cpu(s), Self::Cpu(indexes)) => { - let storage = s.gather(l, indexes, indexes_l, d)?; - Ok(Self::Cpu(storage)) - } - (Self::Cuda(s), Self::Cuda(indexes)) => { - let storage = s.gather(l, indexes, indexes_l, d)?; - Ok(Self::Cuda(storage)) - } - (Self::Metal(s), Self::Metal(indexes)) => { - let storage = s.gather(l, indexes, indexes_l, d)?; - Ok(Self::Metal(storage)) - } - _ => unreachable!(), - } - } - - pub(crate) fn scatter_add( - &self, - l: &Layout, - indexes: &Self, - indexes_l: &Layout, - source: &Self, - source_l: &Layout, - d: usize, - ) -> Result { - self.same_device(indexes, "scatter-add")?; - self.same_device(source, "scatter-add")?; - match (self, indexes, source) { - (Self::Cpu(s), Self::Cpu(indexes), Self::Cpu(source)) => { - let storage = s.scatter_add(l, indexes, indexes_l, source, source_l, d)?; - Ok(Self::Cpu(storage)) - } - (Self::Cuda(s), Self::Cuda(indexes), Self::Cuda(source)) => { - let storage = s.scatter_add(l, indexes, indexes_l, source, source_l, d)?; - Ok(Self::Cuda(storage)) - } - (Self::Metal(s), Self::Metal(indexes), Self::Metal(source)) => { - let storage = s.scatter_add(l, indexes, indexes_l, source, source_l, d)?; - Ok(Self::Metal(storage)) - } - _ => unreachable!(), - } - } - - pub(crate) fn index_add( - &self, - l: &Layout, - indexes: &Self, - indexes_l: &Layout, - source: &Self, - source_l: &Layout, - d: usize, - ) -> Result { - self.same_device(indexes, "index-add")?; - self.same_device(source, "index-add")?; - match (self, indexes, source) { - (Self::Cpu(s), Self::Cpu(indexes), Self::Cpu(source)) => { - let storage = s.index_add(l, indexes, indexes_l, source, source_l, d)?; - Ok(Self::Cpu(storage)) - } - (Self::Cuda(s), Self::Cuda(indexes), Self::Cuda(source)) => { - let storage = s.index_add(l, indexes, indexes_l, source, source_l, d)?; - Ok(Self::Cuda(storage)) - } - (Self::Metal(s), Self::Metal(indexes), Self::Metal(source)) => { - let storage = s.index_add(l, indexes, indexes_l, source, source_l, d)?; - Ok(Self::Metal(storage)) - } - _ => unreachable!(), - } - } - - pub(crate) fn index_select( - &self, - rhs: &Self, - lhs_l: &Layout, - rhs_l: &Layout, - d: usize, - ) -> Result { - self.same_device(rhs, "index-select")?; - match (self, rhs) { - (Self::Cpu(lhs), Self::Cpu(rhs)) => { - let storage = lhs.index_select(rhs, lhs_l, rhs_l, d)?; - Ok(Self::Cpu(storage)) - } - (Self::Cuda(lhs), Self::Cuda(rhs)) => { - let storage = lhs.index_select(rhs, lhs_l, rhs_l, d)?; - Ok(Self::Cuda(storage)) - } - (Self::Metal(lhs), Self::Metal(rhs)) => { - let storage = lhs.index_select(rhs, lhs_l, rhs_l, d)?; - Ok(Self::Metal(storage)) - } - (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { - lhs: lhs.device().location(), - rhs: rhs.device().location(), - op: "index-select", - } - .bt()), - } - } - - pub(crate) fn matmul( - &self, - rhs: &Self, - bmnk: (usize, usize, usize, usize), - lhs_layout: &Layout, - rhs_layout: &Layout, - ) -> Result { - self.same_device(rhs, "matmul")?; - self.same_dtype(rhs, "matmul")?; - match (self, rhs) { - (Self::Cpu(lhs), Self::Cpu(rhs)) => { - let storage = lhs.matmul(rhs, bmnk, lhs_layout, rhs_layout)?; - Ok(Self::Cpu(storage)) - } - (Self::Cuda(lhs), Self::Cuda(rhs)) => { - let storage = lhs.matmul(rhs, bmnk, lhs_layout, rhs_layout)?; - Ok(Self::Cuda(storage)) - } - (Self::Metal(lhs), Self::Metal(rhs)) => { - let storage = lhs.matmul(rhs, bmnk, lhs_layout, rhs_layout)?; - Ok(Self::Metal(storage)) - } - (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { - lhs: lhs.device().location(), - rhs: rhs.device().location(), - op: "matmul", - } - .bt()), - } - } - - // self, the source can be strided whereas dst is contiguous. - pub(crate) fn copy_strided_src( - &self, - dst: &mut Self, - dst_offset: usize, - src_l: &Layout, - ) -> Result<()> { - match (self, dst) { - (Self::Cpu(src), Self::Cpu(dst)) => src.copy_strided_src(dst, dst_offset, src_l), - (Self::Cuda(src), Self::Cuda(dst)) => Ok(src.copy_strided_src(dst, dst_offset, src_l)?), - (Self::Metal(src), Self::Metal(dst)) => { - Ok(src.copy_strided_src(dst, dst_offset, src_l)?) - } - (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { - lhs: lhs.device().location(), - rhs: rhs.device().location(), - op: "copy", - } - .bt()), - } - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/strided_index.rs b/v2/dependency-patches/candle-core-0.4.1/src/strided_index.rs deleted file mode 100644 index eb6a736f83..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/strided_index.rs +++ /dev/null @@ -1,82 +0,0 @@ -use crate::Layout; - -/// An iterator over offset position for items of an N-dimensional arrays stored in a -/// flat buffer using some potential strides. -#[derive(Debug)] -pub struct StridedIndex<'a> { - next_storage_index: Option, - multi_index: Vec, - dims: &'a [usize], - stride: &'a [usize], -} - -impl<'a> StridedIndex<'a> { - pub(crate) fn new(dims: &'a [usize], stride: &'a [usize], start_offset: usize) -> Self { - let elem_count: usize = dims.iter().product(); - let next_storage_index = if elem_count == 0 { - None - } else { - // This applies to the scalar case. - Some(start_offset) - }; - StridedIndex { - next_storage_index, - multi_index: vec![0; dims.len()], - dims, - stride, - } - } - - pub(crate) fn from_layout(l: &'a Layout) -> Self { - Self::new(l.dims(), l.stride(), l.start_offset()) - } -} - -impl<'a> Iterator for StridedIndex<'a> { - type Item = usize; - - fn next(&mut self) -> Option { - let storage_index = match self.next_storage_index { - None => return None, - Some(storage_index) => storage_index, - }; - let mut updated = false; - let mut next_storage_index = storage_index; - for ((multi_i, max_i), stride_i) in self - .multi_index - .iter_mut() - .zip(self.dims.iter()) - .zip(self.stride.iter()) - .rev() - { - let next_i = *multi_i + 1; - if next_i < *max_i { - *multi_i = next_i; - updated = true; - next_storage_index += stride_i; - break; - } else { - next_storage_index -= *multi_i * stride_i; - *multi_i = 0 - } - } - self.next_storage_index = if updated { - Some(next_storage_index) - } else { - None - }; - Some(storage_index) - } -} - -#[derive(Debug)] -pub enum StridedBlocks<'a> { - SingleBlock { - start_offset: usize, - len: usize, - }, - MultipleBlocks { - block_start_index: StridedIndex<'a>, - block_len: usize, - }, -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/tensor.rs b/v2/dependency-patches/candle-core-0.4.1/src/tensor.rs deleted file mode 100644 index 0e2c3e8f32..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/tensor.rs +++ /dev/null @@ -1,2765 +0,0 @@ -//! Tensors are N-dimensional matrixes of elements using a single data type. -#![allow(clippy::redundant_closure_call)] -use crate::backend::{BackendDevice, BackendStorage}; -use crate::op::{ - BackpropOp, BinaryOp, CmpOp, CustomOp1, CustomOp2, CustomOp3, Op, ReduceOp, UnaryOp, -}; -use crate::scalar::TensorOrScalar; -use crate::shape::{Dim, Dims}; -use crate::{bail, storage::Storage, DType, Device, Error, Layout, Result, Shape}; -use std::sync::{Arc, RwLock}; - -/// Unique identifier for tensors. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct TensorId(usize); - -impl TensorId { - fn new() -> Self { - // https://users.rust-lang.org/t/idiomatic-rust-way-to-generate-unique-id/33805 - use std::sync::atomic; - static COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::new(1); - Self(COUNTER.fetch_add(1, atomic::Ordering::Relaxed)) - } -} - -pub struct Tensor_ { - id: TensorId, - // As we provide inner mutability on the tensor content, the alternatives are: - // - Using a mutex, this would have the highest cost when retrieving the storage but would - // prevent errors when concurrent access takes place. Mutex would also be subject to - // deadlocks for example using the current code if the same tensor is used twice by a single - // binary op. - // - Using a refcell unsafe cell would have some intermediary cost, borrow checking would be - // verified dynamically, but the resulting tensors would not be send or sync. - // - Using an unsafe cell would have the lowest cost but undefined behavior on concurrent - // accesses. - // Ideally, we would use Arc for tensors on which we don't plan on modifying the data - // and Arc> for tensors where the data could be modified, e.g. variables but - // that's tricky to encode in the current setup. - storage: Arc>, - layout: Layout, - op: BackpropOp, - is_variable: bool, - dtype: DType, - device: Device, -} - -impl AsRef for Tensor { - fn as_ref(&self) -> &Tensor { - self - } -} - -// Tensors are refcounted so that cloning is cheap when building the op graph. -// Storages are also refcounted independently so that its possible to avoid -// copying the storage for operations that only modify the shape or stride. -#[derive(Clone)] -/// The core struct for manipulating tensors. -/// -/// ```rust -/// use candle_core::{Tensor, DType, Device}; -/// -/// let a = Tensor::arange(0f32, 6f32, &Device::Cpu)?.reshape((2, 3))?; -/// let b = Tensor::arange(0f32, 12f32, &Device::Cpu)?.reshape((3, 4))?; -/// -/// let c = a.matmul(&b)?; -/// # Ok::<(), candle_core::Error>(()) -/// ``` -/// -/// Tensors are reference counted with [`Arc`] so cloning them is cheap. -pub struct Tensor(Arc); - -impl std::ops::Deref for Tensor { - type Target = Tensor_; - - fn deref(&self) -> &Self::Target { - self.0.as_ref() - } -} - -macro_rules! unary_op { - ($fn_name:ident, $op_name:ident) => { - pub fn $fn_name(&self) -> Result { - let shape = self.shape(); - let storage = self - .storage() - .unary_impl::(self.layout())?; - let op = BackpropOp::new1(self, |s| Op::Unary(s, UnaryOp::$op_name)); - Ok(from_storage(storage, shape.clone(), op, false)) - } - }; -} - -macro_rules! binary_op { - ($fn_name:ident, $op_name:ident) => { - pub fn $fn_name(&self, rhs: &Self) -> Result { - let shape = self.same_shape_binary_op(rhs, stringify!($fn_name))?; - let storage = self.storage().binary_impl::( - &*rhs.storage(), - self.layout(), - rhs.layout(), - )?; - let op = BackpropOp::new2(self, rhs, |t1, t2| Op::Binary(t1, t2, BinaryOp::$op_name)); - Ok(from_storage(storage, shape.clone(), op, false)) - } - }; -} - -macro_rules! binary_op_scalar { - ($fn_name:ident, $op_name:ident) => { - pub fn $fn_name(&self, rhs: T) -> Result { - let rhs = match rhs.to_tensor_scalar()? { - crate::scalar::TensorScalar::Tensor(rhs) => rhs, - crate::scalar::TensorScalar::Scalar(rhs) => rhs - .to_dtype(self.dtype())? - .to_device(self.device())? - .broadcast_as(self.shape())?, - }; - let shape = self.same_shape_binary_op(&rhs, stringify!($fn_name))?; - let storage = self.storage().binary_impl::( - &*rhs.storage(), - self.layout(), - rhs.layout(), - )?; - let op = BackpropOp::new2(self, &rhs, |t1, t2| Op::Binary(t1, t2, BinaryOp::$op_name)); - Ok(from_storage(storage, shape.clone(), op, false)) - } - }; -} - -macro_rules! broadcast_binary_op { - ($fn_name:ident, $inner_fn_name:ident) => { - pub fn $fn_name(&self, rhs: &Self) -> Result { - let lhs = self; - let shape = lhs - .shape() - .broadcast_shape_binary_op(rhs.shape(), stringify!($fn_name))?; - let l_broadcast = shape != *lhs.shape(); - let r_broadcast = shape != *rhs.shape(); - match (l_broadcast, r_broadcast) { - (true, true) => lhs - .broadcast_as(&shape)? - .$inner_fn_name(&rhs.broadcast_as(&shape)?), - (false, true) => lhs.$inner_fn_name(&rhs.broadcast_as(&shape)?), - (true, false) => lhs.broadcast_as(&shape)?.$inner_fn_name(rhs), - (false, false) => lhs.$inner_fn_name(rhs), - } - } - }; -} - -/// Creates a fresh tensor structure based on a storage and a shape, this uses contiguous strides. -pub(crate) fn from_storage>( - storage: Storage, - shape: S, - op: BackpropOp, - is_variable: bool, -) -> Tensor { - let dtype = storage.dtype(); - let device = storage.device(); - let tensor_ = Tensor_ { - id: TensorId::new(), - storage: Arc::new(RwLock::new(storage)), - layout: Layout::contiguous(shape), - op, - is_variable, - dtype, - device, - }; - Tensor(Arc::new(tensor_)) -} - -impl Tensor { - pub(crate) fn ones_impl>( - shape: S, - dtype: DType, - device: &Device, - is_variable: bool, - ) -> Result { - let none = BackpropOp::none(); - let shape = shape.into(); - let storage = device.ones(&shape, dtype)?; - Ok(from_storage(storage, shape, none, is_variable)) - } - - /// Creates a new tensor filled with ones. - /// - /// ```rust - /// use candle_core::{Tensor, DType, Device}; - /// let a = Tensor::ones((2, 3), DType::F32, &Device::Cpu)?; - /// let b = Tensor::from_slice(&[1.0f32, 1.0, 1.0, 1.0, 1.0, 1.0], (2, 3), &Device::Cpu)?; - /// // a == b - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn ones>(shape: S, dtype: DType, device: &Device) -> Result { - Self::ones_impl(shape, dtype, device, false) - } - - /// Creates a new tensor filled with ones with same shape, dtype, and device as the other tensor. - /// - /// ```rust - /// use candle_core::{Tensor, DType, Device}; - /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; - /// let b = a.ones_like()?; - /// // b == a + 1 - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn ones_like(&self) -> Result { - Tensor::ones(self.shape(), self.dtype(), self.device()) - } - - // Do not expose outside of the crate, the `is_variable=true` case should only be accessed from - // the variable module. - pub(crate) fn zeros_impl>( - shape: S, - dtype: DType, - device: &Device, - is_variable: bool, - ) -> Result { - let none = BackpropOp::none(); - let shape = shape.into(); - let storage = device.zeros(&shape, dtype)?; - Ok(from_storage(storage, shape, none, is_variable)) - } - - /// Creates a new tensor filled with zeros. - /// - /// ```rust - /// use candle_core::{Tensor, DType, Device}; - /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; - /// let b = Tensor::from_slice(&[0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0], (2, 3), &Device::Cpu)?; - /// // a == b - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn zeros>(shape: S, dtype: DType, device: &Device) -> Result { - Self::zeros_impl(shape, dtype, device, false) - } - - /// Creates a new tensor filled with ones with same shape, dtype, and device as the other - /// tensor. - /// - /// ```rust - /// use candle_core::{Tensor, DType, Device}; - /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; - /// let b = a.zeros_like()?; - /// // b is on CPU f32. - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn zeros_like(&self) -> Result { - Tensor::zeros(self.shape(), self.dtype(), self.device()) - } - - pub(crate) fn rand_impl, T: crate::FloatDType>( - lo: T, - up: T, - s: S, - device: &Device, - is_variable: bool, - ) -> Result { - let s = s.into(); - let storage = device.rand_uniform(lo, up, &s)?; - let none = BackpropOp::none(); - Ok(from_storage(storage, s, none, is_variable)) - } - - pub(crate) fn rand_f64_impl>( - lo: f64, - up: f64, - s: S, - dtype: DType, - device: &Device, - is_variable: bool, - ) -> Result { - let s = s.into(); - let storage = device.rand_uniform_f64(lo, up, &s, dtype)?; - let none = BackpropOp::none(); - Ok(from_storage(storage, s, none, is_variable)) - } - - /// Creates a new tensor initialized with values sampled uniformly between `lo` and `up`. - pub fn rand, T: crate::FloatDType>( - lo: T, - up: T, - s: S, - device: &Device, - ) -> Result { - Self::rand_impl(lo, up, s, device, false) - } - - pub fn rand_like(&self, lo: f64, up: f64) -> Result { - Tensor::rand_f64_impl(lo, up, self.shape(), self.dtype(), self.device(), false) - } - - pub(crate) fn randn_impl, T: crate::FloatDType>( - mean: T, - std: T, - s: S, - device: &Device, - is_variable: bool, - ) -> Result { - let s = s.into(); - let storage = device.rand_normal(mean, std, &s)?; - let none = BackpropOp::none(); - Ok(from_storage(storage, s, none, is_variable)) - } - - pub(crate) fn randn_f64_impl>( - mean: f64, - std: f64, - s: S, - dtype: DType, - device: &Device, - is_variable: bool, - ) -> Result { - let s = s.into(); - let storage = device.rand_normal_f64(mean, std, &s, dtype)?; - let none = BackpropOp::none(); - Ok(from_storage(storage, s, none, is_variable)) - } - - pub fn randn_like(&self, mean: f64, stdev: f64) -> Result { - Tensor::randn_f64_impl( - mean, - stdev, - self.shape(), - self.dtype(), - self.device(), - false, - ) - } - - /// Creates a new tensor initialized with values sampled from a normal distribution with the - /// specified `mean` and standard deviation `std`. - pub fn randn, T: crate::FloatDType>( - mean: T, - std: T, - s: S, - device: &Device, - ) -> Result { - Self::randn_impl(mean, std, s, device, false) - } - - pub(crate) fn new_impl( - array: A, - shape: Shape, - device: &Device, - is_variable: bool, - ) -> Result { - let n: usize = shape.elem_count(); - let buffer_size: usize = array.shape()?.elem_count(); - if buffer_size != n { - return Err(Error::ShapeMismatch { buffer_size, shape }.bt()); - } - let storage = device.storage(array)?; - let none = BackpropOp::none(); - Ok(from_storage(storage, shape, none, is_variable)) - } - - /// Creates a new tensor on the specified device using the content and shape of the input. - pub fn new(array: A, device: &Device) -> Result { - let shape = array.shape()?; - Self::new_impl(array, shape, device, false) - } - - /// Returns a new tensor with all the elements having the same specified value. Note that - /// the tensor is not contiguous so you would have to call `.contiguous()` on it if needed. - pub fn full>( - value: D, - shape: S, - device: &Device, - ) -> Result { - Self::from_vec_impl(vec![value], (), device, false)?.broadcast_as(shape) - } - - /// Creates a new 1D tensor from an iterator. - pub fn from_iter( - iter: impl IntoIterator, - device: &Device, - ) -> Result { - let data = iter.into_iter().collect::>(); - let len = data.len(); - Self::from_vec_impl(data, len, device, false) - } - - /// Creates a new 1D tensor with values from the interval `[start, end)` taken with a common - /// difference `1` from `start`. - pub fn arange(start: D, end: D, device: &Device) -> Result { - Self::arange_step(start, end, D::one(), device) - } - - /// Creates a new 1D tensor with values from the interval `[start, end)` taken with a common - /// difference `step` from `start`. - pub fn arange_step( - start: D, - end: D, - step: D, - device: &Device, - ) -> Result { - if D::is_zero(&step) { - bail!("step cannot be zero") - } - let mut data = vec![]; - let mut current = start; - if step >= D::zero() { - while current < end { - data.push(current); - current += step; - } - } else { - while current > end { - data.push(current); - current += step; - } - } - let len = data.len(); - Self::from_vec_impl(data, len, device, false) - } - - pub(crate) fn from_vec_impl, D: crate::WithDType>( - data: Vec, - shape: S, - device: &Device, - is_variable: bool, - ) -> Result { - let shape = shape.into(); - let buffer_size = data.len(); - if buffer_size != shape.elem_count() { - return Err(Error::ShapeMismatch { buffer_size, shape }.bt()); - } - let storage = device.storage_owned(data)?; - let none = BackpropOp::none(); - Ok(from_storage(storage, shape, none, is_variable)) - } - - /// Creates a new tensor initialized with values from the input vector. The number of elements - /// in this vector must be the same as the number of elements defined by the shape. - /// If the device is cpu, no data copy is made. - pub fn from_vec, D: crate::WithDType>( - data: Vec, - shape: S, - device: &Device, - ) -> Result { - Self::from_vec_impl(data, shape, device, false) - } - - /// Creates a new tensor initialized with values from the input slice. The number of elements - /// in this vector must be the same as the number of elements defined by the shape. - pub fn from_slice, D: crate::WithDType>( - array: &[D], - shape: S, - device: &Device, - ) -> Result { - Self::new_impl(array, shape.into(), device, false) - } - - pub(crate) fn same_shape_binary_op(&self, rhs: &Self, op: &'static str) -> Result<&Shape> { - let lhs = self.shape(); - let rhs = rhs.shape(); - if lhs != rhs { - Err(Error::ShapeMismatchBinaryOp { - lhs: lhs.clone(), - rhs: rhs.clone(), - op, - } - .bt()) - } else { - Ok(lhs) - } - } - - /// Returns true if the computation graph should track this op, that is if it is - /// a variable or if it has some variable as dependencies. - pub fn track_op(&self) -> bool { - self.is_variable || self.op.is_some() - } - - // TODO: Also make an inplace version or a pre-allocated? This could be tricky - // if this can create cycles in the compute graph. - binary_op!(add, Add); - binary_op!(mul, Mul); - binary_op!(sub, Sub); - binary_op!(div, Div); - binary_op_scalar!(maximum, Maximum); - binary_op_scalar!(minimum, Minimum); - broadcast_binary_op!(broadcast_add, add); - broadcast_binary_op!(broadcast_mul, mul); - broadcast_binary_op!(broadcast_sub, sub); - broadcast_binary_op!(broadcast_div, div); - broadcast_binary_op!(broadcast_maximum, maximum); - broadcast_binary_op!(broadcast_minimum, minimum); - broadcast_binary_op!(broadcast_eq, eq); - broadcast_binary_op!(broadcast_ne, ne); - broadcast_binary_op!(broadcast_lt, lt); - broadcast_binary_op!(broadcast_le, le); - broadcast_binary_op!(broadcast_gt, gt); - broadcast_binary_op!(broadcast_ge, ge); - - unary_op!(recip, Recip); - unary_op!(neg, Neg); - unary_op!(exp, Exp); - unary_op!(log, Log); - unary_op!(sin, Sin); - unary_op!(cos, Cos); - unary_op!(tanh, Tanh); - unary_op!(abs, Abs); - unary_op!(sqr, Sqr); - unary_op!(sqrt, Sqrt); - unary_op!(gelu, Gelu); - unary_op!(gelu_erf, GeluErf); - unary_op!(erf, Erf); - unary_op!(relu, Relu); - unary_op!(silu, Silu); - unary_op!(ceil, Ceil); - unary_op!(floor, Floor); - unary_op!(round, Round); - - /// Round element of the input tensor to the nearest integer. - /// - /// If the number of decimals is negative, it specifies the number of positions to the left of - /// the decimal point. - pub fn round_to(&self, decimals: i32) -> Result { - let mult = 10f64.powi(decimals); - (self * mult)?.round()? * (1f64 / mult) - } - - /// Retrieves the single scalar value hold in the tensor. If the tensor contains multiple - /// dimensions, an error is returned instead. - pub fn to_scalar(&self) -> Result { - if self.rank() != 0 { - Err(Error::UnexpectedNumberOfDims { - expected: 0, - got: self.rank(), - shape: self.shape().clone(), - } - .bt())? - } - let from_cpu_storage = |cpu_storage: &crate::CpuStorage| { - let data = S::cpu_storage_as_slice(cpu_storage)?; - Ok::<_, Error>(data[self.layout().start_offset()]) - }; - match &*self.storage() { - Storage::Cpu(cpu_storage) => from_cpu_storage(cpu_storage), - Storage::Cuda(storage) => from_cpu_storage(&storage.to_cpu_storage()?), - Storage::Metal(storage) => from_cpu_storage(&storage.to_cpu_storage()?), - } - } - - /// An alias for `to_scalar`. - pub fn to_vec0(&self) -> Result { - self.to_scalar::() - } - - /// Repeat this tensor along the specified dimensions. - pub fn repeat>(&self, shape: S) -> Result { - // Similar to PyTorch, we extend the number of dimensions of self if needed. - let repeats = shape.into(); - let repeats = repeats.dims(); - let mut inp = if self.rank() < repeats.len() { - let shape = [vec![1; repeats.len() - self.rank()], self.dims().to_vec()].concat(); - self.reshape(shape)? - } else { - self.clone() - }; - for (idx, &repeat) in repeats.iter().enumerate() { - if repeat > 1 { - inp = Tensor::cat(&vec![&inp; repeat], idx)? - } - } - Ok(inp) - } - - /// Creates grids of coordinates specified by the 1D inputs. - /// - /// # Arguments - /// - /// * `args` - A slice of 1D tensors. - /// * `xy_indexing` - Whether to use xy indexing or ij indexing. If xy is selected, the - /// first dimension corresponds to the cardinality of the second input and the second - /// dimension corresponds to the cardinality of the first input. If ij is selected, the - /// dimensions are in the same order as the cardinality of the inputs. - /// - /// # Examples - /// - /// ```rust - /// use candle_core::{Tensor, Device, Shape}; - /// let x = Tensor::new(&[1f32, 2., 3.], &Device::Cpu)?; - /// let y = Tensor::new(&[4f32, 5., 6.], &Device::Cpu)?; - /// - /// let grids_xy = Tensor::meshgrid(&[&x, &y], true)?; - /// - /// assert_eq!(grids_xy.len(), 2); - /// assert_eq!(grids_xy[0].dims(), &[3, 3]); - /// - /// assert_eq!(grids_xy[0].to_vec2::()?, &[[1., 2., 3.], [1., 2., 3.], [1., 2., 3.]]); - /// assert_eq!(grids_xy[1].to_vec2::()?, &[[4., 4., 4.], [5., 5., 5.], [6., 6., 6.]]); - /// - /// let grids_ij = Tensor::meshgrid(&[&x, &y], false)?; - /// - /// assert_eq!(grids_ij[0].to_vec2::()?, &[[1., 1., 1.], [2., 2., 2.], [3., 3., 3.]]); - /// assert_eq!(grids_ij[1].to_vec2::()?, &[[4., 5., 6.], [4., 5., 6.], [4., 5., 6.]]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - /// - /// # Errors - /// - /// * Will return `Err` if `args` contains less than 2 tensors. - /// - pub fn meshgrid>(args: &[A], xy_indexing: bool) -> Result> { - if args.len() <= 1 { - Err(Error::OpRequiresAtLeastTwoTensors { op: "meshgrid" }.bt())? - } - let args: Vec<_> = if xy_indexing { - args.iter().rev().collect() - } else { - args.iter().collect() - }; - - let mut shape = Vec::with_capacity(args.len()); - for arg in args.iter() { - shape.push(arg.as_ref().dims1()?) - } - - let mut grids = Vec::with_capacity(args.len()); - for idx in 0..args.len() { - let mut ones = vec![1usize; args.len()]; - ones[idx] = shape[idx]; - let arg = args[idx].as_ref().reshape(ones)?; - let mut repeats = shape.clone(); - repeats[idx] = 1; - let repeated_tensor = arg.repeat(repeats)?; - grids.push(repeated_tensor); - } - if xy_indexing { - grids.reverse(); - } - Ok(grids) - } - - /// This operation multiplies the input tensor by `mul` then adds `add` and return the result. - /// The input values `mul` and `add` are casted to the appropriate type so some rounding might - /// be performed. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let a = Tensor::new(&[[0f32, 1.], [2., 3.]], &Device::Cpu)?; - /// let a = a.affine(4., -2.)?; - /// assert_eq!(a.to_vec2::()?, &[[-2.0, 2.0], [6.0, 10.0]]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn affine(&self, mul: f64, add: f64) -> Result { - let storage = self.storage().affine(self.layout(), mul, add)?; - let op = BackpropOp::new1(self, |arg| Op::Affine { arg, mul, add }); - Ok(from_storage(storage, self.shape(), op, false)) - } - - /// Applies the Exponential Linear Unit (ELU) function on each element of the input tensor. - pub fn elu(&self, alpha: f64) -> Result { - let storage = self.storage().elu(self.layout(), alpha)?; - let op = BackpropOp::new1(self, |t| Op::Elu(t, alpha)); - Ok(from_storage(storage, self.shape(), op, false)) - } - - /// Raise the tensor to some float exponent `e`. - pub fn powf(&self, e: f64) -> Result { - let storage = self.storage().powf(self.layout(), e)?; - let op = BackpropOp::new1(self, |t| Op::Powf(t, e)); - Ok(from_storage(storage, self.shape(), op, false)) - } - - fn check_dim(&self, dim: usize, op: &'static str) -> Result<()> { - if dim >= self.dims().len() { - Err(Error::DimOutOfRange { - shape: self.shape().clone(), - dim: dim as i32, - op, - } - .bt())? - } else { - Ok(()) - } - } - - /// Split a tensor into the specified number of chunks, this may return less chunks than - /// specified. - pub fn chunk(&self, chunks: usize, dim: D) -> Result> { - let dim = dim.to_index(self.shape(), "chunk")?; - let size = self.dim(dim)?; - if size < chunks { - (0..size).map(|i| self.narrow(dim, i, 1)).collect() - } else { - let chunk_size = size / chunks; - let cnt_additional = size % chunks; - let mut tensors = vec![]; - let mut sum_chunk_size = 0; - for i in 0..chunks { - let chunk_size = if i < cnt_additional { - chunk_size + 1 - } else { - chunk_size - }; - let tensor = self.narrow(dim, sum_chunk_size, chunk_size)?; - tensors.push(tensor); - sum_chunk_size += chunk_size - } - Ok(tensors) - } - } - - /// Returns a new tensor that is a narrowed version of the input, the dimension `dim` - /// ranges from `start` to `start + len`. - pub fn narrow(&self, dim: D, start: usize, len: usize) -> Result { - let dims = self.dims(); - let dim = dim.to_index(self.shape(), "narrow")?; - let err = |msg| { - Err::<(), _>( - Error::NarrowInvalidArgs { - shape: self.shape().clone(), - dim, - start, - len, - msg, - } - .bt(), - ) - }; - if start > dims[dim] { - err("start > dim_len")? - } - if start.saturating_add(len) > dims[dim] { - err("start + len > dim_len")? - } - if start == 0 && dims[dim] == len { - Ok(self.clone()) - } else { - let op = BackpropOp::new1(self, |t| Op::Narrow(t, dim, start, len)); - let layout = self.layout().narrow(dim, start, len)?; - let tensor_ = Tensor_ { - id: TensorId::new(), - storage: self.storage.clone(), - layout, - op, - is_variable: false, - dtype: self.dtype, - device: self.device.clone(), - }; - Ok(Tensor(Arc::new(tensor_))) - } - } - - fn squeeze_dims(self, dims: &[usize]) -> Result { - match dims { - [] => Ok(self), - [i] => self.squeeze(*i), - dims => { - let dims = self - .dims() - .iter() - .enumerate() - .filter_map(|(dim_idx, &v)| { - if dims.contains(&dim_idx) { - None - } else { - Some(v) - } - }) - .collect::>(); - self.reshape(dims) - } - } - } - - fn reduce_impl(&self, dim: D, keepdim: bool, op: ReduceOp) -> Result { - let dim = dim.to_index(self.shape(), op.name())?; - let storage = self.storage().reduce_op(op, self.layout(), &[dim])?; - let mut dims = self.dims().to_vec(); - dims[dim] = 1; - let op = match op { - ReduceOp::Sum | ReduceOp::Min | ReduceOp::Max => { - BackpropOp::new1(self, |arg| Op::Reduce(arg, op, dims.to_vec())) - } - ReduceOp::ArgMin | ReduceOp::ArgMax => BackpropOp::none(), - }; - let res = from_storage(storage, dims, op, false); - if keepdim { - Ok(res) - } else { - res.squeeze_dims(&[dim]) - } - } - - fn sum_impl(&self, sum_dims: D, keepdim: bool) -> Result { - let sum_dims = sum_dims.to_indexes(self.shape(), "sum")?; - let storage = self - .storage() - .reduce_op(ReduceOp::Sum, self.layout(), &sum_dims)?; - let mut dims = self.dims().to_vec(); - for &sum_dim in sum_dims.iter() { - dims[sum_dim] = 1 - } - let op = BackpropOp::new1(self, |a| Op::Reduce(a, ReduceOp::Sum, dims.to_vec())); - let sum = from_storage(storage, dims, op, false); - if keepdim { - Ok(sum) - } else { - sum.squeeze_dims(&sum_dims) - } - } - - /// Roll the tensor input along the given dimension. - /// Elements that are shifted beyond the last position are re-introduced at the first position. - /// - /// ```rust - /// # use candle_core::{Tensor, Device}; - /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; - /// let tensor = tensor.roll(1, 0)?; - /// assert_eq!(tensor.to_vec2::()?, &[[4., 5.], [0., 1.], [2., 3.]]); - /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; - /// let tensor = tensor.roll(-1, 0)?; - /// assert_eq!(tensor.to_vec2::()?, &[[2., 3.], [4., 5.], [0., 1.]]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn roll(&self, shift: i32, dim: D) -> Result - where - D: Dim + Clone, - { - let dim = dim.to_index(self.shape(), "roll")?; - let dim_size = self.dim(dim)?; - let shift = shift.rem_euclid(dim_size as i32) as usize; - if shift == 0 { - Ok(self.clone()) - } else { - let a = self.narrow(dim, 0, dim_size - shift)?; - let b = self.narrow(dim, dim_size - shift, shift)?; - Tensor::cat(&[&b, &a], dim) - } - } - - /// Returns the sum of all elements in the input tensor. The sum is performed over all the - /// input dimensions. - /// - /// The resulting tensor has a shape that is similar to the shape of the input tensor, except - /// that the number of elements for each dimension index in `sum_dims` is 1. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let a = Tensor::new(&[[0f32, 1.], [2., 3.]], &Device::Cpu)?; - /// let s = a.sum_keepdim(0)?; - /// assert_eq!(s.to_vec2::()?, &[[2., 4.]]); - /// let s = a.sum_keepdim(1)?; - /// assert_eq!(s.to_vec2::()?, &[[1.], [5.]]); - /// let s = a.sum_keepdim((0, 1))?; - /// assert_eq!(s.to_vec2::()?, &[[6.]]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn sum_keepdim(&self, sum_dims: D) -> Result { - self.sum_impl(sum_dims, true) - } - - /// Returns the sum of all elements in the input tensor. The sum is performed over all the - /// input dimensions and compared to `sum_keepdim` these dimensions are squeezed rather than - /// kept. - pub fn sum(&self, sum_dims: D) -> Result { - self.sum_impl(sum_dims, false) - } - - /// Returns the mean of all elements in the input tensor. The mean is performed over all the - /// input dimensions. - /// - /// The resulting tensor has a shape that is similar to the shape of the input tensor, except - /// that the number of elements for each dimension index in `mean_dims` is 1. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let a = Tensor::new(&[[0f32, 1.], [2., 3.]], &Device::Cpu)?; - /// let s = a.mean_keepdim(0)?; - /// assert_eq!(s.to_vec2::()?, &[[1., 2.]]); - /// let s = a.mean_keepdim(1)?; - /// assert_eq!(s.to_vec2::()?, &[[0.5], [2.5]]); - /// let s = a.mean_keepdim((0, 1))?; - /// assert_eq!(s.to_vec2::()?, &[[1.5]]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn mean_keepdim(&self, mean_dims: D) -> Result { - let mean_dims = mean_dims.to_indexes(self.shape(), "mean-keepdim")?; - let reduced_dim: usize = mean_dims.iter().map(|i| self.dims()[*i]).product(); - let scale = 1f64 / (reduced_dim as f64); - self.sum_impl(mean_dims, true)? * scale - } - - /// Returns the mean of all elements in the input tensor. The mean is performed over all the - /// input dimensions and compared to `mean_keepdim` these dimensions are squeezed rather than - /// kept. - pub fn mean(&self, mean_dims: D) -> Result { - let mean_dims = mean_dims.to_indexes(self.shape(), "mean")?; - let reduced_dim: usize = mean_dims.iter().map(|i| self.dims()[*i]).product(); - let scale = 1f64 / (reduced_dim as f64); - self.sum_impl(mean_dims, false)? * scale - } - - /// Returns the unbiased variance over the selected dimension. - pub fn var_keepdim(&self, dim: D) -> Result { - let dim = dim.to_index(self.shape(), "var")?; - let mean = self.mean_keepdim(dim)?; - let squares = self.broadcast_sub(&mean)?.sqr()?; - squares.sum_impl(dim, true)? / (self.dim(dim)? - 1) as f64 - } - - /// Returns the unbiased variance over the selected dimension. - pub fn var(&self, dim: D) -> Result { - let dim = dim.to_index(self.shape(), "var")?; - self.var_keepdim(dim)?.squeeze(dim) - } - - /// Gathers the maximum value across the selected dimension. The resulting shape has the same - /// number of dimensions as the original tensor and the select dimension has a single element. - pub fn max_keepdim(&self, dim: D) -> Result { - self.reduce_impl(dim, true, ReduceOp::Max) - } - - /// Similar to `max_keepdim` but the target dimension is squeezed. - pub fn max(&self, dim: D) -> Result { - self.reduce_impl(dim, false, ReduceOp::Max) - } - - /// Gathers the minimum value across the selected dimension. The resulting shape has the same - /// number of dimensions as the original tensor and the select dimension has a single element. - pub fn min_keepdim(&self, dim: D) -> Result { - self.reduce_impl(dim, true, ReduceOp::Min) - } - - /// Similar to `min_keepdim` but the target dimension is squeezed. - pub fn min(&self, dim: D) -> Result { - self.reduce_impl(dim, false, ReduceOp::Min) - } - - pub fn argmax_keepdim(&self, dim: D) -> Result { - self.reduce_impl(dim, true, ReduceOp::ArgMax) - } - - /// Similar to `argmax_keepdim` but the target dimension is squeezed. - pub fn argmax(&self, dim: D) -> Result { - self.reduce_impl(dim, false, ReduceOp::ArgMax) - } - - pub fn argmin_keepdim(&self, dim: D) -> Result { - self.reduce_impl(dim, true, ReduceOp::ArgMin) - } - - /// Similar to `argmin_keepdim` but the target dimension is squeezed. - pub fn argmin(&self, dim: D) -> Result { - self.reduce_impl(dim, false, ReduceOp::ArgMin) - } - - /// Element-wise comparison between two tensors, e.g. equality, greater than, ... The actual - /// comparison operation is specified by the `op` argument. - /// - /// The returned tensor has the same shape as the original tensors and uses `u8` elements. - pub fn cmp(&self, rhs: T, op: CmpOp) -> Result { - let rhs = match rhs.to_tensor_scalar()? { - crate::scalar::TensorScalar::Tensor(rhs) => rhs, - crate::scalar::TensorScalar::Scalar(rhs) => rhs - .to_dtype(self.dtype())? - .to_device(self.device())? - .broadcast_as(self.shape())?, - }; - let shape = self.same_shape_binary_op(&rhs, "cmp")?; - let storage = self - .storage() - .cmp(op, &rhs.storage(), self.layout(), rhs.layout())?; - let op = BackpropOp::new1(self, |a| Op::Cmp(a, op)); - Ok(from_storage(storage, shape.dims(), op, false)) - } - - /// Element-wise equality. - pub fn eq(&self, rhs: T) -> Result { - self.cmp(rhs, CmpOp::Eq) - } - - /// Element-wise non-equality. - pub fn ne(&self, rhs: T) -> Result { - self.cmp(rhs, CmpOp::Ne) - } - - /// Element-wise comparison with lower-than, the returned tensor uses value 1 where `self < - /// rhs` and 0 otherwise. - pub fn lt(&self, rhs: T) -> Result { - self.cmp(rhs, CmpOp::Lt) - } - - /// Element-wise comparison with greater-than, the returned tensor uses value 1 where `self > - /// rhs` and 0 otherwise. - pub fn gt(&self, rhs: T) -> Result { - self.cmp(rhs, CmpOp::Gt) - } - - /// Element-wise comparison with greater-equal, the returned tensor uses value 1 where `self >= - /// rhs` and 0 otherwise. - pub fn ge(&self, rhs: T) -> Result { - self.cmp(rhs, CmpOp::Ge) - } - - /// Element-wise comparison with lower-equal, the returned tensor uses value 1 where `self <= - /// rhs` and 0 otherwise. - pub fn le(&self, rhs: T) -> Result { - self.cmp(rhs, CmpOp::Le) - } - - /// Clamp the tensor values to be between `min` and `max`. - pub fn clamp(&self, min: T1, max: T2) -> Result { - self.maximum(min)?.minimum(max) - } - - /// Interpolate the input tensor to the `target_size` size, taking the value of the nearest element. - /// - /// The input tensor should have three dimensions, `(batch, channels, l)`, the returned - /// tensor also has three dimensions, `(batch, channels, target_size)`. - pub fn interpolate1d(&self, target_size: usize) -> Result { - let (n, c, _l) = self.dims3()?; - let op = BackpropOp::new1(self, |arg| Op::UpsampleNearest1D { arg, target_size }); - let storage = self - .storage() - .upsample_nearest1d(self.layout(), target_size)?; - Ok(from_storage(storage, (n, c, target_size), op, false)) - } - - /// Alias for `interpolate1d`. - pub fn upsample_nearest1d(&self, target_size: usize) -> Result { - self.interpolate1d(target_size) - } - - /// Interpolate the input tensor to the `(target_h, target_w)` size, taking the value of the - /// nearest element. - /// - /// The input tensor should have four dimensions, `(batch, channels, h, w)`, the returned - /// tensor also has four dimensions, `(batch, channels, target_h, target_w)`. - pub fn interpolate2d(&self, target_h: usize, target_w: usize) -> Result { - let (n, c, _h, _w) = self.dims4()?; - let op = BackpropOp::new1(self, |arg| Op::UpsampleNearest2D { - arg, - target_h, - target_w, - }); - let storage = self - .storage() - .upsample_nearest2d(self.layout(), target_h, target_w)?; - Ok(from_storage(storage, (n, c, target_h, target_w), op, false)) - } - - /// Alias for `interpolate2d`. - pub fn upsample_nearest2d(&self, target_h: usize, target_w: usize) -> Result { - self.interpolate2d(target_h, target_w) - } - - /// 2D average pooling over an input tensor with multiple channels. - /// - /// The input tensor should have four dimensions, `(batch, channels, h, w)`, the returned - /// tensor also has four dimensions, `(batch, channels, h', w')`. The pooling is performed on - /// the two last dimensions using a kernel of size `sz`. The returned element is the average - /// value over the kernel window. - pub fn avg_pool2d(&self, sz: T) -> Result { - let sz = sz.to_usize2(); - self.avg_pool2d_with_stride(sz, sz) - } - - /// Same as `avg_pool2d` but with a `stride` that can be set to a value different from the - /// kernel size. - pub fn avg_pool2d_with_stride( - &self, - kernel_size: T, - stride: T, - ) -> Result { - let kernel_size = kernel_size.to_usize2(); - let stride = stride.to_usize2(); - let (n, c, h, w) = self.dims4()?; - if h < kernel_size.0 || w < kernel_size.1 { - bail!("kernel-size {kernel_size:?} is larger than the input size {h},{w}") - } - // https://pytorch.org/docs/stable/generated/torch.nn.AvgPool2d.html#torch.nn.AvgPool2d - let h_out = (h - kernel_size.0) / stride.0 + 1; - let w_out = (w - kernel_size.1) / stride.1 + 1; - let op = BackpropOp::new1(self, |arg| Op::AvgPool2D { - arg, - kernel_size, - stride, - }); - let storage = self - .storage() - .avg_pool2d(self.layout(), kernel_size, stride)?; - Ok(from_storage(storage, (n, c, h_out, w_out), op, false)) - } - - /// 2D max pooling over an input tensor with multiple channels. - /// - /// The input tensor should have four dimensions, `(batch, channels, h, w)`, the returned - /// tensor also has four dimensions, `(batch, channels, h', w')`. The pooling is performed on - /// the two last dimensions using a kernel of size `sz`, the returned element is the maximum - /// value over the kernel window. - pub fn max_pool2d(&self, sz: T) -> Result { - let sz = sz.to_usize2(); - self.max_pool2d_with_stride(sz, sz) - } - - /// Same as `max_pool2d` but with a `stride` that can be set to a value different from the - /// kernel size. - pub fn max_pool2d_with_stride( - &self, - kernel_size: T, - stride: T, - ) -> Result { - let kernel_size = kernel_size.to_usize2(); - let stride = stride.to_usize2(); - let (n, c, h, w) = self.dims4()?; - if h < kernel_size.0 || w < kernel_size.1 { - bail!("kernel-size {kernel_size:?} is larger than the input size {h},{w}") - } - // https://pytorch.org/docs/stable/generated/torch.nn.MaxPool2d.html#torch.nn.MaxPool2d - let h_out = (h - kernel_size.0) / stride.0 + 1; - let w_out = (w - kernel_size.1) / stride.1 + 1; - let op = BackpropOp::new1(self, |arg| Op::MaxPool2D { - arg, - kernel_size, - stride, - }); - let storage = self - .storage() - .max_pool2d(self.layout(), kernel_size, stride)?; - Ok(from_storage(storage, (n, c, h_out, w_out), op, false)) - } - - /// Returns the matrix-multiplication of the input tensor with the other provided tensor. - /// - /// # Arguments - /// - /// * `self` - A tensor with dimensions `b1, b2, ..., bi, m, k`. - /// * `rhs` - A tensor with dimensions `b1, b2, ..., bi, k, n`. - /// - /// The resulting tensor has dimensions `b1, b2, ..., bi, m, n`. - pub fn matmul(&self, rhs: &Self) -> Result { - let a_dims = self.shape().dims(); - let b_dims = rhs.shape().dims(); - - let dim = a_dims.len(); - - if dim < 2 || b_dims.len() != dim { - Err(Error::ShapeMismatchBinaryOp { - lhs: self.shape().clone(), - rhs: rhs.shape().clone(), - op: "matmul", - } - .bt())? - } - - let m = a_dims[dim - 2]; - let k = a_dims[dim - 1]; - let k2 = b_dims[dim - 2]; - let n = b_dims[dim - 1]; - - let c_shape = Shape::from(&a_dims[..dim - 2]).extend(&[m, n]); - let batching: usize = a_dims[..dim - 2].iter().product(); - let batching_b: usize = b_dims[..dim - 2].iter().product(); - if k != k2 || batching != batching_b { - Err(Error::ShapeMismatchBinaryOp { - lhs: self.shape().clone(), - rhs: rhs.shape().clone(), - op: "matmul", - } - .bt())? - } - - let storage = self.storage().matmul( - &rhs.storage(), - (batching, m, n, k), - self.layout(), - rhs.layout(), - )?; - let op = BackpropOp::new2(self, rhs, Op::Matmul); - Ok(from_storage(storage, c_shape, op, false)) - } - - /// Matrix-multiplication with broadcasting support. - /// - /// Compared to `matmul` the two matrixes are allowed to have different dimensions as long as - /// they are compatible for broadcast. E.g. if `self` has shape `(j, 1, n, k)` and `rhs` has - /// shape `(l, k, m)`, the output will have shape `(j, l, n, m)`. - pub fn broadcast_matmul(&self, rhs: &Self) -> Result { - let lhs = self; - let (l_shape, r_shape) = lhs.shape().broadcast_shape_matmul(rhs.shape())?; - let l_broadcast = l_shape != *lhs.shape(); - let r_broadcast = r_shape != *rhs.shape(); - // TODO: Avoid concretising the broadcasted matrixes via contiguous. - match (l_broadcast, r_broadcast) { - (true, true) => lhs - .broadcast_as(&l_shape)? - .contiguous()? - .matmul(&rhs.broadcast_as(&r_shape)?.contiguous()?), - (false, true) => lhs.matmul(&rhs.broadcast_as(&r_shape)?.contiguous()?), - (true, false) => lhs.broadcast_as(&l_shape)?.contiguous()?.matmul(rhs), - (false, false) => lhs.matmul(rhs), - } - } - - /// Returns a tensor with the same shape as the input tensor, the values are taken from - /// `on_true` if the input tensor value is not zero, and `on_false` at the positions where the - /// input tensor is equal to zero. - pub fn where_cond(&self, on_true: &Self, on_false: &Self) -> Result { - let _shap = self.same_shape_binary_op(on_true, "where_cond")?; - let shape = self.same_shape_binary_op(on_false, "where_cond")?; - let storage = self.storage().where_cond( - self.layout(), - &on_true.storage(), - on_true.layout(), - &on_false.storage(), - on_false.layout(), - )?; - let op = BackpropOp::new3(self, on_true, on_false, Op::WhereCond); - Ok(from_storage(storage, shape, op, false)) - } - - /// Returns a tensor with the values from the `self` tensor at the index corresponding to the - /// values hold in the `ids` tensor. - /// - /// # Arguments - /// - /// * `self` - A tensor with dimensions `v, h`. - /// * `ids` - A tensor with dimensions `s` and with integer values between 0 and v (exclusive). - /// - /// The resulting tensor has dimensions `s, h`. `s` is called the sequence length, `v` the - /// vocabulary size, and `h` the hidden size. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let values = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; - /// let ids = Tensor::new(&[2u32, 1u32, 2u32], &Device::Cpu)?; - /// let emb = values.embedding(&ids)?; - /// assert_eq!(emb.to_vec2::()?, &[[4., 5.], [2., 3.], [4., 5.]]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn embedding(&self, ids: &Self) -> Result { - if self.rank() != 2 || ids.rank() != 1 { - Err(Error::ShapeMismatchBinaryOp { - lhs: self.shape().clone(), - rhs: ids.shape().clone(), - op: "embedding", - } - .bt())? - } - self.index_select(ids, 0) - } - - pub fn scatter_add(&self, indexes: &Self, source: &Self, dim: D) -> Result { - let dim = dim.to_index(self.shape(), "scatter-add")?; - let source_dims = source.dims(); - let self_dims = self.dims(); - let mismatch = if source_dims.len() != self_dims.len() { - true - } else { - let mut mismatch = false; - for (i, (&d1, &d2)) in self_dims.iter().zip(source_dims.iter()).enumerate() { - if i != dim && d1 != d2 { - mismatch = true; - break; - } - } - mismatch - }; - if mismatch { - Err(Error::ShapeMismatchBinaryOp { - op: "scatter-add (self, src)", - lhs: self.shape().clone(), - rhs: source.shape().clone(), - } - .bt())? - } - if indexes.dims() != source.dims() { - Err(Error::ShapeMismatchBinaryOp { - op: "scatter-add (indexes, src)", - lhs: indexes.shape().clone(), - rhs: source.shape().clone(), - } - .bt())? - } - let storage = self.storage().scatter_add( - self.layout(), - &indexes.storage(), - indexes.layout(), - &source.storage(), - source.layout(), - dim, - )?; - let op = BackpropOp::new3(self, indexes, source, |t1, t2, t3| { - Op::ScatterAdd(t1, t2, t3, dim) - }); - Ok(from_storage(storage, self.shape(), op, false)) - } - - /// Embeds the values of the `src` tensor into the `self` tensor on the specified dimension. - pub fn slice_scatter(&self, src: &Self, dim: D, start: usize) -> Result { - let dim = dim.to_index(self.shape(), "slice-scatter")?; - if dim == 0 { - self.slice_scatter0(src, start) - } else { - // TODO: Maybe we want to add a more efficient implementation at some point. - self.transpose(0, dim)? - .slice_scatter0(&src.transpose(0, dim)?, start)? - .transpose(0, dim) - } - } - - /// Embeds the values of the `src` tensor into the `self` tensor on the first dimension. - pub fn slice_scatter0(&self, src: &Self, start: usize) -> Result { - if self.dtype() != src.dtype() { - Err(Error::DTypeMismatchBinaryOp { - lhs: self.dtype(), - rhs: src.dtype(), - op: "slice-scatter", - } - .bt())? - } - if self.device().location() != src.device.location() { - Err(Error::DeviceMismatchBinaryOp { - lhs: self.device().location(), - rhs: src.device().location(), - op: "slice-scatter", - } - .bt())? - } - if self.rank() != src.rank() { - Err(Error::UnexpectedNumberOfDims { - expected: self.rank(), - got: src.rank(), - shape: src.shape().clone(), - } - .bt())? - } - let shape_ok = - self.dims() - .iter() - .zip(src.dims().iter()) - .enumerate() - .all(|(dim_idx, (&d1, &d2))| { - if 0 == dim_idx { - d2 + start <= d1 - } else { - d1 == d2 - } - }); - if !shape_ok { - Err(Error::ShapeMismatchBinaryOp { - op: "slice-scatter (self, src)", - lhs: self.shape().clone(), - rhs: src.shape().clone(), - } - .bt())? - } - let mut storage = self.device().zeros(self.shape(), self.dtype())?; - self.storage() - .copy_strided_src(&mut storage, 0, self.layout())?; - let offset = start * src.dims()[1..].iter().product::(); - src.storage() - .copy_strided_src(&mut storage, offset, src.layout())?; - let op = BackpropOp::new2(self, src, |t1, t2| Op::SliceScatter0(t1, t2, start)); - Ok(from_storage(storage, self.shape(), op, false)) - } - - /// Accumulate element from `source` at indexes `indexes` and add them to `self`. - pub fn index_add(&self, indexes: &Self, source: &Self, dim: D) -> Result { - let dim = dim.to_index(self.shape(), "index-add")?; - let source_dims = source.dims(); - let self_dims = self.dims(); - let mismatch = if source_dims.len() != self_dims.len() { - true - } else { - let mut mismatch = false; - for (i, (&d1, &d2)) in self_dims.iter().zip(source_dims.iter()).enumerate() { - if i != dim && d1 != d2 { - mismatch = true; - break; - } - } - mismatch - }; - if mismatch { - Err(Error::ShapeMismatchBinaryOp { - op: "index-add (self, source)", - lhs: self.shape().clone(), - rhs: source.shape().clone(), - } - .bt())? - } - // The number of element in indexes must match the dimension on which the add is - // performed on the source tensor (and the index values from `indexes` are taken from - // the target tensor self) - let indexes_len = indexes.dims1()?; - if source_dims[dim] != indexes_len { - Err(Error::ShapeMismatchBinaryOp { - op: "index-add (ids, source))", - lhs: indexes.shape().clone(), - rhs: source.shape().clone(), - } - .bt())? - } - let storage = self.storage().index_add( - self.layout(), - &indexes.storage(), - indexes.layout(), - &source.storage(), - source.layout(), - dim, - )?; - let op = BackpropOp::new3(self, indexes, source, |t1, t2, t3| { - Op::IndexAdd(t1, t2, t3, dim) - }); - Ok(from_storage(storage, self.shape(), op, false)) - } - - /// Gather values across the target dimension. - /// - /// # Arguments - /// - /// * `self` - The input tensor. - /// * `indexes` - The indices of elements to gather, this should have the same shape as `self` - /// but can have a different number of elements on the target dimension. - /// * `dim` - the target dimension. - /// - /// The resulting tensor has the same shape as `indexes` and use values from `self` indexed on - /// dimension `dim` by the values in `indexes`. - pub fn gather(&self, indexes: &Self, dim: D) -> Result { - let dim = dim.to_index(self.shape(), "gather")?; - let self_dims = self.dims(); - let indexes_dims = indexes.dims(); - let mismatch = if indexes_dims.len() != self_dims.len() { - true - } else { - let mut mismatch = false; - for (i, (&d1, &d2)) in self_dims.iter().zip(indexes_dims.iter()).enumerate() { - if i != dim && d1 != d2 { - mismatch = true; - break; - } - } - mismatch - }; - if mismatch { - Err(Error::ShapeMismatchBinaryOp { - op: "gather", - lhs: self.shape().clone(), - rhs: indexes.shape().clone(), - } - .bt())? - } - let storage = - self.storage() - .gather(self.layout(), &indexes.storage(), indexes.layout(), dim)?; - let op = BackpropOp::new2(self, indexes, |t1, t2| Op::Gather(t1, t2, dim)); - Ok(from_storage(storage, indexes.shape(), op, false)) - } - - /// Select values for the input tensor at the target indexes across the specified dimension. - /// - /// The `indexes` is argument is an int tensor with a single dimension. - /// The output has the same number of dimension as the `self` input. The target dimension of - /// the output has length the length of `indexes` and the values are taken from `self` using - /// the index from `indexes`. Other dimensions have the same number of elements as the input - /// tensor. - pub fn index_select(&self, indexes: &Self, dim: D) -> Result { - let dim = dim.to_index(self.shape(), "index-select")?; - let indexes_len = match indexes.dims() { - [l] => *l, - _ => Err(Error::ShapeMismatchBinaryOp { - lhs: self.shape().clone(), - rhs: indexes.shape().clone(), - op: "index-select", - } - .bt())?, - }; - let storage = self.storage().index_select( - &indexes.storage(), - self.layout(), - indexes.layout(), - dim, - )?; - let mut dims = self.dims().to_vec(); - dims[dim] = indexes_len; - let op = BackpropOp::new2(self, indexes, |t1, t2| Op::IndexSelect(t1, t2, dim)); - Ok(from_storage(storage, dims, op, false)) - } - - /// Returns an iterator over position of the elements in the storage when ranging over the - /// index tuples in lexicographic order. - pub fn strided_index(&self) -> crate::StridedIndex { - self.layout.strided_index() - } - - /// Similar to `strided_index` but returns the position of the start of each contiguous block - /// as well as the length of the contiguous blocks. For a contiguous tensor, the index iterator - /// will only return the start offset and the size would be the number of elements in the - /// tensor. - pub fn strided_blocks(&self) -> crate::StridedBlocks { - self.layout.strided_blocks() - } - - /// Returns the data contained in a 1D tensor as a vector of scalar values. - pub fn to_vec1(&self) -> Result> { - if self.rank() != 1 { - Err(Error::UnexpectedNumberOfDims { - expected: 1, - got: self.rank(), - shape: self.shape().clone(), - } - .bt())? - } - let from_cpu_storage = |cpu_storage: &crate::CpuStorage| { - let data = S::cpu_storage_as_slice(cpu_storage)?; - let data = match self.layout.contiguous_offsets() { - Some((o1, o2)) => data[o1..o2].to_vec(), - None => self.strided_index().map(|i| data[i]).collect(), - }; - Ok::, Error>(data) - }; - match &*self.storage() { - Storage::Cpu(storage) => from_cpu_storage(storage), - Storage::Cuda(storage) => from_cpu_storage(&storage.to_cpu_storage()?), - Storage::Metal(storage) => from_cpu_storage(&storage.to_cpu_storage()?), - } - } - - /// Returns the data contained in a 2D tensor as a vector of vector of scalar values. - pub fn to_vec2(&self) -> Result>> { - let (dim1, dim2) = self.dims2()?; - let from_cpu_storage = |cpu_storage: &crate::CpuStorage| { - let data = S::cpu_storage_as_slice(cpu_storage)?; - let mut rows = vec![]; - match self.layout.contiguous_offsets() { - Some((o1, o2)) => { - let data = &data[o1..o2]; - for idx_row in 0..dim1 { - rows.push(data[idx_row * dim2..(idx_row + 1) * dim2].to_vec()) - } - } - None => { - let mut src_index = self.strided_index(); - for _idx_row in 0..dim1 { - let row = (0..dim2).map(|_| data[src_index.next().unwrap()]).collect(); - rows.push(row) - } - assert!(src_index.next().is_none()); - } - } - Ok(rows) - }; - match &*self.storage() { - Storage::Cpu(storage) => from_cpu_storage(storage), - Storage::Cuda(storage) => from_cpu_storage(&storage.to_cpu_storage()?), - Storage::Metal(storage) => from_cpu_storage(&storage.to_cpu_storage()?), - } - } - - /// Returns the data contained in a 3D tensor. - pub fn to_vec3(&self) -> Result>>> { - let (dim1, dim2, dim3) = self.dims3()?; - let from_cpu_storage = |cpu_storage: &crate::CpuStorage| { - let data = S::cpu_storage_as_slice(cpu_storage)?; - let mut top_rows = vec![]; - match self.layout.contiguous_offsets() { - Some((o1, o2)) => { - let data = &data[o1..o2]; - let dim23 = dim2 * dim3; - for idx1 in 0..dim1 { - let data = &data[idx1 * dim23..(idx1 + 1) * dim23]; - let mut rows = vec![]; - for idx2 in 0..dim2 { - rows.push(data[idx2 * dim3..(idx2 + 1) * dim3].to_vec()) - } - top_rows.push(rows); - } - } - None => { - let mut src_index = self.strided_index(); - for _idx in 0..dim1 { - let mut rows = vec![]; - for _jdx in 0..dim2 { - let row = (0..dim3).map(|_| data[src_index.next().unwrap()]).collect(); - rows.push(row) - } - top_rows.push(rows); - } - assert!(src_index.next().is_none()); - } - } - Ok(top_rows) - }; - match &*self.storage() { - Storage::Cpu(storage) => from_cpu_storage(storage), - Storage::Cuda(storage) => from_cpu_storage(&storage.to_cpu_storage()?), - Storage::Metal(storage) => from_cpu_storage(&storage.to_cpu_storage()?), - } - } - - /// The dtype for the elements stored in the input tensor. - pub fn dtype(&self) -> DType { - self.dtype - } - - /// The device on which the input tensor is located. - pub fn device(&self) -> &Device { - &self.device - } - - /// The tensor shape, i.e. dimension sizes on each axis. - pub fn shape(&self) -> &Shape { - self.layout().shape() - } - - /// The dimension size for this tensor on each axis. - pub fn dims(&self) -> &[usize] { - self.shape().dims() - } - - /// The dimension size for a specified dimension index. - pub fn dim(&self, dim: D) -> Result { - let dim = dim.to_index(self.shape(), "dim")?; - Ok(self.dims()[dim]) - } - - /// The layout of the input tensor, this stores both the shape of the tensor as well as the - /// strides and the start offset to apply to the underlying storage. - pub fn layout(&self) -> &Layout { - &self.layout - } - - pub fn stride(&self) -> &[usize] { - self.layout.stride() - } - - /// The number of dimensions for this tensor, 0 for a scalar tensor, 1 for a 1D tensor, etc. - pub fn rank(&self) -> usize { - self.shape().rank() - } - - /// The number of elements stored in this tensor. - pub fn elem_count(&self) -> usize { - self.shape().elem_count() - } - - /// The unique identifier for this tensor. - pub fn id(&self) -> TensorId { - self.id - } - - /// Whether this tensor is a variable or not. A variable is a tensor for which gradient is - /// tracked and on which backpropagation can be performed. - pub fn is_variable(&self) -> bool { - self.is_variable - } - - pub(crate) fn op(&self) -> &Option { - &self.op - } - - /// Computes the sum of all the elements in this tensor and returns a tensor holding this - /// scalar with zero dimensions. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; - /// let tensor = tensor.sum_all()?; - /// assert_eq!(tensor.to_scalar::()?, 15.); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn sum_all(&self) -> Result { - let dims: Vec<_> = (0..self.rank()).collect(); - self.sum(dims) - } - - pub fn mean_all(&self) -> Result { - self.sum_all()? / self.elem_count() as f64 - } - - fn flatten_( - &self, - start_dim: Option, - end_dim: Option, - ) -> Result { - if self.rank() == 0 { - self.reshape(1) - } else { - let start_dim = match start_dim { - None => 0, - Some(dim) => dim.to_index(self.shape(), "flatten")?, - }; - let end_dim = match end_dim { - None => self.rank() - 1, - Some(dim) => dim.to_index(self.shape(), "flatten")?, - }; - if start_dim < end_dim { - let dims = self.dims(); - let mut dst_dims = dims[..start_dim].to_vec(); - dst_dims.push(dims[start_dim..end_dim + 1].iter().product::()); - if end_dim + 1 < dims.len() { - dst_dims.extend(&dims[end_dim + 1..]); - } - self.reshape(dst_dims) - } else { - Ok(self.clone()) - } - } - } - - /// Flattens the input tensor on the dimension indexes from `start_dim` to `end_dim` (both - /// inclusive). - pub fn flatten(&self, start_dim: D1, end_dim: D2) -> Result { - self.flatten_(Some(start_dim), Some(end_dim)) - } - - /// Flattens the input tensor on the dimension indexes from `0` to `end_dim` (inclusive). - pub fn flatten_to(&self, end_dim: D) -> Result { - self.flatten_(None::, Some(end_dim)) - } - - /// Flattens the input tensor on the dimension indexes from `start_dim` (inclusive) to the last - /// dimension. - pub fn flatten_from(&self, start_dim: D) -> Result { - self.flatten_(Some(start_dim), None::) - } - - /// Flattens the input tensor by reshaping it into a one dimension tensor. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; - /// let tensor = tensor.flatten_all()?; - /// assert_eq!(tensor.to_vec1::()?, &[0., 1., 2., 3., 4., 5.]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn flatten_all(&self) -> Result { - self.flatten_(None::, None::) - } - - /// Returns the sub-tensor fixing the index at `i` on the first dimension. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; - /// let t = tensor.get(0)?; - /// assert_eq!(t.to_vec1::()?, &[0., 1.]); - /// let t = tensor.get(1)?; - /// assert_eq!(t.to_vec1::()?, &[2., 3.]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn get(&self, i: usize) -> Result { - let dims = self.dims(); - if dims.is_empty() { - Ok(self.clone()) - } else { - self.narrow(0, i, 1)?.reshape(&dims[1..]) - } - } - - /// Returns the sub-tensor fixing the index at `index` on the dimension `dim`. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; - /// let t = tensor.get_on_dim(1, 0)?; - /// assert_eq!(t.to_vec1::()?, &[0., 2., 4.]); - /// let t = tensor.get_on_dim(1, 1)?; - /// assert_eq!(t.to_vec1::()?, &[1., 3., 5.]); - /// let t = tensor.get_on_dim(0, 1)?; - /// assert_eq!(t.to_vec1::()?, &[2., 3.]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn get_on_dim(&self, dim: D, index: usize) -> Result { - let dim = dim.to_index(self.shape(), "get_on_dim")?; - self.narrow(dim, index, 1)?.squeeze(dim) - } - - /// Returns a tensor that is a transposed version of the input, the two last dimensions of the - /// input are swapped. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; - /// let tensor = tensor.t()?; - /// assert_eq!(tensor.to_vec2::()?, &[[0.0, 2.0, 4.0], [1.0, 3.0, 5.0]]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn t(&self) -> Result { - let rank = self.rank(); - if rank < 2 { - Err(Error::UnexpectedNumberOfDims { - expected: 2, - got: rank, - shape: self.shape().clone(), - } - .bt())? - } - self.transpose(rank - 2, rank - 1) - } - - /// Returns a tensor that is a transposed version of the input, the given dimensions are - /// swapped. - pub fn transpose(&self, dim1: D1, dim2: D2) -> Result { - let dim1 = dim1.to_index(self.shape(), "transpose")?; - let dim2 = dim2.to_index(self.shape(), "transpose")?; - if dim1 == dim2 { - return Ok(self.clone()); - } - let op = BackpropOp::new1(self, |t| Op::Transpose(t, dim1, dim2)); - let tensor_ = Tensor_ { - id: TensorId::new(), - storage: self.storage.clone(), - layout: self.layout.transpose(dim1, dim2)?, - op, - is_variable: false, - dtype: self.dtype, - device: self.device.clone(), - }; - Ok(Tensor(Arc::new(tensor_))) - } - - /// Returns a tensor with the same data as the input where the dimensions have been permuted. - /// dims must be a permutation, i.e. include each dimension index exactly once. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let tensor = Tensor::arange(0u32, 120u32, &Device::Cpu)?.reshape((2, 3, 4, 5))?; - /// assert_eq!(tensor.dims(), &[2, 3, 4, 5]); - /// let tensor = tensor.permute((2, 3, 1, 0))?; - /// assert_eq!(tensor.dims(), &[4, 5, 3, 2]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn permute(&self, dims: D) -> Result { - let dims = dims.to_indexes(self.shape(), "permute")?; - // O(n^2) permutation check but these arrays are small. - let is_permutation = - dims.len() == self.rank() && (0..dims.len()).all(|i| dims.contains(&i)); - if !is_permutation { - bail!( - "dimension mismatch in permute, tensor {:?}, dims: {:?}", - self.dims(), - dims - ) - } - let op = BackpropOp::new1(self, |t| Op::Permute(t, dims.clone())); - let tensor_ = Tensor_ { - id: TensorId::new(), - storage: self.storage.clone(), - layout: self.layout.permute(&dims)?, - op, - is_variable: false, - dtype: self.dtype, - device: self.device.clone(), - }; - Ok(Tensor(Arc::new(tensor_))) - } - - /// Returns true if the data is stored in a C contiguous (aka row major) way. - pub fn is_contiguous(&self) -> bool { - self.layout.is_contiguous() - } - - /// Returns true if the data is stored in a Fortran contiguous (aka column major) way. - pub fn is_fortran_contiguous(&self) -> bool { - self.layout.is_fortran_contiguous() - } - - /// Compared to clone, this copies the actual storage but may fail because of running out of - /// memory. - pub fn copy(&self) -> Result { - let op = BackpropOp::new1(self, Op::Copy); - let tensor_ = Tensor_ { - id: TensorId::new(), - storage: Arc::new(RwLock::new(self.storage().try_clone(self.layout())?)), - layout: self.layout.clone(), - op, - is_variable: false, - dtype: self.dtype, - device: self.device.clone(), - }; - Ok(Tensor(Arc::new(tensor_))) - } - - /// Returns a new tensor detached from the current graph, gradient are not propagated through - /// this new node. The storage of this tensor is shared with the initial tensor. - /// - /// If the tensor is already detached from the computation graph, the same tensor is returned. - pub fn detach(&self) -> Tensor { - if self.op.is_none() && !self.is_variable { - self.clone() - } else { - let tensor_ = Tensor_ { - id: TensorId::new(), - storage: self.storage.clone(), - layout: self.layout.clone(), - op: BackpropOp::none(), - is_variable: false, - dtype: self.dtype, - device: self.device.clone(), - }; - Tensor(Arc::new(tensor_)) - } - } - - /// If the target device is the same as the tensor device, only a shallow copy is performed. - pub fn to_device(&self, device: &Device) -> Result { - if self.device().same_device(device) { - Ok(self.clone()) - } else { - let storage = match (&*self.storage(), device) { - (Storage::Cpu(storage), Device::Cuda(cuda)) => { - Storage::Cuda(cuda.storage_from_cpu_storage(storage)?) - } - (Storage::Cpu(storage), Device::Metal(metal)) => { - Storage::Metal(metal.storage_from_cpu_storage(storage)?) - } - (Storage::Cuda(storage), Device::Cpu) => Storage::Cpu(storage.to_cpu_storage()?), - (Storage::Metal(storage), Device::Cpu) => Storage::Cpu(storage.to_cpu_storage()?), - (Storage::Cuda(storage), Device::Cuda(cuda)) => { - // TODO: Avoid passing through the cpu storage here, especially if the gpu ids - // are the same. - let cpu_storage = storage.to_cpu_storage()?; - Storage::Cuda(cuda.storage_from_cpu_storage(&cpu_storage)?) - } - (Storage::Cpu(storage), Device::Cpu) => Storage::Cpu(storage.clone()), - _ => { - bail!("not implemented yet") - } - }; - let op = BackpropOp::new1(self, Op::ToDevice); - let tensor_ = Tensor_ { - id: TensorId::new(), - storage: Arc::new(RwLock::new(storage)), - layout: self.layout.clone(), - op, - is_variable: false, - dtype: self.dtype, - device: device.clone(), - }; - Ok(Tensor(Arc::new(tensor_))) - } - } - - /// Returns a new tensor duplicating data from the original tensor. New dimensions are inserted - /// on the left. - pub fn broadcast_left>(&self, left_shape: S) -> Result { - let left_shape = left_shape.into(); - let mut dims = left_shape.into_dims(); - dims.extend(self.dims()); - self.broadcast_as(dims) - } - - /// Broadcast the input tensor to the target shape. This returns an error if the input shape is - /// not compatible with the target shape. - /// - /// If the input shape is `i_1, i_2, ... i_k`, the target shape has to have `k` dimensions or - /// more and shape `j_1, ..., j_l, t_1, t_2, ..., t_k`. The dimensions `j_1` to `j_l` can have - /// any value, the dimension `t_a` must be equal to `i_a` if `i_a` is different from 1. If - /// `i_a` is equal to 1, any value can be used. - pub fn broadcast_as>(&self, shape: S) -> Result { - let tensor_ = Tensor_ { - id: TensorId::new(), - storage: self.storage.clone(), - layout: self.layout.broadcast_as(shape)?, - op: BackpropOp::new1(self, Op::Broadcast), - is_variable: false, - dtype: self.dtype, - device: self.device.clone(), - }; - Ok(Tensor(Arc::new(tensor_))) - } - - /// An alias for broadcast_as. - pub fn expand>(&self, shape: S) -> Result { - self.broadcast_as(shape) - } - - /// Casts the input tensor to the target `dtype`. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let tensor = Tensor::new(3.14159265358979f64, &Device::Cpu)?; - /// assert_eq!(tensor.to_scalar::()?, 3.14159265358979); - /// let tensor = tensor.to_dtype(candle_core::DType::F32)?; - /// assert_eq!(tensor.to_scalar::()?, 3.1415927); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn to_dtype(&self, dtype: DType) -> Result { - if self.dtype() == dtype { - Ok(self.clone()) - } else { - let shape = self.shape(); - let storage = self.storage().to_dtype(self.layout(), dtype)?; - let op = BackpropOp::new1(self, Op::ToDType); - Ok(from_storage(storage, shape.clone(), op, false)) - } - } - - /// Returns a tensor that is in row major order. This is the same as the original tensor if it - /// was already contiguous, otherwise a copy is triggered. - pub fn contiguous(&self) -> Result { - if self.is_contiguous() { - Ok(self.clone()) - } else { - let shape = self.shape(); - let mut storage = self.device().zeros(shape, self.dtype())?; - self.storage() - .copy_strided_src(&mut storage, 0, self.layout())?; - let op = BackpropOp::new1(self, Op::Copy); - Ok(from_storage(storage, shape.clone(), op, false)) - } - } - - /// Create a variable based on the values currently stored in a tensor. The storage is always - /// copied. - pub(crate) fn make_var(&self) -> Result { - let shape = self.shape().clone(); - let mut storage = self.device().zeros(&shape, self.dtype())?; - self.storage() - .copy_strided_src(&mut storage, 0, self.layout())?; - Ok(from_storage(storage, shape, BackpropOp::none(), true)) - } - - /// Reshape returns a tensor with the target shape provided that the number of elements of the - /// original tensor is the same. - /// If the input tensor is contiguous, this is a view on the original data. Otherwise this uses - /// a new storage and copies the data over, the returned tensor is always contiguous. - /// - /// The shape can be specified using a tuple of `usize` and at most one `()` in which case - /// the behavior is the same as when using `-1` in PyTorch: this dimension size is adjusted so - /// as to match the number of elements in the tensor. - /// - /// ```rust - /// # use candle_core::{Tensor, DType, Device, D}; - /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; - /// - /// let c = a.reshape((1, 6))?; - /// assert_eq!(c.shape().dims(), &[1, 6]); - /// - /// let c = a.reshape((3, 2))?; - /// assert_eq!(c.shape().dims(), &[3, 2]); - /// - /// let c = a.reshape((2, (), 1))?; - /// assert_eq!(c.shape().dims(), &[2, 3, 1]); - /// - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn reshape(&self, s: S) -> Result { - let shape = s.into_shape(self.elem_count())?; - if shape.elem_count() != self.elem_count() { - return Err(Error::ShapeMismatchBinaryOp { - lhs: self.shape().clone(), - rhs: shape, - op: "reshape", - } - .bt()); - } - let op = BackpropOp::new1(self, Op::Reshape); - if self.is_contiguous() { - let tensor_ = Tensor_ { - id: TensorId::new(), - storage: self.storage.clone(), - layout: Layout::contiguous_with_offset(shape, self.layout.start_offset()), - op, - is_variable: false, - dtype: self.dtype, - device: self.device.clone(), - }; - Ok(Tensor(Arc::new(tensor_))) - } else { - let mut storage = self.device().zeros(&shape, self.dtype())?; - self.storage() - .copy_strided_src(&mut storage, 0, self.layout())?; - Ok(from_storage(storage, shape, op, false)) - } - } - - /// Creates a new tensor with the specified dimension removed if its size was one. - /// - /// ```rust - /// # use candle_core::{Tensor, DType, Device, D}; - /// let a = Tensor::zeros((2, 3, 1), DType::F32, &Device::Cpu)?; - /// - /// let c = a.squeeze(2)?; - /// assert_eq!(c.shape().dims(), &[2, 3]); - /// - /// let c = a.squeeze(D::Minus1)?; - /// assert_eq!(c.shape().dims(), &[2, 3]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn squeeze(&self, dim: D) -> Result { - // The PyTorch semantics are to return the same tensor if the target dimension - // does not have a size of 1. - let dims = self.dims(); - let dim = dim.to_index(self.shape(), "squeeze")?; - if dims[dim] == 1 { - let mut dims = dims.to_vec(); - dims.remove(dim); - self.reshape(dims) - } else { - Ok(self.clone()) - } - } - - /// Creates a new tensor with a dimension of size one inserted at the specified position. - /// - /// ```rust - /// # use candle_core::{Tensor, DType, Device, D}; - /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; - /// - /// let c = a.unsqueeze(0)?; - /// assert_eq!(c.shape().dims(), &[1, 2, 3]); - /// - /// let c = a.unsqueeze(D::Minus1)?; - /// assert_eq!(c.shape().dims(), &[2, 3, 1]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn unsqueeze(&self, dim: D) -> Result { - let mut dims = self.dims().to_vec(); - let dim = dim.to_index_plus_one(self.shape(), "unsqueeze")?; - // Cannot panic because to_index_plus_one already checks dimensions - dims.insert(dim, 1); - self.reshape(dims) - } - - /// Stacks two or more tensors along a particular dimension. - /// - /// All tensors must have the same rank, and the output has one additional rank - /// - /// ```rust - /// # use candle_core::{Tensor, DType, Device}; - /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; - /// let b = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; - /// - /// let c = Tensor::stack(&[&a, &b], 0)?; - /// assert_eq!(c.shape().dims(), &[2, 2, 3]); - /// - /// let c = Tensor::stack(&[&a, &b], 2)?; - /// assert_eq!(c.shape().dims(), &[2, 3, 2]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn stack, D: Dim>(args: &[A], dim: D) -> Result { - if args.is_empty() { - Err(Error::OpRequiresAtLeastOneTensor { op: "stack" }.bt())? - } - let dim = dim.to_index_plus_one(args[0].as_ref().shape(), "stack")?; - let args = args - .iter() - .map(|t| t.as_ref().unsqueeze(dim)) - .collect::>>()?; - Self::cat(&args, dim) - } - - /// Concatenates two or more tensors along a particular dimension. - /// - /// All tensors must of the same rank, and the output will have - /// the same rank - /// - /// ```rust - /// # use candle_core::{Tensor, DType, Device}; - /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; - /// let b = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; - /// - /// let c = Tensor::cat(&[&a, &b], 0)?; - /// assert_eq!(c.shape().dims(), &[4, 3]); - /// - /// let c = Tensor::cat(&[&a, &b], 1)?; - /// assert_eq!(c.shape().dims(), &[2, 6]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn cat, D: Dim>(args: &[A], dim: D) -> Result { - if args.is_empty() { - Err(Error::OpRequiresAtLeastOneTensor { op: "cat" }.bt())? - } - let arg0 = args[0].as_ref(); - if args.len() == 1 { - return Ok(arg0.clone()); - } - let dim = dim.to_index(arg0.shape(), "cat")?; - for arg in args { - arg.as_ref().check_dim(dim, "cat")?; - } - for (arg_idx, arg) in args.iter().enumerate() { - let arg = arg.as_ref(); - if arg0.rank() != arg.rank() { - Err(Error::UnexpectedNumberOfDims { - expected: arg0.rank(), - got: arg.rank(), - shape: arg.shape().clone(), - } - .bt())? - } - for (dim_idx, (v1, v2)) in arg0 - .shape() - .dims() - .iter() - .zip(arg.shape().dims().iter()) - .enumerate() - { - if dim_idx != dim && v1 != v2 { - Err(Error::ShapeMismatchCat { - dim: dim_idx, - first_shape: arg0.shape().clone(), - n: arg_idx + 1, - nth_shape: arg.shape().clone(), - } - .bt())? - } - } - } - if dim == 0 { - Self::cat0(args) - } else { - // TODO: Avoid these transpositions and have an implementation that works - // for dim != 0... - let args: Vec = args - .iter() - .map(|a| a.as_ref().transpose(0, dim)) - .collect::>>()?; - let cat = Self::cat0(&args)?; - cat.transpose(0, dim) - } - } - - fn cat0>(args: &[A]) -> Result { - if args.is_empty() { - Err(Error::OpRequiresAtLeastOneTensor { op: "cat" }.bt())? - } - let arg0 = args[0].as_ref(); - if args.len() == 1 { - return Ok(arg0.clone()); - } - let rank = arg0.rank(); - let device = arg0.device(); - let dtype = arg0.dtype(); - let first_dims = arg0.shape().dims(); - let mut cat_dims = first_dims.to_vec(); - cat_dims[0] = 0; - let mut offsets = vec![0usize]; - for (arg_idx, arg) in args.iter().enumerate() { - let arg = arg.as_ref(); - if arg.dtype() != dtype { - Err(Error::DTypeMismatchBinaryOp { - lhs: dtype, - rhs: arg.dtype(), - op: "cat", - } - .bt())? - } - if arg.device().location() != device.location() { - Err(Error::DeviceMismatchBinaryOp { - lhs: device.location(), - rhs: arg.device().location(), - op: "cat", - } - .bt())? - } - if rank != arg.rank() { - Err(Error::UnexpectedNumberOfDims { - expected: rank, - got: arg.rank(), - shape: arg.shape().clone(), - } - .bt())? - } - for (dim_idx, (v1, v2)) in arg0 - .shape() - .dims() - .iter() - .zip(arg.shape().dims().iter()) - .enumerate() - { - if dim_idx == 0 { - cat_dims[0] += v2; - } - if dim_idx != 0 && v1 != v2 { - Err(Error::ShapeMismatchCat { - dim: dim_idx, - first_shape: arg0.shape().clone(), - n: arg_idx + 1, - nth_shape: arg.shape().clone(), - } - .bt())? - } - } - let next_offset = offsets.last().unwrap() + arg.elem_count(); - offsets.push(next_offset); - } - let shape = Shape::from(cat_dims); - let op = BackpropOp::new(args, |args| Op::Cat(args, 0)); - let mut storage = device.zeros(&shape, dtype)?; - for (arg, &offset) in args.iter().zip(offsets.iter()) { - let arg = arg.as_ref(); - arg.storage() - .copy_strided_src(&mut storage, offset, arg.layout())?; - } - Ok(from_storage(storage, shape, op, false)) - } - - /// Pad the input tensor using 0s along dimension `dim`. This adds `left` elements before the - /// input tensor values and `right` elements after. - pub fn pad_with_zeros(&self, dim: D, left: usize, right: usize) -> Result { - if left == 0 && right == 0 { - Ok(self.clone()) - } else if left == 0 { - let dim = dim.to_index(self.shape(), "pad_with_zeros")?; - let mut dims = self.dims().to_vec(); - dims[dim] = right; - let right = Tensor::zeros(dims.as_slice(), self.dtype, self.device())?; - Tensor::cat(&[self, &right], dim) - } else if right == 0 { - let dim = dim.to_index(self.shape(), "pad_with_zeros")?; - let mut dims = self.dims().to_vec(); - dims[dim] = left; - let left = Tensor::zeros(dims.as_slice(), self.dtype, self.device())?; - Tensor::cat(&[&left, self], dim) - } else { - let dim = dim.to_index(self.shape(), "pad_with_zeros")?; - let mut dims = self.dims().to_vec(); - dims[dim] = left; - let left = Tensor::zeros(dims.as_slice(), self.dtype, self.device())?; - dims[dim] = right; - let right = Tensor::zeros(dims.as_slice(), self.dtype, self.device())?; - Tensor::cat(&[&left, self, &right], dim) - } - } - - /// Pad the input tensor using same values along dimension `dim`. This adds `left` elements before the - /// input tensor values and `right` elements after. - pub fn pad_with_same(&self, dim: D, left: usize, right: usize) -> Result { - if left == 0 && right == 0 { - Ok(self.clone()) - } else if self.elem_count() == 0 { - bail!("cannot use pad_with_same on an empty tensor") - } else if left == 0 { - let dim = dim.to_index(self.shape(), "pad_with_same")?; - let r = self.narrow(dim, self.dim(dim)? - 1, 1)?; - let mut v = vec![self]; - for _ in 0..right { - v.push(&r) - } - Tensor::cat(&v, dim) - } else if right == 0 { - let dim = dim.to_index(self.shape(), "pad_with_same")?; - let l = self.narrow(dim, 0, 1)?; - let mut v = vec![]; - for _ in 0..left { - v.push(&l) - } - v.push(self); - Tensor::cat(&v, dim) - } else { - let dim = dim.to_index(self.shape(), "pad_with_same")?; - let l = self.narrow(dim, 0, 1)?; - let r = self.narrow(dim, self.dim(dim)? - 1, 1)?; - let mut v = vec![]; - for _ in 0..left { - v.push(&l) - } - v.push(self); - for _ in 0..right { - v.push(&r) - } - Tensor::cat(&v, dim) - } - } - - /// Run the `forward` method of `m` on `self`. - pub fn apply(&self, m: &M) -> Result { - m.forward(self) - } - - /// Run the `forward` method of `m` on `self`. - pub fn apply_t(&self, m: &M, train: bool) -> Result { - m.forward_t(self, train) - } - - pub(crate) fn storage(&self) -> std::sync::RwLockReadGuard<'_, Storage> { - self.storage.read().unwrap() - } - - // If we extend the visibility of this function to be usable outside of this crate, we should - // make it unsafe. - pub(crate) fn storage_mut_and_layout( - &self, - ) -> (std::sync::RwLockWriteGuard<'_, Storage>, &Layout) { - let storage = self.storage.write().unwrap(); - (storage, &self.layout) - } - - /// The storage used by this tensor, together with the layout to use to access it safely. - pub fn storage_and_layout(&self) -> (std::sync::RwLockReadGuard<'_, Storage>, &Layout) { - let storage = self.storage.read().unwrap(); - (storage, &self.layout) - } - - pub(crate) fn same_storage(&self, rhs: &Self) -> bool { - let lhs: &RwLock = self.storage.as_ref(); - let rhs: &RwLock = rhs.storage.as_ref(); - std::ptr::eq(lhs, rhs) - } - - /// Applies a unary custom op without backward support - pub fn apply_op1_no_bwd(&self, c: &C) -> Result { - let (storage, shape) = self.storage().apply_op1(self.layout(), c)?; - Ok(from_storage(storage, shape, BackpropOp::none(), false)) - } - - /// Applies a binary custom op without backward support - pub fn apply_op2_no_bwd(&self, rhs: &Self, c: &C) -> Result { - let (storage, shape) = - self.storage() - .apply_op2(self.layout(), &rhs.storage(), rhs.layout(), c)?; - Ok(from_storage(storage, shape, BackpropOp::none(), false)) - } - - /// Applies a ternary custom op without backward support - pub fn apply_op3_no_bwd(&self, t2: &Self, t3: &Self, c: &C) -> Result { - let (storage, shape) = self.storage().apply_op3( - self.layout(), - &t2.storage(), - t2.layout(), - &t3.storage(), - t3.layout(), - c, - )?; - Ok(from_storage(storage, shape, BackpropOp::none(), false)) - } - - /// Applies a unary custom op. - pub fn apply_op1_arc(&self, c: Arc>) -> Result { - let (storage, shape) = self - .storage() - .apply_op1(self.layout(), c.as_ref().as_ref())?; - let op = BackpropOp::new1(self, |s| Op::CustomOp1(s, c.clone())); - Ok(from_storage(storage, shape, op, false)) - } - - pub fn apply_op1(&self, c: C) -> Result { - self.apply_op1_arc(Arc::new(Box::new(c))) - } - - /// Applies a binary custom op. - pub fn apply_op2_arc( - &self, - rhs: &Self, - c: Arc>, - ) -> Result { - let (storage, shape) = self.storage().apply_op2( - self.layout(), - &rhs.storage(), - rhs.layout(), - c.as_ref().as_ref(), - )?; - let op = BackpropOp::new2(self, rhs, |t1, t2| Op::CustomOp2(t1, t2, c.clone())); - Ok(from_storage(storage, shape, op, false)) - } - - pub fn apply_op2(&self, r: &Self, c: C) -> Result { - self.apply_op2_arc(r, Arc::new(Box::new(c))) - } - - /// Applies a ternary custom op. - pub fn apply_op3_arc( - &self, - t2: &Self, - t3: &Self, - c: Arc>, - ) -> Result { - let (storage, shape) = self.storage().apply_op3( - self.layout(), - &t2.storage(), - t2.layout(), - &t3.storage(), - t3.layout(), - c.as_ref().as_ref(), - )?; - let op = BackpropOp::new3(self, t2, t3, |t1, t2, t3| { - Op::CustomOp3(t1, t2, t3, c.clone()) - }); - Ok(from_storage(storage, shape, op, false)) - } - - pub fn apply_op3( - &self, - t2: &Self, - t3: &Self, - c: C, - ) -> Result { - self.apply_op3_arc(t2, t3, Arc::new(Box::new(c))) - } - - /// Normalize a 'relative' axis value: positive values are kept, negative - /// values means counting the dimensions from the back. - pub fn normalize_axis(&self, axis: i64) -> Result { - let rank = self.rank() as i64; - if rank <= axis { - bail!("axis {axis} is too large, tensor rank {rank}") - } else if 0 <= axis { - Ok(axis as usize) - } else { - let naxis = rank + axis; - if naxis < 0 { - bail!("axis {axis} is too small, tensor rank {rank}") - } - Ok(naxis as usize) - } - } - - /// Returns a lower triangular matrix of ones of size n by n. - pub fn tril2(n: usize, dtype: DType, device: &Device) -> Result { - let t = Tensor::arange(0u32, n as u32, device)?; - let t1 = t.reshape((1, n))?.broadcast_as((n, n))?; - let t2 = t.reshape((n, 1))?.broadcast_as((n, n))?; - t1.le(&t2)?.to_dtype(dtype) - } - - /// Returns an upper triangular matrix of ones of size n by n. - pub fn triu2(n: usize, dtype: DType, device: &Device) -> Result { - let t = Tensor::arange(0u32, n as u32, device)?; - let t1 = t.reshape((1, n))?.broadcast_as((n, n))?; - let t2 = t.reshape((n, 1))?.broadcast_as((n, n))?; - t1.ge(&t2)?.to_dtype(dtype) - } - - /// Returns a matrix with a diagonal of ones of size n by n. - pub fn eye(n: usize, dtype: DType, device: &Device) -> Result { - let t = Tensor::arange(0u32, n as u32, device)?; - let t1 = t.reshape((1, n))?.broadcast_as((n, n))?; - let t2 = t.reshape((n, 1))?.broadcast_as((n, n))?; - t1.eq(&t2)?.to_dtype(dtype) - } - - /// Returns the cumulative sum of elements of the input tensor summed over the specified - /// dimension. - /// - /// This operation is most efficient when dim is the last dimension of the tensor. - pub fn cumsum(&self, dim: D) -> Result { - let dim = dim.to_index(self.shape(), "cumsum")?; - let rank = self.rank(); - if rank == 0 { - return Ok(self.clone()); - } - let n_axis = self.dim(dim)?; - let triu = Tensor::triu2(n_axis, self.dtype(), self.device())?; - if rank == 1 { - self.unsqueeze(0)?.matmul(&triu)?.squeeze(0) - } else { - let last = rank - 1; - let t = self.transpose(dim, last)?; - let t = t.broadcast_matmul(&triu)?; - t.transpose(dim, last) - } - } - - /// Returns a copy of `self` where the values within `ranges` have been replaced with the - /// content of `src`. - pub fn slice_assign>( - &self, - ranges: &[D], - src: &Tensor, - ) -> Result { - let src_dims = src.dims(); - let self_dims = self.dims(); - if self_dims.len() != src_dims.len() { - bail!( - "slice-assign requires input with the same rank {} <> {}", - self_dims.len(), - src_dims.len() - ) - } - if self_dims.len() != ranges.len() { - bail!( - "slice-assign requires input with the same rank as there are ranges {} <> {}", - self_dims.len(), - ranges.len() - ) - } - let mut src = src.clone(); - let mut mask = Self::ones(src.shape(), DType::U8, src.device())?; - for (i, range) in ranges.iter().enumerate() { - let start_included = match range.start_bound() { - std::ops::Bound::Unbounded => 0, - std::ops::Bound::Included(v) => *v, - std::ops::Bound::Excluded(v) => *v + 1, - }; - let end_excluded = match range.end_bound() { - std::ops::Bound::Unbounded => self_dims[i], - std::ops::Bound::Included(v) => *v + 1, - std::ops::Bound::Excluded(v) => *v, - }; - if end_excluded <= start_included { - bail!("slice-assign: empty range for dim {i}, {start_included} {end_excluded}") - } - if self_dims[i] < end_excluded { - bail!( - "slice-assign: upper bound is out of range for dim {i}, {end_excluded} {}", - self_dims[i] - ) - } - if end_excluded - start_included != src_dims[i] { - bail!( - "slice-assign: the range for dim {i} ({start_included}..{end_excluded}) does not match the size of src {}", src_dims[i] - ) - } - src = src.pad_with_zeros(i, start_included, self_dims[i] - end_excluded)?; - mask = mask.pad_with_zeros(i, start_included, self_dims[i] - end_excluded)? - } - mask.where_cond(/* on_true= */ &src, /* on_false= */ self) - } - - /// Returns log(sum(exp(tensor), dim)). - pub fn log_sum_exp(&self, sum_dims: D) -> Result { - let exp = self.exp()?; - let sum = exp.sum(sum_dims)?; - sum.log() - } - - /// Pointwise pow operation. - pub fn pow(&self, rhs: &Tensor) -> Result { - rhs.mul(&self.log()?)?.exp() - } - - /// Broadcasting version of `pow`. - pub fn broadcast_pow(&self, rhs: &Tensor) -> Result { - rhs.broadcast_mul(&self.log()?)?.exp() - } -} - -macro_rules! bin_trait { - ($trait:ident, $fn1:ident, $mul:expr, $add:expr) => { - impl> std::ops::$trait for Tensor { - type Output = Result; - - fn $fn1(self, rhs: B) -> Self::Output { - Tensor::$fn1(&self, rhs.borrow()) - } - } - - impl> std::ops::$trait for &Tensor { - type Output = Result; - - fn $fn1(self, rhs: B) -> Self::Output { - Tensor::$fn1(&self, rhs.borrow()) - } - } - - impl> std::ops::$trait for Result { - type Output = Result; - - fn $fn1(self, rhs: Tensor) -> Self::Output { - Tensor::$fn1(self?.borrow(), &rhs) - } - } - - impl> std::ops::$trait<&Tensor> for Result { - type Output = Result; - - fn $fn1(self, rhs: &Tensor) -> Self::Output { - Tensor::$fn1(self?.borrow(), rhs) - } - } - - impl> std::ops::$trait> for Tensor { - type Output = Result; - - fn $fn1(self, rhs: Result) -> Self::Output { - Tensor::$fn1(&self, rhs?.borrow()) - } - } - - impl> std::ops::$trait> for &Tensor { - type Output = Result; - - fn $fn1(self, rhs: Result) -> Self::Output { - Tensor::$fn1(&self, rhs?.borrow()) - } - } - - impl std::ops::$trait for Tensor { - type Output = Result; - - fn $fn1(self, rhs: f64) -> Self::Output { - self.affine($mul(rhs), $add(rhs)) - } - } - - impl std::ops::$trait for &Tensor { - type Output = Result; - - fn $fn1(self, rhs: f64) -> Self::Output { - self.affine($mul(rhs), $add(rhs)) - } - } - }; -} - -bin_trait!(Add, add, |_| 1., |v| v); -bin_trait!(Sub, sub, |_| 1., |v: f64| -v); -bin_trait!(Mul, mul, |v| v, |_| 0.); -bin_trait!(Div, div, |v| 1. / v, |_| 0.); - -impl std::ops::Add for f64 { - type Output = Result; - - fn add(self, rhs: Tensor) -> Self::Output { - rhs + self - } -} - -impl std::ops::Add<&Tensor> for f64 { - type Output = Result; - - fn add(self, rhs: &Tensor) -> Self::Output { - rhs + self - } -} - -impl std::ops::Mul for f64 { - type Output = Result; - - fn mul(self, rhs: Tensor) -> Self::Output { - rhs * self - } -} - -impl std::ops::Mul<&Tensor> for f64 { - type Output = Result; - - fn mul(self, rhs: &Tensor) -> Self::Output { - rhs * self - } -} - -impl std::ops::Sub for f64 { - type Output = Result; - - fn sub(self, rhs: Tensor) -> Self::Output { - rhs.affine(-1., self) - } -} - -impl std::ops::Sub<&Tensor> for f64 { - type Output = Result; - - fn sub(self, rhs: &Tensor) -> Self::Output { - rhs.affine(-1., self) - } -} - -impl std::ops::Div for f64 { - type Output = Result; - - #[allow(clippy::suspicious_arithmetic_impl)] - fn div(self, rhs: Tensor) -> Self::Output { - rhs.recip()? * self - } -} - -impl std::ops::Div<&Tensor> for f64 { - type Output = Result; - - #[allow(clippy::suspicious_arithmetic_impl)] - fn div(self, rhs: &Tensor) -> Self::Output { - rhs.recip()? * self - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/test_utils.rs b/v2/dependency-patches/candle-core-0.4.1/src/test_utils.rs deleted file mode 100644 index 3b8fb904c0..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/test_utils.rs +++ /dev/null @@ -1,62 +0,0 @@ -use crate::{Result, Tensor}; - -#[macro_export] -macro_rules! test_device { - // TODO: Switch to generating the two last arguments automatically once concat_idents is - // stable. https://github.com/rust-lang/rust/issues/29599 - ($fn_name: ident, $test_cpu: ident, $test_cuda: ident, $test_metal: ident) => { - #[test] - fn $test_cpu() -> Result<()> { - $fn_name(&Device::Cpu) - } - - #[cfg(feature = "cuda")] - #[test] - fn $test_cuda() -> Result<()> { - $fn_name(&Device::new_cuda(0)?) - } - - #[cfg(feature = "metal")] - #[test] - fn $test_metal() -> Result<()> { - $fn_name(&Device::new_metal(0)?) - } - }; -} - -pub fn to_vec0_round(t: &Tensor, digits: i32) -> Result { - let b = 10f32.powi(digits); - let t = t.to_vec0::()?; - Ok(f32::round(t * b) / b) -} - -pub fn to_vec1_round(t: &Tensor, digits: i32) -> Result> { - let b = 10f32.powi(digits); - let t = t.to_vec1::()?; - let t = t.iter().map(|t| f32::round(t * b) / b).collect(); - Ok(t) -} - -pub fn to_vec2_round(t: &Tensor, digits: i32) -> Result>> { - let b = 10f32.powi(digits); - let t = t.to_vec2::()?; - let t = t - .iter() - .map(|t| t.iter().map(|t| f32::round(t * b) / b).collect()) - .collect(); - Ok(t) -} - -pub fn to_vec3_round(t: &Tensor, digits: i32) -> Result>>> { - let b = 10f32.powi(digits); - let t = t.to_vec3::()?; - let t = t - .iter() - .map(|t| { - t.iter() - .map(|t| t.iter().map(|t| f32::round(t * b) / b).collect()) - .collect() - }) - .collect(); - Ok(t) -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/utils.rs b/v2/dependency-patches/candle-core-0.4.1/src/utils.rs deleted file mode 100644 index 78c45a9a9d..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/utils.rs +++ /dev/null @@ -1,44 +0,0 @@ -use std::str::FromStr; - -pub fn get_num_threads() -> usize { - // Respond to the same environment variable as rayon. - match std::env::var("RAYON_NUM_THREADS") - .ok() - .and_then(|s| usize::from_str(&s).ok()) - { - Some(x) if x > 0 => x, - Some(_) | None => num_cpus::get(), - } -} - -pub fn has_accelerate() -> bool { - cfg!(feature = "accelerate") -} - -pub fn has_mkl() -> bool { - cfg!(feature = "mkl") -} - -pub fn cuda_is_available() -> bool { - cfg!(feature = "cuda") -} - -pub fn metal_is_available() -> bool { - cfg!(feature = "metal") -} - -pub fn with_avx() -> bool { - cfg!(target_feature = "avx") -} - -pub fn with_neon() -> bool { - cfg!(target_feature = "neon") -} - -pub fn with_simd128() -> bool { - cfg!(target_feature = "simd128") -} - -pub fn with_f16c() -> bool { - cfg!(target_feature = "f16c") -} diff --git a/v2/dependency-patches/candle-core-0.4.1/src/variable.rs b/v2/dependency-patches/candle-core-0.4.1/src/variable.rs deleted file mode 100644 index bdf8da4aed..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/src/variable.rs +++ /dev/null @@ -1,147 +0,0 @@ -// Variables are wrappers around tensors that can be modified, they are typically used for holding -// weights and being modified by gradient descent. -// We do not expose a public way to create variables as this would break the invariant that the -// tensor within a variable is actually with `is_variable` set to `true`. -use crate::{DType, Device, Error, Result, Shape, Tensor}; - -/// A variable is a wrapper around a tensor, however variables can have their content modified -/// whereas tensors are immutable. -#[derive(Clone, Debug)] -pub struct Var(Tensor); - -impl std::fmt::Display for Var { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Display::fmt(&self.0, f) - } -} - -impl std::ops::Deref for Var { - type Target = Tensor; - - fn deref(&self) -> &Self::Target { - self.0.as_ref() - } -} - -impl Var { - pub fn zeros>(shape: S, dtype: DType, device: &Device) -> Result { - let inner = Tensor::zeros_impl(shape, dtype, device, true)?; - Ok(Self(inner)) - } - - pub fn ones>(shape: S, dtype: DType, device: &Device) -> Result { - let inner = Tensor::ones_impl(shape, dtype, device, true)?; - Ok(Self(inner)) - } - - pub fn from_tensor(t: &Tensor) -> Result { - let inner = t.make_var()?; - Ok(Self(inner)) - } - - pub fn rand_f64>( - lo: f64, - up: f64, - s: S, - dtype: DType, - device: &Device, - ) -> Result { - let inner = Tensor::rand_f64_impl(lo, up, s, dtype, device, true)?; - Ok(Self(inner)) - } - - pub fn randn_f64>( - mean: f64, - std: f64, - s: S, - dtype: DType, - device: &Device, - ) -> Result { - let inner = Tensor::randn_f64_impl(mean, std, s, dtype, device, true)?; - Ok(Self(inner)) - } - - pub fn rand, T: crate::FloatDType>( - lo: T, - up: T, - s: S, - device: &Device, - ) -> Result { - let inner = Tensor::rand_impl(lo, up, s, device, true)?; - Ok(Self(inner)) - } - - pub fn randn, T: crate::FloatDType>( - mean: T, - std: T, - s: S, - device: &Device, - ) -> Result { - let inner = Tensor::randn_impl(mean, std, s, device, true)?; - Ok(Self(inner)) - } - - /// Creates a new tensor on the specified device using the content and shape of the input. - /// This is similar to `new` but the resulting tensor is a variable. - pub fn new(array: A, device: &Device) -> Result { - let shape = array.shape()?; - let inner = Tensor::new_impl(array, shape, device, true)?; - Ok(Self(inner)) - } - - pub fn from_vec, D: crate::WithDType>( - data: Vec, - shape: S, - device: &Device, - ) -> Result { - let inner = Tensor::from_vec_impl(data, shape, device, true)?; - Ok(Self(inner)) - } - - pub fn from_slice, D: crate::WithDType>( - array: &[D], - shape: S, - device: &Device, - ) -> Result { - let inner = Tensor::new_impl(array, shape.into(), device, true)?; - Ok(Self(inner)) - } - - pub fn as_detached_tensor(&self) -> Tensor { - self.0.detach() - } - - pub fn as_tensor(&self) -> &Tensor { - &self.0 - } - - /// Consumes this `Var` and return the underlying tensor. - pub fn into_inner(self) -> Tensor { - self.0 - } - - /// Sets the content of the inner tensor, this does not require a mutable reference as inner - /// mutability is used. - pub fn set(&self, src: &Tensor) -> Result<()> { - if self.same_storage(src) { - let msg = "cannot set a variable to a tensor that is derived from its value"; - Err(Error::CannotSetVar { msg }.bt())? - } - let (mut dst, layout) = self.storage_mut_and_layout(); - if !layout.is_contiguous() { - let msg = "cannot set a non-contiguous variable"; - Err(Error::CannotSetVar { msg }.bt())? - } - let (src, src_l) = src.storage_and_layout(); - if layout.shape() != src_l.shape() { - Err(Error::ShapeMismatchBinaryOp { - lhs: layout.shape().clone(), - rhs: src_l.shape().clone(), - op: "set", - } - .bt())? - } - src.copy_strided_src(&mut dst, layout.start_offset(), src_l)?; - Ok(()) - } -} diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/conv_tests.rs b/v2/dependency-patches/candle-core-0.4.1/tests/conv_tests.rs deleted file mode 100644 index b967515dd9..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/tests/conv_tests.rs +++ /dev/null @@ -1,609 +0,0 @@ -use anyhow::Result; -use candle_core::{test_device, test_utils, Device, IndexOp, Tensor}; - -/* This test is based on the following script. -import torch -torch.manual_seed(4242) - -t = torch.randn((1, 4, 5)) -w = torch.randn((2, 4, 3)) -print(t.flatten()) -print(w.flatten()) -res = torch.nn.functional.conv1d(t, w) -print(res.flatten()) -res = torch.nn.functional.conv1d(t, w, padding=1) -print(res.flatten()) - -w_t = w.transpose(0, 1) -res = torch.nn.functional.conv_transpose1d(t, w_t) -print(res.shape) -print(res) -res = torch.nn.functional.conv_transpose1d(t, w_t, groups=2) -print(res.shape) -print(res) -*/ -fn conv1d(dev: &Device) -> Result<()> { - let t = Tensor::new( - &[ - 0.4056f32, -0.8689, -0.0773, -1.5630, 1.2279, -0.9287, -1.7030, 0.1370, 0.1866, 0.4145, - 1.8025, -0.1536, 2.2013, -0.6836, 0.2477, 1.3127, -0.6957, 0.3278, -1.0124, 0.5599, - ], - dev, - )? - .reshape((1, 4, 5))?; - let w = Tensor::new( - &[ - -0.8404f32, -0.3490, 0.0130, 1.3123, 0.1763, -1.9249, 1.4270, 0.9421, 0.8670, -0.7181, - -1.1111, 0.8869, -1.2429, 1.8357, 1.6052, -1.3844, 0.3951, -1.2036, 0.6686, 1.6261, - -0.6451, -0.0840, -1.4247, 0.5512, - ], - dev, - )? - .reshape((2, 4, 3))?; - let res = t.conv1d(&w, 0, 1, 1, 1)?; - assert_eq!(res.dims(), [1, 2, 3]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [2.6357, -1.3336, 4.1393, -1.1784, 3.5675, 0.5069] - ); - let res = t.conv1d(&w, /*padding*/ 1, 1, 1, 1)?; - assert_eq!(res.dims(), [1, 2, 5]); - // Same as pytorch default padding: use zeros. - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [2.4509, 2.6357, -1.3336, 4.1393, 0.5657, 1.8091, -1.1784, 3.5675, 0.5069, 3.3352] - ); - let res = t.conv_transpose1d(&w.transpose(0, 1)?, 0, 0, 1, 1, 1)?; - assert_eq!(res.dims(), [1, 2, 7]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [ - 0.0699, -1.2899, 8.3018, 5.5873, 2.4572, -2.6143, -0.0706, 1.8765, 4.8318, 1.1538, - 4.7076, -5.9745, -0.8276, 1.621 - ], - ); - let res = t.conv_transpose1d(&w.transpose(0, 1)?, 0, 0, 1, 1, 2)?; - assert_eq!(res.dims(), [1, 4, 7]); - assert_eq!( - test_utils::to_vec2_round(&res.squeeze(0)?, 4)?, - [ - [-1.5596, -1.8099, 2.0407, 4.8764, -0.1743, -0.735, -0.7819], - [0.7816, 3.8152, -0.5926, 2.2515, -5.1844, -0.3157, 1.4721], - [1.6295, 0.52, 6.2611, 0.7109, 2.6315, -1.8793, 0.7113], - [1.0949, 1.0166, 1.7464, 2.4561, -0.79, -0.5119, 0.1488] - ] - ); - Ok(()) -} - -fn conv1d_small(dev: &Device) -> Result<()> { - let t = Tensor::new(&[0.4056f32, -0.8689, -0.0773, -1.5630], dev)?.reshape((1, 1, 4))?; - let w = Tensor::new(&[1f32, 0., 0.], dev)?.reshape((1, 1, 3))?; - let res = t.conv1d(&w, 0, 1, 1, 1)?; - assert_eq!(res.dims(), [1, 1, 2]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [0.4056, -0.8689] - ); - let res = t.conv1d(&w, /*padding*/ 1, 1, 1, 1)?; - assert_eq!(res.dims(), [1, 1, 4]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [0.0, 0.4056, -0.8689, -0.0773], - ); - Ok(()) -} - -/* This test is based on the following script. -import torch -torch.manual_seed(4242) - -t = torch.randn((1, 4, 5, 5)) -w = torch.randn((2, 4, 3, 3)) -print(t.flatten()) -print(w.flatten()) -res = torch.nn.functional.conv2d(t, w) -print(res.flatten()) - -w_t = w.transpose(0, 1) -res = torch.nn.functional.conv_transpose2d(t, w_t) -print(res.shape) -print(res) - -res = torch.nn.functional.conv2d(t, w, dilation=2) -print(res.shape) -print(res[0]) - -res = torch.nn.functional.conv_transpose2d(t, w_t, dilation=2) -print(res.shape) -print(res) -*/ -fn conv2d(dev: &Device) -> Result<()> { - let t = Tensor::new( - &[ - 0.4056f32, -0.8689, -0.0773, -1.5630, -2.8012, -1.5059, 0.3972, 1.0852, 0.4997, 3.0616, - 1.6541, 0.0964, -0.8338, -1.6523, -0.8323, -0.1699, 0.0823, 0.3526, 0.6843, 0.2395, - 1.2279, -0.9287, -1.7030, 0.1370, 0.6047, 0.3770, -0.6266, 0.3529, 2.2013, -0.6836, - 0.2477, 1.3127, -0.2260, 0.2622, -1.2974, -0.8140, -0.8404, -0.3490, 0.0130, 1.3123, - 1.7569, -0.3956, -1.8255, 0.1727, -0.3538, 2.6941, 1.0529, 0.4219, -0.2071, 1.1586, - 0.4717, 0.3865, -0.5690, -0.5010, -0.1310, 0.7796, 0.6630, -0.2021, 2.6090, 0.2049, - 0.6466, -0.5042, -0.0603, -1.6538, -1.2429, 1.8357, 1.6052, -1.3844, 0.3323, -1.3712, - 0.9634, -0.4799, -0.6451, -0.0840, -1.4247, 0.5512, -0.1747, -0.5509, -0.3742, 0.3790, - -0.4431, -0.4720, -0.7890, 0.2620, 0.7875, 0.5377, -0.6779, -0.8088, 1.9098, 1.2006, - -0.8000, -0.4983, 1.5480, 0.8265, -0.1025, 0.5138, 0.5748, 0.3821, -0.4607, 0.0085, - ], - dev, - )?; - let w = Tensor::new( - &[ - -0.9325f32, 0.6451, -0.8537, 0.2378, 0.8764, -0.1832, 0.2987, -0.6488, -0.2273, - -2.4184, -0.1192, -0.4821, -0.5079, -0.5766, -2.4729, 1.6734, 0.4558, 0.2851, 1.1514, - -0.9013, 1.0662, -0.1817, -0.0259, 0.1709, 0.5367, 0.7513, 0.8086, -2.2586, -0.5027, - 0.9141, -1.3086, -1.3343, -1.5669, -0.1657, 0.7958, 0.1432, 0.3896, -0.4501, 0.1667, - 0.0714, -0.0952, 1.2970, -0.1674, -0.3178, 1.0677, 0.3060, 0.7080, 0.1914, 1.1679, - -0.3602, 1.9265, -1.8626, -0.5112, -0.0982, 0.2621, 0.6565, 0.5908, 1.0089, -0.1646, - 1.8032, -0.6286, 0.2016, -0.3370, 1.2555, 0.8009, -0.6488, -0.4652, -1.5685, 1.5860, - 0.5583, 0.4623, 0.6026, - ], - dev, - )?; - let t = t.reshape((1, 4, 5, 5))?; - let w = w.reshape((2, 4, 3, 3))?; - let res = t.conv2d(&w, 0, 1, 1, 1)?; - assert_eq!(res.dims(), [1, 2, 3, 3]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [ - -4.2812, 2.0923, 5.2187, 7.5184, 0.752, -14.9426, 10.0087, 4.391, 0.2918, 1.6715, - 10.389, 3.6023, -4.2808, 0.2672, 5.3646, -5.2023, -2.1955, -9.4075 - ] - ); - let res = t.conv_transpose2d(&w.transpose(0, 1)?, 0, 0, 1, 1)?; - assert_eq!(res.dims(), [1, 2, 7, 7]); - assert_eq!( - test_utils::to_vec3_round(&res.i(0)?, 4)?, - [ - [ - [-1.9918, 2.6797, -0.4599, -1.6037, 1.4131, -2.4012, 2.9277], - [1.8016, -3.5361, 1.0757, 3.5395, -8.2168, -3.2023, 0.5375], - [0.8243, 1.8675, 7.8929, -4.0746, -6.4415, 5.1139, 1.6889], - [0.2722, 8.9679, 3.3477, 1.8514, -4.2896, -3.8228, -7.5632], - [-8.5412, -5.8142, -7.1587, -1.6095, 0.4651, 0.2748, -2.0985], - [2.0833, -0.6482, -12.1692, -4.1284, -2.9765, -0.0656, -4.5114], - [5.307, 2.6957, 2.3087, 1.0478, 0.7808, -1.1519, -0.9579] - ], - [ - [1.089, 0.1872, -0.6408, -0.9897, 0.8503, 1.1019, -0.9211], - [-0.1741, -0.2915, 4.2472, 1.9417, 1.65, 0.6303, -4.7131], - [1.6555, 2.4026, -2.9293, 2.9953, 0.5328, 3.5873, -0.9621], - [-1.4289, -3.2787, 4.1747, -6.0341, -4.6341, -5.7945, 4.142], - [7.5973, 6.4431, 5.9872, 2.1639, -8.6566, 3.3143, -3.4059], - [-0.8775, -3.048, 11.6543, 0.6442, 2.3218, -0.4765, 1.1516], - [-5.5423, -2.5188, 1.0754, -0.0563, -2.9386, -1.1504, 1.0171] - ] - ] - ); - // Dilations. - let res = t.conv2d(&w, 0, 1, 2, 1)?; - assert_eq!(res.dims(), [1, 2, 1, 1]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [2.45, -2.3504], - ); - - // Transpose and dilations. - let res = t.conv_transpose2d(&w.transpose(0, 1)?, 0, 0, 1, 2)?; - assert_eq!(res.dims(), [1, 2, 9, 9]); - assert_eq!( - test_utils::to_vec3_round(&res.i(0)?, 4)?, - [ - [ - [-1.9918, 3.1652, -0.6778, -4.3442, 4.4351, 0.6652, -3.0124, -0.6031, 2.9277], - [2.7036, -1.7156, -0.3969, 1.0516, 1.6381, -2.8886, -0.205, 2.4682, -1.0499], - [-0.9459, 3.1631, 3.707, -4.8369, -8.5166, -1.4496, -2.7559, -3.2698, 1.4376], - [-0.2157, 3.7786, -2.0252, -4.2633, 3.6731, -1.5142, 5.9391, -0.2622, -0.141], - [-6.8121, -3.1744, 1.5945, 3.0637, -9.6088, 1.4446, 2.9489, -3.0082, -7.3822], - [0.2371, 3.3303, 0.3861, 2.2646, -4.6784, 4.1235, -0.0109, 0.3176, -0.03], - [-2.5339, -2.9564, -3.4518, -4.4594, -9.1873, -1.9709, -0.4676, 0.51, -3.5024], - [4.007, 0.3067, -2.2954, 1.1105, -0.1992, 1.6372, -2.9268, 0.2807, -1.2787], - [5.307, 1.1317, 1.3518, 0.9049, 3.8116, -0.4075, -0.8874, -0.2241, -0.9579] - ], - [ - [1.089, -0.6483, 0.0726, -0.4752, -1.3283, 1.7103, 1.0703, 0.1076, -0.9211], - [-0.8629, 0.1376, 0.3202, 2.0955, 0.9696, 2.8988, -1.0012, 1.5049, -0.1278], - [1.9286, -1.5255, -2.9563, 2.4589, 3.3611, -0.6951, 0.3525, -1.7724, -5.9861], - [1.1226, 2.1561, 3.6417, 4.7546, -0.692, 4.4126, -5.1902, 6.0805, 2.3185], - [1.0111, 0.3604, 0.6432, -3.6605, 7.9517, -9.2955, -5.2988, -3.7803, -2.0642], - [3.3172, -1.7967, -3.6576, -2.0942, 1.3158, 0.112, -1.7405, 2.9167, 0.7957], - [5.1001, 1.8995, -1.8639, 1.1262, 9.9629, 2.683, -3.6319, -1.1607, 0.5856], - [-4.8445, -0.5642, 4.2317, 0.0856, 1.2267, -0.5712, 1.736, 1.0997, 0.6908], - [-5.5423, -1.1831, -1.2176, 0.0843, 0.0446, -0.7545, -2.4798, -0.0827, 1.0171] - ] - ] - ); - Ok(()) -} - -/* This test is based on the following script. -import torch -torch.manual_seed(4242) - -t = torch.randn((1, 2, 3, 3)) -w = torch.randn((1, 2, 1, 1)) -print(t.flatten()) -print(w.flatten()) -res = torch.nn.functional.conv2d(t, w) -print(res.flatten()) - -w_t = w.transpose(0, 1) -res = torch.nn.functional.conv_transpose2d(t, w_t) -print(res.shape) -print(res.flatten()) - -t_t = w.transpose(0, 1) -res = torch.nn.functional.conv_transpose2d(t_t, w) -print(res.shape) -print(res.flatten()) -*/ -fn conv2d_small(dev: &Device) -> Result<()> { - let t = Tensor::new( - &[ - 0.4056f32, -0.8689, 0.6843, 0.2395, 1.2279, -0.9287, -1.7030, 0.1370, 0.1866, 0.4145, - -0.6266, 0.3529, 2.2013, -0.6836, 0.2477, 1.3127, -0.6957, 0.3278, - ], - dev, - )?; - let w = Tensor::new(&[-0.9259f32, 1.3017], dev)?; - let t = t.reshape((1, 2, 3, 3))?; - let w = w.reshape((1, 2, 1, 1))?; - let res = t.conv2d(&w, 0, 1, 1, 1)?; - assert_eq!(res.dims(), [1, 1, 3, 3]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [0.164, -0.0111, -0.1742, 2.6437, -2.0268, 1.1823, 3.2855, -1.0324, 0.2539] - ); - let res = t.conv2d(&w, 2, 1, 1, 1)?; - assert_eq!(res.dims(), [1, 1, 7, 7]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [ - 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, - 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.1640, -0.0111, -0.1742, 0.0000, 0.0000, - 0.0000, 0.0000, 2.6437, -2.0268, 1.1823, 0.0000, 0.0000, 0.0000, 0.0000, 3.2855, - -1.0324, 0.2539, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, - 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000 - ] - ); - let res = t.conv_transpose2d(&w.transpose(0, 1)?, 0, 0, 1, 1)?; - assert_eq!(res.dims(), [1, 1, 3, 3]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [0.164, -0.0111, -0.1742, 2.6437, -2.0268, 1.1823, 3.2855, -1.0324, 0.2539], - ); - let res = t.transpose(0, 1)?.conv_transpose2d(&w, 0, 0, 1, 1)?; - assert_eq!(res.dims(), [2, 2, 3, 3]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [ - -0.3755, 0.8045, -0.6336, -0.2218, -1.1369, 0.8599, 1.5768, -0.1268, -0.1728, 0.528, - -1.131, 0.8908, 0.3118, 1.5984, -1.2089, -2.2168, 0.1783, 0.2429, -0.3838, 0.5802, - -0.3268, -2.0382, 0.6329, -0.2293, -1.2154, 0.6441, -0.3035, 0.5396, -0.8156, 0.4594, - 2.8654, -0.8898, 0.3224, 1.7087, -0.9056, 0.4267 - ] - ); - Ok(()) -} - -fn conv2d_smaller(dev: &Device) -> Result<()> { - let t = Tensor::new( - &[ - 0.4056f32, -0.8689, 0.6843, 0.2395, 1.2279, -0.9287, -1.7030, 0.1370, 0.1866, - ], - dev, - )?; - let w = Tensor::new(&[1f32, 1., 1., 1., 1., 1., 1., 1., 1.], dev)?; - let t = t.reshape((1, 1, 3, 3))?; - let w = w.reshape((1, 1, 3, 3))?; - let res = t.conv2d(&w, 0, 1, 1, 1)?; - assert_eq!(res.dims(), [1, 1, 1, 1]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [-0.6197] - ); - Ok(()) -} - -/* This test is based on the following script. -import torch -torch.manual_seed(4242) - -t = torch.randn((1, 2, 4, 2)) -w = torch.randn((1, 2, 1, 1)) -print(t.flatten()) -print(w.flatten()) -res = torch.nn.functional.conv2d(t, w) -print(res.flatten()) -*/ -fn conv2d_non_square(dev: &Device) -> Result<()> { - let t = Tensor::new( - &[ - 0.4056f32, -0.8689, -0.0773, -1.5630, -2.8012, -1.5059, 0.3972, 1.0852, 0.4997, 3.0616, - 1.6541, 0.0964, -0.8338, -1.6523, -0.8323, -0.1699, - ], - dev, - )?; - let w = Tensor::new(&[-1.1351f32, 1.3841], dev)?; - let t = t.reshape((1, 2, 4, 2))?; - let w = w.reshape((1, 2, 1, 1))?; - let res = t.conv2d(&w, 0, 1, 1, 1)?; - assert_eq!(res.dims(), [1, 1, 4, 2]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [0.2312, 5.2238, 2.3772, 1.9076, 2.0256, -0.5776, -1.6028, -1.467] - ); - Ok(()) -} - -/* -import torch -torch.manual_seed(4242) - -t = torch.randn((1, 4, 5, 5), requires_grad=True) -w = torch.randn((2, 4, 3, 3), requires_grad=True) -print(t.flatten()) -print(w.flatten()) -res = torch.nn.functional.conv2d(t, w) -print(res.flatten()) -loss = (res ** 2).sum() -print(loss) -loss.backward() -print(t.grad.shape) -print(t.grad.flatten()) -print(w.grad.shape) -print(w.grad.flatten()) - -t.grad.zero_() -w.grad.zero_() -res = torch.nn.functional.conv2d(t, w, stride=2) -print(res.flatten()) -loss = (res ** 2).sum() -print(loss) -loss.backward() -print(t.grad.shape) -print(t.grad[0]) -print(w.grad.shape) -print(w.grad[0]) -*/ -fn conv2d_grad(dev: &Device) -> Result<()> { - use candle_core::Var; - let t = Var::from_slice( - &[ - 0.4056f32, -0.8689, -0.0773, -1.5630, -2.8012, -1.5059, 0.3972, 1.0852, 0.4997, 3.0616, - 1.6541, 0.0964, -0.8338, -1.6523, -0.8323, -0.1699, 0.0823, 0.3526, 0.6843, 0.2395, - 1.2279, -0.9287, -1.7030, 0.1370, 0.6047, 0.3770, -0.6266, 0.3529, 2.2013, -0.6836, - 0.2477, 1.3127, -0.2260, 0.2622, -1.2974, -0.8140, -0.8404, -0.3490, 0.0130, 1.3123, - 1.7569, -0.3956, -1.8255, 0.1727, -0.3538, 2.6941, 1.0529, 0.4219, -0.2071, 1.1586, - 0.4717, 0.3865, -0.5690, -0.5010, -0.1310, 0.7796, 0.6630, -0.2021, 2.6090, 0.2049, - 0.6466, -0.5042, -0.0603, -1.6538, -1.2429, 1.8357, 1.6052, -1.3844, 0.3323, -1.3712, - 0.9634, -0.4799, -0.6451, -0.0840, -1.4247, 0.5512, -0.1747, -0.5509, -0.3742, 0.3790, - -0.4431, -0.4720, -0.7890, 0.2620, 0.7875, 0.5377, -0.6779, -0.8088, 1.9098, 1.2006, - -0.8000, -0.4983, 1.5480, 0.8265, -0.1025, 0.5138, 0.5748, 0.3821, -0.4607, 0.0085, - ], - (1, 4, 5, 5), - dev, - )?; - let w = Var::from_slice( - &[ - -0.9325f32, 0.6451, -0.8537, 0.2378, 0.8764, -0.1832, 0.2987, -0.6488, -0.2273, - -2.4184, -0.1192, -0.4821, -0.5079, -0.5766, -2.4729, 1.6734, 0.4558, 0.2851, 1.1514, - -0.9013, 1.0662, -0.1817, -0.0259, 0.1709, 0.5367, 0.7513, 0.8086, -2.2586, -0.5027, - 0.9141, -1.3086, -1.3343, -1.5669, -0.1657, 0.7958, 0.1432, 0.3896, -0.4501, 0.1667, - 0.0714, -0.0952, 1.2970, -0.1674, -0.3178, 1.0677, 0.3060, 0.7080, 0.1914, 1.1679, - -0.3602, 1.9265, -1.8626, -0.5112, -0.0982, 0.2621, 0.6565, 0.5908, 1.0089, -0.1646, - 1.8032, -0.6286, 0.2016, -0.3370, 1.2555, 0.8009, -0.6488, -0.4652, -1.5685, 1.5860, - 0.5583, 0.4623, 0.6026, - ], - (2, 4, 3, 3), - dev, - )?; - let res = t.conv2d(&w, 0, 1, 1, 1)?; - let loss = res.sqr()?.sum_all()?; - assert_eq!(test_utils::to_vec0_round(&loss, 2)?, 741.12f32); - let grads = loss.backward()?; - let grad_t = grads.get(&t).unwrap(); - let grad_w = grads.get(&w).unwrap(); - assert_eq!(grad_t.dims(), [1, 4, 5, 5]); - assert_eq!(grad_w.dims(), [2, 4, 3, 3]); - assert_eq!( - test_utils::to_vec1_round(&grad_t.flatten_all()?, 2)?, - [ - 9.29, -2.84, -5.71, 3.38, -7.71, -19.15, 7.02, 29.1, 9.34, 34.73, -22.87, 24.35, - -39.88, -14.01, 21.08, 9.94, 13.63, -34.68, 11.21, -6.26, 7.72, -6.32, -16.64, -1.08, - -20.22, 21.73, -0.37, -4.06, 5.82, -3.65, -30.73, 14.55, 87.7, 31.6, 4.53, -89.78, - -75.37, -57.43, -7.56, 92.96, 18.79, -4.63, -159.75, -42.47, -47.26, 52.88, 37.32, - 49.0, 12.82, 2.01, -8.98, 20.18, 16.62, 12.06, 15.38, 20.0, 2.57, -15.22, 72.62, - -10.75, 2.25, -31.2, 3.75, -0.2, 9.76, -0.68, 5.21, -40.44, -22.59, -61.61, 17.28, - 20.41, 37.55, 5.23, 6.81, 23.54, 23.62, -9.99, -9.13, 4.87, -35.06, -26.1, 63.48, - 25.81, -39.21, -70.68, -46.96, 2.33, 41.81, 82.42, -28.63, -11.78, -35.33, -10.28, - -28.57, -9.13, 7.21, -9.05, -9.62, -11.25 - ] - ); - assert_eq!( - test_utils::to_vec1_round(&grad_w.flatten_all()?, 2)?, - [ - -28.92, -22.88, -141.23, 73.35, 61.07, 47.81, -20.0, -73.71, -41.82, -13.59, 21.5, - 28.72, 28.57, -46.85, -90.19, 143.61, 16.68, 7.43, 18.88, -90.81, -20.29, 54.79, 82.63, - 22.94, 77.81, -16.39, -13.2, 9.34, -40.39, -26.62, 5.33, -60.91, 9.09, -59.37, 7.08, - 58.64, 5.55, 20.52, 2.5, -17.25, -6.8, 22.21, 30.15, -7.52, -37.46, 5.67, 22.58, 9.03, - 47.05, 17.61, 37.31, -98.13, -14.61, -4.8, -6.36, 44.69, 23.34, 8.37, -13.52, 80.05, - -34.24, -16.36, -12.31, 1.92, -33.62, -14.1, -49.23, -7.39, 11.5, -9.98, 9.66, 29.6 - ] - ); - - // Same as before but with stride. - let res = t.conv2d(&w, 0, 2, 1, 1)?; - let loss = res.sqr()?.sum_all()?; - assert_eq!(test_utils::to_vec0_round(&loss, 2)?, 277.16f32); - let grads = loss.backward()?; - let grad_t = grads.get(&t).unwrap(); - let grad_w = grads.get(&w).unwrap(); - assert_eq!(grad_t.dims(), [1, 4, 5, 5]); - assert_eq!(grad_w.dims(), [2, 4, 3, 3]); - assert_eq!( - test_utils::to_vec3_round(&grad_t.i(0)?, 2)?, - [ - [ - [9.29, -7.03, 0.94, 3.49, -7.71], - [-1.8, -7.82, 8.9, 8.46, 7.43], - [-25.84, 22.09, -19.27, -0.22, 1.69], - [4.02, 18.53, -18.37, 2.3, -24.51], - [7.72, -9.68, -12.34, 5.6, -20.22] - ], - [ - [21.73, 3.39, -18.27, 3.86, -3.65], - [8.25, 3.73, 30.73, -8.61, -11.93], - [-72.15, -15.36, -17.53, -12.32, -1.61], - [-22.32, -7.79, -91.82, 6.44, -37.69], - [52.88, 14.44, 42.75, 9.88, 2.01] - ], - [ - [-8.98, 9.91, 6.75, -4.68, 15.38], - [4.93, -0.33, 9.94, -1.46, 14.78], - [13.62, -30.63, 3.96, -3.58, -4.48], - [-14.13, 1.19, -34.43, 3.08, -33.83], - [17.28, 12.94, 31.83, -3.35, 6.81] - ], - [ - [23.54, 6.98, -24.52, 0.52, 4.87], - [9.65, 6.18, 1.71, -25.23, -4.93], - [-54.99, -23.66, 3.19, -3.73, 18.58], - [-21.35, -10.39, -39.88, 28.73, -30.76], - [-9.13, 11.12, -14.0, -8.23, -11.25] - ] - ] - ); - assert_eq!( - test_utils::to_vec3_round(&grad_w.i(0)?, 2)?, - [ - [ - [28.34, -7.91, -45.75], - [21.03, 3.86, 29.86], - [0.72, -36.58, -35.28] - ], - [ - [-16.04, 11.53, -16.38], - [29.62, -16.32, -48.35], - [57.5, 28.29, 25.81] - ], - [ - [2.93, -19.6, 1.57], - [27.15, 53.88, -24.64], - [12.74, -22.6, -26.2] - ], - [ - [-0.18, -14.86, -6.82], - [-19.55, -2.72, 45.9], - [-2.54, 36.97, 27.11] - ] - ] - ); - - // Replicate the issue from https://github.com/huggingface/candle/issues/1212 - let res = t.i((.., .., 0..4, 0..4))?.conv2d(&w, 0, 2, 1, 1)?; - let loss = res.sqr()?.sum_all()?; - assert_eq!(test_utils::to_vec0_round(&loss, 2)?, 21.12f32); - let grads = loss.backward()?; - let grad_t = grads.get(&t).unwrap(); - let grad_w = grads.get(&w).unwrap(); - assert_eq!(grad_t.dims(), [1, 4, 5, 5]); - assert_eq!(grad_w.dims(), [2, 4, 3, 3]); - assert_eq!( - test_utils::to_vec3_round(&grad_t.i(0)?, 2)?, - [ - [ - [9.29, -7.03, 7.87, 0.0, 0.0], - [-1.8, -7.82, 5.9, 0.0, 0.0], - [-3.12, 4.49, 5.52, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.0] - ], - [ - [21.73, 3.39, 4.77, 0.0, 0.0], - [8.25, 3.73, 27.61, 0.0, 0.0], - [-20.55, -5.61, -2.77, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.0] - ], - [ - [-8.98, 9.91, -7.15, 0.0, 0.0], - [4.93, -0.33, 4.56, 0.0, 0.0], - [-6.7, -5.76, -8.05, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.0] - ], - [ - [23.54, 6.98, -10.0, 0.0, 0.0], - [9.65, 6.18, 18.72, 0.0, 0.0], - [3.29, -5.27, 0.79, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.0] - ] - ] - ); - assert_eq!( - test_utils::to_vec3_round(&grad_w.i(0)?, 2)?, - [ - [ - [-3.47, 7.44, 0.66], - [12.89, -3.4, -9.29], - [-14.16, -0.83, 7.14] - ], - [ - [-3.23, 5.37, -3.02], - [-2.12, -11.24, 1.94], - [6.97, 7.2, 2.99] - ], - [ - [-4.04, -3.31, 4.87], - [-6.68, -5.68, 1.73], - [-5.54, 4.32, 0.52] - ], - [[-4.72, 1.5, 4.72], [3.79, 4.04, 6.76], [-4.6, 5.8, 6.93]] - ] - ); - - Ok(()) -} - -test_device!(conv1d, conv1d_cpu, conv1d_gpu, conv1d_metal); -test_device!( - conv1d_small, - conv1d_small_cpu, - conv1d_small_gpu, - conv1d_small_metal -); -test_device!(conv2d, conv2d_cpu, conv2d_gpu, conv2d_metal); -test_device!( - conv2d_non_square, - conv2d_non_square_cpu, - conv2d_non_square_gpu, - conv2d_non_square_metal -); -test_device!( - conv2d_small, - conv2d_small_cpu, - conv2d_small_gpu, - conv2d_small_metal -); -test_device!( - conv2d_smaller, - conv2d_smaller_cpu, - conv2d_smaller_gpu, - conv2d_smaller_metal -); -test_device!( - conv2d_grad, - conv2d_grad_cpu, - conv2d_grad_gpu, - conv2_grad_metal -); diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/custom_op_tests.rs b/v2/dependency-patches/candle-core-0.4.1/tests/custom_op_tests.rs deleted file mode 100644 index cff0aebe78..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/tests/custom_op_tests.rs +++ /dev/null @@ -1,114 +0,0 @@ -use candle_core::backend::BackendStorage; -use candle_core::cpu_backend; -use candle_core::test_utils::to_vec1_round; -use candle_core::{CpuStorage, CustomOp1, DType, Device, Error, Layout, Result, Shape, Tensor}; - -fn fwd(v: T, alpha: f64) -> T { - if v.is_sign_positive() { - v - } else { - let alpha = T::from(alpha).unwrap_or(T::nan()); - (v.exp() - T::one()) * alpha - } -} - -struct Elu { - alpha: f64, -} - -impl CustomOp1 for Elu { - fn name(&self) -> &'static str { - "elu" - } - - fn cpu_fwd(&self, s: &CpuStorage, l: &Layout) -> Result<(CpuStorage, Shape)> { - let storage = candle_core::map_dtype!( - "elu", - s, - |s| cpu_backend::unary_map(s, l, |v| fwd(v, self.alpha)), - (BF16, F16, F32, F64) - ); - Ok((storage, l.shape().clone())) - } -} - -#[test] -fn custom_op1_no_backward() -> Result<()> { - let cpu = &Device::Cpu; - let t = Tensor::arange(0u32, 12u32, cpu)?.to_dtype(DType::F32)?; - let t = (t - 5.)?; - let elu_t = t.apply_op1_no_bwd(&Elu { alpha: 1. })?; - assert_eq!( - to_vec1_round(&elu_t, 4)?, - &[-0.9933, -0.9817, -0.9502, -0.8647, -0.6321, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0] - ); - Ok(()) -} - -// Define a similar struct as Elu but with backward support. -fn bwd(v: T, alpha: f64) -> T { - if v.is_sign_positive() { - T::one() - } else { - let alpha = T::from(alpha).unwrap_or(T::nan()); - v.exp() * alpha - } -} - -struct EluBackward { - alpha: f64, -} - -impl CustomOp1 for EluBackward { - fn name(&self) -> &'static str { - "elu-bwd" - } - - fn cpu_fwd(&self, s: &CpuStorage, l: &Layout) -> Result<(CpuStorage, Shape)> { - let storage = candle_core::map_dtype!( - "elu-bwd", - s, - |s| cpu_backend::unary_map(s, l, |v| bwd(v, self.alpha)), - (BF16, F16, F32, F64) - ); - Ok((storage, l.shape().clone())) - } -} - -struct EluWithBackward(Elu); - -impl EluWithBackward { - fn new(alpha: f64) -> Self { - Self(Elu { alpha }) - } -} - -impl CustomOp1 for EluWithBackward { - fn name(&self) -> &'static str { - "elu" - } - - fn cpu_fwd(&self, s: &CpuStorage, l: &Layout) -> Result<(CpuStorage, Shape)> { - self.0.cpu_fwd(s, l) - } - - fn bwd(&self, arg: &Tensor, _res: &Tensor, grad_res: &Tensor) -> Result> { - let alpha = self.0.alpha; - let bwd = arg.apply_op1(EluBackward { alpha })?; - Ok(Some(grad_res.mul(&bwd)?)) - } -} - -#[test] -fn custom_op1_with_backward() -> Result<()> { - let cpu = &Device::Cpu; - let t = candle_core::Var::new(&[-2f32, 0f32, 2f32], cpu)?; - let elu_t = t.apply_op1(EluWithBackward::new(2.))?; - assert_eq!(to_vec1_round(&elu_t, 4)?, &[-1.7293, 0.0, 2.0]); - - let grads = elu_t.backward()?; - let grad_x = grads.get(&t).unwrap(); - assert_eq!(to_vec1_round(grad_x, 4)?, [0.2707, 1.0, 1.0]); - - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/display_tests.rs b/v2/dependency-patches/candle-core-0.4.1/tests/display_tests.rs deleted file mode 100644 index 8ab8bb6ab7..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/tests/display_tests.rs +++ /dev/null @@ -1,84 +0,0 @@ -use anyhow::Result; -use candle_core::{DType, Device::Cpu, Tensor}; - -#[test] -fn display_scalar() -> Result<()> { - let t = Tensor::new(1234u32, &Cpu)?; - let s = format!("{t}"); - assert_eq!(&s, "[1234]\nTensor[[], u32]"); - let t = t.to_dtype(DType::F32)?.neg()?; - let s = format!("{}", (&t / 10.0)?); - assert_eq!(&s, "[-123.4000]\nTensor[[], f32]"); - let s = format!("{}", (&t / 1e8)?); - assert_eq!(&s, "[-1.2340e-5]\nTensor[[], f32]"); - let s = format!("{}", (&t * 1e8)?); - assert_eq!(&s, "[-1.2340e11]\nTensor[[], f32]"); - let s = format!("{}", (&t * 0.)?); - assert_eq!(&s, "[0.]\nTensor[[], f32]"); - Ok(()) -} - -#[test] -fn display_vector() -> Result<()> { - let t = Tensor::new::<&[u32; 0]>(&[], &Cpu)?; - let s = format!("{t}"); - assert_eq!(&s, "[]\nTensor[[0], u32]"); - let t = Tensor::new(&[0.1234567, 1.0, -1.2, 4.1, f64::NAN], &Cpu)?; - let s = format!("{t}"); - assert_eq!( - &s, - "[ 0.1235, 1.0000, -1.2000, 4.1000, NaN]\nTensor[[5], f64]" - ); - let t = (Tensor::ones(50, DType::F32, &Cpu)? * 42.)?; - let s = format!("\n{t}"); - let expected = r#" -[42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., - 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., - 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., - 42., 42.] -Tensor[[50], f32]"#; - assert_eq!(&s, expected); - let t = (Tensor::ones(11000, DType::F32, &Cpu)? * 42.)?; - let s = format!("{t}"); - assert_eq!( - &s, - "[42., 42., 42., ..., 42., 42., 42.]\nTensor[[11000], f32]" - ); - Ok(()) -} - -#[test] -fn display_multi_dim() -> Result<()> { - let t = (Tensor::ones((200, 100), DType::F32, &Cpu)? * 42.)?; - let s = format!("\n{t}"); - let expected = r#" -[[42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - ... - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.]] -Tensor[[200, 100], f32]"#; - assert_eq!(&s, expected); - let t = t.reshape(&[2, 1, 1, 100, 100])?; - let t = format!("\n{t}"); - let expected = r#" -[[[[[42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - ... - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.]]]], - [[[[42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - ... - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.]]]]] -Tensor[[2, 1, 1, 100, 100], f32]"#; - assert_eq!(&t, expected); - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/fortran_tensor_3d.pth b/v2/dependency-patches/candle-core-0.4.1/tests/fortran_tensor_3d.pth deleted file mode 100644 index bd50b03d9cd74135fa6f53e2bf39da2f7ac0879d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1486 zcmbVMON-M`6uxcRw3C_f{b*;vl}H8cq_1fM4rq~Vu3&5tikp&2uAN|L<|elT3Zfes z`~xoi8LnNr)ZgHra3gq5ax?akg5!b1*QDoseErTjwWv}cq+BK|u|vw_We|$co{j~Z zMnO1kxqGfH?E3L5-|QF23yAmmx15YFK)0-sEj;IlAasuEF|D;O?2<9xF#&?}#<-dQ*_3!wA}nJZmn5c(c}+fVNvudj z=N{boHv9_DnCqRZ5+lR86Bb6WEM~uRZtLQCw=`7nI)k}hL5W)3v?BD(;LAu`VHkz$&w_)rOI+dOSykWlHOd3LLTKWc|6b;pN%gI@N0_} N Result<()> { - let x = Var::new(&[3f32, 1., 4.], device)?; - let x = x.as_tensor(); - let y = (((x * x)? + x * 5f64)? + 4f64)?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!(x.to_vec1::()?, [3., 1., 4.]); - // y = x^2 + 5.x + 4 - assert_eq!(y.to_vec1::()?, [28., 10., 40.]); - // dy/dx = 2.x + 5 - assert_eq!(grad_x.to_vec1::()?, [11., 7., 13.]); - Ok(()) -} - -fn sum_grad(device: &Device) -> Result<()> { - let x = Var::new(&[3f32, 1., 4.], device)?; - let x = x.as_tensor(); - let y = (x.sqr()?.sum_keepdim(0)? * 2.)?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!(y.to_vec1::()?, [52.]); - // y = 2.x^2 so dy/dx = 4.x - assert_eq!(grad_x.to_vec1::()?, &[12., 4., 16.]); - - // Same test as before but squeezing on the last dimension. - let y = (x.sqr()?.sum_keepdim(0)? * 2.)?.squeeze(0)?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!(y.to_scalar::()?, 52.); - // y = 2.x^2 so dy/dx = 4.x - assert_eq!(grad_x.to_vec1::()?, &[12., 4., 16.]); - Ok(()) -} - -fn matmul_grad(device: &Device) -> Result<()> { - let data: Vec<_> = (0..12).map(|i| i as f32).collect(); - let x = Var::from_slice(&data, (2, 2, 3), device)?; - let data: Vec<_> = (0..12).map(|i| i as f32).collect(); - let y = Var::from_slice(&data, (2, 3, 2), device)?; - let c = x.matmul(&y)?; - let grads = c.backward()?; - let grad_x = grads.get(&x).context("no grad for x")?; - let grad_y = grads.get(&y).context("no grad for y")?; - assert_eq!(grad_x.shape(), &Shape::from((2, 2, 3))); - assert_eq!(grad_y.shape(), &Shape::from((2, 3, 2))); - assert_eq!( - &*grad_x.to_vec3::()?, - &[ - [[1., 5., 9.], [1., 5., 9.]], - [[13., 17., 21.], [13., 17., 21.]] - ] - ); - assert_eq!( - &*grad_y.to_vec3::()?, - &[ - [[3., 3.], [5., 5.], [7., 7.]], - [[15., 15.], [17., 17.], [19., 19.]] - ] - ); - Ok(()) -} - -// The simplest gradient descent, using scalar variable. -fn grad_descent(device: &Device) -> Result<()> { - let x = Var::new(0f32, device)?; - let learning_rate = 0.1; - for _step in 0..100 { - let xt = x.as_tensor(); - let c = ((xt - 4.2)? * (xt - 4.2)?)?; - let grads = c.backward()?; - let x_grad = grads.get(&x).context("no grad for x")?; - x.set(&(xt - x_grad * learning_rate)?)? - } - assert_eq!(x.to_scalar::()?, 4.199999); - Ok(()) -} - -fn unary_grad(device: &Device) -> Result<()> { - let x = Var::new(&[3f32, 1., 4., 0.15], device)?; - let x = x.as_tensor(); - let y = (x.log()? + 1.)?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!( - test_utils::to_vec1_round(&y, 4)?, - [2.0986, 1.0, 2.3863, -0.8971] - ); - assert_eq!( - test_utils::to_vec1_round(grad_x, 4)?, - [0.3333, 1.0, 0.25, 6.6667] - ); - let y = x.exp()?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!( - y.to_vec1::()?, - [20.085537, 2.7182817, 54.59815, 1.1618342] - ); - assert_eq!( - grad_x.to_vec1::()?, - [20.085537, 2.7182817, 54.59815, 1.1618342] - ); - let y = x.exp()?.sqr()?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!( - y.to_vec1::()?, - [403.4288, 7.3890557, 2980.9578, 1.3498588] - ); - // exp(x)^2 = exp(2*x) - assert_eq!( - grad_x.to_vec1::()?, - [806.8576, 14.778111, 5961.9155, 2.6997175] - ); - let y = x.sin()?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!( - test_utils::to_vec1_round(&y, 4)?, - [0.1411, 0.8415, -0.7568, 0.1494], - ); - assert_eq!( - test_utils::to_vec1_round(grad_x, 4)?, - [-0.99, 0.5403, -0.6536, 0.9888], - ); - let y = x.cos()?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!( - test_utils::to_vec1_round(&y, 4)?, - [-0.99, 0.5403, -0.6536, 0.9888], - ); - assert_eq!( - test_utils::to_vec1_round(grad_x, 4)?, - [-0.1411, -0.8415, 0.7568, -0.1494], - ); - let y = x.sqr()?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!(y.to_vec1::()?, [9.0, 1.0, 16.0, 0.0225]); - assert_eq!(grad_x.to_vec1::()?, [6.0, 2.0, 8.0, 0.3]); - let y = x.sqr()?.sqrt()?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!(y.to_vec1::()?, [3.0, 1.0, 4.0, 0.15]); - assert_eq!(test_utils::to_vec1_round(grad_x, 4)?, [1.0, 1.0, 1.0, 1.0]); - let y = x.neg()?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!(y.to_vec1::()?, [-3.0, -1.0, -4.0, -0.15]); - assert_eq!(grad_x.to_vec1::()?, [-1.0, -1.0, -1.0, -1.0]); - let y = x.affine(0.2, 1.)?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!(y.to_vec1::()?, [1.6, 1.2, 1.8, 1.03]); - assert_eq!(grad_x.to_vec1::()?, [0.2, 0.2, 0.2, 0.2]); - let y = Tensor::new(1f32, device)?.broadcast_div(x)?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!( - test_utils::to_vec1_round(&y, 4)?, - [0.3333, 1.0, 0.25, 6.6667] - ); - assert_eq!( - grad_x.to_vec1::()?, - [-0.11111111, -1.0, -0.0625, -44.444443], - ); - let y = x.broadcast_div(&Tensor::new(0.5f32, device)?)?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!(y.to_vec1::()?, [6., 2., 8., 0.3]); - assert_eq!(grad_x.to_vec1::()?, [2., 2., 2., 2.]); - - let x = Var::new(&[3f32, 1., 4., 0.15], device)?; - let y = x.powf(2.5)?; - let grads = y.backward()?; - let grad_x = grads.get(&x).context("no grad for x")?; - assert_eq!(test_utils::to_vec1_round(&y, 2)?, [15.59, 1.0, 32.0, 0.01]); - assert_eq!( - test_utils::to_vec1_round(grad_x, 2)?, - [12.99, 2.5, 20.0, 0.15] - ); - - let y = x.tanh()?; - let grads = y.backward()?; - let grad_x = grads.get(&x).context("no grad for x")?; - assert_eq!(test_utils::to_vec1_round(&y, 2)?, [1.0, 0.76, 1.0, 0.15]); - assert_eq!( - test_utils::to_vec1_round(grad_x, 2)?, - [0.01, 0.42, 0.0, 0.98], - ); - - // testing compared to pytorch nn.GELU(approximate = 'tanh') - let y = x.gelu()?; - let grads = y.backward()?; - let grad_x = grads.get(&x).context("no grad for x")?; - assert_eq!( - test_utils::to_vec1_round(&y, 4)?, - [2.9964, 0.8412, 3.9999, 0.0839] - ); - assert_eq!( - test_utils::to_vec1_round(grad_x, 4)?, - [1.0116, 1.0830, 1.0003, 0.6188], - ); - - // Testing compared to pytorch torch.erf - // - // import torch - // x = torch.tensor([3.0, 1.0, 4.0, 0.15], requires_grad=True) - // y = x.erf() - // print(y) - // loss = y.sum() - // loss.backward() - // print(x.grad) - let y = x.erf()?; - let grads = y.backward()?; - let grad_x = grads.get(&x).context("no grad for x")?; - assert_eq!(test_utils::to_vec1_round(&y, 4)?, [1.0, 0.8427, 1.0, 0.168]); - assert_eq!( - test_utils::to_vec1_round(grad_x, 4)?, - [0.0001, 0.4151, 0.0, 1.1033], - ); - - // Testing compared to pytorch nn.GELU(approximate = 'none') - // - // import torch - // import torch.nn.functional as F - // x = torch.tensor([3.0, 1.0, 4.0, 0.15], requires_grad=True) - // y = F.gelu(x, approximate='none') - // print(y) - // loss = y.sum() - // loss.backward() - // print(x.grad) - let y = x.gelu_erf()?; - let grads = y.backward()?; - let grad_x = grads.get(&x).context("no grad for x")?; - assert_eq!( - test_utils::to_vec1_round(&y, 4)?, - [2.9960, 0.8413, 3.9999, 0.0839] - ); - assert_eq!( - test_utils::to_vec1_round(grad_x, 4)?, - [1.0119, 1.0833, 1.0005, 0.6188], - ); - - // Testing compared to pytorch elu - // - // import torch - // import torch.nn.functional as F - // x = torch.tensor([-1.0, 0.0, -2.0, 3.0], requires_grad=True) - // y = F.elu(x, alpha=2.0) - // print(y) - // loss = y.min - // loss = y.sum() - // loss.backward() - // print(x.grad) - let elu_x = Var::new(&[-1.0f32, 0., -2., 3.], device)?; - let y = elu_x.elu(2.)?; - let grads = y.backward()?; - let grad_x = grads.get(&elu_x).context("no grad for x")?; - assert_eq!( - test_utils::to_vec1_round(&y, 4)?, - [-1.2642, 0.0000, -1.7293, 3.0000] - ); - assert_eq!( - test_utils::to_vec1_round(grad_x, 4)?, - [0.7358, 2.0000, 0.2707, 1.0000] - ); - - // testing compared to pytorch nn.Silu() - let y = x.silu()?; - let grads = y.backward()?; - let grad_x = grads.get(&x).context("no grad for x")?; - assert_eq!( - test_utils::to_vec1_round(&y, 4)?, - [2.8577, 0.7311, 3.9281, 0.0806] - ); - assert_eq!( - test_utils::to_vec1_round(grad_x, 4)?, - [1.0881, 0.9277, 1.0527, 0.5747], - ); - - if device.is_cpu() { - let x = Var::new(&[[[1f32, 2., 3.], [4., 5., 6.], [7., 8., 9.]]], device)?; - let y = x.interpolate1d(12)?.reshape(36)?; - - let z = Tensor::new( - &[ - 1_f32, 02., 03., 04., 05., 06., 07., 08., 09., 10., 11., 12., 13., 14., 15., 16., - 17., 18., 19., 20., 21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., - 33., 34., 35., 36., - ], - device, - )?; - - let loss = y.unsqueeze(1)?.transpose(0, 1)?.matmul(&z.unsqueeze(1)?)?; - let grads = loss.backward()?; - let grad_x = grads.get(&x).context("no grad for x")?; - - assert_eq!( - test_utils::to_vec3_round(grad_x, 4)?, - [[[10_f32, 26., 42.], [58., 74., 90.], [106., 122., 138.]]] - ); - } - - // manually checked: see comments - let x = Var::new(&[[[[1f32, 2., 3.], [4., 5., 6.], [7., 8., 9.]]]], device)?; - let y = x.interpolate2d(6, 6)?.reshape(36)?; - - let z = Tensor::new( - &[ - 1_f32, 02., 03., 04., 05., 06., 07., 08., 09., 10., 11., 12., 13., 14., 15., 16., 17., - 18., 19., 20., 21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., 33., 34., - 35., 36., - ], - device, - )?; - // gradient should be - // row 1 - // 1+2+7+8 = 18 - // 3+4+9+10 = 26 - // 5+6+11+12 = 34 - // row 2 - // 13+14+19+20 = 66 - // 15+16+21+22 = 74 - // 17+18+23+24 = 82 - // row 3 - // 25+26+31+32 = 114 - // 27+28+33+34 = 122 - // 29+30+35+36 = 130 - let loss = y.unsqueeze(1)?.transpose(0, 1)?.matmul(&z.unsqueeze(1)?)?; - - let grads = loss.backward()?; - - let grad_x = grads.get(&x).context("no grad for x")?; - assert_eq!( - test_utils::to_vec2_round(&grad_x.flatten(0, 2)?, 4)?, - [[18_f32, 26., 34.], [66., 74., 82.], [114., 122., 130.]] - ); - - // manually checked: see comments - let x = Var::new(&[[[[1f32, 2.], [4., 5.]]]], device)?; - let y = x.interpolate2d(6, 6)?.reshape(36)?; - - let z = Tensor::new( - &[ - 1_f32, 02., 03., 04., 05., 06., 07., 08., 09., 10., 11., 12., 13., 14., 15., 16., 17., - 18., 19., 20., 21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., 33., 34., - 35., 36., - ], - device, - )?; - // gradient should be - // row 1 - // 1+2+3+7+8+9+13+14+15 = 72 - // 4+5+6+10+11+12+16+17+18 = 99 - // row 2 - // 19+20+21+25+26+27+31+32+33 = 234 - // 22+23+24+28+29+30+34+35+36 = 243 - let loss = y.unsqueeze(1)?.transpose(0, 1)?.matmul(&z.unsqueeze(1)?)?; - - let grads = loss.backward()?; - - let grad_x = grads.get(&x).context("no grad for x")?; - assert_eq!( - test_utils::to_vec2_round(&grad_x.flatten(0, 2)?, 4)?, - [[72_f32, 99.], [234., 261.]] - ); - - // manually checked: see comments - let x = Var::new(&[[[[1f32, 2.], [4., 5.]], [[6f32, 7.], [8., 9.]]]], device)?; - - let y = x.interpolate2d(4, 4)?.reshape(32)?; - - #[rustfmt::skip] - let z = Tensor::new( - &[ - 1_f32, 02., 03., 04., - 05., 06., 07., 08., - 09., 10., 11., 12., - 13., 14., 15., 16., - 17., 18., 19., 20., - 21., 22., 23., 24., - 25., 26., 27., 28., - 29., 30., 31., 32. - ], - device, - )?; - // gradient should be - // m1r1 - // 1+2+5+6=14 - // 3+4+7+8=22 - // m1r2 - // 9+10+13+14=46 - // 11+12+15+16=54 - // m2r1 - // 17+18+21+22=78 - // 19+20+23+24=86 - // m2r2 - // 25+26+29+30=110 - // 27+28+31+32=118 - let loss = y.unsqueeze(1)?.transpose(0, 1)?.matmul(&z.unsqueeze(1)?)?; - - let grads = loss.backward()?; - - let grad_x = grads.get(&x).context("no grad for x")?; - - assert_eq!( - test_utils::to_vec3_round(&grad_x.flatten(0, 1)?, 4)?, - [[[14_f32, 22.], [46., 54.]], [[78., 86.], [110., 118.]]] - ); - - // manually checked: see comments - let x = Var::new( - &[[[[1f32, 2.], [4., 5.]]], [[[6f32, 7.], [8., 9.]]]], - device, - )?; - - let y = x.interpolate2d(4, 4)?.reshape(32)?; - - #[rustfmt::skip] - let z = Tensor::new( - &[ - 1_f32, 02., 03., 04., - 05., 06., 07., 08., - 09., 10., 11., 12., - 13., 14., 15., 16., - 17., 18., 19., 20., - 21., 22., 23., 24., - 25., 26., 27., 28., - 29., 30., 31., 32. - ], - device, - )?; - // gradient should be - // m1r1 - // 1+2+5+6=14 - // 3+4+7+8=22 - // m1r2 - // 9+10+13+14=46 - // 11+12+15+16=54 - // m2r1 - // 17+18+21+22=78 - // 19+20+23+24=86 - // m2r2 - // 25+26+29+30=110 - // 27+28+31+32=118 - let loss = y.unsqueeze(1)?.transpose(0, 1)?.matmul(&z.unsqueeze(1)?)?; - - let grads = loss.backward()?; - - let grad_x = grads.get(&x).context("no grad for x")?; - - assert_eq!( - test_utils::to_vec3_round(&grad_x.flatten(0, 1)?, 4)?, - [[[14_f32, 22.], [46., 54.]], [[78., 86.], [110., 118.]]] - ); - Ok(()) -} - -fn binary_grad(device: &Device) -> Result<()> { - let x = Var::new(&[3f32, 1., -4., -1.], device)?; - let x = x.as_tensor(); - // leaky relu - let y = x.maximum(&(x * 0.1)?)?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!(x.to_vec1::()?, [3., 1., -4., -1.]); - assert_eq!(y.to_vec1::()?, [3., 1., -0.4, -0.1]); - assert_eq!(grad_x.to_vec1::()?, [1., 1., 0.1, 0.1]); - - let y = x.minimum(&(x * 0.1)?)?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!(y.to_vec1::()?, [0.3, 0.1, -4., -1.]); - assert_eq!(grad_x.to_vec1::()?, [0.1, 0.1, 1., 1.]); - - // This one is easy to mess up, we want the gradient to be one as it is the identity function. - let y = x.minimum(x)?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!(y.to_vec1::()?, [3., 1., -4., -1.]); - assert_eq!(grad_x.to_vec1::()?, [1., 1., 1., 1.]); - - let x_var = Var::new(&[3f32, 1., -4., -1., 5., 9.], device)?; - let x = x_var.as_tensor(); - let y_var = Var::new(&[2f32, 7., 1.], device)?; - let y = y_var.as_tensor(); - - let ss = x - .reshape((2, 3))? - .slice_scatter0(&y.reshape((1, 3))?, 1)? - .sqr()?; - let grads = ss.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - let grad_y = grads.get(y).context("no grad for y")?; - assert_eq!(ss.to_vec2::()?, [[9., 1., 16.], [4., 49., 1.]]); - assert_eq!(grad_x.to_vec1::()?, [6.0, 2.0, -8.0, 0.0, 0.0, 0.0]); - assert_eq!(grad_y.to_vec1::()?, [4.0, 14.0, 2.0]); - Ok(()) -} - -test_device!( - simple_grad, - simple_grad_cpu, - simple_grad_gpu, - simple_grad_metal -); -test_device!(sum_grad, sum_grad_cpu, sum_grad_gpu, sum_grad_metal); -test_device!( - matmul_grad, - matmul_grad_cpu, - matmul_grad_gpu, - matmul_grad_metal -); -test_device!( - grad_descent, - grad_descent_cpu, - grad_descent_gpu, - grad_descent_metal -); -test_device!(unary_grad, unary_grad_cpu, unary_grad_gpu, unary_grad_metal); -test_device!( - binary_grad, - binary_grad_cpu, - binary_grad_gpu, - binary_grad_metal -); diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/indexing_tests.rs b/v2/dependency-patches/candle-core-0.4.1/tests/indexing_tests.rs deleted file mode 100644 index 047205a31f..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/tests/indexing_tests.rs +++ /dev/null @@ -1,122 +0,0 @@ -use anyhow::Result; -use candle_core::{Device, IndexOp, Tensor}; - -#[test] -fn integer_index() -> Result<()> { - let dev = Device::Cpu; - - let tensor = Tensor::arange(0u32, 2 * 3, &dev)?.reshape((2, 3))?; - let result = tensor.i(1)?; - assert_eq!(result.dims(), &[3]); - assert_eq!(result.to_vec1::()?, &[3, 4, 5]); - - let result = tensor.i((.., 2))?; - assert_eq!(result.dims(), &[2]); - assert_eq!(result.to_vec1::()?, &[2, 5]); - - Ok(()) -} - -#[test] -fn range_index() -> Result<()> { - let dev = Device::Cpu; - // RangeFull - let tensor = Tensor::arange(0u32, 2 * 3, &dev)?.reshape((2, 3))?; - let result = tensor.i(..)?; - assert_eq!(result.dims(), &[2, 3]); - assert_eq!(result.to_vec2::()?, &[[0, 1, 2], [3, 4, 5]]); - - // Range - let tensor = Tensor::arange(0u32, 4 * 3, &dev)?.reshape((4, 3))?; - let result = tensor.i(1..3)?; - assert_eq!(result.dims(), &[2, 3]); - assert_eq!(result.to_vec2::()?, &[[3, 4, 5], [6, 7, 8]]); - - // RangeFrom - let result = tensor.i(2..)?; - assert_eq!(result.dims(), &[2, 3]); - assert_eq!(result.to_vec2::()?, &[[6, 7, 8], [9, 10, 11]]); - - // RangeTo - let result = tensor.i(..2)?; - assert_eq!(result.dims(), &[2, 3]); - assert_eq!(result.to_vec2::()?, &[[0, 1, 2], [3, 4, 5]]); - - // RangeInclusive - let result = tensor.i(1..=2)?; - assert_eq!(result.dims(), &[2, 3]); - assert_eq!(result.to_vec2::()?, &[[3, 4, 5], [6, 7, 8]]); - - // RangeTo - let result = tensor.i(..1)?; - assert_eq!(result.dims(), &[1, 3]); - assert_eq!(result.to_vec2::()?, &[[0, 1, 2]]); - - // RangeToInclusive - let result = tensor.i(..=1)?; - assert_eq!(result.dims(), &[2, 3]); - assert_eq!(result.to_vec2::()?, &[[0, 1, 2], [3, 4, 5]]); - - // Empty range - let result = tensor.i(1..1)?; - assert_eq!(result.dims(), &[0, 3]); - let empty: [[u32; 3]; 0] = []; - assert_eq!(result.to_vec2::()?, &empty); - - // Similar to PyTorch, allow empty ranges when the computed length is negative. - #[allow(clippy::reversed_empty_ranges)] - let result = tensor.i(1..0)?; - assert_eq!(result.dims(), &[0, 3]); - let empty: [[u32; 3]; 0] = []; - assert_eq!(result.to_vec2::()?, &empty); - Ok(()) -} - -#[test] -fn index_3d() -> Result<()> { - let tensor = Tensor::from_iter(0..24u32, &Device::Cpu)?.reshape((2, 3, 4))?; - assert_eq!(tensor.i((0, 0, 0))?.to_scalar::()?, 0); - assert_eq!(tensor.i((1, 0, 0))?.to_scalar::()?, 12); - assert_eq!(tensor.i((0, 1, 0))?.to_scalar::()?, 4); - assert_eq!(tensor.i((0, 1, 3))?.to_scalar::()?, 7); - assert_eq!(tensor.i((0..2, 0, 0))?.to_vec1::()?, &[0, 12]); - assert_eq!( - tensor.i((0..2, .., 0))?.to_vec2::()?, - &[[0, 4, 8], [12, 16, 20]] - ); - assert_eq!( - tensor.i((..2, .., 3))?.to_vec2::()?, - &[[3, 7, 11], [15, 19, 23]] - ); - assert_eq!(tensor.i((1, .., 3))?.to_vec1::()?, &[15, 19, 23]); - Ok(()) -} - -#[test] -fn slice_assign() -> Result<()> { - let dev = Device::Cpu; - - let tensor = Tensor::arange(0u32, 4 * 5, &dev)?.reshape((4, 5))?; - let src = Tensor::arange(0u32, 2 * 3, &dev)?.reshape((3, 2))?; - let out = tensor.slice_assign(&[1..4, 3..5], &src)?; - assert_eq!( - out.to_vec2::()?, - &[ - [0, 1, 2, 3, 4], - [5, 6, 7, 0, 1], - [10, 11, 12, 2, 3], - [15, 16, 17, 4, 5] - ] - ); - let out = tensor.slice_assign(&[0..3, 0..2], &src)?; - assert_eq!( - out.to_vec2::()?, - &[ - [0, 1, 2, 3, 4], - [2, 3, 7, 8, 9], - [4, 5, 12, 13, 14], - [15, 16, 17, 18, 19] - ] - ); - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/layout_tests.rs b/v2/dependency-patches/candle-core-0.4.1/tests/layout_tests.rs deleted file mode 100644 index e06188509c..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/tests/layout_tests.rs +++ /dev/null @@ -1,138 +0,0 @@ -use candle::{test_device, Device, IndexOp, Result, Tensor}; -use candle_core as candle; - -fn contiguous(device: &Device) -> Result<()> { - let tensor = Tensor::arange(0u32, 24u32, device)?.reshape((2, 3, 4))?; - assert_eq!( - tensor.to_vec3::()?, - &[ - [[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]], - [[12, 13, 14, 15], [16, 17, 18, 19], [20, 21, 22, 23]] - ] - ); - assert_eq!( - tensor.t()?.contiguous()?.to_vec3::()?, - &[ - [[0, 4, 8], [1, 5, 9], [2, 6, 10], [3, 7, 11]], - [[12, 16, 20], [13, 17, 21], [14, 18, 22], [15, 19, 23]] - ] - ); - assert_eq!( - tensor.transpose(0, 1)?.contiguous()?.to_vec3::()?, - &[ - [[0, 1, 2, 3], [12, 13, 14, 15]], - [[4, 5, 6, 7], [16, 17, 18, 19]], - [[8, 9, 10, 11], [20, 21, 22, 23]] - ] - ); - assert_eq!( - tensor.transpose(0, 1)?.flatten_all()?.to_vec1::()?, - &[0, 1, 2, 3, 12, 13, 14, 15, 4, 5, 6, 7, 16, 17, 18, 19, 8, 9, 10, 11, 20, 21, 22, 23] - ); - assert_eq!( - tensor - .i(1..)? - .transpose(0, 1)? - .contiguous()? - .to_vec3::()?, - &[[[12, 13, 14, 15]], [[16, 17, 18, 19]], [[20, 21, 22, 23]]] - ); - assert_eq!( - tensor.transpose(0, 2)?.contiguous()?.to_vec3::()?, - &[ - [[0, 12], [4, 16], [8, 20]], - [[1, 13], [5, 17], [9, 21]], - [[2, 14], [6, 18], [10, 22]], - [[3, 15], [7, 19], [11, 23]] - ] - ); - Ok(()) -} - -test_device!(contiguous, contiguous_cpu, contiguous_gpu, contiguous_metal); - -#[test] -fn strided_blocks() -> Result<()> { - use candle::Device::Cpu; - let tensor = Tensor::arange(0u32, 24u32, &Cpu)?.reshape((2, 3, 4))?; - match tensor.strided_blocks() { - candle::StridedBlocks::SingleBlock { start_offset, len } => { - assert_eq!(start_offset, 0); - assert_eq!(len, 24); - } - candle::StridedBlocks::MultipleBlocks { .. } => { - panic!("unexpected block structure") - } - }; - let tensor = Tensor::arange(0u32, 26u32, &Cpu)? - .i(2..)? - .reshape((2, 3, 4))?; - match tensor.strided_blocks() { - candle::StridedBlocks::SingleBlock { start_offset, len } => { - assert_eq!(start_offset, 2); - assert_eq!(len, 24); - } - candle::StridedBlocks::MultipleBlocks { .. } => { - panic!("unexpected block structure") - } - }; - let tensor = Tensor::arange(0u32, 24u32, &Cpu)?.reshape((2, 3, 4))?; - let tensor = tensor.i(1)?; - match tensor.strided_blocks() { - candle::StridedBlocks::SingleBlock { start_offset, len } => { - assert_eq!(start_offset, 12); - assert_eq!(len, 12); - } - candle::StridedBlocks::MultipleBlocks { .. } => { - panic!("unexpected block structure") - } - }; - let tensor = Tensor::arange(0u32, 24u32, &Cpu)?.reshape((2, 3, 4))?; - let tensor = tensor.i((.., 1))?; - match tensor.strided_blocks() { - candle::StridedBlocks::SingleBlock { start_offset, len } => { - assert_eq!(start_offset, 0); - assert_eq!(len, 8); - assert_eq!(tensor.to_vec2::()?, &[[4, 5, 6, 7], [16, 17, 18, 19]]); - } - candle::StridedBlocks::MultipleBlocks { .. } => { - panic!("unexpected block structure") - } - }; - let tensor = Tensor::arange(0u32, 24u32, &Cpu)?.reshape((2, 3, 4))?; - match tensor.t()?.strided_blocks() { - candle::StridedBlocks::SingleBlock { .. } => { - panic!("unexpected block structure") - } - candle::StridedBlocks::MultipleBlocks { - block_start_index, - block_len, - } => { - assert_eq!(block_len, 1); - assert_eq!( - block_start_index.collect::>(), - &[ - 0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11, 12, 16, 20, 13, 17, 21, 14, 18, 22, 15, - 19, 23 - ] - ) - } - }; - let tensor = Tensor::arange(0u32, 24u32, &Cpu)?.reshape((2, 3, 4))?; - match tensor.transpose(0, 1)?.strided_blocks() { - candle::StridedBlocks::SingleBlock { .. } => { - panic!("unexpected block structure") - } - candle::StridedBlocks::MultipleBlocks { - block_start_index, - block_len, - } => { - assert_eq!(block_len, 4); - assert_eq!( - block_start_index.collect::>(), - &[0, 12, 4, 16, 8, 20] - ) - } - }; - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/npy.py b/v2/dependency-patches/candle-core-0.4.1/tests/npy.py deleted file mode 100644 index 0fd2778a15..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/tests/npy.py +++ /dev/null @@ -1,9 +0,0 @@ -import numpy as np -x = np.arange(10) - -# Write a npy file. -np.save("test.npy", x) - -# Write multiple values to a npz file. -values = { "x": x, "x_plus_one": x + 1 } -np.savez("test.npz", **values) diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/pool_tests.rs b/v2/dependency-patches/candle-core-0.4.1/tests/pool_tests.rs deleted file mode 100644 index a3708ec488..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/tests/pool_tests.rs +++ /dev/null @@ -1,114 +0,0 @@ -use candle_core::{test_device, test_utils, Device, IndexOp, Result, Tensor}; - -// https://github.com/huggingface/candle/issues/364 -fn avg_pool2d(dev: &Device) -> Result<()> { - let data: Vec = vec![ - 1., 1., 1., 1., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., - ]; - let t = Tensor::from_vec(data, (1, 1, 4, 4), dev)?; - let pool = t.avg_pool2d(2)?.squeeze(0)?.squeeze(0)?; - assert_eq!(pool.to_vec2::()?, [[0.5f32, 1.], [1., 1.]]); - - let data: Vec = vec![ - 1., 2., 1., 3., 0., 0., 1., 1., 1., 1., 1., 1., 5., 1., 1., 1., - ]; - let t = Tensor::from_vec(data, (1, 1, 2, 8), dev)?; - let pool = t.avg_pool2d(2)?.squeeze(0)?.squeeze(0)?; - assert_eq!(pool.to_vec2::()?, [[5. / 4., 6. / 4., 6. / 4., 1.]]); - Ok(()) -} - -fn max_pool2d(dev: &Device) -> Result<()> { - let data: Vec = vec![ - 1., 2., 1., 3., 0., 0., 1., 1., 1., 1., 1., 1., 5., 1., 1., 1., - ]; - let t = Tensor::from_vec(data, (1, 1, 4, 4), dev)?; - - let pool = t.max_pool2d(2)?.squeeze(0)?.squeeze(0)?; - assert_eq!(pool.to_vec2::()?, [[2f32, 3.], [5., 1.]]); - - let t = t.reshape((1, 1, 2, 8))?; - let pool = t.max_pool2d(2)?.squeeze(0)?.squeeze(0)?; - assert_eq!(pool.to_vec2::()?, [[2.0, 3.0, 5.0, 1.0]]); - Ok(()) -} - -/* This test corresponds to the following PyTorch script. -import torch -torch.manual_seed(4242) - -t = torch.randn((1, 2, 4, 4)) -print(t.flatten()) -res = torch.nn.functional.avg_pool2d(t, 2) -print(res) -*/ -fn avg_pool2d_pytorch(dev: &Device) -> Result<()> { - let t = Tensor::new( - &[ - 0.4056f32, -0.8689, -0.0773, -1.5630, -2.8012, -1.5059, 0.3972, 1.0852, 0.4997, 3.0616, - 1.6541, 0.0964, -0.8338, -1.6523, -0.8323, -0.1699, 0.0823, 0.3526, 0.6843, 0.2395, - 1.2279, -0.9287, -1.7030, 0.1370, 0.6047, 0.3770, -0.6266, 0.3529, 2.2013, -0.6836, - 0.2477, 1.3127, - ], - dev, - )? - .reshape((1, 2, 4, 4))?; - let pool = t.avg_pool2d(2)?.squeeze(0)?; - assert_eq!( - test_utils::to_vec3_round(&pool, 4)?, - [ - [[-1.1926, -0.0395], [0.2688, 0.1871]], - [[0.1835, -0.1606], [0.6249, 0.3217]] - ] - ); - let pool = t.avg_pool2d(3)?.squeeze(0)?; - assert_eq!( - test_utils::to_vec3_round(&pool, 4)?, - [[[0.085]], [[0.0078]]] - ); - - let t = t.reshape((1, 1, 4, 8))?; - let pool = t.avg_pool2d(2)?.squeeze(0)?.squeeze(0)?; - assert_eq!( - test_utils::to_vec2_round(&pool, 4)?, - [ - [0.7745, 0.0276, -1.6983, 0.12], - [0.3542, 0.1625, 0.4542, -0.0014] - ] - ); - Ok(()) -} - -fn upsample_nearest2d(dev: &Device) -> Result<()> { - let t = Tensor::arange(0f32, 6f32, dev)?.reshape((1, 1, 2, 3))?; - let upsampled = t.upsample_nearest2d(4, 6)?.i(0)?.i(0)?; - assert_eq!( - t.i(0)?.i(0)?.to_vec2::()?, - [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]] - ); - assert_eq!( - upsampled.to_vec2::()?, - [ - [0.0, 0.0, 1.0, 1.0, 2.0, 2.0], - [0.0, 0.0, 1.0, 1.0, 2.0, 2.0], - [3.0, 3.0, 4.0, 4.0, 5.0, 5.0], - [3.0, 3.0, 4.0, 4.0, 5.0, 5.0] - ] - ); - Ok(()) -} - -test_device!(avg_pool2d, avg_pool2d_cpu, avg_pool2d_gpu, avg_pool2d_metal); -test_device!( - avg_pool2d_pytorch, - avg_pool2d_pytorch_cpu, - avg_pool2d_pytorch_gpu, - avg_pool2d_pytorch_metal -); -test_device!(max_pool2d, max_pool2d_cpu, max_pool2d_gpu, max_pool2d_metal); -test_device!( - upsample_nearest2d, - upsample_nearest2d_cpu, - upsample_nearest2d_gpu, - upsample_nearest2d_metal -); diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/pth.py b/v2/dependency-patches/candle-core-0.4.1/tests/pth.py deleted file mode 100644 index 5c787c2036..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/tests/pth.py +++ /dev/null @@ -1,37 +0,0 @@ -import torch -from collections import OrderedDict - -# Write a trivial tensor to a pt file -a= torch.tensor([[1,2,3,4], [5,6,7,8]]) -o = OrderedDict() -o["test"] = a - -# Write a trivial tensor to a pt file -torch.save(o, "test.pt") - -############################################################################################################ -# Write a trivial tensor to a pt file with a key -torch.save({"model_state_dict": o}, "test_with_key.pt") - -############################################################################################################ -# Create a tensor with fortran contiguous memory layout -import numpy as np - -# Step 1: Create a 3D NumPy array with Fortran order using a range of numbers -# For example, creating a 2x3x4 array -array_fortran = np.asfortranarray(np.arange(1, 2*3*4 + 1).reshape(2, 3, 4)) - -# Verify the memory order -print("Is Fortran contiguous (F order):", array_fortran.flags['F_CONTIGUOUS']) # Should be True -print("Is C contiguous (C order):", array_fortran.flags['C_CONTIGUOUS']) # Should be False - -# Step 2: Convert the NumPy array to a PyTorch tensor -tensor_fortran = torch.from_numpy(array_fortran) - -# Verify the tensor layout -print("Tensor stride:", tensor_fortran.stride()) # Stride will reflect the Fortran memory layout - -# Step 3: Save the PyTorch tensor to a .pth file -torch.save({"tensor_fortran": tensor_fortran}, 'fortran_tensor_3d.pth') - -print("3D Tensor saved with Fortran layout.") diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/pth_tests.rs b/v2/dependency-patches/candle-core-0.4.1/tests/pth_tests.rs deleted file mode 100644 index 9521f9a05d..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/tests/pth_tests.rs +++ /dev/null @@ -1,31 +0,0 @@ -/// Regression test for pth files not loading on Windows. -#[test] -fn test_pth() { - let tensors = candle_core::pickle::PthTensors::new("tests/test.pt", None).unwrap(); - tensors.get("test").unwrap().unwrap(); -} - -#[test] -fn test_pth_with_key() { - let tensors = - candle_core::pickle::PthTensors::new("tests/test_with_key.pt", Some("model_state_dict")) - .unwrap(); - tensors.get("test").unwrap().unwrap(); -} - -#[test] -fn test_pth_fortran_congiguous() { - let tensors = - candle_core::pickle::PthTensors::new("tests/fortran_tensor_3d.pth", None).unwrap(); - let tensor = tensors.get("tensor_fortran").unwrap().unwrap(); - - assert_eq!(tensor.dims3().unwrap(), (2, 3, 4)); - - assert_eq!( - tensor.to_vec3::().unwrap(), - [ - [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]], - [[13, 14, 15, 16], [17, 18, 19, 20], [21, 22, 23, 24]] - ] - ); -} diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/quantized_tests.rs b/v2/dependency-patches/candle-core-0.4.1/tests/quantized_tests.rs deleted file mode 100644 index 5f7e482516..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/tests/quantized_tests.rs +++ /dev/null @@ -1,1002 +0,0 @@ -use candle_core::{ - bail, - quantized::{self, GgmlDType}, - test_device, - test_utils::to_vec2_round, - Device, Module, Result, Tensor, -}; -use quantized::{k_quants, GgmlType}; -use rand::prelude::*; - -const GGML_TEST_SIZE: usize = 32 * 128; - -const GGML_MAX_QUANTIZATION_TOTAL_ERROR: f32 = 0.002; -const GGML_MAX_QUANTIZATION_TOTAL_ERROR_2BITS: f32 = 0.0075; -const GGML_MAX_QUANTIZATION_TOTAL_ERROR_3BITS: f32 = 0.0040; -const GGML_MAX_DOT_PRODUCT_ERROR: f32 = 0.02; - -fn test_matmul( - device: &Device, - (b, m, n, k): (usize, usize, usize, usize), - dtype: GgmlDType, -) -> Result<()> { - let lhs = (0..(m * k)) - .map(|v| v as f32 / (m * k) as f32) - .collect::>(); - let rhs = (0..(k * n)) - .map(|v| v as f32 / (n * k) as f32) - .collect::>(); - - let lhs = Tensor::from_slice(&lhs, (m, k), device)?; - let rhs = Tensor::from_slice(&rhs, (k, n), device)?; - let mm = lhs.matmul(&rhs)?; - let qtensor = quantized::QTensor::quantize(&rhs.t()?, dtype)?; - let matmul = quantized::QMatMul::from_qtensor(qtensor)?; - let res = matmul.forward(&lhs)?; - - let error: f32 = ((&mm - &res)?.abs()? / &mm.abs()?)? - .sum_all()? - .to_scalar()?; - let error = error / (b * m * n) as f32; - assert!( - error <= 0.02, - "Error {error} is too big. \nExpected:\n {mm} \nFound:\n {res}\n for {dtype:?}" - ); - - Ok(()) -} - -fn quantized_matmul(device: &Device) -> Result<()> { - // TODO Enable this later when we enable cuda. - if device.is_cuda() { - return Ok(()); - } - let (m, k, n) = (3, 64, 4); - let lhs = (0..(m * k)).map(|v| v as f32).collect::>(); - let tensor_lhs = Tensor::from_slice(&lhs, (m, k), device)?; - let mut dst = vec![42.; 3 * 4]; - let mut rhs_t = vec![k_quants::BlockQ4_0::zeros(); 8]; - let rhs = (0..(k * n)).map(|v| v as f32).collect::>(); - k_quants::BlockQ4_0::from_float(&rhs, &mut rhs_t)?; - k_quants::matmul((m, k, n), &lhs, &rhs_t, &mut dst)?; - assert_eq!( - dst.iter().map(|x| x.round()).collect::>(), - &[ - 85120.0, 214562.0, 345455.0, 474748.0, 213475.0, 604465.0, 1000686.0, 1388317.0, - 341876.0, 994283.0, 1655709.0, 2301518.0 - ] - ); - let tensor_rhs = Tensor::from_slice(&rhs, (n, k), device)?.t()?; - let mm = tensor_lhs.matmul(&tensor_rhs)?; - assert_eq!( - mm.to_vec2::()?, - &[ - [85344.0, 214368.0, 343392.0, 472416.0], - [214368.0, 605536.0, 996704.0, 1387872.0], - [343392.0, 996704.0, 1650016.0, 2303328.0] - ] - ); - - let qtensor = quantized::QTensor::quantize(&tensor_rhs.t()?, GgmlDType::Q4_0)?; - let matmul = quantized::QMatMul::from_qtensor(qtensor)?; - let res = matmul.forward(&tensor_lhs)?; - match device { - Device::Metal(_) => assert_eq!( - to_vec2_round(&res, 0)?, - &[ - [84946.0, 214126.0, 344757.0, 473798.0], - [213458.0, 604350.0, 1000469.0, 1387990.0], - [341970.0, 994574.0, 1656181.0, 2302182.0] - ] - ), - _ => assert_eq!( - to_vec2_round(&res, 0)?, - &[ - [85120.0, 214562.0, 345455.0, 474748.0], - [213475.0, 604465.0, 1000686.0, 1388317.0], - [341876.0, 994283.0, 1655709.0, 2301518.0] - ] - ), - } - - test_matmul(device, (1, 3, 4, 256), GgmlDType::Q4_0)?; - - Ok(()) -} - -fn quantized_matmul_neg(device: &Device) -> Result<()> { - // TODO Enable this later when we enable cuda. - if device.is_cuda() { - return Ok(()); - } - let (m, k, n) = (3, 64, 4); - let lhs = (0..(m * k)) - .map(|v| v as f32 - (m * k) as f32 / 2.0) - .collect::>(); - let tensor_lhs = Tensor::from_slice(&lhs, (m, k), device)?; - let mut dst = vec![42.; 3 * 4]; - let mut rhs_t = vec![k_quants::BlockQ4_0::zeros(); 8]; - let rhs = (0..k * n) - .map(|v| v as f32 - (k * n) as f32 / 3.0) - .collect::>(); - let tensor_rhs = Tensor::from_slice(&rhs, (n, k), device)?.t()?; - k_quants::BlockQ4_0::from_float(&rhs, &mut rhs_t)?; - k_quants::matmul((m, k, n), &lhs, &rhs_t, &mut dst)?; - assert_eq!( - dst.iter().map(|x| x.round()).collect::>(), - &[ - 243524.0, -19596.0, -285051.0, -549815.0, 23777.0, 21651.0, 19398.0, 18367.0, - -196472.0, 63012.0, 324585.0, 587902.0 - ] - ); - let mm = tensor_lhs.matmul(&tensor_rhs)?; - assert_eq!( - to_vec2_round(&mm, 0)?, - &[ - [244064.0, -20128.0, -284320.0, -548512.0], - [23563.0, 21515.0, 19467.0, 17419.0], - [-196939.0, 63157.0, 323253.0, 583349.0] - ] - ); - - let qtensor = quantized::QTensor::quantize(&tensor_rhs.t()?, GgmlDType::Q4_0)?; - let matmul = quantized::QMatMul::from_qtensor(qtensor)?; - let res = matmul.forward(&tensor_lhs)?; - match device { - Device::Metal(_) => assert_eq!( - to_vec2_round(&res, 0)?, - &[ - [243666.0, -19714.0, -285433.0, -550453.0], - [23782.0, 21654.0, 19400.0, 18369.0], - [-196102.0, 63022.0, 324233.0, 587191.0] - ] - ), - _ => assert_eq!( - to_vec2_round(&res, 0)?, - &[ - [243524.0, -19596.0, -285051.0, -549815.0], - [23777.0, 21651.0, 19398.0, 18367.0], - [-196472.0, 63012.0, 324585.0, 587902.0] - ] - ), - } - - Ok(()) -} - -test_device!( - quantized_matmul, - quantized_matmul_cpu, - quantized_matmul_cuda, - quantized_matmul_metal -); -test_device!( - quantized_matmul_neg, - quantized_matmul_neg_cpu, - quantized_matmul_neg_cuda, - quantized_matmul_neg_metal -); - -fn quantize_q4_0(device: &Device) -> Result<()> { - let src = (0..32 * 4).map(|v| v as f32).collect::>(); - - let src = Tensor::from_slice(&src, (32 * 4,), device)?; - let quant = quantized::QTensor::quantize(&src, GgmlDType::Q4_0)?; - let dst = quant.dequantize(device)?; - assert_eq!( - dst.to_vec1::()?, - &[ - -0.0, -0.0, 3.875, 3.875, 3.875, 3.875, 7.75, 7.75, 7.75, 7.75, 11.625, 11.625, 11.625, - 11.625, 15.5, 15.5, 15.5, 15.5, 19.375, 19.375, 19.375, 19.375, 23.25, 23.25, 23.25, - 23.25, 27.125, 27.125, 27.125, 27.125, 31.0, 31.0, 31.5, 31.5, 31.5, 31.5, 39.375, - 39.375, 39.375, 39.375, 39.375, 39.375, 39.375, 39.375, 47.25, 47.25, 47.25, 47.25, - 47.25, 47.25, 47.25, 47.25, 55.125, 55.125, 55.125, 55.125, 55.125, 55.125, 55.125, - 55.125, 63.0, 63.0, 63.0, 63.0, 59.375, 59.375, 71.25, 71.25, 71.25, 71.25, 71.25, - 71.25, 71.25, 71.25, 71.25, 71.25, 71.25, 71.25, 83.125, 83.125, 83.125, 83.125, - 83.125, 83.125, 83.125, 83.125, 83.125, 83.125, 83.125, 83.125, 95.0, 95.0, 95.0, 95.0, - 95.0, 95.0, 95.25, 95.25, 95.25, 95.25, 95.25, 95.25, 95.25, 95.25, 111.125, 111.125, - 111.125, 111.125, 111.125, 111.125, 111.125, 111.125, 111.125, 111.125, 111.125, - 111.125, 111.125, 111.125, 111.125, 111.125, 127.0, 127.0, 127.0, 127.0, 127.0, 127.0, - 127.0, 127.0 - ] - ); - ggml_quantization_error_test(GgmlDType::Q4_0, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; - Ok(()) -} - -fn quantize_q4_1(device: &Device) -> Result<()> { - let src = (0..32 * 4).map(|v| v as f32).collect::>(); - let src = Tensor::from_slice(&src, (32 * 4,), device)?; - let quant = quantized::QTensor::quantize(&src, GgmlDType::Q4_1)?; - let dst = quant.dequantize(device)?; - assert_eq!( - round_vector(&dst.to_vec1::()?), - &[ - 0.0, 0.0, 2.066, 2.066, 4.133, 4.133, 6.199, 6.199, 8.266, 8.266, 10.332, 10.332, - 12.398, 12.398, 14.465, 14.465, 16.531, 16.531, 18.598, 18.598, 20.664, 20.664, 22.73, - 22.73, 24.797, 24.797, 26.863, 26.863, 28.93, 28.93, 30.996, 30.996, 32.0, 32.0, - 34.066, 34.066, 36.133, 36.133, 38.199, 38.199, 40.266, 40.266, 42.332, 42.332, 44.398, - 44.398, 46.465, 46.465, 48.531, 48.531, 50.598, 50.598, 52.664, 52.664, 54.73, 54.73, - 56.797, 56.797, 58.863, 58.863, 60.93, 60.93, 62.996, 62.996, 64.0, 64.0, 66.066, - 66.066, 68.133, 68.133, 70.199, 70.199, 72.266, 72.266, 74.332, 74.332, 76.398, 76.398, - 78.465, 78.465, 80.531, 80.531, 82.598, 82.598, 84.664, 84.664, 86.73, 86.73, 88.797, - 88.797, 90.863, 90.863, 92.93, 92.93, 94.996, 94.996, 96.0, 96.0, 98.066, 98.066, - 100.133, 100.133, 102.199, 102.199, 104.266, 104.266, 106.332, 106.332, 108.398, - 108.398, 110.465, 110.465, 112.531, 112.531, 114.598, 114.598, 116.664, 116.664, - 118.73, 118.73, 120.797, 120.797, 122.863, 122.863, 124.93, 124.93, 126.996, 126.996 - ] - ); - ggml_quantization_error_test(GgmlDType::Q4_1, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; - Ok(()) -} - -fn quantize_q5_0(device: &Device) -> Result<()> { - // TODO Enable this later when we enable cuda. - if device.is_cuda() { - return Ok(()); - } - let src = (0..32 * 4).map(|v| v as f32).collect::>(); - let src = Tensor::from_slice(&src, (32 * 4,), device)?; - let quant = quantized::QTensor::quantize(&src, GgmlDType::Q5_0)?; - let dst = quant.dequantize(device)?; - assert_eq!( - round_vector(&dst.to_vec1::()?), - &[ - -0.0, 1.938, 1.938, 3.875, 3.875, 5.813, 5.813, 7.75, 7.75, 9.688, 9.688, 11.625, - 11.625, 13.563, 13.563, 15.5, 15.5, 17.438, 17.438, 19.375, 19.375, 21.313, 21.313, - 23.25, 23.25, 25.188, 25.188, 27.125, 27.125, 29.063, 29.063, 31.0, 31.5, 31.5, 35.438, - 35.438, 35.438, 35.438, 39.375, 39.375, 39.375, 39.375, 43.313, 43.313, 43.313, 43.313, - 47.25, 47.25, 47.25, 47.25, 51.188, 51.188, 51.188, 51.188, 55.125, 55.125, 55.125, - 55.125, 59.063, 59.063, 59.063, 59.063, 63.0, 63.0, 65.313, 65.313, 65.313, 65.313, - 65.313, 71.25, 71.25, 71.25, 71.25, 71.25, 71.25, 77.188, 77.188, 77.188, 77.188, - 77.188, 77.188, 83.125, 83.125, 83.125, 83.125, 83.125, 83.125, 89.063, 89.063, 89.063, - 89.063, 89.063, 89.063, 95.0, 95.0, 95.0, 95.25, 95.25, 95.25, 95.25, 103.188, 103.188, - 103.188, 103.188, 103.188, 103.188, 103.188, 103.188, 111.125, 111.125, 111.125, - 111.125, 111.125, 111.125, 111.125, 111.125, 119.063, 119.063, 119.063, 119.063, - 119.063, 119.063, 119.063, 119.063, 127.0, 127.0, 127.0, 127.0 - ] - ); - ggml_quantization_error_test(GgmlDType::Q5_0, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; - Ok(()) -} - -fn quantize_q5_1(device: &Device) -> Result<()> { - // TODO Enable this later when we enable cuda. - if device.is_cuda() { - return Ok(()); - } - let src = (0..32 * 4).map(|v| v as f32).collect::>(); - let src = Tensor::from_slice(&src, (32 * 4,), device)?; - let quant = quantized::QTensor::quantize(&src, GgmlDType::Q5_1)?; - let dst = quant.dequantize(device)?; - assert_eq!( - round_vector(&dst.to_vec1::()?), - &[ - 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, - 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, - 30.0, 31.0, 32.0, 33.0, 34.0, 35.0, 36.0, 37.0, 38.0, 39.0, 40.0, 41.0, 42.0, 43.0, - 44.0, 45.0, 46.0, 47.0, 48.0, 49.0, 50.0, 51.0, 52.0, 53.0, 54.0, 55.0, 56.0, 57.0, - 58.0, 59.0, 60.0, 61.0, 62.0, 63.0, 64.0, 65.0, 66.0, 67.0, 68.0, 69.0, 70.0, 71.0, - 72.0, 73.0, 74.0, 75.0, 76.0, 77.0, 78.0, 79.0, 80.0, 81.0, 82.0, 83.0, 84.0, 85.0, - 86.0, 87.0, 88.0, 89.0, 90.0, 91.0, 92.0, 93.0, 94.0, 95.0, 96.0, 97.0, 98.0, 99.0, - 100.0, 101.0, 102.0, 103.0, 104.0, 105.0, 106.0, 107.0, 108.0, 109.0, 110.0, 111.0, - 112.0, 113.0, 114.0, 115.0, 116.0, 117.0, 118.0, 119.0, 120.0, 121.0, 122.0, 123.0, - 124.0, 125.0, 126.0, 127.0 - ] - ); - ggml_quantization_error_test(GgmlDType::Q5_1, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; - Ok(()) -} - -fn get_test_vector2(bound: f32, size: usize, device: &Device) -> Result { - assert!( - size % crate::quantized::k_quants::QK_K == 0, - "size must be a multiple of {}", - crate::quantized::k_quants::QK_K - ); - - let src = (0..size) - .map(|v| (v as f32 - size as f32 / 2.) * bound / (size as f32 / 2.)) - .collect::>(); - assert_eq!([src[0], src[size / 2]], [-bound, 0.0]); - Tensor::from_vec(src, (size,), device) -} - -/// Round a vector -fn round_vector(values: &[f32]) -> Vec { - values - .iter() - .map(|x| (1000. * x).round() / 1000.) - .collect::>() -} - -fn compare_with_error(values: &[f32], expected: &[f32], tolerance: f32) { - for (i, (value, expected_value)) in values.iter().zip(expected.iter()).enumerate() { - let difference = (value - expected_value).abs(); - - assert!( - difference < tolerance, - "Error at index {}: value = {}, expected = {}. Difference = {} exceeds tolerance = {}.", - i, - value, - expected_value, - difference, - tolerance - ); - } -} - -/// Creates a vector similar to the ones used in GGML unit tests: -/// https://github.com/ggerganov/llama.cpp/blob/master/tests/test-quantize-fns.cpp#L26-L30 -fn create_ggml_like_vector(offset: f32) -> Vec { - (0..GGML_TEST_SIZE) - .map(|i| 0.1 + 2.0 * (i as f32 + offset).cos()) - .collect() -} - -/// Calculates the root mean square error between two vectors -fn calculate_rmse(a: &[f32], b: &[f32]) -> f32 { - assert_eq!(a.len(), b.len()); - let sum = a - .iter() - .zip(b) - .map(|(a, b)| (a - b).powi(2)) - .sum::() - .sqrt(); - sum / a.len() as f32 -} - -/// Similar to the GGML quantization unit test: -/// https://github.com/ggerganov/llama.cpp/blob/master/tests/test-quantize-fns.cpp#L43-L50 -fn ggml_quantization_error_test(dtype: GgmlDType, device: &Device, max_error: f32) -> Result<()> { - let src = create_ggml_like_vector(0.0); - let src = Tensor::from_slice(&src, (GGML_TEST_SIZE,), device)?; - let quant = quantized::QTensor::quantize(&src, dtype)?; - let dst = quant.dequantize(device)?; - let error = calculate_rmse(&src.to_vec1::()?, &dst.to_vec1::()?); - if error > max_error { - bail!( - "Quantization error {} exceeds max error {}", - error, - max_error - ); - } - Ok(()) -} - -fn quantize_q2k(device: &Device) -> Result<()> { - let dtype = GgmlDType::Q2K; - - let src = get_test_vector2(0.5, 1024, device)?; - let quant = quantized::QTensor::quantize(&src, dtype)?; - let dst = quant.dequantize(device)?; - - let src = src.to_vec1::()?; - let dst = dst.to_vec1::()?; - compare_with_error(dst.as_slice(), src.as_slice(), 0.1); - - // Test some specific values - assert_eq!( - [src[0], src[128], src[256], src[512], src[800], src[1023]], - [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] - ); - let dst = round_vector(&dst); - assert_eq!( - [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], - [-0.499, -0.366, -0.249, 0.0, 0.295, 0.492] - ); - - let src_big = get_test_vector2(128.0, 1024, device)?; - let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; - let dst_big = quant_big.dequantize(device)?; - - let src_big = src_big.to_vec1::()?; - let dst_big = dst_big.to_vec1::()?; - compare_with_error(dst_big.as_slice(), src_big.as_slice(), 6.0); - - ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR_2BITS)?; - Ok(()) -} - -fn quantize_q3k(device: &Device) -> Result<()> { - let dtype = GgmlDType::Q3K; - let src = get_test_vector2(0.5, 1024, device)?; - let quant = quantized::QTensor::quantize(&src, dtype)?; - let dst = quant.dequantize(device)?; - - let src = src.to_vec1::()?; - let dst = dst.to_vec1::()?; - compare_with_error(dst.as_slice(), src.as_slice(), 0.03); - - // Test some specific values - assert_eq!( - [src[0], src[128], src[256], src[512], src[800], src[1023]], - [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] - ); - let dst = round_vector(&dst); - assert_eq!( - [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], - [-0.493, -0.37, -0.243, -0.0, 0.292, 0.492] - ); - - let src_big = get_test_vector2(128.0, 1024, device)?; - let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; - let dst_big = quant_big.dequantize(device)?; - - let src_big = src_big.to_vec1::()?; - let dst_big = dst_big.to_vec1::()?; - compare_with_error(dst_big.as_slice(), src_big.as_slice(), 3.5); - - ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR_3BITS)?; - Ok(()) -} - -fn quantize_q4k(device: &Device) -> Result<()> { - let dtype = GgmlDType::Q4K; - let src = get_test_vector2(0.5, 1024, device)?; - let quant = quantized::QTensor::quantize(&src, dtype)?; - let dst = quant.dequantize(device)?; - - let src = src.to_vec1::()?; - let dst = dst.to_vec1::()?; - compare_with_error(dst.as_slice(), src.as_slice(), 0.017); - - // Test some specific values - assert_eq!( - [src[0], src[128], src[256], src[512], src[800], src[1023]], - [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] - ); - let dst = round_vector(&dst); - assert_eq!( - [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], - [-0.5, -0.373, -0.25, 0.0, 0.288, 0.498] - ); - - let src_big = get_test_vector2(128.0, 1024, device)?; - let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; - let dst_big = quant_big.dequantize(device)?; - - let src_big = src_big.to_vec1::()?; - let dst_big = dst_big.to_vec1::()?; - compare_with_error(dst_big.as_slice(), src_big.as_slice(), 4.5); - - ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; - Ok(()) -} - -fn quantize_q5k(device: &Device) -> Result<()> { - let dtype = GgmlDType::Q5K; - let src = get_test_vector2(0.5, 1024, device)?; - let quant = quantized::QTensor::quantize(&src, dtype)?; - let dst = quant.dequantize(device)?; - - let src = src.to_vec1::()?; - let dst = dst.to_vec1::()?; - compare_with_error(dst.as_slice(), src.as_slice(), 0.009); - - // Test some specific values - assert_eq!( - [src[0], src[128], src[256], src[512], src[800], src[1023]], - [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] - ); - let dst = round_vector(&dst); - assert_eq!( - [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], - [-0.5, -0.373, -0.25, 0.0, 0.279, 0.499] - ); - - let src_big = get_test_vector2(128.0, 1024, device)?; - let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; - let dst_big = quant_big.dequantize(device)?; - - let src_big = src_big.to_vec1::()?; - let dst_big = dst_big.to_vec1::()?; - compare_with_error(dst_big.as_slice(), src_big.as_slice(), 2.5); - - ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; - Ok(()) -} - -fn quantize_q6k(device: &Device) -> Result<()> { - let dtype = GgmlDType::Q6K; - let src = get_test_vector2(0.5, 1024, device)?; - let quant = quantized::QTensor::quantize(&src, dtype)?; - let dst = quant.dequantize(device)?; - - let src = src.to_vec1::()?; - let dst = dst.to_vec1::()?; - compare_with_error(dst.as_slice(), src.as_slice(), 0.008); - - // Test some specific values - assert_eq!( - [src[0], src[128], src[256], src[512], src[800], src[1023]], - [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] - ); - let dst = round_vector(&dst); - assert_eq!( - [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], - [-0.497, -0.372, -0.25, -0.0, 0.284, 0.5] - ); - - let src_big = get_test_vector2(128.0, 1024, device)?; - let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; - let dst_big = quant_big.dequantize(device)?; - - let src_big = src_big.to_vec1::()?; - let dst_big = dst_big.to_vec1::()?; - compare_with_error(dst_big.as_slice(), src_big.as_slice(), 2.0); - - ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; - Ok(()) -} - -fn quantize_q8k(device: &Device) -> Result<()> { - let dtype = GgmlDType::Q8K; - let src = get_test_vector2(0.5, 1024, device)?; - let quant = quantized::QTensor::quantize(&src, dtype)?; - let dst = quant.dequantize(device)?; - - let src = src.to_vec1::()?; - let dst = dst.to_vec1::()?; - compare_with_error(dst.as_slice(), src.as_slice(), 0.008); - - // Test some specific values - assert_eq!( - [src[0], src[128], src[256], src[512], src[800], src[1023]], - [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] - ); - let dst = round_vector(&dst); - assert_eq!( - [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], - [-0.5, -0.375, -0.25, -0.0, 0.281, 0.499] - ); - - let src_big = get_test_vector2(128.0, 1024, device)?; - let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; - let dst_big = quant_big.dequantize(device)?; - - let src_big = src_big.to_vec1::()?; - let dst_big = dst_big.to_vec1::()?; - compare_with_error(dst_big.as_slice(), src_big.as_slice(), 0.6); - - ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; - Ok(()) -} - -test_device!( - quantize_q4_0, - quantize_q4_0_cpu, - quantize_q4_0_cuda, - quantize_q4_0_metal -); -test_device!( - quantize_q4_1, - quantize_q4_1_cpu, - quantize_q4_1_cuda, - quantize_q4_1_metal -); -test_device!( - quantize_q5_0, - quantize_q5_0_cpu, - quantize_q5_0_cuda, - quantize_q5_0_metal -); -test_device!( - quantize_q5_1, - quantize_q5_1_cpu, - quantize_q5_1_cuda, - quantize_q5_1_metal -); -test_device!( - quantize_q2k, - quantize_q2k_cpu, - quantize_q2k_cuda, - quantize_q2k_metal -); -test_device!( - quantize_q3k, - quantize_q3k_cpu, - quantize_q3k_cuda, - quantize_q3k_metal -); -test_device!( - quantize_q4k, - quantize_q4k_cpu, - quantize_q4k_cuda, - quantize_q4k_metal -); -test_device!( - quantize_q5k, - quantize_q5k_cpu, - quantize_q5k_cuda, - quantize_q5k_metal -); -test_device!( - quantize_q6k, - quantize_q6k_cpu, - quantize_q6k_cuda, - quantize_q6k_metal -); -test_device!( - quantize_q8k, - quantize_q8k_cpu, - quantize_q8k_cuda, - quantize_q8k_metal -); - -/// Very simple dot product implementation -fn vec_dot_reference(a: &[f32], b: &[f32]) -> f32 { - a.iter().zip(b).map(|(a, b)| a * b).sum() -} - -/// Returns the error achieved by the GGML matmul unit test. -fn ggml_reference_matmul_error(dtype: GgmlDType) -> Result { - let err = match dtype { - GgmlDType::F16 => 0.000010, - GgmlDType::Q2K => 0.004086, - GgmlDType::Q3K => 0.016148, - GgmlDType::Q4K => 0.002425, - GgmlDType::Q5K => 0.000740, - GgmlDType::Q6K => 0.000952, - GgmlDType::Q4_0 => 0.001143, - GgmlDType::Q4_1 => 0.008, - GgmlDType::Q5_0 => 0.001353, - GgmlDType::Q5_1 => 0.00149, - GgmlDType::Q8_0 => 0.000092, - - // Not from the ggml repo. - GgmlDType::Q8K => 0.00065, - _ => bail!("No GGML results for quantization type {dtype:?}",), - }; - Ok(err) -} - -/// Similar to the GGML matmul unit test: -/// https://github.com/ggerganov/llama.cpp/blob/master/tests/test-quantize-fns.cpp#L76-L91 -fn ggml_matmul_error_test() -> Result<()> { - let a = create_ggml_like_vector(0.0); - let b = create_ggml_like_vector(1.0); - ggml_matmul_error_test_::(a.as_slice(), b.as_slice(), 1.0)?; - // Another example that is more likely to trigger the overflow reported in #1526 - let a = (0..GGML_TEST_SIZE) - .map(|i| i as f32 / GGML_TEST_SIZE as f32) - .collect::>(); - let b = (0..GGML_TEST_SIZE) - .map(|i| i as f32 / GGML_TEST_SIZE as f32) - .collect::>(); - ggml_matmul_error_test_::(a.as_slice(), b.as_slice(), 2.0)?; - Ok(()) -} - -fn ggml_matmul_error_test_(a: &[f32], b: &[f32], err_m: f32) -> Result<()> { - let length = a.len(); - - let mut a_quant = vec![T::zeros(); length / T::BLCK_SIZE]; - let mut b_quant = vec![T::VecDotType::zeros(); length / T::VecDotType::BLCK_SIZE]; - T::from_float(a, &mut a_quant)?; - T::VecDotType::from_float(b, &mut b_quant)?; - - let result = T::vec_dot(length, &a_quant, &b_quant)?; - let result_unopt = T::vec_dot_unopt(length, &a_quant, &b_quant)?; - let reference_result = vec_dot_reference(a, b); - - if (result - result_unopt).abs() / length as f32 > 1e-6 { - bail!( - "the opt and unopt vec-dot returned different values, opt {result}, unopt {result_unopt}" - ) - } - - let error = (result - reference_result).abs() / length as f32; - - let ggml_error = ggml_reference_matmul_error(T::DTYPE)? * err_m; - - if !error.is_finite() || error > GGML_MAX_DOT_PRODUCT_ERROR { - bail!("Dot product error {error} exceeds max error {GGML_MAX_DOT_PRODUCT_ERROR}",); - } - - // We diverge slightly due to different rounding behavior / f16 to f32 conversions in GGML - // => we use a slightly higher error threshold - const ERROR_LENIENCY: f32 = 0.00001; - if error - ERROR_LENIENCY > ggml_error { - bail!( - "Dot product error {} exceeds ggml reference error {}", - error, - ggml_error - ); - } - Ok(()) -} - -#[test] -fn quantized_mm() -> Result<()> { - ggml_matmul_error_test::()?; - ggml_matmul_error_test::()?; - ggml_matmul_error_test::()?; - ggml_matmul_error_test::()?; - ggml_matmul_error_test::()?; - Ok(()) -} - -/// generates random tensors of size `m x k` and `n x k` and calculates their expected matrix multiplication result. -fn get_random_tensors( - m: usize, - k: usize, - n: usize, - device: &Device, -) -> Result<(Tensor, Tensor, Tensor)> { - let mut rng = StdRng::seed_from_u64(314159265358979); - - let lhs = (0..m * k) - .map(|_| rng.gen::() - 0.5) - .collect::>(); - let rhs = (0..n * k) - .map(|_| rng.gen::() - 0.5) - .collect::>(); - - let lhs = Tensor::from_vec(lhs, (m, k), device)?; - let rhs = Tensor::from_vec(rhs, (n, k), device)?; - - let mm = lhs.matmul(&rhs.t()?)?; - Ok((lhs, rhs, mm)) -} - -#[macro_export] -macro_rules! quantized_matmul { - // TODO: Switch to generating the two last arguments automatically once concat_idents is - // stable. https://github.com/rust-lang/rust/issues/29599 - ($fn_name: ident, $fn_name_cpu: ident, $fn_name_cuda: ident, $fn_name_metal: ident, $dtype: expr) => { - fn $fn_name(device: &Device) -> Result<()> { - if device.is_cuda() { - // TODO Enable Cuda GGML sometime maybe. - return Ok(()); - } - test_matmul(device, (1, 3, 4, 256), $dtype)?; - Ok(()) - } - - test_device!($fn_name, $fn_name_cpu, $fn_name_cuda, $fn_name_metal); - }; -} - -quantized_matmul!( - quantized_matmul_q4_0_bis, - quantized_matmul_q4_0_cpu, - quantized_matmul_q4_0_cuda, - quantized_matmul_q4_0_metal, - GgmlDType::Q4_0 -); -quantized_matmul!( - quantized_matmul_q4_1_bis, - quantized_matmul_q4_1_cpu, - quantized_matmul_q4_1_cuda, - quantized_matmul_q4_1_metal, - GgmlDType::Q4_1 -); -quantized_matmul!( - quantized_matmul_q5_0_bis, - quantized_matmul_q5_0_cpu, - quantized_matmul_q5_0_cuda, - quantized_matmul_q5_0_metal, - GgmlDType::Q5_0 -); -quantized_matmul!( - quantized_matmul_q5_1_bis, - quantized_matmul_q5_1_cpu, - quantized_matmul_q5_1_cuda, - quantized_matmul_q5_1_metal, - GgmlDType::Q5_1 -); -quantized_matmul!( - quantized_matmul_q8_0_bis, - quantized_matmul_q8_0_cpu, - quantized_matmul_q8_0_cuda, - quantized_matmul_q8_0_metal, - GgmlDType::Q8_0 -); -// Not implemented in Ggml -// quantized_matmul!( -// quantized_matmul_q8_1_bis, -// quantized_matmul_q8_1_cpu, -// quantized_matmul_q8_1_cuda, -// quantized_matmul_q8_1_metal, -// GgmlDType::Q8_1 -// ); -// TODO This is bugged (also bugged in GGML -quantized_matmul!( - quantized_matmul_q2k_bis, - quantized_matmul_q2k_cpu, - quantized_matmul_q2k_cuda, - quantized_matmul_q2k_metal, - GgmlDType::Q2K -); -quantized_matmul!( - quantized_matmul_q3k_bis, - quantized_matmul_q3k_cpu, - quantized_matmul_q3k_cuda, - quantized_matmul_q3k_metal, - GgmlDType::Q3K -); -quantized_matmul!( - quantized_matmul_q4k_bis, - quantized_matmul_q4k_cpu, - quantized_matmul_q4k_cuda, - quantized_matmul_q4k_metal, - GgmlDType::Q4K -); -quantized_matmul!( - quantized_matmul_q5k_bis, - quantized_matmul_q5k_cpu, - quantized_matmul_q5k_cuda, - quantized_matmul_q5k_metal, - GgmlDType::Q5K -); -quantized_matmul!( - quantized_matmul_q6k_bis, - quantized_matmul_q6k_cpu, - quantized_matmul_q6k_cuda, - quantized_matmul_q6k_metal, - GgmlDType::Q6K -); -// Not implemented on metal -// quantized_matmul!( -// quantized_matmul_q8k_bis, -// quantized_matmul_q8k_cpu, -// quantized_matmul_q8k_cuda, -// quantized_matmul_q8k_metal, -// GgmlDType::Q8K -// ); - -#[test] -fn quantized_matmul_q2k() -> Result<()> { - use k_quants::BlockQ2K; - - let cpu = &Device::Cpu; - let (m, k, n) = (11, 512, 21); - let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); - - let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q2K)?; - let rhs = quantized::QMatMul::from_qtensor(rhs)?; - let mm = rhs.forward(&lhs)?; - - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [0.916, 0.422, 0.215, 1.668]); - - ggml_matmul_error_test::()?; - - Ok(()) -} - -#[test] -fn quantized_matmul_q3k() -> Result<()> { - use k_quants::BlockQ3K; - - let cpu = &Device::Cpu; - let (m, k, n) = (11, 512, 21); - let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); - - let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q3K)?; - let rhs = quantized::QMatMul::from_qtensor(rhs)?; - let mm = rhs.forward(&lhs)?; - - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [1.029, 1.418, -0.314, 1.495]); - - ggml_matmul_error_test::()?; - - Ok(()) -} - -#[test] -fn quantized_matmul_q4k() -> Result<()> { - use k_quants::BlockQ4K; - - let cpu = &Device::Cpu; - let (m, k, n) = (11, 512, 21); - let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); - - let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q4K)?; - let rhs = quantized::QMatMul::from_qtensor(rhs)?; - let mm = rhs.forward(&lhs)?; - - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [1.125, 1.435, -0.201, 1.589]); - - ggml_matmul_error_test::()?; - - Ok(()) -} - -#[test] -fn quantized_matmul_q5k() -> Result<()> { - use k_quants::BlockQ5K; - - let cpu = &Device::Cpu; - let (m, k, n) = (11, 512, 21); - let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); - - let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q5K)?; - let rhs = quantized::QMatMul::from_qtensor(rhs)?; - let mm = rhs.forward(&lhs)?; - - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [1.192, 1.491, -0.18, 1.743]); - - //Expected: 0.000740408897 - ggml_matmul_error_test::()?; - - Ok(()) -} - -#[test] -fn quantized_matmul_q6k() -> Result<()> { - use k_quants::BlockQ6K; - - let cpu = &Device::Cpu; - let (m, k, n) = (11, 512, 21); - let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); - - let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q6K)?; - let rhs = quantized::QMatMul::from_qtensor(rhs)?; - let mm = rhs.forward(&lhs)?; - - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [1.324, 1.49, -0.164, 1.741]); - - ggml_matmul_error_test::()?; - Ok(()) -} - -#[test] -fn quantized_matmul_q8k() -> Result<()> { - use k_quants::BlockQ8K; - - let cpu = &Device::Cpu; - let (m, k, n) = (11, 512, 21); - let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); - - let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q8K)?; - let rhs = quantized::QMatMul::from_qtensor(rhs)?; - let mm = rhs.forward(&lhs)?; - - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [1.266, 1.504, -0.204, 1.7]); - - ggml_matmul_error_test::()?; - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/serialization_tests.rs b/v2/dependency-patches/candle-core-0.4.1/tests/serialization_tests.rs deleted file mode 100644 index 415306f444..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/tests/serialization_tests.rs +++ /dev/null @@ -1,24 +0,0 @@ -use candle_core::{DType, Result, Tensor}; - -#[test] -fn npy() -> Result<()> { - let npy = Tensor::read_npy("tests/test.npy")?; - assert_eq!( - npy.to_dtype(DType::U8)?.to_vec1::()?, - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - ); - Ok(()) -} - -#[test] -fn npz() -> Result<()> { - let npz = Tensor::read_npz("tests/test.npz")?; - assert_eq!(npz.len(), 2); - assert_eq!(npz[0].0, "x"); - assert_eq!(npz[1].0, "x_plus_one"); - assert_eq!( - npz[1].1.to_dtype(DType::U8)?.to_vec1::()?, - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - ); - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/tensor_tests.rs b/v2/dependency-patches/candle-core-0.4.1/tests/tensor_tests.rs deleted file mode 100644 index 40737e7b4d..0000000000 --- a/v2/dependency-patches/candle-core-0.4.1/tests/tensor_tests.rs +++ /dev/null @@ -1,1274 +0,0 @@ -use candle_core::{test_device, test_utils, DType, Device, IndexOp, Result, Tensor, D}; - -fn zeros(device: &Device) -> Result<()> { - let tensor = Tensor::zeros((5, 2), DType::F32, device)?; - let (dim1, dim2) = tensor.dims2()?; - assert_eq!(dim1, 5); - assert_eq!(dim2, 2); - Ok(()) -} - -fn ones(device: &Device) -> Result<()> { - assert_eq!( - Tensor::ones((2, 3), DType::U8, device)?.to_vec2::()?, - [[1, 1, 1], [1, 1, 1]], - ); - assert_eq!( - Tensor::ones((2, 3), DType::U32, device)?.to_vec2::()?, - [[1, 1, 1], [1, 1, 1]], - ); - assert_eq!( - Tensor::ones((2, 3), DType::I64, device)?.to_vec2::()?, - [[1, 1, 1], [1, 1, 1]], - ); - assert_eq!( - Tensor::ones((2, 3), DType::F32, device)?.to_vec2::()?, - [[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]], - ); - assert_eq!( - Tensor::ones((2, 3), DType::F64, device)?.to_vec2::()?, - [[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]], - ); - Ok(()) -} - -fn full(device: &Device) -> Result<()> { - assert_eq!( - Tensor::full(42u32, (2, 3), device)?.to_vec2::()?, - [[42, 42, 42], [42, 42, 42]], - ); - Ok(()) -} - -fn arange(device: &Device) -> Result<()> { - assert_eq!( - Tensor::arange(0u8, 5u8, device)?.to_vec1::()?, - [0, 1, 2, 3, 4], - ); - assert_eq!( - Tensor::arange_step(0u8, 5u8, 2, device)?.to_vec1::()?, - [0, 2, 4], - ); - assert_eq!( - Tensor::arange_step(0u8, 5u8, 3, device)?.to_vec1::()?, - [0, 3], - ); - assert_eq!( - Tensor::arange_step(5i64, 0i64, -1, device)?.to_vec1::()?, - [5, 4, 3, 2, 1], - ); - Ok(()) -} - -fn add_mul(device: &Device) -> Result<()> { - let tensor = Tensor::new(&[3f32, 1., 4.], device)?; - let dim1 = tensor.dims1()?; - assert_eq!(dim1, 3); - let content: Vec = tensor.to_vec1()?; - assert_eq!(content, [3., 1., 4.]); - let tensor = Tensor::add(&tensor, &tensor)?; - let content: Vec = tensor.to_vec1()?; - assert_eq!(content, [6., 2., 8.]); - let tensor = Tensor::mul(&tensor, &tensor)?; - let content: Vec = tensor.to_vec1()?; - assert_eq!(content, [36., 4., 64.]); - Ok(()) -} - -fn tensor_2d(device: &Device) -> Result<()> { - let data = &[[3f32, 1., 4., 1., 5.], [2., 1., 7., 8., 2.]]; - let tensor = Tensor::new(data, device)?; - let dims = tensor.dims2()?; - assert_eq!(dims, (2, 5)); - let content: Vec> = tensor.to_vec2()?; - assert_eq!(content, data); - Ok(()) -} - -fn clamp(device: &Device) -> Result<()> { - let data = &[[3f32, 1., 4., 1., 5.], [2., 1., 7., 8., 2.]]; - let tensor = Tensor::new(data, device)?; - let tensor = tensor.clamp(1.5, 6.2)?; - assert_eq!( - tensor.to_vec2::()?, - [[3.0, 1.5, 4.0, 1.5, 5.0], [2.0, 1.5, 6.2, 6.2, 2.0]], - ); - Ok(()) -} - -fn unary_op(device: &Device) -> Result<()> { - let data = &[[-3f32, 1., 4., -0.1, 0.5], [2.7, -1.8, -0.28, 1.8, 2.8]]; - let tensor = Tensor::new(data, device)?; - assert_eq!( - test_utils::to_vec2_round(&tensor.gelu()?, 4)?, - [ - [-0.0036, 0.8412, 3.9999, -0.046, 0.3457], - [2.6911, -0.0647, -0.1091, 1.7353, 2.7933] - ] - ); - assert_eq!( - test_utils::to_vec2_round(&tensor.gelu_erf()?, 4)?, - [ - [-0.004, 0.8413, 3.9999, -0.046, 0.3457], - [2.6906, -0.0647, -0.1091, 1.7353, 2.7928] - ] - ); - assert_eq!( - test_utils::to_vec2_round(&tensor.erf()?, 4)?, - [ - [-1.0, 0.8427, 1.0, -0.1125, 0.5205], - [0.9999, -0.9891, -0.3079, 0.9891, 0.9999] - ] - ); - assert_eq!( - test_utils::to_vec2_round(&tensor.silu()?, 4)?, - [ - [-0.1423, 0.7311, 3.9281, -0.0475, 0.3112], - [2.53, -0.2553, -0.1205, 1.5447, 2.6395] - ] - ); - assert_eq!( - test_utils::to_vec2_round(&tensor.ceil()?, 4)?, - [[-3.0, 1.0, 4.0, -0.0, 1.0], [3.0, -1.0, -0.0, 2.0, 3.0]] - ); - assert_eq!( - test_utils::to_vec2_round(&tensor.floor()?, 4)?, - [[-3.0, 1.0, 4.0, -1.0, 0.0], [2.0, -2.0, -1.0, 1.0, 2.0]] - ); - assert_eq!( - test_utils::to_vec2_round(&tensor.round()?, 4)?, - [[-3.0, 1.0, 4.0, -0.0, 1.0], [3.0, -2.0, -0.0, 2.0, 3.0]] - ); - let tensor = Tensor::new(&[2997.9246, 314.15926f32], device)?; - assert_eq!( - test_utils::to_vec1_round(&tensor.round_to(2)?, 4)?, - [2997.92, 314.16] - ); - assert_eq!( - test_utils::to_vec1_round(&tensor.round_to(-2)?, 4)?, - [3000.0, 300.] - ); - Ok(()) -} - -fn binary_op(device: &Device) -> Result<()> { - let data = &[[3f32, 1., 4., 1., 5.], [2., 1., 7., 8., 2.]]; - let tensor1 = Tensor::new(data, device)?; - let data2 = &[[5f32, 5., 5., 5., 5.], [2., 1., 7., 8., 2.]]; - let tensor2 = Tensor::new(data2, device)?; - let tensor = (&tensor1 + (&tensor1 * &tensor1)? / (&tensor1 + &tensor2))?; - let dims = tensor.dims2()?; - assert_eq!(dims, (2, 5)); - let content: Vec> = tensor.to_vec2()?; - assert_eq!(content[0], [4.125, 1.1666666, 5.7777777, 1.1666666, 7.5]); - assert_eq!(content[1], [3.0, 1.5, 10.5, 12.0, 3.0]); - #[allow(clippy::eq_op)] - let tensor = (&tensor - &tensor)?; - let content: Vec> = tensor.to_vec2()?; - assert_eq!(content[0], [0., 0., 0., 0., 0.]); - - let min = tensor1.minimum(&(&tensor2 * 0.5)?)?; - let max = tensor1.maximum(&(&tensor2 * 0.5)?)?; - assert_eq!( - min.to_vec2::()?, - [[2.5, 1.0, 2.5, 1.0, 2.5], [1.0, 0.5, 3.5, 4.0, 1.0]], - ); - assert_eq!( - max.to_vec2::()?, - [[3.0, 2.5, 4.0, 2.5, 5.0], [2.0, 1.0, 7.0, 8.0, 2.0]] - ); - Ok(()) -} - -fn transpose(device: &Device) -> Result<()> { - let data = &[[3f32, 1., 4., 1., 5.], [2., 1., 7., 8., 2.]]; - let tensor = Tensor::new(data, device)?.t()?; - let dims = tensor.dims2()?; - assert_eq!(dims, (5, 2)); - assert_eq!( - tensor.to_vec2::()?, - &[[3f32, 2.], [1., 1.], [4., 7.], [1., 8.], [5., 2.]] - ); - assert_eq!(tensor.t()?.to_vec2::()?, data); - assert_eq!(tensor.contiguous()?.t()?.to_vec2::()?, data); - assert_eq!(((tensor + 1.)?.t()? - 1.)?.to_vec2::()?, data); - Ok(()) -} - -fn var(device: &Device) -> Result<()> { - // Values taken from https://pytorch.org/docs/stable/generated/torch.var.html - let data = &[ - [0.2035f32, 1.2959, 1.8101, -0.4644], - [1.5027, -0.3270, 0.5905, 0.6538], - [-1.5745, 1.3330, -0.5596, -0.6548], - [0.1264, -0.5080, 1.6420, 0.1992], - ]; - let tensor = Tensor::new(data, device)?; - assert_eq!( - test_utils::to_vec2_round(&tensor.var_keepdim(1)?, 4)?, - &[[1.0631], [0.559], [1.4893], [0.8258]] - ); - Ok(()) -} - -fn sum(device: &Device) -> Result<()> { - let data = &[[[3u32, 1, 4], [1, 5, 9]], [[2, 1, 7], [8, 2, 8]]]; - let tensor = Tensor::new(data, device)?; - assert_eq!( - tensor.sum_keepdim(2)?.to_vec3::()?, - &[[[8], [15]], [[10], [18]]] - ); - assert_eq!( - tensor.sum_keepdim(0)?.to_vec3::()?, - &[[[5, 2, 11], [9, 7, 17]]], - ); - assert_eq!(tensor.sum_keepdim((0, 2, 1))?.to_vec3::()?, &[[[51]]],); - assert_eq!( - tensor.t()?.sum_keepdim(1)?.t()?.to_vec3::()?, - &[[[8], [15]], [[10], [18]]] - ); - assert_eq!( - tensor.sum_keepdim((2, 1))?.to_vec3::()?, - &[[[8 + 15]], [[10 + 18]]] - ); - let data: Vec = (0..4000u32).collect(); - let tensor = Tensor::new(data.as_slice(), device)?; - assert_eq!(tensor.sum_keepdim(0)?.to_vec1::()?, &[7998000]); - let tensor = tensor.reshape((2000, 2))?; - assert_eq!(tensor.sum_keepdim((0, 1))?.to_vec2::()?, &[[7998000]]); - assert_eq!( - tensor.sum_keepdim(0)?.sum_keepdim(1)?.to_vec2::()?, - &[[7998000]] - ); - assert_eq!( - tensor.sum_keepdim(1)?.sum_keepdim(0)?.to_vec2::()?, - &[[7998000]] - ); - assert_eq!( - tensor.sum_keepdim(0)?.to_vec2::()?, - &[[3998000, 4000000]] - ); - - // Make the tensor non contiguous. - let tensor = tensor.t()?.contiguous()?.t()?; - assert_eq!(tensor.sum_keepdim((0, 1))?.to_vec2::()?, &[[7998000]]); - assert_eq!( - tensor.sum_keepdim(0)?.sum_keepdim(1)?.to_vec2::()?, - &[[7998000]] - ); - assert_eq!( - tensor.sum_keepdim(1)?.sum_keepdim(0)?.to_vec2::()?, - &[[7998000]] - ); - assert_eq!( - tensor.sum_keepdim(0)?.to_vec2::()?, - &[[3998000, 4000000]] - ); - - let t1 = tensor.reshape((200, 5, 4))?; - let t2 = t1.transpose(0, 2)?.contiguous()?.transpose(0, 2)?; - for tensor in [t1, t2] { - assert_eq!( - tensor.sum_keepdim((0, 1, 2))?.to_vec3::()?, - &[[[7998000]]] - ); - assert_eq!( - tensor - .sum_keepdim(0)? - .sum_keepdim(2)? - .sum_keepdim(1)? - .to_vec3::()?, - &[[[7998000]]] - ); - assert_eq!( - tensor - .sum_keepdim(0)? - .sum_keepdim((1, 2))? - .to_vec3::()?, - &[[[7998000]]] - ); - assert_eq!( - tensor - .sum_keepdim(1)? - .sum_keepdim((0, 2))? - .to_vec3::()?, - &[[[7998000]]] - ); - assert_eq!( - tensor.sum_keepdim(0)?.to_vec3::()?, - &[[ - [398000, 398200, 398400, 398600], - [398800, 399000, 399200, 399400], - [399600, 399800, 400000, 400200], - [400400, 400600, 400800, 401000], - [401200, 401400, 401600, 401800] - ]] - ); - } - Ok(()) -} - -fn min(device: &Device) -> Result<()> { - let data = &[[[3u32, 1, 4], [1, 5, 9]], [[2, 1, 7], [8, 2, 8]]]; - let tensor = Tensor::new(data, device)?; - assert_eq!( - tensor.min_keepdim(2)?.to_vec3::()?, - &[[[1], [1]], [[1], [2]]] - ); - assert_eq!( - tensor.min_keepdim(0)?.to_vec3::()?, - &[[[2, 1, 4], [1, 2, 8]]], - ); - let data: Vec = (200..4000u32).collect(); - let tensor = Tensor::new(data.as_slice(), device)?; - assert_eq!(tensor.min_keepdim(0)?.to_vec1::()?, &[200]); - let tensor = tensor.reshape((1900, 2))?; - assert_eq!( - tensor.min_keepdim(0)?.min_keepdim(1)?.to_vec2::()?, - &[[200]] - ); - assert_eq!( - tensor.min_keepdim(1)?.min_keepdim(0)?.to_vec2::()?, - &[[200]] - ); - assert_eq!(tensor.min_keepdim(0)?.to_vec2::()?, &[[200, 201]]); - - // Make the tensor non contiguous. - let tensor = tensor.t()?.contiguous()?.t()?; - assert_eq!( - tensor.min_keepdim(0)?.min_keepdim(1)?.to_vec2::()?, - &[[200]] - ); - assert_eq!( - tensor.min_keepdim(1)?.min_keepdim(0)?.to_vec2::()?, - &[[200]] - ); - assert_eq!(tensor.min_keepdim(0)?.to_vec2::()?, &[[200, 201]]); - - let t1 = tensor.reshape((190, 5, 4))?; - let t2 = t1.transpose(0, 2)?.contiguous()?.transpose(0, 2)?; - for tensor in [t1, t2] { - assert_eq!( - tensor - .min_keepdim(0)? - .min_keepdim(2)? - .min_keepdim(1)? - .to_vec3::()?, - &[[[200]]] - ); - assert_eq!( - tensor.min_keepdim(0)?.to_vec3::()?, - &[[ - [200, 201, 202, 203], - [204, 205, 206, 207], - [208, 209, 210, 211], - [212, 213, 214, 215], - [216, 217, 218, 219] - ]] - ); - } - Ok(()) -} - -fn max(device: &Device) -> Result<()> { - let data = &[[[3u32, 1, 4], [1, 5, 9]], [[2, 1, 7], [8, 2, 8]]]; - let tensor = Tensor::new(data, device)?; - assert_eq!( - tensor.max_keepdim(2)?.to_vec3::()?, - &[[[4], [9]], [[7], [8]]] - ); - assert_eq!( - tensor.max_keepdim(0)?.to_vec3::()?, - &[[[3, 1, 7], [8, 5, 9]]], - ); - let data: Vec = (200..4000u32).collect(); - let tensor = Tensor::new(data.as_slice(), device)?; - assert_eq!(tensor.max_keepdim(0)?.to_vec1::()?, &[3999]); - let tensor = tensor.reshape((1900, 2))?; - assert_eq!( - tensor.max_keepdim(0)?.max_keepdim(1)?.to_vec2::()?, - &[[3999]] - ); - assert_eq!( - tensor.max_keepdim(1)?.max_keepdim(0)?.to_vec2::()?, - &[[3999]] - ); - assert_eq!(tensor.max_keepdim(0)?.to_vec2::()?, &[[3998, 3999]]); - - // Make the tensor non contiguous. - let tensor = tensor.t()?.contiguous()?.t()?; - assert_eq!( - tensor.max_keepdim(0)?.max_keepdim(1)?.to_vec2::()?, - &[[3999]] - ); - assert_eq!( - tensor.max_keepdim(1)?.max_keepdim(0)?.to_vec2::()?, - &[[3999]] - ); - assert_eq!(tensor.max_keepdim(0)?.to_vec2::()?, &[[3998, 3999]]); - - let t1 = tensor.reshape((190, 5, 4))?; - let t2 = t1.transpose(0, 2)?.contiguous()?.transpose(0, 2)?; - for tensor in [t1, t2] { - assert_eq!( - tensor - .max_keepdim(0)? - .max_keepdim(2)? - .max_keepdim(1)? - .to_vec3::()?, - &[[[3999]]] - ); - assert_eq!( - tensor.max_keepdim(0)?.to_vec3::()?, - &[[ - [3980, 3981, 3982, 3983], - [3984, 3985, 3986, 3987], - [3988, 3989, 3990, 3991], - [3992, 3993, 3994, 3995], - [3996, 3997, 3998, 3999] - ]] - ); - } - Ok(()) -} - -fn argmin(device: &Device) -> Result<()> { - let data = &[[[3u32, 1, 4], [1, 5, 9]], [[2, 1, 7], [8, 2, 8]]]; - let tensor = Tensor::new(data, device)?; - assert_eq!( - tensor.argmin_keepdim(2)?.to_vec3::()?, - &[[[1], [0]], [[1], [1]]] - ); - assert_eq!( - tensor.argmin_keepdim(0)?.to_vec3::()?, - &[[[1, 0, 0], [0, 1, 1]]], - ); - let data: Vec = (200..4000u32).collect(); - let tensor = Tensor::new(data.as_slice(), device)?; - assert_eq!(tensor.argmin_keepdim(0)?.to_vec1::()?, &[0]); - let tensor = tensor.reshape((1900, 2))?; - assert_eq!( - tensor - .argmin_keepdim(0)? - .argmin_keepdim(1)? - .to_vec2::()?, - &[[0]] - ); - assert_eq!( - tensor - .argmin_keepdim(1)? - .argmin_keepdim(0)? - .to_vec2::()?, - &[[0]] - ); - assert_eq!(tensor.argmin_keepdim(0)?.to_vec2::()?, &[[0, 0]]); - - // Make the tensor non contiguous. - let tensor = tensor.t()?.contiguous()?.t()?; - assert_eq!( - tensor - .argmin_keepdim(0)? - .argmin_keepdim(1)? - .to_vec2::()?, - &[[0]] - ); - assert_eq!( - tensor - .argmin_keepdim(1)? - .argmin_keepdim(0)? - .to_vec2::()?, - &[[0]] - ); - assert_eq!(tensor.argmin_keepdim(0)?.to_vec2::()?, &[[0, 0]]); - - let t1 = tensor.reshape((190, 5, 4))?; - let t2 = t1.transpose(0, 2)?.contiguous()?.transpose(0, 2)?; - for tensor in [t1, t2] { - assert_eq!( - tensor - .argmin_keepdim(0)? - .argmin_keepdim(2)? - .argmin_keepdim(1)? - .to_vec3::()?, - &[[[0]]] - ); - assert_eq!( - tensor.argmin_keepdim(0)?.to_vec3::()?, - &[[ - [0, 0, 0, 0], - [0, 0, 0, 0], - [0, 0, 0, 0], - [0, 0, 0, 0], - [0, 0, 0, 0], - ]] - ); - } - Ok(()) -} - -fn argmax(device: &Device) -> Result<()> { - let data = &[[[3u32, 1, 4], [1, 5, 9]], [[2, 1, 7], [8, 2, 8]]]; - let tensor = Tensor::new(data, device)?; - assert_eq!( - tensor.argmax_keepdim(2)?.to_vec3::()?, - &[[[2], [2]], [[2], [0]]] - ); - assert_eq!( - tensor.argmax_keepdim(0)?.to_vec3::()?, - &[[[0, 0, 1], [1, 0, 0]]], - ); - let data: Vec = (200..4000u32).collect(); - let tensor = Tensor::new(data.as_slice(), device)?; - assert_eq!(tensor.argmax_keepdim(0)?.to_vec1::()?, &[3799]); - let tensor = tensor.reshape((1900, 2))?; - assert_eq!( - tensor - .argmax_keepdim(0)? - .argmax_keepdim(1)? - .to_vec2::()?, - &[[0]] - ); - assert_eq!( - tensor - .argmax_keepdim(1)? - .argmax_keepdim(0)? - .to_vec2::()?, - &[[0]] - ); - assert_eq!(tensor.argmax_keepdim(0)?.to_vec2::()?, &[[1899, 1899]]); - - // Make the tensor non contiguous. - let tensor = tensor.t()?.contiguous()?.t()?; - assert_eq!( - tensor - .argmax_keepdim(0)? - .argmax_keepdim(1)? - .to_vec2::()?, - &[[0]] - ); - assert_eq!( - tensor - .argmax_keepdim(1)? - .argmax_keepdim(0)? - .to_vec2::()?, - &[[0]] - ); - assert_eq!(tensor.argmax_keepdim(0)?.to_vec2::()?, &[[1899, 1899]]); - - let t1 = tensor.reshape((190, 5, 4))?; - let t2 = t1.transpose(0, 2)?.contiguous()?.transpose(0, 2)?; - for tensor in [t1, t2] { - assert_eq!( - tensor - .argmax_keepdim(0)? - .argmax_keepdim(2)? - .argmax_keepdim(1)? - .to_vec3::()?, - &[[[0]]] - ); - assert_eq!( - tensor.argmax_keepdim(0)?.to_vec3::()?, - &[[ - [189, 189, 189, 189], - [189, 189, 189, 189], - [189, 189, 189, 189], - [189, 189, 189, 189], - [189, 189, 189, 189], - ]] - ); - } - Ok(()) -} - -fn narrow(device: &Device) -> Result<()> { - let data = &[[[3f32, 1., 4.], [1., 5., 9.]], [[2., 1., 7.], [8., 2., 8.]]]; - let tensor = Tensor::new(data, device)?; - assert_eq!( - tensor.narrow(2, 1, 2)?.to_vec3::()?, - &[[[1.0, 4.0], [5.0, 9.0]], [[1.0, 7.0], [2.0, 8.0]]], - ); - assert_eq!( - tensor.narrow(1, 1, 1)?.to_vec3::()?, - &[[[1.0, 5.0, 9.0]], [[8.0, 2.0, 8.0]]], - ); - assert_eq!( - tensor.narrow(0, 0, 1)?.to_vec3::()?, - &[[[3.0, 1.0, 4.0], [1.0, 5.0, 9.0]]], - ); - assert_eq!( - tensor.narrow(0, 1, 1)?.to_vec3::()?, - &[[[2.0, 1.0, 7.0], [8.0, 2.0, 8.0]]], - ); - // The following has been checked against PyTorch via: - // import torch - // t = torch.tensor([[[3., 1., 4.], [1., 5., 9.]], [[2., 1., 7.], [8., 2., 8.]]]) - // t.transpose(-1, -2).narrow(1, 1, 2) - assert_eq!( - tensor.t()?.narrow(1, 1, 2)?.to_vec3::()?, - &[[[1.0, 5.0], [4.0, 9.0]], [[1.0, 2.0], [7.0, 8.0]]], - ); - Ok(()) -} - -fn broadcast(device: &Device) -> Result<()> { - let data = &[3f32, 1., 4.]; - let tensor = Tensor::new(data, device)?; - assert_eq!( - tensor.broadcast_left((3, 1))?.to_vec3::()?, - &[[[3.0, 1.0, 4.0]], [[3.0, 1.0, 4.0]], [[3.0, 1.0, 4.0]]] - ); - Ok(()) -} - -fn cat(device: &Device) -> Result<()> { - // 1D - let t1 = Tensor::new(&[3f32, 1., 4.], device)?; - let t2 = Tensor::new(&[1f32, 5., 9., 2.], device)?; - let t3 = Tensor::new(&[6f32, 5., 3., 5., 8., 9.], device)?; - assert_eq!(Tensor::cat(&[&t1], 0)?.to_vec1::()?, [3f32, 1., 4.],); - assert_eq!( - Tensor::cat(&[&t1, &t2], 0)?.to_vec1::()?, - [3f32, 1., 4., 1., 5., 9., 2.], - ); - assert_eq!( - Tensor::cat(&[&t1, &t2, &t3], 0)?.to_vec1::()?, - [3f32, 1., 4., 1., 5., 9., 2., 6., 5., 3., 5., 8., 9.], - ); - - // 2D - let data = &[[3f32, 1., 4., 1., 5.], [2., 7., 1., 8., 2.]]; - let t1 = Tensor::new(data, device)?; - let data2 = &[[5f32, 5., 5., 5., 5.], [2., 7., 1., 8., 2.]]; - let t2 = Tensor::new(data2, device)?; - assert_eq!( - Tensor::cat(&[&t1, &t2], 0)?.to_vec2::()?, - [ - [3.0, 1.0, 4.0, 1.0, 5.0], - [2.0, 7.0, 1.0, 8.0, 2.0], - [5.0, 5.0, 5.0, 5.0, 5.0], - [2.0, 7.0, 1.0, 8.0, 2.0] - ] - ); - // PyTorch equivalent: - // import torch - // t1 = torch.tensor([[3, 1, 4, 1, 5], [2, 7, 1, 8, 2]]) - // t2 = torch.tensor([[5]*5, [2, 7, 1, 8, 2]]) - // torch.cat([t1.t(), t2.t()], dim=1).t() - assert_eq!( - Tensor::cat(&[&t1.t()?, &t2.t()?], 1)? - .t()? - .to_vec2::()?, - [ - [3.0, 1.0, 4.0, 1.0, 5.0], - [2.0, 7.0, 1.0, 8.0, 2.0], - [5.0, 5.0, 5.0, 5.0, 5.0], - [2.0, 7.0, 1.0, 8.0, 2.0] - ] - ); - assert_eq!( - Tensor::cat(&[&t1, &t2], 1)?.to_vec2::()?, - [ - [3.0, 1.0, 4.0, 1.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0], - [2.0, 7.0, 1.0, 8.0, 2.0, 2.0, 7.0, 1.0, 8.0, 2.0] - ] - ); - Ok(()) -} - -fn embeddings(device: &Device) -> Result<()> { - let ids = Tensor::new(&[0u32, 2u32, 1u32], device)?; - let t = Tensor::new(&[[0f32, 1f32], [2f32, 3f32], [4f32, 5f32]], device)?; - let hs = t.embedding(&ids)?; - assert_eq!(hs.to_vec2::()?, &[[0.0, 1.0], [4.0, 5.0], [2.0, 3.0]]); - let hs = t.index_select(&ids, 0)?; - assert_eq!(hs.to_vec2::()?, &[[0.0, 1.0], [4.0, 5.0], [2.0, 3.0]]); - Ok(()) -} - -fn cmp(device: &Device) -> Result<()> { - let t1 = Tensor::new(&[[0f32, 1f32], [2f32, 3f32], [4f32, 5f32]], device)?; - let t2 = Tensor::new(&[[1f32, 0f32], [3f32, 3f32], [4f32, 7f32]], device)?; - assert_eq!(t1.eq(&t2)?.to_vec2::()?, &[[0, 0], [0, 1], [1, 0]]); - assert_eq!(t1.ne(&t2)?.to_vec2::()?, &[[1, 1], [1, 0], [0, 1]]); - assert_eq!(t1.le(&t2)?.to_vec2::()?, &[[1, 0], [1, 1], [1, 1]]); - assert_eq!(t1.lt(&t2)?.to_vec2::()?, &[[1, 0], [1, 0], [0, 1]]); - assert_eq!(t1.gt(&t2)?.to_vec2::()?, &[[0, 1], [0, 0], [0, 0]]); - assert_eq!(t1.ge(&t2)?.to_vec2::()?, &[[0, 1], [0, 1], [1, 0]]); - Ok(()) -} - -fn index_select(device: &Device) -> Result<()> { - let ids = Tensor::new(&[0u32, 2u32, 1u32], device)?; - let t = Tensor::arange(0f32, 12f32, device)?.reshape((4, 3))?; - assert_eq!( - t.to_vec2::()?, - &[ - [0.0, 1.0, 2.0], - [3.0, 4.0, 5.0], - [6.0, 7.0, 8.0], - [9.0, 10.0, 11.0] - ] - ); - let hs = t.index_select(&ids, 1)?; - assert_eq!( - hs.to_vec2::()?, - &[ - [0.0, 2.0, 1.0], - [3.0, 5.0, 4.0], - [6.0, 8.0, 7.0], - [9.0, 11.0, 10.0] - ] - ); - let hs = t.index_select(&ids, 0)?; - assert_eq!( - hs.to_vec2::()?, - &[[0.0, 1.0, 2.0], [6.0, 7.0, 8.0], [3.0, 4.0, 5.0]] - ); - // Prior to https://github.com/huggingface/candle/pull/1022 - // There would be a bug where the last values in the result tensor would be set to 0. - let ids = Tensor::new(&[0u32, 2u32, 1u32, 0u32, 2u32, 1u32], device)?; - let hs = t.index_select(&ids, 0)?; - assert_eq!( - hs.to_vec2::()?, - &[ - [0.0, 1.0, 2.0], - [6.0, 7.0, 8.0], - [3.0, 4.0, 5.0], - [0.0, 1.0, 2.0], - [6.0, 7.0, 8.0], - [3.0, 4.0, 5.0], - ] - ); - - // Test when selecting dim > 0 with ids size different from elem count of - // target dim in source/input. - let ids = Tensor::new(&[1u32, 0u32, 1u32], device)?; - let t = Tensor::arange(1f32, 5f32, device)?.reshape((2, 2))?; - assert_eq!(t.to_vec2::()?, &[[1.0, 2.0], [3.0, 4.0]]); - let hs = t.index_select(&ids, 1)?; - assert_eq!(hs.to_vec2::()?, &[[2.0, 1.0, 2.0], [4.0, 3.0, 4.0]]); - - Ok(()) -} - -fn index_add(device: &Device) -> Result<()> { - let ids = Tensor::new(&[0u32, 1u32, 1u32], device)?; - let t = Tensor::arange(0f32, 12f32, device)?.reshape((4, 3))?; - assert_eq!( - t.to_vec2::()?, - &[ - [0.0, 1.0, 2.0], - [3.0, 4.0, 5.0], - [6.0, 7.0, 8.0], - [9.0, 10.0, 11.0] - ] - ); - let init = Tensor::ones((4, 2), DType::F32, device)?; - let hs = init.index_add(&ids, &t, 1)?; - assert_eq!( - hs.to_vec2::()?, - &[[1.0, 4.0], [4.0, 10.0], [7.0, 16.0], [10.0, 22.0]], - ); - let init = Tensor::zeros((4, 2), DType::F32, device)?; - let ids = Tensor::new(&[1u32, 0u32, 0u32], device)?; - let hs = init.index_add(&ids, &t, 1)?; - assert_eq!( - hs.to_vec2::()?, - &[[3.0, 0.0], [9.0, 3.0], [15.0, 6.0], [21.0, 9.0]], - ); - - let init = Tensor::zeros((6, 3), DType::F32, device)?; - let ids = Tensor::new(&[5u32, 0u32, 1u32, 0u32], device)?; - let hs = init.index_add(&ids, &t, 0)?; - assert_eq!( - hs.to_vec2::()?, - &[ - [12.0, 14.0, 16.0], - [6.0, 7.0, 8.0], - [0.0, 0.0, 0.0], - [0.0, 0.0, 0.0], - [0.0, 0.0, 0.0], - [0.0, 1.0, 2.0] - ] - ); - Ok(()) -} - -fn slice_scatter(device: &Device) -> Result<()> { - let t = Tensor::arange(0f32, 12f32, device)?.reshape((4, 3))?; - assert_eq!( - t.to_vec2::()?, - &[ - [0.0, 1.0, 2.0], - [3.0, 4.0, 5.0], - [6.0, 7.0, 8.0], - [9.0, 10.0, 11.0] - ] - ); - let src = Tensor::arange(100f32, 106f32, device)?.reshape((2, 3))?; - assert_eq!( - t.slice_scatter0(&src, 0)?.to_vec2::()?, - &[ - [100.0, 101.0, 102.0], - [103.0, 104.0, 105.0], - [6.0, 7.0, 8.0], - [9.0, 10.0, 11.0] - ] - ); - assert_eq!( - t.slice_scatter0(&src, 1)?.to_vec2::()?, - &[ - [0.0, 1.0, 2.0], - [100.0, 101.0, 102.0], - [103.0, 104.0, 105.0], - [9.0, 10.0, 11.0] - ] - ); - assert_eq!( - t.slice_scatter0(&src, 2)?.to_vec2::()?, - &[ - [0.0, 1.0, 2.0], - [3.0, 4.0, 5.0], - [100.0, 101.0, 102.0], - [103.0, 104.0, 105.0], - ] - ); - Ok(()) -} - -fn scatter_add(device: &Device) -> Result<()> { - let t = Tensor::arange(0f32, 12f32, device)?.reshape((4, 3))?; - assert_eq!( - t.to_vec2::()?, - &[ - [0.0, 1.0, 2.0], - [3.0, 4.0, 5.0], - [6.0, 7.0, 8.0], - [9.0, 10.0, 11.0] - ] - ); - let ids = Tensor::new(&[[0u32, 1, 2], [3, 4, 0], [3, 3, 1], [2, 0, 4]], device)?; - let init = Tensor::ones((4, 5), DType::F32, device)?; - let hs = init.scatter_add(&ids, &t, 1)?; - assert_eq!( - hs.to_vec2::()?, - &[ - [1.0, 2.0, 3.0, 1.0, 1.0], - [6.0, 1.0, 1.0, 4.0, 5.0], - [1.0, 9.0, 1.0, 14.0, 1.0], - [11.0, 1.0, 10.0, 1.0, 12.0] - ] - ); - - let init = Tensor::ones((6, 3), DType::F32, device)?; - let hs = init.scatter_add(&ids, &t, 0)?; - assert_eq!( - hs.to_vec2::()?, - &[ - [1.0, 11.0, 6.0], - [1.0, 2.0, 9.0], - [10.0, 1.0, 3.0], - [10.0, 8.0, 1.0], - [1.0, 5.0, 12.0], - [1.0, 1.0, 1.0] - ] - ); - Ok(()) -} - -fn gather(device: &Device) -> Result<()> { - let ids = Tensor::new(&[[0u32], [2u32], [1u32], [0u32]], device)?; - let t = Tensor::arange(0f32, 12f32, device)?.reshape((4, 3))?; - assert_eq!( - t.to_vec2::()?, - &[ - [0.0, 1.0, 2.0], - [3.0, 4.0, 5.0], - [6.0, 7.0, 8.0], - [9.0, 10.0, 11.0] - ] - ); - let hs = t.gather(&ids, 1)?; - assert_eq!(hs.to_vec2::()?, &[[0.0], [5.0], [7.0], [9.0]]); - let ids = Tensor::new( - &[[0u32, 0u32], [2u32, 0u32], [1u32, 1u32], [0u32, 2u32]], - device, - )?; - let hs = t.gather(&ids, 1)?; - assert_eq!( - hs.to_vec2::()?, - &[[0.0, 0.0], [5.0, 3.0], [7.0, 7.0], [9.0, 11.0]] - ); - let ids = Tensor::new(&[[0u32, 2u32, 0u32]], device)?; - let hs = t.gather(&ids, 0)?; - assert_eq!(hs.to_vec2::()?, &[[0.0, 7.0, 2.0]]); - let ids = Tensor::new(&[[0u32, 2u32, 0u32], [0u32, 1u32, 1u32]], device)?; - let hs = t.gather(&ids, 0)?; - assert_eq!(hs.to_vec2::()?, &[[0.0, 7.0, 2.0], [0.0, 4.0, 5.0]]); - Ok(()) -} - -fn matmul(device: &Device) -> Result<()> { - let data = vec![1.0f32, 2.0, 3.0, 4.0]; - let a = Tensor::from_slice(&data, (2, 2), device)?; - let data = vec![1.0f32, 2.0, 3.0, 4.0]; - let b = Tensor::from_slice(&data, (2, 2), device)?; - - let c = a.matmul(&b)?; - assert_eq!(c.to_vec2::()?, &[[7.0f32, 10.0], [15.0, 22.0]]); - - let data = vec![1.0f32, 2.0]; - let a = Tensor::from_slice(&data, (2, 1), device)?; - let data = vec![3.0f32, 4.0]; - let b = Tensor::from_slice(&data, (1, 2), device)?; - let c = a.matmul(&b)?; - assert_eq!(c.to_vec2::()?, &[&[3.0, 4.0], &[6.0, 8.0]]); - - let data: Vec<_> = (0..6).map(|i| i as f32).collect(); - let a = Tensor::from_slice(&data, (2, 3), device)?; - let data: Vec<_> = (0..6).map(|i| (i + 2) as f32).collect(); - let b = Tensor::from_slice(&data, (3, 2), device)?; - let c = a.matmul(&b)?; - assert_eq!(c.to_vec2::()?, &[&[16., 19.], &[52., 64.]]); - - let data: Vec<_> = (0..12).map(|i| i as f32).collect(); - let a = Tensor::from_slice(&data, (2, 2, 3), device)?; - let data: Vec<_> = (0..12).map(|i| (i + 2) as f32).collect(); - let b = Tensor::from_slice(&data, (2, 3, 2), device)?; - let expected = [[[16., 19.], [52., 64.]], [[214., 235.], [304., 334.]]]; - - let c = a.matmul(&b)?; - assert_eq!(c.to_vec3::()?, &expected); - - // Also perform the matmul on contiguous transposed versions. - let a_tt = a.t()?.contiguous()?.t()?; - assert!(!a_tt.is_contiguous()); - assert_eq!(a.dims(), a_tt.dims()); - assert_eq!(a_tt.stride(), &[6, 1, 2]); - - let b_tt = b.t()?.contiguous()?.t()?; - assert!(!b_tt.is_contiguous()); - assert_eq!(b.dims(), b_tt.dims()); - assert_eq!(b_tt.stride(), &[6, 1, 3]); - - assert_eq!(a_tt.matmul(&b)?.to_vec3::()?, &expected); - assert_eq!(a.matmul(&b_tt)?.to_vec3::()?, &expected); - assert_eq!(a_tt.matmul(&b_tt)?.to_vec3::()?, &expected); - Ok(()) -} - -fn broadcast_matmul(device: &Device) -> Result<()> { - let lhs = Tensor::randn(0f32, 1f32, (3, 1, 4, 5), device)?; - let rhs = Tensor::randn(0f32, 1f32, (6, 5, 2), device)?; - let out = lhs.broadcast_matmul(&rhs)?; - assert_eq!(out.dims(), &[3, 6, 4, 2]); - for idx1 in 0..3 { - for idx2 in 0..6 { - let out = out.i((idx1, idx2))?; - let lhs = lhs.i((idx1, 0))?; - let rhs = rhs.i(idx2)?; - let out2 = lhs.matmul(&rhs); - let sum_diff2 = (out - out2)?.sqr()?.sum_all()?; - // With cuda, we see errors of up to ~1e-12. - assert!(sum_diff2.to_vec0::()? < 1e-6) - } - } - Ok(()) -} - -fn broadcasting(device: &Device) -> Result<()> { - let t1 = Tensor::arange(0f32, 24f32, device)?.reshape((4, 2, 3))?; - let t2 = Tensor::new(&[100f32, 200f32], device)?; - let s = t1.broadcast_add(&t2.reshape((2, 1))?)?; - assert_eq!( - s.to_vec3::()?, - &[ - [[100.0, 101.0, 102.0], [203.0, 204.0, 205.0]], - [[106.0, 107.0, 108.0], [209.0, 210.0, 211.0]], - [[112.0, 113.0, 114.0], [215.0, 216.0, 217.0]], - [[118.0, 119.0, 120.0], [221.0, 222.0, 223.0]] - ] - ); - let s = t1.t()?.broadcast_add(&t2)?; - assert_eq!( - s.to_vec3::()?, - &[ - [[100.0, 203.0], [101.0, 204.0], [102.0, 205.0]], - [[106.0, 209.0], [107.0, 210.0], [108.0, 211.0]], - [[112.0, 215.0], [113.0, 216.0], [114.0, 217.0]], - [[118.0, 221.0], [119.0, 222.0], [120.0, 223.0]] - ] - ); - let s = t1.broadcast_sub(&t2.reshape((2, 1))?)?; - assert_eq!( - s.to_vec3::()?, - &[ - [[-100.0, -99.0, -98.0], [-197.0, -196.0, -195.0]], - [[-94.0, -93.0, -92.0], [-191.0, -190.0, -189.0]], - [[-88.0, -87.0, -86.0], [-185.0, -184.0, -183.0]], - [[-82.0, -81.0, -80.0], [-179.0, -178.0, -177.0]] - ] - ); - let s = t1.t()?.broadcast_sub(&t2)?; - assert_eq!( - s.to_vec3::()?, - &[ - [[-100.0, -197.0], [-99.0, -196.0], [-98.0, -195.0]], - [[-94.0, -191.0], [-93.0, -190.0], [-92.0, -189.0]], - [[-88.0, -185.0], [-87.0, -184.0], [-86.0, -183.0]], - [[-82.0, -179.0], [-81.0, -178.0], [-80.0, -177.0]] - ] - ); - // Test a narrowed version as this uses a layout start_offset. - let t1 = t1.i(2..)?; - let s = t1.broadcast_add(&t2.reshape((2, 1))?)?; - assert_eq!( - s.to_vec3::()?, - &[ - [[112.0, 113.0, 114.0], [215.0, 216.0, 217.0]], - [[118.0, 119.0, 120.0], [221.0, 222.0, 223.0]] - ] - ); - let s = t1.t()?.broadcast_add(&t2)?; - assert_eq!( - s.to_vec3::()?, - &[ - [[112.0, 215.0], [113.0, 216.0], [114.0, 217.0]], - [[118.0, 221.0], [119.0, 222.0], [120.0, 223.0]] - ] - ); - let s = t1.broadcast_sub(&t2.reshape((2, 1))?)?; - assert_eq!( - s.to_vec3::()?, - &[ - [[-88.0, -87.0, -86.0], [-185.0, -184.0, -183.0]], - [[-82.0, -81.0, -80.0], [-179.0, -178.0, -177.0]] - ] - ); - let s = t1.t()?.broadcast_sub(&t2)?; - assert_eq!( - s.to_vec3::()?, - &[ - [[-88.0, -185.0], [-87.0, -184.0], [-86.0, -183.0]], - [[-82.0, -179.0], [-81.0, -178.0], [-80.0, -177.0]] - ] - ); - let t3 = Tensor::new(1f32, device)?.broadcast_div(&t2)?; - let s = t1.broadcast_mul(&t2.reshape((2, 1))?)?; - let s_div = t1.broadcast_div(&t3.reshape((2, 1))?)?; - assert_eq!( - s.to_vec3::()?, - &[ - [[1200.0, 1300.0, 1400.0], [3000.0, 3200.0, 3400.0]], - [[1800.0, 1900.0, 2000.0], [4200.0, 4400.0, 4600.0]] - ] - ); - assert_eq!(s.to_vec3::()?, s_div.to_vec3::()?,); - let s = t1.t()?.broadcast_mul(&t2)?; - let s_div = t1.t()?.broadcast_div(&t3)?; - assert_eq!( - s.to_vec3::()?, - &[ - [[1200.0, 3000.0], [1300.0, 3200.0], [1400.0, 3400.0]], - [[1800.0, 4200.0], [1900.0, 4400.0], [2000.0, 4600.0]] - ] - ); - assert_eq!(s.to_vec3::()?, s_div.to_vec3::()?,); - Ok(()) -} - -fn randn(device: &Device) -> Result<()> { - let tensor = Tensor::randn(0f32, 1f32, (5, 3), device)?; - assert_eq!(tensor.dims(), [5, 3]); - let tensor = Tensor::rand(0f32, 1f32, (5, 3), device)?; - assert_eq!(tensor.dims(), [5, 3]); - Ok(()) -} - -test_device!(zeros, zeros_cpu, zeros_gpu, zeros_metal); -test_device!(ones, ones_cpu, ones_gpu, ones_metal); -test_device!(full, full_cpu, full_gpu, full_metal); -test_device!(arange, arange_cpu, arange_gpu, arange_metal); -test_device!(add_mul, add_mul_cpu, add_mul_gpu, add_mul_metal); -test_device!(tensor_2d, tensor_2d_cpu, tensor_2d_gpu, tensor_2d_metal); -test_device!(narrow, narrow_cpu, narrow_gpu, narrow_metal); -test_device!(broadcast, broadcast_cpu, broadcast_gpu, broadcast_metal); -test_device!(cat, cat_cpu, cat_gpu, cat_metal); -test_device!(sum, sum_cpu, sum_gpu, sum_metal); -test_device!(min, min_cpu, min_gpu, min_metal); -test_device!(max, max_cpu, max_gpu, max_metal); -test_device!(argmax, argmax_cpu, argmax_gpu, argmax_metal); -test_device!(argmin, argmin_cpu, argmin_gpu, argmin_metal); -test_device!(transpose, transpose_cpu, transpose_gpu, transpose_metal); -test_device!(unary_op, unary_op_cpu, unary_op_gpu, unary_op_metal); -test_device!(binary_op, binary_op_cpu, binary_op_gpu, binary_op_metal); -test_device!(embeddings, embeddings_cpu, embeddings_gpu, embeddings_metal); -test_device!(cmp, cmp_cpu, cmp_gpu, cmp_metal); -test_device!(matmul, matmul_cpu, matmul_gpu, matmul_metal); -test_device!( - broadcast_matmul, - broadcast_matmul_cpu, - broadcast_matmul_gpu, - broadcast_matmul_metal -); -test_device!( - broadcasting, - broadcasting_cpu, - broadcasting_gpu, - broadcasting_metal -); -test_device!( - index_select, - index_select_cpu, - index_select_gpu, - index_select_metal -); -test_device!(index_add, index_add_cpu, index_add_gpu, index_add_metal); -test_device!(gather, gather_cpu, gather_gpu, gather_metal); -test_device!( - scatter_add, - scatter_add_cpu, - scatter_add_gpu, - scatter_add_metal -); -test_device!( - slice_scatter, - slice_scatter_cpu, - slice_scatter_gpu, - slice_scatter_metal -); -test_device!(randn, randn_cpu, randn_gpu, randn_metal); -test_device!(clamp, clamp_cpu, clamp_gpu, clamp_metal); -test_device!(var, var_cpu, var_gpu, var_metal); - -// There was originally a bug on the CPU implementation for randn -// https://github.com/huggingface/candle/issues/381 -#[test] -fn randn_hasneg() -> Result<()> { - let t = Tensor::randn(0f32, 1f32, 200, &Device::Cpu)?.to_vec1::()?; - if t.iter().all(|&v| v >= 0.) { - candle_core::bail!("all values in tensors are non-negative") - } - Ok(()) -} - -#[test] -fn pad_with_same() -> Result<()> { - let t = Tensor::arange(1f32, 5f32, &Device::Cpu)?.reshape((2, 2))?; - let t0 = t.pad_with_same(0, 1, 2)?; - assert_eq!( - t0.to_vec2::()?, - [[1.0, 2.0], [1.0, 2.0], [3.0, 4.0], [3.0, 4.0], [3.0, 4.0]] - ); - let t1 = t.pad_with_same(1, 1, 2)?; - assert_eq!( - t1.to_vec2::()?, - [[1.0, 1.0, 2.0, 2.0, 2.0], [3.0, 3.0, 4.0, 4.0, 4.0]] - ); - Ok(()) -} - -#[test] -fn i64_abs() -> Result<()> { - let t = Tensor::new(&[-42i64, 1337], &Device::Cpu)?; - let t = t.abs()?; - assert_eq!(t.to_vec1::()?, [42, 1337]); - Ok(()) -} - -#[test] -fn tril_triu_eye() -> Result<()> { - let t = Tensor::tril2(4, DType::F32, &Device::Cpu)?; - assert_eq!( - t.to_vec2::()?, - [ - [1.0, 0.0, 0.0, 0.0], - [1.0, 1.0, 0.0, 0.0], - [1.0, 1.0, 1.0, 0.0], - [1.0, 1.0, 1.0, 1.0] - ], - ); - let t = Tensor::triu2(4, DType::F32, &Device::Cpu)?; - assert_eq!( - t.to_vec2::()?, - [ - [1.0, 1.0, 1.0, 1.0], - [0.0, 1.0, 1.0, 1.0], - [0.0, 0.0, 1.0, 1.0], - [0.0, 0.0, 0.0, 1.0] - ] - ); - let t = Tensor::eye(4, DType::F32, &Device::Cpu)?; - assert_eq!( - t.to_vec2::()?, - [ - [1.0, 0.0, 0.0, 0.0], - [0.0, 1.0, 0.0, 0.0], - [0.0, 0.0, 1.0, 0.0], - [0.0, 0.0, 0.0, 1.0] - ] - ); - Ok(()) -} - -#[test] -fn cumsum() -> Result<()> { - let t = &[3f32, 1., 4., 1., 5.]; - let t = Tensor::new(t, &Device::Cpu)?; - assert_eq!(t.cumsum(0)?.to_vec1::()?, [3., 4., 8., 9., 14.]); - let t = t.unsqueeze(1)?; - assert_eq!( - t.cumsum(0)?.to_vec2::()?, - [[3.0], [4.0], [8.0], [9.0], [14.0]] - ); - assert_eq!( - t.cumsum(1)?.to_vec2::()?, - [[3.0], [1.0], [4.0], [1.0], [5.0]] - ); - let t = &[[3f32, 1., 4., 1., 5.], [2., 1., 7., 8., 2.]]; - let t = Tensor::new(t, &Device::Cpu)?; - assert_eq!( - t.cumsum(1)?.to_vec2::()?, - [[3.0, 4.0, 8.0, 9.0, 14.0], [2.0, 3.0, 10.0, 18.0, 20.0]], - ); - assert_eq!( - t.cumsum(0)?.to_vec2::()?, - [[3.0, 1.0, 4.0, 1.0, 5.0], [5.0, 2.0, 11.0, 9.0, 7.0]] - ); - Ok(()) -} - -/// A helper function for floating point comparison. Both a and b must be 1D Tensor and contains the same amount of data. -/// Assertion passes if the difference of all pairs of a and b is smaller than epsilon. -fn assert_close(a: &Tensor, b: &Tensor, epsilon: f64) -> Result<()> { - let a_vec: Vec = a.to_vec1()?; - let b_vec: Vec = b.to_vec1()?; - - assert_eq!(a_vec.len(), b_vec.len()); - for (a, b) in a_vec.iter().zip(b_vec.iter()) { - assert!((a - b).abs() < epsilon); - } - Ok(()) -} - -#[test] -fn log_sum_exp() -> Result<()> { - let input = Tensor::new(&[[1f64, 2., 3.], [4., 5., 6.]], &Device::Cpu)?; - let output = input.log_sum_exp(D::Minus1)?; - // The expectations obtained from pytorch. - let expected = Tensor::new(&[3.4076, 6.4076], &Device::Cpu)?; - assert_close(&output, &expected, 0.00001)?; - Ok(()) -} - -#[test] -fn pow() -> Result<()> { - let lhs = Tensor::new(&[[1f32, 2., 3.], [4., 5., 6.]], &Device::Cpu)?; - let rhs = (&lhs - 2.)?; - let res = lhs.pow(&rhs)?; - assert_eq!( - test_utils::to_vec2_round(&res, 4)?, - [[1.0, 1.0, 3.0], [16.0, 125.0, 1296.0001]] - ); - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/test.npy b/v2/dependency-patches/candle-core-0.4.1/tests/test.npy deleted file mode 100644 index a3ff5af9540510ac6518b5a2478bc1a40f992371..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 208 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= wXCxM+0{I$-20EHL3bhL411<(AV1&|4P?{M^vp{K9D9r|?*`YKCl;(ue0P!XpoB#j- diff --git a/v2/dependency-patches/candle-core-0.4.1/tests/test.npz b/v2/dependency-patches/candle-core-0.4.1/tests/test.npz deleted file mode 100644 index b6683caab0e0e9e66cb5ca649aab4980a1e650df..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 668 zcmWIWW@Zs#fB;1X8xBpr3qTGCvoeSRzLK#d@ni)#7KxtMe%?72}p)?1S=7iD#D8ZokVdgfgc5ZyS5W1OUF4@IgfaSj3!+p!yZC;|&vLW_h60tGIXYp+tbR@}8WmEgpM zO8o`Bapb}U_$P4S5AY|DxUgf#!FKBdBaPM{^PXqkJlnY0LP(O(mFS~w#7M$MzRTR< z=}BmrTj)(S-V2@=g(2}+5G@kTjR$rRn5(#DL_r#w{fSQL9lLubi?Fp-^@ z94>>866e`Y|{Pa$jw1Xu^ zrRNx70k_WS4cx+wb1GjPK?hw%dsEs^hBod<4~{Qif95DiyLR*qw3Cf~_?D3fMzYN9 ztFwa^VZc~OfcnG1uN{%oLe^7v(W8H=jugv0xSqGFbKN~i0U=}u0!bi1b%=I$q+P)! zs0)!V96JB>^~;lo{263@=RMAm`r(Yk317Bl3cU5P3=H2t&R;?Yko63^$jBImV5w0; z;=m1qcP?1R34F^`|6dDL)%SE=n`qfpi3(kQ`+4)n zfOiOK<3=i3?);)L`uu%4O8-F>T3q}_3HOKJrO>sHs@S6R-4gZ{-|YIcBc%!~D&8vr zH~Cz}JynrK6?RL=X^y;pKQcsFZQ}Y+30Wu-S|m&nlEnez+N;E>iM#fu4OV8_A{%=Ua&@QUk(LBL$$hm%--75a)d;S;$UxT1w^n387>AHq2KYo>>4lqPFX(eg-Z1eK!Yr}cbF5km4 zMqWW11%Nh;xri}WZ5bp#%(Vi@y9JO`dS!(CFxCqQCOo-FBa=WGBHyXa0>t+Ph|6ba t(q?lj%e+;)Gsl?ec^02%3viYRAMz53|Hu{qk4@UZrG#B*5WJt>{S7%wAvgd4 diff --git a/v2/dependency-patches/candle-core-0.9.2/Cargo.toml b/v2/dependency-patches/candle-core-0.9.2/Cargo.toml deleted file mode 100644 index d4278ccc0a..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/Cargo.toml +++ /dev/null @@ -1,262 +0,0 @@ -# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO -# -# When uploading crates to the registry Cargo will automatically -# "normalize" Cargo.toml files for maximal compatibility -# with all versions of Cargo and also rewrite `path` dependencies -# to registry (e.g., crates.io) dependencies. -# -# If you are reading this file be aware that the original Cargo.toml -# will likely look very different (and much more reasonable). -# See Cargo.toml.orig for the original contents. - -[package] -edition = "2021" -name = "candle-core" -version = "0.9.2" -build = false -autolib = false -autobins = false -autoexamples = false -autotests = false -autobenches = false -description = "Minimalist ML framework." -readme = "README.md" -keywords = [ - "blas", - "tensor", - "machine-learning", -] -categories = ["science"] -license = "MIT OR Apache-2.0" -repository = "https://github.com/huggingface/candle" - -[features] -accelerate = [ - "dep:libc", - "dep:accelerate-src", -] -cuda = [ - "cudarc", - "dep:candle-kernels", - "candle-ug?/cuda", - "float8/cuda", -] -cudnn = [ - "cuda", - "cudarc/cudnn", -] -default = [] -metal = [ - "dep:objc2-metal", - "dep:objc2-foundation", - "dep:candle-metal-kernels", - "candle-ug?/metal", -] -mkl = [ - "dep:libc", - "dep:intel-mkl-src", -] -nccl = [ - "cuda", - "cudarc/nccl", -] -ug = ["dep:candle-ug"] - -[lib] -name = "candle_core" -path = "src/lib.rs" - -[[example]] -name = "basics" -path = "examples/basics.rs" - -[[example]] -name = "cuda_basics" -path = "examples/cuda_basics.rs" -required-features = ["cuda"] - -[[example]] -name = "cuda_sum_benchmark" -path = "examples/cuda_sum_benchmark.rs" - -[[example]] -name = "metal_basics" -path = "examples/metal_basics.rs" -required-features = ["metal"] - -[[test]] -name = "bilinear_tests" -path = "tests/bilinear_tests.rs" - -[[test]] -name = "conv_tests" -path = "tests/conv_tests.rs" - -[[test]] -name = "custom_op_tests" -path = "tests/custom_op_tests.rs" - -[[test]] -name = "display_tests" -path = "tests/display_tests.rs" - -[[test]] -name = "grad_tests" -path = "tests/grad_tests.rs" - -[[test]] -name = "indexing_tests" -path = "tests/indexing_tests.rs" - -[[test]] -name = "layout_tests" -path = "tests/layout_tests.rs" - -[[test]] -name = "matmul_tests" -path = "tests/matmul_tests.rs" - -[[test]] -name = "pool_tests" -path = "tests/pool_tests.rs" - -[[test]] -name = "pth_tests" -path = "tests/pth_tests.rs" - -[[test]] -name = "quantized_tests" -path = "tests/quantized_tests.rs" - -[[test]] -name = "serialization_tests" -path = "tests/serialization_tests.rs" - -[[test]] -name = "tensor_tests" -path = "tests/tensor_tests.rs" - -[[bench]] -name = "bench_main" -path = "benches/bench_main.rs" -harness = false - -[dependencies.accelerate-src] -version = "0.3.2" -optional = true - -[dependencies.byteorder] -version = "1.4.3" - -[dependencies.candle-kernels] -version = "0.9.2" -optional = true - -[dependencies.candle-metal-kernels] -version = "0.9.2" -optional = true - -[dependencies.cudarc] -version = "0.19.0" -features = [ - "std", - "cublas", - "cublaslt", - "curand", - "driver", - "nvrtc", - "f16", - "f8", - "cuda-version-from-build-system", - "dynamic-linking", -] -optional = true -default-features = false - -[dependencies.float8] -version = "0.6.0" -features = [ - "num-traits", - "rand_distr", -] - -[dependencies.gemm] -version = "0.19.0" -features = ["wasm-simd128-enable"] - -[dependencies.half] -version = "2.5.0" -features = [ - "num-traits", - "use-intrinsics", - "rand_distr", -] - -[dependencies.intel-mkl-src] -version = "0.8.1" -features = ["mkl-static-lp64-iomp"] -optional = true - -[dependencies.libc] -version = "0.2.147" -optional = true - -[dependencies.libm] -version = "0.2.15" - -[dependencies.memmap2] -version = "0.9.3" -features = ["stable_deref_trait"] - -[dependencies.num-traits] -version = "0.2.15" - -[dependencies.num_cpus] -version = "1.15.0" - -[dependencies.objc2-foundation] -version = "0.3.1" -optional = true - -[dependencies.objc2-metal] -version = "0.3.1" -optional = true - -[dependencies.rand] -version = "0.9.0" - -[dependencies.rand_distr] -version = "0.5.1" - -[dependencies.rayon] -version = "1.7.0" - -[dependencies.safetensors] -version = "0.7.0" - -[dependencies.thiserror] -version = "2" - -[dependencies.yoke] -version = "0.8.1" -features = ["derive"] - -[dependencies.zip] -version = "9.0.1" -default-features = false - -[dev-dependencies.anyhow] -version = "1" -features = ["backtrace"] - -[dev-dependencies.clap] -version = "4.2.4" -features = ["derive"] - -[dev-dependencies.criterion] -version = "0.8" -default-features = false - -[target.'cfg(all(not(target_arch = "wasm32"), not(target_os = "ios")))'.dependencies.candle-ug] -version = "0.9.2" -optional = true diff --git a/v2/dependency-patches/candle-core-0.9.2/Cargo.toml.orig b/v2/dependency-patches/candle-core-0.9.2/Cargo.toml.orig deleted file mode 100644 index f02d15c318..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/Cargo.toml.orig +++ /dev/null @@ -1,70 +0,0 @@ -[package] -name = "candle-core" -version.workspace = true -edition.workspace = true -description.workspace = true -repository.workspace = true -keywords.workspace = true -categories.workspace = true -license.workspace = true -readme = "README.md" - -[dependencies] -accelerate-src = { workspace = true, optional = true } -byteorder = { workspace = true } -candle-kernels = { workspace = true, optional = true } -candle-metal-kernels = { workspace = true, optional = true } -objc2-metal = { workspace = true, optional = true } -objc2-foundation = { workspace = true, optional = true } -cudarc = { workspace = true, optional = true } -gemm = { workspace = true } -half = { workspace = true } -float8 = { workspace = true } -intel-mkl-src = { workspace = true, optional = true } -libc = { workspace = true, optional = true } -libm = { workspace = true } -memmap2 = { workspace = true } -num-traits = { workspace = true } -num_cpus = { workspace = true } -rand = { workspace = true } -rand_distr = { workspace = true } -rayon = { workspace = true } -safetensors = { workspace = true } -thiserror = { workspace = true } -yoke = { workspace = true } -zip = { workspace = true } - -[target.'cfg(all(not(target_arch = "wasm32"), not(target_os = "ios")))'.dependencies] -candle-ug = { workspace = true, optional = true } - -[dev-dependencies] -anyhow = { workspace = true } -clap = { workspace = true } -criterion = { workspace = true } - -[features] -default = [] -cuda = ["cudarc", "dep:candle-kernels", "candle-ug?/cuda", "float8/cuda"] -cudnn = ["cuda", "cudarc/cudnn"] -nccl = ["cuda", "cudarc/nccl"] -mkl = ["dep:libc", "dep:intel-mkl-src"] -accelerate = ["dep:libc", "dep:accelerate-src"] -metal = [ - "dep:objc2-metal", - "dep:objc2-foundation", - "dep:candle-metal-kernels", - "candle-ug?/metal", -] -ug = ["dep:candle-ug"] - -[[bench]] -name = "bench_main" -harness = false - -[[example]] -name = "metal_basics" -required-features = ["metal"] - -[[example]] -name = "cuda_basics" -required-features = ["cuda"] diff --git a/v2/dependency-patches/candle-core-0.9.2/LICENSE b/v2/dependency-patches/candle-core-0.9.2/LICENSE deleted file mode 100644 index 261eeb9e9f..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/v2/dependency-patches/candle-core-0.9.2/README.md b/v2/dependency-patches/candle-core-0.9.2/README.md deleted file mode 100644 index 246f5bae61..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/README.md +++ /dev/null @@ -1,2 +0,0 @@ -# candle -Minimalist ML framework for Rust diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/bench_main.rs b/v2/dependency-patches/candle-core-0.9.2/benches/bench_main.rs deleted file mode 100644 index ec02e4bddb..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/benches/bench_main.rs +++ /dev/null @@ -1,17 +0,0 @@ -mod benchmarks; - -use criterion::criterion_main; - -criterion_main!( - benchmarks::affine::benches, - benchmarks::binary::benches, - benchmarks::broadcast::benches, - benchmarks::copy::benches, - benchmarks::conv_transpose2d::benches, - benchmarks::matmul::benches, - benchmarks::qmatmul::benches, - benchmarks::random::benches, - benchmarks::reduce::benches, - benchmarks::unary::benches, - benchmarks::where_cond::benches, -); diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/affine.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/affine.rs deleted file mode 100644 index 1d6edd4c69..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/affine.rs +++ /dev/null @@ -1,46 +0,0 @@ -use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; -use candle_core::{DType, Device, Tensor}; -use criterion::{criterion_group, Criterion, Throughput}; -use std::hint::black_box; -use std::time::Instant; - -fn run(a: &Tensor) { - a.affine(12.34, 56.78).unwrap(); -} - -fn run_affine_benchmark(c: &mut Criterion, device: &Device, dtype: DType, name: &str) { - let b = 1; - let m = 1024; - let k = 1024; - - let tensor = Tensor::zeros((b, m, k), dtype, device).unwrap(); - - let flops = b * m * k * dtype.size_in_bytes(); - - let mut group = c.benchmark_group(device.bench_name(name)); - group.throughput(Throughput::Bytes(flops as u64)); - group.bench_function("iter", move |b| { - b.iter_custom(|iters| { - let start = Instant::now(); - for _i in 0..iters { - run(black_box(&tensor)); - } - device.sync().unwrap(); - start.elapsed() - }) - }); - group.finish(); -} - -fn criterion_benchmark(c: &mut Criterion) { - let handler = BenchDeviceHandler::new().unwrap(); - for device in handler.devices { - run_affine_benchmark(c, &device, DType::F32, "affine_f32"); - run_affine_benchmark(c, &device, DType::F16, "affine_f16"); - run_affine_benchmark(c, &device, DType::BF16, "affine_bf16"); - #[cfg(not(feature = "metal"))] - run_affine_benchmark(c, &device, DType::F8E4M3, "affine_fp8"); - } -} - -criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/binary.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/binary.rs deleted file mode 100644 index a4953c6332..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/binary.rs +++ /dev/null @@ -1,57 +0,0 @@ -use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; -use candle_core::{DType, Device, Tensor}; -use criterion::{criterion_group, Criterion, Throughput}; -use std::hint::black_box; -use std::time::Instant; - -fn run(lhs: &Tensor, rhs: &Tensor) -> Tensor { - lhs.mul(rhs).unwrap() -} - -fn run_unary_benchmark(c: &mut Criterion, device: &Device, dtype: DType, name: &str) { - let b = 1; - let m = 1024; - let k = 1024; - - let lhs = Tensor::arange(0.0f32, (b * m * k) as f32, device) - .unwrap() - .to_dtype(dtype) - .unwrap() - .reshape((b, m, k)) - .unwrap(); - - let rhs = Tensor::arange(0.0f32, (b * m * k) as f32, device) - .unwrap() - .to_dtype(dtype) - .unwrap() - .reshape((b, m, k)) - .unwrap(); - - let flops = 2 * b * m * k * dtype.size_in_bytes(); - - let mut group = c.benchmark_group(device.bench_name(name)); - group.throughput(Throughput::Bytes(flops as u64)); - group.bench_function("iter", move |b| { - b.iter_custom(|iters| { - let start = Instant::now(); - for _i in 0..iters { - run(black_box(&lhs), black_box(&rhs)); - } - device.sync().unwrap(); - start.elapsed() - }) - }); - group.finish(); -} - -fn criterion_benchmark(c: &mut Criterion) { - let handler = BenchDeviceHandler::new().unwrap(); - for device in handler.devices { - for dtype in [DType::F32, DType::BF16, DType::F16] { - let name = format!("binary_mul_{dtype:?}"); - run_unary_benchmark(c, &device, dtype, &name); - } - } -} - -criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/broadcast.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/broadcast.rs deleted file mode 100644 index 020ed5b259..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/broadcast.rs +++ /dev/null @@ -1,47 +0,0 @@ -use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; -use candle_core::{DType, Device, Tensor}; -use criterion::{criterion_group, Criterion, Throughput}; -use std::hint::black_box; -use std::time::Instant; - -fn run(w: &Tensor, bias: &Tensor) { - w.broadcast_add(bias).unwrap(); -} - -fn run_benchmark(c: &mut Criterion, device: &Device, dtype: DType, name: &str) { - // We simulate a candle-nn style conv2d + bias forward pass. - let batch_size = 1; - let ch = 1; - let m = 126; - let bias_size = 128; - - let x = Tensor::ones((batch_size, ch, m, m), dtype, device).unwrap(); - let bias = Tensor::ones((1, bias_size, 1, 1), dtype, device).unwrap(); - - let flops = batch_size * ch * m * bias_size * dtype.size_in_bytes(); - - let mut group = c.benchmark_group(device.bench_name(name)); - group.throughput(Throughput::Bytes(flops as u64)); - group.bench_function("iter", move |b| { - b.iter_custom(|iters| { - let start = Instant::now(); - for _i in 0..iters { - run(black_box(&x), black_box(&bias)); - } - device.sync().unwrap(); - start.elapsed() - }) - }); - group.finish(); -} - -fn criterion_benchmark(c: &mut Criterion) { - let handler = BenchDeviceHandler::new().unwrap(); - for device in handler.devices { - run_benchmark(c, &device, DType::F32, "broadcast_add_f32"); - run_benchmark(c, &device, DType::F16, "broadcast_add_f16"); - run_benchmark(c, &device, DType::BF16, "broadcast_add_bf16"); - } -} - -criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/conv_transpose2d.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/conv_transpose2d.rs deleted file mode 100644 index 7bd5c5bf06..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/conv_transpose2d.rs +++ /dev/null @@ -1,60 +0,0 @@ -use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; -use candle_core::{DType, Device, Tensor}; -use criterion::{criterion_group, Criterion, Throughput}; -use std::hint::black_box; -use std::time::Instant; - -fn run( - x: &Tensor, - k: &Tensor, - padding: usize, - output_padding: usize, - stride: usize, - dilation: usize, -) { - x.conv_transpose2d(k, padding, output_padding, stride, dilation) - .unwrap(); -} - -fn run_benchmark(c: &mut Criterion, device: &Device, dtype: DType, name: &str) { - let t = Tensor::arange(0.0f32, 10000.0, device) - .unwrap() - .reshape((1, 4, 50, 50)) - .unwrap() - .to_dtype(dtype) - .unwrap(); - - let kernel = Tensor::arange(0.0f32, 100.0, device) - .unwrap() - .reshape((4, 1, 5, 5)) - .unwrap() - .to_dtype(dtype) - .unwrap(); - - let flops = t.dims().iter().product::() * dtype.size_in_bytes(); - - let mut group = c.benchmark_group(device.bench_name(name)); - group.throughput(Throughput::Bytes(flops as u64)); - group.bench_function("iter", move |b| { - b.iter_custom(|iters| { - let start = Instant::now(); - for _i in 0..iters { - run(black_box(&t), black_box(&kernel), 1, 0, 1, 2); - } - device.sync().unwrap(); - start.elapsed() - }) - }); - group.finish(); -} - -fn criterion_benchmark(c: &mut Criterion) { - let handler = BenchDeviceHandler::new().unwrap(); - for device in handler.devices { - run_benchmark(c, &device, DType::F32, "conv_transpose2d_f32"); - run_benchmark(c, &device, DType::F16, "conv_transpose2d_f16"); - run_benchmark(c, &device, DType::BF16, "conv_transpose2d_bf16"); - } -} - -criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/copy.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/copy.rs deleted file mode 100644 index 00eff3dca6..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/copy.rs +++ /dev/null @@ -1,39 +0,0 @@ -use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; -use candle_core::{Device, Tensor, WithDType}; -use criterion::{criterion_group, Criterion, Throughput}; -use std::hint::black_box; -use std::time::Instant; - -fn run_copy_mask_benchmark(c: &mut Criterion, device: &Device, name: &str) { - let batch_size = 128; - let in_seq_len = 1; - let kv_seq_len = 1024; - - let attn_mask = vec![vec![vec![D::zero(); kv_seq_len]; in_seq_len]; batch_size]; - let size_in_bytes = batch_size * in_seq_len * kv_seq_len * D::DTYPE.size_in_bytes(); - - let mut group = c.benchmark_group(device.bench_name(name)); - group.throughput(Throughput::Bytes(size_in_bytes as u64)); - group.bench_function("iter", move |b| { - b.iter_custom(|iters| { - let attn_masks = vec![attn_mask.clone(); iters as usize]; - let start = Instant::now(); - for attn_mask in attn_masks.into_iter() { - let tensor = Tensor::new(black_box(attn_mask), device).unwrap(); - black_box(tensor); - } - device.sync().unwrap(); - start.elapsed() - }) - }); - group.finish(); -} - -fn criterion_benchmark(c: &mut Criterion) { - let handler = BenchDeviceHandler::new().unwrap(); - for device in handler.devices { - run_copy_mask_benchmark::(c, &device, "copy_mask"); - } -} - -criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/matmul.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/matmul.rs deleted file mode 100644 index 08732d2b32..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/matmul.rs +++ /dev/null @@ -1,70 +0,0 @@ -use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; -use candle_core::{DType, Device, Tensor}; -use criterion::{criterion_group, Criterion, Throughput}; -use std::hint::black_box; -use std::time::Instant; - -/// Matmul benchmark shapes covering common GEMM scenarios -const MATMUL_SHAPES: &[(&str, &[usize], &[usize])] = &[ - // Original GEMV test - ("gemv", &[1, 1, 2048], &[1, 2048, 2048]), - // 4D Attention scenarios (multi-head attention) - ("attn_4d_small", &[484, 6, 144, 32], &[484, 6, 32, 144]), - ("attn_4d_large", &[121, 24, 144, 32], &[121, 24, 32, 144]), - // Square matrix tests - ("square_512", &[512, 512], &[512, 512]), - ("square_1024", &[1024, 1024], &[1024, 1024]), - // 3D Batch matmul (attention patterns) - ("batch_1000", &[1000, 144, 32], &[1000, 32, 144]), - // 2D Linear layer scenarios (transformer FFN) - ("linear_large", &[17424, 768], &[768, 3072]), -]; - -fn run(a: &Tensor, b: &Tensor) { - a.broadcast_matmul(b).unwrap(); -} - -fn calculate_flops(shape_a: &[usize], shape_b: &[usize]) -> usize { - let batch: usize = shape_a - .iter() - .take(shape_a.len().saturating_sub(2)) - .product(); - let batch = if batch == 0 { 1 } else { batch }; - let m = shape_a[shape_a.len() - 2]; - let k = shape_a[shape_a.len() - 1]; - let n = shape_b[shape_b.len() - 1]; - 2 * batch * m * k * n -} - -fn run_bench(c: &mut Criterion, device: &Device, name: &str, shape_a: &[usize], shape_b: &[usize]) { - let dtype = DType::F32; - let lhs = Tensor::zeros(shape_a, dtype, device).unwrap(); - let rhs = Tensor::zeros(shape_b, dtype, device).unwrap(); - - let flops = calculate_flops(shape_a, shape_b); - - let mut group = c.benchmark_group(device.bench_name(format!("matmul_{name}"))); - group.throughput(Throughput::Bytes(flops as u64)); - group.bench_function("iter", move |b| { - b.iter_custom(|iters| { - let start = Instant::now(); - for _i in 0..iters { - run(black_box(&lhs), black_box(&rhs)); - } - device.sync().unwrap(); - start.elapsed() - }) - }); - group.finish(); -} - -fn criterion_benchmark(c: &mut Criterion) { - let handler = BenchDeviceHandler::new().unwrap(); - for device in handler.devices { - for (name, shape_a, shape_b) in MATMUL_SHAPES { - run_bench(c, &device, name, shape_a, shape_b); - } - } -} - -criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/mod.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/mod.rs deleted file mode 100644 index 9cc6767a4d..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/mod.rs +++ /dev/null @@ -1,77 +0,0 @@ -pub(crate) mod affine; -pub(crate) mod binary; -pub(crate) mod broadcast; -pub(crate) mod conv_transpose2d; -pub(crate) mod copy; -pub(crate) mod matmul; -pub(crate) mod qmatmul; -pub(crate) mod random; -pub(crate) mod reduce; -pub(crate) mod unary; -pub(crate) mod where_cond; - -use candle_core::{Device, Result}; - -pub(crate) trait BenchDevice { - fn sync(&self) -> Result<()>; - - fn bench_name>(&self, name: S) -> String; -} - -impl BenchDevice for Device { - fn sync(&self) -> Result<()> { - match self { - Device::Cpu => Ok(()), - Device::Cuda(device) => { - #[cfg(feature = "cuda")] - { - use candle_core::backend::BackendDevice; - return Ok(device.synchronize()?); - } - #[cfg(not(feature = "cuda"))] - panic!("Cuda device without cuda feature enabled: {device:?}") - } - Device::Metal(device) => { - #[cfg(feature = "metal")] - return device.wait_until_completed(); - #[cfg(not(feature = "metal"))] - panic!("Metal device without metal feature enabled: {device:?}") - } - } - } - - fn bench_name>(&self, name: S) -> String { - match self { - Device::Cpu => { - let cpu_type = if cfg!(feature = "accelerate") { - "accelerate" - } else if cfg!(feature = "mkl") { - "mkl" - } else { - "cpu" - }; - format!("{}_{}", cpu_type, name.into()) - } - Device::Cuda(_) => format!("cuda_{}", name.into()), - Device::Metal(_) => format!("metal_{}", name.into()), - } - } -} - -struct BenchDeviceHandler { - devices: Vec, -} - -impl BenchDeviceHandler { - pub fn new() -> Result { - let mut devices = Vec::new(); - if cfg!(feature = "metal") { - devices.push(Device::new_metal(0)?); - } else if cfg!(feature = "cuda") { - devices.push(Device::new_cuda(0)?); - } else { - devices.push(Device::Cpu); - } - Ok(Self { devices }) - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/qmatmul.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/qmatmul.rs deleted file mode 100644 index be1d2ad021..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/qmatmul.rs +++ /dev/null @@ -1,73 +0,0 @@ -use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; -use candle_core::{ - quantized::{self, GgmlDType, QMatMul}, - Device, Module, Tensor, -}; -use criterion::{criterion_group, Criterion, Throughput}; -use std::hint::black_box; -use std::time::Instant; - -fn run(matmul: &QMatMul, x: &Tensor) { - matmul.forward(x).unwrap(); -} - -fn run_bench(c: &mut Criterion, device: &Device, dtype: GgmlDType) { - let b = 1; - let m = 1; - let n = 1024; - let k = 1024; - - let lhs = (0..(m * k)) - .map(|v| v as f32 / (m * k) as f32) - .collect::>(); - let rhs = (0..(k * n)) - .map(|v| v as f32 / (n * k) as f32) - .collect::>(); - - let lhs = Tensor::from_slice(&lhs, (m, k), device).unwrap(); - let rhs = Tensor::from_slice(&rhs, (k, n), device).unwrap(); - - let qtensor = quantized::QTensor::quantize(&rhs.t().unwrap(), dtype).unwrap(); - let matmul = quantized::QMatMul::from_qtensor(qtensor).unwrap(); - - let flops = b * m * n * k; - - let mut group = c.benchmark_group(device.bench_name(format!("qmatmul_{dtype:?}"))); - group.sample_size(200); - group.throughput(Throughput::Bytes(flops as u64)); - group.bench_function("iter", move |b| { - b.iter_custom(|iters| { - let start = Instant::now(); - for _i in 0..iters { - run(black_box(&matmul), black_box(&lhs)); - } - device.sync().unwrap(); - start.elapsed() - }) - }); - group.finish(); -} - -fn criterion_benchmark(c: &mut Criterion) { - let handler = BenchDeviceHandler::new().unwrap(); - for device in handler.devices { - for dtype in [ - GgmlDType::F32, - GgmlDType::F16, - GgmlDType::Q4_0, - GgmlDType::Q4_1, - GgmlDType::Q5_0, - GgmlDType::Q5_1, - GgmlDType::Q8_0, - GgmlDType::Q2K, - GgmlDType::Q3K, - GgmlDType::Q4K, - GgmlDType::Q5K, - GgmlDType::Q6K, - ] { - run_bench(c, &device, dtype); - } - } -} - -criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/random.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/random.rs deleted file mode 100644 index 365e051c0e..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/random.rs +++ /dev/null @@ -1,64 +0,0 @@ -use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; -use candle_core::{DType, Device, Tensor}; -use criterion::{criterion_group, Criterion, Throughput}; -use std::hint::black_box; -use std::time::Instant; - -fn rand_uniform(a: &Tensor) { - a.rand_like(-1.0, 123.0).unwrap(); -} - -fn rand_normal(a: &Tensor) { - a.randn_like(100.0, 15.0).unwrap(); -} - -fn run_random_bench(c: &mut Criterion, device: &Device) { - let b = 1; - - let rows = 2048; - let cols = 2048; - - let dtype = DType::F32; - let tensor = Tensor::zeros((b, rows, cols), dtype, device).unwrap(); - - let flops = b * rows * cols * dtype.size_in_bytes(); - - let mut group = c.benchmark_group(device.bench_name("random_uniform")); - group.throughput(Throughput::Bytes(flops as u64)); - group.bench_function("iter", move |benches| { - benches.iter_custom(|iters| { - let start = Instant::now(); - for _i in 0..iters { - rand_uniform(black_box(&tensor)); - } - device.sync().unwrap(); - start.elapsed() - }) - }); - group.finish(); - - let tensor = Tensor::zeros((b, rows, cols), dtype, device).unwrap(); - - let mut group = c.benchmark_group(device.bench_name("random_normal")); - group.throughput(Throughput::Bytes(flops as u64)); - group.bench_function("iter", move |benches| { - benches.iter_custom(|iters| { - let start = Instant::now(); - for _i in 0..iters { - rand_normal(black_box(&tensor)); - } - device.sync().unwrap(); - start.elapsed() - }) - }); - group.finish(); -} - -fn criterion_benchmark(c: &mut Criterion) { - let handler = BenchDeviceHandler::new().unwrap(); - for device in handler.devices { - run_random_bench(c, &device); - } -} - -criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/reduce.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/reduce.rs deleted file mode 100644 index 6020633c00..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/reduce.rs +++ /dev/null @@ -1,159 +0,0 @@ -use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; -use candle_core::{DType, Device, Tensor}; -use criterion::{criterion_group, Criterion, Throughput}; -use half::{bf16, f16}; -use std::hint::black_box; -use std::time::Instant; - -fn run_sum(a: &Tensor) { - a.sum_keepdim(2).unwrap(); -} -fn run_arg_min(a: &Tensor) { - a.argmin_keepdim(2).unwrap(); -} - -fn criterion_benchmark(c: &mut Criterion) { - let handler = BenchDeviceHandler::new().unwrap(); - let (lo, up) = (-1000.0f32, 1000.0f32); - for device in handler.devices { - run_reduce(c, &device, (lo, up), false); - run_reduce(c, &device, (f16::from_f32(lo), f16::from_f32(up)), false); - run_reduce(c, &device, (bf16::from_f32(lo), bf16::from_f32(up)), false); - - run_arg_reduce(c, &device, (lo, up), false); - run_arg_reduce(c, &device, (f16::from_f32(lo), f16::from_f32(up)), false); - run_arg_reduce(c, &device, (bf16::from_f32(lo), bf16::from_f32(up)), false); - - run_reduce(c, &device, (lo, up), true); - run_reduce(c, &device, (f16::from_f32(lo), f16::from_f32(up)), true); - run_reduce(c, &device, (bf16::from_f32(lo), bf16::from_f32(up)), true); - - run_arg_reduce(c, &device, (lo, up), true); - run_arg_reduce(c, &device, (f16::from_f32(lo), f16::from_f32(up)), true); - run_arg_reduce(c, &device, (bf16::from_f32(lo), bf16::from_f32(up)), true); - } -} - -fn run_reduce( - c: &mut Criterion, - device: &Device, - (lo, up): (T, T), - strided: bool, -) { - let b = 1; - let m = 1024; - let k = 1024; - - let a = if strided { - Tensor::rand(lo, up, (b, m, k), device) - .unwrap() - .transpose(0, 2) - .unwrap() - } else { - Tensor::rand(lo, up, (b, m, k), device).unwrap() - }; - - let flops = b * m * k * T::DTYPE.size_in_bytes(); - - let name = match T::DTYPE { - DType::F32 => { - if strided { - "reduce_f32_strided" - } else { - "reduce_f32" - } - } - DType::F16 => { - if strided { - "reduce_f16_strided" - } else { - "reduce_f16" - } - } - DType::BF16 => { - if strided { - "reduce_bf16_strided" - } else { - "reduce_bf16" - } - } - _ => "unknown", - }; - - let mut group = c.benchmark_group(device.bench_name(name)); - group.throughput(Throughput::Bytes(flops as u64)); - group.bench_function("iter", move |b| { - b.iter_custom(|iters| { - let start = Instant::now(); - for _i in 0..iters { - run_sum(black_box(&a)); - } - device.sync().unwrap(); - start.elapsed() - }) - }); - group.finish(); -} - -fn run_arg_reduce( - c: &mut Criterion, - device: &Device, - (lo, up): (T, T), - strided: bool, -) { - let b = 1; - let m = 1024; - let k = 1024; - - let a = if strided { - Tensor::rand(lo, up, (b, m, k), device) - .unwrap() - .transpose(0, 2) - .unwrap() - } else { - Tensor::rand(lo, up, (b, m, k), device).unwrap() - }; - - let flops = b * m * k * T::DTYPE.size_in_bytes(); - - let name = match T::DTYPE { - DType::F32 => { - if strided { - "arg_reduce_f32_strided" - } else { - "arg_reduce_f32" - } - } - DType::F16 => { - if strided { - "arg_reduce_f16_strided" - } else { - "arg_reduce_f16" - } - } - DType::BF16 => { - if strided { - "arg_reduce_bf16_strided" - } else { - "arg_reduce_bf16" - } - } - _ => "unknown", - }; - - let mut group = c.benchmark_group(device.bench_name(name)); - group.throughput(Throughput::Bytes(flops as u64)); - group.bench_function("iter", move |b| { - b.iter_custom(|iters| { - let start = Instant::now(); - for _i in 0..iters { - run_arg_min(black_box(&a)); - } - device.sync().unwrap(); - start.elapsed() - }) - }); - group.finish(); -} - -criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/unary.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/unary.rs deleted file mode 100644 index 65723bb3fd..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/unary.rs +++ /dev/null @@ -1,98 +0,0 @@ -use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; -use candle_core::{DType, Device, Tensor}; -use criterion::{criterion_group, Criterion, Throughput}; -use std::hint::black_box; -use std::time::Instant; - -fn run_sqrt(a: &Tensor) { - a.sqrt().unwrap(); -} - -fn run_unary_benchmark(c: &mut Criterion, device: &Device, dtype: DType, name: &str) { - let b = 1; - let m = 1024; - let k = 1024; - - let tensor = Tensor::arange(0.0f32, (b * m * k) as f32, device) - .unwrap() - .to_dtype(dtype) - .unwrap() - .reshape((b, m, k)) - .unwrap(); - - let flops = b * m * k * dtype.size_in_bytes(); - - let mut group = c.benchmark_group(device.bench_name(name)); - group.throughput(Throughput::Bytes(flops as u64)); - group.bench_function("iter", move |b| { - b.iter_custom(|iters| { - let start = Instant::now(); - for _i in 0..iters { - run_sqrt(black_box(&tensor)); - } - device.sync().unwrap(); - start.elapsed() - }) - }); - group.finish(); -} - -fn run_cast(a: &Tensor, dtype: DType) { - a.to_dtype(dtype).unwrap(); -} - -fn run_cast_benchmark( - c: &mut Criterion, - device: &Device, - dtype: DType, - to_dtype: DType, - name: &str, -) { - let b = 1; - let m = 1024; - let k = 1024; - - let tensor = Tensor::arange(0.0f32, (b * m * k) as f32, device) - .unwrap() - .to_dtype(dtype) - .unwrap() - .reshape((b, m, k)) - .unwrap(); - - let flops = b * m * k * dtype.size_in_bytes(); - - let mut group = c.benchmark_group(device.bench_name(name)); - group.throughput(Throughput::Bytes(flops as u64)); - group.bench_function("iter", move |b| { - b.iter_custom(|iters| { - let start = Instant::now(); - for _i in 0..iters { - run_cast(black_box(&tensor), black_box(to_dtype)); - } - device.sync().unwrap(); - start.elapsed() - }) - }); - group.finish(); -} - -fn criterion_benchmark(c: &mut Criterion) { - let handler = BenchDeviceHandler::new().unwrap(); - for device in handler.devices { - for dtype in [DType::F32, DType::BF16, DType::F16] { - let to_dtype = if matches!(dtype, DType::F32) { - DType::F16 - } else { - DType::F32 - }; - let name = format!("cast_{}_{}", dtype.as_str(), to_dtype.as_str()); - run_cast_benchmark(c, &device, dtype, to_dtype, &name); - } - for dtype in [DType::F32, DType::BF16, DType::F16] { - let name = format!("sqrt_{dtype:?}"); - run_unary_benchmark(c, &device, dtype, &name); - } - } -} - -criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/where_cond.rs b/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/where_cond.rs deleted file mode 100644 index 348c1d8cba..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/benches/benchmarks/where_cond.rs +++ /dev/null @@ -1,65 +0,0 @@ -use crate::benchmarks::{BenchDevice, BenchDeviceHandler}; -use candle_core::{DType, Device, Tensor}; -use criterion::{criterion_group, Criterion, Throughput}; -use std::hint::black_box; -use std::time::Instant; - -fn run(a: &Tensor, b: &Tensor, c: &Tensor) { - a.where_cond(b, c).unwrap(); -} - -const fn create_cond_arr() -> [u8; N] { - let mut arr = [0u8; N]; - let mut i = 0; - while i < N { - arr[i] = (i % 2) as u8; - i += 1; - } - arr -} - -const B: usize = 1; -const M: usize = 1024; -const K: usize = 1024; -const SIZE: usize = B * M * K; - -static DATA: [u8; SIZE] = create_cond_arr::(); - -fn run_where_cond_benchmark(c: &mut Criterion, device: &Device, dtype: DType, name: &str) { - let tensor = Tensor::from_slice(DATA.as_slice(), (B, M, K), device).unwrap(); - let on_true = Tensor::ones((B, M, K), dtype, device).unwrap(); - let on_false = Tensor::zeros((B, M, K), dtype, device).unwrap(); - - let elements = B * M * K; - // E.g. 2 f32 tensors + 1 u8 tensor - let flops = (2 * elements * dtype.size_in_bytes()) + elements; - - let mut group = c.benchmark_group(device.bench_name(name)); - group.throughput(Throughput::Bytes(flops as u64)); - group.bench_function("iter", move |b| { - b.iter_custom(|iters| { - let start = Instant::now(); - for _i in 0..iters { - run( - black_box(&tensor), - black_box(&on_true), - black_box(&on_false), - ); - } - device.sync().unwrap(); - start.elapsed() - }) - }); - group.finish(); -} - -fn criterion_benchmark(c: &mut Criterion) { - let device = BenchDeviceHandler::new().unwrap(); - for d in device.devices { - run_where_cond_benchmark(c, &d, DType::F32, "where_cond_f32"); - run_where_cond_benchmark(c, &d, DType::BF16, "where_cond_bf16"); - run_where_cond_benchmark(c, &d, DType::F16, "where_cond_f16"); - } -} - -criterion_group!(benches, criterion_benchmark); diff --git a/v2/dependency-patches/candle-core-0.9.2/examples/basics.rs b/v2/dependency-patches/candle-core-0.9.2/examples/basics.rs deleted file mode 100644 index e991403441..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/examples/basics.rs +++ /dev/null @@ -1,20 +0,0 @@ -#[cfg(feature = "mkl")] -extern crate intel_mkl_src; - -#[cfg(feature = "accelerate")] -extern crate accelerate_src; - -use anyhow::Result; -use candle_core::{Device, Tensor}; - -fn main() -> Result<()> { - let a = Tensor::new(&[[0.0f32, 1.0, 2.0], [3.0, 4.0, 5.0]], &Device::Cpu)?; - let b = Tensor::new(&[[88.0f32], [99.0]], &Device::Cpu)?; - let new_a = a.slice_scatter(&b, 1, 2)?; - assert_eq!(a.to_vec2::()?, [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); - assert_eq!( - new_a.to_vec2::()?, - [[0.0, 1.0, 88.0], [3.0, 4.0, 99.0]] - ); - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.9.2/examples/cuda_basics.rs b/v2/dependency-patches/candle-core-0.9.2/examples/cuda_basics.rs deleted file mode 100644 index 4eadcdeb82..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/examples/cuda_basics.rs +++ /dev/null @@ -1,23 +0,0 @@ -#[cfg(feature = "accelerate")] -extern crate accelerate_src; - -#[cfg(feature = "mkl")] -extern crate intel_mkl_src; - -use anyhow::Result; -use candle_core::{Device, Tensor}; -// xs: [1024, 64, 1924], c Tensor[dims 128, 64, 8; f32, cuda:0] Conv1dConfig { padding: 0, stride: 4, dilation: 1, groups: 1 } -fn main() -> Result<()> { - let device = Device::new_cuda(0)?; - let x = Tensor::randn(0f32, 1.0, (1024, 64, 1924), &device)?; - let c = Tensor::randn(0f32, 1.0, (128, 64, 8), &device)?; - let _x1 = x.conv1d(&c, 0, 4, 1, 1)?; - drop(_x1); - for _ in 0..20 { - let start_time = std::time::Instant::now(); - let _x1 = x.conv1d(&c, 0, 4, 1, 1)?; - device.synchronize()?; - println!("conv1d: {:?}", start_time.elapsed()); - } - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.9.2/examples/cuda_sum_benchmark.rs b/v2/dependency-patches/candle-core-0.9.2/examples/cuda_sum_benchmark.rs deleted file mode 100644 index 5bd4b4eefe..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/examples/cuda_sum_benchmark.rs +++ /dev/null @@ -1,54 +0,0 @@ -#[cfg(feature = "mkl")] -extern crate intel_mkl_src; - -#[cfg(feature = "accelerate")] -extern crate accelerate_src; - -use std::str::FromStr; - -use anyhow::Result; -use candle_core::{Device, Tensor}; - -fn cos_sin(n: usize, device: &Device) -> Result { - let thetas: Vec<_> = (0..n).map(|i| i as f32 / n as f32).collect(); - let xs: Vec<_> = thetas.iter().map(|t| t.cos().abs()).collect(); - let ys: Vec<_> = thetas.iter().map(|t| t.sin().abs()).collect(); - let xs = Tensor::from_vec(xs, (n, 1), device)?; - let ys = Tensor::from_vec(ys, (1, n), device)?; - let ys = Tensor::cat(&[&ys, &ys, &ys, &ys, &ys, &ys], 1)?; - Ok(xs.matmul(&ys)?) -} - -fn main() -> Result<()> { - let device = Device::new_cuda(0)?; - let args = std::env::args().collect::>(); - let n = if args.len() < 2 { - 2000usize - } else { - usize::from_str(&args[1])? - }; - let xys_cpu = cos_sin(n, &Device::Cpu)?; - let xys = cos_sin(n, &device)?; - println!("{xys_cpu:?} {xys:?}"); - let sum_keepdim_cpu = xys_cpu.sum_keepdim(1)?; - println!("{sum_keepdim_cpu}"); - let sum_keepdim = xys.sum_keepdim(1)?; - println!("{sum_keepdim}"); - let start = std::time::Instant::now(); - let n_iters = 100; - let mut v = 0f32; - for _i in 0..n_iters { - let sum_keepdim = xys.sum_keepdim(1)?; - let sum_keepdim = sum_keepdim.sum_keepdim(0)?; - let sum_keepdim: f32 = sum_keepdim.reshape(&[])?.to_scalar()?; - v += sum_keepdim; - } - let elapsed = start.elapsed(); - if v > 0. { - println!( - "ran {n_iters} iterations, time per iter: {:?} ({v})", - elapsed.div_f64(n_iters as f64) - ); - } - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.9.2/examples/metal_basics.rs b/v2/dependency-patches/candle-core-0.9.2/examples/metal_basics.rs deleted file mode 100644 index 3f433d9c26..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/examples/metal_basics.rs +++ /dev/null @@ -1,28 +0,0 @@ -#[cfg(feature = "accelerate")] -extern crate accelerate_src; - -#[cfg(feature = "mkl")] -extern crate intel_mkl_src; - -use anyhow::Result; -use candle_core::{Device, Tensor}; - -fn main() -> Result<()> { - // This requires the code to be run with MTL_CAPTURE_ENABLED=1 - let device = Device::new_metal(0)?; - let metal_device = match &device { - Device::Metal(m) => m, - _ => anyhow::bail!("unexpected device"), - }; - metal_device.capture("/tmp/candle.gputrace")?; - // This first synchronize ensures that a new command buffer gets created after setting up the - // capture scope. - device.synchronize()?; - let x = Tensor::randn(0f32, 1.0, (128, 128), &device)?; - let x1 = x.add(&x)?; - println!("{x1:?}"); - // This second synchronize ensures that the command buffer gets committed before the end of the - // capture scope. - device.synchronize()?; - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/accelerate.rs b/v2/dependency-patches/candle-core-0.9.2/src/accelerate.rs deleted file mode 100644 index d371d3b33b..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/accelerate.rs +++ /dev/null @@ -1,476 +0,0 @@ -#![allow(dead_code)] -use libc::{c_char, c_double, c_float, c_int, c_long, c_ulong}; - -mod ffi { - use super::*; - extern "C" { - // It would be nice to be able to switch to the NEWLAPACK version of the function but this - // seems to trigger some link error. Available function names can be seen here: - // /Library/Developer/CommandLineTools/SDKs/MacOSX13.3.sdk/System/Library/Frameworks/Accelerate.framework/Versions/A/Accelerate.tbd - #[link_name = "sgemm_"] - pub fn sgemm_ffi( - transa: *const c_char, - transb: *const c_char, - m: *const c_int, - n: *const c_int, - k: *const c_int, - alpha: *const c_float, - a: *const c_float, - lda: *const c_int, - b: *const c_float, - ldb: *const c_int, - beta: *const c_float, - c: *mut c_float, - ldc: *const c_int, - ); - #[link_name = "dgemm_"] - pub fn dgemm_ffi( - transa: *const c_char, - transb: *const c_char, - m: *const c_int, - n: *const c_int, - k: *const c_int, - alpha: *const c_double, - a: *const c_double, - lda: *const c_int, - b: *const c_double, - ldb: *const c_int, - beta: *const c_double, - c: *mut c_double, - ldc: *const c_int, - ); - - pub fn vvexpf(dst: *mut c_float, src: *const c_float, len: *const c_int); - pub fn vvexp(dst: *mut c_double, src: *const c_double, len: *const c_int); - pub fn vvsqrtf(dst: *mut c_float, src: *const c_float, len: *const c_int); - pub fn vvsqrt(dst: *mut c_double, src: *const c_double, len: *const c_int); - pub fn vvsinf(dst: *mut c_float, src: *const c_float, len: *const c_int); - pub fn vvsin(dst: *mut c_double, src: *const c_double, len: *const c_int); - pub fn vvcosf(dst: *mut c_float, src: *const c_float, len: *const c_int); - pub fn vvcos(dst: *mut c_double, src: *const c_double, len: *const c_int); - pub fn vvlogf(dst: *mut c_float, src: *const c_float, len: *const c_int); - pub fn vvlog(dst: *mut c_double, src: *const c_double, len: *const c_int); - pub fn vvtanhf(dst: *mut c_float, src: *const c_float, len: *const c_int); - pub fn vvtanh(dst: *mut c_double, src: *const c_double, len: *const c_int); - - pub fn vDSP_vaddD( - _: *const c_double, - _: c_long, - _: *const c_double, - _: c_long, - _: *mut c_double, - _: c_long, - _: c_ulong, - ); - pub fn vDSP_vadd( - _: *const c_float, - _: c_long, - _: *const c_float, - _: c_long, - _: *mut c_float, - _: c_long, - _: c_ulong, - ); - pub fn vDSP_vsubD( - _: *const c_double, - _: c_long, - _: *const c_double, - _: c_long, - _: *mut c_double, - _: c_long, - _: c_ulong, - ); - pub fn vDSP_vsub( - _: *const c_float, - _: c_long, - _: *const c_float, - _: c_long, - _: *mut c_float, - _: c_long, - _: c_ulong, - ); - pub fn vDSP_vmulD( - _: *const c_double, - _: c_long, - _: *const c_double, - _: c_long, - _: *mut c_double, - _: c_long, - _: c_ulong, - ); - pub fn vDSP_vmul( - _: *const c_float, - _: c_long, - _: *const c_float, - _: c_long, - _: *mut c_float, - _: c_long, - _: c_ulong, - ); - pub fn vDSP_vdivD( - _: *const c_double, - _: c_long, - _: *const c_double, - _: c_long, - _: *mut c_double, - _: c_long, - _: c_ulong, - ); - pub fn vDSP_vdiv( - _: *const c_float, - _: c_long, - _: *const c_float, - _: c_long, - _: *mut c_float, - _: c_long, - _: c_ulong, - ); - pub fn vDSP_vminD( - _: *const c_double, - _: c_long, - _: *const c_double, - _: c_long, - _: *mut c_double, - _: c_long, - _: c_ulong, - ); - pub fn vDSP_vmin( - _: *const c_float, - _: c_long, - _: *const c_float, - _: c_long, - _: *mut c_float, - _: c_long, - _: c_ulong, - ); - pub fn vDSP_vmaxD( - _: *const c_double, - _: c_long, - _: *const c_double, - _: c_long, - _: *mut c_double, - _: c_long, - _: c_ulong, - ); - pub fn vDSP_vmax( - _: *const c_float, - _: c_long, - _: *const c_float, - _: c_long, - _: *mut c_float, - _: c_long, - _: c_ulong, - ); - } -} - -#[allow(clippy::too_many_arguments)] -#[inline] -pub unsafe fn sgemm( - transa: u8, - transb: u8, - m: i32, - n: i32, - k: i32, - alpha: f32, - a: &[f32], - lda: i32, - b: &[f32], - ldb: i32, - beta: f32, - c: &mut [f32], - ldc: i32, -) { - ffi::sgemm_ffi( - &(transa as c_char), - &(transb as c_char), - &m, - &n, - &k, - &alpha, - a.as_ptr(), - &lda, - b.as_ptr(), - &ldb, - &beta, - c.as_mut_ptr(), - &ldc, - ) -} - -#[allow(clippy::too_many_arguments)] -#[inline] -pub unsafe fn dgemm( - transa: u8, - transb: u8, - m: i32, - n: i32, - k: i32, - alpha: f64, - a: &[f64], - lda: i32, - b: &[f64], - ldb: i32, - beta: f64, - c: &mut [f64], - ldc: i32, -) { - ffi::dgemm_ffi( - &(transa as c_char), - &(transb as c_char), - &m, - &n, - &k, - &alpha, - a.as_ptr(), - &lda, - b.as_ptr(), - &ldb, - &beta, - c.as_mut_ptr(), - &ldc, - ) -} - -#[inline] -pub fn vs_exp(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvexpf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} - -#[inline] -pub fn vd_exp(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvexp(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} - -#[inline] -pub fn vs_sqrt(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvsqrtf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} - -#[inline] -pub fn vd_sqrt(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvsqrt(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} - -#[inline] -pub fn vs_sin(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvsinf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} - -#[inline] -pub fn vd_sin(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvsin(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} -#[inline] -pub fn vs_cos(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvcosf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} - -#[inline] -pub fn vd_cos(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvcos(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} -#[inline] -pub fn vs_tanh(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvtanhf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} - -#[inline] -pub fn vd_tanh(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvtanh(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} - -#[inline] -pub fn vs_ln(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvlogf(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} - -#[inline] -pub fn vd_ln(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vvlog(y.as_mut_ptr(), a.as_ptr(), &(a_len as i32)) } -} - -#[inline] -pub fn vs_sqr(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - y.iter_mut().zip(a.iter()).for_each(|(y, a)| *y = *a * *a) -} - -#[inline] -pub fn vd_sqr(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - y.iter_mut().zip(a.iter()).for_each(|(y, a)| *y = *a * *a) -} - -#[inline] -pub fn vs_tanh_inplace(y: &mut [f32]) { - unsafe { ffi::vvtanhf(y.as_mut_ptr(), y.as_ptr(), &(y.len() as i32)) } -} - -#[inline] -pub fn vd_tanh_inplace(y: &mut [f64]) { - unsafe { ffi::vvtanh(y.as_mut_ptr(), y.as_ptr(), &(y.len() as i32)) } -} - -#[inline] -pub fn vs_exp_inplace(y: &mut [f32]) { - unsafe { ffi::vvexpf(y.as_mut_ptr(), y.as_ptr(), &(y.len() as i32)) } -} - -#[inline] -pub fn vd_exp_inplace(y: &mut [f64]) { - unsafe { ffi::vvexp(y.as_mut_ptr(), y.as_ptr(), &(y.len() as i32)) } -} - -#[inline] -pub fn vs_gelu(vs: &[f32], ys: &mut [f32]) { - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = (2.0f32 / std::f32::consts::PI).sqrt() * v * (1.0 + 0.044715 * v * v) - } - vs_tanh_inplace(ys); - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = 0.5 * v * (1.0 + *y) - } -} - -#[inline] -pub fn vd_gelu(vs: &[f64], ys: &mut [f64]) { - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = (2.0f64 / std::f64::consts::PI).sqrt() * v * (1.0 + 0.044715 * v * v) - } - vd_tanh_inplace(ys); - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = 0.5 * v * (1.0 + *y) - } -} - -#[inline] -pub fn vs_silu(vs: &[f32], ys: &mut [f32]) { - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = -v - } - vs_exp_inplace(ys); - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = v / (1.0 + *y) - } -} - -#[inline] -pub fn vd_silu(vs: &[f64], ys: &mut [f64]) { - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = -v - } - vd_exp_inplace(ys); - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = v / (1.0 + *y) - } -} - -macro_rules! binary_op { - ($fn_name:ident, $ty:ty, $accelerate_name:ident) => { - #[inline] - pub fn $fn_name(a: &[$ty], b: &[$ty], y: &mut [$ty]) { - let a_len = a.len(); - let b_len = b.len(); - let y_len = y.len(); - if a_len != y_len || b_len != y_len { - panic!( - "{} a,b,y len mismatch {a_len} {b_len} {y_len}", - stringify!($fn_name) - ); - } - unsafe { - // Weird quirk of accelerate, the rhs comes before the lhs. - ffi::$accelerate_name( - b.as_ptr(), - 1, - a.as_ptr(), - 1, - y.as_mut_ptr(), - 1, - a_len as u64, - ) - } - } - }; -} -binary_op!(vs_add, f32, vDSP_vadd); -binary_op!(vd_add, f64, vDSP_vaddD); -binary_op!(vs_sub, f32, vDSP_vsub); -binary_op!(vd_sub, f64, vDSP_vsubD); -binary_op!(vs_mul, f32, vDSP_vmul); -binary_op!(vd_mul, f64, vDSP_vmulD); -binary_op!(vs_div, f32, vDSP_vdiv); -binary_op!(vd_div, f64, vDSP_vdivD); -binary_op!(vs_max, f32, vDSP_vmax); -binary_op!(vd_max, f64, vDSP_vmaxD); -binary_op!(vs_min, f32, vDSP_vmin); -binary_op!(vd_min, f64, vDSP_vminD); diff --git a/v2/dependency-patches/candle-core-0.9.2/src/backend.rs b/v2/dependency-patches/candle-core-0.9.2/src/backend.rs deleted file mode 100644 index d8ab2b5629..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/backend.rs +++ /dev/null @@ -1,174 +0,0 @@ -//! Traits to Define Backend Behavior -//! -use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; -use crate::{CpuStorage, DType, Layout, Result, Shape}; - -pub trait BackendStorage: Sized { - type Device: BackendDevice; - - fn try_clone(&self, _: &Layout) -> Result; - - fn dtype(&self) -> DType; - - fn device(&self) -> &Self::Device; - - // Maybe this should return a Cow instead so that no copy is done on the cpu case. - fn to_cpu_storage(&self) -> Result; - - fn affine(&self, _: &Layout, _: f64, _: f64) -> Result; - - fn powf(&self, _: &Layout, _: f64) -> Result; - - fn elu(&self, _: &Layout, _: f64) -> Result; - - fn reduce_op(&self, _: ReduceOp, _: &Layout, _: &[usize]) -> Result; - - fn cmp(&self, _: CmpOp, _: &Self, _: &Layout, _: &Layout) -> Result; - - fn to_dtype(&self, _: &Layout, _: DType) -> Result; - - fn unary_impl(&self, _: &Layout) -> Result; - - fn binary_impl(&self, _: &Self, _: &Layout, _: &Layout) -> Result; - - fn where_cond(&self, _: &Layout, _: &Self, _: &Layout, _: &Self, _: &Layout) -> Result; - - fn conv1d( - &self, - _l: &Layout, - _kernel: &Self, - _kernel_l: &Layout, - _params: &crate::conv::ParamsConv1D, - ) -> Result; - - fn conv_transpose1d( - &self, - _l: &Layout, - _kernel: &Self, - _kernel_l: &Layout, - _params: &crate::conv::ParamsConvTranspose1D, - ) -> Result; - - fn conv2d( - &self, - _l: &Layout, - _kernel: &Self, - _kernel_l: &Layout, - _params: &crate::conv::ParamsConv2D, - ) -> Result; - - fn conv_transpose2d( - &self, - _l: &Layout, - _kernel: &Self, - _kernel_l: &Layout, - _params: &crate::conv::ParamsConvTranspose2D, - ) -> Result; - - fn avg_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result; - fn max_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result; - fn upsample_nearest1d(&self, _: &Layout, _: usize) -> Result; - fn upsample_nearest2d(&self, _: &Layout, _: usize, _: usize) -> Result; - fn upsample_bilinear2d( - &self, - _: &Layout, - _: usize, - _: usize, - _: bool, - _: Option, - _: Option, - ) -> Result; - - fn gather(&self, _: &Layout, _: &Self, _: &Layout, _: usize) -> Result; - - fn scatter_set( - &mut self, - _: &Layout, - _: &Self, - _: &Layout, - _: &Self, - _: &Layout, - _: usize, - ) -> Result<()>; - - fn scatter_add_set( - &mut self, - _: &Layout, - _: &Self, - _: &Layout, - _: &Self, - _: &Layout, - _: usize, - ) -> Result<()>; - - fn index_select(&self, _: &Self, _: &Layout, _: &Layout, _: usize) -> Result; - fn index_add( - &self, - _: &Layout, - _: &Self, - _: &Layout, - _: &Self, - _: &Layout, - _: usize, - ) -> Result; - - fn matmul( - &self, - _: &Self, - _: (usize, usize, usize, usize), - _: &Layout, - _: &Layout, - ) -> Result; - - fn copy_strided_src(&self, _: &mut Self, _: usize, _: &Layout) -> Result<()>; - - #[allow(clippy::too_many_arguments)] - // Similar to cudaMemcpy2D, though values are in elements and not in bytes. - fn copy2d( - &self, - _: &mut Self, - _d1: usize, - _d2: usize, - _src_stride1: usize, - _dst_stride1: usize, - _src_offset: usize, - _dst_offset: usize, - ) -> Result<()>; - - fn const_set(&mut self, _: crate::scalar::Scalar, _: &Layout) -> Result<()>; -} - -pub trait BackendDevice: Sized + std::fmt::Debug + Clone { - type Storage: BackendStorage; - - // TODO: Make the usize generic and part of a generic DeviceLocation. - fn new(_: usize) -> Result; - - fn location(&self) -> crate::DeviceLocation; - - fn same_device(&self, _: &Self) -> bool; - - fn zeros_impl(&self, _shape: &Shape, _dtype: DType) -> Result; - - /// # Safety - /// This function is unsafe as it doesn't initialize the underlying data store. - /// The caller should ensure that the data is properly initialized as early as possible - /// after this call. - unsafe fn alloc_uninit(&self, _shape: &Shape, _dtype: DType) -> Result; - - fn storage_from_slice(&self, _: &[T]) -> Result; - - fn storage_from_cpu_storage(&self, _: &CpuStorage) -> Result; - - fn storage_from_cpu_storage_owned(&self, _: CpuStorage) -> Result; - - fn rand_uniform(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result; - - fn rand_normal(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result; - - fn set_seed(&self, _: u64) -> Result<()>; - fn get_current_seed(&self) -> Result; - - /// Synchronize should block until all the operations on the device are completed. - fn synchronize(&self) -> Result<()>; -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/backprop.rs b/v2/dependency-patches/candle-core-0.9.2/src/backprop.rs deleted file mode 100644 index d2310cbe28..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/backprop.rs +++ /dev/null @@ -1,784 +0,0 @@ -//! Methods for backpropagation of gradients. -use crate::op::{BinaryOp, Op, ReduceOp, UnaryOp}; -use crate::{Error, Result, Tensor, TensorId}; -use std::collections::HashMap; - -// arg has been reduced to node via reduce_dims, expand it back to arg. -// This has to handle keepdims. -fn broadcast_back(arg: &Tensor, node: &Tensor, reduced_dims: &[usize]) -> Result { - if arg.rank() == node.rank() { - // keepdim = true - node.broadcast_as(arg.shape()) - } else { - // keepdim = false - // first expand the reduced dims. - node.reshape(reduced_dims)?.broadcast_as(arg.shape()) - } -} - -thread_local! { - static CANDLE_GRAD_DO_NOT_DETACH: bool = { - match std::env::var("CANDLE_GRAD_DO_NOT_DETACH") { - Ok(s) => { - !s.is_empty() && s != "0" - }, - Err(_) => false, - } - } -} - -impl Tensor { - /// Return all the nodes that lead to this value in a topologically sorted vec, the first - /// elements having dependencies on the latter ones, e.g. the first element if any is the - /// argument. - /// This assumes that the op graph is a DAG. - pub fn sorted_nodes(&self) -> Vec<&Tensor> { - // The vec of sorted nodes is passed as an owned value rather than a mutable reference - // to get around some lifetime limitations. - fn walk<'a>( - node: &'a Tensor, - nodes: Vec<&'a Tensor>, - already_seen: &mut HashMap, - ) -> (bool, Vec<&'a Tensor>) { - if let Some(&tg) = already_seen.get(&node.id()) { - return (tg, nodes); - } - let mut track_grad = false; - let mut nodes = if node.is_variable() { - // Do not call recursively on the "leaf" nodes. - track_grad = true; - nodes - } else if node.dtype().is_int() { - nodes - } else if let Some(op) = node.op() { - match op { - Op::IndexAdd(t1, t2, t3, _) - | Op::Scatter(t1, t2, t3, _) - | Op::ScatterAdd(t1, t2, t3, _) - | Op::CustomOp3(t1, t2, t3, _) - | Op::WhereCond(t1, t2, t3) => { - let (tg, nodes) = walk(t1, nodes, already_seen); - track_grad |= tg; - let (tg, nodes) = walk(t2, nodes, already_seen); - track_grad |= tg; - let (tg, nodes) = walk(t3, nodes, already_seen); - track_grad |= tg; - nodes - } - Op::Conv1D { - arg: lhs, - kernel: rhs, - .. - } - | Op::ConvTranspose1D { - arg: lhs, - kernel: rhs, - .. - } - | Op::Conv2D { - arg: lhs, - kernel: rhs, - .. - } - | Op::ConvTranspose2D { - arg: lhs, - kernel: rhs, - .. - } - | Op::CustomOp2(lhs, rhs, _) - | Op::Binary(lhs, rhs, _) - | Op::Gather(lhs, rhs, _) - | Op::IndexSelect(lhs, rhs, _) - | Op::Matmul(lhs, rhs) - | Op::SliceScatter0(lhs, rhs, _) => { - let (tg, nodes) = walk(lhs, nodes, already_seen); - track_grad |= tg; - let (tg, nodes) = walk(rhs, nodes, already_seen); - track_grad |= tg; - nodes - } - Op::Cat(args, _) => args.iter().fold(nodes, |nodes, arg| { - let (tg, nodes) = walk(arg, nodes, already_seen); - track_grad |= tg; - nodes - }), - Op::Affine { arg, mul, .. } => { - if *mul == 0. { - nodes - } else { - let (tg, nodes) = walk(arg, nodes, already_seen); - track_grad |= tg; - nodes - } - } - Op::Unary(_node, UnaryOp::Ceil) - | Op::Unary(_node, UnaryOp::Floor) - | Op::Unary(_node, UnaryOp::Round) - | Op::Unary(_node, UnaryOp::Sign) => nodes, - Op::Reshape(node) - | Op::UpsampleNearest1D { arg: node, .. } - | Op::UpsampleNearest2D { arg: node, .. } - | Op::UpsampleBilinear2D { arg: node, .. } - | Op::AvgPool2D { arg: node, .. } - | Op::MaxPool2D { arg: node, .. } - | Op::Copy(node) - | Op::Broadcast(node) - | Op::Cmp(node, _) - | Op::Reduce(node, ReduceOp::Min | ReduceOp::Sum | ReduceOp::Max, _) - | Op::ToDevice(node) - | Op::Transpose(node, _, _) - | Op::Permute(node, _) - | Op::Narrow(node, _, _, _) - | Op::Unary(node, _) - | Op::Elu(node, _) - | Op::Powf(node, _) - | Op::CustomOp1(node, _) => { - let (tg, nodes) = walk(node, nodes, already_seen); - track_grad |= tg; - nodes - } - Op::ToDType(node) => { - if node.dtype().is_float() { - let (tg, nodes) = walk(node, nodes, already_seen); - track_grad |= tg; - nodes - } else { - nodes - } - } - Op::Reduce(_, ReduceOp::ArgMin | ReduceOp::ArgMax, _) => nodes, - } - } else { - nodes - }; - already_seen.insert(node.id(), track_grad); - if track_grad { - nodes.push(node); - } - (track_grad, nodes) - } - let (_tg, mut nodes) = walk(self, vec![], &mut HashMap::new()); - nodes.reverse(); - nodes - } - - pub fn backward(&self) -> Result { - let sorted_nodes = self.sorted_nodes(); - let mut grads = GradStore::new(); - grads.insert(self, self.ones_like()?.contiguous()?); - for node in sorted_nodes.iter() { - if node.is_variable() { - continue; - } - let grad = grads - .remove(node) - .expect("candle internal error - grad not populated"); - // https://github.com/huggingface/candle/issues/1241 - // Ideally, we would make these operations in place where possible to ensure that we - // do not have to allocate too often. Here we just call `.detach` to avoid computing - // the backprop graph of the backprop itself. This would be an issue for second order - // derivatives but these are out of scope at the moment. - let do_not_detach = CANDLE_GRAD_DO_NOT_DETACH.with(|b| *b); - let grad = if do_not_detach { grad } else { grad.detach() }; - if let Some(op) = node.op() { - match op { - Op::Binary(lhs, rhs, BinaryOp::Add) => { - let lhs_sum_grad = grads.or_insert(lhs)?; - *lhs_sum_grad = lhs_sum_grad.add(&grad)?; - let rhs_sum_grad = grads.or_insert(rhs)?; - *rhs_sum_grad = rhs_sum_grad.add(&grad)?; - } - Op::Binary(lhs, rhs, BinaryOp::Sub) => { - let lhs_sum_grad = grads.or_insert(lhs)?; - *lhs_sum_grad = lhs_sum_grad.add(&grad)?; - let rhs_sum_grad = grads.or_insert(rhs)?; - *rhs_sum_grad = rhs_sum_grad.sub(&grad)?; - } - Op::Binary(lhs, rhs, BinaryOp::Mul) => { - let lhs_grad = grad.mul(rhs)?; - let lhs_sum_grad = grads.or_insert(lhs)?; - *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?; - let rhs_grad = grad.mul(lhs)?; - let rhs_sum_grad = grads.or_insert(rhs)?; - *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?; - } - Op::Binary(lhs, rhs, BinaryOp::Div) => { - let lhs_grad = grad.div(rhs)?; - let lhs_sum_grad = grads.or_insert(lhs)?; - *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?; - let rhs_grad = grad.mul(lhs)?.div(&rhs.sqr()?)?; - let rhs_sum_grad = grads.or_insert(rhs)?; - *rhs_sum_grad = rhs_sum_grad.sub(&rhs_grad)?; - } - Op::Binary(lhs, rhs, BinaryOp::Minimum) - | Op::Binary(lhs, rhs, BinaryOp::Maximum) => { - let mask_lhs = node.eq(lhs)?.to_dtype(grad.dtype())?; - let mask_rhs = node.eq(rhs)?.to_dtype(grad.dtype())?; - - // If both masks are 1 one the same point, we want to scale the - // gradient by 0.5 rather than 1. - let lhs_grad = mask_lhs.mul(&grad)?.div(&(&mask_rhs + 1.)?)?; - let lhs_sum_grad = grads.or_insert(lhs)?; - *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?; - - let rhs_grad = mask_rhs.mul(&grad)?.div(&(&mask_lhs + 1.)?)?; - let rhs_sum_grad = grads.or_insert(rhs)?; - *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?; - } - Op::WhereCond(pred, t, f) => { - let zeros = grad.zeros_like()?; - let t_sum_grad = grads.or_insert(t)?; - let t_grad = pred.where_cond(&grad, &zeros)?; - *t_sum_grad = t_sum_grad.add(&t_grad)?; - let f_sum_grad = grads.or_insert(f)?; - let f_grad = pred.where_cond(&zeros, &grad)?; - *f_sum_grad = f_sum_grad.add(&f_grad)?; - } - Op::Conv1D { - arg, - kernel, - padding, - stride, - dilation, - } => { - // The output height for conv_transpose1d is: - // (l_in - 1) * stride - 2 * padding + dilation * (k_size - 1) + out_padding + 1 - let grad_l_in = grad.dim(2)?; - let k_size = kernel.dim(2)?; - let out_size = - (grad_l_in - 1) * stride + dilation * (k_size - 1) + 1 - 2 * padding; - let out_padding = arg.dim(2)? - out_size; - let grad_arg = grad.conv_transpose1d( - kernel, - *padding, - out_padding, - *stride, - *dilation, - /* groups */ 1, - )?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&grad_arg)?; - - let grad_kernel = arg - .transpose(0, 1)? - .conv1d(&grad.transpose(0, 1)?, *padding, *dilation, *stride, 1)? - .transpose(0, 1)?; - let sum_grad = grads.or_insert(kernel)?; - let (_, _, k0) = kernel.dims3()?; - let (_, _, g_k0) = grad_kernel.dims3()?; - let grad_kernel = if g_k0 != k0 { - grad_kernel.narrow(2, 0, k0)? - } else { - grad_kernel - }; - *sum_grad = sum_grad.add(&grad_kernel)?; - } - Op::Conv2D { - arg, - kernel, - padding, - stride, - dilation, - } => { - // The output height for conv_transpose2d is: - // (i_h - 1) * stride - 2 * padding + dilation * (k_h - 1) + out_padding + 1 - let grad_h = grad.dim(2)?; - let k_h = kernel.dim(2)?; - let out_size = - (grad_h - 1) * stride + dilation * (k_h - 1) + 1 - 2 * padding; - let out_padding = arg.dim(2)? - out_size; - let grad_arg = grad.conv_transpose2d( - kernel, - *padding, - out_padding, - *stride, - *dilation, - )?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&grad_arg)?; - - let grad_kernel = arg - .transpose(0, 1)? - .conv2d(&grad.transpose(0, 1)?, *padding, *dilation, *stride, 1)? - .transpose(0, 1)?; - let sum_grad = grads.or_insert(kernel)?; - let (_, _, k0, k1) = kernel.dims4()?; - let (_, _, g_k0, g_k1) = grad_kernel.dims4()?; - let grad_kernel = if g_k0 != k0 || g_k1 != k1 { - grad_kernel.narrow(2, 0, k0)?.narrow(3, 0, k1)? - } else { - grad_kernel - }; - *sum_grad = sum_grad.add(&grad_kernel)?; - } - Op::ConvTranspose1D { .. } => Err(Error::BackwardNotSupported { - op: "conv-transpose1d", - })?, - Op::ConvTranspose2D { - arg, - kernel, - padding, - stride, - dilation, - output_padding: _output_padding, - } => { - let grad_arg = grad.conv2d(kernel, *padding, *stride, *dilation, 1)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&grad_arg)?; - - let grad_kernel = grad - .transpose(0, 1)? - .conv2d(&arg.transpose(0, 1)?, *padding, *dilation, *stride, 1)? - .transpose(0, 1)?; - let sum_grad = grads.or_insert(kernel)?; - let (_, _, k0, k1) = kernel.dims4()?; - let (_, _, g_k0, g_k1) = grad_kernel.dims4()?; - let grad_kernel = if g_k0 != k0 || g_k1 != k1 { - grad_kernel.narrow(2, 0, k0)?.narrow(3, 0, k1)? - } else { - grad_kernel - }; - *sum_grad = sum_grad.add(&grad_kernel)?; - } - Op::AvgPool2D { - arg, - kernel_size, - stride, - } => { - if kernel_size != stride { - crate::bail!("backward not supported for avgpool2d if ksize {kernel_size:?} != stride {stride:?}") - } - let (_n, _c, h, w) = arg.dims4()?; - let grad_arg = grad.upsample_nearest2d(h, w)?; - let grad_arg = - (grad_arg * (1f64 / (kernel_size.0 * kernel_size.1) as f64))?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&grad_arg)?; - } - Op::MaxPool2D { - arg, - kernel_size, - stride, - } => { - if kernel_size != stride { - crate::bail!("backward not supported for maxpool2d if ksize {kernel_size:?} != stride {stride:?}") - } - let (_n, _c, h, w) = arg.dims4()?; - // For computing the max-pool gradient, we compute a mask where a 1 means - // that the element is the maximum, then we apply this mask to the - // upsampled gradient (taking into account that multiple max may exist so - // we scale the gradient for this case). - let node_upsampled = node.upsample_nearest2d(h, w)?; - let mask = arg.eq(&node_upsampled)?.to_dtype(arg.dtype())?; - let avg = mask.avg_pool2d_with_stride(*kernel_size, *stride)?; - let grad_arg = ((grad * avg)?.upsample_nearest2d(h, w)? * mask)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&grad_arg)?; - } - Op::UpsampleNearest1D { arg, target_size } => { - let (_n, c, size) = arg.dims3()?; - if target_size % size != 0 { - crate::bail!("backward not supported for non integer upscaling factors") - } - let scale = target_size / size; - - let kernel = Tensor::ones((c, 1, scale), arg.dtype(), arg.device())?; - let conv_sum = grad.conv1d(&kernel, 0, scale, 1, c)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = conv_sum; - } - Op::UpsampleNearest2D { - arg, - target_h, - target_w, - } => { - let (_n, c, h, w) = arg.dims4()?; - if target_h % h != 0 || target_w % w != 0 { - crate::bail!("backward not supported for non integer upscaling factors") - } - let scale_h = target_h / h; - let scale_w = target_w / w; - - if scale_h != scale_w { - crate::bail!("backward not supported for non uniform upscaling factors") - }; - let kernel = - Tensor::ones((c, 1, scale_h, scale_w), arg.dtype(), arg.device())?; - let conv_sum = grad.conv2d(&kernel, 0, scale_h, 1, c)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = conv_sum; - } - Op::UpsampleBilinear2D { .. } => { - crate::bail!("backward not supported for upsample_bilinear2d") - } - Op::SliceScatter0(lhs, rhs, start_rhs) => { - let rhs_sum_grad = grads.or_insert(rhs)?; - let rhs_grad = grad.narrow(0, *start_rhs, rhs.dim(0)?)?; - *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?; - - let lhs_sum_grad = grads.or_insert(lhs)?; - let lhs_grad = grad.slice_scatter0(&rhs.zeros_like()?, *start_rhs)?; - *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)? - } - Op::Gather(arg, indexes, dim) => { - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.scatter_add(indexes, &grad, *dim)?; - } - Op::Scatter(init, indexes, src, dim) => { - let init_sum_grad = grads.or_insert(init)?; - *init_sum_grad = init_sum_grad.add(&grad)?; - - let src_grad = grad.gather(indexes, *dim)?; - let src_sum_grad = grads.or_insert(src)?; - *src_sum_grad = src_sum_grad.add(&src_grad)?; - } - Op::ScatterAdd(init, indexes, src, dim) => { - let init_sum_grad = grads.or_insert(init)?; - let mask = init.ones_like()?; - let mask = mask.scatter(indexes, &mask.zeros_like()?, *dim)?; - *init_sum_grad = init_sum_grad.add(&grad.mul(&mask)?)?; - - let src_grad = grad.gather(indexes, *dim)?; - let src_sum_grad = grads.or_insert(src)?; - *src_sum_grad = src_sum_grad.add(&src_grad)?; - } - Op::IndexAdd(init, indexes, src, dim) => { - let init_sum_grad = grads.or_insert(init)?; - *init_sum_grad = init_sum_grad.add(&grad)?; - - let src_grad = grad.index_select(indexes, *dim)?; - let src_sum_grad = grads.or_insert(src)?; - *src_sum_grad = src_sum_grad.add(&src_grad)?; - } - Op::IndexSelect(arg, indexes, dim) => { - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.index_add(indexes, &grad, *dim)?; - } - Op::Matmul(lhs, rhs) => { - // Skipping checks, the op went ok, we can skip - // the matmul size checks for now. - - let lhs_grad = grad.matmul(&rhs.t()?)?; - let lhs_sum_grad = grads.or_insert(lhs)?; - *lhs_sum_grad = lhs_sum_grad.add(&lhs_grad)?; - - let rhs_grad = lhs.t()?.matmul(&grad)?; - let rhs_sum_grad = grads.or_insert(rhs)?; - *rhs_sum_grad = rhs_sum_grad.add(&rhs_grad)?; - } - Op::Cat(args, dim) => { - let mut start_idx = 0; - for arg in args { - let len = arg.dims()[*dim]; - let arg_grad = grad.narrow(*dim, start_idx, len)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&arg_grad)?; - start_idx += len; - } - } - Op::Broadcast(arg) => { - let arg_dims = arg.dims(); - let node_dims = node.dims(); - // The number of dims that have been inserted on the left. - let left_dims = node_dims.len() - arg_dims.len(); - let mut sum_dims: Vec = (0..left_dims).collect(); - for (dim, (node_dim, arg_dim)) in node_dims[left_dims..] - .iter() - .zip(arg_dims.iter()) - .enumerate() - { - if node_dim != arg_dim { - sum_dims.push(dim + left_dims) - } - } - - let mut arg_grad = grad.sum_keepdim(sum_dims.as_slice())?; - for _i in 0..left_dims { - arg_grad = arg_grad.squeeze(0)? - } - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&arg_grad.broadcast_as(sum_grad.dims())?)?; - } - Op::Reduce(arg, ReduceOp::Sum, reduced_dims) => { - let grad = broadcast_back(arg, &grad, reduced_dims)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&grad)?; - } - Op::Reduce(arg, ReduceOp::Max, reduced_dims) => { - let node = broadcast_back(arg, node, reduced_dims)?; - let grad = broadcast_back(arg, &grad, reduced_dims)?; - let grad = node.eq(arg)?.to_dtype(grad.dtype())?.mul(&grad)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&grad.broadcast_as(sum_grad.dims())?)?; - } - Op::Reduce(arg, ReduceOp::Min, reduced_dims) => { - let node = broadcast_back(arg, node, reduced_dims)?; - let grad = broadcast_back(arg, &grad, reduced_dims)?; - let grad = node.eq(arg)?.to_dtype(grad.dtype())?.mul(&grad)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&grad.broadcast_as(sum_grad.dims())?)?; - } - Op::ToDType(arg) => { - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&grad.to_dtype(arg.dtype())?)? - } - Op::Copy(arg) => { - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&grad)? - } - Op::Affine { arg, mul, .. } => { - let arg_grad = grad.affine(*mul, 0.)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&arg_grad)? - } - Op::Unary(arg, UnaryOp::Log) => { - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&(grad / arg)?)? - } - Op::Unary(arg, UnaryOp::Sin) => { - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&(&grad * arg.cos())?)? - } - Op::Unary(arg, UnaryOp::Cos) => { - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.sub(&(&grad * arg.sin())?)? - } - Op::Unary(arg, UnaryOp::Tanh) => { - let sum_grad = grads.or_insert(arg)?; - let minus_dtanh = (node.sqr()? - 1.)?; - *sum_grad = sum_grad.sub(&(&grad * &minus_dtanh)?)? - } - Op::Unary(arg, UnaryOp::Abs) => { - let sum_grad = grads.or_insert(arg)?; - let ones = arg.ones_like()?; - let abs_grad = arg.ge(&arg.zeros_like()?)?.where_cond(&ones, &ones.neg()?); - *sum_grad = sum_grad.add(&(&grad * abs_grad)?)? - } - Op::Unary(arg, UnaryOp::Exp) => { - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&(&grad * *node)?)? - } - Op::Unary(arg, UnaryOp::Neg) => { - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.sub(&grad)? - } - Op::Unary(arg, UnaryOp::Recip) => { - let sum_grad = grads.or_insert(arg)?; - let grad = (grad / arg.sqr()?)?; - *sum_grad = sum_grad.sub(&grad)? - } - &Op::Narrow(ref arg, dim, start_idx, len) => { - let arg_dims = arg.dims(); - let left_pad = if start_idx == 0 { - None - } else { - let mut dims = arg_dims.to_vec(); - dims[dim] = start_idx; - Some(Tensor::zeros(dims, grad.dtype(), grad.device())?) - }; - let right_pad = arg_dims[dim] - start_idx - len; - let right_pad = if right_pad == 0 { - None - } else { - let mut dims = arg_dims.to_vec(); - dims[dim] = right_pad; - Some(Tensor::zeros(dims, grad.dtype(), grad.device())?) - }; - let arg_grad = match (left_pad, right_pad) { - (None, None) => grad, - (Some(l), None) => Tensor::cat(&[&l, &grad], dim)?, - (None, Some(r)) => Tensor::cat(&[&grad, &r], dim)?, - (Some(l), Some(r)) => Tensor::cat(&[&l, &grad, &r], dim)?, - }; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&arg_grad)? - } - Op::Unary(_, UnaryOp::Floor) - | Op::Unary(_, UnaryOp::Round) - | Op::Reduce(_, ReduceOp::ArgMin, _) - | Op::Reduce(_, ReduceOp::ArgMax, _) - | Op::Unary(_, UnaryOp::Sign) - | Op::Cmp(_, _) => {} - Op::Reshape(arg) => { - let arg_grad = grad.reshape(arg.dims())?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&arg_grad)? - } - Op::Unary(_, UnaryOp::Ceil) => Err(Error::BackwardNotSupported { op: "ceil" })?, - Op::Unary(arg, UnaryOp::Gelu) => { - let sum_grad = grads.or_insert(arg)?; - let cube = arg.powf(3.)?; - let tanh = (0.0356774 * &cube + (0.797885 * arg)?)?.tanh()?; - let gelu_grad = (((0.5 * &tanh)? - + (0.0535161 * cube + (0.398942 * arg)?)? * (1. - tanh.powf(2.)?))? - + 0.5)?; - *sum_grad = sum_grad.add(&(&grad * gelu_grad)?)? - } - Op::Unary(arg, UnaryOp::Erf) => { - let sum_grad = grads.or_insert(arg)?; - // d/dx erf(x) = 2/sqrt(pi) * e^(-x^2) - let erf_grad = - (2. / std::f64::consts::PI.sqrt()) * (arg.sqr()?.neg()?).exp()?; - *sum_grad = sum_grad.add(&(&grad * erf_grad)?)? - } - Op::Unary(arg, UnaryOp::GeluErf) => { - let sum_grad = grads.or_insert(arg)?; - // d/dx gelu_erf(x) = 0.5 + 0.398942 e^(-x^2/2) x + 0.5 erf(x/sqrt(2)) - let neg_half_square = (arg.sqr()?.neg()? / 2.)?; - let scaled_exp_arg = (0.398942 * neg_half_square.exp()? * arg)?; - let arg_scaled_sqrt = (arg / 2f64.sqrt())?; - let erf_scaled_sqrt = (0.5 * arg_scaled_sqrt.erf()?)?; - let gelu_erf_grad = (0.5 + scaled_exp_arg + erf_scaled_sqrt)?; - *sum_grad = sum_grad.add(&(&grad * gelu_erf_grad)?)?; - } - Op::Unary(arg, UnaryOp::Relu) => { - let sum_grad = grads.or_insert(arg)?; - let relu_grad = arg.ge(&arg.zeros_like()?)?.to_dtype(arg.dtype())?; - *sum_grad = sum_grad.add(&(&grad * relu_grad)?)? - } - Op::Unary(arg, UnaryOp::Silu) => { - let sum_grad = grads.or_insert(arg)?; - // d/dx silu = sigmoid(x) * (1 + x * (1 - sigmoid(x))) = sigmoid(x) * (1 - node) + node - let sigmoid_arg = (arg.neg()?.exp()? + 1.)?.recip()?; - let silu_grad = &sigmoid_arg * (1. - *node) + *node; - *sum_grad = sum_grad.add(&(&grad * silu_grad)?)? - } - Op::Elu(arg, alpha) => { - // d/dx elu(x) = 1 for x > 0, alpha * e^x for x <= 0 - let sum_grad = grads.or_insert(arg)?; - let zeros = arg.zeros_like()?; - let positive_mask = arg.gt(&zeros)?.to_dtype(arg.dtype())?; - let negative_mask = arg.le(&zeros)?.to_dtype(arg.dtype())?; - // node == alpha * (e^x - 1) for x <= 0, reuse it - let negative_exp_mask = (negative_mask * (*node + *alpha))?; - let combined_mask = (positive_mask + negative_exp_mask)?; - *sum_grad = sum_grad.add(&(grad * combined_mask)?)? - } - Op::Powf(arg, e) => { - let arg_grad = (&(grad * arg.powf(e - 1.)?)? * *e)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&arg_grad)? - } - Op::CustomOp1(arg, c) => { - if let Some(arg_grad) = c.bwd(arg, node, &grad)? { - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&arg_grad)? - } - } - Op::CustomOp2(arg1, arg2, c) => { - let (arg_grad1, arg_grad2) = c.bwd(arg1, arg2, node, &grad)?; - if let Some(arg_grad1) = arg_grad1 { - let sum_grad = grads.or_insert(arg1)?; - *sum_grad = sum_grad.add(&arg_grad1)? - } - if let Some(arg_grad2) = arg_grad2 { - let sum_grad = grads.or_insert(arg2)?; - *sum_grad = sum_grad.add(&arg_grad2)? - } - } - Op::CustomOp3(arg1, arg2, arg3, c) => { - let (arg_grad1, arg_grad2, arg_grad3) = - c.bwd(arg1, arg2, arg3, node, &grad)?; - if let Some(arg_grad1) = arg_grad1 { - let sum_grad = grads.or_insert(arg1)?; - *sum_grad = sum_grad.add(&arg_grad1)? - } - if let Some(arg_grad2) = arg_grad2 { - let sum_grad = grads.or_insert(arg2)?; - *sum_grad = sum_grad.add(&arg_grad2)? - } - if let Some(arg_grad3) = arg_grad3 { - let sum_grad = grads.or_insert(arg3)?; - *sum_grad = sum_grad.add(&arg_grad3)? - } - } - Op::Unary(arg, UnaryOp::Sqr) => { - let arg_grad = arg.mul(&grad)?.affine(2., 0.)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&arg_grad)? - } - Op::Unary(arg, UnaryOp::Sqrt) => { - let arg_grad = grad.div(node)?.affine(0.5, 0.)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&arg_grad)? - } - Op::ToDevice(arg) => { - let sum_grad = grads.or_insert(arg)?; - let arg_grad = grad.to_device(sum_grad.device())?; - *sum_grad = sum_grad.add(&arg_grad)? - } - Op::Transpose(arg, dim1, dim2) => { - let arg_grad = grad.transpose(*dim1, *dim2)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&arg_grad)? - } - Op::Permute(arg, dims) => { - let mut inv_dims = vec![0; dims.len()]; - for (i, &dim_idx) in dims.iter().enumerate() { - inv_dims[dim_idx] = i - } - let arg_grad = grad.permute(inv_dims)?; - let sum_grad = grads.or_insert(arg)?; - *sum_grad = sum_grad.add(&arg_grad)? - } - }; - } - } - Ok(grads) - } -} - -/// A store for gradients, associating a tensor id to the corresponding gradient tensor, used for back propagation. -#[derive(Debug)] -pub struct GradStore(HashMap); - -impl GradStore { - /// Create a new gradient store - fn new() -> Self { - GradStore(HashMap::new()) - } - - /// Get the gradient tensor corresponding to the given tensor id - pub fn get_id(&self, id: TensorId) -> Option<&Tensor> { - self.0.get(&id) - } - - /// Get the gradient tensor associated with the given tensor - pub fn get(&self, tensor: &Tensor) -> Option<&Tensor> { - self.0.get(&tensor.id()) - } - - /// Remove the gradient tensor associated with the given tensor, returning it if it exists - pub fn remove(&mut self, tensor: &Tensor) -> Option { - self.0.remove(&tensor.id()) - } - - /// Insert a gradient tensor associated with the given tensor, returning the previous gradient tensor if it existed - pub fn insert(&mut self, tensor: &Tensor, grad: Tensor) -> Option { - self.0.insert(tensor.id(), grad) - } - - /// Insert a gradient tensor associated with the given tensor id, returning the previous gradient tensor if it existed - pub fn insert_id(&mut self, id: TensorId, grad: Tensor) -> Option { - self.0.insert(id, grad) - } - - /// Get the gradient tensor associated with the given tensor, or, if it does not exist, - /// insert a tensor of zeroes, with the same shape and type as the given tensors and return it - fn or_insert(&mut self, tensor: &Tensor) -> Result<&mut Tensor> { - use std::collections::hash_map::Entry; - let grad = match self.0.entry(tensor.id()) { - Entry::Occupied(entry) => entry.into_mut(), - Entry::Vacant(entry) => { - let grad = tensor.zeros_like()?; - entry.insert(grad) - } - }; - Ok(grad) - } - - /// Get the tensor ids of the stored gradient tensors - pub fn get_ids(&self) -> impl Iterator { - self.0.keys() - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/conv.rs b/v2/dependency-patches/candle-core-0.9.2/src/conv.rs deleted file mode 100644 index 115035ef1c..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/conv.rs +++ /dev/null @@ -1,387 +0,0 @@ -//! 1D and 2D Convolutions -//! -use crate::{op::BackpropOp, op::Op, Error, Result, Tensor}; - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ParamsConv1D { - pub(crate) b_size: usize, - // Maybe we should have a version without l_in as this bit depends on the input and not only on - // the weights. - pub(crate) l_in: usize, - pub(crate) c_out: usize, - pub(crate) c_in: usize, - pub(crate) k_size: usize, - pub(crate) padding: usize, - pub(crate) stride: usize, - pub(crate) dilation: usize, - pub(crate) cudnn_fwd_algo: Option, -} - -impl ParamsConv1D { - pub(crate) fn l_out(&self) -> usize { - (self.l_in + 2 * self.padding - self.dilation * (self.k_size - 1) - 1) / self.stride + 1 - } - - pub(crate) fn out_dims(&self) -> Vec { - let l_out = self.l_out(); - vec![self.b_size, self.c_out, l_out] - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ParamsConvTranspose1D { - pub(crate) b_size: usize, - pub(crate) l_in: usize, - pub(crate) c_out: usize, - pub(crate) c_in: usize, - pub(crate) k_size: usize, - pub(crate) padding: usize, - pub(crate) output_padding: usize, - pub(crate) stride: usize, - pub(crate) dilation: usize, -} - -impl ParamsConvTranspose1D { - pub(crate) fn l_out(&self) -> usize { - (self.l_in - 1) * self.stride - 2 * self.padding - + self.dilation * (self.k_size - 1) - + self.output_padding - + 1 - } - - pub(crate) fn out_dims(&self) -> Vec { - let l_out = self.l_out(); - vec![self.b_size, self.c_out, l_out] - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum CudnnFwdAlgo { - ImplicitGemm, - ImplicitPrecompGemm, - Gemm, - Direct, - Fft, - FftTiling, - Winograd, - WinogradNonFused, - Count, -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ParamsConv2D { - pub(crate) b_size: usize, - pub(crate) i_h: usize, - pub(crate) i_w: usize, - pub(crate) k_h: usize, - pub(crate) k_w: usize, - pub(crate) c_out: usize, - pub(crate) c_in: usize, - pub(crate) padding: usize, - pub(crate) stride: usize, - pub(crate) dilation: usize, - pub cudnn_fwd_algo: Option, -} - -impl ParamsConv2D { - pub(crate) fn out_h(&self) -> usize { - (self.i_h + 2 * self.padding - self.dilation * (self.k_h - 1) - 1) / self.stride + 1 - } - - pub(crate) fn out_w(&self) -> usize { - (self.i_w + 2 * self.padding - self.dilation * (self.k_w - 1) - 1) / self.stride + 1 - } - - pub(crate) fn out_dims(&self) -> Vec { - vec![self.b_size, self.c_out, self.out_h(), self.out_w()] - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ParamsConvTranspose2D { - pub(crate) b_size: usize, - pub(crate) i_h: usize, - pub(crate) i_w: usize, - pub(crate) k_h: usize, - pub(crate) k_w: usize, - pub(crate) c_out: usize, - pub(crate) c_in: usize, - pub(crate) padding: usize, - pub(crate) output_padding: usize, - pub(crate) stride: usize, - pub(crate) dilation: usize, -} - -impl ParamsConvTranspose2D { - pub(crate) fn out_h(&self) -> usize { - (self.i_h - 1) * self.stride + self.dilation * (self.k_h - 1) + self.output_padding + 1 - - 2 * self.padding - } - - pub(crate) fn out_w(&self) -> usize { - (self.i_w - 1) * self.stride + self.dilation * (self.k_w - 1) + self.output_padding + 1 - - 2 * self.padding - } - - pub(crate) fn out_dims(&self) -> Vec { - vec![self.b_size, self.c_out, self.out_h(), self.out_w()] - } -} - -impl Tensor { - fn conv1d_single_group(&self, kernel: &Self, params: &ParamsConv1D) -> Result { - let storage = - self.storage() - .conv1d(self.layout(), &kernel.storage(), kernel.layout(), params)?; - let op = BackpropOp::new2(self, kernel, |arg, kernel| Op::Conv1D { - arg, - kernel, - padding: params.padding, - stride: params.stride, - dilation: params.dilation, - }); - let out_dims = params.out_dims(); - Ok(crate::tensor::from_storage(storage, out_dims, op, false)) - } - - /// Applies a 1D convolution over the input tensor. - pub fn conv1d( - &self, - kernel: &Self, - padding: usize, - stride: usize, - dilation: usize, - groups: usize, - ) -> Result { - self.conv1d_with_algo(kernel, padding, stride, dilation, groups, None) - } - - /// Applies a 1D convolution over the input tensor. - pub fn conv1d_with_algo( - &self, - kernel: &Self, - padding: usize, - stride: usize, - dilation: usize, - groups: usize, - cudnn_fwd_algo: Option, - ) -> Result { - let (c_out, c_in_k, k_size) = kernel.dims3()?; - let (b_size, c_in, l_in) = self.dims3()?; - if c_in != c_in_k * groups { - Err(Error::Conv1dInvalidArgs { - inp_shape: self.shape().clone(), - k_shape: kernel.shape().clone(), - padding, - stride, - msg: "the number of in-channels on the input doesn't match the kernel size", - } - .bt())? - } - - let params = ParamsConv1D { - b_size, - l_in, - c_out: c_out / groups, - c_in: c_in / groups, - k_size, - padding, - stride, - dilation, - cudnn_fwd_algo, - }; - if groups == 1 { - self.conv1d_single_group(kernel, ¶ms) - } else { - let blocks = self.chunk(groups, 1)?; - let kernel = kernel.chunk(groups, 0)?; - let blocks = blocks - .iter() - .zip(&kernel) - .map(|(block, kernel)| block.conv1d_single_group(kernel, ¶ms)) - .collect::>>()?; - Tensor::cat(&blocks, 1) - } - } - - fn conv_transpose1d_single_group( - &self, - kernel: &Self, - params: &ParamsConvTranspose1D, - ) -> Result { - let storage = self.storage().conv_transpose1d( - self.layout(), - &kernel.storage(), - kernel.layout(), - params, - )?; - let op = BackpropOp::new2(self, kernel, |arg, kernel| Op::ConvTranspose1D { - arg, - kernel, - padding: params.padding, - output_padding: params.output_padding, - stride: params.stride, - dilation: params.dilation, - }); - let out_dims = params.out_dims(); - Ok(crate::tensor::from_storage(storage, out_dims, op, false)) - } - - /// Applies a 1D transposed convolution over the input tensor. - pub fn conv_transpose1d( - &self, - kernel: &Self, - padding: usize, - output_padding: usize, - stride: usize, - dilation: usize, - groups: usize, - ) -> Result { - let (c_in_k, c_out, k_size) = kernel.dims3()?; - let (b_size, c_in, l_in) = self.dims3()?; - if c_in != c_in_k { - crate::bail!("in_channel mismatch between input ({c_in}) and kernel ({c_in_k})") - } - if c_in % groups != 0 { - crate::bail!("in_channel {c_in} is not divisible by the number of groups") - } - let params = ParamsConvTranspose1D { - b_size, - l_in, - k_size, - c_out, - c_in: c_in / groups, - padding, - output_padding, - stride, - dilation, - }; - if groups == 1 { - self.conv_transpose1d_single_group(kernel, ¶ms) - } else { - let blocks = self.chunk(groups, 1)?; - let kernel = kernel.chunk(groups, 0)?; - let blocks = blocks - .iter() - .zip(&kernel) - .map(|(block, kernel)| block.conv_transpose1d_single_group(kernel, ¶ms)) - .collect::>>()?; - Tensor::cat(&blocks, 1) - } - } - - fn conv2d_single_group(&self, kernel: &Self, params: &ParamsConv2D) -> Result { - let storage = - self.storage() - .conv2d(self.layout(), &kernel.storage(), kernel.layout(), params)?; - let op = BackpropOp::new2(self, kernel, |arg, kernel| Op::Conv2D { - arg, - kernel, - padding: params.padding, - stride: params.stride, - dilation: params.dilation, - }); - let out_dims = params.out_dims(); - Ok(crate::tensor::from_storage(storage, out_dims, op, false)) - } - - /// Applies a 2D convolution over the input tensor. - pub fn conv2d( - &self, - kernel: &Self, - padding: usize, - stride: usize, - dilation: usize, - groups: usize, - ) -> Result { - self.conv2d_with_algo(kernel, padding, stride, dilation, groups, None) - } - - pub fn conv2d_with_algo( - &self, - kernel: &Self, - padding: usize, - stride: usize, - dilation: usize, - groups: usize, - cudnn_fwd_algo: Option, - ) -> Result { - let (b_size, c_in, i_h, i_w) = self.dims4()?; - let (c_out, c_in_k, k_h, k_w) = kernel.dims4()?; - if c_in != c_in_k * groups { - crate::bail!( - "in_channel mismatch between input ({c_in}, groups {groups}) and kernel ({c_in_k})" - ) - } - let params = ParamsConv2D { - b_size, - i_h, - i_w, - k_h, - k_w, - c_out: c_out / groups, - c_in: c_in / groups, - padding, - stride, - dilation, - cudnn_fwd_algo, - }; - if groups == 1 { - self.conv2d_single_group(kernel, ¶ms) - } else { - let blocks = self.chunk(groups, 1)?; - let kernel = kernel.chunk(groups, 0)?; - let blocks = blocks - .iter() - .zip(&kernel) - .map(|(block, kernel)| block.conv2d_single_group(kernel, ¶ms)) - .collect::>>()?; - Tensor::cat(&blocks, 1) - } - } - - /// Applies a 2D transposed convolution over the input tensor. - pub fn conv_transpose2d( - &self, - kernel: &Self, - padding: usize, - output_padding: usize, - stride: usize, - dilation: usize, - ) -> Result { - let (b_size, c_in, i_h, i_w) = self.dims4()?; - let (c_in_k, c_out, k_h, k_w) = kernel.dims4()?; - if c_in != c_in_k { - crate::bail!("in_channel mismatch between input ({c_in}) and kernel ({c_in_k})") - } - let params = ParamsConvTranspose2D { - b_size, - i_h, - i_w, - k_h, - k_w, - c_out, - c_in, - padding, - output_padding, - stride, - dilation, - }; - let storage = self.storage().conv_transpose2d( - self.layout(), - &kernel.storage(), - kernel.layout(), - ¶ms, - )?; - let op = BackpropOp::new2(self, kernel, |arg, kernel| Op::ConvTranspose2D { - arg, - kernel, - padding: params.padding, - output_padding: params.output_padding, - stride: params.stride, - dilation: params.dilation, - }); - let out_dims = params.out_dims(); - Ok(crate::tensor::from_storage(storage, out_dims, op, false)) - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/convert.rs b/v2/dependency-patches/candle-core-0.9.2/src/convert.rs deleted file mode 100644 index 38e7a7c9a6..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/convert.rs +++ /dev/null @@ -1,166 +0,0 @@ -//! Implement conversion traits for tensors -use crate::{DType, Device, Error, Tensor, WithDType}; -use half::{bf16, f16, slice::HalfFloatSliceExt}; -use std::convert::TryFrom; - -impl TryFrom<&Tensor> for Vec { - type Error = Error; - fn try_from(tensor: &Tensor) -> Result { - tensor.to_vec1::() - } -} - -impl TryFrom<&Tensor> for Vec> { - type Error = Error; - fn try_from(tensor: &Tensor) -> Result { - tensor.to_vec2::() - } -} - -impl TryFrom<&Tensor> for Vec>> { - type Error = Error; - fn try_from(tensor: &Tensor) -> Result { - tensor.to_vec3::() - } -} - -impl TryFrom for Vec { - type Error = Error; - fn try_from(tensor: Tensor) -> Result { - Vec::::try_from(&tensor) - } -} - -impl TryFrom for Vec> { - type Error = Error; - fn try_from(tensor: Tensor) -> Result { - Vec::>::try_from(&tensor) - } -} - -impl TryFrom for Vec>> { - type Error = Error; - fn try_from(tensor: Tensor) -> Result { - Vec::>>::try_from(&tensor) - } -} - -impl TryFrom<&[T]> for Tensor { - type Error = Error; - fn try_from(v: &[T]) -> Result { - Tensor::from_slice(v, v.len(), &Device::Cpu) - } -} - -impl TryFrom> for Tensor { - type Error = Error; - fn try_from(v: Vec) -> Result { - let len = v.len(); - Tensor::from_vec(v, len, &Device::Cpu) - } -} - -macro_rules! from_tensor { - ($typ:ident) => { - impl TryFrom<&Tensor> for $typ { - type Error = Error; - - fn try_from(tensor: &Tensor) -> Result { - tensor.to_scalar::<$typ>() - } - } - - impl TryFrom for $typ { - type Error = Error; - - fn try_from(tensor: Tensor) -> Result { - $typ::try_from(&tensor) - } - } - - impl TryFrom<$typ> for Tensor { - type Error = Error; - - fn try_from(v: $typ) -> Result { - Tensor::new(v, &Device::Cpu) - } - } - }; -} - -from_tensor!(f64); -from_tensor!(f32); -from_tensor!(f16); -from_tensor!(bf16); -from_tensor!(i64); -from_tensor!(i32); -from_tensor!(i16); -from_tensor!(u32); -from_tensor!(u8); - -impl Tensor { - pub fn write_bytes(&self, f: &mut W) -> crate::Result<()> { - use byteorder::{LittleEndian, WriteBytesExt}; - - let vs = self.flatten_all()?; - match self.dtype() { - DType::BF16 => { - let vs = vs.to_vec1::()?; - for &v in vs.reinterpret_cast() { - f.write_u16::(v)? - } - } - DType::F16 => { - let vs = vs.to_vec1::()?; - for &v in vs.reinterpret_cast() { - f.write_u16::(v)? - } - } - DType::F32 => { - // TODO: Avoid using a buffer when data is already on the CPU. - for v in vs.to_vec1::()? { - f.write_f32::(v)? - } - } - DType::F64 => { - for v in vs.to_vec1::()? { - f.write_f64::(v)? - } - } - DType::U32 => { - for v in vs.to_vec1::()? { - f.write_u32::(v)? - } - } - DType::I16 => { - for v in vs.to_vec1::()? { - f.write_i16::(v)? - } - } - DType::I32 => { - for v in vs.to_vec1::()? { - f.write_i32::(v)? - } - } - DType::I64 => { - for v in vs.to_vec1::()? { - f.write_i64::(v)? - } - } - DType::U8 => { - let vs = vs.to_vec1::()?; - f.write_all(&vs)?; - } - DType::F8E4M3 => { - let vs = vs.to_vec1::()?; - for v in vs { - f.write_u8(v.to_bits())? - } - } - DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { - return Err(crate::Error::UnsupportedDTypeForOp(self.dtype(), "write_bytes").bt()) - } - } - Ok(()) - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cpu/avx.rs b/v2/dependency-patches/candle-core-0.9.2/src/cpu/avx.rs deleted file mode 100644 index 113fc14ced..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/cpu/avx.rs +++ /dev/null @@ -1,227 +0,0 @@ -use super::{Cpu, CpuBF16, CpuF16}; -#[cfg(target_arch = "x86")] -use core::arch::x86::*; -#[cfg(target_arch = "x86_64")] -use core::arch::x86_64::*; - -use half::{bf16, f16}; - -pub struct CurrentCpu {} - -const STEP: usize = 32; -const EPR: usize = 8; -const ARR: usize = STEP / EPR; - -impl Cpu for CurrentCpu { - type Unit = __m256; - type Array = [__m256; ARR]; - - const STEP: usize = STEP; - const EPR: usize = EPR; - - fn n() -> usize { - ARR - } - - unsafe fn zero() -> Self::Unit { - _mm256_setzero_ps() - } - - unsafe fn zero_array() -> Self::Array { - [Self::zero(); ARR] - } - - unsafe fn from_f32(v: f32) -> Self::Unit { - _mm256_set1_ps(v) - } - - unsafe fn load(mem_addr: *const f32) -> Self::Unit { - _mm256_loadu_ps(mem_addr) - } - - unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit { - _mm256_add_ps(a, b) - } - - unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit { - _mm256_add_ps(_mm256_mul_ps(b, c), a) - } - - unsafe fn vec_store(mem_addr: *mut f32, a: Self::Unit) { - _mm256_storeu_ps(mem_addr, a); - } - - unsafe fn vec_reduce(mut x: Self::Array, y: *mut f32) { - for i in 0..ARR / 2 { - x[2 * i] = _mm256_add_ps(x[2 * i], x[2 * i + 1]); - } - for i in 0..ARR / 4 { - x[4 * i] = _mm256_add_ps(x[4 * i], x[4 * i + 2]); - } - #[allow(clippy::reversed_empty_ranges)] - for i in 0..ARR / 8 { - x[8 * i] = _mm256_add_ps(x[8 * i], x[8 * i + 4]); - } - let t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]), _mm256_extractf128_ps(x[0], 1)); - let t1 = _mm_hadd_ps(t0, t0); - *y = _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); - } -} - -pub struct CurrentCpuF16 {} -impl CpuF16 for CurrentCpuF16 { - type Unit = __m256; - type Array = [__m256; ARR]; - - const STEP: usize = STEP; - const EPR: usize = EPR; - - fn n() -> usize { - ARR - } - - unsafe fn zero() -> Self::Unit { - _mm256_setzero_ps() - } - - unsafe fn zero_array() -> Self::Array { - [Self::zero(); ARR] - } - - unsafe fn from_f32(v: f32) -> Self::Unit { - _mm256_set1_ps(v) - } - - #[cfg(target_feature = "f16c")] - unsafe fn load(mem_addr: *const f16) -> Self::Unit { - _mm256_cvtph_ps(_mm_loadu_si128(mem_addr as *const __m128i)) - } - - #[cfg(not(target_feature = "f16c"))] - unsafe fn load(mem_addr: *const f16) -> Self::Unit { - let mut tmp = [0.0f32; 8]; - for i in 0..8 { - tmp[i] = (*mem_addr.add(i)).to_f32(); - } - _mm256_loadu_ps(tmp.as_ptr()) - } - - unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit { - _mm256_add_ps(a, b) - } - - unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit { - _mm256_add_ps(_mm256_mul_ps(b, c), a) - } - - #[cfg(target_feature = "f16c")] - unsafe fn vec_store(mem_addr: *mut f16, a: Self::Unit) { - _mm_storeu_si128(mem_addr as *mut __m128i, _mm256_cvtps_ph(a, 0)) - } - - #[cfg(not(target_feature = "f16c"))] - unsafe fn vec_store(mem_addr: *mut f16, a: Self::Unit) { - let mut tmp = [0.0f32; 8]; - _mm256_storeu_ps(tmp.as_mut_ptr(), a); - for i in 0..8 { - *mem_addr.add(i) = f16::from_f32(tmp[i]); - } - } - - unsafe fn vec_reduce(mut x: Self::Array, y: *mut f32) { - let mut offset = ARR >> 1; - for i in 0..offset { - x[i] = _mm256_add_ps(x[i], x[offset + i]); - } - offset >>= 1; - for i in 0..offset { - x[i] = _mm256_add_ps(x[i], x[offset + i]); - } - offset >>= 1; - for i in 0..offset { - x[i] = _mm256_add_ps(x[i], x[offset + i]); - } - let t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]), _mm256_extractf128_ps(x[0], 1)); - let t1 = _mm_hadd_ps(t0, t0); - *y = _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); - } -} - -pub struct CurrentCpuBF16 {} -impl CpuBF16 for CurrentCpuBF16 { - type Unit = __m256; - type Array = [__m256; ARR]; - - const STEP: usize = STEP; - const EPR: usize = EPR; - - fn n() -> usize { - ARR - } - - unsafe fn zero() -> Self::Unit { - _mm256_setzero_ps() - } - - unsafe fn zero_array() -> Self::Array { - [Self::zero(); ARR] - } - - unsafe fn from_f32(v: f32) -> Self::Unit { - _mm256_set1_ps(v) - } - - #[cfg(target_feature = "f16c")] - unsafe fn load(mem_addr: *const bf16) -> Self::Unit { - _mm256_cvtph_ps(_mm_loadu_si128(mem_addr as *const __m128i)) - } - - #[cfg(not(target_feature = "f16c"))] - unsafe fn load(mem_addr: *const bf16) -> Self::Unit { - let mut tmp = [0.0f32; 8]; - for i in 0..8 { - tmp[i] = (*mem_addr.add(i)).to_f32(); - } - _mm256_loadu_ps(tmp.as_ptr()) - } - - unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit { - _mm256_add_ps(a, b) - } - - unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit { - _mm256_add_ps(_mm256_mul_ps(b, c), a) - } - - #[cfg(target_feature = "f16c")] - unsafe fn vec_store(mem_addr: *mut bf16, a: Self::Unit) { - _mm_storeu_si128(mem_addr as *mut __m128i, _mm256_cvtps_ph(a, 0)) - } - - #[cfg(not(target_feature = "f16c"))] - unsafe fn vec_store(mem_addr: *mut bf16, a: Self::Unit) { - let mut tmp = [0.0f32; 8]; - _mm256_storeu_ps(tmp.as_mut_ptr(), a); - for i in 0..8 { - *mem_addr.add(i) = bf16::from_f32(tmp[i]); - } - } - - unsafe fn vec_reduce(mut x: Self::Array, y: *mut f32) { - let mut offset = ARR >> 1; - for i in 0..offset { - x[i] = _mm256_add_ps(x[i], x[offset + i]); - } - offset >>= 1; - for i in 0..offset { - x[i] = _mm256_add_ps(x[i], x[offset + i]); - } - offset >>= 1; - for i in 0..offset { - x[i] = _mm256_add_ps(x[i], x[offset + i]); - } - let t0 = _mm_add_ps(_mm256_castps256_ps128(x[0]), _mm256_extractf128_ps(x[0], 1)); - let t1 = _mm_hadd_ps(t0, t0); - *y = _mm_cvtss_f32(_mm_hadd_ps(t1, t1)); - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cpu/erf.rs b/v2/dependency-patches/candle-core-0.9.2/src/cpu/erf.rs deleted file mode 100644 index 4d05728b0d..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/cpu/erf.rs +++ /dev/null @@ -1,325 +0,0 @@ -#![allow(clippy::excessive_precision)] -// Code taken from https://github.com/statrs-dev/statrs -//! Provides the [error](https://en.wikipedia.org/wiki/Error_function) and -//! related functions - -mod evaluate { - //! Provides functions that don't have a numerical solution and must - //! be solved computationally (e.g. evaluation of a polynomial) - - /// evaluates a polynomial at `z` where `coeff` are the coefficients - /// to a polynomial of order `k` where `k` is the length of `coeff` and the - /// coeffecient - /// to the `k`th power is the `k`th element in coeff. E.g. [3,-1,2] equates to - /// `2z^2 - z + 3` - /// - /// # Remarks - /// - /// Returns 0 for a 0 length coefficient slice - pub fn polynomial(z: f64, coeff: &[f64]) -> f64 { - let n = coeff.len(); - if n == 0 { - return 0.0; - } - - let mut sum = *coeff.last().unwrap(); - for c in coeff[0..n - 1].iter().rev() { - sum = *c + z * sum; - } - sum - } -} -use std::f64; - -/// `erf` calculates the error function at `x`. -pub fn erf_f64(x: f64) -> f64 { - libm::erf(x) -} - -pub fn erf_f32(x: f32) -> f32 { - libm::erff(x) -} - -/// `erf_inv` calculates the inverse error function -/// at `x`. -pub fn erf_inv(x: f64) -> f64 { - if x == 0.0 { - 0.0 - } else if x >= 1.0 { - f64::INFINITY - } else if x <= -1.0 { - f64::NEG_INFINITY - } else if x < 0.0 { - erf_inv_impl(-x, 1.0 + x, -1.0) - } else { - erf_inv_impl(x, 1.0 - x, 1.0) - } -} - -/// `erfc` calculates the complementary error function -/// at `x`. -pub fn erfc_f64(x: f64) -> f64 { - libm::erfc(x) -} - -pub fn erfc_f32(x: f32) -> f32 { - libm::erfcf(x) -} - -/// `erfc_inv` calculates the complementary inverse -/// error function at `x`. -pub fn erfc_inv(x: f64) -> f64 { - if x <= 0.0 { - f64::INFINITY - } else if x >= 2.0 { - f64::NEG_INFINITY - } else if x > 1.0 { - erf_inv_impl(-1.0 + x, 2.0 - x, -1.0) - } else { - erf_inv_impl(1.0 - x, x, 1.0) - } -} - -// ********************************************************** -// ********** Coefficients for erf_inv_impl polynomial ****** -// ********************************************************** - -/// Polynomial coefficients for a numerator of `erf_inv_impl` -/// in the interval [0, 0.5]. -const ERF_INV_IMPL_AN: &[f64] = &[ - -0.000508781949658280665617, - -0.00836874819741736770379, - 0.0334806625409744615033, - -0.0126926147662974029034, - -0.0365637971411762664006, - 0.0219878681111168899165, - 0.00822687874676915743155, - -0.00538772965071242932965, -]; - -/// Polynomial coefficients for a denominator of `erf_inv_impl` -/// in the interval [0, 0.5]. -const ERF_INV_IMPL_AD: &[f64] = &[ - 1.0, - -0.970005043303290640362, - -1.56574558234175846809, - 1.56221558398423026363, - 0.662328840472002992063, - -0.71228902341542847553, - -0.0527396382340099713954, - 0.0795283687341571680018, - -0.00233393759374190016776, - 0.000886216390456424707504, -]; - -/// Polynomial coefficients for a numerator of `erf_inv_impl` -/// in the interval [0.5, 0.75]. -const ERF_INV_IMPL_BN: &[f64] = &[ - -0.202433508355938759655, - 0.105264680699391713268, - 8.37050328343119927838, - 17.6447298408374015486, - -18.8510648058714251895, - -44.6382324441786960818, - 17.445385985570866523, - 21.1294655448340526258, - -3.67192254707729348546, -]; - -/// Polynomial coefficients for a denominator of `erf_inv_impl` -/// in the interval [0.5, 0.75]. -const ERF_INV_IMPL_BD: &[f64] = &[ - 1.0, - 6.24264124854247537712, - 3.9713437953343869095, - -28.6608180499800029974, - -20.1432634680485188801, - 48.5609213108739935468, - 10.8268667355460159008, - -22.6436933413139721736, - 1.72114765761200282724, -]; - -/// Polynomial coefficients for a numerator of `erf_inv_impl` -/// in the interval [0.75, 1] with x less than 3. -const ERF_INV_IMPL_CN: &[f64] = &[ - -0.131102781679951906451, - -0.163794047193317060787, - 0.117030156341995252019, - 0.387079738972604337464, - 0.337785538912035898924, - 0.142869534408157156766, - 0.0290157910005329060432, - 0.00214558995388805277169, - -0.679465575181126350155e-6, - 0.285225331782217055858e-7, - -0.681149956853776992068e-9, -]; - -/// Polynomial coefficients for a denominator of `erf_inv_impl` -/// in the interval [0.75, 1] with x less than 3. -const ERF_INV_IMPL_CD: &[f64] = &[ - 1.0, - 3.46625407242567245975, - 5.38168345707006855425, - 4.77846592945843778382, - 2.59301921623620271374, - 0.848854343457902036425, - 0.152264338295331783612, - 0.01105924229346489121, -]; - -/// Polynomial coefficients for a numerator of `erf_inv_impl` -/// in the interval [0.75, 1] with x between 3 and 6. -const ERF_INV_IMPL_DN: &[f64] = &[ - -0.0350353787183177984712, - -0.00222426529213447927281, - 0.0185573306514231072324, - 0.00950804701325919603619, - 0.00187123492819559223345, - 0.000157544617424960554631, - 0.460469890584317994083e-5, - -0.230404776911882601748e-9, - 0.266339227425782031962e-11, -]; - -/// Polynomial coefficients for a denominator of `erf_inv_impl` -/// in the interval [0.75, 1] with x between 3 and 6. -const ERF_INV_IMPL_DD: &[f64] = &[ - 1.0, - 1.3653349817554063097, - 0.762059164553623404043, - 0.220091105764131249824, - 0.0341589143670947727934, - 0.00263861676657015992959, - 0.764675292302794483503e-4, -]; - -/// Polynomial coefficients for a numerator of `erf_inv_impl` -/// in the interval [0.75, 1] with x between 6 and 18. -const ERF_INV_IMPL_EN: &[f64] = &[ - -0.0167431005076633737133, - -0.00112951438745580278863, - 0.00105628862152492910091, - 0.000209386317487588078668, - 0.149624783758342370182e-4, - 0.449696789927706453732e-6, - 0.462596163522878599135e-8, - -0.281128735628831791805e-13, - 0.99055709973310326855e-16, -]; - -/// Polynomial coefficients for a denominator of `erf_inv_impl` -/// in the interval [0.75, 1] with x between 6 and 18. -const ERF_INV_IMPL_ED: &[f64] = &[ - 1.0, - 0.591429344886417493481, - 0.138151865749083321638, - 0.0160746087093676504695, - 0.000964011807005165528527, - 0.275335474764726041141e-4, - 0.282243172016108031869e-6, -]; - -/// Polynomial coefficients for a numerator of `erf_inv_impl` -/// in the interval [0.75, 1] with x between 18 and 44. -const ERF_INV_IMPL_FN: &[f64] = &[ - -0.0024978212791898131227, - -0.779190719229053954292e-5, - 0.254723037413027451751e-4, - 0.162397777342510920873e-5, - 0.396341011304801168516e-7, - 0.411632831190944208473e-9, - 0.145596286718675035587e-11, - -0.116765012397184275695e-17, -]; - -/// Polynomial coefficients for a denominator of `erf_inv_impl` -/// in the interval [0.75, 1] with x between 18 and 44. -const ERF_INV_IMPL_FD: &[f64] = &[ - 1.0, - 0.207123112214422517181, - 0.0169410838120975906478, - 0.000690538265622684595676, - 0.145007359818232637924e-4, - 0.144437756628144157666e-6, - 0.509761276599778486139e-9, -]; - -/// Polynomial coefficients for a numerator of `erf_inv_impl` -/// in the interval [0.75, 1] with x greater than 44. -const ERF_INV_IMPL_GN: &[f64] = &[ - -0.000539042911019078575891, - -0.28398759004727721098e-6, - 0.899465114892291446442e-6, - 0.229345859265920864296e-7, - 0.225561444863500149219e-9, - 0.947846627503022684216e-12, - 0.135880130108924861008e-14, - -0.348890393399948882918e-21, -]; - -/// Polynomial coefficients for a denominator of `erf_inv_impl` -/// in the interval [0.75, 1] with x greater than 44. -const ERF_INV_IMPL_GD: &[f64] = &[ - 1.0, - 0.0845746234001899436914, - 0.00282092984726264681981, - 0.468292921940894236786e-4, - 0.399968812193862100054e-6, - 0.161809290887904476097e-8, - 0.231558608310259605225e-11, -]; - -// `erf_inv_impl` computes the inverse error function where -// `p`,`q`, and `s` are the first, second, and third intermediate -// parameters respectively -fn erf_inv_impl(p: f64, q: f64, s: f64) -> f64 { - let result = if p <= 0.5 { - let y = 0.0891314744949340820313; - let g = p * (p + 10.0); - let r = evaluate::polynomial(p, ERF_INV_IMPL_AN) / evaluate::polynomial(p, ERF_INV_IMPL_AD); - g * y + g * r - } else if q >= 0.25 { - let y = 2.249481201171875; - let g = (-2.0 * q.ln()).sqrt(); - let xs = q - 0.25; - let r = - evaluate::polynomial(xs, ERF_INV_IMPL_BN) / evaluate::polynomial(xs, ERF_INV_IMPL_BD); - g / (y + r) - } else { - let x = (-q.ln()).sqrt(); - if x < 3.0 { - let y = 0.807220458984375; - let xs = x - 1.125; - let r = evaluate::polynomial(xs, ERF_INV_IMPL_CN) - / evaluate::polynomial(xs, ERF_INV_IMPL_CD); - y * x + r * x - } else if x < 6.0 { - let y = 0.93995571136474609375; - let xs = x - 3.0; - let r = evaluate::polynomial(xs, ERF_INV_IMPL_DN) - / evaluate::polynomial(xs, ERF_INV_IMPL_DD); - y * x + r * x - } else if x < 18.0 { - let y = 0.98362827301025390625; - let xs = x - 6.0; - let r = evaluate::polynomial(xs, ERF_INV_IMPL_EN) - / evaluate::polynomial(xs, ERF_INV_IMPL_ED); - y * x + r * x - } else if x < 44.0 { - let y = 0.99714565277099609375; - let xs = x - 18.0; - let r = evaluate::polynomial(xs, ERF_INV_IMPL_FN) - / evaluate::polynomial(xs, ERF_INV_IMPL_FD); - y * x + r * x - } else { - let y = 0.99941349029541015625; - let xs = x - 44.0; - let r = evaluate::polynomial(xs, ERF_INV_IMPL_GN) - / evaluate::polynomial(xs, ERF_INV_IMPL_GD); - y * x + r * x - } - }; - s * result -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cpu/kernels.rs b/v2/dependency-patches/candle-core-0.9.2/src/cpu/kernels.rs deleted file mode 100644 index bca76adcc8..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/cpu/kernels.rs +++ /dev/null @@ -1,232 +0,0 @@ -pub trait VecOps: num_traits::NumAssign + Copy { - fn min(self, rhs: Self) -> Self; - fn max(self, rhs: Self) -> Self; - - /// Dot-product of two vectors. - /// - /// # Safety - /// - /// The length of `lhs` and `rhs` have to be at least `len`. `res` has to point to a valid - /// element. - #[inline(always)] - unsafe fn vec_dot(lhs: *const Self, rhs: *const Self, res: *mut Self, len: usize) { - *res = Self::zero(); - for i in 0..len { - *res += *lhs.add(i) * *rhs.add(i) - } - } - - /// Sum of all elements in a vector. - /// - /// # Safety - /// - /// The length of `xs` must be at least `len`. `res` has to point to a valid - /// element. - #[inline(always)] - unsafe fn vec_reduce_sum(xs: *const Self, res: *mut Self, len: usize) { - *res = Self::zero(); - for i in 0..len { - *res += *xs.add(i) - } - } - - /// Maximum element in a non-empty vector. - /// - /// # Safety - /// - /// The length of `xs` must be at least `len` and positive. `res` has to point to a valid - /// element. - #[inline(always)] - unsafe fn vec_reduce_max(xs: *const Self, res: *mut Self, len: usize) { - *res = *xs; - for i in 1..len { - *res = (*res).max(*xs.add(i)) - } - } - - /// Minimum element in a non-empty vector. - /// - /// # Safety - /// - /// The length of `xs` must be at least `len` and positive. `res` has to point to a valid - /// element. - #[inline(always)] - unsafe fn vec_reduce_min(xs: *const Self, res: *mut Self, len: usize) { - *res = *xs; - for i in 1..len { - *res = (*res).min(*xs.add(i)) - } - } -} - -impl VecOps for f32 { - #[inline(always)] - fn min(self, other: Self) -> Self { - Self::min(self, other) - } - - #[inline(always)] - fn max(self, other: Self) -> Self { - Self::max(self, other) - } - - #[inline(always)] - unsafe fn vec_dot(lhs: *const Self, rhs: *const Self, res: *mut Self, len: usize) { - super::vec_dot_f32(lhs, rhs, res, len) - } - - #[inline(always)] - unsafe fn vec_reduce_sum(xs: *const Self, res: *mut Self, len: usize) { - super::vec_sum(xs, res, len) - } -} - -impl VecOps for half::f16 { - #[inline(always)] - fn min(self, other: Self) -> Self { - Self::min(self, other) - } - - #[inline(always)] - fn max(self, other: Self) -> Self { - Self::max(self, other) - } - - #[inline(always)] - unsafe fn vec_dot(lhs: *const Self, rhs: *const Self, res: *mut Self, len: usize) { - let mut res_f32 = 0f32; - super::vec_dot_f16(lhs, rhs, &mut res_f32, len); - *res = half::f16::from_f32(res_f32); - } -} - -impl VecOps for f64 { - #[inline(always)] - fn min(self, other: Self) -> Self { - Self::min(self, other) - } - - #[inline(always)] - fn max(self, other: Self) -> Self { - Self::max(self, other) - } -} -impl VecOps for half::bf16 { - #[inline(always)] - fn min(self, other: Self) -> Self { - Self::min(self, other) - } - - #[inline(always)] - fn max(self, other: Self) -> Self { - Self::max(self, other) - } - - #[inline(always)] - unsafe fn vec_dot(lhs: *const Self, rhs: *const Self, res: *mut Self, len: usize) { - let mut res_f32 = 0f32; - super::vec_dot_bf16(lhs, rhs, &mut res_f32, len); - *res = half::bf16::from_f32(res_f32); - } -} -impl VecOps for u8 { - #[inline(always)] - fn min(self, other: Self) -> Self { - ::min(self, other) - } - - #[inline(always)] - fn max(self, other: Self) -> Self { - ::max(self, other) - } -} -impl VecOps for u32 { - #[inline(always)] - fn min(self, other: Self) -> Self { - ::min(self, other) - } - - #[inline(always)] - fn max(self, other: Self) -> Self { - ::max(self, other) - } -} -impl VecOps for i16 { - #[inline(always)] - fn min(self, other: Self) -> Self { - ::min(self, other) - } - - #[inline(always)] - fn max(self, other: Self) -> Self { - ::max(self, other) - } -} -impl VecOps for i32 { - #[inline(always)] - fn min(self, other: Self) -> Self { - ::min(self, other) - } - - #[inline(always)] - fn max(self, other: Self) -> Self { - ::max(self, other) - } -} -impl VecOps for i64 { - #[inline(always)] - fn min(self, other: Self) -> Self { - ::min(self, other) - } - - #[inline(always)] - fn max(self, other: Self) -> Self { - ::max(self, other) - } -} - -impl VecOps for float8::F8E4M3 { - #[inline(always)] - fn min(self, other: Self) -> Self { - Self::min(self, other) - } - - #[inline(always)] - fn max(self, other: Self) -> Self { - Self::max(self, other) - } -} - -#[inline(always)] -pub fn par_for_each(n_threads: usize, func: impl Fn(usize) + Send + Sync) { - if n_threads == 1 { - func(0) - } else { - rayon::scope(|s| { - for thread_idx in 0..n_threads { - let func = &func; - s.spawn(move |_| func(thread_idx)); - } - }) - } -} - -#[inline(always)] -pub fn par_range(lo: usize, up: usize, n_threads: usize, func: impl Fn(usize) + Send + Sync) { - if n_threads == 1 { - for i in lo..up { - func(i) - } - } else { - rayon::scope(|s| { - for thread_idx in 0..n_threads { - let func = &func; - s.spawn(move |_| { - for i in (thread_idx..up).step_by(n_threads) { - func(i) - } - }); - } - }) - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cpu/mod.rs b/v2/dependency-patches/candle-core-0.9.2/src/cpu/mod.rs deleted file mode 100644 index c4864b7a81..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/cpu/mod.rs +++ /dev/null @@ -1,242 +0,0 @@ -//! Traits and methods for CPU-backed Tensors - -pub mod erf; -pub mod kernels; - -#[allow(unused)] -trait Cpu { - type Unit; - type Array; - const STEP: usize; - const EPR: usize; - - fn n() -> usize; - unsafe fn zero() -> Self::Unit; - unsafe fn zero_array() -> Self::Array; - unsafe fn load(mem_addr: *const f32) -> Self::Unit; - unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit; - unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit; - unsafe fn vec_reduce(x: Self::Array, y: *mut f32); - unsafe fn from_f32(v: f32) -> Self::Unit; - unsafe fn vec_store(mem_addr: *mut f32, a: Self::Unit); -} - -#[allow(unused)] -trait CpuF16 { - type Unit; - type Array; - const STEP: usize; - const EPR: usize; - - fn n() -> usize; - unsafe fn zero() -> Self::Unit; - unsafe fn zero_array() -> Self::Array; - unsafe fn load(mem_addr: *const f16) -> Self::Unit; - unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit; - unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit; - unsafe fn vec_reduce(x: Self::Array, y: *mut f32); - unsafe fn from_f32(v: f32) -> Self::Unit; - unsafe fn vec_store(mem_addr: *mut f16, a: Self::Unit); -} - -#[allow(unused)] -trait CpuBF16 { - type Unit; - type Array; - const STEP: usize; - const EPR: usize; - - fn n() -> usize; - unsafe fn zero() -> Self::Unit; - unsafe fn zero_array() -> Self::Array; - unsafe fn load(mem_addr: *const bf16) -> Self::Unit; - unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit; - unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit; - unsafe fn vec_reduce(x: Self::Array, y: *mut f32); - unsafe fn from_f32(v: f32) -> Self::Unit; - unsafe fn vec_store(mem_addr: *mut bf16, a: Self::Unit); -} - -use half::{bf16, f16}; - -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -#[cfg(target_feature = "avx2")] -pub mod avx; -#[cfg(any(target_arch = "x86", target_arch = "x86_64"))] -#[cfg(target_feature = "avx2")] -pub use avx::{CurrentCpu, CurrentCpuBF16, CurrentCpuF16}; - -#[cfg(target_arch = "wasm32")] -#[cfg(target_feature = "simd128")] -pub mod simd128; -#[cfg(target_arch = "wasm32")] -#[cfg(target_feature = "simd128")] -pub use simd128::CurrentCpu; - -#[cfg(any(target_arch = "arm", target_arch = "aarch64"))] -#[cfg(target_feature = "neon")] -pub mod neon; -#[cfg(any(target_arch = "arm", target_arch = "aarch64"))] -#[cfg(target_feature = "neon")] -pub use neon::CurrentCpu; - -#[cfg(any( - target_feature = "neon", - target_feature = "avx2", - target_feature = "simd128" -))] -#[inline(always)] -pub(crate) unsafe fn vec_dot_f32(a_row: *const f32, b_row: *const f32, c: *mut f32, k: usize) { - let np = k & !(CurrentCpu::STEP - 1); - - let mut sum = CurrentCpu::zero_array(); - let mut ax = CurrentCpu::zero_array(); - let mut ay = CurrentCpu::zero_array(); - - for i in (0..np).step_by(CurrentCpu::STEP) { - for j in 0..CurrentCpu::n() { - ax[j] = CurrentCpu::load(a_row.add(i + j * CurrentCpu::EPR)); - ay[j] = CurrentCpu::load(b_row.add(i + j * CurrentCpu::EPR)); - - sum[j] = CurrentCpu::vec_fma(sum[j], ax[j], ay[j]); - } - } - - CurrentCpu::vec_reduce(sum, c); - - // leftovers - for i in np..k { - *c += *a_row.add(i) * (*b_row.add(i)); - } -} - -#[cfg(not(any( - target_feature = "neon", - target_feature = "avx2", - target_feature = "simd128" -)))] -#[inline(always)] -pub(crate) unsafe fn vec_dot_f32(a_row: *const f32, b_row: *const f32, c: *mut f32, k: usize) { - // leftovers - for i in 0..k { - *c += *a_row.add(i) * (*b_row.add(i)); - } -} - -#[cfg(any( - target_feature = "neon", - target_feature = "avx2", - target_feature = "simd128" -))] -#[inline(always)] -pub(crate) unsafe fn vec_sum(row: *const f32, b: *mut f32, k: usize) { - let np = k & !(CurrentCpu::STEP - 1); - - let mut sum = CurrentCpu::zero_array(); - let mut x = CurrentCpu::zero_array(); - - for i in (0..np).step_by(CurrentCpu::STEP) { - for j in 0..CurrentCpu::n() { - x[j] = CurrentCpu::load(row.add(i + j * CurrentCpu::EPR)); - sum[j] = CurrentCpu::vec_add(sum[j], x[j]); - } - } - - CurrentCpu::vec_reduce(sum, b); - - // leftovers - for i in np..k { - *b += *row.add(i) - } -} - -#[cfg(not(any( - target_feature = "neon", - target_feature = "avx2", - target_feature = "simd128" -)))] -#[inline(always)] -pub(crate) unsafe fn vec_sum(row: *const f32, b: *mut f32, k: usize) { - *b = 0f32; - for i in 0..k { - *b += *row.add(i) - } -} - -#[cfg(target_feature = "avx2")] -#[inline(always)] -pub(crate) unsafe fn vec_dot_f16(a_row: *const f16, b_row: *const f16, c: *mut f32, k: usize) { - let mut sumf = 0.0f32; - let np = k & !(CurrentCpuF16::STEP - 1); - - let mut sum = CurrentCpuF16::zero_array(); - let mut ax = CurrentCpuF16::zero_array(); - let mut ay = CurrentCpuF16::zero_array(); - - for i in (0..np).step_by(CurrentCpuF16::STEP) { - for j in 0..CurrentCpuF16::n() { - ax[j] = CurrentCpuF16::load(a_row.add(i + j * CurrentCpuF16::EPR)); - ay[j] = CurrentCpuF16::load(b_row.add(i + j * CurrentCpuF16::EPR)); - - sum[j] = CurrentCpuF16::vec_fma(sum[j], ax[j], ay[j]); - } - } - - CurrentCpuF16::vec_reduce(sum, &mut sumf); - - // leftovers - for i in np..k { - sumf += (*a_row.add(i)).to_f32() * (*b_row.add(i)).to_f32(); - } - *c = sumf; -} - -#[cfg(target_feature = "avx2")] -#[inline(always)] -pub(crate) unsafe fn vec_dot_bf16(a_row: *const bf16, b_row: *const bf16, c: *mut f32, k: usize) { - let mut sumf = 0.0f32; - let np = k & !(CurrentCpuBF16::STEP - 1); - - let mut sum = CurrentCpuBF16::zero_array(); - let mut ax = CurrentCpuBF16::zero_array(); - let mut ay = CurrentCpuBF16::zero_array(); - - for i in (0..np).step_by(CurrentCpuBF16::STEP) { - for j in 0..CurrentCpuBF16::n() { - ax[j] = CurrentCpuBF16::load(a_row.add(i + j * CurrentCpuBF16::EPR)); - ay[j] = CurrentCpuBF16::load(b_row.add(i + j * CurrentCpuBF16::EPR)); - - sum[j] = CurrentCpuBF16::vec_fma(sum[j], ax[j], ay[j]); - } - } - - CurrentCpuBF16::vec_reduce(sum, &mut sumf); - - // leftovers - for i in np..k { - sumf += (*a_row.add(i)).to_f32() * (*b_row.add(i)).to_f32(); - } - *c = sumf; -} - -#[cfg(not(target_feature = "avx2"))] -#[inline(always)] -pub(crate) unsafe fn vec_dot_f16(a_row: *const f16, b_row: *const f16, c: *mut f32, k: usize) { - // leftovers - let mut sum = 0.0; - for i in 0..k { - sum += (*a_row.add(i)).to_f32() * (*b_row.add(i)).to_f32(); - } - *c = sum; -} - -#[cfg(not(target_feature = "avx2"))] -#[inline(always)] -pub(crate) unsafe fn vec_dot_bf16(a_row: *const bf16, b_row: *const bf16, c: *mut f32, k: usize) { - // leftovers - let mut sum = 0.0; - for i in 0..k { - sum += (*a_row.add(i)).to_f32() * (*b_row.add(i)).to_f32(); - } - *c = sum; -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cpu/neon.rs b/v2/dependency-patches/candle-core-0.9.2/src/cpu/neon.rs deleted file mode 100644 index 66b8b45e15..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/cpu/neon.rs +++ /dev/null @@ -1,74 +0,0 @@ -use super::Cpu; -#[cfg(target_arch = "arm")] -use core::arch::arm::*; - -#[cfg(target_arch = "aarch64")] -use core::arch::aarch64::*; - -pub struct CurrentCpu {} - -const STEP: usize = 16; -const EPR: usize = 4; -const ARR: usize = STEP / EPR; - -impl CurrentCpu { - #[cfg(target_arch = "aarch64")] - unsafe fn reduce_one(x: float32x4_t) -> f32 { - vaddvq_f32(x) - } - - #[cfg(target_arch = "arm")] - unsafe fn reduce_one(x: float32x4_t) -> f32 { - vgetq_lane_f32(x, 0) + vgetq_lane_f32(x, 1) + vgetq_lane_f32(x, 2) + vgetq_lane_f32(x, 3) - } -} - -impl Cpu for CurrentCpu { - type Unit = float32x4_t; - type Array = [float32x4_t; ARR]; - - const STEP: usize = STEP; - const EPR: usize = EPR; - - fn n() -> usize { - ARR - } - - unsafe fn zero() -> Self::Unit { - vdupq_n_f32(0.0) - } - - unsafe fn from_f32(x: f32) -> Self::Unit { - vdupq_n_f32(x) - } - - unsafe fn zero_array() -> Self::Array { - [Self::zero(); ARR] - } - - unsafe fn load(mem_addr: *const f32) -> Self::Unit { - vld1q_f32(mem_addr) - } - - unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit { - vaddq_f32(a, b) - } - - unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit { - vfmaq_f32(a, b, c) - } - - unsafe fn vec_store(mem_addr: *mut f32, a: Self::Unit) { - vst1q_f32(mem_addr, a); - } - - unsafe fn vec_reduce(mut x: Self::Array, y: *mut f32) { - for i in 0..ARR / 2 { - x[2 * i] = vaddq_f32(x[2 * i], x[2 * i + 1]); - } - for i in 0..ARR / 4 { - x[4 * i] = vaddq_f32(x[4 * i], x[4 * i + 2]); - } - *y = Self::reduce_one(x[0]); - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cpu/simd128.rs b/v2/dependency-patches/candle-core-0.9.2/src/cpu/simd128.rs deleted file mode 100644 index 64b94588bd..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/cpu/simd128.rs +++ /dev/null @@ -1,64 +0,0 @@ -use super::Cpu; -use core::arch::wasm32::*; - -pub struct CurrentCpu {} - -const STEP: usize = 16; -const EPR: usize = 4; -const ARR: usize = STEP / EPR; - -impl Cpu for CurrentCpu { - type Unit = v128; - type Array = [v128; ARR]; - - const STEP: usize = STEP; - const EPR: usize = EPR; - - fn n() -> usize { - ARR - } - - unsafe fn zero() -> Self::Unit { - f32x4_splat(0.0) - } - - unsafe fn zero_array() -> Self::Array { - [Self::zero(); ARR] - } - - unsafe fn from_f32(v: f32) -> Self::Unit { - f32x4_splat(v) - } - - unsafe fn load(mem_addr: *const f32) -> Self::Unit { - v128_load(mem_addr as *mut v128) - } - - unsafe fn vec_add(a: Self::Unit, b: Self::Unit) -> Self::Unit { - f32x4_add(a, b) - } - - unsafe fn vec_fma(a: Self::Unit, b: Self::Unit, c: Self::Unit) -> Self::Unit { - f32x4_add(f32x4_mul(b, c), a) - } - - unsafe fn vec_store(mem_addr: *mut f32, a: Self::Unit) { - v128_store(mem_addr as *mut v128, a); - } - - unsafe fn vec_reduce(mut x: Self::Array, y: *mut f32) { - for i in 0..ARR / 2 { - x[2 * i] = f32x4_add(x[2 * i], x[2 * i + 1]); - } - for i in 0..ARR / 4 { - x[4 * i] = f32x4_add(x[4 * i], x[4 * i + 2]); - } - for i in 0..ARR / 8 { - x[8 * i] = f32x4_add(x[8 * i], x[8 * i + 4]); - } - *y = f32x4_extract_lane::<0>(x[0]) - + f32x4_extract_lane::<1>(x[0]) - + f32x4_extract_lane::<2>(x[0]) - + f32x4_extract_lane::<3>(x[0]); - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/conv2d.rs b/v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/conv2d.rs deleted file mode 100644 index 70c5ea75fa..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/conv2d.rs +++ /dev/null @@ -1,432 +0,0 @@ -use std::borrow::Cow; - -use rayon::iter::{IntoParallelIterator, ParallelIterator}; - -use crate::{ - conv::ParamsConv2D, - cpu_backend::{copy_strided_src_, Im2Col, Map1, Map2, MatMul}, - shape::dims4, - Layout, Result, WithDType, -}; - -pub(super) struct Conv2D<'a>(pub(super) &'a crate::conv::ParamsConv2D); - -#[allow(dead_code)] -enum Conv2dImpl { - TiledIm2Col, - FullIm2Col, - Direct, -} - -const DEFAULT_CONV2D_IMPL: Conv2dImpl = Conv2dImpl::TiledIm2Col; - -impl Map2 for Conv2D<'_> { - const OP: &'static str = "conv2d"; - fn f( - &self, - inp: &[T], - inp_l: &Layout, - k: &[T], - k_l: &Layout, - ) -> Result> { - let p = self.0; - - // Specialization: pick the best algorithm based on parameters. - // 1x1 convolutions with stride=1, padding=0, dilation=1 - if p.k_h == 1 && p.k_w == 1 && p.stride == 1 && p.padding == 0 && p.dilation == 1 { - return conv2d_1x1(p, inp, inp_l, k, k_l); - } else if p.k_h == 1 && p.k_w == 1 { - // Other 1x1 convolutions for now are assumed faster with full im2col, - // although with large enough input size, tiled will start beating it. - return conv2d_im2col_gemm(p, inp, inp_l, k, k_l); - } - // TODO other cases - - // No fast path, fallback to default general impl. - match DEFAULT_CONV2D_IMPL { - Conv2dImpl::TiledIm2Col => conv2d_tiled(p, inp, inp_l, k, k_l), - Conv2dImpl::Direct => conv2d_direct(p, inp, inp_l, k, k_l), - Conv2dImpl::FullIm2Col => conv2d_im2col_gemm(p, inp, inp_l, k, k_l), - } - } -} - -/// Fast kernel for 1x1 convolutions with stride=1, padding=0, dilation=1 -/// These are just matrix multiplications: [c_out, c_in] @ [c_in, b*h*w] -> [c_out, b*h*w]. -fn conv2d_1x1( - p: &ParamsConv2D, - inp: &[T], - inp_l: &Layout, - k: &[T], - k_l: &Layout, -) -> Result> { - let inp = &inp[inp_l.start_offset()..]; - let inp_stride = inp_l.stride(); - let (inp_s0, inp_s1, inp_s2, inp_s3) = - (inp_stride[0], inp_stride[1], inp_stride[2], inp_stride[3]); - let k = &k[k_l.start_offset()..]; - let k_stride = k_l.stride(); - let (k_s0, k_s1) = (k_stride[0], k_stride[1]); - let (out_h, out_w) = (p.out_h(), p.out_w()); - - let spatial_size = out_h * out_w; - let dst = vec![T::zero(); p.b_size * p.c_out * spatial_size]; - let k_reshaped: Cow<[T]> = if k_s0 == p.c_in && k_s1 == 1 { - // Already contiguous, use slice directly - Cow::Borrowed(&k[..p.c_out * p.c_in]) - } else { - // Reshape kernel to [c_out, c_in] - let mut k_reshaped = Vec::with_capacity(p.c_out * p.c_in); - (0..p.c_out).for_each(|c_out_idx| { - (0..p.c_in).for_each(|c_in_idx| { - let k_idx = c_out_idx * k_s0 + c_in_idx * k_s1; - k_reshaped.push(k[k_idx]); - }); - }); - Cow::Owned(k_reshaped) - }; - let k_layout = Layout::contiguous((p.c_out, p.c_in)); - - // Process each batch - (0..p.b_size).into_par_iter().try_for_each(|b_idx| { - // Reshape input to [c_in, h*w] for this batch - let mut inp_reshaped = Vec::with_capacity(p.c_in * spatial_size); - for c_in_idx in 0..p.c_in { - for h_idx in 0..p.i_h { - for w_idx in 0..p.i_w { - let inp_idx = - b_idx * inp_s0 + c_in_idx * inp_s1 + h_idx * inp_s2 + w_idx * inp_s3; - inp_reshaped.push(inp[inp_idx]); - } - } - } - let inp_layout = Layout::contiguous((p.c_in, spatial_size)); - - // Perform matmul: [c_out, c_in] @ [c_in, spatial_size] -> [c_out, spatial_size] - let matmul = MatMul((1, p.c_out, spatial_size, p.c_in)); - let result = matmul.f(&k_reshaped, &k_layout, &inp_reshaped, &inp_layout)?; - - // Copy result to output - let out_offset = b_idx * p.c_out * spatial_size; - for (i, r) in result.iter().enumerate() { - unsafe { - let ptr = dst.as_ptr().add(out_offset + i) as *mut T; - *ptr = *r; - } - } - Ok::<(), crate::Error>(()) - })?; - - Ok(dst) -} - -/// General tiled convolution implementation using gemm. -/// -/// Similar to full im2col, but instead of materializing the full matrix, we process input/output in tiles, in parallel. -fn conv2d_tiled( - p: &ParamsConv2D, - inp: &[T], - inp_l: &Layout, - k: &[T], - k_l: &Layout, -) -> Result> { - let inp = &inp[inp_l.start_offset()..]; - let (inp_s0, inp_s1, inp_s2, inp_s3) = dims4(inp_l.stride())?; - let k = &k[k_l.start_offset()..]; - let (k_s0, k_s1, k_s2, k_s3) = dims4(k_l.stride())?; - let (out_h, out_w) = (p.out_h(), p.out_w()); - - // Output shape: [b_size, c_out, out_h, out_w]. - let dst = vec![T::zero(); p.b_size * p.c_out * out_h * out_w]; - - // Make contiguous input copy if needed. - let cont_s0 = p.i_h * p.i_w * p.c_in; - let cont_s1 = p.i_w * p.c_in; - let cont_s2 = p.c_in; - let layout_is_valid = inp_l.stride() == [cont_s0, cont_s1, cont_s2, 1]; - let inp_cont: Cow<[T]> = if layout_is_valid { - Cow::Borrowed(inp) - } else { - let mut inp_cont = vec![T::zero(); p.b_size * p.c_in * p.i_h * p.i_w]; - for b_idx in 0..p.b_size { - for h_idx in 0..p.i_h { - for w_idx in 0..p.i_w { - for c_idx in 0..p.c_in { - let src_idx = - b_idx * inp_s0 + c_idx * inp_s1 + h_idx * inp_s2 + w_idx * inp_s3; - let dst_idx = b_idx * cont_s0 + h_idx * cont_s1 + w_idx * cont_s2 + c_idx; - inp_cont[dst_idx] = inp[src_idx] - } - } - } - } - Cow::Owned(inp_cont) - }; - - // shape of k: [c_out, c_in, k_h, k_w] - // strides of k: [k_s0, k_s1, k_s2, k_s3] - // For matmul, we need flattened k in shape [c_out, k_h * k_w * c_in] - // with stride [k_h * k_w * c_in, 1] - let k_size = p.c_in * p.k_h * p.k_w; - let mut k_flat = Vec::with_capacity(p.c_out * k_size); - for dst_c_idx in 0..p.c_out { - for kh in 0..p.k_h { - for kw in 0..p.k_w { - for c_in_idx in 0..p.c_in { - let k_idx = dst_c_idx * k_s0 + c_in_idx * k_s1 + kh * k_s2 + kw * k_s3; - k_flat.push(k[k_idx]); - } - } - } - } - // k_layout: [c_out, k_size] with stride [k_size, 1] - let k_layout = Layout::contiguous((p.c_out, k_size)); - - // TILE_SIZE is number of output pixels (out_h * out_w) per tile. - // Higher tile size can be faster due to better usage of gemm, - // but lower tile sizes enable bigger parallelism across tiles. - // This parameter is impactful and may be dynamic or even runtime tunable in the future. - const TILE_SIZE: usize = 512; - - let total_out_pixels = out_h * out_w; - - // Process batches and tiles in parallel using rayon. - (0..p.b_size).into_par_iter().try_for_each(|b_idx| { - let inp_offset = b_idx * cont_s0; - let out_batch_offset = b_idx * (p.c_out * out_h * out_w); - - let num_tiles = total_out_pixels.div_ceil(TILE_SIZE); - (0..num_tiles).into_par_iter().try_for_each(|tile_idx| { - // Determine actual tile size (may be smaller at the end) { - let tile_start = tile_idx * TILE_SIZE; - let tile_end = (tile_start + TILE_SIZE).min(total_out_pixels); - let tile_size = tile_end - tile_start; - - // Precompute output coordinates. - // Used in both im2col extraction and writing output. - let out_coords: Vec<_> = (tile_start..tile_end) - .map(|idx| (idx / out_w, idx % out_w)) - .collect(); - - // Build im2col tile: [k_size, tile_size] - // This represents the input patches needed for this tile of outputs - let mut col_tile = vec![T::zero(); k_size * tile_size]; - - for (tile_idx, (out_y, out_x)) in out_coords.iter().enumerate() { - // Extract the im2col patch for this output position - for c_in in 0..p.c_in { - let mut patch_offset = c_in; - for kh in 0..p.k_h { - let in_y = - (out_y * p.stride + kh * p.dilation) as isize - p.padding as isize; - if in_y < 0 || in_y >= p.i_h as isize { - // Padding: already zero - patch_offset += p.c_in * p.k_w; - continue; - } - for kw in 0..p.k_w { - let in_x = - (out_x * p.stride + kw * p.dilation) as isize - p.padding as isize; - - if in_x >= 0 && in_x < p.i_w as isize { - let in_y = in_y as usize; - let in_x = in_x as usize; - let inp_idx = inp_offset + in_y * cont_s1 + in_x * cont_s2 + c_in; - let col_idx = patch_offset * tile_size + tile_idx; - col_tile[col_idx] = inp_cont[inp_idx]; - } - // Move to next position (skip c_in channels) - patch_offset += p.c_in; - } - } - } - } - - // Now perform matmul: k_cache [c_out, k_size] @ col_tile [k_size, tile_size] - let matmul = MatMul((1, p.c_out, tile_size, k_size)); - - // Layouts for matmul - // k_flat layout: [c_out, k_size] with stride [k_size, 1] - // col_tile layout: [k_size, tile_size] with stride [tile_size, 1] - let col_layout = Layout::contiguous((k_size, tile_size)); - - // Perform matmul - let result = matmul.f(&k_flat, &k_layout, &col_tile, &col_layout)?; - - // Copy results to output: result is [c_out, tile_size] - for (tile_idx, (out_y, out_x)) in out_coords.iter().enumerate() { - let dst_base = out_batch_offset + out_y * out_w + out_x; - - for c_out_idx in 0..p.c_out { - let dst_idx = dst_base + c_out_idx * (out_h * out_w); - let result_idx = c_out_idx * tile_size + tile_idx; - // SAFETY: Each batch processes a distinct region of the output buffer. - // Within each batch, tiles process non-overlapping output positions. - // Therefore, no two threads will write to the same dst_idx. - unsafe { - let ptr = dst.as_ptr().add(dst_idx) as *mut T; - *ptr = result[result_idx]; - } - } - } - Ok::<(), crate::Error>(()) - }) - })?; - - Ok(dst) -} - -/// General direct convolution impl. Decently fast for small inputs and kernels, but loses to full/tiled gemm. -fn conv2d_direct( - p: &ParamsConv2D, - inp: &[T], - inp_l: &Layout, - k: &[T], - k_l: &Layout, -) -> Result> { - let inp = &inp[inp_l.start_offset()..]; - let (inp_s0, inp_s1, inp_s2, inp_s3) = crate::shape::dims4(inp_l.stride())?; - let k = &k[k_l.start_offset()..]; - let (k_s0, k_s1, k_s2, k_s3) = crate::shape::dims4(k_l.stride())?; - let (out_h, out_w) = (p.out_h(), p.out_w()); - - // Output shape: [b_size, c_out, out_h, out_w]. - let dst = vec![T::zero(); p.b_size * p.c_out * out_h * out_w]; - - // Make contiguous input copy if needed. - let cont_s0 = p.i_h * p.i_w * p.c_in; - let cont_s1 = p.i_w * p.c_in; - let cont_s2 = p.c_in; - let layout_is_valid = inp_l.stride() == [cont_s0, cont_s1, cont_s2, 1]; - let inp_cont: Cow<[T]> = if layout_is_valid { - Cow::Borrowed(inp) - } else { - let mut inp_cont = vec![T::zero(); p.b_size * p.c_in * p.i_h * p.i_w]; - for b_idx in 0..p.b_size { - for h_idx in 0..p.i_h { - for w_idx in 0..p.i_w { - for c_idx in 0..p.c_in { - let src_idx = - b_idx * inp_s0 + c_idx * inp_s1 + h_idx * inp_s2 + w_idx * inp_s3; - let dst_idx = b_idx * cont_s0 + h_idx * cont_s1 + w_idx * cont_s2 + c_idx; - inp_cont[dst_idx] = inp[src_idx] - } - } - } - } - Cow::Owned(inp_cont) - }; - let inp_cont_len = inp_cont.len(); - - let k_cache: Vec> = (0..p.c_out) - .map(|dst_c_idx| { - (0..p.k_h * p.k_w) - .flat_map(|kw_kh| { - let offset_h = kw_kh / p.k_w; - let offset_w = kw_kh % p.k_w; - (0..p.c_in).map(move |c_in_idx| { - k[dst_c_idx * k_s0 + c_in_idx * k_s1 + offset_h * k_s2 + offset_w * k_s3] - }) - }) - .collect() - }) - .collect(); - - for b_idx in 0..p.b_size { - for offset_h in 0..p.k_h { - for offset_w in 0..p.k_w { - let k_offset = offset_h * p.k_w + offset_w; - - (0..p.c_out).into_par_iter().for_each(|dst_c_idx| { - let k_cont = &k_cache[dst_c_idx][k_offset * p.c_in..(k_offset + 1) * p.c_in]; - let base_dst_idx = dst_c_idx * out_w * out_h; - let batch_dst_idx = base_dst_idx + b_idx * p.c_out * out_h * out_w; - let batch_src_idx = b_idx * cont_s0; - - for dst_h in 0..out_h { - let src_h = p.stride * dst_h + offset_h * p.dilation; - if src_h < p.padding || src_h >= p.i_h + p.padding { - continue; - } - let src_h = src_h - p.padding; - let h_dst_idx = batch_dst_idx + dst_h * out_w; - let h_src_idx = batch_src_idx + src_h * cont_s1; - - for dst_w in 0..out_w { - let src_w = p.stride * dst_w + offset_w * p.dilation; - if src_w < p.padding || src_w >= p.i_w + p.padding { - continue; - } - let src_w = src_w - p.padding; - let dst_idx = h_dst_idx + dst_w; - let inp_idx_1 = h_src_idx + src_w * cont_s2; - let inp_idx_2 = (inp_idx_1 + p.c_in).min(inp_cont_len); - let inp_cont = &inp_cont[inp_idx_1..inp_idx_2]; - let mut d = T::zero(); - unsafe { - T::vec_dot(inp_cont.as_ptr(), k_cont.as_ptr(), &mut d, p.c_in); - let ptr = dst.as_ptr().add(dst_idx) as *mut T; - *ptr += d; - } - } - } - }); - } - } - } - - Ok(dst) -} - -#[allow(clippy::uninit_vec)] -fn alloc_uninit_vec(size: usize) -> Vec { - let mut v = Vec::with_capacity(size); - unsafe { v.set_len(size) }; - v -} - -/// Full im2col + gemm convolution implementation. -/// -/// For large inputs im2col and copy_strided_src for output gets expensive. -fn conv2d_im2col_gemm( - p: &ParamsConv2D, - inp: &[T], - inp_l: &Layout, - kernel: &[T], - kernel_l: &Layout, -) -> Result> { - let op = Im2Col { - h_k: p.k_h, - w_k: p.k_w, - padding: p.padding, - stride: p.stride, - dilation: p.dilation, - }; - let col = op.f(inp, inp_l)?; - let b = p.b_size; - let n = p.c_out; - let (h_out, w_out) = (p.out_h(), p.out_w()); - let k = op.h_k * op.w_k * p.c_in; - let m = h_out * w_out; - let col_l = Layout::contiguous((b, m, k)); - let res: Vec = if kernel_l.is_contiguous() { - let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) - .transpose(1, 2)? - .broadcast_as((b, k, n))?; - MatMul((b, m, n, k)).f(&col, &col_l, kernel, &kernel_l)? - } else { - // Make the kernel contiguous if not already the case. - let mut kernel_c = alloc_uninit_vec(kernel_l.shape().elem_count()); - copy_strided_src_(kernel, &mut kernel_c, 0, kernel_l); - let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) - .transpose(1, 2)? - .broadcast_as((b, k, n))?; - MatMul((b, m, n, k)).f(&col, &col_l, &kernel_c, &kernel_l)? - }; - let res_l = Layout::contiguous((b, h_out, w_out, p.c_out)) - .transpose(1, 2)? - .transpose(1, 3)?; - let mut res_t = alloc_uninit_vec(res_l.shape().elem_count()); - copy_strided_src_(&res, &mut res_t, 0, &res_l); - Ok(res_t) -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/mod.rs b/v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/mod.rs deleted file mode 100644 index afb93024ac..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/mod.rs +++ /dev/null @@ -1,3284 +0,0 @@ -//! Implementation of Backend Fns for CPU -use crate::backend::{BackendDevice, BackendStorage}; -use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; -use crate::{DType, Error, IntDType, Layout, Result, Shape, WithDType}; -use float8::F8E4M3; -use half::{bf16, f16}; -use rayon::prelude::*; - -mod utils; -pub use utils::{ - binary_map, binary_map_vec, unary_map, unary_map_vec, Map1, Map1Any, Map2, Map2InPlace, Map2U8, -}; -mod conv2d; -use conv2d::Conv2D; - -const USE_IM2COL_CONV1D: bool = true; -const USE_COL2IM_CONV1D_TR: bool = true; - -// TODO: Maybe we should not implement [Clone] here and instead have an explicit allocator + -// intercept the oom errors to avoid panicking and provide a proper error. -#[derive(Debug, Clone)] -pub enum CpuStorage { - U8(Vec), - U32(Vec), - I16(Vec), - I32(Vec), - I64(Vec), - BF16(Vec), - F16(Vec), - F32(Vec), - F64(Vec), - F8E4M3(Vec), - // Dummy types that store raw bytes - F6E2M3(Vec), - F6E3M2(Vec), - F4(Vec), - F8E8M0(Vec), -} - -#[derive(Debug, Clone)] -pub enum CpuStorageRef<'a> { - U8(&'a [u8]), - U32(&'a [u32]), - I16(&'a [i16]), - I32(&'a [i32]), - I64(&'a [i64]), - BF16(&'a [bf16]), - F16(&'a [f16]), - F32(&'a [f32]), - F64(&'a [f64]), - F8E4M3(&'a [F8E4M3]), - // Dummy types that store raw bytes - F6E2M3(&'a [u8]), - F6E3M2(&'a [u8]), - F4(&'a [u8]), - F8E8M0(&'a [u8]), -} - -#[derive(Debug, Clone)] -pub struct CpuDevice; - -struct Cmp(CmpOp); -impl Map2U8 for Cmp { - const OP: &'static str = "cmp"; - #[inline(always)] - fn f( - &self, - lhs: &[T], - lhs_l: &Layout, - rhs: &[T], - rhs_l: &Layout, - ) -> Result> { - let dst = match self.0 { - CmpOp::Eq => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x == y)), - CmpOp::Ne => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x != y)), - CmpOp::Lt => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x < y)), - CmpOp::Le => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x <= y)), - CmpOp::Gt => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x > y)), - CmpOp::Ge => binary_map(lhs_l, rhs_l, lhs, rhs, |x, y| u8::from(x >= y)), - }; - Ok(dst) - } -} - -struct WCond<'a, T: IntDType>(&'a [T], &'a Layout); - -impl Map2 for WCond<'_, I> { - const OP: &'static str = "where"; - #[inline(always)] - fn f(&self, t: &[T], t_l: &Layout, f: &[T], f_l: &Layout) -> Result> { - let vs = match ( - self.1.contiguous_offsets(), - t_l.contiguous_offsets(), - f_l.contiguous_offsets(), - ) { - (Some((o1, o2)), Some((o_t1, o_t2)), Some((o_f1, o_f2))) => { - let pred = &self.0[o1..o2]; - let t = &t[o_t1..o_t2]; - let f = &f[o_f1..o_f2]; - pred.iter() - .zip(t.iter().zip(f.iter())) - .map(|(p, (&t, &f))| if p.is_true() { t } else { f }) - .collect::>() - } - _ => self - .1 - .strided_index() - .zip(t_l.strided_index().zip(f_l.strided_index())) - .map(|(i_p, (i_t, i_f))| { - if self.0[i_p].is_true() { - t[i_t] - } else { - f[i_f] - } - }) - .collect::>(), - }; - Ok(vs) - } -} - -struct ReduceIndex { - reduce_dim_index: usize, - use_min: bool, - return_index: bool, -} - -impl ReduceIndex { - // The value gets replaced if f(s[current_acc], s[i]) returns true. - #[inline(always)] - fn fold_impl(&self, src: &[T], src_l: &Layout, f: F, g: G) -> Result> - where - T: Clone + Copy, - U: Clone + Copy, - F: Fn(T, T) -> bool, - G: Fn(T, usize) -> U, - { - let reduce_dim_size = src_l.dims()[self.reduce_dim_index]; - let reduce_dim_stride = src_l.stride()[self.reduce_dim_index]; - let dst_len = src_l.shape().elem_count() / reduce_dim_size; - let mut dst: Vec = Vec::with_capacity(dst_len); - let dst_to_set = dst.spare_capacity_mut(); - let dst_to_set = - unsafe { std::mem::transmute::<&mut [std::mem::MaybeUninit], &mut [U]>(dst_to_set) }; - match src_l.contiguous_offsets() { - Some((o1, o2)) => { - let src = &src[o1..o2]; - if reduce_dim_stride == 1 { - for (start_src_i, dst_v) in dst_to_set.iter_mut().enumerate() { - let start_src_i = start_src_i * reduce_dim_size; - let src = &src[start_src_i..start_src_i + reduce_dim_size]; - let mut acc = 0; - let mut val = src[0]; - for (src_i, &s) in src.iter().enumerate() { - if f(val, s) { - acc = src_i; - val = s - } - } - *dst_v = g(val, acc) - } - } else { - for (start_src_i, dst_v) in dst_to_set.iter_mut().enumerate() { - let (p, q) = ( - start_src_i / reduce_dim_stride, - start_src_i % reduce_dim_stride, - ); - // start_src_i = p * reduce_dim_stride + q - let start_src_i = p * reduce_dim_stride * reduce_dim_size + q; - let src = &src[start_src_i..]; - let mut acc = 0; - let mut val = src[0]; - for src_i in 0..reduce_dim_size { - let s = src[src_i * reduce_dim_stride]; - if f(val, s) { - acc = src_i; - val = s - } - } - *dst_v = g(val, acc) - } - } - } - None => { - let l = src_l.narrow(self.reduce_dim_index, 0, 1)?; - for (unstr_index, src_index) in l.strided_index().enumerate() { - let src = &src[src_index..]; - let mut acc = 0; - let mut val = src[0]; - for src_i in 0..reduce_dim_size { - let s = src[src_i * reduce_dim_stride]; - if f(val, s) { - acc = src_i; - val = s - } - } - dst_to_set[unstr_index] = g(val, acc) - } - } - } - unsafe { dst.set_len(dst_len) }; - Ok(dst) - } -} - -impl Map1Any for ReduceIndex { - #[inline(always)] - fn f) -> CpuStorage>( - &self, - src: &[T], - src_l: &Layout, - wrap: W, - ) -> Result { - if src_l.shape().elem_count() == 0 { - Err(Error::EmptyTensor { op: "reduce" }.bt())? - } - let dst = match (self.return_index, self.use_min) { - (false, true) => wrap(self.fold_impl(src, src_l, |x, y| x > y, |v, _i| v)?), - (false, false) => wrap(self.fold_impl(src, src_l, |x, y| x < y, |v, _i| v)?), - (true, true) => { - CpuStorage::U32(self.fold_impl(src, src_l, |x, y| x > y, |_v, i| i as u32)?) - } - (true, false) => { - CpuStorage::U32(self.fold_impl(src, src_l, |x, y| x < y, |_v, i| i as u32)?) - } - }; - Ok(dst) - } -} - -struct ReduceSum<'a> { - dst_shape: &'a Shape, - reduce_dims: &'a [usize], - reduce_dims_and_stride: Vec<(usize, usize)>, -} - -impl ReduceSum<'_> { - #[inline(always)] - fn fold_impl(&self, src: &[T], src_l: &Layout, start_elt: T) -> Result> - where - T: WithDType, - { - let mut dst = vec![start_elt; self.dst_shape.elem_count()]; - match src_l.contiguous_offsets() { - Some((o1, o2)) => { - let src = &src[o1..o2]; - // Handle the case where we reduce over the last dimensions separately as it is - // fairly common and easy to optimize. This rely on the layout being contiguous! - // reduce_dims is sorted, check if it is ranging from a to n-1. - let reduce_over_last_dims = self - .reduce_dims - .iter() - .rev() - .enumerate() - .all(|(i, &v)| v == src_l.shape().rank() - 1 - i); - if reduce_over_last_dims { - let reduce_sz = self - .reduce_dims_and_stride - .iter() - .map(|(u, _)| u) - .product::(); - for (dst_i, dst_v) in dst.iter_mut().enumerate() { - let src_i = dst_i * reduce_sz; - unsafe { - T::vec_reduce_sum( - src[src_i..src_i + reduce_sz].as_ptr(), - dst_v, - reduce_sz, - ) - }; - } - return Ok(dst); - }; - for (unstr_index, &src) in src.iter().enumerate() { - let mut dst_index = unstr_index; - // Set the reduce_dims indexes to 0. - for &(dim, stride) in self.reduce_dims_and_stride.iter() { - // The compiler is able to optimize the following in a single divmod op. - let (pre, post) = (dst_index / stride, dst_index % stride); - dst_index = (pre / dim) * stride + post; - } - dst[dst_index] += src; - } - } - None => { - for (unstr_index, src_index) in src_l.strided_index().enumerate() { - let mut dst_index = unstr_index; - // Set the reduce_dims indexes to 0. - for &(dim, stride) in self.reduce_dims_and_stride.iter() { - // The compiler is able to optimize the following in a single divmod op. - let (pre, post) = (dst_index / stride, dst_index % stride); - dst_index = (pre / dim) * stride + post; - } - dst[dst_index] += src[src_index]; - } - } - } - Ok(dst) - } -} - -impl Map1 for ReduceSum<'_> { - #[inline(always)] - fn f(&self, src: &[T], src_l: &Layout) -> Result> { - self.fold_impl(src, src_l, T::zero()) - } -} - -struct Affine(f64, f64); - -impl Map1 for Affine { - fn f(&self, vs: &[T], layout: &Layout) -> Result> { - let mul = T::from_f64(self.0); - let add = T::from_f64(self.1); - Ok(unary_map(vs, layout, |v| v * mul + add)) - } -} - -struct AvgPool2D((usize, usize), (usize, usize)); - -impl Map1 for AvgPool2D { - fn f(&self, src: &[T], layout: &Layout) -> Result> { - // https://pytorch.org/docs/stable/generated/torch.nn.AvgPool2d.html - let (k_h, k_w) = self.0; - let (s_h, s_w) = self.1; - let (b_sz, c, h, w) = layout.shape().dims4()?; - let stride = layout.stride(); - let (stride_h, stride_w) = (stride[2], stride[3]); - let h_out = (h - k_h) / s_h + 1; - let w_out = (w - k_w) / s_w + 1; - let src_index = layout.start_offset(); - let mut dst = vec![T::zero(); b_sz * c * h_out * w_out]; - let scale = 1f64 / (k_h * k_w) as f64; - let scale = T::from_f64(scale); - for b_idx in 0..b_sz { - let dst = &mut dst[b_idx * c * h_out * w_out..]; - let src_index = src_index + b_idx * stride[0]; - for c_idx in 0..c { - let dst = &mut dst[c_idx * h_out * w_out..]; - let src_index = src_index + c_idx * stride[1]; - for h_idx in 0..h_out { - for w_idx in 0..w_out { - let mut sum = T::zero(); - for m in 0..k_h { - for n in 0..k_w { - let m = s_h * h_idx + m; - let n = s_w * w_idx + n; - sum += src[src_index + m * stride_h + n * stride_w] - } - } - dst[h_idx * w_out + w_idx] = sum * scale; - } - } - } - } - Ok(dst) - } -} - -struct MaxPool2D((usize, usize), (usize, usize)); - -impl Map1 for MaxPool2D { - fn f(&self, src: &[T], layout: &Layout) -> Result> { - // https://pytorch.org/docs/stable/generated/torch.nn.MaxPool2d.html - let (k_h, k_w) = self.0; - let (s_h, s_w) = self.1; - let (b_sz, c, h, w) = layout.shape().dims4()?; - let stride = layout.stride(); - let (stride_h, stride_w) = (stride[2], stride[3]); - let h_out = (h - k_h) / s_h + 1; - let w_out = (w - k_w) / s_w + 1; - let src_index = layout.start_offset(); - let mut dst = vec![T::zero(); b_sz * c * h_out * w_out]; - for b_idx in 0..b_sz { - let dst = &mut dst[b_idx * c * h_out * w_out..]; - let src_index = src_index + b_idx * stride[0]; - for c_idx in 0..c { - let dst = &mut dst[c_idx * h_out * w_out..]; - let src_index = src_index + c_idx * stride[1]; - for h_idx in 0..h_out { - for w_idx in 0..w_out { - let mut largest = - src[src_index + s_h * h_idx * stride_h + s_w * w_idx * stride_w]; - for m in 0..k_h { - for n in 0..k_w { - let m = s_h * h_idx + m; - let n = s_w * w_idx + n; - if largest < src[src_index + m * stride_h + n * stride_w] { - largest = src[src_index + m * stride_h + n * stride_w] - } - } - } - dst[h_idx * w_out + w_idx] = largest; - } - } - } - } - Ok(dst) - } -} - -struct UpsampleNearest1D(usize); - -impl Map1 for UpsampleNearest1D { - fn f(&self, src: &[T], layout: &Layout) -> Result> { - // TODO: Specialized implementation for the case 2*sz? - let dst_sz = self.0; - let (b_sz, c, src_sz) = layout.shape().dims3()?; - let stride = layout.stride(); - let stride_sz = stride[2]; - let src_index = layout.start_offset(); - let scale_sz = src_sz as f64 / dst_sz as f64; - let mut dst = vec![T::zero(); b_sz * c * dst_sz]; - let src_idxs = (0..dst_sz) - .map(|idx| usize::min(src_sz - 1, (idx as f64 * scale_sz) as usize)) - .collect::>(); - for b_idx in 0..b_sz { - let dst = &mut dst[b_idx * c * dst_sz..]; - let src_index = src_index + b_idx * stride[0]; - for c_idx in 0..c { - let dst = &mut dst[c_idx * dst_sz..]; - let src_index = src_index + c_idx * stride[1]; - for (idx, src_idx) in src_idxs.iter().enumerate() { - dst[idx] = src[src_index + src_idx * stride_sz] - } - } - } - Ok(dst) - } -} - -struct UpsampleNearest2D(usize, usize); - -impl Map1 for UpsampleNearest2D { - fn f(&self, src: &[T], layout: &Layout) -> Result> { - // TODO: Specialized implementation for the case 2*h, 2*w? - let (dst_h, dst_w) = (self.0, self.1); - let (b_sz, c, src_h, src_w) = layout.shape().dims4()?; - let stride = layout.stride(); - let (stride_h, stride_w) = (stride[2], stride[3]); - let src_index = layout.start_offset(); - let scale_h = src_h as f64 / dst_h as f64; - let scale_w = src_w as f64 / dst_w as f64; - let mut dst = vec![T::zero(); b_sz * c * dst_h * dst_w]; - let src_h_idxs = (0..dst_h) - .map(|h_idx| usize::min(src_h - 1, (h_idx as f64 * scale_h) as usize)) - .collect::>(); - let src_w_idxs = (0..dst_w) - .map(|w_idx| usize::min(src_w - 1, (w_idx as f64 * scale_w) as usize)) - .collect::>(); - for b_idx in 0..b_sz { - let dst = &mut dst[b_idx * c * dst_h * dst_w..]; - let src_index = src_index + b_idx * stride[0]; - for c_idx in 0..c { - let dst = &mut dst[c_idx * dst_h * dst_w..]; - let src_index = src_index + c_idx * stride[1]; - for (h_idx, src_h_idx) in src_h_idxs.iter().enumerate() { - for (w_idx, src_w_idx) in src_w_idxs.iter().enumerate() { - let src_index = src_index + src_h_idx * stride_h + src_w_idx * stride_w; - dst[h_idx * dst_w + w_idx] = src[src_index] - } - } - } - } - Ok(dst) - } -} - -struct UpsampleBilinear2D { - target_h: usize, - target_w: usize, - align_corners: bool, - scale_h_factor: Option, - scale_w_factor: Option, -} - -impl Map1 for UpsampleBilinear2D { - fn f(&self, src: &[T], layout: &Layout) -> Result> { - let (batch, channels, height_in, width_in) = layout.shape().dims4()?; - let height_out = self.target_h; - let width_out = self.target_w; - - // Early return for identity case - if height_in == height_out && width_in == width_out { - return Ok(src.to_vec()); - } - - let stride = layout.stride(); - let src_offset = layout.start_offset(); - - // Calculate scale factors following PyTorch's area_pixel_compute_scale logic - let scale_h = if self.align_corners { - if height_out > 1 { - (height_in - 1) as f64 / (height_out - 1) as f64 - } else { - 0.0 - } - } else { - // PyTorch's compute_scales_value logic: - // If scale_factor was provided, use 1.0 / scale_factor - // Otherwise, use input_size / output_size - if let Some(scale_factor) = self.scale_h_factor { - 1.0 / scale_factor - } else { - height_in as f64 / height_out as f64 - } - }; - - let scale_w = if self.align_corners { - if width_out > 1 { - (width_in - 1) as f64 / (width_out - 1) as f64 - } else { - 0.0 - } - } else if let Some(scale_factor) = self.scale_w_factor { - 1.0 / scale_factor - } else { - width_in as f64 / width_out as f64 - }; - - // Precompute indices and weights for height - let mut h_indices = Vec::with_capacity(height_out); - for h_out in 0..height_out { - let src_h = if self.align_corners { - scale_h * h_out as f64 - } else { - scale_h * (h_out as f64 + 0.5) - 0.5 - }; - let src_h_clamped = src_h.max(0.0); - let h0 = src_h_clamped.floor() as usize; - let h1 = (h0 + 1).min(height_in - 1); - let weight_h = (src_h_clamped - h0 as f64).clamp(0.0, 1.0); - h_indices.push((h0, h1, weight_h)); - } - - // Precompute indices and weights for width - let mut w_indices = Vec::with_capacity(width_out); - for w_out in 0..width_out { - let src_w = if self.align_corners { - scale_w * w_out as f64 - } else { - scale_w * (w_out as f64 + 0.5) - 0.5 - }; - let src_w_clamped = src_w.max(0.0); - let w0 = src_w_clamped.floor() as usize; - let w1 = (w0 + 1).min(width_in - 1); - let weight_w = (src_w_clamped - w0 as f64).clamp(0.0, 1.0); - w_indices.push((w0, w1, weight_w)); - } - - // Allocate output - let mut dst = vec![T::zero(); batch * channels * height_out * width_out]; - - // Perform bilinear interpolation - for b in 0..batch { - for c in 0..channels { - let base_idx = src_offset + b * stride[0] + c * stride[1]; - let dst_base = (b * channels + c) * height_out * width_out; - - for (h_out, &(h0, h1, weight_h)) in h_indices.iter().enumerate() { - for (w_out, &(w0, w1, weight_w)) in w_indices.iter().enumerate() { - // Get four neighboring pixels - let idx_00 = base_idx + h0 * stride[2] + w0 * stride[3]; - let idx_10 = base_idx + h0 * stride[2] + w1 * stride[3]; - let idx_01 = base_idx + h1 * stride[2] + w0 * stride[3]; - let idx_11 = base_idx + h1 * stride[2] + w1 * stride[3]; - - let v00 = src[idx_00].to_f64(); - let v10 = src[idx_10].to_f64(); - let v01 = src[idx_01].to_f64(); - let v11 = src[idx_11].to_f64(); - - // Bilinear interpolation - let v_top = v00 * (1.0 - weight_w) + v10 * weight_w; - let v_bottom = v01 * (1.0 - weight_w) + v11 * weight_w; - let value = v_top * (1.0 - weight_h) + v_bottom * weight_h; - - dst[dst_base + h_out * width_out + w_out] = T::from_f64(value); - } - } - } - } - - Ok(dst) - } -} - -struct Gather<'a, I: IntDType> { - ids: &'a [I], - ids_l: &'a Layout, - dim: usize, -} - -impl Map1 for Gather<'_, I> { - fn f(&self, src: &[T], src_l: &Layout) -> Result> { - let ids = match self.ids_l.contiguous_offsets() { - Some((a, b)) => &self.ids[a..b], - None => Err(Error::RequiresContiguous { op: "gather" }.bt())?, - }; - let src = match src_l.contiguous_offsets() { - Some((a, b)) => &src[a..b], - None => Err(Error::RequiresContiguous { op: "gather" }.bt())?, - }; - let dim = self.dim; - let ids_dims = self.ids_l.dims(); - let src_dims = src_l.dims(); - let dst_len: usize = ids_dims.iter().product(); - let dst_left_len: usize = ids_dims[..dim].iter().product(); - let dst_dim_len = ids_dims[dim]; - let dst_right_len: usize = ids_dims[dim + 1..].iter().product(); - - let src_dim_len = src_dims[dim]; - let src_right_len: usize = src_dims[dim + 1..].iter().product(); - - let mut dst = vec![T::zero(); dst_len]; - for left_i in 0..dst_left_len { - let start_src_idx = left_i * src_right_len * src_dim_len; - let start_dst_idx = left_i * dst_right_len * dst_dim_len; - for i in 0..dst_dim_len { - let start_dst_idx = start_dst_idx + i * dst_right_len; - for right_i in 0..dst_right_len { - let dst_idx = start_dst_idx + right_i; - let index = ids[dst_idx]; - if index == I::max_value() { - dst[dst_idx] = T::zero(); - } else { - let index = index.as_usize(); - if index >= src_dim_len { - Err(Error::InvalidIndex { - index, - size: src_dim_len, - op: "gather", - } - .bt())? - } - let src_idx = start_src_idx + index * src_right_len + right_i; - dst[dst_idx] = src[src_idx] - } - } - } - } - Ok(dst) - } -} - -struct IndexSelect<'a, T: IntDType> { - ids: &'a [T], - ids_l: &'a Layout, - dim: usize, -} - -impl Map1 for IndexSelect<'_, I> { - fn f(&self, src: &[T], layout: &Layout) -> Result> { - let src = match layout.contiguous_offsets() { - Some((a, b)) => &src[a..b], - None => Err(Error::RequiresContiguous { op: "index-select" }.bt())?, - }; - let dim = self.dim; - let n_ids = match self.ids_l.dims() { - [n_ids] => *n_ids, - d => Err(Error::UnexpectedNumberOfDims { - expected: 1, - got: d.len(), - shape: self.ids_l.shape().clone(), - } - .bt())?, - }; - let stride_ids = self.ids_l.stride()[0]; - let mut dst_dims = layout.dims().to_vec(); - let src_dim = dst_dims[dim]; - dst_dims[dim] = n_ids; - let dst_len: usize = dst_dims.iter().product(); - let left_len: usize = dst_dims[..dim].iter().product(); - let right_len: usize = dst_dims[dim + 1..].iter().product(); - let mut dst = vec![T::zero(); dst_len]; - for left_i in 0..left_len { - let start_src_idx = left_i * right_len * src_dim; - let start_dst_idx = left_i * right_len * n_ids; - for i in 0..n_ids { - let start_dst_idx = start_dst_idx + i * right_len; - let index = self.ids[self.ids_l.start_offset() + stride_ids * i]; - if index == I::max_value() { - dst[start_dst_idx..start_dst_idx + right_len].fill(T::zero()); - } else { - let index = index.as_usize(); - if index >= src_dim { - Err(Error::InvalidIndex { - index, - size: src_dim, - op: "index-select", - } - .bt())? - } - let start_src_idx = start_src_idx + index * right_len; - dst[start_dst_idx..start_dst_idx + right_len] - .copy_from_slice(&src[start_src_idx..start_src_idx + right_len]) - } - } - } - Ok(dst) - } -} - -trait ElemUpdate { - fn f(dst: &mut T, src: T); -} - -struct Set; -struct Add; - -impl ElemUpdate for Set { - fn f(dst: &mut T, src: T) { - *dst = src - } -} - -impl ElemUpdate for Add { - fn f(dst: &mut T, src: T) { - *dst += src - } -} - -struct Scatter<'a, I: IntDType, M: ElemUpdate> { - ids: &'a [I], - ids_l: &'a Layout, - dim: usize, - _phantom: std::marker::PhantomData, -} - -impl<'a, I: IntDType, M: ElemUpdate> Scatter<'a, I, M> { - fn new(ids: &'a [I], ids_l: &'a Layout, dim: usize) -> Self { - Self { - ids, - ids_l, - dim, - _phantom: Default::default(), - } - } -} - -impl Map2InPlace for Scatter<'_, I, M> { - const OP: &'static str = "scatter"; - fn f( - &self, - dst: &mut [T], - dst_l: &Layout, - src: &[T], - src_l: &Layout, - ) -> Result<()> { - let dst = match dst_l.contiguous_offsets() { - None => Err(Error::RequiresContiguous { op: "scatter" }.bt())?, - Some((o1, o2)) => &mut dst[o1..o2], - }; - - let src = match src_l.contiguous_offsets() { - None => Err(Error::RequiresContiguous { op: "scatter" }.bt())?, - Some((o1, o2)) => &src[o1..o2], - }; - - let dim = self.dim; - let ids_dims = self.ids_l.dims(); - let dst_dims = dst_l.dims(); - let dst_dim_len = dst_dims[dim]; - let dst_right_len: usize = dst_dims[dim + 1..].iter().product(); - - let ids_left_len: usize = ids_dims[..dim].iter().product(); - let ids_dim_len = ids_dims[dim]; - let ids_right_len: usize = ids_dims[dim + 1..].iter().product(); - - let ids = match self.ids_l.contiguous_offsets() { - Some((a, b)) => &self.ids[a..b], - None => Err(Error::RequiresContiguous { op: "gather" }.bt())?, - }; - for left_i in 0..ids_left_len { - let start_ids_idx = left_i * ids_right_len * ids_dim_len; - let start_dst_idx = left_i * dst_right_len * dst_dim_len; - for i in 0..ids_dim_len { - let start_ids_idx = start_ids_idx + i * ids_right_len; - for right_i in 0..dst_right_len { - let ids_idx = start_ids_idx + right_i; - let index = ids[ids_idx]; - if index == I::max_value() { - continue; - } - let index = index.as_usize(); - if index >= dst_dim_len { - Err(Error::InvalidIndex { - index, - size: dst_dim_len, - op: "gather", - } - .bt())? - } - let dst_idx = start_dst_idx + index * dst_right_len + right_i; - M::f(&mut dst[dst_idx], src[ids_idx]) - } - } - } - - Ok(()) - } -} - -struct IndexAdd<'a, I: IntDType> { - ids: &'a [I], - dim: usize, -} - -impl Map2 for IndexAdd<'_, I> { - const OP: &'static str = "index-add"; - // https://pytorch.org/docs/stable/generated/torch.Tensor.index_add_.html#torch.Tensor.index_add_ - // v1, l1 -> self - fn f(&self, v1: &[T], l1: &Layout, src: &[T], src_l: &Layout) -> Result> { - let dst_len = l1.shape().elem_count(); - let mut dst = vec![T::zero(); dst_len]; - copy_strided_src_(v1, &mut dst, 0, l1); - let src = match src_l.contiguous_offsets() { - None => Err(Error::RequiresContiguous { op: "index-add" }.bt())?, - Some((o1, o2)) => &src[o1..o2], - }; - let dim = self.dim; - let max_idx = l1.dims()[dim]; - let pre_dim = src_l.dims()[..dim].iter().product::(); - let src_dim_sz = src_l.dims()[dim]; - let post_dim = src_l.dims()[dim + 1..].iter().product::(); - if dim == 0 { - for (src_idx, dst_idx) in self.ids.iter().enumerate() { - if *dst_idx == I::max_value() { - continue; - } - let dst_idx = dst_idx.as_usize(); - if dst_idx >= max_idx { - Err(Error::InvalidIndex { - index: dst_idx, - op: "index-add", - size: max_idx, - })? - } - let src_idx = src_idx * post_dim; - let dst_idx = dst_idx * post_dim; - let src = &src[src_idx..src_idx + post_dim]; - let dst = &mut dst[dst_idx..dst_idx + post_dim]; - for (d, &s) in dst.iter_mut().zip(src.iter()) { - *d += s - } - } - } else { - for (src_idx, dst_idx) in self.ids.iter().enumerate() { - if *dst_idx == I::max_value() { - continue; - } - let dst_idx = dst_idx.as_usize(); - if dst_idx >= max_idx { - Err(Error::InvalidIndex { - index: dst_idx, - op: "index-add", - size: max_idx, - })? - } - for pre_i in 0..pre_dim { - let pre_src_i = (pre_i * src_dim_sz + src_idx) * post_dim; - let pre_dst_i = (pre_i * max_idx + dst_idx) * post_dim; - let src = &src[pre_src_i..pre_src_i + post_dim]; - let dst = &mut dst[pre_dst_i..pre_dst_i + post_dim]; - for (d, &s) in dst.iter_mut().zip(src.iter()) { - *d += s - } - } - } - } - Ok(dst) - } -} - -#[allow(clippy::too_many_arguments)] -fn copy2d_( - src: &[T], - dst: &mut [T], - d1: usize, - d2: usize, - src_stride1: usize, - dst_stride1: usize, - src_offset: usize, - dst_offset: usize, -) { - for i1 in 0..d1 { - let dst_idx = i1 * dst_stride1 + dst_offset; - let src_idx = i1 * src_stride1 + src_offset; - let dst = &mut dst[dst_idx..dst_idx + d2]; - let src = &src[src_idx..src_idx + d2]; - dst.copy_from_slice(src) - } -} - -fn copy_strided_src_(src: &[T], dst: &mut [T], dst_offset: usize, src_l: &Layout) { - match src_l.strided_blocks() { - crate::StridedBlocks::SingleBlock { start_offset, len } => { - let to_copy = (dst.len() - dst_offset).min(len); - dst[dst_offset..dst_offset + to_copy] - .copy_from_slice(&src[start_offset..start_offset + to_copy]) - } - crate::StridedBlocks::MultipleBlocks { - block_start_index, - block_len: 1, - } => { - for (dst_index, src_index) in block_start_index.enumerate() { - let dst_index = dst_index + dst_offset; - if dst_index >= dst.len() { - break; - } - dst[dst_index] = src[src_index] - } - } - crate::StridedBlocks::MultipleBlocks { - block_start_index, - block_len, - } => { - let mut dst_index = dst_offset; - for src_index in block_start_index { - let next_dst_index = dst_index + block_len; - if dst_index >= dst.len() { - break; - } - let to_copy = usize::min(block_len, dst.len() - dst_index); - dst[dst_index..dst_index + to_copy] - .copy_from_slice(&src[src_index..src_index + to_copy]); - dst_index = next_dst_index - } - } - } -} - -struct Conv1D<'a>(&'a crate::conv::ParamsConv1D); - -impl Map2 for Conv1D<'_> { - const OP: &'static str = "conv1d"; - fn f(&self, inp: &[T], inp_l: &Layout, k: &[T], k_l: &Layout) -> Result> { - let p = self.0; - let inp = &inp[inp_l.start_offset()..]; - let k = &k[k_l.start_offset()..]; - let (inp_s0, inp_s1, inp_s2) = crate::shape::dims3(inp_l.stride())?; - let (k_s0, k_s1, k_s2) = crate::shape::dims3(k_l.stride())?; - let l_out = p.l_out(); - let dst_elems = p.c_out * l_out * p.b_size; - // The output shape is [b_size, c_out, l_out] - let dst = vec![T::zero(); dst_elems]; - - // TODO: Avoid making this copy if `inp` already has the appropriate layout. - let mut inp_cont = vec![T::zero(); p.b_size * p.c_in * p.l_in]; - for b_idx in 0..p.b_size { - for src_l in 0..p.l_in { - for src_c_idx in 0..p.c_in { - let inp_idx = b_idx * inp_s0 + src_c_idx * inp_s1 + src_l * inp_s2; - inp_cont[b_idx * p.l_in * p.c_in + src_l * p.c_in + src_c_idx] = inp[inp_idx] - } - } - } - - for offset in 0..p.k_size { - (0..p.c_out).into_par_iter().for_each(|dst_c_idx| { - let dst_idx = dst_c_idx * l_out; - let k_cont = (0..p.c_in) - .map(|c_in_idx| k[dst_c_idx * k_s0 + c_in_idx * k_s1 + offset * k_s2]) - .collect::>(); - for b_idx in 0..p.b_size { - let dst_idx = dst_idx + b_idx * p.c_out * l_out; - for dst_l in 0..l_out { - let dst_idx = dst_idx + dst_l; - let src_l = p.stride * dst_l + offset * p.dilation; - if src_l < p.padding || src_l >= p.padding + p.l_in { - continue; - } - let src_l = src_l - p.padding; - let inp_cont = &inp_cont[b_idx * p.l_in * p.c_in + src_l * p.c_in..]; - assert!(inp_cont.len() >= p.c_in); - assert!(k_cont.len() >= p.c_in); - let mut d = T::zero(); - unsafe { T::vec_dot(inp_cont.as_ptr(), k_cont.as_ptr(), &mut d, p.c_in) } - let dst_p = dst.as_ptr(); - // Safety: dst_idx are uniques per dst_c_idx which is used to parallelise - // the different tasks so no two threads can try to write at the same - // location. - unsafe { - let ptr = dst_p.add(dst_idx) as *mut T; - *ptr += d - } - } - } - }) - } - Ok(dst) - } -} - -struct Im2Col1D { - l_k: usize, - stride: usize, - dilation: usize, - padding: usize, -} - -impl Im2Col1D { - fn l_out(&self, l: usize) -> usize { - (l + 2 * self.padding - self.dilation * (self.l_k - 1) - 1) / self.stride + 1 - } -} - -impl Map1 for Im2Col1D { - fn f(&self, vs: &[T], layout: &Layout) -> Result> { - let &Self { - l_k, - stride, - dilation, - padding, - } = self; - let (b, c, l) = layout.shape().dims3()?; - let l_out = self.l_out(l); - let src = &vs[layout.start_offset()..]; - let mut dst = vec![T::zero(); b * l_out * c * l_k]; - let (src_s0, src_s1, src_s2) = { - let s = layout.stride(); - (s[0], s[1], s[2]) - }; - // TODO: provide specialized kernels for the common use cases. - // - l_k = 1 - // - padding = 0 - // - stride = 1 - // - dilation = 1 - for b_idx in 0..b { - let src_idx = b_idx * src_s0; - let dst_idx = b_idx * l_out * c * l_k; - for l_idx in 0..l_out { - let dst_idx = dst_idx + l_idx * c * l_k; - for c_idx in 0..c { - let dst_idx = dst_idx + c_idx * l_k; - let src_idx = c_idx * src_s1 + src_idx; - for l_k_idx in 0..l_k { - let src_l = l_idx * stride + l_k_idx * dilation; - if padding != 0 && (src_l < padding || src_l >= l + padding) { - continue; - } - let src_l = src_l - padding; - let src_idx = src_idx + src_l * src_s2; - let dst_idx = dst_idx + l_k_idx; - dst[dst_idx] = src[src_idx] - } - } - } - } - Ok(dst) - } -} - -struct Im2Col { - h_k: usize, - w_k: usize, - stride: usize, - dilation: usize, - padding: usize, -} - -impl Im2Col { - fn hw_out(&self, h: usize, w: usize) -> (usize, usize) { - let h_out = (h + 2 * self.padding - self.dilation * (self.h_k - 1) - 1) / self.stride + 1; - let w_out = (w + 2 * self.padding - self.dilation * (self.w_k - 1) - 1) / self.stride + 1; - (h_out, w_out) - } -} - -impl Map1 for Im2Col { - fn f(&self, vs: &[T], layout: &Layout) -> Result> { - let &Self { - h_k, - w_k, - stride, - dilation, - padding, - } = self; - let (b, c, h, w) = layout.shape().dims4()?; - let (h_out, w_out) = self.hw_out(h, w); - let src = &vs[layout.start_offset()..]; - let mut dst = vec![T::zero(); b * h_out * w_out * c * h_k * w_k]; - let (src_s0, src_s1, src_s2, src_s3) = { - let s = layout.stride(); - (s[0], s[1], s[2], s[3]) - }; - // TODO: provide specialized kernels for the common use cases. - // - h_k = w_k = 1 - // - padding = 0 - // - stride = 1 - // - dilation = 1 - for b_idx in 0..b { - let src_idx = b_idx * src_s0; - let dst_idx = b_idx * h_out * w_out * c * h_k * w_k; - for h_idx in 0..h_out { - let dst_idx = dst_idx + h_idx * w_out * c * h_k * w_k; - for w_idx in 0..w_out { - let dst_idx = dst_idx + w_idx * c * h_k * w_k; - for c_idx in 0..c { - let dst_idx = dst_idx + c_idx * h_k * w_k; - let src_idx = c_idx * src_s1 + src_idx; - for h_k_idx in 0..h_k { - let src_h = h_idx * stride + h_k_idx * dilation; - if padding != 0 && (src_h < padding || src_h >= h + padding) { - continue; - } - let src_h = src_h - padding; - let src_idx = src_idx + src_h * src_s2; - let dst_idx = dst_idx + h_k_idx * w_k; - for w_k_idx in 0..w_k { - let src_w = w_idx * stride + w_k_idx * dilation; - if padding != 0 && (src_w < padding || src_w >= w + padding) { - continue; - } - let src_w = src_w - padding; - let src_idx = src_idx + src_w * src_s3; - let dst_idx = dst_idx + w_k_idx; - dst[dst_idx] = src[src_idx] - } - } - } - } - } - } - Ok(dst) - } -} - -struct Col2Im1D { - stride: usize, -} - -impl Map1 for Col2Im1D { - fn f(&self, col: &[T], l: &Layout) -> Result> { - let (b_size, l_in, c_out, k_size) = l.shape().dims4()?; - let stride = self.stride; - let l_out = (l_in - 1) * stride + k_size; - let mut im = vec![T::zero(); b_size * c_out * l_out]; - let (dst_s0, dst_s1) = (c_out * l_out, l_out); - let (src_s0, src_s1, src_s2) = (c_out * k_size * l_in, c_out * k_size, k_size); - for l_in_i in 0..l_in { - for k_i in 0..k_size { - let l_out_i = l_in_i * stride + k_i; - for b_i in 0..b_size { - for c_i in 0..c_out { - let dst_idx = b_i * dst_s0 + c_i * dst_s1 + l_out_i; - let src_idx = b_i * src_s0 + l_in_i * src_s1 + c_i * src_s2 + k_i; - im[dst_idx] += col[src_idx] - } - } - } - } - Ok(im) - } -} - -struct ConvTranspose1D<'a>(&'a crate::conv::ParamsConvTranspose1D); - -impl Map2 for ConvTranspose1D<'_> { - const OP: &'static str = "conv_transpose1d"; - fn f(&self, inp: &[T], inp_l: &Layout, k: &[T], k_l: &Layout) -> Result> { - let p = self.0; - let inp = &inp[inp_l.start_offset()..]; - let k = &k[k_l.start_offset()..]; - let (inp_s0, inp_s1, inp_s2) = crate::shape::dims3(inp_l.stride())?; - let (k_s0, k_s1, k_s2) = crate::shape::dims3(k_l.stride())?; - let l_out = p.l_out(); - - // Output shape: [b_size, c_out, l_out]. - let dst_elems = p.c_out * l_out * p.b_size; - let dst = vec![T::zero(); dst_elems]; - let dst_s0 = p.c_out * l_out; - let dst_s1 = l_out; - let dst_s2 = 1; - - // TODO: Avoid making this copy if `inp` already has the appropriate layout. - let mut inp_cont = vec![T::zero(); p.b_size * p.c_in * p.l_in]; - let cont_s0 = p.l_in * p.c_in; - let cont_s1 = p.c_in; - for b_idx in 0..p.b_size { - for l_idx in 0..p.l_in { - for c_idx in 0..p.c_in { - let src_idx = b_idx * inp_s0 + c_idx * inp_s1 + l_idx * inp_s2; - let dst_idx = b_idx * cont_s0 + l_idx * cont_s1 + c_idx; - inp_cont[dst_idx] = inp[src_idx] - } - } - } - - for k_idx in 0..p.k_size { - (0..p.c_out).into_par_iter().for_each(|dst_c_idx| { - let k_cont = (0..p.c_in) - .map(|c_in_idx| k[c_in_idx * k_s0 + dst_c_idx * k_s1 + k_idx * k_s2]) - .collect::>(); - for b_idx in 0..p.b_size { - for l_idx in 0..p.l_in { - let out_idx = l_idx * p.stride + k_idx * p.dilation; - if out_idx < p.padding { - continue; - } - let out_idx = out_idx - p.padding; - if out_idx < l_out { - let inp_cont = &inp_cont[b_idx * cont_s0 + l_idx * cont_s1..]; - let dst_idx = b_idx * dst_s0 + out_idx * dst_s2 + dst_c_idx * dst_s1; - let mut d = T::zero(); - unsafe { - T::vec_dot(inp_cont.as_ptr(), k_cont.as_ptr(), &mut d, p.c_in) - } - let dst_p = dst.as_ptr(); - // Safety: dst_idx are uniques per dst_c_idx which is used to - // parallelise the different tasks so no two threads can try to - // write at the same location. - unsafe { - let ptr = dst_p.add(dst_idx) as *mut T; - *ptr += d - } - } - } - } - }) - } - Ok(dst) - } -} - -struct ConvTranspose2D<'a>(&'a crate::conv::ParamsConvTranspose2D); - -impl Map2 for ConvTranspose2D<'_> { - const OP: &'static str = "conv_transpose2d"; - fn f(&self, inp: &[T], inp_l: &Layout, k: &[T], k_l: &Layout) -> Result> { - let p = self.0; - let inp = &inp[inp_l.start_offset()..]; - let (inp_s0, inp_s1, inp_s2, inp_s3) = crate::shape::dims4(inp_l.stride())?; - let k = &k[k_l.start_offset()..]; - let (k_s0, k_s1, k_s2, k_s3) = crate::shape::dims4(k_l.stride())?; - let (out_h, out_w) = (p.out_h(), p.out_w()); - - // Output shape: [b_size, c_out, out_h, out_w]. - let dst = vec![T::zero(); p.b_size * p.c_out * out_h * out_w]; - let dst_s0 = p.c_out * out_h * out_w; - let dst_s1 = out_h * out_w; - let dst_s2 = out_w; - let dst_s3 = 1; - - // TODO: Avoid making this copy if `inp` already has the appropriate layout. - let mut inp_cont = vec![T::zero(); p.b_size * p.c_in * p.i_h * p.i_w]; - let cont_s0 = p.i_h * p.i_w * p.c_in; - let cont_s1 = p.i_w * p.c_in; - let cont_s2 = p.c_in; - for b_idx in 0..p.b_size { - for h_idx in 0..p.i_h { - for w_idx in 0..p.i_w { - for c_idx in 0..p.c_in { - let src_idx = - b_idx * inp_s0 + c_idx * inp_s1 + h_idx * inp_s2 + w_idx * inp_s3; - let dst_idx = b_idx * cont_s0 + h_idx * cont_s1 + w_idx * cont_s2 + c_idx; - inp_cont[dst_idx] = inp[src_idx] - } - } - } - } - - for k_y in 0..p.k_h { - for k_x in 0..p.k_w { - (0..p.c_out).into_par_iter().for_each(|dst_c_idx| { - let k_cont = (0..p.c_in) - .map(|c_in_idx| { - k[c_in_idx * k_s0 + dst_c_idx * k_s1 + k_y * k_s2 + k_x * k_s3] - }) - .collect::>(); - for b_idx in 0..p.b_size { - for inp_y in 0..p.i_h { - for inp_x in 0..p.i_w { - let out_x = inp_x * p.stride + k_x * p.dilation; - let out_y = inp_y * p.stride + k_y * p.dilation; - if out_x < p.padding || out_y < p.padding { - continue; - } - let out_x = out_x - p.padding; - let out_y = out_y - p.padding; - if out_x < out_w && out_y < out_h { - let inp_cont = &inp_cont - [b_idx * cont_s0 + inp_y * cont_s1 + inp_x * cont_s2..]; - let dst_idx = b_idx * dst_s0 - + out_y * dst_s2 - + out_x * dst_s3 - + dst_c_idx * dst_s1; - let mut d = T::zero(); - unsafe { - T::vec_dot( - inp_cont.as_ptr(), - k_cont.as_ptr(), - &mut d, - p.c_in, - ) - } - let dst_p = dst.as_ptr(); - // Safety: dst_idx are uniques per dst_c_idx which is used to - // parallelise the different tasks so no two threads can try to - // write at the same location. - unsafe { - let ptr = dst_p.add(dst_idx) as *mut T; - *ptr += d - } - } - } - } - } - }) - } - } - Ok(dst) - } -} - -struct MatMul((usize, usize, usize, usize)); - -impl MatMul { - fn striding_error(&self, lhs_l: &Layout, rhs_l: &Layout, msg: &'static str) -> Error { - Error::MatMulUnexpectedStriding(Box::new(crate::error::MatMulUnexpectedStriding { - lhs_l: lhs_l.clone(), - rhs_l: rhs_l.clone(), - bmnk: self.0, - msg, - })) - .bt() - } - - fn ab_skip(&self, lhs_l: &Layout, rhs_l: &Layout) -> Result<(usize, usize)> { - let lhs_stride = lhs_l.stride(); - let rhs_stride = rhs_l.stride(); - let rank = lhs_stride.len(); - let (_b, m, n, k) = self.0; - let a_skip: usize = match lhs_stride[..rank - 2] { - [s1, stride] if s1 == stride * lhs_l.dims()[1] => stride, - [_, stride] if lhs_l.dims()[0] == 1 => stride, - [stride, _] if lhs_l.dims()[1] == 1 => stride, - [stride] => stride, - [] => m * k, - _ => Err(self.striding_error(lhs_l, rhs_l, "non-contiguous lhs"))?, - }; - let b_skip: usize = match rhs_stride[..rank - 2] { - [s1, stride] if s1 == stride * rhs_l.dims()[1] => stride, - [_, stride] if rhs_l.dims()[0] == 1 => stride, - [stride, _] if rhs_l.dims()[1] == 1 => stride, - [stride] => stride, - [] => n * k, - _ => Err(self.striding_error(lhs_l, rhs_l, "non-contiguous rhs"))?, - }; - Ok((a_skip, b_skip)) - } -} - -impl Map2 for MatMul { - const OP: &'static str = "mat_mul"; - - #[cfg(all(not(feature = "mkl"), not(feature = "accelerate")))] - fn f( - &self, - lhs: &[T], - lhs_l: &Layout, - rhs: &[T], - rhs_l: &Layout, - ) -> Result> { - use gemm::{gemm, Parallelism}; - - match T::DTYPE { - DType::F16 | DType::F32 | DType::F64 => {} - _ => Err(Error::UnsupportedDTypeForOp(T::DTYPE, "matmul").bt())?, - } - - let (b, m, n, k) = self.0; - let lhs = &lhs[lhs_l.start_offset()..]; - let rhs = &rhs[rhs_l.start_offset()..]; - - let lhs_stride = lhs_l.stride(); - let rhs_stride = rhs_l.stride(); - let rank = lhs_stride.len(); - let lhs_cs = lhs_stride[rank - 1]; - let lhs_rs = lhs_stride[rank - 2]; - - let rhs_cs = rhs_stride[rank - 1]; - let rhs_rs = rhs_stride[rank - 2]; - - let (a_skip, b_skip) = self.ab_skip(lhs_l, rhs_l)?; - let c_skip: usize = m * n; - - let dst_shape: Shape = (m, n).into(); - let dst_strides = dst_shape.stride_contiguous(); - let dst_rs = dst_strides[0]; - let dst_cs = dst_strides[1]; - - let mut dst = vec![T::zero(); b * m * n]; - let num_threads = crate::utils::get_num_threads(); - let parallelism = if num_threads > 1 { - Parallelism::Rayon(num_threads) - } else { - Parallelism::None - }; - let (b, m, n, k) = if b_skip == 0 && a_skip == m * k { - // a_skip and c_skip should be updated but step is always 0 so - // it wouldn't matter. - (1, b * m, n, k) - } else if a_skip == 0 && b_skip == n * k { - (1, m, b * n, k) - } else { - (b, m, n, k) - }; - for step in 0..b { - let lhs_p = &lhs[step * a_skip..]; - let rhs_p = &rhs[step * b_skip..]; - let dst_p = &mut dst[step * c_skip..]; - unsafe { - gemm( - /* m: usize = */ m, - /* n: usize = */ n, - /* k: usize = */ k, - /* dst: *mut T = */ dst_p.as_mut_ptr(), - /* dst_cs: isize = */ dst_cs as isize, - /* dst_rs: isize = */ dst_rs as isize, - /* read_dst: bool = */ false, - /* lhs: *const T = */ lhs_p.as_ptr(), - /* lhs_cs: isize = */ lhs_cs as isize, - /* lhs_rs: isize = */ lhs_rs as isize, - /* rhs: *const T = */ rhs_p.as_ptr(), - /* rhs_cs: isize = */ rhs_cs as isize, - /* rhs_rs: isize = */ rhs_rs as isize, - /* alpha: T = */ T::zero(), - /* beta: T = */ T::one(), - /* conj_dst: bool = */ false, - /* conj_lhs: bool = */ false, - /* conj_rhs: bool = */ false, - parallelism, - ) - } - } - Ok(dst) - } - - #[cfg(feature = "accelerate")] - fn f( - &self, - lhs: &[T], - lhs_l: &Layout, - rhs: &[T], - rhs_l: &Layout, - ) -> Result> { - let (b, m, n, k) = self.0; - let lhs = &lhs[lhs_l.start_offset()..]; - let rhs = &rhs[rhs_l.start_offset()..]; - - let lhs_stride = lhs_l.stride(); - let rhs_stride = rhs_l.stride(); - - let (a_skip, b_skip) = self.ab_skip(lhs_l, rhs_l)?; - let c_skip: usize = m * n; - - let rhs_m1 = rhs_stride[rhs_stride.len() - 1]; - let rhs_m2 = rhs_stride[rhs_stride.len() - 2]; - let lhs_m1 = lhs_stride[lhs_stride.len() - 1]; - let lhs_m2 = lhs_stride[lhs_stride.len() - 2]; - - let (lda, transa) = if (rhs_m1 == 1 || n == 1) && (rhs_m2 == n || k == 1) { - (n as i32, b'N') - } else if rhs_m1 == k && rhs_m2 == 1 { - (k as i32, b'T') - } else { - Err(self.striding_error(lhs_l, rhs_l, "non-contiguous rhs"))? - }; - // The b tensor has dims batching, m, k (lhs) - let (ldb, transb) = if (lhs_m1 == 1 || k == 1) && (lhs_m2 == k || m == 1) { - (k as i32, b'N') - } else if lhs_m1 == m && lhs_m2 == 1 { - (m as i32, b'T') - } else { - Err(self.striding_error(lhs_l, rhs_l, "non-contiguous lhs"))? - }; - - let mut dst = vec![T::zero(); b * m * n]; - match T::DTYPE { - DType::F16 => { - crate::bail!("the accelerate backend does not support f16 matmul") - } - DType::F32 => { - for step in 0..b { - let lhs_p = &lhs[step * a_skip..]; - let rhs_p = &rhs[step * b_skip..]; - let dst_p = &mut dst[step * c_skip..]; - unsafe { - let a = rhs_p.as_ptr() as *const f32; - let b = lhs_p.as_ptr() as *const f32; - let c = dst_p.as_mut_ptr() as *mut f32; - let a = std::slice::from_raw_parts(a, a_skip); - let b = std::slice::from_raw_parts(b, b_skip); - let c = std::slice::from_raw_parts_mut(c, c_skip); - crate::accelerate::sgemm( - transa, transb, /* m= */ n as i32, /* n= */ m as i32, - /* k= */ k as i32, /* alpha= */ 1., /* a= */ a, - /* lda= */ lda, /* b= */ b, /* ldb= */ ldb, - /* beta= */ 0., /* c= */ c, /* ldc= */ n as i32, - ) - } - } - } - DType::F64 => { - for step in 0..b { - let lhs_p = &lhs[step * a_skip..]; - let rhs_p = &rhs[step * b_skip..]; - let dst_p = &mut dst[step * c_skip..]; - unsafe { - let a = rhs_p.as_ptr() as *const f64; - let b = lhs_p.as_ptr() as *const f64; - let c = dst_p.as_mut_ptr() as *mut f64; - let a = std::slice::from_raw_parts(a, a_skip); - let b = std::slice::from_raw_parts(b, b_skip); - let c = std::slice::from_raw_parts_mut(c, c_skip); - crate::accelerate::dgemm( - transa, transb, /* m= */ n as i32, /* n= */ m as i32, - /* k= */ k as i32, /* alpha= */ 1., /* a= */ a, - /* lda= */ lda, /* b= */ b, /* ldb= */ ldb, - /* beta= */ 0., /* c= */ c, /* ldc= */ n as i32, - ) - } - } - } - dtype => Err(Error::UnsupportedDTypeForOp(dtype, "matmul").bt())?, - } - Ok(dst) - } - - #[cfg(feature = "mkl")] - fn f( - &self, - lhs: &[T], - lhs_l: &Layout, - rhs: &[T], - rhs_l: &Layout, - ) -> Result> { - let (b, m, n, k) = self.0; - let lhs = &lhs[lhs_l.start_offset()..]; - let rhs = &rhs[rhs_l.start_offset()..]; - - let lhs_stride = lhs_l.stride(); - let rhs_stride = rhs_l.stride(); - - let (a_skip, b_skip) = self.ab_skip(lhs_l, rhs_l)?; - let c_skip: usize = m * n; - - let rhs_m1 = rhs_stride[rhs_stride.len() - 1]; - let rhs_m2 = rhs_stride[rhs_stride.len() - 2]; - let lhs_m1 = lhs_stride[lhs_stride.len() - 1]; - let lhs_m2 = lhs_stride[lhs_stride.len() - 2]; - - let (lda, transa) = if (rhs_m1 == 1 || n == 1) && (rhs_m2 == n || k == 1) { - (n as i32, b'N') - } else if rhs_m1 == k && rhs_m2 == 1 { - (k as i32, b'T') - } else { - Err(self.striding_error(lhs_l, rhs_l, "non-contiguous rhs"))? - }; - // The b tensor has dims batching, m, k (lhs) - let (ldb, transb) = if (lhs_m1 == 1 || k == 1) && (lhs_m2 == k || m == 1) { - (k as i32, b'N') - } else if lhs_m1 == m && lhs_m2 == 1 { - (m as i32, b'T') - } else { - Err(self.striding_error(lhs_l, rhs_l, "non-contiguous lhs"))? - }; - - let mut dst = vec![T::zero(); b * m * n]; - match T::DTYPE { - DType::F16 => { - for step in 0..b { - let lhs_p = &lhs[step * a_skip..]; - let rhs_p = &rhs[step * b_skip..]; - let dst_p = &mut dst[step * c_skip..]; - unsafe { - let a = rhs_p.as_ptr() as *const f16; - let b = lhs_p.as_ptr() as *const f16; - let c = dst_p.as_mut_ptr() as *mut f16; - let a = std::slice::from_raw_parts(a, a_skip); - let b = std::slice::from_raw_parts(b, b_skip); - let c = std::slice::from_raw_parts_mut(c, c_skip); - crate::mkl::hgemm( - transa, - transb, - /* m= */ n as i32, - /* n= */ m as i32, - /* k= */ k as i32, - /* alpha= */ f16::ONE, - /* a= */ a, - /* lda= */ lda, - /* b= */ b, - /* ldb= */ ldb, - /* beta= */ f16::ZERO, - /* c= */ c, - /* ldc= */ n as i32, - ) - } - } - } - DType::F32 => { - for step in 0..b { - let lhs_p = &lhs[step * a_skip..]; - let rhs_p = &rhs[step * b_skip..]; - let dst_p = &mut dst[step * c_skip..]; - unsafe { - let a = rhs_p.as_ptr() as *const f32; - let b = lhs_p.as_ptr() as *const f32; - let c = dst_p.as_mut_ptr() as *mut f32; - let a = std::slice::from_raw_parts(a, a_skip); - let b = std::slice::from_raw_parts(b, b_skip); - let c = std::slice::from_raw_parts_mut(c, c_skip); - crate::mkl::sgemm( - transa, transb, /* m= */ n as i32, /* n= */ m as i32, - /* k= */ k as i32, /* alpha= */ 1., /* a= */ a, - /* lda= */ lda, /* b= */ b, /* ldb= */ ldb, - /* beta= */ 0., /* c= */ c, /* ldc= */ n as i32, - ) - } - } - } - DType::F64 => { - for step in 0..b { - let lhs_p = &lhs[step * a_skip..]; - let rhs_p = &rhs[step * b_skip..]; - let dst_p = &mut dst[step * c_skip..]; - unsafe { - let a = rhs_p.as_ptr() as *const f64; - let b = lhs_p.as_ptr() as *const f64; - let c = dst_p.as_mut_ptr() as *mut f64; - let a = std::slice::from_raw_parts(a, a_skip); - let b = std::slice::from_raw_parts(b, b_skip); - let c = std::slice::from_raw_parts_mut(c, c_skip); - crate::mkl::dgemm( - transa, transb, /* m= */ n as i32, /* n= */ m as i32, - /* k= */ k as i32, /* alpha= */ 1., /* a= */ a, - /* lda= */ lda, /* b= */ b, /* ldb= */ ldb, - /* beta= */ 0., /* c= */ c, /* ldc= */ n as i32, - ) - } - } - } - dtype => Err(Error::UnsupportedDTypeForOp(dtype, "matmul").bt())?, - } - Ok(dst) - } -} - -fn elu(v: T, alpha: T) -> T { - if v.is_sign_positive() { - v - } else { - (v.exp() - T::one()) * alpha - } -} - -impl CpuStorage { - pub fn as_slice(&self) -> Result<&[D]> { - D::cpu_storage_as_slice(self) - } - - pub fn concat(storages: &[CpuStorage]) -> Result { - let storage0 = &storages[0]; - let s = match storage0 { - Self::U8(_) => { - let storages = storages - .iter() - .map(|s| match s { - Self::U8(s) => Ok(s.as_slice()), - _ => crate::bail!("dtype mismatch"), - }) - .collect::>>()? - .concat(); - Self::U8(storages) - } - Self::U32(_) => { - let storages = storages - .iter() - .map(|s| match s { - Self::U32(s) => Ok(s.as_slice()), - _ => crate::bail!("dtype mismatch"), - }) - .collect::>>()? - .concat(); - Self::U32(storages) - } - Self::I16(_) => { - let storages = storages - .iter() - .map(|s| match s { - Self::I16(s) => Ok(s.as_slice()), - _ => crate::bail!("dtype mismatch"), - }) - .collect::>>()? - .concat(); - Self::I16(storages) - } - Self::I32(_) => { - let storages = storages - .iter() - .map(|s| match s { - Self::I32(s) => Ok(s.as_slice()), - _ => crate::bail!("dtype mismatch"), - }) - .collect::>>()? - .concat(); - Self::I32(storages) - } - Self::I64(_) => { - let storages = storages - .iter() - .map(|s| match s { - Self::I64(s) => Ok(s.as_slice()), - _ => crate::bail!("dtype mismatch"), - }) - .collect::>>()? - .concat(); - Self::I64(storages) - } - Self::BF16(_) => { - let storages = storages - .iter() - .map(|s| match s { - Self::BF16(s) => Ok(s.as_slice()), - _ => crate::bail!("dtype mismatch"), - }) - .collect::>>()? - .concat(); - Self::BF16(storages) - } - Self::F16(_) => { - let storages = storages - .iter() - .map(|s| match s { - Self::F16(s) => Ok(s.as_slice()), - _ => crate::bail!("dtype mismatch"), - }) - .collect::>>()? - .concat(); - Self::F16(storages) - } - Self::F32(_) => { - let storages = storages - .iter() - .map(|s| match s { - Self::F32(s) => Ok(s.as_slice()), - _ => crate::bail!("dtype mismatch"), - }) - .collect::>>()? - .concat(); - Self::F32(storages) - } - Self::F64(_) => { - let storages = storages - .iter() - .map(|s| match s { - Self::F64(s) => Ok(s.as_slice()), - _ => crate::bail!("dtype mismatch"), - }) - .collect::>>()? - .concat(); - Self::F64(storages) - } - Self::F8E4M3(_) => { - let storages = storages - .iter() - .map(|s| match s { - Self::F8E4M3(s) => Ok(s.as_slice()), - _ => crate::bail!("dtype mismatch"), - }) - .collect::>>()? - .concat(); - Self::F8E4M3(storages) - } - Self::F6E2M3(_) => { - let storages = storages - .iter() - .map(|s| match s { - Self::F6E2M3(s) => Ok(s.as_slice()), - _ => crate::bail!("dtype mismatch"), - }) - .collect::>>()? - .concat(); - Self::F6E2M3(storages) - } - Self::F6E3M2(_) => { - let storages = storages - .iter() - .map(|s| match s { - Self::F6E3M2(s) => Ok(s.as_slice()), - _ => crate::bail!("dtype mismatch"), - }) - .collect::>>()? - .concat(); - Self::F6E3M2(storages) - } - Self::F4(_) => { - let storages = storages - .iter() - .map(|s| match s { - Self::F4(s) => Ok(s.as_slice()), - _ => crate::bail!("dtype mismatch"), - }) - .collect::>>()? - .concat(); - Self::F4(storages) - } - Self::F8E8M0(_) => { - let storages = storages - .iter() - .map(|s| match s { - Self::F8E8M0(s) => Ok(s.as_slice()), - _ => crate::bail!("dtype mismatch"), - }) - .collect::>>()? - .concat(); - Self::F8E8M0(storages) - } - }; - Ok(s) - } -} - -impl BackendStorage for CpuStorage { - type Device = CpuDevice; - - fn dtype(&self) -> DType { - match self { - Self::U8(_) => DType::U8, - Self::U32(_) => DType::U32, - Self::I16(_) => DType::I16, - Self::I32(_) => DType::I32, - Self::I64(_) => DType::I64, - Self::BF16(_) => DType::BF16, - Self::F16(_) => DType::F16, - Self::F32(_) => DType::F32, - Self::F64(_) => DType::F64, - Self::F8E4M3(_) => DType::F8E4M3, - Self::F6E2M3(_) => DType::F6E2M3, - Self::F6E3M2(_) => DType::F6E3M2, - Self::F4(_) => DType::F4, - Self::F8E8M0(_) => DType::F8E8M0, - } - } - - fn to_dtype(&self, layout: &Layout, dtype: DType) -> Result { - // TODO: find a way around the quadratic number of cases below. - match (self, dtype) { - (Self::U8(storage), DType::BF16) => { - let data = unary_map(storage, layout, |v| bf16::from_f32(v as f32)); - Ok(Self::BF16(data)) - } - (Self::U32(storage), DType::BF16) => { - let data = unary_map(storage, layout, |v| bf16::from_f32(v as f32)); - Ok(Self::BF16(data)) - } - (Self::I64(storage), DType::BF16) => { - let data = unary_map(storage, layout, |v| bf16::from_f32(v as f32)); - Ok(Self::BF16(data)) - } - (Self::BF16(storage), DType::BF16) => { - let data = unary_map(storage, layout, |v| v); - Ok(Self::BF16(data)) - } - (Self::F16(storage), DType::BF16) => { - let data = unary_map(storage, layout, |v| bf16::from_f32(v.to_f32())); - Ok(Self::BF16(data)) - } - (Self::F32(storage), DType::BF16) => { - let data = unary_map(storage, layout, bf16::from_f32); - Ok(Self::BF16(data)) - } - (Self::F64(storage), DType::BF16) => { - let data = unary_map(storage, layout, bf16::from_f64); - Ok(Self::BF16(data)) - } - (Self::U8(storage), DType::F16) => { - let data = unary_map(storage, layout, |v| f16::from_f32(v as f32)); - Ok(Self::F16(data)) - } - (Self::U32(storage), DType::F16) => { - let data = unary_map(storage, layout, |v| f16::from_f32(v as f32)); - Ok(Self::F16(data)) - } - (Self::I64(storage), DType::F16) => { - let data = unary_map(storage, layout, |v| f16::from_f32(v as f32)); - Ok(Self::F16(data)) - } - (Self::BF16(storage), DType::F16) => { - let data = unary_map(storage, layout, |v| f16::from_f32(v.to_f32())); - Ok(Self::F16(data)) - } - (Self::F16(storage), DType::F16) => { - let data = unary_map(storage, layout, |v| v); - Ok(Self::F16(data)) - } - (Self::F32(storage), DType::F16) => { - let data = unary_map(storage, layout, f16::from_f32); - Ok(Self::F16(data)) - } - (Self::F64(storage), DType::F16) => { - let data = unary_map(storage, layout, f16::from_f64); - Ok(Self::F16(data)) - } - (Self::U8(storage), DType::F32) => { - let data = unary_map(storage, layout, |v| v as f32); - Ok(Self::F32(data)) - } - (Self::U32(storage), DType::F32) => { - let data = unary_map(storage, layout, |v| v as f32); - Ok(Self::F32(data)) - } - (Self::I64(storage), DType::F32) => { - let data = unary_map(storage, layout, |v| v as f32); - Ok(Self::F32(data)) - } - (Self::BF16(storage), DType::F32) => { - let data = unary_map(storage, layout, |v| v.to_f32()); - Ok(Self::F32(data)) - } - (Self::F16(storage), DType::F32) => { - let data = unary_map(storage, layout, |v| v.to_f32()); - Ok(Self::F32(data)) - } - (Self::F32(storage), DType::F32) => { - let data = unary_map(storage, layout, |v| v); - Ok(Self::F32(data)) - } - (Self::F64(storage), DType::F32) => { - let data = unary_map(storage, layout, |v| v as f32); - Ok(Self::F32(data)) - } - (Self::U8(storage), DType::U8) => { - let data = unary_map(storage, layout, |v| v); - Ok(Self::U8(data)) - } - (Self::BF16(storage), DType::U8) => { - let data = unary_map(storage, layout, |v| v.to_f32() as u8); - Ok(Self::U8(data)) - } - (Self::F16(storage), DType::U8) => { - let data = unary_map(storage, layout, |v| v.to_f32() as u8); - Ok(Self::U8(data)) - } - (Self::F32(storage), DType::U8) => { - let data = unary_map(storage, layout, |v| v as u8); - Ok(Self::U8(data)) - } - (Self::F64(storage), DType::U8) => { - let data = unary_map(storage, layout, |v| v as u8); - Ok(Self::U8(data)) - } - (Self::U32(storage), DType::U8) => { - let data = unary_map(storage, layout, |v| v as u8); - Ok(Self::U8(data)) - } - (Self::I64(storage), DType::U8) => { - let data = unary_map(storage, layout, |v| v as u8); - Ok(Self::U8(data)) - } - (Self::U8(storage), DType::U32) => { - let data = unary_map(storage, layout, |v| v as u32); - Ok(Self::U32(data)) - } - (Self::U32(storage), DType::U32) => { - let data = unary_map(storage, layout, |v| v); - Ok(Self::U32(data)) - } - (Self::I64(storage), DType::U32) => { - let data = unary_map(storage, layout, |v| v as u32); - Ok(Self::U32(data)) - } - (Self::BF16(storage), DType::U32) => { - let data = unary_map(storage, layout, |v| v.to_f32() as u32); - Ok(Self::U32(data)) - } - (Self::F16(storage), DType::U32) => { - let data = unary_map(storage, layout, |v| v.to_f32() as u32); - Ok(Self::U32(data)) - } - (Self::F32(storage), DType::U32) => { - let data = unary_map(storage, layout, |v| v as u32); - Ok(Self::U32(data)) - } - (Self::F64(storage), DType::U32) => { - let data = unary_map(storage, layout, |v| v as u32); - Ok(Self::U32(data)) - } - (Self::U8(storage), DType::I64) => { - let data = unary_map(storage, layout, |v| v as i64); - Ok(Self::I64(data)) - } - (Self::U32(storage), DType::I64) => { - let data = unary_map(storage, layout, |v| v as i64); - Ok(Self::I64(data)) - } - (Self::I64(storage), DType::I64) => { - let data = unary_map(storage, layout, |v| v); - Ok(Self::I64(data)) - } - (Self::BF16(storage), DType::I64) => { - let data = unary_map(storage, layout, |v| v.to_f32() as i64); - Ok(Self::I64(data)) - } - (Self::F16(storage), DType::I64) => { - let data = unary_map(storage, layout, |v| v.to_f32() as i64); - Ok(Self::I64(data)) - } - (Self::F32(storage), DType::I64) => { - let data = unary_map(storage, layout, |v| v as i64); - Ok(Self::I64(data)) - } - (Self::F64(storage), DType::I64) => { - let data = unary_map(storage, layout, |v| v as i64); - Ok(Self::I64(data)) - } - (Self::U8(storage), DType::F64) => { - let data = unary_map(storage, layout, |v| v as f64); - Ok(Self::F64(data)) - } - (Self::U32(storage), DType::F64) => { - let data = unary_map(storage, layout, |v| v as f64); - Ok(Self::F64(data)) - } - (Self::I64(storage), DType::F64) => { - let data = unary_map(storage, layout, |v| v as f64); - Ok(Self::F64(data)) - } - (Self::BF16(storage), DType::F64) => { - let data = unary_map(storage, layout, |v| v.to_f64()); - Ok(Self::F64(data)) - } - (Self::F16(storage), DType::F64) => { - let data = unary_map(storage, layout, |v| v.to_f64()); - Ok(Self::F64(data)) - } - (Self::F32(storage), DType::F64) => { - let data = unary_map(storage, layout, |v| v as f64); - Ok(Self::F64(data)) - } - (Self::F64(storage), DType::F64) => { - let data = unary_map(storage, layout, |v| v); - Ok(Self::F64(data)) - } - // Conversions to F8E4M3 - (Self::U8(storage), DType::F8E4M3) => { - let data = unary_map(storage, layout, |v| F8E4M3::from_f32(v as f32)); - Ok(Self::F8E4M3(data)) - } - (Self::U32(storage), DType::F8E4M3) => { - let data = unary_map(storage, layout, |v| F8E4M3::from_f32(v as f32)); - Ok(Self::F8E4M3(data)) - } - (Self::I64(storage), DType::F8E4M3) => { - let data = unary_map(storage, layout, |v| F8E4M3::from_f32(v as f32)); - Ok(Self::F8E4M3(data)) - } - (Self::BF16(storage), DType::F8E4M3) => { - let data = unary_map(storage, layout, |v| F8E4M3::from_f32(v.to_f32())); - Ok(Self::F8E4M3(data)) - } - (Self::F16(storage), DType::F8E4M3) => { - let data = unary_map(storage, layout, |v| F8E4M3::from_f32(v.to_f32())); - Ok(Self::F8E4M3(data)) - } - (Self::F32(storage), DType::F8E4M3) => { - let data = unary_map(storage, layout, F8E4M3::from_f32); - Ok(Self::F8E4M3(data)) - } - (Self::F64(storage), DType::F8E4M3) => { - let data = unary_map(storage, layout, F8E4M3::from_f64); - Ok(Self::F8E4M3(data)) - } - (Self::F8E4M3(storage), DType::F8E4M3) => { - let data = unary_map(storage, layout, |v| v); - Ok(Self::F8E4M3(data)) - } - // Conversions from F8E4M3 - (Self::F8E4M3(storage), DType::U8) => { - let data = unary_map(storage, layout, |v| v.to_f32() as u8); - Ok(Self::U8(data)) - } - (Self::F8E4M3(storage), DType::U32) => { - let data = unary_map(storage, layout, |v| v.to_f32() as u32); - Ok(Self::U32(data)) - } - (Self::F8E4M3(storage), DType::I64) => { - let data = unary_map(storage, layout, |v| v.to_f32() as i64); - Ok(Self::I64(data)) - } - (Self::F8E4M3(storage), DType::BF16) => { - let data = unary_map(storage, layout, |v| bf16::from_f32(v.to_f32())); - Ok(Self::BF16(data)) - } - (Self::F8E4M3(storage), DType::F16) => { - let data = unary_map(storage, layout, |v| f16::from_f32(v.to_f32())); - Ok(Self::F16(data)) - } - (Self::F8E4M3(storage), DType::F32) => { - let data = unary_map(storage, layout, |v| v.to_f32()); - Ok(Self::F32(data)) - } - (Self::F8E4M3(storage), DType::F64) => { - let data = unary_map(storage, layout, |v| v.to_f64()); - Ok(Self::F64(data)) - } - // Conversions to I16 - (Self::U8(storage), DType::I16) => { - let data = unary_map(storage, layout, |v| v as i16); - Ok(Self::I16(data)) - } - (Self::U32(storage), DType::I16) => { - let data = unary_map(storage, layout, |v| v as i16); - Ok(Self::I16(data)) - } - (Self::I16(storage), DType::I16) => { - let data = unary_map(storage, layout, |v| v); - Ok(Self::I16(data)) - } - (Self::I32(storage), DType::I16) => { - let data = unary_map(storage, layout, |v| v as i16); - Ok(Self::I16(data)) - } - (Self::I64(storage), DType::I16) => { - let data = unary_map(storage, layout, |v| v as i16); - Ok(Self::I16(data)) - } - (Self::BF16(storage), DType::I16) => { - let data = unary_map(storage, layout, |v| v.to_f32() as i16); - Ok(Self::I16(data)) - } - (Self::F16(storage), DType::I16) => { - let data = unary_map(storage, layout, |v| v.to_f32() as i16); - Ok(Self::I16(data)) - } - (Self::F32(storage), DType::I16) => { - let data = unary_map(storage, layout, |v| v as i16); - Ok(Self::I16(data)) - } - (Self::F64(storage), DType::I16) => { - let data = unary_map(storage, layout, |v| v as i16); - Ok(Self::I16(data)) - } - (Self::F8E4M3(storage), DType::I16) => { - let data = unary_map(storage, layout, |v| v.to_f32() as i16); - Ok(Self::I16(data)) - } - // Conversions to I32 - (Self::U8(storage), DType::I32) => { - let data = unary_map(storage, layout, |v| v as i32); - Ok(Self::I32(data)) - } - (Self::U32(storage), DType::I32) => { - let data = unary_map(storage, layout, |v| v as i32); - Ok(Self::I32(data)) - } - (Self::I16(storage), DType::I32) => { - let data = unary_map(storage, layout, |v| v as i32); - Ok(Self::I32(data)) - } - (Self::I32(storage), DType::I32) => { - let data = unary_map(storage, layout, |v| v); - Ok(Self::I32(data)) - } - (Self::I64(storage), DType::I32) => { - let data = unary_map(storage, layout, |v| v as i32); - Ok(Self::I32(data)) - } - (Self::BF16(storage), DType::I32) => { - let data = unary_map(storage, layout, |v| v.to_f32() as i32); - Ok(Self::I32(data)) - } - (Self::F16(storage), DType::I32) => { - let data = unary_map(storage, layout, |v| v.to_f32() as i32); - Ok(Self::I32(data)) - } - (Self::F32(storage), DType::I32) => { - let data = unary_map(storage, layout, |v| v as i32); - Ok(Self::I32(data)) - } - (Self::F64(storage), DType::I32) => { - let data = unary_map(storage, layout, |v| v as i32); - Ok(Self::I32(data)) - } - (Self::F8E4M3(storage), DType::I32) => { - let data = unary_map(storage, layout, |v| v.to_f32() as i32); - Ok(Self::I32(data)) - } - // Conversions from I16 - (Self::I16(storage), DType::U8) => { - let data = unary_map(storage, layout, |v| v as u8); - Ok(Self::U8(data)) - } - (Self::I16(storage), DType::U32) => { - let data = unary_map(storage, layout, |v| v as u32); - Ok(Self::U32(data)) - } - (Self::I16(storage), DType::I64) => { - let data = unary_map(storage, layout, |v| v as i64); - Ok(Self::I64(data)) - } - (Self::I16(storage), DType::BF16) => { - let data = unary_map(storage, layout, |v| bf16::from_f32(v as f32)); - Ok(Self::BF16(data)) - } - (Self::I16(storage), DType::F16) => { - let data = unary_map(storage, layout, |v| f16::from_f32(v as f32)); - Ok(Self::F16(data)) - } - (Self::I16(storage), DType::F32) => { - let data = unary_map(storage, layout, |v| v as f32); - Ok(Self::F32(data)) - } - (Self::I16(storage), DType::F64) => { - let data = unary_map(storage, layout, |v| v as f64); - Ok(Self::F64(data)) - } - (Self::I16(storage), DType::F8E4M3) => { - let data = unary_map(storage, layout, |v| F8E4M3::from_f32(v as f32)); - Ok(Self::F8E4M3(data)) - } - // Conversions from I32 - (Self::I32(storage), DType::U8) => { - let data = unary_map(storage, layout, |v| v as u8); - Ok(Self::U8(data)) - } - (Self::I32(storage), DType::U32) => { - let data = unary_map(storage, layout, |v| v as u32); - Ok(Self::U32(data)) - } - (Self::I32(storage), DType::I64) => { - let data = unary_map(storage, layout, |v| v as i64); - Ok(Self::I64(data)) - } - (Self::I32(storage), DType::BF16) => { - let data = unary_map(storage, layout, |v| bf16::from_f32(v as f32)); - Ok(Self::BF16(data)) - } - (Self::I32(storage), DType::F16) => { - let data = unary_map(storage, layout, |v| f16::from_f32(v as f32)); - Ok(Self::F16(data)) - } - (Self::I32(storage), DType::F32) => { - let data = unary_map(storage, layout, |v| v as f32); - Ok(Self::F32(data)) - } - (Self::I32(storage), DType::F64) => { - let data = unary_map(storage, layout, |v| v as f64); - Ok(Self::F64(data)) - } - (Self::I32(storage), DType::F8E4M3) => { - let data = unary_map(storage, layout, |v| F8E4M3::from_f32(v as f32)); - Ok(Self::F8E4M3(data)) - } - // Dummy types - return error for all conversions to/from dummy types - (_, DType::F6E2M3) | (_, DType::F6E3M2) | (_, DType::F4) | (_, DType::F8E8M0) => { - Err(Error::UnsupportedDTypeForOp(dtype, "to_dtype").bt()) - } - (Self::F6E2M3(_), _) - | (Self::F6E3M2(_), _) - | (Self::F4(_), _) - | (Self::F8E8M0(_), _) => { - Err(Error::UnsupportedDTypeForOp(self.dtype(), "to_dtype").bt()) - } - } - } - - fn reduce_op(&self, op: ReduceOp, layout: &Layout, reduce_dims: &[usize]) -> Result { - match op { - ReduceOp::Sum => { - let src_dims = layout.dims(); - let mut dst_dims = src_dims.to_vec(); - for &dim in reduce_dims.iter() { - dst_dims[dim] = 1; - } - let dst_shape = Shape::from(dst_dims); - let mut reduce_dims = reduce_dims.to_vec(); - // Sort the reduce_dims as they have to be processed from left to right when converting the - // indexes. - reduce_dims.sort(); - let reduce_dims_and_stride: Vec<_> = reduce_dims - .iter() - .map(|&d| (src_dims[d], src_dims[d + 1..].iter().product::())) - .collect(); - ReduceSum { - dst_shape: &dst_shape, - reduce_dims: &reduce_dims, - reduce_dims_and_stride, - } - .map(self, layout) - } - ReduceOp::Min | ReduceOp::ArgMin | ReduceOp::Max | ReduceOp::ArgMax => { - let reduce_dim_index = match reduce_dims { - [reduce_dim_index] => *reduce_dim_index, - _ => { - let op = match op { - ReduceOp::Min => "min", - ReduceOp::ArgMin => "argmin", - ReduceOp::Max => "max", - ReduceOp::ArgMax => "argmax", - _ => unreachable!(), - }; - let dims = reduce_dims.to_vec(); - Err(Error::OnlySingleDimension { op, dims })? - } - }; - let (use_min, return_index) = match op { - ReduceOp::Min => (true, false), - ReduceOp::ArgMin => (true, true), - ReduceOp::Max => (false, false), - ReduceOp::ArgMax => (false, true), - _ => unreachable!(), - }; - ReduceIndex { - reduce_dim_index, - use_min, - return_index, - } - .map(self, layout) - } - } - } - - fn cmp(&self, op: CmpOp, rhs: &Self, lhs_l: &Layout, rhs_l: &Layout) -> Result { - Cmp(op).map(self, lhs_l, rhs, rhs_l) - } - - fn affine(&self, layout: &Layout, mul: f64, add: f64) -> Result { - Affine(mul, add).map(self, layout) - } - - fn avg_pool2d( - &self, - layout: &Layout, - kernel_size: (usize, usize), - stride: (usize, usize), - ) -> Result { - AvgPool2D(kernel_size, stride).map(self, layout) - } - - fn max_pool2d( - &self, - layout: &Layout, - kernel_size: (usize, usize), - stride: (usize, usize), - ) -> Result { - MaxPool2D(kernel_size, stride).map(self, layout) - } - - fn upsample_nearest1d(&self, layout: &Layout, sz: usize) -> Result { - UpsampleNearest1D(sz).map(self, layout) - } - - fn upsample_nearest2d(&self, layout: &Layout, h: usize, w: usize) -> Result { - UpsampleNearest2D(h, w).map(self, layout) - } - - fn upsample_bilinear2d( - &self, - layout: &Layout, - h: usize, - w: usize, - align_corners: bool, - scale_h: Option, - scale_w: Option, - ) -> Result { - UpsampleBilinear2D { - target_h: h, - target_w: w, - align_corners, - scale_h_factor: scale_h, - scale_w_factor: scale_w, - } - .map(self, layout) - } - - fn powf(&self, layout: &Layout, e: f64) -> Result { - use num_traits::Float; - // TODO: Have some generic map for functions that apply on num_traits::Float elements. - match self { - Self::BF16(storage) => { - let data = unary_map(storage, layout, |v| v.powf(bf16::from_f64(e))); - Ok(Self::BF16(data)) - } - Self::F16(storage) => { - let data = unary_map(storage, layout, |v| v.powf(f16::from_f64(e))); - Ok(Self::F16(data)) - } - Self::F32(storage) => { - let data = unary_map(storage, layout, |v| v.powf(e as f32)); - Ok(Self::F32(data)) - } - Self::F64(storage) => { - let data = unary_map(storage, layout, |v| v.powf(e)); - Ok(Self::F64(data)) - } - Self::F8E4M3(storage) => { - let data = unary_map(storage, layout, |v| v.powf(F8E4M3::from_f64(e))); - Ok(Self::F8E4M3(data)) - } - Self::U8(_) => Err(Error::UnsupportedDTypeForOp(DType::U8, "powf").bt()), - Self::U32(_) => Err(Error::UnsupportedDTypeForOp(DType::U32, "powf").bt()), - Self::I16(_) => Err(Error::UnsupportedDTypeForOp(DType::I16, "powf").bt()), - Self::I32(_) => Err(Error::UnsupportedDTypeForOp(DType::I32, "powf").bt()), - Self::I64(_) => Err(Error::UnsupportedDTypeForOp(DType::I64, "powf").bt()), - Self::F6E2M3(_) => Err(Error::UnsupportedDTypeForOp(DType::F6E2M3, "powf").bt()), - Self::F6E3M2(_) => Err(Error::UnsupportedDTypeForOp(DType::F6E3M2, "powf").bt()), - Self::F4(_) => Err(Error::UnsupportedDTypeForOp(DType::F4, "powf").bt()), - Self::F8E8M0(_) => Err(Error::UnsupportedDTypeForOp(DType::F8E8M0, "powf").bt()), - } - } - - fn elu(&self, layout: &Layout, alpha: f64) -> Result { - // TODO: Have some generic map for functions that apply on num_traits::Float elements. - match self { - Self::BF16(storage) => { - let data = unary_map(storage, layout, |v| elu(v, bf16::from_f64(alpha))); - Ok(Self::BF16(data)) - } - Self::F16(storage) => { - let data = unary_map(storage, layout, |v| elu(v, f16::from_f64(alpha))); - Ok(Self::F16(data)) - } - Self::F32(storage) => { - let data = unary_map(storage, layout, |v| elu(v, f32::from_f64(alpha))); - Ok(Self::F32(data)) - } - Self::F64(storage) => { - let data = unary_map(storage, layout, |v| elu(v, alpha)); - Ok(Self::F64(data)) - } - Self::F8E4M3(storage) => { - let data = unary_map(storage, layout, |v| elu(v, F8E4M3::from_f64(alpha))); - Ok(Self::F8E4M3(data)) - } - Self::U8(_) => Err(Error::UnsupportedDTypeForOp(DType::U8, "elu").bt()), - Self::U32(_) => Err(Error::UnsupportedDTypeForOp(DType::U32, "elu").bt()), - Self::I16(_) => Err(Error::UnsupportedDTypeForOp(DType::I16, "elu").bt()), - Self::I32(_) => Err(Error::UnsupportedDTypeForOp(DType::I32, "elu").bt()), - Self::I64(_) => Err(Error::UnsupportedDTypeForOp(DType::I64, "elu").bt()), - Self::F6E2M3(_) => Err(Error::UnsupportedDTypeForOp(DType::F6E2M3, "elu").bt()), - Self::F6E3M2(_) => Err(Error::UnsupportedDTypeForOp(DType::F6E3M2, "elu").bt()), - Self::F4(_) => Err(Error::UnsupportedDTypeForOp(DType::F4, "elu").bt()), - Self::F8E8M0(_) => Err(Error::UnsupportedDTypeForOp(DType::F8E8M0, "elu").bt()), - } - } - - fn unary_impl(&self, layout: &Layout) -> Result { - match self { - Self::BF16(storage) => { - if B::BF16_VEC { - let data = unary_map_vec(storage, layout, B::bf16, B::bf16_vec); - Ok(Self::BF16(data)) - } else { - let data = unary_map(storage, layout, B::bf16); - Ok(Self::BF16(data)) - } - } - Self::F16(storage) => { - if B::F16_VEC { - let data = unary_map_vec(storage, layout, B::f16, B::f16_vec); - Ok(Self::F16(data)) - } else { - let data = unary_map(storage, layout, B::f16); - Ok(Self::F16(data)) - } - } - Self::F32(storage) => { - if B::F32_VEC { - let data = unary_map_vec(storage, layout, B::f32, B::f32_vec); - Ok(Self::F32(data)) - } else { - let data = unary_map(storage, layout, B::f32); - Ok(Self::F32(data)) - } - } - Self::F64(storage) => { - if B::F64_VEC { - let data = unary_map_vec(storage, layout, B::f64, B::f64_vec); - Ok(Self::F64(data)) - } else { - let data = unary_map(storage, layout, B::f64); - Ok(Self::F64(data)) - } - } - Self::U8(storage) => { - let data = unary_map(storage, layout, B::u8); - Ok(Self::U8(data)) - } - Self::U32(storage) => { - let data = unary_map(storage, layout, B::u32); - Ok(Self::U32(data)) - } - Self::I16(storage) => { - let data = unary_map(storage, layout, B::i16); - Ok(Self::I16(data)) - } - Self::I32(storage) => { - let data = unary_map(storage, layout, B::i32); - Ok(Self::I32(data)) - } - Self::I64(storage) => { - let data = unary_map(storage, layout, B::i64); - Ok(Self::I64(data)) - } - Self::F8E4M3(storage) => { - let data = unary_map(storage, layout, B::f8e4m3); - Ok(Self::F8E4M3(data)) - } - Self::F6E2M3(_) => Err(Error::UnsupportedDTypeForOp(DType::F6E2M3, "unary").bt()), - Self::F6E3M2(_) => Err(Error::UnsupportedDTypeForOp(DType::F6E3M2, "unary").bt()), - Self::F4(_) => Err(Error::UnsupportedDTypeForOp(DType::F4, "unary").bt()), - Self::F8E8M0(_) => Err(Error::UnsupportedDTypeForOp(DType::F8E8M0, "unary").bt()), - } - } - - fn binary_impl( - &self, - rhs: &Self, - lhs_l: &Layout, - rhs_l: &Layout, - ) -> Result { - match (self, rhs) { - (Self::BF16(lhs), Self::BF16(rhs)) => { - let data = if B::BF16_VEC { - binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::bf16, B::bf16_vec) - } else { - binary_map(lhs_l, rhs_l, lhs, rhs, B::bf16) - }; - Ok(Self::BF16(data)) - } - (Self::F16(lhs), Self::F16(rhs)) => { - let data = if B::F16_VEC { - binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::f16, B::f16_vec) - } else { - binary_map(lhs_l, rhs_l, lhs, rhs, B::f16) - }; - Ok(Self::F16(data)) - } - (Self::F32(lhs), Self::F32(rhs)) => { - let data = if B::F32_VEC { - binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::f32, B::f32_vec) - } else { - binary_map(lhs_l, rhs_l, lhs, rhs, B::f32) - }; - Ok(Self::F32(data)) - } - (Self::F64(lhs), Self::F64(rhs)) => { - let data = if B::F64_VEC { - binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::f64, B::f64_vec) - } else { - binary_map(lhs_l, rhs_l, lhs, rhs, B::f64) - }; - Ok(Self::F64(data)) - } - (Self::U32(lhs), Self::U32(rhs)) => { - let data = if B::U32_VEC { - binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::u32, B::u32_vec) - } else { - binary_map(lhs_l, rhs_l, lhs, rhs, B::u32) - }; - Ok(Self::U32(data)) - } - (Self::I16(lhs), Self::I16(rhs)) => { - let data = binary_map(lhs_l, rhs_l, lhs, rhs, B::i16); - Ok(Self::I16(data)) - } - (Self::I32(lhs), Self::I32(rhs)) => { - let data = binary_map(lhs_l, rhs_l, lhs, rhs, B::i32); - Ok(Self::I32(data)) - } - (Self::I64(lhs), Self::I64(rhs)) => { - let data = if B::I64_VEC { - binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::i64, B::i64_vec) - } else { - binary_map(lhs_l, rhs_l, lhs, rhs, B::i64) - }; - Ok(Self::I64(data)) - } - (Self::U8(lhs), Self::U8(rhs)) => { - let data = if B::U8_VEC { - binary_map_vec(lhs_l, rhs_l, lhs, rhs, B::u8, B::u8_vec) - } else { - binary_map(lhs_l, rhs_l, lhs, rhs, B::u8) - }; - Ok(Self::U8(data)) - } - (Self::F8E4M3(lhs), Self::F8E4M3(rhs)) => { - let data = binary_map(lhs_l, rhs_l, lhs, rhs, B::f8e4m3); - Ok(Self::F8E4M3(data)) - } - _ => { - // This should be covered by the dtype check above. - Err(Error::DTypeMismatchBinaryOp { - lhs: self.dtype(), - rhs: rhs.dtype(), - op: B::NAME, - } - .bt()) - } - } - } - - fn copy2d( - &self, - dst: &mut Self, - d1: usize, - d2: usize, - src_s: usize, - dst_s: usize, - src_o: usize, - dst_o: usize, - ) -> Result<()> { - match (self, dst) { - (Self::U8(src), Self::U8(dst)) => copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o), - (Self::U32(src), Self::U32(dst)) => { - copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) - } - (Self::I16(src), Self::I16(dst)) => { - copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) - } - (Self::I32(src), Self::I32(dst)) => { - copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) - } - (Self::I64(src), Self::I64(dst)) => { - copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) - } - (Self::BF16(src), Self::BF16(dst)) => { - copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) - } - (Self::F16(src), Self::F16(dst)) => { - copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) - } - (Self::F32(src), Self::F32(dst)) => { - copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) - } - (Self::F64(src), Self::F64(dst)) => { - copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) - } - (Self::F8E4M3(src), Self::F8E4M3(dst)) => { - copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) - } - (Self::F6E2M3(src), Self::F6E2M3(dst)) => { - copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) - } - (Self::F6E3M2(src), Self::F6E3M2(dst)) => { - copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) - } - (Self::F4(src), Self::F4(dst)) => copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o), - (Self::F8E8M0(src), Self::F8E8M0(dst)) => { - copy2d_(src, dst, d1, d2, src_s, dst_s, src_o, dst_o) - } - (_, dst) => { - return Err(Error::DTypeMismatchBinaryOp { - lhs: self.dtype(), - rhs: dst.dtype(), - op: "copy2d", - } - .bt()); - } - } - Ok(()) - } - - fn copy_strided_src(&self, dst: &mut Self, dst_offset: usize, src_l: &Layout) -> Result<()> { - match (self, dst) { - (Self::U8(src), Self::U8(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), - (Self::U32(src), Self::U32(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), - (Self::I16(src), Self::I16(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), - (Self::I32(src), Self::I32(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), - (Self::I64(src), Self::I64(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), - (Self::BF16(src), Self::BF16(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), - (Self::F16(src), Self::F16(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), - (Self::F32(src), Self::F32(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), - (Self::F64(src), Self::F64(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), - (Self::F8E4M3(src), Self::F8E4M3(dst)) => { - copy_strided_src_(src, dst, dst_offset, src_l) - } - (Self::F6E2M3(src), Self::F6E2M3(dst)) => { - copy_strided_src_(src, dst, dst_offset, src_l) - } - (Self::F6E3M2(src), Self::F6E3M2(dst)) => { - copy_strided_src_(src, dst, dst_offset, src_l) - } - (Self::F4(src), Self::F4(dst)) => copy_strided_src_(src, dst, dst_offset, src_l), - (Self::F8E8M0(src), Self::F8E8M0(dst)) => { - copy_strided_src_(src, dst, dst_offset, src_l) - } - (_, dst) => { - // This should be covered by the dtype check above. - return Err(Error::DTypeMismatchBinaryOp { - lhs: self.dtype(), - rhs: dst.dtype(), - op: "copy_strided", - } - .bt()); - } - } - Ok(()) - } - - fn where_cond( - &self, - layout: &Layout, - t: &Self, - t_l: &Layout, - f: &Self, - f_l: &Layout, - ) -> Result { - match self { - Self::U8(pred) => WCond(pred, layout).map(t, t_l, f, f_l), - Self::U32(pred) => WCond(pred, layout).map(t, t_l, f, f_l), - Self::I16(pred) => WCond(pred, layout).map(t, t_l, f, f_l), - Self::I32(pred) => WCond(pred, layout).map(t, t_l, f, f_l), - Self::I64(pred) => WCond(pred, layout).map(t, t_l, f, f_l), - _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), "where-cond")), - } - } - - fn conv1d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConv1D, - ) -> Result { - if !USE_IM2COL_CONV1D { - return Conv1D(params).map(self, l, kernel, kernel_l); - } - let op = Im2Col1D { - l_k: params.k_size, - padding: params.padding, - stride: params.stride, - dilation: params.dilation, - }; - let col = op.map(self, l)?; - let b = params.b_size; - let n = params.c_out; - let l_out = params.l_out(); - let k = op.l_k * params.c_in; - let m = l_out; - let col_l = Layout::contiguous((b, m, k)); - let res = if kernel_l.is_contiguous() { - let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) - .transpose(1, 2)? - .broadcast_as((b, k, n))?; - col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? - } else { - // Make the kernel contiguous if not already the case. - let mut kernel_c = unsafe { - self.device() - .alloc_uninit(kernel_l.shape(), kernel.dtype())? - }; - kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?; - let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) - .transpose(1, 2)? - .broadcast_as((b, k, n))?; - col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? - }; - let res_l = Layout::contiguous((b, l_out, params.c_out)).transpose(1, 2)?; - let mut res_t = unsafe { self.device().alloc_uninit(res_l.shape(), res.dtype())? }; - res.copy_strided_src(&mut res_t, 0, &res_l)?; - Ok(res_t) - } - - fn conv_transpose1d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConvTranspose1D, - ) -> Result { - let can_use_col2im = kernel_l.is_contiguous() - && params.dilation == 1 - && params.padding == 0 - && params.output_padding == 0; - if USE_COL2IM_CONV1D_TR && can_use_col2im { - let (b_size, c_in, l_in) = l.shape().dims3()?; - let (c_in2, c_out, k_size) = kernel_l.shape().dims3()?; - if !kernel_l.is_contiguous() { - crate::bail!( - "convtr1d: the second argument (kernel) has to be contiguous {kernel_l:?}" - ) - } - if c_in != c_in2 { - crate::bail!( - "convtr1d: shape mismatch on c_in {:?} {:?}", - l.shape(), - kernel_l.shape() - ) - } - let col = { - // This merges the last two dimensions of the kernel together. - let kernel_l_mm = Layout::new( - (b_size, c_in, k_size * c_out).into(), - vec![0, k_size * c_out, 1], - kernel_l.start_offset(), - ); - self.matmul( - kernel, - ( - b_size, - /* m */ l_in, - /* n */ c_out * k_size, - /* k */ c_in, - ), - &l.transpose(1, 2)?, - &kernel_l_mm, - )? - }; - let col_l = Layout::contiguous((b_size, l_in, c_out, k_size)); - Col2Im1D { - stride: params.stride, - } - .map(&col, &col_l) - } else { - ConvTranspose1D(params).map(self, l, kernel, kernel_l) - } - } - - fn conv2d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConv2D, - ) -> Result { - Conv2D(params).map(self, l, kernel, kernel_l) - } - - fn conv_transpose2d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConvTranspose2D, - ) -> Result { - ConvTranspose2D(params).map(self, l, kernel, kernel_l) - } - - fn index_select(&self, ids: &Self, l: &Layout, ids_l: &Layout, dim: usize) -> Result { - match ids { - Self::U8(ids) => IndexSelect { ids, ids_l, dim }.map(self, l), - Self::U32(ids) => IndexSelect { ids, ids_l, dim }.map(self, l), - Self::I64(ids) => IndexSelect { ids, ids_l, dim }.map(self, l), - _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), "index-select").bt()), - } - } - - fn gather(&self, l: &Layout, ids: &Self, ids_l: &Layout, dim: usize) -> Result { - match ids { - Self::U8(ids) => Gather { ids, ids_l, dim }.map(self, l), - Self::U32(ids) => Gather { ids, ids_l, dim }.map(self, l), - Self::I64(ids) => Gather { ids, ids_l, dim }.map(self, l), - _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), "gather").bt()), - } - } - - fn scatter_set( - &mut self, - l: &Layout, - ids: &Self, - ids_l: &Layout, - src: &Self, - src_l: &Layout, - dim: usize, - ) -> Result<()> { - match ids { - Self::U8(ids) => Scatter::<_, Set>::new(ids, ids_l, dim).map(self, l, src, src_l), - Self::U32(ids) => Scatter::<_, Set>::new(ids, ids_l, dim).map(self, l, src, src_l), - Self::I64(ids) => Scatter::<_, Set>::new(ids, ids_l, dim).map(self, l, src, src_l), - _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), "scatter").bt()), - } - } - - fn scatter_add_set( - &mut self, - l: &Layout, - ids: &Self, - ids_l: &Layout, - src: &Self, - src_l: &Layout, - dim: usize, - ) -> Result<()> { - match ids { - Self::U8(ids) => Scatter::<_, Add>::new(ids, ids_l, dim).map(self, l, src, src_l), - Self::U32(ids) => Scatter::<_, Add>::new(ids, ids_l, dim).map(self, l, src, src_l), - Self::I16(ids) => Scatter::<_, Add>::new(ids, ids_l, dim).map(self, l, src, src_l), - Self::I32(ids) => Scatter::<_, Add>::new(ids, ids_l, dim).map(self, l, src, src_l), - Self::I64(ids) => Scatter::<_, Add>::new(ids, ids_l, dim).map(self, l, src, src_l), - _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), "scatter-add").bt()), - } - } - - fn index_add( - &self, - l: &Layout, - ids: &Self, - ids_l: &Layout, - src: &Self, - src_l: &Layout, - dim: usize, - ) -> Result { - match ids { - Self::U8(ids) => { - let ids = match ids_l.contiguous_offsets() { - Some((a, b)) => &ids[a..b], - None => Err(Error::RequiresContiguous { op: "index-add" }.bt())?, - }; - IndexAdd { ids, dim }.map(self, l, src, src_l) - } - Self::U32(ids) => { - let ids = match ids_l.contiguous_offsets() { - Some((a, b)) => &ids[a..b], - None => Err(Error::RequiresContiguous { op: "index-add" }.bt())?, - }; - IndexAdd { ids, dim }.map(self, l, src, src_l) - } - Self::I16(ids) => { - let ids = match ids_l.contiguous_offsets() { - Some((a, b)) => &ids[a..b], - None => Err(Error::RequiresContiguous { op: "index-add" }.bt())?, - }; - IndexAdd { ids, dim }.map(self, l, src, src_l) - } - Self::I32(ids) => { - let ids = match ids_l.contiguous_offsets() { - Some((a, b)) => &ids[a..b], - None => Err(Error::RequiresContiguous { op: "index-add" }.bt())?, - }; - IndexAdd { ids, dim }.map(self, l, src, src_l) - } - Self::I64(ids) => { - let ids = match ids_l.contiguous_offsets() { - Some((a, b)) => &ids[a..b], - None => Err(Error::RequiresContiguous { op: "index-add" }.bt())?, - }; - IndexAdd { ids, dim }.map(self, l, src, src_l) - } - _ => Err(Error::UnsupportedDTypeForOp(self.dtype(), "index-add").bt()), - } - } - - fn matmul( - &self, - rhs: &Self, - bmnk: (usize, usize, usize, usize), - lhs_l: &Layout, - rhs_l: &Layout, - ) -> Result { - MatMul(bmnk).map(self, lhs_l, rhs, rhs_l) - } - - fn device(&self) -> &Self::Device { - &CpuDevice - } - - fn try_clone(&self, _: &Layout) -> Result { - Ok(self.clone()) - } - - fn to_cpu_storage(&self) -> Result { - Ok(self.clone()) - } - - fn const_set(&mut self, s: crate::scalar::Scalar, l: &Layout) -> Result<()> { - use crate::scalar::Scalar; - fn set(src: &mut [T], l: &Layout, s: T) { - match l.strided_blocks() { - crate::StridedBlocks::SingleBlock { start_offset, len } => { - src[start_offset..start_offset + len].fill(s) - } - crate::StridedBlocks::MultipleBlocks { - block_start_index, - block_len: 1, - } => { - for src_index in block_start_index { - src[src_index] = s - } - } - crate::StridedBlocks::MultipleBlocks { - block_start_index, - block_len, - } => { - for src_index in block_start_index { - src[src_index..src_index + block_len].fill(s) - } - } - } - } - match (self, s) { - (Self::BF16(storage), Scalar::BF16(v)) => set(storage, l, v), - (Self::F16(storage), Scalar::F16(v)) => set(storage, l, v), - (Self::F32(storage), Scalar::F32(v)) => set(storage, l, v), - (Self::F64(storage), Scalar::F64(v)) => set(storage, l, v), - (Self::U8(storage), Scalar::U8(v)) => set(storage, l, v), - (Self::U32(storage), Scalar::U32(v)) => set(storage, l, v), - (Self::I16(storage), Scalar::I16(v)) => set(storage, l, v), - (Self::I32(storage), Scalar::I32(v)) => set(storage, l, v), - (Self::I64(storage), Scalar::I64(v)) => set(storage, l, v), - (Self::F8E4M3(storage), Scalar::F8E4M3(v)) => set(storage, l, v), - // Dummy types don't support scalar operations - (Self::F6E2M3(_), _) => { - crate::bail!("const_set not supported for dummy type F6E2M3") - } - (Self::F6E3M2(_), _) => { - crate::bail!("const_set not supported for dummy type F6E3M2") - } - (Self::F4(_), _) => { - crate::bail!("const_set not supported for dummy type F4") - } - (Self::F8E8M0(_), _) => { - crate::bail!("const_set not supported for dummy type F8E8M0") - } - (st, s) => crate::bail!( - "const_set dtype mismatch, expected {:?} but got {:?}", - st.dtype(), - s - ), - } - Ok(()) - } -} - -impl BackendDevice for CpuDevice { - type Storage = CpuStorage; - - fn location(&self) -> crate::DeviceLocation { - crate::DeviceLocation::Cpu - } - - fn same_device(&self, _: &Self) -> bool { - true - } - - fn storage_from_slice(&self, s: &[T]) -> Result { - Ok(T::to_cpu_storage(s)) - } - - fn storage_from_cpu_storage(&self, s: &CpuStorage) -> Result { - Ok(s.clone()) - } - - fn storage_from_cpu_storage_owned(&self, s: CpuStorage) -> Result { - Ok(s) - } - - fn new(_: usize) -> Result { - Ok(Self) - } - - fn set_seed(&self, _seed: u64) -> Result<()> { - crate::bail!("cannot seed the CPU rng with set_seed") - } - - fn get_current_seed(&self) -> Result { - crate::bail!("cannot get the CPU rng seed with get_current_seed") - } - - fn rand_uniform(&self, shape: &Shape, dtype: DType, min: f64, max: f64) -> Result { - use rand::prelude::*; - - let elem_count = shape.elem_count(); - let mut rng = rand::rng(); - match dtype { - DType::U8 - | DType::U32 - | DType::I16 - | DType::I32 - | DType::I64 - | DType::F6E2M3 - | DType::F6E3M2 - | DType::F4 - | DType::F8E8M0 => Err(Error::UnsupportedDTypeForOp(dtype, "rand_uniform").bt()), - DType::BF16 => { - let mut data = Vec::with_capacity(elem_count); - let uniform = rand::distr::Uniform::new(bf16::from_f64(min), bf16::from_f64(max)) - .map_err(Error::wrap)?; - for _i in 0..elem_count { - data.push(rng.sample::(uniform)) - } - Ok(CpuStorage::BF16(data)) - } - DType::F16 => { - let mut data = Vec::with_capacity(elem_count); - let uniform = rand::distr::Uniform::new(f16::from_f64(min), f16::from_f64(max)) - .map_err(Error::wrap)?; - for _i in 0..elem_count { - data.push(rng.sample::(uniform)) - } - Ok(CpuStorage::F16(data)) - } - DType::F8E4M3 => { - let mut data = Vec::with_capacity(elem_count); - let uniform = - rand::distr::Uniform::new(F8E4M3::from_f64(min), F8E4M3::from_f64(max)) - .map_err(Error::wrap)?; - for _i in 0..elem_count { - data.push(rng.sample::(uniform)) - } - Ok(CpuStorage::F8E4M3(data)) - } - DType::F32 => { - let mut data = Vec::with_capacity(elem_count); - let uniform = - rand::distr::Uniform::new(min as f32, max as f32).map_err(Error::wrap)?; - for _i in 0..elem_count { - data.push(rng.sample::(uniform)) - } - Ok(CpuStorage::F32(data)) - } - DType::F64 => { - let mut data = Vec::with_capacity(elem_count); - let uniform = rand::distr::Uniform::new(min, max).map_err(Error::wrap)?; - for _i in 0..elem_count { - data.push(rng.sample::(uniform)) - } - Ok(CpuStorage::F64(data)) - } - } - } - - fn rand_normal(&self, shape: &Shape, dtype: DType, mean: f64, std: f64) -> Result { - use rand::prelude::*; - - let elem_count = shape.elem_count(); - let mut rng = rand::rng(); - match dtype { - DType::U8 - | DType::U32 - | DType::I16 - | DType::I32 - | DType::I64 - | DType::F6E2M3 - | DType::F6E3M2 - | DType::F4 - | DType::F8E8M0 => Err(Error::UnsupportedDTypeForOp(dtype, "rand_normal").bt()), - DType::BF16 => { - let mut data = Vec::with_capacity(elem_count); - let normal = rand_distr::Normal::new(bf16::from_f64(mean), bf16::from_f64(std)) - .map_err(Error::wrap)?; - for _i in 0..elem_count { - data.push(normal.sample(&mut rng)) - } - Ok(CpuStorage::BF16(data)) - } - DType::F16 => { - let mut data = Vec::with_capacity(elem_count); - let normal = rand_distr::Normal::new(f16::from_f64(mean), f16::from_f64(std)) - .map_err(Error::wrap)?; - for _i in 0..elem_count { - data.push(normal.sample(&mut rng)) - } - Ok(CpuStorage::F16(data)) - } - DType::F8E4M3 => { - let mut data = Vec::with_capacity(elem_count); - let normal = rand_distr::Normal::new(F8E4M3::from_f64(mean), F8E4M3::from_f64(std)) - .map_err(Error::wrap)?; - for _i in 0..elem_count { - data.push(normal.sample(&mut rng)) - } - Ok(CpuStorage::F8E4M3(data)) - } - DType::F32 => { - let mut data = Vec::with_capacity(elem_count); - let normal = - rand_distr::Normal::new(mean as f32, std as f32).map_err(Error::wrap)?; - for _i in 0..elem_count { - data.push(normal.sample(&mut rng)) - } - Ok(CpuStorage::F32(data)) - } - DType::F64 => { - let mut data = Vec::with_capacity(elem_count); - let normal = rand_distr::Normal::new(mean, std).map_err(Error::wrap)?; - for _i in 0..elem_count { - data.push(normal.sample(&mut rng)) - } - Ok(CpuStorage::F64(data)) - } - } - } - - #[allow(clippy::uninit_vec)] - unsafe fn alloc_uninit(&self, shape: &Shape, dtype: DType) -> Result { - let elem_count = shape.elem_count(); - // The code below is highly unsafe but hopefully not directly unsound as we only consider - // types that are Copy, not Drop, and for which all bit patterns are proper values. - // It's still pretty risky, see the following for more details: - // https://github.com/rust-lang/rust-clippy/issues/4483 - let storage = match dtype { - DType::U8 => { - let mut v = Vec::with_capacity(elem_count); - v.set_len(elem_count); - CpuStorage::U8(v) - } - DType::U32 => { - let mut v = Vec::with_capacity(elem_count); - v.set_len(elem_count); - CpuStorage::U32(v) - } - DType::I16 => { - let mut v = Vec::with_capacity(elem_count); - v.set_len(elem_count); - CpuStorage::I16(v) - } - DType::I32 => { - let mut v = Vec::with_capacity(elem_count); - v.set_len(elem_count); - CpuStorage::I32(v) - } - DType::I64 => { - let mut v = Vec::with_capacity(elem_count); - v.set_len(elem_count); - CpuStorage::I64(v) - } - DType::BF16 => { - let mut v = Vec::with_capacity(elem_count); - v.set_len(elem_count); - CpuStorage::BF16(v) - } - DType::F16 => { - let mut v = Vec::with_capacity(elem_count); - v.set_len(elem_count); - CpuStorage::F16(v) - } - DType::F32 => { - let mut v = Vec::with_capacity(elem_count); - v.set_len(elem_count); - CpuStorage::F32(v) - } - DType::F64 => { - let mut v = Vec::with_capacity(elem_count); - v.set_len(elem_count); - CpuStorage::F64(v) - } - DType::F8E4M3 => { - let mut v = Vec::with_capacity(elem_count); - v.set_len(elem_count); - CpuStorage::F8E4M3(v) - } - DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { - return Err(Error::UnsupportedDTypeForOp(dtype, "alloc_uninit").bt()) - } - }; - Ok(storage) - } - - fn zeros_impl(&self, shape: &Shape, dtype: DType) -> Result { - let elem_count = shape.elem_count(); - let storage = match dtype { - DType::U8 => CpuStorage::U8(vec![0u8; elem_count]), - DType::U32 => CpuStorage::U32(vec![0u32; elem_count]), - DType::I16 => CpuStorage::I16(vec![0i16; elem_count]), - DType::I32 => CpuStorage::I32(vec![0i32; elem_count]), - DType::I64 => CpuStorage::I64(vec![0i64; elem_count]), - DType::BF16 => CpuStorage::BF16(vec![bf16::ZERO; elem_count]), - DType::F16 => CpuStorage::F16(vec![f16::ZERO; elem_count]), - DType::F32 => CpuStorage::F32(vec![0f32; elem_count]), - DType::F64 => CpuStorage::F64(vec![0f64; elem_count]), - DType::F8E4M3 => CpuStorage::F8E4M3(vec![F8E4M3::ZERO; elem_count]), - DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { - return Err(Error::UnsupportedDTypeForOp(dtype, "zeros").bt()) - } - }; - Ok(storage) - } - - fn synchronize(&self) -> Result<()> { - Ok(()) - } -} - -#[macro_export] -macro_rules! map_dtype { - ($name:expr, $storage:ident, $fn:expr, ($($dtypes:ident),+)) => { - match $storage { - $(CpuStorage::$dtypes(__e) => CpuStorage::$dtypes($fn(__e)),)* - s => Err(Error::UnsupportedDTypeForOp(s.dtype(), $name).bt())?, - } - }; -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/utils.rs b/v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/utils.rs deleted file mode 100644 index 1f800a928b..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/cpu_backend/utils.rs +++ /dev/null @@ -1,409 +0,0 @@ -/// Helper functions to write CPU kernels. -use crate::backend::BackendStorage; -use crate::{Error, Layout, Result, WithDType}; - -type C = super::CpuStorage; -pub trait Map1 { - fn f(&self, vs: &[T], layout: &Layout) -> Result>; - - fn map(&self, vs: &C, layout: &Layout) -> Result { - match vs { - C::U8(vs) => Ok(C::U8(self.f(vs, layout)?)), - C::U32(vs) => Ok(C::U32(self.f(vs, layout)?)), - C::I16(vs) => Ok(C::I16(self.f(vs, layout)?)), - C::I32(vs) => Ok(C::I32(self.f(vs, layout)?)), - C::I64(vs) => Ok(C::I64(self.f(vs, layout)?)), - C::BF16(vs) => Ok(C::BF16(self.f(vs, layout)?)), - C::F16(vs) => Ok(C::F16(self.f(vs, layout)?)), - C::F32(vs) => Ok(C::F32(self.f(vs, layout)?)), - C::F64(vs) => Ok(C::F64(self.f(vs, layout)?)), - C::F8E4M3(vs) => Ok(C::F8E4M3(self.f(vs, layout)?)), - // Dummy types don't support Map1 operations - C::F6E2M3(_) => Err(Error::UnsupportedDTypeForOp(vs.dtype(), "map1").bt()), - C::F6E3M2(_) => Err(Error::UnsupportedDTypeForOp(vs.dtype(), "map1").bt()), - C::F4(_) => Err(Error::UnsupportedDTypeForOp(vs.dtype(), "map1").bt()), - C::F8E8M0(_) => Err(Error::UnsupportedDTypeForOp(vs.dtype(), "map1").bt()), - } - } -} - -pub trait Map1Any { - fn f) -> C>(&self, vs: &[T], layout: &Layout, wrap: W) -> Result; - - fn map(&self, vs: &C, layout: &Layout) -> Result { - match vs { - C::U8(vs) => Ok(self.f(vs, layout, C::U8)?), - C::U32(vs) => Ok(self.f(vs, layout, C::U32)?), - C::I16(vs) => Ok(self.f(vs, layout, C::I16)?), - C::I32(vs) => Ok(self.f(vs, layout, C::I32)?), - C::I64(vs) => Ok(self.f(vs, layout, C::I64)?), - C::BF16(vs) => Ok(self.f(vs, layout, C::BF16)?), - C::F16(vs) => Ok(self.f(vs, layout, C::F16)?), - C::F32(vs) => Ok(self.f(vs, layout, C::F32)?), - C::F64(vs) => Ok(self.f(vs, layout, C::F64)?), - C::F8E4M3(vs) => Ok(self.f(vs, layout, C::F8E4M3)?), - // Dummy types don't support Map1Any operations - C::F6E2M3(_) => Err(Error::UnsupportedDTypeForOp(vs.dtype(), "map1any").bt()), - C::F6E3M2(_) => Err(Error::UnsupportedDTypeForOp(vs.dtype(), "map1any").bt()), - C::F4(_) => Err(Error::UnsupportedDTypeForOp(vs.dtype(), "map1any").bt()), - C::F8E8M0(_) => Err(Error::UnsupportedDTypeForOp(vs.dtype(), "map1any").bt()), - } - } -} - -pub trait Map2 { - const OP: &'static str; - fn f(&self, v1: &[T], l1: &Layout, v2: &[T], l2: &Layout) -> Result>; - - fn map(&self, v1: &C, l1: &Layout, v2: &C, l2: &Layout) -> Result { - match (v1, v2) { - (C::U8(v1), C::U8(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), - (C::U32(v1), C::U32(v2)) => Ok(C::U32(self.f(v1, l1, v2, l2)?)), - (C::I16(v1), C::I16(v2)) => Ok(C::I16(self.f(v1, l1, v2, l2)?)), - (C::I32(v1), C::I32(v2)) => Ok(C::I32(self.f(v1, l1, v2, l2)?)), - (C::I64(v1), C::I64(v2)) => Ok(C::I64(self.f(v1, l1, v2, l2)?)), - (C::BF16(v1), C::BF16(v2)) => Ok(C::BF16(self.f(v1, l1, v2, l2)?)), - (C::F16(v1), C::F16(v2)) => Ok(C::F16(self.f(v1, l1, v2, l2)?)), - (C::F32(v1), C::F32(v2)) => Ok(C::F32(self.f(v1, l1, v2, l2)?)), - (C::F64(v1), C::F64(v2)) => Ok(C::F64(self.f(v1, l1, v2, l2)?)), - (C::F8E4M3(v1), C::F8E4M3(v2)) => Ok(C::F8E4M3(self.f(v1, l1, v2, l2)?)), - _ => Err(Error::DTypeMismatchBinaryOp { - lhs: v1.dtype(), - rhs: v2.dtype(), - op: Self::OP, - } - .bt()), - } - } -} - -pub trait Map2InPlace { - const OP: &'static str; - fn f(&self, v1: &mut [T], l1: &Layout, v2: &[T], l2: &Layout) -> Result<()>; - - fn map(&self, v1: &mut C, l1: &Layout, v2: &C, l2: &Layout) -> Result<()> { - match (v1, v2) { - (C::U8(v1), C::U8(v2)) => self.f(v1, l1, v2, l2)?, - (C::U32(v1), C::U32(v2)) => self.f(v1, l1, v2, l2)?, - (C::I16(v1), C::I16(v2)) => self.f(v1, l1, v2, l2)?, - (C::I32(v1), C::I32(v2)) => self.f(v1, l1, v2, l2)?, - (C::I64(v1), C::I64(v2)) => self.f(v1, l1, v2, l2)?, - (C::BF16(v1), C::BF16(v2)) => self.f(v1, l1, v2, l2)?, - (C::F16(v1), C::F16(v2)) => self.f(v1, l1, v2, l2)?, - (C::F32(v1), C::F32(v2)) => self.f(v1, l1, v2, l2)?, - (C::F64(v1), C::F64(v2)) => self.f(v1, l1, v2, l2)?, - (C::F8E4M3(v1), C::F8E4M3(v2)) => self.f(v1, l1, v2, l2)?, - (v1, v2) => Err(Error::DTypeMismatchBinaryOp { - lhs: v1.dtype(), - rhs: v2.dtype(), - op: Self::OP, - } - .bt())?, - }; - Ok(()) - } -} - -pub trait Map2U8 { - const OP: &'static str; - fn f(&self, v1: &[T], l1: &Layout, v2: &[T], l2: &Layout) -> Result>; - - fn map(&self, v1: &C, l1: &Layout, v2: &C, l2: &Layout) -> Result { - match (v1, v2) { - (C::U8(v1), C::U8(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), - (C::U32(v1), C::U32(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), - (C::I16(v1), C::I16(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), - (C::I32(v1), C::I32(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), - (C::I64(v1), C::I64(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), - (C::BF16(v1), C::BF16(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), - (C::F16(v1), C::F16(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), - (C::F32(v1), C::F32(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), - (C::F64(v1), C::F64(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), - (C::F8E4M3(v1), C::F8E4M3(v2)) => Ok(C::U8(self.f(v1, l1, v2, l2)?)), - _ => Err(Error::DTypeMismatchBinaryOp { - lhs: v1.dtype(), - rhs: v2.dtype(), - op: Self::OP, - } - .bt()), - } - } -} - -pub fn binary_map U>( - lhs_l: &Layout, - rhs_l: &Layout, - lhs: &[T], - rhs: &[T], - mut f: F, -) -> Vec { - match (lhs_l.contiguous_offsets(), rhs_l.contiguous_offsets()) { - (Some((o_l1, o_l2)), Some((o_r1, o_r2))) => lhs[o_l1..o_l2] - .iter() - .zip(rhs[o_r1..o_r2].iter()) - .map(|(&l, &r)| f(l, r)) - .collect(), - (Some((o_l1, o_l2)), None) => { - // TODO: Maybe we want to avoid going through the layout twice. - match rhs_l.offsets_b() { - Some(ob) => { - let mut i_in_block = 0; - let mut i_right_broadcast = 0; - lhs[o_l1..o_l2] - .iter() - .map(|&l| { - let r = unsafe { rhs.get_unchecked(i_in_block + ob.start) }; - i_right_broadcast += 1; - if i_right_broadcast >= ob.right_broadcast { - i_in_block += 1; - i_right_broadcast = 0; - } - if i_in_block >= ob.len { - i_in_block = 0 - } - f(l, *r) - }) - .collect() - } - None => lhs_l - .strided_index() - .zip(rhs_l.strided_index()) - .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) - .collect(), - } - } - (None, Some((o_r1, o_r2))) => { - // TODO: Maybe we want to avoid going through the layout twice. - match lhs_l.offsets_b() { - Some(ob) => { - let mut i_in_block = 0; - let mut i_right_broadcast = 0; - rhs[o_r1..o_r2] - .iter() - .map(|&r| { - let l = unsafe { lhs.get_unchecked(i_in_block + ob.start) }; - i_right_broadcast += 1; - if i_right_broadcast >= ob.right_broadcast { - i_in_block += 1; - i_right_broadcast = 0; - } - if i_in_block >= ob.len { - i_in_block = 0 - } - f(*l, r) - }) - .collect() - } - None => lhs_l - .strided_index() - .zip(rhs_l.strided_index()) - .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) - .collect(), - } - } - _ => lhs_l - .strided_index() - .zip(rhs_l.strided_index()) - .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) - .collect(), - } -} - -// Similar to binary_map but with vectorized variants. -pub fn binary_map_vec T, FV: FnMut(&[T], &[T], &mut [T])>( - lhs_l: &Layout, - rhs_l: &Layout, - lhs: &[T], - rhs: &[T], - mut f: F, - mut f_vec: FV, -) -> Vec { - let el_count = lhs_l.shape().elem_count(); - match (lhs_l.contiguous_offsets(), rhs_l.contiguous_offsets()) { - (Some((o_l1, o_l2)), Some((o_r1, o_r2))) => { - let mut ys: Vec = Vec::with_capacity(el_count); - let ys_to_set = ys.spare_capacity_mut(); - let ys_to_set = unsafe { - std::mem::transmute::<&mut [std::mem::MaybeUninit], &mut [T]>(ys_to_set) - }; - f_vec(&lhs[o_l1..o_l2], &rhs[o_r1..o_r2], ys_to_set); - // SAFETY: values are all set by f_vec. - unsafe { ys.set_len(el_count) }; - ys - } - (Some((o_l1, o_l2)), None) => match rhs_l.offsets_b() { - Some(ob) if ob.right_broadcast == 1 => { - let rhs = &rhs[ob.start..ob.start + ob.len]; - let mut ys: Vec = Vec::with_capacity(el_count); - let ys_to_set = ys.spare_capacity_mut(); - let ys_to_set = unsafe { - std::mem::transmute::<&mut [std::mem::MaybeUninit], &mut [T]>(ys_to_set) - }; - let mut dst_i = 0; - for src_i in (o_l1..o_l2).step_by(ob.len) { - f_vec( - &lhs[src_i..src_i + ob.len], - rhs, - &mut ys_to_set[dst_i..dst_i + ob.len], - ); - dst_i += ob.len; - } - // SAFETY: values are all set by f_vec. - unsafe { ys.set_len(el_count) }; - ys - } - Some(ob) => { - let rhs = &rhs[ob.start..ob.start + ob.len]; - let mut ys = lhs[o_l1..o_l2].to_vec(); - for idx_l in 0..ob.left_broadcast { - let start = idx_l * ob.len * ob.right_broadcast; - for (i, &r) in rhs.iter().enumerate() { - let start = start + i * ob.right_broadcast; - for v in ys[start..start + ob.right_broadcast].iter_mut() { - *v = f(*v, r) - } - } - } - ys - } - None => lhs_l - .strided_index() - .zip(rhs_l.strided_index()) - .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) - .collect(), - }, - (None, Some((o_r1, o_r2))) => match lhs_l.offsets_b() { - Some(ob) if ob.right_broadcast == 1 => { - let lhs = &lhs[ob.start..ob.start + ob.len]; - let mut ys: Vec = Vec::with_capacity(el_count); - let ys_to_set = ys.spare_capacity_mut(); - let ys_to_set = unsafe { - std::mem::transmute::<&mut [std::mem::MaybeUninit], &mut [T]>(ys_to_set) - }; - let mut dst_i = 0; - for src_i in (o_r1..o_r2).step_by(ob.len) { - f_vec( - lhs, - &rhs[src_i..src_i + ob.len], - &mut ys_to_set[dst_i..dst_i + ob.len], - ); - dst_i += ob.len; - } - // SAFETY: values are all set by f_vec. - unsafe { ys.set_len(el_count) }; - ys - } - Some(ob) => { - let lhs = &lhs[ob.start..ob.start + ob.len]; - let mut ys = rhs[o_r1..o_r2].to_vec(); - for idx_l in 0..ob.left_broadcast { - let start = idx_l * ob.len * ob.right_broadcast; - for (i, &l) in lhs.iter().enumerate() { - let start = start + i * ob.right_broadcast; - for v in ys[start..start + ob.right_broadcast].iter_mut() { - *v = f(l, *v) - } - } - } - ys - } - None => lhs_l - .strided_index() - .zip(rhs_l.strided_index()) - .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) - .collect(), - }, - _ => lhs_l - .strided_index() - .zip(rhs_l.strided_index()) - .map(|(lhs_i, rhs_i)| f(lhs[lhs_i], rhs[rhs_i])) - .collect(), - } -} - -pub fn unary_map U>( - vs: &[T], - layout: &Layout, - mut f: F, -) -> Vec { - match layout.strided_blocks() { - crate::StridedBlocks::SingleBlock { start_offset, len } => vs - [start_offset..start_offset + len] - .iter() - .map(|&v| f(v)) - .collect(), - crate::StridedBlocks::MultipleBlocks { - block_start_index, - block_len, - } => { - let mut result = Vec::with_capacity(layout.shape().elem_count()); - // Specialize the case where block_len is one to avoid the second loop. - if block_len == 1 { - for index in block_start_index { - let v = unsafe { vs.get_unchecked(index) }; - result.push(f(*v)) - } - } else { - for index in block_start_index { - for offset in 0..block_len { - let v = unsafe { vs.get_unchecked(index + offset) }; - result.push(f(*v)) - } - } - } - result - } - } -} - -pub fn unary_map_vec U, FV: FnMut(&[T], &mut [U])>( - vs: &[T], - layout: &Layout, - mut f: F, - mut f_vec: FV, -) -> Vec { - match layout.strided_blocks() { - crate::StridedBlocks::SingleBlock { start_offset, len } => { - let mut ys: Vec = Vec::with_capacity(len); - let ys_to_set = ys.spare_capacity_mut(); - let ys_to_set = unsafe { - std::mem::transmute::<&mut [std::mem::MaybeUninit], &mut [U]>(ys_to_set) - }; - f_vec(&vs[start_offset..start_offset + len], ys_to_set); - // SAFETY: values are all set by f_vec. - unsafe { ys.set_len(len) }; - ys - } - crate::StridedBlocks::MultipleBlocks { - block_start_index, - block_len, - } => { - let el_count = layout.shape().elem_count(); - // Specialize the case where block_len is one to avoid the second loop. - if block_len == 1 { - let mut result = Vec::with_capacity(el_count); - for index in block_start_index { - let v = unsafe { vs.get_unchecked(index) }; - result.push(f(*v)) - } - result - } else { - let mut ys: Vec = Vec::with_capacity(el_count); - let ys_to_set = ys.spare_capacity_mut(); - let ys_to_set = unsafe { - std::mem::transmute::<&mut [std::mem::MaybeUninit], &mut [U]>(ys_to_set) - }; - let mut dst_index = 0; - for src_index in block_start_index { - let vs = &vs[src_index..src_index + block_len]; - let ys = &mut ys_to_set[dst_index..dst_index + block_len]; - f_vec(vs, ys); - dst_index += block_len; - } - // SAFETY: values are all set by f_vec. - unsafe { ys.set_len(el_count) }; - ys - } - } - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/cudnn.rs b/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/cudnn.rs deleted file mode 100644 index d7d8770587..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/cudnn.rs +++ /dev/null @@ -1,225 +0,0 @@ -use crate::WithDType; -use cudarc; -use cudarc::cudnn::safe::{ConvForward, Cudnn}; -use cudarc::driver::{CudaSlice, CudaView, DeviceRepr, ValidAsZeroBits}; -use std::cell::RefCell; -use std::collections::HashMap; -use std::sync::Arc; - -// The cudnn handles are stored per thread here rather than on the CudaDevice as they are neither -// send nor sync. -thread_local! { - static CUDNN: RefCell>> = HashMap::new().into(); -} - -impl From for crate::Error { - fn from(err: cudarc::cudnn::CudnnError) -> Self { - crate::Error::wrap(err) - } -} - -impl From for crate::Error { - fn from(err: cudarc::driver::DriverError) -> Self { - crate::Error::wrap(err) - } -} - -pub(crate) fn launch_conv2d< - T: DeviceRepr + WithDType + ValidAsZeroBits + cudarc::cudnn::CudnnDataType, - Y: cudarc::cudnn::CudnnDataType, ->( - src: &CudaView, - src_l: &crate::Layout, - filter: &CudaView, - dst: &mut CudaSlice, - params: &crate::conv::ParamsConv2D, - dev: &crate::cuda_backend::CudaDevice, -) -> crate::Result<()> { - use crate::conv::CudnnFwdAlgo as CandleAlgo; - use cudarc::cudnn::sys::cudnnConvolutionFwdAlgo_t as A; - - let device_id = dev.id(); - let cudnn = CUDNN.with(|cudnn| { - if let Some(cudnn) = cudnn.borrow().get(&device_id) { - return Ok(cudnn.clone()); - } - let c = Cudnn::new(dev.cuda_stream()); - if let Ok(c) = &c { - cudnn.borrow_mut().insert(device_id, c.clone()); - } - c - })?; - let conv = cudnn.create_conv2d::( - /* pad */ [params.padding as i32, params.padding as i32], - /* stride */ [params.stride as i32, params.stride as i32], - /* dilation */ [params.dilation as i32, params.dilation as i32], - cudarc::cudnn::sys::cudnnConvolutionMode_t::CUDNN_CROSS_CORRELATION, - )?; - let x_shape = [ - params.b_size as i32, - params.c_in as i32, - params.i_h as i32, - params.i_w as i32, - ]; - // Note that `src` already starts at the proper offset. - let x = if src_l.is_contiguous() { - cudnn.create_4d_tensor::( - cudarc::cudnn::sys::cudnnTensorFormat_t::CUDNN_TENSOR_NCHW, - x_shape, - )? - } else { - let s = src_l.stride(); - cudnn.create_4d_tensor_ex::( - x_shape, - [s[0] as i32, s[1] as i32, s[2] as i32, s[3] as i32], - )? - }; - let w = cudnn.create_4d_filter::( - cudarc::cudnn::sys::cudnnTensorFormat_t::CUDNN_TENSOR_NCHW, - [ - params.c_out as i32, - params.c_in as i32, - params.k_h as i32, - params.k_w as i32, - ], - )?; - let (w_out, h_out) = (params.out_w() as i32, params.out_h() as i32); - let y = cudnn.create_4d_tensor::( - cudarc::cudnn::sys::cudnnTensorFormat_t::CUDNN_TENSOR_NCHW, - [params.b_size as i32, params.c_out as i32, h_out, w_out], - )?; - let conv2d = ConvForward { - conv: &conv, - x: &x, - w: &w, - y: &y, - }; - let alg = match params.cudnn_fwd_algo { - None => conv2d.pick_algorithm()?, - Some(CandleAlgo::ImplicitGemm) => A::CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM, - Some(CandleAlgo::ImplicitPrecompGemm) => { - A::CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM - } - Some(CandleAlgo::Gemm) => A::CUDNN_CONVOLUTION_FWD_ALGO_GEMM, - Some(CandleAlgo::Direct) => A::CUDNN_CONVOLUTION_FWD_ALGO_DIRECT, - Some(CandleAlgo::Fft) => A::CUDNN_CONVOLUTION_FWD_ALGO_FFT, - Some(CandleAlgo::FftTiling) => A::CUDNN_CONVOLUTION_FWD_ALGO_FFT_TILING, - Some(CandleAlgo::Winograd) => A::CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD, - Some(CandleAlgo::WinogradNonFused) => A::CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED, - Some(CandleAlgo::Count) => A::CUDNN_CONVOLUTION_FWD_ALGO_COUNT, - }; - let workspace_size = conv2d.get_workspace_size(alg)?; - let mut workspace = dev.cuda_stream().alloc_zeros::(workspace_size)?; - unsafe { - conv2d.launch::, _, _, _>( - alg, - Some(&mut workspace), - (T::one(), T::zero()), - src, - filter, - dst, - )?; - } - Ok(()) -} - -pub(crate) fn launch_conv1d< - T: DeviceRepr + WithDType + ValidAsZeroBits + cudarc::cudnn::CudnnDataType, - Y: cudarc::cudnn::CudnnDataType, ->( - src: &CudaView, - src_l: &crate::Layout, - filter: &CudaView, - dst: &mut CudaSlice, - params: &crate::conv::ParamsConv1D, - dev: &crate::cuda_backend::CudaDevice, -) -> crate::Result<()> { - use crate::conv::CudnnFwdAlgo as CandleAlgo; - use cudarc::cudnn::sys::cudnnConvolutionFwdAlgo_t as A; - - let device_id = dev.id(); - let cudnn = CUDNN.with(|cudnn| { - if let Some(cudnn) = cudnn.borrow().get(&device_id) { - return Ok(cudnn.clone()); - } - let c = Cudnn::new(dev.cuda_stream()); - if let Ok(c) = &c { - cudnn.borrow_mut().insert(device_id, c.clone()); - } - c - })?; - let conv = cudnn.create_conv2d::( - /* pad */ [params.padding as i32, 0], - /* stride */ [params.stride as i32, 1], - /* dilation */ [params.dilation as i32, 1], - cudarc::cudnn::sys::cudnnConvolutionMode_t::CUDNN_CROSS_CORRELATION, - )?; - // https://docs.nvidia.com/deeplearning/cudnn/backend/latest/api/cudnn-ops-library.html#cudnnsettensornddescriptor - // > Tensors are restricted to having at least 4 dimensions, and at most CUDNN_DIM_MAX - // > dimensions (defined in cudnn.h). When working with lower dimensional data, it is - // > recommended that the user create a 4D tensor, and set the size along unused dimensions - // > to 1. - let x_shape = [ - params.b_size as i32, - params.c_in as i32, - params.l_in as i32, - 1, - ]; - // Note that `src` already starts at the proper offset. - let x = if src_l.is_contiguous() { - cudnn.create_4d_tensor::( - cudarc::cudnn::sys::cudnnTensorFormat_t::CUDNN_TENSOR_NCHW, - x_shape, - )? - } else { - let s = src_l.stride(); - cudnn.create_4d_tensor_ex::(x_shape, [s[0] as i32, s[1] as i32, s[2] as i32, 1i32])? - }; - let w = cudnn.create_4d_filter::( - cudarc::cudnn::sys::cudnnTensorFormat_t::CUDNN_TENSOR_NCHW, - [ - params.c_out as i32, - params.c_in as i32, - params.k_size as i32, - 1, - ], - )?; - let l_out = params.l_out() as i32; - let y = cudnn.create_4d_tensor::( - cudarc::cudnn::sys::cudnnTensorFormat_t::CUDNN_TENSOR_NCHW, - [params.b_size as i32, params.c_out as i32, l_out, 1], - )?; - let conv1d = ConvForward { - conv: &conv, - x: &x, - w: &w, - y: &y, - }; - let alg = match params.cudnn_fwd_algo { - None => conv1d.pick_algorithm()?, - Some(CandleAlgo::ImplicitGemm) => A::CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_GEMM, - Some(CandleAlgo::ImplicitPrecompGemm) => { - A::CUDNN_CONVOLUTION_FWD_ALGO_IMPLICIT_PRECOMP_GEMM - } - Some(CandleAlgo::Gemm) => A::CUDNN_CONVOLUTION_FWD_ALGO_GEMM, - Some(CandleAlgo::Direct) => A::CUDNN_CONVOLUTION_FWD_ALGO_DIRECT, - Some(CandleAlgo::Fft) => A::CUDNN_CONVOLUTION_FWD_ALGO_FFT, - Some(CandleAlgo::FftTiling) => A::CUDNN_CONVOLUTION_FWD_ALGO_FFT_TILING, - Some(CandleAlgo::Winograd) => A::CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD, - Some(CandleAlgo::WinogradNonFused) => A::CUDNN_CONVOLUTION_FWD_ALGO_WINOGRAD_NONFUSED, - Some(CandleAlgo::Count) => A::CUDNN_CONVOLUTION_FWD_ALGO_COUNT, - }; - let workspace_size = conv1d.get_workspace_size(alg)?; - let mut workspace = dev.cuda_stream().alloc_zeros::(workspace_size)?; - unsafe { - conv1d.launch::, _, _, _>( - alg, - Some(&mut workspace), - (T::one(), T::zero()), - src, - filter, - dst, - )?; - } - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/device.rs b/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/device.rs deleted file mode 100644 index 425fd74f76..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/device.rs +++ /dev/null @@ -1,710 +0,0 @@ -use crate::backend::{BackendDevice, BackendStorage}; -use crate::{CpuStorage, CpuStorageRef, DType, Layout, Result, Shape}; -pub use candle_kernels as kernels; -pub use cudarc; -use cudarc::driver::CudaFunction; -use float8::F8E4M3; -use half::{bf16, f16}; -use std::collections::HashMap; -use std::sync::{Arc, Mutex, RwLock}; - -use super::{CudaError, CudaStorage, CudaStorageSlice, WrapErr}; - -/// Unique identifier for cuda devices. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct DeviceId(usize); - -impl DeviceId { - fn new() -> Self { - // https://users.rust-lang.org/t/idiomatic-rust-way-to-generate-unique-id/33805 - use std::sync::atomic; - static COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::new(1); - Self(COUNTER.fetch_add(1, atomic::Ordering::Relaxed)) - } -} - -struct CudaRng(cudarc::curand::CudaRng); -unsafe impl Send for CudaRng {} - -pub struct ModuleStore { - mdls: [Option>; kernels::ALL_IDS.len()], -} - -#[derive(Clone)] -pub struct CudaDevice { - id: DeviceId, - context: Arc, - modules: Arc>, - custom_modules: Arc>>>, - stream: Arc, - pub(crate) blas: Arc, - curand: Arc>, - seed_value: Arc>, -} - -impl std::fmt::Debug for CudaDevice { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "CudaDevice({:?})", self.id) - } -} - -impl CudaDevice { - #[allow(clippy::missing_safety_doc)] - pub unsafe fn alloc( - &self, - len: usize, - ) -> Result> { - self.stream.alloc::(len).w() - } - - pub fn alloc_zeros( - &self, - len: usize, - ) -> Result> { - self.stream.alloc_zeros::(len).w() - } - - pub fn memcpy_htod< - T: cudarc::driver::DeviceRepr, - Src: cudarc::driver::HostSlice + ?Sized, - Dst: cudarc::driver::DevicePtrMut, - >( - &self, - src: &Src, - dst: &mut Dst, - ) -> Result<()> { - self.stream.memcpy_htod(src, dst).w() - } - - pub fn clone_dtoh>( - &self, - src: &Src, - ) -> Result> { - self.stream.clone_dtoh(src).w() - } - - pub fn memcpy_dtod< - T, - Src: cudarc::driver::DevicePtr, - Dst: cudarc::driver::DevicePtrMut, - >( - &self, - src: &Src, - dst: &mut Dst, - ) -> Result<()> { - self.stream.memcpy_dtod(src, dst).w() - } - - pub fn memcpy_dtoh< - T: cudarc::driver::DeviceRepr, - Src: cudarc::driver::DevicePtr, - Dst: cudarc::driver::HostSlice, - >( - &self, - src: &Src, - dst: &mut Dst, - ) -> Result<()> { - self.stream.memcpy_dtoh(src, dst).w() - } - - pub fn clone_htod + ?Sized>( - &self, - src: &Src, - ) -> Result> { - self.stream.clone_htod(src).w() - } -} - -pub struct CudaFunc { - func: CudaFunction, - stream: Arc, -} - -impl std::ops::Deref for CudaFunc { - type Target = CudaFunction; - - fn deref(&self) -> &Self::Target { - &self.func - } -} - -impl CudaFunc { - pub fn into_cuda_function(self) -> CudaFunction { - self.func - } -} - -#[macro_export] -macro_rules! builder_arg { - ($b:ident, $($arg:expr),*) => { - $( - let __arg = $arg; - $b.arg(&__arg); - )* - }; -} - -impl CudaFunc { - pub fn builder(&self) -> cudarc::driver::LaunchArgs<'_> { - self.stream.launch_builder(&self.func) - } -} - -impl CudaDevice { - pub fn cuda_stream(&self) -> Arc { - self.stream.clone() - } - - /// When turned on, all cuda tensors **created after calling this function** will - /// not track uses via cuda events. - /// - /// # Safety - /// - /// It is up to the user to ensure proper synchronization between multiple streams: - /// - Ensure that no tensor is freed before a use on another stream is finished. - /// - Ensure that a tensor is not used on another stream before allocation on the - /// allocating stream finishes. - /// - Ensure that a tensor is not written two concurrently by multiple streams. - pub unsafe fn disable_event_tracking(&self) { - self.context.disable_event_tracking() - } - - pub fn is_event_tracking(&self) -> bool { - self.context.is_event_tracking() - } - - #[cfg(all(feature = "ug", not(target_arch = "wasm32")))] - pub fn compile( - &self, - func_name: &'static str, - kernel: candle_ug::lang::ssa::Kernel, - ) -> Result { - let mut buf = vec![]; - candle_ug::cuda::code_gen::gen(&mut buf, func_name, &kernel)?; - let cuda_code = String::from_utf8(buf)?; - let opts = cudarc::nvrtc::CompileOptions { - use_fast_math: Some(true), - ..Default::default() - }; - let ptx = cudarc::nvrtc::safe::compile_ptx_with_opts(cuda_code, opts).w()?; - let module = self.context.load_module(ptx).w()?; - let func = module.load_function(func_name).w()?; - Ok(CudaFunc { - func, - stream: self.stream.clone(), - }) - } - - pub fn id(&self) -> DeviceId { - self.id - } - - pub fn get_or_load_custom_func( - &self, - fn_name: &str, - module_name: &str, - ptx: &str, - ) -> Result { - let ms = self.custom_modules.read().unwrap(); - if let Some(mdl) = ms.get(module_name).as_ref() { - let func = mdl.load_function(fn_name).w()?; - return Ok(CudaFunc { - func, - stream: self.stream.clone(), - }); - } - drop(ms); - let mut ms = self.custom_modules.write().unwrap(); - let cuda_module = self.context.load_module(ptx.into()).w()?; - ms.insert(module_name.to_string(), cuda_module.clone()); - let func = cuda_module.load_function(fn_name).w()?; - Ok(CudaFunc { - func, - stream: self.stream.clone(), - }) - } - - pub fn get_or_load_func(&self, fn_name: &str, mdl: &kernels::Module) -> Result { - let ms = self.modules.read().unwrap(); - if let Some(mdl) = ms.mdls[mdl.index()].as_ref() { - let func = mdl.load_function(fn_name).w()?; - return Ok(CudaFunc { - func, - stream: self.stream.clone(), - }); - } - drop(ms); - let mut ms = self.modules.write().unwrap(); - let cuda_module = self.context.load_module(mdl.ptx().into()).w()?; - ms.mdls[mdl.index()] = Some(cuda_module.clone()); - let func = cuda_module.load_function(fn_name).w()?; - Ok(CudaFunc { - func, - stream: self.stream.clone(), - }) - } - - pub fn cublas_handle(&self) -> Arc { - self.blas.clone() - } -} - -impl CudaDevice { - pub fn new_with_stream(ordinal: usize) -> Result { - let context = cudarc::driver::CudaContext::new(ordinal).w()?; - let stream = context.new_stream().w()?; - let blas = cudarc::cublas::CudaBlas::new(stream.clone()).w()?; - let curand = cudarc::curand::CudaRng::new(299792458, stream.clone()).w()?; - let module_store = ModuleStore { - mdls: [const { None }; kernels::ALL_IDS.len()], - }; - Ok(Self { - id: DeviceId::new(), - context, - stream, - blas: Arc::new(blas), - curand: Arc::new(Mutex::new(CudaRng(curand))), - modules: Arc::new(std::sync::RwLock::new(module_store)), - custom_modules: Arc::new(std::sync::RwLock::new(HashMap::new())), - seed_value: Arc::new(RwLock::new(299792458)), - }) - } -} - -impl BackendDevice for CudaDevice { - type Storage = CudaStorage; - - fn new(ordinal: usize) -> Result { - let context = cudarc::driver::CudaContext::new(ordinal).w()?; - let stream = context.default_stream(); - let blas = cudarc::cublas::CudaBlas::new(stream.clone()).w()?; - let curand = cudarc::curand::CudaRng::new(299792458, stream.clone()).w()?; - let module_store = ModuleStore { - mdls: [const { None }; kernels::ALL_IDS.len()], - }; - Ok(Self { - id: DeviceId::new(), - context, - stream, - blas: Arc::new(blas), - curand: Arc::new(Mutex::new(CudaRng(curand))), - modules: Arc::new(std::sync::RwLock::new(module_store)), - custom_modules: Arc::new(std::sync::RwLock::new(HashMap::new())), - seed_value: Arc::new(RwLock::new(299792458)), - }) - } - - fn set_seed(&self, seed: u64) -> Result<()> { - // We do not call set_seed but instead create a new curand object. This ensures that the - // state will be identical and the same random numbers will be generated. - let mut curand = self.curand.lock().unwrap(); - curand.0 = cudarc::curand::CudaRng::new(seed, self.stream.clone()).w()?; - *self.seed_value.write().unwrap() = seed; - Ok(()) - } - - fn get_current_seed(&self) -> Result { - Ok(*self.seed_value.read().unwrap()) - } - - fn location(&self) -> crate::DeviceLocation { - crate::DeviceLocation::Cuda { - gpu_id: self.context.ordinal(), - } - } - - fn same_device(&self, rhs: &Self) -> bool { - self.id == rhs.id - } - - fn zeros_impl(&self, shape: &Shape, dtype: DType) -> Result { - let elem_count = shape.elem_count(); - let slice = match dtype { - DType::U8 => { - let data = self.alloc_zeros::(elem_count)?; - CudaStorageSlice::U8(data) - } - DType::U32 => { - let data = self.alloc_zeros::(elem_count)?; - CudaStorageSlice::U32(data) - } - DType::I16 => { - let data = self.alloc_zeros::(elem_count)?; - CudaStorageSlice::I16(data) - } - DType::I32 => { - let data = self.alloc_zeros::(elem_count)?; - CudaStorageSlice::I32(data) - } - DType::I64 => { - let data = self.alloc_zeros::(elem_count)?; - CudaStorageSlice::I64(data) - } - DType::BF16 => { - let data = self.alloc_zeros::(elem_count)?; - CudaStorageSlice::BF16(data) - } - DType::F16 => { - let data = self.alloc_zeros::(elem_count)?; - CudaStorageSlice::F16(data) - } - DType::F32 => { - let data = self.alloc_zeros::(elem_count)?; - CudaStorageSlice::F32(data) - } - DType::F64 => { - let data = self.alloc_zeros::(elem_count)?; - CudaStorageSlice::F64(data) - } - DType::F8E4M3 => { - let data = self.alloc_zeros::(elem_count)?; - CudaStorageSlice::F8E4M3(data) - } - DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { - return Err( - CudaError::InternalError("Dummy types not supported in CUDA backend").into(), - ) - } - }; - Ok(CudaStorage { - slice, - device: self.clone(), - }) - } - - fn rand_uniform(&self, shape: &Shape, dtype: DType, lo: f64, up: f64) -> Result { - let elem_count = shape.elem_count(); - let curand = self.curand.lock().unwrap(); - let slice = match dtype { - // TODO: Add support for F16 and BF16 though this is likely to require some upstream - // cudarc changes. - DType::U8 - | DType::U32 - | DType::I16 - | DType::I32 - | DType::I64 - | DType::F16 - | DType::BF16 => Err(CudaError::UnsupportedDtype { - dtype, - op: "rand_uniform", - }) - .w()?, - DType::F32 => { - let mut data = unsafe { self.alloc::(elem_count)? }; - curand.0.fill_with_uniform(&mut data).w()?; - CudaStorageSlice::F32(data) - } - DType::F64 => { - let mut data = unsafe { self.alloc::(elem_count)? }; - curand.0.fill_with_uniform(&mut data).w()?; - CudaStorageSlice::F64(data) - } - DType::F8E4M3 | DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { - Err(CudaError::UnsupportedDtype { - dtype, - op: "rand_uniform", - }) - .w()? - } - }; - let slice = if lo == 0. && up == 1.0 { - slice - } else { - use super::utils::Map1; - let layout = Layout::contiguous(shape); - super::Affine(up - lo, lo).map(&slice, self, &layout)? - }; - Ok(CudaStorage { - slice, - device: self.clone(), - }) - } - - fn rand_normal(&self, shape: &Shape, dtype: DType, mean: f64, std: f64) -> Result { - // TODO: Add support for F16 and BF16 though this is likely to require some upstream - // cudarc changes. - let elem_count = shape.elem_count(); - let curand = self.curand.lock().unwrap(); - // curand can only generate an odd number of values. - // https://github.com/huggingface/candle/issues/734 - let elem_count_round = if elem_count % 2 == 1 { - elem_count + 1 - } else { - elem_count - }; - let slice = match dtype { - DType::U8 - | DType::U32 - | DType::I16 - | DType::I32 - | DType::I64 - | DType::F16 - | DType::BF16 => Err(CudaError::UnsupportedDtype { - dtype, - op: "rand_normal", - }) - .w()?, - DType::F32 => { - let mut data = unsafe { self.alloc::(elem_count_round)? }; - curand - .0 - .fill_with_normal(&mut data, mean as f32, std as f32) - .w()?; - CudaStorageSlice::F32(data) - } - DType::F64 => { - let mut data = unsafe { self.alloc::(elem_count_round)? }; - curand.0.fill_with_normal(&mut data, mean, std).w()?; - CudaStorageSlice::F64(data) - } - DType::F8E4M3 | DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { - Err(CudaError::UnsupportedDtype { - dtype, - op: "rand_normal", - }) - .w()? - } - }; - Ok(CudaStorage { - slice, - device: self.clone(), - }) - } - - unsafe fn alloc_uninit(&self, shape: &Shape, dtype: DType) -> Result { - let elem_count = shape.elem_count(); - let slice = match dtype { - DType::U8 => { - let data = self.alloc::(elem_count)?; - CudaStorageSlice::U8(data) - } - DType::U32 => { - let data = self.alloc::(elem_count)?; - CudaStorageSlice::U32(data) - } - DType::I16 => { - let data = self.alloc::(elem_count)?; - CudaStorageSlice::I16(data) - } - DType::I32 => { - let data = self.alloc::(elem_count)?; - CudaStorageSlice::I32(data) - } - DType::I64 => { - let data = self.alloc::(elem_count)?; - CudaStorageSlice::I64(data) - } - DType::BF16 => { - let data = self.alloc::(elem_count)?; - CudaStorageSlice::BF16(data) - } - DType::F16 => { - let data = self.alloc::(elem_count)?; - CudaStorageSlice::F16(data) - } - DType::F32 => { - let data = self.alloc::(elem_count)?; - CudaStorageSlice::F32(data) - } - DType::F64 => { - let data = self.alloc::(elem_count)?; - CudaStorageSlice::F64(data) - } - DType::F8E4M3 => { - let data = self.alloc::(elem_count)?; - CudaStorageSlice::F8E4M3(data) - } - DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { - return Err( - CudaError::InternalError("Dummy types not supported in CUDA backend").into(), - ) - } - }; - Ok(CudaStorage { - slice, - device: self.clone(), - }) - } - - fn storage_from_slice(&self, s: &[T]) -> Result { - let slice = match T::cpu_storage_ref(s) { - CpuStorageRef::U8(storage) => { - let data = self.clone_htod(storage)?; - CudaStorageSlice::U8(data) - } - CpuStorageRef::U32(storage) => { - let data = self.clone_htod(storage)?; - CudaStorageSlice::U32(data) - } - CpuStorageRef::I16(storage) => { - let data = self.clone_htod(storage)?; - CudaStorageSlice::I16(data) - } - CpuStorageRef::I32(storage) => { - let data = self.clone_htod(storage)?; - CudaStorageSlice::I32(data) - } - CpuStorageRef::I64(storage) => { - let data = self.clone_htod(storage)?; - CudaStorageSlice::I64(data) - } - CpuStorageRef::BF16(storage) => { - let data = self.clone_htod(storage)?; - CudaStorageSlice::BF16(data) - } - CpuStorageRef::F16(storage) => { - let data = self.clone_htod(storage)?; - CudaStorageSlice::F16(data) - } - CpuStorageRef::F32(storage) => { - let data = self.clone_htod(storage)?; - CudaStorageSlice::F32(data) - } - CpuStorageRef::F64(storage) => { - let data = self.clone_htod(storage)?; - CudaStorageSlice::F64(data) - } - CpuStorageRef::F8E4M3(storage) => { - let data = self.clone_htod(storage)?; - CudaStorageSlice::F8E4M3(data) - } - CpuStorageRef::F4(_) - | CpuStorageRef::F6E2M3(_) - | CpuStorageRef::F6E3M2(_) - | CpuStorageRef::F8E8M0(_) => { - return Err(CudaError::UnsupportedDtype { - dtype: T::DTYPE, - op: "storage_from_slice", - } - .into()); - } - }; - Ok(CudaStorage { - slice, - device: self.clone(), - }) - } - - fn storage_from_cpu_storage(&self, storage: &CpuStorage) -> Result { - let slice = match storage { - CpuStorage::U8(storage) => { - let data = self.clone_htod(storage)?; - CudaStorageSlice::U8(data) - } - CpuStorage::U32(storage) => { - let data = self.clone_htod(storage)?; - CudaStorageSlice::U32(data) - } - CpuStorage::I16(storage) => { - let data = self.clone_htod(storage)?; - CudaStorageSlice::I16(data) - } - CpuStorage::I32(storage) => { - let data = self.clone_htod(storage)?; - CudaStorageSlice::I32(data) - } - CpuStorage::I64(storage) => { - let data = self.clone_htod(storage)?; - CudaStorageSlice::I64(data) - } - CpuStorage::BF16(storage) => { - let data = self.clone_htod(storage)?; - CudaStorageSlice::BF16(data) - } - CpuStorage::F16(storage) => { - let data = self.clone_htod(storage)?; - CudaStorageSlice::F16(data) - } - CpuStorage::F32(storage) => { - let data = self.clone_htod(storage)?; - CudaStorageSlice::F32(data) - } - CpuStorage::F64(storage) => { - let data = self.clone_htod(storage)?; - CudaStorageSlice::F64(data) - } - CpuStorage::F8E4M3(storage) => { - let data = self.clone_htod(storage)?; - CudaStorageSlice::F8E4M3(data) - } - CpuStorage::F4(_) - | CpuStorage::F6E2M3(_) - | CpuStorage::F6E3M2(_) - | CpuStorage::F8E8M0(_) => { - return Err(CudaError::UnsupportedDtype { - dtype: storage.dtype(), - op: "storage_from_cpu_storage", - } - .into()); - } - }; - Ok(CudaStorage { - slice, - device: self.clone(), - }) - } - - fn storage_from_cpu_storage_owned(&self, storage: CpuStorage) -> Result { - let slice = match storage { - CpuStorage::U8(storage) => { - let data = self.clone_htod(&storage)?; - CudaStorageSlice::U8(data) - } - CpuStorage::U32(storage) => { - let data = self.clone_htod(&storage)?; - CudaStorageSlice::U32(data) - } - CpuStorage::I16(storage) => { - let data = self.clone_htod(&storage)?; - CudaStorageSlice::I16(data) - } - CpuStorage::I32(storage) => { - let data = self.clone_htod(&storage)?; - CudaStorageSlice::I32(data) - } - CpuStorage::I64(storage) => { - let data = self.clone_htod(&storage)?; - CudaStorageSlice::I64(data) - } - CpuStorage::BF16(storage) => { - let data = self.clone_htod(&storage)?; - CudaStorageSlice::BF16(data) - } - CpuStorage::F16(storage) => { - let data = self.clone_htod(&storage)?; - CudaStorageSlice::F16(data) - } - CpuStorage::F32(storage) => { - let data = self.clone_htod(&storage)?; - CudaStorageSlice::F32(data) - } - CpuStorage::F64(storage) => { - let data = self.clone_htod(&storage)?; - CudaStorageSlice::F64(data) - } - CpuStorage::F8E4M3(storage) => { - let data = self.clone_htod(&storage)?; - CudaStorageSlice::F8E4M3(data) - } - CpuStorage::F4(_) - | CpuStorage::F6E2M3(_) - | CpuStorage::F6E3M2(_) - | CpuStorage::F8E8M0(_) => { - return Err(CudaError::UnsupportedDtype { - dtype: storage.dtype(), - op: "storage_from_cpu_storage_owned", - } - .into()); - } - }; - Ok(CudaStorage { - slice, - device: self.clone(), - }) - } - - fn synchronize(&self) -> Result<()> { - self.stream.synchronize().map_err(crate::Error::wrap)?; - Ok(()) - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/error.rs b/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/error.rs deleted file mode 100644 index bd6f8ac6da..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/error.rs +++ /dev/null @@ -1,62 +0,0 @@ -use crate::{DType, Layout}; - -/// cudarc related errors -#[derive(thiserror::Error, Debug)] -pub enum CudaError { - #[error(transparent)] - Cuda(#[from] cudarc::driver::DriverError), - - #[error(transparent)] - Compiler(#[from] cudarc::nvrtc::CompileError), - - #[error(transparent)] - Cublas(#[from] cudarc::cublas::result::CublasError), - - #[error(transparent)] - Curand(#[from] cudarc::curand::result::CurandError), - - #[error("missing kernel '{module_name}'")] - MissingKernel { module_name: String }, - - #[error("unsupported dtype {dtype:?} for {op}")] - UnsupportedDtype { dtype: DType, op: &'static str }, - - #[error("internal error '{0}'")] - InternalError(&'static str), - - #[error("matmul is only supported for contiguous tensors lstride: {lhs_stride:?} rstride: {rhs_stride:?} mnk: {mnk:?}")] - MatMulNonContiguous { - lhs_stride: Layout, - rhs_stride: Layout, - mnk: (usize, usize, usize), - }, - - #[error("{msg}, expected: {expected:?}, got: {got:?}")] - UnexpectedDType { - msg: &'static str, - expected: DType, - got: DType, - }, - - #[error("{cuda} when loading {module_name}")] - Load { - cuda: cudarc::driver::DriverError, - module_name: String, - }, -} - -impl From for crate::Error { - fn from(val: CudaError) -> Self { - crate::Error::Cuda(Box::new(val)).bt() - } -} - -pub trait WrapErr { - fn w(self) -> std::result::Result; -} - -impl> WrapErr for std::result::Result { - fn w(self) -> std::result::Result { - self.map_err(|e| crate::Error::Cuda(Box::new(e.into())).bt()) - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/mod.rs b/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/mod.rs deleted file mode 100644 index d2a6fd56d6..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/mod.rs +++ /dev/null @@ -1,2682 +0,0 @@ -//! Implementation of Backend traits for CUDA device -//! -use crate::backend::{BackendDevice, BackendStorage}; -use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; -use crate::{builder_arg as barg, CpuStorage, DType, Layout, Result, WithDType}; -pub use candle_kernels as kernels; -pub use cudarc; -use cudarc::cublas::{Gemm, GemmConfig, StridedBatchedConfig}; -use cudarc::driver::{ - CudaSlice, DevicePtr, DeviceRepr, LaunchConfig, PushKernelArg, ValidAsZeroBits, -}; -use half::{bf16, f16}; - -#[cfg(feature = "cudnn")] -pub mod cudnn; -mod device; -mod error; -mod utils; -pub use device::{CudaDevice, DeviceId}; -pub use error::{CudaError, WrapErr}; -pub use utils::{Map1, Map1Any, Map2, Map2Any, Map2InPlace, Map3, S}; - -pub enum SlicePtrOrNull { - Ptr(CudaSlice), - Null, -} - -impl SlicePtrOrNull { - pub fn builder_arg<'a, 'b: 'a>(&'b self, builder: &mut cudarc::driver::LaunchArgs<'a>) { - match self { - SlicePtrOrNull::Ptr(slice) => builder.arg(slice), - SlicePtrOrNull::Null => builder.arg(&0usize), - }; - } -} - -impl crate::scalar::Scalar { - pub fn builder_arg<'a, 'b: 'a>(&'b self, builder: &mut cudarc::driver::LaunchArgs<'a>) { - use crate::scalar::Scalar; - match self { - Scalar::U8(v) => builder.arg(v), - Scalar::U32(v) => builder.arg(v), - Scalar::I16(v) => builder.arg(v), - Scalar::I32(v) => builder.arg(v), - Scalar::I64(v) => builder.arg(v), - Scalar::F32(v) => builder.arg(v), - Scalar::F64(v) => builder.arg(v), - Scalar::F16(v) => builder.arg(v), - Scalar::BF16(v) => builder.arg(v), - Scalar::F8E4M3(v) => builder.arg(v), - }; - } -} - -impl SlicePtrOrNull { - pub fn params_from_layout(dev: &CudaDevice, l: &Layout) -> Result { - let ds = if l.is_contiguous() { - SlicePtrOrNull::Null - } else { - SlicePtrOrNull::Ptr(dev.clone_htod(&[l.dims(), l.stride()].concat())?) - }; - Ok(ds) - } -} - -#[derive(Debug)] -pub enum CudaStorageSlice { - U8(CudaSlice), - U32(CudaSlice), - I16(CudaSlice), - I32(CudaSlice), - I64(CudaSlice), - BF16(CudaSlice), - F16(CudaSlice), - F32(CudaSlice), - F64(CudaSlice), - F8E4M3(CudaSlice), - // Dummy types that store raw bytes - F6E2M3(CudaSlice), - F6E3M2(CudaSlice), - F4(CudaSlice), - F8E8M0(CudaSlice), -} - -struct Clone; -impl Map1 for Clone { - fn f( - &self, - s: &CudaSlice, - _: &CudaDevice, - _: &Layout, - ) -> Result> { - s.try_clone().w() - } -} - -pub fn kernel_name(root: &str) -> String { - let dtype = T::DTYPE.as_str(); - format!("{root}_{dtype}") -} - -struct Affine(f64, f64); -impl Map1 for Affine { - fn f( - &self, - src: &CudaSlice, - dev: &CudaDevice, - layout: &Layout, - ) -> Result> { - let shape = layout.shape(); - let dims = shape.dims(); - let el = shape.elem_count(); - let cfg = LaunchConfig::for_num_elems(el as u32); - let ds = SlicePtrOrNull::params_from_layout(dev, layout)?; - let src = &src.slice(layout.start_offset()..); - let func = dev.get_or_load_func(&kernel_name::("affine"), &kernels::AFFINE)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(el)? }; - let mut builder = func.builder(); - barg!(builder, el); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - builder.arg(src); - builder.arg(&out); - barg!(builder, T::from_f64(self.0)); - barg!(builder, T::from_f64(self.1)); - // SAFETY: ffi. - unsafe { builder.launch(cfg).w() }?; - Ok(out) - } -} - -struct Elu(f64); -impl Map1 for Elu { - fn f( - &self, - src: &CudaSlice, - dev: &CudaDevice, - layout: &Layout, - ) -> Result> { - let shape = layout.shape(); - let dims = shape.dims(); - let el = shape.elem_count(); - let cfg = LaunchConfig::for_num_elems(el as u32); - let ds = SlicePtrOrNull::params_from_layout(dev, layout)?; - let src = &src.slice(layout.start_offset()..); - let func = dev.get_or_load_func(&kernel_name::("uelu"), &kernels::UNARY)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(el)? }; - let mut builder = func.builder(); - barg!(builder, el); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - barg!(builder, T::from_f64(self.0)); - builder.arg(src); - builder.arg(&out); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - Ok(out) - } -} - -#[allow(unused)] -struct Im2Col1D { - l_k: usize, - stride: usize, - dilation: usize, - padding: usize, -} - -impl Im2Col1D { - #[allow(unused)] - fn l_out(&self, l: usize) -> usize { - (l + 2 * self.padding - self.dilation * (self.l_k - 1) - 1) / self.stride + 1 - } -} - -impl Map1 for Im2Col1D { - fn f( - &self, - src: &CudaSlice, - dev: &CudaDevice, - layout: &Layout, - ) -> Result> { - let shape = layout.shape(); - let dims = shape.dims(); - let l_out = self.l_out(dims[2]); - let threads = dims[0] * l_out * dims[1]; - let cfg = LaunchConfig::for_num_elems(threads as u32); - let ds = dev.clone_htod(&[dims, layout.stride()].concat())?; - let src = &src.slice(layout.start_offset()..); - let func = dev.get_or_load_func(&kernel_name::("im2col1d"), &kernels::CONV)?; - // SAFETY: Set later by running the kernel. - let dst = unsafe { dev.alloc::(threads * self.l_k)? }; - let mut builder = func.builder(); - barg!(builder, threads); - barg!(builder, l_out); - barg!(builder, self.l_k); - barg!(builder, self.stride); - barg!(builder, self.padding); - barg!(builder, self.dilation); - builder.arg(&ds); - builder.arg(src); - builder.arg(&dst); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - Ok(dst) - } -} - -#[allow(unused)] -struct Im2Col { - h_k: usize, - w_k: usize, - stride: usize, - dilation: usize, - padding: usize, -} - -impl Im2Col { - #[allow(unused)] - fn hw_out(&self, h: usize, w: usize) -> (usize, usize) { - let h_out = (h + 2 * self.padding - self.dilation * (self.h_k - 1) - 1) / self.stride + 1; - let w_out = (w + 2 * self.padding - self.dilation * (self.w_k - 1) - 1) / self.stride + 1; - (h_out, w_out) - } -} - -impl Map1 for Im2Col { - fn f( - &self, - src: &CudaSlice, - dev: &CudaDevice, - layout: &Layout, - ) -> Result> { - let shape = layout.shape(); - let dims = shape.dims(); - let (h_out, w_out) = self.hw_out(dims[2], dims[3]); - let dst_el = dims[0] * h_out * w_out * dims[1] * self.h_k * self.w_k; - let cfg = LaunchConfig::for_num_elems(dst_el as u32); - let ds = dev.clone_htod(&[dims, layout.stride()].concat())?; - let src = &src.slice(layout.start_offset()..); - let func = dev.get_or_load_func(&kernel_name::("im2col"), &kernels::CONV)?; - // SAFETY: Set later by running the kernel. - let dst = unsafe { dev.alloc::(dst_el)? }; - let mut builder = func.builder(); - barg!(builder, dst_el); - barg!(builder, h_out); - barg!(builder, w_out); - barg!(builder, self.h_k); - barg!(builder, self.w_k); - barg!(builder, self.stride); - barg!(builder, self.padding); - barg!(builder, self.dilation); - builder.arg(&ds); - builder.arg(src); - builder.arg(&dst); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - Ok(dst) - } -} - -struct Powf(f64); -impl Map1 for Powf { - fn f( - &self, - src: &CudaSlice, - dev: &CudaDevice, - layout: &Layout, - ) -> Result> { - let shape = layout.shape(); - let dims = shape.dims(); - let el = shape.elem_count(); - let cfg = LaunchConfig::for_num_elems(el as u32); - let ds = SlicePtrOrNull::params_from_layout(dev, layout)?; - let src = &src.slice(layout.start_offset()..); - let func = dev.get_or_load_func(&kernel_name::("upowf"), &kernels::UNARY)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(el)? }; - let mut builder = func.builder(); - barg!(builder, el); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - barg!(builder, T::from_f64(self.0)); - builder.arg(src); - builder.arg(&out); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - Ok(out) - } -} - -struct FastReduce<'a>(&'a [usize], ReduceOp); -impl Map1Any for FastReduce<'_> { - fn f) -> S>( - &self, - src: &CudaSlice, - dev: &CudaDevice, - layout: &Layout, - wrap: W, - ) -> Result { - let src_stride = layout.stride(); - let src_dims = layout.shape().dims(); - let src_el: usize = src_dims.iter().product(); - // Source dims and strides with the sum dims at the end. - let mut dims = vec![]; - let mut stride = vec![]; - let mut dst_el: usize = 1; - for (dim_idx, &d) in src_dims.iter().enumerate() { - if !self.0.contains(&dim_idx) { - dst_el *= d; - dims.push(d); - stride.push(src_stride[dim_idx]); - } - } - for &dim_idx in self.0.iter() { - dims.push(src_dims[dim_idx]); - stride.push(src_stride[dim_idx]); - } - let el_to_sum_per_block = src_el / dst_el; - // The reduction loop requires the shared array to be properly initialized and for - // this we want the number of threads to be a power of two. - let block_dim = usize::min(1024, el_to_sum_per_block).next_power_of_two(); - let cfg = LaunchConfig { - // TODO: Maybe use grid_y if the output is too large? - // TODO: Specialized implementation when reducing on no or all dimensions or when - // reducing only aggregate a small number of elements together. - grid_dim: (dst_el as u32, 1, 1), - block_dim: (block_dim as u32, 1, 1), - shared_mem_bytes: 0, - }; - let ds = dev.clone_htod(&[dims.as_slice(), stride.as_slice()].concat())?; - let src = &src.slice(layout.start_offset()..); - let (name, check_empty, return_index) = match self.1 { - ReduceOp::Sum => ("fast_sum", false, false), - ReduceOp::Min => ("fast_min", true, false), - ReduceOp::Max => ("fast_max", true, false), - ReduceOp::ArgMin => ("fast_argmin", true, true), - ReduceOp::ArgMax => ("fast_argmax", true, true), - }; - if check_empty && layout.shape().elem_count() == 0 { - Err(crate::Error::EmptyTensor { op: "reduce" }.bt())? - } - let func = dev.get_or_load_func(&kernel_name::(name), &kernels::REDUCE)?; - if return_index { - // SAFETY: filled in by the follow up kernel. - let out = unsafe { dev.alloc::(dst_el)? }; - let mut builder = func.builder(); - barg!(builder, src_el); - barg!(builder, el_to_sum_per_block); - barg!(builder, src_dims.len()); - builder.arg(&ds); - builder.arg(src); - builder.arg(&out); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - Ok(S::U32(out)) - } else { - // SAFETY: filled in by the follow up kernel. - let out = unsafe { dev.alloc::(dst_el)? }; - let mut builder = func.builder(); - barg!(builder, src_el); - barg!(builder, el_to_sum_per_block); - barg!(builder, src_dims.len()); - builder.arg(&ds); - builder.arg(src); - builder.arg(&out); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - Ok(wrap(out)) - } - } -} - -impl Map1 for U { - fn f( - &self, - src: &CudaSlice, - dev: &CudaDevice, - layout: &Layout, - ) -> Result> { - let shape = layout.shape(); - let dims = shape.dims(); - let el_count = shape.elem_count(); - let cfg = LaunchConfig::for_num_elems(el_count as u32); - let ds = SlicePtrOrNull::params_from_layout(dev, layout)?; - let src = &src.slice(layout.start_offset()..); - let func = dev.get_or_load_func(&kernel_name::(U::KERNEL), &kernels::UNARY)?; - // SAFETY: Set later by running the kernel. - let mut out = unsafe { dev.alloc::(el_count)? }; - let mut builder = func.builder(); - barg!(builder, el_count); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - builder.arg(src); - builder.arg(&mut out); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - Ok(out) - } -} - -fn slice_ptr(v: &CudaSlice, lo: usize) -> (u64, cudarc::driver::SyncOnDrop<'_>) { - let (_, guard) = v.device_ptr(v.stream()); - let (ptr, _) = v.slice(lo..).device_ptr(v.stream()); - (ptr, guard) -} - -struct IndexSelect<'a>(&'a CudaStorage, &'a Layout, usize); -impl Map1 for IndexSelect<'_> { - fn f( - &self, - src: &CudaSlice, - dev: &CudaDevice, - src_l: &Layout, - ) -> Result> { - let ids_l = &self.1; - let (name, (ids, _guard)) = match &self.0.slice { - CudaStorageSlice::U32(slice) => ("is_u32", slice_ptr(slice, ids_l.start_offset())), - CudaStorageSlice::U8(slice) => ("is_u8", slice_ptr(slice, ids_l.start_offset())), - CudaStorageSlice::I64(slice) => ("is_i64", slice_ptr(slice, ids_l.start_offset())), - _ => Err(CudaError::UnexpectedDType { - msg: "index_select ids should be u8, u32, or i64", - expected: DType::U32, - got: self.0.dtype(), - }) - .w()?, - }; - let ids_shape = ids_l.shape(); - let ids_dims = ids_shape.dims(); - let ds = dev.clone_htod(&[ids_dims, ids_l.stride()].concat())?; - let src = match src_l.contiguous_offsets() { - Some((o1, o2)) => src.slice(o1..o2), - None => Err(crate::Error::RequiresContiguous { op: "index-select" }.bt())?, - }; - let left_size: usize = src_l.dims()[..self.2].iter().product(); - let right_size: usize = src_l.dims()[self.2 + 1..].iter().product(); - let src_dim_size = src_l.dims()[self.2]; - let ids_dim_size = ids_shape.elem_count(); - let dst_el = ids_shape.elem_count() * left_size * right_size; - let cfg = LaunchConfig::for_num_elems(dst_el as u32); - let func = dev.get_or_load_func(&kernel_name::(name), &kernels::INDEXING)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(dst_el)? }; - let mut builder = func.builder(); - barg!(builder, dst_el); - barg!(builder, ids_dims.len()); - builder.arg(&ds); - barg!(builder, ids); - builder.arg(&src); - builder.arg(&out); - barg!(builder, left_size); - barg!(builder, src_dim_size); - barg!(builder, ids_dim_size); - barg!(builder, right_size); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - Ok(out) - } -} - -struct Gather<'a>(&'a CudaStorage, &'a Layout, usize); -impl Map1 for Gather<'_> { - fn f( - &self, - src: &CudaSlice, - dev: &CudaDevice, - src_l: &Layout, - ) -> Result> { - let ids = &self.0; - let ids_l = &self.1; - let dim = self.2; - let (ids_o1, _) = match ids_l.contiguous_offsets() { - Some(o12) => o12, - None => Err(crate::Error::RequiresContiguous { op: "gather" }.bt())?, - }; - let (name, (ids, _guard)) = match &ids.slice { - CudaStorageSlice::U32(slice) => ("gather_u32", slice_ptr(slice, ids_o1)), - CudaStorageSlice::U8(slice) => ("gather_u8", slice_ptr(slice, ids_o1)), - CudaStorageSlice::I64(slice) => ("gather_i64", slice_ptr(slice, ids_o1)), - _ => Err(CudaError::UnexpectedDType { - msg: "gather ids should be u8/u32/i64", - expected: DType::U32, - got: ids.dtype(), - })?, - }; - let el = ids_l.shape().elem_count(); - let cfg = LaunchConfig::for_num_elems(el as u32); - let src = match src_l.contiguous_offsets() { - Some((o1, o2)) => src.slice(o1..o2), - None => Err(crate::Error::RequiresContiguous { op: "gather" }.bt())?, - }; - let left_sz: usize = src_l.dims()[..dim].iter().product(); - let right_sz: usize = src_l.dims()[dim + 1..].iter().product(); - let src_dim_sz = src_l.dims()[dim]; - let ids_dim_sz = ids_l.dims()[dim]; - let func = dev.get_or_load_func(&kernel_name::(name), &kernels::INDEXING)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(el)? }; - let mut builder = func.builder(); - barg!(builder, el); - barg!(builder, ids); - builder.arg(&src); - builder.arg(&out); - barg!(builder, left_sz); - barg!(builder, src_dim_sz); - barg!(builder, ids_dim_sz); - barg!(builder, right_sz); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - Ok(out) - } -} - -struct IndexAdd<'a>(&'a CudaStorage, &'a Layout, usize); -impl Map2InPlace for IndexAdd<'_> { - fn f( - &self, - dst: &mut CudaSlice, - dst_l: &Layout, - src: &CudaSlice, - src_l: &Layout, - dev: &CudaDevice, - ) -> Result<()> { - let ids = &self.0; - let ids_l = &self.1; - let dim = self.2; - let (ids_o1, _) = match ids_l.contiguous_offsets() { - Some(o12) => o12, - None => Err(crate::Error::RequiresContiguous { op: "index-add" }.bt())?, - }; - let (name, (ids, _guard)) = match &ids.slice { - CudaStorageSlice::U32(slice) => ("ia_u32", slice_ptr(slice, ids_o1)), - CudaStorageSlice::I64(slice) => ("ia_i64", slice_ptr(slice, ids_o1)), - CudaStorageSlice::U8(slice) => ("ia_u8", slice_ptr(slice, ids_o1)), - _ => Err(CudaError::UnexpectedDType { - msg: "index-add ids should be u8/u32/i64", - expected: DType::U32, - got: ids.dtype(), - })?, - }; - let dst = match dst_l.contiguous_offsets() { - Some((o1, o2)) => dst.slice(o1..o2), - None => Err(crate::Error::RequiresContiguous { op: "index-add" }.bt())?, - }; - let src = match src_l.contiguous_offsets() { - Some((o1, o2)) => src.slice(o1..o2), - None => Err(crate::Error::RequiresContiguous { op: "index-add" }.bt())?, - }; - let left_sz: usize = src_l.dims()[..dim].iter().product(); - let right_sz: usize = src_l.dims()[dim + 1..].iter().product(); - let src_dim_sz = src_l.dims()[dim]; - let dst_dim_sz = dst_l.dims()[dim]; - let ids_dim_sz = ids_l.dims()[0]; - let cfg = LaunchConfig::for_num_elems((left_sz * right_sz) as u32); - let func = dev.get_or_load_func(&kernel_name::(name), &kernels::INDEXING)?; - let mut builder = func.builder(); - barg!(builder, ids); - barg!(builder, ids_dim_sz); - builder.arg(&src); - builder.arg(&dst); - barg!(builder, left_sz, src_dim_sz, dst_dim_sz, right_sz); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - Ok(()) - } -} - -struct Scatter<'a>(&'a CudaStorage, &'a Layout, usize); -impl Map2InPlace for Scatter<'_> { - fn f( - &self, - dst: &mut CudaSlice, - dst_l: &Layout, - src: &CudaSlice, - src_l: &Layout, - dev: &CudaDevice, - ) -> Result<()> { - let ids = &self.0; - let ids_l = &self.1; - let dim = self.2; - let (ids_o1, _) = match ids_l.contiguous_offsets() { - Some(o12) => o12, - None => Err(crate::Error::RequiresContiguous { op: "scatter" }.bt())?, - }; - let (name, (ids, _guard)) = match &ids.slice { - CudaStorageSlice::U32(slice) => ("s_u32", slice_ptr(slice, ids_o1)), - CudaStorageSlice::I64(slice) => ("s_i64", slice_ptr(slice, ids_o1)), - CudaStorageSlice::U8(slice) => ("s_u8", slice_ptr(slice, ids_o1)), - _ => Err(CudaError::UnexpectedDType { - msg: "scatter ids should be u8/u32/i64", - expected: DType::U32, - got: ids.dtype(), - })?, - }; - let dst = match dst_l.contiguous_offsets() { - Some((o1, o2)) => dst.slice(o1..o2), - None => Err(crate::Error::RequiresContiguous { op: "scatter" }.bt())?, - }; - let src = match src_l.contiguous_offsets() { - Some((o1, o2)) => src.slice(o1..o2), - None => Err(crate::Error::RequiresContiguous { op: "scatter" }.bt())?, - }; - let left_sz: usize = src_l.dims()[..dim].iter().product(); - let right_sz: usize = src_l.dims()[dim + 1..].iter().product(); - let src_dim_sz = src_l.dims()[dim]; - let dst_dim_sz = dst_l.dims()[dim]; - let cfg = LaunchConfig::for_num_elems((left_sz * right_sz) as u32); - let func = dev.get_or_load_func(&kernel_name::(name), &kernels::INDEXING)?; - let mut builder = func.builder(); - barg!(builder, ids); - builder.arg(&src); - builder.arg(&dst); - barg!(builder, left_sz, src_dim_sz, dst_dim_sz, right_sz); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - Ok(()) - } -} - -struct ScatterAdd<'a>(&'a CudaStorage, &'a Layout, usize); -impl Map2InPlace for ScatterAdd<'_> { - fn f( - &self, - dst: &mut CudaSlice, - dst_l: &Layout, - src: &CudaSlice, - src_l: &Layout, - dev: &CudaDevice, - ) -> Result<()> { - let ids = &self.0; - let ids_l = &self.1; - let dim = self.2; - let (ids_o1, _) = match ids_l.contiguous_offsets() { - Some(o12) => o12, - None => Err(crate::Error::RequiresContiguous { op: "scatter-add" }.bt())?, - }; - let (name, (ids, _guard)) = match &ids.slice { - CudaStorageSlice::U32(slice) => ("sa_u32", slice_ptr(slice, ids_o1)), - CudaStorageSlice::I64(slice) => ("sa_i64", slice_ptr(slice, ids_o1)), - CudaStorageSlice::U8(slice) => ("sa_u8", slice_ptr(slice, ids_o1)), - _ => Err(CudaError::UnexpectedDType { - msg: "scatter-add ids should be u8/u32/i64", - expected: DType::U32, - got: ids.dtype(), - })?, - }; - let dst = match dst_l.contiguous_offsets() { - Some((o1, o2)) => dst.slice(o1..o2), - None => Err(crate::Error::RequiresContiguous { op: "scatter-add" }.bt())?, - }; - let src = match src_l.contiguous_offsets() { - Some((o1, o2)) => src.slice(o1..o2), - None => Err(crate::Error::RequiresContiguous { op: "scatter-add" }.bt())?, - }; - let left_sz: usize = src_l.dims()[..dim].iter().product(); - let right_sz: usize = src_l.dims()[dim + 1..].iter().product(); - let src_dim_sz = src_l.dims()[dim]; - let dst_dim_sz = dst_l.dims()[dim]; - let cfg = LaunchConfig::for_num_elems((left_sz * right_sz) as u32); - let func = dev.get_or_load_func(&kernel_name::(name), &kernels::INDEXING)?; - let mut builder = func.builder(); - barg!(builder, ids); - builder.arg(&src); - builder.arg(&dst); - barg!(builder, left_sz, src_dim_sz, dst_dim_sz, right_sz); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - Ok(()) - } -} - -struct Conv1D<'a>(&'a crate::conv::ParamsConv1D); -impl Map2 for Conv1D<'_> { - fn f( - &self, - inp: &CudaSlice, - inp_l: &Layout, - k: &CudaSlice, - k_l: &Layout, - dev: &CudaDevice, - ) -> Result> { - // Kernel shape: (c_out, c_in_k, k_size) - // Input shape: (b_size, c_in, l_in) or (c_in, l_in) - let p = &self.0; - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(k_l.start_offset()..); - let shape = inp_l.shape(); - let dims = shape.dims(); - let el = shape.elem_count(); - let l_out = p.l_out(); - let dst_el = p.c_out * l_out * p.b_size; - let cfg = LaunchConfig::for_num_elems(dst_el as u32); - let func = dev.get_or_load_func(&kernel_name::("conv1d"), &kernels::CONV)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(dst_el)? }; - let ds = if dims.len() == 3 { - [dims, inp_l.stride(), k_l.dims(), k_l.stride()].concat() - } else if dims.len() == 2 { - [&[1], dims, &[1], inp_l.stride(), k_l.dims(), k_l.stride()].concat() - } else { - crate::bail!("unexpected input shape for conv1d {dims:?}") - }; - let ds = dev.clone_htod(&ds)?; - let mut builder = func.builder(); - barg!(builder, el, l_out, p.stride, p.padding, p.dilation); - builder.arg(&ds); - builder.arg(inp); - builder.arg(k); - builder.arg(&out); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - Ok(out) - } -} - -struct Conv2D<'a>(&'a crate::conv::ParamsConv2D); -impl Map2 for Conv2D<'_> { - fn f( - &self, - inp: &CudaSlice, - inp_l: &Layout, - k: &CudaSlice, - k_l: &Layout, - dev: &CudaDevice, - ) -> Result> { - // Kernel shape: (c_out, c_in_k, h_k, w_k) - // Input shape: (b_size, c_in, h_in, w_in) - let p = &self.0; - let (out_w, out_h) = (p.out_w(), p.out_h()); - let dst_el = p.c_out * out_w * out_h * p.b_size; - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(k_l.start_offset()..); - let shape = inp_l.shape(); - let dims = shape.dims(); - let el = shape.elem_count(); - - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(dst_el)? }; - let cfg = LaunchConfig::for_num_elems(dst_el as u32); - let func = dev.get_or_load_func(&kernel_name::("conv2d"), &kernels::CONV)?; - let ds = if dims.len() == 4 { - [dims, inp_l.stride(), k_l.dims(), k_l.stride()].concat() - } else { - crate::bail!("unexpected input shape for conv2d {dims:?}") - }; - let ds = dev.clone_htod(&ds)?; - let mut builder = func.builder(); - barg!(builder, el, out_w, out_h, p.stride, p.padding, p.dilation); - builder.arg(&ds); - builder.arg(inp); - builder.arg(k); - builder.arg(&out); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - Ok(out) - } -} - -struct Col2Im1D { - stride: usize, -} - -impl Map1 for Col2Im1D { - fn f( - &self, - col: &CudaSlice, - dev: &CudaDevice, - l: &Layout, - ) -> Result> { - let (b_size, l_in, c_out, k_size) = l.shape().dims4()?; - let stride = self.stride; - let l_out = (l_in - 1) * stride + k_size; - let dst_el = b_size * c_out * l_out; - let mut im = unsafe { dev.alloc::(dst_el)? }; - - let cfg = LaunchConfig::for_num_elems(dst_el as u32); - let func = dev.get_or_load_func(&kernel_name::("col2im1d"), &kernels::CONV)?; - let mut builder = func.builder(); - barg!(builder, dst_el, l_out, l_in, c_out, k_size, stride); - builder.arg(col); - builder.arg(&mut im); - unsafe { builder.launch(cfg) }.w()?; - Ok(im) - } -} - -struct ConvTranspose1D<'a>(&'a crate::conv::ParamsConvTranspose1D); -impl Map2 for ConvTranspose1D<'_> { - fn f( - &self, - inp: &CudaSlice, - inp_l: &Layout, - k: &CudaSlice, - k_l: &Layout, - dev: &CudaDevice, - ) -> Result> { - // Kernel shape: (c_in_k, c_out, l_k) - // Input shape: (b_size, c_in, l_in) - let p = &self.0; - let l_out = p.l_out(); - let dst_el = p.c_out * l_out * p.b_size; - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(k_l.start_offset()..); - let shape = inp_l.shape(); - let dims = shape.dims(); - let el = shape.elem_count(); - - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(dst_el)? }; - let cfg = LaunchConfig::for_num_elems(dst_el as u32); - let func = dev.get_or_load_func(&kernel_name::("conv_transpose1d"), &kernels::CONV)?; - let ds = if dims.len() == 3 { - [dims, inp_l.stride(), k_l.dims(), k_l.stride()].concat() - } else { - crate::bail!("unexpected input shape for conv_transpose1d {dims:?}") - }; - let ds = dev.clone_htod(&ds)?; - let mut builder = func.builder(); - barg!(builder, el); - barg!(builder, l_out); - barg!(builder, p.stride); - barg!(builder, p.padding); - barg!(builder, p.output_padding); - barg!(builder, p.dilation); - builder.arg(&ds); - builder.arg(inp); - builder.arg(k); - builder.arg(&out); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - Ok(out) - } -} - -struct ConvTranspose2D<'a>(&'a crate::conv::ParamsConvTranspose2D); -impl Map2 for ConvTranspose2D<'_> { - fn f( - &self, - inp: &CudaSlice, - inp_l: &Layout, - k: &CudaSlice, - k_l: &Layout, - dev: &CudaDevice, - ) -> Result> { - // Kernel shape: (c_in_k, c_out, h_k, w_k) - // Input shape: (b_size, c_in, h_in, w_in) - let p = &self.0; - let (out_w, out_h) = (p.out_w(), p.out_h()); - let dst_el = p.c_out * out_w * out_h * p.b_size; - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(k_l.start_offset()..); - let shape = inp_l.shape(); - let dims = shape.dims(); - let el = shape.elem_count(); - - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(dst_el)? }; - let cfg = LaunchConfig::for_num_elems(dst_el as u32); - let func = dev.get_or_load_func(&kernel_name::("conv_transpose2d"), &kernels::CONV)?; - let ds = if dims.len() == 4 { - [dims, inp_l.stride(), k_l.dims(), k_l.stride()].concat() - } else { - crate::bail!("unexpected input shape for conv_transpose2d {dims:?}") - }; - let ds = dev.clone_htod(&ds)?; - let mut builder = func.builder(); - barg!(builder, el); - barg!(builder, out_w); - barg!(builder, out_h); - barg!(builder, p.stride); - barg!(builder, p.padding); - barg!(builder, p.output_padding); - barg!(builder, p.dilation); - builder.arg(&ds); - builder.arg(inp); - builder.arg(k); - builder.arg(&out); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - Ok(out) - } -} - -enum PoolOp { - Max, - Avg, -} - -struct Pool2D { - w_k: usize, - h_k: usize, - w_stride: usize, - h_stride: usize, - op: PoolOp, -} - -impl Map1 for Pool2D { - fn f( - &self, - inp: &CudaSlice, - dev: &CudaDevice, - inp_l: &Layout, - ) -> Result> { - // Input shape: (b_size, c, h, w) - let inp = &inp.slice(inp_l.start_offset()..); - let shape = inp_l.shape(); - let dims = shape.dims(); - let ds = if dims.len() == 4 { - [dims, inp_l.stride()].concat() - } else { - crate::bail!("unexpected input shape for pool {dims:?}") - }; - let el = shape.elem_count(); - let out_w = (dims[2] - self.w_k) / self.w_stride + 1; - let out_h = (dims[3] - self.h_k) / self.h_stride + 1; - let dst_el = out_w * out_h * dims[0] * dims[1]; - let cfg = LaunchConfig::for_num_elems(dst_el as u32); - let kname = match self.op { - PoolOp::Max => "max_pool2d", - PoolOp::Avg => "avg_pool2d", - }; - let func = dev.get_or_load_func(&kernel_name::(kname), &kernels::CONV)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(dst_el)? }; - let ds = dev.clone_htod(&ds)?; - let mut builder = func.builder(); - barg!(builder, el); - barg!(builder, self.w_k); - barg!(builder, self.h_k); - barg!(builder, self.w_stride); - barg!(builder, self.h_stride); - builder.arg(&ds); - builder.arg(inp); - builder.arg(&out); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - Ok(out) - } -} - -struct UpsampleNearest2D(usize, usize); -impl Map1 for UpsampleNearest2D { - fn f( - &self, - inp: &CudaSlice, - dev: &CudaDevice, - inp_l: &Layout, - ) -> Result> { - // Input shape: (b_size, c, h, w) - let inp = &inp.slice(inp_l.start_offset()..); - let shape = inp_l.shape(); - let dims = shape.dims(); - let ds = if dims.len() == 4 { - [dims, inp_l.stride()].concat() - } else { - crate::bail!("unexpected input shape for upsample {dims:?}") - }; - let (out_w, out_h) = (self.0, self.1); - let dst_el = out_w * out_h * dims[0] * dims[1]; - let cfg = LaunchConfig::for_num_elems(dst_el as u32); - let func = dev.get_or_load_func(&kernel_name::("upsample_nearest2d"), &kernels::CONV)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(dst_el)? }; - let ds = dev.clone_htod(&ds)?; - let scale_w = dims[2] as f64 / out_w as f64; - let scale_h = dims[3] as f64 / out_h as f64; - let mut builder = func.builder(); - barg!(builder, out_w); - barg!(builder, out_h); - barg!(builder, scale_w); - barg!(builder, scale_h); - builder.arg(&ds); - builder.arg(inp); - builder.arg(&out); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - Ok(out) - } -} - -struct UpsampleBilinear2D { - out_w: usize, - out_h: usize, - align_corners: bool, - scale_h_factor: Option, - scale_w_factor: Option, -} - -impl Map1 for UpsampleBilinear2D { - fn f( - &self, - inp: &CudaSlice, - dev: &CudaDevice, - inp_l: &Layout, - ) -> Result> { - let inp = &inp.slice(inp_l.start_offset()..); - let shape = inp_l.shape(); - let dims = shape.dims(); - let ds = if dims.len() == 4 { - [dims, inp_l.stride()].concat() - } else { - crate::bail!("unexpected input shape for upsample_bilinear2d {dims:?}") - }; - - let (out_w, out_h) = (self.out_w, self.out_h); - let dst_el = out_w * out_h * dims[0] * dims[1]; - let cfg = LaunchConfig::for_num_elems(dst_el as u32); - let func = - dev.get_or_load_func(&kernel_name::("upsample_bilinear2d"), &kernels::CONV)?; - - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(dst_el)? }; - let ds = dev.clone_htod(&ds)?; - - let mut builder = func.builder(); - barg!(builder, out_w); - barg!(builder, out_h); - barg!(builder, self.align_corners); - barg!(builder, self.scale_h_factor.is_some()); - barg!(builder, self.scale_h_factor.unwrap_or(0.0)); - barg!(builder, self.scale_w_factor.is_some()); - barg!(builder, self.scale_w_factor.unwrap_or(0.0)); - builder.arg(&ds); - builder.arg(inp); - builder.arg(&out); - - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - Ok(out) - } -} - -struct WhereCond<'a>(&'a CudaStorage, &'a Layout); -impl Map2 for WhereCond<'_> { - fn f( - &self, - t: &CudaSlice, - layout_t: &Layout, - f: &CudaSlice, - layout_f: &Layout, - dev: &CudaDevice, - ) -> Result> { - let ids_l = &self.1; - let ((ids, _guard), name) = match &self.0.slice { - CudaStorageSlice::U8(slice) => { - let ptr = slice_ptr(slice, ids_l.start_offset()); - (ptr, "where_u8") - } - CudaStorageSlice::U32(slice) => { - let ptr = slice_ptr(slice, ids_l.start_offset()); - (ptr, "where_u32") - } - CudaStorageSlice::I64(slice) => { - let ptr = slice_ptr(slice, ids_l.start_offset()); - (ptr, "where_i64") - } - _ => Err(CudaError::UnexpectedDType { - msg: "where conditions should be u8/u32/i64", - expected: DType::U32, - got: self.0.dtype(), - }) - .w()?, - }; - let shape = ids_l.shape(); - let dims = shape.dims(); - let el = shape.elem_count(); - let cfg = LaunchConfig::for_num_elems(el as u32); - let ds = - dev.clone_htod(&[dims, ids_l.stride(), layout_t.stride(), layout_f.stride()].concat())?; - let t = &t.slice(layout_t.start_offset()..); - let f = &f.slice(layout_f.start_offset()..); - let func = dev.get_or_load_func(&kernel_name::(name), &kernels::TERNARY)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(el)? }; - let mut builder = func.builder(); - barg!(builder, el); - barg!(builder, dims.len()); - builder.arg(&ds); - barg!(builder, ids); - builder.arg(t); - builder.arg(f); - builder.arg(&out); - // SAFETY: ffi - unsafe { builder.launch(cfg) }.w()?; - Ok(out) - } -} - -impl Map2 for U { - fn f( - &self, - lhs: &CudaSlice, - lhs_l: &Layout, - rhs: &CudaSlice, - rhs_l: &Layout, - dev: &CudaDevice, - ) -> Result> { - let shape = lhs_l.shape(); - let dims = shape.dims(); - let elem_count = shape.elem_count(); - let cfg = LaunchConfig::for_num_elems(elem_count as u32); - let dims_and_strides = if lhs_l.is_contiguous() && rhs_l.is_contiguous() { - SlicePtrOrNull::Null - } else { - SlicePtrOrNull::Ptr(dev.clone_htod(&[dims, lhs_l.stride(), rhs_l.stride()].concat())?) - }; - let lhs = &lhs.slice(lhs_l.start_offset()..); - let rhs = &rhs.slice(rhs_l.start_offset()..); - let func = dev.get_or_load_func(&kernel_name::(U::KERNEL), &kernels::BINARY)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(elem_count)? }; - let mut builder = func.builder(); - barg!(builder, elem_count); - barg!(builder, dims.len()); - dims_and_strides.builder_arg(&mut builder); - builder.arg(lhs); - builder.arg(rhs); - builder.arg(&out); - // SAFETY: ffi - unsafe { builder.launch(cfg) }.w()?; - Ok(out) - } -} - -struct Cmp(CmpOp); -impl Map2Any for Cmp { - fn f( - &self, - lhs: &CudaSlice, - lhs_l: &Layout, - rhs: &CudaSlice, - rhs_l: &Layout, - dev: &CudaDevice, - ) -> Result { - let shape = lhs_l.shape(); - let dims = shape.dims(); - let elem_count = shape.elem_count(); - let cfg = LaunchConfig::for_num_elems(elem_count as u32); - let dims_and_strides = if lhs_l.is_contiguous() && rhs_l.is_contiguous() { - SlicePtrOrNull::Null - } else { - SlicePtrOrNull::Ptr(dev.clone_htod(&[dims, lhs_l.stride(), rhs_l.stride()].concat())?) - }; - let lhs = &lhs.slice(lhs_l.start_offset()..); - let rhs = &rhs.slice(rhs_l.start_offset()..); - let name = match self.0 { - CmpOp::Eq => "eq", - CmpOp::Ne => "ne", - CmpOp::Lt => "lt", - CmpOp::Le => "le", - CmpOp::Gt => "gt", - CmpOp::Ge => "ge", - }; - let func = dev.get_or_load_func(&kernel_name::(name), &kernels::BINARY)?; - // SAFETY: Set later by running the kernel. - let out = unsafe { dev.alloc::(elem_count)? }; - let mut builder = func.builder(); - barg!(builder, elem_count); - barg!(builder, dims.len()); - dims_and_strides.builder_arg(&mut builder); - builder.arg(lhs); - builder.arg(rhs); - builder.arg(&out); - // SAFETY: ffi - unsafe { builder.launch(cfg) }.w()?; - Ok(S::U8(out)) - } -} - -fn slice_src_and_dst<'a, T>( - src: &'a CudaSlice, - src_l: &Layout, - dst: &'a mut CudaSlice, - dst_offset: usize, -) -> ( - cudarc::driver::CudaView<'a, T>, - cudarc::driver::CudaViewMut<'a, T>, -) { - let src_offset = src_l.start_offset(); - let to_copy = dst - .len() - .saturating_sub(dst_offset) - .min(src.len().saturating_sub(src_offset)); - let src = src.slice(src_offset..src_offset + to_copy); - let dst = dst.slice_mut(dst_offset..dst_offset + to_copy); - (src, dst) -} - -#[derive(Debug)] -pub struct CudaStorage { - pub slice: CudaStorageSlice, - pub device: CudaDevice, -} - -pub trait CudaDType: Sized { - fn as_cuda_slice(s: &CudaStorage) -> Result<&CudaSlice>; - fn as_cuda_slice_mut(s: &mut CudaStorage) -> Result<&mut CudaSlice>; - fn wrap_cuda_slice(s: CudaSlice, dev: CudaDevice) -> CudaStorage; -} - -macro_rules! cuda_dtype { - ($ty:ty, $dtype:ident) => { - impl CudaDType for $ty { - fn as_cuda_slice(s: &CudaStorage) -> Result<&CudaSlice> { - match &s.slice { - CudaStorageSlice::$dtype(data) => Ok(&data), - _ => Err(crate::Error::UnexpectedDType { - expected: DType::$dtype, - got: s.dtype(), - msg: "unexpected dtype", - } - .bt()), - } - } - - fn as_cuda_slice_mut(s: &mut CudaStorage) -> Result<&mut CudaSlice> { - match s.slice { - CudaStorageSlice::$dtype(ref mut data) => Ok(data), - _ => Err(crate::Error::UnexpectedDType { - expected: DType::$dtype, - got: s.dtype(), - msg: "unexpected dtype", - } - .bt()), - } - } - - fn wrap_cuda_slice(slice: CudaSlice, device: CudaDevice) -> CudaStorage { - let slice = CudaStorageSlice::$dtype(slice); - CudaStorage { slice, device } - } - } - }; -} -cuda_dtype!(u8, U8); -cuda_dtype!(u32, U32); -cuda_dtype!(i16, I16); -cuda_dtype!(i32, I32); -cuda_dtype!(i64, I64); -cuda_dtype!(f16, F16); -cuda_dtype!(bf16, BF16); -cuda_dtype!(f32, F32); -cuda_dtype!(f64, F64); -cuda_dtype!(float8::F8E4M3, F8E4M3); - -impl CudaStorage { - pub fn wrap_cuda_slice(slice: CudaSlice, device: CudaDevice) -> CudaStorage { - T::wrap_cuda_slice(slice, device) - } - - pub fn as_cuda_slice(&self) -> Result<&CudaSlice> { - T::as_cuda_slice(self) - } - - pub fn as_cuda_slice_mut(&mut self) -> Result<&mut CudaSlice> { - T::as_cuda_slice_mut(self) - } - - pub fn transfer_to_device(&self, dst: &CudaDevice) -> Result { - let dst_stream = dst.cuda_stream(); - let storage_slice = match self.dtype() { - DType::U8 => { - let cuda_slice = self.as_cuda_slice::()?; - let result = dst_stream.clone_dtod(cuda_slice).w()?; - CudaStorageSlice::U8(result) - } - DType::U32 => { - let cuda_slice = self.as_cuda_slice::()?; - let result = dst_stream.clone_dtod(cuda_slice).w()?; - CudaStorageSlice::U32(result) - } - DType::I16 => { - let cuda_slice = self.as_cuda_slice::()?; - let result = dst_stream.clone_dtod(cuda_slice).w()?; - CudaStorageSlice::I16(result) - } - DType::I32 => { - let cuda_slice = self.as_cuda_slice::()?; - let result = dst_stream.clone_dtod(cuda_slice).w()?; - CudaStorageSlice::I32(result) - } - DType::I64 => { - let cuda_slice = self.as_cuda_slice::()?; - let result = dst_stream.clone_dtod(cuda_slice).w()?; - CudaStorageSlice::I64(result) - } - DType::BF16 => { - let cuda_slice = self.as_cuda_slice::()?; - let result = dst_stream.clone_dtod(cuda_slice).w()?; - CudaStorageSlice::BF16(result) - } - DType::F16 => { - let cuda_slice = self.as_cuda_slice::()?; - let result = dst_stream.clone_dtod(cuda_slice).w()?; - CudaStorageSlice::F16(result) - } - DType::F32 => { - let cuda_slice = self.as_cuda_slice::()?; - let result = dst_stream.clone_dtod(cuda_slice).w()?; - CudaStorageSlice::F32(result) - } - DType::F64 => { - let cuda_slice = self.as_cuda_slice::()?; - let result = dst_stream.clone_dtod(cuda_slice).w()?; - CudaStorageSlice::F64(result) - } - DType::F8E4M3 => { - let cuda_slice = self.as_cuda_slice::()?; - let result = dst_stream.clone_dtod(cuda_slice).w()?; - CudaStorageSlice::F8E4M3(result) - } - DType::F6E2M3 => { - let cuda_slice = self.as_cuda_slice::()?; - let result = dst_stream.clone_dtod(cuda_slice).w()?; - CudaStorageSlice::F6E2M3(result) - } - DType::F6E3M2 => { - let cuda_slice = self.as_cuda_slice::()?; - let result = dst_stream.clone_dtod(cuda_slice).w()?; - CudaStorageSlice::F6E3M2(result) - } - DType::F4 => { - let cuda_slice = self.as_cuda_slice::()?; - let result = dst_stream.clone_dtod(cuda_slice).w()?; - CudaStorageSlice::F4(result) - } - DType::F8E8M0 => { - let cuda_slice = self.as_cuda_slice::()?; - let result = dst_stream.clone_dtod(cuda_slice).w()?; - CudaStorageSlice::F8E8M0(result) - } - }; - - Ok(Self { - slice: storage_slice, - device: dst.clone(), - }) - } -} - -fn gemm_config( - alpha: T, - beta: T, - (b, m, n, k): (usize, usize, usize, usize), - lhs_l: &Layout, - rhs_l: &Layout, -) -> Result> { - // https://docs.nvidia.com/cuda/cublas/index.html#cublas-t-gemm - use cudarc::cublas::sys::cublasOperation_t; - - let lhs_stride = lhs_l.stride(); - let rhs_stride = rhs_l.stride(); - let rhs_m1 = rhs_stride[rhs_stride.len() - 1]; - let rhs_m2 = rhs_stride[rhs_stride.len() - 2]; - let lhs_m1 = lhs_stride[lhs_stride.len() - 1]; - let lhs_m2 = lhs_stride[lhs_stride.len() - 2]; - // The a tensor has dims batching, k, n (rhs) - // We also allow for the case where the stride on the minor dimension is not as expected but - // there is a single element. - let (lda, transa) = if (rhs_m1 == 1 || n == 1) && (rhs_m2 == n || k == 1) { - (n as i32, cublasOperation_t::CUBLAS_OP_N) - } else if (rhs_m1 == k || n == 1) && (rhs_m2 == 1 || k == 1) { - (k as i32, cublasOperation_t::CUBLAS_OP_T) - } else { - Err(CudaError::MatMulNonContiguous { - lhs_stride: lhs_l.clone(), - rhs_stride: rhs_l.clone(), - mnk: (m, n, k), - })? - }; - // The b tensor has dims batching, m, k (lhs) - // We also allow for the case where the stride on the minor dimension is not as expected but - // there is a single element. - let (ldb, transb) = if (lhs_m1 == 1 || k == 1) && (lhs_m2 == k || m == 1) { - (k as i32, cublasOperation_t::CUBLAS_OP_N) - } else if (lhs_m1 == m || k == 1) && (lhs_m2 == 1 || m == 1) { - (m as i32, cublasOperation_t::CUBLAS_OP_T) - } else { - Err(CudaError::MatMulNonContiguous { - lhs_stride: lhs_l.clone(), - rhs_stride: rhs_l.clone(), - mnk: (m, n, k), - })? - }; - // The setup below was copied from: - // https://github.com/lebedov/scikit-cuda/blob/7e7300474286019c917a6c8a4bca59405c64fbce/tests/test_cublas.py#L531 - let gemm = GemmConfig { - alpha, - beta, - m: n as i32, - n: m as i32, - k: k as i32, - lda, - ldb, - ldc: n as i32, - transa, - transb, - }; - - let stride_b: usize = match lhs_stride[..lhs_stride.len() - 2] { - [s1, stride] if s1 == stride * lhs_l.dims()[1] => stride, - [_, stride] if lhs_l.dims()[0] == 1 => stride, - [stride, _] if lhs_l.dims()[1] == 1 => stride, - [stride] => stride, - [] => m * k, - _ => Err(CudaError::MatMulNonContiguous { - lhs_stride: lhs_l.clone(), - rhs_stride: rhs_l.clone(), - mnk: (m, n, k), - })?, - }; - let stride_a: usize = match rhs_stride[..rhs_stride.len() - 2] { - [s1, stride] if s1 == stride * rhs_l.dims()[1] => stride, - [_, stride] if rhs_l.dims()[0] == 1 => stride, - [stride, _] if rhs_l.dims()[1] == 1 => stride, - [stride] => stride, - [] => n * k, - _ => Err(CudaError::MatMulNonContiguous { - lhs_stride: lhs_l.clone(), - rhs_stride: rhs_l.clone(), - mnk: (m, n, k), - })?, - }; - Ok(StridedBatchedConfig { - batch_size: b as i32, - gemm, - stride_a: stride_a as i64, - stride_b: stride_b as i64, - stride_c: (m * n) as i64, - }) -} - -impl BackendStorage for CudaStorage { - type Device = CudaDevice; - - fn try_clone(&self, layout: &Layout) -> Result { - let slice = Clone.map(&self.slice, self.device(), layout)?; - let device = self.device.clone(); - Ok(Self { slice, device }) - } - - fn dtype(&self) -> DType { - match self.slice { - CudaStorageSlice::U8(_) => DType::U8, - CudaStorageSlice::U32(_) => DType::U32, - CudaStorageSlice::I16(_) => DType::I16, - CudaStorageSlice::I32(_) => DType::I32, - CudaStorageSlice::I64(_) => DType::I64, - CudaStorageSlice::BF16(_) => DType::BF16, - CudaStorageSlice::F16(_) => DType::F16, - CudaStorageSlice::F32(_) => DType::F32, - CudaStorageSlice::F64(_) => DType::F64, - CudaStorageSlice::F8E4M3(_) => DType::F8E4M3, - CudaStorageSlice::F6E2M3(_) => DType::F6E2M3, - CudaStorageSlice::F6E3M2(_) => DType::F6E3M2, - CudaStorageSlice::F4(_) => DType::F4, - CudaStorageSlice::F8E8M0(_) => DType::F8E8M0, - } - } - - fn device(&self) -> &CudaDevice { - &self.device - } - - fn const_set(&mut self, s: crate::scalar::Scalar, layout: &Layout) -> Result<()> { - let dev = &self.device; - let shape = layout.shape(); - let dims = shape.dims(); - let el_count = shape.elem_count(); - let cfg = LaunchConfig::for_num_elems(el_count as u32); - let ds = SlicePtrOrNull::params_from_layout(dev, layout)?; - let src_o = layout.start_offset(); - let ((src, _guard_src), kernel_name) = match &mut self.slice { - S::U8(s) => (slice_ptr(s, src_o), "const_set_u8"), - S::U32(s) => (slice_ptr(s, src_o), "const_set_u32"), - S::I16(s) => (slice_ptr(s, src_o), "const_set_i16"), - S::I32(s) => (slice_ptr(s, src_o), "const_set_i32"), - S::I64(s) => (slice_ptr(s, src_o), "const_set_i64"), - S::BF16(s) => (slice_ptr(s, src_o), "const_set_bf16"), - S::F16(s) => (slice_ptr(s, src_o), "const_set_f16"), - S::F32(s) => (slice_ptr(s, src_o), "const_set_f32"), - S::F64(s) => (slice_ptr(s, src_o), "const_set_f64"), - S::F8E4M3(s) => (slice_ptr(s, src_o), "const_set_f8_e4m3"), - S::F4(_) | S::F6E2M3(_) | S::F6E3M2(_) | S::F8E8M0(_) => { - return Err(CudaError::UnsupportedDtype { - dtype: self.dtype(), - op: "const_set", - } - .into()); - } - }; - - let func = dev.get_or_load_func(kernel_name, &kernels::FILL)?; - let mut builder = func.builder(); - barg!(builder, el_count); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - s.builder_arg(&mut builder); - barg!(builder, src); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - Ok(()) - } - - fn to_dtype(&self, layout: &Layout, dtype: DType) -> Result { - let shape = layout.shape(); - let dims = shape.dims(); - let el = shape.elem_count(); - let cfg = LaunchConfig::for_num_elems(el as u32); - let dev = self.device(); - let ds = SlicePtrOrNull::params_from_layout(dev, layout)?; - let start_o = layout.start_offset(); - // This returns an i64 rather than a &i64, this is useful to get around some temporary - // lifetime issue and is safe as long as self.slice does not go out of scope before inp - // is used. - let (inp, _guard) = match &self.slice { - CudaStorageSlice::U8(inp) => slice_ptr(inp, start_o), - CudaStorageSlice::U32(inp) => slice_ptr(inp, start_o), - CudaStorageSlice::I16(inp) => slice_ptr(inp, start_o), - CudaStorageSlice::I32(inp) => slice_ptr(inp, start_o), - CudaStorageSlice::I64(inp) => slice_ptr(inp, start_o), - CudaStorageSlice::BF16(inp) => slice_ptr(inp, start_o), - CudaStorageSlice::F16(inp) => slice_ptr(inp, start_o), - CudaStorageSlice::F32(inp) => slice_ptr(inp, start_o), - CudaStorageSlice::F64(inp) => slice_ptr(inp, start_o), - CudaStorageSlice::F8E4M3(inp) => slice_ptr(inp, start_o), - CudaStorageSlice::F4(_) - | CudaStorageSlice::F6E2M3(_) - | CudaStorageSlice::F6E3M2(_) - | CudaStorageSlice::F8E8M0(_) => { - return Err(CudaError::UnsupportedDtype { - dtype: self.dtype(), - op: "to_dtype", - } - .into()); - } - }; - let inp = &inp; - - let kernel_name = format!("cast_{}_{}", self.dtype().as_str(), dtype.as_str()); - let func = dev.get_or_load_func(&kernel_name, &kernels::CAST)?; - let slice = match dtype { - DType::U8 => { - let out = unsafe { dev.alloc::(el)? }; - let mut builder = func.builder(); - barg!(builder, el); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - barg!(builder, *inp); - builder.arg(&out); - unsafe { builder.launch(cfg) }.w()?; - CudaStorageSlice::U8(out) - } - DType::U32 => { - let out = unsafe { dev.alloc::(el)? }; - let mut builder = func.builder(); - barg!(builder, el); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - barg!(builder, *inp); - builder.arg(&out); - unsafe { builder.launch(cfg) }.w()?; - CudaStorageSlice::U32(out) - } - DType::I64 => { - let out = unsafe { dev.alloc::(el)? }; - let mut builder = func.builder(); - barg!(builder, el); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - barg!(builder, *inp); - builder.arg(&out); - unsafe { builder.launch(cfg) }.w()?; - CudaStorageSlice::I64(out) - } - DType::BF16 => { - let out = unsafe { dev.alloc::(el)? }; - let mut builder = func.builder(); - barg!(builder, el); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - barg!(builder, *inp); - builder.arg(&out); - unsafe { builder.launch(cfg) }.w()?; - CudaStorageSlice::BF16(out) - } - DType::F16 => { - let out = unsafe { dev.alloc::(el)? }; - let mut builder = func.builder(); - barg!(builder, el); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - barg!(builder, *inp); - builder.arg(&out); - unsafe { builder.launch(cfg) }.w()?; - CudaStorageSlice::F16(out) - } - DType::F32 => { - let out = unsafe { dev.alloc::(el)? }; - let mut builder = func.builder(); - barg!(builder, el); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - barg!(builder, *inp); - builder.arg(&out); - unsafe { builder.launch(cfg) }.w()?; - CudaStorageSlice::F32(out) - } - DType::F64 => { - let out = unsafe { dev.alloc::(el)? }; - let mut builder = func.builder(); - barg!(builder, el); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - barg!(builder, *inp); - builder.arg(&out); - unsafe { builder.launch(cfg) }.w()?; - CudaStorageSlice::F64(out) - } - DType::F8E4M3 => { - let out = unsafe { dev.alloc::(el)? }; - let mut builder = func.builder(); - barg!(builder, el); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - barg!(builder, *inp); - builder.arg(&out); - unsafe { builder.launch(cfg) }.w()?; - CudaStorageSlice::F8E4M3(out) - } - DType::I16 | DType::I32 => { - return Err(CudaError::InternalError("i16,i32 dtypes are not supported").into()) - } - DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { - return Err( - CudaError::InternalError("Dummy types not supported in CUDA backend").into(), - ) - } - }; - Ok(Self { - slice, - device: dev.clone(), - }) - } - - fn affine(&self, layout: &Layout, mul: f64, add: f64) -> Result { - let device = self.device().clone(); - let slice = Affine(mul, add).map(&self.slice, &device, layout)?; - Ok(Self { slice, device }) - } - - fn powf(&self, layout: &Layout, e: f64) -> Result { - let device = self.device().clone(); - let slice = Powf(e).map(&self.slice, &device, layout)?; - Ok(Self { slice, device }) - } - - fn elu(&self, layout: &Layout, alpha: f64) -> Result { - let device = self.device().clone(); - let slice = Elu(alpha).map(&self.slice, &device, layout)?; - Ok(Self { slice, device }) - } - - fn reduce_op(&self, op: ReduceOp, layout: &Layout, sum_dims: &[usize]) -> Result { - let device = self.device().clone(); - let slice = FastReduce(sum_dims, op).map(&self.slice, &device, layout)?; - Ok(Self { slice, device }) - } - - fn cmp(&self, op: CmpOp, rhs: &Self, lhs_l: &Layout, rhs_l: &Layout) -> Result { - let device = self.device().clone(); - let slice = Cmp(op).map(&self.slice, lhs_l, &rhs.slice, rhs_l, &device)?; - Ok(Self { slice, device }) - } - - fn unary_impl(&self, layout: &Layout) -> Result { - let device = self.device().clone(); - let slice = U::V.map(&self.slice, &device, layout)?; - Ok(Self { slice, device }) - } - - fn binary_impl( - &self, - rhs: &Self, - lhs_l: &Layout, - rhs_l: &Layout, - ) -> Result { - let device = self.device().clone(); - let slice = B::V.map(&self.slice, lhs_l, &rhs.slice, rhs_l, &device)?; - Ok(Self { slice, device }) - } - - fn to_cpu_storage(&self) -> Result { - match &self.slice { - CudaStorageSlice::U8(slice) => { - let cpu_storage = slice.stream().clone_dtoh(slice).w()?; - Ok(CpuStorage::U8(cpu_storage)) - } - CudaStorageSlice::U32(slice) => { - let cpu_storage = slice.stream().clone_dtoh(slice).w()?; - Ok(CpuStorage::U32(cpu_storage)) - } - CudaStorageSlice::I16(slice) => { - let cpu_storage = slice.stream().clone_dtoh(slice).w()?; - Ok(CpuStorage::I16(cpu_storage)) - } - CudaStorageSlice::I32(slice) => { - let cpu_storage = slice.stream().clone_dtoh(slice).w()?; - Ok(CpuStorage::I32(cpu_storage)) - } - CudaStorageSlice::I64(slice) => { - let cpu_storage = slice.stream().clone_dtoh(slice).w()?; - Ok(CpuStorage::I64(cpu_storage)) - } - CudaStorageSlice::BF16(slice) => { - let cpu_storage = slice.stream().clone_dtoh(slice).w()?; - Ok(CpuStorage::BF16(cpu_storage)) - } - CudaStorageSlice::F16(slice) => { - let cpu_storage = slice.stream().clone_dtoh(slice).w()?; - Ok(CpuStorage::F16(cpu_storage)) - } - CudaStorageSlice::F32(slice) => { - let cpu_storage = slice.stream().clone_dtoh(slice).w()?; - Ok(CpuStorage::F32(cpu_storage)) - } - CudaStorageSlice::F64(slice) => { - let cpu_storage = slice.stream().clone_dtoh(slice).w()?; - Ok(CpuStorage::F64(cpu_storage)) - } - CudaStorageSlice::F8E4M3(slice) => { - let cpu_storage = slice.stream().clone_dtoh(slice).w()?; - Ok(CpuStorage::F8E4M3(cpu_storage)) - } - CudaStorageSlice::F4(_) - | CudaStorageSlice::F6E2M3(_) - | CudaStorageSlice::F6E3M2(_) - | CudaStorageSlice::F8E8M0(_) => Err(CudaError::UnsupportedDtype { - dtype: self.dtype(), - op: "to_cpu_storage", - } - .into()), - } - } - - fn where_cond( - &self, - layout: &Layout, - t: &Self, - t_l: &Layout, - f: &Self, - f_l: &Layout, - ) -> Result { - let device = self.device().clone(); - let slice = WhereCond(self, layout).map(&t.slice, t_l, &f.slice, f_l, &device)?; - Ok(Self { slice, device }) - } - - #[cfg(not(feature = "cudnn"))] - fn conv1d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConv1D, - ) -> Result { - const USE_IM2COL_CONV1D: bool = true; - - let device = self.device().clone(); - if !USE_IM2COL_CONV1D { - let slice = Conv1D(params).map(&self.slice, l, &kernel.slice, kernel_l, &device)?; - return Ok(Self { slice, device }); - } - - let col = Im2Col1D { - l_k: params.k_size, - stride: params.stride, - dilation: params.dilation, - padding: params.padding, - } - .map(&self.slice, &device, l)?; - let col = Self { slice: col, device }; - let l_out = params.l_out(); - let b = params.b_size; - let n = params.c_out; - let k = params.k_size * params.c_in; - let m = l_out; - let col_l = Layout::contiguous((b * m, k)); - let res = if kernel_l.is_contiguous() { - let kernel_l = - Layout::contiguous_with_offset((n, k), kernel_l.start_offset()).transpose(0, 1)?; - col.matmul(kernel, (1, b * m, n, k), &col_l, &kernel_l)? - } else { - // Make the kernel contiguous if not already the case. - let mut kernel_c = unsafe { - self.device() - .alloc_uninit(kernel_l.shape(), kernel.dtype())? - }; - kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?; - let kernel_l = - Layout::contiguous_with_offset((n, k), kernel_l.start_offset()).transpose(0, 1)?; - col.matmul(kernel, (1, b * m, n, k), &col_l, &kernel_l)? - }; - let res_l = Layout::contiguous((b, l_out, n)).transpose(1, 2)?; - let mut res_t = unsafe { self.device().alloc_uninit(res_l.shape(), res.dtype())? }; - res.copy_strided_src(&mut res_t, 0, &res_l)?; - Ok(res_t) - } - - #[cfg(feature = "cudnn")] - fn conv1d( - &self, - inp_l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConv1D, - ) -> Result { - let device = self.device().clone(); - if !kernel_l.is_contiguous() { - let slice = Conv1D(params).map(&self.slice, inp_l, &kernel.slice, kernel_l, &device)?; - return Ok(Self { slice, device }); - } - let l_out = params.l_out(); - let dst_el = params.c_out * l_out * params.b_size; - let slice = match (&self.slice, &kernel.slice) { - (S::U8(inp), S::U8(k)) => { - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(kernel_l.start_offset()..); - let mut out = unsafe { device.alloc::(dst_el)? }; - crate::cudnn::launch_conv1d::(inp, inp_l, k, &mut out, params, &device) - .map_err(crate::Error::wrap)?; - S::U8(out) - } - (S::BF16(inp), S::BF16(k)) => { - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(kernel_l.start_offset()..); - let mut out = unsafe { device.alloc::(dst_el)? }; - // Only PSEUDO_BFLOAT16_CONFIG is supported in cudnn, there is no "true bfloat16" - // version. - // https://docs.nvidia.com/deeplearning/cudnn/latest/api/cudnn-cnn-library.html#id88 - crate::cudnn::launch_conv1d::(inp, inp_l, k, &mut out, params, &device) - .map_err(crate::Error::wrap)?; - S::BF16(out) - } - (S::F16(inp), S::F16(k)) => { - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(kernel_l.start_offset()..); - let mut out = unsafe { device.alloc::(dst_el)? }; - crate::cudnn::launch_conv1d::(inp, inp_l, k, &mut out, params, &device) - .map_err(crate::Error::wrap)?; - S::F16(out) - } - (S::F32(inp), S::F32(k)) => { - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(kernel_l.start_offset()..); - let mut out = unsafe { device.alloc::(dst_el)? }; - crate::cudnn::launch_conv1d::(inp, inp_l, k, &mut out, params, &device) - .map_err(crate::Error::wrap)?; - S::F32(out) - } - (S::F64(inp), S::F64(k)) => { - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(kernel_l.start_offset()..); - let mut out = unsafe { device.alloc::(dst_el)? }; - crate::cudnn::launch_conv1d::(inp, inp_l, k, &mut out, params, &device) - .map_err(crate::Error::wrap)?; - S::F64(out) - } - (S::U32(_), S::U32(_)) => Err(CudaError::InternalError("conv1d does not support u32"))?, - (S::I16(_), S::I16(_)) => Err(CudaError::InternalError("conv1d does not support i16"))?, - (S::I32(_), S::I32(_)) => Err(CudaError::InternalError("conv1d does not support i32"))?, - (S::I64(_), S::I64(_)) => Err(CudaError::InternalError("conv1d does not support i64"))?, - (S::F8E4M3(_), S::F8E4M3(_)) => { - Err(CudaError::InternalError("conv1d does not support f8e4m3"))? - } - _ => Err(CudaError::InternalError("dtype mismatch in conv1d"))?, - }; - Ok(Self { slice, device }) - } - - fn conv_transpose1d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConvTranspose1D, - ) -> Result { - const USE_COL2IM_CONV1D_TR: bool = true; - - let device = self.device().clone(); - let can_use_col2im = kernel_l.is_contiguous() - && params.dilation == 1 - && params.padding == 0 - && params.output_padding == 0; - let slice = if USE_COL2IM_CONV1D_TR && can_use_col2im { - let (b_size, c_in, l_in) = l.shape().dims3()?; - let (c_in2, c_out, k_size) = kernel_l.shape().dims3()?; - if !kernel_l.is_contiguous() { - crate::bail!( - "convtr1d: the second argument (kernel) has to be contiguous {kernel_l:?}" - ) - } - if c_in != c_in2 { - crate::bail!( - "convtr1d: shape mismatch on c_in {:?} {:?}", - l.shape(), - kernel_l.shape() - ) - } - let col = { - // This merges the last two dimensions of the kernel together. - let kernel_l_mm = Layout::new( - (b_size, c_in, k_size * c_out).into(), - vec![0, k_size * c_out, 1], - kernel_l.start_offset(), - ); - self.matmul( - kernel, - ( - b_size, - /* m */ l_in, - /* n */ c_out * k_size, - /* k */ c_in, - ), - &l.transpose(1, 2)?, - &kernel_l_mm, - )? - }; - let col_l = Layout::contiguous((b_size, l_in, c_out, k_size)); - Col2Im1D { - stride: params.stride, - } - .map(&col.slice, &device, &col_l)? - } else { - ConvTranspose1D(params).map(&self.slice, l, &kernel.slice, kernel_l, &device)? - }; - Ok(Self { slice, device }) - } - - #[cfg(not(feature = "cudnn"))] - fn conv2d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConv2D, - ) -> Result { - const USE_IM2COL_CONV2D: bool = true; - - let device = self.device().clone(); - if !USE_IM2COL_CONV2D { - let slice = Conv2D(params).map(&self.slice, l, &kernel.slice, kernel_l, &device)?; - return Ok(Self { slice, device }); - } - - let col = Im2Col { - h_k: params.k_h, - w_k: params.k_w, - stride: params.stride, - dilation: params.dilation, - padding: params.padding, - } - .map(&self.slice, &device, l)?; - let col = Self { slice: col, device }; - let h_out = params.out_h(); - let w_out = params.out_w(); - let b = params.b_size; - let n = params.c_out; - let k = params.k_h * params.k_w * params.c_in; - let m = h_out * w_out; - let col_l = Layout::contiguous((b * m, k)); - let res = if kernel_l.is_contiguous() { - let kernel_l = - Layout::contiguous_with_offset((n, k), kernel_l.start_offset()).transpose(0, 1)?; - col.matmul(kernel, (1, b * m, n, k), &col_l, &kernel_l)? - } else { - // Make the kernel contiguous if not already the case. - let mut kernel_c = unsafe { - self.device() - .alloc_uninit(kernel_l.shape(), kernel.dtype())? - }; - kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?; - let kernel_l = - Layout::contiguous_with_offset((n, k), kernel_l.start_offset()).transpose(0, 1)?; - col.matmul(kernel, (1, b * m, n, k), &col_l, &kernel_l)? - }; - let res_l = Layout::contiguous((b, h_out, w_out, n)) - .transpose(1, 2)? - .transpose(1, 3)?; - let mut res_t = unsafe { self.device().alloc_uninit(res_l.shape(), res.dtype())? }; - res.copy_strided_src(&mut res_t, 0, &res_l)?; - Ok(res_t) - } - - #[cfg(feature = "cudnn")] - fn conv2d( - &self, - inp_l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConv2D, - ) -> Result { - let device = self.device().clone(); - if !kernel_l.is_contiguous() { - let slice = Conv2D(params).map(&self.slice, inp_l, &kernel.slice, kernel_l, &device)?; - return Ok(Self { slice, device }); - } - let (out_w, out_h) = (params.out_w(), params.out_h()); - let dst_el = params.c_out * out_w * out_h * params.b_size; - let slice = match (&self.slice, &kernel.slice) { - (S::U8(inp), S::U8(k)) => { - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(kernel_l.start_offset()..); - let mut out = unsafe { device.alloc::(dst_el)? }; - crate::cudnn::launch_conv2d::(inp, inp_l, k, &mut out, params, &device) - .map_err(crate::Error::wrap)?; - S::U8(out) - } - (S::BF16(inp), S::BF16(k)) => { - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(kernel_l.start_offset()..); - let mut out = unsafe { device.alloc::(dst_el)? }; - // Only PSEUDO_BFLOAT16_CONFIG is supported in cudnn, there is no "true bfloat16" - // version. - // https://docs.nvidia.com/deeplearning/cudnn/latest/api/cudnn-cnn-library.html#id88 - crate::cudnn::launch_conv2d::(inp, inp_l, k, &mut out, params, &device) - .map_err(crate::Error::wrap)?; - S::BF16(out) - } - (S::F16(inp), S::F16(k)) => { - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(kernel_l.start_offset()..); - let mut out = unsafe { device.alloc::(dst_el)? }; - crate::cudnn::launch_conv2d::(inp, inp_l, k, &mut out, params, &device) - .map_err(crate::Error::wrap)?; - S::F16(out) - } - (S::F32(inp), S::F32(k)) => { - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(kernel_l.start_offset()..); - let mut out = unsafe { device.alloc::(dst_el)? }; - crate::cudnn::launch_conv2d::(inp, inp_l, k, &mut out, params, &device) - .map_err(crate::Error::wrap)?; - S::F32(out) - } - (S::F64(inp), S::F64(k)) => { - let inp = &inp.slice(inp_l.start_offset()..); - let k = &k.slice(kernel_l.start_offset()..); - let mut out = unsafe { device.alloc::(dst_el)? }; - crate::cudnn::launch_conv2d::(inp, inp_l, k, &mut out, params, &device) - .map_err(crate::Error::wrap)?; - S::F64(out) - } - (S::U32(_), S::U32(_)) => Err(CudaError::InternalError("conv2d does not support u32"))?, - (S::I16(_), S::I16(_)) => Err(CudaError::InternalError("conv2d does not support i16"))?, - (S::I32(_), S::I32(_)) => Err(CudaError::InternalError("conv2d does not support i32"))?, - (S::I64(_), S::I64(_)) => Err(CudaError::InternalError("conv2d does not support i64"))?, - (S::F8E4M3(_), S::F8E4M3(_)) => { - Err(CudaError::InternalError("conv2d does not support f8e4m3"))? - } - _ => Err(CudaError::InternalError("dtype mismatch in conv2d"))?, - }; - Ok(Self { slice, device }) - } - - fn conv_transpose2d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConvTranspose2D, - ) -> Result { - let device = self.device().clone(); - let slice = - ConvTranspose2D(params).map(&self.slice, l, &kernel.slice, kernel_l, &device)?; - Ok(Self { slice, device }) - } - - fn avg_pool2d(&self, l: &Layout, k: (usize, usize), stride: (usize, usize)) -> Result { - let device = self.device().clone(); - let slice = Pool2D { - w_k: k.0, - h_k: k.1, - w_stride: stride.0, - h_stride: stride.1, - op: PoolOp::Avg, - } - .map(&self.slice, &device, l)?; - Ok(Self { slice, device }) - } - - fn max_pool2d(&self, l: &Layout, k: (usize, usize), stride: (usize, usize)) -> Result { - let device = self.device().clone(); - let slice = Pool2D { - w_k: k.0, - h_k: k.1, - w_stride: stride.0, - h_stride: stride.1, - op: PoolOp::Max, - } - .map(&self.slice, &device, l)?; - Ok(Self { slice, device }) - } - - fn upsample_nearest1d(&self, _: &Layout, _out_sz: usize) -> Result { - crate::bail!("upsample-nearest1d is not supported on cuda") - } - - fn upsample_nearest2d(&self, l: &Layout, out_w: usize, out_h: usize) -> Result { - let device = self.device().clone(); - let slice = UpsampleNearest2D(out_w, out_h).map(&self.slice, &device, l)?; - Ok(Self { slice, device }) - } - - fn upsample_bilinear2d( - &self, - l: &Layout, - out_h: usize, - out_w: usize, - align_corners: bool, - scale_h: Option, - scale_w: Option, - ) -> Result { - let device = self.device().clone(); - let slice = UpsampleBilinear2D { - out_w, - out_h, - align_corners, - scale_h_factor: scale_h, - scale_w_factor: scale_w, - } - .map(&self.slice, &device, l)?; - Ok(Self { slice, device }) - } - - fn index_select(&self, ids: &Self, l: &Layout, ids_l: &Layout, dim: usize) -> Result { - let device = self.device().clone(); - let slice = IndexSelect(ids, ids_l, dim).map(&self.slice, &device, l)?; - Ok(Self { slice, device }) - } - fn gather(&self, l: &Layout, ids: &Self, ids_l: &Layout, dim: usize) -> Result { - let device = self.device().clone(); - let slice = Gather(ids, ids_l, dim).map(&self.slice, &device, l)?; - Ok(Self { slice, device }) - } - fn scatter_set( - &mut self, - l: &Layout, - ids: &Self, - ids_l: &Layout, - src: &Self, - src_l: &Layout, - dim: usize, - ) -> Result<()> { - let device = self.device().clone(); - Scatter(ids, ids_l, dim).map(&mut self.slice, l, &src.slice, src_l, &device) - } - fn scatter_add_set( - &mut self, - l: &Layout, - ids: &Self, - ids_l: &Layout, - src: &Self, - src_l: &Layout, - dim: usize, - ) -> Result<()> { - let device = self.device().clone(); - ScatterAdd(ids, ids_l, dim).map(&mut self.slice, l, &src.slice, src_l, &device) - } - fn index_add( - &self, - l: &Layout, - ids: &Self, - ids_l: &Layout, - src: &Self, - src_l: &Layout, - dim: usize, - ) -> Result { - let device = self.device().clone(); - let mut acc = unsafe { device.alloc_uninit(l.shape(), self.dtype())? }; - self.copy_strided_src(&mut acc, 0, l)?; - IndexAdd(ids, ids_l, dim).map(&mut acc.slice, l, &src.slice, src_l, &device)?; - Ok(acc) - } - - fn matmul( - &self, - rhs: &Self, - (b, m, n, k): (usize, usize, usize, usize), - lhs_l: &Layout, - rhs_l: &Layout, - ) -> Result { - let elem_count = b * m * n; - let dev = &self.device; - let slice = match (&self.slice, &rhs.slice) { - (CudaStorageSlice::BF16(lhs), CudaStorageSlice::BF16(rhs)) => { - let lhs = &lhs.slice(lhs_l.start_offset()..); - let rhs = &rhs.slice(rhs_l.start_offset()..); - let cfg = gemm_config(bf16::ONE, bf16::ZERO, (b, m, n, k), lhs_l, rhs_l)?; - let mut out = unsafe { dev.alloc::(elem_count)? }; - unsafe { gemm_strided_batched_bf16(&self.device.blas, cfg, rhs, lhs, &mut out) } - .w()?; - CudaStorageSlice::BF16(out) - } - (CudaStorageSlice::F16(lhs), CudaStorageSlice::F16(rhs)) => { - let lhs = &lhs.slice(lhs_l.start_offset()..); - let rhs = &rhs.slice(rhs_l.start_offset()..); - let cfg = gemm_config(f16::ONE, f16::ZERO, (b, m, n, k), lhs_l, rhs_l)?; - let mut out = unsafe { dev.alloc::(elem_count)? }; - unsafe { gemm_strided_batched_f16(&self.device.blas, cfg, rhs, lhs, &mut out) } - .w()?; - CudaStorageSlice::F16(out) - } - (CudaStorageSlice::F32(lhs), CudaStorageSlice::F32(rhs)) => { - let lhs = &lhs.slice(lhs_l.start_offset()..); - let rhs = &rhs.slice(rhs_l.start_offset()..); - let cfg = gemm_config(1., 0., (b, m, n, k), lhs_l, rhs_l)?; - let mut out = unsafe { dev.alloc::(elem_count)? }; - unsafe { gemm_strided_batched_f32(&self.device.blas, cfg, rhs, lhs, &mut out) } - .w()?; - CudaStorageSlice::F32(out) - } - (CudaStorageSlice::F64(lhs), CudaStorageSlice::F64(rhs)) => { - let lhs = &lhs.slice(lhs_l.start_offset()..); - let rhs = &rhs.slice(rhs_l.start_offset()..); - let cfg = gemm_config(1., 0., (b, m, n, k), lhs_l, rhs_l)?; - let mut out = unsafe { dev.alloc::(elem_count)? }; - unsafe { - self.device - .blas - .gemm_strided_batched(cfg, rhs, lhs, &mut out) - } - .w()?; - CudaStorageSlice::F64(out) - } - _ => Err(CudaError::InternalError("dtype mismatch in matmul op"))?, - }; - let device = dev.clone(); - Ok(Self { slice, device }) - } - - fn copy2d( - &self, - dst: &mut Self, - d1: usize, - d2: usize, - src_s: usize, - dst_s: usize, - src_o: usize, - dst_o: usize, - ) -> Result<()> { - let dev = &self.device; - let d1 = d1 as u32; - let d2 = d2 as u32; - // Nothing to copy so we exit early to avoid launching a kernel and some potential invalid - // argument with a null pointer. - if d1 == 0 || d2 == 0 { - return Ok(()); - } - let dst_s = dst_s as u32; - let src_s = src_s as u32; - let ((src, _guard_src), (dst, _guard_dst), kname) = match (&self.slice, &mut dst.slice) { - (S::U8(s), S::U8(d)) => (slice_ptr(s, src_o), slice_ptr(d, dst_o), "copy2d_u8"), - (S::U32(s), S::U32(d)) => (slice_ptr(s, src_o), slice_ptr(d, dst_o), "copy2d_u32"), - (S::I16(s), S::I16(d)) => (slice_ptr(s, src_o), slice_ptr(d, dst_o), "copy2d_i16"), - (S::I32(s), S::I32(d)) => (slice_ptr(s, src_o), slice_ptr(d, dst_o), "copy2d_i32"), - (S::I64(s), S::I64(d)) => (slice_ptr(s, src_o), slice_ptr(d, dst_o), "copy2d_i64"), - (S::BF16(s), S::BF16(d)) => (slice_ptr(s, src_o), slice_ptr(d, dst_o), "copy2d_bf16"), - (S::F16(s), S::F16(d)) => (slice_ptr(s, src_o), slice_ptr(d, dst_o), "copy2d_f16"), - (S::F32(s), S::F32(d)) => (slice_ptr(s, src_o), slice_ptr(d, dst_o), "copy2d_f32"), - (S::F64(s), S::F64(d)) => (slice_ptr(s, src_o), slice_ptr(d, dst_o), "copy2d_f64"), - (S::F8E4M3(s), S::F8E4M3(d)) => (slice_ptr(s, src_o), slice_ptr(d, dst_o), "copy2d_u8"), - (S::F8E8M0(s), S::F8E8M0(d)) => (slice_ptr(s, src_o), slice_ptr(d, dst_o), "copy2d_u8"), - _ => Err(CudaError::InternalError("dtype mismatch in copy2d"))?, - }; - let func = dev.get_or_load_func(kname, &kernels::FILL)?; - let cfg = LaunchConfig::for_num_elems(d1 * d2); - let mut builder = func.builder(); - barg!(builder, src); - barg!(builder, dst); - barg!(builder, d1); - barg!(builder, d2); - builder.arg(&src_s); - builder.arg(&dst_s); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - Ok(()) - } - - fn copy_strided_src(&self, dst: &mut Self, dst_offset: usize, src_l: &Layout) -> Result<()> { - let src_shape = src_l.shape(); - let dims = src_shape.dims(); - let el_count = src_shape.elem_count(); - if el_count == 0 { - return Ok(()); - } - let cfg = LaunchConfig::for_num_elems(el_count as u32); - let dev = &self.device; - let ds = SlicePtrOrNull::params_from_layout(dev, src_l)?; - match (&self.slice, &mut dst.slice) { - (CudaStorageSlice::BF16(src), CudaStorageSlice::BF16(dst)) => { - let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); - if src_l.is_contiguous() { - dev.memcpy_dtod(&src, &mut dst)? - } else { - let func = dev.get_or_load_func("ucopy_bf16", &kernels::UNARY)?; - let mut builder = func.builder(); - barg!(builder, el_count); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - builder.arg(&src); - builder.arg(&mut dst); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - } - } - (CudaStorageSlice::F16(src), CudaStorageSlice::F16(dst)) => { - let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); - if src_l.is_contiguous() { - dev.memcpy_dtod(&src, &mut dst)? - } else { - let func = dev.get_or_load_func("ucopy_f16", &kernels::UNARY)?; - let mut builder = func.builder(); - barg!(builder, el_count); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - builder.arg(&src); - builder.arg(&mut dst); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - } - } - (CudaStorageSlice::F32(src), CudaStorageSlice::F32(dst)) => { - let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); - if src_l.is_contiguous() { - dev.memcpy_dtod(&src, &mut dst)? - } else { - let func = dev.get_or_load_func("ucopy_f32", &kernels::UNARY)?; - let mut builder = func.builder(); - barg!(builder, el_count); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - builder.arg(&src); - builder.arg(&mut dst); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - } - } - (CudaStorageSlice::U8(src), CudaStorageSlice::U8(dst)) => { - let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); - if src_l.is_contiguous() { - dev.memcpy_dtod(&src, &mut dst)? - } else { - let func = dev.get_or_load_func("ucopy_u8", &kernels::UNARY)?; - let mut builder = func.builder(); - barg!(builder, el_count); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - builder.arg(&src); - builder.arg(&mut dst); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - } - } - (CudaStorageSlice::U32(src), CudaStorageSlice::U32(dst)) => { - let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); - if src_l.is_contiguous() { - dev.memcpy_dtod(&src, &mut dst)? - } else { - let func = dev.get_or_load_func("ucopy_u32", &kernels::UNARY)?; - let mut builder = func.builder(); - barg!(builder, el_count); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - builder.arg(&src); - builder.arg(&mut dst); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - } - } - (CudaStorageSlice::I16(src), CudaStorageSlice::I16(dst)) => { - let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); - if src_l.is_contiguous() { - dev.memcpy_dtod(&src, &mut dst)? - } else { - let func = dev.get_or_load_func("ucopy_i16", &kernels::UNARY)?; - let mut builder = func.builder(); - barg!(builder, el_count); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - builder.arg(&src); - builder.arg(&mut dst); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - } - } - (CudaStorageSlice::I32(src), CudaStorageSlice::I32(dst)) => { - let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); - if src_l.is_contiguous() { - dev.memcpy_dtod(&src, &mut dst)? - } else { - let func = dev.get_or_load_func("ucopy_i32", &kernels::UNARY)?; - let mut builder = func.builder(); - barg!(builder, el_count); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - builder.arg(&src); - builder.arg(&mut dst); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - } - } - (CudaStorageSlice::I64(src), CudaStorageSlice::I64(dst)) => { - let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); - if src_l.is_contiguous() { - dev.memcpy_dtod(&src, &mut dst)? - } else { - let func = dev.get_or_load_func("ucopy_i64", &kernels::UNARY)?; - let mut builder = func.builder(); - barg!(builder, el_count); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - builder.arg(&src); - builder.arg(&mut dst); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - } - } - (CudaStorageSlice::F64(src), CudaStorageSlice::F64(dst)) => { - let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); - if src_l.is_contiguous() { - dev.memcpy_dtod(&src, &mut dst)? - } else { - let func = dev.get_or_load_func("ucopy_f64", &kernels::UNARY)?; - let mut builder = func.builder(); - barg!(builder, el_count); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - builder.arg(&src); - builder.arg(&mut dst); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - } - } - (CudaStorageSlice::F8E4M3(src), CudaStorageSlice::F8E4M3(dst)) => { - let (src, mut dst) = slice_src_and_dst(src, src_l, dst, dst_offset); - if src_l.is_contiguous() { - dev.memcpy_dtod(&src, &mut dst)? - } else { - let func = dev.get_or_load_func("ucopy_f8e4m3", &kernels::UNARY)?; - let mut builder = func.builder(); - barg!(builder, el_count); - barg!(builder, dims.len()); - ds.builder_arg(&mut builder); - builder.arg(&src); - builder.arg(&mut dst); - // SAFETY: ffi. - unsafe { builder.launch(cfg) }.w()?; - } - } - _ => Err(CudaError::InternalError( - "dtype mismatch in copy_strided op", - ))?, - } - Ok(()) - } -} - -// Default for the reduced precision setting is false, similar to pytorch. -// https://github.com/pytorch/pytorch/issues/123157 -static MM_F16_REDUCED_PRECISION: std::sync::atomic::AtomicBool = - std::sync::atomic::AtomicBool::new(false); -static MM_BF16_REDUCED_PRECISION: std::sync::atomic::AtomicBool = - std::sync::atomic::AtomicBool::new(false); -static MM_F32_REDUCED_PRECISION: std::sync::atomic::AtomicBool = - std::sync::atomic::AtomicBool::new(false); - -/// This bool controls whether reduced precision reductions (e.g., with tf32 accumulation type) are -/// allowed with f32 GEMMs. -pub fn gemm_reduced_precision_f32() -> bool { - MM_F32_REDUCED_PRECISION.load(std::sync::atomic::Ordering::Relaxed) -} - -/// This bool controls whether reduced precision reductions (e.g., with tf32 accumulation type) are -/// allowed with f32 GEMMs. -pub fn set_gemm_reduced_precision_f32(b: bool) { - MM_F32_REDUCED_PRECISION.store(b, std::sync::atomic::Ordering::Relaxed) -} - -/// This bool controls whether reduced precision reductions (e.g., with fp16 accumulation type) are -/// allowed with f16 GEMMs. -pub fn gemm_reduced_precision_f16() -> bool { - MM_F16_REDUCED_PRECISION.load(std::sync::atomic::Ordering::Relaxed) -} - -/// This bool controls whether reduced precision reductions (e.g., with fp16 accumulation type) are -/// allowed with f16 GEMMs. -pub fn set_gemm_reduced_precision_f16(b: bool) { - MM_F16_REDUCED_PRECISION.store(b, std::sync::atomic::Ordering::Relaxed) -} - -/// This bool controls whether reduced precision reductions (e.g., with fp16 accumulation type) are -/// allowed with bf16 GEMMs. -pub fn gemm_reduced_precision_bf16() -> bool { - MM_BF16_REDUCED_PRECISION.load(std::sync::atomic::Ordering::Relaxed) -} - -/// This bool controls whether reduced precision reductions (e.g., with fp16 accumulation type) are -/// allowed with bf16 GEMMs. -pub fn set_gemm_reduced_precision_bf16(b: bool) { - MM_BF16_REDUCED_PRECISION.store(b, std::sync::atomic::Ordering::Relaxed) -} - -unsafe fn gemm_strided_batched_f32( - cublas: &cudarc::cublas::CudaBlas, - cfg: StridedBatchedConfig, - a: &cudarc::driver::CudaView, - b: &cudarc::driver::CudaView, - c: &mut CudaSlice, -) -> std::result::Result<(), cudarc::cublas::result::CublasError> { - use cudarc::cublas::sys; - use cudarc::driver::DevicePtrMut; - - let compute_type = if gemm_reduced_precision_f32() { - sys::cublasComputeType_t::CUBLAS_COMPUTE_32F_FAST_TF32 - } else { - sys::cublasComputeType_t::CUBLAS_COMPUTE_32F - }; - let alpha = &cfg.gemm.alpha as *const f32 as *const _; - let beta = &cfg.gemm.beta as *const f32 as *const _; - - let stream = c.stream().clone(); - let (a, _guard_a) = a.device_ptr(&stream); - let (b, _guard_b) = b.device_ptr(&stream); - let (c, _guard_c) = c.device_ptr_mut(&stream); - - cudarc::cublas::result::gemm_strided_batched_ex( - *cublas.handle(), - cfg.gemm.transa, - cfg.gemm.transb, - cfg.gemm.m, - cfg.gemm.n, - cfg.gemm.k, - alpha, - a as *const _, - sys::cudaDataType_t::CUDA_R_32F, - cfg.gemm.lda, - cfg.stride_a, - b as *const _, - sys::cudaDataType_t::CUDA_R_32F, - cfg.gemm.ldb, - cfg.stride_b, - beta, - c as *mut _, - sys::cudaDataType_t::CUDA_R_32F, - cfg.gemm.ldc, - cfg.stride_c, - cfg.batch_size, - compute_type, - sys::cublasGemmAlgo_t::CUBLAS_GEMM_DEFAULT_TENSOR_OP, - ) -} - -unsafe fn gemm_strided_batched_f16( - cublas: &cudarc::cublas::CudaBlas, - cfg: StridedBatchedConfig, - a: &cudarc::driver::CudaView, - b: &cudarc::driver::CudaView, - c: &mut CudaSlice, -) -> std::result::Result<(), cudarc::cublas::result::CublasError> { - use cudarc::cublas::sys; - use cudarc::driver::DevicePtrMut; - - let alpha = cfg.gemm.alpha; - let beta = cfg.gemm.beta; - let alpha_f32: f32 = cfg.gemm.alpha.to_f32(); - let beta_f32: f32 = cfg.gemm.beta.to_f32(); - let (compute_type, alpha, beta) = if gemm_reduced_precision_f16() { - ( - sys::cublasComputeType_t::CUBLAS_COMPUTE_16F, - (&alpha) as *const f16 as *const _, - (&beta) as *const f16 as *const _, - ) - } else { - ( - sys::cublasComputeType_t::CUBLAS_COMPUTE_32F, - (&alpha_f32) as *const f32 as *const _, - (&beta_f32) as *const f32 as *const _, - ) - }; - - let stream = c.stream().clone(); - let (a, _guard_a) = a.device_ptr(&stream); - let (b, _guard_b) = b.device_ptr(&stream); - let (c, _guard_c) = c.device_ptr_mut(&stream); - cudarc::cublas::result::gemm_strided_batched_ex( - *cublas.handle(), - cfg.gemm.transa, - cfg.gemm.transb, - cfg.gemm.m, - cfg.gemm.n, - cfg.gemm.k, - alpha, - a as *const _, - sys::cudaDataType_t::CUDA_R_16F, - cfg.gemm.lda, - cfg.stride_a, - b as *const _, - sys::cudaDataType_t::CUDA_R_16F, - cfg.gemm.ldb, - cfg.stride_b, - beta, - c as *mut _, - sys::cudaDataType_t::CUDA_R_16F, - cfg.gemm.ldc, - cfg.stride_c, - cfg.batch_size, - compute_type, - sys::cublasGemmAlgo_t::CUBLAS_GEMM_DEFAULT_TENSOR_OP, - ) -} - -unsafe fn gemm_strided_batched_bf16( - cublas: &cudarc::cublas::CudaBlas, - cfg: StridedBatchedConfig, - a: &cudarc::driver::CudaView, - b: &cudarc::driver::CudaView, - c: &mut CudaSlice, -) -> std::result::Result<(), cudarc::cublas::result::CublasError> { - use cudarc::cublas::sys; - use cudarc::driver::DevicePtrMut; - - let alpha_f32: f32 = cfg.gemm.alpha.to_f32(); - let beta_f32: f32 = cfg.gemm.beta.to_f32(); - // The type for alpha and beta depends on the computeType. - // https://docs.nvidia.com/cuda/cublas/index.html#cublasgemmstridedbatchedex - let (compute_type, alpha, beta) = if gemm_reduced_precision_bf16() { - ( - sys::cublasComputeType_t::CUBLAS_COMPUTE_32F_FAST_16BF, - (&alpha_f32) as *const f32 as *const _, - (&beta_f32) as *const f32 as *const _, - ) - } else { - ( - sys::cublasComputeType_t::CUBLAS_COMPUTE_32F, - (&alpha_f32) as *const f32 as *const _, - (&beta_f32) as *const f32 as *const _, - ) - }; - - let stream = c.stream().clone(); - let (a, _guard_a) = a.device_ptr(&stream); - let (b, _guard_b) = b.device_ptr(&stream); - let (c, _guard_c) = c.device_ptr_mut(&stream); - cudarc::cublas::result::gemm_strided_batched_ex( - *cublas.handle(), - cfg.gemm.transa, - cfg.gemm.transb, - cfg.gemm.m, - cfg.gemm.n, - cfg.gemm.k, - alpha, - a as *const _, - sys::cudaDataType_t::CUDA_R_16BF, - cfg.gemm.lda, - cfg.stride_a, - b as *const _, - sys::cudaDataType_t::CUDA_R_16BF, - cfg.gemm.ldb, - cfg.stride_b, - beta, - c as *mut _, - sys::cudaDataType_t::CUDA_R_16BF, - cfg.gemm.ldc, - cfg.stride_c, - cfg.batch_size, - compute_type, - sys::cublasGemmAlgo_t::CUBLAS_GEMM_DEFAULT_TENSOR_OP, - ) -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/utils.rs b/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/utils.rs deleted file mode 100644 index 10f8876ab5..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/cuda_backend/utils.rs +++ /dev/null @@ -1,194 +0,0 @@ -/// Helper functions to plug cuda kernels in candle. -use crate::{Layout, Result, WithDType}; -pub use cudarc; -use cudarc::driver::{CudaSlice, DeviceRepr, ValidAsZeroBits}; - -use super::{CudaDevice, CudaError, WrapErr}; - -pub type S = super::CudaStorageSlice; - -pub trait Map1 { - fn f( - &self, - src: &CudaSlice, - dev: &CudaDevice, - layout: &Layout, - ) -> Result>; - - fn map(&self, s: &S, d: &CudaDevice, l: &Layout) -> Result { - let out = match s { - S::U8(s) => S::U8(self.f(s, d, l)?), - S::U32(s) => S::U32(self.f(s, d, l)?), - S::I16(s) => S::I16(self.f(s, d, l)?), - S::I32(s) => S::I32(self.f(s, d, l)?), - S::I64(s) => S::I64(self.f(s, d, l)?), - S::BF16(s) => S::BF16(self.f(s, d, l)?), - S::F16(s) => S::F16(self.f(s, d, l)?), - S::F32(s) => S::F32(self.f(s, d, l)?), - S::F64(s) => S::F64(self.f(s, d, l)?), - S::F8E4M3(s) => S::F8E4M3(self.f(s, d, l)?), - S::F4(_) | S::F6E2M3(_) | S::F6E3M2(_) | S::F8E8M0(_) => { - crate::bail!("Map1 does not uspport this dtype."); - } - }; - Ok(out) - } -} - -pub trait Map2 { - fn f( - &self, - src1: &CudaSlice, - layout1: &Layout, - src2: &CudaSlice, - layout2: &Layout, - dev: &CudaDevice, - ) -> Result>; - - fn map(&self, s1: &S, l1: &Layout, s2: &S, l2: &Layout, d: &CudaDevice) -> Result { - let out = match (s1, s2) { - (S::U8(s1), S::U8(s2)) => S::U8(self.f(s1, l1, s2, l2, d)?), - (S::U32(s1), S::U32(s2)) => S::U32(self.f(s1, l1, s2, l2, d)?), - (S::I16(s1), S::I16(s2)) => S::I16(self.f(s1, l1, s2, l2, d)?), - (S::I32(s1), S::I32(s2)) => S::I32(self.f(s1, l1, s2, l2, d)?), - (S::I64(s1), S::I64(s2)) => S::I64(self.f(s1, l1, s2, l2, d)?), - (S::BF16(s1), S::BF16(s2)) => S::BF16(self.f(s1, l1, s2, l2, d)?), - (S::F16(s1), S::F16(s2)) => S::F16(self.f(s1, l1, s2, l2, d)?), - (S::F32(s1), S::F32(s2)) => S::F32(self.f(s1, l1, s2, l2, d)?), - (S::F64(s1), S::F64(s2)) => S::F64(self.f(s1, l1, s2, l2, d)?), - (S::F8E4M3(s1), S::F8E4M3(s2)) => S::F8E4M3(self.f(s1, l1, s2, l2, d)?), - _ => Err(CudaError::InternalError("dtype mismatch in binary op"))?, - }; - Ok(out) - } -} - -pub trait Map3 { - #[allow(clippy::too_many_arguments)] - fn f( - &self, - src1: &CudaSlice, - layout1: &Layout, - src2: &CudaSlice, - layout2: &Layout, - src3: &CudaSlice, - layout3: &Layout, - dev: &CudaDevice, - ) -> Result>; - - #[allow(clippy::too_many_arguments)] - fn map( - &self, - s1: &S, - l1: &Layout, - s2: &S, - l2: &Layout, - s3: &S, - l3: &Layout, - d: &CudaDevice, - ) -> Result { - let out = match (s1, s2, s3) { - (S::U8(s1), S::U8(s2), S::U8(s3)) => S::U8(self.f(s1, l1, s2, l2, s3, l3, d)?), - (S::U32(s1), S::U32(s2), S::U32(s3)) => S::U32(self.f(s1, l1, s2, l2, s3, l3, d)?), - (S::I64(s1), S::I64(s2), S::I64(s3)) => S::I64(self.f(s1, l1, s2, l2, s3, l3, d)?), - (S::BF16(s1), S::BF16(s2), S::BF16(s3)) => S::BF16(self.f(s1, l1, s2, l2, s3, l3, d)?), - (S::F16(s1), S::F16(s2), S::F16(s3)) => S::F16(self.f(s1, l1, s2, l2, s3, l3, d)?), - (S::F32(s1), S::F32(s2), S::F32(s3)) => S::F32(self.f(s1, l1, s2, l2, s3, l3, d)?), - (S::F64(s1), S::F64(s2), S::F64(s3)) => S::F64(self.f(s1, l1, s2, l2, s3, l3, d)?), - (S::F8E4M3(s1), S::F8E4M3(s2), S::F8E4M3(s3)) => { - S::F8E4M3(self.f(s1, l1, s2, l2, s3, l3, d)?) - } - _ => Err(CudaError::InternalError("dtype mismatch in ternary op"))?, - }; - Ok(out) - } -} - -pub trait Map2InPlace { - fn f( - &self, - dst: &mut CudaSlice, - dst_l: &Layout, - src: &CudaSlice, - src_l: &Layout, - dev: &CudaDevice, - ) -> Result<()>; - - fn map( - &self, - dst: &mut S, - dst_l: &Layout, - src: &S, - src_l: &Layout, - d: &CudaDevice, - ) -> Result<()> { - match (dst, src) { - (S::U8(dst), S::U8(src)) => self.f(dst, dst_l, src, src_l, d), - (S::U32(dst), S::U32(src)) => self.f(dst, dst_l, src, src_l, d), - (S::I16(dst), S::I16(src)) => self.f(dst, dst_l, src, src_l, d), - (S::I32(dst), S::I32(src)) => self.f(dst, dst_l, src, src_l, d), - (S::I64(dst), S::I64(src)) => self.f(dst, dst_l, src, src_l, d), - (S::BF16(dst), S::BF16(src)) => self.f(dst, dst_l, src, src_l, d), - (S::F16(dst), S::F16(src)) => self.f(dst, dst_l, src, src_l, d), - (S::F32(dst), S::F32(src)) => self.f(dst, dst_l, src, src_l, d), - (S::F64(dst), S::F64(src)) => self.f(dst, dst_l, src, src_l, d), - (S::F8E4M3(dst), S::F8E4M3(src)) => self.f(dst, dst_l, src, src_l, d), - _ => Err(CudaError::InternalError("dtype mismatch in binary op"))?, - } - } -} - -pub trait Map1Any { - fn f) -> S>( - &self, - src: &CudaSlice, - dev: &CudaDevice, - layout: &Layout, - wrap: W, - ) -> Result; - - fn map(&self, s: &S, d: &CudaDevice, l: &Layout) -> Result { - let out = match s { - S::U8(s) => self.f(s, d, l, S::U8)?, - S::U32(s) => self.f(s, d, l, S::U32)?, - S::I16(s) => self.f(s, d, l, S::I16)?, - S::I32(s) => self.f(s, d, l, S::I32)?, - S::I64(s) => self.f(s, d, l, S::I64)?, - S::BF16(s) => self.f(s, d, l, S::BF16)?, - S::F16(s) => self.f(s, d, l, S::F16)?, - S::F32(s) => self.f(s, d, l, S::F32)?, - S::F64(s) => self.f(s, d, l, S::F64)?, - S::F8E4M3(s) => self.f(s, d, l, S::F8E4M3)?, - S::F4(_) | S::F6E2M3(_) | S::F6E3M2(_) | S::F8E8M0(_) => { - crate::bail!("Map1 does not uspport this dtype."); - } - }; - Ok(out) - } -} - -pub trait Map2Any { - fn f( - &self, - src1: &CudaSlice, - layout1: &Layout, - src2: &CudaSlice, - layout2: &Layout, - dev: &CudaDevice, - ) -> Result; - - fn map(&self, s1: &S, l1: &Layout, s2: &S, l2: &Layout, d: &CudaDevice) -> Result { - let out = match (s1, s2) { - (S::U8(s1), S::U8(s2)) => self.f(s1, l1, s2, l2, d)?, - (S::U32(s1), S::U32(s2)) => self.f(s1, l1, s2, l2, d)?, - (S::I64(s1), S::I64(s2)) => self.f(s1, l1, s2, l2, d)?, - (S::BF16(s1), S::BF16(s2)) => self.f(s1, l1, s2, l2, d)?, - (S::F16(s1), S::F16(s2)) => self.f(s1, l1, s2, l2, d)?, - (S::F32(s1), S::F32(s2)) => self.f(s1, l1, s2, l2, d)?, - (S::F64(s1), S::F64(s2)) => self.f(s1, l1, s2, l2, d)?, - (S::F8E4M3(s1), S::F8E4M3(s2)) => self.f(s1, l1, s2, l2, d)?, - _ => Err(CudaError::InternalError("dtype mismatch in binary op")).w()?, - }; - Ok(out) - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/custom_op.rs b/v2/dependency-patches/candle-core-0.9.2/src/custom_op.rs deleted file mode 100644 index e76f96806e..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/custom_op.rs +++ /dev/null @@ -1,489 +0,0 @@ -use crate::op::{BackpropOp, Op}; -use crate::tensor::from_storage; -use crate::{CpuStorage, CudaStorage, Layout, MetalStorage, Result, Shape, Tensor}; -use std::sync::Arc; - -/// Unary ops that can be defined in user-land. -pub trait CustomOp1 { - // Box does not support const yet, so use a function to get the name. - fn name(&self) -> &'static str; - - /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn cpu_fwd(&self, storage: &CpuStorage, layout: &Layout) -> Result<(CpuStorage, Shape)>; - - /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn cuda_fwd(&self, _storage: &CudaStorage, _layout: &Layout) -> Result<(CudaStorage, Shape)> { - Err(crate::Error::Cuda( - format!("no cuda implementation for {}", self.name()).into(), - )) - } - - /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn metal_fwd( - &self, - _storage: &MetalStorage, - _layout: &Layout, - ) -> Result<(MetalStorage, Shape)> { - Err(crate::Error::Metal( - format!("no metal implementation for {}", self.name()).into(), - )) - } - - /// This function takes as argument the argument `arg` used in the forward pass, the result - /// produced by the forward operation `res` and the gradient of the result `grad_res`. - /// The function should return the gradient of the argument. - fn bwd(&self, _arg: &Tensor, _res: &Tensor, _grad_res: &Tensor) -> Result> { - Err(crate::Error::BackwardNotSupported { op: self.name() }) - } -} - -pub trait CustomOp2 { - fn name(&self) -> &'static str; - - /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn cpu_fwd( - &self, - s1: &CpuStorage, - l1: &Layout, - s2: &CpuStorage, - l2: &Layout, - ) -> Result<(CpuStorage, Shape)>; - - /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn cuda_fwd( - &self, - _: &CudaStorage, - _: &Layout, - _: &CudaStorage, - _: &Layout, - ) -> Result<(CudaStorage, Shape)> { - Err(crate::Error::Cuda( - format!("no cuda implementation for {}", self.name()).into(), - )) - } - - /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn metal_fwd( - &self, - _: &MetalStorage, - _: &Layout, - _: &MetalStorage, - _: &Layout, - ) -> Result<(MetalStorage, Shape)> { - Err(crate::Error::Metal( - format!("no metal implementation for {}", self.name()).into(), - )) - } - - fn bwd( - &self, - _arg1: &Tensor, - _arg2: &Tensor, - _res: &Tensor, - _grad_res: &Tensor, - ) -> Result<(Option, Option)> { - Err(crate::Error::BackwardNotSupported { op: self.name() }) - } -} - -pub trait CustomOp3 { - fn name(&self) -> &'static str; - - /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn cpu_fwd( - &self, - s1: &CpuStorage, - l1: &Layout, - s2: &CpuStorage, - l2: &Layout, - s3: &CpuStorage, - l3: &Layout, - ) -> Result<(CpuStorage, Shape)>; - - /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn cuda_fwd( - &self, - _: &CudaStorage, - _: &Layout, - _: &CudaStorage, - _: &Layout, - _: &CudaStorage, - _: &Layout, - ) -> Result<(CudaStorage, Shape)> { - Err(crate::Error::Cuda( - format!("no cuda implementation for {}", self.name()).into(), - )) - } - - /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn metal_fwd( - &self, - _: &MetalStorage, - _: &Layout, - _: &MetalStorage, - _: &Layout, - _: &MetalStorage, - _: &Layout, - ) -> Result<(MetalStorage, Shape)> { - Err(crate::Error::Metal( - format!("no metal implementation for {}", self.name()).into(), - )) - } - - fn bwd( - &self, - _arg1: &Tensor, - _arg2: &Tensor, - _arg3: &Tensor, - _res: &Tensor, - _grad_res: &Tensor, - ) -> Result<(Option, Option, Option)> { - Err(crate::Error::BackwardNotSupported { op: self.name() }) - } -} - -impl Tensor { - /// Applies a unary custom op without backward support - pub fn apply_op1_no_bwd(&self, c: &C) -> Result { - let (storage, shape) = self.storage().apply_op1(self.layout(), c)?; - Ok(from_storage(storage, shape, BackpropOp::none(), false)) - } - - /// Applies a binary custom op without backward support - pub fn apply_op2_no_bwd(&self, rhs: &Self, c: &C) -> Result { - let (storage, shape) = - self.storage() - .apply_op2(self.layout(), &rhs.storage(), rhs.layout(), c)?; - Ok(from_storage(storage, shape, BackpropOp::none(), false)) - } - - /// Applies a ternary custom op without backward support - pub fn apply_op3_no_bwd(&self, t2: &Self, t3: &Self, c: &C) -> Result { - let (storage, shape) = self.storage().apply_op3( - self.layout(), - &t2.storage(), - t2.layout(), - &t3.storage(), - t3.layout(), - c, - )?; - Ok(from_storage(storage, shape, BackpropOp::none(), false)) - } - - /// Applies a unary custom op. - pub fn apply_op1_arc(&self, c: Arc>) -> Result { - let (storage, shape) = self - .storage() - .apply_op1(self.layout(), c.as_ref().as_ref())?; - let op = BackpropOp::new1(self, |s| Op::CustomOp1(s, c.clone())); - Ok(from_storage(storage, shape, op, false)) - } - - pub fn apply_op1(&self, c: C) -> Result { - self.apply_op1_arc(Arc::new(Box::new(c))) - } - - /// Applies a binary custom op. - pub fn apply_op2_arc( - &self, - rhs: &Self, - c: Arc>, - ) -> Result { - let (storage, shape) = self.storage().apply_op2( - self.layout(), - &rhs.storage(), - rhs.layout(), - c.as_ref().as_ref(), - )?; - let op = BackpropOp::new2(self, rhs, |t1, t2| Op::CustomOp2(t1, t2, c.clone())); - Ok(from_storage(storage, shape, op, false)) - } - - pub fn apply_op2(&self, r: &Self, c: C) -> Result { - self.apply_op2_arc(r, Arc::new(Box::new(c))) - } - - /// Applies a ternary custom op. - pub fn apply_op3_arc( - &self, - t2: &Self, - t3: &Self, - c: Arc>, - ) -> Result { - let (storage, shape) = self.storage().apply_op3( - self.layout(), - &t2.storage(), - t2.layout(), - &t3.storage(), - t3.layout(), - c.as_ref().as_ref(), - )?; - let op = BackpropOp::new3(self, t2, t3, |t1, t2, t3| { - Op::CustomOp3(t1, t2, t3, c.clone()) - }); - Ok(from_storage(storage, shape, op, false)) - } - - pub fn apply_op3( - &self, - t2: &Self, - t3: &Self, - c: C, - ) -> Result { - self.apply_op3_arc(t2, t3, Arc::new(Box::new(c))) - } -} - -// In place ops. - -/// Unary ops that can be defined in user-land. -/// These ops work in place and as such back-prop is unsupported. -pub trait InplaceOp1 { - // Box does not support const yet, so use a function to get the name. - fn name(&self) -> &'static str; - - /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn cpu_fwd(&self, storage: &mut CpuStorage, layout: &Layout) -> Result<()>; - - /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn cuda_fwd(&self, _storage: &mut CudaStorage, _layout: &Layout) -> Result<()> { - Err(crate::Error::Cuda( - format!("no cuda implementation for {}", self.name()).into(), - )) - } - - /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn metal_fwd(&self, _storage: &mut MetalStorage, _layout: &Layout) -> Result<()> { - Err(crate::Error::Metal( - format!("no metal implementation for {}", self.name()).into(), - )) - } -} - -pub trait InplaceOp2 { - fn name(&self) -> &'static str; - - /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn cpu_fwd(&self, s1: &mut CpuStorage, l1: &Layout, s2: &CpuStorage, l2: &Layout) - -> Result<()>; - - /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn cuda_fwd(&self, _: &mut CudaStorage, _: &Layout, _: &CudaStorage, _: &Layout) -> Result<()> { - Err(crate::Error::Cuda( - format!("no cuda implementation for {}", self.name()).into(), - )) - } - - /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn metal_fwd( - &self, - _: &mut MetalStorage, - _: &Layout, - _: &MetalStorage, - _: &Layout, - ) -> Result<()> { - Err(crate::Error::Metal( - format!("no metal implementation for {}", self.name()).into(), - )) - } -} - -pub trait InplaceOp3 { - fn name(&self) -> &'static str; - - /// The forward pass, as run on a cpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn cpu_fwd( - &self, - s1: &mut CpuStorage, - l1: &Layout, - s2: &CpuStorage, - l2: &Layout, - s3: &CpuStorage, - l3: &Layout, - ) -> Result<()>; - - /// The forward pass, as run on a gpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn cuda_fwd( - &self, - _: &mut CudaStorage, - _: &Layout, - _: &CudaStorage, - _: &Layout, - _: &CudaStorage, - _: &Layout, - ) -> Result<()> { - Err(crate::Error::Cuda( - format!("no cuda implementation for {}", self.name()).into(), - )) - } - - /// The forward pass, as run on a metal gpu device. Note that the storage can use arbitrary strides, - /// offsets etc so the associated layout should be used to access it. - fn metal_fwd( - &self, - _: &mut MetalStorage, - _: &Layout, - _: &MetalStorage, - _: &Layout, - _: &MetalStorage, - _: &Layout, - ) -> Result<()> { - Err(crate::Error::Metal( - format!("no metal implementation for {}", self.name()).into(), - )) - } -} - -impl Tensor { - /// Applies a unary custom op in place. - pub fn inplace_op1(&self, c: &C) -> Result<()> { - self.storage_mut().inplace_op1(self.layout(), c) - } - - /// Applies a unary custom op in place (for the first tensor). - pub fn inplace_op2(&self, rhs: &Self, c: &C) -> Result<()> { - self.storage_mut() - .inplace_op2(self.layout(), &rhs.storage(), rhs.layout(), c) - } - - /// Applies a ternary custom op in place (for the first tensor). - pub fn inplace_op3(&self, t2: &Self, t3: &Self, c: &C) -> Result<()> { - self.storage_mut().inplace_op3( - self.layout(), - &t2.storage(), - t2.layout(), - &t3.storage(), - t3.layout(), - c, - ) - } -} - -#[cfg(feature = "ug")] -pub struct UgIOp1 { - name: &'static str, - #[cfg(feature = "cuda")] - func: cudarc::driver::CudaFunction, - #[cfg(feature = "metal")] - func: candle_metal_kernels::metal::ComputePipeline, -} - -#[cfg(feature = "ug")] -impl UgIOp1 { - #[allow(unused)] - #[cfg(all(not(target_arch = "wasm32"), not(target_os = "ios")))] - pub fn new( - name: &'static str, - kernel: candle_ug::lang::ssa::Kernel, - device: &crate::Device, - ) -> Result { - #[cfg(feature = "cuda")] - { - let device = device.as_cuda_device()?; - let func = device.compile(name, kernel)?; - Ok(Self { - name, - func: func.into_cuda_function(), - }) - } - #[cfg(feature = "metal")] - { - let device = device.as_metal_device()?; - let func = device.compile(name, kernel)?; - Ok(Self { name, func }) - } - #[cfg(not(any(feature = "cuda", feature = "metal")))] - { - Ok(Self { name }) - } - } -} - -#[cfg(feature = "ug")] -impl InplaceOp1 for UgIOp1 { - fn name(&self) -> &'static str { - self.name - } - - fn cpu_fwd(&self, _: &mut CpuStorage, _: &Layout) -> Result<()> { - crate::bail!("ug ops are only supported on metal/cuda at the moment") - } - - #[cfg(feature = "metal")] - fn metal_fwd(&self, sto: &mut MetalStorage, layout: &Layout) -> Result<()> { - use crate::backend::BackendStorage; - use objc2_metal; - - let elem_count = layout.shape().elem_count(); - if sto.dtype() != crate::DType::F32 { - // TODO: support more dtypes. - crate::bail!("input is not a f32 tensor") - } - let device = sto.device(); - let encoder = device.command_encoder()?; - encoder.set_compute_pipeline_state(&self.func); - let (g, b) = if elem_count.is_multiple_of(32) { - (elem_count / 32, 32) - } else { - (elem_count, 1) - }; - let grid_dims = objc2_metal::MTLSize { - width: g, - height: 1, - depth: 1, - }; - let group_dims = candle_metal_kernels::utils::get_block_dims(b, 1, 1); - candle_metal_kernels::utils::set_param(&encoder, 0, (sto.buffer(), 0usize)); - - encoder.use_resource(sto.buffer(), objc2_metal::MTLResourceUsage::Write); - encoder.dispatch_threads(grid_dims, group_dims); - - Ok(()) - } - - #[cfg(feature = "cuda")] - fn cuda_fwd(&self, sto: &mut CudaStorage, layout: &Layout) -> Result<()> { - use crate::cuda_backend::WrapErr; - use cudarc::driver::PushKernelArg; - - let elem_count = layout.shape().elem_count(); - let stream = sto.device.cuda_stream(); - // TODO: support more dtypes. - let sto = sto.as_cuda_slice::()?; - let sto = match layout.contiguous_offsets() { - None => crate::bail!("input has to be contiguous"), - Some((o1, o2)) => sto.slice(o1..o2), - }; - let (g, b) = if elem_count % 32 == 0 { - (elem_count / 32, 32) - } else { - (elem_count, 1) - }; - let cfg = cudarc::driver::LaunchConfig { - grid_dim: (g as u32, 1, 1), - block_dim: (b as u32, 1, 1), - shared_mem_bytes: 0, - }; - let mut builder = stream.launch_builder(&self.func); - builder.arg(&sto); - unsafe { builder.launch(cfg) }.w()?; - Ok(()) - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/device.rs b/v2/dependency-patches/candle-core-0.9.2/src/device.rs deleted file mode 100644 index d0167c61e9..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/device.rs +++ /dev/null @@ -1,502 +0,0 @@ -use crate::backend::BackendDevice; -use crate::cpu_backend::CpuDevice; -use crate::{CpuStorage, DType, Result, Shape, Storage, WithDType}; - -/// A `DeviceLocation` represents a physical device whereas multiple `Device` -/// can live on the same location (typically for cuda devices). -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub enum DeviceLocation { - Cpu, - Cuda { gpu_id: usize }, - Metal { gpu_id: usize }, -} - -/// Cpu, Cuda, or Metal -#[derive(Debug, Clone)] -pub enum Device { - Cpu, - Cuda(crate::CudaDevice), - Metal(crate::MetalDevice), -} - -pub trait NdArray { - fn shape(&self) -> Result; - - fn to_cpu_storage(&self) -> CpuStorage; -} - -impl NdArray for S { - fn shape(&self) -> Result { - Ok(Shape::from(())) - } - - fn to_cpu_storage(&self) -> CpuStorage { - S::to_cpu_storage(&[*self]) - } -} - -impl NdArray for &[S; N] { - fn shape(&self) -> Result { - Ok(Shape::from(self.len())) - } - - fn to_cpu_storage(&self) -> CpuStorage { - S::to_cpu_storage(self.as_slice()) - } -} - -impl NdArray for &[S] { - fn shape(&self) -> Result { - Ok(Shape::from(self.len())) - } - - fn to_cpu_storage(&self) -> CpuStorage { - S::to_cpu_storage(self) - } -} - -impl NdArray for &[[S; N]; M] { - fn shape(&self) -> Result { - Ok(Shape::from((M, N))) - } - - fn to_cpu_storage(&self) -> CpuStorage { - S::to_cpu_storage_owned(self.concat()) - } -} - -impl NdArray - for &[[[S; N3]; N2]; N1] -{ - fn shape(&self) -> Result { - Ok(Shape::from((N1, N2, N3))) - } - - fn to_cpu_storage(&self) -> CpuStorage { - let mut vec = Vec::with_capacity(N1 * N2 * N3); - for i1 in 0..N1 { - for i2 in 0..N2 { - vec.extend(self[i1][i2]) - } - } - S::to_cpu_storage_owned(vec) - } -} - -impl NdArray - for &[[[[S; N4]; N3]; N2]; N1] -{ - fn shape(&self) -> Result { - Ok(Shape::from((N1, N2, N3, N4))) - } - - fn to_cpu_storage(&self) -> CpuStorage { - let mut vec = Vec::with_capacity(N1 * N2 * N3 * N4); - for i1 in 0..N1 { - for i2 in 0..N2 { - for i3 in 0..N3 { - vec.extend(self[i1][i2][i3]) - } - } - } - S::to_cpu_storage_owned(vec) - } -} - -impl NdArray for Vec { - fn shape(&self) -> Result { - Ok(Shape::from(self.len())) - } - - fn to_cpu_storage(&self) -> CpuStorage { - S::to_cpu_storage(self.as_slice()) - } -} - -impl NdArray for Vec<&[S]> { - fn shape(&self) -> Result { - if self.is_empty() { - crate::bail!("empty array") - } - let n = self.len(); - let m = self[0].len(); - for v in self.iter() { - if v.len() != m { - crate::bail!("two elements have different len {m} {}", v.len()) - } - } - Ok(Shape::from((n, m))) - } - - fn to_cpu_storage(&self) -> CpuStorage { - let data = self.iter().copied().flatten().copied().collect::>(); - S::to_cpu_storage_owned(data) - } -} - -impl NdArray for Vec> { - fn shape(&self) -> Result { - if self.is_empty() { - crate::bail!("empty array") - } - let n = self.len(); - let m = self[0].len(); - for v in self.iter() { - if v.len() != m { - crate::bail!("two elements have different len {m} {}", v.len()) - } - } - Ok(Shape::from((n, m))) - } - - fn to_cpu_storage(&self) -> CpuStorage { - let len: usize = self.iter().map(|v| v.len()).sum(); - let mut dst = Vec::with_capacity(len); - for v in self.iter() { - dst.extend(v.iter().copied()); - } - S::to_cpu_storage_owned(dst) - } -} - -impl NdArray for Vec>> { - fn shape(&self) -> Result { - if self.is_empty() { - crate::bail!("empty array") - } - let shape0 = self[0].shape()?; - let n = self.len(); - for v in self.iter() { - let shape = v.shape()?; - if shape != shape0 { - crate::bail!("two elements have different shapes {shape:?} {shape0:?}") - } - } - Ok(Shape::from([[n].as_slice(), shape0.dims()].concat())) - } - - fn to_cpu_storage(&self) -> CpuStorage { - if self.is_empty() { - return S::to_cpu_storage_owned(vec![]); - } - let len: usize = self - .iter() - .map(|v| v.iter().map(|v| v.len()).sum::()) - .sum(); - let mut dst = Vec::with_capacity(len); - for v1 in self.iter() { - for v2 in v1.iter() { - dst.extend(v2.iter().copied()); - } - } - S::to_cpu_storage_owned(dst) - } -} - -impl NdArray for Vec>>> { - fn shape(&self) -> Result { - if self.is_empty() { - crate::bail!("empty array") - } - let shape0 = self[0].shape()?; - let n = self.len(); - for v in self.iter() { - let shape = v.shape()?; - if shape != shape0 { - crate::bail!("two elements have different shapes {shape:?} {shape0:?}") - } - } - Ok(Shape::from([[n].as_slice(), shape0.dims()].concat())) - } - - fn to_cpu_storage(&self) -> CpuStorage { - let len: usize = self - .iter() - .map(|v| { - v.iter() - .map(|v| v.iter().map(|v| v.len()).sum::()) - .sum::() - }) - .sum(); - let mut dst = Vec::with_capacity(len); - for v1 in self.iter() { - for v2 in v1.iter() { - for v3 in v2.iter() { - dst.extend(v3.iter().copied()); - } - } - } - S::to_cpu_storage_owned(dst) - } -} - -impl Device { - pub fn new_cuda(ordinal: usize) -> Result { - Ok(Self::Cuda(crate::CudaDevice::new(ordinal)?)) - } - - pub fn as_cuda_device(&self) -> Result<&crate::CudaDevice> { - match self { - Self::Cuda(d) => Ok(d), - Self::Cpu => crate::bail!("expected a cuda device, got cpu"), - Self::Metal(_) => crate::bail!("expected a cuda device, got Metal"), - } - } - - pub fn as_metal_device(&self) -> Result<&crate::MetalDevice> { - match self { - Self::Cuda(_) => crate::bail!("expected a metal device, got cuda"), - Self::Cpu => crate::bail!("expected a metal device, got cpu"), - Self::Metal(d) => Ok(d), - } - } - - pub fn new_cuda_with_stream(ordinal: usize) -> Result { - Ok(Self::Cuda(crate::CudaDevice::new_with_stream(ordinal)?)) - } - - pub fn new_metal(ordinal: usize) -> Result { - Ok(Self::Metal(crate::MetalDevice::new(ordinal)?)) - } - - pub fn set_seed(&self, seed: u64) -> Result<()> { - match self { - Self::Cpu => CpuDevice.set_seed(seed), - Self::Cuda(c) => c.set_seed(seed), - Self::Metal(m) => m.set_seed(seed), - } - } - - pub fn get_current_seed(&self) -> Result { - match self { - Self::Cpu => CpuDevice.get_current_seed(), - Self::Cuda(c) => c.get_current_seed(), - Self::Metal(m) => m.get_current_seed(), - } - } - - pub fn same_device(&self, rhs: &Self) -> bool { - match (self, rhs) { - (Self::Cpu, Self::Cpu) => true, - (Self::Cuda(lhs), Self::Cuda(rhs)) => lhs.same_device(rhs), - (Self::Metal(lhs), Self::Metal(rhs)) => lhs.same_device(rhs), - _ => false, - } - } - - pub fn location(&self) -> DeviceLocation { - match self { - Self::Cpu => DeviceLocation::Cpu, - Self::Cuda(device) => device.location(), - Device::Metal(device) => device.location(), - } - } - - pub fn is_cpu(&self) -> bool { - matches!(self, Self::Cpu) - } - - pub fn is_cuda(&self) -> bool { - matches!(self, Self::Cuda(_)) - } - - pub fn is_metal(&self) -> bool { - matches!(self, Self::Metal(_)) - } - - pub fn supports_bf16(&self) -> bool { - match self { - Self::Cuda(_) | Self::Metal(_) => true, - Self::Cpu => false, - } - } - - /// Return `BF16` for devices that support it, otherwise default to `F32`. - pub fn bf16_default_to_f32(&self) -> DType { - if self.supports_bf16() { - DType::BF16 - } else { - DType::F32 - } - } - - pub fn cuda_if_available(ordinal: usize) -> Result { - if crate::utils::cuda_is_available() { - Self::new_cuda(ordinal) - } else { - Ok(Self::Cpu) - } - } - - pub fn metal_if_available(ordinal: usize) -> Result { - if crate::utils::metal_is_available() { - Self::new_metal(ordinal) - } else { - Ok(Self::Cpu) - } - } - - pub(crate) fn rand_uniform_f64( - &self, - lo: f64, - up: f64, - shape: &Shape, - dtype: DType, - ) -> Result { - match self { - Device::Cpu => { - let storage = CpuDevice.rand_uniform(shape, dtype, lo, up)?; - Ok(Storage::Cpu(storage)) - } - Device::Cuda(device) => { - // TODO: Remove the special case if we start supporting generating f16/bf16 directly. - if dtype == DType::F16 || dtype == DType::BF16 { - let storage = device.rand_uniform(shape, DType::F32, lo, up)?; - Storage::Cuda(storage).to_dtype(&crate::Layout::contiguous(shape), dtype) - } else { - let storage = device.rand_uniform(shape, dtype, lo, up)?; - Ok(Storage::Cuda(storage)) - } - } - Device::Metal(device) => { - let storage = device.rand_uniform(shape, dtype, lo, up)?; - Ok(Storage::Metal(storage)) - } - } - } - - pub(crate) fn rand_uniform( - &self, - lo: T, - up: T, - shape: &Shape, - ) -> Result { - self.rand_uniform_f64(lo.to_f64(), up.to_f64(), shape, T::DTYPE) - } - - pub(crate) fn rand_normal_f64( - &self, - mean: f64, - std: f64, - shape: &Shape, - dtype: DType, - ) -> Result { - match self { - Device::Cpu => { - let storage = CpuDevice.rand_normal(shape, dtype, mean, std)?; - Ok(Storage::Cpu(storage)) - } - Device::Cuda(device) => { - // TODO: Remove the special case if we start supporting generating f16/bf16 directly. - if dtype == DType::F16 || dtype == DType::BF16 { - let storage = device.rand_normal(shape, DType::F32, mean, std)?; - Storage::Cuda(storage).to_dtype(&crate::Layout::contiguous(shape), dtype) - } else { - let storage = device.rand_normal(shape, dtype, mean, std)?; - Ok(Storage::Cuda(storage)) - } - } - Device::Metal(device) => { - let storage = device.rand_normal(shape, dtype, mean, std)?; - Ok(Storage::Metal(storage)) - } - } - } - - pub(crate) fn rand_normal( - &self, - mean: T, - std: T, - shape: &Shape, - ) -> Result { - self.rand_normal_f64(mean.to_f64(), std.to_f64(), shape, T::DTYPE) - } - - pub(crate) fn zeros(&self, shape: &Shape, dtype: DType) -> Result { - match self { - Device::Cpu => { - let storage = CpuDevice.zeros_impl(shape, dtype)?; - Ok(Storage::Cpu(storage)) - } - Device::Cuda(device) => { - let storage = device.zeros_impl(shape, dtype)?; - Ok(Storage::Cuda(storage)) - } - Device::Metal(device) => { - let storage = device.zeros_impl(shape, dtype)?; - Ok(Storage::Metal(storage)) - } - } - } - - pub(crate) unsafe fn alloc_uninit(&self, shape: &Shape, dtype: DType) -> Result { - match self { - Device::Cpu => { - let storage = CpuDevice.alloc_uninit(shape, dtype)?; - Ok(Storage::Cpu(storage)) - } - Device::Cuda(device) => { - let storage = device.alloc_uninit(shape, dtype)?; - Ok(Storage::Cuda(storage)) - } - Device::Metal(device) => { - let storage = device.alloc_uninit(shape, dtype)?; - Ok(Storage::Metal(storage)) - } - } - } - - pub(crate) fn storage_from_slice(&self, data: &[D]) -> Result { - match self { - Device::Cpu => Ok(Storage::Cpu(data.to_cpu_storage())), - Device::Cuda(device) => { - let storage = device.storage_from_slice(data)?; - Ok(Storage::Cuda(storage)) - } - Device::Metal(device) => { - let storage = device.storage_from_slice(data)?; - Ok(Storage::Metal(storage)) - } - } - } - - pub(crate) fn storage(&self, array: A) -> Result { - match self { - Device::Cpu => Ok(Storage::Cpu(array.to_cpu_storage())), - Device::Cuda(device) => { - let storage = array.to_cpu_storage(); - let storage = device.storage_from_cpu_storage_owned(storage)?; - Ok(Storage::Cuda(storage)) - } - Device::Metal(device) => { - let storage = array.to_cpu_storage(); - let storage = device.storage_from_cpu_storage_owned(storage)?; - Ok(Storage::Metal(storage)) - } - } - } - - pub(crate) fn storage_owned(&self, data: Vec) -> Result { - match self { - Device::Cpu => Ok(Storage::Cpu(S::to_cpu_storage_owned(data))), - Device::Cuda(device) => { - let storage = S::to_cpu_storage_owned(data); - let storage = device.storage_from_cpu_storage_owned(storage)?; - Ok(Storage::Cuda(storage)) - } - Device::Metal(device) => { - let storage = S::to_cpu_storage_owned(data); - let storage = device.storage_from_cpu_storage_owned(storage)?; - Ok(Storage::Metal(storage)) - } - } - } - - pub fn synchronize(&self) -> Result<()> { - match self { - Self::Cpu => Ok(()), - Self::Cuda(d) => d.synchronize(), - Self::Metal(d) => d.synchronize(), - } - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/display.rs b/v2/dependency-patches/candle-core-0.9.2/src/display.rs deleted file mode 100644 index a9b53947f3..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/display.rs +++ /dev/null @@ -1,558 +0,0 @@ -//! Pretty printing of tensors -//! -//! This implementation should be in line with the [PyTorch version](https://github.com/pytorch/pytorch/blob/7b419e8513a024e172eae767e24ec1b849976b13/torch/_tensor_str.py). -//! -use crate::{DType, Result, Tensor, WithDType}; -use half::{bf16, f16}; - -impl Tensor { - fn fmt_dt( - &self, - f: &mut std::fmt::Formatter, - ) -> std::fmt::Result { - let device_str = match self.device().location() { - crate::DeviceLocation::Cpu => "".to_owned(), - crate::DeviceLocation::Cuda { gpu_id } => { - format!(", cuda:{gpu_id}") - } - crate::DeviceLocation::Metal { gpu_id } => { - format!(", metal:{gpu_id}") - } - }; - - write!(f, "Tensor[")?; - match self.dims() { - [] => { - if let Ok(v) = self.to_scalar::() { - write!(f, "{v}")? - } - } - [s] if *s < 10 => { - if let Ok(vs) = self.to_vec1::() { - for (i, v) in vs.iter().enumerate() { - if i > 0 { - write!(f, ", ")?; - } - write!(f, "{v}")?; - } - } - } - dims => { - write!(f, "dims ")?; - for (i, d) in dims.iter().enumerate() { - if i > 0 { - write!(f, ", ")?; - } - write!(f, "{d}")?; - } - } - } - write!(f, "; {}{}]", self.dtype().as_str(), device_str) - } -} - -impl std::fmt::Debug for Tensor { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self.dtype() { - DType::U8 => self.fmt_dt::(f), - DType::U32 => self.fmt_dt::(f), - DType::I16 => self.fmt_dt::(f), - DType::I32 => self.fmt_dt::(f), - DType::I64 => self.fmt_dt::(f), - DType::BF16 => self.fmt_dt::(f), - DType::F16 => self.fmt_dt::(f), - DType::F32 => self.fmt_dt::(f), - DType::F64 => self.fmt_dt::(f), - DType::F8E4M3 => self.fmt_dt::(f), - DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { - write!( - f, - "Tensor[{:?}; dtype={}, unsupported dummy type]", - self.shape(), - self.dtype().as_str() - ) - } - } - } -} - -/// Options for Tensor pretty printing -#[derive(Debug, Clone)] -pub struct PrinterOptions { - pub precision: usize, - pub threshold: usize, - pub edge_items: usize, - pub line_width: usize, - pub sci_mode: Option, -} - -static PRINT_OPTS: std::sync::Mutex = - std::sync::Mutex::new(PrinterOptions::const_default()); - -impl PrinterOptions { - // We cannot use the default trait as it's not const. - const fn const_default() -> Self { - Self { - precision: 4, - threshold: 1000, - edge_items: 3, - line_width: 80, - sci_mode: None, - } - } -} - -pub fn print_options() -> &'static std::sync::Mutex { - &PRINT_OPTS -} - -pub fn set_print_options(options: PrinterOptions) { - *PRINT_OPTS.lock().unwrap() = options -} - -pub fn set_print_options_default() { - *PRINT_OPTS.lock().unwrap() = PrinterOptions::const_default() -} - -pub fn set_print_options_short() { - *PRINT_OPTS.lock().unwrap() = PrinterOptions { - precision: 2, - threshold: 1000, - edge_items: 2, - line_width: 80, - sci_mode: None, - } -} - -pub fn set_print_options_full() { - *PRINT_OPTS.lock().unwrap() = PrinterOptions { - precision: 4, - threshold: usize::MAX, - edge_items: 3, - line_width: 80, - sci_mode: None, - } -} - -pub fn set_line_width(line_width: usize) { - PRINT_OPTS.lock().unwrap().line_width = line_width -} - -pub fn set_precision(precision: usize) { - PRINT_OPTS.lock().unwrap().precision = precision -} - -pub fn set_edge_items(edge_items: usize) { - PRINT_OPTS.lock().unwrap().edge_items = edge_items -} - -pub fn set_threshold(threshold: usize) { - PRINT_OPTS.lock().unwrap().threshold = threshold -} - -pub fn set_sci_mode(sci_mode: Option) { - PRINT_OPTS.lock().unwrap().sci_mode = sci_mode -} - -struct FmtSize { - current_size: usize, -} - -impl FmtSize { - fn new() -> Self { - Self { current_size: 0 } - } - - fn final_size(self) -> usize { - self.current_size - } -} - -impl std::fmt::Write for FmtSize { - fn write_str(&mut self, s: &str) -> std::fmt::Result { - self.current_size += s.len(); - Ok(()) - } -} - -trait TensorFormatter { - type Elem: WithDType; - - fn fmt(&self, v: Self::Elem, max_w: usize, f: &mut T) -> std::fmt::Result; - - fn max_width(&self, to_display: &Tensor) -> usize { - let mut max_width = 1; - if let Ok(vs) = to_display.flatten_all().and_then(|t| t.to_vec1()) { - for &v in vs.iter() { - let mut fmt_size = FmtSize::new(); - let _res = self.fmt(v, 1, &mut fmt_size); - max_width = usize::max(max_width, fmt_size.final_size()) - } - } - max_width - } - - fn write_newline_indent(i: usize, f: &mut std::fmt::Formatter) -> std::fmt::Result { - writeln!(f)?; - for _ in 0..i { - write!(f, " ")? - } - Ok(()) - } - - fn fmt_tensor( - &self, - t: &Tensor, - indent: usize, - max_w: usize, - summarize: bool, - po: &PrinterOptions, - f: &mut std::fmt::Formatter, - ) -> std::fmt::Result { - let dims = t.dims(); - let edge_items = po.edge_items; - write!(f, "[")?; - match dims { - [] => { - if let Ok(v) = t.to_scalar::() { - self.fmt(v, max_w, f)? - } - } - [v] if summarize && *v > 2 * edge_items => { - if let Ok(vs) = t - .narrow(0, 0, edge_items) - .and_then(|t| t.to_vec1::()) - { - for v in vs.into_iter() { - self.fmt(v, max_w, f)?; - write!(f, ", ")?; - } - } - write!(f, "...")?; - if let Ok(vs) = t - .narrow(0, v - edge_items, edge_items) - .and_then(|t| t.to_vec1::()) - { - for v in vs.into_iter() { - write!(f, ", ")?; - self.fmt(v, max_w, f)?; - } - } - } - [_] => { - let elements_per_line = usize::max(1, po.line_width / (max_w + 2)); - if let Ok(vs) = t.to_vec1::() { - for (i, v) in vs.into_iter().enumerate() { - if i > 0 { - if i % elements_per_line == 0 { - write!(f, ",")?; - Self::write_newline_indent(indent, f)? - } else { - write!(f, ", ")?; - } - } - self.fmt(v, max_w, f)? - } - } - } - _ => { - if summarize && dims[0] > 2 * edge_items { - for i in 0..edge_items { - match t.get(i) { - Ok(t) => self.fmt_tensor(&t, indent + 1, max_w, summarize, po, f)?, - Err(e) => write!(f, "{e:?}")?, - } - write!(f, ",")?; - Self::write_newline_indent(indent, f)? - } - write!(f, "...")?; - Self::write_newline_indent(indent, f)?; - for i in dims[0] - edge_items..dims[0] { - match t.get(i) { - Ok(t) => self.fmt_tensor(&t, indent + 1, max_w, summarize, po, f)?, - Err(e) => write!(f, "{e:?}")?, - } - if i + 1 != dims[0] { - write!(f, ",")?; - Self::write_newline_indent(indent, f)? - } - } - } else { - for i in 0..dims[0] { - match t.get(i) { - Ok(t) => self.fmt_tensor(&t, indent + 1, max_w, summarize, po, f)?, - Err(e) => write!(f, "{e:?}")?, - } - if i + 1 != dims[0] { - write!(f, ",")?; - Self::write_newline_indent(indent, f)? - } - } - } - } - } - write!(f, "]")?; - Ok(()) - } -} - -struct FloatFormatter { - int_mode: bool, - sci_mode: bool, - precision: usize, - _phantom: std::marker::PhantomData, -} - -impl FloatFormatter -where - S: WithDType + num_traits::Float + std::fmt::Display, -{ - fn new(t: &Tensor, po: &PrinterOptions) -> Result { - let mut int_mode = true; - let mut sci_mode = false; - - // Rather than containing all values, this should only include - // values that end up being displayed according to [threshold]. - let values = t - .flatten_all()? - .to_vec1()? - .into_iter() - .filter(|v: &S| v.is_finite() && !v.is_zero()) - .collect::>(); - if !values.is_empty() { - let mut nonzero_finite_min = S::max_value(); - let mut nonzero_finite_max = S::min_value(); - for &v in values.iter() { - let v = v.abs(); - if v < nonzero_finite_min { - nonzero_finite_min = v - } - if v > nonzero_finite_max { - nonzero_finite_max = v - } - } - - for &value in values.iter() { - if value.ceil() != value { - int_mode = false; - break; - } - } - if let Some(v1) = S::from(1000.) { - if let Some(v2) = S::from(1e8) { - if let Some(v3) = S::from(1e-4) { - sci_mode = nonzero_finite_max / nonzero_finite_min > v1 - || nonzero_finite_max > v2 - || nonzero_finite_min < v3 - } - } - } - } - - match po.sci_mode { - None => {} - Some(v) => sci_mode = v, - } - Ok(Self { - int_mode, - sci_mode, - precision: po.precision, - _phantom: std::marker::PhantomData, - }) - } -} - -impl TensorFormatter for FloatFormatter -where - S: WithDType + num_traits::Float + std::fmt::Display + std::fmt::LowerExp, -{ - type Elem = S; - - fn fmt(&self, v: Self::Elem, max_w: usize, f: &mut T) -> std::fmt::Result { - if self.sci_mode { - write!( - f, - "{v:width$.prec$e}", - v = v, - width = max_w, - prec = self.precision - ) - } else if self.int_mode { - if v.is_finite() { - write!(f, "{v:width$.0}.", v = v, width = max_w - 1) - } else { - write!(f, "{v:max_w$.0}") - } - } else { - write!( - f, - "{v:width$.prec$}", - v = v, - width = max_w, - prec = self.precision - ) - } - } -} - -struct IntFormatter { - _phantom: std::marker::PhantomData, -} - -impl IntFormatter { - fn new() -> Self { - Self { - _phantom: std::marker::PhantomData, - } - } -} - -impl TensorFormatter for IntFormatter -where - S: WithDType + std::fmt::Display, -{ - type Elem = S; - - fn fmt(&self, v: Self::Elem, max_w: usize, f: &mut T) -> std::fmt::Result { - write!(f, "{v:max_w$}") - } -} - -fn get_summarized_data(t: &Tensor, edge_items: usize) -> Result { - let dims = t.dims(); - if dims.is_empty() { - Ok(t.clone()) - } else if dims.len() == 1 { - if dims[0] > 2 * edge_items { - Tensor::cat( - &[ - t.narrow(0, 0, edge_items)?, - t.narrow(0, dims[0] - edge_items, edge_items)?, - ], - 0, - ) - } else { - Ok(t.clone()) - } - } else if dims[0] > 2 * edge_items { - let mut vs: Vec<_> = (0..edge_items) - .map(|i| get_summarized_data(&t.get(i)?, edge_items)) - .collect::>>()?; - for i in (dims[0] - edge_items)..dims[0] { - vs.push(get_summarized_data(&t.get(i)?, edge_items)?) - } - Tensor::cat(&vs, 0) - } else { - let vs: Vec<_> = (0..dims[0]) - .map(|i| get_summarized_data(&t.get(i)?, edge_items)) - .collect::>>()?; - Tensor::cat(&vs, 0) - } -} - -impl std::fmt::Display for Tensor { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - let po = PRINT_OPTS.lock().unwrap(); - let summarize = self.elem_count() > po.threshold; - let to_display = if summarize { - match get_summarized_data(self, po.edge_items) { - Ok(v) => v, - Err(err) => return write!(f, "{err:?}"), - } - } else { - self.clone() - }; - match self.dtype() { - DType::U8 => { - let tf: IntFormatter = IntFormatter::new(); - let max_w = tf.max_width(&to_display); - tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; - writeln!(f)?; - } - DType::U32 => { - let tf: IntFormatter = IntFormatter::new(); - let max_w = tf.max_width(&to_display); - tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; - writeln!(f)?; - } - DType::I16 => { - let tf: IntFormatter = IntFormatter::new(); - let max_w = tf.max_width(&to_display); - tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; - writeln!(f)?; - } - DType::I32 => { - let tf: IntFormatter = IntFormatter::new(); - let max_w = tf.max_width(&to_display); - tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; - writeln!(f)?; - } - DType::I64 => { - let tf: IntFormatter = IntFormatter::new(); - let max_w = tf.max_width(&to_display); - tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; - writeln!(f)?; - } - DType::BF16 => { - if let Ok(tf) = FloatFormatter::::new(&to_display, &po) { - let max_w = tf.max_width(&to_display); - tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; - writeln!(f)?; - } - } - DType::F16 => { - if let Ok(tf) = FloatFormatter::::new(&to_display, &po) { - let max_w = tf.max_width(&to_display); - tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; - writeln!(f)?; - } - } - DType::F64 => { - if let Ok(tf) = FloatFormatter::::new(&to_display, &po) { - let max_w = tf.max_width(&to_display); - tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; - writeln!(f)?; - } - } - DType::F32 => { - if let Ok(tf) = FloatFormatter::::new(&to_display, &po) { - let max_w = tf.max_width(&to_display); - tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; - writeln!(f)?; - } - } - DType::F8E4M3 => { - if let Ok(tf) = FloatFormatter::::new(&to_display, &po) { - let max_w = tf.max_width(&to_display); - tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; - writeln!(f)?; - } - } - DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { - writeln!( - f, - "Dummy type {} (not supported for display)", - self.dtype().as_str() - )?; - } - }; - - let device_str = match self.device().location() { - crate::DeviceLocation::Cpu => "".to_owned(), - crate::DeviceLocation::Cuda { gpu_id } => { - format!(", cuda:{gpu_id}") - } - crate::DeviceLocation::Metal { gpu_id } => { - format!(", metal:{gpu_id}") - } - }; - - write!( - f, - "Tensor[{:?}, {}{}]", - self.dims(), - self.dtype().as_str(), - device_str - ) - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/dtype.rs b/v2/dependency-patches/candle-core-0.9.2/src/dtype.rs deleted file mode 100644 index 035ca6d503..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/dtype.rs +++ /dev/null @@ -1,294 +0,0 @@ -//! Types for elements that can be stored and manipulated using tensors. -#![allow(clippy::redundant_closure_call)] -use crate::backend::BackendStorage; -use crate::{CpuStorage, CpuStorageRef, Error, Result}; - -/// The different types of elements allowed in tensors. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub enum DType { - // Unsigned 8 bits integer. - U8, - // Unsigned 32 bits integer. - U32, - // Signed 16 bits integer. - I16, - // Signed 32 bits integer. - I32, - // Signed 64 bits integer. - I64, - // Brain floating-point using half precision (16 bits). - BF16, - // Floating-point using half precision (16 bits). - F16, - // Floating-point using single precision (32 bits). - F32, - // Floating-point using double precision (64 bits). - F64, - // 8-bit floating point with 4-bit exponent and 3-bit mantissa. - F8E4M3, - /// 6-bit float with 2 exponent bits and 3 mantissa bits (MX6 format) - F6E2M3, - /// 6-bit float with 3 exponent bits and 2 mantissa bits (MX6 format) - F6E3M2, - /// 4-bit float (MX4 format) - F4, - /// 8-bit float with 8 exponent bits and 0 mantissa bits - F8E8M0, -} - -#[derive(Debug, PartialEq, Eq)] -pub struct DTypeParseError(String); - -impl std::fmt::Display for DTypeParseError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "cannot parse '{}' as a dtype", self.0) - } -} - -impl std::error::Error for DTypeParseError {} - -impl std::str::FromStr for DType { - type Err = DTypeParseError; - fn from_str(s: &str) -> std::result::Result { - match s { - "u8" => Ok(Self::U8), - "u32" => Ok(Self::U32), - "i16" => Ok(Self::I16), - "i32" => Ok(Self::I32), - "i64" => Ok(Self::I64), - "bf16" => Ok(Self::BF16), - "f16" => Ok(Self::F16), - "f32" => Ok(Self::F32), - "f64" => Ok(Self::F64), - "f8e4m3" => Ok(Self::F8E4M3), - "f6e2m3" => Ok(Self::F6E2M3), - "f6e3m2" => Ok(Self::F6E3M2), - "f4" => Ok(Self::F4), - "f8e8m0" => Ok(Self::F8E8M0), - _ => Err(DTypeParseError(s.to_string())), - } - } -} - -impl DType { - /// String representation for dtypes. - pub fn as_str(&self) -> &'static str { - match self { - Self::U8 => "u8", - Self::U32 => "u32", - Self::I16 => "i16", - Self::I32 => "i32", - Self::I64 => "i64", - Self::BF16 => "bf16", - Self::F16 => "f16", - Self::F32 => "f32", - Self::F64 => "f64", - Self::F8E4M3 => "f8e4m3", - Self::F6E2M3 => "f6e2m3", - Self::F6E3M2 => "f6e3m2", - Self::F4 => "f4", - Self::F8E8M0 => "f8e8m0", - } - } - - /// The size used by each element in bytes, i.e. 1 for `U8`, 4 for `F32`. - pub fn size_in_bytes(&self) -> usize { - match self { - Self::U8 => 1, - Self::U32 => 4, - Self::I16 => 2, - Self::I32 => 4, - Self::I64 => 8, - Self::BF16 => 2, - Self::F16 => 2, - Self::F32 => 4, - Self::F64 => 8, - Self::F8E4M3 => 1, - Self::F6E2M3 => 0, // 6 bits - Self::F6E3M2 => 0, // 6 bits - Self::F4 => 0, // 4 bits - Self::F8E8M0 => 1, - } - } - - pub fn is_int(&self) -> bool { - match self { - Self::U8 | Self::U32 | Self::I16 | Self::I32 | Self::I64 => true, - Self::BF16 - | Self::F16 - | Self::F32 - | Self::F64 - | Self::F8E4M3 - | Self::F6E2M3 - | Self::F6E3M2 - | Self::F4 - | Self::F8E8M0 => false, - } - } - - pub fn is_float(&self) -> bool { - match self { - Self::U8 | Self::U32 | Self::I16 | Self::I32 | Self::I64 => false, - Self::BF16 - | Self::F16 - | Self::F32 - | Self::F64 - | Self::F8E4M3 - | Self::F6E2M3 - | Self::F6E3M2 - | Self::F4 - | Self::F8E8M0 => true, - } - } -} - -pub trait WithDType: - Sized - + Copy - + num_traits::NumAssign - + std::cmp::PartialOrd - + std::fmt::Display - + 'static - + Send - + Sync - + std::any::Any - + crate::cpu::kernels::VecOps -{ - const DTYPE: DType; - - fn from_f64(v: f64) -> Self; - fn to_f64(self) -> f64; - fn to_scalar(self) -> crate::scalar::Scalar; - fn cpu_storage_ref(data: &[Self]) -> CpuStorageRef<'_>; - fn to_cpu_storage_owned(data: Vec) -> CpuStorage; - - fn to_cpu_storage(data: &[Self]) -> CpuStorage { - Self::to_cpu_storage_owned(data.to_vec()) - } - - fn cpu_storage_as_slice(s: &CpuStorage) -> Result<&[Self]>; - fn cpu_storage_data(s: CpuStorage) -> Result>; -} - -macro_rules! with_dtype { - ($ty:ty, $dtype:ident, $from_f64:expr, $to_f64:expr) => { - impl WithDType for $ty { - const DTYPE: DType = DType::$dtype; - - fn from_f64(v: f64) -> Self { - $from_f64(v) - } - - fn to_f64(self) -> f64 { - $to_f64(self) - } - - fn to_scalar(self) -> crate::scalar::Scalar { - crate::scalar::Scalar::$dtype(self) - } - - fn cpu_storage_ref(data: &[Self]) -> CpuStorageRef<'_> { - CpuStorageRef::$dtype(data) - } - - fn to_cpu_storage_owned(data: Vec) -> CpuStorage { - CpuStorage::$dtype(data) - } - - fn cpu_storage_data(s: CpuStorage) -> Result> { - match s { - CpuStorage::$dtype(data) => Ok(data), - _ => Err(Error::UnexpectedDType { - expected: DType::$dtype, - got: s.dtype(), - msg: "unexpected dtype", - } - .bt()), - } - } - - fn cpu_storage_as_slice(s: &CpuStorage) -> Result<&[Self]> { - match s { - CpuStorage::$dtype(data) => Ok(data), - _ => Err(Error::UnexpectedDType { - expected: DType::$dtype, - got: s.dtype(), - msg: "unexpected dtype", - } - .bt()), - } - } - } - }; -} -use float8::F8E4M3 as f8e4m3; -use half::{bf16, f16}; - -with_dtype!(u8, U8, |v: f64| v as u8, |v: u8| v as f64); -with_dtype!(u32, U32, |v: f64| v as u32, |v: u32| v as f64); -with_dtype!(i16, I16, |v: f64| v as i16, |v: i16| v as f64); -with_dtype!(i32, I32, |v: f64| v as i32, |v: i32| v as f64); -with_dtype!(i64, I64, |v: f64| v as i64, |v: i64| v as f64); -with_dtype!(f16, F16, f16::from_f64, f16::to_f64); -with_dtype!(bf16, BF16, bf16::from_f64, bf16::to_f64); -with_dtype!(f32, F32, |v: f64| v as f32, |v: f32| v as f64); -with_dtype!(f64, F64, |v: f64| v, |v: f64| v); -with_dtype!(f8e4m3, F8E4M3, f8e4m3::from_f64, |v: f8e4m3| v.to_f64()); - -pub trait IntDType: WithDType + num_traits::Bounded { - fn is_true(&self) -> bool; - fn as_usize(&self) -> usize; -} - -impl IntDType for i64 { - fn is_true(&self) -> bool { - *self != 0 - } - fn as_usize(&self) -> usize { - *self as usize - } -} - -impl IntDType for u32 { - fn is_true(&self) -> bool { - *self != 0 - } - fn as_usize(&self) -> usize { - *self as usize - } -} - -impl IntDType for u8 { - fn is_true(&self) -> bool { - *self != 0 - } - fn as_usize(&self) -> usize { - *self as usize - } -} - -impl IntDType for i16 { - fn is_true(&self) -> bool { - *self != 0 - } - fn as_usize(&self) -> usize { - *self as usize - } -} - -impl IntDType for i32 { - fn is_true(&self) -> bool { - *self != 0 - } - fn as_usize(&self) -> usize { - *self as usize - } -} - -pub trait FloatDType: WithDType {} - -impl FloatDType for f16 {} -impl FloatDType for bf16 {} -impl FloatDType for f32 {} -impl FloatDType for f64 {} -impl FloatDType for f8e4m3 {} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/dummy_cuda_backend.rs b/v2/dependency-patches/candle-core-0.9.2/src/dummy_cuda_backend.rs deleted file mode 100644 index 2aa4585966..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/dummy_cuda_backend.rs +++ /dev/null @@ -1,317 +0,0 @@ -//! Implementation of the Cuda backend when Cuda support has not been compiled in. -//! -#![allow(dead_code)] -use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; -use crate::{CpuStorage, DType, Error, Layout, Result, Shape}; - -#[derive(Debug, Clone)] -pub struct CudaDevice; - -#[derive(Debug)] -pub struct CudaStorage; - -impl CudaStorage { - pub fn transfer_to_device(&self, _dst: &CudaDevice) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } -} - -macro_rules! fail { - () => { - unimplemented!("cuda support has not been enabled, add `cuda` feature to enable.") - }; -} -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct DeviceId(usize); - -impl CudaDevice { - pub fn new_with_stream(_: usize) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - pub fn id(&self) -> DeviceId { - DeviceId(0) - } -} - -impl crate::backend::BackendStorage for CudaStorage { - type Device = CudaDevice; - - fn try_clone(&self, _: &Layout) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn dtype(&self) -> DType { - fail!() - } - - fn device(&self) -> &Self::Device { - fail!() - } - - fn const_set(&mut self, _: crate::scalar::Scalar, _: &Layout) -> Result<()> { - Err(Error::NotCompiledWithCudaSupport) - } - - fn to_cpu_storage(&self) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn affine(&self, _: &Layout, _: f64, _: f64) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn powf(&self, _: &Layout, _: f64) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn elu(&self, _: &Layout, _: f64) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn reduce_op(&self, _: ReduceOp, _: &Layout, _: &[usize]) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn cmp(&self, _: CmpOp, _: &Self, _: &Layout, _: &Layout) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn to_dtype(&self, _: &Layout, _: DType) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn unary_impl(&self, _: &Layout) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn binary_impl(&self, _: &Self, _: &Layout, _: &Layout) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn where_cond(&self, _: &Layout, _: &Self, _: &Layout, _: &Self, _: &Layout) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn conv1d( - &self, - _: &Layout, - _: &Self, - _: &Layout, - _: &crate::conv::ParamsConv1D, - ) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn conv_transpose1d( - &self, - _: &Layout, - _: &Self, - _: &Layout, - _: &crate::conv::ParamsConvTranspose1D, - ) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn conv2d( - &self, - _: &Layout, - _: &Self, - _: &Layout, - _: &crate::conv::ParamsConv2D, - ) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn conv_transpose2d( - &self, - _l: &Layout, - _kernel: &Self, - _kernel_l: &Layout, - _params: &crate::conv::ParamsConvTranspose2D, - ) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn index_select(&self, _: &Self, _: &Layout, _: &Layout, _: usize) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - fn gather(&self, _: &Layout, _: &Self, _: &Layout, _: usize) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn scatter_set( - &mut self, - _: &Layout, - _: &Self, - _: &Layout, - _: &Self, - _: &Layout, - _: usize, - ) -> Result<()> { - Err(Error::NotCompiledWithCudaSupport) - } - - fn scatter_add_set( - &mut self, - _: &Layout, - _: &Self, - _: &Layout, - _: &Self, - _: &Layout, - _: usize, - ) -> Result<()> { - Err(Error::NotCompiledWithCudaSupport) - } - - fn index_add( - &self, - _: &Layout, - _: &Self, - _: &Layout, - _: &Self, - _: &Layout, - _: usize, - ) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn matmul( - &self, - _: &Self, - _: (usize, usize, usize, usize), - _: &Layout, - _: &Layout, - ) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn copy_strided_src(&self, _: &mut Self, _: usize, _: &Layout) -> Result<()> { - Err(Error::NotCompiledWithCudaSupport) - } - - fn copy2d( - &self, - _: &mut Self, - _: usize, - _: usize, - _: usize, - _: usize, - _: usize, - _: usize, - ) -> Result<()> { - Err(Error::NotCompiledWithCudaSupport) - } - - fn avg_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn max_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn upsample_nearest1d(&self, _: &Layout, _: usize) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn upsample_nearest2d(&self, _: &Layout, _: usize, _: usize) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn upsample_bilinear2d( - &self, - _: &Layout, - _: usize, - _: usize, - _: bool, - _: Option, - _: Option, - ) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } -} - -impl crate::backend::BackendDevice for CudaDevice { - type Storage = CudaStorage; - fn new(_: usize) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn set_seed(&self, _: u64) -> Result<()> { - Err(Error::NotCompiledWithCudaSupport) - } - - fn get_current_seed(&self) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn location(&self) -> crate::DeviceLocation { - fail!() - } - - fn same_device(&self, _: &Self) -> bool { - fail!() - } - - fn zeros_impl(&self, _shape: &Shape, _dtype: DType) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - unsafe fn alloc_uninit(&self, _shape: &Shape, _dtype: DType) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn storage_from_slice(&self, _: &[T]) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn storage_from_cpu_storage(&self, _: &CpuStorage) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn storage_from_cpu_storage_owned(&self, _: CpuStorage) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn rand_uniform(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn rand_normal(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - fn synchronize(&self) -> Result<()> { - Ok(()) - } -} - -/// This bool controls whether reduced precision reductions (e.g., with fp16 accumulation type) are -/// allowed with f16 GEMMs. -pub fn gemm_reduced_precision_f16() -> bool { - true -} - -/// This bool controls whether reduced precision reductions (e.g., with fp16 accumulation type) are -/// allowed with f16 GEMMs. -pub fn set_gemm_reduced_precision_f16(_: bool) {} - -/// This bool controls whether reduced precision reductions (e.g., with fp16 accumulation type) are -/// allowed with bf16 GEMMs. -pub fn gemm_reduced_precision_bf16() -> bool { - true -} - -/// This bool controls whether reduced precision reductions (e.g., with fp16 accumulation type) are -/// allowed with bf16 GEMMs. -pub fn set_gemm_reduced_precision_bf16(_: bool) {} - -/// This bool controls whether reduced precision reductions (e.g., with tf32 accumulation type) are -/// allowed with f32 GEMMs. -pub fn gemm_reduced_precision_f32() -> bool { - true -} - -/// This bool controls whether reduced precision reductions (e.g., with tf32 accumulation type) are -/// allowed with f32 GEMMs. -pub fn set_gemm_reduced_precision_f32(_b: bool) {} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/dummy_dtype.rs b/v2/dependency-patches/candle-core-0.9.2/src/dummy_dtype.rs deleted file mode 100644 index 5fdb0961a8..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/dummy_dtype.rs +++ /dev/null @@ -1,268 +0,0 @@ -//! Dummy data types for experimental/future float formats -//! -//! These are placeholder types for experimental floating-point formats -//! that are defined in the safetensors spec but not yet fully implemented. - -use crate::{DType, Error, Result, WithDType}; - -/// 6-bit float with 2 exponent bits and 3 mantissa bits (MX6 format) -/// This is a dummy type. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct F6E2M3; - -/// 6-bit float with 3 exponent bits and 2 mantissa bits (MX6 format) -/// This is a dummy type. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct F6E3M2; - -/// 4-bit float (MX4 format) -/// This is a dummy type. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct F4; - -/// 8-bit float with 8 exponent bits and 0 mantissa bits -/// This is a dummy type. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct F8E8M0; - -// Implement WithDType for dummy types -macro_rules! dummy_with_dtype { - ($ty:ty, $dtype:ident) => { - impl WithDType for $ty { - const DTYPE: DType = DType::$dtype; - - fn from_f64(_v: f64) -> Self { - panic!( - "{} is a dummy type and cannot be constructed", - stringify!($ty) - ) - } - - fn to_f64(self) -> f64 { - panic!( - "{} is a dummy type and cannot be converted", - stringify!($ty) - ) - } - - fn to_scalar(self) -> crate::scalar::Scalar { - panic!( - "{} is a dummy type and cannot be converted to scalar", - stringify!($ty) - ) - } - - fn cpu_storage_ref(_data: &[Self]) -> crate::CpuStorageRef<'_> { - panic!( - "{} is a dummy type and does not support storage", - stringify!($ty) - ) - } - - fn to_cpu_storage_owned(_data: Vec) -> crate::CpuStorage { - panic!( - "{} is a dummy type and does not support storage", - stringify!($ty) - ) - } - - fn cpu_storage_data(_s: crate::CpuStorage) -> Result> { - Err(Error::UnsupportedDTypeForOp(DType::$dtype, "cpu_storage_data").bt()) - } - - fn cpu_storage_as_slice(_s: &crate::CpuStorage) -> Result<&[Self]> { - Err(Error::UnsupportedDTypeForOp(DType::$dtype, "cpu_storage_as_slice").bt()) - } - } - }; -} - -dummy_with_dtype!(F6E2M3, F6E2M3); -dummy_with_dtype!(F6E3M2, F6E3M2); -dummy_with_dtype!(F4, F4); -dummy_with_dtype!(F8E8M0, F8E8M0); - -// Implement NumAssign traits for dummy types -macro_rules! dummy_num_assign { - ($ty:ty) => { - impl std::ops::AddAssign for $ty { - fn add_assign(&mut self, _other: Self) { - panic!( - "{} is a dummy type and does not support operations", - stringify!($ty) - ) - } - } - - impl std::ops::SubAssign for $ty { - fn sub_assign(&mut self, _other: Self) { - panic!( - "{} is a dummy type and does not support operations", - stringify!($ty) - ) - } - } - - impl std::ops::MulAssign for $ty { - fn mul_assign(&mut self, _other: Self) { - panic!( - "{} is a dummy type and does not support operations", - stringify!($ty) - ) - } - } - - impl std::ops::DivAssign for $ty { - fn div_assign(&mut self, _other: Self) { - panic!( - "{} is a dummy type and does not support operations", - stringify!($ty) - ) - } - } - - impl std::ops::RemAssign for $ty { - fn rem_assign(&mut self, _other: Self) { - panic!( - "{} is a dummy type and does not support operations", - stringify!($ty) - ) - } - } - - impl std::ops::Add for $ty { - type Output = Self; - fn add(self, _other: Self) -> Self { - panic!( - "{} is a dummy type and does not support operations", - stringify!($ty) - ) - } - } - - impl std::ops::Sub for $ty { - type Output = Self; - fn sub(self, _other: Self) -> Self { - panic!( - "{} is a dummy type and does not support operations", - stringify!($ty) - ) - } - } - - impl std::ops::Mul for $ty { - type Output = Self; - fn mul(self, _other: Self) -> Self { - panic!( - "{} is a dummy type and does not support operations", - stringify!($ty) - ) - } - } - - impl std::ops::Div for $ty { - type Output = Self; - fn div(self, _other: Self) -> Self { - panic!( - "{} is a dummy type and does not support operations", - stringify!($ty) - ) - } - } - - impl std::ops::Rem for $ty { - type Output = Self; - fn rem(self, _other: Self) -> Self { - panic!( - "{} is a dummy type and does not support operations", - stringify!($ty) - ) - } - } - - impl num_traits::Zero for $ty { - fn zero() -> Self { - panic!( - "{} is a dummy type and does not support operations", - stringify!($ty) - ) - } - - fn is_zero(&self) -> bool { - panic!( - "{} is a dummy type and does not support operations", - stringify!($ty) - ) - } - } - - impl num_traits::One for $ty { - fn one() -> Self { - panic!( - "{} is a dummy type and does not support operations", - stringify!($ty) - ) - } - } - - impl num_traits::Num for $ty { - type FromStrRadixErr = std::num::ParseFloatError; - - fn from_str_radix( - _str: &str, - _radix: u32, - ) -> std::result::Result { - panic!( - "{} is a dummy type and does not support parsing", - stringify!($ty) - ) - } - } - - impl crate::cpu::kernels::VecOps for $ty { - fn min(self, _other: Self) -> Self { - panic!( - "{} is a dummy type and does not support operations", - stringify!($ty) - ) - } - - fn max(self, _other: Self) -> Self { - panic!( - "{} is a dummy type and does not support operations", - stringify!($ty) - ) - } - } - }; -} - -dummy_num_assign!(F6E2M3); -dummy_num_assign!(F6E3M2); -dummy_num_assign!(F4); -dummy_num_assign!(F8E8M0); - -// Display implementations -impl std::fmt::Display for F6E2M3 { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "F6E2M3") - } -} - -impl std::fmt::Display for F6E3M2 { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "F6E3M2") - } -} - -impl std::fmt::Display for F4 { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "F4") - } -} - -impl std::fmt::Display for F8E8M0 { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "F8E8M0") - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/dummy_metal_backend.rs b/v2/dependency-patches/candle-core-0.9.2/src/dummy_metal_backend.rs deleted file mode 100644 index 8c23b580fc..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/dummy_metal_backend.rs +++ /dev/null @@ -1,280 +0,0 @@ -#![allow(dead_code)] -use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; -use crate::{CpuStorage, DType, Error, Layout, Result, Shape}; - -#[derive(Debug, Clone)] -pub struct MetalDevice; - -#[derive(Debug)] -pub struct MetalStorage; - -#[derive(thiserror::Error, Debug)] -pub enum MetalError { - #[error("{0}")] - Message(String), -} - -impl From for MetalError { - fn from(e: String) -> Self { - MetalError::Message(e) - } -} - -macro_rules! fail { - () => { - unimplemented!("metal support has not been enabled, add `metal` feature to enable.") - }; -} - -impl crate::backend::BackendStorage for MetalStorage { - type Device = MetalDevice; - - fn try_clone(&self, _: &Layout) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn dtype(&self) -> DType { - fail!() - } - - fn device(&self) -> &Self::Device { - fail!() - } - - fn const_set(&mut self, _: crate::scalar::Scalar, _: &Layout) -> Result<()> { - Err(Error::NotCompiledWithMetalSupport) - } - - fn to_cpu_storage(&self) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn affine(&self, _: &Layout, _: f64, _: f64) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn powf(&self, _: &Layout, _: f64) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn elu(&self, _: &Layout, _: f64) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn reduce_op(&self, _: ReduceOp, _: &Layout, _: &[usize]) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn cmp(&self, _: CmpOp, _: &Self, _: &Layout, _: &Layout) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn to_dtype(&self, _: &Layout, _: DType) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn unary_impl(&self, _: &Layout) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn binary_impl(&self, _: &Self, _: &Layout, _: &Layout) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn where_cond(&self, _: &Layout, _: &Self, _: &Layout, _: &Self, _: &Layout) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn conv1d( - &self, - _: &Layout, - _: &Self, - _: &Layout, - _: &crate::conv::ParamsConv1D, - ) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn conv_transpose1d( - &self, - _l: &Layout, - _kernel: &Self, - _kernel_l: &Layout, - _params: &crate::conv::ParamsConvTranspose1D, - ) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn conv2d( - &self, - _: &Layout, - _: &Self, - _: &Layout, - _: &crate::conv::ParamsConv2D, - ) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn conv_transpose2d( - &self, - _l: &Layout, - _kernel: &Self, - _kernel_l: &Layout, - _params: &crate::conv::ParamsConvTranspose2D, - ) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn index_select(&self, _: &Self, _: &Layout, _: &Layout, _: usize) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - fn gather(&self, _: &Layout, _: &Self, _: &Layout, _: usize) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn scatter_set( - &mut self, - _: &Layout, - _: &Self, - _: &Layout, - _: &Self, - _: &Layout, - _: usize, - ) -> Result<()> { - Err(Error::NotCompiledWithMetalSupport) - } - - fn scatter_add_set( - &mut self, - _: &Layout, - _: &Self, - _: &Layout, - _: &Self, - _: &Layout, - _: usize, - ) -> Result<()> { - Err(Error::NotCompiledWithMetalSupport) - } - - fn index_add( - &self, - _: &Layout, - _: &Self, - _: &Layout, - _: &Self, - _: &Layout, - _: usize, - ) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn matmul( - &self, - _: &Self, - _: (usize, usize, usize, usize), - _: &Layout, - _: &Layout, - ) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn copy_strided_src(&self, _: &mut Self, _: usize, _: &Layout) -> Result<()> { - Err(Error::NotCompiledWithMetalSupport) - } - - fn copy2d( - &self, - _: &mut Self, - _: usize, - _: usize, - _: usize, - _: usize, - _: usize, - _: usize, - ) -> Result<()> { - Err(Error::NotCompiledWithMetalSupport) - } - - fn avg_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn max_pool2d(&self, _: &Layout, _: (usize, usize), _: (usize, usize)) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn upsample_nearest1d(&self, _: &Layout, _: usize) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn upsample_nearest2d(&self, _: &Layout, _: usize, _: usize) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn upsample_bilinear2d( - &self, - _: &Layout, - _: usize, - _: usize, - _: bool, - _: Option, - _: Option, - ) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } -} - -impl crate::backend::BackendDevice for MetalDevice { - type Storage = MetalStorage; - fn new(_: usize) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn set_seed(&self, _: u64) -> Result<()> { - Err(Error::NotCompiledWithMetalSupport) - } - - fn get_current_seed(&self) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn location(&self) -> crate::DeviceLocation { - fail!() - } - - fn same_device(&self, _: &Self) -> bool { - fail!() - } - - fn zeros_impl(&self, _shape: &Shape, _dtype: DType) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - unsafe fn alloc_uninit(&self, _shape: &Shape, _dtype: DType) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn storage_from_slice(&self, _: &[T]) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn storage_from_cpu_storage(&self, _: &CpuStorage) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn storage_from_cpu_storage_owned(&self, _: CpuStorage) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn rand_uniform(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn rand_normal(&self, _: &Shape, _: DType, _: f64, _: f64) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - fn synchronize(&self) -> Result<()> { - Ok(()) - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/error.rs b/v2/dependency-patches/candle-core-0.9.2/src/error.rs deleted file mode 100644 index 9f10ca29cf..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/error.rs +++ /dev/null @@ -1,394 +0,0 @@ -//! Candle-specific Error and Result -use std::{convert::Infallible, fmt::Display}; - -use crate::{DType, DeviceLocation, Layout, MetalError, Shape}; - -#[derive(Debug, Clone)] -pub struct MatMulUnexpectedStriding { - pub lhs_l: Layout, - pub rhs_l: Layout, - pub bmnk: (usize, usize, usize, usize), - pub msg: &'static str, -} - -impl std::fmt::Debug for Error { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{self}") - } -} - -/// Main library error type. -#[derive(thiserror::Error)] -pub enum Error { - // === DType Errors === - #[error("{msg}, expected: {expected:?}, got: {got:?}")] - UnexpectedDType { - msg: &'static str, - expected: DType, - got: DType, - }, - - #[error("dtype mismatch in {op}, lhs: {lhs:?}, rhs: {rhs:?}")] - DTypeMismatchBinaryOp { - lhs: DType, - rhs: DType, - op: &'static str, - }, - - #[error("unsupported dtype {0:?} for op {1}")] - UnsupportedDTypeForOp(DType, &'static str), - - // === Dimension Index Errors === - #[error("{op}: dimension index {dim} out of range for shape {shape:?}")] - DimOutOfRange { - shape: Shape, - dim: i32, - op: &'static str, - }, - - #[error("{op}: duplicate dim index {dims:?} for shape {shape:?}")] - DuplicateDimIndex { - shape: Shape, - dims: Vec, - op: &'static str, - }, - - // === Shape Errors === - #[error("unexpected rank, expected: {expected}, got: {got} ({shape:?})")] - UnexpectedNumberOfDims { - expected: usize, - got: usize, - shape: Shape, - }, - - #[error("{msg}, expected: {expected:?}, got: {got:?}")] - UnexpectedShape { - msg: String, - expected: Shape, - got: Shape, - }, - - #[error( - "Shape mismatch, got buffer of size {buffer_size} which is compatible with shape {shape:?}" - )] - ShapeMismatch { buffer_size: usize, shape: Shape }, - - #[error("shape mismatch in {op}, lhs: {lhs:?}, rhs: {rhs:?}")] - ShapeMismatchBinaryOp { - lhs: Shape, - rhs: Shape, - op: &'static str, - }, - - #[error("shape mismatch in cat for dim {dim}, shape for arg 1: {first_shape:?} shape for arg {n}: {nth_shape:?}")] - ShapeMismatchCat { - dim: usize, - first_shape: Shape, - n: usize, - nth_shape: Shape, - }, - - #[error("Cannot divide tensor of shape {shape:?} equally along dim {dim} into {n_parts}")] - ShapeMismatchSplit { - shape: Shape, - dim: usize, - n_parts: usize, - }, - - #[error("{op} can only be performed on a single dimension")] - OnlySingleDimension { op: &'static str, dims: Vec }, - - #[error("empty tensor for {op}")] - EmptyTensor { op: &'static str }, - - // === Device Errors === - #[error("device mismatch in {op}, lhs: {lhs:?}, rhs: {rhs:?}")] - DeviceMismatchBinaryOp { - lhs: DeviceLocation, - rhs: DeviceLocation, - op: &'static str, - }, - - // === Op Specific Errors === - #[error("narrow invalid args {msg}: {shape:?}, dim: {dim}, start: {start}, len:{len}")] - NarrowInvalidArgs { - shape: Shape, - dim: usize, - start: usize, - len: usize, - msg: &'static str, - }, - - #[error("conv1d invalid args {msg}: inp: {inp_shape:?}, k: {k_shape:?}, pad: {padding}, stride: {stride}")] - Conv1dInvalidArgs { - inp_shape: Shape, - k_shape: Shape, - padding: usize, - stride: usize, - msg: &'static str, - }, - - #[error("{op} invalid index {index} with dim size {size}")] - InvalidIndex { - op: &'static str, - index: usize, - size: usize, - }, - - #[error("cannot broadcast {src_shape:?} to {dst_shape:?}")] - BroadcastIncompatibleShapes { src_shape: Shape, dst_shape: Shape }, - - #[error("cannot set variable {msg}")] - CannotSetVar { msg: &'static str }, - - // Box indirection to avoid large variant. - #[error("{0:?}")] - MatMulUnexpectedStriding(Box), - - #[error("{op} only supports contiguous tensors")] - RequiresContiguous { op: &'static str }, - - #[error("{op} expects at least one tensor")] - OpRequiresAtLeastOneTensor { op: &'static str }, - - #[error("{op} expects at least two tensors")] - OpRequiresAtLeastTwoTensors { op: &'static str }, - - #[error("backward is not supported for {op}")] - BackwardNotSupported { op: &'static str }, - - // === Other Errors === - #[error("the candle crate has not been built with cuda support")] - NotCompiledWithCudaSupport, - - #[error("the candle crate has not been built with metal support")] - NotCompiledWithMetalSupport, - - #[error("cannot find tensor {path}")] - CannotFindTensor { path: String }, - - // === Wrapped Errors === - #[error(transparent)] - Cuda(Box), - - #[error("Metal error {0}")] - Metal(#[from] MetalError), - - #[cfg(all(not(target_arch = "wasm32"), not(target_os = "ios"), feature = "ug"))] - #[error(transparent)] - Ug(#[from] candle_ug::Error), - - #[error(transparent)] - TryFromIntError(#[from] core::num::TryFromIntError), - - #[error("npy/npz error {0}")] - Npy(String), - - /// Zip file format error. - #[error(transparent)] - Zip(#[from] zip::result::ZipError), - - /// Integer parse error. - #[error(transparent)] - ParseInt(#[from] std::num::ParseIntError), - - /// Utf8 parse error. - #[error(transparent)] - FromUtf8(#[from] std::string::FromUtf8Error), - - /// I/O error. - #[error(transparent)] - Io(#[from] std::io::Error), - - /// SafeTensor error. - #[error(transparent)] - SafeTensor(#[from] safetensors::SafeTensorError), - - #[error("unsupported safetensor dtype {0:?}")] - UnsupportedSafeTensorDtype(safetensors::Dtype), - - /// Arbitrary errors wrapping. - #[error("{0}")] - Wrapped(Box), - - /// Arbitrary errors wrapping with context. - #[error("{wrapped:?}\n{context:?}")] - WrappedContext { - wrapped: Box, - context: String, - }, - - #[error("{context}\n{inner}")] - Context { - inner: Box, - context: Box, - }, - - /// Adding path information to an error. - #[error("path: {path:?} {inner}")] - WithPath { - inner: Box, - path: std::path::PathBuf, - }, - - #[error("{inner}\n{backtrace}")] - WithBacktrace { - inner: Box, - backtrace: Box, - }, - - /// User generated error message, typically created via `bail!`. - #[error("{0}")] - Msg(String), - - #[error("unwrap none")] - UnwrapNone, -} - -pub type Result = std::result::Result; - -impl Error { - pub fn wrap(err: impl std::fmt::Display + Send + Sync + 'static) -> Self { - Self::Wrapped(Box::new(err)).bt() - } - - pub fn msg(err: impl std::fmt::Display) -> Self { - Self::Msg(err.to_string()).bt() - } - - pub fn debug(err: impl std::fmt::Debug) -> Self { - Self::Msg(format!("{err:?}")).bt() - } - - pub fn bt(self) -> Self { - let backtrace = std::backtrace::Backtrace::capture(); - match backtrace.status() { - std::backtrace::BacktraceStatus::Disabled - | std::backtrace::BacktraceStatus::Unsupported => self, - _ => Self::WithBacktrace { - inner: Box::new(self), - backtrace: Box::new(backtrace), - }, - } - } - - pub fn with_path>(self, p: P) -> Self { - Self::WithPath { - inner: Box::new(self), - path: p.as_ref().to_path_buf(), - } - } - - pub fn context(self, c: impl std::fmt::Display + Send + Sync + 'static) -> Self { - Self::Context { - inner: Box::new(self), - context: Box::new(c), - } - } -} - -#[macro_export] -macro_rules! bail { - ($msg:literal $(,)?) => { - return Err($crate::Error::Msg(format!($msg).into()).bt()) - }; - ($err:expr $(,)?) => { - return Err($crate::Error::Msg(format!($err).into()).bt()) - }; - ($fmt:expr, $($arg:tt)*) => { - return Err($crate::Error::Msg(format!($fmt, $($arg)*).into()).bt()) - }; -} - -pub fn zip(r1: Result, r2: Result) -> Result<(T, U)> { - match (r1, r2) { - (Ok(r1), Ok(r2)) => Ok((r1, r2)), - (Err(e), _) => Err(e), - (_, Err(e)) => Err(e), - } -} - -pub(crate) mod private { - pub trait Sealed {} - - impl Sealed for std::result::Result where E: std::error::Error {} - impl Sealed for Option {} -} - -/// Attach more context to an error. -/// -/// Inspired by [`anyhow::Context`]. -pub trait Context: private::Sealed { - /// Wrap the error value with additional context. - fn context(self, context: C) -> std::result::Result - where - C: Display + Send + Sync + 'static; - - /// Wrap the error value with additional context that is evaluated lazily - /// only once an error does occur. - fn with_context(self, f: F) -> std::result::Result - where - C: Display + Send + Sync + 'static, - F: FnOnce() -> C; -} - -impl Context for std::result::Result -where - E: std::error::Error + Send + Sync + 'static, -{ - fn context(self, context: C) -> std::result::Result - where - C: Display + Send + Sync + 'static, - { - // Not using map_err to save 2 useless frames off the captured backtrace - // in ext_context. - match self { - Ok(ok) => Ok(ok), - Err(error) => Err(Error::WrappedContext { - wrapped: Box::new(error), - context: context.to_string(), - } - .bt()), - } - } - - fn with_context(self, context: F) -> std::result::Result - where - C: Display + Send + Sync + 'static, - F: FnOnce() -> C, - { - match self { - Ok(ok) => Ok(ok), - Err(error) => Err(Error::WrappedContext { - wrapped: Box::new(error), - context: context().to_string(), - } - .bt()), - } - } -} - -impl Context for Option { - fn context(self, context: C) -> std::result::Result - where - C: Display + Send + Sync + 'static, - { - // Not using ok_or_else to save 2 useless frames off the captured - // backtrace. - match self { - Some(ok) => Ok(ok), - None => Err(Error::msg(context).bt()), - } - } - - fn with_context(self, context: F) -> std::result::Result - where - C: Display + Send + Sync + 'static, - F: FnOnce() -> C, - { - match self { - Some(v) => Ok(v), - None => Err(Error::UnwrapNone.context(context()).bt()), - } - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/indexer.rs b/v2/dependency-patches/candle-core-0.9.2/src/indexer.rs deleted file mode 100644 index d6cd6debf8..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/indexer.rs +++ /dev/null @@ -1,257 +0,0 @@ -use crate::{Error, Tensor}; -use std::ops::{ - Bound, Range, RangeBounds, RangeFrom, RangeFull, RangeInclusive, RangeTo, RangeToInclusive, -}; - -impl Tensor { - /// Intended to be use by the trait `.i()` - /// - /// ``` - /// # use candle_core::{Tensor, DType, Device, IndexOp}; - /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; - /// - /// let c = a.i(0..1)?; - /// assert_eq!(c.shape().dims(), &[1, 3]); - /// - /// let c = a.i(0)?; - /// assert_eq!(c.shape().dims(), &[3]); - /// - /// let c = a.i((.., ..2) )?; - /// assert_eq!(c.shape().dims(), &[2, 2]); - /// - /// let c = a.i((.., ..=2))?; - /// assert_eq!(c.shape().dims(), &[2, 3]); - /// - /// # Ok::<(), candle_core::Error>(()) - /// ``` - fn index(&self, indexers: &[TensorIndexer]) -> Result { - let mut x = self.clone(); - let dims = self.shape().dims(); - let mut current_dim = 0; - for (i, indexer) in indexers.iter().enumerate() { - x = match indexer { - TensorIndexer::Select(n) => x.narrow(current_dim, *n, 1)?.squeeze(current_dim)?, - TensorIndexer::Narrow(left_bound, right_bound) => { - let start = match left_bound { - Bound::Included(n) => *n, - Bound::Excluded(n) => *n + 1, - Bound::Unbounded => 0, - }; - let stop = match right_bound { - Bound::Included(n) => *n + 1, - Bound::Excluded(n) => *n, - Bound::Unbounded => dims[i], - }; - let out = x.narrow(current_dim, start, stop.saturating_sub(start))?; - current_dim += 1; - out - } - TensorIndexer::IndexSelect(indexes) => { - if indexes.rank() != 1 { - crate::bail!("multi-dimensional tensor indexing is not supported") - } - let out = x.index_select(&indexes.to_device(x.device())?, current_dim)?; - current_dim += 1; - out - } - TensorIndexer::Err(e) => crate::bail!("indexing error {e:?}"), - }; - } - Ok(x) - } -} - -#[derive(Debug)] -/// Generic structure used to index a slice of the tensor -pub enum TensorIndexer { - /// This selects the elements for which an index has some specific value. - Select(usize), - /// This is a regular slice, purely indexing a chunk of the tensor - Narrow(Bound, Bound), - /// Indexing via a 1d tensor - IndexSelect(Tensor), - Err(Error), -} - -impl From for TensorIndexer { - fn from(index: usize) -> Self { - TensorIndexer::Select(index) - } -} - -impl From<&[u32]> for TensorIndexer { - fn from(index: &[u32]) -> Self { - match Tensor::new(index, &crate::Device::Cpu) { - Ok(tensor) => TensorIndexer::IndexSelect(tensor), - Err(e) => TensorIndexer::Err(e), - } - } -} - -impl From> for TensorIndexer { - fn from(index: Vec) -> Self { - let len = index.len(); - match Tensor::from_vec(index, len, &crate::Device::Cpu) { - Ok(tensor) => TensorIndexer::IndexSelect(tensor), - Err(e) => TensorIndexer::Err(e), - } - } -} - -impl From<&Tensor> for TensorIndexer { - fn from(tensor: &Tensor) -> Self { - TensorIndexer::IndexSelect(tensor.clone()) - } -} - -trait RB: RangeBounds {} -impl RB for Range {} -impl RB for RangeFrom {} -impl RB for RangeFull {} -impl RB for RangeInclusive {} -impl RB for RangeTo {} -impl RB for RangeToInclusive {} - -impl From for TensorIndexer { - fn from(range: T) -> Self { - use std::ops::Bound::*; - let start = match range.start_bound() { - Included(idx) => Included(*idx), - Excluded(idx) => Excluded(*idx), - Unbounded => Unbounded, - }; - let end = match range.end_bound() { - Included(idx) => Included(*idx), - Excluded(idx) => Excluded(*idx), - Unbounded => Unbounded, - }; - TensorIndexer::Narrow(start, end) - } -} - -/// Trait used to implement multiple signatures for ease of use of the slicing -/// of a tensor -pub trait IndexOp { - /// Returns a slicing iterator which are the chunks of data necessary to - /// reconstruct the desired tensor. - fn i(&self, index: T) -> Result; -} - -impl IndexOp for Tensor -where - T: Into, -{ - ///```rust - /// use candle_core::{Tensor, DType, Device, IndexOp}; - /// let a = Tensor::new(&[ - /// [0., 1.], - /// [2., 3.], - /// [4., 5.] - /// ], &Device::Cpu)?; - /// - /// let b = a.i(0)?; - /// assert_eq!(b.shape().dims(), &[2]); - /// assert_eq!(b.to_vec1::()?, &[0., 1.]); - /// - /// let c = a.i(..2)?; - /// assert_eq!(c.shape().dims(), &[2, 2]); - /// assert_eq!(c.to_vec2::()?, &[ - /// [0., 1.], - /// [2., 3.] - /// ]); - /// - /// let d = a.i(1..)?; - /// assert_eq!(d.shape().dims(), &[2, 2]); - /// assert_eq!(d.to_vec2::()?, &[ - /// [2., 3.], - /// [4., 5.] - /// ]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - fn i(&self, index: T) -> Result { - self.index(&[index.into()]) - } -} - -impl IndexOp<(A,)> for Tensor -where - A: Into, -{ - ///```rust - /// use candle_core::{Tensor, DType, Device, IndexOp}; - /// let a = Tensor::new(&[ - /// [0f32, 1.], - /// [2. , 3.], - /// [4. , 5.] - /// ], &Device::Cpu)?; - /// - /// let b = a.i((0,))?; - /// assert_eq!(b.shape().dims(), &[2]); - /// assert_eq!(b.to_vec1::()?, &[0., 1.]); - /// - /// let c = a.i((..2,))?; - /// assert_eq!(c.shape().dims(), &[2, 2]); - /// assert_eq!(c.to_vec2::()?, &[ - /// [0., 1.], - /// [2., 3.] - /// ]); - /// - /// let d = a.i((1..,))?; - /// assert_eq!(d.shape().dims(), &[2, 2]); - /// assert_eq!(d.to_vec2::()?, &[ - /// [2., 3.], - /// [4., 5.] - /// ]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - fn i(&self, (a,): (A,)) -> Result { - self.index(&[a.into()]) - } -} -#[allow(non_snake_case)] -impl IndexOp<(A, B)> for Tensor -where - A: Into, - B: Into, -{ - ///```rust - /// use candle_core::{Tensor, DType, Device, IndexOp}; - /// let a = Tensor::new(&[[0f32, 1., 2.], [3., 4., 5.], [6., 7., 8.]], &Device::Cpu)?; - /// - /// let b = a.i((1, 0))?; - /// assert_eq!(b.to_vec0::()?, 3.); - /// - /// let c = a.i((..2, 1))?; - /// assert_eq!(c.shape().dims(), &[2]); - /// assert_eq!(c.to_vec1::()?, &[1., 4.]); - /// - /// let d = a.i((2.., ..))?; - /// assert_eq!(d.shape().dims(), &[1, 3]); - /// assert_eq!(d.to_vec2::()?, &[[6., 7., 8.]]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - fn i(&self, (a, b): (A, B)) -> Result { - self.index(&[a.into(), b.into()]) - } -} - -macro_rules! index_op_tuple { - ($doc:tt, $($t:ident),+) => { - #[allow(non_snake_case)] - impl<$($t),*> IndexOp<($($t,)*)> for Tensor - where - $($t: Into,)* - { - #[doc=$doc] - fn i(&self, ($($t,)*): ($($t,)*)) -> Result { - self.index(&[$($t.into(),)*]) - } - } - }; -} - -index_op_tuple!("see [TensorIndex#method.i]", A, B, C); -index_op_tuple!("see [TensorIndex#method.i]", A, B, C, D); -index_op_tuple!("see [TensorIndex#method.i]", A, B, C, D, E); -index_op_tuple!("see [TensorIndex#method.i]", A, B, C, D, E, F); -index_op_tuple!("see [TensorIndex#method.i]", A, B, C, D, E, F, G); diff --git a/v2/dependency-patches/candle-core-0.9.2/src/layout.rs b/v2/dependency-patches/candle-core-0.9.2/src/layout.rs deleted file mode 100644 index 91e50481ec..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/layout.rs +++ /dev/null @@ -1,278 +0,0 @@ -//! Tensor Layouts including contiguous or sparse strides -use crate::{Error, Result, Shape}; - -#[derive(Debug, PartialEq, Eq, Clone)] -pub struct Layout { - shape: Shape, - // The strides are given in number of elements and not in bytes. - stride: Vec, - start_offset: usize, -} - -impl Layout { - pub fn new(shape: Shape, stride: Vec, start_offset: usize) -> Self { - Self { - shape, - stride, - start_offset, - } - } - - pub fn contiguous_with_offset>(shape: S, start_offset: usize) -> Self { - let shape = shape.into(); - let stride = shape.stride_contiguous(); - Self { - shape, - stride, - start_offset, - } - } - - pub fn contiguous>(shape: S) -> Self { - Self::contiguous_with_offset(shape, 0) - } - - pub fn dims(&self) -> &[usize] { - self.shape.dims() - } - - /// The dimension size for a specified dimension index. - pub fn dim(&self, dim: D) -> Result { - let dim = dim.to_index(&self.shape, "dim")?; - Ok(self.dims()[dim]) - } - - pub fn shape(&self) -> &Shape { - &self.shape - } - - pub fn stride(&self) -> &[usize] { - &self.stride - } - - pub fn start_offset(&self) -> usize { - self.start_offset - } - - /// Returns the appropriate start and stop offset if the data is stored in a C - /// contiguous (aka row major) way. - pub fn contiguous_offsets(&self) -> Option<(usize, usize)> { - if self.is_contiguous() { - let start_o = self.start_offset; - Some((start_o, start_o + self.shape.elem_count())) - } else { - None - } - } - - /// Returns true if the data is stored in a C contiguous (aka row major) way. - /// Note that this does not implies that the start offset is 0 or that there are no extra - /// elements at the end of the storage. - pub fn is_contiguous(&self) -> bool { - self.shape.is_contiguous(&self.stride) - } - - /// Returns true if the data is stored in a Fortran contiguous (aka column major) way. - pub fn is_fortran_contiguous(&self) -> bool { - self.shape.is_fortran_contiguous(&self.stride) - } - - pub fn narrow(&self, dim: usize, start: usize, len: usize) -> Result { - let dims = self.shape().dims(); - if dim >= dims.len() { - Err(Error::DimOutOfRange { - shape: self.shape().clone(), - dim: dim as i32, - op: "narrow", - } - .bt())? - } - if start + len > dims[dim] { - Err(Error::NarrowInvalidArgs { - shape: self.shape.clone(), - dim, - start, - len, - msg: "start + len > dim_len", - } - .bt())? - } - let mut dims = dims.to_vec(); - dims[dim] = len; - Ok(Self { - shape: Shape::from(dims), - stride: self.stride.clone(), - start_offset: self.start_offset + self.stride[dim] * start, - }) - } - - pub fn transpose(&self, dim1: usize, dim2: usize) -> Result { - let rank = self.shape.rank(); - if rank <= dim1 || rank <= dim2 { - Err(Error::UnexpectedNumberOfDims { - expected: usize::max(dim1, dim2), - got: rank, - shape: self.shape().clone(), - } - .bt())? - } - let mut stride = self.stride().to_vec(); - let mut dims = self.shape().dims().to_vec(); - dims.swap(dim1, dim2); - stride.swap(dim1, dim2); - Ok(Self { - shape: Shape::from(dims), - stride, - start_offset: self.start_offset, - }) - } - - pub fn permute(&self, idxs: &[usize]) -> Result { - let is_permutation = - idxs.len() == self.shape.rank() && (0..idxs.len()).all(|i| idxs.contains(&i)); - if !is_permutation { - crate::bail!( - "dimension mismatch in permute, tensor {:?}, dims: {:?}", - self.dims(), - idxs - ) - } - let stride = self.stride(); - let dims = self.shape().dims(); - let mut perm_stride = stride.to_vec(); - let mut perm_dims = dims.to_vec(); - for (i, &idx) in idxs.iter().enumerate() { - perm_stride[i] = stride[idx]; - perm_dims[i] = dims[idx]; - } - Ok(Self { - shape: Shape::from(perm_dims), - stride: perm_stride, - start_offset: self.start_offset, - }) - } - - pub fn broadcast_as>(&self, shape: S) -> Result { - let shape = shape.into(); - if shape.rank() < self.shape().rank() { - return Err(Error::BroadcastIncompatibleShapes { - src_shape: self.shape().clone(), - dst_shape: shape, - } - .bt()); - } - let added_dims = shape.rank() - self.shape().rank(); - let mut stride = vec![0; added_dims]; - for (&dst_dim, (&src_dim, &src_stride)) in shape.dims()[added_dims..] - .iter() - .zip(self.dims().iter().zip(self.stride())) - { - let s = if dst_dim == src_dim { - src_stride - } else if src_dim != 1 { - return Err(Error::BroadcastIncompatibleShapes { - src_shape: self.shape().clone(), - dst_shape: shape, - } - .bt()); - } else { - 0 - }; - stride.push(s) - } - Ok(Self { - shape, - stride, - start_offset: self.start_offset, - }) - } - - pub(crate) fn strided_index(&self) -> crate::StridedIndex<'_> { - crate::StridedIndex::from_layout(self) - } - - pub(crate) fn strided_blocks(&self) -> crate::StridedBlocks<'_> { - let mut block_len = 1; - let mut contiguous_dims = 0; // These are counted from the right. - for (&stride, &dim) in self.stride().iter().zip(self.dims().iter()).rev() { - if stride != block_len { - break; - } - block_len *= dim; - contiguous_dims += 1; - } - let index_dims = self.dims().len() - contiguous_dims; - if index_dims == 0 { - crate::StridedBlocks::SingleBlock { - start_offset: self.start_offset, - len: block_len, - } - } else { - let block_start_index = crate::StridedIndex::new( - &self.dims()[..index_dims], - &self.stride[..index_dims], - self.start_offset, - ); - crate::StridedBlocks::MultipleBlocks { - block_start_index, - block_len, - } - } - } - - // Returns the contiguous offsets with broadcast if applicable. - pub(crate) fn offsets_b(&self) -> Option { - let mut left_broadcast = 1; - let mut right_broadcast = 1; - let strides = self.stride(); - let dims = self.dims(); - let mut start_cont = 0; - let mut end_cont = dims.len(); - for (&s, &d) in strides.iter().zip(dims.iter()) { - if s != 0 { - break; - } - start_cont += 1; - left_broadcast *= d; - } - if start_cont == dims.len() { - return Some(ContiguousOffsetsWithBroadcast { - start: self.start_offset, - len: 1, - left_broadcast, - right_broadcast: 1, - }); - } - for (&s, &d) in strides.iter().zip(dims.iter()).rev() { - if s != 0 { - break; - } - end_cont -= 1; - right_broadcast *= d; - } - // Check that the inner dims are contiguous - let strides = &strides[start_cont..end_cont]; - let dims = &dims[start_cont..end_cont]; - let mut len = 1; - for (&stride, &dim) in strides.iter().zip(dims.iter()).rev() { - if stride != len { - return None; - } - len *= dim; - } - Some(ContiguousOffsetsWithBroadcast { - start: self.start_offset, - len, - left_broadcast, - right_broadcast, - }) - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct ContiguousOffsetsWithBroadcast { - pub start: usize, - pub len: usize, - pub left_broadcast: usize, - pub right_broadcast: usize, -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/lib.rs b/v2/dependency-patches/candle-core-0.9.2/src/lib.rs deleted file mode 100644 index 791933d25a..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/lib.rs +++ /dev/null @@ -1,177 +0,0 @@ -//! ML framework for Rust -//! -//! ```rust -//! use candle_core::{Tensor, DType, Device}; -//! # use candle_core::Error; -//! # fn main() -> Result<(), Error>{ -//! -//! let a = Tensor::arange(0f32, 6f32, &Device::Cpu)?.reshape((2, 3))?; -//! let b = Tensor::arange(0f32, 12f32, &Device::Cpu)?.reshape((3, 4))?; -//! let c = a.matmul(&b)?; -//! -//! # Ok(())} -//! ``` -//! -//! ## Features -//! -//! - Simple syntax (looks and feels like PyTorch) -//! - CPU and Cuda backends (and M1 support) -//! - Enable serverless (CPU) small and fast deployments -//! - Model training -//! - Distributed computing (NCCL). -//! - Models out of the box (Llama, Whisper, Falcon, ...) -//! -//! ## FAQ -//! -//! - Why Candle? -//! -//! Candle stems from the need to reduce binary size in order to *enable serverless* -//! possible by making the whole engine smaller than PyTorch very large library volume -//! -//! And simply *removing Python* from production workloads. -//! Python can really add overhead in more complex workflows and the [GIL](https://www.backblaze.com/blog/the-python-gil-past-present-and-future/) is a notorious source of headaches. -//! -//! Rust is cool, and a lot of the HF ecosystem already has Rust crates [safetensors](https://github.com/huggingface/safetensors) and [tokenizers](https://github.com/huggingface/tokenizers) -//! -//! ## Other Crates -//! -//! Candle consists of a number of crates. This crate holds core the common data structures but you may wish -//! to look at the docs for the other crates which can be found here: -//! -//! - [candle-core](https://docs.rs/candle-core/). Core Datastructures and DataTypes. -//! - [candle-nn](https://docs.rs/candle-nn/). Building blocks for Neural Nets. -//! - [candle-datasets](https://docs.rs/candle-datasets/). Rust access to commonly used Datasets like MNIST. -//! - [candle-examples](https://docs.rs/candle-examples/). Examples of Candle in Use. -//! - [candle-onnx](https://docs.rs/candle-onnx/). Loading and using ONNX models. -//! - [candle-pyo3](https://docs.rs/candle-pyo3/). Access to Candle from Python. -//! - [candle-transformers](https://docs.rs/candle-transformers/). Candle implementation of many published transformer models. -//! - -#[cfg(feature = "accelerate")] -mod accelerate; -pub mod backend; -pub mod backprop; -pub mod conv; -mod convert; -pub mod cpu; -pub mod cpu_backend; -#[cfg(feature = "cuda")] -pub mod cuda_backend; -mod custom_op; -mod device; -pub mod display; -mod dtype; -pub mod dummy_cuda_backend; -pub mod dummy_dtype; -mod dummy_metal_backend; -pub mod error; -mod indexer; -pub mod layout; -#[cfg(feature = "metal")] -pub mod metal_backend; -#[cfg(feature = "mkl")] -mod mkl; -pub mod npy; -pub mod op; -pub mod pickle; -pub mod quantized; -pub mod safetensors; -pub mod scalar; -pub mod shape; -mod sort; -mod storage; -pub mod streaming; -mod strided_index; -mod tensor; -mod tensor_cat; -pub mod test_utils; -pub mod utils; -mod variable; - -#[cfg(feature = "cudnn")] -pub use cuda_backend::cudnn; - -pub use cpu_backend::{CpuStorage, CpuStorageRef}; -#[cfg(feature = "ug")] -pub use custom_op::UgIOp1; -pub use custom_op::{CustomOp1, CustomOp2, CustomOp3, InplaceOp1, InplaceOp2, InplaceOp3}; -pub use device::{Device, DeviceLocation, NdArray}; -pub use dtype::{DType, DTypeParseError, FloatDType, IntDType, WithDType}; -pub use dummy_dtype::{F4, F6E2M3, F6E3M2, F8E8M0}; -pub use error::{Context, Error, Result}; -pub use indexer::{IndexOp, TensorIndexer}; -pub use layout::Layout; -pub use shape::{Shape, D}; -pub use storage::Storage; -pub use streaming::{StreamTensor, StreamingBinOp, StreamingModule}; -pub use strided_index::{StridedBlocks, StridedIndex}; -pub use tensor::{Tensor, TensorId}; -pub use variable::Var; - -#[cfg(feature = "cuda")] -pub use cuda_backend as cuda; - -#[cfg(not(feature = "cuda"))] -pub use dummy_cuda_backend as cuda; - -pub use cuda::{CudaDevice, CudaStorage}; - -#[cfg(feature = "metal")] -pub use metal_backend::{MetalDevice, MetalError, MetalStorage}; - -#[cfg(not(feature = "metal"))] -pub use dummy_metal_backend::{MetalDevice, MetalError, MetalStorage}; - -#[cfg(feature = "mkl")] -extern crate intel_mkl_src; - -#[cfg(feature = "accelerate")] -extern crate accelerate_src; - -pub trait ToUsize2 { - fn to_usize2(self) -> (usize, usize); -} - -impl ToUsize2 for usize { - fn to_usize2(self) -> (usize, usize) { - (self, self) - } -} - -impl ToUsize2 for (usize, usize) { - fn to_usize2(self) -> (usize, usize) { - self - } -} - -/// Defining a module with forward method using a single argument. -pub trait Module { - fn forward(&self, xs: &Tensor) -> Result; -} - -impl Result> Module for T { - fn forward(&self, xs: &Tensor) -> Result { - self(xs) - } -} - -impl Module for Option<&M> { - fn forward(&self, xs: &Tensor) -> Result { - match self { - None => Ok(xs.clone()), - Some(m) => m.forward(xs), - } - } -} - -/// A single forward method using a single single tensor argument and a flag to -/// separate the training and evaluation behaviors. -pub trait ModuleT { - fn forward_t(&self, xs: &Tensor, train: bool) -> Result; -} - -impl ModuleT for M { - fn forward_t(&self, xs: &Tensor, _train: bool) -> Result { - self.forward(xs) - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/metal_backend/device.rs b/v2/dependency-patches/candle-core-0.9.2/src/metal_backend/device.rs deleted file mode 100644 index 1728c5a4e0..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/metal_backend/device.rs +++ /dev/null @@ -1,287 +0,0 @@ -use crate::{DType, Result}; - -#[cfg(feature = "ug")] -use candle_metal_kernels::metal::ComputePipeline; -use candle_metal_kernels::{ - metal::{ - BlitCommandEncoder, Buffer, BufferMap, Commands, ComputeCommandEncoder, Device, - MTLResourceOptions, - }, - Kernels, -}; -use objc2_foundation::NSURL; -use objc2_metal::{MTLCaptureDescriptor, MTLCaptureDestination, MTLCaptureManager}; - -use std::path::Path; -use std::sync::{Arc, Mutex, RwLock}; - -use super::MetalError; - -/// Unique identifier for metal devices. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct DeviceId(usize); - -impl DeviceId { - pub(crate) fn new() -> Self { - // https://users.rust-lang.org/t/idiomatic-rust-way-to-generate-unique-id/33805 - use std::sync::atomic; - static COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::new(1); - Self(COUNTER.fetch_add(1, atomic::Ordering::Relaxed)) - } -} - -#[derive(Clone)] -pub struct MetalDevice { - /// Unique identifier, the registryID is not sufficient as it identifies the GPU rather than - /// the device itself. - pub(crate) id: DeviceId, - - /// Raw metal device: - pub(crate) device: Device, - - pub(crate) commands: Arc>, - - /// Simple allocator struct. - /// The buffers are stored in size buckets since ML tends to use similar shapes over and over. - /// We store the buffers in [`Arc`] because it's much faster than Obj-c internal ref counting - /// (could be linked to FFI communication overhead). - /// - /// Whenever a buffer has a strong_count==1, we can reuse it, it means it was dropped in the - /// graph calculation, and only we the allocator kept a reference to it, therefore it's free - /// to be reused. However, in order for this to work, we need to guarantee the order of - /// operation, so that this buffer is not being used by another kernel at the same time. - /// Arc is the CPU reference count, it doesn't mean anything on the GPU side of things. - /// - /// Whenever we actually allocate a new buffer, we make a full sweep to clean up unused buffers - /// (strong_count = 1). - pub(crate) buffers: Arc>, - - /// Simple keeper struct to keep track of the already compiled kernels so we can reuse them. - /// Heavily used by [`candle_metal_kernels`] - pub(crate) kernels: Arc, - /// Seed for random number generation. - pub(crate) seed: Arc>, - /// Last seed value set on this device. - pub(crate) seed_value: Arc>, -} - -// Resource options used for creating buffers. Shared storage mode allows both CPU and GPU to access the buffer. -pub const RESOURCE_OPTIONS: MTLResourceOptions = - objc2_metal::MTLResourceOptions(MTLResourceOptions::StorageModeShared.bits()); -//| MTLResourceOptions::HazardTrackingModeUntracked.bits(), -//); - -// Resource options used for `new_private_buffer`. This uses `private` where supported. -#[cfg(target_os = "ios")] -pub const PRIVATE_RESOURCE_OPTIONS: MTLResourceOptions = MTLResourceOptions::StorageModeShared; -#[cfg(not(target_os = "ios"))] -pub const PRIVATE_RESOURCE_OPTIONS: MTLResourceOptions = MTLResourceOptions::StorageModePrivate; - -impl std::fmt::Debug for MetalDevice { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "MetalDevice({:?})", self.id) - } -} - -impl std::ops::Deref for MetalDevice { - type Target = Device; - - fn deref(&self) -> &Self::Target { - &self.device - } -} - -impl MetalDevice { - #[cfg(all(feature = "ug", not(target_arch = "wasm32"), not(target_os = "ios")))] - pub fn compile( - &self, - func_name: &'static str, - kernel: candle_ug::lang::ssa::Kernel, - ) -> Result { - let mut buf = vec![]; - candle_ug::metal::code_gen::gen(&mut buf, func_name, &kernel)?; - let metal_code = String::from_utf8(buf)?; - let lib = self - .device - .new_library_with_source(&metal_code, None) - .map_err(MetalError::from)?; - let func = lib - .get_function(func_name, None) - .map_err(MetalError::from)?; - let pl = self - .device - .new_compute_pipeline_state_with_function(&func) - .map_err(MetalError::from)?; - Ok(pl) - } - - pub fn id(&self) -> DeviceId { - self.id - } - - pub fn metal_device(&self) -> &Device { - &self.device - } - - fn drop_unused_buffers(&self) -> Result<()> { - let mut buffers = self.buffers.write().map_err(MetalError::from)?; - for subbuffers in buffers.values_mut() { - let newbuffers = subbuffers - .iter() - .filter(|s| Arc::strong_count(*s) > 1) - .map(Arc::clone) - .collect(); - *subbuffers = newbuffers; - } - Ok(()) - } - - pub fn command_encoder(&self) -> Result { - let commands = self.commands.write().map_err(MetalError::from)?; - let (flush, command_encoder) = commands.command_encoder().map_err(MetalError::from)?; - if flush { - self.drop_unused_buffers()? - } - Ok(command_encoder) - } - - pub fn blit_command_encoder(&self) -> Result { - let commands = self.commands.write().map_err(MetalError::from)?; - let (flush, command_encoder) = commands.blit_command_encoder().map_err(MetalError::from)?; - if flush { - self.drop_unused_buffers()? - } - Ok(command_encoder) - } - - pub fn wait_until_completed(&self) -> Result<()> { - let commands = self.commands.write().map_err(MetalError::from)?; - commands.wait_until_completed().map_err(MetalError::from)?; - Ok(()) - } - - pub fn kernels(&self) -> &Kernels { - &self.kernels - } - - pub fn device(&self) -> &Device { - &self.device - } - - /// Creates a new buffer (not necessarily zeroed). - pub fn new_buffer( - &self, - element_count: usize, - dtype: DType, - _name: &str, - ) -> Result> { - let size = element_count * dtype.size_in_bytes(); - self.allocate_buffer(size) - } - - /// Creates a new private buffer (not necessarily zeroed). - /// - /// This is intentionally not in the Metal buffer pool to allow the efficient implementation of persistent buffers. - pub fn new_private_buffer( - &self, - element_count: usize, - dtype: DType, - _name: &str, - ) -> Result> { - let size = element_count * dtype.size_in_bytes(); - let buffer = self - .device - .new_buffer(size, PRIVATE_RESOURCE_OPTIONS) - .map_err(MetalError::from)?; - Ok(Arc::new(buffer)) - } - - /// Creates a new buffer from data. - /// - /// Does not require synchronization, as [newBufferWithBytes](https://developer.apple.com/documentation/metal/mtldevice/1433429-newbufferwithbytes) - /// allocates the buffer and copies over the existing data before returning the MTLBuffer. - pub fn new_buffer_with_data(&self, data: &[T]) -> Result> { - let size = core::mem::size_of_val(data); - let new_buffer = self - .device - .new_buffer_with_data(data.as_ptr().cast(), size, RESOURCE_OPTIONS) - .map_err(MetalError::from)?; - let mut buffers = self.buffers.write().map_err(MetalError::from)?; - - let subbuffers = buffers.entry(size).or_insert(vec![]); - - let new_buffer = Arc::new(new_buffer); - subbuffers.push(new_buffer.clone()); - Ok(new_buffer) - } - - pub fn allocate_zeros(&self, size_in_bytes: usize) -> Result> { - let buffer = self.allocate_buffer(size_in_bytes)?; - let blit = self.blit_command_encoder()?; - blit.set_label("zeros"); - blit.fill_buffer(&buffer, (0, buffer.length()), 0); - blit.end_encoding(); - Ok(buffer) - } - - /// The critical allocator algorithm - pub fn allocate_buffer(&self, size: usize) -> Result> { - let mut buffers = self.buffers.write().map_err(MetalError::from)?; - if let Some(b) = find_available_buffer(size, &buffers) { - // Cloning also ensures we increment the strong count - return Ok(b.clone()); - } - let size = buf_size(size); - let subbuffers = buffers.entry(size).or_insert(vec![]); - - let new_buffer = self - .device - .new_buffer(size, RESOURCE_OPTIONS) - .map_err(MetalError::from)?; - let new_buffer = Arc::new(new_buffer); - subbuffers.push(new_buffer.clone()); - Ok(new_buffer) - } - - /// Create a metal GPU capture trace on [`path`]. - pub fn capture>(&self, path: P) -> Result<()> { - let capture = unsafe { MTLCaptureManager::sharedCaptureManager() }; - let descriptor = MTLCaptureDescriptor::new(); - descriptor.setDestination(MTLCaptureDestination::GPUTraceDocument); - descriptor.set_capture_device(self.device().as_ref()); - // The [set_output_url] call requires an absolute path so we convert it if needed. - if path.as_ref().is_absolute() { - let url = NSURL::from_file_path(path); - descriptor.setOutputURL(url.as_deref()); - } else { - let path = std::env::current_dir()?.join(path); - let url = NSURL::from_file_path(path); - descriptor.setOutputURL(url.as_deref()); - } - - capture - .startCaptureWithDescriptor_error(&descriptor) - .map_err(|e| MetalError::from(e.to_string()))?; - Ok(()) - } -} - -fn buf_size(size: usize) -> usize { - size.saturating_sub(1).next_power_of_two() -} - -fn find_available_buffer(size: usize, buffers: &BufferMap) -> Option> { - let mut best_buffer: Option<&Arc> = None; - let mut best_buffer_size = usize::MAX; - for (buffer_size, subbuffers) in buffers.iter() { - if buffer_size >= &size && buffer_size < &best_buffer_size { - for sub in subbuffers { - if Arc::strong_count(sub) == 1 { - best_buffer = Some(sub); - best_buffer_size = *buffer_size; - } - } - } - } - best_buffer.cloned() -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/metal_backend/mod.rs b/v2/dependency-patches/candle-core-0.9.2/src/metal_backend/mod.rs deleted file mode 100644 index 363ffa9f7a..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/metal_backend/mod.rs +++ /dev/null @@ -1,2137 +0,0 @@ -//! Implementation of Backend traits for Metal -//! -use crate::backend::{BackendDevice, BackendStorage}; -use crate::conv::{ParamsConv1D, ParamsConv2D, ParamsConvTranspose1D, ParamsConvTranspose2D}; -use crate::op::{BinaryOpT, CmpOp, ReduceOp, UnaryOpT}; -use crate::{CpuStorage, CpuStorageRef, DType, Error, Layout, Result, Shape}; -use candle_metal_kernels::{ - metal::{Buffer, Commands, Device}, - BufferOffset, CallConvTranspose2dCfg, Kernels, RESOURCE_OPTIONS, -}; -use objc2_foundation::NSRange; -use std::collections::HashMap; -use std::ffi::c_void; -use std::sync::{Arc, Mutex, PoisonError, RwLock, TryLockError}; - -mod device; -pub use device::{DeviceId, MetalDevice}; - -pub fn buffer_o<'a>(buffer: &'a Buffer, l: &Layout, dtype: DType) -> BufferOffset<'a> { - BufferOffset { - buffer, - offset_in_bytes: l.start_offset() * dtype.size_in_bytes(), - } -} -/// Simple way to catch lock error without -/// depending on T -#[derive(thiserror::Error, Debug)] -pub enum LockError { - #[error("{0}")] - Poisoned(String), - #[error("Would block")] - WouldBlock, -} - -impl From> for MetalError { - fn from(value: TryLockError) -> Self { - match value { - TryLockError::Poisoned(p) => MetalError::LockError(LockError::Poisoned(p.to_string())), - TryLockError::WouldBlock => MetalError::LockError(LockError::WouldBlock), - } - } -} - -impl From> for MetalError { - fn from(p: PoisonError) -> Self { - MetalError::LockError(LockError::Poisoned(p.to_string())) - } -} - -/// Metal related errors -#[derive(thiserror::Error, Debug)] -pub enum MetalError { - #[error("{0}")] - Message(String), - #[error(transparent)] - KernelError(#[from] candle_metal_kernels::MetalKernelError), - #[error("{0:?}")] - LockError(LockError), - #[error("{msg}, expected: {expected:?}, got: {got:?}")] - UnexpectedDType { - msg: &'static str, - expected: DType, - got: DType, - }, -} - -impl From for MetalError { - fn from(e: String) -> Self { - MetalError::Message(e) - } -} - -#[derive(Debug, Clone)] -pub struct MetalStorage { - /// The actual buffer containing the data. - buffer: Arc, - /// a reference to the device owning this buffer - device: MetalDevice, - /// The count of allocated elements in the buffer - count: usize, - /// The dtype is kept since buffers are untyped. - dtype: DType, -} - -impl BackendStorage for MetalStorage { - type Device = MetalDevice; - - fn try_clone(&self, _: &Layout) -> Result { - Ok(self.clone()) - } - - fn dtype(&self) -> DType { - self.dtype - } - - fn device(&self) -> &Self::Device { - &self.device - } - - fn to_cpu_storage(&self) -> Result { - match self.dtype { - DType::U8 => Ok(CpuStorage::U8(self.to_cpu()?)), - DType::U32 => Ok(CpuStorage::U32(self.to_cpu()?)), - DType::I16 => Ok(CpuStorage::I16(self.to_cpu()?)), - DType::I32 => Ok(CpuStorage::I32(self.to_cpu()?)), - DType::I64 => Ok(CpuStorage::I64(self.to_cpu()?)), - DType::F16 => Ok(CpuStorage::F16(self.to_cpu()?)), - DType::BF16 => Ok(CpuStorage::BF16(self.to_cpu()?)), - DType::F32 => Ok(CpuStorage::F32(self.to_cpu()?)), - DType::F64 => Ok(CpuStorage::F64(self.to_cpu()?)), - DType::F8E4M3 => Ok(CpuStorage::F8E4M3(self.to_cpu()?)), - DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { - Err(crate::Error::UnsupportedDTypeForOp(self.dtype, "to_cpu_storage").bt()) - } - } - } - - fn affine(&self, layout: &Layout, mul: f64, add: f64) -> Result { - let device = self.device().clone(); - - let shape = layout.shape(); - let el = shape.elem_count(); - let dtype = self.dtype; - - let buffer = device.new_buffer(el, self.dtype, "affine")?; - let encoder = self.device.command_encoder()?; - encoder.set_label("affine"); - let src = buffer_o(&self.buffer, layout, dtype); - if layout.is_contiguous() { - let name = match self.dtype { - DType::F32 => "affine_f32", - DType::F16 => "affine_f16", - DType::BF16 => "affine_bf16", - DType::U8 => "affine_u8", - DType::U32 => "affine_u32", - DType::I64 => "affine_i64", - dtype => crate::bail!("Metal contiguous affine {dtype:?} not implemented"), - }; - candle_metal_kernels::call_affine( - &device.device, - &encoder, - &device.kernels, - name, - self.dtype.size_in_bytes(), - el, - src, - &buffer, - mul as f32, - add as f32, - ) - .map_err(MetalError::from)?; - } else { - let name = match self.dtype { - DType::F32 => "affine_f32_strided", - DType::F16 => "affine_f16_strided", - DType::BF16 => "affine_bf16_strided", - DType::U8 => "affine_u8_strided", - DType::U32 => "affine_u32_strided", - DType::I64 => "affine_i64_strided", - dtype => crate::bail!("Metal strided affine {dtype:?} not implemented"), - }; - candle_metal_kernels::call_affine_strided( - &device.device, - &encoder, - &device.kernels, - name, - layout.dims(), - src, - layout.stride(), - &buffer, - mul as f32, - add as f32, - ) - .map_err(MetalError::from)?; - } - Ok(Self::new(buffer, device.clone(), el, dtype)) - } - - fn powf(&self, layout: &Layout, pow: f64) -> Result { - let device = self.device().clone(); - - let shape = layout.shape(); - let el = shape.elem_count(); - let dtype = self.dtype; - - let buffer = device.new_buffer(el, self.dtype, "powf")?; - let encoder = self.device.command_encoder()?; - encoder.set_label("powf"); - let src = buffer_o(&self.buffer, layout, dtype); - if layout.is_contiguous() { - let name = match self.dtype { - DType::F32 => "powf_f32", - DType::F16 => "powf_f16", - DType::BF16 => "powf_bf16", - dtype => crate::bail!("Metal contiguous powf {dtype:?} not implemented"), - }; - candle_metal_kernels::call_powf( - &device.device, - &encoder, - &device.kernels, - name, - self.dtype.size_in_bytes(), - el, - src, - &buffer, - pow as f32, - ) - .map_err(MetalError::from)?; - } else { - let name = match self.dtype { - DType::F32 => "powf_f32_strided", - DType::F16 => "powf_f16_strided", - DType::BF16 => "powf_bf16_strided", - dtype => crate::bail!("Metal strided powf {dtype:?} not implemented"), - }; - candle_metal_kernels::call_powf_strided( - &device.device, - &encoder, - &device.kernels, - name, - layout.dims(), - src, - layout.stride(), - &buffer, - pow as f32, - ) - .map_err(MetalError::from)?; - } - Ok(Self::new(buffer, device.clone(), el, dtype)) - } - - fn elu(&self, layout: &Layout, alpha: f64) -> Result { - let device = self.device().clone(); - - let shape = layout.shape(); - let el = shape.elem_count(); - let dtype = self.dtype; - - let buffer = device.new_buffer(el, self.dtype, "elu")?; - let encoder = self.device.command_encoder()?; - encoder.set_label("elu"); - let src = buffer_o(&self.buffer, layout, self.dtype); - if layout.is_contiguous() { - let name = match self.dtype { - DType::F32 => "elu_f32", - DType::F16 => "elu_f16", - DType::BF16 => "elu_bf16", - dtype => crate::bail!("Metal contiguous elu {dtype:?} not implemented"), - }; - candle_metal_kernels::call_elu( - &device.device, - &encoder, - &device.kernels, - name, - self.dtype.size_in_bytes(), - el, - src, - &buffer, - alpha as f32, - ) - .map_err(MetalError::from)?; - } else { - let name = match self.dtype { - DType::F32 => "elu_f32_strided", - DType::F16 => "elu_f16_strided", - DType::BF16 => "elu_bf16_strided", - dtype => crate::bail!("Metal strided elu {dtype:?} not implemented"), - }; - candle_metal_kernels::call_elu_strided( - &device.device, - &encoder, - &device.kernels, - name, - layout.dims(), - src, - layout.stride(), - &buffer, - alpha as f32, - ) - .map_err(MetalError::from)?; - } - Ok(Self::new(buffer, device.clone(), el, dtype)) - } - - fn reduce_op(&self, op: ReduceOp, layout: &Layout, sum_dims: &[usize]) -> Result { - let device = self.device.clone(); - - let src_stride = layout.stride(); - let src_dims = layout.shape().dims(); - // Source dims and strides with the sum dims at the end. - let mut dims = vec![]; - let mut stride = vec![]; - let mut dst_el: usize = 1; - for (dim_idx, &d) in src_dims.iter().enumerate() { - if !sum_dims.contains(&dim_idx) { - dst_el *= d; - dims.push(d); - stride.push(src_stride[dim_idx]); - } - } - - for &dim_idx in sum_dims.iter() { - dims.push(src_dims[dim_idx]); - stride.push(src_stride[dim_idx]); - } - - let reduction_shape = Shape::from(dims.clone()); - - if layout.is_contiguous() && reduction_shape.is_contiguous(&stride) { - let (name, check_empty, return_index) = match (op, self.dtype) { - (ReduceOp::Sum, DType::F32) => ("fast_sum_f32", false, false), - (ReduceOp::Min, DType::F32) => ("fast_min_f32", true, false), - (ReduceOp::Max, DType::F32) => ("fast_max_f32", true, false), - (ReduceOp::ArgMin, DType::F32) => ("fast_argmin_f32", true, true), - (ReduceOp::ArgMax, DType::F32) => ("fast_argmax_f32", true, true), - (ReduceOp::Sum, DType::U32) => ("fast_sum_u32", false, false), - (ReduceOp::Min, DType::U32) => ("fast_min_u32", true, false), - (ReduceOp::Max, DType::U32) => ("fast_max_u32", true, false), - (ReduceOp::ArgMin, DType::U32) => ("fast_argmin_u32", true, true), - (ReduceOp::ArgMax, DType::U32) => ("fast_argmax_u32", true, true), - (ReduceOp::Sum, DType::F16) => ("fast_sum_f16", false, false), - (ReduceOp::Min, DType::F16) => ("fast_min_f16", true, false), - (ReduceOp::Max, DType::F16) => ("fast_max_f16", true, false), - (ReduceOp::ArgMin, DType::F16) => ("fast_argmin_f16", true, true), - (ReduceOp::ArgMax, DType::F16) => ("fast_argmax_f16", true, true), - (ReduceOp::Sum, DType::BF16) => ("fast_sum_bf16", false, false), - (ReduceOp::Min, DType::BF16) => ("fast_min_bf16", true, false), - (ReduceOp::Max, DType::BF16) => ("fast_max_bf16", true, false), - (ReduceOp::ArgMin, DType::BF16) => ("fast_argmin_bf16", true, true), - (ReduceOp::ArgMax, DType::BF16) => ("fast_argmax_bf16", true, true), - (ReduceOp::Sum, DType::I64) => ("fast_sum_i64", false, false), - (ReduceOp::Min, DType::I64) => ("fast_min_i64", true, false), - (ReduceOp::Max, DType::I64) => ("fast_max_i64", true, false), - (ReduceOp::ArgMin, DType::I64) => ("fast_argmin_i64", true, true), - (ReduceOp::ArgMax, DType::I64) => ("fast_argmax_i64", true, true), - (ReduceOp::Sum, DType::U8) => ("fast_sum_u8", false, false), - (ReduceOp::Min, DType::U8) => ("fast_min_u8", true, false), - (ReduceOp::Max, DType::U8) => ("fast_max_u8", true, false), - (ReduceOp::ArgMin, DType::U8) => ("fast_argmin_u8", true, true), - (ReduceOp::ArgMax, DType::U8) => ("fast_argmax_u8", true, true), - (k, dtype) => { - crate::bail!("Metal contiguous reduce op {k:?} {dtype:?} not implemented") - } - }; - if check_empty && layout.shape().elem_count() == 0 { - Err(crate::Error::EmptyTensor { op: "reduce" }.bt())? - } - let dtype = if return_index { DType::U32 } else { self.dtype }; - let buffer = device.new_buffer(dst_el, dtype, "reduce")?; - let encoder = self.device.command_encoder()?; - encoder.set_label("reduce"); - let src = buffer_o(&self.buffer, layout, self.dtype); - candle_metal_kernels::call_reduce_contiguous( - &device.device, - &encoder, - &device.kernels, - name, - src_dims, - dst_el, - src, - &buffer, - ) - .map_err(MetalError::from)?; - - return Ok(Self::new(buffer, device, dst_el, dtype)); - } - - let (name, check_empty, return_index) = match (op, self.dtype) { - (ReduceOp::Sum, DType::F32) => ("fast_sum_f32_strided", false, false), - (ReduceOp::Min, DType::F32) => ("fast_min_f32_strided", true, false), - (ReduceOp::Max, DType::F32) => ("fast_max_f32_strided", true, false), - (ReduceOp::ArgMin, DType::F32) => ("fast_argmin_f32_strided", true, true), - (ReduceOp::ArgMax, DType::F32) => ("fast_argmax_f32_strided", true, true), - (ReduceOp::Sum, DType::U32) => ("fast_sum_u32_strided", false, false), - (ReduceOp::Min, DType::U32) => ("fast_min_u32_strided", true, false), - (ReduceOp::Max, DType::U32) => ("fast_max_u32_strided", true, false), - (ReduceOp::ArgMin, DType::U32) => ("fast_argmin_u32_strided", true, true), - (ReduceOp::ArgMax, DType::U32) => ("fast_argmax_u32_strided", true, true), - (ReduceOp::Sum, DType::F16) => ("fast_sum_f16_strided", false, false), - (ReduceOp::Min, DType::F16) => ("fast_min_f16_strided", true, false), - (ReduceOp::Max, DType::F16) => ("fast_max_f16_strided", true, false), - (ReduceOp::ArgMin, DType::F16) => ("fast_argmin_f16_strided", true, true), - (ReduceOp::ArgMax, DType::F16) => ("fast_argmax_f16_strided", true, true), - (ReduceOp::Sum, DType::BF16) => ("fast_sum_bf16_strided", false, false), - (ReduceOp::Min, DType::BF16) => ("fast_min_bf16_strided", true, false), - (ReduceOp::Max, DType::BF16) => ("fast_max_bf16_strided", true, false), - (ReduceOp::ArgMin, DType::BF16) => ("fast_argmin_bf16_strided", true, true), - (ReduceOp::ArgMax, DType::BF16) => ("fast_argmax_bf16_strided", true, true), - (ReduceOp::Sum, DType::I64) => ("fast_sum_i64_strided", false, false), - (ReduceOp::Min, DType::I64) => ("fast_min_i64_strided", true, false), - (ReduceOp::Max, DType::I64) => ("fast_max_i64_strided", true, false), - (ReduceOp::ArgMin, DType::I64) => ("fast_argmin_i64_strided", true, true), - (ReduceOp::ArgMax, DType::I64) => ("fast_argmax_i64_strided", true, true), - (ReduceOp::Sum, DType::U8) => ("fast_sum_u8_strided", false, false), - (ReduceOp::Min, DType::U8) => ("fast_min_u8_strided", true, false), - (ReduceOp::Max, DType::U8) => ("fast_max_u8_strided", true, false), - (ReduceOp::ArgMin, DType::U8) => ("fast_argmin_u8_strided", true, true), - (ReduceOp::ArgMax, DType::U8) => ("fast_argmax_u8_strided", true, true), - (k, dtype) => crate::bail!("Metal strided reduce op {k:?} {dtype:?} not implemented"), - }; - if check_empty && layout.shape().elem_count() == 0 { - Err(crate::Error::EmptyTensor { op: "reduce" }.bt())? - } - let dtype = if return_index { DType::U32 } else { self.dtype }; - let buffer = device.new_buffer(dst_el, dtype, "reduce")?; - let encoder = self.device.command_encoder()?; - encoder.set_label("reduce"); - let src = buffer_o(&self.buffer, layout, self.dtype); - candle_metal_kernels::call_reduce_strided( - &device.device, - &encoder, - &device.kernels, - name, - &dims, - &stride, - dst_el, - src, - &buffer, - ) - .map_err(MetalError::from)?; - - Ok(Self::new(buffer, device, dst_el, dtype)) - } - - fn cmp(&self, op: CmpOp, rhs: &Self, lhs_l: &Layout, rhs_l: &Layout) -> Result { - let name = match op { - CmpOp::Eq => "eq", - CmpOp::Ne => "ne", - CmpOp::Le => "le", - CmpOp::Ge => "ge", - CmpOp::Lt => "lt", - CmpOp::Gt => "gt", - }; - self.binary(name, rhs, lhs_l, rhs_l) - } - - fn const_set(&mut self, s: crate::scalar::Scalar, l: &Layout) -> Result<()> { - use crate::scalar::Scalar; - fn set( - self_: &mut MetalStorage, - s: S, - l: &Layout, - ) -> Result<()> { - let device = self_.device(); - let dtype = self_.dtype; - let shape = l.shape(); - let el_count = shape.elem_count(); - let encoder = device.command_encoder()?; - encoder.set_label("const-set"); - let dst = buffer_o(&self_.buffer, l, self_.dtype); - - if l.is_contiguous() { - use candle_metal_kernels::unary::contiguous; - let kernel_name = match dtype { - DType::F16 => contiguous::const_set::HALF, - DType::BF16 => contiguous::const_set::BFLOAT, - DType::F32 => contiguous::const_set::FLOAT, - DType::I64 => contiguous::const_set::I64, - DType::U32 => contiguous::const_set::U32, - DType::U8 => contiguous::const_set::U8, - DType::F8E4M3 => crate::bail!("unsupported const-set f8e4m3"), - DType::F64 => crate::bail!("unsupported const-set f64"), - DType::F4 - | DType::F6E2M3 - | DType::F6E3M2 - | DType::F8E8M0 - | DType::I16 - | DType::I32 => { - return Err(Error::UnsupportedDTypeForOp(dtype, "const-set").bt()) - } - }; - candle_metal_kernels::call_const_set_contiguous( - &device.device, - &encoder, - &device.kernels, - kernel_name, - dtype.size_in_bytes(), - el_count, - s, - dst, - ) - .map_err(MetalError::from)?; - } else { - use candle_metal_kernels::unary::strided; - let kernel_name = match dtype { - DType::F16 => strided::const_set::HALF, - DType::BF16 => strided::const_set::BFLOAT, - DType::F32 => strided::const_set::FLOAT, - DType::I64 => strided::const_set::I64, - DType::U32 => strided::const_set::U32, - DType::U8 => strided::const_set::U8, - DType::F8E4M3 => crate::bail!("unsupported const-set f8e4m3"), - DType::F64 => crate::bail!("unsupported const-set f64"), - DType::F4 - | DType::F6E2M3 - | DType::F6E3M2 - | DType::F8E8M0 - | DType::I16 - | DType::I32 => { - return Err(Error::UnsupportedDTypeForOp(dtype, "const-set").bt()) - } - }; - candle_metal_kernels::call_const_set_strided( - &device.device, - &encoder, - &device.kernels, - kernel_name, - l.dims(), - s, - l.stride(), - dst, - ) - .map_err(MetalError::from)?; - } - Ok(()) - } - match (self.dtype, s) { - (DType::U8, Scalar::U8(s)) => set(self, s, l), - (DType::U32, Scalar::U32(s)) => set(self, s, l), - (DType::I64, Scalar::I64(s)) => set(self, s, l), - (DType::F16, Scalar::F16(s)) => set(self, s, l), - (DType::BF16, Scalar::BF16(s)) => set(self, s, l), - (DType::F32, Scalar::F32(s)) => set(self, s, l), - (DType::F64, Scalar::F64(s)) => set(self, s, l), - _ => crate::bail!("dtype mismatch, expected {:?}, got {:?}", self.dtype, s), - } - } - - fn to_dtype(&self, layout: &Layout, dtype: DType) -> Result { - let device = self.device(); - let shape = layout.shape(); - let el_count = shape.elem_count(); - let buffer = device.new_buffer(el_count, dtype, "to_dtype")?; - let encoder = device.command_encoder()?; - encoder.set_label("to_dtype"); - let src = buffer_o(&self.buffer, layout, self.dtype); - if layout.is_contiguous() { - let kernel_name = match (self.dtype, dtype) { - (DType::U32, DType::BF16) => "cast_u32_bf16", - (DType::U32, DType::F16) => "cast_u32_f16", - (DType::U32, DType::F32) => "cast_u32_f32", - (DType::U32, DType::I64) => "cast_u32_i64", - (DType::U32, DType::U8) => "cast_u32_u8", - - (DType::U8, DType::BF16) => "cast_u8_bf16", - (DType::U8, DType::F16) => "cast_u8_f16", - (DType::U8, DType::F32) => "cast_u8_f32", - (DType::U8, DType::I64) => "cast_u8_i64", - (DType::U8, DType::U32) => "cast_u8_u32", - - (DType::F32, DType::BF16) => "cast_f32_bf16", - (DType::F32, DType::F16) => "cast_f32_f16", - (DType::F32, DType::I64) => "cast_f32_i64", - (DType::F32, DType::U32) => "cast_f32_u32", - (DType::F32, DType::U8) => "cast_f32_u8", - - (DType::I64, DType::BF16) => "cast_i64_bf16", - (DType::I64, DType::F16) => "cast_i64_f16", - (DType::I64, DType::F32) => "cast_i64_f32", - (DType::I64, DType::U32) => "cast_i64_u32", - (DType::I64, DType::U8) => "cast_i64_u8", - - (DType::F16, DType::BF16) => "cast_f16_bf16", - (DType::F16, DType::F32) => "cast_f16_f32", - (DType::F16, DType::I64) => "cast_f16_i64", - (DType::F16, DType::U32) => "cast_f16_u32", - (DType::F16, DType::U8) => "cast_f16_u8", - - (DType::BF16, DType::F16) => "cast_bf16_f16", - (DType::BF16, DType::F32) => "cast_bf16_f32", - (DType::BF16, DType::I64) => "cast_bf16_i64", - (DType::BF16, DType::U32) => "cast_bf16_u32", - (DType::BF16, DType::U8) => "cast_bf16_u8", - - (left, right) => { - crate::bail!("Metal contiguous to_dtype {left:?} {right:?} not implemented") - } - }; - candle_metal_kernels::call_cast_contiguous( - &device.device, - &encoder, - &device.kernels, - kernel_name, - self.dtype.size_in_bytes(), - el_count, - src, - &buffer, - ) - .map_err(MetalError::from)?; - } else { - let kernel_name = match (self.dtype, dtype) { - (DType::BF16, DType::F16) => "cast_bf16_f16_strided", - (DType::BF16, DType::F32) => "cast_bf16_f32_strided", - (DType::BF16, DType::I64) => "cast_bf16_i64_strided", - (DType::BF16, DType::U32) => "cast_bf16_u32_strided", - (DType::BF16, DType::U8) => "cast_bf16_u8_strided", - - (DType::F16, DType::BF16) => "cast_f16_bf16_strided", - (DType::F16, DType::F32) => "cast_f16_f32_strided", - (DType::F16, DType::I64) => "cast_f16_i64_strided", - (DType::F16, DType::U32) => "cast_f16_u32_strided", - (DType::F16, DType::U8) => "cast_f16_u8_strided", - - (DType::F32, DType::BF16) => "cast_f32_bf16_strided", - (DType::F32, DType::F16) => "cast_f32_f16_strided", - (DType::F32, DType::I64) => "cast_f32_i64_strided", - (DType::F32, DType::U32) => "cast_f32_u32_strided", - (DType::F32, DType::U8) => "cast_f32_u8_strided", - - (DType::I64, DType::F32) => "cast_i64_f32_strided", - (DType::I64, DType::BF16) => "cast_i64_bf16_strided", - (DType::I64, DType::F16) => "cast_i64_f16_strided", - (DType::I64, DType::U32) => "cast_i64_u32_strided", - (DType::I64, DType::U8) => "cast_i64_u8_strided", - - (DType::U32, DType::BF16) => "cast_u32_bf16_strided", - (DType::U32, DType::F16) => "cast_u32_f16_strided", - (DType::U32, DType::F32) => "cast_u32_f32_strided", - (DType::U32, DType::I64) => "cast_u32_i64_strided", - (DType::U32, DType::U8) => "cast_u32_u8_strided", - - (DType::U8, DType::BF16) => "cast_u8_bf16_strided", - (DType::U8, DType::F16) => "cast_u8_f16_strided", - (DType::U8, DType::F32) => "cast_u8_f32_strided", - (DType::U8, DType::I64) => "cast_u8_i64_strided", - (DType::U8, DType::U32) => "cast_u8_u32_strided", - - (left, right) => { - crate::bail!("Metal strided to_dtype {left:?} {right:?} not implemented") - } - }; - candle_metal_kernels::call_cast_strided( - &device.device, - &encoder, - &device.kernels, - kernel_name, - layout.dims(), - src, - layout.stride(), - &buffer, - ) - .map_err(MetalError::from)?; - } - Ok(Self::new(buffer, device.clone(), el_count, dtype)) - } - - fn unary_impl(&self, layout: &Layout) -> Result { - let device = self.device(); - let dtype = self.dtype; - let shape = layout.shape(); - let el_count = shape.elem_count(); - let buffer = device.new_buffer(el_count, dtype, B::KERNEL)?; - let encoder = device.command_encoder()?; - encoder.set_label(B::KERNEL); - let src = buffer_o(&self.buffer, layout, self.dtype); - - if layout.is_contiguous() { - use candle_metal_kernels::unary::contiguous; - let kernel_name = match (B::KERNEL, dtype) { - ("uabs", DType::F16) => contiguous::abs::HALF, - ("uabs", DType::F32) => contiguous::abs::FLOAT, - ("uabs", DType::BF16) => contiguous::abs::BFLOAT, - ("uceil", DType::F16) => contiguous::ceil::HALF, - ("uceil", DType::F32) => contiguous::ceil::FLOAT, - ("uceil", DType::BF16) => contiguous::ceil::BFLOAT, - ("ucos", DType::F16) => contiguous::cos::HALF, - ("ucos", DType::F32) => contiguous::cos::FLOAT, - ("ucos", DType::BF16) => contiguous::cos::BFLOAT, - ("uerf", DType::F16) => contiguous::erf::HALF, - ("uerf", DType::F32) => contiguous::erf::FLOAT, - ("uerf", DType::BF16) => contiguous::erf::BFLOAT, - ("uexp", DType::F16) => contiguous::exp::HALF, - ("uexp", DType::F32) => contiguous::exp::FLOAT, - ("uexp", DType::BF16) => contiguous::exp::BFLOAT, - ("ufloor", DType::F16) => contiguous::floor::HALF, - ("ufloor", DType::F32) => contiguous::floor::FLOAT, - ("ufloor", DType::BF16) => contiguous::floor::BFLOAT, - ("ugelu_erf", DType::F16) => contiguous::gelu_erf::HALF, - ("ugelu_erf", DType::F32) => contiguous::gelu_erf::FLOAT, - ("ugelu_erf", DType::BF16) => contiguous::gelu_erf::BFLOAT, - ("ugelu", DType::F16) => contiguous::gelu::HALF, - ("ugelu", DType::F32) => contiguous::gelu::FLOAT, - ("ugelu", DType::BF16) => contiguous::gelu::BFLOAT, - ("ulog", DType::F16) => contiguous::log::HALF, - ("ulog", DType::F32) => contiguous::log::FLOAT, - ("ulog", DType::BF16) => contiguous::log::BFLOAT, - ("uneg", DType::F16) => contiguous::neg::HALF, - ("uneg", DType::F32) => contiguous::neg::FLOAT, - ("uneg", DType::BF16) => contiguous::neg::BFLOAT, - ("urecip", DType::F16) => contiguous::recip::HALF, - ("urecip", DType::F32) => contiguous::recip::FLOAT, - ("urecip", DType::BF16) => contiguous::recip::BFLOAT, - ("urelu", DType::F16) => contiguous::relu::HALF, - ("urelu", DType::F32) => contiguous::relu::FLOAT, - ("urelu", DType::BF16) => contiguous::relu::BFLOAT, - ("uround", DType::F16) => contiguous::round::HALF, - ("uround", DType::F32) => contiguous::round::FLOAT, - ("uround", DType::BF16) => contiguous::round::BFLOAT, - ("usilu", DType::F16) => contiguous::silu::HALF, - ("usilu", DType::F32) => contiguous::silu::FLOAT, - ("usilu", DType::BF16) => contiguous::silu::BFLOAT, - ("usin", DType::F16) => contiguous::sin::HALF, - ("usin", DType::F32) => contiguous::sin::FLOAT, - ("usin", DType::BF16) => contiguous::sin::BFLOAT, - ("usqr", DType::F16) => contiguous::sqr::HALF, - ("usqr", DType::F32) => contiguous::sqr::FLOAT, - ("usqr", DType::BF16) => contiguous::sqr::BFLOAT, - ("usqrt", DType::F16) => contiguous::sqrt::HALF, - ("usqrt", DType::F32) => contiguous::sqrt::FLOAT, - ("usqrt", DType::BF16) => contiguous::sqrt::BFLOAT, - ("utanh", DType::F16) => contiguous::tanh::HALF, - ("utanh", DType::F32) => contiguous::tanh::FLOAT, - ("utanh", DType::BF16) => contiguous::tanh::BFLOAT, - ("usign", DType::F16) => contiguous::sign::HALF, - ("usign", DType::F32) => contiguous::sign::FLOAT, - ("usign", DType::BF16) => contiguous::sign::BFLOAT, - ("usign", DType::I64) => contiguous::sign::I64, - (name, dtype) => { - crate::bail!("Metal contiguous unary {name} {dtype:?} not implemented") - } - }; - - candle_metal_kernels::call_unary_contiguous( - &device.device, - &encoder, - &device.kernels, - kernel_name, - dtype.size_in_bytes(), - el_count, - src, - &buffer, - ) - .map_err(MetalError::from)?; - } else { - use candle_metal_kernels::unary::strided; - let kernel_name = match (B::KERNEL, dtype) { - ("ucos", DType::F32) => strided::cos::FLOAT, - ("usin", DType::F32) => strided::sin::FLOAT, - ("usqr", DType::F32) => strided::sqr::FLOAT, - ("usqrt", DType::F32) => strided::sqrt::FLOAT, - ("uneg", DType::F32) => strided::neg::FLOAT, - ("uexp", DType::F32) => strided::exp::FLOAT, - ("ulog", DType::F32) => strided::log::FLOAT, - ("ugelu", DType::F32) => strided::gelu::FLOAT, - ("ugelu_erf", DType::F32) => strided::gelu_erf::FLOAT, - ("uerf", DType::F32) => strided::erf::FLOAT, - ("usilu", DType::F32) => strided::silu::FLOAT, - ("uabs", DType::F32) => strided::abs::FLOAT, - ("uceil", DType::F32) => strided::ceil::FLOAT, - ("ufloor", DType::F32) => strided::floor::FLOAT, - ("urelu", DType::F32) => strided::relu::FLOAT, - ("uround", DType::F32) => strided::round::FLOAT, - ("utanh", DType::F32) => strided::tanh::FLOAT, - - ("ucos", DType::F16) => strided::cos::HALF, - ("usin", DType::F16) => strided::sin::HALF, - ("usqr", DType::F16) => strided::sqr::HALF, - ("usqrt", DType::F16) => strided::sqrt::HALF, - ("uneg", DType::F16) => strided::neg::HALF, - ("uexp", DType::F16) => strided::exp::HALF, - ("ulog", DType::F16) => strided::log::HALF, - ("ugelu", DType::F16) => strided::gelu::HALF, - ("ugelu_erf", DType::F16) => strided::gelu_erf::HALF, - ("uerf", DType::F16) => strided::erf::HALF, - ("usilu", DType::F16) => strided::silu::HALF, - ("uabs", DType::F16) => strided::abs::HALF, - ("uceil", DType::F16) => strided::ceil::HALF, - ("ufloor", DType::F16) => strided::floor::HALF, - ("urelu", DType::F16) => strided::relu::HALF, - ("uround", DType::F16) => strided::round::HALF, - ("utanh", DType::F16) => strided::tanh::HALF, - - ("ucos", DType::BF16) => strided::cos::BFLOAT, - ("usin", DType::BF16) => strided::sin::BFLOAT, - ("usqr", DType::BF16) => strided::sqr::BFLOAT, - ("usqrt", DType::BF16) => strided::sqrt::BFLOAT, - ("uneg", DType::BF16) => strided::neg::BFLOAT, - ("uexp", DType::BF16) => strided::exp::BFLOAT, - ("ulog", DType::BF16) => strided::log::BFLOAT, - ("ugelu", DType::BF16) => strided::gelu::BFLOAT, - ("ugelu_erf", DType::BF16) => strided::gelu_erf::BFLOAT, - ("uerf", DType::BF16) => strided::erf::BFLOAT, - ("usilu", DType::BF16) => strided::silu::BFLOAT, - ("uabs", DType::BF16) => strided::abs::BFLOAT, - ("uceil", DType::BF16) => strided::ceil::BFLOAT, - ("ufloor", DType::BF16) => strided::floor::BFLOAT, - ("urelu", DType::BF16) => strided::relu::BFLOAT, - ("uround", DType::BF16) => strided::round::BFLOAT, - ("utanh", DType::BF16) => strided::tanh::BFLOAT, - - (name, dtype) => { - crate::bail!("Metal strided unary {name} {dtype:?} not implemented") - } - }; - let dst = BufferOffset::zero_offset(&buffer); - candle_metal_kernels::call_unary_strided( - &device.device, - &encoder, - &device.kernels, - kernel_name, - layout.dims(), - src, - layout.stride(), - dst, - ) - .map_err(MetalError::from)?; - } - - Ok(Self::new(buffer, device.clone(), el_count, dtype)) - } - - fn binary_impl( - &self, - rhs: &Self, - lhs_l: &Layout, - rhs_l: &Layout, - ) -> Result { - self.binary(B::KERNEL, rhs, lhs_l, rhs_l) - } - - fn where_cond( - &self, - layout: &Layout, - t: &Self, - t_l: &Layout, - f: &Self, - f_l: &Layout, - ) -> Result { - let device = self.device.clone(); - let shape = t_l.shape(); - let dims = shape.dims(); - let el = shape.elem_count(); - let dtype = t.dtype; - let buffer = self.device.new_buffer(el, dtype, "where")?; - let encoder = self.device.command_encoder()?; - encoder.set_label("where"); - if t.dtype() != f.dtype() { - crate::bail!( - "Invalid where: different dtypes for values {:?} != {:?}", - t.dtype(), - f.dtype() - ); - } - let name = match (self.dtype, t.dtype()) { - (DType::U8, DType::F32) => "where_u8_f32", - (DType::U32, DType::F32) => "where_u32_f32", - (DType::U8, DType::BF16) => "where_u8_bf16", - (DType::U8, DType::F16) => "where_u8_f16", - (DType::U8, DType::I64) => "where_u8_i64", - (DType::U8, DType::U32) => "where_u8_u32", - (DType::U8, DType::U8) => "where_u8_u8", - (left, right) => crate::bail!("Metal where_cond {left:?} {right:?} not implemented"), - }; - let src = buffer_o(&self.buffer, layout, self.dtype); - let t = buffer_o(&t.buffer, t_l, t.dtype); - let f = buffer_o(&f.buffer, f_l, f.dtype); - candle_metal_kernels::call_where_cond( - &device.device, - &encoder, - &device.kernels, - name, - dtype.size_in_bytes(), - dims, - src, - layout.stride(), - layout.is_contiguous(), - t, - t_l.stride(), - t_l.is_contiguous(), - f, - f_l.stride(), - f_l.is_contiguous(), - &buffer, - ) - .map_err(MetalError::from)?; - Ok(Self::new(buffer, device, el, dtype)) - } - - fn conv1d( - &self, - layout: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &ParamsConv1D, - ) -> Result { - let device = self.device().clone(); - let shape = layout.shape(); - let dims = shape.dims(); - let strides = layout.stride(); - - let stride = params.stride; - let dilation = params.dilation; - let padding = params.padding; - let k_size = params.k_size; - let l_out = (dims[2] + 2 * padding - dilation * (k_size - 1) - 1) / stride + 1; - let dst_el = dims[0] * l_out * dims[1] * k_size; - let dst = self - .device - .new_buffer(dst_el, self.dtype, "conv1d_im2col")?; - let encoder = self.device.command_encoder()?; - encoder.set_label("conv1d_im2col"); - let name = match self.dtype { - DType::F32 => "im2col1d_f32", - DType::F16 => "im2col1d_f16", - DType::BF16 => "im2col1d_bf16", - DType::U8 => "im2col1d_u8", - DType::U32 => "im2col1d_u32", - dtype => crate::bail!("Metal conv1d {dtype:?} not implemented"), - }; - let src = buffer_o(&self.buffer, layout, self.dtype); - candle_metal_kernels::call_im2col1d_strided( - &self.device.device, - &encoder, - &self.device.kernels, - name, - layout.shape().dims(), - strides, - (k_size, stride, padding, dilation), - src, - &dst, - ) - .map_err(MetalError::from)?; - drop(encoder); - let col = Self { - buffer: dst, - device, - count: dst_el, - dtype: self.dtype, - }; - let l_out = params.l_out(); - let b = params.b_size; - let n = params.c_out; - let k = params.k_size * params.c_in; - let m = l_out; - let col_l = Layout::contiguous((b, m, k)); - let res = if kernel_l.is_contiguous() { - let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) - .transpose(1, 2)? - .broadcast_as((b, k, n))?; - col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? - } else { - // Make the kernel contiguous if not already the case. - let mut kernel_c = self.device().zeros_impl(kernel_l.shape(), kernel.dtype())?; - kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?; - let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) - .transpose(1, 2)? - .broadcast_as((b, k, n))?; - col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? - }; - let res_l = Layout::contiguous((b, l_out, n)).transpose(1, 2)?; - let mut res_t = self.device().zeros_impl(res_l.shape(), res.dtype())?; - res.copy_strided_src(&mut res_t, 0, &res_l)?; - Ok(res_t) - } - - fn conv_transpose1d( - &self, - layout: &Layout, - k: &Self, - k_layout: &Layout, - params: &ParamsConvTranspose1D, - ) -> Result { - const USE_COL2IM_CONV1D_TR: bool = true; - - let can_use_col2im = k_layout.is_contiguous() - && params.dilation == 1 - && params.padding == 0 - && params.output_padding == 0; - let l_out = params.l_out(); - let dst_el = params.c_out * l_out * params.b_size; - - let buffer = if USE_COL2IM_CONV1D_TR && can_use_col2im { - let (b_size, c_in, l_in) = layout.shape().dims3()?; - let (c_in2, c_out, k_size) = k_layout.shape().dims3()?; - if c_in != c_in2 { - crate::bail!( - "convtr1d: shape mismatch on c_in {:?} {:?}", - layout.shape(), - k_layout.shape() - ) - } - let buffer = self - .device - .new_buffer(dst_el, self.dtype, "conv_transpose1d")?; - - let name = match self.dtype { - DType::F32 => "col2im1d_f32", - DType::F16 => "col2im1d_f16", - DType::BF16 => "col2im1d_bf16", - DType::U32 => "col2im1d_u32", - DType::U8 => "col2im1d_u8", - dtype => crate::bail!("metal col2im1d {dtype:?} not implemented"), - }; - let col = { - // This merges the last two dimensions of the kernel together. - let kernel_l_mm = Layout::new( - (b_size, c_in, k_size * c_out).into(), - vec![0, k_size * c_out, 1], - k_layout.start_offset(), - ); - self.matmul( - k, - (b_size, l_in, c_out * k_size, c_in), - &layout.transpose(1, 2)?, - &kernel_l_mm, - )? - }; - // It is important for the command encoder to be obtained *after* the matmul - // kernel has run, otherwise we might use a command-buffer that has been committed - // already resulting in the following error. - // _status < MTLCommandBufferStatusCommitted > - // -[IOGPUMetalCommandBuffer setCurrentCommandEncoder:] - let encoder = self.device.command_encoder()?; - encoder.set_label("col2im1d"); - candle_metal_kernels::call_col2im1d( - &self.device.device, - &encoder, - &self.device.kernels, - name, - &[b_size, l_in, c_out, k_size], - params.k_size, - params.stride, - BufferOffset::zero_offset(&col.buffer), - &buffer, - ) - .map_err(MetalError::from)?; - buffer - } else { - let buffer = self - .device - .new_buffer(dst_el, self.dtype, "conv_transpose1d")?; - - let encoder = self.device.command_encoder()?; - encoder.set_label("conv_transpose1d"); - let name = match self.dtype { - DType::F32 => "conv_transpose1d_f32", - DType::F16 => "conv_transpose1d_f16", - DType::BF16 => "conv_transpose1d_bf16", - DType::U32 => "conv_transpose1d_u32", - DType::U8 => "conv_transpose1d_u8", - dtype => crate::bail!("Metal conv_transpose1d {dtype:?} not implemented"), - }; - candle_metal_kernels::call_conv_transpose1d( - &self.device.device, - &encoder, - &self.device.kernels, - name, - params.dilation, - params.stride, - params.padding, - params.output_padding, - params.c_out, - l_out, - params.b_size, - layout.dims(), - layout.stride(), - k_layout.dims(), - k_layout.stride(), - &self.buffer, - layout.start_offset() * self.dtype.size_in_bytes(), - &k.buffer, - k_layout.start_offset() * k.dtype.size_in_bytes(), - &buffer, - ) - .map_err(MetalError::from)?; - buffer - }; - Ok(Self::new(buffer, self.device.clone(), dst_el, self.dtype)) - } - - fn conv2d( - &self, - layout: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &ParamsConv2D, - ) -> Result { - let device = self.device().clone(); - let shape = layout.shape(); - let dims = shape.dims(); - - let stride = params.stride; - let dilation = params.dilation; - let padding = params.padding; - let h_k = params.k_h; - let w_k = params.k_w; - let h = dims[2]; - let w = dims[3]; - let h_out = (h + 2 * padding - dilation * (h_k - 1) - 1) / stride + 1; - let w_out = (w + 2 * padding - dilation * (w_k - 1) - 1) / stride + 1; - let dst_el = dims[0] * h_out * w_out * dims[1] * h_k * w_k; - - let dst = self - .device - .new_buffer(dst_el, self.dtype, "conv2d_im2col")?; - let encoder = self.device.command_encoder()?; - encoder.set_label("conv2d_im2col"); - let name = match self.dtype { - DType::F32 => "im2col_f32", - DType::F16 => "im2col_f16", - DType::BF16 => "im2col_bf16", - DType::U8 => "im2col_u8", - DType::U32 => "im2col_u32", - dtype => crate::bail!("Metal conv2d {dtype:?} not implemented"), - }; - let src = buffer_o(&self.buffer, layout, self.dtype); - candle_metal_kernels::call_im2col_strided( - &self.device.device, - &encoder, - &self.device.kernels, - name, - layout.shape().dims(), - layout.stride(), - (h_k, w_k, stride, padding, dilation), - src, - &dst, - ) - .map_err(MetalError::from)?; - drop(encoder); - let col = Self { - buffer: dst, - device, - count: dst_el, - dtype: self.dtype, - }; - let h_out = params.out_h(); - let w_out = params.out_w(); - let b = params.b_size; - let n = params.c_out; - let k = params.k_h * params.k_w * params.c_in; - let m = h_out * w_out; - let col_l = Layout::contiguous((b, m, k)); - let res = if kernel_l.is_contiguous() { - let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) - .transpose(1, 2)? - .broadcast_as((b, k, n))?; - col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? - } else { - // Make the kernel contiguous if not already the case. - let mut kernel_c = self.device().zeros_impl(kernel_l.shape(), kernel.dtype())?; - kernel.copy_strided_src(&mut kernel_c, 0, kernel_l)?; - let kernel_l = Layout::contiguous_with_offset((1, n, k), kernel_l.start_offset()) - .transpose(1, 2)? - .broadcast_as((b, k, n))?; - col.matmul(kernel, (b, m, n, k), &col_l, &kernel_l)? - }; - let res_l = Layout::contiguous((b, h_out, w_out, n)) - .transpose(1, 2)? - .transpose(1, 3)?; - let mut res_t = self.device().zeros_impl(res_l.shape(), res.dtype())?; - res.copy_strided_src(&mut res_t, 0, &res_l)?; - Ok(res_t) - } - - fn conv_transpose2d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &ParamsConvTranspose2D, - ) -> Result { - // Kernel shape: (c_in_k, c_out, h_k, w_k) - // Input shape: (b_size, c_in, h_in, w_in) - let (out_w, out_h) = (params.out_w(), params.out_h()); - let dst_el = params.c_out * out_w * out_h * params.b_size; - - let dims = l.dims(); - if dims.len() != 4 { - crate::bail!("unexpected input shape for conv_transpose2d {dims:?}, expected 4") - } - - let k_dims = kernel_l.dims(); - if k_dims.len() != 4 { - crate::bail!("unexpected kernel shape for conv_transpose2d {k_dims:?}, expected 4") - } - - let buffer = self - .device - .new_buffer(dst_el, self.dtype, "conv_transpose2d")?; - - let encoder = self.device.command_encoder()?; - encoder.set_label("conv_transpose2d"); - - let name = match self.dtype { - DType::F32 => "conv_transpose2d_f32", - DType::F16 => "conv_transpose2d_f16", - DType::BF16 => "conv_transpose2d_bf16", - dtype => crate::bail!("Metal conv_transpose2d {dtype:?} not implemented"), - }; - - candle_metal_kernels::call_conv_transpose2d( - &self.device.device, - &encoder, - &self.device.kernels, - name, - CallConvTranspose2dCfg { - dilation: params.dilation, - stride: params.stride, - padding: params.padding, - output_padding: params.output_padding, - c_out: params.c_out, - out_h, - out_w, - b_size: params.b_size, - input_dims: l.dims(), - input_stride: l.stride(), - kernel_dims: kernel_l.dims(), - kernel_stride: kernel_l.stride(), - input_offset: l.start_offset() * self.dtype.size_in_bytes(), - kernel_offset: kernel_l.start_offset() * kernel.dtype.size_in_bytes(), - }, - &self.buffer, - &kernel.buffer, - &buffer, - ) - .map_err(MetalError::from)?; - Ok(Self::new(buffer, self.device.clone(), dst_el, self.dtype)) - } - - fn avg_pool2d( - &self, - inp_l: &Layout, - (w_k, h_k): (usize, usize), - (w_stride, h_stride): (usize, usize), - ) -> Result { - let shape = inp_l.shape(); - let (b_size, channels, width, height) = shape.dims4()?; - let strides = inp_l.stride(); - let name = match self.dtype { - DType::F32 => "avg_pool2d_f32", - DType::F16 => "avg_pool2d_f16", - DType::BF16 => "avg_pool2d_bf16", - DType::U8 => "avg_pool2d_u8", - DType::U32 => "avg_pool2d_u32", - dtype => crate::bail!("Metal avg_pool2d {dtype:?} not implemented"), - }; - let out_w = (width - w_k) / w_stride + 1; - let out_h = (height - h_k) / h_stride + 1; - let dst_el = out_w * out_h * b_size * channels; - let buffer = self.device.new_buffer(dst_el, self.dtype, "avg_pool2d")?; - let encoder = self.device.command_encoder()?; - encoder.set_label("avg_pool2d"); - candle_metal_kernels::call_pool2d( - &self.device.device, - &encoder, - &self.device.kernels, - name, - inp_l.dims(), - strides, - out_w, - out_h, - w_k, - h_k, - w_stride, - h_stride, - &self.buffer, - &buffer, - ) - .map_err(MetalError::from)?; - Ok(Self::new(buffer, self.device.clone(), dst_el, self.dtype)) - } - - fn max_pool2d( - &self, - inp_l: &Layout, - (w_k, h_k): (usize, usize), - (w_stride, h_stride): (usize, usize), - ) -> Result { - let shape = inp_l.shape(); - let (b_size, channels, width, height) = shape.dims4()?; - let strides = inp_l.stride(); - let name = match self.dtype { - DType::F32 => "max_pool2d_f32", - DType::F16 => "max_pool2d_f16", - DType::BF16 => "max_pool2d_bf16", - DType::U8 => "max_pool2d_u8", - DType::U32 => "max_pool2d_u32", - dtype => crate::bail!("Metal max_pool2d {dtype:?} not implemented"), - }; - let out_w = (width - w_k) / w_stride + 1; - let out_h = (height - h_k) / h_stride + 1; - let dst_el = out_w * out_h * b_size * channels; - let buffer = self.device.new_buffer(dst_el, self.dtype, "max_pool2d")?; - let encoder = self.device.command_encoder()?; - encoder.set_label("max_pool2d"); - candle_metal_kernels::call_pool2d( - &self.device.device, - &encoder, - &self.device.kernels, - name, - inp_l.dims(), - strides, - out_w, - out_h, - w_k, - h_k, - w_stride, - h_stride, - &self.buffer, - &buffer, - ) - .map_err(MetalError::from)?; - Ok(Self::new(buffer, self.device.clone(), dst_el, self.dtype)) - } - - fn upsample_nearest1d(&self, _: &Layout, _: usize) -> Result { - crate::bail!("Metal upsample_nearest1d not implemented") - } - - fn upsample_nearest2d(&self, inp_l: &Layout, out_w: usize, out_h: usize) -> Result { - // let inp = &inp.slice(inp_l.start_offset()..); - let shape = inp_l.shape(); - let dims = shape.dims(); - let strides = inp_l.stride(); - if dims.len() != 4 { - crate::bail!("unexpected input shape for upsample {dims:?}") - } - let name = match self.dtype { - DType::F32 => "upsample_nearest2d_f32", - DType::F16 => "upsample_nearest2d_f16", - DType::BF16 => "upsample_nearest2d_bf16", - DType::U8 => "upsample_nearest2d_u8", - DType::U32 => "upsample_nearest2d_u32", - dtype => crate::bail!("Metal upsample_nearest2d {dtype:?} not implemented"), - }; - - let dst_el = out_w * out_h * dims[0] * dims[1]; - let buffer = self - .device - .new_buffer(dst_el, self.dtype, "upsample_nearest2d")?; - let encoder = self.device.command_encoder()?; - encoder.set_label("upsample_nearest2d"); - let src = buffer_o(&self.buffer, inp_l, self.dtype); - candle_metal_kernels::call_upsample_nearest_2d( - &self.device.device, - &encoder, - &self.device.kernels, - name, - dims, - strides, - out_w, - out_h, - src, - &buffer, - ) - .map_err(MetalError::from)?; - Ok(Self::new(buffer, self.device.clone(), dst_el, self.dtype)) - } - - fn upsample_bilinear2d( - &self, - inp_l: &Layout, - out_h: usize, - out_w: usize, - align_corners: bool, - scale_h: Option, - scale_w: Option, - ) -> Result { - let shape = inp_l.shape(); - let dims = shape.dims(); - let strides = inp_l.stride(); - - if dims.len() != 4 { - crate::bail!("unexpected input shape for upsample_bilinear2d {dims:?}") - } - - let name = match self.dtype { - DType::F32 => "upsample_bilinear2d_f32", - DType::F16 => "upsample_bilinear2d_f16", - DType::BF16 => "upsample_bilinear2d_bf16", - DType::U8 => "upsample_bilinear2d_u8", - DType::U32 => "upsample_bilinear2d_u32", - dtype => crate::bail!("Metal upsample_bilinear2d {dtype:?} not implemented"), - }; - - let dst_el = out_w * out_h * dims[0] * dims[1]; - let buffer = self - .device - .new_buffer(dst_el, self.dtype, "upsample_bilinear2d")?; - - let encoder = self.device.command_encoder()?; - encoder.set_label("upsample_bilinear2d"); - - let src = buffer_o(&self.buffer, inp_l, self.dtype); - candle_metal_kernels::call_upsample_bilinear_2d( - &self.device.device, - &encoder, - &self.device.kernels, - name, - dims, - strides, - out_w, - out_h, - align_corners, - scale_h, - scale_w, - src, - &buffer, - ) - .map_err(MetalError::from)?; - - Ok(Self::new(buffer, self.device.clone(), dst_el, self.dtype)) - } - - fn gather(&self, src_l: &Layout, ids: &Self, ids_l: &Layout, dim: usize) -> Result { - if !ids_l.is_contiguous() { - return Err(crate::Error::RequiresContiguous { op: "gather" }.bt()); - }; - let ids_el = ids_l.dims()[dim]; - let dst_el = ids_l.shape().elem_count(); - let dtype = self.dtype; - let device = self.device(); - let buffer = device.new_buffer(dst_el, dtype, "gather")?; - let name = match (ids.dtype, self.dtype) { - (DType::U8, DType::U8) => "gather_u8_u8", - (DType::U8, DType::F32) => "gather_u8_f32", - (DType::U8, DType::F16) => "gather_u8_f16", - (DType::U8, DType::BF16) => "gather_u8_bf16", - (DType::U8, DType::U32) => "gather_u8_u32", - (DType::U8, DType::I64) => "gather_u8_i64", - (DType::U32, DType::F32) => "gather_u32_f32", - (DType::U32, DType::F16) => "gather_u32_f16", - (DType::U32, DType::BF16) => "gather_u32_bf16", - (DType::U32, DType::U32) => "gather_u32_u32", - (DType::U32, DType::I64) => "gather_u32_i64", - (DType::I64, DType::F32) => "gather_i64_f32", - (DType::I64, DType::F16) => "gather_i64_f16", - (DType::I64, DType::BF16) => "gather_i64_bf16", - (DType::I64, DType::U32) => "gather_i64_u32", - (DType::I64, DType::I64) => "gather_i64_i64", - (left, right) => crate::bail!("Metal gather {left:?} {right:?} not implemented"), - }; - let encoder = self.device.command_encoder()?; - encoder.set_label("gather"); - let src = buffer_o(&self.buffer, src_l, dtype); - let ids = buffer_o(&ids.buffer, ids_l, ids.dtype); - candle_metal_kernels::call_gather( - &device.device, - &encoder, - &self.device.kernels, - name, - src_l.dims(), - ids_el, - dim, - src, - ids, - &buffer, - ) - .map_err(MetalError::from)?; - Ok(Self::new(buffer, device.clone(), dst_el, dtype)) - } - - fn scatter_set( - &mut self, - l: &Layout, - ids: &Self, - ids_l: &Layout, - src: &Self, - src_l: &Layout, - dim: usize, - ) -> Result<()> { - if !l.is_contiguous() || !ids_l.is_contiguous() || !src_l.is_contiguous() { - return Err(crate::Error::RequiresContiguous { op: "scatter" }.bt()); - }; - let name = match (ids.dtype, self.dtype) { - (DType::U8, DType::F32) => "s_u8_f32", - (DType::U8, DType::F16) => "s_u8_f16", - (DType::U8, DType::BF16) => "s_u8_bf16", - (DType::U32, DType::U32) => "s_u32_u32", - (DType::U32, DType::F32) => "s_u32_f32", - (DType::U32, DType::F16) => "s_u32_f16", - (DType::U32, DType::BF16) => "s_u32_bf16", - (DType::I64, DType::F32) => "s_i64_f32", - (DType::I64, DType::F16) => "s_i64_f16", - (DType::I64, DType::BF16) => "s_i64_bf16", - _ => Err(MetalError::UnexpectedDType { - msg: "scatter ids should be u8/u32/i64", - expected: DType::U32, - got: ids.dtype(), - })?, - }; - let encoder = self.device.command_encoder()?; - encoder.set_label("scatter"); - let dst = buffer_o(&self.buffer, l, self.dtype); - let src = buffer_o(&src.buffer, src_l, src.dtype); - let ids = buffer_o(&ids.buffer, ids_l, ids.dtype); - candle_metal_kernels::call_scatter( - &self.device.device, - &encoder, - &self.device.kernels, - name, - src_l.dims(), - l.dims(), - dim, - src, - ids, - dst, - ) - .map_err(MetalError::from)?; - Ok(()) - } - - fn scatter_add_set( - &mut self, - l: &Layout, - ids: &Self, - ids_l: &Layout, - src: &Self, - src_l: &Layout, - dim: usize, - ) -> Result<()> { - if !l.is_contiguous() || !ids_l.is_contiguous() || !src_l.is_contiguous() { - return Err(crate::Error::RequiresContiguous { op: "scatter-add" }.bt()); - }; - let name = match (ids.dtype, self.dtype) { - (DType::U8, DType::F32) => "sa_u8_f32", - (DType::U8, DType::F16) => "sa_u8_f16", - (DType::U8, DType::BF16) => "sa_u8_bf16", - (DType::U32, DType::U32) => "sa_u32_u32", - (DType::U32, DType::F32) => "sa_u32_f32", - (DType::U32, DType::F16) => "sa_u32_f16", - (DType::U32, DType::BF16) => "sa_u32_bf16", - (DType::I64, DType::F32) => "sa_i64_f32", - (DType::I64, DType::F16) => "sa_i64_f16", - (DType::I64, DType::BF16) => "sa_i64_bf16", - _ => Err(MetalError::UnexpectedDType { - msg: "scatter-add ids should be u8/u32/i64", - expected: DType::U32, - got: ids.dtype(), - })?, - }; - let encoder = self.device.command_encoder()?; - encoder.set_label("scatter_add"); - let dst = buffer_o(&self.buffer, l, self.dtype); - let src = buffer_o(&src.buffer, src_l, src.dtype); - let ids = buffer_o(&ids.buffer, ids_l, ids.dtype); - candle_metal_kernels::call_scatter( - &self.device.device, - &encoder, - &self.device.kernels, - name, - src_l.dims(), - l.dims(), - dim, - src, - ids, - dst, - ) - .map_err(MetalError::from)?; - Ok(()) - } - - fn index_select(&self, ids: &Self, src_l: &Layout, ids_l: &Layout, dim: usize) -> Result { - if !ids_l.is_contiguous() { - crate::bail!("Metal index_select requires contiguous ids") - } - let left_size: usize = src_l.dims()[..dim].iter().product(); - let right_size: usize = src_l.dims()[dim + 1..].iter().product(); - let ids_el = ids_l.shape().elem_count(); - let dst_el = ids_el * left_size * right_size; - let dtype = self.dtype; - let device = self.device(); - let buffer = device.new_buffer(dst_el, dtype, "index_select")?; - let name = match (ids.dtype, self.dtype) { - (DType::U8, DType::U8) => "is_u8_u8", - (DType::U8, DType::U32) => "is_u8_u32", - (DType::U8, DType::I64) => "is_u8_i64", - (DType::U8, DType::BF16) => "is_u8_bf16", - (DType::U8, DType::F32) => "is_u8_f32", - (DType::U8, DType::F16) => "is_u8_f16", - - (DType::U32, DType::U8) => "is_u32_u8", - (DType::U32, DType::U32) => "is_u32_u32", - (DType::U32, DType::I64) => "is_u32_i64", - (DType::U32, DType::F32) => "is_u32_f32", - (DType::U32, DType::F16) => "is_u32_f16", - (DType::U32, DType::BF16) => "is_u32_bf16", - - (DType::I64, DType::U8) => "is_i64_u8", - (DType::I64, DType::U32) => "is_i64_u32", - (DType::I64, DType::I64) => "is_i64_i64", - (DType::I64, DType::F32) => "is_i64_f32", - (DType::I64, DType::F16) => "is_i64_f16", - (DType::I64, DType::BF16) => "is_i64_bf16", - - (left, right) => { - crate::bail!("Metal contiguous index_select {left:?} {right:?} not implemented") - } - }; - let encoder = self.device.command_encoder()?; - let src = buffer_o(&self.buffer, src_l, dtype); - let ids = buffer_o(&ids.buffer, ids_l, ids.dtype); - candle_metal_kernels::call_index_select( - &device.device, - &encoder, - &self.device.kernels, - name, - src_l.dims(), - ids_el, - dim, - src_l.is_contiguous(), - src_l.dims(), - src_l.stride(), - src, - ids, - &buffer, - ) - .map_err(MetalError::from)?; - Ok(Self::new(buffer, device.clone(), dst_el, dtype)) - } - - fn index_add( - &self, - l: &Layout, - ids: &Self, - ids_l: &Layout, - src: &Self, - src_l: &Layout, - dim: usize, - ) -> Result { - let mut acc = self.device.zeros_impl(l.shape(), self.dtype())?; - self.copy_strided_src(&mut acc, 0, l)?; - if !ids_l.is_contiguous() || !src_l.is_contiguous() { - return Err(crate::Error::RequiresContiguous { op: "index-add" }.bt()); - }; - let name = match (ids.dtype, self.dtype) { - (DType::I64, DType::BF16) => "ia_i64_bf16", - (DType::I64, DType::F16) => "ia_i64_f16", - (DType::I64, DType::F32) => "ia_i64_f32", - (DType::I64, DType::I64) => "ia_i64_i64", - (DType::I64, DType::U32) => "ia_i64_u32", - (DType::I64, DType::U8) => "ia_i64_u8", - - (DType::U32, DType::BF16) => "ia_u32_bf16", - (DType::U32, DType::F16) => "ia_u32_f16", - (DType::U32, DType::F32) => "ia_u32_f32", - (DType::U32, DType::I64) => "ia_u32_i64", - (DType::U32, DType::U32) => "ia_u32_u32", - (DType::U32, DType::U8) => "ia_u32_u8", - - (DType::U8, DType::BF16) => "ia_u8_bf16", - (DType::U8, DType::F16) => "ia_u8_f16", - (DType::U8, DType::F32) => "ia_u8_f32", - (DType::U8, DType::I64) => "ia_u8_i64", - (DType::U8, DType::U32) => "ia_u8_u32", - (DType::U8, DType::U8) => "ia_u8_u8", - - _ => Err(MetalError::UnexpectedDType { - msg: "index-add ids should be u8/u32/i64", - expected: DType::U32, - got: ids.dtype(), - })?, - }; - let encoder = self.device.command_encoder()?; - encoder.set_label("index_add"); - let src = buffer_o(&src.buffer, src_l, src.dtype); - let ids = buffer_o(&ids.buffer, ids_l, ids.dtype); - candle_metal_kernels::call_index_add( - &self.device.device, - &encoder, - &self.device.kernels, - name, - src_l.dims(), - l.dims(), - ids_l.dims(), - dim, - src, - ids, - &acc.buffer, - ) - .map_err(MetalError::from)?; - Ok(acc) - } - - fn matmul( - &self, - rhs: &Self, - (b, m, n, k): (usize, usize, usize, usize), - lhs_l: &Layout, - rhs_l: &Layout, - ) -> Result { - let buffer = self.device.new_buffer(b * m * n, self.dtype, "matmul")?; - let encoder = self.device.command_encoder()?; - encoder.set_label("matmul"); - let dtype = match self.dtype { - DType::F32 => candle_metal_kernels::GemmDType::F32, - DType::F16 => candle_metal_kernels::GemmDType::F16, - DType::BF16 => candle_metal_kernels::GemmDType::BF16, - dtype => { - return Err( - MetalError::Message(format!("mlx matmul doesn't support {dtype:?}")).into(), - ) - } - }; - candle_metal_kernels::call_mlx_gemm( - &self.device.device, - &encoder, - &self.device.kernels, - dtype, - (b, m, n, k), - lhs_l.stride(), - lhs_l.start_offset() * self.dtype.size_in_bytes(), - &self.buffer, - rhs_l.stride(), - rhs_l.start_offset() * rhs.dtype.size_in_bytes(), - &rhs.buffer, - &buffer, - ) - .map_err(MetalError::from)?; - - Ok(Self::new( - buffer, - self.device.clone(), - b * m * n, - self.dtype(), - )) - } - - fn copy2d( - &self, - dst: &mut Self, - d1: usize, - d2: usize, - src_s: usize, - dst_s: usize, - src_o: usize, - dst_o: usize, - ) -> Result<()> { - if self.dtype() != dst.dtype() { - crate::bail!( - "copy2d with inconsistent dtypes {:?} {:?}", - self.dtype(), - dst.dtype() - ) - } - if src_s == d2 && dst_s == d2 { - let blit = self.device.blit_command_encoder()?; - blit.set_label("copy2d_contiguous"); - let src_offset = src_o * self.dtype.size_in_bytes(); - let length = d1 * d2 * self.dtype.size_in_bytes(); - let dst_offset = dst_o * dst.dtype().size_in_bytes(); - blit.copy_from_buffer(&self.buffer, src_offset, dst.buffer(), dst_offset, length); - blit.end_encoding(); - } else { - let el_count = d1 * d2; - if el_count == 0 { - return Ok(()); - } - let kernel_name = match self.dtype { - DType::F32 => candle_metal_kernels::copy2d::FLOAT, - DType::F16 => candle_metal_kernels::copy2d::HALF, - DType::BF16 => candle_metal_kernels::copy2d::BFLOAT, - DType::I64 => candle_metal_kernels::copy2d::I64, - DType::U32 => candle_metal_kernels::copy2d::U32, - DType::U8 => candle_metal_kernels::copy2d::U8, - dtype => crate::bail!("Metal copy2d {dtype:?} not implemented"), - }; - let encoder = self.device.command_encoder()?; - encoder.set_label("copy2d"); - candle_metal_kernels::call_copy2d( - &self.device.device, - &encoder, - &self.device.kernels, - kernel_name, - &self.buffer, - &dst.buffer, - d1, - d2, - src_s, - dst_s, - src_o * self.dtype.size_in_bytes(), - dst_o * self.dtype.size_in_bytes(), - ) - .map_err(MetalError::from)?; - } - Ok(()) - } - - fn copy_strided_src(&self, dst: &mut Self, dst_offset: usize, src_l: &Layout) -> Result<()> { - if src_l.is_contiguous() && self.dtype == dst.dtype() { - let blit = self.device.blit_command_encoder()?; - blit.set_label("copy_contiguous"); - let src_offset = src_l.start_offset() * self.dtype.size_in_bytes(); - let length = src_l.shape().elem_count() * self.dtype.size_in_bytes(); - let dst_offset = dst_offset * dst.dtype().size_in_bytes(); - blit.copy_from_buffer(&self.buffer, src_offset, dst.buffer(), dst_offset, length); - blit.end_encoding(); - } else { - let src_shape = src_l.shape(); - let el_count = src_shape.elem_count(); - if el_count == 0 { - return Ok(()); - } - let kernel_name = match self.dtype { - DType::F32 => candle_metal_kernels::unary::strided::copy::FLOAT, - DType::F16 => candle_metal_kernels::unary::strided::copy::HALF, - DType::BF16 => candle_metal_kernels::unary::strided::copy::BFLOAT, - DType::I64 => candle_metal_kernels::unary::strided::copy::I64, - DType::U32 => candle_metal_kernels::unary::strided::copy::U32, - DType::U8 => candle_metal_kernels::unary::strided::copy::U8, - dtype => crate::bail!("Metal copy_strided {dtype:?} not implemented"), - }; - let src = buffer_o(&self.buffer, src_l, self.dtype); - let dst = BufferOffset { - buffer: &dst.buffer, - offset_in_bytes: dst_offset * dst.dtype.size_in_bytes(), - }; - let encoder = self.device.command_encoder()?; - encoder.set_label("copy_strided"); - candle_metal_kernels::call_unary_strided( - &self.device.device, - &encoder, - &self.device.kernels, - kernel_name, - src_l.dims(), - src, - src_l.stride(), - dst, - ) - .map_err(MetalError::from)?; - } - Ok(()) - } -} - -impl MetalStorage { - pub fn new(buffer: Arc, device: MetalDevice, count: usize, dtype: DType) -> Self { - Self { - buffer, - device, - count, - dtype, - } - } - - pub fn buffer(&self) -> &Buffer { - &self.buffer - } - - pub fn binary( - &self, - op: &'static str, - rhs: &Self, - lhs_l: &Layout, - rhs_l: &Layout, - ) -> Result { - fn kernel_name(op: &'static str, dtype: &DType, suffix: &str) -> String { - format!("{op}_{}{}", dtype.as_str(), suffix) - } - let device = self.device(); - let shape = lhs_l.shape(); - let el_count = shape.elem_count(); - let encoder = device.command_encoder()?; - let lhs = buffer_o(&self.buffer, lhs_l, self.dtype); - let rhs = buffer_o(&rhs.buffer, rhs_l, rhs.dtype); - - let dtype = match op { - "eq" | "ne" | "le" | "lt" | "ge" | "gt" => DType::U8, - _ => self.dtype, - }; - let lhs_contiguous = lhs_l.is_contiguous(); - let rhs_contiguous = rhs_l.is_contiguous(); - - let buffer = if lhs_contiguous && rhs_contiguous { - let kernel = kernel_name(op, &self.dtype, ""); - let buffer = device.new_buffer(el_count, dtype, op)?; - candle_metal_kernels::call_binary_contiguous( - &device.device, - &encoder, - &device.kernels, - kernel, - self.dtype.size_in_bytes(), - el_count, - lhs, - rhs, - &buffer, - ) - .map_err(MetalError::from)?; - buffer - } else { - let strided_suffix = if lhs_contiguous { - "_rstrided" - } else if rhs_contiguous { - "_lstrided" - } else { - "_strided" - }; - let kernel = kernel_name(op, &self.dtype, strided_suffix); - let buffer = device.new_buffer(el_count, dtype, op)?; - candle_metal_kernels::call_binary_strided( - &device.device, - &encoder, - &device.kernels, - kernel, - self.dtype.size_in_bytes(), - lhs_l.dims(), - lhs, - lhs_l.stride(), - rhs, - rhs_l.stride(), - &buffer, - ) - .map_err(MetalError::from)?; - buffer - }; - encoder.set_label("binary"); - Ok(Self::new(buffer, device.clone(), el_count, dtype)) - } - - pub(crate) fn to_cpu(&self) -> Result> { - let size = self.count * self.dtype.size_in_bytes(); - let buffer = self.device.allocate_buffer(size)?; - { - let blit = self.device.blit_command_encoder()?; - blit.set_label("blit_to_cpu"); - blit.copy_from_buffer(&self.buffer, 0, &buffer, 0, size); - blit.end_encoding(); - } - self.device.wait_until_completed()?; - Ok(read_to_vec(&buffer, self.count)) - } -} - -impl BackendDevice for MetalDevice { - type Storage = MetalStorage; - - fn new(ordinal: usize) -> Result { - let device = Device::all().swap_remove(ordinal); - let command_queue = device.new_command_queue().map_err(MetalError::from)?; - let kernels = Arc::new(Kernels::new()); - let seed = Arc::new(Mutex::new( - device - .new_buffer_with_data( - [299792458u64].as_ptr() as *const c_void, - 4, - RESOURCE_OPTIONS, - ) - .map_err(MetalError::from)?, - )); - let commands = Commands::new(command_queue).map_err(MetalError::from)?; - Ok(Self { - id: DeviceId::new(), - device, - commands: Arc::new(RwLock::new(commands)), - buffers: Arc::new(RwLock::new(HashMap::new())), - kernels, - seed, - seed_value: Arc::new(RwLock::new(299792458)), - }) - } - - fn location(&self) -> crate::DeviceLocation { - crate::DeviceLocation::Metal { - gpu_id: self.registry_id() as usize, - } - } - - fn same_device(&self, rhs: &Self) -> bool { - self.id == rhs.id - } - - unsafe fn alloc_uninit(&self, shape: &Shape, dtype: DType) -> Result { - let buffer = self.new_buffer(shape.elem_count(), dtype, "alloc-uninit")?; - Ok(MetalStorage::new( - buffer, - self.clone(), - shape.elem_count(), - dtype, - )) - } - - fn zeros_impl(&self, shape: &Shape, dtype: DType) -> Result { - let size = shape.elem_count() * dtype.size_in_bytes(); - let buffer = self.allocate_zeros(size)?; - Ok(MetalStorage::new( - buffer, - self.clone(), - shape.elem_count(), - dtype, - )) - } - - fn storage_from_slice(&self, s: &[T]) -> Result { - let (count, buffer) = match T::cpu_storage_ref(s) { - CpuStorageRef::U8(storage) => (storage.len(), self.new_buffer_with_data(storage)), - CpuStorageRef::U32(storage) => (storage.len(), self.new_buffer_with_data(storage)), - CpuStorageRef::I16(storage) => (storage.len(), self.new_buffer_with_data(storage)), - CpuStorageRef::I32(storage) => (storage.len(), self.new_buffer_with_data(storage)), - CpuStorageRef::I64(storage) => (storage.len(), self.new_buffer_with_data(storage)), - CpuStorageRef::BF16(storage) => (storage.len(), self.new_buffer_with_data(storage)), - CpuStorageRef::F16(storage) => (storage.len(), self.new_buffer_with_data(storage)), - CpuStorageRef::F32(storage) => (storage.len(), self.new_buffer_with_data(storage)), - CpuStorageRef::F64(storage) => (storage.len(), self.new_buffer_with_data(storage)), - CpuStorageRef::F8E4M3(storage) => (storage.len(), self.new_buffer_with_data(storage)), - CpuStorageRef::F6E2M3(_) - | CpuStorageRef::F6E3M2(_) - | CpuStorageRef::F4(_) - | CpuStorageRef::F8E8M0(_) => { - return Err(Error::UnsupportedDTypeForOp(T::DTYPE, "to_dtype").bt()) - } - }; - Ok(Self::Storage::new(buffer?, self.clone(), count, T::DTYPE)) - } - - fn storage_from_cpu_storage(&self, storage: &CpuStorage) -> Result { - let (count, buffer) = match storage { - CpuStorage::U8(storage) => (storage.len(), self.new_buffer_with_data(storage)), - CpuStorage::U32(storage) => (storage.len(), self.new_buffer_with_data(storage)), - CpuStorage::I16(storage) => (storage.len(), self.new_buffer_with_data(storage)), - CpuStorage::I32(storage) => (storage.len(), self.new_buffer_with_data(storage)), - CpuStorage::I64(storage) => (storage.len(), self.new_buffer_with_data(storage)), - CpuStorage::BF16(storage) => (storage.len(), self.new_buffer_with_data(storage)), - CpuStorage::F16(storage) => (storage.len(), self.new_buffer_with_data(storage)), - CpuStorage::F32(storage) => (storage.len(), self.new_buffer_with_data(storage)), - CpuStorage::F64(storage) => (storage.len(), self.new_buffer_with_data(storage)), - CpuStorage::F8E4M3(storage) => (storage.len(), self.new_buffer_with_data(storage)), - CpuStorage::F6E2M3(_) - | CpuStorage::F6E3M2(_) - | CpuStorage::F4(_) - | CpuStorage::F8E8M0(_) => { - return Err(Error::UnsupportedDTypeForOp(storage.dtype(), "to_dtype").bt()) - } - }; - Ok(Self::Storage::new( - buffer?, - self.clone(), - count, - storage.dtype(), - )) - } - - fn storage_from_cpu_storage_owned(&self, storage: CpuStorage) -> Result { - self.storage_from_cpu_storage(&storage) - } - - fn rand_uniform( - &self, - shape: &Shape, - dtype: DType, - min: f64, - max: f64, - ) -> Result { - let name = match dtype { - DType::F32 => "rand_uniform_f32", - DType::F16 => "rand_uniform_f16", - DType::BF16 => "rand_uniform_bf16", - dtype => crate::bail!("rand_uniform not implemented for {dtype:?}"), - }; - let buffer = self.new_buffer(shape.elem_count(), dtype, "rand_uniform")?; - let encoder = self.command_encoder()?; - encoder.set_label("rand_uniform"); - candle_metal_kernels::call_random_uniform( - &self.device, - &encoder, - &self.kernels, - name, - min as f32, - max as f32, - shape.elem_count(), - &self.seed.lock().unwrap(), - &buffer, - ) - .map_err(MetalError::from)?; - - Ok(Self::Storage::new( - buffer, - self.clone(), - shape.elem_count(), - dtype, - )) - } - - fn rand_normal( - &self, - shape: &Shape, - dtype: DType, - mean: f64, - stddev: f64, - ) -> Result { - let name = match dtype { - DType::F32 => "rand_normal_f32", - DType::F16 => "rand_normal_f16", - DType::BF16 => "rand_normal_bf16", - dtype => crate::bail!("rand_uniform not implemented for {dtype:?}"), - }; - let buffer = self.new_buffer(shape.elem_count(), dtype, "rand_normal")?; - let encoder = self.command_encoder()?; - encoder.set_label("rand_normal"); - candle_metal_kernels::call_random_normal( - &self.device, - &encoder, - &self.kernels, - name, - mean as f32, - stddev as f32, - shape.elem_count(), - &self.seed.lock().unwrap(), - &buffer, - ) - .map_err(MetalError::from)?; - - Ok(Self::Storage::new( - buffer, - self.clone(), - shape.elem_count(), - dtype, - )) - } - - fn set_seed(&self, seed: u64) -> Result<()> { - *self.seed_value.write().unwrap() = seed; - - let seed_buffer = self.seed.try_lock().map_err(MetalError::from)?; - let contents = seed_buffer.data(); - unsafe { - std::ptr::copy_nonoverlapping([seed].as_ptr(), contents as *mut u64, 1); - } - seed_buffer.did_modify_range(NSRange::new(0, 8)); - - Ok(()) - } - - fn get_current_seed(&self) -> Result { - Ok(*self.seed_value.read().unwrap()) - } - - fn synchronize(&self) -> Result<()> { - self.wait_until_completed() - } -} - -fn read_to_vec(buffer: &Buffer, n: usize) -> Vec { - let ptr = buffer.contents() as *const T; - assert!(!ptr.is_null()); - let slice = unsafe { std::slice::from_raw_parts(ptr, n) }; - slice.to_vec() -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/mkl.rs b/v2/dependency-patches/candle-core-0.9.2/src/mkl.rs deleted file mode 100644 index 359add741c..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/mkl.rs +++ /dev/null @@ -1,418 +0,0 @@ -#![allow(dead_code)] -use libc::{c_char, c_double, c_float, c_int}; - -mod ffi { - use super::*; - extern "C" { - pub fn vsTanh(n: c_int, a: *const c_float, y: *mut c_float); - pub fn vdTanh(n: c_int, a: *const c_double, y: *mut c_double); - pub fn vsExp(n: c_int, a: *const c_float, y: *mut c_float); - pub fn vdExp(n: c_int, a: *const c_double, y: *mut c_double); - pub fn vsLn(n: c_int, a: *const c_float, y: *mut c_float); - pub fn vdLn(n: c_int, a: *const c_double, y: *mut c_double); - pub fn vsSin(n: c_int, a: *const c_float, y: *mut c_float); - pub fn vdSin(n: c_int, a: *const c_double, y: *mut c_double); - pub fn vsCos(n: c_int, a: *const c_float, y: *mut c_float); - pub fn vdCos(n: c_int, a: *const c_double, y: *mut c_double); - pub fn vsSqrt(n: c_int, a: *const c_float, y: *mut c_float); - pub fn vdSqrt(n: c_int, a: *const c_double, y: *mut c_double); - - pub fn vsAdd(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); - pub fn vdAdd(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); - pub fn vsSub(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); - pub fn vdSub(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); - pub fn vsMul(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); - pub fn vdMul(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); - pub fn vsDiv(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); - pub fn vdDiv(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); - pub fn vsFmax(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); - pub fn vdFmax(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); - pub fn vsFmin(n: c_int, a: *const c_float, b: *const c_float, y: *mut c_float); - pub fn vdFmin(n: c_int, a: *const c_double, b: *const c_double, y: *mut c_double); - - pub fn sgemm_( - transa: *const c_char, - transb: *const c_char, - m: *const c_int, - n: *const c_int, - k: *const c_int, - alpha: *const c_float, - a: *const c_float, - lda: *const c_int, - b: *const c_float, - ldb: *const c_int, - beta: *const c_float, - c: *mut c_float, - ldc: *const c_int, - ); - pub fn dgemm_( - transa: *const c_char, - transb: *const c_char, - m: *const c_int, - n: *const c_int, - k: *const c_int, - alpha: *const c_double, - a: *const c_double, - lda: *const c_int, - b: *const c_double, - ldb: *const c_int, - beta: *const c_double, - c: *mut c_double, - ldc: *const c_int, - ); - pub fn hgemm_( - transa: *const c_char, - transb: *const c_char, - m: *const c_int, - n: *const c_int, - k: *const c_int, - alpha: *const half::f16, - a: *const half::f16, - lda: *const c_int, - b: *const half::f16, - ldb: *const c_int, - beta: *const half::f16, - c: *mut half::f16, - ldc: *const c_int, - ); - } -} - -#[allow(clippy::too_many_arguments)] -#[inline] -pub unsafe fn sgemm( - transa: u8, - transb: u8, - m: i32, - n: i32, - k: i32, - alpha: f32, - a: &[f32], - lda: i32, - b: &[f32], - ldb: i32, - beta: f32, - c: &mut [f32], - ldc: i32, -) { - ffi::sgemm_( - &(transa as c_char), - &(transb as c_char), - &m, - &n, - &k, - &alpha, - a.as_ptr(), - &lda, - b.as_ptr(), - &ldb, - &beta, - c.as_mut_ptr(), - &ldc, - ) -} - -#[allow(clippy::too_many_arguments)] -#[inline] -pub unsafe fn dgemm( - transa: u8, - transb: u8, - m: i32, - n: i32, - k: i32, - alpha: f64, - a: &[f64], - lda: i32, - b: &[f64], - ldb: i32, - beta: f64, - c: &mut [f64], - ldc: i32, -) { - ffi::dgemm_( - &(transa as c_char), - &(transb as c_char), - &m, - &n, - &k, - &alpha, - a.as_ptr(), - &lda, - b.as_ptr(), - &ldb, - &beta, - c.as_mut_ptr(), - &ldc, - ) -} - -#[allow(clippy::too_many_arguments)] -#[inline] -pub unsafe fn hgemm( - transa: u8, - transb: u8, - m: i32, - n: i32, - k: i32, - alpha: half::f16, - a: &[half::f16], - lda: i32, - b: &[half::f16], - ldb: i32, - beta: half::f16, - c: &mut [half::f16], - ldc: i32, -) { - ffi::hgemm_( - &(transa as c_char), - &(transb as c_char), - &m, - &n, - &k, - &alpha, - a.as_ptr(), - &lda, - b.as_ptr(), - &ldb, - &beta, - c.as_mut_ptr(), - &ldc, - ) -} - -#[inline] -pub fn vs_exp(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vsExp(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vd_exp(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vdExp(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vs_ln(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vsLn(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vd_ln(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vdLn(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vs_sin(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vsSin(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vd_sin(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vdSin(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vs_cos(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vsCos(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vd_cos(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vdCos(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vs_sqrt(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vsSqrt(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vd_sqrt(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vdSqrt(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vs_sqr(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vsMul(a_len as i32, a.as_ptr(), a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vd_sqr(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vdMul(a_len as i32, a.as_ptr(), a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vs_tanh(a: &[f32], y: &mut [f32]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vsTanh(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vd_tanh(a: &[f64], y: &mut [f64]) { - let a_len = a.len(); - let y_len = y.len(); - if a_len != y_len { - panic!("a and y have different lengths {a_len} <> {y_len}") - } - unsafe { ffi::vdTanh(a_len as i32, a.as_ptr(), y.as_mut_ptr()) } -} - -// The vector functions from mkl can be performed in place by using the same array for input and -// output. -// https://www.intel.com/content/www/us/en/docs/onemkl/developer-reference-c/2023-2/vector-mathematical-functions.html -#[inline] -pub fn vs_tanh_inplace(y: &mut [f32]) { - unsafe { ffi::vsTanh(y.len() as i32, y.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vd_tanh_inplace(y: &mut [f64]) { - unsafe { ffi::vdTanh(y.len() as i32, y.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vs_exp_inplace(y: &mut [f32]) { - unsafe { ffi::vsExp(y.len() as i32, y.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vd_exp_inplace(y: &mut [f64]) { - unsafe { ffi::vdExp(y.len() as i32, y.as_ptr(), y.as_mut_ptr()) } -} - -#[inline] -pub fn vs_gelu(vs: &[f32], ys: &mut [f32]) { - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = (2.0f32 / std::f32::consts::PI).sqrt() * v * (1.0 + 0.044715 * v * v) - } - vs_tanh_inplace(ys); - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = 0.5 * v * (1.0 + *y) - } -} - -#[inline] -pub fn vd_gelu(vs: &[f64], ys: &mut [f64]) { - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = (2.0f64 / std::f64::consts::PI).sqrt() * v * (1.0 + 0.044715 * v * v) - } - vd_tanh_inplace(ys); - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = 0.5 * v * (1.0 + *y) - } -} - -#[inline] -pub fn vs_silu(vs: &[f32], ys: &mut [f32]) { - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = -v - } - vs_exp_inplace(ys); - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = v / (1.0 + *y) - } -} - -#[inline] -pub fn vd_silu(vs: &[f64], ys: &mut [f64]) { - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = -v - } - vd_exp_inplace(ys); - for (&v, y) in vs.iter().zip(ys.iter_mut()) { - *y = v / (1.0 + *y) - } -} - -macro_rules! binary_op { - ($fn_name:ident, $ty:ty, $mkl_name:ident) => { - #[inline] - pub fn $fn_name(a: &[$ty], b: &[$ty], y: &mut [$ty]) { - let a_len = a.len(); - let b_len = b.len(); - let y_len = y.len(); - if a_len != y_len || b_len != y_len { - panic!( - "{} a,b,y len mismatch {a_len} {b_len} {y_len}", - stringify!($fn_name) - ); - } - unsafe { ffi::$mkl_name(a_len as i32, a.as_ptr(), b.as_ptr(), y.as_mut_ptr()) } - } - }; -} -binary_op!(vs_add, f32, vsAdd); -binary_op!(vd_add, f64, vdAdd); -binary_op!(vs_sub, f32, vsSub); -binary_op!(vd_sub, f64, vdSub); -binary_op!(vs_mul, f32, vsMul); -binary_op!(vd_mul, f64, vdMul); -binary_op!(vs_div, f32, vsDiv); -binary_op!(vd_div, f64, vdDiv); -binary_op!(vs_max, f32, vsFmax); -binary_op!(vd_max, f64, vdFmax); -binary_op!(vs_min, f32, vsFmin); -binary_op!(vd_min, f64, vdFmin); diff --git a/v2/dependency-patches/candle-core-0.9.2/src/npy.rs b/v2/dependency-patches/candle-core-0.9.2/src/npy.rs deleted file mode 100644 index 6035f32734..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/npy.rs +++ /dev/null @@ -1,477 +0,0 @@ -//! Numpy support for tensors. -//! -//! The spec for the npy format can be found in -//! [npy-format](https://docs.scipy.org/doc/numpy-1.14.2/neps/npy-format.html). -//! The functions from this module can be used to read tensors from npy/npz files -//! or write tensors to these files. A npy file contains a single tensor (unnamed) -//! whereas a npz file can contain multiple named tensors. npz files are also compressed. -//! -//! These two formats are easy to use in Python using the numpy library. -//! -//! ```python -//! import numpy as np -//! x = np.arange(10) -//! -//! # Write a npy file. -//! np.save("test.npy", x) -//! -//! # Read a value from the npy file. -//! x = np.load("test.npy") -//! -//! # Write multiple values to a npz file. -//! values = { "x": x, "x_plus_one": x + 1 } -//! np.savez("test.npz", **values) -//! -//! # Load multiple values from a npz file. -//! values = np.loadz("test.npz") -//! ``` -use crate::{DType, Device, Error, Result, Shape, Tensor}; -use byteorder::{LittleEndian, ReadBytesExt}; -use half::{bf16, f16, slice::HalfFloatSliceExt}; -use std::collections::HashMap; -use std::fs::File; -use std::io::{BufReader, Read, Write}; -use std::path::Path; - -const NPY_MAGIC_STRING: &[u8] = b"\x93NUMPY"; -const NPY_SUFFIX: &str = ".npy"; - -fn read_header(reader: &mut R) -> Result { - let mut magic_string = vec![0u8; NPY_MAGIC_STRING.len()]; - reader.read_exact(&mut magic_string)?; - if magic_string != NPY_MAGIC_STRING { - return Err(Error::Npy("magic string mismatch".to_string())); - } - let mut version = [0u8; 2]; - reader.read_exact(&mut version)?; - let header_len_len = match version[0] { - 1 => 2, - 2 => 4, - otherwise => return Err(Error::Npy(format!("unsupported version {otherwise}"))), - }; - let mut header_len = vec![0u8; header_len_len]; - reader.read_exact(&mut header_len)?; - let header_len = header_len - .iter() - .rev() - .fold(0_usize, |acc, &v| 256 * acc + v as usize); - let mut header = vec![0u8; header_len]; - reader.read_exact(&mut header)?; - Ok(String::from_utf8_lossy(&header).to_string()) -} - -#[derive(Debug, PartialEq)] -struct Header { - descr: DType, - fortran_order: bool, - shape: Vec, -} - -impl Header { - fn shape(&self) -> Shape { - Shape::from(self.shape.as_slice()) - } - - fn to_string(&self) -> Result { - let fortran_order = if self.fortran_order { "True" } else { "False" }; - let mut shape = self - .shape - .iter() - .map(|x| x.to_string()) - .collect::>() - .join(","); - let descr = match self.descr { - DType::BF16 => Err(Error::Npy("bf16 is not supported".into()))?, - DType::F16 => "f2", - DType::F32 => "f4", - DType::F64 => "f8", - DType::I16 => "i2", - DType::I32 => "i4", - DType::I64 => "i8", - DType::U32 => "u4", - DType::U8 => "u1", - DType::F8E4M3 => Err(Error::Npy("f8e4m3 is not supported".into()))?, - DType::F6E2M3 => Err(Error::Npy("f6e2m3 is not supported".into()))?, - DType::F6E3M2 => Err(Error::Npy("f6e3m2 is not supported".into()))?, - DType::F4 => Err(Error::Npy("f4 is not supported".into()))?, - DType::F8E8M0 => Err(Error::Npy("f8e8m0 is not supported".into()))?, - }; - if !shape.is_empty() { - shape.push(',') - } - Ok(format!( - "{{'descr': '<{descr}', 'fortran_order': {fortran_order}, 'shape': ({shape}), }}" - )) - } - - // Hacky parser for the npy header, a typical example would be: - // {'descr': ' Result
{ - let header = - header.trim_matches(|c: char| c == '{' || c == '}' || c == ',' || c.is_whitespace()); - - let mut parts: Vec = vec![]; - let mut start_index = 0usize; - let mut cnt_parenthesis = 0i64; - for (index, c) in header.char_indices() { - match c { - '(' => cnt_parenthesis += 1, - ')' => cnt_parenthesis -= 1, - ',' => { - if cnt_parenthesis == 0 { - parts.push(header[start_index..index].to_owned()); - start_index = index + 1; - } - } - _ => {} - } - } - parts.push(header[start_index..].to_owned()); - let mut part_map: HashMap = HashMap::new(); - for part in parts.iter() { - let part = part.trim(); - if !part.is_empty() { - match part.split(':').collect::>().as_slice() { - [key, value] => { - let key = key.trim_matches(|c: char| c == '\'' || c.is_whitespace()); - let value = value.trim_matches(|c: char| c == '\'' || c.is_whitespace()); - let _ = part_map.insert(key.to_owned(), value.to_owned()); - } - _ => return Err(Error::Npy(format!("unable to parse header {header}"))), - } - } - } - let fortran_order = match part_map.get("fortran_order") { - None => false, - Some(fortran_order) => match fortran_order.as_ref() { - "False" => false, - "True" => true, - _ => return Err(Error::Npy(format!("unknown fortran_order {fortran_order}"))), - }, - }; - let descr = match part_map.get("descr") { - None => return Err(Error::Npy("no descr in header".to_string())), - Some(descr) => { - if descr.is_empty() { - return Err(Error::Npy("empty descr".to_string())); - } - if descr.starts_with('>') { - return Err(Error::Npy(format!("little-endian descr {descr}"))); - } - // the only supported types in tensor are: - // float64, float32, float16, - // complex64, complex128, - // int64, int32, int16, int8, - // uint8, and bool. - match descr.trim_matches(|c: char| c == '=' || c == '<' || c == '|') { - "e" | "f2" => DType::F16, - "f" | "f4" => DType::F32, - "d" | "f8" => DType::F64, - "i" | "i4" => DType::I32, - "q" | "i8" => DType::I64, - "h" | "i2" => DType::I16, - // "b" | "i1" => DType::S8, - "B" | "u1" => DType::U8, - "I" | "u4" => DType::U32, - "?" | "b1" => DType::U8, - // "F" | "F4" => DType::C64, - // "D" | "F8" => DType::C128, - descr => return Err(Error::Npy(format!("unrecognized descr {descr}"))), - } - } - }; - let shape = match part_map.get("shape") { - None => return Err(Error::Npy("no shape in header".to_string())), - Some(shape) => { - let shape = shape.trim_matches(|c: char| c == '(' || c == ')' || c == ','); - if shape.is_empty() { - vec![] - } else { - shape - .split(',') - .map(|v| v.trim().parse::()) - .collect::, _>>()? - } - } - }; - Ok(Header { - descr, - fortran_order, - shape, - }) - } -} - -impl Tensor { - // TODO: Add the possibility to read directly to a device? - pub(crate) fn from_reader( - shape: Shape, - dtype: DType, - reader: &mut R, - ) -> Result { - let elem_count = shape.elem_count(); - match dtype { - DType::BF16 => { - let mut data_t = vec![bf16::ZERO; elem_count]; - reader.read_u16_into::(data_t.reinterpret_cast_mut())?; - Tensor::from_vec(data_t, shape, &Device::Cpu) - } - DType::F16 => { - let mut data_t = vec![f16::ZERO; elem_count]; - reader.read_u16_into::(data_t.reinterpret_cast_mut())?; - Tensor::from_vec(data_t, shape, &Device::Cpu) - } - DType::F32 => { - let mut data_t = vec![0f32; elem_count]; - reader.read_f32_into::(&mut data_t)?; - Tensor::from_vec(data_t, shape, &Device::Cpu) - } - DType::F64 => { - let mut data_t = vec![0f64; elem_count]; - reader.read_f64_into::(&mut data_t)?; - Tensor::from_vec(data_t, shape, &Device::Cpu) - } - DType::U8 => { - let mut data_t = vec![0u8; elem_count]; - reader.read_exact(&mut data_t)?; - Tensor::from_vec(data_t, shape, &Device::Cpu) - } - DType::U32 => { - let mut data_t = vec![0u32; elem_count]; - reader.read_u32_into::(&mut data_t)?; - Tensor::from_vec(data_t, shape, &Device::Cpu) - } - DType::I16 => { - let mut data_t = vec![0i16; elem_count]; - reader.read_i16_into::(&mut data_t)?; - Tensor::from_vec(data_t, shape, &Device::Cpu) - } - DType::I32 => { - let mut data_t = vec![0i32; elem_count]; - reader.read_i32_into::(&mut data_t)?; - Tensor::from_vec(data_t, shape, &Device::Cpu) - } - DType::I64 => { - let mut data_t = vec![0i64; elem_count]; - reader.read_i64_into::(&mut data_t)?; - Tensor::from_vec(data_t, shape, &Device::Cpu) - } - DType::F8E4M3 => { - let mut data_t = vec![0u8; elem_count]; - reader.read_exact(&mut data_t)?; - let data_f8: Vec = - data_t.into_iter().map(float8::F8E4M3::from_bits).collect(); - Tensor::from_vec(data_f8, shape, &Device::Cpu) - } - DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { - Err(Error::UnsupportedDTypeForOp(dtype, "from_reader").bt()) - } - } - } - - /// Reads a npy file and return the stored multi-dimensional array as a tensor. - pub fn read_npy>(path: T) -> Result { - let mut reader = File::open(path.as_ref())?; - let header = read_header(&mut reader)?; - let header = Header::parse(&header)?; - if header.fortran_order { - return Err(Error::Npy("fortran order not supported".to_string())); - } - Self::from_reader(header.shape(), header.descr, &mut reader) - } - - /// Reads a npz file and returns the stored multi-dimensional arrays together with their names. - pub fn read_npz>(path: T) -> Result> { - let zip_reader = BufReader::new(File::open(path.as_ref())?); - let mut zip = zip::ZipArchive::new(zip_reader)?; - let mut result = vec![]; - for i in 0..zip.len() { - let mut reader = zip.by_index(i)?; - let name = { - let name = reader.name()?; - name.strip_suffix(NPY_SUFFIX).unwrap_or(&name).to_owned() - }; - let header = read_header(&mut reader)?; - let header = Header::parse(&header)?; - if header.fortran_order { - return Err(Error::Npy("fortran order not supported".to_string())); - } - let s = Self::from_reader(header.shape(), header.descr, &mut reader)?; - result.push((name, s)) - } - Ok(result) - } - - /// Reads a npz file and returns the stored multi-dimensional arrays for some specified names. - pub fn read_npz_by_name>(path: T, names: &[&str]) -> Result> { - let zip_reader = BufReader::new(File::open(path.as_ref())?); - let mut zip = zip::ZipArchive::new(zip_reader)?; - let mut result = vec![]; - for name in names.iter() { - let mut reader = match zip.by_name(&format!("{name}{NPY_SUFFIX}")) { - Ok(reader) => reader, - Err(_) => Err(Error::Npy(format!( - "no array for {name} in {:?}", - path.as_ref() - )))?, - }; - let header = read_header(&mut reader)?; - let header = Header::parse(&header)?; - if header.fortran_order { - return Err(Error::Npy("fortran order not supported".to_string())); - } - let s = Self::from_reader(header.shape(), header.descr, &mut reader)?; - result.push(s) - } - Ok(result) - } - - fn write(&self, f: &mut T) -> Result<()> { - f.write_all(NPY_MAGIC_STRING)?; - f.write_all(&[1u8, 0u8])?; - let header = Header { - descr: self.dtype(), - fortran_order: false, - shape: self.dims().to_vec(), - }; - let mut header = header.to_string()?; - let pad = 16 - (NPY_MAGIC_STRING.len() + 5 + header.len()) % 16; - for _ in 0..pad % 16 { - header.push(' ') - } - header.push('\n'); - f.write_all(&[(header.len() % 256) as u8, (header.len() / 256) as u8])?; - f.write_all(header.as_bytes())?; - self.write_bytes(f) - } - - /// Writes a multi-dimensional array in the npy format. - pub fn write_npy>(&self, path: T) -> Result<()> { - let mut f = File::create(path.as_ref())?; - self.write(&mut f) - } - - /// Writes multiple multi-dimensional arrays using the npz format. - pub fn write_npz, T: AsRef, P: AsRef>( - ts: &[(S, T)], - path: P, - ) -> Result<()> { - let mut zip = zip::ZipWriter::new(File::create(path.as_ref())?); - let options: zip::write::FileOptions<()> = - zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); - - for (name, tensor) in ts.iter() { - zip.start_file(format!("{}.npy", name.as_ref()), options)?; - tensor.as_ref().write(&mut zip)? - } - Ok(()) - } -} - -/// Lazy tensor loader. -pub struct NpzTensors { - index_per_name: HashMap, - path: std::path::PathBuf, - // We do not store a zip reader as it needs mutable access to extract data. Instead we - // re-create a zip reader for each tensor. -} - -impl NpzTensors { - pub fn new>(path: T) -> Result { - let path = path.as_ref().to_owned(); - let zip_reader = BufReader::new(File::open(&path)?); - let mut zip = zip::ZipArchive::new(zip_reader)?; - let mut index_per_name = HashMap::new(); - for i in 0..zip.len() { - let file = zip.by_index(i)?; - let name = { - let name = file.name()?; - name.strip_suffix(NPY_SUFFIX).unwrap_or(&name).to_owned() - }; - index_per_name.insert(name, i); - } - Ok(Self { - index_per_name, - path, - }) - } - - pub fn names(&self) -> Vec<&String> { - self.index_per_name.keys().collect() - } - - /// This only returns the shape and dtype for a named tensor. Compared to `get`, this avoids - /// reading the whole tensor data. - pub fn get_shape_and_dtype(&self, name: &str) -> Result<(Shape, DType)> { - let index = match self.index_per_name.get(name) { - None => crate::bail!("cannot find tensor {name}"), - Some(index) => *index, - }; - let zip_reader = BufReader::new(File::open(&self.path)?); - let mut zip = zip::ZipArchive::new(zip_reader)?; - let mut reader = zip.by_index(index)?; - let header = read_header(&mut reader)?; - let header = Header::parse(&header)?; - Ok((header.shape(), header.descr)) - } - - pub fn get(&self, name: &str) -> Result> { - let index = match self.index_per_name.get(name) { - None => return Ok(None), - Some(index) => *index, - }; - // We hope that the file has not changed since first reading it. - let zip_reader = BufReader::new(File::open(&self.path)?); - let mut zip = zip::ZipArchive::new(zip_reader)?; - let mut reader = zip.by_index(index)?; - let header = read_header(&mut reader)?; - let header = Header::parse(&header)?; - if header.fortran_order { - return Err(Error::Npy("fortran order not supported".to_string())); - } - let tensor = Tensor::from_reader(header.shape(), header.descr, &mut reader)?; - Ok(Some(tensor)) - } -} - -#[cfg(test)] -mod tests { - use super::Header; - - #[test] - fn parse() { - let h = "{'descr': ' &'static str { - match self { - Self::ArgMax => "argmax", - Self::ArgMin => "argmin", - Self::Min => "min", - Self::Max => "max", - Self::Sum => "sum", - } - } -} - -// These ops return the same type as their input type. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum BinaryOp { - Add, - Mul, - Sub, - Div, - Maximum, - Minimum, -} - -// Unary ops with no argument -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum UnaryOp { - Exp, - Log, - Sin, - Cos, - Abs, - Neg, - Recip, - Sqr, - Sqrt, - Gelu, - GeluErf, - Erf, - Relu, - Silu, - Tanh, - Floor, - Ceil, - Round, - Sign, -} - -#[derive(Clone)] -pub enum Op { - Binary(Tensor, Tensor, BinaryOp), - Unary(Tensor, UnaryOp), - Cmp(Tensor, CmpOp), - // The third argument is the reduced shape with `keepdim=true`. - Reduce(Tensor, ReduceOp, Vec), - Matmul(Tensor, Tensor), - Gather(Tensor, Tensor, usize), - Scatter(Tensor, Tensor, Tensor, usize), - ScatterAdd(Tensor, Tensor, Tensor, usize), - IndexSelect(Tensor, Tensor, usize), - IndexAdd(Tensor, Tensor, Tensor, usize), - WhereCond(Tensor, Tensor, Tensor), - - #[allow(dead_code)] - Conv1D { - arg: Tensor, - kernel: Tensor, - padding: usize, - stride: usize, - dilation: usize, - }, - - #[allow(dead_code)] - ConvTranspose1D { - arg: Tensor, - kernel: Tensor, - padding: usize, - output_padding: usize, - stride: usize, - dilation: usize, - }, - - #[allow(dead_code)] - Conv2D { - arg: Tensor, - kernel: Tensor, - padding: usize, - stride: usize, - dilation: usize, - }, - - #[allow(dead_code)] - ConvTranspose2D { - arg: Tensor, - kernel: Tensor, - padding: usize, - output_padding: usize, - stride: usize, - dilation: usize, - }, - - AvgPool2D { - arg: Tensor, - kernel_size: (usize, usize), - stride: (usize, usize), - }, - - MaxPool2D { - arg: Tensor, - kernel_size: (usize, usize), - stride: (usize, usize), - }, - - UpsampleNearest1D { - arg: Tensor, - target_size: usize, - }, - UpsampleNearest2D { - arg: Tensor, - target_h: usize, - target_w: usize, - }, - UpsampleBilinear2D { - arg: Tensor, - target_h: usize, - target_w: usize, - align_corners: bool, - }, - - Cat(Vec, usize), - - #[allow(dead_code)] // add is currently unused. - Affine { - arg: Tensor, - mul: f64, - add: f64, - }, - ToDType(Tensor), - Copy(Tensor), - Broadcast(Tensor), - Narrow(Tensor, usize, usize, usize), - SliceScatter0(Tensor, Tensor, usize), - Reshape(Tensor), - ToDevice(Tensor), - Transpose(Tensor, usize, usize), - Permute(Tensor, Vec), - Elu(Tensor, f64), - Powf(Tensor, f64), - CustomOp1( - Tensor, - std::sync::Arc>, - ), - CustomOp2( - Tensor, - Tensor, - std::sync::Arc>, - ), - CustomOp3( - Tensor, - Tensor, - Tensor, - std::sync::Arc>, - ), -} - -pub trait UnaryOpT { - const NAME: &'static str; - const KERNEL: &'static str; - const V: Self; - fn bf16(v1: bf16) -> bf16; - fn f16(v1: f16) -> f16; - fn f32(v1: f32) -> f32; - fn f64(v1: f64) -> f64; - fn u8(v1: u8) -> u8; - fn u32(v1: u32) -> u32; - fn i16(v1: i16) -> i16; - fn i32(v1: i32) -> i32; - fn i64(v1: i64) -> i64; - fn f8e4m3(v1: f8e4m3) -> f8e4m3; - - // There is no very good way to represent optional function in traits so we go for an explicit - // boolean flag to mark the function as existing. - const BF16_VEC: bool = false; - fn bf16_vec(_xs: &[bf16], _ys: &mut [bf16]) {} - const F16_VEC: bool = false; - fn f16_vec(_xs: &[f16], _ys: &mut [f16]) {} - const F32_VEC: bool = false; - fn f32_vec(_xs: &[f32], _ys: &mut [f32]) {} - const F64_VEC: bool = false; - fn f64_vec(_xs: &[f64], _ys: &mut [f64]) {} -} - -pub trait BinaryOpT { - const NAME: &'static str; - const KERNEL: &'static str; - const V: Self; - fn bf16(v1: bf16, v2: bf16) -> bf16; - fn f16(v1: f16, v2: f16) -> f16; - fn f32(v1: f32, v2: f32) -> f32; - fn f64(v1: f64, v2: f64) -> f64; - fn u8(v1: u8, v2: u8) -> u8; - fn u32(v1: u32, v2: u32) -> u32; - fn i16(v1: i16, v2: i16) -> i16; - fn i32(v1: i32, v2: i32) -> i32; - fn i64(v1: i64, v2: i64) -> i64; - fn f8e4m3(v1: f8e4m3, v2: f8e4m3) -> f8e4m3; - - const BF16_VEC: bool = false; - fn bf16_vec(_xs1: &[bf16], _xs2: &[bf16], _ys: &mut [bf16]) {} - const F16_VEC: bool = false; - fn f16_vec(_xs1: &[f16], _xs2: &[f16], _ys: &mut [f16]) {} - const F32_VEC: bool = false; - fn f32_vec(_xs1: &[f32], _xs2: &[f32], _ys: &mut [f32]) {} - const F64_VEC: bool = false; - fn f64_vec(_xs1: &[f64], _xs2: &[f64], _ys: &mut [f64]) {} - const U8_VEC: bool = false; - fn u8_vec(_xs1: &[u8], _xs2: &[u8], _ys: &mut [u8]) {} - const U32_VEC: bool = false; - fn u32_vec(_xs1: &[u32], _xs2: &[u32], _ys: &mut [u32]) {} - const I64_VEC: bool = false; - fn i64_vec(_xs1: &[i64], _xs2: &[i64], _ys: &mut [i64]) {} -} - -pub struct Add; -pub struct Div; -pub struct Mul; -pub struct Sub; -pub struct Maximum; -pub struct Minimum; -pub struct Exp; -pub struct Log; -pub struct Sin; -pub struct Cos; -pub struct Abs; -pub struct Neg; -pub struct Recip; -pub struct Sqr; -pub struct Sqrt; -pub struct Gelu; -pub struct GeluErf; -pub struct Erf; -pub struct Relu; -pub struct Silu; -pub struct Tanh; -pub struct Floor; -pub struct Ceil; -pub struct Round; -pub struct Sign; - -macro_rules! bin_op { - ($op:ident, $name: literal, $e: expr, $f32_vec: ident, $f64_vec: ident) => { - impl BinaryOpT for $op { - const NAME: &'static str = $name; - const KERNEL: &'static str = concat!("b", $name); - const V: Self = $op; - #[inline(always)] - fn bf16(v1: bf16, v2: bf16) -> bf16 { - $e(v1, v2) - } - #[inline(always)] - fn f16(v1: f16, v2: f16) -> f16 { - $e(v1, v2) - } - #[inline(always)] - fn f32(v1: f32, v2: f32) -> f32 { - $e(v1, v2) - } - #[inline(always)] - fn f64(v1: f64, v2: f64) -> f64 { - $e(v1, v2) - } - #[inline(always)] - fn u8(v1: u8, v2: u8) -> u8 { - $e(v1, v2) - } - #[inline(always)] - fn u32(v1: u32, v2: u32) -> u32 { - $e(v1, v2) - } - #[inline(always)] - fn i16(v1: i16, v2: i16) -> i16 { - $e(v1, v2) - } - #[inline(always)] - fn i32(v1: i32, v2: i32) -> i32 { - $e(v1, v2) - } - #[inline(always)] - fn i64(v1: i64, v2: i64) -> i64 { - $e(v1, v2) - } - #[inline(always)] - fn f8e4m3(v1: f8e4m3, v2: f8e4m3) -> f8e4m3 { - $e(v1, v2) - } - - #[cfg(feature = "mkl")] - const F32_VEC: bool = true; - #[cfg(feature = "mkl")] - const F64_VEC: bool = true; - #[cfg(feature = "mkl")] - #[inline(always)] - fn f32_vec(xs1: &[f32], xs2: &[f32], ys: &mut [f32]) { - crate::mkl::$f32_vec(xs1, xs2, ys) - } - #[cfg(feature = "mkl")] - #[inline(always)] - fn f64_vec(xs1: &[f64], xs2: &[f64], ys: &mut [f64]) { - crate::mkl::$f64_vec(xs1, xs2, ys) - } - - #[cfg(feature = "accelerate")] - const F32_VEC: bool = true; - #[cfg(feature = "accelerate")] - const F64_VEC: bool = true; - #[cfg(feature = "accelerate")] - #[inline(always)] - fn f32_vec(xs1: &[f32], xs2: &[f32], ys: &mut [f32]) { - crate::accelerate::$f32_vec(xs1, xs2, ys) - } - #[cfg(feature = "accelerate")] - #[inline(always)] - fn f64_vec(xs1: &[f64], xs2: &[f64], ys: &mut [f64]) { - crate::accelerate::$f64_vec(xs1, xs2, ys) - } - } - }; -} - -bin_op!(Add, "add", |v1, v2| v1 + v2, vs_add, vd_add); -bin_op!(Sub, "sub", |v1, v2| v1 - v2, vs_sub, vd_sub); -bin_op!(Mul, "mul", |v1, v2| v1 * v2, vs_mul, vd_mul); -bin_op!(Div, "div", |v1, v2| v1 / v2, vs_div, vd_div); -bin_op!( - Minimum, - "minimum", - |v1, v2| if v1 > v2 { v2 } else { v1 }, - vs_min, - vd_min -); -bin_op!( - Maximum, - "maximum", - |v1, v2| if v1 < v2 { v2 } else { v1 }, - vs_max, - vd_max -); - -#[allow(clippy::redundant_closure_call)] -macro_rules! unary_op { - ($op: ident, $name: literal, $a: ident, $e: expr) => { - impl UnaryOpT for $op { - const NAME: &'static str = $name; - const KERNEL: &'static str = concat!("u", $name); - const V: Self = $op; - #[inline(always)] - fn bf16($a: bf16) -> bf16 { - $e - } - #[inline(always)] - fn f16($a: f16) -> f16 { - $e - } - #[inline(always)] - fn f32($a: f32) -> f32 { - $e - } - #[inline(always)] - fn f64($a: f64) -> f64 { - $e - } - #[inline(always)] - fn u8(_: u8) -> u8 { - todo!("no unary function for u8") - } - #[inline(always)] - fn u32(_: u32) -> u32 { - todo!("no unary function for u32") - } - #[inline(always)] - fn i16(_: i16) -> i16 { - todo!("no unary function for i16") - } - #[inline(always)] - fn i32(_: i32) -> i32 { - todo!("no unary function for i32") - } - #[inline(always)] - fn i64(_: i64) -> i64 { - todo!("no unary function for i64") - } - #[inline(always)] - fn f8e4m3($a: f8e4m3) -> f8e4m3 { - $e - } - } - }; - - ($op: ident, $name: literal, $a: ident, $e: expr, $f32_vec:ident, $f64_vec:ident) => { - impl UnaryOpT for $op { - const NAME: &'static str = $name; - const KERNEL: &'static str = concat!("u", $name); - const V: Self = $op; - #[inline(always)] - fn bf16($a: bf16) -> bf16 { - $e - } - #[inline(always)] - fn f16($a: f16) -> f16 { - $e - } - #[inline(always)] - fn f32($a: f32) -> f32 { - $e - } - #[inline(always)] - fn f64($a: f64) -> f64 { - $e - } - #[inline(always)] - fn u8(_: u8) -> u8 { - todo!("no unary function for u8") - } - #[inline(always)] - fn u32(_: u32) -> u32 { - todo!("no unary function for u32") - } - #[inline(always)] - fn i16(_: i16) -> i16 { - todo!("no unary function for i16") - } - #[inline(always)] - fn i32(_: i32) -> i32 { - todo!("no unary function for i32") - } - #[inline(always)] - fn i64(_: i64) -> i64 { - todo!("no unary function for i64") - } - #[inline(always)] - fn f8e4m3($a: f8e4m3) -> f8e4m3 { - $e - } - - #[cfg(feature = "mkl")] - const F32_VEC: bool = true; - #[cfg(feature = "mkl")] - const F64_VEC: bool = true; - #[cfg(feature = "mkl")] - #[inline(always)] - fn f32_vec(xs: &[f32], ys: &mut [f32]) { - crate::mkl::$f32_vec(xs, ys) - } - #[cfg(feature = "mkl")] - #[inline(always)] - fn f64_vec(xs: &[f64], ys: &mut [f64]) { - crate::mkl::$f64_vec(xs, ys) - } - - #[cfg(feature = "accelerate")] - const F32_VEC: bool = true; - #[cfg(feature = "accelerate")] - const F64_VEC: bool = true; - #[cfg(feature = "accelerate")] - #[inline(always)] - fn f32_vec(xs: &[f32], ys: &mut [f32]) { - crate::accelerate::$f32_vec(xs, ys) - } - #[cfg(feature = "accelerate")] - #[inline(always)] - fn f64_vec(xs: &[f64], ys: &mut [f64]) { - crate::accelerate::$f64_vec(xs, ys) - } - } - }; -} - -unary_op!(Exp, "exp", v, v.exp(), vs_exp, vd_exp); -unary_op!(Log, "log", v, v.ln(), vs_ln, vd_ln); -unary_op!(Sin, "sin", v, v.sin(), vs_sin, vd_sin); -unary_op!(Cos, "cos", v, v.cos(), vs_cos, vd_cos); -unary_op!(Tanh, "tanh", v, v.tanh(), vs_tanh, vd_tanh); -unary_op!(Neg, "neg", v, -v); -unary_op!(Recip, "recip", v, v.recip()); -unary_op!(Sqr, "sqr", v, v * v, vs_sqr, vd_sqr); -unary_op!(Sqrt, "sqrt", v, v.sqrt(), vs_sqrt, vd_sqrt); - -// Hardcode the value for sqrt(2/pi) -// https://github.com/huggingface/candle/issues/1982 -#[allow(clippy::excessive_precision)] -const SQRT_TWO_OVER_PI_F32: f32 = 0.79788456080286535587989211986876373; -#[allow(clippy::excessive_precision)] -const SQRT_TWO_OVER_PI_F64: f64 = 0.79788456080286535587989211986876373; - -/// Tanh based approximation of the `gelu` operation -/// GeluErf is the more precise one. -/// -impl UnaryOpT for Gelu { - const NAME: &'static str = "gelu"; - const V: Self = Gelu; - #[inline(always)] - fn bf16(v: bf16) -> bf16 { - bf16::from_f32_const(0.5) - * v - * (bf16::ONE - + bf16::tanh( - bf16::from_f32_const(SQRT_TWO_OVER_PI_F32) - * v - * (bf16::ONE + bf16::from_f32_const(0.044715) * v * v), - )) - } - #[inline(always)] - fn f16(v: f16) -> f16 { - f16::from_f32_const(0.5) - * v - * (f16::ONE - + f16::tanh( - f16::from_f32_const(SQRT_TWO_OVER_PI_F32) - * v - * (f16::ONE + f16::from_f32_const(0.044715) * v * v), - )) - } - #[inline(always)] - fn f32(v: f32) -> f32 { - 0.5 * v * (1.0 + f32::tanh(SQRT_TWO_OVER_PI_F32 * v * (1.0 + 0.044715 * v * v))) - } - #[inline(always)] - fn f64(v: f64) -> f64 { - 0.5 * v * (1.0 + f64::tanh(SQRT_TWO_OVER_PI_F64 * v * (1.0 + 0.044715 * v * v))) - } - #[inline(always)] - fn u8(_: u8) -> u8 { - 0 - } - #[inline(always)] - fn u32(_: u32) -> u32 { - 0 - } - #[inline(always)] - fn i16(_: i16) -> i16 { - 0 - } - #[inline(always)] - fn i32(_: i32) -> i32 { - 0 - } - #[inline(always)] - fn i64(_: i64) -> i64 { - 0 - } - #[inline(always)] - fn f8e4m3(v: f8e4m3) -> f8e4m3 { - f8e4m3::from_f32(0.5) - * v - * (f8e4m3::ONE - + f8e4m3::tanh( - f8e4m3::from_f32(SQRT_TWO_OVER_PI_F32) - * v - * (f8e4m3::ONE + f8e4m3::from_f32(0.044715) * v * v), - )) - } - const KERNEL: &'static str = "ugelu"; - - #[cfg(feature = "mkl")] - const F32_VEC: bool = true; - - #[cfg(feature = "mkl")] - #[inline(always)] - fn f32_vec(xs: &[f32], ys: &mut [f32]) { - crate::mkl::vs_gelu(xs, ys) - } - - #[cfg(feature = "mkl")] - const F64_VEC: bool = true; - - #[cfg(feature = "mkl")] - #[inline(always)] - fn f64_vec(xs: &[f64], ys: &mut [f64]) { - crate::mkl::vd_gelu(xs, ys) - } - - #[cfg(feature = "accelerate")] - const F32_VEC: bool = true; - - #[cfg(feature = "accelerate")] - #[inline(always)] - fn f32_vec(xs: &[f32], ys: &mut [f32]) { - crate::accelerate::vs_gelu(xs, ys) - } - - #[cfg(feature = "accelerate")] - const F64_VEC: bool = true; - - #[cfg(feature = "accelerate")] - #[inline(always)] - fn f64_vec(xs: &[f64], ys: &mut [f64]) { - crate::accelerate::vd_gelu(xs, ys) - } -} - -/// `erf` operation -/// -impl UnaryOpT for Erf { - const NAME: &'static str = "erf"; - const KERNEL: &'static str = "uerf"; - const V: Self = Erf; - #[inline(always)] - fn bf16(v: bf16) -> bf16 { - bf16::from_f64(Self::f64(v.to_f64())) - } - #[inline(always)] - fn f16(v: f16) -> f16 { - f16::from_f64(Self::f64(v.to_f64())) - } - #[inline(always)] - fn f32(v: f32) -> f32 { - crate::cpu::erf::erf_f32(v) - } - #[inline(always)] - fn f64(v: f64) -> f64 { - crate::cpu::erf::erf_f64(v) - } - #[inline(always)] - fn u8(_: u8) -> u8 { - 0 - } - #[inline(always)] - fn u32(_: u32) -> u32 { - 0 - } - #[inline(always)] - fn i16(_: i16) -> i16 { - 0 - } - #[inline(always)] - fn i32(_: i32) -> i32 { - 0 - } - #[inline(always)] - fn i64(_: i64) -> i64 { - 0 - } - #[inline(always)] - fn f8e4m3(v: f8e4m3) -> f8e4m3 { - f8e4m3::from_f64(Self::f64(v.to_f64())) - } -} - -/// Silu operation -impl UnaryOpT for Silu { - const NAME: &'static str = "silu"; - const V: Self = Silu; - #[inline(always)] - fn bf16(v: bf16) -> bf16 { - v / (bf16::ONE + (-v).exp()) - } - #[inline(always)] - fn f16(v: f16) -> f16 { - v / (f16::ONE + (-v).exp()) - } - #[inline(always)] - fn f32(v: f32) -> f32 { - v / (1.0 + (-v).exp()) - } - #[inline(always)] - fn f64(v: f64) -> f64 { - v / (1.0 + (-v).exp()) - } - #[inline(always)] - fn u8(_: u8) -> u8 { - 0 - } - #[inline(always)] - fn u32(_: u32) -> u32 { - 0 - } - #[inline(always)] - fn i16(_: i16) -> i16 { - 0 - } - #[inline(always)] - fn i32(_: i32) -> i32 { - 0 - } - #[inline(always)] - fn i64(_: i64) -> i64 { - 0 - } - #[inline(always)] - fn f8e4m3(v: f8e4m3) -> f8e4m3 { - v / (f8e4m3::ONE + (-v).exp()) - } - const KERNEL: &'static str = "usilu"; - - #[cfg(feature = "mkl")] - const F32_VEC: bool = true; - - #[cfg(feature = "mkl")] - #[inline(always)] - fn f32_vec(xs: &[f32], ys: &mut [f32]) { - crate::mkl::vs_silu(xs, ys) - } - - #[cfg(feature = "mkl")] - const F64_VEC: bool = true; - - #[cfg(feature = "mkl")] - #[inline(always)] - fn f64_vec(xs: &[f64], ys: &mut [f64]) { - crate::mkl::vd_silu(xs, ys) - } - - #[cfg(feature = "accelerate")] - const F32_VEC: bool = true; - - #[cfg(feature = "accelerate")] - #[inline(always)] - fn f32_vec(xs: &[f32], ys: &mut [f32]) { - crate::accelerate::vs_silu(xs, ys) - } - - #[cfg(feature = "accelerate")] - const F64_VEC: bool = true; - - #[cfg(feature = "accelerate")] - #[inline(always)] - fn f64_vec(xs: &[f64], ys: &mut [f64]) { - crate::accelerate::vd_silu(xs, ys) - } -} - -impl UnaryOpT for Abs { - const NAME: &'static str = "abs"; - const KERNEL: &'static str = "uabs"; - const V: Self = Abs; - #[inline(always)] - fn bf16(v: bf16) -> bf16 { - v.abs() - } - #[inline(always)] - fn f16(v: f16) -> f16 { - v.abs() - } - #[inline(always)] - fn f32(v: f32) -> f32 { - v.abs() - } - #[inline(always)] - fn f64(v: f64) -> f64 { - v.abs() - } - #[inline(always)] - fn u8(v: u8) -> u8 { - v - } - #[inline(always)] - fn u32(v: u32) -> u32 { - v - } - #[inline(always)] - fn i16(v: i16) -> i16 { - v.abs() - } - #[inline(always)] - fn i32(v: i32) -> i32 { - v.abs() - } - #[inline(always)] - fn i64(v: i64) -> i64 { - v.abs() - } - #[inline(always)] - fn f8e4m3(v: f8e4m3) -> f8e4m3 { - v.abs() - } -} - -impl UnaryOpT for Ceil { - const NAME: &'static str = "ceil"; - const KERNEL: &'static str = "uceil"; - const V: Self = Ceil; - #[inline(always)] - fn bf16(v: bf16) -> bf16 { - v.ceil() - } - #[inline(always)] - fn f16(v: f16) -> f16 { - v.ceil() - } - #[inline(always)] - fn f32(v: f32) -> f32 { - v.ceil() - } - #[inline(always)] - fn f64(v: f64) -> f64 { - v.ceil() - } - #[inline(always)] - fn u8(v: u8) -> u8 { - v - } - #[inline(always)] - fn u32(v: u32) -> u32 { - v - } - #[inline(always)] - fn i16(v: i16) -> i16 { - v - } - #[inline(always)] - fn i32(v: i32) -> i32 { - v - } - #[inline(always)] - fn i64(v: i64) -> i64 { - v - } - #[inline(always)] - fn f8e4m3(v: f8e4m3) -> f8e4m3 { - v.ceil() - } -} - -impl UnaryOpT for Floor { - const NAME: &'static str = "floor"; - const KERNEL: &'static str = "ufloor"; - const V: Self = Floor; - #[inline(always)] - fn bf16(v: bf16) -> bf16 { - v.floor() - } - #[inline(always)] - fn f16(v: f16) -> f16 { - v.floor() - } - #[inline(always)] - fn f32(v: f32) -> f32 { - v.floor() - } - #[inline(always)] - fn f64(v: f64) -> f64 { - v.floor() - } - #[inline(always)] - fn u8(v: u8) -> u8 { - v - } - #[inline(always)] - fn u32(v: u32) -> u32 { - v - } - #[inline(always)] - fn i16(v: i16) -> i16 { - v - } - #[inline(always)] - fn i32(v: i32) -> i32 { - v - } - #[inline(always)] - fn i64(v: i64) -> i64 { - v - } - #[inline(always)] - fn f8e4m3(v: f8e4m3) -> f8e4m3 { - v.floor() - } -} - -impl UnaryOpT for Round { - const NAME: &'static str = "round"; - const KERNEL: &'static str = "uround"; - const V: Self = Round; - #[inline(always)] - fn bf16(v: bf16) -> bf16 { - v.round() - } - #[inline(always)] - fn f16(v: f16) -> f16 { - v.round() - } - #[inline(always)] - fn f32(v: f32) -> f32 { - v.round() - } - #[inline(always)] - fn f64(v: f64) -> f64 { - v.round() - } - #[inline(always)] - fn u8(v: u8) -> u8 { - v - } - #[inline(always)] - fn u32(v: u32) -> u32 { - v - } - #[inline(always)] - fn i16(v: i16) -> i16 { - v - } - #[inline(always)] - fn i32(v: i32) -> i32 { - v - } - #[inline(always)] - fn i64(v: i64) -> i64 { - v - } - #[inline(always)] - fn f8e4m3(v: f8e4m3) -> f8e4m3 { - v.round() - } -} - -impl UnaryOpT for GeluErf { - const NAME: &'static str = "gelu_erf"; - const KERNEL: &'static str = "ugelu_erf"; - const V: Self = GeluErf; - #[inline(always)] - fn bf16(v: bf16) -> bf16 { - bf16::from_f64(Self::f64(v.to_f64())) - } - #[inline(always)] - fn f16(v: f16) -> f16 { - f16::from_f64(Self::f64(v.to_f64())) - } - #[inline(always)] - fn f32(v: f32) -> f32 { - (crate::cpu::erf::erf_f32(v * std::f32::consts::FRAC_1_SQRT_2) + 1.) * 0.5 * v - } - #[inline(always)] - fn f64(v: f64) -> f64 { - (crate::cpu::erf::erf_f64(v * std::f64::consts::FRAC_1_SQRT_2) + 1.) * 0.5 * v - } - #[inline(always)] - fn u8(_: u8) -> u8 { - 0 - } - #[inline(always)] - fn u32(_: u32) -> u32 { - 0 - } - #[inline(always)] - fn i16(_: i16) -> i16 { - 0 - } - #[inline(always)] - fn i32(_: i32) -> i32 { - 0 - } - #[inline(always)] - fn i64(_: i64) -> i64 { - 0 - } - #[inline(always)] - fn f8e4m3(v: f8e4m3) -> f8e4m3 { - f8e4m3::from_f32(Self::f32(v.to_f32())) - } -} - -impl UnaryOpT for Relu { - const NAME: &'static str = "relu"; - const KERNEL: &'static str = "urelu"; - const V: Self = Relu; - #[inline(always)] - fn bf16(v: bf16) -> bf16 { - v.max(bf16::ZERO) - } - #[inline(always)] - fn f16(v: f16) -> f16 { - v.max(f16::ZERO) - } - #[inline(always)] - fn f32(v: f32) -> f32 { - v.max(0f32) - } - #[inline(always)] - fn f64(v: f64) -> f64 { - v.max(0f64) - } - #[inline(always)] - fn u8(v: u8) -> u8 { - v - } - #[inline(always)] - fn u32(v: u32) -> u32 { - v - } - #[inline(always)] - fn i16(v: i16) -> i16 { - v.max(0) - } - #[inline(always)] - fn i32(v: i32) -> i32 { - v.max(0) - } - #[inline(always)] - fn i64(v: i64) -> i64 { - v.max(0) - } - #[inline(always)] - fn f8e4m3(v: f8e4m3) -> f8e4m3 { - v.max(f8e4m3::ZERO) - } -} - -/// `BackpropOp` is a wrapper around `Option`. The main goal is to ensure that dependencies are -/// properly checked when creating a new value -#[derive(Clone)] -pub struct BackpropOp(Option); - -impl BackpropOp { - pub fn none() -> Self { - BackpropOp(None) - } - - pub(crate) fn new1(arg: &Tensor, f: impl Fn(Tensor) -> Op) -> Self { - let op = if arg.track_op() { - Some(f(arg.clone())) - } else { - None - }; - Self(op) - } - - pub(crate) fn new2(arg1: &Tensor, arg2: &Tensor, f: impl Fn(Tensor, Tensor) -> Op) -> Self { - let op = if arg1.track_op() || arg2.track_op() { - Some(f(arg1.clone(), arg2.clone())) - } else { - None - }; - Self(op) - } - - pub(crate) fn new3( - arg1: &Tensor, - arg2: &Tensor, - arg3: &Tensor, - f: impl Fn(Tensor, Tensor, Tensor) -> Op, - ) -> Self { - let op = if arg1.track_op() || arg2.track_op() || arg3.track_op() { - Some(f(arg1.clone(), arg2.clone(), arg3.clone())) - } else { - None - }; - Self(op) - } - - pub(crate) fn new>(args: &[A], f: impl Fn(Vec) -> Op) -> Self { - let op = if args.iter().any(|arg| arg.as_ref().track_op()) { - let args: Vec = args.iter().map(|arg| arg.as_ref().clone()).collect(); - Some(f(args)) - } else { - None - }; - Self(op) - } - - pub(crate) fn is_none(&self) -> bool { - self.0.is_none() - } -} - -impl std::ops::Deref for BackpropOp { - type Target = Option; - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -impl UnaryOpT for Sign { - const NAME: &'static str = "sign"; - const KERNEL: &'static str = "usign"; - const V: Self = Sign; - #[inline(always)] - fn bf16(v: bf16) -> bf16 { - bf16::from((v > bf16::ZERO) as i8) - bf16::from((v < bf16::ZERO) as i8) - } - #[inline(always)] - fn f16(v: f16) -> f16 { - f16::from((v > f16::ZERO) as i8) - f16::from((v < f16::ZERO) as i8) - } - #[inline(always)] - fn f32(v: f32) -> f32 { - f32::from(v > 0.) - f32::from(v < 0.) - } - #[inline(always)] - fn f64(v: f64) -> f64 { - f64::from(v > 0.) - f64::from(v < 0.) - } - #[inline(always)] - fn u8(v: u8) -> u8 { - u8::min(1, v) - } - #[inline(always)] - fn u32(v: u32) -> u32 { - u32::min(1, v) - } - #[inline(always)] - fn i16(v: i16) -> i16 { - (v > 0) as i16 - (v < 0) as i16 - } - #[inline(always)] - fn i32(v: i32) -> i32 { - (v > 0) as i32 - (v < 0) as i32 - } - #[inline(always)] - fn i64(v: i64) -> i64 { - (v > 0) as i64 - (v < 0) as i64 - } - #[inline(always)] - fn f8e4m3(v: f8e4m3) -> f8e4m3 { - if v > f8e4m3::ZERO { - f8e4m3::ONE - } else if v < f8e4m3::ZERO { - -f8e4m3::ONE - } else { - f8e4m3::ZERO - } - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/pickle.rs b/v2/dependency-patches/candle-core-0.9.2/src/pickle.rs deleted file mode 100644 index 8a41edfbc6..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/pickle.rs +++ /dev/null @@ -1,841 +0,0 @@ -//! Just enough pickle support to be able to read PyTorch checkpoints. -// This hardcodes objects that are required for tensor reading, we may want to make this a bit more -// composable/tensor agnostic at some point. -use crate::{Context, DType, Error as E, Layout, Result, Tensor}; -use byteorder::{LittleEndian, ReadBytesExt}; -use std::collections::HashMap; -use std::io::BufRead; - -const VERBOSE: bool = false; - -// https://docs.juliahub.com/Pickle/LAUNc/0.1.0/opcode/ -#[repr(u8)] -#[derive(Debug, Eq, PartialEq, Clone)] -pub enum OpCode { - // https://github.com/python/cpython/blob/ed25f097160b5cbb0c9a1f9a746d2f1bbc96515a/Lib/pickletools.py#L2123 - Proto = 0x80, - Global = b'c', - BinPut = b'q', - LongBinPut = b'r', - EmptyTuple = b')', - Reduce = b'R', - Mark = b'(', - BinUnicode = b'X', - BinInt = b'J', - Tuple = b't', - BinPersId = b'Q', - BinInt1 = b'K', - BinInt2 = b'M', - Tuple1 = 0x85, - Tuple2 = 0x86, - Tuple3 = 0x87, - NewTrue = 0x88, - NewFalse = 0x89, - None = b'N', - BinGet = b'h', - LongBinGet = b'j', - SetItem = b's', - SetItems = b'u', - EmptyDict = b'}', - Dict = b'd', - Build = b'b', - Stop = b'.', - NewObj = 0x81, - EmptyList = b']', - BinFloat = b'G', - Append = b'a', - Appends = b'e', - Long1 = 0x8a, -} - -// Avoid using FromPrimitive so as not to drag another dependency. -impl TryFrom for OpCode { - type Error = u8; - fn try_from(value: u8) -> std::result::Result { - match value { - 0x80 => Ok(Self::Proto), - b'c' => Ok(Self::Global), - b'q' => Ok(Self::BinPut), - b'r' => Ok(Self::LongBinPut), - b')' => Ok(Self::EmptyTuple), - b'R' => Ok(Self::Reduce), - b'(' => Ok(Self::Mark), - b'X' => Ok(Self::BinUnicode), - b'J' => Ok(Self::BinInt), - b't' => Ok(Self::Tuple), - b'Q' => Ok(Self::BinPersId), - b'K' => Ok(Self::BinInt1), - b'M' => Ok(Self::BinInt2), - b'N' => Ok(Self::None), - 0x85 => Ok(Self::Tuple1), - 0x86 => Ok(Self::Tuple2), - 0x87 => Ok(Self::Tuple3), - 0x88 => Ok(Self::NewTrue), - 0x89 => Ok(Self::NewFalse), - b'h' => Ok(Self::BinGet), - b'j' => Ok(Self::LongBinGet), - b's' => Ok(Self::SetItem), - b'u' => Ok(Self::SetItems), - b'}' => Ok(Self::EmptyDict), - b'd' => Ok(Self::EmptyDict), - b'b' => Ok(Self::Build), - b'.' => Ok(Self::Stop), - 0x81 => Ok(Self::NewObj), - b']' => Ok(Self::EmptyList), - b'G' => Ok(Self::BinFloat), - b'a' => Ok(Self::Append), - b'e' => Ok(Self::Appends), - 0x8a => Ok(Self::Long1), - value => Err(value), - } - } -} - -fn read_to_newline(r: &mut R) -> Result> { - let mut data: Vec = Vec::with_capacity(32); - r.read_until(b'\n', &mut data)?; - data.pop(); - if data.last() == Some(&b'\r') { - data.pop(); - } - Ok(data) -} - -#[derive(Debug, Clone, PartialEq)] -pub enum Object { - Class { - module_name: String, - class_name: String, - }, - Int(i32), - Long(i64), - Float(f64), - Unicode(String), - Bool(bool), - None, - Tuple(Vec), - List(Vec), - Mark, - Dict(Vec<(Object, Object)>), - Reduce { - callable: Box, - args: Box, - }, - Build { - callable: Box, - args: Box, - }, - PersistentLoad(Box), -} - -type OResult = std::result::Result; - -impl Object { - pub fn unicode(self) -> OResult { - match self { - Self::Unicode(t) => Ok(t), - _ => Err(self), - } - } - - pub fn reduce(self) -> OResult<(Self, Self)> { - match self { - Self::Reduce { callable, args } => Ok((*callable, *args)), - _ => Err(self), - } - } - - pub fn none(self) -> OResult<()> { - match self { - Self::None => Ok(()), - _ => Err(self), - } - } - - pub fn persistent_load(self) -> OResult { - match self { - Self::PersistentLoad(t) => Ok(*t), - _ => Err(self), - } - } - - pub fn bool(self) -> OResult { - match self { - Self::Bool(t) => Ok(t), - _ => Err(self), - } - } - - pub fn int(self) -> OResult { - match self { - Self::Int(t) => Ok(t), - _ => Err(self), - } - } - - pub fn int_or_long(self) -> OResult { - match self { - Self::Int(t) => Ok(t as i64), - Self::Long(t) => Ok(t), - _ => Err(self), - } - } - - pub fn tuple(self) -> OResult> { - match self { - Self::Tuple(t) => Ok(t), - _ => Err(self), - } - } - - pub fn dict(self) -> OResult> { - match self { - Self::Dict(t) => Ok(t), - _ => Err(self), - } - } - - pub fn class(self) -> OResult<(String, String)> { - match self { - Self::Class { - module_name, - class_name, - } => Ok((module_name, class_name)), - _ => Err(self), - } - } - - pub fn into_tensor_info( - self, - name: Self, - dir_name: &std::path::Path, - ) -> Result> { - let name = match name.unicode() { - Ok(name) => name, - Err(_) => return Ok(None), - }; - let (callable, args) = match self.reduce() { - Ok(callable_args) => callable_args, - _ => return Ok(None), - }; - let (callable, args) = match callable { - Object::Class { - module_name, - class_name, - } if module_name == "torch._tensor" && class_name == "_rebuild_from_type_v2" => { - let mut args = args.tuple()?; - let callable = args.remove(0); - let args = args.remove(1); - (callable, args) - } - Object::Class { - module_name, - class_name, - } if module_name == "torch._utils" && class_name == "_rebuild_parameter" => { - let mut args = args.tuple()?; - args.remove(0).reduce()? - } - _ => (callable, args), - }; - match callable { - Object::Class { - module_name, - class_name, - } if module_name == "torch._utils" && class_name == "_rebuild_tensor_v2" => {} - _ => return Ok(None), - }; - let (layout, dtype, file_path, storage_size) = rebuild_args(args)?; - Ok(Some(TensorInfo { - name, - dtype, - layout, - path: format!("{}/{}", dir_name.to_string_lossy(), file_path), - storage_size, - })) - } -} - -impl TryFrom for String { - type Error = Object; - fn try_from(value: Object) -> std::result::Result { - match value { - Object::Unicode(s) => Ok(s), - other => Err(other), - } - } -} - -impl TryFrom for usize { - type Error = Object; - fn try_from(value: Object) -> std::result::Result { - match value { - Object::Int(s) if s >= 0 => Ok(s as usize), - other => Err(other), - } - } -} - -impl> TryFrom for Vec { - type Error = Object; - fn try_from(value: Object) -> std::result::Result { - match value { - Object::Tuple(values) => { - // This does not return the appropriate value in the error case but instead return - // the object related to the first error. - values - .into_iter() - .map(|v| T::try_from(v)) - .collect::, Self::Error>>() - } - other => Err(other), - } - } -} - -#[derive(Debug)] -pub struct Stack { - stack: Vec, - memo: HashMap, -} - -impl Stack { - pub fn empty() -> Self { - Self { - stack: Vec::with_capacity(512), - memo: HashMap::new(), - } - } - - pub fn stack(&self) -> &[Object] { - self.stack.as_slice() - } - - pub fn read_loop(&mut self, r: &mut R) -> Result<()> { - loop { - if self.read(r)? { - break; - } - } - Ok(()) - } - - pub fn finalize(mut self) -> Result { - self.pop() - } - - fn push(&mut self, obj: Object) { - self.stack.push(obj) - } - - fn pop(&mut self) -> Result { - match self.stack.pop() { - None => crate::bail!("unexpected empty stack"), - Some(obj) => Ok(obj), - } - } - - // https://docs.juliahub.com/Pickle/LAUNc/0.1.0/opcode/#Pickle.OpCodes.BUILD - fn build(&mut self) -> Result<()> { - let args = self.pop()?; - let obj = self.pop()?; - let obj = match (obj, args) { - (Object::Dict(mut obj), Object::Dict(mut args)) => { - obj.append(&mut args); - Object::Dict(obj) - } - (obj, args) => Object::Build { - callable: Box::new(obj), - args: Box::new(args), - }, - }; - self.push(obj); - Ok(()) - } - - fn reduce(&mut self) -> Result<()> { - let args = self.pop()?; - let callable = self.pop()?; - #[allow(clippy::single_match)] - let reduced = match &callable { - Object::Class { - module_name, - class_name, - } => { - if module_name == "collections" - && (class_name == "OrderedDict" || class_name == "defaultdict") - { - // TODO: have a separate ordered dict and a separate default dict. - Some(Object::Dict(vec![])) - } else { - None - } - } - _ => None, - }; - let reduced = reduced.unwrap_or_else(|| Object::Reduce { - callable: Box::new(callable), - args: Box::new(args), - }); - self.push(reduced); - Ok(()) - } - - fn last(&mut self) -> Result<&mut Object> { - match self.stack.last_mut() { - None => crate::bail!("unexpected empty stack"), - Some(obj) => Ok(obj), - } - } - - fn memo_get(&self, id: u32) -> Result { - match self.memo.get(&id) { - None => crate::bail!("missing object in memo {id}"), - Some(obj) => { - // Maybe we should use refcounting rather than doing potential large clones here. - Ok(obj.clone()) - } - } - } - - fn memo_put(&mut self, id: u32) -> Result<()> { - let obj = self.last()?.clone(); - self.memo.insert(id, obj); - Ok(()) - } - - fn persistent_load(&self, id: Object) -> Result { - Ok(Object::PersistentLoad(Box::new(id))) - } - - fn new_obj(&self, class: Object, args: Object) -> Result { - Ok(Object::Reduce { - callable: Box::new(class), - args: Box::new(args), - }) - } - - fn pop_to_marker(&mut self) -> Result> { - let mut mark_idx = None; - for (idx, obj) in self.stack.iter().enumerate().rev() { - if obj == &Object::Mark { - mark_idx = Some(idx); - break; - } - } - match mark_idx { - Some(mark_idx) => { - let objs = self.stack.split_off(mark_idx + 1); - self.stack.pop(); - Ok(objs) - } - None => { - crate::bail!("marker object not found") - } - } - } - - pub fn read(&mut self, r: &mut R) -> Result { - let op_code = match OpCode::try_from(r.read_u8()?) { - Ok(op_code) => op_code, - Err(op_code) => { - crate::bail!("unknown op-code {op_code}") - } - }; - // println!("op: {op_code:?}"); - // println!("{:?}", self.stack); - match op_code { - OpCode::Proto => { - let version = r.read_u8()?; - if VERBOSE { - println!("proto {version}"); - } - } - OpCode::Global => { - let module_name = read_to_newline(r)?; - let class_name = read_to_newline(r)?; - let module_name = String::from_utf8_lossy(&module_name).to_string(); - let class_name = String::from_utf8_lossy(&class_name).to_string(); - self.push(Object::Class { - module_name, - class_name, - }) - } - OpCode::BinInt1 => { - let arg = r.read_u8()?; - self.push(Object::Int(arg as i32)) - } - OpCode::BinInt2 => { - let arg = r.read_u16::()?; - self.push(Object::Int(arg as i32)) - } - OpCode::BinInt => { - let arg = r.read_i32::()?; - self.push(Object::Int(arg)) - } - OpCode::BinFloat => { - // Somehow floats are encoded using BigEndian whereas int types use LittleEndian. - // https://github.com/python/cpython/blob/0c80da4c14d904a367968955544dd6ae58c8101c/Lib/pickletools.py#L855 - // https://github.com/pytorch/pytorch/blob/372d078f361e726bb4ac0884ac334b04c58179ef/torch/_weights_only_unpickler.py#L243 - let arg = r.read_f64::()?; - self.push(Object::Float(arg)) - } - OpCode::BinUnicode => { - let len = r.read_u32::()?; - let mut data = vec![0u8; len as usize]; - r.read_exact(&mut data)?; - let data = String::from_utf8(data).map_err(E::wrap)?; - self.push(Object::Unicode(data)) - } - OpCode::BinPersId => { - let id = self.pop()?; - let obj = self.persistent_load(id)?; - self.push(obj) - } - OpCode::Tuple => { - let objs = self.pop_to_marker()?; - self.push(Object::Tuple(objs)) - } - OpCode::Tuple1 => { - let obj = self.pop()?; - self.push(Object::Tuple(vec![obj])) - } - OpCode::Tuple2 => { - let obj2 = self.pop()?; - let obj1 = self.pop()?; - self.push(Object::Tuple(vec![obj1, obj2])) - } - OpCode::Tuple3 => { - let obj3 = self.pop()?; - let obj2 = self.pop()?; - let obj1 = self.pop()?; - self.push(Object::Tuple(vec![obj1, obj2, obj3])) - } - OpCode::NewTrue => self.push(Object::Bool(true)), - OpCode::NewFalse => self.push(Object::Bool(false)), - OpCode::Append => { - let value = self.pop()?; - let pylist = self.last()?; - if let Object::List(d) = pylist { - d.push(value) - } else { - crate::bail!("expected a list, got {pylist:?}") - } - } - OpCode::Appends => { - let objs = self.pop_to_marker()?; - let pylist = self.last()?; - if let Object::List(d) = pylist { - d.extend(objs) - } else { - crate::bail!("expected a list, got {pylist:?}") - } - } - OpCode::SetItem => { - let value = self.pop()?; - let key = self.pop()?; - let pydict = self.last()?; - if let Object::Dict(d) = pydict { - d.push((key, value)) - } else { - crate::bail!("expected a dict, got {pydict:?}") - } - } - OpCode::SetItems => { - let mut objs = self.pop_to_marker()?; - let pydict = self.last()?; - if let Object::Dict(d) = pydict { - if objs.len() % 2 != 0 { - crate::bail!("setitems: not an even number of objects") - } - while let Some(value) = objs.pop() { - let key = objs.pop().context("empty objs")?; - d.push((key, value)) - } - } else { - crate::bail!("expected a dict, got {pydict:?}") - } - } - OpCode::None => self.push(Object::None), - OpCode::Stop => { - return Ok(true); - } - OpCode::Build => self.build()?, - OpCode::EmptyDict => self.push(Object::Dict(vec![])), - OpCode::Dict => { - let mut objs = self.pop_to_marker()?; - let mut pydict = vec![]; - if objs.len() % 2 != 0 { - crate::bail!("setitems: not an even number of objects") - } - while let Some(value) = objs.pop() { - let key = objs.pop().context("empty objs")?; - pydict.push((key, value)) - } - self.push(Object::Dict(pydict)) - } - OpCode::Mark => self.push(Object::Mark), - OpCode::Reduce => self.reduce()?, - OpCode::EmptyTuple => self.push(Object::Tuple(vec![])), - OpCode::EmptyList => self.push(Object::List(vec![])), - OpCode::BinGet => { - let arg = r.read_u8()?; - let obj = self.memo_get(arg as u32)?; - self.push(obj) - } - OpCode::LongBinGet => { - let arg = r.read_u32::()?; - let obj = self.memo_get(arg)?; - self.push(obj) - } - OpCode::BinPut => { - let arg = r.read_u8()?; - self.memo_put(arg as u32)? - } - OpCode::LongBinPut => { - let arg = r.read_u32::()?; - self.memo_put(arg)? - } - OpCode::NewObj => { - let args = self.pop()?; - let class = self.pop()?; - let obj = self.new_obj(class, args)?; - self.push(obj) - } - OpCode::Long1 => { - let n_bytes = r.read_u8()?; - let mut v = 0; - // Decode the next n bytes in little endian - for i in 0..n_bytes { - v |= (r.read_u8()? as i64) << (i * 8); - } - self.push(Object::Long(v)) - } - } - Ok(false) - } -} - -impl From for E { - fn from(value: Object) -> Self { - E::Msg(format!("conversion error on {value:?}")) - } -} - -// https://github.com/pytorch/pytorch/blob/4eac43d046ded0f0a5a5fa8db03eb40f45bf656e/torch/_utils.py#L198 -// Arguments: storage, storage_offset, size, stride, requires_grad, backward_hooks -fn rebuild_args(args: Object) -> Result<(Layout, DType, String, usize)> { - let mut args = args.tuple()?; - let stride = Vec::::try_from(args.remove(3))?; - let size = Vec::::try_from(args.remove(2))?; - let offset = args.remove(1).int_or_long()? as usize; - let storage = args.remove(0).persistent_load()?; - let mut storage = storage.tuple()?; - let storage_size = storage.remove(4).int_or_long()? as usize; - let path = storage.remove(2).unicode()?; - let (_module_name, class_name) = storage.remove(1).class()?; - let dtype = match class_name.as_str() { - "FloatStorage" => DType::F32, - "DoubleStorage" => DType::F64, - "HalfStorage" => DType::F16, - "BFloat16Storage" => DType::BF16, - "ByteStorage" => DType::U8, - "LongStorage" => DType::I64, - other => { - crate::bail!("unsupported storage type {other}") - } - }; - let layout = Layout::new( - crate::Shape::from(size), - stride, - offset * dtype.size_in_bytes(), - ); - Ok((layout, dtype, path, storage_size)) -} - -#[derive(Debug, Clone)] -pub struct TensorInfo { - pub name: String, - pub dtype: DType, - pub layout: Layout, - pub path: String, - pub storage_size: usize, -} - -/// Read the tensor info from a .pth file. -/// -/// # Arguments -/// * `file` - The path to the .pth file. -/// * `verbose` - Whether to print debug information. -/// * `key` - Optional key to retrieve `state_dict` from the pth file. -pub fn read_pth_tensor_info>( - file: P, - verbose: bool, - key: Option<&str>, -) -> Result> { - let file = std::fs::File::open(file)?; - let zip_reader = std::io::BufReader::new(file); - let mut zip = zip::ZipArchive::new(zip_reader)?; - let zip_file_names = zip - .file_names() - .map(|name| name.map(|name| name.into_owned())) - .collect::>>()?; - - let mut tensor_infos = vec![]; - for file_name in zip_file_names.iter() { - if !file_name.ends_with("data.pkl") { - continue; - } - let dir_name = std::path::PathBuf::from(file_name.strip_suffix(".pkl").context("no .pkl")?); - let reader = zip.by_name(file_name)?; - let mut reader = std::io::BufReader::new(reader); - let mut stack = Stack::empty(); - stack.read_loop(&mut reader)?; - let obj = stack.finalize()?; - if VERBOSE || verbose { - println!("{obj:#?}"); - } - - let obj = match obj { - Object::Build { callable, args } => match *callable { - Object::Reduce { callable, args: _ } => match *callable { - Object::Class { - module_name, - class_name, - } if module_name == "__torch__" && class_name == "Module" => *args, - _ => continue, - }, - _ => continue, - }, - obj => obj, - }; - - // If key is provided, then we need to extract the state_dict from the object. - let obj = if let Some(key) = key { - if let Object::Dict(key_values) = obj { - key_values - .into_iter() - .find(|(k, _)| *k == Object::Unicode(key.to_owned())) - .map(|(_, v)| v) - .ok_or_else(|| E::Msg(format!("key {key} not found")))? - } else { - obj - } - } else { - obj - }; - - // If the object is a dict, then we can extract the tensor info from it. - // NOTE: We are assuming that the `obj` is state_dict by this stage. - if let Object::Dict(key_values) = obj { - for (name, value) in key_values.into_iter() { - match value.into_tensor_info(name, &dir_name) { - Ok(Some(tensor_info)) => tensor_infos.push(tensor_info), - Ok(None) => {} - Err(err) => eprintln!("skipping: {err:?}"), - } - } - } - } - Ok(tensor_infos) -} - -/// Lazy tensor loader. -pub struct PthTensors { - tensor_infos: HashMap, - path: std::path::PathBuf, - // We do not store a zip reader as it needs mutable access to extract data. Instead we - // re-create a zip reader for each tensor. -} - -impl PthTensors { - pub fn new>(path: P, key: Option<&str>) -> Result { - let tensor_infos = read_pth_tensor_info(path.as_ref(), false, key)?; - let tensor_infos = tensor_infos - .into_iter() - .map(|ti| (ti.name.to_string(), ti)) - .collect(); - let path = path.as_ref().to_owned(); - Ok(Self { tensor_infos, path }) - } - - pub fn tensor_infos(&self) -> &HashMap { - &self.tensor_infos - } - - pub fn get(&self, name: &str) -> Result> { - use std::io::Read; - let tensor_info = match self.tensor_infos.get(name) { - None => return Ok(None), - Some(tensor_info) => tensor_info, - }; - // We hope that the file has not changed since first reading it. - let zip_reader = std::io::BufReader::new(std::fs::File::open(&self.path)?); - let mut zip = zip::ZipArchive::new(zip_reader)?; - let mut reader = zip.by_name(&tensor_info.path)?; - let is_fortran_contiguous = tensor_info.layout.is_fortran_contiguous(); - let rank = tensor_info.layout.shape().rank(); - - // Reading the data is a bit tricky as it can be strided, for now only support the basic - // case and when the tensor is fortran contiguous. - if !tensor_info.layout.is_contiguous() && !is_fortran_contiguous { - crate::bail!( - "cannot retrieve non-contiguous tensors {:?}", - tensor_info.layout - ) - } - let start_offset = tensor_info.layout.start_offset(); - if start_offset > 0 { - std::io::copy( - &mut reader.by_ref().take(start_offset as u64), - &mut std::io::sink(), - )?; - } - let tensor = Tensor::from_reader( - tensor_info.layout.shape().clone(), - tensor_info.dtype, - &mut reader, - )?; - - if rank > 1 && is_fortran_contiguous { - // Reverse the shape, e.g. Shape(2, 3, 4) -> Shape(4, 3, 2) - let shape_reversed: Vec<_> = tensor_info.layout.dims().iter().rev().cloned().collect(); - let tensor = tensor.reshape(shape_reversed)?; - - // Permute (transpose) the dimensions, e.g. Shape(4, 3, 2) -> Shape(2, 3, 4) - let dim_indices_reversed: Vec<_> = (0..rank).rev().collect(); - let tensor = tensor.permute(dim_indices_reversed)?; - Ok(Some(tensor)) - } else { - Ok(Some(tensor)) - } - } -} - -/// Read all the tensors from a PyTorch pth file with a given key. -/// -/// # Arguments -/// * `path` - Path to the pth file. -/// * `key` - Optional key to retrieve `state_dict` from the pth file. Sometimes the pth file -/// contains multiple objects and the state_dict is the one we are interested in. -pub fn read_all_with_key>( - path: P, - key: Option<&str>, -) -> Result> { - let pth = PthTensors::new(path, key)?; - let tensor_names = pth.tensor_infos.keys(); - let mut tensors = Vec::with_capacity(tensor_names.len()); - for name in tensor_names { - if let Some(tensor) = pth.get(name)? { - tensors.push((name.to_string(), tensor)) - } - } - Ok(tensors) -} - -/// Read all the tensors from a PyTorch pth file. -/// -/// # Arguments -/// * `path` - Path to the pth file. -pub fn read_all>(path: P) -> Result> { - read_all_with_key(path, None) -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/avx.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/avx.rs deleted file mode 100644 index 527941eb6d..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/quantized/avx.rs +++ /dev/null @@ -1,669 +0,0 @@ -use super::k_quants::{ - BlockQ2K, BlockQ3K, BlockQ4K, BlockQ4_0, BlockQ5K, BlockQ6K, BlockQ8K, BlockQ8_0, QK8_0, QK_K, -}; -use byteorder::{ByteOrder, LittleEndian}; -use half::f16; - -#[cfg(target_arch = "x86")] -use core::arch::x86::*; -#[cfg(target_arch = "x86_64")] -use core::arch::x86_64::*; - -#[inline(always)] -pub(crate) unsafe fn sum_i16_pairs_float(x: __m256i) -> __m256 { - let ones = _mm256_set1_epi16(1); - let summed_pairs = _mm256_madd_epi16(ones, x); - _mm256_cvtepi32_ps(summed_pairs) -} - -#[inline(always)] -pub(crate) unsafe fn mul_sum_us8_pairs_float(ax: __m256i, sy: __m256i) -> __m256 { - let dot = _mm256_maddubs_epi16(ax, sy); - sum_i16_pairs_float(dot) -} - -#[inline(always)] -pub(crate) unsafe fn hsum_float_8(x: __m256) -> f32 { - let res = _mm256_extractf128_ps(x, 1); - let res = _mm_add_ps(res, _mm256_castps256_ps128(x)); - let res = _mm_add_ps(res, _mm_movehl_ps(res, res)); - let res = _mm_add_ss(res, _mm_movehdup_ps(res)); - _mm_cvtss_f32(res) -} - -#[inline(always)] -pub(crate) unsafe fn bytes_from_nibbles_32(rsi: *const u8) -> __m256i { - let tmp = _mm_loadu_si128(rsi as *const __m128i); - let bytes = _mm256_insertf128_si256::<1>(_mm256_castsi128_si256(tmp), _mm_srli_epi16(tmp, 4)); - let low_mask = _mm256_set1_epi8(0xF); - _mm256_and_si256(low_mask, bytes) -} - -#[inline(always)] -pub(crate) unsafe fn mul_sum_i8_pairs_float(x: __m256i, y: __m256i) -> __m256 { - let ax = _mm256_sign_epi8(x, x); - let sy = _mm256_sign_epi8(y, x); - mul_sum_us8_pairs_float(ax, sy) -} - -#[inline(always)] -pub(crate) fn vec_dot_q4_0_q8_0(n: usize, xs: &[BlockQ4_0], ys: &[BlockQ8_0]) -> f32 { - debug_assert!( - n.is_multiple_of(QK8_0), - "vec_dot_q4_0_q8_0: {n} is not divisible by {QK8_0}" - ); - unsafe { - let mut acc = _mm256_setzero_ps(); - for (x, y) in xs.iter().zip(ys.iter()) { - let d = _mm256_set1_ps(f16::to_f32(x.d) * f16::to_f32(y.d)); - let bx = bytes_from_nibbles_32(x.qs.as_ptr()); - let off = _mm256_set1_epi8(8); - let bx = _mm256_sub_epi8(bx, off); - let by = _mm256_loadu_si256(y.qs.as_ptr() as *const __m256i); - let q = mul_sum_i8_pairs_float(bx, by); - acc = _mm256_fmadd_ps(d, q, acc); - } - hsum_float_8(acc) - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q8_0_q8_0(n: usize, xs: &[BlockQ8_0], ys: &[BlockQ8_0]) -> f32 { - debug_assert!( - n.is_multiple_of(QK8_0), - "vec_dot_q8_0_q8_0: {n} is not divisible by {QK8_0}" - ); - unsafe { - let mut acc = _mm256_setzero_ps(); - for (x, y) in xs.iter().zip(ys.iter()) { - let d = _mm256_set1_ps(f16::to_f32(x.d) * f16::to_f32(y.d)); - let bx = _mm256_loadu_si256(x.qs.as_ptr() as *const __m256i); - let by = _mm256_loadu_si256(y.qs.as_ptr() as *const __m256i); - let q = mul_sum_i8_pairs_float(bx, by); - acc = _mm256_fmadd_ps(d, q, acc); - } - hsum_float_8(acc) - } -} - -#[inline(always)] -unsafe fn get_scale_shuffle(i: usize) -> __m128i { - const K_SHUFFLE: [u8; 128] = [ - 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, - 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 5, 5, 5, 5, 5, 5, 5, 5, 6, 6, 6, 6, 6, 6, 6, 6, 7, 7, 7, 7, - 7, 7, 7, 7, 8, 8, 8, 8, 8, 8, 8, 8, 9, 9, 9, 9, 9, 9, 9, 9, 10, 10, 10, 10, 10, 10, 10, 10, - 11, 11, 11, 11, 11, 11, 11, 11, 12, 12, 12, 12, 12, 12, 12, 12, 13, 13, 13, 13, 13, 13, 13, - 13, 14, 14, 14, 14, 14, 14, 14, 14, 15, 15, 15, 15, 15, 15, 15, 15, - ]; - _mm_loadu_si128((K_SHUFFLE.as_ptr() as *const __m128i).add(i)) -} - -#[inline(always)] -unsafe fn get_scale_shuffle_k4(i: usize) -> __m256i { - const K_SHUFFLE: [u8; 256] = [ - 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, - 0, 1, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, - 2, 3, 2, 3, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, - 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, - 6, 7, 6, 7, 6, 7, 6, 7, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, - 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, - 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 10, 11, 12, 13, 12, 13, 12, 13, - 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, - 13, 12, 13, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, - 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, - ]; - _mm256_loadu_si256((K_SHUFFLE.as_ptr() as *const __m256i).add(i)) -} - -#[inline(always)] -unsafe fn get_scale_shuffle_q3k(i: usize) -> __m256i { - const K_SHUFFLE: [u8; 128] = [ - 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, 2, 3, - 2, 3, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 4, 5, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, 6, 7, - 6, 7, 6, 7, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 8, 9, 10, 11, 10, 11, 10, 11, 10, 11, - 10, 11, 10, 11, 10, 11, 10, 11, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, 13, 12, - 13, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, 14, 15, - ]; - _mm256_loadu_si256((K_SHUFFLE.as_ptr() as *const __m256i).add(i)) -} - -#[inline(always)] -pub(crate) fn vec_dot_q6k_q8k(n: usize, xs: &[BlockQ6K], ys: &[BlockQ8K]) -> f32 { - debug_assert!( - n.is_multiple_of(QK_K), - "vec_dot_q6k_8k: {n} is not divisible by {QK_K}" - ); - - unsafe { - let m4 = _mm256_set1_epi8(0xF); - let m2 = _mm256_set1_epi8(3); - let m32s = _mm256_set1_epi8(32); - let mut acc = _mm256_setzero_ps(); - for (x, y) in xs.iter().zip(ys.iter()) { - let d = y.d * x.d.to_f32(); - let mut q4 = x.ql.as_ptr(); - let mut qh = x.qh.as_ptr(); - let mut q8 = y.qs.as_ptr(); - - let scales = _mm_loadu_si128(x.scales.as_ptr() as *const __m128i); - let mut sumi = _mm256_setzero_si256(); - - for j in 0..QK_K / 128 { - let is = j * 4; - let scale_0 = _mm_shuffle_epi8(scales, get_scale_shuffle(is)); - let scale_1 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 1)); - let scale_2 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 2)); - let scale_3 = _mm_shuffle_epi8(scales, get_scale_shuffle(is + 3)); - - let q4bits1 = _mm256_loadu_si256(q4 as *const __m256i); - q4 = q4.add(32); - let q4bits2 = _mm256_loadu_si256(q4 as *const __m256i); - q4 = q4.add(32); - let q4bits_h = _mm256_loadu_si256(qh as *const __m256i); - qh = qh.add(32); - - let q4h_0 = _mm256_slli_epi16(_mm256_and_si256(q4bits_h, m2), 4); - let q4h_1 = - _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bits_h, 2), m2), 4); - let q4h_2 = - _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bits_h, 4), m2), 4); - let q4h_3 = - _mm256_slli_epi16(_mm256_and_si256(_mm256_srli_epi16(q4bits_h, 6), m2), 4); - - let q4_0 = _mm256_or_si256(_mm256_and_si256(q4bits1, m4), q4h_0); - let q4_1 = _mm256_or_si256(_mm256_and_si256(q4bits2, m4), q4h_1); - let q4_2 = - _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits1, 4), m4), q4h_2); - let q4_3 = - _mm256_or_si256(_mm256_and_si256(_mm256_srli_epi16(q4bits2, 4), m4), q4h_3); - - let q8_0 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let q8_1 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let q8_2 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let q8_3 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - - let q8s_0 = _mm256_maddubs_epi16(m32s, q8_0); - let q8s_1 = _mm256_maddubs_epi16(m32s, q8_1); - let q8s_2 = _mm256_maddubs_epi16(m32s, q8_2); - let q8s_3 = _mm256_maddubs_epi16(m32s, q8_3); - - let p16_0 = _mm256_maddubs_epi16(q4_0, q8_0); - let p16_1 = _mm256_maddubs_epi16(q4_1, q8_1); - let p16_2 = _mm256_maddubs_epi16(q4_2, q8_2); - let p16_3 = _mm256_maddubs_epi16(q4_3, q8_3); - - let p16_0 = _mm256_sub_epi16(p16_0, q8s_0); - let p16_1 = _mm256_sub_epi16(p16_1, q8s_1); - let p16_2 = _mm256_sub_epi16(p16_2, q8s_2); - let p16_3 = _mm256_sub_epi16(p16_3, q8s_3); - - let p16_0 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_0), p16_0); - let p16_1 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_1), p16_1); - let p16_2 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_2), p16_2); - let p16_3 = _mm256_madd_epi16(_mm256_cvtepi8_epi16(scale_3), p16_3); - - sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1)); - sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_2, p16_3)); - } - acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc); - } - hsum_float_8(acc) - } -} - -#[inline(always)] -unsafe fn mm256_set_m128i(a: __m128i, b: __m128i) -> __m256i { - _mm256_insertf128_si256(_mm256_castsi128_si256(b), a, 1) -} - -#[inline(always)] -pub(crate) fn vec_dot_q2k_q8k(n: usize, xs: &[BlockQ2K], ys: &[BlockQ8K]) -> f32 { - debug_assert!( - n.is_multiple_of(QK_K), - "vec_dot_q2k_q8k: {n} is not divisible by {QK_K}" - ); - - unsafe { - let m3 = _mm256_set1_epi8(3); - let m4 = _mm_set1_epi8(0xF); - - let mut acc = _mm256_setzero_ps(); - - for (x, y) in xs.iter().zip(ys.iter()) { - let d = y.d * x.d.to_f32(); - let dmin = -y.d * x.dmin.to_f32(); - - let mut q2 = x.qs.as_ptr(); - let mut q8 = y.qs.as_ptr(); - - let mins_and_scales = _mm_loadu_si128(x.scales.as_ptr() as *const __m128i); - let scales8 = _mm_and_si128(mins_and_scales, m4); - let mins8 = _mm_and_si128(_mm_srli_epi16(mins_and_scales, 4), m4); - let mins = _mm256_cvtepi8_epi16(mins8); - let prod = - _mm256_madd_epi16(mins, _mm256_loadu_si256(y.bsums.as_ptr() as *const __m256i)); - - acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&dmin), _mm256_cvtepi32_ps(prod), acc); - - let all_scales = _mm256_cvtepi8_epi16(scales8); - let l_scales = _mm256_extracti128_si256(all_scales, 0); - let h_scales = _mm256_extracti128_si256(all_scales, 1); - let scales = [ - mm256_set_m128i(l_scales, l_scales), - mm256_set_m128i(h_scales, h_scales), - ]; - - let mut sumi = _mm256_setzero_si256(); - - for scale in scales { - let q2bits = _mm256_loadu_si256(q2 as *const __m256i); - q2 = q2.add(32); - - let q8_0 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let q8_1 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let q8_2 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let q8_3 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - - let q2_0 = _mm256_and_si256(q2bits, m3); - let q2_1 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 2), m3); - let q2_2 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 4), m3); - let q2_3 = _mm256_and_si256(_mm256_srli_epi16(q2bits, 6), m3); - - let p0 = _mm256_maddubs_epi16(q2_0, q8_0); - let p1 = _mm256_maddubs_epi16(q2_1, q8_1); - let p2 = _mm256_maddubs_epi16(q2_2, q8_2); - let p3 = _mm256_maddubs_epi16(q2_3, q8_3); - - let p0 = - _mm256_madd_epi16(_mm256_shuffle_epi8(scale, get_scale_shuffle_q3k(0)), p0); - let p1 = - _mm256_madd_epi16(_mm256_shuffle_epi8(scale, get_scale_shuffle_q3k(1)), p1); - let p2 = - _mm256_madd_epi16(_mm256_shuffle_epi8(scale, get_scale_shuffle_q3k(2)), p2); - let p3 = - _mm256_madd_epi16(_mm256_shuffle_epi8(scale, get_scale_shuffle_q3k(3)), p3); - - let p0 = _mm256_add_epi32(p0, p1); - let p2 = _mm256_add_epi32(p2, p3); - - sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p0, p2)); - } - acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc); - } - - hsum_float_8(acc) - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q3k_q8k(n: usize, xs: &[BlockQ3K], ys: &[BlockQ8K]) -> f32 { - debug_assert!( - n.is_multiple_of(QK_K), - "vec_dot_q3k_q8k: {n} is not divisible by {QK_K}" - ); - - const KMASK1: u32 = 0x03030303; - const KMASK2: u32 = 0x0f0f0f0f; - - let mut aux = [0u32; 3]; - - unsafe { - let m3 = _mm256_set1_epi8(3); - let mone = _mm256_set1_epi8(1); - let m32 = _mm_set1_epi8(32); - - let mut acc = _mm256_setzero_ps(); - for (x, y) in xs.iter().zip(ys.iter()) { - let d = y.d * x.d.to_f32(); - - let mut q3 = x.qs.as_ptr(); - let mut q8 = y.qs.as_ptr(); - - LittleEndian::read_u32_into(&x.scales, &mut aux); - let scales128 = _mm_set_epi32( - (((aux[1] >> 4) & KMASK2) | (((aux[2] >> 6) & KMASK1) << 4)) as i32, - (((aux[0] >> 4) & KMASK2) | (((aux[2] >> 4) & KMASK1) << 4)) as i32, - ((aux[1] & KMASK2) | (((aux[2] >> 2) & KMASK1) << 4)) as i32, - ((aux[0] & KMASK2) | (((aux[2]) & KMASK1) << 4)) as i32, - ); - let scales128 = _mm_sub_epi8(scales128, m32); - let all_scales = _mm256_cvtepi8_epi16(scales128); - let l_scales = _mm256_extracti128_si256(all_scales, 0); - let h_scales = _mm256_extracti128_si256(all_scales, 1); - let scales = [ - mm256_set_m128i(l_scales, l_scales), - mm256_set_m128i(h_scales, h_scales), - ]; - - // high bit - let hbits = _mm256_loadu_si256(x.hmask.as_ptr() as *const __m256i); - - let mut sumi = _mm256_setzero_si256(); - - for (j, scale) in scales.iter().enumerate() { - // load low 2 bits - let q3bits = _mm256_loadu_si256(q3 as *const __m256i); - q3 = q3.add(32); - - // Prepare low and high bits - // We hardcode the shifts here to avoid loading them into a separate register - let q3l_0 = _mm256_and_si256(q3bits, m3); - let q3h_0 = if j == 0 { - _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 0)), 0) - } else { - _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 4)), 4) - }; - let q3h_0 = _mm256_slli_epi16(q3h_0, 2); - - let q3l_1 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 2), m3); - let q3h_1 = if j == 0 { - _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 1)), 1) - } else { - _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 5)), 5) - }; - let q3h_1 = _mm256_slli_epi16(q3h_1, 2); - - let q3l_2 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 4), m3); - let q3h_2 = if j == 0 { - _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 2)), 2) - } else { - _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 6)), 6) - }; - let q3h_2 = _mm256_slli_epi16(q3h_2, 2); - - let q3l_3 = _mm256_and_si256(_mm256_srli_epi16(q3bits, 6), m3); - let q3h_3 = if j == 0 { - _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 3)), 3) - } else { - _mm256_srli_epi16(_mm256_andnot_si256(hbits, _mm256_slli_epi16(mone, 7)), 7) - }; - let q3h_3 = _mm256_slli_epi16(q3h_3, 2); - - // load Q8 quants - let q8_0 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let q8_1 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let q8_2 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let q8_3 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - - // Dot product: we multiply the 2 low bits and 1 high bit part separately, so we - // can use _mm256_maddubs_epi16, and then subtract. The high bit part has the 2 - // already subtracted (and so, it is zero if the high bit was not set, and 2 if the - // high bit was set) - let q8s_0 = _mm256_maddubs_epi16(q3h_0, q8_0); - let q8s_1 = _mm256_maddubs_epi16(q3h_1, q8_1); - let q8s_2 = _mm256_maddubs_epi16(q3h_2, q8_2); - let q8s_3 = _mm256_maddubs_epi16(q3h_3, q8_3); - - let p16_0 = _mm256_maddubs_epi16(q3l_0, q8_0); - let p16_1 = _mm256_maddubs_epi16(q3l_1, q8_1); - let p16_2 = _mm256_maddubs_epi16(q3l_2, q8_2); - let p16_3 = _mm256_maddubs_epi16(q3l_3, q8_3); - - let p16_0 = _mm256_sub_epi16(p16_0, q8s_0); - let p16_1 = _mm256_sub_epi16(p16_1, q8s_1); - let p16_2 = _mm256_sub_epi16(p16_2, q8s_2); - let p16_3 = _mm256_sub_epi16(p16_3, q8s_3); - - // multiply with scales - let p16_0 = - _mm256_madd_epi16(_mm256_shuffle_epi8(*scale, get_scale_shuffle_q3k(0)), p16_0); - let p16_1 = - _mm256_madd_epi16(_mm256_shuffle_epi8(*scale, get_scale_shuffle_q3k(1)), p16_1); - let p16_2 = - _mm256_madd_epi16(_mm256_shuffle_epi8(*scale, get_scale_shuffle_q3k(2)), p16_2); - let p16_3 = - _mm256_madd_epi16(_mm256_shuffle_epi8(*scale, get_scale_shuffle_q3k(3)), p16_3); - - // accumulate - let p16_0 = _mm256_add_epi32(p16_0, p16_1); - let p16_2 = _mm256_add_epi32(p16_2, p16_3); - sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_2)); - } - - // multiply with block scale and accumulate - acc = _mm256_fmadd_ps(_mm256_broadcast_ss(&d), _mm256_cvtepi32_ps(sumi), acc); - } - hsum_float_8(acc) - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q4k_q8k(n: usize, xs: &[BlockQ4K], ys: &[BlockQ8K]) -> f32 { - debug_assert!( - n.is_multiple_of(QK_K), - "vec_dot_q4k_q8k: {n} is not divisible by {QK_K}" - ); - let mut utmp = [0u32; 4]; - const KMASK1: u32 = 0x3f3f3f3f; - const KMASK2: u32 = 0x0f0f0f0f; - const KMASK3: u32 = 0x03030303; - - unsafe { - let m4 = _mm256_set1_epi8(0xF); - - let mut acc = _mm256_setzero_ps(); - let mut acc_m = _mm_setzero_ps(); - - for (x, y) in xs.iter().zip(ys.iter()) { - let d = y.d * x.d.to_f32(); - let dmin = -y.d * x.dmin.to_f32(); - - LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); - - utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); - let uaux = utmp[1] & KMASK1; - utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); - utmp[2] = uaux; - utmp[0] &= KMASK1; - - let mut q4 = x.qs.as_ptr(); - let mut q8 = y.qs.as_ptr(); - - let mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32( - utmp[3] as i32, - utmp[2] as i32, - utmp[1] as i32, - utmp[0] as i32, - )); - - let q8sums = _mm256_loadu_si256(y.bsums.as_ptr() as *const __m256i); - let q8s = _mm_hadd_epi16( - _mm256_extracti128_si256(q8sums, 0), - _mm256_extracti128_si256(q8sums, 1), - ); - let prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s); - acc_m = _mm_fmadd_ps(_mm_set1_ps(dmin), _mm_cvtepi32_ps(prod), acc_m); - - let sc128 = _mm256_extracti128_si256(mins_and_scales, 0); - let scales = mm256_set_m128i(sc128, sc128); - - let mut sumi = _mm256_setzero_si256(); - - for j in 0..QK_K / 64 { - let scale_l = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2 * j)); - let scale_h = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2 * j + 1)); - - let q4bits = _mm256_loadu_si256(q4 as *const __m256i); - q4 = q4.add(32); - let q4l = _mm256_and_si256(q4bits, m4); - let q4h = _mm256_and_si256(_mm256_srli_epi16(q4bits, 4), m4); - - let q8l = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let p16l = _mm256_maddubs_epi16(q4l, q8l); - let p16l = _mm256_madd_epi16(scale_l, p16l); - sumi = _mm256_add_epi32(sumi, p16l); - - let q8h = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let p16h = _mm256_maddubs_epi16(q4h, q8h); - let p16h = _mm256_madd_epi16(scale_h, p16h); - sumi = _mm256_add_epi32(sumi, p16h); - } - - let vd = _mm256_set1_ps(d); - acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc); - } - - let acc_m = _mm_add_ps(acc_m, _mm_movehl_ps(acc_m, acc_m)); - let acc_m = _mm_add_ss(acc_m, _mm_movehdup_ps(acc_m)); - - hsum_float_8(acc) + _mm_cvtss_f32(acc_m) - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q5k_q8k(n: usize, xs: &[BlockQ5K], ys: &[BlockQ8K]) -> f32 { - debug_assert!( - n.is_multiple_of(QK_K), - "vec_dot_q5k_q8k: {n} is not divisible by {QK_K}" - ); - let mut utmp = [0u32; 4]; - const KMASK1: u32 = 0x3f3f3f3f; - const KMASK2: u32 = 0x0f0f0f0f; - const KMASK3: u32 = 0x03030303; - - unsafe { - let m4 = _mm256_set1_epi8(0xF); - let mzero = _mm_setzero_si128(); - let mone = _mm256_set1_epi8(1); - - let mut acc = _mm256_setzero_ps(); - let mut summs = 0.0; - - for (x, y) in xs.iter().zip(ys.iter()) { - let d = y.d * x.d.to_f32(); - let dmin = -y.d * x.dmin.to_f32(); - - LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); - - utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); - let uaux = utmp[1] & KMASK1; - utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); - utmp[2] = uaux; - utmp[0] &= KMASK1; - - let mut q5 = x.qs.as_ptr(); - let mut q8 = y.qs.as_ptr(); - - let mins_and_scales = _mm256_cvtepu8_epi16(_mm_set_epi32( - utmp[3] as i32, - utmp[2] as i32, - utmp[1] as i32, - utmp[0] as i32, - )); - - let q8sums = _mm256_loadu_si256(y.bsums.as_ptr() as *const __m256i); - let q8s = _mm_hadd_epi16( - _mm256_extracti128_si256(q8sums, 0), - _mm256_extracti128_si256(q8sums, 1), - ); - let prod = _mm_madd_epi16(_mm256_extracti128_si256(mins_and_scales, 1), q8s); - let hsum = _mm_hadd_epi32(_mm_hadd_epi32(prod, mzero), mzero); - summs += dmin * _mm_extract_epi32(hsum, 0) as f32; - - let sc128 = _mm256_extracti128_si256(mins_and_scales, 0); - let scales = mm256_set_m128i(sc128, sc128); - - let hbits = _mm256_loadu_si256(x.qh.as_ptr() as *const __m256i); - let mut hmask = mone; - - let mut sumi = _mm256_setzero_si256(); - - for j in 0..QK_K / 64 { - let scale_0 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2 * j)); - let scale_1 = _mm256_shuffle_epi8(scales, get_scale_shuffle_k4(2 * j + 1)); - - let q5bits = _mm256_loadu_si256(q5 as *const __m256i); - q5 = q5.add(32); - - //Similar to q3k we hardcode the shifts here to avoid loading them into a separate register - let q5l_0 = _mm256_and_si256(q5bits, m4); - let q5l_0_shift_input = _mm256_and_si256(hbits, hmask); - let q5l_0_right_shift = match j { - 0 => _mm256_srli_epi16(q5l_0_shift_input, 0), - 1 => _mm256_srli_epi16(q5l_0_shift_input, 2), - 2 => _mm256_srli_epi16(q5l_0_shift_input, 4), - 3 => _mm256_srli_epi16(q5l_0_shift_input, 6), - _ => unreachable!(), - }; - let q5h_0 = _mm256_slli_epi16(q5l_0_right_shift, 4); - let q5_0 = _mm256_add_epi8(q5l_0, q5h_0); - hmask = _mm256_slli_epi16(hmask, 1); - - let q5l_1 = _mm256_and_si256(_mm256_srli_epi16(q5bits, 4), m4); - let q5l_1_shift_input = _mm256_and_si256(hbits, hmask); - let q5l_1_right_shift = match j { - 0 => _mm256_srli_epi16(q5l_1_shift_input, 1), - 1 => _mm256_srli_epi16(q5l_1_shift_input, 3), - 2 => _mm256_srli_epi16(q5l_1_shift_input, 5), - 3 => _mm256_srli_epi16(q5l_1_shift_input, 7), - _ => unreachable!(), - }; - - let q5h_1 = _mm256_slli_epi16(q5l_1_right_shift, 4); - let q5_1 = _mm256_add_epi8(q5l_1, q5h_1); - hmask = _mm256_slli_epi16(hmask, 1); - - let q8_0 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - let q8_1 = _mm256_loadu_si256(q8 as *const __m256i); - q8 = q8.add(32); - - let p16_0 = _mm256_maddubs_epi16(q5_0, q8_0); - let p16_1 = _mm256_maddubs_epi16(q5_1, q8_1); - - let p16_0 = _mm256_madd_epi16(scale_0, p16_0); - let p16_1 = _mm256_madd_epi16(scale_1, p16_1); - - sumi = _mm256_add_epi32(sumi, _mm256_add_epi32(p16_0, p16_1)); - } - let vd = _mm256_set1_ps(d); - acc = _mm256_fmadd_ps(vd, _mm256_cvtepi32_ps(sumi), acc); - } - hsum_float_8(acc) + summs - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q8k_q8k(n: usize, xs: &[BlockQ8K], ys: &[BlockQ8K]) -> f32 { - debug_assert!( - n.is_multiple_of(QK_K), - "vec_dot_q8k_8k: {n} is not divisible by {QK_K}" - ); - unsafe { - let mut acc = _mm256_setzero_ps(); - for (xs, ys) in xs.iter().zip(ys.iter()) { - let mut sumi = _mm256_setzero_si256(); - let x_qs = xs.qs.as_ptr(); - let y_qs = ys.qs.as_ptr(); - for j in (0..QK_K).step_by(32) { - let xs = _mm256_loadu_si256(x_qs.add(j) as *const __m256i); - let ys = _mm256_loadu_si256(y_qs.add(j) as *const __m256i); - - let xs0 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(xs, 0)); - let ys0 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(ys, 0)); - sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(xs0, ys0)); - - let xs1 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(xs, 1)); - let ys1 = _mm256_cvtepi8_epi16(_mm256_extracti128_si256(ys, 1)); - sumi = _mm256_add_epi32(sumi, _mm256_madd_epi16(xs1, ys1)); - } - let d = _mm256_set1_ps(xs.d * ys.d); - acc = _mm256_fmadd_ps(d, _mm256_cvtepi32_ps(sumi), acc); - } - hsum_float_8(acc) - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/cuda.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/cuda.rs deleted file mode 100644 index c47aaf9d24..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/quantized/cuda.rs +++ /dev/null @@ -1,983 +0,0 @@ -use super::{GgmlDType, QStorage}; -use crate::quantized::k_quants::GgmlType; -use crate::{backend::BackendDevice, cuda_backend::WrapErr}; -use crate::{builder_arg as barg, CudaDevice, CudaStorage, Result}; -use half::f16; - -use cudarc::driver::{CudaSlice, CudaView, PushKernelArg}; - -#[derive(Clone, Debug)] -struct PaddedCudaSlice { - inner: CudaSlice, - len: usize, -} - -#[derive(Clone, Debug)] -pub struct QCudaStorage { - data: PaddedCudaSlice, - dtype: GgmlDType, - device: CudaDevice, -} - -static FORCE_DMMV: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false); - -pub fn set_force_dmmv(f: bool) { - FORCE_DMMV.store(f, std::sync::atomic::Ordering::Relaxed) -} - -pub const WARP_SIZE: usize = 32; -pub const MMQ_X_Q4_0_AMPERE: usize = 4; -pub const MMQ_Y_Q4_0_AMPERE: usize = 32; -pub const NWARPS_Q4_0_AMPERE: usize = 4; -pub const GGML_CUDA_MMV_X: usize = 32; -pub const GGML_CUDA_MMV_Y: usize = 1; -pub const CUDA_QUANTIZE_BLOCK_SIZE: usize = 256; -pub const CUDA_DEQUANTIZE_BLOCK_SIZE: usize = 256; -pub const MATRIX_ROW_PADDING: usize = 512; - -fn ceil_div(p: usize, q: usize) -> usize { - p.div_ceil(q) -} - -fn pad(p: usize, q: usize) -> usize { - ceil_div(p, q) * q -} - -fn quantize_q8_1( - src: &CudaView, - dst: &mut CudaSlice, - k: usize, - ky: usize, - dev: &CudaDevice, -) -> Result<()> { - let kx_padded = pad(k, MATRIX_ROW_PADDING); - let num_blocks = ceil_div(kx_padded, CUDA_QUANTIZE_BLOCK_SIZE); - - let total_rows = ky; - // Get Q8_1 metadata. - let q8_1_block_size = GgmlDType::Q8_1.block_size(); - let q8_1_type_size = GgmlDType::Q8_1.type_size(); - - // Calculate the size of the output buffer in bytes. - let num_blocks_per_row = kx_padded / q8_1_block_size; - let dst_row_size_bytes = num_blocks_per_row * q8_1_type_size; - - const CHUNK_SIZE: usize = 65535; // gridDim.y limit - let func = dev.get_or_load_func("quantize_q8_1", &candle_kernels::QUANTIZED)?; - - let mut rows_processed = 0; - while rows_processed < total_rows { - // --- calculate the number of rows for this chunk --- - let remaining_rows = total_rows - rows_processed; - // This is our gridDim.y, now <= 65535 - let rows_in_chunk = std::cmp::min(CHUNK_SIZE, remaining_rows); - - // --- slice the source (f32) tensor by elements --- - let src_start_elem = rows_processed * k; - let src_num_elems = rows_in_chunk * k; - let src_chunk = src.slice(src_start_elem..(src_start_elem + src_num_elems)); - - // --- slice the destination (u8) tensor by bytes --- - let dst_start_byte = rows_processed * dst_row_size_bytes; - let dst_num_bytes = rows_in_chunk * dst_row_size_bytes; - let dst_chunk = dst.slice(dst_start_byte..(dst_start_byte + dst_num_bytes)); - - let cfg = cudarc::driver::LaunchConfig { - grid_dim: (num_blocks as u32, rows_in_chunk as u32, 1), - block_dim: (CUDA_QUANTIZE_BLOCK_SIZE as u32, 1, 1), - shared_mem_bytes: 0, - }; - - let mut builder = func.builder(); - builder.arg(&src_chunk); - builder.arg(&dst_chunk); - barg!(builder, k as i32, kx_padded as i32); - unsafe { builder.launch(cfg) }.w()?; - - rows_processed += rows_in_chunk; - } - - Ok(()) -} - -fn dequantize_f32( - data: &PaddedCudaSlice, - dtype: GgmlDType, - elem_count: usize, - dev: &CudaDevice, -) -> Result { - let nb = elem_count.div_ceil(256); - let (kernel_name, is_k, block_dim, num_blocks) = match dtype { - GgmlDType::Q4_0 => ("dequantize_block_q4_0_f32", false, 32, nb), - GgmlDType::Q4_1 => ("dequantize_block_q4_1_f32", false, 32, nb), - GgmlDType::Q5_0 => ( - "dequantize_block_q5_0_f32", - false, - CUDA_DEQUANTIZE_BLOCK_SIZE, - ceil_div(elem_count, 2 * CUDA_DEQUANTIZE_BLOCK_SIZE), - ), - GgmlDType::Q5_1 => ( - "dequantize_block_q5_1_f32", - false, - CUDA_DEQUANTIZE_BLOCK_SIZE, - ceil_div(elem_count, 2 * CUDA_DEQUANTIZE_BLOCK_SIZE), - ), - GgmlDType::Q8_0 => ("dequantize_block_q8_0_f32", false, 32, nb), - GgmlDType::Q2K => ("dequantize_block_q2_K_f32", true, 64, nb), - GgmlDType::Q3K => ("dequantize_block_q3_K_f32", true, 64, nb), - GgmlDType::Q4K => ("dequantize_block_q4_K_f32", true, 32, nb), - GgmlDType::Q5K => ("dequantize_block_q5_K_f32", true, 64, nb), - GgmlDType::Q6K => ("dequantize_block_q6_K_f32", true, 64, nb), - GgmlDType::Q8K => ("dequantize_block_q8_K_f32", true, 32, nb), - _ => crate::bail!("unsupported dtype for dequantize {dtype:?}"), - }; - let func = dev.get_or_load_func(kernel_name, &candle_kernels::QUANTIZED)?; - let dst = unsafe { dev.alloc::(elem_count)? }; - // See e.g. - // https://github.com/ggerganov/llama.cpp/blob/cbbd1efa06f8c09f9dff58ff9d9af509cc4c152b/ggml-cuda.cu#L7270 - let cfg = cudarc::driver::LaunchConfig { - grid_dim: (num_blocks as u32, 1, 1), - block_dim: (block_dim as u32, 1, 1), - shared_mem_bytes: 0, - }; - - if is_k { - let mut builder = func.builder(); - builder.arg(&data.inner); - builder.arg(&dst); - unsafe { builder.launch(cfg) }.w()?; - } else { - let nb32 = match dtype { - GgmlDType::Q5_0 | GgmlDType::Q5_1 => elem_count, - _ => elem_count / 32, - }; - let mut builder = func.builder(); - builder.arg(&data.inner); - builder.arg(&dst); - barg!(builder, nb32 as i32); - unsafe { builder.launch(cfg) }.w()?; - } - Ok(CudaStorage::wrap_cuda_slice(dst, dev.clone())) -} - -fn dequantize_f16( - data: &PaddedCudaSlice, - dtype: GgmlDType, - elem_count: usize, - dev: &CudaDevice, -) -> Result { - let nb = elem_count.div_ceil(256); - let (kernel_name, is_k, block_dim, num_blocks) = match dtype { - GgmlDType::Q4_0 => ("dequantize_block_q4_0_f16", false, 32, nb), - GgmlDType::Q4_1 => ("dequantize_block_q4_1_f16", false, 32, nb), - GgmlDType::Q5_0 => ( - "dequantize_block_q5_0_f16", - false, - CUDA_DEQUANTIZE_BLOCK_SIZE, - ceil_div(elem_count, 2 * CUDA_DEQUANTIZE_BLOCK_SIZE), - ), - GgmlDType::Q5_1 => ( - "dequantize_block_q5_1_f16", - false, - CUDA_DEQUANTIZE_BLOCK_SIZE, - ceil_div(elem_count, 2 * CUDA_DEQUANTIZE_BLOCK_SIZE), - ), - GgmlDType::Q8_0 => ("dequantize_block_q8_0_f16", false, 32, nb), - GgmlDType::Q2K => ("dequantize_block_q2_K_f16", true, 64, nb), - GgmlDType::Q3K => ("dequantize_block_q3_K_f16", true, 64, nb), - GgmlDType::Q4K => ("dequantize_block_q4_K_f16", true, 32, nb), - GgmlDType::Q5K => ("dequantize_block_q5_K_f16", true, 64, nb), - GgmlDType::Q6K => ("dequantize_block_q6_K_f16", true, 64, nb), - GgmlDType::Q8K => ("dequantize_block_q8_K_f16", true, 32, nb), - _ => crate::bail!("unsupported dtype for dequantize {dtype:?}"), - }; - let func = dev.get_or_load_func(kernel_name, &candle_kernels::QUANTIZED)?; - let dst = unsafe { dev.alloc::(elem_count)? }; - // See e.g. - // https://github.com/ggerganov/llama.cpp/blob/cbbd1efa06f8c09f9dff58ff9d9af509cc4c152b/ggml-cuda.cu#L7270 - let cfg = cudarc::driver::LaunchConfig { - grid_dim: (num_blocks as u32, 1, 1), - block_dim: (block_dim as u32, 1, 1), - shared_mem_bytes: 0, - }; - - if is_k { - let mut builder = func.builder(); - builder.arg(&data.inner); - builder.arg(&dst); - unsafe { builder.launch(cfg) }.w()?; - } else { - let nb32 = match dtype { - GgmlDType::Q5_0 | GgmlDType::Q5_1 => elem_count, - _ => elem_count / 32, - }; - let mut builder = func.builder(); - builder.arg(&data.inner); - builder.arg(&dst); - barg!(builder, nb32 as i32); - unsafe { builder.launch(cfg) }.w()?; - } - Ok(CudaStorage::wrap_cuda_slice(dst, dev.clone())) -} - -fn dequantize_mul_mat_vec( - data: &PaddedCudaSlice, - y: &CudaView, - dtype: GgmlDType, - ncols: usize, - nrows: usize, - dev: &CudaDevice, -) -> Result { - let data_elems = data.len / dtype.type_size() * dtype.block_size(); - if data_elems < ncols * nrows { - crate::bail!("unexpected data size {}, ncols {ncols} {nrows}", data_elems) - } - if y.len() != ncols { - crate::bail!("unexpected y size {}, ncols {ncols} {nrows}", y.len()) - } - let kernel_name = match dtype { - GgmlDType::Q4_0 => "dequantize_mul_mat_vec_q4_0_cuda", - GgmlDType::Q4_1 => "dequantize_mul_mat_vec_q4_1_cuda", - GgmlDType::Q5_0 => "dequantize_mul_mat_vec_q5_0_cuda", - GgmlDType::Q5_1 => "dequantize_mul_mat_vec_q5_1_cuda", - GgmlDType::Q8_0 => "dequantize_mul_mat_vec_q8_0_cuda", - GgmlDType::Q2K => "dequantize_mul_mat_vec_q2_k", - GgmlDType::Q3K => "dequantize_mul_mat_vec_q3_k", - GgmlDType::Q4K => "dequantize_mul_mat_vec_q4_k", - GgmlDType::Q5K => "dequantize_mul_mat_vec_q5_k", - GgmlDType::Q6K => "dequantize_mul_mat_vec_q6_k", - _ => crate::bail!("unsupported dtype for quantized matmul {dtype:?}"), - }; - let func = dev.get_or_load_func(kernel_name, &candle_kernels::QUANTIZED)?; - let dst = unsafe { dev.alloc::(nrows)? }; - let block_num_y = ceil_div(nrows, GGML_CUDA_MMV_Y); - let cfg = cudarc::driver::LaunchConfig { - grid_dim: (block_num_y as u32, 1, 1), - block_dim: (WARP_SIZE as u32, GGML_CUDA_MMV_Y as u32, 1), - shared_mem_bytes: 0, - }; - - let mut builder = func.builder(); - builder.arg(&data.inner); - builder.arg(y); - builder.arg(&dst); - barg!(builder, ncols as i32, nrows as i32); - unsafe { builder.launch(cfg) }.w()?; - Ok(CudaStorage::wrap_cuda_slice(dst, dev.clone())) -} - -fn mul_mat_vec_via_q8_1( - data: &PaddedCudaSlice, - y: &CudaView, - dtype: GgmlDType, - ncols: usize, - nrows: usize, - b_size: usize, - dev: &CudaDevice, -) -> Result { - let data_elems = data.len / dtype.type_size() * dtype.block_size(); - if data_elems < ncols * nrows { - crate::bail!("unexpected data size {}, ncols {ncols} {nrows}", data_elems) - } - if y.len() != ncols * b_size { - crate::bail!("unexpected y size {}, ncols {ncols} {nrows}", y.len()) - } - if b_size == 0 || b_size > 8 { - crate::bail!("only bsize between 1 and 8 are supported, got {b_size}") - } - // Start by quantizing y - let ncols_padded = pad(ncols, MATRIX_ROW_PADDING); - let y_size_in_bytes = - b_size * ncols_padded * GgmlDType::Q8_1.type_size() / GgmlDType::Q8_1.block_size(); - let mut y_q8_1 = unsafe { dev.alloc::(y_size_in_bytes)? }; - quantize_q8_1(y, &mut y_q8_1, ncols, b_size, dev)?; - - let kernel_name = match dtype { - GgmlDType::Q4_0 => "mul_mat_vec_q4_0_q8_1_cuda", - GgmlDType::Q4_1 => "mul_mat_vec_q4_1_q8_1_cuda", - GgmlDType::Q5_0 => "mul_mat_vec_q5_0_q8_1_cuda", - GgmlDType::Q5_1 => "mul_mat_vec_q5_1_q8_1_cuda", - GgmlDType::Q8_0 => "mul_mat_vec_q8_0_q8_1_cuda", - GgmlDType::Q2K => "mul_mat_vec_q2_K_q8_1_cuda", - GgmlDType::Q3K => "mul_mat_vec_q3_K_q8_1_cuda", - GgmlDType::Q4K => "mul_mat_vec_q4_K_q8_1_cuda", - GgmlDType::Q5K => "mul_mat_vec_q5_K_q8_1_cuda", - GgmlDType::Q6K => "mul_mat_vec_q6_K_q8_1_cuda", - _ => crate::bail!("unsupported dtype for quantized matmul {dtype:?}"), - }; - let kernel_name = format!("{kernel_name}{b_size}"); - let func = dev.get_or_load_func(&kernel_name, &candle_kernels::QUANTIZED)?; - let dst = unsafe { dev.alloc::(nrows * b_size)? }; - // https://github.com/ggerganov/llama.cpp/blob/facb8b56f8fd3bb10a693bf0943ae9d69d0828ef/ggml-cuda/mmvq.cu#L98 - let (nblocks, nwarps) = match b_size { - 1 => (nrows as u32, 4), - 2..=4 => ((nrows as u32).div_ceil(2), 4), - 5..=8 => ((nrows as u32).div_ceil(2), 2), - _ => crate::bail!("unexpected bsize {b_size}"), - }; - let cfg = cudarc::driver::LaunchConfig { - grid_dim: (nblocks, 1, 1), - block_dim: (WARP_SIZE as u32, nwarps, 1), - shared_mem_bytes: 0, - }; - - let mut builder = func.builder(); - builder.arg(&data.inner); - builder.arg(&y_q8_1); - builder.arg(&dst); - barg!( - builder, - /* ncols_x */ ncols as i32, - /* nrows_x */ nrows as i32, - /* nrows_y */ ncols_padded as i32, - /* nrows_dst */ nrows as i32 - ); - unsafe { builder.launch(cfg) }.w()?; - Ok(CudaStorage::wrap_cuda_slice(dst, dev.clone())) -} - -#[allow(clippy::too_many_arguments)] -fn mul_mat_via_q8_1( - data: &PaddedCudaSlice, - y: &CudaView, - dtype: GgmlDType, - x_rows: usize, - x_cols: usize, - y_rows: usize, - y_cols: usize, - dev: &CudaDevice, -) -> Result { - let data_elems = data.len / dtype.type_size() * dtype.block_size(); - if data_elems < x_rows * x_cols { - crate::bail!("unexpected lhs size {}, {x_rows} {x_cols}", data_elems) - } - if y.len() != y_rows * y_cols { - crate::bail!("unexpected y size {}, {y_rows} {y_cols}", y.len()) - } - if x_cols != y_rows { - crate::bail!("unexpected x/y size {x_rows} {x_cols} {y_rows} {y_cols}") - } - let k = x_cols; - // Start by quantizing y - let k_padded = pad(k, MATRIX_ROW_PADDING); - let y_size_in_bytes = - k_padded * y_cols * GgmlDType::Q8_1.type_size() / GgmlDType::Q8_1.block_size(); - let mut y_q8_1 = unsafe { dev.alloc::(y_size_in_bytes)? }; - quantize_q8_1(y, &mut y_q8_1, k, y_cols, dev)?; - - let (kernel_name, mmq_x, mmq_y) = match dtype { - GgmlDType::Q4_0 => ("mul_mat_q4_0", 64, 128), - GgmlDType::Q4_1 => ("mul_mat_q4_1", 64, 128), - GgmlDType::Q5_0 => ("mul_mat_q5_0", 128, 64), - GgmlDType::Q5_1 => ("mul_mat_q5_1", 128, 64), - GgmlDType::Q8_0 => ("mul_mat_q8_0", 128, 64), - GgmlDType::Q2K => ("mul_mat_q2_K", 64, 128), - GgmlDType::Q3K => ("mul_mat_q3_K", 128, 128), - GgmlDType::Q4K => ("mul_mat_q4_K", 64, 128), - GgmlDType::Q5K => ("mul_mat_q5_K", 64, 128), - GgmlDType::Q6K => ("mul_mat_q6_K", 64, 64), - _ => crate::bail!("unsupported dtype for quantized matmul {dtype:?}"), - }; - let func = dev.get_or_load_func(kernel_name, &candle_kernels::QUANTIZED)?; - let dst = unsafe { dev.alloc::(x_rows * y_cols)? }; - let cfg = cudarc::driver::LaunchConfig { - grid_dim: ( - ceil_div(x_rows, mmq_y) as u32, - ceil_div(y_cols, mmq_x) as u32, - 1, - ), - block_dim: (WARP_SIZE as u32, 4, 1), - shared_mem_bytes: 0, - }; - - let mut builder = func.builder(); - builder.arg(/* vx */ &data.inner); - builder.arg(/* vy */ &y_q8_1); - builder.arg(/* dst */ &dst); - barg!( - builder, - /* ncols_x */ x_cols as i32, - /* nrows_x */ x_rows as i32, - /* ncols_y */ y_cols as i32, - /* nrows_y */ k_padded as i32, - /* nrows_dst */ x_rows as i32 - ); - unsafe { builder.launch(cfg) }.w()?; - Ok(CudaStorage::wrap_cuda_slice(dst, dev.clone())) -} - -#[allow(clippy::too_many_arguments)] -fn indexed_moe_forward_fused_q8_1_input( - weight: &CudaView, - w_shape: &crate::Shape, //[num_experts, n, k] - w_dtype: GgmlDType, - input: &CudaSlice, - in_shape: &crate::Shape, //[batch, topk or 1, k] - ids: &CudaView, - idx_shape: &crate::Shape, //[batch, topk] - dev: &CudaDevice, -) -> Result<(CudaStorage, crate::Shape)> { - let (_, n, k) = w_shape.dims3()?; - let batch = in_shape.dims()[0]; - let input_dim1 = in_shape.dims()[1]; - - let topk = idx_shape.dims()[1]; - assert!(batch == idx_shape.dims()[0], "batch dim not match!"); - - // Quantize input into q8_1. - let total_rows = batch * input_dim1; - let k_padded = pad(k, MATRIX_ROW_PADDING); - // Get Q8_1 metadata. - let q8_1_block_size = GgmlDType::Q8_1.block_size(); - let q8_1_type_size = GgmlDType::Q8_1.type_size(); - - // Calculate the size of the output buffer in bytes. - let num_blocks_per_row = k_padded / q8_1_block_size; - let dst_row_size_bytes = num_blocks_per_row * q8_1_type_size; - let y_size_in_bytes = total_rows * dst_row_size_bytes; - let mut input_quant = unsafe { dev.alloc::(y_size_in_bytes)? }; - - let input_view = input.slice(0..); - quantize_q8_1(&input_view, &mut input_quant, k, total_rows, dev)?; - - // output buffer - let outsize = batch * topk * n; - let out = unsafe { dev.alloc::(outsize)? }; - - let kernel_name = match w_dtype { - GgmlDType::Q2K => "indexed_moe_forward_q2k_q8_1", - GgmlDType::Q3K => "indexed_moe_forward_q3k_q8_1", - GgmlDType::Q4K => "indexed_moe_forward_q4k_q8_1", - GgmlDType::Q5K => "indexed_moe_forward_q5k_q8_1", - GgmlDType::Q6K => "indexed_moe_forward_q6k_q8_1", - GgmlDType::Q8_0 => "indexed_moe_forward_q8_0_q8_1", - _ => crate::bail!("unsupported dtype for indexed_moe_forward {w_dtype:?}"), - }; - let func = dev.get_or_load_func(kernel_name, &candle_kernels::QUANTIZED)?; - let (nblocks, nwarps) = (n as u32, 4); - let cfg = cudarc::driver::LaunchConfig { - grid_dim: (nblocks, batch as u32, topk as u32), - block_dim: (WARP_SIZE as u32, nwarps, 1), - shared_mem_bytes: 0, - }; - - let mut builder = func.builder(); - builder.arg(weight); - builder.arg(&input_quant); - builder.arg(ids); - builder.arg(&out); - - barg!( - builder, - n as i32, - k as i32, - batch as i32, - topk as i32, - k_padded as i32, - input_dim1 as i32 - ); - unsafe { builder.launch(cfg) }.w()?; - - let mut out_shape = in_shape.dims().to_vec(); - out_shape.pop(); - out_shape.push(n); - out_shape[1] = topk; - Ok(( - CudaStorage::wrap_cuda_slice(out, dev.clone()), - out_shape.into(), - )) -} - -impl QCudaStorage { - pub fn indexed_moe_forward( - &self, - self_shape: &crate::Shape, //[num_experts, n, k] - input: &CudaStorage, //[batch, topk or 1, k] - input_l: &crate::Layout, - ids: &CudaStorage, //[batch, topk] - ids_l: &crate::Layout, - ) -> Result<(CudaStorage, crate::Shape)> { - if matches!( - self.dtype(), - GgmlDType::Q8_0 - | GgmlDType::Q2K - | GgmlDType::Q3K - | GgmlDType::Q4K - | GgmlDType::Q5K - | GgmlDType::Q6K - ) { - let input_storage = input.as_cuda_slice::()?; - let ids_storage = ids.as_cuda_slice::()?; - indexed_moe_forward_fused_q8_1_input( - &self.data.inner.slice(0..), - self_shape, //[num_experts, n, k] - self.dtype(), - input_storage, - input_l.shape(), //[batch, topk or 1, k] - &ids_storage.slice(0..), - ids_l.shape(), //[batch, topk] - &self.device, - ) - } else { - crate::bail!( - "The given quantized dtype {:?} is not supported for indexed_moe_forward!", - self.dtype() - ); - } - } - - pub fn zeros(device: &CudaDevice, el_count: usize, dtype: GgmlDType) -> Result { - let size_in_bytes = ceil_div(el_count, dtype.block_size()) * dtype.type_size(); - let padded_size_in_bytes = - ceil_div(el_count + MATRIX_ROW_PADDING, dtype.block_size()) * dtype.type_size(); - let inner = device.alloc_zeros::(padded_size_in_bytes)?; - Ok(QCudaStorage { - data: PaddedCudaSlice { - inner, - len: size_in_bytes, - }, - device: device.clone(), - dtype, - }) - } - - pub fn dtype(&self) -> GgmlDType { - self.dtype - } - - pub fn device(&self) -> &CudaDevice { - &self.device - } - - pub fn dequantize(&self, elem_count: usize) -> Result { - fn deq(buffer: &[u8], n: usize, dst: &mut [f32]) { - let slice = unsafe { std::slice::from_raw_parts(buffer.as_ptr() as *const T, n) }; - let vec = slice.to_vec(); - T::to_float(&vec, dst) - } - - let fast_kernel = matches!( - self.dtype, - GgmlDType::Q4_0 - | GgmlDType::Q4_1 - | GgmlDType::Q5_0 - | GgmlDType::Q5_1 - | GgmlDType::Q8_0 - | GgmlDType::Q2K - | GgmlDType::Q3K - | GgmlDType::Q4K - | GgmlDType::Q5K - | GgmlDType::Q6K - | GgmlDType::Q8K - ); - if fast_kernel { - return dequantize_f32(&self.data, self.dtype, elem_count, self.device()); - } - // Run the dequantization on cpu. - - let buffer = self - .device - .clone_dtoh(&self.data.inner.slice(..self.data.len))?; - let mut out = vec![0.0; elem_count]; - let block_len = elem_count / self.dtype.block_size(); - match self.dtype { - GgmlDType::F32 => deq::(&buffer, block_len, &mut out), - GgmlDType::F16 => deq::(&buffer, block_len, &mut out), - GgmlDType::BF16 => deq::(&buffer, block_len, &mut out), - GgmlDType::Q4_0 => deq::(&buffer, block_len, &mut out), - GgmlDType::Q4_1 => deq::(&buffer, block_len, &mut out), - GgmlDType::Q5_0 => deq::(&buffer, block_len, &mut out), - GgmlDType::Q5_1 => deq::(&buffer, block_len, &mut out), - GgmlDType::Q8_0 => deq::(&buffer, block_len, &mut out), - GgmlDType::Q8_1 => deq::(&buffer, block_len, &mut out), - GgmlDType::Q2K => deq::(&buffer, block_len, &mut out), - GgmlDType::Q3K => deq::(&buffer, block_len, &mut out), - GgmlDType::Q4K => deq::(&buffer, block_len, &mut out), - GgmlDType::Q5K => deq::(&buffer, block_len, &mut out), - GgmlDType::Q6K => deq::(&buffer, block_len, &mut out), - GgmlDType::Q8K => deq::(&buffer, block_len, &mut out), - } - - self.device - .storage_from_cpu_storage(&crate::CpuStorage::F32(out)) - } - - pub fn dequantize_f16(&self, elem_count: usize) -> Result { - dequantize_f16(&self.data, self.dtype, elem_count, self.device()) - } - - pub fn quantize(&mut self, src: &CudaStorage) -> Result<()> { - // Run the quantization on cpu. - let src = match &src.slice { - crate::cuda_backend::CudaStorageSlice::F32(data) => self.device.clone_dtoh(data)?, - _ => crate::bail!("only f32 can be quantized"), - }; - let src_len = src.len(); - let src = crate::Storage::Cpu(crate::CpuStorage::F32(src)); - let mut qcpu_storage = crate::Device::Cpu.qzeros(src_len, self.dtype)?; - qcpu_storage.quantize(&src)?; - let data = qcpu_storage.data()?; - let padded_len = - data.len() + MATRIX_ROW_PADDING * self.dtype.type_size() / self.dtype.block_size(); - let mut inner = unsafe { self.device.alloc::(padded_len)? }; - self.device - .memcpy_htod(&*data, &mut inner.slice_mut(..data.len()))?; - self.data = PaddedCudaSlice { - inner, - len: data.len(), - }; - Ok(()) - } - - pub fn quantize_imatrix( - &mut self, - src: &CudaStorage, - imatrix_weights: &[f32], - n_per_row: usize, - ) -> Result<()> { - // Run the quantization on cpu. - let src = match &src.slice { - crate::cuda_backend::CudaStorageSlice::F32(data) => self.device.clone_dtoh(data)?, - _ => crate::bail!("only f32 can be quantized"), - }; - let src_len = src.len(); - let src = crate::Storage::Cpu(crate::CpuStorage::F32(src)); - let mut qcpu_storage = crate::Device::Cpu.qzeros(src_len, self.dtype)?; - qcpu_storage.quantize_imatrix(&src, imatrix_weights, n_per_row)?; - let data = qcpu_storage.data()?; - let padded_len = - data.len() + MATRIX_ROW_PADDING * self.dtype.type_size() / self.dtype.block_size(); - let mut inner = unsafe { self.device.alloc::(padded_len)? }; - self.device - .memcpy_htod(&*data, &mut inner.slice_mut(..data.len()))?; - self.data = PaddedCudaSlice { - inner, - len: data.len(), - }; - Ok(()) - } - - pub fn quantize_imatrix_onto( - &mut self, - src: &crate::CpuStorage, - imatrix_weights: &[f32], - n_per_row: usize, - ) -> Result<()> { - // Run the quantization on cpu. - let src_len = src.as_slice::()?.len(); - let mut qcpu_storage = crate::Device::Cpu.qzeros(src_len, self.dtype)?; - - if let QStorage::Cpu(storage) = &mut qcpu_storage { - storage.from_float_imatrix(src.as_slice::()?, imatrix_weights, n_per_row); - } else { - unreachable!() - } - - let data = qcpu_storage.data()?; - let padded_len = - data.len() + MATRIX_ROW_PADDING * self.dtype.type_size() / self.dtype.block_size(); - let mut inner = unsafe { self.device.alloc::(padded_len)? }; - self.device - .memcpy_htod(&*data, &mut inner.slice_mut(..data.len()))?; - self.data = PaddedCudaSlice { - inner, - len: data.len(), - }; - Ok(()) - } - - pub fn quantize_onto(&mut self, src: &crate::CpuStorage) -> Result<()> { - // Run the quantization on cpu. - let src_len = src.as_slice::()?.len(); - let mut qcpu_storage = crate::Device::Cpu.qzeros(src_len, self.dtype)?; - - if let QStorage::Cpu(storage) = &mut qcpu_storage { - storage.from_float(src.as_slice::()?); - } else { - unreachable!() - } - - let data = qcpu_storage.data()?; - let padded_len = - data.len() + MATRIX_ROW_PADDING * self.dtype.type_size() / self.dtype.block_size(); - let mut inner = unsafe { self.device.alloc::(padded_len)? }; - self.device - .memcpy_htod(&*data, &mut inner.slice_mut(..data.len()))?; - self.data = PaddedCudaSlice { - inner, - len: data.len(), - }; - Ok(()) - } - - pub fn storage_size_in_bytes(&self) -> usize { - self.data.len - } - - pub fn fwd( - &self, - self_shape: &crate::Shape, - storage: &CudaStorage, - layout: &crate::Layout, - ) -> Result<(CudaStorage, crate::Shape)> { - let max_bm = if FORCE_DMMV.load(std::sync::atomic::Ordering::Relaxed) { - 1 - } else { - 8 - }; - let use_vec_kernel = match layout.shape().dims() { - [b, m, _k] => b * m <= max_bm, - [b, _k] => *b <= max_bm, - _ => false, - }; - if use_vec_kernel { - self.dequantize_matmul_vec(self_shape, storage, layout) - } else { - self.dequantize_matmul(self_shape, storage, layout) - } - } - - pub fn data(&self) -> Result> { - let mut out = vec![0u8; self.data.len]; - self.device - .memcpy_dtoh(&self.data.inner.slice(..self.data.len), &mut out)?; - Ok(out) - } - - pub fn device_ptr(&self) -> Result<*const u8> { - use cudarc::driver::DevicePtr; - Ok(self.data.inner.device_ptr(self.data.inner.stream()).0 as *const u8) - } -} - -impl QCudaStorage { - fn dequantize_matmul_vec( - &self, - self_shape: &crate::Shape, - rhs: &CudaStorage, - rhs_l: &crate::Layout, - ) -> Result<(CudaStorage, crate::Shape)> { - let (nrows, ncols) = self_shape.dims2()?; - let rhs = rhs.as_cuda_slice::()?; - let rhs = match rhs_l.contiguous_offsets() { - Some((o1, o2)) => rhs.slice(o1..o2), - None => Err(crate::Error::RequiresContiguous { op: "dmmv" }.bt())?, - }; - let (b_size, k) = match rhs_l.shape().dims() { - [b, m, k] => (b * m, *k), - [b, k] => (*b, *k), - _ => crate::bail!("unexpected rhs shape in dmmv {:?}", rhs_l.shape()), - }; - if ncols != k { - crate::bail!("mismatch on matmul dim {self_shape:?} {:?}", rhs_l.shape()) - } - - let out = if FORCE_DMMV.load(std::sync::atomic::Ordering::Relaxed) { - dequantize_mul_mat_vec(&self.data, &rhs, self.dtype, ncols, nrows, self.device())? - } else { - mul_mat_vec_via_q8_1( - &self.data, - &rhs, - self.dtype, - ncols, - nrows, - b_size, - self.device(), - )? - }; - let mut out_shape = rhs_l.shape().dims().to_vec(); - out_shape.pop(); - out_shape.push(nrows); - Ok((out, out_shape.into())) - } - - fn dequantize_matmul( - &self, - self_shape: &crate::Shape, - storage: &CudaStorage, - layout: &crate::Layout, - ) -> Result<(CudaStorage, crate::Shape)> { - use crate::backend::BackendStorage; - let (n, k) = self_shape.dims2()?; - let (b, m, k2) = match layout.shape().dims() { - &[b, m, k2] => (b, m, k2), - &[m, k2] => (1, m, k2), - s => crate::bail!("unexpected shape for input {s:?}"), - }; - if k2 != k { - crate::bail!("mismatch on matmul dim {self_shape:?} {:?}", layout.shape()) - } - - let out = if FORCE_DMMV.load(std::sync::atomic::Ordering::Relaxed) { - let data_f32 = self.dequantize(n * k)?; - let rhs_l = crate::Layout::new((k, n).into(), vec![1, k], 0).broadcast_as((b, k, n))?; - storage.matmul(&data_f32, (b, m, n, k), layout, &rhs_l)? - } else { - let storage = storage.as_cuda_slice::()?; - let storage = match layout.contiguous_offsets() { - Some((o1, o2)) => storage.slice(o1..o2), - None => Err(crate::Error::RequiresContiguous { - op: "quantized-matmul", - } - .bt())?, - }; - mul_mat_via_q8_1( - &self.data, - &storage, - self.dtype, - /* x_rows */ n, - /* x_cols */ k, - /* y_rows */ k, - /* y_cols */ b * m, - self.device(), - )? - }; - let mut out_shape = layout.shape().dims().to_vec(); - out_shape.pop(); - out_shape.push(n); - Ok((out, out_shape.into())) - } -} - -pub fn load_quantized( - device: &CudaDevice, - data: &[T], -) -> Result { - let data = unsafe { - std::slice::from_raw_parts(data.as_ptr() as *const u8, core::mem::size_of_val(data)) - }; - let dtype = T::DTYPE; - let padded_len = data.len() + MATRIX_ROW_PADDING * dtype.type_size() / dtype.block_size(); - let mut inner = unsafe { device.alloc::(padded_len)? }; - device.memcpy_htod(data, &mut inner.slice_mut(..data.len()))?; - Ok(QStorage::Cuda(QCudaStorage { - data: PaddedCudaSlice { - inner, - len: data.len(), - }, - device: device.clone(), - dtype, - })) -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn cuda_quantize_q8_1() -> Result<()> { - let dev = CudaDevice::new(0)?; - let el = 256; - let el_padded = pad(el, MATRIX_ROW_PADDING); - let y_size_in_bytes = - el_padded * GgmlDType::Q8_1.type_size() / GgmlDType::Q8_1.block_size(); - let mut y_q8_1 = unsafe { dev.alloc::(y_size_in_bytes)? }; - let vs: Vec = (0..el).map(|v| v as f32).collect(); - let y = dev.clone_htod(&vs)?; - quantize_q8_1(&y.as_view(), &mut y_q8_1, el, 1, &dev)?; - Ok(()) - } - - #[test] - fn cuda_mmv_q8_1() -> Result<()> { - let dev = CudaDevice::new(0)?; - let ncols = 256; - let vs: Vec = (0..ncols).map(|v| v as f32).collect(); - let y = dev.clone_htod(&vs)?; - let mut xs = QCudaStorage::zeros(&dev, ncols, GgmlDType::Q4_0)?; - xs.quantize(&CudaStorage::wrap_cuda_slice(y.clone(), dev.clone()))?; - let cuda_storage = mul_mat_vec_via_q8_1( - &xs.data, - &y.as_view(), - /* dtype */ GgmlDType::Q4_0, - /* ncols */ ncols, - /* nrows */ 1, - /* b_size */ 1, - &dev, - )?; - let vs = cuda_storage.as_cuda_slice::()?; - let vs = dev.clone_dtoh(&vs.as_view())?; - assert_eq!(vs.len(), 1); - // for n = 255, n.(n+1).(2n+1) / 6 = 5559680 - // Q8 means 1/256 precision. - assert_eq!(vs[0], 5561664.5); - - let cuda_storage = dequantize_mul_mat_vec( - &xs.data, - &y.as_view(), - /* dtype */ GgmlDType::Q4_0, - /* ncols */ ncols, - /* nrows */ 1, - &dev, - )?; - let vs = cuda_storage.as_cuda_slice::()?; - let vs = dev.clone_dtoh(&vs.as_view())?; - assert_eq!(vs.len(), 1); - assert_eq!(vs[0], 5561851.0); - Ok(()) - } - - #[test] - fn cuda_mm_q8_1() -> Result<()> { - let dev = CudaDevice::new(0)?; - let ncols = 256; - let vs: Vec = (0..ncols * 4).map(|v| v as f32 / 4.).collect(); - let y = dev.clone_htod(&vs)?; - let mut xs = QCudaStorage::zeros(&dev, ncols * 4, GgmlDType::Q4_0)?; - xs.quantize(&CudaStorage::wrap_cuda_slice(y.clone(), dev.clone()))?; - let cuda_storage = mul_mat_via_q8_1( - &xs.data, - &y.as_view(), - /* dtype */ GgmlDType::Q4_0, - /* x_rows */ 4, - /* x_cols */ ncols, - /* y_rows */ ncols, - /* y_cols */ 4, - &dev, - )?; - let vs = cuda_storage.as_cuda_slice::()?; - let vs = dev.clone_dtoh(&vs.as_view())?; - - /* - x = torch.tensor([float(v) for v in range(1024)]).reshape(4, 256) - x @ x.t() / 16 - tensor([[ 347480.0000, 869720.0000, 1391960.0000, 1914200.0000], - [ 869720.0000, 2440536.0000, 4011352.0000, 5582166.5000], - [ 1391960.0000, 4011352.0000, 6630742.0000, 9250132.0000], - [ 1914200.0000, 5582166.5000, 9250132.0000, 12918099.0000]]) - */ - assert_eq!(vs.len(), 16); - assert_eq!(vs[0], 347604.0); - assert_eq!(vs[1], 888153.06); - assert_eq!(vs[4], 869780.7); - assert_eq!(vs[5], 2483145.0); - assert_eq!(vs[11], 9407368.0); - assert_eq!(vs[14], 9470856.0); - assert_eq!(vs[15], 13138824.0); - Ok(()) - } - - // The following test used to fail under compute-sanitizer until #2526. - #[test] - fn cuda_mm_q8_1_pad() -> Result<()> { - let dev = CudaDevice::new(0)?; - let (x_rows, ncols, y_cols) = (4, 16, 2048); - let vs: Vec = (0..ncols * y_cols).map(|v| v as f32 / 256.).collect(); - let y = dev.clone_htod(&vs)?; - let mut xs = QCudaStorage::zeros(&dev, ncols * x_rows, GgmlDType::Q4_0)?; - xs.quantize(&CudaStorage::wrap_cuda_slice(y.clone(), dev.clone()))?; - let cuda_storage = mul_mat_via_q8_1( - &xs.data, - &y.as_view(), - /* dtype */ GgmlDType::Q4_0, - /* x_rows */ x_rows, - /* x_cols */ ncols, - /* y_rows */ ncols, - /* y_cols */ y_cols, - &dev, - )?; - let vs = cuda_storage.as_cuda_slice::()?; - let _vs = dev.clone_dtoh(&vs.as_view())?; - Ok(()) - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/dummy_cuda.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/dummy_cuda.rs deleted file mode 100644 index 04f19f9fcb..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/quantized/dummy_cuda.rs +++ /dev/null @@ -1,95 +0,0 @@ -#![allow(unused)] -use super::GgmlDType; -use crate::{CudaDevice, CudaStorage, Error, Result}; - -pub struct QCudaStorage { - dtype: GgmlDType, - device: CudaDevice, -} - -impl QCudaStorage { - pub fn zeros(_: &CudaDevice, _: usize, _: GgmlDType) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - pub fn dtype(&self) -> GgmlDType { - self.dtype - } - - pub fn device(&self) -> &CudaDevice { - &self.device - } - - pub fn dequantize(&self, _elem_count: usize) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - pub fn dequantize_f16(&self, _elem_count: usize) -> Result { - Err(Error::NotCompiledWithCudaSupport) - } - - pub fn quantize(&mut self, _src: &CudaStorage) -> Result<()> { - Err(Error::NotCompiledWithCudaSupport) - } - - pub fn quantize_imatrix( - &mut self, - _src: &CudaStorage, - _imatrix_weights: &[f32], - _n_per_row: usize, - ) -> Result<()> { - Err(Error::NotCompiledWithCudaSupport) - } - - pub fn quantize_imatrix_onto( - &mut self, - _src: &crate::CpuStorage, - _imatrix_weights: &[f32], - _n_per_row: usize, - ) -> Result<()> { - Err(Error::NotCompiledWithCudaSupport) - } - - pub fn quantize_onto(&mut self, _src: &crate::CpuStorage) -> Result<()> { - Err(Error::NotCompiledWithCudaSupport) - } - - pub fn device_ptr(&self) -> Result<*const u8> { - Err(Error::NotCompiledWithCudaSupport) - } - - pub fn storage_size_in_bytes(&self) -> usize { - 0 - } - - pub fn fwd( - &self, - _self_shape: &crate::Shape, - _storage: &CudaStorage, - _layout: &crate::Layout, - ) -> Result<(CudaStorage, crate::Shape)> { - Err(Error::NotCompiledWithCudaSupport) - } - - pub fn data(&self) -> Result> { - Err(Error::NotCompiledWithCudaSupport) - } - - pub fn indexed_moe_forward( - &self, - _: &crate::Shape, - _: &CudaStorage, - _: &crate::Layout, - _: &CudaStorage, - _: &crate::Layout, - ) -> Result<(CudaStorage, crate::Shape)> { - Err(Error::NotCompiledWithCudaSupport) - } -} - -pub fn load_quantized( - _device: &CudaDevice, - _data: &[T], -) -> Result { - Err(Error::NotCompiledWithCudaSupport) -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/dummy_metal.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/dummy_metal.rs deleted file mode 100644 index 6f470e9099..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/quantized/dummy_metal.rs +++ /dev/null @@ -1,87 +0,0 @@ -#![allow(unused)] -use super::GgmlDType; -use crate::{Error, MetalDevice, MetalStorage, Result}; - -pub struct QMetalStorage { - dtype: GgmlDType, - device: MetalDevice, -} - -impl QMetalStorage { - pub fn zeros(_: &MetalDevice, _: usize, _: GgmlDType) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - pub fn dtype(&self) -> GgmlDType { - self.dtype - } - - pub fn device(&self) -> &MetalDevice { - &self.device - } - - pub fn dequantize(&self, _elem_count: usize) -> Result { - Err(Error::NotCompiledWithMetalSupport) - } - - pub fn quantize(&mut self, _src: &MetalStorage) -> Result<()> { - Err(Error::NotCompiledWithMetalSupport) - } - - pub fn quantize_imatrix( - &mut self, - _src: &MetalStorage, - _imatrix_weights: &[f32], - _n_per_row: usize, - ) -> Result<()> { - Err(Error::NotCompiledWithMetalSupport) - } - - pub fn quantize_imatrix_onto( - &mut self, - _src: &crate::CpuStorage, - _imatrix_weights: &[f32], - _n_per_row: usize, - ) -> Result<()> { - Err(Error::NotCompiledWithMetalSupport) - } - - pub fn quantize_onto(&mut self, _src: &crate::CpuStorage) -> Result<()> { - Err(Error::NotCompiledWithCudaSupport) - } - - pub fn storage_size_in_bytes(&self) -> usize { - 0 - } - - pub fn fwd( - &self, - _self_shape: &crate::Shape, - _storage: &MetalStorage, - _layout: &crate::Layout, - ) -> Result<(MetalStorage, crate::Shape)> { - Err(Error::NotCompiledWithMetalSupport) - } - - pub fn data(&self) -> Result> { - Err(Error::NotCompiledWithMetalSupport) - } - - pub fn indexed_moe_forward( - &self, - _: &crate::Shape, - _: &MetalStorage, - _: &crate::Layout, - _: &MetalStorage, - _: &crate::Layout, - ) -> Result<(MetalStorage, crate::Shape)> { - Err(Error::NotCompiledWithMetalSupport) - } -} - -pub fn load_quantized( - _device: &MetalDevice, - _data: &[T], -) -> Result { - Err(Error::NotCompiledWithMetalSupport) -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/ggml_file.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/ggml_file.rs deleted file mode 100644 index ea5ec02578..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/quantized/ggml_file.rs +++ /dev/null @@ -1,266 +0,0 @@ -//! Support for the GGML file format. - -use super::{k_quants, GgmlDType, QStorage}; -use crate::{Device, Result}; -use byteorder::{LittleEndian, ReadBytesExt}; -use std::collections::HashMap; - -// https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/llama.h#L37 -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Magic { - Ggjt, - Ggla, - Ggmf, - Ggml, - Ggsn, -} - -impl TryFrom for Magic { - type Error = crate::Error; - fn try_from(value: u32) -> Result { - let magic = match value { - 0x67676a74 => Self::Ggjt, - 0x67676c61 => Self::Ggla, - 0x67676d66 => Self::Ggmf, - 0x67676d6c => Self::Ggml, - 0x6767736e => Self::Ggsn, - _ => crate::bail!("unknown magic {value:08x}"), - }; - Ok(magic) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum VersionedMagic { - GgmlUnversioned, - GgmfV1, - GgjtV1, - GgjtV2, - GgjtV3, -} - -impl VersionedMagic { - fn read(reader: &mut R) -> Result { - let magic = reader.read_u32::()?; - let magic = Magic::try_from(magic)?; - if magic == Magic::Ggml { - return Ok(Self::GgmlUnversioned); - } - let version = reader.read_u32::()?; - let versioned_magic = match (magic, version) { - (Magic::Ggmf, 1) => Self::GgmfV1, - (Magic::Ggjt, 1) => Self::GgjtV1, - (Magic::Ggjt, 2) => Self::GgjtV2, - (Magic::Ggjt, 3) => Self::GgjtV3, - _ => crate::bail!("ggml: unsupported magic/version {magic:?}/{version}"), - }; - Ok(versioned_magic) - } - - fn align32(&self) -> bool { - match self { - Self::GgmlUnversioned | Self::GgmfV1 => false, - Self::GgjtV1 | Self::GgjtV2 | Self::GgjtV3 => true, - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq)] -pub struct HParams { - pub n_vocab: u32, - pub n_embd: u32, - pub n_mult: u32, - pub n_head: u32, - pub n_layer: u32, - pub n_rot: u32, - pub ftype: u32, -} - -impl HParams { - fn read(reader: &mut R) -> Result { - let n_vocab = reader.read_u32::()?; - let n_embd = reader.read_u32::()?; - let n_mult = reader.read_u32::()?; - let n_head = reader.read_u32::()?; - let n_layer = reader.read_u32::()?; - let n_rot = reader.read_u32::()?; - let ftype = reader.read_u32::()?; - Ok(Self { - n_vocab, - n_embd, - n_mult, - n_head, - n_layer, - n_rot, - ftype, - }) - } -} - -#[derive(Debug, Clone, PartialEq)] -pub struct Vocab { - pub token_score_pairs: Vec<(Vec, f32)>, -} - -impl Vocab { - fn read(reader: &mut R, n_vocab: usize) -> Result { - // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/llama.cpp#L556 - let mut token_score_pairs = Vec::with_capacity(n_vocab); - for _index in 0..n_vocab { - let len = reader.read_u32::()? as usize; - let mut word = vec![0u8; len]; - reader.read_exact(&mut word)?; - let score = reader.read_f32::()?; - token_score_pairs.push((word, score)) - } - Ok(Self { token_score_pairs }) - } -} - -fn from_raw_data( - raw_data: &[u8], - size_in_bytes: usize, - dims: Vec, - device: &Device, -) -> Result { - let raw_data_ptr = raw_data.as_ptr(); - let n_blocks = size_in_bytes / std::mem::size_of::(); - let data = unsafe { std::slice::from_raw_parts(raw_data_ptr as *const T, n_blocks) }; - let data: QStorage = match device { - Device::Cpu => QStorage::Cpu(Box::new(data.to_vec())), - Device::Metal(metal) => super::metal::load_quantized(metal, data)?, - Device::Cuda(cuda) => super::cuda::load_quantized(cuda, data)?, - }; - super::QTensor::new(data, dims) -} - -/// Creates a [Tensor] from a raw GGML tensor. -pub fn qtensor_from_ggml( - ggml_dtype: GgmlDType, - raw_data: &[u8], - dims: Vec, - device: &Device, -) -> Result { - let tensor_elems = dims.iter().product::(); - let block_size = ggml_dtype.block_size(); - if tensor_elems % block_size != 0 { - crate::bail!( - "the number of elements {tensor_elems} is not divisible by the block size {block_size}" - ) - } - let size_in_bytes = tensor_elems / block_size * ggml_dtype.type_size(); - - match ggml_dtype { - GgmlDType::F32 => from_raw_data::(raw_data, size_in_bytes, dims, device), - GgmlDType::F16 => from_raw_data::(raw_data, size_in_bytes, dims, device), - GgmlDType::BF16 => from_raw_data::(raw_data, size_in_bytes, dims, device), - GgmlDType::Q4_0 => { - from_raw_data::(raw_data, size_in_bytes, dims, device) - } - GgmlDType::Q4_1 => { - from_raw_data::(raw_data, size_in_bytes, dims, device) - } - GgmlDType::Q5_0 => { - from_raw_data::(raw_data, size_in_bytes, dims, device) - } - GgmlDType::Q5_1 => { - from_raw_data::(raw_data, size_in_bytes, dims, device) - } - GgmlDType::Q8_0 => { - from_raw_data::(raw_data, size_in_bytes, dims, device) - } - GgmlDType::Q2K => { - from_raw_data::(raw_data, size_in_bytes, dims, device) - } - GgmlDType::Q3K => { - from_raw_data::(raw_data, size_in_bytes, dims, device) - } - GgmlDType::Q4K => { - from_raw_data::(raw_data, size_in_bytes, dims, device) - } - GgmlDType::Q5K => { - from_raw_data::(raw_data, size_in_bytes, dims, device) - } - GgmlDType::Q6K => { - from_raw_data::(raw_data, size_in_bytes, dims, device) - } - _ => crate::bail!("quantized type {ggml_dtype:?} is not supported yet"), - } -} - -fn read_one_tensor( - reader: &mut R, - magic: VersionedMagic, - device: &Device, -) -> Result<(String, super::QTensor)> { - let n_dims = reader.read_u32::()?; - let name_len = reader.read_u32::()?; - let ggml_dtype = reader.read_u32::()?; - let ggml_dtype = GgmlDType::from_u32(ggml_dtype)?; - let mut dims = vec![0u32; n_dims as usize]; - reader.read_u32_into::(&mut dims)?; - // The dimensions are stored in reverse order, see for example: - // https://github.com/ggerganov/llama.cpp/blob/b5ffb2849d23afe73647f68eec7b68187af09be6/convert.py#L969 - dims.reverse(); - let mut name = vec![0u8; name_len as usize]; - reader.read_exact(&mut name)?; - let name = String::from_utf8_lossy(&name).into_owned(); - - if magic.align32() { - let pos = reader.stream_position()?; - reader.seek(std::io::SeekFrom::Current(((32 - pos % 32) % 32) as i64))?; - } - let dims = dims.iter().map(|&u| u as usize).collect::>(); - let tensor_elems = dims.iter().product::(); - let size_in_bytes = tensor_elems * ggml_dtype.type_size() / ggml_dtype.block_size(); - // TODO: Mmap version to avoid copying the data around? - let mut raw_data = vec![0u8; size_in_bytes]; - reader.read_exact(&mut raw_data)?; - match qtensor_from_ggml(ggml_dtype, &raw_data, dims, device) { - Ok(tensor) => Ok((name, tensor)), - Err(e) => crate::bail!("Error creating tensor {name}: {e}"), - } -} - -pub struct Content { - pub magic: VersionedMagic, - pub hparams: HParams, - pub vocab: Vocab, - pub tensors: HashMap, - pub device: Device, -} - -impl Content { - pub fn read( - reader: &mut R, - device: &Device, - ) -> Result { - // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/llama.cpp#L505 - let last_position = reader.seek(std::io::SeekFrom::End(0))?; - reader.seek(std::io::SeekFrom::Start(0))?; - let magic = VersionedMagic::read(reader)?; - let hparams = HParams::read(reader)?; - let vocab = Vocab::read(reader, hparams.n_vocab as usize)?; - let mut tensors = HashMap::new(); - - while reader.stream_position()? != last_position { - let (name, tensor) = read_one_tensor(reader, magic, device)?; - tensors.insert(name, tensor); - } - let device = device.clone(); - Ok(Self { - magic, - hparams, - vocab, - tensors, - device, - }) - } - - pub fn remove(&mut self, name: &str) -> Result { - match self.tensors.remove(name) { - None => crate::bail!("cannot find tensor with name '{name}'"), - Some(tensor) => Ok(tensor), - } - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/gguf_file.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/gguf_file.rs deleted file mode 100644 index 197e43cfe3..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/quantized/gguf_file.rs +++ /dev/null @@ -1,539 +0,0 @@ -//! Support for the [GGUF file format](https://github.com/philpax/ggml/blob/gguf-spec/docs/gguf.md). -//! -//! Spec: https://github.com/ggml-org/ggml/blob/master/docs/gguf.md - -use super::{GgmlDType, QTensor}; -use crate::{Context, Device, Result}; -use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; -use std::collections::HashMap; - -pub const DEFAULT_ALIGNMENT: u64 = 32; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Magic { - Gguf, -} - -impl TryFrom for Magic { - type Error = crate::Error; - fn try_from(value: u32) -> Result { - let magic = match value { - 0x46554747 | 0x47475546 => Self::Gguf, - _ => crate::bail!("unknown magic 0x{value:08x}"), - }; - Ok(magic) - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum VersionedMagic { - GgufV1, - GgufV2, - GgufV3, -} - -impl VersionedMagic { - fn read(reader: &mut R) -> Result { - let magic = reader.read_u32::()?; - let magic = Magic::try_from(magic)?; - let version = reader.read_u32::()?; - let versioned_magic = match (magic, version) { - (Magic::Gguf, 1) => Self::GgufV1, - (Magic::Gguf, 2) => Self::GgufV2, - (Magic::Gguf, 3) => Self::GgufV3, - _ => crate::bail!("gguf: unsupported magic/version {magic:?}/{version}"), - }; - Ok(versioned_magic) - } -} - -#[derive(Debug)] -pub struct TensorInfo { - pub ggml_dtype: GgmlDType, - pub shape: crate::Shape, - pub offset: u64, -} - -impl TensorInfo { - pub fn read( - &self, - reader: &mut R, - tensor_data_offset: u64, - device: &Device, - ) -> Result { - let tensor_elems = self.shape.elem_count(); - let block_size = self.ggml_dtype.block_size(); - if !tensor_elems.is_multiple_of(block_size) { - crate::bail!( - "the number of elements {tensor_elems} is not divisible by the block size {block_size}" - ) - } - let size_in_bytes = tensor_elems / block_size * self.ggml_dtype.type_size(); - let mut raw_data = vec![0u8; size_in_bytes]; - reader.seek(std::io::SeekFrom::Start(tensor_data_offset + self.offset))?; - reader.read_exact(&mut raw_data)?; - super::ggml_file::qtensor_from_ggml( - self.ggml_dtype, - &raw_data, - self.shape.dims().to_vec(), - device, - ) - } -} - -#[derive(Debug)] -pub struct Content { - pub magic: VersionedMagic, - pub metadata: HashMap, - pub tensor_infos: HashMap, - pub tensor_data_offset: u64, -} - -fn read_string(reader: &mut R, magic: &VersionedMagic) -> Result { - let len = match magic { - VersionedMagic::GgufV1 => reader.read_u32::()? as usize, - VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => { - reader.read_u64::()? as usize - } - }; - let mut v = vec![0u8; len]; - reader.read_exact(&mut v)?; - // GGUF strings are supposed to be non-null terminated but in practice this happens. - while let Some(0) = v.last() { - v.pop(); - } - // GGUF strings are utf8 encoded but there are cases that don't seem to be valid. - Ok(String::from_utf8_lossy(&v).into_owned()) -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum ValueType { - // The value is a 8-bit unsigned integer. - U8, - // The value is a 8-bit signed integer. - I8, - // The value is a 16-bit unsigned little-endian integer. - U16, - // The value is a 16-bit signed little-endian integer. - I16, - // The value is a 32-bit unsigned little-endian integer. - U32, - // The value is a 32-bit signed little-endian integer. - I32, - // The value is a 64-bit unsigned little-endian integer. - U64, - // The value is a 64-bit signed little-endian integer. - I64, - // The value is a 32-bit IEEE754 floating point number. - F32, - // The value is a 64-bit IEEE754 floating point number. - F64, - // The value is a boolean. - // 1-byte value where 0 is false and 1 is true. - // Anything else is invalid, and should be treated as either the model being invalid or the reader being buggy. - Bool, - // The value is a UTF-8 non-null-terminated string, with length prepended. - String, - // The value is an array of other values, with the length and type prepended. - // Arrays can be nested, and the length of the array is the number of elements in the array, not the number of bytes. - Array, -} - -#[derive(Debug, Clone)] -pub enum Value { - U8(u8), - I8(i8), - U16(u16), - I16(i16), - U32(u32), - I32(i32), - U64(u64), - I64(i64), - F32(f32), - F64(f64), - Bool(bool), - String(String), - Array(Vec), -} - -impl Value { - pub fn value_type(&self) -> ValueType { - match self { - Self::U8(_) => ValueType::U8, - Self::I8(_) => ValueType::I8, - Self::U16(_) => ValueType::U16, - Self::I16(_) => ValueType::I16, - Self::U32(_) => ValueType::U32, - Self::I32(_) => ValueType::I32, - Self::U64(_) => ValueType::U64, - Self::I64(_) => ValueType::I64, - Self::F32(_) => ValueType::F32, - Self::F64(_) => ValueType::F64, - Self::Bool(_) => ValueType::Bool, - Self::String(_) => ValueType::String, - Self::Array(_) => ValueType::Array, - } - } - - pub fn to_u8(&self) -> Result { - match self { - Self::U8(v) => Ok(*v), - v => crate::bail!("not a u8 {v:?}"), - } - } - - pub fn to_i8(&self) -> Result { - match self { - Self::I8(v) => Ok(*v), - v => crate::bail!("not a i8 {v:?}"), - } - } - - pub fn to_u16(&self) -> Result { - match self { - Self::U16(v) => Ok(*v), - v => crate::bail!("not a u16 {v:?}"), - } - } - - pub fn to_i16(&self) -> Result { - match self { - Self::I16(v) => Ok(*v), - v => crate::bail!("not a i16 {v:?}"), - } - } - - pub fn to_u32(&self) -> Result { - match self { - Self::U32(v) => Ok(*v), - v => crate::bail!("not a u32 {v:?}"), - } - } - - pub fn to_i32(&self) -> Result { - match self { - Self::I32(v) => Ok(*v), - v => crate::bail!("not a i32 {v:?}"), - } - } - - /// This will also automatically upcast any integral types which will not truncate. - pub fn to_u64(&self) -> Result { - match self { - Self::U64(v) => Ok(*v), - // Autoupcast cases here - Self::U8(v) => Ok(*v as u64), - Self::U16(v) => Ok(*v as u64), - Self::U32(v) => Ok(*v as u64), - Self::Bool(v) => Ok(*v as u64), - v => crate::bail!("not a u64 or upcastable to u64 {v:?}"), - } - } - - pub fn to_i64(&self) -> Result { - match self { - Self::I64(v) => Ok(*v), - v => crate::bail!("not a i64 {v:?}"), - } - } - - pub fn to_f32(&self) -> Result { - match self { - Self::F32(v) => Ok(*v), - v => crate::bail!("not a f32 {v:?}"), - } - } - - pub fn to_f64(&self) -> Result { - match self { - Self::F64(v) => Ok(*v), - v => crate::bail!("not a f64 {v:?}"), - } - } - - pub fn to_bool(&self) -> Result { - match self { - Self::Bool(v) => Ok(*v), - v => crate::bail!("not a bool {v:?}"), - } - } - - pub fn to_vec(&self) -> Result<&Vec> { - match self { - Self::Array(v) => Ok(v), - v => crate::bail!("not a vec {v:?}"), - } - } - - pub fn to_string(&self) -> Result<&String> { - match self { - Self::String(v) => Ok(v), - v => crate::bail!("not a string {v:?}"), - } - } - - fn read( - reader: &mut R, - value_type: ValueType, - magic: &VersionedMagic, - ) -> Result { - let v = match value_type { - ValueType::U8 => Self::U8(reader.read_u8()?), - ValueType::I8 => Self::I8(reader.read_i8()?), - ValueType::U16 => Self::U16(reader.read_u16::()?), - ValueType::I16 => Self::I16(reader.read_i16::()?), - ValueType::U32 => Self::U32(reader.read_u32::()?), - ValueType::I32 => Self::I32(reader.read_i32::()?), - ValueType::U64 => Self::U64(reader.read_u64::()?), - ValueType::I64 => Self::I64(reader.read_i64::()?), - ValueType::F32 => Self::F32(reader.read_f32::()?), - ValueType::F64 => Self::F64(reader.read_f64::()?), - ValueType::Bool => match reader.read_u8()? { - 0 => Self::Bool(false), - 1 => Self::Bool(true), - b => crate::bail!("unexpected bool value {b}"), - }, - ValueType::String => Self::String(read_string(reader, magic)?), - ValueType::Array => { - let value_type = reader.read_u32::()?; - let value_type = ValueType::from_u32(value_type)?; - let len = match magic { - VersionedMagic::GgufV1 => reader.read_u32::()? as usize, - VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => { - reader.read_u64::()? as usize - } - }; - let mut vs = Vec::with_capacity(len); - for _ in 0..len { - vs.push(Value::read(reader, value_type, magic)?) - } - Self::Array(vs) - } - }; - Ok(v) - } - - fn write(&self, w: &mut W) -> Result<()> { - match self { - &Self::U8(v) => w.write_u8(v)?, - &Self::I8(v) => w.write_i8(v)?, - &Self::U16(v) => w.write_u16::(v)?, - &Self::I16(v) => w.write_i16::(v)?, - &Self::U32(v) => w.write_u32::(v)?, - &Self::I32(v) => w.write_i32::(v)?, - &Self::U64(v) => w.write_u64::(v)?, - &Self::I64(v) => w.write_i64::(v)?, - &Self::F32(v) => w.write_f32::(v)?, - &Self::F64(v) => w.write_f64::(v)?, - &Self::Bool(v) => w.write_u8(u8::from(v))?, - Self::String(v) => write_string(w, v.as_str())?, - Self::Array(v) => { - // The `Value` type does not enforce that all the values in an Array have the same - // type. - let value_type = if v.is_empty() { - // Doesn't matter, the array is empty. - ValueType::U32 - } else { - let value_type: std::collections::HashSet<_> = - v.iter().map(|elem| elem.value_type()).collect(); - if value_type.len() != 1 { - crate::bail!("multiple value-types in the same array {value_type:?}") - } - value_type.into_iter().next().context("empty value_type")? - }; - w.write_u32::(value_type.to_u32())?; - w.write_u64::(v.len() as u64)?; - for elem in v.iter() { - elem.write(w)? - } - } - } - Ok(()) - } -} - -impl ValueType { - fn from_u32(v: u32) -> Result { - let v = match v { - 0 => Self::U8, - 1 => Self::I8, - 2 => Self::U16, - 3 => Self::I16, - 4 => Self::U32, - 5 => Self::I32, - 6 => Self::F32, - 7 => Self::Bool, - 8 => Self::String, - 9 => Self::Array, - 10 => Self::U64, - 11 => Self::I64, - 12 => Self::F64, - v => crate::bail!("unrecognized value-type {v:#08x}"), - }; - Ok(v) - } - - fn to_u32(self) -> u32 { - match self { - Self::U8 => 0, - Self::I8 => 1, - Self::U16 => 2, - Self::I16 => 3, - Self::U32 => 4, - Self::I32 => 5, - Self::F32 => 6, - Self::Bool => 7, - Self::String => 8, - Self::Array => 9, - Self::U64 => 10, - Self::I64 => 11, - Self::F64 => 12, - } - } -} - -impl Content { - pub fn read(reader: &mut R) -> Result { - let magic = VersionedMagic::read(reader)?; - - let tensor_count = match magic { - VersionedMagic::GgufV1 => reader.read_u32::()? as usize, - VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => { - reader.read_u64::()? as usize - } - }; - let metadata_kv_count = match magic { - VersionedMagic::GgufV1 => reader.read_u32::()? as usize, - VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => { - reader.read_u64::()? as usize - } - }; - - let mut metadata = HashMap::new(); - for _idx in 0..metadata_kv_count { - let key = read_string(reader, &magic)?; - let value_type = reader.read_u32::()?; - let value_type = ValueType::from_u32(value_type)?; - let value = Value::read(reader, value_type, &magic)?; - metadata.insert(key, value); - } - let mut tensor_infos = HashMap::new(); - for _idx in 0..tensor_count { - let tensor_name = read_string(reader, &magic)?; - let n_dimensions = reader.read_u32::()?; - - let mut dimensions: Vec = match magic { - VersionedMagic::GgufV1 => { - let mut dimensions = vec![0; n_dimensions as usize]; - reader.read_u32_into::(&mut dimensions)?; - dimensions.into_iter().map(|c| c as usize).collect() - } - VersionedMagic::GgufV2 | VersionedMagic::GgufV3 => { - let mut dimensions = vec![0; n_dimensions as usize]; - reader.read_u64_into::(&mut dimensions)?; - dimensions.into_iter().map(|c| c as usize).collect() - } - }; - - dimensions.reverse(); - let ggml_dtype = reader.read_u32::()?; - let ggml_dtype = GgmlDType::from_u32(ggml_dtype)?; - let offset = reader.read_u64::()?; - tensor_infos.insert( - tensor_name, - TensorInfo { - shape: crate::Shape::from(dimensions), - offset, - ggml_dtype, - }, - ); - } - let position = reader.stream_position()?; - let alignment = match metadata.get("general.alignment") { - Some(Value::U8(v)) => *v as u64, - Some(Value::U16(v)) => *v as u64, - Some(Value::U32(v)) => *v as u64, - Some(Value::I8(v)) if *v >= 0 => *v as u64, - Some(Value::I16(v)) if *v >= 0 => *v as u64, - Some(Value::I32(v)) if *v >= 0 => *v as u64, - _ => DEFAULT_ALIGNMENT, - }; - let tensor_data_offset = position.div_ceil(alignment) * alignment; - Ok(Self { - magic, - metadata, - tensor_infos, - tensor_data_offset, - }) - } - - pub fn tensor( - &self, - reader: &mut R, - name: &str, - device: &Device, - ) -> Result { - let tensor_info = match self.tensor_infos.get(name) { - Some(tensor_info) => tensor_info, - None => crate::bail!("cannot find tensor info for {name}"), - }; - tensor_info.read(reader, self.tensor_data_offset, device) - } -} - -fn write_string(w: &mut W, str: &str) -> Result<()> { - let bytes = str.as_bytes(); - w.write_u64::(bytes.len() as u64)?; - w.write_all(bytes)?; - Ok(()) -} - -pub fn write( - w: &mut W, - metadata: &[(&str, &Value)], - tensors: &[(&str, &QTensor)], -) -> Result<()> { - w.write_u32::(0x46554747)?; - w.write_u32::(2)?; // version 2. - w.write_u64::(tensors.len() as u64)?; - w.write_u64::(metadata.len() as u64)?; - for (name, value) in metadata.iter() { - write_string(w, name)?; - w.write_u32::(value.value_type().to_u32())?; - value.write(w)?; - } - let mut offset = 0usize; - let mut offsets = Vec::with_capacity(tensors.len()); - for (name, tensor) in tensors.iter() { - write_string(w, name)?; - let dims = tensor.shape().dims(); - w.write_u32::(dims.len() as u32)?; - for &dim in dims.iter().rev() { - w.write_u64::(dim as u64)?; - } - w.write_u32::(tensor.dtype().to_u32())?; - w.write_u64::(offset as u64)?; - offsets.push(offset); - let size_in_bytes = tensor.storage_size_in_bytes(); - let padding = 31 - (31 + size_in_bytes) % 32; - offset += size_in_bytes + padding; - } - let pos = w.stream_position()? as usize; - let padding = 31 - (31 + pos) % 32; - w.write_all(&vec![0u8; padding])?; - let tensor_start_pos = w.stream_position()? as usize; - for (offset, (_name, tensor)) in offsets.iter().zip(tensors.iter()) { - let pos = w.stream_position()? as usize; - if tensor_start_pos + offset != pos { - crate::bail!( - "internal error, unexpected current position {tensor_start_pos} {offset} {pos}" - ) - } - let data = tensor.data()?; - let size_in_bytes = data.len(); - w.write_all(&data)?; - let padding = 31 - (31 + size_in_bytes) % 32; - w.write_all(&vec![0u8; padding])?; - } - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/imatrix_file.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/imatrix_file.rs deleted file mode 100644 index ed228b74ce..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/quantized/imatrix_file.rs +++ /dev/null @@ -1,85 +0,0 @@ -use std::collections::HashMap; -use std::fs::File; -use std::io::{Cursor, Read}; -use std::path::Path; - -use byteorder::{LittleEndian, ReadBytesExt}; - -use crate::Result; - -pub fn load_imatrix>(fname: P) -> Result>> { - let mut all_data = HashMap::new(); - - let mut file = File::open(&fname).map_err(|e| { - crate::Error::msg(format!( - "Failed to open {}: {}", - fname.as_ref().display(), - e - )) - })?; - let mut buffer = Vec::new(); - file.read_to_end(&mut buffer).map_err(|e| { - crate::Error::msg(format!( - "Failed to read file {}: {}", - fname.as_ref().display(), - e - )) - })?; - - let mut cursor = Cursor::new(buffer); - - let n_entries = cursor - .read_i32::() - .map_err(|e| crate::Error::msg(format!("Failed to read number of entries: {e}")))? - as usize; - - if n_entries < 1 { - crate::bail!("No data in file {}", fname.as_ref().display()); - } - - for i in 0..n_entries { - // Read length of the name - let len = cursor.read_i32::().map_err(|e| { - crate::Error::msg(format!( - "Failed to read name length for entry {}: {}", - i + 1, - e - )) - })? as usize; - - // Read the name - let mut name_buf = vec![0u8; len]; - cursor.read_exact(&mut name_buf).map_err(|e| { - crate::Error::msg(format!("Failed to read name for entry {}: {}", i + 1, e)) - })?; - let name = String::from_utf8(name_buf).map_err(|e| { - crate::Error::msg(format!("Invalid UTF-8 name for entry {}: {}", i + 1, e)) - })?; - - // Read ncall and nval - let ncall = cursor.read_i32::().map_err(|e| { - crate::Error::msg(format!("Failed to read ncall for entry {}: {}", i + 1, e)) - })? as usize; - - let nval = cursor.read_i32::().map_err(|e| { - crate::Error::msg(format!("Failed to read nval for entry {}: {}", i + 1, e)) - })? as usize; - - if nval < 1 { - crate::bail!("Invalid nval for entry {}: {}", i + 1, nval); - } - - let mut data = Vec::with_capacity(nval); - for _ in 0..nval { - let v = cursor.read_f32::().unwrap(); - if ncall == 0 { - data.push(v); - } else { - data.push(v / ncall as f32); - } - } - all_data.insert(name, data); - } - - Ok(all_data) -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/k_quants.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/k_quants.rs deleted file mode 100644 index 9069b23667..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/quantized/k_quants.rs +++ /dev/null @@ -1,2505 +0,0 @@ -use super::utils::{ - get_scale_min_k4, group_for_dequantization, group_for_quantization, make_q3_quants, - make_qkx1_quants, make_qx_quants, nearest_int, -}; -use super::GgmlDType; -use crate::quantized::utils::{make_qkx3_quants, make_qp_quants}; -use crate::Result; -use byteorder::{ByteOrder, LittleEndian}; -use half::{bf16, f16, slice::HalfFloatSliceExt}; -use rayon::prelude::*; - -// Default to QK_K 256 rather than 64. -pub const QK_K: usize = 256; -pub const K_SCALE_SIZE: usize = 12; - -pub const QK4_0: usize = 32; -pub const QK4_1: usize = 32; -pub const QK5_0: usize = 32; -pub const QK5_1: usize = 32; -pub const QK8_0: usize = 32; -pub const QK8_1: usize = 32; - -pub trait GgmlType: Sized + Clone + Send + Sync { - const DTYPE: GgmlDType; - const BLCK_SIZE: usize; - const DIRECT_COPY: bool = false; - type VecDotType: GgmlType; - - // This is only safe for types that include immediate values such as float/int/... - fn zeros() -> Self { - unsafe { std::mem::MaybeUninit::zeroed().assume_init() } - } - fn to_float(xs: &[Self], ys: &mut [f32]); - fn from_float(xs: &[f32], ys: &mut [Self]); - fn from_float_imatrix( - _xs: &[f32], - _ys: &mut [Self], - _imatrix_weights: &[f32], - _n_per_row: usize, - ) { - panic!( - "`from_float_imatrix` is unimplemented for {:?}", - Self::DTYPE - ); - } - - fn direct_copy(_xs: &[f32], _ys: &mut [Self]) {} - - /// Dot product used as a building block for quantized mat-mul. - /// n is the number of elements to be considered. - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32; - - /// Generic implementation of the dot product without simd optimizations. - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32; -} - -#[derive(Debug, Clone, PartialEq)] -#[repr(C)] -pub struct BlockQ4_0 { - pub(crate) d: f16, - pub(crate) qs: [u8; QK4_0 / 2], -} -const _: () = assert!(std::mem::size_of::() == 18); - -#[derive(Debug, Clone, PartialEq)] -#[repr(C)] -pub struct BlockQ4_1 { - pub(crate) d: f16, - pub(crate) m: f16, - pub(crate) qs: [u8; QK4_1 / 2], -} -const _: () = assert!(std::mem::size_of::() == 20); - -#[derive(Debug, Clone, PartialEq)] -#[repr(C)] -pub struct BlockQ5_0 { - pub(crate) d: f16, - pub(crate) qh: [u8; 4], - pub(crate) qs: [u8; QK5_0 / 2], -} -const _: () = assert!(std::mem::size_of::() == 22); - -#[derive(Debug, Clone, PartialEq)] -#[repr(C)] -pub struct BlockQ5_1 { - pub(crate) d: f16, - pub(crate) m: f16, - pub(crate) qh: [u8; 4], - pub(crate) qs: [u8; QK5_1 / 2], -} -const _: () = assert!(std::mem::size_of::() == 24); - -#[derive(Debug, Clone, PartialEq)] -#[repr(C)] -pub struct BlockQ8_0 { - pub(crate) d: f16, - pub(crate) qs: [i8; QK8_0], -} -const _: () = assert!(std::mem::size_of::() == 34); - -#[derive(Debug, Clone, PartialEq)] -#[repr(C)] -pub struct BlockQ8_1 { - pub(crate) d: f16, - pub(crate) s: f16, - pub(crate) qs: [i8; QK8_1], -} -const _: () = assert!(std::mem::size_of::() == 36); - -#[derive(Debug, Clone, PartialEq)] -#[repr(C)] -pub struct BlockQ2K { - pub(crate) scales: [u8; QK_K / 16], - pub(crate) qs: [u8; QK_K / 4], - pub(crate) d: f16, - pub(crate) dmin: f16, -} -const _: () = assert!(QK_K / 16 + QK_K / 4 + 2 * 2 == std::mem::size_of::()); - -#[derive(Debug, Clone, PartialEq)] -#[repr(C)] -pub struct BlockQ3K { - pub(crate) hmask: [u8; QK_K / 8], - pub(crate) qs: [u8; QK_K / 4], - pub(crate) scales: [u8; 12], - pub(crate) d: f16, -} -const _: () = assert!(QK_K / 8 + QK_K / 4 + 12 + 2 == std::mem::size_of::()); - -#[derive(Debug, Clone, PartialEq)] -// https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/k_quants.h#L82 -#[repr(C)] -pub struct BlockQ4K { - pub(crate) d: f16, - pub(crate) dmin: f16, - pub(crate) scales: [u8; K_SCALE_SIZE], - pub(crate) qs: [u8; QK_K / 2], -} -const _: () = assert!(QK_K / 2 + K_SCALE_SIZE + 2 * 2 == std::mem::size_of::()); - -#[derive(Debug, Clone, PartialEq)] -#[repr(C)] -pub struct BlockQ5K { - pub(crate) d: f16, - pub(crate) dmin: f16, - pub(crate) scales: [u8; K_SCALE_SIZE], - pub(crate) qh: [u8; QK_K / 8], - pub(crate) qs: [u8; QK_K / 2], -} -const _: () = - assert!(QK_K / 8 + QK_K / 2 + 2 * 2 + K_SCALE_SIZE == std::mem::size_of::()); - -#[derive(Debug, Clone, PartialEq)] -#[repr(C)] -pub struct BlockQ6K { - pub(crate) ql: [u8; QK_K / 2], - pub(crate) qh: [u8; QK_K / 4], - pub(crate) scales: [i8; QK_K / 16], - pub(crate) d: f16, -} -const _: () = assert!(3 * QK_K / 4 + QK_K / 16 + 2 == std::mem::size_of::()); - -#[derive(Debug, Clone, PartialEq)] -#[repr(C)] -pub struct BlockQ8K { - pub(crate) d: f32, - pub(crate) qs: [i8; QK_K], - pub(crate) bsums: [i16; QK_K / 16], -} -const _: () = assert!(4 + QK_K + QK_K / 16 * 2 == std::mem::size_of::()); - -impl GgmlType for BlockQ4_0 { - const DTYPE: GgmlDType = GgmlDType::Q4_0; - const BLCK_SIZE: usize = QK4_0; - type VecDotType = BlockQ8_0; - - // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1525 - fn to_float(xs: &[Self], ys: &mut [f32]) { - let k = ys.len(); - let qk = Self::BLCK_SIZE; - debug_assert!( - k.is_multiple_of(qk), - "dequantize_row_q4_0: {k} is not divisible by {qk}" - ); - - let nb = k / qk; - for i in 0..nb { - let d = xs[i].d.to_f32(); - - for j in 0..(qk / 2) { - let x0 = (xs[i].qs[j] & 0x0F) as i16 - 8; - let x1 = (xs[i].qs[j] >> 4) as i16 - 8; - - ys[i * qk + j] = (x0 as f32) * d; - ys[i * qk + j + qk / 2] = (x1 as f32) * d; - } - } - } - - fn from_float(xs: &[f32], ys: &mut [Self]) { - // quantize_row_q4_0 - let qk = Self::BLCK_SIZE; - let k = xs.len(); - debug_assert!(k.is_multiple_of(qk), "{k} is not divisible by {qk}"); - debug_assert_eq!( - ys.len(), - k / qk, - "size mismatch {} {} {}", - xs.len(), - ys.len(), - qk, - ); - for (i, ys) in ys.iter_mut().enumerate() { - let mut amax = 0f32; - let mut max = 0f32; - - let xs = &xs[i * qk..(i + 1) * qk]; - for &x in xs.iter() { - if amax < x.abs() { - amax = x.abs(); - max = x; - } - } - let d = max / -8.0; - let id = if d != 0f32 { 1. / d } else { 0. }; - ys.d = f16::from_f32(d); - - for (j, q) in ys.qs.iter_mut().enumerate() { - let x0 = xs[j] * id; - let x1 = xs[qk / 2 + j] * id; - let xi0 = u8::min(15, (x0 + 8.5) as u8); - let xi1 = u8::min(15, (x1 + 8.5) as u8); - *q = xi0 | (xi1 << 4) - } - } - } - - // https://github.com/ggerganov/llama.cpp/blob/b5ffb2849d23afe73647f68eec7b68187af09be6/ggml.c#L2361C10-L2361C122 - #[allow(unreachable_code)] - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - #[cfg(target_feature = "avx2")] - return super::avx::vec_dot_q4_0_q8_0(n, xs, ys); - - #[cfg(target_feature = "neon")] - return super::neon::vec_dot_q4_0_q8_0(n, xs, ys); - - #[cfg(target_feature = "simd128")] - return super::simd128::vec_dot_q4_0_q8_0(n, xs, ys); - - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - debug_assert!( - n.is_multiple_of(QK8_0), - "vec_dot_q4_0_q8_0: {n} is not divisible by {QK8_0}" - ); - // Generic implementation. - let mut sumf = 0f32; - for (xs, ys) in xs.iter().zip(ys.iter()) { - let mut sum_i = 0; - for j in 0..QK8_0 / 2 { - let v0 = (xs.qs[j] & 0x0F) as i32 - 8; - let v1 = (xs.qs[j] >> 4) as i32 - 8; - sum_i += v0 * ys.qs[j] as i32 + v1 * ys.qs[j + QK8_0 / 2] as i32 - } - sumf += sum_i as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d) - } - sumf - } -} - -impl GgmlType for BlockQ4_1 { - const DTYPE: GgmlDType = GgmlDType::Q4_1; - const BLCK_SIZE: usize = QK4_1; - type VecDotType = BlockQ8_1; - - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - // ggml_vec_dot_q4_1_q8_1 - let qk = QK8_1; - debug_assert!( - n.is_multiple_of(qk), - "vec_dot_q4_1_q8_1: {n} is not divisible by {qk}" - ); - debug_assert!( - (n / qk).is_multiple_of(2), - "vec_dot_q4_1_q8_1: {n}, nb is not divisible by 2" - ); - - // Generic implementation. - let mut sumf = 0f32; - - for (xs, ys) in xs.iter().zip(ys.iter()) { - let mut sumi = 0i32; - - for j in 0..qk / 2 { - let v0 = xs.qs[j] as i32 & 0x0F; - let v1 = xs.qs[j] as i32 >> 4; - sumi += (v0 * ys.qs[j] as i32) + (v1 * ys.qs[j + qk / 2] as i32); - } - - sumf += sumi as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d) - + f16::to_f32(xs.m) * f16::to_f32(ys.s) - } - sumf - } - - fn from_float(xs: &[f32], ys: &mut [Self]) { - // quantize_row_q4_1 - let qk = Self::BLCK_SIZE; - - debug_assert_eq!( - ys.len() * qk, - xs.len(), - "size mismatch {} {} {}", - xs.len(), - ys.len(), - qk, - ); - for (i, ys) in ys.iter_mut().enumerate() { - let xs = &xs[i * qk..(i + 1) * qk]; - - let mut min = f32::INFINITY; - let mut max = f32::NEG_INFINITY; - for &x in xs.iter() { - min = f32::min(x, min); - max = f32::max(x, max); - } - let d = (max - min) / ((1 << 4) - 1) as f32; - let id = if d != 0f32 { 1. / d } else { 0. }; - ys.d = f16::from_f32(d); - ys.m = f16::from_f32(min); - - for (j, q) in ys.qs.iter_mut().take(qk / 2).enumerate() { - let x0 = (xs[j] - min) * id; - let x1 = (xs[qk / 2 + j] - min) * id; - - let xi0 = u8::min(15, (x0 + 0.5) as u8); - let xi1 = u8::min(15, (x1 + 0.5) as u8); - - *q = xi0 | (xi1 << 4); - } - } - } - - // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1545 - fn to_float(xs: &[Self], ys: &mut [f32]) { - let k = ys.len(); - debug_assert!( - k.is_multiple_of(QK4_1), - "dequantize_row_q4_1: {k} is not divisible by {QK4_1}" - ); - - let nb = k / QK4_1; - for i in 0..nb { - let d = xs[i].d.to_f32(); - let m = xs[i].m.to_f32(); - - for j in 0..(QK4_1 / 2) { - let x0 = xs[i].qs[j] & 0x0F; - let x1 = xs[i].qs[j] >> 4; - - ys[i * QK4_1 + j] = (x0 as f32) * d + m; - ys[i * QK4_1 + j + QK4_1 / 2] = (x1 as f32) * d + m; - } - } - } -} - -impl GgmlType for BlockQ5_0 { - const DTYPE: GgmlDType = GgmlDType::Q5_0; - const BLCK_SIZE: usize = QK5_0; - type VecDotType = BlockQ8_0; - - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - let qk = Self::BLCK_SIZE; - - debug_assert!( - n.is_multiple_of(qk), - "vec_dot_q5_0_q8_0: {n} is not divisible by {qk}" - ); - debug_assert!( - (n / qk).is_multiple_of(2), - "vec_dot_q5_0_q8_0: {n}, nb is not divisible by 2" - ); - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(_n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - // Generic implementation. - let mut sumf = 0f32; - - for (xs, ys) in xs.iter().zip(ys.iter()) { - let qh = LittleEndian::read_u32(&xs.qh); - let mut sumi = 0i32; - - for j in 0..Self::BLCK_SIZE / 2 { - let xh_0 = (((qh & (1u32 << j)) >> j) << 4) as u8; - let xh_1 = ((qh & (1u32 << (j + 16))) >> (j + 12)) as u8; - - let x0 = ((xs.qs[j] & 0x0F) as i32 | xh_0 as i32) - 16; - let x1 = ((xs.qs[j] >> 4) as i32 | xh_1 as i32) - 16; - - sumi += (x0 * ys.qs[j] as i32) + (x1 * ys.qs[j + Self::BLCK_SIZE / 2] as i32); - } - - sumf += sumi as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d) - } - sumf - } - - fn from_float(xs: &[f32], ys: &mut [Self]) { - // quantize_row_q5_0 - debug_assert_eq!( - ys.len() * Self::BLCK_SIZE, - xs.len(), - "size mismatch {} {} {}", - xs.len(), - ys.len(), - Self::BLCK_SIZE, - ); - for (i, ys) in ys.iter_mut().enumerate() { - let xs = &xs[i * Self::BLCK_SIZE..(i + 1) * Self::BLCK_SIZE]; - - let mut amax = 0f32; - let mut max = 0f32; - for &x in xs.iter() { - if amax < x.abs() { - amax = x.abs(); - max = x; - } - } - let d = max / -16.; - let id = if d != 0f32 { 1. / d } else { 0. }; - ys.d = f16::from_f32(d); - let mut qh = 0u32; - for j in 0..Self::BLCK_SIZE / 2 { - let x0 = xs[j] * id; - let x1 = xs[j + Self::BLCK_SIZE / 2] * id; - let xi0 = ((x0 + 16.5) as i8).min(31) as u8; - let xi1 = ((x1 + 16.5) as i8).min(31) as u8; - ys.qs[j] = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4); - qh |= ((xi0 as u32 & 0x10) >> 4) << j; - qh |= ((xi1 as u32 & 0x10) >> 4) << (j + Self::BLCK_SIZE / 2); - } - LittleEndian::write_u32(&mut ys.qh, qh) - } - } - - // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1566 - fn to_float(xs: &[Self], ys: &mut [f32]) { - let k = ys.len(); - debug_assert!( - k.is_multiple_of(QK5_0), - "dequantize_row_q5_0: {k} is not divisible by {QK5_0}" - ); - let nb = k / QK5_0; - for i in 0..nb { - let d = xs[i].d.to_f32(); - let qh: u32 = LittleEndian::read_u32(&xs[i].qh); - - for j in 0..(QK5_0 / 2) { - let xh_0 = (((qh >> j) << 4) & 0x10) as u8; - let xh_1 = ((qh >> (j + 12)) & 0x10) as u8; - - let x0 = ((xs[i].qs[j] & 0x0F) | xh_0) as i32 - 16; - let x1 = ((xs[i].qs[j] >> 4) | xh_1) as i32 - 16; - - ys[i * QK5_0 + j] = (x0 as f32) * d; - ys[i * QK5_0 + j + QK5_0 / 2] = (x1 as f32) * d; - } - } - } -} - -impl GgmlType for BlockQ5_1 { - const DTYPE: GgmlDType = GgmlDType::Q5_1; - const BLCK_SIZE: usize = QK5_1; - type VecDotType = BlockQ8_1; - - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - let qk = Self::BLCK_SIZE; - debug_assert!( - n.is_multiple_of(qk), - "vec_dot_q5_1_q8_1: {n} is not divisible by {qk}" - ); - debug_assert!( - (n / qk).is_multiple_of(2), - "vec_dot_q5_1_q8_1: {n}, nb is not divisible by 2" - ); - - // Generic implementation. - let mut sumf = 0f32; - - for (xs, ys) in xs.iter().zip(ys.iter()) { - let qh = LittleEndian::read_u32(&xs.qh); - let mut sumi = 0i32; - - for j in 0..Self::BLCK_SIZE / 2 { - let xh_0 = ((qh >> j) << 4) & 0x10; - let xh_1 = (qh >> (j + 12)) & 0x10; - - let x0 = (xs.qs[j] as i32 & 0xF) | xh_0 as i32; - let x1 = (xs.qs[j] as i32 >> 4) | xh_1 as i32; - - sumi += (x0 * ys.qs[j] as i32) + (x1 * ys.qs[j + Self::BLCK_SIZE / 2] as i32); - } - - sumf += sumi as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d) - + f16::to_f32(xs.m) * f16::to_f32(ys.s) - } - sumf - } - - fn from_float(xs: &[f32], ys: &mut [Self]) { - // quantize_row_q5_1 - let qk = Self::BLCK_SIZE; - debug_assert_eq!( - ys.len() * qk, - xs.len(), - "size mismatch {} {} {}", - xs.len(), - ys.len(), - qk, - ); - for (i, ys) in ys.iter_mut().enumerate() { - let xs = &xs[i * qk..(i + 1) * qk]; - - let mut min = f32::INFINITY; - let mut max = f32::NEG_INFINITY; - for &x in xs.iter() { - min = f32::min(x, min); - max = f32::max(x, max); - } - let d = (max - min) / ((1 << 5) - 1) as f32; - let id = if d != 0f32 { 1. / d } else { 0. }; - ys.d = f16::from_f32(d); - ys.m = f16::from_f32(min); - - let mut qh = 0u32; - for (j, q) in ys.qs.iter_mut().take(qk / 2).enumerate() { - let x0 = (xs[j] - min) * id; - let x1 = (xs[qk / 2 + j] - min) * id; - - let xi0 = (x0 + 0.5) as u8; - let xi1 = (x1 + 0.5) as u8; - - *q = (xi0 & 0x0F) | ((xi1 & 0x0F) << 4); - // get the 5-th bit and store it in qh at the right position - qh |= ((xi0 as u32 & 0x10) >> 4) << j; - qh |= ((xi1 as u32 & 0x10) >> 4) << (j + qk / 2); - } - LittleEndian::write_u32(&mut ys.qh, qh); - } - } - - // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1592 - fn to_float(xs: &[Self], ys: &mut [f32]) { - let k = ys.len(); - debug_assert!( - k.is_multiple_of(QK5_1), - "dequantize_row_q5_1: {k} is not divisible by {QK5_1}" - ); - - let nb = k / QK5_1; - for i in 0..nb { - let d = xs[i].d.to_f32(); - let m = xs[i].m.to_f32(); - let qh: u32 = LittleEndian::read_u32(&xs[i].qh); - - for j in 0..(QK5_1 / 2) { - let xh_0 = (((qh >> j) << 4) & 0x10) as u8; - let xh_1 = ((qh >> (j + 12)) & 0x10) as u8; - - let x0 = (xs[i].qs[j] & 0x0F) | xh_0; - let x1 = (xs[i].qs[j] >> 4) | xh_1; - - ys[i * QK5_1 + j] = (x0 as f32) * d + m; - ys[i * QK5_1 + j + QK5_1 / 2] = (x1 as f32) * d + m; - } - } - } -} - -impl GgmlType for BlockQ8_0 { - const DTYPE: GgmlDType = GgmlDType::Q8_0; - const BLCK_SIZE: usize = QK8_0; - type VecDotType = BlockQ8_0; - - // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L1619 - fn to_float(xs: &[Self], ys: &mut [f32]) { - let k = ys.len(); - debug_assert!( - k.is_multiple_of(QK8_0), - "dequantize_row_q8_0: {k} is not divisible by {QK8_0}" - ); - - let nb = k / QK8_0; - - for i in 0..nb { - let d = xs[i].d.to_f32(); - - for j in 0..QK8_0 { - ys[i * QK8_0 + j] = xs[i].qs[j] as f32 * d; - } - } - } - - fn from_float(xs: &[f32], ys: &mut [Self]) { - // quantize_row_q8_0 - let k = xs.len(); - debug_assert!( - k.is_multiple_of(Self::BLCK_SIZE), - "{k} is not divisible by {}", - Self::BLCK_SIZE - ); - debug_assert_eq!( - ys.len(), - k / Self::BLCK_SIZE, - "size mismatch {} {} {}", - xs.len(), - ys.len(), - Self::BLCK_SIZE - ); - for (i, ys) in ys.iter_mut().enumerate() { - let mut amax = 0f32; - let xs = &xs[i * Self::BLCK_SIZE..(i + 1) * Self::BLCK_SIZE]; - for &x in xs.iter() { - amax = amax.max(x.abs()) - } - let d = amax / ((1 << 7) - 1) as f32; - let id = if d != 0f32 { 1. / d } else { 0. }; - ys.d = f16::from_f32(d); - for (y, &x) in ys.qs.iter_mut().zip(xs.iter()) { - *y = f32::round(x * id) as i8 - } - } - } - - #[allow(unreachable_code)] - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - #[cfg(target_feature = "avx2")] - return super::avx::vec_dot_q8_0_q8_0(n, xs, ys); - - #[cfg(target_feature = "neon")] - return super::neon::vec_dot_q8_0_q8_0(n, xs, ys); - - #[cfg(target_feature = "simd128")] - return super::simd128::vec_dot_q8_0_q8_0(n, xs, ys); - - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - debug_assert!( - n.is_multiple_of(QK8_0), - "vec_dot_q8_0_q8_0: {n} is not divisible by {QK8_0}" - ); - - // Generic implementation. - let mut sumf = 0f32; - for (xs, ys) in xs.iter().zip(ys.iter()) { - let sum_i = xs - .qs - .iter() - .zip(ys.qs.iter()) - .map(|(&x, &y)| x as i32 * y as i32) - .sum::(); - sumf += sum_i as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d) - } - sumf - } -} - -impl GgmlType for BlockQ8_1 { - const DTYPE: GgmlDType = GgmlDType::Q8_1; - const BLCK_SIZE: usize = QK8_1; - type VecDotType = BlockQ8_1; - - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - debug_assert!( - n.is_multiple_of(QK8_1), - "vec_dot_q8_1_q8_1: {n} is not divisible by {QK8_1}" - ); - - // Generic implementation. - let mut sumf = 0f32; - for (xs, ys) in xs.iter().zip(ys.iter()) { - let sum_i = xs - .qs - .iter() - .zip(ys.qs.iter()) - .map(|(&x, &y)| x as i32 * y as i32) - .sum::(); - sumf += sum_i as f32 * f16::to_f32(xs.d) * f16::to_f32(ys.d) - } - sumf - } - - fn from_float(xs: &[f32], ys: &mut [Self]) { - // quantize_row_q8_1 - debug_assert_eq!( - ys.len() * Self::BLCK_SIZE, - xs.len(), - "size mismatch {} {} {}", - xs.len(), - ys.len(), - Self::BLCK_SIZE - ); - for (i, ys) in ys.iter_mut().enumerate() { - let mut amax = 0f32; - let xs = &xs[i * Self::BLCK_SIZE..(i + 1) * Self::BLCK_SIZE]; - for &x in xs.iter() { - amax = amax.max(x.abs()) - } - let d = amax / ((1 << 7) - 1) as f32; - let id = if d != 0f32 { 1. / d } else { 0. }; - ys.d = f16::from_f32(d); - let mut sum = 0i32; - for j in 0..Self::BLCK_SIZE / 2 { - let v0 = xs[j] * id; - let v1 = xs[j + Self::BLCK_SIZE / 2] * id; - ys.qs[j] = f32::round(v0) as i8; - ys.qs[j + Self::BLCK_SIZE / 2] = f32::round(v1) as i8; - sum += ys.qs[j] as i32 + ys.qs[j + Self::BLCK_SIZE / 2] as i32; - } - ys.s = f16::from_f32(sum as f32) * ys.d; - } - } - - fn to_float(_xs: &[Self], _ys: &mut [f32]) { - unimplemented!("no support for vec-dot on Q8_1") - } -} - -impl GgmlType for BlockQ2K { - const DTYPE: GgmlDType = GgmlDType::Q2K; - const BLCK_SIZE: usize = QK_K; - type VecDotType = BlockQ8K; - - #[allow(unreachable_code)] - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - #[cfg(target_feature = "avx2")] - return super::avx::vec_dot_q2k_q8k(n, xs, ys); - - #[cfg(target_feature = "neon")] - return super::neon::vec_dot_q2k_q8k(n, xs, ys); - - #[cfg(target_feature = "simd128")] - return super::simd128::vec_dot_q2k_q8k(n, xs, ys); - - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - debug_assert!( - n.is_multiple_of(QK_K), - "vec_dot_q2k_q8k: {n} is not divisible by {QK_K}" - ); - - let mut sumf = 0.0; - for (x, y) in xs.iter().zip(ys.iter()) { - let mut q2: &[_] = &x.qs; - let mut q8: &[_] = &y.qs; - let sc = &x.scales; - - let mut summs = 0; - for (bsum, scale) in y.bsums.iter().zip(sc) { - summs += *bsum as i32 * ((scale >> 4) as i32); - } - - let dall = y.d * x.d.to_f32(); - let dmin = y.d * x.dmin.to_f32(); - - let mut isum = 0; - let mut is = 0; - for _ in 0..(QK_K / 128) { - let mut shift = 0; - for _ in 0..4 { - let d = (sc[is] & 0xF) as i32; - is += 1; - let mut isuml = 0; - for l in 0..16 { - isuml += q8[l] as i32 * (((q2[l] >> shift) & 3) as i32); - } - isum += d * isuml; - let d = (sc[is] & 0xF) as i32; - is += 1; - isuml = 0; - for l in 16..32 { - isuml += q8[l] as i32 * (((q2[l] >> shift) & 3) as i32); - } - isum += d * isuml; - shift += 2; - // adjust the indexing - q8 = &q8[32..]; - } - // adjust the indexing - q2 = &q2[32..]; - } - sumf += dall * isum as f32 - dmin * summs as f32; - } - - sumf - } - - // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L279 - fn from_float(xs: &[f32], ys: &mut [Self]) { - const Q4SCALE: f32 = 15.0; - - for (block, x) in group_for_quantization(xs, ys) { - //calculate scales and mins - let mut mins: [f32; QK_K / 16] = [0.0; QK_K / 16]; - let mut scales: [f32; QK_K / 16] = [0.0; QK_K / 16]; - - for (j, x_scale_slice) in x.chunks(16).enumerate() { - (scales[j], mins[j]) = make_qkx1_quants(3, 5, x_scale_slice); - } - // get max scale and max min and ensure they are >= 0.0 - let max_scale = scales.iter().fold(0.0, |max, &val| val.max(max)); - let max_min = mins.iter().fold(0.0, |max, &val| val.max(max)); - - if max_scale > 0.0 { - let iscale = Q4SCALE / max_scale; - for (j, scale) in scales.iter().enumerate().take(QK_K / 16) { - block.scales[j] = nearest_int(iscale * scale) as u8; - } - block.d = f16::from_f32(max_scale / Q4SCALE); - } else { - for j in 0..QK_K / 16 { - block.scales[j] = 0; - } - block.d = f16::from_f32(0.0); - } - - if max_min > 0.0 { - let iscale = Q4SCALE / max_min; - for (j, scale) in block.scales.iter_mut().enumerate() { - let l = nearest_int(iscale * mins[j]) as u8; - *scale |= l << 4; - } - block.dmin = f16::from_f32(max_min / Q4SCALE); - } else { - block.dmin = f16::from_f32(0.0); - } - - let mut big_l: [u8; QK_K] = [0; QK_K]; - - for j in 0..QK_K / 16 { - let d = block.d.to_f32() * (block.scales[j] & 0xF) as f32; - if d == 0.0 { - continue; - } - let dm = block.dmin.to_f32() * (block.scales[j] >> 4) as f32; - for ii in 0..16 { - let ll = nearest_int((x[16 * j + ii] + dm) / d).clamp(0, 3); - big_l[16 * j + ii] = ll as u8; - } - } - - for j in (0..QK_K).step_by(128) { - for ll in 0..32 { - block.qs[j / 4 + ll] = big_l[j + ll] - | (big_l[j + ll + 32] << 2) - | (big_l[j + ll + 64] << 4) - | (big_l[j + ll + 96] << 6); - } - } - } - } - - fn from_float_imatrix(xs: &[f32], ys: &mut [Self], imatrix_weights: &[f32], n_per_row: usize) { - for (sblk_idx, (block, x)) in group_for_quantization(xs, ys).into_iter().enumerate() { - let mut mins: [f32; QK_K / 16] = [0.0; QK_K / 16]; - let mut scales: [f32; QK_K / 16] = [0.0; QK_K / 16]; - let mut weights: [f32; 16] = [0.0; 16]; - let mut sw: [f32; QK_K / 16] = [0.0; QK_K / 16]; - let mut ls: [u8; QK_K / 16] = [0; QK_K / 16]; - let mut lm: [u8; QK_K / 16] = [0; QK_K / 16]; - - let sum_x2 = x.iter().map(|x| x * x).sum::(); - let sigma2 = sum_x2 / QK_K as f32; - for (j, x_scale_slice) in x.chunks_exact(16).enumerate() { - for (l, (w_elem, x_elem)) in weights.iter_mut().zip(x_scale_slice).enumerate() { - let imatrix_row = sblk_idx % (n_per_row / QK_K); - let imatrix_w = imatrix_weights[imatrix_row * QK_K + 16 * j + l]; - *w_elem = imatrix_w * (sigma2 + x_elem * x_elem).sqrt(); - } - let sumw = weights.iter().sum::(); - sw[j] = sumw; - (scales[j], mins[j]) = - make_qkx3_quants(3, x_scale_slice, Some(&weights), -0.9, 0.05, 36, false); - } - - let d_block = make_qp_quants(QK_K / 16, 15, &scales, &mut ls, &sw); - let m_block = make_qp_quants(QK_K / 16, 15, &mins, &mut lm, &sw); - - block.d = f16::from_f32(d_block); - block.dmin = f16::from_f32(m_block); - - for j in 0..QK_K / 16 { - block.scales[j] = ls[j] | (lm[j] << 4); - } - - let mut big_l: [u8; QK_K] = [0; QK_K]; - - for j in 0..QK_K / 16 { - let d = block.d.to_f32() * (block.scales[j] & 0xF) as f32; - if d == 0.0 { - continue; - } - let dm = block.dmin.to_f32() * (block.scales[j] >> 4) as f32; - for ii in 0..16 { - let ll = nearest_int((x[16 * j + ii] + dm) / d).clamp(0, 3); - big_l[16 * j + ii] = ll as u8; - } - } - - for j in (0..QK_K).step_by(128) { - for ll in 0..32 { - block.qs[j / 4 + ll] = big_l[j + ll] - | (big_l[j + ll + 32] << 2) - | (big_l[j + ll + 64] << 4) - | (big_l[j + ll + 96] << 6); - } - } - } - } - // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L354 - fn to_float(xs: &[Self], ys: &mut [f32]) { - for (block, y) in group_for_dequantization(xs, ys) { - let d = block.d.to_f32(); - let min = block.dmin.to_f32(); - - let mut is = 0; - - for (y_block, qs) in y.chunks_exact_mut(128).zip(block.qs.chunks_exact(32)) { - // Step by 32 over q. - let mut shift = 0; - let mut y_block_index = 0; - for _j in 0..4 { - let sc = block.scales[is]; - is += 1; - let dl = d * (sc & 0xF) as f32; - let ml = min * (sc >> 4) as f32; - for q in &qs[..16] { - let y = dl * ((q >> shift) & 3) as f32 - ml; - y_block[y_block_index] = y; - y_block_index += 1; - } - - let sc = block.scales[is]; - is += 1; - let dl = d * (sc & 0xF) as f32; - let ml = min * (sc >> 4) as f32; - for q in &qs[16..] { - let y = dl * ((q >> shift) & 3) as f32 - ml; - y_block[y_block_index] = y; - y_block_index += 1; - } - - shift += 2; - } - } - } - } -} - -impl GgmlType for BlockQ3K { - const DTYPE: GgmlDType = GgmlDType::Q3K; - const BLCK_SIZE: usize = QK_K; - type VecDotType = BlockQ8K; - - #[allow(unreachable_code)] - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - #[cfg(target_feature = "avx2")] - return super::avx::vec_dot_q3k_q8k(n, xs, ys); - - #[cfg(target_feature = "neon")] - return super::neon::vec_dot_q3k_q8k(n, xs, ys); - - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - debug_assert!( - n.is_multiple_of(QK_K), - "vec_dot_q3k_q8k: {n} is not divisible by {QK_K}" - ); - - const KMASK1: u32 = 0x03030303; - const KMASK2: u32 = 0x0f0f0f0f; - - let mut aux8: [i8; QK_K] = [0; QK_K]; - let mut aux16: [i16; 8] = [0; 8]; - let mut sums: [f32; 8] = [0.0; 8]; - let mut aux32: [i32; 8] = [0; 8]; - - let mut auxs: [u32; 4] = [0; 4]; - - for (x, y) in xs.iter().zip(ys.iter()) { - let mut q3: &[u8] = &x.qs; - let hmask: &[u8] = &x.hmask; - let mut q8: &[i8] = &y.qs; - - aux32.fill(0); - let mut a = &mut aux8[..]; - - let mut m = 1; - //Like the GGML original this is written this way to enable the compiler to vectorize it. - for _ in 0..QK_K / 128 { - a.iter_mut() - .take(32) - .zip(q3) - .for_each(|(a_val, q3_val)| *a_val = (q3_val & 3) as i8); - a.iter_mut() - .take(32) - .zip(hmask) - .for_each(|(a_val, hmask_val)| { - *a_val -= if hmask_val & m != 0 { 0 } else { 4 } - }); - a = &mut a[32..]; - m <<= 1; - - a.iter_mut() - .take(32) - .zip(q3) - .for_each(|(a_val, q3_val)| *a_val = ((q3_val >> 2) & 3) as i8); - a.iter_mut() - .take(32) - .zip(hmask) - .for_each(|(a_val, hmask_val)| { - *a_val -= if hmask_val & m != 0 { 0 } else { 4 } - }); - a = &mut a[32..]; - m <<= 1; - - a.iter_mut() - .take(32) - .zip(q3) - .for_each(|(a_val, q3_val)| *a_val = ((q3_val >> 4) & 3) as i8); - a.iter_mut() - .take(32) - .zip(hmask) - .for_each(|(a_val, hmask_val)| { - *a_val -= if hmask_val & m != 0 { 0 } else { 4 } - }); - a = &mut a[32..]; - m <<= 1; - - a.iter_mut() - .take(32) - .zip(q3) - .for_each(|(a_val, q3_val)| *a_val = ((q3_val >> 6) & 3) as i8); - a.iter_mut() - .take(32) - .zip(hmask) - .for_each(|(a_val, hmask_val)| { - *a_val -= if hmask_val & m != 0 { 0 } else { 4 } - }); - a = &mut a[32..]; - m <<= 1; - q3 = &q3[32..]; - } - - a = &mut aux8[..]; - - LittleEndian::read_u32_into(&x.scales, &mut auxs[0..3]); - - let tmp = auxs[2]; - auxs[2] = ((auxs[0] >> 4) & KMASK2) | (((tmp >> 4) & KMASK1) << 4); - auxs[3] = ((auxs[1] >> 4) & KMASK2) | (((tmp >> 6) & KMASK1) << 4); - auxs[0] = (auxs[0] & KMASK2) | (((tmp) & KMASK1) << 4); - auxs[1] = (auxs[1] & KMASK2) | (((tmp >> 2) & KMASK1) << 4); - - for aux in auxs { - for scale in aux.to_le_bytes() { - let scale = i8::from_be_bytes([scale]); - for l in 0..8 { - aux16[l] = q8[l] as i16 * a[l] as i16; - } - for l in 0..8 { - aux32[l] += (scale as i32 - 32) * aux16[l] as i32; - } - q8 = &q8[8..]; - a = &mut a[8..]; - - for l in 0..8 { - aux16[l] = q8[l] as i16 * a[l] as i16; - } - for l in 0..8 { - aux32[l] += (scale as i32 - 32) * aux16[l] as i32; - } - q8 = &q8[8..]; - a = &mut a[8..]; - } - } - let d = x.d.to_f32() * y.d; - for l in 0..8 { - sums[l] += d * aux32[l] as f32; - } - } - - sums.iter().sum() - } - - fn from_float(xs: &[f32], ys: &mut [Self]) { - for (block, x) in group_for_quantization(xs, ys) { - let mut scales: [f32; QK_K / 16] = [0.0; QK_K / 16]; - for (j, x_scale_slice) in x.chunks_exact(16).enumerate() { - scales[j] = make_q3_quants(x_scale_slice, 4, true); - } - - // Get max scale by absolute value. - let mut max_scale: f32 = 0.0; - for &scale in scales.iter() { - if scale.abs() > max_scale.abs() { - max_scale = scale; - } - } - - block.scales.fill(0); - - if max_scale != 0.0 { - let iscale = -32.0 / max_scale; - for (j, scale) in scales.iter().enumerate() { - let l_val = nearest_int(iscale * scale); - let l_val = l_val.clamp(-32, 31) + 32; - if j < 8 { - block.scales[j] = (l_val & 0xF) as u8; - } else { - block.scales[j - 8] |= ((l_val & 0xF) << 4) as u8; - } - let l_val = l_val >> 4; - block.scales[j % 4 + 8] |= (l_val << (2 * (j / 4))) as u8; - } - block.d = f16::from_f32(1.0 / iscale); - } else { - block.d = f16::from_f32(0.0); - } - - let mut l: [i8; QK_K] = [0; QK_K]; - - for j in 0..QK_K / 16 { - let sc = if j < 8 { - block.scales[j] & 0xF - } else { - block.scales[j - 8] >> 4 - }; - let sc = (sc | (((block.scales[8 + j % 4] >> (2 * (j / 4))) & 3) << 4)) as i8 - 32; - let d = block.d.to_f32() * sc as f32; - if d != 0.0 { - for ii in 0..16 { - let l_val = nearest_int(x[16 * j + ii] / d); - l[16 * j + ii] = (l_val.clamp(-4, 3) + 4) as i8; - } - } - } - - block.hmask.fill(0); - let mut m = 0; - let mut hm = 1; - - for ll in l.iter_mut() { - if *ll > 3 { - block.hmask[m] |= hm; - *ll -= 4; - } - m += 1; - if m == QK_K / 8 { - m = 0; - hm <<= 1; - } - } - - for j in (0..QK_K).step_by(128) { - for l_val in 0..32 { - block.qs[j / 4 + l_val] = (l[j + l_val] - | (l[j + l_val + 32] << 2) - | (l[j + l_val + 64] << 4) - | (l[j + l_val + 96] << 6)) - as u8; - } - } - } - } - - fn from_float_imatrix(xs: &[f32], ys: &mut [Self], imatrix_weights: &[f32], n_per_row: usize) { - for (sblk_idx, (block, x)) in group_for_quantization(xs, ys).into_iter().enumerate() { - let mut scales: [f32; QK_K / 16] = [0.0; QK_K / 16]; - let mut weights: [f32; 16] = [0.0; 16]; - let mut sw: [f32; QK_K / 16] = [0.0; QK_K / 16]; - let mut ls: [i8; QK_K / 16] = [0; QK_K / 16]; - let mut l: [i8; QK_K] = [0; QK_K]; - - let sum_x2 = x.iter().map(|x| x * x).sum::(); - let sigma2 = 2. * sum_x2 / QK_K as f32; - - for (j, x_scale_slice) in x.chunks_exact(16).enumerate() { - for (l_idx, (w_elem, x_elem)) in weights.iter_mut().zip(x_scale_slice).enumerate() { - let imatrix_row = sblk_idx % (n_per_row / QK_K); - let imatrix_w = imatrix_weights[imatrix_row * QK_K + 16 * j + l_idx]; - *w_elem = imatrix_w * (sigma2 + x_elem * x_elem).sqrt(); - } - let sumw = weights.iter().sum::(); - sw[j] = sumw; - scales[j] = unsafe { - make_qx_quants( - 16, - 4, - x_scale_slice.as_ptr(), - l.as_mut_ptr().add(16 * j), - 1, - weights.as_ptr(), - ) - }; - } - - block.scales.fill(0); - let d_block = unsafe { - make_qx_quants( - QK_K / 16, - 32, - scales.as_ptr(), - ls.as_mut_ptr(), - 1, - sw.as_ptr(), - ) - }; - block.d = f16::from_f32(d_block); - for (j, l_val) in ls.iter().enumerate().take(QK_K / 16) { - if j < 8 { - block.scales[j] = (l_val & 0xF) as u8; - } else { - block.scales[j - 8] |= ((l_val & 0xF) << 4) as u8; - } - let l_val = l_val >> 4; - block.scales[j % 4 + 8] |= (l_val << (2 * (j / 4))) as u8; - } - - for j in 0..QK_K / 16 { - let sc = if j < 8 { - block.scales[j] & 0xF - } else { - block.scales[j - 8] >> 4 - }; - let sc = (sc | (((block.scales[8 + j % 4] >> (2 * (j / 4))) & 3) << 4)) as i8 - 32; - let d = block.d.to_f32() * sc as f32; - if d != 0.0 { - for ii in 0..16 { - let l_val = nearest_int(x[16 * j + ii] / d); - l[16 * j + ii] = (l_val.clamp(-4, 3) + 4) as i8; - } - } - } - - block.hmask.fill(0); - let mut m = 0; - let mut hm = 1; - - for ll in l.iter_mut() { - if *ll > 3 { - block.hmask[m] |= hm; - *ll -= 4; - } - m += 1; - if m == QK_K / 8 { - m = 0; - hm <<= 1; - } - } - - for j in (0..QK_K).step_by(128) { - for l_val in 0..32 { - block.qs[j / 4 + l_val] = (l[j + l_val] - | (l[j + l_val + 32] << 2) - | (l[j + l_val + 64] << 4) - | (l[j + l_val + 96] << 6)) - as u8; - } - } - } - } - - // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L533 - fn to_float(xs: &[Self], ys: &mut [f32]) { - const KMASK1: u32 = 0x03030303; - const KMASK2: u32 = 0x0f0f0f0f; - - for (block, y) in group_for_dequantization(xs, ys) { - //Reconstruct the scales - let mut aux = [0; 4]; - LittleEndian::read_u32_into(&block.scales, &mut aux[0..3]); - - let tmp = aux[2]; - aux[2] = ((aux[0] >> 4) & KMASK2) | (((tmp >> 4) & KMASK1) << 4); - aux[3] = ((aux[1] >> 4) & KMASK2) | (((tmp >> 6) & KMASK1) << 4); - aux[0] = (aux[0] & KMASK2) | (((tmp) & KMASK1) << 4); - aux[1] = (aux[1] & KMASK2) | (((tmp >> 2) & KMASK1) << 4); - - //Transfer the scales into an i8 array - let scales: &mut [i8] = - unsafe { std::slice::from_raw_parts_mut(aux.as_mut_ptr() as *mut i8, 16) }; - - let d_all = block.d.to_f32(); - let mut m = 1; - let mut is = 0; - - // Dequantize both 128 long blocks - // 32 qs values per 128 long block - // Each 16 elements get a scale - for (y, qs) in y.chunks_exact_mut(128).zip(block.qs.chunks_exact(32)) { - let mut shift = 0; - for shift_scoped_y in y.chunks_exact_mut(32) { - for (scale_index, scale_scoped_y) in - shift_scoped_y.chunks_exact_mut(16).enumerate() - { - let dl = d_all * (scales[is] as f32 - 32.0); - for (i, inner_y) in scale_scoped_y.iter_mut().enumerate() { - let new_y = dl - * (((qs[i + 16 * scale_index] >> shift) & 3) as i8 - - if (block.hmask[i + 16 * scale_index] & m) == 0 { - 4 - } else { - 0 - }) as f32; - *inner_y = new_y; - } - // 16 block finished => advance scale index - is += 1; - } - // 32 block finished => increase shift and m - shift += 2; - m <<= 1; - } - } - } - } -} - -impl GgmlType for BlockQ4K { - const DTYPE: GgmlDType = GgmlDType::Q4K; - const BLCK_SIZE: usize = QK_K; - type VecDotType = BlockQ8K; - - #[allow(unreachable_code)] - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - #[cfg(target_feature = "avx2")] - return super::avx::vec_dot_q4k_q8k(n, xs, ys); - - #[cfg(target_feature = "neon")] - return super::neon::vec_dot_q4k_q8k(n, xs, ys); - - #[cfg(target_feature = "simd128")] - return super::simd128::vec_dot_q4k_q8k(n, xs, ys); - - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - debug_assert!( - n.is_multiple_of(QK_K), - "vec_dot_q4k_q8k: {n} is not divisible by {QK_K}" - ); - - const KMASK1: u32 = 0x3f3f3f3f; - const KMASK2: u32 = 0x0f0f0f0f; - const KMASK3: u32 = 0x03030303; - - let mut utmp: [u32; 4] = [0; 4]; - let mut scales: [u8; 8] = [0; 8]; - let mut mins: [u8; 8] = [0; 8]; - - let mut aux8: [i8; QK_K] = [0; QK_K]; - let mut aux16: [i16; 8] = [0; 8]; - let mut sums: [f32; 8] = [0.0; 8]; - let mut aux32: [i32; 8] = [0; 8]; - - let mut sumf = 0.0; - for (y, x) in ys.iter().zip(xs.iter()) { - let q4 = &x.qs; - let q8 = &y.qs; - aux32.fill(0); - - let mut a = &mut aux8[..]; - let mut q4 = &q4[..]; - for _ in 0..QK_K / 64 { - for l in 0..32 { - a[l] = (q4[l] & 0xF) as i8; - } - a = &mut a[32..]; - for l in 0..32 { - a[l] = (q4[l] >> 4) as i8; - } - a = &mut a[32..]; - q4 = &q4[32..]; - } - - LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); - - utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); - let uaux = utmp[1] & KMASK1; - utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); - utmp[2] = uaux; - utmp[0] &= KMASK1; - - //extract scales and mins - LittleEndian::write_u32_into(&utmp[0..2], &mut scales); - LittleEndian::write_u32_into(&utmp[2..4], &mut mins); - - let mut sumi = 0; - for j in 0..QK_K / 16 { - sumi += y.bsums[j] as i32 * mins[j / 2] as i32; - } - - let mut a = &mut aux8[..]; - let mut q8 = &q8[..]; - - for scale in scales { - let scale = scale as i32; - for _ in 0..4 { - for l in 0..8 { - aux16[l] = q8[l] as i16 * a[l] as i16; - } - for l in 0..8 { - aux32[l] += scale * aux16[l] as i32; - } - q8 = &q8[8..]; - a = &mut a[8..]; - } - } - let d = x.d.to_f32() * y.d; - for l in 0..8 { - sums[l] += d * aux32[l] as f32; - } - let dmin = x.dmin.to_f32() * y.d; - sumf -= dmin * sumi as f32; - } - sumf + sums.iter().sum::() - } - - fn from_float(xs: &[f32], ys: &mut [Self]) { - for (block, x) in group_for_quantization(xs, ys) { - let mut mins: [f32; QK_K / 32] = [0.0; QK_K / 32]; - let mut scales: [f32; QK_K / 32] = [0.0; QK_K / 32]; - - for (j, x_scale_slice) in x.chunks_exact(32).enumerate() { - (scales[j], mins[j]) = make_qkx1_quants(15, 5, x_scale_slice); - } - - // get max scale and max min and ensure they are >= 0.0 - let max_scale = scales.iter().fold(0.0, |max, &val| val.max(max)); - let max_min = mins.iter().fold(0.0, |max, &val| val.max(max)); - - let inv_scale = if max_scale > 0.0 { - 63.0 / max_scale - } else { - 0.0 - }; - let inv_min = if max_min > 0.0 { 63.0 / max_min } else { 0.0 }; - - for j in 0..QK_K / 32 { - let ls = nearest_int(inv_scale * scales[j]).min(63) as u8; - let lm = nearest_int(inv_min * mins[j]).min(63) as u8; - if j < 4 { - block.scales[j] = ls; - block.scales[j + 4] = lm; - } else { - block.scales[j + 4] = (ls & 0xF) | ((lm & 0xF) << 4); - block.scales[j - 4] |= (ls >> 4) << 6; - block.scales[j] |= (lm >> 4) << 6; - } - } - - block.d = f16::from_f32(max_scale / 63.0); - block.dmin = f16::from_f32(max_min / 63.0); - - let mut l: [u8; QK_K] = [0; QK_K]; - - for j in 0..QK_K / 32 { - let (sc, m) = get_scale_min_k4(j, &block.scales); - let d = block.d.to_f32() * sc as f32; - if d != 0.0 { - let dm = block.dmin.to_f32() * m as f32; - for ii in 0..32 { - let l_val = nearest_int((x[32 * j + ii] + dm) / d); - l[32 * j + ii] = l_val.clamp(0, 15) as u8; - } - } - } - - let q = &mut block.qs; - for j in (0..QK_K).step_by(64) { - for l_val in 0..32 { - let offset_index = (j / 64) * 32 + l_val; - q[offset_index] = l[j + l_val] | (l[j + l_val + 32] << 4); - } - } - } - } - - fn from_float_imatrix(xs: &[f32], ys: &mut [Self], imatrix_weights: &[f32], n_per_row: usize) { - for (sblk_idx, (block, x)) in group_for_quantization(xs, ys).into_iter().enumerate() { - let mut mins: [f32; QK_K / 32] = [0.0; QK_K / 32]; - let mut scales: [f32; QK_K / 32] = [0.0; QK_K / 32]; - let mut weights: [f32; 32] = [0.0; 32]; - let mut sw: [f32; QK_K / 32] = [0.0; QK_K / 32]; - let mut ls: [u8; QK_K / 32] = [0; QK_K / 32]; - let mut lm: [u8; QK_K / 32] = [0; QK_K / 32]; - - let sum_x2 = x.iter().map(|x| x * x).sum::(); - let sigma2 = 2. * sum_x2 / QK_K as f32; - - for (j, x_scale_slice) in x.chunks_exact(32).enumerate() { - for (l, (w_elem, x_elem)) in weights.iter_mut().zip(x_scale_slice).enumerate() { - let imatrix_row = sblk_idx % (n_per_row / QK_K); - let imatrix_w = imatrix_weights[imatrix_row * QK_K + 32 * j + l]; - *w_elem = imatrix_w * (sigma2 + x_elem * x_elem).sqrt(); - } - let sumw = weights.iter().sum::(); - sw[j] = sumw; - (scales[j], mins[j]) = - make_qkx3_quants(15, x_scale_slice, Some(&weights), -0.9, 0.05, 36, false); - } - - let d_block = make_qp_quants(QK_K / 32, 63, &scales, &mut ls, &sw); - let m_block = make_qp_quants(QK_K / 32, 63, &mins, &mut lm, &sw); - for j in 0..QK_K / 32 { - let ls_val = ls[j]; - let lm_val = lm[j]; - if j < 4 { - block.scales[j] = ls_val; - block.scales[j + 4] = lm_val; - } else { - block.scales[j + 4] = (ls_val & 0xF) | ((lm_val & 0xF) << 4); - block.scales[j - 4] |= (ls_val >> 4) << 6; - block.scales[j] |= (lm_val >> 4) << 6; - } - } - - block.d = f16::from_f32(d_block); - block.dmin = f16::from_f32(m_block); - - let mut l: [u8; QK_K] = [0; QK_K]; - for j in 0..QK_K / 32 { - let (sc, m) = get_scale_min_k4(j, &block.scales); - let d = block.d.to_f32() * sc as f32; - if d != 0.0 { - let dm = block.dmin.to_f32() * m as f32; - for ii in 0..32 { - let l_val = nearest_int((x[32 * j + ii] + dm) / d); - l[32 * j + ii] = l_val.clamp(0, 15) as u8; - } - } - } - - let q = &mut block.qs; - for j in (0..QK_K).step_by(64) { - for l_val in 0..32 { - let offset_index = (j / 64) * 32 + l_val; - q[offset_index] = l[j + l_val] | (l[j + l_val + 32] << 4); - } - } - } - } - // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L735 - fn to_float(xs: &[Self], ys: &mut [f32]) { - for (block, y) in group_for_dequantization(xs, ys) { - let d = block.d.to_f32(); - let min = block.dmin.to_f32(); - let q = &block.qs; - let mut is = 0; - let mut ys_index = 0; - - for j in (0..QK_K).step_by(64) { - let q = &q[j / 2..j / 2 + 32]; - let (sc, m) = get_scale_min_k4(is, &block.scales); - let d1 = d * sc as f32; - let m1 = min * m as f32; - let (sc, m) = get_scale_min_k4(is + 1, &block.scales); - let d2 = d * sc as f32; - let m2 = min * m as f32; - for q in q { - y[ys_index] = d1 * (q & 0xF) as f32 - m1; - ys_index += 1; - } - for q in q { - y[ys_index] = d2 * (q >> 4) as f32 - m2; - ys_index += 1; - } - is += 2; - } - } - } -} - -// https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L928 -impl GgmlType for BlockQ5K { - const DTYPE: GgmlDType = GgmlDType::Q5K; - const BLCK_SIZE: usize = QK_K; - type VecDotType = BlockQ8K; - - #[allow(unreachable_code)] - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - #[cfg(target_feature = "avx2")] - return super::avx::vec_dot_q5k_q8k(n, xs, ys); - - #[cfg(target_feature = "neon")] - return super::neon::vec_dot_q5k_q8k(n, xs, ys); - - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - debug_assert!( - n.is_multiple_of(QK_K), - "vec_dot_q5k_q8k: {n} is not divisible by {QK_K}" - ); - - const KMASK1: u32 = 0x3f3f3f3f; - const KMASK2: u32 = 0x0f0f0f0f; - const KMASK3: u32 = 0x03030303; - - let mut utmp: [u32; 4] = [0; 4]; - let mut scales: [u8; 8] = [0; 8]; - let mut mins: [u8; 8] = [0; 8]; - - let mut aux8: [i8; QK_K] = [0; QK_K]; - let mut aux16: [i16; 8] = [0; 8]; - let mut sums: [f32; 8] = [0.0; 8]; - let mut aux32: [i32; 8] = [0; 8]; - - let mut sumf = 0.0; - for (y, x) in ys.iter().zip(xs.iter()) { - let q5 = &x.qs; - let hm = &x.qh; - let q8 = &y.qs; - aux32.fill(0); - - let mut a = &mut aux8[..]; - let mut q5 = &q5[..]; - let mut m = 1u8; - - for _ in 0..QK_K / 64 { - for l in 0..32 { - a[l] = (q5[l] & 0xF) as i8; - a[l] += if hm[l] & m != 0 { 16 } else { 0 }; - } - a = &mut a[32..]; - m <<= 1; - for l in 0..32 { - a[l] = (q5[l] >> 4) as i8; - a[l] += if hm[l] & m != 0 { 16 } else { 0 }; - } - a = &mut a[32..]; - m <<= 1; - q5 = &q5[32..]; - } - - LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); - - utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); - let uaux = utmp[1] & KMASK1; - utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); - utmp[2] = uaux; - utmp[0] &= KMASK1; - - //extract scales and mins - LittleEndian::write_u32_into(&utmp[0..2], &mut scales); - LittleEndian::write_u32_into(&utmp[2..4], &mut mins); - - let mut sumi = 0; - for j in 0..QK_K / 16 { - sumi += y.bsums[j] as i32 * mins[j / 2] as i32; - } - - let mut a = &mut aux8[..]; - let mut q8 = &q8[..]; - - for scale in scales { - let scale = scale as i32; - for _ in 0..4 { - for l in 0..8 { - aux16[l] = q8[l] as i16 * a[l] as i16; - } - for l in 0..8 { - aux32[l] += scale * aux16[l] as i32; - } - q8 = &q8[8..]; - a = &mut a[8..]; - } - } - let d = x.d.to_f32() * y.d; - for l in 0..8 { - sums[l] += d * aux32[l] as f32; - } - let dmin = x.dmin.to_f32() * y.d; - sumf -= dmin * sumi as f32; - } - sumf + sums.iter().sum::() - } - - // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L793 - fn from_float(xs: &[f32], ys: &mut [Self]) { - for (block, x) in group_for_quantization(xs, ys) { - let mut mins: [f32; QK_K / 32] = [0.0; QK_K / 32]; - let mut scales: [f32; QK_K / 32] = [0.0; QK_K / 32]; - - for (j, x_scale_slice) in x.chunks_exact(32).enumerate() { - (scales[j], mins[j]) = make_qkx1_quants(31, 5, x_scale_slice); - } - - // get max scale and max min and ensure they are >= 0.0 - let max_scale = scales.iter().fold(0.0, |max, &val| val.max(max)); - let max_min = mins.iter().fold(0.0, |max, &val| val.max(max)); - - let inv_scale = if max_scale > 0.0 { - 63.0 / max_scale - } else { - 0.0 - }; - let inv_min = if max_min > 0.0 { 63.0 / max_min } else { 0.0 }; - for j in 0..QK_K / 32 { - let ls = nearest_int(inv_scale * scales[j]).min(63) as u8; - let lm = nearest_int(inv_min * mins[j]).min(63) as u8; - if j < 4 { - block.scales[j] = ls; - block.scales[j + 4] = lm; - } else { - block.scales[j + 4] = (ls & 0xF) | ((lm & 0xF) << 4); - block.scales[j - 4] |= (ls >> 4) << 6; - block.scales[j] |= (lm >> 4) << 6; - } - } - block.d = f16::from_f32(max_scale / 63.0); - block.dmin = f16::from_f32(max_min / 63.0); - - let mut l: [u8; QK_K] = [0; QK_K]; - for j in 0..QK_K / 32 { - let (sc, m) = get_scale_min_k4(j, &block.scales); - let d = block.d.to_f32() * sc as f32; - if d == 0.0 { - continue; - } - let dm = block.dmin.to_f32() * m as f32; - for ii in 0..32 { - let ll = nearest_int((x[32 * j + ii] + dm) / d); - l[32 * j + ii] = ll.clamp(0, 31) as u8; - } - } - - let qh = &mut block.qh; - let ql = &mut block.qs; - qh.fill(0); - - let mut m1 = 1; - let mut m2 = 2; - for n in (0..QK_K).step_by(64) { - let offset = (n / 64) * 32; - for j in 0..32 { - let mut l1 = l[n + j]; - if l1 > 15 { - l1 -= 16; - qh[j] |= m1; - } - let mut l2 = l[n + j + 32]; - if l2 > 15 { - l2 -= 16; - qh[j] |= m2; - } - ql[offset + j] = l1 | (l2 << 4); - } - m1 <<= 2; - m2 <<= 2; - } - } - } - - fn from_float_imatrix(xs: &[f32], ys: &mut [Self], imatrix_weights: &[f32], n_per_row: usize) { - for (sblk_idx, (block, x)) in group_for_quantization(xs, ys).into_iter().enumerate() { - let mut mins: [f32; QK_K / 32] = [0.0; QK_K / 32]; - let mut scales: [f32; QK_K / 32] = [0.0; QK_K / 32]; - let mut weights: [f32; 32] = [0.0; 32]; - let mut sw: [f32; QK_K / 32] = [0.0; QK_K / 32]; - let mut ls: [u8; QK_K / 32] = [0; QK_K / 32]; - let mut lm: [u8; QK_K / 32] = [0; QK_K / 32]; - - let sum_x2 = x.iter().map(|x| x * x).sum::(); - let sigma2 = 2. * sum_x2 / QK_K as f32; - - for (j, x_scale_slice) in x.chunks_exact(32).enumerate() { - for (l, (w_elem, x_elem)) in weights.iter_mut().zip(x_scale_slice).enumerate() { - let imatrix_row = sblk_idx % (n_per_row / QK_K); - let imatrix_w = imatrix_weights[imatrix_row * QK_K + 32 * j + l]; - *w_elem = imatrix_w * (sigma2 + x_elem * x_elem).sqrt(); - } - let sumw = weights.iter().sum::(); - sw[j] = sumw; - (scales[j], mins[j]) = - make_qkx3_quants(31, x_scale_slice, Some(&weights), -0.9, 0.05, 36, false); - } - - let d_block = make_qp_quants(QK_K / 32, 63, &scales, &mut ls, &sw); - let m_block = make_qp_quants(QK_K / 32, 63, &mins, &mut lm, &sw); - for j in 0..QK_K / 32 { - let ls_val = ls[j].min(63); - let lm_val = lm[j].min(63); - if j < 4 { - block.scales[j] = ls_val; - block.scales[j + 4] = lm_val; - } else { - block.scales[j + 4] = (ls_val & 0xF) | ((lm_val & 0xF) << 4); - block.scales[j - 4] |= (ls_val >> 4) << 6; - block.scales[j] |= (lm_val >> 4) << 6; - } - } - - block.d = f16::from_f32(d_block); - block.dmin = f16::from_f32(m_block); - - let mut l: [u8; QK_K] = [0; QK_K]; - for j in 0..QK_K / 32 { - let (sc, m) = get_scale_min_k4(j, &block.scales); - let d = block.d.to_f32() * sc as f32; - if d != 0.0 { - let dm = block.dmin.to_f32() * m as f32; - for ii in 0..32 { - let l_val = nearest_int((x[32 * j + ii] + dm) / d); - l[32 * j + ii] = l_val.clamp(0, 31) as u8; - } - } - } - - let qh = &mut block.qh; - let ql = &mut block.qs; - qh.fill(0); - - let mut m1 = 1; - let mut m2 = 2; - for n in (0..QK_K).step_by(64) { - let offset = (n / 64) * 32; - for j in 0..32 { - let mut l1 = l[n + j]; - if l1 > 15 { - l1 -= 16; - qh[j] |= m1; - } - let mut l2 = l[n + j + 32]; - if l2 > 15 { - l2 -= 16; - qh[j] |= m2; - } - ql[offset + j] = l1 | (l2 << 4); - } - m1 <<= 2; - m2 <<= 2; - } - } - } - - // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L928 - fn to_float(xs: &[Self], ys: &mut [f32]) { - for (block, y) in group_for_dequantization(xs, ys) { - let d = block.d.to_f32(); - let min = block.dmin.to_f32(); - let ql = &block.qs; - let qh = &block.qh; - let mut is = 0; - let mut u1 = 1; - let mut u2 = 2; - let mut ys_index = 0; - - for j in (0..QK_K).step_by(64) { - let ql = &ql[j / 2..j / 2 + 32]; - let (sc, m) = get_scale_min_k4(is, &block.scales); - let d1 = d * sc as f32; - let m1 = min * m as f32; - let (sc, m) = get_scale_min_k4(is + 1, &block.scales); - let d2 = d * sc as f32; - let m2 = min * m as f32; - for (ql, qh) in ql.iter().zip(qh) { - let to_add = if qh & u1 != 0 { 16f32 } else { 0f32 }; - y[ys_index] = d1 * ((ql & 0xF) as f32 + to_add) - m1; - ys_index += 1; - } - for (ql, qh) in ql.iter().zip(qh) { - let to_add = if qh & u2 != 0 { 16f32 } else { 0f32 }; - y[ys_index] = d2 * ((ql >> 4) as f32 + to_add) - m2; - ys_index += 1; - } - is += 2; - u1 <<= 2; - u2 <<= 2; - } - } - } -} - -impl GgmlType for BlockQ6K { - const DTYPE: GgmlDType = GgmlDType::Q6K; - const BLCK_SIZE: usize = QK_K; - type VecDotType = BlockQ8K; - - #[allow(unreachable_code)] - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - #[cfg(target_feature = "avx2")] - return super::avx::vec_dot_q6k_q8k(n, xs, ys); - - #[cfg(target_feature = "neon")] - return super::neon::vec_dot_q6k_q8k(n, xs, ys); - - #[cfg(target_feature = "simd128")] - return super::simd128::vec_dot_q6k_q8k(n, xs, ys); - - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - debug_assert!( - n.is_multiple_of(QK_K), - "vec_dot_q6k_q8k: {n} is not divisible by {QK_K}" - ); - - let mut aux8 = [0i8; QK_K]; - let mut aux16 = [0i16; 8]; - let mut sums = [0f32; 8]; - let mut aux32 = [0f32; 8]; - - for (x, y) in xs.iter().zip(ys.iter()) { - let q4 = &x.ql; - let qh = &x.qh; - let q8 = &y.qs; - aux32.fill(0f32); - - for j in (0..QK_K).step_by(128) { - let aux8 = &mut aux8[j..]; - let q4 = &q4[j / 2..]; - let qh = &qh[j / 4..]; - for l in 0..32 { - aux8[l] = (((q4[l] & 0xF) | ((qh[l] & 3) << 4)) as i32 - 32) as i8; - aux8[l + 32] = - (((q4[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) as i32 - 32) as i8; - aux8[l + 64] = (((q4[l] >> 4) | (((qh[l] >> 4) & 3) << 4)) as i32 - 32) as i8; - aux8[l + 96] = - (((q4[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) as i32 - 32) as i8; - } - } - - for (j, &scale) in x.scales.iter().enumerate() { - let scale = scale as f32; - let q8 = &q8[16 * j..]; - let aux8 = &aux8[16 * j..]; - for l in 0..8 { - aux16[l] = q8[l] as i16 * aux8[l] as i16; - } - for l in 0..8 { - aux32[l] += scale * aux16[l] as f32 - } - let q8 = &q8[8..]; - let aux8 = &aux8[8..]; - for l in 0..8 { - aux16[l] = q8[l] as i16 * aux8[l] as i16; - } - for l in 0..8 { - aux32[l] += scale * aux16[l] as f32 - } - } - - let d = x.d.to_f32() * y.d; - for (sum, &a) in sums.iter_mut().zip(aux32.iter()) { - *sum += a * d; - } - } - sums.iter().sum() - } - - fn from_float(xs: &[f32], ys: &mut [Self]) { - debug_assert_eq!( - xs.len(), - ys.len() * Self::BLCK_SIZE, - "quantize_row_q6k: size mismatch {} {} {}", - xs.len(), - ys.len(), - Self::BLCK_SIZE - ); - let mut l = [0i8; QK_K]; - let mut scales = [0f32; QK_K / 16]; - let mut x = xs.as_ptr(); - let l = l.as_mut_ptr(); - unsafe { - for y in ys.iter_mut() { - let mut max_scale = 0f32; - let mut max_abs_scale = 0f32; - for (ib, scale_) in scales.iter_mut().enumerate() { - let scale = - make_qx_quants(16, 32, x.add(16 * ib), l.add(16 * ib), 1, std::ptr::null()); - *scale_ = scale; - let abs_scale = scale.abs(); - if abs_scale > max_abs_scale { - max_abs_scale = abs_scale; - max_scale = scale - } - } - - let iscale = -128f32 / max_scale; - y.d = f16::from_f32(1.0 / iscale); - - for (y_scale, scale) in y.scales.iter_mut().zip(scales.iter()) { - *y_scale = nearest_int(iscale * scale).min(127) as i8 - } - - for (j, &y_scale) in y.scales.iter().enumerate() { - let d = y.d.to_f32() * y_scale as f32; - if d == 0. { - continue; - } - for ii in 0..16 { - let ll = nearest_int(*x.add(16 * j + ii) / d).clamp(-32, 31); - *l.add(16 * j + ii) = (ll + 32) as i8 - } - } - - let mut ql = y.ql.as_mut_ptr(); - let mut qh = y.qh.as_mut_ptr(); - - for j in (0..QK_K).step_by(128) { - for l_idx in 0..32 { - let q1 = *l.add(j + l_idx) & 0xF; - let q2 = *l.add(j + l_idx + 32) & 0xF; - let q3 = *l.add(j + l_idx + 64) & 0xF; - let q4 = *l.add(j + l_idx + 96) & 0xF; - *ql.add(l_idx) = (q1 | (q3 << 4)) as u8; - *ql.add(l_idx + 32) = (q2 | (q4 << 4)) as u8; - *qh.add(l_idx) = ((*l.add(j + l_idx) >> 4) - | ((*l.add(j + l_idx + 32) >> 4) << 2) - | ((*l.add(j + l_idx + 64) >> 4) << 4) - | ((*l.add(j + l_idx + 96) >> 4) << 6)) - as u8; - } - ql = ql.add(64); - qh = qh.add(32); - } - - x = x.add(QK_K) - } - } - } - - fn from_float_imatrix(xs: &[f32], ys: &mut [Self], imatrix_weights: &[f32], n_per_row: usize) { - debug_assert_eq!( - xs.len(), - ys.len() * Self::BLCK_SIZE, - "quantize_row_q6k imatrix: size mismatch {} {} {}", - xs.len(), - ys.len(), - Self::BLCK_SIZE - ); - let mut l = [0i8; QK_K]; - let mut scales = [0f32; QK_K / 16]; - let mut x = xs.as_ptr(); - let imatrix_weights = imatrix_weights.as_ptr(); - let l = l.as_mut_ptr(); - unsafe { - for (sblk_idx, y) in ys.iter_mut().enumerate() { - let mut max_scale = 0f32; - let mut max_abs_scale = 0f32; - for (ib, scale_) in scales.iter_mut().enumerate() { - let imatrix_row = sblk_idx % (n_per_row / QK_K); - let scale = make_qx_quants( - 16, - 32, - x.add(16 * ib), - l.add(16 * ib), - 1, - imatrix_weights.add(QK_K * imatrix_row + 16 * ib), - ); - *scale_ = scale; - let abs_scale = scale.abs(); - if abs_scale > max_abs_scale { - max_abs_scale = abs_scale; - max_scale = scale - } - } - - let iscale = -128f32 / max_scale; - y.d = f16::from_f32(1.0 / iscale); - - for (y_scale, scale) in y.scales.iter_mut().zip(scales.iter()) { - *y_scale = nearest_int(iscale * scale).min(127) as i8 - } - - for (j, &y_scale) in y.scales.iter().enumerate() { - let d = y.d.to_f32() * y_scale as f32; - if d == 0. { - continue; - } - for ii in 0..16 { - let ll = nearest_int(*x.add(16 * j + ii) / d).clamp(-32, 31); - *l.add(16 * j + ii) = (ll + 32) as i8 - } - } - - let mut ql = y.ql.as_mut_ptr(); - let mut qh = y.qh.as_mut_ptr(); - - for j in (0..QK_K).step_by(128) { - for l_idx in 0..32 { - let q1 = *l.add(j + l_idx) & 0xF; - let q2 = *l.add(j + l_idx + 32) & 0xF; - let q3 = *l.add(j + l_idx + 64) & 0xF; - let q4 = *l.add(j + l_idx + 96) & 0xF; - *ql.add(l_idx) = (q1 | (q3 << 4)) as u8; - *ql.add(l_idx + 32) = (q2 | (q4 << 4)) as u8; - *qh.add(l_idx) = ((*l.add(j + l_idx) >> 4) - | ((*l.add(j + l_idx + 32) >> 4) << 2) - | ((*l.add(j + l_idx + 64) >> 4) << 4) - | ((*l.add(j + l_idx + 96) >> 4) << 6)) - as u8; - } - ql = ql.add(64); - qh = qh.add(32); - } - - x = x.add(QK_K) - } - } - } - - // https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L1067 - fn to_float(xs: &[Self], ys: &mut [f32]) { - let k = ys.len(); - debug_assert!( - k.is_multiple_of(QK_K), - "dequantize_row_q6k: {k} is not divisible by {QK_K}" - ); - - for (idx_x, x) in xs.iter().enumerate() { - let d = x.d.to_f32(); - let ql = &x.ql; - let qh = &x.qh; - let sc = &x.scales; - for n in (0..QK_K).step_by(128) { - let idx = n / 128; - let ys = &mut ys[idx_x * QK_K + n..]; - let sc = &sc[8 * idx..]; - let ql = &ql[64 * idx..]; - let qh = &qh[32 * idx..]; - for l in 0..32 { - let is = l / 16; - let q1 = ((ql[l] & 0xF) | ((qh[l] & 3) << 4)) as i8 - 32; - let q2 = ((ql[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) as i8 - 32; - let q3 = ((ql[l] >> 4) | (((qh[l] >> 4) & 3) << 4)) as i8 - 32; - let q4 = ((ql[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) as i8 - 32; - ys[l] = d * sc[is] as f32 * q1 as f32; - ys[l + 32] = d * sc[is + 2] as f32 * q2 as f32; - ys[l + 64] = d * sc[is + 4] as f32 * q3 as f32; - ys[l + 96] = d * sc[is + 6] as f32 * q4 as f32; - } - } - } - } -} - -impl GgmlType for BlockQ8K { - const DTYPE: GgmlDType = GgmlDType::Q8K; - const BLCK_SIZE: usize = QK_K; - type VecDotType = BlockQ8K; - - #[allow(unreachable_code)] - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - #[cfg(target_feature = "avx2")] - return super::avx::vec_dot_q8k_q8k(n, xs, ys); - - #[cfg(target_feature = "neon")] - return super::neon::vec_dot_q8k_q8k(n, xs, ys); - - #[cfg(target_feature = "simd128")] - return super::simd128::vec_dot_q8k_q8k(n, xs, ys); - - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - debug_assert!( - n.is_multiple_of(QK_K), - "vec_dot_q8k_q8k: {n} is not divisible by {QK_K}" - ); - // Generic implementation. - let mut sumf = 0f32; - for (xs, ys) in xs.iter().zip(ys.iter()) { - let sum_i = xs - .qs - .iter() - .zip(ys.qs.iter()) - .map(|(&x, &y)| x as i32 * y as i32) - .sum::(); - sumf += sum_i as f32 * xs.d * ys.d - } - sumf - } - - fn from_float(xs: &[f32], ys: &mut [Self]) { - let k = xs.len(); - debug_assert!( - k.is_multiple_of(QK_K), - "quantize_row_q8k: {k} is not divisible by {QK_K}" - ); - for (i, y) in ys.iter_mut().enumerate() { - let mut max = 0f32; - let mut amax = 0f32; - let xs = &xs[i * QK_K..(i + 1) * QK_K]; - for &x in xs.iter() { - if amax < x.abs() { - amax = x.abs(); - max = x; - } - } - if amax == 0f32 { - y.d = 0f32; - y.qs.fill(0) - } else { - let iscale = -128f32 / max; - for (j, q) in y.qs.iter_mut().enumerate() { - // ggml uses nearest_int with bit magic here, maybe we want the same - // but we would have to test and benchmark it. - let v = (iscale * xs[j]).round(); - *q = v.min(127.) as i8 - } - for j in 0..QK_K / 16 { - let mut sum = 0i32; - for ii in 0..16 { - sum += y.qs[j * 16 + ii] as i32 - } - y.bsums[j] = sum as i16 - } - y.d = 1.0 / iscale - } - } - } - - fn to_float(xs: &[Self], ys: &mut [f32]) { - let k = ys.len(); - debug_assert!( - k.is_multiple_of(QK_K), - "dequantize_row_q8k: {k} is not divisible by {QK_K}" - ); - for (i, x) in xs.iter().enumerate() { - for (j, &q) in x.qs.iter().enumerate() { - ys[i * QK_K + j] = x.d * q as f32 - } - } - } -} - -// https://github.com/ggml-org/llama.cpp/blob/aa3ee0eb0b80efca126cedf9bcb4fb5864b46ce3/ggml/src/ggml-cpu/ggml-cpu.c#L1205 -pub fn matmul( - (m, k, n): (usize, usize, usize), - lhs: &[f32], - rhs_t: &[T], - dst: &mut [f32], -) -> Result<()> { - debug_assert_eq!( - T::BLCK_SIZE, - T::VecDotType::BLCK_SIZE, - "Mismatched block sizes" - ); - debug_assert_eq!( - m * k, - lhs.len(), - "unexpected lhs length {} ({m},{k},{n})", - lhs.len() - ); - let k_in_blocks = k.div_ceil(T::BLCK_SIZE); - - // TODO: Pre-allocate this. - let mut lhs_b = vec![T::VecDotType::zeros(); m * k_in_blocks]; - // f32, f16, and bf16 support direct copy - if T::DIRECT_COPY { - T::VecDotType::direct_copy(lhs, &mut lhs_b); - } else { - for row_idx in 0..m { - let lhs_b_mut = &mut lhs_b[row_idx * k_in_blocks..(row_idx + 1) * k_in_blocks]; - let lhs = &lhs[row_idx * k..(row_idx + 1) * k]; - T::VecDotType::from_float(lhs, lhs_b_mut) - } - } - - for row_idx in 0..m { - let lhs_row = &lhs_b[row_idx * k_in_blocks..(row_idx + 1) * k_in_blocks]; - let dst_row = &mut dst[row_idx * n..(row_idx + 1) * n]; - - dst_row - .into_par_iter() - .enumerate() - .with_min_len(128) - .with_max_len(512) - .for_each(|(col_idx, dst)| { - let rhs_col = &rhs_t[col_idx * k_in_blocks..(col_idx + 1) * k_in_blocks]; - *dst = T::vec_dot(k, rhs_col, lhs_row); - }); - } - Ok(()) -} - -pub fn matmul_f16( - mkn: (usize, usize, usize), - lhs: &[f16], - rhs_t: &[T], - dst: &mut [f16], -) -> Result<()> { - let (m, k, n) = mkn; - if m * k != lhs.len() { - crate::bail!("unexpected lhs length {} {mkn:?}", lhs.len()); - } - - let k_in_lhs_blocks = k.div_ceil(T::BLCK_SIZE); - let k_in_rhs_blocks = k.div_ceil(T::VecDotType::BLCK_SIZE); - let mut lhs_b = vec![T::VecDotType::zeros(); m * k_in_lhs_blocks]; - for row_idx in 0..m { - let lhs_b = &mut lhs_b[row_idx * k_in_lhs_blocks..(row_idx + 1) * k_in_lhs_blocks]; - let lhs = &lhs[row_idx * k..(row_idx + 1) * k]; - let lhs_f32: Vec<_> = lhs.iter().map(|&x| x.to_f32()).collect(); - T::VecDotType::from_float(&lhs_f32, lhs_b); - } - let lhs_b = lhs_b.as_slice(); - - for row_idx in 0..m { - let lhs_row = &lhs_b[row_idx * k_in_lhs_blocks..(row_idx + 1) * k_in_lhs_blocks]; - let dst_row = &mut dst[row_idx * n..(row_idx + 1) * n]; - - for (col_idx, dst) in dst_row.iter_mut().enumerate() { - let rhs_col = &rhs_t[col_idx * k_in_rhs_blocks..(col_idx + 1) * k_in_rhs_blocks]; - let value = T::vec_dot(k, rhs_col, lhs_row); - *dst = f16::from_f32(value); - } - } - Ok(()) -} - -impl GgmlType for f32 { - const DTYPE: GgmlDType = GgmlDType::F32; - const BLCK_SIZE: usize = 1; - const DIRECT_COPY: bool = true; - type VecDotType = f32; - - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - debug_assert!(xs.len() >= n, "size mismatch xs {} < {n}", xs.len()); - debug_assert!(ys.len() >= n, "size mismatch ys {} < {n}", ys.len()); - let mut res = 0f32; - unsafe { crate::cpu::vec_dot_f32(xs.as_ptr(), ys.as_ptr(), &mut res, n) }; - res - } - - fn from_float(xs: &[f32], ys: &mut [Self]) { - debug_assert_eq!( - xs.len(), - ys.len(), - "size mismatch xs {} != ys {}", - xs.len(), - ys.len() - ); - ys.copy_from_slice(xs); - } - - fn to_float(xs: &[Self], ys: &mut [f32]) { - debug_assert_eq!( - xs.len(), - ys.len(), - "size mismatch xs {} != ys {}", - xs.len(), - ys.len() - ); - ys.copy_from_slice(xs); - } - - fn direct_copy(xs: &[f32], ys: &mut [Self]) { - Self::from_float(xs, ys) - } -} - -impl GgmlType for f16 { - const DTYPE: GgmlDType = GgmlDType::F16; - const BLCK_SIZE: usize = 1; - const DIRECT_COPY: bool = true; - type VecDotType = f16; - - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - debug_assert!(xs.len() >= n, "size mismatch xs {} < {n}", xs.len()); - debug_assert!(ys.len() >= n, "size mismatch ys {} < {n}", ys.len()); - let mut res = 0f32; - unsafe { crate::cpu::vec_dot_f16(xs.as_ptr(), ys.as_ptr(), &mut res, n) }; - res - } - - fn from_float(xs: &[f32], ys: &mut [Self]) { - debug_assert_eq!( - xs.len(), - ys.len(), - "size mismatch xs {} != ys {}", - xs.len(), - ys.len() - ); - ys.convert_from_f32_slice(xs); - } - - fn to_float(xs: &[Self], ys: &mut [f32]) { - debug_assert_eq!( - xs.len(), - ys.len(), - "size mismatch xs {} != ys {}", - xs.len(), - ys.len() - ); - xs.convert_to_f32_slice(ys); - } - - fn direct_copy(xs: &[f32], ys: &mut [Self]) { - Self::from_float(xs, ys) - } -} - -impl GgmlType for bf16 { - const DTYPE: GgmlDType = GgmlDType::BF16; - const BLCK_SIZE: usize = 1; - const DIRECT_COPY: bool = true; - type VecDotType = bf16; - - fn vec_dot(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - Self::vec_dot_unopt(n, xs, ys) - } - - fn vec_dot_unopt(n: usize, xs: &[Self], ys: &[Self::VecDotType]) -> f32 { - debug_assert!(xs.len() >= n, "size mismatch xs {} < {n}", xs.len()); - debug_assert!(ys.len() >= n, "size mismatch ys {} < {n}", ys.len()); - let mut res = 0f32; - unsafe { crate::cpu::vec_dot_bf16(xs.as_ptr(), ys.as_ptr(), &mut res, n) }; - res - } - - fn from_float(xs: &[f32], ys: &mut [Self]) { - debug_assert_eq!( - xs.len(), - ys.len(), - "size mismatch xs {} != ys {}", - xs.len(), - ys.len() - ); - ys.convert_from_f32_slice(xs); - } - - fn to_float(xs: &[Self], ys: &mut [f32]) { - debug_assert_eq!( - xs.len(), - ys.len(), - "size mismatch xs {} != ys {}", - xs.len(), - ys.len() - ); - xs.convert_to_f32_slice(ys); - } - - fn direct_copy(xs: &[f32], ys: &mut [Self]) { - Self::from_float(xs, ys) - } -} - -macro_rules! verify_block_size { - ( $block_type:ident ) => { - const _: () = - assert!($block_type::BLCK_SIZE == <$block_type as GgmlType>::VecDotType::BLCK_SIZE); - }; -} - -macro_rules! verify_block_sizes { - ( $( $block_type:ident ),* ) => { - $( - verify_block_size!($block_type); - )* - }; -} - -verify_block_sizes!( - BlockQ4_0, BlockQ4_1, BlockQ5_0, BlockQ5_1, BlockQ8_0, BlockQ8_1, BlockQ2K, BlockQ3K, BlockQ4K, - BlockQ5K, BlockQ6K, BlockQ8K, f32, f16, bf16 -); diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/metal.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/metal.rs deleted file mode 100644 index ad746ef0e3..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/quantized/metal.rs +++ /dev/null @@ -1,391 +0,0 @@ -use super::{GgmlDType, QStorage}; -use crate::backend::BackendStorage; -use crate::{DType, MetalDevice, MetalStorage, Result, Shape, D}; -use candle_metal_kernels::metal::Buffer; -use std::sync::Arc; - -pub struct QMetalStorage { - dtype: GgmlDType, - device: MetalDevice, - buffer: Arc, -} - -impl QMetalStorage { - pub fn zeros(device: &MetalDevice, elem_count: usize, dtype: GgmlDType) -> Result { - let size = elem_count * dtype.type_size() / dtype.block_size(); - let buffer = device.allocate_zeros(size)?; - Ok(Self { - buffer, - device: device.clone(), - dtype, - }) - } - - pub fn dtype(&self) -> GgmlDType { - self.dtype - } - - pub fn device(&self) -> &MetalDevice { - &self.device - } - - pub fn buffer(&self) -> &Buffer { - &self.buffer - } - - pub fn dequantize(&self, elem_count: usize) -> Result { - use crate::quantized::k_quants::GgmlType; - - let buffer = self.device.allocate_buffer(self.buffer.length())?; - let blit = self.device.blit_command_encoder()?; - blit.set_label("blit_to_cpu"); - blit.copy_from_buffer(&self.buffer, 0, &buffer, 0, self.buffer.length()); - blit.end_encoding(); - self.device.wait_until_completed()?; - let mut out = vec![0.0; elem_count]; - let block_len = elem_count / self.dtype.block_size(); - match self.dtype { - GgmlDType::F32 => { - let vec: Vec = read_to_vec(&buffer, block_len); - f32::to_float(&vec, &mut out); - } - GgmlDType::F16 => { - let vec: Vec = read_to_vec(&buffer, block_len); - half::f16::to_float(&vec, &mut out); - } - GgmlDType::BF16 => { - let vec: Vec = read_to_vec(&buffer, block_len); - half::bf16::to_float(&vec, &mut out); - } - GgmlDType::Q4_0 => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ4_0::to_float(&vec, &mut out); - } - GgmlDType::Q4_1 => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ4_1::to_float(&vec, &mut out); - } - GgmlDType::Q5_0 => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ5_0::to_float(&vec, &mut out); - } - GgmlDType::Q5_1 => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ5_1::to_float(&vec, &mut out); - } - GgmlDType::Q8_0 => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ8_0::to_float(&vec, &mut out); - } - GgmlDType::Q8_1 => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ8_1::to_float(&vec, &mut out); - } - GgmlDType::Q2K => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ2K::to_float(&vec, &mut out); - } - GgmlDType::Q3K => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ3K::to_float(&vec, &mut out); - } - GgmlDType::Q4K => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ4K::to_float(&vec, &mut out); - } - GgmlDType::Q5K => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ5K::to_float(&vec, &mut out); - } - GgmlDType::Q6K => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ6K::to_float(&vec, &mut out); - } - GgmlDType::Q8K => { - let vec: Vec = read_to_vec(&buffer, block_len); - crate::quantized::BlockQ8K::to_float(&vec, &mut out); - } - } - - let buffer = self.device.new_buffer_with_data(&out)?; - Ok(MetalStorage::new( - buffer, - self.device.clone(), - elem_count, - DType::F32, - )) - } - - pub fn quantize(&mut self, src: &MetalStorage) -> Result<()> { - // Quantization only happens on CPU for now. - let src = src.to_cpu::()?; - let elem_count = src.len(); - let src = crate::Storage::Cpu(crate::CpuStorage::F32(src)); - let mut qcpu_storage = crate::Device::Cpu.qzeros(elem_count, self.dtype)?; - qcpu_storage.quantize(&src)?; - let buffer = self.device.new_buffer_with_data(&qcpu_storage.data()?)?; - self.buffer = buffer; - Ok(()) - } - - pub fn quantize_imatrix( - &mut self, - src: &MetalStorage, - imatrix_weights: &[f32], - n_per_row: usize, - ) -> Result<()> { - // Quantization only happens on CPU for now. - let src = src.to_cpu::()?; - let elem_count = src.len(); - let src = crate::Storage::Cpu(crate::CpuStorage::F32(src)); - let mut qcpu_storage = crate::Device::Cpu.qzeros(elem_count, self.dtype)?; - qcpu_storage.quantize_imatrix(&src, imatrix_weights, n_per_row)?; - let buffer = self.device.new_buffer_with_data(&qcpu_storage.data()?)?; - self.buffer = buffer; - Ok(()) - } - - pub fn quantize_imatrix_onto( - &mut self, - src: &crate::CpuStorage, - imatrix_weights: &[f32], - n_per_row: usize, - ) -> Result<()> { - // Quantization only happens on CPU for now. - let elem_count = src.as_slice::()?.len(); - let mut qcpu_storage = crate::Device::Cpu.qzeros(elem_count, self.dtype)?; - - if let QStorage::Cpu(storage) = &mut qcpu_storage { - storage.from_float_imatrix(src.as_slice::()?, imatrix_weights, n_per_row); - } else { - unreachable!() - } - - let buffer = self.device.new_buffer_with_data(&qcpu_storage.data()?)?; - self.buffer = buffer; - Ok(()) - } - - pub fn quantize_onto(&mut self, src: &crate::CpuStorage) -> Result<()> { - // Quantization only happens on CPU for now. - let elem_count = src.as_slice::()?.len(); - let mut qcpu_storage = crate::Device::Cpu.qzeros(elem_count, self.dtype)?; - - if let QStorage::Cpu(storage) = &mut qcpu_storage { - storage.from_float(src.as_slice::()?); - } else { - unreachable!() - } - - let buffer = self.device.new_buffer_with_data(&qcpu_storage.data()?)?; - self.buffer = buffer; - Ok(()) - } - - pub fn storage_size_in_bytes(&self) -> usize { - self.buffer.length() - } - - fn fwd_mv( - &self, - self_shape: &Shape, - storage: &MetalStorage, - layout: &crate::Layout, - ) -> Result<(MetalStorage, Shape)> { - use crate::MetalError; - - if !layout.is_contiguous() { - crate::bail!("input tensor is not contiguous {layout:?}") - } - let src_shape = layout.shape(); - // self is transposed so n is first then k. - if src_shape.rank() < 2 { - crate::bail!("input tensor has only one dimension {layout:?}") - } - let (n, k) = self_shape.dims2()?; - let mut dst_shape = src_shape.dims().to_vec(); - - // We always use a single batch dimension and stack all the tensors in the batch on the - // second dimension as the implementation in candle-metal-kernels doesn't handle batch - // properly. - let m = match dst_shape.len() { - 3 => dst_shape[0] * dst_shape[1], - 2 => dst_shape[0], - n => crate::bail!("Invalid rank {n} for quantized matmul metal"), - }; - let last_k = dst_shape.pop().unwrap(); - if last_k != k { - crate::bail!("input tensor {layout:?} incompatible with {:?}", self_shape) - } - dst_shape.push(n); - let dst_shape = Shape::from(dst_shape); - let device = storage.device().clone(); - let dst = device.new_buffer(dst_shape.elem_count(), DType::F32, "qmatmul")?; - let encoder = device.command_encoder()?; - // In some cases it would be better to use the mm variant, though it has its drawbacks - // around memory alignment. - for batch_id in 0..m { - candle_metal_kernels::call_quantized_matmul_mv_t( - device.device(), - &encoder, - device.kernels(), - self.dtype.into(), - (1, 1, n, k), - storage.buffer(), - (layout.start_offset() + batch_id * k) * storage.dtype().size_in_bytes(), - &self.buffer, - batch_id * n * DType::F32.size_in_bytes(), - &dst, - ) - .map_err(MetalError::from)?; - } - let dst_storage = crate::MetalStorage::new(dst, device, dst_shape.elem_count(), DType::F32); - Ok((dst_storage, dst_shape)) - } - - pub fn fwd( - &self, - self_shape: &Shape, - storage: &MetalStorage, - layout: &crate::Layout, - ) -> Result<(MetalStorage, Shape)> { - use crate::MetalError; - - if !layout.is_contiguous() { - crate::bail!("input tensor is not contiguous {layout:?}") - } - let src_shape = layout.shape(); - // self is transposed so n is first then k. - if src_shape.rank() < 2 { - crate::bail!("input tensor has only one dimension {layout:?}") - } - let n = self_shape.dim(D::Minus2)?; - let k = self_shape.dim(D::Minus1)?; - let mut dst_shape = src_shape.dims().to_vec(); - - if src_shape.rank() < self_shape.rank() { - crate::bail!( - "input rank ({}) must be >= weight rank ({})", - src_shape.rank(), - self_shape.rank() - ) - } - - if src_shape.dim(D::Minus2)? == 1 { - return self.fwd_mv(self_shape, storage, layout); - } - - let last_k = dst_shape.pop().unwrap(); - if last_k != k { - crate::bail!("input tensor {layout:?} incompatible with {:?}", self_shape) - } - dst_shape.push(n); - let dst_shape = Shape::from(dst_shape); - let device = storage.device().clone(); - let dst = device.new_buffer(dst_shape.elem_count(), DType::F32, "qmatmul")?; - let encoder = device.command_encoder()?; - - assert_eq!(storage.dtype(), DType::F32); - - if self_shape.rank() > 4 { - crate::bail!("weight rank ({}) must be <= 4", self_shape.rank()) - } - let src0_l = crate::Layout::contiguous( - [vec![1; 4 - self_shape.rank()], self_shape.dims().to_vec()].concat(), - ); - let src0_stride = src0_l - .stride() - .iter() - .map(|x| { - (*x as f32 * (self.dtype.type_size() as f32 / self.dtype.block_size() as f32)) - as usize - }) - .collect::>(); - - if src_shape.rank() > 4 { - crate::bail!("weight rank ({}) must be <= 4", src_shape.rank()) - } - let src1_l = crate::Layout::contiguous( - [vec![1; 4 - src_shape.rank()], src_shape.dims().to_vec()].concat(), - ); - - candle_metal_kernels::call_quantized_matmul_mm_t( - device.device(), - &encoder, - device.kernels(), - self.dtype.into(), - src0_l.dims(), - &src0_stride, - &self.buffer, - src1_l.dims(), - &src1_l - .stride() - .iter() - .map(|x| x * DType::F32.size_in_bytes()) - .collect::>(), - storage.buffer(), - src1_l.start_offset() * storage.dtype().size_in_bytes(), - dst_shape.dims(), - 0, - &dst, - ) - .map_err(MetalError::from)?; - - let dst_storage = crate::MetalStorage::new(dst, device, dst_shape.elem_count(), DType::F32); - Ok((dst_storage, dst_shape)) - } - - pub fn data(&self) -> Result> { - let buffer = self.device.allocate_buffer(self.buffer.length())?; - { - let blit = self.device.blit_command_encoder()?; - blit.set_label("blit_to_cpu"); - blit.copy_from_buffer(&self.buffer, 0, &buffer, 0, self.buffer.length()); - blit.end_encoding(); - } - self.device.wait_until_completed()?; - Ok(read_to_vec::(&buffer, self.storage_size_in_bytes())) - } -} - -pub fn load_quantized( - device: &MetalDevice, - data: &[T], -) -> Result { - let buffer = device.new_buffer_with_data(data)?; - let device = device.clone(); - Ok(QStorage::Metal(QMetalStorage { - dtype: T::DTYPE, - device, - buffer, - })) -} - -fn read_to_vec(buffer: &Buffer, n: usize) -> Vec { - let ptr = buffer.contents() as *const T; - assert!(!ptr.is_null()); - let slice = unsafe { std::slice::from_raw_parts(ptr, n) }; - slice.to_vec() -} - -impl From for candle_metal_kernels::GgmlDType { - fn from(value: GgmlDType) -> Self { - match value { - GgmlDType::Q4_0 => candle_metal_kernels::GgmlDType::Q4_0, - GgmlDType::Q4_1 => candle_metal_kernels::GgmlDType::Q4_1, - GgmlDType::Q5_0 => candle_metal_kernels::GgmlDType::Q5_0, - GgmlDType::Q5_1 => candle_metal_kernels::GgmlDType::Q5_1, - GgmlDType::Q8_0 => candle_metal_kernels::GgmlDType::Q8_0, - GgmlDType::Q8_1 => candle_metal_kernels::GgmlDType::Q8_1, - GgmlDType::Q2K => candle_metal_kernels::GgmlDType::Q2K, - GgmlDType::Q3K => candle_metal_kernels::GgmlDType::Q3K, - GgmlDType::Q4K => candle_metal_kernels::GgmlDType::Q4K, - GgmlDType::Q5K => candle_metal_kernels::GgmlDType::Q5K, - GgmlDType::Q6K => candle_metal_kernels::GgmlDType::Q6K, - GgmlDType::Q8K => candle_metal_kernels::GgmlDType::Q8K, - GgmlDType::F16 => candle_metal_kernels::GgmlDType::F16, - GgmlDType::F32 => candle_metal_kernels::GgmlDType::F32, - GgmlDType::BF16 => candle_metal_kernels::GgmlDType::F16, - } - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/mod.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/mod.rs deleted file mode 100644 index 7316d29871..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/quantized/mod.rs +++ /dev/null @@ -1,881 +0,0 @@ -use crate::{ - backend::BackendStorage, CpuStorage, DType, Device, Result, Shape, Storage, Tensor, D, -}; -use k_quants::*; -use std::borrow::Cow; - -#[cfg(target_feature = "avx2")] -pub mod avx; -mod dummy_cuda; -mod dummy_metal; -pub mod ggml_file; -pub mod gguf_file; -pub mod imatrix_file; -pub mod k_quants; -#[cfg(feature = "metal")] -pub mod metal; -#[cfg(not(feature = "metal"))] -mod metal { - pub use super::dummy_metal::*; -} -#[cfg(feature = "cuda")] -pub mod cuda; -#[cfg(not(feature = "cuda"))] -mod cuda { - pub use super::dummy_cuda::*; -} - -#[cfg(target_feature = "neon")] -pub mod neon; -#[cfg(target_feature = "simd128")] -pub mod simd128; -pub mod utils; -use half::{bf16, f16}; - -pub use k_quants::GgmlType; - -fn as_t_slice(data: Cow<'_, [u8]>) -> &[T] { - let size = std::mem::size_of::(); - assert_eq!( - data.len() % size, - 0, - "Data length must be a multiple of T's size" - ); - let ptr = data.as_ptr(); - assert_eq!( - (ptr as usize) % std::mem::align_of::(), - 0, - "Data pointer must be aligned to T's alignment" - ); - unsafe { std::slice::from_raw_parts(ptr as *const T, data.len() / size) } -} - -pub struct QTensor { - storage: QStorage, - shape: Shape, -} - -impl Device { - fn qzeros(&self, elem_count: usize, dtype: GgmlDType) -> Result { - match self { - Device::Cpu => { - let storage = dtype.cpu_zeros(elem_count); - Ok(QStorage::Cpu(storage)) - } - Device::Metal(metal) => { - let storage = metal::QMetalStorage::zeros(metal, elem_count, dtype)?; - Ok(QStorage::Metal(storage)) - } - Device::Cuda(cuda) => { - let storage = cuda::QCudaStorage::zeros(cuda, elem_count, dtype)?; - Ok(QStorage::Cuda(storage)) - } - } - } -} - -pub enum QStorage { - Cpu(Box), - Metal(metal::QMetalStorage), - Cuda(cuda::QCudaStorage), -} - -impl QStorage { - pub fn from_data(data: Cow<'_, [u8]>, device: &Device, dtype: GgmlDType) -> Result { - match device { - Device::Cpu => Ok(Self::Cpu(dtype.from_data(data))), - Device::Metal(d) => match dtype { - GgmlDType::F32 => metal::load_quantized(d, as_t_slice::(data)), - GgmlDType::F16 => metal::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q4_0 => metal::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q4_1 => metal::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q5_0 => metal::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q5_1 => metal::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q8_0 => metal::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q8_1 => metal::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q2K => metal::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q3K => metal::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q4K => metal::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q5K => metal::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q6K => metal::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q8K => metal::load_quantized(d, as_t_slice::(data)), - GgmlDType::BF16 => metal::load_quantized(d, as_t_slice::(data)), - }, - Device::Cuda(d) => match dtype { - GgmlDType::F32 => cuda::load_quantized(d, as_t_slice::(data)), - GgmlDType::F16 => cuda::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q4_0 => cuda::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q4_1 => cuda::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q5_0 => cuda::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q5_1 => cuda::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q8_0 => cuda::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q8_1 => cuda::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q2K => cuda::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q3K => cuda::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q4K => cuda::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q5K => cuda::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q6K => cuda::load_quantized(d, as_t_slice::(data)), - GgmlDType::Q8K => cuda::load_quantized(d, as_t_slice::(data)), - GgmlDType::BF16 => cuda::load_quantized(d, as_t_slice::(data)), - }, - } - } - - fn block_size(&self) -> usize { - match self { - QStorage::Cpu(storage) => storage.block_size(), - QStorage::Metal(storage) => storage.dtype().block_size(), - QStorage::Cuda(storage) => storage.dtype().block_size(), - } - } - - fn dtype(&self) -> GgmlDType { - match self { - QStorage::Cpu(storage) => storage.dtype(), - QStorage::Metal(storage) => storage.dtype(), - QStorage::Cuda(storage) => storage.dtype(), - } - } - - fn device(&self) -> Device { - match self { - QStorage::Cpu(_storage) => Device::Cpu, - QStorage::Metal(storage) => Device::Metal(storage.device().clone()), - QStorage::Cuda(storage) => Device::Cuda(storage.device().clone()), - } - } - - fn size_in_bytes(&self) -> usize { - match self { - QStorage::Cpu(storage) => storage.storage_size_in_bytes(), - QStorage::Metal(storage) => storage.storage_size_in_bytes(), - QStorage::Cuda(storage) => storage.storage_size_in_bytes(), - } - } - - fn quantize(&mut self, src: &Storage) -> Result<()> { - match (self, src) { - (QStorage::Cpu(storage), Storage::Cpu(src)) => { - storage.from_float(src.as_slice::()?); - } - (QStorage::Metal(storage), Storage::Metal(src)) => storage.quantize(src)?, - (QStorage::Cuda(storage), Storage::Cuda(src)) => storage.quantize(src)?, - _ => crate::bail!("Invalid quantize storage locations do not match"), - } - Ok(()) - } - - fn quantize_imatrix( - &mut self, - src: &Storage, - imatrix_weights: &[f32], - n_per_row: usize, - ) -> Result<()> { - match (self, src) { - (QStorage::Cpu(storage), Storage::Cpu(src)) => { - storage.from_float_imatrix(src.as_slice::()?, imatrix_weights, n_per_row); - } - (QStorage::Metal(storage), Storage::Metal(src)) => { - storage.quantize_imatrix(src, imatrix_weights, n_per_row)? - } - (QStorage::Cuda(storage), Storage::Cuda(src)) => { - storage.quantize_imatrix(src, imatrix_weights, n_per_row)? - } - _ => crate::bail!("Invalid quantize storage locations do not match"), - } - Ok(()) - } - - fn quantize_onto(&mut self, src: &Storage) -> Result<()> { - match (self, src) { - (QStorage::Cpu(storage), Storage::Cpu(src)) => { - storage.from_float(src.as_slice::()?); - } - (QStorage::Metal(storage), Storage::Cpu(src)) => storage.quantize_onto(src)?, - (QStorage::Cuda(storage), Storage::Cpu(src)) => storage.quantize_onto(src)?, - _ => crate::bail!("Invalid quantize source storage locations: not on cpu"), - } - Ok(()) - } - - fn quantize_imatrix_onto( - &mut self, - src: &Storage, - imatrix_weights: &[f32], - n_per_row: usize, - ) -> Result<()> { - match (self, src) { - (QStorage::Cpu(storage), Storage::Cpu(src)) => { - storage.from_float_imatrix(src.as_slice::()?, imatrix_weights, n_per_row); - } - (QStorage::Metal(storage), Storage::Cpu(src)) => { - storage.quantize_imatrix_onto(src, imatrix_weights, n_per_row)? - } - (QStorage::Cuda(storage), Storage::Cpu(src)) => { - storage.quantize_imatrix_onto(src, imatrix_weights, n_per_row)? - } - _ => crate::bail!("Invalid quantize storage locations do not match"), - } - Ok(()) - } - - fn dequantize(&self, elem_count: usize) -> Result { - match self { - QStorage::Cpu(storage) => Ok(Storage::Cpu(storage.dequantize(elem_count)?)), - QStorage::Metal(storage) => Ok(Storage::Metal(storage.dequantize(elem_count)?)), - QStorage::Cuda(storage) => Ok(Storage::Cuda(storage.dequantize(elem_count)?)), - } - } - - fn data(&self) -> Result> { - match self { - QStorage::Cpu(storage) => { - let data_ptr = storage.as_ptr(); - let size_in_bytes = storage.storage_size_in_bytes(); - let data = unsafe { std::slice::from_raw_parts(data_ptr, size_in_bytes) }; - Ok(Cow::from(data)) - } - QStorage::Cuda(storage) => Ok(Cow::from(storage.data()?)), - QStorage::Metal(storage) => Ok(Cow::from(storage.data()?)), - } - } - - pub fn device_ptr(&self) -> Result<*const u8> { - match self { - QStorage::Cuda(storage) => storage.device_ptr(), - QStorage::Metal(_) | QStorage::Cpu(_) => { - crate::bail!("not implemented"); - } - } - } -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum GgmlDType { - F32, - F16, - BF16, - Q4_0, - Q4_1, - Q5_0, - Q5_1, - Q8_0, - Q8_1, - Q2K, - Q3K, - Q4K, - Q5K, - Q6K, - Q8K, -} - -impl GgmlDType { - pub(crate) fn from_u32(u: u32) -> Result { - let dtype = match u { - 0 => Self::F32, - 1 => Self::F16, - 2 => Self::Q4_0, - 3 => Self::Q4_1, - 6 => Self::Q5_0, - 7 => Self::Q5_1, - 8 => Self::Q8_0, - 9 => Self::Q8_1, - 10 => Self::Q2K, - 11 => Self::Q3K, - 12 => Self::Q4K, - 13 => Self::Q5K, - 14 => Self::Q6K, - 15 => Self::Q8K, - // https://github.com/ggerganov/ggml/blob/29d87fc6676e7ed0cdfdec0804b06001d9c2bb44/include/ggml.h#L389 - 30 => Self::BF16, - _ => crate::bail!("unknown dtype for tensor {u}"), - }; - Ok(dtype) - } - - pub(crate) fn to_u32(self) -> u32 { - match self { - Self::F32 => 0, - Self::F16 => 1, - Self::Q4_0 => 2, - Self::Q4_1 => 3, - Self::Q5_0 => 6, - Self::Q5_1 => 7, - Self::Q8_0 => 8, - Self::Q8_1 => 9, - Self::Q2K => 10, - Self::Q3K => 11, - Self::Q4K => 12, - Self::Q5K => 13, - Self::Q6K => 14, - Self::Q8K => 15, - // https://github.com/ggerganov/ggml/blob/29d87fc6676e7ed0cdfdec0804b06001d9c2bb44/include/ggml.h#L389 - Self::BF16 => 30, - } - } - - /// The block dtype - pub fn cpu_zeros(&self, elem_count: usize) -> Box { - match self { - Self::F32 => Box::new(vec![f32::zeros(); elem_count]), - Self::F16 => Box::new(vec![f16::zeros(); elem_count]), - Self::Q4_0 => Box::new(vec![BlockQ4_0::zeros(); elem_count / BlockQ4_0::BLCK_SIZE]), - Self::Q4_1 => Box::new(vec![BlockQ4_1::zeros(); elem_count / BlockQ4_1::BLCK_SIZE]), - Self::Q5_0 => Box::new(vec![BlockQ5_0::zeros(); elem_count / BlockQ5_0::BLCK_SIZE]), - Self::Q5_1 => Box::new(vec![BlockQ5_1::zeros(); elem_count / BlockQ5_1::BLCK_SIZE]), - Self::Q8_0 => Box::new(vec![BlockQ8_0::zeros(); elem_count / BlockQ8_0::BLCK_SIZE]), - Self::Q8_1 => Box::new(vec![BlockQ8_1::zeros(); elem_count / BlockQ8_1::BLCK_SIZE]), - Self::Q2K => Box::new(vec![BlockQ2K::zeros(); elem_count / BlockQ2K::BLCK_SIZE]), - Self::Q3K => Box::new(vec![BlockQ3K::zeros(); elem_count / BlockQ3K::BLCK_SIZE]), - Self::Q4K => Box::new(vec![BlockQ4K::zeros(); elem_count / BlockQ4K::BLCK_SIZE]), - Self::Q5K => Box::new(vec![BlockQ5K::zeros(); elem_count / BlockQ5K::BLCK_SIZE]), - Self::Q6K => Box::new(vec![BlockQ6K::zeros(); elem_count / BlockQ6K::BLCK_SIZE]), - Self::Q8K => Box::new(vec![BlockQ8K::zeros(); elem_count / BlockQ8K::BLCK_SIZE]), - Self::BF16 => Box::new(vec![bf16::zeros(); elem_count]), - } - } - - pub fn from_data(&self, data: Cow<'_, [u8]>) -> Box { - match self { - Self::F32 => Box::new(as_t_slice::(data).to_vec()), - Self::F16 => Box::new(as_t_slice::(data).to_vec()), - Self::Q4_0 => Box::new(as_t_slice::(data).to_vec()), - Self::Q4_1 => Box::new(as_t_slice::(data).to_vec()), - Self::Q5_0 => Box::new(as_t_slice::(data).to_vec()), - Self::Q5_1 => Box::new(as_t_slice::(data).to_vec()), - Self::Q8_0 => Box::new(as_t_slice::(data).to_vec()), - Self::Q8_1 => Box::new(as_t_slice::(data).to_vec()), - Self::Q2K => Box::new(as_t_slice::(data).to_vec()), - Self::Q3K => Box::new(as_t_slice::(data).to_vec()), - Self::Q4K => Box::new(as_t_slice::(data).to_vec()), - Self::Q5K => Box::new(as_t_slice::(data).to_vec()), - Self::Q6K => Box::new(as_t_slice::(data).to_vec()), - Self::Q8K => Box::new(as_t_slice::(data).to_vec()), - Self::BF16 => Box::new(as_t_slice::(data).to_vec()), - } - } - - /// The type size for blocks in bytes. - pub fn type_size(&self) -> usize { - use k_quants::*; - match self { - Self::F32 => 4, - Self::F16 | Self::BF16 => 2, - Self::Q4_0 => std::mem::size_of::(), - Self::Q4_1 => std::mem::size_of::(), - Self::Q5_0 => std::mem::size_of::(), - Self::Q5_1 => std::mem::size_of::(), - // https://github.com/ggerganov/llama.cpp/blob/468ea24fb4633a0d681f7ac84089566c1c6190cb/ggml.c#L932 - Self::Q8_0 => std::mem::size_of::(), - Self::Q8_1 => std::mem::size_of::(), - Self::Q2K => std::mem::size_of::(), - Self::Q3K => std::mem::size_of::(), - Self::Q4K => std::mem::size_of::(), - Self::Q5K => std::mem::size_of::(), - Self::Q6K => std::mem::size_of::(), - Self::Q8K => std::mem::size_of::(), - } - } - - /// The block size, i.e. the number of elements stored in each block. - pub fn block_size(&self) -> usize { - match self { - Self::F32 => 1, - Self::F16 | Self::BF16 => 1, - Self::Q4_0 => k_quants::QK4_0, - Self::Q4_1 => k_quants::QK4_1, - Self::Q5_0 => k_quants::QK5_0, - Self::Q5_1 => k_quants::QK5_1, - Self::Q8_0 => k_quants::QK8_0, - Self::Q8_1 => k_quants::QK8_1, - Self::Q2K | Self::Q3K | Self::Q4K | Self::Q5K | Self::Q6K | Self::Q8K => k_quants::QK_K, - } - } -} - -// A version of GgmlType without `vec_dot` so that it can be dyn boxed. -pub trait QuantizedType: Send + Sync { - fn dtype(&self) -> GgmlDType; - fn matmul_t(&self, mkn: (usize, usize, usize), lhs: &[f32], dst: &mut [f32]) -> Result<()>; - fn matmul_t_f16(&self, mkn: (usize, usize, usize), lhs: &[f16], dst: &mut [f16]) -> Result<()>; - fn dequantize(&self, elem_count: usize) -> Result; - fn storage_size_in_bytes(&self) -> usize; - fn as_ptr(&self) -> *const u8; - fn block_size(&self) -> usize; - #[allow(clippy::wrong_self_convention)] - fn from_float(&mut self, xs: &[f32]); - #[allow(clippy::wrong_self_convention)] - fn from_float_imatrix(&mut self, xs: &[f32], imatrix_weights: &[f32], n_per_row: usize); - fn size(&self) -> usize; -} - -impl QuantizedType for Vec { - fn matmul_t(&self, mkn: (usize, usize, usize), lhs: &[f32], dst: &mut [f32]) -> Result<()> { - k_quants::matmul(mkn, lhs, self.as_slice(), dst) - } - fn matmul_t_f16(&self, mkn: (usize, usize, usize), lhs: &[f16], dst: &mut [f16]) -> Result<()> { - k_quants::matmul_f16(mkn, lhs, self.as_slice(), dst) - } - - fn size(&self) -> usize { - self.len() * core::mem::size_of::() - } - - fn from_float(&mut self, xs: &[f32]) { - T::from_float(xs, self) - } - - fn from_float_imatrix(&mut self, xs: &[f32], imatrix_weights: &[f32], n_per_row: usize) { - T::from_float_imatrix(xs, self, imatrix_weights, n_per_row) - } - - fn dtype(&self) -> GgmlDType { - T::DTYPE - } - - fn block_size(&self) -> usize { - T::BLCK_SIZE - } - - fn dequantize(&self, elem_count: usize) -> Result { - let mut ys = vec![0.0f32; elem_count]; - T::to_float(self.as_slice(), &mut ys); - Ok(CpuStorage::F32(ys)) - } - - fn storage_size_in_bytes(&self) -> usize { - self.len() * std::mem::size_of::() - } - - fn as_ptr(&self) -> *const u8 { - self.as_ptr() as *const u8 - } -} - -impl std::fmt::Debug for QTensor { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "QTensor[{:?}; {:?}]", self.shape, self.dtype()) - } -} - -fn check_shape(shape: &Shape, block_size: usize) -> Result<()> { - let dims = shape.dims(); - if dims.is_empty() { - crate::bail!("scalar tensor cannot be quantized {shape:?}") - } - if !dims[dims.len() - 1].is_multiple_of(block_size) { - crate::bail!( - "quantized tensor must have their last dim divisible by block size {shape:?} {}", - block_size - ) - } - Ok(()) -} - -impl QTensor { - pub fn new>(storage: QStorage, shape: S) -> Result { - let shape = shape.into(); - check_shape(&shape, storage.block_size())?; - Ok(Self { storage, shape }) - } - - pub fn quantize(src: &Tensor, dtype: GgmlDType) -> Result { - let shape = src.shape(); - let block_size = dtype.block_size(); - check_shape(shape, block_size)?; - let src = src.to_dtype(crate::DType::F32)?.flatten_all()?; - let elem_count = shape.elem_count(); - if !elem_count.is_multiple_of(block_size) { - crate::bail!( - "tensor size ({shape:?}) is not divisible by block size {}", - block_size - ) - } - let mut storage = src.device().qzeros(elem_count, dtype)?; - storage.quantize(&src.storage())?; - Ok(Self { - storage, - shape: shape.clone(), - }) - } - - pub fn quantize_imatrix( - src: &Tensor, - imatrix_weights: &[f32], - dtype: GgmlDType, - ) -> Result { - // (n_per_row/QK_K-1)*QK_K+(QK_K/32-1)*32+32=n_per_row - // Size of imatrix == last dim of tensor - let n_per_row = src.dim(D::Minus1)?; - if imatrix_weights.len() != n_per_row { - crate::bail!( - "imatrix weights must have the same length {} as the last dim of src {}", - imatrix_weights.len(), - src.dim(D::Minus1)? - ); - } - - let shape = src.shape(); - let block_size = dtype.block_size(); - check_shape(shape, block_size)?; - let src = src.to_dtype(crate::DType::F32)?.flatten_all()?; - let elem_count = shape.elem_count(); - if !elem_count.is_multiple_of(block_size) { - crate::bail!( - "tensor size ({shape:?}) is not divisible by block size {}", - block_size - ); - } - let mut storage = src.device().qzeros(elem_count, dtype)?; - storage.quantize_imatrix(&src.storage(), imatrix_weights, n_per_row)?; - Ok(Self { - storage, - shape: shape.clone(), - }) - } - - /// Quantize `src` (currently on the CPU) to a QTensor on `dev` - pub fn quantize_imatrix_onto( - src: &Tensor, - imatrix_weights: &[f32], - dtype: GgmlDType, - dev: &Device, - ) -> Result { - if !src.device().is_cpu() { - crate::bail!( - "`quantize_onto` expects a `src` to be on the cpu, got {:?}.", - src.device() - ) - } - // (n_per_row/QK_K-1)*QK_K+(QK_K/32-1)*32+32=n_per_row - // Size of imatrix == last dim of tensor - let n_per_row = src.dim(D::Minus1)?; - if imatrix_weights.len() != n_per_row { - crate::bail!( - "imatrix weights must have the same length {} as the last dim of src {}", - imatrix_weights.len(), - src.dim(D::Minus1)? - ); - } - let shape = src.shape(); - let block_size = dtype.block_size(); - check_shape(shape, block_size)?; - let src = src.to_dtype(crate::DType::F32)?.flatten_all()?; - let elem_count = shape.elem_count(); - if !elem_count.is_multiple_of(block_size) { - crate::bail!( - "tensor size ({shape:?}) is not divisible by block size {}", - block_size - ) - } - // storage is on the `dev`, src is on `cpu` - let mut storage = dev.qzeros(elem_count, dtype)?; - storage.quantize_imatrix_onto(&src.storage(), imatrix_weights, n_per_row)?; - Ok(Self { - storage, - shape: shape.clone(), - }) - } - - /// Quantize `src` (currently on the CPU) to a QTensor on `dev` - pub fn quantize_onto(src: &Tensor, dtype: GgmlDType, dev: &Device) -> Result { - if !src.device().is_cpu() { - crate::bail!( - "`quantize_onto` expects a `src` to be on the cpu, got {:?}.", - src.device() - ) - } - let shape = src.shape(); - let block_size = dtype.block_size(); - check_shape(shape, block_size)?; - let src = src.to_dtype(crate::DType::F32)?.flatten_all()?; - let elem_count = shape.elem_count(); - if !elem_count.is_multiple_of(block_size) { - crate::bail!( - "tensor size ({shape:?}) is not divisible by block size {}", - block_size - ) - } - // storage is on the `dev`, src is on `cpu` - let mut storage = dev.qzeros(elem_count, dtype)?; - storage.quantize_onto(&src.storage())?; - Ok(Self { - storage, - shape: shape.clone(), - }) - } - - pub fn dtype(&self) -> GgmlDType { - self.storage.dtype() - } - - pub fn device(&self) -> Device { - self.storage.device() - } - - pub fn rank(&self) -> usize { - self.shape.rank() - } - - pub fn shape(&self) -> &Shape { - &self.shape - } - - pub fn dequantize(&self, device: &Device) -> Result { - let storage = self.storage.dequantize(self.shape.elem_count())?; - let none = crate::op::BackpropOp::none(); - crate::tensor::from_storage(storage, self.shape.clone(), none, false).to_device(device) - } - - pub fn dequantize_f16(&self, device: &Device) -> Result { - // In the CUDA case, we have a specialized kernel as this can be useful for volta - // architectures. https://github.com/huggingface/candle/issues/2136 - match &self.storage { - QStorage::Cuda(s) => { - let s = s.dequantize_f16(self.shape.elem_count())?; - let none = crate::op::BackpropOp::none(); - crate::tensor::from_storage(Storage::Cuda(s), self.shape.clone(), none, false) - .to_device(device) - } - _ => { - let s = self.dequantize(device)?.to_dtype(crate::DType::F16)?; - Ok(s) - } - } - } - - pub fn storage_size_in_bytes(&self) -> usize { - self.storage.size_in_bytes() - } - - pub fn data(&self) -> Result> { - self.storage.data() - } - - pub fn indexed_moe_forward(&self, x: &Tensor, ids: &Tensor) -> Result { - match &self.storage { - QStorage::Cuda(s) => match (&*x.storage(), &*ids.storage()) { - (Storage::Cuda(x_storage), Storage::Cuda(ids_storage)) => { - let (storage, out_shape) = s.indexed_moe_forward( - self.shape(), - x_storage, - x.layout(), - ids_storage, - ids.layout(), - )?; - Ok(crate::tensor::from_storage( - Storage::Cuda(storage), - out_shape, - crate::op::BackpropOp::none(), - false, - )) - } - _ => { - panic!("Non-cuda indexed_moe_forward is not implemented!"); - } - }, - _ => { - panic!("indexed_moe_forward is not implemented in this platform!"); - } - } - } - - pub fn device_ptr(&self) -> Result<*const u8> { - match &self.storage { - QStorage::Cuda(storage) => storage.device_ptr(), - QStorage::Metal(_) | QStorage::Cpu(_) => { - crate::bail!("not implemented"); - } - } - } -} - -#[derive(Clone, Debug)] -pub enum QMatMul { - QTensor(std::sync::Arc), - Tensor(Tensor), - TensorF16(Tensor), -} - -thread_local! { - static DEQUANTIZE_ALL: bool = { - match std::env::var("CANDLE_DEQUANTIZE_ALL") { - Ok(s) => { - !s.is_empty() && s != "0" - }, - Err(_) => false, - } - } -} - -thread_local! { - static DEQUANTIZE_ALL_F16: bool = { - match std::env::var("CANDLE_DEQUANTIZE_ALL_F16") { - Ok(s) => { - !s.is_empty() && s != "0" - }, - Err(_) => false, - } - } -} - -impl QMatMul { - pub fn from_arc(qtensor: std::sync::Arc) -> Result { - let dequantize = match qtensor.dtype() { - GgmlDType::F32 | GgmlDType::F16 | GgmlDType::BF16 => true, - _ => DEQUANTIZE_ALL.with(|b| *b), - }; - let t = if dequantize { - let tensor = qtensor.dequantize(&qtensor.device())?; - Self::Tensor(tensor) - } else if DEQUANTIZE_ALL_F16.with(|b| *b) { - let tensor = qtensor.dequantize_f16(&qtensor.device())?; - Self::TensorF16(tensor) - } else { - Self::QTensor(qtensor) - }; - Ok(t) - } - - pub fn from_qtensor(qtensor: QTensor) -> Result { - Self::from_arc(std::sync::Arc::new(qtensor)) - } - - pub fn dequantize_f16(&self) -> Result { - match self { - Self::QTensor(t) => t.dequantize_f16(&t.device()), - Self::Tensor(t) => t.to_dtype(DType::F16), - Self::TensorF16(t) => Ok(t.clone()), - } - } - - pub fn forward_via_f16(&self, xs: &Tensor) -> Result { - let w = self.dequantize_f16()?; - let in_dtype = xs.dtype(); - let w = match *xs.dims() { - [b1, b2, _, _] => w.broadcast_left((b1, b2))?.t()?, - [bsize, _, _] => w.broadcast_left(bsize)?.t()?, - _ => w.t()?, - }; - xs.to_dtype(DType::F16)?.matmul(&w)?.to_dtype(in_dtype) - } - - pub fn indexed_moe_forward(&self, x: &Tensor, ids: &Tensor) -> Result { - match self { - Self::QTensor(t) => t.indexed_moe_forward(x, ids), - _ => { - panic!("Not implemented!") - } - } - } -} - -impl crate::CustomOp1 for QTensor { - fn name(&self) -> &'static str { - "qmatmul" - } - - fn cpu_fwd( - &self, - storage: &crate::CpuStorage, - layout: &crate::Layout, - ) -> Result<(crate::CpuStorage, Shape)> { - if !layout.is_contiguous() { - crate::bail!("input tensor is not contiguous {layout:?}") - } - let src_shape = layout.shape(); - // self is transposed so n is first then k. - let (n, k) = self.shape.dims2()?; - if src_shape.rank() < 2 { - crate::bail!("input tensor has only one dimension {layout:?}") - } - let mut dst_shape = src_shape.dims().to_vec(); - let last_k = dst_shape.pop().unwrap(); - if last_k != k { - crate::bail!("input tensor {layout:?} incompatible with {:?}", self.shape) - } - dst_shape.push(n); - let dst_shape = Shape::from(dst_shape); - #[allow(clippy::infallible_destructuring_match)] - let self_storage = match &self.storage { - QStorage::Cpu(storage) => storage, - QStorage::Metal(_) | QStorage::Cuda(_) => crate::bail!("Invalid storage"), - }; - match storage.dtype() { - DType::F32 => { - let slice = storage.as_slice::()?; - let slice = - &slice[layout.start_offset()..layout.start_offset() + src_shape.elem_count()]; - let mut dst_storage = vec![0f32; dst_shape.elem_count()]; - self_storage.matmul_t( - (dst_shape.elem_count() / n, k, n), - slice, - &mut dst_storage, - )?; - Ok((crate::CpuStorage::F32(dst_storage), dst_shape)) - } - DType::F16 => { - let slice = storage.as_slice::()?; - let slice = - &slice[layout.start_offset()..layout.start_offset() + src_shape.elem_count()]; - let mut dst_storage = vec![f16::ZERO; dst_shape.elem_count()]; - self_storage.matmul_t_f16( - (dst_shape.elem_count() / n, k, n), - slice, - &mut dst_storage, - )?; - Ok((crate::CpuStorage::F16(dst_storage), dst_shape)) - } - _ => crate::bail!("Expected f32/f16"), - } - } - - fn metal_fwd( - &self, - storage: &crate::MetalStorage, - layout: &crate::Layout, - ) -> Result<(crate::MetalStorage, Shape)> { - let self_storage = match &self.storage { - QStorage::Metal(metal) => metal, - _ => unreachable!("Cannot call metal matmul on non metal QTensor"), - }; - self_storage.fwd(&self.shape, storage, layout) - } - - fn cuda_fwd( - &self, - storage: &crate::CudaStorage, - layout: &crate::Layout, - ) -> Result<(crate::CudaStorage, Shape)> { - let self_storage = match &self.storage { - QStorage::Cuda(cuda) => cuda, - _ => unreachable!("Cannot call cuda matmul on non cuda QTensor"), - }; - self_storage.fwd(&self.shape, storage, layout) - } -} - -impl crate::Module for QMatMul { - fn forward(&self, xs: &Tensor) -> Result { - match self { - Self::QTensor(t) => xs.apply_op1_no_bwd(t.as_ref()), - Self::Tensor(w) => { - let w = match *xs.dims() { - [b1, b2, _, _] => w.broadcast_left((b1, b2))?.t()?, - [bsize, _, _] => w.broadcast_left(bsize)?.t()?, - _ => w.t()?, - }; - xs.matmul(&w) - } - Self::TensorF16(w) => { - let in_dtype = xs.dtype(); - let w = match *xs.dims() { - [b1, b2, _, _] => w.broadcast_left((b1, b2))?.t()?, - [bsize, _, _] => w.broadcast_left(bsize)?.t()?, - _ => w.t()?, - }; - xs.to_dtype(DType::F16)?.matmul(&w)?.to_dtype(in_dtype) - } - } - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/neon.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/neon.rs deleted file mode 100644 index 63196769f5..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/quantized/neon.rs +++ /dev/null @@ -1,615 +0,0 @@ -use super::k_quants::{ - BlockQ2K, BlockQ3K, BlockQ4K, BlockQ4_0, BlockQ5K, BlockQ6K, BlockQ8K, BlockQ8_0, QK8_0, QK_K, -}; -use byteorder::{ByteOrder, LittleEndian}; - -#[allow(unused_imports)] -#[cfg(target_arch = "arm")] -use core::arch::arm::*; - -#[allow(unused_imports)] -#[cfg(target_arch = "aarch64")] -use core::arch::aarch64::*; - -#[inline(always)] -unsafe fn vdotq_s32(a: int8x16_t, b: int8x16_t) -> int32x4_t { - // TODO: dotprod - let p0 = vmull_s8(vget_low_s8(a), vget_low_s8(b)); - let p1 = vmull_s8(vget_high_s8(a), vget_high_s8(b)); - vaddq_s32(vpaddlq_s16(p0), vpaddlq_s16(p1)) -} - -#[inline(always)] -pub(crate) fn vec_dot_q4_0_q8_0(n: usize, xs: &[BlockQ4_0], ys: &[BlockQ8_0]) -> f32 { - debug_assert!( - n.is_multiple_of(QK8_0), - "vec_dot_q4_0_q8_0: {n} is not divisible by {QK8_0}" - ); - let nb = n / QK8_0; - unsafe { - let mut sumv0 = vdupq_n_f32(0.0f32); - for i in 0..nb { - let x0 = &xs[i]; - let y0 = &ys[i]; - - let m4b = vdupq_n_u8(0x0F); - let s8b = vdupq_n_s8(0x8); - - let v0_0 = vld1q_u8(x0.qs.as_ptr()); - - // 4-bit -> 8-bit - let v0_0l = vreinterpretq_s8_u8(vandq_u8(v0_0, m4b)); - let v0_0h = vreinterpretq_s8_u8(vshrq_n_u8(v0_0, 4)); - - // sub 8 - let v0_0ls = vsubq_s8(v0_0l, s8b); - let v0_0hs = vsubq_s8(v0_0h, s8b); - - // load y - let v1_0l = vld1q_s8(y0.qs.as_ptr()); - let v1_0h = vld1q_s8(y0.qs.as_ptr().add(16)); - - let pl0 = vdotq_s32(v0_0ls, v1_0l); - let ph0 = vdotq_s32(v0_0hs, v1_0h); - sumv0 = vmlaq_n_f32( - sumv0, - vcvtq_f32_s32(vaddq_s32(pl0, ph0)), - x0.d.to_f32() * y0.d.to_f32(), - ); - } - vaddvq_f32(sumv0) - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q8_0_q8_0(n: usize, xs: &[BlockQ8_0], ys: &[BlockQ8_0]) -> f32 { - debug_assert!( - n.is_multiple_of(QK8_0), - "vec_dot_q8_0_q8_0: {n} is not divisible by {QK8_0}" - ); - let nb = n / QK8_0; - unsafe { - let mut sumv0 = vdupq_n_f32(0.0f32); - for i in 0..nb { - let x0 = &xs[i]; - let y0 = &ys[i]; - - let x0_0 = vld1q_s8(x0.qs.as_ptr()); - let x0_1 = vld1q_s8(x0.qs.as_ptr().add(16)); - - // load y - let y0_0 = vld1q_s8(y0.qs.as_ptr()); - let y0_1 = vld1q_s8(y0.qs.as_ptr().add(16)); - - let p0 = vdotq_s32(x0_0, y0_0); - let p1 = vdotq_s32(x0_1, y0_1); - - sumv0 = vmlaq_n_f32( - sumv0, - vcvtq_f32_s32(vaddq_s32(p0, p1)), - x0.d.to_f32() * y0.d.to_f32(), - ); - } - vaddvq_f32(sumv0) - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q8k_q8k(n: usize, xs: &[BlockQ8K], ys: &[BlockQ8K]) -> f32 { - debug_assert!( - n.is_multiple_of(QK_K), - "vec_dot_q8k_q8k: {n} is not divisible by {QK_K}" - ); - let mut sumf = 0f32; - for (xs, ys) in xs.iter().zip(ys.iter()) { - unsafe { - let mut sum_i = vdupq_n_s32(0); - let scale = xs.d * ys.d; - let xs = xs.qs.as_ptr(); - let ys = ys.qs.as_ptr(); - for i in (0..QK_K).step_by(16) { - let xs = vld1q_s8(xs.add(i)); - let ys = vld1q_s8(ys.add(i)); - let xy = vdotq_s32(xs, ys); - sum_i = vaddq_s32(sum_i, xy) - } - sumf += vaddvq_s32(sum_i) as f32 * scale - } - } - sumf -} - -#[inline(always)] -pub(crate) fn vec_dot_q6k_q8k(n: usize, xs: &[BlockQ6K], ys: &[BlockQ8K]) -> f32 { - debug_assert!( - n.is_multiple_of(QK_K), - "vec_dot_q6k_q8k: {n} is not divisible by {QK_K}" - ); - let mut sum = 0f32; - unsafe { - let m4b = vdupq_n_u8(0xF); - - let mone = vdupq_n_u8(3); - - for (x, y) in xs.iter().zip(ys.iter()) { - let d_all = x.d.to_f32(); - - let mut q6 = x.ql.as_ptr(); - let mut qh = x.qh.as_ptr(); - let mut q8 = y.qs.as_ptr(); - - let mut scale = x.scales.as_ptr(); - - let q8sums = vld1q_s16_x2(y.bsums.as_ptr()); - let scales = vld1q_s8(scale); - let q6scales = int16x8x2_t( - vmovl_s8(vget_low_s8(scales)), - vmovl_s8(vget_high_s8(scales)), - ); - - let prod = vaddq_s32( - vaddq_s32( - vmull_s16(vget_low_s16(q8sums.0), vget_low_s16(q6scales.0)), - vmull_s16(vget_high_s16(q8sums.0), vget_high_s16(q6scales.0)), - ), - vaddq_s32( - vmull_s16(vget_low_s16(q8sums.1), vget_low_s16(q6scales.1)), - vmull_s16(vget_high_s16(q8sums.1), vget_high_s16(q6scales.1)), - ), - ); - let isum_mins = vaddvq_s32(prod); - - let mut isum = 0i32; - - for _j in 0..QK_K / 128 { - let qhbits = vld1q_u8_x2(qh); - qh = qh.add(32); - let q6bits = vld1q_u8_x4(q6); - q6 = q6.add(64); - let q8bytes = vld1q_s8_x4(q8); - q8 = q8.add(64); - - let q6h_0 = vshlq_n_u8(vandq_u8(mone, qhbits.0), 4); - let q6h_1 = vshlq_n_u8(vandq_u8(mone, qhbits.1), 4); - let shifted = vshrq_n_u8(qhbits.0, 2); - let q6h_2 = vshlq_n_u8(vandq_u8(mone, shifted), 4); - let shifted = vshrq_n_u8(qhbits.1, 2); - let q6h_3 = vshlq_n_u8(vandq_u8(mone, shifted), 4); - - let q6bytes_0 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.0, m4b), q6h_0)); - let q6bytes_1 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.1, m4b), q6h_1)); - let q6bytes_2 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.2, m4b), q6h_2)); - let q6bytes_3 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q6bits.3, m4b), q6h_3)); - - let p0 = vdotq_s32(q6bytes_0, q8bytes.0); - let p1 = vdotq_s32(q6bytes_1, q8bytes.1); - let (scale0, scale1) = (*scale as i32, *scale.add(1) as i32); - isum += vaddvq_s32(p0) * scale0 + vaddvq_s32(p1) * scale1; - scale = scale.add(2); - - let p2 = vdotq_s32(q6bytes_2, q8bytes.2); - let p3 = vdotq_s32(q6bytes_3, q8bytes.3); - let (scale0, scale1) = (*scale as i32, *scale.add(1) as i32); - isum += vaddvq_s32(p2) * scale0 + vaddvq_s32(p3) * scale1; - scale = scale.add(2); - - let q8bytes = vld1q_s8_x4(q8); - q8 = q8.add(64); - - let shifted = vshrq_n_u8(qhbits.0, 4); - let q6h_0 = vshlq_n_u8(vandq_u8(mone, shifted), 4); - let shifted = vshrq_n_u8(qhbits.1, 4); - let q6h_1 = vshlq_n_u8(vandq_u8(mone, shifted), 4); - let shifted = vshrq_n_u8(qhbits.0, 6); - let q6h_2 = vshlq_n_u8(vandq_u8(mone, shifted), 4); - let shifted = vshrq_n_u8(qhbits.1, 6); - let q6h_3 = vshlq_n_u8(vandq_u8(mone, shifted), 4); - - let q6bytes_0 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.0, 4), q6h_0)); - let q6bytes_1 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.1, 4), q6h_1)); - let q6bytes_2 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.2, 4), q6h_2)); - let q6bytes_3 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q6bits.3, 4), q6h_3)); - - let p0 = vdotq_s32(q6bytes_0, q8bytes.0); - let p1 = vdotq_s32(q6bytes_1, q8bytes.1); - let (scale0, scale1) = (*scale as i32, *scale.add(1) as i32); - isum += vaddvq_s32(p0) * scale0 + vaddvq_s32(p1) * scale1; - scale = scale.add(2); - - let p2 = vdotq_s32(q6bytes_2, q8bytes.2); - let p3 = vdotq_s32(q6bytes_3, q8bytes.3); - let (scale0, scale1) = (*scale as i32, *scale.add(1) as i32); - isum += vaddvq_s32(p2) * scale0 + vaddvq_s32(p3) * scale1; - scale = scale.add(2); - } - sum += d_all * y.d * ((isum - 32 * isum_mins) as f32); - } - } - sum -} - -#[inline(always)] -pub(crate) fn vec_dot_q5k_q8k(n: usize, xs: &[BlockQ5K], ys: &[BlockQ8K]) -> f32 { - debug_assert!( - n.is_multiple_of(QK_K), - "vec_dot_q5k_q8k: {n} is not divisible by {QK_K}" - ); - let mut sumf = 0f32; - let mut utmp = [0u32; 4]; - const KMASK1: u32 = 0x3f3f3f3f; - const KMASK2: u32 = 0x0f0f0f0f; - const KMASK3: u32 = 0x03030303; - - unsafe { - let m4b = vdupq_n_u8(0xF); - let mone = vdupq_n_u8(1); - let mtwo = vdupq_n_u8(2); - - for (x, y) in xs.iter().zip(ys.iter()) { - let d = y.d * x.d.to_f32(); - let dmin = y.d * x.dmin.to_f32(); - - let q8sums = vpaddq_s16( - vld1q_s16(y.bsums.as_ptr()), - vld1q_s16(y.bsums.as_ptr().add(8)), - ); - - LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); - - utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); - let uaux = utmp[1] & KMASK1; - utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); - utmp[2] = uaux; - utmp[0] &= KMASK1; - - let mins8 = vld1_u8((utmp.as_ptr() as *const u8).add(8)); - let mins = vreinterpretq_s16_u16(vmovl_u8(mins8)); - let prod = vaddq_s32( - vmull_s16(vget_low_s16(q8sums), vget_low_s16(mins)), - vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)), - ); - let sumi_mins = vaddvq_s32(prod); - - let mut scales = utmp.as_ptr() as *const u8; - - let mut q5 = x.qs.as_ptr(); - let mut q8 = y.qs.as_ptr(); - - let mut qhbits = vld1q_u8_x2(x.qh.as_ptr()); - - let mut sumi = 0i32; - - for _j in 0..QK_K / 64 { - let q5bits = vld1q_u8_x2(q5); - q5 = q5.add(32); - let q8bytes = vld1q_s8_x4(q8); - q8 = q8.add(64); - - let q5h_0 = vshlq_n_u8(vandq_u8(mone, qhbits.0), 4); - let q5h_1 = vshlq_n_u8(vandq_u8(mone, qhbits.1), 4); - let q5h_2 = vshlq_n_u8(vandq_u8(mtwo, qhbits.0), 3); - let q5h_3 = vshlq_n_u8(vandq_u8(mtwo, qhbits.1), 3); - qhbits.0 = vshrq_n_u8(qhbits.0, 2); - qhbits.1 = vshrq_n_u8(qhbits.1, 2); - - let q5bytes_0 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q5bits.0, m4b), q5h_0)); - let q5bytes_1 = vreinterpretq_s8_u8(vorrq_u8(vandq_u8(q5bits.1, m4b), q5h_1)); - let q5bytes_2 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.0, 4), q5h_2)); - let q5bytes_3 = vreinterpretq_s8_u8(vorrq_u8(vshrq_n_u8(q5bits.1, 4), q5h_3)); - - let p0 = vdotq_s32(q5bytes_0, q8bytes.0); - let p1 = vdotq_s32(q5bytes_1, q8bytes.1); - sumi += vaddvq_s32(vaddq_s32(p0, p1)) * *scales as i32; - scales = scales.add(1); - - let p2 = vdotq_s32(q5bytes_2, q8bytes.2); - let p3 = vdotq_s32(q5bytes_3, q8bytes.3); - sumi += vaddvq_s32(vaddq_s32(p2, p3)) * *scales as i32; - scales = scales.add(1); - } - sumf += d * sumi as f32 - dmin * sumi_mins as f32; - } - } - sumf -} - -#[inline(always)] -pub(crate) fn vec_dot_q4k_q8k(n: usize, xs: &[BlockQ4K], ys: &[BlockQ8K]) -> f32 { - debug_assert!( - n.is_multiple_of(QK_K), - "vec_dot_q4k_q8k: {n} is not divisible by {QK_K}" - ); - let mut sumf = 0f32; - let mut utmp = [0u32; 4]; - let mut scales = [0u8; 16]; - const KMASK1: u32 = 0x3f3f3f3f; - const KMASK2: u32 = 0x0f0f0f0f; - const KMASK3: u32 = 0x03030303; - - unsafe { - let m4b = vdupq_n_u8(0xF); - - for (x, y) in xs.iter().zip(ys.iter()) { - let d = y.d * x.d.to_f32(); - let dmin = y.d * x.dmin.to_f32(); - - let q8sums = vpaddq_s16( - vld1q_s16(y.bsums.as_ptr()), - vld1q_s16(y.bsums.as_ptr().add(8)), - ); - - LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); - - let mins8 = vld1_u32( - [ - utmp[1] & KMASK1, - ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4), - ] - .as_ptr(), - ); - utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); - utmp[0] &= KMASK1; - - let mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins8))); - let prod = vaddq_s32( - vmull_s16(vget_low_s16(q8sums), vget_low_s16(mins)), - vmull_s16(vget_high_s16(q8sums), vget_high_s16(mins)), - ); - sumf -= dmin * vaddvq_s32(prod) as f32; - - LittleEndian::write_u32_into(&utmp, &mut scales); - - let mut q4 = x.qs.as_ptr(); - let mut q8 = y.qs.as_ptr(); - - let mut sumi1 = 0i32; - let mut sumi2 = 0i32; - - for j in 0..QK_K / 64 { - let q4bits = vld1q_u8_x2(q4); - q4 = q4.add(32); - let q8bytes = vld1q_s8_x2(q8); - q8 = q8.add(32); - let q4bytes = int8x16x2_t( - vreinterpretq_s8_u8(vandq_u8(q4bits.0, m4b)), - vreinterpretq_s8_u8(vandq_u8(q4bits.1, m4b)), - ); - let p0 = vdotq_s32(q4bytes.0, q8bytes.0); - let p1 = vdotq_s32(q4bytes.1, q8bytes.1); - sumi1 += vaddvq_s32(vaddq_s32(p0, p1)) * scales[2 * j] as i32; - - let q8bytes = vld1q_s8_x2(q8); - q8 = q8.add(32); - let q4bytes = int8x16x2_t( - vreinterpretq_s8_u8(vshrq_n_u8(q4bits.0, 4)), - vreinterpretq_s8_u8(vshrq_n_u8(q4bits.1, 4)), - ); - let p2 = vdotq_s32(q4bytes.0, q8bytes.0); - let p3 = vdotq_s32(q4bytes.1, q8bytes.1); - sumi2 += vaddvq_s32(vaddq_s32(p2, p3)) * scales[2 * j + 1] as i32; - } - sumf += d * (sumi1 + sumi2) as f32; - } - } - sumf -} - -#[inline(always)] -pub(crate) fn vec_dot_q3k_q8k(n: usize, xs: &[BlockQ3K], ys: &[BlockQ8K]) -> f32 { - debug_assert!( - n.is_multiple_of(QK_K), - "vec_dot_q3k_q8k: {n} is not divisible by {QK_K}" - ); - let mut sumf = 0f32; - let mut utmp = [0u32; 4]; - let mut aux = [0u32; 3]; - const KMASK1: u32 = 0x03030303; - const KMASK2: u32 = 0x0f0f0f0f; - - unsafe { - let m3b = vdupq_n_u8(0x3); - let m0 = vdupq_n_u8(1); - let m1 = vshlq_n_u8(m0, 1); - let m2 = vshlq_n_u8(m0, 2); - let m3 = vshlq_n_u8(m0, 3); - for (x, y) in xs.iter().zip(ys.iter()) { - let d = y.d * x.d.to_f32(); - let mut q3 = x.qs.as_ptr(); - let qh = x.hmask.as_ptr(); - let mut q8 = y.qs.as_ptr(); - - let mut qhbits = vld1q_u8_x2(qh); - - let mut isum = 0i32; - - // Set up scales - LittleEndian::read_u32_into(&x.scales, &mut aux); - - utmp[3] = ((aux[1] >> 4) & KMASK2) | (((aux[2] >> 6) & KMASK1) << 4); - utmp[2] = ((aux[0] >> 4) & KMASK2) | (((aux[2] >> 4) & KMASK1) << 4); - utmp[1] = (aux[1] & KMASK2) | (((aux[2] >> 2) & KMASK1) << 4); - utmp[0] = (aux[0] & KMASK2) | ((aux[2] & KMASK1) << 4); - - let mut scale = utmp.as_mut_ptr() as *mut i8; - for j in 0..16 { - *scale.add(j) -= 32i8 - } - - for j in 0..QK_K / 128 { - let q3bits = vld1q_u8_x2(q3); - q3 = q3.add(32); - let q8bytes_1 = vld1q_s8_x4(q8); - q8 = q8.add(64); - let q8bytes_2 = vld1q_s8_x4(q8); - q8 = q8.add(64); - - let q3h_0 = vshlq_n_u8(vbicq_u8(m0, qhbits.0), 2); - let q3h_1 = vshlq_n_u8(vbicq_u8(m0, qhbits.1), 2); - let q3h_2 = vshlq_n_u8(vbicq_u8(m1, qhbits.0), 1); - let q3h_3 = vshlq_n_u8(vbicq_u8(m1, qhbits.1), 1); - - let q3bytes_0 = vsubq_s8( - vreinterpretq_s8_u8(vandq_u8(q3bits.0, m3b)), - vreinterpretq_s8_u8(q3h_0), - ); - let q3bytes_1 = vsubq_s8( - vreinterpretq_s8_u8(vandq_u8(q3bits.1, m3b)), - vreinterpretq_s8_u8(q3h_1), - ); - let q3bytes_2 = vsubq_s8( - vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.0, 2), m3b)), - vreinterpretq_s8_u8(q3h_2), - ); - let q3bytes_3 = vsubq_s8( - vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.1, 2), m3b)), - vreinterpretq_s8_u8(q3h_3), - ); - - let p0 = vdotq_s32(q3bytes_0, q8bytes_1.0); - let p1 = vdotq_s32(q3bytes_1, q8bytes_1.1); - let p2 = vdotq_s32(q3bytes_2, q8bytes_1.2); - let p3 = vdotq_s32(q3bytes_3, q8bytes_1.3); - isum += vaddvq_s32(p0) * *scale as i32 - + vaddvq_s32(p1) * *scale.add(1) as i32 - + vaddvq_s32(p2) * *scale.add(2) as i32 - + vaddvq_s32(p3) * *scale.add(3) as i32; - scale = scale.add(4); - - let q3h_0 = vbicq_u8(m2, qhbits.0); - let q3h_1 = vbicq_u8(m2, qhbits.1); - let q3h_2 = vshrq_n_u8(vbicq_u8(m3, qhbits.0), 1); - let q3h_3 = vshrq_n_u8(vbicq_u8(m3, qhbits.1), 1); - - let q3bytes_0 = vsubq_s8( - vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.0, 4), m3b)), - vreinterpretq_s8_u8(q3h_0), - ); - let q3bytes_1 = vsubq_s8( - vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.1, 4), m3b)), - vreinterpretq_s8_u8(q3h_1), - ); - let q3bytes_2 = vsubq_s8( - vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.0, 6), m3b)), - vreinterpretq_s8_u8(q3h_2), - ); - let q3bytes_3 = vsubq_s8( - vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q3bits.1, 6), m3b)), - vreinterpretq_s8_u8(q3h_3), - ); - - let p0 = vdotq_s32(q3bytes_0, q8bytes_2.0); - let p1 = vdotq_s32(q3bytes_1, q8bytes_2.1); - let p2 = vdotq_s32(q3bytes_2, q8bytes_2.2); - let p3 = vdotq_s32(q3bytes_3, q8bytes_2.3); - isum += vaddvq_s32(p0) * *scale as i32 - + vaddvq_s32(p1) * *scale.add(1) as i32 - + vaddvq_s32(p2) * *scale.add(2) as i32 - + vaddvq_s32(p3) * *scale.add(3) as i32; - scale = scale.add(4); - - if j == 0 { - qhbits.0 = vshrq_n_u8(qhbits.0, 4); - qhbits.1 = vshrq_n_u8(qhbits.1, 4); - } - } - sumf += d * isum as f32; - } - } - sumf -} - -#[inline(always)] -pub(crate) fn vec_dot_q2k_q8k(n: usize, xs: &[BlockQ2K], ys: &[BlockQ8K]) -> f32 { - debug_assert!( - n.is_multiple_of(QK_K), - "vec_dot_q2k_q8k: {n} is not divisible by {QK_K}" - ); - let mut sumf = 0f32; - let mut aux = [0u8; 16]; - - unsafe { - let m3 = vdupq_n_u8(0x3); - let m4 = vdupq_n_u8(0xF); - - for (x, y) in xs.iter().zip(ys.iter()) { - let d = y.d * x.d.to_f32(); - let dmin = -y.d * x.dmin.to_f32(); - - let mut q2 = x.qs.as_ptr(); - let mut q8 = y.qs.as_ptr(); - let sc = x.scales.as_ptr(); - - let mins_and_scales = vld1q_u8(sc); - let scales = vandq_u8(mins_and_scales, m4); - vst1q_u8(aux.as_mut_ptr(), scales); - - let mins = vshrq_n_u8(mins_and_scales, 4); - let q8sums = vld1q_s16_x2(y.bsums.as_ptr()); - let mins16 = int16x8x2_t( - vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(mins))), - vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(mins))), - ); - let s0 = vaddq_s32( - vmull_s16(vget_low_s16(mins16.0), vget_low_s16(q8sums.0)), - vmull_s16(vget_high_s16(mins16.0), vget_high_s16(q8sums.0)), - ); - let s1 = vaddq_s32( - vmull_s16(vget_low_s16(mins16.1), vget_low_s16(q8sums.1)), - vmull_s16(vget_high_s16(mins16.1), vget_high_s16(q8sums.1)), - ); - sumf += dmin * vaddvq_s32(vaddq_s32(s0, s1)) as f32; - - let mut isum = 0i32; - let mut is = 0usize; - - // TODO: dotprod - for _j in 0..QK_K / 128 { - let q2bits = vld1q_u8_x2(q2); - q2 = q2.add(32); - - let q8bytes = vld1q_s8_x2(q8); - q8 = q8.add(32); - let mut q2bytes = int8x16x2_t( - vreinterpretq_s8_u8(vandq_u8(q2bits.0, m3)), - vreinterpretq_s8_u8(vandq_u8(q2bits.1, m3)), - ); - isum += multiply_accum_with_scale(&aux, is, 0, q2bytes, q8bytes); - - let q8bytes = vld1q_s8_x2(q8); - q8 = q8.add(32); - q2bytes.0 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.0, 2), m3)); - q2bytes.1 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.1, 2), m3)); - isum += multiply_accum_with_scale(&aux, is, 2, q2bytes, q8bytes); - - let q8bytes = vld1q_s8_x2(q8); - q8 = q8.add(32); - q2bytes.0 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.0, 4), m3)); - q2bytes.1 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.1, 4), m3)); - isum += multiply_accum_with_scale(&aux, is, 4, q2bytes, q8bytes); - - let q8bytes = vld1q_s8_x2(q8); - q8 = q8.add(32); - q2bytes.0 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.0, 6), m3)); - q2bytes.1 = vreinterpretq_s8_u8(vandq_u8(vshrq_n_u8(q2bits.1, 6), m3)); - isum += multiply_accum_with_scale(&aux, is, 6, q2bytes, q8bytes); - - is += 8; - } - sumf += d * isum as f32; - } - } - sumf -} - -#[inline(always)] -unsafe fn multiply_accum_with_scale( - aux: &[u8; 16], - is: usize, - index: usize, - q2bytes: int8x16x2_t, - q8bytes: int8x16x2_t, -) -> i32 { - let p1 = vdotq_s32(q2bytes.0, q8bytes.0); - let p2 = vdotq_s32(q2bytes.1, q8bytes.1); - vaddvq_s32(p1) * aux[is + index] as i32 + vaddvq_s32(p2) * aux[is + 1 + index] as i32 -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/simd128.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/simd128.rs deleted file mode 100644 index 4c02f9919e..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/quantized/simd128.rs +++ /dev/null @@ -1,418 +0,0 @@ -use super::k_quants::{BlockQ2K, BlockQ4K, BlockQ4_0, BlockQ6K, BlockQ8K, BlockQ8_0, QK8_0, QK_K}; -use byteorder::{ByteOrder, LittleEndian}; -use half::f16; - -use core::arch::wasm32::*; - -#[inline(always)] -pub(crate) fn vec_dot_q4_0_q8_0(n: usize, xs: &[BlockQ4_0], ys: &[BlockQ8_0]) -> f32 { - debug_assert!( - n.is_multiple_of(QK8_0), - "vec_dot_q4_0_q8_0: {n} is not divisible by {QK8_0}" - ); - unsafe { - let mut acc = f32x4_splat(0.0f32); - for (x, y) in xs.iter().zip(ys.iter()) { - let x1234 = v128_load(x.qs.as_ptr() as *const v128); - let x12 = v128_and(x1234, u8x16_splat(0x0F)); - let x12 = i8x16_sub(x12, i8x16_splat(8)); - let x34 = u8x16_shr(x1234, 4); - let x34 = i8x16_sub(x34, i8x16_splat(8)); - - let x1 = i16x8_extend_low_i8x16(x12); - let y1 = i16x8_load_extend_i8x8(y.qs.as_ptr()); - let sum_xy = i32x4_dot_i16x8(x1, y1); - - let x2 = i16x8_extend_high_i8x16(x12); - let y2 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(8)); - let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x2, y2)); - - let x3 = i16x8_extend_low_i8x16(x34); - let y3 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(16)); - let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x3, y3)); - - let x4 = i16x8_extend_high_i8x16(x34); - let y4 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(24)); - let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x4, y4)); - - let sum_xy = f32x4_convert_i32x4(sum_xy); - - // f32x4_relaxed_madd is nightly only. - let d = f32x4_splat(f16::to_f32(x.d) * f16::to_f32(y.d)); - let scaled = f32x4_mul(sum_xy, d); - acc = f32x4_add(acc, scaled) - } - let res = f32x4_extract_lane::<0>(acc) - + f32x4_extract_lane::<1>(acc) - + f32x4_extract_lane::<2>(acc) - + f32x4_extract_lane::<3>(acc); - res - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q8_0_q8_0(n: usize, xs: &[BlockQ8_0], ys: &[BlockQ8_0]) -> f32 { - debug_assert!( - n.is_multiple_of(QK8_0), - "vec_dot_q8_0_q8_0: {n} is not divisible by {QK8_0}" - ); - unsafe { - let mut acc = f32x4_splat(0.0f32); - for (x, y) in xs.iter().zip(ys.iter()) { - let x1 = i16x8_load_extend_i8x8(x.qs.as_ptr()); - let y1 = i16x8_load_extend_i8x8(y.qs.as_ptr()); - let sum_xy = i32x4_dot_i16x8(x1, y1); - - let x2 = i16x8_load_extend_i8x8(x.qs.as_ptr().add(8)); - let y2 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(8)); - let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x2, y2)); - - let x3 = i16x8_load_extend_i8x8(x.qs.as_ptr().add(16)); - let y3 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(16)); - let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x3, y3)); - - let x4 = i16x8_load_extend_i8x8(x.qs.as_ptr().add(24)); - let y4 = i16x8_load_extend_i8x8(y.qs.as_ptr().add(24)); - let sum_xy = i32x4_add(sum_xy, i32x4_dot_i16x8(x4, y4)); - - let sum_xy = f32x4_convert_i32x4(sum_xy); - - // f32x4_relaxed_madd is nightly only. - let d = f32x4_splat(f16::to_f32(x.d) * f16::to_f32(y.d)); - let scaled = f32x4_mul(sum_xy, d); - acc = f32x4_add(acc, scaled) - } - let res = f32x4_extract_lane::<0>(acc) - + f32x4_extract_lane::<1>(acc) - + f32x4_extract_lane::<2>(acc) - + f32x4_extract_lane::<3>(acc); - res - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q2k_q8k(n: usize, xs: &[BlockQ2K], ys: &[BlockQ8K]) -> f32 { - debug_assert!( - n.is_multiple_of(QK_K), - "vec_dot_q2k_q8k: {n} is not divisible by {QK_K}" - ); - unsafe { - let mut sumf = f32x4_splat(0f32); - for (x, y) in xs.iter().zip(ys.iter()) { - let mut q2: &[_] = &x.qs; - let mut q8: &[_] = &y.qs; - let sc = &x.scales; - - let mut summs = i32x4_splat(0); - for i in (0..(QK_K / 16)).step_by(4) { - let bsums = i32x4_load_extend_i16x4(y.bsums.as_ptr().add(i)); - let scales = i32x4_shr( - i32x4( - sc[i] as i32, - sc[i + 1] as i32, - sc[i + 2] as i32, - sc[i + 3] as i32, - ), - 4, - ); - summs = i32x4_add(summs, i32x4_mul(bsums, scales)) - } - let summs = f32x4_convert_i32x4(summs); - - let dall = y.d * x.d.to_f32(); - let dmin = y.d * x.dmin.to_f32(); - - let mut isum = i32x4_splat(0); - let mut is = 0; - for _ in 0..(QK_K / 128) { - let mut shift = 0; - for _ in 0..4 { - let d = (sc[is] & 0xF) as i32; - is += 1; - let mut isuml = i16x8_splat(0); - for l in (0..16).step_by(8) { - let q8 = i16x8_load_extend_i8x8(q8.as_ptr().add(l)); - let q2 = i16x8_load_extend_u8x8(q2.as_ptr().add(l)); - let q2 = v128_and(i16x8_shr(q2, shift), i16x8_splat(3)); - isuml = i16x8_add(isuml, i16x8_mul(q2, q8)) - } - let dd = i32x4_splat(d); - isum = i32x4_add(isum, i32x4_mul(i32x4_extend_low_i16x8(isuml), dd)); - isum = i32x4_add(isum, i32x4_mul(i32x4_extend_high_i16x8(isuml), dd)); - let d = (sc[is] & 0xF) as i32; - is += 1; - let mut isuml = i16x8_splat(0); - for l in (16..32).step_by(8) { - let q8 = i16x8_load_extend_i8x8(q8.as_ptr().add(l)); - let q2 = i16x8_load_extend_u8x8(q2.as_ptr().add(l)); - let q2 = v128_and(i16x8_shr(q2, shift), i16x8_splat(3)); - isuml = i16x8_add(isuml, i16x8_mul(q2, q8)) - } - let dd = i32x4_splat(d); - isum = i32x4_add(isum, i32x4_mul(i32x4_extend_low_i16x8(isuml), dd)); - isum = i32x4_add(isum, i32x4_mul(i32x4_extend_high_i16x8(isuml), dd)); - shift += 2; - // adjust the indexing - q8 = &q8[32..]; - } - // adjust the indexing - q2 = &q2[32..]; - } - let isum = f32x4_convert_i32x4(isum); - sumf = f32x4_add( - sumf, - f32x4_sub( - f32x4_mul(isum, f32x4_splat(dall)), - f32x4_mul(summs, f32x4_splat(dmin)), - ), - ); - } - let sumf = f32x4_extract_lane::<0>(sumf) - + f32x4_extract_lane::<1>(sumf) - + f32x4_extract_lane::<2>(sumf) - + f32x4_extract_lane::<3>(sumf); - sumf - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q4k_q8k(n: usize, xs: &[BlockQ4K], ys: &[BlockQ8K]) -> f32 { - debug_assert!( - n.is_multiple_of(QK_K), - "vec_dot_q4k_q8k: {n} is not divisible by {QK_K}" - ); - const KMASK1: u32 = 0x3f3f3f3f; - const KMASK2: u32 = 0x0f0f0f0f; - const KMASK3: u32 = 0x03030303; - - let mut utmp: [u32; 4] = [0; 4]; - let mut scales: [u8; 8] = [0; 8]; - let mut mins: [u8; 8] = [0; 8]; - - let mut aux8: [u8; QK_K] = [0; QK_K]; - let mut sums = f32x4_splat(0f32); - unsafe { - for (y, x) in ys.iter().zip(xs.iter()) { - let q4 = &x.qs; - let q8 = &y.qs; - - for j in 0..QK_K / 64 { - let q4_1 = v128_load(q4.as_ptr().add(32 * j) as *const v128); - let q4_2 = v128_load(q4.as_ptr().add(32 * j + 16) as *const v128); - v128_store( - aux8.as_mut_ptr().add(64 * j) as *mut v128, - v128_and(q4_1, u8x16_splat(0x0F)), - ); - v128_store( - aux8.as_mut_ptr().add(64 * j + 16) as *mut v128, - v128_and(q4_2, u8x16_splat(0x0F)), - ); - v128_store( - aux8.as_mut_ptr().add(64 * j + 32) as *mut v128, - u8x16_shr(q4_1, 4), - ); - v128_store( - aux8.as_mut_ptr().add(64 * j + 48) as *mut v128, - u8x16_shr(q4_2, 4), - ); - } - - LittleEndian::read_u32_into(&x.scales, &mut utmp[0..3]); - - utmp[3] = ((utmp[2] >> 4) & KMASK2) | (((utmp[1] >> 6) & KMASK3) << 4); - let uaux = utmp[1] & KMASK1; - utmp[1] = (utmp[2] & KMASK2) | (((utmp[0] >> 6) & KMASK3) << 4); - utmp[2] = uaux; - utmp[0] &= KMASK1; - - //extract scales and mins - LittleEndian::write_u32_into(&utmp[0..2], &mut scales); - LittleEndian::write_u32_into(&utmp[2..4], &mut mins); - - let mut sumi = i32x4_splat(0); - for j in (0..QK_K / 16).step_by(4) { - let bsums = i32x4_load_extend_i16x4(y.bsums.as_ptr().add(j)); - let (m1, m2) = (mins[j / 2] as i32, mins[j / 2 + 1] as i32); - let mins = i32x4(m1, m1, m2, m2); - sumi = i32x4_add(sumi, i32x4_mul(bsums, mins)); - } - - let mut aux32 = i32x4_splat(0i32); - for (scale_i, scale) in scales.iter().enumerate() { - let scale = i32x4_splat(*scale as i32); - for j in 0..4 { - let i = 32 * scale_i + 8 * j; - let q8 = i16x8_load_extend_i8x8(q8.as_ptr().add(i)); - let aux8 = i16x8_load_extend_u8x8(aux8.as_ptr().add(i)); - let aux16 = i16x8_mul(q8, aux8); - aux32 = i32x4_add(aux32, i32x4_mul(scale, i32x4_extend_low_i16x8(aux16))); - aux32 = i32x4_add(aux32, i32x4_mul(scale, i32x4_extend_high_i16x8(aux16))); - } - } - let aux32 = f32x4_convert_i32x4(aux32); - let d = f32x4_splat(x.d.to_f32() * y.d); - sums = f32x4_add(sums, f32x4_mul(aux32, d)); - let dmin = x.dmin.to_f32() * y.d; - let dmin = f32x4_splat(dmin); - let sumi = f32x4_convert_i32x4(sumi); - sums = f32x4_sub(sums, f32x4_mul(sumi, dmin)); - } - let sums = f32x4_extract_lane::<0>(sums) - + f32x4_extract_lane::<1>(sums) - + f32x4_extract_lane::<2>(sums) - + f32x4_extract_lane::<3>(sums); - sums - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q6k_q8k(n: usize, xs: &[BlockQ6K], ys: &[BlockQ8K]) -> f32 { - debug_assert!( - n.is_multiple_of(QK_K), - "vec_dot_q6k_q8k: {n} is not divisible by {QK_K}" - ); - let mut aux8 = [0i8; QK_K]; - unsafe { - let mut sums = f32x4_splat(0f32); - - for (x, y) in xs.iter().zip(ys.iter()) { - let q4 = &x.ql; - let qh = &x.qh; - let q8 = &y.qs; - let mut aux32 = f32x4_splat(0f32); - - for j in (0..QK_K).step_by(128) { - let aux8 = aux8.as_mut_ptr().add(j); - let q4 = &q4.as_ptr().add(j / 2); - let qh = &qh.as_ptr().add(j / 4); - for l in (0..32).step_by(16) { - // aux8[l] = (((q4[l] & 0xF) | ((qh[l] & 3) << 4)) as i32 - 32) as i8; - let a8 = v128_or( - v128_and(v128_load(q4.add(l) as *const v128), u8x16_splat(0xF)), - u8x16_shl( - v128_and(v128_load(qh.add(l) as *const v128), u8x16_splat(3)), - 4, - ), - ); - let a8_low = i16x8_sub(i16x8_extend_low_u8x16(a8), i16x8_splat(32)); - let a8_high = i16x8_sub(i16x8_extend_high_u8x16(a8), i16x8_splat(32)); - v128_store( - aux8.add(l) as *mut v128, - i8x16_narrow_i16x8(a8_low, a8_high), - ); - - // aux8[l + 32] = - // (((q4[l + 32] & 0xF) | (((qh[l] >> 2) & 3) << 4)) as i32 - 32) as i8; - let a8 = v128_or( - v128_and(v128_load(q4.add(l + 32) as *const v128), u8x16_splat(0xF)), - u8x16_shl( - v128_and( - u8x16_shr(v128_load(qh.add(l) as *const v128), 2), - u8x16_splat(3), - ), - 4, - ), - ); - let a8_low = i16x8_sub(i16x8_extend_low_u8x16(a8), i16x8_splat(32)); - let a8_high = i16x8_sub(i16x8_extend_high_u8x16(a8), i16x8_splat(32)); - v128_store( - aux8.add(l + 32) as *mut v128, - i8x16_narrow_i16x8(a8_low, a8_high), - ); - - // aux8[l + 64] = (((q4[l] >> 4) | (((qh[l] >> 4) & 3) << 4)) as i32 - 32) as i8; - let a8 = v128_or( - u8x16_shr(v128_load(q4.add(l) as *const v128), 4), - u8x16_shl( - v128_and( - u8x16_shr(v128_load(qh.add(l) as *const v128), 4), - u8x16_splat(3), - ), - 4, - ), - ); - let a8_low = i16x8_sub(i16x8_extend_low_u8x16(a8), i16x8_splat(32)); - let a8_high = i16x8_sub(i16x8_extend_high_u8x16(a8), i16x8_splat(32)); - v128_store( - aux8.add(l + 64) as *mut v128, - i8x16_narrow_i16x8(a8_low, a8_high), - ); - - // aux8[l + 96] = - // (((q4[l + 32] >> 4) | (((qh[l] >> 6) & 3) << 4)) as i32 - 32) as i8; - let a8 = v128_or( - u8x16_shr(v128_load(q4.add(l + 32) as *const v128), 4), - u8x16_shl( - v128_and( - u8x16_shr(v128_load(qh.add(l) as *const v128), 6), - u8x16_splat(3), - ), - 4, - ), - ); - let a8_low = i16x8_sub(i16x8_extend_low_u8x16(a8), i16x8_splat(32)); - let a8_high = i16x8_sub(i16x8_extend_high_u8x16(a8), i16x8_splat(32)); - v128_store( - aux8.add(l + 96) as *mut v128, - i8x16_narrow_i16x8(a8_low, a8_high), - ); - } - } - - for (j, &scale) in x.scales.iter().enumerate() { - let scale = f32x4_splat(scale as f32); - for offset in [0, 8] { - let aux16 = i16x8_mul( - i16x8_load_extend_i8x8(q8.as_ptr().add(16 * j + offset)), - i16x8_load_extend_i8x8(aux8.as_ptr().add(16 * j + offset)), - ); - aux32 = f32x4_add( - aux32, - f32x4_mul(f32x4_convert_i32x4(i32x4_extend_low_i16x8(aux16)), scale), - ); - aux32 = f32x4_add( - aux32, - f32x4_mul(f32x4_convert_i32x4(i32x4_extend_high_i16x8(aux16)), scale), - ); - } - } - - let d = f32x4_splat(x.d.to_f32() * y.d); - sums = f32x4_add(sums, f32x4_mul(aux32, d)); - } - let sums = f32x4_extract_lane::<0>(sums) - + f32x4_extract_lane::<1>(sums) - + f32x4_extract_lane::<2>(sums) - + f32x4_extract_lane::<3>(sums); - sums - } -} - -#[inline(always)] -pub(crate) fn vec_dot_q8k_q8k(n: usize, xs: &[BlockQ8K], ys: &[BlockQ8K]) -> f32 { - debug_assert!( - n.is_multiple_of(QK_K), - "vec_dot_q8k_q8k: {n} is not divisible by {QK_K}" - ); - unsafe { - let mut acc = f32x4_splat(0.0f32); - for (xs, ys) in xs.iter().zip(ys.iter()) { - let x_qs = xs.qs.as_ptr(); - let y_qs = ys.qs.as_ptr(); - let mut sumi = i32x4_splat(0); - for j in (0..QK_K).step_by(8) { - let xs = i16x8_load_extend_i8x8(x_qs.add(j)); - let ys = i16x8_load_extend_i8x8(y_qs.add(j)); - let sum_xy = i32x4_dot_i16x8(xs, ys); - sumi = i32x4_add(sumi, sum_xy) - } - let d = f32x4_splat(xs.d * ys.d); - acc = f32x4_add(acc, f32x4_mul(f32x4_convert_i32x4(sumi), d)) - } - let res = f32x4_extract_lane::<0>(acc) - + f32x4_extract_lane::<1>(acc) - + f32x4_extract_lane::<2>(acc) - + f32x4_extract_lane::<3>(acc); - res - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/quantized/utils.rs b/v2/dependency-patches/candle-core-0.9.2/src/quantized/utils.rs deleted file mode 100644 index 9dd9c0918a..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/quantized/utils.rs +++ /dev/null @@ -1,562 +0,0 @@ -pub(super) fn nearest_int(v: f32) -> i32 { - v.round() as i32 -} - -/// Validates that the input and output are the right size and returns an iterator which maps each -/// input region `xs` to its corresponding output block in `ys`. Each output region is guaranteed -/// to be `T::BLCK_SIZE` long. -pub(super) fn group_for_quantization<'a, 'b, T: super::k_quants::GgmlType>( - xs: &'b [f32], - ys: &'a mut [T], -) -> Vec<(&'a mut T, &'b [f32])> { - let block_size = T::BLCK_SIZE; - let dtype = T::DTYPE; - - let expected_blocks = xs.len() / block_size; - let actual_blocks = ys.len(); - - // Validate that the input is the right size - debug_assert_eq!( - expected_blocks, - actual_blocks, - "quantize {dtype:?}: expected {expected_blocks} blocks but only {actual_blocks} were provided!"); - - ys.iter_mut().zip(xs.chunks_exact(block_size)).collect() -} - -/// Validates that the input and output are the right size and returns an iterator which maps each -/// input block `xs` to its corresponding output region in `ys`. Each output region is guaranteed -/// to be `T::BLCK_SIZE` long. -pub(super) fn group_for_dequantization<'a, 'b, T: super::k_quants::GgmlType>( - xs: &'a [T], - ys: &'b mut [f32], -) -> Vec<(&'a T, &'b mut [f32])> { - let block_size = T::BLCK_SIZE; - let dtype = T::DTYPE; - - let actual_output_len = ys.len(); - let expected_output_len = xs.len() * block_size; - // Validate that the output is the right size - debug_assert_eq!( - expected_output_len, - actual_output_len, - "dequantize {dtype:?}: ys (len = {actual_output_len}) does not match the expected length of {expected_output_len}!" - ); - - // Zip the blocks and outputs together - xs.iter().zip(ys.chunks_exact_mut(block_size)).collect() -} - -pub(super) fn get_scale_min_k4(j: usize, q: &[u8]) -> (u8, u8) { - if j < 4 { - let d = q[j] & 63; - let m = q[j + 4] & 63; - (d, m) - } else { - let d = (q[j + 4] & 0xF) | ((q[j - 4] >> 6) << 4); - let m = (q[j + 4] >> 4) | ((q[j] >> 6) << 4); - (d, m) - } -} - -pub(super) unsafe fn make_qx_quants( - n: usize, - nmax: i32, - x: *const f32, - ls: *mut i8, - rmse_type: i32, - qw: *const f32, -) -> f32 { - let mut max = 0f32; - let mut amax = 0f32; - for i in 0..n { - let x = *x.add(i); - let ax = x.abs(); - if ax > amax { - amax = ax; - max = x; - } - } - if amax == 0. { - // all zero - for i in 0..n { - *ls.add(i) = 0; - } - return 0.; - } - let mut iscale = -(nmax as f32) / max; - if rmse_type == 0 { - for i in 0..n { - let x = *x.add(i); - let l = nearest_int(iscale * x); - *ls.add(i) = (nmax + l.clamp(-nmax, nmax - 1)) as i8; - } - return 1.0 / iscale; - } - let weight_type = rmse_type % 2; - let mut sumlx = 0f32; - let mut suml2 = 0f32; - for i in 0..n { - let x = *x.add(i); - let l = nearest_int(iscale * x); - let l = l.clamp(-nmax, nmax - 1); - *ls.add(i) = (l + nmax) as i8; - let w = if !qw.is_null() { - *qw.add(i) - } else if weight_type == 1 { - x * x - } else { - 1.0 - }; - let l = l as f32; - sumlx += w * x * l; - suml2 += w * l * l; - } - let mut scale = sumlx / suml2; - let mut best = scale * sumlx; - for _itry in 0..3 { - let iscale = 1.0 / scale; - let mut slx = 0f32; - let mut sl2 = 0f32; - let mut changed = false; - for i in 0..n { - let x = *x.add(i); - let l = nearest_int(iscale * x); - let l = l.clamp(-nmax, nmax - 1); - if l + nmax != *ls.add(i) as i32 { - changed = true; - } - let w = if !qw.is_null() { - *qw.add(i) - } else if weight_type == 1 { - x * x - } else { - 1.0 - }; - let l = l as f32; - slx += w * x * l; - sl2 += w * l * l; - } - if !changed || sl2 == 0.0 || slx * slx <= best * sl2 { - break; - } - for i in 0..n { - let x = *x.add(i); - let l = nearest_int(iscale * x); - *ls.add(i) = (nmax + l.clamp(-nmax, nmax - 1)) as i8; - } - sumlx = slx; - suml2 = sl2; - scale = sumlx / suml2; - best = scale * sumlx; - } - for _itry in 0..5 { - let mut n_changed = 0; - for i in 0..n { - let x = *x.add(i); - let w = if !qw.is_null() { - *qw.add(i) - } else if weight_type == 1 { - x * x - } else { - 1.0 - }; - let l = *ls.add(i) as i32 - nmax; - let mut slx = sumlx - w * x * l as f32; - if slx > 0. { - let mut sl2 = suml2 - w * l as f32 * l as f32; - let new_l = nearest_int(x * sl2 / slx); - let new_l = new_l.clamp(-nmax, nmax - 1); - if new_l != l { - slx += w * x * new_l as f32; - sl2 += w * new_l as f32 * new_l as f32; - if sl2 > 0. && slx * slx * suml2 > sumlx * sumlx * sl2 { - *ls.add(i) = (nmax + new_l) as i8; - sumlx = slx; - suml2 = sl2; - scale = sumlx / suml2; - best = scale * sumlx; - n_changed += 1; - } - } - } - } - if n_changed == 0 { - break; - } - } - if rmse_type < 3 { - return scale; - } - for is in -4..4 { - if is == 0 { - continue; - } - iscale = -(nmax as f32 + 0.1f32 * is as f32) / max; - let mut sumlx = 0.; - let mut suml2 = 0.; - for i in 0..n { - let x = *x.add(i); - let l = nearest_int(iscale * x); - let l = l.clamp(-nmax, nmax - 1); - let w = if !qw.is_null() { - *qw.add(i) - } else if weight_type == 1 { - x * x - } else { - 1.0 - }; - let l = l as f32; - sumlx += w * x * l; - suml2 += w * l * l; - } - if suml2 > 0. && sumlx * sumlx > best * suml2 { - for i in 0..n { - let x = *x.add(i); - let l = nearest_int(iscale * x); - *ls.add(i) = (nmax + l.clamp(-nmax, nmax - 1)) as i8; - } - scale = sumlx / suml2; - best = scale * sumlx; - } - } - scale -} - -// https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L224 -pub(super) fn make_qkx1_quants(nmax: i32, ntry: usize, x: &[f32]) -> (f32, f32) { - let n = x.len(); - let mut l = vec![0; n]; - // Get min/max - let min = *x - .iter() - .take(n) - .min_by(|a, b| a.total_cmp(b)) - .unwrap_or(&x[0]); - let max = *x.iter().max_by(|a, b| a.total_cmp(b)).unwrap_or(&x[0]); - - // If min == max, all values are the same => nothing to do here - if max == min { - return (0.0, 0.0); - } - - // Ensure min <= 0.0 - let mut min = min.min(0.); - - // Compute scale and inverse scale - let mut iscale = nmax as f32 / (max - min); - let mut scale = 1.0 / iscale; - - for _ in 0..ntry { - let mut sumlx = 0.0; - let mut suml2 = 0; - let mut did_change = false; - - for (i, value) in x.iter().enumerate().take(n) { - let li = nearest_int(iscale * (value - min)).clamp(0, nmax); - let clamped_li = li as u8; - if clamped_li != l[i] { - l[i] = clamped_li; - did_change = true; - } - sumlx += (value - min) * li as f32; - suml2 += li * li; - } - scale = sumlx / suml2 as f32; - - let sum: f32 = x - .iter() - .take(n) - .zip(l.iter().take(n)) - .map(|(xi, &li)| xi - scale * li as f32) - .sum(); - - min = sum / n as f32; - if min > 0.0 { - min = 0.0; - } - iscale = 1.0 / scale; - if !did_change { - break; - } - } - (scale, -min) -} - -// https://github.com/ggerganov/llama.cpp/blob/8183159cf3def112f6d1fe94815fce70e1bffa12/k_quants.c#L165 -pub(super) fn make_q3_quants(x: &[f32], nmax: i32, do_rmse: bool) -> f32 { - let n = x.len(); - let mut l = vec![0i8; n]; - - let mut max = 0.0; - let mut amax = 0.0; - for &xi in x.iter().take(n) { - let ax = xi.abs(); - if ax > amax { - amax = ax; - max = xi; - } - } - - if amax == 0.0 { - return 0.0; - } - - let iscale = -(nmax as f32) / max; - if do_rmse { - let mut sumlx = 0.0; - let mut suml2 = 0.0; - for i in 0..n { - let li = (iscale * x[i]).round() as i32; - let li = li.clamp(-nmax, nmax - 1); - l[i] = li as i8; - let w = x[i] * x[i]; - sumlx += w * x[i] * li as f32; - suml2 += w * (li * li) as f32; - } - for _ in 0..5 { - let mut n_changed = 0; - for i in 0..n { - let w = x[i] * x[i]; - let mut slx = sumlx - w * x[i] * l[i] as f32; - if slx > 0.0 { - let mut sl2 = suml2 - w * (l[i] as i32 * l[i] as i32) as f32; - let mut new_l = (x[i] * sl2 / slx).round() as i32; - new_l = new_l.clamp(-nmax, nmax - 1); - if new_l != l[i] as i32 { - slx += w * x[i] * new_l as f32; - sl2 += w * (new_l * new_l) as f32; - if sl2 > 0.0 && slx * slx * suml2 > sumlx * sumlx * sl2 { - l[i] = new_l as i8; - sumlx = slx; - suml2 = sl2; - n_changed += 1; - } - } - } - } - if n_changed == 0 { - break; - } - } - for li in l.iter_mut() { - *li += nmax as i8; - } - return sumlx / suml2; - } - for i in 0..n { - let li = (iscale * x[i]).round() as i32; - l[i] = (li.clamp(-nmax, nmax - 1) + nmax) as i8; - } - 1.0 / iscale -} - -// https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/ggml/src/ggml-quants.c#L744 -/// (scale, min) -pub(super) fn make_qkx3_quants( - nmax: i32, - x: &[f32], - weights: Option<&[f32]>, - rmin: f32, - rdelta: f32, - nstep: usize, - use_mad: bool, -) -> (f32, f32) { - let n = x.len(); - let mut l: [u8; 32] = [0; 32]; - let mut l_aux: [u8; 32] = [0; 32]; - - let mut min_val = x[0]; - let mut max_val = x[0]; - let mut sum_w = match weights { - Some(w) => w[0], - None => x[0] * x[0], - }; - let mut sum_x = sum_w * x[0]; - - for i in 1..n { - if x[i] < min_val { - min_val = x[i]; - } - if x[i] > max_val { - max_val = x[i]; - } - let w = match weights { - Some(w) => w[i], - None => x[i] * x[i], - }; - sum_w += w; - sum_x += w * x[i]; - } - - if min_val > 0.0 { - min_val = 0.0; - } - - if max_val <= min_val { - return (0.0, -min_val); - } - - let mut iscale = nmax as f32 / (max_val - min_val); - let mut scale = 1.0 / iscale; - let mut best_mad = 0.0; - - for i in 0..n { - let l_val = nearest_int(iscale * (x[i] - min_val)).clamp(0, nmax) as u8; - l[i] = l_val; - let diff = scale * (l_val as f32) + min_val - x[i]; - let diff = if use_mad { diff.abs() } else { diff * diff }; - let w = match weights { - Some(w) => w[i], - None => x[i] * x[i], - }; - best_mad += w * diff; - } - - if nstep < 1 { - return (scale, -min_val); - } - - for is in 0..=nstep { - iscale = (rmin + rdelta * is as f32 + nmax as f32) / (max_val - min_val); - let (mut sum_l, mut sum_l2, mut sum_xl) = (0.0, 0.0, 0.0); - - for i in 0..n { - let l_val = nearest_int(iscale * (x[i] - min_val)).clamp(0, nmax) as u8; - l_aux[i] = l_val; - let w = match weights { - Some(w) => w[i], - None => x[i] * x[i], - }; - sum_l += w * l_val as f32; - sum_l2 += w * (l_val as f32).powi(2); - sum_xl += w * l_val as f32 * x[i]; - } - - let d = sum_w * sum_l2 - sum_l * sum_l; - if d > 0.0 { - let mut this_scale = (sum_w * sum_xl - sum_x * sum_l) / d; - let mut this_min = (sum_l2 * sum_x - sum_l * sum_xl) / d; - - if this_min > 0.0 { - this_min = 0.0; - this_scale = sum_xl / sum_l2; - } - - let mut mad = 0.0; - for i in 0..n { - let diff = this_scale * (l_aux[i] as f32) + this_min - x[i]; - let diff = if use_mad { diff.abs() } else { diff * diff }; - let w = match weights { - Some(w) => w[i], - None => x[i] * x[i], - }; - mad += w * diff; - } - - if mad < best_mad { - l.copy_from_slice(&l_aux); - best_mad = mad; - scale = this_scale; - min_val = this_min; - } - } - } - - (scale, -min_val) -} - -// https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/ggml/src/ggml-quants.c#L827 -pub(super) fn make_qp_quants( - n: usize, - nmax: u8, - x: &[f32], - l: &mut [u8], - quant_weights: &[f32], -) -> f32 { - assert_eq!(x.len(), n); - assert_eq!(l.len(), n); - assert_eq!(quant_weights.len(), n); - - let max = x.iter().copied().fold(0.0, f32::max); - if max == 0.0 { - l.iter_mut().for_each(|li| *li = 0); - return 0.0; - } - - let mut iscale = nmax as f32 / max; - for (xi, li) in x.iter().zip(l.iter_mut()) { - *li = nearest_int(iscale * xi) as u8; - } - - let scale = 1.0 / iscale; - let mut best_mse = x - .iter() - .zip(l.iter()) - .zip(quant_weights.iter()) - .map(|((&xi, &li), &w)| { - let diff = xi - scale * li as f32; - w * diff * diff - }) - .sum::(); - - for is in -4..=4 { - if is == 0 { - continue; - } - let iscale_is = (0.1 * is as f32 + nmax as f32) / max; - let scale_is = 1.0 / iscale_is; - - let mse = x - .iter() - .zip(quant_weights.iter()) - .map(|(&xi, &w)| { - let mut li = nearest_int(iscale_is * xi) as u8; - li = li.min(nmax); - let diff = xi - scale_is * li as f32; - w * diff * diff - }) - .sum::(); - - if mse < best_mse { - best_mse = mse; - iscale = iscale_is; - } - } - - let mut sumlx = 0.0; - let mut suml2 = 0.0; - for ((xi, li), &w) in x.iter().zip(l.iter_mut()).zip(quant_weights.iter()) { - let mut li_new = (iscale * xi).round() as u8; - li_new = li_new.min(nmax); - *li = li_new; - sumlx += w * xi * li_new as f32; - suml2 += w * (li_new as f32).powi(2); - } - - for _ in 0..5 { - let mut n_changed = 0; - for ((xi, li), &w) in x.iter().zip(l.iter_mut()).zip(quant_weights.iter()) { - let mut slx = sumlx - w * xi * *li as f32; - let mut sl2 = suml2 - w * (*li as f32).powi(2); - if slx > 0.0 && sl2 > 0.0 { - let new_li = (nearest_int(xi * sl2 / slx) as u8).min(nmax); - if new_li != *li { - slx += w * xi * new_li as f32; - sl2 += w * (new_li as f32).powi(2); - if slx.powi(2) * suml2 > sumlx.powi(2) * sl2 { - *li = new_li; - sumlx = slx; - suml2 = sl2; - n_changed += 1; - } - } - } - } - if n_changed == 0 { - break; - } - } - - sumlx / suml2 -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/safetensors.rs b/v2/dependency-patches/candle-core-0.9.2/src/safetensors.rs deleted file mode 100644 index bec233b614..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/safetensors.rs +++ /dev/null @@ -1,644 +0,0 @@ -//! Module to load `safetensor` files into CPU/GPU memory. -//! -//! There are multiple ways to load tensors from safetensor files: -//! - `load` function for loading directly into memory and returning a HashMap of tensors -//! - `MmapedSafetensors` for memory mapping files and avoiding full allocation -//! - `SliceSafetensors` for working with in-memory buffers -//! - `BufferedSafetensors` for owning a buffer of data -//! -//! Tensors can also be serialized to safetensor format using the `save` function or -//! `Tensor::save_safetensors` method. -//! -use crate::op::BackpropOp; -use crate::storage::Storage; -use crate::tensor::from_storage; -use crate::{DType, Device, Error, Result, Tensor, WithDType}; -use safetensors::tensor as st; -use safetensors::tensor::SafeTensors; -use std::borrow::Cow; -use std::collections::HashMap; -use std::path::Path; - -impl From for st::Dtype { - fn from(value: DType) -> Self { - match value { - DType::U8 => st::Dtype::U8, - DType::U32 => st::Dtype::U32, - DType::I16 => st::Dtype::I16, - DType::I32 => st::Dtype::I32, - DType::I64 => st::Dtype::I64, - DType::BF16 => st::Dtype::BF16, - DType::F16 => st::Dtype::F16, - DType::F32 => st::Dtype::F32, - DType::F64 => st::Dtype::F64, - DType::F8E4M3 => st::Dtype::F8_E4M3, - DType::F6E2M3 => st::Dtype::F6_E2M3, - DType::F6E3M2 => st::Dtype::F6_E3M2, - DType::F4 => st::Dtype::F4, - DType::F8E8M0 => st::Dtype::F8_E8M0, - } - } -} - -impl TryFrom for DType { - type Error = Error; - fn try_from(value: st::Dtype) -> Result { - match value { - st::Dtype::U8 => Ok(DType::U8), - st::Dtype::U32 => Ok(DType::U32), - st::Dtype::I16 => Ok(DType::I16), - st::Dtype::I32 => Ok(DType::I32), - st::Dtype::I64 => Ok(DType::I64), - st::Dtype::BF16 => Ok(DType::BF16), - st::Dtype::F16 => Ok(DType::F16), - st::Dtype::F32 => Ok(DType::F32), - st::Dtype::F64 => Ok(DType::F64), - st::Dtype::F8_E4M3 => Ok(DType::F8E4M3), - st::Dtype::F6_E2M3 => Ok(DType::F6E2M3), - st::Dtype::F6_E3M2 => Ok(DType::F6E3M2), - st::Dtype::F4 => Ok(DType::F4), - st::Dtype::F8_E8M0 => Ok(DType::F8E8M0), - dtype => Err(Error::UnsupportedSafeTensorDtype(dtype)), - } - } -} - -impl st::View for Tensor { - fn dtype(&self) -> st::Dtype { - self.dtype().into() - } - fn shape(&self) -> &[usize] { - self.shape().dims() - } - - fn data(&self) -> Cow<'_, [u8]> { - // This copies data from GPU to CPU. - // TODO: Avoid the unwrap here. - Cow::Owned(convert_back(self).unwrap()) - } - - fn data_len(&self) -> usize { - let n: usize = self.shape().elem_count(); - let bytes_per_element = self.dtype().size_in_bytes(); - n * bytes_per_element - } -} - -impl st::View for &Tensor { - fn dtype(&self) -> st::Dtype { - (*self).dtype().into() - } - fn shape(&self) -> &[usize] { - self.dims() - } - - fn data(&self) -> Cow<'_, [u8]> { - // This copies data from GPU to CPU. - // TODO: Avoid the unwrap here. - Cow::Owned(convert_back(self).unwrap()) - } - - fn data_len(&self) -> usize { - let n: usize = self.dims().iter().product(); - let bytes_per_element = (*self).dtype().size_in_bytes(); - n * bytes_per_element - } -} - -impl Tensor { - pub fn save_safetensors>(&self, name: &str, filename: P) -> Result<()> { - let data = [(name, self.clone())]; - Ok(st::serialize_to_file(data, None, filename.as_ref())?) - } -} - -fn convert_slice(data: &[u8], shape: &[usize], device: &Device) -> Result { - let size_in_bytes = T::DTYPE.size_in_bytes(); - let elem_count = data.len() / size_in_bytes; - if (data.as_ptr() as usize).is_multiple_of(size_in_bytes) { - // SAFETY This is safe because we just checked that this - // was correctly aligned. - let data: &[T] = - unsafe { std::slice::from_raw_parts(data.as_ptr() as *const T, elem_count) }; - Tensor::from_slice(data, shape, device) - } else { - // XXX: We need to specify `T` here, otherwise the compiler will infer u8 because of the following cast - // Making this vector too small to fit a full f16/f32/f64 weights, resulting in out-of-bounds access - let mut c: Vec = Vec::with_capacity(elem_count); - // SAFETY: We just created c, so the allocated memory is necessarily - // contiguous and non overlapping with the view's data. - // We're downgrading the `c` pointer from T to u8, which removes alignment - // constraints. - unsafe { - std::ptr::copy_nonoverlapping(data.as_ptr(), c.as_mut_ptr() as *mut u8, data.len()); - c.set_len(elem_count) - } - Tensor::from_slice(&c, shape, device) - } -} - -fn convert_slice_with_cast Result>( - data: &[u8], - shape: &[usize], - device: &Device, - conv: F, -) -> Result { - let size_in_bytes = std::mem::size_of::(); - let elem_count = data.len() / size_in_bytes; - if (data.as_ptr() as usize).is_multiple_of(size_in_bytes) { - // SAFETY This is safe because we just checked that this - // was correctly aligned. - let data: &[T] = - unsafe { std::slice::from_raw_parts(data.as_ptr() as *const T, elem_count) }; - let data = data.iter().map(|t| conv(*t)).collect::>>()?; - Tensor::from_vec(data, shape, device) - } else { - // XXX: We need to specify `T` here, otherwise the compiler will infer u8 because of the following cast - // Making this vector too small to fit a full f16/f32/f64 weights, resulting in out-of-bounds access - let mut c: Vec = Vec::with_capacity(elem_count); - // SAFETY: We just created c, so the allocated memory is necessarily - // contiguous and non overlapping with the view's data. - // We're downgrading the `c` pointer from T to u8, which removes alignment - // constraints. - unsafe { - std::ptr::copy_nonoverlapping(data.as_ptr(), c.as_mut_ptr() as *mut u8, data.len()); - c.set_len(elem_count) - } - let c = c.into_iter().map(conv).collect::>>()?; - Tensor::from_vec(c, shape, device) - } -} - -fn convert_with_cast_ Result>( - view: &st::TensorView<'_>, - device: &Device, - conv: F, -) -> Result { - convert_slice_with_cast::(view.data(), view.shape(), device, conv) -} - -fn convert_(view: &st::TensorView<'_>, device: &Device) -> Result { - convert_slice::(view.data(), view.shape(), device) -} - -fn convert_back_(mut vs: Vec) -> Vec { - let size_in_bytes = T::DTYPE.size_in_bytes(); - let length = vs.len() * size_in_bytes; - let capacity = vs.capacity() * size_in_bytes; - let ptr = vs.as_mut_ptr() as *mut u8; - // Don't run the destructor for Vec - std::mem::forget(vs); - // SAFETY: - // - // Every T is larger than u8, so there is no issue regarding alignment. - // This re-interpret the Vec as a Vec. - unsafe { Vec::from_raw_parts(ptr, length, capacity) } -} - -pub trait Load { - fn load(&self, device: &Device) -> Result; -} - -impl Load for st::TensorView<'_> { - fn load(&self, device: &Device) -> Result { - convert(self, device) - } -} - -impl Tensor { - pub fn from_raw_buffer( - data: &[u8], - dtype: DType, - shape: &[usize], - device: &Device, - ) -> Result { - match dtype { - DType::U8 => convert_slice::(data, shape, device), - DType::U32 => convert_slice::(data, shape, device), - DType::I16 => convert_slice::(data, shape, device), - DType::I32 => convert_slice::(data, shape, device), - DType::I64 => convert_slice::(data, shape, device), - DType::BF16 => convert_slice::(data, shape, device), - DType::F16 => convert_slice::(data, shape, device), - DType::F32 => convert_slice::(data, shape, device), - DType::F64 => convert_slice::(data, shape, device), - DType::F8E4M3 => convert_slice::(data, shape, device), - DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { - // For dummy types, create storage with raw bytes - let storage = match device { - Device::Cpu => { - let cpu_storage = match dtype { - DType::F6E2M3 => crate::cpu_backend::CpuStorage::F6E2M3(data.to_vec()), - DType::F6E3M2 => crate::cpu_backend::CpuStorage::F6E3M2(data.to_vec()), - DType::F4 => crate::cpu_backend::CpuStorage::F4(data.to_vec()), - DType::F8E8M0 => crate::cpu_backend::CpuStorage::F8E8M0(data.to_vec()), - _ => unreachable!(), - }; - Storage::Cpu(cpu_storage) - } - #[cfg(feature = "cuda")] - Device::Cuda(device) => { - let mut slice = unsafe { device.alloc::(data.len())? }; - device.memcpy_htod(data, &mut slice)?; - - let slice = match dtype { - DType::F6E2M3 => crate::cuda_backend::CudaStorageSlice::F6E2M3(slice), - DType::F6E3M2 => crate::cuda_backend::CudaStorageSlice::F6E3M2(slice), - DType::F4 => crate::cuda_backend::CudaStorageSlice::F4(slice), - DType::F8E8M0 => crate::cuda_backend::CudaStorageSlice::F8E8M0(slice), - _ => unreachable!(), - }; - let storage = crate::cuda_backend::CudaStorage { - slice, - device: device.clone(), - }; - Storage::Cuda(storage) - } - #[cfg(not(feature = "cuda"))] - Device::Cuda(_) => { - return Err(Error::Msg("CUDA support not compiled".to_string())); - } - #[cfg(feature = "metal")] - Device::Metal(device) => { - let buffer = device.new_buffer_with_data(data)?; - - let storage = crate::metal_backend::MetalStorage::new( - buffer, - device.clone(), - data.len(), - dtype, - ); - Storage::Metal(storage) - } - #[cfg(not(feature = "metal"))] - Device::Metal(_) => { - return Err(Error::Msg("Metal support not compiled".to_string())); - } - }; - - let op = BackpropOp::none(); - Ok(from_storage(storage, shape, op, false)) - } - } - } -} - -fn convert(view: &st::TensorView<'_>, device: &Device) -> Result { - match view.dtype() { - st::Dtype::U8 => convert_::(view, device), - st::Dtype::U16 => { - let conv = |x| Ok(u32::from(x)); - convert_with_cast_::(view, device, conv) - } - st::Dtype::U32 => convert_::(view, device), - st::Dtype::I16 => convert_::(view, device), - st::Dtype::I32 => convert_::(view, device), - st::Dtype::I64 => convert_::(view, device), - st::Dtype::BF16 => convert_::(view, device), - st::Dtype::F16 => convert_::(view, device), - st::Dtype::F32 => convert_::(view, device), - st::Dtype::F64 => convert_::(view, device), - st::Dtype::F8_E4M3 => convert_::(view, device), - st::Dtype::F6_E2M3 | st::Dtype::F6_E3M2 | st::Dtype::F4 | st::Dtype::F8_E8M0 => { - // For dummy types, we need to handle loading by creating a dummy tensor - // Since these types don't have actual data representation, we'll create - // a tensor that indicates it's a dummy type - convert_dummy(view, device) - } - dtype => Err(Error::UnsupportedSafeTensorDtype(dtype)), - } -} - -fn convert_dummy(view: &st::TensorView<'_>, device: &Device) -> Result { - // For dummy types, we'll create the appropriate storage variant that preserves - // both the raw data and the correct dtype - let (dtype, _dtype_name) = match view.dtype() { - st::Dtype::F6_E2M3 => (DType::F6E2M3, "F6_E2M3 (MX6)"), - st::Dtype::F6_E3M2 => (DType::F6E3M2, "F6_E3M2 (MX6)"), - st::Dtype::F4 => (DType::F4, "F4 (MX4)"), - st::Dtype::F8_E8M0 => (DType::F8E8M0, "F8_E8M0"), - _ => unreachable!("convert_dummy called with non-dummy dtype"), - }; - - // Load the raw bytes - let data = view.data(); - let shape = view.shape(); - - // Create storage with the appropriate dummy type variant - let storage = match device { - Device::Cpu => { - let cpu_storage = match dtype { - DType::F6E2M3 => crate::cpu_backend::CpuStorage::F6E2M3(data.to_vec()), - DType::F6E3M2 => crate::cpu_backend::CpuStorage::F6E3M2(data.to_vec()), - DType::F4 => crate::cpu_backend::CpuStorage::F4(data.to_vec()), - DType::F8E8M0 => crate::cpu_backend::CpuStorage::F8E8M0(data.to_vec()), - _ => unreachable!(), - }; - Storage::Cpu(cpu_storage) - } - #[cfg(feature = "cuda")] - Device::Cuda(device) => { - let mut slice = unsafe { device.alloc::(data.len())? }; - device.memcpy_htod(data, &mut slice)?; - - let slice = match dtype { - DType::F6E2M3 => crate::cuda_backend::CudaStorageSlice::F6E2M3(slice), - DType::F6E3M2 => crate::cuda_backend::CudaStorageSlice::F6E3M2(slice), - DType::F4 => crate::cuda_backend::CudaStorageSlice::F4(slice), - DType::F8E8M0 => crate::cuda_backend::CudaStorageSlice::F8E8M0(slice), - _ => unreachable!(), - }; - let storage = crate::cuda_backend::CudaStorage { - slice, - device: device.clone(), - }; - Storage::Cuda(storage) - } - #[cfg(not(feature = "cuda"))] - Device::Cuda(_) => { - return Err(Error::Msg("CUDA support not compiled".to_string())); - } - #[cfg(feature = "metal")] - Device::Metal(device) => { - let buffer = device.new_buffer_with_data(data)?; - - let storage = - crate::metal_backend::MetalStorage::new(buffer, device.clone(), data.len(), dtype); - Storage::Metal(storage) - } - #[cfg(not(feature = "metal"))] - Device::Metal(_) => { - return Err(Error::Msg("Metal support not compiled".to_string())); - } - }; - - // Create tensor with correct dtype - let op = BackpropOp::none(); - Ok(from_storage(storage, shape, op, false)) -} - -fn convert_back(tensor: &Tensor) -> Result> { - // TODO: This makes an unnecessary copy when the tensor is on the cpu. - let tensor = tensor.flatten_all()?; - match tensor.dtype() { - DType::U8 => Ok(convert_back_::(tensor.to_vec1()?)), - DType::U32 => Ok(convert_back_::(tensor.to_vec1()?)), - DType::I16 => Ok(convert_back_::(tensor.to_vec1()?)), - DType::I32 => Ok(convert_back_::(tensor.to_vec1()?)), - DType::I64 => Ok(convert_back_::(tensor.to_vec1()?)), - DType::F16 => Ok(convert_back_::(tensor.to_vec1()?)), - DType::BF16 => Ok(convert_back_::(tensor.to_vec1()?)), - DType::F32 => Ok(convert_back_::(tensor.to_vec1()?)), - DType::F64 => Ok(convert_back_::(tensor.to_vec1()?)), - DType::F8E4M3 => Ok(convert_back_::(tensor.to_vec1()?)), - DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { - Err(Error::Msg("Internal error: dtype mismatch in storage".to_string()).bt()) - } - } -} - -pub fn load>(filename: P, device: &Device) -> Result> { - let data = std::fs::read(filename.as_ref())?; - load_buffer(&data[..], device) -} - -pub fn load_buffer(data: &[u8], device: &Device) -> Result> { - let st = safetensors::SafeTensors::deserialize(data)?; - st.tensors() - .into_iter() - .map(|(name, view)| Ok((name, view.load(device)?))) - .collect() -} - -pub fn save + Ord + std::fmt::Display, P: AsRef>( - tensors: &HashMap, - filename: P, -) -> Result<()> { - Ok(st::serialize_to_file(tensors, None, filename.as_ref())?) -} - -#[derive(yoke::Yokeable)] -struct SafeTensors_<'a>(SafeTensors<'a>); - -pub struct MmapedSafetensors { - safetensors: Vec, memmap2::Mmap>>, - routing: Option>, -} - -impl MmapedSafetensors { - /// Creates a wrapper around a memory mapped file and deserialize the safetensors header. - /// - /// # Safety - /// - /// The unsafe is inherited from [`memmap2::MmapOptions`]. - pub unsafe fn new>(p: P) -> Result { - let p = p.as_ref(); - let file = std::fs::File::open(p).map_err(|e| Error::from(e).with_path(p))?; - let file = memmap2::MmapOptions::new() - .map(&file) - .map_err(|e| Error::from(e).with_path(p))?; - let safetensors = yoke::Yoke::, memmap2::Mmap>::try_attach_to_cart( - file, - |data: &[u8]| { - let st = safetensors::SafeTensors::deserialize(data) - .map_err(|e| Error::from(e).with_path(p))?; - Ok::<_, Error>(SafeTensors_(st)) - }, - )?; - Ok(Self { - safetensors: vec![safetensors], - routing: None, - }) - } - - /// Creates a wrapper around multiple memory mapped file and deserialize the safetensors headers. - /// - /// If a tensor name appears in multiple files, the last entry is returned. - /// - /// # Safety - /// - /// The unsafe is inherited from [`memmap2::MmapOptions`]. - pub unsafe fn multi>(paths: &[P]) -> Result { - let mut routing = HashMap::new(); - let mut safetensors = vec![]; - for (index, p) in paths.iter().enumerate() { - let p = p.as_ref(); - let file = std::fs::File::open(p).map_err(|e| Error::from(e).with_path(p))?; - let file = memmap2::MmapOptions::new() - .map(&file) - .map_err(|e| Error::from(e).with_path(p))?; - let data = yoke::Yoke::, memmap2::Mmap>::try_attach_to_cart( - file, - |data: &[u8]| { - let st = safetensors::SafeTensors::deserialize(data) - .map_err(|e| Error::from(e).with_path(p))?; - Ok::<_, Error>(SafeTensors_(st)) - }, - )?; - for k in data.get().0.names() { - routing.insert(k.to_string(), index); - } - safetensors.push(data) - } - Ok(Self { - safetensors, - routing: Some(routing), - }) - } - - pub fn load(&self, name: &str, dev: &Device) -> Result { - self.get(name)?.load(dev) - } - - pub fn tensors(&self) -> Vec<(String, st::TensorView<'_>)> { - let mut tensors = vec![]; - for safetensors in self.safetensors.iter() { - tensors.push(safetensors.get().0.tensors()) - } - tensors.into_iter().flatten().collect() - } - - pub fn get(&self, name: &str) -> Result> { - let index = match &self.routing { - None => 0, - Some(routing) => { - let index = routing.get(name).ok_or_else(|| { - Error::CannotFindTensor { - path: name.to_string(), - } - .bt() - })?; - *index - } - }; - Ok(self.safetensors[index].get().0.tensor(name)?) - } -} - -pub struct SliceSafetensors<'a> { - safetensors: SafeTensors<'a>, -} - -impl<'a> SliceSafetensors<'a> { - /// Creates a wrapper around a binary buffer and deserialize the safetensors header. - pub fn new(buffer: &'a [u8]) -> Result { - let safetensors = safetensors::SafeTensors::deserialize(buffer)?; - Ok(Self { safetensors }) - } - - pub fn load(&self, name: &str, dev: &Device) -> Result { - self.safetensors.tensor(name)?.load(dev) - } - - pub fn tensors(&self) -> Vec<(String, st::TensorView<'_>)> { - self.safetensors.tensors() - } - - pub fn get(&self, name: &str) -> Result> { - Ok(self.safetensors.tensor(name)?) - } -} - -pub struct BufferedSafetensors { - safetensors: yoke::Yoke, Vec>, -} - -impl BufferedSafetensors { - /// Creates a wrapper around a binary buffer and deserialize the safetensors header. - pub fn new(buffer: Vec) -> Result { - let safetensors = yoke::Yoke::, Vec>::try_attach_to_cart( - buffer, - |data: &[u8]| { - let st = safetensors::SafeTensors::deserialize(data)?; - Ok::<_, Error>(SafeTensors_(st)) - }, - )?; - Ok(Self { safetensors }) - } - - pub fn load(&self, name: &str, dev: &Device) -> Result { - self.get(name)?.load(dev) - } - - pub fn tensors(&self) -> Vec<(String, st::TensorView<'_>)> { - self.safetensors.get().0.tensors() - } - - pub fn get(&self, name: &str) -> Result> { - Ok(self.safetensors.get().0.tensor(name)?) - } -} - -pub struct MmapedFile { - path: std::path::PathBuf, - inner: memmap2::Mmap, -} - -impl MmapedFile { - /// Creates a wrapper around a memory mapped file from which you can retrieve - /// tensors using [`MmapedFile::deserialize`] - /// - /// # Safety - /// - /// The unsafe is inherited from [`memmap2::MmapOptions`]. - pub unsafe fn new>(p: P) -> Result { - let p = p.as_ref(); - let file = std::fs::File::open(p).map_err(|e| Error::from(e).with_path(p))?; - let inner = memmap2::MmapOptions::new() - .map(&file) - .map_err(|e| Error::from(e).with_path(p))?; - Ok(Self { - inner, - path: p.to_path_buf(), - }) - } - - pub fn deserialize(&self) -> Result> { - let st = safetensors::SafeTensors::deserialize(&self.inner) - .map_err(|e| Error::from(e).with_path(&self.path))?; - Ok(st) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::collections::HashMap; - - #[test] - fn save_single_tensor() { - let t = Tensor::zeros((2, 2), DType::F32, &Device::Cpu).unwrap(); - t.save_safetensors("t", "t.safetensors").unwrap(); - let bytes = std::fs::read("t.safetensors").unwrap(); - assert_eq!(bytes, b"@\0\0\0\0\0\0\0{\"t\":{\"dtype\":\"F32\",\"shape\":[2,2],\"data_offsets\":[0,16]}} \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"); - std::fs::remove_file("t.safetensors").unwrap(); - } - - #[test] - fn save_load_multiple_tensors() { - let t = Tensor::zeros((2, 2), DType::F32, &Device::Cpu).unwrap(); - let u = Tensor::zeros((1, 2), DType::F32, &Device::Cpu).unwrap(); - let map: HashMap<_, _> = [("t", t), ("u", u)].into_iter().collect(); - save(&map, "multi.safetensors").unwrap(); - - let weights = load("multi.safetensors", &Device::Cpu).unwrap(); - assert_eq!(weights.get("t").unwrap().dims(), &[2, 2]); - assert_eq!(weights.get("u").unwrap().dims(), &[1, 2]); - let bytes = std::fs::read("multi.safetensors").unwrap(); - assert_eq!(bytes, b"x\0\0\0\0\0\0\0{\"t\":{\"dtype\":\"F32\",\"shape\":[2,2],\"data_offsets\":[0,16]},\"u\":{\"dtype\":\"F32\",\"shape\":[1,2],\"data_offsets\":[16,24]}} \0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"); - std::fs::remove_file("multi.safetensors").unwrap(); - } - - #[test] - fn load_u8() { - let bytes = b"8\0\0\0\0\0\0\0{\"x\":{\"dtype\":\"U8\",\"shape\":[2],\"data_offsets\":[0,2]}} \x01\x03"; - std::fs::write("test_u8.safetensors", bytes).unwrap(); - let weights = load("test_u8.safetensors", &Device::Cpu).unwrap(); - let tensor = weights.get("x").unwrap(); - assert_eq!(tensor.dims(), &[2]); - assert_eq!(tensor.dtype(), DType::U8); - let data: Vec = tensor.to_vec1().unwrap(); - assert_eq!(data, vec![1, 3]); - std::fs::remove_file("test_u8.safetensors").unwrap(); - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/scalar.rs b/v2/dependency-patches/candle-core-0.9.2/src/scalar.rs deleted file mode 100644 index 5c512c03b9..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/scalar.rs +++ /dev/null @@ -1,115 +0,0 @@ -//! TensorScalar Enum and Trait -//! -use crate::{DType, Result, Tensor, WithDType}; -use float8::F8E4M3 as f8e4m3; -use half::{bf16, f16}; - -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum Scalar { - U8(u8), - U32(u32), - I16(i16), - I32(i32), - I64(i64), - BF16(bf16), - F16(f16), - F32(f32), - F64(f64), - F8E4M3(f8e4m3), -} - -impl From for Scalar { - fn from(value: T) -> Self { - value.to_scalar() - } -} - -impl Scalar { - pub fn zero(dtype: DType) -> Self { - match dtype { - DType::U8 => Scalar::U8(0), - DType::U32 => Scalar::U32(0), - DType::I16 => Scalar::I16(0), - DType::I32 => Scalar::I32(0), - DType::I64 => Scalar::I64(0), - DType::BF16 => Scalar::BF16(bf16::ZERO), - DType::F16 => Scalar::F16(f16::ZERO), - DType::F32 => Scalar::F32(0.0), - DType::F64 => Scalar::F64(0.0), - DType::F8E4M3 => Scalar::F8E4M3(f8e4m3::ZERO), - DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { - panic!("Cannot create zero scalar for dummy type {dtype:?}") - } - } - } - - pub fn one(dtype: DType) -> Self { - match dtype { - DType::U8 => Scalar::U8(1), - DType::U32 => Scalar::U32(1), - DType::I16 => Scalar::I16(1), - DType::I32 => Scalar::I32(1), - DType::I64 => Scalar::I64(1), - DType::BF16 => Scalar::BF16(bf16::ONE), - DType::F16 => Scalar::F16(f16::ONE), - DType::F32 => Scalar::F32(1.0), - DType::F64 => Scalar::F64(1.0), - DType::F8E4M3 => Scalar::F8E4M3(f8e4m3::ONE), - DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { - panic!("Cannot create one scalar for dummy type {dtype:?}") - } - } - } - - pub fn dtype(&self) -> DType { - match self { - Scalar::U8(_) => DType::U8, - Scalar::U32(_) => DType::U32, - Scalar::I16(_) => DType::I16, - Scalar::I32(_) => DType::I32, - Scalar::I64(_) => DType::I64, - Scalar::BF16(_) => DType::BF16, - Scalar::F16(_) => DType::F16, - Scalar::F32(_) => DType::F32, - Scalar::F64(_) => DType::F64, - Scalar::F8E4M3(_) => DType::F8E4M3, - } - } - - pub fn to_f64(&self) -> f64 { - match self { - Scalar::U8(v) => *v as f64, - Scalar::U32(v) => *v as f64, - Scalar::I16(v) => *v as f64, - Scalar::I32(v) => *v as f64, - Scalar::I64(v) => *v as f64, - Scalar::BF16(v) => v.to_f64(), - Scalar::F16(v) => v.to_f64(), - Scalar::F32(v) => *v as f64, - Scalar::F64(v) => *v, - Scalar::F8E4M3(v) => v.to_f64(), - } - } -} - -pub enum TensorScalar { - Tensor(Tensor), - Scalar(Tensor), -} - -pub trait TensorOrScalar { - fn to_tensor_scalar(self) -> Result; -} - -impl TensorOrScalar for &Tensor { - fn to_tensor_scalar(self) -> Result { - Ok(TensorScalar::Tensor(self.clone())) - } -} - -impl TensorOrScalar for T { - fn to_tensor_scalar(self) -> Result { - let scalar = Tensor::new(self, &crate::Device::Cpu)?; - Ok(TensorScalar::Scalar(scalar)) - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/shape.rs b/v2/dependency-patches/candle-core-0.9.2/src/shape.rs deleted file mode 100644 index b9e731266f..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/shape.rs +++ /dev/null @@ -1,634 +0,0 @@ -//! The shape of a tensor is a tuple with the size of each of its dimensions. -#![allow(clippy::redundant_closure_call)] -use crate::{Error, Result}; - -#[derive(Clone, PartialEq, Eq)] -pub struct Shape(Vec); - -pub const SCALAR: Shape = Shape(vec![]); - -impl std::fmt::Debug for Shape { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:?}", &self.dims()) - } -} - -impl From<&[usize; C]> for Shape { - fn from(dims: &[usize; C]) -> Self { - Self(dims.to_vec()) - } -} - -impl From<&[usize]> for Shape { - fn from(dims: &[usize]) -> Self { - Self(dims.to_vec()) - } -} - -impl From<&Shape> for Shape { - fn from(shape: &Shape) -> Self { - Self(shape.0.to_vec()) - } -} - -impl From<()> for Shape { - fn from(_: ()) -> Self { - Self(vec![]) - } -} - -impl From for Shape { - fn from(d1: usize) -> Self { - Self(vec![d1]) - } -} - -macro_rules! impl_from_tuple { - ($tuple:ty, $($index:tt),+) => { - impl From<$tuple> for Shape { - fn from(d: $tuple) -> Self { - Self(vec![$(d.$index,)+]) - } - } - } -} - -impl_from_tuple!((usize,), 0); -impl_from_tuple!((usize, usize), 0, 1); -impl_from_tuple!((usize, usize, usize), 0, 1, 2); -impl_from_tuple!((usize, usize, usize, usize), 0, 1, 2, 3); -impl_from_tuple!((usize, usize, usize, usize, usize), 0, 1, 2, 3, 4); -impl_from_tuple!((usize, usize, usize, usize, usize, usize), 0, 1, 2, 3, 4, 5); - -impl From> for Shape { - fn from(dims: Vec) -> Self { - Self(dims) - } -} - -macro_rules! extract_dims { - ($fn_name:ident, $cnt:tt, $dims:expr, $out_type:ty) => { - pub fn $fn_name(dims: &[usize]) -> Result<$out_type> { - if dims.len() != $cnt { - Err(Error::UnexpectedNumberOfDims { - expected: $cnt, - got: dims.len(), - shape: Shape::from(dims), - } - .bt()) - } else { - Ok($dims(dims)) - } - } - - impl Shape { - pub fn $fn_name(&self) -> Result<$out_type> { - $fn_name(self.0.as_slice()) - } - } - - impl crate::Tensor { - pub fn $fn_name(&self) -> Result<$out_type> { - self.shape().$fn_name() - } - } - - impl std::convert::TryInto<$out_type> for Shape { - type Error = crate::Error; - fn try_into(self) -> std::result::Result<$out_type, Self::Error> { - self.$fn_name() - } - } - }; -} - -impl Shape { - pub fn from_dims(dims: &[usize]) -> Self { - Self(dims.to_vec()) - } - - /// The rank is the number of dimensions, 0 for a scalar value, 1 for a vector, etc. - pub fn rank(&self) -> usize { - self.0.len() - } - - pub fn into_dims(self) -> Vec { - self.0 - } - - /// The dimensions as a slice of `usize`. - pub fn dims(&self) -> &[usize] { - &self.0 - } - - /// The dimension size for a specified dimension index. - pub fn dim(&self, dim: D) -> Result { - let dim = dim.to_index(self, "dim")?; - Ok(self.dims()[dim]) - } - - /// The total number of elements, this is the product of all dimension sizes. - pub fn elem_count(&self) -> usize { - self.0.iter().product() - } - - /// The strides given in number of elements for a contiguous n-dimensional - /// arrays using this shape. - pub(crate) fn stride_contiguous(&self) -> Vec { - let mut stride: Vec<_> = self - .0 - .iter() - .rev() - .scan(1, |prod, u| { - let prod_pre_mult = *prod; - *prod *= u; - Some(prod_pre_mult) - }) - .collect(); - stride.reverse(); - stride - } - - /// Returns true if the strides are C contiguous (aka row major). - pub fn is_contiguous(&self, stride: &[usize]) -> bool { - if self.0.len() != stride.len() { - return false; - } - let mut acc = 1; - for (&stride, &dim) in stride.iter().zip(self.0.iter()).rev() { - if dim > 1 && stride != acc { - return false; - } - acc *= dim; - } - true - } - - /// Returns true if the strides are Fortran contiguous (aka column major). - pub fn is_fortran_contiguous(&self, stride: &[usize]) -> bool { - if self.0.len() != stride.len() { - return false; - } - let mut acc = 1; - for (&stride, &dim) in stride.iter().zip(self.0.iter()) { - if dim > 1 && stride != acc { - return false; - } - acc *= dim; - } - true - } - - /// Modifies the shape by adding a list of additional dimensions at the end of the existing - /// dimensions. - pub fn extend(mut self, additional_dims: &[usize]) -> Self { - self.0.extend(additional_dims); - self - } - - /// Check whether the two shapes are compatible for broadcast, and if it is the case return the - /// broadcasted shape. This is to be used for binary pointwise ops. - pub fn broadcast_shape_binary_op(&self, rhs: &Self, op: &'static str) -> Result { - let lhs = self; - let lhs_dims = lhs.dims(); - let rhs_dims = rhs.dims(); - let lhs_ndims = lhs_dims.len(); - let rhs_ndims = rhs_dims.len(); - let bcast_ndims = usize::max(lhs_ndims, rhs_ndims); - let mut bcast_dims = vec![0; bcast_ndims]; - for (idx, bcast_value) in bcast_dims.iter_mut().enumerate() { - let rev_idx = bcast_ndims - idx; - let l_value = if lhs_ndims < rev_idx { - 1 - } else { - lhs_dims[lhs_ndims - rev_idx] - }; - let r_value = if rhs_ndims < rev_idx { - 1 - } else { - rhs_dims[rhs_ndims - rev_idx] - }; - *bcast_value = if l_value == r_value { - l_value - } else if l_value == 1 { - r_value - } else if r_value == 1 { - l_value - } else { - Err(Error::ShapeMismatchBinaryOp { - lhs: lhs.clone(), - rhs: rhs.clone(), - op, - } - .bt())? - } - } - Ok(Shape::from(bcast_dims)) - } - - pub(crate) fn broadcast_shape_matmul(&self, rhs: &Self) -> Result<(Shape, Shape)> { - let lhs = self; - let lhs_dims = lhs.dims(); - let rhs_dims = rhs.dims(); - if lhs_dims.len() < 2 || rhs_dims.len() < 2 { - crate::bail!("only 2d matrixes are supported {lhs:?} {rhs:?}") - } - let (m, lhs_k) = (lhs_dims[lhs_dims.len() - 2], lhs_dims[lhs_dims.len() - 1]); - let (rhs_k, n) = (rhs_dims[rhs_dims.len() - 2], rhs_dims[rhs_dims.len() - 1]); - if lhs_k != rhs_k { - crate::bail!("different inner dimensions in broadcast matmul {lhs:?} {rhs:?}") - } - - let lhs_b = Self::from(&lhs_dims[..lhs_dims.len() - 2]); - let rhs_b = Self::from(&rhs_dims[..rhs_dims.len() - 2]); - let bcast = lhs_b.broadcast_shape_binary_op(&rhs_b, "broadcast_matmul")?; - let bcast_dims = bcast.dims(); - - let bcast_lhs = [bcast_dims, &[m, lhs_k]].concat(); - let bcast_rhs = [bcast_dims, &[rhs_k, n]].concat(); - Ok((Shape::from(bcast_lhs), Shape::from(bcast_rhs))) - } -} - -pub trait Dim { - fn to_index(&self, shape: &Shape, op: &'static str) -> Result; - fn to_index_plus_one(&self, shape: &Shape, op: &'static str) -> Result; -} - -impl Dim for usize { - fn to_index(&self, shape: &Shape, op: &'static str) -> Result { - let dim = *self; - if dim >= shape.dims().len() { - Err(Error::DimOutOfRange { - shape: shape.clone(), - dim: dim as i32, - op, - } - .bt())? - } else { - Ok(dim) - } - } - - fn to_index_plus_one(&self, shape: &Shape, op: &'static str) -> Result { - let dim = *self; - if dim > shape.dims().len() { - Err(Error::DimOutOfRange { - shape: shape.clone(), - dim: dim as i32, - op, - } - .bt())? - } else { - Ok(dim) - } - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub enum D { - Minus1, - Minus2, - Minus(usize), -} - -impl D { - fn out_of_range(&self, shape: &Shape, op: &'static str) -> Error { - let dim = match self { - Self::Minus1 => -1, - Self::Minus2 => -2, - Self::Minus(u) => -(*u as i32), - }; - Error::DimOutOfRange { - shape: shape.clone(), - dim, - op, - } - .bt() - } -} - -impl Dim for D { - fn to_index(&self, shape: &Shape, op: &'static str) -> Result { - let rank = shape.rank(); - match self { - Self::Minus1 if rank >= 1 => Ok(rank - 1), - Self::Minus2 if rank >= 2 => Ok(rank - 2), - Self::Minus(u) if *u > 0 && rank >= *u => Ok(rank - *u), - _ => Err(self.out_of_range(shape, op)), - } - } - - fn to_index_plus_one(&self, shape: &Shape, op: &'static str) -> Result { - let rank = shape.rank(); - match self { - Self::Minus1 => Ok(rank), - Self::Minus2 if rank >= 1 => Ok(rank - 1), - Self::Minus(u) if *u > 0 && rank + 1 >= *u => Ok(rank + 1 - *u), - _ => Err(self.out_of_range(shape, op)), - } - } -} - -pub trait Dims: Sized { - fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result>; - - fn to_indexes(self, shape: &Shape, op: &'static str) -> Result> { - let dims = self.to_indexes_internal(shape, op)?; - for (i, &dim) in dims.iter().enumerate() { - if dims[..i].contains(&dim) { - Err(Error::DuplicateDimIndex { - shape: shape.clone(), - dims: dims.clone(), - op, - } - .bt())? - } - if dim >= shape.rank() { - Err(Error::DimOutOfRange { - shape: shape.clone(), - dim: dim as i32, - op, - } - .bt())? - } - } - Ok(dims) - } -} - -impl Dims for Vec { - fn to_indexes_internal(self, _: &Shape, _: &'static str) -> Result> { - Ok(self) - } -} - -impl Dims for [usize; N] { - fn to_indexes_internal(self, _: &Shape, _: &'static str) -> Result> { - Ok(self.to_vec()) - } -} - -impl Dims for &[usize] { - fn to_indexes_internal(self, _: &Shape, _: &'static str) -> Result> { - Ok(self.to_vec()) - } -} - -impl Dims for () { - fn to_indexes_internal(self, _: &Shape, _: &'static str) -> Result> { - Ok(vec![]) - } -} - -impl Dims for D { - fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { - let dim = self.to_index(shape, op)?; - Ok(vec![dim]) - } -} - -impl Dims for (D,) { - fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { - let dim = self.0.to_index(shape, op)?; - Ok(vec![dim]) - } -} - -impl Dims for (D1, D2) { - fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { - let d0 = self.0.to_index(shape, op)?; - let d1 = self.1.to_index(shape, op)?; - Ok(vec![d0, d1]) - } -} - -impl Dims for (D1, D2, D3) { - fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { - let d0 = self.0.to_index(shape, op)?; - let d1 = self.1.to_index(shape, op)?; - let d2 = self.2.to_index(shape, op)?; - Ok(vec![d0, d1, d2]) - } -} - -impl Dims for (D1, D2, D3, D4) { - fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { - let d0 = self.0.to_index(shape, op)?; - let d1 = self.1.to_index(shape, op)?; - let d2 = self.2.to_index(shape, op)?; - let d3 = self.3.to_index(shape, op)?; - Ok(vec![d0, d1, d2, d3]) - } -} - -impl Dims for (D1, D2, D3, D4, D5) { - fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { - let d0 = self.0.to_index(shape, op)?; - let d1 = self.1.to_index(shape, op)?; - let d2 = self.2.to_index(shape, op)?; - let d3 = self.3.to_index(shape, op)?; - let d4 = self.4.to_index(shape, op)?; - Ok(vec![d0, d1, d2, d3, d4]) - } -} - -impl Dims for (D1, D2, D3, D4, D5, D6) { - fn to_indexes_internal(self, shape: &Shape, op: &'static str) -> Result> { - let d0 = self.0.to_index(shape, op)?; - let d1 = self.1.to_index(shape, op)?; - let d2 = self.2.to_index(shape, op)?; - let d3 = self.3.to_index(shape, op)?; - let d4 = self.4.to_index(shape, op)?; - let d5 = self.5.to_index(shape, op)?; - Ok(vec![d0, d1, d2, d3, d4, d5]) - } -} - -extract_dims!(dims0, 0, |_: &[usize]| (), ()); -extract_dims!(dims1, 1, |d: &[usize]| d[0], usize); -extract_dims!(dims2, 2, |d: &[usize]| (d[0], d[1]), (usize, usize)); -extract_dims!( - dims3, - 3, - |d: &[usize]| (d[0], d[1], d[2]), - (usize, usize, usize) -); -extract_dims!( - dims4, - 4, - |d: &[usize]| (d[0], d[1], d[2], d[3]), - (usize, usize, usize, usize) -); -extract_dims!( - dims5, - 5, - |d: &[usize]| (d[0], d[1], d[2], d[3], d[4]), - (usize, usize, usize, usize, usize) -); - -pub trait ShapeWithOneHole { - fn into_shape(self, el_count: usize) -> Result; -} - -impl> ShapeWithOneHole for S { - fn into_shape(self, _el_count: usize) -> Result { - Ok(self.into()) - } -} - -impl ShapeWithOneHole for ((),) { - fn into_shape(self, el_count: usize) -> Result { - Ok(el_count.into()) - } -} - -fn hole_size(el_count: usize, prod_d: usize, s: &dyn std::fmt::Debug) -> Result { - if prod_d == 0 { - crate::bail!("cannot reshape tensor of {el_count} elements to {s:?}") - } - if !el_count.is_multiple_of(prod_d) { - crate::bail!("cannot reshape tensor with {el_count} elements to {s:?}") - } - Ok(el_count / prod_d) -} - -impl ShapeWithOneHole for ((), usize) { - fn into_shape(self, el_count: usize) -> Result { - let ((), d1) = self; - Ok((hole_size(el_count, d1, &self)?, d1).into()) - } -} - -impl ShapeWithOneHole for (usize, ()) { - fn into_shape(self, el_count: usize) -> Result { - let (d1, ()) = self; - Ok((d1, hole_size(el_count, d1, &self)?).into()) - } -} - -impl ShapeWithOneHole for ((), usize, usize) { - fn into_shape(self, el_count: usize) -> Result { - let ((), d1, d2) = self; - Ok((hole_size(el_count, d1 * d2, &self)?, d1, d2).into()) - } -} - -impl ShapeWithOneHole for (usize, (), usize) { - fn into_shape(self, el_count: usize) -> Result { - let (d1, (), d2) = self; - Ok((d1, hole_size(el_count, d1 * d2, &self)?, d2).into()) - } -} - -impl ShapeWithOneHole for (usize, usize, ()) { - fn into_shape(self, el_count: usize) -> Result { - let (d1, d2, ()) = self; - Ok((d1, d2, hole_size(el_count, d1 * d2, &self)?).into()) - } -} - -impl ShapeWithOneHole for ((), usize, usize, usize) { - fn into_shape(self, el_count: usize) -> Result { - let ((), d1, d2, d3) = self; - let d = hole_size(el_count, d1 * d2 * d3, &self)?; - Ok((d, d1, d2, d3).into()) - } -} - -impl ShapeWithOneHole for (usize, (), usize, usize) { - fn into_shape(self, el_count: usize) -> Result { - let (d1, (), d2, d3) = self; - let d = hole_size(el_count, d1 * d2 * d3, &self)?; - Ok((d1, d, d2, d3).into()) - } -} - -impl ShapeWithOneHole for (usize, usize, (), usize) { - fn into_shape(self, el_count: usize) -> Result { - let (d1, d2, (), d3) = self; - let d = hole_size(el_count, d1 * d2 * d3, &self)?; - Ok((d1, d2, d, d3).into()) - } -} - -impl ShapeWithOneHole for (usize, usize, usize, ()) { - fn into_shape(self, el_count: usize) -> Result { - let (d1, d2, d3, ()) = self; - let d = hole_size(el_count, d1 * d2 * d3, &self)?; - Ok((d1, d2, d3, d).into()) - } -} - -impl ShapeWithOneHole for ((), usize, usize, usize, usize) { - fn into_shape(self, el_count: usize) -> Result { - let ((), d1, d2, d3, d4) = self; - let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?; - Ok((d, d1, d2, d3, d4).into()) - } -} - -impl ShapeWithOneHole for (usize, (), usize, usize, usize) { - fn into_shape(self, el_count: usize) -> Result { - let (d1, (), d2, d3, d4) = self; - let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?; - Ok((d1, d, d2, d3, d4).into()) - } -} - -impl ShapeWithOneHole for (usize, usize, (), usize, usize) { - fn into_shape(self, el_count: usize) -> Result { - let (d1, d2, (), d3, d4) = self; - let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?; - Ok((d1, d2, d, d3, d4).into()) - } -} - -impl ShapeWithOneHole for (usize, usize, usize, (), usize) { - fn into_shape(self, el_count: usize) -> Result { - let (d1, d2, d3, (), d4) = self; - let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?; - Ok((d1, d2, d3, d, d4).into()) - } -} - -impl ShapeWithOneHole for (usize, usize, usize, usize, ()) { - fn into_shape(self, el_count: usize) -> Result { - let (d1, d2, d3, d4, ()) = self; - let d = hole_size(el_count, d1 * d2 * d3 * d4, &self)?; - Ok((d1, d2, d3, d4, d).into()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn stride() { - let shape = Shape::from(()); - assert_eq!(shape.stride_contiguous(), Vec::::new()); - let shape = Shape::from(42); - assert_eq!(shape.stride_contiguous(), [1]); - let shape = Shape::from((42, 1337)); - assert_eq!(shape.stride_contiguous(), [1337, 1]); - let shape = Shape::from((299, 792, 458)); - assert_eq!(shape.stride_contiguous(), [458 * 792, 458, 1]); - } - - #[test] - fn test_from_tuple() { - let shape = Shape::from((2,)); - assert_eq!(shape.dims(), &[2]); - let shape = Shape::from((2, 3)); - assert_eq!(shape.dims(), &[2, 3]); - let shape = Shape::from((2, 3, 4)); - assert_eq!(shape.dims(), &[2, 3, 4]); - let shape = Shape::from((2, 3, 4, 5)); - assert_eq!(shape.dims(), &[2, 3, 4, 5]); - let shape = Shape::from((2, 3, 4, 5, 6)); - assert_eq!(shape.dims(), &[2, 3, 4, 5, 6]); - let shape = Shape::from((2, 3, 4, 5, 6, 7)); - assert_eq!(shape.dims(), &[2, 3, 4, 5, 6, 7]); - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/sort.rs b/v2/dependency-patches/candle-core-0.9.2/src/sort.rs deleted file mode 100644 index 19d783874b..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/sort.rs +++ /dev/null @@ -1,290 +0,0 @@ -use crate::{Result, Tensor}; -use rayon::prelude::*; - -#[derive(Debug, Clone, Copy)] -struct ArgSort { - asc: bool, - last_dim: usize, -} - -impl ArgSort { - fn asort(&self, vs: &[T], layout: &crate::Layout) -> Vec { - #[allow(clippy::uninit_vec)] - // Safety: indexes are set later in the parallelized section. - let mut sort_indexes = unsafe { - let el_count = layout.shape().elem_count(); - let mut v = Vec::with_capacity(el_count); - v.set_len(el_count); - v - }; - if self.asc { - sort_indexes - .par_chunks_exact_mut(self.last_dim) - .zip(vs.par_chunks_exact(self.last_dim)) - .for_each(|(indexes, vs)| { - indexes - .iter_mut() - .enumerate() - .for_each(|(i, v)| *v = i as u32); - indexes.sort_by(|&i, &j| { - vs[i as usize] - .partial_cmp(&vs[j as usize]) - .unwrap_or(std::cmp::Ordering::Greater) - }) - }); - } else { - sort_indexes - .par_chunks_exact_mut(self.last_dim) - .zip(vs.par_chunks_exact(self.last_dim)) - .for_each(|(indexes, vs)| { - indexes - .iter_mut() - .enumerate() - .for_each(|(i, v)| *v = i as u32); - indexes.sort_by(|&j, &i| { - vs[i as usize] - .partial_cmp(&vs[j as usize]) - .unwrap_or(std::cmp::Ordering::Greater) - }) - }); - } - sort_indexes - } -} - -#[cfg(feature = "cuda")] -mod cuda { - use super::*; - use crate::cuda_backend::cudarc::driver::{ - CudaSlice, DeviceRepr, LaunchConfig, ValidAsZeroBits, - }; - use crate::cuda_backend::{kernel_name, kernels, CudaStorageSlice as S, WrapErr}; - use crate::{CudaDevice, WithDType}; - - impl crate::cuda_backend::Map1Any for ArgSort { - fn f) -> S>( - &self, - src: &CudaSlice, - dev: &CudaDevice, - layout: &crate::Layout, - _wrap: W, - ) -> Result { - use cudarc::driver::PushKernelArg; - - let slice = match layout.contiguous_offsets() { - None => crate::bail!("input has to be contiguous"), - Some((o1, o2)) => src.slice(o1..o2), - }; - let elem_count = layout.shape().elem_count(); - let dst = unsafe { dev.alloc::(elem_count)? }; - let func = if self.asc { - dev.get_or_load_func(&kernel_name::("asort_asc"), &kernels::SORT)? - } else { - dev.get_or_load_func(&kernel_name::("asort_desc"), &kernels::SORT)? - }; - let ncols = self.last_dim; - let nrows = elem_count / ncols; - let ncols_pad = next_power_of_2(ncols); - // Limit block dim to 1024 threads, which is the maximum on modern CUDA gpus. - let block_dim = ncols_pad.min(1024); - let cfg = LaunchConfig { - grid_dim: (nrows as u32, 1, 1), - block_dim: (block_dim as u32, 1, 1), - shared_mem_bytes: (ncols_pad * std::mem::size_of::()) as u32, - }; - let stream = dev.cuda_stream(); - let mut builder = stream.launch_builder(&func); - let ncols = ncols as i32; - let ncols_pad = ncols_pad as i32; - builder.arg(&slice).arg(&dst).arg(&ncols).arg(&ncols_pad); - unsafe { builder.launch(cfg) }.w()?; - Ok(S::U32(dst)) - } - } -} - -impl crate::CustomOp1 for ArgSort { - fn name(&self) -> &'static str { - "argsort" - } - - fn cpu_fwd( - &self, - storage: &crate::CpuStorage, - layout: &crate::Layout, - ) -> Result<(crate::CpuStorage, crate::Shape)> { - let sort_indexes = match storage { - crate::CpuStorage::U8(vs) => self.asort(vs, layout), - crate::CpuStorage::U32(vs) => self.asort(vs, layout), - crate::CpuStorage::I16(vs) => self.asort(vs, layout), - crate::CpuStorage::I32(vs) => self.asort(vs, layout), - crate::CpuStorage::I64(vs) => self.asort(vs, layout), - crate::CpuStorage::BF16(vs) => self.asort(vs, layout), - crate::CpuStorage::F16(vs) => self.asort(vs, layout), - crate::CpuStorage::F32(vs) => self.asort(vs, layout), - crate::CpuStorage::F64(vs) => self.asort(vs, layout), - crate::CpuStorage::F8E4M3(vs) => self.asort(vs, layout), - // Dummy types don't support sorting - crate::CpuStorage::F6E2M3(_) => { - return Err( - crate::Error::UnsupportedDTypeForOp(crate::DType::F6E2M3, "argsort").bt(), - ) - } - crate::CpuStorage::F6E3M2(_) => { - return Err( - crate::Error::UnsupportedDTypeForOp(crate::DType::F6E3M2, "argsort").bt(), - ) - } - crate::CpuStorage::F4(_) => { - return Err(crate::Error::UnsupportedDTypeForOp(crate::DType::F4, "argsort").bt()) - } - crate::CpuStorage::F8E8M0(_) => { - return Err( - crate::Error::UnsupportedDTypeForOp(crate::DType::F8E8M0, "argsort").bt(), - ) - } - }; - let sort_indexes = crate::CpuStorage::U32(sort_indexes); - Ok((sort_indexes, layout.shape().into())) - } - - #[cfg(feature = "cuda")] - fn cuda_fwd( - &self, - storage: &crate::CudaStorage, - layout: &crate::Layout, - ) -> Result<(crate::CudaStorage, crate::Shape)> { - use crate::backend::BackendStorage; - use crate::cuda_backend::Map1Any; - let dev = storage.device(); - let slice = self.map(&storage.slice, dev, layout)?; - let dst = crate::cuda_backend::CudaStorage { - slice, - device: dev.clone(), - }; - Ok((dst, layout.shape().clone())) - } - - #[cfg(feature = "metal")] - fn metal_fwd( - &self, - storage: &crate::MetalStorage, - layout: &crate::Layout, - ) -> Result<(crate::MetalStorage, crate::Shape)> { - use crate::backend::BackendStorage; - use crate::DType; - - let name = { - if self.asc { - match storage.dtype() { - DType::BF16 => "asort_asc_bf16", - DType::F16 => "asort_asc_f16", - DType::F32 => "asort_asc_f32", - DType::F64 => "asort_asc_f64", - DType::U8 => "asort_asc_u8", - DType::U32 => "asort_asc_u32", - DType::I16 => "asort_asc_i16", - DType::I32 => "asort_asc_i32", - DType::I64 => "asort_asc_i64", - DType::F8E4M3 => crate::bail!("Metal device does not yet support F8E4M3."), - DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { - return Err( - crate::Error::UnsupportedDTypeForOp(storage.dtype(), "argsort").bt(), - ) - } - } - } else { - match storage.dtype() { - DType::BF16 => "asort_desc_bf16", - DType::F16 => "asort_desc_f16", - DType::F32 => "asort_desc_f32", - DType::F64 => "asort_desc_f64", - DType::U8 => "asort_desc_u8", - DType::U32 => "asort_desc_u32", - DType::I16 => "asort_desc_i16", - DType::I32 => "asort_desc_i32", - DType::I64 => "asort_desc_i64", - DType::F8E4M3 => crate::bail!("Metal device does not yet support F8E4M3."), - DType::F6E2M3 | DType::F6E3M2 | DType::F4 | DType::F8E8M0 => { - return Err( - crate::Error::UnsupportedDTypeForOp(storage.dtype(), "argsort").bt(), - ) - } - } - } - }; - let device = storage.device(); - let kernels = device.kernels(); - let command_encoder = device.command_encoder()?; - let el = layout.shape().elem_count(); - let ncols = self.last_dim; - let nrows = el / ncols; - let src = crate::metal_backend::buffer_o(storage.buffer(), layout, storage.dtype()); - let dst = device.new_buffer(el, DType::U32, "asort")?; - let mut ncols_pad = 1; - while ncols_pad < ncols { - ncols_pad *= 2; - } - candle_metal_kernels::call_arg_sort( - device.metal_device(), - &command_encoder, - kernels, - name, - nrows, - ncols, - ncols_pad, - src, - &dst, - ) - .map_err(crate::Error::wrap)?; - let dst = crate::MetalStorage::new(dst, device.clone(), el, DType::U32); - Ok((dst, layout.shape().clone())) - } -} - -#[allow(unused)] -fn next_power_of_2(x: usize) -> usize { - let mut n = 1; - while n < x { - n *= 2 - } - n -} - -impl Tensor { - /// Returns the indices that sort the tensor along the last dimension. - /// - /// If `asc` is `true`, sorting is in ascending order. Otherwise sorting is performed in - /// descending order. The sort is unstable so there is no guarantees on the final order when it - /// comes to ties. - pub fn arg_sort_last_dim(&self, asc: bool) -> Result { - if !self.is_contiguous() { - return Err(crate::Error::RequiresContiguous { - op: "arg_sort_last_dim", - }); - } - let last_dim = match self.dims().last() { - None => crate::bail!("empty last-dim in arg-sort"), - Some(last_dim) => *last_dim, - }; - // No need for a backward pass for arg sort. - self.apply_op1_no_bwd(&ArgSort { asc, last_dim }) - } - - /// Sorts the tensor along the last dimension, returns the sorted tensor together with the - /// sorted indexes. - /// - /// If `asc` is `true`, sorting is in ascending order. Otherwise sorting is performed in - /// descending order. The sort is unstable so there is no guarantees on the final order when it - /// comes to ties. - pub fn sort_last_dim(&self, asc: bool) -> Result<(Tensor, Tensor)> { - if !self.is_contiguous() { - return Err(crate::Error::RequiresContiguous { - op: "sort_last_dim", - }); - } - let asort = self.arg_sort_last_dim(asc)?; - let sorted = self.gather(&asort, crate::D::Minus1)?; - Ok((sorted, asort)) - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/storage.rs b/v2/dependency-patches/candle-core-0.9.2/src/storage.rs deleted file mode 100644 index 4646dc88ea..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/storage.rs +++ /dev/null @@ -1,849 +0,0 @@ -use crate::backend::BackendStorage; -use crate::op::{self, CmpOp, ReduceOp}; -use crate::scalar::Scalar; -use crate::{CpuStorage, CudaStorage, DType, Device, Error, Layout, MetalStorage, Result, Shape}; -use crate::{CustomOp1, CustomOp2, CustomOp3, InplaceOp1, InplaceOp2, InplaceOp3}; - -// We do not want to implement Clone on Storage as cloning may fail because of -// out of memory. Instead try_clone should be used. -#[derive(Debug)] -pub enum Storage { - Cpu(CpuStorage), - Cuda(CudaStorage), - Metal(MetalStorage), -} - -impl Storage { - pub fn try_clone(&self, layout: &Layout) -> Result { - match self { - Self::Cpu(storage) => Ok(Self::Cpu(storage.clone())), - Self::Cuda(storage) => { - let storage = storage.try_clone(layout)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = storage.try_clone(layout)?; - Ok(Self::Metal(storage)) - } - } - } - - pub fn device(&self) -> Device { - match self { - Self::Cpu(_) => Device::Cpu, - Self::Cuda(storage) => Device::Cuda(storage.device().clone()), - Self::Metal(storage) => Device::Metal(storage.device().clone()), - } - } - - pub fn dtype(&self) -> DType { - match self { - Self::Cpu(storage) => storage.dtype(), - Self::Cuda(storage) => storage.dtype(), - Self::Metal(storage) => storage.dtype(), - } - } - - pub(crate) fn same_device(&self, rhs: &Self, op: &'static str) -> Result<()> { - let lhs_device = self.device(); - let rhs_device = rhs.device(); - let lhs = lhs_device.location(); - let rhs = rhs_device.location(); - let same_device = if self.device().is_metal() { - // On metal, we require the device to be exactly the same rather than - // having the same location. In cuda this is not necessary as all CudaDevice on the - // same GPU will use the same cuda stream. - lhs_device.same_device(&rhs_device) - } else { - lhs == rhs - }; - if !same_device { - Err(Error::DeviceMismatchBinaryOp { lhs, rhs, op }.bt()) - } else { - Ok(()) - } - } - - pub(crate) fn same_dtype(&self, rhs: &Self, op: &'static str) -> Result<()> { - let lhs = self.dtype(); - let rhs = rhs.dtype(); - if lhs != rhs { - Err(Error::DTypeMismatchBinaryOp { lhs, rhs, op }.bt()) - } else { - Ok(()) - } - } - - pub(crate) fn const_set(&mut self, v: Scalar, l: &Layout) -> Result<()> { - match self { - Storage::Cpu(storage) => storage.const_set(v, l), - Storage::Cuda(storage) => storage.const_set(v, l), - Storage::Metal(storage) => storage.const_set(v, l), - } - } - - pub(crate) fn affine(&self, layout: &Layout, mul: f64, add: f64) -> Result { - match self { - Storage::Cpu(storage) => { - let storage = storage.affine(layout, mul, add)?; - Ok(Self::Cpu(storage)) - } - Self::Cuda(storage) => { - let storage = storage.affine(layout, mul, add)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = storage.affine(layout, mul, add)?; - Ok(Self::Metal(storage)) - } - } - } - - pub(crate) fn powf(&self, layout: &Layout, alpha: f64) -> Result { - match self { - Storage::Cpu(storage) => { - let storage = storage.powf(layout, alpha)?; - Ok(Self::Cpu(storage)) - } - Self::Cuda(storage) => { - let storage = storage.powf(layout, alpha)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = storage.powf(layout, alpha)?; - Ok(Self::Metal(storage)) - } - } - } - - pub(crate) fn elu(&self, layout: &Layout, alpha: f64) -> Result { - match self { - Storage::Cpu(storage) => { - let storage = storage.elu(layout, alpha)?; - Ok(Self::Cpu(storage)) - } - Self::Cuda(storage) => { - let storage = storage.elu(layout, alpha)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = storage.elu(layout, alpha)?; - Ok(Self::Metal(storage)) - } - } - } - - pub(crate) fn cmp( - &self, - op: CmpOp, - rhs: &Self, - lhs_layout: &Layout, - rhs_layout: &Layout, - ) -> Result { - self.same_device(rhs, "cmp")?; - self.same_dtype(rhs, "cmp")?; - match (self, rhs) { - (Storage::Cpu(lhs), Storage::Cpu(rhs)) => { - let storage = lhs.cmp(op, rhs, lhs_layout, rhs_layout)?; - Ok(Self::Cpu(storage)) - } - (Self::Cuda(lhs), Self::Cuda(rhs)) => { - let storage = lhs.cmp(op, rhs, lhs_layout, rhs_layout)?; - Ok(Self::Cuda(storage)) - } - (Self::Metal(lhs), Self::Metal(rhs)) => { - let storage = lhs.cmp(op, rhs, lhs_layout, rhs_layout)?; - Ok(Self::Metal(storage)) - } - (lhs, rhs) => { - // Should not happen because of the same device check above but we're defensive - // anyway. - Err(Error::DeviceMismatchBinaryOp { - lhs: lhs.device().location(), - rhs: rhs.device().location(), - op: "cmp", - } - .bt()) - } - } - } - - pub(crate) fn reduce_op(&self, op: ReduceOp, layout: &Layout, s: &[usize]) -> Result { - match self { - Storage::Cpu(storage) => { - let storage = storage.reduce_op(op, layout, s)?; - Ok(Self::Cpu(storage)) - } - Self::Cuda(storage) => { - let storage = storage.reduce_op(op, layout, s)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = storage.reduce_op(op, layout, s)?; - Ok(Self::Metal(storage)) - } - } - } - - pub(crate) fn to_dtype(&self, layout: &Layout, dtype: DType) -> Result { - match self { - Storage::Cpu(storage) => { - let storage = storage.to_dtype(layout, dtype)?; - Ok(Self::Cpu(storage)) - } - Self::Cuda(storage) => { - let storage = storage.to_dtype(layout, dtype)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = storage.to_dtype(layout, dtype)?; - Ok(Self::Metal(storage)) - } - } - } - - pub(crate) fn apply_op1(&self, l: &Layout, c: &dyn CustomOp1) -> Result<(Self, Shape)> { - match self { - Self::Cpu(storage) => { - let (storage, shape) = c.cpu_fwd(storage, l)?; - Ok((Self::Cpu(storage), shape)) - } - Self::Cuda(storage) => { - let (storage, shape) = c.cuda_fwd(storage, l)?; - Ok((Self::Cuda(storage), shape)) - } - Self::Metal(storage) => { - let (storage, shape) = c.metal_fwd(storage, l)?; - Ok((Self::Metal(storage), shape)) - } - } - } - - pub(crate) fn apply_op2( - &self, - l1: &Layout, - t2: &Self, - l2: &Layout, - c: &dyn CustomOp2, - ) -> Result<(Self, Shape)> { - self.same_device(t2, c.name())?; - match (self, t2) { - (Self::Cpu(s1), Self::Cpu(s2)) => { - let (s, shape) = c.cpu_fwd(s1, l1, s2, l2)?; - Ok((Self::Cpu(s), shape)) - } - (Self::Cuda(s1), Self::Cuda(s2)) => { - let (s, shape) = c.cuda_fwd(s1, l1, s2, l2)?; - Ok((Self::Cuda(s), shape)) - } - (Self::Metal(s1), Self::Metal(s2)) => { - let (s, shape) = c.metal_fwd(s1, l1, s2, l2)?; - Ok((Self::Metal(s), shape)) - } - _ => unreachable!(), - } - } - - pub(crate) fn apply_op3( - &self, - l1: &Layout, - t2: &Self, - l2: &Layout, - t3: &Self, - l3: &Layout, - c: &dyn CustomOp3, - ) -> Result<(Self, Shape)> { - self.same_device(t2, c.name())?; - self.same_device(t3, c.name())?; - match (self, t2, t3) { - (Self::Cpu(s1), Self::Cpu(s2), Self::Cpu(s3)) => { - let (s, shape) = c.cpu_fwd(s1, l1, s2, l2, s3, l3)?; - Ok((Self::Cpu(s), shape)) - } - (Self::Cuda(s1), Self::Cuda(s2), Self::Cuda(s3)) => { - let (s, shape) = c.cuda_fwd(s1, l1, s2, l2, s3, l3)?; - Ok((Self::Cuda(s), shape)) - } - (Self::Metal(s1), Self::Metal(s2), Self::Metal(s3)) => { - let (s, shape) = c.metal_fwd(s1, l1, s2, l2, s3, l3)?; - Ok((Self::Metal(s), shape)) - } - _ => unreachable!(), - } - } - - pub(crate) fn inplace_op1(&mut self, l: &Layout, c: &dyn InplaceOp1) -> Result<()> { - match self { - Self::Cpu(storage) => c.cpu_fwd(storage, l), - Self::Cuda(storage) => c.cuda_fwd(storage, l), - Self::Metal(storage) => c.metal_fwd(storage, l), - } - } - - pub(crate) fn inplace_op2( - &mut self, - l1: &Layout, - t2: &Self, - l2: &Layout, - c: &dyn InplaceOp2, - ) -> Result<()> { - self.same_device(t2, c.name())?; - match (self, t2) { - (Self::Cpu(s1), Self::Cpu(s2)) => c.cpu_fwd(s1, l1, s2, l2), - (Self::Cuda(s1), Self::Cuda(s2)) => c.cuda_fwd(s1, l1, s2, l2), - (Self::Metal(s1), Self::Metal(s2)) => c.metal_fwd(s1, l1, s2, l2), - _ => unreachable!(), - } - } - - pub(crate) fn inplace_op3( - &mut self, - l1: &Layout, - t2: &Self, - l2: &Layout, - t3: &Self, - l3: &Layout, - c: &dyn InplaceOp3, - ) -> Result<()> { - self.same_device(t2, c.name())?; - self.same_device(t3, c.name())?; - match (self, t2, t3) { - (Self::Cpu(s1), Self::Cpu(s2), Self::Cpu(s3)) => c.cpu_fwd(s1, l1, s2, l2, s3, l3), - (Self::Cuda(s1), Self::Cuda(s2), Self::Cuda(s3)) => c.cuda_fwd(s1, l1, s2, l2, s3, l3), - (Self::Metal(s1), Self::Metal(s2), Self::Metal(s3)) => { - c.metal_fwd(s1, l1, s2, l2, s3, l3) - } - _ => unreachable!(), - } - } - - pub(crate) fn unary_impl(&self, layout: &Layout) -> Result { - match self { - Storage::Cpu(storage) => { - let storage = storage.unary_impl::(layout)?; - Ok(Self::Cpu(storage)) - } - Self::Cuda(storage) => { - let storage = storage.unary_impl::(layout)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = storage.unary_impl::(layout)?; - Ok(Self::Metal(storage)) - } - } - } - - pub(crate) fn binary_impl( - &self, - rhs: &Self, - lhs_layout: &Layout, - rhs_layout: &Layout, - ) -> Result { - self.same_device(rhs, B::NAME)?; - self.same_dtype(rhs, B::NAME)?; - match (self, rhs) { - (Storage::Cpu(lhs), Storage::Cpu(rhs)) => { - let storage = lhs.binary_impl::(rhs, lhs_layout, rhs_layout)?; - Ok(Self::Cpu(storage)) - } - (Self::Cuda(lhs), Self::Cuda(rhs)) => { - let storage = lhs.binary_impl::(rhs, lhs_layout, rhs_layout)?; - Ok(Self::Cuda(storage)) - } - (Self::Metal(lhs), Self::Metal(rhs)) => { - let storage = lhs.binary_impl::(rhs, lhs_layout, rhs_layout)?; - Ok(Self::Metal(storage)) - } - (lhs, rhs) => { - // Should not happen because of the same device check above but we're defensive - // anyway. - Err(Error::DeviceMismatchBinaryOp { - lhs: lhs.device().location(), - rhs: rhs.device().location(), - op: B::NAME, - } - .bt()) - } - } - } - - pub(crate) fn conv1d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConv1D, - ) -> Result { - self.same_device(kernel, "conv1d")?; - self.same_dtype(kernel, "conv1d")?; - match (self, &kernel) { - (Storage::Cpu(inp), Storage::Cpu(kernel)) => { - let s = inp.conv1d(l, kernel, kernel_l, params)?; - Ok(Self::Cpu(s)) - } - (Storage::Cuda(inp), Storage::Cuda(kernel)) => { - let s = inp.conv1d(l, kernel, kernel_l, params)?; - Ok(Self::Cuda(s)) - } - (Storage::Metal(inp), Storage::Metal(kernel)) => { - let s = inp.conv1d(l, kernel, kernel_l, params)?; - Ok(Self::Metal(s)) - } - (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { - lhs: lhs.device().location(), - rhs: rhs.device().location(), - op: "conv1d", - } - .bt()), - } - } - - pub(crate) fn conv_transpose1d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConvTranspose1D, - ) -> Result { - self.same_device(kernel, "conv-transpose1d")?; - self.same_dtype(kernel, "conv-transpose1d")?; - match (self, &kernel) { - (Storage::Cpu(inp), Storage::Cpu(kernel)) => { - let s = inp.conv_transpose1d(l, kernel, kernel_l, params)?; - Ok(Self::Cpu(s)) - } - (Storage::Cuda(inp), Storage::Cuda(kernel)) => { - let s = inp.conv_transpose1d(l, kernel, kernel_l, params)?; - Ok(Self::Cuda(s)) - } - (Storage::Metal(inp), Storage::Metal(kernel)) => { - let s = inp.conv_transpose1d(l, kernel, kernel_l, params)?; - Ok(Self::Metal(s)) - } - (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { - lhs: lhs.device().location(), - rhs: rhs.device().location(), - op: "conv-transpose1d", - } - .bt()), - } - } - - pub(crate) fn conv2d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConv2D, - ) -> Result { - self.same_device(kernel, "conv2d")?; - self.same_dtype(kernel, "conv2d")?; - match (self, &kernel) { - (Storage::Cpu(inp), Storage::Cpu(kernel)) => { - let s = inp.conv2d(l, kernel, kernel_l, params)?; - Ok(Self::Cpu(s)) - } - (Storage::Cuda(inp), Storage::Cuda(kernel)) => { - let s = inp.conv2d(l, kernel, kernel_l, params)?; - Ok(Self::Cuda(s)) - } - (Storage::Metal(inp), Storage::Metal(kernel)) => { - let s = inp.conv2d(l, kernel, kernel_l, params)?; - Ok(Self::Metal(s)) - } - (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { - lhs: lhs.device().location(), - rhs: rhs.device().location(), - op: "conv2d", - } - .bt()), - } - } - - pub(crate) fn conv_transpose2d( - &self, - l: &Layout, - kernel: &Self, - kernel_l: &Layout, - params: &crate::conv::ParamsConvTranspose2D, - ) -> Result { - self.same_device(kernel, "conv_transpose2d")?; - self.same_dtype(kernel, "conv_transpose2d")?; - match (self, &kernel) { - (Storage::Cpu(inp), Storage::Cpu(kernel)) => { - let s = inp.conv_transpose2d(l, kernel, kernel_l, params)?; - Ok(Self::Cpu(s)) - } - (Storage::Cuda(inp), Storage::Cuda(kernel)) => { - let s = inp.conv_transpose2d(l, kernel, kernel_l, params)?; - Ok(Self::Cuda(s)) - } - (Storage::Metal(inp), Storage::Metal(kernel)) => { - let s = inp.conv_transpose2d(l, kernel, kernel_l, params)?; - Ok(Self::Metal(s)) - } - (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { - lhs: lhs.device().location(), - rhs: rhs.device().location(), - op: "conv_transpose2d", - } - .bt()), - } - } - - pub(crate) fn avg_pool2d( - &self, - layout: &Layout, - kernel_size: (usize, usize), - stride: (usize, usize), - ) -> Result { - match self { - Storage::Cpu(storage) => { - let storage = storage.avg_pool2d(layout, kernel_size, stride)?; - Ok(Self::Cpu(storage)) - } - Self::Cuda(storage) => { - let storage = storage.avg_pool2d(layout, kernel_size, stride)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = storage.avg_pool2d(layout, kernel_size, stride)?; - Ok(Self::Metal(storage)) - } - } - } - - pub(crate) fn max_pool2d( - &self, - layout: &Layout, - kernel_size: (usize, usize), - stride: (usize, usize), - ) -> Result { - match self { - Storage::Cpu(storage) => { - let storage = storage.max_pool2d(layout, kernel_size, stride)?; - Ok(Self::Cpu(storage)) - } - Self::Cuda(storage) => { - let storage = storage.max_pool2d(layout, kernel_size, stride)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = storage.max_pool2d(layout, kernel_size, stride)?; - Ok(Self::Metal(storage)) - } - } - } - - pub(crate) fn upsample_nearest1d(&self, layout: &Layout, sz: usize) -> Result { - match self { - Storage::Cpu(storage) => { - let storage = storage.upsample_nearest1d(layout, sz)?; - Ok(Self::Cpu(storage)) - } - Self::Cuda(storage) => { - let storage = storage.upsample_nearest1d(layout, sz)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = storage.upsample_nearest1d(layout, sz)?; - Ok(Self::Metal(storage)) - } - } - } - - pub(crate) fn upsample_nearest2d(&self, layout: &Layout, h: usize, w: usize) -> Result { - match self { - Storage::Cpu(storage) => { - let storage = storage.upsample_nearest2d(layout, h, w)?; - Ok(Self::Cpu(storage)) - } - Self::Cuda(storage) => { - let storage = storage.upsample_nearest2d(layout, h, w)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = storage.upsample_nearest2d(layout, h, w)?; - Ok(Self::Metal(storage)) - } - } - } - - pub(crate) fn upsample_bilinear2d( - &self, - layout: &Layout, - h: usize, - w: usize, - align_corners: bool, - scale_h: Option, - scale_w: Option, - ) -> Result { - match self { - Storage::Cpu(storage) => { - let storage = - storage.upsample_bilinear2d(layout, h, w, align_corners, scale_h, scale_w)?; - Ok(Self::Cpu(storage)) - } - Self::Cuda(storage) => { - let storage = - storage.upsample_bilinear2d(layout, h, w, align_corners, scale_h, scale_w)?; - Ok(Self::Cuda(storage)) - } - Self::Metal(storage) => { - let storage = - storage.upsample_bilinear2d(layout, h, w, align_corners, scale_h, scale_w)?; - Ok(Self::Metal(storage)) - } - } - } - - pub(crate) fn where_cond( - &self, - layout: &Layout, - t: &Self, - layout_t: &Layout, - f: &Self, - layout_f: &Layout, - ) -> Result { - self.same_device(t, "where")?; - self.same_device(f, "where")?; - t.same_dtype(f, "where")?; - match (self, t, f) { - (Storage::Cpu(cond), Storage::Cpu(t), Storage::Cpu(f)) => { - let storage = cond.where_cond(layout, t, layout_t, f, layout_f)?; - Ok(Self::Cpu(storage)) - } - (Self::Cuda(cond), Self::Cuda(t), Self::Cuda(f)) => { - let storage = cond.where_cond(layout, t, layout_t, f, layout_f)?; - Ok(Self::Cuda(storage)) - } - (Self::Metal(cond), Self::Metal(t), Self::Metal(f)) => { - let storage = cond.where_cond(layout, t, layout_t, f, layout_f)?; - Ok(Self::Metal(storage)) - } - (_, lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { - lhs: lhs.device().location(), - rhs: rhs.device().location(), - op: "where", - } - .bt()), - } - } - - pub(crate) fn gather( - &self, - l: &Layout, - indexes: &Self, - indexes_l: &Layout, - d: usize, - ) -> Result { - self.same_device(indexes, "index-add")?; - match (self, indexes) { - (Self::Cpu(s), Self::Cpu(indexes)) => { - let storage = s.gather(l, indexes, indexes_l, d)?; - Ok(Self::Cpu(storage)) - } - (Self::Cuda(s), Self::Cuda(indexes)) => { - let storage = s.gather(l, indexes, indexes_l, d)?; - Ok(Self::Cuda(storage)) - } - (Self::Metal(s), Self::Metal(indexes)) => { - let storage = s.gather(l, indexes, indexes_l, d)?; - Ok(Self::Metal(storage)) - } - _ => unreachable!(), - } - } - - pub(crate) fn scatter_set( - &mut self, - l: &Layout, - indexes: &Self, - indexes_l: &Layout, - source: &Self, - source_l: &Layout, - d: usize, - ) -> Result<()> { - self.same_device(indexes, "scatter-set")?; - self.same_device(source, "scatter-set")?; - match (self, indexes, source) { - (Self::Cpu(s), Self::Cpu(indexes), Self::Cpu(source)) => { - s.scatter_set(l, indexes, indexes_l, source, source_l, d)?; - } - (Self::Cuda(s), Self::Cuda(indexes), Self::Cuda(source)) => { - s.scatter_set(l, indexes, indexes_l, source, source_l, d)?; - } - (Self::Metal(s), Self::Metal(indexes), Self::Metal(source)) => { - s.scatter_set(l, indexes, indexes_l, source, source_l, d)?; - } - _ => unreachable!(), - } - Ok(()) - } - - pub(crate) fn scatter_add( - &mut self, - l: &Layout, - indexes: &Self, - indexes_l: &Layout, - source: &Self, - source_l: &Layout, - d: usize, - ) -> Result<()> { - self.same_device(indexes, "scatter-add")?; - self.same_device(source, "scatter-add")?; - match (self, indexes, source) { - (Self::Cpu(s), Self::Cpu(indexes), Self::Cpu(source)) => { - s.scatter_add_set(l, indexes, indexes_l, source, source_l, d)?; - } - (Self::Cuda(s), Self::Cuda(indexes), Self::Cuda(source)) => { - s.scatter_add_set(l, indexes, indexes_l, source, source_l, d)?; - } - (Self::Metal(s), Self::Metal(indexes), Self::Metal(source)) => { - s.scatter_add_set(l, indexes, indexes_l, source, source_l, d)?; - } - _ => unreachable!(), - } - Ok(()) - } - - pub(crate) fn index_add( - &self, - l: &Layout, - indexes: &Self, - indexes_l: &Layout, - source: &Self, - source_l: &Layout, - d: usize, - ) -> Result { - self.same_device(indexes, "index-add")?; - self.same_device(source, "index-add")?; - match (self, indexes, source) { - (Self::Cpu(s), Self::Cpu(indexes), Self::Cpu(source)) => { - let storage = s.index_add(l, indexes, indexes_l, source, source_l, d)?; - Ok(Self::Cpu(storage)) - } - (Self::Cuda(s), Self::Cuda(indexes), Self::Cuda(source)) => { - let storage = s.index_add(l, indexes, indexes_l, source, source_l, d)?; - Ok(Self::Cuda(storage)) - } - (Self::Metal(s), Self::Metal(indexes), Self::Metal(source)) => { - let storage = s.index_add(l, indexes, indexes_l, source, source_l, d)?; - Ok(Self::Metal(storage)) - } - _ => unreachable!(), - } - } - - pub(crate) fn index_select( - &self, - rhs: &Self, - lhs_l: &Layout, - rhs_l: &Layout, - d: usize, - ) -> Result { - self.same_device(rhs, "index-select")?; - match (self, rhs) { - (Self::Cpu(lhs), Self::Cpu(rhs)) => { - let storage = lhs.index_select(rhs, lhs_l, rhs_l, d)?; - Ok(Self::Cpu(storage)) - } - (Self::Cuda(lhs), Self::Cuda(rhs)) => { - let storage = lhs.index_select(rhs, lhs_l, rhs_l, d)?; - Ok(Self::Cuda(storage)) - } - (Self::Metal(lhs), Self::Metal(rhs)) => { - let storage = lhs.index_select(rhs, lhs_l, rhs_l, d)?; - Ok(Self::Metal(storage)) - } - (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { - lhs: lhs.device().location(), - rhs: rhs.device().location(), - op: "index-select", - } - .bt()), - } - } - - pub(crate) fn matmul( - &self, - rhs: &Self, - bmnk: (usize, usize, usize, usize), - lhs_layout: &Layout, - rhs_layout: &Layout, - ) -> Result { - self.same_device(rhs, "matmul")?; - self.same_dtype(rhs, "matmul")?; - match (self, rhs) { - (Self::Cpu(lhs), Self::Cpu(rhs)) => { - let storage = lhs.matmul(rhs, bmnk, lhs_layout, rhs_layout)?; - Ok(Self::Cpu(storage)) - } - (Self::Cuda(lhs), Self::Cuda(rhs)) => { - let storage = lhs.matmul(rhs, bmnk, lhs_layout, rhs_layout)?; - Ok(Self::Cuda(storage)) - } - (Self::Metal(lhs), Self::Metal(rhs)) => { - let storage = lhs.matmul(rhs, bmnk, lhs_layout, rhs_layout)?; - Ok(Self::Metal(storage)) - } - (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { - lhs: lhs.device().location(), - rhs: rhs.device().location(), - op: "matmul", - } - .bt()), - } - } - - // self, the source can be strided whereas dst is contiguous. - pub(crate) fn copy_strided_src( - &self, - dst: &mut Self, - dst_offset: usize, - src_l: &Layout, - ) -> Result<()> { - match (self, dst) { - (Self::Cpu(src), Self::Cpu(dst)) => src.copy_strided_src(dst, dst_offset, src_l), - (Self::Cuda(src), Self::Cuda(dst)) => Ok(src.copy_strided_src(dst, dst_offset, src_l)?), - (Self::Metal(src), Self::Metal(dst)) => { - Ok(src.copy_strided_src(dst, dst_offset, src_l)?) - } - (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { - lhs: lhs.device().location(), - rhs: rhs.device().location(), - op: "copy", - } - .bt()), - } - } - - #[allow(clippy::too_many_arguments)] - pub(crate) fn copy2d( - &self, - dst: &mut Self, - d1: usize, - d2: usize, - src_s: usize, - dst_s: usize, - src_o: usize, - dst_o: usize, - ) -> Result<()> { - match (self, dst) { - (Self::Cpu(src), Self::Cpu(dst)) => src.copy2d(dst, d1, d2, src_s, dst_s, src_o, dst_o), - (Self::Cuda(src), Self::Cuda(dst)) => { - Ok(src.copy2d(dst, d1, d2, src_s, dst_s, src_o, dst_o)?) - } - (Self::Metal(src), Self::Metal(dst)) => { - Ok(src.copy2d(dst, d1, d2, src_s, dst_s, src_o, dst_o)?) - } - (lhs, rhs) => Err(Error::DeviceMismatchBinaryOp { - lhs: lhs.device().location(), - rhs: rhs.device().location(), - op: "copy2d", - } - .bt()), - } - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/streaming.rs b/v2/dependency-patches/candle-core-0.9.2/src/streaming.rs deleted file mode 100644 index f4c0a9ff0b..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/streaming.rs +++ /dev/null @@ -1,208 +0,0 @@ -//! StreamTensror useful for streaming ops. -//! -use crate::{Result, Shape, Tensor}; - -pub trait Dim: crate::shape::Dim + Copy {} -impl Dim for T {} - -/// A stream tensor is used in streaming module. It can either contain an actual tensor or be -/// empty. -#[derive(Clone)] -pub struct StreamTensor(Option); - -impl std::fmt::Debug for StreamTensor { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match &self.0 { - Some(t) => write!(f, "{:?}", t.shape()), - None => write!(f, "Empty"), - } - } -} - -impl std::convert::From> for StreamTensor { - fn from(value: Option) -> Self { - Self(value) - } -} - -impl std::convert::From for StreamTensor { - fn from(value: Tensor) -> Self { - Self(Some(value)) - } -} - -impl std::convert::From<()> for StreamTensor { - fn from(_value: ()) -> Self { - Self(None) - } -} - -impl StreamTensor { - pub fn empty() -> Self { - Self(None) - } - - pub fn from_tensor(tensor: Tensor) -> Self { - Self(Some(tensor)) - } - - pub fn shape(&self) -> Option<&Shape> { - self.0.as_ref().map(|t| t.shape()) - } - - pub fn cat2(&self, rhs: &Self, dim: D) -> Result { - let xs = match (&self.0, &rhs.0) { - (Some(lhs), Some(rhs)) => { - let xs = Tensor::cat(&[lhs, rhs], dim)?; - Some(xs) - } - (Some(xs), None) | (None, Some(xs)) => Some(xs.clone()), - (None, None) => None, - }; - Ok(Self(xs)) - } - - pub fn seq_len(&self, dim: D) -> Result { - match &self.0 { - None => Ok(0), - Some(v) => v.dim(dim), - } - } - - pub fn reset(&mut self) { - self.0 = None - } - - pub fn narrow(&self, dim: D, offset: usize, len: usize) -> Result { - let t = match &self.0 { - None => None, - Some(t) => { - let seq_len = t.dim(dim)?; - if seq_len <= offset { - None - } else { - let t = t.narrow(dim, offset, usize::min(len, seq_len - offset))?; - Some(t) - } - } - }; - Ok(Self(t)) - } - - /// Splits the Streaming Tensor on the time axis `dim` with the first `lhs_len` elements - /// returned in the first output and the remaining in the second output. - pub fn split(&self, dim: D, lhs_len: usize) -> Result<(Self, Self)> { - match &self.0 { - None => Ok((Self::empty(), Self::empty())), - Some(t) => { - let seq_len = t.dim(dim)?; - let lhs_len = usize::min(seq_len, lhs_len); - if lhs_len == 0 { - Ok((Self::empty(), t.clone().into())) - } else { - let lhs = Self::from_tensor(t.narrow(dim, 0, lhs_len)?); - let rhs_len = seq_len - lhs_len; - let rhs = if rhs_len == 0 { - Self::empty() - } else { - Self::from_tensor(t.narrow(dim, lhs_len, rhs_len)?) - }; - Ok((lhs, rhs)) - } - } - } - } - - pub fn as_option(&self) -> Option<&Tensor> { - self.0.as_ref() - } - - pub fn apply(&self, m: &M) -> Result { - match &self.0 { - None => Ok(Self::empty()), - Some(t) => Ok(Self::from_tensor(t.apply(m)?)), - } - } -} - -/// Streaming modules take as input a stream tensor and return a stream tensor. They may perform -/// some internal buffering so that enough data has been received for the module to be able to -/// perform some operations. -pub trait StreamingModule { - // TODO: Should we also have a flush method? - fn step(&mut self, xs: &StreamTensor) -> Result; - fn reset_state(&mut self); -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub enum BinOp { - Add, - Mul, - Sub, - Div, -} - -#[derive(Debug, Clone)] -pub struct StreamingBinOp { - prev_lhs: StreamTensor, - prev_rhs: StreamTensor, - pub op: BinOp, - pub dim: crate::D, -} - -impl StreamingBinOp { - pub fn new(op: BinOp, dim: crate::D) -> Self { - Self { - prev_lhs: StreamTensor::empty(), - prev_rhs: StreamTensor::empty(), - op, - dim, - } - } - - pub fn reset_state(&mut self) { - self.prev_lhs.reset(); - self.prev_rhs.reset(); - } - - pub fn forward(&self, lhs: &Tensor, rhs: &Tensor) -> Result { - match self.op { - BinOp::Add => Tensor::add(lhs, rhs), - BinOp::Mul => Tensor::mul(lhs, rhs), - BinOp::Sub => Tensor::sub(lhs, rhs), - BinOp::Div => Tensor::div(lhs, rhs), - } - } - - pub fn step(&mut self, lhs: &StreamTensor, rhs: &StreamTensor) -> Result { - let lhs = StreamTensor::cat2(&self.prev_lhs, lhs, self.dim)?; - let rhs = StreamTensor::cat2(&self.prev_rhs, rhs, self.dim)?; - let lhs_len = lhs.seq_len(self.dim)?; - let rhs_len = rhs.seq_len(self.dim)?; - let common_len = usize::min(lhs_len, rhs_len); - let (lhs, prev_lhs) = lhs.split(self.dim, common_len)?; - let (rhs, prev_rhs) = rhs.split(self.dim, common_len)?; - let ys = match (lhs.0, rhs.0) { - (Some(lhs), Some(rhs)) => { - let ys = self.forward(&lhs, &rhs)?; - StreamTensor::from_tensor(ys) - } - (None, None) => StreamTensor::empty(), - (lhs, rhs) => crate::bail!("INTERNAL ERROR inconsistent lhs and rhs {lhs:?} {rhs:?}"), - }; - self.prev_lhs = prev_lhs; - self.prev_rhs = prev_rhs; - Ok(ys) - } -} - -/// Simple wrapper that doesn't do any buffering. -pub struct Map(T); - -impl StreamingModule for Map { - fn reset_state(&mut self) {} - - fn step(&mut self, xs: &StreamTensor) -> Result { - xs.apply(&self.0) - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/strided_index.rs b/v2/dependency-patches/candle-core-0.9.2/src/strided_index.rs deleted file mode 100644 index a31d406a43..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/strided_index.rs +++ /dev/null @@ -1,94 +0,0 @@ -use crate::Layout; - -/// An iterator over offset position for items of an N-dimensional arrays stored in a -/// flat buffer using some potential strides. -#[derive(Debug)] -pub struct StridedIndex<'a> { - next_storage_index: Option, - multi_index: Vec, - dims: &'a [usize], - stride: &'a [usize], - remaining: usize, -} - -impl<'a> StridedIndex<'a> { - pub(crate) fn new(dims: &'a [usize], stride: &'a [usize], start_offset: usize) -> Self { - let elem_count: usize = dims.iter().product(); - let next_storage_index = if elem_count == 0 { - None - } else { - // This applies to the scalar case. - Some(start_offset) - }; - StridedIndex { - next_storage_index, - multi_index: vec![0; dims.len()], - dims, - stride, - remaining: elem_count, - } - } - - pub(crate) fn from_layout(l: &'a Layout) -> Self { - Self::new(l.dims(), l.stride(), l.start_offset()) - } -} - -impl Iterator for StridedIndex<'_> { - type Item = usize; - - #[inline] - fn next(&mut self) -> Option { - let storage_index = self.next_storage_index?; - let mut updated = false; - let mut next_storage_index = storage_index; - for ((multi_i, max_i), stride_i) in self - .multi_index - .iter_mut() - .zip(self.dims.iter()) - .zip(self.stride.iter()) - .rev() - { - let next_i = *multi_i + 1; - if next_i < *max_i { - *multi_i = next_i; - updated = true; - next_storage_index += stride_i; - break; - } else { - next_storage_index -= *multi_i * stride_i; - *multi_i = 0 - } - } - self.remaining -= 1; - self.next_storage_index = if updated { - Some(next_storage_index) - } else { - None - }; - Some(storage_index) - } - - #[inline] - fn size_hint(&self) -> (usize, Option) { - (self.remaining, Some(self.remaining)) - } -} - -impl ExactSizeIterator for StridedIndex<'_> { - fn len(&self) -> usize { - self.remaining - } -} - -#[derive(Debug)] -pub enum StridedBlocks<'a> { - SingleBlock { - start_offset: usize, - len: usize, - }, - MultipleBlocks { - block_start_index: StridedIndex<'a>, - block_len: usize, - }, -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/tensor.rs b/v2/dependency-patches/candle-core-0.9.2/src/tensor.rs deleted file mode 100644 index 52b7f79b77..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/tensor.rs +++ /dev/null @@ -1,3116 +0,0 @@ -//! Tensors are N-dimensional matrixes of elements using a single data type. -#![allow(clippy::redundant_closure_call)] -use crate::backend::{BackendDevice, BackendStorage}; -use crate::op::{BackpropOp, BinaryOp, CmpOp, Op, ReduceOp, UnaryOp}; -use crate::scalar::TensorOrScalar; -use crate::shape::{Dim, Dims, ShapeWithOneHole}; -use crate::{bail, storage::Storage, DType, Device, Error, Layout, Result, Shape}; -use std::sync::{Arc, RwLock}; - -/// Unique identifier for tensors. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] -pub struct TensorId(usize); - -impl TensorId { - fn new() -> Self { - // https://users.rust-lang.org/t/idiomatic-rust-way-to-generate-unique-id/33805 - use std::sync::atomic; - static COUNTER: atomic::AtomicUsize = atomic::AtomicUsize::new(1); - Self(COUNTER.fetch_add(1, atomic::Ordering::Relaxed)) - } -} - -pub struct Tensor_ { - id: TensorId, - // As we provide inner mutability on the tensor content, the alternatives are: - // - Using a mutex, this would have the highest cost when retrieving the storage but would - // prevent errors when concurrent access takes place. Mutex would also be subject to - // deadlocks for example using the current code if the same tensor is used twice by a single - // binary op. - // - Using a refcell unsafe cell would have some intermediary cost, borrow checking would be - // verified dynamically, but the resulting tensors would not be send or sync. - // - Using an unsafe cell would have the lowest cost but undefined behavior on concurrent - // accesses. - // Ideally, we would use Arc for tensors on which we don't plan on modifying the data - // and Arc> for tensors where the data could be modified, e.g. variables but - // that's tricky to encode in the current setup. - storage: Arc>, - layout: Layout, - op: BackpropOp, - is_variable: bool, - dtype: DType, - device: Device, -} - -impl AsRef for Tensor { - fn as_ref(&self) -> &Tensor { - self - } -} - -// Tensors are refcounted so that cloning is cheap when building the op graph. -// Storages are also refcounted independently so that its possible to avoid -// copying the storage for operations that only modify the shape or stride. -#[derive(Clone)] -/// The core struct for manipulating tensors. -/// -/// ```rust -/// use candle_core::{Tensor, DType, Device}; -/// -/// let a = Tensor::arange(0f32, 6f32, &Device::Cpu)?.reshape((2, 3))?; -/// let b = Tensor::arange(0f32, 12f32, &Device::Cpu)?.reshape((3, 4))?; -/// -/// let c = a.matmul(&b)?; -/// # Ok::<(), candle_core::Error>(()) -/// ``` -/// -/// Tensors are reference counted with [`Arc`] so cloning them is cheap. -pub struct Tensor(Arc); - -impl std::ops::Deref for Tensor { - type Target = Tensor_; - - fn deref(&self) -> &Self::Target { - self.0.as_ref() - } -} - -macro_rules! unary_op { - ($fn_name:ident, $op_name:ident) => { - pub fn $fn_name(&self) -> Result { - let shape = self.shape(); - if shape.elem_count() == 0 { - return Ok(self.clone()); - } - let storage = self - .storage() - .unary_impl::(self.layout())?; - let op = BackpropOp::new1(self, |s| Op::Unary(s, UnaryOp::$op_name)); - Ok(from_storage(storage, shape.clone(), op, false)) - } - }; -} - -macro_rules! binary_op { - ($fn_name:ident, $op_name:ident) => { - pub fn $fn_name(&self, rhs: &Self) -> Result { - let shape = self.same_shape_binary_op(rhs, stringify!($fn_name))?; - if shape.elem_count() == 0 { - return Ok(self.clone()); - } - let storage = self.storage().binary_impl::( - &*rhs.storage(), - self.layout(), - rhs.layout(), - )?; - let op = BackpropOp::new2(self, rhs, |t1, t2| Op::Binary(t1, t2, BinaryOp::$op_name)); - Ok(from_storage(storage, shape.clone(), op, false)) - } - }; -} - -macro_rules! binary_op_scalar { - ($fn_name:ident, $op_name:ident) => { - pub fn $fn_name(&self, rhs: T) -> Result { - let rhs = match rhs.to_tensor_scalar()? { - crate::scalar::TensorScalar::Tensor(rhs) => rhs, - crate::scalar::TensorScalar::Scalar(rhs) => rhs - .to_dtype(self.dtype())? - .to_device(self.device())? - .broadcast_as(self.shape())?, - }; - let shape = self.same_shape_binary_op(&rhs, stringify!($fn_name))?; - if self.elem_count() == 0 { - return Ok(self.clone()); - } - let storage = self.storage().binary_impl::( - &*rhs.storage(), - self.layout(), - rhs.layout(), - )?; - let op = BackpropOp::new2(self, &rhs, |t1, t2| Op::Binary(t1, t2, BinaryOp::$op_name)); - Ok(from_storage(storage, shape.clone(), op, false)) - } - }; -} - -macro_rules! broadcast_binary_op { - ($fn_name:ident, $inner_fn_name:ident) => { - pub fn $fn_name(&self, rhs: &Self) -> Result { - let lhs = self; - let shape = lhs - .shape() - .broadcast_shape_binary_op(rhs.shape(), stringify!($fn_name))?; - let l_broadcast = shape != *lhs.shape(); - let r_broadcast = shape != *rhs.shape(); - match (l_broadcast, r_broadcast) { - (true, true) => lhs - .broadcast_as(&shape)? - .$inner_fn_name(&rhs.broadcast_as(&shape)?), - (false, true) => lhs.$inner_fn_name(&rhs.broadcast_as(&shape)?), - (true, false) => lhs.broadcast_as(&shape)?.$inner_fn_name(rhs), - (false, false) => lhs.$inner_fn_name(rhs), - } - } - }; -} - -/// Creates a fresh tensor structure based on a storage and a shape, this uses contiguous strides. -pub(crate) fn from_storage>( - storage: Storage, - shape: S, - op: BackpropOp, - is_variable: bool, -) -> Tensor { - let dtype = storage.dtype(); - let device = storage.device(); - let tensor_ = Tensor_ { - id: TensorId::new(), - storage: Arc::new(RwLock::new(storage)), - layout: Layout::contiguous(shape), - op, - is_variable, - dtype, - device, - }; - Tensor(Arc::new(tensor_)) -} - -impl Tensor { - pub(crate) fn ones_impl>( - shape: S, - dtype: DType, - device: &Device, - is_variable: bool, - ) -> Result { - let none = BackpropOp::none(); - let shape = shape.into(); - let mut storage = unsafe { device.alloc_uninit(&shape, dtype)? }; - let layout = Layout::contiguous(shape.clone()); - storage.const_set(crate::scalar::Scalar::one(dtype), &layout)?; - Ok(from_storage(storage, shape, none, is_variable)) - } - - /// Creates a new tensor filled with ones. - /// - /// ```rust - /// use candle_core::{Tensor, DType, Device}; - /// let a = Tensor::ones((2, 3), DType::F32, &Device::Cpu)?; - /// let b = Tensor::from_slice(&[1.0f32, 1.0, 1.0, 1.0, 1.0, 1.0], (2, 3), &Device::Cpu)?; - /// // a == b - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn ones>(shape: S, dtype: DType, device: &Device) -> Result { - Self::ones_impl(shape, dtype, device, false) - } - - pub fn const_set(&self, value: crate::scalar::Scalar) -> Result<()> { - self.storage_mut().const_set(value, self.layout()) - } - - pub fn zero_set(&self) -> Result<()> { - self.const_set(crate::scalar::Scalar::zero(self.dtype())) - } - - pub fn one_set(&self) -> Result<()> { - self.const_set(crate::scalar::Scalar::one(self.dtype())) - } - - /// Creates a new tensor filled with ones with same shape, dtype, and device as the other tensor. - /// - /// ```rust - /// use candle_core::{Tensor, DType, Device}; - /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; - /// let b = a.ones_like()?; - /// // b == a + 1 - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn ones_like(&self) -> Result { - Tensor::ones(self.shape(), self.dtype(), self.device()) - } - - // Do not expose outside of the crate, the `is_variable=true` case should only be accessed from - // the variable module. - pub(crate) fn zeros_impl>( - shape: S, - dtype: DType, - device: &Device, - is_variable: bool, - ) -> Result { - let none = BackpropOp::none(); - let shape = shape.into(); - let storage = device.zeros(&shape, dtype)?; - Ok(from_storage(storage, shape, none, is_variable)) - } - - /// Creates a new tensor filled with zeros. - /// - /// ```rust - /// use candle_core::{Tensor, DType, Device}; - /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; - /// let b = Tensor::from_slice(&[0.0f32, 0.0, 0.0, 0.0, 0.0, 0.0], (2, 3), &Device::Cpu)?; - /// // a == b - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn zeros>(shape: S, dtype: DType, device: &Device) -> Result { - Self::zeros_impl(shape, dtype, device, false) - } - - /// Creates a new tensor filled with zeros with same shape, dtype, and device as the other - /// tensor. - /// - /// ```rust - /// use candle_core::{Tensor, DType, Device}; - /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; - /// let b = a.zeros_like()?; - /// // b is on CPU f32. - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn zeros_like(&self) -> Result { - Tensor::zeros(self.shape(), self.dtype(), self.device()) - } - - // Do not expose outside of the crate, the `is_variable=true` case should only be accessed from - // the variable module. - pub(crate) unsafe fn empty_impl>( - shape: S, - dtype: DType, - device: &Device, - is_variable: bool, - ) -> Result { - let none = BackpropOp::none(); - let shape = shape.into(); - let storage = device.alloc_uninit(&shape, dtype)?; - Ok(from_storage(storage, shape, none, is_variable)) - } - - /// Creates a new tensor filled with uninitialized memory. - /// - /// # Safety - /// This returns uninitialized memory. - /// - /// ```rust - /// use candle_core::{Tensor, DType, Device}; - /// let a = unsafe { Tensor::empty((2, 3), DType::F32, &Device::Cpu)? }; - /// // a == b - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub unsafe fn empty>(shape: S, dtype: DType, device: &Device) -> Result { - Self::empty_impl(shape, dtype, device, false) - } - - /// Creates a new tensor filled with uninitialized memory of the same shape, dtype, and device as the other - /// tensor. - /// - /// # Safety - /// This returns uninitialized memory. - /// - /// ```rust - /// use candle_core::{Tensor, DType, Device}; - /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; - /// let b = unsafe { a.empty_like()? }; - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub unsafe fn empty_like(&self) -> Result { - Tensor::empty(self.shape(), self.dtype(), self.device()) - } - - pub(crate) fn rand_impl, T: crate::FloatDType>( - lo: T, - up: T, - s: S, - device: &Device, - is_variable: bool, - ) -> Result { - let s = s.into(); - let storage = device.rand_uniform(lo, up, &s)?; - let none = BackpropOp::none(); - Ok(from_storage(storage, s, none, is_variable)) - } - - pub(crate) fn rand_f64_impl>( - lo: f64, - up: f64, - s: S, - dtype: DType, - device: &Device, - is_variable: bool, - ) -> Result { - let s = s.into(); - let storage = device.rand_uniform_f64(lo, up, &s, dtype)?; - let none = BackpropOp::none(); - Ok(from_storage(storage, s, none, is_variable)) - } - - /// Creates a new tensor initialized with values sampled uniformly between `lo` and `up`. - pub fn rand, T: crate::FloatDType>( - lo: T, - up: T, - s: S, - device: &Device, - ) -> Result { - Self::rand_impl(lo, up, s, device, false) - } - - pub fn rand_like(&self, lo: f64, up: f64) -> Result { - Tensor::rand_f64_impl(lo, up, self.shape(), self.dtype(), self.device(), false) - } - - pub(crate) fn randn_impl, T: crate::FloatDType>( - mean: T, - std: T, - s: S, - device: &Device, - is_variable: bool, - ) -> Result { - let s = s.into(); - let storage = device.rand_normal(mean, std, &s)?; - let none = BackpropOp::none(); - Ok(from_storage(storage, s, none, is_variable)) - } - - pub(crate) fn randn_f64_impl>( - mean: f64, - std: f64, - s: S, - dtype: DType, - device: &Device, - is_variable: bool, - ) -> Result { - let s = s.into(); - let storage = device.rand_normal_f64(mean, std, &s, dtype)?; - let none = BackpropOp::none(); - Ok(from_storage(storage, s, none, is_variable)) - } - - pub fn randn_like(&self, mean: f64, stdev: f64) -> Result { - Tensor::randn_f64_impl( - mean, - stdev, - self.shape(), - self.dtype(), - self.device(), - false, - ) - } - - /// Creates a new tensor initialized with values sampled from a normal distribution with the - /// specified `mean` and standard deviation `std`. - pub fn randn, T: crate::FloatDType>( - mean: T, - std: T, - s: S, - device: &Device, - ) -> Result { - Self::randn_impl(mean, std, s, device, false) - } - - pub(crate) fn new_impl( - array: A, - shape: Shape, - device: &Device, - is_variable: bool, - ) -> Result { - let n: usize = shape.elem_count(); - let buffer_size: usize = array.shape()?.elem_count(); - if buffer_size != n { - return Err(Error::ShapeMismatch { buffer_size, shape }.bt()); - } - let storage = device.storage(array)?; - let none = BackpropOp::none(); - Ok(from_storage(storage, shape, none, is_variable)) - } - - /// Creates a new tensor on the specified device using the content and shape of the input. - pub fn new(array: A, device: &Device) -> Result { - let shape = array.shape()?; - Self::new_impl(array, shape, device, false) - } - - /// Returns a new tensor with all the elements having the same specified value. - ///```rust - /// use candle_core::{Tensor, Device}; - /// let a = Tensor::full(3.5, (2, 4), &Device::Cpu)?; - /// - /// assert_eq!(a.to_vec2::()?, &[ - /// [3.5, 3.5, 3.5, 3.5], - /// [3.5, 3.5, 3.5, 3.5], - /// ]); - /// # Ok::<(), candle_core::Error>(()) - pub fn full>( - value: D, - shape: S, - device: &Device, - ) -> Result { - let none = BackpropOp::none(); - let shape = shape.into(); - let mut storage = unsafe { device.alloc_uninit(&shape, D::DTYPE)? }; - let layout = Layout::contiguous(shape.clone()); - storage.const_set(value.to_scalar(), &layout)?; - Ok(from_storage(storage, shape, none, false)) - } - - /// Creates a new 1D tensor from an iterator. - ///```rust - /// use candle_core::{Tensor, Device}; - /// let a = Tensor::from_iter( [1.0, 2.0, 3.0, 4.0].into_iter(), &Device::Cpu)?; - /// - /// assert_eq!(a.to_vec1::()?, &[1.0, 2.0, 3.0, 4.0]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn from_iter( - iter: impl IntoIterator, - device: &Device, - ) -> Result { - let data = iter.into_iter().collect::>(); - let len = data.len(); - Self::from_vec_impl(data, len, device, false) - } - - /// Creates a new 1D tensor with values from the interval `[start, end)` taken with a common - /// difference `1` from `start`. - ///```rust - /// use candle_core::{Tensor, Device}; - /// let a = Tensor::arange(2., 5., &Device::Cpu)?; - /// - /// assert_eq!(a.to_vec1::()?, &[2., 3., 4.]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn arange(start: D, end: D, device: &Device) -> Result { - Self::arange_step(start, end, D::one(), device) - } - - /// Creates a new 1D tensor with values from the interval `[start, end)` taken with a common - /// difference `step` from `start`. - ///```rust - /// use candle_core::{Tensor, Device}; - /// let a = Tensor::arange_step(2.0, 4.0, 0.5, &Device::Cpu)?; - /// - /// assert_eq!(a.to_vec1::()?, &[2.0, 2.5, 3.0, 3.5]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn arange_step( - start: D, - end: D, - step: D, - device: &Device, - ) -> Result { - if D::is_zero(&step) { - bail!("step cannot be zero") - } - let mut data = vec![]; - let mut current = start; - if step >= D::zero() { - while current < end { - data.push(current); - current += step; - } - } else { - while current > end { - data.push(current); - current += step; - } - } - let len = data.len(); - Self::from_vec_impl(data, len, device, false) - } - - pub(crate) fn from_vec_impl( - data: Vec, - shape: S, - device: &Device, - is_variable: bool, - ) -> Result { - let shape = shape.into_shape(data.len())?; - let storage = device.storage_owned(data)?; - let none = BackpropOp::none(); - Ok(from_storage(storage, shape, none, is_variable)) - } - - /// Creates a new tensor initialized with values from the input vector. The number of elements - /// in this vector must be the same as the number of elements defined by the shape. - /// If the device is cpu, no data copy is made. - ///```rust - /// use candle_core::{Tensor, Device}; - /// let a = Tensor::from_vec(vec!{1., 2., 3., 4., 5., 6.}, (2, 3), &Device::Cpu)?; - /// - /// assert_eq!(a.to_vec2::()?, &[ - /// [1., 2., 3.], - /// [4., 5., 6.] - /// ]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn from_vec( - data: Vec, - shape: S, - device: &Device, - ) -> Result { - Self::from_vec_impl(data, shape, device, false) - } - - /// Creates a new tensor initialized with values from the input slice. The number of elements - /// in this vector must be the same as the number of elements defined by the shape. - ///```rust - /// use candle_core::{Tensor, Device}; - /// let values = vec![1., 2., 3., 4., 5., 6., 7., 8.]; - /// let a = Tensor::from_slice(&values[1..7], (2, 3), &Device::Cpu)?; - /// - /// assert_eq!(a.to_vec2::()?, &[ - /// [2., 3., 4.], - /// [5., 6., 7.] - /// ]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn from_slice( - array: &[D], - shape: S, - device: &Device, - ) -> Result { - let shape = shape.into_shape(array.len())?; - let storage = device.storage_from_slice(array)?; - let none = BackpropOp::none(); - Ok(from_storage(storage, shape, none, false)) - } - - pub(crate) fn same_shape_binary_op(&self, rhs: &Self, op: &'static str) -> Result<&Shape> { - let lhs = self.shape(); - let rhs = rhs.shape(); - if lhs != rhs { - Err(Error::ShapeMismatchBinaryOp { - lhs: lhs.clone(), - rhs: rhs.clone(), - op, - } - .bt()) - } else { - Ok(lhs) - } - } - - /// Returns true if the computation graph should track this op, that is if it is - /// a variable or if it has some variable as dependencies. - pub fn track_op(&self) -> bool { - self.is_variable || self.op.is_some() - } - - /// Creates a fresh tensor structure based on a storage and a shape. - /// - /// # Note - /// - This uses contiguous strides - /// - Ensure the shape is compatible with the shape of the storage. - pub fn from_storage>( - storage: Storage, - shape: S, - op: BackpropOp, - is_variable: bool, - ) -> Tensor { - from_storage(storage, shape, op, is_variable) - } - - // TODO: Also make an inplace version or a pre-allocated? This could be tricky - // if this can create cycles in the compute graph. - binary_op!(add, Add); - binary_op!(mul, Mul); - binary_op!(sub, Sub); - binary_op!(div, Div); - binary_op_scalar!(maximum, Maximum); - binary_op_scalar!(minimum, Minimum); - broadcast_binary_op!(broadcast_add, add); - broadcast_binary_op!(broadcast_mul, mul); - broadcast_binary_op!(broadcast_sub, sub); - broadcast_binary_op!(broadcast_div, div); - broadcast_binary_op!(broadcast_maximum, maximum); - broadcast_binary_op!(broadcast_minimum, minimum); - broadcast_binary_op!(broadcast_eq, eq); - broadcast_binary_op!(broadcast_ne, ne); - broadcast_binary_op!(broadcast_lt, lt); - broadcast_binary_op!(broadcast_le, le); - broadcast_binary_op!(broadcast_gt, gt); - broadcast_binary_op!(broadcast_ge, ge); - - unary_op!(recip, Recip); - unary_op!(neg, Neg); - unary_op!(exp, Exp); - unary_op!(log, Log); - unary_op!(sin, Sin); - unary_op!(cos, Cos); - unary_op!(tanh, Tanh); - unary_op!(abs, Abs); - unary_op!(sqr, Sqr); - unary_op!(sqrt, Sqrt); - unary_op!(gelu, Gelu); - unary_op!(gelu_erf, GeluErf); - unary_op!(erf, Erf); - unary_op!(relu, Relu); - unary_op!(silu, Silu); - unary_op!(ceil, Ceil); - unary_op!(floor, Floor); - unary_op!(round, Round); - unary_op!(sign, Sign); - - /// Round element of the input tensor to the nearest integer. - /// - /// If the number of decimals is negative, it specifies the number of positions to the left of - /// the decimal point. - pub fn round_to(&self, decimals: i32) -> Result { - let mult = 10f64.powi(decimals); - (self * mult)?.round()? * (1f64 / mult) - } - - /// Retrieves the single scalar value hold in the tensor. If the tensor contains multiple - /// dimensions, an error is returned instead. - pub fn to_scalar(&self) -> Result { - if self.rank() != 0 { - Err(Error::UnexpectedNumberOfDims { - expected: 0, - got: self.rank(), - shape: self.shape().clone(), - } - .bt())? - } - let from_cpu_storage = |cpu_storage: &crate::CpuStorage| { - let data = S::cpu_storage_as_slice(cpu_storage)?; - Ok::<_, Error>(data[self.layout().start_offset()]) - }; - match &*self.storage() { - Storage::Cpu(cpu_storage) => from_cpu_storage(cpu_storage), - Storage::Cuda(storage) => from_cpu_storage(&storage.to_cpu_storage()?), - Storage::Metal(storage) => from_cpu_storage(&storage.to_cpu_storage()?), - } - } - - /// An alias for `to_scalar`. - pub fn to_vec0(&self) -> Result { - self.to_scalar::() - } - - /// Repeat this tensor along the specified dimensions. - pub fn repeat>(&self, shape: S) -> Result { - // Similar to PyTorch, we extend the number of dimensions of self if needed. - let repeats = shape.into(); - let repeats = repeats.dims(); - let mut inp = if self.rank() < repeats.len() { - let shape = [vec![1; repeats.len() - self.rank()], self.dims().to_vec()].concat(); - self.reshape(shape)? - } else { - self.clone() - }; - for (idx, &repeat) in repeats.iter().enumerate() { - if repeat > 1 { - inp = Tensor::cat(&vec![&inp; repeat], idx)? - } - } - Ok(inp) - } - - /// Creates grids of coordinates specified by the 1D inputs. - /// - /// # Arguments - /// - /// * `args` - A slice of 1D tensors. - /// * `xy_indexing` - Whether to use xy indexing or ij indexing. If xy is selected, the - /// first dimension corresponds to the cardinality of the second input and the second - /// dimension corresponds to the cardinality of the first input. If ij is selected, the - /// dimensions are in the same order as the cardinality of the inputs. - /// - /// # Examples - /// - /// ```rust - /// use candle_core::{Tensor, Device, Shape}; - /// let x = Tensor::new(&[1f32, 2., 3.], &Device::Cpu)?; - /// let y = Tensor::new(&[4f32, 5., 6.], &Device::Cpu)?; - /// - /// let grids_xy = Tensor::meshgrid(&[&x, &y], true)?; - /// - /// assert_eq!(grids_xy.len(), 2); - /// assert_eq!(grids_xy[0].dims(), &[3, 3]); - /// - /// assert_eq!(grids_xy[0].to_vec2::()?, &[[1., 2., 3.], [1., 2., 3.], [1., 2., 3.]]); - /// assert_eq!(grids_xy[1].to_vec2::()?, &[[4., 4., 4.], [5., 5., 5.], [6., 6., 6.]]); - /// - /// let grids_ij = Tensor::meshgrid(&[&x, &y], false)?; - /// - /// assert_eq!(grids_ij[0].to_vec2::()?, &[[1., 1., 1.], [2., 2., 2.], [3., 3., 3.]]); - /// assert_eq!(grids_ij[1].to_vec2::()?, &[[4., 5., 6.], [4., 5., 6.], [4., 5., 6.]]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - /// - /// # Errors - /// - /// * Will return `Err` if `args` contains less than 2 tensors. - /// - pub fn meshgrid>(args: &[A], xy_indexing: bool) -> Result> { - if args.len() <= 1 { - Err(Error::OpRequiresAtLeastTwoTensors { op: "meshgrid" }.bt())? - } - let args: Vec<_> = if xy_indexing { - args.iter().rev().collect() - } else { - args.iter().collect() - }; - - let mut shape = Vec::with_capacity(args.len()); - for arg in args.iter() { - shape.push(arg.as_ref().dims1()?) - } - - let mut grids = Vec::with_capacity(args.len()); - for idx in 0..args.len() { - let mut ones = vec![1usize; args.len()]; - ones[idx] = shape[idx]; - let arg = args[idx].as_ref().reshape(ones)?; - let mut repeats = shape.clone(); - repeats[idx] = 1; - let repeated_tensor = arg.repeat(repeats)?; - grids.push(repeated_tensor); - } - if xy_indexing { - grids.reverse(); - } - Ok(grids) - } - - /// This operation multiplies the input tensor by `mul` then adds `add` and return the result. - /// The input values `mul` and `add` are casted to the appropriate type so some rounding might - /// be performed. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let a = Tensor::new(&[[0f32, 1.], [2., 3.]], &Device::Cpu)?; - /// let a = a.affine(4., -2.)?; - /// assert_eq!(a.to_vec2::()?, &[[-2.0, 2.0], [6.0, 10.0]]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn affine(&self, mul: f64, add: f64) -> Result { - if self.elem_count() == 0 { - return Ok(self.clone()); - } - let storage = self.storage().affine(self.layout(), mul, add)?; - let op = BackpropOp::new1(self, |arg| Op::Affine { arg, mul, add }); - Ok(from_storage(storage, self.shape(), op, false)) - } - - /// Applies the Exponential Linear Unit (ELU) function on each element of the input tensor. - pub fn elu(&self, alpha: f64) -> Result { - if self.elem_count() == 0 { - return Ok(self.clone()); - } - let storage = self.storage().elu(self.layout(), alpha)?; - let op = BackpropOp::new1(self, |t| Op::Elu(t, alpha)); - Ok(from_storage(storage, self.shape(), op, false)) - } - - /// Raise the tensor to some float exponent `e`. - pub fn powf(&self, e: f64) -> Result { - if self.elem_count() == 0 { - return Ok(self.clone()); - } - let storage = self.storage().powf(self.layout(), e)?; - let op = BackpropOp::new1(self, |t| Op::Powf(t, e)); - Ok(from_storage(storage, self.shape(), op, false)) - } - - pub(crate) fn check_dim(&self, dim: usize, op: &'static str) -> Result<()> { - if dim >= self.dims().len() { - Err(Error::DimOutOfRange { - shape: self.shape().clone(), - dim: dim as i32, - op, - } - .bt())? - } else { - Ok(()) - } - } - - /// Split a tensor into the specified number of chunks, this may return less chunks than - /// specified. - pub fn chunk(&self, chunks: usize, dim: D) -> Result> { - let dim = dim.to_index(self.shape(), "chunk")?; - let size = self.dim(dim)?; - if size < chunks { - (0..size).map(|i| self.narrow(dim, i, 1)).collect() - } else { - let chunk_size = size / chunks; - let cnt_additional = size % chunks; - let mut tensors = vec![]; - let mut sum_chunk_size = 0; - for i in 0..chunks { - let chunk_size = if i < cnt_additional { - chunk_size + 1 - } else { - chunk_size - }; - let tensor = self.narrow(dim, sum_chunk_size, chunk_size)?; - tensors.push(tensor); - sum_chunk_size += chunk_size - } - Ok(tensors) - } - } - - /// Returns a new tensor that is a narrowed version of the input, the dimension `dim` - /// ranges from `start` to `start + len`. - /// ``` - /// use candle_core::{Tensor, Device}; - /// let a = Tensor::new(&[ - /// [0f32, 1., 2.], - /// [3. , 4., 5.], - /// [6. , 7., 8.] - /// ], &Device::Cpu)?; - /// - /// let b = a.narrow(0, 1, 2)?; - /// assert_eq!(b.shape().dims(), &[2, 3]); - /// assert_eq!(b.to_vec2::()?, &[ - /// [3., 4., 5.], - /// [6., 7., 8.] - /// ]); - /// - /// let c = a.narrow(1, 1, 1)?; - /// assert_eq!(c.shape().dims(), &[3, 1]); - /// assert_eq!(c.to_vec2::()?, &[ - /// [1.], - /// [4.], - /// [7.] - /// ]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn narrow(&self, dim: D, start: usize, len: usize) -> Result { - let dims = self.dims(); - let dim = dim.to_index(self.shape(), "narrow")?; - let err = |msg| { - Err::<(), _>( - Error::NarrowInvalidArgs { - shape: self.shape().clone(), - dim, - start, - len, - msg, - } - .bt(), - ) - }; - if start > dims[dim] { - err("start > dim_len")? - } - if start.saturating_add(len) > dims[dim] { - err("start + len > dim_len")? - } - if start == 0 && dims[dim] == len { - Ok(self.clone()) - } else { - let op = BackpropOp::new1(self, |t| Op::Narrow(t, dim, start, len)); - let layout = self.layout().narrow(dim, start, len)?; - let tensor_ = Tensor_ { - id: TensorId::new(), - storage: self.storage.clone(), - layout, - op, - is_variable: false, - dtype: self.dtype, - device: self.device.clone(), - }; - Ok(Tensor(Arc::new(tensor_))) - } - } - - fn squeeze_dims(self, dims: &[usize]) -> Result { - match dims { - [] => Ok(self), - [i] => self.squeeze(*i), - dims => { - let dims = self - .dims() - .iter() - .enumerate() - .filter_map(|(dim_idx, &v)| { - if dims.contains(&dim_idx) { - None - } else { - Some(v) - } - }) - .collect::>(); - self.reshape(dims) - } - } - } - - fn reduce_impl(&self, dim: D, keepdim: bool, op: ReduceOp) -> Result { - let dim = dim.to_index(self.shape(), op.name())?; - let storage = self.storage().reduce_op(op, self.layout(), &[dim])?; - let mut dims = self.dims().to_vec(); - dims[dim] = 1; - let op = match op { - ReduceOp::Sum | ReduceOp::Min | ReduceOp::Max => { - BackpropOp::new1(self, |arg| Op::Reduce(arg, op, dims.to_vec())) - } - ReduceOp::ArgMin | ReduceOp::ArgMax => BackpropOp::none(), - }; - let res = from_storage(storage, dims, op, false); - if keepdim { - Ok(res) - } else { - res.squeeze_dims(&[dim]) - } - } - - fn sum_impl(&self, sum_dims: D, keepdim: bool) -> Result { - let sum_dims = sum_dims.to_indexes(self.shape(), "sum")?; - let storage = self - .storage() - .reduce_op(ReduceOp::Sum, self.layout(), &sum_dims)?; - let mut dims = self.dims().to_vec(); - for &sum_dim in sum_dims.iter() { - dims[sum_dim] = 1 - } - let op = BackpropOp::new1(self, |a| Op::Reduce(a, ReduceOp::Sum, dims.to_vec())); - let sum = from_storage(storage, dims, op, false); - if keepdim { - Ok(sum) - } else { - sum.squeeze_dims(&sum_dims) - } - } - - /// Roll the tensor input along the given dimension. - /// Elements that are shifted beyond the last position are re-introduced at the first position. - /// - /// ```rust - /// # use candle_core::{Tensor, Device}; - /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; - /// let tensor = tensor.roll(1, 0)?; - /// assert_eq!(tensor.to_vec2::()?, &[[4., 5.], [0., 1.], [2., 3.]]); - /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; - /// let tensor = tensor.roll(-1, 0)?; - /// assert_eq!(tensor.to_vec2::()?, &[[2., 3.], [4., 5.], [0., 1.]]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn roll(&self, shift: i32, dim: D) -> Result - where - D: Dim + Clone, - { - let dim = dim.to_index(self.shape(), "roll")?; - let dim_size = self.dim(dim)?; - let shift = shift.rem_euclid(dim_size as i32) as usize; - if shift == 0 { - Ok(self.clone()) - } else { - let a = self.narrow(dim, 0, dim_size - shift)?; - let b = self.narrow(dim, dim_size - shift, shift)?; - Tensor::cat(&[&b, &a], dim) - } - } - - /// Returns the sum of all elements in the input tensor. The sum is performed over all the - /// input dimensions. - /// - /// The resulting tensor has a shape that is similar to the shape of the input tensor, except - /// that the number of elements for each dimension index in `sum_dims` is 1. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let a = Tensor::new(&[[0f32, 1.], [2., 3.]], &Device::Cpu)?; - /// let s = a.sum_keepdim(0)?; - /// assert_eq!(s.to_vec2::()?, &[[2., 4.]]); - /// let s = a.sum_keepdim(1)?; - /// assert_eq!(s.to_vec2::()?, &[[1.], [5.]]); - /// let s = a.sum_keepdim((0, 1))?; - /// assert_eq!(s.to_vec2::()?, &[[6.]]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn sum_keepdim(&self, sum_dims: D) -> Result { - self.sum_impl(sum_dims, true) - } - - /// Returns the sum of all elements in the input tensor. The sum is performed over all the - /// input dimensions and compared to `sum_keepdim` these dimensions are squeezed rather than - /// kept. - pub fn sum(&self, sum_dims: D) -> Result { - self.sum_impl(sum_dims, false) - } - - /// Returns the mean of all elements in the input tensor. The mean is performed over all the - /// input dimensions. - /// - /// The resulting tensor has a shape that is similar to the shape of the input tensor, except - /// that the number of elements for each dimension index in `mean_dims` is 1. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let a = Tensor::new(&[[0f32, 1.], [2., 3.]], &Device::Cpu)?; - /// let s = a.mean_keepdim(0)?; - /// assert_eq!(s.to_vec2::()?, &[[1., 2.]]); - /// let s = a.mean_keepdim(1)?; - /// assert_eq!(s.to_vec2::()?, &[[0.5], [2.5]]); - /// let s = a.mean_keepdim((0, 1))?; - /// assert_eq!(s.to_vec2::()?, &[[1.5]]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn mean_keepdim(&self, mean_dims: D) -> Result { - let mean_dims = mean_dims.to_indexes(self.shape(), "mean-keepdim")?; - let reduced_dim: usize = mean_dims.iter().map(|i| self.dims()[*i]).product(); - let scale = 1f64 / (reduced_dim as f64); - self.sum_impl(mean_dims, true)? * scale - } - - /// Returns the mean of all elements in the input tensor. The mean is performed over all the - /// input dimensions and compared to `mean_keepdim` these dimensions are squeezed rather than - /// kept. - pub fn mean(&self, mean_dims: D) -> Result { - let mean_dims = mean_dims.to_indexes(self.shape(), "mean")?; - let reduced_dim: usize = mean_dims.iter().map(|i| self.dims()[*i]).product(); - let scale = 1f64 / (reduced_dim as f64); - self.sum_impl(mean_dims, false)? * scale - } - - /// Returns the unbiased variance over the selected dimension. - pub fn var_keepdim(&self, dim: D) -> Result { - let dim = dim.to_index(self.shape(), "var")?; - let mean = self.mean_keepdim(dim)?; - let squares = self.broadcast_sub(&mean)?.sqr()?; - squares.sum_impl(dim, true)? / (self.dim(dim)? - 1) as f64 - } - - /// Returns the unbiased variance over the selected dimension. - pub fn var(&self, dim: D) -> Result { - let dim = dim.to_index(self.shape(), "var")?; - self.var_keepdim(dim)?.squeeze(dim) - } - - /// Gathers the maximum value across the selected dimension. The resulting shape has the same - /// number of dimensions as the original tensor and the select dimension has a single element. - pub fn max_keepdim(&self, dim: D) -> Result { - self.reduce_impl(dim, true, ReduceOp::Max) - } - - /// Similar to `max_keepdim` but the target dimension is squeezed. - pub fn max(&self, dim: D) -> Result { - self.reduce_impl(dim, false, ReduceOp::Max) - } - - /// Gathers the minimum value across the selected dimension. The resulting shape has the same - /// number of dimensions as the original tensor and the select dimension has a single element. - pub fn min_keepdim(&self, dim: D) -> Result { - self.reduce_impl(dim, true, ReduceOp::Min) - } - - /// Similar to `min_keepdim` but the target dimension is squeezed. - pub fn min(&self, dim: D) -> Result { - self.reduce_impl(dim, false, ReduceOp::Min) - } - - pub fn argmax_keepdim(&self, dim: D) -> Result { - self.reduce_impl(dim, true, ReduceOp::ArgMax) - } - - /// Similar to `argmax_keepdim` but the target dimension is squeezed. - pub fn argmax(&self, dim: D) -> Result { - self.reduce_impl(dim, false, ReduceOp::ArgMax) - } - - pub fn argmin_keepdim(&self, dim: D) -> Result { - self.reduce_impl(dim, true, ReduceOp::ArgMin) - } - - /// Similar to `argmin_keepdim` but the target dimension is squeezed. - pub fn argmin(&self, dim: D) -> Result { - self.reduce_impl(dim, false, ReduceOp::ArgMin) - } - - /// Element-wise comparison between two tensors, e.g. equality, greater than, ... The actual - /// comparison operation is specified by the `op` argument. - /// - /// The returned tensor has the same shape as the original tensors and uses `u8` elements. - pub fn cmp(&self, rhs: T, op: CmpOp) -> Result { - let rhs = match rhs.to_tensor_scalar()? { - crate::scalar::TensorScalar::Tensor(rhs) => rhs, - crate::scalar::TensorScalar::Scalar(rhs) => rhs - .to_dtype(self.dtype())? - .to_device(self.device())? - .broadcast_as(self.shape())?, - }; - let shape = self.same_shape_binary_op(&rhs, "cmp")?; - let storage = self - .storage() - .cmp(op, &rhs.storage(), self.layout(), rhs.layout())?; - let op = BackpropOp::new1(self, |a| Op::Cmp(a, op)); - Ok(from_storage(storage, shape.dims(), op, false)) - } - - /// Element-wise equality. - pub fn eq(&self, rhs: T) -> Result { - self.cmp(rhs, CmpOp::Eq) - } - - /// Element-wise non-equality. - pub fn ne(&self, rhs: T) -> Result { - self.cmp(rhs, CmpOp::Ne) - } - - /// Element-wise comparison with lower-than, the returned tensor uses value 1 where `self < - /// rhs` and 0 otherwise. - pub fn lt(&self, rhs: T) -> Result { - self.cmp(rhs, CmpOp::Lt) - } - - /// Element-wise comparison with greater-than, the returned tensor uses value 1 where `self > - /// rhs` and 0 otherwise. - pub fn gt(&self, rhs: T) -> Result { - self.cmp(rhs, CmpOp::Gt) - } - - /// Element-wise comparison with greater-equal, the returned tensor uses value 1 where `self >= - /// rhs` and 0 otherwise. - pub fn ge(&self, rhs: T) -> Result { - self.cmp(rhs, CmpOp::Ge) - } - - /// Element-wise comparison with lower-equal, the returned tensor uses value 1 where `self <= - /// rhs` and 0 otherwise. - pub fn le(&self, rhs: T) -> Result { - self.cmp(rhs, CmpOp::Le) - } - - /// Clamp the tensor values to be between `min` and `max`. - pub fn clamp(&self, min: T1, max: T2) -> Result { - self.maximum(min)?.minimum(max) - } - - /// Interpolate the input tensor to the `target_size` size, taking the value of the nearest element. - /// - /// The input tensor should have three dimensions, `(batch, channels, l)`, the returned - /// tensor also has three dimensions, `(batch, channels, target_size)`. - pub fn interpolate1d(&self, target_size: usize) -> Result { - let (n, c, _l) = self.dims3()?; - let op = BackpropOp::new1(self, |arg| Op::UpsampleNearest1D { arg, target_size }); - let storage = self - .storage() - .upsample_nearest1d(self.layout(), target_size)?; - Ok(from_storage(storage, (n, c, target_size), op, false)) - } - - /// Alias for `interpolate1d`. - pub fn upsample_nearest1d(&self, target_size: usize) -> Result { - self.interpolate1d(target_size) - } - - /// Interpolate the input tensor to the `(target_h, target_w)` size, taking the value of the - /// nearest element. - /// - /// The input tensor should have four dimensions, `(batch, channels, h, w)`, the returned - /// tensor also has four dimensions, `(batch, channels, target_h, target_w)`. - pub fn interpolate2d(&self, target_h: usize, target_w: usize) -> Result { - let (n, c, _h, _w) = self.dims4()?; - let op = BackpropOp::new1(self, |arg| Op::UpsampleNearest2D { - arg, - target_h, - target_w, - }); - let storage = self - .storage() - .upsample_nearest2d(self.layout(), target_h, target_w)?; - Ok(from_storage(storage, (n, c, target_h, target_w), op, false)) - } - - /// Alias for `interpolate2d`. - pub fn upsample_nearest2d(&self, target_h: usize, target_w: usize) -> Result { - self.interpolate2d(target_h, target_w) - } - - /// Bilinear interpolation to resize the input tensor to the specified size. - /// - /// The input tensor should have four dimensions: `(batch, channels, h, w)`. - /// The returned tensor also has four dimensions: `(batch, channels, target_h, target_w)`. - /// - /// # Arguments - /// - /// * `target_h` - Target height - /// * `target_w` - Target width - /// * `align_corners` - If true, corner pixels are aligned. If false (default), - /// pixels are treated as areas (matches PyTorch default behavior). - /// - /// # Example - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// # fn main() -> candle_core::Result<()> { - /// let t = Tensor::arange(0f32, 16f32, &Device::Cpu)?.reshape((1, 1, 4, 4))?; - /// let upsampled = t.upsample_bilinear2d(8, 8, false)?; - /// assert_eq!(upsampled.dims(), &[1, 1, 8, 8]); - /// # Ok(()) - /// # } - /// ``` - pub fn upsample_bilinear2d( - &self, - target_h: usize, - target_w: usize, - align_corners: bool, - ) -> Result { - let (n, c, _h, _w) = self.dims4()?; - let op = BackpropOp::new1(self, |arg| Op::UpsampleBilinear2D { - arg, - target_h, - target_w, - align_corners, - }); - // Pass None for scale factors (size mode) - let storage = self.storage().upsample_bilinear2d( - self.layout(), - target_h, - target_w, - align_corners, - None, - None, - )?; - Ok(from_storage(storage, (n, c, target_h, target_w), op, false)) - } - - /// Bilinear interpolation using scale factors. - /// - /// Similar to `upsample_bilinear2d` but uses scale factors instead of absolute sizes. - /// This matches PyTorch's `interpolate(scale_factor=...)` behavior. - /// - /// # Arguments - /// - /// * `scale_h` - Height scaling factor - /// * `scale_w` - Width scaling factor - /// * `align_corners` - If true, corner pixels are aligned - /// - /// # Example - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// # fn main() -> candle_core::Result<()> { - /// let t = Tensor::arange(0f32, 16f32, &Device::Cpu)?.reshape((1, 1, 4, 4))?; - /// // Scale by 2x in both dimensions - /// let upsampled = t.upsample_bilinear2d_with_scale(2.0, 2.0, false)?; - /// assert_eq!(upsampled.dims(), &[1, 1, 8, 8]); - /// # Ok(()) - /// # } - /// ``` - pub fn upsample_bilinear2d_with_scale( - &self, - scale_h: f64, - scale_w: f64, - align_corners: bool, - ) -> Result { - let (n, c, height_in, width_in) = self.dims4()?; - - // Calculate output size (floor, matching PyTorch) - let height_out = (height_in as f64 * scale_h).floor() as usize; - let width_out = (width_in as f64 * scale_w).floor() as usize; - - // Early return if size unchanged - if height_in == height_out && width_in == width_out { - return Ok(self.clone()); - } - - let op = BackpropOp::new1(self, |arg| Op::UpsampleBilinear2D { - arg, - target_h: height_out, - target_w: width_out, - align_corners, - }); - - // Pass original scale factors (scale_factor mode) - // This ensures PyTorch-compatible scale calculation - let storage = self.storage().upsample_bilinear2d( - self.layout(), - height_out, - width_out, - align_corners, - Some(scale_h), - Some(scale_w), - )?; - Ok(from_storage( - storage, - (n, c, height_out, width_out), - op, - false, - )) - } - - /// 2D average pooling over an input tensor with multiple channels. - /// - /// The input tensor should have four dimensions, `(batch, channels, h, w)`, the returned - /// tensor also has four dimensions, `(batch, channels, h', w')`. The pooling is performed on - /// the two last dimensions using a kernel of size `sz`. The returned element is the average - /// value over the kernel window. - pub fn avg_pool2d(&self, sz: T) -> Result { - let sz = sz.to_usize2(); - self.avg_pool2d_with_stride(sz, sz) - } - - /// Same as `avg_pool2d` but with a `stride` that can be set to a value different from the - /// kernel size. - pub fn avg_pool2d_with_stride( - &self, - kernel_size: T, - stride: T, - ) -> Result { - let kernel_size = kernel_size.to_usize2(); - let stride = stride.to_usize2(); - let (n, c, h, w) = self.dims4()?; - if h < kernel_size.0 || w < kernel_size.1 { - bail!("kernel-size {kernel_size:?} is larger than the input size {h},{w}") - } - // https://pytorch.org/docs/stable/generated/torch.nn.AvgPool2d.html#torch.nn.AvgPool2d - let h_out = (h - kernel_size.0) / stride.0 + 1; - let w_out = (w - kernel_size.1) / stride.1 + 1; - let op = BackpropOp::new1(self, |arg| Op::AvgPool2D { - arg, - kernel_size, - stride, - }); - let storage = self - .storage() - .avg_pool2d(self.layout(), kernel_size, stride)?; - Ok(from_storage(storage, (n, c, h_out, w_out), op, false)) - } - - /// 2D max pooling over an input tensor with multiple channels. - /// - /// The input tensor should have four dimensions, `(batch, channels, h, w)`, the returned - /// tensor also has four dimensions, `(batch, channels, h', w')`. The pooling is performed on - /// the two last dimensions using a kernel of size `sz`, the returned element is the maximum - /// value over the kernel window. - pub fn max_pool2d(&self, sz: T) -> Result { - let sz = sz.to_usize2(); - self.max_pool2d_with_stride(sz, sz) - } - - /// Same as `max_pool2d` but with a `stride` that can be set to a value different from the - /// kernel size. - pub fn max_pool2d_with_stride( - &self, - kernel_size: T, - stride: T, - ) -> Result { - let kernel_size = kernel_size.to_usize2(); - let stride = stride.to_usize2(); - let (n, c, h, w) = self.dims4()?; - if h < kernel_size.0 || w < kernel_size.1 { - bail!("kernel-size {kernel_size:?} is larger than the input size {h},{w}") - } - // https://pytorch.org/docs/stable/generated/torch.nn.MaxPool2d.html#torch.nn.MaxPool2d - let h_out = (h - kernel_size.0) / stride.0 + 1; - let w_out = (w - kernel_size.1) / stride.1 + 1; - let op = BackpropOp::new1(self, |arg| Op::MaxPool2D { - arg, - kernel_size, - stride, - }); - let storage = self - .storage() - .max_pool2d(self.layout(), kernel_size, stride)?; - Ok(from_storage(storage, (n, c, h_out, w_out), op, false)) - } - - /// Computes the dot product of two 1D tensors. - /// - /// - If inputs are 1D vectors (`[n]`), returns their scalar dot product. - /// - Panics if shapes are not compatible - /// - Not supported for integer dtypes - /// - /// # Example (vectors) - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let t1 = Tensor::new(&[1.0, 2.0, 3.0], &Device::Cpu)?; - /// let t2 = Tensor::new(&[4.0, 5.0, 6.0], &Device::Cpu)?; - /// let res = t1.dot(&t2)?; - /// assert_eq!(res.to_scalar::()?, 32.); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn dot(&self, rhs: &Self) -> Result { - if self.dims().len() != 1 || rhs.dims().len() != 1 { - return Err(Error::ShapeMismatchBinaryOp { - lhs: self.shape().clone(), - rhs: rhs.shape().clone(), - op: "dot", - }); - } - - (self * rhs).and_then(|ret| ret.sum_all()) - } - - /// Computes the **Frobenius norm** (L2 norm of all elements) of the tensor. - /// - Output is `sqrt(sum(x^2))`. - /// - Always returns a scalar (`[]` shape). - /// - /// # Example - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let t = Tensor::new(&[[3., 4.], [0., 0.]], &Device::Cpu)?; - /// let norm = t.norm()?; - /// assert_eq!(norm.to_scalar::()?, 5.); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn norm(&self) -> Result { - if self.dtype().is_int() { - bail!("norm not supported for integer dtypes"); - } - - self.sqr().and_then(|x| x.sum_all()).and_then(|x| x.sqrt()) - } - - /// Performs strict matrix-vector multiplication (`[m, n] * [n] = [m]`). - /// - /// - If `self` is a matrix (`[m, n]`) and `rhs` is a vector (`[n]`), returns a vector (`[m]`). - /// - **No broadcasting**: Panics if `self` is not 2D or if `rhs` is not 1D with matching size. - /// - /// # Example - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let mat = Tensor::new(&[[1., 2., 3.], [4., 5., 6.]], &Device::Cpu)?; - /// let vec = Tensor::new(&[1., 1., 1.], &Device::Cpu)?; - /// let res = mat.mv(&vec)?; - /// assert_eq!(res.to_vec1::()?, [6., 15.]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn mv(&self, rhs: &Self) -> Result { - // Strict shape checks - let lhs_dims = self.dims(); - let rhs_dims = rhs.dims(); - if lhs_dims.len() != 2 || rhs_dims.len() != 1 || lhs_dims[1] != rhs_dims[0] { - return Err(Error::ShapeMismatchBinaryOp { - lhs: self.shape().clone(), - rhs: rhs.shape().clone(), - op: "mv", - }); - } - - // Direct matmul after ensuring rhs is column vector - self.matmul(&rhs.unsqueeze(1)?)?.squeeze(1) - } - - /// Returns the matrix-multiplication of the input tensor with the other provided tensor. - /// - /// # Arguments - /// - /// * `self` - A tensor with dimensions `b1, b2, ..., bi, m, k`. - /// * `rhs` - A tensor with dimensions `b1, b2, ..., bi, k, n`. - /// - /// The resulting tensor has dimensions `b1, b2, ..., bi, m, n`. - pub fn matmul(&self, rhs: &Self) -> Result { - let a_dims = self.shape().dims(); - let b_dims = rhs.shape().dims(); - - let dim = a_dims.len(); - - if dim < 2 || b_dims.len() != dim { - Err(Error::ShapeMismatchBinaryOp { - lhs: self.shape().clone(), - rhs: rhs.shape().clone(), - op: "matmul", - } - .bt())? - } - - let m = a_dims[dim - 2]; - let k = a_dims[dim - 1]; - let k2 = b_dims[dim - 2]; - let n = b_dims[dim - 1]; - - let c_shape = Shape::from(&a_dims[..dim - 2]).extend(&[m, n]); - if c_shape.elem_count() == 0 || k == 0 { - return Tensor::zeros(c_shape, self.dtype(), self.device()); - } - let batching: usize = a_dims[..dim - 2].iter().product(); - let batching_b: usize = b_dims[..dim - 2].iter().product(); - if k != k2 || batching != batching_b { - Err(Error::ShapeMismatchBinaryOp { - lhs: self.shape().clone(), - rhs: rhs.shape().clone(), - op: "matmul", - } - .bt())? - } - - let storage = self.storage().matmul( - &rhs.storage(), - (batching, m, n, k), - self.layout(), - rhs.layout(), - )?; - let op = BackpropOp::new2(self, rhs, Op::Matmul); - Ok(from_storage(storage, c_shape, op, false)) - } - - /// Matrix-multiplication with broadcasting support. - /// - /// Compared to `matmul` the two matrixes are allowed to have different dimensions as long as - /// they are compatible for broadcast. E.g. if `self` has shape `(j, 1, n, k)` and `rhs` has - /// shape `(l, k, m)`, the output will have shape `(j, l, n, m)`. - pub fn broadcast_matmul(&self, rhs: &Self) -> Result { - let lhs = self; - let (l_shape, r_shape) = lhs.shape().broadcast_shape_matmul(rhs.shape())?; - let l_broadcast = l_shape != *lhs.shape(); - let r_broadcast = r_shape != *rhs.shape(); - // TODO: Avoid concretising the broadcasted matrixes via contiguous. - match (l_broadcast, r_broadcast) { - (true, true) => lhs - .broadcast_as(&l_shape)? - .contiguous()? - .matmul(&rhs.broadcast_as(&r_shape)?.contiguous()?), - (false, true) => lhs.matmul(&rhs.broadcast_as(&r_shape)?.contiguous()?), - (true, false) => lhs.broadcast_as(&l_shape)?.contiguous()?.matmul(rhs), - (false, false) => lhs.matmul(rhs), - } - } - - /// Returns a tensor with the same shape as the input tensor, the values are taken from - /// `on_true` if the input tensor value is not zero, and `on_false` at the positions where the - /// input tensor is equal to zero. - pub fn where_cond(&self, on_true: &Self, on_false: &Self) -> Result { - let _shap = self.same_shape_binary_op(on_true, "where_cond")?; - let shape = self.same_shape_binary_op(on_false, "where_cond")?; - let storage = self.storage().where_cond( - self.layout(), - &on_true.storage(), - on_true.layout(), - &on_false.storage(), - on_false.layout(), - )?; - let op = BackpropOp::new3(self, on_true, on_false, Op::WhereCond); - Ok(from_storage(storage, shape, op, false)) - } - - /// Returns a tensor with the values from the `self` tensor at the index corresponding to the - /// values hold in the `ids` tensor. - /// - /// # Arguments - /// - /// * `self` - A tensor with dimensions `v, h`. - /// * `ids` - A tensor with dimensions `s` and with integer values between 0 and v (exclusive). - /// - /// The resulting tensor has dimensions `s, h`. `s` is called the sequence length, `v` the - /// vocabulary size, and `h` the hidden size. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let values = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; - /// let ids = Tensor::new(&[2u32, 1u32, 2u32], &Device::Cpu)?; - /// let emb = values.embedding(&ids)?; - /// assert_eq!(emb.to_vec2::()?, &[[4., 5.], [2., 3.], [4., 5.]]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn embedding(&self, ids: &Self) -> Result { - if self.rank() != 2 || ids.rank() != 1 { - Err(Error::ShapeMismatchBinaryOp { - lhs: self.shape().clone(), - rhs: ids.shape().clone(), - op: "embedding", - } - .bt())? - } - self.index_select(ids, 0) - } - - fn scatter_checks(&self, indexes: &Self, source: &Self, dim: usize) -> Result<()> { - let source_dims = source.dims(); - let self_dims = self.dims(); - let mismatch = if source_dims.len() != self_dims.len() { - true - } else { - let mut mismatch = false; - for (i, (&d1, &d2)) in self_dims.iter().zip(source_dims.iter()).enumerate() { - if i != dim && d1 != d2 { - mismatch = true; - break; - } - } - mismatch - }; - if mismatch { - Err(Error::ShapeMismatchBinaryOp { - op: "scatter (self, src)", - lhs: self.shape().clone(), - rhs: source.shape().clone(), - } - .bt())? - } - if indexes.dims() != source.dims() { - Err(Error::ShapeMismatchBinaryOp { - op: "scatter (indexes, src)", - lhs: indexes.shape().clone(), - rhs: source.shape().clone(), - } - .bt())? - } - Ok(()) - } - - pub fn scatter(&self, indexes: &Self, source: &Self, dim: D) -> Result { - let dim = dim.to_index(self.shape(), "scatter")?; - self.scatter_checks(indexes, source, dim)?; - let shape = self.shape(); - let mut storage = unsafe { self.device().alloc_uninit(shape, self.dtype())? }; - self.storage() - .copy_strided_src(&mut storage, 0, self.layout())?; - let layout = Layout::contiguous(shape); - storage.scatter_set( - &layout, - &indexes.storage(), - indexes.layout(), - &source.storage(), - source.layout(), - dim, - )?; - let op = BackpropOp::new3(self, indexes, source, |t1, t2, t3| { - Op::Scatter(t1, t2, t3, dim) - }); - Ok(from_storage(storage, self.shape(), op, false)) - } - - pub fn scatter_set(&self, indexes: &Self, source: &Self, dim: D) -> Result<()> { - if self.same_storage(source) { - crate::bail!("cannot use slice_set when self and src share their storage") - } - let dim = dim.to_index(self.shape(), "scatter-set")?; - self.scatter_checks(indexes, source, dim)?; - self.storage_mut().scatter_set( - self.layout(), - &indexes.storage(), - indexes.layout(), - &source.storage(), - source.layout(), - dim, - )?; - Ok(()) - } - - pub fn scatter_add(&self, indexes: &Self, source: &Self, dim: D) -> Result { - let dim = dim.to_index(self.shape(), "scatter-add")?; - self.scatter_checks(indexes, source, dim)?; - let shape = self.shape(); - let mut storage = unsafe { self.device().alloc_uninit(shape, self.dtype())? }; - self.storage() - .copy_strided_src(&mut storage, 0, self.layout())?; - let layout = Layout::contiguous(shape); - storage.scatter_add( - &layout, - &indexes.storage(), - indexes.layout(), - &source.storage(), - source.layout(), - dim, - )?; - let op = BackpropOp::new3(self, indexes, source, |t1, t2, t3| { - Op::ScatterAdd(t1, t2, t3, dim) - }); - Ok(from_storage(storage, self.shape(), op, false)) - } - - pub fn scatter_add_set(&self, indexes: &Self, source: &Self, dim: D) -> Result<()> { - if self.same_storage(source) { - crate::bail!("cannot use slice_set when self and src share their storage") - } - let dim = dim.to_index(self.shape(), "scatter-add-set")?; - self.scatter_checks(indexes, source, dim)?; - self.storage_mut().scatter_add( - self.layout(), - &indexes.storage(), - indexes.layout(), - &source.storage(), - source.layout(), - dim, - )?; - Ok(()) - } - - /// Embeds the values of the `src` tensor into the `self` tensor on the specified dimension. - pub fn slice_scatter(&self, src: &Self, dim: D, start: usize) -> Result { - let dim = dim.to_index(self.shape(), "slice-scatter")?; - if dim == 0 { - self.slice_scatter0(src, start) - } else { - // TODO: Maybe we want to add a more efficient implementation at some point. - self.transpose(0, dim)? - .slice_scatter0(&src.transpose(0, dim)?, start)? - .transpose(0, dim) - } - } - - /// Embeds the values of the `src` tensor into the `self` tensor on the first dimension. - pub fn slice_scatter0(&self, src: &Self, start: usize) -> Result { - if self.dtype() != src.dtype() { - Err(Error::DTypeMismatchBinaryOp { - lhs: self.dtype(), - rhs: src.dtype(), - op: "slice-scatter", - } - .bt())? - } - if self.device().location() != src.device.location() { - Err(Error::DeviceMismatchBinaryOp { - lhs: self.device().location(), - rhs: src.device().location(), - op: "slice-scatter", - } - .bt())? - } - if self.rank() != src.rank() { - Err(Error::UnexpectedNumberOfDims { - expected: self.rank(), - got: src.rank(), - shape: src.shape().clone(), - } - .bt())? - } - let shape_ok = - self.dims() - .iter() - .zip(src.dims().iter()) - .enumerate() - .all(|(dim_idx, (&d1, &d2))| { - if 0 == dim_idx { - d2 + start <= d1 - } else { - d1 == d2 - } - }); - if !shape_ok { - Err(Error::ShapeMismatchBinaryOp { - op: "slice-scatter (self, src)", - lhs: self.shape().clone(), - rhs: src.shape().clone(), - } - .bt())? - } - let mut storage = unsafe { self.device().alloc_uninit(self.shape(), self.dtype())? }; - self.storage() - .copy_strided_src(&mut storage, 0, self.layout())?; - let offset = start * src.dims()[1..].iter().product::(); - src.storage() - .copy_strided_src(&mut storage, offset, src.layout())?; - let op = BackpropOp::new2(self, src, |t1, t2| Op::SliceScatter0(t1, t2, start)); - Ok(from_storage(storage, self.shape(), op, false)) - } - - /// Accumulate element from `source` at indexes `indexes` and add them to `self`. - pub fn index_add(&self, indexes: &Self, source: &Self, dim: D) -> Result { - let dim = dim.to_index(self.shape(), "index-add")?; - let source_dims = source.dims(); - let self_dims = self.dims(); - let mismatch = if source_dims.len() != self_dims.len() { - true - } else { - let mut mismatch = false; - for (i, (&d1, &d2)) in self_dims.iter().zip(source_dims.iter()).enumerate() { - if i != dim && d1 != d2 { - mismatch = true; - break; - } - } - mismatch - }; - if mismatch { - Err(Error::ShapeMismatchBinaryOp { - op: "index-add (self, source)", - lhs: self.shape().clone(), - rhs: source.shape().clone(), - } - .bt())? - } - // The number of element in indexes must match the dimension on which the add is - // performed on the source tensor (and the index values from `indexes` are taken from - // the target tensor self) - let indexes_len = indexes.dims1()?; - if source_dims[dim] != indexes_len { - Err(Error::ShapeMismatchBinaryOp { - op: "index-add (ids, source))", - lhs: indexes.shape().clone(), - rhs: source.shape().clone(), - } - .bt())? - } - let storage = self.storage().index_add( - self.layout(), - &indexes.storage(), - indexes.layout(), - &source.storage(), - source.layout(), - dim, - )?; - let op = BackpropOp::new3(self, indexes, source, |t1, t2, t3| { - Op::IndexAdd(t1, t2, t3, dim) - }); - Ok(from_storage(storage, self.shape(), op, false)) - } - - /// Gather values across the target dimension. - /// - /// # Arguments - /// - /// * `self` - The input tensor. - /// * `indexes` - The indices of elements to gather, this should have same number of dimensions as `self` - /// and indexes.dims()[d] <= self.dims()[d] for all dimensions d != dim - /// * `dim` - the target dimension. - /// - /// The resulting tensor has the same shape as `indexes` and use values from `self` indexed on - /// dimension `dim` by the values in `indexes`. - pub fn gather(&self, indexes: &Self, dim: D) -> Result { - let dim = dim.to_index(self.shape(), "gather")?; - - let self_dims = self.dims(); - let indexes_dims = indexes.dims(); - let mismatch = if indexes_dims.len() != self_dims.len() { - true - } else { - let mut mismatch = false; - for (i, (&d1, &d2)) in self_dims.iter().zip(indexes_dims.iter()).enumerate() { - if i != dim && d1 < d2 { - mismatch = true; - break; - } - } - mismatch - }; - if mismatch { - Err(Error::ShapeMismatchBinaryOp { - op: "gather", - lhs: self.shape().clone(), - rhs: indexes.shape().clone(), - } - .bt())? - } - let storage = - self.storage() - .gather(self.layout(), &indexes.storage(), indexes.layout(), dim)?; - let op = BackpropOp::new2(self, indexes, |t1, t2| Op::Gather(t1, t2, dim)); - Ok(from_storage(storage, indexes.shape(), op, false)) - } - - /// Select values for the input tensor at the target indexes across the specified dimension. - /// - /// The `indexes` is argument is an int tensor with a single dimension. - /// The output has the same number of dimension as the `self` input. The target dimension of - /// the output has length the length of `indexes` and the values are taken from `self` using - /// the index from `indexes`. Other dimensions have the same number of elements as the input - /// tensor. - pub fn index_select(&self, indexes: &Self, dim: D) -> Result { - let dim = dim.to_index(self.shape(), "index-select")?; - let indexes_len = match indexes.dims() { - [l] => *l, - _ => Err(Error::ShapeMismatchBinaryOp { - lhs: self.shape().clone(), - rhs: indexes.shape().clone(), - op: "index-select", - } - .bt())?, - }; - let storage = self.storage().index_select( - &indexes.storage(), - self.layout(), - indexes.layout(), - dim, - )?; - let mut dims = self.dims().to_vec(); - dims[dim] = indexes_len; - let op = BackpropOp::new2(self, indexes, |t1, t2| Op::IndexSelect(t1, t2, dim)); - Ok(from_storage(storage, dims, op, false)) - } - - /// Returns an iterator over position of the elements in the storage when ranging over the - /// index tuples in lexicographic order. - pub fn strided_index(&self) -> crate::StridedIndex<'_> { - self.layout.strided_index() - } - - /// Similar to `strided_index` but returns the position of the start of each contiguous block - /// as well as the length of the contiguous blocks. For a contiguous tensor, the index iterator - /// will only return the start offset and the size would be the number of elements in the - /// tensor. - pub fn strided_blocks(&self) -> crate::StridedBlocks<'_> { - self.layout.strided_blocks() - } - - /// Returns the data contained in a 1D tensor as a vector of scalar values. - pub fn to_vec1(&self) -> Result> { - if self.rank() != 1 { - Err(Error::UnexpectedNumberOfDims { - expected: 1, - got: self.rank(), - shape: self.shape().clone(), - } - .bt())? - } - let from_cpu_storage = |cpu_storage: &crate::CpuStorage| { - let data = S::cpu_storage_as_slice(cpu_storage)?; - let data = match self.layout.contiguous_offsets() { - Some((o1, o2)) => data[o1..o2].to_vec(), - None => self.strided_index().map(|i| data[i]).collect(), - }; - Ok::, Error>(data) - }; - match &*self.storage() { - Storage::Cpu(storage) => from_cpu_storage(storage), - Storage::Cuda(storage) => from_cpu_storage(&storage.to_cpu_storage()?), - Storage::Metal(storage) => from_cpu_storage(&storage.to_cpu_storage()?), - } - } - - /// Returns the data contained in a 2D tensor as a vector of vector of scalar values. - pub fn to_vec2(&self) -> Result>> { - let (dim1, dim2) = self.dims2()?; - let from_cpu_storage = |cpu_storage: &crate::CpuStorage| { - let data = S::cpu_storage_as_slice(cpu_storage)?; - let mut rows = vec![]; - match self.layout.contiguous_offsets() { - Some((o1, o2)) => { - let data = &data[o1..o2]; - for idx_row in 0..dim1 { - rows.push(data[idx_row * dim2..(idx_row + 1) * dim2].to_vec()) - } - } - None => { - let mut src_index = self.strided_index(); - for _idx_row in 0..dim1 { - let row = (0..dim2).map(|_| data[src_index.next().unwrap()]).collect(); - rows.push(row) - } - assert!(src_index.next().is_none()); - } - } - Ok(rows) - }; - match &*self.storage() { - Storage::Cpu(storage) => from_cpu_storage(storage), - Storage::Cuda(storage) => from_cpu_storage(&storage.to_cpu_storage()?), - Storage::Metal(storage) => from_cpu_storage(&storage.to_cpu_storage()?), - } - } - - /// Returns the data contained in a 3D tensor. - pub fn to_vec3(&self) -> Result>>> { - let (dim1, dim2, dim3) = self.dims3()?; - let from_cpu_storage = |cpu_storage: &crate::CpuStorage| { - let data = S::cpu_storage_as_slice(cpu_storage)?; - let mut top_rows = vec![]; - match self.layout.contiguous_offsets() { - Some((o1, o2)) => { - let data = &data[o1..o2]; - let dim23 = dim2 * dim3; - for idx1 in 0..dim1 { - let data = &data[idx1 * dim23..(idx1 + 1) * dim23]; - let mut rows = vec![]; - for idx2 in 0..dim2 { - rows.push(data[idx2 * dim3..(idx2 + 1) * dim3].to_vec()) - } - top_rows.push(rows); - } - } - None => { - let mut src_index = self.strided_index(); - for _idx in 0..dim1 { - let mut rows = vec![]; - for _jdx in 0..dim2 { - let row = (0..dim3).map(|_| data[src_index.next().unwrap()]).collect(); - rows.push(row) - } - top_rows.push(rows); - } - assert!(src_index.next().is_none()); - } - } - Ok(top_rows) - }; - match &*self.storage() { - Storage::Cpu(storage) => from_cpu_storage(storage), - Storage::Cuda(storage) => from_cpu_storage(&storage.to_cpu_storage()?), - Storage::Metal(storage) => from_cpu_storage(&storage.to_cpu_storage()?), - } - } - - /// The dtype for the elements stored in the input tensor. - pub fn dtype(&self) -> DType { - self.dtype - } - - /// The device on which the input tensor is located. - pub fn device(&self) -> &Device { - &self.device - } - - /// The tensor shape, i.e. dimension sizes on each axis. - pub fn shape(&self) -> &Shape { - self.layout().shape() - } - - /// The dimension size for this tensor on each axis. - pub fn dims(&self) -> &[usize] { - self.shape().dims() - } - - /// The dimension size for a specified dimension index. - pub fn dim(&self, dim: D) -> Result { - let dim = dim.to_index(self.shape(), "dim")?; - Ok(self.dims()[dim]) - } - - /// The layout of the input tensor, this stores both the shape of the tensor as well as the - /// strides and the start offset to apply to the underlying storage. - pub fn layout(&self) -> &Layout { - &self.layout - } - - pub fn stride(&self) -> &[usize] { - self.layout.stride() - } - - /// The number of dimensions for this tensor, 0 for a scalar tensor, 1 for a 1D tensor, etc. - pub fn rank(&self) -> usize { - self.shape().rank() - } - - /// The number of elements stored in this tensor. - pub fn elem_count(&self) -> usize { - self.shape().elem_count() - } - - /// The unique identifier for this tensor. - pub fn id(&self) -> TensorId { - self.id - } - - /// Whether this tensor is a variable or not. A variable is a tensor for which gradient is - /// tracked and on which backpropagation can be performed. - pub fn is_variable(&self) -> bool { - self.is_variable - } - - pub(crate) fn op(&self) -> &Option { - &self.op - } - - /// Computes the max of all the elements in this tensor and returns a tensor holding this - /// scalar with zero dimensions. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; - /// let tensor = tensor.max_all()?; - /// assert_eq!(tensor.to_scalar::()?, 5.); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn max_all(&self) -> Result { - if self.rank() == 0 { - Ok(self.clone()) - } else { - self.flatten_all()?.max(0) - } - } - - /// Computes the min of all the elements in this tensor and returns a tensor holding this - /// scalar with zero dimensions. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; - /// let tensor = tensor.min_all()?; - /// assert_eq!(tensor.to_scalar::()?, 0.); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn min_all(&self) -> Result { - if self.rank() == 0 { - Ok(self.clone()) - } else { - self.flatten_all()?.min(0) - } - } - - /// Computes the sum of all the elements in this tensor and returns a tensor holding this - /// scalar with zero dimensions. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; - /// let tensor = tensor.sum_all()?; - /// assert_eq!(tensor.to_scalar::()?, 15.); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn sum_all(&self) -> Result { - let dims: Vec<_> = (0..self.rank()).collect(); - self.sum(dims) - } - - pub fn mean_all(&self) -> Result { - self.sum_all()? / self.elem_count() as f64 - } - - fn flatten_( - &self, - start_dim: Option, - end_dim: Option, - ) -> Result { - if self.rank() == 0 { - self.reshape(1) - } else { - let start_dim = match start_dim { - None => 0, - Some(dim) => dim.to_index(self.shape(), "flatten")?, - }; - let end_dim = match end_dim { - None => self.rank() - 1, - Some(dim) => dim.to_index(self.shape(), "flatten")?, - }; - if start_dim < end_dim { - let dims = self.dims(); - let mut dst_dims = dims[..start_dim].to_vec(); - dst_dims.push(dims[start_dim..end_dim + 1].iter().product::()); - if end_dim + 1 < dims.len() { - dst_dims.extend(&dims[end_dim + 1..]); - } - self.reshape(dst_dims) - } else { - Ok(self.clone()) - } - } - } - - /// Flattens the input tensor on the dimension indexes from `start_dim` to `end_dim` (both - /// inclusive). - pub fn flatten(&self, start_dim: D1, end_dim: D2) -> Result { - self.flatten_(Some(start_dim), Some(end_dim)) - } - - /// Flattens the input tensor on the dimension indexes from `0` to `end_dim` (inclusive). - pub fn flatten_to(&self, end_dim: D) -> Result { - self.flatten_(None::, Some(end_dim)) - } - - /// Flattens the input tensor on the dimension indexes from `start_dim` (inclusive) to the last - /// dimension. - pub fn flatten_from(&self, start_dim: D) -> Result { - self.flatten_(Some(start_dim), None::) - } - - /// Flattens the input tensor by reshaping it into a one dimension tensor. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; - /// let tensor = tensor.flatten_all()?; - /// assert_eq!(tensor.to_vec1::()?, &[0., 1., 2., 3., 4., 5.]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn flatten_all(&self) -> Result { - self.flatten_(None::, None::) - } - - /// Returns the sub-tensor fixing the index at `i` on the first dimension. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; - /// let t = tensor.get(0)?; - /// assert_eq!(t.to_vec1::()?, &[0., 1.]); - /// let t = tensor.get(1)?; - /// assert_eq!(t.to_vec1::()?, &[2., 3.]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn get(&self, i: usize) -> Result { - let dims = self.dims(); - if dims.is_empty() { - Ok(self.clone()) - } else { - self.narrow(0, i, 1)?.reshape(&dims[1..]) - } - } - - /// Returns the sub-tensor fixing the index at `index` on the dimension `dim`. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; - /// let t = tensor.get_on_dim(1, 0)?; - /// assert_eq!(t.to_vec1::()?, &[0., 2., 4.]); - /// let t = tensor.get_on_dim(1, 1)?; - /// assert_eq!(t.to_vec1::()?, &[1., 3., 5.]); - /// let t = tensor.get_on_dim(0, 1)?; - /// assert_eq!(t.to_vec1::()?, &[2., 3.]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn get_on_dim(&self, dim: D, index: usize) -> Result { - let dim = dim.to_index(self.shape(), "get_on_dim")?; - self.narrow(dim, index, 1)?.squeeze(dim) - } - - /// Returns a tensor that is a transposed version of the input, the two last dimensions of the - /// input are swapped. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let tensor = Tensor::new(&[[0f32, 1.], [2., 3.], [4., 5.]], &Device::Cpu)?; - /// let tensor = tensor.t()?; - /// assert_eq!(tensor.to_vec2::()?, &[[0.0, 2.0, 4.0], [1.0, 3.0, 5.0]]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn t(&self) -> Result { - let rank = self.rank(); - if rank < 2 { - Err(Error::UnexpectedNumberOfDims { - expected: 2, - got: rank, - shape: self.shape().clone(), - } - .bt())? - } - self.transpose(rank - 2, rank - 1) - } - - /// Returns a tensor that is a transposed version of the input, the given dimensions are - /// swapped. - pub fn transpose(&self, dim1: D1, dim2: D2) -> Result { - let dim1 = dim1.to_index(self.shape(), "transpose")?; - let dim2 = dim2.to_index(self.shape(), "transpose")?; - if dim1 == dim2 { - return Ok(self.clone()); - } - let op = BackpropOp::new1(self, |t| Op::Transpose(t, dim1, dim2)); - let tensor_ = Tensor_ { - id: TensorId::new(), - storage: self.storage.clone(), - layout: self.layout.transpose(dim1, dim2)?, - op, - is_variable: false, - dtype: self.dtype, - device: self.device.clone(), - }; - Ok(Tensor(Arc::new(tensor_))) - } - - /// Returns a tensor with the same data as the input where the dimensions have been permuted. - /// dims must be a permutation, i.e. include each dimension index exactly once. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let tensor = Tensor::arange(0u32, 120u32, &Device::Cpu)?.reshape((2, 3, 4, 5))?; - /// assert_eq!(tensor.dims(), &[2, 3, 4, 5]); - /// let tensor = tensor.permute((2, 3, 1, 0))?; - /// assert_eq!(tensor.dims(), &[4, 5, 3, 2]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn permute(&self, dims: D) -> Result { - let dims = dims.to_indexes(self.shape(), "permute")?; - // O(n^2) permutation check but these arrays are small. - let is_permutation = - dims.len() == self.rank() && (0..dims.len()).all(|i| dims.contains(&i)); - if !is_permutation { - bail!( - "dimension mismatch in permute, tensor {:?}, dims: {:?}", - self.dims(), - dims - ) - } - let op = BackpropOp::new1(self, |t| Op::Permute(t, dims.clone())); - let tensor_ = Tensor_ { - id: TensorId::new(), - storage: self.storage.clone(), - layout: self.layout.permute(&dims)?, - op, - is_variable: false, - dtype: self.dtype, - device: self.device.clone(), - }; - Ok(Tensor(Arc::new(tensor_))) - } - - /// Returns true if the data is stored in a C contiguous (aka row major) way. - pub fn is_contiguous(&self) -> bool { - self.layout.is_contiguous() - } - - /// Returns true if the data is stored in a Fortran contiguous (aka column major) way. - pub fn is_fortran_contiguous(&self) -> bool { - self.layout.is_fortran_contiguous() - } - - /// Compared to clone, this copies the actual storage but may fail because of running out of - /// memory. - pub fn copy(&self) -> Result { - let op = BackpropOp::new1(self, Op::Copy); - let tensor_ = Tensor_ { - id: TensorId::new(), - storage: Arc::new(RwLock::new(self.storage().try_clone(self.layout())?)), - layout: self.layout.clone(), - op, - is_variable: false, - dtype: self.dtype, - device: self.device.clone(), - }; - Ok(Tensor(Arc::new(tensor_))) - } - - /// Returns a new tensor detached from the current graph, gradient are not propagated through - /// this new node. The storage of this tensor is shared with the initial tensor. - /// - /// If the tensor is already detached from the computation graph, the same tensor is returned. - pub fn detach(&self) -> Tensor { - if self.op.is_none() && !self.is_variable { - self.clone() - } else { - let tensor_ = Tensor_ { - id: TensorId::new(), - storage: self.storage.clone(), - layout: self.layout.clone(), - op: BackpropOp::none(), - is_variable: false, - dtype: self.dtype, - device: self.device.clone(), - }; - Tensor(Arc::new(tensor_)) - } - } - - /// If the target device is the same as the tensor device, only a shallow copy is performed. - pub fn to_device(&self, device: &Device) -> Result { - if self.device().same_device(device) { - Ok(self.clone()) - } else { - let storage = match (&*self.storage(), device) { - (Storage::Cpu(storage), Device::Cuda(cuda)) => { - Storage::Cuda(cuda.storage_from_cpu_storage(storage)?) - } - (Storage::Cpu(storage), Device::Metal(metal)) => { - Storage::Metal(metal.storage_from_cpu_storage(storage)?) - } - (Storage::Cuda(storage), Device::Cpu) => Storage::Cpu(storage.to_cpu_storage()?), - (Storage::Metal(storage), Device::Cpu) => Storage::Cpu(storage.to_cpu_storage()?), - (Storage::Cuda(storage), Device::Cuda(cuda)) => { - // can't clone storage if it's the same device because of the underlying device ptr - let dst_storage = storage.transfer_to_device(cuda)?; - Storage::Cuda(dst_storage) - } - (Storage::Cpu(storage), Device::Cpu) => Storage::Cpu(storage.clone()), - _ => { - bail!( - "not implemented yet, self.device: {:?}, device: {:?}", - self.device(), - device - ) - } - }; - let op = BackpropOp::new1(self, Op::ToDevice); - let tensor_ = Tensor_ { - id: TensorId::new(), - storage: Arc::new(RwLock::new(storage)), - layout: self.layout.clone(), - op, - is_variable: false, - dtype: self.dtype, - device: device.clone(), - }; - Ok(Tensor(Arc::new(tensor_))) - } - } - - /// Returns a new tensor duplicating data from the original tensor. New dimensions are inserted - /// on the left. - pub fn broadcast_left>(&self, left_shape: S) -> Result { - let left_shape = left_shape.into(); - let mut dims = left_shape.into_dims(); - dims.extend(self.dims()); - self.broadcast_as(dims) - } - - /// Broadcast the input tensor to the target shape. This returns an error if the input shape is - /// not compatible with the target shape. - /// - /// If the input shape is `i_1, i_2, ... i_k`, the target shape has to have `k` dimensions or - /// more and shape `j_1, ..., j_l, t_1, t_2, ..., t_k`. The dimensions `j_1` to `j_l` can have - /// any value, the dimension `t_a` must be equal to `i_a` if `i_a` is different from 1. If - /// `i_a` is equal to 1, any value can be used. - pub fn broadcast_as>(&self, shape: S) -> Result { - let tensor_ = Tensor_ { - id: TensorId::new(), - storage: self.storage.clone(), - layout: self.layout.broadcast_as(shape)?, - op: BackpropOp::new1(self, Op::Broadcast), - is_variable: false, - dtype: self.dtype, - device: self.device.clone(), - }; - Ok(Tensor(Arc::new(tensor_))) - } - - /// An alias for broadcast_as. - pub fn expand>(&self, shape: S) -> Result { - self.broadcast_as(shape) - } - - /// Casts the input tensor to the target `dtype`. - /// - /// ```rust - /// use candle_core::{Tensor, Device}; - /// let tensor = Tensor::new(3.14159265358979f64, &Device::Cpu)?; - /// assert_eq!(tensor.to_scalar::()?, 3.14159265358979); - /// let tensor = tensor.to_dtype(candle_core::DType::F32)?; - /// assert_eq!(tensor.to_scalar::()?, 3.1415927); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn to_dtype(&self, dtype: DType) -> Result { - if self.dtype() == dtype { - Ok(self.clone()) - } else { - let shape = self.shape(); - let storage = self.storage().to_dtype(self.layout(), dtype)?; - let op = BackpropOp::new1(self, Op::ToDType); - Ok(from_storage(storage, shape.clone(), op, false)) - } - } - - /// Returns a tensor that is in row major order. This is the same as the original tensor if it - /// was already contiguous, otherwise a copy is triggered. - pub fn contiguous(&self) -> Result { - if self.is_contiguous() { - Ok(self.clone()) - } else { - let shape = self.shape(); - let mut storage = unsafe { self.device().alloc_uninit(shape, self.dtype())? }; - self.storage() - .copy_strided_src(&mut storage, 0, self.layout())?; - let op = BackpropOp::new1(self, Op::Copy); - Ok(from_storage(storage, shape.clone(), op, false)) - } - } - - /// Returns a tensor that is in row major order. This always makes a copy. - pub fn force_contiguous(&self) -> Result { - let shape = self.shape(); - let mut storage = unsafe { self.device().alloc_uninit(shape, self.dtype())? }; - self.storage() - .copy_strided_src(&mut storage, 0, self.layout())?; - let op = BackpropOp::new1(self, Op::Copy); - Ok(from_storage(storage, shape.clone(), op, false)) - } - - /// Create a variable based on the values currently stored in a tensor. The storage is always - /// copied. - pub(crate) fn make_var(&self) -> Result { - let shape = self.shape().clone(); - let mut storage = unsafe { self.device().alloc_uninit(&shape, self.dtype())? }; - self.storage() - .copy_strided_src(&mut storage, 0, self.layout())?; - Ok(from_storage(storage, shape, BackpropOp::none(), true)) - } - - /// Reshape returns a tensor with the target shape provided that the number of elements of the - /// original tensor is the same. - /// If the input tensor is contiguous, this is a view on the original data. Otherwise this uses - /// a new storage and copies the data over, the returned tensor is always contiguous. - /// - /// The shape can be specified using a tuple of `usize` and at most one `()` in which case - /// the behavior is the same as when using `-1` in PyTorch: this dimension size is adjusted so - /// as to match the number of elements in the tensor. - /// - /// ```rust - /// # use candle_core::{Tensor, DType, Device, D}; - /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; - /// - /// let c = a.reshape((1, 6))?; - /// assert_eq!(c.shape().dims(), &[1, 6]); - /// - /// let c = a.reshape((3, 2))?; - /// assert_eq!(c.shape().dims(), &[3, 2]); - /// - /// let c = a.reshape((2, (), 1))?; - /// assert_eq!(c.shape().dims(), &[2, 3, 1]); - /// - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn reshape(&self, s: S) -> Result { - let shape = s.into_shape(self.elem_count())?; - if shape.elem_count() != self.elem_count() { - return Err(Error::ShapeMismatchBinaryOp { - lhs: self.shape().clone(), - rhs: shape, - op: "reshape", - } - .bt()); - } - let op = BackpropOp::new1(self, Op::Reshape); - if self.is_contiguous() { - let tensor_ = Tensor_ { - id: TensorId::new(), - storage: self.storage.clone(), - layout: Layout::contiguous_with_offset(shape, self.layout.start_offset()), - op, - is_variable: false, - dtype: self.dtype, - device: self.device.clone(), - }; - Ok(Tensor(Arc::new(tensor_))) - } else { - let mut storage = unsafe { self.device().alloc_uninit(&shape, self.dtype())? }; - self.storage() - .copy_strided_src(&mut storage, 0, self.layout())?; - Ok(from_storage(storage, shape, op, false)) - } - } - - /// Creates a new tensor with the specified dimension removed if its size was one. - /// - /// ```rust - /// # use candle_core::{Tensor, DType, Device, D}; - /// let a = Tensor::zeros((2, 3, 1), DType::F32, &Device::Cpu)?; - /// - /// let c = a.squeeze(2)?; - /// assert_eq!(c.shape().dims(), &[2, 3]); - /// - /// let c = a.squeeze(D::Minus1)?; - /// assert_eq!(c.shape().dims(), &[2, 3]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn squeeze(&self, dim: D) -> Result { - // The PyTorch semantics are to return the same tensor if the target dimension - // does not have a size of 1. - let dims = self.dims(); - let dim = dim.to_index(self.shape(), "squeeze")?; - if dims[dim] == 1 { - let mut dims = dims.to_vec(); - let mut strides = self.stride().to_vec(); - dims.remove(dim); - strides.remove(dim); - let tensor_ = Tensor_ { - id: TensorId::new(), - storage: self.storage.clone(), - layout: Layout::new(dims.into(), strides, self.layout.start_offset()), - op: BackpropOp::new1(self, Op::Reshape), - is_variable: false, - dtype: self.dtype, - device: self.device.clone(), - }; - Ok(Tensor(Arc::new(tensor_))) - } else { - Ok(self.clone()) - } - } - - /// Creates a new tensor with a dimension of size one inserted at the specified position. - /// - /// ```rust - /// # use candle_core::{Tensor, DType, Device, D}; - /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; - /// - /// let c = a.unsqueeze(0)?; - /// assert_eq!(c.shape().dims(), &[1, 2, 3]); - /// - /// let c = a.unsqueeze(D::Minus1)?; - /// assert_eq!(c.shape().dims(), &[2, 3, 1]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn unsqueeze(&self, dim: D) -> Result { - let mut dims = self.dims().to_vec(); - let mut strides = self.stride().to_vec(); - let dim = dim.to_index_plus_one(self.shape(), "unsqueeze")?; - // Cannot panic because to_index_plus_one already checks dimensions - dims.insert(dim, 1); - // Any stride would work here, but we pick one so as to maximize the probability to remain - // C contiguous. - let stride = if dim < strides.len() { strides[dim] } else { 1 }; - strides.insert(dim, stride); - let tensor_ = Tensor_ { - id: TensorId::new(), - storage: self.storage.clone(), - layout: Layout::new(dims.into(), strides, self.layout.start_offset()), - op: BackpropOp::new1(self, Op::Reshape), - is_variable: false, - dtype: self.dtype, - device: self.device.clone(), - }; - Ok(Tensor(Arc::new(tensor_))) - } - - /// Stacks two or more tensors along a particular dimension. - /// - /// All tensors must have the same rank, and the output has one additional rank - /// - /// ```rust - /// # use candle_core::{Tensor, DType, Device}; - /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; - /// let b = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; - /// - /// let c = Tensor::stack(&[&a, &b], 0)?; - /// assert_eq!(c.shape().dims(), &[2, 2, 3]); - /// - /// let c = Tensor::stack(&[&a, &b], 2)?; - /// assert_eq!(c.shape().dims(), &[2, 3, 2]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn stack, D: Dim>(args: &[A], dim: D) -> Result { - if args.is_empty() { - Err(Error::OpRequiresAtLeastOneTensor { op: "stack" }.bt())? - } - let dim = dim.to_index_plus_one(args[0].as_ref().shape(), "stack")?; - let args = args - .iter() - .map(|t| t.as_ref().unsqueeze(dim)) - .collect::>>()?; - Self::cat(&args, dim) - } - - /// Pad the input tensor using 0s along dimension `dim`. This adds `left` elements before the - /// input tensor values and `right` elements after. - pub fn pad_with_zeros(&self, dim: D, left: usize, right: usize) -> Result { - if left == 0 && right == 0 { - Ok(self.clone()) - } else if left == 0 { - let dim = dim.to_index(self.shape(), "pad_with_zeros")?; - let mut dims = self.dims().to_vec(); - dims[dim] = right; - let right = Tensor::zeros(dims.as_slice(), self.dtype, self.device())?; - Tensor::cat(&[self, &right], dim) - } else if right == 0 { - let dim = dim.to_index(self.shape(), "pad_with_zeros")?; - let mut dims = self.dims().to_vec(); - dims[dim] = left; - let left = Tensor::zeros(dims.as_slice(), self.dtype, self.device())?; - Tensor::cat(&[&left, self], dim) - } else { - let dim = dim.to_index(self.shape(), "pad_with_zeros")?; - let mut dims = self.dims().to_vec(); - dims[dim] = left; - let left = Tensor::zeros(dims.as_slice(), self.dtype, self.device())?; - dims[dim] = right; - let right = Tensor::zeros(dims.as_slice(), self.dtype, self.device())?; - Tensor::cat(&[&left, self, &right], dim) - } - } - - /// Pad the input tensor using same values along dimension `dim`. This adds `left` elements before the - /// input tensor values and `right` elements after. - pub fn pad_with_same(&self, dim: D, left: usize, right: usize) -> Result { - if left == 0 && right == 0 { - Ok(self.clone()) - } else if self.elem_count() == 0 { - bail!("cannot use pad_with_same on an empty tensor") - } else if left == 0 { - let dim = dim.to_index(self.shape(), "pad_with_same")?; - let r = self.narrow(dim, self.dim(dim)? - 1, 1)?; - let mut v = vec![self]; - for _ in 0..right { - v.push(&r) - } - Tensor::cat(&v, dim) - } else if right == 0 { - let dim = dim.to_index(self.shape(), "pad_with_same")?; - let l = self.narrow(dim, 0, 1)?; - let mut v = vec![]; - for _ in 0..left { - v.push(&l) - } - v.push(self); - Tensor::cat(&v, dim) - } else { - let dim = dim.to_index(self.shape(), "pad_with_same")?; - let l = self.narrow(dim, 0, 1)?; - let r = self.narrow(dim, self.dim(dim)? - 1, 1)?; - let mut v = vec![]; - for _ in 0..left { - v.push(&l) - } - v.push(self); - for _ in 0..right { - v.push(&r) - } - Tensor::cat(&v, dim) - } - } - - /// Run the `forward` method of `m` on `self`. - pub fn apply(&self, m: &M) -> Result { - m.forward(self) - } - - /// Run the `forward` method of `m` on `self`. - pub fn apply_t(&self, m: &M, train: bool) -> Result { - m.forward_t(self, train) - } - - pub(crate) fn storage(&self) -> std::sync::RwLockReadGuard<'_, Storage> { - self.storage.read().unwrap() - } - - pub(crate) fn storage_mut(&self) -> std::sync::RwLockWriteGuard<'_, Storage> { - self.storage.write().unwrap() - } - - // If we extend the visibility of this function to be usable outside of this crate, we should - // make it unsafe. - pub(crate) fn storage_mut_and_layout( - &self, - ) -> (std::sync::RwLockWriteGuard<'_, Storage>, &Layout) { - let storage = self.storage.write().unwrap(); - (storage, &self.layout) - } - - /// The storage used by this tensor, together with the layout to use to access it safely. - pub fn storage_and_layout(&self) -> (std::sync::RwLockReadGuard<'_, Storage>, &Layout) { - let storage = self.storage.read().unwrap(); - (storage, &self.layout) - } - - pub(crate) fn same_storage(&self, rhs: &Self) -> bool { - let lhs: &RwLock = self.storage.as_ref(); - let rhs: &RwLock = rhs.storage.as_ref(); - std::ptr::eq(lhs, rhs) - } - - /// Normalize a 'relative' axis value: positive values are kept, negative - /// values means counting the dimensions from the back. - pub fn normalize_axis(&self, axis: i64) -> Result { - let rank = self.rank() as i64; - if rank <= axis { - bail!("axis {axis} is too large, tensor rank {rank}") - } else if 0 <= axis { - Ok(axis as usize) - } else { - let naxis = rank + axis; - if naxis < 0 { - bail!("axis {axis} is too small, tensor rank {rank}") - } - Ok(naxis as usize) - } - } - - /// Returns a lower triangular matrix of ones of size n by n. - pub fn tril2(n: usize, dtype: DType, device: &Device) -> Result { - let t = Tensor::arange(0u32, n as u32, device)?; - let t1 = t.reshape((1, n))?.broadcast_as((n, n))?; - let t2 = t.reshape((n, 1))?.broadcast_as((n, n))?; - t1.le(&t2)?.to_dtype(dtype) - } - - /// Returns an upper triangular matrix of ones of size n by n. - pub fn triu2(n: usize, dtype: DType, device: &Device) -> Result { - let t = Tensor::arange(0u32, n as u32, device)?; - let t1 = t.reshape((1, n))?.broadcast_as((n, n))?; - let t2 = t.reshape((n, 1))?.broadcast_as((n, n))?; - t1.ge(&t2)?.to_dtype(dtype) - } - - /// Returns a matrix with a diagonal of ones of size n by n. - pub fn eye(n: usize, dtype: DType, device: &Device) -> Result { - let t = Tensor::arange(0u32, n as u32, device)?; - let t1 = t.reshape((1, n))?.broadcast_as((n, n))?; - let t2 = t.reshape((n, 1))?.broadcast_as((n, n))?; - t1.eq(&t2)?.to_dtype(dtype) - } - - /// Returns the cumulative sum of elements of the input tensor summed over the specified - /// dimension. - /// - /// This operation is most efficient when dim is the last dimension of the tensor. - pub fn cumsum(&self, dim: D) -> Result { - let dim = dim.to_index(self.shape(), "cumsum")?; - let rank = self.rank(); - if rank == 0 { - return Ok(self.clone()); - } - let n_axis = self.dim(dim)?; - let triu = Tensor::triu2(n_axis, self.dtype(), self.device())?; - if rank == 1 { - self.unsqueeze(0)?.matmul(&triu)?.squeeze(0) - } else { - let last = rank - 1; - let t = self.transpose(dim, last)?; - let t = t.broadcast_matmul(&triu)?; - t.transpose(dim, last) - } - } - - /// Returns a copy of `self` where the values within `ranges` have been replaced with the - /// content of `src`. - pub fn slice_assign>( - &self, - ranges: &[D], - src: &Tensor, - ) -> Result { - let src_dims = src.dims(); - let self_dims = self.dims(); - if self_dims.len() != src_dims.len() { - bail!( - "slice-assign requires input with the same rank {} <> {}", - self_dims.len(), - src_dims.len() - ) - } - if self_dims.len() != ranges.len() { - bail!( - "slice-assign requires input with the same rank as there are ranges {} <> {}", - self_dims.len(), - ranges.len() - ) - } - let mut src = src.clone(); - let mut mask = Self::ones(src.shape(), DType::U8, src.device())?; - for (i, range) in ranges.iter().enumerate() { - let start_included = match range.start_bound() { - std::ops::Bound::Unbounded => 0, - std::ops::Bound::Included(v) => *v, - std::ops::Bound::Excluded(v) => *v + 1, - }; - let end_excluded = match range.end_bound() { - std::ops::Bound::Unbounded => self_dims[i], - std::ops::Bound::Included(v) => *v + 1, - std::ops::Bound::Excluded(v) => *v, - }; - if end_excluded <= start_included { - bail!("slice-assign: empty range for dim {i}, {start_included} {end_excluded}") - } - if self_dims[i] < end_excluded { - bail!( - "slice-assign: upper bound is out of range for dim {i}, {end_excluded} {}", - self_dims[i] - ) - } - if end_excluded - start_included != src_dims[i] { - bail!( - "slice-assign: the range for dim {i} ({start_included}..{end_excluded}) does not match the size of src {}", src_dims[i] - ) - } - src = src.pad_with_zeros(i, start_included, self_dims[i] - end_excluded)?; - mask = mask.pad_with_zeros(i, start_included, self_dims[i] - end_excluded)? - } - mask.where_cond(/* on_true= */ &src, /* on_false= */ self) - } - - /// Returns log(sum(exp(tensor), dim)). - pub fn log_sum_exp(&self, sum_dims: D) -> Result { - let sum_dims = sum_dims.to_indexes(self.shape(), "log-sum-exp")?; - if sum_dims.is_empty() { - return Ok(self.clone()); - } - let max = sum_dims[1..] - .iter() - .try_fold(self.max_keepdim(sum_dims[0])?, |max, &dim| { - max.max_keepdim(dim) - })?; - let exp = self.broadcast_sub(&max)?.exp()?; - let sum = exp.sum(sum_dims.clone())?; - - sum.log()? + max.squeeze_dims(&sum_dims) - } - - /// Pointwise pow operation. - pub fn pow(&self, rhs: &Tensor) -> Result { - rhs.mul(&self.log()?)?.exp() - } - - /// Broadcasting version of `pow`. - pub fn broadcast_pow(&self, rhs: &Tensor) -> Result { - rhs.broadcast_mul(&self.log()?)?.exp() - } - - /// Returns a new tensor with the order of elements reversed along the specified dimensions. - /// This function makes a copy of the tensor’s data. - /// - /// ```rust - /// # use candle_core::{Tensor, Device}; - /// let t = Tensor::arange(0., 6., &Device::Cpu)?.reshape((2, 3))?; - /// assert_eq!(t.to_vec2::()?, &[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]]); - /// let t_flipped = t.flip(&[0])?; - /// assert_eq!(t_flipped.to_vec2::()?, &[[3.0, 4.0, 5.0], [0.0, 1.0, 2.0]]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn flip(&self, dims: &[usize]) -> Result { - let mut result = self.clone(); - for &dim in dims.iter() { - let size = result.dim(dim)?; - let indices: Vec = (0..size).rev().map(|x| x as i64).collect(); - let indices_tensor = Tensor::from_vec(indices, (size,), result.device())?; - result = result.index_select(&indices_tensor, dim)?; - } - Ok(result) - } - - /// Returns a view of which contains all slices of size `size` from self tensor in the dimension - /// `dim` and stepped by `step`. - pub fn unfold(&self, dim: D, size: usize, step: usize) -> Result { - // https://github.com/pytorch/pytorch/blob/75b0720a97ac5d82e8a7a1a6ae7c5f7a87d7183d/aten/src/ATen/native/TensorShape.cpp#L3785-L3804 - let mut sizes = self.dims().to_vec(); - let mut strides = self.stride().to_vec(); - - let dim = dim.to_index(self.shape(), "unfold")?; - - let max_len = if self.dims().is_empty() { - 1 - } else { - sizes[dim] - }; - if size > max_len { - bail!( - "unsqueeze: maximum size for tensor at dimension {dim} is {max_len} but size is {size}" - ) - } - sizes.push(size); - strides.push(if self.dims().is_empty() { - 1 - } else { - strides[dim] - }); - - if !self.dims().is_empty() { - sizes[dim] = ((sizes[dim] as f32 - size as f32) / step as f32 + 1.) as usize; - strides[dim] *= step; - } - - let tensor_ = Tensor_ { - id: TensorId::new(), - storage: self.storage.clone(), - layout: Layout::new(sizes.into(), strides, self.layout.start_offset()), - op: BackpropOp::new1(self, Op::Reshape), - is_variable: false, - dtype: self.dtype, - device: self.device.clone(), - }; - Ok(Tensor(Arc::new(tensor_))) - } -} - -macro_rules! bin_trait { - ($trait:ident, $fn1:ident, $mul:expr, $add:expr) => { - impl> std::ops::$trait for Tensor { - type Output = Result; - - fn $fn1(self, rhs: B) -> Self::Output { - Tensor::$fn1(&self, rhs.borrow()) - } - } - - impl> std::ops::$trait for &Tensor { - type Output = Result; - - fn $fn1(self, rhs: B) -> Self::Output { - Tensor::$fn1(&self, rhs.borrow()) - } - } - - impl> std::ops::$trait for Result { - type Output = Result; - - fn $fn1(self, rhs: Tensor) -> Self::Output { - Tensor::$fn1(self?.borrow(), &rhs) - } - } - - impl> std::ops::$trait<&Tensor> for Result { - type Output = Result; - - fn $fn1(self, rhs: &Tensor) -> Self::Output { - Tensor::$fn1(self?.borrow(), rhs) - } - } - - impl> std::ops::$trait> for Tensor { - type Output = Result; - - fn $fn1(self, rhs: Result) -> Self::Output { - Tensor::$fn1(&self, rhs?.borrow()) - } - } - - impl> std::ops::$trait> for &Tensor { - type Output = Result; - - fn $fn1(self, rhs: Result) -> Self::Output { - Tensor::$fn1(&self, rhs?.borrow()) - } - } - - impl std::ops::$trait for Tensor { - type Output = Result; - - fn $fn1(self, rhs: f64) -> Self::Output { - self.affine($mul(rhs), $add(rhs)) - } - } - - impl std::ops::$trait for &Tensor { - type Output = Result; - - fn $fn1(self, rhs: f64) -> Self::Output { - self.affine($mul(rhs), $add(rhs)) - } - } - }; -} - -bin_trait!(Add, add, |_| 1., |v| v); -bin_trait!(Sub, sub, |_| 1., |v: f64| -v); -bin_trait!(Mul, mul, |v| v, |_| 0.); -bin_trait!(Div, div, |v| 1. / v, |_| 0.); - -impl std::ops::Add for f64 { - type Output = Result; - - fn add(self, rhs: Tensor) -> Self::Output { - rhs + self - } -} - -impl std::ops::Add<&Tensor> for f64 { - type Output = Result; - - fn add(self, rhs: &Tensor) -> Self::Output { - rhs + self - } -} - -impl std::ops::Mul for f64 { - type Output = Result; - - fn mul(self, rhs: Tensor) -> Self::Output { - rhs * self - } -} - -impl std::ops::Mul<&Tensor> for f64 { - type Output = Result; - - fn mul(self, rhs: &Tensor) -> Self::Output { - rhs * self - } -} - -impl std::ops::Sub for f64 { - type Output = Result; - - fn sub(self, rhs: Tensor) -> Self::Output { - rhs.affine(-1., self) - } -} - -impl std::ops::Sub<&Tensor> for f64 { - type Output = Result; - - fn sub(self, rhs: &Tensor) -> Self::Output { - rhs.affine(-1., self) - } -} - -impl std::ops::Div for f64 { - type Output = Result; - - #[allow(clippy::suspicious_arithmetic_impl)] - fn div(self, rhs: Tensor) -> Self::Output { - rhs.recip()? * self - } -} - -impl std::ops::Div<&Tensor> for f64 { - type Output = Result; - - #[allow(clippy::suspicious_arithmetic_impl)] - fn div(self, rhs: &Tensor) -> Self::Output { - rhs.recip()? * self - } -} - -impl> From<(Storage, S)> for Tensor { - fn from((storage, shape): (Storage, S)) -> Self { - from_storage(storage, shape, BackpropOp::none(), false) - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/tensor_cat.rs b/v2/dependency-patches/candle-core-0.9.2/src/tensor_cat.rs deleted file mode 100644 index 520b246f5e..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/tensor_cat.rs +++ /dev/null @@ -1,303 +0,0 @@ -use crate::{shape::Dim, Context, Error, Result, Shape, Tensor}; - -impl Tensor { - /// Concatenates two or more tensors along a particular dimension. - /// - /// All tensors must of the same rank, and the output will have - /// the same rank - /// - /// ```rust - /// # use candle_core::{Tensor, DType, Device}; - /// let a = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; - /// let b = Tensor::zeros((2, 3), DType::F32, &Device::Cpu)?; - /// - /// let c = Tensor::cat(&[&a, &b], 0)?; - /// assert_eq!(c.shape().dims(), &[4, 3]); - /// - /// let c = Tensor::cat(&[&a, &b], 1)?; - /// assert_eq!(c.shape().dims(), &[2, 6]); - /// # Ok::<(), candle_core::Error>(()) - /// ``` - pub fn cat, D: Dim>(args: &[A], dim: D) -> Result { - if args.is_empty() { - Err(Error::OpRequiresAtLeastOneTensor { op: "cat" }.bt())? - } - let arg0 = args[0].as_ref(); - if args.len() == 1 { - return Ok(arg0.clone()); - } - let dim = dim.to_index(arg0.shape(), "cat")?; - for arg in args { - arg.as_ref().check_dim(dim, "cat")?; - } - for (arg_idx, arg) in args.iter().enumerate() { - let arg = arg.as_ref(); - if arg0.rank() != arg.rank() { - Err(Error::UnexpectedNumberOfDims { - expected: arg0.rank(), - got: arg.rank(), - shape: arg.shape().clone(), - } - .bt())? - } - for (dim_idx, (v1, v2)) in arg0 - .shape() - .dims() - .iter() - .zip(arg.shape().dims().iter()) - .enumerate() - { - if dim_idx != dim && v1 != v2 { - Err(Error::ShapeMismatchCat { - dim: dim_idx, - first_shape: arg0.shape().clone(), - n: arg_idx + 1, - nth_shape: arg.shape().clone(), - } - .bt())? - } - } - } - let all_contiguous = args.iter().all(|v| v.as_ref().is_contiguous()); - if all_contiguous { - Self::cat_contiguous(args, dim) - } else if dim == 0 { - Self::cat0(args) - } else { - let args: Vec = args - .iter() - .map(|a| a.as_ref().transpose(0, dim)) - .collect::>>()?; - let cat = Self::cat0(&args)?; - cat.transpose(0, dim) - } - } - - fn cat0>(args: &[A]) -> Result { - if args.is_empty() { - Err(Error::OpRequiresAtLeastOneTensor { op: "cat" }.bt())? - } - let arg0 = args[0].as_ref(); - if args.len() == 1 { - return Ok(arg0.clone()); - } - let rank = arg0.rank(); - let device = arg0.device(); - let dtype = arg0.dtype(); - let first_dims = arg0.shape().dims(); - let mut cat_dims = first_dims.to_vec(); - cat_dims[0] = 0; - let mut offsets = vec![0usize]; - for (arg_idx, arg) in args.iter().enumerate() { - let arg = arg.as_ref(); - if arg.dtype() != dtype { - Err(Error::DTypeMismatchBinaryOp { - lhs: dtype, - rhs: arg.dtype(), - op: "cat", - } - .bt())? - } - if arg.device().location() != device.location() { - Err(Error::DeviceMismatchBinaryOp { - lhs: device.location(), - rhs: arg.device().location(), - op: "cat", - } - .bt())? - } - if rank != arg.rank() { - Err(Error::UnexpectedNumberOfDims { - expected: rank, - got: arg.rank(), - shape: arg.shape().clone(), - } - .bt())? - } - for (dim_idx, (v1, v2)) in arg0 - .shape() - .dims() - .iter() - .zip(arg.shape().dims().iter()) - .enumerate() - { - if dim_idx == 0 { - cat_dims[0] += v2; - } - if dim_idx != 0 && v1 != v2 { - Err(Error::ShapeMismatchCat { - dim: dim_idx, - first_shape: arg0.shape().clone(), - n: arg_idx + 1, - nth_shape: arg.shape().clone(), - } - .bt())? - } - } - let next_offset = offsets.last().context("empty offsets")? + arg.elem_count(); - offsets.push(next_offset); - } - let shape = Shape::from(cat_dims); - let op = crate::op::BackpropOp::new(args, |args| crate::op::Op::Cat(args, 0)); - let mut storage = unsafe { device.alloc_uninit(&shape, dtype)? }; - for (arg, &offset) in args.iter().zip(offsets.iter()) { - let arg = arg.as_ref(); - arg.storage() - .copy_strided_src(&mut storage, offset, arg.layout())?; - } - Ok(crate::tensor::from_storage(storage, shape, op, false)) - } - - fn cat_contiguous>(args: &[A], dim: usize) -> Result { - if args.is_empty() { - Err(Error::OpRequiresAtLeastOneTensor { op: "cat" }.bt())? - } - let arg0 = args[0].as_ref(); - if args.len() == 1 { - return Ok(arg0.clone()); - } - let rank = arg0.rank(); - let device = arg0.device(); - let dtype = arg0.dtype(); - let first_dims = arg0.shape().dims(); - let mut cat_dims = first_dims.to_vec(); - cat_dims[dim] = 0; - for (arg_idx, arg) in args.iter().enumerate() { - let arg = arg.as_ref(); - if arg.dtype() != dtype { - Err(Error::DTypeMismatchBinaryOp { - lhs: dtype, - rhs: arg.dtype(), - op: "cat", - } - .bt())? - } - if arg.device().location() != device.location() { - Err(Error::DeviceMismatchBinaryOp { - lhs: device.location(), - rhs: arg.device().location(), - op: "cat", - } - .bt())? - } - if rank != arg.rank() { - Err(Error::UnexpectedNumberOfDims { - expected: rank, - got: arg.rank(), - shape: arg.shape().clone(), - } - .bt())? - } - for (dim_idx, (v1, v2)) in arg0 - .shape() - .dims() - .iter() - .zip(arg.shape().dims().iter()) - .enumerate() - { - if dim_idx == dim { - cat_dims[dim] += v2; - } - if dim_idx != dim && v1 != v2 { - Err(Error::ShapeMismatchCat { - dim: dim_idx, - first_shape: arg0.shape().clone(), - n: arg_idx + 1, - nth_shape: arg.shape().clone(), - } - .bt())? - } - } - } - let cat_target_dim_len = cat_dims[dim]; - let block_size: usize = cat_dims.iter().skip(1 + dim).product(); - let shape = Shape::from(cat_dims); - let op = crate::op::BackpropOp::new(args, |args| crate::op::Op::Cat(args, dim)); - let mut storage = unsafe { device.alloc_uninit(&shape, dtype)? }; - let mut dst_o = 0; - for arg in args.iter() { - let arg = arg.as_ref(); - let arg_dims = arg.shape().dims(); - let d1: usize = arg_dims.iter().take(dim).product(); - let d2 = block_size * arg_dims[dim]; - let dst_s = block_size * cat_target_dim_len; - let src_o = arg.layout().start_offset(); - arg.storage().copy2d( - &mut storage, - d1, - d2, - /* src_s */ d2, - dst_s, - src_o, - dst_o, - )?; - dst_o += d2; - } - Ok(crate::tensor::from_storage(storage, shape, op, false)) - } - - /// Set the values on `self` using values from `src`. The copy starts at the specified - /// `offset` for the target dimension `dim` on `self`. - /// `self` and `src` must have the same shape except on dimension `dim` where the `self` size - /// has to be greater than or equal to `offset` plus the `src` size. - /// - /// Note that this modifies `self` in place and as such is not compatible with - /// back-propagation. - pub fn slice_set(&self, src: &Self, dim: D, offset: usize) -> Result<()> { - let dim = dim.to_index(self.shape(), "slice-set")?; - if !self.is_contiguous() || !src.is_contiguous() { - Err(Error::RequiresContiguous { op: "slice-set" }.bt())? - } - if self.same_storage(src) { - crate::bail!("cannot use slice_set when self and src share their storage") - } - if self.dtype() != src.dtype() { - Err(Error::DTypeMismatchBinaryOp { - lhs: self.dtype(), - rhs: src.dtype(), - op: "slice-set", - } - .bt())? - } - if self.device().location() != src.device().location() { - Err(Error::DeviceMismatchBinaryOp { - lhs: self.device().location(), - rhs: src.device().location(), - op: "slice-set", - } - .bt())? - } - if self.rank() != src.rank() { - Err(Error::UnexpectedNumberOfDims { - expected: self.rank(), - got: src.rank(), - shape: self.shape().clone(), - } - .bt())? - } - for (dim_idx, (v1, v2)) in self.dims().iter().zip(src.dims().iter()).enumerate() { - if dim_idx == dim && *v2 + offset > *v1 { - crate::bail!("shape mismatch on target dim, dst: {v1}, src: {v2} + {offset}") - } - if dim_idx != dim && v1 != v2 { - crate::bail!("shape mismatch on dim {dim_idx}, {v1} <> {v2}") - } - } - let block_size: usize = src.dims().iter().skip(1 + dim).product(); - let d1: usize = src.dims().iter().take(dim).product(); - let d2 = block_size * src.dims()[dim]; - let dst_o = self.layout().start_offset() + offset * block_size; - let src_o = src.layout().start_offset(); - src.storage().copy2d( - &mut self.storage_mut(), - d1, - d2, - /* src_s */ d2, - /* dst_s */ block_size * self.dims()[dim], - src_o, - dst_o, - )?; - - Ok(()) - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/test_utils.rs b/v2/dependency-patches/candle-core-0.9.2/src/test_utils.rs deleted file mode 100644 index e331399f4d..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/test_utils.rs +++ /dev/null @@ -1,71 +0,0 @@ -use crate::{Result, Tensor}; - -#[macro_export] -macro_rules! test_device { - // TODO: Switch to generating the two last arguments automatically once concat_idents is - // stable. https://github.com/rust-lang/rust/issues/29599 - ($fn_name: ident, $test_cpu: ident, $test_cuda: ident, $test_metal: ident) => { - #[test] - fn $test_cpu() -> Result<()> { - $fn_name(&Device::Cpu) - } - - #[cfg(feature = "cuda")] - #[test] - fn $test_cuda() -> Result<()> { - $fn_name(&Device::new_cuda(0)?) - } - - #[cfg(feature = "metal")] - #[test] - fn $test_metal() -> Result<()> { - $fn_name(&Device::new_metal(0)?) - } - }; -} - -pub fn assert_tensor_eq(t1: &Tensor, t2: &Tensor) -> Result<()> { - assert_eq!(t1.shape(), t2.shape()); - // Default U8 may not be large enough to hold the sum (`t.sum_all` defaults to the dtype of `t`) - let eq_tensor = t1.eq(t2)?.to_dtype(crate::DType::U32)?; - let all_equal = eq_tensor.sum_all()?; - assert_eq!(all_equal.to_scalar::()?, eq_tensor.elem_count() as u32); - Ok(()) -} - -pub fn to_vec0_round(t: &Tensor, digits: i32) -> Result { - let b = 10f32.powi(digits); - let t = t.to_vec0::()?; - Ok(f32::round(t * b) / b) -} - -pub fn to_vec1_round(t: &Tensor, digits: i32) -> Result> { - let b = 10f32.powi(digits); - let t = t.to_vec1::()?; - let t = t.iter().map(|t| f32::round(t * b) / b).collect(); - Ok(t) -} - -pub fn to_vec2_round(t: &Tensor, digits: i32) -> Result>> { - let b = 10f32.powi(digits); - let t = t.to_vec2::()?; - let t = t - .iter() - .map(|t| t.iter().map(|t| f32::round(t * b) / b).collect()) - .collect(); - Ok(t) -} - -pub fn to_vec3_round(t: &Tensor, digits: i32) -> Result>>> { - let b = 10f32.powi(digits); - let t = t.to_vec3::()?; - let t = t - .iter() - .map(|t| { - t.iter() - .map(|t| t.iter().map(|t| f32::round(t * b) / b).collect()) - .collect() - }) - .collect(); - Ok(t) -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/utils.rs b/v2/dependency-patches/candle-core-0.9.2/src/utils.rs deleted file mode 100644 index 4a58db8113..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/utils.rs +++ /dev/null @@ -1,45 +0,0 @@ -//! Useful functions for checking features. -use std::str::FromStr; - -pub fn get_num_threads() -> usize { - // Respond to the same environment variable as rayon. - match std::env::var("RAYON_NUM_THREADS") - .ok() - .and_then(|s| usize::from_str(&s).ok()) - { - Some(x) if x > 0 => x, - Some(_) | None => num_cpus::get(), - } -} - -pub fn has_accelerate() -> bool { - cfg!(feature = "accelerate") -} - -pub fn has_mkl() -> bool { - cfg!(feature = "mkl") -} - -pub fn cuda_is_available() -> bool { - cfg!(feature = "cuda") -} - -pub fn metal_is_available() -> bool { - cfg!(feature = "metal") -} - -pub fn with_avx() -> bool { - cfg!(target_feature = "avx2") -} - -pub fn with_neon() -> bool { - cfg!(target_feature = "neon") -} - -pub fn with_simd128() -> bool { - cfg!(target_feature = "simd128") -} - -pub fn with_f16c() -> bool { - cfg!(target_feature = "f16c") -} diff --git a/v2/dependency-patches/candle-core-0.9.2/src/variable.rs b/v2/dependency-patches/candle-core-0.9.2/src/variable.rs deleted file mode 100644 index 1e4880e59a..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/src/variable.rs +++ /dev/null @@ -1,152 +0,0 @@ -// Variables are wrappers around tensors that can be modified, they are typically used for holding -// weights and being modified by gradient descent. -// We do not expose a public way to create variables as this would break the invariant that the -// tensor within a variable is actually with `is_variable` set to `true`. -use crate::{DType, Device, Error, Result, Shape, Tensor}; - -/// A variable is a wrapper around a tensor, however variables can have their content modified -/// whereas tensors are immutable. -#[derive(Clone, Debug)] -pub struct Var(Tensor); - -impl std::fmt::Display for Var { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - std::fmt::Display::fmt(&self.0, f) - } -} - -impl std::ops::Deref for Var { - type Target = Tensor; - - fn deref(&self) -> &Self::Target { - self.0.as_ref() - } -} - -impl Var { - pub fn zeros>(shape: S, dtype: DType, device: &Device) -> Result { - let inner = Tensor::zeros_impl(shape, dtype, device, true)?; - Ok(Self(inner)) - } - - pub fn ones>(shape: S, dtype: DType, device: &Device) -> Result { - let inner = Tensor::ones_impl(shape, dtype, device, true)?; - Ok(Self(inner)) - } - - // Convert a tensor to a variable, if the tensor is already a variable then it is returned as is. - pub fn from_tensor(t: &Tensor) -> Result { - if t.is_variable() { - Ok(Self(t.clone())) - } else { - let inner = t.make_var()?; - Ok(Self(inner)) - } - } - - pub fn rand_f64>( - lo: f64, - up: f64, - s: S, - dtype: DType, - device: &Device, - ) -> Result { - let inner = Tensor::rand_f64_impl(lo, up, s, dtype, device, true)?; - Ok(Self(inner)) - } - - pub fn randn_f64>( - mean: f64, - std: f64, - s: S, - dtype: DType, - device: &Device, - ) -> Result { - let inner = Tensor::randn_f64_impl(mean, std, s, dtype, device, true)?; - Ok(Self(inner)) - } - - pub fn rand, T: crate::FloatDType>( - lo: T, - up: T, - s: S, - device: &Device, - ) -> Result { - let inner = Tensor::rand_impl(lo, up, s, device, true)?; - Ok(Self(inner)) - } - - pub fn randn, T: crate::FloatDType>( - mean: T, - std: T, - s: S, - device: &Device, - ) -> Result { - let inner = Tensor::randn_impl(mean, std, s, device, true)?; - Ok(Self(inner)) - } - - /// Creates a new tensor on the specified device using the content and shape of the input. - /// This is similar to `new` but the resulting tensor is a variable. - pub fn new(array: A, device: &Device) -> Result { - let shape = array.shape()?; - let inner = Tensor::new_impl(array, shape, device, true)?; - Ok(Self(inner)) - } - - pub fn from_vec, D: crate::WithDType>( - data: Vec, - shape: S, - device: &Device, - ) -> Result { - let inner = Tensor::from_vec_impl(data, shape, device, true)?; - Ok(Self(inner)) - } - - pub fn from_slice, D: crate::WithDType>( - array: &[D], - shape: S, - device: &Device, - ) -> Result { - let inner = Tensor::new_impl(array, shape.into(), device, true)?; - Ok(Self(inner)) - } - - pub fn as_detached_tensor(&self) -> Tensor { - self.0.detach() - } - - pub fn as_tensor(&self) -> &Tensor { - &self.0 - } - - /// Consumes this `Var` and return the underlying tensor. - pub fn into_inner(self) -> Tensor { - self.0 - } - - /// Sets the content of the inner tensor, this does not require a mutable reference as inner - /// mutability is used. - pub fn set(&self, src: &Tensor) -> Result<()> { - if self.same_storage(src) { - let msg = "cannot set a variable to a tensor that is derived from its value"; - Err(Error::CannotSetVar { msg }.bt())? - } - let (mut dst, layout) = self.storage_mut_and_layout(); - if !layout.is_contiguous() { - let msg = "cannot set a non-contiguous variable"; - Err(Error::CannotSetVar { msg }.bt())? - } - let (src, src_l) = src.storage_and_layout(); - if layout.shape() != src_l.shape() { - Err(Error::ShapeMismatchBinaryOp { - lhs: layout.shape().clone(), - rhs: src_l.shape().clone(), - op: "set", - } - .bt())? - } - src.copy_strided_src(&mut dst, layout.start_offset(), src_l)?; - Ok(()) - } -} diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/bilinear_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/bilinear_tests.rs deleted file mode 100644 index 3716df6ba4..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/tests/bilinear_tests.rs +++ /dev/null @@ -1,525 +0,0 @@ -use candle_core::{test_device, Device, IndexOp, Result, Tensor}; - -// ============================================================================ -// PyTorch Exact Comparison Tests -// ============================================================================ -// These tests compare against exact PyTorch outputs to ensure correctness - -/* Test corresponds to PyTorch: -import torch -import torch.nn.functional as F -input = torch.arange(16, dtype=torch.float32).reshape(1, 1, 4, 4) -output = F.interpolate(input, size=(8, 8), mode='bilinear', align_corners=False) -*/ -fn bilinear_pytorch_2x_upscale(dev: &Device) -> Result<()> { - let input = Tensor::arange(0f32, 16f32, dev)?.reshape((1, 1, 4, 4))?; - let output = input.upsample_bilinear2d(8, 8, false)?; - - // PyTorch expected output (verified from PyTorch 2.10.0) - let expected = Tensor::new( - &[ - 0.0000f32, 0.2500, 0.7500, 1.2500, 1.7500, 2.2500, 2.7500, 3.0000, 1.0000, 1.2500, - 1.7500, 2.2500, 2.7500, 3.2500, 3.7500, 4.0000, 3.0000, 3.2500, 3.7500, 4.2500, 4.7500, - 5.2500, 5.7500, 6.0000, 5.0000, 5.2500, 5.7500, 6.2500, 6.7500, 7.2500, 7.7500, 8.0000, - 7.0000, 7.2500, 7.7500, 8.2500, 8.7500, 9.2500, 9.7500, 10.0000, 9.0000, 9.2500, - 9.7500, 10.2500, 10.7500, 11.2500, 11.7500, 12.0000, 11.0000, 11.2500, 11.7500, - 12.2500, 12.7500, 13.2500, 13.7500, 14.0000, 12.0000, 12.2500, 12.7500, 13.2500, - 13.7500, 14.2500, 14.7500, 15.0000, - ], - dev, - )? - .reshape((1, 1, 8, 8))?; - - let diff = (&output - &expected)?.abs()?.flatten_all()?.max(0)?; - let max_diff = diff.to_vec0::()?; - - assert!( - max_diff < 1e-4, - "Max difference {} exceeds threshold 1e-4", - max_diff - ); - Ok(()) -} - -/* Test corresponds to PyTorch: -import torch -import torch.nn.functional as F -input = torch.arange(64, dtype=torch.float32).reshape(1, 1, 8, 8) -output = F.interpolate(input, size=(4, 4), mode='bilinear', align_corners=False) -*/ -fn bilinear_pytorch_downscale(dev: &Device) -> Result<()> { - let input = Tensor::arange(0f32, 64f32, dev)?.reshape((1, 1, 8, 8))?; - let output = input.upsample_bilinear2d(4, 4, false)?; - - // PyTorch expected output - let expected = Tensor::new( - &[ - 4.5f32, 6.5, 8.5, 10.5, 20.5, 22.5, 24.5, 26.5, 36.5, 38.5, 40.5, 42.5, 52.5, 54.5, - 56.5, 58.5, - ], - dev, - )? - .reshape((1, 1, 4, 4))?; - - let diff = (&output - &expected)?.abs()?.flatten_all()?.max(0)?; - let max_diff = diff.to_vec0::()?; - - assert!( - max_diff < 1e-4, - "Max difference {} exceeds threshold 1e-4", - max_diff - ); - Ok(()) -} - -/* Test corresponds to PyTorch: -import torch -import torch.nn.functional as F -torch.manual_seed(42) -input = torch.randn(1, 2, 4, 4, dtype=torch.float32) -output = F.interpolate(input, size=(8, 8), mode='bilinear', align_corners=False) -*/ -fn bilinear_pytorch_multi_channel(dev: &Device) -> Result<()> { - // Using fixed seed data from PyTorch (seed=42) - let input = Tensor::new( - &[ - // Channel 0 - 1.9269f32, 1.4873, 0.9007, -2.1055, 0.6784, -1.2345, -0.0431, -1.6047, -0.7521, 1.6487, - -0.3925, -1.4036, -0.7279, -0.5594, -0.7688, 0.7624, // Channel 1 - 1.6423f32, -0.1596, -0.4974, 0.4396, -0.7581, 1.0783, 0.8008, 1.6806, 1.2791, 1.2964, - 0.6105, 1.3347, -0.2316, 0.0418, -0.2516, 0.8599, - ], - dev, - )? - .reshape((1, 2, 4, 4))?; - - let output = input.upsample_bilinear2d(8, 8, false)?; - - assert_eq!(output.dims(), &[1, 2, 8, 8]); - - // Verify output is finite and in reasonable range - let output_vec = output.flatten_all()?.to_vec1::()?; - for &val in &output_vec { - assert!(val.is_finite(), "Output contains non-finite value"); - } - - // Check first row of channel 0 from PyTorch output - let output_ch0_row0 = output.i((0, 0, 0, ..))?.to_vec1::()?; - let expected_ch0_row0 = [ - 1.9269f32, 1.8170, 1.5972, 1.3406, 1.0474, 0.1492, -1.3540, -2.1055, - ]; - - for (i, (&out, &exp)) in output_ch0_row0 - .iter() - .zip(expected_ch0_row0.iter()) - .enumerate() - { - let diff = (out - exp).abs(); - assert!( - diff < 1e-3, - "Channel 0, row 0, index {} differs: got {}, expected {}, diff {}", - i, - out, - exp, - diff - ); - } - - // Check first row of channel 1 from PyTorch output - let output_ch1_row0 = output.i((0, 1, 0, ..))?.to_vec1::()?; - let expected_ch1_row0 = [ - 1.6423f32, 1.1918, 0.2909, -0.2440, -0.4129, -0.2632, 0.2053, 0.4396, - ]; - - for (i, (&out, &exp)) in output_ch1_row0 - .iter() - .zip(expected_ch1_row0.iter()) - .enumerate() - { - let diff = (out - exp).abs(); - assert!( - diff < 1e-3, - "Channel 1, row 0, index {} differs: got {}, expected {}, diff {}", - i, - out, - exp, - diff - ); - } - - Ok(()) -} - -/* Test corresponds to PyTorch: -import torch -import torch.nn.functional as F -input = torch.tensor([[[[1.0, 2.0], [3.0, 4.0]]]], dtype=torch.float32) -output = F.interpolate(input, size=(4, 4), mode='bilinear', align_corners=True) -*/ -fn bilinear_pytorch_align_corners_true(dev: &Device) -> Result<()> { - let input = Tensor::from_vec(vec![1.0f32, 2.0, 3.0, 4.0], (1, 1, 2, 2), dev)?; - let output = input.upsample_bilinear2d(4, 4, true)?; - - // PyTorch expected output with align_corners=True - let expected = Tensor::new( - &[ - 1.0f32, 1.3333, 1.6667, 2.0, 1.6667, 2.0, 2.3333, 2.6667, 2.3333, 2.6667, 3.0, 3.3333, - 3.0, 3.3333, 3.6667, 4.0, - ], - dev, - )? - .reshape((1, 1, 4, 4))?; - - let diff = (&output - &expected)?.abs()?.flatten_all()?.max(0)?; - let max_diff = diff.to_vec0::()?; - - assert!( - max_diff < 1e-3, - "Max difference {} exceeds threshold 1e-3", - max_diff - ); - - // Verify corners are exactly preserved with align_corners=True - let output_vec = output.flatten_all()?.to_vec1::()?; - assert!( - (output_vec[0] - 1.0).abs() < 1e-5, - "Top-left corner not preserved" - ); - assert!( - (output_vec[3] - 2.0).abs() < 1e-5, - "Top-right corner not preserved" - ); - assert!( - (output_vec[12] - 3.0).abs() < 1e-5, - "Bottom-left corner not preserved" - ); - assert!( - (output_vec[15] - 4.0).abs() < 1e-5, - "Bottom-right corner not preserved" - ); - - Ok(()) -} - -/* Test corresponds to PyTorch: -import torch -import torch.nn.functional as F -input = torch.arange(16, dtype=torch.float32).reshape(1, 1, 4, 4) -output = F.interpolate(input, scale_factor=2.0, mode='bilinear', align_corners=False) -*/ -fn bilinear_pytorch_scale_factor(dev: &Device) -> Result<()> { - let input = Tensor::arange(0f32, 16f32, dev)?.reshape((1, 1, 4, 4))?; - let output_scale = input.upsample_bilinear2d_with_scale(2.0, 2.0, false)?; - let output_size = input.upsample_bilinear2d(8, 8, false)?; - - // scale_factor=2.0 should produce identical results to size=(8, 8) - let diff = (&output_scale - &output_size)? - .abs()? - .flatten_all()? - .max(0)?; - let max_diff = diff.to_vec0::()?; - - assert!( - max_diff < 1e-6, - "scale_factor and size methods differ by {}", - max_diff - ); - - Ok(()) -} - -/* Test corresponds to PyTorch: -import torch -import torch.nn.functional as F -input = torch.arange(24, dtype=torch.float32).reshape(1, 1, 4, 6) -output = F.interpolate(input, size=(8, 12), mode='bilinear', align_corners=False) -*/ -fn bilinear_pytorch_non_square_exact(dev: &Device) -> Result<()> { - let input = Tensor::arange(0f32, 24f32, dev)?.reshape((1, 1, 4, 6))?; - let output = input.upsample_bilinear2d(8, 12, false)?; - - // PyTorch expected output (verified from PyTorch 2.10.0) - #[rustfmt::skip] - let expected = Tensor::new( - &[ - 0.0f32, 0.25, 0.75, 1.25, 1.75, 2.25, 2.75, 3.25, 3.75, 4.25, 4.75, 5.0, - 1.5, 1.75, 2.25, 2.75, 3.25, 3.75, 4.25, 4.75, 5.25, 5.75, 6.25, 6.5, - 4.5, 4.75, 5.25, 5.75, 6.25, 6.75, 7.25, 7.75, 8.25, 8.75, 9.25, 9.5, - 7.5, 7.75, 8.25, 8.75, 9.25, 9.75, 10.25, 10.75, 11.25, 11.75, 12.25, 12.5, - 10.5, 10.75, 11.25, 11.75, 12.25, 12.75, 13.25, 13.75, 14.25, 14.75, 15.25, 15.5, - 13.5, 13.75, 14.25, 14.75, 15.25, 15.75, 16.25, 16.75, 17.25, 17.75, 18.25, 18.5, - 16.5, 16.75, 17.25, 17.75, 18.25, 18.75, 19.25, 19.75, 20.25, 20.75, 21.25, 21.5, - 18.0, 18.25, 18.75, 19.25, 19.75, 20.25, 20.75, 21.25, 21.75, 22.25, 22.75, 23.0, - ], - dev, - )? - .reshape((1, 1, 8, 12))?; - - let diff = (&output - &expected)?.abs()?.flatten_all()?.max(0)?; - let max_diff = diff.to_vec0::()?; - - assert!( - max_diff < 1e-4, - "Max difference {} exceeds threshold 1e-4", - max_diff - ); - Ok(()) -} - -/* Test corresponds to PyTorch: -import torch -import torch.nn.functional as F -input = torch.tensor([[[[5.0]]]], dtype=torch.float32) -output = F.interpolate(input, size=(3, 3), mode='bilinear', align_corners=False) -*/ -fn bilinear_pytorch_tiny_1x1_to_3x3(dev: &Device) -> Result<()> { - let input = Tensor::new(&[5.0f32], dev)?.reshape((1, 1, 1, 1))?; - let output = input.upsample_bilinear2d(3, 3, false)?; - - // PyTorch expected output: all values should be 5.0 - let expected = Tensor::new(&[5.0f32, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0], dev)? - .reshape((1, 1, 3, 3))?; - - let diff = (&output - &expected)?.abs()?.flatten_all()?.max(0)?; - let max_diff = diff.to_vec0::()?; - - assert!( - max_diff < 1e-6, - "Max difference {} exceeds threshold 1e-6", - max_diff - ); - Ok(()) -} - -/* Test corresponds to PyTorch: -import torch -import torch.nn.functional as F -input = torch.tensor([[[[2.0, 8.0]]]], dtype=torch.float32) -output = F.interpolate(input, size=(3, 6), mode='bilinear', align_corners=False) -*/ -fn bilinear_pytorch_tiny_1x2_to_3x6(dev: &Device) -> Result<()> { - let input = Tensor::new(&[2.0f32, 8.0], dev)?.reshape((1, 1, 1, 2))?; - let output = input.upsample_bilinear2d(3, 6, false)?; - - // PyTorch expected output - #[rustfmt::skip] - let expected = Tensor::new( - &[ - 2.0f32, 2.0, 4.0, 6.0, 8.0, 8.0, - 2.0, 2.0, 4.0, 6.0, 8.0, 8.0, - 2.0, 2.0, 4.0, 6.0, 8.0, 8.0, - ], - dev, - )? - .reshape((1, 1, 3, 6))?; - - let diff = (&output - &expected)?.abs()?.flatten_all()?.max(0)?; - let max_diff = diff.to_vec0::()?; - - assert!( - max_diff < 1e-6, - "Max difference {} exceeds threshold 1e-6", - max_diff - ); - Ok(()) -} - -/* Test corresponds to PyTorch: -import torch -import torch.nn.functional as F -torch.manual_seed(123) -input = torch.randn(1, 1, 64, 64, dtype=torch.float32) -output = F.interpolate(input, size=(128, 128), mode='bilinear', align_corners=False) -*/ -fn bilinear_pytorch_large_64x64_to_128x128(dev: &Device) -> Result<()> { - // Test large tensor for numerical stability - // We'll just verify dimensions and that output is finite - use candle_core::DType; - - let input = Tensor::randn(0f32, 1f32, (1, 1, 64, 64), dev)?; - let output = input.upsample_bilinear2d(128, 128, false)?; - - assert_eq!(output.dims(), &[1, 1, 128, 128]); - assert_eq!(output.dtype(), DType::F32); - - // Verify all values are finite - let output_vec = output.flatten_all()?.to_vec1::()?; - for &val in &output_vec { - assert!( - val.is_finite(), - "Large tensor output contains non-finite value" - ); - } - - // Verify output is in reasonable range (should be similar to input range) - let min_val = output_vec.iter().copied().fold(f32::INFINITY, f32::min); - let max_val = output_vec.iter().copied().fold(f32::NEG_INFINITY, f32::max); - - assert!( - min_val > -10.0 && max_val < 10.0, - "Large tensor output values out of expected range: min={}, max={}", - min_val, - max_val - ); - - Ok(()) -} - -// ============================================================================ -// Dimension and Shape Tests (Consolidated) -// ============================================================================ -// These tests verify correct output dimensions for various input configurations - -fn bilinear_output_dimensions(dev: &Device) -> Result<()> { - // Test 1: Non-square dimensions - let t1 = Tensor::arange(0f32, 32f32, dev)?.reshape((1, 1, 4, 8))?; - let out1 = t1.upsample_bilinear2d(6, 12, false)?; - assert_eq!(out1.dims(), &[1, 1, 6, 12], "Non-square upscale failed"); - - // Test 2: Batch processing - let t2 = Tensor::arange(0f32, 192f32, dev)?.reshape((4, 3, 4, 4))?; - let out2 = t2.upsample_bilinear2d(8, 8, false)?; - assert_eq!(out2.dims(), &[4, 3, 8, 8], "Batch processing failed"); - - // Test 3: Asymmetric scale factors - let t3 = Tensor::arange(0f32, 16f32, dev)?.reshape((1, 1, 4, 4))?; - let out3 = t3.upsample_bilinear2d_with_scale(2.0, 3.0, false)?; - assert_eq!(out3.dims(), &[1, 1, 8, 12], "Asymmetric scale failed"); - - // Test 4: Fractional scale factors - let t4 = Tensor::arange(0f32, 16f32, dev)?.reshape((1, 1, 4, 4))?; - let out4 = t4.upsample_bilinear2d_with_scale(1.5, 1.5, false)?; - assert_eq!(out4.dims(), &[1, 1, 6, 6], "Fractional scale failed"); - - // Test 5: Single pixel output - let t5 = Tensor::arange(0f32, 16f32, dev)?.reshape((1, 1, 4, 4))?; - let out5 = t5.upsample_bilinear2d(1, 1, false)?; - assert_eq!(out5.dims(), &[1, 1, 1, 1], "Single pixel output failed"); - let val = out5.flatten_all()?.to_vec1::()?[0]; - assert!(val.is_finite(), "Single pixel value is not finite"); - - // Test 6: Large scale factor - let t6 = Tensor::arange(0f32, 4f32, dev)?.reshape((1, 1, 2, 2))?; - let out6 = t6.upsample_bilinear2d_with_scale(5.0, 5.0, false)?; - assert_eq!(out6.dims(), &[1, 1, 10, 10], "Large scale factor failed"); - - Ok(()) -} - -// ============================================================================ -// Special Behavior Tests -// ============================================================================ - -fn bilinear_identity(dev: &Device) -> Result<()> { - // Test that upsampling to the same size returns an identical tensor - let t = Tensor::arange(0f32, 16f32, dev)?.reshape((1, 1, 4, 4))?; - let output = t.upsample_bilinear2d(4, 4, false)?; - - let diff = (&t - &output)?.abs()?.flatten_all()?.max(0)?; - assert!(diff.to_vec0::()? < 1e-6); - Ok(()) -} - -fn bilinear_align_corners_difference(dev: &Device) -> Result<()> { - // Test that align_corners parameter produces different results - let t = Tensor::arange(0f32, 16f32, dev)?.reshape((1, 1, 4, 4))?; - - let output_false = t.upsample_bilinear2d(8, 8, false)?; - let output_true = t.upsample_bilinear2d(8, 8, true)?; - - // Results should be different between align_corners modes - let diff = (&output_false - &output_true)?.abs()?.sum_all()?; - assert!(diff.to_vec0::()? > 0.1); - Ok(()) -} - -// ============================================================================ -// Test Device Macros -// ============================================================================ - -// PyTorch exact comparison tests -test_device!( - bilinear_pytorch_2x_upscale, - bilinear_pytorch_2x_upscale_cpu, - bilinear_pytorch_2x_upscale_gpu, - bilinear_pytorch_2x_upscale_metal -); - -test_device!( - bilinear_pytorch_downscale, - bilinear_pytorch_downscale_cpu, - bilinear_pytorch_downscale_gpu, - bilinear_pytorch_downscale_metal -); - -test_device!( - bilinear_pytorch_multi_channel, - bilinear_pytorch_multi_channel_cpu, - bilinear_pytorch_multi_channel_gpu, - bilinear_pytorch_multi_channel_metal -); - -test_device!( - bilinear_pytorch_align_corners_true, - bilinear_pytorch_align_corners_true_cpu, - bilinear_pytorch_align_corners_true_gpu, - bilinear_pytorch_align_corners_true_metal -); - -test_device!( - bilinear_pytorch_scale_factor, - bilinear_pytorch_scale_factor_cpu, - bilinear_pytorch_scale_factor_gpu, - bilinear_pytorch_scale_factor_metal -); - -test_device!( - bilinear_pytorch_non_square_exact, - bilinear_pytorch_non_square_exact_cpu, - bilinear_pytorch_non_square_exact_gpu, - bilinear_pytorch_non_square_exact_metal -); - -test_device!( - bilinear_pytorch_tiny_1x1_to_3x3, - bilinear_pytorch_tiny_1x1_to_3x3_cpu, - bilinear_pytorch_tiny_1x1_to_3x3_gpu, - bilinear_pytorch_tiny_1x1_to_3x3_metal -); - -test_device!( - bilinear_pytorch_tiny_1x2_to_3x6, - bilinear_pytorch_tiny_1x2_to_3x6_cpu, - bilinear_pytorch_tiny_1x2_to_3x6_gpu, - bilinear_pytorch_tiny_1x2_to_3x6_metal -); - -test_device!( - bilinear_pytorch_large_64x64_to_128x128, - bilinear_pytorch_large_64x64_to_128x128_cpu, - bilinear_pytorch_large_64x64_to_128x128_gpu, - bilinear_pytorch_large_64x64_to_128x128_metal -); - -// Dimension tests (consolidated) -test_device!( - bilinear_output_dimensions, - bilinear_output_dimensions_cpu, - bilinear_output_dimensions_gpu, - bilinear_output_dimensions_metal -); - -// Special behavior tests -test_device!( - bilinear_identity, - bilinear_identity_cpu, - bilinear_identity_gpu, - bilinear_identity_metal -); - -test_device!( - bilinear_align_corners_difference, - bilinear_align_corners_difference_cpu, - bilinear_align_corners_difference_gpu, - bilinear_align_corners_difference_metal -); diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/conv_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/conv_tests.rs deleted file mode 100644 index e492bebab8..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/tests/conv_tests.rs +++ /dev/null @@ -1,894 +0,0 @@ -use anyhow::Result; -use candle_core::{test_device, test_utils, Device, IndexOp, Tensor}; - -/* This test is based on the following script. -import torch -torch.manual_seed(4242) - -t = torch.randn((1, 4, 5)) -w = torch.randn((2, 4, 3)) -print(t.flatten()) -print(w.flatten()) -res = torch.nn.functional.conv1d(t, w) -print(res.flatten()) -res = torch.nn.functional.conv1d(t, w, padding=1) -print(res.flatten()) - -w_t = w.transpose(0, 1) -res = torch.nn.functional.conv_transpose1d(t, w_t) -print(res.shape) -print(res) -res = torch.nn.functional.conv_transpose1d(t, w_t, groups=2) -print(res.shape) -print(res) -*/ -fn conv1d(dev: &Device) -> Result<()> { - let t = Tensor::new( - &[ - 0.4056f32, -0.8689, -0.0773, -1.5630, 1.2279, -0.9287, -1.7030, 0.1370, 0.1866, 0.4145, - 1.8025, -0.1536, 2.2013, -0.6836, 0.2477, 1.3127, -0.6957, 0.3278, -1.0124, 0.5599, - ], - dev, - )? - .reshape((1, 4, 5))?; - let w = Tensor::new( - &[ - -0.8404f32, -0.3490, 0.0130, 1.3123, 0.1763, -1.9249, 1.4270, 0.9421, 0.8670, -0.7181, - -1.1111, 0.8869, -1.2429, 1.8357, 1.6052, -1.3844, 0.3951, -1.2036, 0.6686, 1.6261, - -0.6451, -0.0840, -1.4247, 0.5512, - ], - dev, - )? - .reshape((2, 4, 3))?; - let res = t.conv1d(&w, 0, 1, 1, 1)?; - assert_eq!(res.dims(), [1, 2, 3]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [2.6357, -1.3336, 4.1393, -1.1784, 3.5675, 0.5069] - ); - let res = t.conv1d(&w, /*padding*/ 1, 1, 1, 1)?; - assert_eq!(res.dims(), [1, 2, 5]); - // Same as pytorch default padding: use zeros. - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [2.4509, 2.6357, -1.3336, 4.1393, 0.5657, 1.8091, -1.1784, 3.5675, 0.5069, 3.3352] - ); - let res = { - let t = Tensor::cat(&[&t.zeros_like()?, &t, &t.zeros_like()?], 0)?; - t.conv1d(&w, /*padding*/ 1, 1, 1, 1)? - }; - assert_eq!(res.dims(), [3, 2, 5]); - // Same as pytorch default padding: use zeros. - assert_eq!( - test_utils::to_vec1_round(&res.i(0)?.flatten_all()?, 4)?, - [0., 0., 0., 0., 0., 0., 0., 0., 0., 0.] - ); - assert_eq!( - test_utils::to_vec1_round(&res.i(1)?.flatten_all()?, 4)?, - [2.4509, 2.6357, -1.3336, 4.1393, 0.5657, 1.8091, -1.1784, 3.5675, 0.5069, 3.3352] - ); - - let w = w.transpose(0, 1)?; - // The CPU kernels applied in the contiguous and non contiguous cases are different. - for w in [w.clone(), w.contiguous()?] { - let res = t.conv_transpose1d(&w, 0, 0, 1, 1, 1)?; - assert_eq!(res.dims(), [1, 2, 7]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [ - 0.0699, -1.2899, 8.3018, 5.5873, 2.4572, -2.6143, -0.0706, 1.8765, 4.8318, 1.1538, - 4.7076, -5.9745, -0.8276, 1.621 - ], - ); - let res = t.conv_transpose1d(&w, 0, 0, 1, 1, 2)?; - assert_eq!(res.dims(), [1, 4, 7]); - assert_eq!( - test_utils::to_vec2_round(&res.squeeze(0)?, 4)?, - [ - [-1.5596, -1.8099, 2.0407, 4.8764, -0.1743, -0.735, -0.7819], - [0.7816, 3.8152, -0.5926, 2.2515, -5.1844, -0.3157, 1.4721], - [1.6295, 0.52, 6.2611, 0.7109, 2.6315, -1.8793, 0.7113], - [1.0949, 1.0166, 1.7464, 2.4561, -0.79, -0.5119, 0.1488] - ] - ); - } - Ok(()) -} - -fn conv1d_small(dev: &Device) -> Result<()> { - let t = Tensor::new(&[0.4056f32, -0.8689, -0.0773, -1.5630], dev)?.reshape((1, 1, 4))?; - let w = Tensor::new(&[1f32, 0., 0.], dev)?.reshape((1, 1, 3))?; - let res = t.conv1d(&w, 0, 1, 1, 1)?; - assert_eq!(res.dims(), [1, 1, 2]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [0.4056, -0.8689] - ); - let res = t.conv1d(&w, /*padding*/ 1, 1, 1, 1)?; - assert_eq!(res.dims(), [1, 1, 4]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [0.0, 0.4056, -0.8689, -0.0773], - ); - Ok(()) -} - -/* This test is based on the following script. -import torch -torch.manual_seed(4242) - -t = torch.randn((1, 4, 5, 5)) -w = torch.randn((2, 4, 3, 3)) -print(t.flatten()) -print(w.flatten()) -res = torch.nn.functional.conv2d(t, w) -print(res.flatten()) - -w_t = w.transpose(0, 1) -res = torch.nn.functional.conv_transpose2d(t, w_t) -print(res.shape) -print(res) - -res = torch.nn.functional.conv2d(t, w, dilation=2) -print(res.shape) -print(res[0]) - -res = torch.nn.functional.conv_transpose2d(t, w_t, dilation=2) -print(res.shape) -print(res) -*/ -fn conv2d(dev: &Device) -> Result<()> { - let t = Tensor::new( - &[ - 0.4056f32, -0.8689, -0.0773, -1.5630, -2.8012, -1.5059, 0.3972, 1.0852, 0.4997, 3.0616, - 1.6541, 0.0964, -0.8338, -1.6523, -0.8323, -0.1699, 0.0823, 0.3526, 0.6843, 0.2395, - 1.2279, -0.9287, -1.7030, 0.1370, 0.6047, 0.3770, -0.6266, 0.3529, 2.2013, -0.6836, - 0.2477, 1.3127, -0.2260, 0.2622, -1.2974, -0.8140, -0.8404, -0.3490, 0.0130, 1.3123, - 1.7569, -0.3956, -1.8255, 0.1727, -0.3538, 2.6941, 1.0529, 0.4219, -0.2071, 1.1586, - 0.4717, 0.3865, -0.5690, -0.5010, -0.1310, 0.7796, 0.6630, -0.2021, 2.6090, 0.2049, - 0.6466, -0.5042, -0.0603, -1.6538, -1.2429, 1.8357, 1.6052, -1.3844, 0.3323, -1.3712, - 0.9634, -0.4799, -0.6451, -0.0840, -1.4247, 0.5512, -0.1747, -0.5509, -0.3742, 0.3790, - -0.4431, -0.4720, -0.7890, 0.2620, 0.7875, 0.5377, -0.6779, -0.8088, 1.9098, 1.2006, - -0.8, -0.4983, 1.5480, 0.8265, -0.1025, 0.5138, 0.5748, 0.3821, -0.4607, 0.0085, - ], - dev, - )?; - let w = Tensor::new( - &[ - -0.9325f32, 0.6451, -0.8537, 0.2378, 0.8764, -0.1832, 0.2987, -0.6488, -0.2273, - -2.4184, -0.1192, -0.4821, -0.5079, -0.5766, -2.4729, 1.6734, 0.4558, 0.2851, 1.1514, - -0.9013, 1.0662, -0.1817, -0.0259, 0.1709, 0.5367, 0.7513, 0.8086, -2.2586, -0.5027, - 0.9141, -1.3086, -1.3343, -1.5669, -0.1657, 0.7958, 0.1432, 0.3896, -0.4501, 0.1667, - 0.0714, -0.0952, 1.2970, -0.1674, -0.3178, 1.0677, 0.3060, 0.7080, 0.1914, 1.1679, - -0.3602, 1.9265, -1.8626, -0.5112, -0.0982, 0.2621, 0.6565, 0.5908, 1.0089, -0.1646, - 1.8032, -0.6286, 0.2016, -0.3370, 1.2555, 0.8009, -0.6488, -0.4652, -1.5685, 1.5860, - 0.5583, 0.4623, 0.6026, - ], - dev, - )?; - let t = t.reshape((1, 4, 5, 5))?; - let w = w.reshape((2, 4, 3, 3))?; - let res = t.conv2d(&w, 0, 1, 1, 1)?; - assert_eq!(res.dims(), [1, 2, 3, 3]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [ - -4.2812, 2.0923, 5.2187, 7.5184, 0.752, -14.9426, 10.0087, 4.391, 0.2918, 1.6715, - 10.389, 3.6023, -4.2808, 0.2672, 5.3646, -5.2023, -2.1955, -9.4075 - ] - ); - let res = { - let t = Tensor::cat(&[&t.zeros_like()?, &t, &t.zeros_like()?], 0)?; - t.conv2d(&w, 0, 1, 1, 1)? - }; - assert_eq!(res.dims(), [3, 2, 3, 3]); - assert_eq!( - test_utils::to_vec1_round(&res.i(0)?.flatten_all()?, 4)?, - [0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.] - ); - assert_eq!( - test_utils::to_vec1_round(&res.i(1)?.flatten_all()?, 4)?, - [ - -4.2812, 2.0923, 5.2187, 7.5184, 0.752, -14.9426, 10.0087, 4.391, 0.2918, 1.6715, - 10.389, 3.6023, -4.2808, 0.2672, 5.3646, -5.2023, -2.1955, -9.4075 - ] - ); - - let res = t.conv_transpose2d(&w.transpose(0, 1)?, 0, 0, 1, 1)?; - - assert_eq!(res.dims(), [1, 2, 7, 7]); - assert_eq!( - test_utils::to_vec3_round(&res.i(0)?, 4)?, - [ - [ - [-1.9918, 2.6797, -0.4599, -1.6037, 1.4131, -2.4012, 2.9277], - [1.8016, -3.5361, 1.0757, 3.5395, -8.2168, -3.2023, 0.5375], - [0.8243, 1.8675, 7.8929, -4.0746, -6.4415, 5.1139, 1.6889], - [0.2722, 8.9679, 3.3477, 1.8514, -4.2896, -3.8228, -7.5632], - [-8.5412, -5.8142, -7.1587, -1.6095, 0.4651, 0.2748, -2.0985], - [2.0833, -0.6482, -12.1692, -4.1284, -2.9765, -0.0656, -4.5114], - [5.307, 2.6957, 2.3087, 1.0478, 0.7808, -1.1519, -0.9579] - ], - [ - [1.089, 0.1872, -0.6408, -0.9897, 0.8503, 1.1019, -0.9211], - [-0.1741, -0.2915, 4.2472, 1.9417, 1.65, 0.6303, -4.7131], - [1.6555, 2.4026, -2.9293, 2.9953, 0.5328, 3.5873, -0.9621], - [-1.4289, -3.2787, 4.1747, -6.0341, -4.6341, -5.7945, 4.142], - [7.5973, 6.4431, 5.9872, 2.1639, -8.6566, 3.3143, -3.4059], - [-0.8775, -3.048, 11.6543, 0.6442, 2.3218, -0.4765, 1.1516], - [-5.5423, -2.5188, 1.0754, -0.0563, -2.9386, -1.1504, 1.0171] - ] - ] - ); - - // Dilations. - let res = t.conv2d(&w, 0, 1, 2, 1)?; - assert_eq!(res.dims(), [1, 2, 1, 1]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [2.45, -2.3504], - ); - - // Transpose and dilations. - let res = t.conv_transpose2d(&w.transpose(0, 1)?, 0, 0, 1, 2)?; - assert_eq!(res.dims(), [1, 2, 9, 9]); - assert_eq!( - test_utils::to_vec3_round(&res.i(0)?, 4)?, - [ - [ - [-1.9918, 3.1652, -0.6778, -4.3442, 4.4351, 0.6652, -3.0124, -0.6031, 2.9277], - [2.7036, -1.7156, -0.3969, 1.0516, 1.6381, -2.8886, -0.205, 2.4682, -1.0499], - [-0.9459, 3.1631, 3.707, -4.8369, -8.5166, -1.4496, -2.7559, -3.2698, 1.4376], - [-0.2157, 3.7786, -2.0252, -4.2633, 3.6731, -1.5142, 5.9391, -0.2622, -0.141], - [-6.8121, -3.1744, 1.5945, 3.0637, -9.6088, 1.4446, 2.9489, -3.0082, -7.3822], - [0.2371, 3.3303, 0.3861, 2.2646, -4.6784, 4.1235, -0.0109, 0.3176, -0.03], - [-2.5339, -2.9564, -3.4518, -4.4594, -9.1873, -1.9709, -0.4676, 0.51, -3.5024], - [4.007, 0.3067, -2.2954, 1.1105, -0.1992, 1.6372, -2.9268, 0.2807, -1.2787], - [5.307, 1.1317, 1.3518, 0.9049, 3.8116, -0.4075, -0.8874, -0.2241, -0.9579] - ], - [ - [1.089, -0.6483, 0.0726, -0.4752, -1.3283, 1.7103, 1.0703, 0.1076, -0.9211], - [-0.8629, 0.1376, 0.3202, 2.0955, 0.9696, 2.8988, -1.0012, 1.5049, -0.1278], - [1.9286, -1.5255, -2.9563, 2.4589, 3.3611, -0.6951, 0.3525, -1.7724, -5.9861], - [1.1226, 2.1561, 3.6417, 4.7546, -0.692, 4.4126, -5.1902, 6.0805, 2.3185], - [1.0111, 0.3604, 0.6432, -3.6605, 7.9517, -9.2955, -5.2988, -3.7803, -2.0642], - [3.3172, -1.7967, -3.6576, -2.0942, 1.3158, 0.112, -1.7405, 2.9167, 0.7957], - [5.1001, 1.8995, -1.8639, 1.1262, 9.9629, 2.683, -3.6319, -1.1607, 0.5856], - [-4.8445, -0.5642, 4.2317, 0.0856, 1.2267, -0.5712, 1.736, 1.0997, 0.6908], - [-5.5423, -1.1831, -1.2176, 0.0843, 0.0446, -0.7545, -2.4798, -0.0827, 1.0171] - ] - ] - ); - - Ok(()) -} - -/* This test is based on the following script. -import torch -torch.manual_seed(4242) - -t = torch.randn((1, 2, 3, 3)) -w = torch.randn((1, 2, 1, 1)) -print(t.flatten()) -print(w.flatten()) -res = torch.nn.functional.conv2d(t, w) -print(res.flatten()) - -w_t = w.transpose(0, 1) -res = torch.nn.functional.conv_transpose2d(t, w_t) -print(res.shape) -print(res.flatten()) - -t_t = w.transpose(0, 1) -res = torch.nn.functional.conv_transpose2d(t_t, w) -print(res.shape) -print(res.flatten()) -*/ -fn conv2d_small(dev: &Device) -> Result<()> { - let t = Tensor::new( - &[ - 0.4056f32, -0.8689, 0.6843, 0.2395, 1.2279, -0.9287, -1.7030, 0.1370, 0.1866, 0.4145, - -0.6266, 0.3529, 2.2013, -0.6836, 0.2477, 1.3127, -0.6957, 0.3278, - ], - dev, - )?; - let w = Tensor::new(&[-0.9259f32, 1.3017], dev)?; - let t = t.reshape((1, 2, 3, 3))?; - let w = w.reshape((1, 2, 1, 1))?; - let res = t.conv2d(&w, 0, 1, 1, 1)?; - assert_eq!(res.dims(), [1, 1, 3, 3]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [0.164, -0.0111, -0.1742, 2.6437, -2.0268, 1.1823, 3.2855, -1.0324, 0.2539] - ); - let res = t.conv2d(&w, 2, 1, 1, 1)?; - assert_eq!(res.dims(), [1, 1, 7, 7]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [ - 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.1640, - -0.0111, -0.1742, 0.0, 0.0, 0.0, 0.0, 2.6437, -2.0268, 1.1823, 0.0, 0.0, 0.0, 0.0, - 3.2855, -1.0324, 0.2539, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, - 0.0, 0.0, 0.0, 0.0 - ] - ); - - let res = t.conv_transpose2d(&w.transpose(0, 1)?, 0, 0, 1, 1)?; - assert_eq!(res.dims(), [1, 1, 3, 3]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [0.164, -0.0111, -0.1742, 2.6437, -2.0268, 1.1823, 3.2855, -1.0324, 0.2539], - ); - let res = t.transpose(0, 1)?.conv_transpose2d(&w, 0, 0, 1, 1)?; - assert_eq!(res.dims(), [2, 2, 3, 3]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [ - -0.3755, 0.8045, -0.6336, -0.2218, -1.1369, 0.8599, 1.5768, -0.1268, -0.1728, 0.528, - -1.131, 0.8908, 0.3118, 1.5984, -1.2089, -2.2168, 0.1783, 0.2429, -0.3838, 0.5802, - -0.3268, -2.0382, 0.6329, -0.2293, -1.2154, 0.6441, -0.3035, 0.5396, -0.8156, 0.4594, - 2.8654, -0.8898, 0.3224, 1.7087, -0.9056, 0.4267 - ] - ); - Ok(()) -} - -fn conv2d_smaller(dev: &Device) -> Result<()> { - let t = Tensor::new( - &[ - 0.4056f32, -0.8689, 0.6843, 0.2395, 1.2279, -0.9287, -1.7030, 0.1370, 0.1866, - ], - dev, - )?; - let w = Tensor::new(&[1f32, 1., 1., 1., 1., 1., 1., 1., 1.], dev)?; - let t = t.reshape((1, 1, 3, 3))?; - let w = w.reshape((1, 1, 3, 3))?; - let res = t.conv2d(&w, 0, 1, 1, 1)?; - assert_eq!(res.dims(), [1, 1, 1, 1]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [-0.6197] - ); - Ok(()) -} - -/* This test is based on the following script. -import torch -torch.manual_seed(4242) - -t = torch.randn((1, 2, 4, 2)) -w = torch.randn((1, 2, 1, 1)) -print(t.flatten()) -print(w.flatten()) -res = torch.nn.functional.conv2d(t, w) -print(res.flatten()) -*/ -fn conv2d_non_square(dev: &Device) -> Result<()> { - let t = Tensor::new( - &[ - 0.4056f32, -0.8689, -0.0773, -1.5630, -2.8012, -1.5059, 0.3972, 1.0852, 0.4997, 3.0616, - 1.6541, 0.0964, -0.8338, -1.6523, -0.8323, -0.1699, - ], - dev, - )?; - let w = Tensor::new(&[-1.1351f32, 1.3841], dev)?; - let t = t.reshape((1, 2, 4, 2))?; - let w = w.reshape((1, 2, 1, 1))?; - let res = t.conv2d(&w, 0, 1, 1, 1)?; - assert_eq!(res.dims(), [1, 1, 4, 2]); - assert_eq!( - test_utils::to_vec1_round(&res.flatten_all()?, 4)?, - [0.2312, 5.2238, 2.3772, 1.9076, 2.0256, -0.5776, -1.6028, -1.467] - ); - Ok(()) -} - -/* -import torch -torch.manual_seed(4242) - -t = torch.randn((1, 4, 5, 5), requires_grad=True) -w = torch.randn((2, 4, 3, 3), requires_grad=True) -print(t.flatten()) -print(w.flatten()) -res = torch.nn.functional.conv2d(t, w) -print(res.flatten()) -loss = (res ** 2).sum() -print(loss) -loss.backward() -print(t.grad.shape) -print(t.grad.flatten()) -print(w.grad.shape) -print(w.grad.flatten()) - -t.grad.zero_() -w.grad.zero_() -res = torch.nn.functional.conv2d(t, w, stride=2) -print(res.flatten()) -loss = (res ** 2).sum() -print(loss) -loss.backward() -print(t.grad.shape) -print(t.grad[0]) -print(w.grad.shape) -print(w.grad[0]) -*/ -fn conv2d_grad(dev: &Device) -> Result<()> { - // conv-transposes are not implemented for metal - use candle_core::Var; - let t = Var::from_slice( - &[ - 0.4056f32, -0.8689, -0.0773, -1.5630, -2.8012, -1.5059, 0.3972, 1.0852, 0.4997, 3.0616, - 1.6541, 0.0964, -0.8338, -1.6523, -0.8323, -0.1699, 0.0823, 0.3526, 0.6843, 0.2395, - 1.2279, -0.9287, -1.7030, 0.1370, 0.6047, 0.3770, -0.6266, 0.3529, 2.2013, -0.6836, - 0.2477, 1.3127, -0.2260, 0.2622, -1.2974, -0.8140, -0.8404, -0.3490, 0.0130, 1.3123, - 1.7569, -0.3956, -1.8255, 0.1727, -0.3538, 2.6941, 1.0529, 0.4219, -0.2071, 1.1586, - 0.4717, 0.3865, -0.5690, -0.5010, -0.1310, 0.7796, 0.6630, -0.2021, 2.6090, 0.2049, - 0.6466, -0.5042, -0.0603, -1.6538, -1.2429, 1.8357, 1.6052, -1.3844, 0.3323, -1.3712, - 0.9634, -0.4799, -0.6451, -0.0840, -1.4247, 0.5512, -0.1747, -0.5509, -0.3742, 0.3790, - -0.4431, -0.4720, -0.7890, 0.2620, 0.7875, 0.5377, -0.6779, -0.8088, 1.9098, 1.2006, - -0.8, -0.4983, 1.5480, 0.8265, -0.1025, 0.5138, 0.5748, 0.3821, -0.4607, 0.0085, - ], - (1, 4, 5, 5), - dev, - )?; - let w = Var::from_slice( - &[ - -0.9325f32, 0.6451, -0.8537, 0.2378, 0.8764, -0.1832, 0.2987, -0.6488, -0.2273, - -2.4184, -0.1192, -0.4821, -0.5079, -0.5766, -2.4729, 1.6734, 0.4558, 0.2851, 1.1514, - -0.9013, 1.0662, -0.1817, -0.0259, 0.1709, 0.5367, 0.7513, 0.8086, -2.2586, -0.5027, - 0.9141, -1.3086, -1.3343, -1.5669, -0.1657, 0.7958, 0.1432, 0.3896, -0.4501, 0.1667, - 0.0714, -0.0952, 1.2970, -0.1674, -0.3178, 1.0677, 0.3060, 0.7080, 0.1914, 1.1679, - -0.3602, 1.9265, -1.8626, -0.5112, -0.0982, 0.2621, 0.6565, 0.5908, 1.0089, -0.1646, - 1.8032, -0.6286, 0.2016, -0.3370, 1.2555, 0.8009, -0.6488, -0.4652, -1.5685, 1.5860, - 0.5583, 0.4623, 0.6026, - ], - (2, 4, 3, 3), - dev, - )?; - let res = t.conv2d(&w, 0, 1, 1, 1)?; - let loss = res.sqr()?.sum_all()?; - assert_eq!(test_utils::to_vec0_round(&loss, 2)?, 741.12f32); - let grads = loss.backward()?; - let grad_t = grads.get(&t).unwrap(); - let grad_w = grads.get(&w).unwrap(); - assert_eq!(grad_t.dims(), [1, 4, 5, 5]); - assert_eq!(grad_w.dims(), [2, 4, 3, 3]); - assert_eq!( - test_utils::to_vec1_round(&grad_t.flatten_all()?, 2)?, - [ - 9.29, -2.84, -5.71, 3.38, -7.71, -19.15, 7.02, 29.1, 9.34, 34.73, -22.87, 24.35, - -39.88, -14.01, 21.08, 9.94, 13.63, -34.68, 11.21, -6.26, 7.72, -6.32, -16.64, -1.08, - -20.22, 21.73, -0.37, -4.06, 5.82, -3.65, -30.73, 14.55, 87.7, 31.6, 4.53, -89.78, - -75.37, -57.43, -7.56, 92.96, 18.79, -4.63, -159.75, -42.47, -47.26, 52.88, 37.32, - 49.0, 12.82, 2.01, -8.98, 20.18, 16.62, 12.06, 15.38, 20.0, 2.57, -15.22, 72.62, - -10.75, 2.25, -31.2, 3.75, -0.2, 9.76, -0.68, 5.21, -40.44, -22.59, -61.61, 17.28, - 20.41, 37.55, 5.23, 6.81, 23.54, 23.62, -9.99, -9.13, 4.87, -35.06, -26.1, 63.48, - 25.81, -39.21, -70.68, -46.96, 2.33, 41.81, 82.42, -28.63, -11.78, -35.33, -10.28, - -28.57, -9.13, 7.21, -9.05, -9.62, -11.25 - ] - ); - assert_eq!( - test_utils::to_vec1_round(&grad_w.flatten_all()?, 2)?, - [ - -28.92, -22.88, -141.23, 73.35, 61.07, 47.81, -20.0, -73.71, -41.82, -13.59, 21.5, - 28.72, 28.57, -46.85, -90.19, 143.61, 16.68, 7.43, 18.88, -90.81, -20.29, 54.79, 82.63, - 22.94, 77.81, -16.39, -13.2, 9.34, -40.39, -26.62, 5.33, -60.91, 9.09, -59.37, 7.08, - 58.64, 5.55, 20.52, 2.5, -17.25, -6.8, 22.21, 30.15, -7.52, -37.46, 5.67, 22.58, 9.03, - 47.05, 17.61, 37.31, -98.13, -14.61, -4.8, -6.36, 44.69, 23.34, 8.37, -13.52, 80.05, - -34.24, -16.36, -12.31, 1.92, -33.62, -14.1, -49.23, -7.39, 11.5, -9.98, 9.66, 29.6 - ] - ); - - // Same as before but with stride. - let res = t.conv2d(&w, 0, 2, 1, 1)?; - let loss = res.sqr()?.sum_all()?; - assert_eq!(test_utils::to_vec0_round(&loss, 2)?, 277.16f32); - let grads = loss.backward()?; - let grad_t = grads.get(&t).unwrap(); - let grad_w = grads.get(&w).unwrap(); - assert_eq!(grad_t.dims(), [1, 4, 5, 5]); - assert_eq!(grad_w.dims(), [2, 4, 3, 3]); - assert_eq!( - test_utils::to_vec3_round(&grad_t.i(0)?, 2)?, - [ - [ - [9.29, -7.03, 0.94, 3.49, -7.71], - [-1.8, -7.82, 8.9, 8.46, 7.43], - [-25.84, 22.09, -19.27, -0.22, 1.69], - [4.02, 18.53, -18.37, 2.3, -24.51], - [7.72, -9.68, -12.34, 5.6, -20.22] - ], - [ - [21.73, 3.39, -18.27, 3.86, -3.65], - [8.25, 3.73, 30.73, -8.61, -11.93], - [-72.15, -15.36, -17.53, -12.32, -1.61], - [-22.32, -7.79, -91.82, 6.44, -37.69], - [52.88, 14.44, 42.75, 9.88, 2.01] - ], - [ - [-8.98, 9.91, 6.75, -4.68, 15.38], - [4.93, -0.33, 9.94, -1.46, 14.78], - [13.62, -30.63, 3.96, -3.58, -4.48], - [-14.13, 1.19, -34.43, 3.08, -33.83], - [17.28, 12.94, 31.83, -3.35, 6.81] - ], - [ - [23.54, 6.98, -24.52, 0.52, 4.87], - [9.65, 6.18, 1.71, -25.23, -4.93], - [-54.99, -23.66, 3.19, -3.73, 18.58], - [-21.35, -10.39, -39.88, 28.73, -30.76], - [-9.13, 11.12, -14.0, -8.23, -11.25] - ] - ] - ); - assert_eq!( - test_utils::to_vec3_round(&grad_w.i(0)?, 2)?, - [ - [ - [28.34, -7.91, -45.75], - [21.03, 3.86, 29.86], - [0.72, -36.58, -35.28] - ], - [ - [-16.04, 11.53, -16.38], - [29.62, -16.32, -48.35], - [57.5, 28.29, 25.81] - ], - [ - [2.93, -19.6, 1.57], - [27.15, 53.88, -24.64], - [12.74, -22.6, -26.2] - ], - [ - [-0.18, -14.86, -6.82], - [-19.55, -2.72, 45.9], - [-2.54, 36.97, 27.11] - ] - ] - ); - - // Replicate the issue from https://github.com/huggingface/candle/issues/1212 - let res = t.i((.., .., 0..4, 0..4))?.conv2d(&w, 0, 2, 1, 1)?; - let loss = res.sqr()?.sum_all()?; - assert_eq!(test_utils::to_vec0_round(&loss, 2)?, 21.12f32); - let grads = loss.backward()?; - let grad_t = grads.get(&t).unwrap(); - let grad_w = grads.get(&w).unwrap(); - assert_eq!(grad_t.dims(), [1, 4, 5, 5]); - assert_eq!(grad_w.dims(), [2, 4, 3, 3]); - assert_eq!( - test_utils::to_vec3_round(&grad_t.i(0)?, 2)?, - [ - [ - [9.29, -7.03, 7.87, 0.0, 0.0], - [-1.8, -7.82, 5.9, 0.0, 0.0], - [-3.12, 4.49, 5.52, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.0] - ], - [ - [21.73, 3.39, 4.77, 0.0, 0.0], - [8.25, 3.73, 27.61, 0.0, 0.0], - [-20.55, -5.61, -2.77, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.0] - ], - [ - [-8.98, 9.91, -7.15, 0.0, 0.0], - [4.93, -0.33, 4.56, 0.0, 0.0], - [-6.7, -5.76, -8.05, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.0] - ], - [ - [23.54, 6.98, -10.0, 0.0, 0.0], - [9.65, 6.18, 18.72, 0.0, 0.0], - [3.29, -5.27, 0.79, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.0], - [0.0, 0.0, 0.0, 0.0, 0.0] - ] - ] - ); - assert_eq!( - test_utils::to_vec3_round(&grad_w.i(0)?, 2)?, - [ - [ - [-3.47, 7.44, 0.66], - [12.89, -3.4, -9.29], - [-14.16, -0.83, 7.14] - ], - [ - [-3.23, 5.37, -3.02], - [-2.12, -11.24, 1.94], - [6.97, 7.2, 2.99] - ], - [ - [-4.04, -3.31, 4.87], - [-6.68, -5.68, 1.73], - [-5.54, 4.32, 0.52] - ], - [[-4.72, 1.5, 4.72], [3.79, 4.04, 6.76], [-4.6, 5.8, 6.93]] - ] - ); - - // Conv Transpose 2d Test - //tested against following python - - // import torch - // torch.manual_seed(4242) - // padding = 4 - // outpadding = 2 - // dilation = 3 - // stride = 3 - // input = torch.randn((1, 4, 7, 5), requires_grad=True) - // kernel = torch.randn((4, 2, 3, 5), requires_grad=True) - // print("input", input.flatten()) - // print("kernel", kernel.flatten()) - // res = torch.nn.functional.conv_transpose2d( - // input, - // kernel, - // stride=stride, - // padding=padding, - // dilation=dilation, - // output_padding=outpadding, - // ) - // res.retain_grad() - // print(res.shape) - // loss = (res**2).sum() - // print(loss) - // loss.backward() - // print(input.grad.shape) - // print("input grad", torch.round(input.grad, decimals=1)) - // print(kernel.grad.shape) - // print("kernel grad", torch.round(kernel.grad.flatten(), decimals=1)) - - let padding = 4; - let outpadding = 2; - let dilation = 3; - let stride = 3; - - let t = Var::from_slice( - &[ - 0.4056_f32, -0.8689, -0.0773, -1.5630, -2.8012, -1.5059, 0.3972, 1.0852, 0.4997, - 3.0616, 1.6541, 0.0964, -0.8338, -1.6523, -0.8323, -0.1699, 0.0823, 0.3526, 0.6843, - 0.2395, 1.2279, -0.9287, -1.7030, 0.1370, 0.6047, 0.3770, -0.6266, 0.3529, 2.2013, - -0.6836, 0.2477, 1.3127, -0.2260, 0.2622, -1.2974, -0.8140, -0.8404, -0.3490, 0.0130, - 1.3123, 1.7569, -0.3956, -1.8255, 0.1727, -0.3538, 2.6941, 1.0529, 0.4219, -0.2071, - 1.1586, 0.4717, 0.3865, -0.5690, -0.5010, -0.1310, 0.7796, 0.6630, -0.2021, 2.6090, - 0.2049, 0.6466, -0.5042, -0.0603, -1.6538, -1.2429, 1.8357, 1.6052, -1.3844, 0.3323, - -1.3712, 0.9634, -0.4799, -0.6451, -0.0840, -1.4247, 0.5512, -0.1747, -0.5509, -0.3742, - 0.3790, -0.4431, -0.4720, -0.7890, 0.2620, 0.5411, -1.1715, -2.4997, 2.3249, -0.8912, - -0.4733, -0.5701, -2.8888, -1.4112, -0.5471, -0.9234, -1.1660, 0.4189, -0.7465, - -0.6473, 0.1402, 0.7875, 0.5377, -0.6779, -0.8088, -0.4864, -0.2312, 0.9279, 0.1264, - 1.5480, 0.8265, -0.1025, 0.5138, -0.2512, 0.1576, 1.2705, 0.3641, -0.9325, 0.6451, - -0.8537, 0.2378, 0.1794, 0.2752, -0.3687, -1.1149, -0.1410, -0.5829, -0.0892, 1.4258, - -2.2789, 0.5270, 0.1825, 1.7007, -0.5263, -0.2954, 0.4440, 0.5537, 0.3492, 0.6186, - 1.6475, 0.2219, - ], - (1, 4, 7, 5), - dev, - )?; - - #[rustfmt::skip] - let w = Var::from_slice( - &[ - -1.1744_f32, 0.3266, 2.5893, 1.0142, 0.1763, 0.7752, 0.6604, 0.2029, -0.2145, 0.7234, - -0.3441, -1.5400, -0.6333, 0.6613, 0.2083, 0.6230, -1.7002, 0.3393, 0.4049, 1.0762, - 0.2723, 1.4181, 0.0029, -0.2122, 1.7668, 1.4168, 0.3320, -0.2719, 0.7932, -0.7204, - 0.4447, 0.1211, 0.5908, 1.0089, -0.1646, 1.8033, -0.6286, 0.2016, -0.3370, 1.2555, - 0.8009, -0.6488, -0.4652, -1.5685, 1.5860, 0.5583, 0.4623, 0.6026, 0.8828, 2.4990, - 0.6811, -0.3369, 1.3320, 1.7669, -1.1067, 1.2958, -0.9415, -0.9655, -0.4462, 0.7181, - 0.5181, -1.1658, -1.8467, -0.7763, 1.2769, 0.8651, 0.9890, 1.5092, 0.7207, -0.8481, - 0.7417, 0.3375, -1.2685, 1.4572, 1.0915, 0.1093, -0.8550, -0.5831, -0.6309, -0.2509, - 0.5220, -0.0914, 0.7900, 0.1096, 0.3258, 0.2723, -1.0942, -0.3393, -0.1653, 0.5732, - -0.8014, 1.8194, -1.9023, 0.2127, 1.8636, -0.8979, 0.1927, -0.2778, 0.3105, 0.0071, - -1.1823, 0.2476, -0.7178, -1.3821, 1.0769, -0.4376, -0.9967, -0.1227, 1.6197, -1.0604, - 0.1372, 0.8141, -0.6163, 0.7304, -0.8285, 2.0636, -0.7176, 0.2495, -0.2581, -0.4478, - ], - (4, 2, 3, 5), - dev, - )?; - let res = t.conv_transpose2d(&w, padding, outpadding, stride, dilation)?; - let loss = res.sqr()?.sum_all()?; - assert_eq!(test_utils::to_vec0_round(&loss, 0)?, 2904.0); - let grads = loss.backward()?; - - let grad_t = grads.get(&t).unwrap(); - let grad_w = grads.get(&w).unwrap(); - assert_eq!(grad_t.dims(), [1, 4, 7, 5]); - assert_eq!(grad_w.dims(), [4, 2, 3, 5]); - - assert_eq!( - test_utils::to_vec1_round(&grad_w.flatten_all()?, 1)?, - [ - // torch gets 89.1 - -89.0, -135.3, 136.7, 102.0, -53.4, 117.9, 118.6, -43.9, -218.0, -58.5, -114.3, -150.0, - -15.6, 172.1, 66.3, -64.3, -27.9, -19.8, 31.7, 62.1, 5.5, 92.6, 28.2, -29.6, 55.9, - 52.7, -72.7, -119.8, 53.8, -25.5, 128.8, 19.3, 68.0, 190.9, -64.1, -86.2, -111.2, - 106.6, -67.7, 37.8, 115.9, 50.4, -77.7, -54.9, 22.3, -4.6, 89.8, 61.7, 122.4, 192.6, - -27.8, -104.6, 57.0, 166.4, 27.1, 6.1, 18.7, -93.2, 31.5, 168.2, -3.7, -99.5, -55.5, - -10.8, 17.5, 20.8, 16.9, 43.8, 42.0, -89.2, 18.8, -9.6, -84.1, 212.6, 19.7, -50.0, - -52.0, -40.0, -166.6, -73.2, -10.8, -73.3, 31.5, -23.4, -79.3, -27.0, -84.4, -42.9, - -20.3, 51.8, -16.7, 76.3, -120.5, -65.8, 96.5, -10.7, -45.9, -88.1, 65.4, -7.0, -1.5, - 92.8, -25.1, -114.2, -5.8, -14.8, -51.2, -20.7, 54.2, -79.8, 47.7, -29.2, -8.8, 53.5, - -28.4, 85.0, -18.3, 107.0, 28.3, -71.8 - ] - ); - - assert_eq!( - test_utils::to_vec3_round(&grad_t.i(0)?, 1)?, - [ - [ - [32.3, -41.6, -24.0, 14.1, 17.6], - [-11.8, 72.5, 87.6, 46.4, 61.5], - [115.0, 108.5, -48.6, -63.4, -50.0], - [51.3, 5.4, 31.3, 91.1, -30.9], - [52.7, 92.8, -68.0, -47.0, 83.0], - // pytorch gets -107.1 - [-10.2, -107.0, -5.4, 213.1, -31.4], - [-2.4, 65.1, 9.2, -146.2, -24.2] - ], - [ - [-72.6, -63.9, -61.9, 45.3, 33.0], - [79.3, -0.5, -26.2, 78.2, 42.7], - [90.9, 141.6, 40.1, -62.7, 37.0], - [32.8, 198.2, -0.8, -31.1, 27.3], - // torch gets 48.0 - [34.5, 34.9, -47.9, 127.6, -12.3], - [-61.4, -3.2, -2.9, -10.9, -16.6], - [74.6, 60.1, -68.9, 34.5, -50.4] - ], - [ - [37.5, -56.9, -43.6, -13.5, -9.9], - [40.0, 97.3, 28.6, 14.2, -30.1], - [-22.3, -126.3, -68.8, -8.2, 26.1], - [-32.9, 37.3, 108.5, -54.8, 29.6], - [34.9, -176.9, -125.0, -28.3, -13.9], - [-54.9, 142.6, 62.1, -80.4, -65.6], - [7.4, -91.1, -67.6, 35.0, 39.7] - ], - [ - [-57.2, -40.9, -10.1, 32.6, 29.4], - [18.7, -18.0, 29.5, -1.2, 59.2], - [-14.0, -74.4, 19.8, -117.0, 58.2], - [-21.8, 163.5, -71.1, -99.0, 80.9], - [-58.9, -10.9, 93.8, -139.6, 98.0], - // torch gets 54.5 - [-54.4, 135.3, 6.0, -79.1, 134.6], - [27.5, -76.0, 43.4, -2.8, -7.8] - ] - ] - ); - - // Test the same, but then with the following properties, t & w are unmodified. - let padding = 1; - let outpadding = 1; - let dilation = 1; - let stride = 2; - - let res = t.conv_transpose2d(&w, padding, outpadding, stride, dilation)?; - let loss = res.sqr()?.sum_all()?; - assert_eq!(test_utils::to_vec0_round(&loss, 0)?, 3627.0); // torch gives 3626.8560 - - let grads = loss.backward()?; - - let grad_t = grads.get(&t).unwrap(); - let grad_w = grads.get(&w).unwrap(); - assert_eq!(grad_t.dims(), [1, 4, 7, 5]); - assert_eq!(grad_w.dims(), [4, 2, 3, 5]); - - #[rustfmt::skip] - assert_eq!( - test_utils::to_vec3_round(&grad_t.i(0)?, 1)?, - [ - [ - [ 13.2, -40.7, -9.7, -47.3, -82.7], - [ -98.2, 9.7, 57.7, -6.2, 180.7], - [ 100.2, 24.1, 3.7, -100.5, -48.1], - [ -0.3, 13.5, -2.9, 80.0, -49.8], - [ 47.2, -25.6, -74.4, 61.2, -18.4], - [ 4.6, -69.5, 27.9, 66.5, -88.1], - // 4th column on next row; torch is 4.2 - [ -12.0, 79.2, -40.0, 4.1, -97.1], - ], - [ - [ -42.2, -36.5, -51.1, 7.5, 32.3], - [ 74.1, -44.6, -68.8, 19.5, 7.7], - [ 137.1, 54.2, 153.8, -58.0, 45.5], - [ 24.4, -56.8, 9.7, -41.0, -14.5], - [ -3.7, 72.6, 8.3, 134.8, 40.5], - [ 43.2, -56.9, -47.5, -89.4, -95.4], - [ 68.2, 108.1, -80.0, 57.0, -121.1] - ], - [ - [ 31.1, -11.4, -34.8, 33.1, -44.2], - [ 29.4, -31.6, -40.2, 13.7, 13.1], - [ -0.8, -83.8, -7.8, -17.3, 78.2], - [ 12.0, -118.7, 137.5, -76.7, 50.8], - [ -28.7, -114.2, -3.7, -96.3, -13.8], - [ -31.8, 28.5, -14.3, 4.6, 13.4], - [ 28.0, -0.2, -38.9, -29.7, -59.0] - ], - [ - [ -16.8, 38.5, 15.5, 26.6, 48.9], - [ 14.5, 49.6, -24.8, 65.6, 61.7], - [ 22.1, -64.7, -4.3, -51.0, 36.3], - [ 31.0, -88.9, 47.1, -123.5, -3.8], - [ -14.8, -39.8, 128.2, -110.3, 42.6], - // 1st column on next row; torch is -7.2 - [ -7.1, 95.3, -21.3, -58.7, -13.9], - [ 26.9, 21.3, 16.1, 70.3, 32.1] - ] - ] - ); - - #[rustfmt::skip] - assert_eq!( - test_utils::to_vec1_round(&grad_w.flatten_all()?, 1)?, - [ - // 2nd value; torch gets -3.2, 3rd value; torch gets 221.8 - -2.460e+01, -3.100e+00, 2.219e+02, 7.400e+00, 5.620e+01, - 7.420e+01, 7.830e+01, 8.900e+00, 1.050e+01, 2.810e+01, - 5.100e+00, -1.046e+02, -1.572e+02, 8.710e+01, -9.840e+01, - -4.230e+01, -1.898e+02, 1.860e+01, -3.570e+01, 9.810e+01, - 4.680e+01, 1.182e+02, 4.020e+01, -1.900e+00, 1.508e+02, - 1.094e+02, 1.018e+02, -4.620e+01, 1.591e+02, -2.320e+01, - // 5th value; torch gets 7.1 - -8.450e+01, -4.600e+00, 6.330e+01, 1.123e+02, -7.000e+00, - 1.101e+02, -6.620e+01, 2.090e+01, -5.120e+01, 8.990e+01, - 9.050e+01, -6.990e+01, 6.800e+01, -9.250e+01, 1.380e+02, - 4.720e+01, 4.710e+01, 6.210e+01, 8.870e+01, 2.098e+02, - 3.870e+01, -1.390e+01, 6.270e+01, 1.484e+02, -9.920e+01, - -4.200e+01, -1.505e+02, -1.480e+01, -2.620e+01, 8.220e+01, - -3.350e+01, -2.260e+01, -1.198e+02, -5.080e+01, 1.259e+02, - 5.600e+01, 9.270e+01, 1.209e+02, 6.590e+01, -8.330e+01, - 7.000e+00, -2.600e+01, -1.133e+02, 3.870e+01, 4.020e+01, - -6.300e+00, -8.710e+01, -5.150e+01, -8.510e+01, 2.000e-01, - 3.640e+01, -6.100e+00, 6.590e+01, -2.700e+00, 6.550e+01, - // 4th value; torch gets 3.8 - 5.300e+00, -6.760e+01, -4.270e+01, -3.900e+00, 2.880e+01, - 5.260e+01, 6.170e+01, -1.203e+02, -1.610e+01, 7.740e+01, - -1.008e+02, -1.070e+01, -9.900e+00, 3.300e+00, -2.620e+01, - -4.440e+01, 2.580e+01, -6.920e+01, -4.220e+01, 1.108e+02, - 1.240e+01, -3.440e+01, -2.800e+00, 7.880e+01, -6.690e+01, - 1.480e+01, 2.310e+01, -4.260e+01, -1.500e+00, -4.760e+01, - 5.350e+01, -2.260e+01, 8.000e-01, -3.840e+01, -2.500e+00 - ] - ); - - Ok(()) -} - -test_device!(conv1d, conv1d_cpu, conv1d_gpu, conv1d_metal); -test_device!( - conv1d_small, - conv1d_small_cpu, - conv1d_small_gpu, - conv1d_small_metal -); -test_device!(conv2d, conv2d_cpu, conv2d_gpu, conv2d_metal); -test_device!( - conv2d_non_square, - conv2d_non_square_cpu, - conv2d_non_square_gpu, - conv2d_non_square_metal -); -test_device!( - conv2d_small, - conv2d_small_cpu, - conv2d_small_gpu, - conv2d_small_metal -); -test_device!( - conv2d_smaller, - conv2d_smaller_cpu, - conv2d_smaller_gpu, - conv2d_smaller_metal -); -test_device!( - conv2d_grad, - conv2d_grad_cpu, - conv2d_grad_gpu, - conv2_grad_metal -); diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/custom_op_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/custom_op_tests.rs deleted file mode 100644 index cea9b90cac..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/tests/custom_op_tests.rs +++ /dev/null @@ -1,182 +0,0 @@ -use candle_core::backend::BackendStorage; -use candle_core::cpu_backend; -use candle_core::test_utils::to_vec1_round; -use candle_core::{CpuStorage, CustomOp1, DType, Device, Error, Layout, Result, Shape, Tensor}; - -fn fwd(v: T, alpha: f64) -> T { - if v.is_sign_positive() { - v - } else { - let alpha = T::from(alpha).unwrap_or(T::nan()); - (v.exp() - T::one()) * alpha - } -} - -struct Elu { - alpha: f64, -} - -impl CustomOp1 for Elu { - fn name(&self) -> &'static str { - "elu" - } - - fn cpu_fwd(&self, s: &CpuStorage, l: &Layout) -> Result<(CpuStorage, Shape)> { - let storage = candle_core::map_dtype!( - "elu", - s, - |s| cpu_backend::unary_map(s, l, |v| fwd(v, self.alpha)), - (F8E4M3, BF16, F16, F32, F64) - ); - Ok((storage, l.shape().clone())) - } -} - -#[test] -fn custom_op1_no_backward() -> Result<()> { - let cpu = &Device::Cpu; - let t = Tensor::arange(0u32, 12u32, cpu)?.to_dtype(DType::F32)?; - let t = (t - 5.)?; - let elu_t = t.apply_op1_no_bwd(&Elu { alpha: 1. })?; - assert_eq!( - to_vec1_round(&elu_t, 4)?, - &[-0.9933, -0.9817, -0.9502, -0.8647, -0.6321, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0] - ); - Ok(()) -} - -// Define a similar struct as Elu but with backward support. -fn bwd(v: T, alpha: f64) -> T { - if v.is_sign_positive() { - T::one() - } else { - let alpha = T::from(alpha).unwrap_or(T::nan()); - v.exp() * alpha - } -} - -struct EluBackward { - alpha: f64, -} - -impl CustomOp1 for EluBackward { - fn name(&self) -> &'static str { - "elu-bwd" - } - - fn cpu_fwd(&self, s: &CpuStorage, l: &Layout) -> Result<(CpuStorage, Shape)> { - let storage = candle_core::map_dtype!( - "elu-bwd", - s, - |s| cpu_backend::unary_map(s, l, |v| bwd(v, self.alpha)), - (F8E4M3, BF16, F16, F32, F64) - ); - Ok((storage, l.shape().clone())) - } -} - -struct EluWithBackward(Elu); - -impl EluWithBackward { - fn new(alpha: f64) -> Self { - Self(Elu { alpha }) - } -} - -impl CustomOp1 for EluWithBackward { - fn name(&self) -> &'static str { - "elu" - } - - fn cpu_fwd(&self, s: &CpuStorage, l: &Layout) -> Result<(CpuStorage, Shape)> { - self.0.cpu_fwd(s, l) - } - - fn bwd(&self, arg: &Tensor, _res: &Tensor, grad_res: &Tensor) -> Result> { - let alpha = self.0.alpha; - let bwd = arg.apply_op1(EluBackward { alpha })?; - Ok(Some(grad_res.mul(&bwd)?)) - } -} - -#[test] -fn custom_op1_with_backward() -> Result<()> { - let cpu = &Device::Cpu; - let t = candle_core::Var::new(&[-2f32, 0f32, 2f32], cpu)?; - let elu_t = t.apply_op1(EluWithBackward::new(2.))?; - assert_eq!(to_vec1_round(&elu_t, 4)?, &[-1.7293, 0.0, 2.0]); - - let grads = elu_t.backward()?; - let grad_x = grads.get(&t).unwrap(); - assert_eq!(to_vec1_round(grad_x, 4)?, [0.2707, 1.0, 1.0]); - - Ok(()) -} - -impl candle_core::InplaceOp1 for Elu { - fn name(&self) -> &'static str { - "elu" - } - - fn cpu_fwd(&self, s: &mut CpuStorage, _l: &Layout) -> Result<()> { - let alpha = self.alpha; - match s { - CpuStorage::F8E4M3(s) => s.iter_mut().for_each(|v| *v = fwd(*v, alpha)), - CpuStorage::BF16(s) => s.iter_mut().for_each(|v| *v = fwd(*v, alpha)), - CpuStorage::F16(s) => s.iter_mut().for_each(|v| *v = fwd(*v, alpha)), - CpuStorage::F32(s) => s.iter_mut().for_each(|v| *v = fwd(*v, alpha)), - CpuStorage::F64(s) => s.iter_mut().for_each(|v| *v = fwd(*v, alpha)), - _ => candle_core::bail!("unsupported dtype for inplace elu"), - } - Ok(()) - } -} - -#[test] -fn inplace_op1() -> Result<()> { - let cpu = &Device::Cpu; - let t = Tensor::arange(0u32, 12u32, cpu)?.to_dtype(DType::F32)?; - let t = (t - 5.)?; - t.inplace_op1(&Elu { alpha: 1. })?; - assert_eq!( - to_vec1_round(&t, 4)?, - &[-0.9933, -0.9817, -0.9502, -0.8647, -0.6321, 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0] - ); - Ok(()) -} - -#[cfg(all(feature = "ug", any(feature = "cuda", feature = "metal")))] -#[allow(clippy::approx_constant)] -#[test] -fn ug_op() -> Result<()> { - let kernel = { - use candle_ug::lang::op; - - let layout = candle_ug::Layout::from_shape(&[12]); - let ptr = op::Arg::ptr(candle_ug::DType::F32); - let src = op::load(ptr.id(), layout.clone(), candle_ug::DType::F32)?; - let src = op::unary(op::UnaryOp::Exp, src)?; - let st = op::store(ptr.id(), layout, src)?; - let kernel = op::Kernel::new("exp".to_string(), vec![ptr], vec![st]); - let opts: candle_ug::lower_op::Opts = Default::default(); - kernel.lower(&opts)? - }; - let device = if candle_core::utils::cuda_is_available() { - Device::new_cuda(0)? - } else if candle_core::utils::metal_is_available() { - Device::new_metal(0)? - } else { - candle_core::bail!("metal/cuda is mandatory for this test") - }; - let op = candle_core::UgIOp1::new("test", kernel, &device)?; - let t = Tensor::arange(0u32, 12u32, &device)?.to_dtype(DType::F32)?; - t.inplace_op1(&op)?; - assert_eq!( - to_vec1_round(&t, 2)?, - &[ - 1.0, 2.72, 7.39, 20.09, 54.6, 148.41, 403.43, 1096.63, 2980.96, 8103.08, 22026.47, - 59874.13 - ] - ); - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/display_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/display_tests.rs deleted file mode 100644 index 8ab8bb6ab7..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/tests/display_tests.rs +++ /dev/null @@ -1,84 +0,0 @@ -use anyhow::Result; -use candle_core::{DType, Device::Cpu, Tensor}; - -#[test] -fn display_scalar() -> Result<()> { - let t = Tensor::new(1234u32, &Cpu)?; - let s = format!("{t}"); - assert_eq!(&s, "[1234]\nTensor[[], u32]"); - let t = t.to_dtype(DType::F32)?.neg()?; - let s = format!("{}", (&t / 10.0)?); - assert_eq!(&s, "[-123.4000]\nTensor[[], f32]"); - let s = format!("{}", (&t / 1e8)?); - assert_eq!(&s, "[-1.2340e-5]\nTensor[[], f32]"); - let s = format!("{}", (&t * 1e8)?); - assert_eq!(&s, "[-1.2340e11]\nTensor[[], f32]"); - let s = format!("{}", (&t * 0.)?); - assert_eq!(&s, "[0.]\nTensor[[], f32]"); - Ok(()) -} - -#[test] -fn display_vector() -> Result<()> { - let t = Tensor::new::<&[u32; 0]>(&[], &Cpu)?; - let s = format!("{t}"); - assert_eq!(&s, "[]\nTensor[[0], u32]"); - let t = Tensor::new(&[0.1234567, 1.0, -1.2, 4.1, f64::NAN], &Cpu)?; - let s = format!("{t}"); - assert_eq!( - &s, - "[ 0.1235, 1.0000, -1.2000, 4.1000, NaN]\nTensor[[5], f64]" - ); - let t = (Tensor::ones(50, DType::F32, &Cpu)? * 42.)?; - let s = format!("\n{t}"); - let expected = r#" -[42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., - 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., - 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., - 42., 42.] -Tensor[[50], f32]"#; - assert_eq!(&s, expected); - let t = (Tensor::ones(11000, DType::F32, &Cpu)? * 42.)?; - let s = format!("{t}"); - assert_eq!( - &s, - "[42., 42., 42., ..., 42., 42., 42.]\nTensor[[11000], f32]" - ); - Ok(()) -} - -#[test] -fn display_multi_dim() -> Result<()> { - let t = (Tensor::ones((200, 100), DType::F32, &Cpu)? * 42.)?; - let s = format!("\n{t}"); - let expected = r#" -[[42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - ... - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.]] -Tensor[[200, 100], f32]"#; - assert_eq!(&s, expected); - let t = t.reshape(&[2, 1, 1, 100, 100])?; - let t = format!("\n{t}"); - let expected = r#" -[[[[[42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - ... - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.]]]], - [[[[42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - ... - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.]]]]] -Tensor[[2, 1, 1, 100, 100], f32]"#; - assert_eq!(&t, expected); - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/fortran_tensor_3d.pth b/v2/dependency-patches/candle-core-0.9.2/tests/fortran_tensor_3d.pth deleted file mode 100644 index bd50b03d9cd74135fa6f53e2bf39da2f7ac0879d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1486 zcmbVMON-M`6uxcRw3C_f{b*;vl}H8cq_1fM4rq~Vu3&5tikp&2uAN|L<|elT3Zfes z`~xoi8LnNr)ZgHra3gq5ax?akg5!b1*QDoseErTjwWv}cq+BK|u|vw_We|$co{j~Z zMnO1kxqGfH?E3L5-|QF23yAmmx15YFK)0-sEj;IlAasuEF|D;O?2<9xF#&?}#<-dQ*_3!wA}nJZmn5c(c}+fVNvudj z=N{boHv9_DnCqRZ5+lR86Bb6WEM~uRZtLQCw=`7nI)k}hL5W)3v?BD(;LAu`VHkz$&w_)rOI+dOSykWlHOd3LLTKWc|6b;pN%gI@N0_} N Result<()> { - let x = Var::new(&[3f32, 1., 4.], device)?; - let x = x.as_tensor(); - let y = (((x * x)? + x * 5f64)? + 4f64)?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!(x.to_vec1::()?, [3., 1., 4.]); - // y = x^2 + 5.x + 4 - assert_eq!(y.to_vec1::()?, [28., 10., 40.]); - // dy/dx = 2.x + 5 - assert_eq!(grad_x.to_vec1::()?, [11., 7., 13.]); - Ok(()) -} - -fn sum_grad(device: &Device) -> Result<()> { - let x = Var::new(&[3f32, 1., 4.], device)?; - let x = x.as_tensor(); - let y = (x.sqr()?.sum_keepdim(0)? * 2.)?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!(y.to_vec1::()?, [52.]); - // y = 2.x^2 so dy/dx = 4.x - assert_eq!(grad_x.to_vec1::()?, &[12., 4., 16.]); - - // Same test as before but squeezing on the last dimension. - let y = (x.sqr()?.sum_keepdim(0)? * 2.)?.squeeze(0)?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!(y.to_scalar::()?, 52.); - // y = 2.x^2 so dy/dx = 4.x - assert_eq!(grad_x.to_vec1::()?, &[12., 4., 16.]); - Ok(()) -} - -fn matmul_grad(device: &Device) -> Result<()> { - let data: Vec<_> = (0..12).map(|i| i as f32).collect(); - let x = Var::from_slice(&data, (2, 2, 3), device)?; - let data: Vec<_> = (0..12).map(|i| i as f32).collect(); - let y = Var::from_slice(&data, (2, 3, 2), device)?; - let c = x.matmul(&y)?; - let grads = c.backward()?; - let grad_x = grads.get(&x).context("no grad for x")?; - let grad_y = grads.get(&y).context("no grad for y")?; - assert_eq!(grad_x.shape(), &Shape::from((2, 2, 3))); - assert_eq!(grad_y.shape(), &Shape::from((2, 3, 2))); - assert_eq!( - &*grad_x.to_vec3::()?, - &[ - [[1., 5., 9.], [1., 5., 9.]], - [[13., 17., 21.], [13., 17., 21.]] - ] - ); - assert_eq!( - &*grad_y.to_vec3::()?, - &[ - [[3., 3.], [5., 5.], [7., 7.]], - [[15., 15.], [17., 17.], [19., 19.]] - ] - ); - Ok(()) -} - -// The simplest gradient descent, using scalar variable. -fn grad_descent(device: &Device) -> Result<()> { - let x = Var::new(0f32, device)?; - let learning_rate = 0.1; - for _step in 0..100 { - let xt = x.as_tensor(); - let c = ((xt - 4.2)? * (xt - 4.2)?)?; - let grads = c.backward()?; - let x_grad = grads.get(&x).context("no grad for x")?; - x.set(&(xt - x_grad * learning_rate)?)? - } - assert_eq!(x.to_scalar::()?, 4.199999); - Ok(()) -} - -fn unary_grad(device: &Device) -> Result<()> { - let x = Var::new(&[3f32, 1., 4., 0.15], device)?; - let x = x.as_tensor(); - let y = (x.log()? + 1.)?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!( - test_utils::to_vec1_round(&y, 4)?, - [2.0986, 1.0, 2.3863, -0.8971] - ); - assert_eq!( - test_utils::to_vec1_round(grad_x, 4)?, - [0.3333, 1.0, 0.25, 6.6667] - ); - let y = x.exp()?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!( - test_utils::to_vec1_round(&y, 4)?, - [20.0855, 2.7183, 54.5982, 1.1618] - ); - assert_eq!( - test_utils::to_vec1_round(grad_x, 4)?, - [20.0855, 2.7183, 54.5982, 1.1618] - ); - let y = x.exp()?.sqr()?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!( - test_utils::to_vec1_round(&y, 3)?, - [403.429, 7.389, 2980.958, 1.35] - ); - // exp(x)^2 = exp(2*x) - assert_eq!( - test_utils::to_vec1_round(grad_x, 2)?, - [806.86, 14.78, 5961.92, 2.7] - ); - let y = x.sin()?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!( - test_utils::to_vec1_round(&y, 4)?, - [0.1411, 0.8415, -0.7568, 0.1494], - ); - assert_eq!( - test_utils::to_vec1_round(grad_x, 4)?, - [-0.99, 0.5403, -0.6536, 0.9888], - ); - let y = x.cos()?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!( - test_utils::to_vec1_round(&y, 4)?, - [-0.99, 0.5403, -0.6536, 0.9888], - ); - assert_eq!( - test_utils::to_vec1_round(grad_x, 4)?, - [-0.1411, -0.8415, 0.7568, -0.1494], - ); - let y = x.sqr()?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!(y.to_vec1::()?, [9.0, 1.0, 16.0, 0.0225]); - assert_eq!(grad_x.to_vec1::()?, [6.0, 2.0, 8.0, 0.3]); - let y = x.sqr()?.sqrt()?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!(y.to_vec1::()?, [3.0, 1.0, 4.0, 0.15]); - assert_eq!(test_utils::to_vec1_round(grad_x, 4)?, [1.0, 1.0, 1.0, 1.0]); - let y = x.neg()?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!(y.to_vec1::()?, [-3.0, -1.0, -4.0, -0.15]); - assert_eq!(grad_x.to_vec1::()?, [-1.0, -1.0, -1.0, -1.0]); - let y = x.affine(0.2, 1.)?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!(y.to_vec1::()?, [1.6, 1.2, 1.8, 1.03]); - assert_eq!(grad_x.to_vec1::()?, [0.2, 0.2, 0.2, 0.2]); - let y = Tensor::new(1f32, device)?.broadcast_div(x)?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!( - test_utils::to_vec1_round(&y, 4)?, - [0.3333, 1.0, 0.25, 6.6667] - ); - assert_eq!( - grad_x.to_vec1::()?, - [-0.11111111, -1.0, -0.0625, -44.444443], - ); - let y = x.broadcast_div(&Tensor::new(0.5f32, device)?)?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!(y.to_vec1::()?, [6., 2., 8., 0.3]); - assert_eq!(grad_x.to_vec1::()?, [2., 2., 2., 2.]); - - let x = Var::new(&[3f32, 1., 4., 0.15], device)?; - let y = x.powf(2.5)?; - let grads = y.backward()?; - let grad_x = grads.get(&x).context("no grad for x")?; - assert_eq!(test_utils::to_vec1_round(&y, 2)?, [15.59, 1.0, 32.0, 0.01]); - assert_eq!( - test_utils::to_vec1_round(grad_x, 2)?, - [12.99, 2.5, 20.0, 0.15] - ); - - let y = x.tanh()?; - let grads = y.backward()?; - let grad_x = grads.get(&x).context("no grad for x")?; - assert_eq!(test_utils::to_vec1_round(&y, 2)?, [1.0, 0.76, 1.0, 0.15]); - assert_eq!( - test_utils::to_vec1_round(grad_x, 2)?, - [0.01, 0.42, 0.0, 0.98], - ); - - // testing compared to pytorch nn.GELU(approximate = 'tanh') - let y = x.gelu()?; - let grads = y.backward()?; - let grad_x = grads.get(&x).context("no grad for x")?; - assert_eq!( - test_utils::to_vec1_round(&y, 4)?, - [2.9964, 0.8412, 3.9999, 0.0839] - ); - assert_eq!( - test_utils::to_vec1_round(grad_x, 4)?, - [1.0116, 1.0830, 1.0003, 0.6188], - ); - - // Testing compared to pytorch torch.erf - // - // import torch - // x = torch.tensor([3.0, 1.0, 4.0, 0.15], requires_grad=True) - // y = x.erf() - // print(y) - // loss = y.sum() - // loss.backward() - // print(x.grad) - let y = x.erf()?; - let grads = y.backward()?; - let grad_x = grads.get(&x).context("no grad for x")?; - assert_eq!(test_utils::to_vec1_round(&y, 4)?, [1.0, 0.8427, 1.0, 0.168]); - assert_eq!( - test_utils::to_vec1_round(grad_x, 4)?, - [0.0001, 0.4151, 0.0, 1.1033], - ); - - // Testing compared to pytorch nn.GELU(approximate = 'none') - // - // import torch - // import torch.nn.functional as F - // x = torch.tensor([3.0, 1.0, 4.0, 0.15], requires_grad=True) - // y = F.gelu(x, approximate='none') - // print(y) - // loss = y.sum() - // loss.backward() - // print(x.grad) - let y = x.gelu_erf()?; - let grads = y.backward()?; - let grad_x = grads.get(&x).context("no grad for x")?; - assert_eq!( - test_utils::to_vec1_round(&y, 4)?, - [2.9960, 0.8413, 3.9999, 0.0839] - ); - assert_eq!( - test_utils::to_vec1_round(grad_x, 4)?, - [1.0119, 1.0833, 1.0005, 0.6188], - ); - - // Testing compared to pytorch elu - // - // import torch - // import torch.nn.functional as F - // x = torch.tensor([-1.0, 0.0, -2.0, 3.0], requires_grad=True) - // y = F.elu(x, alpha=2.0) - // print(y) - // loss = y.min - // loss = y.sum() - // loss.backward() - // print(x.grad) - let elu_x = Var::new(&[-1.0f32, 0., -2., 3.], device)?; - let y = elu_x.elu(2.)?; - let grads = y.backward()?; - let grad_x = grads.get(&elu_x).context("no grad for x")?; - - assert_eq!( - test_utils::to_vec1_round(&y, 4)?, - [-1.2642, 0.0000, -1.7293, 3.0000] - ); - assert_eq!( - test_utils::to_vec1_round(grad_x, 4)?, - [0.7358, 2.0000, 0.2707, 1.0000] - ); - - // testing compared to pytorch nn.Silu() - let y = x.silu()?; - let grads = y.backward()?; - let grad_x = grads.get(&x).context("no grad for x")?; - assert_eq!( - test_utils::to_vec1_round(&y, 4)?, - [2.8577, 0.7311, 3.9281, 0.0806] - ); - assert_eq!( - test_utils::to_vec1_round(grad_x, 4)?, - [1.0881, 0.9277, 1.0527, 0.5747], - ); - - if device.is_cpu() { - let x = Var::new(&[[[1f32, 2., 3.], [4., 5., 6.], [7., 8., 9.]]], device)?; - let y = x.interpolate1d(12)?.reshape(36)?; - - let z = Tensor::new( - &[ - 1_f32, 02., 03., 04., 05., 06., 07., 08., 09., 10., 11., 12., 13., 14., 15., 16., - 17., 18., 19., 20., 21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., - 33., 34., 35., 36., - ], - device, - )?; - - let loss = y.unsqueeze(1)?.transpose(0, 1)?.matmul(&z.unsqueeze(1)?)?; - let grads = loss.backward()?; - let grad_x = grads.get(&x).context("no grad for x")?; - - assert_eq!( - test_utils::to_vec3_round(grad_x, 4)?, - [[[10_f32, 26., 42.], [58., 74., 90.], [106., 122., 138.]]] - ); - } - - // manually checked: see comments - let x = Var::new(&[[[[1f32, 2., 3.], [4., 5., 6.], [7., 8., 9.]]]], device)?; - let y = x.interpolate2d(6, 6)?.reshape(36)?; - - let z = Tensor::new( - &[ - 1_f32, 02., 03., 04., 05., 06., 07., 08., 09., 10., 11., 12., 13., 14., 15., 16., 17., - 18., 19., 20., 21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., 33., 34., - 35., 36., - ], - device, - )?; - // gradient should be - // row 1 - // 1+2+7+8 = 18 - // 3+4+9+10 = 26 - // 5+6+11+12 = 34 - // row 2 - // 13+14+19+20 = 66 - // 15+16+21+22 = 74 - // 17+18+23+24 = 82 - // row 3 - // 25+26+31+32 = 114 - // 27+28+33+34 = 122 - // 29+30+35+36 = 130 - let loss = y.unsqueeze(1)?.transpose(0, 1)?.matmul(&z.unsqueeze(1)?)?; - - let grads = loss.backward()?; - - let grad_x = grads.get(&x).context("no grad for x")?; - assert_eq!( - test_utils::to_vec2_round(&grad_x.flatten(0, 2)?, 4)?, - [[18_f32, 26., 34.], [66., 74., 82.], [114., 122., 130.]] - ); - - // manually checked: see comments - let x = Var::new(&[[[[1f32, 2.], [4., 5.]]]], device)?; - let y = x.interpolate2d(6, 6)?.reshape(36)?; - - let z = Tensor::new( - &[ - 1_f32, 02., 03., 04., 05., 06., 07., 08., 09., 10., 11., 12., 13., 14., 15., 16., 17., - 18., 19., 20., 21., 22., 23., 24., 25., 26., 27., 28., 29., 30., 31., 32., 33., 34., - 35., 36., - ], - device, - )?; - // gradient should be - // row 1 - // 1+2+3+7+8+9+13+14+15 = 72 - // 4+5+6+10+11+12+16+17+18 = 99 - // row 2 - // 19+20+21+25+26+27+31+32+33 = 234 - // 22+23+24+28+29+30+34+35+36 = 243 - let loss = y.unsqueeze(1)?.transpose(0, 1)?.matmul(&z.unsqueeze(1)?)?; - - let grads = loss.backward()?; - - let grad_x = grads.get(&x).context("no grad for x")?; - assert_eq!( - test_utils::to_vec2_round(&grad_x.flatten(0, 2)?, 4)?, - [[72_f32, 99.], [234., 261.]] - ); - - // manually checked: see comments - let x = Var::new(&[[[[1f32, 2.], [4., 5.]], [[6f32, 7.], [8., 9.]]]], device)?; - - let y = x.interpolate2d(4, 4)?.reshape(32)?; - - #[rustfmt::skip] - let z = Tensor::new( - &[ - 1_f32, 02., 03., 04., - 05., 06., 07., 08., - 09., 10., 11., 12., - 13., 14., 15., 16., - 17., 18., 19., 20., - 21., 22., 23., 24., - 25., 26., 27., 28., - 29., 30., 31., 32. - ], - device, - )?; - // gradient should be - // m1r1 - // 1+2+5+6=14 - // 3+4+7+8=22 - // m1r2 - // 9+10+13+14=46 - // 11+12+15+16=54 - // m2r1 - // 17+18+21+22=78 - // 19+20+23+24=86 - // m2r2 - // 25+26+29+30=110 - // 27+28+31+32=118 - let loss = y.unsqueeze(1)?.transpose(0, 1)?.matmul(&z.unsqueeze(1)?)?; - - let grads = loss.backward()?; - - let grad_x = grads.get(&x).context("no grad for x")?; - - assert_eq!( - test_utils::to_vec3_round(&grad_x.flatten(0, 1)?, 4)?, - [[[14_f32, 22.], [46., 54.]], [[78., 86.], [110., 118.]]] - ); - - // manually checked: see comments - let x = Var::new( - &[[[[1f32, 2.], [4., 5.]]], [[[6f32, 7.], [8., 9.]]]], - device, - )?; - - let y = x.interpolate2d(4, 4)?.reshape(32)?; - - #[rustfmt::skip] - let z = Tensor::new( - &[ - 1_f32, 02., 03., 04., - 05., 06., 07., 08., - 09., 10., 11., 12., - 13., 14., 15., 16., - 17., 18., 19., 20., - 21., 22., 23., 24., - 25., 26., 27., 28., - 29., 30., 31., 32. - ], - device, - )?; - // gradient should be - // m1r1 - // 1+2+5+6=14 - // 3+4+7+8=22 - // m1r2 - // 9+10+13+14=46 - // 11+12+15+16=54 - // m2r1 - // 17+18+21+22=78 - // 19+20+23+24=86 - // m2r2 - // 25+26+29+30=110 - // 27+28+31+32=118 - let loss = y.unsqueeze(1)?.transpose(0, 1)?.matmul(&z.unsqueeze(1)?)?; - - let grads = loss.backward()?; - - let grad_x = grads.get(&x).context("no grad for x")?; - - assert_eq!( - test_utils::to_vec3_round(&grad_x.flatten(0, 1)?, 4)?, - [[[14_f32, 22.], [46., 54.]], [[78., 86.], [110., 118.]]] - ); - Ok(()) -} - -fn binary_grad(device: &Device) -> Result<()> { - let x = Var::new(&[3f32, 1., -4., -1.], device)?; - let x = x.as_tensor(); - // leaky relu - let y = x.maximum(&(x * 0.1)?)?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!(x.to_vec1::()?, [3., 1., -4., -1.]); - assert_eq!(y.to_vec1::()?, [3., 1., -0.4, -0.1]); - assert_eq!(grad_x.to_vec1::()?, [1., 1., 0.1, 0.1]); - - let y = x.minimum(&(x * 0.1)?)?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!(y.to_vec1::()?, [0.3, 0.1, -4., -1.]); - assert_eq!(grad_x.to_vec1::()?, [0.1, 0.1, 1., 1.]); - - // This one is easy to mess up, we want the gradient to be one as it is the identity function. - let y = x.minimum(x)?; - let grads = y.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - assert_eq!(y.to_vec1::()?, [3., 1., -4., -1.]); - assert_eq!(grad_x.to_vec1::()?, [1., 1., 1., 1.]); - - let x_var = Var::new(&[3f32, 1., -4., -1., 5., 9.], device)?; - let x = x_var.as_tensor(); - let y_var = Var::new(&[2f32, 7., 1.], device)?; - let y = y_var.as_tensor(); - - let ss = x - .reshape((2, 3))? - .slice_scatter0(&y.reshape((1, 3))?, 1)? - .sqr()?; - let grads = ss.backward()?; - let grad_x = grads.get(x).context("no grad for x")?; - let grad_y = grads.get(y).context("no grad for y")?; - assert_eq!(ss.to_vec2::()?, [[9., 1., 16.], [4., 49., 1.]]); - assert_eq!(grad_x.to_vec1::()?, [6.0, 2.0, -8.0, 0.0, 0.0, 0.0]); - assert_eq!(grad_y.to_vec1::()?, [4.0, 14.0, 2.0]); - Ok(()) -} - -#[test] -fn test_flip_backprop() -> Result<()> { - let device = &Device::Cpu; - - // Create a tensor (leaf node) that requires gradients - let x = Var::ones((2, 2), DType::F64, device)?; - let weights = Tensor::arange(1.0, 5.0, device)?.reshape((2, 2))?; - - let y = x.matmul(&weights)?; - let expected_y = Tensor::from_vec(vec![4.0, 6.0, 4.0, 6.0], (2, 2), device)?; - candle_core::test_utils::assert_tensor_eq(&y, &expected_y)?; - - let z = y.flip(&[1])?; - let expected_z = Tensor::from_vec(vec![6.0, 4.0, 6.0, 4.0], (2, 2), device)?; - candle_core::test_utils::assert_tensor_eq(&z, &expected_z)?; - - let loss = z.sum_all()?; - - let grad_store = loss.backward()?; - let grad_x = grad_store.get_id(x.id()).unwrap(); - - let flipped_weights = weights.flip(&[1])?; - let dloss_dy = Tensor::ones((2, 2), DType::F64, device)?; - // dloss/dx = dloss/dy @ dy/dx = ones @ weight.flip.T - let expected_grad = dloss_dy.matmul(&flipped_weights.t()?)?; - candle_core::test_utils::assert_tensor_eq(grad_x, &expected_grad)?; - - Ok(()) -} - -test_device!( - simple_grad, - simple_grad_cpu, - simple_grad_gpu, - simple_grad_metal -); -test_device!(sum_grad, sum_grad_cpu, sum_grad_gpu, sum_grad_metal); -test_device!( - matmul_grad, - matmul_grad_cpu, - matmul_grad_gpu, - matmul_grad_metal -); -test_device!( - grad_descent, - grad_descent_cpu, - grad_descent_gpu, - grad_descent_metal -); -test_device!(unary_grad, unary_grad_cpu, unary_grad_gpu, unary_grad_metal); -test_device!( - binary_grad, - binary_grad_cpu, - binary_grad_gpu, - binary_grad_metal -); diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/indexing_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/indexing_tests.rs deleted file mode 100644 index 047205a31f..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/tests/indexing_tests.rs +++ /dev/null @@ -1,122 +0,0 @@ -use anyhow::Result; -use candle_core::{Device, IndexOp, Tensor}; - -#[test] -fn integer_index() -> Result<()> { - let dev = Device::Cpu; - - let tensor = Tensor::arange(0u32, 2 * 3, &dev)?.reshape((2, 3))?; - let result = tensor.i(1)?; - assert_eq!(result.dims(), &[3]); - assert_eq!(result.to_vec1::()?, &[3, 4, 5]); - - let result = tensor.i((.., 2))?; - assert_eq!(result.dims(), &[2]); - assert_eq!(result.to_vec1::()?, &[2, 5]); - - Ok(()) -} - -#[test] -fn range_index() -> Result<()> { - let dev = Device::Cpu; - // RangeFull - let tensor = Tensor::arange(0u32, 2 * 3, &dev)?.reshape((2, 3))?; - let result = tensor.i(..)?; - assert_eq!(result.dims(), &[2, 3]); - assert_eq!(result.to_vec2::()?, &[[0, 1, 2], [3, 4, 5]]); - - // Range - let tensor = Tensor::arange(0u32, 4 * 3, &dev)?.reshape((4, 3))?; - let result = tensor.i(1..3)?; - assert_eq!(result.dims(), &[2, 3]); - assert_eq!(result.to_vec2::()?, &[[3, 4, 5], [6, 7, 8]]); - - // RangeFrom - let result = tensor.i(2..)?; - assert_eq!(result.dims(), &[2, 3]); - assert_eq!(result.to_vec2::()?, &[[6, 7, 8], [9, 10, 11]]); - - // RangeTo - let result = tensor.i(..2)?; - assert_eq!(result.dims(), &[2, 3]); - assert_eq!(result.to_vec2::()?, &[[0, 1, 2], [3, 4, 5]]); - - // RangeInclusive - let result = tensor.i(1..=2)?; - assert_eq!(result.dims(), &[2, 3]); - assert_eq!(result.to_vec2::()?, &[[3, 4, 5], [6, 7, 8]]); - - // RangeTo - let result = tensor.i(..1)?; - assert_eq!(result.dims(), &[1, 3]); - assert_eq!(result.to_vec2::()?, &[[0, 1, 2]]); - - // RangeToInclusive - let result = tensor.i(..=1)?; - assert_eq!(result.dims(), &[2, 3]); - assert_eq!(result.to_vec2::()?, &[[0, 1, 2], [3, 4, 5]]); - - // Empty range - let result = tensor.i(1..1)?; - assert_eq!(result.dims(), &[0, 3]); - let empty: [[u32; 3]; 0] = []; - assert_eq!(result.to_vec2::()?, &empty); - - // Similar to PyTorch, allow empty ranges when the computed length is negative. - #[allow(clippy::reversed_empty_ranges)] - let result = tensor.i(1..0)?; - assert_eq!(result.dims(), &[0, 3]); - let empty: [[u32; 3]; 0] = []; - assert_eq!(result.to_vec2::()?, &empty); - Ok(()) -} - -#[test] -fn index_3d() -> Result<()> { - let tensor = Tensor::from_iter(0..24u32, &Device::Cpu)?.reshape((2, 3, 4))?; - assert_eq!(tensor.i((0, 0, 0))?.to_scalar::()?, 0); - assert_eq!(tensor.i((1, 0, 0))?.to_scalar::()?, 12); - assert_eq!(tensor.i((0, 1, 0))?.to_scalar::()?, 4); - assert_eq!(tensor.i((0, 1, 3))?.to_scalar::()?, 7); - assert_eq!(tensor.i((0..2, 0, 0))?.to_vec1::()?, &[0, 12]); - assert_eq!( - tensor.i((0..2, .., 0))?.to_vec2::()?, - &[[0, 4, 8], [12, 16, 20]] - ); - assert_eq!( - tensor.i((..2, .., 3))?.to_vec2::()?, - &[[3, 7, 11], [15, 19, 23]] - ); - assert_eq!(tensor.i((1, .., 3))?.to_vec1::()?, &[15, 19, 23]); - Ok(()) -} - -#[test] -fn slice_assign() -> Result<()> { - let dev = Device::Cpu; - - let tensor = Tensor::arange(0u32, 4 * 5, &dev)?.reshape((4, 5))?; - let src = Tensor::arange(0u32, 2 * 3, &dev)?.reshape((3, 2))?; - let out = tensor.slice_assign(&[1..4, 3..5], &src)?; - assert_eq!( - out.to_vec2::()?, - &[ - [0, 1, 2, 3, 4], - [5, 6, 7, 0, 1], - [10, 11, 12, 2, 3], - [15, 16, 17, 4, 5] - ] - ); - let out = tensor.slice_assign(&[0..3, 0..2], &src)?; - assert_eq!( - out.to_vec2::()?, - &[ - [0, 1, 2, 3, 4], - [2, 3, 7, 8, 9], - [4, 5, 12, 13, 14], - [15, 16, 17, 18, 19] - ] - ); - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/layout_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/layout_tests.rs deleted file mode 100644 index bc67f7defc..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/tests/layout_tests.rs +++ /dev/null @@ -1,152 +0,0 @@ -use candle::{test_device, Device, IndexOp, Result, Tensor}; -use candle_core as candle; - -fn contiguous(device: &Device) -> Result<()> { - let tensor = Tensor::arange(0u32, 24u32, device)?.reshape((2, 3, 4))?; - assert_eq!( - tensor.to_vec3::()?, - &[ - [[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 10, 11]], - [[12, 13, 14, 15], [16, 17, 18, 19], [20, 21, 22, 23]] - ] - ); - assert_eq!( - tensor.t()?.contiguous()?.to_vec3::()?, - &[ - [[0, 4, 8], [1, 5, 9], [2, 6, 10], [3, 7, 11]], - [[12, 16, 20], [13, 17, 21], [14, 18, 22], [15, 19, 23]] - ] - ); - assert_eq!( - tensor.transpose(0, 1)?.contiguous()?.to_vec3::()?, - &[ - [[0, 1, 2, 3], [12, 13, 14, 15]], - [[4, 5, 6, 7], [16, 17, 18, 19]], - [[8, 9, 10, 11], [20, 21, 22, 23]] - ] - ); - assert_eq!( - tensor.transpose(0, 1)?.flatten_all()?.to_vec1::()?, - &[0, 1, 2, 3, 12, 13, 14, 15, 4, 5, 6, 7, 16, 17, 18, 19, 8, 9, 10, 11, 20, 21, 22, 23] - ); - assert_eq!( - tensor - .i(1..)? - .transpose(0, 1)? - .contiguous()? - .to_vec3::()?, - &[[[12, 13, 14, 15]], [[16, 17, 18, 19]], [[20, 21, 22, 23]]] - ); - assert_eq!( - tensor.transpose(0, 2)?.contiguous()?.to_vec3::()?, - &[ - [[0, 12], [4, 16], [8, 20]], - [[1, 13], [5, 17], [9, 21]], - [[2, 14], [6, 18], [10, 22]], - [[3, 15], [7, 19], [11, 23]] - ] - ); - Ok(()) -} - -test_device!(contiguous, contiguous_cpu, contiguous_gpu, contiguous_metal); - -#[test] -fn strided_blocks() -> Result<()> { - use candle::Device::Cpu; - let tensor = Tensor::arange(0u32, 24u32, &Cpu)?.reshape((2, 3, 4))?; - match tensor.strided_blocks() { - candle::StridedBlocks::SingleBlock { start_offset, len } => { - assert_eq!(start_offset, 0); - assert_eq!(len, 24); - } - candle::StridedBlocks::MultipleBlocks { .. } => { - panic!("unexpected block structure") - } - }; - let tensor = Tensor::arange(0u32, 26u32, &Cpu)? - .i(2..)? - .reshape((2, 3, 4))?; - match tensor.strided_blocks() { - candle::StridedBlocks::SingleBlock { start_offset, len } => { - assert_eq!(start_offset, 2); - assert_eq!(len, 24); - } - candle::StridedBlocks::MultipleBlocks { .. } => { - panic!("unexpected block structure") - } - }; - let tensor = Tensor::arange(0u32, 24u32, &Cpu)?.reshape((2, 3, 4))?; - let tensor = tensor.i(1)?; - match tensor.strided_blocks() { - candle::StridedBlocks::SingleBlock { start_offset, len } => { - assert_eq!(start_offset, 12); - assert_eq!(len, 12); - } - candle::StridedBlocks::MultipleBlocks { .. } => { - panic!("unexpected block structure") - } - }; - let tensor = Tensor::arange(0u32, 24u32, &Cpu)?.reshape((2, 3, 4))?; - let tensor = tensor.i((.., 1))?.contiguous()?; - match tensor.strided_blocks() { - candle::StridedBlocks::SingleBlock { start_offset, len } => { - assert_eq!(start_offset, 0); - assert_eq!(len, 8); - assert_eq!(tensor.to_vec2::()?, &[[4, 5, 6, 7], [16, 17, 18, 19]]); - } - candle::StridedBlocks::MultipleBlocks { .. } => { - panic!("unexpected block structure") - } - }; - let tensor = Tensor::arange(0u32, 24u32, &Cpu)?.reshape((2, 3, 4))?; - let tensor = tensor.i((.., 1))?; - match tensor.strided_blocks() { - candle::StridedBlocks::SingleBlock { .. } => { - panic!("unexpected block structure") - } - candle::StridedBlocks::MultipleBlocks { - block_len, - block_start_index, - } => { - assert_eq!(block_len, 4); - assert_eq!(block_start_index.collect::>(), &[4, 16]) - } - }; - let tensor = Tensor::arange(0u32, 24u32, &Cpu)?.reshape((2, 3, 4))?; - match tensor.t()?.strided_blocks() { - candle::StridedBlocks::SingleBlock { .. } => { - panic!("unexpected block structure") - } - candle::StridedBlocks::MultipleBlocks { - block_start_index, - block_len, - } => { - assert_eq!(block_len, 1); - assert_eq!( - block_start_index.collect::>(), - &[ - 0, 4, 8, 1, 5, 9, 2, 6, 10, 3, 7, 11, 12, 16, 20, 13, 17, 21, 14, 18, 22, 15, - 19, 23 - ] - ) - } - }; - let tensor = Tensor::arange(0u32, 24u32, &Cpu)?.reshape((2, 3, 4))?; - match tensor.transpose(0, 1)?.strided_blocks() { - candle::StridedBlocks::SingleBlock { .. } => { - panic!("unexpected block structure") - } - candle::StridedBlocks::MultipleBlocks { - block_start_index, - block_len, - } => { - assert_eq!(block_len, 4); - assert_eq!( - block_start_index.collect::>(), - &[0, 12, 4, 16, 8, 20] - ) - } - }; - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/matmul_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/matmul_tests.rs deleted file mode 100644 index aa2189f7a0..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/tests/matmul_tests.rs +++ /dev/null @@ -1,146 +0,0 @@ -use candle_core::{test_device, DType, Device, IndexOp, Result, Tensor}; - -fn matmul(device: &Device) -> Result<()> { - let data = vec![1.0f32, 2.0, 3.0, 4.0]; - let a = Tensor::from_slice(&data, (2, 2), device)?; - let data = vec![1.0f32, 2.0, 3.0, 4.0]; - let b = Tensor::from_slice(&data, (2, 2), device)?; - - let c = a.matmul(&b)?; - assert_eq!(c.to_vec2::()?, &[[7.0f32, 10.0], [15.0, 22.0]]); - - let data = vec![1.0f32, 2.0]; - let a = Tensor::from_slice(&data, (2, 1), device)?; - let data = vec![3.0f32, 4.0]; - let b = Tensor::from_slice(&data, (1, 2), device)?; - let c = a.matmul(&b)?; - assert_eq!(c.to_vec2::()?, &[&[3.0, 4.0], &[6.0, 8.0]]); - - let data: Vec<_> = (0..6).map(|i| i as f32).collect(); - let a = Tensor::from_slice(&data, (2, 3), device)?; - let data: Vec<_> = (0..6).map(|i| (i + 2) as f32).collect(); - let b = Tensor::from_slice(&data, (3, 2), device)?; - let c = a.matmul(&b)?; - assert_eq!(c.to_vec2::()?, &[&[16., 19.], &[52., 64.]]); - - let data: Vec<_> = (0..12).map(|i| i as f32).collect(); - let a = Tensor::from_slice(&data, (2, 2, 3), device)?; - let data: Vec<_> = (0..12).map(|i| (i + 2) as f32).collect(); - let b = Tensor::from_slice(&data, (2, 3, 2), device)?; - let expected = [[[16., 19.], [52., 64.]], [[214., 235.], [304., 334.]]]; - - let c = a.matmul(&b)?; - assert_eq!(c.to_vec3::()?, &expected); - - // Also perform the matmul on contiguous transposed versions. - let a_tt = a.t()?.contiguous()?.t()?; - assert!(!a_tt.is_contiguous()); - assert_eq!(a.dims(), a_tt.dims()); - assert_eq!(a_tt.stride(), &[6, 1, 2]); - - let b_tt = b.t()?.contiguous()?.t()?; - assert!(!b_tt.is_contiguous()); - assert_eq!(b.dims(), b_tt.dims()); - assert_eq!(b_tt.stride(), &[6, 1, 3]); - - assert_eq!(a_tt.matmul(&b)?.to_vec3::()?, &expected); - assert_eq!(a.matmul(&b_tt)?.to_vec3::()?, &expected); - assert_eq!(a_tt.matmul(&b_tt)?.to_vec3::()?, &expected); - Ok(()) -} - -fn matmul_bf16(device: &Device) -> Result<()> { - if !device.supports_bf16() { - return Ok(()); - } - let data = vec![1.0f32, 2.0, 3.0, 4.0]; - let a = Tensor::from_slice(&data, (2, 2), device)?.to_dtype(DType::BF16)?; - let data = vec![1.0f32, 2.0, 3.0, 4.0]; - let b = Tensor::from_slice(&data, (2, 2), device)?.to_dtype(DType::BF16)?; - - let c = a.matmul(&b)?.to_dtype(DType::F32)?; - assert_eq!(c.to_vec2::()?, &[[7.0f32, 10.0], [15.0, 22.0]]); - Ok(()) -} - -fn broadcast_matmul(device: &Device) -> Result<()> { - let lhs = Tensor::randn(0f32, 1f32, (3, 1, 4, 5), device)?; - let rhs = Tensor::randn(0f32, 1f32, (6, 5, 2), device)?; - let out = lhs.broadcast_matmul(&rhs)?; - assert_eq!(out.dims(), &[3, 6, 4, 2]); - for idx1 in 0..3 { - for idx2 in 0..6 { - let out = out.i((idx1, idx2))?; - let lhs = lhs.i((idx1, 0))?; - let rhs = rhs.i(idx2)?; - let out2 = lhs.matmul(&rhs); - let sum_diff2 = (out - out2)?.sqr()?.sum_all()?; - // With cuda, we see errors of up to ~1e-12. - assert!(sum_diff2.to_vec0::()? < 1e-6) - } - } - Ok(()) -} - -#[test] -fn tensor_dot() -> Result<()> { - let lhs = Tensor::new(&[1., 2., 3.], &Device::Cpu)?; - let rhs = Tensor::new(&[4., 5., 6.], &Device::Cpu)?; - let expected = Tensor::new(32., &Device::Cpu)?; - let dot_ret = lhs.dot(&rhs)?; - candle_core::test_utils::assert_tensor_eq(&dot_ret, &expected)?; - Ok(()) -} - -#[test] -fn tensor_mv() -> Result<()> { - let mat = Tensor::new(&[[1., 2., 3.], [4., 5., 6.]], &Device::Cpu)?; - let vec = Tensor::new(&[1., 1., 1.], &Device::Cpu)?; - let expected = Tensor::new(&[6., 15.], &Device::Cpu)?; - let mv_ret = mat.mv(&vec)?; - candle_core::test_utils::assert_tensor_eq(&mv_ret, &expected)?; - Ok(()) -} - -// https://github.com/huggingface/candle/issues/1948 -fn squeeze_mm(device: &Device) -> Result<()> { - let seq_len = 8_usize; - let a = Tensor::zeros((1, seq_len, 16), DType::F32, device)?; - let x = a.i((.., seq_len - 1, ..))?; - let w = Tensor::zeros((32, 16), DType::F32, device)?.t()?; - let x = x.matmul(&w)?; - assert_eq!(x.dims(), &[1, 32]); - Ok(()) -} - -// https://github.com/huggingface/candle/issues/1992 -fn mm_layout(device: &Device) -> Result<()> { - let a = Tensor::arange(0f32, 16f32, device)?.reshape((1, 1, 4, 4))?; - let b = Tensor::arange(0f32, 8f32, device)?.reshape((1, 1, 4, 2))?; - let mm1 = a.matmul(&b)?; - // Forces the layout to be: - // shape: [1, 1, 4, 2], stride: [8, 2, 2, 1], start_offset: 0 - // This is still a contiguous matrix but matmul checks are only the two last dimensions have - // non 1 sizes but matmul check may be reluctant to handle it. - let b = b.transpose(1, 2)?.force_contiguous()?.transpose(1, 2)?; - let mm2 = a.matmul(&b)?; - let diff = (mm1 - mm2)?.abs()?.sum_all()?.to_vec0::()?; - assert_eq!(diff, 0.); - Ok(()) -} - -test_device!(matmul, matmul_cpu, matmul_gpu, matmul_metal); -test_device!( - matmul_bf16, - matmul_bf16_cpu, - matmul_bf16_gpu, - matmul_bf16_metal -); -test_device!( - broadcast_matmul, - broadcast_matmul_cpu, - broadcast_matmul_gpu, - broadcast_matmul_metal -); -test_device!(squeeze_mm, squeeze_mm_cpu, squeeze_mm_gpu, squeeze_mm_metal); -test_device!(mm_layout, mm_layout_cpu, mm_layout_gpu, mm_layout_metal); diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/npy.py b/v2/dependency-patches/candle-core-0.9.2/tests/npy.py deleted file mode 100644 index 0fd2778a15..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/tests/npy.py +++ /dev/null @@ -1,9 +0,0 @@ -import numpy as np -x = np.arange(10) - -# Write a npy file. -np.save("test.npy", x) - -# Write multiple values to a npz file. -values = { "x": x, "x_plus_one": x + 1 } -np.savez("test.npz", **values) diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/pool_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/pool_tests.rs deleted file mode 100644 index 1edb7d353b..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/tests/pool_tests.rs +++ /dev/null @@ -1,117 +0,0 @@ -use candle_core::{test_device, test_utils, Device, IndexOp, Result, Tensor}; - -// https://github.com/huggingface/candle/issues/364 -fn avg_pool2d(dev: &Device) -> Result<()> { - let data: Vec = vec![ - 1., 1., 1., 1., 0., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., - ]; - let t = Tensor::from_vec(data, (1, 1, 4, 4), dev)?; - let pool = t.avg_pool2d(2)?.squeeze(0)?.squeeze(0)?; - assert_eq!(pool.to_vec2::()?, [[0.5f32, 1.], [1., 1.]]); - - let data: Vec = vec![ - 1., 2., 1., 3., 0., 0., 1., 1., 1., 1., 1., 1., 5., 1., 1., 1., - ]; - let t = Tensor::from_vec(data, (1, 1, 2, 8), dev)?; - let pool = t.avg_pool2d(2)?.squeeze(0)?.squeeze(0)?; - assert_eq!(pool.to_vec2::()?, [[5. / 4., 6. / 4., 6. / 4., 1.]]); - Ok(()) -} - -fn max_pool2d(dev: &Device) -> Result<()> { - let data: Vec = vec![ - 1., 2., 1., 3., 0., 0., 1., 1., 1., 1., 1., 1., 5., 1., 1., 1., - ]; - let t = Tensor::from_vec(data, (1, 1, 4, 4), dev)?; - - let pool = t.max_pool2d(2)?.squeeze(0)?.squeeze(0)?; - assert_eq!(pool.to_vec2::()?, [[2f32, 3.], [5., 1.]]); - - let t = t.reshape((1, 1, 2, 8))?; - let pool = t.max_pool2d(2)?.squeeze(0)?.squeeze(0)?; - assert_eq!(pool.to_vec2::()?, [[2.0, 3.0, 5.0, 1.0]]); - Ok(()) -} - -/* This test corresponds to the following PyTorch script. -import torch -torch.manual_seed(4242) - -t = torch.randn((1, 2, 4, 4)) -print(t.flatten()) -res = torch.nn.functional.avg_pool2d(t, 2) -print(res) -*/ -fn avg_pool2d_pytorch(dev: &Device) -> Result<()> { - if dev.is_metal() { - return Ok(()); - } - let t = Tensor::new( - &[ - 0.4056f32, -0.8689, -0.0773, -1.5630, -2.8012, -1.5059, 0.3972, 1.0852, 0.4997, 3.0616, - 1.6541, 0.0964, -0.8338, -1.6523, -0.8323, -0.1699, 0.0823, 0.3526, 0.6843, 0.2395, - 1.2279, -0.9287, -1.7030, 0.1370, 0.6047, 0.3770, -0.6266, 0.3529, 2.2013, -0.6836, - 0.2477, 1.3127, - ], - dev, - )? - .reshape((1, 2, 4, 4))?; - let pool = t.avg_pool2d(2)?.squeeze(0)?; - assert_eq!( - test_utils::to_vec3_round(&pool, 4)?, - [ - [[-1.1926, -0.0395], [0.2688, 0.1871]], - [[0.1835, -0.1606], [0.6249, 0.3217]] - ] - ); - let pool = t.avg_pool2d(3)?.squeeze(0)?; - assert_eq!( - test_utils::to_vec3_round(&pool, 4)?, - [[[0.085]], [[0.0078]]] - ); - - let t = t.reshape((1, 1, 4, 8))?; - let pool = t.avg_pool2d(2)?.squeeze(0)?.squeeze(0)?; - assert_eq!( - test_utils::to_vec2_round(&pool, 4)?, - [ - [0.7745, 0.0276, -1.6983, 0.12], - [0.3542, 0.1625, 0.4542, -0.0014] - ] - ); - Ok(()) -} - -fn upsample_nearest2d(dev: &Device) -> Result<()> { - let t = Tensor::arange(0f32, 6f32, dev)?.reshape((1, 1, 2, 3))?; - let upsampled = t.upsample_nearest2d(4, 6)?.i(0)?.i(0)?; - assert_eq!( - t.i(0)?.i(0)?.to_vec2::()?, - [[0.0, 1.0, 2.0], [3.0, 4.0, 5.0]] - ); - assert_eq!( - upsampled.to_vec2::()?, - [ - [0.0, 0.0, 1.0, 1.0, 2.0, 2.0], - [0.0, 0.0, 1.0, 1.0, 2.0, 2.0], - [3.0, 3.0, 4.0, 4.0, 5.0, 5.0], - [3.0, 3.0, 4.0, 4.0, 5.0, 5.0] - ] - ); - Ok(()) -} - -test_device!(avg_pool2d, avg_pool2d_cpu, avg_pool2d_gpu, avg_pool2d_metal); -test_device!( - avg_pool2d_pytorch, - avg_pool2d_pytorch_cpu, - avg_pool2d_pytorch_gpu, - avg_pool2d_pytorch_metal -); -test_device!(max_pool2d, max_pool2d_cpu, max_pool2d_gpu, max_pool2d_metal); -test_device!( - upsample_nearest2d, - upsample_nearest2d_cpu, - upsample_nearest2d_gpu, - upsample_nearest2d_metal -); diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/pth.py b/v2/dependency-patches/candle-core-0.9.2/tests/pth.py deleted file mode 100644 index 5c787c2036..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/tests/pth.py +++ /dev/null @@ -1,37 +0,0 @@ -import torch -from collections import OrderedDict - -# Write a trivial tensor to a pt file -a= torch.tensor([[1,2,3,4], [5,6,7,8]]) -o = OrderedDict() -o["test"] = a - -# Write a trivial tensor to a pt file -torch.save(o, "test.pt") - -############################################################################################################ -# Write a trivial tensor to a pt file with a key -torch.save({"model_state_dict": o}, "test_with_key.pt") - -############################################################################################################ -# Create a tensor with fortran contiguous memory layout -import numpy as np - -# Step 1: Create a 3D NumPy array with Fortran order using a range of numbers -# For example, creating a 2x3x4 array -array_fortran = np.asfortranarray(np.arange(1, 2*3*4 + 1).reshape(2, 3, 4)) - -# Verify the memory order -print("Is Fortran contiguous (F order):", array_fortran.flags['F_CONTIGUOUS']) # Should be True -print("Is C contiguous (C order):", array_fortran.flags['C_CONTIGUOUS']) # Should be False - -# Step 2: Convert the NumPy array to a PyTorch tensor -tensor_fortran = torch.from_numpy(array_fortran) - -# Verify the tensor layout -print("Tensor stride:", tensor_fortran.stride()) # Stride will reflect the Fortran memory layout - -# Step 3: Save the PyTorch tensor to a .pth file -torch.save({"tensor_fortran": tensor_fortran}, 'fortran_tensor_3d.pth') - -print("3D Tensor saved with Fortran layout.") diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/pth_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/pth_tests.rs deleted file mode 100644 index 7ea3d1420e..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/tests/pth_tests.rs +++ /dev/null @@ -1,31 +0,0 @@ -/// Regression test for pth files not loading on Windows. -#[test] -fn test_pth() { - let tensors = candle_core::pickle::PthTensors::new("tests/test.pt", None).unwrap(); - tensors.get("test").unwrap().unwrap(); -} - -#[test] -fn test_pth_with_key() { - let tensors = - candle_core::pickle::PthTensors::new("tests/test_with_key.pt", Some("model_state_dict")) - .unwrap(); - tensors.get("test").unwrap().unwrap(); -} - -#[test] -fn test_pth_fortran_contiguous() { - let tensors = - candle_core::pickle::PthTensors::new("tests/fortran_tensor_3d.pth", None).unwrap(); - let tensor = tensors.get("tensor_fortran").unwrap().unwrap(); - - assert_eq!(tensor.dims3().unwrap(), (2, 3, 4)); - - assert_eq!( - tensor.to_vec3::().unwrap(), - [ - [[1, 2, 3, 4], [5, 6, 7, 8], [9, 10, 11, 12]], - [[13, 14, 15, 16], [17, 18, 19, 20], [21, 22, 23, 24]] - ] - ); -} diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/quantized_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/quantized_tests.rs deleted file mode 100644 index c69034a579..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/tests/quantized_tests.rs +++ /dev/null @@ -1,1402 +0,0 @@ -use candle_core::{ - bail, - quantized::{self, GgmlDType}, - test_device, - test_utils::to_vec2_round, - DType, Device, IndexOp, Module, Result, Tensor, Var, -}; -use quantized::{k_quants, GgmlType}; -use rand::prelude::*; - -const GGML_TEST_SIZE: usize = 32 * 128; - -const GGML_MAX_QUANTIZATION_TOTAL_ERROR: f32 = 0.002; -const GGML_MAX_QUANTIZATION_TOTAL_ERROR_2BITS: f32 = 0.0075; -const GGML_MAX_QUANTIZATION_TOTAL_ERROR_3BITS: f32 = 0.0040; -const GGML_MAX_DOT_PRODUCT_ERROR: f32 = 0.02; - -fn test_matmul( - device: &Device, - (b, m, n, k): (usize, usize, usize, usize), - dtype: GgmlDType, -) -> Result<()> { - if (device.is_cuda() || device.is_metal()) - && (dtype == GgmlDType::Q8_1 || dtype == GgmlDType::Q8K) - { - return Ok(()); - } - - let lhs = (0..(m * k)) - .map(|v| v as f32 / (m * k) as f32) - .collect::>(); - let rhs = (0..(k * n)) - .map(|v| v as f32 / (n * k) as f32) - .collect::>(); - - let lhs = Tensor::from_slice(&lhs, (m, k), device)?; - let rhs = Tensor::from_slice(&rhs, (k, n), device)?; - let mm = lhs.matmul(&rhs)?; - let qtensor = quantized::QTensor::quantize(&rhs.t()?, dtype)?; - let matmul = quantized::QMatMul::from_qtensor(qtensor)?; - let res = matmul.forward(&lhs)?; - - let error: f32 = ((&mm - &res)?.abs()? / &mm.abs()?)? - .sum_all()? - .to_scalar()?; - let error = error / (b * m * n) as f32; - assert!( - error <= 0.02, - "Error {error} is too big. \nExpected:\n {mm} \nFound:\n {res}\n for {dtype:?}" - ); - - Ok(()) -} - -#[cfg(feature = "metal")] -#[test] -fn test_matmul_mm() -> Result<()> { - let dtype = GgmlDType::Q8_0; - let device = Device::new_metal(0)?; - - let m = 32; - let n = 32; - let k = 32; - let lhs = (0..(m * k)) - .map(|v| v as f32 / (m * k) as f32) - .collect::>(); - let rhs = (0..(k * n)) - .map(|v| v as f32 / (n * k) as f32) - .collect::>(); - - let lhs = Tensor::from_slice(&lhs, (m, k), &device)?; - let rhs = Tensor::from_slice(&rhs, (1, 1, k, n), &device)?.repeat((5, 20, 1, 1))?; - let mm = lhs.broadcast_matmul(&rhs)?; - let qtensor = quantized::QTensor::quantize(&lhs.t()?, dtype)?; - let matmul = quantized::QMatMul::from_qtensor(qtensor)?; - let res = matmul.forward(&rhs)?; - - let error: f32 = ((&mm - &res)?.abs()? / &mm.abs()?)? - .sum_all()? - .to_scalar()?; - - let error = error / res.elem_count() as f32; - assert!( - error <= 0.001, - "Error {error} is too big. \nExpected:\n {mm} \nFound:\n {res}\n for {dtype:?}" - ); - - Ok(()) -} - -fn quantized_matmul(device: &Device) -> Result<()> { - let (m, k, n) = (3, 64, 4); - let lhs_s = (0..(m * k)).map(|v| v as f32).collect::>(); - let lhs = Tensor::from_slice(&lhs_s, (m, k), device)?; - let mut dst = vec![42.; 3 * 4]; - let mut rhs_t = vec![k_quants::BlockQ4_0::zeros(); 8]; - let rhs = (0..(k * n)).map(|v| v as f32).collect::>(); - k_quants::BlockQ4_0::from_float(&rhs, &mut rhs_t); - k_quants::matmul((m, k, n), &lhs_s, &rhs_t, &mut dst)?; - assert_eq!( - dst.iter().map(|x| x.round()).collect::>(), - &[ - 85120.0, 214562.0, 345455.0, 474748.0, 213475.0, 604465.0, 1000686.0, 1388317.0, - 341876.0, 994283.0, 1655709.0, 2301518.0 - ] - ); - let tensor_rhs = Tensor::from_slice(&rhs, (n, k), device)?.t()?; - let mm = lhs.matmul(&tensor_rhs)?; - assert_eq!( - mm.to_vec2::()?, - &[ - [85344.0, 214368.0, 343392.0, 472416.0], - [214368.0, 605536.0, 996704.0, 1387872.0], - [343392.0, 996704.0, 1650016.0, 2303328.0] - ] - ); - - let qtensor = quantized::QTensor::quantize(&tensor_rhs.t()?, GgmlDType::Q4_0)?; - let matmul = quantized::QMatMul::from_qtensor(qtensor)?; - let res = matmul.forward(&lhs)?; - match device { - Device::Metal(_) => assert_eq!( - to_vec2_round(&res, 0)?, - &[ - [84946.0, 214126.0, 344757.0, 473798.0], - [213458.0, 604350.0, 1000469.0, 1387990.0], - [341970.0, 994574.0, 1656181.0, 2302182.0] - ] - ), - Device::Cuda(_) => assert_eq!( - to_vec2_round(&res, 0)?, - &[ - [84866.0, 214045.0, 344676.0, 473707.0], - [213425.0, 604313.0, 1000431.0, 1387960.0], - [342030.0, 994630.0, 1656248.0, 2302250.0] - ] - ), - Device::Cpu => assert_eq!( - to_vec2_round(&res, 0)?, - &[ - [85120.0, 214562.0, 345455.0, 474748.0], - [213475.0, 604465.0, 1000686.0, 1388317.0], - [341876.0, 994283.0, 1655709.0, 2301518.0] - ] - ), - } - test_matmul(device, (1, 3, 4, 256), GgmlDType::Q4_0)?; - Ok(()) -} - -fn quantized_matmul_neg(device: &Device) -> Result<()> { - let (m, k, n) = (3, 64, 4); - let lhs_s = (0..(m * k)) - .map(|v| v as f32 - (m * k) as f32 / 2.0) - .collect::>(); - let lhs = Tensor::from_slice(&lhs_s, (m, k), device)?; - let mut dst = vec![42.; 3 * 4]; - let mut rhs_t = vec![k_quants::BlockQ4_0::zeros(); 8]; - let rhs = (0..k * n) - .map(|v| v as f32 - (k * n) as f32 / 3.0) - .collect::>(); - let tensor_rhs = Tensor::from_slice(&rhs, (n, k), device)?.t()?; - k_quants::BlockQ4_0::from_float(&rhs, &mut rhs_t); - k_quants::matmul((m, k, n), &lhs_s, &rhs_t, &mut dst)?; - assert_eq!( - dst.iter().map(|x| x.round()).collect::>(), - &[ - 243524.0, -19596.0, -285051.0, -549815.0, 23777.0, 21651.0, 19398.0, 18367.0, - -196472.0, 63012.0, 324585.0, 587902.0 - ] - ); - let mm = lhs.matmul(&tensor_rhs)?; - assert_eq!( - to_vec2_round(&mm, 0)?, - &[ - [244064.0, -20128.0, -284320.0, -548512.0], - [23563.0, 21515.0, 19467.0, 17419.0], - [-196939.0, 63157.0, 323253.0, 583349.0] - ] - ); - - let qtensor = quantized::QTensor::quantize(&tensor_rhs.t()?, GgmlDType::Q4_0)?; - let matmul = quantized::QMatMul::from_qtensor(qtensor)?; - let res = matmul.forward(&lhs)?; - match device { - Device::Metal(_) => assert_eq!( - to_vec2_round(&res, 0)?, - &[ - [243659.0, -19716.0, -285444.0, -550439.0], - [23779.0, 21653.0, 19404.0, 18349.0], - [-196101.0, 63021.0, 324252.0, 587137.0] - ] - ), - Device::Cuda(_) => assert_eq!( - to_vec2_round(&res, 0)?, - &[ - [243740.0, -19762.0, -285476.0, -550498.0], - [23774.0, 21645.0, 19395.0, 18364.0], - [-196045.0, 63030.0, 324120.0, 587079.0] - ] - ), - Device::Cpu => assert_eq!( - to_vec2_round(&res, 0)?, - &[ - [243524.0, -19596.0, -285051.0, -549815.0], - [23777.0, 21651.0, 19398.0, 18367.0], - [-196472.0, 63012.0, 324585.0, 587902.0] - ] - ), - } - let lhs2 = Tensor::stack(&[&lhs, &lhs], 0)?; - let res2 = matmul.forward(&lhs2)?; - let res2 = res2.i(1)?; - let diff = (&res - res2)?.abs()?.mean_all()?.to_vec0::()? / res.elem_count() as f32; - if device.is_cuda() { - assert!(diff < 0.1); - } else { - assert!(diff < 0.96); - } - Ok(()) -} - -fn qmm_batch(dev: &Device) -> Result<()> { - let (lhs, rhs, _mm) = get_random_tensors(2, 256, 6, dev)?; - let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q2K)?; - let rhs = quantized::QMatMul::from_qtensor(rhs)?; - let mm = rhs.forward(&lhs)?; - assert_eq!(mm.shape().dims(), [2, 6]); - let lhs2 = Tensor::cat(&[&lhs, &lhs], 0)?; - let mm2 = rhs.forward(&lhs2)?; - assert_eq!(mm2.shape().dims(), [4, 6]); - let diff2 = (mm2.i(2..)? - &mm)?.abs()?.sum_all()?.to_vec0::()?; - assert_eq!(diff2, 0.0); - let lhs3 = Tensor::cat(&[&lhs2, &lhs], 0)?; - let mm3 = rhs.forward(&lhs3)?; - assert_eq!(mm3.shape().dims(), [6, 6]); - let diff3 = (mm3.i(2..4)? - &mm)?.abs()?.sum_all()?.to_vec0::()?; - assert_eq!(diff3, 0.0); - let diff3 = (mm3.i(4..)? - &mm)?.abs()?.sum_all()?.to_vec0::()?; - assert_eq!(diff3, 0.0); - let lhs4 = Tensor::cat(&[&lhs3, &lhs3], 0)?; - let mm4 = rhs.forward(&lhs4)?; - assert_eq!(mm4.shape().dims(), [12, 6]); - let diff4 = (mm4.i(..6)? - &mm3)?.abs()?.sum_all()?.to_vec0::()?; - if dev.is_cuda() { - // We use a different kernel for sizes from 1 to 8 on cuda which explains - // the difference here. - assert!(0. < diff4 && diff4 < 1e-4) - } else { - assert_eq!(diff4, 0.0) - }; - let diff4 = (mm4.i(6..)? - &mm4.i(..6)?)? - .abs()? - .sum_all()? - .to_vec0::()?; - assert_eq!(diff4, 0.0); - Ok(()) -} - -test_device!(quantized_matmul, qmm_cpu, qmm_cuda, qmm_metal); -test_device!(quantized_matmul_neg, qmm_n_cpu, qmm_n_cuda, qmm_n_metal); -test_device!(qmm_batch, qmm_b_cpu, qmm_b_cuda, qmm_b_metal); - -fn quantize_q4_0(device: &Device) -> Result<()> { - let src = (0..32 * 4).map(|v| v as f32).collect::>(); - - let src = Tensor::from_slice(&src, (32 * 4,), device)?; - let quant = quantized::QTensor::quantize(&src, GgmlDType::Q4_0)?; - let dst = quant.dequantize(device)?; - let dst_f16 = quant.dequantize_f16(device)?; - let diff = (dst.to_dtype(DType::F16)? - dst_f16)? - .to_dtype(DType::F32)? - .abs()? - .sum_all()? - .to_vec0::()?; - assert_eq!(diff, 0.); - assert_eq!( - dst.to_vec1::()?, - &[ - -0.0, -0.0, 3.875, 3.875, 3.875, 3.875, 7.75, 7.75, 7.75, 7.75, 11.625, 11.625, 11.625, - 11.625, 15.5, 15.5, 15.5, 15.5, 19.375, 19.375, 19.375, 19.375, 23.25, 23.25, 23.25, - 23.25, 27.125, 27.125, 27.125, 27.125, 31.0, 31.0, 31.5, 31.5, 31.5, 31.5, 39.375, - 39.375, 39.375, 39.375, 39.375, 39.375, 39.375, 39.375, 47.25, 47.25, 47.25, 47.25, - 47.25, 47.25, 47.25, 47.25, 55.125, 55.125, 55.125, 55.125, 55.125, 55.125, 55.125, - 55.125, 63.0, 63.0, 63.0, 63.0, 59.375, 59.375, 71.25, 71.25, 71.25, 71.25, 71.25, - 71.25, 71.25, 71.25, 71.25, 71.25, 71.25, 71.25, 83.125, 83.125, 83.125, 83.125, - 83.125, 83.125, 83.125, 83.125, 83.125, 83.125, 83.125, 83.125, 95.0, 95.0, 95.0, 95.0, - 95.0, 95.0, 95.25, 95.25, 95.25, 95.25, 95.25, 95.25, 95.25, 95.25, 111.125, 111.125, - 111.125, 111.125, 111.125, 111.125, 111.125, 111.125, 111.125, 111.125, 111.125, - 111.125, 111.125, 111.125, 111.125, 111.125, 127.0, 127.0, 127.0, 127.0, 127.0, 127.0, - 127.0, 127.0 - ] - ); - ggml_quantization_error_test(GgmlDType::Q4_0, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; - Ok(()) -} - -fn quantize_q4_1(device: &Device) -> Result<()> { - let src = (0..32 * 4).map(|v| v as f32).collect::>(); - let src = Tensor::from_slice(&src, (32 * 4,), device)?; - let quant = quantized::QTensor::quantize(&src, GgmlDType::Q4_1)?; - let dst = quant.dequantize(device)?; - let dst_f16 = quant.dequantize_f16(device)?; - let diff = (dst.to_dtype(DType::F16)? - dst_f16)? - .to_dtype(DType::F32)? - .abs()? - .sum_all()? - .to_vec0::()?; - assert_eq!(diff, 0.); - assert_eq!( - round_vector(&dst.to_vec1::()?), - &[ - 0.0, 0.0, 2.066, 2.066, 4.133, 4.133, 6.199, 6.199, 8.266, 8.266, 10.332, 10.332, - 12.398, 12.398, 14.465, 14.465, 16.531, 16.531, 18.598, 18.598, 20.664, 20.664, 22.73, - 22.73, 24.797, 24.797, 26.863, 26.863, 28.93, 28.93, 30.996, 30.996, 32.0, 32.0, - 34.066, 34.066, 36.133, 36.133, 38.199, 38.199, 40.266, 40.266, 42.332, 42.332, 44.398, - 44.398, 46.465, 46.465, 48.531, 48.531, 50.598, 50.598, 52.664, 52.664, 54.73, 54.73, - 56.797, 56.797, 58.863, 58.863, 60.93, 60.93, 62.996, 62.996, 64.0, 64.0, 66.066, - 66.066, 68.133, 68.133, 70.199, 70.199, 72.266, 72.266, 74.332, 74.332, 76.398, 76.398, - 78.465, 78.465, 80.531, 80.531, 82.598, 82.598, 84.664, 84.664, 86.73, 86.73, 88.797, - 88.797, 90.863, 90.863, 92.93, 92.93, 94.996, 94.996, 96.0, 96.0, 98.066, 98.066, - 100.133, 100.133, 102.199, 102.199, 104.266, 104.266, 106.332, 106.332, 108.398, - 108.398, 110.465, 110.465, 112.531, 112.531, 114.598, 114.598, 116.664, 116.664, - 118.73, 118.73, 120.797, 120.797, 122.863, 122.863, 124.93, 124.93, 126.996, 126.996 - ] - ); - ggml_quantization_error_test(GgmlDType::Q4_1, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; - Ok(()) -} - -fn quantize_q5_0(device: &Device) -> Result<()> { - let src = (0..32 * 4).map(|v| v as f32).collect::>(); - let src = Tensor::from_slice(&src, (32 * 4,), device)?; - let quant = quantized::QTensor::quantize(&src, GgmlDType::Q5_0)?; - let dst = quant.dequantize(device)?; - let dst_f16 = quant.dequantize_f16(device)?; - let diff = (dst.to_dtype(DType::F16)? - dst_f16)? - .to_dtype(DType::F32)? - .abs()? - .sum_all()? - .to_vec0::()?; - assert_eq!(diff, 0.); - assert_eq!( - round_vector(&dst.to_vec1::()?), - &[ - -0.0, 1.938, 1.938, 3.875, 3.875, 5.813, 5.813, 7.75, 7.75, 9.688, 9.688, 11.625, - 11.625, 13.563, 13.563, 15.5, 15.5, 17.438, 17.438, 19.375, 19.375, 21.313, 21.313, - 23.25, 23.25, 25.188, 25.188, 27.125, 27.125, 29.063, 29.063, 31.0, 31.5, 31.5, 35.438, - 35.438, 35.438, 35.438, 39.375, 39.375, 39.375, 39.375, 43.313, 43.313, 43.313, 43.313, - 47.25, 47.25, 47.25, 47.25, 51.188, 51.188, 51.188, 51.188, 55.125, 55.125, 55.125, - 55.125, 59.063, 59.063, 59.063, 59.063, 63.0, 63.0, 65.313, 65.313, 65.313, 65.313, - 65.313, 71.25, 71.25, 71.25, 71.25, 71.25, 71.25, 77.188, 77.188, 77.188, 77.188, - 77.188, 77.188, 83.125, 83.125, 83.125, 83.125, 83.125, 83.125, 89.063, 89.063, 89.063, - 89.063, 89.063, 89.063, 95.0, 95.0, 95.0, 95.25, 95.25, 95.25, 95.25, 103.188, 103.188, - 103.188, 103.188, 103.188, 103.188, 103.188, 103.188, 111.125, 111.125, 111.125, - 111.125, 111.125, 111.125, 111.125, 111.125, 119.063, 119.063, 119.063, 119.063, - 119.063, 119.063, 119.063, 119.063, 127.0, 127.0, 127.0, 127.0 - ] - ); - ggml_quantization_error_test(GgmlDType::Q5_0, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; - Ok(()) -} - -fn quantize_q5_1(device: &Device) -> Result<()> { - let src = (0..32 * 4).map(|v| v as f32).collect::>(); - let src = Tensor::from_slice(&src, (32 * 4,), device)?; - let quant = quantized::QTensor::quantize(&src, GgmlDType::Q5_1)?; - let dst = quant.dequantize(device)?; - let dst_f16 = quant.dequantize_f16(device)?; - let diff = (dst.to_dtype(DType::F16)? - dst_f16)? - .to_dtype(DType::F32)? - .abs()? - .sum_all()? - .to_vec0::()?; - assert_eq!(diff, 0.); - assert_eq!( - round_vector(&dst.to_vec1::()?), - &[ - 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0, 12.0, 13.0, 14.0, 15.0, - 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, - 30.0, 31.0, 32.0, 33.0, 34.0, 35.0, 36.0, 37.0, 38.0, 39.0, 40.0, 41.0, 42.0, 43.0, - 44.0, 45.0, 46.0, 47.0, 48.0, 49.0, 50.0, 51.0, 52.0, 53.0, 54.0, 55.0, 56.0, 57.0, - 58.0, 59.0, 60.0, 61.0, 62.0, 63.0, 64.0, 65.0, 66.0, 67.0, 68.0, 69.0, 70.0, 71.0, - 72.0, 73.0, 74.0, 75.0, 76.0, 77.0, 78.0, 79.0, 80.0, 81.0, 82.0, 83.0, 84.0, 85.0, - 86.0, 87.0, 88.0, 89.0, 90.0, 91.0, 92.0, 93.0, 94.0, 95.0, 96.0, 97.0, 98.0, 99.0, - 100.0, 101.0, 102.0, 103.0, 104.0, 105.0, 106.0, 107.0, 108.0, 109.0, 110.0, 111.0, - 112.0, 113.0, 114.0, 115.0, 116.0, 117.0, 118.0, 119.0, 120.0, 121.0, 122.0, 123.0, - 124.0, 125.0, 126.0, 127.0 - ] - ); - ggml_quantization_error_test(GgmlDType::Q5_1, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; - Ok(()) -} - -fn get_test_vector2(bound: f32, size: usize, device: &Device) -> Result { - assert!( - size.is_multiple_of(crate::quantized::k_quants::QK_K), - "size must be a multiple of {}", - crate::quantized::k_quants::QK_K - ); - - let src = (0..size) - .map(|v| (v as f32 - size as f32 / 2.) * bound / (size as f32 / 2.)) - .collect::>(); - assert_eq!([src[0], src[size / 2]], [-bound, 0.0]); - Tensor::from_vec(src, (size,), device) -} - -/// Round a vector -fn round_vector(values: &[f32]) -> Vec { - values - .iter() - .map(|x| (1000. * x).round() / 1000.) - .collect::>() -} - -fn compare_with_error(values: &[f32], expected: &[f32], tolerance: f32) { - for (i, (value, expected_value)) in values.iter().zip(expected.iter()).enumerate() { - let difference = (value - expected_value).abs(); - - assert!( - difference < tolerance, - "Error at index {i}: value = {value}, expected = {expected_value}. Difference = {difference} exceeds tolerance = {tolerance}." - ); - } -} - -/// Creates a vector similar to the ones used in GGML unit tests: -/// https://github.com/ggerganov/llama.cpp/blob/master/tests/test-quantize-fns.cpp#L26-L30 -fn create_ggml_like_vector(offset: f32) -> Vec { - (0..GGML_TEST_SIZE) - .map(|i| 0.1 + 2.0 * (i as f32 + offset).cos()) - .collect() -} - -/// Calculates the root mean square error between two vectors -fn calculate_rmse(a: &[f32], b: &[f32]) -> f32 { - assert_eq!(a.len(), b.len()); - let sum = a - .iter() - .zip(b) - .map(|(a, b)| (a - b).powi(2)) - .sum::() - .sqrt(); - sum / a.len() as f32 -} - -/// Similar to the GGML quantization unit test: -/// https://github.com/ggerganov/llama.cpp/blob/master/tests/test-quantize-fns.cpp#L43-L50 -fn ggml_quantization_error_test(dtype: GgmlDType, device: &Device, max_error: f32) -> Result<()> { - let src = create_ggml_like_vector(0.0); - let src = Tensor::from_slice(&src, (GGML_TEST_SIZE,), device)?; - let quant = quantized::QTensor::quantize(&src, dtype)?; - let dst = quant.dequantize(device)?; - let dst_f16 = quant.dequantize_f16(device)?; - let diff = (dst.to_dtype(DType::F16)? - dst_f16)? - .to_dtype(DType::F32)? - .abs()? - .sum_all()? - .to_vec0::()?; - assert_eq!(diff, 0.); - let error = calculate_rmse(&src.to_vec1::()?, &dst.to_vec1::()?); - if error > max_error { - bail!( - "Quantization error {} exceeds max error {}", - error, - max_error - ); - } - Ok(()) -} - -#[test] -fn imatrix_quantize_q6k() -> Result<()> { - let cpu = &Device::Cpu; - - let mut row_counts = 0f64; - let mut ncall = 0f64; - let mut values = Tensor::zeros((768,), DType::F32, cpu)?; - - for _ in 0..10 { - let lhs = Var::from_tensor(&Tensor::randn(0f32, 1f32, (1024, 512), cpu)?)?; - let rhs = Var::from_tensor(&Tensor::randn(0f32, 1f32, (512, 768), cpu)?)?; - let res = lhs.matmul(&rhs)?; - - // https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/examples/imatrix/imatrix.cpp#L180-L186 - values = (values + res.sqr()?.sum(0)?)?; - row_counts += res.dim(0)? as f64; - ncall += 1.; - } - - // https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/examples/imatrix/imatrix.cpp#L275 - let out = ((values / row_counts)? * ncall)?; - let imatrix = out.to_vec1::()?; - - let xs = Tensor::randn(0f32, 1f32, (1024, 768), cpu)?; - - let quant1 = quantized::QTensor::quantize(&xs, GgmlDType::Q6K)?; - let quant2 = quantized::QTensor::quantize_imatrix(&xs, &imatrix, GgmlDType::Q6K)?; - - let dequant1 = quant1.dequantize(cpu)?; - let dequant2 = quant2.dequantize(cpu)?; - - let err1 = (dequant1 - &xs)?.abs()?.mean_all()?.to_scalar::()?; - let err2 = (dequant2 - &xs)?.abs()?.mean_all()?.to_scalar::()?; - assert!(err2 < err1, "err2 {err2} > err1 {err1}"); - - Ok(()) -} - -#[test] -fn imatrix_quantize_q5k() -> Result<()> { - let cpu = &Device::Cpu; - - let mut row_counts = 0f64; - let mut ncall = 0f64; - let mut values = Tensor::zeros((768,), DType::F32, cpu)?; - - for _ in 0..10 { - let lhs = Var::from_tensor(&Tensor::randn(0f32, 1f32, (1024, 512), cpu)?)?; - let rhs = Var::from_tensor(&Tensor::randn(0f32, 1f32, (512, 768), cpu)?)?; - let res = lhs.matmul(&rhs)?; - - // https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/examples/imatrix/imatrix.cpp#L180-L186 - values = (values + res.sqr()?.sum(0)?)?; - row_counts += res.dim(0)? as f64; - ncall += 1.; - } - - // https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/examples/imatrix/imatrix.cpp#L275 - let out = ((values / row_counts)? * ncall)?; - let imatrix = out.to_vec1::()?; - - let xs = Tensor::randn(0f32, 1f32, (1024, 768), cpu)?; - - let quant1 = quantized::QTensor::quantize(&xs, GgmlDType::Q5K)?; - let quant2 = quantized::QTensor::quantize_imatrix(&xs, &imatrix, GgmlDType::Q5K)?; - - let dequant1 = quant1.dequantize(cpu)?; - let dequant2 = quant2.dequantize(cpu)?; - - let err1 = (dequant1 - &xs)?.abs()?.mean_all()?.to_scalar::()?; - let err2 = (dequant2 - &xs)?.abs()?.mean_all()?.to_scalar::()?; - assert!(err2 < err1, "err2 {err2} > err1 {err1}"); - - Ok(()) -} - -#[test] -fn imatrix_quantize_q4k() -> Result<()> { - // let data = - // quantized::imatrix_file::load_imatrix("../Llama-3.2-3B-Instruct.imatrix").unwrap(); - // for (name, weights) in &data { - // println!("{name}, {} elems", weights.len()); - // } - // dbg!(&data["blk.0.attn_q.weight"].len()); - - let cpu = &Device::Cpu; - - let mut row_counts = 0f64; - let mut ncall = 0f64; - let mut values = Tensor::zeros((768,), DType::F32, cpu)?; - - for _ in 0..10 { - let lhs = Var::from_tensor(&Tensor::randn(0f32, 1f32, (1024, 512), cpu)?)?; - let rhs = Var::from_tensor(&Tensor::randn(0f32, 1f32, (512, 768), cpu)?)?; - let res = lhs.matmul(&rhs)?; - - // https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/examples/imatrix/imatrix.cpp#L180-L186 - values = (values + res.sqr()?.sum(0)?)?; - row_counts += res.dim(0)? as f64; - ncall += 1.; - } - - // https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/examples/imatrix/imatrix.cpp#L275 - let out = ((values / row_counts)? * ncall)?; - let imatrix = out.to_vec1::()?; - - let xs = Tensor::randn(0f32, 1f32, (1024, 768), cpu)?; - - let quant1 = quantized::QTensor::quantize(&xs, GgmlDType::Q4K)?; - let quant2 = quantized::QTensor::quantize_imatrix(&xs, &imatrix, GgmlDType::Q4K)?; - - let dequant1 = quant1.dequantize(cpu)?; - let dequant2 = quant2.dequantize(cpu)?; - - let err1 = (dequant1 - &xs)?.abs()?.mean_all()?.to_scalar::()?; - let err2 = (dequant2 - &xs)?.abs()?.mean_all()?.to_scalar::()?; - assert!(err2 < err1, "err2 {err2} > err1 {err1}"); - - Ok(()) -} - -#[test] -fn imatrix_quantize_q3k() -> Result<()> { - let cpu = &Device::Cpu; - - let mut row_counts = 0f64; - let mut ncall = 0f64; - let mut values = Tensor::zeros((768,), DType::F32, cpu)?; - - for _ in 0..10 { - let lhs = Var::from_tensor(&Tensor::randn(0f32, 1f32, (1024, 512), cpu)?)?; - let rhs = Var::from_tensor(&Tensor::randn(0f32, 1f32, (512, 768), cpu)?)?; - let res = lhs.matmul(&rhs)?; - - // https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/examples/imatrix/imatrix.cpp#L180-L186 - values = (values + res.sqr()?.sum(0)?)?; - row_counts += res.dim(0)? as f64; - ncall += 1.; - } - - // https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/examples/imatrix/imatrix.cpp#L275 - let out = ((values / row_counts)? * ncall)?; - let imatrix = out.to_vec1::()?; - - let xs = Tensor::randn(0f32, 1f32, (1024, 768), cpu)?; - - let quant1 = quantized::QTensor::quantize(&xs, GgmlDType::Q3K)?; - let quant2 = quantized::QTensor::quantize_imatrix(&xs, &imatrix, GgmlDType::Q3K)?; - - let dequant1 = quant1.dequantize(cpu)?; - let dequant2 = quant2.dequantize(cpu)?; - - let err1 = (dequant1 - &xs)?.abs()?.mean_all()?.to_scalar::()?; - let err2 = (dequant2 - &xs)?.abs()?.mean_all()?.to_scalar::()?; - assert!(err2 < err1, "err2 {err2} > err1 {err1}"); - - Ok(()) -} - -#[test] -fn imatrix_quantize_q2k() -> Result<()> { - let cpu = &Device::Cpu; - - let mut row_counts = 0f64; - let mut ncall = 0f64; - let mut values = Tensor::zeros((768,), DType::F32, cpu)?; - - for _ in 0..10 { - let lhs = Var::from_tensor(&Tensor::randn(0f32, 1f32, (1024, 512), cpu)?)?; - let rhs = Var::from_tensor(&Tensor::randn(0f32, 1f32, (512, 768), cpu)?)?; - let res = lhs.matmul(&rhs)?; - - // https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/examples/imatrix/imatrix.cpp#L180-L186 - values = (values + res.sqr()?.sum(0)?)?; - row_counts += res.dim(0)? as f64; - ncall += 1.; - } - - // https://github.com/ggerganov/llama.cpp/blob/678d7994f4da0af3d29046be99950ac999ee9762/examples/imatrix/imatrix.cpp#L275 - let out = ((values / row_counts)? * ncall)?; - let imatrix = out.to_vec1::()?; - - let xs = Tensor::randn(0f32, 1f32, (1024, 768), cpu)?; - - let quant1 = quantized::QTensor::quantize(&xs, GgmlDType::Q2K)?; - let quant2 = quantized::QTensor::quantize_imatrix(&xs, &imatrix, GgmlDType::Q2K)?; - - let dequant1 = quant1.dequantize(cpu)?; - let dequant2 = quant2.dequantize(cpu)?; - - let err1 = (dequant1 - &xs)?.abs()?.mean_all()?.to_scalar::()?; - let err2 = (dequant2 - &xs)?.abs()?.mean_all()?.to_scalar::()?; - assert!(err2 < err1, "err2 {err2} > err1 {err1}"); - - Ok(()) -} - -fn quantize_q2k(device: &Device) -> Result<()> { - let dtype = GgmlDType::Q2K; - - let src = get_test_vector2(0.5, 1024, device)?; - let quant = quantized::QTensor::quantize(&src, dtype)?; - let dst = quant.dequantize(device)?; - let dst_f16 = quant.dequantize_f16(device)?; - let diff = (dst.to_dtype(DType::F16)? - dst_f16)? - .to_dtype(DType::F32)? - .abs()? - .sum_all()? - .to_vec0::()?; - assert_eq!(diff, 0.); - - let src = src.to_vec1::()?; - let dst = dst.to_vec1::()?; - compare_with_error(dst.as_slice(), src.as_slice(), 0.1); - - // Test some specific values - assert_eq!( - [src[0], src[128], src[256], src[512], src[800], src[1023]], - [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] - ); - let dst = round_vector(&dst); - assert_eq!( - [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], - [-0.499, -0.366, -0.249, 0.0, 0.295, 0.492] - ); - - let src_big = get_test_vector2(128.0, 1024, device)?; - let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; - let dst_big = quant_big.dequantize(device)?; - let dst_big_f16 = quant_big.dequantize_f16(device)?; - let diff = (dst_big.to_dtype(DType::F16)? - dst_big_f16)? - .to_dtype(DType::F32)? - .abs()? - .sum_all()? - .to_vec0::()?; - assert_eq!(diff, 0.); - - let src_big = src_big.to_vec1::()?; - let dst_big = dst_big.to_vec1::()?; - compare_with_error(dst_big.as_slice(), src_big.as_slice(), 6.0); - - ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR_2BITS)?; - Ok(()) -} - -fn quantize_q3k(device: &Device) -> Result<()> { - let dtype = GgmlDType::Q3K; - let src = get_test_vector2(0.5, 1024, device)?; - let quant = quantized::QTensor::quantize(&src, dtype)?; - let dst = quant.dequantize(device)?; - let dst_f16 = quant.dequantize_f16(device)?; - let diff = (dst.to_dtype(DType::F16)? - dst_f16)? - .to_dtype(DType::F32)? - .abs()? - .sum_all()? - .to_vec0::()?; - assert_eq!(diff, 0.); - - let src = src.to_vec1::()?; - let dst = dst.to_vec1::()?; - compare_with_error(dst.as_slice(), src.as_slice(), 0.03); - - // Test some specific values - assert_eq!( - [src[0], src[128], src[256], src[512], src[800], src[1023]], - [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] - ); - let dst = round_vector(&dst); - assert_eq!( - [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], - [-0.493, -0.37, -0.243, -0.0, 0.292, 0.492] - ); - - let src_big = get_test_vector2(128.0, 1024, device)?; - let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; - let dst_big = quant_big.dequantize(device)?; - let dst_big_f16 = quant_big.dequantize_f16(device)?; - let diff = (dst_big.to_dtype(DType::F16)? - dst_big_f16)? - .to_dtype(DType::F32)? - .abs()? - .sum_all()? - .to_vec0::()?; - assert_eq!(diff, 0.); - - let src_big = src_big.to_vec1::()?; - let dst_big = dst_big.to_vec1::()?; - compare_with_error(dst_big.as_slice(), src_big.as_slice(), 3.5); - - ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR_3BITS)?; - Ok(()) -} - -fn quantize_q4k(device: &Device) -> Result<()> { - let dtype = GgmlDType::Q4K; - let src = get_test_vector2(0.5, 1024, device)?; - let quant = quantized::QTensor::quantize(&src, dtype)?; - let dst = quant.dequantize(device)?; - let dst_f16 = quant.dequantize_f16(device)?; - let diff = (dst.to_dtype(DType::F16)? - dst_f16)? - .to_dtype(DType::F32)? - .abs()? - .sum_all()? - .to_vec0::()?; - assert_eq!(diff, 0.); - - let src = src.to_vec1::()?; - let dst = dst.to_vec1::()?; - compare_with_error(dst.as_slice(), src.as_slice(), 0.017); - - // Test some specific values - assert_eq!( - [src[0], src[128], src[256], src[512], src[800], src[1023]], - [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] - ); - let dst = round_vector(&dst); - assert_eq!( - [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], - [-0.5, -0.373, -0.25, 0.0, 0.288, 0.498] - ); - - let src_big = get_test_vector2(128.0, 1024, device)?; - let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; - let dst_big = quant_big.dequantize(device)?; - let dst_big_f16 = quant_big.dequantize_f16(device)?; - let diff = (dst_big.to_dtype(DType::F16)? - dst_big_f16)? - .to_dtype(DType::F32)? - .abs()? - .sum_all()? - .to_vec0::()?; - assert_eq!(diff, 0.); - - let src_big = src_big.to_vec1::()?; - let dst_big = dst_big.to_vec1::()?; - compare_with_error(dst_big.as_slice(), src_big.as_slice(), 4.5); - - ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; - Ok(()) -} - -fn quantize_q5k(device: &Device) -> Result<()> { - let dtype = GgmlDType::Q5K; - let src = get_test_vector2(0.5, 1024, device)?; - let quant = quantized::QTensor::quantize(&src, dtype)?; - let dst = quant.dequantize(device)?; - let dst_f16 = quant.dequantize_f16(device)?; - let diff = (dst.to_dtype(DType::F16)? - dst_f16)? - .to_dtype(DType::F32)? - .abs()? - .sum_all()? - .to_vec0::()?; - assert_eq!(diff, 0.); - - let src = src.to_vec1::()?; - let dst = dst.to_vec1::()?; - compare_with_error(dst.as_slice(), src.as_slice(), 0.009); - - // Test some specific values - assert_eq!( - [src[0], src[128], src[256], src[512], src[800], src[1023]], - [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] - ); - let dst = round_vector(&dst); - assert_eq!( - [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], - [-0.5, -0.373, -0.25, 0.0, 0.279, 0.499] - ); - - let src_big = get_test_vector2(128.0, 1024, device)?; - let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; - let dst_big = quant_big.dequantize(device)?; - let dst_big_f16 = quant_big.dequantize_f16(device)?; - let diff = (dst_big.to_dtype(DType::F16)? - dst_big_f16)? - .to_dtype(DType::F32)? - .abs()? - .sum_all()? - .to_vec0::()?; - assert_eq!(diff, 0.); - - let src_big = src_big.to_vec1::()?; - let dst_big = dst_big.to_vec1::()?; - compare_with_error(dst_big.as_slice(), src_big.as_slice(), 2.5); - - ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; - Ok(()) -} - -fn quantize_q6k(device: &Device) -> Result<()> { - let dtype = GgmlDType::Q6K; - let src = get_test_vector2(0.5, 1024, device)?; - let quant = quantized::QTensor::quantize(&src, dtype)?; - let dst = quant.dequantize(device)?; - let dst_f16 = quant.dequantize_f16(device)?; - let diff = (dst.to_dtype(DType::F16)? - dst_f16)? - .to_dtype(DType::F32)? - .abs()? - .sum_all()? - .to_vec0::()?; - assert_eq!(diff, 0.); - - let src = src.to_vec1::()?; - let dst = dst.to_vec1::()?; - compare_with_error(dst.as_slice(), src.as_slice(), 0.008); - - // Test some specific values - assert_eq!( - [src[0], src[128], src[256], src[512], src[800], src[1023]], - [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] - ); - let dst = round_vector(&dst); - assert_eq!( - [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], - [-0.497, -0.372, -0.25, -0.0, 0.284, 0.5] - ); - - let src_big = get_test_vector2(128.0, 1024, device)?; - let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; - let dst_big = quant_big.dequantize(device)?; - let dst_big_f16 = quant_big.dequantize_f16(device)?; - let diff = (dst_big.to_dtype(DType::F16)? - dst_big_f16)? - .to_dtype(DType::F32)? - .abs()? - .sum_all()? - .to_vec0::()?; - assert_eq!(diff, 0.); - - let src_big = src_big.to_vec1::()?; - let dst_big = dst_big.to_vec1::()?; - compare_with_error(dst_big.as_slice(), src_big.as_slice(), 2.0); - - ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; - Ok(()) -} - -fn quantize_q8k(device: &Device) -> Result<()> { - let dtype = GgmlDType::Q8K; - let src = get_test_vector2(0.5, 1024, device)?; - let quant = quantized::QTensor::quantize(&src, dtype)?; - let dst = quant.dequantize(device)?; - let dst_f16 = quant.dequantize_f16(device)?; - let diff = (dst.to_dtype(DType::F16)? - dst_f16)? - .to_dtype(DType::F32)? - .abs()? - .sum_all()? - .to_vec0::()?; - assert_eq!(diff, 0.); - - let src = src.to_vec1::()?; - let dst = dst.to_vec1::()?; - compare_with_error(dst.as_slice(), src.as_slice(), 0.008); - - // Test some specific values - assert_eq!( - [src[0], src[128], src[256], src[512], src[800], src[1023]], - [-0.5, -0.375, -0.25, 0.0, 0.28125, 0.49902344] - ); - let dst = round_vector(&dst); - assert_eq!( - [dst[0], dst[128], dst[256], dst[512], dst[800], dst[1023]], - [-0.5, -0.375, -0.25, -0.0, 0.281, 0.499] - ); - - let src_big = get_test_vector2(128.0, 1024, device)?; - let quant_big = quantized::QTensor::quantize(&src_big, dtype)?; - let dst_big = quant_big.dequantize(device)?; - let dst_big_f16 = quant_big.dequantize_f16(device)?; - let diff = (dst_big.to_dtype(DType::F16)? - dst_big_f16)? - .to_dtype(DType::F32)? - .abs()? - .sum_all()? - .to_vec0::()?; - assert_eq!(diff, 0.); - - let src_big = src_big.to_vec1::()?; - let dst_big = dst_big.to_vec1::()?; - compare_with_error(dst_big.as_slice(), src_big.as_slice(), 0.6); - - ggml_quantization_error_test(dtype, device, GGML_MAX_QUANTIZATION_TOTAL_ERROR)?; - Ok(()) -} - -test_device!( - quantize_q4_0, - quantize_q4_0_cpu, - quantize_q4_0_cuda, - quantize_q4_0_metal -); -test_device!( - quantize_q4_1, - quantize_q4_1_cpu, - quantize_q4_1_cuda, - quantize_q4_1_metal -); -test_device!( - quantize_q5_0, - quantize_q5_0_cpu, - quantize_q5_0_cuda, - quantize_q5_0_metal -); -test_device!( - quantize_q5_1, - quantize_q5_1_cpu, - quantize_q5_1_cuda, - quantize_q5_1_metal -); -test_device!( - quantize_q2k, - quantize_q2k_cpu, - quantize_q2k_cuda, - quantize_q2k_metal -); -test_device!( - quantize_q3k, - quantize_q3k_cpu, - quantize_q3k_cuda, - quantize_q3k_metal -); -test_device!( - quantize_q4k, - quantize_q4k_cpu, - quantize_q4k_cuda, - quantize_q4k_metal -); -test_device!( - quantize_q5k, - quantize_q5k_cpu, - quantize_q5k_cuda, - quantize_q5k_metal -); -test_device!( - quantize_q6k, - quantize_q6k_cpu, - quantize_q6k_cuda, - quantize_q6k_metal -); -test_device!( - quantize_q8k, - quantize_q8k_cpu, - quantize_q8k_cuda, - quantize_q8k_metal -); - -/// Very simple dot product implementation -fn vec_dot_reference(a: &[f32], b: &[f32]) -> f32 { - a.iter().zip(b).map(|(a, b)| a * b).sum() -} - -/// Returns the error achieved by the GGML matmul unit test. -fn ggml_reference_matmul_error(dtype: GgmlDType) -> Result { - let err = match dtype { - GgmlDType::F32 => 0.000000, - GgmlDType::F16 => 0.000010, - GgmlDType::BF16 => 0.000200, - GgmlDType::Q2K => 0.004086, - GgmlDType::Q3K => 0.016148, - GgmlDType::Q4K => 0.002425, - GgmlDType::Q5K => 0.000740, - GgmlDType::Q6K => 0.000952, - GgmlDType::Q4_0 => 0.001143, - GgmlDType::Q4_1 => 0.008, - GgmlDType::Q5_0 => 0.001353, - GgmlDType::Q5_1 => 0.00149, - GgmlDType::Q8_0 => 0.000092, - GgmlDType::Q8_1 => 0.000092, - - // Not from the ggml repo. - GgmlDType::Q8K => 0.00065, - }; - Ok(err) -} - -/// Similar to the GGML matmul unit test: -/// https://github.com/ggerganov/llama.cpp/blob/master/tests/test-quantize-fns.cpp#L76-L91 -fn ggml_matmul_error_test() -> Result<()> { - let a = create_ggml_like_vector(0.0); - let b = create_ggml_like_vector(1.0); - ggml_matmul_error_test_::(a.as_slice(), b.as_slice(), 1.0)?; - // Another example that is more likely to trigger the overflow reported in #1526 - let a = (0..GGML_TEST_SIZE) - .map(|i| i as f32 / GGML_TEST_SIZE as f32) - .collect::>(); - let b = (0..GGML_TEST_SIZE) - .map(|i| i as f32 / GGML_TEST_SIZE as f32) - .collect::>(); - ggml_matmul_error_test_::(a.as_slice(), b.as_slice(), 2.0)?; - Ok(()) -} - -fn ggml_matmul_error_test_(a: &[f32], b: &[f32], err_m: f32) -> Result<()> { - let length = a.len(); - - let mut a_quant = vec![T::zeros(); length / T::BLCK_SIZE]; - let mut b_quant = vec![T::VecDotType::zeros(); length / T::VecDotType::BLCK_SIZE]; - T::from_float(a, &mut a_quant); - T::VecDotType::from_float(b, &mut b_quant); - - let result = T::vec_dot(length, &a_quant, &b_quant); - let result_unopt = T::vec_dot_unopt(length, &a_quant, &b_quant); - - if (result - result_unopt).abs() / length as f32 > 1e-6 { - bail!( - "the opt and unopt vec-dot returned different values, opt: {result} vs unopt: {result_unopt}" - ) - } - - let mut dst = vec![0.0f32; 1]; - crate::k_quants::matmul((1, length, 1), b, &a_quant, &mut dst)?; - let result_matmul = dst[0]; - - if (result_matmul - result).abs() / length as f32 > 1e-6 { - bail!( - "calling matmul vs calling vec-dot directly returned different values, matmul: {result_matmul} vs vec-dot: {result}" - ) - } - - let reference_result = vec_dot_reference(a, b); - - let verify_result = |result: f32, source: &str| { - let error = (result - reference_result).abs() / length as f32; - let ggml_error = ggml_reference_matmul_error(T::DTYPE)? * err_m; - if !error.is_finite() || error > GGML_MAX_DOT_PRODUCT_ERROR { - bail!("Dot product with dtype {:?} error {error} exceeds max error {GGML_MAX_DOT_PRODUCT_ERROR}. Source: {source}", T::DTYPE); - } - // We diverge slightly due to different rounding behavior / f16 to f32 conversions in GGML - // => we use a slightly higher error threshold - const ERROR_LENIENCY: f32 = 0.00001; - if error - ERROR_LENIENCY > ggml_error { - bail!( - "Dot product with dtype {:?} error {error} exceeds ggml reference error {ggml_error}. Source: {source}", - T::DTYPE, - ); - } - Ok(()) - }; - - verify_result(result, "vec-dot")?; - verify_result(result_matmul, "matmul")?; - Ok(()) -} - -#[test] -fn quantized_mm() -> Result<()> { - ggml_matmul_error_test::()?; - ggml_matmul_error_test::()?; - //ggml_matmul_error_test::()?; TODO: Fails on ubuntu and windows. Check CpuBF16 impl - ggml_matmul_error_test::()?; - ggml_matmul_error_test::()?; - ggml_matmul_error_test::()?; - ggml_matmul_error_test::()?; - ggml_matmul_error_test::()?; - ggml_matmul_error_test::()?; - Ok(()) -} - -/// generates random tensors of size `m x k` and `n x k` and calculates their expected matrix multiplication result. -fn get_random_tensors( - m: usize, - k: usize, - n: usize, - device: &Device, -) -> Result<(Tensor, Tensor, Tensor)> { - let mut rng = StdRng::seed_from_u64(314159265358979); - - let lhs = (0..m * k) - .map(|_| rng.random::() - 0.5) - .collect::>(); - let rhs = (0..n * k) - .map(|_| rng.random::() - 0.5) - .collect::>(); - - let lhs = Tensor::from_vec(lhs, (m, k), device)?; - let rhs = Tensor::from_vec(rhs, (n, k), device)?; - - let mm = lhs.matmul(&rhs.t()?)?; - Ok((lhs, rhs, mm)) -} - -#[macro_export] -macro_rules! quantized_matmul { - // TODO: Switch to generating the two last arguments automatically once concat_idents is - // stable. https://github.com/rust-lang/rust/issues/29599 - ($fn_name: ident, $fn_name_cpu: ident, $fn_name_cuda: ident, $fn_name_metal: ident, $dtype: expr) => { - fn $fn_name(device: &Device) -> Result<()> { - test_matmul(device, (1, 3, 4, 256), $dtype)?; - Ok(()) - } - - test_device!($fn_name, $fn_name_cpu, $fn_name_cuda, $fn_name_metal); - }; -} - -quantized_matmul!( - quantized_matmul_q4_0_bis, - quantized_matmul_q4_0_cpu, - quantized_matmul_q4_0_cuda, - quantized_matmul_q4_0_metal, - GgmlDType::Q4_0 -); -quantized_matmul!( - quantized_matmul_q4_1_bis, - quantized_matmul_q4_1_cpu, - quantized_matmul_q4_1_cuda, - quantized_matmul_q4_1_metal, - GgmlDType::Q4_1 -); -quantized_matmul!( - quantized_matmul_q5_0_bis, - quantized_matmul_q5_0_cpu, - quantized_matmul_q5_0_cuda, - quantized_matmul_q5_0_metal, - GgmlDType::Q5_0 -); -quantized_matmul!( - quantized_matmul_q5_1_bis, - quantized_matmul_q5_1_cpu, - quantized_matmul_q5_1_cuda, - quantized_matmul_q5_1_metal, - GgmlDType::Q5_1 -); -quantized_matmul!( - quantized_matmul_q8_0_bis, - quantized_matmul_q8_0_cpu, - quantized_matmul_q8_0_cuda, - quantized_matmul_q8_0_metal, - GgmlDType::Q8_0 -); -quantized_matmul!( - quantized_matmul_q8_1_bis, - quantized_matmul_q8_1_cpu, - quantized_matmul_q8_1_cuda, - quantized_matmul_q8_1_metal, - GgmlDType::Q8_1 -); -quantized_matmul!( - quantized_matmul_q2k_bis, - quantized_matmul_q2k_cpu, - quantized_matmul_q2k_cuda, - quantized_matmul_q2k_metal, - GgmlDType::Q2K -); -quantized_matmul!( - quantized_matmul_q3k_bis, - quantized_matmul_q3k_cpu, - quantized_matmul_q3k_cuda, - quantized_matmul_q3k_metal, - GgmlDType::Q3K -); -quantized_matmul!( - quantized_matmul_q4k_bis, - quantized_matmul_q4k_cpu, - quantized_matmul_q4k_cuda, - quantized_matmul_q4k_metal, - GgmlDType::Q4K -); -quantized_matmul!( - quantized_matmul_q5k_bis, - quantized_matmul_q5k_cpu, - quantized_matmul_q5k_cuda, - quantized_matmul_q5k_metal, - GgmlDType::Q5K -); -quantized_matmul!( - quantized_matmul_q6k_bis, - quantized_matmul_q6k_cpu, - quantized_matmul_q6k_cuda, - quantized_matmul_q6k_metal, - GgmlDType::Q6K -); -// Not implemented on metal -quantized_matmul!( - quantized_matmul_q8k_bis, - quantized_matmul_q8k_cpu, - quantized_matmul_q8k_cuda, - quantized_matmul_q8k_metal, - GgmlDType::Q8K -); - -#[test] -fn quantized_matmul_q2k() -> Result<()> { - use k_quants::BlockQ2K; - - let cpu = &Device::Cpu; - let (m, k, n) = (11, 512, 21); - let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); - - let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q2K)?; - let rhs = quantized::QMatMul::from_qtensor(rhs)?; - let mm = rhs.forward(&lhs)?; - - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [0.916, 0.422, 0.215, 1.668]); - - ggml_matmul_error_test::()?; - - Ok(()) -} - -#[test] -fn quantized_matmul_q3k() -> Result<()> { - use k_quants::BlockQ3K; - - let cpu = &Device::Cpu; - let (m, k, n) = (11, 512, 21); - let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); - - let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q3K)?; - let rhs = quantized::QMatMul::from_qtensor(rhs)?; - let mm = rhs.forward(&lhs)?; - - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [1.029, 1.418, -0.314, 1.495]); - - ggml_matmul_error_test::()?; - - Ok(()) -} - -#[test] -fn quantized_matmul_q4k() -> Result<()> { - use k_quants::BlockQ4K; - - let cpu = &Device::Cpu; - let (m, k, n) = (11, 512, 21); - let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); - - let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q4K)?; - let rhs = quantized::QMatMul::from_qtensor(rhs)?; - let mm = rhs.forward(&lhs)?; - - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [1.125, 1.435, -0.201, 1.589]); - - ggml_matmul_error_test::()?; - - Ok(()) -} - -#[test] -fn quantized_matmul_q5k() -> Result<()> { - use k_quants::BlockQ5K; - - let cpu = &Device::Cpu; - let (m, k, n) = (11, 512, 21); - let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); - - let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q5K)?; - let rhs = quantized::QMatMul::from_qtensor(rhs)?; - let mm = rhs.forward(&lhs)?; - - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [1.192, 1.491, -0.18, 1.743]); - - //Expected: 0.000740408897 - ggml_matmul_error_test::()?; - - Ok(()) -} - -#[test] -fn quantized_matmul_q6k() -> Result<()> { - use k_quants::BlockQ6K; - - let cpu = &Device::Cpu; - let (m, k, n) = (11, 512, 21); - let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); - - let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q6K)?; - let rhs = quantized::QMatMul::from_qtensor(rhs)?; - let mm = rhs.forward(&lhs)?; - - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [1.324, 1.49, -0.164, 1.741]); - - ggml_matmul_error_test::()?; - Ok(()) -} - -#[test] -fn quantized_matmul_q8k() -> Result<()> { - use k_quants::BlockQ8K; - - let cpu = &Device::Cpu; - let (m, k, n) = (11, 512, 21); - let (lhs, rhs, mm) = get_random_tensors(m, k, n, cpu)?; - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [1.262, 1.513, -0.208, 1.702]); - - let rhs = quantized::QTensor::quantize(&rhs, GgmlDType::Q8K)?; - let rhs = quantized::QMatMul::from_qtensor(rhs)?; - let mm = rhs.forward(&lhs)?; - - assert_eq!(mm.dims(), [m, n]); - let dst = mm.flatten_all()?.to_vec1::()?; - let dst = round_vector(&[dst[0], dst[m * n / 3], dst[m * n * 2 / 3], dst[m * n - 1]]); - assert_eq!(dst, [1.266, 1.504, -0.204, 1.7]); - - ggml_matmul_error_test::()?; - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/serialization_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/serialization_tests.rs deleted file mode 100644 index f81350e6e1..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/tests/serialization_tests.rs +++ /dev/null @@ -1,71 +0,0 @@ -use candle_core::{DType, Result, Tensor}; - -struct TmpFile(std::path::PathBuf); - -impl TmpFile { - fn create(base: &str) -> TmpFile { - let filename = std::env::temp_dir().join(format!( - "candle-{}-{}-{:?}", - base, - std::process::id(), - std::thread::current().id(), - )); - TmpFile(filename) - } -} - -impl std::convert::AsRef for TmpFile { - fn as_ref(&self) -> &std::path::Path { - self.0.as_path() - } -} - -impl Drop for TmpFile { - fn drop(&mut self) { - std::fs::remove_file(&self.0).unwrap() - } -} - -#[test] -fn npy() -> Result<()> { - let npy = Tensor::read_npy("tests/test.npy")?; - assert_eq!( - npy.to_dtype(DType::U8)?.to_vec1::()?, - [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - ); - Ok(()) -} - -#[test] -fn npz() -> Result<()> { - let npz = Tensor::read_npz("tests/test.npz")?; - assert_eq!(npz.len(), 2); - assert_eq!(npz[0].0, "x"); - assert_eq!(npz[1].0, "x_plus_one"); - assert_eq!( - npz[1].1.to_dtype(DType::U8)?.to_vec1::()?, - [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - ); - Ok(()) -} - -#[test] -fn safetensors() -> Result<()> { - use candle_core::safetensors::Load; - - let tmp_file = TmpFile::create("st"); - let t = Tensor::arange(0f32, 24f32, &candle_core::Device::Cpu)?; - t.save_safetensors("t", &tmp_file)?; - // Load from file. - let st = candle_core::safetensors::load(&tmp_file, &candle_core::Device::Cpu)?; - let t2 = st.get("t").unwrap(); - let diff = (&t - t2)?.abs()?.sum_all()?.to_vec0::()?; - assert_eq!(diff, 0f32); - // Load from bytes. - let bytes = std::fs::read(tmp_file)?; - let st = candle_core::safetensors::SliceSafetensors::new(&bytes)?; - let t2 = st.get("t").unwrap().load(&candle_core::Device::Cpu); - let diff = (&t - t2)?.abs()?.sum_all()?.to_vec0::()?; - assert_eq!(diff, 0f32); - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/tensor_tests.rs b/v2/dependency-patches/candle-core-0.9.2/tests/tensor_tests.rs deleted file mode 100644 index 5422353b0e..0000000000 --- a/v2/dependency-patches/candle-core-0.9.2/tests/tensor_tests.rs +++ /dev/null @@ -1,2062 +0,0 @@ -use candle_core::{test_device, test_utils, DType, Device, IndexOp, Result, Tensor, D}; -use float8::F8E4M3; - -fn zeros(device: &Device) -> Result<()> { - let tensor = Tensor::zeros((5, 2), DType::F32, device)?; - let (dim1, dim2) = tensor.dims2()?; - assert_eq!(dim1, 5); - assert_eq!(dim2, 2); - Ok(()) -} - -fn ones(device: &Device) -> Result<()> { - assert_eq!( - Tensor::ones((2, 3), DType::U8, device)?.to_vec2::()?, - [[1, 1, 1], [1, 1, 1]], - ); - assert_eq!( - Tensor::ones((2, 3), DType::U32, device)?.to_vec2::()?, - [[1, 1, 1], [1, 1, 1]], - ); - assert_eq!( - Tensor::ones((2, 3), DType::I64, device)?.to_vec2::()?, - [[1, 1, 1], [1, 1, 1]], - ); - assert_eq!( - Tensor::ones((2, 3), DType::F32, device)?.to_vec2::()?, - [[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]], - ); - if !device.is_metal() { - assert_eq!( - Tensor::ones((2, 3), DType::F64, device)?.to_vec2::()?, - [[1.0, 1.0, 1.0], [1.0, 1.0, 1.0]], - ); - } - assert_eq!( - Tensor::ones((2, 3), DType::F16, device)?.to_vec2::()?, - [ - [ - half::f16::from_f32(1.0), - half::f16::from_f32(1.0), - half::f16::from_f32(1.0) - ], - [ - half::f16::from_f32(1.0), - half::f16::from_f32(1.0), - half::f16::from_f32(1.0) - ] - ], - ); - assert_eq!( - Tensor::ones((2, 3), DType::BF16, device)?.to_vec2::()?, - [ - [ - half::bf16::from_f32(1.0), - half::bf16::from_f32(1.0), - half::bf16::from_f32(1.0) - ], - [ - half::bf16::from_f32(1.0), - half::bf16::from_f32(1.0), - half::bf16::from_f32(1.0) - ] - ], - ); - - if !device.is_metal() { - assert_eq!( - Tensor::ones((2, 3), DType::F8E4M3, device)?.to_vec2::()?, - [ - [ - F8E4M3::from_f32(1.), - F8E4M3::from_f32(1.), - F8E4M3::from_f32(1.) - ], - [ - F8E4M3::from_f32(1.), - F8E4M3::from_f32(1.), - F8E4M3::from_f32(1.) - ] - ], - ); - } - Ok(()) -} - -fn full(device: &Device) -> Result<()> { - let tensor = Tensor::zeros((3, 4), DType::U32, device)?; - tensor.const_set(42u32.into())?; - assert_eq!( - tensor.to_vec2::()?, - [[42, 42, 42, 42], [42, 42, 42, 42], [42, 42, 42, 42]] - ); - tensor.i((.., 2))?.const_set(1337u32.into())?; - assert_eq!( - tensor.to_vec2::()?, - [[42, 42, 1337, 42], [42, 42, 1337, 42], [42, 42, 1337, 42]] - ); - tensor.i((2, ..))?.const_set(1u32.into())?; - assert_eq!( - tensor.to_vec2::()?, - [[42, 42, 1337, 42], [42, 42, 1337, 42], [1, 1, 1, 1]] - ); - Ok(()) -} - -fn const_set(device: &Device) -> Result<()> { - assert_eq!( - Tensor::full(42u32, (2, 3), device)?.to_vec2::()?, - [[42, 42, 42], [42, 42, 42]], - ); - Ok(()) -} - -fn arange(device: &Device) -> Result<()> { - assert_eq!( - Tensor::arange(0u8, 5u8, device)?.to_vec1::()?, - [0, 1, 2, 3, 4], - ); - assert_eq!( - Tensor::arange_step(0u8, 5u8, 2, device)?.to_vec1::()?, - [0, 2, 4], - ); - assert_eq!( - Tensor::arange_step(0u8, 5u8, 3, device)?.to_vec1::()?, - [0, 3], - ); - assert_eq!( - Tensor::arange_step(5i64, 0i64, -1, device)?.to_vec1::()?, - [5, 4, 3, 2, 1], - ); - - if !device.is_metal() { - assert_eq!( - Tensor::arange_step( - F8E4M3::from_f32(0.), - F8E4M3::from_f32(5.), - F8E4M3::from_f32(2.), - device - )? - .to_vec1::()?, - [ - F8E4M3::from_f32(0.), - F8E4M3::from_f32(2.), - F8E4M3::from_f32(4.), - ], - ); - } - - Ok(()) -} - -fn add_mul(device: &Device) -> Result<()> { - let tensor = Tensor::new(&[3f32, 1., 4.], device)?; - let dim1 = tensor.dims1()?; - assert_eq!(dim1, 3); - let content: Vec = tensor.to_vec1()?; - assert_eq!(content, [3., 1., 4.]); - let tensor = Tensor::add(&tensor, &tensor)?; - let content: Vec = tensor.to_vec1()?; - assert_eq!(content, [6., 2., 8.]); - let tensor = Tensor::mul(&tensor, &tensor)?; - let content: Vec = tensor.to_vec1()?; - assert_eq!(content, [36., 4., 64.]); - Ok(()) -} - -fn tensor_2d(device: &Device) -> Result<()> { - let data = &[[3f32, 1., 4., 1., 5.], [2., 1., 7., 8., 2.]]; - let tensor = Tensor::new(data, device)?; - let dims = tensor.dims2()?; - assert_eq!(dims, (2, 5)); - let content: Vec> = tensor.to_vec2()?; - assert_eq!(content, data); - Ok(()) -} - -fn clamp(device: &Device) -> Result<()> { - let data = &[[3f32, 1., 4., 1., 5.], [2., 1., 7., 8., 2.]]; - let tensor = Tensor::new(data, device)?; - let tensor = tensor.clamp(1.5, 6.2)?; - assert_eq!( - tensor.to_vec2::()?, - [[3.0, 1.5, 4.0, 1.5, 5.0], [2.0, 1.5, 6.2, 6.2, 2.0]], - ); - Ok(()) -} - -fn asort(device: &Device) -> Result<()> { - let data = &[[3f32, 1., 4., 1.1, 5.], [2.1, 1., 7., 8., 2.]]; - let tensor = Tensor::new(data, device)?; - let indexes = tensor.arg_sort_last_dim(true)?; - assert_eq!( - indexes.to_vec2::()?, - [[1, 3, 0, 2, 4], [1, 4, 0, 2, 3]], - ); - let indexes = tensor.arg_sort_last_dim(false)?; - assert_eq!( - indexes.to_vec2::()?, - [[4, 2, 0, 3, 1], [3, 2, 0, 4, 1]], - ); - let (sorted, indexes) = tensor.sort_last_dim(true)?; - assert_eq!( - indexes.to_vec2::()?, - [[1, 3, 0, 2, 4], [1, 4, 0, 2, 3]], - ); - assert_eq!( - sorted.to_vec2::()?, - [[1.0, 1.1, 3.0, 4.0, 5.0], [1.0, 2.0, 2.1, 7.0, 8.0]] - ); - let (sorted, indexes) = tensor.sort_last_dim(false)?; - assert_eq!( - indexes.to_vec2::()?, - [[4, 2, 0, 3, 1], [3, 2, 0, 4, 1]], - ); - assert_eq!( - sorted.to_vec2::()?, - [[5.0, 4.0, 3.0, 1.1, 1.0], [8.0, 7.0, 2.1, 2.0, 1.0]] - ); - Ok(()) -} - -/// Test sorting a large tensor that exceeds 1024 elements. -fn asort_big(device: &Device) -> Result<()> { - // Skip on metal for now - if device.is_metal() { - return Ok(()); - } - const SIZE: usize = 2000; - let data: Vec = (0..SIZE).map(|x| (SIZE - x) as f32).collect(); - let tensor = Tensor::new(data.as_slice(), device)?; - - let indexes = tensor.arg_sort_last_dim(true)?; - let expected_indexes: Vec = (0..SIZE).rev().map(|x| x as u32).collect(); - assert_eq!(indexes.to_vec1::()?, expected_indexes); - - let indexes = tensor.arg_sort_last_dim(false)?; - let expected_indexes: Vec = (0..SIZE).map(|x| x as u32).collect(); - assert_eq!(indexes.to_vec1::()?, expected_indexes); - Ok(()) -} - -fn unary_op(device: &Device) -> Result<()> { - let data = &[[-3f32, 1., 4., -0.1, 0.5], [2.7, -1.8, -0.28, 1.8, 2.8]]; - let tensor = Tensor::new(data, device)?; - assert_eq!( - test_utils::to_vec2_round(&tensor.gelu()?, 4)?, - [ - [-0.0036, 0.8412, 3.9999, -0.046, 0.3457], - [2.6911, -0.0647, -0.1091, 1.7353, 2.7933] - ] - ); - let t_f16 = tensor.to_dtype(DType::F16)?.gelu()?.to_dtype(DType::F32)?; - let max_diff = (tensor.gelu()? - t_f16)?.flatten_all()?.max(0)?; - assert!(max_diff.to_vec0::()? < 5e-3); - assert_eq!( - test_utils::to_vec2_round(&tensor.gelu_erf()?, 4)?, - [ - [-0.004, 0.8413, 3.9999, -0.046, 0.3457], - [2.6906, -0.0647, -0.1091, 1.7353, 2.7928] - ] - ); - assert_eq!( - test_utils::to_vec2_round(&tensor.erf()?, 4)?, - [ - [-1.0, 0.8427, 1.0, -0.1125, 0.5205], - [0.9999, -0.9891, -0.3079, 0.9891, 0.9999] - ] - ); - assert_eq!( - test_utils::to_vec2_round(&tensor.silu()?, 4)?, - [ - [-0.1423, 0.7311, 3.9281, -0.0475, 0.3112], - [2.53, -0.2553, -0.1205, 1.5447, 2.6395] - ] - ); - assert_eq!( - test_utils::to_vec2_round(&tensor.ceil()?, 4)?, - [[-3.0, 1.0, 4.0, -0.0, 1.0], [3.0, -1.0, -0.0, 2.0, 3.0]] - ); - assert_eq!( - test_utils::to_vec2_round(&tensor.floor()?, 4)?, - [[-3.0, 1.0, 4.0, -1.0, 0.0], [2.0, -2.0, -1.0, 1.0, 2.0]] - ); - assert_eq!( - test_utils::to_vec2_round(&tensor.round()?, 4)?, - [[-3.0, 1.0, 4.0, -0.0, 1.0], [3.0, -2.0, -0.0, 2.0, 3.0]] - ); - let tensor = Tensor::new(&[2997.9246, 314.15926f32], device)?; - assert_eq!( - test_utils::to_vec1_round(&tensor.round_to(2)?, 4)?, - [2997.92, 314.16] - ); - assert_eq!( - test_utils::to_vec1_round(&tensor.round_to(-2)?, 4)?, - [3000.0, 300.] - ); - let tensor = Tensor::new( - &[-1.01f32, -0.9, -0.1, 0.0, -0.0, 0.1, 0.9, 1.0, 1.1], - device, - )?; - assert_eq!( - tensor.sign()?.to_vec1::()?, - [-1., -1., -1., 0., 0., 1., 1., 1., 1.] - ); - let tensor = Tensor::new(&[-1.0f32, 0., -2., 3.], device)?; - let y = tensor.elu(2.)?; - assert_eq!( - test_utils::to_vec1_round(&y, 4)?, - [-1.2642, 0.0000, -1.7293, 3.0000] - ); - // This test failed on metal prior to the following PR: - // https://github.com/huggingface/candle/pull/2490 - let y = tensor.reshape((2, 2))?.t()?.elu(2.)?.flatten_all()?; - assert_eq!( - test_utils::to_vec1_round(&y, 4)?, - [-1.2642, -1.7293, 0.0000, 3.0000] - ); - Ok(()) -} - -fn binary_op(device: &Device) -> Result<()> { - let data = &[[3f32, 1., 4., 1., 5.], [2., 1., 7., 8., 2.]]; - let tensor1 = Tensor::new(data, device)?; - let data2 = &[[5f32, 5., 5., 5., 5.], [2., 1., 7., 8., 2.]]; - let tensor2 = Tensor::new(data2, device)?; - let tensor = (&tensor1 + (&tensor1 * &tensor1)? / (&tensor1 + &tensor2))?; - let dims = tensor.dims2()?; - assert_eq!(dims, (2, 5)); - let content: Vec> = tensor.to_vec2()?; - assert_eq!(content[0], [4.125, 1.1666666, 5.7777777, 1.1666666, 7.5]); - assert_eq!(content[1], [3.0, 1.5, 10.5, 12.0, 3.0]); - #[allow(clippy::eq_op)] - let tensor = (&tensor - &tensor)?; - let content: Vec> = tensor.to_vec2()?; - assert_eq!(content[0], [0., 0., 0., 0., 0.]); - - let min = tensor1.minimum(&(&tensor2 * 0.5)?)?; - let max = tensor1.maximum(&(&tensor2 * 0.5)?)?; - assert_eq!( - min.to_vec2::()?, - [[2.5, 1.0, 2.5, 1.0, 2.5], [1.0, 0.5, 3.5, 4.0, 1.0]], - ); - assert_eq!( - max.to_vec2::()?, - [[3.0, 2.5, 4.0, 2.5, 5.0], [2.0, 1.0, 7.0, 8.0, 2.0]] - ); - Ok(()) -} - -fn ternary_op(device: &Device) -> Result<()> { - let data = &[[0u8, 1, 0, 1, 0], [1, 1, 1, 0, 0]]; - let ids = Tensor::new(data, device)?; - let data = &[[0f32, 1., 2., 3., 4.], [5., 6., 7., 8., 9.]]; - let a = Tensor::new(data, device)?; - let data = &[[10f32, 11., 12., 13., 14.], [15., 16., 17., 18., 19.]]; - let b = Tensor::new(data, device)?; - let tensor = ids.where_cond(&a, &b)?; - let dims = tensor.dims(); - assert_eq!(dims, [2, 5]); - let result: Vec = tensor.flatten_all()?.to_vec1()?; - assert_eq!(result, [10., 1., 12., 3., 14., 5., 6., 7., 18., 19.]); - Ok(()) -} - -fn transpose(device: &Device) -> Result<()> { - let data = &[[3f32, 1., 4., 1., 5.], [2., 1., 7., 8., 2.]]; - let tensor = Tensor::new(data, device)?.t()?; - let dims = tensor.dims2()?; - assert_eq!(dims, (5, 2)); - assert_eq!( - tensor.to_vec2::()?, - &[[3f32, 2.], [1., 1.], [4., 7.], [1., 8.], [5., 2.]] - ); - assert_eq!(tensor.t()?.to_vec2::()?, data); - assert_eq!(tensor.contiguous()?.t()?.to_vec2::()?, data); - assert_eq!(((tensor + 1.)?.t()? - 1.)?.to_vec2::()?, data); - Ok(()) -} - -fn var(device: &Device) -> Result<()> { - // Values taken from https://pytorch.org/docs/stable/generated/torch.var.html - let data = &[ - [0.2035f32, 1.2959, 1.8101, -0.4644], - [1.5027, -0.3270, 0.5905, 0.6538], - [-1.5745, 1.3330, -0.5596, -0.6548], - [0.1264, -0.5080, 1.6420, 0.1992], - ]; - let tensor = Tensor::new(data, device)?; - assert_eq!( - test_utils::to_vec2_round(&tensor.var_keepdim(1)?, 4)?, - &[[1.0631], [0.559], [1.4893], [0.8258]] - ); - Ok(()) -} - -fn sum(device: &Device) -> Result<()> { - let data = &[[[3u32, 1, 4], [1, 5, 9]], [[2, 1, 7], [8, 2, 8]]]; - let tensor = Tensor::new(data, device)?; - assert_eq!( - tensor.sum_keepdim(2)?.to_vec3::()?, - &[[[8], [15]], [[10], [18]]] - ); - assert_eq!( - tensor.sum_keepdim(0)?.to_vec3::()?, - &[[[5, 2, 11], [9, 7, 17]]], - ); - assert_eq!(tensor.sum_keepdim((0, 2, 1))?.to_vec3::()?, &[[[51]]],); - assert_eq!( - tensor.t()?.sum_keepdim(1)?.t()?.to_vec3::()?, - &[[[8], [15]], [[10], [18]]] - ); - assert_eq!( - tensor.sum_keepdim((2, 1))?.to_vec3::()?, - &[[[8 + 15]], [[10 + 18]]] - ); - let data: Vec = (0..4000u32).collect(); - let tensor = Tensor::new(data.as_slice(), device)?; - assert_eq!(tensor.sum_keepdim(0)?.to_vec1::()?, &[7998000]); - let tensor = tensor.reshape((2000, 2))?; - assert_eq!(tensor.sum_keepdim((0, 1))?.to_vec2::()?, &[[7998000]]); - assert_eq!( - tensor.sum_keepdim(0)?.sum_keepdim(1)?.to_vec2::()?, - &[[7998000]] - ); - assert_eq!( - tensor.sum_keepdim(1)?.sum_keepdim(0)?.to_vec2::()?, - &[[7998000]] - ); - assert_eq!( - tensor.sum_keepdim(0)?.to_vec2::()?, - &[[3998000, 4000000]] - ); - - // Make the tensor non contiguous. - let tensor = tensor.t()?.contiguous()?.t()?; - assert_eq!(tensor.sum_keepdim((0, 1))?.to_vec2::()?, &[[7998000]]); - assert_eq!( - tensor.sum_keepdim(0)?.sum_keepdim(1)?.to_vec2::()?, - &[[7998000]] - ); - assert_eq!( - tensor.sum_keepdim(1)?.sum_keepdim(0)?.to_vec2::()?, - &[[7998000]] - ); - assert_eq!( - tensor.sum_keepdim(0)?.to_vec2::()?, - &[[3998000, 4000000]] - ); - - let t1 = tensor.reshape((200, 5, 4))?; - let t2 = t1.transpose(0, 2)?.contiguous()?.transpose(0, 2)?; - for tensor in [t1, t2] { - assert_eq!( - tensor.sum_keepdim((0, 1, 2))?.to_vec3::()?, - &[[[7998000]]] - ); - assert_eq!( - tensor - .sum_keepdim(0)? - .sum_keepdim(2)? - .sum_keepdim(1)? - .to_vec3::()?, - &[[[7998000]]] - ); - assert_eq!( - tensor - .sum_keepdim(0)? - .sum_keepdim((1, 2))? - .to_vec3::()?, - &[[[7998000]]] - ); - assert_eq!( - tensor - .sum_keepdim(1)? - .sum_keepdim((0, 2))? - .to_vec3::()?, - &[[[7998000]]] - ); - assert_eq!( - tensor.sum_keepdim(0)?.to_vec3::()?, - &[[ - [398000, 398200, 398400, 398600], - [398800, 399000, 399200, 399400], - [399600, 399800, 400000, 400200], - [400400, 400600, 400800, 401000], - [401200, 401400, 401600, 401800] - ]] - ); - } - Ok(()) -} - -fn min(device: &Device) -> Result<()> { - let data = &[[[3u32, 1, 4], [1, 5, 9]], [[2, 1, 7], [8, 2, 8]]]; - let tensor = Tensor::new(data, device)?; - assert_eq!( - tensor.min_keepdim(2)?.to_vec3::()?, - &[[[1], [1]], [[1], [2]]] - ); - assert_eq!( - tensor.min_keepdim(0)?.to_vec3::()?, - &[[[2, 1, 4], [1, 2, 8]]], - ); - let data: Vec = (200..4000u32).collect(); - let tensor = Tensor::new(data.as_slice(), device)?; - assert_eq!(tensor.min_keepdim(0)?.to_vec1::()?, &[200]); - let tensor = tensor.reshape((1900, 2))?; - assert_eq!( - tensor.min_keepdim(0)?.min_keepdim(1)?.to_vec2::()?, - &[[200]] - ); - assert_eq!( - tensor.min_keepdim(1)?.min_keepdim(0)?.to_vec2::()?, - &[[200]] - ); - assert_eq!(tensor.min_keepdim(0)?.to_vec2::()?, &[[200, 201]]); - - // Make the tensor non contiguous. - let tensor = tensor.t()?.contiguous()?.t()?; - assert_eq!( - tensor.min_keepdim(0)?.min_keepdim(1)?.to_vec2::()?, - &[[200]] - ); - assert_eq!( - tensor.min_keepdim(1)?.min_keepdim(0)?.to_vec2::()?, - &[[200]] - ); - assert_eq!(tensor.min_keepdim(0)?.to_vec2::()?, &[[200, 201]]); - - let t1 = tensor.reshape((190, 5, 4))?; - let t2 = t1.transpose(0, 2)?.contiguous()?.transpose(0, 2)?; - for tensor in [t1, t2] { - assert_eq!( - tensor - .min_keepdim(0)? - .min_keepdim(2)? - .min_keepdim(1)? - .to_vec3::()?, - &[[[200]]] - ); - assert_eq!( - tensor.min_keepdim(0)?.to_vec3::()?, - &[[ - [200, 201, 202, 203], - [204, 205, 206, 207], - [208, 209, 210, 211], - [212, 213, 214, 215], - [216, 217, 218, 219] - ]] - ); - } - Ok(()) -} - -fn max(device: &Device) -> Result<()> { - let data = &[[[3u32, 1, 4], [1, 5, 9]], [[2, 1, 7], [8, 2, 8]]]; - let tensor = Tensor::new(data, device)?; - assert_eq!( - tensor.max_keepdim(2)?.to_vec3::()?, - &[[[4], [9]], [[7], [8]]] - ); - assert_eq!( - tensor.max_keepdim(0)?.to_vec3::()?, - &[[[3, 1, 7], [8, 5, 9]]], - ); - let data: Vec = (200..4000u32).collect(); - let tensor = Tensor::new(data.as_slice(), device)?; - assert_eq!(tensor.max_keepdim(0)?.to_vec1::()?, &[3999]); - let tensor = tensor.reshape((1900, 2))?; - assert_eq!( - tensor.max_keepdim(0)?.max_keepdim(1)?.to_vec2::()?, - &[[3999]] - ); - assert_eq!( - tensor.max_keepdim(1)?.max_keepdim(0)?.to_vec2::()?, - &[[3999]] - ); - assert_eq!(tensor.max_keepdim(0)?.to_vec2::()?, &[[3998, 3999]]); - - // Make the tensor non contiguous. - let tensor = tensor.t()?.contiguous()?.t()?; - assert_eq!( - tensor.max_keepdim(0)?.max_keepdim(1)?.to_vec2::()?, - &[[3999]] - ); - assert_eq!( - tensor.max_keepdim(1)?.max_keepdim(0)?.to_vec2::()?, - &[[3999]] - ); - assert_eq!(tensor.max_keepdim(0)?.to_vec2::()?, &[[3998, 3999]]); - - let t1 = tensor.reshape((190, 5, 4))?; - let t2 = t1.transpose(0, 2)?.contiguous()?.transpose(0, 2)?; - for tensor in [t1, t2] { - assert_eq!( - tensor - .max_keepdim(0)? - .max_keepdim(2)? - .max_keepdim(1)? - .to_vec3::()?, - &[[[3999]]] - ); - assert_eq!( - tensor.max_keepdim(0)?.to_vec3::()?, - &[[ - [3980, 3981, 3982, 3983], - [3984, 3985, 3986, 3987], - [3988, 3989, 3990, 3991], - [3992, 3993, 3994, 3995], - [3996, 3997, 3998, 3999] - ]] - ); - } - Ok(()) -} - -fn argmin(device: &Device) -> Result<()> { - let data = &[[[3u32, 1, 4], [1, 5, 9]], [[2, 1, 7], [8, 2, 8]]]; - let tensor = Tensor::new(data, device)?; - assert_eq!( - tensor.argmin_keepdim(2)?.to_vec3::()?, - &[[[1], [0]], [[1], [1]]] - ); - assert_eq!( - tensor.argmin_keepdim(0)?.to_vec3::()?, - &[[[1, 0, 0], [0, 1, 1]]], - ); - let data: Vec = (200..4000u32).collect(); - let tensor = Tensor::new(data.as_slice(), device)?; - assert_eq!(tensor.argmin_keepdim(0)?.to_vec1::()?, &[0]); - let tensor = tensor.reshape((1900, 2))?; - assert_eq!( - tensor - .argmin_keepdim(0)? - .argmin_keepdim(1)? - .to_vec2::()?, - &[[0]] - ); - assert_eq!( - tensor - .argmin_keepdim(1)? - .argmin_keepdim(0)? - .to_vec2::()?, - &[[0]] - ); - assert_eq!(tensor.argmin_keepdim(0)?.to_vec2::()?, &[[0, 0]]); - - // Make the tensor non contiguous. - let tensor = tensor.t()?.contiguous()?.t()?; - assert_eq!( - tensor - .argmin_keepdim(0)? - .argmin_keepdim(1)? - .to_vec2::()?, - &[[0]] - ); - assert_eq!( - tensor - .argmin_keepdim(1)? - .argmin_keepdim(0)? - .to_vec2::()?, - &[[0]] - ); - assert_eq!(tensor.argmin_keepdim(0)?.to_vec2::()?, &[[0, 0]]); - - let t1 = tensor.reshape((190, 5, 4))?; - let t2 = t1.transpose(0, 2)?.contiguous()?.transpose(0, 2)?; - for tensor in [t1, t2] { - assert_eq!( - tensor - .argmin_keepdim(0)? - .argmin_keepdim(2)? - .argmin_keepdim(1)? - .to_vec3::()?, - &[[[0]]] - ); - assert_eq!( - tensor.argmin_keepdim(0)?.to_vec3::()?, - &[[ - [0, 0, 0, 0], - [0, 0, 0, 0], - [0, 0, 0, 0], - [0, 0, 0, 0], - [0, 0, 0, 0], - ]] - ); - } - Ok(()) -} - -fn argmax(device: &Device) -> Result<()> { - let data = &[[[3u32, 1, 4], [1, 5, 9]], [[2, 1, 7], [8, 2, 8]]]; - let tensor = Tensor::new(data, device)?; - assert_eq!( - tensor.argmax_keepdim(2)?.to_vec3::()?, - &[[[2], [2]], [[2], [0]]] - ); - assert_eq!( - tensor.argmax_keepdim(0)?.to_vec3::()?, - &[[[0, 0, 1], [1, 0, 0]]], - ); - let data: Vec = (200..4000u32).collect(); - let tensor = Tensor::new(data.as_slice(), device)?; - assert_eq!(tensor.argmax_keepdim(0)?.to_vec1::()?, &[3799]); - let tensor = tensor.reshape((1900, 2))?; - assert_eq!( - tensor - .argmax_keepdim(0)? - .argmax_keepdim(1)? - .to_vec2::()?, - &[[0]] - ); - assert_eq!( - tensor - .argmax_keepdim(1)? - .argmax_keepdim(0)? - .to_vec2::()?, - &[[0]] - ); - assert_eq!(tensor.argmax_keepdim(0)?.to_vec2::()?, &[[1899, 1899]]); - - // Make the tensor non contiguous. - let tensor = tensor.t()?.contiguous()?.t()?; - assert_eq!( - tensor - .argmax_keepdim(0)? - .argmax_keepdim(1)? - .to_vec2::()?, - &[[0]] - ); - assert_eq!( - tensor - .argmax_keepdim(1)? - .argmax_keepdim(0)? - .to_vec2::()?, - &[[0]] - ); - assert_eq!(tensor.argmax_keepdim(0)?.to_vec2::()?, &[[1899, 1899]]); - - let t1 = tensor.reshape((190, 5, 4))?; - let t2 = t1.transpose(0, 2)?.contiguous()?.transpose(0, 2)?; - for tensor in [t1, t2] { - assert_eq!( - tensor - .argmax_keepdim(0)? - .argmax_keepdim(2)? - .argmax_keepdim(1)? - .to_vec3::()?, - &[[[0]]] - ); - assert_eq!( - tensor.argmax_keepdim(0)?.to_vec3::()?, - &[[ - [189, 189, 189, 189], - [189, 189, 189, 189], - [189, 189, 189, 189], - [189, 189, 189, 189], - [189, 189, 189, 189], - ]] - ); - } - Ok(()) -} - -fn narrow(device: &Device) -> Result<()> { - let data = &[[[3f32, 1., 4.], [1., 5., 9.]], [[2., 1., 7.], [8., 2., 8.]]]; - let tensor = Tensor::new(data, device)?; - assert_eq!( - tensor.narrow(2, 1, 2)?.to_vec3::()?, - &[[[1.0, 4.0], [5.0, 9.0]], [[1.0, 7.0], [2.0, 8.0]]], - ); - assert_eq!( - tensor.narrow(1, 1, 1)?.to_vec3::()?, - &[[[1.0, 5.0, 9.0]], [[8.0, 2.0, 8.0]]], - ); - assert_eq!( - tensor.narrow(0, 0, 1)?.to_vec3::()?, - &[[[3.0, 1.0, 4.0], [1.0, 5.0, 9.0]]], - ); - assert_eq!( - tensor.narrow(0, 1, 1)?.to_vec3::()?, - &[[[2.0, 1.0, 7.0], [8.0, 2.0, 8.0]]], - ); - // The following has been checked against PyTorch via: - // import torch - // t = torch.tensor([[[3., 1., 4.], [1., 5., 9.]], [[2., 1., 7.], [8., 2., 8.]]]) - // t.transpose(-1, -2).narrow(1, 1, 2) - assert_eq!( - tensor.t()?.narrow(1, 1, 2)?.to_vec3::()?, - &[[[1.0, 5.0], [4.0, 9.0]], [[1.0, 2.0], [7.0, 8.0]]], - ); - Ok(()) -} - -fn broadcast(device: &Device) -> Result<()> { - let data = &[3f32, 1., 4.]; - let tensor = Tensor::new(data, device)?; - assert_eq!( - tensor.broadcast_left((3, 1))?.to_vec3::()?, - &[[[3.0, 1.0, 4.0]], [[3.0, 1.0, 4.0]], [[3.0, 1.0, 4.0]]] - ); - Ok(()) -} - -fn slice_set(device: &Device) -> Result<()> { - let (b, h, max_t, d) = (2, 4, 7, 3); - let cache = Tensor::zeros((b, h, max_t, d), DType::F32, device)?; - let tensor = Tensor::randn(0f32, 1f32, (b, h, 4, d), device)?; - cache.slice_set(&tensor, 2, 0)?; - let cache_t = cache.narrow(2, 0, 4)?; - let diff = (cache_t - &tensor)?.abs()?.sum_all()?.to_vec0::()?; - assert_eq!(diff, 0.); - cache.slice_set(&tensor, 2, 1)?; - let cache_t = cache.narrow(2, 1, 4)?; - let diff = (cache_t - &tensor)?.abs()?.sum_all()?.to_vec0::()?; - assert_eq!(diff, 0.); - let ones = Tensor::ones((b, h, 1, d), DType::F32, device)?; - cache.slice_set(&ones, 2, 6)?; - let diff = cache.narrow(2, 5, 1)?.abs()?.sum_all()?.to_vec0::()?; - assert_eq!(diff, 0.); - let diff = (cache.narrow(2, 6, 1)? - 1.)? - .abs()? - .sum_all()? - .to_vec0::()?; - assert_eq!(diff, 0.); - // This used to create a deadlock rather than returning an actual error. - assert!(cache.slice_set(&cache, 0, 0).is_err()); - Ok(()) -} - -fn cat(device: &Device) -> Result<()> { - // 1D - let t1 = Tensor::new(&[3f32, 1., 4.], device)?; - let t2 = Tensor::new(&[1f32, 5., 9., 2.], device)?; - let t3 = Tensor::new(&[6f32, 5., 3., 5., 8., 9.], device)?; - assert_eq!(Tensor::cat(&[&t1], 0)?.to_vec1::()?, [3f32, 1., 4.],); - assert_eq!( - Tensor::cat(&[&t1, &t2], 0)?.to_vec1::()?, - [3f32, 1., 4., 1., 5., 9., 2.], - ); - assert_eq!( - Tensor::cat(&[&t1, &t2, &t3], 0)?.to_vec1::()?, - [3f32, 1., 4., 1., 5., 9., 2., 6., 5., 3., 5., 8., 9.], - ); - - // 2D - let data = &[[3f32, 1., 4., 1., 5.], [2., 7., 1., 8., 2.]]; - let t1 = Tensor::new(data, device)?; - let data2 = &[[5f32, 5., 5., 5., 5.], [2., 7., 1., 8., 2.]]; - let t2 = Tensor::new(data2, device)?; - assert_eq!( - Tensor::cat(&[&t1, &t2], 0)?.to_vec2::()?, - [ - [3.0, 1.0, 4.0, 1.0, 5.0], - [2.0, 7.0, 1.0, 8.0, 2.0], - [5.0, 5.0, 5.0, 5.0, 5.0], - [2.0, 7.0, 1.0, 8.0, 2.0] - ] - ); - // PyTorch equivalent: - // import torch - // t1 = torch.tensor([[3, 1, 4, 1, 5], [2, 7, 1, 8, 2]]) - // t2 = torch.tensor([[5]*5, [2, 7, 1, 8, 2]]) - // torch.cat([t1.t(), t2.t()], dim=1).t() - assert_eq!( - Tensor::cat(&[&t1.t()?, &t2.t()?], 1)? - .t()? - .to_vec2::()?, - [ - [3.0, 1.0, 4.0, 1.0, 5.0], - [2.0, 7.0, 1.0, 8.0, 2.0], - [5.0, 5.0, 5.0, 5.0, 5.0], - [2.0, 7.0, 1.0, 8.0, 2.0] - ] - ); - assert_eq!( - Tensor::cat(&[&t1, &t2], 1)?.to_vec2::()?, - [ - [3.0, 1.0, 4.0, 1.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0], - [2.0, 7.0, 1.0, 8.0, 2.0, 2.0, 7.0, 1.0, 8.0, 2.0] - ] - ); - - // 3D - let t1 = Tensor::arange(0, 48i64, device)?.reshape((2, 6, 4))?; - let t2 = Tensor::arange(100, 124i64, device)?.reshape((2, 3, 4))?; - let t3 = Tensor::arange(10000, 10032i64, device)?.reshape((2, 4, 4))?; - - let t_cat = Tensor::cat(&[&t1, &t2, &t3], 1)?; - - let t1 = t1.t()?.contiguous()?.t()?; - let t2 = t2.t()?.contiguous()?.t()?; - let t3 = t3.t()?.contiguous()?.t()?; - let t_cat2 = Tensor::cat(&[&t1, &t2, &t3], 1)?; - - let diff = t_cat.eq(&t_cat2)?.to_dtype(DType::F32)?.sum_all()?; - assert_eq!(diff.to_vec0::()?, 104.0); - assert_eq!(t_cat.i((0, 0, 0))?.to_vec0::()?, 0); - assert_eq!(t_cat.i((0, 4, 0))?.to_vec0::()?, 16); - assert_eq!(t_cat.i((0, 5, 0))?.to_vec0::()?, 20); - assert_eq!(t_cat.i((1, 5, 0))?.to_vec0::()?, 44); - assert_eq!(t_cat.i((0, 6, 0))?.to_vec0::()?, 100); - assert_eq!(t_cat.i((1, 6, 0))?.to_vec0::()?, 112); - assert_eq!(t_cat.i((0, 6, 1))?.to_vec0::()?, 101); - assert_eq!(t_cat.i((0, 7, 1))?.to_vec0::()?, 105); - assert_eq!(t_cat.i((0, 12, 1))?.to_vec0::()?, 10013); - assert_eq!(t_cat.i((1, 12, 3))?.to_vec0::()?, 10031); - Ok(()) -} - -fn embeddings(device: &Device) -> Result<()> { - let ids = Tensor::new(&[0u32, 2u32, 1u32], device)?; - let t = Tensor::new(&[[0f32, 1f32], [2f32, 3f32], [4f32, 5f32]], device)?; - let hs = t.embedding(&ids)?; - assert_eq!(hs.to_vec2::()?, &[[0.0, 1.0], [4.0, 5.0], [2.0, 3.0]]); - let hs = t.index_select(&ids, 0)?; - assert_eq!(hs.to_vec2::()?, &[[0.0, 1.0], [4.0, 5.0], [2.0, 3.0]]); - let hs = t.index_select(&ids.to_dtype(DType::I64)?, 0)?; - assert_eq!(hs.to_vec2::()?, &[[0.0, 1.0], [4.0, 5.0], [2.0, 3.0]]); - let ids = Tensor::new(&[u32::MAX, 2u32, u32::MAX], device)?; - let hs = t.index_select(&ids, 0)?; - assert_eq!(hs.to_vec2::()?, &[[0.0, 0.0], [4.0, 5.0], [0.0, 0.0]]); - Ok(()) -} - -#[test] -fn index_select_fail() -> Result<()> { - // Check that an error is properly reported on out of bounds. - let ids = Tensor::new(&[4u32, 2u32, 1u32], &Device::Cpu)?; - let t = Tensor::new(&[[0f32, 1f32], [2f32, 3f32], [4f32, 5f32]], &Device::Cpu)?; - let hs = t.index_select(&ids, 0); - assert!(hs.is_err()); - Ok(()) -} - -// The test below triggers an unwinding panic as there is a panic within the -// #[cfg(feature = "cuda")] -// #[test] -// #[should_panic] -// fn index_select_fail_gpu() { -// // Check that a panic happens for out of bounds in cuda -// if let Ok(device) = Device::new_cuda(0) { -// if let Ok(ids) = Tensor::new(&[4u32, 2u32, 1u32], &device) { -// if let Ok(t) = Tensor::new(&[[0f32, 1f32], [2f32, 3f32], [4f32, 5f32]], &device) { -// let _ = t.index_select(&ids, 0); -// } -// } -// } -// } - -fn cmp(device: &Device) -> Result<()> { - let t1 = Tensor::new(&[[0f32, 1f32], [2f32, 3f32], [4f32, 5f32]], device)?; - let t2 = Tensor::new(&[[1f32, 0f32], [3f32, 3f32], [4f32, 7f32]], device)?; - assert_eq!(t1.eq(&t2)?.to_vec2::()?, &[[0, 0], [0, 1], [1, 0]]); - assert_eq!(t1.ne(&t2)?.to_vec2::()?, &[[1, 1], [1, 0], [0, 1]]); - assert_eq!(t1.le(&t2)?.to_vec2::()?, &[[1, 0], [1, 1], [1, 1]]); - assert_eq!(t1.lt(&t2)?.to_vec2::()?, &[[1, 0], [1, 0], [0, 1]]); - assert_eq!(t1.gt(&t2)?.to_vec2::()?, &[[0, 1], [0, 0], [0, 0]]); - assert_eq!(t1.ge(&t2)?.to_vec2::()?, &[[0, 1], [0, 1], [1, 0]]); - Ok(()) -} - -fn index_select(device: &Device) -> Result<()> { - let ids = Tensor::new(&[0u32, 2u32, 1u32], device)?; - let t = Tensor::arange(0f32, 12f32, device)?.reshape((4, 3))?; - assert_eq!( - t.to_vec2::()?, - &[ - [0.0, 1.0, 2.0], - [3.0, 4.0, 5.0], - [6.0, 7.0, 8.0], - [9.0, 10.0, 11.0] - ] - ); - for dtype in [DType::U8, DType::U32, DType::I64] { - let ids = ids.to_dtype(dtype)?; - let hs = t.index_select(&ids, 1)?; - assert_eq!( - hs.to_vec2::()?, - &[ - [0.0, 2.0, 1.0], - [3.0, 5.0, 4.0], - [6.0, 8.0, 7.0], - [9.0, 11.0, 10.0] - ] - ); - let hs = t.index_select(&ids, 0)?; - assert_eq!( - hs.to_vec2::()?, - &[[0.0, 1.0, 2.0], [6.0, 7.0, 8.0], [3.0, 4.0, 5.0]] - ); - // Prior to https://github.com/huggingface/candle/pull/1022 - // There would be a bug where the last values in the result tensor would be set to 0. - let ids = Tensor::new(&[0u32, 2u32, 1u32, 0u32, 2u32, 1u32], device)?; - let hs = t.index_select(&ids, 0)?; - assert_eq!( - hs.to_vec2::()?, - &[ - [0.0, 1.0, 2.0], - [6.0, 7.0, 8.0], - [3.0, 4.0, 5.0], - [0.0, 1.0, 2.0], - [6.0, 7.0, 8.0], - [3.0, 4.0, 5.0], - ] - ); - - // Test when selecting dim > 0 with ids size different from elem count of - // target dim in source/input. - let ids = Tensor::new(&[1u32, 0u32, 1u32], device)?; - let t = Tensor::arange(1f32, 5f32, device)?.reshape((2, 2))?; - assert_eq!(t.to_vec2::()?, &[[1.0, 2.0], [3.0, 4.0]]); - let hs = t.index_select(&ids, 1)?; - assert_eq!(hs.to_vec2::()?, &[[2.0, 1.0, 2.0], [4.0, 3.0, 4.0]]); - } - - Ok(()) -} - -fn index_add(device: &Device) -> Result<()> { - let ids = Tensor::new(&[0u32, 1u32, 1u32], device)?; - let t = Tensor::arange(0f32, 12f32, device)?.reshape((4, 3))?; - assert_eq!( - t.to_vec2::()?, - &[ - [0.0, 1.0, 2.0], - [3.0, 4.0, 5.0], - [6.0, 7.0, 8.0], - [9.0, 10.0, 11.0] - ] - ); - let init = Tensor::ones((4, 2), DType::F32, device)?; - let hs = init.index_add(&ids, &t, 1)?; - assert_eq!( - hs.to_vec2::()?, - &[[1.0, 4.0], [4.0, 10.0], [7.0, 16.0], [10.0, 22.0]], - ); - let init = Tensor::zeros((4, 2), DType::F32, device)?; - let ids = Tensor::new(&[1u32, 0u32, 0u32], device)?; - let hs = init.index_add(&ids, &t, 1)?; - assert_eq!( - hs.to_vec2::()?, - &[[3.0, 0.0], [9.0, 3.0], [15.0, 6.0], [21.0, 9.0]], - ); - - let init = Tensor::zeros((6, 3), DType::F32, device)?; - let ids = Tensor::new(&[5u32, 0u32, 1u32, 0u32], device)?; - let hs = init.index_add(&ids, &t, 0)?; - assert_eq!( - hs.to_vec2::()?, - &[ - [12.0, 14.0, 16.0], - [6.0, 7.0, 8.0], - [0.0, 0.0, 0.0], - [0.0, 0.0, 0.0], - [0.0, 0.0, 0.0], - [0.0, 1.0, 2.0] - ] - ); - Ok(()) -} - -fn slice_scatter(device: &Device) -> Result<()> { - let t = Tensor::arange(0f32, 12f32, device)?.reshape((4, 3))?; - assert_eq!( - t.to_vec2::()?, - &[ - [0.0, 1.0, 2.0], - [3.0, 4.0, 5.0], - [6.0, 7.0, 8.0], - [9.0, 10.0, 11.0] - ] - ); - let src = Tensor::arange(100f32, 106f32, device)?.reshape((2, 3))?; - assert_eq!( - t.slice_scatter0(&src, 0)?.to_vec2::()?, - &[ - [100.0, 101.0, 102.0], - [103.0, 104.0, 105.0], - [6.0, 7.0, 8.0], - [9.0, 10.0, 11.0] - ] - ); - assert_eq!( - t.slice_scatter0(&src, 1)?.to_vec2::()?, - &[ - [0.0, 1.0, 2.0], - [100.0, 101.0, 102.0], - [103.0, 104.0, 105.0], - [9.0, 10.0, 11.0] - ] - ); - assert_eq!( - t.slice_scatter0(&src, 2)?.to_vec2::()?, - &[ - [0.0, 1.0, 2.0], - [3.0, 4.0, 5.0], - [100.0, 101.0, 102.0], - [103.0, 104.0, 105.0], - ] - ); - Ok(()) -} - -fn scatter(device: &Device) -> Result<()> { - let t = Tensor::arange(0f32, 12f32, device)?.reshape((4, 3))?; - assert_eq!( - t.to_vec2::()?, - &[ - [0.0, 1.0, 2.0], - [3.0, 4.0, 5.0], - [6.0, 7.0, 8.0], - [9.0, 10.0, 11.0] - ] - ); - let ids = Tensor::new(&[[0u32, 1, 2], [3, 4, 0], [3, 3, 1], [2, 0, 4]], device)?; - let init = Tensor::ones((4, 5), DType::F32, device)?; - let hs = init.scatter_add(&ids, &t, 1)?; - assert_eq!( - hs.to_vec2::()?, - &[ - [1.0, 2.0, 3.0, 1.0, 1.0], - [6.0, 1.0, 1.0, 4.0, 5.0], - [1.0, 9.0, 1.0, 14.0, 1.0], - [11.0, 1.0, 10.0, 1.0, 12.0] - ] - ); - - let hs = init.scatter(&ids, &t, 1)?; - assert_eq!( - hs.to_vec2::()?, - &[ - [0.0, 1.0, 2.0, 1.0, 1.0], - [5.0, 1.0, 1.0, 3.0, 4.0], - [1.0, 8.0, 1.0, 7.0, 1.0], - [10.0, 1.0, 9.0, 1.0, 11.0] - ] - ); - - let init = Tensor::ones((6, 3), DType::F32, device)?; - let hs = init.scatter_add(&ids, &t, 0)?; - assert_eq!( - hs.to_vec2::()?, - &[ - [1.0, 11.0, 6.0], - [1.0, 2.0, 9.0], - [10.0, 1.0, 3.0], - [10.0, 8.0, 1.0], - [1.0, 5.0, 12.0], - [1.0, 1.0, 1.0] - ] - ); - let hs = init.scatter(&ids, &t, 0)?; - assert_eq!( - hs.to_vec2::()?, - &[ - [0.0, 10.0, 5.0], - [1.0, 1.0, 8.0], - [9.0, 1.0, 2.0], - [6.0, 7.0, 1.0], - [1.0, 4.0, 11.0], - [1.0, 1.0, 1.0] - ] - ); - - let hs = { - let ids = Tensor::new( - &[ - [0u32, u32::MAX, 2], - [3, 4, u32::MAX], - [3, 3, 1], - [u32::MAX, u32::MAX, 4], - ], - device, - )?; - init.scatter(&ids, &t, 0)? - }; - assert_eq!( - hs.to_vec2::()?, - &[ - [0.0, 1.0, 1.0], - [1.0, 1.0, 8.0], - [1.0, 1.0, 2.0], - [6.0, 7.0, 1.0], - [1.0, 4.0, 11.0], - [1.0, 1.0, 1.0] - ] - ); - - init.scatter_set(&ids, &t, 0)?; - assert_eq!( - init.to_vec2::()?, - &[ - [0.0, 10.0, 5.0], - [1.0, 1.0, 8.0], - [9.0, 1.0, 2.0], - [6.0, 7.0, 1.0], - [1.0, 4.0, 11.0], - [1.0, 1.0, 1.0] - ] - ); - - Ok(()) -} - -fn gather(device: &Device) -> Result<()> { - let ids = Tensor::new(&[[0u32], [2u32], [1u32], [0u32]], device)?; - let t = Tensor::arange(0f32, 12f32, device)?.reshape((4, 3))?; - assert_eq!( - t.to_vec2::()?, - &[ - [0.0, 1.0, 2.0], - [3.0, 4.0, 5.0], - [6.0, 7.0, 8.0], - [9.0, 10.0, 11.0] - ] - ); - let hs = t.gather(&ids, 1)?; - assert_eq!(hs.to_vec2::()?, &[[0.0], [5.0], [7.0], [9.0]]); - let ids = Tensor::new( - &[[0u32, 0u32], [2u32, 0u32], [1u32, 1u32], [0u32, 2u32]], - device, - )?; - let hs = t.gather(&ids, 1)?; - assert_eq!( - hs.to_vec2::()?, - &[[0.0, 0.0], [5.0, 3.0], [7.0, 7.0], [9.0, 11.0]] - ); - let ids = Tensor::new(&[[0u32, 2u32, 0u32]], device)?; - let hs = t.gather(&ids, 0)?; - assert_eq!(hs.to_vec2::()?, &[[0.0, 7.0, 2.0]]); - let ids = Tensor::new(&[[0u32, 2u32, 0u32], [0u32, 1u32, 1u32]], device)?; - let hs = t.gather(&ids, 0)?; - assert_eq!(hs.to_vec2::()?, &[[0.0, 7.0, 2.0], [0.0, 4.0, 5.0]]); - - let hs = { - let ids = Tensor::new( - &[ - [0u32, 0u32], - [2u32, u32::MAX], - [u32::MAX, 1u32], - [0u32, 2u32], - ], - device, - )?; - t.gather(&ids, 1)? - }; - assert_eq!( - hs.to_vec2::()?, - &[[0.0, 0.0], [5.0, 0.0], [0.0, 7.0], [9.0, 11.0]] - ); - - // Random data - - // Dim: 0 - let t = Tensor::new( - &[ - [ - [108_f32, -47., 16., -56., -83., -130., 210.], - [253., 95., 151., 228., -210., -123., -127.], - [-9., -217., 2., -78., 163., 245., -204.], - [-246., 79., -238., 88., -226., -184., 171.], - [8., -48., -153., 234., -34., 166., -153.], - [124., 0., -10., -61., -242., -15., -238.], - ], - [ - [12., -64., -199., 244., -240., 156., -128.], - [173., -57., 4., -198., 233., -110., 238.], - [95., 82., 0., 240., 53., -211., 209.], - [-122., 167., -212., 227., -144., 61., 118.], - [-63., -146., 200., 244., 168., -167., 116.], - [-125., -147., 110., -253., -178., -250., -18.], - ], - [ - [57., 86., -50., 56., 92., 205., -78.], - [-137., -156., -18., 248., -61., -239., 14.], - [-248., -30., -50., -70., -251., 250., -83.], - [-221., 67., 72., 59., -24., -154., 232.], - [-144., -23., -74., 5., 93., 171., 205.], - [46., -77., -38., -226., 246., 161., -17.], - ], - [ - [-153., -231., -236., 161., 126., 2., -22.], - [-229., -41., 209., 164., 234., 160., 57.], - [223., 254., -186., -162., -46., -160., -102.], - [65., 30., 213., -253., 59., 224., -154.], - [-82., -203., -177., 17., 31., -256., -246.], - [176., -135., -65., 54., -56., 210., 76.], - ], - [ - [-10., -245., 168., 124., -14., -33., -178.], - [25., -43., -39., 132., -89., 169., 179.], - [187., -215., 32., -133., 87., -7., -168.], - [-224., -215., -5., -230., -58., -162., 128.], - [158., -137., -122., -100., -202., -83., 136.], - [30., -185., -144., 250., 209., -40., 127.], - ], - [ - [-196., 108., -245., 122., 146., -228., 62.], - [-1., -66., 160., 137., 13., -172., -21.], - [244., 199., -164., 28., 119., -175., 198.], - [-62., 253., -162., 195., -95., -230., -211.], - [123., -72., -26., -107., -139., 64., 245.], - [11., -126., -182., 108., -12., 184., -127.], - ], - [ - [-159., 126., 176., 161., 73., -111., -138.], - [-187., 214., -217., -33., -223., -201., -212.], - [-61., -120., -166., -172., -95., 53., 196.], - [-33., 86., 134., -152., 154., -53., 74.], - [186., -28., -154., -174., 141., -109., 217.], - [82., 35., 252., 145., 181., 74., -87.], - ], - ], - device, - )?; - - let ids = Tensor::new( - &[ - [ - [6_u32, 6, 4, 3, 4, 4, 6], - [3, 3, 2, 4, 4, 4, 6], - [3, 3, 0, 2, 4, 6, 4], - [2, 5, 1, 2, 6, 6, 1], - [2, 1, 6, 5, 3, 2, 3], - [6, 1, 0, 1, 0, 2, 6], - ], - [ - [4, 6, 4, 3, 3, 3, 2], - [4, 3, 2, 4, 4, 4, 6], - [2, 3, 0, 2, 4, 6, 4], - [6, 5, 1, 2, 6, 6, 1], - [4, 1, 6, 5, 3, 2, 3], - [1, 1, 0, 1, 0, 2, 6], - ], - [ - [3, 6, 4, 3, 3, 3, 2], - [2, 3, 2, 4, 4, 4, 6], - [4, 3, 0, 2, 4, 6, 4], - [0, 5, 1, 2, 6, 6, 1], - [6, 1, 6, 5, 3, 2, 3], - [4, 1, 0, 1, 0, 2, 6], - ], - [ - [0, 6, 4, 3, 3, 3, 2], - [5, 3, 2, 4, 4, 4, 6], - [0, 3, 0, 2, 4, 6, 4], - [3, 5, 1, 2, 6, 6, 1], - [0, 1, 6, 5, 3, 2, 3], - [3, 1, 0, 1, 0, 2, 6], - ], - ], - device, - )?; - - let hs = t.gather(&ids, 0)?; - assert_eq!( - hs.to_vec3::()?, - &[ - [ - [-159_f32, 126., 168., 161., -14., -33., -138.], - [-229., -41., -18., 132., -89., 169., -212.], - [223., 254., 2., -70., 87., 53., -168.], - [-221., 253., -212., 59., 154., -53., 118.], - [-144., -146., -154., -107., 31., 171., -246.], - [82., -147., -10., -253., -242., 161., -87.] - ], - [ - [-10., 126., 168., 161., 126., 2., -78.], - [25., -41., -18., 132., -89., 169., -212.], - [-248., 254., 2., -70., 87., 53., -168.], - [-33., 253., -212., 59., 154., -53., 118.], - [158., -146., -154., -107., 31., 171., -246.], - [-125., -147., -10., -253., -242., 161., -87.] - ], - [ - [-153., 126., 168., 161., 126., 2., -78.], - [-137., -41., -18., 132., -89., 169., -212.], - [187., 254., 2., -70., 87., 53., -168.], - [-246., 253., -212., 59., 154., -53., 118.], - [186., -146., -154., -107., 31., 171., -246.], - [30., -147., -10., -253., -242., 161., -87.] - ], - [ - [108., 126., 168., 161., 126., 2., -78.], - [-1., -41., -18., 132., -89., 169., -212.], - [-9., 254., 2., -70., 87., 53., -168.], - [65., 253., -212., 59., 154., -53., 118.], - [8., -146., -154., -107., 31., 171., -246.], - [176., -147., -10., -253., -242., 161., -87.] - ] - ] - ); - - // Dim: 1 - let t = Tensor::new( - &[ - [ - [-117_f32, -175., 69., -163.], - [200., 242., -21., -67.], - [179., 150., -126., -75.], - [-118., 38., -138., -13.], - [-221., 136., -185., 180.], - [58., 182., -204., -149.], - ], - [ - [3., -148., -58., -154.], - [-43., 45., -108., 4.], - [-69., -249., -71., -21.], - [80., 110., -152., -235.], - [-88., 7., 92., -250.], - [-186., 207., -242., 98.], - ], - [ - [238., 19., 64., -242.], - [-150., -97., 218., 58.], - [111., -233., 204., -212.], - [-242., -232., 83., 42.], - [153., 62., -251., 219.], - [-117., 36., -119., 10.], - ], - [ - [215., 159., -169., -27.], - [-83., 101., -88., 169.], - [-205., 93., 225., -64.], - [-162., 240., 214., 23.], - [-112., 6., 21., 245.], - [-38., 113., 93., 215.], - ], - [ - [91., -188., -148., 101.], - [74., 203., -35., 55.], - [-116., -130., -153., -96.], - [58., 22., -45., -194.], - [-221., -134., 73., 159.], - [-203., -254., 31., 235.], - ], - [ - [105., -53., 61., 186.], - [-195., 234., 75., -1.], - [51., 139., 160., -108.], - [-173., -167., 161., 19.], - [83., -246., 156., -222.], - [109., 39., -149., 137.], - ], - ], - device, - )?; - - let ids = Tensor::new( - &[ - [[4_u32, 4, 4, 2]], - [[0, 4, 4, 3]], - [[1, 5, 3, 4]], - [[0, 3, 3, 2]], - [[1, 1, 5, 2]], - [[1, 4, 5, 4]], - ], - device, - )?; - - let hs = t.gather(&ids, 1)?; - assert_eq!( - hs.to_vec3::()?, - &[ - [[-221., 136., -185., -75.]], - [[3., 7., 92., -235.]], - [[-150., 36., 83., 219.]], - [[215., 240., 214., -64.]], - [[74., 203., 31., -96.]], - [[-195., -246., -149., -222.]] - ] - ); - - // Dim: 2 - let t = Tensor::new( - &[ - [[-162_f32, 202.], [-126., -39.], [35., -65.], [1., 80.]], - [[37., 248.], [-191., 89.], [117., -40.], [-217., 220.]], - ], - device, - )?; - - let ids = Tensor::new(&[[[1_u32], [0], [1], [1]], [[0], [1], [0], [1]]], device)?; - - let hs = t.gather(&ids, 2)?; - assert_eq!( - hs.to_vec3::()?, - &[ - [[202.], [-126.], [-65.], [80.]], - [[37.], [89.], [117.], [220.]] - ] - ); - - let t = Tensor::new( - &[ - [[-21_f32, -197.], [194., 122.]], - [[255., -106.], [-191., 250.]], - [[33., -117.], [43., 10.]], - [[-130., 238.], [-217., -92.]], - ], - device, - )?; - - let ids = Tensor::new( - &[ - [[0_u32, 1], [1, 0]], - [[1, 0], [0, 1]], - [[0, 1], [0, 1]], - [[1, 0], [1, 0]], - ], - device, - )?; - - let hs = t.gather(&ids, 2)?; - assert_eq!( - hs.to_vec3::()?, - &[ - [[-21., -197.], [122., 194.]], - [[-106., 255.], [-191., 250.]], - [[33., -117.], [43., 10.]], - [[238., -130.], [-92., -217.]] - ] - ); - - Ok(()) -} - -fn broadcasting(device: &Device) -> Result<()> { - let t1 = Tensor::arange(0f32, 24f32, device)?.reshape((4, 2, 3))?; - let t2 = Tensor::new(&[100f32, 200f32], device)?; - let s = t1.broadcast_add(&t2.reshape((2, 1))?)?; - assert_eq!( - s.to_vec3::()?, - &[ - [[100.0, 101.0, 102.0], [203.0, 204.0, 205.0]], - [[106.0, 107.0, 108.0], [209.0, 210.0, 211.0]], - [[112.0, 113.0, 114.0], [215.0, 216.0, 217.0]], - [[118.0, 119.0, 120.0], [221.0, 222.0, 223.0]] - ] - ); - let s = t1.t()?.broadcast_add(&t2)?; - assert_eq!( - s.to_vec3::()?, - &[ - [[100.0, 203.0], [101.0, 204.0], [102.0, 205.0]], - [[106.0, 209.0], [107.0, 210.0], [108.0, 211.0]], - [[112.0, 215.0], [113.0, 216.0], [114.0, 217.0]], - [[118.0, 221.0], [119.0, 222.0], [120.0, 223.0]] - ] - ); - let s = t1.broadcast_sub(&t2.reshape((2, 1))?)?; - assert_eq!( - s.to_vec3::()?, - &[ - [[-100.0, -99.0, -98.0], [-197.0, -196.0, -195.0]], - [[-94.0, -93.0, -92.0], [-191.0, -190.0, -189.0]], - [[-88.0, -87.0, -86.0], [-185.0, -184.0, -183.0]], - [[-82.0, -81.0, -80.0], [-179.0, -178.0, -177.0]] - ] - ); - let s = t1.t()?.broadcast_sub(&t2)?; - assert_eq!( - s.to_vec3::()?, - &[ - [[-100.0, -197.0], [-99.0, -196.0], [-98.0, -195.0]], - [[-94.0, -191.0], [-93.0, -190.0], [-92.0, -189.0]], - [[-88.0, -185.0], [-87.0, -184.0], [-86.0, -183.0]], - [[-82.0, -179.0], [-81.0, -178.0], [-80.0, -177.0]] - ] - ); - // Test a narrowed version as this uses a layout start_offset. - let t1 = t1.i(2..)?; - let s = t1.broadcast_add(&t2.reshape((2, 1))?)?; - assert_eq!( - s.to_vec3::()?, - &[ - [[112.0, 113.0, 114.0], [215.0, 216.0, 217.0]], - [[118.0, 119.0, 120.0], [221.0, 222.0, 223.0]] - ] - ); - let s = t1.t()?.broadcast_add(&t2)?; - assert_eq!( - s.to_vec3::()?, - &[ - [[112.0, 215.0], [113.0, 216.0], [114.0, 217.0]], - [[118.0, 221.0], [119.0, 222.0], [120.0, 223.0]] - ] - ); - let s = t1.broadcast_sub(&t2.reshape((2, 1))?)?; - assert_eq!( - s.to_vec3::()?, - &[ - [[-88.0, -87.0, -86.0], [-185.0, -184.0, -183.0]], - [[-82.0, -81.0, -80.0], [-179.0, -178.0, -177.0]] - ] - ); - let s = t1.t()?.broadcast_sub(&t2)?; - assert_eq!( - s.to_vec3::()?, - &[ - [[-88.0, -185.0], [-87.0, -184.0], [-86.0, -183.0]], - [[-82.0, -179.0], [-81.0, -178.0], [-80.0, -177.0]] - ] - ); - let t3 = Tensor::new(1f32, device)?.broadcast_div(&t2)?; - let s = t1.broadcast_mul(&t2.reshape((2, 1))?)?; - let s_div = t1.broadcast_div(&t3.reshape((2, 1))?)?; - assert_eq!( - s.to_vec3::()?, - &[ - [[1200.0, 1300.0, 1400.0], [3000.0, 3200.0, 3400.0]], - [[1800.0, 1900.0, 2000.0], [4200.0, 4400.0, 4600.0]] - ] - ); - assert_eq!(s.to_vec3::()?, s_div.to_vec3::()?,); - let s = t1.t()?.broadcast_mul(&t2)?; - let s_div = t1.t()?.broadcast_div(&t3)?; - assert_eq!( - s.to_vec3::()?, - &[ - [[1200.0, 3000.0], [1300.0, 3200.0], [1400.0, 3400.0]], - [[1800.0, 4200.0], [1900.0, 4400.0], [2000.0, 4600.0]] - ] - ); - assert_eq!(s.to_vec3::()?, s_div.to_vec3::()?,); - Ok(()) -} - -fn randn(device: &Device) -> Result<()> { - let tensor = Tensor::randn(0f32, 1f32, (5, 3), device)?; - assert_eq!(tensor.dims(), [5, 3]); - // Check that the seed gets updated by checking that - // a new series of numbers is generated each time - let tensor2 = Tensor::randn(0f32, 1f32, (5, 3), device)?; - assert_ne!(tensor.to_vec2::()?, tensor2.to_vec2::()?); - let tensor = Tensor::rand(0f32, 1f32, (5, 3), device)?; - assert_eq!(tensor.dims(), [5, 3]); - // Check that the seed gets updated by checking that - // a new series of numbers is generated each time - let tensor2 = Tensor::rand(0f32, 1f32, (5, 3), device)?; - assert_ne!(tensor.to_vec2::()?, tensor2.to_vec2::()?); - // We do not expect deterministic elements at any index. - // There once was a bug that had a deterministic zero element in evenly sized tensors. - const N: usize = 2; - let v = (0..100) - .map(|_| Tensor::randn(0f32, 1f32, N, device).and_then(|t| t.to_vec1::())) - .collect::>>()?; - assert!( - (0..N).all(|i| v.windows(2).any(|pair| pair[0][i] != pair[1][i])), - "There are deterministic values in the randn tensors" - ); - let v = (0..100) - .map(|_| Tensor::rand(0f32, 1f32, N, device).and_then(|t| t.to_vec1::())) - .collect::>>()?; - assert!( - (0..N).all(|i| v.windows(2).any(|pair| pair[0][i] != pair[1][i])), - "There are deterministic values in the rand tensors" - ); - Ok(()) -} - -fn zero_dim(device: &Device) -> Result<()> { - let t = Tensor::zeros((4, 0, 1), DType::F32, device)?; - assert_eq!(t.dims3()?, (4, 0, 1)); - let t2 = Tensor::zeros((4, 3, 1), DType::F32, device)?; - let t_cat = Tensor::cat(&[&t, &t2], 1)?; - assert_eq!(t_cat.dims3()?, (4, 3, 1)); - let t_cat = Tensor::cat(&[&t, &t], 1)?; - assert_eq!(t_cat.dims3()?, (4, 0, 1)); - let t_unary = t.sqrt()?; - assert_eq!(t_unary.dims3()?, (4, 0, 1)); - let t_plus = (&t + 1.)?; - assert_eq!(t_plus.dims3()?, (4, 0, 1)); - let t_mm = t2.matmul(&t.t()?)?; - assert_eq!(t_mm.dims3()?, (4, 3, 0)); - let t_mm = t.matmul(&t2.t()?)?; - assert_eq!(t_mm.dims3()?, (4, 0, 3)); - let t_mm = t.t()?.matmul(&t)?; - assert_eq!(t_mm.dims3()?, (4, 1, 1)); - Ok(()) -} - -test_device!(zeros, zeros_cpu, zeros_gpu, zeros_metal); -test_device!(ones, ones_cpu, ones_gpu, ones_metal); -test_device!(full, full_cpu, full_gpu, full_metal); -test_device!(const_set, cs_cpu, cs_gpu, cs_metal); -test_device!(arange, arange_cpu, arange_gpu, arange_metal); -test_device!(add_mul, add_mul_cpu, add_mul_gpu, add_mul_metal); -test_device!(tensor_2d, tensor_2d_cpu, tensor_2d_gpu, tensor_2d_metal); -test_device!(narrow, narrow_cpu, narrow_gpu, narrow_metal); -test_device!(broadcast, broadcast_cpu, broadcast_gpu, broadcast_metal); -test_device!(slice_set, ss_cpu, ss_gpu, ss_metal); -test_device!(cat, cat_cpu, cat_gpu, cat_metal); -test_device!(sum, sum_cpu, sum_gpu, sum_metal); -test_device!(min, min_cpu, min_gpu, min_metal); -test_device!(max, max_cpu, max_gpu, max_metal); -test_device!(argmax, argmax_cpu, argmax_gpu, argmax_metal); -test_device!(argmin, argmin_cpu, argmin_gpu, argmin_metal); -test_device!(transpose, transpose_cpu, transpose_gpu, transpose_metal); -test_device!(unary_op, unary_op_cpu, unary_op_gpu, unary_op_metal); -test_device!(binary_op, binary_op_cpu, binary_op_gpu, binary_op_metal); -test_device!(ternary_op, ternary_op_cpu, ternary_op_gpu, ternary_op_metal); -test_device!(embeddings, embeddings_cpu, embeddings_gpu, embeddings_metal); -test_device!(cmp, cmp_cpu, cmp_gpu, cmp_metal); -test_device!( - broadcasting, - broadcasting_cpu, - broadcasting_gpu, - broadcasting_metal -); -test_device!( - index_select, - index_select_cpu, - index_select_gpu, - index_select_metal -); -test_device!(index_add, index_add_cpu, index_add_gpu, index_add_metal); -test_device!(gather, gather_cpu, gather_gpu, gather_metal); -test_device!(scatter, scatter_cpu, scatter_gpu, scatter_metal); -test_device!( - slice_scatter, - slice_scatter_cpu, - slice_scatter_gpu, - slice_scatter_metal -); -test_device!(randn, randn_cpu, randn_gpu, randn_metal); -test_device!(clamp, clamp_cpu, clamp_gpu, clamp_metal); -test_device!(asort, asort_cpu, asort_gpu, asort_metal); -test_device!(asort_big, asort_big_cpu, asort_big_gpu, asort_big_metal); -test_device!(var, var_cpu, var_gpu, var_metal); -test_device!(zero_dim, zero_dim_cpu, zero_dim_gpu, zero_dim_metal); - -fn tensor_send_sync(device: &Device) -> Result<()> { - let tensor = Tensor::new(vec![1.0f32, 2.0, 3.0], device)?; - - for _ in 0..10 { - let tensor = tensor.clone(); - std::thread::spawn(move || { - let new = tensor.add(&tensor).unwrap(); - let result: Vec = new.to_vec1().unwrap(); - assert_eq!(result, vec![2.0f32, 4.0, 6.0]); - }); - } - let result: Vec = tensor.to_vec1().unwrap(); - assert_eq!(result, vec![1.0f32, 2.0, 3.0]); - - let tensor = Tensor::new(vec![1.0f32, 2.0, 3.0], device)?; - tensor.device().synchronize().unwrap(); - - let new = std::thread::spawn(move || { - let new = tensor.add(&tensor).unwrap(); - new.device().synchronize().unwrap(); - new - }) - .join() - .unwrap(); - let result: Vec = new.to_vec1().unwrap(); - assert_eq!(result, vec![2.0f32, 4.0, 6.0]); - - Ok(()) -} -test_device!( - tensor_send_sync, - tensor_send_sync_cpu, - tensor_send_sync_gpu, - tensor_send_sync_metal -); - -// There was originally a bug on the CPU implementation for randn -// https://github.com/huggingface/candle/issues/381 -#[test] -fn randn_hasneg() -> Result<()> { - let t = Tensor::randn(0f32, 1f32, 200, &Device::Cpu)?.to_vec1::()?; - if t.iter().all(|&v| v >= 0.) { - candle_core::bail!("all values in tensors are non-negative") - } - Ok(()) -} - -#[test] -fn pad_with_same() -> Result<()> { - let t = Tensor::arange(1f32, 5f32, &Device::Cpu)?.reshape((2, 2))?; - let t0 = t.pad_with_same(0, 1, 2)?; - assert_eq!( - t0.to_vec2::()?, - [[1.0, 2.0], [1.0, 2.0], [3.0, 4.0], [3.0, 4.0], [3.0, 4.0]] - ); - let t1 = t.pad_with_same(1, 1, 2)?; - assert_eq!( - t1.to_vec2::()?, - [[1.0, 1.0, 2.0, 2.0, 2.0], [3.0, 3.0, 4.0, 4.0, 4.0]] - ); - Ok(()) -} - -#[test] -fn i64_abs() -> Result<()> { - let t = Tensor::new(&[-42i64, 1337], &Device::Cpu)?; - let t = t.abs()?; - assert_eq!(t.to_vec1::()?, [42, 1337]); - Ok(()) -} - -#[test] -fn tril_triu_eye() -> Result<()> { - let t = Tensor::tril2(4, DType::F32, &Device::Cpu)?; - assert_eq!( - t.to_vec2::()?, - [ - [1.0, 0.0, 0.0, 0.0], - [1.0, 1.0, 0.0, 0.0], - [1.0, 1.0, 1.0, 0.0], - [1.0, 1.0, 1.0, 1.0] - ], - ); - let t = Tensor::triu2(4, DType::F32, &Device::Cpu)?; - assert_eq!( - t.to_vec2::()?, - [ - [1.0, 1.0, 1.0, 1.0], - [0.0, 1.0, 1.0, 1.0], - [0.0, 0.0, 1.0, 1.0], - [0.0, 0.0, 0.0, 1.0] - ] - ); - let t = Tensor::eye(4, DType::F32, &Device::Cpu)?; - assert_eq!( - t.to_vec2::()?, - [ - [1.0, 0.0, 0.0, 0.0], - [0.0, 1.0, 0.0, 0.0], - [0.0, 0.0, 1.0, 0.0], - [0.0, 0.0, 0.0, 1.0] - ] - ); - Ok(()) -} - -#[test] -fn cumsum() -> Result<()> { - let t = &[3f32, 1., 4., 1., 5.]; - let t = Tensor::new(t, &Device::Cpu)?; - assert_eq!(t.cumsum(0)?.to_vec1::()?, [3., 4., 8., 9., 14.]); - let t = t.unsqueeze(1)?; - assert_eq!( - t.cumsum(0)?.to_vec2::()?, - [[3.0], [4.0], [8.0], [9.0], [14.0]] - ); - assert_eq!( - t.cumsum(1)?.to_vec2::()?, - [[3.0], [1.0], [4.0], [1.0], [5.0]] - ); - let t = &[[3f32, 1., 4., 1., 5.], [2., 1., 7., 8., 2.]]; - let t = Tensor::new(t, &Device::Cpu)?; - assert_eq!( - t.cumsum(1)?.to_vec2::()?, - [[3.0, 4.0, 8.0, 9.0, 14.0], [2.0, 3.0, 10.0, 18.0, 20.0]], - ); - assert_eq!( - t.cumsum(0)?.to_vec2::()?, - [[3.0, 1.0, 4.0, 1.0, 5.0], [5.0, 2.0, 11.0, 9.0, 7.0]] - ); - Ok(()) -} - -/// A helper function for floating point comparison. Both a and b must be 1D Tensor and contains the same amount of data. -/// Assertion passes if the difference of all pairs of a and b is smaller than epsilon. -fn assert_close(a: &Tensor, b: &Tensor, epsilon: f64) -> Result<()> { - let a_vec: Vec = a.to_vec1()?; - let b_vec: Vec = b.to_vec1()?; - - assert_eq!(a_vec.len(), b_vec.len()); - for (a, b) in a_vec.iter().zip(b_vec.iter()) { - assert!((a - b).abs() < epsilon); - } - Ok(()) -} - -#[test] -fn log_sum_exp() -> Result<()> { - let input = Tensor::new( - &[ - [[1f64, 2., 3.], [4., 5., 6.]], - [[-1000.0, -999.0, -1001.0], [1000.0, 999.0, 1001.0]], - ], - &Device::Cpu, - )?; - - let output = input.log_sum_exp(D::Minus1)?; - // The expectations obtained from pytorch. - let expected = Tensor::new(&[[3.4076, 6.4076], [-998.5924, 1001.4076]], &Device::Cpu)?; - assert_eq!(output.dims(), expected.dims()); - assert_close(&output.flatten_all()?, &expected.flatten_all()?, 0.00001)?; - - assert_eq!( - input.log_sum_exp((0, 1))?.to_vec1::()?, - [1000.0, 999.0, 1001.0] - ); - assert_eq!( - input.log_sum_exp(())?.to_vec3::()?, - input.to_vec3::()? - ); - - Ok(()) -} - -#[test] -fn pow() -> Result<()> { - let lhs = Tensor::new(&[[1f32, 2., 3.], [4., 5., 6.]], &Device::Cpu)?; - let rhs = (&lhs - 2.)?; - let res = lhs.pow(&rhs)?; - assert_eq!( - test_utils::to_vec2_round(&res, 3)?, - [[1.0, 1.0, 3.0], [16.0, 125.0, 1296.0]] - ); - Ok(()) -} - -#[test] -fn test_flip_1d() -> Result<()> { - // 1D: [0, 1, 2, 3, 4] - let t = Tensor::arange(0.0, 5.0, &Device::Cpu)?.reshape((5,))?; - let flipped = t.flip(&[0])?; - // Expected: [4, 3, 2, 1, 0] - let expected = Tensor::from_vec(vec![4.0, 3.0, 2.0, 1.0, 0.0], (5,), &Device::Cpu)?; - candle_core::test_utils::assert_tensor_eq(&flipped, &expected)?; - Ok(()) -} - -#[test] -fn test_flip_2d() -> Result<()> { - // 2D: - // [[0, 1, 2], - // [3, 4, 5]] - let t = Tensor::arange(0.0, 6.0, &Device::Cpu)?.reshape((2, 3))?; - let flipped = t.flip(&[0, 1])?; - // Expected: - // [[5, 4, 3], - // [2, 1, 0]] - let expected = Tensor::from_vec(vec![5.0, 4.0, 3.0, 2.0, 1.0, 0.0], (2, 3), &Device::Cpu)?; - candle_core::test_utils::assert_tensor_eq(&flipped, &expected)?; - Ok(()) -} - -#[test] -fn test_flip_3d_channels() -> Result<()> { - // 3D: - // [[[0,1,2], - // [3,4,5]], - // - // [[6,7,8], - // [9,10,11]]] - let t = Tensor::arange(0.0, 12.0, &Device::Cpu)?.reshape((2, 2, 3))?; - let flipped = t.flip(&[2])?; - // Expected: - // [[[2,1,0], - // [5,4,3]], - // - // [[8,7,6], - // [11,10,9]]] - let expected = Tensor::from_vec( - vec![2.0, 1.0, 0.0, 5.0, 4.0, 3.0, 8.0, 7.0, 6.0, 11.0, 10.0, 9.0], - (2, 2, 3), - &Device::Cpu, - )?; - candle_core::test_utils::assert_tensor_eq(&flipped, &expected)?; - Ok(()) -} - -#[test] -fn tensor_new() -> Result<()> { - let t1 = Tensor::new(vec![1f32, 2.0, 3.0], &Device::Cpu)?; - assert_eq!(t1.to_vec1::()?, [1.0, 2.0, 3.0]); - let t2 = Tensor::new(vec![vec![1f32, 2., 3.], vec![4., 5., 6.]], &Device::Cpu)?; - assert_eq!(t2.to_vec2::()?, [[1., 2., 3.], [4., 5., 6.]]); - let t3 = Tensor::new( - vec![ - vec![vec![1f32, 2., 3.], vec![4., 5., 6.]], - vec![vec![3f32, 1., 4.], vec![1., 5., 9.]], - ], - &Device::Cpu, - )?; - assert_eq!( - t3.to_vec3::()?, - [ - [[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]], - [[3.0, 1.0, 4.0], [1.0, 5.0, 9.0]] - ] - ); - Ok(()) -} - -#[test] -fn tensor_norm() -> Result<()> { - let t = Tensor::new(&[[3., 4.], [0., 0.]], &Device::Cpu)?; - let norm = t.norm()?; - assert_eq!(norm.to_scalar::()?, 5.); - Ok(()) -} - -#[cfg(feature = "cuda")] -#[test] -fn transfers_cuda_to_device() -> Result<()> { - use rand::seq::SliceRandom; - - let devices = cudarc::driver::safe::CudaContext::device_count() - .map_err(candle_core::cuda::CudaError::from)?; - if devices < 2 { - return Ok(()); - } - let first = Device::new_cuda(0)?; - - let mut data: Vec = (0..262144).collect(); - let mut rng = rand::rng(); - data.shuffle(&mut rng); - - let t1 = Tensor::from_vec(data, (512, 512), &first)?; - let second = Device::new_cuda(1)?; - let t2 = t1.to_device(&second)?; - - assert_ne!( - t1.device().as_cuda_device()?.id(), - t2.device().as_cuda_device()?.id() - ); - Ok(()) -} - -#[cfg(feature = "cuda")] -#[test] -fn allocates_twice_when_transferring_to_same_device() -> Result<()> { - use std::{ops::Deref, sync::RwLockReadGuard}; - - use candle_core::Storage; - use rand::seq::SliceRandom; - - let first = Device::new_cuda(0)?; - let second = Device::new_cuda(0)?; - - let mut data: Vec = (0..262144).collect(); - let mut rng = rand::rng(); - data.shuffle(&mut rng); - - let t1 = Tensor::from_vec(data, (512, 512), &first)?; - let t2 = t1.to_device(&second)?; - - let (storage1, _) = t1.storage_and_layout(); - let (storage2, _) = t2.storage_and_layout(); - let extract = |s: RwLockReadGuard<'_, Storage>| match &s.deref() { - Storage::Cuda(c) => { - use cudarc::driver::DevicePtr; - let slice = c.as_cuda_slice::().unwrap(); - let ptr = slice.device_ptr(slice.stream()).0; - ptr - } - _ => unimplemented!(), - }; - let id1 = extract(storage1); - let id2 = extract(storage2); - assert_ne!(id1, id2); - Ok(()) -} diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/test.npy b/v2/dependency-patches/candle-core-0.9.2/tests/test.npy deleted file mode 100644 index a3ff5af9540510ac6518b5a2478bc1a40f992371..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 208 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlWC!@qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= wXCxM+0{I$-20EHL3bhL411<(AV1&|4P?{M^vp{K9D9r|?*`YKCl;(ue0P!XpoB#j- diff --git a/v2/dependency-patches/candle-core-0.9.2/tests/test.npz b/v2/dependency-patches/candle-core-0.9.2/tests/test.npz deleted file mode 100644 index b6683caab0e0e9e66cb5ca649aab4980a1e650df..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 668 zcmWIWW@Zs#fB;1X8xBpr3qTGCvoeSRzLK#d@ni)#7KxtMe%?72}p)?1S=7iD#D8ZokVdgfgc5ZyS5W1OUF4@IgfaSj3!+p!yZC;|&vLW_h60tGIXYp+tbR@}8WmEgpM zO8o`Bapb}U_$P4S5AY|DxUgf#!FKBdBaPM{^PXqkJlnY0LP(O(mFS~w#7M$MzRTR< z=}BmrTj)(S-V2@=g(2}+5G@kTjR$rRn5(#DL_r#w{fSQL9lLubi?Fp-^@ z94>>866e`Y|{Pa$jw1Xu^ zrRNx70k_WS4cx+wb1GjPK?hw%dsEs^hBod<4~{Qif95DiyLR*qw3Cf~_?D3fMzYN9 ztFwa^VZc~OfcnG1uN{%oLe^7v(W8H=jugv0xSqGFbKN~i0U=}u0!bi1b%=I$q+P)! zs0)!V96JB>^~;lo{263@=RMAm`r(Yk317Bl3cU5P3=H2t&R;?Yko63^$jBImV5w0; z;=m1qcP?1R34F^`|6dDL)%SE=n`qfpi3(kQ`+4)n zfOiOK<3=i3?);)L`uu%4O8-F>T3q}_3HOKJrO>sHs@S6R-4gZ{-|YIcBc%!~D&8vr zH~Cz}JynrK6?RL=X^y;pKQcsFZQ}Y+30Wu-S|m&nlEnez+N;E>iM#fu4OV8_A{%=Ua&@QUk(LBL$$hm%--75a)d;S;$UxT1w^n387>AHq2KYo>>4lqPFX(eg-Z1eK!Yr}cbF5km4 zMqWW11%Nh;xri}WZ5bp#%(Vi@y9JO`dS!(CFxCqQCOo-FBa=WGBHyXa0>t+Ph|6ba t(q?lj%e+;)Gsl?ec^02%3viYRAMz53|Hu{qk4@UZrG#B*5WJt>{S7%wAvgd4 diff --git a/v2/dependency-patches/compat-tests/Cargo.lock b/v2/dependency-patches/compat-tests/Cargo.lock deleted file mode 100644 index b47c10c170..0000000000 --- a/v2/dependency-patches/compat-tests/Cargo.lock +++ /dev/null @@ -1,887 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 4 - -[[package]] -name = "autocfg" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bitflags" -version = "2.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" - -[[package]] -name = "bytemuck" -version = "1.25.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95832e849adfb21180ccb6826a99da14e5d266ae5c2e668e1602cf234f153797" -dependencies = [ - "bytemuck_derive", -] - -[[package]] -name = "bytemuck_derive" -version = "1.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a1f896587b6f2c069c73d2f0913e2d590c3990285cd2f0b6aa02b786b4c679c" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.9", -] - -[[package]] -name = "byteorder" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" - -[[package]] -name = "candle-core" -version = "0.4.1" -dependencies = [ - "byteorder", - "gemm", - "half", - "memmap2", - "num-traits", - "num_cpus", - "rand", - "rand_distr", - "rayon", - "safetensors", - "thiserror", - "yoke", - "zip", -] - -[[package]] -name = "cfg-if" -version = "1.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" - -[[package]] -name = "cfg_aliases" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527" - -[[package]] -name = "crc32fast" -version = "1.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055" -dependencies = [ - "cfg-if", -] - -[[package]] -name = "crossbeam-deque" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" -dependencies = [ - "crossbeam-epoch", - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-epoch" -version = "0.9.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" -dependencies = [ - "crossbeam-utils", -] - -[[package]] -name = "crossbeam-utils" -version = "0.8.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" - -[[package]] -name = "crunchy" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" - -[[package]] -name = "dyn-stack" -version = "0.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e53799688f5632f364f8fb387488dd05db9fe45db7011be066fc20e7027f8b" -dependencies = [ - "bytemuck", - "reborrow", -] - -[[package]] -name = "either" -version = "1.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" - -[[package]] -name = "enum-as-inner" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "equivalent" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" - -[[package]] -name = "errno" -version = "0.3.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" -dependencies = [ - "libc", - "windows-sys", -] - -[[package]] -name = "fastrand" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da7c62ceae207dd37ea5b845da6a0696c799f85e97da1ab5b7910be3c1c80223" - -[[package]] -name = "gemm" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ab24cc62135b40090e31a76a9b2766a501979f3070fa27f689c27ec04377d32" -dependencies = [ - "dyn-stack", - "gemm-c32", - "gemm-c64", - "gemm-common", - "gemm-f16", - "gemm-f32", - "gemm-f64", - "num-complex", - "num-traits", - "paste", - "raw-cpuid", - "seq-macro", -] - -[[package]] -name = "gemm-c32" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9c030d0b983d1e34a546b86e08f600c11696fde16199f971cd46c12e67512c0" -dependencies = [ - "dyn-stack", - "gemm-common", - "num-complex", - "num-traits", - "paste", - "raw-cpuid", - "seq-macro", -] - -[[package]] -name = "gemm-c64" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbb5f2e79fefb9693d18e1066a557b4546cd334b226beadc68b11a8f9431852a" -dependencies = [ - "dyn-stack", - "gemm-common", - "num-complex", - "num-traits", - "paste", - "raw-cpuid", - "seq-macro", -] - -[[package]] -name = "gemm-common" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2e7ea062c987abcd8db95db917b4ffb4ecdfd0668471d8dc54734fdff2354e8" -dependencies = [ - "bytemuck", - "dyn-stack", - "half", - "num-complex", - "num-traits", - "once_cell", - "paste", - "pulp", - "raw-cpuid", - "rayon", - "seq-macro", - "sysctl", -] - -[[package]] -name = "gemm-f16" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ca4c06b9b11952071d317604acb332e924e817bd891bec8dfb494168c7cedd4" -dependencies = [ - "dyn-stack", - "gemm-common", - "gemm-f32", - "half", - "num-complex", - "num-traits", - "paste", - "raw-cpuid", - "rayon", - "seq-macro", -] - -[[package]] -name = "gemm-f32" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9a69f51aaefbd9cf12d18faf273d3e982d9d711f60775645ed5c8047b4ae113" -dependencies = [ - "dyn-stack", - "gemm-common", - "num-complex", - "num-traits", - "paste", - "raw-cpuid", - "seq-macro", -] - -[[package]] -name = "gemm-f64" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa397a48544fadf0b81ec8741e5c0fba0043008113f71f2034def1935645d2b0" -dependencies = [ - "dyn-stack", - "gemm-common", - "num-complex", - "num-traits", - "paste", - "raw-cpuid", - "seq-macro", -] - -[[package]] -name = "getrandom" -version = "0.2.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" -dependencies = [ - "cfg-if", - "libc", - "wasi", -] - -[[package]] -name = "getrandom" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" -dependencies = [ - "cfg-if", - "libc", - "r-efi", -] - -[[package]] -name = "half" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc52e53916c08643f1b56ec082790d1e86a32e58dc5268f897f313fbae7b4872" -dependencies = [ - "bytemuck", - "cfg-if", - "crunchy", - "num-traits", - "rand", - "rand_distr", -] - -[[package]] -name = "hashbrown" -version = "0.17.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "hermit-abi" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17592d60ebacc7d5e169f4663c5f84f9161cc90328abcfe8456f41e4dfcb284" - -[[package]] -name = "indexmap" -version = "2.14.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" -dependencies = [ - "equivalent", - "hashbrown", -] - -[[package]] -name = "itoa" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" - -[[package]] -name = "libc" -version = "0.2.191" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" - -[[package]] -name = "libm" -version = "0.2.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" - -[[package]] -name = "linux-raw-sys" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" - -[[package]] -name = "memchr" -version = "2.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" - -[[package]] -name = "memmap2" -version = "0.9.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" -dependencies = [ - "libc", - "stable_deref_trait", -] - -[[package]] -name = "nix" -version = "0.31.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf20d2fde8ff38632c426f1165ed7436270b44f199fc55284c38276f9db47c3d" -dependencies = [ - "bitflags 2.13.2", - "cfg-if", - "cfg_aliases", - "libc", -] - -[[package]] -name = "num-complex" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" -dependencies = [ - "bytemuck", - "num-traits", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", - "libm", -] - -[[package]] -name = "num_cpus" -version = "1.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" -dependencies = [ - "hermit-abi", - "libc", -] - -[[package]] -name = "once_cell" -version = "1.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" - -[[package]] -name = "paste" -version = "1.0.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" - -[[package]] -name = "ppv-lite86" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9" -dependencies = [ - "zerocopy", -] - -[[package]] -name = "proc-macro2" -version = "1.0.107" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "pulp" -version = "0.18.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0a01a0dc67cf4558d279f0c25b0962bd08fc6dec0137699eae304103e882fe6" -dependencies = [ - "bytemuck", - "libm", - "num-complex", - "reborrow", -] - -[[package]] -name = "quote" -version = "1.0.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - -[[package]] -name = "rand" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c" -dependencies = [ - "libc", - "rand_chacha", - "rand_core", -] - -[[package]] -name = "rand_chacha" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.6.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" -dependencies = [ - "getrandom 0.2.17", -] - -[[package]] -name = "rand_distr" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" -dependencies = [ - "num-traits", - "rand", -] - -[[package]] -name = "raw-cpuid" -version = "10.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" -dependencies = [ - "bitflags 1.3.2", -] - -[[package]] -name = "rayon" -version = "1.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" -dependencies = [ - "either", - "rayon-core", -] - -[[package]] -name = "rayon-core" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" -dependencies = [ - "crossbeam-deque", - "crossbeam-utils", -] - -[[package]] -name = "reborrow" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" - -[[package]] -name = "rustix" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891efababe418670775f199f0d233d84843c227a0949a883ce15b37c78d6629d" -dependencies = [ - "bitflags 2.13.2", - "errno", - "libc", - "linux-raw-sys", - "windows-sys", -] - -[[package]] -name = "ruview-candle04-zip9-compat-test" -version = "0.1.0" -dependencies = [ - "candle-core", - "half", - "tempfile", -] - -[[package]] -name = "safetensors" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44560c11236a6130a46ce36c836a62936dc81ebf8c36a37947423571be0e55b6" -dependencies = [ - "serde", - "serde_json", -] - -[[package]] -name = "same-file" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" -dependencies = [ - "winapi-util", -] - -[[package]] -name = "seq-macro" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" - -[[package]] -name = "serde" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4148590afebada386688f18773da617792bf2ef03ffc1e4cbd2b1d45b023e0ba" -dependencies = [ - "serde_core", - "serde_derive", -] - -[[package]] -name = "serde_core" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67dca2c9c51e58a4791a4b1ed58308b39c64224d349a935ab5039aa360942a48" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.229" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.9", -] - -[[package]] -name = "serde_json" -version = "1.0.154" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" -dependencies = [ - "itoa", - "memchr", - "serde", - "serde_core", - "zmij", -] - -[[package]] -name = "stable_deref_trait" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" - -[[package]] -name = "syn" -version = "2.0.119" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "3.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "synstructure" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "synstructure" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901704edd0dfe137f1987838ee4f259e4e063c31371bdb423f7ae38ec6f77f02" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.9", -] - -[[package]] -name = "sysctl" -version = "0.5.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea" -dependencies = [ - "bitflags 2.13.2", - "byteorder", - "enum-as-inner", - "libc", - "thiserror", - "walkdir", -] - -[[package]] -name = "tempfile" -version = "3.27.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" -dependencies = [ - "fastrand", - "getrandom 0.4.3", - "once_cell", - "rustix", - "windows-sys", -] - -[[package]] -name = "thiserror" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6aaf5339b578ea85b50e080feb250a3e8ae8cfcdff9a461c9ec2904bc923f52" -dependencies = [ - "thiserror-impl", -] - -[[package]] -name = "thiserror-impl" -version = "1.0.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "typed-path" -version = "0.12.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54ec2755506471f54baf5e4bb5499d8e510cc1caf271270e778a736bd2d7336c" - -[[package]] -name = "unicode-ident" -version = "1.0.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" - -[[package]] -name = "walkdir" -version = "2.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" -dependencies = [ - "same-file", - "winapi-util", -] - -[[package]] -name = "wasi" -version = "0.11.1+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" - -[[package]] -name = "winapi-util" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "windows-link" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" - -[[package]] -name = "windows-sys" -version = "0.61.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" -dependencies = [ - "windows-link", -] - -[[package]] -name = "yoke" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" -dependencies = [ - "serde", - "stable_deref_trait", - "yoke-derive", - "zerofrom", -] - -[[package]] -name = "yoke-derive" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", - "synstructure 0.13.2", -] - -[[package]] -name = "zerocopy" -version = "0.8.63" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.63" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - -[[package]] -name = "zerofrom" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec05a11813ea801ff6d75110ad09cd0824ddba17dfe17128ea0d5f68e6c5272" -dependencies = [ - "zerofrom-derive", -] - -[[package]] -name = "zerofrom-derive" -version = "0.1.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f75b4683f6c7f45248d4d64056a24298c6281e0993356d7d1b4a1a962ef10d4a" -dependencies = [ - "proc-macro2", - "quote", - "syn 3.0.9", - "synstructure 0.14.0", -] - -[[package]] -name = "zip" -version = "9.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e" -dependencies = [ - "crc32fast", - "indexmap", - "memchr", - "nix", - "typed-path", -] - -[[package]] -name = "zmij" -version = "1.0.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29666d0abbfad1e3dc4dcf6144730dd3a3ab225bbbdac83319345b1b44ccfc1b" diff --git a/v2/dependency-patches/compat-tests/Cargo.toml b/v2/dependency-patches/compat-tests/Cargo.toml deleted file mode 100644 index 2b4b879f50..0000000000 --- a/v2/dependency-patches/compat-tests/Cargo.toml +++ /dev/null @@ -1,13 +0,0 @@ -[package] -name = "ruview-candle04-zip9-compat-test" -version = "0.1.0" -edition = "2021" -publish = false - -# Isolated because the legacy optional backend predates half's rand0.9 -# migration; the main workspace's burn backend requires half2.7. -[workspace] -[dependencies] -candle-core = { path = "../candle-core-0.4.1", default-features = false } -half = "=2.3.1" -tempfile = "3" diff --git a/v2/dependency-patches/compat-tests/src/lib.rs b/v2/dependency-patches/compat-tests/src/lib.rs deleted file mode 100644 index e9386bad6a..0000000000 --- a/v2/dependency-patches/compat-tests/src/lib.rs +++ /dev/null @@ -1,14 +0,0 @@ -// ZIP9 compatibility must preserve actual tensor names and values, not just compile. -use candle_core::{Device, Tensor}; -#[test] -fn npz_roundtrip_retains_unicode_and_nested_tensor_names() -> candle_core::Result<()> { - let directory = tempfile::tempdir()?; - let path = directory.path().join("tensors.npz"); - let tensor = Tensor::new(&[1f32, -2.5, 3.25], &Device::Cpu)?; - Tensor::write_npz(&[("nested/room-α", &tensor)], &path)?; - let restored = Tensor::read_npz(&path)?; - assert_eq!(restored.len(), 1); - assert_eq!(restored[0].0, "nested/room-α"); - assert_eq!(restored[0].1.to_vec1::()?, vec![1., -2.5, 3.25]); - Ok(()) -} diff --git a/v2/dependency-patches/compatibility.patch b/v2/dependency-patches/compatibility.patch deleted file mode 100644 index 2ca57831b3..0000000000 --- a/v2/dependency-patches/compatibility.patch +++ /dev/null @@ -1,185 +0,0 @@ ---- upstream/candle-core-0.4.1/Cargo.toml -+++ patched/candle-core-0.4.1/Cargo.toml -@@ -104,7 +104,7 @@ - features = ["derive"] - - [dependencies.zip] --version = "0.6.6" -+version = "9.0.1" - default-features = false - - [dev-dependencies.anyhow] ---- upstream/candle-core-0.4.1/src/npy.rs -+++ patched/candle-core-0.4.1/src/npy.rs -@@ -261,8 +261,8 @@ - for i in 0..zip.len() { - let mut reader = zip.by_index(i)?; - let name = { -- let name = reader.name(); -- name.strip_suffix(NPY_SUFFIX).unwrap_or(name).to_owned() -+ let name = reader.name()?; -+ name.strip_suffix(NPY_SUFFIX).unwrap_or(&name).to_owned() - }; - let header = read_header(&mut reader)?; - let header = Header::parse(&header)?; -@@ -331,7 +331,7 @@ - ) -> Result<()> { - let mut zip = zip::ZipWriter::new(File::create(path.as_ref())?); - let options = -- zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Stored); -+ zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); - - for (name, tensor) in ts.iter() { - zip.start_file(format!("{}.npy", name.as_ref()), options)?; -@@ -358,8 +358,8 @@ - for i in 0..zip.len() { - let file = zip.by_index(i)?; - let name = { -- let name = file.name(); -- name.strip_suffix(NPY_SUFFIX).unwrap_or(name).to_owned() -+ let name = file.name()?; -+ name.strip_suffix(NPY_SUFFIX).unwrap_or(&name).to_owned() - }; - index_per_name.insert(name, i); - } ---- upstream/candle-core-0.4.1/src/pickle.rs -+++ patched/candle-core-0.4.1/src/pickle.rs -@@ -653,8 +653,8 @@ - let mut zip = zip::ZipArchive::new(zip_reader)?; - let zip_file_names = zip - .file_names() -- .map(|f| f.to_string()) -- .collect::>(); -+ .map(|name| name.map(|name| name.into_owned())) -+ .collect::>>()?; - - let mut tensor_infos = vec![]; - for file_name in zip_file_names.iter() { ---- upstream/candle-core-0.9.2/Cargo.toml -+++ patched/candle-core-0.9.2/Cargo.toml -@@ -242,7 +242,7 @@ - features = ["derive"] - - [dependencies.zip] --version = "7.2.0" -+version = "9.0.1" - default-features = false - - [dev-dependencies.anyhow] ---- upstream/candle-core-0.9.2/src/npy.rs -+++ patched/candle-core-0.9.2/src/npy.rs -@@ -288,8 +288,8 @@ - for i in 0..zip.len() { - let mut reader = zip.by_index(i)?; - let name = { -- let name = reader.name(); -- name.strip_suffix(NPY_SUFFIX).unwrap_or(name).to_owned() -+ let name = reader.name()?; -+ name.strip_suffix(NPY_SUFFIX).unwrap_or(&name).to_owned() - }; - let header = read_header(&mut reader)?; - let header = Header::parse(&header)?; -@@ -358,7 +358,7 @@ - ) -> Result<()> { - let mut zip = zip::ZipWriter::new(File::create(path.as_ref())?); - let options: zip::write::FileOptions<()> = -- zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Stored); -+ zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); - - for (name, tensor) in ts.iter() { - zip.start_file(format!("{}.npy", name.as_ref()), options)?; -@@ -385,8 +385,8 @@ - for i in 0..zip.len() { - let file = zip.by_index(i)?; - let name = { -- let name = file.name(); -- name.strip_suffix(NPY_SUFFIX).unwrap_or(name).to_owned() -+ let name = file.name()?; -+ name.strip_suffix(NPY_SUFFIX).unwrap_or(&name).to_owned() - }; - index_per_name.insert(name, i); - } ---- upstream/candle-core-0.9.2/src/pickle.rs -+++ patched/candle-core-0.9.2/src/pickle.rs -@@ -677,8 +677,8 @@ - let mut zip = zip::ZipArchive::new(zip_reader)?; - let zip_file_names = zip - .file_names() -- .map(|f| f.to_string()) -- .collect::>(); -+ .map(|name| name.map(|name| name.into_owned())) -+ .collect::>>()?; - - let mut tensor_infos = vec![]; - for file_name in zip_file_names.iter() { ---- upstream/tch-0.24.0/Cargo.toml -+++ patched/tch-0.24.0/Cargo.toml -@@ -240,7 +240,7 @@ - version = "0.24.0" - - [dependencies.zip] --version = "0.6" -+version = "9.0.1" - - [dev-dependencies.anyhow] - version = "^1.0.60" ---- upstream/tch-0.24.0/src/tensor/npy.rs -+++ patched/tch-0.24.0/src/tensor/npy.rs -@@ -188,8 +188,8 @@ - for i in 0..zip.len() { - let mut reader = zip.by_index(i).unwrap(); - let name = { -- let name = reader.name(); -- name.strip_suffix(NPY_SUFFIX).unwrap_or(name).to_owned() -+ let name = reader.name()?; -+ name.strip_suffix(NPY_SUFFIX).unwrap_or(&name).to_owned() - }; - let header = read_header(&mut reader)?; - let header = Header::parse(&header)?; -@@ -236,7 +236,7 @@ - ) -> Result<(), TchError> { - let mut zip = zip::ZipWriter::new(File::create(path.as_ref())?); - let options = -- zip::write::FileOptions::default().compression_method(zip::CompressionMethod::Stored); -+ zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); - - for (name, tensor) in ts.iter() { - zip.start_file(format!("{}.npy", name.as_ref()), options)?; ---- upstream/torch-sys-0.24.0/Cargo.toml -+++ patched/torch-sys-0.24.0/Cargo.toml -@@ -76,4 +76,4 @@ - optional = true - - [build-dependencies.zip] --version = "0.6" -+version = "9.0.1" ---- upstream/torch-sys-0.24.0/build.rs -+++ patched/torch-sys-0.24.0/build.rs -@@ -121,25 +121,8 @@ - let file = fs::File::open(&filename)?; - let buf = io::BufReader::new(file); - let mut archive = zip::ZipArchive::new(buf)?; -- for i in 0..archive.len() { -- let mut file = archive.by_index(i)?; -- let outpath = outpath.as_ref().join(file.mangled_name()); -- if !file.name().ends_with('/') { -- println!( -- "File {} extracted to \"{}\" ({} bytes)", -- i, -- outpath.as_path().display(), -- file.size() -- ); -- if let Some(p) = outpath.parent() { -- if !p.exists() { -- fs::create_dir_all(p)?; -- } -- } -- let mut outfile = fs::File::create(&outpath)?; -- io::copy(&mut file, &mut outfile)?; -- } -- } -+ // ZIP 9's checked extraction rejects path/symlink escapes and unsafe permissions. -+ archive.extract(outpath.as_ref())?; - - // This is if we're unzipping a python wheel. - if outpath.as_ref().join("torch").exists() { diff --git a/v2/dependency-patches/patches.json b/v2/dependency-patches/patches.json deleted file mode 100644 index ac5fcd607b..0000000000 --- a/v2/dependency-patches/patches.json +++ /dev/null @@ -1,1877 +0,0 @@ -{ - "schema": 1, - "packages": [ - { - "package": "candle-core-0.4.1", - "archiveSha256": "6f1b20174c1707e20f4cb364a355b449803c03e9b0c9193324623cf9787a4e00", - "changedFiles": [ - "Cargo.toml", - "src/npy.rs", - "src/pickle.rs" - ], - "files": [ - { - "path": "Cargo.toml", - "upstreamSha256": "73aa7197c515b16c9391b15beaf7cec96ae2d9023b1fa445ae39d8f3e7c835f1", - "sha256": "8da30d82317883eaba530173eba8fa663eba366e05f60728156ca79c3b38409f" - }, - { - "path": "Cargo.toml.orig", - "upstreamSha256": "f63b2a021badc93735c8fbce9922d08cb171f1dea8cc6cc47c0c0e89165e427f", - "sha256": "f63b2a021badc93735c8fbce9922d08cb171f1dea8cc6cc47c0c0e89165e427f" - }, - { - "path": "LICENSE", - "upstreamSha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4", - "sha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4" - }, - { - "path": "README.md", - "upstreamSha256": "9b00007b1d568125e67d6f2ff5c40acd8227dec4c62681cbc87775768a125200", - "sha256": "9b00007b1d568125e67d6f2ff5c40acd8227dec4c62681cbc87775768a125200" - }, - { - "path": "benches/bench_main.rs", - "upstreamSha256": "74b8b99df438f178eaa64bdff5c94eefbc67d2bedac347e3978feb32430db22f", - "sha256": "74b8b99df438f178eaa64bdff5c94eefbc67d2bedac347e3978feb32430db22f" - }, - { - "path": "benches/benchmarks/affine.rs", - "upstreamSha256": "2bd50867a662b354c63531fa2cf33dbc62cdc5a27006fe2defae655076d19e7a", - "sha256": "2bd50867a662b354c63531fa2cf33dbc62cdc5a27006fe2defae655076d19e7a" - }, - { - "path": "benches/benchmarks/matmul.rs", - "upstreamSha256": "268cd905c3e1a2d76b87bdee52a57adcae8f60accb61faea28492c10533bd2a5", - "sha256": "268cd905c3e1a2d76b87bdee52a57adcae8f60accb61faea28492c10533bd2a5" - }, - { - "path": "benches/benchmarks/mod.rs", - "upstreamSha256": "6e4538e0bd00bc1c90aad32485ad71b9944b47c1bffc6f9d85ac2a1977a1f852", - "sha256": "6e4538e0bd00bc1c90aad32485ad71b9944b47c1bffc6f9d85ac2a1977a1f852" - }, - { - "path": "benches/benchmarks/random.rs", - "upstreamSha256": "d19e9712cabd73fd2f9b630b8e0dbdd227b987282b70b86026e6789b985a193b", - "sha256": "d19e9712cabd73fd2f9b630b8e0dbdd227b987282b70b86026e6789b985a193b" - }, - { - "path": "benches/benchmarks/where_cond.rs", - "upstreamSha256": "9da8cedbf0c0cd5026f3b75b5c8bf99a3620216ff8c1de5b9866083decf1bbf7", - "sha256": "9da8cedbf0c0cd5026f3b75b5c8bf99a3620216ff8c1de5b9866083decf1bbf7" - }, - { - "path": "examples/basics.rs", - "upstreamSha256": "834936869ca3542647c1c0feaafce578708cda17381b8010fb9ce2c1b78ef838", - "sha256": "834936869ca3542647c1c0feaafce578708cda17381b8010fb9ce2c1b78ef838" - }, - { - "path": "examples/cuda_basics.rs", - "upstreamSha256": "ef49742e6d60eefc67369bb2ad7148afa6af3fcb8a90614d6988163cc2894505", - "sha256": "ef49742e6d60eefc67369bb2ad7148afa6af3fcb8a90614d6988163cc2894505" - }, - { - "path": "examples/cuda_sum_benchmark.rs", - "upstreamSha256": "8c1ccac409d77695793186588274f8b08c2c66337f0751485634a0329a55a9b7", - "sha256": "8c1ccac409d77695793186588274f8b08c2c66337f0751485634a0329a55a9b7" - }, - { - "path": "examples/tensor-tools.rs", - "upstreamSha256": "7a1d7b40bf6f8bd6ef288357a23fe5030f1907629cd046e5746a1a516dc15f67", - "sha256": "7a1d7b40bf6f8bd6ef288357a23fe5030f1907629cd046e5746a1a516dc15f67" - }, - { - "path": "src/accelerate.rs", - "upstreamSha256": "9be99929bb6553d9baf6b6a5a56fa66bfdbee122f76c4792a00e09ac8a0d638a", - "sha256": "9be99929bb6553d9baf6b6a5a56fa66bfdbee122f76c4792a00e09ac8a0d638a" - }, - { - "path": "src/backend.rs", - "upstreamSha256": "4ea8f8411ee365aadcf07c3dfa3e5b7b9120d888e638103a783b9b0c51b83572", - "sha256": "4ea8f8411ee365aadcf07c3dfa3e5b7b9120d888e638103a783b9b0c51b83572" - }, - { - "path": "src/backprop.rs", - "upstreamSha256": "e40a3a24153e8b3c589447a90e1598d7a2f3c41e0eec6851c54259ccf1bb7615", - "sha256": "e40a3a24153e8b3c589447a90e1598d7a2f3c41e0eec6851c54259ccf1bb7615" - }, - { - "path": "src/conv.rs", - "upstreamSha256": "973c9176dad4060b7ec749c080a9ce4098c05266bff585b0ac2df28c1050379e", - "sha256": "973c9176dad4060b7ec749c080a9ce4098c05266bff585b0ac2df28c1050379e" - }, - { - "path": "src/convert.rs", - "upstreamSha256": "859ba5f6c8a9ae69dd1f5442904cd86202d98c24d96d55c9c1cfe04ff37b50a0", - "sha256": "859ba5f6c8a9ae69dd1f5442904cd86202d98c24d96d55c9c1cfe04ff37b50a0" - }, - { - "path": "src/cpu/avx.rs", - "upstreamSha256": "6a30f8f1c6a7c6b103ce7d675d84f550a622369839b115399f65d72a504cbd10", - "sha256": "6a30f8f1c6a7c6b103ce7d675d84f550a622369839b115399f65d72a504cbd10" - }, - { - "path": "src/cpu/erf.rs", - "upstreamSha256": "d71df728dca01fd96e9fffef7778149ac1ac776d65434d0087f9ae9bb266224c", - "sha256": "d71df728dca01fd96e9fffef7778149ac1ac776d65434d0087f9ae9bb266224c" - }, - { - "path": "src/cpu/kernels.rs", - "upstreamSha256": "ac7cbf3b316c2ff0b532d3c74a2b1e086bbf074899ffebeaa9d97bf337577c6a", - "sha256": "ac7cbf3b316c2ff0b532d3c74a2b1e086bbf074899ffebeaa9d97bf337577c6a" - }, - { - "path": "src/cpu/mod.rs", - "upstreamSha256": "ce333c5327a3250d3efd04dac983f27341480c13822f1a63e24832f928bc932b", - "sha256": "ce333c5327a3250d3efd04dac983f27341480c13822f1a63e24832f928bc932b" - }, - { - "path": "src/cpu/neon.rs", - "upstreamSha256": "c180dffb8a342c2f026d8fdcc235c7d9e8f0aac8a5f6f03de4c2f4de8dda609c", - "sha256": "c180dffb8a342c2f026d8fdcc235c7d9e8f0aac8a5f6f03de4c2f4de8dda609c" - }, - { - "path": "src/cpu/simd128.rs", - "upstreamSha256": "8ed8cff924086325c4d4af5a12746a7ce4a5a05f18cdeaa28bc0ff9c64060906", - "sha256": "8ed8cff924086325c4d4af5a12746a7ce4a5a05f18cdeaa28bc0ff9c64060906" - }, - { - "path": "src/cpu_backend.rs", - "upstreamSha256": "c136829a246ef9088e84926c73b8721327473d893756f7bd7da2dd549b88ad7f", - "sha256": "c136829a246ef9088e84926c73b8721327473d893756f7bd7da2dd549b88ad7f" - }, - { - "path": "src/cuda_backend.rs", - "upstreamSha256": "f765d6c163aa9c1072a46b9a92ebdc83a5bcf580f76f0e3bbdd68450fa1f5be6", - "sha256": "f765d6c163aa9c1072a46b9a92ebdc83a5bcf580f76f0e3bbdd68450fa1f5be6" - }, - { - "path": "src/cudnn.rs", - "upstreamSha256": "d4381e8361653afc0dc9f9ec55c1881e36ec53504f9dbf3d76e0c9834ef4ec00", - "sha256": "d4381e8361653afc0dc9f9ec55c1881e36ec53504f9dbf3d76e0c9834ef4ec00" - }, - { - "path": "src/device.rs", - "upstreamSha256": "5fa4b7ff27f477d8b74687bed8904feee6c8bc12757cbf653c35c7a5e093dab3", - "sha256": "5fa4b7ff27f477d8b74687bed8904feee6c8bc12757cbf653c35c7a5e093dab3" - }, - { - "path": "src/display.rs", - "upstreamSha256": "ed33ff8bce426676a73df05713f14c2139f6312fa953a34ee8793f4ac56d16e0", - "sha256": "ed33ff8bce426676a73df05713f14c2139f6312fa953a34ee8793f4ac56d16e0" - }, - { - "path": "src/dtype.rs", - "upstreamSha256": "349c6efecf22cd3f1160cd0d38758400d6835c881481a2e88af716e9550c547c", - "sha256": "349c6efecf22cd3f1160cd0d38758400d6835c881481a2e88af716e9550c547c" - }, - { - "path": "src/dummy_cuda_backend.rs", - "upstreamSha256": "e9df849b9597e770954e251e318f7f784fe103b8ccfbf4fcfd0a07b63e78e4ed", - "sha256": "e9df849b9597e770954e251e318f7f784fe103b8ccfbf4fcfd0a07b63e78e4ed" - }, - { - "path": "src/dummy_metal_backend.rs", - "upstreamSha256": "c7b8f2574ec053142b1331ee993f54a6ab12f324fa4adf55f72e89d7569763c6", - "sha256": "c7b8f2574ec053142b1331ee993f54a6ab12f324fa4adf55f72e89d7569763c6" - }, - { - "path": "src/error.rs", - "upstreamSha256": "4423c8adff65933639b571ddae12acacd94d74eea0e93747dec917ec0b32464a", - "sha256": "4423c8adff65933639b571ddae12acacd94d74eea0e93747dec917ec0b32464a" - }, - { - "path": "src/indexer.rs", - "upstreamSha256": "1636ab33e7197eb11eff58f94b6bf6c19919680659a1c0265c9245ed95217779", - "sha256": "1636ab33e7197eb11eff58f94b6bf6c19919680659a1c0265c9245ed95217779" - }, - { - "path": "src/layout.rs", - "upstreamSha256": "b5de2481ae06d8e40b02a53fae66a3bc45e7f61de531cadc77d3b04cc796de4a", - "sha256": "b5de2481ae06d8e40b02a53fae66a3bc45e7f61de531cadc77d3b04cc796de4a" - }, - { - "path": "src/lib.rs", - "upstreamSha256": "b657d353d87d9b20cf65877260084fa4db4600fa9a2db910998ba440805e1923", - "sha256": "b657d353d87d9b20cf65877260084fa4db4600fa9a2db910998ba440805e1923" - }, - { - "path": "src/metal_backend.rs", - "upstreamSha256": "b2a6adfccaf0a7f88cf9b64c4cd67414be755a1fbf15e90d20facfdf15c297bc", - "sha256": "b2a6adfccaf0a7f88cf9b64c4cd67414be755a1fbf15e90d20facfdf15c297bc" - }, - { - "path": "src/mkl.rs", - "upstreamSha256": "c80564bd6139736e0e28336ba3130f386da963d62238eb45f2f1a15c506c2254", - "sha256": "c80564bd6139736e0e28336ba3130f386da963d62238eb45f2f1a15c506c2254" - }, - { - "path": "src/npy.rs", - "upstreamSha256": "b1c0cba48be9752eefe4ee940b37f477ea15650bc2e3257f2c98dd9a7c9fa2fb", - "sha256": "ae2eb805486716804a17003a2d0075c9db164968aaa2014cd4e0b6fbf5c45f19" - }, - { - "path": "src/op.rs", - "upstreamSha256": "170fb16bf46153a5c7fda99405a32b5b84a901a9fd6a8dcb5609c2230b842c03", - "sha256": "170fb16bf46153a5c7fda99405a32b5b84a901a9fd6a8dcb5609c2230b842c03" - }, - { - "path": "src/pickle.rs", - "upstreamSha256": "0ae3f2cb79983853437156fefa7a23ec9e42610d1e68fcb5e6b55b783b9ba84c", - "sha256": "70503eeb3f83ef0fce94989ebe39c062fc6087382bbe890acd18c820a4c7b73a" - }, - { - "path": "src/quantized/avx.rs", - "upstreamSha256": "24caf149e175e12ac4b55f94074f051518c94d85d3b6e749581e1241b15b5bf5", - "sha256": "24caf149e175e12ac4b55f94074f051518c94d85d3b6e749581e1241b15b5bf5" - }, - { - "path": "src/quantized/cuda.rs", - "upstreamSha256": "7083e8ee864ea7841c43732ed0a9019aab7824d859803d84e9c6e7a9ef93b197", - "sha256": "7083e8ee864ea7841c43732ed0a9019aab7824d859803d84e9c6e7a9ef93b197" - }, - { - "path": "src/quantized/dummy_cuda.rs", - "upstreamSha256": "b3d3fc2d28cb380fcb76703ad8ff47432456491b8ad97e65fbd19657ac90f3ad", - "sha256": "b3d3fc2d28cb380fcb76703ad8ff47432456491b8ad97e65fbd19657ac90f3ad" - }, - { - "path": "src/quantized/dummy_metal.rs", - "upstreamSha256": "fcdbac00595c9035874d28c4539aee6143629ee4bd8ca128ebe3614e87eb4bca", - "sha256": "fcdbac00595c9035874d28c4539aee6143629ee4bd8ca128ebe3614e87eb4bca" - }, - { - "path": "src/quantized/ggml_file.rs", - "upstreamSha256": "63153bc7242b8af7a24fa6e89fcbd634122d789aa35f8b631791a8e1308bc9d7", - "sha256": "63153bc7242b8af7a24fa6e89fcbd634122d789aa35f8b631791a8e1308bc9d7" - }, - { - "path": "src/quantized/gguf_file.rs", - "upstreamSha256": "4328832e0470f18d7cd0783ac9c94e8fbecce828111f706082f8837015625b37", - "sha256": "4328832e0470f18d7cd0783ac9c94e8fbecce828111f706082f8837015625b37" - }, - { - "path": "src/quantized/k_quants.rs", - "upstreamSha256": "1c5abc34ffdfb3aba16905b3c79b7f71242e2b2a927b67be39923b636ab81f30", - "sha256": "1c5abc34ffdfb3aba16905b3c79b7f71242e2b2a927b67be39923b636ab81f30" - }, - { - "path": "src/quantized/metal.rs", - "upstreamSha256": "f75fcb39ee6b13e0d6f99661b5e29e9df7b823ac601dfd70560bc1ec47c60272", - "sha256": "f75fcb39ee6b13e0d6f99661b5e29e9df7b823ac601dfd70560bc1ec47c60272" - }, - { - "path": "src/quantized/mod.rs", - "upstreamSha256": "9f4542e55e4e2297fdbcd3f46072f450a70b3851cba413425bfdb82ed96e9707", - "sha256": "9f4542e55e4e2297fdbcd3f46072f450a70b3851cba413425bfdb82ed96e9707" - }, - { - "path": "src/quantized/neon.rs", - "upstreamSha256": "e76ef3a2ae350cffe35764845481e6fbb1553da3cd34b500e7e6ecc127a41c70", - "sha256": "e76ef3a2ae350cffe35764845481e6fbb1553da3cd34b500e7e6ecc127a41c70" - }, - { - "path": "src/quantized/simd128.rs", - "upstreamSha256": "6afe4c9677961ddc5cbeaaeb3aa48a8af200134cea8d39e43c239e0df5d1d3d5", - "sha256": "6afe4c9677961ddc5cbeaaeb3aa48a8af200134cea8d39e43c239e0df5d1d3d5" - }, - { - "path": "src/quantized/utils.rs", - "upstreamSha256": "622e915fbcc13b03d827284fc802076b2955c53744ff6d7a9a881a4509d2f0c5", - "sha256": "622e915fbcc13b03d827284fc802076b2955c53744ff6d7a9a881a4509d2f0c5" - }, - { - "path": "src/safetensors.rs", - "upstreamSha256": "71a4bbfbf5b0ba5445df8c030b825dce30b3cf96de1f1e5c1e68c87aad2e046b", - "sha256": "71a4bbfbf5b0ba5445df8c030b825dce30b3cf96de1f1e5c1e68c87aad2e046b" - }, - { - "path": "src/scalar.rs", - "upstreamSha256": "26158f228b848a3b8fc61c86b37903175835c39a00986a6304b79dca31e9a3e9", - "sha256": "26158f228b848a3b8fc61c86b37903175835c39a00986a6304b79dca31e9a3e9" - }, - { - "path": "src/shape.rs", - "upstreamSha256": "76679ea3209dd844dc58f79f6686b1339bb59fa2558e9179d4f2e7f000973e8b", - "sha256": "76679ea3209dd844dc58f79f6686b1339bb59fa2558e9179d4f2e7f000973e8b" - }, - { - "path": "src/storage.rs", - "upstreamSha256": "b9916e7360569ab2807317eecd302e26e5f1e0bbca1f0e25f8cb2958c0fe21fc", - "sha256": "b9916e7360569ab2807317eecd302e26e5f1e0bbca1f0e25f8cb2958c0fe21fc" - }, - { - "path": "src/strided_index.rs", - "upstreamSha256": "86e85bdfc9d38c1e7d69935015079b4a1b6b546e5f444297e9f127a98241cfa4", - "sha256": "86e85bdfc9d38c1e7d69935015079b4a1b6b546e5f444297e9f127a98241cfa4" - }, - { - "path": "src/tensor.rs", - "upstreamSha256": "282929063ff6299e9a0feffc456c48a9a0869cd941a9de3e5d84a3550d1879ec", - "sha256": "282929063ff6299e9a0feffc456c48a9a0869cd941a9de3e5d84a3550d1879ec" - }, - { - "path": "src/test_utils.rs", - "upstreamSha256": "1f4483bb02ab7fdd90828b947d4d5029bbbe400387438dceea9ea6171841929d", - "sha256": "1f4483bb02ab7fdd90828b947d4d5029bbbe400387438dceea9ea6171841929d" - }, - { - "path": "src/utils.rs", - "upstreamSha256": "ba1c0946bcc179d4b7a2f7c74cefa0a212187272d9c6e245af20a51e73cdfa8a", - "sha256": "ba1c0946bcc179d4b7a2f7c74cefa0a212187272d9c6e245af20a51e73cdfa8a" - }, - { - "path": "src/variable.rs", - "upstreamSha256": "4872d2dff978932f73b03bd6721cf85d2b4944f3745f049699baadfb7ad9e9af", - "sha256": "4872d2dff978932f73b03bd6721cf85d2b4944f3745f049699baadfb7ad9e9af" - }, - { - "path": "tests/conv_tests.rs", - "upstreamSha256": "d286576393be58478de44d1ea78419d8fb0e6c7c6c9042a38043d706065e8fb2", - "sha256": "d286576393be58478de44d1ea78419d8fb0e6c7c6c9042a38043d706065e8fb2" - }, - { - "path": "tests/custom_op_tests.rs", - "upstreamSha256": "1b9db66297b8c19536206027e382f808d499e639a453a58e05cf1b9a54f32879", - "sha256": "1b9db66297b8c19536206027e382f808d499e639a453a58e05cf1b9a54f32879" - }, - { - "path": "tests/display_tests.rs", - "upstreamSha256": "cf59d2ee9954a639db5c43ade847d4629503b706282419fae8c6481790c41e9b", - "sha256": "cf59d2ee9954a639db5c43ade847d4629503b706282419fae8c6481790c41e9b" - }, - { - "path": "tests/fortran_tensor_3d.pth", - "upstreamSha256": "f5178bcd53eb6cb54c9a5e825492c717bf4e7bbadb9f0f3833b6c5c6764f7a38", - "sha256": "f5178bcd53eb6cb54c9a5e825492c717bf4e7bbadb9f0f3833b6c5c6764f7a38" - }, - { - "path": "tests/grad_tests.rs", - "upstreamSha256": "91a17f47c667766551f90175d01b79595dcc3c8fcde43e809ce48a9fbf65ebde", - "sha256": "91a17f47c667766551f90175d01b79595dcc3c8fcde43e809ce48a9fbf65ebde" - }, - { - "path": "tests/indexing_tests.rs", - "upstreamSha256": "3d9e855b36a3e0b8683777b3d13c6d34859773f47e5ff309d612cbfcafdd6be4", - "sha256": "3d9e855b36a3e0b8683777b3d13c6d34859773f47e5ff309d612cbfcafdd6be4" - }, - { - "path": "tests/layout_tests.rs", - "upstreamSha256": "7fb5e266dae460593c076067e733a010666328b9353d42e62cff395df41f9073", - "sha256": "7fb5e266dae460593c076067e733a010666328b9353d42e62cff395df41f9073" - }, - { - "path": "tests/npy.py", - "upstreamSha256": "e2547b01e13fc3807991286645dba5eea7c5b3d76e821be2cfa7822be7a2f818", - "sha256": "e2547b01e13fc3807991286645dba5eea7c5b3d76e821be2cfa7822be7a2f818" - }, - { - "path": "tests/pool_tests.rs", - "upstreamSha256": "61947694711e0fa576606cddd50df7099705de7741ce8395dba316189ae21129", - "sha256": "61947694711e0fa576606cddd50df7099705de7741ce8395dba316189ae21129" - }, - { - "path": "tests/pth.py", - "upstreamSha256": "7ef7b4686691b33173f376f22daa121e92c91bec172994afe3deebd53e77f0b7", - "sha256": "7ef7b4686691b33173f376f22daa121e92c91bec172994afe3deebd53e77f0b7" - }, - { - "path": "tests/pth_tests.rs", - "upstreamSha256": "08496d5434bfb2382497f1e5d7f83ebe149bd4030b06882912d457d579d011a8", - "sha256": "08496d5434bfb2382497f1e5d7f83ebe149bd4030b06882912d457d579d011a8" - }, - { - "path": "tests/quantized_tests.rs", - "upstreamSha256": "c4208384d84b3bab5398f05bf99ab02fdcbcf54ccc43a801a07deada1e1269b3", - "sha256": "c4208384d84b3bab5398f05bf99ab02fdcbcf54ccc43a801a07deada1e1269b3" - }, - { - "path": "tests/serialization_tests.rs", - "upstreamSha256": "623c851de37f9d6c5c8fc64da257cf1a3359274851d25b1637abc1d6fe29d48c", - "sha256": "623c851de37f9d6c5c8fc64da257cf1a3359274851d25b1637abc1d6fe29d48c" - }, - { - "path": "tests/tensor_tests.rs", - "upstreamSha256": "9f19a2046372d16316a8452db504241ba6145720362504bf3b48456c17307dab", - "sha256": "9f19a2046372d16316a8452db504241ba6145720362504bf3b48456c17307dab" - }, - { - "path": "tests/test.npy", - "upstreamSha256": "f314ddddf18999d4890f4fec8bb6539fc62b6f41861fa85e1c04c98a03097dda", - "sha256": "f314ddddf18999d4890f4fec8bb6539fc62b6f41861fa85e1c04c98a03097dda" - }, - { - "path": "tests/test.npz", - "upstreamSha256": "999497e0c309302915e4db2b16118ce097c68aac66c08058fac8db3d58c7809a", - "sha256": "999497e0c309302915e4db2b16118ce097c68aac66c08058fac8db3d58c7809a" - }, - { - "path": "tests/test.pt", - "upstreamSha256": "2bc164fa96f78ef8fcb8eab5fdae9b991efe946b9b63071fdf1769dba5d3fe25", - "sha256": "2bc164fa96f78ef8fcb8eab5fdae9b991efe946b9b63071fdf1769dba5d3fe25" - }, - { - "path": "tests/test_with_key.pt", - "upstreamSha256": "2775d95f17d2630e2534e3ae09462624955031407a67f0f3b52115c49601b747", - "sha256": "2775d95f17d2630e2534e3ae09462624955031407a67f0f3b52115c49601b747" - } - ] - }, - { - "package": "candle-core-0.9.2", - "archiveSha256": "c15b675b80d994b2eadb20a4bbe434eabeb454eac3ee5e2b4cf6f147ee9be091", - "changedFiles": [ - "Cargo.toml", - "src/npy.rs", - "src/pickle.rs" - ], - "files": [ - { - "path": "Cargo.toml", - "upstreamSha256": "fbb9960d0496ea8cb11aead7548a70af655af410d5e3f7f094882dd11738f4f9", - "sha256": "d22d8e98c28b136040aac1d306599911635da862a2b3ac2f895aa7b19d92cff3" - }, - { - "path": "Cargo.toml.orig", - "upstreamSha256": "b7d9510ca0933845e9f1acad1b5504aff030e34a6877cf8900bf6bd70ee0484d", - "sha256": "b7d9510ca0933845e9f1acad1b5504aff030e34a6877cf8900bf6bd70ee0484d" - }, - { - "path": "LICENSE", - "upstreamSha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4", - "sha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4" - }, - { - "path": "README.md", - "upstreamSha256": "9b00007b1d568125e67d6f2ff5c40acd8227dec4c62681cbc87775768a125200", - "sha256": "9b00007b1d568125e67d6f2ff5c40acd8227dec4c62681cbc87775768a125200" - }, - { - "path": "benches/bench_main.rs", - "upstreamSha256": "0228bb14b13ee0cd2d7e907d8e66017fa291d81d39316c183f4bf5aed0657518", - "sha256": "0228bb14b13ee0cd2d7e907d8e66017fa291d81d39316c183f4bf5aed0657518" - }, - { - "path": "benches/benchmarks/affine.rs", - "upstreamSha256": "260d6aeb9faf50808ac278dd93664dd24769bcd5c53333490d742cdff298844c", - "sha256": "260d6aeb9faf50808ac278dd93664dd24769bcd5c53333490d742cdff298844c" - }, - { - "path": "benches/benchmarks/binary.rs", - "upstreamSha256": "86335f2ca24005c215d590797cfde35c6a3bba21978b0cf184ac906c6f255e40", - "sha256": "86335f2ca24005c215d590797cfde35c6a3bba21978b0cf184ac906c6f255e40" - }, - { - "path": "benches/benchmarks/broadcast.rs", - "upstreamSha256": "9da8fde24dcbed559dfd44e7ec6607762ed2a7f80857a057f619838a4c723618", - "sha256": "9da8fde24dcbed559dfd44e7ec6607762ed2a7f80857a057f619838a4c723618" - }, - { - "path": "benches/benchmarks/conv_transpose2d.rs", - "upstreamSha256": "f57723066b6fc173d2957181e2e54a06505213303f82577b10d482983048981a", - "sha256": "f57723066b6fc173d2957181e2e54a06505213303f82577b10d482983048981a" - }, - { - "path": "benches/benchmarks/copy.rs", - "upstreamSha256": "406c04fa73a664a785dd7d9c6a4ad4d09b4c0c0b52072b8e429566485fd61938", - "sha256": "406c04fa73a664a785dd7d9c6a4ad4d09b4c0c0b52072b8e429566485fd61938" - }, - { - "path": "benches/benchmarks/matmul.rs", - "upstreamSha256": "8ceb3d0e312a82770f85414459009d0385043eb1fa384f8dd34476b84cf42bbb", - "sha256": "8ceb3d0e312a82770f85414459009d0385043eb1fa384f8dd34476b84cf42bbb" - }, - { - "path": "benches/benchmarks/mod.rs", - "upstreamSha256": "28ed8ad6b0e938364c62d64456dd98bd84faca4b038b7f3661ebe013555833ac", - "sha256": "28ed8ad6b0e938364c62d64456dd98bd84faca4b038b7f3661ebe013555833ac" - }, - { - "path": "benches/benchmarks/qmatmul.rs", - "upstreamSha256": "cd71f24bc0dbdd716f36fec4658a444cde64fdeacca1cdf63bbf1501ede12e40", - "sha256": "cd71f24bc0dbdd716f36fec4658a444cde64fdeacca1cdf63bbf1501ede12e40" - }, - { - "path": "benches/benchmarks/random.rs", - "upstreamSha256": "0f68d50a3da9f1096ab00c6a1141b58454b335604d6c2ff3d39af830a47b9138", - "sha256": "0f68d50a3da9f1096ab00c6a1141b58454b335604d6c2ff3d39af830a47b9138" - }, - { - "path": "benches/benchmarks/reduce.rs", - "upstreamSha256": "8a0ce9e9f10ca9d210db3c424325dd082ccc08fd288080f0b579ab1c7faa8fc6", - "sha256": "8a0ce9e9f10ca9d210db3c424325dd082ccc08fd288080f0b579ab1c7faa8fc6" - }, - { - "path": "benches/benchmarks/unary.rs", - "upstreamSha256": "40153b0fe906af2235e0a356719a208d82b5910853df93957f8e18bcb61bcef1", - "sha256": "40153b0fe906af2235e0a356719a208d82b5910853df93957f8e18bcb61bcef1" - }, - { - "path": "benches/benchmarks/where_cond.rs", - "upstreamSha256": "41be1e61510cfdfbdc74cbb523ec8756fad12eb53a3b6add8f032999e98370aa", - "sha256": "41be1e61510cfdfbdc74cbb523ec8756fad12eb53a3b6add8f032999e98370aa" - }, - { - "path": "examples/basics.rs", - "upstreamSha256": "6c85b5c338290f8d2eeeb548209fc4f4d4052fa4bfd2ea07846731cd9c36d88d", - "sha256": "6c85b5c338290f8d2eeeb548209fc4f4d4052fa4bfd2ea07846731cd9c36d88d" - }, - { - "path": "examples/cuda_basics.rs", - "upstreamSha256": "ec4aa80f89acf913ca74828fdf19ceb24808406794e98caa3f524dd23ce66cd4", - "sha256": "ec4aa80f89acf913ca74828fdf19ceb24808406794e98caa3f524dd23ce66cd4" - }, - { - "path": "examples/cuda_sum_benchmark.rs", - "upstreamSha256": "f53810234d3b90cbef0cc23c7ac1ad8141a9a93d1e86f7c358f5d32fcc5c76cc", - "sha256": "f53810234d3b90cbef0cc23c7ac1ad8141a9a93d1e86f7c358f5d32fcc5c76cc" - }, - { - "path": "examples/metal_basics.rs", - "upstreamSha256": "4f2cb274449cf1b6326b1ed2211f1076d55e621fcc81cdd501eca72e7203351e", - "sha256": "4f2cb274449cf1b6326b1ed2211f1076d55e621fcc81cdd501eca72e7203351e" - }, - { - "path": "src/accelerate.rs", - "upstreamSha256": "9be99929bb6553d9baf6b6a5a56fa66bfdbee122f76c4792a00e09ac8a0d638a", - "sha256": "9be99929bb6553d9baf6b6a5a56fa66bfdbee122f76c4792a00e09ac8a0d638a" - }, - { - "path": "src/backend.rs", - "upstreamSha256": "822a96a62ef7d300141b7d439807482bd40bded3f592e9e72c1e108f43ff63cc", - "sha256": "822a96a62ef7d300141b7d439807482bd40bded3f592e9e72c1e108f43ff63cc" - }, - { - "path": "src/backprop.rs", - "upstreamSha256": "4f370f420aaea68bf23976f807c9a47a97c36d46f17f18ef82132d43a9847a52", - "sha256": "4f370f420aaea68bf23976f807c9a47a97c36d46f17f18ef82132d43a9847a52" - }, - { - "path": "src/conv.rs", - "upstreamSha256": "31b8870d37dad9d2962985657121e85bc350830555d20054700921ea689a0d9f", - "sha256": "31b8870d37dad9d2962985657121e85bc350830555d20054700921ea689a0d9f" - }, - { - "path": "src/convert.rs", - "upstreamSha256": "664562f7fc98008b1cbf4815aacf79311141a40d9a003f86ecdb8cebcf6abbae", - "sha256": "664562f7fc98008b1cbf4815aacf79311141a40d9a003f86ecdb8cebcf6abbae" - }, - { - "path": "src/cpu/avx.rs", - "upstreamSha256": "d06f480b9ec3553568c0ef4b32e30d9f0dc7371f6db94473b42724f2b2519d45", - "sha256": "d06f480b9ec3553568c0ef4b32e30d9f0dc7371f6db94473b42724f2b2519d45" - }, - { - "path": "src/cpu/erf.rs", - "upstreamSha256": "e9d7e063b6e1b3af2b785b4b26ea038e512608a380edbb0d65d0253f0f70dbdc", - "sha256": "e9d7e063b6e1b3af2b785b4b26ea038e512608a380edbb0d65d0253f0f70dbdc" - }, - { - "path": "src/cpu/kernels.rs", - "upstreamSha256": "ad187fd28c7b491ae982e5e9c714331b89c91056c370f6a7417cc79dfda7aab1", - "sha256": "ad187fd28c7b491ae982e5e9c714331b89c91056c370f6a7417cc79dfda7aab1" - }, - { - "path": "src/cpu/mod.rs", - "upstreamSha256": "667e12b8807e5b73cb3b84f4078029e1060fc2b4147122584c24dc0c083c3738", - "sha256": "667e12b8807e5b73cb3b84f4078029e1060fc2b4147122584c24dc0c083c3738" - }, - { - "path": "src/cpu/neon.rs", - "upstreamSha256": "c180dffb8a342c2f026d8fdcc235c7d9e8f0aac8a5f6f03de4c2f4de8dda609c", - "sha256": "c180dffb8a342c2f026d8fdcc235c7d9e8f0aac8a5f6f03de4c2f4de8dda609c" - }, - { - "path": "src/cpu/simd128.rs", - "upstreamSha256": "8ed8cff924086325c4d4af5a12746a7ce4a5a05f18cdeaa28bc0ff9c64060906", - "sha256": "8ed8cff924086325c4d4af5a12746a7ce4a5a05f18cdeaa28bc0ff9c64060906" - }, - { - "path": "src/cpu_backend/conv2d.rs", - "upstreamSha256": "367f0c7e9380215879e6c95cffabbd9df76206d96716f8c8ccfdded2ac646bf1", - "sha256": "367f0c7e9380215879e6c95cffabbd9df76206d96716f8c8ccfdded2ac646bf1" - }, - { - "path": "src/cpu_backend/mod.rs", - "upstreamSha256": "93512c3d7c4ea7b1a0b74300eea658f0afb7d51850cec928a995ebeeacde66b5", - "sha256": "93512c3d7c4ea7b1a0b74300eea658f0afb7d51850cec928a995ebeeacde66b5" - }, - { - "path": "src/cpu_backend/utils.rs", - "upstreamSha256": "85b05a7f3d6934a115f72f1caf977d7bbdd680715939fb3bdfbbf24af5ac15b1", - "sha256": "85b05a7f3d6934a115f72f1caf977d7bbdd680715939fb3bdfbbf24af5ac15b1" - }, - { - "path": "src/cuda_backend/cudnn.rs", - "upstreamSha256": "3e3237559a6148567a369effe6e9616bbc12a6b67c13c09b2d52e6f6382f04b8", - "sha256": "3e3237559a6148567a369effe6e9616bbc12a6b67c13c09b2d52e6f6382f04b8" - }, - { - "path": "src/cuda_backend/device.rs", - "upstreamSha256": "40f4bb9512f9001a983802129b6cfee16381a24b7876f13ba02cd49b6dfe1925", - "sha256": "40f4bb9512f9001a983802129b6cfee16381a24b7876f13ba02cd49b6dfe1925" - }, - { - "path": "src/cuda_backend/error.rs", - "upstreamSha256": "b60bce29f50ce640fb1932009bd447f0ff7fc0e293900befb05a9d8df0e43413", - "sha256": "b60bce29f50ce640fb1932009bd447f0ff7fc0e293900befb05a9d8df0e43413" - }, - { - "path": "src/cuda_backend/mod.rs", - "upstreamSha256": "bc2bd5fe3702bfccb49d64af07dc4197013d6617c720643cd2a14b718f4134e2", - "sha256": "bc2bd5fe3702bfccb49d64af07dc4197013d6617c720643cd2a14b718f4134e2" - }, - { - "path": "src/cuda_backend/utils.rs", - "upstreamSha256": "24ceb7c9e50389a911e9aaf4d4f01e92ca3f2a06743adfa72981ba870a0574d9", - "sha256": "24ceb7c9e50389a911e9aaf4d4f01e92ca3f2a06743adfa72981ba870a0574d9" - }, - { - "path": "src/custom_op.rs", - "upstreamSha256": "c0a4953dad882f3a392070fa6bbabeaa582d2576d50bf349e12a97d680161ff1", - "sha256": "c0a4953dad882f3a392070fa6bbabeaa582d2576d50bf349e12a97d680161ff1" - }, - { - "path": "src/device.rs", - "upstreamSha256": "0929b7e72e83dea86a40f43f55a61eb2ca0dcb2d1c3c2d6e698bd18cf801754b", - "sha256": "0929b7e72e83dea86a40f43f55a61eb2ca0dcb2d1c3c2d6e698bd18cf801754b" - }, - { - "path": "src/display.rs", - "upstreamSha256": "17d38758a849310137f69069a0806224b9fdb682c0b44196b6d5885327515211", - "sha256": "17d38758a849310137f69069a0806224b9fdb682c0b44196b6d5885327515211" - }, - { - "path": "src/dtype.rs", - "upstreamSha256": "7aa09f4fb008d8ad5c2bbc83cfcd4bbcaab8f5e351d5f4ba6b93e7a1f1f2fe76", - "sha256": "7aa09f4fb008d8ad5c2bbc83cfcd4bbcaab8f5e351d5f4ba6b93e7a1f1f2fe76" - }, - { - "path": "src/dummy_cuda_backend.rs", - "upstreamSha256": "842836efb0c1ca8097d1a5c11140207f185b7f877603e98091afdf7d494fee47", - "sha256": "842836efb0c1ca8097d1a5c11140207f185b7f877603e98091afdf7d494fee47" - }, - { - "path": "src/dummy_dtype.rs", - "upstreamSha256": "50bcae6eeba464291a8d85020e967056b42cc7199658999de7159d0c4e8190e0", - "sha256": "50bcae6eeba464291a8d85020e967056b42cc7199658999de7159d0c4e8190e0" - }, - { - "path": "src/dummy_metal_backend.rs", - "upstreamSha256": "e59c40e1aea0210adac21316af7b7e9b838d8357da9492045f530a139d35acd7", - "sha256": "e59c40e1aea0210adac21316af7b7e9b838d8357da9492045f530a139d35acd7" - }, - { - "path": "src/error.rs", - "upstreamSha256": "e6605678f4093e0867e73a5579aa8d046d13d65bd609d5851edf42d44051927d", - "sha256": "e6605678f4093e0867e73a5579aa8d046d13d65bd609d5851edf42d44051927d" - }, - { - "path": "src/indexer.rs", - "upstreamSha256": "2df921d3e8284540a34c197b692b384ed5d6a25047d3685535f01c227718a3e0", - "sha256": "2df921d3e8284540a34c197b692b384ed5d6a25047d3685535f01c227718a3e0" - }, - { - "path": "src/layout.rs", - "upstreamSha256": "1bf7fca1aa8759cfe4663a63ed9cce5dcc7ae8da84fc5a2e043697a279980db7", - "sha256": "1bf7fca1aa8759cfe4663a63ed9cce5dcc7ae8da84fc5a2e043697a279980db7" - }, - { - "path": "src/lib.rs", - "upstreamSha256": "a7d035c1a1153b3acdee0c9378f5d2fbd14eb2e16e9e3f2fb27db687030640ab", - "sha256": "a7d035c1a1153b3acdee0c9378f5d2fbd14eb2e16e9e3f2fb27db687030640ab" - }, - { - "path": "src/metal_backend/device.rs", - "upstreamSha256": "3ca33c5b6eec88fa8893f14726bd60ed259c60f42b7646464b375f8c9ff2434f", - "sha256": "3ca33c5b6eec88fa8893f14726bd60ed259c60f42b7646464b375f8c9ff2434f" - }, - { - "path": "src/metal_backend/mod.rs", - "upstreamSha256": "16bc99ee096c581eaa83ce6cf9d007674d4eb6bb5b6e30560ccb62a3daac07a1", - "sha256": "16bc99ee096c581eaa83ce6cf9d007674d4eb6bb5b6e30560ccb62a3daac07a1" - }, - { - "path": "src/mkl.rs", - "upstreamSha256": "c80564bd6139736e0e28336ba3130f386da963d62238eb45f2f1a15c506c2254", - "sha256": "c80564bd6139736e0e28336ba3130f386da963d62238eb45f2f1a15c506c2254" - }, - { - "path": "src/npy.rs", - "upstreamSha256": "e53194fd98b92c5c2bc4948cc39bcd59e63b3b477d550c2a8e4bc9c66db67bca", - "sha256": "f638519dc260eb834cdbce9db7f6b3e570bf7589bf40bf6dca055f35b4ceb86e" - }, - { - "path": "src/op.rs", - "upstreamSha256": "8787d8e941217e500b7f653aa4252c41a0e15459c31f7373d0dc896fce4c4521", - "sha256": "8787d8e941217e500b7f653aa4252c41a0e15459c31f7373d0dc896fce4c4521" - }, - { - "path": "src/pickle.rs", - "upstreamSha256": "7fcb931fb616c35c7817e5461cc6173d7c6635339223c360a6c22fb077c07d63", - "sha256": "3b98dee96ce63c91da0644b2d63040e924b92bb17a49a3f15ade096ce2d49a91" - }, - { - "path": "src/quantized/avx.rs", - "upstreamSha256": "69cb5408571eb64812cb6386f0e24a7ff39fdc16482d44ee9f31fb40e536665c", - "sha256": "69cb5408571eb64812cb6386f0e24a7ff39fdc16482d44ee9f31fb40e536665c" - }, - { - "path": "src/quantized/cuda.rs", - "upstreamSha256": "a085e61faf3d383fe332775673d691b585cabcb12701351a75839d9ebb8196c5", - "sha256": "a085e61faf3d383fe332775673d691b585cabcb12701351a75839d9ebb8196c5" - }, - { - "path": "src/quantized/dummy_cuda.rs", - "upstreamSha256": "534ff012566fea238b247715c278352fa1860202ea9c8db32636469769f6dd97", - "sha256": "534ff012566fea238b247715c278352fa1860202ea9c8db32636469769f6dd97" - }, - { - "path": "src/quantized/dummy_metal.rs", - "upstreamSha256": "a9c727095f509046d1a50d131eee1529f9aa97ab306ceb1a44a1e827ea10109d", - "sha256": "a9c727095f509046d1a50d131eee1529f9aa97ab306ceb1a44a1e827ea10109d" - }, - { - "path": "src/quantized/ggml_file.rs", - "upstreamSha256": "dfc48f822e4393bc836efcf782f4a6bb36168ac9d641324a054f13ee862aadac", - "sha256": "dfc48f822e4393bc836efcf782f4a6bb36168ac9d641324a054f13ee862aadac" - }, - { - "path": "src/quantized/gguf_file.rs", - "upstreamSha256": "a263c9076e3e233457cc3d7780e3240c6448c7b5b8c517e480353691be5ff186", - "sha256": "a263c9076e3e233457cc3d7780e3240c6448c7b5b8c517e480353691be5ff186" - }, - { - "path": "src/quantized/imatrix_file.rs", - "upstreamSha256": "4ad710e64a0c9d3e5c2be1f6ca0a760035babb66c5fb91774acd1d7a30288d66", - "sha256": "4ad710e64a0c9d3e5c2be1f6ca0a760035babb66c5fb91774acd1d7a30288d66" - }, - { - "path": "src/quantized/k_quants.rs", - "upstreamSha256": "736a4b74c26c8c3be2917ec4c4ca88cf87e892c6b3278f5a466726bdf84f21b1", - "sha256": "736a4b74c26c8c3be2917ec4c4ca88cf87e892c6b3278f5a466726bdf84f21b1" - }, - { - "path": "src/quantized/metal.rs", - "upstreamSha256": "cacf5336009a8f83e13fa32a3f1c5e28ab5a5d52fc91d9ec4ddc265a1bcc7eb4", - "sha256": "cacf5336009a8f83e13fa32a3f1c5e28ab5a5d52fc91d9ec4ddc265a1bcc7eb4" - }, - { - "path": "src/quantized/mod.rs", - "upstreamSha256": "82fe93c2b9c0625c9b7147a60e977acaff841a7e4e139f631e6c06fc5eb0e8a0", - "sha256": "82fe93c2b9c0625c9b7147a60e977acaff841a7e4e139f631e6c06fc5eb0e8a0" - }, - { - "path": "src/quantized/neon.rs", - "upstreamSha256": "2a46cd5cf8f296c3673f61380bac49fd98232472ae5db60dc7babc788fdd0be7", - "sha256": "2a46cd5cf8f296c3673f61380bac49fd98232472ae5db60dc7babc788fdd0be7" - }, - { - "path": "src/quantized/simd128.rs", - "upstreamSha256": "5c8000c6dea9803de8befde33b15970c1b1dcbf83b914c6261a3e9c2b658c1da", - "sha256": "5c8000c6dea9803de8befde33b15970c1b1dcbf83b914c6261a3e9c2b658c1da" - }, - { - "path": "src/quantized/utils.rs", - "upstreamSha256": "33a4ff45fafb4baecbd6b95f68fd373d447283be19e38e63f1f4da3434f3a47b", - "sha256": "33a4ff45fafb4baecbd6b95f68fd373d447283be19e38e63f1f4da3434f3a47b" - }, - { - "path": "src/safetensors.rs", - "upstreamSha256": "1259250e0899d842d34e0a2495c414fb63efdee0d78e74092a47648c5380bfb3", - "sha256": "1259250e0899d842d34e0a2495c414fb63efdee0d78e74092a47648c5380bfb3" - }, - { - "path": "src/scalar.rs", - "upstreamSha256": "d98ca484e6fd0580f2b8bb2c3c43283671496042aa9d58115829052532d6200c", - "sha256": "d98ca484e6fd0580f2b8bb2c3c43283671496042aa9d58115829052532d6200c" - }, - { - "path": "src/shape.rs", - "upstreamSha256": "0fb3b21b63f348efcc18222cecbc3b5ea510d14c50f728c9471a30e24901ba74", - "sha256": "0fb3b21b63f348efcc18222cecbc3b5ea510d14c50f728c9471a30e24901ba74" - }, - { - "path": "src/sort.rs", - "upstreamSha256": "4c1263ab7b6f80a1acd05813a4afd4ef2aa98d44378458829dfc403bccfdbb92", - "sha256": "4c1263ab7b6f80a1acd05813a4afd4ef2aa98d44378458829dfc403bccfdbb92" - }, - { - "path": "src/storage.rs", - "upstreamSha256": "6241271357ee152f0244e47053f40de811832fca89e8aece6420bbc1b5c485c4", - "sha256": "6241271357ee152f0244e47053f40de811832fca89e8aece6420bbc1b5c485c4" - }, - { - "path": "src/streaming.rs", - "upstreamSha256": "abd0de65d894b617e6e3fb0c0787f220582bcb1f8eb9341c33bb74aaa23bf165", - "sha256": "abd0de65d894b617e6e3fb0c0787f220582bcb1f8eb9341c33bb74aaa23bf165" - }, - { - "path": "src/strided_index.rs", - "upstreamSha256": "b4884a6ad8d6fcb043b69b47b8203a035f40616057677802df32c4ee3799445b", - "sha256": "b4884a6ad8d6fcb043b69b47b8203a035f40616057677802df32c4ee3799445b" - }, - { - "path": "src/tensor.rs", - "upstreamSha256": "25b95a8deecffa87d55325d9ae258b9552778a7eb24a653b7c1438340aa21487", - "sha256": "25b95a8deecffa87d55325d9ae258b9552778a7eb24a653b7c1438340aa21487" - }, - { - "path": "src/tensor_cat.rs", - "upstreamSha256": "8c97ffbac8ca2fef78d5d1440e05df7915e6b1c2ae34ef891a36e6a6e989f497", - "sha256": "8c97ffbac8ca2fef78d5d1440e05df7915e6b1c2ae34ef891a36e6a6e989f497" - }, - { - "path": "src/test_utils.rs", - "upstreamSha256": "b3eec41e2864344b9e0a0eec4f0be70f8273617b99684d41315809530327debb", - "sha256": "b3eec41e2864344b9e0a0eec4f0be70f8273617b99684d41315809530327debb" - }, - { - "path": "src/utils.rs", - "upstreamSha256": "bdfc884fb12a14eb2b5a0dfafebd7951596ac9361f97c1cd93c4284507eec9e6", - "sha256": "bdfc884fb12a14eb2b5a0dfafebd7951596ac9361f97c1cd93c4284507eec9e6" - }, - { - "path": "src/variable.rs", - "upstreamSha256": "b9b10b5d2d4a81d1ab920f9337e556a634bae7e484a0b04ea855f9a903fc87ec", - "sha256": "b9b10b5d2d4a81d1ab920f9337e556a634bae7e484a0b04ea855f9a903fc87ec" - }, - { - "path": "tests/bilinear_tests.rs", - "upstreamSha256": "6844aa75cbe436d753391e3839619adca77a1f83d6ae78957261181fabbefe9a", - "sha256": "6844aa75cbe436d753391e3839619adca77a1f83d6ae78957261181fabbefe9a" - }, - { - "path": "tests/conv_tests.rs", - "upstreamSha256": "f303e81c201c3b450d95b3b437d128ee4cea4344c3aa31f3d2f09b81d3e9ad8d", - "sha256": "f303e81c201c3b450d95b3b437d128ee4cea4344c3aa31f3d2f09b81d3e9ad8d" - }, - { - "path": "tests/custom_op_tests.rs", - "upstreamSha256": "fa13ae08edf71d1c15a0a7509652367ee53a5960b722d7ee503494455fa4173d", - "sha256": "fa13ae08edf71d1c15a0a7509652367ee53a5960b722d7ee503494455fa4173d" - }, - { - "path": "tests/display_tests.rs", - "upstreamSha256": "cf59d2ee9954a639db5c43ade847d4629503b706282419fae8c6481790c41e9b", - "sha256": "cf59d2ee9954a639db5c43ade847d4629503b706282419fae8c6481790c41e9b" - }, - { - "path": "tests/fortran_tensor_3d.pth", - "upstreamSha256": "f5178bcd53eb6cb54c9a5e825492c717bf4e7bbadb9f0f3833b6c5c6764f7a38", - "sha256": "f5178bcd53eb6cb54c9a5e825492c717bf4e7bbadb9f0f3833b6c5c6764f7a38" - }, - { - "path": "tests/grad_tests.rs", - "upstreamSha256": "f7fc99b411d75c914532236ee1d5ba9f783dc6a4165d40e9145d4110fe7a21c8", - "sha256": "f7fc99b411d75c914532236ee1d5ba9f783dc6a4165d40e9145d4110fe7a21c8" - }, - { - "path": "tests/indexing_tests.rs", - "upstreamSha256": "3d9e855b36a3e0b8683777b3d13c6d34859773f47e5ff309d612cbfcafdd6be4", - "sha256": "3d9e855b36a3e0b8683777b3d13c6d34859773f47e5ff309d612cbfcafdd6be4" - }, - { - "path": "tests/layout_tests.rs", - "upstreamSha256": "25e6347e2c647838331254ad21d437f6ab59bc26dd98110c696d365f05daa18e", - "sha256": "25e6347e2c647838331254ad21d437f6ab59bc26dd98110c696d365f05daa18e" - }, - { - "path": "tests/matmul_tests.rs", - "upstreamSha256": "7499f96b1d2343ab919076f57f6907f85391af5c2e55d057673fb24ea59a061e", - "sha256": "7499f96b1d2343ab919076f57f6907f85391af5c2e55d057673fb24ea59a061e" - }, - { - "path": "tests/npy.py", - "upstreamSha256": "e2547b01e13fc3807991286645dba5eea7c5b3d76e821be2cfa7822be7a2f818", - "sha256": "e2547b01e13fc3807991286645dba5eea7c5b3d76e821be2cfa7822be7a2f818" - }, - { - "path": "tests/pool_tests.rs", - "upstreamSha256": "b574c67ce8486578d55b3f27070f83bf1c1137173b9d3762d5cd2e7d25c9f8b3", - "sha256": "b574c67ce8486578d55b3f27070f83bf1c1137173b9d3762d5cd2e7d25c9f8b3" - }, - { - "path": "tests/pth.py", - "upstreamSha256": "7ef7b4686691b33173f376f22daa121e92c91bec172994afe3deebd53e77f0b7", - "sha256": "7ef7b4686691b33173f376f22daa121e92c91bec172994afe3deebd53e77f0b7" - }, - { - "path": "tests/pth_tests.rs", - "upstreamSha256": "32440ce8dcb64ddf469e7cf122f4d19aa0ae484650c20d127d147fb99052fa35", - "sha256": "32440ce8dcb64ddf469e7cf122f4d19aa0ae484650c20d127d147fb99052fa35" - }, - { - "path": "tests/quantized_tests.rs", - "upstreamSha256": "908ffb8070f75f123edcf7a19586e9c6172bd749b670a1fb53f3bfd523bceb34", - "sha256": "908ffb8070f75f123edcf7a19586e9c6172bd749b670a1fb53f3bfd523bceb34" - }, - { - "path": "tests/serialization_tests.rs", - "upstreamSha256": "98db2da64586ec61234767421a792e712f2c072fe2382b1ff589c771b03b7d57", - "sha256": "98db2da64586ec61234767421a792e712f2c072fe2382b1ff589c771b03b7d57" - }, - { - "path": "tests/tensor_tests.rs", - "upstreamSha256": "8ce0f9b1406a9c30a1d2600dc88887875fec89b3c87e4eea21c971535b4bc261", - "sha256": "8ce0f9b1406a9c30a1d2600dc88887875fec89b3c87e4eea21c971535b4bc261" - }, - { - "path": "tests/test.npy", - "upstreamSha256": "f314ddddf18999d4890f4fec8bb6539fc62b6f41861fa85e1c04c98a03097dda", - "sha256": "f314ddddf18999d4890f4fec8bb6539fc62b6f41861fa85e1c04c98a03097dda" - }, - { - "path": "tests/test.npz", - "upstreamSha256": "999497e0c309302915e4db2b16118ce097c68aac66c08058fac8db3d58c7809a", - "sha256": "999497e0c309302915e4db2b16118ce097c68aac66c08058fac8db3d58c7809a" - }, - { - "path": "tests/test.pt", - "upstreamSha256": "2bc164fa96f78ef8fcb8eab5fdae9b991efe946b9b63071fdf1769dba5d3fe25", - "sha256": "2bc164fa96f78ef8fcb8eab5fdae9b991efe946b9b63071fdf1769dba5d3fe25" - }, - { - "path": "tests/test_with_key.pt", - "upstreamSha256": "2775d95f17d2630e2534e3ae09462624955031407a67f0f3b52115c49601b747", - "sha256": "2775d95f17d2630e2534e3ae09462624955031407a67f0f3b52115c49601b747" - } - ] - }, - { - "package": "tch-0.24.0", - "archiveSha256": "0d3f84a069d8ba16dbf720b61e8bf131d90ffb8e958a664eae8e4993c5c2fa6f", - "changedFiles": [ - "Cargo.toml", - "src/tensor/npy.rs" - ], - "files": [ - { - "path": ".cargo/config.toml", - "upstreamSha256": "c5d69e9af7086ce85ae357f712437ede39c9c95a4348af8eec5931d2e16df9fa", - "sha256": "c5d69e9af7086ce85ae357f712437ede39c9c95a4348af8eec5931d2e16df9fa" - }, - { - "path": ".github/workflows/rust-ci.yml", - "upstreamSha256": "813732a2cbae7f5d4b1e471e3c3ac4a76a88c3aac6e70503dc866104aa806516", - "sha256": "813732a2cbae7f5d4b1e471e3c3ac4a76a88c3aac6e70503dc866104aa806516" - }, - { - "path": ".gitignore", - "upstreamSha256": "12b8a657f57a9113bfa5d6334d53a75883b18a4b5268cb0f0c27c10390acc68c", - "sha256": "12b8a657f57a9113bfa5d6334d53a75883b18a4b5268cb0f0c27c10390acc68c" - }, - { - "path": ".python-version", - "upstreamSha256": "7b55f8e67b5623c4bef3fa691288da9437d79d3aba156de48d481db32ac7d16d", - "sha256": "7b55f8e67b5623c4bef3fa691288da9437d79d3aba156de48d481db32ac7d16d" - }, - { - "path": "CHANGELOG.md", - "upstreamSha256": "d58a09e672d3e0f9019fd241037a646c969242af316a7633475a664b98fad9e0", - "sha256": "d58a09e672d3e0f9019fd241037a646c969242af316a7633475a664b98fad9e0" - }, - { - "path": "Cargo.toml", - "upstreamSha256": "adbef029a0a2df62d5b355d1261a989cb94c58fabb0adfabbf71f7daa12c56a2", - "sha256": "e2abaea0e6fc4018ebe60e48d416dfbfd3f5f0ef42cc6cfe98cba1e7b800c47f" - }, - { - "path": "Cargo.toml.orig", - "upstreamSha256": "510cefeb66e0aeb030fa23912084d3e468d547f863a65b45631980b7ecb1842d", - "sha256": "510cefeb66e0aeb030fa23912084d3e468d547f863a65b45631980b7ecb1842d" - }, - { - "path": "LICENSE-APACHE", - "upstreamSha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4", - "sha256": "c71d239df91726fc519c6eb72d318ec65820627232b2f796219e87dcf35d0ab4" - }, - { - "path": "LICENSE-MIT", - "upstreamSha256": "23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3", - "sha256": "23f18e03dc49df91622fe2a76176497404e46ced8a715d9d2b67a7446571cca3" - }, - { - "path": "Makefile", - "upstreamSha256": "7697845148dd4145ffa3945b4a0ec4f5ec640707742def245caff108ef4826b9", - "sha256": "7697845148dd4145ffa3945b4a0ec4f5ec640707742def245caff108ef4826b9" - }, - { - "path": "README.md", - "upstreamSha256": "7de992bd0004550851085eca2c326d3caa68d768fec8fdf614893cb6c9ddfe84", - "sha256": "7de992bd0004550851085eca2c326d3caa68d768fec8fdf614893cb6c9ddfe84" - }, - { - "path": "build.rs", - "upstreamSha256": "c3485073dba6a1b483b781e9699852f4fcfbe7410a5502521b2d4e985fa79250", - "sha256": "c3485073dba6a1b483b781e9699852f4fcfbe7410a5502521b2d4e985fa79250" - }, - { - "path": "clippy.toml", - "upstreamSha256": "7e0750989c182ad8742564a5f3862174984c140e373cf904f8b0c2079dfe2f85", - "sha256": "7e0750989c182ad8742564a5f3862174984c140e373cf904f8b0c2079dfe2f85" - }, - { - "path": "dune-project", - "upstreamSha256": "4d3262e21b391caa29d21ee71b946c3d7c213680169b6e953feee12aca082666", - "sha256": "4d3262e21b391caa29d21ee71b946c3d7c213680169b6e953feee12aca082666" - }, - { - "path": "examples/basics.rs", - "upstreamSha256": "5059e17e83429de3e04f2937f6053032b8ac2e876b2b1d1e5230d952d06dff95", - "sha256": "5059e17e83429de3e04f2937f6053032b8ac2e876b2b1d1e5230d952d06dff95" - }, - { - "path": "examples/char-rnn/README.md", - "upstreamSha256": "e5c0121e55a3d0c71615329c096de82d177281bc97baafab3f30dcf733aa9763", - "sha256": "e5c0121e55a3d0c71615329c096de82d177281bc97baafab3f30dcf733aa9763" - }, - { - "path": "examples/char-rnn/main.rs", - "upstreamSha256": "cfb75afc5557d11a9ba595eff5e34dbfe3c5d9fbf690349e3b51d886af3df40d", - "sha256": "cfb75afc5557d11a9ba595eff5e34dbfe3c5d9fbf690349e3b51d886af3df40d" - }, - { - "path": "examples/cifar/main.rs", - "upstreamSha256": "a4ee823bb1d8e6faac1a4e38221e849b5dc702f9fb6d9b1f3b8ebeccb6bb3991", - "sha256": "a4ee823bb1d8e6faac1a4e38221e849b5dc702f9fb6d9b1f3b8ebeccb6bb3991" - }, - { - "path": "examples/custom-optimizer/README.md", - "upstreamSha256": "d05debefba2b5ba3600cc5496f669b3a916effe43aaf0b95c3420b1551f123c6", - "sha256": "d05debefba2b5ba3600cc5496f669b3a916effe43aaf0b95c3420b1551f123c6" - }, - { - "path": "examples/custom-optimizer/main.rs", - "upstreamSha256": "4927a02258f46a1b24e62eac1e44591aa6efa8ace7d291b167f7b29416e68646", - "sha256": "4927a02258f46a1b24e62eac1e44591aa6efa8ace7d291b167f7b29416e68646" - }, - { - "path": "examples/custom-optimizer/sparse_adam.rs", - "upstreamSha256": "fc00de4d91983408f1a263a9a0295043e5df7c53cabbca4a4ea06f646eaa97f5", - "sha256": "fc00de4d91983408f1a263a9a0295043e5df7c53cabbca4a4ea06f646eaa97f5" - }, - { - "path": "examples/gan/main.rs", - "upstreamSha256": "5cf6987046c7f1a32c8ea61528438311e1f8a6003efaf038e95ee87d5ffc96c2", - "sha256": "5cf6987046c7f1a32c8ea61528438311e1f8a6003efaf038e95ee87d5ffc96c2" - }, - { - "path": "examples/jit/README.md", - "upstreamSha256": "5e3dfe6c37d99face010555ddbee4aeaf86326ffda33834bde6bb823a6673fbe", - "sha256": "5e3dfe6c37d99face010555ddbee4aeaf86326ffda33834bde6bb823a6673fbe" - }, - { - "path": "examples/jit/main.rs", - "upstreamSha256": "f95f8c7f8e17d2d48f546723c9dcd347504f7e0af56d88dd8968d3be3e479e1d", - "sha256": "f95f8c7f8e17d2d48f546723c9dcd347504f7e0af56d88dd8968d3be3e479e1d" - }, - { - "path": "examples/jit/resnet.py", - "upstreamSha256": "d6e57912e48abce72ef364b781d87bb0b0d367f800cffe4a84b7597f908d78cf", - "sha256": "d6e57912e48abce72ef364b781d87bb0b0d367f800cffe4a84b7597f908d78cf" - }, - { - "path": "examples/jit-quantized/README.md", - "upstreamSha256": "f051686f643f3df136667a69a418f9d3d9455d66307b3ffc62c307c17feca810", - "sha256": "f051686f643f3df136667a69a418f9d3d9455d66307b3ffc62c307c17feca810" - }, - { - "path": "examples/jit-quantized/main.rs", - "upstreamSha256": "39f0e44f0fe0261a57b720e8dcaf270b16c0cc8e97bb407951f538cff9b0b4d7", - "sha256": "39f0e44f0fe0261a57b720e8dcaf270b16c0cc8e97bb407951f538cff9b0b4d7" - }, - { - "path": "examples/jit-quantized/resnet.py", - "upstreamSha256": "b05fecadefb77685721373bf421e4753b5c6037a2e48364859f9c96f71a50cfc", - "sha256": "b05fecadefb77685721373bf421e4753b5c6037a2e48364859f9c96f71a50cfc" - }, - { - "path": "examples/jit-trace/README.md", - "upstreamSha256": "fc0de268c396414553f3c828fc607769e441751bf48d9a63058e2a4b13028c9b", - "sha256": "fc0de268c396414553f3c828fc607769e441751bf48d9a63058e2a4b13028c9b" - }, - { - "path": "examples/jit-trace/main.rs", - "upstreamSha256": "dc4ebf024747929f4f69eead423ff02ecdb96996fa21d11566fac048e7cf88d2", - "sha256": "dc4ebf024747929f4f69eead423ff02ecdb96996fa21d11566fac048e7cf88d2" - }, - { - "path": "examples/jit-trace/test.py", - "upstreamSha256": "a9f5c1940e886695b8eeb17344ae7c1beb0a008e9a638fd5206928bc4ca0b272", - "sha256": "a9f5c1940e886695b8eeb17344ae7c1beb0a008e9a638fd5206928bc4ca0b272" - }, - { - "path": "examples/jit-train/README.md", - "upstreamSha256": "12c99ed805300fe8d94bb8ff25a8cb85b4d97a623546f5281057e43328945832", - "sha256": "12c99ed805300fe8d94bb8ff25a8cb85b4d97a623546f5281057e43328945832" - }, - { - "path": "examples/jit-train/main.rs", - "upstreamSha256": "aaa2677aa66d63836d2115fa2e1d1db656503997b95fb6af58baf204e1ab6278", - "sha256": "aaa2677aa66d63836d2115fa2e1d1db656503997b95fb6af58baf204e1ab6278" - }, - { - "path": "examples/jit-train/resnet.py", - "upstreamSha256": "4c1fba01663d8cf36fc18b98ba478049bfc945a26e7fd9e89369ea7db297a6f7", - "sha256": "4c1fba01663d8cf36fc18b98ba478049bfc945a26e7fd9e89369ea7db297a6f7" - }, - { - "path": "examples/llama/convert_checkpoint.py", - "upstreamSha256": "16406a2e91c95ba5ee36690fe89cfc09020f011b0b6c2ee5e8e9bc8f177869ff", - "sha256": "16406a2e91c95ba5ee36690fe89cfc09020f011b0b6c2ee5e8e9bc8f177869ff" - }, - { - "path": "examples/llama/main.rs", - "upstreamSha256": "bc295bfe11070eb5f9d17e9fd077b4ea99b6388018526a1bf2fc3fd71052b418", - "sha256": "bc295bfe11070eb5f9d17e9fd077b4ea99b6388018526a1bf2fc3fd71052b418" - }, - { - "path": "examples/llama/sentencepiece.rs", - "upstreamSha256": "0df84fe132840454f6c20ef5db0679409f4081d294a8bdcd027637f2fd7dc6d9", - "sha256": "0df84fe132840454f6c20ef5db0679409f4081d294a8bdcd027637f2fd7dc6d9" - }, - { - "path": "examples/memory_test.rs", - "upstreamSha256": "5a7c82419512ebc1e484155a94b0ad0b254f53aaf6d3b2b5f04e41d886a10219", - "sha256": "5a7c82419512ebc1e484155a94b0ad0b254f53aaf6d3b2b5f04e41d886a10219" - }, - { - "path": "examples/min-gpt/README.md", - "upstreamSha256": "6324404f9fc300b1b4050d8f10cf891ccd92ab19b021ead52d713f40768bbaa8", - "sha256": "6324404f9fc300b1b4050d8f10cf891ccd92ab19b021ead52d713f40768bbaa8" - }, - { - "path": "examples/min-gpt/main.rs", - "upstreamSha256": "f493826895f9f804e56c766c1f5b80a1f300b17a21e9f6c9df049a39ea686fa4", - "sha256": "f493826895f9f804e56c766c1f5b80a1f300b17a21e9f6c9df049a39ea686fa4" - }, - { - "path": "examples/mnist/README.md", - "upstreamSha256": "c0e42d7b5df9a263ae99e90ce90a1aefe67d3b10d6d406e251460265106ad8f7", - "sha256": "c0e42d7b5df9a263ae99e90ce90a1aefe67d3b10d6d406e251460265106ad8f7" - }, - { - "path": "examples/mnist/main.rs", - "upstreamSha256": "0aecfe7aca1920ec90901f87474612bff5aa72a6877793cfb25f5b2f08de1b6e", - "sha256": "0aecfe7aca1920ec90901f87474612bff5aa72a6877793cfb25f5b2f08de1b6e" - }, - { - "path": "examples/mnist/mnist_conv.rs", - "upstreamSha256": "6a8e430eb80f87662136c56d60df916b9df025818f86cb5e2cf69e6eaefc7004", - "sha256": "6a8e430eb80f87662136c56d60df916b9df025818f86cb5e2cf69e6eaefc7004" - }, - { - "path": "examples/mnist/mnist_linear.rs", - "upstreamSha256": "afa61ca3c1f9bfa914a0ef54ebdfbfa3c5a2a70c60895cb0f34600a097eeb45c", - "sha256": "afa61ca3c1f9bfa914a0ef54ebdfbfa3c5a2a70c60895cb0f34600a097eeb45c" - }, - { - "path": "examples/mnist/mnist_nn.rs", - "upstreamSha256": "93c169a72cd9aefe7a728ace6aa82653859637444d2bcbeb2681ff5633960c97", - "sha256": "93c169a72cd9aefe7a728ace6aa82653859637444d2bcbeb2681ff5633960c97" - }, - { - "path": "examples/neural-style-transfer/README.md", - "upstreamSha256": "e7e7c64e392cbb75ddae3712e74a0dc5139a106020b6d560be5a609941b91463", - "sha256": "e7e7c64e392cbb75ddae3712e74a0dc5139a106020b6d560be5a609941b91463" - }, - { - "path": "examples/neural-style-transfer/main.rs", - "upstreamSha256": "20df81430d7282e02f3db986e110c093635697759ba0ab148b01e7d17b21d89c", - "sha256": "20df81430d7282e02f3db986e110c093635697759ba0ab148b01e7d17b21d89c" - }, - { - "path": "examples/pretrained-models/main.rs", - "upstreamSha256": "e440848e0a1b8917ec3e4fd684377a83df73a38acc6f8beead8cec32568fc3bb", - "sha256": "e440848e0a1b8917ec3e4fd684377a83df73a38acc6f8beead8cec32568fc3bb" - }, - { - "path": "examples/reinforcement-learning/README.md", - "upstreamSha256": "fa06d2e419c0e0b9f19d8945fff4855e3494ecca5cb468e81aa01faa62671950", - "sha256": "fa06d2e419c0e0b9f19d8945fff4855e3494ecca5cb468e81aa01faa62671950" - }, - { - "path": "examples/reinforcement-learning/a2c.rs", - "upstreamSha256": "b009bbdff77a7fda8970886e571f15215549230cbb3581527b5dd1a754c5c01e", - "sha256": "b009bbdff77a7fda8970886e571f15215549230cbb3581527b5dd1a754c5c01e" - }, - { - "path": "examples/reinforcement-learning/atari_wrappers.py", - "upstreamSha256": "6d8c6fb8f9777096c991948fa10cdbf71ffce72c56cb9a8da05ed8f89fdfda96", - "sha256": "6d8c6fb8f9777096c991948fa10cdbf71ffce72c56cb9a8da05ed8f89fdfda96" - }, - { - "path": "examples/reinforcement-learning/ddpg.rs", - "upstreamSha256": "9868b522a46facddd3cb1ad85adb9f802e139c2d6f1529a44c09d15f880d7f96", - "sha256": "9868b522a46facddd3cb1ad85adb9f802e139c2d6f1529a44c09d15f880d7f96" - }, - { - "path": "examples/reinforcement-learning/gym_env.rs", - "upstreamSha256": "07ae70893e77e48b3eee0835b590ddc90bcee4e58e33981d94fb272cbb3ed087", - "sha256": "07ae70893e77e48b3eee0835b590ddc90bcee4e58e33981d94fb272cbb3ed087" - }, - { - "path": "examples/reinforcement-learning/main.rs", - "upstreamSha256": "a8bb5e2b12a4c784899ab84ab2c3812457277b96a7d6d58dd15a068d0da56bd3", - "sha256": "a8bb5e2b12a4c784899ab84ab2c3812457277b96a7d6d58dd15a068d0da56bd3" - }, - { - "path": "examples/reinforcement-learning/policy_gradient.rs", - "upstreamSha256": "85d66e592aab4f433efd9e534b73a82c0160d0422ee318bb43176240e8b7ca29", - "sha256": "85d66e592aab4f433efd9e534b73a82c0160d0422ee318bb43176240e8b7ca29" - }, - { - "path": "examples/reinforcement-learning/ppo.rs", - "upstreamSha256": "cfcf5bac4e000e8e891455c3cc08af3b31a29c386596c5351d2a0e7cbd62f715", - "sha256": "cfcf5bac4e000e8e891455c3cc08af3b31a29c386596c5351d2a0e7cbd62f715" - }, - { - "path": "examples/reinforcement-learning/vec_gym_env.rs", - "upstreamSha256": "c9e8e9502d8f40636969fa6b87cb7150467e06134c7de6c95e4e6681618bc01c", - "sha256": "c9e8e9502d8f40636969fa6b87cb7150467e06134c7de6c95e4e6681618bc01c" - }, - { - "path": "examples/stable-diffusion/README.md", - "upstreamSha256": "8320c7f0f009dcae5d4e73ca8d6e06e013ec8c1ca1c5c1e72908b4e117791db3", - "sha256": "8320c7f0f009dcae5d4e73ca8d6e06e013ec8c1ca1c5c1e72908b4e117791db3" - }, - { - "path": "examples/stable-diffusion/main.rs", - "upstreamSha256": "6cc1de257c7721e944ed38ae6eec27de55450d5443bb0726003ff184117dd78e", - "sha256": "6cc1de257c7721e944ed38ae6eec27de55450d5443bb0726003ff184117dd78e" - }, - { - "path": "examples/tensor-tools.rs", - "upstreamSha256": "3667cb780fb9d978733703a57438f311ffcc6b527e06ea6e940de7d2080e8452", - "sha256": "3667cb780fb9d978733703a57438f311ffcc6b527e06ea6e940de7d2080e8452" - }, - { - "path": "examples/transfer-learning/README.md", - "upstreamSha256": "55d10f559ba7ad67c09f2e1785d8a0d7581c367544561571d21f33cbff89efd9", - "sha256": "55d10f559ba7ad67c09f2e1785d8a0d7581c367544561571d21f33cbff89efd9" - }, - { - "path": "examples/transfer-learning/main.rs", - "upstreamSha256": "15611d564628ab805b85bd16fcf36bbc2fd319e66ff0600063f7d452eb3991f7", - "sha256": "15611d564628ab805b85bd16fcf36bbc2fd319e66ff0600063f7d452eb3991f7" - }, - { - "path": "examples/translation/dataset.rs", - "upstreamSha256": "71dc84d0624442607c4da514e1d3d6bf4ce68115fb05a9a9ddc6f57e08e27021", - "sha256": "71dc84d0624442607c4da514e1d3d6bf4ce68115fb05a9a9ddc6f57e08e27021" - }, - { - "path": "examples/translation/lang.rs", - "upstreamSha256": "afd22986d5685bff5c676b52f8df174500cb0571b67ca9fa86fcd9a02a08d84c", - "sha256": "afd22986d5685bff5c676b52f8df174500cb0571b67ca9fa86fcd9a02a08d84c" - }, - { - "path": "examples/translation/main.rs", - "upstreamSha256": "218f2f42e308f4e82ada9ac593b5b5e0e1ad81f2d8a31f2ca2122268acbdc5d4", - "sha256": "218f2f42e308f4e82ada9ac593b5b5e0e1ad81f2d8a31f2ca2122268acbdc5d4" - }, - { - "path": "examples/vae/main.rs", - "upstreamSha256": "c2918c72fac34a963dfa0af5f95e80c388504a1f2c88880bb3e2edf2554e2c57", - "sha256": "c2918c72fac34a963dfa0af5f95e80c388504a1f2c88880bb3e2edf2554e2c57" - }, - { - "path": "examples/yolo/coco_classes.rs", - "upstreamSha256": "ea234f9868307a4f40ce6db5b9de801173143ee18b76ec19c0839192cb0f6b70", - "sha256": "ea234f9868307a4f40ce6db5b9de801173143ee18b76ec19c0839192cb0f6b70" - }, - { - "path": "examples/yolo/darknet.rs", - "upstreamSha256": "4625b95c621dea03ff14025b835c751a8ba983a34ec72faafb3a40c2de56146b", - "sha256": "4625b95c621dea03ff14025b835c751a8ba983a34ec72faafb3a40c2de56146b" - }, - { - "path": "examples/yolo/main.rs", - "upstreamSha256": "c3c8ca9439f1301e0b4c51d90b0c3bcae6d187edf56f3e7e129a716ad86dbf4b", - "sha256": "c3c8ca9439f1301e0b4c51d90b0c3bcae6d187edf56f3e7e129a716ad86dbf4b" - }, - { - "path": "examples/yolo/yolo-v3.cfg", - "upstreamSha256": "d9ec1f4e2dc3d9644429a2bcd7046ca5441938a742899b6ba917e076f6a6ed9b", - "sha256": "d9ec1f4e2dc3d9644429a2bcd7046ca5441938a742899b6ba917e076f6a6ed9b" - }, - { - "path": "gen/.ocamlformat", - "upstreamSha256": "ade33328770bc5abae551708c82b1dbd5692bcc34f620194cc0e4da0f129a799", - "sha256": "ade33328770bc5abae551708c82b1dbd5692bcc34f620194cc0e4da0f129a799" - }, - { - "path": "gen/dune", - "upstreamSha256": "bdb0b14b2925f3dfc8a3773d68107d6b7152adbeebcdb7089684291cd555ea97", - "sha256": "bdb0b14b2925f3dfc8a3773d68107d6b7152adbeebcdb7089684291cd555ea97" - }, - { - "path": "gen/gen.ml", - "upstreamSha256": "9b6cedf6f67922aa181175be8d2fea30a38ae3a61de63a6ab5be786b4ced2261", - "sha256": "9b6cedf6f67922aa181175be8d2fea30a38ae3a61de63a6ab5be786b4ced2261" - }, - { - "path": "gen/gen.mli", - "upstreamSha256": "473ecfe5ccae7c03cd11744977d1c5829a89027a1608b28378226bf4746afbd6", - "sha256": "473ecfe5ccae7c03cd11744977d1c5829a89027a1608b28378226bf4746afbd6" - }, - { - "path": "main.py", - "upstreamSha256": "683aef4a3ee7db81e8324b2bf7e865f1033a196509e46e0d56e0a1eb345da3a9", - "sha256": "683aef4a3ee7db81e8324b2bf7e865f1033a196509e46e0d56e0a1eb345da3a9" - }, - { - "path": "pyproject.toml", - "upstreamSha256": "55524021b952764e06e4f99ef6d6266df2b5a0a9bb34bb04ae48e47d5d4b1182", - "sha256": "55524021b952764e06e4f99ef6d6266df2b5a0a9bb34bb04ae48e47d5d4b1182" - }, - { - "path": "rustfmt.toml", - "upstreamSha256": "976b4ed388544f55349256eb91d07e16f4bcce42ce325d09bf090cc003ec72aa", - "sha256": "976b4ed388544f55349256eb91d07e16f4bcce42ce325d09bf090cc003ec72aa" - }, - { - "path": "src/data.rs", - "upstreamSha256": "442dda726016da0e48a8aba2d37ba8724c76564e70e5a2527ef0e4637605cedf", - "sha256": "442dda726016da0e48a8aba2d37ba8724c76564e70e5a2527ef0e4637605cedf" - }, - { - "path": "src/error.rs", - "upstreamSha256": "176156fe1a1c60f46ec4d1eace32d23454917cde98e3ebd6c3e7509e5f9855bb", - "sha256": "176156fe1a1c60f46ec4d1eace32d23454917cde98e3ebd6c3e7509e5f9855bb" - }, - { - "path": "src/lib.rs", - "upstreamSha256": "af659efd91873381e49a86cc2c13f030851363f24479ee498c6f157e3a45b4dd", - "sha256": "af659efd91873381e49a86cc2c13f030851363f24479ee498c6f157e3a45b4dd" - }, - { - "path": "src/nn/batch_norm.rs", - "upstreamSha256": "b98a8cc4f570a00c1c6d06700ceb413bfe186b9f56b5ba0a9b97d0bb24a5213d", - "sha256": "b98a8cc4f570a00c1c6d06700ceb413bfe186b9f56b5ba0a9b97d0bb24a5213d" - }, - { - "path": "src/nn/conv.rs", - "upstreamSha256": "4de67d22462918cb67eee394eeaec3659dbd47ea8934304a3272c94b4ecd9c39", - "sha256": "4de67d22462918cb67eee394eeaec3659dbd47ea8934304a3272c94b4ecd9c39" - }, - { - "path": "src/nn/conv_transpose.rs", - "upstreamSha256": "d881e48332ff94ae5a751d939cae64f7da1728b974010da8cc04995b214bb24c", - "sha256": "d881e48332ff94ae5a751d939cae64f7da1728b974010da8cc04995b214bb24c" - }, - { - "path": "src/nn/func.rs", - "upstreamSha256": "0353023499b3424d5b270c240c169d03050a26189a00b89f8ebd0cacfa142558", - "sha256": "0353023499b3424d5b270c240c169d03050a26189a00b89f8ebd0cacfa142558" - }, - { - "path": "src/nn/group_norm.rs", - "upstreamSha256": "5361fd0be596bb29baf44a6803b8be0bcbc5a3e946d580e029d83528ace04777", - "sha256": "5361fd0be596bb29baf44a6803b8be0bcbc5a3e946d580e029d83528ace04777" - }, - { - "path": "src/nn/init.rs", - "upstreamSha256": "8568346e1f460d0d15be56988608a44cd8d153537ecd9d055544881b286aaa57", - "sha256": "8568346e1f460d0d15be56988608a44cd8d153537ecd9d055544881b286aaa57" - }, - { - "path": "src/nn/layer_norm.rs", - "upstreamSha256": "00165a19464fa8f2917725bdd494ba7e73ff5107f58e67dd9e5eaa644ed395ab", - "sha256": "00165a19464fa8f2917725bdd494ba7e73ff5107f58e67dd9e5eaa644ed395ab" - }, - { - "path": "src/nn/linear.rs", - "upstreamSha256": "1195ebadc53485ae45abd68f9c817274330fcfa12d501d81fe8944654e66bc78", - "sha256": "1195ebadc53485ae45abd68f9c817274330fcfa12d501d81fe8944654e66bc78" - }, - { - "path": "src/nn/mod.rs", - "upstreamSha256": "f64e448ed6fe04391178347f4f8eb44283994cdd933babcbd4b2a1f3dc033c30", - "sha256": "f64e448ed6fe04391178347f4f8eb44283994cdd933babcbd4b2a1f3dc033c30" - }, - { - "path": "src/nn/module.rs", - "upstreamSha256": "f2d8a75740c0ab58362f5177708bf1e838c8a77dcacb8ffd0ff241b2069020c4", - "sha256": "f2d8a75740c0ab58362f5177708bf1e838c8a77dcacb8ffd0ff241b2069020c4" - }, - { - "path": "src/nn/optimizer.rs", - "upstreamSha256": "de0149e1e30e3ff1616ee48a84971f413e0f21baa6f89e4f24303f7b822c4353", - "sha256": "de0149e1e30e3ff1616ee48a84971f413e0f21baa6f89e4f24303f7b822c4353" - }, - { - "path": "src/nn/rnn.rs", - "upstreamSha256": "4d3351230b3678ff1a49158d6d88b0c407f30468dd4c46c6dece343c38d46353", - "sha256": "4d3351230b3678ff1a49158d6d88b0c407f30468dd4c46c6dece343c38d46353" - }, - { - "path": "src/nn/sequential.rs", - "upstreamSha256": "a1a7db1e8addeffa920ca44183d8304158dd71b62ac697de1bbf2875f71ae180", - "sha256": "a1a7db1e8addeffa920ca44183d8304158dd71b62ac697de1bbf2875f71ae180" - }, - { - "path": "src/nn/sparse.rs", - "upstreamSha256": "a20f93d1f004f29aaa8022cf0f4ead95c71d622eb1f4429c7ebf52f3f93c71f5", - "sha256": "a20f93d1f004f29aaa8022cf0f4ead95c71d622eb1f4429c7ebf52f3f93c71f5" - }, - { - "path": "src/nn/var_store.rs", - "upstreamSha256": "aaf0c8ba59a7852ae93c02b5c65fe3972bca279937e6fd6f20a358aa455ab2c7", - "sha256": "aaf0c8ba59a7852ae93c02b5c65fe3972bca279937e6fd6f20a358aa455ab2c7" - }, - { - "path": "src/tensor/convert.rs", - "upstreamSha256": "32933952a4f072c08499a23a17e7c781f3643fe6a5ad691855c735f85844aae3", - "sha256": "32933952a4f072c08499a23a17e7c781f3643fe6a5ad691855c735f85844aae3" - }, - { - "path": "src/tensor/display.rs", - "upstreamSha256": "1b25afd950ec9c85a68007f9699986dcd4696a198a0c03ae0b47c2b1da6c9b77", - "sha256": "1b25afd950ec9c85a68007f9699986dcd4696a198a0c03ae0b47c2b1da6c9b77" - }, - { - "path": "src/tensor/index.rs", - "upstreamSha256": "b7d86aea010c98a0c14a5c1899bcba0100039b7c992d1ac52f8b57ac56116173", - "sha256": "b7d86aea010c98a0c14a5c1899bcba0100039b7c992d1ac52f8b57ac56116173" - }, - { - "path": "src/tensor/iter.rs", - "upstreamSha256": "d069c87a72eef94643421a791865cc9d2313ac027809fac16ed601b00f266b6a", - "sha256": "d069c87a72eef94643421a791865cc9d2313ac027809fac16ed601b00f266b6a" - }, - { - "path": "src/tensor/mod.rs", - "upstreamSha256": "c6023b05b03cfe8d806bf2a7d36bb4a377092383eff07fe49218e68fda6aaf91", - "sha256": "c6023b05b03cfe8d806bf2a7d36bb4a377092383eff07fe49218e68fda6aaf91" - }, - { - "path": "src/tensor/npy.rs", - "upstreamSha256": "6f9c3c4dd4a57f6ba7c464e7159c83048a7a287641166dc44f7011b9b40e5728", - "sha256": "11320e9c6d0621874f6e9814063cf793e9a25e24b215ed9761c4e7f39d2e787c" - }, - { - "path": "src/tensor/ops.rs", - "upstreamSha256": "0b653a7c9e135a5b98ca41ffa612d5ded438b8b08d99c1568077c6bb32a1abaa", - "sha256": "0b653a7c9e135a5b98ca41ffa612d5ded438b8b08d99c1568077c6bb32a1abaa" - }, - { - "path": "src/tensor/safetensors.rs", - "upstreamSha256": "68526ff9e266ce26ab543a37e8e221cdd88596a55f263f90d2cd9144789fd63d", - "sha256": "68526ff9e266ce26ab543a37e8e221cdd88596a55f263f90d2cd9144789fd63d" - }, - { - "path": "src/vision/alexnet.rs", - "upstreamSha256": "8d56ae9d6bdc4546969f115fba41e3fb8d86ac1bc441151648830c60bdb88182", - "sha256": "8d56ae9d6bdc4546969f115fba41e3fb8d86ac1bc441151648830c60bdb88182" - }, - { - "path": "src/vision/cifar.rs", - "upstreamSha256": "6f36e04700a6ce5638cedee14d3303433682f97cbc4ee7ae3c618b91749bdadc", - "sha256": "6f36e04700a6ce5638cedee14d3303433682f97cbc4ee7ae3c618b91749bdadc" - }, - { - "path": "src/vision/convmixer.rs", - "upstreamSha256": "4440d6bb206528ecaf790168e3f867f75c9bb8432bba61bf75872c25e3ce0648", - "sha256": "4440d6bb206528ecaf790168e3f867f75c9bb8432bba61bf75872c25e3ce0648" - }, - { - "path": "src/vision/dataset.rs", - "upstreamSha256": "71bfc17de04742e9e45f6552a098674008e3044cd1d23e9fd3016891d958d83d", - "sha256": "71bfc17de04742e9e45f6552a098674008e3044cd1d23e9fd3016891d958d83d" - }, - { - "path": "src/vision/densenet.rs", - "upstreamSha256": "f8c427602f2d8a202f0da8fe0274b3e2ac1cad890874a96ac6adc4734b3a2941", - "sha256": "f8c427602f2d8a202f0da8fe0274b3e2ac1cad890874a96ac6adc4734b3a2941" - }, - { - "path": "src/vision/dinov2.rs", - "upstreamSha256": "a5f28c0e4f2b06175f69209522b273e9b38c55fd757f6258b60ecda0cebf5e9d", - "sha256": "a5f28c0e4f2b06175f69209522b273e9b38c55fd757f6258b60ecda0cebf5e9d" - }, - { - "path": "src/vision/efficientnet.rs", - "upstreamSha256": "acf386a6e4d34918b159c23dfb93b2ac6f4534d52e692eb8768060b51e59a736", - "sha256": "acf386a6e4d34918b159c23dfb93b2ac6f4534d52e692eb8768060b51e59a736" - }, - { - "path": "src/vision/export_dinov2.py", - "upstreamSha256": "b3a8011f7dbb73acf5b3f1d9517cde392343df29528caf786151164419c09758", - "sha256": "b3a8011f7dbb73acf5b3f1d9517cde392343df29528caf786151164419c09758" - }, - { - "path": "src/vision/export_model.py", - "upstreamSha256": "81aeaf9851b7dc1975e60a554a5f6b87a371444425eb2461a80e35202c76b213", - "sha256": "81aeaf9851b7dc1975e60a554a5f6b87a371444425eb2461a80e35202c76b213" - }, - { - "path": "src/vision/image.rs", - "upstreamSha256": "13db66d8de4845d235322af48c02f80e46b441c0a06be0b94e13da56e43433f7", - "sha256": "13db66d8de4845d235322af48c02f80e46b441c0a06be0b94e13da56e43433f7" - }, - { - "path": "src/vision/imagenet.rs", - "upstreamSha256": "afd4332f031b911513023bbab02f6b143dbf0e1b113ccfdfbf2c2fd03436c33d", - "sha256": "afd4332f031b911513023bbab02f6b143dbf0e1b113ccfdfbf2c2fd03436c33d" - }, - { - "path": "src/vision/inception.rs", - "upstreamSha256": "a2486d88d91adcbacfe583e12facb4f1a78b1eb15a5a4848bce5dd5f165f2bfd", - "sha256": "a2486d88d91adcbacfe583e12facb4f1a78b1eb15a5a4848bce5dd5f165f2bfd" - }, - { - "path": "src/vision/mnist.rs", - "upstreamSha256": "74dfd3ea86e9913a977087a52ad511952df1f3c0354712bd7f5f8be1763becf7", - "sha256": "74dfd3ea86e9913a977087a52ad511952df1f3c0354712bd7f5f8be1763becf7" - }, - { - "path": "src/vision/mobilenet.rs", - "upstreamSha256": "14bd40b7642181f6ddaf548f60d76e4ca78cff6256b4996119bbc9c149c72cec", - "sha256": "14bd40b7642181f6ddaf548f60d76e4ca78cff6256b4996119bbc9c149c72cec" - }, - { - "path": "src/vision/mod.rs", - "upstreamSha256": "32cf8ebd17678f80673dac2f2a9718b164e9ef1d0b1a6f6f8d9dd3b98d99449d", - "sha256": "32cf8ebd17678f80673dac2f2a9718b164e9ef1d0b1a6f6f8d9dd3b98d99449d" - }, - { - "path": "src/vision/resnet.rs", - "upstreamSha256": "dca15a6d5cfbe7f37e942c173087fdaeb19a347bfca2718ce45721ef244d2433", - "sha256": "dca15a6d5cfbe7f37e942c173087fdaeb19a347bfca2718ce45721ef244d2433" - }, - { - "path": "src/vision/rust_image.rs", - "upstreamSha256": "2b13844118406c0de90c44d0e0d989a53481a5f919d328269366c7debfdea46a", - "sha256": "2b13844118406c0de90c44d0e0d989a53481a5f919d328269366c7debfdea46a" - }, - { - "path": "src/vision/squeezenet.rs", - "upstreamSha256": "406e5036ab0688b5279bbc149f89c6032c722d9bf5db51d07d58fd8b89b7b547", - "sha256": "406e5036ab0688b5279bbc149f89c6032c722d9bf5db51d07d58fd8b89b7b547" - }, - { - "path": "src/vision/vgg.rs", - "upstreamSha256": "4f35d9d27df98eb1879394e8451b29a99c01a46c94f28b00079884d8b1fb4fb5", - "sha256": "4f35d9d27df98eb1879394e8451b29a99c01a46c94f28b00079884d8b1fb4fb5" - }, - { - "path": "src/wrappers/device.rs", - "upstreamSha256": "c3294ab621b92affb11323257fd3fc21877a9404c0826143b4800e5bfaf42142", - "sha256": "c3294ab621b92affb11323257fd3fc21877a9404c0826143b4800e5bfaf42142" - }, - { - "path": "src/wrappers/image.rs", - "upstreamSha256": "368abc92ebb07db9fdc89d4b068e81e8b48fa07bb6a52ff9d82e1220bad7275d", - "sha256": "368abc92ebb07db9fdc89d4b068e81e8b48fa07bb6a52ff9d82e1220bad7275d" - }, - { - "path": "src/wrappers/jit.rs", - "upstreamSha256": "f3cf318bb0574235b455577cfaa4fcbb89f213d4889f990813b5fe65ec9b7567", - "sha256": "f3cf318bb0574235b455577cfaa4fcbb89f213d4889f990813b5fe65ec9b7567" - }, - { - "path": "src/wrappers/kind.rs", - "upstreamSha256": "93d2a7f93983863ef95f3064a51997604a57244474cbe6362bd9a881b6c71fb4", - "sha256": "93d2a7f93983863ef95f3064a51997604a57244474cbe6362bd9a881b6c71fb4" - }, - { - "path": "src/wrappers/layout.rs", - "upstreamSha256": "b6ca7feb4408dcb0b78fc16679013f0f6764d843e89504e231d8c308754b0b06", - "sha256": "b6ca7feb4408dcb0b78fc16679013f0f6764d843e89504e231d8c308754b0b06" - }, - { - "path": "src/wrappers/mod.rs", - "upstreamSha256": "6a94ed3c44fb1fb1db729cfe938f41e2cf47a7fa64e4d5245d7c1a168835e513", - "sha256": "6a94ed3c44fb1fb1db729cfe938f41e2cf47a7fa64e4d5245d7c1a168835e513" - }, - { - "path": "src/wrappers/optimizer.rs", - "upstreamSha256": "c04cf4f4113a340d2e1ac7f67230eed51d7aff1139027ead196f665ec467ceba", - "sha256": "c04cf4f4113a340d2e1ac7f67230eed51d7aff1139027ead196f665ec467ceba" - }, - { - "path": "src/wrappers/python.rs", - "upstreamSha256": "85e660baa0735954d4003fad47d24a8066451131c56303bf9fee88025eb44bd3", - "sha256": "85e660baa0735954d4003fad47d24a8066451131c56303bf9fee88025eb44bd3" - }, - { - "path": "src/wrappers/scalar.rs", - "upstreamSha256": "cf532532527c50cc6fba9688b0a5d8addb191c248bd251fcca57c468647f3be2", - "sha256": "cf532532527c50cc6fba9688b0a5d8addb191c248bd251fcca57c468647f3be2" - }, - { - "path": "src/wrappers/stream.rs", - "upstreamSha256": "9af1abd309cd2c5b2b0d1e6a42505f983d081e5edbebb82c327b2b48dd1c2ae1", - "sha256": "9af1abd309cd2c5b2b0d1e6a42505f983d081e5edbebb82c327b2b48dd1c2ae1" - }, - { - "path": "src/wrappers/tensor.rs", - "upstreamSha256": "965eea22e487b9cc801c6c96730dd297c9f18d22c2fc8104bc698e5d4eceeb60", - "sha256": "965eea22e487b9cc801c6c96730dd297c9f18d22c2fc8104bc698e5d4eceeb60" - }, - { - "path": "src/wrappers/tensor_fallible_generated.rs", - "upstreamSha256": "ecb190d97a1c7004403e60b3774ad45747f9d5207d69d336c087ee39e73a75fc", - "sha256": "ecb190d97a1c7004403e60b3774ad45747f9d5207d69d336c087ee39e73a75fc" - }, - { - "path": "src/wrappers/tensor_generated.rs", - "upstreamSha256": "bd97c25060b73d68bf3c18de04cbea04420df20713b764b085ad989a174bcdbd", - "sha256": "bd97c25060b73d68bf3c18de04cbea04420df20713b764b085ad989a174bcdbd" - }, - { - "path": "src/wrappers/utils.rs", - "upstreamSha256": "121da922d4856db46f5fb2029bc1c11901ab49a4c4e0570c86eddf82eaf7a2bd", - "sha256": "121da922d4856db46f5fb2029bc1c11901ab49a4c4e0570c86eddf82eaf7a2bd" - }, - { - "path": "tests/autocast.rs", - "upstreamSha256": "42f8866420500d6cf9df5162efcd31abc6f6fb6f23988446efce24b75b93b179", - "sha256": "42f8866420500d6cf9df5162efcd31abc6f6fb6f23988446efce24b75b93b179" - }, - { - "path": "tests/create_jit_models.py", - "upstreamSha256": "493e5de4979bc309866d2ca4826f5220612dded6c560db2317cfc5a11092fcd3", - "sha256": "493e5de4979bc309866d2ca4826f5220612dded6c560db2317cfc5a11092fcd3" - }, - { - "path": "tests/data_tests.rs", - "upstreamSha256": "0b0381105f737084b25e114cbb54c39ceb83c23caa4d22c9ed3dfda9d18c7430", - "sha256": "0b0381105f737084b25e114cbb54c39ceb83c23caa4d22c9ed3dfda9d18c7430" - }, - { - "path": "tests/device_tests.rs", - "upstreamSha256": "75cb156293b0b0b8e029957e509bef8226f7d34409821410605a01c9e33f4e10", - "sha256": "75cb156293b0b0b8e029957e509bef8226f7d34409821410605a01c9e33f4e10" - }, - { - "path": "tests/display_tests.rs", - "upstreamSha256": "397c8a9c034c22b7609cfa8d7673a22a8c2c1c389e24af4bc1d61503639e56fc", - "sha256": "397c8a9c034c22b7609cfa8d7673a22a8c2c1c389e24af4bc1d61503639e56fc" - }, - { - "path": "tests/foo.pt", - "upstreamSha256": "1f951cccc226891f6aff1135dca62ba44c40a36683ec1bbc4011405c4eb4aa58", - "sha256": "1f951cccc226891f6aff1135dca62ba44c40a36683ec1bbc4011405c4eb4aa58" - }, - { - "path": "tests/foo1.pt", - "upstreamSha256": "e2d0f81eeb4f85715d994d6b29fd5543f492d49f28d46ad2ed7e755c0eb07970", - "sha256": "e2d0f81eeb4f85715d994d6b29fd5543f492d49f28d46ad2ed7e755c0eb07970" - }, - { - "path": "tests/foo2.pt", - "upstreamSha256": "7054526195c7750e723530868695c4bab77322834a3444a0fef1173d12536286", - "sha256": "7054526195c7750e723530868695c4bab77322834a3444a0fef1173d12536286" - }, - { - "path": "tests/foo3.pt", - "upstreamSha256": "f9e41c74d9447a5e519e3634239134ee7894b8579daf5d1bfe52625cd20b14ab", - "sha256": "f9e41c74d9447a5e519e3634239134ee7894b8579daf5d1bfe52625cd20b14ab" - }, - { - "path": "tests/foo4.pt", - "upstreamSha256": "96098cabdd9c11975d0326cfb1f4a65d98bad10e96b4f930a94be562a01808d2", - "sha256": "96098cabdd9c11975d0326cfb1f4a65d98bad10e96b4f930a94be562a01808d2" - }, - { - "path": "tests/foo5.pt", - "upstreamSha256": "0955c843a9d227ad995190153d4790439ef76c1123cc0a561e49a5f82af0fdf6", - "sha256": "0955c843a9d227ad995190153d4790439ef76c1123cc0a561e49a5f82af0fdf6" - }, - { - "path": "tests/foo6.pt", - "upstreamSha256": "6db11ded23a342b83b117483dab248edb22cbf2d58df3e1953edc599e581e339", - "sha256": "6db11ded23a342b83b117483dab248edb22cbf2d58df3e1953edc599e581e339" - }, - { - "path": "tests/foo7.pt", - "upstreamSha256": "9700f83cc5cdda151d455d36241cac2a78c6ee565790e1332252a7742df6e332", - "sha256": "9700f83cc5cdda151d455d36241cac2a78c6ee565790e1332252a7742df6e332" - }, - { - "path": "tests/foo8.pt", - "upstreamSha256": "75ad1bef7fd6a8dce6310d0ff6baec481f8f28a8974e1531885183ad415c2b03", - "sha256": "75ad1bef7fd6a8dce6310d0ff6baec481f8f28a8974e1531885183ad415c2b03" - }, - { - "path": "tests/jit_tests.rs", - "upstreamSha256": "99318c64f8f63d05796b823f04836ff67e1b0552128b85ac0cb05799dfc82f5a", - "sha256": "99318c64f8f63d05796b823f04836ff67e1b0552128b85ac0cb05799dfc82f5a" - }, - { - "path": "tests/linear/bs.npy", - "upstreamSha256": "1362db3d95c18b4b5a48bd7a9e2c9ec64a293bb13d2869a9b7a78579f448984e", - "sha256": "1362db3d95c18b4b5a48bd7a9e2c9ec64a293bb13d2869a9b7a78579f448984e" - }, - { - "path": "tests/linear/gen-test-data.py", - "upstreamSha256": "defc2eaf96a98e41e96e5ef12d679fbe1539b7a6279600b331d14438f876dd8e", - "sha256": "defc2eaf96a98e41e96e5ef12d679fbe1539b7a6279600b331d14438f876dd8e" - }, - { - "path": "tests/linear/in.npy", - "upstreamSha256": "a30a0baf260237fcb7e31d9759e1cbfd81857baa41fe5e9b6924d9a9f0ffe29e", - "sha256": "a30a0baf260237fcb7e31d9759e1cbfd81857baa41fe5e9b6924d9a9f0ffe29e" - }, - { - "path": "tests/linear/out.npy", - "upstreamSha256": "0f4b8a9fb298b824565611fabc2e96cbc95c8f15823efe4d05ff4c98c9f5055a", - "sha256": "0f4b8a9fb298b824565611fabc2e96cbc95c8f15823efe4d05ff4c98c9f5055a" - }, - { - "path": "tests/linear/ws.npy", - "upstreamSha256": "3aa3fe5195a384602ea414b135360984b2f632698d1a1b5be65dc1667dc9b654", - "sha256": "3aa3fe5195a384602ea414b135360984b2f632698d1a1b5be65dc1667dc9b654" - }, - { - "path": "tests/nn_tests.rs", - "upstreamSha256": "381c7c7fed56e64eaed0011a810683a6e407ac68906c3c2446020b7190f7b93f", - "sha256": "381c7c7fed56e64eaed0011a810683a6e407ac68906c3c2446020b7190f7b93f" - }, - { - "path": "tests/serialization_tests.rs", - "upstreamSha256": "f4ff52054041f6bfe757755fe0589f14935969e16cb2250dffe20be7d84084e6", - "sha256": "f4ff52054041f6bfe757755fe0589f14935969e16cb2250dffe20be7d84084e6" - }, - { - "path": "tests/tensor_indexing.rs", - "upstreamSha256": "b841fd6555b7f05c214cd2cfc74703f07e89d95e61f47aa29164cdfd1cbd0d44", - "sha256": "b841fd6555b7f05c214cd2cfc74703f07e89d95e61f47aa29164cdfd1cbd0d44" - }, - { - "path": "tests/tensor_tests.rs", - "upstreamSha256": "9ab0cc35b177f518d127458968d823df7b0c010bb5fc83245a1a1a2f272627af", - "sha256": "9ab0cc35b177f518d127458968d823df7b0c010bb5fc83245a1a1a2f272627af" - }, - { - "path": "tests/test_utils.rs", - "upstreamSha256": "728c5a88016b7f563acfb58215df2c1be64d17e4c33ce9ff585e7a0ada5d9e20", - "sha256": "728c5a88016b7f563acfb58215df2c1be64d17e4c33ce9ff585e7a0ada5d9e20" - }, - { - "path": "tests/var_store.rs", - "upstreamSha256": "35bc81d5c57e5021062e8f1559dfe930c8042382f3feebccf3e49f23a9002733", - "sha256": "35bc81d5c57e5021062e8f1559dfe930c8042382f3feebccf3e49f23a9002733" - }, - { - "path": "tests/vision_tests.rs", - "upstreamSha256": "b060ec32a2022bbb56240a46a02b7d3c59cb7e7d993ce5be174b00abd2e96267", - "sha256": "b060ec32a2022bbb56240a46a02b7d3c59cb7e7d993ce5be174b00abd2e96267" - }, - { - "path": "uv.lock", - "upstreamSha256": "00db4fd5ee16bd783595b34dc75ae6f10076f41019311cdbc421f182d628f216", - "sha256": "00db4fd5ee16bd783595b34dc75ae6f10076f41019311cdbc421f182d628f216" - } - ] - }, - { - "package": "torch-sys-0.24.0", - "archiveSha256": "f4ba78777379cf09aaa79708c63e477cf0f95e021d04360c6821f1a9f56173f7", - "changedFiles": [ - "Cargo.toml", - "build.rs" - ], - "files": [ - { - "path": "Cargo.toml", - "upstreamSha256": "90b61bb60089252c0444199ae8716bce42ed8438d96354794813c1b93f2b938e", - "sha256": "a1ba42411b026533f14ba4345c6b0b796dee6d651a345a92aaaa6352326d3703" - }, - { - "path": "Cargo.toml.orig", - "upstreamSha256": "178be14ef5df386b02ef91c6e8dfdb25acdf98c33474fd0feb512cf831e01a41", - "sha256": "178be14ef5df386b02ef91c6e8dfdb25acdf98c33474fd0feb512cf831e01a41" - }, - { - "path": "build.rs", - "upstreamSha256": "3b4de10540ab27d2a85e0226100c2a50bff9fe69a5e83b213a8398f6ed9ee8a8", - "sha256": "3d92b1ad6895ca15022c0b7fc84927d0c70810de8c72a6447d5a431b49095212" - }, - { - "path": "libtch/CMakeLists.txt", - "upstreamSha256": "5cbec8dde02e45be4789454d42d35fe7365785e6ed2fdca6499bd844ef4137a1", - "sha256": "5cbec8dde02e45be4789454d42d35fe7365785e6ed2fdca6499bd844ef4137a1" - }, - { - "path": "libtch/stb_image.h", - "upstreamSha256": "599c64066c333d9b125d65f3394de642a42cf4888c3626535dcf62fcfaee1227", - "sha256": "599c64066c333d9b125d65f3394de642a42cf4888c3626535dcf62fcfaee1227" - }, - { - "path": "libtch/stb_image_resize.h", - "upstreamSha256": "5ef7cd32cfbd9de3f87606711fdf8543958302cafc95905f9397d4bee3ad67cb", - "sha256": "5ef7cd32cfbd9de3f87606711fdf8543958302cafc95905f9397d4bee3ad67cb" - }, - { - "path": "libtch/stb_image_write.h", - "upstreamSha256": "dd0ec918c4cc4477a24a2fd268b8a55c3da6767ba50f4e7a9142b9d8e2592487", - "sha256": "dd0ec918c4cc4477a24a2fd268b8a55c3da6767ba50f4e7a9142b9d8e2592487" - }, - { - "path": "libtch/torch_api.cpp", - "upstreamSha256": "3b416a34ff5a6dda9ec6e88ef8e6e8735228e2501ec3c2a01919d93217eaccea", - "sha256": "3b416a34ff5a6dda9ec6e88ef8e6e8735228e2501ec3c2a01919d93217eaccea" - }, - { - "path": "libtch/torch_api.h", - "upstreamSha256": "b5120b143c396a0ed0c1ca19f3cad1dbe2b605351e2ab5eba2b536e17131bb97", - "sha256": "b5120b143c396a0ed0c1ca19f3cad1dbe2b605351e2ab5eba2b536e17131bb97" - }, - { - "path": "libtch/torch_api_generated.cpp", - "upstreamSha256": "029a25dbc9429277ba03c1b516ee4ce78671e8c620597beaf602647fccb13208", - "sha256": "029a25dbc9429277ba03c1b516ee4ce78671e8c620597beaf602647fccb13208" - }, - { - "path": "libtch/torch_api_generated.h", - "upstreamSha256": "0049a691ffb0fe5822385046367af9f9076fe9814c11da2dccbe052ef1e01af6", - "sha256": "0049a691ffb0fe5822385046367af9f9076fe9814c11da2dccbe052ef1e01af6" - }, - { - "path": "libtch/torch_python.cpp", - "upstreamSha256": "fdfd9bb88687fb47e7066a9358af97292a7fdb3c6222c62f6a8b1f554b613b2f", - "sha256": "fdfd9bb88687fb47e7066a9358af97292a7fdb3c6222c62f6a8b1f554b613b2f" - }, - { - "path": "libtch/torch_python.h", - "upstreamSha256": "da330a4c846a15fddc4ab3142ecaa535273724393bad963aa19cf622c5237e2f", - "sha256": "da330a4c846a15fddc4ab3142ecaa535273724393bad963aa19cf622c5237e2f" - }, - { - "path": "src/c_generated.rs", - "upstreamSha256": "8c8581ec7338e28c74fd605dfa07d3fffebbc6a81730904b223d7c5361a4846f", - "sha256": "8c8581ec7338e28c74fd605dfa07d3fffebbc6a81730904b223d7c5361a4846f" - }, - { - "path": "src/cuda.rs", - "upstreamSha256": "5041c0ab989d13aa3a800c4b3070c74d7e587b7b884e66f431a83e5dfcbd699e", - "sha256": "5041c0ab989d13aa3a800c4b3070c74d7e587b7b884e66f431a83e5dfcbd699e" - }, - { - "path": "src/io.rs", - "upstreamSha256": "fcf37f38ce57acddfa4141fd392dcd5a45a106aa4e76344cfd3902909759bcf6", - "sha256": "fcf37f38ce57acddfa4141fd392dcd5a45a106aa4e76344cfd3902909759bcf6" - }, - { - "path": "src/lib.rs", - "upstreamSha256": "3b01047fc37f8f8de98fed728f0492286da49ac346c3ea1457d97597f1ca8ecf", - "sha256": "3b01047fc37f8f8de98fed728f0492286da49ac346c3ea1457d97597f1ca8ecf" - }, - { - "path": "src/python.rs", - "upstreamSha256": "1f11411d6f1f8664f85a48d10957dfbf79667af2ccb9bd548f749cd597df5f8c", - "sha256": "1f11411d6f1f8664f85a48d10957dfbf79667af2ccb9bd548f749cd597df5f8c" - }, - { - "path": "src/traits.rs", - "upstreamSha256": "c986a2595e1ee98545579e9b46f14e0059aab4d7c02554c9d53c47d74af15ae0", - "sha256": "c986a2595e1ee98545579e9b46f14e0059aab4d7c02554c9d53c47d74af15ae0" - } - ] - } - ] -} diff --git a/v2/dependency-patches/tch-0.24.0/.cargo/config.toml b/v2/dependency-patches/tch-0.24.0/.cargo/config.toml deleted file mode 100644 index e3033cae22..0000000000 --- a/v2/dependency-patches/tch-0.24.0/.cargo/config.toml +++ /dev/null @@ -1,2 +0,0 @@ -[target.x86_64-unknown-linux-gnu] -rustflags = ["-C", "link-arg=-Wl,--allow-shlib-undefined", "-C", "link-arg=-fno-lto"] diff --git a/v2/dependency-patches/tch-0.24.0/.github/workflows/rust-ci.yml b/v2/dependency-patches/tch-0.24.0/.github/workflows/rust-ci.yml deleted file mode 100644 index 74e75bf4fa..0000000000 --- a/v2/dependency-patches/tch-0.24.0/.github/workflows/rust-ci.yml +++ /dev/null @@ -1,74 +0,0 @@ -on: [push, pull_request] - -name: Continuous integration - -jobs: - check: - name: Check - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - rust: [stable, nightly] - steps: - - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: ${{ matrix.rust }} - override: true - - uses: actions-rs/cargo@v1 - with: - command: check - args: --features download-libtorch - - test: - name: Test Suite - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - rust: [stable, nightly] - steps: - - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: ${{ matrix.rust }} - override: true - - uses: actions-rs/cargo@v1 - with: - command: test - args: --features download-libtorch - - fmt: - name: Rustfmt - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - - run: rustup component add rustfmt - - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --all -- --check - - clippy: - name: Clippy - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: stable - override: true - - run: rustup component add clippy - - uses: actions-rs/cargo@v1 - with: - command: clippy - args: --examples --tests --all-features --all -- -D warnings diff --git a/v2/dependency-patches/tch-0.24.0/.gitignore b/v2/dependency-patches/tch-0.24.0/.gitignore deleted file mode 100644 index d201b4d7d5..0000000000 --- a/v2/dependency-patches/tch-0.24.0/.gitignore +++ /dev/null @@ -1,19 +0,0 @@ -target/ -_build/ -data/ -.idea/ -gen/.merlin -**/*.rs.bk -*.swp -*.swo -Cargo.lock -__pycache__ -*~ -.*~ -.vscode/ - -*.ot -*.safetensors -*.so -*.dylib -llama-tokenizer.json diff --git a/v2/dependency-patches/tch-0.24.0/.python-version b/v2/dependency-patches/tch-0.24.0/.python-version deleted file mode 100644 index e4fba21835..0000000000 --- a/v2/dependency-patches/tch-0.24.0/.python-version +++ /dev/null @@ -1 +0,0 @@ -3.12 diff --git a/v2/dependency-patches/tch-0.24.0/CHANGELOG.md b/v2/dependency-patches/tch-0.24.0/CHANGELOG.md deleted file mode 100644 index 0e3a534f5d..0000000000 --- a/v2/dependency-patches/tch-0.24.0/CHANGELOG.md +++ /dev/null @@ -1,143 +0,0 @@ -# Changelog -This documents the main changes to the `tch` crate. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## Unreleased -### Changed - -## v0.22.0 -### Changed -- PyTorch v2.9 support - -## v0.21.0 -### Changed -- PyTorch v2.8 support - -## v0.20.0 -### Changed -- PyTorch v2.7 support - -## v0.19.0 -### Changed -- PyTorch v2.6 support - -## v0.18.1 -### Changed -- PyTorch v2.5.1 support - -## v0.18.0 -### Changed -- PyTorch v2.5 support - -## v0.17.0 -### Changed -- PyTorch v2.4 support - -## v0.16.0 -### Changed -- PyTorch v2.3 support - -## v0.15.0 -### Changed -- PyTorch v2.2 support - -## v0.14.0 -### Changed -- PyTorch v2.1 support - [803](https://github.com/LaurentMazare/tch-rs/pull/803). -- Add a `pyo3-tch` crate for interacting with Python via PyO3 - [730](https://github.com/LaurentMazare/tch-rs/pull/730). -- Expose the cuda fuser enabled flag, - [728](https://github.com/LaurentMazare/tch-rs/pull/728). -- Improved the safetensor error wrapping, - [720](https://github.com/LaurentMazare/tch-rs/pull/720). - -## v0.13.0 - 2023-05-18 -### Added -- Support static linking in the build script, - [712](https://github.com/LaurentMazare/tch-rs/pull/712). -- Make the libtorch download opt-in rather than a default behavior. The libtorch - library download can still be triggered by enabling the `download-libtorch` - feature, [707](https://github.com/LaurentMazare/tch-rs/pull/707). -- Rename the `of_...` conversion functions to `from_...` so as to be closer to - the Rust best practices, - [706](https://github.com/LaurentMazare/tch-rs/pull/706). This is a breaking - change and will require modifying calls such as `of_slice` to be `from_slice` - instead. -- Expose some functions so that Python extensions that operates on PyTorch - tensors can be written with `tch`, - [704](https://github.com/LaurentMazare/tch-rs/pull/704). -- Rework the torch-sys build script making it easier to leverage a Python - PyTorch install as a source for libtorch, - [703](https://github.com/LaurentMazare/tch-rs/pull/703). - -## v0.12.0 - 2023-05-10 -### Changed -- EfficientNet models have been reworked, pre-trained models used `safetensors` - weight by default, [679](https://github.com/LaurentMazare/tch-rs/pull/679). -- None can be used for nullable scalar types, - [680](https://github.com/LaurentMazare/tch-rs/pull/680). -- Automated conversion of list arguments: all the generated functions that take - as input a slice of int or float can now be used directly with int values or - fixed length arrays [682](https://github.com/LaurentMazare/tch-rs/pull/682). -- Replace the `From` traits with some `TryFrom` versions, - [683](https://github.com/LaurentMazare/tch-rs/pull/683). This is a breaking - change, note that also the old version would flatten the tensor if needed to - reduce the number of dimensions, this has to be done explicitely with the new - version. - -## v0.11.0 - 2023-03-20 -### Added -- Adapt to C++ PyTorch library (`libtorch`) version `v2.0.0`. -### Changed -- Update the `half` dependency to version `2`, [646](https://github.com/LaurentMazare/tch-rs/pull/646). - -## v0.10.3 - 2023-02-23 -### Added -- Add some helper functions in a utils module to check the available devices and versions. - -## v0.10.2 - 2023-02-19 -### Added -- Add the `eps` and `amsgrad` options to Adam, [600](https://github.com/LaurentMazare/tch-rs/pull/600). -### Changed -- Fix loading of `VarStore` when using `Mps` devices, [623](https://github.com/LaurentMazare/tch-rs/pull/623). -- Use `ureq` instead of `curl` to reduce compile times, [620](https://github.com/LaurentMazare/tch-rs/pull/620). -- Fix the handling of dicts in TorchScript, [597](https://github.com/LaurentMazare/tch-rs/issues/597). - -## v0.10.1 - 2022-12-12 -### Changed -- Default `vs.load` to use the Python weight format when the file extension is `.pt` or `.bin`. - -## v0.10.0 - 2022-12-12 -### Added -- Expose functions for setting manual seeds for CUDA devices, [#500](https://github.com/LaurentMazare/tch-rs/pull/500). -- Expose functions for triggering manual sync, [#500](https://github.com/LaurentMazare/tch-rs/pull/500). -- Add some functions to load Python weight files. -### Changed -- Extending the Kaiming initialization, [#573](https://github.com/LaurentMazare/tch-rs/pull/573). - -## v0.9.0 - 2022-11-05 -### Changed -- Adapt to C++ PyTorch library (`libtorch`) version `v1.13.0`. - -## v0.8.0 - 2022-07-04 -### Changed -- Adapt to C++ PyTorch library (`libtorch`) version `v1.12.0`. - -## v0.7.2 - 2022-05-16 -### Changed -- Adapt to C++ PyTorch library (`libtorch`) version `v1.11.0`. - -## v0.6.1 - 2021-10-25 -### Changed -- Adapt to C++ PyTorch library (`libtorch`) version `v1.10.0`. - -## v0.5.0 - 2021-06-25 -### Changed -- Adapt to C++ PyTorch library (`libtorch`) version `v1.9.0`. - -## v0.4.1 - 2021-05-08 -### Changed -- Adapt to C++ PyTorch library (`libtorch`) version `v1.8.1`. diff --git a/v2/dependency-patches/tch-0.24.0/Cargo.toml b/v2/dependency-patches/tch-0.24.0/Cargo.toml deleted file mode 100644 index c7c18fd6b3..0000000000 --- a/v2/dependency-patches/tch-0.24.0/Cargo.toml +++ /dev/null @@ -1,246 +0,0 @@ -# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO -# -# When uploading crates to the registry Cargo will automatically -# "normalize" Cargo.toml files for maximal compatibility -# with all versions of Cargo and also rewrite `path` dependencies -# to registry (e.g., crates.io) dependencies. -# -# If you are reading this file be aware that the original Cargo.toml -# will likely look very different (and much more reasonable). -# See Cargo.toml.orig for the original contents. - -[package] -edition = "2021" -name = "tch" -version = "0.24.0" -authors = ["Laurent Mazare "] -build = "build.rs" -exclude = ["examples/stable-diffusion/media/*"] -autolib = false -autobins = false -autoexamples = false -autotests = false -autobenches = false -description = "Rust wrappers for the PyTorch C++ api (libtorch)." -readme = "README.md" -keywords = [ - "pytorch", - "deep-learning", - "machine-learning", -] -categories = ["science"] -license = "MIT/Apache-2.0" -repository = "https://github.com/LaurentMazare/tch-rs" - -[package.metadata.docs.rs] -features = ["doc-only"] - -[features] -cuda-tests = [] -doc-only = ["torch-sys/doc-only"] -download-libtorch = ["torch-sys/download-libtorch"] -python-extension = ["torch-sys/python-extension"] -rl-python = ["cpython"] - -[lib] -name = "tch" -path = "src/lib.rs" - -[[example]] -name = "basics" -path = "examples/basics.rs" - -[[example]] -name = "char-rnn" -path = "examples/char-rnn/main.rs" - -[[example]] -name = "cifar" -path = "examples/cifar/main.rs" - -[[example]] -name = "custom-optimizer" -path = "examples/custom-optimizer/main.rs" - -[[example]] -name = "gan" -path = "examples/gan/main.rs" - -[[example]] -name = "jit" -path = "examples/jit/main.rs" - -[[example]] -name = "jit-quantized" -path = "examples/jit-quantized/main.rs" - -[[example]] -name = "jit-trace" -path = "examples/jit-trace/main.rs" - -[[example]] -name = "jit-train" -path = "examples/jit-train/main.rs" - -[[example]] -name = "llama" -path = "examples/llama/main.rs" -required-features = [ - "regex", - "clap", - "serde_json", - "memmap2", -] - -[[example]] -name = "memory_test" -path = "examples/memory_test.rs" - -[[example]] -name = "min-gpt" -path = "examples/min-gpt/main.rs" - -[[example]] -name = "mnist" -path = "examples/mnist/main.rs" - -[[example]] -name = "neural-style-transfer" -path = "examples/neural-style-transfer/main.rs" - -[[example]] -name = "pretrained-models" -path = "examples/pretrained-models/main.rs" - -[[example]] -name = "reinforcement-learning" -path = "examples/reinforcement-learning/main.rs" -required-features = ["rl-python"] - -[[example]] -name = "stable-diffusion" -path = "examples/stable-diffusion/main.rs" -required-features = ["regex"] - -[[example]] -name = "tensor-tools" -path = "examples/tensor-tools.rs" - -[[example]] -name = "transfer-learning" -path = "examples/transfer-learning/main.rs" - -[[example]] -name = "translation" -path = "examples/translation/main.rs" - -[[example]] -name = "vae" -path = "examples/vae/main.rs" - -[[example]] -name = "yolo" -path = "examples/yolo/main.rs" - -[[test]] -name = "autocast" -path = "tests/autocast.rs" - -[[test]] -name = "data_tests" -path = "tests/data_tests.rs" - -[[test]] -name = "device_tests" -path = "tests/device_tests.rs" - -[[test]] -name = "display_tests" -path = "tests/display_tests.rs" - -[[test]] -name = "jit_tests" -path = "tests/jit_tests.rs" - -[[test]] -name = "nn_tests" -path = "tests/nn_tests.rs" - -[[test]] -name = "serialization_tests" -path = "tests/serialization_tests.rs" - -[[test]] -name = "tensor_indexing" -path = "tests/tensor_indexing.rs" - -[[test]] -name = "tensor_tests" -path = "tests/tensor_tests.rs" - -[[test]] -name = "test_utils" -path = "tests/test_utils.rs" - -[[test]] -name = "var_store" -path = "tests/var_store.rs" - -[[test]] -name = "vision_tests" -path = "tests/vision_tests.rs" - -[dependencies.clap] -version = "4.2.4" -features = ["derive"] -optional = true - -[dependencies.cpython] -version = "0.7.1" -optional = true - -[dependencies.half] -version = "2" - -[dependencies.image] -version = "0.24.5" -optional = true - -[dependencies.lazy_static] -version = "1.3.0" - -[dependencies.libc] -version = "0.2.0" - -[dependencies.memmap2] -version = "0.6.1" -optional = true - -[dependencies.ndarray] -version = "0.16.1" - -[dependencies.rand] -version = "0.8" - -[dependencies.regex] -version = "1.6.0" -optional = true - -[dependencies.safetensors] -version = "0.3.0" - -[dependencies.serde_json] -version = "1.0.96" -optional = true - -[dependencies.thiserror] -version = "1" - -[dependencies.torch-sys] -version = "0.24.0" - -[dependencies.zip] -version = "9.0.1" - -[dev-dependencies.anyhow] -version = "^1.0.60" diff --git a/v2/dependency-patches/tch-0.24.0/Cargo.toml.orig b/v2/dependency-patches/tch-0.24.0/Cargo.toml.orig deleted file mode 100644 index e39ee700fd..0000000000 --- a/v2/dependency-patches/tch-0.24.0/Cargo.toml.orig +++ /dev/null @@ -1,67 +0,0 @@ -[package] -name = "tch" -version = "0.24.0" -authors = ["Laurent Mazare "] -edition = "2021" -build = "build.rs" - -description = "Rust wrappers for the PyTorch C++ api (libtorch)." -repository = "https://github.com/LaurentMazare/tch-rs" -keywords = ["pytorch", "deep-learning", "machine-learning"] -categories = ["science"] -license = "MIT/Apache-2.0" -readme = "README.md" - -exclude = [ - "examples/stable-diffusion/media/*", -] - -[dependencies] -lazy_static = "1.3.0" -libc = "0.2.0" -ndarray = "0.16.1" -rand = "0.8" -thiserror = "1" -torch-sys = { version = "0.24.0", path = "torch-sys" } -zip = "0.6" -half = "2" -safetensors = "0.3.0" - -cpython = { version = "0.7.1", optional = true } -regex = { version = "1.6.0", optional = true } -image = { version = "0.24.5", optional = true } -clap = { version = "4.2.4", features = ["derive"], optional = true } -serde_json = { version = "1.0.96", optional = true } -memmap2 = { version = "0.6.1", optional = true } - -[dev-dependencies] -anyhow = "^1.0.60" - -[workspace] -members = [ - "torch-sys", - "pyo3-tch", - "examples/python-extension", -] - -[features] -download-libtorch = ["torch-sys/download-libtorch"] -python-extension = ["torch-sys/python-extension"] -rl-python = ["cpython"] -doc-only = ["torch-sys/doc-only"] -cuda-tests = [] - -[package.metadata.docs.rs] -features = [ "doc-only" ] - -[[example]] -name = "reinforcement-learning" -required-features = ["rl-python"] - -[[example]] -name = "stable-diffusion" -required-features = ["regex"] - -[[example]] -name = "llama" -required-features = ["regex", "clap", "serde_json", "memmap2"] diff --git a/v2/dependency-patches/tch-0.24.0/LICENSE-APACHE b/v2/dependency-patches/tch-0.24.0/LICENSE-APACHE deleted file mode 100644 index 261eeb9e9f..0000000000 --- a/v2/dependency-patches/tch-0.24.0/LICENSE-APACHE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/v2/dependency-patches/tch-0.24.0/LICENSE-MIT b/v2/dependency-patches/tch-0.24.0/LICENSE-MIT deleted file mode 100644 index 31aa79387f..0000000000 --- a/v2/dependency-patches/tch-0.24.0/LICENSE-MIT +++ /dev/null @@ -1,23 +0,0 @@ -Permission is hereby granted, free of charge, to any -person obtaining a copy of this software and associated -documentation files (the "Software"), to deal in the -Software without restriction, including without -limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software -is furnished to do so, subject to the following -conditions: - -The above copyright notice and this permission notice -shall be included in all copies or substantial portions -of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF -ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED -TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A -PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT -SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY -CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR -IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. diff --git a/v2/dependency-patches/tch-0.24.0/Makefile b/v2/dependency-patches/tch-0.24.0/Makefile deleted file mode 100644 index 0c06cffb8a..0000000000 --- a/v2/dependency-patches/tch-0.24.0/Makefile +++ /dev/null @@ -1,13 +0,0 @@ -test: .FORCE - cargo test - -clean: .FORCE - cargo clean - -gen: .FORCE - dune exec gen/gen.exe - rustfmt src/wrappers/tensor_fallible_generated.rs - rustfmt src/wrappers/tensor_generated.rs - rustfmt torch-sys/src/c_generated.rs - -.FORCE: diff --git a/v2/dependency-patches/tch-0.24.0/README.md b/v2/dependency-patches/tch-0.24.0/README.md deleted file mode 100644 index 777afa57d5..0000000000 --- a/v2/dependency-patches/tch-0.24.0/README.md +++ /dev/null @@ -1,372 +0,0 @@ -# tch-rs -Rust bindings for the C++ api of PyTorch. The goal of the `tch` crate is to -provide some thin wrappers around the C++ PyTorch api (a.k.a. libtorch). It -aims at staying as close as possible to the original C++ api. More idiomatic -rust bindings could then be developed on top of this. The -[documentation](https://docs.rs/tch/) can be found on docs.rs. - -[![Build Status](https://github.com/LaurentMazare/tch-rs/workflows/Continuous%20integration/badge.svg)](https://github.com/LaurentMazare/tch-rs/actions) -[![Latest version](https://img.shields.io/crates/v/tch.svg)](https://crates.io/crates/tch) -[![Documentation](https://docs.rs/tch/badge.svg)](https://docs.rs/tch) -[![Dependency Status](https://deps.rs/repo/github/LaurentMazare/tch-rs/status.svg)](https://deps.rs/repo/github/LaurentMazare/tch-rs) -![License](https://img.shields.io/crates/l/tch.svg) -[changelog](https://github.com/LaurentMazare/tch-rs/blob/main/CHANGELOG.md) - - -The code generation part for the C api on top of libtorch comes from -[ocaml-torch](https://github.com/LaurentMazare/ocaml-torch). - -## Getting Started - -This crate requires the C++ PyTorch library (libtorch) in version *v2.11.0* to be available on -your system. You can either: - -- Use the system-wide libtorch installation (default). -- Install libtorch manually and let the build script know about it via the `LIBTORCH` environment variable. -- Use a Python PyTorch install, to do this set `LIBTORCH_USE_PYTORCH=1`. -- When a system-wide libtorch can't be found and `LIBTORCH` is not set, the - build script can download a pre-built binary version of libtorch by using - the `download-libtorch` feature. By default a CPU version is used. The - `TORCH_CUDA_VERSION` environment variable can be set to `cu117` in order to - get a pre-built binary using CUDA 11.7. - -### System-wide Libtorch - -On linux platforms, the build script will look for a system-wide libtorch -library in `/usr/lib/libtorch.so`. - -### Python PyTorch Install - -If the `LIBTORCH_USE_PYTORCH` environment variable is set, the active python -interpreter is called to retrieve information about the torch python package. -This version is then linked against. - -### Libtorch Manual Install - -- Get `libtorch` from the -[PyTorch website download section](https://pytorch.org/get-started/locally/) and extract -the content of the zip file. -- For Linux and macOS users, add the following to your `.bashrc` or equivalent, where `/path/to/libtorch` -is the path to the directory that was created when unzipping the file. -```bash -export LIBTORCH=/path/to/libtorch -``` -The header files location can also be specified separately from the shared library via -the following: -```bash -# LIBTORCH_INCLUDE must contain `include` directory. -export LIBTORCH_INCLUDE=/path/to/libtorch/ -# LIBTORCH_LIB must contain `lib` directory. -export LIBTORCH_LIB=/path/to/libtorch/ -``` -- For Windows users, assuming that `X:\path\to\libtorch` is the unzipped libtorch directory. - - Navigate to Control Panel -> View advanced system settings -> Environment variables. - - Create the `LIBTORCH` variable and set it to `X:\path\to\libtorch`. - - Append `X:\path\to\libtorch\lib` to the `Path` variable. - - If you prefer to temporarily set environment variables, in PowerShell you can run -```powershell -$Env:LIBTORCH = "X:\path\to\libtorch" -$Env:Path += ";X:\path\to\libtorch\lib" -``` -- You should now be able to run some examples, e.g. `cargo run --example basics`. - -### Windows Specific Notes - -As per [the pytorch docs](https://pytorch.org/cppdocs/installing.html) the Windows debug and release builds are not ABI-compatible. This could lead to some segfaults if the incorrect version of libtorch is used. - -It is recommended to use the MSVC Rust toolchain (e.g. by installing `stable-x86_64-pc-windows-msvc` via rustup) rather than a MinGW based one as PyTorch has compatibilities issues with MinGW. - -### Static Linking - -When setting environment variable `LIBTORCH_STATIC=1`, `libtorch` is statically -linked rather than using the dynamic libraries. The pre-compiled artifacts don't -seem to include `libtorch.a` by default so this would have to be compiled -manually, e.g. via the following: - -```bash -git clone -b v2.11.0 --recurse-submodule https://github.com/pytorch/pytorch.git pytorch-static --depth 1 -cd pytorch-static -USE_CUDA=OFF BUILD_SHARED_LIBS=OFF python setup.py build -# export LIBTORCH to point at the build directory in pytorch-static. -``` - -## Examples - -### Basic Tensor Operations - -This crate provides a tensor type which wraps PyTorch tensors. Here is a minimal -example of how to perform some tensor operations. - -```rust -use tch::Tensor; - -fn main() { - let t = Tensor::from_slice(&[3, 1, 4, 1, 5]); - let t = t * 2; - t.print(); -} -``` - -### Training a Model via Gradient Descent - -PyTorch provides automatic differentiation for most tensor operations -it supports. This is commonly used to train models using gradient -descent. The optimization is performed over variables which are created -via a `nn::VarStore` by defining their shapes and initializations. - -In the example below `my_module` uses two variables `x1` and `x2` -which initial values are 0. The forward pass applied to tensor `xs` -returns `xs * x1 + exp(xs) * x2`. - -Once the model has been generated, a `nn::Sgd` optimizer is created. -Then on each step of the training loop: - -- The forward pass is applied to a mini-batch of data. -- A loss is computed as the mean square error between the model output and the mini-batch ground truth. -- Finally an optimization step is performed: gradients are computed and variables from the `VarStore` are modified accordingly. - - -```rust -use tch::nn::{Module, OptimizerConfig}; -use tch::{kind, nn, Device, Tensor}; - -fn my_module(p: nn::Path, dim: i64) -> impl nn::Module { - let x1 = p.zeros("x1", &[dim]); - let x2 = p.zeros("x2", &[dim]); - nn::func(move |xs| xs * &x1 + xs.exp() * &x2) -} - -fn gradient_descent() { - let vs = nn::VarStore::new(Device::Cpu); - let my_module = my_module(vs.root(), 7); - let mut opt = nn::Sgd::default().build(&vs, 1e-2).unwrap(); - for _idx in 1..50 { - // Dummy mini-batches made of zeros. - let xs = Tensor::zeros(&[7], kind::FLOAT_CPU); - let ys = Tensor::zeros(&[7], kind::FLOAT_CPU); - let loss = (my_module.forward(&xs) - ys).pow_tensor_scalar(2).sum(kind::Kind::Float); - opt.backward_step(&loss); - } -} -``` - -### Writing a Simple Neural Network - -The `nn` api can be used to create neural network architectures, e.g. the following code defines -a simple model with one hidden layer and trains it on the MNIST dataset using the Adam optimizer. - -```rust -use anyhow::Result; -use tch::{nn, nn::Module, nn::OptimizerConfig, Device}; - -const IMAGE_DIM: i64 = 784; -const HIDDEN_NODES: i64 = 128; -const LABELS: i64 = 10; - -fn net(vs: &nn::Path) -> impl Module { - nn::seq() - .add(nn::linear( - vs / "layer1", - IMAGE_DIM, - HIDDEN_NODES, - Default::default(), - )) - .add_fn(|xs| xs.relu()) - .add(nn::linear(vs, HIDDEN_NODES, LABELS, Default::default())) -} - -pub fn run() -> Result<()> { - let m = tch::vision::mnist::load_dir("data")?; - let vs = nn::VarStore::new(Device::Cpu); - let net = net(&vs.root()); - let mut opt = nn::Adam::default().build(&vs, 1e-3)?; - for epoch in 1..200 { - let loss = net - .forward(&m.train_images) - .cross_entropy_for_logits(&m.train_labels); - opt.backward_step(&loss); - let test_accuracy = net - .forward(&m.test_images) - .accuracy_for_logits(&m.test_labels); - println!( - "epoch: {:4} train loss: {:8.5} test acc: {:5.2}%", - epoch, - f64::from(&loss), - 100. * f64::from(&test_accuracy), - ); - } - Ok(()) -} -``` - -More details on the training loop can be found in the -[detailed tutorial](https://github.com/LaurentMazare/tch-rs/tree/master/examples/mnist). - -### Using some Pre-Trained Model - -The [pretrained-models example](https://github.com/LaurentMazare/tch-rs/tree/master/examples/pretrained-models/main.rs) -illustrates how to use some pre-trained computer vision model on an image. -The weights - which have been extracted from the PyTorch implementation - can be -downloaded here [resnet18.ot](https://github.com/LaurentMazare/tch-rs/releases/download/mw/resnet18.ot) -and here [resnet34.ot](https://github.com/LaurentMazare/tch-rs/releases/download/mw/resnet34.ot). - -The example can then be run via the following command: -```bash -cargo run --example pretrained-models -- resnet18.ot tiger.jpg -``` -This should print the top 5 imagenet categories for the image. The code for this example is pretty simple. - -```rust - // First the image is loaded and resized to 224x224. - let image = imagenet::load_image_and_resize(image_file)?; - - // A variable store is created to hold the model parameters. - let vs = tch::nn::VarStore::new(tch::Device::Cpu); - - // Then the model is built on this variable store, and the weights are loaded. - let resnet18 = tch::vision::resnet::resnet18(vs.root(), imagenet::CLASS_COUNT); - vs.load(weight_file)?; - - // Apply the forward pass of the model to get the logits and convert them - // to probabilities via a softmax. - let output = resnet18 - .forward_t(&image.unsqueeze(0), /*train=*/ false) - .softmax(-1); - - // Finally print the top 5 categories and their associated probabilities. - for (probability, class) in imagenet::top(&output, 5).iter() { - println!("{:50} {:5.2}%", class, 100.0 * probability) - } -``` -### Importing Pre-Trained Weights from PyTorch Using SafeTensors - -`safetensors` is a new simple format by HuggingFace for storing tensors. It does not rely on Python's `pickle` module, and therefore the tensors are not bound to the specific classes and the exact directory structure used when the model is saved. It is also zero-copy, which means that reading the file will require no more memory than the original file. - -For more information on `safetensors`, please check out https://github.com/huggingface/safetensors - -#### Installing `safetensors` - -You can install `safetensors` via the pip manager: - -``` -pip install safetensors -``` - -#### Exporting weights in PyTorch - -```python -import torchvision -from safetensors import torch as stt - -model = torchvision.models.resnet18(pretrained=True) -stt.save_file(model.state_dict(), 'resnet18.safetensors') -``` - -*Note: the filename of the export must be named with a `.safetensors` suffix for it to be properly decoded by `tch`.* - -#### Importing weights in `tch` - -```rust -use anyhow::Result; -use tch::{ - Device, - Kind, - nn::VarStore, - vision::{ - imagenet, - resnet::resnet18, - } -}; - -fn main() -> Result<()> { - // Create the model and load the pre-trained weights - let mut vs = VarStore::new(Device::cuda_if_available()); - let model = resnet18(&vs.root(), 1000); - vs.load("resnet18.safetensors")?; - - // Load the image file and resize it to the usual imagenet dimension of 224x224. - let image = imagenet::load_image_and_resize224("dog.jpg")? - .to_device(vs.device()); - - // Apply the forward pass of the model to get the logits - let output = image - .unsqueeze(0) - .apply_t(&model, false) - .softmax(-1, Kind::Float); - - // Print the top 5 categories for this image. - for (probability, class) in imagenet::top(&output, 5).iter() { - println!("{:50} {:5.2}%", class, 100.0 * probability) - } - - Ok(()) -} -``` - -Further examples include: -* A simplified version of - [char-rnn](https://github.com/LaurentMazare/tch-rs/blob/master/examples/char-rnn) - illustrating character level language modeling using Recurrent Neural Networks. -* [Neural style transfer](https://github.com/LaurentMazare/tch-rs/blob/master/examples/neural-style-transfer) - uses a pre-trained VGG-16 model to compose an image in the style of another image (pre-trained weights: - [vgg16.ot](https://github.com/LaurentMazare/tch-rs/releases/download/mw/vgg16.ot)). -* Some [ResNet examples on CIFAR-10](https://github.com/LaurentMazare/tch-rs/tree/master/examples/cifar). -* A [tutorial](https://github.com/LaurentMazare/tch-rs/tree/master/examples/jit) - showing how to deploy/run some Python trained models using - [TorchScript JIT](https://pytorch.org/docs/stable/jit.html). -* Some [Reinforcement Learning](https://github.com/LaurentMazare/tch-rs/blob/master/examples/reinforcement-learning) - examples using the [OpenAI Gym](https://github.com/openai/gym) environment. This includes a policy gradient - example as well as an A2C implementation that can run on Atari games. -* A [Transfer Learning Tutorial](https://github.com/LaurentMazare/tch-rs/blob/master/examples/transfer-learning) - shows how to finetune a pre-trained ResNet model on a very small dataset. -* A [simplified version of GPT](https://github.com/LaurentMazare/tch-rs/blob/master/examples/min-gpt) - similar to minGPT. -* A [Stable Diffusion](https://github.com/LaurentMazare/diffusers-rs) - implementation following the lines of hugginface's diffusers library. - -External material: -* A [tutorial](http://vegapit.com/article/how-to-use-torch-in-rust-with-tch-rs) showing how to use Torch to compute option prices and greeks. -* [tchrs-opencv-webcam-inference](https://github.com/metobom/tchrs-opencv-webcam-inference) uses `tch-rs` and `opencv` to run inference - on a webcam feed for some Python trained model based on mobilenet v3. - -## FAQ - -### What are the best practices for Python to Rust model translations? - -See some details in [this thread](https://github.com/LaurentMazare/tch-rs/issues/549#issuecomment-1296840898). - -### How to get this to work on a M1/M2 mac? - -Check this [issue](https://github.com/LaurentMazare/tch-rs/issues/488). - -### Compilation is slow, torch-sys seems to be rebuilt every time cargo gets run. -See this [issue](https://github.com/LaurentMazare/tch-rs/issues/596), this could -be caused by rust-analyzer not knowing about the proper environment variables -like `LIBTORCH` and `LD_LIBRARY_PATH`. - -### Using Rust/tch code from Python. -It is possible to call Rust/tch code from Python via PyO3, -[tch-ext](https://github.com/LaurentMazare/tch-ext) provides an example of such -a Python extension. - -### Error loading shared libraries. - -If you get an error about not finding some shared libraries when running the generated binaries -(e.g. -` error while loading shared libraries: libtorch_cpu.so: cannot open shared object file: No such file or directory`). -You can try adding the following to your `.bashrc` where `/path/to/libtorch` is the path to your -libtorch install. -``` -# For Linux -export LD_LIBRARY_PATH=/path/to/libtorch/lib:$LD_LIBRARY_PATH -# For macOS -export DYLD_LIBRARY_PATH=/path/to/libtorch/lib:$DYLD_LIBRARY_PATH -``` - -## License -`tch-rs` is distributed under the terms of both the MIT license -and the Apache license (version 2.0), at your option. - -See [LICENSE-APACHE](LICENSE-APACHE), [LICENSE-MIT](LICENSE-MIT) for more -details. diff --git a/v2/dependency-patches/tch-0.24.0/build.rs b/v2/dependency-patches/tch-0.24.0/build.rs deleted file mode 100644 index 181d58e23b..0000000000 --- a/v2/dependency-patches/tch-0.24.0/build.rs +++ /dev/null @@ -1,13 +0,0 @@ -fn main() { - let os = std::env::var("CARGO_CFG_TARGET_OS").expect("Unable to get TARGET_OS"); - match os.as_str() { - "linux" | "windows" => { - if let Some(lib_path) = std::env::var_os("DEP_TCH_LIBTORCH_LIB") { - println!("cargo:rustc-link-arg=-Wl,-rpath={}", lib_path.to_string_lossy()); - } - println!("cargo:rustc-link-arg=-Wl,--no-as-needed"); - println!("cargo:rustc-link-arg=-ltorch"); - } - _ => {} - } -} diff --git a/v2/dependency-patches/tch-0.24.0/clippy.toml b/v2/dependency-patches/tch-0.24.0/clippy.toml deleted file mode 100644 index 0d4e02f0b0..0000000000 --- a/v2/dependency-patches/tch-0.24.0/clippy.toml +++ /dev/null @@ -1 +0,0 @@ -too-many-arguments-threshold = 20 diff --git a/v2/dependency-patches/tch-0.24.0/dune-project b/v2/dependency-patches/tch-0.24.0/dune-project deleted file mode 100644 index a26d6e2739..0000000000 --- a/v2/dependency-patches/tch-0.24.0/dune-project +++ /dev/null @@ -1 +0,0 @@ -(lang dune 1.6) diff --git a/v2/dependency-patches/tch-0.24.0/examples/basics.rs b/v2/dependency-patches/tch-0.24.0/examples/basics.rs deleted file mode 100644 index 469ce73f0c..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/basics.rs +++ /dev/null @@ -1,36 +0,0 @@ -use tch::{kind, Tensor}; - -fn grad_example() { - let mut x = Tensor::from(2.0).set_requires_grad(true); - let y = &x * &x + &x + 36; - println!("{}", y.double_value(&[])); - x.zero_grad(); - y.backward(); - let mut dy_over_dx = x.grad(); - dy_over_dx.print(); - let t2 = dy_over_dx.clamp_(1.23, 1.24); - dy_over_dx.print(); - t2.print(); - println!("Grad {}", dy_over_dx.double_value(&[])); -} - -fn main() { - println!("Cuda available: {}", tch::Cuda::is_available()); - println!("Cudnn available: {}", tch::Cuda::cudnn_is_available()); - let device = tch::Device::cuda_if_available(); - let t = Tensor::from_slice(&[3, 1, 4, 1, 5]).to(device); - t.print(); - let t = Tensor::randn([5, 4], kind::FLOAT_CPU); - t.print(); - (&t + 1.5).print(); - (&t + 2.5).print(); - let mut t = Tensor::from_slice(&[1.1f32, 2.1, 3.1]); - t += 42; - t.print(); - println!("{:?} {}", t.size(), t.double_value(&[1])); - grad_example(); - println!("has_mps: {}", tch::utils::has_mps()); - println!("has_vulkan: {}", tch::utils::has_vulkan()); - println!("version_cudnn: {}", tch::utils::version_cudnn()); - println!("version_cudart: {}", tch::utils::version_cudart()); -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/char-rnn/README.md b/v2/dependency-patches/tch-0.24.0/examples/char-rnn/README.md deleted file mode 100644 index 764e4b06c7..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/char-rnn/README.md +++ /dev/null @@ -1,43 +0,0 @@ -This example implements a character-level language model heavily inspired -by [char-rnn](https://github.com/karpathy/char-rnn). -- For training, the model takes as input a text file and learns to predict the - next character following a given sequence. -- For generation, the model uses a fixed seed, returns the next character - distribution from which a character is randomly sampled and this is iterated - to generate a text. - -At the end of each training epoch, some sample text is generated and printed. - -Any text file can be used as an input, as long as it's large enough for training. -A typical example would be the -[tiny Shakespeare dataset](https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt). -The training text file should be stored in `data/input.txt`. - -Compiling and running the example is done via the following command lines: -```bash -cargo run --example char-rnn -``` - -Here is an example of generated data when training on the Shakespeare dataset after only 5 epochs. -``` -CAMILLO: -Nurse! Wast thou, to eat to be proud it. -My lords, he's gone; hers' falsehsof with a temper, -And all these silk of this issuel, or puck -And the new-vangance is danlike to take and turn; -Let him be through my brother and your arries -It sets do so, a knowledge itself have done! - -HENRY BOLINGBROKE: -Come, come, when, sir, along is war, -Horsein your downrial his worshiph, fair sprenched -feignaconal take for thee, dispatch'd with living -Ingracious enching you. Then, great duke -Can klecked found my propter heart, and fees? - -NORTHUMBERLAND: -Have youths of face, I know, that living sons -Through us within the house of the dear inn: -Before thy fire and loving eyes,-- -``` - diff --git a/v2/dependency-patches/tch-0.24.0/examples/char-rnn/main.rs b/v2/dependency-patches/tch-0.24.0/examples/char-rnn/main.rs deleted file mode 100644 index d9020421a2..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/char-rnn/main.rs +++ /dev/null @@ -1,68 +0,0 @@ -/* This example uses the tinyshakespeare dataset which can be downloaded at: - https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt - - It has been heavily inspired by https://github.com/karpathy/char-rnn -*/ - -use anyhow::Result; -use tch::data::TextData; -use tch::nn::{Linear, Module, OptimizerConfig, LSTM, RNN}; -use tch::{nn, Device, Kind, Tensor}; - -const LEARNING_RATE: f64 = 0.01; -const HIDDEN_SIZE: i64 = 256; -const SEQ_LEN: i64 = 180; -const BATCH_SIZE: i64 = 256; -const EPOCHS: i64 = 100; -const SAMPLING_LEN: i64 = 1024; - -/// Generates some sample string using the LSTM + Linear model. -fn sample(data: &TextData, lstm: &LSTM, linear: &Linear, device: Device) -> String { - let labels = data.labels(); - let mut state = lstm.zero_state(1); - let mut last_label = 0i64; - let mut result = String::new(); - for _index in 0..SAMPLING_LEN { - let input = Tensor::zeros([1, labels], (Kind::Float, device)); - let _ = input.narrow(1, last_label, 1).fill_(1.0); - state = lstm.step(&input, &state); - let sampled_y = linear - .forward(&state.h()) - .squeeze_dim(0) - .softmax(-1, Kind::Float) - .multinomial(1, false); - last_label = i64::try_from(sampled_y).unwrap(); - result.push(data.label_to_char(last_label)) - } - result -} - -pub fn main() -> Result<()> { - let device = Device::cuda_if_available(); - let vs = nn::VarStore::new(device); - let data = TextData::new("data/input.txt")?; - let labels = data.labels(); - println!("Dataset loaded, {labels} labels."); - let lstm = nn::lstm(vs.root(), labels, HIDDEN_SIZE, Default::default()); - let linear = nn::linear(vs.root(), HIDDEN_SIZE, labels, Default::default()); - let mut opt = nn::Adam::default().build(&vs, LEARNING_RATE)?; - for epoch in 1..(1 + EPOCHS) { - let mut sum_loss = 0.; - let mut cnt_loss = 0.; - for batch in data.iter_shuffle(SEQ_LEN + 1, BATCH_SIZE) { - let xs_onehot = batch.narrow(1, 0, SEQ_LEN).onehot(labels); - let ys = batch.narrow(1, 1, SEQ_LEN).to_kind(Kind::Int64); - let (lstm_out, _) = lstm.seq(&xs_onehot.to_device(device)); - let logits = linear.forward(&lstm_out); - let loss = logits - .view([BATCH_SIZE * SEQ_LEN, labels]) - .cross_entropy_for_logits(&ys.to_device(device).view([BATCH_SIZE * SEQ_LEN])); - opt.backward_step_clip(&loss, 0.5); - sum_loss += f64::try_from(loss)?; - cnt_loss += 1.0; - } - println!("Epoch: {} loss: {:5.3}", epoch, sum_loss / cnt_loss); - println!("Sample: {}", sample(&data, &lstm, &linear, device)); - } - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/cifar/main.rs b/v2/dependency-patches/tch-0.24.0/examples/cifar/main.rs deleted file mode 100644 index 0e39c44fdd..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/cifar/main.rs +++ /dev/null @@ -1,72 +0,0 @@ -/* Training various models on the CIFAR-10 dataset. - - The dataset can be downloaded from https://www.cs.toronto.edu/~kriz/cifar.html, files - should be placed in the data/ directory. - - The resnet model reaches 95.4% accuracy. -*/ - -use anyhow::Result; -use tch::nn::{FuncT, ModuleT, OptimizerConfig, SequentialT}; -use tch::{nn, Device}; - -fn conv_bn(vs: &nn::Path, c_in: i64, c_out: i64) -> SequentialT { - let conv2d_cfg = nn::ConvConfig { padding: 1, bias: false, ..Default::default() }; - nn::seq_t() - .add(nn::conv2d(vs, c_in, c_out, 3, conv2d_cfg)) - .add(nn::batch_norm2d(vs, c_out, Default::default())) - .add_fn(|x| x.relu()) -} - -fn layer<'a>(vs: &nn::Path, c_in: i64, c_out: i64) -> FuncT<'a> { - let pre = conv_bn(&vs.sub("pre"), c_in, c_out); - let block1 = conv_bn(&vs.sub("b1"), c_out, c_out); - let block2 = conv_bn(&vs.sub("b2"), c_out, c_out); - nn::func_t(move |xs, train| { - let pre = xs.apply_t(&pre, train).max_pool2d_default(2); - let ys = pre.apply_t(&block1, train).apply_t(&block2, train); - pre + ys - }) -} - -fn fast_resnet(vs: &nn::Path) -> SequentialT { - nn::seq_t() - .add(conv_bn(&vs.sub("pre"), 3, 64)) - .add(layer(&vs.sub("layer1"), 64, 128)) - .add(conv_bn(&vs.sub("inter"), 128, 256)) - .add_fn(|x| x.max_pool2d_default(2)) - .add(layer(&vs.sub("layer2"), 256, 512)) - .add_fn(|x| x.max_pool2d_default(4).flat_view()) - .add(nn::linear(vs.sub("linear"), 512, 10, Default::default())) - .add_fn(|x| x * 0.125) -} - -fn learning_rate(epoch: i64) -> f64 { - if epoch < 50 { - 0.1 - } else if epoch < 100 { - 0.01 - } else { - 0.001 - } -} - -pub fn main() -> Result<()> { - let m = tch::vision::cifar::load_dir("data")?; - let vs = nn::VarStore::new(Device::cuda_if_available()); - let net = fast_resnet(&vs.root()); - let mut opt = - nn::Sgd { momentum: 0.9, dampening: 0., wd: 5e-4, nesterov: true }.build(&vs, 0.)?; - for epoch in 1..150 { - opt.set_lr(learning_rate(epoch)); - for (bimages, blabels) in m.train_iter(64).shuffle().to_device(vs.device()) { - let bimages = tch::vision::dataset::augmentation(&bimages, true, 4, 8); - let loss = net.forward_t(&bimages, true).cross_entropy_for_logits(&blabels); - opt.backward_step(&loss); - } - let test_accuracy = - net.batch_accuracy_for_logits(&m.test_images, &m.test_labels, vs.device(), 512); - println!("epoch: {:4} test acc: {:5.2}%", epoch, 100. * test_accuracy,); - } - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/README.md b/v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/README.md deleted file mode 100644 index aff46c67b3..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/README.md +++ /dev/null @@ -1,21 +0,0 @@ -## Custom Optimizer - Sparse Adam optimizer - -This example implements an Adam optimizer for sparse and dense gradients. This -is useful for large embedding matrices, where the gradient is very sparse. -Instead of updating the whole embedding, just a small portion of the matrix is -updated and therefore a significant speed-up in training time is gained. - -For the dense update step it uses the `addcdiv_` (fraction and then add assign -in-place) and `addcmul_` (multiply and then add assign in-place) functions and -for the sparse part `index_select` and `index_add` to reduce the necessary -work. The sparse gradient update subroutine is only faster for sparse matrices. -There is a `force_sparse` parameter, which enforces that the sparse subroutine -is used, even for dense gradients. This is only for testing purposes, because -the problem is actual of dense nature. - -As a toy example, a MNIST classification problem is solved with help of the -Adam implementation. For further information take a look at the [python -implementation](https://github.com/pytorch/pytorch/blob/master/torch/optim/sparse_adam.py) -and at [Adam: A Method for Stochastic -Optimization](https://arxiv.org/abs/1412.6980). The Adam optimizer should reach -97% in about 170 epochs. diff --git a/v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/main.rs b/v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/main.rs deleted file mode 100644 index 8e40d27cc4..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/main.rs +++ /dev/null @@ -1,50 +0,0 @@ -// This should rearch 97% accuracy. - -mod sparse_adam; - -use anyhow::Result; -use tch::{nn, nn::Module, Device}; - -const IMAGE_DIM: i64 = 784; -const HIDDEN_NODES: i64 = 128; -const LABELS: i64 = 10; - -fn net(vs: &nn::Path) -> impl Module { - nn::seq() - .add(nn::linear(vs / "layer1", IMAGE_DIM, HIDDEN_NODES, Default::default())) - .add_fn(|xs| xs.relu()) - .add(nn::linear(vs, HIDDEN_NODES, LABELS, Default::default())) -} - -pub fn run() -> Result<()> { - let m = tch::vision::mnist::load_dir("data")?; - let vs = nn::VarStore::new(Device::Cpu); - let net = net(&vs.root()); - - // force a sparse update step (in order to test on dense problem) - let force_sparse = false; - // create a custom optimizer with learning rate `0.005`, beta_1 `0.9` and beta2 `0.999` - let mut opt = sparse_adam::SparseAdam::new(&vs, 5e-3, 0.9, 0.999, 1e-8, force_sparse); - - for epoch in 1..200 { - let loss = net.forward(&m.train_images).cross_entropy_for_logits(&m.train_labels); - - // call custom optimizer - opt.zero_grad(); - loss.backward(); - opt.step(); - - let test_accuracy = net.forward(&m.test_images).accuracy_for_logits(&m.test_labels); - println!( - "epoch: {:4} train loss: {:8.5} test acc: {:5.2}%", - epoch, - f64::try_from(&loss)?, - 100. * f64::try_from(&test_accuracy)?, - ); - } - Ok(()) -} - -fn main() { - run().unwrap(); -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/sparse_adam.rs b/v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/sparse_adam.rs deleted file mode 100644 index 3e5a88f474..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/custom-optimizer/sparse_adam.rs +++ /dev/null @@ -1,147 +0,0 @@ -use std::sync::{Arc, Mutex}; -use tch::nn::{VarStore, Variables}; -use tch::{no_grad, Device, Kind, Tensor}; - -/// Buffer of first/second order moment for the Adam optimizer -struct Buffer { - pub first_moment: Tensor, - pub second_moment: Tensor, - idx: usize, -} - -impl Buffer { - pub fn new(size: &[i64]) -> Buffer { - Buffer { - first_moment: Tensor::zeros(size, (Kind::Float, Device::Cpu)), - second_moment: Tensor::zeros(size, (Kind::Float, Device::Cpu)), - idx: 1, - } - } - - // Return and increment the timestep - pub fn inc(&mut self) -> usize { - let old_val = self.idx; - self.idx += 1; - - old_val - } -} - -/// Sparse-Adam optimizer supporting sparse and dense gradients -/// -/// This demonstrates how a custom optimizer can be implemented. It handles a shared copy of the -/// variables store `vars` and updates it by traversing the computation graph. -pub struct SparseAdam { - lr: f64, - beta1: f64, - beta2: f64, - eps: f64, - force_sparse: bool, - - vars: Arc>, - buffers: Vec, -} - -impl SparseAdam { - pub fn new( - vs: &VarStore, - lr: f64, - beta1: f64, - beta2: f64, - eps: f64, - force_sparse: bool, - ) -> SparseAdam { - let vars = vs.variables_.clone(); - - // create buffers for every trainable variable - let buffers = vars - .lock() - .unwrap() - .trainable_variables - .iter() - .map(|x| Buffer::new(&x.tensor.size())) - .collect(); - - SparseAdam { lr, beta1, beta2, eps, force_sparse, vars, buffers } - } - - /// Ensure that the gradient update is not part of the autograd routine - pub fn step(&mut self) { - no_grad(|| self._step()); - } - - pub fn _step(&mut self) { - let mut vars = self.vars.lock().unwrap(); - - // iterate through all trainable variables - for (var, buffer) in vars.trainable_variables.iter_mut().zip(&mut self.buffers) { - let mut grad = var.tensor.grad(); - - // calculate both bias correction values - let buffer_idx = buffer.inc(); - let bias_correction1 = 1.0 - self.beta1.powf(buffer_idx as f64); - let bias_correction2 = 1.0 - self.beta2.powf(buffer_idx as f64); - - // check whether the gradient is sparse - if grad.is_sparse() || self.force_sparse { - // convert matrix to sparse matrix if necessary - if !grad.is_sparse() { - grad = grad.to_sparse_sparse_dim(1); - } - - // deduplicate coordinates in the sparse matrix - let grad = grad.coalesce(); - // get indices and values of the sparse gradient - let indices = grad.indices().squeeze(); - let values = grad.values(); - - // for SGD we would do: - //tensor.index_add_(0, &indices, &(-self.lr * &values)); - - // update both moments - // old = b*old + (1-b) * new <==> old += (1-b) * (new - old) - let update_first_moment = - (1.0 - self.beta1) * (&values - buffer.first_moment.index_select(0, &indices)); - let update_second_moment = (1.0 - self.beta2) - * (&values * &values - buffer.second_moment.index_select(0, &indices)); - - let _ = buffer.first_moment.index_add_(0, &indices, &update_first_moment); - let _ = buffer.second_moment.index_add_(0, &indices, &update_second_moment); - - // first part of update step -lr * m_t / (1-b_1^t) - let part1 = - buffer.first_moment.index_select(0, &indices) * (-self.lr / bias_correction1); - // second part of update step sqrt(v_t / (1-b_2^t)) + eps - let part2 = (buffer.second_moment.index_select(0, &indices) / bias_correction2) - .sqrt() - + self.eps; - - let _ = var.tensor.index_add_(0, &indices, &(part1 / part2)); - } else { - // update first moment - buffer.first_moment *= self.beta1; - buffer.first_moment += (1.0 - self.beta1) * &grad; - // update second raw moment - buffer.second_moment *= self.beta2; - let scaled_grad = grad * (1.0 - self.beta2).sqrt(); - let _ = buffer.second_moment.addcmul_(&scaled_grad, &scaled_grad); - - // first part of update step -lr * m_t / (1-b_1^t) - let part1 = &buffer.first_moment * (-self.lr / bias_correction1); - // second part of update step sqrt(v_t / (1-b_2^t)) + eps - let part2 = (&buffer.second_moment / bias_correction2).sqrt() + self.eps; - - // calculate fraction and update parameters - let _ = var.tensor.addcdiv_(&part1, &part2); - } - } - } - - // zero the gradient of all trainable variables - pub fn zero_grad(&mut self) { - let mut vars = self.vars.lock().unwrap(); - for var in &mut vars.trainable_variables { - var.tensor.zero_grad(); - } - } -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/gan/main.rs b/v2/dependency-patches/tch-0.24.0/examples/gan/main.rs deleted file mode 100644 index c0f14f6bd7..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/gan/main.rs +++ /dev/null @@ -1,148 +0,0 @@ -// Realtivistic DCGAN. -// https://github.com/AlexiaJM/RelativisticGAN -// -// TODO: override the initializations if this does not converge well. -use anyhow::{bail, Result}; -use tch::{kind, nn, nn::OptimizerConfig, Device, Kind, Tensor}; - -const IMG_SIZE: i64 = 64; -const LATENT_DIM: i64 = 128; -const BATCH_SIZE: i64 = 32; -const LEARNING_RATE: f64 = 1e-4; -const BATCHES: i64 = 100000000; - -fn tr2d(p: nn::Path, c_in: i64, c_out: i64, padding: i64, stride: i64) -> nn::ConvTranspose2D { - let cfg = nn::ConvTransposeConfig { stride, padding, bias: false, ..Default::default() }; - nn::conv_transpose2d(p, c_in, c_out, 4, cfg) -} - -fn conv2d(p: nn::Path, c_in: i64, c_out: i64, padding: i64, stride: i64) -> nn::Conv2D { - let cfg = nn::ConvConfig { stride, padding, bias: false, ..Default::default() }; - nn::conv2d(p, c_in, c_out, 4, cfg) -} - -fn generator(p: nn::Path) -> impl nn::ModuleT { - nn::seq_t() - .add(tr2d(&p / "tr1", LATENT_DIM, 1024, 0, 1)) - .add(nn::batch_norm2d(&p / "bn1", 1024, Default::default())) - .add_fn(|xs| xs.relu()) - .add(tr2d(&p / "tr2", 1024, 512, 1, 2)) - .add(nn::batch_norm2d(&p / "bn2", 512, Default::default())) - .add_fn(|xs| xs.relu()) - .add(tr2d(&p / "tr3", 512, 256, 1, 2)) - .add(nn::batch_norm2d(&p / "bn3", 256, Default::default())) - .add_fn(|xs| xs.relu()) - .add(tr2d(&p / "tr4", 256, 128, 1, 2)) - .add(nn::batch_norm2d(&p / "bn4", 128, Default::default())) - .add_fn(|xs| xs.relu()) - .add(tr2d(&p / "tr5", 128, 3, 1, 2)) - .add_fn(|xs| xs.tanh()) -} - -fn leaky_relu(xs: &Tensor) -> Tensor { - xs.maximum(&(xs * 0.2)) -} - -fn discriminator(p: nn::Path) -> impl nn::ModuleT { - nn::seq_t() - .add(conv2d(&p / "conv1", 3, 128, 1, 2)) - .add_fn(leaky_relu) - .add(conv2d(&p / "conv2", 128, 256, 1, 2)) - .add(nn::batch_norm2d(&p / "bn2", 256, Default::default())) - .add_fn(leaky_relu) - .add(conv2d(&p / "conv3", 256, 512, 1, 2)) - .add(nn::batch_norm2d(&p / "bn3", 512, Default::default())) - .add_fn(leaky_relu) - .add(conv2d(&p / "conv4", 512, 1024, 1, 2)) - .add(nn::batch_norm2d(&p / "bn4", 1024, Default::default())) - .add_fn(leaky_relu) - .add(conv2d(&p / "conv5", 1024, 1, 0, 1)) -} - -fn mse_loss(x: &Tensor, y: &Tensor) -> Tensor { - let diff = x - y; - (&diff * &diff).mean(Kind::Float) -} - -// Generate a 2D matrix of images from a tensor with multiple images. -fn image_matrix(imgs: &Tensor, sz: i64) -> Result { - let imgs = ((imgs + 1.) * 127.5).clamp(0., 255.).to_kind(Kind::Uint8); - let mut ys: Vec = vec![]; - for i in 0..sz { - ys.push(Tensor::cat(&(0..sz).map(|j| imgs.narrow(0, 4 * i + j, 1)).collect::>(), 2)) - } - Ok(Tensor::cat(&ys, 3).squeeze_dim(0)) -} - -pub fn main() -> Result<()> { - let device = Device::cuda_if_available(); - let args: Vec<_> = std::env::args().collect(); - let image_dir = match args.as_slice() { - [_, d] => d.to_owned(), - _ => bail!("usage: main image-dataset-dir"), - }; - let images = tch::vision::image::load_dir(image_dir, IMG_SIZE, IMG_SIZE)?; - println!("loaded dataset: {images:?}"); - let train_size = images.size()[0]; - - let random_batch_images = || { - let index = Tensor::randint(train_size, [BATCH_SIZE], kind::INT64_CPU); - images.index_select(0, &index).to_device(device).to_kind(Kind::Float) / 127.5 - 1. - }; - let rand_latent = || { - (Tensor::rand([BATCH_SIZE, LATENT_DIM, 1, 1], kind::FLOAT_CPU) * 2.0 - 1.0) - .to_device(device) - }; - - let mut generator_vs = nn::VarStore::new(device); - let generator = generator(generator_vs.root()); - let mut opt_g = nn::adam(0.5, 0.999, 0.).build(&generator_vs, LEARNING_RATE)?; - - let mut discriminator_vs = nn::VarStore::new(device); - let discriminator = discriminator(discriminator_vs.root()); - let mut opt_d = nn::adam(0.5, 0.999, 0.).build(&discriminator_vs, LEARNING_RATE)?; - - let fixed_noise = rand_latent(); - - for index in 0..BATCHES { - discriminator_vs.unfreeze(); - generator_vs.freeze(); - let discriminator_loss = { - let batch_images = random_batch_images(); - let y_pred = batch_images.apply_t(&discriminator, true); - let y_pred_fake = rand_latent() - .apply_t(&generator, true) - .copy() - .detach() - .apply_t(&discriminator, true); - mse_loss(&y_pred, &(y_pred_fake.mean(Kind::Float) + 1)) - + mse_loss(&y_pred_fake, &(y_pred.mean(Kind::Float) - 1)) - }; - opt_d.backward_step(&discriminator_loss); - - discriminator_vs.freeze(); - generator_vs.unfreeze(); - - let generator_loss = { - let batch_images = random_batch_images(); - let y_pred = batch_images.apply_t(&discriminator, true); - let y_pred_fake = rand_latent().apply_t(&generator, true).apply_t(&discriminator, true); - mse_loss(&y_pred, &(y_pred_fake.mean(Kind::Float) - 1)) - + mse_loss(&y_pred_fake, &(y_pred.mean(Kind::Float) + 1)) - }; - opt_g.backward_step(&generator_loss); - - if index % 1000 == 0 { - let imgs = fixed_noise - .apply_t(&generator, true) - .view([-1, 3, IMG_SIZE, IMG_SIZE]) - .to_device(Device::Cpu); - tch::vision::image::save(&image_matrix(&imgs, 4)?, format!("relout{index}.png"))? - } - if index % 100 == 0 { - println!("{index}") - }; - } - - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit-quantized/README.md b/v2/dependency-patches/tch-0.24.0/examples/jit-quantized/README.md deleted file mode 100644 index 181b1eceba..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/jit-quantized/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# Loading and Running a Quantized PyTorch Model in Rust - -This builds upon the [main JIT tutorial](https://github.com/LaurentMazare/tch-rs/blob/master/examples/jit/README.md) -to load and run quantized PyTorch models. - -After a model has been successfully trained, they must often be deployed to -resource-constrained devices (such as cheap cloud computing instances or mobile -phones). A common technique used to speed up inference on devices lacking a GPU -is to quantize the model's weight matrices and perform some or all of its operations -using INT8 tensors (rather than FP32). - -## Quantizing a Python PyTorch Model -The details of quantizing a PyTorch model are described in the PyTorch [documentation](https://pytorch.org/docs/stable/quantization.html). In this example we simply export a model available within torchvision. -```python -import torch -import torchvision - -model = torchvision.models.quantization.resnet18( - pretrained=True, quantize=True -) -model.eval() -example = torch.rand(1, 3, 224, 224) -traced_script_module = torch.jit.trace(model, example) -traced_script_module.save("resnet18_int8.pt") -``` - -## Loading a Quantized Torch Script Model from Rust -The process for loading a quantized Torch Script model is almost identical to that -outlined in the [main jit tutorial](https://github.com/LaurentMazare/tch-rs/blob/master/examples/jit/README.md). -The only difference is that we must specify a quantization engine (or backend) to -use during inference. The two engines currently supported by PyTorch are [FBGEMM](https://github.com/pytorch/FBGEMM) -(for inference on x86 architectures) and [QNNPACK](https://github.com/pytorch/QNNPACK) -(for inference on ARM architectures). - -This can be done using the following: -```rust -tch::QEngine::FBGEMM.set()?, -// or -tch::QEngine::QNNPACK.set()?, -``` diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit-quantized/main.rs b/v2/dependency-patches/tch-0.24.0/examples/jit-quantized/main.rs deleted file mode 100644 index 02c349f215..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/jit-quantized/main.rs +++ /dev/null @@ -1,51 +0,0 @@ -// This example illustrates how to use a quantized PyTorch model trained and -// exported using the Python JIT API. -// See https://pytorch.org/tutorials/advanced/cpp_export.html and -// https://pytorch.org/docs/stable/quantization.html for more details. -use anyhow::{bail, Result}; -use std::time::SystemTime; -use tch::vision::imagenet; - -const NRUNS: i32 = 10; - -pub fn main() -> Result<()> { - let args: Vec<_> = std::env::args().collect(); - let (model_file, image_file, qengine) = match args.as_slice() { - [_, m, i, q] => (m.to_owned(), i.to_owned(), Some(q.to_owned())), - [_, m, i] => (m.to_owned(), i.to_owned(), None), - _ => bail!("usage: main model.pt image.jpg [fbgemm | qnnpack]"), - }; - - // Set quantization engine - match qengine { - None => (), - Some(qengine) => match qengine.as_str() { - "fbgemm" => tch::QEngine::FBGEMM.set()?, - "qnnpack" => tch::QEngine::QNNPACK.set()?, - _ => bail!("qengine should be one of 'fbgemm' or 'qnnpack' or ommitted"), - }, - }; - - // Load the image file and resize it to the usual imagenet dimension of 224x224. - let image = imagenet::load_image_and_resize224(image_file)?; - - // Load the Python saved module. - let model = tch::CModule::load(model_file)?; - - // Measure the average inference time. - let now = SystemTime::now(); - for _ in 1..NRUNS { - let _output = image.unsqueeze(0).apply(&model); - } - println!("Mean Inference Time: {} ms", now.elapsed().unwrap().as_millis() / NRUNS as u128); - - // Apply the forward pass of the model to get the logits. - let output = image.unsqueeze(0).apply(&model).softmax(-1, tch::Kind::Float); - - // Print the top 5 categories for this image. - println!("Top 5 Predictions:"); - for (probability, class) in imagenet::top(&output, 5).iter() { - println!("{:50} {:5.2}%", class, 100.0 * probability) - } - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit-quantized/resnet.py b/v2/dependency-patches/tch-0.24.0/examples/jit-quantized/resnet.py deleted file mode 100644 index 4b3f79ba67..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/jit-quantized/resnet.py +++ /dev/null @@ -1,22 +0,0 @@ -import torch -import torchvision - - -def save_resnet18_script_module( - filename: str, pretrained: bool, quantize: bool -) -> None: - """Save quantizable resnet18 as traced script module.""" - model = torchvision.models.quantization.resnet18( - pretrained=pretrained, quantize=quantize - ) - model.eval() - example = torch.rand(1, 3, 224, 224) - traced_script_module = torch.jit.trace(model, example) - traced_script_module.save(filename) - - -# Save fp32 model -save_resnet18_script_module("resnet18_fp32.pt", True, False) - -# Save int8 model -save_resnet18_script_module("resnet18_int8.pt", True, True) diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit-trace/README.md b/v2/dependency-patches/tch-0.24.0/examples/jit-trace/README.md deleted file mode 100644 index 1dd829dc63..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/jit-trace/README.md +++ /dev/null @@ -1,27 +0,0 @@ -To run this example, you will have to unzip the [MNIST data -files](http://yann.lecun.com/exdb/mnist/) in `data/`. - -# Saving a trained model by tracing - -This tutorial illustrates how to serialize a model trained in Rust as a TorchScript program. - -## Training and serializing a TorchScript program in Rust. - -By running the following command, you can train a model on the MNIST dataset and serialize it as a TorchScript program in Rust. - -```bash -cargo run --example jit-trace -``` - -It will save the model as `model.pt`. - -## Loading the model in Python - -You can load and run the saved model in Python. -`test.py` is an example Python script to load the model from `model.pt` and run it on the MNIST dataset. - -You can run the script by the following command. -Please ensure that `torch` and `torchvision` are installed in your environment before running. -```bash -python ./examples/jit-trace/test.py -``` diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit-trace/main.rs b/v2/dependency-patches/tch-0.24.0/examples/jit-trace/main.rs deleted file mode 100644 index d27dddf113..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/jit-trace/main.rs +++ /dev/null @@ -1,63 +0,0 @@ -use anyhow::Result; -use tch::{kind::FLOAT_CUDA, nn, nn::ModuleT, nn::OptimizerConfig, CModule, Device, Tensor}; - -#[derive(Debug)] -struct Net { - conv1: nn::Conv2D, - conv2: nn::Conv2D, - fc1: nn::Linear, - fc2: nn::Linear, -} - -impl Net { - fn new(vs: &nn::Path) -> Net { - let conv1 = nn::conv2d(vs, 1, 32, 5, Default::default()); - let conv2 = nn::conv2d(vs, 32, 64, 5, Default::default()); - let fc1 = nn::linear(vs, 1024, 1024, Default::default()); - let fc2 = nn::linear(vs, 1024, 10, Default::default()); - Net { conv1, conv2, fc1, fc2 } - } -} - -impl nn::ModuleT for Net { - fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor { - xs.view([-1, 1, 28, 28]) - .apply(&self.conv1) - .max_pool2d_default(2) - .apply(&self.conv2) - .max_pool2d_default(2) - .view([-1, 1024]) - .apply(&self.fc1) - .relu() - .dropout(0.5, train) - .apply(&self.fc2) - } -} - -fn main() -> Result<()> { - let m = tch::vision::mnist::load_dir("data")?; - let mut vs = nn::VarStore::new(Device::cuda_if_available()); - let net = Net::new(&vs.root()); - let mut opt = nn::Adam::default().build(&vs, 1e-4)?; - for epoch in 1..10 { - for (bimages, blabels) in m.train_iter(256).shuffle().to_device(vs.device()) { - let loss = net.forward_t(&bimages, true).cross_entropy_for_logits(&blabels); - opt.backward_step(&loss); - } - let test_accuracy = - net.batch_accuracy_for_logits(&m.test_images, &m.test_labels, vs.device(), 1024); - println!("epoch: {:4} test acc: {:5.2}%", epoch, 100. * test_accuracy,); - } - - vs.freeze(); - let mut closure = |input: &[Tensor]| vec![net.forward_t(&input[0], false)]; - let model = CModule::create_by_tracing( - "MyModule", - "forward", - &[Tensor::zeros([784], FLOAT_CUDA)], - &mut closure, - )?; - model.save("model.pt")?; - - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit-trace/test.py b/v2/dependency-patches/tch-0.24.0/examples/jit-trace/test.py deleted file mode 100644 index 497187deee..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/jit-trace/test.py +++ /dev/null @@ -1,45 +0,0 @@ -# The following code is partially copied from the official example. -# https://github.com/pytorch/examples/blob/ca1bd9167f7216e087532160fc5b98643d53f87e/mnist/main.py - -import torch -from torchvision import datasets, transforms -import torch.nn.functional as F - - -def test(model, device, test_loader): - model.eval() - test_loss = 0 - correct = 0 - with torch.no_grad(): - for data, target in test_loader: - data, target = data.to(device), target.to(device) - output = model(data) - test_loss += F.nll_loss( - output, target, reduction="sum" - ).item() # sum up batch loss - pred = output.argmax( - dim=1, keepdim=True - ) # get the index of the max log-probability - correct += pred.eq(target.view_as(pred)).sum().item() - - test_loss /= len(test_loader.dataset) - - print( - "\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\n".format( - test_loss, - correct, - len(test_loader.dataset), - 100.0 * correct / len(test_loader.dataset), - ) - ) - - -model = torch.jit.load("model.pt") -device = torch.device("cuda") -test_kwargs = {"batch_size": 100} -transform = transforms.Compose( - [transforms.ToTensor(), transforms.Normalize((0.1307,), (0.3081,))] -) -dataset = datasets.MNIST("./data", train=False, download=True, transform=transform) -test_loader = torch.utils.data.DataLoader(dataset, **test_kwargs) -test(model, device, test_loader) diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit-train/README.md b/v2/dependency-patches/tch-0.24.0/examples/jit-train/README.md deleted file mode 100644 index a07f9ea14a..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/jit-train/README.md +++ /dev/null @@ -1,136 +0,0 @@ -To run this example, you will have to unzip the [MNIST data -files](http://yann.lecun.com/exdb/mnist/) in `data/`. - -# Loading and Training a PyTorch Model in Rust - -In this tutorial this is illustrated using a demo model that has not been pre-trained. -We start by defining and serializing the model using the Python api. -The resulting model file is later loaded from Rust and trained on the MNIST dataset. - -## Converting a Python PyTorch Model to Torch Script - -There are various ways to create the Torch Script as detailed -in the original [tutorial](https://pytorch.org/tutorials/advanced/cpp_export.html). - -Here we will use scripting. The following Python script runs the -defined model on some random image and creates the Torch Script -file based on this evaluation. - -```python -import torch -from torch.nn import Module - - -class DemoModule(Module): - def __init__(self): - super().__init__() - self.batch_norm = torch.nn.BatchNorm2d(1) - self.conv1 = torch.nn.Conv2d(1, 8, kernel_size=(5, 5), padding=(2, 2)) - self.conv2 = torch.nn.Conv2d(8, 16, kernel_size=(5, 5), padding=(2, 2)) - self.flatten = torch.nn.Flatten() - self.dropout = torch.nn.Dropout() - self.linear1 = torch.nn.Linear(16 * 28 * 28, 100) - self.linear2 = torch.nn.Linear(100, 10) - - def forward(self, x): - x = self.batch_norm(x) - x = self.conv1(x) - x = self.conv2(x) - x = self.flatten(x) - x = self.dropout(x) - x = self.linear1(x) - return self.linear2(x) - - -traced_script_module = torch.jit.script(DemoModule()) -traced_script_module.save("model.pt") -``` - -The last line creates the `model.pt` Torch Script file which includes both the model -architecture and the weight values. - -## Loading the Torch Script Model from Rust - -The `model.pt` file can then be loaded and trained from Rust. - -```rust -fn train_and_save_model(dataset: &Dataset, device: Device) -> Result<()> { - let vs = VarStore::new(device); - - let mut trainable = TrainableCModule::load("model.pt", vs.root())?; - trainable.set_train(); - let initial_acc = trainable.batch_accuracy_for_logits( - &dataset.test_images, - &dataset.test_labels, - vs.device(), - 1024, - ); - println!("Initial accuracy: {:5.2}%", 100. * initial_acc); - - let mut opt = Adam::default().build(&vs, 1e-4)?; - for epoch in 1..20 { - for (images, labels) in dataset - .train_iter(128) - .shuffle() - .to_device(vs.device()) - .take(50) - { - let loss = trainable - .forward_t(&images, true) - .cross_entropy_for_logits(&labels); - opt.backward_step(&loss); - } - let test_accuracy = trainable.batch_accuracy_for_logits( - &dataset.test_images, - &dataset.test_labels, - vs.device(), - 1024, - ); - println!("epoch: {:4} test acc: {:5.2}%", epoch, 100. * test_accuracy,); - } - - trainable.save("trained_model.pt")?; - Ok(()) -} -``` - -After the model was trained, the updated Torch Script is saved to file `trained_model.pt`. -This can be used for executing the module later on either via Python or -using Rust as per the following. - -```rust -fn load_trained_and_test_acc(dataset: &Dataset, device: Device) -> Result<()> { - let mut module = CModule::load_on_device("trained_model.pt", device)?; - module.set_eval(); - let accuracy = - module.batch_accuracy_for_logits(&dataset.test_images, &dataset.test_labels, device, 1024); - println!("Updated accuracy: {:5.2}%", 100. * accuracy); - Ok(()) -} -``` - -Below is the typical output that this example would produce. - -``` -Initial accuracy: 13.11% -epoch: 1 test acc: 87.11% -epoch: 2 test acc: 90.16% -epoch: 3 test acc: 90.31% -epoch: 4 test acc: 90.02% -epoch: 5 test acc: 90.66% -epoch: 6 test acc: 91.00% -epoch: 7 test acc: 91.48% -epoch: 8 test acc: 91.10% -epoch: 9 test acc: 91.13% -epoch: 10 test acc: 91.35% -epoch: 11 test acc: 91.04% -epoch: 12 test acc: 91.43% -epoch: 13 test acc: 91.34% -epoch: 14 test acc: 91.05% -epoch: 15 test acc: 91.23% -epoch: 16 test acc: 91.30% -epoch: 17 test acc: 91.45% -epoch: 18 test acc: 91.70% -epoch: 19 test acc: 91.44% -Updated accuracy: 91.93% -``` diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit-train/main.rs b/v2/dependency-patches/tch-0.24.0/examples/jit-train/main.rs deleted file mode 100644 index 8e7f9e3761..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/jit-train/main.rs +++ /dev/null @@ -1,57 +0,0 @@ -use anyhow::Result; -use tch::nn::{Adam, ModuleT, OptimizerConfig, VarStore}; -use tch::vision::dataset::Dataset; -use tch::TrainableCModule; -use tch::{CModule, Device}; - -fn train_and_save_model(dataset: &Dataset, device: Device) -> Result<()> { - let vs = VarStore::new(device); - let mut trainable = TrainableCModule::load("model.pt", vs.root())?; - trainable.set_train(); - let initial_acc = trainable.batch_accuracy_for_logits( - &dataset.test_images, - &dataset.test_labels, - vs.device(), - 1024, - ); - println!("Initial accuracy: {:5.2}%", 100. * initial_acc); - - let mut opt = Adam::default().build(&vs, 1e-4)?; - for epoch in 1..20 { - for (images, labels) in dataset.train_iter(128).shuffle().to_device(vs.device()).take(50) { - let loss = trainable.forward_t(&images, true).cross_entropy_for_logits(&labels); - opt.backward_step(&loss); - } - let test_accuracy = trainable.batch_accuracy_for_logits( - &dataset.test_images, - &dataset.test_labels, - vs.device(), - 1024, - ); - println!("epoch: {:4} test acc: {:5.2}%", epoch, 100. * test_accuracy,); - } - trainable.save("trained_model.pt")?; - Ok(()) -} - -fn load_trained_and_test_acc(dataset: &Dataset, device: Device) -> Result<()> { - let mut module = CModule::load_on_device("trained_model.pt", device)?; - module.set_eval(); - let accuracy = - module.batch_accuracy_for_logits(&dataset.test_images, &dataset.test_labels, device, 1024); - println!("Updated accuracy: {:5.2}%", 100. * accuracy); - Ok(()) -} - -fn main() -> Result<()> { - let dataset = tch::vision::mnist::load_dir("data")?; - let dataset = Dataset { - train_images: dataset.train_images.view([-1, 1, 28, 28]), - test_images: dataset.test_images.view([-1, 1, 28, 28]), - ..dataset - }; - let device = Device::Cpu; - train_and_save_model(&dataset, device)?; - load_trained_and_test_acc(&dataset, device)?; - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit-train/resnet.py b/v2/dependency-patches/tch-0.24.0/examples/jit-train/resnet.py deleted file mode 100644 index fe389d8127..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/jit-train/resnet.py +++ /dev/null @@ -1,27 +0,0 @@ -import torch -from torch.nn import Module - - -class DemoModule(Module): - def __init__(self): - super().__init__() - self.batch_norm = torch.nn.BatchNorm2d(1) - self.conv1 = torch.nn.Conv2d(1, 8, kernel_size=(5, 5), padding=(2, 2)) - self.conv2 = torch.nn.Conv2d(8, 16, kernel_size=(5, 5), padding=(2, 2)) - self.flatten = torch.nn.Flatten() - self.dropout = torch.nn.Dropout() - self.linear1 = torch.nn.Linear(16 * 28 * 28, 100) - self.linear2 = torch.nn.Linear(100, 10) - - def forward(self, x): - x = self.batch_norm(x) - x = self.conv1(x) - x = self.conv2(x) - x = self.flatten(x) - x = self.dropout(x) - x = self.linear1(x) - return self.linear2(x) - - -traced_script_module = torch.jit.script(DemoModule()) -traced_script_module.save("model.pt") diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit/README.md b/v2/dependency-patches/tch-0.24.0/examples/jit/README.md deleted file mode 100644 index 9e2ea5bf36..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/jit/README.md +++ /dev/null @@ -1,116 +0,0 @@ -# Loading and Running a PyTorch Model in Rust - -This tutorial follows the steps of the -[Loading a PyTorch Model in C++ tutorial](https://pytorch.org/tutorials/advanced/cpp_export.html). - -PyTorch models are commonly written and trained in Python. The trained model can then be -serialized in a [Torch Script](https://pytorch.org/docs/stable/jit.html) file. -The Torch Script file contains a description of the model architecture as well as -trained weights. This file can be loaded from Rust to run inference for the saved -model. - -In this tutorial this is illustrated using a ResNet-18 model that has been trained on the -ImageNet dataset. We start by loading and serializing the model using the Python api. -The resulting model file is later loaded from Rust and run on some given image. - -## Converting a Python PyTorch Model to Torch Script - -There are various ways to create the Torch Script as detailed -in the original [tutorial](https://pytorch.org/tutorials/advanced/cpp_export.html). - -Here we will use tracing. The following python script runs the -pre-trained ResNet-18 model on some random image and uses tracing to create -the Torch Script file based on this evaluation. - -```python -import torch -import torchvision - -model = torchvision.models.resnet18(pretrained=True) -model.eval() -example = torch.rand(1, 3, 224, 224) -traced_script_module = torch.jit.trace(model, example) -traced_script_module.save("model.pt") -``` - -Note that `model.eval()` is useful to ensure that the saved model is -in testing mode rather than in training mode. This has an impact on the -batch-norm layers. - -The last line creates the `model.pt` Torch Script file which includes both the model -architecture and the trained weight values. - -## Loading the Torch Script Model from Rust - -The `model.pt` file can then be loaded and executed from Rust. - -```rust -pub fn main() -> anyhow::Result<()> { - let args: Vec<_> = std::env::args().collect(); - let (model_file, image_file) = match args.as_slice() { - [_, m, i] => (m.to_owned(), i.to_owned()), - _ => bail!("usage: main model.pt image.jpg"), - }; - let image = imagenet::load_image_and_resize(image_file)?; - let model = tch::CModule::load(model_file)?; - let output = model.forward_ts(&[image.unsqueeze(0)])?.softmax(-1); - for (probability, class) in imagenet::top(&output, 5).iter() { - println!("{:50} {:5.2}%", class, 100.0 * probability) - } - Ok(()) -} -``` - -Let us have a closer look at what this code is doing. The first couple lines -extract the model and image filenames from command line arguments -Then the image is loaded, resized to 224x224, and converted to a tensor -using ImageNet normalization. - -```rust - let image = imagenet::load_image_and_resize(image_file)?; -``` - -The exported model is loaded. - -```rust - let model = tch::CModule::load(model_file)?; -``` - -Now we can run the model on the image tensor. This returns the logits for each -of the ImageNet 1000 classes. A softmax is applied to get the associated -probabilities. - -```rust - let output = model.forward_ts(&[image.unsqueeze(0)])?.softmax(-1); -``` - -Alternatively, one can write the following instead as `tch::CModule` can be -used as any other module via apply when there is only a single input. -```rust - let output = image.unsqueeze(0).apply(&model).softmax(-1); -``` - -And finally we print the 5 classes with the highest probabilities. -```rust - for (probability, class) in imagenet::top(&output, 5).iter() { - println!("{:50} {:5.2}%", class, 100.0 * probability) - } -``` - -Cargo can be used to run this example. -```bash -cargo run --example jit model.pt image.jpg -``` - -This results in the Rust code printing the top 5 predicted labels -as well as the associated probabilities. - -``` -tiger, Panthera tigris 96.33% -tiger cat 3.56% -zebra 0.09% -jaguar, panther, Panthera onca, Felis onca 0.01% -tabby, tabby cat 0.01% -``` - -![tiger](https://upload.wikimedia.org/wikipedia/commons/thumb/3/3b/Royal_Bengal_Tiger_at_Kanha_National_Park.jpg/800px-Royal_Bengal_Tiger_at_Kanha_National_Park.jpg) diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit/main.rs b/v2/dependency-patches/tch-0.24.0/examples/jit/main.rs deleted file mode 100644 index fb8e2e79d3..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/jit/main.rs +++ /dev/null @@ -1,27 +0,0 @@ -// This example illustrates how to use a PyTorch model trained and exported using the -// Python JIT API. -// See https://pytorch.org/tutorials/advanced/cpp_export.html for more details. -use anyhow::{bail, Result}; -use tch::vision::imagenet; - -pub fn main() -> Result<()> { - let args: Vec<_> = std::env::args().collect(); - let (model_file, image_file) = match args.as_slice() { - [_, m, i] => (m.to_owned(), i.to_owned()), - _ => bail!("usage: main model.pt image.jpg"), - }; - // Load the image file and resize it to the usual imagenet dimension of 224x224. - let image = imagenet::load_image_and_resize224(image_file)?; - - // Load the Python saved module. - let model = tch::CModule::load(model_file)?; - - // Apply the forward pass of the model to get the logits. - let output = image.unsqueeze(0).apply(&model).softmax(-1, tch::Kind::Float); - - // Print the top 5 categories for this image. - for (probability, class) in imagenet::top(&output, 5).iter() { - println!("{:50} {:5.2}%", class, 100.0 * probability) - } - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/jit/resnet.py b/v2/dependency-patches/tch-0.24.0/examples/jit/resnet.py deleted file mode 100644 index eaaa834f80..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/jit/resnet.py +++ /dev/null @@ -1,8 +0,0 @@ -import torch -import torchvision - -model = torchvision.models.resnet18(pretrained=True) -model.eval() -example = torch.rand(1, 3, 224, 224) -traced_script_module = torch.jit.trace(model, example) -traced_script_module.save("model.pt") diff --git a/v2/dependency-patches/tch-0.24.0/examples/llama/convert_checkpoint.py b/v2/dependency-patches/tch-0.24.0/examples/llama/convert_checkpoint.py deleted file mode 100644 index 423034304f..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/llama/convert_checkpoint.py +++ /dev/null @@ -1,63 +0,0 @@ -# Adapted from https://github.com/Lightning-AI/lit-llama/blob/main/scripts/convert_checkpoint.py -import sys -import torch -from typing import Dict -from pathlib import Path -from safetensors.torch import save_file - -def convert_state_dict(state_dict: Dict[str, torch.Tensor], dtype: torch.dtype = torch.float16) -> Dict[str, torch.Tensor]: - print("start conv") - - def get_and_remove(key): - v = state_dict[key].to(dtype) - del state_dict[key] - return v - - converted = {} - converted["transformer.wte.weight"] = get_and_remove("tok_embeddings.weight") - converted["lm_head.weight"] = get_and_remove("output.weight") - converted["transformer.ln_f.scale"] = get_and_remove("norm.weight") - - for layer_idx in sorted(set([k.split(".")[1] for k in state_dict if k.startswith("layers")])): - print(layer_idx) - - # attention - # the wq, wk, wv from the FB model are stacked in our model as c_attn - converted[f"transformer.h.{layer_idx}.attn.c_attn.weight"] = torch.cat( - ( - get_and_remove(f"layers.{layer_idx}.attention.wq.weight"), - get_and_remove(f"layers.{layer_idx}.attention.wk.weight"), - get_and_remove(f"layers.{layer_idx}.attention.wv.weight"), - ) - ) - converted[f"transformer.h.{layer_idx}.attn.c_proj.weight"] = get_and_remove( - f"layers.{layer_idx}.attention.wo.weight" - ) - # mlp - converted[f"transformer.h.{layer_idx}.mlp.c_fc1.weight"] = get_and_remove( - f"layers.{layer_idx}.feed_forward.w1.weight" - ) - converted[f"transformer.h.{layer_idx}.mlp.c_proj.weight"] = get_and_remove( - f"layers.{layer_idx}.feed_forward.w2.weight" - ) - converted[f"transformer.h.{layer_idx}.mlp.c_fc2.weight"] = get_and_remove( - f"layers.{layer_idx}.feed_forward.w3.weight" - ) - # rms norm - converted[f"transformer.h.{layer_idx}.rms_1.scale"] = get_and_remove(f"layers.{layer_idx}.attention_norm.weight") - converted[f"transformer.h.{layer_idx}.rms_2.scale"] = get_and_remove(f"layers.{layer_idx}.ffn_norm.weight") - return converted - -def convert_weights(llama_ckpt, *, output_st: Path = Path("llama.safetensors"), dtype: str = "float16") -> None: - dt = getattr(torch, dtype, None) - if not isinstance(dt, torch.dtype): - raise ValueError(f"{dtype} is not a valid dtype.") - checkpoint = torch.load(llama_ckpt, map_location="cpu") - converted = convert_state_dict(checkpoint, dtype=dt) - del checkpoint - save_file(converted, output_st) - -if __name__ == "__main__": - if len(sys.argv) != 2: - raise ValueError(f"usage: convert_checkpoint.py ..../LLaMA/7B/consolidated.00.pth") - convert_weights(sys.argv[1]) diff --git a/v2/dependency-patches/tch-0.24.0/examples/llama/main.rs b/v2/dependency-patches/tch-0.24.0/examples/llama/main.rs deleted file mode 100644 index 3303859c06..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/llama/main.rs +++ /dev/null @@ -1,397 +0,0 @@ -// An implementation of LLaMA https://github.com/facebookresearch/llama -// This only contains the inference part. -// -// This is based on nanoGPT in a similar way to: -// https://github.com/Lightning-AI/lit-llama/blob/main/lit_llama/model.py -// -// The tokenizer config can be retrieved from: -// https://huggingface.co/hf-internal-testing/llama-tokenizer/blob/main/tokenizer.json -// -// In order to convert the llama weights to a .safetensors file, run: -// python examples/llama/convert_checkpoint.py ..../LLaMA/7B/consolidated.00.pth -use anyhow::Result; -use clap::Parser; -use tch::nn::{self, Module}; -use tch::{Device, Kind, Tensor}; - -mod sentencepiece; -use sentencepiece::Tokenizer; - -const CONTEXT_SIZE: usize = 512; -const START_PROMPT: &str = r" -EDWARD: -I wonder how our princely father 'scaped, -Or whether he be 'scaped away or no -From Clifford's and Northumberland's pursuit: -Had he been ta'en, we should have heard the news; -Had he been slain, we should have heard the news; -Or had he 'scaped, methinks we should have heard -The happy tidings of his good escape. -How fares my brother? why is he so sad? - -RICHARD: -I cannot joy, until I be resolved -Where our right valiant father is become. -I saw him in the battle range about; -And watch'd him how he singled Clifford forth. -Methought he bore him in the thickest troop -As doth a lion in a herd of neat; -Or as a bear, encompass'd round with dogs, -Who having pinch'd a few and made them cry, -The rest stand all aloof, and bark at him. -So fared our father with his enemies; -So fled his enemies my warlike father: -Methinks, 'tis prize enough to be his son. -See how the morning opes her golden gates, -And takes her farewell of the glorious sun! -How well resembles it the prime of youth, -Trimm'd like a younker prancing to his love! - -EDWARD: -Dazzle mine eyes, or do I see three suns? - -RICHARD: -Three glorious suns, each one a perfect sun; -Not separated with the racking clouds, -But sever'd in a pale clear-shining sky. -See, see! they join, embrace, and seem to kiss, -As if they vow'd some league inviolable: -Now are they but one lamp, one light, one sun. -In this the heaven figures some event. - -EDWARD: -'Tis wondrous strange, the like yet never heard of. -I think it cites us, brother, to the field, -That we, the sons of brave Plantagenet, -Each one already blazing by our meeds, -Should notwithstanding join our lights together -And over-shine the earth as this the world. -Whate'er it bodes, henceforward will I bear -Upon my target three fair-shining suns. -"; - -#[allow(dead_code)] -struct Config { - block_size: usize, - vocab_size: usize, - n_layer: usize, - n_head: usize, - n_embd: usize, -} - -#[allow(dead_code)] -impl Config { - fn config_7b() -> Self { - Self { block_size: 4096, vocab_size: 32000, n_layer: 32, n_head: 32, n_embd: 4096 } - } - - fn config_13b() -> Self { - Self { block_size: 4096, vocab_size: 32000, n_layer: 40, n_head: 40, n_embd: 5120 } - } - - fn config_30b() -> Self { - Self { block_size: 4096, vocab_size: 32000, n_layer: 60, n_head: 52, n_embd: 6656 } - } - - fn config_65b() -> Self { - Self { block_size: 4096, vocab_size: 32000, n_layer: 80, n_head: 64, n_embd: 8192 } - } -} - -#[derive(Debug)] -struct RmsNorm { - scale: Tensor, - size: i64, -} - -impl RmsNorm { - fn new(vs: nn::Path, size: i64) -> Self { - let scale = vs.zeros("scale", &[size]); - Self { scale, size } - } -} - -impl Module for RmsNorm { - fn forward(&self, xs: &Tensor) -> Tensor { - let norm_xs = (xs * xs).mean_dim(-1, true, Kind::Float); - let xs_normed = xs * (norm_xs + 1e-5).rsqrt(); - let scale = self.scale.reshape([1, 1, self.size]); - scale * xs_normed - } -} - -#[derive(Debug)] -struct Mlp { - c_fc1: nn::Linear, - c_fc2: nn::Linear, - c_proj: nn::Linear, -} - -impl Mlp { - fn new(vs: nn::Path, n_embd: i64) -> Self { - let n_hidden = 8 * n_embd / 3; - let n_hidden = (n_hidden - 1) / 256 * 256 + 256; - let c = nn::LinearConfig { bias: false, ..Default::default() }; - let c_fc1 = nn::linear(&vs / "c_fc1", n_embd, n_hidden, c); - let c_fc2 = nn::linear(&vs / "c_fc2", n_embd, n_hidden, c); - let c_proj = nn::linear(&vs / "c_proj", n_hidden, n_embd, c); - Self { c_fc1, c_fc2, c_proj } - } -} - -impl Module for Mlp { - fn forward(&self, xs: &Tensor) -> Tensor { - let xs = xs.apply(&self.c_fc1).silu() * xs.apply(&self.c_fc2); - xs.apply(&self.c_proj) - } -} - -#[derive(Debug)] -struct CausalSelfAttention { - c_attn: nn::Linear, - c_proj: nn::Linear, - n_head: i64, - n_embd: i64, - device: Device, -} - -impl CausalSelfAttention { - fn new(vs: nn::Path, n_head: i64, n_embd: i64) -> Self { - let c = nn::LinearConfig { bias: false, ..Default::default() }; - let c_attn = nn::linear(&vs / "c_attn", n_embd, 3 * n_embd, c); - let c_proj = nn::linear(&vs / "c_proj", n_embd, n_embd, c); - Self { c_attn, c_proj, n_head, n_embd, device: vs.device() } - } - - fn apply_rotary_emb(&self, x: &Tensor, freqs_cis: &Tensor) -> Tensor { - let mut dims = x.size(); - let v = dims.pop().unwrap(); - dims.push(v / 2); - dims.push(2); - let x = x.reshape(&dims); - let re_x = x.slice(-1, 0, 1, 1); - let im_x = x.slice(-1, 1, 2, 1); - let re_f = freqs_cis.slice(-1, 0, 1, 1); - let im_f = freqs_cis.slice(-1, 1, 2, 1); - let re = &re_x * &re_f - &im_x * &im_f; - let im = &re_x * &im_f + &im_x * &re_f; - let rope = Tensor::cat(&[&re, &im], -1); - // TODO: Add the flatten op. - let mut dims = rope.size(); - let v1 = dims.pop().unwrap(); - let v2 = dims.pop().unwrap(); - dims.push(v1 * v2); - rope.reshape(&dims) - } - - fn forward(&self, x: &Tensor, freqs_cis: &Tensor) -> Tensor { - let (b, t, c) = x.size3().unwrap(); - let kind = x.kind(); - let qkv = self.c_attn.forward(x); - let n_embd = self.n_embd; - let q = qkv.slice(2, 0, n_embd, 1); - let k = qkv.slice(2, n_embd, 2 * n_embd, 1); - let v = qkv.slice(2, 2 * n_embd, 3 * n_embd, 1); - let target_dim = [b, t, self.n_head, c / self.n_head]; - let k = k.reshape(target_dim).transpose(1, 2); - let q = q.reshape(target_dim).transpose(1, 2); - let v = v.reshape(target_dim).transpose(1, 2); - let q = self.apply_rotary_emb(&q, freqs_cis); - let k = self.apply_rotary_emb(&k, freqs_cis); - let k_shape = k.size(); - let att: Tensor = q.matmul(&k.transpose(-2, -1)) / (*k_shape.last().unwrap() as f64).sqrt(); - let mask = Tensor::ones([t, t], (kind, self.device)).tril(0).reshape([1, 1, t, t]); - let att = att.masked_fill(&mask.eq(0.), f64::NEG_INFINITY); - let y = att.softmax(-1, kind).matmul(&v); - let y = y.transpose(1, 2).reshape([b, t, c]); - self.c_proj.forward(&y) - } -} - -#[derive(Debug)] -struct Block { - rms_1: RmsNorm, - attn: CausalSelfAttention, - rms_2: RmsNorm, - mlp: Mlp, -} - -impl Block { - fn new(vs: nn::Path, config: &Config) -> Self { - let rms_1 = RmsNorm::new(&vs / "rms_1", config.n_embd as i64); - let attn = - CausalSelfAttention::new(&vs / "attn", config.n_head as i64, config.n_embd as i64); - let rms_2 = RmsNorm::new(&vs / "rms_2", config.n_embd as i64); - let mlp = Mlp::new(&vs / "mlp", config.n_embd as i64); - Self { rms_1, attn, rms_2, mlp } - } - - fn forward(&self, x: &Tensor, freqs_cis: &Tensor) -> Tensor { - let x = self.attn.forward(&self.rms_1.forward(x), freqs_cis) + x; - self.mlp.forward(&self.rms_2.forward(&x)) + x - } -} - -#[derive(Debug)] -struct Llama { - wte: nn::Embedding, - blocks: Vec, - ln_f: RmsNorm, - lm_head: nn::Linear, -} - -impl Llama { - fn new(vs: nn::Path, config: &Config) -> Self { - let c = nn::LinearConfig { bias: false, ..Default::default() }; - let lm_head = - nn::linear(&vs / "lm_head", config.n_embd as i64, config.vocab_size as i64, c); - let wte = nn::embedding( - &vs / "transformer" / "wte", - config.vocab_size as i64, - config.n_embd as i64, - Default::default(), - ); - let blocks = (0..config.n_layer) - .map(|i| Block::new(&vs / "transformer" / "h" / i, config)) - .collect::>(); - let ln_f = RmsNorm::new(&vs / "transformer" / "ln_f", config.n_embd as i64); - Self { wte, blocks, ln_f, lm_head } - } - - fn forward(&self, x: &Tensor, freqs_cis: &Tensor) -> Tensor { - let (_, t) = x.size2().unwrap(); - let mut x = self.wte.forward(x); - for block in self.blocks.iter() { - x = block.forward(&x, freqs_cis); - } - let x = self.ln_f.forward(&x); - let x = x.slice(1, t - 1, t, 1); - self.lm_head.forward(&x) - } -} - -fn precompute_freqs_cis(config: &Config) -> Tensor { - let seq_len = CONTEXT_SIZE; - let n_elem = config.n_embd / config.n_head; - let theta: Vec<_> = - (0..n_elem).step_by(2).map(|i| 1f32 / 10000f32.powf(i as f32 / n_elem as f32)).collect(); - let arange: Vec<_> = (0..seq_len).map(|c| c as f32).collect(); - let theta = Tensor::from_slice(&theta); - let arange = Tensor::from_slice(&arange); - let idx_theta = arange.outer(&theta); - let shape = [1, 1, seq_len as i64, n_elem as i64 / 2, 1]; - let idx_theta_cos = idx_theta.cos().reshape(shape); - let idx_theta_sin = idx_theta.sin().reshape(shape); - Tensor::cat(&[&idx_theta_cos, &idx_theta_sin], -1) -} - -fn llama(vs: nn::Path, args: Args) -> impl Module { - let config = Config::config_7b(); - let freqs_cis = precompute_freqs_cis(&config).to_device(vs.device()); - let llama = Llama::new(vs, &config); - nn::func(move |xs| { - let logits = llama.forward(xs, &freqs_cis); - (logits / args.temperature).softmax(-1, Kind::Float) - }) -} - -#[derive(Debug, Clone, Copy, clap::ValueEnum)] -enum CompKind { - Float, - Half, - #[clap(name = "bfloat16")] - BFloat16, - #[clap(name = "qint8")] - QInt8, -} - -impl CompKind { - fn to_kind(self) -> Kind { - match self { - Self::Float => Kind::Float, - Self::Half => Kind::Half, - Self::BFloat16 => Kind::BFloat16, - Self::QInt8 => Kind::QInt8, - } - } -} - -#[derive(clap::Parser, Debug, Clone, Copy)] -#[command(author, version, about, long_about = None)] -struct Args { - /// Run on CPU rather than on GPU. - #[arg(long)] - cpu: bool, - - /// Use this type of float for computations, float/half/bfloat16/... - #[arg(long, value_enum, default_value = "float")] - kind: CompKind, - - /// The temperature used to generate samples. - #[arg(long, default_value_t = 1.0)] - temperature: f64, - - /// The length of the sample to generate (in tokens). - #[arg(long, default_value_t = 100)] - sample_len: usize, -} - -fn main() -> Result<()> { - let _no_grad = tch::no_grad_guard(); - let args = Args::parse(); - let tokenizer = Tokenizer::from_file("llama-tokenizer.json")?; - let mut tokens = tokenizer.encode(START_PROMPT)?; - let mut new_tokens = vec![]; - let start_build = std::time::Instant::now(); - let device = if args.cpu { - Device::Cpu - } else if tch::utils::has_mps() { - Device::Mps - } else { - Device::Cuda(0) - }; - let mut vs = nn::VarStore::new(device); - let llama = llama(vs.root(), args); - // When loading the weights, assume that they are in the float16 - // format. - vs.set_kind(Kind::Half); - println!("generated the model in {:?}", start_build.elapsed()); - - let start_load = std::time::Instant::now(); - // Instead of the standard `VarStore::load`, we use a mmap version that - // should improve memory efficiency. - { - let file = std::fs::File::open("llama.safetensors")?; - let content = unsafe { memmap2::MmapOptions::new().map(&file)? }; - let safetensors = safetensors::SafeTensors::deserialize(&content)?; - - let mut variables = vs.variables_.lock().unwrap(); - for (name, var) in variables.named_variables.iter_mut() { - let view = safetensors.tensor(name)?; - let size: Vec = view.shape().iter().map(|&x| x as i64).collect(); - let kind: Kind = view.dtype().try_into()?; - // Using from_blob here instead of from_data_size avoids some unnecessary copy. - let src_tensor = - unsafe { Tensor::from_blob(view.data().as_ptr(), &size, &[], kind, Device::Cpu) }; - var.f_copy_(&src_tensor)?; - } - } - println!("loaded weights in {:?}", start_load.elapsed()); - - vs.set_kind(args.kind.to_kind()); - println!("switched to {:?}", args.kind); - - for index in 0..args.sample_len { - let ctxt: Vec<_> = - tokens[tokens.len().saturating_sub(CONTEXT_SIZE)..].iter().map(|c| *c as i64).collect(); - let ctxt = Tensor::from_slice(&ctxt).reshape([1, -1]); - let logits = llama.forward(&ctxt); - let sampled_y = logits.get(0).get(0).multinomial(1, true); - let next_token = i64::try_from(&sampled_y)? as usize; - tokens.push(next_token); - new_tokens.push(next_token); - println!("{} token: {} '{}'", index + 1, next_token, tokenizer.decode(&[next_token])); - } - println!("----\n{}\n----", tokenizer.decode(&new_tokens)); - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/llama/sentencepiece.rs b/v2/dependency-patches/tch-0.24.0/examples/llama/sentencepiece.rs deleted file mode 100644 index 3eb6f0e55b..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/llama/sentencepiece.rs +++ /dev/null @@ -1,149 +0,0 @@ -// A very naive sentencepiece encoder/decoder, this only supports the BPE model and not the unigram -// one. -use anyhow::{bail, Context, Result}; -use std::collections::{HashMap, HashSet}; -pub struct Tokenizer { - encoder: HashMap, usize>, - // TODO: Maybe use a vec instead of a hashmap? - decoder: HashMap, - bpe_ranks: HashMap<(Vec, Vec), usize>, -} - -const DELIM: char = '▁'; - -impl Tokenizer { - pub fn from_file>(path: P) -> Result { - let reader = std::io::BufReader::new(std::fs::File::open(path)?); - let config: serde_json::Value = serde_json::from_reader(reader)?; - let model = config.get("model").context("no model key")?; - let type_ = - model.get("type").context("no model.type key")?.as_str().context("not a string")?; - if type_ != "BPE" { - bail!(format!("model type is not BPE: {type_}")) - } - let vocab = model - .get("vocab") - .context("no model.vocab key")? - .as_object() - .context("model.vocab not an object")?; - let single_chars: HashSet = vocab - .iter() - .filter_map(|(key, _)| { - let b = key.as_bytes(); - if b.len() == 1 { - Some(b[0]) - } else { - None - } - }) - .collect(); - let encoder = vocab - .iter() - .rev() - .map(|(key, value)| { - let key = key - .strip_prefix("<0x") - .and_then(|s| s.strip_suffix('>')) - .and_then(|s| u8::from_str_radix(s, 16).ok()) - .and_then(|s| if single_chars.contains(&s) { None } else { Some(s) }) - .map_or_else(|| key.as_bytes().to_vec(), |s| vec![s]); - value.as_i64().context("not an int").map(|v| (key, v as usize)) - }) - .collect::>>()?; - let bpe_ranks = model - .get("merges") - .context("no model.merges key")? - .as_array() - .context("model.merges not an array")? - .iter() - .enumerate() - .map(|(i, value)| { - let value = value.as_str().context("not a string")?; - match value.split_once(' ') { - Some((v1, v2)) => { - let key = (v1.as_bytes().to_vec(), v2.as_bytes().to_vec()); - Ok((key, i)) - } - None => bail!(format!("no space in merge '{value}'")), - } - }) - .collect::>>()?; - let decoder = encoder - .iter() - .map(|(k, v)| (*v, String::from_utf8_lossy(k).replace(DELIM, " "))) - .collect(); - Ok(Self { encoder, decoder, bpe_ranks }) - } - - fn get_pairs(word: &[Vec]) -> HashSet<(Vec, Vec)> { - let mut pairs = HashSet::new(); - for (i, v) in word.iter().enumerate() { - if i > 0 { - pairs.insert((word[i - 1].clone(), v.clone())); - } - } - pairs - } - - fn bpe(&self, s: &str) -> Vec { - let mut buffer = [0u8; 4]; - let mut word: Vec> = vec![]; - for c in s.chars() { - let buffer = c.encode_utf8(&mut buffer); - word.push(buffer.as_bytes().to_vec()) - } - if word.is_empty() { - return Vec::new(); - } - while word.len() > 1 { - let mut current_min = None; - let pairs = Self::get_pairs(&word); - for p in pairs.iter() { - match self.bpe_ranks.get(p) { - None => {} - Some(v) => { - let should_replace = match current_min { - None => true, - Some((current_min, _)) => v < current_min, - }; - if should_replace { - current_min = Some((v, p)) - } - } - } - } - let (first, second) = match current_min { - None => break, - Some((_v, (first, second))) => (first, second), - }; - let mut new_word = vec![]; - let mut index = 0; - while index < word.len() { - let w = &word[index]; - if index + 1 < word.len() && w == first && &word[index + 1] == second { - let mut first_and_second = first.clone(); - first_and_second.extend_from_slice(second); - new_word.push(first_and_second); - index += 2 - } else { - new_word.push(w.clone()); - index += 1 - } - } - word = new_word - } - word.iter().filter_map(|x| self.encoder.get(x)).copied().collect() - } - - // Run bpe on the whole string, very very inefficient but should be good enough for - // prompts. The original string should first be split on whitespace/... - pub fn encode(&self, s: &str) -> Result> { - let mut buffer = [0u8; 4]; - let s = format!("{DELIM}{}", s.replace(' ', DELIM.encode_utf8(&mut buffer))); - Ok(self.bpe(&s)) - } - - pub fn decode(&self, tokens: &[usize]) -> String { - tokens.iter().map(|token| self.decoder[token].as_str()).collect() - } -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/memory_test.rs b/v2/dependency-patches/tch-0.24.0/examples/memory_test.rs deleted file mode 100644 index f2bd0cae8b..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/memory_test.rs +++ /dev/null @@ -1,19 +0,0 @@ -// This example is useful to check for memory leaks. -// A large number of tensors are created either on the cpu or on the gpu and one -// can monitor the main memory usage or the gpu memory at the same time. -use tch::Tensor; - -fn main() { - let a: Vec = std::env::args().collect(); - let device = match a.iter().map(|x| x.as_str()).collect::>().as_slice() { - [_] => tch::Device::Cpu, - [_, "cpu"] => tch::Device::Cpu, - [_, "gpu"] => tch::Device::Cuda(0), - _ => panic!("usage: main cpu|gpu"), - }; - let slice = vec![0; 1_000_000]; - for i in 1..1_000_000 { - let t = Tensor::from_slice(&slice).to_device(device); - println!("{} {:?}", i, t.size()) - } -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/min-gpt/README.md b/v2/dependency-patches/tch-0.24.0/examples/min-gpt/README.md deleted file mode 100644 index 33e94f2647..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/min-gpt/README.md +++ /dev/null @@ -1,78 +0,0 @@ -## minGPT-rs - -This example implements a character-level language model using -a simplified version of [minGPT](https://github.com/karpathy/minGPT). -All the model details as well as the training loop can be found -in `main.rs`. - -Besides the model being different, this example is in line with the LSTM -version [char-rnn](https://github.com/LaurentMazare/tch-rs/tree/master/examples/char-rnn). -The same comments apply: -- For training, the model takes as input a text file and learns to predict the - next character following a given sequence. -- For generation, the model uses a fixed seed, returns the next character - distribution from which a character is randomly sampled and this is iterated - to generate a text. - -At the end of each training epoch, some sample text is generated and printed. - -Any text file can be used as an input, as long as it's large enough for training. -A typical example would be the -[tiny Shakespeare dataset](https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt). -The training text file should be stored in `data/input.txt`. - -Compiling and running the example is done via the following command lines: -```bash -cargo run --example min-gpt -``` - -Here is an example of generated data when training on the Shakespeare dataset at -the end of the first epoch. - -``` -LORD: -My nown of none, by God's mother, -No more of this country, nor than I -Who have strengthen lamented to make it stay -there; and yet it will no more but abide. - -AUTOLYCUS: -Very true, and but a month old. - -DORCAS: -Bless me from marrying a usurer! - -AUTOLYCUS: -Here's the midwife's name to't, one Mistress -Overdone's own house, for here be made. - -EXETER: -The doubt is that which calls so coldly? - -GLOUCESTER: -I grant ye. - -LADY ANNE: -Dost grant me, hedgehog? then, God grant me too -Thou mayst be damned for that wicked deed! -O, he was gentle, mild, and virtuous! - -GLOUCESTER: -The fitter for the King of heaven, that hath him. - -LADY ANNE: -He is in heaven, where thou shalt never come. - -GLOUCESTER: -Let him thank me, or I am sure yet; -But send me France and my true Jove's hands. - -WARWICK: -And I choose Clarence only for protector. - -KING HENRY VI: -Warwick and Clarence give me both your hands: -Now join your hands together, and your father's death -And hardening of my brother's son -It reep her to crave the hire, here my mother -``` diff --git a/v2/dependency-patches/tch-0.24.0/examples/min-gpt/main.rs b/v2/dependency-patches/tch-0.24.0/examples/min-gpt/main.rs deleted file mode 100644 index 66ddd985b2..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/min-gpt/main.rs +++ /dev/null @@ -1,212 +0,0 @@ -/* This example uses the tinyshakespeare dataset which can be downloaded at: - https://raw.githubusercontent.com/karpathy/char-rnn/master/data/tinyshakespeare/input.txt - - This is mostly a rust port of https://github.com/karpathy/minGPT -*/ - -use anyhow::{bail, Result}; -use tch::data::TextData; -use tch::nn::{ModuleT, OptimizerConfig}; -use tch::{nn, Device, IndexOp, Kind, Tensor}; - -const LEARNING_RATE: f64 = 0.0003; -const BLOCK_SIZE: i64 = 128; -const BATCH_SIZE: i64 = 64; -const EPOCHS: i64 = 100; -const SAMPLING_LEN: i64 = 4096; - -#[derive(Debug, Copy, Clone)] -struct Config { - vocab_size: i64, - n_embd: i64, - n_head: i64, - n_layer: i64, - block_size: i64, - attn_pdrop: f64, - resid_pdrop: f64, - embd_pdrop: f64, -} - -// Weight decay only applies to the weight matrixes in the linear layers -const NO_WEIGHT_DECAY_GROUP: usize = 0; -const WEIGHT_DECAY_GROUP: usize = 1; - -// Custom linear layer so that different groups can be used for weight -// and biases. -#[derive(Debug)] -struct Linear { - pub ws: Tensor, - pub bs: Tensor, -} - -impl nn::Module for Linear { - fn forward(&self, xs: &Tensor) -> Tensor { - xs.matmul(&self.ws.tr()) + &self.bs - } -} - -fn linear(vs: nn::Path, in_dim: i64, out_dim: i64) -> Linear { - let wd = vs.set_group(WEIGHT_DECAY_GROUP); - let no_wd = vs.set_group(NO_WEIGHT_DECAY_GROUP); - Linear { - ws: wd.randn("weight", &[out_dim, in_dim], 0.0, 0.02), - bs: no_wd.zeros("bias", &[out_dim]), - } -} - -fn linear_no_bias(vs: nn::Path, in_dim: i64, out_dim: i64) -> Linear { - let wd = vs.set_group(WEIGHT_DECAY_GROUP); - let no_wd = vs.set_group(NO_WEIGHT_DECAY_GROUP); - Linear { - ws: wd.randn("weight", &[out_dim, in_dim], 0.0, 0.02), - bs: no_wd.zeros_no_train("bias", &[out_dim]), - } -} - -fn causal_self_attention(p: &nn::Path, cfg: Config) -> impl ModuleT { - let key = linear(p / "key", cfg.n_embd, cfg.n_embd); - let query = linear(p / "query", cfg.n_embd, cfg.n_embd); - let value = linear(p / "value", cfg.n_embd, cfg.n_embd); - let proj = linear(p / "proj", cfg.n_embd, cfg.n_embd); - let mask_init = - Tensor::ones([cfg.block_size, cfg.block_size], (Kind::Float, p.device())).tril(0); - let mask_init = mask_init.view([1, 1, cfg.block_size, cfg.block_size]); - // let mask = p.var_copy("mask", &mask_init); - let mask = mask_init; - nn::func_t(move |xs, train| { - let (sz_b, sz_t, sz_c) = xs.size3().unwrap(); - let sizes = [sz_b, sz_t, cfg.n_head, sz_c / cfg.n_head]; - let k = xs.apply(&key).view(sizes).transpose(1, 2); - let q = xs.apply(&query).view(sizes).transpose(1, 2); - let v = xs.apply(&value).view(sizes).transpose(1, 2); - let att = q.matmul(&k.transpose(-2, -1)) * (1.0 / f64::sqrt(sizes[3] as f64)); - let att = att.masked_fill(&mask.i((.., .., ..sz_t, ..sz_t)).eq(0.), f64::NEG_INFINITY); - let att = att.softmax(-1, Kind::Float).dropout(cfg.attn_pdrop, train); - let ys = att.matmul(&v).transpose(1, 2).contiguous().view([sz_b, sz_t, sz_c]); - ys.apply(&proj).dropout(cfg.resid_pdrop, train) - }) -} - -fn block(p: &nn::Path, cfg: Config) -> impl ModuleT { - let ln1 = nn::layer_norm(p / "ln1", vec![cfg.n_embd], Default::default()); - let ln2 = nn::layer_norm(p / "ln2", vec![cfg.n_embd], Default::default()); - let attn = causal_self_attention(p, cfg); - let lin1 = linear(p / "lin1", cfg.n_embd, 4 * cfg.n_embd); - let lin2 = linear(p / "lin2", 4 * cfg.n_embd, cfg.n_embd); - nn::func_t(move |xs, train| { - let xs = xs + xs.apply(&ln1).apply_t(&attn, train); - let ys = - xs.apply(&ln2).apply(&lin1).gelu("none").apply(&lin2).dropout(cfg.resid_pdrop, train); - xs + ys - }) -} - -fn gpt(p: nn::Path, cfg: Config) -> impl ModuleT { - let p = &p.set_group(NO_WEIGHT_DECAY_GROUP); - let tok_emb = nn::embedding(p / "tok_emb", cfg.vocab_size, cfg.n_embd, Default::default()); - let pos_emb = p.zeros("pos_emb", &[1, cfg.block_size, cfg.n_embd]); - let ln_f = nn::layer_norm(p / "ln_f", vec![cfg.n_embd], Default::default()); - let head = linear_no_bias(p / "head", cfg.n_embd, cfg.vocab_size); - let mut blocks = nn::seq_t(); - for block_idx in 0..cfg.n_layer { - blocks = blocks.add(block(&(p / block_idx), cfg)); - } - nn::func_t(move |xs, train| { - let (_sz_b, sz_t) = xs.size2().unwrap(); - let tok_emb = xs.apply(&tok_emb); - let pos_emb = pos_emb.i((.., ..sz_t, ..)); - (tok_emb + pos_emb) - .dropout(cfg.embd_pdrop, train) - .apply_t(&blocks, train) - .apply(&ln_f) - .apply(&head) - }) -} - -/// Generates some sample string using the GPT model. -fn sample(data: &TextData, gpt: &impl ModuleT, input: Tensor) -> String { - let mut input = input; - let mut result = String::new(); - for _index in 0..SAMPLING_LEN { - let logits = input.apply_t(gpt, false).i((0, -1, ..)); - let sampled_y = logits.softmax(-1, Kind::Float).multinomial(1, true); - let last_label = i64::try_from(&sampled_y).unwrap(); - result.push(data.label_to_char(last_label)); - input = Tensor::cat(&[input, sampled_y.view([1, 1])], 1).narrow(1, 1, BLOCK_SIZE); - } - result -} - -pub fn main() -> Result<()> { - let device = Device::cuda_if_available(); - let mut vs = nn::VarStore::new(device); - let data = TextData::new("data/input.txt")?; - let labels = data.labels(); - println!("Dataset loaded, {labels} labels."); - let cfg = Config { - vocab_size: labels, - n_embd: 512, - n_head: 8, - n_layer: 8, - block_size: BLOCK_SIZE, - attn_pdrop: 0.1, - resid_pdrop: 0.1, - embd_pdrop: 0.1, - }; - let gpt = gpt(vs.root() / "gpt", cfg); - let args: Vec<_> = std::env::args().collect(); - if args.len() < 2 { - bail!("usage: main (train|predict weights.ot seqstart)") - } - match args[1].as_str() { - "train" => { - let mut opt = nn::AdamW::default().build(&vs, LEARNING_RATE)?; - opt.set_weight_decay_group(NO_WEIGHT_DECAY_GROUP, 0.0); - opt.set_weight_decay_group(WEIGHT_DECAY_GROUP, 0.1); - let mut idx = 0; - for epoch in 1..(1 + EPOCHS) { - let mut sum_loss = 0.; - let mut cnt_loss = 0.; - for batch in data.iter_shuffle(BLOCK_SIZE + 1, BATCH_SIZE) { - let xs = batch.narrow(1, 0, BLOCK_SIZE).to_kind(Kind::Int64).to_device(device); - let ys = batch.narrow(1, 1, BLOCK_SIZE).to_kind(Kind::Int64).to_device(device); - let logits = xs.apply_t(&gpt, true); - let loss = logits - .view([BATCH_SIZE * BLOCK_SIZE, labels]) - .cross_entropy_for_logits(&ys.view([BATCH_SIZE * BLOCK_SIZE])); - opt.backward_step_clip(&loss, 0.5); - sum_loss += f64::try_from(loss)?; - cnt_loss += 1.0; - idx += 1; - if idx % 10000 == 0 { - println!("Epoch: {} loss: {:5.3}", epoch, sum_loss / cnt_loss); - let input = Tensor::zeros([1, BLOCK_SIZE], (Kind::Int64, device)); - println!("Sample: {}", sample(&data, &gpt, input)); - if let Err(err) = vs.save(format!("gpt{idx}.ot")) { - println!("error while saving {err}"); - } - sum_loss = 0.; - cnt_loss = 0.; - } - } - } - } - "predict" => { - vs.load(args[2].as_str())?; - let seqstart = args[3].as_str(); - let input = Tensor::zeros([1, BLOCK_SIZE], (Kind::Int64, device)); - for (idx, c) in seqstart.chars().rev().enumerate() { - let idx = idx as i64; - if idx >= BLOCK_SIZE { - break; - } - let _filled = - input.i((0, BLOCK_SIZE - 1 - idx)).fill_(data.char_to_label(c)? as i64); - } - println!("Sample: {}", sample(&data, &gpt, input)); - } - _ => bail!("usage: main (train|predict weights.ot seqstart)"), - }; - - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/mnist/README.md b/v2/dependency-patches/tch-0.24.0/examples/mnist/README.md deleted file mode 100644 index b9e7d5409c..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/mnist/README.md +++ /dev/null @@ -1,100 +0,0 @@ -The examples in this directory have been adapted from the [TensorFlow -tutorials](https://www.tensorflow.org/versions/r0.7/tutorials/mnist/pros/index.html). -To execute these examples, you will have to unzip the [MNIST data -files](http://yann.lecun.com/exdb/mnist/) in `data/`. - -## Linear Classifier - -The code can be found in `mnist_linear.ml`. - -We first load the MNIST data. This is done using the MNIST helper module, -labels are returned as a vector of integer. Train images and labels are used -when training the model. Test images and labels are used to estimate the -validation error. - -```rust -let m = vision::mnist::load_dir("data").unwrap(); -``` - -After that two tensors are initialized to hold the weights and biases for the -linear classifier. `requires_grad` is used when creating the tensors to inform -torch that we will compute some gradients with respect to these tensors. - -```rust -let mut ws = Tensor::zeros(&[IMAGE_DIM, LABELS], kind::FLOAT_CPU).set_requires_grad(true); -let mut bs = Tensor::zeros(&[LABELS], kind::FLOAT_CPU).set_requires_grad(true); -``` - -Using these the model is defined as multiplying an input by the weight matrix -and adding the bias. A softmax function is used to transform the output into a -probability distribution. - -```rust - let logits = m.train_images.mm(&ws) + &bs; -``` - -We use gradient descent to minimize cross-entropy with respect to variables -`ws` and `bs` and iterate this a couple hundred times. - -Rather than using an optimizer we perform the gradient descent updates manually. -This is only to illustrate how gradients can be computed and used. Other examples -such as `mnist_nn.rs` or `mnist_conv.rs` use an Adam optimizer. - -```rust - for epoch in 1..200 { - let logits = m.train_images.mm(&ws) + &bs; - let loss = logits.log_softmax(-1).nll_loss(&m.train_labels); - ws.zero_grad(); - bs.zero_grad(); - loss.backward(); - no_grad(|| { - ws += ws.grad() * (-1); - bs += bs.grad() * (-1); - }); -``` - -Running this code should build a model that has ~92% accuracy. - -## A Simple Neural-Network - -The code can be found in `mnist_nn.rs`, accuracy should reach ~96%. - -## Convolutional Neural-Network - -The code can be found in `mnist_conv.rs`, accuracy should reach ~99%. - -When building models with multiple weights and bias parameters we use -a variable store to keep track of these variables and let the optimizer -know about them. The variable store is created in the first line of the -following snippet, then the model is built using this variable store -and finally we create an optimizer that will performs gradient descent -over the parameters that have been added to the variable store when -creating the network. - -```rust - let vs = nn::VarStore::new(Device::cuda_if_available()); - let net = Net::new(&vs.root()); - let opt = nn::Optimizer::adam(&vs, 1e-4, Default::default()); -``` - -Note that this will automatically run on a GPU when available. -For a convolutional model we cannot run on all the training images in -a single step as this would requires lots of memory. Instead we -run on mini-batches. An iterator makes it easy to loop over all -the training images shuffled and grouped by mini-batches. -This is done in the main training loop: - -```rust - for epoch in 1..100 { - for (bimages, blabels) in m.train_iter(256).shuffle().to_device(vs.device()) { - let loss = net - .forward_t(&bimages, true) - .cross_entropy_for_logits(&blabels); - opt.backward_step(&loss); - } - let test_accuracy = - net.batch_accuracy_for_logits(&m.test_images, &m.test_labels, vs.device(), 1024); - println!("epoch: {:4} test acc: {:5.2}%", epoch, 100. * test_accuracy,); - } -``` - diff --git a/v2/dependency-patches/tch-0.24.0/examples/mnist/main.rs b/v2/dependency-patches/tch-0.24.0/examples/mnist/main.rs deleted file mode 100644 index 88e0483991..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/mnist/main.rs +++ /dev/null @@ -1,25 +0,0 @@ -/* Some very simple models trained on the MNIST dataset. - The 4 following dataset files can be downloaded from http://yann.lecun.com/exdb/mnist/ - These files should be extracted in the 'data' directory. - train-images-idx3-ubyte.gz - train-labels-idx1-ubyte.gz - t10k-images-idx3-ubyte.gz - t10k-labels-idx1-ubyte.gz -*/ - -use anyhow::Result; - -mod mnist_conv; -mod mnist_linear; -mod mnist_nn; - -fn main() -> Result<()> { - let args: Vec = std::env::args().collect(); - let model = if args.len() < 2 { None } else { Some(args[1].as_str()) }; - match model { - None => mnist_nn::run(), - Some("linear") => mnist_linear::run(), - Some("conv") => mnist_conv::run(), - Some(_) => mnist_nn::run(), - } -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_conv.rs b/v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_conv.rs deleted file mode 100644 index f4b366946a..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_conv.rs +++ /dev/null @@ -1,54 +0,0 @@ -// CNN model. This should rearch 99.1% accuracy. - -use anyhow::Result; -use tch::{nn, nn::ModuleT, nn::OptimizerConfig, Device, Tensor}; - -#[derive(Debug)] -struct Net { - conv1: nn::Conv2D, - conv2: nn::Conv2D, - fc1: nn::Linear, - fc2: nn::Linear, -} - -impl Net { - fn new(vs: &nn::Path) -> Net { - let conv1 = nn::conv2d(vs, 1, 32, 5, Default::default()); - let conv2 = nn::conv2d(vs, 32, 64, 5, Default::default()); - let fc1 = nn::linear(vs, 1024, 1024, Default::default()); - let fc2 = nn::linear(vs, 1024, 10, Default::default()); - Net { conv1, conv2, fc1, fc2 } - } -} - -impl nn::ModuleT for Net { - fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor { - xs.view([-1, 1, 28, 28]) - .apply(&self.conv1) - .max_pool2d_default(2) - .apply(&self.conv2) - .max_pool2d_default(2) - .view([-1, 1024]) - .apply(&self.fc1) - .relu() - .dropout(0.5, train) - .apply(&self.fc2) - } -} - -pub fn run() -> Result<()> { - let m = tch::vision::mnist::load_dir("data")?; - let vs = nn::VarStore::new(Device::cuda_if_available()); - let net = Net::new(&vs.root()); - let mut opt = nn::Adam::default().build(&vs, 1e-4)?; - for epoch in 1..100 { - for (bimages, blabels) in m.train_iter(256).shuffle().to_device(vs.device()) { - let loss = net.forward_t(&bimages, true).cross_entropy_for_logits(&blabels); - opt.backward_step(&loss); - } - let test_accuracy = - net.batch_accuracy_for_logits(&m.test_images, &m.test_labels, vs.device(), 1024); - println!("epoch: {:4} test acc: {:5.2}%", epoch, 100. * test_accuracy,); - } - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_linear.rs b/v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_linear.rs deleted file mode 100644 index cd21117c34..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_linear.rs +++ /dev/null @@ -1,42 +0,0 @@ -// This should rearch 91.5% accuracy. - -use anyhow::Result; -use tch::{kind, no_grad, vision, Kind, Tensor}; - -const IMAGE_DIM: i64 = 784; -const LABELS: i64 = 10; - -pub fn run() -> Result<()> { - let m = vision::mnist::load_dir("data")?; - println!("train-images: {:?}", m.train_images.size()); - println!("train-labels: {:?}", m.train_labels.size()); - println!("test-images: {:?}", m.test_images.size()); - println!("test-labels: {:?}", m.test_labels.size()); - let mut ws = Tensor::zeros([IMAGE_DIM, LABELS], kind::FLOAT_CPU).set_requires_grad(true); - let mut bs = Tensor::zeros([LABELS], kind::FLOAT_CPU).set_requires_grad(true); - for epoch in 1..200 { - let logits = m.train_images.mm(&ws) + &bs; - let loss = logits.log_softmax(-1, Kind::Float).nll_loss(&m.train_labels); - ws.zero_grad(); - bs.zero_grad(); - loss.backward(); - no_grad(|| { - ws += ws.grad() * (-1); - bs += bs.grad() * (-1); - }); - let test_logits = m.test_images.mm(&ws) + &bs; - let test_accuracy = test_logits - .argmax(Some(-1), false) - .eq_tensor(&m.test_labels) - .to_kind(Kind::Float) - .mean(Kind::Float) - .double_value(&[]); - println!( - "epoch: {:4} train loss: {:8.5} test acc: {:5.2}%", - epoch, - loss.double_value(&[]), - 100. * test_accuracy - ); - } - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_nn.rs b/v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_nn.rs deleted file mode 100644 index 0c799633a3..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/mnist/mnist_nn.rs +++ /dev/null @@ -1,34 +0,0 @@ -// This should rearch 97% accuracy. - -use anyhow::Result; -use tch::{nn, nn::Module, nn::OptimizerConfig, Device}; - -const IMAGE_DIM: i64 = 784; -const HIDDEN_NODES: i64 = 128; -const LABELS: i64 = 10; - -fn net(vs: &nn::Path) -> impl Module { - nn::seq() - .add(nn::linear(vs / "layer1", IMAGE_DIM, HIDDEN_NODES, Default::default())) - .add_fn(|xs| xs.relu()) - .add(nn::linear(vs, HIDDEN_NODES, LABELS, Default::default())) -} - -pub fn run() -> Result<()> { - let m = tch::vision::mnist::load_dir("data")?; - let vs = nn::VarStore::new(Device::Cpu); - let net = net(&vs.root()); - let mut opt = nn::Adam::default().build(&vs, 1e-3)?; - for epoch in 1..200 { - let loss = net.forward(&m.train_images).cross_entropy_for_logits(&m.train_labels); - opt.backward_step(&loss); - let test_accuracy = net.forward(&m.test_images).accuracy_for_logits(&m.test_labels); - println!( - "epoch: {:4} train loss: {:8.5} test acc: {:5.2}%", - epoch, - f64::try_from(&loss)?, - 100. * f64::try_from(&test_accuracy)?, - ); - } - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/neural-style-transfer/README.md b/v2/dependency-patches/tch-0.24.0/examples/neural-style-transfer/README.md deleted file mode 100644 index 2f6a8e0010..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/neural-style-transfer/README.md +++ /dev/null @@ -1,162 +0,0 @@ -## Neural Style Transfer in Rust - -This example shows how to implement -[Neural Style Transfer](https://arxiv.org/abs/1508.06576) using PyTorch and Rust, -it is very close to the -[Neural Transfer using PyTorch tutorial](https://pytorch.org/tutorials/advanced/neural_style_tutorial.html) -which contains far more details. - -The Neural Style Transfer algorithm takes as input a content image and a style image and -produces some new images looking similar to the content image but using the style -from the second image. - -### Running the Code -We use a pre-trained VGG-16 convolutional network, the weights can be found in -[vgg16.ot](https://github.com/LaurentMazare/tch-rs/releases/download/mw/vgg16.ot). -Running the example can be done with the following command, this will create -some new images in the current directory. - -```bash -cargo run --example neural-style-transfer style.jpg content.jpg vgg16.ot -``` - -Here is some example content and output images using [the Starry Night](https://en.wikipedia.org/wiki/The_Starry_Night) -as a style image. - -![20 Fenchurch Street](https://raw.githubusercontent.com/LaurentMazare/ocaml-torch/master/examples/neural_transfer/20fen.jpg) -![Starry 20 Fen](https://raw.githubusercontent.com/LaurentMazare/ocaml-torch/master/examples/neural_transfer/20fen-starry.jpg) - -([original image source](https://commons.wikimedia.org/wiki/File:Walkie-Talkie_-_Sept_2015.jpg)) - -### Neural Style Transfer - -The Neural Style Transfer algorithm works by gradient descent starting from an -initial image -- we use the content image as a starting point but could also -use random noise. - -This gradient descent aims at minimizing a loss composed of two parts: the -style loss evaluates how much our current image is similar to the style image -in terms of style; the content loss focuses on whether our current -image has a content close to the reference content image. - -In order to define these distances we use a pre-trained convolutional neural -network. This network is run on the current image as well as on the style and -content images. Intuitively the features extracted by the lower layers are -more about style and the features extracted by the upper layers about content. - -The content loss is computed using the mean-square error on the extracted -features for some upper layers. This error helps ensuring that the extracted -features happen at the same on-screen places between the content and the -current images. - -For the style loss we do not really care about spatial location of the style -features. So rather than the mean-square error we use some Gram matrix to -define a loss that is location independent. - -```rust -fn gram_matrix(m: &Tensor) -> Tensor { - let (a, b, c, d) = m.size4().unwrap(); - let m = m.view(&[a * b, c * d]); - let g = m.matmul(&m.tr()); - g / (a * b * c * d) -} - -fn style_loss(m1: &Tensor, m2: &Tensor) -> Tensor { - gram_matrix(m1).mse_loss(&gram_matrix(m2), 1) -} -``` - -### Loading the Model and Images - -We start by creating a *device* which can be either a CPU or a GPU depending on -what is available. - -```rust - let device = Device::cuda_if_available(); -``` - -Then we create a variable store on this device, this variable store will be -used to hold the pre-trained variables for the VGG-16 model. Using this -variable store we create the model and load the weights from the weight file. -Finally we *freeze* the variable store ensuring that the model weights are not -modified when running the optimizer later in the process. - -```rust - let mut net_vs = tch::nn::VarStore::new(device); - let net = vgg::vgg16(&net_vs.root(), imagenet::CLASS_COUNT); - net_vs.load(weights_file)?; - net_vs.freeze(); -``` - -Next we load the style and content image, and move them to the device. Note -that the `unsqueeze` method is used to add a batch-dimension of size 1 so that -the model can be run on these images. - -```rust - let style_img = imagenet::load_image(style_img)? - .unsqueeze(0) - .to_device(device); - let content_img = imagenet::load_image(content_img)? - .unsqueeze(0) - .to_device(device); -``` - -### Running the Model on the Content and Style Images - -We now run the model on the style and content images. These calls returns some -vector containing the extracted features for each of the model layers. -```rust - let style_layers = net.forward_all_t(&style_img, false, Some(max_layer)); - let content_layers = net.forward_all_t(&content_img, false, Some(max_layer)); -``` - -As we will use gradient descent to optimize an image, we create a second -variable store to hold this image as a variable. The initial value for this -variable is obtained by copying the content image. - -```rust - let vs = nn::VarStore::new(device); - let input_var = vs.root().var_copy("img", &content_img); -``` - -### Gradient Descent Loop - -We use Adam for optimization: creating an optimizer requires to pass it the -variable store that we want to optimize on. - -```rust - let opt = nn::Adam::default().build(&vs, LEARNING_RATE)?; -``` - -In the gradient descent loop the style and content losses are computed, -summed together, and run an Adam optimization step is performed. -We also regularly print the current loss value and write the current image -to a file. - -```rust - for step_idx in 1..(1 + TOTAL_STEPS) { - // ... compute style_loss and content_loss ... - let loss = style_loss * STYLE_WEIGHT + content_loss; - opt.backward_step(&loss); - if step_idx % 1000 == 0 { - println!("{} {}", step_idx, f64::from(loss)); - imagenet::save_image(&input_var, &format!("out{}.jpg", step_idx))?; - } - } -``` - -In order to compute the losses, the model is evaluated on the current image. -The style loss is then extracted from the style layers and the content loss from -the content layers. - -```rust - let input_layers = net.forward_all_t(&input_var, false, Some(max_layer)); - let style_loss: Tensor = STYLE_INDEXES - .iter() - .map(|&i| style_loss(&input_layers[i], &style_layers[i])) - .sum(); - let content_loss: Tensor = CONTENT_INDEXES - .iter() - .map(|&i| input_layers[i].mse_loss(&content_layers[i], 1)) - .sum(); -``` diff --git a/v2/dependency-patches/tch-0.24.0/examples/neural-style-transfer/main.rs b/v2/dependency-patches/tch-0.24.0/examples/neural-style-transfer/main.rs deleted file mode 100644 index 0951e51243..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/neural-style-transfer/main.rs +++ /dev/null @@ -1,72 +0,0 @@ -// This is inspired by the Neural Style tutorial from PyTorch.org -// https://pytorch.org/tutorials/advanced/neural_style_tutorial.html -// The pre-trained weights for the VGG16 model can be downloaded from: -// https://github.com/LaurentMazare/tch-rs/releases/download/mw/vgg16.ot -use anyhow::{bail, Result}; -use tch::vision::{imagenet, vgg}; -use tch::{nn, nn::OptimizerConfig, Device, Tensor}; - -const STYLE_WEIGHT: f64 = 1e6; -const LEARNING_RATE: f64 = 1e-1; -const TOTAL_STEPS: i64 = 3000; -const STYLE_INDEXES: [usize; 5] = [0, 2, 5, 7, 10]; -const CONTENT_INDEXES: [usize; 1] = [7]; - -fn gram_matrix(m: &Tensor) -> Tensor { - let (a, b, c, d) = m.size4().unwrap(); - let m = m.view([a * b, c * d]); - let g = m.matmul(&m.tr()); - g / (a * b * c * d) -} - -fn style_loss(m1: &Tensor, m2: &Tensor) -> Tensor { - gram_matrix(m1).mse_loss(&gram_matrix(m2), tch::Reduction::Mean) -} - -pub fn main() -> Result<()> { - let device = Device::cuda_if_available(); - let args: Vec<_> = std::env::args().collect(); - let (style_img, content_img, weights) = match args.as_slice() { - [_, s, c, w] => (s.to_owned(), c.to_owned(), w.to_owned()), - _ => bail!("usage: main style.jpg content.jpg vgg16.ot"), - }; - - let mut net_vs = tch::nn::VarStore::new(device); - let net = vgg::vgg16(&net_vs.root(), imagenet::CLASS_COUNT); - net_vs.load(&weights).unwrap_or_else(|_| panic!("Could not load weights file {}", &weights)); - net_vs.freeze(); - - let style_img = imagenet::load_image(&style_img) - .unwrap_or_else(|_| panic!("Could not load the style file {}", &style_img)) - .unsqueeze(0) - .to_device(device); - let content_img = imagenet::load_image(&content_img) - .unwrap_or_else(|_| panic!("Could not load the content file {}", &content_img)) - .unsqueeze(0) - .to_device(device); - let max_layer = STYLE_INDEXES.iter().max().unwrap() + 1; - let style_layers = net.forward_all_t(&style_img, false, Some(max_layer)); - let content_layers = net.forward_all_t(&content_img, false, Some(max_layer)); - - let vs = nn::VarStore::new(device); - let input_var = vs.root().var_copy("img", &content_img); - let mut opt = nn::Adam::default().build(&vs, LEARNING_RATE)?; - - for step_idx in 1..(1 + TOTAL_STEPS) { - let input_layers = net.forward_all_t(&input_var, false, Some(max_layer)); - let style_loss: Tensor = - STYLE_INDEXES.iter().map(|&i| style_loss(&input_layers[i], &style_layers[i])).sum(); - let content_loss: Tensor = CONTENT_INDEXES - .iter() - .map(|&i| input_layers[i].mse_loss(&content_layers[i], tch::Reduction::Mean)) - .sum(); - let loss = style_loss * STYLE_WEIGHT + content_loss; - opt.backward_step(&loss); - if step_idx % 1000 == 0 { - println!("{} {}", step_idx, f64::try_from(loss)?); - imagenet::save_image(&input_var, format!("out{step_idx}.jpg"))?; - } - } - - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/pretrained-models/main.rs b/v2/dependency-patches/tch-0.24.0/examples/pretrained-models/main.rs deleted file mode 100644 index 3984bfda36..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/pretrained-models/main.rs +++ /dev/null @@ -1,81 +0,0 @@ -// This example illustrates how to use pre-trained vision models. -// model to get the imagenet label for some image. -// -// The pre-trained weight files containing the pre-trained weights can be found here: -// https://github.com/LaurentMazare/tch-rs/releases/download/mw/resnet18.ot -// https://github.com/LaurentMazare/tch-rs/releases/download/mw/resnet34.ot -// https://github.com/LaurentMazare/tch-rs/releases/download/mw/densenet121.ot -// https://github.com/LaurentMazare/tch-rs/releases/download/mw/vgg13.ot -// https://github.com/LaurentMazare/tch-rs/releases/download/mw/vgg16.ot -// https://github.com/LaurentMazare/tch-rs/releases/download/mw/vgg19.ot -// https://github.com/LaurentMazare/tch-rs/releases/download/mw/squeezenet1_0.ot -// https://github.com/LaurentMazare/tch-rs/releases/download/mw/squeezenet1_1.ot -// https://github.com/LaurentMazare/tch-rs/releases/download/mw/alexnet.ot -// https://github.com/LaurentMazare/tch-rs/releases/download/mw/inception-v3.ot -// https://github.com/LaurentMazare/tch-rs/releases/download/mw/mobilenet-v2.ot -// https://github.com/LaurentMazare/ocaml-torch/releases/download/v0.1-unstable/efficientnet-b0.safetensors -// https://github.com/LaurentMazare/ocaml-torch/releases/download/v0.1-unstable/efficientnet-b1.safetensors -// https://github.com/LaurentMazare/ocaml-torch/releases/download/v0.1-unstable/efficientnet-b2.safetensors -// https://github.com/LaurentMazare/ocaml-torch/releases/download/v0.1-unstable/efficientnet-b3.safetensors -// https://github.com/LaurentMazare/ocaml-torch/releases/download/v0.1-unstable/efficientnet-b4.safetensors -// https://github.com/LaurentMazare/ocaml-torch/releases/download/v0.1-unstable/convmixer1536_20.ot -// https://github.com/LaurentMazare/ocaml-torch/releases/download/v0.1-unstable/convmixer1024_20.ot -// In order to obtain the dinov2 weights, e.g. dinov2_vits14.safetensors, run the -// src/vision/export_dinov2.py -use anyhow::{bail, Context, Result}; -use tch::nn::ModuleT; -use tch::vision::{ - alexnet, convmixer, densenet, dinov2, efficientnet, imagenet, inception, mobilenet, resnet, - squeezenet, vgg, -}; - -pub fn main() -> Result<()> { - let args: Vec<_> = std::env::args().collect(); - let (weights, image) = match args.as_slice() { - [_, w, i] => (std::path::Path::new(w), i.to_owned()), - _ => bail!("usage: main resnet18.ot image.jpg"), - }; - // Load the image file and resize it to the usual imagenet dimension of 224x224. - let image = imagenet::load_image_and_resize224(image)?; - - // Create the model and load the weights from the file. - let mut vs = tch::nn::VarStore::new(tch::Device::Cpu); - let net: Box = - match weights.file_stem().context("no stem")?.to_str().context("invalid stem")? { - "resnet18" => Box::new(resnet::resnet18(&vs.root(), imagenet::CLASS_COUNT)), - "resnet34" => Box::new(resnet::resnet34(&vs.root(), imagenet::CLASS_COUNT)), - "densenet121" => Box::new(densenet::densenet121(&vs.root(), imagenet::CLASS_COUNT)), - "vgg13" => Box::new(vgg::vgg13(&vs.root(), imagenet::CLASS_COUNT)), - "vgg16" => Box::new(vgg::vgg16(&vs.root(), imagenet::CLASS_COUNT)), - "vgg19" => Box::new(vgg::vgg19(&vs.root(), imagenet::CLASS_COUNT)), - "squeezenet1_0" => Box::new(squeezenet::v1_0(&vs.root(), imagenet::CLASS_COUNT)), - "squeezenet1_1" => Box::new(squeezenet::v1_1(&vs.root(), imagenet::CLASS_COUNT)), - "alexnet" => Box::new(alexnet::alexnet(&vs.root(), imagenet::CLASS_COUNT)), - "inception-v3" => Box::new(inception::v3(&vs.root(), imagenet::CLASS_COUNT)), - "mobilenet-v2" => Box::new(mobilenet::v2(&vs.root(), imagenet::CLASS_COUNT)), - "efficientnet-b0" => Box::new(efficientnet::b0(&vs.root(), imagenet::CLASS_COUNT)), - // Maybe the higher resolution models should be handled differently. - "efficientnet-b1" => Box::new(efficientnet::b1(&vs.root(), imagenet::CLASS_COUNT)), - "efficientnet-b2" => Box::new(efficientnet::b2(&vs.root(), imagenet::CLASS_COUNT)), - "efficientnet-b3" => Box::new(efficientnet::b3(&vs.root(), imagenet::CLASS_COUNT)), - "efficientnet-b4" => Box::new(efficientnet::b4(&vs.root(), imagenet::CLASS_COUNT)), - "efficientnet-b5" => Box::new(efficientnet::b5(&vs.root(), imagenet::CLASS_COUNT)), - "efficientnet-b6" => Box::new(efficientnet::b6(&vs.root(), imagenet::CLASS_COUNT)), - "efficientnet-b7" => Box::new(efficientnet::b7(&vs.root(), imagenet::CLASS_COUNT)), - "convmixer1536_20" => Box::new(convmixer::c1536_20(&vs.root(), imagenet::CLASS_COUNT)), - "convmixer1024_20" => Box::new(convmixer::c1024_20(&vs.root(), imagenet::CLASS_COUNT)), - "dinov2_vits14" => Box::new(dinov2::vit_small(&vs.root())), - s => bail!("unknown model for {s}, use a weight file named e.g. resnet18.ot"), - }; - vs.load(weights)?; - - // Apply the forward pass of the model to get the logits. - let output = - net.forward_t(&image.unsqueeze(0), /* train= */ false).softmax(-1, tch::Kind::Float); // Convert to probability. - - // Print the top 5 categories for this image. - for (probability, class) in imagenet::top(&output, 5).iter() { - println!("{:50} {:5.2}%", class, 100.0 * probability) - } - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/README.md b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/README.md deleted file mode 100644 index 7fe1c1735e..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/README.md +++ /dev/null @@ -1,36 +0,0 @@ -# Reinforcement Learning Examples - -These examples illustrate how to implement a couple reinforcement learning -algorithms to play Atari games. -This uses the [OpenAI Gym](https://github.com/openai/gym) through its Python -api so the gym Python package has to be installed. -The Python library is executed in Rust using -[rust-cpython](https://github.com/dgrunwald/rust-cpython). - -The policy gradient example uses the `CartPole-v0` environment. It can be -run using the following command. - -```bash -cargo run --example reinforcement-learning --features=python pg -``` - -The A2C example can use any Atari game supported in the OpenAI gym. -Training can be launched by running: - -```bash -cargo run --example reinforcement-learning --features=python a2c -``` - -This produces some weight files in the current directory. - -If you get some Python import error when using conda, this can -be related to `rust-cpython` not being able to use the Python -runtime as a shared library. `rust-cpython` uses -`Py_ENABLE_SHARED` to determine the link mode which does not -seem to work properly with conda as reported in this -[issue](https://github.com/conda/conda-build/issues/2738). - -## A2C Agent Playing Breakout -[![Breakout](https://img.youtube.com/vi/Zk6j7fC1C6M/0.jpg)](https://www.youtube.com/watch?v=Zk6j7fC1C6M) -## A2C Agent Playing SpaceInvaders -[![SpaceInvaders](https://img.youtube.com/vi/p16n4w3aE8k/0.jpg)](https://www.youtube.com/watch?v=p16n4w3aE8k) diff --git a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/a2c.rs b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/a2c.rs deleted file mode 100644 index a5b8ef533f..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/a2c.rs +++ /dev/null @@ -1,171 +0,0 @@ -/* Advantage Actor Critic (A2C) model. - A2C is a synchronous variant of Asynchronous the Advantage Actor Critic (A3C) - model introduced by DeepMind in https://arxiv.org/abs/1602.01783 - - See https://blog.openai.com/baselines-acktr-a2c/ for a reference - python implementation. -*/ -use super::vec_gym_env::VecGymEnv; -use tch::kind::{FLOAT_CPU, INT64_CPU}; -use tch::{nn, nn::OptimizerConfig, Kind::Float, Tensor}; - -const ENV_NAME: &str = "SpaceInvadersNoFrameskip-v4"; -const NPROCS: i64 = 16; -const NSTEPS: i64 = 5; -const NSTACK: i64 = 4; -const UPDATES: i64 = 1000000; - -type Model = Box (Tensor, Tensor)>; - -fn model(p: &nn::Path, nact: i64) -> Model { - let stride = |s| nn::ConvConfig { stride: s, ..Default::default() }; - let seq = nn::seq() - .add(nn::conv2d(p / "c1", NSTACK, 32, 8, stride(4))) - .add_fn(|xs| xs.relu()) - .add(nn::conv2d(p / "c2", 32, 64, 4, stride(2))) - .add_fn(|xs| xs.relu()) - .add(nn::conv2d(p / "c3", 64, 64, 3, stride(1))) - .add_fn(|xs| xs.relu().flat_view()) - .add(nn::linear(p / "l1", 3136, 512, Default::default())) - .add_fn(|xs| xs.relu()); - let critic = nn::linear(p / "cl", 512, 1, Default::default()); - let actor = nn::linear(p / "al", 512, nact, Default::default()); - let device = p.device(); - Box::new(move |xs: &Tensor| { - let xs = xs.to_device(device).apply(&seq); - (xs.apply(&critic), xs.apply(&actor)) - }) -} - -#[derive(Debug)] -struct FrameStack { - data: Tensor, - nprocs: i64, - nstack: i64, -} - -impl FrameStack { - fn new(nprocs: i64, nstack: i64) -> FrameStack { - FrameStack { data: Tensor::zeros([nprocs, nstack, 84, 84], FLOAT_CPU), nprocs, nstack } - } - - fn update<'a>(&'a mut self, img: &Tensor, masks: Option<&Tensor>) -> &'a Tensor { - if let Some(masks) = masks { - self.data *= masks.view([self.nprocs, 1, 1, 1]) - }; - let slice = |i| self.data.narrow(1, i, 1); - for i in 1..self.nstack { - slice(i - 1).copy_(&slice(i)) - } - slice(self.nstack - 1).copy_(img); - &self.data - } -} - -/// Trains an agent using A2C. -pub fn train() -> cpython::PyResult<()> { - let env = VecGymEnv::new(ENV_NAME, None, NPROCS)?; - println!("action space: {}", env.action_space()); - println!("observation space: {:?}", env.observation_space()); - - let device = tch::Device::cuda_if_available(); - let vs = nn::VarStore::new(device); - let model = model(&vs.root(), env.action_space()); - let mut opt = nn::Adam::default().build(&vs, 1e-4).unwrap(); - - let mut sum_rewards = Tensor::zeros([NPROCS], FLOAT_CPU); - let mut total_rewards = 0f64; - let mut total_episodes = 0f64; - - let mut frame_stack = FrameStack::new(NPROCS, NSTACK); - let _ = frame_stack.update(&env.reset()?, None); - let s_states = Tensor::zeros([NSTEPS + 1, NPROCS, NSTACK, 84, 84], FLOAT_CPU); - for update_index in 0..UPDATES { - s_states.get(0).copy_(&s_states.get(-1)); - let s_values = Tensor::zeros([NSTEPS, NPROCS], FLOAT_CPU); - let s_rewards = Tensor::zeros([NSTEPS, NPROCS], FLOAT_CPU); - let s_actions = Tensor::zeros([NSTEPS, NPROCS], INT64_CPU); - let s_masks = Tensor::zeros([NSTEPS, NPROCS], FLOAT_CPU); - for s in 0..NSTEPS { - let (critic, actor) = tch::no_grad(|| model(&s_states.get(s))); - let probs = actor.softmax(-1, Float); - let actions = probs.multinomial(1, true).squeeze_dim(-1); - let step = env.step(Vec::::try_from(&actions).unwrap())?; - - sum_rewards += &step.reward; - total_rewards += f64::try_from((&sum_rewards * &step.is_done).sum(Float)).unwrap(); - total_episodes += f64::try_from(step.is_done.sum(Float)).unwrap(); - - let masks = Tensor::from(1f32) - step.is_done; - sum_rewards *= &masks; - let obs = frame_stack.update(&step.obs, Some(&masks)); - s_actions.get(s).copy_(&actions); - s_values.get(s).copy_(&critic.squeeze_dim(-1)); - s_states.get(s + 1).copy_(obs); - s_rewards.get(s).copy_(&step.reward); - s_masks.get(s).copy_(&masks); - } - let s_returns = { - let r = Tensor::zeros([NSTEPS + 1, NPROCS], FLOAT_CPU); - let critic = tch::no_grad(|| model(&s_states.get(-1)).0); - r.get(-1).copy_(&critic.view([NPROCS])); - for s in (0..NSTEPS).rev() { - let r_s = s_rewards.get(s) + r.get(s + 1) * s_masks.get(s) * 0.99; - r.get(s).copy_(&r_s); - } - r - }; - let (critic, actor) = - model(&s_states.narrow(0, 0, NSTEPS).view([NSTEPS * NPROCS, NSTACK, 84, 84])); - let critic = critic.view([NSTEPS, NPROCS]); - let actor = actor.view([NSTEPS, NPROCS, -1]); - let log_probs = actor.log_softmax(-1, Float); - let probs = actor.softmax(-1, Float); - let action_log_probs = { - let index = s_actions.unsqueeze(-1).to_device(device); - log_probs.gather(2, &index, false).squeeze_dim(-1) - }; - let dist_entropy = (-log_probs * probs).sum_dim_intlist(-1, false, Float).mean(Float); - let advantages = s_returns.narrow(0, 0, NSTEPS).to_device(device) - critic; - let value_loss = (&advantages * &advantages).mean(Float); - let action_loss = (-advantages.detach() * action_log_probs).mean(Float); - let loss = value_loss * 0.5 + action_loss - dist_entropy * 0.01; - opt.backward_step_clip(&loss, 0.5); - if update_index > 0 && update_index % 500 == 0 { - println!("{} {:.0} {}", update_index, total_episodes, total_rewards / total_episodes); - total_rewards = 0.; - total_episodes = 0.; - } - if update_index > 0 && update_index % 10000 == 0 { - if let Err(err) = vs.save(format!("a2c{update_index}.ot")) { - println!("error while saving {err}") - } - } - } - Ok(()) -} - -/// Samples some trajectory for a trained agent, the observation frames are stored in /dev/shm. -pub fn sample>(weight_file: T) -> cpython::PyResult<()> { - let env = VecGymEnv::new(ENV_NAME, Some("/dev/shm"), 1)?; - println!("action space: {}", env.action_space()); - println!("observation space: {:?}", env.observation_space()); - - let mut vs = nn::VarStore::new(tch::Device::Cpu); - let model = model(&vs.root(), env.action_space()); - vs.load(weight_file).unwrap(); - - let mut frame_stack = FrameStack::new(1, NSTACK); - let mut obs = frame_stack.update(&env.reset()?, None); - - for _index in 0..5000 { - let (_critic, actor) = tch::no_grad(|| model(obs)); - let probs = actor.softmax(-1, Float); - let actions = probs.multinomial(1, true).squeeze_dim(-1); - let step = env.step(Vec::::try_from(&actions).unwrap())?; - - let masks = Tensor::from(1f32) - step.is_done; - obs = frame_stack.update(&step.obs, Some(&masks)); - } - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/atari_wrappers.py b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/atari_wrappers.py deleted file mode 100644 index ad1d2704e2..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/atari_wrappers.py +++ /dev/null @@ -1,308 +0,0 @@ -import gym -import numpy as np -from collections import deque -from PIL import Image -from multiprocessing import Process, Pipe - -# atari_wrappers.py -class NoopResetEnv(gym.Wrapper): - def __init__(self, env, noop_max=30): - """Sample initial states by taking random number of no-ops on reset. - No-op is assumed to be action 0. - """ - gym.Wrapper.__init__(self, env) - self.noop_max = noop_max - self.override_num_noops = None - assert env.unwrapped.get_action_meanings()[0] == 'NOOP' - - def reset(self): - """ Do no-op action for a number of steps in [1, noop_max].""" - self.env.reset() - if self.override_num_noops is not None: - noops = self.override_num_noops - else: - noops = self.unwrapped.np_random.integers(1, self.noop_max + 1) #pylint: disable=E1101 - assert noops > 0 - obs = None - for _ in range(noops): - obs, _, done, _ = self.env.step(0) - if done: - obs = self.env.reset() - return obs - -class FireResetEnv(gym.Wrapper): - def __init__(self, env): - """Take action on reset for environments that are fixed until firing.""" - gym.Wrapper.__init__(self, env) - assert env.unwrapped.get_action_meanings()[1] == 'FIRE' - assert len(env.unwrapped.get_action_meanings()) >= 3 - - def reset(self): - self.env.reset() - obs, _, done, _ = self.env.step(1) - if done: - self.env.reset() - obs, _, done, _ = self.env.step(2) - if done: - self.env.reset() - return obs - -class ImageSaver(gym.Wrapper): - def __init__(self, env, img_path, rank): - gym.Wrapper.__init__(self, env) - self._cnt = 0 - self._img_path = img_path - self._rank = rank - - def step(self, action): - step_result = self.env.step(action) - obs, _, _, _ = step_result - img = Image.fromarray(obs, 'RGB') - img.save('%s/out%d-%05d.png' % (self._img_path, self._rank, self._cnt)) - self._cnt += 1 - return step_result - -class EpisodicLifeEnv(gym.Wrapper): - def __init__(self, env): - """Make end-of-life == end-of-episode, but only reset on true game over. - Done by DeepMind for the DQN and co. since it helps value estimation. - """ - gym.Wrapper.__init__(self, env) - self.lives = 0 - self.was_real_done = True - - def step(self, action): - obs, reward, done, info = self.env.step(action) - self.was_real_done = done - # check current lives, make loss of life terminal, - # then update lives to handle bonus lives - lives = self.env.unwrapped.ale.lives() - if lives < self.lives and lives > 0: - # for Qbert somtimes we stay in lives == 0 condtion for a few frames - # so its important to keep lives > 0, so that we only reset once - # the environment advertises done. - done = True - self.lives = lives - return obs, reward, done, info - - def reset(self): - """Reset only when lives are exhausted. - This way all states are still reachable even though lives are episodic, - and the learner need not know about any of this behind-the-scenes. - """ - if self.was_real_done: - obs = self.env.reset() - else: - # no-op step to advance from terminal/lost life state - obs, _, _, _ = self.env.step(0) - self.lives = self.env.unwrapped.ale.lives() - return obs - -class MaxAndSkipEnv(gym.Wrapper): - def __init__(self, env, skip=4): - """Return only every `skip`-th frame""" - gym.Wrapper.__init__(self, env) - # most recent raw observations (for max pooling across time steps) - self._obs_buffer = deque(maxlen=2) - self._skip = skip - - def step(self, action): - """Repeat action, sum reward, and max over last observations.""" - total_reward = 0.0 - done = None - for _ in range(self._skip): - obs, reward, done, info = self.env.step(action) - self._obs_buffer.append(obs) - total_reward += reward - if done: - break - max_frame = np.max(np.stack(self._obs_buffer), axis=0) - - return max_frame, total_reward, done, info - - def reset(self): - """Clear past frame buffer and init. to first obs. from inner env.""" - self._obs_buffer.clear() - obs = self.env.reset() - self._obs_buffer.append(obs) - return obs - -class ClipRewardEnv(gym.RewardWrapper): - def reward(self, reward): - """Bin reward to {+1, 0, -1} by its sign.""" - return np.sign(reward) - -class WarpFrame(gym.ObservationWrapper): - def __init__(self, env): - """Warp frames to 84x84 as done in the Nature paper and later work.""" - gym.ObservationWrapper.__init__(self, env) - self.res = 84 - self.observation_space = gym.spaces.Box(low=0, high=255, shape=(self.res, self.res, 1), dtype='uint8') - - def observation(self, obs): - frame = np.dot(obs.astype('float32'), np.array([0.299, 0.587, 0.114], 'float32')) - frame = np.array(Image.fromarray(frame).resize((self.res, self.res), - resample=Image.BILINEAR), dtype=np.uint8) - return frame.reshape((self.res, self.res, 1)) - -class FrameStack(gym.Wrapper): - def __init__(self, env, k): - """Buffer observations and stack across channels (last axis).""" - gym.Wrapper.__init__(self, env) - self.k = k - self.frames = deque([], maxlen=k) - shp = env.observation_space.shape - assert shp[2] == 1 # can only stack 1-channel frames - self.observation_space = gym.spaces.Box(low=0, high=255, shape=(shp[0], shp[1], k), dtype='uint8') - - def reset(self): - """Clear buffer and re-fill by duplicating the first observation.""" - ob = self.env.reset() - for _ in range(self.k): self.frames.append(ob) - return self.observation() - - def step(self, action): - ob, reward, done, info = self.env.step(action) - self.frames.append(ob) - return self.observation(), reward, done, info - - def observation(self): - assert len(self.frames) == self.k - return np.concatenate(self.frames, axis=2) - -def wrap_deepmind(env, episode_life=True, clip_rewards=True): - """Configure environment for DeepMind-style Atari. - - Note: this does not include frame stacking!""" - assert 'NoFrameskip' in env.spec.id # required for DeepMind-style skip - if episode_life: - env = EpisodicLifeEnv(env) - env = NoopResetEnv(env, noop_max=30) - env = MaxAndSkipEnv(env, skip=4) - if 'FIRE' in env.unwrapped.get_action_meanings(): - env = FireResetEnv(env) - env = WarpFrame(env) - if clip_rewards: - env = ClipRewardEnv(env) - return env - -# envs.py -def make_env(env_id, img_dir, seed, rank): - def _thunk(): - env = gym.make(env_id) - env.seed(seed + rank) - if img_dir is not None: - env = ImageSaver(env, img_dir, rank) - env = wrap_deepmind(env) - env = WrapPyTorch(env) - return env - - return _thunk - -class WrapPyTorch(gym.ObservationWrapper): - def __init__(self, env=None): - super(WrapPyTorch, self).__init__(env) - self.observation_space = gym.spaces.Box(0.0, 1.0, [1, 84, 84], dtype='float32') - - def observation(self, observation): - return observation.transpose(2, 0, 1) - -# vecenv.py -class VecEnv(object): - """ - Vectorized environment base class - """ - def step(self, vac): - """ - Apply sequence of actions to sequence of environments - actions -> (observations, rewards, news) - - where 'news' is a boolean vector indicating whether each element is new. - """ - raise NotImplementedError - def reset(self): - """ - Reset all environments - """ - raise NotImplementedError - def close(self): - pass - -# subproc_vec_env.py -def worker(remote, env_fn_wrapper): - env = env_fn_wrapper.x() - while True: - cmd, data = remote.recv() - if cmd == 'step': - ob, reward, done, info = env.step(data) - if done: - ob = env.reset() - remote.send((ob, reward, done, info)) - elif cmd == 'reset': - ob = env.reset() - remote.send(ob) - elif cmd == 'close': - remote.close() - break - elif cmd == 'get_spaces': - remote.send((env.action_space, env.observation_space)) - else: - raise NotImplementedError - -class CloudpickleWrapper(object): - """ - Uses cloudpickle to serialize contents (otherwise multiprocessing tries to use pickle) - """ - def __init__(self, x): - self.x = x - def __getstate__(self): - import cloudpickle - return cloudpickle.dumps(self.x) - def __setstate__(self, ob): - import pickle - self.x = pickle.loads(ob) - -class SubprocVecEnv(VecEnv): - def __init__(self, env_fns): - """ - envs: list of gym environments to run in subprocesses - """ - nenvs = len(env_fns) - self.remotes, self.work_remotes = zip(*[Pipe() for _ in range(nenvs)]) - self.ps = [Process(target=worker, args=(work_remote, CloudpickleWrapper(env_fn))) - for (work_remote, env_fn) in zip(self.work_remotes, env_fns)] - for p in self.ps: - p.start() - - self.remotes[0].send(('get_spaces', None)) - self.action_space, self.observation_space = self.remotes[0].recv() - - - def step(self, actions): - for remote, action in zip(self.remotes, actions): - remote.send(('step', action)) - results = [remote.recv() for remote in self.remotes] - obs, rews, dones, infos = zip(*results) - return np.stack(obs), np.stack(rews), np.stack(dones), infos - - def reset(self): - for remote in self.remotes: - remote.send(('reset', None)) - return np.stack([remote.recv() for remote in self.remotes]) - - def close(self): - for remote in self.remotes: - remote.send(('close', None)) - for p in self.ps: - p.join() - - @property - def num_envs(self): - return len(self.remotes) - -# Create the environment. -def make(env_name, img_dir, num_processes): - envs = SubprocVecEnv([ - make_env(env_name, img_dir, 1337, i) for i in range(num_processes) - ]) - return envs diff --git a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/ddpg.rs b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/ddpg.rs deleted file mode 100644 index 743411bde7..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/ddpg.rs +++ /dev/null @@ -1,343 +0,0 @@ -/* Deep Deterministic Policy Gradient. - - Continuous control with deep reinforcement learning, Lillicrap et al. 2015 - https://arxiv.org/abs/1509.02971 - - See https://spinningup.openai.com/en/latest/algorithms/ddpg.html for a - reference python implementation. -*/ -use super::gym_env::GymEnv; -use tch::{ - kind::{FLOAT_CPU, INT64_CPU}, - nn, - nn::OptimizerConfig, - Device, - Kind::Float, - Tensor, -}; - -// The impact of the q value of the next state on the current state's q value. -const GAMMA: f64 = 0.99; -// The weight for updating the target networks. -const TAU: f64 = 0.005; -// The capacity of the replay buffer used for sampling training data. -const REPLAY_BUFFER_CAPACITY: usize = 100_000; -// The training batch size for each training iteration. -const TRAINING_BATCH_SIZE: usize = 100; -// The total number of episodes. -const MAX_EPISODES: usize = 100; -// The maximum length of an episode. -const EPISODE_LENGTH: usize = 200; -// The number of training iterations after one episode finishes. -const TRAINING_ITERATIONS: usize = 200; - -// Ornstein-Uhlenbeck process parameters. -const MU: f64 = 0.0; -const THETA: f64 = 0.15; -const SIGMA: f64 = 0.1; - -const ACTOR_LEARNING_RATE: f64 = 1e-4; -const CRITIC_LEARNING_RATE: f64 = 1e-3; - -struct OuNoise { - mu: f64, - theta: f64, - sigma: f64, - state: Tensor, -} - -impl OuNoise { - fn new(mu: f64, theta: f64, sigma: f64, num_actions: usize) -> Self { - let state = Tensor::ones([num_actions as _], FLOAT_CPU); - Self { mu, theta, sigma, state } - } - - fn sample(&mut self) -> &Tensor { - let dx = self.theta * (self.mu - &self.state) - + self.sigma * Tensor::randn(self.state.size(), FLOAT_CPU); - self.state += dx; - &self.state - } -} - -struct ReplayBuffer { - obs: Tensor, - next_obs: Tensor, - rewards: Tensor, - actions: Tensor, - capacity: usize, - len: usize, - i: usize, -} - -impl ReplayBuffer { - fn new(capacity: usize, num_obs: usize, num_actions: usize) -> Self { - Self { - obs: Tensor::zeros([capacity as _, num_obs as _], FLOAT_CPU), - next_obs: Tensor::zeros([capacity as _, num_obs as _], FLOAT_CPU), - rewards: Tensor::zeros([capacity as _, 1], FLOAT_CPU), - actions: Tensor::zeros([capacity as _, num_actions as _], FLOAT_CPU), - capacity, - len: 0, - i: 0, - } - } - - fn push(&mut self, obs: &Tensor, actions: &Tensor, reward: &Tensor, next_obs: &Tensor) { - let i = self.i % self.capacity; - self.obs.get(i as _).copy_(obs); - self.rewards.get(i as _).copy_(reward); - self.actions.get(i as _).copy_(actions); - self.next_obs.get(i as _).copy_(next_obs); - self.i += 1; - if self.len < self.capacity { - self.len += 1; - } - } - - fn random_batch(&self, batch_size: usize) -> Option<(Tensor, Tensor, Tensor, Tensor)> { - if self.len < 3 { - return None; - } - - let batch_size = batch_size.min(self.len - 1); - let batch_indexes = Tensor::randint((self.len - 2) as _, [batch_size as _], INT64_CPU); - - let states = self.obs.index_select(0, &batch_indexes); - let next_states = self.next_obs.index_select(0, &batch_indexes); - let actions = self.actions.index_select(0, &batch_indexes); - let rewards = self.rewards.index_select(0, &batch_indexes); - - Some((states, actions, rewards, next_states)) - } -} - -struct Actor { - var_store: nn::VarStore, - network: nn::Sequential, - device: Device, - num_obs: usize, - num_actions: usize, - opt: nn::Optimizer, - learning_rate: f64, -} - -impl Clone for Actor { - fn clone(&self) -> Self { - let mut new = Self::new(self.num_obs, self.num_actions, self.learning_rate); - new.var_store.copy(&self.var_store).unwrap(); - new - } -} - -impl Actor { - fn new(num_obs: usize, num_actions: usize, learning_rate: f64) -> Self { - let var_store = nn::VarStore::new(tch::Device::Cpu); - let opt = nn::Adam::default().build(&var_store, learning_rate).unwrap(); - let p = &var_store.root(); - Self { - network: nn::seq() - .add(nn::linear(p / "al1", num_obs as _, 400, Default::default())) - .add_fn(|xs| xs.relu()) - .add(nn::linear(p / "al2", 400, 300, Default::default())) - .add_fn(|xs| xs.relu()) - .add(nn::linear(p / "al3", 300, num_actions as _, Default::default())) - .add_fn(|xs| xs.tanh()), - device: p.device(), - num_obs, - num_actions, - var_store, - opt, - learning_rate, - } - } - - fn forward(&self, obs: &Tensor) -> Tensor { - obs.to_device(self.device).apply(&self.network) - } -} - -struct Critic { - var_store: nn::VarStore, - network: nn::Sequential, - device: Device, - num_obs: usize, - num_actions: usize, - opt: nn::Optimizer, - learning_rate: f64, -} - -impl Clone for Critic { - fn clone(&self) -> Self { - let mut new = Self::new(self.num_obs, self.num_actions, self.learning_rate); - new.var_store.copy(&self.var_store).unwrap(); - new - } -} - -impl Critic { - fn new(num_obs: usize, num_actions: usize, learning_rate: f64) -> Self { - let var_store = nn::VarStore::new(tch::Device::Cpu); - let opt = nn::Adam::default().build(&var_store, 1e-3).unwrap(); - let p = &var_store.root(); - Self { - network: nn::seq() - .add(nn::linear(p / "cl1", (num_obs + num_actions) as _, 400, Default::default())) - .add_fn(|xs| xs.relu()) - .add(nn::linear(p / "cl2", 400, 300, Default::default())) - .add_fn(|xs| xs.relu()) - .add(nn::linear(p / "cl3", 300, 1, Default::default())), - device: p.device(), - var_store, - num_obs, - num_actions, - opt, - learning_rate, - } - } - - fn forward(&self, obs: &Tensor, actions: &Tensor) -> Tensor { - let xs = Tensor::cat(&[actions.copy(), obs.copy()], 1); - xs.to_device(self.device).apply(&self.network) - } -} - -fn track(dest: &mut nn::VarStore, src: &nn::VarStore, tau: f64) { - tch::no_grad(|| { - for (dest, src) in - dest.trainable_variables().iter_mut().zip(src.trainable_variables().iter()) - { - dest.copy_(&(tau * src + (1.0 - tau) * &*dest)); - } - }) -} - -struct Agent { - actor: Actor, - actor_target: Actor, - - critic: Critic, - critic_target: Critic, - - replay_buffer: ReplayBuffer, - - ou_noise: OuNoise, - - train: bool, - - gamma: f64, - tau: f64, -} - -impl Agent { - fn new( - actor: Actor, - critic: Critic, - ou_noise: OuNoise, - replay_buffer_capacity: usize, - train: bool, - gamma: f64, - tau: f64, - ) -> Self { - let actor_target = actor.clone(); - let critic_target = critic.clone(); - let replay_buffer = - ReplayBuffer::new(replay_buffer_capacity, actor.num_obs, actor.num_actions); - Self { - actor, - actor_target, - critic, - critic_target, - replay_buffer, - ou_noise, - train, - gamma, - tau, - } - } - - fn actions(&mut self, obs: &Tensor) -> Tensor { - let mut actions = tch::no_grad(|| self.actor.forward(obs)); - if self.train { - actions += self.ou_noise.sample(); - } - actions - } - - fn remember(&mut self, obs: &Tensor, actions: &Tensor, reward: &Tensor, next_obs: &Tensor) { - self.replay_buffer.push(obs, actions, reward, next_obs); - } - - fn train(&mut self, batch_size: usize) { - let (states, actions, rewards, next_states) = - match self.replay_buffer.random_batch(batch_size) { - Some(v) => v, - _ => return, // We don't have enough samples for training yet. - }; - - let mut q_target = - self.critic_target.forward(&next_states, &self.actor_target.forward(&next_states)); - q_target = rewards + (self.gamma * q_target).detach(); - - let q = self.critic.forward(&states, &actions); - - let diff = q_target - q; - let critic_loss = (&diff * &diff).mean(Float); - - self.critic.opt.zero_grad(); - critic_loss.backward(); - self.critic.opt.step(); - - let actor_loss = -self.critic.forward(&states, &self.actor.forward(&states)).mean(Float); - - self.actor.opt.zero_grad(); - actor_loss.backward(); - self.actor.opt.step(); - - track(&mut self.critic_target.var_store, &self.critic.var_store, self.tau); - track(&mut self.actor_target.var_store, &self.actor.var_store, self.tau); - } -} - -pub fn run() -> cpython::PyResult<()> { - let env = GymEnv::new("Pendulum-v1")?; - println!("action space: {}", env.action_space()); - println!("observation space: {:?}", env.observation_space()); - - let num_obs = env.observation_space().iter().product::() as usize; - let num_actions = env.action_space() as usize; - - let actor = Actor::new(num_obs, num_actions, ACTOR_LEARNING_RATE); - let critic = Critic::new(num_obs, num_actions, CRITIC_LEARNING_RATE); - let ou_noise = OuNoise::new(MU, THETA, SIGMA, num_actions); - let mut agent = Agent::new(actor, critic, ou_noise, REPLAY_BUFFER_CAPACITY, true, GAMMA, TAU); - - for episode in 0..MAX_EPISODES { - let mut obs = env.reset()?; - - let mut total_reward = 0.0; - for _ in 0..EPISODE_LENGTH { - let mut actions = 2.0 * f64::try_from(agent.actions(&obs)).unwrap(); - actions = actions.clamp(-2.0, 2.0); - - let action_vec = vec![actions]; - let step = env.step(&action_vec)?; - total_reward += step.reward; - - agent.remember(&obs, &actions.into(), &step.reward.into(), &step.obs); - - if step.is_done { - break; - } - obs = step.obs; - } - - println!("episode {episode} with total reward of {total_reward}"); - - for _ in 0..TRAINING_ITERATIONS { - agent.train(TRAINING_BATCH_SIZE); - } - } - - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/gym_env.rs b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/gym_env.rs deleted file mode 100644 index f11c13304c..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/gym_env.rs +++ /dev/null @@ -1,78 +0,0 @@ -//! Wrappers around the Python API of the OpenAI gym. -use cpython::{NoArgs, ObjectProtocol, PyObject, PyResult, Python, ToPyObject}; -use tch::Tensor; - -/// The return value for a step. -#[derive(Debug)] -pub struct Step { - pub obs: Tensor, - pub action: A, - pub reward: f64, - pub is_done: bool, -} - -impl Step { - /// Returns a copy of this step changing the observation tensor. - pub fn copy_with_obs(&self, obs: &Tensor) -> Step { - Step { obs: obs.copy(), action: self.action, reward: self.reward, is_done: self.is_done } - } -} - -/// An OpenAI Gym session. -pub struct GymEnv { - env: PyObject, - action_space: i64, - observation_space: Vec, -} - -impl GymEnv { - /// Creates a new session of the specified OpenAI Gym environment. - pub fn new(name: &str) -> PyResult { - let gil = Python::acquire_gil(); - let py = gil.python(); - let gym = py.import("gym")?; - let env = gym.call(py, "make", (name,), None)?; - let _ = env.call_method(py, "seed", (42,), None)?; - let action_space = env.getattr(py, "action_space")?; - let action_space = if let Ok(val) = action_space.getattr(py, "n") { - val.extract(py)? - } else { - let action_space: Vec = action_space.getattr(py, "shape")?.extract(py)?; - action_space[0] - }; - let observation_space = env.getattr(py, "observation_space")?; - let observation_space = observation_space.getattr(py, "shape")?.extract(py)?; - Ok(GymEnv { env, action_space, observation_space }) - } - - /// Resets the environment, returning the observation tensor. - pub fn reset(&self) -> PyResult { - let gil = Python::acquire_gil(); - let py = gil.python(); - let obs = self.env.call_method(py, "reset", NoArgs, None)?; - Ok(Tensor::from_slice(&obs.extract::>(py)?)) - } - - /// Applies an environment step using the specified action. - pub fn step(&self, action: A) -> PyResult> { - let gil = Python::acquire_gil(); - let py = gil.python(); - let step = self.env.call_method(py, "step", (action,), None)?; - Ok(Step { - obs: Tensor::from_slice(&step.get_item(py, 0)?.extract::>(py)?), - reward: step.get_item(py, 1)?.extract(py)?, - is_done: step.get_item(py, 2)?.extract(py)?, - action, - }) - } - - /// Returns the number of allowed actions for this environment. - pub fn action_space(&self) -> i64 { - self.action_space - } - - /// Returns the shape of the observation tensors. - pub fn observation_space(&self) -> &[i64] { - &self.observation_space - } -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/main.rs b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/main.rs deleted file mode 100644 index 724d804f95..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/main.rs +++ /dev/null @@ -1,20 +0,0 @@ -mod a2c; -mod ddpg; -mod gym_env; -mod policy_gradient; -mod ppo; -mod vec_gym_env; - -fn main() -> cpython::PyResult<()> { - let a: Vec = std::env::args().collect(); - match a.iter().map(|x| x.as_str()).collect::>().as_slice() { - [_, "a2c"] => a2c::train()?, - [_, "a2c-sample", weight_file] => a2c::sample(weight_file)?, - [_, "pg"] => policy_gradient::run()?, - [_, "ppo"] => ppo::train()?, - [_, "ppo-sample", weight_file] => ppo::sample(weight_file)?, - [_, "ddpg"] => ddpg::run()?, - _ => println!("usage: main pg|a2c|a2c-sample|ppo|ppo-sample|ddpg"), - } - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/policy_gradient.rs b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/policy_gradient.rs deleted file mode 100644 index 81945ce8e3..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/policy_gradient.rs +++ /dev/null @@ -1,84 +0,0 @@ -// Policy gradient example. -// This uses OpenAI Gym environment through rust-cpython. -// -// This is adapted from OpenAI Spinning Up series: -// https://spinningup.openai.com/en/latest/spinningup/rl_intro3.html -// A TensorFlow Python implementation can be found here: -// https://github.com/openai/spinningup/blob/master/spinup/examples/pg_math/2_rtg_pg.py - -use super::gym_env::{GymEnv, Step}; -use tch::{nn, nn::OptimizerConfig, Kind::Float, Tensor}; - -fn model(p: &nn::Path, input_shape: &[i64], nact: i64) -> impl nn::Module { - let nin = input_shape.iter().product::(); - nn::seq() - .add(nn::linear(p / "lin1", nin, 32, Default::default())) - .add_fn(|xs| xs.tanh()) - .add(nn::linear(p / "lin2", 32, nact, Default::default())) -} - -fn accumulate_rewards(steps: &[Step]) -> Vec { - let mut rewards: Vec = steps.iter().map(|s| s.reward).collect(); - let mut acc_reward = 0f64; - for (i, reward) in rewards.iter_mut().enumerate().rev() { - if steps[i].is_done { - acc_reward = 0.0; - } - acc_reward += *reward; - *reward = acc_reward; - } - rewards -} - -/// Trains an agent using the policy gradient algorithm. -pub fn run() -> cpython::PyResult<()> { - let env = GymEnv::new("CartPole-v0")?; - println!("action space: {:?}", env.action_space()); - println!("observation space: {:?}", env.observation_space()); - - let vs = nn::VarStore::new(tch::Device::Cpu); - let model = model(&vs.root(), env.observation_space(), env.action_space()); - let mut opt = nn::Adam::default().build(&vs, 1e-2).unwrap(); - - for epoch_idx in 0..50 { - let mut obs = env.reset()?; - let mut steps: Vec> = vec![]; - // Perform some rollouts with the current model. - loop { - let action = tch::no_grad(|| { - obs.unsqueeze(0).apply(&model).softmax(1, Float).multinomial(1, true) - }); - let action = i64::try_from(action).unwrap(); - let step = env.step(action)?; - steps.push(step.copy_with_obs(&obs)); - obs = if step.is_done { env.reset()? } else { step.obs }; - if step.is_done && steps.len() > 5000 { - break; - } - } - let sum_r: f64 = steps.iter().map(|s| s.reward).sum(); - let episodes: i64 = steps.iter().map(|s| s.is_done as i64).sum(); - println!( - "epoch: {:<3} episodes: {:<5} avg reward per episode: {:.2}", - epoch_idx, - episodes, - sum_r / episodes as f64 - ); - - // Train the model via policy gradient on the rollout data. - let batch_size = steps.len() as i64; - let actions: Vec = steps.iter().map(|s| s.action).collect(); - let actions = Tensor::from_slice(&actions).unsqueeze(1); - let rewards = accumulate_rewards(&steps); - let rewards = Tensor::from_slice(&rewards).to_kind(Float); - let action_mask = - Tensor::zeros([batch_size, 2], tch::kind::FLOAT_CPU).scatter_value(1, &actions, 1.0); - let obs: Vec = steps.into_iter().map(|s| s.obs).collect(); - let logits = Tensor::stack(&obs, 0).apply(&model); - let log_probs = - (action_mask * logits.log_softmax(1, Float)).sum_dim_intlist(1, false, Float); - let loss = -(rewards * log_probs).mean(Float); - opt.backward_step(&loss) - } - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/ppo.rs b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/ppo.rs deleted file mode 100644 index d1b462eda6..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/ppo.rs +++ /dev/null @@ -1,181 +0,0 @@ -/* Proximal Policy Optimization (PPO) model. - - Proximal Policy Optimization Algorithms, Schulman et al. 2017 - https://arxiv.org/abs/1707.06347 - - See https://spinningup.openai.com/en/latest/algorithms/ppo.html for a - reference python implementation. -*/ -use super::vec_gym_env::VecGymEnv; -use tch::kind::{FLOAT_CPU, INT64_CPU}; -use tch::{nn, nn::OptimizerConfig, Kind, Tensor}; - -const ENV_NAME: &str = "SpaceInvadersNoFrameskip-v4"; -const NPROCS: i64 = 8; -const NSTEPS: i64 = 256; -const NSTACK: i64 = 4; -const UPDATES: i64 = 1000000; -const OPTIM_BATCHSIZE: i64 = 64; -const OPTIM_EPOCHS: i64 = 4; - -type Model = Box (Tensor, Tensor)>; - -fn model(p: &nn::Path, nact: i64) -> Model { - let stride = |s| nn::ConvConfig { stride: s, ..Default::default() }; - let seq = nn::seq() - .add(nn::conv2d(p / "c1", NSTACK, 32, 8, stride(4))) - .add_fn(|xs| xs.relu()) - .add(nn::conv2d(p / "c2", 32, 64, 4, stride(2))) - .add_fn(|xs| xs.relu()) - .add(nn::conv2d(p / "c3", 64, 64, 3, stride(1))) - .add_fn(|xs| xs.relu().flat_view()) - .add(nn::linear(p / "l1", 3136, 512, Default::default())) - .add_fn(|xs| xs.relu()); - let critic = nn::linear(p / "cl", 512, 1, Default::default()); - let actor = nn::linear(p / "al", 512, nact, Default::default()); - let device = p.device(); - Box::new(move |xs: &Tensor| { - let xs = xs.to_device(device).apply(&seq); - (xs.apply(&critic), xs.apply(&actor)) - }) -} - -#[derive(Debug)] -struct FrameStack { - data: Tensor, - nprocs: i64, - nstack: i64, -} - -impl FrameStack { - fn new(nprocs: i64, nstack: i64) -> FrameStack { - FrameStack { data: Tensor::zeros([nprocs, nstack, 84, 84], FLOAT_CPU), nprocs, nstack } - } - - fn update<'a>(&'a mut self, img: &Tensor, masks: Option<&Tensor>) -> &'a Tensor { - if let Some(masks) = masks { - self.data *= masks.view([self.nprocs, 1, 1, 1]) - }; - let slice = |i| self.data.narrow(1, i, 1); - for i in 1..self.nstack { - slice(i - 1).copy_(&slice(i)) - } - slice(self.nstack - 1).copy_(img); - &self.data - } -} - -pub fn train() -> cpython::PyResult<()> { - let env = VecGymEnv::new(ENV_NAME, None, NPROCS)?; - println!("action space: {}", env.action_space()); - println!("observation space: {:?}", env.observation_space()); - - let device = tch::Device::cuda_if_available(); - let vs = nn::VarStore::new(device); - let model = model(&vs.root(), env.action_space()); - let mut opt = nn::Adam::default().build(&vs, 1e-4).unwrap(); - - let mut sum_rewards = Tensor::zeros([NPROCS], FLOAT_CPU); - let mut total_rewards = 0f64; - let mut total_episodes = 0f64; - - let mut frame_stack = FrameStack::new(NPROCS, NSTACK); - let _ = frame_stack.update(&env.reset()?, None); - let s_states = Tensor::zeros([NSTEPS + 1, NPROCS, NSTACK, 84, 84], FLOAT_CPU); - - let train_size = NSTEPS * NPROCS; - for update_index in 0..UPDATES { - s_states.get(0).copy_(&s_states.get(-1)); - let s_values = Tensor::zeros([NSTEPS, NPROCS], FLOAT_CPU); - let s_rewards = Tensor::zeros([NSTEPS, NPROCS], FLOAT_CPU); - let s_actions = Tensor::zeros([NSTEPS, NPROCS], INT64_CPU); - let s_masks = Tensor::zeros([NSTEPS, NPROCS], FLOAT_CPU); - for s in 0..NSTEPS { - let (critic, actor) = tch::no_grad(|| model(&s_states.get(s))); - let probs = actor.softmax(-1, Kind::Float); - let actions = probs.multinomial(1, true).squeeze_dim(-1); - let step = env.step(Vec::::try_from(&actions).unwrap())?; - - sum_rewards += &step.reward; - total_rewards += - f64::try_from((&sum_rewards * &step.is_done).sum(Kind::Float)).unwrap(); - total_episodes += f64::try_from(step.is_done.sum(Kind::Float)).unwrap(); - - let masks = Tensor::from(1f32) - step.is_done; - sum_rewards *= &masks; - let obs = frame_stack.update(&step.obs, Some(&masks)); - s_actions.get(s).copy_(&actions); - s_values.get(s).copy_(&critic.squeeze_dim(-1)); - s_states.get(s + 1).copy_(obs); - s_rewards.get(s).copy_(&step.reward); - s_masks.get(s).copy_(&masks); - } - let states = s_states.narrow(0, 0, NSTEPS).view([train_size, NSTACK, 84, 84]); - let returns = { - let r = Tensor::zeros([NSTEPS + 1, NPROCS], FLOAT_CPU); - let critic = tch::no_grad(|| model(&s_states.get(-1)).0); - r.get(-1).copy_(&critic.view([NPROCS])); - for s in (0..NSTEPS).rev() { - let r_s = s_rewards.get(s) + r.get(s + 1) * s_masks.get(s) * 0.99; - r.get(s).copy_(&r_s); - } - r.narrow(0, 0, NSTEPS).view([train_size, 1]) - }; - let actions = s_actions.view([train_size]); - for _index in 0..OPTIM_EPOCHS { - let batch_indexes = Tensor::randint(train_size, [OPTIM_BATCHSIZE], INT64_CPU); - let states = states.index_select(0, &batch_indexes); - let actions = actions.index_select(0, &batch_indexes); - let returns = returns.index_select(0, &batch_indexes); - let (critic, actor) = model(&states); - let log_probs = actor.log_softmax(-1, Kind::Float); - let probs = actor.softmax(-1, Kind::Float); - let action_log_probs = { - let index = actions.unsqueeze(-1).to_device(device); - log_probs.gather(-1, &index, false).squeeze_dim(-1) - }; - let dist_entropy = - (-log_probs * probs).sum_dim_intlist(-1, false, Kind::Float).mean(Kind::Float); - let advantages = returns.to_device(device) - critic; - let value_loss = (&advantages * &advantages).mean(Kind::Float); - let action_loss = (-advantages.detach() * action_log_probs).mean(Kind::Float); - let loss = value_loss * 0.5 + action_loss - dist_entropy * 0.01; - opt.backward_step_clip(&loss, 0.5); - } - if update_index > 0 && update_index % 25 == 0 { - println!("{} {:.0} {}", update_index, total_episodes, total_rewards / total_episodes); - total_rewards = 0.; - total_episodes = 0.; - } - if update_index > 0 && update_index % 1000 == 0 { - if let Err(err) = vs.save(format!("trpo{update_index}.ot")) { - println!("error while saving {err}") - } - } - } - Ok(()) -} - -pub fn sample>(weight_file: T) -> cpython::PyResult<()> { - let env = VecGymEnv::new(ENV_NAME, Some("/dev/shm"), 1)?; - println!("action space: {}", env.action_space()); - println!("observation space: {:?}", env.observation_space()); - - let mut vs = nn::VarStore::new(tch::Device::Cpu); - let model = model(&vs.root(), env.action_space()); - vs.load(weight_file).unwrap(); - - let mut frame_stack = FrameStack::new(1, NSTACK); - let mut obs = frame_stack.update(&env.reset()?, None); - - for _index in 0..5000 { - let (_critic, actor) = tch::no_grad(|| model(obs)); - let probs = actor.softmax(-1, Kind::Float); - let actions = probs.multinomial(1, true).squeeze_dim(-1); - let step = env.step(Vec::::try_from(&actions).unwrap())?; - - let masks = Tensor::from(1f32) - step.is_done; - obs = frame_stack.update(&step.obs, Some(&masks)); - } - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/vec_gym_env.rs b/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/vec_gym_env.rs deleted file mode 100644 index e13d378740..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/reinforcement-learning/vec_gym_env.rs +++ /dev/null @@ -1,66 +0,0 @@ -// Vectorized version of the gym environment. -use cpython::{buffer::PyBuffer, NoArgs, ObjectProtocol, PyObject, PyResult, Python}; -use tch::Tensor; - -#[derive(Debug)] -pub struct Step { - pub obs: Tensor, - pub reward: Tensor, - pub is_done: Tensor, -} - -pub struct VecGymEnv { - env: PyObject, - action_space: i64, - observation_space: Vec, -} - -impl VecGymEnv { - pub fn new(name: &str, img_dir: Option<&str>, nprocesses: i64) -> PyResult { - let gil = Python::acquire_gil(); - let py = gil.python(); - let sys = py.import("sys")?; - let path = sys.get(py, "path")?; - let _ = path.call_method(py, "append", ("examples/reinforcement-learning",), None)?; - let gym = py.import("atari_wrappers")?; - let env = gym.call(py, "make", (name, img_dir, nprocesses), None)?; - let action_space = env.getattr(py, "action_space")?; - let action_space = action_space.getattr(py, "n")?.extract(py)?; - let observation_space = env.getattr(py, "observation_space")?; - let observation_space: Vec = observation_space.getattr(py, "shape")?.extract(py)?; - let observation_space = - [vec![nprocesses].as_slice(), observation_space.as_slice()].concat(); - Ok(VecGymEnv { env, action_space, observation_space }) - } - - pub fn reset(&self) -> PyResult { - let gil = Python::acquire_gil(); - let py = gil.python(); - let obs = self.env.call_method(py, "reset", NoArgs, None)?; - let obs = obs.call_method(py, "flatten", NoArgs, None)?; - let obs = Tensor::from_slice(&obs.extract::>(py)?); - Ok(obs.view_(&self.observation_space)) - } - - pub fn step(&self, action: Vec) -> PyResult { - let gil = Python::acquire_gil(); - let py = gil.python(); - let step = self.env.call_method(py, "step", (action,), None)?; - let obs = step.get_item(py, 0)?.call_method(py, "flatten", NoArgs, None)?; - let obs_buffer = PyBuffer::get(py, &obs)?; - let obs_vec: Vec = obs_buffer.to_vec(py)?; - let obs = - Tensor::from_slice(&obs_vec).view_(&self.observation_space).to_kind(tch::Kind::Float); - let reward = Tensor::from_slice(&step.get_item(py, 1)?.extract::>(py)?); - let is_done = Tensor::from_slice(&step.get_item(py, 2)?.extract::>(py)?); - Ok(Step { obs, reward, is_done }) - } - - pub fn action_space(&self) -> i64 { - self.action_space - } - - pub fn observation_space(&self) -> &[i64] { - &self.observation_space - } -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/stable-diffusion/README.md b/v2/dependency-patches/tch-0.24.0/examples/stable-diffusion/README.md deleted file mode 100644 index af9f4d8463..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/stable-diffusion/README.md +++ /dev/null @@ -1,11 +0,0 @@ -![rusty robot holding a torch](media/robot13.jpg) - -_A rusty robot holding a fire torch_, generated by stable diffusion using Rust and libtorch. - - -__IMPORTANT NOTE__ - -This example has now been moved in its own repo -[diffusers-rs](https://github.com/LaurentMazare/diffusers-rs). -The new repo includes new functionalities like inpainting, pre-packaged weights -so that it's not necessary to run the conversion steps below anymore, etc. diff --git a/v2/dependency-patches/tch-0.24.0/examples/stable-diffusion/main.rs b/v2/dependency-patches/tch-0.24.0/examples/stable-diffusion/main.rs deleted file mode 100644 index 47ab967c10..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/stable-diffusion/main.rs +++ /dev/null @@ -1,2512 +0,0 @@ -// Stable Diffusion implementation inspired: -// - Huggingface's amazing diffuser Python api: https://huggingface.co/blog/annotated-diffusion -// - Huggingface's (also amazing) blog post: https://huggingface.co/blog/annotated-diffusion -// - The "Grokking Stable Diffusion" notebook by Jonathan Whitaker. -// https://colab.research.google.com/drive/1dlgggNa5Mz8sEAGU0wFCHhGLFooW_pf1?usp=sharing -// -// In order to run this, first download the following and extract the file in data/ -// -// mkdir -p data && cd data -// wget https://github.com/openai/CLIP/raw/main/clip/bpe_simple_vocab_16e6.txt.gz -// gunzip bpe_simple_vocab_16e6.txt.gz -// -// Download and convert the weights: -// -// 1. Clip Encoding Weights -// wget https://huggingface.co/openai/clip-vit-large-patch14/resolve/main/pytorch_model.bin -// From python, extract the weights and save them as a .npz file. -// import numpy as np -// import torch -// model = torch.load("./pytorch_model.bin") -// np.savez("./pytorch_model.npz", **{k: v.numpy() for k, v in model.items() if "text_model" in k}) -// -// Then use tensor_tools to convert this to a .ot file that tch can use. -// cargo run --release --example tensor-tools cp ./data/pytorch_model.npz ./data/pytorch_model.ot -// -// 2. VAE and Unet Weights -// https://huggingface.co/CompVis/stable-diffusion-v1-4/blob/main/vae/diffusion_pytorch_model.bin -// https://huggingface.co/CompVis/stable-diffusion-v1-4/blob/main/unet/diffusion_pytorch_model.bin -// -// import numpy as np -// import torch -// model = torch.load("./vae.bin") -// np.savez("./vae.npz", **{k: v.numpy() for k, v in model.items()}) -// model = torch.load("./unet.bin") -// np.savez("./unet.npz", **{k: v.numpy() for k, v in model.items()}) -// -// cargo run --release --example tensor-tools cp ./data/vae.npz ./data/vae.ot -// cargo run --release --example tensor-tools cp ./data/unet.npz ./data/unet.ot -// TODO: fix tensor_tools so that it works properly there. -// TODO: Split this file, probably in a way similar to huggingface/diffusers. -use std::collections::{HashMap, HashSet}; -use std::io::BufRead; -use tch::{kind, nn, nn::Module, Device, Kind, Tensor}; - -// The config details can be found in the "text_config" section of this json file: -// https://huggingface.co/openai/clip-vit-large-patch14/blob/main/config.json -// "hidden_act": "quick_gelu" -const VOCAB_SIZE: i64 = 49408; -const EMBED_DIM: i64 = 768; // a.k.a. config.hidden_size -const INTERMEDIATE_SIZE: i64 = 3072; -const MAX_POSITION_EMBEDDINGS: usize = 77; -const NUM_HIDDEN_LAYERS: i64 = 12; -const NUM_ATTENTION_HEADS: i64 = 12; - -const HEIGHT: i64 = 512; -const WIDTH: i64 = 512; -const GUIDANCE_SCALE: f64 = 7.5; - -const BYTES_TO_UNICODE: [(u8, char); 256] = [ - (33, '!'), - (34, '"'), - (35, '#'), - (36, '$'), - (37, '%'), - (38, '&'), - (39, '\''), - (40, '('), - (41, ')'), - (42, '*'), - (43, '+'), - (44, ','), - (45, '-'), - (46, '.'), - (47, '/'), - (48, '0'), - (49, '1'), - (50, '2'), - (51, '3'), - (52, '4'), - (53, '5'), - (54, '6'), - (55, '7'), - (56, '8'), - (57, '9'), - (58, ':'), - (59, ';'), - (60, '<'), - (61, '='), - (62, '>'), - (63, '?'), - (64, '@'), - (65, 'A'), - (66, 'B'), - (67, 'C'), - (68, 'D'), - (69, 'E'), - (70, 'F'), - (71, 'G'), - (72, 'H'), - (73, 'I'), - (74, 'J'), - (75, 'K'), - (76, 'L'), - (77, 'M'), - (78, 'N'), - (79, 'O'), - (80, 'P'), - (81, 'Q'), - (82, 'R'), - (83, 'S'), - (84, 'T'), - (85, 'U'), - (86, 'V'), - (87, 'W'), - (88, 'X'), - (89, 'Y'), - (90, 'Z'), - (91, '['), - (92, '\\'), - (93, ']'), - (94, '^'), - (95, '_'), - (96, '`'), - (97, 'a'), - (98, 'b'), - (99, 'c'), - (100, 'd'), - (101, 'e'), - (102, 'f'), - (103, 'g'), - (104, 'h'), - (105, 'i'), - (106, 'j'), - (107, 'k'), - (108, 'l'), - (109, 'm'), - (110, 'n'), - (111, 'o'), - (112, 'p'), - (113, 'q'), - (114, 'r'), - (115, 's'), - (116, 't'), - (117, 'u'), - (118, 'v'), - (119, 'w'), - (120, 'x'), - (121, 'y'), - (122, 'z'), - (123, '{'), - (124, '|'), - (125, '}'), - (126, '~'), - (161, '¡'), - (162, '¢'), - (163, '£'), - (164, '¤'), - (165, '¥'), - (166, '¦'), - (167, '§'), - (168, '¨'), - (169, '©'), - (170, 'ª'), - (171, '«'), - (172, '¬'), - (174, '®'), - (175, '¯'), - (176, '°'), - (177, '±'), - (178, '²'), - (179, '³'), - (180, '´'), - (181, 'µ'), - (182, '¶'), - (183, '·'), - (184, '¸'), - (185, '¹'), - (186, 'º'), - (187, '»'), - (188, '¼'), - (189, '½'), - (190, '¾'), - (191, '¿'), - (192, 'À'), - (193, 'Á'), - (194, 'Â'), - (195, 'Ã'), - (196, 'Ä'), - (197, 'Å'), - (198, 'Æ'), - (199, 'Ç'), - (200, 'È'), - (201, 'É'), - (202, 'Ê'), - (203, 'Ë'), - (204, 'Ì'), - (205, 'Í'), - (206, 'Î'), - (207, 'Ï'), - (208, 'Ð'), - (209, 'Ñ'), - (210, 'Ò'), - (211, 'Ó'), - (212, 'Ô'), - (213, 'Õ'), - (214, 'Ö'), - (215, '×'), - (216, 'Ø'), - (217, 'Ù'), - (218, 'Ú'), - (219, 'Û'), - (220, 'Ü'), - (221, 'Ý'), - (222, 'Þ'), - (223, 'ß'), - (224, 'à'), - (225, 'á'), - (226, 'â'), - (227, 'ã'), - (228, 'ä'), - (229, 'å'), - (230, 'æ'), - (231, 'ç'), - (232, 'è'), - (233, 'é'), - (234, 'ê'), - (235, 'ë'), - (236, 'ì'), - (237, 'í'), - (238, 'î'), - (239, 'ï'), - (240, 'ð'), - (241, 'ñ'), - (242, 'ò'), - (243, 'ó'), - (244, 'ô'), - (245, 'õ'), - (246, 'ö'), - (247, '÷'), - (248, 'ø'), - (249, 'ù'), - (250, 'ú'), - (251, 'û'), - (252, 'ü'), - (253, 'ý'), - (254, 'þ'), - (255, 'ÿ'), - (0, 'Ā'), - (1, 'ā'), - (2, 'Ă'), - (3, 'ă'), - (4, 'Ą'), - (5, 'ą'), - (6, 'Ć'), - (7, 'ć'), - (8, 'Ĉ'), - (9, 'ĉ'), - (10, 'Ċ'), - (11, 'ċ'), - (12, 'Č'), - (13, 'č'), - (14, 'Ď'), - (15, 'ď'), - (16, 'Đ'), - (17, 'đ'), - (18, 'Ē'), - (19, 'ē'), - (20, 'Ĕ'), - (21, 'ĕ'), - (22, 'Ė'), - (23, 'ė'), - (24, 'Ę'), - (25, 'ę'), - (26, 'Ě'), - (27, 'ě'), - (28, 'Ĝ'), - (29, 'ĝ'), - (30, 'Ğ'), - (31, 'ğ'), - (32, 'Ġ'), - (127, 'ġ'), - (128, 'Ģ'), - (129, 'ģ'), - (130, 'Ĥ'), - (131, 'ĥ'), - (132, 'Ħ'), - (133, 'ħ'), - (134, 'Ĩ'), - (135, 'ĩ'), - (136, 'Ī'), - (137, 'ī'), - (138, 'Ĭ'), - (139, 'ĭ'), - (140, 'Į'), - (141, 'į'), - (142, 'İ'), - (143, 'ı'), - (144, 'IJ'), - (145, 'ij'), - (146, 'Ĵ'), - (147, 'ĵ'), - (148, 'Ķ'), - (149, 'ķ'), - (150, 'ĸ'), - (151, 'Ĺ'), - (152, 'ĺ'), - (153, 'Ļ'), - (154, 'ļ'), - (155, 'Ľ'), - (156, 'ľ'), - (157, 'Ŀ'), - (158, 'ŀ'), - (159, 'Ł'), - (160, 'ł'), - (173, 'Ń'), -]; - -const PAT: &str = - r"<\|startoftext\|>|<\|endoftext\|>|'s|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+"; - -// This is mostly a Rust rewrite of the original Python CLIP code. -// https://github.com/openai/CLIP/blob/main/clip/simple_tokenizer.py -struct Tokenizer { - re: regex::Regex, - encoder: HashMap, - decoder: HashMap, - bpe_ranks: HashMap<(String, String), usize>, - start_of_text_token: usize, - end_of_text_token: usize, -} - -impl Tokenizer { - fn create>(bpe_path: T) -> anyhow::Result { - let bpe_file = std::fs::File::open(bpe_path)?; - let bpe_lines: Result, _> = std::io::BufReader::new(bpe_file).lines().collect(); - let bpe_lines = bpe_lines?; - let bpe_lines: Result, _> = bpe_lines[1..49152 - 256 - 2 + 1] - .iter() - .map(|line| { - let vs: Vec<_> = line.split_whitespace().collect(); - if vs.len() != 2 { - anyhow::bail!("expected two items got {} '{}'", vs.len(), line) - } - Ok((vs[0].to_string(), vs[1].to_string())) - }) - .collect(); - let bpe_lines = bpe_lines?; - let mut vocab: Vec = Vec::new(); - for (_index, elem) in BYTES_TO_UNICODE { - vocab.push(elem.into()) - } - for (_index, elem) in BYTES_TO_UNICODE { - vocab.push(format!("{elem}")); - } - for elem in bpe_lines.iter() { - vocab.push(format!("{}{}", elem.0, elem.1)) - } - let start_of_text_token = vocab.len(); - vocab.push("<|startoftext|>".to_string()); - let end_of_text_token = vocab.len(); - vocab.push("<|endoftext|>".to_string()); - let encoder: HashMap<_, _> = vocab.into_iter().enumerate().map(|(i, v)| (v, i)).collect(); - let decoder: HashMap<_, _> = encoder.iter().map(|(k, v)| (*v, k.clone())).collect(); - let bpe_ranks: HashMap<_, _> = - bpe_lines.into_iter().enumerate().map(|(i, v)| (v, i)).collect(); - let re = regex::Regex::new(PAT)?; - let tokenizer = - Tokenizer { encoder, re, bpe_ranks, decoder, start_of_text_token, end_of_text_token }; - Ok(tokenizer) - } - - fn get_pairs(word: &[String]) -> HashSet<(String, String)> { - let mut pairs = HashSet::new(); - for (i, v) in word.iter().enumerate() { - if i > 0 { - pairs.insert((word[i - 1].clone(), v.clone())); - } - } - pairs - } - - fn bpe(&self, token: &str) -> Vec { - let mut word: Vec = token.chars().map(|x| x.to_string()).collect(); - if word.is_empty() { - return Vec::new(); - } - let last_index = word.len() - 1; - word[last_index] = format!("{}", word[last_index]); - while word.len() > 1 { - let mut current_min = None; - let pairs = Self::get_pairs(&word); - for p in pairs.iter() { - match self.bpe_ranks.get(p) { - None => {} - Some(v) => { - let should_replace = match current_min { - None => true, - Some((current_min, _)) => v < current_min, - }; - if should_replace { - current_min = Some((v, p)) - } - } - } - } - let (first, second) = match current_min { - None => break, - Some((_v, (first, second))) => (first, second), - }; - let mut new_word = vec![]; - let mut index = 0; - while index < word.len() { - let w = &word[index]; - if index + 1 < word.len() && w == first && &word[index + 1] == second { - new_word.push(format!("{first}{second}")); - index += 2 - } else { - new_word.push(w.clone()); - index += 1 - } - } - word = new_word - } - word.iter().map(|x| *self.encoder.get(x).unwrap()).collect() - } - - fn encode(&self, s: &str, pad_size_to: Option) -> anyhow::Result> { - let s = s.to_lowercase(); - let mut bpe_tokens: Vec = vec![self.start_of_text_token]; - for token in self.re.captures_iter(&s) { - let token = token.get(0).unwrap().as_str(); - bpe_tokens.extend(self.bpe(token)) - } - match pad_size_to { - None => bpe_tokens.push(self.end_of_text_token), - Some(pad_size_to) => { - bpe_tokens.resize_with( - std::cmp::min(bpe_tokens.len(), pad_size_to - 1), - Default::default, - ); - while bpe_tokens.len() < pad_size_to { - bpe_tokens.push(self.end_of_text_token) - } - } - } - Ok(bpe_tokens) - } - - fn decode(&self, tokens: &[usize]) -> String { - let s: String = tokens.iter().map(|token| self.decoder[token].as_str()).collect(); - s.replace("", " ") - } -} - -// CLIP Text Model -// https://github.com/huggingface/transformers/blob/674f750a57431222fa2832503a108df3badf1564/src/transformers/models/clip/modeling_clip.py -#[derive(Debug)] -struct ClipTextEmbeddings { - token_embedding: nn::Embedding, - position_embedding: nn::Embedding, - position_ids: Tensor, -} - -impl ClipTextEmbeddings { - fn new(vs: nn::Path) -> Self { - let token_embedding = - nn::embedding(&vs / "token_embedding", VOCAB_SIZE, EMBED_DIM, Default::default()); - let position_embedding = nn::embedding( - &vs / "position_embedding", - MAX_POSITION_EMBEDDINGS as i64, - EMBED_DIM, - Default::default(), - ); - let position_ids = - Tensor::arange(MAX_POSITION_EMBEDDINGS as i64, (Kind::Int64, vs.device())) - .expand([1, -1], false); - ClipTextEmbeddings { token_embedding, position_embedding, position_ids } - } -} - -impl Module for ClipTextEmbeddings { - fn forward(&self, xs: &Tensor) -> Tensor { - let token_embedding = self.token_embedding.forward(xs); - let position_embedding = self.position_embedding.forward(&self.position_ids); - token_embedding + position_embedding - } -} - -fn quick_gelu(xs: &Tensor) -> Tensor { - xs * (xs * 1.702).sigmoid() -} - -#[derive(Debug)] -struct ClipAttention { - k_proj: nn::Linear, - v_proj: nn::Linear, - q_proj: nn::Linear, - out_proj: nn::Linear, - head_dim: i64, - scale: f64, -} - -impl ClipAttention { - fn new(vs: nn::Path) -> Self { - let k_proj = nn::linear(&vs / "k_proj", EMBED_DIM, EMBED_DIM, Default::default()); - let v_proj = nn::linear(&vs / "v_proj", EMBED_DIM, EMBED_DIM, Default::default()); - let q_proj = nn::linear(&vs / "q_proj", EMBED_DIM, EMBED_DIM, Default::default()); - let out_proj = nn::linear(&vs / "out_proj", EMBED_DIM, EMBED_DIM, Default::default()); - let head_dim = EMBED_DIM / NUM_ATTENTION_HEADS; - let scale = (head_dim as f64).powf(-0.5); - ClipAttention { k_proj, v_proj, q_proj, out_proj, head_dim, scale } - } - - fn shape(&self, xs: &Tensor, seq_len: i64, bsz: i64) -> Tensor { - xs.view((bsz, seq_len, NUM_ATTENTION_HEADS, self.head_dim)).transpose(1, 2).contiguous() - } - - fn forward(&self, xs: &Tensor, causal_attention_mask: &Tensor) -> Tensor { - let (bsz, tgt_len, embed_dim) = xs.size3().unwrap(); - let query_states = xs.apply(&self.q_proj) * self.scale; - let proj_shape = (bsz * NUM_ATTENTION_HEADS, -1, self.head_dim); - let query_states = self.shape(&query_states, tgt_len, bsz).view(proj_shape); - let key_states = self.shape(&xs.apply(&self.k_proj), -1, bsz).view(proj_shape); - let value_states = self.shape(&xs.apply(&self.v_proj), -1, bsz).view(proj_shape); - let attn_weights = query_states.bmm(&key_states.transpose(1, 2)); - - let src_len = key_states.size()[1]; - let attn_weights = - attn_weights.view((bsz, NUM_ATTENTION_HEADS, tgt_len, src_len)) + causal_attention_mask; - let attn_weights = attn_weights.view((bsz * NUM_ATTENTION_HEADS, tgt_len, src_len)); - let attn_weights = attn_weights.softmax(-1, Kind::Float); - - let attn_output = attn_weights.bmm(&value_states); - attn_output - .view((bsz, NUM_ATTENTION_HEADS, tgt_len, self.head_dim)) - .transpose(1, 2) - .reshape([bsz, tgt_len, embed_dim]) - .apply(&self.out_proj) - } -} - -#[derive(Debug)] -struct ClipMlp { - fc1: nn::Linear, - fc2: nn::Linear, -} - -impl ClipMlp { - fn new(vs: nn::Path) -> Self { - let fc1 = nn::linear(&vs / "fc1", EMBED_DIM, INTERMEDIATE_SIZE, Default::default()); - let fc2 = nn::linear(&vs / "fc2", INTERMEDIATE_SIZE, EMBED_DIM, Default::default()); - ClipMlp { fc1, fc2 } - } -} - -impl Module for ClipMlp { - fn forward(&self, xs: &Tensor) -> Tensor { - let xs = xs.apply(&self.fc1); - quick_gelu(&xs).apply(&self.fc2) - } -} - -#[derive(Debug)] -struct ClipEncoderLayer { - self_attn: ClipAttention, - layer_norm1: nn::LayerNorm, - mlp: ClipMlp, - layer_norm2: nn::LayerNorm, -} - -impl ClipEncoderLayer { - fn new(vs: nn::Path) -> Self { - let self_attn = ClipAttention::new(&vs / "self_attn"); - let layer_norm1 = nn::layer_norm(&vs / "layer_norm1", vec![EMBED_DIM], Default::default()); - let mlp = ClipMlp::new(&vs / "mlp"); - let layer_norm2 = nn::layer_norm(&vs / "layer_norm2", vec![EMBED_DIM], Default::default()); - ClipEncoderLayer { self_attn, layer_norm1, mlp, layer_norm2 } - } - - fn forward(&self, xs: &Tensor, causal_attention_mask: &Tensor) -> Tensor { - let residual = xs; - let xs = self.layer_norm1.forward(xs); - let xs = self.self_attn.forward(&xs, causal_attention_mask); - let xs = xs + residual; - - let residual = &xs; - let xs = self.layer_norm2.forward(&xs); - let xs = self.mlp.forward(&xs); - xs + residual - } -} - -#[derive(Debug)] -struct ClipEncoder { - layers: Vec, -} - -impl ClipEncoder { - fn new(vs: nn::Path) -> Self { - let vs = &vs / "layers"; - let mut layers: Vec = Vec::new(); - for index in 0..NUM_HIDDEN_LAYERS { - let layer = ClipEncoderLayer::new(&vs / index); - layers.push(layer) - } - ClipEncoder { layers } - } - - fn forward(&self, xs: &Tensor, causal_attention_mask: &Tensor) -> Tensor { - let mut xs = xs.shallow_clone(); - for layer in self.layers.iter() { - xs = layer.forward(&xs, causal_attention_mask) - } - xs - } -} - -#[derive(Debug)] -struct ClipTextTransformer { - embeddings: ClipTextEmbeddings, - encoder: ClipEncoder, - final_layer_norm: nn::LayerNorm, -} - -impl ClipTextTransformer { - fn new(vs: nn::Path) -> Self { - let vs = &vs / "text_model"; - let embeddings = ClipTextEmbeddings::new(&vs / "embeddings"); - let encoder = ClipEncoder::new(&vs / "encoder"); - let final_layer_norm = - nn::layer_norm(&vs / "final_layer_norm", vec![EMBED_DIM], Default::default()); - ClipTextTransformer { embeddings, encoder, final_layer_norm } - } - - // https://github.com/huggingface/transformers/blob/674f750a57431222fa2832503a108df3badf1564/src/transformers/models/clip/modeling_clip.py#L678 - fn build_causal_attention_mask(bsz: i64, seq_len: i64, device: Device) -> Tensor { - let mut mask = Tensor::ones([bsz, seq_len, seq_len], (Kind::Float, device)); - mask.fill_(f32::MIN as f64).triu_(1).unsqueeze(1) - } -} - -impl Module for ClipTextTransformer { - fn forward(&self, xs: &Tensor) -> Tensor { - let (bsz, seq_len) = xs.size2().unwrap(); - let xs = self.embeddings.forward(xs); - let causal_attention_mask = Self::build_causal_attention_mask(bsz, seq_len, xs.device()); - let xs = self.encoder.forward(&xs, &causal_attention_mask); - xs.apply(&self.final_layer_norm) - } -} - -#[derive(Debug)] -struct GeGlu { - proj: nn::Linear, -} - -impl GeGlu { - fn new(vs: nn::Path, dim_in: i64, dim_out: i64) -> Self { - let proj = nn::linear(&vs / "proj", dim_in, dim_out * 2, Default::default()); - Self { proj } - } -} - -impl Module for GeGlu { - fn forward(&self, xs: &Tensor) -> Tensor { - let hidden_states_and_gate = xs.apply(&self.proj).chunk(2, -1); - &hidden_states_and_gate[0] * hidden_states_and_gate[1].gelu("none") - } -} - -#[derive(Debug)] -struct FeedForward { - project_in: GeGlu, - linear: nn::Linear, -} - -impl FeedForward { - // The glu parameter in the python code is unused? - // https://github.com/huggingface/diffusers/blob/d3d22ce5a894becb951eec03e663951b28d45135/src/diffusers/models/attention.py#L347 - fn new(vs: nn::Path, dim: i64, dim_out: Option, mult: i64) -> Self { - let inner_dim = dim * mult; - let dim_out = dim_out.unwrap_or(dim); - let vs = &vs / "net"; - let project_in = GeGlu::new(&vs / 0, dim, inner_dim); - let linear = nn::linear(&vs / 2, inner_dim, dim_out, Default::default()); - Self { project_in, linear } - } -} - -impl Module for FeedForward { - fn forward(&self, xs: &Tensor) -> Tensor { - xs.apply(&self.project_in).apply(&self.linear) - } -} - -#[derive(Debug)] -struct CrossAttention { - to_q: nn::Linear, - to_k: nn::Linear, - to_v: nn::Linear, - to_out: nn::Linear, - heads: i64, - scale: f64, -} - -impl CrossAttention { - // Defaults should be heads = 8, dim_head = 64, context_dim = None - fn new( - vs: nn::Path, - query_dim: i64, - context_dim: Option, - heads: i64, - dim_head: i64, - ) -> Self { - let no_bias = nn::LinearConfig { bias: false, ..Default::default() }; - let inner_dim = dim_head * heads; - let context_dim = context_dim.unwrap_or(query_dim); - let scale = 1.0 / f64::sqrt(dim_head as f64); - let to_q = nn::linear(&vs / "to_q", query_dim, inner_dim, no_bias); - let to_k = nn::linear(&vs / "to_k", context_dim, inner_dim, no_bias); - let to_v = nn::linear(&vs / "to_v", context_dim, inner_dim, no_bias); - let to_out = nn::linear(&vs / "to_out" / 0, inner_dim, query_dim, Default::default()); - Self { to_q, to_k, to_v, to_out, heads, scale } - } - - fn reshape_heads_to_batch_dim(&self, xs: &Tensor) -> Tensor { - let (batch_size, seq_len, dim) = xs.size3().unwrap(); - xs.reshape([batch_size, seq_len, self.heads, dim / self.heads]) - .permute([0, 2, 1, 3]) - .reshape([batch_size * self.heads, seq_len, dim / self.heads]) - } - - fn reshape_batch_dim_to_heads(&self, xs: &Tensor) -> Tensor { - let (batch_size, seq_len, dim) = xs.size3().unwrap(); - xs.reshape([batch_size / self.heads, self.heads, seq_len, dim]) - .permute([0, 2, 1, 3]) - .reshape([batch_size / self.heads, seq_len, dim * self.heads]) - } - - fn attention(&self, query: &Tensor, key: &Tensor, value: &Tensor) -> Tensor { - let xs = query - .matmul(&(key.transpose(-1, -2) * self.scale)) - .softmax(-1, Kind::Float) - .matmul(value); - self.reshape_batch_dim_to_heads(&xs) - } - - fn forward(&self, xs: &Tensor, context: Option<&Tensor>) -> Tensor { - let query = xs.apply(&self.to_q); - let context = context.unwrap_or(xs); - let key = context.apply(&self.to_k); - let value = context.apply(&self.to_v); - let query = self.reshape_heads_to_batch_dim(&query); - let key = self.reshape_heads_to_batch_dim(&key); - let value = self.reshape_heads_to_batch_dim(&value); - self.attention(&query, &key, &value).apply(&self.to_out) - } -} - -#[derive(Debug)] -struct BasicTransformerBlock { - attn1: CrossAttention, - ff: FeedForward, - attn2: CrossAttention, - norm1: nn::LayerNorm, - norm2: nn::LayerNorm, - norm3: nn::LayerNorm, -} - -impl BasicTransformerBlock { - fn new(vs: nn::Path, dim: i64, n_heads: i64, d_head: i64, context_dim: Option) -> Self { - let attn1 = CrossAttention::new(&vs / "attn1", dim, None, n_heads, d_head); - let ff = FeedForward::new(&vs / "ff", dim, None, 4); - let attn2 = CrossAttention::new(&vs / "attn2", dim, context_dim, n_heads, d_head); - let norm1 = nn::layer_norm(&vs / "norm1", vec![dim], Default::default()); - let norm2 = nn::layer_norm(&vs / "norm2", vec![dim], Default::default()); - let norm3 = nn::layer_norm(&vs / "norm3", vec![dim], Default::default()); - Self { attn1, ff, attn2, norm1, norm2, norm3 } - } - - fn forward(&self, xs: &Tensor, context: Option<&Tensor>) -> Tensor { - let xs = self.attn1.forward(&xs.apply(&self.norm1), None) + xs; - let xs = self.attn2.forward(&xs.apply(&self.norm2), context) + xs; - xs.apply(&self.norm3).apply(&self.ff) + xs - } -} - -#[derive(Debug, Clone, Copy)] -struct SpatialTransformerConfig { - depth: i64, - num_groups: i64, - context_dim: Option, -} - -impl Default for SpatialTransformerConfig { - fn default() -> Self { - Self { depth: 1, num_groups: 32, context_dim: None } - } -} - -#[derive(Debug)] -struct SpatialTransformer { - norm: nn::GroupNorm, - proj_in: nn::Conv2D, - transformer_blocks: Vec, - proj_out: nn::Conv2D, - #[allow(dead_code)] - config: SpatialTransformerConfig, -} - -impl SpatialTransformer { - fn new( - vs: nn::Path, - in_channels: i64, - n_heads: i64, - d_head: i64, - config: SpatialTransformerConfig, - ) -> Self { - let inner_dim = n_heads * d_head; - let group_cfg = nn::GroupNormConfig { eps: 1e-6, affine: true, ..Default::default() }; - let norm = nn::group_norm(&vs / "norm", config.num_groups, in_channels, group_cfg); - let conv_cfg = nn::ConvConfig { stride: 1, padding: 0, ..Default::default() }; - let proj_in = nn::conv2d(&vs / "proj_in", in_channels, inner_dim, 1, conv_cfg); - let mut transformer_blocks = vec![]; - let vs_tb = &vs / "transformer_blocks"; - for index in 0..config.depth { - let tb = BasicTransformerBlock::new( - &vs_tb / index, - inner_dim, - n_heads, - d_head, - config.context_dim, - ); - transformer_blocks.push(tb) - } - let proj_out = nn::conv2d(&vs / "proj_out", inner_dim, in_channels, 1, conv_cfg); - Self { norm, proj_in, transformer_blocks, proj_out, config } - } - - fn forward(&self, xs: &Tensor, context: Option<&Tensor>) -> Tensor { - let (batch, _channel, height, weight) = xs.size4().unwrap(); - let residual = xs; - let xs = xs.apply(&self.norm).apply(&self.proj_in); - let inner_dim = xs.size()[1]; - let mut xs = xs.permute([0, 2, 3, 1]).view((batch, height * weight, inner_dim)); - for block in self.transformer_blocks.iter() { - xs = block.forward(&xs, context) - } - let xs = - xs.view((batch, height, weight, inner_dim)).permute([0, 3, 1, 2]).apply(&self.proj_out); - xs + residual - } -} - -#[derive(Debug, Clone, Copy)] -struct AttentionBlockConfig { - num_head_channels: Option, - num_groups: i64, - rescale_output_factor: f64, - eps: f64, -} - -impl Default for AttentionBlockConfig { - fn default() -> Self { - Self { num_head_channels: None, num_groups: 32, rescale_output_factor: 1., eps: 1e-5 } - } -} - -#[derive(Debug)] -struct AttentionBlock { - group_norm: nn::GroupNorm, - query: nn::Linear, - key: nn::Linear, - value: nn::Linear, - proj_attn: nn::Linear, - channels: i64, - num_heads: i64, - config: AttentionBlockConfig, -} - -impl AttentionBlock { - fn new(vs: nn::Path, channels: i64, config: AttentionBlockConfig) -> Self { - let num_head_channels = config.num_head_channels.unwrap_or(channels); - let num_heads = channels / num_head_channels; - let group_cfg = nn::GroupNormConfig { eps: config.eps, affine: true, ..Default::default() }; - let group_norm = nn::group_norm(&vs / "group_norm", config.num_groups, channels, group_cfg); - let query = nn::linear(&vs / "query", channels, channels, Default::default()); - let key = nn::linear(&vs / "key", channels, channels, Default::default()); - let value = nn::linear(&vs / "value", channels, channels, Default::default()); - let proj_attn = nn::linear(&vs / "proj_attn", channels, channels, Default::default()); - Self { group_norm, query, key, value, proj_attn, channels, num_heads, config } - } - - fn transpose_for_scores(&self, xs: Tensor) -> Tensor { - let (batch, t, _h_times_d) = xs.size3().unwrap(); - xs.view((batch, t, self.num_heads, -1)).permute([0, 2, 1, 3]) - } -} - -impl Module for AttentionBlock { - fn forward(&self, xs: &Tensor) -> Tensor { - let residual = xs; - let (batch, channel, height, width) = xs.size4().unwrap(); - let xs = xs.apply(&self.group_norm).view((batch, channel, height * width)).transpose(1, 2); - - let query_proj = xs.apply(&self.query); - let key_proj = xs.apply(&self.key); - let value_proj = xs.apply(&self.value); - - let query_states = self.transpose_for_scores(query_proj); - let key_states = self.transpose_for_scores(key_proj); - let value_states = self.transpose_for_scores(value_proj); - - let scale = f64::powf((self.channels as f64) / (self.num_heads as f64), -0.25); - let attention_scores = - (query_states * scale).matmul(&(key_states.transpose(-1, -2) * scale)); - let attention_probs = attention_scores.softmax(-1, Kind::Float); - - let xs = attention_probs.matmul(&value_states); - let xs = xs.permute([0, 2, 1, 3]).contiguous(); - let mut new_xs_shape = xs.size(); - new_xs_shape.pop(); - new_xs_shape.pop(); - new_xs_shape.push(self.channels); - - let xs = xs - .view(new_xs_shape.as_slice()) - .apply(&self.proj_attn) - .transpose(-1, -2) - .view((batch, channel, height, width)); - (xs + residual) / self.config.rescale_output_factor - } -} - -#[derive(Debug)] -struct Downsample2D { - conv: Option, - padding: i64, -} - -impl Downsample2D { - fn new( - vs: nn::Path, - in_channels: i64, - use_conv: bool, - out_channels: i64, - padding: i64, - ) -> Self { - let conv = if use_conv { - let config = nn::ConvConfig { stride: 2, padding, ..Default::default() }; - let conv = nn::conv2d(&vs / "conv", in_channels, out_channels, 3, config); - Some(conv) - } else { - None - }; - Downsample2D { conv, padding } - } -} - -impl Module for Downsample2D { - fn forward(&self, xs: &Tensor) -> Tensor { - match &self.conv { - None => xs.avg_pool2d([2, 2], [2, 2], [0, 0], false, true, None), - Some(conv) => { - if self.padding == 0 { - xs.pad([0, 1, 0, 1], "constant", Some(0.)).apply(conv) - } else { - xs.apply(conv) - } - } - } - } -} - -// This does not support the conv-transpose mode. -#[derive(Debug)] -struct Upsample2D { - conv: nn::Conv2D, -} - -impl Upsample2D { - fn new(vs: nn::Path, in_channels: i64, out_channels: i64) -> Self { - let config = nn::ConvConfig { padding: 1, ..Default::default() }; - let conv = nn::conv2d(&vs / "conv", in_channels, out_channels, 3, config); - Self { conv } - } -} - -impl Upsample2D { - fn forward(&self, xs: &Tensor, size: Option<(i64, i64)>) -> Tensor { - let xs = match size { - None => { - // The following does not work and it's tricky to pass no fixed - // dimensions so hack our way around this. - // xs.upsample_nearest2d(&[], Some(2.), Some(2.) - let (_bsize, _channels, h, w) = xs.size4().unwrap(); - xs.upsample_nearest2d([2 * h, 2 * w], Some(2.), Some(2.)) - } - Some((h, w)) => xs.upsample_nearest2d([h, w], None, None), - }; - xs.apply(&self.conv) - } -} - -#[derive(Debug, Clone, Copy)] -struct ResnetBlock2DConfig { - out_channels: Option, - temb_channels: Option, - groups: i64, - groups_out: Option, - eps: f64, - use_in_shortcut: Option, - // non_linearity: silu - output_scale_factor: f64, -} - -impl Default for ResnetBlock2DConfig { - fn default() -> Self { - Self { - out_channels: None, - temb_channels: Some(512), - groups: 32, - groups_out: None, - eps: 1e-6, - use_in_shortcut: None, - output_scale_factor: 1., - } - } -} - -#[derive(Debug)] -struct ResnetBlock2D { - norm1: nn::GroupNorm, - conv1: nn::Conv2D, - norm2: nn::GroupNorm, - conv2: nn::Conv2D, - time_emb_proj: Option, - conv_shortcut: Option, - config: ResnetBlock2DConfig, -} - -impl ResnetBlock2D { - fn new(vs: nn::Path, in_channels: i64, config: ResnetBlock2DConfig) -> Self { - let out_channels = config.out_channels.unwrap_or(in_channels); - let conv_cfg = nn::ConvConfig { stride: 1, padding: 1, ..Default::default() }; - let group_cfg = nn::GroupNormConfig { eps: config.eps, affine: true, ..Default::default() }; - let norm1 = nn::group_norm(&vs / "norm1", config.groups, in_channels, group_cfg); - let conv1 = nn::conv2d(&vs / "conv1", in_channels, out_channels, 3, conv_cfg); - let groups_out = config.groups_out.unwrap_or(config.groups); - let norm2 = nn::group_norm(&vs / "norm2", groups_out, out_channels, group_cfg); - let conv2 = nn::conv2d(&vs / "conv2", out_channels, out_channels, 3, conv_cfg); - let use_in_shortcut = config.use_in_shortcut.unwrap_or(in_channels != out_channels); - let conv_shortcut = if use_in_shortcut { - let conv_cfg = nn::ConvConfig { stride: 1, padding: 0, ..Default::default() }; - Some(nn::conv2d(&vs / "conv_shortcut", in_channels, out_channels, 1, conv_cfg)) - } else { - None - }; - let time_emb_proj = config.temb_channels.map(|temb_channels| { - nn::linear(&vs / "time_emb_proj", temb_channels, out_channels, Default::default()) - }); - Self { norm1, conv1, norm2, conv2, time_emb_proj, config, conv_shortcut } - } - - fn forward(&self, xs: &Tensor, temb: Option<&Tensor>) -> Tensor { - let shortcut_xs = match &self.conv_shortcut { - Some(conv_shortcut) => xs.apply(conv_shortcut), - None => xs.shallow_clone(), - }; - let xs = xs.apply(&self.norm1).silu().apply(&self.conv1); - let xs = match (temb, &self.time_emb_proj) { - (Some(temb), Some(time_emb_proj)) => { - temb.silu().apply(time_emb_proj).unsqueeze(-1).unsqueeze(-1) + xs - } - _ => xs, - }; - let xs = xs.apply(&self.norm2).silu().apply(&self.conv2); - (shortcut_xs + xs) / self.config.output_scale_factor - } -} - -#[derive(Debug, Clone, Copy)] -struct DownEncoderBlock2DConfig { - num_layers: i64, - resnet_eps: f64, - resnet_groups: i64, - output_scale_factor: f64, - add_downsample: bool, - downsample_padding: i64, -} - -impl Default for DownEncoderBlock2DConfig { - fn default() -> Self { - Self { - num_layers: 1, - resnet_eps: 1e-6, - resnet_groups: 32, - output_scale_factor: 1., - add_downsample: true, - downsample_padding: 1, - } - } -} - -#[derive(Debug)] -struct DownEncoderBlock2D { - resnets: Vec, - downsampler: Option, - #[allow(dead_code)] - config: DownEncoderBlock2DConfig, -} - -impl DownEncoderBlock2D { - fn new( - vs: nn::Path, - in_channels: i64, - out_channels: i64, - config: DownEncoderBlock2DConfig, - ) -> Self { - let resnets: Vec<_> = { - let vs = &vs / "resnets"; - let conv_cfg = ResnetBlock2DConfig { - eps: config.resnet_eps, - out_channels: Some(out_channels), - groups: config.resnet_groups, - output_scale_factor: config.output_scale_factor, - temb_channels: None, - ..Default::default() - }; - (0..(config.num_layers)) - .map(|i| { - let in_channels = if i == 0 { in_channels } else { out_channels }; - ResnetBlock2D::new(&vs / i, in_channels, conv_cfg) - }) - .collect() - }; - let downsampler = if config.add_downsample { - let downsample = Downsample2D::new( - &(&vs / "downsamplers") / 0, - out_channels, - true, - out_channels, - config.downsample_padding, - ); - Some(downsample) - } else { - None - }; - Self { resnets, downsampler, config } - } -} - -impl Module for DownEncoderBlock2D { - fn forward(&self, xs: &Tensor) -> Tensor { - let mut xs = xs.shallow_clone(); - for resnet in self.resnets.iter() { - xs = resnet.forward(&xs, None) - } - match &self.downsampler { - Some(downsampler) => xs.apply(downsampler), - None => xs, - } - } -} - -#[derive(Debug, Clone, Copy)] -struct UpDecoderBlock2DConfig { - num_layers: i64, - resnet_eps: f64, - resnet_groups: i64, - output_scale_factor: f64, - add_upsample: bool, -} - -impl Default for UpDecoderBlock2DConfig { - fn default() -> Self { - Self { - num_layers: 1, - resnet_eps: 1e-6, - resnet_groups: 32, - output_scale_factor: 1., - add_upsample: true, - } - } -} - -#[derive(Debug)] -struct UpDecoderBlock2D { - resnets: Vec, - upsampler: Option, - #[allow(dead_code)] - config: UpDecoderBlock2DConfig, -} - -impl UpDecoderBlock2D { - fn new( - vs: nn::Path, - in_channels: i64, - out_channels: i64, - config: UpDecoderBlock2DConfig, - ) -> Self { - let resnets: Vec<_> = { - let vs = &vs / "resnets"; - let conv_cfg = ResnetBlock2DConfig { - out_channels: Some(out_channels), - eps: config.resnet_eps, - groups: config.resnet_groups, - output_scale_factor: config.output_scale_factor, - temb_channels: None, - ..Default::default() - }; - (0..(config.num_layers)) - .map(|i| { - let in_channels = if i == 0 { in_channels } else { out_channels }; - ResnetBlock2D::new(&vs / i, in_channels, conv_cfg) - }) - .collect() - }; - let upsampler = if config.add_upsample { - let upsample = Upsample2D::new(&vs / "upsamplers" / 0, out_channels, out_channels); - Some(upsample) - } else { - None - }; - Self { resnets, upsampler, config } - } -} - -impl Module for UpDecoderBlock2D { - fn forward(&self, xs: &Tensor) -> Tensor { - let mut xs = xs.shallow_clone(); - for resnet in self.resnets.iter() { - xs = resnet.forward(&xs, None) - } - match &self.upsampler { - Some(upsampler) => upsampler.forward(&xs, None), - None => xs, - } - } -} - -#[derive(Debug, Clone, Copy)] -struct UNetMidBlock2DConfig { - num_layers: i64, - resnet_eps: f64, - resnet_groups: Option, - attn_num_head_channels: Option, - // attention_type "default" - output_scale_factor: f64, -} - -impl Default for UNetMidBlock2DConfig { - fn default() -> Self { - Self { - num_layers: 1, - resnet_eps: 1e-6, - resnet_groups: Some(32), - attn_num_head_channels: Some(1), - output_scale_factor: 1., - } - } -} - -#[derive(Debug)] -struct UNetMidBlock2D { - resnet: ResnetBlock2D, - attn_resnets: Vec<(AttentionBlock, ResnetBlock2D)>, - #[allow(dead_code)] - config: UNetMidBlock2DConfig, -} - -impl UNetMidBlock2D { - fn new( - vs: nn::Path, - in_channels: i64, - temb_channels: Option, - config: UNetMidBlock2DConfig, - ) -> Self { - let vs_resnets = &vs / "resnets"; - let vs_attns = &vs / "attentions"; - let resnet_groups = config.resnet_groups.unwrap_or_else(|| i64::min(in_channels / 4, 32)); - let resnet_cfg = ResnetBlock2DConfig { - eps: config.resnet_eps, - groups: resnet_groups, - output_scale_factor: config.output_scale_factor, - temb_channels, - ..Default::default() - }; - let resnet = ResnetBlock2D::new(&vs_resnets / "0", in_channels, resnet_cfg); - let attn_cfg = AttentionBlockConfig { - num_head_channels: config.attn_num_head_channels, - num_groups: resnet_groups, - rescale_output_factor: config.output_scale_factor, - eps: config.resnet_eps, - }; - let mut attn_resnets = vec![]; - for index in 0..config.num_layers { - let attn = AttentionBlock::new(&vs_attns / index, in_channels, attn_cfg); - let resnet = ResnetBlock2D::new(&vs_resnets / (index + 1), in_channels, resnet_cfg); - attn_resnets.push((attn, resnet)) - } - Self { resnet, attn_resnets, config } - } - - fn forward(&self, xs: &Tensor, temb: Option<&Tensor>) -> Tensor { - let mut xs = self.resnet.forward(xs, temb); - for (attn, resnet) in self.attn_resnets.iter() { - xs = resnet.forward(&xs.apply(attn), temb) - } - xs - } -} - -#[derive(Debug, Clone, Copy)] -struct UNetMidBlock2DCrossAttnConfig { - num_layers: i64, - resnet_eps: f64, - resnet_groups: Option, - attn_num_head_channels: i64, - // attention_type "default" - output_scale_factor: f64, - cross_attn_dim: i64, -} - -impl Default for UNetMidBlock2DCrossAttnConfig { - fn default() -> Self { - Self { - num_layers: 1, - resnet_eps: 1e-6, - resnet_groups: Some(32), - attn_num_head_channels: 1, - output_scale_factor: 1., - cross_attn_dim: 1280, - } - } -} - -#[derive(Debug)] -struct UNetMidBlock2DCrossAttn { - resnet: ResnetBlock2D, - attn_resnets: Vec<(SpatialTransformer, ResnetBlock2D)>, - #[allow(dead_code)] - config: UNetMidBlock2DCrossAttnConfig, -} - -impl UNetMidBlock2DCrossAttn { - fn new( - vs: nn::Path, - in_channels: i64, - temb_channels: Option, - config: UNetMidBlock2DCrossAttnConfig, - ) -> Self { - let vs_resnets = &vs / "resnets"; - let vs_attns = &vs / "attentions"; - let resnet_groups = config.resnet_groups.unwrap_or_else(|| i64::min(in_channels / 4, 32)); - let resnet_cfg = ResnetBlock2DConfig { - eps: config.resnet_eps, - groups: resnet_groups, - output_scale_factor: config.output_scale_factor, - temb_channels, - ..Default::default() - }; - let resnet = ResnetBlock2D::new(&vs_resnets / "0", in_channels, resnet_cfg); - let n_heads = config.attn_num_head_channels; - let attn_cfg = SpatialTransformerConfig { - depth: 1, - num_groups: resnet_groups, - context_dim: Some(config.cross_attn_dim), - }; - let mut attn_resnets = vec![]; - for index in 0..config.num_layers { - let attn = SpatialTransformer::new( - &vs_attns / index, - in_channels, - n_heads, - in_channels / n_heads, - attn_cfg, - ); - let resnet = ResnetBlock2D::new(&vs_resnets / (index + 1), in_channels, resnet_cfg); - attn_resnets.push((attn, resnet)) - } - Self { resnet, attn_resnets, config } - } - - fn forward( - &self, - xs: &Tensor, - temb: Option<&Tensor>, - encoder_hidden_states: Option<&Tensor>, - ) -> Tensor { - let mut xs = self.resnet.forward(xs, temb); - for (attn, resnet) in self.attn_resnets.iter() { - xs = resnet.forward(&attn.forward(&xs, encoder_hidden_states), temb) - } - xs - } -} - -#[derive(Debug, Clone)] -struct EncoderConfig { - // down_block_types: DownEncoderBlock2D - block_out_channels: Vec, - layers_per_block: i64, - norm_num_groups: i64, - double_z: bool, -} - -impl Default for EncoderConfig { - fn default() -> Self { - Self { - block_out_channels: vec![64], - layers_per_block: 2, - norm_num_groups: 32, - double_z: true, - } - } -} - -#[derive(Debug)] -struct Encoder { - conv_in: nn::Conv2D, - down_blocks: Vec, - mid_block: UNetMidBlock2D, - conv_norm_out: nn::GroupNorm, - conv_out: nn::Conv2D, - #[allow(dead_code)] - config: EncoderConfig, -} - -impl Encoder { - fn new(vs: nn::Path, in_channels: i64, out_channels: i64, config: EncoderConfig) -> Self { - let conv_cfg = nn::ConvConfig { stride: 1, padding: 1, ..Default::default() }; - let conv_in = - nn::conv2d(&vs / "conv_in", in_channels, config.block_out_channels[0], 3, conv_cfg); - let mut down_blocks = vec![]; - let vs_down_blocks = &vs / "down_blocks"; - for index in 0..config.block_out_channels.len() { - let out_channels = config.block_out_channels[index]; - let in_channels = if index > 0 { - config.block_out_channels[index - 1] - } else { - config.block_out_channels[0] - }; - let is_final = index + 1 == config.block_out_channels.len(); - let cfg = DownEncoderBlock2DConfig { - num_layers: config.layers_per_block, - resnet_eps: 1e-6, - resnet_groups: config.norm_num_groups, - add_downsample: !is_final, - downsample_padding: 0, - ..Default::default() - }; - let down_block = - DownEncoderBlock2D::new(&vs_down_blocks / index, in_channels, out_channels, cfg); - down_blocks.push(down_block) - } - let last_block_out_channels = *config.block_out_channels.last().unwrap(); - let mid_cfg = UNetMidBlock2DConfig { - resnet_eps: 1e-6, - output_scale_factor: 1., - attn_num_head_channels: None, - resnet_groups: Some(config.norm_num_groups), - ..Default::default() - }; - let mid_block = - UNetMidBlock2D::new(&vs / "mid_block", last_block_out_channels, None, mid_cfg); - let group_cfg = nn::GroupNormConfig { eps: 1e-6, ..Default::default() }; - let conv_norm_out = nn::group_norm( - &vs / "conv_norm_out", - config.norm_num_groups, - last_block_out_channels, - group_cfg, - ); - let conv_out_channels = if config.double_z { 2 * out_channels } else { out_channels }; - let conv_cfg = nn::ConvConfig { padding: 1, ..Default::default() }; - let conv_out = - nn::conv2d(&vs / "conv_out", last_block_out_channels, conv_out_channels, 3, conv_cfg); - Self { conv_in, down_blocks, mid_block, conv_norm_out, conv_out, config } - } -} - -impl Module for Encoder { - fn forward(&self, xs: &Tensor) -> Tensor { - let mut xs = xs.apply(&self.conv_in); - for down_block in self.down_blocks.iter() { - xs = xs.apply(down_block) - } - self.mid_block.forward(&xs, None).apply(&self.conv_norm_out).silu().apply(&self.conv_out) - } -} - -#[derive(Debug, Clone)] -struct DecoderConfig { - // up_block_types: UpDecoderBlock2D - block_out_channels: Vec, - layers_per_block: i64, - norm_num_groups: i64, -} - -impl Default for DecoderConfig { - fn default() -> Self { - Self { block_out_channels: vec![64], layers_per_block: 2, norm_num_groups: 32 } - } -} - -#[derive(Debug)] -struct Decoder { - conv_in: nn::Conv2D, - up_blocks: Vec, - mid_block: UNetMidBlock2D, - conv_norm_out: nn::GroupNorm, - conv_out: nn::Conv2D, - #[allow(dead_code)] - config: DecoderConfig, -} - -impl Decoder { - fn new(vs: nn::Path, in_channels: i64, out_channels: i64, config: DecoderConfig) -> Self { - let n_block_out_channels = config.block_out_channels.len(); - let last_block_out_channels = *config.block_out_channels.last().unwrap(); - let conv_cfg = nn::ConvConfig { stride: 1, padding: 1, ..Default::default() }; - let conv_in = - nn::conv2d(&vs / "conv_in", in_channels, last_block_out_channels, 3, conv_cfg); - let mid_cfg = UNetMidBlock2DConfig { - resnet_eps: 1e-6, - output_scale_factor: 1., - attn_num_head_channels: None, - resnet_groups: Some(config.norm_num_groups), - ..Default::default() - }; - let mid_block = - UNetMidBlock2D::new(&vs / "mid_block", last_block_out_channels, None, mid_cfg); - let mut up_blocks = vec![]; - let vs_up_blocks = &vs / "up_blocks"; - let reversed_block_out_channels: Vec<_> = - config.block_out_channels.iter().copied().rev().collect(); - for index in 0..n_block_out_channels { - let out_channels = reversed_block_out_channels[index]; - let in_channels = if index > 0 { - reversed_block_out_channels[index - 1] - } else { - reversed_block_out_channels[0] - }; - let is_final = index + 1 == n_block_out_channels; - let cfg = UpDecoderBlock2DConfig { - num_layers: config.layers_per_block + 1, - resnet_eps: 1e-6, - resnet_groups: config.norm_num_groups, - add_upsample: !is_final, - ..Default::default() - }; - let up_block = - UpDecoderBlock2D::new(&vs_up_blocks / index, in_channels, out_channels, cfg); - up_blocks.push(up_block) - } - let group_cfg = nn::GroupNormConfig { eps: 1e-6, ..Default::default() }; - let conv_norm_out = nn::group_norm( - &vs / "conv_norm_out", - config.norm_num_groups, - config.block_out_channels[0], - group_cfg, - ); - let conv_cfg = nn::ConvConfig { padding: 1, ..Default::default() }; - let conv_out = - nn::conv2d(&vs / "conv_out", config.block_out_channels[0], out_channels, 3, conv_cfg); - Self { conv_in, up_blocks, mid_block, conv_norm_out, conv_out, config } - } -} - -impl Module for Decoder { - fn forward(&self, xs: &Tensor) -> Tensor { - let mut xs = self.mid_block.forward(&xs.apply(&self.conv_in), None); - for up_block in self.up_blocks.iter() { - xs = xs.apply(up_block) - } - xs.apply(&self.conv_norm_out).silu().apply(&self.conv_out) - } -} - -#[derive(Debug, Clone)] -struct AutoEncoderKLConfig { - block_out_channels: Vec, - layers_per_block: i64, - latent_channels: i64, - norm_num_groups: i64, -} - -impl Default for AutoEncoderKLConfig { - fn default() -> Self { - Self { - block_out_channels: vec![64], - layers_per_block: 1, - latent_channels: 4, - norm_num_groups: 32, - } - } -} - -// https://github.com/huggingface/diffusers/blob/970e30606c2944e3286f56e8eb6d3dc6d1eb85f7/src/diffusers/models/vae.py#L485 -// This implementation is specific to the config used in stable-diffusion-v1-4 -// https://huggingface.co/CompVis/stable-diffusion-v1-4/blob/main/vae/config.json -#[derive(Debug)] -struct AutoEncoderKL { - encoder: Encoder, - decoder: Decoder, - quant_conv: nn::Conv2D, - post_quant_conv: nn::Conv2D, - #[allow(dead_code)] - config: AutoEncoderKLConfig, -} - -impl AutoEncoderKL { - fn new(vs: nn::Path, in_channels: i64, out_channels: i64, config: AutoEncoderKLConfig) -> Self { - let latent_channels = config.latent_channels; - let encoder_cfg = EncoderConfig { - block_out_channels: config.block_out_channels.clone(), - layers_per_block: config.layers_per_block, - norm_num_groups: config.norm_num_groups, - double_z: true, - }; - let encoder = Encoder::new(&vs / "encoder", in_channels, latent_channels, encoder_cfg); - let decoder_cfg = DecoderConfig { - block_out_channels: config.block_out_channels.clone(), - layers_per_block: config.layers_per_block, - norm_num_groups: config.norm_num_groups, - }; - let decoder = Decoder::new(&vs / "decoder", latent_channels, out_channels, decoder_cfg); - let conv_cfg = Default::default(); - let quant_conv = - nn::conv2d(&vs / "quant_conv", 2 * latent_channels, 2 * latent_channels, 1, conv_cfg); - let post_quant_conv = - nn::conv2d(&vs / "post_quant_conv", latent_channels, latent_channels, 1, conv_cfg); - Self { encoder, decoder, quant_conv, post_quant_conv, config } - } - - // Returns the parameters of the latent distribution. - #[allow(dead_code)] - fn encode(&self, xs: &Tensor) -> Tensor { - xs.apply(&self.encoder).apply(&self.quant_conv) - } - - /// Takes as input some sampled values. - fn decode(&self, xs: &Tensor) -> Tensor { - xs.apply(&self.post_quant_conv).apply(&self.decoder) - } -} - -#[derive(Debug, Clone, Copy)] -struct DownBlock2DConfig { - num_layers: i64, - resnet_eps: f64, - // resnet_time_scale_shift: "default" - // resnet_act_fn: "swish" - resnet_groups: i64, - output_scale_factor: f64, - add_downsample: bool, - downsample_padding: i64, -} - -impl Default for DownBlock2DConfig { - fn default() -> Self { - Self { - num_layers: 1, - resnet_eps: 1e-6, - resnet_groups: 32, - output_scale_factor: 1., - add_downsample: true, - downsample_padding: 1, - } - } -} - -#[derive(Debug)] -struct DownBlock2D { - resnets: Vec, - downsampler: Option, - #[allow(dead_code)] - config: DownBlock2DConfig, -} - -impl DownBlock2D { - fn new( - vs: nn::Path, - in_channels: i64, - out_channels: i64, - temb_channels: Option, - config: DownBlock2DConfig, - ) -> Self { - let vs_resnets = &vs / "resnets"; - let resnet_cfg = ResnetBlock2DConfig { - out_channels: Some(out_channels), - eps: config.resnet_eps, - output_scale_factor: config.output_scale_factor, - temb_channels, - ..Default::default() - }; - let resnets = (0..config.num_layers) - .map(|i| { - let in_channels = if i == 0 { in_channels } else { out_channels }; - ResnetBlock2D::new(&vs_resnets / i, in_channels, resnet_cfg) - }) - .collect(); - let downsampler = if config.add_downsample { - let downsampler = Downsample2D::new( - &vs / "downsamplers" / 0, - out_channels, - true, - out_channels, - config.downsample_padding, - ); - Some(downsampler) - } else { - None - }; - Self { resnets, downsampler, config } - } - - fn forward(&self, xs: &Tensor, temb: Option<&Tensor>) -> (Tensor, Vec) { - let mut xs = xs.shallow_clone(); - let mut output_states = vec![]; - for resnet in self.resnets.iter() { - xs = resnet.forward(&xs, temb); - output_states.push(xs.shallow_clone()); - } - let xs = match &self.downsampler { - Some(downsampler) => { - let xs = xs.apply(downsampler); - output_states.push(xs.shallow_clone()); - xs - } - None => xs, - }; - (xs, output_states) - } -} - -#[derive(Debug, Clone, Copy)] -struct CrossAttnDownBlock2DConfig { - downblock: DownBlock2DConfig, - attn_num_head_channels: i64, - cross_attention_dim: i64, - // attention_type: "default" -} - -impl Default for CrossAttnDownBlock2DConfig { - fn default() -> Self { - Self { downblock: Default::default(), attn_num_head_channels: 1, cross_attention_dim: 1280 } - } -} - -#[derive(Debug)] -struct CrossAttnDownBlock2D { - downblock: DownBlock2D, - attentions: Vec, - #[allow(dead_code)] - config: CrossAttnDownBlock2DConfig, -} - -impl CrossAttnDownBlock2D { - fn new( - vs: nn::Path, - in_channels: i64, - out_channels: i64, - temb_channels: Option, - config: CrossAttnDownBlock2DConfig, - ) -> Self { - let downblock = DownBlock2D::new( - vs.clone(), - in_channels, - out_channels, - temb_channels, - config.downblock, - ); - let n_heads = config.attn_num_head_channels; - let cfg = SpatialTransformerConfig { - depth: 1, - context_dim: Some(config.cross_attention_dim), - num_groups: config.downblock.resnet_groups, - }; - let vs_attn = &vs / "attentions"; - let attentions = (0..config.downblock.num_layers) - .map(|i| { - SpatialTransformer::new( - &vs_attn / i, - out_channels, - n_heads, - out_channels / n_heads, - cfg, - ) - }) - .collect(); - Self { downblock, attentions, config } - } - - fn forward( - &self, - xs: &Tensor, - temb: Option<&Tensor>, - encoder_hidden_states: Option<&Tensor>, - ) -> (Tensor, Vec) { - let mut output_states = vec![]; - let mut xs = xs.shallow_clone(); - for (resnet, attn) in self.downblock.resnets.iter().zip(self.attentions.iter()) { - xs = resnet.forward(&xs, temb); - xs = attn.forward(&xs, encoder_hidden_states); - output_states.push(xs.shallow_clone()); - } - let xs = match &self.downblock.downsampler { - Some(downsampler) => { - let xs = xs.apply(downsampler); - output_states.push(xs.shallow_clone()); - xs - } - None => xs, - }; - (xs, output_states) - } -} - -#[derive(Debug, Clone, Copy)] -struct UpBlock2DConfig { - num_layers: i64, - resnet_eps: f64, - // resnet_time_scale_shift: "default" - // resnet_act_fn: "swish" - resnet_groups: i64, - output_scale_factor: f64, - add_upsample: bool, -} - -impl Default for UpBlock2DConfig { - fn default() -> Self { - Self { - num_layers: 1, - resnet_eps: 1e-6, - resnet_groups: 32, - output_scale_factor: 1., - add_upsample: true, - } - } -} - -#[derive(Debug)] -struct UpBlock2D { - resnets: Vec, - upsampler: Option, - #[allow(dead_code)] - config: UpBlock2DConfig, -} - -impl UpBlock2D { - fn new( - vs: nn::Path, - in_channels: i64, - prev_output_channels: i64, - out_channels: i64, - temb_channels: Option, - config: UpBlock2DConfig, - ) -> Self { - let vs_resnets = &vs / "resnets"; - let resnet_cfg = ResnetBlock2DConfig { - out_channels: Some(out_channels), - temb_channels, - eps: config.resnet_eps, - output_scale_factor: config.output_scale_factor, - ..Default::default() - }; - let resnets = (0..config.num_layers) - .map(|i| { - let res_skip_channels = - if i == config.num_layers - 1 { in_channels } else { out_channels }; - let resnet_in_channels = if i == 0 { prev_output_channels } else { out_channels }; - let in_channels = resnet_in_channels + res_skip_channels; - ResnetBlock2D::new(&vs_resnets / i, in_channels, resnet_cfg) - }) - .collect(); - let upsampler = if config.add_upsample { - let upsampler = Upsample2D::new(&vs / "upsamplers" / 0, out_channels, out_channels); - Some(upsampler) - } else { - None - }; - Self { resnets, upsampler, config } - } - - fn forward( - &self, - xs: &Tensor, - res_xs: &[Tensor], - temb: Option<&Tensor>, - upsample_size: Option<(i64, i64)>, - ) -> Tensor { - let mut xs = xs.shallow_clone(); - for (index, resnet) in self.resnets.iter().enumerate() { - xs = Tensor::cat(&[&xs, &res_xs[res_xs.len() - index - 1]], 1); - xs = resnet.forward(&xs, temb); - } - match &self.upsampler { - Some(upsampler) => upsampler.forward(&xs, upsample_size), - None => xs, - } - } -} - -#[derive(Debug, Clone, Copy)] -struct CrossAttnUpBlock2DConfig { - upblock: UpBlock2DConfig, - attn_num_head_channels: i64, - cross_attention_dim: i64, - // attention_type: "default" -} - -impl Default for CrossAttnUpBlock2DConfig { - fn default() -> Self { - Self { upblock: Default::default(), attn_num_head_channels: 1, cross_attention_dim: 1280 } - } -} - -#[derive(Debug)] -struct CrossAttnUpBlock2D { - upblock: UpBlock2D, - attentions: Vec, - #[allow(dead_code)] - config: CrossAttnUpBlock2DConfig, -} - -impl CrossAttnUpBlock2D { - fn new( - vs: nn::Path, - in_channels: i64, - prev_output_channels: i64, - out_channels: i64, - temb_channels: Option, - config: CrossAttnUpBlock2DConfig, - ) -> Self { - let upblock = UpBlock2D::new( - vs.clone(), - in_channels, - prev_output_channels, - out_channels, - temb_channels, - config.upblock, - ); - let n_heads = config.attn_num_head_channels; - let cfg = SpatialTransformerConfig { - depth: 1, - context_dim: Some(config.cross_attention_dim), - num_groups: config.upblock.resnet_groups, - }; - let vs_attn = &vs / "attentions"; - let attentions = (0..config.upblock.num_layers) - .map(|i| { - SpatialTransformer::new( - &vs_attn / i, - out_channels, - n_heads, - out_channels / n_heads, - cfg, - ) - }) - .collect(); - Self { upblock, attentions, config } - } - - fn forward( - &self, - xs: &Tensor, - res_xs: &[Tensor], - temb: Option<&Tensor>, - upsample_size: Option<(i64, i64)>, - encoder_hidden_states: Option<&Tensor>, - ) -> Tensor { - let mut xs = xs.shallow_clone(); - for (index, resnet) in self.upblock.resnets.iter().enumerate() { - xs = Tensor::cat(&[&xs, &res_xs[res_xs.len() - index - 1]], 1); - xs = resnet.forward(&xs, temb); - xs = self.attentions[index].forward(&xs, encoder_hidden_states); - } - match &self.upblock.upsampler { - Some(upsampler) => upsampler.forward(&xs, upsample_size), - None => xs, - } - } -} - -#[derive(Debug)] -struct Timesteps { - num_channels: i64, - flip_sin_to_cos: bool, - downscale_freq_shift: f64, - device: Device, -} - -impl Timesteps { - fn new( - num_channels: i64, - flip_sin_to_cos: bool, - downscale_freq_shift: f64, - device: Device, - ) -> Self { - Self { num_channels, flip_sin_to_cos, downscale_freq_shift, device } - } -} - -impl Module for Timesteps { - fn forward(&self, xs: &Tensor) -> Tensor { - let half_dim = self.num_channels / 2; - let exponent = Tensor::arange(half_dim, (Kind::Float, self.device)) * -f64::ln(10000.); - let exponent = exponent / (half_dim as f64 - self.downscale_freq_shift); - let emb = exponent.exp(); - // emb = timesteps[:, None].float() * emb[None, :] - let emb = xs.unsqueeze(-1) * emb.unsqueeze(0); - let emb = if self.flip_sin_to_cos { - Tensor::cat(&[emb.cos(), emb.sin()], -1) - } else { - Tensor::cat(&[emb.sin(), emb.cos()], -1) - }; - if self.num_channels % 2 == 1 { - emb.pad([0, 1, 0, 0], "constant", None) - } else { - emb - } - } -} - -#[derive(Debug)] -struct TimestepEmbedding { - linear_1: nn::Linear, - linear_2: nn::Linear, -} - -impl TimestepEmbedding { - // act_fn: "silu" - fn new(vs: nn::Path, channel: i64, time_embed_dim: i64) -> Self { - let linear_cfg = Default::default(); - let linear_1 = nn::linear(&vs / "linear_1", channel, time_embed_dim, linear_cfg); - let linear_2 = nn::linear(&vs / "linear_2", time_embed_dim, time_embed_dim, linear_cfg); - Self { linear_1, linear_2 } - } -} - -impl Module for TimestepEmbedding { - fn forward(&self, xs: &Tensor) -> Tensor { - xs.apply(&self.linear_1).silu().apply(&self.linear_2) - } -} - -#[derive(Debug, Clone, Copy)] -struct BlockConfig { - out_channels: i64, - use_cross_attn: bool, -} - -#[derive(Debug, Clone)] -struct UNet2DConditionModelConfig { - center_input_sample: bool, - flip_sin_to_cos: bool, - freq_shift: f64, - blocks: Vec, - layers_per_block: i64, - downsample_padding: i64, - mid_block_scale_factor: f64, - norm_num_groups: i64, - norm_eps: f64, - cross_attention_dim: i64, - attention_head_dim: i64, -} - -impl Default for UNet2DConditionModelConfig { - fn default() -> Self { - Self { - center_input_sample: false, - flip_sin_to_cos: true, - freq_shift: 0., - blocks: vec![ - BlockConfig { out_channels: 320, use_cross_attn: true }, - BlockConfig { out_channels: 640, use_cross_attn: true }, - BlockConfig { out_channels: 1280, use_cross_attn: true }, - BlockConfig { out_channels: 1280, use_cross_attn: false }, - ], - layers_per_block: 2, - downsample_padding: 1, - mid_block_scale_factor: 1., - norm_num_groups: 32, - norm_eps: 1e-5, - cross_attention_dim: 1280, - attention_head_dim: 8, - } - } -} - -#[derive(Debug)] -enum UNetDownBlock { - Basic(DownBlock2D), - CrossAttn(CrossAttnDownBlock2D), -} - -#[derive(Debug)] -enum UNetUpBlock { - Basic(UpBlock2D), - CrossAttn(CrossAttnUpBlock2D), -} - -#[derive(Debug)] -struct UNet2DConditionModel { - conv_in: nn::Conv2D, - time_proj: Timesteps, - time_embedding: TimestepEmbedding, - down_blocks: Vec, - mid_block: UNetMidBlock2DCrossAttn, - up_blocks: Vec, - conv_norm_out: nn::GroupNorm, - conv_out: nn::Conv2D, - config: UNet2DConditionModelConfig, -} - -impl UNet2DConditionModel { - fn new( - vs: nn::Path, - in_channels: i64, - out_channels: i64, - config: UNet2DConditionModelConfig, - ) -> Self { - let n_blocks = config.blocks.len(); - let b_channels = config.blocks[0].out_channels; - let bl_channels = config.blocks.last().unwrap().out_channels; - let time_embed_dim = b_channels * 4; - let conv_cfg = nn::ConvConfig { stride: 1, padding: 1, ..Default::default() }; - let conv_in = nn::conv2d(&vs / "conv_in", in_channels, b_channels, 3, conv_cfg); - - let time_proj = - Timesteps::new(b_channels, config.flip_sin_to_cos, config.freq_shift, vs.device()); - let time_embedding = - TimestepEmbedding::new(&vs / "time_embedding", b_channels, time_embed_dim); - - let vs_db = &vs / "down_blocks"; - let down_blocks = (0..n_blocks) - .map(|i| { - let BlockConfig { out_channels, use_cross_attn } = config.blocks[i]; - let in_channels = - if i > 0 { config.blocks[i - 1].out_channels } else { b_channels }; - let db_cfg = DownBlock2DConfig { - num_layers: config.layers_per_block, - resnet_eps: config.norm_eps, - resnet_groups: config.norm_num_groups, - add_downsample: i < n_blocks - 1, - downsample_padding: config.downsample_padding, - ..Default::default() - }; - if use_cross_attn { - let config = CrossAttnDownBlock2DConfig { - downblock: db_cfg, - attn_num_head_channels: config.attention_head_dim, - cross_attention_dim: config.cross_attention_dim, - }; - let block = CrossAttnDownBlock2D::new( - &vs_db / i, - in_channels, - out_channels, - Some(time_embed_dim), - config, - ); - UNetDownBlock::CrossAttn(block) - } else { - let block = DownBlock2D::new( - &vs_db / i, - in_channels, - out_channels, - Some(time_embed_dim), - db_cfg, - ); - UNetDownBlock::Basic(block) - } - }) - .collect(); - - let mid_cfg = UNetMidBlock2DCrossAttnConfig { - resnet_eps: config.norm_eps, - output_scale_factor: config.mid_block_scale_factor, - cross_attn_dim: config.cross_attention_dim, - attn_num_head_channels: config.attention_head_dim, - resnet_groups: Some(config.norm_num_groups), - ..Default::default() - }; - let mid_block = UNetMidBlock2DCrossAttn::new( - &vs / "mid_block", - bl_channels, - Some(time_embed_dim), - mid_cfg, - ); - - let vs_ub = &vs / "up_blocks"; - let up_blocks = (0..n_blocks) - .map(|i| { - let BlockConfig { out_channels, use_cross_attn } = config.blocks[n_blocks - 1 - i]; - let prev_out_channels = - if i > 0 { config.blocks[n_blocks - i].out_channels } else { bl_channels }; - let in_channels = { - let index = if i == n_blocks - 1 { 0 } else { n_blocks - i - 2 }; - config.blocks[index].out_channels - }; - let ub_cfg = UpBlock2DConfig { - num_layers: config.layers_per_block + 1, - resnet_eps: config.norm_eps, - resnet_groups: config.norm_num_groups, - add_upsample: i < n_blocks - 1, - ..Default::default() - }; - if use_cross_attn { - let config = CrossAttnUpBlock2DConfig { - upblock: ub_cfg, - attn_num_head_channels: config.attention_head_dim, - cross_attention_dim: config.cross_attention_dim, - }; - let block = CrossAttnUpBlock2D::new( - &vs_ub / i, - in_channels, - prev_out_channels, - out_channels, - Some(time_embed_dim), - config, - ); - UNetUpBlock::CrossAttn(block) - } else { - let block = UpBlock2D::new( - &vs_ub / i, - in_channels, - prev_out_channels, - out_channels, - Some(time_embed_dim), - ub_cfg, - ); - UNetUpBlock::Basic(block) - } - }) - .collect(); - - let group_cfg = nn::GroupNormConfig { eps: config.norm_eps, ..Default::default() }; - let conv_norm_out = - nn::group_norm(&vs / "conv_norm_out", config.norm_num_groups, b_channels, group_cfg); - let conv_out = nn::conv2d(&vs / "conv_out", b_channels, out_channels, 3, conv_cfg); - Self { - conv_in, - time_proj, - time_embedding, - down_blocks, - mid_block, - up_blocks, - conv_norm_out, - conv_out, - config, - } - } -} - -impl UNet2DConditionModel { - fn forward(&self, xs: &Tensor, timestep: f64, encoder_hidden_states: &Tensor) -> Tensor { - let (bsize, _channels, height, width) = xs.size4().unwrap(); - let device = xs.device(); - let n_blocks = self.config.blocks.len(); - let num_upsamplers = n_blocks - 1; - let default_overall_up_factor = 2i64.pow(num_upsamplers as u32); - let forward_upsample_size = - height % default_overall_up_factor != 0 || width % default_overall_up_factor != 0; - // 0. center input if necessary - let xs = if self.config.center_input_sample { xs * 2.0 - 1.0 } else { xs.shallow_clone() }; - // 1. time - let emb = (Tensor::ones([bsize], (Kind::Float, device)) * timestep) - .apply(&self.time_proj) - .apply(&self.time_embedding); - // 2. pre-process - let xs = xs.apply(&self.conv_in); - // 3. down - let mut down_block_res_xs = vec![xs.shallow_clone()]; - let mut xs = xs; - for down_block in self.down_blocks.iter() { - let (_xs, res_xs) = match down_block { - UNetDownBlock::Basic(b) => b.forward(&xs, Some(&emb)), - UNetDownBlock::CrossAttn(b) => { - b.forward(&xs, Some(&emb), Some(encoder_hidden_states)) - } - }; - down_block_res_xs.extend(res_xs); - xs = _xs; - } - // 4. mid - let xs = self.mid_block.forward(&xs, Some(&emb), Some(encoder_hidden_states)); - // 5. up - let mut xs = xs; - let mut upsample_size = None; - for (i, up_block) in self.up_blocks.iter().enumerate() { - let n_resnets = match up_block { - UNetUpBlock::Basic(b) => b.resnets.len(), - UNetUpBlock::CrossAttn(b) => b.upblock.resnets.len(), - }; - let res_xs = down_block_res_xs.split_off(down_block_res_xs.len() - n_resnets); - if i < n_blocks - 1 && forward_upsample_size { - let (_, _, h, w) = down_block_res_xs.last().unwrap().size4().unwrap(); - upsample_size = Some((h, w)) - } - xs = match up_block { - UNetUpBlock::Basic(b) => b.forward(&xs, &res_xs, Some(&emb), upsample_size), - UNetUpBlock::CrossAttn(b) => { - b.forward(&xs, &res_xs, Some(&emb), upsample_size, Some(encoder_hidden_states)) - } - }; - } - // 6. post-process - xs.apply(&self.conv_norm_out).silu().apply(&self.conv_out) - } -} - -// TODO: LMSDiscreteScheduler -// https://github.com/huggingface/diffusers/blob/32bf4fdc4386809c870528cb261028baae012d27/src/diffusers/schedulers/scheduling_lms_discrete.py#L47 - -fn build_clip_transformer(device: Device) -> anyhow::Result { - let mut vs = nn::VarStore::new(device); - let text_model = ClipTextTransformer::new(vs.root()); - vs.load("data/pytorch_model.ot")?; - Ok(text_model) -} - -fn build_vae(device: Device) -> anyhow::Result { - let mut vs_ae = nn::VarStore::new(device); - // https://huggingface.co/CompVis/stable-diffusion-v1-4/blob/main/vae/config.json - let autoencoder_cfg = AutoEncoderKLConfig { - block_out_channels: vec![128, 256, 512, 512], - layers_per_block: 2, - latent_channels: 4, - norm_num_groups: 32, - }; - let autoencoder = AutoEncoderKL::new(vs_ae.root(), 3, 3, autoencoder_cfg); - vs_ae.load("data/vae.ot")?; - Ok(autoencoder) -} - -fn build_unet(device: Device) -> anyhow::Result { - let mut vs_unet = nn::VarStore::new(device); - // https://huggingface.co/CompVis/stable-diffusion-v1-4/blob/main/unet/config.json - let unet_cfg = UNet2DConditionModelConfig { - attention_head_dim: 8, - blocks: vec![ - BlockConfig { out_channels: 320, use_cross_attn: true }, - BlockConfig { out_channels: 640, use_cross_attn: true }, - BlockConfig { out_channels: 1280, use_cross_attn: true }, - BlockConfig { out_channels: 1280, use_cross_attn: false }, - ], - center_input_sample: false, - cross_attention_dim: 768, - downsample_padding: 1, - flip_sin_to_cos: true, - freq_shift: 0., - layers_per_block: 2, - mid_block_scale_factor: 1., - norm_eps: 1e-5, - norm_num_groups: 32, - }; - let unet = UNet2DConditionModel::new(vs_unet.root(), 4, 4, unet_cfg); - vs_unet.load("data/unet.ot")?; - Ok(unet) -} - -#[derive(Debug, Clone, Copy)] -enum BetaSchedule { - #[allow(dead_code)] - Linear, - ScaledLinear, -} - -#[derive(Debug, Clone, Copy)] -struct DDIMSchedulerConfig { - beta_start: f64, - beta_end: f64, - beta_schedule: BetaSchedule, - eta: f64, -} - -impl Default for DDIMSchedulerConfig { - fn default() -> Self { - Self { - beta_start: 0.00085f64, - beta_end: 0.012f64, - beta_schedule: BetaSchedule::ScaledLinear, - eta: 0., - } - } -} - -#[derive(Debug, Clone)] -struct DDIMScheduler { - timesteps: Vec, - alphas_cumprod: Vec, - step_ratio: usize, - #[allow(dead_code)] - config: DDIMSchedulerConfig, -} - -// clip_sample: False, set_alpha_to_one: False -impl DDIMScheduler { - fn new(inference_steps: usize, train_timesteps: usize, config: DDIMSchedulerConfig) -> Self { - let step_ratio = train_timesteps / inference_steps; - // TODO: Remove this hack which aimed at matching the behavior of diffusers==0.2.4 - let timesteps = (0..(inference_steps + 1)).map(|s| s * step_ratio).rev().collect(); - let betas = match config.beta_schedule { - BetaSchedule::ScaledLinear => Tensor::linspace( - config.beta_start.sqrt(), - config.beta_end.sqrt(), - train_timesteps as i64, - kind::FLOAT_CPU, - ) - .square(), - BetaSchedule::Linear => Tensor::linspace( - config.beta_start, - config.beta_end, - train_timesteps as i64, - kind::FLOAT_CPU, - ), - }; - let alphas: Tensor = 1.0 - betas; - let alphas_cumprod = Vec::::try_from(alphas.cumprod(0, Kind::Double)).unwrap(); - Self { alphas_cumprod, timesteps, step_ratio, config } - } - - // https://github.com/huggingface/diffusers/blob/6e099e2c8ce4c4f5c7318e970a8c093dc5c7046e/src/diffusers/schedulers/scheduling_ddim.py#L195 - fn step(&self, model_output: &Tensor, timestep: usize, sample: &Tensor) -> Tensor { - let prev_timestep = timestep.saturating_sub(self.step_ratio); - - let alpha_prod_t = self.alphas_cumprod[timestep]; - let alpha_prod_t_prev = self.alphas_cumprod[prev_timestep]; - let beta_prod_t = 1. - alpha_prod_t; - let beta_prod_t_prev = 1. - alpha_prod_t_prev; - - let pred_original_sample = - (sample - beta_prod_t.sqrt() * model_output) / alpha_prod_t.sqrt(); - - let variance = (beta_prod_t_prev / beta_prod_t) * (1. - alpha_prod_t / alpha_prod_t_prev); - let std_dev_t = self.config.eta * variance.sqrt(); - - let pred_sample_direction = - (1. - alpha_prod_t_prev - std_dev_t * std_dev_t).sqrt() * model_output; - let prev_sample = alpha_prod_t_prev.sqrt() * pred_original_sample + pred_sample_direction; - if self.config.eta > 0. { - &prev_sample + Tensor::randn_like(&prev_sample) * std_dev_t - } else { - prev_sample - } - } -} - -fn main() -> anyhow::Result<()> { - println!("Cuda available: {}", tch::Cuda::is_available()); - println!("Cudnn available: {}", tch::Cuda::cudnn_is_available()); - // TODO: Switch to using claps to allow more flags? - let mut prompt = "A rusty robot holding a fire torch in its hand".to_string(); - let mut device = Device::cuda_if_available(); - for arg in std::env::args().skip(1) { - if arg.as_str() == "cpu" { - device = Device::Cpu; - } else { - prompt = arg; - } - } - let n_steps = 30; - let scheduler = DDIMScheduler::new(n_steps, 1000, Default::default()); - - let tokenizer = Tokenizer::create("data/bpe_simple_vocab_16e6.txt")?; - let tokens = tokenizer.encode(&prompt, Some(MAX_POSITION_EMBEDDINGS))?; - let str = tokenizer.decode(&tokens); - println!("Str: {str}"); - let tokens: Vec = tokens.iter().map(|x| *x as i64).collect(); - let tokens = Tensor::from_slice(&tokens).view((1, -1)).to(device); - let uncond_tokens = tokenizer.encode("", Some(MAX_POSITION_EMBEDDINGS))?; - let uncond_tokens: Vec = uncond_tokens.iter().map(|x| *x as i64).collect(); - let uncond_tokens = Tensor::from_slice(&uncond_tokens).view((1, -1)).to(device); - println!("Tokens: {tokens:?}"); - - let no_grad_guard = tch::no_grad_guard(); - println!("Building the Clip transformer."); - let text_model = build_clip_transformer(device)?; - let text_embeddings = text_model.forward(&tokens); - let uncond_embeddings = text_model.forward(&uncond_tokens); - let text_embeddings = Tensor::cat(&[uncond_embeddings, text_embeddings], 0); - println!("Text embeddings: {text_embeddings:?}"); - - println!("Building the autoencoder."); - let vae = build_vae(device)?; - println!("Building the unet."); - let unet = build_unet(device)?; - - let bsize = 1; - // DETERMINISTIC SEEDING - tch::manual_seed(32); - let mut latents = Tensor::randn([bsize, 4, HEIGHT / 8, WIDTH / 8], (Kind::Float, device)); - - for (timestep_index, ×tep) in scheduler.timesteps.iter().enumerate() { - println!("Timestep {timestep_index} {timestep} {latents:?}"); - let latent_model_input = Tensor::cat(&[&latents, &latents], 0); - let noise_pred = unet.forward(&latent_model_input, timestep as f64, &text_embeddings); - let noise_pred = noise_pred.chunk(2, 0); - let (noise_pred_uncond, noise_pred_text) = (&noise_pred[0], &noise_pred[1]); - let noise_pred = noise_pred_uncond + (noise_pred_text - noise_pred_uncond) * GUIDANCE_SCALE; - latents = scheduler.step(&noise_pred, timestep, &latents); - - let image = vae.decode(&(&latents / 0.18215)); - let image = (image / 2 + 0.5).clamp(0., 1.).to_device(Device::Cpu); - let image = (image * 255.).to_kind(Kind::Uint8); - tch::vision::image::save(&image, format!("sd_{timestep_index}.png"))? - } - - drop(no_grad_guard); - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/tensor-tools.rs b/v2/dependency-patches/tch-0.24.0/examples/tensor-tools.rs deleted file mode 100644 index 3c6c6ffed3..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/tensor-tools.rs +++ /dev/null @@ -1,79 +0,0 @@ -// A small tensor tool utility. -// -// - List the content of some npy/npz/ot file. -// tensor-tools ls a.npy b.npz c.ot -// -// - Convert a npz file to an ot file. -// tensor-tools cp src.npz dst.ot -// Or the other way around. -// tensor-tools cp src.ot dst.npz - -use anyhow::{bail, ensure, Result}; - -pub fn main() -> Result<()> { - let args: Vec<_> = std::env::args().collect(); - ensure!(args.len() >= 2, "usage: {} (ls|cp) ...", args[0]); - match args[1].as_str() { - "ls" => { - for filename in args.iter().skip(2) { - if filename.ends_with(".npy") { - let tensor = tch::Tensor::read_npy(filename)?; - println!("{filename}: {tensor:?}"); - } else if filename.ends_with(".npz") { - let tensors = tch::Tensor::read_npz(filename)?; - for (name, tensor) in tensors.iter() { - println!("{filename}: {name} {tensor:?}") - } - } else if filename.ends_with(".safetensors") { - let tensors = tch::Tensor::read_safetensors(filename)?; - for (name, tensor) in tensors.iter() { - println!("{filename}: {name} {tensor:?}") - } - } else if filename.ends_with(".ot") { - let tensors = tch::Tensor::load_multi(filename)?; - for (name, tensor) in tensors.iter() { - println!("{filename}: {name} {tensor:?}") - } - } else if filename.ends_with(".bin") || filename.ends_with(".zip") { - let tensors = tch::Tensor::loadz_multi(filename)?; - for (name, tensor) in tensors.iter() { - println!("{filename}: {name} {tensor:?}") - } - } else { - bail!("unhandled file {}", filename); - } - } - } - "cp" => { - ensure!(args.len() == 4, "usage: {} cp src.ot dst.npz", args[0]); - let src_filename = &args[2]; - let dst_filename = &args[3]; - let tensors = if src_filename.ends_with(".npz") { - tch::Tensor::read_npz(src_filename)? - } else if src_filename.ends_with(".safetensors") { - tch::Tensor::read_safetensors(src_filename)? - } else if src_filename.ends_with(".ot") { - tch::Tensor::load_multi(src_filename)? - } else if src_filename.ends_with(".bin") || src_filename.ends_with(".zip") { - tch::Tensor::loadz_multi(src_filename)? - } else { - bail!("unhandled file {}", src_filename) - }; - for (name, tensor) in tensors.iter() { - println!("{src_filename}: {name} {tensor:?}") - } - if dst_filename.ends_with(".npz") { - tch::Tensor::write_npz(&tensors, dst_filename)? - } else if dst_filename.ends_with(".safetensors") { - tch::Tensor::write_safetensors(&tensors, dst_filename)? - } else if dst_filename.ends_with(".ot") { - tch::Tensor::save_multi(&tensors, dst_filename)? - } else { - bail!("unhandled file {}", dst_filename) - }; - } - _ => bail!("usage: {} (ls|cp) ...", args[0]), - } - - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/transfer-learning/README.md b/v2/dependency-patches/tch-0.24.0/examples/transfer-learning/README.md deleted file mode 100644 index 1861dfcacb..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/transfer-learning/README.md +++ /dev/null @@ -1,141 +0,0 @@ -# Transfer Learning Tutorial - -This tutorial follows the lines of the -[PyTorch Transfer Learning Tutorial](https://pytorch.org/tutorials/beginner/transfer_learning_tutorial.html). - -We will use transfer learning to leverage a pretrained ResNet model on a small dataset. -This dataset is made of images of ants and bees that we want to classify, -there are roughly 240 training images and 150 validation images each of them of size -224x224. The dataset is so small that training even the simplest convolutional neural -network on it would be very difficult. - -Instead the original tutorial proposes two alternatives to train the classifier. - -- *Finetuning the pretrained model.* We start from a ResNet-18 model pretrained on ImageNet -1000 categories, replace the last layer by a binary classifier and train the resulting model -as usual. -- *Using a pretrained model as feature extractor.* The pretrained model weights are frozen and -we run this model and store the outputs of the last layer before the final classifier. -We then train a binary classifier on the resulting features. - -We will focus on the second alternative but first we need to get the code -building and running and we also have to download the dataset and pretrained -weights. - -## Installation Instructions -Run the following commands to download the latest tch-rs version -and run the tests, this installs the CPU version of libtorch if necessary. - -```bash -git clone https://github.com/LaurentMazare/tch-rs.git -cd tch-rs -cargo test -``` - -The ants and bees dataset can be downloaded [here](https://download.pytorch.org/tutorial/hymenoptera_data.zip). -You can download the weights for a ResNet-18 network pretrained on ImageNet, -[resnet18.ot](https://github.com/LaurentMazare/tch-rs/releases/download/untagged-eb220e5c19f9bb250bd1/resnet18.ot). - -Once this is done and the dataset has been extracted we can build and run the code with: -```bash -cargo run --example transfer-learning resnet18.ot hymenoptera_data -``` - -## Loading the Data - -Let us now have a look at the code from `main.rs`. -The dataset is loaded via some helper functions. - -```rust -let dataset = imagenet::load_from_dir(dataset_dir)?; -println!("{:?}", dataset); -``` - -The `println!` macro prints the dimensions of the tensors that have -been created. For training the tensor has shape `211x3x224x224`, this -corresponds to 211 images of height and width both 224 with 3 channels -(PyTorch uses the NCHW ordering for image data). The testing image -tensor has dimensions `127` so there are 127 images with the -same size as used in training. - - -## Using a Pretrained ResNet as Feature Extractor - -The pixel data from the dataset is converted to features by running -a pre-trained ResNet model. - -```rust -let mut vs = tch::nn::VarStore::new(tch::Device::Cpu); -let net = resnet::resnet18_no_final_layer(&vs.root()); -vs.load(weights)?; - -let train_images = tch::no_grad(|| dataset.train_images.apply_t(&net, false)); -let test_images = tch::no_grad(|| dataset.test_images.apply_t(&net, false)); -``` - -This snippet performs the following steps: -- A variable store `vs` is created to hold the network weights. -- A ResNet-18 model is created using this variable store. At this point the - model weights are randomly initialized. -- `vs.load(weights)` loads the weights stored in a given file and copy their values - to some tensors. Tensors are named in the serialized file in a way that matches - the names we used when creating the ResNet model. -- Finally for each tensor of the training and testing datasets, `apply` - performs a forward pass on the model and returns the resulting tensor. In this - case the result is a vector of 512 values per sample. -The `no_grad` closure informs PyTorch that there is no need to keep a graph of the -forward pass as we do not plan on asking for gradients. - -## Training a Linear Layer on top of the Extracted Features - -Now that we have precomputed the output of the ResNet model on our training and -testing images we will train a linear binary classifier to recognize ants vs bees. - -We start by defining a model, for this we need a variable store to hold the -trainable variables. - -```rust -let vs = tch::nn::VarStore::new(tch::Device::Cpu); -let linear = nn::linear(vs.root(), 512, dataset.labels, Default::default()); -``` - -We will use stochastic gradient descent to minimize the cross-entropy loss -on the classification task. To do this we create a `sgd` optimizer and then -iterate on the training dataset. After each epoch the accuracy is computed -on the testing set and printed. - -```rust -let sgd = nn::Sgd::default().build(&vs, 1e-3)?; -for epoch_idx in 1..1001 { - let predicted = train_images.apply(&linear); - let loss = predicted.cross_entropy_for_logits(&dataset.train_labels); - sgd.backward_step(&loss); - - let test_accuracy = test_images - .apply(&linear) - .accuracy_for_logits(&dataset.test_labels); - println!("{} {:.2}%", epoch_idx, 100. * f64::from(test_accuracy)); -} -``` - -On each training step the model output is computed through a forward pass. The -cross-entropy loss is then evaluated on the resulting logits using the training labels. -The backward pass then evaluates gradients for the trainable variables of our -model and these variables are updated by the optimizer. -```rust - let predicted = train_images.apply(&linear); - let loss = predicted.cross_entropy_for_logits(&dataset.train_labels); - sgd.backward_step(&loss); -``` - -After each epoch the accuracy is evaluated on the testing set and printed out. -```rust - let test_accuracy = test_images - .apply(&linear) - .accuracy_for_logits(&dataset.test_labels); - println!("{} {:.2}%", epoch_idx, 100. * f64::from(test_accuracy)); -``` - -This should result in a `94.5%` accuracy on the testing set. -The whole code for this example can be found in [main.rs](main.rs). - diff --git a/v2/dependency-patches/tch-0.24.0/examples/transfer-learning/main.rs b/v2/dependency-patches/tch-0.24.0/examples/transfer-learning/main.rs deleted file mode 100644 index bb773e50e5..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/transfer-learning/main.rs +++ /dev/null @@ -1,42 +0,0 @@ -// This example illustrates how to use transfer learning to fine tune a pre-trained -// imagenet model on another dataset. -// -// The pre-trained weight files containing the pre-trained weights can be found here: -// https://github.com/LaurentMazare/tch-rs/releases/download/untagged-eb220e5c19f9bb250bd1/resnet18.ot -use anyhow::{bail, Result}; -use tch::nn::{self, OptimizerConfig}; -use tch::vision::{imagenet, resnet}; - -pub fn main() -> Result<()> { - let args: Vec<_> = std::env::args().collect(); - let (weights, dataset_dir) = match args.as_slice() { - [_, w, d] => (std::path::Path::new(w), d.to_owned()), - _ => bail!("usage: main resnet18.ot dataset-path"), - }; - // Load the dataset and resize it to the usual imagenet dimension of 224x224. - let dataset = imagenet::load_from_dir(dataset_dir)?; - println!("{dataset:?}"); - - // Create the model and load the weights from the file. - let mut vs = tch::nn::VarStore::new(tch::Device::Cpu); - let net = resnet::resnet18_no_final_layer(&vs.root()); - vs.load(weights)?; - - // Pre-compute the final activations. - let train_images = tch::no_grad(|| dataset.train_images.apply_t(&net, false)); - let test_images = tch::no_grad(|| dataset.test_images.apply_t(&net, false)); - - let vs = nn::VarStore::new(tch::Device::Cpu); - let linear = nn::linear(vs.root(), 512, dataset.labels, Default::default()); - let mut sgd = nn::Sgd::default().build(&vs, 1e-3)?; - - for epoch_idx in 1..1001 { - let predicted = train_images.apply(&linear); - let loss = predicted.cross_entropy_for_logits(&dataset.train_labels); - sgd.backward_step(&loss); - - let test_accuracy = test_images.apply(&linear).accuracy_for_logits(&dataset.test_labels); - println!("{} {:.2}%", epoch_idx, 100. * f64::try_from(test_accuracy)?); - } - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/translation/dataset.rs b/v2/dependency-patches/tch-0.24.0/examples/translation/dataset.rs deleted file mode 100644 index cf2f61e135..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/translation/dataset.rs +++ /dev/null @@ -1,115 +0,0 @@ -use super::lang; -use anyhow::{bail, Result}; -use std::fs::File; -use std::io::{BufRead, BufReader}; - -const PREFIXES: [&str; 12] = [ - "i am ", - "i m ", - "you are ", - "you re", - "he is ", - "he s ", - "she is ", - "she s ", - "we are ", - "we re ", - "they are ", - "they re ", -]; - -#[derive(Debug)] -pub struct Dataset { - input_lang: lang::Lang, - output_lang: lang::Lang, - pairs: Vec<(String, String)>, -} - -fn normalize(s: &str) -> String { - s.to_lowercase() - .chars() - .map(|c| match c { - '!' => " !".to_string(), - '.' => " .".to_string(), - '?' => " ?".to_string(), - _ => { - if c.is_alphanumeric() { - c.to_string() - } else { - " ".to_string() - } - } - }) - .collect() -} - -fn to_indexes(s: &str, lang: &lang::Lang) -> Vec { - let mut res = s.split_whitespace().filter_map(|x| lang.get_index(x)).collect::>(); - res.push(lang.eos_token()); - res -} - -fn filter_prefix(s: &str) -> bool { - PREFIXES.iter().any(|prefix| s.starts_with(prefix)) -} - -fn read_pairs(ilang: &str, olang: &str, max_length: usize) -> Result> { - let file = File::open(format!("data/{ilang}-{olang}.txt"))?; - let mut res: Vec<(String, String)> = vec![]; - for line in BufReader::new(file).lines() { - let line = line?; - match line.split('\t').collect::>().as_slice() { - [lhs, rhs] => { - let lhs = normalize(lhs); - let rhs = normalize(rhs); - if lhs.split_whitespace().count() < max_length - && rhs.split_whitespace().count() < max_length - && (filter_prefix(&lhs) || filter_prefix(&rhs)) - { - res.push((lhs, rhs)) - } - } - _ => bail!("a line does not contain a single tab {}", line), - } - } - Ok(res) -} - -impl Dataset { - pub fn new(ilang: &str, olang: &str, max_length: usize) -> Result { - let pairs = read_pairs(ilang, olang, max_length)?; - let mut input_lang = lang::Lang::new(ilang); - let mut output_lang = lang::Lang::new(olang); - for (lhs, rhs) in pairs.iter() { - input_lang.add_sentence(lhs); - output_lang.add_sentence(rhs); - } - let dataset = Dataset { input_lang, output_lang, pairs }; - Ok(dataset) - } - - pub fn input_lang(&self) -> &lang::Lang { - &self.input_lang - } - - pub fn output_lang(&self) -> &lang::Lang { - &self.output_lang - } - - pub fn reverse(self) -> Self { - Dataset { - input_lang: self.output_lang, - output_lang: self.input_lang, - pairs: self.pairs.into_iter().map(|(x, y)| (y, x)).collect(), - } - } - - pub fn pairs(&self) -> Vec<(Vec, Vec)> { - self.pairs - .iter() - .map(|(lhs, rhs)| { - (to_indexes(lhs, &self.input_lang), to_indexes(rhs, &self.output_lang)) - }) - .collect::>() - } -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/translation/lang.rs b/v2/dependency-patches/tch-0.24.0/examples/translation/lang.rs deleted file mode 100644 index b85371d605..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/translation/lang.rs +++ /dev/null @@ -1,73 +0,0 @@ -use std::collections::HashMap; - -const SOS_TOKEN: &str = "SOS"; -const EOS_TOKEN: &str = "EOS"; - -#[derive(Debug)] -pub struct Lang { - name: String, - word_to_index_and_count: HashMap, - index_to_word: HashMap, -} - -impl Lang { - fn add_word(&mut self, word: &str) { - if !word.is_empty() { - match self.word_to_index_and_count.get_mut(word) { - None => { - let length = self.word_to_index_and_count.len(); - self.word_to_index_and_count.insert(word.to_string(), (length, 1)); - self.index_to_word.insert(length, word.to_owned()); - } - Some((_, cnt)) => { - *cnt += 1; - } - } - } - } - - pub fn new(name: &str) -> Lang { - let mut lang = Lang { - name: name.to_string(), - word_to_index_and_count: HashMap::new(), - index_to_word: HashMap::new(), - }; - lang.add_word(SOS_TOKEN); - lang.add_word(EOS_TOKEN); - lang - } - - pub fn add_sentence(&mut self, sentence: &str) { - for word in sentence.split_whitespace() { - self.add_word(word); - } - } - - pub fn len(&self) -> usize { - self.index_to_word.len() - } - - pub fn sos_token(&self) -> usize { - self.word_to_index_and_count.get(SOS_TOKEN).unwrap().0 - } - - pub fn eos_token(&self) -> usize { - self.word_to_index_and_count.get(EOS_TOKEN).unwrap().0 - } - - pub fn name(&self) -> &str { - &self.name - } - - pub fn get_index(&self, word: &str) -> Option { - self.word_to_index_and_count.get(word).map(|x| x.0) - } - - pub fn seq_to_string(&self, seq: &[usize]) -> String { - let mut res: Vec = vec![]; - for s in seq.iter() { - res.push(self.index_to_word.get(s).unwrap().to_string()); - } - res.join(" ") - } -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/translation/main.rs b/v2/dependency-patches/tch-0.24.0/examples/translation/main.rs deleted file mode 100644 index 504fe7061a..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/translation/main.rs +++ /dev/null @@ -1,224 +0,0 @@ -/* Translation with a Sequence to Sequence Model and Attention. - - This follows the line of the PyTorch tutorial: - https://pytorch.org/tutorials/intermediate/seq2seq_translation_tutorial.html - And trains a Sequence to Sequence (seq2seq) model using attention to - perform translation between French and English. - - The dataset can be downloaded from the following link: - https://download.pytorch.org/tutorial/data.zip - The eng-fra.txt file should be moved in the data directory. -*/ -use anyhow::Result; -use rand::prelude::*; -use tch::nn::{GRUState, Module, OptimizerConfig, RNN}; -use tch::{nn, Device, Kind, Tensor}; - -mod dataset; -use dataset::Dataset; -mod lang; -use lang::Lang; - -const MAX_LENGTH: usize = 10; -const LEARNING_RATE: f64 = 0.001; -const SAMPLES: usize = 100_000; -const HIDDEN_SIZE: usize = 256; - -struct Encoder { - embedding: nn::Embedding, - gru: nn::GRU, -} - -impl Encoder { - fn new(vs: nn::Path, in_dim: usize, hidden_dim: usize) -> Self { - let in_dim = in_dim as i64; - let hidden_dim = hidden_dim as i64; - let gru = nn::gru(&vs, hidden_dim, hidden_dim, Default::default()); - let embedding = nn::embedding(&vs, in_dim, hidden_dim, Default::default()); - Encoder { embedding, gru } - } - - fn forward(&self, xs: &Tensor, state: &GRUState) -> (Tensor, GRUState) { - let xs = self.embedding.forward(xs).view([1, -1]); - let state = self.gru.step(&xs, state); - (state.value().squeeze_dim(1), state) - } -} - -struct Decoder { - device: Device, - embedding: nn::Embedding, - gru: nn::GRU, - attn: nn::Linear, - attn_combine: nn::Linear, - linear: nn::Linear, -} - -impl Decoder { - fn new(vs: nn::Path, hidden_dim: usize, out_dim: usize) -> Self { - let hidden_dim = hidden_dim as i64; - let out_dim = out_dim as i64; - Decoder { - device: vs.device(), - embedding: nn::embedding(&vs, out_dim, hidden_dim, Default::default()), - gru: nn::gru(&vs, hidden_dim, hidden_dim, Default::default()), - attn: nn::linear(&vs, hidden_dim * 2, MAX_LENGTH as i64, Default::default()), - attn_combine: nn::linear(&vs, hidden_dim * 2, hidden_dim, Default::default()), - linear: nn::linear(&vs, hidden_dim, out_dim, Default::default()), - } - } - - fn forward( - &self, - xs: &Tensor, - state: &GRUState, - enc_outputs: &Tensor, - is_training: bool, - ) -> (Tensor, GRUState) { - let xs = self.embedding.forward(xs).dropout(0.1, is_training).view([1, -1]); - let attn_weights = - Tensor::cat(&[&xs, &state.value().squeeze_dim(1)], 1).apply(&self.attn).unsqueeze(0); - let (sz1, sz2, sz3) = enc_outputs.size3().unwrap(); - let enc_outputs = if sz2 == MAX_LENGTH as i64 { - enc_outputs.shallow_clone() - } else { - let shape = [sz1, MAX_LENGTH as i64 - sz2, sz3]; - let zeros = Tensor::zeros(shape, (Kind::Float, self.device)); - Tensor::cat(&[enc_outputs, &zeros], 1) - }; - let attn_applied = attn_weights.bmm(&enc_outputs).squeeze_dim(1); - let xs = Tensor::cat(&[&xs, &attn_applied], 1).apply(&self.attn_combine).relu(); - let state = self.gru.step(&xs, state); - (self.linear.forward(&state.value()).log_softmax(-1, Kind::Float).squeeze_dim(1), state) - } -} - -struct Model { - encoder: Encoder, - decoder: Decoder, - decoder_start: Tensor, - decoder_eos: usize, - device: Device, -} - -impl Model { - fn new(vs: nn::Path, ilang: &Lang, olang: &Lang, hidden_dim: usize) -> Self { - Model { - encoder: Encoder::new(&vs / "enc", ilang.len(), hidden_dim), - decoder: Decoder::new(&vs / "dec", hidden_dim, olang.len()), - decoder_start: Tensor::from_slice(&[olang.sos_token() as i64]).to_device(vs.device()), - decoder_eos: olang.eos_token(), - device: vs.device(), - } - } - - #[allow(clippy::assign_op_pattern)] - fn train_loss(&self, input_: &[usize], target: &[usize], rng: &mut ThreadRng) -> Tensor { - let mut state = self.encoder.gru.zero_state(1); - let mut enc_outputs = vec![]; - for &s in input_.iter() { - let s = Tensor::from_slice(&[s as i64]).to_device(self.device); - let (out, state_) = self.encoder.forward(&s, &state); - enc_outputs.push(out); - state = state_; - } - let enc_outputs = Tensor::stack(&enc_outputs, 1); - let use_teacher_forcing: bool = rng.gen(); - let mut loss = Tensor::from(0f32).to_device(self.device); - let mut prev = self.decoder_start.shallow_clone(); - for &s in target.iter() { - let (output, state_) = self.decoder.forward(&prev, &state, &enc_outputs, true); - state = state_; - let target_tensor = Tensor::from_slice(&[s as i64]).to_device(self.device); - loss = loss + output.nll_loss(&target_tensor); - let (_, output) = output.topk(1, -1, true, true); - if self.decoder_eos == i64::try_from(&output).unwrap() as usize { - break; - } - prev = if use_teacher_forcing { target_tensor } else { output }; - } - loss - } - - fn predict(&self, input_: &[usize]) -> Vec { - let mut state = self.encoder.gru.zero_state(1); - let mut enc_outputs = vec![]; - for &s in input_.iter() { - let s = Tensor::from_slice(&[s as i64]).to_device(self.device); - let (out, state_) = self.encoder.forward(&s, &state); - enc_outputs.push(out); - state = state_; - } - let enc_outputs = Tensor::stack(&enc_outputs, 1); - let mut prev = self.decoder_start.shallow_clone(); - let mut output_seq: Vec = vec![]; - for _i in 0..MAX_LENGTH { - let (output, state_) = self.decoder.forward(&prev, &state, &enc_outputs, true); - let (_, output) = output.topk(1, -1, true, true); - let output_ = i64::try_from(&output).unwrap() as usize; - output_seq.push(output_); - if self.decoder_eos == output_ { - break; - } - state = state_; - prev = output; - } - output_seq - } -} - -struct LossStats { - total_loss: f64, - samples: usize, -} - -impl LossStats { - fn new() -> LossStats { - LossStats { total_loss: 0., samples: 0 } - } - - fn update(&mut self, loss: f64) { - self.total_loss += loss; - self.samples += 1; - } - - fn avg_and_reset(&mut self) -> f64 { - let avg = self.total_loss / self.samples as f64; - self.total_loss = 0.; - self.samples = 0; - avg - } -} - -pub fn main() -> Result<()> { - let dataset = Dataset::new("eng", "fra", MAX_LENGTH)?.reverse(); - let ilang = dataset.input_lang(); - let olang = dataset.output_lang(); - let pairs = dataset.pairs(); - println!("Input: {} {} words.", ilang.name(), ilang.len()); - println!("Output: {} {} words.", olang.name(), olang.len()); - println!("Pairs: {}.", pairs.len()); - let mut rng = thread_rng(); - let device = Device::cuda_if_available(); - let vs = nn::VarStore::new(device); - let model = Model::new(vs.root(), ilang, olang, HIDDEN_SIZE); - let mut opt = nn::Adam::default().build(&vs, LEARNING_RATE)?; - let mut loss_stats = LossStats::new(); - for idx in 1..=SAMPLES { - let (input_, target) = pairs.choose(&mut rng).unwrap(); - let loss = model.train_loss(input_, target, &mut rng); - opt.backward_step(&loss); - loss_stats.update(f64::try_from(loss)? / target.len() as f64); - if idx % 1000 == 0 { - println!("{} {}", idx, loss_stats.avg_and_reset()); - for _pred_index in 1..5 { - let (input_, target) = pairs.choose(&mut rng).unwrap(); - let predict = model.predict(input_); - println!("in: {}", ilang.seq_to_string(input_)); - println!("tgt: {}", olang.seq_to_string(target)); - println!("out: {}", olang.seq_to_string(&predict)); - } - } - } - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/vae/main.rs b/v2/dependency-patches/tch-0.24.0/examples/vae/main.rs deleted file mode 100644 index 96b7562799..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/vae/main.rs +++ /dev/null @@ -1,95 +0,0 @@ -/* Variational Auto-Encoder on MNIST. - The implementation is based on: - https://github.com/pytorch/examples/blob/master/vae/main.py - - The 4 following dataset files can be downloaded from http://yann.lecun.com/exdb/mnist/ - These files should be extracted in the 'data' directory. - train-images-idx3-ubyte.gz - train-labels-idx1-ubyte.gz - t10k-images-idx3-ubyte.gz - t10k-labels-idx1-ubyte.gz -*/ - -use anyhow::Result; -use tch::{nn, nn::Module, nn::OptimizerConfig, Kind, Reduction, Tensor}; - -struct Vae { - fc1: nn::Linear, - fc21: nn::Linear, - fc22: nn::Linear, - fc3: nn::Linear, - fc4: nn::Linear, -} - -impl Vae { - fn new(vs: &nn::Path) -> Self { - Vae { - fc1: nn::linear(vs / "fc1", 784, 400, Default::default()), - fc21: nn::linear(vs / "fc21", 400, 20, Default::default()), - fc22: nn::linear(vs / "fc22", 400, 20, Default::default()), - fc3: nn::linear(vs / "fc3", 20, 400, Default::default()), - fc4: nn::linear(vs / "fc4", 400, 784, Default::default()), - } - } - - fn encode(&self, xs: &Tensor) -> (Tensor, Tensor) { - let h1 = xs.apply(&self.fc1).relu(); - (self.fc21.forward(&h1), self.fc22.forward(&h1)) - } - - fn decode(&self, zs: &Tensor) -> Tensor { - zs.apply(&self.fc3).relu().apply(&self.fc4).sigmoid() - } - - fn forward(&self, xs: &Tensor) -> (Tensor, Tensor, Tensor) { - let (mu, logvar) = self.encode(&xs.view([-1, 784])); - let std = (&logvar * 0.5).exp(); - let eps = std.randn_like(); - (self.decode(&(&mu + eps * std)), mu, logvar) - } -} - -// Reconstruction + KL divergence losses summed over all elements and batch dimension. -fn loss(recon_x: &Tensor, x: &Tensor, mu: &Tensor, logvar: &Tensor) -> Tensor { - let bce = recon_x.binary_cross_entropy::(&x.view([-1, 784]), None, Reduction::Sum); - // See Appendix B from VAE paper: - // Kingma and Welling. Auto-Encoding Variational Bayes. ICLR, 2014 - // https://arxiv.org/abs/1312.6114 - // 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2) - let kld = -0.5 * (1i64 + logvar - mu.pow_tensor_scalar(2) - logvar.exp()).sum(Kind::Float); - bce + kld -} - -// Generate a 2D matrix of images from a tensor with multiple images. -fn image_matrix(imgs: &Tensor, sz: i64) -> Result { - let imgs = (imgs * 256.).clamp(0., 255.).to_kind(Kind::Uint8); - let mut ys: Vec = vec![]; - for i in 0..sz { - ys.push(Tensor::cat(&(0..sz).map(|j| imgs.narrow(0, 4 * i + j, 1)).collect::>(), 2)) - } - Ok(Tensor::cat(&ys, 3).squeeze_dim(0)) -} - -pub fn main() -> Result<()> { - let device = tch::Device::cuda_if_available(); - let m = tch::vision::mnist::load_dir("data")?; - let vs = nn::VarStore::new(device); - let vae = Vae::new(&vs.root()); - let mut opt = nn::Adam::default().build(&vs, 1e-3)?; - for epoch in 1..21 { - let mut train_loss = 0f64; - let mut samples = 0f64; - for (bimages, _) in m.train_iter(128).shuffle().to_device(vs.device()) { - let (recon_batch, mu, logvar) = vae.forward(&bimages); - let loss = loss(&recon_batch, &bimages, &mu, &logvar); - opt.backward_step(&loss); - train_loss += f64::try_from(&loss)?; - samples += bimages.size()[0] as f64; - } - println!("Epoch: {}, loss: {}", epoch, train_loss / samples); - let s = Tensor::randn([64, 20], tch::kind::FLOAT_CPU).to(device); - let s = vae.decode(&s).to(tch::Device::Cpu).view([64, 1, 28, 28]); - tch::vision::image::save(&image_matrix(&s, 8)?, format!("s_{epoch}.png"))? - } - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/yolo/coco_classes.rs b/v2/dependency-patches/tch-0.24.0/examples/yolo/coco_classes.rs deleted file mode 100644 index 0075352492..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/yolo/coco_classes.rs +++ /dev/null @@ -1,82 +0,0 @@ -pub const NAMES: [&str; 80] = [ - "person", - "bicycle", - "car", - "motorbike", - "aeroplane", - "bus", - "train", - "truck", - "boat", - "traffic light", - "fire hydrant", - "stop sign", - "parking meter", - "bench", - "bird", - "cat", - "dog", - "horse", - "sheep", - "cow", - "elephant", - "bear", - "zebra", - "giraffe", - "backpack", - "umbrella", - "handbag", - "tie", - "suitcase", - "frisbee", - "skis", - "snowboard", - "sports ball", - "kite", - "baseball bat", - "baseball glove", - "skateboard", - "surfboard", - "tennis racket", - "bottle", - "wine glass", - "cup", - "fork", - "knife", - "spoon", - "bowl", - "banana", - "apple", - "sandwich", - "orange", - "broccoli", - "carrot", - "hot dog", - "pizza", - "donut", - "cake", - "chair", - "sofa", - "pottedplant", - "bed", - "diningtable", - "toilet", - "tvmonitor", - "laptop", - "mouse", - "remote", - "keyboard", - "cell phone", - "microwave", - "oven", - "toaster", - "sink", - "refrigerator", - "book", - "clock", - "vase", - "scissors", - "teddy bear", - "hair drier", - "toothbrush", -]; diff --git a/v2/dependency-patches/tch-0.24.0/examples/yolo/darknet.rs b/v2/dependency-patches/tch-0.24.0/examples/yolo/darknet.rs deleted file mode 100644 index 283b1cadf3..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/yolo/darknet.rs +++ /dev/null @@ -1,286 +0,0 @@ -use anyhow::{bail, ensure, Result}; -use std::collections::BTreeMap; -use std::fs::File; -use std::io::{BufRead, BufReader}; -use std::path::Path; -use tch::{ - nn, - nn::{FuncT, ModuleT}, - Tensor, -}; - -#[derive(Debug)] -struct Block { - block_type: String, - parameters: BTreeMap, -} - -impl Block { - fn get(&self, key: &str) -> Result<&str> { - match self.parameters.get(key) { - None => bail!("cannot find {} in {}", key, self.block_type), - Some(value) => Ok(value), - } - } -} - -#[derive(Debug)] -pub struct Darknet { - blocks: Vec, - parameters: BTreeMap, -} - -impl Darknet { - fn get(&self, key: &str) -> Result<&str> { - match self.parameters.get(key) { - None => bail!("cannot find {} in net parameters", key), - Some(value) => Ok(value), - } - } -} - -struct Accumulator { - block_type: Option, - parameters: BTreeMap, - net: Darknet, -} - -impl Accumulator { - fn new() -> Accumulator { - Accumulator { - block_type: None, - parameters: BTreeMap::new(), - net: Darknet { blocks: vec![], parameters: BTreeMap::new() }, - } - } - - fn finish_block(&mut self) { - match &self.block_type { - None => (), - Some(block_type) => { - if block_type == "net" { - self.net.parameters = self.parameters.clone(); - } else { - let block = Block { - block_type: block_type.to_string(), - parameters: self.parameters.clone(), - }; - self.net.blocks.push(block); - } - self.parameters.clear(); - } - } - self.block_type = None; - } -} - -pub fn parse_config>(path: T) -> Result { - let file = File::open(path.as_ref())?; - let mut acc = Accumulator::new(); - for line in BufReader::new(file).lines() { - let line = line?; - if line.is_empty() || line.starts_with('#') { - continue; - } - let line = line.trim(); - if line.starts_with('[') { - ensure!(line.ends_with(']'), "line does not end with ']' {}", line); - let line = &line[1..line.len() - 1]; - acc.finish_block(); - acc.block_type = Some(line.to_string()); - } else { - let key_value: Vec<&str> = line.splitn(2, '=').collect(); - ensure!(key_value.len() == 2, "missing equal {}", line); - let prev = acc - .parameters - .insert(key_value[0].trim().to_owned(), key_value[1].trim().to_owned()); - ensure!(prev.is_none(), "multiple value for key {}", line); - } - } - acc.finish_block(); - Ok(acc.net) -} - -enum Bl { - Layer(Box), - Route(Vec), - Shortcut(usize), - Yolo(i64, Vec<(i64, i64)>), -} - -fn conv(vs: nn::Path, index: usize, p: i64, b: &Block) -> Result<(i64, Bl)> { - let activation = b.get("activation")?; - let filters = b.get("filters")?.parse::()?; - let pad = b.get("pad")?.parse::()?; - let size = b.get("size")?.parse::()?; - let stride = b.get("stride")?.parse::()?; - let pad = if pad != 0 { (size - 1) / 2 } else { 0 }; - let (bn, bias) = match b.parameters.get("batch_normalize") { - Some(p) if p.parse::()? != 0 => { - let vs = &vs / format!("batch_norm_{index}"); - let bn = nn::batch_norm2d(vs, filters, Default::default()); - (Some(bn), false) - } - Some(_) | None => (None, true), - }; - let conv_cfg = nn::ConvConfig { stride, padding: pad, bias, ..Default::default() }; - let vs = &vs / format!("conv_{index}"); - let conv = nn::conv2d(vs, p, filters, size, conv_cfg); - let leaky = match activation { - "leaky" => true, - "linear" => false, - otherwise => bail!("unsupported activation {}", otherwise), - }; - let func = nn::func_t(move |xs, train| { - let xs = xs.apply(&conv); - let xs = match &bn { - Some(bn) => xs.apply_t(bn, train), - None => xs, - }; - if leaky { - xs.maximum(&(&xs * 0.1)) - } else { - xs - } - }); - Ok((filters, Bl::Layer(Box::new(func)))) -} - -fn upsample(prev_channels: i64) -> Result<(i64, Bl)> { - let layer = nn::func_t(|xs, _is_training| { - let (_n, _c, h, w) = xs.size4().unwrap(); - xs.upsample_nearest2d([2 * h, 2 * w], 2.0, 2.0) - }); - Ok((prev_channels, Bl::Layer(Box::new(layer)))) -} - -fn int_list_of_string(s: &str) -> Result> { - let res: Result, _> = s.split(',').map(|xs| xs.trim().parse::()).collect(); - Ok(res?) -} - -fn usize_of_index(index: usize, i: i64) -> usize { - if i >= 0 { - i as usize - } else { - (index as i64 + i) as usize - } -} - -fn route(index: usize, p: &[(i64, Bl)], block: &Block) -> Result<(i64, Bl)> { - let layers = int_list_of_string(block.get("layers")?)?; - let layers: Vec = layers.into_iter().map(|l| usize_of_index(index, l)).collect(); - let channels = layers.iter().map(|&l| p[l].0).sum(); - Ok((channels, Bl::Route(layers))) -} - -fn shortcut(index: usize, p: i64, block: &Block) -> Result<(i64, Bl)> { - let from = block.get("from")?.parse::()?; - Ok((p, Bl::Shortcut(usize_of_index(index, from)))) -} - -fn yolo(p: i64, block: &Block) -> Result<(i64, Bl)> { - let classes = block.get("classes")?.parse::()?; - let flat = int_list_of_string(block.get("anchors")?)?; - ensure!(flat.len() % 2 == 0, "even number of anchors"); - let anchors: Vec<_> = (0..(flat.len() / 2)).map(|i| (flat[2 * i], flat[2 * i + 1])).collect(); - let mask = int_list_of_string(block.get("mask")?)?; - let anchors = mask.into_iter().map(|i| anchors[i as usize]).collect(); - Ok((p, Bl::Yolo(classes, anchors))) -} - -// Apply f to a slice of tensor xs and replace xs values with f output. -fn slice_apply_and_set(xs: &mut Tensor, start: i64, len: i64, f: F) -where - F: FnOnce(&Tensor) -> Tensor, -{ - let mut slice = xs.narrow(2, start, len); - let src = f(&slice); - slice.copy_(&src) -} - -fn detect(xs: &Tensor, image_height: i64, classes: i64, anchors: &[(i64, i64)]) -> Tensor { - let (bsize, _channels, height, _width) = xs.size4().unwrap(); - let stride = image_height / height; - let grid_size = image_height / stride; - let bbox_attrs = 5 + classes; - let nanchors = anchors.len() as i64; - let mut xs = xs - .view((bsize, bbox_attrs * nanchors, grid_size * grid_size)) - .transpose(1, 2) - .contiguous() - .view((bsize, grid_size * grid_size * nanchors, bbox_attrs)); - let grid = Tensor::arange(grid_size, tch::kind::FLOAT_CPU); - let a = grid.repeat([grid_size, 1]); - let b = a.tr().contiguous(); - let x_offset = a.view((-1, 1)); - let y_offset = b.view((-1, 1)); - let xy_offset = - Tensor::cat(&[x_offset, y_offset], 1).repeat([1, nanchors]).view((-1, 2)).unsqueeze(0); - let anchors: Vec = anchors - .iter() - .flat_map(|&(x, y)| vec![x as f32 / stride as f32, y as f32 / stride as f32].into_iter()) - .collect(); - let anchors = - Tensor::from_slice(&anchors).view((-1, 2)).repeat([grid_size * grid_size, 1]).unsqueeze(0); - slice_apply_and_set(&mut xs, 0, 2, |xs| xs.sigmoid() + xy_offset); - slice_apply_and_set(&mut xs, 4, 1 + classes, Tensor::sigmoid); - slice_apply_and_set(&mut xs, 2, 2, |xs| xs.exp() * anchors); - slice_apply_and_set(&mut xs, 0, 4, |xs| xs * stride); - xs -} - -impl Darknet { - pub fn height(&self) -> Result { - let image_height = self.get("height")?.parse::()?; - Ok(image_height) - } - - pub fn width(&self) -> Result { - let image_width = self.get("width")?.parse::()?; - Ok(image_width) - } - - pub fn build_model(&self, vs: &nn::Path) -> Result> { - let mut blocks: Vec<(i64, Bl)> = vec![]; - let mut prev_channels: i64 = 3; - for (index, block) in self.blocks.iter().enumerate() { - let channels_and_bl = match block.block_type.as_str() { - "convolutional" => conv(vs / index, index, prev_channels, block)?, - "upsample" => upsample(prev_channels)?, - "shortcut" => shortcut(index, prev_channels, block)?, - "route" => route(index, &blocks, block)?, - "yolo" => yolo(prev_channels, block)?, - otherwise => bail!("unsupported block type {}", otherwise), - }; - prev_channels = channels_and_bl.0; - blocks.push(channels_and_bl); - } - let image_height = self.height()?; - let func = nn::func_t(move |xs, train| { - let mut prev_ys: Vec = vec![]; - let mut detections: Vec = vec![]; - for (_, b) in blocks.iter() { - let ys = match b { - Bl::Layer(l) => { - let xs = prev_ys.last().unwrap_or(xs); - l.forward_t(xs, train) - } - Bl::Route(layers) => { - let layers: Vec<_> = layers.iter().map(|&i| &prev_ys[i]).collect(); - Tensor::cat(&layers, 1) - } - Bl::Shortcut(from) => prev_ys.last().unwrap() + prev_ys.get(*from).unwrap(), - Bl::Yolo(classes, anchors) => { - let xs = prev_ys.last().unwrap_or(xs); - detections.push(detect(xs, image_height, *classes, anchors)); - Tensor::default() - } - }; - prev_ys.push(ys); - } - Tensor::cat(&detections, 1) - }); - Ok(func) - } -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/yolo/main.rs b/v2/dependency-patches/tch-0.24.0/examples/yolo/main.rs deleted file mode 100644 index a84281f19d..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/yolo/main.rs +++ /dev/null @@ -1,153 +0,0 @@ -// The pre-trained weights can be downloaded here: -// https://github.com/LaurentMazare/ocaml-torch/releases/download/v0.1-unstable/yolo-v3.ot -// -// These weights have been generated by using the Python version at: -// https://github.com/ayooshkathuria/YOLO_v3_tutorial_from_scratch -// The weights are exported in npz format by adding the following code in detect.py after the -// model.load_weights(...) call. -// -// ```python -// def remove_prefix(text, prefix): -// return text[text.startswith(prefix) and len(prefix):] -// nps = {} -// for k, v in model.state_dict().items(): -// k = remove_prefix(k, 'module_list.') -// nps[k] = v.detach().numpy() -// np.savez('yolo-v3.ot', **nps) -// ``` -// -// Then tch-rs tensor-tools example can be used to convert the .npz weights -// to the requested .ot file. -// cargo run --example tensor-tools cp yolo-v3.ot.npz yolo-v3.ot - -mod coco_classes; -mod darknet; - -use anyhow::{ensure, Result}; -use tch::nn::ModuleT; -use tch::vision::image; -use tch::Tensor; -const CONFIG_NAME: &str = "examples/yolo/yolo-v3.cfg"; -const CONFIDENCE_THRESHOLD: f64 = 0.5; -const NMS_THRESHOLD: f64 = 0.4; - -#[derive(Debug, Clone, Copy)] -struct Bbox { - xmin: f64, - ymin: f64, - xmax: f64, - ymax: f64, - confidence: f64, -} - -// Intersection over union of two bounding boxes. -fn iou(b1: &Bbox, b2: &Bbox) -> f64 { - let b1_area = (b1.xmax - b1.xmin + 1.) * (b1.ymax - b1.ymin + 1.); - let b2_area = (b2.xmax - b2.xmin + 1.) * (b2.ymax - b2.ymin + 1.); - let i_xmin = b1.xmin.max(b2.xmin); - let i_xmax = b1.xmax.min(b2.xmax); - let i_ymin = b1.ymin.max(b2.ymin); - let i_ymax = b1.ymax.min(b2.ymax); - let i_area = (i_xmax - i_xmin + 1.).max(0.) * (i_ymax - i_ymin + 1.).max(0.); - i_area / (b1_area + b2_area - i_area) -} - -// Assumes x1 <= x2 and y1 <= y2 -pub fn draw_rect(t: &mut Tensor, x1: i64, x2: i64, y1: i64, y2: i64) { - let color = Tensor::from_slice(&[0., 0., 1.]).view([3, 1, 1]); - t.narrow(2, x1, x2 - x1).narrow(1, y1, y2 - y1).copy_(&color) -} - -pub fn report(pred: &Tensor, img: &Tensor, w: i64, h: i64) -> Result { - let (npreds, pred_size) = pred.size2()?; - let nclasses = (pred_size - 5) as usize; - // The bounding boxes grouped by (maximum) class index. - let mut bboxes: Vec> = (0..nclasses).map(|_| vec![]).collect(); - // Extract the bounding boxes for which confidence is above the threshold. - for index in 0..npreds { - let pred = Vec::::try_from(pred.get(index))?; - let confidence = pred[4]; - if confidence > CONFIDENCE_THRESHOLD { - let mut class_index = 0; - for i in 0..nclasses { - if pred[5 + i] > pred[5 + class_index] { - class_index = i - } - } - if pred[class_index + 5] > 0. { - let bbox = Bbox { - xmin: pred[0] - pred[2] / 2., - ymin: pred[1] - pred[3] / 2., - xmax: pred[0] + pred[2] / 2., - ymax: pred[1] + pred[3] / 2., - confidence, - }; - bboxes[class_index].push(bbox) - } - } - } - // Perform non-maximum suppression. - for bboxes_for_class in bboxes.iter_mut() { - bboxes_for_class.sort_by(|b1, b2| b2.confidence.partial_cmp(&b1.confidence).unwrap()); - let mut current_index = 0; - for index in 0..bboxes_for_class.len() { - let mut drop = false; - for prev_index in 0..current_index { - let iou = iou(&bboxes_for_class[prev_index], &bboxes_for_class[index]); - if iou > NMS_THRESHOLD { - drop = true; - break; - } - } - if !drop { - bboxes_for_class.swap(current_index, index); - current_index += 1; - } - } - bboxes_for_class.truncate(current_index); - } - // Annotate the original image and print boxes information. - let (_, initial_h, initial_w) = img.size3()?; - let mut img = img.to_kind(tch::Kind::Float) / 255.; - let w_ratio = initial_w as f64 / w as f64; - let h_ratio = initial_h as f64 / h as f64; - for (class_index, bboxes_for_class) in bboxes.iter().enumerate() { - for b in bboxes_for_class.iter() { - println!("{}: {:?}", coco_classes::NAMES[class_index], b); - let xmin = ((b.xmin * w_ratio) as i64).clamp(0, initial_w - 1); - let ymin = ((b.ymin * h_ratio) as i64).clamp(0, initial_h - 1); - let xmax = ((b.xmax * w_ratio) as i64).clamp(0, initial_w - 1); - let ymax = ((b.ymax * h_ratio) as i64).clamp(0, initial_h - 1); - draw_rect(&mut img, xmin, xmax, ymin, ymax.min(ymin + 2)); - draw_rect(&mut img, xmin, xmax, ymin.max(ymax - 2), ymax); - draw_rect(&mut img, xmin, xmax.min(xmin + 2), ymin, ymax); - draw_rect(&mut img, xmin.max(xmax - 2), xmax, ymin, ymax); - } - } - Ok((img * 255.).to_kind(tch::Kind::Uint8)) -} - -pub fn main() -> Result<()> { - let args: Vec<_> = std::env::args().collect(); - ensure!(args.len() >= 3, "usage: main yolo-v3.ot img.jpg ..."); - - // Create the model and load the weights from the file. - let mut vs = tch::nn::VarStore::new(tch::Device::Cpu); - let darknet = darknet::parse_config(CONFIG_NAME)?; - let model = darknet.build_model(&vs.root())?; - vs.load(&args[1])?; - - for (index, image) in args.iter().skip(2).enumerate() { - // Load the image file and resize it. - let original_image = image::load(image)?; - let net_width = darknet.width()?; - let net_height = darknet.height()?; - let image = image::resize(&original_image, net_width, net_height)?; - let image = image.unsqueeze(0).to_kind(tch::Kind::Float) / 255.; - let predictions = model.forward_t(&image, false).squeeze(); - let image = report(&predictions, &original_image, net_width, net_height)?; - image::save(&image, format!("output-{index:05}.jpg"))?; - println!("Converted {index}"); - } - Ok(()) -} diff --git a/v2/dependency-patches/tch-0.24.0/examples/yolo/yolo-v3.cfg b/v2/dependency-patches/tch-0.24.0/examples/yolo/yolo-v3.cfg deleted file mode 100644 index 6772dddbf8..0000000000 --- a/v2/dependency-patches/tch-0.24.0/examples/yolo/yolo-v3.cfg +++ /dev/null @@ -1,790 +0,0 @@ -[net] -# Testing -batch=1 -subdivisions=1 -# Training -# batch=64 -# subdivisions=16 -width= 416 - -height = 416 -channels=3 -momentum=0.9 -decay=0.0005 -angle=0 -saturation = 1.5 -exposure = 1.5 -hue=.1 - -learning_rate=0.001 -burn_in=1000 -max_batches = 500200 -policy=steps -steps=400000,450000 -scales=.1,.1 - -[convolutional] -batch_normalize=1 -filters=32 -size=3 -stride=1 -pad=1 -activation=leaky - -# Downsample - -[convolutional] -batch_normalize=1 -filters=64 -size=3 -stride=2 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=32 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=64 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - -# Downsample - -[convolutional] -batch_normalize=1 -filters=128 -size=3 -stride=2 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=64 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=128 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - -[convolutional] -batch_normalize=1 -filters=64 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=128 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - -# Downsample - -[convolutional] -batch_normalize=1 -filters=256 -size=3 -stride=2 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=128 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=256 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - -[convolutional] -batch_normalize=1 -filters=128 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=256 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - -[convolutional] -batch_normalize=1 -filters=128 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=256 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - -[convolutional] -batch_normalize=1 -filters=128 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=256 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - - -[convolutional] -batch_normalize=1 -filters=128 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=256 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - -[convolutional] -batch_normalize=1 -filters=128 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=256 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - -[convolutional] -batch_normalize=1 -filters=128 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=256 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - -[convolutional] -batch_normalize=1 -filters=128 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=256 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - -# Downsample - -[convolutional] -batch_normalize=1 -filters=512 -size=3 -stride=2 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=256 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=512 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - - -[convolutional] -batch_normalize=1 -filters=256 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=512 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - - -[convolutional] -batch_normalize=1 -filters=256 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=512 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - - -[convolutional] -batch_normalize=1 -filters=256 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=512 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - -[convolutional] -batch_normalize=1 -filters=256 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=512 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - - -[convolutional] -batch_normalize=1 -filters=256 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=512 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - - -[convolutional] -batch_normalize=1 -filters=256 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=512 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - -[convolutional] -batch_normalize=1 -filters=256 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=512 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - -# Downsample - -[convolutional] -batch_normalize=1 -filters=1024 -size=3 -stride=2 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=512 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=1024 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - -[convolutional] -batch_normalize=1 -filters=512 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=1024 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - -[convolutional] -batch_normalize=1 -filters=512 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=1024 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - -[convolutional] -batch_normalize=1 -filters=512 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=1024 -size=3 -stride=1 -pad=1 -activation=leaky - -[shortcut] -from=-3 -activation=linear - -###################### - -[convolutional] -batch_normalize=1 -filters=512 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -size=3 -stride=1 -pad=1 -filters=1024 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=512 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -size=3 -stride=1 -pad=1 -filters=1024 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=512 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -size=3 -stride=1 -pad=1 -filters=1024 -activation=leaky - -[convolutional] -size=1 -stride=1 -pad=1 -filters=255 -activation=linear - - -[yolo] -mask = 6,7,8 -anchors = 10,13, 16,30, 33,23, 30,61, 62,45, 59,119, 116,90, 156,198, 373,326 -classes=80 -num=9 -jitter=.3 -ignore_thresh = .5 -truth_thresh = 1 -random=1 - - -[route] -layers = -4 - -[convolutional] -batch_normalize=1 -filters=256 -size=1 -stride=1 -pad=1 -activation=leaky - -[upsample] -stride=2 - -[route] -layers = -1, 61 - - - -[convolutional] -batch_normalize=1 -filters=256 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -size=3 -stride=1 -pad=1 -filters=512 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=256 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -size=3 -stride=1 -pad=1 -filters=512 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=256 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -size=3 -stride=1 -pad=1 -filters=512 -activation=leaky - -[convolutional] -size=1 -stride=1 -pad=1 -filters=255 -activation=linear - - -[yolo] -mask = 3,4,5 -anchors = 10,13, 16,30, 33,23, 30,61, 62,45, 59,119, 116,90, 156,198, 373,326 -classes=80 -num=9 -jitter=.3 -ignore_thresh = .5 -truth_thresh = 1 -random=1 - - - -[route] -layers = -4 - -[convolutional] -batch_normalize=1 -filters=128 -size=1 -stride=1 -pad=1 -activation=leaky - -[upsample] -stride=2 - -[route] -layers = -1, 36 - - - -[convolutional] -batch_normalize=1 -filters=128 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -size=3 -stride=1 -pad=1 -filters=256 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=128 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -size=3 -stride=1 -pad=1 -filters=256 -activation=leaky - -[convolutional] -batch_normalize=1 -filters=128 -size=1 -stride=1 -pad=1 -activation=leaky - -[convolutional] -batch_normalize=1 -size=3 -stride=1 -pad=1 -filters=256 -activation=leaky - -[convolutional] -size=1 -stride=1 -pad=1 -filters=255 -activation=linear - - -[yolo] -mask = 0,1,2 -anchors = 10,13, 16,30, 33,23, 30,61, 62,45, 59,119, 116,90, 156,198, 373,326 -classes=80 -num=9 -jitter=.3 -ignore_thresh = .5 -truth_thresh = 1 -random=1 - diff --git a/v2/dependency-patches/tch-0.24.0/gen/.ocamlformat b/v2/dependency-patches/tch-0.24.0/gen/.ocamlformat deleted file mode 100644 index 6d3685839c..0000000000 --- a/v2/dependency-patches/tch-0.24.0/gen/.ocamlformat +++ /dev/null @@ -1,2 +0,0 @@ -profile = janestreet -let-binding-spacing = compact diff --git a/v2/dependency-patches/tch-0.24.0/gen/dune b/v2/dependency-patches/tch-0.24.0/gen/dune deleted file mode 100644 index b5b2c527be..0000000000 --- a/v2/dependency-patches/tch-0.24.0/gen/dune +++ /dev/null @@ -1,3 +0,0 @@ -(executables - (names gen) - (libraries base stdio yaml)) diff --git a/v2/dependency-patches/tch-0.24.0/gen/gen.ml b/v2/dependency-patches/tch-0.24.0/gen/gen.ml deleted file mode 100644 index 55ec8a93fc..0000000000 --- a/v2/dependency-patches/tch-0.24.0/gen/gen.ml +++ /dev/null @@ -1,892 +0,0 @@ -(* Automatically generate the C++ -> C -> rust bindings. - This takes as input the Descriptions.yaml file that gets generated when - building PyTorch from source. - - Run with: dune exec gen/gen.exe -*) -open Base -open Stdio - -let excluded_functions = - Set.of_list - (module String) - [ "multi_margin_loss" - ; "multi_margin_loss_out" - ; "log_softmax_backward_data" - ; "softmax_backward_data" - ; "clone" - ; "copy" - ; "copy_out" - ; "copy_" - ; "conv_transpose2d_backward_out" - ; "conv_transpose3d_backward_out" - ; "slow_conv_transpose2d_backward_out" - ; "slow_conv_transpose3d_backward_out" - ; "slow_conv3d_backward_out" - ; "normal" - ; "_cufft_set_plan_cache_max_size" - ; "_cufft_clear_plan_cache" - ; "backward" - ; "_amp_non_finite_check_and_unscale_" - ; "_cummin_helper" - ; "_cummax_helper" - ; "retain_grad" - ; "_validate_sparse_coo_tensor_args" - ; "_sparse_semi_structured_addmm" - ; "_backward" - ; "size" - ; "stride" - ; "_assert_async" - ; "gradient" - ; "linalg_vector_norm" - ; "linalg_vector_norm_out" - ; "linalg_matrix_norm" - ; "linalg_matrix_norm_out" - ; "linalg__powsum" - (* Deactivate normal_out, bernoulli_out as these result in some - ambiguous function calls. *) - ; "normal_out" - ; "bernoulli_out" - ; "nested_tensor" - ; "arange_out" - ] - -let no_tensor_options = - Set.of_list - (module String) - [ "zeros_like" - ; "empty_like" - ; "full_like" - ; "ones_like" - ; "rand_like" - ; "randint_like" - ; "randn_like" - ] - -let prefixed_functions = - Set.of_list - (module String) - [ "add" - ; "add_" - ; "div" - ; "div_" - ; "mul" - ; "mul_" - ; "sub" - ; "sub_" - ; "nll_loss" - ; "to_mkldnn" - ] - -(* By default, scalar argument that have a default value are not available on - the Rust side, this is to preserve the Rust api simplicity assuming that - these scalars arguments are not often overriden. - Adding function name [foo] in [with_optional_scalar_args] results in having - explicit scalar arguments even if a default is present. *) -let with_optional_scalar_args = Set.of_list (module String) [ "arange"; "baddbmm" ] - -let excluded_prefixes = - [ "_thnn_" - ; "_th_" - ; "thnn_" - ; "th_" - ; "_foreach" - ; "_amp_foreach" - ; "_nested_tensor" - ; "_fused_adam" - ; "_fused_adagrad" - ; "sym_" - ; "_fused_sgd" - ; "fbgemm_" - ] - -let excluded_suffixes = [ "_forward"; "_forward_out" ] -let yaml_error yaml ~msg = Printf.failwithf "%s, %s" msg (Yaml.to_string_exn yaml) () - -let extract_bool = function - | `Bool b -> b - | `String "true" -> true - | `String "false" -> false - | yaml -> yaml_error yaml ~msg:"expected bool" - -let extract_list = function - | `A l -> l - | yaml -> yaml_error yaml ~msg:"expected list" - -let extract_map = function - | `O map -> Map.of_alist_exn (module String) map - | yaml -> yaml_error yaml ~msg:"expected map" - -let extract_string = function - | `String s -> s - (* The yaml spec for torch uses n which is converted to a bool. *) - | `Bool b -> if b then "y" else "n" - | `Float f -> Float.to_string f - | yaml -> yaml_error yaml ~msg:"expected string" - -module Func = struct - type arg_type = - | Bool - | Int64 - | Int64Option - | Double - | DoubleOption - | Tensor - | TensorOption (* Tensor.t option *) - | IntList - | IntListOption - | DoubleList - | TensorOptList - | TensorList - | TensorOptions (* Tensor kind and device *) - | Scalar - | ScalarType - | ScalarTypeOption - | Device - | String - | Layout - | LayoutOption - - type arg = - { arg_name : string - ; arg_type : arg_type - ; default_value : string option - } - - type t = - { name : string - ; operator_name : string - ; overload_name : string - ; args : arg list - ; returns : [ `fixed of int | `dynamic | `bool | `int64_t | `double | `nothing ] - ; (* number of tensors that are returned *) - kind : [ `function_ | `method_ ] - } - - let arg_type_of_string str ~is_nullable = - match String.lowercase str with - | "bool" -> Some Bool - | "int64_t" -> Some (if is_nullable then Int64Option else Int64) - | "double" -> Some (if is_nullable then DoubleOption else Double) - | "at::tensor" -> Some (if is_nullable then TensorOption else Tensor) - | "at::tensoroptions" -> Some TensorOptions - | "at::intarrayref" -> Some (if is_nullable then IntListOption else IntList) - | "at::arrayref" -> Some DoubleList - | "const c10::list<::std::optional> &" - | "const c10::list> &" -> Some TensorOptList - | "const at::itensorlistref &" | "at::tensorlist" -> Some TensorList - | "at::device" -> Some Device - | "const at::scalar &" | "at::scalar" -> Some Scalar - | "at::scalartype" -> if is_nullable then Some ScalarTypeOption else Some ScalarType - | "c10::string_view" -> Some String - | "at::layout" -> Some (if is_nullable then LayoutOption else Layout) - | _ -> None - - let c_typed_args_list t = - List.map t.args ~f:(fun { arg_name; arg_type; _ } -> - match arg_type with - | IntList | IntListOption -> - Printf.sprintf "int64_t *%s_data, int %s_len" arg_name arg_name - | DoubleList -> Printf.sprintf "double *%s_data, int %s_len" arg_name arg_name - | TensorOptList | TensorList -> - Printf.sprintf "tensor *%s_data, int %s_len" arg_name arg_name - | TensorOptions -> Printf.sprintf "int %s_kind, int %s_device" arg_name arg_name - | String -> Printf.sprintf "char* %s_ptr, int %s_len" arg_name arg_name - | Int64Option -> Printf.sprintf "int64_t %s_v, uint8_t %s_null" arg_name arg_name - | DoubleOption -> Printf.sprintf "double %s_v, uint8_t %s_null" arg_name arg_name - | otherwise -> - let simple_type_cstring = - match otherwise with - | Bool -> "int" - | Int64 -> "int64_t" - | Double -> "double" - | Tensor -> "tensor" - | TensorOption -> "tensor" - | ScalarType -> "int" - | ScalarTypeOption -> "int" - | Device -> "int" - | Scalar -> "scalar" - | Layout | LayoutOption -> "int8_t" - | Int64Option - | DoubleOption - | String - | IntList - | IntListOption - | DoubleList - | TensorOptList - | TensorList - | TensorOptions -> assert false - in - Printf.sprintf "%s %s" simple_type_cstring arg_name) - |> String.concat ~sep:", " - - let c_args_list args = - List.map args ~f:(fun { arg_name; arg_type; _ } -> - match arg_type with - | Scalar | Tensor -> "*" ^ arg_name - | Layout -> Printf.sprintf "static_cast(%s)" arg_name - | LayoutOption -> - Printf.sprintf - "(%s == -1 ? c10::nullopt : \ - c10::optional(static_cast(%s)))" - arg_name - arg_name - | TensorOption -> Printf.sprintf "(%s ? ::std::optional(*%s) : ::std::nullopt)" arg_name arg_name - | Bool -> "(bool)" ^ arg_name - | IntList -> Printf.sprintf "torch::IntArrayRef(%s_data, %s_len)" arg_name arg_name - | IntListOption -> - Printf.sprintf - "%s_data == nullptr ? c10::nullopt : \ - c10::optional(torch::IntArrayRef(%s_data, %s_len))" - arg_name - arg_name - arg_name - | DoubleList -> - Printf.sprintf "at::ArrayRef(%s_data, %s_len)" arg_name arg_name - | String -> Printf.sprintf "std::string(%s_ptr, %s_len)" arg_name arg_name - | TensorOptList -> - Printf.sprintf "of_carray_tensor_opt(%s_data, %s_len)" arg_name arg_name - | TensorList -> Printf.sprintf "of_carray_tensor(%s_data, %s_len)" arg_name arg_name - | TensorOptions -> - Printf.sprintf - "at::device(device_of_int(%s_device)).dtype(at::ScalarType(%s_kind))" - arg_name - arg_name - | Int64Option -> - Printf.sprintf - "%s_null ? c10::nullopt : c10::optional(%s_v)" - arg_name - arg_name - | DoubleOption -> - Printf.sprintf - "%s_null ? c10::nullopt : c10::optional(%s_v)" - arg_name - arg_name - | ScalarType -> Printf.sprintf "at::ScalarType(%s)" arg_name - | ScalarTypeOption -> - Printf.sprintf - "%s < 0 ? c10::nullopt : c10::optional(at::ScalarType(%s))" - arg_name - arg_name - | Device -> Printf.sprintf "device_of_int(%s)" arg_name - | _ -> arg_name) - |> String.concat ~sep:", " - - let c_call t = - match t.kind with - | `function_ -> Printf.sprintf "torch::%s(%s)" t.name (c_args_list t.args) - | `method_ -> - (match t.args with - | head :: tail -> - Printf.sprintf "%s->%s(%s)" head.arg_name t.name (c_args_list tail) - | [] -> - Printf.failwithf "Method calls should have at least one argument %s" t.name ()) - - let replace_map = - Map.of_alist_exn - (module String) - [ "t", "tr" - ; "where", "where_" - ; "view", "view_" - ; "unsafe", "unsafe_" - ; "to_device", "to_device_" - ] - - let rust_name name = - let name = - Map.find replace_map name - |> Option.value ~default:name - |> String.lowercase - |> String.substr_replace_all ~pattern:"__" ~with_:"_" - in - if String.is_prefix name ~prefix:"_" then "internal" ^ name else name - - let operator_name t = - match String.lowercase t.operator_name with - | "scatter_reduce" -> - (* scatter_reduce is both an operator name and also obtained from the - scatter operator when using the reduce overload. *) - "_scatter_reduce" - | "scatter_reduce_" -> "_scatter_reduce_" - | other -> other - - let c_rust_args_list t = - List.map t.args ~f:(fun arg -> - let an = arg.arg_name in - let single_param = Printf.sprintf "%s_: %s" an in - match arg.arg_type with - | Bool -> single_param "c_int" - | Layout | LayoutOption -> single_param "i8" - | Int64 -> single_param "i64" - | Double -> single_param "f64" - | Tensor -> single_param "*mut C_tensor" - | TensorOption -> single_param "*mut C_tensor" - | Scalar -> single_param "*mut C_scalar" - | ScalarType | ScalarTypeOption -> single_param "c_int" - | Device -> single_param "c_int" - | String -> Printf.sprintf "%s_ptr: *const u8, %s_len: c_int" an an - | IntList | IntListOption -> - Printf.sprintf "%s_data: *const i64, %s_len: c_int" an an - | DoubleList -> Printf.sprintf "%s_data: *const f64, %s_len: c_int" an an - | TensorOptList -> - Printf.sprintf "%s_data: *const *mut C_tensor, %s_len: c_int" an an - | TensorList -> Printf.sprintf "%s_data: *const *mut C_tensor, %s_len: c_int" an an - | Int64Option -> Printf.sprintf "%s_v: i64, %s_null: i8" an an - | DoubleOption -> Printf.sprintf "%s_v: f64, %s_null: i8" an an - | TensorOptions -> Printf.sprintf "%s_kind: c_int, %s_device: c_int" an an) - |> String.concat ~sep:", " - - let self_name = "self" - let input_name = "input" - - let self_tensor arg = - match arg.arg_type with - | Tensor -> String.( = ) arg.arg_name self_name - | _ -> false - - let input_tensor arg = - match arg.arg_type with - | Tensor -> String.( = ) arg.arg_name input_name - | _ -> false - - let type_parameters t = - let needs_scalar_parameter = - List.exists t.args ~f:(fun arg -> - match arg.arg_type with - | Scalar -> true - | _ -> false) - in - let needs_type_parameter = - List.exists t.args ~f:(fun arg -> - match arg.arg_type with - | TensorOptList | TensorList | TensorOption -> true - | _ -> false) - in - let type_parameter = if needs_type_parameter then [ "T: Borrow" ] else [] in - let scalar_parameter = if needs_scalar_parameter then [ "S: Into" ] else [] in - let parameters = type_parameter @ scalar_parameter in - match parameters with - | [] -> "" - | p -> "<" ^ String.concat p ~sep:", " ^ ">" - - let rust_args_list t = - match List.partition_tf t.args ~f:self_tensor with - | [ self ], args_list -> Some self, args_list - | _, _ -> - (match List.partition_tf t.args ~f:input_tensor with - | [ self ], args_list -> Some self, args_list - | _, _ -> None, t.args) - - let rust_typed_args_list t = - let to_string args = - List.map args ~f:(fun arg -> - let rust_arg_type = - match arg.arg_type with - | Bool -> "bool" - | Layout -> "Layout" - | LayoutOption -> "Option" - | Int64 -> - if String.( = ) arg.arg_name "reduction" then "crate::Reduction" else "i64" - | Double -> "f64" - | Tensor -> "&Tensor" - | TensorOption -> "Option" - | IntList -> "impl IntList" - | IntListOption -> "impl IntListOption" - | DoubleList -> "impl DoubleList" - | TensorOptList -> "&[Option]" - | TensorList -> "&[T]" - | String -> "&str" - | TensorOptions -> "(Kind, Device)" - | Int64Option -> "impl Into>" - | DoubleOption -> "impl Into>" - | Scalar -> "S" - | ScalarType -> "Kind" - | ScalarTypeOption -> "impl Into>" - | Device -> "Device" - in - Printf.sprintf "%s: %s" (rust_name arg.arg_name) rust_arg_type) - |> String.concat ~sep:", " - in - let self_arg = - if String.is_suffix t.name ~suffix:"_" || String.( = ) t.name "set_data" - then "&mut self" - else "&self" - in - match List.partition_tf t.args ~f:self_tensor with - | [ self ], args_list -> - Some self.arg_name, Printf.sprintf "%s, %s" self_arg (to_string args_list) - | _, _ -> - (match List.partition_tf t.args ~f:input_tensor with - | [ self ], args_list -> - Some self.arg_name, Printf.sprintf "%s, %s" self_arg (to_string args_list) - | _, _ -> None, to_string t.args) - - let rust_return_type t ~fallible = - let returns = - match t.returns with - | `nothing -> None - | `fixed 1 -> Some "Tensor" - | `fixed v -> - List.init v ~f:(fun _ -> "Tensor") - |> String.concat ~sep:", " - |> Printf.sprintf "(%s)" - |> Option.some - | `dynamic -> Some "Vec" - | `bool -> Some "bool" - | `int64_t -> Some "i64" - | `double -> Some "f64" - in - match returns with - | None -> if fallible then Printf.sprintf " -> Result<(), TchError>" else "" - | Some returns -> - if fallible - then Printf.sprintf " -> Result<%s, TchError>" returns - else Printf.sprintf " -> %s" returns - - let rust_binding_args t ~self = - List.map t.args ~f:(fun arg -> - let name = - if Option.value_map self ~default:false ~f:(String.( = ) arg.arg_name) - then "self" - else rust_name arg.arg_name - in - match arg.arg_type with - | Tensor -> Printf.sprintf "%s.c_tensor" name - | Scalar -> Printf.sprintf "%s.into().c_scalar" name - | Bool -> Printf.sprintf "if %s { 1 } else { 0 }" name - | ScalarType -> Printf.sprintf "%s.c_int()" name - | ScalarTypeOption -> Printf.sprintf "%s.into().map_or(-1, |s| s.c_int())" name - | Device -> Printf.sprintf "%s.c_int()" name - | TensorOptions -> Printf.sprintf "%s.0.c_int(), %s.1.c_int()" name name - | Int64Option -> Printf.sprintf "%s.unwrap_or(0i64), %s.is_none() as i8" name name - | DoubleOption -> - Printf.sprintf "%s.unwrap_or(std::f64::NAN), %s.is_none() as i8" name name - | String -> Printf.sprintf "%s.as_ptr(), %s.len() as i32" name name - | IntList | IntListOption | DoubleList -> - Printf.sprintf "%s.as_ptr(), %s.len_i32()" name name - | TensorOptList -> - Printf.sprintf "ptr_list_opt(%s).as_ptr(), %s.len() as i32" name name - | TensorList -> Printf.sprintf "ptr_list(%s).as_ptr(), %s.len() as i32" name name - | TensorOption -> - Printf.sprintf - "%s.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor)" - name - | Int64 when String.( = ) name "reduction" -> "reduction.to_int()" - | Layout -> Printf.sprintf "%s.to_i8()" name - | LayoutOption -> Printf.sprintf "%s.map_or(-1, |s| s.to_i8())" name - | _ -> name) - |> String.concat ~sep:",\n " -end - -exception Not_a_simple_arg - -let read_yaml filename = - let funcs = - (* Split the file to avoid Yaml.of_string_exn segfaulting. *) - In_channel.with_file filename ~f:In_channel.input_lines - |> List.group ~break:(fun _ l -> String.length l > 0 && Char.( = ) l.[0] '-') - |> List.concat_map ~f:(fun lines -> - Yaml.of_string_exn (String.concat lines ~sep:"\n") |> extract_list) - in - printf "Read %s, got %d functions.\n%!" filename (List.length funcs); - List.filter_map funcs ~f:(fun yaml -> - let map = extract_map yaml in - let name = Map.find_exn map "name" |> extract_string in - let operator_name = Map.find_exn map "operator_name" |> extract_string in - let overload_name = Map.find_exn map "overload_name" |> extract_string in - let deprecated = Map.find_exn map "deprecated" |> extract_bool in - let method_of = - Map.find_exn map "method_of" |> extract_list |> List.map ~f:extract_string - in - let arguments = Map.find_exn map "arguments" |> extract_list in - let returns = - let is_tensor returns = - let returns = extract_map returns in - let return_type = Map.find_exn returns "dynamic_type" |> extract_string in - String.( = ) return_type "at::Tensor" - in - let returns = Map.find_exn map "returns" |> extract_list in - if List.is_empty returns - then Some `nothing - else if List.for_all returns ~f:is_tensor - then Some (`fixed (List.length returns)) - else ( - match returns with - | [ returns ] -> - let return_type = - Map.find_exn (extract_map returns) "dynamic_type" |> extract_string - in - (match return_type with - | "bool" -> Some `bool - | "int64_t" -> Some `int64_t - | "double" -> Some `double - | "at::TensorList" | "dynamic_type: const c10::List> &" - -> Some `dynamic - | _ -> None) - | [] | _ :: _ :: _ -> None) - in - let kind = - if List.exists method_of ~f:(String.( = ) "namespace") - then Some `function_ - else if List.exists method_of ~f:(String.( = ) "Tensor") - then Some `method_ - else None - in - if (not deprecated) - && (not - (List.exists excluded_prefixes ~f:(fun prefix -> - String.is_prefix name ~prefix))) - && (not - (List.exists excluded_suffixes ~f:(fun suffix -> - String.is_suffix name ~suffix))) - && not (Set.mem excluded_functions name) - then - Option.both returns kind - |> Option.bind ~f:(fun (returns, kind) -> - try - let args ~with_optional_scalar_args = - List.filter_map arguments ~f:(fun arg -> - let arg = extract_map arg in - let arg_name = Map.find_exn arg "name" |> extract_string in - let arg_type = Map.find_exn arg "dynamic_type" |> extract_string in - let is_nullable = - Map.find arg "is_nullable" - |> Option.value_map ~default:false ~f:extract_bool - in - let default_value = - Map.find arg "default" |> Option.map ~f:extract_string - in - match Func.arg_type_of_string arg_type ~is_nullable with - | Some Scalar when Option.is_some default_value && not is_nullable -> - if with_optional_scalar_args - then Some { Func.arg_name; arg_type = Scalar; default_value } - else None - | Some TensorOptions - when Option.is_some default_value && Set.mem no_tensor_options name -> - None - | Some arg_type -> - let arg_name = - match arg_name, arg_type with - | "self", Scalar -> "self_scalar" - | _, _ -> arg_name - in - Some { Func.arg_name; arg_type; default_value } - | None -> - if Option.is_some default_value then None else raise Not_a_simple_arg) - in - let args = - args ~with_optional_scalar_args:(Set.mem with_optional_scalar_args name) - in - Some [ { Func.name; operator_name; overload_name; args; returns; kind } ] - with - | Not_a_simple_arg -> None) - else None) - -let p out_channel s = - Printf.ksprintf - (fun line -> - Out_channel.output_string out_channel line; - Out_channel.output_char out_channel '\n') - s - -let write_cpp funcs filename = - Out_channel.with_file (filename ^ ".cpp") ~f:(fun out_cpp -> - Out_channel.with_file (filename ^ ".h") ~f:(fun out_h -> - let pc s = p out_cpp s in - let ph s = p out_h s in - pc "// THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT BY HAND!"; - pc "#include \"%s.h\"" (Stdlib.Filename.basename filename); - pc ""; - ph "// THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT BY HAND!"; - ph "#include \"torch_api.h\""; - ph ""; - ph "extern \"C\" {"; - Map.iteri funcs ~f:(fun ~key:exported_name ~data:func -> - let c_typed_args_list = Func.c_typed_args_list func in - match func.returns with - | `nothing -> - pc "void atg_%s(%s) {" exported_name c_typed_args_list; - pc " PROTECT("; - pc " %s;" (Func.c_call func); - pc " )"; - pc "}"; - pc ""; - ph "void atg_%s(%s);" exported_name c_typed_args_list - | `fixed ntensors -> - pc "void atg_%s(tensor *out__, %s) {" exported_name c_typed_args_list; - pc " PROTECT("; - pc " auto outputs__ = %s;" (Func.c_call func); - if ntensors = 1 - then pc " out__[0] = new torch::Tensor(outputs__);" - else - for i = 0 to ntensors - 1 do - pc " out__[%d] = new torch::Tensor(std::get<%d>(outputs__));" i i - done; - pc " )"; - pc "}"; - pc ""; - ph "void atg_%s(tensor *, %s);" exported_name c_typed_args_list - | `dynamic -> - pc "tensor *atg_%s(%s) {" exported_name c_typed_args_list; - pc " PROTECT("; - pc " auto outputs__ = %s;" (Func.c_call func); - (* the returned type is a C++ vector of tensors *) - pc " int sz = outputs__.size();"; - pc - " torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * \ - sizeof(torch::Tensor*));"; - pc " for (int i = 0; i < sz; ++i)"; - pc " out__[i] = new torch::Tensor(outputs__[i]);"; - pc " out__[sz] = nullptr;"; - pc " return out__;"; - pc " )"; - pc " return nullptr;"; - pc "}"; - pc ""; - ph "tensor *atg_%s(%s);" exported_name c_typed_args_list - | (`bool | `int64_t | `double) as returns -> - let c_type = - match returns with - | `bool -> "int" - | `int64_t -> "int64_t" - | `double -> "double" - in - pc "%s atg_%s(%s) {" c_type exported_name c_typed_args_list; - pc " PROTECT("; - pc " return %s;" (Func.c_call func); - pc " )"; - pc " return 0;"; - pc "}"; - pc ""; - ph "%s atg_%s(%s);" c_type exported_name c_typed_args_list); - ph "}")) - -let write_fallible_wrapper funcs filename = - Out_channel.with_file filename ~f:(fun out_ml -> - let pm s = p out_ml s in - pm "/* THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT BY HAND! */"; - pm "#![allow(clippy::all)]"; - pm "use torch_sys::*;"; - pm "use torch_sys::c_generated::*;"; - pm "use crate::{Device, Kind, Scalar, TchError, Tensor, Layout};"; - pm "use std::convert::Into;"; - pm "use std::borrow::Borrow;"; - pm ""; - pm "fn ptr_list_opt>(l: &[Option]) -> Vec<*mut C_tensor> {"; - pm - " l.iter().map(|x| x.as_ref().map_or(std::ptr::null_mut(), |x| \ - x.borrow().c_tensor)).collect()"; - pm "}"; - pm ""; - pm "fn ptr_list>(l: &[T]) -> Vec<*mut C_tensor> {"; - pm " l.iter().map(|x| x.borrow().c_tensor).collect()"; - pm "}"; - pm ""; - pm "impl Tensor {"; - Map.iteri funcs ~f:(fun ~key:exported_name ~data:(func : Func.t) -> - let rust_name = Func.rust_name exported_name in - let self, rust_args_list = Func.rust_typed_args_list func in - pm ""; - pm " pub fn f_%s%s(" rust_name (Func.type_parameters func); - pm " %s" rust_args_list; - pm " )%s {" (Func.rust_return_type func ~fallible:true); - List.iter func.args ~f:(fun arg -> - match arg.arg_type with - | DoubleOption | Int64Option -> - let arg_name = Func.rust_name arg.arg_name in - pm " let %s = %s.into();" arg_name arg_name - | _ -> ()); - match func.returns with - | `dynamic -> - pm " let c_tensors = unsafe_torch_err!("; - pm " atg_%s(" exported_name; - pm " %s));" (Func.rust_binding_args func ~self); - pm " let mut r__ = vec![];"; - pm " let mut i = 0;"; - pm " loop {"; - pm " let c__ = unsafe{*c_tensors.add(i)};"; - pm " if c__.is_null() { break }"; - pm " r__.push(Tensor {c_tensor: c__});"; - pm " i += 1;"; - pm " }"; - pm " unsafe{libc::free(c_tensors as *mut libc::c_void)}"; - pm " Ok(r__)"; - pm " }" - | `nothing -> - pm " unsafe_torch_err!("; - pm " atg_%s(" exported_name; - pm " %s" (Func.rust_binding_args func ~self); - pm " ));"; - pm " Ok(())"; - pm " }" - | `fixed ntensors -> - pm " let mut c_tensors = [std::ptr::null_mut(); %d];" ntensors; - pm " unsafe_torch_err!("; - pm " atg_%s(c_tensors.as_mut_ptr()," exported_name; - pm " %s" (Func.rust_binding_args func ~self); - pm " ));"; - let returns = - if ntensors = 1 - then "Tensor { c_tensor: c_tensors[0] }" - else - List.init ntensors ~f:(Printf.sprintf "Tensor { c_tensor: c_tensors[%d] }") - |> String.concat ~sep:", " - |> Printf.sprintf "(%s)" - in - pm " Ok(%s)" returns; - pm " }" - | (`bool | `int64_t | `double) as returns -> - let is_bool = - match returns with - | `bool -> true - | `int64_t | `double -> false - in - pm " let return_;"; - pm " unsafe_torch_err!("; - pm " return_ = atg_%s(" exported_name; - pm " %s" (Func.rust_binding_args func ~self); - pm " ));"; - let return_ = if is_bool then "return_ != 0" else "return_" in - pm " Ok(%s)" return_; - pm " }"); - pm "}") - -let write_wrapper funcs filename = - Out_channel.with_file filename ~f:(fun out_ml -> - let pm s = p out_ml s in - pm "/* THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT BY HAND! */"; - pm "#![allow(clippy::all)]"; - pm "use crate::{Device, Kind, Scalar, Tensor, Layout};"; - pm "use std::convert::Into;"; - pm "use std::borrow::Borrow;"; - pm "use torch_sys::*;"; - pm ""; - pm "impl Tensor {"; - Map.iteri funcs ~f:(fun ~key:exported_name ~data:(func : Func.t) -> - let rust_name = Func.rust_name exported_name in - let rust_name, fallible_rust_name = - if Set.mem prefixed_functions func.name - then "g_" ^ rust_name, "f_" ^ rust_name - else rust_name, "f_" ^ rust_name - in - pm ""; - pm " pub fn %s%s(" rust_name (Func.type_parameters func); - let _self, rust_args_list = Func.rust_typed_args_list func in - pm " %s" rust_args_list; - pm " )%s {" (Func.rust_return_type func ~fallible:false); - let self, rust_args_list = Func.rust_args_list func in - let self = if Option.is_some self then "self." else "Tensor::" in - let rust_args_list = - List.map rust_args_list ~f:(fun arg -> Func.rust_name arg.Func.arg_name) - |> String.concat ~sep:", " - in - pm " %s%s(%s).unwrap()" self fallible_rust_name rust_args_list; - pm " }"); - pm "}") - -let write_ffi funcs filename = - Out_channel.with_file filename ~f:(fun out_ml -> - let pm s = p out_ml s in - pm "/* THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT BY HAND! */"; - pm "#[allow(clippy::all)]"; - pm "use crate::{C_scalar, C_tensor};"; - pm "use libc::c_int;"; - pm ""; - pm "extern \"C\" {"; - Map.iteri funcs ~f:(fun ~key:exported_name ~data:func -> - match func.Func.returns with - | `nothing -> pm " pub fn atg_%s(%s);" exported_name (Func.c_rust_args_list func) - | `fixed _ -> - pm - " pub fn atg_%s(out__: *mut *mut C_tensor, %s);" - exported_name - (Func.c_rust_args_list func) - | `dynamic -> - pm - " pub fn atg_%s(%s) -> *mut *mut C_tensor;" - exported_name - (Func.c_rust_args_list func) - | (`bool | `int64_t | `double) as returns -> - let rust_type = - match returns with - | `bool -> "c_int" - | `int64_t -> "i64" - | `double -> "f64" - in - pm - " pub fn atg_%s(%s) -> %s;" - exported_name - (Func.c_rust_args_list func) - rust_type); - pm "}") - -let methods = - let c name args = - { Func.name - ; operator_name = name - ; overload_name = "" - ; args - ; returns = `fixed 1 - ; kind = `method_ - } - in - let ca arg_name arg_type = { Func.arg_name; arg_type; default_value = None } in - [ c "grad" [ ca "self" Tensor ] - ; c "set_requires_grad" [ ca "self" Tensor; ca "r" Bool ] - ; c "toType" [ ca "self" Tensor; ca "scalar_type" ScalarType ] - ; c "to" [ ca "self" Tensor; ca "device" Device ] - ] - -let run - ~yaml_filename - ~cpp_filename - ~ffi_filename - ~wrapper_filename - ~fallible_wrapper_filename - = - let funcs = read_yaml yaml_filename |> List.concat in - let funcs = methods @ funcs in - printf "Generating code for %d functions.\n%!" (List.length funcs); - (* Generate some unique names for overloaded functions. *) - let funcs = - List.map funcs ~f:(fun func -> Func.operator_name func, func) - |> Map.of_alist_multi (module String) - |> Map.to_alist - |> List.concat_map ~f:(fun (name, funcs) -> - match funcs with - | [] -> assert false - | [ func ] -> [ name, func ] - | funcs -> - let has_empty_overload = - List.exists funcs ~f:(fun (func : Func.t) -> - String.is_empty func.overload_name) - in - List.sort funcs ~compare:(fun (f1 : Func.t) (f2 : Func.t) -> - match Int.compare (String.length f1.name) (String.length f2.name) with - | 0 -> Int.compare (List.length f1.args) (List.length f2.args) - | cmp -> cmp) - |> List.mapi ~f:(fun index (func : Func.t) -> - let operator_name = Func.operator_name func in - let overload_name = String.lowercase func.overload_name in - let name = - if String.is_empty overload_name || (index = 0 && not has_empty_overload) - then operator_name - else if String.is_suffix operator_name ~suffix:"_" - then operator_name ^ overload_name ^ "_" - else operator_name ^ "_" ^ overload_name - in - name, func)) - |> Map.of_alist_exn (module String) - in - write_cpp funcs cpp_filename; - write_ffi funcs ffi_filename; - write_wrapper funcs wrapper_filename; - write_fallible_wrapper funcs fallible_wrapper_filename - -let () = - run - ~yaml_filename:"third_party/pytorch/Declarations-v2.11.0.yaml" - ~cpp_filename:"torch-sys/libtch/torch_api_generated" - ~ffi_filename:"torch-sys/src/c_generated.rs" - ~wrapper_filename:"src/wrappers/tensor_generated.rs" - ~fallible_wrapper_filename:"src/wrappers/tensor_fallible_generated.rs" diff --git a/v2/dependency-patches/tch-0.24.0/gen/gen.mli b/v2/dependency-patches/tch-0.24.0/gen/gen.mli deleted file mode 100644 index 6d56210951..0000000000 --- a/v2/dependency-patches/tch-0.24.0/gen/gen.mli +++ /dev/null @@ -1 +0,0 @@ -(* Intentionally left blank. *) diff --git a/v2/dependency-patches/tch-0.24.0/main.py b/v2/dependency-patches/tch-0.24.0/main.py deleted file mode 100644 index 8bbdbfe955..0000000000 --- a/v2/dependency-patches/tch-0.24.0/main.py +++ /dev/null @@ -1,6 +0,0 @@ -def main(): - print("Hello from tch-rs!") - - -if __name__ == "__main__": - main() diff --git a/v2/dependency-patches/tch-0.24.0/pyproject.toml b/v2/dependency-patches/tch-0.24.0/pyproject.toml deleted file mode 100644 index a8e15e75a2..0000000000 --- a/v2/dependency-patches/tch-0.24.0/pyproject.toml +++ /dev/null @@ -1,9 +0,0 @@ -[project] -name = "tch-rs" -version = "0.1.0" -description = "Rust bindings for PyTorch" -readme = "README.md" -requires-python = ">=3.12" -dependencies = [ - "torch==2.11.0", -] diff --git a/v2/dependency-patches/tch-0.24.0/rustfmt.toml b/v2/dependency-patches/tch-0.24.0/rustfmt.toml deleted file mode 100644 index 8e8f481d13..0000000000 --- a/v2/dependency-patches/tch-0.24.0/rustfmt.toml +++ /dev/null @@ -1,2 +0,0 @@ -use_small_heuristics = "Max" -edition = "2021" diff --git a/v2/dependency-patches/tch-0.24.0/src/data.rs b/v2/dependency-patches/tch-0.24.0/src/data.rs deleted file mode 100644 index 97256e8018..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/data.rs +++ /dev/null @@ -1,203 +0,0 @@ -//! Dataset iterators. -use crate::{kind, kind::Kind, Device, IndexOp, TchError, Tensor}; -use std::collections::HashMap; - -/// An iterator over a pair of tensors which have the same first dimension -/// size. -/// The typical use case is to iterate over batches. Each batch is a pair -/// containing a (potentially random) slice of each of the two input -/// tensors. -#[derive(Debug)] -pub struct Iter2 { - xs: Tensor, - ys: Tensor, - batch_index: i64, - batch_size: i64, - total_size: i64, - device: Device, - return_smaller_last_batch: bool, -} - -impl Iter2 { - /// Returns a new iterator. - /// - /// This takes as input two tensors which first dimension must match. The - /// returned iterator can be used to range over mini-batches of data of - /// specified size. - /// An error is returned if `xs` and `ys` have different first dimension - /// sizes. - /// - /// # Arguments - /// - /// * `xs` - the features to be used by the model. - /// * `ys` - the targets that the model attempts to predict. - /// * `batch_size` - the size of batches to be returned. - pub fn f_new(xs: &Tensor, ys: &Tensor, batch_size: i64) -> Result { - let total_size = xs.size()[0]; - if ys.size()[0] != total_size { - return Err(TchError::Shape(format!( - "different dimension for the two inputs {xs:?} {ys:?}" - ))); - } - Ok(Iter2 { - xs: xs.shallow_clone(), - ys: ys.shallow_clone(), - batch_index: 0, - batch_size, - total_size, - device: Device::Cpu, - return_smaller_last_batch: false, - }) - } - - /// Returns a new iterator. - /// - /// This takes as input two tensors which first dimension must match. The - /// returned iterator can be used to range over mini-batches of data of - /// specified size. - /// Panics if `xs` and `ys` have different first dimension sizes. - /// - /// # Arguments - /// - /// * `xs` - the features to be used by the model. - /// * `ys` - the targets that the model attempts to predict. - /// * `batch_size` - the size of batches to be returned. - pub fn new(xs: &Tensor, ys: &Tensor, batch_size: i64) -> Iter2 { - Iter2::f_new(xs, ys, batch_size).unwrap() - } - - /// Shuffles the dataset. - /// - /// The iterator would still run over the whole dataset but the order in - /// which elements are grouped in mini-batches is randomized. - pub fn shuffle(&mut self) -> &mut Iter2 { - let index = Tensor::randperm(self.total_size, (Kind::Int64, self.device)); - self.xs = self.xs.index_select(0, &index); - self.ys = self.ys.index_select(0, &index); - self - } - - /// Transfers the mini-batches to a specified device. - #[allow(clippy::wrong_self_convention)] - pub fn to_device(&mut self, device: Device) -> &mut Iter2 { - self.device = device; - self - } - - /// When set, returns the last batch even if smaller than the batch size. - pub fn return_smaller_last_batch(&mut self) -> &mut Iter2 { - self.return_smaller_last_batch = true; - self - } -} - -impl Iterator for Iter2 { - type Item = (Tensor, Tensor); - - fn next(&mut self) -> Option { - let start = self.batch_index * self.batch_size; - let size = std::cmp::min(self.batch_size, self.total_size - start); - if size <= 0 || (!self.return_smaller_last_batch && size < self.batch_size) { - None - } else { - self.batch_index += 1; - Some(( - self.xs.i(start..start + size).to_device(self.device), - self.ys.i(start..start + size).to_device(self.device), - )) - } - } -} - -/// Text data holder. -#[derive(Debug)] -pub struct TextData { - data: Tensor, - char_for_label: Vec, - label_for_char: HashMap, -} - -/// Text data iterator. -#[derive(Debug)] -pub struct TextDataIter { - data: Tensor, - seq_len: i64, - batch_index: i64, - batch_size: i64, - indexes: Tensor, - indexes_len: i64, -} - -impl TextData { - /// Creates a text dataset from a file. - pub fn new>(filename: P) -> Result { - let mut buffer = std::fs::read(&filename).map_err(|err| { - std::io::Error::new(err.kind(), format!("{:?} {err}", filename.as_ref())) - })?; - - let mut label_for_char = HashMap::::new(); - let mut char_for_label = Vec::::new(); - for c in buffer.iter_mut() { - *c = *label_for_char.entry(*c).or_insert_with(|| { - let label = char_for_label.len() as u8; - char_for_label.push(*c as char); - label - }) - } - - Ok(TextData { data: Tensor::from_slice(&buffer), char_for_label, label_for_char }) - } - - /// Returns the number of different characters/labels used by the dataset. - pub fn labels(&self) -> i64 { - self.char_for_label.len() as i64 - } - - /// Returns a shallow copy of the data. - pub fn data(&self) -> Tensor { - self.data.shallow_clone() - } - - pub fn label_to_char(&self, label: i64) -> char { - self.char_for_label[label as usize] - } - - pub fn char_to_label(&self, c: char) -> Result { - match self.label_for_char.get(&(c as u8)) { - None => Err(TchError::Convert(format!("cannot find char {c}"))), - Some(v) => Ok(*v), - } - } - - /// Returns a batch iterator over the dataset. - /// Each sample is made of seq_len characters. - pub fn iter_shuffle(&self, seq_len: i64, batch_size: i64) -> TextDataIter { - let indexes_len = self.data.size()[0] - seq_len + 1; - TextDataIter { - data: self.data.shallow_clone(), - seq_len, - batch_index: 0, - batch_size, - indexes: Tensor::randperm(indexes_len, kind::INT64_CPU), - indexes_len, - } - } -} - -impl Iterator for TextDataIter { - type Item = Tensor; - - fn next(&mut self) -> Option { - let start = self.batch_index * self.batch_size; - let size = std::cmp::min(self.batch_size, self.indexes_len - start); - if size < self.batch_size { - None - } else { - self.batch_index += 1; - let indexes = Vec::::try_from(&self.indexes.i(start..start + size)).unwrap(); - let batch: Vec<_> = indexes.iter().map(|&i| self.data.i(i..i + self.seq_len)).collect(); - let batch: Vec<_> = batch.iter().collect(); - Some(Tensor::stack(&batch, 0)) - } - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/error.rs b/v2/dependency-patches/tch-0.24.0/src/error.rs deleted file mode 100644 index 1f7cc3e3d9..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/error.rs +++ /dev/null @@ -1,74 +0,0 @@ -use std::ffi::NulError; -use std::io; -use std::num::ParseIntError; - -use thiserror::Error; -use zip::result::ZipError; - -/// Main library error type. -#[derive(Error, Debug)] -pub enum TchError { - /// Conversion error. - #[error("conversion error: {0}")] - Convert(String), - - /// Invalid file format. - #[error("invalid file format: {0}")] - FileFormat(String), - - /// Missing tensor with name. - #[error("cannot find the tensor named {0} in {1}")] - TensorNameNotFound(String, String), - - /// I/O error. - #[error(transparent)] - Io(#[from] io::Error), - - /// Tensor kind error. - #[error("tensor kind error: {0}")] - Kind(String), - - /// Missing image. - #[error("no image found in {0}")] - MissingImage(String), - - /// Null pointer. - #[error(transparent)] - Nul(#[from] NulError), - - /// Integer parse error. - #[error(transparent)] - ParseInt(#[from] ParseIntError), - - /// Invalid shape. - #[error("invalid shape: {0}")] - Shape(String), - - /// Unknown kind - #[error("unknown kind: {0}")] - UnknownKind(libc::c_int), - - /// Errors returned by the Torch C++ API. - #[error("Internal torch error: {0}")] - Torch(String), - - /// Zip file format error. - #[error(transparent)] - Zip(#[from] ZipError), - - #[error(transparent)] - NdArray(#[from] ndarray::ShapeError), - - /// Errors returned by the safetensors library. - #[error("safetensors error {path}: {err}")] - SafeTensorError { path: String, err: safetensors::SafeTensorError }, -} - -impl TchError { - pub fn path_context(&self, path_name: &str) -> Self { - match self { - TchError::Torch(error) => TchError::Torch(format!("{path_name}: {error}")), - _ => unimplemented!(), - } - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/lib.rs b/v2/dependency-patches/tch-0.24.0/src/lib.rs deleted file mode 100644 index 83caa31849..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/lib.rs +++ /dev/null @@ -1,32 +0,0 @@ -#[macro_use] -extern crate lazy_static; - -pub mod data; - -mod error; -pub use error::TchError; -pub type Result = std::result::Result; - -pub(crate) mod wrappers; -pub use wrappers::device::{Cuda, Device}; -pub use wrappers::jit::{self, CModule, IValue, TrainableCModule}; -pub use wrappers::kind::{self, Kind}; -pub use wrappers::layout::Layout; -pub use wrappers::optimizer::COptimizer; -#[cfg(feature = "python-extension")] -pub use wrappers::python; -pub use wrappers::scalar::Scalar; -pub use wrappers::utils; -pub use wrappers::{ - get_num_interop_threads, get_num_threads, manual_seed, set_num_interop_threads, - set_num_threads, QEngine, -}; - -mod tensor; -pub use tensor::{ - autocast, display, index, no_grad, no_grad_guard, with_grad, IndexOp, NewAxis, NoGradGuard, - Reduction, Shape, Tensor, TensorIndexer, -}; - -pub mod nn; -pub mod vision; diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/batch_norm.rs b/v2/dependency-patches/tch-0.24.0/src/nn/batch_norm.rs deleted file mode 100644 index 8a8d2ceec6..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/nn/batch_norm.rs +++ /dev/null @@ -1,132 +0,0 @@ -//! A batch-normalization layer. -use crate::Tensor; -use std::borrow::Borrow; - -/// Batch-normalization config. -#[derive(Debug, Clone, Copy)] -pub struct BatchNormConfig { - pub cudnn_enabled: bool, - pub eps: f64, - pub momentum: f64, - pub affine: bool, - pub ws_init: super::Init, - pub bs_init: super::Init, -} - -impl Default for BatchNormConfig { - fn default() -> Self { - BatchNormConfig { - cudnn_enabled: true, - eps: 1e-5, - momentum: 0.1, - affine: true, - ws_init: super::Init::Uniform { lo: 0., up: 1. }, - bs_init: super::Init::Const(0.), - } - } -} - -/// A batch-normalization layer. -#[derive(Debug)] -pub struct BatchNorm { - config: BatchNormConfig, - pub running_mean: Tensor, - pub running_var: Tensor, - pub ws: Option, - pub bs: Option, - pub nd: usize, -} - -fn batch_norm<'a, T: Borrow>>( - vs: T, - nd: usize, - out_dim: i64, - config: BatchNormConfig, -) -> BatchNorm { - let vs = vs.borrow(); - let (ws, bs) = if config.affine { - let ws = vs.var("weight", &[out_dim], config.ws_init); - let bs = vs.var("bias", &[out_dim], config.bs_init); - (Some(ws), Some(bs)) - } else { - (None, None) - }; - BatchNorm { - config, - running_mean: vs.zeros_no_train("running_mean", &[out_dim]), - running_var: vs.ones_no_train("running_var", &[out_dim]), - ws, - bs, - nd, - } -} - -/// Applies Batch Normalization over a three dimension input. -/// -/// The input shape is assumed to be (N, C, L). Normalization -/// is performed over the first batch dimension N. -pub fn batch_norm1d<'a, T: Borrow>>( - vs: T, - out_dim: i64, - config: BatchNormConfig, -) -> BatchNorm { - batch_norm(vs, 1, out_dim, config) -} - -/// Applies Batch Normalization over a four dimension input. -/// -/// The input shape is assumed to be (N, C, H, W). Normalization -/// is performed over the first batch dimension N. -pub fn batch_norm2d<'a, T: Borrow>>( - vs: T, - out_dim: i64, - config: BatchNormConfig, -) -> BatchNorm { - batch_norm(vs, 2, out_dim, config) -} - -/// Applies Batch Normalization over a five dimension input. -/// -/// The input shape is assumed to be (N, C, D, H, W). Normalization -/// is performed over the first batch dimension N. -pub fn batch_norm3d<'a, T: Borrow>>( - vs: T, - out_dim: i64, - config: BatchNormConfig, -) -> BatchNorm { - batch_norm(vs, 3, out_dim, config) -} - -impl super::module::ModuleT for BatchNorm { - fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor { - let dim = xs.dim(); - if self.nd == 1 && dim != 2 && dim != 3 { - panic!( - "as nd={}, expected an input tensor with 2 or 3 dims, got {} ({:?})", - self.nd, - dim, - xs.size() - ) - } - if self.nd > 1 && xs.dim() != self.nd + 2 { - panic!( - "as nd={}, expected an input tensor with {} dims, got {} ({:?})", - self.nd, - self.nd + 2, - dim, - xs.size() - ) - }; - Tensor::batch_norm( - xs, - self.ws.as_ref(), - self.bs.as_ref(), - Some(&self.running_mean), - Some(&self.running_var), - train, - self.config.momentum, - self.config.eps, - self.config.cudnn_enabled, - ) - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/conv.rs b/v2/dependency-patches/tch-0.24.0/src/nn/conv.rs deleted file mode 100644 index 6c99248fd6..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/nn/conv.rs +++ /dev/null @@ -1,242 +0,0 @@ -//! N-dimensional convolution layers. -use super::Path; -use crate::{TchError, Tensor}; -use std::borrow::Borrow; - -/// How padding is performed by convolution operations -/// on the edge of the input tensor. -#[derive(Debug, Clone, Copy)] -pub enum PaddingMode { - Zeros, - Reflect, - Replicate, - Circular, -} - -impl PaddingMode { - fn to_string(self) -> &'static str { - // This has to match the internal representation used on the C++ - // side. - match self { - // The default value when using constant is zero. - PaddingMode::Zeros => "constant", - PaddingMode::Reflect => "reflect", - PaddingMode::Replicate => "replicate", - PaddingMode::Circular => "circular", - } - } - - pub fn f_pad( - self, - xs: &Tensor, - reversed_padding_repeated_twice: &[i64], - ) -> Result { - xs.f_pad(reversed_padding_repeated_twice, self.to_string(), None) - } - - pub fn pad(self, xs: &Tensor, reversed_padding_repeated_twice: &[i64]) -> Tensor { - xs.pad(reversed_padding_repeated_twice, self.to_string(), None) - } -} - -/// Generic convolution config. -#[allow(clippy::upper_case_acronyms)] -#[derive(Debug, Clone, Copy)] -pub struct ConvConfigND { - pub stride: ND, - pub padding: ND, - pub dilation: ND, - pub groups: i64, - pub bias: bool, - pub ws_init: super::Init, - pub bs_init: super::Init, - pub padding_mode: PaddingMode, -} - -/// Convolution config using the same parameters on all dimensions. -pub type ConvConfig = ConvConfigND; - -impl Default for ConvConfig { - fn default() -> Self { - ConvConfig { - stride: 1, - padding: 0, - dilation: 1, - groups: 1, - bias: true, - ws_init: super::init::DEFAULT_KAIMING_UNIFORM, - bs_init: super::Init::Const(0.), - padding_mode: PaddingMode::Zeros, - } - } -} - -impl Default for ConvConfigND<[i64; 2]> { - fn default() -> Self { - ConvConfigND::<[i64; 2]> { - stride: [1, 1], - padding: [0, 0], - dilation: [1, 1], - groups: 1, - bias: true, - ws_init: super::init::DEFAULT_KAIMING_UNIFORM, - bs_init: super::Init::Const(0.), - padding_mode: PaddingMode::Zeros, - } - } -} - -/// The default convolution config without bias. -pub fn no_bias() -> ConvConfig { - ConvConfig { bias: false, ..Default::default() } -} - -// Use const generics when they have landed in stable rust. -/// A N-dimensional convolution layer. -#[derive(Debug)] -pub struct Conv { - pub ws: Tensor, - pub bs: Option, - reversed_padding_repeated_twice: Vec, - config: ConvConfigND, -} - -/// One dimension convolution layer. -pub type Conv1D = Conv<[i64; 1]>; - -/// Two dimensions convolution layer. -pub type Conv2D = Conv<[i64; 2]>; - -/// Three dimensions convolution layer. -pub type Conv3D = Conv<[i64; 3]>; - -/// Creates a new convolution layer for any number of dimensions. -pub fn conv<'a, ND: std::convert::AsRef<[i64]>, T: Borrow>>( - vs: T, - in_dim: i64, - out_dim: i64, - ksizes: ND, - config: ConvConfigND, -) -> Conv { - let vs = vs.borrow(); - let bs = if config.bias { Some(vs.var("bias", &[out_dim], config.bs_init)) } else { None }; - let mut weight_size = vec![out_dim, in_dim / config.groups]; - weight_size.extend(ksizes.as_ref().iter()); - let ws = vs.var("weight", weight_size.as_slice(), config.ws_init); - let mut reversed_padding_repeated_twice = vec![]; - for &v in config.padding.as_ref().iter().rev() { - reversed_padding_repeated_twice.push(v) - } - for &v in config.padding.as_ref().iter().rev() { - reversed_padding_repeated_twice.push(v) - } - Conv { ws, bs, config, reversed_padding_repeated_twice } -} - -trait Create: std::convert::AsRef<[i64]> + std::marker::Sized { - fn make_array(i: i64) -> Self; - - fn conv<'a, T: Borrow>>( - vs: T, - in_dim: i64, - out_dim: i64, - ksize: i64, - config: ConvConfig, - ) -> Conv { - let config = ConvConfigND:: { - stride: Self::make_array(config.stride), - padding: Self::make_array(config.padding), - dilation: Self::make_array(config.dilation), - groups: config.groups, - bias: config.bias, - ws_init: config.ws_init, - bs_init: config.bs_init, - padding_mode: config.padding_mode, - }; - conv(vs, in_dim, out_dim, Self::make_array(ksize), config) - } -} - -impl Create for [i64; 1] { - fn make_array(i: i64) -> Self { - [i] - } -} - -impl Create for [i64; 2] { - fn make_array(i: i64) -> Self { - [i, i] - } -} - -impl Create for [i64; 3] { - fn make_array(i: i64) -> Self { - [i, i, i] - } -} - -/// Creates a new one dimension convolution layer. -pub fn conv1d<'a, T: Borrow>>(vs: T, i: i64, o: i64, k: i64, c: ConvConfig) -> Conv1D { - <[i64; 1]>::conv(vs, i, o, k, c) -} - -/// Creates a new two dimension convolution layer. -pub fn conv2d<'a, T: Borrow>>(vs: T, i: i64, o: i64, k: i64, c: ConvConfig) -> Conv2D { - <[i64; 2]>::conv(vs, i, o, k, c) -} - -/// Creates a new three dimension convolution layer. -pub fn conv3d<'a, T: Borrow>>(vs: T, i: i64, o: i64, k: i64, c: ConvConfig) -> Conv3D { - <[i64; 3]>::conv(vs, i, o, k, c) -} - -impl super::module::Module for Conv1D { - fn forward(&self, xs: &Tensor) -> Tensor { - let (xs, padding) = match self.config.padding_mode { - PaddingMode::Zeros => (xs.shallow_clone(), self.config.padding), - p => (p.pad(xs, &self.reversed_padding_repeated_twice), [0]), - }; - xs.conv1d( - &self.ws, - self.bs.as_ref(), - self.config.stride, - padding, - self.config.dilation, - self.config.groups, - ) - } -} - -impl super::module::Module for Conv2D { - fn forward(&self, xs: &Tensor) -> Tensor { - let (xs, padding) = match self.config.padding_mode { - PaddingMode::Zeros => (xs.shallow_clone(), self.config.padding), - p => (p.pad(xs, &self.reversed_padding_repeated_twice), [0, 0]), - }; - xs.conv2d( - &self.ws, - self.bs.as_ref(), - self.config.stride, - padding, - self.config.dilation, - self.config.groups, - ) - } -} - -impl super::module::Module for Conv3D { - fn forward(&self, xs: &Tensor) -> Tensor { - let (xs, padding) = match self.config.padding_mode { - PaddingMode::Zeros => (xs.shallow_clone(), self.config.padding), - p => (p.pad(xs, &self.reversed_padding_repeated_twice), [0, 0, 0]), - }; - xs.conv3d( - &self.ws, - self.bs.as_ref(), - self.config.stride, - padding, - self.config.dilation, - self.config.groups, - ) - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/conv_transpose.rs b/v2/dependency-patches/tch-0.24.0/src/nn/conv_transpose.rs deleted file mode 100644 index f183eaecfa..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/nn/conv_transpose.rs +++ /dev/null @@ -1,189 +0,0 @@ -//! A two dimension transposed convolution layer. -use super::Path; -use crate::Tensor; -use std::borrow::Borrow; - -/// A generic transposed convolution configuration. -#[allow(clippy::upper_case_acronyms)] -#[derive(Debug, Clone, Copy)] -pub struct ConvTransposeConfigND { - pub stride: ND, - pub padding: ND, - pub output_padding: ND, - pub groups: i64, - pub bias: bool, - pub dilation: ND, - pub ws_init: super::Init, - pub bs_init: super::Init, -} - -/// A transposed convolution configuration using the same values on each dimension. -pub type ConvTransposeConfig = ConvTransposeConfigND; - -impl Default for ConvTransposeConfig { - fn default() -> Self { - ConvTransposeConfigND { - stride: 1, - padding: 0, - output_padding: 0, - dilation: 1, - groups: 1, - bias: true, - ws_init: super::init::DEFAULT_KAIMING_UNIFORM, - bs_init: super::Init::Const(0.), - } - } -} - -/// A generic transposed convolution layer. -#[allow(clippy::upper_case_acronyms)] -#[derive(Debug)] -pub struct ConvTransposeND { - pub ws: Tensor, - pub bs: Option, - config: ConvTransposeConfigND, -} - -/// A one dimension transposed convolution layer. -pub type ConvTranspose1D = ConvTransposeND<[i64; 1]>; - -/// A two dimension transposed convolution layer. -pub type ConvTranspose2D = ConvTransposeND<[i64; 2]>; - -/// A three dimension transposed convolution layer. -pub type ConvTranspose3D = ConvTransposeND<[i64; 3]>; - -fn conv_transpose<'a, ND: std::convert::AsRef<[i64]>, T: Borrow>>( - vs: T, - in_dim: i64, - out_dim: i64, - ksizes: ND, - config: ConvTransposeConfigND, -) -> ConvTransposeND { - let vs = vs.borrow(); - let bs = if config.bias { Some(vs.var("bias", &[out_dim], config.bs_init)) } else { None }; - let mut weight_size = vec![in_dim, out_dim / config.groups]; - weight_size.extend(ksizes.as_ref().iter()); - let ws = vs.var("weight", weight_size.as_slice(), config.ws_init); - ConvTransposeND { ws, bs, config } -} - -trait Create: std::convert::AsRef<[i64]> + std::marker::Sized { - fn make_array(i: i64) -> Self; - - fn conv_transpose<'a, T: Borrow>>( - vs: T, - in_dim: i64, - out_dim: i64, - ksize: i64, - config: ConvTransposeConfig, - ) -> ConvTransposeND { - let config = ConvTransposeConfigND:: { - stride: Self::make_array(config.stride), - padding: Self::make_array(config.padding), - output_padding: Self::make_array(config.output_padding), - dilation: Self::make_array(config.dilation), - groups: config.groups, - bias: config.bias, - ws_init: config.ws_init, - bs_init: config.bs_init, - }; - conv_transpose(vs, in_dim, out_dim, Self::make_array(ksize), config) - } -} - -impl Create for [i64; 1] { - fn make_array(i: i64) -> Self { - [i] - } -} - -impl Create for [i64; 2] { - fn make_array(i: i64) -> Self { - [i, i] - } -} - -impl Create for [i64; 3] { - fn make_array(i: i64) -> Self { - [i, i, i] - } -} - -/// Creates a one dimension transposed convolution layer. -pub fn conv_transpose1d<'a, T: Borrow>>( - vs: T, - i: i64, - o: i64, - k: i64, - c: ConvTransposeConfig, -) -> ConvTranspose1D { - <[i64; 1]>::conv_transpose(vs, i, o, k, c) -} - -/// Creates a two dimension transposed convolution layer. -pub fn conv_transpose2d<'a, T: Borrow>>( - vs: T, - i: i64, - o: i64, - k: i64, - c: ConvTransposeConfig, -) -> ConvTranspose2D { - <[i64; 2]>::conv_transpose(vs, i, o, k, c) -} - -/// Creates a three dimension transposed convolution layer. -pub fn conv_transpose3d<'a, T: Borrow>>( - vs: T, - i: i64, - o: i64, - k: i64, - c: ConvTransposeConfig, -) -> ConvTranspose3D { - <[i64; 3]>::conv_transpose(vs, i, o, k, c) -} - -impl super::module::Module for ConvTranspose1D { - fn forward(&self, xs: &Tensor) -> Tensor { - Tensor::conv_transpose1d( - xs, - &self.ws, - self.bs.as_ref(), - self.config.stride, - self.config.padding, - self.config.output_padding, - self.config.groups, - self.config.dilation, - ) - } -} - -impl super::module::Module for ConvTranspose2D { - fn forward(&self, xs: &Tensor) -> Tensor { - Tensor::conv_transpose2d( - xs, - &self.ws, - self.bs.as_ref(), - self.config.stride, - self.config.padding, - self.config.output_padding, - self.config.groups, - self.config.dilation, - ) - } -} - -impl super::module::Module for ConvTranspose3D { - fn forward(&self, xs: &Tensor) -> Tensor { - Tensor::conv_transpose3d( - xs, - &self.ws, - self.bs.as_ref(), - self.config.stride, - self.config.padding, - self.config.output_padding, - self.config.groups, - self.config.dilation, - ) - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/func.rs b/v2/dependency-patches/tch-0.24.0/src/nn/func.rs deleted file mode 100644 index 123be59c0c..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/nn/func.rs +++ /dev/null @@ -1,51 +0,0 @@ -//! Layers defined by closures. -use crate::Tensor; - -/// A layer defined by a simple closure. -pub struct Func<'a> { - f: Box Tensor + Send>, -} - -impl std::fmt::Debug for Func<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "func") - } -} - -pub fn func<'a, F>(f: F) -> Func<'a> -where - F: 'a + Fn(&Tensor) -> Tensor + Send, -{ - Func { f: Box::new(f) } -} - -impl super::module::Module for Func<'_> { - fn forward(&self, xs: &Tensor) -> Tensor { - (*self.f)(xs) - } -} - -/// A layer defined by a closure with an additional training parameter. -#[allow(clippy::type_complexity)] -pub struct FuncT<'a> { - f: Box Tensor + Send>, -} - -impl std::fmt::Debug for FuncT<'_> { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "funcT") - } -} - -pub fn func_t<'a, F>(f: F) -> FuncT<'a> -where - F: 'a + Fn(&Tensor, bool) -> Tensor + Send, -{ - FuncT { f: Box::new(f) } -} - -impl super::module::ModuleT for FuncT<'_> { - fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor { - (*self.f)(xs, train) - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/group_norm.rs b/v2/dependency-patches/tch-0.24.0/src/nn/group_norm.rs deleted file mode 100644 index f02af14ee9..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/nn/group_norm.rs +++ /dev/null @@ -1,66 +0,0 @@ -//! A group-normalization layer. -//! Group Normalization -use crate::Tensor; -use std::borrow::Borrow; - -/// Group-normalization config. -#[derive(Debug, Clone, Copy)] -pub struct GroupNormConfig { - pub cudnn_enabled: bool, - pub eps: f64, - pub affine: bool, - pub ws_init: super::Init, - pub bs_init: super::Init, -} - -impl Default for GroupNormConfig { - fn default() -> Self { - GroupNormConfig { - cudnn_enabled: true, - eps: 1e-5, - affine: true, - ws_init: super::Init::Const(1.), - bs_init: super::Init::Const(0.), - } - } -} - -/// A group-normalization layer. -#[derive(Debug)] -pub struct GroupNorm { - config: GroupNormConfig, - pub ws: Option, - pub bs: Option, - pub num_groups: i64, - pub num_channels: i64, -} - -pub fn group_norm<'a, T: Borrow>>( - vs: T, - num_groups: i64, - num_channels: i64, - config: GroupNormConfig, -) -> GroupNorm { - let vs = vs.borrow(); - let (ws, bs) = if config.affine { - let ws = vs.var("weight", &[num_channels], config.ws_init); - let bs = vs.var("bias", &[num_channels], config.bs_init); - (Some(ws), Some(bs)) - } else { - (None, None) - }; - GroupNorm { config, ws, bs, num_groups, num_channels } -} - -impl super::module::Module for GroupNorm { - fn forward(&self, xs: &Tensor) -> Tensor { - Tensor::group_norm( - xs, - self.num_groups, - self.ws.as_ref(), - self.bs.as_ref(), - self.config.eps, - self.config.cudnn_enabled, - ) - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/init.rs b/v2/dependency-patches/tch-0.24.0/src/nn/init.rs deleted file mode 100644 index 89c43f9da9..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/nn/init.rs +++ /dev/null @@ -1,212 +0,0 @@ -//! Variable initialization. -use crate::{Device, Kind, TchError, Tensor}; - -/// Number of features as input or output of a layer. -/// In Kaiming initialization, choosing `FanIn` preserves -/// the magnitude of the variance of the weights in the -/// forward pass, choosing `FanOut` preserves this -/// magnitude in the backward pass. -#[derive(Debug, Copy, Clone)] -pub enum FanInOut { - FanIn, - FanOut, -} - -impl FanInOut { - /// Compute the fan-in or fan-out value for a weight tensor of - /// the specified dimensions. - /// - pub fn for_weight_dims(&self, dims: &[i64]) -> i64 { - let receptive_field_size: i64 = dims.iter().skip(2).product(); - match &self { - FanInOut::FanIn => { - if dims.len() < 2 { - 1 - } else { - dims[1] * receptive_field_size - } - } - FanInOut::FanOut => { - if dims.is_empty() { - 1 - } else { - dims[0] * receptive_field_size - } - } - } - } -} - -#[derive(Debug, Copy, Clone)] -pub enum NormalOrUniform { - Normal, - Uniform, -} - -/// The non-linear function that follows this layer. ReLU is the -/// recommended value. -#[derive(Debug, Copy, Clone)] -pub enum NonLinearity { - ReLU, - Linear, - Sigmoid, - Tanh, - SELU, - ExplicitGain(f64), -} - -impl NonLinearity { - pub fn gain(&self) -> f64 { - match *self { - NonLinearity::ReLU => 2f64.sqrt(), - NonLinearity::Tanh => 5. / 3., - NonLinearity::Linear | NonLinearity::Sigmoid => 1., - NonLinearity::SELU => 0.75, - NonLinearity::ExplicitGain(g) => g, - } - } -} - -/// Variable initializations. -#[derive(Debug, Copy, Clone)] -pub enum Init { - /// Constant value. - Const(f64), - - /// Random normal with some mean and standard deviation. - Randn { mean: f64, stdev: f64 }, - - /// Uniform initialization between some lower and upper bounds. - Uniform { lo: f64, up: f64 }, - - /// Kaiming uniform initialization. - /// See "Delving deep into rectifiers: Surpassing human-level performance on ImageNet classification" - /// He, K. et al. (2015). This uses a uniform distribution. - Kaiming { dist: NormalOrUniform, fan: FanInOut, non_linearity: NonLinearity }, - - /// Orthogonal initialization - Orthogonal { gain: f64 }, -} - -pub const DEFAULT_KAIMING_UNIFORM: Init = Init::Kaiming { - dist: NormalOrUniform::Uniform, - fan: FanInOut::FanIn, - non_linearity: NonLinearity::ReLU, -}; - -pub const DEFAULT_KAIMING_NORMAL: Init = Init::Kaiming { - dist: NormalOrUniform::Normal, - fan: FanInOut::FanIn, - non_linearity: NonLinearity::ReLU, -}; - -/// Creates a new float tensor with the specified shape, device, and initialization. -pub fn f_init(i: Init, dims: &[i64], device: Device, kind: Kind) -> Result { - match i { - Init::Const(cst) => { - // Optimize the case for which a single C++ code can be done. - if cst == 0. { - Tensor::f_zeros(dims, (kind, device)) - } else if (cst - 1.).abs() <= f64::EPSILON { - Tensor::f_ones(dims, (kind, device)) - } else { - Tensor::f_ones(dims, (kind, device)).map(|t| t * cst) - } - } - Init::Uniform { lo, up } => Tensor::f_zeros(dims, (kind, device))?.f_uniform_(lo, up), - Init::Randn { mean, stdev } => { - if mean == 0. && (stdev - 1.).abs() <= f64::EPSILON { - Tensor::f_randn(dims, (kind, device)) - } else { - Tensor::f_randn(dims, (kind, device)).map(|t| t * stdev + mean) - } - } - Init::Kaiming { dist, fan, non_linearity } => { - let fan = fan.for_weight_dims(dims); - let gain = non_linearity.gain(); - let std = gain / (fan as f64).sqrt(); - match dist { - NormalOrUniform::Uniform => { - let bound = 3f64.sqrt() * std; - Tensor::f_zeros(dims, (kind, device))?.f_uniform_(-bound, bound) - } - NormalOrUniform::Normal => { - let randn = Tensor::f_randn(dims, (kind, device))?; - Ok(randn * std) - } - } - } - Init::Orthogonal { gain } => { - if dims.len() < 2 { - return Err(TchError::Shape( - "Only tensors with 2 or more dimensions are supported".to_string(), - )); - } - let rows = dims[0]; - let cols: i64 = dims.iter().skip(1).product(); - - let mut flattened = - Tensor::f_empty([rows, cols], (kind, device))?.f_normal_(0.0, 1.0)?; - let flattened = if rows < cols { flattened.f_t_()? } else { flattened }; - - let (mut q, r) = Tensor::f_linalg_qr(&flattened, "reduced")?; - let d = r.f_diag(0)?; - let ph = d.f_sign()?; - q *= ph; - - let mut q = if rows < cols { q.f_t_()? } else { q }; - crate::no_grad(|| q *= gain); - - q.f_contiguous() - } - } -} - -/// Creates a new float tensor with the specified shape, device, and initialization. -pub fn init(i: Init, dims: &[i64], device: Device) -> Tensor { - f_init(i, dims, device, Kind::Float).unwrap() -} - -impl Init { - /// Re-initializes an existing tensor with the specified initialization - pub fn set(self, tensor: &mut Tensor) { - match self { - Init::Const(cst) => { - let _ = tensor.fill_(cst); - } - Init::Uniform { lo, up } => { - let _ = tensor.uniform_(lo, up); - } - Init::Kaiming { dist, fan, non_linearity } => { - let fan = fan.for_weight_dims(&tensor.size()); - let gain = non_linearity.gain(); - let std = gain / (fan as f64).sqrt(); - match dist { - NormalOrUniform::Uniform => { - let bound = 3f64.sqrt() * std; - let _ = tensor.uniform_(-bound, bound); - } - NormalOrUniform::Normal => { - tensor.copy_(&(tensor.randn_like() * std)); - } - } - } - Init::Randn { mean, stdev } => { - tensor.copy_(&(tensor.randn_like() * stdev + mean)); - } - Init::Orthogonal { gain } => { - let q = - f_init(Init::Orthogonal { gain }, &tensor.size(), tensor.device(), Kind::Float) - .unwrap(); - crate::no_grad(|| tensor.view_as(&q).copy_(&q)); - } - } - } -} - -impl Tensor { - /// Re-initializes the tensor using the specified initialization. - pub fn init(&mut self, i: Init) { - i.set(self) - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/layer_norm.rs b/v2/dependency-patches/tch-0.24.0/src/nn/layer_norm.rs deleted file mode 100644 index b522f872a6..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/nn/layer_norm.rs +++ /dev/null @@ -1,65 +0,0 @@ -//! A layer-normalization layer. -use crate::Tensor; -use std::borrow::Borrow; - -/// Layer-normalization config. -#[derive(Debug, Clone, Copy)] -pub struct LayerNormConfig { - pub cudnn_enabled: bool, - pub eps: f64, - pub elementwise_affine: bool, - pub ws_init: super::Init, - pub bs_init: super::Init, -} - -impl Default for LayerNormConfig { - fn default() -> Self { - LayerNormConfig { - cudnn_enabled: true, - eps: 1e-5, - elementwise_affine: true, - ws_init: super::Init::Const(1.), - bs_init: super::Init::Const(0.), - } - } -} - -/// A layer-normalization layer. -#[derive(Debug)] -pub struct LayerNorm { - config: LayerNormConfig, - pub ws: Option, - pub bs: Option, - pub normalized_shape: Vec, -} - -pub fn layer_norm<'a, T: Borrow>>( - vs: T, - normalized_shape: Vec, - config: LayerNormConfig, -) -> LayerNorm { - let vs = vs.borrow(); - - let (ws, bs) = if config.elementwise_affine { - let ws = vs.var("weight", normalized_shape.as_slice(), config.ws_init); - let bs = vs.var("bias", normalized_shape.as_slice(), config.bs_init); - (Some(ws), Some(bs)) - } else { - (None, None) - }; - - LayerNorm { config, ws, bs, normalized_shape } -} - -impl super::module::Module for LayerNorm { - fn forward(&self, xs: &Tensor) -> Tensor { - Tensor::layer_norm( - xs, - self.normalized_shape.as_slice(), - self.ws.as_ref(), - self.bs.as_ref(), - self.config.eps, - self.config.cudnn_enabled, - ) - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/linear.rs b/v2/dependency-patches/tch-0.24.0/src/nn/linear.rs deleted file mode 100644 index e5d385ba51..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/nn/linear.rs +++ /dev/null @@ -1,74 +0,0 @@ -//! A linear fully-connected layer. -use crate::Tensor; -use std::borrow::Borrow; - -/// Configuration for a linear layer. -#[derive(Debug, Clone, Copy)] -pub struct LinearConfig { - pub ws_init: super::Init, - pub bs_init: Option, - pub bias: bool, -} - -impl Default for LinearConfig { - fn default() -> Self { - LinearConfig { ws_init: super::init::DEFAULT_KAIMING_UNIFORM, bs_init: None, bias: true } - } -} - -/// A linear fully-connected layer. -#[derive(Debug)] -pub struct Linear { - pub ws: Tensor, - pub bs: Option, -} - -/// Creates a new linear layer. -pub fn linear<'a, T: Borrow>>( - vs: T, - in_dim: i64, - out_dim: i64, - c: LinearConfig, -) -> Linear { - let vs = vs.borrow(); - let bs = if c.bias { - let bs_init = c.bs_init.unwrap_or_else(|| { - let bound = 1.0 / (in_dim as f64).sqrt(); - super::Init::Uniform { lo: -bound, up: bound } - }); - Some(vs.var("bias", &[out_dim], bs_init)) - } else { - None - }; - - Linear { ws: vs.var("weight", &[out_dim, in_dim], c.ws_init), bs } -} - -impl super::module::Module for Linear { - fn forward(&self, xs: &Tensor) -> Tensor { - xs.linear(&self.ws, self.bs.as_ref()) - } -} - -#[test] -fn matches_pytorch() { - use crate::nn::Module; - - let input = Tensor::read_npy("tests/linear/in.npy").unwrap(); - let expected_output = Tensor::read_npy("tests/linear/out.npy").unwrap(); - let ws = Tensor::read_npy("tests/linear/ws.npy").unwrap(); - let bs = Some(Tensor::read_npy("tests/linear/bs.npy").unwrap()); - - let original_output = - if let Some(bias) = &bs { input.matmul(&ws.tr()) + bias } else { input.matmul(&ws.tr()) }; - - let linear = Linear { ws, bs }; - let output = linear.forward(&input); - - let delta_output: f32 = (&output - &expected_output).norm().try_into().unwrap(); - let delta_original: f32 = (&original_output - &expected_output).norm().try_into().unwrap(); - - // The `matmul()` implementation is close, but `linear()` is at least as close or closer. - assert!(output.allclose(&expected_output, 1e-5, 1e-8, false)); - assert!(delta_output <= delta_original); -} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/mod.rs b/v2/dependency-patches/tch-0.24.0/src/nn/mod.rs deleted file mode 100644 index 9343462d7e..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/nn/mod.rs +++ /dev/null @@ -1,69 +0,0 @@ -//! A small neural-network library based on Torch. -//! -//! This library tries to stay as close as possible to the original -//! Python and C++ implementations. -pub mod init; -pub use init::{f_init, init, Init}; - -mod var_store; -pub use var_store::{Path, VarStore, Variables}; - -mod module; -pub use module::{Module, ModuleT}; - -mod linear; -pub use linear::*; - -mod conv; -pub use conv::*; - -mod conv_transpose; -pub use conv_transpose::*; - -mod batch_norm; -pub use batch_norm::*; - -mod group_norm; -pub use group_norm::*; - -mod layer_norm; -pub use layer_norm::*; - -mod sparse; -pub use sparse::*; - -mod rnn; -pub use rnn::*; - -mod func; -pub use func::*; - -mod sequential; -pub use sequential::*; - -mod optimizer; -pub use optimizer::{ - adam, adamw, rms_prop, sgd, Adam, AdamW, Optimizer, OptimizerConfig, RmsProp, Sgd, -}; - -/// An identity layer. This just propagates its tensor input as output. -#[derive(Debug)] -pub struct Id(); - -impl ModuleT for Id { - fn forward_t(&self, xs: &crate::Tensor, _train: bool) -> crate::Tensor { - xs.shallow_clone() - } -} - -impl Module for crate::CModule { - fn forward(&self, xs: &crate::Tensor) -> crate::Tensor { - self.forward_ts(&[xs]).unwrap() - } -} - -impl ModuleT for crate::TrainableCModule { - fn forward_t(&self, xs: &crate::Tensor, _train: bool) -> crate::Tensor { - self.inner.forward_ts(&[xs]).unwrap() - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/module.rs b/v2/dependency-patches/tch-0.24.0/src/nn/module.rs deleted file mode 100644 index 6301cbfacf..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/nn/module.rs +++ /dev/null @@ -1,67 +0,0 @@ -//! Basic module traits defining the forward pass. -use crate::{data::Iter2, Device, Tensor}; - -/// The simplest module trait, defining a forward function. -pub trait Module: std::fmt::Debug + Send { - fn forward(&self, xs: &Tensor) -> Tensor; -} - -/// Module trait with an additional train parameter. -/// -/// The train parameter is commonly used to have different behavior between training -/// and evaluation, e.g. when using dropout or batch-normalization. -pub trait ModuleT: std::fmt::Debug + Send { - fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor; - - fn batch_accuracy_for_logits( - &self, - xs: &Tensor, - ys: &Tensor, - d: Device, - batch_size: i64, - ) -> f64 { - let _no_grad = crate::no_grad_guard(); - let mut sum_accuracy = 0f64; - let mut sample_count = 0f64; - for (xs, ys) in Iter2::new(xs, ys, batch_size).return_smaller_last_batch() { - let acc = self.forward_t(&xs.to_device(d), false).accuracy_for_logits(&ys.to_device(d)); - let size = xs.size()[0] as f64; - sum_accuracy += f64::try_from(&acc).unwrap() * size; - sample_count += size; - } - sum_accuracy / sample_count - } -} - -impl ModuleT for T -where - T: Module, -{ - fn forward_t(&self, xs: &Tensor, _train: bool) -> Tensor { - self.forward(xs) - } -} - -impl Tensor { - pub fn apply(&self, m: &M) -> Tensor { - m.forward(self) - } - - pub fn apply_t(&self, m: &M, train: bool) -> Tensor { - m.forward_t(self, train) - } - - pub fn apply_opt(&self, m: &Option) -> Tensor { - match m { - Some(m) => m.forward(self), - None => self.shallow_clone(), - } - } - - pub fn apply_opt_t(&self, m: &Option, train: bool) -> Tensor { - match m { - Some(m) => m.forward_t(self, train), - None => self.shallow_clone(), - } - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/optimizer.rs b/v2/dependency-patches/tch-0.24.0/src/nn/optimizer.rs deleted file mode 100644 index a4ceb5d28d..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/nn/optimizer.rs +++ /dev/null @@ -1,324 +0,0 @@ -//! Optimizers to be used for gradient-descent based training. -use super::var_store::{VarStore, Variables}; -use crate::wrappers::optimizer::COptimizer; -use crate::{TchError, Tensor}; -use std::sync::{Arc, Mutex}; - -/// An optimizer to run gradient descent. -#[derive(Debug)] -pub struct Optimizer { - opt: COptimizer, - variables: Arc>, - variables_in_optimizer: usize, -} - -/// Optimizer configurations. These configs can be used to build optimizer. -pub trait OptimizerConfig -where - Self: std::marker::Sized, -{ - fn build_copt(&self, lr: f64) -> Result; - - /// Builds an optimizer with the specified learning rate handling variables stored in `vs`. - fn build(self, vs: &VarStore, lr: f64) -> Result { - let mut opt = self.build_copt(lr)?; - let v = vs.variables_.lock().unwrap(); - for var in &v.trainable_variables { - opt.add_parameters(&var.tensor, var.group)?; - } - Ok(Optimizer { - opt, - variables: vs.variables_.clone(), - variables_in_optimizer: v.trainable_variables.len(), - }) - } -} - -/// Parameters for the SGD optimizer. -#[derive(Debug, Copy, Clone)] -pub struct Sgd { - pub momentum: f64, - pub dampening: f64, - pub wd: f64, - pub nesterov: bool, -} - -impl Default for Sgd { - fn default() -> Self { - Sgd { momentum: 0., dampening: 0., wd: 0., nesterov: false } - } -} - -/// Creates the configuration for a Stochastic Gradient Descent (SGD) optimizer. -pub fn sgd(momentum: f64, dampening: f64, wd: f64, nesterov: bool) -> Sgd { - Sgd { momentum, dampening, wd, nesterov } -} - -impl OptimizerConfig for Sgd { - fn build_copt(&self, lr: f64) -> Result { - COptimizer::sgd(lr, self.momentum, self.dampening, self.wd, self.nesterov) - } -} - -/// Parameters for the Adam optimizer. -#[derive(Debug, Copy, Clone)] -pub struct Adam { - pub beta1: f64, - pub beta2: f64, - pub wd: f64, - pub eps: f64, - pub amsgrad: bool, -} - -impl Default for Adam { - fn default() -> Self { - Adam { beta1: 0.9, beta2: 0.999, wd: 0., eps: 1e-8, amsgrad: false } - } -} - -/// Creates the configuration for the Adam optimizer. -pub fn adam(beta1: f64, beta2: f64, wd: f64) -> Adam { - Adam { beta1, beta2, wd, eps: 1e-8, amsgrad: false } -} - -impl Adam { - pub fn beta1(mut self, b: f64) -> Self { - self.beta1 = b; - self - } - - pub fn beta2(mut self, b: f64) -> Self { - self.beta2 = b; - self - } - - pub fn wd(mut self, w: f64) -> Self { - self.wd = w; - self - } - - pub fn eps(mut self, e: f64) -> Self { - self.eps = e; - self - } - - pub fn amsgrad(mut self, a: bool) -> Self { - self.amsgrad = a; - self - } -} - -impl OptimizerConfig for Adam { - fn build_copt(&self, lr: f64) -> Result { - COptimizer::adam(lr, self.beta1, self.beta2, self.wd, self.eps, self.amsgrad) - } -} - -/// Parameters for the AdamW optimizer. -#[derive(Debug, Copy, Clone)] -pub struct AdamW { - pub beta1: f64, - pub beta2: f64, - pub wd: f64, - pub eps: f64, - pub amsgrad: bool, -} - -impl Default for AdamW { - fn default() -> Self { - AdamW { beta1: 0.9, beta2: 0.999, wd: 0.01, eps: 1e-8, amsgrad: false } - } -} - -/// Creates the configuration for the AdamW optimizer. -pub fn adamw(beta1: f64, beta2: f64, wd: f64) -> AdamW { - AdamW { beta1, beta2, wd, eps: 1e-8, amsgrad: false } -} - -impl AdamW { - pub fn beta1(mut self, b: f64) -> Self { - self.beta1 = b; - self - } - - pub fn beta2(mut self, b: f64) -> Self { - self.beta2 = b; - self - } - - pub fn wd(mut self, w: f64) -> Self { - self.wd = w; - self - } - - pub fn eps(mut self, e: f64) -> Self { - self.eps = e; - self - } - - pub fn amsgrad(mut self, a: bool) -> Self { - self.amsgrad = a; - self - } -} - -impl OptimizerConfig for AdamW { - fn build_copt(&self, lr: f64) -> Result { - COptimizer::adamw(lr, self.beta1, self.beta2, self.wd, self.eps, self.amsgrad) - } -} - -/// Parameters for the RmsProp optimizer. -#[derive(Debug, Copy, Clone)] -pub struct RmsProp { - pub alpha: f64, - pub eps: f64, - pub wd: f64, - pub momentum: f64, - pub centered: bool, -} - -impl Default for RmsProp { - fn default() -> Self { - RmsProp { alpha: 0.99, eps: 1e-8, wd: 0., momentum: 0., centered: false } - } -} - -/// Creates the configuration for the RmsProp optimizer. -pub fn rms_prop(alpha: f64, eps: f64, wd: f64, momentum: f64, centered: bool) -> RmsProp { - RmsProp { alpha, eps, wd, momentum, centered } -} - -impl OptimizerConfig for RmsProp { - fn build_copt(&self, lr: f64) -> Result { - COptimizer::rms_prop(lr, self.alpha, self.eps, self.wd, self.momentum, self.centered) - } -} - -impl Optimizer { - fn add_missing_variables(&mut self) { - let v = self.variables.lock().unwrap(); - if v.trainable_variables.len() > self.variables_in_optimizer { - for var in &v.trainable_variables[self.variables_in_optimizer..] { - self.opt.add_parameters(&var.tensor, var.group).unwrap(); - } - self.variables_in_optimizer = v.trainable_variables.len(); - } - } - - /// Zeroes the gradient for the tensors tracked by this optimizer. - pub fn zero_grad(&mut self) { - self.add_missing_variables(); - self.opt.zero_grad().unwrap() - } - - /// Clips gradient value at some specified maximum value. - pub fn clip_grad_value(&self, max: f64) { - let v = self.variables.lock().unwrap(); - for var in v.trainable_variables.iter() { - let mut grad = var.tensor.grad(); - if grad.defined() { - let _t = grad.clamp_(-max, max); - } - } - } - - /// Clips gradient L2 norm over all trainable parameters. - /// - /// The norm is computed over all gradients together, as if they were - /// concatenated into a single vector. - pub fn clip_grad_norm(&self, max: f64) { - crate::no_grad(|| { - let v = self.variables.lock().unwrap(); - let mut norms = vec![]; - for var in v.trainable_variables.iter() { - let grad = var.tensor.grad(); - if grad.defined() { - norms.push(grad.norm()); - } - } - let total_norm = f64::try_from(Tensor::stack(&norms, 0).norm()).unwrap(); - let clip_coef = max / (total_norm + 1e-6); - if clip_coef < 1.0 { - for var in v.trainable_variables.iter() { - let mut grad = var.tensor.grad(); - if grad.defined() { - let _t = grad.g_mul_scalar_(clip_coef); - } - } - } - }) - } - - /// Performs an optimization step, updating the tracked tensors based on their gradients. - pub fn step(&mut self) { - self.add_missing_variables(); - self.opt.step().unwrap() - } - - /// Applies a backward step pass, update the gradients, and performs an optimization step. - pub fn backward_step(&mut self, loss: &Tensor) { - self.add_missing_variables(); - self.opt.zero_grad().unwrap(); - loss.backward(); - self.opt.step().unwrap() - } - - /// Applies a backward step pass, update the gradients, and performs an optimization step. - /// - /// The gradients are clipped based on `max` before being applied. - pub fn backward_step_clip(&mut self, loss: &Tensor, max: f64) { - self.add_missing_variables(); - self.opt.zero_grad().unwrap(); - loss.backward(); - self.clip_grad_value(max); - self.opt.step().unwrap() - } - - /// Applies a backward step pass, update the gradients, and performs an optimization step. - /// - /// The gradients L2 norm is clipped based on `max`. - pub fn backward_step_clip_norm(&mut self, loss: &Tensor, max: f64) { - self.add_missing_variables(); - self.opt.zero_grad().unwrap(); - loss.backward(); - self.clip_grad_norm(max); - self.opt.step().unwrap() - } - - /// Sets the optimizer learning rate. - pub fn set_lr(&mut self, lr: f64) { - self.opt.set_learning_rate(lr).unwrap() - } - - /// Sets the optimizer momentum. - pub fn set_momentum(&mut self, m: f64) { - self.opt.set_momentum(m).unwrap() - } - - /// Sets the optimizer learning rate for a parameter group. - pub fn set_lr_group(&mut self, group: usize, lr: f64) { - self.opt.set_learning_rate_group(group, lr).unwrap() - } - - /// Sets the optimizer momentum. - pub fn set_momentum_group(&mut self, group: usize, m: f64) { - self.opt.set_momentum_group(group, m).unwrap() - } - - /// Returns all the trainable variables for this optimizer. - pub fn trainable_variables(&self) -> Vec { - let variables = self.variables.lock().unwrap(); - variables.trainable_variables.iter().map(|v| v.tensor.shallow_clone()).collect() - } - - /// Sets the optimizer weight decay. - pub fn set_weight_decay(&mut self, weight_decay: f64) { - self.opt.set_weight_decay(weight_decay).unwrap() - } - - /// Sets the optimizer weight decay. - pub fn set_weight_decay_group(&mut self, group: usize, weight_decay: f64) { - self.opt.set_weight_decay_group(group, weight_decay).unwrap() - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/rnn.rs b/v2/dependency-patches/tch-0.24.0/src/nn/rnn.rs deleted file mode 100644 index f74fd7f016..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/nn/rnn.rs +++ /dev/null @@ -1,285 +0,0 @@ -//! Recurrent Neural Networks -use crate::{Device, Tensor}; -use std::borrow::Borrow; - -/// Trait for Recurrent Neural Networks. -#[allow(clippy::upper_case_acronyms)] -pub trait RNN { - type State; - - /// A zero state from which the recurrent network is usually initialized. - fn zero_state(&self, batch_dim: i64) -> Self::State; - - /// Applies a single step of the recurrent network. - /// - /// The input should have dimensions [batch_size, features]. - fn step(&self, input: &Tensor, state: &Self::State) -> Self::State; - - /// Applies multiple steps of the recurrent network. - /// - /// The input should have dimensions [batch_size, seq_len, features]. - /// The initial state is the result of applying zero_state. - fn seq(&self, input: &Tensor) -> (Tensor, Self::State) { - let batch_dim = input.size()[0]; - let state = self.zero_state(batch_dim); - self.seq_init(input, &state) - } - - /// Applies multiple steps of the recurrent network. - /// - /// The input should have dimensions [batch_size, seq_len, features]. - fn seq_init(&self, input: &Tensor, state: &Self::State) -> (Tensor, Self::State); -} - -/// The state for a LSTM network, this contains two tensors. -#[allow(clippy::upper_case_acronyms)] -#[derive(Debug)] -pub struct LSTMState(pub (Tensor, Tensor)); - -impl LSTMState { - /// The hidden state vector, which is also the output of the LSTM. - pub fn h(&self) -> Tensor { - (self.0).0.shallow_clone() - } - - /// The cell state vector. - pub fn c(&self) -> Tensor { - (self.0).1.shallow_clone() - } -} - -// The GRU and LSTM layers share the same config. -/// Configuration for the GRU and LSTM layers. -#[allow(clippy::upper_case_acronyms)] -#[derive(Debug, Clone, Copy)] -pub struct RNNConfig { - pub has_biases: bool, - pub num_layers: i64, - pub dropout: f64, - pub train: bool, - pub bidirectional: bool, - pub batch_first: bool, - pub w_ih_init: super::Init, - pub w_hh_init: super::Init, - pub b_ih_init: Option, - pub b_hh_init: Option, -} - -impl Default for RNNConfig { - fn default() -> Self { - RNNConfig { - has_biases: true, - num_layers: 1, - dropout: 0., - train: true, - bidirectional: false, - batch_first: true, - w_ih_init: super::init::DEFAULT_KAIMING_UNIFORM, - w_hh_init: super::init::DEFAULT_KAIMING_UNIFORM, - b_ih_init: Some(super::Init::Const(0.)), - b_hh_init: Some(super::Init::Const(0.)), - } - } -} - -fn rnn_weights<'a, T: Borrow>>( - vs: T, - in_dim: i64, - hidden_dim: i64, - gate_dim: i64, - num_directions: i64, - c: RNNConfig, -) -> Vec { - let vs = vs.borrow(); - let mut flat_weights = vec![]; - for layer_idx in 0..c.num_layers { - for direction_idx in 0..num_directions { - let in_dim = if layer_idx == 0 { in_dim } else { hidden_dim * num_directions }; - let suffix = if direction_idx == 1 { "_reverse" } else { "" }; - let w_ih = vs.var( - &format!("weight_ih_l{layer_idx}{suffix}"), - &[gate_dim, in_dim], - c.w_ih_init, - ); - let w_hh = vs.var( - &format!("weight_hh_l{layer_idx}{suffix}"), - &[gate_dim, hidden_dim], - c.w_hh_init, - ); - flat_weights.push(w_ih); - flat_weights.push(w_hh); - if c.has_biases { - let b_ih = vs.var( - &format!("bias_ih_l{layer_idx}{suffix}"), - &[gate_dim], - c.b_ih_init.unwrap(), - ); - let b_hh = vs.var( - &format!("bias_hh_l{layer_idx}{suffix}"), - &[gate_dim], - c.b_hh_init.unwrap(), - ); - flat_weights.push(b_ih); - flat_weights.push(b_hh); - } - } - } - flat_weights -} - -/// A Long Short-Term Memory (LSTM) layer. -/// -/// -#[allow(clippy::upper_case_acronyms)] -#[derive(Debug)] -pub struct LSTM { - flat_weights: Vec, - hidden_dim: i64, - config: RNNConfig, - device: Device, -} - -/// Creates a LSTM layer. -pub fn lstm<'a, T: Borrow>>( - vs: T, - in_dim: i64, - hidden_dim: i64, - c: RNNConfig, -) -> LSTM { - let vs = vs.borrow(); - let num_directions = if c.bidirectional { 2 } else { 1 }; - let gate_dim = 4 * hidden_dim; - let flat_weights = rnn_weights(vs, in_dim, hidden_dim, gate_dim, num_directions, c); - - if vs.device().is_cuda() && crate::Cuda::cudnn_is_available() { - let _ = Tensor::internal_cudnn_rnn_flatten_weight( - &flat_weights, - 4, - in_dim, - 2, /* 2 for LSTM see rnn.cpp in pytorch */ - hidden_dim, - 0, /* disables projections */ - c.num_layers, - c.batch_first, - c.bidirectional, - ); - } - LSTM { flat_weights, hidden_dim, config: c, device: vs.device() } -} - -impl RNN for LSTM { - type State = LSTMState; - - fn zero_state(&self, batch_dim: i64) -> LSTMState { - let num_directions = if self.config.bidirectional { 2 } else { 1 }; - let layer_dim = self.config.num_layers * num_directions; - let shape = [layer_dim, batch_dim, self.hidden_dim]; - let zeros = Tensor::zeros(shape, (self.flat_weights[0].kind(), self.device)); - LSTMState((zeros.shallow_clone(), zeros.shallow_clone())) - } - - fn step(&self, input: &Tensor, in_state: &LSTMState) -> LSTMState { - let input = input.unsqueeze(1); - let (_output, state) = self.seq_init(&input, in_state); - state - } - - fn seq_init(&self, input: &Tensor, in_state: &LSTMState) -> (Tensor, LSTMState) { - let LSTMState((h, c)) = in_state; - let flat_weights = self.flat_weights.iter().collect::>(); - let (output, h, c) = input.lstm( - &[h, c], - &flat_weights, - self.config.has_biases, - self.config.num_layers, - self.config.dropout, - self.config.train, - self.config.bidirectional, - self.config.batch_first, - ); - (output, LSTMState((h, c))) - } -} - -/// A GRU state, this contains a single tensor. -#[allow(clippy::upper_case_acronyms)] -#[derive(Debug)] -pub struct GRUState(pub Tensor); - -impl GRUState { - pub fn value(&self) -> Tensor { - self.0.shallow_clone() - } -} - -/// A Gated Recurrent Unit (GRU) layer. -/// -/// -#[allow(clippy::upper_case_acronyms)] -#[derive(Debug)] -pub struct GRU { - flat_weights: Vec, - hidden_dim: i64, - config: RNNConfig, - device: Device, -} - -/// Creates a new GRU layer. -pub fn gru<'a, T: Borrow>>( - vs: T, - in_dim: i64, - hidden_dim: i64, - c: RNNConfig, -) -> GRU { - let vs = vs.borrow(); - let num_directions = if c.bidirectional { 2 } else { 1 }; - let gate_dim = 3 * hidden_dim; - let flat_weights = rnn_weights(vs, in_dim, hidden_dim, gate_dim, num_directions, c); - - if vs.device().is_cuda() && crate::Cuda::cudnn_is_available() { - let _ = Tensor::internal_cudnn_rnn_flatten_weight( - &flat_weights, - 4, - in_dim, - 3, /* 3 for GRU see rnn.cpp in pytorch */ - hidden_dim, - 0, /* disables projections */ - c.num_layers, - c.batch_first, - c.bidirectional, - ); - } - GRU { flat_weights, hidden_dim, config: c, device: vs.device() } -} - -impl RNN for GRU { - type State = GRUState; - - fn zero_state(&self, batch_dim: i64) -> GRUState { - let num_directions = if self.config.bidirectional { 2 } else { 1 }; - let layer_dim = self.config.num_layers * num_directions; - let shape = [layer_dim, batch_dim, self.hidden_dim]; - GRUState(Tensor::zeros(shape, (self.flat_weights[0].kind(), self.device))) - } - - fn step(&self, input: &Tensor, in_state: &GRUState) -> GRUState { - let input = input.unsqueeze(1); - let (_output, state) = self.seq_init(&input, in_state); - state - } - - fn seq_init(&self, input: &Tensor, in_state: &GRUState) -> (Tensor, GRUState) { - let GRUState(h) = in_state; - let (output, h) = input.gru( - h, - &self.flat_weights, - self.config.has_biases, - self.config.num_layers, - self.config.dropout, - self.config.train, - self.config.bidirectional, - self.config.batch_first, - ); - (output, GRUState(h)) - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/sequential.rs b/v2/dependency-patches/tch-0.24.0/src/nn/sequential.rs deleted file mode 100644 index ea1be81a15..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/nn/sequential.rs +++ /dev/null @@ -1,149 +0,0 @@ -//! A sequential layer used to chain multiple layers and closures. -use super::{Module, ModuleT}; -use crate::Tensor; - -/// A sequential layer combining multiple other layers. -#[derive(Debug)] -pub struct Sequential { - layers: Vec>, -} - -/// Creates a new empty sequential layer. -pub fn seq() -> Sequential { - Sequential { layers: vec![] } -} - -impl Sequential { - /// The number of sub-layers embedded in this layer. - pub fn len(&self) -> i64 { - self.layers.len() as i64 - } - - /// Returns true if this layer does not have any sub-layer. - pub fn is_empty(&self) -> bool { - self.layers.is_empty() - } -} - -impl Module for Sequential { - fn forward(&self, xs: &Tensor) -> Tensor { - if self.layers.is_empty() { - xs.shallow_clone() - } else { - let xs = self.layers[0].forward(xs); - self.layers.iter().skip(1).fold(xs, |xs, layer| layer.forward(&xs)) - } - } -} - -impl Sequential { - /// Appends a layer after all the current layers. - #[allow(clippy::should_implement_trait)] - pub fn add(mut self, layer: M) -> Self { - self.layers.push(Box::new(layer)); - self - } - - /// Appends a closure after all the current layers. - pub fn add_fn(self, f: F) -> Self - where - F: 'static + Fn(&Tensor) -> Tensor + Send, - { - self.add(super::func(f)) - } - - /// Applies the forward pass and returns the output for each layer. - pub fn forward_all(&self, xs: &Tensor, n: Option) -> Vec { - if self.layers.is_empty() { - vec![xs.shallow_clone()] - } else { - let n = n.unwrap_or(self.layers.len()); - let xs = self.layers[0].forward(xs); - let mut vec = vec![]; - let out = self.layers.iter().take(n).skip(1).fold(xs, |xs, layer| { - let out = layer.forward(&xs); - vec.push(xs); - out - }); - vec.push(out); - vec - } - } -} - -/// A sequential layer combining new layers with support for a training mode. -#[derive(Debug)] -pub struct SequentialT { - layers: Vec>, -} - -/// Creates a new empty sequential layer. -pub fn seq_t() -> SequentialT { - SequentialT { layers: vec![] } -} - -impl SequentialT { - /// The number of sub-layers embedded in this layer. - pub fn len(&self) -> i64 { - self.layers.len() as i64 - } - - /// Returns true if this layer does not have any sub-layer. - pub fn is_empty(&self) -> bool { - self.layers.is_empty() - } -} - -impl ModuleT for SequentialT { - fn forward_t(&self, xs: &Tensor, train: bool) -> Tensor { - if self.layers.is_empty() { - xs.shallow_clone() - } else { - let xs = self.layers[0].forward_t(xs, train); - self.layers.iter().skip(1).fold(xs, |xs, layer| layer.forward_t(&xs, train)) - } - } -} - -impl SequentialT { - /// Appends a layer after all the current layers. - #[allow(clippy::should_implement_trait)] - pub fn add(mut self, layer: M) -> Self { - self.layers.push(Box::new(layer)); - self - } - - /// Appends a closure after all the current layers. - pub fn add_fn(self, f: F) -> Self - where - F: 'static + Fn(&Tensor) -> Tensor + Send, - { - self.add(super::func(f)) - } - - /// Appends a closure after all the current layers. - pub fn add_fn_t(self, f: F) -> Self - where - F: 'static + Fn(&Tensor, bool) -> Tensor + Send, - { - self.add(super::func_t(f)) - } - - /// Applies the forward pass and returns the output for each layer. - pub fn forward_all_t(&self, xs: &Tensor, train: bool, n: Option) -> Vec { - if self.layers.is_empty() { - vec![xs.shallow_clone()] - } else { - let n = n.unwrap_or(self.layers.len()); - let xs = self.layers[0].forward_t(xs, train); - let mut vec = vec![]; - let out = self.layers.iter().take(n).skip(1).fold(xs, |xs, layer| { - let out = layer.forward_t(&xs, train); - vec.push(xs); - out - }); - vec.push(out); - vec - } - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/sparse.rs b/v2/dependency-patches/tch-0.24.0/src/nn/sparse.rs deleted file mode 100644 index 6a6eb08fc8..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/nn/sparse.rs +++ /dev/null @@ -1,55 +0,0 @@ -//! Sparse Layers -use crate::Tensor; -use std::borrow::Borrow; - -/// Configuration option for an embedding layer. -#[derive(Debug, Clone, Copy)] -pub struct EmbeddingConfig { - pub sparse: bool, - pub scale_grad_by_freq: bool, - pub ws_init: super::Init, - pub padding_idx: i64, -} - -impl Default for EmbeddingConfig { - fn default() -> Self { - EmbeddingConfig { - sparse: false, - scale_grad_by_freq: false, - ws_init: super::Init::Randn { mean: 0., stdev: 1. }, - padding_idx: -1, - } - } -} - -/// An embedding layer. -/// -/// An embedding layer acts as a simple lookup table that stores embeddings. -/// This is commonly used to store word embeddings. -#[derive(Debug)] -pub struct Embedding { - pub ws: Tensor, - config: EmbeddingConfig, -} - -pub fn embedding<'a, T: Borrow>>( - vs: T, - num_embeddings: i64, - embedding_dim: i64, - config: EmbeddingConfig, -) -> Embedding { - let vs = vs.borrow(); - Embedding { ws: vs.var("weight", &[num_embeddings, embedding_dim], config.ws_init), config } -} - -impl super::module::Module for Embedding { - fn forward(&self, xs: &Tensor) -> Tensor { - Tensor::embedding( - &self.ws, - xs, - self.config.padding_idx, - self.config.scale_grad_by_freq, - self.config.sparse, - ) - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/nn/var_store.rs b/v2/dependency-patches/tch-0.24.0/src/nn/var_store.rs deleted file mode 100644 index 0f51bfdef7..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/nn/var_store.rs +++ /dev/null @@ -1,935 +0,0 @@ -//! Variable stores. -use super::Init; -use crate::tensor::Tensor; -use crate::wrappers::stream::ReadSeekAdapter; -use crate::{Device, Kind, TchError}; -use std::collections::hash_map::Entry::{Occupied, Vacant}; -use std::collections::HashMap; -use std::io::{Read, Seek}; -use std::ops::Div; -use std::sync::{Arc, Mutex, MutexGuard}; - -/// The separator is used to separate path elements in the tensor names. -const SEP: char = '.'; - -#[derive(Debug)] -pub struct Var { - pub tensor: Tensor, - pub group: usize, -} - -// When the variable store is frozen, the trainable_variables vector -// still contains the same tensors however these tensors are set not -// to require gradients. -#[derive(Debug)] -pub struct Variables { - pub named_variables: HashMap, - pub trainable_variables: Vec, -} - -/// A VarStore is used to store variables used by one or multiple layers. -/// It specifies a single device where all variables are stored. -#[derive(Debug)] -pub struct VarStore { - pub variables_: Arc>, - device: Device, - kind: Kind, -} - -/// A variable store with an associated path for variables naming. -#[derive(Debug, Clone)] -pub struct Path<'a> { - path: Vec, - group: usize, - var_store: &'a VarStore, -} - -/// An Entry holds an entry corresponding to a given name in Path. -#[derive(Debug)] -pub struct Entry<'a> { - name: &'a str, - variables: MutexGuard<'a, Variables>, - // This field holds the mutex lock - path: &'a Path<'a>, -} - -impl VarStore { - /// Creates a new var-store located on the specified device. - pub fn new(device: Device) -> VarStore { - let variables = - Variables { named_variables: HashMap::new(), trainable_variables: Vec::new() }; - VarStore { variables_: Arc::new(Mutex::new(variables)), device, kind: Kind::Float } - } - - pub fn merge(var_stores: Vec<(VarStore, Option<&str>)>) -> Result { - let mut new_var_store = VarStore::new(Device::Cpu); - - if var_stores.is_empty() { - Ok(new_var_store) - } else { - let mut new_variables = - Variables { named_variables: HashMap::new(), trainable_variables: Vec::new() }; - let device = var_stores[0].0.device(); - - for (var_store, prefix) in var_stores { - if var_store.device() != device { - return Err(TchError::Torch(format!( - "All VarStores must be on the same device, got {:?} and {:?}", - device, - var_store.device() - ))); - } - for (var_name, var) in var_store.variables() { - let new_var_name = format!("{}{}", prefix.unwrap_or(""), var_name); - match new_variables.named_variables.entry(new_var_name) { - Occupied(v) => { - return Err(TchError::Torch(format!( - "Duplicate variable name found: {}. Provide a unique prefix to allow merge operation", - v.key(), - ))); - } - Vacant(v) => { - v.insert(var); - } - } - } - for trainable_var in - var_store.variables_.lock().unwrap().trainable_variables.drain(..) - { - new_variables.trainable_variables.push(trainable_var); - } - } - new_var_store.variables_ = Arc::new(Mutex::new(new_variables)); - new_var_store.device = device; - - Ok(new_var_store) - } - } - - /// Gets the device for this var-store. - pub fn device(&self) -> Device { - self.device - } - - /// Gets the default kind of new variables - pub fn kind(&self) -> Kind { - self.kind - } - - /// Returns the number of tensors currently stored on this var-store. - pub fn len(&self) -> usize { - let variables = self.variables_.lock().unwrap(); - variables.named_variables.len() - } - - /// Returns true if no tensors are currently stored on this var-store. - pub fn is_empty(&self) -> bool { - let variables = self.variables_.lock().unwrap(); - variables.named_variables.is_empty() - } - - /// Returns all the trainable variables for this var-store. - pub fn trainable_variables(&self) -> Vec { - let variables = self.variables_.lock().unwrap(); - variables.trainable_variables.iter().map(|v| v.tensor.shallow_clone()).collect() - } - - /// Returns all variables along with their names. - pub fn variables(&self) -> HashMap { - let variables = self.variables_.lock().unwrap(); - variables - .named_variables - .iter() - .map(|(name, v)| (name.clone(), v.shallow_clone())) - .collect() - } - - /// Gets the root path for this variable store. - /// - /// Variables are named and organized using paths. This function returns - /// the top level path for the var store and can be combined with '/' - /// to create sub-paths. - pub fn root(&self) -> Path<'_> { - Path { path: vec![], group: 0, var_store: self } - } - - /// Saves the var-store variable values to a file. - /// - /// Weight values for all the tensors currently stored in the - /// var-store are saved in the given file. - /// - /// If the given path ends with the suffix `.safetensors`, the file will - /// be saved in safetensors format. Otherwise, libtorch C++ module format - /// will be used. Note that saving in pickle format (`.pt` extension) is - /// not supported by the C++ API of Torch. - pub fn save>(&self, path: T) -> Result<(), TchError> { - let variables = self.variables_.lock().unwrap(); - let named_tensors = variables.named_variables.iter().collect::>(); - match path.as_ref().extension().and_then(|x| x.to_str()) { - Some("safetensors") => Tensor::write_safetensors(named_tensors.as_slice(), path), - Some(_) | None => Tensor::save_multi(named_tensors.as_slice(), path), - } - } - - /// Saves the var-store variable values to a stream. - /// - /// Weight values for all the tensors currently stored in the - /// var-store gets saved in the given stream. - pub fn save_to_stream(&self, stream: W) -> Result<(), TchError> { - let variables = self.variables_.lock().unwrap(); - let named_tensors = variables.named_variables.iter().collect::>(); - Tensor::save_multi_to_stream(named_tensors.as_slice(), stream) - } - - fn named_tensors>( - &self, - path: T, - ) -> Result, TchError> { - let named_tensors = match path.as_ref().extension().and_then(|x| x.to_str()) { - Some("bin") | Some("pt") => Tensor::loadz_multi_with_device(&path, self.device), - Some("safetensors") => Tensor::read_safetensors(path), - Some(_) | None => Tensor::load_multi_with_device(&path, self.device), - }; - Ok(named_tensors?.into_iter().collect()) - } - - /// Copies the data from source tensor to destination - /// - /// Updates the precision of the destination to match the source - fn copy_data_with_precision_update(src: &Tensor, dst: &mut Tensor) -> Result<(), TchError> { - dst.set_data(&dst.to_kind(src.kind())); - dst.f_copy_(src) - } - - fn load_internal>(&mut self, path: T) -> Result<(), TchError> { - let named_tensors = self.named_tensors(&path)?; - let mut variables = self.variables_.lock().unwrap(); - for (name, var) in variables.named_variables.iter_mut() { - match named_tensors.get(name) { - Some(src) => crate::no_grad(|| { - Self::copy_data_with_precision_update(src, var) - .map_err(|e| e.path_context(name)) - })?, - None => { - return Err(TchError::TensorNameNotFound( - name.to_string(), - path.as_ref().to_string_lossy().into_owned(), - )); - } - } - } - Ok(()) - } - - /// Loads the var-store variable values from a file. - /// - /// Weight values for all the tensors currently stored in the - /// var-store are loaded from the given file. Note that the set of - /// variables stored in the var-store is not changed, only the values - /// for these tensors are modified. - /// - /// The format of the file is deduced from the file extension: - /// - `.safetensors`: The file is assumed to be in safetensors format. - /// - `.bin` or `.pt`: The file is assumed to be in pickle format. - /// - Otherwise, the file is assumed to be in libtorch C++ module format. - pub fn load>(&mut self, path: T) -> Result<(), TchError> { - if self.device != Device::Mps { - self.load_internal(path) - } else { - // Current workaround to allow loading in MPS device. - // On new libtorch releases check if direct loading becomes possible and revert - // See (https://github.com/LaurentMazare/tch-rs/issues/609#issuecomment-1427071598). - self.set_device(Device::Cpu); - let or_error = self.load_internal(path); - // Be cautious not to early exit so as to ensure that the device is set back to Mps - // even on errors. - self.set_device(Device::Mps); - or_error - } - } - - /// Loads the var-store variable values from a stream. - /// - /// Weight values for all the tensors currently stored in the - /// var-store gets loaded from the given stream. Note that the set of - /// variables stored in the var-store is not changed, only the values - /// for these tensors are modified. - pub fn load_from_stream(&mut self, stream: S) -> Result<(), TchError> { - let adapter = ReadSeekAdapter::new(stream); - let named_tensors = Tensor::load_multi_from_stream_with_device(adapter, self.device)?; - let named_tensors: HashMap<_, _> = named_tensors.into_iter().collect(); - let mut variables = self.variables_.lock().unwrap(); - for (name, var) in variables.named_variables.iter_mut() { - match named_tensors.get(name) { - Some(src) => crate::no_grad(|| { - Self::copy_data_with_precision_update(src, var) - .map_err(|e| e.path_context(name)) - })?, - None => { - return Err(TchError::TensorNameNotFound( - name.to_string(), - "source stream".to_string(), - )); - } - } - } - Ok(()) - } - - /// Loads the var-store variable values from a file if it exists. - /// - /// Weight values for the tensors currently stored in the var-store and the given file get - /// loaded from the given file. If a variable in the var store is not present in the given file, - /// it is skipped and its values are not updated. This method should be used if pre-trained - /// weight for only parts of the model are available. - /// Note that the set of variables stored in the var-store is not changed, only the values - /// for these tensors are modified. - /// - /// Returns a String Vector containing the names of missing variables. - pub fn load_partial>( - &mut self, - path: T, - ) -> Result, TchError> { - let named_tensors = self.named_tensors(&path)?; - let mut variables = self.variables_.lock().unwrap(); - let mut missing_variables = Vec::new(); - for (name, var) in variables.named_variables.iter_mut() { - match named_tensors.get(name) { - Some(src) => crate::no_grad(|| { - Self::copy_data_with_precision_update(src, var) - .map_err(|e| e.path_context(name)) - })?, - None => { - missing_variables.push(name.to_owned()); - } - } - } - Ok(missing_variables) - } - - /// Freezes a var store. - /// - /// Gradients for the variables in this store are not tracked - /// anymore. - pub fn freeze(&mut self) { - let variables = self.variables_.lock().unwrap(); - for variable in variables.trainable_variables.iter() { - let _v = variable.tensor.set_requires_grad(false); - } - } - - /// Unfreezes a var store. - /// - /// Gradients for the variables in this store are tracked again. - pub fn unfreeze(&mut self) { - let variables = self.variables_.lock().unwrap(); - for variable in variables.trainable_variables.iter() { - let _v = variable.tensor.set_requires_grad(true); - } - } - - /// Casts all variables in a var store to the target kind and sets the default kind - /// for new variables. - /// - /// For floating-point conversion, methods `half`, `bfloat16`, `float` and `double` - /// should be preferred as they ensure only float-like variables will be converted - /// to the target type. - pub fn set_kind(&mut self, kind: Kind) { - self.root().set_kind(kind); - self.kind = kind; - } - - /// Casts all float-like variable of a var store to half-precision (Half kind). - pub fn half(&mut self) { - self.root().half(); - } - - /// Casts all float-like variable of a var store to bfloat16-precision (BFloat16 kind). - pub fn bfloat16(&mut self) { - self.root().bfloat16(); - } - - /// Casts all float-like variable of a var store to single-precision (Float kind). - pub fn float(&mut self) { - self.root().float(); - } - - /// Casts all float-like variable of a var store to single-precision (Double kind). - pub fn double(&mut self) { - self.root().double(); - } - - /// Migrates a var store and all its tensor to a target device. - pub fn set_device(&mut self, device: Device) { - let mut variables = self.variables_.lock().unwrap(); - for (_, variable) in variables.named_variables.iter_mut() { - variable.set_data(&variable.to_device(device)); - } - self.device = device - } - - /// Copies variable values from a source var store to this var store. - /// - /// All the variables in this var store have to exist with the same - /// name in the source var store, otherwise an error is returned. - pub fn copy(&mut self, src: &VarStore) -> Result<(), TchError> { - let mut variables = self.variables_.lock().unwrap(); - let src_variables = src.variables_.lock().unwrap(); - let device = self.device; - for name in variables.named_variables.keys() { - if !src_variables.named_variables.contains_key(name) { - return Err(TchError::TensorNameNotFound( - name.to_string(), - "src var-store".to_string(), - )); - } - } - for (name, var) in variables.named_variables.iter_mut() { - let src_var = src_variables.named_variables.get(name).unwrap(); - crate::no_grad(|| var.f_copy_(&src_var.to_device(device)))?; - } - Ok(()) - } -} - -impl<'a> Path<'a> { - /// Get the components of the path. - pub fn components(&self) -> impl Iterator { - self.path.iter().map(String::as_str) - } - - /// Gets a sub-path of the given path. - pub fn sub(&self, s: T) -> Path<'a> { - let s = s.to_string(); - if s.chars().any(|x| x == SEP) { - panic!("sub name cannot contain {SEP} {s}"); - } - let mut path = self.path.clone(); - path.push(s); - Path { path, group: self.group, var_store: self.var_store } - } - - pub fn set_group(&self, group: usize) -> Path<'a> { - Path { path: self.path.clone(), group, var_store: self.var_store } - } - - /// Gets the device where the var-store variables are stored. - pub fn device(&self) -> Device { - self.var_store.device - } - - /// Gets the default kind of new variables - pub fn kind(&self) -> Kind { - self.var_store.kind - } - - pub fn path(&self, name: &str) -> String { - if name.chars().any(|x| x == SEP) { - panic!("variable name cannot contain {SEP} {name}"); - } - if self.path.is_empty() { - name.to_string() - } else { - format!("{}{}{}", self.path.join(&SEP.to_string()), SEP, name) - } - } - - /// Casts all variables in a var store sub-path to the target kind . - /// - /// Only the variable in the path sub-tree are cast to the target kind: - /// other var store variables are unaffected. For floating-point conversion, methods - /// `half`, `bfloat16`, `float` and `double` should be preferred as they ensure only - /// float-like variables will be converted to the target type. - pub fn set_kind(&mut self, kind: Kind) { - let path_root = self.path.join(SEP.to_string().as_str()); - let mut variables = self.var_store.variables_.lock().unwrap(); - for (variable_name, variable) in variables.named_variables.iter_mut() { - if variable_name.starts_with(&path_root) { - variable.set_data(&variable.to_kind(kind)); - } - } - } - - /// Casts all float-like variables in a var store sub-path to the target kind . - /// - /// Only the float-like variable in the path sub-tree are cast to the target kind: - /// other var store variables are unaffected - fn set_float_kind(&mut self, kind: Kind) { - let path_root = self.path.join(SEP.to_string().as_str()); - let mut variables = self.var_store.variables_.lock().unwrap(); - for (variable_name, variable) in variables.named_variables.iter_mut() { - if variable_name.starts_with(&path_root) & variable.is_floating_point() { - variable.set_data(&variable.to_kind(kind)); - } - } - } - - /// Casts all float-like variables in a var store sub-path to half-precision (Half kind). - /// - /// Only the variable in the path sub-tree are cast to half-precision: - /// other var store variables are unaffected - pub fn half(&mut self) { - self.set_float_kind(Kind::Half); - } - - /// Casts all float-like variables in a var store sub-path to bfloat16-precision (BFloat16 kind). - /// - /// Only the variable in the path sub-tree are cast to bfloat16-precision: - /// other var store variables are unaffected - pub fn bfloat16(&mut self) { - self.set_float_kind(Kind::BFloat16); - } - - /// Casts all float-like variables in a var store sub-path to single-precision (Float kind). - /// - /// Only the variable in the path sub-tree are cast to single-precision: - /// other var store variables are unaffected - pub fn float(&mut self) { - self.set_float_kind(Kind::Float); - } - - /// Casts all float-like variables in a var store sub-path to double-precision (Double kind). - /// - /// Only the variable in the path sub-tree are cast to double-precision: - /// other var store variables are unaffected - pub fn double(&mut self) { - self.set_float_kind(Kind::Double); - } - - pub fn add(&self, name: &str, tensor: Tensor, trainable: bool) -> Tensor { - let path = self.path(name); - let mut variables = self.var_store.variables_.lock().unwrap(); - let path = if variables.named_variables.contains_key(&path) { - format!("{}__{}", path, variables.named_variables.len()) - } else { - path - }; - let tensor = if trainable { tensor.set_requires_grad(true) } else { tensor }; - if trainable { - let var = Var { tensor: tensor.shallow_clone(), group: self.group }; - variables.trainable_variables.push(var); - }; - variables.named_variables.insert(path, tensor.shallow_clone()); - tensor - } - - fn get_or_add_with_lock( - &self, - name: &str, - tensor: Tensor, - trainable: bool, - mut variables: MutexGuard, - ) -> Tensor { - let path = self.path(name); - if let Some(var) = variables.named_variables.get(&path) { - return var.shallow_clone(); - } - - let tensor = if trainable { tensor.set_requires_grad(true) } else { tensor }; - if trainable { - let var = Var { tensor: tensor.shallow_clone(), group: self.group }; - variables.trainable_variables.push(var); - } - variables.named_variables.insert(path, tensor.shallow_clone()); - tensor - } - - /// Creates a new variable initialized with zeros. - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable will not be trainable so - /// gradients will not be tracked. - /// The variable uses a float tensor initialized with zeros. - pub fn f_zeros_no_train(&self, name: &str, dims: &[i64]) -> Result { - let z = Tensor::f_zeros(dims, (Kind::Float, self.device()))?; - Ok(self.add(name, z, false)) - } - - /// Creates a new variable initialized with ones. - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable will not be trainable so - /// gradients will not be tracked. - /// The variable uses a float tensor initialized with ones. - pub fn f_ones_no_train(&self, name: &str, dims: &[i64]) -> Result { - let o = Tensor::f_ones(dims, (Kind::Float, self.device()))?; - Ok(self.add(name, o, false)) - } - - /// Creates a new variable. - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable is trainable, its gradient - /// will be tracked. - /// The variable uses a float tensor initialized as per the - /// related argument. - pub fn f_var(&self, name: &str, dims: &[i64], init: Init) -> Result { - let v = super::f_init(init, dims, self.device(), self.kind())?; - Ok(self.add(name, v, true)) - } - - /// Creates a new variable initialized with zeros. - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable is trainable, its gradient - /// will be tracked. - /// The variable uses a float tensor initialized with zeros. - pub fn f_zeros(&self, name: &str, dims: &[i64]) -> Result { - self.f_var(name, dims, Init::Const(0.)) - } - - /// Creates a new variable initialized with ones. - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable is trainable, its gradient - /// will be tracked. - /// The variable uses a float tensor initialized with ones. - pub fn f_ones(&self, name: &str, dims: &[i64]) -> Result { - self.f_var(name, dims, Init::Const(1.)) - } - - /// Creates a new variable initialized randomly with normal distribution. - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable is trainable, its gradient - /// will be tracked. - /// The variable uses a float tensor initialized randomly using a - /// standard normal distribution. - pub fn f_randn_standard(&self, name: &str, dims: &[i64]) -> Result { - let init = Init::Randn { mean: 0., stdev: 1. }; - self.f_var(name, dims, init) - } - - /// Creates a new variable initialized randomly with normal distribution. - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable is trainable, its gradient - /// will be tracked. - /// The variable uses a float tensor initialized randomly using a - /// normal distribution with the specified mean and standard deviation. - pub fn f_randn( - &self, - name: &str, - dims: &[i64], - mean: f64, - stdev: f64, - ) -> Result { - self.f_var(name, dims, Init::Randn { mean, stdev }) - } - - /// Creates a new variable initialized randomly with uniform distribution. - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable is trainable, its gradient - /// will be tracked. - /// The variable uses a float tensor initialized randomly using a - /// uniform distribution between the specified bounds. - pub fn f_uniform( - &self, - name: &str, - dims: &[i64], - lo: f64, - up: f64, - ) -> Result { - self.f_var(name, dims, Init::Uniform { lo, up }) - } - - /// Creates a new variable initialized randomly with kaiming uniform. - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable is trainable, its gradient - /// will be tracked. - /// The variable uses a float tensor initialized randomly using a - /// uniform distribution which bounds follow Kaiming initialization. - pub fn f_kaiming_uniform(&self, name: &str, dims: &[i64]) -> Result { - self.f_var(name, dims, super::init::DEFAULT_KAIMING_UNIFORM) - } - - /// Creates a new variable initialized randomly with kaiming normal. - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable is trainable, its gradient - /// will be tracked. - /// The variable uses a float tensor initialized randomly using a - /// normal distribution which stdev follow Kaiming initialization. - pub fn f_kaiming_normal(&self, name: &str, dims: &[i64]) -> Result { - self.f_var(name, dims, super::init::DEFAULT_KAIMING_NORMAL) - } - - /// Creates a new variable initialized randomly with an orthogonal matrix - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable is trainable, its gradient - /// will be tracked. - /// The variable uses a float tensor initialized randomly with an orthogonal - /// matrix as described in *Exact solutions to the nonlinear dynamics - /// of learning in deep linear neural networks* - Saxe, A. et. al. (2013). - /// The input tensor must have at least 2 dimensions, and for tensors - /// with more than 2 dimensions the trailing dimensions are flattened. - pub fn f_orthogonal(&self, name: &str, dims: &[i64], gain: f64) -> Result { - self.f_var(name, dims, Init::Orthogonal { gain }) - } - - /// Creates a new variable initialized by copying an existing tensor. - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable is trainable, its gradient - /// will be tracked. - /// The variable uses a float tensor initialized by copying some - /// given tensor. - pub fn f_var_copy(&self, name: &str, t: &Tensor) -> Result { - let mut v = self.f_zeros(name, &t.size())?; - crate::no_grad(|| v.f_copy_(t))?; - Ok(v) - } - - /// Creates a new variable initialized with zeros. - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable will not be trainable so - /// gradients will not be tracked. - /// The variable uses a float tensor initialized with zeros. - pub fn zeros_no_train(&self, name: &str, dims: &[i64]) -> Tensor { - self.f_zeros_no_train(name, dims).unwrap() - } - - /// Creates a new variable initialized with ones. - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable will not be trainable so - /// gradients will not be tracked. - /// The variable uses a float tensor initialized with ones. - pub fn ones_no_train(&self, name: &str, dims: &[i64]) -> Tensor { - self.f_ones_no_train(name, dims).unwrap() - } - - /// Creates a new variable. - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable is trainable, its gradient - /// will be tracked. - /// The variable uses a float tensor initialized as per the - /// related argument. - pub fn var(&self, name: &str, dims: &[i64], init: Init) -> Tensor { - self.f_var(name, dims, init).unwrap() - } - - /// Creates a new variable initialized with zeros. - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable is trainable, its gradient - /// will be tracked. - /// The variable uses a float tensor initialized with zeros. - pub fn zeros(&self, name: &str, dims: &[i64]) -> Tensor { - self.f_zeros(name, dims).unwrap() - } - - /// Creates a new variable initialized with ones. - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable is trainable, its gradient - /// will be tracked. - /// The variable uses a float tensor initialized with ones. - pub fn ones(&self, name: &str, dims: &[i64]) -> Tensor { - self.f_ones(name, dims).unwrap() - } - - /// Creates a new variable initialized randomly with normal distribution. - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable is trainable, its gradient - /// will be tracked. - /// The variable uses a float tensor initialized randomly using a - /// standard normal distribution. - pub fn randn_standard(&self, name: &str, dims: &[i64]) -> Tensor { - self.f_randn_standard(name, dims).unwrap() - } - - /// Creates a new variable initialized randomly with normal distribution. - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable is trainable, its gradient - /// will be tracked. - /// The variable uses a float tensor initialized randomly using a - /// normal distribution with the specified mean and standard deviation. - pub fn randn(&self, name: &str, dims: &[i64], mean: f64, stdev: f64) -> Tensor { - self.f_randn(name, dims, mean, stdev).unwrap() - } - - /// Creates a new variable initialized randomly with uniform distribution. - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable is trainable, its gradient - /// will be tracked. - /// The variable uses a float tensor initialized randomly using a - /// uniform distribution between the specified bounds. - pub fn uniform(&self, name: &str, dims: &[i64], lo: f64, up: f64) -> Tensor { - self.f_uniform(name, dims, lo, up).unwrap() - } - - /// Creates a new variable initialized randomly with kaiming uniform. - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable is trainable, its gradient - /// will be tracked. - /// The variable uses a float tensor initialized randomly using a - /// uniform distribution which bounds follow Kaiming initialization. - pub fn kaiming_uniform(&self, name: &str, dims: &[i64]) -> Tensor { - self.f_kaiming_uniform(name, dims).unwrap() - } - - /// Creates a new variable initialized randomly with kaiming normal. - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable is trainable, its gradient - /// will be tracked. - /// The variable uses a float tensor initialized randomly using a - /// normal distribution which stdev follow Kaiming initialization. - pub fn kaiming_normal(&self, name: &str, dims: &[i64]) -> Tensor { - self.f_kaiming_normal(name, dims).unwrap() - } - - /// Creates a new variable initialized randomly with an orthogonal matrix - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable is trainable, its gradient - /// will be tracked. - /// The variable uses a float tensor initialized randomly with an orthogonal - /// matrix as described in *Exact solutions to the nonlinear dynamics - /// of learning in deep linear neural networks* - Saxe, A. et. al. (2013). - /// The input tensor must have at least 2 dimensions, and for tensors - /// with more than 2 dimensions the trailing dimensions are flattened. - pub fn orthogonal(&self, name: &str, dims: &[i64], gain: f64) -> Tensor { - self.f_orthogonal(name, dims, gain).unwrap() - } - - /// Creates a new variable initialized by copying an existing tensor. - /// - /// The new variable is named according to the name parameter and - /// has the specified shape. The variable is trainable, its gradient - /// will be tracked. - /// The variable uses a float tensor initialized by copying some - /// given tensor. - pub fn var_copy(&self, name: &str, t: &Tensor) -> Tensor { - self.f_var_copy(name, t).unwrap() - } - - /// Gets the tensor corresponding to a given name if present. - pub fn get(&self, name: &str) -> Option { - let path = self.path(name); - let variables = self.var_store.variables_.lock().unwrap(); - variables.named_variables.get(&path).map(|v| v.shallow_clone()) - } - - /// Gets the entry corresponding to a given name for in-place manipulation. - pub fn entry<'b>(&'b self, name: &'b str) -> Entry<'b> { - let variables = self.var_store.variables_.lock().unwrap(); - Entry { name, variables, path: self } - } -} - -impl Entry<'_> { - /// Returns the existing entry if, otherwise create a new variable. - /// - /// If this entry name matches the name of a variables stored in the - /// var store, the corresponding tensor is returned. Otherwise a new - /// variable is added to the var-store with the entry name and is - /// initialized according to the init parameter. - pub fn or_var(self, dims: &[i64], init: Init) -> Tensor { - let v = super::init(init, dims, self.path.device()); - self.path.get_or_add_with_lock(self.name, v, true, self.variables) - } - - /// Returns the existing entry if, otherwise create a new variable. - pub fn or_var_copy(self, tensor: &Tensor) -> Tensor { - let mut v = self.or_zeros(&tensor.size()); - crate::no_grad(|| v.copy_(tensor)); - v - } - - /// Returns the existing entry if, otherwise create a new variable. - pub fn or_kaiming_uniform(self, dims: &[i64]) -> Tensor { - self.or_var(dims, super::init::DEFAULT_KAIMING_NORMAL) - } - - /// Returns the existing entry if, otherwise create a new variable. - pub fn or_kaiming_normal(self, dims: &[i64]) -> Tensor { - self.or_var(dims, super::init::DEFAULT_KAIMING_NORMAL) - } - - /// Returns the existing entry if, otherwise create a new variable. - pub fn or_orthogonal(self, dims: &[i64], gain: f64) -> Tensor { - self.or_var(dims, Init::Orthogonal { gain }) - } - - /// Returns the existing entry if, otherwise create a new variable. - pub fn or_ones(self, dims: &[i64]) -> Tensor { - self.or_var(dims, Init::Const(1.)) - } - - /// Returns the existing entry if, otherwise create a new variable. - pub fn or_ones_no_train(self, dims: &[i64]) -> Tensor { - let o = Tensor::ones(dims, (Kind::Float, self.path.device())); - self.path.get_or_add_with_lock(self.name, o, true, self.variables) - } - - /// Returns the existing entry if, otherwise create a new variable. - pub fn or_randn(self, dims: &[i64], mean: f64, stdev: f64) -> Tensor { - self.or_var(dims, Init::Randn { mean, stdev }) - } - - /// Returns the existing entry if, otherwise create a new variable. - pub fn or_randn_standard(self, dims: &[i64]) -> Tensor { - let init = Init::Randn { mean: 0., stdev: 1. }; - self.or_var(dims, init) - } - - /// Returns the existing entry if, otherwise create a new variable. - pub fn or_uniform(self, dims: &[i64], lo: f64, up: f64) -> Tensor { - self.or_var(dims, Init::Uniform { lo, up }) - } - - /// Returns the existing entry if, otherwise create a new variable. - pub fn or_zeros(self, dims: &[i64]) -> Tensor { - self.or_var(dims, Init::Const(0.)) - } - - /// Returns the existing entry if, otherwise create a new variable. - pub fn or_zeros_no_train(self, dims: &[i64]) -> Tensor { - let z = Tensor::zeros(dims, (Kind::Float, self.path.device())); - self.path.get_or_add_with_lock(self.name, z, true, self.variables) - } -} - -impl<'a, T> Div for &mut Path<'a> -where - T: std::string::ToString, -{ - type Output = Path<'a>; - - fn div(self, rhs: T) -> Self::Output { - self.sub(rhs.to_string()) - } -} - -impl<'a, T> Div for &Path<'a> -where - T: std::string::ToString, -{ - type Output = Path<'a>; - - fn div(self, rhs: T) -> Self::Output { - self.sub(rhs.to_string()) - } -} - -impl<'a, T> Div for Path<'a> -where - T: std::string::ToString, -{ - type Output = Path<'a>; - - fn div(self, rhs: T) -> Self::Output { - self.sub(rhs.to_string()) - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/tensor/convert.rs b/v2/dependency-patches/tch-0.24.0/src/tensor/convert.rs deleted file mode 100644 index a0b8c3101f..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/tensor/convert.rs +++ /dev/null @@ -1,182 +0,0 @@ -//! Implement conversion traits for tensors -use super::Tensor; -use crate::{kind::Element, TchError}; -use half::{bf16, f16}; -use std::convert::{TryFrom, TryInto}; - -impl TryFrom<&Tensor> for Vec { - type Error = TchError; - fn try_from(tensor: &Tensor) -> Result { - let size = tensor.size(); - if size.len() != 1 { - Err(TchError::Convert(format!( - "Attempting to convert a Tensor with {} dimensions to flat vector", - size.len() - )))?; - } - let numel = size[0] as usize; - let mut vec = vec![T::ZERO; numel]; - tensor.f_to_kind(T::KIND)?.f_copy_data(&mut vec, numel)?; - Ok(vec) - } -} - -impl TryFrom<&Tensor> for Vec> { - type Error = TchError; - fn try_from(tensor: &Tensor) -> Result { - let (s1, s2) = tensor.size2()?; - let s1 = s1 as usize; - let s2 = s2 as usize; - let num_elem = s1 * s2; - // TODO: Try to remove this intermediary copy. - let mut all_elems = vec![T::ZERO; num_elem]; - tensor.f_to_kind(T::KIND)?.f_copy_data(&mut all_elems, num_elem)?; - let out = (0..s1).map(|i1| (0..s2).map(|i2| all_elems[i1 * s2 + i2]).collect()).collect(); - Ok(out) - } -} - -impl TryFrom<&Tensor> for Vec>> { - type Error = TchError; - fn try_from(tensor: &Tensor) -> Result { - let (s1, s2, s3) = tensor.size3()?; - let s1 = s1 as usize; - let s2 = s2 as usize; - let s3 = s3 as usize; - let num_elem = s1 * s2 * s3; - // TODO: Try to remove this intermediary copy. - let mut all_elems = vec![T::ZERO; num_elem]; - tensor.f_to_kind(T::KIND)?.f_copy_data(&mut all_elems, num_elem)?; - let out = (0..s1) - .map(|i1| { - (0..s2) - .map(|i2| (0..s3).map(|i3| all_elems[i1 * s2 * s3 + i2 * s3 + i3]).collect()) - .collect() - }) - .collect(); - Ok(out) - } -} - -impl TryFrom for Vec { - type Error = TchError; - fn try_from(tensor: Tensor) -> Result { - Vec::::try_from(&tensor) - } -} - -impl TryFrom for Vec> { - type Error = TchError; - fn try_from(tensor: Tensor) -> Result { - Vec::>::try_from(&tensor) - } -} - -impl TryFrom for Vec>> { - type Error = TchError; - fn try_from(tensor: Tensor) -> Result { - Vec::>>::try_from(&tensor) - } -} - -macro_rules! from_tensor { - ($typ:ident) => { - impl TryFrom<&Tensor> for $typ { - type Error = TchError; - - fn try_from(tensor: &Tensor) -> Result { - let numel = tensor.numel(); - if numel != 1 { - return Err(TchError::Convert(format!( - "expected exactly one element, got {}", - numel - ))); - } - let mut vec = [$typ::ZERO; 1]; - tensor - .f_to_device(crate::Device::Cpu)? - .f_to_kind($typ::KIND)? - .f_copy_data(&mut vec, numel)?; - Ok(vec[0]) - } - } - - impl TryFrom for $typ { - type Error = TchError; - - fn try_from(tensor: Tensor) -> Result { - $typ::try_from(&tensor) - } - } - }; -} - -from_tensor!(f64); -from_tensor!(f32); -from_tensor!(f16); -from_tensor!(i64); -from_tensor!(i32); -from_tensor!(i16); -from_tensor!(i8); -from_tensor!(u8); -from_tensor!(bool); -from_tensor!(bf16); - -impl TryInto> for &Tensor { - type Error = TchError; - - fn try_into(self) -> Result, Self::Error> { - let num_elem = self.numel(); - let mut vec = vec![T::ZERO; num_elem]; - self.f_to_kind(T::KIND)?.f_copy_data(&mut vec, num_elem)?; - let shape: Vec = self.size().iter().map(|s| *s as usize).collect(); - Ok(ndarray::ArrayD::from_shape_vec(ndarray::IxDyn(&shape), vec)?) - } -} - -impl TryFrom<&ndarray::ArrayBase> for Tensor -where - T: ndarray::Data, - T::Elem: Element, - D: ndarray::Dimension, -{ - type Error = TchError; - - fn try_from(value: &ndarray::ArrayBase) -> Result { - let slice = value - .as_slice() - .ok_or_else(|| TchError::Convert("cannot convert to slice".to_string()))?; - let tn = Self::f_from_slice(slice)?; - let shape: Vec = value.shape().iter().map(|s| *s as i64).collect(); - tn.f_reshape(shape) - } -} - -impl TryFrom> for Tensor -where - T: ndarray::Data, - T::Elem: Element, - D: ndarray::Dimension, -{ - type Error = TchError; - - fn try_from(value: ndarray::ArrayBase) -> Result { - Self::try_from(&value) - } -} - -impl TryFrom<&Vec> for Tensor { - type Error = TchError; - - fn try_from(value: &Vec) -> Result { - Self::f_from_slice(value.as_slice()) - } -} - -impl TryFrom> for Tensor { - type Error = TchError; - - fn try_from(value: Vec) -> Result { - Self::try_from(&value) - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/tensor/display.rs b/v2/dependency-patches/tch-0.24.0/src/tensor/display.rs deleted file mode 100644 index 572d68b0b1..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/tensor/display.rs +++ /dev/null @@ -1,434 +0,0 @@ -/// Pretty printing of tensors -/// This implementation should be in line with the PyTorch version. -/// https://github.com/pytorch/pytorch/blob/7b419e8513a024e172eae767e24ec1b849976b13/torch/_tensor_str.py -use crate::{Kind, Tensor}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum BasicKind { - Float, - Int, - Bool, - Complex, -} - -impl BasicKind { - fn for_tensor(t: &Tensor) -> BasicKind { - match t.f_kind() { - Err(_) => BasicKind::Complex, - Ok(kind) => match kind { - Kind::Int | Kind::Int8 | Kind::Uint8 | Kind::Int16 | Kind::Int64 => BasicKind::Int, - Kind::BFloat16 - | Kind::QInt8 - | Kind::QUInt8 - | Kind::QInt32 - | Kind::Half - | Kind::Float - | Kind::Double - | Kind::Float8e5m2 - | Kind::Float8e4m3fn - | Kind::Float8e5m2fnuz - | Kind::Float8e4m3fnuz => BasicKind::Float, - Kind::Bool => BasicKind::Bool, - Kind::ComplexHalf | Kind::ComplexFloat | Kind::ComplexDouble => BasicKind::Complex, - }, - } - } - - fn _is_floating_point(&self) -> bool { - match self { - BasicKind::Float => true, - BasicKind::Bool | BasicKind::Int | BasicKind::Complex => false, - } - } -} - -impl std::fmt::Debug for Tensor { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - if self.defined() { - match self.f_kind() { - Err(err) => write!(f, "Tensor[{:?}, {:?}]", self.size(), err), - Ok(kind) => { - let (is_int, is_float) = match kind { - Kind::Int | Kind::Int8 | Kind::Uint8 | Kind::Int16 | Kind::Int64 => { - (true, false) - } - Kind::BFloat16 - | Kind::QInt8 - | Kind::QUInt8 - | Kind::QInt32 - | Kind::Half - | Kind::Float - | Kind::Double - | Kind::Float8e5m2 - | Kind::Float8e4m3fn - | Kind::Float8e5m2fnuz - | Kind::Float8e4m3fnuz => (false, true), - Kind::Bool - | Kind::ComplexHalf - | Kind::ComplexFloat - | Kind::ComplexDouble => (false, false), - }; - match (self.size().as_slice(), is_int, is_float) { - ([], true, false) => write!(f, "[{}]", i64::try_from(self).unwrap()), - ([s], true, false) if *s < 10 => { - write!(f, "{:?}", Vec::::try_from(self).unwrap()) - } - ([], false, true) => write!(f, "[{}]", f64::try_from(self).unwrap()), - ([s], false, true) if *s < 10 => { - write!(f, "{:?}", Vec::::try_from(self).unwrap()) - } - _ => write!(f, "Tensor[{:?}, {:?}]", self.size(), kind), - } - } - } - } else { - write!(f, "Tensor[Undefined]") - } - } -} - -/// Options for Tensor pretty printing -pub struct PrinterOptions { - precision: usize, - threshold: usize, - edge_items: usize, - line_width: usize, - sci_mode: Option, -} - -lazy_static! { - static ref PRINT_OPTS: std::sync::Mutex = - std::sync::Mutex::new(Default::default()); -} - -pub fn set_print_options(options: PrinterOptions) { - *PRINT_OPTS.lock().unwrap() = options -} - -pub fn set_print_options_default() { - *PRINT_OPTS.lock().unwrap() = Default::default() -} - -pub fn set_print_options_short() { - *PRINT_OPTS.lock().unwrap() = PrinterOptions { - precision: 2, - threshold: 1000, - edge_items: 2, - line_width: 80, - sci_mode: None, - } -} - -pub fn set_print_options_full() { - *PRINT_OPTS.lock().unwrap() = PrinterOptions { - precision: 4, - threshold: usize::MAX, - edge_items: 3, - line_width: 80, - sci_mode: None, - } -} - -impl Default for PrinterOptions { - fn default() -> Self { - Self { precision: 4, threshold: 1000, edge_items: 3, line_width: 80, sci_mode: None } - } -} - -trait TensorFormatter { - type Elem; - - fn fmt(&self, v: Self::Elem, max_w: usize, f: &mut T) -> std::fmt::Result; - - fn value(tensor: &Tensor) -> Self::Elem; - - fn values(tensor: &Tensor) -> Vec; - - fn max_width(&self, to_display: &Tensor) -> usize { - let mut max_width = 1; - for v in Self::values(to_display) { - let mut fmt_size = FmtSize::new(); - let _res = self.fmt(v, 1, &mut fmt_size); - max_width = usize::max(max_width, fmt_size.final_size()) - } - max_width - } - - fn write_newline_indent(i: usize, f: &mut std::fmt::Formatter) -> std::fmt::Result { - writeln!(f)?; - for _ in 0..i { - write!(f, " ")? - } - Ok(()) - } - - fn fmt_tensor( - &self, - t: &Tensor, - indent: usize, - max_w: usize, - summarize: bool, - po: &PrinterOptions, - f: &mut std::fmt::Formatter, - ) -> std::fmt::Result { - let size = t.size(); - let edge_items = po.edge_items as i64; - write!(f, "[")?; - match size.as_slice() { - [] => self.fmt(Self::value(t), max_w, f)?, - [v] if summarize && *v > 2 * edge_items => { - for v in Self::values(&t.slice(0, None, Some(edge_items), 1)).into_iter() { - self.fmt(v, max_w, f)?; - write!(f, ", ")?; - } - write!(f, "...")?; - for v in Self::values(&t.slice(0, Some(-edge_items), None, 1)).into_iter() { - write!(f, ", ")?; - self.fmt(v, max_w, f)? - } - } - [_] => { - let elements_per_line = usize::max(1, po.line_width / (max_w + 2)); - for (i, v) in Self::values(t).into_iter().enumerate() { - if i > 0 { - if i % elements_per_line == 0 { - write!(f, ",")?; - Self::write_newline_indent(indent, f)? - } else { - write!(f, ", ")?; - } - } - self.fmt(v, max_w, f)? - } - } - _ => { - if summarize && size[0] > 2 * edge_items { - for i in 0..edge_items { - self.fmt_tensor(&t.get(i), indent + 1, max_w, summarize, po, f)?; - write!(f, ",")?; - Self::write_newline_indent(indent, f)? - } - write!(f, "...")?; - Self::write_newline_indent(indent, f)?; - for i in size[0] - edge_items..size[0] { - self.fmt_tensor(&t.get(i), indent + 1, max_w, summarize, po, f)?; - if i + 1 != size[0] { - write!(f, ",")?; - Self::write_newline_indent(indent, f)? - } - } - } else { - for i in 0..size[0] { - self.fmt_tensor(&t.get(i), indent + 1, max_w, summarize, po, f)?; - if i + 1 != size[0] { - write!(f, ",")?; - Self::write_newline_indent(indent, f)? - } - } - } - } - } - write!(f, "]")?; - Ok(()) - } -} - -struct FloatFormatter { - int_mode: bool, - sci_mode: bool, - precision: usize, -} - -struct FmtSize { - current_size: usize, -} - -impl FmtSize { - fn new() -> Self { - Self { current_size: 0 } - } - - fn final_size(self) -> usize { - self.current_size - } -} - -impl std::fmt::Write for FmtSize { - fn write_str(&mut self, s: &str) -> std::fmt::Result { - self.current_size += s.len(); - Ok(()) - } -} - -impl FloatFormatter { - fn new(t: &Tensor, po: &PrinterOptions) -> Self { - let mut int_mode = true; - let mut sci_mode = false; - - let _guard = crate::no_grad_guard(); - let t = t.to_device(crate::Device::Cpu); - - // Rather than containing all values, this should only include - // values that end up being displayed according to [threshold]. - let nonzero_finite_vals = { - let t = t.reshape([-1]); - t.masked_select(&t.isfinite().logical_and(&t.ne(0.))) - }; - - let values = Vec::::try_from(&nonzero_finite_vals).unwrap(); - if nonzero_finite_vals.numel() > 0 { - let nonzero_finite_abs = nonzero_finite_vals.abs(); - let nonzero_finite_min = nonzero_finite_abs.min().double_value(&[]); - let nonzero_finite_max = nonzero_finite_abs.max().double_value(&[]); - - for &value in values.iter() { - if value.ceil() != value { - int_mode = false; - break; - } - } - - sci_mode = nonzero_finite_max / nonzero_finite_min > 1000. - || nonzero_finite_max > 1e8 - || nonzero_finite_min < 1e-4 - } - - match po.sci_mode { - None => {} - Some(v) => sci_mode = v, - } - Self { int_mode, sci_mode, precision: po.precision } - } -} - -impl TensorFormatter for FloatFormatter { - type Elem = f64; - - fn fmt(&self, v: Self::Elem, max_w: usize, f: &mut T) -> std::fmt::Result { - if self.sci_mode { - write!(f, "{v:width$.prec$e}", v = v, width = max_w, prec = self.precision) - } else if self.int_mode { - if v.is_finite() { - write!(f, "{v:width$.0}.", v = v, width = max_w - 1) - } else { - write!(f, "{v:max_w$.0}") - } - } else { - write!(f, "{v:width$.prec$}", v = v, width = max_w, prec = self.precision) - } - } - - fn value(tensor: &Tensor) -> Self::Elem { - tensor.double_value(&[]) - } - - fn values(tensor: &Tensor) -> Vec { - Vec::::try_from(tensor.reshape(-1)).unwrap() - } -} - -struct IntFormatter; - -impl TensorFormatter for IntFormatter { - type Elem = i64; - - fn fmt(&self, v: Self::Elem, max_w: usize, f: &mut T) -> std::fmt::Result { - write!(f, "{v:max_w$}") - } - - fn value(tensor: &Tensor) -> Self::Elem { - tensor.int64_value(&[]) - } - - fn values(tensor: &Tensor) -> Vec { - Vec::::try_from(tensor.reshape(-1)).unwrap() - } -} - -struct BoolFormatter; - -impl TensorFormatter for BoolFormatter { - type Elem = bool; - - fn fmt(&self, v: Self::Elem, max_w: usize, f: &mut T) -> std::fmt::Result { - let v = if v { "true" } else { "false" }; - write!(f, "{v:max_w$}") - } - - fn value(tensor: &Tensor) -> Self::Elem { - tensor.int64_value(&[]) != 0 - } - - fn values(tensor: &Tensor) -> Vec { - Vec::::try_from(tensor.reshape(-1)).unwrap() - } -} - -fn get_summarized_data(t: &Tensor, edge_items: i64) -> Tensor { - let size = t.size(); - if size.is_empty() { - t.shallow_clone() - } else if size.len() == 1 { - if size[0] > 2 * edge_items { - Tensor::cat( - &[t.slice(0, None, Some(edge_items), 1), t.slice(0, Some(-edge_items), None, 1)], - 0, - ) - } else { - t.shallow_clone() - } - } else if size[0] > 2 * edge_items { - let mut vs: Vec<_> = - (0..edge_items).map(|i| get_summarized_data(&t.get(i), edge_items)).collect(); - for i in (size[0] - edge_items)..size[0] { - vs.push(get_summarized_data(&t.get(i), edge_items)) - } - Tensor::stack(&vs, 0) - } else { - let vs: Vec<_> = (0..size[0]).map(|i| get_summarized_data(&t.get(i), edge_items)).collect(); - Tensor::stack(&vs, 0) - } -} - -impl std::fmt::Display for Tensor { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - if self.defined() { - let po = PRINT_OPTS.lock().unwrap(); - let summarize = self.numel() > po.threshold; - let basic_kind = BasicKind::for_tensor(self); - let to_display = if summarize { - get_summarized_data(self, po.edge_items as i64) - } else { - self.shallow_clone() - }; - match basic_kind { - BasicKind::Int => { - let tf = IntFormatter; - let max_w = tf.max_width(&to_display); - tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; - writeln!(f)?; - } - BasicKind::Float => { - let tf = FloatFormatter::new(&to_display, &po); - let max_w = tf.max_width(&to_display); - tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; - writeln!(f)?; - } - BasicKind::Bool => { - let tf = BoolFormatter; - let max_w = tf.max_width(&to_display); - tf.fmt_tensor(self, 1, max_w, summarize, &po, f)?; - writeln!(f)?; - } - BasicKind::Complex => {} - }; - let kind = match self.f_kind() { - Ok(kind) => format!("{kind:?}"), - Err(err) => format!("{err:?}"), - }; - write!(f, "Tensor[{:?}, {}]", self.size(), kind) - } else { - write!(f, "Tensor[Undefined]") - } - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/tensor/index.rs b/v2/dependency-patches/tch-0.24.0/src/tensor/index.rs deleted file mode 100644 index 2bed77724d..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/tensor/index.rs +++ /dev/null @@ -1,372 +0,0 @@ -//! Indexing operations -//! -//! This module defines the `i` indexing operation. This can be used in various -//! scenarios. -//! -//! Using an integer index returns the slice obtained by selecting elements with -//! the specified index. Negative values can be used for the index, and `..` can -//! be used to get all the indexes from a given dimension. -//! -//! ```ignore -//! use crate::tch::{IndexOp, Tensor}; -//! let tensor = Tensor::from_slice(&[1, 2, 3, 4, 5, 6]).view((2, 3)); -//! let t = tensor.i(1); -//! let t = tensor.i((.., -2)); -//! ``` -//! -//! Indexes like `1..`, `..1`, or `1..2`, can be used to narrow a dimension. -//! -//! ```ignore -//! use crate::tch::{IndexOp, Tensor}; -//! let tensor = Tensor::from_slice(&[1, 2, 3, 4, 5, 6]).view((2, 3)); -//! let t = tensor.i((.., 1..)); -//! assert_eq!(t.size(), [2, 2]); -//! assert_eq!(Vec::::from(t.contiguous().view(-1)), [2, 3, 5, 6]); -//! let t = tensor.i((..1, ..)); -//! assert_eq!(t.size(), [1, 3]); -//! assert_eq!(Vec::::from(t.contiguous().view(-1)), [1, 2, 3]); -//! let t = tensor.i((.., 1..2)); -//! assert_eq!(t.size(), [2, 1]); -//! assert_eq!(Vec::::from(t.contiguous().view(-1)), [2, 5]); -//! let t = tensor.i((.., 1..=2)); -//! assert_eq!(t.size(), [2, 2]); -//! assert_eq!(Vec::::from(t.contiguous().view(-1)), [2, 3, 5, 6]); -//! ``` -//! -//! The `NewAxis` index can be used to insert a dimension. -//! -//! ```ignore -//! use crate::tch::{IndexOp, NewAxis, Tensor}; -//! let tensor = Tensor::from_slice(&[1, 2, 3, 4, 5, 6]).view((2, 3)); -//! let t = tensor.i((NewAxis,)); -//! assert_eq!(t.size(), &[1, 2, 3]); -//! let t = tensor.i((.., .., NewAxis)); -//! assert_eq!(t.size(), &[2, 3, 1]); -//! ``` -//! -//! Unlike NumPy, the `i` operation does not support advanced indexing. -//! The result can be different from NumPy with same set of arguments. -//! For example, `tensor.i(..1, vec![0, 3], vec![2, 1, 3])` does narrowing -//! on first dimension, and index selection on second and third dimensions. -//! The analogous NumPy indexing `array[:1, [0, 3], [2, 1, 3]]` throws -//! shape mismatch error due to advanced indexing rule. Another distinction -//! is that `i` guarantees the input and result tensor shares the same -//! underlying storage, while NumPy may copy the tensor in certain scenarios. -use crate::{Result, TchError, Tensor}; -use std::ops::{ - Bound, Range, RangeBounds, RangeFrom, RangeFull, RangeInclusive, RangeTo, RangeToInclusive, -}; - -#[derive(Debug, PartialEq, Eq)] -pub struct NewAxis; - -#[derive(Debug, PartialEq)] -pub enum TensorIndexer { - Select(i64), - Narrow(Bound, Bound), - IndexSelect(Tensor), - InsertNewAxis, -} - -impl From for TensorIndexer { - fn from(_index: NewAxis) -> Self { - TensorIndexer::InsertNewAxis - } -} - -impl From for TensorIndexer { - fn from(index: i64) -> Self { - TensorIndexer::Select(index) - } -} - -impl From<&[i64]> for TensorIndexer { - fn from(index: &[i64]) -> Self { - let tensor = index.into(); - TensorIndexer::IndexSelect(tensor) - } -} - -impl From> for TensorIndexer { - fn from(index: Vec) -> Self { - let tensor = Tensor::from_slice(&index); - TensorIndexer::IndexSelect(tensor) - } -} - -impl From<&Tensor> for TensorIndexer { - fn from(tensor: &Tensor) -> Self { - TensorIndexer::IndexSelect(tensor.shallow_clone()) - } -} - -macro_rules! impl_from_range { - ($range_type:ty) => { - impl From<$range_type> for TensorIndexer { - fn from(range: $range_type) -> Self { - use std::ops::Bound::*; - - let start = match range.start_bound() { - Included(idx) => Included(*idx), - Excluded(idx) => Excluded(*idx), - Unbounded => Unbounded, - }; - - let end = match range.end_bound() { - Included(idx) => Included(*idx), - Excluded(idx) => Excluded(*idx), - Unbounded => Unbounded, - }; - - TensorIndexer::Narrow(start, end) - } - } - }; -} - -impl_from_range!(Range); -impl_from_range!(RangeFrom); -impl_from_range!(RangeFull); -impl_from_range!(RangeInclusive); -impl_from_range!(RangeTo); -impl_from_range!(RangeToInclusive); - -pub trait IndexOp { - fn i(&self, index: T) -> Tensor; - fn f_i(&self, index: T) -> Result; -} - -impl IndexOp for Tensor -where - A: Into, -{ - fn i(&self, index: A) -> Tensor { - self.f_i(index).unwrap() - } - - fn f_i(&self, index: A) -> Result { - self.f_indexer(&[index.into()]) - } -} - -impl IndexOp<(A,)> for Tensor -where - A: Into, -{ - fn i(&self, index: (A,)) -> Tensor { - self.f_i(index).unwrap() - } - - fn f_i(&self, index: (A,)) -> Result { - let idx_a = index.0.into(); - self.f_indexer(&[idx_a]) - } -} - -impl IndexOp<(A, B)> for Tensor -where - A: Into, - B: Into, -{ - fn i(&self, index: (A, B)) -> Tensor { - self.f_i(index).unwrap() - } - - fn f_i(&self, index: (A, B)) -> Result { - let idx_a = index.0.into(); - let idx_b = index.1.into(); - self.f_indexer(&[idx_a, idx_b]) - } -} - -impl IndexOp<(A, B, C)> for Tensor -where - A: Into, - B: Into, - C: Into, -{ - fn i(&self, index: (A, B, C)) -> Tensor { - self.f_i(index).unwrap() - } - - fn f_i(&self, index: (A, B, C)) -> Result { - let idx_a = index.0.into(); - let idx_b = index.1.into(); - let idx_c = index.2.into(); - self.f_indexer(&[idx_a, idx_b, idx_c]) - } -} - -impl IndexOp<(A, B, C, D)> for Tensor -where - A: Into, - B: Into, - C: Into, - D: Into, -{ - fn i(&self, index: (A, B, C, D)) -> Tensor { - self.f_i(index).unwrap() - } - - fn f_i(&self, index: (A, B, C, D)) -> Result { - let idx_a = index.0.into(); - let idx_b = index.1.into(); - let idx_c = index.2.into(); - let idx_d = index.3.into(); - self.f_indexer(&[idx_a, idx_b, idx_c, idx_d]) - } -} - -impl IndexOp<(A, B, C, D, E)> for Tensor -where - A: Into, - B: Into, - C: Into, - D: Into, - E: Into, -{ - fn i(&self, index: (A, B, C, D, E)) -> Tensor { - self.f_i(index).unwrap() - } - - fn f_i(&self, index: (A, B, C, D, E)) -> Result { - let idx_a = index.0.into(); - let idx_b = index.1.into(); - let idx_c = index.2.into(); - let idx_d = index.3.into(); - let idx_e = index.4.into(); - self.f_indexer(&[idx_a, idx_b, idx_c, idx_d, idx_e]) - } -} - -impl IndexOp<(A, B, C, D, E, F)> for Tensor -where - A: Into, - B: Into, - C: Into, - D: Into, - E: Into, - F: Into, -{ - fn i(&self, index: (A, B, C, D, E, F)) -> Tensor { - self.f_i(index).unwrap() - } - - fn f_i(&self, index: (A, B, C, D, E, F)) -> Result { - let idx_a = index.0.into(); - let idx_b = index.1.into(); - let idx_c = index.2.into(); - let idx_d = index.3.into(); - let idx_e = index.4.into(); - let idx_f = index.5.into(); - self.f_indexer(&[idx_a, idx_b, idx_c, idx_d, idx_e, idx_f]) - } -} - -impl IndexOp<(A, B, C, D, E, F, G)> for Tensor -where - A: Into, - B: Into, - C: Into, - D: Into, - E: Into, - F: Into, - G: Into, -{ - fn i(&self, index: (A, B, C, D, E, F, G)) -> Tensor { - self.f_i(index).unwrap() - } - - fn f_i(&self, index: (A, B, C, D, E, F, G)) -> Result { - let idx_a = index.0.into(); - let idx_b = index.1.into(); - let idx_c = index.2.into(); - let idx_d = index.3.into(); - let idx_e = index.4.into(); - let idx_f = index.5.into(); - let idx_g = index.6.into(); - self.f_indexer(&[idx_a, idx_b, idx_c, idx_d, idx_e, idx_f, idx_g]) - } -} - -impl Tensor { - fn f_indexer(&self, index_spec: &[TensorIndexer]) -> Result { - use std::ops::Bound::*; - use TensorIndexer::*; - - // Make sure n. non-newaxis does not exceed n. of dimensions - let n_newaxis = index_spec.iter().filter(|spec| *spec == &InsertNewAxis).count(); - - if index_spec.len() > self.size().len() + n_newaxis { - return Err(TchError::Shape(format!( - "too many indices for tensor of dimension {}", - self.size().len() - ))); - } - - // Make sure tensors conform the format - for spec in index_spec.iter() { - use super::Kind::*; - if let IndexSelect(tensor) = spec { - if tensor.size().len() != 1 { - return Err(TchError::Shape( - "Multi-dimensional tensor is not supported for indexing".to_string(), - )); - } - match tensor.f_kind()? { - Int64 => {} - Int16 => {} - Int8 => {} - Int => {} - _ => { - return Err(TchError::Kind(format!("the kind of tensors used as indices must be one of {Int64:?}, {Int16:?}, {Int8:?}, {Int:?}"))); - } - } - } - } - - // Apply indexing from left to right - let mut curr_tensor = self.shallow_clone(); - let mut curr_idx: i64 = 0; - - for spec in index_spec.iter() { - let (next_tensor, next_idx) = match spec { - InsertNewAxis => (curr_tensor.unsqueeze(curr_idx), curr_idx + 1), - Select(index) => ( - curr_tensor.select(curr_idx, *index), - curr_idx, // not advanced because select() squeezes dimension - ), - Narrow(start, end) => { - if let Some((start, length)) = match (start, end) { - (Unbounded, Unbounded) => None, - (Included(start), Unbounded) => { - let dim_len = curr_tensor.size()[curr_idx as usize]; - Some((*start, dim_len - *start)) - } - (Excluded(start), Unbounded) => { - let dim_len = curr_tensor.size()[curr_idx as usize]; - Some((*start + 1, dim_len - *start - 1)) - } - (Unbounded, Included(end)) => Some((0, *end + 1)), - (Unbounded, Excluded(end)) => Some((0, *end)), - (Included(start), Included(end)) => Some((*start, *end - *start + 1)), - (Included(start), Excluded(end)) => Some((*start, *end - *start)), - (Excluded(start), Included(end)) => Some((*start + 1, *end - *start)), - (Excluded(start), Excluded(end)) => Some((*start + 1, *end - *start - 1)), - } { - (curr_tensor.f_narrow(curr_idx, start, length.max(0))?, curr_idx + 1) - } else { - (curr_tensor, curr_idx + 1) - } - } - IndexSelect(index_tensor) => { - let index_tensor = index_tensor.to_device(curr_tensor.device()); - (curr_tensor.index_select(curr_idx, &index_tensor), curr_idx + 1) - } - }; - curr_tensor = next_tensor; - curr_idx = next_idx; - } - Ok(curr_tensor) - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/tensor/iter.rs b/v2/dependency-patches/tch-0.24.0/src/tensor/iter.rs deleted file mode 100644 index bf0efdd8bd..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/tensor/iter.rs +++ /dev/null @@ -1,62 +0,0 @@ -use super::Tensor; -use crate::TchError; - -pub struct Iter { - index: i64, - len: i64, - content: Tensor, - phantom: std::marker::PhantomData, -} - -impl Tensor { - pub fn iter(&self) -> Result, TchError> { - Ok(Iter { - index: 0, - len: self.size1()?, - content: self.shallow_clone(), - phantom: std::marker::PhantomData, - }) - } -} - -impl std::iter::Iterator for Iter { - type Item = i64; - fn next(&mut self) -> Option { - if self.index >= self.len { - return None; - } - let v = self.content.int64_value(&[self.index]); - self.index += 1; - Some(v) - } -} - -impl std::iter::Iterator for Iter { - type Item = f64; - fn next(&mut self) -> Option { - if self.index >= self.len { - return None; - } - let v = self.content.double_value(&[self.index]); - self.index += 1; - Some(v) - } -} - -impl std::iter::Sum for Tensor { - fn sum>(mut iter: I) -> Tensor { - match iter.next() { - None => Tensor::from(0.), - Some(t) => iter.fold(t, |acc, x| x + acc), - } - } -} - -impl<'a> std::iter::Sum<&'a Tensor> for Tensor { - fn sum>(mut iter: I) -> Tensor { - match iter.next() { - None => Tensor::from(0.), - Some(t) => iter.fold(t.shallow_clone(), |acc, x| x + acc), - } - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/tensor/mod.rs b/v2/dependency-patches/tch-0.24.0/src/tensor/mod.rs deleted file mode 100644 index 839e9d18a1..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/tensor/mod.rs +++ /dev/null @@ -1,253 +0,0 @@ -//! A Torch tensor. -use crate::{Device, Kind, TchError}; - -mod convert; -pub mod display; -pub mod index; -mod iter; -mod npy; -mod ops; -mod safetensors; - -pub use super::wrappers::tensor::{ - autocast, no_grad, no_grad_guard, with_grad, NoGradGuard, Reduction, Tensor, -}; -pub use index::{IndexOp, NewAxis, TensorIndexer}; - -pub trait Shape { - fn to_shape(&self) -> Box<[i64]>; -} - -macro_rules! impl_shape { - ($v:expr) => { - impl Shape for [i64; $v] { - fn to_shape(&self) -> Box<[i64]> { - Box::new(*self) - } - } - }; -} - -impl_shape!(0); -impl_shape!(1); -impl_shape!(2); -impl_shape!(3); -impl_shape!(4); -impl_shape!(5); -impl_shape!(6); - -impl Shape for () { - fn to_shape(&self) -> Box<[i64]> { - Box::new([]) - } -} - -impl Shape for &[i64] { - fn to_shape(&self) -> Box<[i64]> { - (*self).into() - } -} - -impl Shape for i64 { - fn to_shape(&self) -> Box<[i64]> { - Box::new([*self]) - } -} - -impl Shape for usize { - fn to_shape(&self) -> Box<[i64]> { - Box::new([*self as i64]) - } -} - -impl Shape for i32 { - fn to_shape(&self) -> Box<[i64]> { - Box::new([i64::from(*self)]) - } -} - -impl Shape for (i64,) { - fn to_shape(&self) -> Box<[i64]> { - Box::new([self.0]) - } -} - -impl Shape for (i64, i64) { - fn to_shape(&self) -> Box<[i64]> { - Box::new([self.0, self.1]) - } -} - -impl Shape for (i64, i64, i64) { - fn to_shape(&self) -> Box<[i64]> { - Box::new([self.0, self.1, self.2]) - } -} - -impl Shape for (i64, i64, i64, i64) { - fn to_shape(&self) -> Box<[i64]> { - Box::new([self.0, self.1, self.2, self.3]) - } -} - -impl Tensor { - pub fn f_view(&self, s: T) -> Result { - self.f_view_(&s.to_shape()) - } - - pub fn view(&self, s: T) -> Tensor { - self.view_(&s.to_shape()) - } - - pub fn f_zero_pad1d(&self, left: i64, right: i64) -> Result { - if self.dim() != 3 { - return Err(TchError::Shape(format!( - "expected a 3 dimension tensor, got {:?}", - self.size() - ))); - } - self.f_constant_pad_nd([left, right]) - } - - pub fn zero_pad1d(&self, left: i64, right: i64) -> Tensor { - self.f_zero_pad1d(left, right).unwrap() - } - - pub fn f_zero_pad2d( - &self, - left: i64, - right: i64, - top: i64, - bottom: i64, - ) -> Result { - if self.dim() != 4 { - return Err(TchError::Shape(format!( - "expected a 4 dimension tensor, got {:?}", - self.size() - ))); - } - self.f_constant_pad_nd([left, right, top, bottom]) - } - - pub fn zero_pad2d(&self, left: i64, right: i64, top: i64, bottom: i64) -> Tensor { - self.f_zero_pad2d(left, right, top, bottom).unwrap() - } -} - -impl From<&[T]> for Tensor { - fn from(v: &[T]) -> Tensor { - Tensor::from_slice(v) - } -} - -impl From for Tensor { - fn from(v: T) -> Tensor { - Tensor::from_slice(&[v]).view(()) - } -} -impl Tensor { - /// Casts a tensor to a specified kind. - pub fn to_kind(&self, kind: Kind) -> Tensor { - self.totype(kind) - } - - pub fn f_to_kind(&self, kind: Kind) -> Result { - self.f_totype(kind) - } - - pub fn nll_loss(&self, targets: &Tensor) -> Tensor { - self.g_nll_loss::(targets, None, Reduction::Mean, -100) - } -} - -impl Tensor { - /// Computes the cross-entropy loss based on some logits and targets. - pub fn cross_entropy_for_logits(&self, targets: &Tensor) -> Tensor { - self.log_softmax(-1, Kind::Float).nll_loss(targets) - } - - /// Returns the average accuracy for some given logits assuming that - /// targets represent ground-truth. - pub fn accuracy_for_logits(&self, targets: &Tensor) -> Tensor { - self.argmax(-1, false).eq_tensor(targets).to_kind(Kind::Float).mean(Kind::Float) - } - - pub fn random_batch(&self, batch_size: i64) -> Tensor { - let len: i64 = self.size()[0]; - let index = Tensor::randint(len, [batch_size], (Kind::Int64, self.device())); - self.index_select(0, &index) - } - - pub fn random_batch2(t1: &Tensor, t2: &Tensor, batch_size: i64) -> (Tensor, Tensor) { - let len1: i64 = t1.size()[0]; - let len2: i64 = t2.size()[0]; - if len1 != len2 { - panic!("random_batch2: shape mismatch {:?} {:?}", t1.size(), t2.size()) - } - let device1 = t1.device(); - let device2 = t2.device(); - if device1 != device2 { - panic!("random_batch2: device mismatch {device1:?} {device2:?}") - } - let index = Tensor::randint(len1, [batch_size], (Kind::Int64, device1)); - let batch1 = t1.index_select(0, &index); - let batch2 = t2.index_select(0, &index); - (batch1, batch2) - } - - /// Moves a tensor to a specified device. - pub fn to_device(&self, device: Device) -> Tensor { - self.to(device) - } - - pub fn f_to_device(&self, device: Device) -> Result { - self.f_to(device) - } - - pub fn avg_pool2d_default(&self, ksize: i64) -> Tensor { - self.avg_pool2d([ksize, ksize], [ksize, ksize], [0, 0], false, true, 1) - } - - pub fn max_pool2d_default(&self, ksize: i64) -> Tensor { - self.max_pool2d([ksize, ksize], [ksize, ksize], [0, 0], [1, 1], false) - } - - /// Flattens a tensor. - /// - /// This returns a flattened version of the given tensor. The first dimension - /// is preserved as it is assumed to be the mini-batch dimension. - pub fn flat_view(&self) -> Tensor { - self.view((self.size()[0], -1)) - } - - /// Converts a tensor to a one-hot encoded version. - /// - /// If the input has a size [N1, N2, ..., Nk], the returned tensor has a size - /// [N1, ..., Nk, labels]. The returned tensor uses float values. - /// Elements of the input vector are expected to be between 0 and labels-1. - pub fn onehot(&self, labels: i64) -> Tensor { - Tensor::zeros([self.size(), vec![labels]].concat(), (Kind::Float, self.device())) - .scatter_value_(-1, &self.unsqueeze(-1).to_kind(Kind::Int64), 1.0) - } - - /// Copies a tensor to a newly allocated tensor using the same shape and device. - pub fn copy(&self) -> Tensor { - let mut result = self.zeros_like(); - result.copy_(self); - result - } - - /// Copies the data from a two dimensional slice in a tensor object. - pub fn from_slice2(v: &[U]) -> Tensor - where - T: crate::kind::Element, - U: AsRef<[T]>, - { - let inner: Vec = v.iter().map(|v| Tensor::from_slice(v.as_ref())).collect(); - Tensor::stack(&inner, 0) - } - - pub fn to_mkldnn(&self) -> Tensor { - self.g_to_mkldnn(self.kind()) - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/tensor/npy.rs b/v2/dependency-patches/tch-0.24.0/src/tensor/npy.rs deleted file mode 100644 index 855d834cd5..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/tensor/npy.rs +++ /dev/null @@ -1,277 +0,0 @@ -//! Numpy support for tensors. -//! -//! This module implements the support for reading and writing `.npy` and `.npz` files. The file -//! format spec can be found at: -//! . -use crate::{Kind, TchError, Tensor}; -use std::collections::HashMap; -use std::fs::File; -use std::io::{BufReader, Read, Write}; -use std::path::Path; - -const NPY_MAGIC_STRING: &[u8] = b"\x93NUMPY"; -const NPY_SUFFIX: &str = ".npy"; - -fn read_header(reader: &mut R) -> Result { - let mut magic_string = vec![0u8; NPY_MAGIC_STRING.len()]; - reader.read_exact(&mut magic_string)?; - if magic_string != NPY_MAGIC_STRING { - return Err(TchError::FileFormat("magic string mismatch".to_string())); - } - let mut version = [0u8; 2]; - reader.read_exact(&mut version)?; - let header_len_len = match version[0] { - 1 => 2, - 2 => 4, - otherwise => return Err(TchError::FileFormat(format!("unsupported version {otherwise}"))), - }; - let mut header_len = vec![0u8; header_len_len]; - reader.read_exact(&mut header_len)?; - let header_len = header_len.iter().rev().fold(0_usize, |acc, &v| 256 * acc + v as usize); - let mut header = vec![0u8; header_len]; - reader.read_exact(&mut header)?; - Ok(String::from_utf8_lossy(&header).to_string()) -} - -#[derive(Debug, PartialEq)] -struct Header { - descr: Kind, - fortran_order: bool, - shape: Vec, -} - -impl Header { - fn to_string(&self) -> Result { - let fortran_order = if self.fortran_order { "True" } else { "False" }; - let mut shape = self.shape.iter().map(|x| x.to_string()).collect::>().join(","); - let descr = match self.descr { - Kind::Half => "f2", - Kind::Float => "f4", - Kind::Double => "f8", - Kind::Int => "i4", - Kind::Int64 => "i8", - Kind::Int16 => "i2", - Kind::Int8 => "i1", - Kind::Uint8 => "u1", - descr => return Err(TchError::FileFormat(format!("unsupported kind {descr:?}"))), - }; - if !shape.is_empty() { - shape.push(',') - } - Ok(format!( - "{{'descr': '<{descr}', 'fortran_order': {fortran_order}, 'shape': ({shape}), }}" - )) - } - - // Hacky parser for the npy header, a typical example would be: - // {'descr': ' Result { - let header = - header.trim_matches(|c: char| c == '{' || c == '}' || c == ',' || c.is_whitespace()); - - let mut parts: Vec = vec![]; - let mut start_index = 0usize; - let mut cnt_parenthesis = 0i64; - for (index, c) in header.char_indices() { - match c { - '(' => cnt_parenthesis += 1, - ')' => cnt_parenthesis -= 1, - ',' => { - if cnt_parenthesis == 0 { - parts.push(header[start_index..index].to_owned()); - start_index = index + 1; - } - } - _ => {} - } - } - parts.push(header[start_index..].to_owned()); - let mut part_map: HashMap = HashMap::new(); - for part in parts.iter() { - let part = part.trim(); - if !part.is_empty() { - match part.split(':').collect::>().as_slice() { - [key, value] => { - let key = key.trim_matches(|c: char| c == '\'' || c.is_whitespace()); - let value = value.trim_matches(|c: char| c == '\'' || c.is_whitespace()); - let _ = part_map.insert(key.to_owned(), value.to_owned()); - } - _ => { - return Err(TchError::FileFormat(format!( - "unable to parse header {header}" - ))) - } - } - } - } - let fortran_order = match part_map.get("fortran_order") { - None => false, - Some(fortran_order) => match fortran_order.as_ref() { - "False" => false, - "True" => true, - _ => { - return Err(TchError::FileFormat(format!( - "unknown fortran_order {fortran_order}" - ))) - } - }, - }; - let descr = match part_map.get("descr") { - None => return Err(TchError::FileFormat("no descr in header".to_string())), - Some(descr) => { - if descr.is_empty() { - return Err(TchError::FileFormat("empty descr".to_string())); - } - if descr.starts_with('>') { - return Err(TchError::FileFormat(format!("little-endian descr {descr}"))); - } - // the only supported types in tensor are: - // float64, float32, float16, - // complex64, complex128, - // int64, int32, int16, int8, - // uint8, and bool. - match descr.trim_matches(|c: char| c == '=' || c == '<' || c == '|') { - "e" | "f2" => Kind::Half, - "f" | "f4" => Kind::Float, - "d" | "f8" => Kind::Double, - "i" | "i4" => Kind::Int, - "q" | "i8" => Kind::Int64, - "h" | "i2" => Kind::Int16, - "b" | "i1" => Kind::Int8, - "B" | "u1" => Kind::Uint8, - "?" | "b1" => Kind::Bool, - "F" | "F4" | "c8" => Kind::ComplexFloat, - "D" | "F8" | "c16" => Kind::ComplexDouble, - descr => { - return Err(TchError::FileFormat(format!("unrecognized descr {descr}"))) - } - } - } - }; - let shape = match part_map.get("shape") { - None => return Err(TchError::FileFormat("no shape in header".to_string())), - Some(shape) => { - let shape = shape.trim_matches(|c: char| c == '(' || c == ')' || c == ','); - if shape.is_empty() { - vec![] - } else { - shape - .split(',') - .map(|v| v.trim().parse::()) - .collect::, _>>()? - } - } - }; - Ok(Header { descr, fortran_order, shape }) - } -} - -impl crate::Tensor { - /// Reads a npy file and return the stored tensor. - pub fn read_npy>(path: T) -> Result { - let mut reader = File::open(path.as_ref())?; - let header = read_header(&mut reader)?; - let header = Header::parse(&header)?; - if header.fortran_order { - return Err(TchError::FileFormat("fortran order not supported".to_string())); - } - let mut data: Vec = vec![]; - reader.read_to_end(&mut data)?; - Tensor::f_from_data_size(&data, &header.shape, header.descr) - } - - /// Reads a npz file and returns some named tensors. - pub fn read_npz>(path: T) -> Result, TchError> { - let zip_reader = BufReader::new(File::open(path.as_ref())?); - let mut zip = zip::ZipArchive::new(zip_reader)?; - let mut result = vec![]; - for i in 0..zip.len() { - let mut reader = zip.by_index(i).unwrap(); - let name = { - let name = reader.name()?; - name.strip_suffix(NPY_SUFFIX).unwrap_or(&name).to_owned() - }; - let header = read_header(&mut reader)?; - let header = Header::parse(&header)?; - if header.fortran_order { - return Err(TchError::FileFormat("fortran order not supported".to_string())); - } - let mut data: Vec = vec![]; - reader.read_to_end(&mut data)?; - let tensor = Tensor::f_from_data_size(&data, &header.shape, header.descr)?; - result.push((name, tensor)) - } - Ok(result) - } - - fn write(&self, f: &mut T) -> Result<(), TchError> { - f.write_all(NPY_MAGIC_STRING)?; - f.write_all(&[1u8, 0u8])?; - let kind = self.f_kind()?; - let header = Header { descr: kind, fortran_order: false, shape: self.size() }; - let mut header = header.to_string()?; - let pad = 16 - (NPY_MAGIC_STRING.len() + 5 + header.len()) % 16; - for _ in 0..pad % 16 { - header.push(' ') - } - header.push('\n'); - f.write_all(&[(header.len() % 256) as u8, (header.len() / 256) as u8])?; - f.write_all(header.as_bytes())?; - let numel = self.numel(); - let mut content = vec![0u8; numel * kind.elt_size_in_bytes()]; - self.f_copy_data_u8(&mut content, numel)?; - f.write_all(&content)?; - Ok(()) - } - - /// Writes a tensor in the npy format so that it can be read using python. - pub fn write_npy>(&self, path: T) -> Result<(), TchError> { - let mut f = File::create(path.as_ref())?; - self.write(&mut f) - } - - pub fn write_npz, T: AsRef, P: AsRef>( - ts: &[(S, T)], - path: P, - ) -> Result<(), TchError> { - let mut zip = zip::ZipWriter::new(File::create(path.as_ref())?); - let options = - zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); - - for (name, tensor) in ts.iter() { - zip.start_file(format!("{}.npy", name.as_ref()), options)?; - tensor.as_ref().write(&mut zip)? - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::Header; - - #[test] - fn parse() { - let h = "{'descr': '(v: T) -> T { - v -} - -fn neg(t: Tensor) -> Tensor { - t.neg() -} - -fn inv(t: Tensor) -> Tensor { - t.pow_tensor_scalar(-1) -} - -macro_rules! impl_op { - ($trait:ident, $func:ident, $op:ident) => { - impl $trait for Tensor { - type Output = Tensor; - - fn $func(self, rhs: Tensor) -> Self::Output { - self.$op(&rhs) - } - } - - impl $trait<&Tensor> for Tensor { - type Output = Tensor; - - fn $func(self, rhs: &Tensor) -> Self::Output { - self.$op(rhs) - } - } - - impl<'a> $trait<&Tensor> for &'a Tensor { - type Output = Tensor; - - fn $func(self, rhs: &Tensor) -> Self::Output { - self.$op(rhs) - } - } - - impl $trait for &Tensor { - type Output = Tensor; - - fn $func(self, rhs: Tensor) -> Self::Output { - self.$op(&rhs) - } - } - }; -} - -impl Add for &Tensor -where - S: Into, -{ - type Output = Tensor; - - fn add(self, rhs: S) -> Self::Output { - self.g_add_scalar(rhs) - } -} - -impl Add for Tensor -where - S: Into, -{ - type Output = Tensor; - - fn add(self, rhs: S) -> Self::Output { - (&self).add(rhs) - } -} - -impl Sub for &Tensor -where - S: Into, -{ - type Output = Tensor; - - fn sub(self, rhs: S) -> Self::Output { - self.g_sub_scalar(rhs) - } -} - -impl Sub for Tensor -where - S: Into, -{ - type Output = Tensor; - - fn sub(self, rhs: S) -> Self::Output { - (&self).sub(rhs) - } -} - -impl Mul for &Tensor -where - S: Into, -{ - type Output = Tensor; - - fn mul(self, rhs: S) -> Self::Output { - self.g_mul_scalar(rhs) - } -} - -impl Mul for Tensor -where - S: Into, -{ - type Output = Tensor; - - fn mul(self, rhs: S) -> Self::Output { - (&self).mul(rhs) - } -} - -impl Div for &Tensor -where - S: Into, -{ - type Output = Tensor; - - fn div(self, rhs: S) -> Self::Output { - self.g_div_scalar(rhs) - } -} - -impl Div for Tensor -where - S: Into, -{ - type Output = Tensor; - - fn div(self, rhs: S) -> Self::Output { - (&self).div(rhs) - } -} - -macro_rules! impl_op_basic { - /* rev such that rev(op(b, a)) = op(a, b) */ - ($trait:ident, $func:ident, $op:ident, $rev:ident) => { - impl $trait for i32 { - type Output = Tensor; - - fn $func(self, rhs: Tensor) -> Self::Output { - self.$func(&rhs) - } - } - - impl $trait for i64 { - type Output = Tensor; - - fn $func(self, rhs: Tensor) -> Self::Output { - self.$func(&rhs) - } - } - - impl $trait for f32 { - type Output = Tensor; - - fn $func(self, rhs: Tensor) -> Self::Output { - self.$func(&rhs) - } - } - - impl $trait for f64 { - type Output = Tensor; - - fn $func(self, rhs: Tensor) -> Self::Output { - self.$func(&rhs) - } - } - - impl $trait<&Tensor> for i32 { - type Output = Tensor; - - fn $func(self, rhs: &Tensor) -> Self::Output { - $rev(rhs.$op(self as i64)) - } - } - - impl $trait<&Tensor> for i64 { - type Output = Tensor; - - fn $func(self, rhs: &Tensor) -> Self::Output { - $rev(rhs.$op(self)) - } - } - - impl $trait<&Tensor> for f32 { - type Output = Tensor; - - fn $func(self, rhs: &Tensor) -> Self::Output { - $rev(rhs.$op(self as f64)) - } - } - - impl $trait<&Tensor> for f64 { - type Output = Tensor; - - fn $func(self, rhs: &Tensor) -> Self::Output { - $rev(rhs.$op(self)) - } - } - }; -} - -macro_rules! impl_op_assign { - ($trait:ident, $func:ident, $op:ident) => { - impl $trait for Tensor { - fn $func(&mut self, rhs: Tensor) { - let _ = self.$op(&rhs); - } - } - - impl $trait<&Tensor> for Tensor { - fn $func(&mut self, rhs: &Tensor) { - let _ = self.$op(rhs); - } - } - }; -} - -macro_rules! impl_op_assign_basic { - ($trait:ident, $func:ident, $op:ident) => { - impl $trait for Tensor { - fn $func(&mut self, rhs: i32) { - let _ = self.$op(rhs as i64); - } - } - - impl $trait for Tensor { - fn $func(&mut self, rhs: i64) { - let _ = self.$op(rhs); - } - } - - impl $trait for Tensor { - fn $func(&mut self, rhs: f32) { - let _ = self.$op(rhs as f64); - } - } - - impl $trait for Tensor { - fn $func(&mut self, rhs: f64) { - let _ = self.$op(rhs); - } - } - }; -} - -impl_op!(Add, add, g_add); -impl_op_basic!(Add, add, g_add_scalar, id); -impl_op_assign!(AddAssign, add_assign, g_add_); -impl_op_assign_basic!(AddAssign, add_assign, g_add_scalar_); - -impl_op!(Mul, mul, g_mul); -impl_op_basic!(Mul, mul, g_mul_scalar, id); -impl_op_assign!(MulAssign, mul_assign, g_mul_); -impl_op_assign_basic!(MulAssign, mul_assign, g_mul_scalar_); - -impl_op!(Div, div, g_div); -impl_op_basic!(Div, div, g_div_scalar, inv); -impl_op_assign!(DivAssign, div_assign, g_div_); -impl_op_assign_basic!(DivAssign, div_assign, g_div_scalar_); - -impl_op!(Sub, sub, g_sub); -impl_op_basic!(Sub, sub, g_sub_scalar, neg); -impl_op_assign!(SubAssign, sub_assign, g_sub_); -impl_op_assign_basic!(SubAssign, sub_assign, g_sub_scalar_); - -impl Neg for Tensor { - type Output = Tensor; - - fn neg(self) -> Tensor { - self.f_neg().unwrap() - } -} - -impl Neg for &Tensor { - type Output = Tensor; - - fn neg(self) -> Tensor { - self.f_neg().unwrap() - } -} - -impl PartialEq for Tensor { - fn eq(&self, other: &Tensor) -> bool { - if self.size() != other.size() { - return false; - } - match self.f_eq_tensor(other) { - Err(_) => false, - Ok(v) => match v.f_all() { - Err(_) => false, - Ok(v) => match i64::try_from(v) { - Err(_) => false, - Ok(v) => v > 0, - }, - }, - } - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/tensor/safetensors.rs b/v2/dependency-patches/tch-0.24.0/src/tensor/safetensors.rs deleted file mode 100644 index 9ca4984a9a..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/tensor/safetensors.rs +++ /dev/null @@ -1,183 +0,0 @@ -//! Safetensors support for tensors. -//! -//! This module implements reading and writing tensors in the `.safetensors` format. -//! -use crate::nn::VarStore; -use crate::{Kind, TchError, Tensor}; - -use std::convert::{TryFrom, TryInto}; -use std::path::Path; - -use safetensors::tensor::{Dtype, SafeTensors, TensorView, View}; - -impl TryFrom for Dtype { - type Error = TchError; - fn try_from(kind: Kind) -> Result { - let dtype = match kind { - Kind::Bool => Dtype::BOOL, - Kind::Uint8 => Dtype::U8, - Kind::Int8 => Dtype::I8, - Kind::Int16 => Dtype::I16, - Kind::Int => Dtype::I32, - Kind::Int64 => Dtype::I64, - Kind::BFloat16 => Dtype::BF16, - Kind::Half => Dtype::F16, - Kind::Float => Dtype::F32, - Kind::Double => Dtype::F64, - kind => return Err(TchError::Convert(format!("unsupported kind ({kind:?})"))), - }; - Ok(dtype) - } -} - -impl TryFrom for Kind { - type Error = TchError; - fn try_from(dtype: Dtype) -> Result { - let kind = match dtype { - Dtype::BOOL => Kind::Bool, - Dtype::U8 => Kind::Uint8, - Dtype::I8 => Kind::Int8, - Dtype::I16 => Kind::Int16, - Dtype::I32 => Kind::Int, - Dtype::I64 => Kind::Int64, - Dtype::BF16 => Kind::BFloat16, - Dtype::F16 => Kind::Half, - Dtype::F32 => Kind::Float, - Dtype::F64 => Kind::Double, - dtype => return Err(TchError::Convert(format!("unsupported dtype {dtype:?}"))), - }; - Ok(kind) - } -} - -impl<'a> TryFrom> for Tensor { - type Error = TchError; - fn try_from(view: TensorView<'a>) -> Result { - let size: Vec = view.shape().iter().map(|&x| x as i64).collect(); - let kind: Kind = view.dtype().try_into()?; - Tensor::f_from_data_size(view.data(), &size, kind) - } -} - -struct SafeView<'a> { - tensor: &'a Tensor, - shape: Vec, - dtype: Dtype, -} - -impl<'a> TryFrom<&'a Tensor> for SafeView<'a> { - type Error = TchError; - - fn try_from(tensor: &'a Tensor) -> Result { - if tensor.is_sparse() { - return Err(TchError::Convert("Cannot save sparse tensors".to_string())); - } - - if !tensor.is_contiguous() { - return Err(TchError::Convert("Cannot save non contiguous tensors".to_string())); - } - - let dtype = tensor.kind().try_into()?; - let shape = tensor.size().iter().map(|&x| x as usize).collect(); - Ok(Self { tensor, shape, dtype }) - } -} - -impl View for SafeView<'_> { - fn dtype(&self) -> Dtype { - self.dtype - } - fn shape(&self) -> &[usize] { - &self.shape - } - - fn data(&self) -> std::borrow::Cow<'_, [u8]> { - let mut data = vec![0; self.data_len()]; - let numel = self.tensor.numel(); - self.tensor.f_copy_data_u8(&mut data, numel).unwrap(); - data.into() - } - - fn data_len(&self) -> usize { - self.tensor.numel() * self.tensor.kind().elt_size_in_bytes() - } -} - -fn wrap_err>(path: P, err: safetensors::SafeTensorError) -> TchError { - TchError::SafeTensorError { path: path.as_ref().to_string_lossy().to_string(), err } -} - -impl crate::Tensor { - /// Reads a safetensors file and returns some named tensors. - pub fn read_safetensors>(path: T) -> Result, TchError> { - let file = std::fs::read(&path).map_err(|e| wrap_err(&path, e.into()))?; - let safetensors = SafeTensors::deserialize(&file).map_err(|e| wrap_err(&path, e))?; - safetensors.tensors().into_iter().map(|(name, view)| Ok((name, view.try_into()?))).collect() - } - - /// Writes a tensor in the safetensors format. - pub fn write_safetensors, T: AsRef, P: AsRef>( - tensors: &[(S, T)], - path: P, - ) -> Result<(), TchError> { - let views = tensors - .iter() - .map(|(name, tensor)| { - Ok::<(&str, SafeView), TchError>((name.as_ref(), tensor.as_ref().try_into()?)) - }) - .collect::, _>>()?; - safetensors::tensor::serialize_to_file(views, &None, path.as_ref()) - .map_err(|e| wrap_err(path, e))?; - Ok(()) - } -} - -impl VarStore { - /// Read data from safe tensor file, missing tensors will raise a error. - pub fn read_safetensors>(&self, path: T) -> Result<(), TchError> { - let file = std::fs::read(&path).map_err(|e| wrap_err(&path, e.into()))?; - let safetensors = SafeTensors::deserialize(&file).map_err(|e| wrap_err(&path, e))?; - for (name, tensor) in self.variables_.lock().unwrap().named_variables.iter_mut() { - let view = safetensors.tensor(name).map_err(|e| wrap_err(&path, e))?; - let data: Tensor = view.try_into()?; - tensor.f_copy_(&data)? - } - Ok(()) - } - - pub fn fill_safetensors>(&self, path: P) -> Result<(), TchError> { - for (name, tensor) in Tensor::read_safetensors(path)? { - if let Some(s) = self.variables_.lock().unwrap().named_variables.get_mut(&name) { - s.f_copy_(&tensor)? - } - } - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use std::convert::TryInto; - - use crate::Kind; - use safetensors::Dtype; - - #[test] - fn parse() { - // From Kind to Dtype - assert_eq!(TryInto::::try_into(Kind::Double).unwrap(), Dtype::F64); - assert_eq!(TryInto::::try_into(Kind::Float).unwrap(), Dtype::F32); - assert_eq!(TryInto::::try_into(Kind::Half).unwrap(), Dtype::F16); - - assert_eq!(TryInto::::try_into(Kind::Int8).unwrap(), Dtype::I8); - assert_eq!(TryInto::::try_into(Kind::Uint8).unwrap(), Dtype::U8); - - // From Dtype to Kind - assert_eq!(TryInto::::try_into(Dtype::F64).unwrap(), Kind::Double); - assert_eq!(TryInto::::try_into(Dtype::F32).unwrap(), Kind::Float); - assert_eq!(TryInto::::try_into(Dtype::F16).unwrap(), Kind::Half); - - assert_eq!(TryInto::::try_into(Dtype::I8).unwrap(), Kind::Int8); - assert_eq!(TryInto::::try_into(Dtype::U8).unwrap(), Kind::Uint8); - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/alexnet.rs b/v2/dependency-patches/tch-0.24.0/src/vision/alexnet.rs deleted file mode 100644 index 6338ef0a60..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/vision/alexnet.rs +++ /dev/null @@ -1,44 +0,0 @@ -//! AlexNet. -//! -use crate::{nn, nn::Conv2D, nn::ModuleT, Tensor}; - -fn conv2d(p: nn::Path, c_in: i64, c_out: i64, ksize: i64, padding: i64, stride: i64) -> Conv2D { - let conv2d_cfg = nn::ConvConfig { stride, padding, ..Default::default() }; - nn::conv2d(p, c_in, c_out, ksize, conv2d_cfg) -} - -fn max_pool2d(xs: Tensor, ksize: i64, stride: i64) -> Tensor { - xs.max_pool2d([ksize, ksize], [stride, stride], [0, 0], [1, 1], false) -} - -fn features(p: nn::Path) -> impl ModuleT { - nn::seq_t() - .add(conv2d(&p / "0", 3, 64, 11, 2, 4)) - .add_fn(|xs| max_pool2d(xs.relu(), 3, 2)) - .add(conv2d(&p / "3", 64, 192, 5, 1, 2)) - .add_fn(|xs| max_pool2d(xs.relu(), 3, 2)) - .add(conv2d(&p / "6", 192, 384, 3, 1, 1)) - .add_fn(|xs| xs.relu()) - .add(conv2d(&p / "8", 384, 256, 3, 1, 1)) - .add_fn(|xs| xs.relu()) - .add(conv2d(&p / "10", 256, 256, 3, 1, 1)) - .add_fn(|xs| max_pool2d(xs.relu(), 3, 2)) -} - -fn classifier(p: nn::Path, nclasses: i64) -> impl ModuleT { - nn::seq_t() - .add_fn_t(|xs, train| xs.dropout(0.5, train)) - .add(nn::linear(&p / "1", 256 * 6 * 6, 4096, Default::default())) - .add_fn(|xs| xs.relu()) - .add_fn_t(|xs, train| xs.dropout(0.5, train)) - .add(nn::linear(&p / "4", 4096, 4096, Default::default())) - .add_fn(|xs| xs.relu()) - .add(nn::linear(&p / "6", 4096, nclasses, Default::default())) -} - -pub fn alexnet(p: &nn::Path, nclasses: i64) -> impl ModuleT { - nn::seq_t() - .add(features(p / "features")) - .add_fn(|xs| xs.adaptive_avg_pool2d([6, 6]).flat_view()) - .add(classifier(p / "classifier", nclasses)) -} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/cifar.rs b/v2/dependency-patches/tch-0.24.0/src/vision/cifar.rs deleted file mode 100644 index 112345d10c..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/vision/cifar.rs +++ /dev/null @@ -1,64 +0,0 @@ -//! The CIFAR-10 dataset. -//! -//! The files can be downloaded from the following page: -//! -//! The binary version of the dataset is used. -use super::dataset::Dataset; -use crate::{kind, IndexOp, Kind, Tensor}; -use std::fs::File; -use std::io::{BufReader, Read, Result}; - -const W: i64 = 32; -const H: i64 = 32; -const C: i64 = 3; -const BYTES_PER_IMAGE: i64 = W * H * C + 1; -const SAMPLES_PER_FILE: i64 = 10000; - -fn read_file_(filename: &std::path::Path) -> Result<(Tensor, Tensor)> { - let mut buf_reader = BufReader::new(File::open(filename)?); - let mut data = vec![0u8; (SAMPLES_PER_FILE * BYTES_PER_IMAGE) as usize]; - buf_reader.read_exact(&mut data)?; - let content = Tensor::from_slice(&data); - let images = Tensor::zeros([SAMPLES_PER_FILE, C, H, W], kind::FLOAT_CPU); - let labels = Tensor::zeros([SAMPLES_PER_FILE], kind::INT64_CPU); - for index in 0..SAMPLES_PER_FILE { - let content_offset = BYTES_PER_IMAGE * index; - labels.i(index).copy_(&content.i(content_offset)); - images.i(index).copy_( - &content - .narrow(0, 1 + content_offset, BYTES_PER_IMAGE - 1) - .view((C, H, W)) - .to_kind(Kind::Float), - ); - } - Ok((images.to_kind(Kind::Float) / 255.0, labels)) -} - -fn read_file(filename: &std::path::Path) -> Result<(Tensor, Tensor)> { - read_file_(filename) - .map_err(|err| std::io::Error::new(err.kind(), format!("{filename:?} {err}"))) -} - -pub fn load_dir>(dir: T) -> Result { - let dir = dir.as_ref(); - let (test_images, test_labels) = read_file(&dir.join("test_batch.bin"))?; - let train_images_and_labels = [ - "data_batch_1.bin", - "data_batch_2.bin", - "data_batch_3.bin", - "data_batch_4.bin", - "data_batch_5.bin", - ] - .iter() - .map(|x| read_file(&dir.join(x))) - .collect::>>()?; - let (train_images, train_labels): (Vec<_>, Vec<_>) = - train_images_and_labels.into_iter().unzip(); - Ok(Dataset { - train_images: Tensor::cat(&train_images, 0), - train_labels: Tensor::cat(&train_labels, 0), - test_images, - test_labels, - labels: 10, - }) -} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/convmixer.rs b/v2/dependency-patches/tch-0.24.0/src/vision/convmixer.rs deleted file mode 100644 index ce0379bbba..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/vision/convmixer.rs +++ /dev/null @@ -1,49 +0,0 @@ -//! ConvMixer implementation. -//! -//! See "Patches Are All You Need?", submitted to ICLR'21 -//! -use crate::nn; - -fn block(p: nn::Path, dim: i64, kernel_size: i64) -> impl nn::ModuleT { - let conv2d_cfg = nn::ConvConfig { groups: dim, ..Default::default() }; - let p_fn = &(&p / "0") / "fn"; - let conv1 = - crate::vision::efficientnet::conv2d_same(&p_fn / "0", dim, dim, kernel_size, conv2d_cfg); - let bn1 = nn::batch_norm2d(&p_fn / "2", dim, Default::default()); - let conv2 = nn::conv2d(&p / "1", dim, dim, 1, Default::default()); - let bn2 = nn::batch_norm2d(&p / "3", dim, Default::default()); - nn::func_t(move |xs, train| { - let ys = xs.apply(&conv1).gelu("none").apply_t(&bn1, train); - (xs + ys).apply(&conv2).gelu("none").apply_t(&bn2, train) - }) -} - -fn convmixer( - p: &nn::Path, - nclasses: i64, - dim: i64, - depth: i64, - kernel_size: i64, - patch_size: i64, -) -> nn::FuncT<'static> { - let conv2d_cfg = nn::ConvConfig { stride: patch_size, ..Default::default() }; - let conv1 = nn::conv2d(p / "0", 3, dim, patch_size, conv2d_cfg); - let bn1 = nn::batch_norm2d(p / "2", dim, Default::default()); - let blocks: Vec<_> = (0..depth).map(|index| block(p / (3 + index), dim, kernel_size)).collect(); - let fc = nn::linear(p / "25", dim, nclasses, Default::default()); - nn::func_t(move |xs, train| { - let mut xs = xs.apply(&conv1).gelu("none").apply_t(&bn1, train); - for block in blocks.iter() { - xs = xs.apply_t(block, train) - } - xs.adaptive_avg_pool2d([1, 1]).flat_view().apply(&fc) - }) -} - -pub fn c1536_20(p: &nn::Path, nclasses: i64) -> nn::FuncT<'static> { - convmixer(p, nclasses, 1536, 20, 9, 7) -} - -pub fn c1024_20(p: &nn::Path, nclasses: i64) -> nn::FuncT<'static> { - convmixer(p, nclasses, 1024, 20, 9, 14) -} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/dataset.rs b/v2/dependency-patches/tch-0.24.0/src/vision/dataset.rs deleted file mode 100644 index 782401248c..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/vision/dataset.rs +++ /dev/null @@ -1,95 +0,0 @@ -//! A simple dataset structure shared by various computer vision datasets. -use crate::data::Iter2; -use crate::{IndexOp, Tensor}; -use rand::Rng; - -#[derive(Debug)] -pub struct Dataset { - pub train_images: Tensor, - pub train_labels: Tensor, - pub test_images: Tensor, - pub test_labels: Tensor, - pub labels: i64, -} - -impl Dataset { - pub fn train_iter(&self, batch_size: i64) -> Iter2 { - Iter2::new(&self.train_images, &self.train_labels, batch_size) - } - - pub fn test_iter(&self, batch_size: i64) -> Iter2 { - Iter2::new(&self.test_images, &self.test_labels, batch_size) - } -} - -/// Randomly applies horizontal flips -/// This expects a 4 dimension NCHW tensor and returns a tensor with -/// an identical shape. -pub fn random_flip(t: &Tensor) -> Tensor { - let size = t.size(); - if size.len() != 4 { - panic!("unexpected shape for tensor {t:?}") - } - let output = t.zeros_like(); - for batch_index in 0..size[0] { - let mut output_view = output.i(batch_index); - let t_view = t.i(batch_index); - let src = if rand::random() { t_view } else { t_view.flip([2]) }; - output_view.copy_(&src) - } - output -} - -/// Pad the image using reflections and take some random crops. -/// This expects a 4 dimension NCHW tensor and returns a tensor with -/// an identical shape. -pub fn random_crop(t: &Tensor, pad: i64) -> Tensor { - let size = t.size(); - if size.len() != 4 { - panic!("unexpected shape for tensor {t:?}") - } - let sz_h = size[2]; - let sz_w = size[3]; - let padded = t.reflection_pad2d([pad, pad, pad, pad]); - let output = t.zeros_like(); - for bindex in 0..size[0] { - let mut output_view = output.i(bindex); - let start_w = rand::thread_rng().gen_range(0..2 * pad); - let start_h = rand::thread_rng().gen_range(0..2 * pad); - let src = padded.i((bindex, .., start_h..start_h + sz_h, start_w..start_w + sz_w)); - output_view.copy_(&src) - } - output -} - -/// Applies cutout: randomly remove some square areas in the original images. -/// -pub fn random_cutout(t: &Tensor, sz: i64) -> Tensor { - let size = t.size(); - if size.len() != 4 || sz > size[2] || sz > size[3] { - panic!("unexpected shape for tensor {t:?} {sz}") - } - let mut output = t.zeros_like(); - output.copy_(t); - for bindex in 0..size[0] { - let start_h = rand::thread_rng().gen_range(0..size[2] - sz + 1); - let start_w = rand::thread_rng().gen_range(0..size[3] - sz + 1); - let _output = - output.i((bindex, .., start_h..start_h + sz, start_w..start_w + sz)).fill_(0.0); - } - output -} - -pub fn augmentation(t: &Tensor, flip: bool, crop: i64, cutout: i64) -> Tensor { - let mut t = t.shallow_clone(); - if flip { - t = random_flip(&t); - } - if crop > 0 { - t = random_crop(&t, crop); - } - if cutout > 0 { - t = random_cutout(&t, cutout); - } - t -} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/densenet.rs b/v2/dependency-patches/tch-0.24.0/src/vision/densenet.rs deleted file mode 100644 index 03b95dc3a6..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/vision/densenet.rs +++ /dev/null @@ -1,96 +0,0 @@ -//! DenseNet implementation. -//! -//! See "Densely Connected Convolutional Networks", Huang et al 2016. -//! -//! -//! Pre-trained weights for the densenet-121 variants can be found here: -//! -use crate::{nn, nn::Conv2D, nn::ModuleT, Tensor}; - -fn conv2d(p: nn::Path, c_in: i64, c_out: i64, ksize: i64, padding: i64, stride: i64) -> Conv2D { - let conv2d_cfg = nn::ConvConfig { stride, padding, bias: false, ..Default::default() }; - nn::conv2d(p, c_in, c_out, ksize, conv2d_cfg) -} - -fn dense_layer(p: nn::Path, c_in: i64, bn_size: i64, growth: i64) -> impl ModuleT { - let c_inter = bn_size * growth; - let bn1 = nn::batch_norm2d(&p / "norm1", c_in, Default::default()); - let conv1 = conv2d(&p / "conv1", c_in, c_inter, 1, 0, 1); - let bn2 = nn::batch_norm2d(&p / "norm2", c_inter, Default::default()); - let conv2 = conv2d(&p / "conv2", c_inter, growth, 3, 1, 1); - nn::func_t(move |xs, train| { - let ys = - xs.apply_t(&bn1, train).relu().apply(&conv1).apply_t(&bn2, train).relu().apply(&conv2); - Tensor::cat(&[xs, &ys], 1) - }) -} - -fn dense_block(p: nn::Path, c_in: i64, bn_size: i64, growth: i64, nlayers: i64) -> impl ModuleT { - let mut seq = nn::seq_t(); - for i in 0..nlayers { - seq = seq.add(dense_layer( - &p / &format!("denselayer{}", 1 + i), - c_in + i * growth, - bn_size, - growth, - )); - } - seq -} - -fn transition(p: nn::Path, c_in: i64, c_out: i64) -> impl ModuleT { - nn::seq_t() - .add(nn::batch_norm2d(&p / "norm", c_in, Default::default())) - .add_fn(|xs| xs.relu()) - .add(conv2d(&p / "conv", c_in, c_out, 1, 0, 1)) - .add_fn(|xs| xs.avg_pool2d_default(2)) -} - -fn densenet( - p: &nn::Path, - c_in: i64, - bn_size: i64, - growth: i64, - block_config: &[i64], - c_out: i64, -) -> impl ModuleT { - let fp = p / "features"; - let mut seq = nn::seq_t() - .add(conv2d(&fp / "conv0", 3, c_in, 7, 3, 2)) - .add(nn::batch_norm2d(&fp / "norm0", c_in, Default::default())) - .add_fn(|xs| xs.relu().max_pool2d([3, 3], [2, 2], [1, 1], [1, 1], false)); - let mut nfeat = c_in; - for (i, &nlayers) in block_config.iter().enumerate() { - seq = seq.add(dense_block( - &fp / &format!("denseblock{}", 1 + i), - nfeat, - bn_size, - growth, - nlayers, - )); - nfeat += nlayers * growth; - if i + 1 != block_config.len() { - seq = seq.add(transition(&fp / &format!("transition{}", 1 + i), nfeat, nfeat / 2)); - nfeat /= 2 - } - } - seq.add(nn::batch_norm2d(&fp / "norm5", nfeat, Default::default())) - .add_fn(|xs| xs.relu().avg_pool2d([7, 7], [1, 1], [0, 0], false, true, 1).flat_view()) - .add(nn::linear(p / "classifier", nfeat, c_out, Default::default())) -} - -pub fn densenet121(p: &nn::Path, nclasses: i64) -> impl ModuleT { - densenet(p, 64, 4, 32, &[6, 12, 24, 16], nclasses) -} - -pub fn densenet161(p: &nn::Path, nclasses: i64) -> impl ModuleT { - densenet(p, 96, 4, 48, &[6, 12, 36, 24], nclasses) -} - -pub fn densenet169(p: &nn::Path, nclasses: i64) -> impl ModuleT { - densenet(p, 64, 4, 32, &[6, 12, 32, 32], nclasses) -} - -pub fn densenet201(p: &nn::Path, nclasses: i64) -> impl ModuleT { - densenet(p, 64, 4, 32, &[6, 12, 48, 32], nclasses) -} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/dinov2.rs b/v2/dependency-patches/tch-0.24.0/src/vision/dinov2.rs deleted file mode 100644 index c6a00d82f0..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/vision/dinov2.rs +++ /dev/null @@ -1,234 +0,0 @@ -//! DINOv2: Learning Robust Visual Features without Supervision -//! https://github.com/facebookresearch/dinov2 -//! The weights can be extracted from pre-trained Python models -//! using `python src/vision/export_dinov2.py`. -// TODO: use swiglu. -use crate::{nn, IndexOp, Kind, Tensor}; - -const IMG_SIZE: i64 = 518; -const PATCH_SIZE: i64 = 14; -const NUM_CLASSES: i64 = 1000; - -#[derive(Debug)] -struct Attention { - qkv: nn::Linear, - proj: nn::Linear, - num_heads: i64, - scale: f64, -} - -impl Attention { - fn new(vs: nn::Path, dim: i64, num_heads: i64, qkv_bias: bool, proj_bias: bool) -> Self { - let qkv_config = nn::LinearConfig { bias: qkv_bias, ..Default::default() }; - let proj_config = nn::LinearConfig { bias: proj_bias, ..Default::default() }; - let qkv = nn::linear(&vs / "qkv", dim, dim * 3, qkv_config); - let proj = nn::linear(&vs / "proj", dim, dim, proj_config); - let scale = 1. / ((dim / num_heads) as f64).sqrt(); - Self { qkv, proj, num_heads, scale } - } -} - -impl nn::Module for Attention { - fn forward(&self, xs: &Tensor) -> Tensor { - let (b, n, c) = xs.size3().unwrap(); - let qkv = self - .qkv - .forward(xs) - .reshape([b, n, 3, self.num_heads, c / self.num_heads]) - .permute([2, 0, 3, 1, 4]); - let q = qkv.get(0) * self.scale; - let k = qkv.get(1); - let v = qkv.get(2); - let attn = q.matmul(&k.transpose(-2, -1)).softmax(-1, Kind::Float); - attn.matmul(&v).transpose(1, 2).reshape([b, n, c]).apply(&self.proj) - } -} - -#[derive(Debug)] -struct LayerScale { - gamma: Tensor, -} - -impl LayerScale { - fn new(vs: nn::Path, dim: i64) -> Self { - let gamma = vs.var("gamma", &[dim], nn::Init::Const(0.)); - Self { gamma } - } -} - -impl nn::Module for LayerScale { - fn forward(&self, xs: &Tensor) -> Tensor { - xs * &self.gamma - } -} - -#[derive(Debug)] -struct Mlp { - fc1: nn::Linear, - fc2: nn::Linear, -} - -impl Mlp { - fn new(vs: nn::Path, in_features: i64, hidden_features: i64, bias: bool) -> Self { - let out_features = in_features; - let config = nn::LinearConfig { bias, ..Default::default() }; - let fc1 = nn::linear(&vs / "fc1", in_features, hidden_features, config); - let fc2 = nn::linear(&vs / "fc2", hidden_features, out_features, config); - Self { fc1, fc2 } - } -} - -impl nn::Module for Mlp { - fn forward(&self, xs: &Tensor) -> Tensor { - xs.apply(&self.fc1).gelu("none").apply(&self.fc2) - } -} - -#[derive(Debug)] -struct Block { - norm1: nn::LayerNorm, - attn: Attention, - ls1: LayerScale, - norm2: nn::LayerNorm, - mlp: Mlp, - ls2: LayerScale, -} - -impl Block { - fn new(vs: nn::Path, dim: i64, num_heads: i64) -> Self { - let norm1 = nn::layer_norm(&vs / "norm1", vec![dim], Default::default()); - let attn = Attention::new(&vs / "attn", dim, num_heads, true, true); - let ls1 = LayerScale::new(&vs / "ls1", dim); - let norm2 = nn::layer_norm(&vs / "norm2", vec![dim], Default::default()); - let mlp = Mlp::new(&vs / "mlp", dim, dim * 4, true); - let ls2 = LayerScale::new(&vs / "ls2", dim); - Self { norm1, attn, ls1, norm2, mlp, ls2 } - } -} - -impl nn::Module for Block { - fn forward(&self, xs: &Tensor) -> Tensor { - let xs = xs + xs.apply(&self.norm1).apply(&self.attn).apply(&self.ls1); - &xs + xs.apply(&self.norm2).apply(&self.mlp).apply(&self.ls2) - } -} - -#[derive(Debug)] -struct PatchEmbed { - proj: nn::Conv2D, - patch_size: (i64, i64), - num_patches: i64, -} - -impl PatchEmbed { - fn new(vs: nn::Path, img_size: i64, patch_size: i64, in_chans: i64, embed_dim: i64) -> Self { - let config = nn::ConvConfig { stride: patch_size, ..Default::default() }; - let proj = nn::conv2d(vs / "proj", in_chans, embed_dim, patch_size, config); - let num_patches = (img_size / patch_size) * (img_size / patch_size); - Self { proj, patch_size: (patch_size, patch_size), num_patches } - } -} - -impl nn::Module for PatchEmbed { - fn forward(&self, xs: &Tensor) -> Tensor { - let (_b, _c, h, w) = xs.size4().unwrap(); - let (patch_h, patch_w) = self.patch_size; - if (h % patch_h) != 0 { - panic!("image height {h} is not a multiple of patch height {patch_h}") - } - if (w % patch_w) != 0 { - panic!("image width {w} is not a multiple of patch width {patch_w}") - } - let xs = xs.apply(&self.proj); - let (b, c, h, w) = xs.size4().unwrap(); - // flatten embeddings. - xs.reshape([b, c, h * w]).transpose(1, 2) - } -} - -#[derive(Debug)] -pub struct DinoVisionTransformer { - patch_embed: PatchEmbed, - cls_token: Tensor, - pos_embed: Tensor, - blocks: Vec, - norm: nn::LayerNorm, - head: nn::Linear, -} - -impl DinoVisionTransformer { - pub fn new(vs: &nn::Path, depth: usize, embed_dim: i64, num_heads: i64) -> Self { - let patch_embed = PatchEmbed::new(vs / "patch_embed", IMG_SIZE, PATCH_SIZE, 3, embed_dim); - let cls_token = vs.var("cls_token", &[1, 1, embed_dim], nn::Init::Const(0.)); - let num_tokens = 1; - let pos_embed = vs.var( - "pos_embed", - &[1, patch_embed.num_patches + num_tokens, embed_dim], - nn::Init::Const(0.), - ); - let head = nn::linear(vs / "head", 2 * embed_dim, NUM_CLASSES, Default::default()); - let norm = nn::layer_norm(vs / "norm", vec![embed_dim], Default::default()); - let blocks = - (0..depth).map(|i| Block::new(vs / "blocks" / i, embed_dim, num_heads)).collect(); - Self { patch_embed, cls_token, pos_embed, blocks, norm, head } - } - - pub fn interpolate_pos_encoding(&self, xs: &Tensor, w: i64, h: i64) -> Tensor { - let npatch = xs.size()[1] - 1; - let n = self.pos_embed.size()[1] - 1; - let sqrt_n = (n as f64).sqrt(); - if npatch == n && w == h { - return xs.shallow_clone(); - } - let class_pos_embed = self.pos_embed.i((.., ..1)); - let patch_pos_embed = self.pos_embed.i((.., 1..)); - let dim = *xs.size().last().unwrap(); - let (w0, h0) = ((w / PATCH_SIZE) as f64 + 0.1, (h / PATCH_SIZE) as f64 + 0.1); - let patch_pos_embed = patch_pos_embed - .reshape([1, sqrt_n as i64, sqrt_n as i64, dim]) - .permute([0, 3, 1, 2]) - .upsample_bicubic2d([w0 as i64, h0 as i64], false, w0 / sqrt_n, h0 / sqrt_n); - let patch_pos_embed = patch_pos_embed.permute([0, 2, 3, 1]).reshape([1, -1, dim]); - Tensor::cat(&[class_pos_embed, patch_pos_embed], 1) - } - - pub fn prepare_tokens_with_mask(&self, xs: &Tensor) -> Tensor { - let (b, _nc, w, h) = xs.size4().unwrap(); - let xs = xs.apply(&self.patch_embed); - let xs = Tensor::concat(&[self.cls_token.expand([b, -1, -1], false), xs], 1); - &xs + &self.interpolate_pos_encoding(&xs, w, h) - } - - pub fn extract_features(&self, xs: &Tensor) -> Tensor { - let mut xs = self.prepare_tokens_with_mask(xs); - for blk in self.blocks.iter() { - xs = xs.apply(blk) - } - let xs = xs.apply(&self.norm); - let xs_norm_clstoken = xs.i((.., 0)); - let xs_norm_patchtokens = xs.i((.., 1..)).mean_dim(1, false, None); - Tensor::concat(&[xs_norm_clstoken, xs_norm_patchtokens], -1) - } -} - -impl nn::Module for DinoVisionTransformer { - fn forward(&self, xs: &Tensor) -> Tensor { - self.extract_features(xs).apply(&self.head) - } -} - -pub fn vit_small(vs: &nn::Path) -> DinoVisionTransformer { - DinoVisionTransformer::new(vs, 12, 384, 6) -} - -pub fn vit_base(vs: &nn::Path) -> DinoVisionTransformer { - DinoVisionTransformer::new(vs, 12, 768, 12) -} - -pub fn vit_large(vs: &nn::Path) -> DinoVisionTransformer { - DinoVisionTransformer::new(vs, 24, 1024, 16) -} - -pub fn vit_giant(vs: &nn::Path) -> DinoVisionTransformer { - DinoVisionTransformer::new(vs, 40, 1536, 24) -} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/efficientnet.rs b/v2/dependency-patches/tch-0.24.0/src/vision/efficientnet.rs deleted file mode 100644 index d38c4a41c6..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/vision/efficientnet.rs +++ /dev/null @@ -1,329 +0,0 @@ -//! EfficientNet implementation. -use crate::nn::{self, ConvConfig, Module, ModuleT}; -use crate::Tensor; - -const BATCH_NORM_MOMENTUM: f64 = 0.99; -const BATCH_NORM_EPSILON: f64 = 1e-3; - -// Based on the Python version from torchvision. -// https://github.com/pytorch/vision/blob/0d75d9e5516f446c9c0ef93bd4ed9fea13992d06/torchvision/models/efficientnet.py#L47 -#[derive(Debug, Clone, Copy)] -pub struct MBConvConfig { - expand_ratio: f64, - kernel: i64, - stride: i64, - input_channels: i64, - out_channels: i64, - num_layers: usize, -} - -fn make_divisible(v: f64, divisor: i64) -> i64 { - let min_value = divisor; - let new_v = i64::max(min_value, (v + divisor as f64 * 0.5) as i64 / divisor * divisor); - if (new_v as f64) < 0.9 * v { - new_v + divisor - } else { - new_v - } -} - -fn bneck_confs(width_mult: f64, depth_mult: f64) -> Vec { - let bneck_conf = |e, k, s, i, o, n| { - let input_channels = make_divisible(i as f64 * width_mult, 8); - let out_channels = make_divisible(o as f64 * width_mult, 8); - let num_layers = (n as f64 * depth_mult).ceil() as usize; - MBConvConfig { - expand_ratio: e, - kernel: k, - stride: s, - input_channels, - out_channels, - num_layers, - } - }; - vec![ - bneck_conf(1., 3, 1, 32, 16, 1), - bneck_conf(6., 3, 2, 16, 24, 2), - bneck_conf(6., 5, 2, 24, 40, 2), - bneck_conf(6., 3, 2, 40, 80, 3), - bneck_conf(6., 5, 1, 80, 112, 3), - bneck_conf(6., 5, 2, 112, 192, 4), - bneck_conf(6., 3, 1, 192, 320, 1), - ] -} - -impl MBConvConfig { - fn b0() -> Vec { - bneck_confs(1.0, 1.0) - } - fn b1() -> Vec { - bneck_confs(1.0, 1.1) - } - fn b2() -> Vec { - bneck_confs(1.1, 1.2) - } - fn b3() -> Vec { - bneck_confs(1.2, 1.4) - } - fn b4() -> Vec { - bneck_confs(1.4, 1.8) - } - fn b5() -> Vec { - bneck_confs(1.6, 2.2) - } - fn b6() -> Vec { - bneck_confs(1.8, 2.6) - } - fn b7() -> Vec { - bneck_confs(2.0, 3.1) - } -} - -/// Conv2D with same padding. -#[derive(Debug)] -struct Conv2DSame { - conv2d: nn::Conv2D, - s: i64, - k: i64, -} - -impl Conv2DSame { - fn new(vs: nn::Path, i: i64, o: i64, k: i64, stride: i64, groups: i64, b: bool) -> Self { - let conv_config = nn::ConvConfig { stride, groups, bias: b, ..Default::default() }; - let conv2d = nn::conv2d(vs, i, o, k, conv_config); - Self { conv2d, s: stride, k } - } -} - -impl Module for Conv2DSame { - fn forward(&self, xs: &Tensor) -> Tensor { - let s = self.s; - let k = self.k; - let size = xs.size(); - let ih = size[2]; - let iw = size[3]; - let oh = (ih + s - 1) / s; - let ow = (iw + s - 1) / s; - let pad_h = i64::max((oh - 1) * s + k - ih, 0); - let pad_w = i64::max((ow - 1) * s + k - iw, 0); - if pad_h > 0 || pad_w > 0 { - xs.zero_pad2d(pad_w / 2, pad_w - pad_w / 2, pad_h / 2, pad_h - pad_h / 2) - .apply(&self.conv2d) - } else { - xs.apply(&self.conv2d) - } - } -} - -#[derive(Debug)] -struct ConvNormActivation { - conv2d: Conv2DSame, - bn2d: nn::BatchNorm, - activation: bool, -} - -impl ConvNormActivation { - fn new(vs: nn::Path, i: i64, o: i64, k: i64, stride: i64, groups: i64) -> Self { - let conv2d = Conv2DSame::new(&vs / 0, i, o, k, stride, groups, false); - let bn_config = nn::BatchNormConfig { - momentum: 1.0 - BATCH_NORM_MOMENTUM, - eps: BATCH_NORM_EPSILON, - ..Default::default() - }; - let bn2d = nn::batch_norm2d(&vs / 1, o, bn_config); - Self { conv2d, bn2d, activation: true } - } - - fn no_activation(self) -> Self { - Self { activation: false, ..self } - } -} - -impl ModuleT for ConvNormActivation { - fn forward_t(&self, xs: &Tensor, t: bool) -> Tensor { - let xs = xs.apply(&self.conv2d).apply_t(&self.bn2d, t); - if self.activation { - xs.swish() - } else { - xs - } - } -} - -#[derive(Debug)] -struct SqueezeExcitation { - fc1: Conv2DSame, - fc2: Conv2DSame, -} - -impl SqueezeExcitation { - fn new(vs: nn::Path, input_channels: i64, squeeze_channels: i64) -> Self { - let fc1 = Conv2DSame::new(&vs / "fc1", input_channels, squeeze_channels, 1, 1, 1, true); - let fc2 = Conv2DSame::new(&vs / "fc2", squeeze_channels, input_channels, 1, 1, 1, true); - Self { fc1, fc2 } - } -} - -impl ModuleT for SqueezeExcitation { - fn forward_t(&self, xs: &Tensor, t: bool) -> Tensor { - let scale = xs - .adaptive_avg_pool2d([1, 1]) - .apply_t(&self.fc1, t) - .swish() - .apply_t(&self.fc2, t) - .sigmoid(); - scale * xs - } -} - -#[derive(Debug)] -struct MBConv { - expand_cna: Option, - depthwise_cna: ConvNormActivation, - squeeze_excitation: SqueezeExcitation, - project_cna: ConvNormActivation, - config: MBConvConfig, -} - -impl MBConv { - fn new(vs: nn::Path, c: MBConvConfig) -> Self { - let vs = &vs / "block"; - let exp = make_divisible(c.input_channels as f64 * c.expand_ratio, 8); - let expand_cna = if exp != c.input_channels { - Some(ConvNormActivation::new(&vs / 0, c.input_channels, exp, 1, 1, 1)) - } else { - None - }; - let start_index = if expand_cna.is_some() { 1 } else { 0 }; - let depthwise_cna = - ConvNormActivation::new(&vs / start_index, exp, exp, c.kernel, c.stride, exp); - let squeeze_channels = i64::max(1, c.input_channels / 4); - let squeeze_excitation = - SqueezeExcitation::new(&vs / (start_index + 1), exp, squeeze_channels); - let project_cna = - ConvNormActivation::new(&vs / (start_index + 2), exp, c.out_channels, 1, 1, 1) - .no_activation(); - Self { expand_cna, depthwise_cna, squeeze_excitation, project_cna, config: c } - } -} - -impl ModuleT for MBConv { - fn forward_t(&self, xs: &Tensor, t: bool) -> Tensor { - let use_res_connect = - self.config.stride == 1 && self.config.input_channels == self.config.out_channels; - let ys = match &self.expand_cna { - Some(expand_cna) => xs.apply_t(expand_cna, t), - None => xs.shallow_clone(), - }; - let ys = ys - .apply_t(&self.depthwise_cna, t) - .apply_t(&self.squeeze_excitation, t) - .apply_t(&self.project_cna, t); - if use_res_connect { - ys + xs - } else { - ys - } - } -} - -impl Tensor { - fn swish(&self) -> Tensor { - self * self.sigmoid() - } -} - -#[derive(Debug)] -struct EfficientNet { - init_cna: ConvNormActivation, - blocks: Vec, - final_cna: ConvNormActivation, - classifier: nn::Linear, -} - -impl EfficientNet { - fn new(p: &nn::Path, configs: Vec, nclasses: i64) -> Self { - let f_p = p / "features"; - let first_in_c = configs[0].input_channels; - let last_out_c = configs.last().unwrap().out_channels; - let final_out_c = 4 * last_out_c; - let init_cna = ConvNormActivation::new(&f_p / 0, 3, first_in_c, 3, 2, 1); - let nconfigs = configs.len(); - let mut blocks = vec![]; - for (index, cnf) in configs.into_iter().enumerate() { - let f_p = &f_p / (index + 1); - for r_index in 0..cnf.num_layers { - let cnf = if r_index == 0 { - cnf - } else { - MBConvConfig { input_channels: cnf.out_channels, stride: 1, ..cnf } - }; - blocks.push(MBConv::new(&f_p / r_index, cnf)) - } - } - let final_cna = - ConvNormActivation::new(&f_p / (nconfigs + 1), last_out_c, final_out_c, 1, 1, 1); - let classifier = - nn::linear(p / "classifier" / 1, final_out_c, nclasses, Default::default()); - Self { init_cna, blocks, final_cna, classifier } - } -} - -impl ModuleT for EfficientNet { - fn forward_t(&self, xs: &Tensor, t: bool) -> Tensor { - let mut xs = xs.apply_t(&self.init_cna, t); - for block in self.blocks.iter() { - xs = xs.apply_t(block, t) - } - xs.apply_t(&self.final_cna, t) - .adaptive_avg_pool2d([1, 1]) - .squeeze_dim(-1) - .squeeze_dim(-1) - .apply(&self.classifier) - } -} - -pub fn b0(p: &nn::Path, nclasses: i64) -> impl ModuleT { - EfficientNet::new(p, MBConvConfig::b0(), nclasses) -} -pub fn b1(p: &nn::Path, nclasses: i64) -> impl ModuleT { - EfficientNet::new(p, MBConvConfig::b1(), nclasses) -} -pub fn b2(p: &nn::Path, nclasses: i64) -> impl ModuleT { - EfficientNet::new(p, MBConvConfig::b2(), nclasses) -} -pub fn b3(p: &nn::Path, nclasses: i64) -> impl ModuleT { - EfficientNet::new(p, MBConvConfig::b3(), nclasses) -} -pub fn b4(p: &nn::Path, nclasses: i64) -> impl ModuleT { - EfficientNet::new(p, MBConvConfig::b4(), nclasses) -} -pub fn b5(p: &nn::Path, nclasses: i64) -> impl ModuleT { - EfficientNet::new(p, MBConvConfig::b5(), nclasses) -} -pub fn b6(p: &nn::Path, nclasses: i64) -> impl ModuleT { - EfficientNet::new(p, MBConvConfig::b6(), nclasses) -} -pub fn b7(p: &nn::Path, nclasses: i64) -> impl ModuleT { - EfficientNet::new(p, MBConvConfig::b7(), nclasses) -} - -#[allow(clippy::many_single_char_names)] -pub fn conv2d_same(vs: nn::Path, i: i64, o: i64, k: i64, c: ConvConfig) -> impl Module { - let conv2d = nn::conv2d(vs, i, o, k, c); - let s = c.stride; - nn::func(move |xs| { - let size = xs.size(); - let ih = size[2]; - let iw = size[3]; - let oh = (ih + s - 1) / s; - let ow = (iw + s - 1) / s; - let pad_h = i64::max((oh - 1) * s + k - ih, 0); - let pad_w = i64::max((ow - 1) * s + k - iw, 0); - if pad_h > 0 || pad_w > 0 { - xs.zero_pad2d(pad_w / 2, pad_w - pad_w / 2, pad_h / 2, pad_h - pad_h / 2).apply(&conv2d) - } else { - xs.apply(&conv2d) - } - }) -} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/export_dinov2.py b/v2/dependency-patches/tch-0.24.0/src/vision/export_dinov2.py deleted file mode 100644 index b3cf1815a8..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/vision/export_dinov2.py +++ /dev/null @@ -1,16 +0,0 @@ -import torch -from safetensors.torch import save_file - -def normalize_key(k): - if k.startswith("backbone."): - k = k[9:] - if k.startswith("linear_head."): - k = k[7:] - return k - -for model_size in ["small", "base", "large", "giant"]: - letter = model_size[0] - dinov2_vits14 = torch.hub.load('facebookresearch/dinov2', f'dinov2_vit{letter}14_lc', layers=1) - weights = dinov2_vits14.state_dict() - weights = {normalize_key(k): v for k, v in weights.items()} - save_file(weights, f"dinov2_vit{letter}14.safetensors") diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/export_model.py b/v2/dependency-patches/tch-0.24.0/src/vision/export_model.py deleted file mode 100644 index e2fc3580b9..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/vision/export_model.py +++ /dev/null @@ -1,16 +0,0 @@ -# This script exports pre-trained model weights in the safetensors format. -import numpy as np -import torch -import torchvision -from safetensors import torch as stt - -m = torchvision.models.efficientnet_b0(pretrained=True) -stt.save_file(m.state_dict(), 'efficientnet-b0.safetensors') -m = torchvision.models.efficientnet_b1(pretrained=True) -stt.save_file(m.state_dict(), 'efficientnet-b1.safetensors') -m = torchvision.models.efficientnet_b2(pretrained=True) -stt.save_file(m.state_dict(), 'efficientnet-b2.safetensors') -m = torchvision.models.efficientnet_b3(pretrained=True) -stt.save_file(m.state_dict(), 'efficientnet-b3.safetensors') -m = torchvision.models.efficientnet_b4(pretrained=True) -stt.save_file(m.state_dict(), 'efficientnet-b4.safetensors') diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/image.rs b/v2/dependency-patches/tch-0.24.0/src/vision/image.rs deleted file mode 100644 index fcc830eb80..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/vision/image.rs +++ /dev/null @@ -1,146 +0,0 @@ -//! Utility functions to manipulate images. -use crate::wrappers::image::{load_hwc, load_hwc_from_mem, resize_hwc, save_hwc}; -use crate::{Device, TchError, Tensor}; -use std::io; -use std::path::Path; - -pub(crate) fn hwc_to_chw(tensor: &Tensor) -> Tensor { - tensor.permute([2, 0, 1]) -} - -pub(crate) fn chw_to_hwc(tensor: &Tensor) -> Tensor { - tensor.permute([1, 2, 0]) -} - -/// Loads an image from a file. -/// -/// On success returns a tensor of shape [channel, height, width]. -pub fn load>(path: T) -> Result { - let tensor = load_hwc(path)?; - Ok(hwc_to_chw(&tensor)) -} - -/// Loads an image from memory. -/// -/// On success returns a tensor of shape [channel, height, width]. -pub fn load_from_memory(img_data: &[u8]) -> Result { - let tensor = load_hwc_from_mem(img_data)?; - Ok(hwc_to_chw(&tensor)) -} - -/// Saves an image to a file. -/// -/// This expects as input a tensor of shape [channel, height, width]. -/// The image format is based on the filename suffix, supported suffixes -/// are jpg, png, tga, and bmp. -/// The tensor input should be of kind UInt8 with values ranging from -/// 0 to 255. -pub fn save>(t: &Tensor, path: T) -> Result<(), TchError> { - let t = t.to_kind(crate::Kind::Uint8); - match t.size().as_slice() { - [1, _, _, _] => save_hwc(&chw_to_hwc(&t.squeeze_dim(0)).to_device(Device::Cpu), path), - [_, _, _] => save_hwc(&chw_to_hwc(&t).to_device(Device::Cpu), path), - sz => Err(TchError::FileFormat(format!("unexpected size for image tensor {sz:?}"))), - } -} - -/// Resizes an image. -/// -/// This expects as input a tensor of shape [channel, height, width] and returns -/// a tensor of shape [channel, out_h, out_w]. -pub fn resize(t: &Tensor, out_w: i64, out_h: i64) -> Result { - Ok(hwc_to_chw(&resize_hwc(&chw_to_hwc(t), out_w, out_h)?)) -} - -pub fn resize_preserve_aspect_ratio_hwc( - t: &Tensor, - out_w: i64, - out_h: i64, -) -> Result { - let tensor_size = t.size(); - let (w, h) = (tensor_size[0], tensor_size[1]); - if w * out_h == h * out_w { - Ok(hwc_to_chw(&resize_hwc(t, out_w, out_h)?)) - } else { - let (resize_w, resize_h) = { - let ratio_w = out_w as f64 / w as f64; - let ratio_h = out_h as f64 / h as f64; - let ratio = ratio_w.max(ratio_h); - ((ratio * h as f64) as i64, (ratio * w as f64) as i64) - }; - let resize_w = i64::max(resize_w, out_w); - let resize_h = i64::max(resize_h, out_h); - let t = hwc_to_chw(&resize_hwc(t, resize_w, resize_h)?); - let t = if resize_w == out_w { t } else { t.f_narrow(2, (resize_w - out_w) / 2, out_w)? }; - let t = if resize_h == out_h { t } else { t.f_narrow(1, (resize_h - out_h) / 2, out_h)? }; - Ok(t) - } -} - -/// Resize an image, preserve the aspect ratio by taking a center crop. -/// -/// This expects as input a tensor of shape [channel, height, width] and returns -pub fn resize_preserve_aspect_ratio( - t: &Tensor, - out_w: i64, - out_h: i64, -) -> Result { - resize_preserve_aspect_ratio_hwc(&chw_to_hwc(t), out_w, out_h) -} - -/// Loads and resize an image, preserve the aspect ratio by taking a center crop. -pub fn load_and_resize>( - path: T, - out_w: i64, - out_h: i64, -) -> Result { - let tensor = load_hwc(path)?; - resize_preserve_aspect_ratio_hwc(&tensor, out_w, out_h) -} - -/// Loads and resize an image from memory, preserve the aspect ratio by taking a center crop. -pub fn load_and_resize_from_memory( - img_data: &[u8], - out_w: i64, - out_h: i64, -) -> Result { - let tensor = load_hwc_from_mem(img_data)?; - resize_preserve_aspect_ratio_hwc(&tensor, out_w, out_h) -} - -fn visit_dirs(dir: &Path, files: &mut Vec) -> Result<(), TchError> { - if dir.is_dir() { - for entry in std::fs::read_dir(dir)? { - let entry = entry?; - let path = entry.path(); - if path.is_dir() { - visit_dirs(&path, files)?; - } else if entry - .file_name() - .to_str() - .is_some_and(|s| s.ends_with(".png") || s.ends_with(".jpg")) - { - files.push(entry); - } - } - } - Ok(()) -} - -/// Loads all the images in a directory. -pub fn load_dir>(path: T, out_w: i64, out_h: i64) -> Result { - let mut files: Vec = vec![]; - visit_dirs(path.as_ref(), &mut files)?; - if files.is_empty() { - return Err(TchError::Io(io::Error::new( - io::ErrorKind::NotFound, - format!("no image found in {:?}", path.as_ref(),), - ))); - } - let v: Vec<_> = files - .iter() - // Silently discard image errors. - .filter_map(|x| load_and_resize(x.path(), out_w, out_h).ok()) - .collect(); - Ok(Tensor::stack(&v, 0)) -} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/imagenet.rs b/v2/dependency-patches/tch-0.24.0/src/vision/imagenet.rs deleted file mode 100644 index e4b93d4060..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/vision/imagenet.rs +++ /dev/null @@ -1,1170 +0,0 @@ -//! Helper functions for ImageNet like datasets. -use super::dataset::Dataset; -use crate::{kind, Device, Kind, TchError, Tensor}; -use std::io; -use std::path::Path; -use std::sync::Mutex; - -lazy_static! { - static ref IMAGENET_MEAN: Mutex = - Mutex::new(Tensor::from_slice(&[0.485f32, 0.456, 0.406]).view((3, 1, 1))); - static ref IMAGENET_STD: Mutex = - Mutex::new(Tensor::from_slice(&[0.229f32, 0.224, 0.225]).view((3, 1, 1))); -} - -pub fn normalize(tensor: &Tensor) -> Result { - let mean = IMAGENET_MEAN.lock().unwrap(); - let std = IMAGENET_STD.lock().unwrap(); - (tensor.to_kind(Kind::Float) / 255.0).f_sub(&mean)?.f_div(&std) -} - -pub fn unnormalize(tensor: &Tensor) -> Result { - let mean = IMAGENET_MEAN.lock().unwrap(); - let std = IMAGENET_STD.lock().unwrap(); - let tensor = (tensor.f_mul(&std)?.f_add(&mean)? * 255.0).clamp(0., 255.0).to_kind(Kind::Uint8); - Ok(tensor) -} - -/// Saves an image to a path. -/// This unapplies the ImageNet normalization. -pub fn save_image>(tensor: &Tensor, path: T) -> Result<(), TchError> { - super::image::save(&unnormalize(&tensor.to_device(Device::Cpu))?, path) -} - -/// Loads an image from a file and applies the ImageNet normalization. -pub fn load_image>(path: T) -> Result { - normalize(&super::image::load(path)?) -} - -/// Loads an image from memory and applies the ImageNet normalization. -pub fn load_image_from_memory(img_data: &[u8]) -> Result { - normalize(&super::image::load_from_memory(img_data)?) -} - -/// Loads an image from a file and resize it to 224x224. -/// This applies the ImageNet normalization. -pub fn load_image_and_resize224>(path: T) -> Result { - normalize(&super::image::load_and_resize(path, 224, 224)?) -} - -/// Loads an image from memory and resize it to 224x224. -/// This applies the ImageNet normalization. -pub fn load_image_and_resize224_from_memory(img_data: &[u8]) -> Result { - normalize(&super::image::load_and_resize_from_memory(img_data, 224, 224)?) -} - -/// Loads an image from a file and resize it to the specified width and height. -/// This applies the ImageNet normalization. -pub fn load_image_and_resize>(path: T, w: i64, h: i64) -> Result { - normalize(&super::image::load_and_resize(path, w, h)?) -} - -/// Loads an image from memory and resize it to the specified width and height. -/// This applies the ImageNet normalization. -pub fn load_image_and_resize_from_memory( - img_data: &[u8], - w: i64, - h: i64, -) -> Result { - normalize(&super::image::load_and_resize_from_memory(img_data, w, h)?) -} - -fn has_image_suffix>(path: T) -> bool { - match path.as_ref().extension() { - None => false, - Some(extension) => match extension.to_str() { - Some("jpg") | Some("jpeg") | Some("png") | Some("JPG") | Some("JPEG") | Some("PNG") => { - true - } - Some(_) | None => false, - }, - } -} - -fn load_images_from_dir(dir: std::path::PathBuf) -> Result { - let mut images: Vec = vec![]; - for image in std::fs::read_dir(&dir)? { - match image { - Err(_) => {} - Ok(image) => { - let image = image.path(); - if !has_image_suffix(&image) { - continue; - } - match load_image_and_resize224(image) { - Err(_) => {} - Ok(image) => images.push(image), - } - } - } - } - if images.is_empty() { - return Err(TchError::Io(io::Error::new( - io::ErrorKind::NotFound, - format!("no image found in {dir:?}"), - ))); - } - Tensor::f_stack(&images, 0) -} - -/// Loads a dataset from a directory. -/// -/// This assumes that the directory contains two subdirectories named train and val. -/// In each of these datasets, there should be a subdirectory per class named -/// in the same way. -/// The ImageNet normalization is applied, image are resized to 224x224. -pub fn load_from_dir>(dir: T) -> Result { - let train_path = dir.as_ref().join("train"); - let valid_path = dir.as_ref().join("val"); - let classes = std::fs::read_dir(&valid_path)? - .filter_map(|d| d.ok().map(|d| d.path())) - .filter(|d| d.is_dir()) - .filter_map(|d| d.file_name().map(|d| d.to_os_string())) - .collect::>(); - println!("classes: {classes:?}"); - let mut train_images: Vec = vec![]; - let mut train_labels: Vec = vec![]; - let mut test_images: Vec = vec![]; - let mut test_labels: Vec = vec![]; - for (label_index, label_dir) in classes.iter().enumerate() { - let label_index = label_index as i64; - let images = load_images_from_dir(train_path.join(label_dir))?; - let nimages = images.size()[0]; - train_images.push(images); - train_labels.push(Tensor::ones([nimages], kind::INT64_CPU) * label_index); - let images = load_images_from_dir(valid_path.join(label_dir))?; - let nimages = images.size()[0]; - test_images.push(images); - test_labels.push(Tensor::ones([nimages], kind::INT64_CPU) * label_index); - } - Ok(Dataset { - train_images: Tensor::f_cat(&train_images, 0)?, - train_labels: Tensor::f_cat(&train_labels, 0)?, - test_images: Tensor::f_cat(&test_images, 0)?, - test_labels: Tensor::f_cat(&test_labels, 0)?, - labels: classes.len() as i64, - }) -} - -pub const CLASS_COUNT: i64 = 1000; - -pub const CLASSES: [&str; 1000] = [ - "tench, Tinca tinca", - "goldfish, Carassius auratus", - "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", - "tiger shark, Galeocerdo cuvieri", - "hammerhead, hammerhead shark", - "electric ray, crampfish, numbfish, torpedo", - "stingray", - "cock", - "hen", - "ostrich, Struthio camelus", - "brambling, Fringilla montifringilla", - "goldfinch, Carduelis carduelis", - "house finch, linnet, Carpodacus mexicanus", - "junco, snowbird", - "indigo bunting, indigo finch, indigo bird, Passerina cyanea", - "robin, American robin, Turdus migratorius", - "bulbul", - "jay", - "magpie", - "chickadee", - "water ouzel, dipper", - "kite", - "bald eagle, American eagle, Haliaeetus leucocephalus", - "vulture", - "great grey owl, great gray owl, Strix nebulosa", - "European fire salamander, Salamandra salamandra", - "common newt, Triturus vulgaris", - "eft", - "spotted salamander, Ambystoma maculatum", - "axolotl, mud puppy, Ambystoma mexicanum", - "bullfrog, Rana catesbeiana", - "tree frog, tree-frog", - "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", - "loggerhead, loggerhead turtle, Caretta caretta", - "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", - "mud turtle", - "terrapin", - "box turtle, box tortoise", - "banded gecko", - "common iguana, iguana, Iguana iguana", - "American chameleon, anole, Anolis carolinensis", - "whiptail, whiptail lizard", - "agama", - "frilled lizard, Chlamydosaurus kingi", - "alligator lizard", - "Gila monster, Heloderma suspectum", - "green lizard, Lacerta viridis", - "African chameleon, Chamaeleo chamaeleon", - "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", - "African crocodile, Nile crocodile, Crocodylus niloticus", - "American alligator, Alligator mississipiensis", - "triceratops", - "thunder snake, worm snake, Carphophis amoenus", - "ringneck snake, ring-necked snake, ring snake", - "hognose snake, puff adder, sand viper", - "green snake, grass snake", - "king snake, kingsnake", - "garter snake, grass snake", - "water snake", - "vine snake", - "night snake, Hypsiglena torquata", - "boa constrictor, Constrictor constrictor", - "rock python, rock snake, Python sebae", - "Indian cobra, Naja naja", - "green mamba", - "sea snake", - "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", - "diamondback, diamondback rattlesnake, Crotalus adamanteus", - "sidewinder, horned rattlesnake, Crotalus cerastes", - "trilobite", - "harvestman, daddy longlegs, Phalangium opilio", - "scorpion", - "black and gold garden spider, Argiope aurantia", - "barn spider, Araneus cavaticus", - "garden spider, Aranea diademata", - "black widow, Latrodectus mactans", - "tarantula", - "wolf spider, hunting spider", - "tick", - "centipede", - "black grouse", - "ptarmigan", - "ruffed grouse, partridge, Bonasa umbellus", - "prairie chicken, prairie grouse, prairie fowl", - "peacock", - "quail", - "partridge", - "African grey, African gray, Psittacus erithacus", - "macaw", - "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", - "lorikeet", - "coucal", - "bee eater", - "hornbill", - "hummingbird", - "jacamar", - "toucan", - "drake", - "red-breasted merganser, Mergus serrator", - "goose", - "black swan, Cygnus atratus", - "tusker", - "echidna, spiny anteater, anteater", - "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", - "wallaby, brush kangaroo", - "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", - "wombat", - "jellyfish", - "sea anemone, anemone", - "brain coral", - "flatworm, platyhelminth", - "nematode, nematode worm, roundworm", - "conch", - "snail", - "slug", - "sea slug, nudibranch", - "chiton, coat-of-mail shell, sea cradle, polyplacophore", - "chambered nautilus, pearly nautilus, nautilus", - "Dungeness crab, Cancer magister", - "rock crab, Cancer irroratus", - "fiddler crab", - "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", - "American lobster, Northern lobster, Maine lobster, Homarus americanus", - "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", - "crayfish, crawfish, crawdad, crawdaddy", - "hermit crab", - "isopod", - "white stork, Ciconia ciconia", - "black stork, Ciconia nigra", - "spoonbill", - "flamingo", - "little blue heron, Egretta caerulea", - "American egret, great white heron, Egretta albus", - "bittern", - "crane", - "limpkin, Aramus pictus", - "European gallinule, Porphyrio porphyrio", - "American coot, marsh hen, mud hen, water hen, Fulica americana", - "bustard", - "ruddy turnstone, Arenaria interpres", - "red-backed sandpiper, dunlin, Erolia alpina", - "redshank, Tringa totanus", - "dowitcher", - "oystercatcher, oyster catcher", - "pelican", - "king penguin, Aptenodytes patagonica", - "albatross, mollymawk", - "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", - "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", - "dugong, Dugong dugon", - "sea lion", - "Chihuahua", - "Japanese spaniel", - "Maltese dog, Maltese terrier, Maltese", - "Pekinese, Pekingese, Peke", - "Shih-Tzu", - "Blenheim spaniel", - "papillon", - "toy terrier", - "Rhodesian ridgeback", - "Afghan hound, Afghan", - "basset, basset hound", - "beagle", - "bloodhound, sleuthhound", - "bluetick", - "black-and-tan coonhound", - "Walker hound, Walker foxhound", - "English foxhound", - "redbone", - "borzoi, Russian wolfhound", - "Irish wolfhound", - "Italian greyhound", - "whippet", - "Ibizan hound, Ibizan Podenco", - "Norwegian elkhound, elkhound", - "otterhound, otter hound", - "Saluki, gazelle hound", - "Scottish deerhound, deerhound", - "Weimaraner", - "Staffordshire bullterrier, Staffordshire bull terrier", - "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", - "Bedlington terrier", - "Border terrier", - "Kerry blue terrier", - "Irish terrier", - "Norfolk terrier", - "Norwich terrier", - "Yorkshire terrier", - "wire-haired fox terrier", - "Lakeland terrier", - "Sealyham terrier, Sealyham", - "Airedale, Airedale terrier", - "cairn, cairn terrier", - "Australian terrier", - "Dandie Dinmont, Dandie Dinmont terrier", - "Boston bull, Boston terrier", - "miniature schnauzer", - "giant schnauzer", - "standard schnauzer", - "Scotch terrier, Scottish terrier, Scottie", - "Tibetan terrier, chrysanthemum dog", - "silky terrier, Sydney silky", - "soft-coated wheaten terrier", - "West Highland white terrier", - "Lhasa, Lhasa apso", - "flat-coated retriever", - "curly-coated retriever", - "golden retriever", - "Labrador retriever", - "Chesapeake Bay retriever", - "German short-haired pointer", - "vizsla, Hungarian pointer", - "English setter", - "Irish setter, red setter", - "Gordon setter", - "Brittany spaniel", - "clumber, clumber spaniel", - "English springer, English springer spaniel", - "Welsh springer spaniel", - "cocker spaniel, English cocker spaniel, cocker", - "Sussex spaniel", - "Irish water spaniel", - "kuvasz", - "schipperke", - "groenendael", - "malinois", - "briard", - "kelpie", - "komondor", - "Old English sheepdog, bobtail", - "Shetland sheepdog, Shetland sheep dog, Shetland", - "collie", - "Border collie", - "Bouvier des Flandres, Bouviers des Flandres", - "Rottweiler", - "German shepherd, German shepherd dog, German police dog, alsatian", - "Doberman, Doberman pinscher", - "miniature pinscher", - "Greater Swiss Mountain dog", - "Bernese mountain dog", - "Appenzeller", - "EntleBucher", - "boxer", - "bull mastiff", - "Tibetan mastiff", - "French bulldog", - "Great Dane", - "Saint Bernard, St Bernard", - "Eskimo dog, husky", - "malamute, malemute, Alaskan malamute", - "Siberian husky", - "dalmatian, coach dog, carriage dog", - "affenpinscher, monkey pinscher, monkey dog", - "basenji", - "pug, pug-dog", - "Leonberg", - "Newfoundland, Newfoundland dog", - "Great Pyrenees", - "Samoyed, Samoyede", - "Pomeranian", - "chow, chow chow", - "keeshond", - "Brabancon griffon", - "Pembroke, Pembroke Welsh corgi", - "Cardigan, Cardigan Welsh corgi", - "toy poodle", - "miniature poodle", - "standard poodle", - "Mexican hairless", - "timber wolf, grey wolf, gray wolf, Canis lupus", - "white wolf, Arctic wolf, Canis lupus tundrarum", - "red wolf, maned wolf, Canis rufus, Canis niger", - "coyote, prairie wolf, brush wolf, Canis latrans", - "dingo, warrigal, warragal, Canis dingo", - "dhole, Cuon alpinus", - "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", - "hyena, hyaena", - "red fox, Vulpes vulpes", - "kit fox, Vulpes macrotis", - "Arctic fox, white fox, Alopex lagopus", - "grey fox, gray fox, Urocyon cinereoargenteus", - "tabby, tabby cat", - "tiger cat", - "Persian cat", - "Siamese cat, Siamese", - "Egyptian cat", - "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", - "lynx, catamount", - "leopard, Panthera pardus", - "snow leopard, ounce, Panthera uncia", - "jaguar, panther, Panthera onca, Felis onca", - "lion, king of beasts, Panthera leo", - "tiger, Panthera tigris", - "cheetah, chetah, Acinonyx jubatus", - "brown bear, bruin, Ursus arctos", - "American black bear, black bear, Ursus americanus, Euarctos americanus", - "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", - "sloth bear, Melursus ursinus, Ursus ursinus", - "mongoose", - "meerkat, mierkat", - "tiger beetle", - "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", - "ground beetle, carabid beetle", - "long-horned beetle, longicorn, longicorn beetle", - "leaf beetle, chrysomelid", - "dung beetle", - "rhinoceros beetle", - "weevil", - "fly", - "bee", - "ant, emmet, pismire", - "grasshopper, hopper", - "cricket", - "walking stick, walkingstick, stick insect", - "cockroach, roach", - "mantis, mantid", - "cicada, cicala", - "leafhopper", - "lacewing, lacewing fly", - "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", - "damselfly", - "admiral", - "ringlet, ringlet butterfly", - "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", - "cabbage butterfly", - "sulphur butterfly, sulfur butterfly", - "lycaenid, lycaenid butterfly", - "starfish, sea star", - "sea urchin", - "sea cucumber, holothurian", - "wood rabbit, cottontail, cottontail rabbit", - "hare", - "Angora, Angora rabbit", - "hamster", - "porcupine, hedgehog", - "fox squirrel, eastern fox squirrel, Sciurus niger", - "marmot", - "beaver", - "guinea pig, Cavia cobaya", - "sorrel", - "zebra", - "hog, pig, grunter, squealer, Sus scrofa", - "wild boar, boar, Sus scrofa", - "warthog", - "hippopotamus, hippo, river horse, Hippopotamus amphibius", - "ox", - "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", - "bison", - "ram, tup", - "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", - "ibex, Capra ibex", - "hartebeest", - "impala, Aepyceros melampus", - "gazelle", - "Arabian camel, dromedary, Camelus dromedarius", - "llama", - "weasel", - "mink", - "polecat, fitch, foulmart, foumart, Mustela putorius", - "black-footed ferret, ferret, Mustela nigripes", - "otter", - "skunk, polecat, wood pussy", - "badger", - "armadillo", - "three-toed sloth, ai, Bradypus tridactylus", - "orangutan, orang, orangutang, Pongo pygmaeus", - "gorilla, Gorilla gorilla", - "chimpanzee, chimp, Pan troglodytes", - "gibbon, Hylobates lar", - "siamang, Hylobates syndactylus, Symphalangus syndactylus", - "guenon, guenon monkey", - "patas, hussar monkey, Erythrocebus patas", - "baboon", - "macaque", - "langur", - "colobus, colobus monkey", - "proboscis monkey, Nasalis larvatus", - "marmoset", - "capuchin, ringtail, Cebus capucinus", - "howler monkey, howler", - "titi, titi monkey", - "spider monkey, Ateles geoffroyi", - "squirrel monkey, Saimiri sciureus", - "Madagascar cat, ring-tailed lemur, Lemur catta", - "indri, indris, Indri indri, Indri brevicaudatus", - "Indian elephant, Elephas maximus", - "African elephant, Loxodonta africana", - "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", - "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", - "barracouta, snoek", - "eel", - "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", - "rock beauty, Holocanthus tricolor", - "anemone fish", - "sturgeon", - "gar, garfish, garpike, billfish, Lepisosteus osseus", - "lionfish", - "puffer, pufferfish, blowfish, globefish", - "abacus", - "abaya", - "academic gown, academic robe, judge's robe", - "accordion, piano accordion, squeeze box", - "acoustic guitar", - "aircraft carrier, carrier, flattop, attack aircraft carrier", - "airliner", - "airship, dirigible", - "altar", - "ambulance", - "amphibian, amphibious vehicle", - "analog clock", - "apiary, bee house", - "apron", - "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", - "assault rifle, assault gun", - "backpack, back pack, knapsack, packsack, rucksack, haversack", - "bakery, bakeshop, bakehouse", - "balance beam, beam", - "balloon", - "ballpoint, ballpoint pen, ballpen, Biro", - "Band Aid", - "banjo", - "bannister, banister, balustrade, balusters, handrail", - "barbell", - "barber chair", - "barbershop", - "barn", - "barometer", - "barrel, cask", - "barrow, garden cart, lawn cart, wheelbarrow", - "baseball", - "basketball", - "bassinet", - "bassoon", - "bathing cap, swimming cap", - "bath towel", - "bathtub, bathing tub, bath, tub", - "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", - "beacon, lighthouse, beacon light, pharos", - "beaker", - "bearskin, busby, shako", - "beer bottle", - "beer glass", - "bell cote, bell cot", - "bib", - "bicycle-built-for-two, tandem bicycle, tandem", - "bikini, two-piece", - "binder, ring-binder", - "binoculars, field glasses, opera glasses", - "birdhouse", - "boathouse", - "bobsled, bobsleigh, bob", - "bolo tie, bolo, bola tie, bola", - "bonnet, poke bonnet", - "bookcase", - "bookshop, bookstore, bookstall", - "bottlecap", - "bow", - "bow tie, bow-tie, bowtie", - "brass, memorial tablet, plaque", - "brassiere, bra, bandeau", - "breakwater, groin, groyne, mole, bulwark, seawall, jetty", - "breastplate, aegis, egis", - "broom", - "bucket, pail", - "buckle", - "bulletproof vest", - "bullet train, bullet", - "butcher shop, meat market", - "cab, hack, taxi, taxicab", - "caldron, cauldron", - "candle, taper, wax light", - "cannon", - "canoe", - "can opener, tin opener", - "cardigan", - "car mirror", - "carousel, carrousel, merry-go-round, roundabout, whirligig", - "carpenter's kit, tool kit", - "carton", - "car wheel", - "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", - "cassette", - "cassette player", - "castle", - "catamaran", - "CD player", - "cello, violoncello", - "cellular telephone, cellular phone, cellphone, cell, mobile phone", - "chain", - "chainlink fence", - "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", - "chain saw, chainsaw", - "chest", - "chiffonier, commode", - "chime, bell, gong", - "china cabinet, china closet", - "Christmas stocking", - "church, church building", - "cinema, movie theater, movie theatre, movie house, picture palace", - "cleaver, meat cleaver, chopper", - "cliff dwelling", - "cloak", - "clog, geta, patten, sabot", - "cocktail shaker", - "coffee mug", - "coffeepot", - "coil, spiral, volute, whorl, helix", - "combination lock", - "computer keyboard, keypad", - "confectionery, confectionary, candy store", - "container ship, containership, container vessel", - "convertible", - "corkscrew, bottle screw", - "cornet, horn, trumpet, trump", - "cowboy boot", - "cowboy hat, ten-gallon hat", - "cradle", - "crane", - "crash helmet", - "crate", - "crib, cot", - "Crock Pot", - "croquet ball", - "crutch", - "cuirass", - "dam, dike, dyke", - "desk", - "desktop computer", - "dial telephone, dial phone", - "diaper, nappy, napkin", - "digital clock", - "digital watch", - "dining table, board", - "dishrag, dishcloth", - "dishwasher, dish washer, dishwashing machine", - "disk brake, disc brake", - "dock, dockage, docking facility", - "dogsled, dog sled, dog sleigh", - "dome", - "doormat, welcome mat", - "drilling platform, offshore rig", - "drum, membranophone, tympan", - "drumstick", - "dumbbell", - "Dutch oven", - "electric fan, blower", - "electric guitar", - "electric locomotive", - "entertainment center", - "envelope", - "espresso maker", - "face powder", - "feather boa, boa", - "file, file cabinet, filing cabinet", - "fireboat", - "fire engine, fire truck", - "fire screen, fireguard", - "flagpole, flagstaff", - "flute, transverse flute", - "folding chair", - "football helmet", - "forklift", - "fountain", - "fountain pen", - "four-poster", - "freight car", - "French horn, horn", - "frying pan, frypan, skillet", - "fur coat", - "garbage truck, dustcart", - "gasmask, respirator, gas helmet", - "gas pump, gasoline pump, petrol pump, island dispenser", - "goblet", - "go-kart", - "golf ball", - "golfcart, golf cart", - "gondola", - "gong, tam-tam", - "gown", - "grand piano, grand", - "greenhouse, nursery, glasshouse", - "grille, radiator grille", - "grocery store, grocery, food market, market", - "guillotine", - "hair slide", - "hair spray", - "half track", - "hammer", - "hamper", - "hand blower, blow dryer, blow drier, hair dryer, hair drier", - "hand-held computer, hand-held microcomputer", - "handkerchief, hankie, hanky, hankey", - "hard disc, hard disk, fixed disk", - "harmonica, mouth organ, harp, mouth harp", - "harp", - "harvester, reaper", - "hatchet", - "holster", - "home theater, home theatre", - "honeycomb", - "hook, claw", - "hoopskirt, crinoline", - "horizontal bar, high bar", - "horse cart, horse-cart", - "hourglass", - "iPod", - "iron, smoothing iron", - "jack-o'-lantern", - "jean, blue jean, denim", - "jeep, landrover", - "jersey, T-shirt, tee shirt", - "jigsaw puzzle", - "jinrikisha, ricksha, rickshaw", - "joystick", - "kimono", - "knee pad", - "knot", - "lab coat, laboratory coat", - "ladle", - "lampshade, lamp shade", - "laptop, laptop computer", - "lawn mower, mower", - "lens cap, lens cover", - "letter opener, paper knife, paperknife", - "library", - "lifeboat", - "lighter, light, igniter, ignitor", - "limousine, limo", - "liner, ocean liner", - "lipstick, lip rouge", - "Loafer", - "lotion", - "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", - "loupe, jeweler's loupe", - "lumbermill, sawmill", - "magnetic compass", - "mailbag, postbag", - "mailbox, letter box", - "maillot", - "maillot, tank suit", - "manhole cover", - "maraca", - "marimba, xylophone", - "mask", - "matchstick", - "maypole", - "maze, labyrinth", - "measuring cup", - "medicine chest, medicine cabinet", - "megalith, megalithic structure", - "microphone, mike", - "microwave, microwave oven", - "military uniform", - "milk can", - "minibus", - "miniskirt, mini", - "minivan", - "missile", - "mitten", - "mixing bowl", - "mobile home, manufactured home", - "Model T", - "modem", - "monastery", - "monitor", - "moped", - "mortar", - "mortarboard", - "mosque", - "mosquito net", - "motor scooter, scooter", - "mountain bike, all-terrain bike, off-roader", - "mountain tent", - "mouse, computer mouse", - "mousetrap", - "moving van", - "muzzle", - "nail", - "neck brace", - "necklace", - "nipple", - "notebook, notebook computer", - "obelisk", - "oboe, hautboy, hautbois", - "ocarina, sweet potato", - "odometer, hodometer, mileometer, milometer", - "oil filter", - "organ, pipe organ", - "oscilloscope, scope, cathode-ray oscilloscope, CRO", - "overskirt", - "oxcart", - "oxygen mask", - "packet", - "paddle, boat paddle", - "paddlewheel, paddle wheel", - "padlock", - "paintbrush", - "pajama, pyjama, pj's, jammies", - "palace", - "panpipe, pandean pipe, syrinx", - "paper towel", - "parachute, chute", - "parallel bars, bars", - "park bench", - "parking meter", - "passenger car, coach, carriage", - "patio, terrace", - "pay-phone, pay-station", - "pedestal, plinth, footstall", - "pencil box, pencil case", - "pencil sharpener", - "perfume, essence", - "Petri dish", - "photocopier", - "pick, plectrum, plectron", - "pickelhaube", - "picket fence, paling", - "pickup, pickup truck", - "pier", - "piggy bank, penny bank", - "pill bottle", - "pillow", - "ping-pong ball", - "pinwheel", - "pirate, pirate ship", - "pitcher, ewer", - "plane, carpenter's plane, woodworking plane", - "planetarium", - "plastic bag", - "plate rack", - "plow, plough", - "plunger, plumber's helper", - "Polaroid camera, Polaroid Land camera", - "pole", - "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", - "poncho", - "pool table, billiard table, snooker table", - "pop bottle, soda bottle", - "pot, flowerpot", - "potter's wheel", - "power drill", - "prayer rug, prayer mat", - "printer", - "prison, prison house", - "projectile, missile", - "projector", - "puck, hockey puck", - "punching bag, punch bag, punching ball, punchball", - "purse", - "quill, quill pen", - "quilt, comforter, comfort, puff", - "racer, race car, racing car", - "racket, racquet", - "radiator", - "radio, wireless", - "radio telescope, radio reflector", - "rain barrel", - "recreational vehicle, RV, R.V.", - "reel", - "reflex camera", - "refrigerator, icebox", - "remote control, remote", - "restaurant, eating house, eating place, eatery", - "revolver, six-gun, six-shooter", - "rifle", - "rocking chair, rocker", - "rotisserie", - "rubber eraser, rubber, pencil eraser", - "rugby ball", - "rule, ruler", - "running shoe", - "safe", - "safety pin", - "saltshaker, salt shaker", - "sandal", - "sarong", - "sax, saxophone", - "scabbard", - "scale, weighing machine", - "school bus", - "schooner", - "scoreboard", - "screen, CRT screen", - "screw", - "screwdriver", - "seat belt, seatbelt", - "sewing machine", - "shield, buckler", - "shoe shop, shoe-shop, shoe store", - "shoji", - "shopping basket", - "shopping cart", - "shovel", - "shower cap", - "shower curtain", - "ski", - "ski mask", - "sleeping bag", - "slide rule, slipstick", - "sliding door", - "slot, one-armed bandit", - "snorkel", - "snowmobile", - "snowplow, snowplough", - "soap dispenser", - "soccer ball", - "sock", - "solar dish, solar collector, solar furnace", - "sombrero", - "soup bowl", - "space bar", - "space heater", - "space shuttle", - "spatula", - "speedboat", - "spider web, spider's web", - "spindle", - "sports car, sport car", - "spotlight, spot", - "stage", - "steam locomotive", - "steel arch bridge", - "steel drum", - "stethoscope", - "stole", - "stone wall", - "stopwatch, stop watch", - "stove", - "strainer", - "streetcar, tram, tramcar, trolley, trolley car", - "stretcher", - "studio couch, day bed", - "stupa, tope", - "submarine, pigboat, sub, U-boat", - "suit, suit of clothes", - "sundial", - "sunglass", - "sunglasses, dark glasses, shades", - "sunscreen, sunblock, sun blocker", - "suspension bridge", - "swab, swob, mop", - "sweatshirt", - "swimming trunks, bathing trunks", - "swing", - "switch, electric switch, electrical switch", - "syringe", - "table lamp", - "tank, army tank, armored combat vehicle, armoured combat vehicle", - "tape player", - "teapot", - "teddy, teddy bear", - "television, television system", - "tennis ball", - "thatch, thatched roof", - "theater curtain, theatre curtain", - "thimble", - "thresher, thrasher, threshing machine", - "throne", - "tile roof", - "toaster", - "tobacco shop, tobacconist shop, tobacconist", - "toilet seat", - "torch", - "totem pole", - "tow truck, tow car, wrecker", - "toyshop", - "tractor", - "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", - "tray", - "trench coat", - "tricycle, trike, velocipede", - "trimaran", - "tripod", - "triumphal arch", - "trolleybus, trolley coach, trackless trolley", - "trombone", - "tub, vat", - "turnstile", - "typewriter keyboard", - "umbrella", - "unicycle, monocycle", - "upright, upright piano", - "vacuum, vacuum cleaner", - "vase", - "vault", - "velvet", - "vending machine", - "vestment", - "viaduct", - "violin, fiddle", - "volleyball", - "waffle iron", - "wall clock", - "wallet, billfold, notecase, pocketbook", - "wardrobe, closet, press", - "warplane, military plane", - "washbasin, handbasin, washbowl, lavabo, wash-hand basin", - "washer, automatic washer, washing machine", - "water bottle", - "water jug", - "water tower", - "whiskey jug", - "whistle", - "wig", - "window screen", - "window shade", - "Windsor tie", - "wine bottle", - "wing", - "wok", - "wooden spoon", - "wool, woolen, woollen", - "worm fence, snake fence, snake-rail fence, Virginia fence", - "wreck", - "yawl", - "yurt", - "web site, website, internet site, site", - "comic book", - "crossword puzzle, crossword", - "street sign", - "traffic light, traffic signal, stoplight", - "book jacket, dust cover, dust jacket, dust wrapper", - "menu", - "plate", - "guacamole", - "consomme", - "hot pot, hotpot", - "trifle", - "ice cream, icecream", - "ice lolly, lolly, lollipop, popsicle", - "French loaf", - "bagel, beigel", - "pretzel", - "cheeseburger", - "hotdog, hot dog, red hot", - "mashed potato", - "head cabbage", - "broccoli", - "cauliflower", - "zucchini, courgette", - "spaghetti squash", - "acorn squash", - "butternut squash", - "cucumber, cuke", - "artichoke, globe artichoke", - "bell pepper", - "cardoon", - "mushroom", - "Granny Smith", - "strawberry", - "orange", - "lemon", - "fig", - "pineapple, ananas", - "banana", - "jackfruit, jak, jack", - "custard apple", - "pomegranate", - "hay", - "carbonara", - "chocolate sauce, chocolate syrup", - "dough", - "meat loaf, meatloaf", - "pizza, pizza pie", - "potpie", - "burrito", - "red wine", - "espresso", - "cup", - "eggnog", - "alp", - "bubble", - "cliff, drop, drop-off", - "coral reef", - "geyser", - "lakeside, lakeshore", - "promontory, headland, head, foreland", - "sandbar, sand bar", - "seashore, coast, seacoast, sea-coast", - "valley, vale", - "volcano", - "ballplayer, baseball player", - "groom, bridegroom", - "scuba diver", - "rapeseed", - "daisy", - "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", - "corn", - "acorn", - "hip, rose hip, rosehip", - "buckeye, horse chestnut, conker", - "coral fungus", - "agaric", - "gyromitra", - "stinkhorn, carrion fungus", - "earthstar", - "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", - "bolete", - "ear, spike, capitulum", - "toilet tissue, toilet paper, bathroom tissue", -]; - -/// Returns the top k classes as well as the associated scores. -pub fn top(tensor: &Tensor, k: i64) -> Vec<(f64, String)> { - let tensor = match tensor.size().as_slice() { - [CLASS_COUNT] => tensor.shallow_clone(), - [1, CLASS_COUNT] => tensor.view((CLASS_COUNT,)), - [1, 1, CLASS_COUNT] => tensor.view((CLASS_COUNT,)), - _ => panic!("unexpected tensor shape {tensor:?}"), - }; - let (values, indexes) = tensor.topk(k, 0, true, true); - let values = Vec::::try_from(values).unwrap(); - let indexes = Vec::::try_from(indexes).unwrap(); - values - .iter() - .zip(indexes.iter()) - .map(|(&value, &index)| (value, CLASSES[index as usize].to_owned())) - .collect() -} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/inception.rs b/v2/dependency-patches/tch-0.24.0/src/vision/inception.rs deleted file mode 100644 index 7bc8738f57..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/vision/inception.rs +++ /dev/null @@ -1,154 +0,0 @@ -//! InceptionV3. -use crate::{nn, nn::ModuleT, Tensor}; - -fn conv_bn(p: nn::Path, c_in: i64, c_out: i64, ksize: i64, pad: i64, stride: i64) -> impl ModuleT { - let conv2d_cfg = nn::ConvConfig { stride, padding: pad, bias: false, ..Default::default() }; - let bn_cfg = nn::BatchNormConfig { eps: 0.001, ..Default::default() }; - nn::seq_t() - .add(nn::conv2d(&p / "conv", c_in, c_out, ksize, conv2d_cfg)) - .add(nn::batch_norm2d(&p / "bn", c_out, bn_cfg)) - .add_fn(|xs| xs.relu()) -} - -fn conv_bn2(p: nn::Path, c_in: i64, c_out: i64, ksize: [i64; 2], pad: [i64; 2]) -> impl ModuleT { - let conv2d_cfg = - nn::ConvConfigND::<[i64; 2]> { padding: pad, bias: false, ..Default::default() }; - let bn_cfg = nn::BatchNormConfig { eps: 0.001, ..Default::default() }; - nn::seq_t() - .add(nn::conv(&p / "conv", c_in, c_out, ksize, conv2d_cfg)) - .add(nn::batch_norm2d(&p / "bn", c_out, bn_cfg)) - .add_fn(|xs| xs.relu()) -} - -fn max_pool2d(xs: &Tensor, ksize: i64, stride: i64) -> Tensor { - xs.max_pool2d([ksize, ksize], [stride, stride], [0, 0], [1, 1], false) -} - -fn inception_a(p: nn::Path, c_in: i64, c_pool: i64) -> impl ModuleT { - let b1 = conv_bn(&p / "branch1x1", c_in, 64, 1, 0, 1); - let b2_1 = conv_bn(&p / "branch5x5_1", c_in, 48, 1, 0, 1); - let b2_2 = conv_bn(&p / "branch5x5_2", 48, 64, 5, 2, 1); - let b3_1 = conv_bn(&p / "branch3x3dbl_1", c_in, 64, 1, 0, 1); - let b3_2 = conv_bn(&p / "branch3x3dbl_2", 64, 96, 3, 1, 1); - let b3_3 = conv_bn(&p / "branch3x3dbl_3", 96, 96, 3, 1, 1); - let bpool = conv_bn(&p / "branch_pool", c_in, c_pool, 1, 0, 1); - nn::func_t(move |xs, tr| { - let b1 = xs.apply_t(&b1, tr); - let b2 = xs.apply_t(&b2_1, tr).apply_t(&b2_2, tr); - let b3 = xs.apply_t(&b3_1, tr).apply_t(&b3_2, tr).apply_t(&b3_3, tr); - let bpool = xs.avg_pool2d([3, 3], [1, 1], [1, 1], false, true, 9).apply_t(&bpool, tr); - Tensor::cat(&[b1, b2, b3, bpool], 1) - }) -} - -fn inception_b(p: nn::Path, c_in: i64) -> impl ModuleT { - let b1 = conv_bn(&p / "branch3x3", c_in, 384, 3, 0, 2); - let b2_1 = conv_bn(&p / "branch3x3dbl_1", c_in, 64, 1, 0, 1); - let b2_2 = conv_bn(&p / "branch3x3dbl_2", 64, 96, 3, 1, 1); - let b2_3 = conv_bn(&p / "branch3x3dbl_3", 96, 96, 3, 0, 2); - nn::func_t(move |xs, tr| { - let b1 = xs.apply_t(&b1, tr); - let b2 = xs.apply_t(&b2_1, tr).apply_t(&b2_2, tr).apply_t(&b2_3, tr); - let bpool = max_pool2d(xs, 3, 2); - Tensor::cat(&[b1, b2, bpool], 1) - }) -} - -fn inception_c(p: nn::Path, c_in: i64, c7: i64) -> impl ModuleT { - let b1 = conv_bn(&p / "branch1x1", c_in, 192, 1, 0, 1); - - let b2_1 = conv_bn(&p / "branch7x7_1", c_in, c7, 1, 0, 1); - let b2_2 = conv_bn2(&p / "branch7x7_2", c7, c7, [1, 7], [0, 3]); - let b2_3 = conv_bn2(&p / "branch7x7_3", c7, 192, [7, 1], [3, 0]); - - let b3_1 = conv_bn(&p / "branch7x7dbl_1", c_in, c7, 1, 0, 1); - let b3_2 = conv_bn2(&p / "branch7x7dbl_2", c7, c7, [7, 1], [3, 0]); - let b3_3 = conv_bn2(&p / "branch7x7dbl_3", c7, c7, [1, 7], [0, 3]); - let b3_4 = conv_bn2(&p / "branch7x7dbl_4", c7, c7, [7, 1], [3, 0]); - let b3_5 = conv_bn2(&p / "branch7x7dbl_5", c7, 192, [1, 7], [0, 3]); - - let bpool = conv_bn(&p / "branch_pool", c_in, 192, 1, 0, 1); - - nn::func_t(move |xs, tr| { - let b1 = xs.apply_t(&b1, tr); - let b2 = xs.apply_t(&b2_1, tr).apply_t(&b2_2, tr).apply_t(&b2_3, tr); - let b3 = xs - .apply_t(&b3_1, tr) - .apply_t(&b3_2, tr) - .apply_t(&b3_3, tr) - .apply_t(&b3_4, tr) - .apply_t(&b3_5, tr); - let bpool = xs.avg_pool2d([3, 3], [1, 1], [1, 1], false, true, 9).apply_t(&bpool, tr); - Tensor::cat(&[b1, b2, b3, bpool], 1) - }) -} - -fn inception_d(p: nn::Path, c_in: i64) -> impl ModuleT { - let b1_1 = conv_bn(&p / "branch3x3_1", c_in, 192, 1, 0, 1); - let b1_2 = conv_bn(&p / "branch3x3_2", 192, 320, 3, 0, 2); - - let b2_1 = conv_bn(&p / "branch7x7x3_1", c_in, 192, 1, 0, 1); - let b2_2 = conv_bn2(&p / "branch7x7x3_2", 192, 192, [1, 7], [0, 3]); - let b2_3 = conv_bn2(&p / "branch7x7x3_3", 192, 192, [7, 1], [3, 0]); - let b2_4 = conv_bn(&p / "branch7x7x3_4", 192, 192, 3, 0, 2); - - nn::func_t(move |xs, tr| { - let b1 = xs.apply_t(&b1_1, tr).apply_t(&b1_2, tr); - let b2 = xs.apply_t(&b2_1, tr).apply_t(&b2_2, tr).apply_t(&b2_3, tr).apply_t(&b2_4, tr); - let bpool = max_pool2d(xs, 3, 2); - Tensor::cat(&[b1, b2, bpool], 1) - }) -} - -fn inception_e(p: nn::Path, c_in: i64) -> impl ModuleT { - let b1 = conv_bn(&p / "branch1x1", c_in, 320, 1, 0, 1); - - let b2_1 = conv_bn(&p / "branch3x3_1", c_in, 384, 1, 0, 1); - let b2_2a = conv_bn2(&p / "branch3x3_2a", 384, 384, [1, 3], [0, 1]); - let b2_2b = conv_bn2(&p / "branch3x3_2b", 384, 384, [3, 1], [1, 0]); - - let b3_1 = conv_bn(&p / "branch3x3dbl_1", c_in, 448, 1, 0, 1); - let b3_2 = conv_bn(&p / "branch3x3dbl_2", 448, 384, 3, 1, 1); - let b3_3a = conv_bn2(&p / "branch3x3dbl_3a", 384, 384, [1, 3], [0, 1]); - let b3_3b = conv_bn2(&p / "branch3x3dbl_3b", 384, 384, [3, 1], [1, 0]); - - let bpool = conv_bn(&p / "branch_pool", c_in, 192, 1, 0, 1); - - nn::func_t(move |xs, tr| { - let b1 = xs.apply_t(&b1, tr); - - let b2 = xs.apply_t(&b2_1, tr); - let b2 = Tensor::cat(&[b2.apply_t(&b2_2a, tr), b2.apply_t(&b2_2b, tr)], 1); - - let b3 = xs.apply_t(&b3_1, tr).apply_t(&b3_2, tr); - let b3 = Tensor::cat(&[b3.apply_t(&b3_3a, tr), b3.apply_t(&b3_3b, tr)], 1); - - let bpool = xs.avg_pool2d([3, 3], [1, 1], [1, 1], false, true, 9).apply_t(&bpool, tr); - - Tensor::cat(&[b1, b2, b3, bpool], 1) - }) -} - -pub fn v3(p: &nn::Path, nclasses: i64) -> impl ModuleT { - nn::seq_t() - .add(conv_bn(p / "Conv2d_1a_3x3", 3, 32, 3, 0, 2)) - .add(conv_bn(p / "Conv2d_2a_3x3", 32, 32, 3, 0, 1)) - .add(conv_bn(p / "Conv2d_2b_3x3", 32, 64, 3, 1, 1)) - .add_fn(|xs| max_pool2d(&xs.relu(), 3, 2)) - .add(conv_bn(p / "Conv2d_3b_1x1", 64, 80, 1, 0, 1)) - .add(conv_bn(p / "Conv2d_4a_3x3", 80, 192, 3, 0, 1)) - .add_fn(|xs| max_pool2d(&xs.relu(), 3, 2)) - .add(inception_a(p / "Mixed_5b", 192, 32)) - .add(inception_a(p / "Mixed_5c", 256, 64)) - .add(inception_a(p / "Mixed_5d", 288, 64)) - .add(inception_b(p / "Mixed_6a", 288)) - .add(inception_c(p / "Mixed_6b", 768, 128)) - .add(inception_c(p / "Mixed_6c", 768, 160)) - .add(inception_c(p / "Mixed_6d", 768, 160)) - .add(inception_c(p / "Mixed_6e", 768, 192)) - .add(inception_d(p / "Mixed_7a", 768)) - .add(inception_e(p / "Mixed_7b", 1280)) - .add(inception_e(p / "Mixed_7c", 2048)) - .add_fn_t(|xs, train| xs.adaptive_avg_pool2d([1, 1]).dropout(0.5, train).flat_view()) - .add(nn::linear(p / "fc", 2048, nclasses, Default::default())) -} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/mnist.rs b/v2/dependency-patches/tch-0.24.0/src/vision/mnist.rs deleted file mode 100644 index 30e3cc9a94..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/vision/mnist.rs +++ /dev/null @@ -1,69 +0,0 @@ -//! The MNIST hand-written digit dataset. -//! -//! The files can be obtained from the following link: -//! -use super::dataset::Dataset; -use crate::{Kind, Tensor}; -use std::fs::File; -use std::io::{self, BufReader, Read, Result}; - -fn read_u32(reader: &mut T) -> Result { - let mut b = vec![0u8; 4]; - reader.read_exact(&mut b)?; - let (result, _) = - b.iter().rev().fold((0u64, 1u64), |(s, basis), &x| (s + basis * u64::from(x), basis * 256)); - Ok(result as u32) -} - -fn check_magic_number(reader: &mut T, expected: u32) -> Result<()> { - let magic_number = read_u32(reader)?; - if magic_number != expected { - return Err(io::Error::other(format!( - "incorrect magic number {magic_number} != {expected}" - ))); - } - Ok(()) -} - -fn read_labels_(filename: &std::path::Path) -> Result { - let mut buf_reader = BufReader::new(File::open(filename)?); - check_magic_number(&mut buf_reader, 2049)?; - let samples = read_u32(&mut buf_reader)?; - let mut data = vec![0u8; samples as usize]; - buf_reader.read_exact(&mut data)?; - Ok(Tensor::from_slice(&data).to_kind(Kind::Int64)) -} - -fn read_images_(filename: &std::path::Path) -> Result { - let mut buf_reader = BufReader::new(File::open(filename)?); - check_magic_number(&mut buf_reader, 2051)?; - let samples = read_u32(&mut buf_reader)?; - let rows = read_u32(&mut buf_reader)?; - let cols = read_u32(&mut buf_reader)?; - let data_len = samples * rows * cols; - let mut data = vec![0u8; data_len as usize]; - buf_reader.read_exact(&mut data)?; - let tensor = Tensor::from_slice(&data) - .view((i64::from(samples), i64::from(rows * cols))) - .to_kind(Kind::Float); - Ok(tensor / 255.) -} - -fn read_labels(filename: &std::path::Path) -> Result { - read_labels_(filename) - .map_err(|err| std::io::Error::new(err.kind(), format!("{filename:?} {err}"))) -} - -fn read_images(filename: &std::path::Path) -> Result { - read_images_(filename) - .map_err(|err| std::io::Error::new(err.kind(), format!("{filename:?} {err}"))) -} - -pub fn load_dir>(dir: T) -> Result { - let dir = dir.as_ref(); - let train_images = read_images(&dir.join("train-images-idx3-ubyte"))?; - let train_labels = read_labels(&dir.join("train-labels-idx1-ubyte"))?; - let test_images = read_images(&dir.join("t10k-images-idx3-ubyte"))?; - let test_labels = read_labels(&dir.join("t10k-labels-idx1-ubyte"))?; - Ok(Dataset { train_images, train_labels, test_images, test_labels, labels: 10 }) -} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/mobilenet.rs b/v2/dependency-patches/tch-0.24.0/src/vision/mobilenet.rs deleted file mode 100644 index 31eea884da..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/vision/mobilenet.rs +++ /dev/null @@ -1,83 +0,0 @@ -//! MobileNet V2 implementation. -//! -use crate::nn::{self, ModuleT}; - -#[allow(clippy::identity_op)] -// Conv2D + BatchNorm2D + ReLU6 -fn cbr(p: nn::Path, c_in: i64, c_out: i64, ks: i64, stride: i64, g: i64) -> impl ModuleT { - let conv2d = nn::ConvConfig { - stride, - padding: (ks - 1) / 2, - groups: g, - bias: false, - ..Default::default() - }; - nn::seq_t() - .add(nn::conv2d(&p / 0, c_in, c_out, ks, conv2d)) - .add(nn::batch_norm2d(&p / 1, c_out, Default::default())) - .add_fn(|xs| xs.relu().clamp_max(6.)) -} - -// Inverted Residual block. -fn inv(p: nn::Path, c_in: i64, c_out: i64, stride: i64, er: i64) -> impl ModuleT { - let c_hidden = er * c_in; - let mut conv = nn::seq_t(); - let mut id = 0; - if er != 1 { - conv = conv.add(cbr(&p / id, c_in, c_hidden, 1, 1, 1)); - id += 1; - } - conv = conv - .add(cbr(&p / id, c_hidden, c_hidden, 3, stride, c_hidden)) - .add(nn::conv2d(&p / (id + 1), c_hidden, c_out, 1, nn::no_bias())) - .add(nn::batch_norm2d(&p / (id + 2), c_out, Default::default())); - nn::func_t(move |xs, train| { - let ys = xs.apply_t(&conv, train); - if stride == 1 && c_in == c_out { - xs + ys - } else { - ys - } - }) -} - -const INVERTED_RESIDUAL_SETTINGS: [(i64, i64, i64, i64); 7] = [ - (1, 16, 1, 1), - (6, 24, 2, 2), - (6, 32, 3, 2), - (6, 64, 4, 2), - (6, 96, 3, 1), - (6, 160, 3, 2), - (6, 320, 1, 1), -]; - -#[allow(clippy::identity_op)] -pub fn v2(p: &nn::Path, nclasses: i64) -> impl ModuleT { - let f_p = p / "features"; - let c_p = p / "classifier"; - let mut c_in = 32; - let mut features = nn::seq_t().add(cbr(&f_p / "0", 3, c_in, 3, 2, 1)); - let mut layer_id = 1; - for &(er, c_out, n, stride) in INVERTED_RESIDUAL_SETTINGS.iter() { - for i in 0..n { - let stride = if i == 0 { stride } else { 1 }; - let f_p = &f_p / layer_id; - features = features.add(inv(&f_p / "conv", c_in, c_out, stride, er)); - c_in = c_out; - layer_id += 1; - } - } - features = features.add(cbr(&f_p / layer_id, c_in, 1280, 1, 1, 1)); - let classifier = nn::seq_t().add_fn_t(|xs, train| xs.dropout(0.2, train)).add(nn::linear( - &c_p / 1, - 1280, - nclasses, - Default::default(), - )); - nn::func_t(move |xs, train| { - xs.apply_t(&features, train) - .mean_dim(Some([2i64].as_slice()), false, crate::Kind::Float) - .mean_dim(Some([2i64].as_slice()), false, crate::Kind::Float) - .apply_t(&classifier, train) - }) -} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/mod.rs b/v2/dependency-patches/tch-0.24.0/src/vision/mod.rs deleted file mode 100644 index cbfbb436aa..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/vision/mod.rs +++ /dev/null @@ -1,34 +0,0 @@ -//! The `vision` module groups functions and models related to -//! computer vision. -pub mod dataset; - -pub mod image; - -pub mod mnist; - -pub mod cifar; - -pub mod alexnet; - -pub mod inception; - -pub mod resnet; - -pub mod densenet; - -pub mod vgg; - -pub mod squeezenet; - -pub mod mobilenet; - -pub mod imagenet; - -pub mod efficientnet; - -pub mod convmixer; - -pub mod dinov2; - -#[cfg(feature = "image")] -mod rust_image; diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/resnet.rs b/v2/dependency-patches/tch-0.24.0/src/vision/resnet.rs deleted file mode 100644 index 09c0f28406..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/vision/resnet.rs +++ /dev/null @@ -1,183 +0,0 @@ -//! ResNet implementation. -//! -//! See "Deep Residual Learning for Image Recognition" He et al. 2015 -//! -use crate::{nn, nn::Conv2D, nn::FuncT, nn::ModuleT}; - -fn conv2d(p: nn::Path, c_in: i64, c_out: i64, ksize: i64, padding: i64, stride: i64) -> Conv2D { - let conv2d_cfg = nn::ConvConfig { stride, padding, bias: false, ..Default::default() }; - nn::conv2d(p, c_in, c_out, ksize, conv2d_cfg) -} - -fn downsample(p: nn::Path, c_in: i64, c_out: i64, stride: i64) -> impl ModuleT { - if stride != 1 || c_in != c_out { - nn::seq_t().add(conv2d(&p / "0", c_in, c_out, 1, 0, stride)).add(nn::batch_norm2d( - &p / "1", - c_out, - Default::default(), - )) - } else { - nn::seq_t() - } -} - -fn basic_block(p: nn::Path, c_in: i64, c_out: i64, stride: i64) -> impl ModuleT { - let conv1 = conv2d(&p / "conv1", c_in, c_out, 3, 1, stride); - let bn1 = nn::batch_norm2d(&p / "bn1", c_out, Default::default()); - let conv2 = conv2d(&p / "conv2", c_out, c_out, 3, 1, 1); - let bn2 = nn::batch_norm2d(&p / "bn2", c_out, Default::default()); - let downsample = downsample(&p / "downsample", c_in, c_out, stride); - nn::func_t(move |xs, train| { - let ys = xs.apply(&conv1).apply_t(&bn1, train).relu().apply(&conv2).apply_t(&bn2, train); - (xs.apply_t(&downsample, train) + ys).relu() - }) -} - -fn basic_layer(p: nn::Path, c_in: i64, c_out: i64, stride: i64, cnt: i64) -> impl ModuleT { - let mut layer = nn::seq_t().add(basic_block(&p / "0", c_in, c_out, stride)); - for block_index in 1..cnt { - layer = layer.add(basic_block(&p / &block_index.to_string(), c_out, c_out, 1)) - } - layer -} - -fn resnet( - p: &nn::Path, - nclasses: Option, - c1: i64, - c2: i64, - c3: i64, - c4: i64, -) -> FuncT<'static> { - let conv1 = conv2d(p / "conv1", 3, 64, 7, 3, 2); - let bn1 = nn::batch_norm2d(p / "bn1", 64, Default::default()); - let layer1 = basic_layer(p / "layer1", 64, 64, 1, c1); - let layer2 = basic_layer(p / "layer2", 64, 128, 2, c2); - let layer3 = basic_layer(p / "layer3", 128, 256, 2, c3); - let layer4 = basic_layer(p / "layer4", 256, 512, 2, c4); - let fc = nclasses.map(|n| nn::linear(p / "fc", 512, n, Default::default())); - nn::func_t(move |xs, train| { - xs.apply(&conv1) - .apply_t(&bn1, train) - .relu() - .max_pool2d([3, 3], [2, 2], [1, 1], [1, 1], false) - .apply_t(&layer1, train) - .apply_t(&layer2, train) - .apply_t(&layer3, train) - .apply_t(&layer4, train) - .adaptive_avg_pool2d([1, 1]) - .flat_view() - .apply_opt(&fc) - }) -} - -/// Creates a ResNet-18 model. -/// -/// Pre-trained weights can be downloaded at the following link: -/// -pub fn resnet18(p: &nn::Path, num_classes: i64) -> FuncT<'static> { - resnet(p, Some(num_classes), 2, 2, 2, 2) -} - -pub fn resnet18_no_final_layer(p: &nn::Path) -> FuncT<'static> { - resnet(p, None, 2, 2, 2, 2) -} - -/// Creates a ResNet-34 model. -/// -/// Pre-trained weights can be downloaded at the following link: -/// -pub fn resnet34(p: &nn::Path, num_classes: i64) -> FuncT<'static> { - resnet(p, Some(num_classes), 3, 4, 6, 3) -} - -pub fn resnet34_no_final_layer(p: &nn::Path) -> FuncT<'static> { - resnet(p, None, 3, 4, 6, 3) -} - -// Bottleneck versions for ResNet 50, 101, and 152. - -fn bottleneck_block(p: nn::Path, c_in: i64, c_out: i64, stride: i64, e: i64) -> impl ModuleT { - let e_dim = e * c_out; - let conv1 = conv2d(&p / "conv1", c_in, c_out, 1, 0, 1); - let bn1 = nn::batch_norm2d(&p / "bn1", c_out, Default::default()); - let conv2 = conv2d(&p / "conv2", c_out, c_out, 3, 1, stride); - let bn2 = nn::batch_norm2d(&p / "bn2", c_out, Default::default()); - let conv3 = conv2d(&p / "conv3", c_out, e_dim, 1, 0, 1); - let bn3 = nn::batch_norm2d(&p / "bn3", e_dim, Default::default()); - let downsample = downsample(&p / "downsample", c_in, e_dim, stride); - nn::func_t(move |xs, train| { - let ys = xs - .apply(&conv1) - .apply_t(&bn1, train) - .relu() - .apply(&conv2) - .apply_t(&bn2, train) - .relu() - .apply(&conv3) - .apply_t(&bn3, train); - (xs.apply_t(&downsample, train) + ys).relu() - }) -} - -fn bottleneck_layer(p: nn::Path, c_in: i64, c_out: i64, stride: i64, cnt: i64) -> impl ModuleT { - let mut layer = nn::seq_t().add(bottleneck_block(&p / "0", c_in, c_out, stride, 4)); - for block_index in 1..cnt { - layer = layer.add(bottleneck_block(&p / &block_index.to_string(), 4 * c_out, c_out, 1, 4)) - } - layer -} - -fn bottleneck_resnet( - p: &nn::Path, - nclasses: Option, - c1: i64, - c2: i64, - c3: i64, - c4: i64, -) -> impl ModuleT { - let conv1 = conv2d(p / "conv1", 3, 64, 7, 3, 2); - let bn1 = nn::batch_norm2d(p / "bn1", 64, Default::default()); - let layer1 = bottleneck_layer(p / "layer1", 64, 64, 1, c1); - let layer2 = bottleneck_layer(p / "layer2", 4 * 64, 128, 2, c2); - let layer3 = bottleneck_layer(p / "layer3", 4 * 128, 256, 2, c3); - let layer4 = bottleneck_layer(p / "layer4", 4 * 256, 512, 2, c4); - let fc = nclasses.map(|n| nn::linear(p / "fc", 4 * 512, n, Default::default())); - nn::func_t(move |xs, train| { - xs.apply(&conv1) - .apply_t(&bn1, train) - .relu() - .max_pool2d([3, 3], [2, 2], [1, 1], [1, 1], false) - .apply_t(&layer1, train) - .apply_t(&layer2, train) - .apply_t(&layer3, train) - .apply_t(&layer4, train) - .adaptive_avg_pool2d([1, 1]) - .flat_view() - .apply_opt(&fc) - }) -} - -pub fn resnet50(p: &nn::Path, num_classes: i64) -> impl ModuleT { - bottleneck_resnet(p, Some(num_classes), 3, 4, 6, 3) -} - -pub fn resnet50_no_final_layer(p: &nn::Path) -> impl ModuleT { - bottleneck_resnet(p, None, 3, 4, 6, 3) -} - -pub fn resnet101(p: &nn::Path, num_classes: i64) -> impl ModuleT { - bottleneck_resnet(p, Some(num_classes), 3, 4, 23, 3) -} - -pub fn resnet101_no_final_layer(p: &nn::Path) -> impl ModuleT { - bottleneck_resnet(p, None, 3, 4, 23, 3) -} - -pub fn resnet152(p: &nn::Path, num_classes: i64) -> impl ModuleT { - bottleneck_resnet(p, Some(num_classes), 3, 8, 36, 3) -} - -pub fn resnet150_no_final_layer(p: &nn::Path) -> impl ModuleT { - bottleneck_resnet(p, None, 3, 8, 36, 3) -} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/rust_image.rs b/v2/dependency-patches/tch-0.24.0/src/vision/rust_image.rs deleted file mode 100644 index 0d78b258ad..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/vision/rust_image.rs +++ /dev/null @@ -1,145 +0,0 @@ -/// Conversion helpers for the image crate. -use std::convert::TryFrom; - -use image::{ - DynamicImage, EncodableLayout, GrayAlphaImage, GrayImage, Rgb32FImage, RgbImage, Rgba32FImage, - RgbaImage, -}; - -use crate::vision::image::{chw_to_hwc, hwc_to_chw}; -use crate::{Kind, TchError, Tensor}; - -impl<'i> TryFrom<&'i DynamicImage> for Tensor { - type Error = TchError; - - fn try_from(image: &'i DynamicImage) -> Result { - match image { - DynamicImage::ImageLuma8(gray) => Tensor::try_from(gray), - DynamicImage::ImageLumaA8(gray_a) => Tensor::try_from(gray_a), - DynamicImage::ImageRgb8(rgb) => Tensor::try_from(rgb), - DynamicImage::ImageRgba8(rgba) => Tensor::try_from(rgba), - DynamicImage::ImageRgb32F(rgb) => Tensor::try_from(rgb), - DynamicImage::ImageRgba32F(rgba) => Tensor::try_from(rgba), - _ => Err(TchError::Convert("unsupported DynamicImage variant".to_string())), - } - } -} - -impl<'i> TryFrom<&'i GrayImage> for Tensor { - type Error = TchError; - - /// `h * w` => `1 * 1 * h * w` - fn try_from(gray: &'i GrayImage) -> Result { - let size = &[gray.height() as i64, gray.width() as i64, 1]; - let tensor = Tensor::f_from_data_size(gray.as_bytes(), size, Kind::Uint8)?; - Ok(hwc_to_chw(&tensor)) - } -} - -impl<'i> TryFrom<&'i GrayAlphaImage> for Tensor { - type Error = TchError; - - /// `2 * h * w` => `1 * 2 * h * w` - fn try_from(gray: &'i GrayAlphaImage) -> Result { - let size = &[gray.height() as i64, gray.width() as i64, 2]; - let tensor = Tensor::f_from_data_size(gray.as_bytes(), size, Kind::Uint8)?; - Ok(hwc_to_chw(&tensor)) - } -} - -impl<'i> TryFrom<&'i RgbImage> for Tensor { - type Error = TchError; - - /// `h * w * 3` => `1 * 3 * h * w` - fn try_from(rgb: &'i RgbImage) -> Result { - let size = &[rgb.height() as i64, rgb.width() as i64, 3]; - let tensor = Tensor::f_from_data_size(rgb.as_raw(), size, Kind::Uint8)?; - Ok(hwc_to_chw(&tensor)) - } -} - -impl<'i> TryFrom<&'i RgbaImage> for Tensor { - type Error = TchError; - - /// `h * w * 4` => `1 * 4 * h * w` - fn try_from(rgb: &'i RgbaImage) -> Result { - let kind = Kind::Uint8; - let size = &[rgb.height() as i64, rgb.width() as i64, 4]; - let tensor = Tensor::f_from_data_size(rgb.as_raw(), size, kind)?; - Ok(hwc_to_chw(&tensor)) - } -} - -impl<'i> TryFrom<&'i Tensor> for RgbImage { - type Error = TchError; - - /// `1 * 3 * h * w` => `h * w * 3` - fn try_from(value: &'i Tensor) -> Result { - let tensor = assert_tensor_as_image(value, Kind::Uint8, 3)?; - let width = tensor.size()[1] as u32; - let height = tensor.size()[0] as u32; - let length = (width * height * 3) as usize; - let mut buffer = vec![0; length]; - tensor.f_copy_data(&mut buffer, length)?; - RgbImage::from_raw(width, height, buffer) - .ok_or_else(|| TchError::Convert("Failed to convert tensor to image".to_string())) - } -} - -impl<'i> TryFrom<&'i Rgb32FImage> for Tensor { - type Error = TchError; - - /// `h * w * 3` => `1 * 3 * h * w` - fn try_from(rgb: &'i Rgb32FImage) -> Result { - let kind = Kind::Float; - let size = &[rgb.height() as i64, rgb.width() as i64, 3]; - let tensor = Tensor::f_from_data_size(rgb.as_bytes(), size, kind)?; - Ok(hwc_to_chw(&tensor)) - } -} - -impl<'i> TryFrom<&'i Rgba32FImage> for Tensor { - type Error = TchError; - - /// `h * w * 4` => `1 * 4 * h * w` - fn try_from(rgb: &'i Rgba32FImage) -> Result { - let kind = Kind::Float; - let size = &[rgb.height() as i64, rgb.width() as i64, 4]; - let tensor = Tensor::f_from_data_size(rgb.as_bytes(), size, kind)?; - Ok(hwc_to_chw(&tensor)) - } -} - -impl<'i> TryFrom<&'i Tensor> for Rgb32FImage { - type Error = TchError; - - /// `1 * 3 * h * w` => `h * w * 3` - fn try_from(value: &'i Tensor) -> Result { - let tensor = assert_tensor_as_image(value, Kind::Float, 3)?; - let width = tensor.size()[1] as u32; - let height = tensor.size()[0] as u32; - let length = (width * height * 3) as usize; - let mut buffer = vec![0.0; length]; - tensor.f_copy_data(&mut buffer, length)?; - Rgb32FImage::from_raw(width, height, buffer) - .ok_or_else(|| TchError::Convert("Failed to convert tensor to image".to_string())) - } -} - -#[inline] -fn assert_tensor_as_image(tensor: &Tensor, except: Kind, channel: i64) -> Result { - let kind = tensor.kind(); - let size = tensor.size(); - if size.len() != 3 { - let msg = format!("Tensor size is `{size:?}`, except rank 3 tensor"); - Err(TchError::Convert(msg)) - } else if size[0] != channel { - let msg = format!("Tensor size is `{size:?}`, except {channel} channels"); - Err(TchError::Convert(msg)) - } else if kind != except { - let msg = format!("Tensor kind is `{kind:?}`, except `{except:?}` tensor"); - Err(TchError::Convert(msg)) - } else { - Ok(chw_to_hwc(tensor)) - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/squeezenet.rs b/v2/dependency-patches/tch-0.24.0/src/vision/squeezenet.rs deleted file mode 100644 index d2b9ce86e3..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/vision/squeezenet.rs +++ /dev/null @@ -1,68 +0,0 @@ -//! SqueezeNet implementation. - -use crate::{nn, nn::Module, nn::ModuleT, Tensor}; - -fn max_pool2d(xs: &Tensor) -> Tensor { - xs.max_pool2d([3, 3], [2, 2], [0, 0], [1, 1], true) -} - -fn fire(p: nn::Path, c_in: i64, c_squeeze: i64, c_exp1: i64, c_exp3: i64) -> impl Module { - let cfg3 = nn::ConvConfig { padding: 1, ..Default::default() }; - let squeeze = nn::conv2d(&p / "squeeze", c_in, c_squeeze, 1, Default::default()); - let exp1 = nn::conv2d(&p / "expand1x1", c_squeeze, c_exp1, 1, Default::default()); - let exp3 = nn::conv2d(&p / "expand3x3", c_squeeze, c_exp3, 3, cfg3); - nn::func(move |xs| { - let xs = xs.apply(&squeeze).relu(); - Tensor::cat(&[xs.apply(&exp1).relu(), xs.apply(&exp3).relu()], 1) - }) -} - -fn squeezenet(p: &nn::Path, v1_0: bool, nclasses: i64) -> impl ModuleT { - let f_p = p / "features"; - let c_p = p / "classifier"; - let initial_conv_cfg = nn::ConvConfig { stride: 2, ..Default::default() }; - let final_conv_cfg = nn::ConvConfig { stride: 1, ..Default::default() }; - let features = if v1_0 { - nn::seq_t() - .add(nn::conv2d(&f_p / "0", 3, 96, 7, initial_conv_cfg)) - .add_fn(|xs| xs.relu()) - .add_fn(max_pool2d) - .add(fire(&f_p / "3", 96, 16, 64, 64)) - .add(fire(&f_p / "4", 128, 16, 64, 64)) - .add(fire(&f_p / "5", 128, 32, 128, 128)) - .add_fn(max_pool2d) - .add(fire(&f_p / "7", 256, 32, 128, 128)) - .add(fire(&f_p / "8", 256, 48, 192, 192)) - .add(fire(&f_p / "9", 384, 48, 192, 192)) - .add(fire(&f_p / "10", 384, 64, 256, 256)) - .add_fn(max_pool2d) - .add(fire(&f_p / "12", 512, 64, 256, 256)) - } else { - nn::seq_t() - .add(nn::conv2d(&f_p / "0", 3, 64, 3, initial_conv_cfg)) - .add_fn(|xs| xs.relu()) - .add_fn(max_pool2d) - .add(fire(&f_p / "3", 64, 16, 64, 64)) - .add(fire(&f_p / "4", 128, 16, 64, 64)) - .add_fn(max_pool2d) - .add(fire(&f_p / "6", 128, 32, 128, 128)) - .add(fire(&f_p / "7", 256, 32, 128, 128)) - .add_fn(max_pool2d) - .add(fire(&f_p / "9", 256, 48, 192, 192)) - .add(fire(&f_p / "10", 384, 48, 192, 192)) - .add(fire(&f_p / "11", 384, 64, 256, 256)) - .add(fire(&f_p / "12", 512, 64, 256, 256)) - }; - features - .add_fn_t(|xs, train| xs.dropout(0.5, train)) - .add(nn::conv2d(&c_p / "1", 512, nclasses, 1, final_conv_cfg)) - .add_fn(|xs| xs.relu().adaptive_avg_pool2d([1, 1]).flat_view()) -} - -pub fn v1_0(p: &nn::Path, nclasses: i64) -> impl ModuleT { - squeezenet(p, true, nclasses) -} - -pub fn v1_1(p: &nn::Path, nclasses: i64) -> impl ModuleT { - squeezenet(p, false, nclasses) -} diff --git a/v2/dependency-patches/tch-0.24.0/src/vision/vgg.rs b/v2/dependency-patches/tch-0.24.0/src/vision/vgg.rs deleted file mode 100644 index 7adf0bd02b..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/vision/vgg.rs +++ /dev/null @@ -1,100 +0,0 @@ -//! VGG models. -//! -//! Pre-trained weights for the vgg-13/16/19 models can be found here: -//! -//! -//! -use crate::{nn, nn::Conv2D, nn::SequentialT}; - -// Each list element contains multiple convolutions with some specified number -// of features followed by a single max-pool layer. -fn layers_a() -> Vec> { - vec![vec![64], vec![128], vec![256, 256], vec![512, 512], vec![512, 512]] -} - -fn layers_b() -> Vec> { - vec![vec![64, 64], vec![128, 128], vec![256, 256], vec![512, 512], vec![512, 512]] -} -fn layers_d() -> Vec> { - vec![ - vec![64, 64], - vec![128, 128], - vec![256, 256, 256], - vec![512, 512, 512], - vec![512, 512, 512], - ] -} -fn layers_e() -> Vec> { - vec![ - vec![64, 64], - vec![128, 128], - vec![256, 256, 256, 256], - vec![512, 512, 512, 512], - vec![512, 512, 512, 512], - ] -} - -fn conv2d(p: nn::Path, c_in: i64, c_out: i64) -> Conv2D { - let conv2d_cfg = nn::ConvConfig { stride: 1, padding: 1, ..Default::default() }; - nn::conv2d(p, c_in, c_out, 3, conv2d_cfg) -} - -fn vgg(p: &nn::Path, cfg: Vec>, nclasses: i64, batch_norm: bool) -> SequentialT { - let c = p / "classifier"; - let mut seq = nn::seq_t(); - let f = p / "features"; - let mut c_in = 3; - for channels in cfg.into_iter() { - for &c_out in channels.iter() { - let l = seq.len(); - seq = seq.add(conv2d(&f / &l.to_string(), c_in, c_out)); - if batch_norm { - let l = seq.len(); - seq = seq.add(nn::batch_norm2d(&f / &l.to_string(), c_out, Default::default())); - }; - seq = seq.add_fn(|xs| xs.relu()); - c_in = c_out; - } - seq = seq.add_fn(|xs| xs.max_pool2d_default(2)); - } - seq.add_fn(|xs| xs.flat_view()) - .add(nn::linear(&c / "0", 512 * 7 * 7, 4096, Default::default())) - .add_fn(|xs| xs.relu()) - .add_fn_t(|xs, train| xs.dropout(0.5, train)) - .add(nn::linear(&c / "3", 4096, 4096, Default::default())) - .add_fn(|xs| xs.relu()) - .add_fn_t(|xs, train| xs.dropout(0.5, train)) - .add(nn::linear(&c / "6", 4096, nclasses, Default::default())) -} - -pub fn vgg11(p: &nn::Path, nclasses: i64) -> SequentialT { - vgg(p, layers_a(), nclasses, false) -} - -pub fn vgg11_bn(p: &nn::Path, nclasses: i64) -> SequentialT { - vgg(p, layers_a(), nclasses, true) -} - -pub fn vgg13(p: &nn::Path, nclasses: i64) -> SequentialT { - vgg(p, layers_b(), nclasses, false) -} - -pub fn vgg13_bn(p: &nn::Path, nclasses: i64) -> SequentialT { - vgg(p, layers_b(), nclasses, true) -} - -pub fn vgg16(p: &nn::Path, nclasses: i64) -> SequentialT { - vgg(p, layers_d(), nclasses, false) -} - -pub fn vgg16_bn(p: &nn::Path, nclasses: i64) -> SequentialT { - vgg(p, layers_d(), nclasses, true) -} - -pub fn vgg19(p: &nn::Path, nclasses: i64) -> SequentialT { - vgg(p, layers_e(), nclasses, false) -} - -pub fn vgg19_bn(p: &nn::Path, nclasses: i64) -> SequentialT { - vgg(p, layers_e(), nclasses, true) -} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/device.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/device.rs deleted file mode 100644 index 9dadd192e7..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/wrappers/device.rs +++ /dev/null @@ -1,120 +0,0 @@ -//! Devices on which tensor computations are run. - -/// A torch device. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub enum Device { - /// The main CPU device. - Cpu, - /// The main GPU device. - Cuda(usize), - /// The main MPS device. - Mps, - /// The main Vulkan device. - Vulkan, -} - -/// Cuda related helper functions. -pub enum Cuda {} -impl Cuda { - /// Returns the number of CUDA devices available. - pub fn device_count() -> i64 { - let res = unsafe_torch!(torch_sys::cuda::atc_cuda_device_count()); - i64::from(res) - } - - /// Returns true if at least one CUDA device is available. - pub fn is_available() -> bool { - unsafe_torch!(torch_sys::cuda::atc_cuda_is_available()) != 0 - } - - /// Returns true if CUDA is available, and CuDNN is available. - pub fn cudnn_is_available() -> bool { - unsafe_torch!(torch_sys::cuda::atc_cudnn_is_available()) != 0 - } - - /// Sets the seed for the current GPU. - /// - /// # Arguments - /// - /// * `seed` - An unsigned 64bit int to be used as seed. - pub fn manual_seed(seed: u64) { - unsafe_torch!(torch_sys::cuda::atc_manual_seed(seed)); - } - - /// Sets the seed for all available GPUs. - /// - /// # Arguments - /// - /// * `seed` - An unsigned 64bit int to be used as seed. - pub fn manual_seed_all(seed: u64) { - unsafe_torch!(torch_sys::cuda::atc_manual_seed_all(seed)); - } - - /// Waits for all kernels in all streams on a CUDA device to complete. - /// - /// # Arguments - /// - /// * `device_index` - A signed 64bit int to indice which device to wait for. - pub fn synchronize(device_index: i64) { - unsafe_torch!(torch_sys::cuda::atc_synchronize(device_index)); - } - - /// Returns true if cudnn is enabled by the user. - /// - /// This does not indicate whether cudnn is actually usable. - pub fn user_enabled_cudnn() -> bool { - unsafe_torch!(torch_sys::cuda::atc_user_enabled_cudnn()) != 0 - } - - /// Enable or disable cudnn. - pub fn set_user_enabled_cudnn(b: bool) { - unsafe_torch!(torch_sys::cuda::atc_set_user_enabled_cudnn(i32::from(b))) - } - - /// Sets cudnn benchmark mode. - /// - /// When set cudnn will try to optimize the generators durning - /// the first network runs and then use the optimized architecture - /// in the following runs. This can result in significant performance - /// improvements. - pub fn cudnn_set_benchmark(b: bool) { - unsafe_torch!(torch_sys::cuda::atc_set_benchmark_cudnn(i32::from(b))) - } -} - -impl Device { - pub(super) fn c_int(self) -> libc::c_int { - match self { - Device::Cpu => -1, - Device::Cuda(device_index) => device_index as libc::c_int, - Device::Mps => -2, - Device::Vulkan => -3, - } - } - - pub(super) fn from_c_int(v: libc::c_int) -> Self { - match v { - -1 => Device::Cpu, - -2 => Device::Mps, - -3 => Device::Vulkan, - index if index >= 0 => Device::Cuda(index as usize), - _ => panic!("unexpected device {v}"), - } - } - - /// Returns a GPU device if available, else default to CPU. - pub fn cuda_if_available() -> Device { - if Cuda::is_available() { - Device::Cuda(0) - } else { - Device::Cpu - } - } - - pub fn is_cuda(self) -> bool { - match self { - Device::Cuda(_) => true, - Device::Cpu | Device::Mps | Device::Vulkan => false, - } - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/image.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/image.rs deleted file mode 100644 index bc4a7b186e..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/wrappers/image.rs +++ /dev/null @@ -1,35 +0,0 @@ -use super::tensor::Tensor; -use super::utils::path_to_cstring; -use crate::TchError; -use libc::c_int; -use std::path::Path; - -/// On success returns a tensor of shape [width, height, channels]. -pub fn load_hwc>(path: T) -> Result { - let path = path_to_cstring(path)?; - let c_tensor = unsafe_torch_err!(torch_sys::at_load_image(path.as_ptr())); - Ok(Tensor { c_tensor }) -} - -/// On success returns a tensor of shape [width, height, channels]. -pub fn load_hwc_from_mem(data: &[u8]) -> Result { - let c_tensor = - unsafe_torch_err!(torch_sys::at_load_image_from_memory(data.as_ptr(), data.len())); - Ok(Tensor { c_tensor }) -} - -/// Expects a tensor of shape [width, height, channels]. -pub fn save_hwc>(t: &Tensor, path: T) -> Result<(), TchError> { - let path = path_to_cstring(path)?; - let _ = unsafe_torch_err!(torch_sys::at_save_image(t.c_tensor, path.as_ptr())); - Ok(()) -} - -/// Expects a tensor of shape [width, height, channels]. -/// On success returns a tensor of shape [width, height, channels]. -pub fn resize_hwc(t: &Tensor, out_w: i64, out_h: i64) -> Result { - let c_tensor = unsafe_torch_err!({ - torch_sys::at_resize_image(t.c_tensor, out_w as c_int, out_h as c_int) - }); - Ok(Tensor { c_tensor }) -} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/jit.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/jit.rs deleted file mode 100644 index f3c316a837..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/wrappers/jit.rs +++ /dev/null @@ -1,879 +0,0 @@ -//! JIT interface to run model trained/saved using PyTorch Python API. -use super::utils::{path_to_cstring, ptr_to_string}; -use super::{device::Device, kind::Kind}; -use crate::{nn::Path, TchError, Tensor}; -use libc::{c_int, c_void}; -use std::borrow::Borrow; -use std::convert::TryFrom; -use torch_sys::*; - -/// Argument and output values for JIT models. These represent arbitrary values, -/// e.g. tensors, atomic values, pairs of values, etc. -#[derive(Debug, PartialEq)] -#[non_exhaustive] -pub enum IValue { - None, - Tensor(crate::Tensor), - Double(f64), - Int(i64), - Bool(bool), - Tuple(Vec), - IntList(Vec), - DoubleList(Vec), - BoolList(Vec), - String(String), - StringList(Vec), - TensorList(Vec), - GenericList(Vec), - // We use a vec to represent dictionaries as f64 does not implement - // Eq or Hash out of the box in rust. TODO: improve this? - GenericDict(Vec<(IValue, IValue)>), - Object(Object), - Device(Device), -} - -impl IValue { - fn type_str(self) -> &'static str { - match self { - IValue::None => "None", - IValue::Tensor(_) => "Tensor", - IValue::Double(_) => "Double", - IValue::Int(_) => "Int", - IValue::Bool(_) => "Bool", - IValue::Tuple(_) => "Tuple", - IValue::IntList(_) => "IntList", - IValue::DoubleList(_) => "DoubleList", - IValue::BoolList(_) => "BoolList", - IValue::String(_) => "String", - IValue::StringList(_) => "StringList", - IValue::TensorList(_) => "TensorList", - IValue::GenericList(_) => "GenericList", - IValue::GenericDict(_) => "GenericDict", - IValue::Object(_) => "Object", - IValue::Device(_) => "Device", - } - } -} - -impl From<()> for IValue { - fn from((): ()) -> Self { - IValue::None - } -} - -impl, T2: Into> From<(T1, T2)> for IValue { - fn from((p1, p2): (T1, T2)) -> Self { - IValue::Tuple(vec![p1.into(), p2.into()]) - } -} - -impl, T2: Into, T3: Into> From<(T1, T2, T3)> for IValue { - fn from((p1, p2, p3): (T1, T2, T3)) -> Self { - IValue::Tuple(vec![p1.into(), p2.into(), p3.into()]) - } -} - -impl, T2: Into, T3: Into, T4: Into> From<(T1, T2, T3, T4)> - for IValue -{ - fn from((p1, p2, p3, p4): (T1, T2, T3, T4)) -> Self { - IValue::Tuple(vec![p1.into(), p2.into(), p3.into(), p4.into()]) - } -} - -impl TryFrom for (T1, T2) -where - T1: TryFrom, - TchError: From, - T2: TryFrom, - TchError: From, -{ - type Error = TchError; - fn try_from(value: IValue) -> Result { - match value { - IValue::GenericList(mut vec) | IValue::Tuple(mut vec) => { - if vec.len() == 2 { - let t2 = T2::try_from(vec.pop().unwrap())?; - let t1 = T1::try_from(vec.pop().unwrap())?; - Ok((t1, t2)) - } else { - Err(TchError::Kind(format!( - "unable to unpack ivalue, expected a tuple of len 2 got {}", - vec.len() - ))) - } - } - _ => Err(TchError::Kind(format!( - "unable to unpack ivalue, expected a tuple got {}", - value.type_str() - ))), - } - } -} - -impl TryFrom for (T1, T2, T3) -where - T1: TryFrom, - TchError: From, - T2: TryFrom, - TchError: From, - T3: TryFrom, - TchError: From, -{ - type Error = TchError; - fn try_from(value: IValue) -> Result { - match value { - IValue::GenericList(mut vec) | IValue::Tuple(mut vec) => { - if vec.len() == 3 { - let t3 = T3::try_from(vec.pop().unwrap())?; - let t2 = T2::try_from(vec.pop().unwrap())?; - let t1 = T1::try_from(vec.pop().unwrap())?; - Ok((t1, t2, t3)) - } else { - Err(TchError::Kind(format!( - "unable to unpack ivalue, expected a tuple of len 3 got {}", - vec.len() - ))) - } - } - _ => Err(TchError::Kind(format!( - "unable to unpack ivalue, expected a tuple got {}", - value.type_str() - ))), - } - } -} - -impl TryFrom for (T1, T2, T3, T4) -where - T1: TryFrom, - TchError: From, - T2: TryFrom, - TchError: From, - T3: TryFrom, - TchError: From, - T4: TryFrom, - TchError: From, -{ - type Error = TchError; - fn try_from(value: IValue) -> Result { - match value { - IValue::GenericList(mut vec) | IValue::Tuple(mut vec) => { - if vec.len() == 4 { - let t4 = T4::try_from(vec.pop().unwrap())?; - let t3 = T3::try_from(vec.pop().unwrap())?; - let t2 = T2::try_from(vec.pop().unwrap())?; - let t1 = T1::try_from(vec.pop().unwrap())?; - Ok((t1, t2, t3, t4)) - } else { - Err(TchError::Kind(format!( - "unable to unpack ivalue, expected a tuple of len 4 got {}", - vec.len() - ))) - } - } - _ => Err(TchError::Kind(format!( - "unable to unpack ivalue, expected a tuple got {}", - value.type_str() - ))), - } - } -} - -macro_rules! impl_from { - ($type_:ty, $cons:ident) => { - impl From<$type_> for IValue { - fn from(v: $type_) -> Self { - IValue::$cons(v) - } - } - - impl TryFrom for $type_ { - type Error = TchError; - fn try_from(value: IValue) -> Result<$type_, TchError> { - match value { - IValue::$cons(t) => Ok(t), - _ => Err(TchError::Kind(format!( - "unable to unpack ivalue, expected {} got {}", - std::stringify!($cons), - value.type_str() - ))), - } - } - } - - // A generic trait for Option would seem nicer but because - // of E0119, this is currently hard to do. - // See https://github.com/rust-lang/rust/issues/50133 - impl TryFrom for Option<$type_> { - type Error = TchError; - fn try_from(value: IValue) -> Result { - match value { - IValue::None => Ok(None), - IValue::$cons(t) => Ok(Some(t)), - _ => Err(TchError::Kind(format!( - "unable to unpack ivalue, expected {} or None got {}", - std::stringify!($cons), - value.type_str() - ))), - } - } - } - }; -} - -impl_from!(i64, Int); -impl_from!(f64, Double); -impl_from!(bool, Bool); -impl_from!(String, String); -impl_from!(Tensor, Tensor); -impl_from!(Vec, IntList); -impl_from!(Vec, DoubleList); -impl_from!(Vec, BoolList); -impl_from!(Vec, StringList); -impl_from!(Vec, TensorList); -impl_from!(Vec, GenericList); -impl_from!(Vec<(IValue, IValue)>, GenericDict); -impl_from!(Object, Object); -impl_from!(Device, Device); - -impl From<&str> for IValue { - fn from(s: &str) -> Self { - IValue::String(s.to_string()) - } -} - -impl IValue { - #![allow(unused_unsafe)] - pub(super) fn to_c(&self) -> Result<*mut CIValue, TchError> { - let c = unsafe_torch_err!(match self { - IValue::Tensor(tensor) => ati_tensor(tensor.c_tensor), - IValue::Int(i) => ati_int(*i), - IValue::None => ati_none(), - IValue::Double(f) => ati_double(*f), - IValue::Bool(b) => ati_bool(i32::from(*b)), - IValue::Tuple(v) => { - let v = v.iter().map(Self::to_c).collect::, TchError>>()?; - let tuple = ati_tuple(v.as_ptr(), v.len() as c_int); - for x in v { - ati_free(x); - } - - tuple - } - IValue::GenericList(v) => { - let v = v.iter().map(Self::to_c).collect::, TchError>>()?; - let list = ati_generic_list(v.as_ptr(), v.len() as c_int); - for x in v { - ati_free(x); - } - list - } - IValue::IntList(v) => ati_int_list(v.as_ptr(), v.len() as c_int), - IValue::DoubleList(v) => ati_double_list(v.as_ptr(), v.len() as c_int), - IValue::BoolList(v) => { - let v: Vec = v.iter().map(|&b| libc::c_char::from(b)).collect(); - ati_bool_list(v.as_ptr(), v.len() as c_int) - } - IValue::TensorList(v) => { - let v = v.iter().map(|t| t.c_tensor).collect::>(); - ati_tensor_list(v.as_ptr(), v.len() as c_int) - } - IValue::String(string) => { - let c_str = std::ffi::CString::new(string.as_str())?; - ati_string(c_str.as_ptr()) - } - IValue::StringList(strings) => { - let mut v = vec![]; - for s in strings { - v.push(std::ffi::CString::new(s.as_str())?); - } - let v_ptr: Vec<_> = v.iter().map(|s| s.as_ptr()).collect(); - ati_string_list(v_ptr.as_ptr(), v.len() as c_int) - } - IValue::GenericDict(dict) => { - let v = dict - .iter() - .flat_map(|(k, v)| vec![Self::to_c(k), Self::to_c(v)]) - .collect::, TchError>>()?; - let dict = ati_generic_dict(v.as_ptr(), dict.len() as c_int); - for x in v { - ati_free(x); - } - dict - } - IValue::Object(Object { c_ivalue }) => { - // Clone the object if necessary before passing the pointer to the C++ side. - unsafe_torch_err!(ati_clone(*c_ivalue)) - } - IValue::Device(device) => { - ati_device(device.c_int()) - } - }); - Ok(c) - } - - // This consumes the pointer and frees the associated memory (unless it is an Object). - pub(super) fn from_c(c_ivalue: *mut CIValue) -> Result { - let mut free = true; - let tag = unsafe_torch_err!(ati_tag(c_ivalue)); - let v = match tag { - 0 => IValue::None, - 1 => { - let c_tensor = unsafe_torch_err!(ati_to_tensor(c_ivalue)); - IValue::Tensor(crate::Tensor { c_tensor }) - } - 2 => IValue::Double(unsafe_torch_err!(ati_to_double(c_ivalue))), - 3 => IValue::Int(unsafe_torch_err!(ati_to_int(c_ivalue))), - 4 => { - let b = unsafe_torch_err!(ati_to_bool(c_ivalue)); - if b < 0 { - return Err(TchError::Kind(format!("unexpected bool value {b}"))); - } - IValue::Bool(b != 0) - } - 5 => { - let len = unsafe_torch_err!(ati_tuple_length(c_ivalue)); - let mut c_ivalues: Vec<_> = - (0..len).map(|_| std::ptr::null_mut::()).collect(); - unsafe_torch_err!(ati_to_tuple(c_ivalue, c_ivalues.as_mut_ptr(), len)); - let vec: Result, _> = - c_ivalues.iter().map(|&c_ivalue| Self::from_c(c_ivalue)).collect(); - IValue::Tuple(vec?) - } - 6 => { - let len = unsafe_torch_err!(ati_length(c_ivalue)); - let mut c_array = vec![0i64; len as usize]; - unsafe_torch_err!(ati_to_int_list(c_ivalue, c_array.as_mut_ptr(), len)); - IValue::IntList(c_array) - } - 7 => { - let len = unsafe_torch_err!(ati_length(c_ivalue)); - let mut c_array = vec![0f64; len as usize]; - unsafe_torch_err!(ati_to_double_list(c_ivalue, c_array.as_mut_ptr(), len)); - IValue::DoubleList(c_array) - } - 8 => { - let len = unsafe_torch_err!(ati_length(c_ivalue)); - let mut c_array = vec![0_i8; len as usize]; - let c_array_ptr = c_array.as_mut_ptr() as *mut libc::c_char; - unsafe_torch_err!(ati_to_bool_list(c_ivalue, c_array_ptr, len)); - IValue::BoolList(c_array.iter().map(|&x| x != 0).collect()) - } - 9 => { - let ptr = unsafe_torch_err!(ati_to_string(c_ivalue)); - let string = match unsafe { ptr_to_string(ptr) } { - None => return Err(TchError::Kind("nullptr representation".to_string())), - Some(s) => s, - }; - IValue::String(string) - } - 10 => { - let len = unsafe_torch_err!(ati_length(c_ivalue)); - let mut c_tensors: Vec<_> = - (0..len).map(|_| std::ptr::null_mut::()).collect(); - unsafe_torch_err!(ati_to_tensor_list(c_ivalue, c_tensors.as_mut_ptr(), len)); - let vec: Vec<_> = c_tensors.iter().map(|&c_tensor| Tensor { c_tensor }).collect(); - IValue::TensorList(vec) - } - 12 => { - let len = unsafe_torch_err!(ati_length(c_ivalue)); - let mut c_ivalues: Vec<_> = - (0..len).map(|_| std::ptr::null_mut::()).collect(); - unsafe_torch_err!(ati_to_generic_list(c_ivalue, c_ivalues.as_mut_ptr(), len)); - let vec: Result, _> = - c_ivalues.iter().map(|&c_ivalue| Self::from_c(c_ivalue)).collect(); - IValue::GenericList(vec?) - } - 13 => { - let len = unsafe_torch_err!(ati_length(c_ivalue)); - let mut c_ivalues: Vec<_> = - (0..2 * len).map(|_| std::ptr::null_mut::()).collect(); - unsafe_torch_err!(ati_to_generic_dict(c_ivalue, c_ivalues.as_mut_ptr(), len)); - let mut res: Vec<(IValue, IValue)> = vec![]; - for i in 0..(len as usize) { - let key = Self::from_c(c_ivalues[2 * i])?; - let value = Self::from_c(c_ivalues[2 * i + 1])?; - res.push((key, value)) - } - IValue::GenericDict(res) - } - 14 => { - free = false; - IValue::Object(Object { c_ivalue }) - } - _ => return Err(TchError::Kind(format!("unhandled tag {tag}"))), - }; - if free { - unsafe_torch_err!(ati_free(c_ivalue)); - } - Ok(v) - } -} - -/// A jit PyTorch module. -/// -/// These modules can be created via the -/// [TorchScript python api](https://pytorch.org/docs/stable/jit.html). -#[derive(Debug)] -pub struct CModule { - pub(super) c_module: *mut CModule_, -} - -unsafe impl Send for CModule {} - -unsafe impl Sync for CModule {} - -impl Drop for CModule { - fn drop(&mut self) { - unsafe_torch!(atm_free(self.c_module)) - } -} - -impl CModule { - /// Loads a PyTorch saved JIT model from a file. - pub fn load>(path: T) -> Result { - let path = path_to_cstring(path)?; - let c_module = unsafe_torch_err!(atm_load(path.as_ptr())); - Ok(CModule { c_module }) - } - - /// Loads a PyTorch saved JIT model from a file onto the given device. - /// - /// This function loads the model directly on the specified device, - /// which means it also allows loading a GPU model on the CPU without having a CUDA enabled GPU. - pub fn load_on_device>( - path: T, - device: Device, - ) -> Result { - let path = path_to_cstring(path)?; - let c_module = unsafe_torch_err!(atm_load_on_device(path.as_ptr(), device.c_int())); - Ok(CModule { c_module }) - } - - /// Loads a PyTorch saved JIT model from a read instance. - pub fn load_data(f: &mut T) -> Result { - let mut buffer = Vec::new(); - f.read_to_end(&mut buffer)?; - let buffer_ptr = buffer.as_ptr() as *const libc::c_char; - let c_module = unsafe_torch_err!(atm_load_str(buffer_ptr, buffer.len())); - Ok(CModule { c_module }) - } - - /// Loads a PyTorch saved JIT model from a read instance. - /// - /// This function loads the model directly on the specified device, - /// which means it also allows loading a GPU model on the CPU without having a CUDA enabled GPU. - pub fn load_data_on_device( - f: &mut T, - device: Device, - ) -> Result { - let mut buffer = Vec::new(); - f.read_to_end(&mut buffer)?; - let buffer_ptr = buffer.as_ptr() as *const libc::c_char; - let c_module = - unsafe_torch_err!(atm_load_str_on_device(buffer_ptr, buffer.len(), device.c_int())); - Ok(CModule { c_module }) - } - - /// Performs the forward pass for a model on some specified tensor inputs. This is equivalent - /// to calling method_ts with the 'forward' method name, and returns a single tensor. - pub fn forward_ts>(&self, ts: &[T]) -> Result { - let ts: Vec<_> = ts.iter().map(|x| x.borrow().c_tensor).collect(); - let c_tensor = - unsafe_torch_err!(atm_forward(self.c_module, ts.as_ptr(), ts.len() as c_int)); - Ok(Tensor { c_tensor }) - } - - /// Performs the forward pass for a model on some specified ivalue inputs. This is equivalent - /// to calling method_is with the 'forward' method name, and returns an arbitrary ivalue. - pub fn forward_is>(&self, ts: &[T]) -> Result { - let ts = ts.iter().map(|x| x.borrow().to_c()).collect::, TchError>>()?; - let c_ivalue = - unsafe_torch_err!(atm_forward_(self.c_module, ts.as_ptr(), ts.len() as c_int)); - for x in ts { - unsafe { ati_free(x) } - } - IValue::from_c(c_ivalue) - } - - /// Runs a specified entry point for a model on some given tensor inputs. - pub fn method_ts>( - &self, - method_name: &str, - ts: &[T], - ) -> Result { - let ts: Vec<_> = ts.iter().map(|x| x.borrow().c_tensor).collect(); - let method_name = std::ffi::CString::new(method_name)?; - let c_tensor = unsafe_torch_err!(atm_method( - self.c_module, - method_name.as_ptr(), - ts.as_ptr(), - ts.len() as c_int - )); - Ok(Tensor { c_tensor }) - } - - /// Runs a specified entry point for a model on some given ivalue inputs. - pub fn method_is>( - &self, - method_name: &str, - ts: &[T], - ) -> Result { - let ts = ts.iter().map(|x| x.borrow().to_c()).collect::, TchError>>()?; - let method_name = std::ffi::CString::new(method_name)?; - let c_ivalue = unsafe_torch_err!(atm_method_( - self.c_module, - method_name.as_ptr(), - ts.as_ptr(), - ts.len() as c_int - )); - for x in ts { - unsafe { ati_free(x) } - } - IValue::from_c(c_ivalue) - } - - /// Create a specified custom JIT class object with the given class name, eg: `__torch__.foo.Bar` - pub fn create_class_is>( - &self, - clz_name: &str, - ts: &[T], - ) -> Result { - let ts = ts.iter().map(|x| x.borrow().to_c()).collect::, TchError>>()?; - let clz_name = std::ffi::CString::new(clz_name)?; - let c_ivalue = unsafe_torch_err!(atm_create_class_( - self.c_module, - clz_name.as_ptr(), - ts.as_ptr(), - ts.len() as c_int - )); - for x in ts { - unsafe { ati_free(x) } - } - IValue::from_c(c_ivalue) - } - - /// Switches the module to evaluation mode. - pub fn f_set_eval(&mut self) -> Result<(), TchError> { - unsafe_torch_err!(atm_eval(self.c_module)); - Ok(()) - } - - /// Switches the module to evaluation mode. - pub fn set_eval(&mut self) { - self.f_set_eval().unwrap(); - } - - /// Switches the module to training mode. - pub fn f_set_train(&mut self) -> Result<(), TchError> { - unsafe_torch_err!(atm_train(self.c_module)); - Ok(()) - } - - /// Switches the module to training mode. - pub fn set_train(&mut self) { - self.f_set_train().unwrap(); - } - - /// Moves the module to a different device and converts the kind. - pub fn to(&mut self, device: Device, kind: Kind, non_blocking: bool) { - unsafe_torch!(atm_to(self.c_module, device.c_int(), kind.c_int(), non_blocking)); - } - - /// Saves a module to a given path. - pub fn save>(&self, path: T) -> Result<(), TchError> { - let path = path_to_cstring(path)?; - unsafe_torch_err!(atm_save(self.c_module, path.as_ptr())); - Ok(()) - } - - /// Loads some named tensors from a module - pub fn named_parameters(&self) -> Result, TchError> { - let mut v: Vec<(String, Tensor)> = vec![]; - unsafe_torch_err!(atm_named_parameters( - self.c_module, - &mut v as *mut _ as *mut c_void, - super::tensor::add_callback - )); - Ok(v) - } - - /// Create a new module by tracing the application of the specified function on - /// the given inputs. - pub fn create_by_tracing( - modl_name: &str, - fn_name: &str, - inputs: &[Tensor], - closure: &mut F, - ) -> Result - where - F: FnMut(&[Tensor]) -> Vec, - { - let modl_name = std::ffi::CString::new(modl_name)?; - let fn_name = std::ffi::CString::new(fn_name)?; - let c_inputs = inputs.iter().map(|tensor| tensor.c_tensor).collect::>(); - let c_module = unsafe_torch_err!(atm_create_for_tracing( - modl_name.as_ptr(), - c_inputs.as_ptr(), - c_inputs.len() as c_int - )); - let outputs = closure(inputs); - let c_outputs = outputs.iter().map(|tensor| tensor.c_tensor).collect::>(); - unsafe_torch_err!(atm_end_tracing( - c_module, - fn_name.as_ptr(), - c_outputs.as_ptr(), - c_outputs.len() as c_int, - )); - Ok(CModule { c_module }) - } -} - -/// The trainable version of a jit PyTorch module. -/// -/// These modules can be created via the -/// [TorchScript python api](https://pytorch.org/docs/stable/jit.html). -#[derive(Debug)] -pub struct TrainableCModule { - pub(crate) inner: CModule, -} - -impl TrainableCModule { - /// Loads a PyTorch saved JIT module from a file. - /// - /// This function also adds the tensors from the JIT module to the VarStore path - /// passed as argument so that the module can be trained. - pub fn load>(module_path: T, path: Path) -> Result { - let inner = CModule::load_on_device(module_path, path.device())?; - for (name, tensor) in inner.named_parameters()? { - let requires_grad = tensor.requires_grad(); - let _t = path.add(&name.replace('.', "_"), tensor, requires_grad); - } - Ok(TrainableCModule { inner }) - } - - /// Loads a PyTorch saved JIT model from a read instance. - /// - /// This function also adds the tensors from the JIT module to the VarStore path - /// passed as argument so that the module can be trained. - pub fn load_data(data: &mut T, path: Path) -> Result { - let inner = CModule::load_data_on_device(data, path.device())?; - for (name, tensor) in inner.named_parameters()? { - let requires_grad = tensor.requires_grad(); - let _t = path.add(&name.replace('.', "_"), tensor, requires_grad); - } - Ok(TrainableCModule { inner }) - } - - pub fn save>(&self, module_path: T) -> Result<(), TchError> { - self.inner.save(module_path) - } - - /// Switches the module to training mode. - pub fn f_set_train(&mut self) -> Result<(), TchError> { - self.inner.f_set_train() - } - - /// Switches the module to training mode. - pub fn set_train(&mut self) { - self.inner.set_train() - } - - /// Switches the module to evaluation mode. - pub fn f_set_eval(&mut self) -> Result<(), TchError> { - self.inner.f_set_eval() - } - - /// Switches the module to evaluation mode. - pub fn set_eval(&mut self) { - self.inner.set_eval() - } - - /// Performs the forward pass for a model on some specified tensor inputs. - pub fn forward_ts>(&self, ts: &[T]) -> Result { - self.inner.forward_ts(ts) - } - - /// Performs the forward pass for a model on some specified ivalue inputs. - pub fn forward_is>(&self, ts: &[T]) -> Result { - self.inner.forward_is(ts) - } - - /// Runs a specified entry point for a model on some given tensor inputs. - pub fn method_ts>( - &self, - method_name: &str, - ts: &[T], - ) -> Result { - self.inner.method_ts(method_name, ts) - } - - /// Runs a specified entry point for a model on some given ivalue inputs. - pub fn method_is>( - &self, - method_name: &str, - ts: &[T], - ) -> Result { - self.inner.method_is(method_name, ts) - } -} - -/// Returns whether profiling mode is set or not. -pub fn f_get_profiling_mode() -> Result { - Ok(unsafe_torch_err!(atm_get_profiling_mode()) != 0) -} - -/// Returns whether profiling mode is set or not. -pub fn get_profiling_mode() -> bool { - f_get_profiling_mode().unwrap() -} - -/// Activates or deactivates the profiling mode. -pub fn f_set_profiling_mode(b: bool) -> Result<(), TchError> { - unsafe_torch_err!(atm_set_profiling_mode(b as c_int)); - Ok(()) -} - -/// Activates or deactivates the profiling mode. -pub fn set_profiling_mode(b: bool) { - f_set_profiling_mode(b).unwrap() -} - -pub fn f_fuser_cuda_set_enabled(enabled: bool) -> Result<(), TchError> { - unsafe_torch_err!(atm_fuser_cuda_set_enabled(enabled)); - Ok(()) -} - -pub fn fuser_cuda_set_enabled(enabled: bool) { - f_fuser_cuda_set_enabled(enabled).unwrap() -} - -pub fn f_fuser_cuda_is_enabled() -> Result { - let b = unsafe_torch_err!(atm_fuser_cuda_is_enabled()); - Ok(b) -} - -pub fn fuser_cuda_is_enabled() -> bool { - f_fuser_cuda_is_enabled().unwrap() -} - -pub fn f_set_tensor_expr_fuser_enabled(b: bool) -> Result<(), TchError> { - unsafe_torch_err!(atm_set_tensor_expr_fuser_enabled(b as c_int)); - Ok(()) -} - -pub fn set_tensor_expr_fuser_enabled(b: bool) { - f_set_tensor_expr_fuser_enabled(b).unwrap() -} - -pub fn f_get_tensor_expr_fuser_enabled() -> Result { - Ok(unsafe_torch_err!(atm_get_tensor_expr_fuser_enabled())) -} - -pub fn get_tensor_expr_fuser_enabled() -> bool { - f_get_tensor_expr_fuser_enabled().unwrap() -} - -/// Enables or disables the graph executor optimizer for the current thread. -/// -/// # Arguments -/// -/// * `b` - A boolean that if true enables the graph executor optimizer for the current thread. -/// -/// This function returns an error if it is not possible to enable or disable the graph executor optimizer. -pub fn f_set_graph_executor_optimize(b: bool) -> Result<(), TchError> { - unsafe_torch_err!(at_set_graph_executor_optimize(b)); - Ok(()) -} - -/// Enables or disables the graph executor optimizer for the current thread. -/// -/// # Arguments -/// -/// * `b` - A boolean that if true enables the graph executor optimizer for the current thread. -/// -/// This panics if it is not possible to enable or disable the graph executor optimizer. -pub fn set_graph_executor_optimize(b: bool) { - f_set_graph_executor_optimize(b).unwrap(); -} - -#[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Debug, PartialEq)] -pub struct Object { - c_ivalue: *mut CIValue, -} - -impl Object { - /// Applies the specified method to the object. The method takes as argument an arbitrary - /// number of ivalues and returns an ivalue. - pub fn method_is>( - &self, - method_name: &str, - ts: &[T], - ) -> Result { - let ts = ts.iter().map(|x| x.borrow().to_c()).collect::, TchError>>()?; - let method_name = std::ffi::CString::new(method_name)?; - let c_ivalue = unsafe_torch_err!(ati_object_method_( - self.c_ivalue, - method_name.as_ptr(), - ts.as_ptr(), - ts.len() as c_int - )); - for x in ts { - unsafe { ati_free(x) } - } - IValue::from_c(c_ivalue) - } - - /// Retrieves the specified attribute from an object as an ivalue. - pub fn getattr(&self, attr_name: &str) -> Result { - let property_name = std::ffi::CString::new(attr_name)?; - let c_ivalue = - unsafe_torch_err!(ati_object_getattr_(self.c_ivalue, property_name.as_ptr())); - if c_ivalue.is_null() { - return Err(TchError::Torch(format!( - "Object.getattr(\"{attr_name}\") returned CIValue nullptr" - ))); - } - IValue::from_c(c_ivalue) - } -} - -impl Drop for Object { - fn drop(&mut self) { - unsafe_torch!(ati_free(self.c_ivalue)) - } -} - -#[cfg(test)] -mod tests { - use super::IValue; - use std::f64::consts; - - fn round_trip>(t: T) { - let ivalue: IValue = t.into(); - let ivalue2 = IValue::from_c(ivalue.to_c().unwrap()).unwrap(); - assert_eq!(ivalue, ivalue2); - } - #[test] - fn ivalue_round_trip() { - round_trip(()); - round_trip(true); - round_trip(false); - round_trip(-1); - round_trip(42); - round_trip(15); - round_trip("".to_string()); - round_trip("foobar".to_string()); - round_trip((42, consts::PI)); - round_trip(vec![42, 1337]); - round_trip(vec![consts::E, consts::PI, 299792458.00001]); - round_trip((vec![true, false, true, true], vec![consts::E, consts::PI, 299792458.00001])); - round_trip(vec![IValue::from(42), IValue::from("foobar")]); - round_trip(vec![ - (IValue::from(42), IValue::from("foobar")), - (IValue::from("foo"), IValue::from("bar")), - ]); - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/kind.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/kind.rs deleted file mode 100644 index 390c623fec..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/wrappers/kind.rs +++ /dev/null @@ -1,176 +0,0 @@ -//! The different kind of elements supported in Torch. - -use half; - -/// The different kind of elements that a Tensor can hold. -#[allow(clippy::upper_case_acronyms)] -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub enum Kind { - Uint8, - Int8, - Int16, - Int, - Int64, - Half, - Float, - Double, - ComplexHalf, - ComplexFloat, - ComplexDouble, - Bool, - QInt8, - QUInt8, - QInt32, - BFloat16, - Float8e5m2, - Float8e4m3fn, - Float8e5m2fnuz, - Float8e4m3fnuz, -} - -impl Kind { - pub(super) fn c_int(self) -> libc::c_int { - // These values should be in sync with include/c10/core/ScalarType.h - // https://github.com/pytorch/pytorch/blob/a8d6afb511a69687bbb2b7e88a3cf67917e1697e/c10/core/ScalarType.h#L57 - match self { - Kind::Uint8 => 0, - Kind::Int8 => 1, - Kind::Int16 => 2, - Kind::Int => 3, - Kind::Int64 => 4, - Kind::Half => 5, - Kind::Float => 6, - Kind::Double => 7, - Kind::ComplexHalf => 8, - Kind::ComplexFloat => 9, - Kind::ComplexDouble => 10, - Kind::Bool => 11, - Kind::QInt8 => 12, - Kind::QUInt8 => 13, - Kind::QInt32 => 14, - Kind::BFloat16 => 15, - Kind::Float8e5m2 => 23, - Kind::Float8e4m3fn => 24, - Kind::Float8e5m2fnuz => 25, - Kind::Float8e4m3fnuz => 26, - } - } - - pub(super) fn from_c_int(v: libc::c_int) -> Result { - match v { - 0 => Ok(Kind::Uint8), - 1 => Ok(Kind::Int8), - 2 => Ok(Kind::Int16), - 3 => Ok(Kind::Int), - 4 => Ok(Kind::Int64), - 5 => Ok(Kind::Half), - 6 => Ok(Kind::Float), - 7 => Ok(Kind::Double), - 8 => Ok(Kind::ComplexHalf), - 9 => Ok(Kind::ComplexFloat), - 10 => Ok(Kind::ComplexDouble), - 11 => Ok(Kind::Bool), - 12 => Ok(Kind::QInt8), - 13 => Ok(Kind::QUInt8), - 14 => Ok(Kind::QInt32), - 15 => Ok(Kind::BFloat16), - 23 => Ok(Kind::Float8e5m2), - 24 => Ok(Kind::Float8e4m3fn), - 25 => Ok(Kind::Float8e5m2fnuz), - 26 => Ok(Kind::Float8e4m3fnuz), - _ => Err(crate::TchError::UnknownKind(v)), - } - } - - pub fn elt_size_in_bytes(self) -> usize { - match self { - Kind::Uint8 => 1, - Kind::Int8 => 1, - Kind::Int16 => 2, - Kind::Int => 4, - Kind::Int64 => 8, - Kind::Half => 2, - Kind::Float => 4, - Kind::Double => 8, - Kind::ComplexHalf => 4, - Kind::ComplexFloat => 8, - Kind::ComplexDouble => 16, - Kind::Bool => 1, - Kind::QInt8 => 1, - Kind::QUInt8 => 1, - Kind::QInt32 => 4, - Kind::BFloat16 => 2, - Kind::Float8e5m2 => 1, - Kind::Float8e4m3fn => 1, - Kind::Float8e5m2fnuz => 1, - Kind::Float8e4m3fnuz => 1, - } - } -} - -pub const FLOAT_CPU: (Kind, crate::Device) = (Kind::Float, crate::Device::Cpu); -pub const DOUBLE_CPU: (Kind, crate::Device) = (Kind::Double, crate::Device::Cpu); -pub const INT64_CPU: (Kind, crate::Device) = (Kind::Int64, crate::Device::Cpu); - -pub const FLOAT_CUDA: (Kind, crate::Device) = (Kind::Float, crate::Device::Cuda(0)); -pub const DOUBLE_CUDA: (Kind, crate::Device) = (Kind::Double, crate::Device::Cuda(0)); -pub const INT64_CUDA: (Kind, crate::Device) = (Kind::Int64, crate::Device::Cuda(0)); - -/// Kinds for tensor elements -/// -/// # Safety -/// The specified Kind must be for a type that has the same length as Self. -pub unsafe trait Element: Clone { - const KIND: Kind; - const ZERO: Self; -} - -unsafe impl Element for u8 { - const KIND: Kind = Kind::Uint8; - const ZERO: Self = 0; -} - -unsafe impl Element for i8 { - const KIND: Kind = Kind::Int8; - const ZERO: Self = 0; -} - -unsafe impl Element for i16 { - const KIND: Kind = Kind::Int16; - const ZERO: Self = 0; -} - -unsafe impl Element for i32 { - const KIND: Kind = Kind::Int; - const ZERO: Self = 0; -} - -unsafe impl Element for i64 { - const KIND: Kind = Kind::Int64; - const ZERO: Self = 0; -} - -unsafe impl Element for half::f16 { - const KIND: Kind = Kind::Half; - const ZERO: Self = half::f16::ZERO; -} - -unsafe impl Element for half::bf16 { - const KIND: Kind = Kind::Half; - const ZERO: Self = half::bf16::ZERO; -} - -unsafe impl Element for f32 { - const KIND: Kind = Kind::Float; - const ZERO: Self = 0.; -} - -unsafe impl Element for f64 { - const KIND: Kind = Kind::Double; - const ZERO: Self = 0.; -} - -unsafe impl Element for bool { - const KIND: Kind = Kind::Bool; - const ZERO: Self = false; -} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/layout.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/layout.rs deleted file mode 100644 index 5c26aded2a..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/wrappers/layout.rs +++ /dev/null @@ -1,29 +0,0 @@ -/// A tensor layout. - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub enum Layout { - Strided, - Sparse, - SparseCsr, - Mkldnn, - SparseCsc, - SparseBsr, - SparseBsc, - NumOptions, -} - -impl Layout { - // This should be kept in sync with include/c10/core/Layout.h - pub fn to_i8(&self) -> i8 { - match self { - Self::Strided => 0, - Self::Sparse => 1, - Self::SparseCsr => 2, - Self::Mkldnn => 3, - Self::SparseCsc => 4, - Self::SparseBsr => 5, - Self::SparseBsc => 6, - Self::NumOptions => 7, - } - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/mod.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/mod.rs deleted file mode 100644 index a3b69b9b9f..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/wrappers/mod.rs +++ /dev/null @@ -1,21 +0,0 @@ -#[macro_use] -pub mod utils; - -pub use utils::{ - get_num_interop_threads, get_num_threads, manual_seed, set_num_interop_threads, - set_num_threads, QEngine, -}; - -pub(crate) mod device; -pub(crate) mod image; -pub mod jit; -pub mod kind; -pub(crate) mod layout; -pub(crate) mod optimizer; -#[cfg(feature = "python-extension")] -pub mod python; -pub(crate) mod scalar; -pub(crate) mod stream; -pub(crate) mod tensor; -pub(crate) mod tensor_fallible_generated; -pub(crate) mod tensor_generated; diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/optimizer.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/optimizer.rs deleted file mode 100644 index c55d04fc4b..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/wrappers/optimizer.rs +++ /dev/null @@ -1,129 +0,0 @@ -use super::tensor::Tensor; -use crate::TchError; - -pub struct COptimizer { - c_optimizer: *mut torch_sys::C_optimizer, -} - -unsafe impl Send for COptimizer {} - -impl std::fmt::Debug for COptimizer { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - write!(f, "optimizer") - } -} - -impl COptimizer { - pub fn adam( - lr: f64, - beta1: f64, - beta2: f64, - wd: f64, - eps: f64, - amsgrad: bool, - ) -> Result { - let c_optimizer = - unsafe_torch_err!(torch_sys::ato_adam(lr, beta1, beta2, wd, eps, amsgrad)); - Ok(COptimizer { c_optimizer }) - } - - pub fn adamw( - lr: f64, - beta1: f64, - beta2: f64, - wd: f64, - eps: f64, - amsgrad: bool, - ) -> Result { - let c_optimizer = - unsafe_torch_err!(torch_sys::ato_adamw(lr, beta1, beta2, wd, eps, amsgrad)); - Ok(COptimizer { c_optimizer }) - } - - // Maybe we should use the builder pattern to provide default values for these ? - pub fn rms_prop( - lr: f64, - alpha: f64, - eps: f64, - wd: f64, - momentum: f64, - centered: bool, - ) -> Result { - let centered = i32::from(centered); - let c_optimizer = - unsafe_torch_err!(torch_sys::ato_rms_prop(lr, alpha, eps, wd, momentum, centered)); - Ok(COptimizer { c_optimizer }) - } - - pub fn sgd( - lr: f64, - momentum: f64, - dampening: f64, - wd: f64, - nesterov: bool, - ) -> Result { - let nesterov = i32::from(nesterov); - let c_optimizer = - unsafe_torch_err!(torch_sys::ato_sgd(lr, momentum, dampening, wd, nesterov)); - Ok(COptimizer { c_optimizer }) - } - - pub fn add_parameters(&mut self, t: &Tensor, group: usize) -> Result<(), TchError> { - unsafe_torch_err!(torch_sys::ato_add_parameters(self.c_optimizer, t.c_tensor, group)); - Ok(()) - } - - pub fn set_learning_rate(&mut self, lr: f64) -> Result<(), TchError> { - unsafe_torch_err!(torch_sys::ato_set_learning_rate(self.c_optimizer, lr)); - Ok(()) - } - - pub fn set_learning_rate_group(&mut self, group: usize, lr: f64) -> Result<(), TchError> { - unsafe_torch_err!(torch_sys::ato_set_learning_rate_group(self.c_optimizer, group, lr)); - Ok(()) - } - - pub fn set_momentum(&mut self, m: f64) -> Result<(), TchError> { - unsafe_torch_err!(torch_sys::ato_set_momentum(self.c_optimizer, m)); - Ok(()) - } - - pub fn set_momentum_group(&mut self, group: usize, m: f64) -> Result<(), TchError> { - unsafe_torch_err!(torch_sys::ato_set_momentum_group(self.c_optimizer, group, m)); - Ok(()) - } - - pub fn set_weight_decay(&mut self, weight_decay: f64) -> Result<(), TchError> { - unsafe_torch_err!(torch_sys::ato_set_weight_decay(self.c_optimizer, weight_decay)); - Ok(()) - } - - pub fn set_weight_decay_group( - &mut self, - group: usize, - weight_decay: f64, - ) -> Result<(), TchError> { - unsafe_torch_err!(torch_sys::ato_set_weight_decay_group( - self.c_optimizer, - group, - weight_decay - )); - Ok(()) - } - - pub fn zero_grad(&self) -> Result<(), TchError> { - unsafe_torch_err!(torch_sys::ato_zero_grad(self.c_optimizer)); - Ok(()) - } - - pub fn step(&self) -> Result<(), TchError> { - unsafe_torch_err!(torch_sys::ato_step(self.c_optimizer)); - Ok(()) - } -} - -impl Drop for COptimizer { - fn drop(&mut self) { - unsafe { torch_sys::ato_free(self.c_optimizer) } - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/python.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/python.rs deleted file mode 100644 index 136625d63c..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/wrappers/python.rs +++ /dev/null @@ -1,34 +0,0 @@ -use crate::{TchError, Tensor}; -use torch_sys::python::{self, C_pyobject}; - -pub type CPyObject = C_pyobject; - -/// Check whether an object is a wrapped tensor or not. -/// -/// # Safety -/// Undefined behavior if the given pointer is not a valid PyObject. -pub unsafe fn pyobject_check(pyobject: *mut CPyObject) -> Result { - let v = unsafe_torch_err!(python::thp_variable_check(pyobject)); - Ok(v) -} - -impl Tensor { - /// Wrap a tensor in a Python object. - pub fn pyobject_wrap(&self) -> Result<*mut CPyObject, TchError> { - let v = unsafe_torch_err!(python::thp_variable_wrap(self.c_tensor)); - Ok(v) - } - - /// Unwrap a tensor stored in a Python object. This returns `Ok(None)` if - /// the object is not a wrapped tensor. - /// - /// # Safety - /// Undefined behavior if the given pointer is not a valid PyObject. - pub unsafe fn pyobject_unpack(pyobject: *mut CPyObject) -> Result, TchError> { - if !pyobject_check(pyobject)? { - return Ok(None); - } - let v = unsafe_torch_err!(python::thp_variable_unpack(pyobject)); - Ok(Some(Tensor::from_ptr(v))) - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/scalar.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/scalar.rs deleted file mode 100644 index 3ae275919b..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/wrappers/scalar.rs +++ /dev/null @@ -1,111 +0,0 @@ -//! Scalar elements. - -use crate::TchError; - -/// A single scalar value. -pub struct Scalar { - pub(super) c_scalar: *mut torch_sys::C_scalar, -} - -impl Scalar { - /// Creates an integer scalar. - pub fn int(v: i64) -> Scalar { - let c_scalar = unsafe_torch!(torch_sys::ats_int(v)); - Scalar { c_scalar } - } - - /// Creates a float scalar scalar. - pub fn float(v: f64) -> Scalar { - let c_scalar = unsafe_torch!(torch_sys::ats_float(v)); - Scalar { c_scalar } - } - - /// Returns an integer value. - pub fn to_int(&self) -> Result { - let i = unsafe_torch_err!(torch_sys::ats_to_int(self.c_scalar)); - Ok(i) - } - - /// Returns a float value. - pub fn to_float(&self) -> Result { - let f = unsafe_torch_err!(torch_sys::ats_to_float(self.c_scalar)); - Ok(f) - } - - /// Returns a string representation of the scalar. - pub fn to_string(&self) -> Result { - let s = unsafe_torch_err!({ - super::utils::ptr_to_string(torch_sys::ats_to_string(self.c_scalar)) - }); - match s { - None => Err(TchError::Kind("nullptr representation".to_string())), - Some(s) => Ok(s), - } - } -} - -impl std::fmt::Debug for Scalar { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - match self.to_string() { - Err(_) => write!(f, "err"), - Ok(s) => write!(f, "scalar<{s}>"), - } - } -} - -impl Drop for Scalar { - fn drop(&mut self) { - unsafe_torch!(torch_sys::ats_free(self.c_scalar)) - } -} - -impl From for Scalar { - fn from(v: i64) -> Scalar { - Scalar::int(v) - } -} - -impl From for Scalar { - fn from(v: f64) -> Scalar { - Scalar::float(v) - } -} - -impl From for i64 { - fn from(s: Scalar) -> i64 { - Self::from(&s) - } -} - -impl From for f64 { - fn from(s: Scalar) -> f64 { - Self::from(&s) - } -} - -impl From<&Scalar> for i64 { - fn from(s: &Scalar) -> i64 { - s.to_int().unwrap() - } -} - -impl From<&Scalar> for f64 { - fn from(s: &Scalar) -> f64 { - s.to_float().unwrap() - } -} - -#[cfg(test)] -mod tests { - use super::Scalar; - #[test] - fn scalar() { - let pi = Scalar::float(std::f64::consts::PI); - assert_eq!(i64::from(&pi), 3); - assert_eq!(f64::from(&pi), std::f64::consts::PI); - let leet = Scalar::int(1337); - assert_eq!(i64::from(&leet), 1337); - assert_eq!(f64::from(&leet), 1337.); - assert_eq!(&format!("{pi:?}"), "scalar<3.14159>"); - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/stream.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/stream.rs deleted file mode 100644 index 4468d328eb..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/wrappers/stream.rs +++ /dev/null @@ -1,26 +0,0 @@ -use std::io::{Read, Result, Seek, SeekFrom}; -use torch_sys::io::ReadStream; - -pub struct ReadSeekAdapter { - inner: T, -} - -impl ReadSeekAdapter { - pub fn new(inner: T) -> Self { - Self { inner } - } -} - -impl Read for ReadSeekAdapter { - fn read(&mut self, buf: &mut [u8]) -> Result { - self.inner.read(buf) - } -} - -impl Seek for ReadSeekAdapter { - fn seek(&mut self, pos: SeekFrom) -> Result { - self.inner.seek(pos) - } -} - -impl ReadStream for ReadSeekAdapter {} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/tensor.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/tensor.rs deleted file mode 100644 index 3d5cf7fc74..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/wrappers/tensor.rs +++ /dev/null @@ -1,883 +0,0 @@ -use super::stream::ReadSeekAdapter; -use super::utils::{path_to_cstring, ptr_to_string}; -use super::{ - device::{Cuda, Device}, - kind, - kind::Kind, -}; -use crate::TchError; -use libc::{c_char, c_int, c_void}; -use std::borrow::Borrow; -use std::io::{Read, Seek, Write}; -use std::path::Path; -use torch_sys::io::ReadStream; -use torch_sys::*; - -/// A tensor object. -#[must_use] -pub struct Tensor { - pub(super) c_tensor: *mut C_tensor, -} - -unsafe impl Send for Tensor {} - -pub extern "C" fn add_callback(data: *mut c_void, name: *const c_char, c_tensor: *mut C_tensor) { - let name = unsafe { std::ffi::CStr::from_ptr(name).to_str().unwrap() }; - let name = name.replace('|', "."); - let v: &mut Vec<(String, Tensor)> = unsafe { &mut *(data as *mut Vec<(String, Tensor)>) }; - v.push((name, Tensor { c_tensor })) -} - -impl Tensor { - /// Creates a new tensor. - pub fn new() -> Tensor { - let c_tensor = unsafe_torch!(at_new_tensor()); - Tensor { c_tensor } - } - - /// Creates a new tensor from the pointer to an existing C++ tensor. - /// - /// # Safety - /// - /// The caller must ensures that the pointer outlives the Rust - /// object. - pub unsafe fn from_ptr(c_tensor: *mut C_tensor) -> Self { - Self { c_tensor } - } - - /// Creates a new tensor from the pointer to an existing C++ tensor. - /// - /// # Safety - /// - /// A shallow copy of the pointer is made so there is no need for - /// this pointer to remain valid for the whole lifetime of the Rust - /// object. - pub unsafe fn clone_from_ptr(c_tensor: *mut C_tensor) -> Self { - let c_tensor = at_shallow_clone(c_tensor); - crate::wrappers::utils::read_and_clean_error().unwrap(); - Self { c_tensor } - } - - /// Returns a pointer to the underlying C++ tensor. - /// - /// The caller must ensures that the Rust tensor object outlives - /// this pointer. - pub fn as_ptr(&self) -> *const C_tensor { - self.c_tensor - } - - /// Returns a mutable pointer to the underlying C++ tensor. - /// - /// The caller must ensures that the Rust tensor object outlives - /// this pointer. - pub fn as_mut_ptr(&mut self) -> *mut C_tensor { - self.c_tensor - } - - /// Returns the number of dimension of the tensor. - pub fn dim(&self) -> usize { - unsafe_torch!(at_dim(self.c_tensor)) - } - - /// Returns the shape of the input tensor. - pub fn size(&self) -> Vec { - let dim = unsafe_torch!(at_dim(self.c_tensor)); - let mut sz = vec![0i64; dim]; - unsafe_torch!(at_shape(self.c_tensor, sz.as_mut_ptr())); - sz - } - - /// Returns the tensor size for single dimension tensors. - pub fn size1(&self) -> Result { - match self.size().as_slice() { - &[s0] => Ok(s0), - size => Err(TchError::Shape(format!("expected one dim, got {size:?}"))), - } - } - - /// Returns the tensor sizes for two dimension tensors. - pub fn size2(&self) -> Result<(i64, i64), TchError> { - match self.size().as_slice() { - &[s0, s1] => Ok((s0, s1)), - size => Err(TchError::Shape(format!("expected two dims, got {size:?}"))), - } - } - - /// Returns the tensor sizes for three dimension tensors. - pub fn size3(&self) -> Result<(i64, i64, i64), TchError> { - match self.size().as_slice() { - &[s0, s1, s2] => Ok((s0, s1, s2)), - size => Err(TchError::Shape(format!("expected three dims, got {size:?}"))), - } - } - - /// Returns the tensor sizes for four dimension tensors. - pub fn size4(&self) -> Result<(i64, i64, i64, i64), TchError> { - match self.size().as_slice() { - &[s0, s1, s2, s3] => Ok((s0, s1, s2, s3)), - size => Err(TchError::Shape(format!("expected four dims, got {size:?}"))), - } - } - - /// Returns the tensor sizes for five dimension tensors. - pub fn size5(&self) -> Result<(i64, i64, i64, i64, i64), TchError> { - match self.size().as_slice() { - &[s0, s1, s2, s3, s4] => Ok((s0, s1, s2, s3, s4)), - size => Err(TchError::Shape(format!("expected five dims, got {size:?}"))), - } - } - - /// Returns the tensor sizes for six dimension tensors. - pub fn size6(&self) -> Result<(i64, i64, i64, i64, i64, i64), TchError> { - match self.size().as_slice() { - &[s0, s1, s2, s3, s4, s5] => Ok((s0, s1, s2, s3, s4, s5)), - size => Err(TchError::Shape(format!("expected six dims, got {size:?}"))), - } - } - - /// Returns the stride of the input tensor. - pub fn stride(&self) -> Vec { - let dim = unsafe_torch!(at_dim(self.c_tensor)); - let mut sz = vec![0i64; dim]; - unsafe_torch!(at_stride(self.c_tensor, sz.as_mut_ptr())); - sz - } - - /// Returns the tensor strides for single dimension tensors. - pub fn stride1(&self) -> Result { - match self.stride().as_slice() { - &[s0] => Ok(s0), - size => Err(TchError::Shape(format!("expected one dim, got {size:?}"))), - } - } - - /// Returns the tensor strides for two dimension tensors. - pub fn stride2(&self) -> Result<(i64, i64), TchError> { - match self.stride().as_slice() { - &[s0, s1] => Ok((s0, s1)), - size => Err(TchError::Shape(format!("expected two dims, got {size:?}"))), - } - } - - /// Returns the tensor strides for three dimension tensors. - pub fn stride3(&self) -> Result<(i64, i64, i64), TchError> { - match self.stride().as_slice() { - &[s0, s1, s2] => Ok((s0, s1, s2)), - size => Err(TchError::Shape(format!("expected three dims, got {size:?}"))), - } - } - - /// Returns the tensor strides for four dimension tensors. - pub fn stride4(&self) -> Result<(i64, i64, i64, i64), TchError> { - match self.stride().as_slice() { - &[s0, s1, s2, s3] => Ok((s0, s1, s2, s3)), - size => Err(TchError::Shape(format!("expected four dims, got {size:?}"))), - } - } - - /// Returns the tensor strides for five dimension tensors. - pub fn stride5(&self) -> Result<(i64, i64, i64, i64, i64), TchError> { - match self.stride().as_slice() { - &[s0, s1, s2, s3, s4] => Ok((s0, s1, s2, s3, s4)), - size => Err(TchError::Shape(format!("expected five dims, got {size:?}"))), - } - } - - /// Returns the tensor strides for six dimension tensors. - pub fn stride6(&self) -> Result<(i64, i64, i64, i64, i64, i64), TchError> { - match self.stride().as_slice() { - &[s0, s1, s2, s3, s4, s5] => Ok((s0, s1, s2, s3, s4, s5)), - size => Err(TchError::Shape(format!("expected six dims, got {size:?}"))), - } - } - - /// Returns the kind of elements stored in the input tensor. Returns - /// an error on undefined tensors and unsupported data types. - pub fn f_kind(&self) -> Result { - let kind = unsafe_torch!(at_scalar_type(self.c_tensor)); - Kind::from_c_int(kind) - } - - /// Returns the kind of elements stored in the input tensor. Panics - /// an error on undefined tensors and unsupported data types. - pub fn kind(&self) -> Kind { - self.f_kind().unwrap() - } - - /// Returns the device on which the input tensor is located. - pub fn device(&self) -> Device { - let device = unsafe_torch!(at_device(self.c_tensor)); - Device::from_c_int(device) - } - - /// Prints the input tensor. - /// - /// Caution: this uses the C++ printer which prints the whole tensor even if - /// it is very large. - pub fn print(&self) { - unsafe_torch!(at_print(self.c_tensor)) - } - - /// Returns a double value on tensors holding a single element. An error is - /// returned otherwise. - pub fn f_double_value(&self, idx: &[i64]) -> Result { - Ok(unsafe_torch_err!({ - at_double_value_at_indexes(self.c_tensor, idx.as_ptr(), idx.len() as i32) - })) - } - - /// Returns an int value on tensors holding a single element. An error is - /// returned otherwise. - pub fn f_int64_value(&self, idx: &[i64]) -> Result { - Ok(unsafe_torch_err!({ - at_int64_value_at_indexes(self.c_tensor, idx.as_ptr(), idx.len() as i32) - })) - } - - /// Returns a double value on tensors holding a single element. Panics otherwise. - pub fn double_value(&self, idx: &[i64]) -> f64 { - self.f_double_value(idx).unwrap() - } - - /// Returns an int value on tensors holding a single element. Panics otherwise. - pub fn int64_value(&self, idx: &[i64]) -> i64 { - self.f_int64_value(idx).unwrap() - } - - /// Returns true if gradient are currently tracked for this tensor. - pub fn requires_grad(&self) -> bool { - unsafe_torch!(at_requires_grad(self.c_tensor)) != 0 - } - - /// Returns the address of the first element of this tensor. - pub fn data_ptr(&self) -> *mut c_void { - unsafe_torch!(at_data_ptr(self.c_tensor)) - } - - /// Returns true if the tensor is defined. - pub fn defined(&self) -> bool { - unsafe_torch!(at_defined(self.c_tensor) != 0) - } - - /// Returns true if the tensor is compatible with MKL-DNN (oneDNN). - pub fn is_mkldnn(&self) -> bool { - unsafe_torch!(at_is_mkldnn(self.c_tensor) != 0) - } - - /// Returns true if the tensor is sparse. - pub fn is_sparse(&self) -> bool { - unsafe_torch!(at_is_sparse(self.c_tensor) != 0) - } - - // Returns true if the tensor if contiguous - pub fn is_contiguous(&self) -> bool { - unsafe_torch!(at_is_contiguous(self.c_tensor) != 0) - } - - /// Zeroes the gradient tensor attached to this tensor if defined. - pub fn zero_grad(&mut self) { - let mut grad = self.grad(); - if grad.defined() { - let _ = grad.detach_().zero_(); - } - } - - /// Runs the backward pass, populating the gradient tensors for tensors - /// which gradients are tracked. - /// - /// Gradients tracking can be turned on via `set_requires_grad`. - pub fn f_backward(&self) -> Result<(), TchError> { - unsafe_torch_err!(at_backward(self.c_tensor, 0, 0)); - Ok(()) - } - - /// Runs the backward pass, populating the gradient tensors for tensors - /// which gradients are tracked. - /// - /// Gradients tracking can be turned on via `set_requires_grad`. - /// Panics if the C++ api returns an exception. - pub fn backward(&self) { - self.f_backward().unwrap() - } - - pub fn f_run_backward( - tensors: &[T1], - inputs: &[T2], - keep_graph: bool, - create_graph: bool, - ) -> Result, TchError> - where - T1: Borrow, - T2: Borrow, - { - let mut outputs = vec![std::ptr::null_mut(); inputs.len()]; - let tensors: Vec<_> = tensors.iter().map(|x| x.borrow().c_tensor).collect(); - let inputs: Vec<_> = inputs.iter().map(|x| x.borrow().c_tensor).collect(); - unsafe_torch_err!(at_run_backward( - tensors.as_ptr(), - tensors.len() as c_int, - inputs.as_ptr(), - inputs.len() as c_int, - outputs.as_mut_ptr(), - keep_graph as c_int, - create_graph as c_int, - )); - Ok(outputs.into_iter().map(|c_tensor| Tensor { c_tensor }).collect()) - } - - pub fn run_backward( - tensors: &[T1], - inputs: &[T2], - keep_graph: bool, - create_graph: bool, - ) -> Vec - where - T1: Borrow, - T2: Borrow, - { - Tensor::f_run_backward(tensors, inputs, keep_graph, create_graph).unwrap() - } - - /// Copies `numel` elements from `self` to `dst`. - pub fn f_copy_data_u8(&self, dst: &mut [u8], numel: usize) -> Result<(), TchError> { - let elt_size_in_bytes = self.f_kind()?.elt_size_in_bytes(); - if dst.len() < numel * elt_size_in_bytes { - return Err(TchError::Shape(format!("slice len < {numel}"))); - } - unsafe_torch_err!(at_copy_data( - self.c_tensor, - dst.as_mut_ptr() as *const c_void, - numel, - elt_size_in_bytes, - )); - Ok(()) - } - - /// Unscale tensor while checking for infinities. - /// - /// `found_inf` is a singleton tensor that is used to record the - /// presence of infinite values. `inv_scale` is a scalar containing - /// the inverse scaling factor. This method is only available - /// for CUDA tensors. - pub fn f_internal_amp_non_finite_check_and_unscale( - &mut self, - found_inf: &mut Tensor, - inv_scale: &Tensor, - ) -> Result<(), TchError> { - unsafe_torch_err!(at__amp_non_finite_check_and_unscale( - self.c_tensor, - found_inf.c_tensor, - inv_scale.c_tensor - )); - - Ok(()) - } - - /// Unscale tensor while checking for infinities. - /// - /// `found_inf` is a singleton tensor that is used to record the - /// presence of infinite values. `inv_scale` is a scalar containing - /// the inverse scaling factor. This method is only available - /// for CUDA tensors. - pub fn internal_amp_non_finite_check_and_unscale( - &mut self, - found_inf: &mut Tensor, - inv_scale: &Tensor, - ) { - self.f_internal_amp_non_finite_check_and_unscale(found_inf, inv_scale).unwrap() - } - - /// Copies `numel` elements from `self` to `dst`. - pub fn copy_data_u8(&self, dst: &mut [u8], numel: usize) { - self.f_copy_data_u8(dst, numel).unwrap() - } - - /// Copies `numel` elements from `self` to `dst`. - pub fn f_copy_data( - &self, - dst: &mut [T], - numel: usize, - ) -> Result<(), TchError> { - if T::KIND != self.f_kind()? { - return Err(TchError::Kind(format!( - "incoherent elt kind, {:?} != {:?}", - self.f_kind(), - T::KIND - ))); - } - if dst.len() < numel { - return Err(TchError::Shape(format!("slice len < {numel}"))); - } - unsafe_torch_err!(at_copy_data( - self.c_tensor, - dst.as_mut_ptr() as *const c_void, - numel, - T::KIND.elt_size_in_bytes(), - )); - Ok(()) - } - - /// Copies `numel` elements from `self` to `dst`. - pub fn copy_data(&self, dst: &mut [T], numel: usize) { - self.f_copy_data(dst, numel).unwrap() - } - - /// Returns the total number of elements stored in a tensor. - pub fn numel(&self) -> usize { - self.size().iter().product::() as usize - } - - // This is similar to vec_... but faster as it directly blits the data. - /// Converts a slice to a tensor. - pub fn f_from_slice(data: &[T]) -> Result { - let data_len = data.len(); - let data = data.as_ptr() as *const c_void; - let c_tensor = unsafe_torch_err!(at_tensor_of_data( - data, - [data_len as i64].as_ptr(), - 1, - T::KIND.elt_size_in_bytes(), - T::KIND.c_int(), - )); - Ok(Tensor { c_tensor }) - } - - /// Converts a slice to a tensor. - pub fn from_slice(data: &[T]) -> Tensor { - Self::f_from_slice(data).unwrap() - } - - /// Converts some byte data to a tensor with some specified kind and shape. - pub fn f_from_data_size(data: &[u8], size: &[i64], kind: Kind) -> Result { - let data = data.as_ptr() as *const c_void; - let elt_size_in_bytes = kind.elt_size_in_bytes(); - let c_tensor = unsafe_torch_err!(at_tensor_of_data( - data, - size.as_ptr(), - size.len(), - elt_size_in_bytes, - kind.c_int(), - )); - Ok(Tensor { c_tensor }) - } - - /// Creates a tensor from data that is assumed to be initialized. - /// Resize operations are not allowed on this tensor without copying the data first. - /// An empty strides slice will result in using the default strides. - /// # Safety - /// Behavior is undefined if `data` points to invalid data. - pub unsafe fn f_from_blob( - data: *const u8, - size: &[i64], - strides: &[i64], - kind: Kind, - device: Device, - ) -> Result { - let data = data as *const c_void; - #[allow(unused_unsafe)] - let c_tensor = unsafe_torch_err!(at_tensor_of_blob( - data, - size.as_ptr(), - size.len(), - strides.as_ptr(), - strides.len(), - kind.c_int(), - device.c_int() - )); - Ok(Tensor { c_tensor }) - } - - /// Creates a tensor from data that is assumed to be initialized. - /// Resize operations are not allowed on this tensor without copying the data first. - /// An empty strides slice will result in using the default strides. - /// # Safety - /// Behavior is undefined if `data` points to invalid data. - pub unsafe fn from_blob( - data: *const u8, - size: &[i64], - strides: &[i64], - kind: Kind, - device: Device, - ) -> Tensor { - Self::f_from_blob(data, size, strides, kind, device).unwrap() - } - - /// Converts some byte data to a tensor with some specified kind and shape. - pub fn from_data_size(data: &[u8], size: &[i64], kind: Kind) -> Tensor { - Self::f_from_data_size(data, size, kind).unwrap() - } - - /// Returns a new tensor that share storage with the input tensor. - pub fn shallow_clone(&self) -> Tensor { - let c_tensor = unsafe_torch!(at_shallow_clone(self.c_tensor)); - Tensor { c_tensor } - } - - /// Gets the sub-tensor at the given index. - pub fn f_get(&self, index: i64) -> Result { - let c_tensor = unsafe_torch_err!(at_get(self.c_tensor, index as c_int)); - Ok(Tensor { c_tensor }) - } - - /// Gets the sub-tensor at the given index. - pub fn get(&self, index: i64) -> Tensor { - self.f_get(index).unwrap() - } - - /// Copies values from the argument tensor to the input tensor. - pub fn f_copy_(&mut self, src: &Tensor) -> Result<(), TchError> { - unsafe_torch_err!(at_copy_(self.c_tensor, src.c_tensor)); - Ok(()) - } - - /// Copies values from the argument tensor to the input tensor. - pub fn copy_(&mut self, src: &Tensor) { - self.f_copy_(src).unwrap() - } - - /// Loads a tensor from a file. - /// - /// The file format is the same as the one used by the PyTorch C++ API. - pub fn load>(path: T) -> Result { - let path = path_to_cstring(path)?; - let c_tensor = unsafe_torch_err!(at_load(path.as_ptr())); - Ok(Tensor { c_tensor }) - } - - /// Loads a tensor from a stream. - /// - /// The file format is the same as the one used by the PyTorch C++ API. - pub fn load_from_stream(stream: T) -> Result { - let adapter = ReadSeekAdapter::new(stream); - let boxed_stream: Box> = Box::new(Box::new(adapter)); - let c_tensor = - unsafe_torch_err!(at_load_from_stream(Box::into_raw(boxed_stream) as *mut c_void,)); - Ok(Tensor { c_tensor }) - } - - /// Saves a tensor to a file. - /// - /// The file format is the same as the one used by the PyTorch C++ API. - pub fn save>(&self, path: T) -> Result<(), TchError> { - let path = path_to_cstring(path)?; - unsafe_torch_err!(at_save(self.c_tensor, path.as_ptr())); - Ok(()) - } - - /// Saves a tensor to a stream. - /// - /// The file format is the same as the one used by the PyTorch C++ API. - pub fn save_to_stream(&self, stream: W) -> Result<(), TchError> { - let boxed_stream: Box> = Box::new(Box::new(stream)); - unsafe_torch_err!(at_save_to_stream( - self.c_tensor, - Box::into_raw(boxed_stream) as *mut c_void, - )); - Ok(()) - } - - /// Saves some named tensors to a file - /// - /// The file format is the same as the one used by the PyTorch C++ API. - pub fn save_multi, T: AsRef, P: AsRef>( - named_tensors: &[(S, T)], - path: P, - ) -> Result<(), TchError> { - let path = path_to_cstring(path)?; - let c_tensors = named_tensors.iter().map(|nt| nt.1.as_ref().c_tensor).collect::>(); - let names = named_tensors - .iter() - .map(|nt| nt.0.as_ref().replace('.', "|").into_bytes()) - .map(std::ffi::CString::new) - .collect::, _>>()?; - let name_ptrs = names.iter().map(|n| n.as_ptr()).collect::>(); - unsafe_torch_err!(at_save_multi( - c_tensors.as_ptr(), - name_ptrs.as_ptr(), - names.len() as i32, - path.as_ptr(), - )); - Ok(()) - } - - /// Saves some named tensors to a stream - /// - /// The file format is the same as the one used by the PyTorch C++ API. - pub fn save_multi_to_stream, T: AsRef, W: Write>( - named_tensors: &[(S, T)], - stream: W, - ) -> Result<(), TchError> { - let boxed_stream: Box> = Box::new(Box::new(stream)); - let c_tensors = named_tensors.iter().map(|nt| nt.1.as_ref().c_tensor).collect::>(); - let names = named_tensors - .iter() - .map(|nt| nt.0.as_ref().replace('.', "|").into_bytes()) - .map(std::ffi::CString::new) - .collect::, _>>()?; - let name_ptrs = names.iter().map(|n| n.as_ptr()).collect::>(); - unsafe_torch_err!(at_save_multi_to_stream( - c_tensors.as_ptr(), - name_ptrs.as_ptr(), - names.len() as i32, - Box::into_raw(boxed_stream) as *mut c_void, - )); - Ok(()) - } - - /// Loads some named tensors from a file - /// - /// The file format is the same as the one used for modules in the PyTorch C++ API. - /// It commonly uses the .ot extension. - pub fn load_multi>(path: T) -> Result, TchError> { - let path = path_to_cstring(path)?; - let mut v: Vec<(String, Tensor)> = vec![]; - unsafe_torch_err!(at_load_callback( - path.as_ptr(), - &mut v as *mut _ as *mut c_void, - add_callback - )); - Ok(v) - } - - /// Loads some named tensors from a file to a given device - /// - /// The file format is the same as the one used for modules in the PyTorch C++ API. - /// It commonly uses the .ot extension. - pub fn load_multi_with_device>( - path: T, - device: Device, - ) -> Result, TchError> { - let path = path_to_cstring(path)?; - let mut v: Vec<(String, Tensor)> = vec![]; - unsafe_torch_err!(at_load_callback_with_device( - path.as_ptr(), - &mut v as *mut _ as *mut c_void, - add_callback, - device.c_int(), - )); - Ok(v) - } - - /// Loads some named tensors from a zip file - /// - /// The expected file format is a zip archive containing a data.pkl file describing - /// the embedded tensors. These are commonly used with the .bin extension to export - /// PyTorch models and weights using the Python api. - pub fn loadz_multi>(path: T) -> Result, TchError> { - let path = path_to_cstring(path)?; - let mut v: Vec<(String, Tensor)> = vec![]; - unsafe_torch_err!(at_loadz_callback( - path.as_ptr(), - &mut v as *mut _ as *mut c_void, - add_callback - )); - Ok(v) - } - - /// Loads some named tensors from a zip file to a given device - /// - /// The expected file format is a zip archive containing a data.pkl file describing - /// the embedded tensors. These are commonly used with the .bin extension to export - /// PyTorch models and weights using the Python api. - pub fn loadz_multi_with_device>( - path: T, - device: Device, - ) -> Result, TchError> { - let path = path_to_cstring(path)?; - let mut v: Vec<(String, Tensor)> = vec![]; - unsafe_torch_err!(at_loadz_callback_with_device( - path.as_ptr(), - &mut v as *mut _ as *mut c_void, - add_callback, - device.c_int(), - )); - Ok(v) - } - - /// Loads some named tensors from a stream - /// - /// The file format is the same as the one used by the PyTorch C++ API. - pub fn load_multi_from_stream( - stream: T, - ) -> Result, TchError> { - let adapter = ReadSeekAdapter::new(stream); - let boxed_stream: Box> = Box::new(Box::new(adapter)); - let mut v: Vec<(String, Tensor)> = vec![]; - unsafe_torch_err!(at_load_from_stream_callback( - Box::into_raw(boxed_stream) as *mut c_void, - &mut v as *mut _ as *mut c_void, - add_callback, - false, - 0, - )); - Ok(v) - } - - /// Loads some named tensors from a stream to a given device - /// - /// The file format is the same as the one used by the PyTorch C++ API. - pub fn load_multi_from_stream_with_device( - stream: T, - device: Device, - ) -> Result, TchError> { - let adapter = ReadSeekAdapter::new(stream); - let boxed_stream: Box> = Box::new(Box::new(adapter)); - let mut v: Vec<(String, Tensor)> = vec![]; - unsafe_torch_err!(at_load_from_stream_callback( - Box::into_raw(boxed_stream) as *mut c_void, - &mut v as *mut _ as *mut c_void, - add_callback, - true, - device.c_int(), - )); - Ok(v) - } - - /// Returns a string representation for the tensor. - /// - /// The representation will contain all the tensor element hence may be huge for - /// large tensors. - pub fn to_string(&self, lw: i64) -> Result { - let s = - unsafe_torch_err!(ptr_to_string(torch_sys::at_to_string(self.c_tensor, lw as c_int))); - match s { - None => Err(TchError::Kind("nullptr representation".to_string())), - Some(s) => Ok(s), - } - } -} - -impl Default for Tensor { - fn default() -> Self { - Self::new() - } -} - -impl Drop for Tensor { - fn drop(&mut self) { - unsafe_torch!(at_free(self.c_tensor)) - } -} - -fn autocast_clear_cache() { - unsafe_torch!(at_autocast_clear_cache()) -} - -fn autocast_decrement_nesting() -> isize { - unsafe_torch!(at_autocast_decrement_nesting() as isize) -} - -fn autocast_increment_nesting() -> isize { - unsafe_torch!(at_autocast_increment_nesting() as isize) -} - -fn autocast_is_enabled() -> bool { - unsafe_torch!(at_autocast_is_enabled() != 0) -} - -fn autocast_set_enabled(b: bool) -> bool { - unsafe_torch!(at_autocast_set_enabled(i32::from(b)) != 0) -} - -/// Runs a closure in mixed precision. -pub fn autocast(enabled: bool, f: F) -> T -where - F: FnOnce() -> T, -{ - if !Cuda::is_available() { - return f(); - } - - // Check whether we are using CUDA. - let prev = autocast_is_enabled(); - autocast_set_enabled(enabled); - autocast_increment_nesting(); - - let result = f(); - - if autocast_decrement_nesting() == 0 { - autocast_clear_cache(); - } - autocast_set_enabled(prev); - - result -} - -fn grad_set_enabled(b: bool) -> bool { - unsafe_torch!(at_grad_set_enabled(i32::from(b)) != 0) -} - -/// Runs a closure without keeping track of gradients. -pub fn no_grad(f: F) -> T -where - F: FnOnce() -> T, -{ - let prev = grad_set_enabled(false); - let result = f(); - let _false = grad_set_enabled(prev); - result -} - -/// Runs a closure explicitly keeping track of gradients, this could be -/// run within a no_grad closure for example. -pub fn with_grad(f: F) -> T -where - F: FnOnce() -> T, -{ - let prev = grad_set_enabled(true); - let result = f(); - let _false = grad_set_enabled(prev); - result -} - -/// A RAII guard that prevents gradient tracking until deallocated. -pub struct NoGradGuard { - enabled: bool, -} - -/// Disables gradient tracking, this will be enabled back when the -/// returned value gets deallocated. -/// Note that it is important to bind this to a name like `_guard` -/// and not to `_` as the latter would immediately drop the guard. -/// See -/// for more details. -pub fn no_grad_guard() -> NoGradGuard { - NoGradGuard { enabled: grad_set_enabled(false) } -} - -impl std::convert::AsRef for Tensor { - fn as_ref(&self) -> &Self { - self - } -} - -impl Drop for NoGradGuard { - fn drop(&mut self) { - let _enabled = grad_set_enabled(self.enabled); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub enum Reduction { - /// Do not reduce. - None, - /// Mean of losses. - Mean, - /// Sum of losses. - Sum, - /// Escape hatch in case new options become available. - Other(i64), -} - -impl Reduction { - // This has to stay in sync with - // pytorch/aten/src/ATen/core/Reduction.h - pub fn to_int(self) -> i64 { - match self { - Reduction::None => 0, - Reduction::Mean => 1, - Reduction::Sum => 2, - Reduction::Other(i) => i, - } - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/tensor_fallible_generated.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/tensor_fallible_generated.rs deleted file mode 100644 index 2aff06e73e..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/wrappers/tensor_fallible_generated.rs +++ /dev/null @@ -1,38995 +0,0 @@ -/* THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT BY HAND! */ -#![allow(clippy::all)] -use crate::{Device, Kind, Layout, Scalar, TchError, Tensor}; -use std::borrow::Borrow; -use std::convert::Into; -use torch_sys::c_generated::*; -use torch_sys::*; - -fn ptr_list_opt>(l: &[Option]) -> Vec<*mut C_tensor> { - l.iter().map(|x| x.as_ref().map_or(std::ptr::null_mut(), |x| x.borrow().c_tensor)).collect() -} - -fn ptr_list>(l: &[T]) -> Vec<*mut C_tensor> { - l.iter().map(|x| x.borrow().c_tensor).collect() -} - -impl Tensor { - pub fn f_internal_and_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___and__( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_and_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___and__tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_iand_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___iand__( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_iand_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___iand__tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_ilshift_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___ilshift__( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_ilshift_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___ilshift__tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_ior_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___ior__( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_ior_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___ior__tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_irshift_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___irshift__( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_irshift_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___irshift__tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_ixor_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___ixor__( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_ixor_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___ixor__tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_lshift_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___lshift__( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_lshift_scalar_out_>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___lshift__scalar_out_( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_lshift_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___lshift__tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_lshift_tensor_out_( - &self, - out: &Tensor, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___lshift__tensor_out_( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_or_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___or__(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_or_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___or__tensor_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_rshift_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___rshift__( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_rshift_scalar_out_>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___rshift__scalar_out_( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_rshift_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___rshift__tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_rshift_tensor_out_( - &self, - out: &Tensor, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___rshift__tensor_out_( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_xor_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___xor__( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_xor_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg___xor__tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_adaptive_avg_pool2d( - &self, - output_size: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__adaptive_avg_pool2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_adaptive_avg_pool2d_backward( - &self, - grad_output: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__adaptive_avg_pool2d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_adaptive_avg_pool2d_backward_out( - &self, - out: &Tensor, - grad_output: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__adaptive_avg_pool2d_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad_output.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_adaptive_avg_pool2d_out( - &self, - out: &Tensor, - output_size: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__adaptive_avg_pool2d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_adaptive_avg_pool3d( - &self, - output_size: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__adaptive_avg_pool3d( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_adaptive_avg_pool3d_backward( - &self, - grad_output: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__adaptive_avg_pool3d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_adaptive_avg_pool3d_backward_out( - &self, - out: &Tensor, - grad_output: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__adaptive_avg_pool3d_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad_output.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_adaptive_avg_pool3d_out( - &self, - out: &Tensor, - output_size: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__adaptive_avg_pool3d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_add_batch_dim(&self, batch_dim: i64, level: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__add_batch_dim( - c_tensors.as_mut_ptr(), - self.c_tensor, - batch_dim, - level - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_add_relu(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__add_relu(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_add_relu_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__add_relu_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_add_relu_out( - &self, - out: &Tensor, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__add_relu_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_add_relu_scalar>( - &self, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__add_relu_scalar( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_add_relu_scalar_>( - &mut self, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__add_relu_scalar_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_add_relu_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__add_relu_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_addmm_activation( - &self, - mat1: &Tensor, - mat2: &Tensor, - use_gelu: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__addmm_activation( - c_tensors.as_mut_ptr(), - self.c_tensor, - mat1.c_tensor, - mat2.c_tensor, - if use_gelu { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_addmm_activation_out( - &self, - out: &Tensor, - mat1: &Tensor, - mat2: &Tensor, - use_gelu: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__addmm_activation_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mat1.c_tensor, - mat2.c_tensor, - if use_gelu { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_aminmax(&self) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__aminmax(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_aminmax_dim( - &self, - dim: i64, - keepdim: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__aminmax_dim( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - if keepdim { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_aminmax_dim_out( - &self, - out0: &Tensor, - out1: &Tensor, - dim: i64, - keepdim: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__aminmax_dim_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - self.c_tensor, - dim, - if keepdim { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_aminmax_out( - &self, - out0: &Tensor, - out1: &Tensor, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__aminmax_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - self.c_tensor - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_amp_update_scale( - &self, - growth_tracker: &Tensor, - found_inf: &Tensor, - scale_growth_factor: f64, - scale_backoff_factor: f64, - growth_interval: i64, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__amp_update_scale( - c_tensors.as_mut_ptr(), - self.c_tensor, - growth_tracker.c_tensor, - found_inf.c_tensor, - scale_growth_factor, - scale_backoff_factor, - growth_interval - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_amp_update_scale_( - &mut self, - growth_tracker: &Tensor, - found_inf: &Tensor, - scale_growth_factor: f64, - scale_backoff_factor: f64, - growth_interval: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__amp_update_scale_( - c_tensors.as_mut_ptr(), - self.c_tensor, - growth_tracker.c_tensor, - found_inf.c_tensor, - scale_growth_factor, - scale_backoff_factor, - growth_interval - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_amp_update_scale_out( - &self, - out: &Tensor, - growth_tracker: &Tensor, - found_inf: &Tensor, - scale_growth_factor: f64, - scale_backoff_factor: f64, - growth_interval: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__amp_update_scale_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - growth_tracker.c_tensor, - found_inf.c_tensor, - scale_growth_factor, - scale_backoff_factor, - growth_interval - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_assert_scalar>( - self_scalar: S, - assert_msg: &str, - ) -> Result<(), TchError> { - unsafe_torch_err!(atg__assert_scalar( - self_scalar.into().c_scalar, - assert_msg.as_ptr(), - assert_msg.len() as i32 - )); - Ok(()) - } - - pub fn f_internal_assert_tensor_metadata( - a: &Tensor, - size: impl IntListOption, - stride: impl IntListOption, - dtype: impl Into>, - device: Device, - layout: Option, - ) -> Result<(), TchError> { - unsafe_torch_err!(atg__assert_tensor_metadata( - a.c_tensor, - size.as_ptr(), - size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - dtype.into().map_or(-1, |s| s.c_int()), - device.c_int(), - layout.map_or(-1, |s| s.to_i8()) - )); - Ok(()) - } - - pub fn f_internal_autocast_to_full_precision( - &self, - cuda_enabled: bool, - cpu_enabled: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__autocast_to_full_precision( - c_tensors.as_mut_ptr(), - self.c_tensor, - if cuda_enabled { 1 } else { 0 }, - if cpu_enabled { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_autocast_to_reduced_precision( - &self, - cuda_enabled: bool, - cpu_enabled: bool, - cuda_dtype: Kind, - cpu_dtype: Kind, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__autocast_to_reduced_precision( - c_tensors.as_mut_ptr(), - self.c_tensor, - if cuda_enabled { 1 } else { 0 }, - if cpu_enabled { 1 } else { 0 }, - cuda_dtype.c_int(), - cpu_dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_batch_norm_no_update>( - &self, - weight: Option, - bias: Option, - running_mean: Option, - running_var: Option, - momentum: f64, - eps: f64, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg__batch_norm_no_update( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - momentum, - eps - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_internal_batch_norm_no_update_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - out3: &Tensor, - weight: Option, - bias: Option, - running_mean: Option, - running_var: Option, - momentum: f64, - eps: f64, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg__batch_norm_no_update_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - out2.c_tensor, - out3.c_tensor, - self.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - momentum, - eps - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_internal_batch_norm_with_update>( - &self, - weight: Option, - bias: Option, - running_mean: &Tensor, - running_var: &Tensor, - momentum: f64, - eps: f64, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg__batch_norm_with_update( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_mean.c_tensor, - running_var.c_tensor, - momentum, - eps - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_internal_batch_norm_with_update_functional>( - &self, - weight: Option, - bias: Option, - running_mean: &Tensor, - running_var: &Tensor, - momentum: f64, - eps: f64, - ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 6]; - unsafe_torch_err!(atg__batch_norm_with_update_functional( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_mean.c_tensor, - running_var.c_tensor, - momentum, - eps - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - Tensor { c_tensor: c_tensors[4] }, - Tensor { c_tensor: c_tensors[5] }, - )) - } - - pub fn f_internal_batch_norm_with_update_out>( - &self, - out: &Tensor, - save_mean: &Tensor, - save_invstd: &Tensor, - reserve: &Tensor, - weight: Option, - bias: Option, - running_mean: &Tensor, - running_var: &Tensor, - momentum: f64, - eps: f64, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg__batch_norm_with_update_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - save_mean.c_tensor, - save_invstd.c_tensor, - reserve.c_tensor, - self.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_mean.c_tensor, - running_var.c_tensor, - momentum, - eps - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_internal_cast_byte(&self, non_blocking: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__cast_byte( - c_tensors.as_mut_ptr(), - self.c_tensor, - if non_blocking { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_cast_char(&self, non_blocking: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__cast_char( - c_tensors.as_mut_ptr(), - self.c_tensor, - if non_blocking { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_cast_double(&self, non_blocking: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__cast_double( - c_tensors.as_mut_ptr(), - self.c_tensor, - if non_blocking { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_cast_float(&self, non_blocking: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__cast_float( - c_tensors.as_mut_ptr(), - self.c_tensor, - if non_blocking { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_cast_half(&self, non_blocking: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__cast_half( - c_tensors.as_mut_ptr(), - self.c_tensor, - if non_blocking { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_cast_int(&self, non_blocking: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__cast_int( - c_tensors.as_mut_ptr(), - self.c_tensor, - if non_blocking { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_cast_long(&self, non_blocking: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__cast_long( - c_tensors.as_mut_ptr(), - self.c_tensor, - if non_blocking { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_cast_short(&self, non_blocking: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__cast_short( - c_tensors.as_mut_ptr(), - self.c_tensor, - if non_blocking { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_cdist_backward( - grad: &Tensor, - x1: &Tensor, - x2: &Tensor, - p: f64, - cdist: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__cdist_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - x1.c_tensor, - x2.c_tensor, - p, - cdist.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_cdist_backward_out( - out: &Tensor, - grad: &Tensor, - x1: &Tensor, - x2: &Tensor, - p: f64, - cdist: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__cdist_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad.c_tensor, - x1.c_tensor, - x2.c_tensor, - p, - cdist.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_cholesky_solve_helper( - &self, - a: &Tensor, - upper: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__cholesky_solve_helper( - c_tensors.as_mut_ptr(), - self.c_tensor, - a.c_tensor, - if upper { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_cholesky_solve_helper_out( - &self, - out: &Tensor, - a: &Tensor, - upper: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__cholesky_solve_helper_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - a.c_tensor, - if upper { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_chunk_cat>( - tensors: &[T], - dim: i64, - num_chunks: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__chunk_cat( - c_tensors.as_mut_ptr(), - ptr_list(tensors).as_ptr(), - tensors.len() as i32, - dim, - num_chunks - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_chunk_cat_out>( - out: &Tensor, - tensors: &[T], - dim: i64, - num_chunks: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__chunk_cat_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - ptr_list(tensors).as_ptr(), - tensors.len() as i32, - dim, - num_chunks - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_coalesce(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__coalesce(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_coalesce_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__coalesce_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_coalesced(&self, coalesced: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__coalesced( - c_tensors.as_mut_ptr(), - self.c_tensor, - if coalesced { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_coalesced_(&mut self, coalesced: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__coalesced_( - c_tensors.as_mut_ptr(), - self.c_tensor, - if coalesced { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_coalesced_out( - &self, - out: &Tensor, - coalesced: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__coalesced_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - if coalesced { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_compute_linear_combination( - &self, - coefficients: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__compute_linear_combination( - c_tensors.as_mut_ptr(), - self.c_tensor, - coefficients.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_compute_linear_combination_out( - &self, - out: &Tensor, - coefficients: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__compute_linear_combination_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - coefficients.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_conj(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__conj(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_conj_copy(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__conj_copy(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_conj_copy_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__conj_copy_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_conj_physical(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__conj_physical(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_conj_physical_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__conj_physical_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_conv_depthwise2d>( - &self, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__conv_depthwise2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_conv_depthwise2d_out>( - &self, - out: &Tensor, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__conv_depthwise2d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_convert_indices_from_coo_to_csr( - &self, - size: i64, - out_int32: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__convert_indices_from_coo_to_csr( - c_tensors.as_mut_ptr(), - self.c_tensor, - size, - if out_int32 { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_convert_indices_from_coo_to_csr_out( - &self, - out: &Tensor, - size: i64, - out_int32: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__convert_indices_from_coo_to_csr_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - size, - if out_int32 { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_convert_indices_from_csr_to_coo( - crow_indices: &Tensor, - col_indices: &Tensor, - out_int32: bool, - transpose: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__convert_indices_from_csr_to_coo( - c_tensors.as_mut_ptr(), - crow_indices.c_tensor, - col_indices.c_tensor, - if out_int32 { 1 } else { 0 }, - if transpose { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_convert_indices_from_csr_to_coo_out( - out: &Tensor, - crow_indices: &Tensor, - col_indices: &Tensor, - out_int32: bool, - transpose: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__convert_indices_from_csr_to_coo_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - crow_indices.c_tensor, - col_indices.c_tensor, - if out_int32 { 1 } else { 0 }, - if transpose { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_convert_weight_to_int4pack( - &self, - innerktiles: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__convert_weight_to_int4pack( - c_tensors.as_mut_ptr(), - self.c_tensor, - innerktiles - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_convert_weight_to_int4pack_for_cpu( - &self, - innerktiles: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__convert_weight_to_int4pack_for_cpu( - c_tensors.as_mut_ptr(), - self.c_tensor, - innerktiles - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_convolution>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - transposed: bool, - output_padding: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - cudnn_enabled: bool, - allow_tf32: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__convolution( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if transposed { 1 } else { 0 }, - output_padding.as_ptr(), - output_padding.len_i32(), - groups, - if benchmark { 1 } else { 0 }, - if deterministic { 1 } else { 0 }, - if cudnn_enabled { 1 } else { 0 }, - if allow_tf32 { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_convolution_deprecated>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - transposed: bool, - output_padding: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - cudnn_enabled: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__convolution_deprecated( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if transposed { 1 } else { 0 }, - output_padding.as_ptr(), - output_padding.len_i32(), - groups, - if benchmark { 1 } else { 0 }, - if deterministic { 1 } else { 0 }, - if cudnn_enabled { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_convolution_mode>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: &str, - dilation: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__convolution_mode( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len() as i32, - dilation.as_ptr(), - dilation.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_convolution_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - transposed: bool, - output_padding: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - cudnn_enabled: bool, - allow_tf32: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__convolution_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if transposed { 1 } else { 0 }, - output_padding.as_ptr(), - output_padding.len_i32(), - groups, - if benchmark { 1 } else { 0 }, - if deterministic { 1 } else { 0 }, - if cudnn_enabled { 1 } else { 0 }, - if allow_tf32 { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_copy_from( - &self, - dst: &Tensor, - non_blocking: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__copy_from( - c_tensors.as_mut_ptr(), - self.c_tensor, - dst.c_tensor, - if non_blocking { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_copy_from_and_resize(&self, dst: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__copy_from_and_resize( - c_tensors.as_mut_ptr(), - self.c_tensor, - dst.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_copy_from_and_resize_out( - &self, - out: &Tensor, - dst: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__copy_from_and_resize_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dst.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_copy_from_out( - &self, - out: &Tensor, - dst: &Tensor, - non_blocking: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__copy_from_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dst.c_tensor, - if non_blocking { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_cslt_compress(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__cslt_compress(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_cslt_sparse_mm>( - compressed_a: &Tensor, - dense_b: &Tensor, - bias: Option, - alpha: Option, - out_dtype: impl Into>, - transpose_result: bool, - alg_id: i64, - split_k: i64, - split_k_mode: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__cslt_sparse_mm( - c_tensors.as_mut_ptr(), - compressed_a.c_tensor, - dense_b.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - alpha.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - out_dtype.into().map_or(-1, |s| s.c_int()), - if transpose_result { 1 } else { 0 }, - alg_id, - split_k, - split_k_mode - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_cslt_sparse_mm_search>( - compressed_a: &Tensor, - dense_b: &Tensor, - bias: Option, - alpha: Option, - out_dtype: impl Into>, - transpose_result: bool, - ) -> Result { - let return_; - unsafe_torch_err!( - return_ = atg__cslt_sparse_mm_search( - compressed_a.c_tensor, - dense_b.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - alpha.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - out_dtype.into().map_or(-1, |s| s.c_int()), - if transpose_result { 1 } else { 0 } - ) - ); - Ok(return_) - } - - pub fn f_internal_ctc_loss( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: impl IntList, - target_lengths: impl IntList, - blank: i64, - zero_infinity: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__ctc_loss( - c_tensors.as_mut_ptr(), - log_probs.c_tensor, - targets.c_tensor, - input_lengths.as_ptr(), - input_lengths.len_i32(), - target_lengths.as_ptr(), - target_lengths.len_i32(), - blank, - if zero_infinity { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_ctc_loss_backward( - grad: &Tensor, - log_probs: &Tensor, - targets: &Tensor, - input_lengths: impl IntList, - target_lengths: impl IntList, - neg_log_likelihood: &Tensor, - log_alpha: &Tensor, - blank: i64, - zero_infinity: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__ctc_loss_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - log_probs.c_tensor, - targets.c_tensor, - input_lengths.as_ptr(), - input_lengths.len_i32(), - target_lengths.as_ptr(), - target_lengths.len_i32(), - neg_log_likelihood.c_tensor, - log_alpha.c_tensor, - blank, - if zero_infinity { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_ctc_loss_backward_out( - out: &Tensor, - grad: &Tensor, - log_probs: &Tensor, - targets: &Tensor, - input_lengths: impl IntList, - target_lengths: impl IntList, - neg_log_likelihood: &Tensor, - log_alpha: &Tensor, - blank: i64, - zero_infinity: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__ctc_loss_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad.c_tensor, - log_probs.c_tensor, - targets.c_tensor, - input_lengths.as_ptr(), - input_lengths.len_i32(), - target_lengths.as_ptr(), - target_lengths.len_i32(), - neg_log_likelihood.c_tensor, - log_alpha.c_tensor, - blank, - if zero_infinity { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_ctc_loss_backward_tensor( - grad: &Tensor, - log_probs: &Tensor, - targets: &Tensor, - input_lengths: &Tensor, - target_lengths: &Tensor, - neg_log_likelihood: &Tensor, - log_alpha: &Tensor, - blank: i64, - zero_infinity: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__ctc_loss_backward_tensor( - c_tensors.as_mut_ptr(), - grad.c_tensor, - log_probs.c_tensor, - targets.c_tensor, - input_lengths.c_tensor, - target_lengths.c_tensor, - neg_log_likelihood.c_tensor, - log_alpha.c_tensor, - blank, - if zero_infinity { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_ctc_loss_out( - out0: &Tensor, - out1: &Tensor, - log_probs: &Tensor, - targets: &Tensor, - input_lengths: impl IntList, - target_lengths: impl IntList, - blank: i64, - zero_infinity: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__ctc_loss_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - log_probs.c_tensor, - targets.c_tensor, - input_lengths.as_ptr(), - input_lengths.len_i32(), - target_lengths.as_ptr(), - target_lengths.len_i32(), - blank, - if zero_infinity { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_ctc_loss_tensor( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: &Tensor, - target_lengths: &Tensor, - blank: i64, - zero_infinity: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__ctc_loss_tensor( - c_tensors.as_mut_ptr(), - log_probs.c_tensor, - targets.c_tensor, - input_lengths.c_tensor, - target_lengths.c_tensor, - blank, - if zero_infinity { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_ctc_loss_tensor_out( - out0: &Tensor, - out1: &Tensor, - log_probs: &Tensor, - targets: &Tensor, - input_lengths: &Tensor, - target_lengths: &Tensor, - blank: i64, - zero_infinity: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__ctc_loss_tensor_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - log_probs.c_tensor, - targets.c_tensor, - input_lengths.c_tensor, - target_lengths.c_tensor, - blank, - if zero_infinity { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_cudnn_attention_backward( - grad_out: &Tensor, - query: &Tensor, - key: &Tensor, - value: &Tensor, - out: &Tensor, - logsumexp: &Tensor, - philox_seed: &Tensor, - philox_offset: &Tensor, - attn_bias: &Tensor, - cum_seq_q: &Tensor, - cum_seq_k: &Tensor, - max_q: i64, - max_k: i64, - dropout_p: f64, - is_causal: bool, - scale: impl Into>, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let scale = scale.into(); - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__cudnn_attention_backward( - c_tensors.as_mut_ptr(), - grad_out.c_tensor, - query.c_tensor, - key.c_tensor, - value.c_tensor, - out.c_tensor, - logsumexp.c_tensor, - philox_seed.c_tensor, - philox_offset.c_tensor, - attn_bias.c_tensor, - cum_seq_q.c_tensor, - cum_seq_k.c_tensor, - max_q, - max_k, - dropout_p, - if is_causal { 1 } else { 0 }, - scale.unwrap_or(std::f64::NAN), - scale.is_none() as i8 - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_cudnn_ctc_loss( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: impl IntList, - target_lengths: impl IntList, - blank: i64, - deterministic: bool, - zero_infinity: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__cudnn_ctc_loss( - c_tensors.as_mut_ptr(), - log_probs.c_tensor, - targets.c_tensor, - input_lengths.as_ptr(), - input_lengths.len_i32(), - target_lengths.as_ptr(), - target_lengths.len_i32(), - blank, - if deterministic { 1 } else { 0 }, - if zero_infinity { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_cudnn_ctc_loss_out( - out0: &Tensor, - out1: &Tensor, - log_probs: &Tensor, - targets: &Tensor, - input_lengths: impl IntList, - target_lengths: impl IntList, - blank: i64, - deterministic: bool, - zero_infinity: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__cudnn_ctc_loss_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - log_probs.c_tensor, - targets.c_tensor, - input_lengths.as_ptr(), - input_lengths.len_i32(), - target_lengths.as_ptr(), - target_lengths.len_i32(), - blank, - if deterministic { 1 } else { 0 }, - if zero_infinity { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_cudnn_ctc_loss_tensor( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: &Tensor, - target_lengths: &Tensor, - blank: i64, - deterministic: bool, - zero_infinity: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__cudnn_ctc_loss_tensor( - c_tensors.as_mut_ptr(), - log_probs.c_tensor, - targets.c_tensor, - input_lengths.c_tensor, - target_lengths.c_tensor, - blank, - if deterministic { 1 } else { 0 }, - if zero_infinity { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_cudnn_init_dropout_state( - dropout: f64, - train: bool, - dropout_seed: i64, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__cudnn_init_dropout_state( - c_tensors.as_mut_ptr(), - dropout, - if train { 1 } else { 0 }, - dropout_seed, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_cudnn_init_dropout_state_out( - out: &Tensor, - dropout: f64, - train: bool, - dropout_seed: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__cudnn_init_dropout_state_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - dropout, - if train { 1 } else { 0 }, - dropout_seed - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_cudnn_rnn>( - &self, - weight: &[T], - weight_stride0: i64, - weight_buf: Option, - hx: &Tensor, - cx: Option, - mode: i64, - hidden_size: i64, - proj_size: i64, - num_layers: i64, - batch_first: bool, - dropout: f64, - train: bool, - bidirectional: bool, - batch_sizes: impl IntList, - dropout_state: Option, - ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 5]; - unsafe_torch_err!(atg__cudnn_rnn( - c_tensors.as_mut_ptr(), - self.c_tensor, - ptr_list(weight).as_ptr(), - weight.len() as i32, - weight_stride0, - weight_buf.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - hx.c_tensor, - cx.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - mode, - hidden_size, - proj_size, - num_layers, - if batch_first { 1 } else { 0 }, - dropout, - if train { 1 } else { 0 }, - if bidirectional { 1 } else { 0 }, - batch_sizes.as_ptr(), - batch_sizes.len_i32(), - dropout_state.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - Tensor { c_tensor: c_tensors[4] }, - )) - } - - pub fn f_internal_cudnn_rnn_flatten_weight>( - weight_arr: &[T], - weight_stride0: i64, - input_size: i64, - mode: i64, - hidden_size: i64, - proj_size: i64, - num_layers: i64, - batch_first: bool, - bidirectional: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__cudnn_rnn_flatten_weight( - c_tensors.as_mut_ptr(), - ptr_list(weight_arr).as_ptr(), - weight_arr.len() as i32, - weight_stride0, - input_size, - mode, - hidden_size, - proj_size, - num_layers, - if batch_first { 1 } else { 0 }, - if bidirectional { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_cudnn_rnn_flatten_weight_out>( - out: &Tensor, - weight_arr: &[T], - weight_stride0: i64, - input_size: i64, - mode: i64, - hidden_size: i64, - proj_size: i64, - num_layers: i64, - batch_first: bool, - bidirectional: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__cudnn_rnn_flatten_weight_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - ptr_list(weight_arr).as_ptr(), - weight_arr.len() as i32, - weight_stride0, - input_size, - mode, - hidden_size, - proj_size, - num_layers, - if batch_first { 1 } else { 0 }, - if bidirectional { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_cudnn_rnn_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - out3: &Tensor, - out4: &Tensor, - weight: &[T], - weight_stride0: i64, - weight_buf: Option, - hx: &Tensor, - cx: Option, - mode: i64, - hidden_size: i64, - proj_size: i64, - num_layers: i64, - batch_first: bool, - dropout: f64, - train: bool, - bidirectional: bool, - batch_sizes: impl IntList, - dropout_state: Option, - ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 5]; - unsafe_torch_err!(atg__cudnn_rnn_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - out2.c_tensor, - out3.c_tensor, - out4.c_tensor, - self.c_tensor, - ptr_list(weight).as_ptr(), - weight.len() as i32, - weight_stride0, - weight_buf.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - hx.c_tensor, - cx.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - mode, - hidden_size, - proj_size, - num_layers, - if batch_first { 1 } else { 0 }, - dropout, - if train { 1 } else { 0 }, - if bidirectional { 1 } else { 0 }, - batch_sizes.as_ptr(), - batch_sizes.len_i32(), - dropout_state.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - Tensor { c_tensor: c_tensors[4] }, - )) - } - - pub fn f_internal_debug_has_internal_overlap(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg__debug_has_internal_overlap(self.c_tensor)); - Ok(return_) - } - - pub fn f_internal_dim_arange(like: &Tensor, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__dim_arange(c_tensors.as_mut_ptr(), like.c_tensor, dim)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_dimi(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg__dimi(self.c_tensor)); - Ok(return_) - } - - pub fn f_internal_dimv(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg__dimv(self.c_tensor)); - Ok(return_) - } - - pub fn f_internal_dirichlet_grad( - x: &Tensor, - alpha: &Tensor, - total: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__dirichlet_grad( - c_tensors.as_mut_ptr(), - x.c_tensor, - alpha.c_tensor, - total.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_dirichlet_grad_out( - out: &Tensor, - x: &Tensor, - alpha: &Tensor, - total: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__dirichlet_grad_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - alpha.c_tensor, - total.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_dyn_quant_matmul_4bit( - inp: &Tensor, - packed_weights: &Tensor, - block_size: i64, - in_features: i64, - out_features: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__dyn_quant_matmul_4bit( - c_tensors.as_mut_ptr(), - inp.c_tensor, - packed_weights.c_tensor, - block_size, - in_features, - out_features - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_dyn_quant_pack_4bit_weight>( - weights: &Tensor, - scales_zeros: &Tensor, - bias: Option, - block_size: i64, - in_features: i64, - out_features: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__dyn_quant_pack_4bit_weight( - c_tensors.as_mut_ptr(), - weights.c_tensor, - scales_zeros.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - block_size, - in_features, - out_features - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_efficient_attention_backward>( - grad_out_: &Tensor, - query: &Tensor, - key: &Tensor, - value: &Tensor, - bias: Option, - out: &Tensor, - cu_seqlens_q: Option, - cu_seqlens_k: Option, - max_seqlen_q: i64, - max_seqlen_k: i64, - logsumexp: &Tensor, - dropout_p: f64, - philox_seed: &Tensor, - philox_offset: &Tensor, - custom_mask_type: i64, - bias_requires_grad: bool, - scale: impl Into>, - num_splits_key: impl Into>, - window_size: impl Into>, - shared_storage_dqdkdv: bool, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let scale = scale.into(); - let num_splits_key = num_splits_key.into(); - let window_size = window_size.into(); - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg__efficient_attention_backward( - c_tensors.as_mut_ptr(), - grad_out_.c_tensor, - query.c_tensor, - key.c_tensor, - value.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - out.c_tensor, - cu_seqlens_q.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - cu_seqlens_k.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - max_seqlen_q, - max_seqlen_k, - logsumexp.c_tensor, - dropout_p, - philox_seed.c_tensor, - philox_offset.c_tensor, - custom_mask_type, - if bias_requires_grad { 1 } else { 0 }, - scale.unwrap_or(std::f64::NAN), - scale.is_none() as i8, - num_splits_key.unwrap_or(0i64), - num_splits_key.is_none() as i8, - window_size.unwrap_or(0i64), - window_size.is_none() as i8, - if shared_storage_dqdkdv { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_internal_efficientzerotensor( - size: impl IntList, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__efficientzerotensor( - c_tensors.as_mut_ptr(), - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_efficientzerotensor_out( - out: &Tensor, - size: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__efficientzerotensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_embedding_bag>( - weight: &Tensor, - indices: &Tensor, - offsets: &Tensor, - scale_grad_by_freq: bool, - mode: i64, - sparse: bool, - per_sample_weights: Option, - include_last_offset: bool, - padding_idx: i64, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg__embedding_bag( - c_tensors.as_mut_ptr(), - weight.c_tensor, - indices.c_tensor, - offsets.c_tensor, - if scale_grad_by_freq { 1 } else { 0 }, - mode, - if sparse { 1 } else { 0 }, - per_sample_weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if include_last_offset { 1 } else { 0 }, - padding_idx - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_internal_embedding_bag_backward>( - grad: &Tensor, - indices: &Tensor, - offsets: &Tensor, - offset2bag: &Tensor, - bag_size: &Tensor, - maximum_indices: &Tensor, - num_weights: i64, - scale_grad_by_freq: bool, - mode: i64, - sparse: bool, - per_sample_weights: Option, - padding_idx: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__embedding_bag_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - indices.c_tensor, - offsets.c_tensor, - offset2bag.c_tensor, - bag_size.c_tensor, - maximum_indices.c_tensor, - num_weights, - if scale_grad_by_freq { 1 } else { 0 }, - mode, - if sparse { 1 } else { 0 }, - per_sample_weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - padding_idx - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_embedding_bag_dense_backward>( - grad: &Tensor, - indices: &Tensor, - offset2bag: &Tensor, - bag_size: &Tensor, - maximum_indices: &Tensor, - num_weights: i64, - scale_grad_by_freq: bool, - mode: i64, - per_sample_weights: Option, - padding_idx: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__embedding_bag_dense_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - indices.c_tensor, - offset2bag.c_tensor, - bag_size.c_tensor, - maximum_indices.c_tensor, - num_weights, - if scale_grad_by_freq { 1 } else { 0 }, - mode, - per_sample_weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - padding_idx - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_embedding_bag_dense_backward_out>( - out: &Tensor, - grad: &Tensor, - indices: &Tensor, - offset2bag: &Tensor, - bag_size: &Tensor, - maximum_indices: &Tensor, - num_weights: i64, - scale_grad_by_freq: bool, - mode: i64, - per_sample_weights: Option, - padding_idx: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__embedding_bag_dense_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad.c_tensor, - indices.c_tensor, - offset2bag.c_tensor, - bag_size.c_tensor, - maximum_indices.c_tensor, - num_weights, - if scale_grad_by_freq { 1 } else { 0 }, - mode, - per_sample_weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - padding_idx - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_embedding_bag_forward_only>( - weight: &Tensor, - indices: &Tensor, - offsets: &Tensor, - scale_grad_by_freq: bool, - mode: i64, - sparse: bool, - per_sample_weights: Option, - include_last_offset: bool, - padding_idx: i64, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg__embedding_bag_forward_only( - c_tensors.as_mut_ptr(), - weight.c_tensor, - indices.c_tensor, - offsets.c_tensor, - if scale_grad_by_freq { 1 } else { 0 }, - mode, - if sparse { 1 } else { 0 }, - per_sample_weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if include_last_offset { 1 } else { 0 }, - padding_idx - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_internal_embedding_bag_forward_only_out>( - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - out3: &Tensor, - weight: &Tensor, - indices: &Tensor, - offsets: &Tensor, - scale_grad_by_freq: bool, - mode: i64, - sparse: bool, - per_sample_weights: Option, - include_last_offset: bool, - padding_idx: i64, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg__embedding_bag_forward_only_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - out2.c_tensor, - out3.c_tensor, - weight.c_tensor, - indices.c_tensor, - offsets.c_tensor, - if scale_grad_by_freq { 1 } else { 0 }, - mode, - if sparse { 1 } else { 0 }, - per_sample_weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if include_last_offset { 1 } else { 0 }, - padding_idx - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_internal_embedding_bag_out>( - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - out3: &Tensor, - weight: &Tensor, - indices: &Tensor, - offsets: &Tensor, - scale_grad_by_freq: bool, - mode: i64, - sparse: bool, - per_sample_weights: Option, - include_last_offset: bool, - padding_idx: i64, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg__embedding_bag_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - out2.c_tensor, - out3.c_tensor, - weight.c_tensor, - indices.c_tensor, - offsets.c_tensor, - if scale_grad_by_freq { 1 } else { 0 }, - mode, - if sparse { 1 } else { 0 }, - per_sample_weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if include_last_offset { 1 } else { 0 }, - padding_idx - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_internal_embedding_bag_per_sample_weights_backward( - grad: &Tensor, - weight: &Tensor, - indices: &Tensor, - offsets: &Tensor, - offset2bag: &Tensor, - mode: i64, - padding_idx: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__embedding_bag_per_sample_weights_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - weight.c_tensor, - indices.c_tensor, - offsets.c_tensor, - offset2bag.c_tensor, - mode, - padding_idx - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_embedding_bag_per_sample_weights_backward_out( - out: &Tensor, - grad: &Tensor, - weight: &Tensor, - indices: &Tensor, - offsets: &Tensor, - offset2bag: &Tensor, - mode: i64, - padding_idx: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__embedding_bag_per_sample_weights_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad.c_tensor, - weight.c_tensor, - indices.c_tensor, - offsets.c_tensor, - offset2bag.c_tensor, - mode, - padding_idx - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_embedding_bag_sparse_backward>( - grad: &Tensor, - indices: &Tensor, - offsets: &Tensor, - offset2bag: &Tensor, - bag_size: &Tensor, - num_weights: i64, - scale_grad_by_freq: bool, - mode: i64, - per_sample_weights: Option, - padding_idx: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__embedding_bag_sparse_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - indices.c_tensor, - offsets.c_tensor, - offset2bag.c_tensor, - bag_size.c_tensor, - num_weights, - if scale_grad_by_freq { 1 } else { 0 }, - mode, - per_sample_weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - padding_idx - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_empty_affine_quantized( - size: impl IntList, - options: (Kind, Device), - scale: f64, - zero_point: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__empty_affine_quantized( - c_tensors.as_mut_ptr(), - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int(), - scale, - zero_point - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_empty_affine_quantized_out( - out: &Tensor, - size: impl IntList, - scale: f64, - zero_point: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__empty_affine_quantized_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - size.as_ptr(), - size.len_i32(), - scale, - zero_point - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_empty_per_channel_affine_quantized( - size: impl IntList, - scales: &Tensor, - zero_points: &Tensor, - axis: i64, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__empty_per_channel_affine_quantized( - c_tensors.as_mut_ptr(), - size.as_ptr(), - size.len_i32(), - scales.c_tensor, - zero_points.c_tensor, - axis, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_empty_per_channel_affine_quantized_out( - out: &Tensor, - size: impl IntList, - scales: &Tensor, - zero_points: &Tensor, - axis: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__empty_per_channel_affine_quantized_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - size.as_ptr(), - size.len_i32(), - scales.c_tensor, - zero_points.c_tensor, - axis - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_euclidean_dist(x1: &Tensor, x2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__euclidean_dist(c_tensors.as_mut_ptr(), x1.c_tensor, x2.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_euclidean_dist_out( - out: &Tensor, - x1: &Tensor, - x2: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__euclidean_dist_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x1.c_tensor, - x2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_fake_quantize_learnable_per_channel_affine( - &self, - scale: &Tensor, - zero_point: &Tensor, - axis: i64, - quant_min: i64, - quant_max: i64, - grad_factor: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__fake_quantize_learnable_per_channel_affine( - c_tensors.as_mut_ptr(), - self.c_tensor, - scale.c_tensor, - zero_point.c_tensor, - axis, - quant_min, - quant_max, - grad_factor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_fake_quantize_learnable_per_channel_affine_backward( - &self, - grad: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - axis: i64, - quant_min: i64, - quant_max: i64, - grad_factor: f64, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__fake_quantize_learnable_per_channel_affine_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - self.c_tensor, - scale.c_tensor, - zero_point.c_tensor, - axis, - quant_min, - quant_max, - grad_factor - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_fake_quantize_learnable_per_channel_affine_out( - &self, - out: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - axis: i64, - quant_min: i64, - quant_max: i64, - grad_factor: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__fake_quantize_learnable_per_channel_affine_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - scale.c_tensor, - zero_point.c_tensor, - axis, - quant_min, - quant_max, - grad_factor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_fake_quantize_learnable_per_tensor_affine( - &self, - scale: &Tensor, - zero_point: &Tensor, - quant_min: i64, - quant_max: i64, - grad_factor: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__fake_quantize_learnable_per_tensor_affine( - c_tensors.as_mut_ptr(), - self.c_tensor, - scale.c_tensor, - zero_point.c_tensor, - quant_min, - quant_max, - grad_factor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_fake_quantize_learnable_per_tensor_affine_backward( - &self, - grad: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - quant_min: i64, - quant_max: i64, - grad_factor: f64, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__fake_quantize_learnable_per_tensor_affine_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - self.c_tensor, - scale.c_tensor, - zero_point.c_tensor, - quant_min, - quant_max, - grad_factor - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_fake_quantize_learnable_per_tensor_affine_out( - &self, - out: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - quant_min: i64, - quant_max: i64, - grad_factor: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__fake_quantize_learnable_per_tensor_affine_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - scale.c_tensor, - zero_point.c_tensor, - quant_min, - quant_max, - grad_factor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_fake_quantize_per_tensor_affine_cachemask_tensor_qparams( - &self, - scale: &Tensor, - zero_point: &Tensor, - fake_quant_enabled: &Tensor, - quant_min: i64, - quant_max: i64, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__fake_quantize_per_tensor_affine_cachemask_tensor_qparams( - c_tensors.as_mut_ptr(), - self.c_tensor, - scale.c_tensor, - zero_point.c_tensor, - fake_quant_enabled.c_tensor, - quant_min, - quant_max - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_fake_quantize_per_tensor_affine_cachemask_tensor_qparams_out( - &self, - out0: &Tensor, - out1: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - fake_quant_enabled: &Tensor, - quant_min: i64, - quant_max: i64, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__fake_quantize_per_tensor_affine_cachemask_tensor_qparams_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - self.c_tensor, - scale.c_tensor, - zero_point.c_tensor, - fake_quant_enabled.c_tensor, - quant_min, - quant_max - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_fft_c2c( - &self, - dim: impl IntList, - normalization: i64, - forward: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__fft_c2c( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - normalization, - if forward { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_fft_c2c_out( - &self, - out: &Tensor, - dim: impl IntList, - normalization: i64, - forward: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__fft_c2c_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - normalization, - if forward { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_fft_c2r( - &self, - dim: impl IntList, - normalization: i64, - last_dim_size: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__fft_c2r( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - normalization, - last_dim_size - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_fft_c2r_out( - &self, - out: &Tensor, - dim: impl IntList, - normalization: i64, - last_dim_size: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__fft_c2r_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - normalization, - last_dim_size - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_fft_r2c( - &self, - dim: impl IntList, - normalization: i64, - onesided: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__fft_r2c( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - normalization, - if onesided { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_fft_r2c_out( - &self, - out: &Tensor, - dim: impl IntList, - normalization: i64, - onesided: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__fft_r2c_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - normalization, - if onesided { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_fill_mem_eff_dropout_mask_( - &mut self, - dropout_p: f64, - seed: i64, - offset: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__fill_mem_eff_dropout_mask_( - c_tensors.as_mut_ptr(), - self.c_tensor, - dropout_p, - seed, - offset - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_flash_attention_backward( - grad_out: &Tensor, - query: &Tensor, - key: &Tensor, - value: &Tensor, - out: &Tensor, - logsumexp: &Tensor, - cum_seq_q: &Tensor, - cum_seq_k: &Tensor, - max_q: i64, - max_k: i64, - dropout_p: f64, - is_causal: bool, - rng_state: &Tensor, - unused: &Tensor, - scale: impl Into>, - window_size_left: impl Into>, - window_size_right: impl Into>, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let scale = scale.into(); - let window_size_left = window_size_left.into(); - let window_size_right = window_size_right.into(); - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__flash_attention_backward( - c_tensors.as_mut_ptr(), - grad_out.c_tensor, - query.c_tensor, - key.c_tensor, - value.c_tensor, - out.c_tensor, - logsumexp.c_tensor, - cum_seq_q.c_tensor, - cum_seq_k.c_tensor, - max_q, - max_k, - dropout_p, - if is_causal { 1 } else { 0 }, - rng_state.c_tensor, - unused.c_tensor, - scale.unwrap_or(std::f64::NAN), - scale.is_none() as i8, - window_size_left.unwrap_or(0i64), - window_size_left.is_none() as i8, - window_size_right.unwrap_or(0i64), - window_size_right.is_none() as i8 - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_foobar( - &self, - arg1: bool, - arg2: bool, - arg3: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__foobar( - c_tensors.as_mut_ptr(), - self.c_tensor, - if arg1 { 1 } else { 0 }, - if arg2 { 1 } else { 0 }, - if arg3 { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_foobar_out( - &self, - out: &Tensor, - arg1: bool, - arg2: bool, - arg3: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__foobar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - if arg1 { 1 } else { 0 }, - if arg2 { 1 } else { 0 }, - if arg3 { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_functional_assert_async( - &self, - assert_msg: &str, - dep_token: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__functional_assert_async( - c_tensors.as_mut_ptr(), - self.c_tensor, - assert_msg.as_ptr(), - assert_msg.len() as i32, - dep_token.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_functional_assert_scalar>( - self_scalar: S, - assert_msg: &str, - dep_token: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__functional_assert_scalar( - c_tensors.as_mut_ptr(), - self_scalar.into().c_scalar, - assert_msg.as_ptr(), - assert_msg.len() as i32, - dep_token.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_functional_sym_constrain_range>( - size: S, - min: impl Into>, - max: impl Into>, - dep_token: &Tensor, - ) -> Result { - let min = min.into(); - let max = max.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__functional_sym_constrain_range( - c_tensors.as_mut_ptr(), - size.into().c_scalar, - min.unwrap_or(0i64), - min.is_none() as i8, - max.unwrap_or(0i64), - max.is_none() as i8, - dep_token.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_functional_sym_constrain_range_for_size>( - size: S, - min: impl Into>, - max: impl Into>, - dep_token: &Tensor, - ) -> Result { - let min = min.into(); - let max = max.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__functional_sym_constrain_range_for_size( - c_tensors.as_mut_ptr(), - size.into().c_scalar, - min.unwrap_or(0i64), - min.is_none() as i8, - max.unwrap_or(0i64), - max.is_none() as i8, - dep_token.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_fused_dropout(&self, p: f64) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__fused_dropout(c_tensors.as_mut_ptr(), self.c_tensor, p)); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_fused_dropout_out( - &self, - out0: &Tensor, - out1: &Tensor, - p: f64, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__fused_dropout_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - self.c_tensor, - p - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_fused_moving_avg_obs_fq_helper( - &self, - observer_on: &Tensor, - fake_quant_on: &Tensor, - running_min: &Tensor, - running_max: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - averaging_const: f64, - quant_min: i64, - quant_max: i64, - ch_axis: i64, - per_row_fake_quant: bool, - symmetric_quant: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__fused_moving_avg_obs_fq_helper( - c_tensors.as_mut_ptr(), - self.c_tensor, - observer_on.c_tensor, - fake_quant_on.c_tensor, - running_min.c_tensor, - running_max.c_tensor, - scale.c_tensor, - zero_point.c_tensor, - averaging_const, - quant_min, - quant_max, - ch_axis, - if per_row_fake_quant { 1 } else { 0 }, - if symmetric_quant { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_fused_moving_avg_obs_fq_helper_functional( - &self, - observer_on: &Tensor, - fake_quant_on: &Tensor, - running_min: &Tensor, - running_max: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - averaging_const: f64, - quant_min: i64, - quant_max: i64, - ch_axis: i64, - per_row_fake_quant: bool, - symmetric_quant: bool, - ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 6]; - unsafe_torch_err!(atg__fused_moving_avg_obs_fq_helper_functional( - c_tensors.as_mut_ptr(), - self.c_tensor, - observer_on.c_tensor, - fake_quant_on.c_tensor, - running_min.c_tensor, - running_max.c_tensor, - scale.c_tensor, - zero_point.c_tensor, - averaging_const, - quant_min, - quant_max, - ch_axis, - if per_row_fake_quant { 1 } else { 0 }, - if symmetric_quant { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - Tensor { c_tensor: c_tensors[4] }, - Tensor { c_tensor: c_tensors[5] }, - )) - } - - pub fn f_internal_fused_moving_avg_obs_fq_helper_out( - &self, - out0: &Tensor, - out1: &Tensor, - observer_on: &Tensor, - fake_quant_on: &Tensor, - running_min: &Tensor, - running_max: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - averaging_const: f64, - quant_min: i64, - quant_max: i64, - ch_axis: i64, - per_row_fake_quant: bool, - symmetric_quant: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__fused_moving_avg_obs_fq_helper_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - self.c_tensor, - observer_on.c_tensor, - fake_quant_on.c_tensor, - running_min.c_tensor, - running_max.c_tensor, - scale.c_tensor, - zero_point.c_tensor, - averaging_const, - quant_min, - quant_max, - ch_axis, - if per_row_fake_quant { 1 } else { 0 }, - if symmetric_quant { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_fused_rms_norm>( - &self, - normalized_shape: impl IntList, - weight: Option, - eps: impl Into>, - ) -> Result<(Tensor, Tensor), TchError> { - let eps = eps.into(); - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__fused_rms_norm( - c_tensors.as_mut_ptr(), - self.c_tensor, - normalized_shape.as_ptr(), - normalized_shape.len_i32(), - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - eps.unwrap_or(std::f64::NAN), - eps.is_none() as i8 - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_fused_sdp_choice>( - query: &Tensor, - key: &Tensor, - value: &Tensor, - attn_mask: Option, - dropout_p: f64, - is_causal: bool, - scale: impl Into>, - enable_gqa: bool, - ) -> Result { - let scale = scale.into(); - let return_; - unsafe_torch_err!( - return_ = atg__fused_sdp_choice( - query.c_tensor, - key.c_tensor, - value.c_tensor, - attn_mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - dropout_p, - if is_causal { 1 } else { 0 }, - scale.unwrap_or(std::f64::NAN), - scale.is_none() as i8, - if enable_gqa { 1 } else { 0 } - ) - ); - Ok(return_) - } - - pub fn f_internal_fw_primal(&self, level: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__fw_primal(c_tensors.as_mut_ptr(), self.c_tensor, level)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_fw_primal_copy(&self, level: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__fw_primal_copy(c_tensors.as_mut_ptr(), self.c_tensor, level)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_fw_primal_copy_out( - &self, - out: &Tensor, - level: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__fw_primal_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - level - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_gather_sparse_backward( - &self, - dim: i64, - index: &Tensor, - grad: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__gather_sparse_backward( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - grad.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_grid_sampler_2d_cpu_fallback( - &self, - grid: &Tensor, - interpolation_mode: i64, - padding_mode: i64, - align_corners: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__grid_sampler_2d_cpu_fallback( - c_tensors.as_mut_ptr(), - self.c_tensor, - grid.c_tensor, - interpolation_mode, - padding_mode, - if align_corners { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_grid_sampler_2d_cpu_fallback_backward( - &self, - grad_output: &Tensor, - grid: &Tensor, - interpolation_mode: i64, - padding_mode: i64, - align_corners: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__grid_sampler_2d_cpu_fallback_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - grid.c_tensor, - interpolation_mode, - padding_mode, - if align_corners { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_grid_sampler_2d_cpu_fallback_out( - &self, - out: &Tensor, - grid: &Tensor, - interpolation_mode: i64, - padding_mode: i64, - align_corners: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__grid_sampler_2d_cpu_fallback_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - grid.c_tensor, - interpolation_mode, - padding_mode, - if align_corners { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_grouped_mm>( - &self, - mat2: &Tensor, - offs: Option, - bias: Option, - out_dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__grouped_mm( - c_tensors.as_mut_ptr(), - self.c_tensor, - mat2.c_tensor, - offs.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - out_dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_has_compatible_shallow_copy_type( - &self, - from: &Tensor, - ) -> Result { - let return_; - unsafe_torch_err!( - return_ = atg__has_compatible_shallow_copy_type(self.c_tensor, from.c_tensor) - ); - Ok(return_ != 0) - } - - pub fn f_internal_has_same_storage_numel(&self, other: &Tensor) -> Result { - let return_; - unsafe_torch_err!(return_ = atg__has_same_storage_numel(self.c_tensor, other.c_tensor)); - Ok(return_ != 0) - } - - pub fn f_internal_histogramdd_bin_edges>( - &self, - bins: impl IntList, - range: impl DoubleList, - weight: Option, - density: bool, - ) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg__histogramdd_bin_edges( - self.c_tensor, - bins.as_ptr(), - bins.len_i32(), - range.as_ptr(), - range.len_i32(), - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if density { 1 } else { 0 } - )); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_internal_histogramdd_bin_edges_out>( - &self, - out: &[T], - bins: impl IntList, - range: impl DoubleList, - weight: Option, - density: bool, - ) -> Result<(), TchError> { - unsafe_torch_err!(atg__histogramdd_bin_edges_out( - ptr_list(out).as_ptr(), - out.len() as i32, - self.c_tensor, - bins.as_ptr(), - bins.len_i32(), - range.as_ptr(), - range.len_i32(), - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if density { 1 } else { 0 } - )); - Ok(()) - } - - pub fn f_internal_histogramdd_from_bin_cts>( - &self, - bins: impl IntList, - range: impl DoubleList, - weight: Option, - density: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__histogramdd_from_bin_cts( - c_tensors.as_mut_ptr(), - self.c_tensor, - bins.as_ptr(), - bins.len_i32(), - range.as_ptr(), - range.len_i32(), - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if density { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_histogramdd_from_bin_cts_out>( - &self, - out: &Tensor, - bins: impl IntList, - range: impl DoubleList, - weight: Option, - density: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__histogramdd_from_bin_cts_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - bins.as_ptr(), - bins.len_i32(), - range.as_ptr(), - range.len_i32(), - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if density { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_histogramdd_from_bin_tensors>( - &self, - bins: &[T], - weight: Option, - density: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__histogramdd_from_bin_tensors( - c_tensors.as_mut_ptr(), - self.c_tensor, - ptr_list(bins).as_ptr(), - bins.len() as i32, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if density { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_histogramdd_from_bin_tensors_out>( - &self, - out: &Tensor, - bins: &[T], - weight: Option, - density: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__histogramdd_from_bin_tensors_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - ptr_list(bins).as_ptr(), - bins.len() as i32, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if density { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_index_put_impl>( - &self, - indices: &[Option], - values: &Tensor, - accumulate: bool, - unsafe_: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__index_put_impl( - c_tensors.as_mut_ptr(), - self.c_tensor, - ptr_list_opt(indices).as_ptr(), - indices.len() as i32, - values.c_tensor, - if accumulate { 1 } else { 0 }, - if unsafe_ { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_index_put_impl_>( - &mut self, - indices: &[Option], - values: &Tensor, - accumulate: bool, - unsafe_: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__index_put_impl_( - c_tensors.as_mut_ptr(), - self.c_tensor, - ptr_list_opt(indices).as_ptr(), - indices.len() as i32, - values.c_tensor, - if accumulate { 1 } else { 0 }, - if unsafe_ { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_index_put_impl_out>( - &self, - out: &Tensor, - indices: &[Option], - values: &Tensor, - accumulate: bool, - unsafe_: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__index_put_impl_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - ptr_list_opt(indices).as_ptr(), - indices.len() as i32, - values.c_tensor, - if accumulate { 1 } else { 0 }, - if unsafe_ { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_indices(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__indices(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_indices_copy(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__indices_copy(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_indices_copy_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__indices_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_int_mm(&self, mat2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__int_mm(c_tensors.as_mut_ptr(), self.c_tensor, mat2.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_int_mm_out(&self, out: &Tensor, mat2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__int_mm_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mat2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_is_all_true(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__is_all_true(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_is_any_true(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__is_any_true(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_is_zerotensor(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg__is_zerotensor(self.c_tensor)); - Ok(return_ != 0) - } - - pub fn f_internal_lazy_clone(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__lazy_clone(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_linalg_check_errors( - info: &Tensor, - api_name: &str, - is_matrix: bool, - ) -> Result<(), TchError> { - unsafe_torch_err!(atg__linalg_check_errors( - info.c_tensor, - api_name.as_ptr(), - api_name.len() as i32, - if is_matrix { 1 } else { 0 } - )); - Ok(()) - } - - pub fn f_internal_linalg_det(a: &Tensor) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__linalg_det(c_tensors.as_mut_ptr(), a.c_tensor)); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_linalg_det_result( - result: &Tensor, - lu: &Tensor, - pivots: &Tensor, - a: &Tensor, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__linalg_det_result( - c_tensors.as_mut_ptr(), - result.c_tensor, - lu.c_tensor, - pivots.c_tensor, - a.c_tensor - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_linalg_eigh( - a: &Tensor, - uplo: &str, - compute_v: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__linalg_eigh( - c_tensors.as_mut_ptr(), - a.c_tensor, - uplo.as_ptr(), - uplo.len() as i32, - if compute_v { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_linalg_eigh_eigenvalues( - eigenvalues: &Tensor, - eigenvectors: &Tensor, - a: &Tensor, - uplo: &str, - compute_v: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__linalg_eigh_eigenvalues( - c_tensors.as_mut_ptr(), - eigenvalues.c_tensor, - eigenvectors.c_tensor, - a.c_tensor, - uplo.as_ptr(), - uplo.len() as i32, - if compute_v { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_linalg_eigvals(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__linalg_eigvals(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_linalg_slogdet( - a: &Tensor, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg__linalg_slogdet(c_tensors.as_mut_ptr(), a.c_tensor)); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_internal_linalg_slogdet_sign( - sign: &Tensor, - logabsdet: &Tensor, - lu: &Tensor, - pivots: &Tensor, - a: &Tensor, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg__linalg_slogdet_sign( - c_tensors.as_mut_ptr(), - sign.c_tensor, - logabsdet.c_tensor, - lu.c_tensor, - pivots.c_tensor, - a.c_tensor - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_internal_linalg_solve_ex( - a: &Tensor, - b: &Tensor, - left: bool, - check_errors: bool, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg__linalg_solve_ex( - c_tensors.as_mut_ptr(), - a.c_tensor, - b.c_tensor, - if left { 1 } else { 0 }, - if check_errors { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_internal_linalg_solve_ex_result( - result: &Tensor, - lu: &Tensor, - pivots: &Tensor, - info: &Tensor, - a: &Tensor, - b: &Tensor, - left: bool, - check_errors: bool, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg__linalg_solve_ex_result( - c_tensors.as_mut_ptr(), - result.c_tensor, - lu.c_tensor, - pivots.c_tensor, - info.c_tensor, - a.c_tensor, - b.c_tensor, - if left { 1 } else { 0 }, - if check_errors { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_internal_linalg_svd( - a: &Tensor, - full_matrices: bool, - compute_uv: bool, - driver: &str, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__linalg_svd( - c_tensors.as_mut_ptr(), - a.c_tensor, - if full_matrices { 1 } else { 0 }, - if compute_uv { 1 } else { 0 }, - driver.as_ptr(), - driver.len() as i32 - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_linalg_svd_u( - u: &Tensor, - s: &Tensor, - vh: &Tensor, - a: &Tensor, - full_matrices: bool, - compute_uv: bool, - driver: &str, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__linalg_svd_u( - c_tensors.as_mut_ptr(), - u.c_tensor, - s.c_tensor, - vh.c_tensor, - a.c_tensor, - if full_matrices { 1 } else { 0 }, - if compute_uv { 1 } else { 0 }, - driver.as_ptr(), - driver.len() as i32 - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_log_softmax( - &self, - dim: i64, - half_to_float: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__log_softmax( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - if half_to_float { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_log_softmax_backward_data( - grad_output: &Tensor, - output: &Tensor, - dim: i64, - input_dtype: Kind, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__log_softmax_backward_data( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - output.c_tensor, - dim, - input_dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_log_softmax_backward_data_out( - out: &Tensor, - grad_output: &Tensor, - output: &Tensor, - dim: i64, - input_dtype: Kind, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__log_softmax_backward_data_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad_output.c_tensor, - output.c_tensor, - dim, - input_dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_log_softmax_out( - &self, - out: &Tensor, - dim: i64, - half_to_float: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__log_softmax_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - if half_to_float { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_logcumsumexp(&self, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__logcumsumexp(c_tensors.as_mut_ptr(), self.c_tensor, dim)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_logcumsumexp_out(&self, out: &Tensor, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__logcumsumexp_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_lstm_mps>( - &self, - hx: &[T], - params: &[T], - has_biases: bool, - num_layers: i64, - dropout: f64, - train: bool, - bidirectional: bool, - batch_first: bool, - ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 6]; - unsafe_torch_err!(atg__lstm_mps( - c_tensors.as_mut_ptr(), - self.c_tensor, - ptr_list(hx).as_ptr(), - hx.len() as i32, - ptr_list(params).as_ptr(), - params.len() as i32, - if has_biases { 1 } else { 0 }, - num_layers, - dropout, - if train { 1 } else { 0 }, - if bidirectional { 1 } else { 0 }, - if batch_first { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - Tensor { c_tensor: c_tensors[4] }, - Tensor { c_tensor: c_tensors[5] }, - )) - } - - pub fn f_internal_lstm_mps_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - out3: &Tensor, - out4: &Tensor, - out5: &Tensor, - hx: &[T], - params: &[T], - has_biases: bool, - num_layers: i64, - dropout: f64, - train: bool, - bidirectional: bool, - batch_first: bool, - ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 6]; - unsafe_torch_err!(atg__lstm_mps_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - out2.c_tensor, - out3.c_tensor, - out4.c_tensor, - out5.c_tensor, - self.c_tensor, - ptr_list(hx).as_ptr(), - hx.len() as i32, - ptr_list(params).as_ptr(), - params.len() as i32, - if has_biases { 1 } else { 0 }, - num_layers, - dropout, - if train { 1 } else { 0 }, - if bidirectional { 1 } else { 0 }, - if batch_first { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - Tensor { c_tensor: c_tensors[4] }, - Tensor { c_tensor: c_tensors[5] }, - )) - } - - pub fn f_internal_lu_with_info( - &self, - pivot: bool, - check_errors: bool, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__lu_with_info( - c_tensors.as_mut_ptr(), - self.c_tensor, - if pivot { 1 } else { 0 }, - if check_errors { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_make_dep_token(options: (Kind, Device)) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__make_dep_token( - c_tensors.as_mut_ptr(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_make_dual( - primal: &Tensor, - tangent: &Tensor, - level: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__make_dual( - c_tensors.as_mut_ptr(), - primal.c_tensor, - tangent.c_tensor, - level - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_make_dual_copy( - primal: &Tensor, - tangent: &Tensor, - level: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__make_dual_copy( - c_tensors.as_mut_ptr(), - primal.c_tensor, - tangent.c_tensor, - level - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_make_dual_copy_out( - out: &Tensor, - primal: &Tensor, - tangent: &Tensor, - level: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__make_dual_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - primal.c_tensor, - tangent.c_tensor, - level - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_make_per_channel_quantized_tensor( - &self, - scale: &Tensor, - zero_point: &Tensor, - axis: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__make_per_channel_quantized_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - scale.c_tensor, - zero_point.c_tensor, - axis - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_make_per_channel_quantized_tensor_out( - &self, - out: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - axis: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__make_per_channel_quantized_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - scale.c_tensor, - zero_point.c_tensor, - axis - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_make_per_tensor_quantized_tensor( - &self, - scale: f64, - zero_point: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__make_per_tensor_quantized_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - scale, - zero_point - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_make_per_tensor_quantized_tensor_out( - &self, - out: &Tensor, - scale: f64, - zero_point: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__make_per_tensor_quantized_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - scale, - zero_point - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_masked_scale(&self, mask: &Tensor, scale: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__masked_scale( - c_tensors.as_mut_ptr(), - self.c_tensor, - mask.c_tensor, - scale - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_masked_scale_out( - &self, - out: &Tensor, - mask: &Tensor, - scale: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__masked_scale_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mask.c_tensor, - scale - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_masked_softmax( - &self, - mask: &Tensor, - dim: impl Into>, - mask_type: impl Into>, - ) -> Result { - let dim = dim.into(); - let mask_type = mask_type.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__masked_softmax( - c_tensors.as_mut_ptr(), - self.c_tensor, - mask.c_tensor, - dim.unwrap_or(0i64), - dim.is_none() as i8, - mask_type.unwrap_or(0i64), - mask_type.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_masked_softmax_backward( - grad_output: &Tensor, - output: &Tensor, - mask: &Tensor, - dim: impl Into>, - ) -> Result { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__masked_softmax_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - output.c_tensor, - mask.c_tensor, - dim.unwrap_or(0i64), - dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_masked_softmax_backward_out( - out: &Tensor, - grad_output: &Tensor, - output: &Tensor, - mask: &Tensor, - dim: impl Into>, - ) -> Result { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__masked_softmax_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad_output.c_tensor, - output.c_tensor, - mask.c_tensor, - dim.unwrap_or(0i64), - dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_masked_softmax_out( - &self, - out: &Tensor, - mask: &Tensor, - dim: impl Into>, - mask_type: impl Into>, - ) -> Result { - let dim = dim.into(); - let mask_type = mask_type.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__masked_softmax_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mask.c_tensor, - dim.unwrap_or(0i64), - dim.is_none() as i8, - mask_type.unwrap_or(0i64), - mask_type.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_mixed_dtypes_linear>( - &self, - weight: &Tensor, - scale: &Tensor, - bias: Option, - activation: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__mixed_dtypes_linear( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - scale.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - activation.as_ptr(), - activation.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_mkldnn_reshape(&self, shape: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__mkldnn_reshape( - c_tensors.as_mut_ptr(), - self.c_tensor, - shape.as_ptr(), - shape.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_mkldnn_reshape_out( - &self, - out: &Tensor, - shape: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__mkldnn_reshape_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - shape.as_ptr(), - shape.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_mkldnn_transpose(&self, dim0: i64, dim1: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__mkldnn_transpose(c_tensors.as_mut_ptr(), self.c_tensor, dim0, dim1)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_mkldnn_transpose_( - &mut self, - dim0: i64, - dim1: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__mkldnn_transpose_( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim0, - dim1 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_mkldnn_transpose_out( - &self, - out: &Tensor, - dim0: i64, - dim1: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__mkldnn_transpose_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim0, - dim1 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_mps_convolution>( - &self, - weight: &Tensor, - bias: Option, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__mps_convolution( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - padding.as_ptr(), - padding.len_i32(), - stride.as_ptr(), - stride.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_mps_convolution_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__mps_convolution_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - padding.as_ptr(), - padding.len_i32(), - stride.as_ptr(), - stride.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_mps_convolution_transpose( - &self, - weight: &Tensor, - padding: impl IntList, - output_padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__mps_convolution_transpose( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - padding.as_ptr(), - padding.len_i32(), - output_padding.as_ptr(), - output_padding.len_i32(), - stride.as_ptr(), - stride.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_mps_convolution_transpose_out( - &self, - out: &Tensor, - weight: &Tensor, - padding: impl IntList, - output_padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__mps_convolution_transpose_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - padding.as_ptr(), - padding.len_i32(), - output_padding.as_ptr(), - output_padding.len_i32(), - stride.as_ptr(), - stride.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_native_batch_norm_legit>( - &self, - weight: Option, - bias: Option, - running_mean: &Tensor, - running_var: &Tensor, - training: bool, - momentum: f64, - eps: f64, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__native_batch_norm_legit( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_mean.c_tensor, - running_var.c_tensor, - if training { 1 } else { 0 }, - momentum, - eps - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_native_batch_norm_legit_functional>( - &self, - weight: Option, - bias: Option, - running_mean: &Tensor, - running_var: &Tensor, - training: bool, - momentum: f64, - eps: f64, - ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 5]; - unsafe_torch_err!(atg__native_batch_norm_legit_functional( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_mean.c_tensor, - running_var.c_tensor, - if training { 1 } else { 0 }, - momentum, - eps - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - Tensor { c_tensor: c_tensors[4] }, - )) - } - - pub fn f_internal_native_batch_norm_legit_no_stats>( - &self, - weight: Option, - bias: Option, - training: bool, - momentum: f64, - eps: f64, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__native_batch_norm_legit_no_stats( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if training { 1 } else { 0 }, - momentum, - eps - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_native_batch_norm_legit_no_stats_out>( - &self, - out: &Tensor, - save_mean: &Tensor, - save_invstd: &Tensor, - weight: Option, - bias: Option, - training: bool, - momentum: f64, - eps: f64, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__native_batch_norm_legit_no_stats_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - save_mean.c_tensor, - save_invstd.c_tensor, - self.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if training { 1 } else { 0 }, - momentum, - eps - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_native_batch_norm_legit_no_training>( - &self, - weight: Option, - bias: Option, - running_mean: &Tensor, - running_var: &Tensor, - momentum: f64, - eps: f64, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__native_batch_norm_legit_no_training( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_mean.c_tensor, - running_var.c_tensor, - momentum, - eps - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_native_batch_norm_legit_no_training_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - weight: Option, - bias: Option, - running_mean: &Tensor, - running_var: &Tensor, - momentum: f64, - eps: f64, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__native_batch_norm_legit_no_training_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - out2.c_tensor, - self.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_mean.c_tensor, - running_var.c_tensor, - momentum, - eps - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_native_batch_norm_legit_out>( - &self, - out: &Tensor, - save_mean: &Tensor, - save_invstd: &Tensor, - weight: Option, - bias: Option, - running_mean: &Tensor, - running_var: &Tensor, - training: bool, - momentum: f64, - eps: f64, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__native_batch_norm_legit_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - save_mean.c_tensor, - save_invstd.c_tensor, - self.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_mean.c_tensor, - running_var.c_tensor, - if training { 1 } else { 0 }, - momentum, - eps - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_native_multi_head_attention>( - query: &Tensor, - key: &Tensor, - value: &Tensor, - embed_dim: i64, - num_head: i64, - qkv_weight: &Tensor, - qkv_bias: &Tensor, - proj_weight: &Tensor, - proj_bias: &Tensor, - mask: Option, - need_weights: bool, - average_attn_weights: bool, - mask_type: impl Into>, - ) -> Result<(Tensor, Tensor), TchError> { - let mask_type = mask_type.into(); - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__native_multi_head_attention( - c_tensors.as_mut_ptr(), - query.c_tensor, - key.c_tensor, - value.c_tensor, - embed_dim, - num_head, - qkv_weight.c_tensor, - qkv_bias.c_tensor, - proj_weight.c_tensor, - proj_bias.c_tensor, - mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if need_weights { 1 } else { 0 }, - if average_attn_weights { 1 } else { 0 }, - mask_type.unwrap_or(0i64), - mask_type.is_none() as i8 - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_native_multi_head_attention_out>( - out0: &Tensor, - out1: &Tensor, - query: &Tensor, - key: &Tensor, - value: &Tensor, - embed_dim: i64, - num_head: i64, - qkv_weight: &Tensor, - qkv_bias: &Tensor, - proj_weight: &Tensor, - proj_bias: &Tensor, - mask: Option, - need_weights: bool, - average_attn_weights: bool, - mask_type: impl Into>, - ) -> Result<(Tensor, Tensor), TchError> { - let mask_type = mask_type.into(); - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__native_multi_head_attention_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - query.c_tensor, - key.c_tensor, - value.c_tensor, - embed_dim, - num_head, - qkv_weight.c_tensor, - qkv_bias.c_tensor, - proj_weight.c_tensor, - proj_bias.c_tensor, - mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if need_weights { 1 } else { 0 }, - if average_attn_weights { 1 } else { 0 }, - mask_type.unwrap_or(0i64), - mask_type.is_none() as i8 - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_neg_view(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__neg_view(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_neg_view_copy(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__neg_view_copy(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_neg_view_copy_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__neg_view_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nested_compute_contiguous_strides_offsets( - nested_size: &Tensor, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__nested_compute_contiguous_strides_offsets( - c_tensors.as_mut_ptr(), - nested_size.c_tensor - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_nested_from_padded( - padded: &Tensor, - cpu_nested_shape_example: &Tensor, - fuse_transform_0213: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nested_from_padded( - c_tensors.as_mut_ptr(), - padded.c_tensor, - cpu_nested_shape_example.c_tensor, - if fuse_transform_0213 { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nested_from_padded_and_nested_example( - padded: &Tensor, - nt_example: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nested_from_padded_and_nested_example( - c_tensors.as_mut_ptr(), - padded.c_tensor, - nt_example.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nested_from_padded_and_nested_example_out( - out: &Tensor, - padded: &Tensor, - nt_example: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nested_from_padded_and_nested_example_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - padded.c_tensor, - nt_example.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nested_from_padded_out( - out: &Tensor, - padded: &Tensor, - cpu_nested_shape_example: &Tensor, - fuse_transform_0213: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nested_from_padded_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - padded.c_tensor, - cpu_nested_shape_example.c_tensor, - if fuse_transform_0213 { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nested_from_padded_tensor>( - padded: &Tensor, - offsets: &Tensor, - dummy: &Tensor, - ragged_idx: i64, - min_seqlen: Option, - max_seqlen: Option, - sum_s: impl Into>, - ) -> Result { - let sum_s = sum_s.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nested_from_padded_tensor( - c_tensors.as_mut_ptr(), - padded.c_tensor, - offsets.c_tensor, - dummy.c_tensor, - ragged_idx, - min_seqlen.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - max_seqlen.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - sum_s.unwrap_or(0i64), - sum_s.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nested_get_jagged_dummy(any: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nested_get_jagged_dummy(c_tensors.as_mut_ptr(), any.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nested_get_lengths(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nested_get_lengths(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nested_get_max_seqlen(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nested_get_max_seqlen(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nested_get_min_seqlen(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nested_get_min_seqlen(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nested_get_offsets(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nested_get_offsets(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nested_get_ragged_idx(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg__nested_get_ragged_idx(self.c_tensor)); - Ok(return_) - } - - pub fn f_internal_nested_get_values(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nested_get_values(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nested_get_values_copy(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nested_get_values_copy(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nested_get_values_copy_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nested_get_values_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nested_select_backward( - &self, - grad_output: &Tensor, - dim: i64, - index: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nested_select_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - dim, - index - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nested_sum_backward( - &self, - grad: &Tensor, - dim: impl IntListOption, - keepdim: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nested_sum_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nested_view_from_buffer( - &self, - nested_size: &Tensor, - nested_strides: &Tensor, - offsets: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nested_view_from_buffer( - c_tensors.as_mut_ptr(), - self.c_tensor, - nested_size.c_tensor, - nested_strides.c_tensor, - offsets.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nested_view_from_buffer_copy( - &self, - nested_size: &Tensor, - nested_strides: &Tensor, - offsets: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nested_view_from_buffer_copy( - c_tensors.as_mut_ptr(), - self.c_tensor, - nested_size.c_tensor, - nested_strides.c_tensor, - offsets.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nested_view_from_buffer_copy_out( - &self, - out: &Tensor, - nested_size: &Tensor, - nested_strides: &Tensor, - offsets: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nested_view_from_buffer_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - nested_size.c_tensor, - nested_strides.c_tensor, - offsets.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nested_view_from_jagged>( - &self, - offsets: &Tensor, - dummy: &Tensor, - lengths: Option, - ragged_idx: i64, - min_seqlen: Option, - max_seqlen: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nested_view_from_jagged( - c_tensors.as_mut_ptr(), - self.c_tensor, - offsets.c_tensor, - dummy.c_tensor, - lengths.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - ragged_idx, - min_seqlen.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - max_seqlen.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nested_view_from_jagged_copy>( - &self, - offsets: &Tensor, - dummy: &Tensor, - lengths: Option, - ragged_idx: i64, - min_seqlen: Option, - max_seqlen: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nested_view_from_jagged_copy( - c_tensors.as_mut_ptr(), - self.c_tensor, - offsets.c_tensor, - dummy.c_tensor, - lengths.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - ragged_idx, - min_seqlen.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - max_seqlen.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nested_view_from_jagged_copy_out>( - &self, - out: &Tensor, - offsets: &Tensor, - dummy: &Tensor, - lengths: Option, - ragged_idx: i64, - min_seqlen: Option, - max_seqlen: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nested_view_from_jagged_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - offsets.c_tensor, - dummy.c_tensor, - lengths.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - ragged_idx, - min_seqlen.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - max_seqlen.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_new_zeros_with_same_feature_meta( - &self, - other: &Tensor, - self_num_batch_dims: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__new_zeros_with_same_feature_meta( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor, - self_num_batch_dims - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_new_zeros_with_same_feature_meta_out( - &self, - out: &Tensor, - other: &Tensor, - self_num_batch_dims: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__new_zeros_with_same_feature_meta_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor, - self_num_batch_dims - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nnpack_available() -> Result { - let return_; - unsafe_torch_err!(return_ = atg__nnpack_available()); - Ok(return_ != 0) - } - - pub fn f_internal_nnpack_spatial_convolution>( - &self, - weight: &Tensor, - bias: Option, - padding: impl IntList, - stride: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nnpack_spatial_convolution( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - padding.as_ptr(), - padding.len_i32(), - stride.as_ptr(), - stride.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nnpack_spatial_convolution_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - padding: impl IntList, - stride: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__nnpack_spatial_convolution_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - padding.as_ptr(), - padding.len_i32(), - stride.as_ptr(), - stride.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_nnz(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg__nnz(self.c_tensor)); - Ok(return_) - } - - pub fn f_internal_pack_padded_sequence( - &self, - lengths: &Tensor, - batch_first: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__pack_padded_sequence( - c_tensors.as_mut_ptr(), - self.c_tensor, - lengths.c_tensor, - if batch_first { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_pack_padded_sequence_backward( - grad: &Tensor, - input_size: impl IntList, - batch_sizes: &Tensor, - batch_first: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__pack_padded_sequence_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - input_size.as_ptr(), - input_size.len_i32(), - batch_sizes.c_tensor, - if batch_first { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_pack_padded_sequence_out( - &self, - out0: &Tensor, - out1: &Tensor, - lengths: &Tensor, - batch_first: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__pack_padded_sequence_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - self.c_tensor, - lengths.c_tensor, - if batch_first { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_pad_circular(&self, pad: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__pad_circular( - c_tensors.as_mut_ptr(), - self.c_tensor, - pad.as_ptr(), - pad.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_pad_enum( - &self, - pad: impl IntList, - mode: i64, - value: impl Into>, - ) -> Result { - let value = value.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__pad_enum( - c_tensors.as_mut_ptr(), - self.c_tensor, - pad.as_ptr(), - pad.len_i32(), - mode, - value.unwrap_or(std::f64::NAN), - value.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_pad_packed_sequence>( - data: &Tensor, - batch_sizes: &Tensor, - batch_first: bool, - padding_value: S, - total_length: i64, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__pad_packed_sequence( - c_tensors.as_mut_ptr(), - data.c_tensor, - batch_sizes.c_tensor, - if batch_first { 1 } else { 0 }, - padding_value.into().c_scalar, - total_length - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_pdist_backward( - &self, - grad: &Tensor, - p: f64, - pdist: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__pdist_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - self.c_tensor, - p, - pdist.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_pdist_backward_out( - &self, - out: &Tensor, - grad: &Tensor, - p: f64, - pdist: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__pdist_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad.c_tensor, - self.c_tensor, - p, - pdist.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_pin_memory(&self, device: Device) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__pin_memory(c_tensors.as_mut_ptr(), self.c_tensor, device.c_int())); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_pin_memory_out( - &self, - out: &Tensor, - device: Device, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__pin_memory_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - device.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_prelu_kernel(&self, weight: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__prelu_kernel( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_prelu_kernel_backward( - &self, - grad_output: &Tensor, - weight: &Tensor, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__prelu_kernel_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - weight.c_tensor - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_print(s: &str) -> Result<(), TchError> { - unsafe_torch_err!(atg__print(s.as_ptr(), s.len() as i32)); - Ok(()) - } - - pub fn f_internal_propagate_xla_data(&self, output: &Tensor) -> Result<(), TchError> { - unsafe_torch_err!(atg__propagate_xla_data(self.c_tensor, output.c_tensor)); - Ok(()) - } - - pub fn f_internal_remove_batch_dim( - &self, - level: i64, - batch_size: i64, - out_dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__remove_batch_dim( - c_tensors.as_mut_ptr(), - self.c_tensor, - level, - batch_size, - out_dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_reshape_alias( - &self, - size: impl IntList, - stride: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__reshape_alias( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32(), - stride.as_ptr(), - stride.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_reshape_alias_copy( - &self, - size: impl IntList, - stride: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__reshape_alias_copy( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32(), - stride.as_ptr(), - stride.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_reshape_alias_copy_out( - &self, - out: &Tensor, - size: impl IntList, - stride: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__reshape_alias_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - size.as_ptr(), - size.len_i32(), - stride.as_ptr(), - stride.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_reshape_copy(&self, size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__reshape_copy( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_reshape_from_tensor(&self, shape: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__reshape_from_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - shape.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_resize_output( - &self, - size: impl IntList, - device: Device, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__resize_output( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32(), - device.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_resize_output_( - &mut self, - size: impl IntList, - device: Device, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__resize_output_( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32(), - device.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_resize_output_out( - &self, - out: &Tensor, - size: impl IntList, - device: Device, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__resize_output_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - size.as_ptr(), - size.len_i32(), - device.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_rowwise_prune( - weight: &Tensor, - mask: &Tensor, - compressed_indices_dtype: Kind, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__rowwise_prune( - c_tensors.as_mut_ptr(), - weight.c_tensor, - mask.c_tensor, - compressed_indices_dtype.c_int() - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_safe_softmax( - &self, - dim: i64, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__safe_softmax( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sample_dirichlet(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sample_dirichlet(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sample_dirichlet_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sample_dirichlet_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_saturate_weight_to_fp16(weight: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__saturate_weight_to_fp16(c_tensors.as_mut_ptr(), weight.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_scaled_dot_product_attention_math>( - query: &Tensor, - key: &Tensor, - value: &Tensor, - attn_mask: Option, - dropout_p: f64, - is_causal: bool, - dropout_mask: Option, - scale: impl Into>, - enable_gqa: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let scale = scale.into(); - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__scaled_dot_product_attention_math( - c_tensors.as_mut_ptr(), - query.c_tensor, - key.c_tensor, - value.c_tensor, - attn_mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - dropout_p, - if is_causal { 1 } else { 0 }, - dropout_mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - scale.unwrap_or(std::f64::NAN), - scale.is_none() as i8, - if enable_gqa { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_scaled_dot_product_attention_math_for_mps>( - query: &Tensor, - key: &Tensor, - value: &Tensor, - attn_mask: Option, - dropout_p: f64, - is_causal: bool, - dropout_mask: Option, - scale: impl Into>, - ) -> Result<(Tensor, Tensor), TchError> { - let scale = scale.into(); - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__scaled_dot_product_attention_math_for_mps( - c_tensors.as_mut_ptr(), - query.c_tensor, - key.c_tensor, - value.c_tensor, - attn_mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - dropout_p, - if is_causal { 1 } else { 0 }, - dropout_mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - scale.unwrap_or(std::f64::NAN), - scale.is_none() as i8 - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_scaled_dot_product_cudnn_attention_backward( - grad_out: &Tensor, - query: &Tensor, - key: &Tensor, - value: &Tensor, - out: &Tensor, - logsumexp: &Tensor, - philox_seed: &Tensor, - philox_offset: &Tensor, - attn_bias: &Tensor, - cum_seq_q: &Tensor, - cum_seq_k: &Tensor, - max_q: i64, - max_k: i64, - dropout_p: f64, - is_causal: bool, - scale: impl Into>, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let scale = scale.into(); - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__scaled_dot_product_cudnn_attention_backward( - c_tensors.as_mut_ptr(), - grad_out.c_tensor, - query.c_tensor, - key.c_tensor, - value.c_tensor, - out.c_tensor, - logsumexp.c_tensor, - philox_seed.c_tensor, - philox_offset.c_tensor, - attn_bias.c_tensor, - cum_seq_q.c_tensor, - cum_seq_k.c_tensor, - max_q, - max_k, - dropout_p, - if is_causal { 1 } else { 0 }, - scale.unwrap_or(std::f64::NAN), - scale.is_none() as i8 - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_scaled_dot_product_efficient_attention>( - query: &Tensor, - key: &Tensor, - value: &Tensor, - attn_bias: Option, - compute_log_sumexp: bool, - dropout_p: f64, - is_causal: bool, - scale: impl Into>, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let scale = scale.into(); - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg__scaled_dot_product_efficient_attention( - c_tensors.as_mut_ptr(), - query.c_tensor, - key.c_tensor, - value.c_tensor, - attn_bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if compute_log_sumexp { 1 } else { 0 }, - dropout_p, - if is_causal { 1 } else { 0 }, - scale.unwrap_or(std::f64::NAN), - scale.is_none() as i8 - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_internal_scaled_dot_product_flash_attention_backward( - grad_out: &Tensor, - query: &Tensor, - key: &Tensor, - value: &Tensor, - out: &Tensor, - logsumexp: &Tensor, - cum_seq_q: &Tensor, - cum_seq_k: &Tensor, - max_q: i64, - max_k: i64, - dropout_p: f64, - is_causal: bool, - philox_seed: &Tensor, - philox_offset: &Tensor, - scale: impl Into>, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let scale = scale.into(); - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__scaled_dot_product_flash_attention_backward( - c_tensors.as_mut_ptr(), - grad_out.c_tensor, - query.c_tensor, - key.c_tensor, - value.c_tensor, - out.c_tensor, - logsumexp.c_tensor, - cum_seq_q.c_tensor, - cum_seq_k.c_tensor, - max_q, - max_k, - dropout_p, - if is_causal { 1 } else { 0 }, - philox_seed.c_tensor, - philox_offset.c_tensor, - scale.unwrap_or(std::f64::NAN), - scale.is_none() as i8 - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_scaled_dot_product_flash_attention_for_cpu>( - query: &Tensor, - key: &Tensor, - value: &Tensor, - dropout_p: f64, - is_causal: bool, - attn_mask: Option, - scale: impl Into>, - ) -> Result<(Tensor, Tensor), TchError> { - let scale = scale.into(); - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__scaled_dot_product_flash_attention_for_cpu( - c_tensors.as_mut_ptr(), - query.c_tensor, - key.c_tensor, - value.c_tensor, - dropout_p, - if is_causal { 1 } else { 0 }, - attn_mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - scale.unwrap_or(std::f64::NAN), - scale.is_none() as i8 - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_scaled_dot_product_flash_attention_for_cpu_backward>( - grad_out: &Tensor, - query: &Tensor, - key: &Tensor, - value: &Tensor, - out: &Tensor, - logsumexp: &Tensor, - dropout_p: f64, - is_causal: bool, - attn_mask: Option, - scale: impl Into>, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let scale = scale.into(); - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__scaled_dot_product_flash_attention_for_cpu_backward( - c_tensors.as_mut_ptr(), - grad_out.c_tensor, - query.c_tensor, - key.c_tensor, - value.c_tensor, - out.c_tensor, - logsumexp.c_tensor, - dropout_p, - if is_causal { 1 } else { 0 }, - attn_mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - scale.unwrap_or(std::f64::NAN), - scale.is_none() as i8 - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_scaled_grouped_mm>( - &self, - mat2: &Tensor, - scale_a: &Tensor, - scale_b: &Tensor, - offs: Option, - bias: Option, - scale_result: Option, - out_dtype: impl Into>, - use_fast_accum: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__scaled_grouped_mm( - c_tensors.as_mut_ptr(), - self.c_tensor, - mat2.c_tensor, - scale_a.c_tensor, - scale_b.c_tensor, - offs.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - scale_result.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - out_dtype.into().map_or(-1, |s| s.c_int()), - if use_fast_accum { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_scaled_grouped_mm_v2>( - &self, - mat2: &Tensor, - scale_a: &[T], - recipe_a: impl IntList, - swizzle_a: impl IntList, - scale_b: &[T], - recipe_b: impl IntList, - swizzle_b: impl IntList, - offs: Option, - bias: Option, - out_dtype: impl Into>, - contraction_dim: impl IntList, - use_fast_accum: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__scaled_grouped_mm_v2( - c_tensors.as_mut_ptr(), - self.c_tensor, - mat2.c_tensor, - ptr_list(scale_a).as_ptr(), - scale_a.len() as i32, - recipe_a.as_ptr(), - recipe_a.len_i32(), - swizzle_a.as_ptr(), - swizzle_a.len_i32(), - ptr_list(scale_b).as_ptr(), - scale_b.len() as i32, - recipe_b.as_ptr(), - recipe_b.len_i32(), - swizzle_b.as_ptr(), - swizzle_b.len_i32(), - offs.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - out_dtype.into().map_or(-1, |s| s.c_int()), - contraction_dim.as_ptr(), - contraction_dim.len_i32(), - if use_fast_accum { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_scaled_mm>( - &self, - mat2: &Tensor, - scale_a: &Tensor, - scale_b: &Tensor, - bias: Option, - scale_result: Option, - out_dtype: impl Into>, - use_fast_accum: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__scaled_mm( - c_tensors.as_mut_ptr(), - self.c_tensor, - mat2.c_tensor, - scale_a.c_tensor, - scale_b.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - scale_result.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - out_dtype.into().map_or(-1, |s| s.c_int()), - if use_fast_accum { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_scaled_mm_out>( - &self, - out: &Tensor, - mat2: &Tensor, - scale_a: &Tensor, - scale_b: &Tensor, - bias: Option, - scale_result: Option, - out_dtype: impl Into>, - use_fast_accum: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__scaled_mm_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mat2.c_tensor, - scale_a.c_tensor, - scale_b.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - scale_result.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - out_dtype.into().map_or(-1, |s| s.c_int()), - if use_fast_accum { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_scaled_mm_v2>( - &self, - mat2: &Tensor, - scale_a: &[T], - recipe_a: impl IntList, - swizzle_a: impl IntList, - scale_b: &[T], - recipe_b: impl IntList, - swizzle_b: impl IntList, - bias: Option, - out_dtype: impl Into>, - contraction_dim: impl IntList, - use_fast_accum: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__scaled_mm_v2( - c_tensors.as_mut_ptr(), - self.c_tensor, - mat2.c_tensor, - ptr_list(scale_a).as_ptr(), - scale_a.len() as i32, - recipe_a.as_ptr(), - recipe_a.len_i32(), - swizzle_a.as_ptr(), - swizzle_a.len_i32(), - ptr_list(scale_b).as_ptr(), - scale_b.len() as i32, - recipe_b.as_ptr(), - recipe_b.len_i32(), - swizzle_b.as_ptr(), - swizzle_b.len_i32(), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - out_dtype.into().map_or(-1, |s| s.c_int()), - contraction_dim.as_ptr(), - contraction_dim.len_i32(), - if use_fast_accum { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_scaled_mm_v2_out>( - &self, - out: &Tensor, - mat2: &Tensor, - scale_a: &[T], - recipe_a: impl IntList, - swizzle_a: impl IntList, - scale_b: &[T], - recipe_b: impl IntList, - swizzle_b: impl IntList, - bias: Option, - out_dtype: impl Into>, - contraction_dim: impl IntList, - use_fast_accum: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__scaled_mm_v2_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mat2.c_tensor, - ptr_list(scale_a).as_ptr(), - scale_a.len() as i32, - recipe_a.as_ptr(), - recipe_a.len_i32(), - swizzle_a.as_ptr(), - swizzle_a.len_i32(), - ptr_list(scale_b).as_ptr(), - scale_b.len() as i32, - recipe_b.as_ptr(), - recipe_b.len_i32(), - swizzle_b.as_ptr(), - swizzle_b.len_i32(), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - out_dtype.into().map_or(-1, |s| s.c_int()), - contraction_dim.as_ptr(), - contraction_dim.len_i32(), - if use_fast_accum { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_scatter_reduce( - &self, - dim: i64, - index: &Tensor, - src: &Tensor, - reduce: &str, - include_self: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__scatter_reduce( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - src.c_tensor, - reduce.as_ptr(), - reduce.len() as i32, - if include_self { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_scatter_reduce_( - &mut self, - dim: i64, - index: &Tensor, - src: &Tensor, - reduce: &str, - include_self: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__scatter_reduce_( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - src.c_tensor, - reduce.as_ptr(), - reduce.len() as i32, - if include_self { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_scatter_reduce_two_out( - &self, - out: &Tensor, - dim: i64, - index: &Tensor, - src: &Tensor, - reduce: &str, - include_self: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__scatter_reduce_two_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - index.c_tensor, - src.c_tensor, - reduce.as_ptr(), - reduce.len() as i32, - if include_self { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_segment_reduce_backward, S: Into>( - grad: &Tensor, - output: &Tensor, - data: &Tensor, - reduce: &str, - lengths: Option, - offsets: Option, - axis: i64, - initial: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__segment_reduce_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - output.c_tensor, - data.c_tensor, - reduce.as_ptr(), - reduce.len() as i32, - lengths.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - offsets.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - axis, - initial.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_segment_reduce_backward_out, S: Into>( - out: &Tensor, - grad: &Tensor, - output: &Tensor, - data: &Tensor, - reduce: &str, - lengths: Option, - offsets: Option, - axis: i64, - initial: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__segment_reduce_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad.c_tensor, - output.c_tensor, - data.c_tensor, - reduce.as_ptr(), - reduce.len() as i32, - lengths.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - offsets.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - axis, - initial.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_shape_as_tensor(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__shape_as_tensor(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_slow_conv2d_backward( - &self, - grad_input: &Tensor, - grad_weight: &Tensor, - grad_bias: &Tensor, - grad_output: &Tensor, - weight: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__slow_conv2d_backward( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_weight.c_tensor, - grad_bias.c_tensor, - grad_output.c_tensor, - self.c_tensor, - weight.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32() - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_sobol_engine_draw( - quasi: &Tensor, - n: i64, - sobolstate: &Tensor, - dimension: i64, - num_generated: i64, - dtype: impl Into>, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__sobol_engine_draw( - c_tensors.as_mut_ptr(), - quasi.c_tensor, - n, - sobolstate.c_tensor, - dimension, - num_generated, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_sobol_engine_ff_( - &mut self, - n: i64, - sobolstate: &Tensor, - dimension: i64, - num_generated: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sobol_engine_ff_( - c_tensors.as_mut_ptr(), - self.c_tensor, - n, - sobolstate.c_tensor, - dimension, - num_generated - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sobol_engine_initialize_state_( - &mut self, - dimension: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sobol_engine_initialize_state_( - c_tensors.as_mut_ptr(), - self.c_tensor, - dimension - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sobol_engine_scramble_( - &mut self, - ltm: &Tensor, - dimension: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sobol_engine_scramble_( - c_tensors.as_mut_ptr(), - self.c_tensor, - ltm.c_tensor, - dimension - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_softmax(&self, dim: i64, half_to_float: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__softmax( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - if half_to_float { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_softmax_backward_data( - grad_output: &Tensor, - output: &Tensor, - dim: i64, - input_dtype: Kind, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__softmax_backward_data( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - output.c_tensor, - dim, - input_dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_softmax_backward_data_out( - grad_input: &Tensor, - grad_output: &Tensor, - output: &Tensor, - dim: i64, - input_dtype: Kind, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__softmax_backward_data_out( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - output.c_tensor, - dim, - input_dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_softmax_out( - &self, - out: &Tensor, - dim: i64, - half_to_float: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__softmax_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - if half_to_float { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_addmm( - &self, - mat1: &Tensor, - mat2: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_addmm( - c_tensors.as_mut_ptr(), - self.c_tensor, - mat1.c_tensor, - mat2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_addmm_out( - &self, - out: &Tensor, - mat1: &Tensor, - mat2: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_addmm_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mat1.c_tensor, - mat2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_broadcast_to(&self, size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_broadcast_to( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_broadcast_to_copy( - &self, - size: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_broadcast_to_copy( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_broadcast_to_copy_out( - &self, - out: &Tensor, - size: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_broadcast_to_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_bsc_tensor_unsafe( - ccol_indices: &Tensor, - row_indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_bsc_tensor_unsafe( - c_tensors.as_mut_ptr(), - ccol_indices.c_tensor, - row_indices.c_tensor, - values.c_tensor, - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_bsr_tensor_unsafe( - crow_indices: &Tensor, - col_indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_bsr_tensor_unsafe( - c_tensors.as_mut_ptr(), - crow_indices.c_tensor, - col_indices.c_tensor, - values.c_tensor, - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_compressed_tensor_unsafe( - compressed_indices: &Tensor, - plain_indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_compressed_tensor_unsafe( - c_tensors.as_mut_ptr(), - compressed_indices.c_tensor, - plain_indices.c_tensor, - values.c_tensor, - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_compressed_tensor_with_dims( - nnz: i64, - dense_dim: i64, - size: impl IntList, - blocksize: impl IntList, - index_dtype: Kind, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_compressed_tensor_with_dims( - c_tensors.as_mut_ptr(), - nnz, - dense_dim, - size.as_ptr(), - size.len_i32(), - blocksize.as_ptr(), - blocksize.len_i32(), - index_dtype.c_int(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_coo_tensor_unsafe( - indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - is_coalesced: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_coo_tensor_unsafe( - c_tensors.as_mut_ptr(), - indices.c_tensor, - values.c_tensor, - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int(), - if is_coalesced { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_coo_tensor_with_dims( - sparse_dim: i64, - dense_dim: i64, - size: impl IntList, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_coo_tensor_with_dims( - c_tensors.as_mut_ptr(), - sparse_dim, - dense_dim, - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_coo_tensor_with_dims_and_tensors( - sparse_dim: i64, - dense_dim: i64, - size: impl IntList, - indices: &Tensor, - values: &Tensor, - options: (Kind, Device), - is_coalesced: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_coo_tensor_with_dims_and_tensors( - c_tensors.as_mut_ptr(), - sparse_dim, - dense_dim, - size.as_ptr(), - size.len_i32(), - indices.c_tensor, - values.c_tensor, - options.0.c_int(), - options.1.c_int(), - if is_coalesced { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_coo_tensor_with_dims_and_tensors_out( - out: &Tensor, - sparse_dim: i64, - dense_dim: i64, - size: impl IntList, - indices: &Tensor, - values: &Tensor, - is_coalesced: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_coo_tensor_with_dims_and_tensors_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - sparse_dim, - dense_dim, - size.as_ptr(), - size.len_i32(), - indices.c_tensor, - values.c_tensor, - if is_coalesced { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_coo_tensor_with_dims_out( - out: &Tensor, - sparse_dim: i64, - dense_dim: i64, - size: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_coo_tensor_with_dims_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - sparse_dim, - dense_dim, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_csc_tensor_unsafe( - ccol_indices: &Tensor, - row_indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_csc_tensor_unsafe( - c_tensors.as_mut_ptr(), - ccol_indices.c_tensor, - row_indices.c_tensor, - values.c_tensor, - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_csr_prod( - &self, - dim: impl IntList, - keepdim: bool, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_csr_prod( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 }, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_csr_prod_dim_dtype_out( - &self, - out: &Tensor, - dim: impl IntList, - keepdim: bool, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_csr_prod_dim_dtype_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 }, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_csr_sum( - &self, - dim: impl IntList, - keepdim: bool, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_csr_sum( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 }, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_csr_sum_dim_dtype_out( - &self, - out: &Tensor, - dim: impl IntList, - keepdim: bool, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_csr_sum_dim_dtype_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 }, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_csr_tensor_unsafe( - crow_indices: &Tensor, - col_indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_csr_tensor_unsafe( - c_tensors.as_mut_ptr(), - crow_indices.c_tensor, - col_indices.c_tensor, - values.c_tensor, - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_log_softmax( - &self, - dim: i64, - half_to_float: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_log_softmax( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - if half_to_float { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_log_softmax_backward_data( - &self, - grad_output: &Tensor, - output: &Tensor, - dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_log_softmax_backward_data( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - output.c_tensor, - dim, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_log_softmax_backward_data_out( - &self, - out: &Tensor, - grad_output: &Tensor, - output: &Tensor, - dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_log_softmax_backward_data_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad_output.c_tensor, - output.c_tensor, - dim, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_log_softmax_int( - &self, - dim: i64, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_log_softmax_int( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_log_softmax_out( - &self, - out: &Tensor, - dim: i64, - half_to_float: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_log_softmax_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - if half_to_float { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_mask_projection( - &self, - mask: &Tensor, - accumulate_matches: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_mask_projection( - c_tensors.as_mut_ptr(), - self.c_tensor, - mask.c_tensor, - if accumulate_matches { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_mask_projection_out( - &self, - out: &Tensor, - mask: &Tensor, - accumulate_matches: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_mask_projection_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mask.c_tensor, - if accumulate_matches { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_mm(sparse: &Tensor, dense: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_mm(c_tensors.as_mut_ptr(), sparse.c_tensor, dense.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_mm_reduce( - sparse: &Tensor, - dense: &Tensor, - reduce: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_mm_reduce( - c_tensors.as_mut_ptr(), - sparse.c_tensor, - dense.c_tensor, - reduce.as_ptr(), - reduce.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_mm_reduce_impl( - &self, - other: &Tensor, - reduce: &str, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__sparse_mm_reduce_impl( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor, - reduce.as_ptr(), - reduce.len() as i32 - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_sparse_semi_structured_apply( - &self, - thread_masks: &Tensor, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__sparse_semi_structured_apply( - c_tensors.as_mut_ptr(), - self.c_tensor, - thread_masks.c_tensor - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_sparse_semi_structured_apply_dense( - &self, - thread_masks: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_semi_structured_apply_dense( - c_tensors.as_mut_ptr(), - self.c_tensor, - thread_masks.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_semi_structured_linear>( - &self, - weight: &Tensor, - meta: &Tensor, - bias: Option, - activation: &str, - out_dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_semi_structured_linear( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - meta.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - activation.as_ptr(), - activation.len() as i32, - out_dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_semi_structured_mm( - mat1: &Tensor, - mat1_meta: &Tensor, - mat2: &Tensor, - out_dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_semi_structured_mm( - c_tensors.as_mut_ptr(), - mat1.c_tensor, - mat1_meta.c_tensor, - mat2.c_tensor, - out_dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_semi_structured_tile( - &self, - algorithm: &str, - use_cutlass: bool, - ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 5]; - unsafe_torch_err!(atg__sparse_semi_structured_tile( - c_tensors.as_mut_ptr(), - self.c_tensor, - algorithm.as_ptr(), - algorithm.len() as i32, - if use_cutlass { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - Tensor { c_tensor: c_tensors[4] }, - )) - } - - pub fn f_internal_sparse_softmax( - &self, - dim: i64, - half_to_float: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_softmax( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - if half_to_float { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_softmax_backward_data( - &self, - grad_output: &Tensor, - output: &Tensor, - dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_softmax_backward_data( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - output.c_tensor, - dim, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_softmax_backward_data_out( - &self, - out: &Tensor, - grad_output: &Tensor, - output: &Tensor, - dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_softmax_backward_data_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad_output.c_tensor, - output.c_tensor, - dim, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_softmax_int( - &self, - dim: i64, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_softmax_int( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_softmax_out( - &self, - out: &Tensor, - dim: i64, - half_to_float: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_softmax_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - if half_to_float { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_sparse_matmul(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_sparse_matmul( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_sparse_matmul_out( - &self, - out: &Tensor, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_sparse_matmul_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_sum(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_sum(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_sum_backward( - &self, - grad: &Tensor, - dim: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_sum_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_sum_backward_out( - &self, - out: &Tensor, - grad: &Tensor, - dim: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_sum_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_sum_dim(&self, dim: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_sum_dim( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_sum_dim_dtype( - &self, - dim: impl IntList, - dtype: Kind, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_sum_dim_dtype( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_sum_dim_out( - &self, - out: &Tensor, - dim: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_sum_dim_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_sparse_sum_dtype(&self, dtype: Kind) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__sparse_sum_dtype( - c_tensors.as_mut_ptr(), - self.c_tensor, - dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_spdiags( - diagonals: &Tensor, - offsets: &Tensor, - shape: impl IntList, - layout: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__spdiags( - c_tensors.as_mut_ptr(), - diagonals.c_tensor, - offsets.c_tensor, - shape.as_ptr(), - shape.len_i32(), - layout.map_or(-1, |s| s.to_i8()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_spdiags_out( - out: &Tensor, - diagonals: &Tensor, - offsets: &Tensor, - shape: impl IntList, - layout: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__spdiags_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - diagonals.c_tensor, - offsets.c_tensor, - shape.as_ptr(), - shape.len_i32(), - layout.map_or(-1, |s| s.to_i8()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_spsolve(a: &Tensor, b: &Tensor, left: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__spsolve( - c_tensors.as_mut_ptr(), - a.c_tensor, - b.c_tensor, - if left { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_stack>( - tensors: &[T], - dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__stack( - c_tensors.as_mut_ptr(), - ptr_list(tensors).as_ptr(), - tensors.len() as i32, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_stack_out>( - out: &Tensor, - tensors: &[T], - dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__stack_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - ptr_list(tensors).as_ptr(), - tensors.len() as i32, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_standard_gamma(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__standard_gamma(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_standard_gamma_grad(&self, output: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__standard_gamma_grad( - c_tensors.as_mut_ptr(), - self.c_tensor, - output.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_standard_gamma_grad_out( - &self, - out: &Tensor, - output: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__standard_gamma_grad_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - output.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_standard_gamma_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__standard_gamma_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_test_ambiguous_defaults( - dummy: &Tensor, - a: i64, - b: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__test_ambiguous_defaults( - c_tensors.as_mut_ptr(), - dummy.c_tensor, - a, - b - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_test_ambiguous_defaults_b( - dummy: &Tensor, - a: i64, - b: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__test_ambiguous_defaults_b( - c_tensors.as_mut_ptr(), - dummy.c_tensor, - a, - b.as_ptr(), - b.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_test_autograd_multiple_dispatch(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__test_autograd_multiple_dispatch( - c_tensors.as_mut_ptr(), - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_test_autograd_multiple_dispatch_fullcoverage_out( - &self, - out: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__test_autograd_multiple_dispatch_fullcoverage_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_test_autograd_multiple_dispatch_ntonly( - &self, - b: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__test_autograd_multiple_dispatch_ntonly( - c_tensors.as_mut_ptr(), - self.c_tensor, - if b { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_test_autograd_multiple_dispatch_view(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__test_autograd_multiple_dispatch_view( - c_tensors.as_mut_ptr(), - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_test_autograd_multiple_dispatch_view_copy(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__test_autograd_multiple_dispatch_view_copy( - c_tensors.as_mut_ptr(), - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_test_autograd_multiple_dispatch_view_copy_out( - &self, - out: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__test_autograd_multiple_dispatch_view_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_test_check_tensor(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__test_check_tensor(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_test_functorch_fallback(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__test_functorch_fallback( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_test_functorch_fallback_out( - &self, - out: &Tensor, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__test_functorch_fallback_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_test_optional_filled_intlist( - values: &Tensor, - addends: impl IntListOption, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__test_optional_filled_intlist( - c_tensors.as_mut_ptr(), - values.c_tensor, - addends.as_ptr(), - addends.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_test_optional_filled_intlist_out( - out: &Tensor, - values: &Tensor, - addends: impl IntListOption, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__test_optional_filled_intlist_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - values.c_tensor, - addends.as_ptr(), - addends.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_test_optional_floatlist( - values: &Tensor, - addends: impl DoubleList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__test_optional_floatlist( - c_tensors.as_mut_ptr(), - values.c_tensor, - addends.as_ptr(), - addends.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_test_optional_floatlist_out( - out: &Tensor, - values: &Tensor, - addends: impl DoubleList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__test_optional_floatlist_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - values.c_tensor, - addends.as_ptr(), - addends.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_test_optional_intlist( - values: &Tensor, - addends: impl IntListOption, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__test_optional_intlist( - c_tensors.as_mut_ptr(), - values.c_tensor, - addends.as_ptr(), - addends.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_test_optional_intlist_out( - out: &Tensor, - values: &Tensor, - addends: impl IntListOption, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__test_optional_intlist_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - values.c_tensor, - addends.as_ptr(), - addends.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_test_parallel_materialize( - &self, - num_parallel: i64, - skip_first: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__test_parallel_materialize( - c_tensors.as_mut_ptr(), - self.c_tensor, - num_parallel, - if skip_first { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_test_serialization_subcmul( - &self, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__test_serialization_subcmul( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_test_string_default( - dummy: &Tensor, - a: &str, - b: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__test_string_default( - c_tensors.as_mut_ptr(), - dummy.c_tensor, - a.as_ptr(), - a.len() as i32, - b.as_ptr(), - b.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_test_warn_in_autograd(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__test_warn_in_autograd(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_test_warn_in_autograd_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__test_warn_in_autograd_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_to_copy( - &self, - options: (Kind, Device), - non_blocking: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__to_copy( - c_tensors.as_mut_ptr(), - self.c_tensor, - options.0.c_int(), - options.1.c_int(), - if non_blocking { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_to_copy_out( - &self, - out: &Tensor, - non_blocking: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__to_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - if non_blocking { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_to_cpu>(tensors: &[T]) -> Result, TchError> { - let c_tensors = - unsafe_torch_err!(atg__to_cpu(ptr_list(tensors).as_ptr(), tensors.len() as i32)); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_internal_to_dense( - &self, - dtype: impl Into>, - masked_grad: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__to_dense( - c_tensors.as_mut_ptr(), - self.c_tensor, - dtype.into().map_or(-1, |s| s.c_int()), - if masked_grad { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_to_dense_out( - &self, - out: &Tensor, - dtype: impl Into>, - masked_grad: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__to_dense_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dtype.into().map_or(-1, |s| s.c_int()), - if masked_grad { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_to_sparse( - &self, - layout: Option, - blocksize: impl IntListOption, - dense_dim: impl Into>, - ) -> Result { - let dense_dim = dense_dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__to_sparse( - c_tensors.as_mut_ptr(), - self.c_tensor, - layout.map_or(-1, |s| s.to_i8()), - blocksize.as_ptr(), - blocksize.len_i32(), - dense_dim.unwrap_or(0i64), - dense_dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_to_sparse_bsc( - &self, - blocksize: impl IntList, - dense_dim: impl Into>, - ) -> Result { - let dense_dim = dense_dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__to_sparse_bsc( - c_tensors.as_mut_ptr(), - self.c_tensor, - blocksize.as_ptr(), - blocksize.len_i32(), - dense_dim.unwrap_or(0i64), - dense_dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_to_sparse_bsc_out( - &self, - out: &Tensor, - blocksize: impl IntList, - dense_dim: impl Into>, - ) -> Result { - let dense_dim = dense_dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__to_sparse_bsc_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - blocksize.as_ptr(), - blocksize.len_i32(), - dense_dim.unwrap_or(0i64), - dense_dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_to_sparse_bsr( - &self, - blocksize: impl IntList, - dense_dim: impl Into>, - ) -> Result { - let dense_dim = dense_dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__to_sparse_bsr( - c_tensors.as_mut_ptr(), - self.c_tensor, - blocksize.as_ptr(), - blocksize.len_i32(), - dense_dim.unwrap_or(0i64), - dense_dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_to_sparse_bsr_out( - &self, - out: &Tensor, - blocksize: impl IntList, - dense_dim: impl Into>, - ) -> Result { - let dense_dim = dense_dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__to_sparse_bsr_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - blocksize.as_ptr(), - blocksize.len_i32(), - dense_dim.unwrap_or(0i64), - dense_dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_to_sparse_csc( - &self, - dense_dim: impl Into>, - ) -> Result { - let dense_dim = dense_dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__to_sparse_csc( - c_tensors.as_mut_ptr(), - self.c_tensor, - dense_dim.unwrap_or(0i64), - dense_dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_to_sparse_csc_out( - &self, - out: &Tensor, - dense_dim: impl Into>, - ) -> Result { - let dense_dim = dense_dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__to_sparse_csc_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dense_dim.unwrap_or(0i64), - dense_dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_to_sparse_csr( - &self, - dense_dim: impl Into>, - ) -> Result { - let dense_dim = dense_dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__to_sparse_csr( - c_tensors.as_mut_ptr(), - self.c_tensor, - dense_dim.unwrap_or(0i64), - dense_dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_to_sparse_csr_out( - &self, - out: &Tensor, - dense_dim: impl Into>, - ) -> Result { - let dense_dim = dense_dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__to_sparse_csr_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dense_dim.unwrap_or(0i64), - dense_dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_to_sparse_out( - &self, - out: &Tensor, - layout: Option, - blocksize: impl IntListOption, - dense_dim: impl Into>, - ) -> Result { - let dense_dim = dense_dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__to_sparse_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - layout.map_or(-1, |s| s.to_i8()), - blocksize.as_ptr(), - blocksize.len_i32(), - dense_dim.unwrap_or(0i64), - dense_dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_to_sparse_semi_structured( - dense: &Tensor, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__to_sparse_semi_structured(c_tensors.as_mut_ptr(), dense.c_tensor)); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_to_sparse_sparse_dim(&self, sparse_dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__to_sparse_sparse_dim( - c_tensors.as_mut_ptr(), - self.c_tensor, - sparse_dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_to_sparse_sparse_dim_out( - &self, - out: &Tensor, - sparse_dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__to_sparse_sparse_dim_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - sparse_dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_transform_bias_rescale_qkv( - qkv: &Tensor, - qkv_bias: &Tensor, - num_heads: i64, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__transform_bias_rescale_qkv( - c_tensors.as_mut_ptr(), - qkv.c_tensor, - qkv_bias.c_tensor, - num_heads - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_transform_bias_rescale_qkv_out( - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - qkv: &Tensor, - qkv_bias: &Tensor, - num_heads: i64, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__transform_bias_rescale_qkv_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - out2.c_tensor, - qkv.c_tensor, - qkv_bias.c_tensor, - num_heads - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_transformer_encoder_layer_fwd>( - src: &Tensor, - embed_dim: i64, - num_heads: i64, - qkv_weight: &Tensor, - qkv_bias: &Tensor, - proj_weight: &Tensor, - proj_bias: &Tensor, - use_gelu: bool, - norm_first: bool, - eps: f64, - norm_weight_1: &Tensor, - norm_bias_1: &Tensor, - norm_weight_2: &Tensor, - norm_bias_2: &Tensor, - ffn_weight_1: &Tensor, - ffn_bias_1: &Tensor, - ffn_weight_2: &Tensor, - ffn_bias_2: &Tensor, - mask: Option, - mask_type: impl Into>, - ) -> Result { - let mask_type = mask_type.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__transformer_encoder_layer_fwd( - c_tensors.as_mut_ptr(), - src.c_tensor, - embed_dim, - num_heads, - qkv_weight.c_tensor, - qkv_bias.c_tensor, - proj_weight.c_tensor, - proj_bias.c_tensor, - if use_gelu { 1 } else { 0 }, - if norm_first { 1 } else { 0 }, - eps, - norm_weight_1.c_tensor, - norm_bias_1.c_tensor, - norm_weight_2.c_tensor, - norm_bias_2.c_tensor, - ffn_weight_1.c_tensor, - ffn_bias_1.c_tensor, - ffn_weight_2.c_tensor, - ffn_bias_2.c_tensor, - mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - mask_type.unwrap_or(0i64), - mask_type.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_transformer_encoder_layer_fwd_out>( - out: &Tensor, - src: &Tensor, - embed_dim: i64, - num_heads: i64, - qkv_weight: &Tensor, - qkv_bias: &Tensor, - proj_weight: &Tensor, - proj_bias: &Tensor, - use_gelu: bool, - norm_first: bool, - eps: f64, - norm_weight_1: &Tensor, - norm_bias_1: &Tensor, - norm_weight_2: &Tensor, - norm_bias_2: &Tensor, - ffn_weight_1: &Tensor, - ffn_bias_1: &Tensor, - ffn_weight_2: &Tensor, - ffn_bias_2: &Tensor, - mask: Option, - mask_type: impl Into>, - ) -> Result { - let mask_type = mask_type.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__transformer_encoder_layer_fwd_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - src.c_tensor, - embed_dim, - num_heads, - qkv_weight.c_tensor, - qkv_bias.c_tensor, - proj_weight.c_tensor, - proj_bias.c_tensor, - if use_gelu { 1 } else { 0 }, - if norm_first { 1 } else { 0 }, - eps, - norm_weight_1.c_tensor, - norm_bias_1.c_tensor, - norm_weight_2.c_tensor, - norm_bias_2.c_tensor, - ffn_weight_1.c_tensor, - ffn_bias_1.c_tensor, - ffn_weight_2.c_tensor, - ffn_bias_2.c_tensor, - mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - mask_type.unwrap_or(0i64), - mask_type.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_trilinear( - i1: &Tensor, - i2: &Tensor, - i3: &Tensor, - expand1: impl IntList, - expand2: impl IntList, - expand3: impl IntList, - sumdim: impl IntList, - unroll_dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__trilinear( - c_tensors.as_mut_ptr(), - i1.c_tensor, - i2.c_tensor, - i3.c_tensor, - expand1.as_ptr(), - expand1.len_i32(), - expand2.as_ptr(), - expand2.len_i32(), - expand3.as_ptr(), - expand3.len_i32(), - sumdim.as_ptr(), - sumdim.len_i32(), - unroll_dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_trilinear_out( - out: &Tensor, - i1: &Tensor, - i2: &Tensor, - i3: &Tensor, - expand1: impl IntList, - expand2: impl IntList, - expand3: impl IntList, - sumdim: impl IntList, - unroll_dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__trilinear_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - i1.c_tensor, - i2.c_tensor, - i3.c_tensor, - expand1.as_ptr(), - expand1.len_i32(), - expand2.as_ptr(), - expand2.len_i32(), - expand3.as_ptr(), - expand3.len_i32(), - sumdim.as_ptr(), - sumdim.len_i32(), - unroll_dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_triton_multi_head_attention>( - query: &Tensor, - key: &Tensor, - value: &Tensor, - embed_dim: i64, - num_head: i64, - qkv_weight: &Tensor, - qkv_bias: &Tensor, - proj_weight: &Tensor, - proj_bias: &Tensor, - mask: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__triton_multi_head_attention( - c_tensors.as_mut_ptr(), - query.c_tensor, - key.c_tensor, - value.c_tensor, - embed_dim, - num_head, - qkv_weight.c_tensor, - qkv_bias.c_tensor, - proj_weight.c_tensor, - proj_bias.c_tensor, - mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_triton_multi_head_attention_out>( - out: &Tensor, - query: &Tensor, - key: &Tensor, - value: &Tensor, - embed_dim: i64, - num_head: i64, - qkv_weight: &Tensor, - qkv_bias: &Tensor, - proj_weight: &Tensor, - proj_bias: &Tensor, - mask: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__triton_multi_head_attention_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - query.c_tensor, - key.c_tensor, - value.c_tensor, - embed_dim, - num_head, - qkv_weight.c_tensor, - qkv_bias.c_tensor, - proj_weight.c_tensor, - proj_bias.c_tensor, - mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_triton_scaled_dot_attention( - q: &Tensor, - k: &Tensor, - v: &Tensor, - dropout_p: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__triton_scaled_dot_attention( - c_tensors.as_mut_ptr(), - q.c_tensor, - k.c_tensor, - v.c_tensor, - dropout_p - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_triton_scaled_dot_attention_out( - out: &Tensor, - q: &Tensor, - k: &Tensor, - v: &Tensor, - dropout_p: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__triton_scaled_dot_attention_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - q.c_tensor, - k.c_tensor, - v.c_tensor, - dropout_p - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_unique( - &self, - sorted: bool, - return_inverse: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__unique( - c_tensors.as_mut_ptr(), - self.c_tensor, - if sorted { 1 } else { 0 }, - if return_inverse { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_unique2( - &self, - sorted: bool, - return_inverse: bool, - return_counts: bool, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__unique2( - c_tensors.as_mut_ptr(), - self.c_tensor, - if sorted { 1 } else { 0 }, - if return_inverse { 1 } else { 0 }, - if return_counts { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_unique2_out( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - sorted: bool, - return_inverse: bool, - return_counts: bool, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg__unique2_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - out2.c_tensor, - self.c_tensor, - if sorted { 1 } else { 0 }, - if return_inverse { 1 } else { 0 }, - if return_counts { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_internal_unique_out( - &self, - out0: &Tensor, - out1: &Tensor, - sorted: bool, - return_inverse: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__unique_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - self.c_tensor, - if sorted { 1 } else { 0 }, - if return_inverse { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_unpack_dual(dual: &Tensor, level: i64) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__unpack_dual(c_tensors.as_mut_ptr(), dual.c_tensor, level)); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_unsafe_index>( - &self, - indices: &[Option], - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__unsafe_index( - c_tensors.as_mut_ptr(), - self.c_tensor, - ptr_list_opt(indices).as_ptr(), - indices.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_unsafe_index_put>( - &self, - indices: &[Option], - values: &Tensor, - accumulate: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__unsafe_index_put( - c_tensors.as_mut_ptr(), - self.c_tensor, - ptr_list_opt(indices).as_ptr(), - indices.len() as i32, - values.c_tensor, - if accumulate { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_unsafe_masked_index, S: Into>( - &self, - mask: &Tensor, - indices: &[Option], - fill: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__unsafe_masked_index( - c_tensors.as_mut_ptr(), - self.c_tensor, - mask.c_tensor, - ptr_list_opt(indices).as_ptr(), - indices.len() as i32, - fill.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_unsafe_masked_index_put_accumulate>( - &self, - mask: &Tensor, - indices: &[Option], - values: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__unsafe_masked_index_put_accumulate( - c_tensors.as_mut_ptr(), - self.c_tensor, - mask.c_tensor, - ptr_list_opt(indices).as_ptr(), - indices.len() as i32, - values.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_unsafe_view(&self, size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__unsafe_view( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_unsafe_view_out( - &self, - out: &Tensor, - size: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__unsafe_view_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_bicubic2d_aa( - &self, - output_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_bicubic2d_aa( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_bicubic2d_aa_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_bicubic2d_aa_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_bicubic2d_aa_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_bicubic2d_aa_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_bicubic2d_aa_out( - &self, - out: &Tensor, - output_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_bicubic2d_aa_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_bicubic2d_aa_vec( - &self, - output_size: impl IntListOption, - align_corners: bool, - scale_factors: impl DoubleList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_bicubic2d_aa_vec( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - if align_corners { 1 } else { 0 }, - scale_factors.as_ptr(), - scale_factors.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_bilinear2d_aa( - &self, - output_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_bilinear2d_aa( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_bilinear2d_aa_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_bilinear2d_aa_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_bilinear2d_aa_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_bilinear2d_aa_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_bilinear2d_aa_out( - &self, - out: &Tensor, - output_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_bilinear2d_aa_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_bilinear2d_aa_vec( - &self, - output_size: impl IntListOption, - align_corners: bool, - scale_factors: impl DoubleList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_bilinear2d_aa_vec( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - if align_corners { 1 } else { 0 }, - scale_factors.as_ptr(), - scale_factors.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_nearest_exact1d( - &self, - output_size: impl IntList, - scales: impl Into>, - ) -> Result { - let scales = scales.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_nearest_exact1d( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - scales.unwrap_or(std::f64::NAN), - scales.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_nearest_exact1d_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales: impl Into>, - ) -> Result { - let scales = scales.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_nearest_exact1d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - scales.unwrap_or(std::f64::NAN), - scales.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_nearest_exact1d_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales: impl Into>, - ) -> Result { - let scales = scales.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_nearest_exact1d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - scales.unwrap_or(std::f64::NAN), - scales.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_nearest_exact1d_out( - &self, - out: &Tensor, - output_size: impl IntList, - scales: impl Into>, - ) -> Result { - let scales = scales.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_nearest_exact1d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - scales.unwrap_or(std::f64::NAN), - scales.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_nearest_exact1d_vec( - &self, - output_size: impl IntListOption, - scale_factors: impl DoubleList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_nearest_exact1d_vec( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - scale_factors.as_ptr(), - scale_factors.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_nearest_exact2d( - &self, - output_size: impl IntList, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_nearest_exact2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_nearest_exact2d_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_nearest_exact2d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_nearest_exact2d_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_nearest_exact2d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_nearest_exact2d_out( - &self, - out: &Tensor, - output_size: impl IntList, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_nearest_exact2d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_nearest_exact2d_vec( - &self, - output_size: impl IntListOption, - scale_factors: impl DoubleList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_nearest_exact2d_vec( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - scale_factors.as_ptr(), - scale_factors.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_nearest_exact3d( - &self, - output_size: impl IntList, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_d = scales_d.into(); - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_nearest_exact3d( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - scales_d.unwrap_or(std::f64::NAN), - scales_d.is_none() as i8, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_nearest_exact3d_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_d = scales_d.into(); - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_nearest_exact3d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - scales_d.unwrap_or(std::f64::NAN), - scales_d.is_none() as i8, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_nearest_exact3d_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_d = scales_d.into(); - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_nearest_exact3d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - scales_d.unwrap_or(std::f64::NAN), - scales_d.is_none() as i8, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_nearest_exact3d_out( - &self, - out: &Tensor, - output_size: impl IntList, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_d = scales_d.into(); - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_nearest_exact3d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - scales_d.unwrap_or(std::f64::NAN), - scales_d.is_none() as i8, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_upsample_nearest_exact3d_vec( - &self, - output_size: impl IntListOption, - scale_factors: impl DoubleList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__upsample_nearest_exact3d_vec( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - scale_factors.as_ptr(), - scale_factors.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_use_cudnn_ctc_loss( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: impl IntList, - target_lengths: impl IntList, - blank: i64, - ) -> Result { - let return_; - unsafe_torch_err!( - return_ = atg__use_cudnn_ctc_loss( - log_probs.c_tensor, - targets.c_tensor, - input_lengths.as_ptr(), - input_lengths.len_i32(), - target_lengths.as_ptr(), - target_lengths.len_i32(), - blank - ) - ); - Ok(return_ != 0) - } - - pub fn f_internal_use_cudnn_ctc_loss_tensor( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: &Tensor, - target_lengths: &Tensor, - blank: i64, - ) -> Result { - let return_; - unsafe_torch_err!( - return_ = atg__use_cudnn_ctc_loss_tensor( - log_probs.c_tensor, - targets.c_tensor, - input_lengths.c_tensor, - target_lengths.c_tensor, - blank - ) - ); - Ok(return_ != 0) - } - - pub fn f_internal_use_cudnn_rnn_flatten_weight() -> Result { - let return_; - unsafe_torch_err!(return_ = atg__use_cudnn_rnn_flatten_weight()); - Ok(return_ != 0) - } - - pub fn f_internal_use_miopen_ctc_loss( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: impl IntList, - target_lengths: impl IntList, - blank: i64, - ) -> Result { - let return_; - unsafe_torch_err!( - return_ = atg__use_miopen_ctc_loss( - log_probs.c_tensor, - targets.c_tensor, - input_lengths.as_ptr(), - input_lengths.len_i32(), - target_lengths.as_ptr(), - target_lengths.len_i32(), - blank - ) - ); - Ok(return_ != 0) - } - - pub fn f_internal_use_miopen_ctc_loss_tensor( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: &Tensor, - target_lengths: &Tensor, - blank: i64, - ) -> Result { - let return_; - unsafe_torch_err!( - return_ = atg__use_miopen_ctc_loss_tensor( - log_probs.c_tensor, - targets.c_tensor, - input_lengths.c_tensor, - target_lengths.c_tensor, - blank - ) - ); - Ok(return_ != 0) - } - - pub fn f_internal_validate_compressed_sparse_indices( - is_crow: bool, - compressed_idx: &Tensor, - plain_idx: &Tensor, - cdim: i64, - dim: i64, - nnz: i64, - ) -> Result<(), TchError> { - unsafe_torch_err!(atg__validate_compressed_sparse_indices( - if is_crow { 1 } else { 0 }, - compressed_idx.c_tensor, - plain_idx.c_tensor, - cdim, - dim, - nnz - )); - Ok(()) - } - - pub fn f_internal_validate_sparse_bsc_tensor_args( - ccol_indices: &Tensor, - row_indices: &Tensor, - values: &Tensor, - size: impl IntList, - check_pinning: bool, - ) -> Result<(), TchError> { - unsafe_torch_err!(atg__validate_sparse_bsc_tensor_args( - ccol_indices.c_tensor, - row_indices.c_tensor, - values.c_tensor, - size.as_ptr(), - size.len_i32(), - if check_pinning { 1 } else { 0 } - )); - Ok(()) - } - - pub fn f_internal_validate_sparse_bsr_tensor_args( - crow_indices: &Tensor, - col_indices: &Tensor, - values: &Tensor, - size: impl IntList, - check_pinning: bool, - ) -> Result<(), TchError> { - unsafe_torch_err!(atg__validate_sparse_bsr_tensor_args( - crow_indices.c_tensor, - col_indices.c_tensor, - values.c_tensor, - size.as_ptr(), - size.len_i32(), - if check_pinning { 1 } else { 0 } - )); - Ok(()) - } - - pub fn f_internal_validate_sparse_compressed_tensor_args( - compressed_indices: &Tensor, - plain_indices: &Tensor, - values: &Tensor, - size: impl IntList, - layout: Layout, - check_pinning: bool, - ) -> Result<(), TchError> { - unsafe_torch_err!(atg__validate_sparse_compressed_tensor_args( - compressed_indices.c_tensor, - plain_indices.c_tensor, - values.c_tensor, - size.as_ptr(), - size.len_i32(), - layout.to_i8(), - if check_pinning { 1 } else { 0 } - )); - Ok(()) - } - - pub fn f_internal_validate_sparse_csc_tensor_args( - ccol_indices: &Tensor, - row_indices: &Tensor, - values: &Tensor, - size: impl IntList, - check_pinning: bool, - ) -> Result<(), TchError> { - unsafe_torch_err!(atg__validate_sparse_csc_tensor_args( - ccol_indices.c_tensor, - row_indices.c_tensor, - values.c_tensor, - size.as_ptr(), - size.len_i32(), - if check_pinning { 1 } else { 0 } - )); - Ok(()) - } - - pub fn f_internal_validate_sparse_csr_tensor_args( - crow_indices: &Tensor, - col_indices: &Tensor, - values: &Tensor, - size: impl IntList, - check_pinning: bool, - ) -> Result<(), TchError> { - unsafe_torch_err!(atg__validate_sparse_csr_tensor_args( - crow_indices.c_tensor, - col_indices.c_tensor, - values.c_tensor, - size.as_ptr(), - size.len_i32(), - if check_pinning { 1 } else { 0 } - )); - Ok(()) - } - - pub fn f_internal_values(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__values(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_values_copy(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__values_copy(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_values_copy_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__values_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_version(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg__version(self.c_tensor)); - Ok(return_) - } - - pub fn f_internal_weight_int4pack_mm( - &self, - mat2: &Tensor, - qgroupsize: i64, - qscaleandzeros: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__weight_int4pack_mm( - c_tensors.as_mut_ptr(), - self.c_tensor, - mat2.c_tensor, - qgroupsize, - qscaleandzeros.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_weight_int4pack_mm_for_cpu( - &self, - mat2: &Tensor, - qgroupsize: i64, - qscaleandzeros: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__weight_int4pack_mm_for_cpu( - c_tensors.as_mut_ptr(), - self.c_tensor, - mat2.c_tensor, - qgroupsize, - qscaleandzeros.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_weight_int4pack_mm_with_scales_and_zeros( - &self, - mat2: &Tensor, - qgroupsize: i64, - qscale: &Tensor, - qzeros: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__weight_int4pack_mm_with_scales_and_zeros( - c_tensors.as_mut_ptr(), - self.c_tensor, - mat2.c_tensor, - qgroupsize, - qscale.c_tensor, - qzeros.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_weight_int8pack_mm( - &self, - mat2: &Tensor, - scales: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__weight_int8pack_mm( - c_tensors.as_mut_ptr(), - self.c_tensor, - mat2.c_tensor, - scales.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_weight_norm(v: &Tensor, g: &Tensor, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__weight_norm(c_tensors.as_mut_ptr(), v.c_tensor, g.c_tensor, dim)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_weight_norm_differentiable_backward( - grad_w: &Tensor, - saved_v: &Tensor, - saved_g: &Tensor, - saved_norms: &Tensor, - dim: i64, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__weight_norm_differentiable_backward( - c_tensors.as_mut_ptr(), - grad_w.c_tensor, - saved_v.c_tensor, - saved_g.c_tensor, - saved_norms.c_tensor, - dim - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_weight_norm_interface( - v: &Tensor, - g: &Tensor, - dim: i64, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__weight_norm_interface( - c_tensors.as_mut_ptr(), - v.c_tensor, - g.c_tensor, - dim - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_weight_norm_interface_backward( - grad_w: &Tensor, - saved_v: &Tensor, - saved_g: &Tensor, - saved_norms: &Tensor, - dim: i64, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__weight_norm_interface_backward( - c_tensors.as_mut_ptr(), - grad_w.c_tensor, - saved_v.c_tensor, - saved_g.c_tensor, - saved_norms.c_tensor, - dim - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_weight_norm_interface_backward_out( - out0: &Tensor, - out1: &Tensor, - grad_w: &Tensor, - saved_v: &Tensor, - saved_g: &Tensor, - saved_norms: &Tensor, - dim: i64, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__weight_norm_interface_backward_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - grad_w.c_tensor, - saved_v.c_tensor, - saved_g.c_tensor, - saved_norms.c_tensor, - dim - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_weight_norm_interface_out( - out0: &Tensor, - out1: &Tensor, - v: &Tensor, - g: &Tensor, - dim: i64, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg__weight_norm_interface_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - v.c_tensor, - g.c_tensor, - dim - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_internal_wrapped_linear_prepack( - weight: &Tensor, - weight_scale: &Tensor, - weight_zero_point: &Tensor, - bias: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__wrapped_linear_prepack( - c_tensors.as_mut_ptr(), - weight.c_tensor, - weight_scale.c_tensor, - weight_zero_point.c_tensor, - bias.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_internal_wrapped_quantized_linear_prepacked( - &self, - input_scale: &Tensor, - input_zero_point: &Tensor, - packed_weight: &Tensor, - output_scale: &Tensor, - output_zero_point: &Tensor, - out_channel: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg__wrapped_quantized_linear_prepacked( - c_tensors.as_mut_ptr(), - self.c_tensor, - input_scale.c_tensor, - input_zero_point.c_tensor, - packed_weight.c_tensor, - output_scale.c_tensor, - output_zero_point.c_tensor, - out_channel - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_abs(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_abs(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_abs_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_abs_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_abs_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_abs_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_absolute(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_absolute(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_absolute_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_absolute_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_absolute_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_absolute_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_acos(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_acos(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_acos_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_acos_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_acos_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_acos_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_acosh(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_acosh(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_acosh_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_acosh_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_acosh_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_acosh_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_adaptive_avg_pool1d(&self, output_size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_adaptive_avg_pool1d( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_adaptive_avg_pool1d_out( - &self, - out: &Tensor, - output_size: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_adaptive_avg_pool1d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_adaptive_avg_pool2d(&self, output_size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_adaptive_avg_pool2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_adaptive_avg_pool2d_out( - &self, - out: &Tensor, - output_size: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_adaptive_avg_pool2d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_adaptive_avg_pool3d(&self, output_size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_adaptive_avg_pool3d( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_adaptive_avg_pool3d_backward( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_adaptive_avg_pool3d_backward( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_adaptive_avg_pool3d_out( - &self, - out: &Tensor, - output_size: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_adaptive_avg_pool3d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_adaptive_max_pool1d( - &self, - output_size: impl IntList, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_adaptive_max_pool1d( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32() - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_adaptive_max_pool2d( - &self, - output_size: impl IntList, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_adaptive_max_pool2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32() - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_adaptive_max_pool2d_backward( - &self, - grad_output: &Tensor, - indices: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_adaptive_max_pool2d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - indices.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_adaptive_max_pool2d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - indices: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_adaptive_max_pool2d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - indices.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_adaptive_max_pool2d_out( - &self, - out: &Tensor, - indices: &Tensor, - output_size: impl IntList, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_adaptive_max_pool2d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - indices.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32() - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_adaptive_max_pool3d( - &self, - output_size: impl IntList, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_adaptive_max_pool3d( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32() - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_adaptive_max_pool3d_backward( - &self, - grad_output: &Tensor, - indices: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_adaptive_max_pool3d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - indices.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_adaptive_max_pool3d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - indices: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_adaptive_max_pool3d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - indices.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_adaptive_max_pool3d_out( - &self, - out: &Tensor, - indices: &Tensor, - output_size: impl IntList, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_adaptive_max_pool3d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - indices.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32() - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_add(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_add(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_add_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_add_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_add_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_add_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_add_scalar>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_add_scalar( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_add_scalar_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_add_scalar_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_add_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_add_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_addbmm(&self, batch1: &Tensor, batch2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_addbmm( - c_tensors.as_mut_ptr(), - self.c_tensor, - batch1.c_tensor, - batch2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_addbmm_(&mut self, batch1: &Tensor, batch2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_addbmm_( - c_tensors.as_mut_ptr(), - self.c_tensor, - batch1.c_tensor, - batch2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_addbmm_out( - &self, - out: &Tensor, - batch1: &Tensor, - batch2: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_addbmm_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - batch1.c_tensor, - batch2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_addcdiv(&self, tensor1: &Tensor, tensor2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_addcdiv( - c_tensors.as_mut_ptr(), - self.c_tensor, - tensor1.c_tensor, - tensor2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_addcdiv_(&mut self, tensor1: &Tensor, tensor2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_addcdiv_( - c_tensors.as_mut_ptr(), - self.c_tensor, - tensor1.c_tensor, - tensor2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_addcdiv_out( - &self, - out: &Tensor, - tensor1: &Tensor, - tensor2: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_addcdiv_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - tensor1.c_tensor, - tensor2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_addcmul(&self, tensor1: &Tensor, tensor2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_addcmul( - c_tensors.as_mut_ptr(), - self.c_tensor, - tensor1.c_tensor, - tensor2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_addcmul_(&mut self, tensor1: &Tensor, tensor2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_addcmul_( - c_tensors.as_mut_ptr(), - self.c_tensor, - tensor1.c_tensor, - tensor2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_addcmul_out( - &self, - out: &Tensor, - tensor1: &Tensor, - tensor2: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_addcmul_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - tensor1.c_tensor, - tensor2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_addmm(&self, mat1: &Tensor, mat2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_addmm( - c_tensors.as_mut_ptr(), - self.c_tensor, - mat1.c_tensor, - mat2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_addmm_(&mut self, mat1: &Tensor, mat2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_addmm_( - c_tensors.as_mut_ptr(), - self.c_tensor, - mat1.c_tensor, - mat2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_addmm_dtype( - &self, - mat1: &Tensor, - mat2: &Tensor, - out_dtype: Kind, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_addmm_dtype( - c_tensors.as_mut_ptr(), - self.c_tensor, - mat1.c_tensor, - mat2.c_tensor, - out_dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_addmm_dtype_out( - &self, - out: &Tensor, - mat1: &Tensor, - mat2: &Tensor, - out_dtype: Kind, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_addmm_dtype_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mat1.c_tensor, - mat2.c_tensor, - out_dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_addmm_out( - &self, - out: &Tensor, - mat1: &Tensor, - mat2: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_addmm_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mat1.c_tensor, - mat2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_addmv(&self, mat: &Tensor, vec: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_addmv( - c_tensors.as_mut_ptr(), - self.c_tensor, - mat.c_tensor, - vec.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_addmv_(&mut self, mat: &Tensor, vec: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_addmv_( - c_tensors.as_mut_ptr(), - self.c_tensor, - mat.c_tensor, - vec.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_addmv_out( - &self, - out: &Tensor, - mat: &Tensor, - vec: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_addmv_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mat.c_tensor, - vec.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_addr(&self, vec1: &Tensor, vec2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_addr( - c_tensors.as_mut_ptr(), - self.c_tensor, - vec1.c_tensor, - vec2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_addr_(&mut self, vec1: &Tensor, vec2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_addr_( - c_tensors.as_mut_ptr(), - self.c_tensor, - vec1.c_tensor, - vec2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_addr_out( - &self, - out: &Tensor, - vec1: &Tensor, - vec2: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_addr_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - vec1.c_tensor, - vec2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_adjoint(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_adjoint(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_affine_grid_generator( - theta: &Tensor, - size: impl IntList, - align_corners: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_affine_grid_generator( - c_tensors.as_mut_ptr(), - theta.c_tensor, - size.as_ptr(), - size.len_i32(), - if align_corners { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_affine_grid_generator_backward( - grad: &Tensor, - size: impl IntList, - align_corners: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_affine_grid_generator_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - size.as_ptr(), - size.len_i32(), - if align_corners { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_affine_grid_generator_out( - out: &Tensor, - theta: &Tensor, - size: impl IntList, - align_corners: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_affine_grid_generator_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - theta.c_tensor, - size.as_ptr(), - size.len_i32(), - if align_corners { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_alias(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_alias(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_alias_copy(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_alias_copy(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_alias_copy_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_alias_copy_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_align_as(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_align_as(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_align_tensors>(tensors: &[T]) -> Result, TchError> { - let c_tensors = - unsafe_torch_err!(atg_align_tensors(ptr_list(tensors).as_ptr(), tensors.len() as i32)); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_all(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_all(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_all_all_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_all_all_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_all_dim(&self, dim: i64, keepdim: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_all_dim( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_all_dims(&self, dim: impl IntListOption, keepdim: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_all_dims( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_all_dims_out( - &self, - out: &Tensor, - dim: impl IntListOption, - keepdim: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_all_dims_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_all_out(&self, out: &Tensor, dim: i64, keepdim: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_all_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_allclose( - &self, - other: &Tensor, - rtol: f64, - atol: f64, - equal_nan: bool, - ) -> Result { - let return_; - unsafe_torch_err!( - return_ = atg_allclose( - self.c_tensor, - other.c_tensor, - rtol, - atol, - if equal_nan { 1 } else { 0 } - ) - ); - Ok(return_ != 0) - } - - pub fn f_alpha_dropout(&self, p: f64, train: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_alpha_dropout( - c_tensors.as_mut_ptr(), - self.c_tensor, - p, - if train { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_alpha_dropout_(&mut self, p: f64, train: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_alpha_dropout_( - c_tensors.as_mut_ptr(), - self.c_tensor, - p, - if train { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_amax(&self, dim: impl IntList, keepdim: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_amax( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_amax_out( - &self, - out: &Tensor, - dim: impl IntList, - keepdim: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_amax_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_amin(&self, dim: impl IntList, keepdim: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_amin( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_amin_out( - &self, - out: &Tensor, - dim: impl IntList, - keepdim: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_amin_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_aminmax( - &self, - dim: impl Into>, - keepdim: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_aminmax( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.unwrap_or(0i64), - dim.is_none() as i8, - if keepdim { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_aminmax_out( - &self, - min: &Tensor, - max: &Tensor, - dim: impl Into>, - keepdim: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_aminmax_out( - c_tensors.as_mut_ptr(), - min.c_tensor, - max.c_tensor, - self.c_tensor, - dim.unwrap_or(0i64), - dim.is_none() as i8, - if keepdim { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_angle(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_angle(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_angle_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_angle_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_any(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_any(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_any_all_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_any_all_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_any_dim(&self, dim: i64, keepdim: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_any_dim( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_any_dims(&self, dim: impl IntListOption, keepdim: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_any_dims( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_any_dims_out( - &self, - out: &Tensor, - dim: impl IntListOption, - keepdim: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_any_dims_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_any_out(&self, out: &Tensor, dim: i64, keepdim: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_any_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arange>(end: S, options: (Kind, Device)) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arange( - c_tensors.as_mut_ptr(), - end.into().c_scalar, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arange_start>( - start: S, - end: S, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arange_start( - c_tensors.as_mut_ptr(), - start.into().c_scalar, - end.into().c_scalar, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arange_start_step>( - start: S, - end: S, - step: S, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arange_start_step( - c_tensors.as_mut_ptr(), - start.into().c_scalar, - end.into().c_scalar, - step.into().c_scalar, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arccos(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arccos(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arccos_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arccos_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arccos_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arccos_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arccosh(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arccosh(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arccosh_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arccosh_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arccosh_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arccosh_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arcsin(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arcsin(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arcsin_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arcsin_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arcsin_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arcsin_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arcsinh(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arcsinh(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arcsinh_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arcsinh_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arcsinh_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arcsinh_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arctan(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arctan(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arctan2(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arctan2(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arctan2_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arctan2_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arctan2_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arctan2_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arctan_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arctan_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arctan_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arctan_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arctanh(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arctanh(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arctanh_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arctanh_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_arctanh_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_arctanh_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_argmax(&self, dim: impl Into>, keepdim: bool) -> Result { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_argmax( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.unwrap_or(0i64), - dim.is_none() as i8, - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_argmax_out( - &self, - out: &Tensor, - dim: impl Into>, - keepdim: bool, - ) -> Result { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_argmax_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.unwrap_or(0i64), - dim.is_none() as i8, - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_argmin(&self, dim: impl Into>, keepdim: bool) -> Result { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_argmin( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.unwrap_or(0i64), - dim.is_none() as i8, - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_argmin_out( - &self, - out: &Tensor, - dim: impl Into>, - keepdim: bool, - ) -> Result { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_argmin_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.unwrap_or(0i64), - dim.is_none() as i8, - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_argsort(&self, dim: i64, descending: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_argsort( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - if descending { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_argsort_stable( - &self, - stable: bool, - dim: i64, - descending: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_argsort_stable( - c_tensors.as_mut_ptr(), - self.c_tensor, - if stable { 1 } else { 0 }, - dim, - if descending { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_argsort_stable_out( - &self, - out: &Tensor, - stable: bool, - dim: i64, - descending: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_argsort_stable_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - if stable { 1 } else { 0 }, - dim, - if descending { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_argwhere(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_argwhere(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_as_strided( - &self, - size: impl IntList, - stride: impl IntList, - storage_offset: impl Into>, - ) -> Result { - let storage_offset = storage_offset.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_as_strided( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - storage_offset.unwrap_or(0i64), - storage_offset.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_as_strided_( - &mut self, - size: impl IntList, - stride: impl IntList, - storage_offset: impl Into>, - ) -> Result { - let storage_offset = storage_offset.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_as_strided_( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - storage_offset.unwrap_or(0i64), - storage_offset.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_as_strided_copy( - &self, - size: impl IntList, - stride: impl IntList, - storage_offset: impl Into>, - ) -> Result { - let storage_offset = storage_offset.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_as_strided_copy( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - storage_offset.unwrap_or(0i64), - storage_offset.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_as_strided_copy_out( - &self, - out: &Tensor, - size: impl IntList, - stride: impl IntList, - storage_offset: impl Into>, - ) -> Result { - let storage_offset = storage_offset.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_as_strided_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - size.as_ptr(), - size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - storage_offset.unwrap_or(0i64), - storage_offset.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_as_strided_scatter( - &self, - src: &Tensor, - size: impl IntList, - stride: impl IntList, - storage_offset: impl Into>, - ) -> Result { - let storage_offset = storage_offset.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_as_strided_scatter( - c_tensors.as_mut_ptr(), - self.c_tensor, - src.c_tensor, - size.as_ptr(), - size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - storage_offset.unwrap_or(0i64), - storage_offset.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_as_strided_scatter_out( - &self, - out: &Tensor, - src: &Tensor, - size: impl IntList, - stride: impl IntList, - storage_offset: impl Into>, - ) -> Result { - let storage_offset = storage_offset.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_as_strided_scatter_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - src.c_tensor, - size.as_ptr(), - size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - storage_offset.unwrap_or(0i64), - storage_offset.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_asin(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_asin(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_asin_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_asin_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_asin_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_asin_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_asinh(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_asinh(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_asinh_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_asinh_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_asinh_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_asinh_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_atan(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_atan(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_atan2(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_atan2(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_atan2_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_atan2_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_atan2_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_atan2_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_atan_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_atan_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_atan_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_atan_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_atanh(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_atanh(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_atanh_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_atanh_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_atanh_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_atanh_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_atleast_1d(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_atleast_1d(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_atleast_1d_sequence>( - tensors: &[T], - ) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_atleast_1d_sequence( - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_atleast_2d(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_atleast_2d(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_atleast_2d_sequence>( - tensors: &[T], - ) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_atleast_2d_sequence( - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_atleast_3d(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_atleast_3d(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_atleast_3d_sequence>( - tensors: &[T], - ) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_atleast_3d_sequence( - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_avg_pool1d( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ceil_mode: bool, - count_include_pad: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_avg_pool1d( - c_tensors.as_mut_ptr(), - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - if ceil_mode { 1 } else { 0 }, - if count_include_pad { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_avg_pool1d_out( - &self, - out: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ceil_mode: bool, - count_include_pad: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_avg_pool1d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - if ceil_mode { 1 } else { 0 }, - if count_include_pad { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_avg_pool2d( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ceil_mode: bool, - count_include_pad: bool, - divisor_override: impl Into>, - ) -> Result { - let divisor_override = divisor_override.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_avg_pool2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - if ceil_mode { 1 } else { 0 }, - if count_include_pad { 1 } else { 0 }, - divisor_override.unwrap_or(0i64), - divisor_override.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_avg_pool2d_backward( - &self, - grad_output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ceil_mode: bool, - count_include_pad: bool, - divisor_override: impl Into>, - ) -> Result { - let divisor_override = divisor_override.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_avg_pool2d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - if ceil_mode { 1 } else { 0 }, - if count_include_pad { 1 } else { 0 }, - divisor_override.unwrap_or(0i64), - divisor_override.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_avg_pool2d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ceil_mode: bool, - count_include_pad: bool, - divisor_override: impl Into>, - ) -> Result { - let divisor_override = divisor_override.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_avg_pool2d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - if ceil_mode { 1 } else { 0 }, - if count_include_pad { 1 } else { 0 }, - divisor_override.unwrap_or(0i64), - divisor_override.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_avg_pool2d_out( - &self, - out: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ceil_mode: bool, - count_include_pad: bool, - divisor_override: impl Into>, - ) -> Result { - let divisor_override = divisor_override.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_avg_pool2d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - if ceil_mode { 1 } else { 0 }, - if count_include_pad { 1 } else { 0 }, - divisor_override.unwrap_or(0i64), - divisor_override.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_avg_pool3d( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ceil_mode: bool, - count_include_pad: bool, - divisor_override: impl Into>, - ) -> Result { - let divisor_override = divisor_override.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_avg_pool3d( - c_tensors.as_mut_ptr(), - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - if ceil_mode { 1 } else { 0 }, - if count_include_pad { 1 } else { 0 }, - divisor_override.unwrap_or(0i64), - divisor_override.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_avg_pool3d_backward( - &self, - grad_output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ceil_mode: bool, - count_include_pad: bool, - divisor_override: impl Into>, - ) -> Result { - let divisor_override = divisor_override.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_avg_pool3d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - if ceil_mode { 1 } else { 0 }, - if count_include_pad { 1 } else { 0 }, - divisor_override.unwrap_or(0i64), - divisor_override.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_avg_pool3d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ceil_mode: bool, - count_include_pad: bool, - divisor_override: impl Into>, - ) -> Result { - let divisor_override = divisor_override.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_avg_pool3d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - if ceil_mode { 1 } else { 0 }, - if count_include_pad { 1 } else { 0 }, - divisor_override.unwrap_or(0i64), - divisor_override.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_avg_pool3d_out( - &self, - out: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ceil_mode: bool, - count_include_pad: bool, - divisor_override: impl Into>, - ) -> Result { - let divisor_override = divisor_override.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_avg_pool3d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - if ceil_mode { 1 } else { 0 }, - if count_include_pad { 1 } else { 0 }, - divisor_override.unwrap_or(0i64), - divisor_override.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_baddbmm>( - &self, - batch1: &Tensor, - batch2: &Tensor, - beta: S, - alpha: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_baddbmm( - c_tensors.as_mut_ptr(), - self.c_tensor, - batch1.c_tensor, - batch2.c_tensor, - beta.into().c_scalar, - alpha.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_baddbmm_(&mut self, batch1: &Tensor, batch2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_baddbmm_( - c_tensors.as_mut_ptr(), - self.c_tensor, - batch1.c_tensor, - batch2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_baddbmm_dtype>( - &self, - batch1: &Tensor, - batch2: &Tensor, - out_dtype: Kind, - beta: S, - alpha: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_baddbmm_dtype( - c_tensors.as_mut_ptr(), - self.c_tensor, - batch1.c_tensor, - batch2.c_tensor, - out_dtype.c_int(), - beta.into().c_scalar, - alpha.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_baddbmm_dtype_out( - &self, - out: &Tensor, - batch1: &Tensor, - batch2: &Tensor, - out_dtype: Kind, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_baddbmm_dtype_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - batch1.c_tensor, - batch2.c_tensor, - out_dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_baddbmm_out( - &self, - out: &Tensor, - batch1: &Tensor, - batch2: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_baddbmm_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - batch1.c_tensor, - batch2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bartlett_window( - window_length: i64, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bartlett_window( - c_tensors.as_mut_ptr(), - window_length, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bartlett_window_out(out: &Tensor, window_length: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bartlett_window_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - window_length - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bartlett_window_periodic( - window_length: i64, - periodic: bool, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bartlett_window_periodic( - c_tensors.as_mut_ptr(), - window_length, - if periodic { 1 } else { 0 }, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bartlett_window_periodic_out( - out: &Tensor, - window_length: i64, - periodic: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bartlett_window_periodic_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - window_length, - if periodic { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_batch_norm>( - &self, - weight: Option, - bias: Option, - running_mean: Option, - running_var: Option, - training: bool, - momentum: f64, - eps: f64, - cudnn_enabled: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_batch_norm( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if training { 1 } else { 0 }, - momentum, - eps, - if cudnn_enabled { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_batch_norm_backward_elemt>( - &self, - grad_out: &Tensor, - mean: &Tensor, - invstd: &Tensor, - weight: Option, - sum_dy: &Tensor, - sum_dy_xmu: &Tensor, - count: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_batch_norm_backward_elemt( - c_tensors.as_mut_ptr(), - grad_out.c_tensor, - self.c_tensor, - mean.c_tensor, - invstd.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - sum_dy.c_tensor, - sum_dy_xmu.c_tensor, - count.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_batch_norm_backward_elemt_out>( - &self, - out: &Tensor, - grad_out: &Tensor, - mean: &Tensor, - invstd: &Tensor, - weight: Option, - sum_dy: &Tensor, - sum_dy_xmu: &Tensor, - count: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_batch_norm_backward_elemt_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad_out.c_tensor, - self.c_tensor, - mean.c_tensor, - invstd.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - sum_dy.c_tensor, - sum_dy_xmu.c_tensor, - count.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_batch_norm_backward_reduce>( - &self, - grad_out: &Tensor, - mean: &Tensor, - invstd: &Tensor, - weight: Option, - input_g: bool, - weight_g: bool, - bias_g: bool, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg_batch_norm_backward_reduce( - c_tensors.as_mut_ptr(), - grad_out.c_tensor, - self.c_tensor, - mean.c_tensor, - invstd.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if input_g { 1 } else { 0 }, - if weight_g { 1 } else { 0 }, - if bias_g { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_batch_norm_backward_reduce_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - out3: &Tensor, - grad_out: &Tensor, - mean: &Tensor, - invstd: &Tensor, - weight: Option, - input_g: bool, - weight_g: bool, - bias_g: bool, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg_batch_norm_backward_reduce_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - out2.c_tensor, - out3.c_tensor, - grad_out.c_tensor, - self.c_tensor, - mean.c_tensor, - invstd.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if input_g { 1 } else { 0 }, - if weight_g { 1 } else { 0 }, - if bias_g { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_batch_norm_elemt>( - &self, - weight: Option, - bias: Option, - mean: &Tensor, - invstd: &Tensor, - eps: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_batch_norm_elemt( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - mean.c_tensor, - invstd.c_tensor, - eps - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_batch_norm_elemt_out>( - &self, - out: &Tensor, - weight: Option, - bias: Option, - mean: &Tensor, - invstd: &Tensor, - eps: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_batch_norm_elemt_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - mean.c_tensor, - invstd.c_tensor, - eps - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_batch_norm_gather_stats>( - &self, - mean: &Tensor, - invstd: &Tensor, - running_mean: Option, - running_var: Option, - momentum: f64, - eps: f64, - count: i64, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_batch_norm_gather_stats( - c_tensors.as_mut_ptr(), - self.c_tensor, - mean.c_tensor, - invstd.c_tensor, - running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - momentum, - eps, - count - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_batch_norm_gather_stats_out>( - &self, - out0: &Tensor, - out1: &Tensor, - mean: &Tensor, - invstd: &Tensor, - running_mean: Option, - running_var: Option, - momentum: f64, - eps: f64, - count: i64, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_batch_norm_gather_stats_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - self.c_tensor, - mean.c_tensor, - invstd.c_tensor, - running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - momentum, - eps, - count - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_batch_norm_gather_stats_with_counts>( - &self, - mean: &Tensor, - invstd: &Tensor, - running_mean: Option, - running_var: Option, - momentum: f64, - eps: f64, - counts: &Tensor, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_batch_norm_gather_stats_with_counts( - c_tensors.as_mut_ptr(), - self.c_tensor, - mean.c_tensor, - invstd.c_tensor, - running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - momentum, - eps, - counts.c_tensor - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_batch_norm_gather_stats_with_counts_out>( - &self, - out0: &Tensor, - out1: &Tensor, - mean: &Tensor, - invstd: &Tensor, - running_mean: Option, - running_var: Option, - momentum: f64, - eps: f64, - counts: &Tensor, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_batch_norm_gather_stats_with_counts_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - self.c_tensor, - mean.c_tensor, - invstd.c_tensor, - running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - momentum, - eps, - counts.c_tensor - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_batch_norm_stats(&self, eps: f64) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_batch_norm_stats(c_tensors.as_mut_ptr(), self.c_tensor, eps)); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_batch_norm_stats_out( - &self, - out0: &Tensor, - out1: &Tensor, - eps: f64, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_batch_norm_stats_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - self.c_tensor, - eps - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_batch_norm_update_stats>( - &self, - running_mean: Option, - running_var: Option, - momentum: f64, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_batch_norm_update_stats( - c_tensors.as_mut_ptr(), - self.c_tensor, - running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - momentum - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_batch_norm_update_stats_out>( - &self, - out0: &Tensor, - out1: &Tensor, - running_mean: Option, - running_var: Option, - momentum: f64, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_batch_norm_update_stats_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - self.c_tensor, - running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - momentum - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_bernoulli(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bernoulli(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bernoulli_(&mut self, p: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bernoulli_(c_tensors.as_mut_ptr(), self.c_tensor, p.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bernoulli_float_(&mut self, p: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bernoulli_float_(c_tensors.as_mut_ptr(), self.c_tensor, p)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bernoulli_p(&self, p: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bernoulli_p(c_tensors.as_mut_ptr(), self.c_tensor, p)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bernoulli_tensor(&self, p: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bernoulli_tensor(c_tensors.as_mut_ptr(), self.c_tensor, p.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bilinear>( - input1: &Tensor, - input2: &Tensor, - weight: &Tensor, - bias: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bilinear( - c_tensors.as_mut_ptr(), - input1.c_tensor, - input2.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_binary_cross_entropy>( - &self, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_binary_cross_entropy( - c_tensors.as_mut_ptr(), - self.c_tensor, - target.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_binary_cross_entropy_backward>( - &self, - grad_output: &Tensor, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_binary_cross_entropy_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - target.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_binary_cross_entropy_backward_grad_input>( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_binary_cross_entropy_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - target.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_binary_cross_entropy_out>( - &self, - out: &Tensor, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_binary_cross_entropy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - target.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_binary_cross_entropy_with_logits>( - &self, - target: &Tensor, - weight: Option, - pos_weight: Option, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_binary_cross_entropy_with_logits( - c_tensors.as_mut_ptr(), - self.c_tensor, - target.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - pos_weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_binary_cross_entropy_with_logits_out>( - &self, - out: &Tensor, - target: &Tensor, - weight: Option, - pos_weight: Option, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_binary_cross_entropy_with_logits_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - target.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - pos_weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bincount>( - &self, - weights: Option, - minlength: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bincount( - c_tensors.as_mut_ptr(), - self.c_tensor, - weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - minlength - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bincount_out>( - &self, - out: &Tensor, - weights: Option, - minlength: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bincount_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - minlength - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_binomial(count: &Tensor, prob: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_binomial(c_tensors.as_mut_ptr(), count.c_tensor, prob.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_binomial_out(out: &Tensor, count: &Tensor, prob: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_binomial_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - count.c_tensor, - prob.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_and>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_and( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_and_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_and_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_and_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_and_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_and_scalar_tensor>( - self_scalar: S, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_and_scalar_tensor( - c_tensors.as_mut_ptr(), - self_scalar.into().c_scalar, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_and_scalar_tensor_out>( - out: &Tensor, - self_scalar: S, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_and_scalar_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self_scalar.into().c_scalar, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_and_tensor(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_and_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_and_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_and_tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_and_tensor_out( - &self, - out: &Tensor, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_and_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_left_shift(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_left_shift( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_left_shift_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_left_shift_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_left_shift_scalar_tensor>( - self_scalar: S, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_left_shift_scalar_tensor( - c_tensors.as_mut_ptr(), - self_scalar.into().c_scalar, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_left_shift_scalar_tensor_out>( - out: &Tensor, - self_scalar: S, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_left_shift_scalar_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self_scalar.into().c_scalar, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_left_shift_tensor_out( - &self, - out: &Tensor, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_left_shift_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_left_shift_tensor_scalar>( - &self, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_left_shift_tensor_scalar( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_left_shift_tensor_scalar_>( - &mut self, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_left_shift_tensor_scalar_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_left_shift_tensor_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_left_shift_tensor_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_not(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_not(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_not_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_not_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_not_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_not_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_or>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_or( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_or_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_or_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_or_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_or_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_or_scalar_tensor>( - self_scalar: S, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_or_scalar_tensor( - c_tensors.as_mut_ptr(), - self_scalar.into().c_scalar, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_or_scalar_tensor_out>( - out: &Tensor, - self_scalar: S, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_or_scalar_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self_scalar.into().c_scalar, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_or_tensor(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_or_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_or_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_or_tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_or_tensor_out( - &self, - out: &Tensor, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_or_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_right_shift(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_right_shift( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_right_shift_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_right_shift_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_right_shift_scalar_tensor>( - self_scalar: S, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_right_shift_scalar_tensor( - c_tensors.as_mut_ptr(), - self_scalar.into().c_scalar, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_right_shift_scalar_tensor_out>( - out: &Tensor, - self_scalar: S, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_right_shift_scalar_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self_scalar.into().c_scalar, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_right_shift_tensor_out( - &self, - out: &Tensor, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_right_shift_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_right_shift_tensor_scalar>( - &self, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_right_shift_tensor_scalar( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_right_shift_tensor_scalar_>( - &mut self, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_right_shift_tensor_scalar_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_right_shift_tensor_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_right_shift_tensor_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_xor>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_xor( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_xor_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_xor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_xor_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_xor_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_xor_scalar_tensor>( - self_scalar: S, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_xor_scalar_tensor( - c_tensors.as_mut_ptr(), - self_scalar.into().c_scalar, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_xor_scalar_tensor_out>( - out: &Tensor, - self_scalar: S, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_xor_scalar_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self_scalar.into().c_scalar, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_xor_tensor(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_xor_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_xor_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_xor_tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bitwise_xor_tensor_out( - &self, - out: &Tensor, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bitwise_xor_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_blackman_window( - window_length: i64, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_blackman_window( - c_tensors.as_mut_ptr(), - window_length, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_blackman_window_out(out: &Tensor, window_length: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_blackman_window_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - window_length - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_blackman_window_periodic( - window_length: i64, - periodic: bool, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_blackman_window_periodic( - c_tensors.as_mut_ptr(), - window_length, - if periodic { 1 } else { 0 }, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_blackman_window_periodic_out( - out: &Tensor, - window_length: i64, - periodic: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_blackman_window_periodic_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - window_length, - if periodic { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_block_diag>(tensors: &[T]) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_block_diag( - c_tensors.as_mut_ptr(), - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_block_diag_out>( - out: &Tensor, - tensors: &[T], - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_block_diag_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bmm(&self, mat2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bmm(c_tensors.as_mut_ptr(), self.c_tensor, mat2.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bmm_dtype(&self, mat2: &Tensor, out_dtype: Kind) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bmm_dtype( - c_tensors.as_mut_ptr(), - self.c_tensor, - mat2.c_tensor, - out_dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bmm_dtype_out( - &self, - out: &Tensor, - mat2: &Tensor, - out_dtype: Kind, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bmm_dtype_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mat2.c_tensor, - out_dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bmm_out(&self, out: &Tensor, mat2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bmm_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mat2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_broadcast_tensors>(tensors: &[T]) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_broadcast_tensors( - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_broadcast_to(&self, size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_broadcast_to( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bucketize( - &self, - boundaries: &Tensor, - out_int32: bool, - right: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bucketize( - c_tensors.as_mut_ptr(), - self.c_tensor, - boundaries.c_tensor, - if out_int32 { 1 } else { 0 }, - if right { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bucketize_scalar>( - self_scalar: S, - boundaries: &Tensor, - out_int32: bool, - right: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bucketize_scalar( - c_tensors.as_mut_ptr(), - self_scalar.into().c_scalar, - boundaries.c_tensor, - if out_int32 { 1 } else { 0 }, - if right { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bucketize_scalar_out>( - out: &Tensor, - self_scalar: S, - boundaries: &Tensor, - out_int32: bool, - right: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bucketize_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self_scalar.into().c_scalar, - boundaries.c_tensor, - if out_int32 { 1 } else { 0 }, - if right { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_bucketize_tensor_out( - &self, - out: &Tensor, - boundaries: &Tensor, - out_int32: bool, - right: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_bucketize_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - boundaries.c_tensor, - if out_int32 { 1 } else { 0 }, - if right { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_can_cast(from_: Kind, to: Kind) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_can_cast(from_.c_int(), to.c_int())); - Ok(return_ != 0) - } - - pub fn f_cartesian_prod>(tensors: &[T]) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cartesian_prod( - c_tensors.as_mut_ptr(), - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cat>(tensors: &[T], dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cat( - c_tensors.as_mut_ptr(), - ptr_list(tensors).as_ptr(), - tensors.len() as i32, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cat_out>( - out: &Tensor, - tensors: &[T], - dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cat_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - ptr_list(tensors).as_ptr(), - tensors.len() as i32, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cauchy(&self, median: f64, sigma: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cauchy(c_tensors.as_mut_ptr(), self.c_tensor, median, sigma)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cauchy_(&mut self, median: f64, sigma: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cauchy_(c_tensors.as_mut_ptr(), self.c_tensor, median, sigma)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cauchy_out(&self, out: &Tensor, median: f64, sigma: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cauchy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - median, - sigma - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ccol_indices(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ccol_indices(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ccol_indices_copy(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ccol_indices_copy(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ccol_indices_copy_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ccol_indices_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cdist( - x1: &Tensor, - x2: &Tensor, - p: f64, - compute_mode: impl Into>, - ) -> Result { - let compute_mode = compute_mode.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cdist( - c_tensors.as_mut_ptr(), - x1.c_tensor, - x2.c_tensor, - p, - compute_mode.unwrap_or(0i64), - compute_mode.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ceil(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ceil(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ceil_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ceil_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ceil_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ceil_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_celu(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_celu(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_celu_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_celu_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_celu_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_celu_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_chain_matmul>(matrices: &[T]) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_chain_matmul( - c_tensors.as_mut_ptr(), - ptr_list(matrices).as_ptr(), - matrices.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_chain_matmul_out>( - out: &Tensor, - matrices: &[T], - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_chain_matmul_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - ptr_list(matrices).as_ptr(), - matrices.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_chalf(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_chalf(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_channel_shuffle(&self, groups: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_channel_shuffle(c_tensors.as_mut_ptr(), self.c_tensor, groups)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_channel_shuffle_out(&self, out: &Tensor, groups: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_channel_shuffle_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cholesky(&self, upper: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cholesky( - c_tensors.as_mut_ptr(), - self.c_tensor, - if upper { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cholesky_inverse(&self, upper: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cholesky_inverse( - c_tensors.as_mut_ptr(), - self.c_tensor, - if upper { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cholesky_inverse_out(&self, out: &Tensor, upper: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cholesky_inverse_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - if upper { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cholesky_out(&self, out: &Tensor, upper: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cholesky_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - if upper { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cholesky_solve(&self, input2: &Tensor, upper: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cholesky_solve( - c_tensors.as_mut_ptr(), - self.c_tensor, - input2.c_tensor, - if upper { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cholesky_solve_out( - &self, - out: &Tensor, - input2: &Tensor, - upper: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cholesky_solve_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - input2.c_tensor, - if upper { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_choose_qparams_optimized( - &self, - numel: i64, - n_bins: i64, - ratio: f64, - bit_width: i64, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_choose_qparams_optimized( - c_tensors.as_mut_ptr(), - self.c_tensor, - numel, - n_bins, - ratio, - bit_width - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_chunk(&self, chunks: i64, dim: i64) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_chunk(self.c_tensor, chunks, dim)); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_clamp>(&self, min: S, max: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clamp( - c_tensors.as_mut_ptr(), - self.c_tensor, - min.into().c_scalar, - max.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clamp_>(&mut self, min: S, max: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clamp_( - c_tensors.as_mut_ptr(), - self.c_tensor, - min.into().c_scalar, - max.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clamp_max>(&self, max: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clamp_max( - c_tensors.as_mut_ptr(), - self.c_tensor, - max.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clamp_max_>(&mut self, max: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clamp_max_( - c_tensors.as_mut_ptr(), - self.c_tensor, - max.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clamp_max_out>( - &self, - out: &Tensor, - max: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clamp_max_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - max.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clamp_max_tensor(&self, max: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clamp_max_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - max.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clamp_max_tensor_(&mut self, max: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clamp_max_tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - max.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clamp_max_tensor_out(&self, out: &Tensor, max: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clamp_max_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - max.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clamp_min>(&self, min: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clamp_min( - c_tensors.as_mut_ptr(), - self.c_tensor, - min.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clamp_min_>(&mut self, min: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clamp_min_( - c_tensors.as_mut_ptr(), - self.c_tensor, - min.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clamp_min_out>( - &self, - out: &Tensor, - min: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clamp_min_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - min.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clamp_min_tensor(&self, min: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clamp_min_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - min.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clamp_min_tensor_(&mut self, min: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clamp_min_tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - min.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clamp_min_tensor_out(&self, out: &Tensor, min: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clamp_min_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - min.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clamp_out>( - &self, - out: &Tensor, - min: S, - max: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clamp_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - min.into().c_scalar, - max.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clamp_tensor>( - &self, - min: Option, - max: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clamp_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - min.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - max.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clamp_tensor_>( - &mut self, - min: Option, - max: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clamp_tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - min.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - max.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clamp_tensor_out>( - &self, - out: &Tensor, - min: Option, - max: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clamp_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - min.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - max.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clip>(&self, min: S, max: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clip( - c_tensors.as_mut_ptr(), - self.c_tensor, - min.into().c_scalar, - max.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clip_>(&mut self, min: S, max: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clip_( - c_tensors.as_mut_ptr(), - self.c_tensor, - min.into().c_scalar, - max.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clip_out>( - &self, - out: &Tensor, - min: S, - max: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clip_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - min.into().c_scalar, - max.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clip_tensor>( - &self, - min: Option, - max: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clip_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - min.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - max.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clip_tensor_>( - &mut self, - min: Option, - max: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clip_tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - min.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - max.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clip_tensor_out>( - &self, - out: &Tensor, - min: Option, - max: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clip_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - min.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - max.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_clone(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_clone(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_coalesce(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_coalesce(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_col2im( - &self, - output_size: impl IntList, - kernel_size: impl IntList, - dilation: impl IntList, - padding: impl IntList, - stride: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_col2im( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - kernel_size.as_ptr(), - kernel_size.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - padding.as_ptr(), - padding.len_i32(), - stride.as_ptr(), - stride.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_col2im_out( - &self, - out: &Tensor, - output_size: impl IntList, - kernel_size: impl IntList, - dilation: impl IntList, - padding: impl IntList, - stride: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_col2im_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - kernel_size.as_ptr(), - kernel_size.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - padding.as_ptr(), - padding.len_i32(), - stride.as_ptr(), - stride.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_col_indices(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_col_indices(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_col_indices_copy(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_col_indices_copy(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_col_indices_copy_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_col_indices_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_column_stack>(tensors: &[T]) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_column_stack( - c_tensors.as_mut_ptr(), - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_column_stack_out>( - out: &Tensor, - tensors: &[T], - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_column_stack_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_combinations(&self, r: i64, with_replacement: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_combinations( - c_tensors.as_mut_ptr(), - self.c_tensor, - r, - if with_replacement { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_complex(real: &Tensor, imag: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_complex(c_tensors.as_mut_ptr(), real.c_tensor, imag.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_complex_out(out: &Tensor, real: &Tensor, imag: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_complex_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - real.c_tensor, - imag.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_concat>(tensors: &[T], dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_concat( - c_tensors.as_mut_ptr(), - ptr_list(tensors).as_ptr(), - tensors.len() as i32, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_concat_out>( - out: &Tensor, - tensors: &[T], - dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_concat_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - ptr_list(tensors).as_ptr(), - tensors.len() as i32, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_concatenate>(tensors: &[T], dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_concatenate( - c_tensors.as_mut_ptr(), - ptr_list(tensors).as_ptr(), - tensors.len() as i32, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_concatenate_out>( - out: &Tensor, - tensors: &[T], - dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_concatenate_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - ptr_list(tensors).as_ptr(), - tensors.len() as i32, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_conj(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_conj(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_conj_physical(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_conj_physical(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_conj_physical_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_conj_physical_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_conj_physical_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_conj_physical_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_constant_pad_nd(&self, pad: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_constant_pad_nd( - c_tensors.as_mut_ptr(), - self.c_tensor, - pad.as_ptr(), - pad.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_constant_pad_nd_out( - &self, - out: &Tensor, - pad: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_constant_pad_nd_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - pad.as_ptr(), - pad.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_contiguous(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_contiguous(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_conv1d>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_conv1d( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_conv1d_padding>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: &str, - dilation: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_conv1d_padding( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len() as i32, - dilation.as_ptr(), - dilation.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_conv2d>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_conv2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_conv2d_padding>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: &str, - dilation: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_conv2d_padding( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len() as i32, - dilation.as_ptr(), - dilation.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_conv3d>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_conv3d( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_conv3d_padding>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: &str, - dilation: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_conv3d_padding( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len() as i32, - dilation.as_ptr(), - dilation.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_conv_depthwise3d>( - &self, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_conv_depthwise3d( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_conv_depthwise3d_out>( - &self, - out: &Tensor, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_conv_depthwise3d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_conv_tbc(&self, weight: &Tensor, bias: &Tensor, pad: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_conv_tbc( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.c_tensor, - pad - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_conv_tbc_backward( - &self, - input: &Tensor, - weight: &Tensor, - bias: &Tensor, - pad: i64, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_conv_tbc_backward( - c_tensors.as_mut_ptr(), - self.c_tensor, - input.c_tensor, - weight.c_tensor, - bias.c_tensor, - pad - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_conv_tbc_out( - &self, - out: &Tensor, - weight: &Tensor, - bias: &Tensor, - pad: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_conv_tbc_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - bias.c_tensor, - pad - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_conv_transpose1d>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - output_padding: impl IntList, - groups: i64, - dilation: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_conv_transpose1d( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - output_padding.as_ptr(), - output_padding.len_i32(), - groups, - dilation.as_ptr(), - dilation.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_conv_transpose2d>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - output_padding: impl IntList, - groups: i64, - dilation: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_conv_transpose2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - output_padding.as_ptr(), - output_padding.len_i32(), - groups, - dilation.as_ptr(), - dilation.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_conv_transpose3d>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - output_padding: impl IntList, - groups: i64, - dilation: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_conv_transpose3d( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - output_padding.as_ptr(), - output_padding.len_i32(), - groups, - dilation.as_ptr(), - dilation.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_convolution>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - transposed: bool, - output_padding: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_convolution( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if transposed { 1 } else { 0 }, - output_padding.as_ptr(), - output_padding.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_convolution_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - transposed: bool, - output_padding: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_convolution_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if transposed { 1 } else { 0 }, - output_padding.as_ptr(), - output_padding.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_convolution_overrideable>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - transposed: bool, - output_padding: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_convolution_overrideable( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if transposed { 1 } else { 0 }, - output_padding.as_ptr(), - output_padding.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_convolution_overrideable_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - transposed: bool, - output_padding: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_convolution_overrideable_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if transposed { 1 } else { 0 }, - output_padding.as_ptr(), - output_padding.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_copy_sparse_to_sparse( - &self, - src: &Tensor, - non_blocking: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_copy_sparse_to_sparse( - c_tensors.as_mut_ptr(), - self.c_tensor, - src.c_tensor, - if non_blocking { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_copy_sparse_to_sparse_( - &mut self, - src: &Tensor, - non_blocking: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_copy_sparse_to_sparse_( - c_tensors.as_mut_ptr(), - self.c_tensor, - src.c_tensor, - if non_blocking { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_copy_sparse_to_sparse_out( - &self, - out: &Tensor, - src: &Tensor, - non_blocking: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_copy_sparse_to_sparse_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - src.c_tensor, - if non_blocking { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_copysign(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_copysign(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_copysign_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_copysign_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_copysign_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_copysign_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_copysign_scalar>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_copysign_scalar( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_copysign_scalar_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_copysign_scalar_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_copysign_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_copysign_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_corrcoef(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_corrcoef(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cos(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cos(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cos_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cos_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cos_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cos_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cosh(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cosh(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cosh_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cosh_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cosh_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cosh_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cosine_embedding_loss( - input1: &Tensor, - input2: &Tensor, - target: &Tensor, - margin: f64, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cosine_embedding_loss( - c_tensors.as_mut_ptr(), - input1.c_tensor, - input2.c_tensor, - target.c_tensor, - margin, - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cosine_similarity( - x1: &Tensor, - x2: &Tensor, - dim: i64, - eps: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cosine_similarity( - c_tensors.as_mut_ptr(), - x1.c_tensor, - x2.c_tensor, - dim, - eps - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_count_nonzero(&self, dim: impl Into>) -> Result { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_count_nonzero( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.unwrap_or(0i64), - dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_count_nonzero_dim_intlist(&self, dim: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_count_nonzero_dim_intlist( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_count_nonzero_dim_intlist_out( - &self, - out: &Tensor, - dim: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_count_nonzero_dim_intlist_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_count_nonzero_out( - &self, - out: &Tensor, - dim: impl Into>, - ) -> Result { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_count_nonzero_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.unwrap_or(0i64), - dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cov>( - &self, - correction: i64, - fweights: Option, - aweights: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cov( - c_tensors.as_mut_ptr(), - self.c_tensor, - correction, - fweights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - aweights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cross(&self, other: &Tensor, dim: impl Into>) -> Result { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cross( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor, - dim.unwrap_or(0i64), - dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cross_entropy_loss>( - &self, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ignore_index: i64, - label_smoothing: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cross_entropy_loss( - c_tensors.as_mut_ptr(), - self.c_tensor, - target.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - reduction.to_int(), - ignore_index, - label_smoothing - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cross_out( - &self, - out: &Tensor, - other: &Tensor, - dim: impl Into>, - ) -> Result { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cross_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor, - dim.unwrap_or(0i64), - dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_crow_indices(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_crow_indices(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_crow_indices_copy(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_crow_indices_copy(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_crow_indices_copy_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_crow_indices_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ctc_loss( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: impl IntList, - target_lengths: impl IntList, - blank: i64, - reduction: crate::Reduction, - zero_infinity: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ctc_loss( - c_tensors.as_mut_ptr(), - log_probs.c_tensor, - targets.c_tensor, - input_lengths.as_ptr(), - input_lengths.len_i32(), - target_lengths.as_ptr(), - target_lengths.len_i32(), - blank, - reduction.to_int(), - if zero_infinity { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ctc_loss_tensor( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: &Tensor, - target_lengths: &Tensor, - blank: i64, - reduction: crate::Reduction, - zero_infinity: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ctc_loss_tensor( - c_tensors.as_mut_ptr(), - log_probs.c_tensor, - targets.c_tensor, - input_lengths.c_tensor, - target_lengths.c_tensor, - blank, - reduction.to_int(), - if zero_infinity { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cudnn_affine_grid_generator( - theta: &Tensor, - n: i64, - c: i64, - h: i64, - w: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cudnn_affine_grid_generator( - c_tensors.as_mut_ptr(), - theta.c_tensor, - n, - c, - h, - w - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cudnn_affine_grid_generator_backward( - grad: &Tensor, - n: i64, - c: i64, - h: i64, - w: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cudnn_affine_grid_generator_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - n, - c, - h, - w - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cudnn_affine_grid_generator_backward_out( - out: &Tensor, - grad: &Tensor, - n: i64, - c: i64, - h: i64, - w: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cudnn_affine_grid_generator_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad.c_tensor, - n, - c, - h, - w - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cudnn_affine_grid_generator_out( - out: &Tensor, - theta: &Tensor, - n: i64, - c: i64, - h: i64, - w: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cudnn_affine_grid_generator_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - theta.c_tensor, - n, - c, - h, - w - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cudnn_batch_norm>( - &self, - weight: &Tensor, - bias: Option, - running_mean: Option, - running_var: Option, - training: bool, - exponential_average_factor: f64, - epsilon: f64, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg_cudnn_batch_norm( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if training { 1 } else { 0 }, - exponential_average_factor, - epsilon - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_cudnn_batch_norm_backward>( - &self, - grad_output: &Tensor, - weight: &Tensor, - running_mean: Option, - running_var: Option, - save_mean: Option, - save_var: Option, - epsilon: f64, - reservespace: &Tensor, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_cudnn_batch_norm_backward( - c_tensors.as_mut_ptr(), - self.c_tensor, - grad_output.c_tensor, - weight.c_tensor, - running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - save_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - save_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - epsilon, - reservespace.c_tensor - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_cudnn_batch_norm_backward_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - grad_output: &Tensor, - weight: &Tensor, - running_mean: Option, - running_var: Option, - save_mean: Option, - save_var: Option, - epsilon: f64, - reservespace: &Tensor, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_cudnn_batch_norm_backward_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - out2.c_tensor, - self.c_tensor, - grad_output.c_tensor, - weight.c_tensor, - running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - save_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - save_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - epsilon, - reservespace.c_tensor - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_cudnn_batch_norm_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - out3: &Tensor, - weight: &Tensor, - bias: Option, - running_mean: Option, - running_var: Option, - training: bool, - exponential_average_factor: f64, - epsilon: f64, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg_cudnn_batch_norm_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - out2.c_tensor, - out3.c_tensor, - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if training { 1 } else { 0 }, - exponential_average_factor, - epsilon - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_cudnn_convolution( - &self, - weight: &Tensor, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - allow_tf32: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cudnn_convolution( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - padding.as_ptr(), - padding.len_i32(), - stride.as_ptr(), - stride.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups, - if benchmark { 1 } else { 0 }, - if deterministic { 1 } else { 0 }, - if allow_tf32 { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cudnn_convolution_add_relu, S: Into>( - &self, - weight: &Tensor, - z: &Tensor, - alpha: S, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cudnn_convolution_add_relu( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - z.c_tensor, - alpha.into().c_scalar, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cudnn_convolution_add_relu_out, S: Into>( - &self, - out: &Tensor, - weight: &Tensor, - z: &Tensor, - alpha: S, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cudnn_convolution_add_relu_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - z.c_tensor, - alpha.into().c_scalar, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cudnn_convolution_out( - &self, - out: &Tensor, - weight: &Tensor, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - allow_tf32: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cudnn_convolution_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - padding.as_ptr(), - padding.len_i32(), - stride.as_ptr(), - stride.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups, - if benchmark { 1 } else { 0 }, - if deterministic { 1 } else { 0 }, - if allow_tf32 { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cudnn_convolution_relu>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cudnn_convolution_relu( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cudnn_convolution_relu_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cudnn_convolution_relu_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cudnn_convolution_transpose( - &self, - weight: &Tensor, - padding: impl IntList, - output_padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - allow_tf32: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cudnn_convolution_transpose( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - padding.as_ptr(), - padding.len_i32(), - output_padding.as_ptr(), - output_padding.len_i32(), - stride.as_ptr(), - stride.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups, - if benchmark { 1 } else { 0 }, - if deterministic { 1 } else { 0 }, - if allow_tf32 { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cudnn_convolution_transpose_out( - &self, - out: &Tensor, - weight: &Tensor, - padding: impl IntList, - output_padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - allow_tf32: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cudnn_convolution_transpose_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - padding.as_ptr(), - padding.len_i32(), - output_padding.as_ptr(), - output_padding.len_i32(), - stride.as_ptr(), - stride.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups, - if benchmark { 1 } else { 0 }, - if deterministic { 1 } else { 0 }, - if allow_tf32 { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cudnn_grid_sampler(&self, grid: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cudnn_grid_sampler( - c_tensors.as_mut_ptr(), - self.c_tensor, - grid.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cudnn_grid_sampler_backward( - &self, - grid: &Tensor, - grad_output: &Tensor, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_cudnn_grid_sampler_backward( - c_tensors.as_mut_ptr(), - self.c_tensor, - grid.c_tensor, - grad_output.c_tensor - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_cudnn_grid_sampler_backward_out( - &self, - out0: &Tensor, - out1: &Tensor, - grid: &Tensor, - grad_output: &Tensor, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_cudnn_grid_sampler_backward_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - self.c_tensor, - grid.c_tensor, - grad_output.c_tensor - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_cudnn_grid_sampler_out( - &self, - out: &Tensor, - grid: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cudnn_grid_sampler_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - grid.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cudnn_is_acceptable(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_cudnn_is_acceptable(self.c_tensor)); - Ok(return_ != 0) - } - - pub fn f_cummax(&self, dim: i64) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_cummax(c_tensors.as_mut_ptr(), self.c_tensor, dim)); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_cummax_out( - &self, - values: &Tensor, - indices: &Tensor, - dim: i64, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_cummax_out( - c_tensors.as_mut_ptr(), - values.c_tensor, - indices.c_tensor, - self.c_tensor, - dim - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_cummaxmin_backward( - &self, - grad: &Tensor, - indices: &Tensor, - dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cummaxmin_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - self.c_tensor, - indices.c_tensor, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cummin(&self, dim: i64) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_cummin(c_tensors.as_mut_ptr(), self.c_tensor, dim)); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_cummin_out( - &self, - values: &Tensor, - indices: &Tensor, - dim: i64, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_cummin_out( - c_tensors.as_mut_ptr(), - values.c_tensor, - indices.c_tensor, - self.c_tensor, - dim - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_cumprod(&self, dim: i64, dtype: impl Into>) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cumprod( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cumprod_( - &mut self, - dim: i64, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cumprod_( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cumprod_backward( - &self, - grad: &Tensor, - dim: i64, - output: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cumprod_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - self.c_tensor, - dim, - output.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cumprod_out( - &self, - out: &Tensor, - dim: i64, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cumprod_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cumsum(&self, dim: i64, dtype: impl Into>) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cumsum( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cumsum_( - &mut self, - dim: i64, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cumsum_( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cumsum_out( - &self, - out: &Tensor, - dim: i64, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cumsum_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cumulative_trapezoid(y: &Tensor, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cumulative_trapezoid(c_tensors.as_mut_ptr(), y.c_tensor, dim)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_cumulative_trapezoid_x(y: &Tensor, x: &Tensor, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_cumulative_trapezoid_x( - c_tensors.as_mut_ptr(), - y.c_tensor, - x.c_tensor, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_data(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_data(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_deg2rad(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_deg2rad(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_deg2rad_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_deg2rad_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_deg2rad_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_deg2rad_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_dense_dim(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_dense_dim(self.c_tensor)); - Ok(return_) - } - - pub fn f_dequantize(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_dequantize(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_dequantize_self_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_dequantize_self_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_dequantize_tensors>(tensors: &[T]) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_dequantize_tensors( - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_dequantize_tensors_out>( - out: &[T], - tensors: &[T], - ) -> Result<(), TchError> { - unsafe_torch_err!(atg_dequantize_tensors_out( - ptr_list(out).as_ptr(), - out.len() as i32, - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - Ok(()) - } - - pub fn f_det(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_det(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_detach(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_detach(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_detach_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_detach_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_detach_copy(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_detach_copy(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_detach_copy_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_detach_copy_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_diag(&self, diagonal: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_diag(c_tensors.as_mut_ptr(), self.c_tensor, diagonal)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_diag_embed(&self, offset: i64, dim1: i64, dim2: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_diag_embed( - c_tensors.as_mut_ptr(), - self.c_tensor, - offset, - dim1, - dim2 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_diag_embed_out( - &self, - out: &Tensor, - offset: i64, - dim1: i64, - dim2: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_diag_embed_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - offset, - dim1, - dim2 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_diag_out(&self, out: &Tensor, diagonal: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_diag_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - diagonal - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_diagflat(&self, offset: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_diagflat(c_tensors.as_mut_ptr(), self.c_tensor, offset)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_diagonal(&self, offset: i64, dim1: i64, dim2: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_diagonal(c_tensors.as_mut_ptr(), self.c_tensor, offset, dim1, dim2)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_diagonal_backward( - grad_output: &Tensor, - input_sizes: impl IntList, - offset: i64, - dim1: i64, - dim2: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_diagonal_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - input_sizes.as_ptr(), - input_sizes.len_i32(), - offset, - dim1, - dim2 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_diagonal_backward_out( - out: &Tensor, - grad_output: &Tensor, - input_sizes: impl IntList, - offset: i64, - dim1: i64, - dim2: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_diagonal_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad_output.c_tensor, - input_sizes.as_ptr(), - input_sizes.len_i32(), - offset, - dim1, - dim2 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_diagonal_copy(&self, offset: i64, dim1: i64, dim2: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_diagonal_copy( - c_tensors.as_mut_ptr(), - self.c_tensor, - offset, - dim1, - dim2 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_diagonal_copy_out( - &self, - out: &Tensor, - offset: i64, - dim1: i64, - dim2: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_diagonal_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - offset, - dim1, - dim2 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_diagonal_scatter( - &self, - src: &Tensor, - offset: i64, - dim1: i64, - dim2: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_diagonal_scatter( - c_tensors.as_mut_ptr(), - self.c_tensor, - src.c_tensor, - offset, - dim1, - dim2 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_diagonal_scatter_out( - &self, - out: &Tensor, - src: &Tensor, - offset: i64, - dim1: i64, - dim2: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_diagonal_scatter_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - src.c_tensor, - offset, - dim1, - dim2 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_diff>( - &self, - n: i64, - dim: i64, - prepend: Option, - append: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_diff( - c_tensors.as_mut_ptr(), - self.c_tensor, - n, - dim, - prepend.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - append.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_diff_out>( - &self, - out: &Tensor, - n: i64, - dim: i64, - prepend: Option, - append: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_diff_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - n, - dim, - prepend.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - append.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_digamma(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_digamma(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_digamma_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_digamma_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_digamma_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_digamma_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_dist(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_dist(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_dist_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_dist_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_div(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_div(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_div_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_div_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_div_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_div_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_div_out_mode( - &self, - out: &Tensor, - other: &Tensor, - rounding_mode: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_div_out_mode( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor, - rounding_mode.as_ptr(), - rounding_mode.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_div_scalar>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_div_scalar( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_div_scalar_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_div_scalar_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_div_scalar_mode>( - &self, - other: S, - rounding_mode: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_div_scalar_mode( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar, - rounding_mode.as_ptr(), - rounding_mode.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_div_scalar_mode_>( - &mut self, - other: S, - rounding_mode: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_div_scalar_mode_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar, - rounding_mode.as_ptr(), - rounding_mode.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_div_scalar_mode_out>( - &self, - out: &Tensor, - other: S, - rounding_mode: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_div_scalar_mode_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar, - rounding_mode.as_ptr(), - rounding_mode.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_div_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_div_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_div_tensor_mode( - &self, - other: &Tensor, - rounding_mode: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_div_tensor_mode( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor, - rounding_mode.as_ptr(), - rounding_mode.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_div_tensor_mode_( - &mut self, - other: &Tensor, - rounding_mode: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_div_tensor_mode_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor, - rounding_mode.as_ptr(), - rounding_mode.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_divide(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_divide(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_divide_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_divide_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_divide_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_divide_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_divide_out_mode( - &self, - out: &Tensor, - other: &Tensor, - rounding_mode: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_divide_out_mode( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor, - rounding_mode.as_ptr(), - rounding_mode.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_divide_scalar>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_divide_scalar( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_divide_scalar_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_divide_scalar_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_divide_scalar_mode>( - &self, - other: S, - rounding_mode: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_divide_scalar_mode( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar, - rounding_mode.as_ptr(), - rounding_mode.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_divide_scalar_mode_>( - &mut self, - other: S, - rounding_mode: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_divide_scalar_mode_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar, - rounding_mode.as_ptr(), - rounding_mode.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_divide_tensor_mode( - &self, - other: &Tensor, - rounding_mode: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_divide_tensor_mode( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor, - rounding_mode.as_ptr(), - rounding_mode.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_divide_tensor_mode_( - &mut self, - other: &Tensor, - rounding_mode: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_divide_tensor_mode_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor, - rounding_mode.as_ptr(), - rounding_mode.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_dot(&self, tensor: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_dot(c_tensors.as_mut_ptr(), self.c_tensor, tensor.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_dot_out(&self, out: &Tensor, tensor: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_dot_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - tensor.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_dropout(&self, p: f64, train: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_dropout( - c_tensors.as_mut_ptr(), - self.c_tensor, - p, - if train { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_dropout_(&mut self, p: f64, train: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_dropout_( - c_tensors.as_mut_ptr(), - self.c_tensor, - p, - if train { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_dsplit(&self, sections: i64) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_dsplit(self.c_tensor, sections)); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_dsplit_array(&self, indices: impl IntList) -> Result, TchError> { - let c_tensors = - unsafe_torch_err!(atg_dsplit_array(self.c_tensor, indices.as_ptr(), indices.len_i32())); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_dstack>(tensors: &[T]) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_dstack( - c_tensors.as_mut_ptr(), - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_dstack_out>( - out: &Tensor, - tensors: &[T], - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_dstack_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_einsum>( - equation: &str, - tensors: &[T], - path: impl IntListOption, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_einsum( - c_tensors.as_mut_ptr(), - equation.as_ptr(), - equation.len() as i32, - ptr_list(tensors).as_ptr(), - tensors.len() as i32, - path.as_ptr(), - path.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_elu(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_elu(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_elu_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_elu_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_elu_backward>( - grad_output: &Tensor, - alpha: S, - scale: S, - input_scale: S, - is_result: bool, - self_or_result: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_elu_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - alpha.into().c_scalar, - scale.into().c_scalar, - input_scale.into().c_scalar, - if is_result { 1 } else { 0 }, - self_or_result.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_elu_backward_grad_input>( - grad_input: &Tensor, - grad_output: &Tensor, - alpha: S, - scale: S, - input_scale: S, - is_result: bool, - self_or_result: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_elu_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - alpha.into().c_scalar, - scale.into().c_scalar, - input_scale.into().c_scalar, - if is_result { 1 } else { 0 }, - self_or_result.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_elu_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_elu_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_embedding( - weight: &Tensor, - indices: &Tensor, - padding_idx: i64, - scale_grad_by_freq: bool, - sparse: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_embedding( - c_tensors.as_mut_ptr(), - weight.c_tensor, - indices.c_tensor, - padding_idx, - if scale_grad_by_freq { 1 } else { 0 }, - if sparse { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_embedding_backward( - grad: &Tensor, - indices: &Tensor, - num_weights: i64, - padding_idx: i64, - scale_grad_by_freq: bool, - sparse: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_embedding_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - indices.c_tensor, - num_weights, - padding_idx, - if scale_grad_by_freq { 1 } else { 0 }, - if sparse { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_embedding_bag>( - weight: &Tensor, - indices: &Tensor, - offsets: &Tensor, - scale_grad_by_freq: bool, - mode: i64, - sparse: bool, - per_sample_weights: Option, - include_last_offset: bool, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg_embedding_bag( - c_tensors.as_mut_ptr(), - weight.c_tensor, - indices.c_tensor, - offsets.c_tensor, - if scale_grad_by_freq { 1 } else { 0 }, - mode, - if sparse { 1 } else { 0 }, - per_sample_weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if include_last_offset { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_embedding_bag_padding_idx>( - weight: &Tensor, - indices: &Tensor, - offsets: &Tensor, - scale_grad_by_freq: bool, - mode: i64, - sparse: bool, - per_sample_weights: Option, - include_last_offset: bool, - padding_idx: impl Into>, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let padding_idx = padding_idx.into(); - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg_embedding_bag_padding_idx( - c_tensors.as_mut_ptr(), - weight.c_tensor, - indices.c_tensor, - offsets.c_tensor, - if scale_grad_by_freq { 1 } else { 0 }, - mode, - if sparse { 1 } else { 0 }, - per_sample_weights.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if include_last_offset { 1 } else { 0 }, - padding_idx.unwrap_or(0i64), - padding_idx.is_none() as i8 - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_embedding_dense_backward( - grad_output: &Tensor, - indices: &Tensor, - num_weights: i64, - padding_idx: i64, - scale_grad_by_freq: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_embedding_dense_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - indices.c_tensor, - num_weights, - padding_idx, - if scale_grad_by_freq { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_embedding_dense_backward_out( - out: &Tensor, - grad_output: &Tensor, - indices: &Tensor, - num_weights: i64, - padding_idx: i64, - scale_grad_by_freq: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_embedding_dense_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad_output.c_tensor, - indices.c_tensor, - num_weights, - padding_idx, - if scale_grad_by_freq { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_embedding_out( - out: &Tensor, - weight: &Tensor, - indices: &Tensor, - padding_idx: i64, - scale_grad_by_freq: bool, - sparse: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_embedding_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - weight.c_tensor, - indices.c_tensor, - padding_idx, - if scale_grad_by_freq { 1 } else { 0 }, - if sparse { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_embedding_renorm( - &self, - indices: &Tensor, - max_norm: f64, - norm_type: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_embedding_renorm( - c_tensors.as_mut_ptr(), - self.c_tensor, - indices.c_tensor, - max_norm, - norm_type - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_embedding_renorm_( - &mut self, - indices: &Tensor, - max_norm: f64, - norm_type: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_embedding_renorm_( - c_tensors.as_mut_ptr(), - self.c_tensor, - indices.c_tensor, - max_norm, - norm_type - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_embedding_renorm_out( - &self, - out: &Tensor, - indices: &Tensor, - max_norm: f64, - norm_type: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_embedding_renorm_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - indices.c_tensor, - max_norm, - norm_type - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_embedding_sparse_backward( - grad: &Tensor, - indices: &Tensor, - num_weights: i64, - padding_idx: i64, - scale_grad_by_freq: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_embedding_sparse_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - indices.c_tensor, - num_weights, - padding_idx, - if scale_grad_by_freq { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_empty(size: impl IntList, options: (Kind, Device)) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_empty( - c_tensors.as_mut_ptr(), - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_empty_like(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_empty_like(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_empty_like_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_empty_like_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_empty_out(out: &Tensor, size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_empty_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_empty_permuted( - size: impl IntList, - physical_layout: impl IntList, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_empty_permuted( - c_tensors.as_mut_ptr(), - size.as_ptr(), - size.len_i32(), - physical_layout.as_ptr(), - physical_layout.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_empty_permuted_out( - out: &Tensor, - size: impl IntList, - physical_layout: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_empty_permuted_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - size.as_ptr(), - size.len_i32(), - physical_layout.as_ptr(), - physical_layout.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_empty_quantized( - size: impl IntList, - qtensor: &Tensor, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_empty_quantized( - c_tensors.as_mut_ptr(), - size.as_ptr(), - size.len_i32(), - qtensor.c_tensor, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_empty_quantized_out( - out: &Tensor, - size: impl IntList, - qtensor: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_empty_quantized_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - size.as_ptr(), - size.len_i32(), - qtensor.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_empty_strided( - size: impl IntList, - stride: impl IntList, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_empty_strided( - c_tensors.as_mut_ptr(), - size.as_ptr(), - size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_empty_strided_out( - out: &Tensor, - size: impl IntList, - stride: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_empty_strided_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - size.as_ptr(), - size.len_i32(), - stride.as_ptr(), - stride.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_eq>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_eq(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_eq_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_eq_(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_eq_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_eq_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_eq_tensor(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_eq_tensor(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_eq_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_eq_tensor_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_eq_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_eq_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_equal(&self, other: &Tensor) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_equal(self.c_tensor, other.c_tensor)); - Ok(return_ != 0) - } - - pub fn f_erf(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_erf(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_erf_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_erf_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_erf_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_erf_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_erfc(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_erfc(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_erfc_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_erfc_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_erfc_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_erfc_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_erfinv(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_erfinv(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_erfinv_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_erfinv_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_erfinv_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_erfinv_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_exp(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_exp(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_exp2(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_exp2(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_exp2_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_exp2_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_exp2_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_exp2_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_exp_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_exp_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_exp_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_exp_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_expand(&self, size: impl IntList, implicit: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_expand( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32(), - if implicit { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_expand_as(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_expand_as(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_expand_copy(&self, size: impl IntList, implicit: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_expand_copy( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32(), - if implicit { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_expand_copy_out( - &self, - out: &Tensor, - size: impl IntList, - implicit: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_expand_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - size.as_ptr(), - size.len_i32(), - if implicit { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_expm1(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_expm1(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_expm1_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_expm1_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_expm1_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_expm1_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_exponential(&self, lambd: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_exponential(c_tensors.as_mut_ptr(), self.c_tensor, lambd)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_exponential_(&mut self, lambd: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_exponential_(c_tensors.as_mut_ptr(), self.c_tensor, lambd)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_exponential_out(&self, out: &Tensor, lambd: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_exponential_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - lambd - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_eye(n: i64, options: (Kind, Device)) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_eye(c_tensors.as_mut_ptr(), n, options.0.c_int(), options.1.c_int())); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_eye_m(n: i64, m: i64, options: (Kind, Device)) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_eye_m( - c_tensors.as_mut_ptr(), - n, - m, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_eye_m_out(out: &Tensor, n: i64, m: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_eye_m_out(c_tensors.as_mut_ptr(), out.c_tensor, n, m)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_eye_out(out: &Tensor, n: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_eye_out(c_tensors.as_mut_ptr(), out.c_tensor, n)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fake_quantize_per_channel_affine( - &self, - scale: &Tensor, - zero_point: &Tensor, - axis: i64, - quant_min: i64, - quant_max: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fake_quantize_per_channel_affine( - c_tensors.as_mut_ptr(), - self.c_tensor, - scale.c_tensor, - zero_point.c_tensor, - axis, - quant_min, - quant_max - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fake_quantize_per_channel_affine_cachemask( - &self, - scale: &Tensor, - zero_point: &Tensor, - axis: i64, - quant_min: i64, - quant_max: i64, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_fake_quantize_per_channel_affine_cachemask( - c_tensors.as_mut_ptr(), - self.c_tensor, - scale.c_tensor, - zero_point.c_tensor, - axis, - quant_min, - quant_max - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_fake_quantize_per_channel_affine_cachemask_backward( - grad: &Tensor, - mask: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fake_quantize_per_channel_affine_cachemask_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - mask.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fake_quantize_per_channel_affine_cachemask_out( - &self, - out0: &Tensor, - out1: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - axis: i64, - quant_min: i64, - quant_max: i64, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_fake_quantize_per_channel_affine_cachemask_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - self.c_tensor, - scale.c_tensor, - zero_point.c_tensor, - axis, - quant_min, - quant_max - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_fake_quantize_per_tensor_affine( - &self, - scale: f64, - zero_point: i64, - quant_min: i64, - quant_max: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fake_quantize_per_tensor_affine( - c_tensors.as_mut_ptr(), - self.c_tensor, - scale, - zero_point, - quant_min, - quant_max - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fake_quantize_per_tensor_affine_cachemask( - &self, - scale: f64, - zero_point: i64, - quant_min: i64, - quant_max: i64, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_fake_quantize_per_tensor_affine_cachemask( - c_tensors.as_mut_ptr(), - self.c_tensor, - scale, - zero_point, - quant_min, - quant_max - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_fake_quantize_per_tensor_affine_cachemask_backward( - grad: &Tensor, - mask: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fake_quantize_per_tensor_affine_cachemask_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - mask.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fake_quantize_per_tensor_affine_cachemask_out( - &self, - out0: &Tensor, - out1: &Tensor, - scale: f64, - zero_point: i64, - quant_min: i64, - quant_max: i64, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_fake_quantize_per_tensor_affine_cachemask_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - self.c_tensor, - scale, - zero_point, - quant_min, - quant_max - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_fake_quantize_per_tensor_affine_tensor_qparams( - &self, - scale: &Tensor, - zero_point: &Tensor, - quant_min: i64, - quant_max: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fake_quantize_per_tensor_affine_tensor_qparams( - c_tensors.as_mut_ptr(), - self.c_tensor, - scale.c_tensor, - zero_point.c_tensor, - quant_min, - quant_max - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_feature_alpha_dropout(&self, p: f64, train: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_feature_alpha_dropout( - c_tensors.as_mut_ptr(), - self.c_tensor, - p, - if train { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_feature_alpha_dropout_(&mut self, p: f64, train: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_feature_alpha_dropout_( - c_tensors.as_mut_ptr(), - self.c_tensor, - p, - if train { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_feature_dropout(&self, p: f64, train: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_feature_dropout( - c_tensors.as_mut_ptr(), - self.c_tensor, - p, - if train { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_feature_dropout_(&mut self, p: f64, train: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_feature_dropout_( - c_tensors.as_mut_ptr(), - self.c_tensor, - p, - if train { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_fft( - &self, - n: impl Into>, - dim: i64, - norm: &str, - ) -> Result { - let n = n.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_fft( - c_tensors.as_mut_ptr(), - self.c_tensor, - n.unwrap_or(0i64), - n.is_none() as i8, - dim, - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_fft2( - &self, - s: impl IntListOption, - dim: impl IntList, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_fft2( - c_tensors.as_mut_ptr(), - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_fft2_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntList, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_fft2_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_fft_out( - &self, - out: &Tensor, - n: impl Into>, - dim: i64, - norm: &str, - ) -> Result { - let n = n.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_fft_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - n.unwrap_or(0i64), - n.is_none() as i8, - dim, - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_fftfreq(n: i64, d: f64, options: (Kind, Device)) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_fftfreq( - c_tensors.as_mut_ptr(), - n, - d, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_fftfreq_out(out: &Tensor, n: i64, d: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_fftfreq_out(c_tensors.as_mut_ptr(), out.c_tensor, n, d)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_fftn( - &self, - s: impl IntListOption, - dim: impl IntListOption, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_fftn( - c_tensors.as_mut_ptr(), - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_fftn_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntListOption, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_fftn_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_fftshift(&self, dim: impl IntListOption) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_fftshift( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_hfft( - &self, - n: impl Into>, - dim: i64, - norm: &str, - ) -> Result { - let n = n.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_hfft( - c_tensors.as_mut_ptr(), - self.c_tensor, - n.unwrap_or(0i64), - n.is_none() as i8, - dim, - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_hfft2( - &self, - s: impl IntListOption, - dim: impl IntList, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_hfft2( - c_tensors.as_mut_ptr(), - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_hfft2_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntList, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_hfft2_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_hfft_out( - &self, - out: &Tensor, - n: impl Into>, - dim: i64, - norm: &str, - ) -> Result { - let n = n.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_hfft_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - n.unwrap_or(0i64), - n.is_none() as i8, - dim, - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_hfftn( - &self, - s: impl IntListOption, - dim: impl IntListOption, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_hfftn( - c_tensors.as_mut_ptr(), - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_hfftn_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntListOption, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_hfftn_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_ifft( - &self, - n: impl Into>, - dim: i64, - norm: &str, - ) -> Result { - let n = n.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_ifft( - c_tensors.as_mut_ptr(), - self.c_tensor, - n.unwrap_or(0i64), - n.is_none() as i8, - dim, - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_ifft2( - &self, - s: impl IntListOption, - dim: impl IntList, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_ifft2( - c_tensors.as_mut_ptr(), - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_ifft2_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntList, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_ifft2_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_ifft_out( - &self, - out: &Tensor, - n: impl Into>, - dim: i64, - norm: &str, - ) -> Result { - let n = n.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_ifft_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - n.unwrap_or(0i64), - n.is_none() as i8, - dim, - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_ifftn( - &self, - s: impl IntListOption, - dim: impl IntListOption, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_ifftn( - c_tensors.as_mut_ptr(), - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_ifftn_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntListOption, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_ifftn_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_ifftshift(&self, dim: impl IntListOption) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_ifftshift( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_ihfft( - &self, - n: impl Into>, - dim: i64, - norm: &str, - ) -> Result { - let n = n.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_ihfft( - c_tensors.as_mut_ptr(), - self.c_tensor, - n.unwrap_or(0i64), - n.is_none() as i8, - dim, - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_ihfft2( - &self, - s: impl IntListOption, - dim: impl IntList, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_ihfft2( - c_tensors.as_mut_ptr(), - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_ihfft2_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntList, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_ihfft2_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_ihfft_out( - &self, - out: &Tensor, - n: impl Into>, - dim: i64, - norm: &str, - ) -> Result { - let n = n.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_ihfft_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - n.unwrap_or(0i64), - n.is_none() as i8, - dim, - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_ihfftn( - &self, - s: impl IntListOption, - dim: impl IntListOption, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_ihfftn( - c_tensors.as_mut_ptr(), - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_ihfftn_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntListOption, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_ihfftn_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_irfft( - &self, - n: impl Into>, - dim: i64, - norm: &str, - ) -> Result { - let n = n.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_irfft( - c_tensors.as_mut_ptr(), - self.c_tensor, - n.unwrap_or(0i64), - n.is_none() as i8, - dim, - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_irfft2( - &self, - s: impl IntListOption, - dim: impl IntList, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_irfft2( - c_tensors.as_mut_ptr(), - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_irfft2_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntList, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_irfft2_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_irfft_out( - &self, - out: &Tensor, - n: impl Into>, - dim: i64, - norm: &str, - ) -> Result { - let n = n.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_irfft_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - n.unwrap_or(0i64), - n.is_none() as i8, - dim, - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_irfftn( - &self, - s: impl IntListOption, - dim: impl IntListOption, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_irfftn( - c_tensors.as_mut_ptr(), - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_irfftn_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntListOption, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_irfftn_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_rfft( - &self, - n: impl Into>, - dim: i64, - norm: &str, - ) -> Result { - let n = n.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_rfft( - c_tensors.as_mut_ptr(), - self.c_tensor, - n.unwrap_or(0i64), - n.is_none() as i8, - dim, - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_rfft2( - &self, - s: impl IntListOption, - dim: impl IntList, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_rfft2( - c_tensors.as_mut_ptr(), - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_rfft2_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntList, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_rfft2_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_rfft_out( - &self, - out: &Tensor, - n: impl Into>, - dim: i64, - norm: &str, - ) -> Result { - let n = n.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_rfft_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - n.unwrap_or(0i64), - n.is_none() as i8, - dim, - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_rfftfreq(n: i64, d: f64, options: (Kind, Device)) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_rfftfreq( - c_tensors.as_mut_ptr(), - n, - d, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_rfftfreq_out(out: &Tensor, n: i64, d: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_rfftfreq_out(c_tensors.as_mut_ptr(), out.c_tensor, n, d)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_rfftn( - &self, - s: impl IntListOption, - dim: impl IntListOption, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_rfftn( - c_tensors.as_mut_ptr(), - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fft_rfftn_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntListOption, - norm: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fft_rfftn_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - s.as_ptr(), - s.len_i32(), - dim.as_ptr(), - dim.len_i32(), - norm.as_ptr(), - norm.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fill>(&self, value: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fill(c_tensors.as_mut_ptr(), self.c_tensor, value.into().c_scalar)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fill_>(&mut self, value: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fill_(c_tensors.as_mut_ptr(), self.c_tensor, value.into().c_scalar)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fill_diagonal_>( - &mut self, - fill_value: S, - wrap: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fill_diagonal_( - c_tensors.as_mut_ptr(), - self.c_tensor, - fill_value.into().c_scalar, - if wrap { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fill_scalar_out>( - &self, - out: &Tensor, - value: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fill_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - value.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fill_tensor(&self, value: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fill_tensor(c_tensors.as_mut_ptr(), self.c_tensor, value.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fill_tensor_(&mut self, value: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fill_tensor_(c_tensors.as_mut_ptr(), self.c_tensor, value.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fill_tensor_out(&self, out: &Tensor, value: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fill_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - value.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fix(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fix(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fix_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fix_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fix_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fix_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_flatten(&self, start_dim: i64, end_dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_flatten(c_tensors.as_mut_ptr(), self.c_tensor, start_dim, end_dim)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_flatten_dense_tensors>(tensors: &[T]) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_flatten_dense_tensors( - c_tensors.as_mut_ptr(), - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_flip(&self, dims: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_flip( - c_tensors.as_mut_ptr(), - self.c_tensor, - dims.as_ptr(), - dims.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_flip_out(&self, out: &Tensor, dims: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_flip_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dims.as_ptr(), - dims.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fliplr(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fliplr(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_flipud(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_flipud(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_float_power(&self, exponent: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_float_power( - c_tensors.as_mut_ptr(), - self.c_tensor, - exponent.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_float_power_>(&mut self, exponent: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_float_power_( - c_tensors.as_mut_ptr(), - self.c_tensor, - exponent.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_float_power_scalar>( - self_scalar: S, - exponent: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_float_power_scalar( - c_tensors.as_mut_ptr(), - self_scalar.into().c_scalar, - exponent.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_float_power_scalar_out>( - out: &Tensor, - self_scalar: S, - exponent: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_float_power_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self_scalar.into().c_scalar, - exponent.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_float_power_tensor_(&mut self, exponent: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_float_power_tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - exponent.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_float_power_tensor_scalar>( - &self, - exponent: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_float_power_tensor_scalar( - c_tensors.as_mut_ptr(), - self.c_tensor, - exponent.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_float_power_tensor_scalar_out>( - &self, - out: &Tensor, - exponent: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_float_power_tensor_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - exponent.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_float_power_tensor_tensor_out( - &self, - out: &Tensor, - exponent: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_float_power_tensor_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - exponent.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_floor(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_floor(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_floor_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_floor_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_floor_divide(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_floor_divide(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_floor_divide_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_floor_divide_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_floor_divide_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_floor_divide_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_floor_divide_scalar>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_floor_divide_scalar( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_floor_divide_scalar_>( - &mut self, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_floor_divide_scalar_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_floor_divide_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_floor_divide_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_floor_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_floor_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fmax(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fmax(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fmax_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fmax_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fmin(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fmin(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fmin_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fmin_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fmod>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fmod(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fmod_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fmod_(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fmod_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fmod_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fmod_tensor(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fmod_tensor(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fmod_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fmod_tensor_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fmod_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fmod_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_frac(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_frac(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_frac_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_frac_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_frac_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_frac_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fractional_max_pool2d( - &self, - kernel_size: impl IntList, - output_size: impl IntList, - random_samples: &Tensor, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_fractional_max_pool2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - output_size.as_ptr(), - output_size.len_i32(), - random_samples.c_tensor - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_fractional_max_pool2d_backward( - &self, - grad_output: &Tensor, - kernel_size: impl IntList, - output_size: impl IntList, - indices: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fractional_max_pool2d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - output_size.as_ptr(), - output_size.len_i32(), - indices.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fractional_max_pool2d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - kernel_size: impl IntList, - output_size: impl IntList, - indices: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fractional_max_pool2d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - output_size.as_ptr(), - output_size.len_i32(), - indices.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fractional_max_pool2d_output( - &self, - output: &Tensor, - indices: &Tensor, - kernel_size: impl IntList, - output_size: impl IntList, - random_samples: &Tensor, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_fractional_max_pool2d_output( - c_tensors.as_mut_ptr(), - output.c_tensor, - indices.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - output_size.as_ptr(), - output_size.len_i32(), - random_samples.c_tensor - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_fractional_max_pool3d( - &self, - kernel_size: impl IntList, - output_size: impl IntList, - random_samples: &Tensor, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_fractional_max_pool3d( - c_tensors.as_mut_ptr(), - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - output_size.as_ptr(), - output_size.len_i32(), - random_samples.c_tensor - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_fractional_max_pool3d_backward( - &self, - grad_output: &Tensor, - kernel_size: impl IntList, - output_size: impl IntList, - indices: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fractional_max_pool3d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - output_size.as_ptr(), - output_size.len_i32(), - indices.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fractional_max_pool3d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - kernel_size: impl IntList, - output_size: impl IntList, - indices: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fractional_max_pool3d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - output_size.as_ptr(), - output_size.len_i32(), - indices.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fractional_max_pool3d_output( - &self, - output: &Tensor, - indices: &Tensor, - kernel_size: impl IntList, - output_size: impl IntList, - random_samples: &Tensor, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_fractional_max_pool3d_output( - c_tensors.as_mut_ptr(), - output.c_tensor, - indices.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - output_size.as_ptr(), - output_size.len_i32(), - random_samples.c_tensor - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_frexp(&self) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_frexp(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_frexp_tensor_out( - &self, - mantissa: &Tensor, - exponent: &Tensor, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_frexp_tensor_out( - c_tensors.as_mut_ptr(), - mantissa.c_tensor, - exponent.c_tensor, - self.c_tensor - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_frobenius_norm(&self, dim: impl IntList, keepdim: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_frobenius_norm( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_frobenius_norm_out( - &self, - out: &Tensor, - dim: impl IntList, - keepdim: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_frobenius_norm_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_from_file( - filename: &str, - shared: bool, - size: impl Into>, - options: (Kind, Device), - ) -> Result { - let size = size.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_from_file( - c_tensors.as_mut_ptr(), - filename.as_ptr(), - filename.len() as i32, - if shared { 1 } else { 0 }, - size.unwrap_or(0i64), - size.is_none() as i8, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_from_file_out( - out: &Tensor, - filename: &str, - shared: bool, - size: impl Into>, - ) -> Result { - let size = size.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_from_file_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - filename.as_ptr(), - filename.len() as i32, - if shared { 1 } else { 0 }, - size.unwrap_or(0i64), - size.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_full>( - size: impl IntList, - fill_value: S, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_full( - c_tensors.as_mut_ptr(), - size.as_ptr(), - size.len_i32(), - fill_value.into().c_scalar, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_full_like>(&self, fill_value: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_full_like( - c_tensors.as_mut_ptr(), - self.c_tensor, - fill_value.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_full_like_out>( - &self, - out: &Tensor, - fill_value: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_full_like_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - fill_value.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_full_out>( - out: &Tensor, - size: impl IntList, - fill_value: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_full_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - size.as_ptr(), - size.len_i32(), - fill_value.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_fused_moving_avg_obs_fake_quant( - &self, - observer_on: &Tensor, - fake_quant_on: &Tensor, - running_min: &Tensor, - running_max: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - averaging_const: f64, - quant_min: i64, - quant_max: i64, - ch_axis: i64, - per_row_fake_quant: bool, - symmetric_quant: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_fused_moving_avg_obs_fake_quant( - c_tensors.as_mut_ptr(), - self.c_tensor, - observer_on.c_tensor, - fake_quant_on.c_tensor, - running_min.c_tensor, - running_max.c_tensor, - scale.c_tensor, - zero_point.c_tensor, - averaging_const, - quant_min, - quant_max, - ch_axis, - if per_row_fake_quant { 1 } else { 0 }, - if symmetric_quant { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_gather( - &self, - dim: i64, - index: &Tensor, - sparse_grad: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_gather( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - if sparse_grad { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_gather_backward( - &self, - grad: &Tensor, - dim: i64, - index: &Tensor, - sparse_grad: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_gather_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - self.c_tensor, - dim, - index.c_tensor, - if sparse_grad { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_gather_out( - &self, - out: &Tensor, - dim: i64, - index: &Tensor, - sparse_grad: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_gather_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - index.c_tensor, - if sparse_grad { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_gcd(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_gcd(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_gcd_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_gcd_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_gcd_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_gcd_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ge>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ge(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ge_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ge_(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ge_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ge_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ge_tensor(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ge_tensor(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ge_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ge_tensor_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ge_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ge_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_gelu(&self, approximate: &str) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_gelu( - c_tensors.as_mut_ptr(), - self.c_tensor, - approximate.as_ptr(), - approximate.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_gelu_(&mut self, approximate: &str) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_gelu_( - c_tensors.as_mut_ptr(), - self.c_tensor, - approximate.as_ptr(), - approximate.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_gelu_backward( - &self, - grad_output: &Tensor, - approximate: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_gelu_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - approximate.as_ptr(), - approximate.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_gelu_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - approximate: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_gelu_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - approximate.as_ptr(), - approximate.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_gelu_out(&self, out: &Tensor, approximate: &str) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_gelu_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - approximate.as_ptr(), - approximate.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_geometric(&self, p: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_geometric(c_tensors.as_mut_ptr(), self.c_tensor, p)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_geometric_(&mut self, p: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_geometric_(c_tensors.as_mut_ptr(), self.c_tensor, p)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_geometric_out(&self, out: &Tensor, p: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_geometric_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - p - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_geqrf(&self) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_geqrf(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_geqrf_a(&self, a: &Tensor, tau: &Tensor) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_geqrf_a( - c_tensors.as_mut_ptr(), - a.c_tensor, - tau.c_tensor, - self.c_tensor - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_ger(&self, vec2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ger(c_tensors.as_mut_ptr(), self.c_tensor, vec2.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ger_out(&self, out: &Tensor, vec2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ger_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - vec2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_glu(&self, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_glu(c_tensors.as_mut_ptr(), self.c_tensor, dim)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_glu_backward(&self, grad_output: &Tensor, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_glu_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_glu_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_glu_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_glu_backward_jvp( - grad_x: &Tensor, - grad_glu: &Tensor, - x: &Tensor, - dgrad_glu: &Tensor, - dx: &Tensor, - dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_glu_backward_jvp( - c_tensors.as_mut_ptr(), - grad_x.c_tensor, - grad_glu.c_tensor, - x.c_tensor, - dgrad_glu.c_tensor, - dx.c_tensor, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_glu_backward_jvp_out( - out: &Tensor, - grad_x: &Tensor, - grad_glu: &Tensor, - x: &Tensor, - dgrad_glu: &Tensor, - dx: &Tensor, - dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_glu_backward_jvp_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad_x.c_tensor, - grad_glu.c_tensor, - x.c_tensor, - dgrad_glu.c_tensor, - dx.c_tensor, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_glu_jvp(glu: &Tensor, x: &Tensor, dx: &Tensor, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_glu_jvp( - c_tensors.as_mut_ptr(), - glu.c_tensor, - x.c_tensor, - dx.c_tensor, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_glu_jvp_out( - out: &Tensor, - glu: &Tensor, - x: &Tensor, - dx: &Tensor, - dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_glu_jvp_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - glu.c_tensor, - x.c_tensor, - dx.c_tensor, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_glu_out(&self, out: &Tensor, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_glu_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor, dim)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_grad(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_grad(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_greater>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_greater( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_greater_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_greater_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_greater_equal>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_greater_equal( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_greater_equal_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_greater_equal_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_greater_equal_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_greater_equal_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_greater_equal_tensor(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_greater_equal_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_greater_equal_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_greater_equal_tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_greater_equal_tensor_out( - &self, - out: &Tensor, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_greater_equal_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_greater_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_greater_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_greater_tensor(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_greater_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_greater_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_greater_tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_greater_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_greater_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_grid_sampler( - &self, - grid: &Tensor, - interpolation_mode: i64, - padding_mode: i64, - align_corners: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_grid_sampler( - c_tensors.as_mut_ptr(), - self.c_tensor, - grid.c_tensor, - interpolation_mode, - padding_mode, - if align_corners { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_grid_sampler_2d( - &self, - grid: &Tensor, - interpolation_mode: i64, - padding_mode: i64, - align_corners: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_grid_sampler_2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - grid.c_tensor, - interpolation_mode, - padding_mode, - if align_corners { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_grid_sampler_2d_out( - &self, - out: &Tensor, - grid: &Tensor, - interpolation_mode: i64, - padding_mode: i64, - align_corners: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_grid_sampler_2d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - grid.c_tensor, - interpolation_mode, - padding_mode, - if align_corners { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_grid_sampler_3d( - &self, - grid: &Tensor, - interpolation_mode: i64, - padding_mode: i64, - align_corners: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_grid_sampler_3d( - c_tensors.as_mut_ptr(), - self.c_tensor, - grid.c_tensor, - interpolation_mode, - padding_mode, - if align_corners { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_grid_sampler_3d_out( - &self, - out: &Tensor, - grid: &Tensor, - interpolation_mode: i64, - padding_mode: i64, - align_corners: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_grid_sampler_3d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - grid.c_tensor, - interpolation_mode, - padding_mode, - if align_corners { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_group_norm>( - &self, - num_groups: i64, - weight: Option, - bias: Option, - eps: f64, - cudnn_enabled: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_group_norm( - c_tensors.as_mut_ptr(), - self.c_tensor, - num_groups, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - eps, - if cudnn_enabled { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_gru>( - &self, - hx: &Tensor, - params: &[T], - has_biases: bool, - num_layers: i64, - dropout: f64, - train: bool, - bidirectional: bool, - batch_first: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_gru( - c_tensors.as_mut_ptr(), - self.c_tensor, - hx.c_tensor, - ptr_list(params).as_ptr(), - params.len() as i32, - if has_biases { 1 } else { 0 }, - num_layers, - dropout, - if train { 1 } else { 0 }, - if bidirectional { 1 } else { 0 }, - if batch_first { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_gru_cell>( - &self, - hx: &Tensor, - w_ih: &Tensor, - w_hh: &Tensor, - b_ih: Option, - b_hh: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_gru_cell( - c_tensors.as_mut_ptr(), - self.c_tensor, - hx.c_tensor, - w_ih.c_tensor, - w_hh.c_tensor, - b_ih.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - b_hh.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_gru_data>( - data: &Tensor, - batch_sizes: &Tensor, - hx: &Tensor, - params: &[T], - has_biases: bool, - num_layers: i64, - dropout: f64, - train: bool, - bidirectional: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_gru_data( - c_tensors.as_mut_ptr(), - data.c_tensor, - batch_sizes.c_tensor, - hx.c_tensor, - ptr_list(params).as_ptr(), - params.len() as i32, - if has_biases { 1 } else { 0 }, - num_layers, - dropout, - if train { 1 } else { 0 }, - if bidirectional { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_gt>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_gt(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_gt_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_gt_(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_gt_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_gt_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_gt_tensor(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_gt_tensor(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_gt_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_gt_tensor_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_gt_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_gt_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hamming_window( - window_length: i64, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hamming_window( - c_tensors.as_mut_ptr(), - window_length, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hamming_window_out(out: &Tensor, window_length: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hamming_window_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - window_length - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hamming_window_periodic( - window_length: i64, - periodic: bool, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hamming_window_periodic( - c_tensors.as_mut_ptr(), - window_length, - if periodic { 1 } else { 0 }, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hamming_window_periodic_alpha( - window_length: i64, - periodic: bool, - alpha: f64, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hamming_window_periodic_alpha( - c_tensors.as_mut_ptr(), - window_length, - if periodic { 1 } else { 0 }, - alpha, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hamming_window_periodic_alpha_beta( - window_length: i64, - periodic: bool, - alpha: f64, - beta: f64, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hamming_window_periodic_alpha_beta( - c_tensors.as_mut_ptr(), - window_length, - if periodic { 1 } else { 0 }, - alpha, - beta, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hamming_window_periodic_alpha_beta_out( - out: &Tensor, - window_length: i64, - periodic: bool, - alpha: f64, - beta: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hamming_window_periodic_alpha_beta_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - window_length, - if periodic { 1 } else { 0 }, - alpha, - beta - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hamming_window_periodic_alpha_out( - out: &Tensor, - window_length: i64, - periodic: bool, - alpha: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hamming_window_periodic_alpha_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - window_length, - if periodic { 1 } else { 0 }, - alpha - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hamming_window_periodic_out( - out: &Tensor, - window_length: i64, - periodic: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hamming_window_periodic_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - window_length, - if periodic { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hann_window(window_length: i64, options: (Kind, Device)) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hann_window( - c_tensors.as_mut_ptr(), - window_length, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hann_window_out(out: &Tensor, window_length: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hann_window_out(c_tensors.as_mut_ptr(), out.c_tensor, window_length)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hann_window_periodic( - window_length: i64, - periodic: bool, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hann_window_periodic( - c_tensors.as_mut_ptr(), - window_length, - if periodic { 1 } else { 0 }, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hann_window_periodic_out( - out: &Tensor, - window_length: i64, - periodic: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hann_window_periodic_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - window_length, - if periodic { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hardshrink(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hardshrink(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hardshrink_backward>( - &self, - grad_out: &Tensor, - lambd: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hardshrink_backward( - c_tensors.as_mut_ptr(), - grad_out.c_tensor, - self.c_tensor, - lambd.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hardshrink_backward_grad_input>( - &self, - grad_input: &Tensor, - grad_out: &Tensor, - lambd: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hardshrink_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_out.c_tensor, - self.c_tensor, - lambd.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hardshrink_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hardshrink_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hardsigmoid(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hardsigmoid(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hardsigmoid_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hardsigmoid_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hardsigmoid_backward(&self, grad_output: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hardsigmoid_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hardsigmoid_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hardsigmoid_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hardsigmoid_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hardsigmoid_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hardswish(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hardswish(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hardswish_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hardswish_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hardswish_backward(&self, grad_output: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hardswish_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hardswish_backward_out( - &self, - out: &Tensor, - grad_output: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hardswish_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad_output.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hardswish_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hardswish_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hardtanh(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hardtanh(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hardtanh_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hardtanh_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hardtanh_backward>( - &self, - grad_output: &Tensor, - min_val: S, - max_val: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hardtanh_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - min_val.into().c_scalar, - max_val.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hardtanh_backward_grad_input>( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - min_val: S, - max_val: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hardtanh_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - min_val.into().c_scalar, - max_val.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hardtanh_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hardtanh_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hash_tensor( - &self, - dim: impl IntList, - keepdim: bool, - mode: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hash_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 }, - mode - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hash_tensor_out( - &self, - out: &Tensor, - dim: impl IntList, - keepdim: bool, - mode: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hash_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 }, - mode - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_heaviside(&self, values: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_heaviside(c_tensors.as_mut_ptr(), self.c_tensor, values.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_heaviside_(&mut self, values: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_heaviside_(c_tensors.as_mut_ptr(), self.c_tensor, values.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_heaviside_out(&self, out: &Tensor, values: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_heaviside_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - values.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hinge_embedding_loss( - &self, - target: &Tensor, - margin: f64, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hinge_embedding_loss( - c_tensors.as_mut_ptr(), - self.c_tensor, - target.c_tensor, - margin, - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_histc(&self, bins: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_histc(c_tensors.as_mut_ptr(), self.c_tensor, bins)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_histc_out(&self, out: &Tensor, bins: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_histc_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor, bins)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_histogram>( - &self, - bins: &Tensor, - weight: Option, - density: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_histogram( - c_tensors.as_mut_ptr(), - self.c_tensor, - bins.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if density { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_histogram_bin_ct>( - &self, - bins: i64, - range: impl DoubleList, - weight: Option, - density: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_histogram_bin_ct( - c_tensors.as_mut_ptr(), - self.c_tensor, - bins, - range.as_ptr(), - range.len_i32(), - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if density { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_histogram_bin_ct_out>( - &self, - hist: &Tensor, - bin_edges: &Tensor, - bins: i64, - range: impl DoubleList, - weight: Option, - density: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_histogram_bin_ct_out( - c_tensors.as_mut_ptr(), - hist.c_tensor, - bin_edges.c_tensor, - self.c_tensor, - bins, - range.as_ptr(), - range.len_i32(), - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if density { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_histogram_bins_tensor_out>( - &self, - hist: &Tensor, - bin_edges: &Tensor, - bins: &Tensor, - weight: Option, - density: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_histogram_bins_tensor_out( - c_tensors.as_mut_ptr(), - hist.c_tensor, - bin_edges.c_tensor, - self.c_tensor, - bins.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if density { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_hsplit(&self, sections: i64) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_hsplit(self.c_tensor, sections)); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_hsplit_array(&self, indices: impl IntList) -> Result, TchError> { - let c_tensors = - unsafe_torch_err!(atg_hsplit_array(self.c_tensor, indices.as_ptr(), indices.len_i32())); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_hspmm(mat1: &Tensor, mat2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hspmm(c_tensors.as_mut_ptr(), mat1.c_tensor, mat2.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hspmm_out(out: &Tensor, mat1: &Tensor, mat2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hspmm_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - mat1.c_tensor, - mat2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hstack>(tensors: &[T]) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hstack( - c_tensors.as_mut_ptr(), - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hstack_out>( - out: &Tensor, - tensors: &[T], - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hstack_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_huber_loss( - &self, - target: &Tensor, - reduction: crate::Reduction, - delta: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_huber_loss( - c_tensors.as_mut_ptr(), - self.c_tensor, - target.c_tensor, - reduction.to_int(), - delta - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_huber_loss_backward( - &self, - grad_output: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - delta: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_huber_loss_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - target.c_tensor, - reduction.to_int(), - delta - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_huber_loss_backward_out( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - delta: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_huber_loss_backward_out( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - target.c_tensor, - reduction.to_int(), - delta - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_huber_loss_out( - &self, - out: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - delta: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_huber_loss_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - target.c_tensor, - reduction.to_int(), - delta - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hypot(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hypot(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hypot_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hypot_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_hypot_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_hypot_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_i0(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_i0(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_i0_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_i0_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_i0_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_i0_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_igamma(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_igamma(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_igamma_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_igamma_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_igamma_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_igamma_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_igammac(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_igammac(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_igammac_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_igammac_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_igammac_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_igammac_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_im2col( - &self, - kernel_size: impl IntList, - dilation: impl IntList, - padding: impl IntList, - stride: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_im2col( - c_tensors.as_mut_ptr(), - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - padding.as_ptr(), - padding.len_i32(), - stride.as_ptr(), - stride.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_im2col_out( - &self, - out: &Tensor, - kernel_size: impl IntList, - dilation: impl IntList, - padding: impl IntList, - stride: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_im2col_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - padding.as_ptr(), - padding.len_i32(), - stride.as_ptr(), - stride.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_imag(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_imag(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index>(&self, indices: &[Option]) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index( - c_tensors.as_mut_ptr(), - self.c_tensor, - ptr_list_opt(indices).as_ptr(), - indices.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index_add( - &self, - dim: i64, - index: &Tensor, - source: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index_add( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - source.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index_add_( - &mut self, - dim: i64, - index: &Tensor, - source: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index_add_( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - source.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index_add_out( - &self, - out: &Tensor, - dim: i64, - index: &Tensor, - source: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index_add_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - index.c_tensor, - source.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index_copy( - &self, - dim: i64, - index: &Tensor, - source: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index_copy( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - source.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index_copy_( - &mut self, - dim: i64, - index: &Tensor, - source: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index_copy_( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - source.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index_copy_out( - &self, - out: &Tensor, - dim: i64, - index: &Tensor, - source: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - index.c_tensor, - source.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index_fill>( - &self, - dim: i64, - index: &Tensor, - value: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index_fill( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - value.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index_fill_>( - &mut self, - dim: i64, - index: &Tensor, - value: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index_fill_( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - value.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index_fill_int_scalar_out>( - &self, - out: &Tensor, - dim: i64, - index: &Tensor, - value: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index_fill_int_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - index.c_tensor, - value.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index_fill_int_tensor( - &self, - dim: i64, - index: &Tensor, - value: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index_fill_int_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - value.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index_fill_int_tensor_( - &mut self, - dim: i64, - index: &Tensor, - value: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index_fill_int_tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - value.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index_fill_int_tensor_out( - &self, - out: &Tensor, - dim: i64, - index: &Tensor, - value: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index_fill_int_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - index.c_tensor, - value.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index_put>( - &self, - indices: &[Option], - values: &Tensor, - accumulate: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index_put( - c_tensors.as_mut_ptr(), - self.c_tensor, - ptr_list_opt(indices).as_ptr(), - indices.len() as i32, - values.c_tensor, - if accumulate { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index_put_>( - &mut self, - indices: &[Option], - values: &Tensor, - accumulate: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index_put_( - c_tensors.as_mut_ptr(), - self.c_tensor, - ptr_list_opt(indices).as_ptr(), - indices.len() as i32, - values.c_tensor, - if accumulate { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index_put_out>( - &self, - out: &Tensor, - indices: &[Option], - values: &Tensor, - accumulate: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index_put_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - ptr_list_opt(indices).as_ptr(), - indices.len() as i32, - values.c_tensor, - if accumulate { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index_reduce( - &self, - dim: i64, - index: &Tensor, - source: &Tensor, - reduce: &str, - include_self: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index_reduce( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - source.c_tensor, - reduce.as_ptr(), - reduce.len() as i32, - if include_self { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index_reduce_( - &mut self, - dim: i64, - index: &Tensor, - source: &Tensor, - reduce: &str, - include_self: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index_reduce_( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - source.c_tensor, - reduce.as_ptr(), - reduce.len() as i32, - if include_self { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index_reduce_out( - &self, - out: &Tensor, - dim: i64, - index: &Tensor, - source: &Tensor, - reduce: &str, - include_self: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index_reduce_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - index.c_tensor, - source.c_tensor, - reduce.as_ptr(), - reduce.len() as i32, - if include_self { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index_select(&self, dim: i64, index: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index_select( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index_select_backward( - grad: &Tensor, - self_sizes: impl IntList, - dim: i64, - index: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index_select_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - self_sizes.as_ptr(), - self_sizes.len_i32(), - dim, - index.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index_select_out( - &self, - out: &Tensor, - dim: i64, - index: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index_select_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - index.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_index_tensor_out>( - &self, - out: &Tensor, - indices: &[Option], - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_index_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - ptr_list_opt(indices).as_ptr(), - indices.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_indices(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_indices(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_indices_copy(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_indices_copy(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_indices_copy_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_indices_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_infinitely_differentiable_gelu_backward( - &self, - grad: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_infinitely_differentiable_gelu_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_inner(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_inner(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_inner_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_inner_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_instance_norm>( - &self, - weight: Option, - bias: Option, - running_mean: Option, - running_var: Option, - use_input_stats: bool, - momentum: f64, - eps: f64, - cudnn_enabled: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_instance_norm( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if use_input_stats { 1 } else { 0 }, - momentum, - eps, - if cudnn_enabled { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_int_repr(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_int_repr(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_int_repr_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_int_repr_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_inverse(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_inverse(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_inverse_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_inverse_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_is_coalesced(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_is_coalesced(self.c_tensor)); - Ok(return_ != 0) - } - - pub fn f_is_complex(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_is_complex(self.c_tensor)); - Ok(return_ != 0) - } - - pub fn f_is_conj(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_is_conj(self.c_tensor)); - Ok(return_ != 0) - } - - pub fn f_is_distributed(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_is_distributed(self.c_tensor)); - Ok(return_ != 0) - } - - pub fn f_is_floating_point(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_is_floating_point(self.c_tensor)); - Ok(return_ != 0) - } - - pub fn f_is_inference(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_is_inference(self.c_tensor)); - Ok(return_ != 0) - } - - pub fn f_is_leaf(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_is_leaf(self.c_tensor)); - Ok(return_ != 0) - } - - pub fn f_is_neg(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_is_neg(self.c_tensor)); - Ok(return_ != 0) - } - - pub fn f_is_nonzero(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_is_nonzero(self.c_tensor)); - Ok(return_ != 0) - } - - pub fn f_is_pinned(&self, device: Device) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_is_pinned(self.c_tensor, device.c_int())); - Ok(return_ != 0) - } - - pub fn f_is_same_size(&self, other: &Tensor) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_is_same_size(self.c_tensor, other.c_tensor)); - Ok(return_ != 0) - } - - pub fn f_is_set_to(&self, tensor: &Tensor) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_is_set_to(self.c_tensor, tensor.c_tensor)); - Ok(return_ != 0) - } - - pub fn f_is_signed(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_is_signed(self.c_tensor)); - Ok(return_ != 0) - } - - pub fn f_is_vulkan_available() -> Result { - let return_; - unsafe_torch_err!(return_ = atg_is_vulkan_available()); - Ok(return_ != 0) - } - - pub fn f_isclose( - &self, - other: &Tensor, - rtol: f64, - atol: f64, - equal_nan: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_isclose( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor, - rtol, - atol, - if equal_nan { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_isfinite(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_isfinite(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_isin( - elements: &Tensor, - test_elements: &Tensor, - assume_unique: bool, - invert: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_isin( - c_tensors.as_mut_ptr(), - elements.c_tensor, - test_elements.c_tensor, - if assume_unique { 1 } else { 0 }, - if invert { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_isin_scalar_tensor>( - element: S, - test_elements: &Tensor, - assume_unique: bool, - invert: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_isin_scalar_tensor( - c_tensors.as_mut_ptr(), - element.into().c_scalar, - test_elements.c_tensor, - if assume_unique { 1 } else { 0 }, - if invert { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_isin_scalar_tensor_out>( - out: &Tensor, - element: S, - test_elements: &Tensor, - assume_unique: bool, - invert: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_isin_scalar_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - element.into().c_scalar, - test_elements.c_tensor, - if assume_unique { 1 } else { 0 }, - if invert { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_isin_tensor_scalar>( - elements: &Tensor, - test_element: S, - assume_unique: bool, - invert: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_isin_tensor_scalar( - c_tensors.as_mut_ptr(), - elements.c_tensor, - test_element.into().c_scalar, - if assume_unique { 1 } else { 0 }, - if invert { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_isin_tensor_scalar_out>( - out: &Tensor, - elements: &Tensor, - test_element: S, - assume_unique: bool, - invert: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_isin_tensor_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - elements.c_tensor, - test_element.into().c_scalar, - if assume_unique { 1 } else { 0 }, - if invert { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_isin_tensor_tensor_out( - out: &Tensor, - elements: &Tensor, - test_elements: &Tensor, - assume_unique: bool, - invert: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_isin_tensor_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - elements.c_tensor, - test_elements.c_tensor, - if assume_unique { 1 } else { 0 }, - if invert { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_isinf(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_isinf(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_isinf_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_isinf_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_isnan(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_isnan(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_isnan_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_isnan_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_isneginf(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_isneginf(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_isneginf_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_isneginf_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_isposinf(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_isposinf(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_isposinf_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_isposinf_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_isreal(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_isreal(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_istft>( - &self, - n_fft: i64, - hop_length: impl Into>, - win_length: impl Into>, - window: Option, - center: bool, - normalized: bool, - onesided: bool, - length: impl Into>, - return_complex: bool, - ) -> Result { - let hop_length = hop_length.into(); - let win_length = win_length.into(); - let length = length.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_istft( - c_tensors.as_mut_ptr(), - self.c_tensor, - n_fft, - hop_length.unwrap_or(0i64), - hop_length.is_none() as i8, - win_length.unwrap_or(0i64), - win_length.is_none() as i8, - window.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if center { 1 } else { 0 }, - if normalized { 1 } else { 0 }, - if onesided { 1 } else { 0 }, - length.unwrap_or(0i64), - length.is_none() as i8, - if return_complex { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_kaiser_window( - window_length: i64, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_kaiser_window( - c_tensors.as_mut_ptr(), - window_length, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_kaiser_window_beta( - window_length: i64, - periodic: bool, - beta: f64, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_kaiser_window_beta( - c_tensors.as_mut_ptr(), - window_length, - if periodic { 1 } else { 0 }, - beta, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_kaiser_window_beta_out( - out: &Tensor, - window_length: i64, - periodic: bool, - beta: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_kaiser_window_beta_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - window_length, - if periodic { 1 } else { 0 }, - beta - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_kaiser_window_out(out: &Tensor, window_length: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_kaiser_window_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - window_length - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_kaiser_window_periodic( - window_length: i64, - periodic: bool, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_kaiser_window_periodic( - c_tensors.as_mut_ptr(), - window_length, - if periodic { 1 } else { 0 }, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_kaiser_window_periodic_out( - out: &Tensor, - window_length: i64, - periodic: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_kaiser_window_periodic_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - window_length, - if periodic { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_kl_div( - &self, - target: &Tensor, - reduction: crate::Reduction, - log_target: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_kl_div( - c_tensors.as_mut_ptr(), - self.c_tensor, - target.c_tensor, - reduction.to_int(), - if log_target { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_kron(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_kron(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_kron_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_kron_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_kthvalue( - &self, - k: i64, - dim: i64, - keepdim: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_kthvalue( - c_tensors.as_mut_ptr(), - self.c_tensor, - k, - dim, - if keepdim { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_kthvalue_values( - &self, - values: &Tensor, - indices: &Tensor, - k: i64, - dim: i64, - keepdim: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_kthvalue_values( - c_tensors.as_mut_ptr(), - values.c_tensor, - indices.c_tensor, - self.c_tensor, - k, - dim, - if keepdim { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_l1_loss( - &self, - target: &Tensor, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_l1_loss( - c_tensors.as_mut_ptr(), - self.c_tensor, - target.c_tensor, - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_layer_norm>( - &self, - normalized_shape: impl IntList, - weight: Option, - bias: Option, - eps: f64, - cudnn_enable: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_layer_norm( - c_tensors.as_mut_ptr(), - self.c_tensor, - normalized_shape.as_ptr(), - normalized_shape.len_i32(), - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - eps, - if cudnn_enable { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lcm(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lcm(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lcm_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lcm_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lcm_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lcm_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ldexp(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ldexp(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ldexp_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ldexp_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ldexp_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ldexp_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_le>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_le(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_le_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_le_(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_le_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_le_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_le_tensor(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_le_tensor(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_le_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_le_tensor_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_le_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_le_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_leaky_relu(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_leaky_relu(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_leaky_relu_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_leaky_relu_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_leaky_relu_backward>( - &self, - grad_output: &Tensor, - negative_slope: S, - self_is_result: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_leaky_relu_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - negative_slope.into().c_scalar, - if self_is_result { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_leaky_relu_backward_grad_input>( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - negative_slope: S, - self_is_result: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_leaky_relu_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - negative_slope.into().c_scalar, - if self_is_result { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_leaky_relu_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_leaky_relu_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lerp>(&self, end: &Tensor, weight: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lerp( - c_tensors.as_mut_ptr(), - self.c_tensor, - end.c_tensor, - weight.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lerp_>( - &mut self, - end: &Tensor, - weight: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lerp_( - c_tensors.as_mut_ptr(), - self.c_tensor, - end.c_tensor, - weight.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lerp_scalar_out>( - &self, - out: &Tensor, - end: &Tensor, - weight: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lerp_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - end.c_tensor, - weight.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lerp_tensor(&self, end: &Tensor, weight: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lerp_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - end.c_tensor, - weight.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lerp_tensor_(&mut self, end: &Tensor, weight: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lerp_tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - end.c_tensor, - weight.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lerp_tensor_out( - &self, - out: &Tensor, - end: &Tensor, - weight: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lerp_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - end.c_tensor, - weight.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_less>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_less(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_less_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_less_(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_less_equal>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_less_equal( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_less_equal_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_less_equal_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_less_equal_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_less_equal_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_less_equal_tensor(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_less_equal_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_less_equal_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_less_equal_tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_less_equal_tensor_out( - &self, - out: &Tensor, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_less_equal_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_less_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_less_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_less_tensor(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_less_tensor(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_less_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_less_tensor_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_less_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_less_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lgamma(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lgamma(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lgamma_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lgamma_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lgamma_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lgamma_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lift(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lift(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lift_fresh(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lift_fresh(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lift_fresh_copy(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lift_fresh_copy(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lift_fresh_copy_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lift_fresh_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lift_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lift_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_cholesky(&self, upper: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_cholesky( - c_tensors.as_mut_ptr(), - self.c_tensor, - if upper { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_cholesky_ex( - &self, - upper: bool, - check_errors: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_linalg_cholesky_ex( - c_tensors.as_mut_ptr(), - self.c_tensor, - if upper { 1 } else { 0 }, - if check_errors { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_linalg_cholesky_ex_l( - &self, - l: &Tensor, - info: &Tensor, - upper: bool, - check_errors: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_linalg_cholesky_ex_l( - c_tensors.as_mut_ptr(), - l.c_tensor, - info.c_tensor, - self.c_tensor, - if upper { 1 } else { 0 }, - if check_errors { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_linalg_cholesky_out(&self, out: &Tensor, upper: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_cholesky_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - if upper { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_cond>(&self, p: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_cond( - c_tensors.as_mut_ptr(), - self.c_tensor, - p.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_cond_out>( - &self, - out: &Tensor, - p: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_cond_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - p.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_cond_p_str(&self, p: &str) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_cond_p_str( - c_tensors.as_mut_ptr(), - self.c_tensor, - p.as_ptr(), - p.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_cond_p_str_out(&self, out: &Tensor, p: &str) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_cond_p_str_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - p.as_ptr(), - p.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_cross(&self, other: &Tensor, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_cross( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_cross_out( - &self, - out: &Tensor, - other: &Tensor, - dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_cross_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_det(a: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_det(c_tensors.as_mut_ptr(), a.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_det_out(out: &Tensor, a: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_det_out(c_tensors.as_mut_ptr(), out.c_tensor, a.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_diagonal( - a: &Tensor, - offset: i64, - dim1: i64, - dim2: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_diagonal( - c_tensors.as_mut_ptr(), - a.c_tensor, - offset, - dim1, - dim2 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_eig(&self) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_linalg_eig(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_linalg_eig_out( - &self, - eigenvalues: &Tensor, - eigenvectors: &Tensor, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_linalg_eig_out( - c_tensors.as_mut_ptr(), - eigenvalues.c_tensor, - eigenvectors.c_tensor, - self.c_tensor - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_linalg_eigh(&self, uplo: &str) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_linalg_eigh( - c_tensors.as_mut_ptr(), - self.c_tensor, - uplo.as_ptr(), - uplo.len() as i32 - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_linalg_eigh_eigvals( - &self, - eigvals: &Tensor, - eigvecs: &Tensor, - uplo: &str, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_linalg_eigh_eigvals( - c_tensors.as_mut_ptr(), - eigvals.c_tensor, - eigvecs.c_tensor, - self.c_tensor, - uplo.as_ptr(), - uplo.len() as i32 - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_linalg_eigvals(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_eigvals(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_eigvals_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_eigvals_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_eigvalsh(&self, uplo: &str) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_eigvalsh( - c_tensors.as_mut_ptr(), - self.c_tensor, - uplo.as_ptr(), - uplo.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_eigvalsh_out(&self, out: &Tensor, uplo: &str) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_eigvalsh_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - uplo.as_ptr(), - uplo.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_householder_product(&self, tau: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_householder_product( - c_tensors.as_mut_ptr(), - self.c_tensor, - tau.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_householder_product_out( - &self, - out: &Tensor, - tau: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_householder_product_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - tau.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_inv(a: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_inv(c_tensors.as_mut_ptr(), a.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_inv_ex(a: &Tensor, check_errors: bool) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_linalg_inv_ex( - c_tensors.as_mut_ptr(), - a.c_tensor, - if check_errors { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_linalg_inv_ex_inverse( - inverse: &Tensor, - info: &Tensor, - a: &Tensor, - check_errors: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_linalg_inv_ex_inverse( - c_tensors.as_mut_ptr(), - inverse.c_tensor, - info.c_tensor, - a.c_tensor, - if check_errors { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_linalg_inv_out(out: &Tensor, a: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_inv_out(c_tensors.as_mut_ptr(), out.c_tensor, a.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_ldl_factor(&self, hermitian: bool) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_linalg_ldl_factor( - c_tensors.as_mut_ptr(), - self.c_tensor, - if hermitian { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_linalg_ldl_factor_ex( - &self, - hermitian: bool, - check_errors: bool, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_linalg_ldl_factor_ex( - c_tensors.as_mut_ptr(), - self.c_tensor, - if hermitian { 1 } else { 0 }, - if check_errors { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_linalg_ldl_factor_ex_out( - &self, - ld: &Tensor, - pivots: &Tensor, - info: &Tensor, - hermitian: bool, - check_errors: bool, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_linalg_ldl_factor_ex_out( - c_tensors.as_mut_ptr(), - ld.c_tensor, - pivots.c_tensor, - info.c_tensor, - self.c_tensor, - if hermitian { 1 } else { 0 }, - if check_errors { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_linalg_ldl_factor_out( - &self, - ld: &Tensor, - pivots: &Tensor, - hermitian: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_linalg_ldl_factor_out( - c_tensors.as_mut_ptr(), - ld.c_tensor, - pivots.c_tensor, - self.c_tensor, - if hermitian { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_linalg_ldl_solve( - ld: &Tensor, - pivots: &Tensor, - b: &Tensor, - hermitian: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_ldl_solve( - c_tensors.as_mut_ptr(), - ld.c_tensor, - pivots.c_tensor, - b.c_tensor, - if hermitian { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_ldl_solve_out( - out: &Tensor, - ld: &Tensor, - pivots: &Tensor, - b: &Tensor, - hermitian: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_ldl_solve_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - ld.c_tensor, - pivots.c_tensor, - b.c_tensor, - if hermitian { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_lstsq( - &self, - b: &Tensor, - rcond: impl Into>, - driver: &str, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let rcond = rcond.into(); - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg_linalg_lstsq( - c_tensors.as_mut_ptr(), - self.c_tensor, - b.c_tensor, - rcond.unwrap_or(std::f64::NAN), - rcond.is_none() as i8, - driver.as_ptr(), - driver.len() as i32 - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_linalg_lstsq_out( - &self, - solution: &Tensor, - residuals: &Tensor, - rank: &Tensor, - singular_values: &Tensor, - b: &Tensor, - rcond: impl Into>, - driver: &str, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let rcond = rcond.into(); - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg_linalg_lstsq_out( - c_tensors.as_mut_ptr(), - solution.c_tensor, - residuals.c_tensor, - rank.c_tensor, - singular_values.c_tensor, - self.c_tensor, - b.c_tensor, - rcond.unwrap_or(std::f64::NAN), - rcond.is_none() as i8, - driver.as_ptr(), - driver.len() as i32 - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_linalg_lu(a: &Tensor, pivot: bool) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_linalg_lu( - c_tensors.as_mut_ptr(), - a.c_tensor, - if pivot { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_linalg_lu_factor(a: &Tensor, pivot: bool) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_linalg_lu_factor( - c_tensors.as_mut_ptr(), - a.c_tensor, - if pivot { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_linalg_lu_factor_ex( - a: &Tensor, - pivot: bool, - check_errors: bool, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_linalg_lu_factor_ex( - c_tensors.as_mut_ptr(), - a.c_tensor, - if pivot { 1 } else { 0 }, - if check_errors { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_linalg_lu_factor_ex_out( - lu: &Tensor, - pivots: &Tensor, - info: &Tensor, - a: &Tensor, - pivot: bool, - check_errors: bool, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_linalg_lu_factor_ex_out( - c_tensors.as_mut_ptr(), - lu.c_tensor, - pivots.c_tensor, - info.c_tensor, - a.c_tensor, - if pivot { 1 } else { 0 }, - if check_errors { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_linalg_lu_factor_out( - lu: &Tensor, - pivots: &Tensor, - a: &Tensor, - pivot: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_linalg_lu_factor_out( - c_tensors.as_mut_ptr(), - lu.c_tensor, - pivots.c_tensor, - a.c_tensor, - if pivot { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_linalg_lu_out( - p: &Tensor, - l: &Tensor, - u: &Tensor, - a: &Tensor, - pivot: bool, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_linalg_lu_out( - c_tensors.as_mut_ptr(), - p.c_tensor, - l.c_tensor, - u.c_tensor, - a.c_tensor, - if pivot { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_linalg_lu_solve( - lu: &Tensor, - pivots: &Tensor, - b: &Tensor, - left: bool, - adjoint: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_lu_solve( - c_tensors.as_mut_ptr(), - lu.c_tensor, - pivots.c_tensor, - b.c_tensor, - if left { 1 } else { 0 }, - if adjoint { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_lu_solve_out( - out: &Tensor, - lu: &Tensor, - pivots: &Tensor, - b: &Tensor, - left: bool, - adjoint: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_lu_solve_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - lu.c_tensor, - pivots.c_tensor, - b.c_tensor, - if left { 1 } else { 0 }, - if adjoint { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_matmul(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_matmul(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_matmul_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_matmul_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_matrix_exp(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_matrix_exp(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_matrix_exp_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_matrix_exp_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_matrix_power(&self, n: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_matrix_power(c_tensors.as_mut_ptr(), self.c_tensor, n)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_matrix_power_out(&self, out: &Tensor, n: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_matrix_power_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - n - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_matrix_rank(&self, tol: f64, hermitian: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_matrix_rank( - c_tensors.as_mut_ptr(), - self.c_tensor, - tol, - if hermitian { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_matrix_rank_atol_rtol_float( - &self, - atol: impl Into>, - rtol: impl Into>, - hermitian: bool, - ) -> Result { - let atol = atol.into(); - let rtol = rtol.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_matrix_rank_atol_rtol_float( - c_tensors.as_mut_ptr(), - self.c_tensor, - atol.unwrap_or(std::f64::NAN), - atol.is_none() as i8, - rtol.unwrap_or(std::f64::NAN), - rtol.is_none() as i8, - if hermitian { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_matrix_rank_atol_rtol_float_out( - &self, - out: &Tensor, - atol: impl Into>, - rtol: impl Into>, - hermitian: bool, - ) -> Result { - let atol = atol.into(); - let rtol = rtol.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_matrix_rank_atol_rtol_float_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - atol.unwrap_or(std::f64::NAN), - atol.is_none() as i8, - rtol.unwrap_or(std::f64::NAN), - rtol.is_none() as i8, - if hermitian { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_matrix_rank_atol_rtol_tensor>( - &self, - atol: Option, - rtol: Option, - hermitian: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_matrix_rank_atol_rtol_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - atol.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - rtol.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if hermitian { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_matrix_rank_atol_rtol_tensor_out>( - &self, - out: &Tensor, - atol: Option, - rtol: Option, - hermitian: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_matrix_rank_atol_rtol_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - atol.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - rtol.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if hermitian { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_matrix_rank_out( - &self, - out: &Tensor, - tol: f64, - hermitian: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_matrix_rank_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - tol, - if hermitian { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_matrix_rank_out_tol_tensor( - &self, - out: &Tensor, - tol: &Tensor, - hermitian: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_matrix_rank_out_tol_tensor( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - tol.c_tensor, - if hermitian { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_matrix_rank_tol_tensor( - &self, - tol: &Tensor, - hermitian: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_matrix_rank_tol_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - tol.c_tensor, - if hermitian { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_multi_dot>(tensors: &[T]) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_multi_dot( - c_tensors.as_mut_ptr(), - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_multi_dot_out>( - out: &Tensor, - tensors: &[T], - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_multi_dot_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_norm>( - &self, - ord: S, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_norm( - c_tensors.as_mut_ptr(), - self.c_tensor, - ord.into().c_scalar, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 }, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_norm_ord_str( - &self, - ord: &str, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_norm_ord_str( - c_tensors.as_mut_ptr(), - self.c_tensor, - ord.as_ptr(), - ord.len() as i32, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 }, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_norm_ord_str_out( - &self, - out: &Tensor, - ord: &str, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_norm_ord_str_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - ord.as_ptr(), - ord.len() as i32, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 }, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_norm_out>( - &self, - out: &Tensor, - ord: S, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_norm_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - ord.into().c_scalar, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 }, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_pinv(&self, rcond: f64, hermitian: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_pinv( - c_tensors.as_mut_ptr(), - self.c_tensor, - rcond, - if hermitian { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_pinv_atol_rtol_float( - &self, - atol: impl Into>, - rtol: impl Into>, - hermitian: bool, - ) -> Result { - let atol = atol.into(); - let rtol = rtol.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_pinv_atol_rtol_float( - c_tensors.as_mut_ptr(), - self.c_tensor, - atol.unwrap_or(std::f64::NAN), - atol.is_none() as i8, - rtol.unwrap_or(std::f64::NAN), - rtol.is_none() as i8, - if hermitian { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_pinv_atol_rtol_float_out( - &self, - out: &Tensor, - atol: impl Into>, - rtol: impl Into>, - hermitian: bool, - ) -> Result { - let atol = atol.into(); - let rtol = rtol.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_pinv_atol_rtol_float_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - atol.unwrap_or(std::f64::NAN), - atol.is_none() as i8, - rtol.unwrap_or(std::f64::NAN), - rtol.is_none() as i8, - if hermitian { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_pinv_atol_rtol_tensor>( - &self, - atol: Option, - rtol: Option, - hermitian: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_pinv_atol_rtol_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - atol.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - rtol.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if hermitian { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_pinv_atol_rtol_tensor_out>( - &self, - out: &Tensor, - atol: Option, - rtol: Option, - hermitian: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_pinv_atol_rtol_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - atol.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - rtol.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if hermitian { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_pinv_out( - &self, - out: &Tensor, - rcond: f64, - hermitian: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_pinv_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - rcond, - if hermitian { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_pinv_out_rcond_tensor( - &self, - out: &Tensor, - rcond: &Tensor, - hermitian: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_pinv_out_rcond_tensor( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - rcond.c_tensor, - if hermitian { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_pinv_rcond_tensor( - &self, - rcond: &Tensor, - hermitian: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_pinv_rcond_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - rcond.c_tensor, - if hermitian { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_qr(a: &Tensor, mode: &str) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_linalg_qr( - c_tensors.as_mut_ptr(), - a.c_tensor, - mode.as_ptr(), - mode.len() as i32 - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_linalg_qr_out( - q: &Tensor, - r: &Tensor, - a: &Tensor, - mode: &str, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_linalg_qr_out( - c_tensors.as_mut_ptr(), - q.c_tensor, - r.c_tensor, - a.c_tensor, - mode.as_ptr(), - mode.len() as i32 - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_linalg_slogdet(a: &Tensor) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_linalg_slogdet(c_tensors.as_mut_ptr(), a.c_tensor)); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_linalg_slogdet_out( - sign: &Tensor, - logabsdet: &Tensor, - a: &Tensor, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_linalg_slogdet_out( - c_tensors.as_mut_ptr(), - sign.c_tensor, - logabsdet.c_tensor, - a.c_tensor - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_linalg_solve(a: &Tensor, b: &Tensor, left: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_solve( - c_tensors.as_mut_ptr(), - a.c_tensor, - b.c_tensor, - if left { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_solve_ex( - a: &Tensor, - b: &Tensor, - left: bool, - check_errors: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_linalg_solve_ex( - c_tensors.as_mut_ptr(), - a.c_tensor, - b.c_tensor, - if left { 1 } else { 0 }, - if check_errors { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_linalg_solve_ex_out( - result: &Tensor, - info: &Tensor, - a: &Tensor, - b: &Tensor, - left: bool, - check_errors: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_linalg_solve_ex_out( - c_tensors.as_mut_ptr(), - result.c_tensor, - info.c_tensor, - a.c_tensor, - b.c_tensor, - if left { 1 } else { 0 }, - if check_errors { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_linalg_solve_out( - out: &Tensor, - a: &Tensor, - b: &Tensor, - left: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_solve_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - a.c_tensor, - b.c_tensor, - if left { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_solve_triangular( - &self, - b: &Tensor, - upper: bool, - left: bool, - unitriangular: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_solve_triangular( - c_tensors.as_mut_ptr(), - self.c_tensor, - b.c_tensor, - if upper { 1 } else { 0 }, - if left { 1 } else { 0 }, - if unitriangular { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_solve_triangular_out( - &self, - out: &Tensor, - b: &Tensor, - upper: bool, - left: bool, - unitriangular: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_solve_triangular_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - b.c_tensor, - if upper { 1 } else { 0 }, - if left { 1 } else { 0 }, - if unitriangular { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_svd( - a: &Tensor, - full_matrices: bool, - driver: &str, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_linalg_svd( - c_tensors.as_mut_ptr(), - a.c_tensor, - if full_matrices { 1 } else { 0 }, - driver.as_ptr(), - driver.len() as i32 - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_linalg_svd_u( - u: &Tensor, - s: &Tensor, - vh: &Tensor, - a: &Tensor, - full_matrices: bool, - driver: &str, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_linalg_svd_u( - c_tensors.as_mut_ptr(), - u.c_tensor, - s.c_tensor, - vh.c_tensor, - a.c_tensor, - if full_matrices { 1 } else { 0 }, - driver.as_ptr(), - driver.len() as i32 - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_linalg_svdvals(a: &Tensor, driver: &str) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_svdvals( - c_tensors.as_mut_ptr(), - a.c_tensor, - driver.as_ptr(), - driver.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_svdvals_out( - out: &Tensor, - a: &Tensor, - driver: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_svdvals_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - a.c_tensor, - driver.as_ptr(), - driver.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_tensorinv(&self, ind: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_tensorinv(c_tensors.as_mut_ptr(), self.c_tensor, ind)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_tensorinv_out(&self, out: &Tensor, ind: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_tensorinv_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - ind - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_tensorsolve( - &self, - other: &Tensor, - dims: impl IntListOption, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_tensorsolve( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor, - dims.as_ptr(), - dims.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_tensorsolve_out( - &self, - out: &Tensor, - other: &Tensor, - dims: impl IntListOption, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_tensorsolve_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor, - dims.as_ptr(), - dims.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_vander(x: &Tensor, n: impl Into>) -> Result { - let n = n.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_vander( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.unwrap_or(0i64), - n.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_vecdot(x: &Tensor, y: &Tensor, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_vecdot(c_tensors.as_mut_ptr(), x.c_tensor, y.c_tensor, dim)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linalg_vecdot_out( - out: &Tensor, - x: &Tensor, - y: &Tensor, - dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linalg_vecdot_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - y.c_tensor, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linear>( - &self, - weight: &Tensor, - bias: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linear( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linear_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linear_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linspace>( - start: S, - end: S, - steps: i64, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linspace( - c_tensors.as_mut_ptr(), - start.into().c_scalar, - end.into().c_scalar, - steps, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linspace_out>( - out: &Tensor, - start: S, - end: S, - steps: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linspace_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - start.into().c_scalar, - end.into().c_scalar, - steps - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linspace_scalar_tensor>( - start: S, - end: &Tensor, - steps: i64, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linspace_scalar_tensor( - c_tensors.as_mut_ptr(), - start.into().c_scalar, - end.c_tensor, - steps, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linspace_scalar_tensor_out>( - out: &Tensor, - start: S, - end: &Tensor, - steps: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linspace_scalar_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - start.into().c_scalar, - end.c_tensor, - steps - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linspace_tensor_scalar>( - start: &Tensor, - end: S, - steps: i64, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linspace_tensor_scalar( - c_tensors.as_mut_ptr(), - start.c_tensor, - end.into().c_scalar, - steps, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linspace_tensor_scalar_out>( - out: &Tensor, - start: &Tensor, - end: S, - steps: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linspace_tensor_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - start.c_tensor, - end.into().c_scalar, - steps - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linspace_tensor_tensor( - start: &Tensor, - end: &Tensor, - steps: i64, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linspace_tensor_tensor( - c_tensors.as_mut_ptr(), - start.c_tensor, - end.c_tensor, - steps, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_linspace_tensor_tensor_out( - out: &Tensor, - start: &Tensor, - end: &Tensor, - steps: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_linspace_tensor_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - start.c_tensor, - end.c_tensor, - steps - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_log(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_log(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_log10(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_log10(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_log10_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_log10_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_log10_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_log10_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_log1p(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_log1p(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_log1p_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_log1p_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_log1p_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_log1p_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_log2(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_log2(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_log2_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_log2_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_log2_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_log2_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_log_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_log_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_log_normal(&self, mean: f64, std: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_log_normal(c_tensors.as_mut_ptr(), self.c_tensor, mean, std)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_log_normal_(&mut self, mean: f64, std: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_log_normal_(c_tensors.as_mut_ptr(), self.c_tensor, mean, std)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_log_normal_out(&self, out: &Tensor, mean: f64, std: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_log_normal_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mean, - std - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_log_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_log_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_log_sigmoid(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_log_sigmoid(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_log_sigmoid_backward( - &self, - grad_output: &Tensor, - buffer: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_log_sigmoid_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - buffer.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_log_sigmoid_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - buffer: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_log_sigmoid_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - buffer.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_log_sigmoid_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_log_sigmoid_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_log_softmax( - &self, - dim: i64, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_log_softmax( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_log_softmax_int_out( - &self, - out: &Tensor, - dim: i64, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_log_softmax_int_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logaddexp(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logaddexp(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logaddexp2(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logaddexp2(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logaddexp2_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logaddexp2_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logaddexp_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logaddexp_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logcumsumexp(&self, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logcumsumexp(c_tensors.as_mut_ptr(), self.c_tensor, dim)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logcumsumexp_out(&self, out: &Tensor, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logcumsumexp_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logdet(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logdet(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logical_and(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logical_and(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logical_and_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logical_and_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logical_and_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logical_and_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logical_not(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logical_not(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logical_not_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logical_not_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logical_not_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logical_not_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logical_or(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logical_or(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logical_or_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logical_or_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logical_or_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logical_or_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logical_xor(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logical_xor(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logical_xor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logical_xor_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logical_xor_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logical_xor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logit(&self, eps: impl Into>) -> Result { - let eps = eps.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logit( - c_tensors.as_mut_ptr(), - self.c_tensor, - eps.unwrap_or(std::f64::NAN), - eps.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logit_(&mut self, eps: impl Into>) -> Result { - let eps = eps.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logit_( - c_tensors.as_mut_ptr(), - self.c_tensor, - eps.unwrap_or(std::f64::NAN), - eps.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logit_backward( - &self, - grad_output: &Tensor, - eps: impl Into>, - ) -> Result { - let eps = eps.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logit_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - eps.unwrap_or(std::f64::NAN), - eps.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logit_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - eps: impl Into>, - ) -> Result { - let eps = eps.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logit_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - eps.unwrap_or(std::f64::NAN), - eps.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logit_out( - &self, - out: &Tensor, - eps: impl Into>, - ) -> Result { - let eps = eps.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logit_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - eps.unwrap_or(std::f64::NAN), - eps.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logspace>( - start: S, - end: S, - steps: i64, - base: f64, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logspace( - c_tensors.as_mut_ptr(), - start.into().c_scalar, - end.into().c_scalar, - steps, - base, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logspace_out>( - out: &Tensor, - start: S, - end: S, - steps: i64, - base: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logspace_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - start.into().c_scalar, - end.into().c_scalar, - steps, - base - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logspace_scalar_tensor>( - start: S, - end: &Tensor, - steps: i64, - base: f64, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logspace_scalar_tensor( - c_tensors.as_mut_ptr(), - start.into().c_scalar, - end.c_tensor, - steps, - base, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logspace_scalar_tensor_out>( - out: &Tensor, - start: S, - end: &Tensor, - steps: i64, - base: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logspace_scalar_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - start.into().c_scalar, - end.c_tensor, - steps, - base - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logspace_tensor_scalar>( - start: &Tensor, - end: S, - steps: i64, - base: f64, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logspace_tensor_scalar( - c_tensors.as_mut_ptr(), - start.c_tensor, - end.into().c_scalar, - steps, - base, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logspace_tensor_scalar_out>( - out: &Tensor, - start: &Tensor, - end: S, - steps: i64, - base: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logspace_tensor_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - start.c_tensor, - end.into().c_scalar, - steps, - base - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logspace_tensor_tensor( - start: &Tensor, - end: &Tensor, - steps: i64, - base: f64, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logspace_tensor_tensor( - c_tensors.as_mut_ptr(), - start.c_tensor, - end.c_tensor, - steps, - base, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logspace_tensor_tensor_out( - out: &Tensor, - start: &Tensor, - end: &Tensor, - steps: i64, - base: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logspace_tensor_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - start.c_tensor, - end.c_tensor, - steps, - base - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logsumexp(&self, dim: impl IntList, keepdim: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logsumexp( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_logsumexp_out( - &self, - out: &Tensor, - dim: impl IntList, - keepdim: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_logsumexp_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lstm>( - &self, - hx: &[T], - params: &[T], - has_biases: bool, - num_layers: i64, - dropout: f64, - train: bool, - bidirectional: bool, - batch_first: bool, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_lstm( - c_tensors.as_mut_ptr(), - self.c_tensor, - ptr_list(hx).as_ptr(), - hx.len() as i32, - ptr_list(params).as_ptr(), - params.len() as i32, - if has_biases { 1 } else { 0 }, - num_layers, - dropout, - if train { 1 } else { 0 }, - if bidirectional { 1 } else { 0 }, - if batch_first { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_lstm_cell>( - &self, - hx: &[T], - w_ih: &Tensor, - w_hh: &Tensor, - b_ih: Option, - b_hh: Option, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_lstm_cell( - c_tensors.as_mut_ptr(), - self.c_tensor, - ptr_list(hx).as_ptr(), - hx.len() as i32, - w_ih.c_tensor, - w_hh.c_tensor, - b_ih.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - b_hh.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_lstm_data>( - data: &Tensor, - batch_sizes: &Tensor, - hx: &[T], - params: &[T], - has_biases: bool, - num_layers: i64, - dropout: f64, - train: bool, - bidirectional: bool, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_lstm_data( - c_tensors.as_mut_ptr(), - data.c_tensor, - batch_sizes.c_tensor, - ptr_list(hx).as_ptr(), - hx.len() as i32, - ptr_list(params).as_ptr(), - params.len() as i32, - if has_biases { 1 } else { 0 }, - num_layers, - dropout, - if train { 1 } else { 0 }, - if bidirectional { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_lstm_mps_backward>( - &self, - out0: &Tensor, - out1: &[T], - out2: &[T], - grad_y: Option, - grad_hy: Option, - grad_cy: Option, - z_state: &Tensor, - cell_state_fwd: &Tensor, - layersoutputs: &Tensor, - hx: &[T], - params: &[T], - has_biases: bool, - num_layers: i64, - dropout: f64, - train: bool, - bidirectional: bool, - batch_first: bool, - ) -> Result<(), TchError> { - unsafe_torch_err!(atg_lstm_mps_backward( - out0.c_tensor, - ptr_list(out1).as_ptr(), - out1.len() as i32, - ptr_list(out2).as_ptr(), - out2.len() as i32, - grad_y.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - grad_hy.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - grad_cy.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - z_state.c_tensor, - cell_state_fwd.c_tensor, - self.c_tensor, - layersoutputs.c_tensor, - ptr_list(hx).as_ptr(), - hx.len() as i32, - ptr_list(params).as_ptr(), - params.len() as i32, - if has_biases { 1 } else { 0 }, - num_layers, - dropout, - if train { 1 } else { 0 }, - if bidirectional { 1 } else { 0 }, - if batch_first { 1 } else { 0 } - )); - Ok(()) - } - - pub fn f_lt>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lt(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lt_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lt_(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lt_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lt_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lt_tensor(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lt_tensor(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lt_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lt_tensor_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lt_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lt_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lu_solve(&self, lu_data: &Tensor, lu_pivots: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lu_solve( - c_tensors.as_mut_ptr(), - self.c_tensor, - lu_data.c_tensor, - lu_pivots.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lu_solve_out( - &self, - out: &Tensor, - lu_data: &Tensor, - lu_pivots: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_lu_solve_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - lu_data.c_tensor, - lu_pivots.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_lu_unpack( - lu_data: &Tensor, - lu_pivots: &Tensor, - unpack_data: bool, - unpack_pivots: bool, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_lu_unpack( - c_tensors.as_mut_ptr(), - lu_data.c_tensor, - lu_pivots.c_tensor, - if unpack_data { 1 } else { 0 }, - if unpack_pivots { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_lu_unpack_out( - p: &Tensor, - l: &Tensor, - u: &Tensor, - lu_data: &Tensor, - lu_pivots: &Tensor, - unpack_data: bool, - unpack_pivots: bool, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_lu_unpack_out( - c_tensors.as_mut_ptr(), - p.c_tensor, - l.c_tensor, - u.c_tensor, - lu_data.c_tensor, - lu_pivots.c_tensor, - if unpack_data { 1 } else { 0 }, - if unpack_pivots { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_margin_ranking_loss( - input1: &Tensor, - input2: &Tensor, - target: &Tensor, - margin: f64, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_margin_ranking_loss( - c_tensors.as_mut_ptr(), - input1.c_tensor, - input2.c_tensor, - target.c_tensor, - margin, - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_masked_fill>( - &self, - mask: &Tensor, - value: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_masked_fill( - c_tensors.as_mut_ptr(), - self.c_tensor, - mask.c_tensor, - value.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_masked_fill_>( - &mut self, - mask: &Tensor, - value: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_masked_fill_( - c_tensors.as_mut_ptr(), - self.c_tensor, - mask.c_tensor, - value.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_masked_fill_scalar_out>( - &self, - out: &Tensor, - mask: &Tensor, - value: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_masked_fill_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mask.c_tensor, - value.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_masked_fill_tensor(&self, mask: &Tensor, value: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_masked_fill_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - mask.c_tensor, - value.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_masked_fill_tensor_( - &mut self, - mask: &Tensor, - value: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_masked_fill_tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - mask.c_tensor, - value.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_masked_fill_tensor_out( - &self, - out: &Tensor, - mask: &Tensor, - value: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_masked_fill_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mask.c_tensor, - value.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_masked_scatter(&self, mask: &Tensor, source: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_masked_scatter( - c_tensors.as_mut_ptr(), - self.c_tensor, - mask.c_tensor, - source.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_masked_scatter_( - &mut self, - mask: &Tensor, - source: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_masked_scatter_( - c_tensors.as_mut_ptr(), - self.c_tensor, - mask.c_tensor, - source.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_masked_scatter_backward( - grad_output: &Tensor, - mask: &Tensor, - sizes: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_masked_scatter_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - mask.c_tensor, - sizes.as_ptr(), - sizes.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_masked_scatter_out( - &self, - out: &Tensor, - mask: &Tensor, - source: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_masked_scatter_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mask.c_tensor, - source.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_masked_select(&self, mask: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_masked_select(c_tensors.as_mut_ptr(), self.c_tensor, mask.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_masked_select_backward( - &self, - grad: &Tensor, - mask: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_masked_select_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - self.c_tensor, - mask.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_masked_select_out(&self, out: &Tensor, mask: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_masked_select_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mask.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_matmul(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_matmul(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_matmul_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_matmul_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_matrix_exp(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_matrix_exp(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_matrix_exp_backward(&self, grad: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_matrix_exp_backward( - c_tensors.as_mut_ptr(), - self.c_tensor, - grad.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_matrix_h(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_matrix_h(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_matrix_power(&self, n: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_matrix_power(c_tensors.as_mut_ptr(), self.c_tensor, n)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_matrix_power_out(&self, out: &Tensor, n: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_matrix_power_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - n - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_max(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_max(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_max_dim(&self, dim: i64, keepdim: bool) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_max_dim( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - if keepdim { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_max_dim_max( - &self, - max: &Tensor, - max_values: &Tensor, - dim: i64, - keepdim: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_max_dim_max( - c_tensors.as_mut_ptr(), - max.c_tensor, - max_values.c_tensor, - self.c_tensor, - dim, - if keepdim { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_max_other(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_max_other(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_max_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_max_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_max_pool1d( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_max_pool1d( - c_tensors.as_mut_ptr(), - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_max_pool1d_with_indices( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_max_pool1d_with_indices( - c_tensors.as_mut_ptr(), - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_max_pool2d( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_max_pool2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_max_pool2d_backward( - &self, - grad_output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_max_pool2d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_max_pool2d_backward_out( - &self, - out: &Tensor, - grad_output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_max_pool2d_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad_output.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_max_pool2d_with_indices( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_max_pool2d_with_indices( - c_tensors.as_mut_ptr(), - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_max_pool2d_with_indices_backward( - &self, - grad_output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - indices: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_max_pool2d_with_indices_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 }, - indices.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_max_pool2d_with_indices_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - indices: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_max_pool2d_with_indices_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 }, - indices.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_max_pool2d_with_indices_out( - &self, - out: &Tensor, - indices: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_max_pool2d_with_indices_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - indices.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_max_pool3d( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_max_pool3d( - c_tensors.as_mut_ptr(), - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_max_pool3d_with_indices( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_max_pool3d_with_indices( - c_tensors.as_mut_ptr(), - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_max_pool3d_with_indices_backward( - &self, - grad_output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - indices: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_max_pool3d_with_indices_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 }, - indices.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_max_pool3d_with_indices_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - indices: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_max_pool3d_with_indices_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 }, - indices.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_max_pool3d_with_indices_out( - &self, - out: &Tensor, - indices: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_max_pool3d_with_indices_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - indices.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_max_unary_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_max_unary_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_max_unpool2d( - &self, - indices: &Tensor, - output_size: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_max_unpool2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - indices.c_tensor, - output_size.as_ptr(), - output_size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_max_unpool2d_out( - &self, - out: &Tensor, - indices: &Tensor, - output_size: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_max_unpool2d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - indices.c_tensor, - output_size.as_ptr(), - output_size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_max_unpool3d( - &self, - indices: &Tensor, - output_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_max_unpool3d( - c_tensors.as_mut_ptr(), - self.c_tensor, - indices.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_max_unpool3d_out( - &self, - out: &Tensor, - indices: &Tensor, - output_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_max_unpool3d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - indices.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_maximum(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_maximum(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_maximum_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_maximum_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mean(&self, dtype: impl Into>) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mean( - c_tensors.as_mut_ptr(), - self.c_tensor, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mean_dim( - &self, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mean_dim( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 }, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mean_dtype_out( - &self, - out: &Tensor, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mean_dtype_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mean_out( - &self, - out: &Tensor, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mean_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 }, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_median(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_median(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_median_dim(&self, dim: i64, keepdim: bool) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_median_dim( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - if keepdim { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_median_dim_values( - &self, - values: &Tensor, - indices: &Tensor, - dim: i64, - keepdim: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_median_dim_values( - c_tensors.as_mut_ptr(), - values.c_tensor, - indices.c_tensor, - self.c_tensor, - dim, - if keepdim { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_median_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_median_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_meshgrid>(tensors: &[T]) -> Result, TchError> { - let c_tensors = - unsafe_torch_err!(atg_meshgrid(ptr_list(tensors).as_ptr(), tensors.len() as i32)); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_meshgrid_indexing>( - tensors: &[T], - indexing: &str, - ) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_meshgrid_indexing( - ptr_list(tensors).as_ptr(), - tensors.len() as i32, - indexing.as_ptr(), - indexing.len() as i32 - )); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_mh(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mh(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_min(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_min(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_min_dim(&self, dim: i64, keepdim: bool) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_min_dim( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - if keepdim { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_min_dim_min( - &self, - min: &Tensor, - min_indices: &Tensor, - dim: i64, - keepdim: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_min_dim_min( - c_tensors.as_mut_ptr(), - min.c_tensor, - min_indices.c_tensor, - self.c_tensor, - dim, - if keepdim { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_min_other(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_min_other(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_min_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_min_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_min_unary_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_min_unary_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_minimum(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_minimum(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_minimum_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_minimum_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_miopen_batch_norm>( - &self, - weight: &Tensor, - bias: Option, - running_mean: Option, - running_var: Option, - training: bool, - exponential_average_factor: f64, - epsilon: f64, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_miopen_batch_norm( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if training { 1 } else { 0 }, - exponential_average_factor, - epsilon - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_miopen_batch_norm_backward>( - &self, - grad_output: &Tensor, - weight: &Tensor, - running_mean: Option, - running_var: Option, - save_mean: Option, - save_var: Option, - epsilon: f64, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_miopen_batch_norm_backward( - c_tensors.as_mut_ptr(), - self.c_tensor, - grad_output.c_tensor, - weight.c_tensor, - running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - save_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - save_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - epsilon - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_miopen_batch_norm_backward_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - grad_output: &Tensor, - weight: &Tensor, - running_mean: Option, - running_var: Option, - save_mean: Option, - save_var: Option, - epsilon: f64, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_miopen_batch_norm_backward_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - out2.c_tensor, - self.c_tensor, - grad_output.c_tensor, - weight.c_tensor, - running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - save_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - save_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - epsilon - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_miopen_batch_norm_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - weight: &Tensor, - bias: Option, - running_mean: Option, - running_var: Option, - training: bool, - exponential_average_factor: f64, - epsilon: f64, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_miopen_batch_norm_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - out2.c_tensor, - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if training { 1 } else { 0 }, - exponential_average_factor, - epsilon - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_miopen_convolution>( - &self, - weight: &Tensor, - bias: Option, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_miopen_convolution( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - padding.as_ptr(), - padding.len_i32(), - stride.as_ptr(), - stride.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups, - if benchmark { 1 } else { 0 }, - if deterministic { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_miopen_convolution_add_relu, S: Into>( - &self, - weight: &Tensor, - z: &Tensor, - alpha: S, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_miopen_convolution_add_relu( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - z.c_tensor, - alpha.into().c_scalar, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_miopen_convolution_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_miopen_convolution_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - padding.as_ptr(), - padding.len_i32(), - stride.as_ptr(), - stride.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups, - if benchmark { 1 } else { 0 }, - if deterministic { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_miopen_convolution_relu>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_miopen_convolution_relu( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_miopen_convolution_transpose>( - &self, - weight: &Tensor, - bias: Option, - padding: impl IntList, - output_padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_miopen_convolution_transpose( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - padding.as_ptr(), - padding.len_i32(), - output_padding.as_ptr(), - output_padding.len_i32(), - stride.as_ptr(), - stride.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups, - if benchmark { 1 } else { 0 }, - if deterministic { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_miopen_convolution_transpose_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - padding: impl IntList, - output_padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_miopen_convolution_transpose_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - padding.as_ptr(), - padding.len_i32(), - output_padding.as_ptr(), - output_padding.len_i32(), - stride.as_ptr(), - stride.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups, - if benchmark { 1 } else { 0 }, - if deterministic { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_miopen_ctc_loss( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: impl IntList, - target_lengths: impl IntList, - blank: i64, - deterministic: bool, - zero_infinity: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_miopen_ctc_loss( - c_tensors.as_mut_ptr(), - log_probs.c_tensor, - targets.c_tensor, - input_lengths.as_ptr(), - input_lengths.len_i32(), - target_lengths.as_ptr(), - target_lengths.len_i32(), - blank, - if deterministic { 1 } else { 0 }, - if zero_infinity { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_miopen_ctc_loss_out( - out0: &Tensor, - out1: &Tensor, - log_probs: &Tensor, - targets: &Tensor, - input_lengths: impl IntList, - target_lengths: impl IntList, - blank: i64, - deterministic: bool, - zero_infinity: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_miopen_ctc_loss_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - log_probs.c_tensor, - targets.c_tensor, - input_lengths.as_ptr(), - input_lengths.len_i32(), - target_lengths.as_ptr(), - target_lengths.len_i32(), - blank, - if deterministic { 1 } else { 0 }, - if zero_infinity { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_miopen_ctc_loss_tensor( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: &Tensor, - target_lengths: &Tensor, - blank: i64, - deterministic: bool, - zero_infinity: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_miopen_ctc_loss_tensor( - c_tensors.as_mut_ptr(), - log_probs.c_tensor, - targets.c_tensor, - input_lengths.c_tensor, - target_lengths.c_tensor, - blank, - if deterministic { 1 } else { 0 }, - if zero_infinity { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_miopen_depthwise_convolution>( - &self, - weight: &Tensor, - bias: Option, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_miopen_depthwise_convolution( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - padding.as_ptr(), - padding.len_i32(), - stride.as_ptr(), - stride.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups, - if benchmark { 1 } else { 0 }, - if deterministic { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_miopen_depthwise_convolution_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_miopen_depthwise_convolution_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - padding.as_ptr(), - padding.len_i32(), - stride.as_ptr(), - stride.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups, - if benchmark { 1 } else { 0 }, - if deterministic { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_miopen_rnn>( - &self, - weight: &[T], - weight_stride0: i64, - hx: &Tensor, - cx: Option, - mode: i64, - hidden_size: i64, - num_layers: i64, - batch_first: bool, - dropout: f64, - train: bool, - bidirectional: bool, - batch_sizes: impl IntList, - dropout_state: Option, - ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 5]; - unsafe_torch_err!(atg_miopen_rnn( - c_tensors.as_mut_ptr(), - self.c_tensor, - ptr_list(weight).as_ptr(), - weight.len() as i32, - weight_stride0, - hx.c_tensor, - cx.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - mode, - hidden_size, - num_layers, - if batch_first { 1 } else { 0 }, - dropout, - if train { 1 } else { 0 }, - if bidirectional { 1 } else { 0 }, - batch_sizes.as_ptr(), - batch_sizes.len_i32(), - dropout_state.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - Tensor { c_tensor: c_tensors[4] }, - )) - } - - pub fn f_miopen_rnn_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - out3: &Tensor, - out4: &Tensor, - weight: &[T], - weight_stride0: i64, - hx: &Tensor, - cx: Option, - mode: i64, - hidden_size: i64, - num_layers: i64, - batch_first: bool, - dropout: f64, - train: bool, - bidirectional: bool, - batch_sizes: impl IntList, - dropout_state: Option, - ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 5]; - unsafe_torch_err!(atg_miopen_rnn_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - out2.c_tensor, - out3.c_tensor, - out4.c_tensor, - self.c_tensor, - ptr_list(weight).as_ptr(), - weight.len() as i32, - weight_stride0, - hx.c_tensor, - cx.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - mode, - hidden_size, - num_layers, - if batch_first { 1 } else { 0 }, - dropout, - if train { 1 } else { 0 }, - if bidirectional { 1 } else { 0 }, - batch_sizes.as_ptr(), - batch_sizes.len_i32(), - dropout_state.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - Tensor { c_tensor: c_tensors[4] }, - )) - } - - pub fn f_mish(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mish(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mish_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mish_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mish_backward(&self, grad_output: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mish_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mish_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mish_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_adaptive_avg_pool2d( - &self, - output_size: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mkldnn_adaptive_avg_pool2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_adaptive_avg_pool2d_backward( - &self, - grad_output: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mkldnn_adaptive_avg_pool2d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_adaptive_avg_pool2d_backward_out( - &self, - out: &Tensor, - grad_output: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mkldnn_adaptive_avg_pool2d_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad_output.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_adaptive_avg_pool2d_out( - &self, - out: &Tensor, - output_size: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mkldnn_adaptive_avg_pool2d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_convolution>( - &self, - weight: &Tensor, - bias: Option, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mkldnn_convolution( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - padding.as_ptr(), - padding.len_i32(), - stride.as_ptr(), - stride.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_convolution_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mkldnn_convolution_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - padding.as_ptr(), - padding.len_i32(), - stride.as_ptr(), - stride.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_linear>( - &self, - weight: &Tensor, - bias: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mkldnn_linear( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_linear_backward_input( - input_size: impl IntList, - grad_output: &Tensor, - weight: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mkldnn_linear_backward_input( - c_tensors.as_mut_ptr(), - input_size.as_ptr(), - input_size.len_i32(), - grad_output.c_tensor, - weight.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_linear_backward_input_out( - out: &Tensor, - input_size: impl IntList, - grad_output: &Tensor, - weight: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mkldnn_linear_backward_input_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - input_size.as_ptr(), - input_size.len_i32(), - grad_output.c_tensor, - weight.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_linear_backward_weights( - &self, - grad_output: &Tensor, - weight: &Tensor, - bias_defined: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_mkldnn_linear_backward_weights( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - weight.c_tensor, - if bias_defined { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_mkldnn_linear_backward_weights_out( - &self, - out0: &Tensor, - out1: &Tensor, - grad_output: &Tensor, - weight: &Tensor, - bias_defined: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_mkldnn_linear_backward_weights_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - grad_output.c_tensor, - self.c_tensor, - weight.c_tensor, - if bias_defined { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_mkldnn_linear_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mkldnn_linear_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_max_pool2d( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mkldnn_max_pool2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_max_pool2d_backward( - &self, - grad_output: &Tensor, - output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mkldnn_max_pool2d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - output.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_max_pool2d_backward_out( - &self, - out: &Tensor, - grad_output: &Tensor, - output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mkldnn_max_pool2d_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad_output.c_tensor, - output.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_max_pool2d_out( - &self, - out: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mkldnn_max_pool2d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_max_pool3d( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mkldnn_max_pool3d( - c_tensors.as_mut_ptr(), - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_max_pool3d_backward( - &self, - grad_output: &Tensor, - output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mkldnn_max_pool3d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - output.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_max_pool3d_backward_out( - &self, - out: &Tensor, - grad_output: &Tensor, - output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mkldnn_max_pool3d_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad_output.c_tensor, - output.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_max_pool3d_out( - &self, - out: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mkldnn_max_pool3d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_reorder_conv2d_weight( - &self, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - input_size: impl IntListOption, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mkldnn_reorder_conv2d_weight( - c_tensors.as_mut_ptr(), - self.c_tensor, - padding.as_ptr(), - padding.len_i32(), - stride.as_ptr(), - stride.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups, - input_size.as_ptr(), - input_size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_reorder_conv2d_weight_out( - &self, - out: &Tensor, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - input_size: impl IntListOption, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mkldnn_reorder_conv2d_weight_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - padding.as_ptr(), - padding.len_i32(), - stride.as_ptr(), - stride.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups, - input_size.as_ptr(), - input_size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_reorder_conv3d_weight( - &self, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - input_size: impl IntListOption, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mkldnn_reorder_conv3d_weight( - c_tensors.as_mut_ptr(), - self.c_tensor, - padding.as_ptr(), - padding.len_i32(), - stride.as_ptr(), - stride.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups, - input_size.as_ptr(), - input_size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_reorder_conv3d_weight_out( - &self, - out: &Tensor, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - input_size: impl IntListOption, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mkldnn_reorder_conv3d_weight_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - padding.as_ptr(), - padding.len_i32(), - stride.as_ptr(), - stride.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - groups, - input_size.as_ptr(), - input_size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mkldnn_rnn_layer( - &self, - weight0: &Tensor, - weight1: &Tensor, - weight2: &Tensor, - weight3: &Tensor, - hx_: &Tensor, - cx_: &Tensor, - reverse: bool, - batch_sizes: impl IntList, - mode: i64, - hidden_size: i64, - num_layers: i64, - has_biases: bool, - bidirectional: bool, - batch_first: bool, - train: bool, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg_mkldnn_rnn_layer( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight0.c_tensor, - weight1.c_tensor, - weight2.c_tensor, - weight3.c_tensor, - hx_.c_tensor, - cx_.c_tensor, - if reverse { 1 } else { 0 }, - batch_sizes.as_ptr(), - batch_sizes.len_i32(), - mode, - hidden_size, - num_layers, - if has_biases { 1 } else { 0 }, - if bidirectional { 1 } else { 0 }, - if batch_first { 1 } else { 0 }, - if train { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_mkldnn_rnn_layer_backward>( - &self, - weight1: &Tensor, - weight2: &Tensor, - weight3: &Tensor, - weight4: &Tensor, - hx_: &Tensor, - cx_tmp: &Tensor, - output: &Tensor, - hy_: &Tensor, - cy_: &Tensor, - grad_output: Option, - grad_hy: Option, - grad_cy: Option, - reverse: bool, - mode: i64, - hidden_size: i64, - num_layers: i64, - has_biases: bool, - train: bool, - bidirectional: bool, - batch_sizes: impl IntList, - batch_first: bool, - workspace: &Tensor, - ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 7]; - unsafe_torch_err!(atg_mkldnn_rnn_layer_backward( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight1.c_tensor, - weight2.c_tensor, - weight3.c_tensor, - weight4.c_tensor, - hx_.c_tensor, - cx_tmp.c_tensor, - output.c_tensor, - hy_.c_tensor, - cy_.c_tensor, - grad_output.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - grad_hy.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - grad_cy.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if reverse { 1 } else { 0 }, - mode, - hidden_size, - num_layers, - if has_biases { 1 } else { 0 }, - if train { 1 } else { 0 }, - if bidirectional { 1 } else { 0 }, - batch_sizes.as_ptr(), - batch_sizes.len_i32(), - if batch_first { 1 } else { 0 }, - workspace.c_tensor - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - Tensor { c_tensor: c_tensors[4] }, - Tensor { c_tensor: c_tensors[5] }, - Tensor { c_tensor: c_tensors[6] }, - )) - } - - pub fn f_mkldnn_rnn_layer_backward_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - out3: &Tensor, - out4: &Tensor, - out5: &Tensor, - out6: &Tensor, - weight1: &Tensor, - weight2: &Tensor, - weight3: &Tensor, - weight4: &Tensor, - hx_: &Tensor, - cx_tmp: &Tensor, - output: &Tensor, - hy_: &Tensor, - cy_: &Tensor, - grad_output: Option, - grad_hy: Option, - grad_cy: Option, - reverse: bool, - mode: i64, - hidden_size: i64, - num_layers: i64, - has_biases: bool, - train: bool, - bidirectional: bool, - batch_sizes: impl IntList, - batch_first: bool, - workspace: &Tensor, - ) -> Result<(Tensor, Tensor, Tensor, Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 7]; - unsafe_torch_err!(atg_mkldnn_rnn_layer_backward_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - out2.c_tensor, - out3.c_tensor, - out4.c_tensor, - out5.c_tensor, - out6.c_tensor, - self.c_tensor, - weight1.c_tensor, - weight2.c_tensor, - weight3.c_tensor, - weight4.c_tensor, - hx_.c_tensor, - cx_tmp.c_tensor, - output.c_tensor, - hy_.c_tensor, - cy_.c_tensor, - grad_output.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - grad_hy.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - grad_cy.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if reverse { 1 } else { 0 }, - mode, - hidden_size, - num_layers, - if has_biases { 1 } else { 0 }, - if train { 1 } else { 0 }, - if bidirectional { 1 } else { 0 }, - batch_sizes.as_ptr(), - batch_sizes.len_i32(), - if batch_first { 1 } else { 0 }, - workspace.c_tensor - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - Tensor { c_tensor: c_tensors[4] }, - Tensor { c_tensor: c_tensors[5] }, - Tensor { c_tensor: c_tensors[6] }, - )) - } - - pub fn f_mkldnn_rnn_layer_out( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - out3: &Tensor, - weight0: &Tensor, - weight1: &Tensor, - weight2: &Tensor, - weight3: &Tensor, - hx_: &Tensor, - cx_: &Tensor, - reverse: bool, - batch_sizes: impl IntList, - mode: i64, - hidden_size: i64, - num_layers: i64, - has_biases: bool, - bidirectional: bool, - batch_first: bool, - train: bool, - ) -> Result<(Tensor, Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 4]; - unsafe_torch_err!(atg_mkldnn_rnn_layer_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - out2.c_tensor, - out3.c_tensor, - self.c_tensor, - weight0.c_tensor, - weight1.c_tensor, - weight2.c_tensor, - weight3.c_tensor, - hx_.c_tensor, - cx_.c_tensor, - if reverse { 1 } else { 0 }, - batch_sizes.as_ptr(), - batch_sizes.len_i32(), - mode, - hidden_size, - num_layers, - if has_biases { 1 } else { 0 }, - if bidirectional { 1 } else { 0 }, - if batch_first { 1 } else { 0 }, - if train { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - Tensor { c_tensor: c_tensors[3] }, - )) - } - - pub fn f_mm(&self, mat2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mm(c_tensors.as_mut_ptr(), self.c_tensor, mat2.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mm_dtype(&self, mat2: &Tensor, out_dtype: Kind) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mm_dtype( - c_tensors.as_mut_ptr(), - self.c_tensor, - mat2.c_tensor, - out_dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mm_dtype_out( - &self, - out: &Tensor, - mat2: &Tensor, - out_dtype: Kind, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mm_dtype_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mat2.c_tensor, - out_dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mm_out(&self, out: &Tensor, mat2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mm_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mat2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mode(&self, dim: i64, keepdim: bool) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_mode( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - if keepdim { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_mode_values( - &self, - values: &Tensor, - indices: &Tensor, - dim: i64, - keepdim: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_mode_values( - c_tensors.as_mut_ptr(), - values.c_tensor, - indices.c_tensor, - self.c_tensor, - dim, - if keepdim { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_moveaxis( - &self, - source: impl IntList, - destination: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_moveaxis( - c_tensors.as_mut_ptr(), - self.c_tensor, - source.as_ptr(), - source.len_i32(), - destination.as_ptr(), - destination.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_moveaxis_int(&self, source: i64, destination: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_moveaxis_int( - c_tensors.as_mut_ptr(), - self.c_tensor, - source, - destination - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_movedim( - &self, - source: impl IntList, - destination: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_movedim( - c_tensors.as_mut_ptr(), - self.c_tensor, - source.as_ptr(), - source.len_i32(), - destination.as_ptr(), - destination.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_movedim_int(&self, source: i64, destination: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_movedim_int( - c_tensors.as_mut_ptr(), - self.c_tensor, - source, - destination - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mse_loss( - &self, - target: &Tensor, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mse_loss( - c_tensors.as_mut_ptr(), - self.c_tensor, - target.c_tensor, - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mse_loss_backward( - &self, - grad_output: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mse_loss_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - target.c_tensor, - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mse_loss_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mse_loss_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - target.c_tensor, - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mse_loss_out( - &self, - out: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mse_loss_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - target.c_tensor, - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_msort(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_msort(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_msort_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_msort_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mt(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mt(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mul(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mul(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mul_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mul_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mul_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mul_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mul_scalar>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mul_scalar( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mul_scalar_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mul_scalar_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mul_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mul_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_multi_margin_loss_backward, S: Into>( - &self, - grad_output: &Tensor, - target: &Tensor, - p: S, - margin: S, - weight: Option, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_multi_margin_loss_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - target.c_tensor, - p.into().c_scalar, - margin.into().c_scalar, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_multi_margin_loss_backward_grad_input, S: Into>( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - target: &Tensor, - p: S, - margin: S, - weight: Option, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_multi_margin_loss_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - target.c_tensor, - p.into().c_scalar, - margin.into().c_scalar, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_multilabel_margin_loss( - &self, - target: &Tensor, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_multilabel_margin_loss( - c_tensors.as_mut_ptr(), - self.c_tensor, - target.c_tensor, - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_multilabel_margin_loss_backward( - &self, - grad_output: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - is_target: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_multilabel_margin_loss_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - target.c_tensor, - reduction.to_int(), - is_target.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_multilabel_margin_loss_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - is_target: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_multilabel_margin_loss_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - target.c_tensor, - reduction.to_int(), - is_target.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_multilabel_margin_loss_out( - &self, - out: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_multilabel_margin_loss_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - target.c_tensor, - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_multinomial(&self, num_samples: i64, replacement: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_multinomial( - c_tensors.as_mut_ptr(), - self.c_tensor, - num_samples, - if replacement { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_multinomial_out( - &self, - out: &Tensor, - num_samples: i64, - replacement: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_multinomial_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - num_samples, - if replacement { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_multiply(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_multiply(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_multiply_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_multiply_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_multiply_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_multiply_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_multiply_scalar>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_multiply_scalar( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_multiply_scalar_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_multiply_scalar_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mv(&self, vec: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mv(c_tensors.as_mut_ptr(), self.c_tensor, vec.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mv_out(&self, out: &Tensor, vec: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mv_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - vec.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mvlgamma(&self, p: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mvlgamma(c_tensors.as_mut_ptr(), self.c_tensor, p)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mvlgamma_(&mut self, p: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mvlgamma_(c_tensors.as_mut_ptr(), self.c_tensor, p)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_mvlgamma_out(&self, out: &Tensor, p: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_mvlgamma_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor, p)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nan_to_num( - &self, - nan: impl Into>, - posinf: impl Into>, - neginf: impl Into>, - ) -> Result { - let nan = nan.into(); - let posinf = posinf.into(); - let neginf = neginf.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nan_to_num( - c_tensors.as_mut_ptr(), - self.c_tensor, - nan.unwrap_or(std::f64::NAN), - nan.is_none() as i8, - posinf.unwrap_or(std::f64::NAN), - posinf.is_none() as i8, - neginf.unwrap_or(std::f64::NAN), - neginf.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nan_to_num_( - &mut self, - nan: impl Into>, - posinf: impl Into>, - neginf: impl Into>, - ) -> Result { - let nan = nan.into(); - let posinf = posinf.into(); - let neginf = neginf.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nan_to_num_( - c_tensors.as_mut_ptr(), - self.c_tensor, - nan.unwrap_or(std::f64::NAN), - nan.is_none() as i8, - posinf.unwrap_or(std::f64::NAN), - posinf.is_none() as i8, - neginf.unwrap_or(std::f64::NAN), - neginf.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nan_to_num_out( - &self, - out: &Tensor, - nan: impl Into>, - posinf: impl Into>, - neginf: impl Into>, - ) -> Result { - let nan = nan.into(); - let posinf = posinf.into(); - let neginf = neginf.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nan_to_num_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - nan.unwrap_or(std::f64::NAN), - nan.is_none() as i8, - posinf.unwrap_or(std::f64::NAN), - posinf.is_none() as i8, - neginf.unwrap_or(std::f64::NAN), - neginf.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nanmean( - &self, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nanmean( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 }, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nanmean_out( - &self, - out: &Tensor, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nanmean_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 }, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nanmedian(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nanmedian(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nanmedian_dim(&self, dim: i64, keepdim: bool) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_nanmedian_dim( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - if keepdim { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_nanmedian_dim_values( - &self, - values: &Tensor, - indices: &Tensor, - dim: i64, - keepdim: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_nanmedian_dim_values( - c_tensors.as_mut_ptr(), - values.c_tensor, - indices.c_tensor, - self.c_tensor, - dim, - if keepdim { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_nanmedian_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nanmedian_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nanquantile( - &self, - q: &Tensor, - dim: impl Into>, - keepdim: bool, - interpolation: &str, - ) -> Result { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nanquantile( - c_tensors.as_mut_ptr(), - self.c_tensor, - q.c_tensor, - dim.unwrap_or(0i64), - dim.is_none() as i8, - if keepdim { 1 } else { 0 }, - interpolation.as_ptr(), - interpolation.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nanquantile_out( - &self, - out: &Tensor, - q: &Tensor, - dim: impl Into>, - keepdim: bool, - interpolation: &str, - ) -> Result { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nanquantile_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - q.c_tensor, - dim.unwrap_or(0i64), - dim.is_none() as i8, - if keepdim { 1 } else { 0 }, - interpolation.as_ptr(), - interpolation.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nanquantile_scalar( - &self, - q: f64, - dim: impl Into>, - keepdim: bool, - interpolation: &str, - ) -> Result { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nanquantile_scalar( - c_tensors.as_mut_ptr(), - self.c_tensor, - q, - dim.unwrap_or(0i64), - dim.is_none() as i8, - if keepdim { 1 } else { 0 }, - interpolation.as_ptr(), - interpolation.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nanquantile_scalar_out( - &self, - out: &Tensor, - q: f64, - dim: impl Into>, - keepdim: bool, - interpolation: &str, - ) -> Result { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nanquantile_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - q, - dim.unwrap_or(0i64), - dim.is_none() as i8, - if keepdim { 1 } else { 0 }, - interpolation.as_ptr(), - interpolation.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nansum( - &self, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nansum( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 }, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nansum_out( - &self, - out: &Tensor, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nansum_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 }, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_narrow(&self, dim: i64, start: i64, length: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_narrow(c_tensors.as_mut_ptr(), self.c_tensor, dim, start, length)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_narrow_copy(&self, dim: i64, start: i64, length: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_narrow_copy( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - start, - length - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_narrow_copy_out( - &self, - out: &Tensor, - dim: i64, - start: i64, - length: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_narrow_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - start, - length - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_narrow_tensor( - &self, - dim: i64, - start: &Tensor, - length: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_narrow_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - start.c_tensor, - length - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_native_batch_norm>( - &self, - weight: Option, - bias: Option, - running_mean: Option, - running_var: Option, - training: bool, - momentum: f64, - eps: f64, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_native_batch_norm( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if training { 1 } else { 0 }, - momentum, - eps - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_native_batch_norm_out>( - &self, - out: &Tensor, - save_mean: &Tensor, - save_invstd: &Tensor, - weight: Option, - bias: Option, - running_mean: Option, - running_var: Option, - training: bool, - momentum: f64, - eps: f64, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_native_batch_norm_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - save_mean.c_tensor, - save_invstd.c_tensor, - self.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_mean.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - running_var.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if training { 1 } else { 0 }, - momentum, - eps - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_native_channel_shuffle(&self, groups: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_native_channel_shuffle( - c_tensors.as_mut_ptr(), - self.c_tensor, - groups - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_native_dropout(&self, p: f64, train: bool) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_native_dropout( - c_tensors.as_mut_ptr(), - self.c_tensor, - p, - if train { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_native_dropout_backward( - grad_output: &Tensor, - mask: &Tensor, - scale: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_native_dropout_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - mask.c_tensor, - scale - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_native_dropout_backward_out( - out: &Tensor, - grad_output: &Tensor, - mask: &Tensor, - scale: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_native_dropout_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad_output.c_tensor, - mask.c_tensor, - scale - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_native_dropout_out( - &self, - out0: &Tensor, - out1: &Tensor, - p: f64, - train: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_native_dropout_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - self.c_tensor, - p, - if train { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_native_group_norm>( - &self, - weight: Option, - bias: Option, - n: i64, - c: i64, - hxw: i64, - group: i64, - eps: f64, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_native_group_norm( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - n, - c, - hxw, - group, - eps - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_native_group_norm_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - weight: Option, - bias: Option, - n: i64, - c: i64, - hxw: i64, - group: i64, - eps: f64, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_native_group_norm_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - out2.c_tensor, - self.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - n, - c, - hxw, - group, - eps - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_native_layer_norm>( - &self, - normalized_shape: impl IntList, - weight: Option, - bias: Option, - eps: f64, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_native_layer_norm( - c_tensors.as_mut_ptr(), - self.c_tensor, - normalized_shape.as_ptr(), - normalized_shape.len_i32(), - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - eps - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_native_layer_norm_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - normalized_shape: impl IntList, - weight: Option, - bias: Option, - eps: f64, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_native_layer_norm_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - out2.c_tensor, - self.c_tensor, - normalized_shape.as_ptr(), - normalized_shape.len_i32(), - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - eps - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_native_norm(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_native_norm(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_native_norm_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_native_norm_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_native_norm_scalaropt_dim_dtype>( - &self, - p: S, - dim: impl IntList, - keepdim: bool, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_native_norm_scalaropt_dim_dtype( - c_tensors.as_mut_ptr(), - self.c_tensor, - p.into().c_scalar, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 }, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_native_norm_scalaropt_dim_dtype_out>( - &self, - out: &Tensor, - p: S, - dim: impl IntList, - keepdim: bool, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_native_norm_scalaropt_dim_dtype_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - p.into().c_scalar, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 }, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ne>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ne(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ne_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ne_(c_tensors.as_mut_ptr(), self.c_tensor, other.into().c_scalar)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ne_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ne_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ne_tensor(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ne_tensor(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ne_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ne_tensor_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ne_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ne_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_neg(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_neg(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_neg_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_neg_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_neg_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_neg_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_negative(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_negative(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_negative_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_negative_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_negative_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_negative_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nested_to_padded_tensor( - &self, - padding: f64, - output_size: impl IntListOption, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nested_to_padded_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - padding, - output_size.as_ptr(), - output_size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_new_empty( - &self, - size: impl IntList, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_new_empty( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_new_empty_out(&self, out: &Tensor, size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_new_empty_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_new_empty_strided( - &self, - size: impl IntList, - stride: impl IntList, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_new_empty_strided( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_new_empty_strided_out( - &self, - out: &Tensor, - size: impl IntList, - stride: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_new_empty_strided_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - size.as_ptr(), - size.len_i32(), - stride.as_ptr(), - stride.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_new_full>( - &self, - size: impl IntList, - fill_value: S, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_new_full( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32(), - fill_value.into().c_scalar, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_new_full_out>( - &self, - out: &Tensor, - size: impl IntList, - fill_value: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_new_full_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - size.as_ptr(), - size.len_i32(), - fill_value.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_new_ones( - &self, - size: impl IntList, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_new_ones( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_new_ones_out(&self, out: &Tensor, size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_new_ones_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_new_zeros( - &self, - size: impl IntList, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_new_zeros( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_new_zeros_out(&self, out: &Tensor, size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_new_zeros_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nextafter(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nextafter(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nextafter_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nextafter_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nextafter_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nextafter_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nll_loss>( - &self, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ignore_index: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nll_loss( - c_tensors.as_mut_ptr(), - self.c_tensor, - target.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - reduction.to_int(), - ignore_index - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nll_loss2d>( - &self, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ignore_index: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nll_loss2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - target.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - reduction.to_int(), - ignore_index - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nll_loss2d_backward>( - &self, - grad_output: &Tensor, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ignore_index: i64, - total_weight: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nll_loss2d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - target.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - reduction.to_int(), - ignore_index, - total_weight.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nll_loss2d_backward_grad_input>( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ignore_index: i64, - total_weight: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nll_loss2d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - target.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - reduction.to_int(), - ignore_index, - total_weight.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nll_loss2d_out>( - &self, - out: &Tensor, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ignore_index: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nll_loss2d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - target.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - reduction.to_int(), - ignore_index - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nll_loss_backward>( - &self, - grad_output: &Tensor, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ignore_index: i64, - total_weight: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nll_loss_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - target.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - reduction.to_int(), - ignore_index, - total_weight.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nll_loss_backward_grad_input>( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ignore_index: i64, - total_weight: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nll_loss_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - target.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - reduction.to_int(), - ignore_index, - total_weight.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nll_loss_nd>( - &self, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ignore_index: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nll_loss_nd( - c_tensors.as_mut_ptr(), - self.c_tensor, - target.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - reduction.to_int(), - ignore_index - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nll_loss_out>( - &self, - out: &Tensor, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ignore_index: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nll_loss_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - target.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - reduction.to_int(), - ignore_index - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nonzero(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nonzero(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nonzero_numpy(&self) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_nonzero_numpy(self.c_tensor)); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_nonzero_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nonzero_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nonzero_static(&self, size: i64, fill_value: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nonzero_static( - c_tensors.as_mut_ptr(), - self.c_tensor, - size, - fill_value - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nonzero_static_out( - &self, - out: &Tensor, - size: i64, - fill_value: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nonzero_static_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - size, - fill_value - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_norm(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_norm(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_norm_dtype_out>( - &self, - out: &Tensor, - p: S, - dim: impl IntList, - keepdim: bool, - dtype: Kind, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_norm_dtype_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - p.into().c_scalar, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 }, - dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_norm_except_dim(v: &Tensor, pow: i64, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_norm_except_dim(c_tensors.as_mut_ptr(), v.c_tensor, pow, dim)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_norm_out>( - &self, - out: &Tensor, - p: S, - dim: impl IntList, - keepdim: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_norm_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - p.into().c_scalar, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_norm_scalar_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_norm_scalar_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_norm_scalaropt_dim>( - &self, - p: S, - dim: impl IntList, - keepdim: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_norm_scalaropt_dim( - c_tensors.as_mut_ptr(), - self.c_tensor, - p.into().c_scalar, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_norm_scalaropt_dim_dtype>( - &self, - p: S, - dim: impl IntList, - keepdim: bool, - dtype: Kind, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_norm_scalaropt_dim_dtype( - c_tensors.as_mut_ptr(), - self.c_tensor, - p.into().c_scalar, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 }, - dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_norm_scalaropt_dtype>( - &self, - p: S, - dtype: Kind, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_norm_scalaropt_dtype( - c_tensors.as_mut_ptr(), - self.c_tensor, - p.into().c_scalar, - dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_norm_scalaropt_dtype_out>( - &self, - out: &Tensor, - p: S, - dtype: Kind, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_norm_scalaropt_dtype_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - p.into().c_scalar, - dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_normal_(&mut self, mean: f64, std: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_normal_(c_tensors.as_mut_ptr(), self.c_tensor, mean, std)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_normal_functional(&self, mean: f64, std: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_normal_functional(c_tensors.as_mut_ptr(), self.c_tensor, mean, std)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_not_equal>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_not_equal( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_not_equal_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_not_equal_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_not_equal_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_not_equal_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_not_equal_tensor(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_not_equal_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_not_equal_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_not_equal_tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_not_equal_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_not_equal_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nuclear_norm(&self, keepdim: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nuclear_norm( - c_tensors.as_mut_ptr(), - self.c_tensor, - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nuclear_norm_dim(&self, dim: impl IntList, keepdim: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nuclear_norm_dim( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nuclear_norm_dim_out( - &self, - out: &Tensor, - dim: impl IntList, - keepdim: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nuclear_norm_dim_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_nuclear_norm_out(&self, out: &Tensor, keepdim: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_nuclear_norm_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_numpy_t(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_numpy_t(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_one_hot(&self, num_classes: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_one_hot(c_tensors.as_mut_ptr(), self.c_tensor, num_classes)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ones(size: impl IntList, options: (Kind, Device)) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ones( - c_tensors.as_mut_ptr(), - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ones_like(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ones_like(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ones_like_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ones_like_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ones_out(out: &Tensor, size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ones_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_orgqr(&self, input2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_orgqr(c_tensors.as_mut_ptr(), self.c_tensor, input2.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_orgqr_out(&self, out: &Tensor, input2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_orgqr_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - input2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ormqr( - &self, - input2: &Tensor, - input3: &Tensor, - left: bool, - transpose: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ormqr( - c_tensors.as_mut_ptr(), - self.c_tensor, - input2.c_tensor, - input3.c_tensor, - if left { 1 } else { 0 }, - if transpose { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ormqr_out( - &self, - out: &Tensor, - input2: &Tensor, - input3: &Tensor, - left: bool, - transpose: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ormqr_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - input2.c_tensor, - input3.c_tensor, - if left { 1 } else { 0 }, - if transpose { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_outer(&self, vec2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_outer(c_tensors.as_mut_ptr(), self.c_tensor, vec2.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_outer_out(&self, out: &Tensor, vec2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_outer_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - vec2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_output_nr(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_output_nr(self.c_tensor)); - Ok(return_) - } - - pub fn f_pad( - &self, - pad: impl IntList, - mode: &str, - value: impl Into>, - ) -> Result { - let value = value.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_pad( - c_tensors.as_mut_ptr(), - self.c_tensor, - pad.as_ptr(), - pad.len_i32(), - mode.as_ptr(), - mode.len() as i32, - value.unwrap_or(std::f64::NAN), - value.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_pad_sequence>( - sequences: &[T], - batch_first: bool, - padding_value: f64, - padding_side: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_pad_sequence( - c_tensors.as_mut_ptr(), - ptr_list(sequences).as_ptr(), - sequences.len() as i32, - if batch_first { 1 } else { 0 }, - padding_value, - padding_side.as_ptr(), - padding_side.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_pairwise_distance( - x1: &Tensor, - x2: &Tensor, - p: f64, - eps: f64, - keepdim: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_pairwise_distance( - c_tensors.as_mut_ptr(), - x1.c_tensor, - x2.c_tensor, - p, - eps, - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_pdist(&self, p: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_pdist(c_tensors.as_mut_ptr(), self.c_tensor, p)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_permute(&self, dims: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_permute( - c_tensors.as_mut_ptr(), - self.c_tensor, - dims.as_ptr(), - dims.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_permute_copy(&self, dims: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_permute_copy( - c_tensors.as_mut_ptr(), - self.c_tensor, - dims.as_ptr(), - dims.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_permute_copy_out(&self, out: &Tensor, dims: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_permute_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dims.as_ptr(), - dims.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_pin_memory(&self, device: Device) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_pin_memory(c_tensors.as_mut_ptr(), self.c_tensor, device.c_int())); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_pinverse(&self, rcond: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_pinverse(c_tensors.as_mut_ptr(), self.c_tensor, rcond)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_pixel_shuffle(&self, upscale_factor: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_pixel_shuffle(c_tensors.as_mut_ptr(), self.c_tensor, upscale_factor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_pixel_shuffle_out( - &self, - out: &Tensor, - upscale_factor: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_pixel_shuffle_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - upscale_factor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_pixel_unshuffle(&self, downscale_factor: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_pixel_unshuffle( - c_tensors.as_mut_ptr(), - self.c_tensor, - downscale_factor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_pixel_unshuffle_out( - &self, - out: &Tensor, - downscale_factor: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_pixel_unshuffle_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - downscale_factor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_poisson(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_poisson(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_poisson_nll_loss( - &self, - target: &Tensor, - log_input: bool, - full: bool, - eps: f64, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_poisson_nll_loss( - c_tensors.as_mut_ptr(), - self.c_tensor, - target.c_tensor, - if log_input { 1 } else { 0 }, - if full { 1 } else { 0 }, - eps, - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_poisson_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_poisson_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_polar(abs: &Tensor, angle: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_polar(c_tensors.as_mut_ptr(), abs.c_tensor, angle.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_polar_out(out: &Tensor, abs: &Tensor, angle: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_polar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - abs.c_tensor, - angle.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_polygamma(&self, n: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_polygamma(c_tensors.as_mut_ptr(), n, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_polygamma_(&mut self, n: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_polygamma_(c_tensors.as_mut_ptr(), self.c_tensor, n)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_polygamma_out(&self, out: &Tensor, n: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_polygamma_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - n, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_positive(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_positive(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_pow(&self, exponent: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_pow(c_tensors.as_mut_ptr(), self.c_tensor, exponent.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_pow_>(&mut self, exponent: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_pow_( - c_tensors.as_mut_ptr(), - self.c_tensor, - exponent.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_pow_scalar>( - self_scalar: S, - exponent: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_pow_scalar( - c_tensors.as_mut_ptr(), - self_scalar.into().c_scalar, - exponent.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_pow_scalar_out>( - out: &Tensor, - self_scalar: S, - exponent: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_pow_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self_scalar.into().c_scalar, - exponent.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_pow_tensor_(&mut self, exponent: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_pow_tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - exponent.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_pow_tensor_scalar>(&self, exponent: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_pow_tensor_scalar( - c_tensors.as_mut_ptr(), - self.c_tensor, - exponent.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_pow_tensor_scalar_out>( - &self, - out: &Tensor, - exponent: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_pow_tensor_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - exponent.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_pow_tensor_tensor_out( - &self, - out: &Tensor, - exponent: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_pow_tensor_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - exponent.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_prelu(&self, weight: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_prelu(c_tensors.as_mut_ptr(), self.c_tensor, weight.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_prod(&self, dtype: impl Into>) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_prod( - c_tensors.as_mut_ptr(), - self.c_tensor, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_prod_dim_int( - &self, - dim: i64, - keepdim: bool, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_prod_dim_int( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - if keepdim { 1 } else { 0 }, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_prod_int_out( - &self, - out: &Tensor, - dim: i64, - keepdim: bool, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_prod_int_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - if keepdim { 1 } else { 0 }, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_prod_out( - &self, - out: &Tensor, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_prod_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_put( - &self, - index: &Tensor, - source: &Tensor, - accumulate: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_put( - c_tensors.as_mut_ptr(), - self.c_tensor, - index.c_tensor, - source.c_tensor, - if accumulate { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_put_( - &mut self, - index: &Tensor, - source: &Tensor, - accumulate: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_put_( - c_tensors.as_mut_ptr(), - self.c_tensor, - index.c_tensor, - source.c_tensor, - if accumulate { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_put_out( - &self, - out: &Tensor, - index: &Tensor, - source: &Tensor, - accumulate: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_put_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - index.c_tensor, - source.c_tensor, - if accumulate { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_q_per_channel_axis(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_q_per_channel_axis(self.c_tensor)); - Ok(return_) - } - - pub fn f_q_per_channel_scales(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_q_per_channel_scales(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_q_per_channel_scales_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_q_per_channel_scales_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_q_per_channel_zero_points(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_q_per_channel_zero_points(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_q_per_channel_zero_points_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_q_per_channel_zero_points_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_q_scale(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_q_scale(self.c_tensor)); - Ok(return_) - } - - pub fn f_q_zero_point(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_q_zero_point(self.c_tensor)); - Ok(return_) - } - - pub fn f_qr(&self, some: bool) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_qr(c_tensors.as_mut_ptr(), self.c_tensor, if some { 1 } else { 0 })); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_qr_q(&self, q: &Tensor, r: &Tensor, some: bool) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_qr_q( - c_tensors.as_mut_ptr(), - q.c_tensor, - r.c_tensor, - self.c_tensor, - if some { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_quantile( - &self, - q: &Tensor, - dim: impl Into>, - keepdim: bool, - interpolation: &str, - ) -> Result { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantile( - c_tensors.as_mut_ptr(), - self.c_tensor, - q.c_tensor, - dim.unwrap_or(0i64), - dim.is_none() as i8, - if keepdim { 1 } else { 0 }, - interpolation.as_ptr(), - interpolation.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_quantile_out( - &self, - out: &Tensor, - q: &Tensor, - dim: impl Into>, - keepdim: bool, - interpolation: &str, - ) -> Result { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantile_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - q.c_tensor, - dim.unwrap_or(0i64), - dim.is_none() as i8, - if keepdim { 1 } else { 0 }, - interpolation.as_ptr(), - interpolation.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_quantile_scalar( - &self, - q: f64, - dim: impl Into>, - keepdim: bool, - interpolation: &str, - ) -> Result { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantile_scalar( - c_tensors.as_mut_ptr(), - self.c_tensor, - q, - dim.unwrap_or(0i64), - dim.is_none() as i8, - if keepdim { 1 } else { 0 }, - interpolation.as_ptr(), - interpolation.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_quantile_scalar_out( - &self, - out: &Tensor, - q: f64, - dim: impl Into>, - keepdim: bool, - interpolation: &str, - ) -> Result { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantile_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - q, - dim.unwrap_or(0i64), - dim.is_none() as i8, - if keepdim { 1 } else { 0 }, - interpolation.as_ptr(), - interpolation.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_quantize_per_channel( - &self, - scales: &Tensor, - zero_points: &Tensor, - axis: i64, - dtype: Kind, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantize_per_channel( - c_tensors.as_mut_ptr(), - self.c_tensor, - scales.c_tensor, - zero_points.c_tensor, - axis, - dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_quantize_per_channel_out( - &self, - out: &Tensor, - scales: &Tensor, - zero_points: &Tensor, - axis: i64, - dtype: Kind, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantize_per_channel_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - scales.c_tensor, - zero_points.c_tensor, - axis, - dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_quantize_per_tensor( - &self, - scale: f64, - zero_point: i64, - dtype: Kind, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantize_per_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - scale, - zero_point, - dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_quantize_per_tensor_dynamic( - &self, - dtype: Kind, - reduce_range: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantize_per_tensor_dynamic( - c_tensors.as_mut_ptr(), - self.c_tensor, - dtype.c_int(), - if reduce_range { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_quantize_per_tensor_dynamic_out( - &self, - out: &Tensor, - dtype: Kind, - reduce_range: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantize_per_tensor_dynamic_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dtype.c_int(), - if reduce_range { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_quantize_per_tensor_out( - &self, - out: &Tensor, - scale: f64, - zero_point: i64, - dtype: Kind, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantize_per_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - scale, - zero_point, - dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_quantize_per_tensor_tensor_qparams( - &self, - scale: &Tensor, - zero_point: &Tensor, - dtype: Kind, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantize_per_tensor_tensor_qparams( - c_tensors.as_mut_ptr(), - self.c_tensor, - scale.c_tensor, - zero_point.c_tensor, - dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_quantize_per_tensor_tensor_qparams_out( - &self, - out: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - dtype: Kind, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantize_per_tensor_tensor_qparams_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - scale.c_tensor, - zero_point.c_tensor, - dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_quantize_per_tensor_tensors>( - tensors: &[T], - scales: &Tensor, - zero_points: &Tensor, - dtype: Kind, - ) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_quantize_per_tensor_tensors( - ptr_list(tensors).as_ptr(), - tensors.len() as i32, - scales.c_tensor, - zero_points.c_tensor, - dtype.c_int() - )); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_quantize_per_tensor_tensors_out>( - out: &[T], - tensors: &[T], - scales: &Tensor, - zero_points: &Tensor, - dtype: Kind, - ) -> Result<(), TchError> { - unsafe_torch_err!(atg_quantize_per_tensor_tensors_out( - ptr_list(out).as_ptr(), - out.len() as i32, - ptr_list(tensors).as_ptr(), - tensors.len() as i32, - scales.c_tensor, - zero_points.c_tensor, - dtype.c_int() - )); - Ok(()) - } - - pub fn f_quantized_batch_norm>( - &self, - weight: Option, - bias: Option, - mean: &Tensor, - var: &Tensor, - eps: f64, - output_scale: f64, - output_zero_point: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantized_batch_norm( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - mean.c_tensor, - var.c_tensor, - eps, - output_scale, - output_zero_point - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_quantized_batch_norm_out>( - &self, - out: &Tensor, - weight: Option, - bias: Option, - mean: &Tensor, - var: &Tensor, - eps: f64, - output_scale: f64, - output_zero_point: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantized_batch_norm_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - mean.c_tensor, - var.c_tensor, - eps, - output_scale, - output_zero_point - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_quantized_gru_cell>( - &self, - hx: &Tensor, - w_ih: &Tensor, - w_hh: &Tensor, - b_ih: &Tensor, - b_hh: &Tensor, - packed_ih: &Tensor, - packed_hh: &Tensor, - col_offsets_ih: &Tensor, - col_offsets_hh: &Tensor, - scale_ih: S, - scale_hh: S, - zero_point_ih: S, - zero_point_hh: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantized_gru_cell( - c_tensors.as_mut_ptr(), - self.c_tensor, - hx.c_tensor, - w_ih.c_tensor, - w_hh.c_tensor, - b_ih.c_tensor, - b_hh.c_tensor, - packed_ih.c_tensor, - packed_hh.c_tensor, - col_offsets_ih.c_tensor, - col_offsets_hh.c_tensor, - scale_ih.into().c_scalar, - scale_hh.into().c_scalar, - zero_point_ih.into().c_scalar, - zero_point_hh.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_quantized_lstm_cell, S: Into>( - &self, - hx: &[T], - w_ih: &Tensor, - w_hh: &Tensor, - b_ih: &Tensor, - b_hh: &Tensor, - packed_ih: &Tensor, - packed_hh: &Tensor, - col_offsets_ih: &Tensor, - col_offsets_hh: &Tensor, - scale_ih: S, - scale_hh: S, - zero_point_ih: S, - zero_point_hh: S, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_quantized_lstm_cell( - c_tensors.as_mut_ptr(), - self.c_tensor, - ptr_list(hx).as_ptr(), - hx.len() as i32, - w_ih.c_tensor, - w_hh.c_tensor, - b_ih.c_tensor, - b_hh.c_tensor, - packed_ih.c_tensor, - packed_hh.c_tensor, - col_offsets_ih.c_tensor, - col_offsets_hh.c_tensor, - scale_ih.into().c_scalar, - scale_hh.into().c_scalar, - zero_point_ih.into().c_scalar, - zero_point_hh.into().c_scalar - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_quantized_max_pool1d( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantized_max_pool1d( - c_tensors.as_mut_ptr(), - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_quantized_max_pool1d_out( - &self, - out: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantized_max_pool1d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_quantized_max_pool2d( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantized_max_pool2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_quantized_max_pool2d_out( - &self, - out: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantized_max_pool2d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_quantized_max_pool3d( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantized_max_pool3d( - c_tensors.as_mut_ptr(), - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_quantized_max_pool3d_out( - &self, - out: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantized_max_pool3d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32(), - if ceil_mode { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_quantized_rnn_relu_cell>( - &self, - hx: &Tensor, - w_ih: &Tensor, - w_hh: &Tensor, - b_ih: &Tensor, - b_hh: &Tensor, - packed_ih: &Tensor, - packed_hh: &Tensor, - col_offsets_ih: &Tensor, - col_offsets_hh: &Tensor, - scale_ih: S, - scale_hh: S, - zero_point_ih: S, - zero_point_hh: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantized_rnn_relu_cell( - c_tensors.as_mut_ptr(), - self.c_tensor, - hx.c_tensor, - w_ih.c_tensor, - w_hh.c_tensor, - b_ih.c_tensor, - b_hh.c_tensor, - packed_ih.c_tensor, - packed_hh.c_tensor, - col_offsets_ih.c_tensor, - col_offsets_hh.c_tensor, - scale_ih.into().c_scalar, - scale_hh.into().c_scalar, - zero_point_ih.into().c_scalar, - zero_point_hh.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_quantized_rnn_tanh_cell>( - &self, - hx: &Tensor, - w_ih: &Tensor, - w_hh: &Tensor, - b_ih: &Tensor, - b_hh: &Tensor, - packed_ih: &Tensor, - packed_hh: &Tensor, - col_offsets_ih: &Tensor, - col_offsets_hh: &Tensor, - scale_ih: S, - scale_hh: S, - zero_point_ih: S, - zero_point_hh: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_quantized_rnn_tanh_cell( - c_tensors.as_mut_ptr(), - self.c_tensor, - hx.c_tensor, - w_ih.c_tensor, - w_hh.c_tensor, - b_ih.c_tensor, - b_hh.c_tensor, - packed_ih.c_tensor, - packed_hh.c_tensor, - col_offsets_ih.c_tensor, - col_offsets_hh.c_tensor, - scale_ih.into().c_scalar, - scale_hh.into().c_scalar, - zero_point_ih.into().c_scalar, - zero_point_hh.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rad2deg(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rad2deg(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rad2deg_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rad2deg_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rad2deg_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rad2deg_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rand(size: impl IntList, options: (Kind, Device)) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rand( - c_tensors.as_mut_ptr(), - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rand_like(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rand_like(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rand_like_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rand_like_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rand_out(out: &Tensor, size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rand_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_randint( - high: i64, - size: impl IntList, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_randint( - c_tensors.as_mut_ptr(), - high, - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_randint_like(&self, high: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_randint_like(c_tensors.as_mut_ptr(), self.c_tensor, high)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_randint_like_low_dtype(&self, low: i64, high: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_randint_like_low_dtype( - c_tensors.as_mut_ptr(), - self.c_tensor, - low, - high - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_randint_like_low_dtype_out( - &self, - out: &Tensor, - low: i64, - high: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_randint_like_low_dtype_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - low, - high - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_randint_like_out(&self, out: &Tensor, high: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_randint_like_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - high - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_randint_like_tensor(&self, high: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_randint_like_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - high.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_randint_like_tensor_out( - &self, - out: &Tensor, - high: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_randint_like_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - high.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_randint_low( - low: i64, - high: i64, - size: impl IntList, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_randint_low( - c_tensors.as_mut_ptr(), - low, - high, - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_randint_low_out( - out: &Tensor, - low: i64, - high: i64, - size: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_randint_low_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - low, - high, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_randint_out(out: &Tensor, high: i64, size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_randint_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - high, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_randn(size: impl IntList, options: (Kind, Device)) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_randn( - c_tensors.as_mut_ptr(), - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_randn_like(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_randn_like(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_randn_like_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_randn_like_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_randn_out(out: &Tensor, size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_randn_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_random(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_random(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_random_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_random_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_random_from(&self, from: i64, to: impl Into>) -> Result { - let to = to.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_random_from( - c_tensors.as_mut_ptr(), - self.c_tensor, - from, - to.unwrap_or(0i64), - to.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_random_from_( - &mut self, - from: i64, - to: impl Into>, - ) -> Result { - let to = to.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_random_from_( - c_tensors.as_mut_ptr(), - self.c_tensor, - from, - to.unwrap_or(0i64), - to.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_random_from_out( - &self, - out: &Tensor, - from: i64, - to: impl Into>, - ) -> Result { - let to = to.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_random_from_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - from, - to.unwrap_or(0i64), - to.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_random_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_random_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_random_to(&self, to: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_random_to(c_tensors.as_mut_ptr(), self.c_tensor, to)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_random_to_(&mut self, to: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_random_to_(c_tensors.as_mut_ptr(), self.c_tensor, to)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_random_to_out(&self, out: &Tensor, to: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_random_to_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - to - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_randperm(n: i64, options: (Kind, Device)) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_randperm( - c_tensors.as_mut_ptr(), - n, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_randperm_out(out: &Tensor, n: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_randperm_out(c_tensors.as_mut_ptr(), out.c_tensor, n)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_range>( - start: S, - end: S, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_range( - c_tensors.as_mut_ptr(), - start.into().c_scalar, - end.into().c_scalar, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_range_out>( - out: &Tensor, - start: S, - end: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_range_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - start.into().c_scalar, - end.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_range_out_>( - out: &Tensor, - start: S, - end: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_range_out_( - c_tensors.as_mut_ptr(), - out.c_tensor, - start.into().c_scalar, - end.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_range_step>( - start: S, - end: S, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_range_step( - c_tensors.as_mut_ptr(), - start.into().c_scalar, - end.into().c_scalar, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_ravel(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_ravel(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_real(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_real(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_reciprocal(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_reciprocal(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_reciprocal_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_reciprocal_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_reciprocal_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_reciprocal_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_reflection_pad1d(&self, padding: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_reflection_pad1d( - c_tensors.as_mut_ptr(), - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_reflection_pad1d_backward( - &self, - grad_output: &Tensor, - padding: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_reflection_pad1d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_reflection_pad1d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - padding: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_reflection_pad1d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_reflection_pad1d_out( - &self, - out: &Tensor, - padding: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_reflection_pad1d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_reflection_pad2d(&self, padding: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_reflection_pad2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_reflection_pad2d_backward( - &self, - grad_output: &Tensor, - padding: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_reflection_pad2d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_reflection_pad2d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - padding: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_reflection_pad2d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_reflection_pad2d_out( - &self, - out: &Tensor, - padding: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_reflection_pad2d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_reflection_pad3d(&self, padding: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_reflection_pad3d( - c_tensors.as_mut_ptr(), - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_reflection_pad3d_backward( - &self, - grad_output: &Tensor, - padding: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_reflection_pad3d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_reflection_pad3d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - padding: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_reflection_pad3d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_reflection_pad3d_out( - &self, - out: &Tensor, - padding: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_reflection_pad3d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_relu(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_relu(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_relu6(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_relu6(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_relu6_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_relu6_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_relu_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_relu_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_relu_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_relu_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_remainder>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_remainder( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_remainder_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_remainder_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_remainder_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_remainder_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_remainder_scalar_tensor>( - self_scalar: S, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_remainder_scalar_tensor( - c_tensors.as_mut_ptr(), - self_scalar.into().c_scalar, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_remainder_scalar_tensor_out>( - out: &Tensor, - self_scalar: S, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_remainder_scalar_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self_scalar.into().c_scalar, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_remainder_tensor(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_remainder_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_remainder_tensor_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_remainder_tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_remainder_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_remainder_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_renorm>( - &self, - p: S, - dim: i64, - maxnorm: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_renorm( - c_tensors.as_mut_ptr(), - self.c_tensor, - p.into().c_scalar, - dim, - maxnorm.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_renorm_>( - &mut self, - p: S, - dim: i64, - maxnorm: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_renorm_( - c_tensors.as_mut_ptr(), - self.c_tensor, - p.into().c_scalar, - dim, - maxnorm.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_renorm_out>( - &self, - out: &Tensor, - p: S, - dim: i64, - maxnorm: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_renorm_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - p.into().c_scalar, - dim, - maxnorm.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_repeat(&self, repeats: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_repeat( - c_tensors.as_mut_ptr(), - self.c_tensor, - repeats.as_ptr(), - repeats.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_repeat_interleave( - repeats: &Tensor, - output_size: impl Into>, - ) -> Result { - let output_size = output_size.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_repeat_interleave( - c_tensors.as_mut_ptr(), - repeats.c_tensor, - output_size.unwrap_or(0i64), - output_size.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_repeat_interleave_self_int( - &self, - repeats: i64, - dim: impl Into>, - output_size: impl Into>, - ) -> Result { - let dim = dim.into(); - let output_size = output_size.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_repeat_interleave_self_int( - c_tensors.as_mut_ptr(), - self.c_tensor, - repeats, - dim.unwrap_or(0i64), - dim.is_none() as i8, - output_size.unwrap_or(0i64), - output_size.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_repeat_interleave_self_tensor( - &self, - repeats: &Tensor, - dim: impl Into>, - output_size: impl Into>, - ) -> Result { - let dim = dim.into(); - let output_size = output_size.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_repeat_interleave_self_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - repeats.c_tensor, - dim.unwrap_or(0i64), - dim.is_none() as i8, - output_size.unwrap_or(0i64), - output_size.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_repeat_interleave_tensor_out( - out: &Tensor, - repeats: &Tensor, - output_size: impl Into>, - ) -> Result { - let output_size = output_size.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_repeat_interleave_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - repeats.c_tensor, - output_size.unwrap_or(0i64), - output_size.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_repeat_out(&self, out: &Tensor, repeats: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_repeat_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - repeats.as_ptr(), - repeats.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_replication_pad1d(&self, padding: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_replication_pad1d( - c_tensors.as_mut_ptr(), - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_replication_pad1d_backward( - &self, - grad_output: &Tensor, - padding: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_replication_pad1d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_replication_pad1d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - padding: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_replication_pad1d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_replication_pad1d_out( - &self, - out: &Tensor, - padding: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_replication_pad1d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_replication_pad2d(&self, padding: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_replication_pad2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_replication_pad2d_backward( - &self, - grad_output: &Tensor, - padding: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_replication_pad2d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_replication_pad2d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - padding: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_replication_pad2d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_replication_pad2d_out( - &self, - out: &Tensor, - padding: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_replication_pad2d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_replication_pad3d(&self, padding: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_replication_pad3d( - c_tensors.as_mut_ptr(), - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_replication_pad3d_backward( - &self, - grad_output: &Tensor, - padding: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_replication_pad3d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_replication_pad3d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - padding: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_replication_pad3d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_replication_pad3d_out( - &self, - out: &Tensor, - padding: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_replication_pad3d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_requires_grad_(&mut self, requires_grad: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_requires_grad_( - c_tensors.as_mut_ptr(), - self.c_tensor, - if requires_grad { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_reshape(&self, shape: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_reshape( - c_tensors.as_mut_ptr(), - self.c_tensor, - shape.as_ptr(), - shape.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_reshape_as(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_reshape_as(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_resize(&self, size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_resize( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_resize_(&mut self, size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_resize_( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_resize_as(&self, the_template: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_resize_as( - c_tensors.as_mut_ptr(), - self.c_tensor, - the_template.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_resize_as_(&mut self, the_template: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_resize_as_( - c_tensors.as_mut_ptr(), - self.c_tensor, - the_template.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_resize_as_out(&self, out: &Tensor, the_template: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_resize_as_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - the_template.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_resize_as_sparse(&self, the_template: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_resize_as_sparse( - c_tensors.as_mut_ptr(), - self.c_tensor, - the_template.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_resize_as_sparse_(&mut self, the_template: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_resize_as_sparse_( - c_tensors.as_mut_ptr(), - self.c_tensor, - the_template.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_resize_as_sparse_out( - &self, - out: &Tensor, - the_template: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_resize_as_sparse_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - the_template.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_resize_out(&self, out: &Tensor, size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_resize_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_resolve_conj(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_resolve_conj(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_resolve_neg(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_resolve_neg(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_retains_grad(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_retains_grad(self.c_tensor)); - Ok(return_ != 0) - } - - pub fn f_rms_norm>( - &self, - normalized_shape: impl IntList, - weight: Option, - eps: impl Into>, - ) -> Result { - let eps = eps.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rms_norm( - c_tensors.as_mut_ptr(), - self.c_tensor, - normalized_shape.as_ptr(), - normalized_shape.len_i32(), - weight.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - eps.unwrap_or(std::f64::NAN), - eps.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rnn_relu>( - &self, - hx: &Tensor, - params: &[T], - has_biases: bool, - num_layers: i64, - dropout: f64, - train: bool, - bidirectional: bool, - batch_first: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_rnn_relu( - c_tensors.as_mut_ptr(), - self.c_tensor, - hx.c_tensor, - ptr_list(params).as_ptr(), - params.len() as i32, - if has_biases { 1 } else { 0 }, - num_layers, - dropout, - if train { 1 } else { 0 }, - if bidirectional { 1 } else { 0 }, - if batch_first { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_rnn_relu_cell>( - &self, - hx: &Tensor, - w_ih: &Tensor, - w_hh: &Tensor, - b_ih: Option, - b_hh: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rnn_relu_cell( - c_tensors.as_mut_ptr(), - self.c_tensor, - hx.c_tensor, - w_ih.c_tensor, - w_hh.c_tensor, - b_ih.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - b_hh.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rnn_relu_data>( - data: &Tensor, - batch_sizes: &Tensor, - hx: &Tensor, - params: &[T], - has_biases: bool, - num_layers: i64, - dropout: f64, - train: bool, - bidirectional: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_rnn_relu_data( - c_tensors.as_mut_ptr(), - data.c_tensor, - batch_sizes.c_tensor, - hx.c_tensor, - ptr_list(params).as_ptr(), - params.len() as i32, - if has_biases { 1 } else { 0 }, - num_layers, - dropout, - if train { 1 } else { 0 }, - if bidirectional { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_rnn_tanh>( - &self, - hx: &Tensor, - params: &[T], - has_biases: bool, - num_layers: i64, - dropout: f64, - train: bool, - bidirectional: bool, - batch_first: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_rnn_tanh( - c_tensors.as_mut_ptr(), - self.c_tensor, - hx.c_tensor, - ptr_list(params).as_ptr(), - params.len() as i32, - if has_biases { 1 } else { 0 }, - num_layers, - dropout, - if train { 1 } else { 0 }, - if bidirectional { 1 } else { 0 }, - if batch_first { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_rnn_tanh_cell>( - &self, - hx: &Tensor, - w_ih: &Tensor, - w_hh: &Tensor, - b_ih: Option, - b_hh: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rnn_tanh_cell( - c_tensors.as_mut_ptr(), - self.c_tensor, - hx.c_tensor, - w_ih.c_tensor, - w_hh.c_tensor, - b_ih.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - b_hh.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rnn_tanh_data>( - data: &Tensor, - batch_sizes: &Tensor, - hx: &Tensor, - params: &[T], - has_biases: bool, - num_layers: i64, - dropout: f64, - train: bool, - bidirectional: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_rnn_tanh_data( - c_tensors.as_mut_ptr(), - data.c_tensor, - batch_sizes.c_tensor, - hx.c_tensor, - ptr_list(params).as_ptr(), - params.len() as i32, - if has_biases { 1 } else { 0 }, - num_layers, - dropout, - if train { 1 } else { 0 }, - if bidirectional { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_roll(&self, shifts: impl IntList, dims: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_roll( - c_tensors.as_mut_ptr(), - self.c_tensor, - shifts.as_ptr(), - shifts.len_i32(), - dims.as_ptr(), - dims.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_roll_out( - &self, - out: &Tensor, - shifts: impl IntList, - dims: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_roll_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - shifts.as_ptr(), - shifts.len_i32(), - dims.as_ptr(), - dims.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rot90(&self, k: i64, dims: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rot90( - c_tensors.as_mut_ptr(), - self.c_tensor, - k, - dims.as_ptr(), - dims.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rot90_out( - &self, - out: &Tensor, - k: i64, - dims: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rot90_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - k, - dims.as_ptr(), - dims.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_round(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_round(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_round_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_round_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_round_decimals(&self, decimals: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_round_decimals(c_tensors.as_mut_ptr(), self.c_tensor, decimals)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_round_decimals_(&mut self, decimals: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_round_decimals_(c_tensors.as_mut_ptr(), self.c_tensor, decimals)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_round_decimals_out(&self, out: &Tensor, decimals: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_round_decimals_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - decimals - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_round_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_round_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_row_indices(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_row_indices(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_row_indices_copy(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_row_indices_copy(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_row_indices_copy_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_row_indices_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_row_stack>(tensors: &[T]) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_row_stack( - c_tensors.as_mut_ptr(), - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_row_stack_out>( - out: &Tensor, - tensors: &[T], - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_row_stack_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rrelu(&self, training: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rrelu( - c_tensors.as_mut_ptr(), - self.c_tensor, - if training { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rrelu_(&mut self, training: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rrelu_( - c_tensors.as_mut_ptr(), - self.c_tensor, - if training { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rrelu_with_noise(&self, noise: &Tensor, training: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rrelu_with_noise( - c_tensors.as_mut_ptr(), - self.c_tensor, - noise.c_tensor, - if training { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rrelu_with_noise_( - &mut self, - noise: &Tensor, - training: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rrelu_with_noise_( - c_tensors.as_mut_ptr(), - self.c_tensor, - noise.c_tensor, - if training { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rrelu_with_noise_backward>( - &self, - grad_output: &Tensor, - noise: &Tensor, - lower: S, - upper: S, - training: bool, - self_is_result: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rrelu_with_noise_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - noise.c_tensor, - lower.into().c_scalar, - upper.into().c_scalar, - if training { 1 } else { 0 }, - if self_is_result { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rrelu_with_noise_backward_out>( - &self, - out: &Tensor, - grad_output: &Tensor, - noise: &Tensor, - lower: S, - upper: S, - training: bool, - self_is_result: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rrelu_with_noise_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad_output.c_tensor, - self.c_tensor, - noise.c_tensor, - lower.into().c_scalar, - upper.into().c_scalar, - if training { 1 } else { 0 }, - if self_is_result { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rrelu_with_noise_functional( - &self, - noise: &Tensor, - training: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_rrelu_with_noise_functional( - c_tensors.as_mut_ptr(), - self.c_tensor, - noise.c_tensor, - if training { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_rrelu_with_noise_out( - &self, - out: &Tensor, - noise: &Tensor, - training: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rrelu_with_noise_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - noise.c_tensor, - if training { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rsqrt(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rsqrt(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rsqrt_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rsqrt_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rsqrt_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rsqrt_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rsub(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rsub(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rsub_scalar>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rsub_scalar( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rsub_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rsub_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_rsub_tensor_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_rsub_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_scalar_tensor>( - s: S, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_scalar_tensor( - c_tensors.as_mut_ptr(), - s.into().c_scalar, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_scalar_tensor_out>(out: &Tensor, s: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_scalar_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - s.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_scaled_dot_product_attention>( - query: &Tensor, - key: &Tensor, - value: &Tensor, - attn_mask: Option, - dropout_p: f64, - is_causal: bool, - scale: impl Into>, - enable_gqa: bool, - ) -> Result { - let scale = scale.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_scaled_dot_product_attention( - c_tensors.as_mut_ptr(), - query.c_tensor, - key.c_tensor, - value.c_tensor, - attn_mask.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - dropout_p, - if is_causal { 1 } else { 0 }, - scale.unwrap_or(std::f64::NAN), - scale.is_none() as i8, - if enable_gqa { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_scatter(&self, dim: i64, index: &Tensor, src: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_scatter( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - src.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_scatter_( - &mut self, - dim: i64, - index: &Tensor, - src: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_scatter_( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - src.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_scatter_add( - &self, - dim: i64, - index: &Tensor, - src: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_scatter_add( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - src.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_scatter_add_( - &mut self, - dim: i64, - index: &Tensor, - src: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_scatter_add_( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - src.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_scatter_add_out( - &self, - out: &Tensor, - dim: i64, - index: &Tensor, - src: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_scatter_add_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - index.c_tensor, - src.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_scatter_reduce( - &self, - dim: i64, - index: &Tensor, - src: &Tensor, - reduce: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_scatter_reduce( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - src.c_tensor, - reduce.as_ptr(), - reduce.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_scatter_reduce_( - &mut self, - dim: i64, - index: &Tensor, - src: &Tensor, - reduce: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_scatter_reduce_( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - src.c_tensor, - reduce.as_ptr(), - reduce.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_scatter_reduce_out( - &self, - out: &Tensor, - dim: i64, - index: &Tensor, - src: &Tensor, - reduce: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_scatter_reduce_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - index.c_tensor, - src.c_tensor, - reduce.as_ptr(), - reduce.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_scatter_src_out( - &self, - out: &Tensor, - dim: i64, - index: &Tensor, - src: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_scatter_src_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - index.c_tensor, - src.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_scatter_value>( - &self, - dim: i64, - index: &Tensor, - value: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_scatter_value( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - value.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_scatter_value_>( - &mut self, - dim: i64, - index: &Tensor, - value: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_scatter_value_( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - value.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_scatter_value_out>( - &self, - out: &Tensor, - dim: i64, - index: &Tensor, - value: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_scatter_value_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - index.c_tensor, - value.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_scatter_value_reduce>( - &self, - dim: i64, - index: &Tensor, - value: S, - reduce: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_scatter_value_reduce( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - value.into().c_scalar, - reduce.as_ptr(), - reduce.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_scatter_value_reduce_>( - &mut self, - dim: i64, - index: &Tensor, - value: S, - reduce: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_scatter_value_reduce_( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - index.c_tensor, - value.into().c_scalar, - reduce.as_ptr(), - reduce.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_scatter_value_reduce_out>( - &self, - out: &Tensor, - dim: i64, - index: &Tensor, - value: S, - reduce: &str, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_scatter_value_reduce_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - index.c_tensor, - value.into().c_scalar, - reduce.as_ptr(), - reduce.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_searchsorted>( - &self, - sorted_sequence: &Tensor, - out_int32: bool, - right: bool, - side: &str, - sorter: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_searchsorted( - c_tensors.as_mut_ptr(), - sorted_sequence.c_tensor, - self.c_tensor, - if out_int32 { 1 } else { 0 }, - if right { 1 } else { 0 }, - side.as_ptr(), - side.len() as i32, - sorter.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_searchsorted_scalar, S: Into>( - sorted_sequence: &Tensor, - self_scalar: S, - out_int32: bool, - right: bool, - side: &str, - sorter: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_searchsorted_scalar( - c_tensors.as_mut_ptr(), - sorted_sequence.c_tensor, - self_scalar.into().c_scalar, - if out_int32 { 1 } else { 0 }, - if right { 1 } else { 0 }, - side.as_ptr(), - side.len() as i32, - sorter.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_searchsorted_scalar_out, S: Into>( - out: &Tensor, - sorted_sequence: &Tensor, - self_scalar: S, - out_int32: bool, - right: bool, - side: &str, - sorter: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_searchsorted_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - sorted_sequence.c_tensor, - self_scalar.into().c_scalar, - if out_int32 { 1 } else { 0 }, - if right { 1 } else { 0 }, - side.as_ptr(), - side.len() as i32, - sorter.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_searchsorted_tensor_out>( - &self, - out: &Tensor, - sorted_sequence: &Tensor, - out_int32: bool, - right: bool, - side: &str, - sorter: Option, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_searchsorted_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - sorted_sequence.c_tensor, - self.c_tensor, - if out_int32 { 1 } else { 0 }, - if right { 1 } else { 0 }, - side.as_ptr(), - side.len() as i32, - sorter.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_segment_reduce, S: Into>( - data: &Tensor, - reduce: &str, - lengths: Option, - indices: Option, - offsets: Option, - axis: i64, - unsafe_: bool, - initial: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_segment_reduce( - c_tensors.as_mut_ptr(), - data.c_tensor, - reduce.as_ptr(), - reduce.len() as i32, - lengths.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - indices.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - offsets.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - axis, - if unsafe_ { 1 } else { 0 }, - initial.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_segment_reduce_out, S: Into>( - out: &Tensor, - data: &Tensor, - reduce: &str, - lengths: Option, - indices: Option, - offsets: Option, - axis: i64, - unsafe_: bool, - initial: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_segment_reduce_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - data.c_tensor, - reduce.as_ptr(), - reduce.len() as i32, - lengths.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - indices.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - offsets.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - axis, - if unsafe_ { 1 } else { 0 }, - initial.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_select(&self, dim: i64, index: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_select(c_tensors.as_mut_ptr(), self.c_tensor, dim, index)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_select_backward( - grad_output: &Tensor, - input_sizes: impl IntList, - dim: i64, - index: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_select_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - input_sizes.as_ptr(), - input_sizes.len_i32(), - dim, - index - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_select_backward_out( - out: &Tensor, - grad_output: &Tensor, - input_sizes: impl IntList, - dim: i64, - index: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_select_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad_output.c_tensor, - input_sizes.as_ptr(), - input_sizes.len_i32(), - dim, - index - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_select_copy(&self, dim: i64, index: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_select_copy(c_tensors.as_mut_ptr(), self.c_tensor, dim, index)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_select_copy_int_out( - &self, - out: &Tensor, - dim: i64, - index: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_select_copy_int_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - index - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_select_scatter(&self, src: &Tensor, dim: i64, index: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_select_scatter( - c_tensors.as_mut_ptr(), - self.c_tensor, - src.c_tensor, - dim, - index - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_select_scatter_out( - &self, - out: &Tensor, - src: &Tensor, - dim: i64, - index: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_select_scatter_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - src.c_tensor, - dim, - index - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_selu(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_selu(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_selu_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_selu_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_set(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_set(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_set_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_set_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_set_data(&mut self, new_data: &Tensor) -> Result<(), TchError> { - unsafe_torch_err!(atg_set_data(self.c_tensor, new_data.c_tensor)); - Ok(()) - } - - pub fn f_set_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_set_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_set_requires_grad(&self, r: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_set_requires_grad( - c_tensors.as_mut_ptr(), - self.c_tensor, - if r { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_set_source_tensor(&self, source: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_set_source_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - source.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_set_source_tensor_(&mut self, source: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_set_source_tensor_( - c_tensors.as_mut_ptr(), - self.c_tensor, - source.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_set_source_tensor_out( - &self, - out: &Tensor, - source: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_set_source_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - source.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_set_source_tensor_storage_offset_( - &mut self, - source: &Tensor, - storage_offset: i64, - size: impl IntList, - stride: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_set_source_tensor_storage_offset_( - c_tensors.as_mut_ptr(), - self.c_tensor, - source.c_tensor, - storage_offset, - size.as_ptr(), - size.len_i32(), - stride.as_ptr(), - stride.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sgn(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sgn(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sgn_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sgn_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sgn_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sgn_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sigmoid(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sigmoid(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sigmoid_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sigmoid_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sigmoid_backward(grad_output: &Tensor, output: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sigmoid_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - output.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sigmoid_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sigmoid_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - output.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sigmoid_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sigmoid_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sign(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sign(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sign_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sign_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sign_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sign_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_signbit(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_signbit(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_signbit_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_signbit_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_silu(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_silu(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_silu_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_silu_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_silu_backward(&self, grad_output: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_silu_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_silu_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_silu_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_silu_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_silu_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sin(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sin(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sin_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sin_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sin_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sin_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sinc(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sinc(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sinc_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sinc_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sinc_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sinc_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sinh(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sinh(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sinh_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sinh_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sinh_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sinh_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_slice( - &self, - dim: i64, - start: impl Into>, - end: impl Into>, - step: i64, - ) -> Result { - let start = start.into(); - let end = end.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_slice( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - start.unwrap_or(0i64), - start.is_none() as i8, - end.unwrap_or(0i64), - end.is_none() as i8, - step - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_slice_backward( - grad_output: &Tensor, - input_sizes: impl IntList, - dim: i64, - start: i64, - end: i64, - step: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_slice_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - input_sizes.as_ptr(), - input_sizes.len_i32(), - dim, - start, - end, - step - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_slice_backward_out( - out: &Tensor, - grad_output: &Tensor, - input_sizes: impl IntList, - dim: i64, - start: i64, - end: i64, - step: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_slice_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad_output.c_tensor, - input_sizes.as_ptr(), - input_sizes.len_i32(), - dim, - start, - end, - step - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_slice_copy( - &self, - dim: i64, - start: impl Into>, - end: impl Into>, - step: i64, - ) -> Result { - let start = start.into(); - let end = end.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_slice_copy( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - start.unwrap_or(0i64), - start.is_none() as i8, - end.unwrap_or(0i64), - end.is_none() as i8, - step - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_slice_copy_tensor_out( - &self, - out: &Tensor, - dim: i64, - start: impl Into>, - end: impl Into>, - step: i64, - ) -> Result { - let start = start.into(); - let end = end.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_slice_copy_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - start.unwrap_or(0i64), - start.is_none() as i8, - end.unwrap_or(0i64), - end.is_none() as i8, - step - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_slice_inverse( - &self, - src: &Tensor, - dim: i64, - start: impl Into>, - end: impl Into>, - step: i64, - ) -> Result { - let start = start.into(); - let end = end.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_slice_inverse( - c_tensors.as_mut_ptr(), - self.c_tensor, - src.c_tensor, - dim, - start.unwrap_or(0i64), - start.is_none() as i8, - end.unwrap_or(0i64), - end.is_none() as i8, - step - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_slice_scatter( - &self, - src: &Tensor, - dim: i64, - start: impl Into>, - end: impl Into>, - step: i64, - ) -> Result { - let start = start.into(); - let end = end.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_slice_scatter( - c_tensors.as_mut_ptr(), - self.c_tensor, - src.c_tensor, - dim, - start.unwrap_or(0i64), - start.is_none() as i8, - end.unwrap_or(0i64), - end.is_none() as i8, - step - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_slice_scatter_out( - &self, - out: &Tensor, - src: &Tensor, - dim: i64, - start: impl Into>, - end: impl Into>, - step: i64, - ) -> Result { - let start = start.into(); - let end = end.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_slice_scatter_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - src.c_tensor, - dim, - start.unwrap_or(0i64), - start.is_none() as i8, - end.unwrap_or(0i64), - end.is_none() as i8, - step - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_slogdet(&self) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_slogdet(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_slogdet_out( - &self, - sign: &Tensor, - logabsdet: &Tensor, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_slogdet_out( - c_tensors.as_mut_ptr(), - sign.c_tensor, - logabsdet.c_tensor, - self.c_tensor - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_slow_conv3d>( - &self, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_slow_conv3d( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_slow_conv3d_out>( - &self, - out: &Tensor, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_slow_conv3d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_slow_conv_dilated2d>( - &self, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_slow_conv_dilated2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_slow_conv_dilated2d_out>( - &self, - out: &Tensor, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_slow_conv_dilated2d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_slow_conv_dilated3d>( - &self, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_slow_conv_dilated3d( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_slow_conv_dilated3d_out>( - &self, - out: &Tensor, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_slow_conv_dilated3d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_slow_conv_transpose2d>( - &self, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - output_padding: impl IntList, - dilation: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_slow_conv_transpose2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - output_padding.as_ptr(), - output_padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_slow_conv_transpose2d_out>( - &self, - out: &Tensor, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - output_padding: impl IntList, - dilation: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_slow_conv_transpose2d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - output_padding.as_ptr(), - output_padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_slow_conv_transpose3d>( - &self, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - output_padding: impl IntList, - dilation: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_slow_conv_transpose3d( - c_tensors.as_mut_ptr(), - self.c_tensor, - weight.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - output_padding.as_ptr(), - output_padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_slow_conv_transpose3d_out>( - &self, - out: &Tensor, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - output_padding: impl IntList, - dilation: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_slow_conv_transpose3d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - weight.c_tensor, - kernel_size.as_ptr(), - kernel_size.len_i32(), - bias.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - stride.as_ptr(), - stride.len_i32(), - padding.as_ptr(), - padding.len_i32(), - output_padding.as_ptr(), - output_padding.len_i32(), - dilation.as_ptr(), - dilation.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_smm(&self, mat2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_smm(c_tensors.as_mut_ptr(), self.c_tensor, mat2.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_smooth_l1_loss( - &self, - target: &Tensor, - reduction: crate::Reduction, - beta: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_smooth_l1_loss( - c_tensors.as_mut_ptr(), - self.c_tensor, - target.c_tensor, - reduction.to_int(), - beta - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_smooth_l1_loss_backward( - &self, - grad_output: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - beta: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_smooth_l1_loss_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - target.c_tensor, - reduction.to_int(), - beta - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_smooth_l1_loss_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - beta: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_smooth_l1_loss_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - target.c_tensor, - reduction.to_int(), - beta - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_smooth_l1_loss_out( - &self, - out: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - beta: f64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_smooth_l1_loss_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - target.c_tensor, - reduction.to_int(), - beta - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_soft_margin_loss( - &self, - target: &Tensor, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_soft_margin_loss( - c_tensors.as_mut_ptr(), - self.c_tensor, - target.c_tensor, - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_soft_margin_loss_backward( - &self, - grad_output: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_soft_margin_loss_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - target.c_tensor, - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_soft_margin_loss_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_soft_margin_loss_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - target.c_tensor, - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_soft_margin_loss_out( - &self, - out: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_soft_margin_loss_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - target.c_tensor, - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_softmax(&self, dim: i64, dtype: impl Into>) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_softmax( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_softmax_int_out( - &self, - out: &Tensor, - dim: i64, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_softmax_int_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_softplus(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_softplus(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_softplus_backward>( - &self, - grad_output: &Tensor, - beta: S, - threshold: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_softplus_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - beta.into().c_scalar, - threshold.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_softplus_backward_grad_input>( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - beta: S, - threshold: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_softplus_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - beta.into().c_scalar, - threshold.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_softplus_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_softplus_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_softshrink(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_softshrink(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_softshrink_backward>( - &self, - grad_output: &Tensor, - lambd: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_softshrink_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - lambd.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_softshrink_backward_grad_input>( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - lambd: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_softshrink_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - lambd.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_softshrink_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_softshrink_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sort(&self, dim: i64, descending: bool) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_sort( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - if descending { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_sort_stable( - &self, - stable: bool, - dim: i64, - descending: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_sort_stable( - c_tensors.as_mut_ptr(), - self.c_tensor, - if stable { 1 } else { 0 }, - dim, - if descending { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_sort_values( - &self, - values: &Tensor, - indices: &Tensor, - dim: i64, - descending: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_sort_values( - c_tensors.as_mut_ptr(), - values.c_tensor, - indices.c_tensor, - self.c_tensor, - dim, - if descending { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_sort_values_stable( - &self, - values: &Tensor, - indices: &Tensor, - stable: bool, - dim: i64, - descending: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_sort_values_stable( - c_tensors.as_mut_ptr(), - values.c_tensor, - indices.c_tensor, - self.c_tensor, - if stable { 1 } else { 0 }, - dim, - if descending { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_sparse_bsc_tensor( - ccol_indices: &Tensor, - row_indices: &Tensor, - values: &Tensor, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_bsc_tensor( - c_tensors.as_mut_ptr(), - ccol_indices.c_tensor, - row_indices.c_tensor, - values.c_tensor, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_bsc_tensor_ccol_row_value_size( - ccol_indices: &Tensor, - row_indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_bsc_tensor_ccol_row_value_size( - c_tensors.as_mut_ptr(), - ccol_indices.c_tensor, - row_indices.c_tensor, - values.c_tensor, - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_bsr_tensor( - crow_indices: &Tensor, - col_indices: &Tensor, - values: &Tensor, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_bsr_tensor( - c_tensors.as_mut_ptr(), - crow_indices.c_tensor, - col_indices.c_tensor, - values.c_tensor, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_bsr_tensor_crow_col_value_size( - crow_indices: &Tensor, - col_indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_bsr_tensor_crow_col_value_size( - c_tensors.as_mut_ptr(), - crow_indices.c_tensor, - col_indices.c_tensor, - values.c_tensor, - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_compressed_tensor( - compressed_indices: &Tensor, - plain_indices: &Tensor, - values: &Tensor, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_compressed_tensor( - c_tensors.as_mut_ptr(), - compressed_indices.c_tensor, - plain_indices.c_tensor, - values.c_tensor, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_compressed_tensor_comp_plain_value_size( - compressed_indices: &Tensor, - plain_indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_compressed_tensor_comp_plain_value_size( - c_tensors.as_mut_ptr(), - compressed_indices.c_tensor, - plain_indices.c_tensor, - values.c_tensor, - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_coo_tensor( - size: impl IntList, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_coo_tensor( - c_tensors.as_mut_ptr(), - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_coo_tensor_indices( - indices: &Tensor, - values: &Tensor, - options: (Kind, Device), - is_coalesced: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_coo_tensor_indices( - c_tensors.as_mut_ptr(), - indices.c_tensor, - values.c_tensor, - options.0.c_int(), - options.1.c_int(), - if is_coalesced { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_coo_tensor_indices_size( - indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - is_coalesced: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_coo_tensor_indices_size( - c_tensors.as_mut_ptr(), - indices.c_tensor, - values.c_tensor, - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int(), - if is_coalesced { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_coo_tensor_size_out( - out: &Tensor, - size: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_coo_tensor_size_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_csc_tensor( - ccol_indices: &Tensor, - row_indices: &Tensor, - values: &Tensor, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_csc_tensor( - c_tensors.as_mut_ptr(), - ccol_indices.c_tensor, - row_indices.c_tensor, - values.c_tensor, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_csc_tensor_ccol_row_value_size( - ccol_indices: &Tensor, - row_indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_csc_tensor_ccol_row_value_size( - c_tensors.as_mut_ptr(), - ccol_indices.c_tensor, - row_indices.c_tensor, - values.c_tensor, - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_csr_tensor( - crow_indices: &Tensor, - col_indices: &Tensor, - values: &Tensor, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_csr_tensor( - c_tensors.as_mut_ptr(), - crow_indices.c_tensor, - col_indices.c_tensor, - values.c_tensor, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_csr_tensor_crow_col_value_size( - crow_indices: &Tensor, - col_indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_csr_tensor_crow_col_value_size( - c_tensors.as_mut_ptr(), - crow_indices.c_tensor, - col_indices.c_tensor, - values.c_tensor, - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_dim(&self) -> Result { - let return_; - unsafe_torch_err!(return_ = atg_sparse_dim(self.c_tensor)); - Ok(return_) - } - - pub fn f_sparse_mask(&self, mask: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_mask(c_tensors.as_mut_ptr(), self.c_tensor, mask.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_mask_out(&self, out: &Tensor, mask: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_mask_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mask.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_resize( - &self, - size: impl IntList, - sparse_dim: i64, - dense_dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_resize( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32(), - sparse_dim, - dense_dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_resize_( - &mut self, - size: impl IntList, - sparse_dim: i64, - dense_dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_resize_( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32(), - sparse_dim, - dense_dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_resize_and_clear( - &self, - size: impl IntList, - sparse_dim: i64, - dense_dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_resize_and_clear( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32(), - sparse_dim, - dense_dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_resize_and_clear_( - &mut self, - size: impl IntList, - sparse_dim: i64, - dense_dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_resize_and_clear_( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32(), - sparse_dim, - dense_dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_resize_and_clear_out( - &self, - out: &Tensor, - size: impl IntList, - sparse_dim: i64, - dense_dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_resize_and_clear_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - size.as_ptr(), - size.len_i32(), - sparse_dim, - dense_dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_resize_out( - &self, - out: &Tensor, - size: impl IntList, - sparse_dim: i64, - dense_dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_resize_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - size.as_ptr(), - size.len_i32(), - sparse_dim, - dense_dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_sampled_addmm(&self, mat1: &Tensor, mat2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_sampled_addmm( - c_tensors.as_mut_ptr(), - self.c_tensor, - mat1.c_tensor, - mat2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sparse_sampled_addmm_out( - &self, - out: &Tensor, - mat1: &Tensor, - mat2: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sparse_sampled_addmm_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mat1.c_tensor, - mat2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_airy_ai(x: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_airy_ai(c_tensors.as_mut_ptr(), x.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_airy_ai_out(out: &Tensor, x: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_airy_ai_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_bessel_j0(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_bessel_j0(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_bessel_j0_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_bessel_j0_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_bessel_j1(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_bessel_j1(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_bessel_j1_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_bessel_j1_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_bessel_y0(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_bessel_y0(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_bessel_y0_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_bessel_y0_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_bessel_y1(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_bessel_y1(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_bessel_y1_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_bessel_y1_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_t(x: &Tensor, n: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_t( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_t_n_scalar>( - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_t_n_scalar( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_t_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_t_n_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_t_out( - out: &Tensor, - x: &Tensor, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_t_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_t_x_scalar>( - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_t_x_scalar( - c_tensors.as_mut_ptr(), - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_t_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_t_x_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_u(x: &Tensor, n: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_u( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_u_n_scalar>( - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_u_n_scalar( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_u_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_u_n_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_u_out( - out: &Tensor, - x: &Tensor, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_u_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_u_x_scalar>( - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_u_x_scalar( - c_tensors.as_mut_ptr(), - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_u_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_u_x_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_v(x: &Tensor, n: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_v( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_v_n_scalar>( - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_v_n_scalar( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_v_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_v_n_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_v_out( - out: &Tensor, - x: &Tensor, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_v_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_v_x_scalar>( - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_v_x_scalar( - c_tensors.as_mut_ptr(), - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_v_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_v_x_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_w(x: &Tensor, n: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_w( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_w_n_scalar>( - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_w_n_scalar( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_w_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_w_n_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_w_out( - out: &Tensor, - x: &Tensor, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_w_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_w_x_scalar>( - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_w_x_scalar( - c_tensors.as_mut_ptr(), - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_chebyshev_polynomial_w_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_chebyshev_polynomial_w_x_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_digamma(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_digamma(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_digamma_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_digamma_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_entr(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_entr(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_entr_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_entr_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_erf(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_erf(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_erf_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_erf_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_erfc(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_erfc(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_erfc_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_erfc_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_erfcx(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_erfcx(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_erfcx_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_erfcx_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_erfinv(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_erfinv(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_erfinv_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_erfinv_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_exp2(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_exp2(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_exp2_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_exp2_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_expit(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_expit(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_expit_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_expit_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_expm1(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_expm1(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_expm1_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_expm1_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_gammainc(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_gammainc( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_gammainc_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_gammainc_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_gammaincc(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_gammaincc( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_gammaincc_out( - &self, - out: &Tensor, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_gammaincc_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_gammaln(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_gammaln(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_gammaln_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_gammaln_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_hermite_polynomial_h(x: &Tensor, n: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_hermite_polynomial_h( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_hermite_polynomial_h_n_scalar>( - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_hermite_polynomial_h_n_scalar( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_hermite_polynomial_h_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_hermite_polynomial_h_n_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_hermite_polynomial_h_out( - out: &Tensor, - x: &Tensor, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_hermite_polynomial_h_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_hermite_polynomial_h_x_scalar>( - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_hermite_polynomial_h_x_scalar( - c_tensors.as_mut_ptr(), - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_hermite_polynomial_h_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_hermite_polynomial_h_x_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_hermite_polynomial_he(x: &Tensor, n: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_hermite_polynomial_he( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_hermite_polynomial_he_n_scalar>( - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_hermite_polynomial_he_n_scalar( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_hermite_polynomial_he_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_hermite_polynomial_he_n_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_hermite_polynomial_he_out( - out: &Tensor, - x: &Tensor, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_hermite_polynomial_he_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_hermite_polynomial_he_x_scalar>( - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_hermite_polynomial_he_x_scalar( - c_tensors.as_mut_ptr(), - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_hermite_polynomial_he_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_hermite_polynomial_he_x_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_i0(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_i0(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_i0_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_i0_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_i0e(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_i0e(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_i0e_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_i0e_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_i1(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_i1(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_i1_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_i1_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_i1e(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_i1e(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_i1e_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_i1e_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_laguerre_polynomial_l(x: &Tensor, n: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_laguerre_polynomial_l( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_laguerre_polynomial_l_n_scalar>( - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_laguerre_polynomial_l_n_scalar( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_laguerre_polynomial_l_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_laguerre_polynomial_l_n_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_laguerre_polynomial_l_out( - out: &Tensor, - x: &Tensor, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_laguerre_polynomial_l_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_laguerre_polynomial_l_x_scalar>( - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_laguerre_polynomial_l_x_scalar( - c_tensors.as_mut_ptr(), - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_laguerre_polynomial_l_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_laguerre_polynomial_l_x_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_legendre_polynomial_p(x: &Tensor, n: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_legendre_polynomial_p( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_legendre_polynomial_p_n_scalar>( - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_legendre_polynomial_p_n_scalar( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_legendre_polynomial_p_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_legendre_polynomial_p_n_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_legendre_polynomial_p_out( - out: &Tensor, - x: &Tensor, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_legendre_polynomial_p_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_legendre_polynomial_p_x_scalar>( - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_legendre_polynomial_p_x_scalar( - c_tensors.as_mut_ptr(), - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_legendre_polynomial_p_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_legendre_polynomial_p_x_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_log1p(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_log1p(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_log1p_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_log1p_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_log_ndtr(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_log_ndtr(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_log_ndtr_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_log_ndtr_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_log_softmax( - &self, - dim: i64, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_log_softmax( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_logit(&self, eps: impl Into>) -> Result { - let eps = eps.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_logit( - c_tensors.as_mut_ptr(), - self.c_tensor, - eps.unwrap_or(std::f64::NAN), - eps.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_logit_out( - &self, - out: &Tensor, - eps: impl Into>, - ) -> Result { - let eps = eps.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_logit_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - eps.unwrap_or(std::f64::NAN), - eps.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_logsumexp( - &self, - dim: impl IntList, - keepdim: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_logsumexp( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_logsumexp_out( - &self, - out: &Tensor, - dim: impl IntList, - keepdim: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_logsumexp_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_modified_bessel_i0(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_modified_bessel_i0(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_modified_bessel_i0_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_modified_bessel_i0_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_modified_bessel_i1(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_modified_bessel_i1(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_modified_bessel_i1_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_modified_bessel_i1_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_modified_bessel_k0(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_modified_bessel_k0(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_modified_bessel_k0_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_modified_bessel_k0_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_modified_bessel_k1(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_modified_bessel_k1(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_modified_bessel_k1_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_modified_bessel_k1_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_multigammaln(&self, p: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_multigammaln(c_tensors.as_mut_ptr(), self.c_tensor, p)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_multigammaln_out(&self, out: &Tensor, p: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_multigammaln_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - p - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_ndtr(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_ndtr(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_ndtr_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_ndtr_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_ndtri(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_ndtri(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_ndtri_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_ndtri_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_polygamma(&self, n: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_polygamma(c_tensors.as_mut_ptr(), n, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_polygamma_out(&self, out: &Tensor, n: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_polygamma_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - n, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_psi(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_psi(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_psi_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_psi_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_round(&self, decimals: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_round(c_tensors.as_mut_ptr(), self.c_tensor, decimals)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_round_out(&self, out: &Tensor, decimals: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_round_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - decimals - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_scaled_modified_bessel_k0(x: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_scaled_modified_bessel_k0( - c_tensors.as_mut_ptr(), - x.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_scaled_modified_bessel_k0_out( - out: &Tensor, - x: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_scaled_modified_bessel_k0_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_scaled_modified_bessel_k1(x: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_scaled_modified_bessel_k1( - c_tensors.as_mut_ptr(), - x.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_scaled_modified_bessel_k1_out( - out: &Tensor, - x: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_scaled_modified_bessel_k1_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_t( - x: &Tensor, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_t( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_t_n_scalar>( - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_t_n_scalar( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_t_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_t_n_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_t_out( - out: &Tensor, - x: &Tensor, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_t_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_t_x_scalar>( - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_t_x_scalar( - c_tensors.as_mut_ptr(), - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_t_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_t_x_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_u( - x: &Tensor, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_u( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_u_n_scalar>( - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_u_n_scalar( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_u_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_u_n_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_u_out( - out: &Tensor, - x: &Tensor, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_u_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_u_x_scalar>( - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_u_x_scalar( - c_tensors.as_mut_ptr(), - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_u_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_u_x_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_v( - x: &Tensor, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_v( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_v_n_scalar>( - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_v_n_scalar( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_v_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_v_n_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_v_out( - out: &Tensor, - x: &Tensor, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_v_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_v_x_scalar>( - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_v_x_scalar( - c_tensors.as_mut_ptr(), - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_v_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_v_x_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_w( - x: &Tensor, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_w( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_w_n_scalar>( - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_w_n_scalar( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_w_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_w_n_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_w_out( - out: &Tensor, - x: &Tensor, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_w_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_w_x_scalar>( - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_w_x_scalar( - c_tensors.as_mut_ptr(), - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_shifted_chebyshev_polynomial_w_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_shifted_chebyshev_polynomial_w_x_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.into().c_scalar, - n.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_sinc(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_sinc(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_sinc_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_sinc_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_softmax( - &self, - dim: i64, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_softmax( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_spherical_bessel_j0(x: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_spherical_bessel_j0(c_tensors.as_mut_ptr(), x.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_spherical_bessel_j0_out(out: &Tensor, x: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_spherical_bessel_j0_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - x.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_xlog1py(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_xlog1py( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_xlog1py_other_scalar>( - &self, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_xlog1py_other_scalar( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_xlog1py_other_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_xlog1py_other_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_xlog1py_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_xlog1py_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_xlog1py_self_scalar>( - self_scalar: S, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_xlog1py_self_scalar( - c_tensors.as_mut_ptr(), - self_scalar.into().c_scalar, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_xlog1py_self_scalar_out>( - out: &Tensor, - self_scalar: S, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_xlog1py_self_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self_scalar.into().c_scalar, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_xlogy(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_xlogy(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_xlogy_other_scalar>( - &self, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_xlogy_other_scalar( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_xlogy_other_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_xlogy_other_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_xlogy_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_xlogy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_xlogy_self_scalar>( - self_scalar: S, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_xlogy_self_scalar( - c_tensors.as_mut_ptr(), - self_scalar.into().c_scalar, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_xlogy_self_scalar_out>( - out: &Tensor, - self_scalar: S, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_xlogy_self_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self_scalar.into().c_scalar, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_zeta(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_zeta(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_zeta_other_scalar>( - &self, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_zeta_other_scalar( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_zeta_other_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_zeta_other_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_zeta_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_zeta_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_zeta_self_scalar>( - self_scalar: S, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_zeta_self_scalar( - c_tensors.as_mut_ptr(), - self_scalar.into().c_scalar, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_special_zeta_self_scalar_out>( - out: &Tensor, - self_scalar: S, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_special_zeta_self_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self_scalar.into().c_scalar, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_split(&self, split_size: i64, dim: i64) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_split(self.c_tensor, split_size, dim)); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_split_copy(&self, split_size: i64, dim: i64) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_split_copy(self.c_tensor, split_size, dim)); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_split_copy_tensor_out>( - &self, - out: &[T], - split_size: i64, - dim: i64, - ) -> Result<(), TchError> { - unsafe_torch_err!(atg_split_copy_tensor_out( - ptr_list(out).as_ptr(), - out.len() as i32, - self.c_tensor, - split_size, - dim - )); - Ok(()) - } - - pub fn f_split_sizes( - &self, - split_size: impl IntList, - dim: i64, - ) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_split_sizes( - self.c_tensor, - split_size.as_ptr(), - split_size.len_i32(), - dim - )); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_split_with_sizes( - &self, - split_sizes: impl IntList, - dim: i64, - ) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_split_with_sizes( - self.c_tensor, - split_sizes.as_ptr(), - split_sizes.len_i32(), - dim - )); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_split_with_sizes_copy( - &self, - split_sizes: impl IntList, - dim: i64, - ) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_split_with_sizes_copy( - self.c_tensor, - split_sizes.as_ptr(), - split_sizes.len_i32(), - dim - )); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_split_with_sizes_copy_out>( - &self, - out: &[T], - split_sizes: impl IntList, - dim: i64, - ) -> Result<(), TchError> { - unsafe_torch_err!(atg_split_with_sizes_copy_out( - ptr_list(out).as_ptr(), - out.len() as i32, - self.c_tensor, - split_sizes.as_ptr(), - split_sizes.len_i32(), - dim - )); - Ok(()) - } - - pub fn f_sqrt(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sqrt(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sqrt_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sqrt_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sqrt_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sqrt_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_square(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_square(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_square_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_square_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_square_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_square_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_squeeze(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_squeeze(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_squeeze_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_squeeze_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_squeeze_copy(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_squeeze_copy(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_squeeze_copy_dim(&self, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_squeeze_copy_dim(c_tensors.as_mut_ptr(), self.c_tensor, dim)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_squeeze_copy_dim_out(&self, out: &Tensor, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_squeeze_copy_dim_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_squeeze_copy_dims(&self, dim: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_squeeze_copy_dims( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_squeeze_copy_dims_out( - &self, - out: &Tensor, - dim: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_squeeze_copy_dims_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_squeeze_copy_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_squeeze_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_squeeze_dim(&self, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_squeeze_dim(c_tensors.as_mut_ptr(), self.c_tensor, dim)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_squeeze_dim_(&mut self, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_squeeze_dim_(c_tensors.as_mut_ptr(), self.c_tensor, dim)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_squeeze_dims(&self, dim: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_squeeze_dims( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_squeeze_dims_(&mut self, dim: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_squeeze_dims_( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sspaddmm(&self, mat1: &Tensor, mat2: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sspaddmm( - c_tensors.as_mut_ptr(), - self.c_tensor, - mat1.c_tensor, - mat2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sspaddmm_out( - &self, - out: &Tensor, - mat1: &Tensor, - mat2: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sspaddmm_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - mat1.c_tensor, - mat2.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_stack>(tensors: &[T], dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_stack( - c_tensors.as_mut_ptr(), - ptr_list(tensors).as_ptr(), - tensors.len() as i32, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_stack_out>( - out: &Tensor, - tensors: &[T], - dim: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_stack_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - ptr_list(tensors).as_ptr(), - tensors.len() as i32, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_std(&self, unbiased: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_std( - c_tensors.as_mut_ptr(), - self.c_tensor, - if unbiased { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_std_correction>( - &self, - dim: impl IntListOption, - correction: S, - keepdim: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_std_correction( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - correction.into().c_scalar, - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_std_correction_out>( - &self, - out: &Tensor, - dim: impl IntListOption, - correction: S, - keepdim: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_std_correction_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - correction.into().c_scalar, - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_std_dim( - &self, - dim: impl IntListOption, - unbiased: bool, - keepdim: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_std_dim( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if unbiased { 1 } else { 0 }, - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_std_mean(&self, unbiased: bool) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_std_mean( - c_tensors.as_mut_ptr(), - self.c_tensor, - if unbiased { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_std_mean_correction>( - &self, - dim: impl IntListOption, - correction: S, - keepdim: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_std_mean_correction( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - correction.into().c_scalar, - if keepdim { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_std_mean_correction_out>( - &self, - out0: &Tensor, - out1: &Tensor, - dim: impl IntListOption, - correction: S, - keepdim: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_std_mean_correction_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - correction.into().c_scalar, - if keepdim { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_std_mean_dim( - &self, - dim: impl IntListOption, - unbiased: bool, - keepdim: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_std_mean_dim( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if unbiased { 1 } else { 0 }, - if keepdim { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_std_out( - &self, - out: &Tensor, - dim: impl IntListOption, - unbiased: bool, - keepdim: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_std_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if unbiased { 1 } else { 0 }, - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_stft>( - &self, - n_fft: i64, - hop_length: impl Into>, - win_length: impl Into>, - window: Option, - normalized: bool, - onesided: bool, - return_complex: bool, - align_to_window: bool, - ) -> Result { - let hop_length = hop_length.into(); - let win_length = win_length.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_stft( - c_tensors.as_mut_ptr(), - self.c_tensor, - n_fft, - hop_length.unwrap_or(0i64), - hop_length.is_none() as i8, - win_length.unwrap_or(0i64), - win_length.is_none() as i8, - window.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if normalized { 1 } else { 0 }, - if onesided { 1 } else { 0 }, - if return_complex { 1 } else { 0 }, - if align_to_window { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_stft_center>( - &self, - n_fft: i64, - hop_length: impl Into>, - win_length: impl Into>, - window: Option, - center: bool, - pad_mode: &str, - normalized: bool, - onesided: bool, - return_complex: bool, - align_to_window: bool, - ) -> Result { - let hop_length = hop_length.into(); - let win_length = win_length.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_stft_center( - c_tensors.as_mut_ptr(), - self.c_tensor, - n_fft, - hop_length.unwrap_or(0i64), - hop_length.is_none() as i8, - win_length.unwrap_or(0i64), - win_length.is_none() as i8, - window.as_ref().map_or(std::ptr::null_mut(), |t| t.borrow().c_tensor), - if center { 1 } else { 0 }, - pad_mode.as_ptr(), - pad_mode.len() as i32, - if normalized { 1 } else { 0 }, - if onesided { 1 } else { 0 }, - if return_complex { 1 } else { 0 }, - if align_to_window { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sub(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sub(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sub_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sub_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sub_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sub_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sub_scalar>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sub_scalar( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sub_scalar_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sub_scalar_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sub_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sub_scalar_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_subtract(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_subtract(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_subtract_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_subtract_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_subtract_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_subtract_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_subtract_scalar>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_subtract_scalar( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_subtract_scalar_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_subtract_scalar_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sum(&self, dtype: impl Into>) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sum( - c_tensors.as_mut_ptr(), - self.c_tensor, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sum_dim_intlist( - &self, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sum_dim_intlist( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 }, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sum_intlist_out( - &self, - out: &Tensor, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sum_intlist_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if keepdim { 1 } else { 0 }, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sum_out( - &self, - out: &Tensor, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sum_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_sum_to_size(&self, size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_sum_to_size( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_svd( - &self, - some: bool, - compute_uv: bool, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_svd( - c_tensors.as_mut_ptr(), - self.c_tensor, - if some { 1 } else { 0 }, - if compute_uv { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_svd_u( - &self, - u: &Tensor, - s: &Tensor, - v: &Tensor, - some: bool, - compute_uv: bool, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_svd_u( - c_tensors.as_mut_ptr(), - u.c_tensor, - s.c_tensor, - v.c_tensor, - self.c_tensor, - if some { 1 } else { 0 }, - if compute_uv { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_swapaxes(&self, axis0: i64, axis1: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_swapaxes(c_tensors.as_mut_ptr(), self.c_tensor, axis0, axis1)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_swapaxes_(&mut self, axis0: i64, axis1: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_swapaxes_(c_tensors.as_mut_ptr(), self.c_tensor, axis0, axis1)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_swapdims(&self, dim0: i64, dim1: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_swapdims(c_tensors.as_mut_ptr(), self.c_tensor, dim0, dim1)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_swapdims_(&mut self, dim0: i64, dim1: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_swapdims_(c_tensors.as_mut_ptr(), self.c_tensor, dim0, dim1)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_tr(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_t(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_t_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_t_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_t_copy(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_t_copy(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_t_copy_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_t_copy_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_take(&self, index: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_take(c_tensors.as_mut_ptr(), self.c_tensor, index.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_take_along_dim( - &self, - indices: &Tensor, - dim: impl Into>, - ) -> Result { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_take_along_dim( - c_tensors.as_mut_ptr(), - self.c_tensor, - indices.c_tensor, - dim.unwrap_or(0i64), - dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_take_along_dim_out( - &self, - out: &Tensor, - indices: &Tensor, - dim: impl Into>, - ) -> Result { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_take_along_dim_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - indices.c_tensor, - dim.unwrap_or(0i64), - dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_take_out(&self, out: &Tensor, index: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_take_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - index.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_tan(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_tan(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_tan_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_tan_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_tan_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_tan_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_tanh(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_tanh(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_tanh_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_tanh_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_tanh_backward(grad_output: &Tensor, output: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_tanh_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - output.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_tanh_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_tanh_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - output.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_tanh_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_tanh_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_tensor_split(&self, sections: i64, dim: i64) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_tensor_split(self.c_tensor, sections, dim)); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_tensor_split_indices( - &self, - indices: impl IntList, - dim: i64, - ) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_tensor_split_indices( - self.c_tensor, - indices.as_ptr(), - indices.len_i32(), - dim - )); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_tensor_split_tensor_indices_or_sections( - &self, - tensor_indices_or_sections: &Tensor, - dim: i64, - ) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_tensor_split_tensor_indices_or_sections( - self.c_tensor, - tensor_indices_or_sections.c_tensor, - dim - )); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_tensordot( - &self, - other: &Tensor, - dims_self: impl IntList, - dims_other: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_tensordot( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor, - dims_self.as_ptr(), - dims_self.len_i32(), - dims_other.as_ptr(), - dims_other.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_tensordot_out( - &self, - out: &Tensor, - other: &Tensor, - dims_self: impl IntList, - dims_other: impl IntList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_tensordot_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor, - dims_self.as_ptr(), - dims_self.len_i32(), - dims_other.as_ptr(), - dims_other.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_threshold>(&self, threshold: S, value: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_threshold( - c_tensors.as_mut_ptr(), - self.c_tensor, - threshold.into().c_scalar, - value.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_threshold_>( - &mut self, - threshold: S, - value: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_threshold_( - c_tensors.as_mut_ptr(), - self.c_tensor, - threshold.into().c_scalar, - value.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_threshold_backward>( - &self, - grad_output: &Tensor, - threshold: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_threshold_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - self.c_tensor, - threshold.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_threshold_backward_grad_input>( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - threshold: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_threshold_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - self.c_tensor, - threshold.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_threshold_out>( - &self, - out: &Tensor, - threshold: S, - value: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_threshold_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - threshold.into().c_scalar, - value.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_tile(&self, dims: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_tile( - c_tensors.as_mut_ptr(), - self.c_tensor, - dims.as_ptr(), - dims.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_to(&self, device: Device) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_to(c_tensors.as_mut_ptr(), self.c_tensor, device.c_int())); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_to_dense( - &self, - dtype: impl Into>, - masked_grad: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_to_dense( - c_tensors.as_mut_ptr(), - self.c_tensor, - dtype.into().map_or(-1, |s| s.c_int()), - if masked_grad { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_to_dense_backward( - &self, - grad: &Tensor, - masked_grad: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_to_dense_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - self.c_tensor, - if masked_grad { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_to_device_( - &self, - device: Device, - dtype: Kind, - non_blocking: bool, - copy: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_to_device( - c_tensors.as_mut_ptr(), - self.c_tensor, - device.c_int(), - dtype.c_int(), - if non_blocking { 1 } else { 0 }, - if copy { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_to_dtype( - &self, - dtype: Kind, - non_blocking: bool, - copy: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_to_dtype( - c_tensors.as_mut_ptr(), - self.c_tensor, - dtype.c_int(), - if non_blocking { 1 } else { 0 }, - if copy { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_to_dtype_layout( - &self, - options: (Kind, Device), - non_blocking: bool, - copy: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_to_dtype_layout( - c_tensors.as_mut_ptr(), - self.c_tensor, - options.0.c_int(), - options.1.c_int(), - if non_blocking { 1 } else { 0 }, - if copy { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_to_mkldnn(&self, dtype: impl Into>) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_to_mkldnn( - c_tensors.as_mut_ptr(), - self.c_tensor, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_to_mkldnn_backward(&self, grad: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_to_mkldnn_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_to_mkldnn_out( - &self, - out: &Tensor, - dtype: impl Into>, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_to_mkldnn_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dtype.into().map_or(-1, |s| s.c_int()) - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_to_other( - &self, - other: &Tensor, - non_blocking: bool, - copy: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_to_other( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.c_tensor, - if non_blocking { 1 } else { 0 }, - if copy { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_to_padded_tensor( - &self, - padding: f64, - output_size: impl IntListOption, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_to_padded_tensor( - c_tensors.as_mut_ptr(), - self.c_tensor, - padding, - output_size.as_ptr(), - output_size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_to_padded_tensor_out( - &self, - out: &Tensor, - padding: f64, - output_size: impl IntListOption, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_to_padded_tensor_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - padding, - output_size.as_ptr(), - output_size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_to_sparse( - &self, - layout: Option, - blocksize: impl IntListOption, - dense_dim: impl Into>, - ) -> Result { - let dense_dim = dense_dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_to_sparse( - c_tensors.as_mut_ptr(), - self.c_tensor, - layout.map_or(-1, |s| s.to_i8()), - blocksize.as_ptr(), - blocksize.len_i32(), - dense_dim.unwrap_or(0i64), - dense_dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_to_sparse_bsc( - &self, - blocksize: impl IntList, - dense_dim: impl Into>, - ) -> Result { - let dense_dim = dense_dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_to_sparse_bsc( - c_tensors.as_mut_ptr(), - self.c_tensor, - blocksize.as_ptr(), - blocksize.len_i32(), - dense_dim.unwrap_or(0i64), - dense_dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_to_sparse_bsr( - &self, - blocksize: impl IntList, - dense_dim: impl Into>, - ) -> Result { - let dense_dim = dense_dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_to_sparse_bsr( - c_tensors.as_mut_ptr(), - self.c_tensor, - blocksize.as_ptr(), - blocksize.len_i32(), - dense_dim.unwrap_or(0i64), - dense_dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_to_sparse_csc(&self, dense_dim: impl Into>) -> Result { - let dense_dim = dense_dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_to_sparse_csc( - c_tensors.as_mut_ptr(), - self.c_tensor, - dense_dim.unwrap_or(0i64), - dense_dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_to_sparse_csr(&self, dense_dim: impl Into>) -> Result { - let dense_dim = dense_dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_to_sparse_csr( - c_tensors.as_mut_ptr(), - self.c_tensor, - dense_dim.unwrap_or(0i64), - dense_dim.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_to_sparse_sparse_dim(&self, sparse_dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_to_sparse_sparse_dim( - c_tensors.as_mut_ptr(), - self.c_tensor, - sparse_dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_topk( - &self, - k: i64, - dim: i64, - largest: bool, - sorted: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_topk( - c_tensors.as_mut_ptr(), - self.c_tensor, - k, - dim, - if largest { 1 } else { 0 }, - if sorted { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_topk_values( - &self, - values: &Tensor, - indices: &Tensor, - k: i64, - dim: i64, - largest: bool, - sorted: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_topk_values( - c_tensors.as_mut_ptr(), - values.c_tensor, - indices.c_tensor, - self.c_tensor, - k, - dim, - if largest { 1 } else { 0 }, - if sorted { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_totype(&self, scalar_type: Kind) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_totype(c_tensors.as_mut_ptr(), self.c_tensor, scalar_type.c_int())); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_trace(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_trace(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_trace_backward(grad: &Tensor, sizes: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_trace_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - sizes.as_ptr(), - sizes.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_trace_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_trace_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_transpose(&self, dim0: i64, dim1: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_transpose(c_tensors.as_mut_ptr(), self.c_tensor, dim0, dim1)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_transpose_(&mut self, dim0: i64, dim1: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_transpose_(c_tensors.as_mut_ptr(), self.c_tensor, dim0, dim1)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_transpose_copy(&self, dim0: i64, dim1: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_transpose_copy(c_tensors.as_mut_ptr(), self.c_tensor, dim0, dim1)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_transpose_copy_int_out( - &self, - out: &Tensor, - dim0: i64, - dim1: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_transpose_copy_int_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim0, - dim1 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_trapezoid(y: &Tensor, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_trapezoid(c_tensors.as_mut_ptr(), y.c_tensor, dim)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_trapezoid_x(y: &Tensor, x: &Tensor, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_trapezoid_x(c_tensors.as_mut_ptr(), y.c_tensor, x.c_tensor, dim)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_trapz(y: &Tensor, x: &Tensor, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_trapz(c_tensors.as_mut_ptr(), y.c_tensor, x.c_tensor, dim)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_trapz_dx(y: &Tensor, dx: f64, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_trapz_dx(c_tensors.as_mut_ptr(), y.c_tensor, dx, dim)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_triangular_solve( - &self, - a: &Tensor, - upper: bool, - transpose: bool, - unitriangular: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_triangular_solve( - c_tensors.as_mut_ptr(), - self.c_tensor, - a.c_tensor, - if upper { 1 } else { 0 }, - if transpose { 1 } else { 0 }, - if unitriangular { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_triangular_solve_x( - &self, - x: &Tensor, - m: &Tensor, - a: &Tensor, - upper: bool, - transpose: bool, - unitriangular: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_triangular_solve_x( - c_tensors.as_mut_ptr(), - x.c_tensor, - m.c_tensor, - self.c_tensor, - a.c_tensor, - if upper { 1 } else { 0 }, - if transpose { 1 } else { 0 }, - if unitriangular { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_tril(&self, diagonal: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_tril(c_tensors.as_mut_ptr(), self.c_tensor, diagonal)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_tril_(&mut self, diagonal: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_tril_(c_tensors.as_mut_ptr(), self.c_tensor, diagonal)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_tril_indices( - row: i64, - col: i64, - offset: i64, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_tril_indices( - c_tensors.as_mut_ptr(), - row, - col, - offset, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_tril_indices_out( - out: &Tensor, - row: i64, - col: i64, - offset: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_tril_indices_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - row, - col, - offset - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_tril_out(&self, out: &Tensor, diagonal: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_tril_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - diagonal - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_triplet_margin_loss( - anchor: &Tensor, - positive: &Tensor, - negative: &Tensor, - margin: f64, - p: f64, - eps: f64, - swap: bool, - reduction: crate::Reduction, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_triplet_margin_loss( - c_tensors.as_mut_ptr(), - anchor.c_tensor, - positive.c_tensor, - negative.c_tensor, - margin, - p, - eps, - if swap { 1 } else { 0 }, - reduction.to_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_triu(&self, diagonal: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_triu(c_tensors.as_mut_ptr(), self.c_tensor, diagonal)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_triu_(&mut self, diagonal: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_triu_(c_tensors.as_mut_ptr(), self.c_tensor, diagonal)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_triu_indices( - row: i64, - col: i64, - offset: i64, - options: (Kind, Device), - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_triu_indices( - c_tensors.as_mut_ptr(), - row, - col, - offset, - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_triu_indices_out( - out: &Tensor, - row: i64, - col: i64, - offset: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_triu_indices_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - row, - col, - offset - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_triu_out(&self, out: &Tensor, diagonal: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_triu_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - diagonal - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_true_divide(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_true_divide(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_true_divide_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_true_divide_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_true_divide_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_true_divide_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_true_divide_scalar>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_true_divide_scalar( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_true_divide_scalar_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_true_divide_scalar_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_trunc(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_trunc(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_trunc_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_trunc_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_trunc_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_trunc_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_type_as(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_type_as(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_unbind(&self, dim: i64) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_unbind(self.c_tensor, dim)); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_unbind_copy(&self, dim: i64) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_unbind_copy(self.c_tensor, dim)); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_unbind_copy_int_out>( - &self, - out: &[T], - dim: i64, - ) -> Result<(), TchError> { - unsafe_torch_err!(atg_unbind_copy_int_out( - ptr_list(out).as_ptr(), - out.len() as i32, - self.c_tensor, - dim - )); - Ok(()) - } - - pub fn f_unflatten(&self, dim: i64, sizes: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_unflatten( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - sizes.as_ptr(), - sizes.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_unflatten_dense_tensors>( - flat: &Tensor, - tensors: &[T], - ) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_unflatten_dense_tensors( - flat.c_tensor, - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_unfold(&self, dimension: i64, size: i64, step: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_unfold(c_tensors.as_mut_ptr(), self.c_tensor, dimension, size, step)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_unfold_backward( - grad_in: &Tensor, - input_sizes: impl IntList, - dim: i64, - size: i64, - step: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_unfold_backward( - c_tensors.as_mut_ptr(), - grad_in.c_tensor, - input_sizes.as_ptr(), - input_sizes.len_i32(), - dim, - size, - step - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_unfold_backward_out( - out: &Tensor, - grad_in: &Tensor, - input_sizes: impl IntList, - dim: i64, - size: i64, - step: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_unfold_backward_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - grad_in.c_tensor, - input_sizes.as_ptr(), - input_sizes.len_i32(), - dim, - size, - step - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_unfold_copy(&self, dimension: i64, size: i64, step: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_unfold_copy( - c_tensors.as_mut_ptr(), - self.c_tensor, - dimension, - size, - step - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_unfold_copy_out( - &self, - out: &Tensor, - dimension: i64, - size: i64, - step: i64, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_unfold_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dimension, - size, - step - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_uniform(&self, from: f64, to: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_uniform(c_tensors.as_mut_ptr(), self.c_tensor, from, to)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_uniform_(&mut self, from: f64, to: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_uniform_(c_tensors.as_mut_ptr(), self.c_tensor, from, to)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_uniform_out(&self, out: &Tensor, from: f64, to: f64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_uniform_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - from, - to - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_unique_consecutive( - &self, - return_inverse: bool, - return_counts: bool, - dim: impl Into>, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_unique_consecutive( - c_tensors.as_mut_ptr(), - self.c_tensor, - if return_inverse { 1 } else { 0 }, - if return_counts { 1 } else { 0 }, - dim.unwrap_or(0i64), - dim.is_none() as i8 - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_unique_consecutive_out( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - return_inverse: bool, - return_counts: bool, - dim: impl Into>, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let dim = dim.into(); - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_unique_consecutive_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - out2.c_tensor, - self.c_tensor, - if return_inverse { 1 } else { 0 }, - if return_counts { 1 } else { 0 }, - dim.unwrap_or(0i64), - dim.is_none() as i8 - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_unique_dim( - &self, - dim: i64, - sorted: bool, - return_inverse: bool, - return_counts: bool, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_unique_dim( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - if sorted { 1 } else { 0 }, - if return_inverse { 1 } else { 0 }, - if return_counts { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_unique_dim_consecutive( - &self, - dim: i64, - return_inverse: bool, - return_counts: bool, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_unique_dim_consecutive( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim, - if return_inverse { 1 } else { 0 }, - if return_counts { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_unique_dim_consecutive_out( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - dim: i64, - return_inverse: bool, - return_counts: bool, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_unique_dim_consecutive_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - out2.c_tensor, - self.c_tensor, - dim, - if return_inverse { 1 } else { 0 }, - if return_counts { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_unique_dim_out( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - dim: i64, - sorted: bool, - return_inverse: bool, - return_counts: bool, - ) -> Result<(Tensor, Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 3]; - unsafe_torch_err!(atg_unique_dim_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - out2.c_tensor, - self.c_tensor, - dim, - if sorted { 1 } else { 0 }, - if return_inverse { 1 } else { 0 }, - if return_counts { 1 } else { 0 } - )); - Ok(( - Tensor { c_tensor: c_tensors[0] }, - Tensor { c_tensor: c_tensors[1] }, - Tensor { c_tensor: c_tensors[2] }, - )) - } - - pub fn f_unsafe_chunk(&self, chunks: i64, dim: i64) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_unsafe_chunk(self.c_tensor, chunks, dim)); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_unsafe_split(&self, split_size: i64, dim: i64) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_unsafe_split(self.c_tensor, split_size, dim)); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_unsafe_split_tensor_out>( - &self, - out: &[T], - split_size: i64, - dim: i64, - ) -> Result<(), TchError> { - unsafe_torch_err!(atg_unsafe_split_tensor_out( - ptr_list(out).as_ptr(), - out.len() as i32, - self.c_tensor, - split_size, - dim - )); - Ok(()) - } - - pub fn f_unsafe_split_with_sizes( - &self, - split_sizes: impl IntList, - dim: i64, - ) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_unsafe_split_with_sizes( - self.c_tensor, - split_sizes.as_ptr(), - split_sizes.len_i32(), - dim - )); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_unsafe_split_with_sizes_out>( - &self, - out: &[T], - split_sizes: impl IntList, - dim: i64, - ) -> Result<(), TchError> { - unsafe_torch_err!(atg_unsafe_split_with_sizes_out( - ptr_list(out).as_ptr(), - out.len() as i32, - self.c_tensor, - split_sizes.as_ptr(), - split_sizes.len_i32(), - dim - )); - Ok(()) - } - - pub fn f_unsqueeze(&self, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_unsqueeze(c_tensors.as_mut_ptr(), self.c_tensor, dim)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_unsqueeze_(&mut self, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_unsqueeze_(c_tensors.as_mut_ptr(), self.c_tensor, dim)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_unsqueeze_copy(&self, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_unsqueeze_copy(c_tensors.as_mut_ptr(), self.c_tensor, dim)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_unsqueeze_copy_out(&self, out: &Tensor, dim: i64) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_unsqueeze_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_bicubic2d( - &self, - output_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_bicubic2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_bicubic2d_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_bicubic2d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_bicubic2d_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_bicubic2d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_bicubic2d_out( - &self, - out: &Tensor, - output_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_bicubic2d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_bicubic2d_vec( - &self, - output_size: impl IntListOption, - align_corners: bool, - scale_factors: impl DoubleList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_bicubic2d_vec( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - if align_corners { 1 } else { 0 }, - scale_factors.as_ptr(), - scale_factors.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_bilinear2d( - &self, - output_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_bilinear2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_bilinear2d_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_bilinear2d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_bilinear2d_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_bilinear2d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_bilinear2d_out( - &self, - out: &Tensor, - output_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_bilinear2d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_bilinear2d_vec( - &self, - output_size: impl IntListOption, - align_corners: bool, - scale_factors: impl DoubleList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_bilinear2d_vec( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - if align_corners { 1 } else { 0 }, - scale_factors.as_ptr(), - scale_factors.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_bilinear2d_vec_out( - &self, - out: &Tensor, - output_size: impl IntListOption, - align_corners: bool, - scale_factors: impl DoubleList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_bilinear2d_vec_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - if align_corners { 1 } else { 0 }, - scale_factors.as_ptr(), - scale_factors.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_linear1d( - &self, - output_size: impl IntList, - align_corners: bool, - scales: impl Into>, - ) -> Result { - let scales = scales.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_linear1d( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales.unwrap_or(std::f64::NAN), - scales.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_linear1d_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales: impl Into>, - ) -> Result { - let scales = scales.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_linear1d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales.unwrap_or(std::f64::NAN), - scales.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_linear1d_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales: impl Into>, - ) -> Result { - let scales = scales.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_linear1d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales.unwrap_or(std::f64::NAN), - scales.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_linear1d_out( - &self, - out: &Tensor, - output_size: impl IntList, - align_corners: bool, - scales: impl Into>, - ) -> Result { - let scales = scales.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_linear1d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales.unwrap_or(std::f64::NAN), - scales.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_linear1d_vec( - &self, - output_size: impl IntListOption, - align_corners: bool, - scale_factors: impl DoubleList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_linear1d_vec( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - if align_corners { 1 } else { 0 }, - scale_factors.as_ptr(), - scale_factors.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_nearest1d( - &self, - output_size: impl IntList, - scales: impl Into>, - ) -> Result { - let scales = scales.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_nearest1d( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - scales.unwrap_or(std::f64::NAN), - scales.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_nearest1d_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales: impl Into>, - ) -> Result { - let scales = scales.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_nearest1d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - scales.unwrap_or(std::f64::NAN), - scales.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_nearest1d_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales: impl Into>, - ) -> Result { - let scales = scales.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_nearest1d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - scales.unwrap_or(std::f64::NAN), - scales.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_nearest1d_out( - &self, - out: &Tensor, - output_size: impl IntList, - scales: impl Into>, - ) -> Result { - let scales = scales.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_nearest1d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - scales.unwrap_or(std::f64::NAN), - scales.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_nearest1d_vec( - &self, - output_size: impl IntListOption, - scale_factors: impl DoubleList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_nearest1d_vec( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - scale_factors.as_ptr(), - scale_factors.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_nearest2d( - &self, - output_size: impl IntList, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_nearest2d( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_nearest2d_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_nearest2d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_nearest2d_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_nearest2d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_nearest2d_out( - &self, - out: &Tensor, - output_size: impl IntList, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_nearest2d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_nearest2d_vec( - &self, - output_size: impl IntListOption, - scale_factors: impl DoubleList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_nearest2d_vec( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - scale_factors.as_ptr(), - scale_factors.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_nearest2d_vec_out( - &self, - out: &Tensor, - output_size: impl IntListOption, - scale_factors: impl DoubleList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_nearest2d_vec_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - scale_factors.as_ptr(), - scale_factors.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_nearest3d( - &self, - output_size: impl IntList, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_d = scales_d.into(); - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_nearest3d( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - scales_d.unwrap_or(std::f64::NAN), - scales_d.is_none() as i8, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_nearest3d_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_d = scales_d.into(); - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_nearest3d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - scales_d.unwrap_or(std::f64::NAN), - scales_d.is_none() as i8, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_nearest3d_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_d = scales_d.into(); - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_nearest3d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - scales_d.unwrap_or(std::f64::NAN), - scales_d.is_none() as i8, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_nearest3d_out( - &self, - out: &Tensor, - output_size: impl IntList, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_d = scales_d.into(); - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_nearest3d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - scales_d.unwrap_or(std::f64::NAN), - scales_d.is_none() as i8, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_nearest3d_vec( - &self, - output_size: impl IntListOption, - scale_factors: impl DoubleList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_nearest3d_vec( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - scale_factors.as_ptr(), - scale_factors.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_trilinear3d( - &self, - output_size: impl IntList, - align_corners: bool, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_d = scales_d.into(); - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_trilinear3d( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales_d.unwrap_or(std::f64::NAN), - scales_d.is_none() as i8, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_trilinear3d_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_d = scales_d.into(); - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_trilinear3d_backward( - c_tensors.as_mut_ptr(), - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales_d.unwrap_or(std::f64::NAN), - scales_d.is_none() as i8, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_trilinear3d_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_d = scales_d.into(); - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_trilinear3d_backward_grad_input( - c_tensors.as_mut_ptr(), - grad_input.c_tensor, - grad_output.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - input_size.as_ptr(), - input_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales_d.unwrap_or(std::f64::NAN), - scales_d.is_none() as i8, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_trilinear3d_out( - &self, - out: &Tensor, - output_size: impl IntList, - align_corners: bool, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Result { - let scales_d = scales_d.into(); - let scales_h = scales_h.into(); - let scales_w = scales_w.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_trilinear3d_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - if align_corners { 1 } else { 0 }, - scales_d.unwrap_or(std::f64::NAN), - scales_d.is_none() as i8, - scales_h.unwrap_or(std::f64::NAN), - scales_h.is_none() as i8, - scales_w.unwrap_or(std::f64::NAN), - scales_w.is_none() as i8 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_upsample_trilinear3d_vec( - &self, - output_size: impl IntListOption, - align_corners: bool, - scale_factors: impl DoubleList, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_upsample_trilinear3d_vec( - c_tensors.as_mut_ptr(), - self.c_tensor, - output_size.as_ptr(), - output_size.len_i32(), - if align_corners { 1 } else { 0 }, - scale_factors.as_ptr(), - scale_factors.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_value_selecting_reduction_backward( - grad: &Tensor, - dim: i64, - indices: &Tensor, - sizes: impl IntList, - keepdim: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_value_selecting_reduction_backward( - c_tensors.as_mut_ptr(), - grad.c_tensor, - dim, - indices.c_tensor, - sizes.as_ptr(), - sizes.len_i32(), - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_values(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_values(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_values_copy(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_values_copy(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_values_copy_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_values_copy_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_vander( - x: &Tensor, - n: impl Into>, - increasing: bool, - ) -> Result { - let n = n.into(); - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_vander( - c_tensors.as_mut_ptr(), - x.c_tensor, - n.unwrap_or(0i64), - n.is_none() as i8, - if increasing { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_var(&self, unbiased: bool) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_var( - c_tensors.as_mut_ptr(), - self.c_tensor, - if unbiased { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_var_correction>( - &self, - dim: impl IntListOption, - correction: S, - keepdim: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_var_correction( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - correction.into().c_scalar, - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_var_correction_out>( - &self, - out: &Tensor, - dim: impl IntListOption, - correction: S, - keepdim: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_var_correction_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - correction.into().c_scalar, - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_var_dim( - &self, - dim: impl IntListOption, - unbiased: bool, - keepdim: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_var_dim( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if unbiased { 1 } else { 0 }, - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_var_mean(&self, unbiased: bool) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_var_mean( - c_tensors.as_mut_ptr(), - self.c_tensor, - if unbiased { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_var_mean_correction>( - &self, - dim: impl IntListOption, - correction: S, - keepdim: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_var_mean_correction( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - correction.into().c_scalar, - if keepdim { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_var_mean_correction_out>( - &self, - out0: &Tensor, - out1: &Tensor, - dim: impl IntListOption, - correction: S, - keepdim: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_var_mean_correction_out( - c_tensors.as_mut_ptr(), - out0.c_tensor, - out1.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - correction.into().c_scalar, - if keepdim { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_var_mean_dim( - &self, - dim: impl IntListOption, - unbiased: bool, - keepdim: bool, - ) -> Result<(Tensor, Tensor), TchError> { - let mut c_tensors = [std::ptr::null_mut(); 2]; - unsafe_torch_err!(atg_var_mean_dim( - c_tensors.as_mut_ptr(), - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if unbiased { 1 } else { 0 }, - if keepdim { 1 } else { 0 } - )); - Ok((Tensor { c_tensor: c_tensors[0] }, Tensor { c_tensor: c_tensors[1] })) - } - - pub fn f_var_out( - &self, - out: &Tensor, - dim: impl IntListOption, - unbiased: bool, - keepdim: bool, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_var_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dim.as_ptr(), - dim.len_i32(), - if unbiased { 1 } else { 0 }, - if keepdim { 1 } else { 0 } - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_vdot(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_vdot(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_vdot_out(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_vdot_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_view_(&self, size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_view( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_view_as(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_view_as(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_view_as_complex(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_view_as_complex(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_view_as_complex_copy(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_view_as_complex_copy(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_view_as_complex_copy_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_view_as_complex_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_view_as_real(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_view_as_real(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_view_as_real_copy(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_view_as_real_copy(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_view_as_real_copy_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_view_as_real_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_view_copy(&self, size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_view_copy( - c_tensors.as_mut_ptr(), - self.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_view_copy_dtype(&self, dtype: Kind) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_view_copy_dtype( - c_tensors.as_mut_ptr(), - self.c_tensor, - dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_view_copy_dtype_out(&self, out: &Tensor, dtype: Kind) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_view_copy_dtype_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - dtype.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_view_copy_out(&self, out: &Tensor, size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_view_copy_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_view_dtype(&self, dtype: Kind) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_view_dtype(c_tensors.as_mut_ptr(), self.c_tensor, dtype.c_int())); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_vsplit(&self, sections: i64) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_vsplit(self.c_tensor, sections)); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_vsplit_array(&self, indices: impl IntList) -> Result, TchError> { - let c_tensors = - unsafe_torch_err!(atg_vsplit_array(self.c_tensor, indices.as_ptr(), indices.len_i32())); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_vstack>(tensors: &[T]) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_vstack( - c_tensors.as_mut_ptr(), - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_vstack_out>( - out: &Tensor, - tensors: &[T], - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_vstack_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - ptr_list(tensors).as_ptr(), - tensors.len() as i32 - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_where_(condition: &Tensor) -> Result, TchError> { - let c_tensors = unsafe_torch_err!(atg_where(condition.c_tensor)); - let mut r__ = vec![]; - let mut i = 0; - loop { - let c__ = unsafe { *c_tensors.add(i) }; - if c__.is_null() { - break; - } - r__.push(Tensor { c_tensor: c__ }); - i += 1; - } - unsafe { libc::free(c_tensors as *mut libc::c_void) } - Ok(r__) - } - - pub fn f_where_scalar>( - condition: &Tensor, - self_scalar: S, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_where_scalar( - c_tensors.as_mut_ptr(), - condition.c_tensor, - self_scalar.into().c_scalar, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_where_scalarother>( - &self, - condition: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_where_scalarother( - c_tensors.as_mut_ptr(), - condition.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_where_scalarself>( - condition: &Tensor, - self_scalar: S, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_where_scalarself( - c_tensors.as_mut_ptr(), - condition.c_tensor, - self_scalar.into().c_scalar, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_where_self(&self, condition: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_where_self( - c_tensors.as_mut_ptr(), - condition.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_where_self_out( - &self, - out: &Tensor, - condition: &Tensor, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_where_self_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - condition.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_xlogy(&self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_xlogy(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_xlogy_(&mut self, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_xlogy_(c_tensors.as_mut_ptr(), self.c_tensor, other.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_xlogy_outscalar_other>( - &self, - out: &Tensor, - other: S, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_xlogy_outscalar_other( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_xlogy_outscalar_self>( - out: &Tensor, - self_scalar: S, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_xlogy_outscalar_self( - c_tensors.as_mut_ptr(), - out.c_tensor, - self_scalar.into().c_scalar, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_xlogy_outtensor(&self, out: &Tensor, other: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_xlogy_outtensor( - c_tensors.as_mut_ptr(), - out.c_tensor, - self.c_tensor, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_xlogy_scalar_other>(&self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_xlogy_scalar_other( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_xlogy_scalar_other_>(&mut self, other: S) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_xlogy_scalar_other_( - c_tensors.as_mut_ptr(), - self.c_tensor, - other.into().c_scalar - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_xlogy_scalar_self>( - self_scalar: S, - other: &Tensor, - ) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_xlogy_scalar_self( - c_tensors.as_mut_ptr(), - self_scalar.into().c_scalar, - other.c_tensor - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_zero(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_zero(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_zero_(&mut self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_zero_(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_zero_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_zero_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_zeros(size: impl IntList, options: (Kind, Device)) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_zeros( - c_tensors.as_mut_ptr(), - size.as_ptr(), - size.len_i32(), - options.0.c_int(), - options.1.c_int() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_zeros_like(&self) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_zeros_like(c_tensors.as_mut_ptr(), self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_zeros_like_out(&self, out: &Tensor) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_zeros_like_out(c_tensors.as_mut_ptr(), out.c_tensor, self.c_tensor)); - Ok(Tensor { c_tensor: c_tensors[0] }) - } - - pub fn f_zeros_out(out: &Tensor, size: impl IntList) -> Result { - let mut c_tensors = [std::ptr::null_mut(); 1]; - unsafe_torch_err!(atg_zeros_out( - c_tensors.as_mut_ptr(), - out.c_tensor, - size.as_ptr(), - size.len_i32() - )); - Ok(Tensor { c_tensor: c_tensors[0] }) - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/tensor_generated.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/tensor_generated.rs deleted file mode 100644 index e71463f387..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/wrappers/tensor_generated.rs +++ /dev/null @@ -1,19703 +0,0 @@ -/* THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT BY HAND! */ -#![allow(clippy::all)] -use crate::{Device, Kind, Layout, Scalar, Tensor}; -use std::borrow::Borrow; -use std::convert::Into; -use torch_sys::*; - -impl Tensor { - pub fn internal_and_>(&mut self, other: S) -> Tensor { - self.f_internal_and_(other).unwrap() - } - - pub fn internal_and_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_internal_and_tensor_(other).unwrap() - } - - pub fn internal_iand_>(&mut self, other: S) -> Tensor { - self.f_internal_iand_(other).unwrap() - } - - pub fn internal_iand_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_internal_iand_tensor_(other).unwrap() - } - - pub fn internal_ilshift_>(&mut self, other: S) -> Tensor { - self.f_internal_ilshift_(other).unwrap() - } - - pub fn internal_ilshift_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_internal_ilshift_tensor_(other).unwrap() - } - - pub fn internal_ior_>(&mut self, other: S) -> Tensor { - self.f_internal_ior_(other).unwrap() - } - - pub fn internal_ior_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_internal_ior_tensor_(other).unwrap() - } - - pub fn internal_irshift_>(&mut self, other: S) -> Tensor { - self.f_internal_irshift_(other).unwrap() - } - - pub fn internal_irshift_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_internal_irshift_tensor_(other).unwrap() - } - - pub fn internal_ixor_>(&mut self, other: S) -> Tensor { - self.f_internal_ixor_(other).unwrap() - } - - pub fn internal_ixor_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_internal_ixor_tensor_(other).unwrap() - } - - pub fn internal_lshift_>(&mut self, other: S) -> Tensor { - self.f_internal_lshift_(other).unwrap() - } - - pub fn internal_lshift_scalar_out_>(&self, out: &Tensor, other: S) -> Tensor { - self.f_internal_lshift_scalar_out_(out, other).unwrap() - } - - pub fn internal_lshift_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_internal_lshift_tensor_(other).unwrap() - } - - pub fn internal_lshift_tensor_out_(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_internal_lshift_tensor_out_(out, other).unwrap() - } - - pub fn internal_or_>(&mut self, other: S) -> Tensor { - self.f_internal_or_(other).unwrap() - } - - pub fn internal_or_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_internal_or_tensor_(other).unwrap() - } - - pub fn internal_rshift_>(&mut self, other: S) -> Tensor { - self.f_internal_rshift_(other).unwrap() - } - - pub fn internal_rshift_scalar_out_>(&self, out: &Tensor, other: S) -> Tensor { - self.f_internal_rshift_scalar_out_(out, other).unwrap() - } - - pub fn internal_rshift_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_internal_rshift_tensor_(other).unwrap() - } - - pub fn internal_rshift_tensor_out_(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_internal_rshift_tensor_out_(out, other).unwrap() - } - - pub fn internal_xor_>(&mut self, other: S) -> Tensor { - self.f_internal_xor_(other).unwrap() - } - - pub fn internal_xor_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_internal_xor_tensor_(other).unwrap() - } - - pub fn internal_adaptive_avg_pool2d(&self, output_size: impl IntList) -> Tensor { - self.f_internal_adaptive_avg_pool2d(output_size).unwrap() - } - - pub fn internal_adaptive_avg_pool2d_backward(&self, grad_output: &Tensor) -> Tensor { - self.f_internal_adaptive_avg_pool2d_backward(grad_output).unwrap() - } - - pub fn internal_adaptive_avg_pool2d_backward_out( - &self, - out: &Tensor, - grad_output: &Tensor, - ) -> Tensor { - self.f_internal_adaptive_avg_pool2d_backward_out(out, grad_output).unwrap() - } - - pub fn internal_adaptive_avg_pool2d_out( - &self, - out: &Tensor, - output_size: impl IntList, - ) -> Tensor { - self.f_internal_adaptive_avg_pool2d_out(out, output_size).unwrap() - } - - pub fn internal_adaptive_avg_pool3d(&self, output_size: impl IntList) -> Tensor { - self.f_internal_adaptive_avg_pool3d(output_size).unwrap() - } - - pub fn internal_adaptive_avg_pool3d_backward(&self, grad_output: &Tensor) -> Tensor { - self.f_internal_adaptive_avg_pool3d_backward(grad_output).unwrap() - } - - pub fn internal_adaptive_avg_pool3d_backward_out( - &self, - out: &Tensor, - grad_output: &Tensor, - ) -> Tensor { - self.f_internal_adaptive_avg_pool3d_backward_out(out, grad_output).unwrap() - } - - pub fn internal_adaptive_avg_pool3d_out( - &self, - out: &Tensor, - output_size: impl IntList, - ) -> Tensor { - self.f_internal_adaptive_avg_pool3d_out(out, output_size).unwrap() - } - - pub fn internal_add_batch_dim(&self, batch_dim: i64, level: i64) -> Tensor { - self.f_internal_add_batch_dim(batch_dim, level).unwrap() - } - - pub fn internal_add_relu(&self, other: &Tensor) -> Tensor { - self.f_internal_add_relu(other).unwrap() - } - - pub fn internal_add_relu_(&mut self, other: &Tensor) -> Tensor { - self.f_internal_add_relu_(other).unwrap() - } - - pub fn internal_add_relu_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_internal_add_relu_out(out, other).unwrap() - } - - pub fn internal_add_relu_scalar>(&self, other: S) -> Tensor { - self.f_internal_add_relu_scalar(other).unwrap() - } - - pub fn internal_add_relu_scalar_>(&mut self, other: S) -> Tensor { - self.f_internal_add_relu_scalar_(other).unwrap() - } - - pub fn internal_add_relu_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_internal_add_relu_scalar_out(out, other).unwrap() - } - - pub fn internal_addmm_activation( - &self, - mat1: &Tensor, - mat2: &Tensor, - use_gelu: bool, - ) -> Tensor { - self.f_internal_addmm_activation(mat1, mat2, use_gelu).unwrap() - } - - pub fn internal_addmm_activation_out( - &self, - out: &Tensor, - mat1: &Tensor, - mat2: &Tensor, - use_gelu: bool, - ) -> Tensor { - self.f_internal_addmm_activation_out(out, mat1, mat2, use_gelu).unwrap() - } - - pub fn internal_aminmax(&self) -> (Tensor, Tensor) { - self.f_internal_aminmax().unwrap() - } - - pub fn internal_aminmax_dim(&self, dim: i64, keepdim: bool) -> (Tensor, Tensor) { - self.f_internal_aminmax_dim(dim, keepdim).unwrap() - } - - pub fn internal_aminmax_dim_out( - &self, - out0: &Tensor, - out1: &Tensor, - dim: i64, - keepdim: bool, - ) -> (Tensor, Tensor) { - self.f_internal_aminmax_dim_out(out0, out1, dim, keepdim).unwrap() - } - - pub fn internal_aminmax_out(&self, out0: &Tensor, out1: &Tensor) -> (Tensor, Tensor) { - self.f_internal_aminmax_out(out0, out1).unwrap() - } - - pub fn internal_amp_update_scale( - &self, - growth_tracker: &Tensor, - found_inf: &Tensor, - scale_growth_factor: f64, - scale_backoff_factor: f64, - growth_interval: i64, - ) -> (Tensor, Tensor) { - self.f_internal_amp_update_scale( - growth_tracker, - found_inf, - scale_growth_factor, - scale_backoff_factor, - growth_interval, - ) - .unwrap() - } - - pub fn internal_amp_update_scale_( - &mut self, - growth_tracker: &Tensor, - found_inf: &Tensor, - scale_growth_factor: f64, - scale_backoff_factor: f64, - growth_interval: i64, - ) -> Tensor { - self.f_internal_amp_update_scale_( - growth_tracker, - found_inf, - scale_growth_factor, - scale_backoff_factor, - growth_interval, - ) - .unwrap() - } - - pub fn internal_amp_update_scale_out( - &self, - out: &Tensor, - growth_tracker: &Tensor, - found_inf: &Tensor, - scale_growth_factor: f64, - scale_backoff_factor: f64, - growth_interval: i64, - ) -> Tensor { - self.f_internal_amp_update_scale_out( - out, - growth_tracker, - found_inf, - scale_growth_factor, - scale_backoff_factor, - growth_interval, - ) - .unwrap() - } - - pub fn internal_assert_scalar>(self_scalar: S, assert_msg: &str) { - Tensor::f_internal_assert_scalar(self_scalar, assert_msg).unwrap() - } - - pub fn internal_assert_tensor_metadata( - a: &Tensor, - size: impl IntListOption, - stride: impl IntListOption, - dtype: impl Into>, - device: Device, - layout: Option, - ) { - Tensor::f_internal_assert_tensor_metadata(a, size, stride, dtype, device, layout).unwrap() - } - - pub fn internal_autocast_to_full_precision( - &self, - cuda_enabled: bool, - cpu_enabled: bool, - ) -> Tensor { - self.f_internal_autocast_to_full_precision(cuda_enabled, cpu_enabled).unwrap() - } - - pub fn internal_autocast_to_reduced_precision( - &self, - cuda_enabled: bool, - cpu_enabled: bool, - cuda_dtype: Kind, - cpu_dtype: Kind, - ) -> Tensor { - self.f_internal_autocast_to_reduced_precision( - cuda_enabled, - cpu_enabled, - cuda_dtype, - cpu_dtype, - ) - .unwrap() - } - - pub fn internal_batch_norm_no_update>( - &self, - weight: Option, - bias: Option, - running_mean: Option, - running_var: Option, - momentum: f64, - eps: f64, - ) -> (Tensor, Tensor, Tensor, Tensor) { - self.f_internal_batch_norm_no_update(weight, bias, running_mean, running_var, momentum, eps) - .unwrap() - } - - pub fn internal_batch_norm_no_update_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - out3: &Tensor, - weight: Option, - bias: Option, - running_mean: Option, - running_var: Option, - momentum: f64, - eps: f64, - ) -> (Tensor, Tensor, Tensor, Tensor) { - self.f_internal_batch_norm_no_update_out( - out0, - out1, - out2, - out3, - weight, - bias, - running_mean, - running_var, - momentum, - eps, - ) - .unwrap() - } - - pub fn internal_batch_norm_with_update>( - &self, - weight: Option, - bias: Option, - running_mean: &Tensor, - running_var: &Tensor, - momentum: f64, - eps: f64, - ) -> (Tensor, Tensor, Tensor, Tensor) { - self.f_internal_batch_norm_with_update( - weight, - bias, - running_mean, - running_var, - momentum, - eps, - ) - .unwrap() - } - - pub fn internal_batch_norm_with_update_functional>( - &self, - weight: Option, - bias: Option, - running_mean: &Tensor, - running_var: &Tensor, - momentum: f64, - eps: f64, - ) -> (Tensor, Tensor, Tensor, Tensor, Tensor, Tensor) { - self.f_internal_batch_norm_with_update_functional( - weight, - bias, - running_mean, - running_var, - momentum, - eps, - ) - .unwrap() - } - - pub fn internal_batch_norm_with_update_out>( - &self, - out: &Tensor, - save_mean: &Tensor, - save_invstd: &Tensor, - reserve: &Tensor, - weight: Option, - bias: Option, - running_mean: &Tensor, - running_var: &Tensor, - momentum: f64, - eps: f64, - ) -> (Tensor, Tensor, Tensor, Tensor) { - self.f_internal_batch_norm_with_update_out( - out, - save_mean, - save_invstd, - reserve, - weight, - bias, - running_mean, - running_var, - momentum, - eps, - ) - .unwrap() - } - - pub fn internal_cast_byte(&self, non_blocking: bool) -> Tensor { - self.f_internal_cast_byte(non_blocking).unwrap() - } - - pub fn internal_cast_char(&self, non_blocking: bool) -> Tensor { - self.f_internal_cast_char(non_blocking).unwrap() - } - - pub fn internal_cast_double(&self, non_blocking: bool) -> Tensor { - self.f_internal_cast_double(non_blocking).unwrap() - } - - pub fn internal_cast_float(&self, non_blocking: bool) -> Tensor { - self.f_internal_cast_float(non_blocking).unwrap() - } - - pub fn internal_cast_half(&self, non_blocking: bool) -> Tensor { - self.f_internal_cast_half(non_blocking).unwrap() - } - - pub fn internal_cast_int(&self, non_blocking: bool) -> Tensor { - self.f_internal_cast_int(non_blocking).unwrap() - } - - pub fn internal_cast_long(&self, non_blocking: bool) -> Tensor { - self.f_internal_cast_long(non_blocking).unwrap() - } - - pub fn internal_cast_short(&self, non_blocking: bool) -> Tensor { - self.f_internal_cast_short(non_blocking).unwrap() - } - - pub fn internal_cdist_backward( - grad: &Tensor, - x1: &Tensor, - x2: &Tensor, - p: f64, - cdist: &Tensor, - ) -> Tensor { - Tensor::f_internal_cdist_backward(grad, x1, x2, p, cdist).unwrap() - } - - pub fn internal_cdist_backward_out( - out: &Tensor, - grad: &Tensor, - x1: &Tensor, - x2: &Tensor, - p: f64, - cdist: &Tensor, - ) -> Tensor { - Tensor::f_internal_cdist_backward_out(out, grad, x1, x2, p, cdist).unwrap() - } - - pub fn internal_cholesky_solve_helper(&self, a: &Tensor, upper: bool) -> Tensor { - self.f_internal_cholesky_solve_helper(a, upper).unwrap() - } - - pub fn internal_cholesky_solve_helper_out( - &self, - out: &Tensor, - a: &Tensor, - upper: bool, - ) -> Tensor { - self.f_internal_cholesky_solve_helper_out(out, a, upper).unwrap() - } - - pub fn internal_chunk_cat>( - tensors: &[T], - dim: i64, - num_chunks: i64, - ) -> Tensor { - Tensor::f_internal_chunk_cat(tensors, dim, num_chunks).unwrap() - } - - pub fn internal_chunk_cat_out>( - out: &Tensor, - tensors: &[T], - dim: i64, - num_chunks: i64, - ) -> Tensor { - Tensor::f_internal_chunk_cat_out(out, tensors, dim, num_chunks).unwrap() - } - - pub fn internal_coalesce(&self) -> Tensor { - self.f_internal_coalesce().unwrap() - } - - pub fn internal_coalesce_out(&self, out: &Tensor) -> Tensor { - self.f_internal_coalesce_out(out).unwrap() - } - - pub fn internal_coalesced(&self, coalesced: bool) -> Tensor { - self.f_internal_coalesced(coalesced).unwrap() - } - - pub fn internal_coalesced_(&mut self, coalesced: bool) -> Tensor { - self.f_internal_coalesced_(coalesced).unwrap() - } - - pub fn internal_coalesced_out(&self, out: &Tensor, coalesced: bool) -> Tensor { - self.f_internal_coalesced_out(out, coalesced).unwrap() - } - - pub fn internal_compute_linear_combination(&self, coefficients: &Tensor) -> Tensor { - self.f_internal_compute_linear_combination(coefficients).unwrap() - } - - pub fn internal_compute_linear_combination_out( - &self, - out: &Tensor, - coefficients: &Tensor, - ) -> Tensor { - self.f_internal_compute_linear_combination_out(out, coefficients).unwrap() - } - - pub fn internal_conj(&self) -> Tensor { - self.f_internal_conj().unwrap() - } - - pub fn internal_conj_copy(&self) -> Tensor { - self.f_internal_conj_copy().unwrap() - } - - pub fn internal_conj_copy_out(&self, out: &Tensor) -> Tensor { - self.f_internal_conj_copy_out(out).unwrap() - } - - pub fn internal_conj_physical(&self) -> Tensor { - self.f_internal_conj_physical().unwrap() - } - - pub fn internal_conj_physical_out(&self, out: &Tensor) -> Tensor { - self.f_internal_conj_physical_out(out).unwrap() - } - - pub fn internal_conv_depthwise2d>( - &self, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ) -> Tensor { - self.f_internal_conv_depthwise2d(weight, kernel_size, bias, stride, padding, dilation) - .unwrap() - } - - pub fn internal_conv_depthwise2d_out>( - &self, - out: &Tensor, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ) -> Tensor { - self.f_internal_conv_depthwise2d_out( - out, - weight, - kernel_size, - bias, - stride, - padding, - dilation, - ) - .unwrap() - } - - pub fn internal_convert_indices_from_coo_to_csr(&self, size: i64, out_int32: bool) -> Tensor { - self.f_internal_convert_indices_from_coo_to_csr(size, out_int32).unwrap() - } - - pub fn internal_convert_indices_from_coo_to_csr_out( - &self, - out: &Tensor, - size: i64, - out_int32: bool, - ) -> Tensor { - self.f_internal_convert_indices_from_coo_to_csr_out(out, size, out_int32).unwrap() - } - - pub fn internal_convert_indices_from_csr_to_coo( - crow_indices: &Tensor, - col_indices: &Tensor, - out_int32: bool, - transpose: bool, - ) -> Tensor { - Tensor::f_internal_convert_indices_from_csr_to_coo( - crow_indices, - col_indices, - out_int32, - transpose, - ) - .unwrap() - } - - pub fn internal_convert_indices_from_csr_to_coo_out( - out: &Tensor, - crow_indices: &Tensor, - col_indices: &Tensor, - out_int32: bool, - transpose: bool, - ) -> Tensor { - Tensor::f_internal_convert_indices_from_csr_to_coo_out( - out, - crow_indices, - col_indices, - out_int32, - transpose, - ) - .unwrap() - } - - pub fn internal_convert_weight_to_int4pack(&self, innerktiles: i64) -> Tensor { - self.f_internal_convert_weight_to_int4pack(innerktiles).unwrap() - } - - pub fn internal_convert_weight_to_int4pack_for_cpu(&self, innerktiles: i64) -> Tensor { - self.f_internal_convert_weight_to_int4pack_for_cpu(innerktiles).unwrap() - } - - pub fn internal_convolution>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - transposed: bool, - output_padding: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - cudnn_enabled: bool, - allow_tf32: bool, - ) -> Tensor { - self.f_internal_convolution( - weight, - bias, - stride, - padding, - dilation, - transposed, - output_padding, - groups, - benchmark, - deterministic, - cudnn_enabled, - allow_tf32, - ) - .unwrap() - } - - pub fn internal_convolution_deprecated>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - transposed: bool, - output_padding: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - cudnn_enabled: bool, - ) -> Tensor { - self.f_internal_convolution_deprecated( - weight, - bias, - stride, - padding, - dilation, - transposed, - output_padding, - groups, - benchmark, - deterministic, - cudnn_enabled, - ) - .unwrap() - } - - pub fn internal_convolution_mode>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: &str, - dilation: impl IntList, - groups: i64, - ) -> Tensor { - self.f_internal_convolution_mode(weight, bias, stride, padding, dilation, groups).unwrap() - } - - pub fn internal_convolution_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - transposed: bool, - output_padding: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - cudnn_enabled: bool, - allow_tf32: bool, - ) -> Tensor { - self.f_internal_convolution_out( - out, - weight, - bias, - stride, - padding, - dilation, - transposed, - output_padding, - groups, - benchmark, - deterministic, - cudnn_enabled, - allow_tf32, - ) - .unwrap() - } - - pub fn internal_copy_from(&self, dst: &Tensor, non_blocking: bool) -> Tensor { - self.f_internal_copy_from(dst, non_blocking).unwrap() - } - - pub fn internal_copy_from_and_resize(&self, dst: &Tensor) -> Tensor { - self.f_internal_copy_from_and_resize(dst).unwrap() - } - - pub fn internal_copy_from_and_resize_out(&self, out: &Tensor, dst: &Tensor) -> Tensor { - self.f_internal_copy_from_and_resize_out(out, dst).unwrap() - } - - pub fn internal_copy_from_out(&self, out: &Tensor, dst: &Tensor, non_blocking: bool) -> Tensor { - self.f_internal_copy_from_out(out, dst, non_blocking).unwrap() - } - - pub fn internal_cslt_compress(&self) -> Tensor { - self.f_internal_cslt_compress().unwrap() - } - - pub fn internal_cslt_sparse_mm>( - compressed_a: &Tensor, - dense_b: &Tensor, - bias: Option, - alpha: Option, - out_dtype: impl Into>, - transpose_result: bool, - alg_id: i64, - split_k: i64, - split_k_mode: i64, - ) -> Tensor { - Tensor::f_internal_cslt_sparse_mm( - compressed_a, - dense_b, - bias, - alpha, - out_dtype, - transpose_result, - alg_id, - split_k, - split_k_mode, - ) - .unwrap() - } - - pub fn internal_cslt_sparse_mm_search>( - compressed_a: &Tensor, - dense_b: &Tensor, - bias: Option, - alpha: Option, - out_dtype: impl Into>, - transpose_result: bool, - ) -> i64 { - Tensor::f_internal_cslt_sparse_mm_search( - compressed_a, - dense_b, - bias, - alpha, - out_dtype, - transpose_result, - ) - .unwrap() - } - - pub fn internal_ctc_loss( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: impl IntList, - target_lengths: impl IntList, - blank: i64, - zero_infinity: bool, - ) -> (Tensor, Tensor) { - Tensor::f_internal_ctc_loss( - log_probs, - targets, - input_lengths, - target_lengths, - blank, - zero_infinity, - ) - .unwrap() - } - - pub fn internal_ctc_loss_backward( - grad: &Tensor, - log_probs: &Tensor, - targets: &Tensor, - input_lengths: impl IntList, - target_lengths: impl IntList, - neg_log_likelihood: &Tensor, - log_alpha: &Tensor, - blank: i64, - zero_infinity: bool, - ) -> Tensor { - Tensor::f_internal_ctc_loss_backward( - grad, - log_probs, - targets, - input_lengths, - target_lengths, - neg_log_likelihood, - log_alpha, - blank, - zero_infinity, - ) - .unwrap() - } - - pub fn internal_ctc_loss_backward_out( - out: &Tensor, - grad: &Tensor, - log_probs: &Tensor, - targets: &Tensor, - input_lengths: impl IntList, - target_lengths: impl IntList, - neg_log_likelihood: &Tensor, - log_alpha: &Tensor, - blank: i64, - zero_infinity: bool, - ) -> Tensor { - Tensor::f_internal_ctc_loss_backward_out( - out, - grad, - log_probs, - targets, - input_lengths, - target_lengths, - neg_log_likelihood, - log_alpha, - blank, - zero_infinity, - ) - .unwrap() - } - - pub fn internal_ctc_loss_backward_tensor( - grad: &Tensor, - log_probs: &Tensor, - targets: &Tensor, - input_lengths: &Tensor, - target_lengths: &Tensor, - neg_log_likelihood: &Tensor, - log_alpha: &Tensor, - blank: i64, - zero_infinity: bool, - ) -> Tensor { - Tensor::f_internal_ctc_loss_backward_tensor( - grad, - log_probs, - targets, - input_lengths, - target_lengths, - neg_log_likelihood, - log_alpha, - blank, - zero_infinity, - ) - .unwrap() - } - - pub fn internal_ctc_loss_out( - out0: &Tensor, - out1: &Tensor, - log_probs: &Tensor, - targets: &Tensor, - input_lengths: impl IntList, - target_lengths: impl IntList, - blank: i64, - zero_infinity: bool, - ) -> (Tensor, Tensor) { - Tensor::f_internal_ctc_loss_out( - out0, - out1, - log_probs, - targets, - input_lengths, - target_lengths, - blank, - zero_infinity, - ) - .unwrap() - } - - pub fn internal_ctc_loss_tensor( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: &Tensor, - target_lengths: &Tensor, - blank: i64, - zero_infinity: bool, - ) -> (Tensor, Tensor) { - Tensor::f_internal_ctc_loss_tensor( - log_probs, - targets, - input_lengths, - target_lengths, - blank, - zero_infinity, - ) - .unwrap() - } - - pub fn internal_ctc_loss_tensor_out( - out0: &Tensor, - out1: &Tensor, - log_probs: &Tensor, - targets: &Tensor, - input_lengths: &Tensor, - target_lengths: &Tensor, - blank: i64, - zero_infinity: bool, - ) -> (Tensor, Tensor) { - Tensor::f_internal_ctc_loss_tensor_out( - out0, - out1, - log_probs, - targets, - input_lengths, - target_lengths, - blank, - zero_infinity, - ) - .unwrap() - } - - pub fn internal_cudnn_attention_backward( - grad_out: &Tensor, - query: &Tensor, - key: &Tensor, - value: &Tensor, - out: &Tensor, - logsumexp: &Tensor, - philox_seed: &Tensor, - philox_offset: &Tensor, - attn_bias: &Tensor, - cum_seq_q: &Tensor, - cum_seq_k: &Tensor, - max_q: i64, - max_k: i64, - dropout_p: f64, - is_causal: bool, - scale: impl Into>, - ) -> (Tensor, Tensor, Tensor) { - Tensor::f_internal_cudnn_attention_backward( - grad_out, - query, - key, - value, - out, - logsumexp, - philox_seed, - philox_offset, - attn_bias, - cum_seq_q, - cum_seq_k, - max_q, - max_k, - dropout_p, - is_causal, - scale, - ) - .unwrap() - } - - pub fn internal_cudnn_ctc_loss( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: impl IntList, - target_lengths: impl IntList, - blank: i64, - deterministic: bool, - zero_infinity: bool, - ) -> (Tensor, Tensor) { - Tensor::f_internal_cudnn_ctc_loss( - log_probs, - targets, - input_lengths, - target_lengths, - blank, - deterministic, - zero_infinity, - ) - .unwrap() - } - - pub fn internal_cudnn_ctc_loss_out( - out0: &Tensor, - out1: &Tensor, - log_probs: &Tensor, - targets: &Tensor, - input_lengths: impl IntList, - target_lengths: impl IntList, - blank: i64, - deterministic: bool, - zero_infinity: bool, - ) -> (Tensor, Tensor) { - Tensor::f_internal_cudnn_ctc_loss_out( - out0, - out1, - log_probs, - targets, - input_lengths, - target_lengths, - blank, - deterministic, - zero_infinity, - ) - .unwrap() - } - - pub fn internal_cudnn_ctc_loss_tensor( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: &Tensor, - target_lengths: &Tensor, - blank: i64, - deterministic: bool, - zero_infinity: bool, - ) -> (Tensor, Tensor) { - Tensor::f_internal_cudnn_ctc_loss_tensor( - log_probs, - targets, - input_lengths, - target_lengths, - blank, - deterministic, - zero_infinity, - ) - .unwrap() - } - - pub fn internal_cudnn_init_dropout_state( - dropout: f64, - train: bool, - dropout_seed: i64, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_internal_cudnn_init_dropout_state(dropout, train, dropout_seed, options).unwrap() - } - - pub fn internal_cudnn_init_dropout_state_out( - out: &Tensor, - dropout: f64, - train: bool, - dropout_seed: i64, - ) -> Tensor { - Tensor::f_internal_cudnn_init_dropout_state_out(out, dropout, train, dropout_seed).unwrap() - } - - pub fn internal_cudnn_rnn>( - &self, - weight: &[T], - weight_stride0: i64, - weight_buf: Option, - hx: &Tensor, - cx: Option, - mode: i64, - hidden_size: i64, - proj_size: i64, - num_layers: i64, - batch_first: bool, - dropout: f64, - train: bool, - bidirectional: bool, - batch_sizes: impl IntList, - dropout_state: Option, - ) -> (Tensor, Tensor, Tensor, Tensor, Tensor) { - self.f_internal_cudnn_rnn( - weight, - weight_stride0, - weight_buf, - hx, - cx, - mode, - hidden_size, - proj_size, - num_layers, - batch_first, - dropout, - train, - bidirectional, - batch_sizes, - dropout_state, - ) - .unwrap() - } - - pub fn internal_cudnn_rnn_flatten_weight>( - weight_arr: &[T], - weight_stride0: i64, - input_size: i64, - mode: i64, - hidden_size: i64, - proj_size: i64, - num_layers: i64, - batch_first: bool, - bidirectional: bool, - ) -> Tensor { - Tensor::f_internal_cudnn_rnn_flatten_weight( - weight_arr, - weight_stride0, - input_size, - mode, - hidden_size, - proj_size, - num_layers, - batch_first, - bidirectional, - ) - .unwrap() - } - - pub fn internal_cudnn_rnn_flatten_weight_out>( - out: &Tensor, - weight_arr: &[T], - weight_stride0: i64, - input_size: i64, - mode: i64, - hidden_size: i64, - proj_size: i64, - num_layers: i64, - batch_first: bool, - bidirectional: bool, - ) -> Tensor { - Tensor::f_internal_cudnn_rnn_flatten_weight_out( - out, - weight_arr, - weight_stride0, - input_size, - mode, - hidden_size, - proj_size, - num_layers, - batch_first, - bidirectional, - ) - .unwrap() - } - - pub fn internal_cudnn_rnn_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - out3: &Tensor, - out4: &Tensor, - weight: &[T], - weight_stride0: i64, - weight_buf: Option, - hx: &Tensor, - cx: Option, - mode: i64, - hidden_size: i64, - proj_size: i64, - num_layers: i64, - batch_first: bool, - dropout: f64, - train: bool, - bidirectional: bool, - batch_sizes: impl IntList, - dropout_state: Option, - ) -> (Tensor, Tensor, Tensor, Tensor, Tensor) { - self.f_internal_cudnn_rnn_out( - out0, - out1, - out2, - out3, - out4, - weight, - weight_stride0, - weight_buf, - hx, - cx, - mode, - hidden_size, - proj_size, - num_layers, - batch_first, - dropout, - train, - bidirectional, - batch_sizes, - dropout_state, - ) - .unwrap() - } - - pub fn internal_debug_has_internal_overlap(&self) -> i64 { - self.f_internal_debug_has_internal_overlap().unwrap() - } - - pub fn internal_dim_arange(like: &Tensor, dim: i64) -> Tensor { - Tensor::f_internal_dim_arange(like, dim).unwrap() - } - - pub fn internal_dimi(&self) -> i64 { - self.f_internal_dimi().unwrap() - } - - pub fn internal_dimv(&self) -> i64 { - self.f_internal_dimv().unwrap() - } - - pub fn internal_dirichlet_grad(x: &Tensor, alpha: &Tensor, total: &Tensor) -> Tensor { - Tensor::f_internal_dirichlet_grad(x, alpha, total).unwrap() - } - - pub fn internal_dirichlet_grad_out( - out: &Tensor, - x: &Tensor, - alpha: &Tensor, - total: &Tensor, - ) -> Tensor { - Tensor::f_internal_dirichlet_grad_out(out, x, alpha, total).unwrap() - } - - pub fn internal_dyn_quant_matmul_4bit( - inp: &Tensor, - packed_weights: &Tensor, - block_size: i64, - in_features: i64, - out_features: i64, - ) -> Tensor { - Tensor::f_internal_dyn_quant_matmul_4bit( - inp, - packed_weights, - block_size, - in_features, - out_features, - ) - .unwrap() - } - - pub fn internal_dyn_quant_pack_4bit_weight>( - weights: &Tensor, - scales_zeros: &Tensor, - bias: Option, - block_size: i64, - in_features: i64, - out_features: i64, - ) -> Tensor { - Tensor::f_internal_dyn_quant_pack_4bit_weight( - weights, - scales_zeros, - bias, - block_size, - in_features, - out_features, - ) - .unwrap() - } - - pub fn internal_efficient_attention_backward>( - grad_out_: &Tensor, - query: &Tensor, - key: &Tensor, - value: &Tensor, - bias: Option, - out: &Tensor, - cu_seqlens_q: Option, - cu_seqlens_k: Option, - max_seqlen_q: i64, - max_seqlen_k: i64, - logsumexp: &Tensor, - dropout_p: f64, - philox_seed: &Tensor, - philox_offset: &Tensor, - custom_mask_type: i64, - bias_requires_grad: bool, - scale: impl Into>, - num_splits_key: impl Into>, - window_size: impl Into>, - shared_storage_dqdkdv: bool, - ) -> (Tensor, Tensor, Tensor, Tensor) { - Tensor::f_internal_efficient_attention_backward( - grad_out_, - query, - key, - value, - bias, - out, - cu_seqlens_q, - cu_seqlens_k, - max_seqlen_q, - max_seqlen_k, - logsumexp, - dropout_p, - philox_seed, - philox_offset, - custom_mask_type, - bias_requires_grad, - scale, - num_splits_key, - window_size, - shared_storage_dqdkdv, - ) - .unwrap() - } - - pub fn internal_efficientzerotensor(size: impl IntList, options: (Kind, Device)) -> Tensor { - Tensor::f_internal_efficientzerotensor(size, options).unwrap() - } - - pub fn internal_efficientzerotensor_out(out: &Tensor, size: impl IntList) -> Tensor { - Tensor::f_internal_efficientzerotensor_out(out, size).unwrap() - } - - pub fn internal_embedding_bag>( - weight: &Tensor, - indices: &Tensor, - offsets: &Tensor, - scale_grad_by_freq: bool, - mode: i64, - sparse: bool, - per_sample_weights: Option, - include_last_offset: bool, - padding_idx: i64, - ) -> (Tensor, Tensor, Tensor, Tensor) { - Tensor::f_internal_embedding_bag( - weight, - indices, - offsets, - scale_grad_by_freq, - mode, - sparse, - per_sample_weights, - include_last_offset, - padding_idx, - ) - .unwrap() - } - - pub fn internal_embedding_bag_backward>( - grad: &Tensor, - indices: &Tensor, - offsets: &Tensor, - offset2bag: &Tensor, - bag_size: &Tensor, - maximum_indices: &Tensor, - num_weights: i64, - scale_grad_by_freq: bool, - mode: i64, - sparse: bool, - per_sample_weights: Option, - padding_idx: i64, - ) -> Tensor { - Tensor::f_internal_embedding_bag_backward( - grad, - indices, - offsets, - offset2bag, - bag_size, - maximum_indices, - num_weights, - scale_grad_by_freq, - mode, - sparse, - per_sample_weights, - padding_idx, - ) - .unwrap() - } - - pub fn internal_embedding_bag_dense_backward>( - grad: &Tensor, - indices: &Tensor, - offset2bag: &Tensor, - bag_size: &Tensor, - maximum_indices: &Tensor, - num_weights: i64, - scale_grad_by_freq: bool, - mode: i64, - per_sample_weights: Option, - padding_idx: i64, - ) -> Tensor { - Tensor::f_internal_embedding_bag_dense_backward( - grad, - indices, - offset2bag, - bag_size, - maximum_indices, - num_weights, - scale_grad_by_freq, - mode, - per_sample_weights, - padding_idx, - ) - .unwrap() - } - - pub fn internal_embedding_bag_dense_backward_out>( - out: &Tensor, - grad: &Tensor, - indices: &Tensor, - offset2bag: &Tensor, - bag_size: &Tensor, - maximum_indices: &Tensor, - num_weights: i64, - scale_grad_by_freq: bool, - mode: i64, - per_sample_weights: Option, - padding_idx: i64, - ) -> Tensor { - Tensor::f_internal_embedding_bag_dense_backward_out( - out, - grad, - indices, - offset2bag, - bag_size, - maximum_indices, - num_weights, - scale_grad_by_freq, - mode, - per_sample_weights, - padding_idx, - ) - .unwrap() - } - - pub fn internal_embedding_bag_forward_only>( - weight: &Tensor, - indices: &Tensor, - offsets: &Tensor, - scale_grad_by_freq: bool, - mode: i64, - sparse: bool, - per_sample_weights: Option, - include_last_offset: bool, - padding_idx: i64, - ) -> (Tensor, Tensor, Tensor, Tensor) { - Tensor::f_internal_embedding_bag_forward_only( - weight, - indices, - offsets, - scale_grad_by_freq, - mode, - sparse, - per_sample_weights, - include_last_offset, - padding_idx, - ) - .unwrap() - } - - pub fn internal_embedding_bag_forward_only_out>( - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - out3: &Tensor, - weight: &Tensor, - indices: &Tensor, - offsets: &Tensor, - scale_grad_by_freq: bool, - mode: i64, - sparse: bool, - per_sample_weights: Option, - include_last_offset: bool, - padding_idx: i64, - ) -> (Tensor, Tensor, Tensor, Tensor) { - Tensor::f_internal_embedding_bag_forward_only_out( - out0, - out1, - out2, - out3, - weight, - indices, - offsets, - scale_grad_by_freq, - mode, - sparse, - per_sample_weights, - include_last_offset, - padding_idx, - ) - .unwrap() - } - - pub fn internal_embedding_bag_out>( - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - out3: &Tensor, - weight: &Tensor, - indices: &Tensor, - offsets: &Tensor, - scale_grad_by_freq: bool, - mode: i64, - sparse: bool, - per_sample_weights: Option, - include_last_offset: bool, - padding_idx: i64, - ) -> (Tensor, Tensor, Tensor, Tensor) { - Tensor::f_internal_embedding_bag_out( - out0, - out1, - out2, - out3, - weight, - indices, - offsets, - scale_grad_by_freq, - mode, - sparse, - per_sample_weights, - include_last_offset, - padding_idx, - ) - .unwrap() - } - - pub fn internal_embedding_bag_per_sample_weights_backward( - grad: &Tensor, - weight: &Tensor, - indices: &Tensor, - offsets: &Tensor, - offset2bag: &Tensor, - mode: i64, - padding_idx: i64, - ) -> Tensor { - Tensor::f_internal_embedding_bag_per_sample_weights_backward( - grad, - weight, - indices, - offsets, - offset2bag, - mode, - padding_idx, - ) - .unwrap() - } - - pub fn internal_embedding_bag_per_sample_weights_backward_out( - out: &Tensor, - grad: &Tensor, - weight: &Tensor, - indices: &Tensor, - offsets: &Tensor, - offset2bag: &Tensor, - mode: i64, - padding_idx: i64, - ) -> Tensor { - Tensor::f_internal_embedding_bag_per_sample_weights_backward_out( - out, - grad, - weight, - indices, - offsets, - offset2bag, - mode, - padding_idx, - ) - .unwrap() - } - - pub fn internal_embedding_bag_sparse_backward>( - grad: &Tensor, - indices: &Tensor, - offsets: &Tensor, - offset2bag: &Tensor, - bag_size: &Tensor, - num_weights: i64, - scale_grad_by_freq: bool, - mode: i64, - per_sample_weights: Option, - padding_idx: i64, - ) -> Tensor { - Tensor::f_internal_embedding_bag_sparse_backward( - grad, - indices, - offsets, - offset2bag, - bag_size, - num_weights, - scale_grad_by_freq, - mode, - per_sample_weights, - padding_idx, - ) - .unwrap() - } - - pub fn internal_empty_affine_quantized( - size: impl IntList, - options: (Kind, Device), - scale: f64, - zero_point: i64, - ) -> Tensor { - Tensor::f_internal_empty_affine_quantized(size, options, scale, zero_point).unwrap() - } - - pub fn internal_empty_affine_quantized_out( - out: &Tensor, - size: impl IntList, - scale: f64, - zero_point: i64, - ) -> Tensor { - Tensor::f_internal_empty_affine_quantized_out(out, size, scale, zero_point).unwrap() - } - - pub fn internal_empty_per_channel_affine_quantized( - size: impl IntList, - scales: &Tensor, - zero_points: &Tensor, - axis: i64, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_internal_empty_per_channel_affine_quantized( - size, - scales, - zero_points, - axis, - options, - ) - .unwrap() - } - - pub fn internal_empty_per_channel_affine_quantized_out( - out: &Tensor, - size: impl IntList, - scales: &Tensor, - zero_points: &Tensor, - axis: i64, - ) -> Tensor { - Tensor::f_internal_empty_per_channel_affine_quantized_out( - out, - size, - scales, - zero_points, - axis, - ) - .unwrap() - } - - pub fn internal_euclidean_dist(x1: &Tensor, x2: &Tensor) -> Tensor { - Tensor::f_internal_euclidean_dist(x1, x2).unwrap() - } - - pub fn internal_euclidean_dist_out(out: &Tensor, x1: &Tensor, x2: &Tensor) -> Tensor { - Tensor::f_internal_euclidean_dist_out(out, x1, x2).unwrap() - } - - pub fn internal_fake_quantize_learnable_per_channel_affine( - &self, - scale: &Tensor, - zero_point: &Tensor, - axis: i64, - quant_min: i64, - quant_max: i64, - grad_factor: f64, - ) -> Tensor { - self.f_internal_fake_quantize_learnable_per_channel_affine( - scale, - zero_point, - axis, - quant_min, - quant_max, - grad_factor, - ) - .unwrap() - } - - pub fn internal_fake_quantize_learnable_per_channel_affine_backward( - &self, - grad: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - axis: i64, - quant_min: i64, - quant_max: i64, - grad_factor: f64, - ) -> (Tensor, Tensor, Tensor) { - self.f_internal_fake_quantize_learnable_per_channel_affine_backward( - grad, - scale, - zero_point, - axis, - quant_min, - quant_max, - grad_factor, - ) - .unwrap() - } - - pub fn internal_fake_quantize_learnable_per_channel_affine_out( - &self, - out: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - axis: i64, - quant_min: i64, - quant_max: i64, - grad_factor: f64, - ) -> Tensor { - self.f_internal_fake_quantize_learnable_per_channel_affine_out( - out, - scale, - zero_point, - axis, - quant_min, - quant_max, - grad_factor, - ) - .unwrap() - } - - pub fn internal_fake_quantize_learnable_per_tensor_affine( - &self, - scale: &Tensor, - zero_point: &Tensor, - quant_min: i64, - quant_max: i64, - grad_factor: f64, - ) -> Tensor { - self.f_internal_fake_quantize_learnable_per_tensor_affine( - scale, - zero_point, - quant_min, - quant_max, - grad_factor, - ) - .unwrap() - } - - pub fn internal_fake_quantize_learnable_per_tensor_affine_backward( - &self, - grad: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - quant_min: i64, - quant_max: i64, - grad_factor: f64, - ) -> (Tensor, Tensor, Tensor) { - self.f_internal_fake_quantize_learnable_per_tensor_affine_backward( - grad, - scale, - zero_point, - quant_min, - quant_max, - grad_factor, - ) - .unwrap() - } - - pub fn internal_fake_quantize_learnable_per_tensor_affine_out( - &self, - out: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - quant_min: i64, - quant_max: i64, - grad_factor: f64, - ) -> Tensor { - self.f_internal_fake_quantize_learnable_per_tensor_affine_out( - out, - scale, - zero_point, - quant_min, - quant_max, - grad_factor, - ) - .unwrap() - } - - pub fn internal_fake_quantize_per_tensor_affine_cachemask_tensor_qparams( - &self, - scale: &Tensor, - zero_point: &Tensor, - fake_quant_enabled: &Tensor, - quant_min: i64, - quant_max: i64, - ) -> (Tensor, Tensor) { - self.f_internal_fake_quantize_per_tensor_affine_cachemask_tensor_qparams( - scale, - zero_point, - fake_quant_enabled, - quant_min, - quant_max, - ) - .unwrap() - } - - pub fn internal_fake_quantize_per_tensor_affine_cachemask_tensor_qparams_out( - &self, - out0: &Tensor, - out1: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - fake_quant_enabled: &Tensor, - quant_min: i64, - quant_max: i64, - ) -> (Tensor, Tensor) { - self.f_internal_fake_quantize_per_tensor_affine_cachemask_tensor_qparams_out( - out0, - out1, - scale, - zero_point, - fake_quant_enabled, - quant_min, - quant_max, - ) - .unwrap() - } - - pub fn internal_fft_c2c(&self, dim: impl IntList, normalization: i64, forward: bool) -> Tensor { - self.f_internal_fft_c2c(dim, normalization, forward).unwrap() - } - - pub fn internal_fft_c2c_out( - &self, - out: &Tensor, - dim: impl IntList, - normalization: i64, - forward: bool, - ) -> Tensor { - self.f_internal_fft_c2c_out(out, dim, normalization, forward).unwrap() - } - - pub fn internal_fft_c2r( - &self, - dim: impl IntList, - normalization: i64, - last_dim_size: i64, - ) -> Tensor { - self.f_internal_fft_c2r(dim, normalization, last_dim_size).unwrap() - } - - pub fn internal_fft_c2r_out( - &self, - out: &Tensor, - dim: impl IntList, - normalization: i64, - last_dim_size: i64, - ) -> Tensor { - self.f_internal_fft_c2r_out(out, dim, normalization, last_dim_size).unwrap() - } - - pub fn internal_fft_r2c( - &self, - dim: impl IntList, - normalization: i64, - onesided: bool, - ) -> Tensor { - self.f_internal_fft_r2c(dim, normalization, onesided).unwrap() - } - - pub fn internal_fft_r2c_out( - &self, - out: &Tensor, - dim: impl IntList, - normalization: i64, - onesided: bool, - ) -> Tensor { - self.f_internal_fft_r2c_out(out, dim, normalization, onesided).unwrap() - } - - pub fn internal_fill_mem_eff_dropout_mask_( - &mut self, - dropout_p: f64, - seed: i64, - offset: i64, - ) -> Tensor { - self.f_internal_fill_mem_eff_dropout_mask_(dropout_p, seed, offset).unwrap() - } - - pub fn internal_flash_attention_backward( - grad_out: &Tensor, - query: &Tensor, - key: &Tensor, - value: &Tensor, - out: &Tensor, - logsumexp: &Tensor, - cum_seq_q: &Tensor, - cum_seq_k: &Tensor, - max_q: i64, - max_k: i64, - dropout_p: f64, - is_causal: bool, - rng_state: &Tensor, - unused: &Tensor, - scale: impl Into>, - window_size_left: impl Into>, - window_size_right: impl Into>, - ) -> (Tensor, Tensor, Tensor) { - Tensor::f_internal_flash_attention_backward( - grad_out, - query, - key, - value, - out, - logsumexp, - cum_seq_q, - cum_seq_k, - max_q, - max_k, - dropout_p, - is_causal, - rng_state, - unused, - scale, - window_size_left, - window_size_right, - ) - .unwrap() - } - - pub fn internal_foobar(&self, arg1: bool, arg2: bool, arg3: bool) -> Tensor { - self.f_internal_foobar(arg1, arg2, arg3).unwrap() - } - - pub fn internal_foobar_out(&self, out: &Tensor, arg1: bool, arg2: bool, arg3: bool) -> Tensor { - self.f_internal_foobar_out(out, arg1, arg2, arg3).unwrap() - } - - pub fn internal_functional_assert_async(&self, assert_msg: &str, dep_token: &Tensor) -> Tensor { - self.f_internal_functional_assert_async(assert_msg, dep_token).unwrap() - } - - pub fn internal_functional_assert_scalar>( - self_scalar: S, - assert_msg: &str, - dep_token: &Tensor, - ) -> Tensor { - Tensor::f_internal_functional_assert_scalar(self_scalar, assert_msg, dep_token).unwrap() - } - - pub fn internal_functional_sym_constrain_range>( - size: S, - min: impl Into>, - max: impl Into>, - dep_token: &Tensor, - ) -> Tensor { - Tensor::f_internal_functional_sym_constrain_range(size, min, max, dep_token).unwrap() - } - - pub fn internal_functional_sym_constrain_range_for_size>( - size: S, - min: impl Into>, - max: impl Into>, - dep_token: &Tensor, - ) -> Tensor { - Tensor::f_internal_functional_sym_constrain_range_for_size(size, min, max, dep_token) - .unwrap() - } - - pub fn internal_fused_dropout(&self, p: f64) -> (Tensor, Tensor) { - self.f_internal_fused_dropout(p).unwrap() - } - - pub fn internal_fused_dropout_out( - &self, - out0: &Tensor, - out1: &Tensor, - p: f64, - ) -> (Tensor, Tensor) { - self.f_internal_fused_dropout_out(out0, out1, p).unwrap() - } - - pub fn internal_fused_moving_avg_obs_fq_helper( - &self, - observer_on: &Tensor, - fake_quant_on: &Tensor, - running_min: &Tensor, - running_max: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - averaging_const: f64, - quant_min: i64, - quant_max: i64, - ch_axis: i64, - per_row_fake_quant: bool, - symmetric_quant: bool, - ) -> (Tensor, Tensor) { - self.f_internal_fused_moving_avg_obs_fq_helper( - observer_on, - fake_quant_on, - running_min, - running_max, - scale, - zero_point, - averaging_const, - quant_min, - quant_max, - ch_axis, - per_row_fake_quant, - symmetric_quant, - ) - .unwrap() - } - - pub fn internal_fused_moving_avg_obs_fq_helper_functional( - &self, - observer_on: &Tensor, - fake_quant_on: &Tensor, - running_min: &Tensor, - running_max: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - averaging_const: f64, - quant_min: i64, - quant_max: i64, - ch_axis: i64, - per_row_fake_quant: bool, - symmetric_quant: bool, - ) -> (Tensor, Tensor, Tensor, Tensor, Tensor, Tensor) { - self.f_internal_fused_moving_avg_obs_fq_helper_functional( - observer_on, - fake_quant_on, - running_min, - running_max, - scale, - zero_point, - averaging_const, - quant_min, - quant_max, - ch_axis, - per_row_fake_quant, - symmetric_quant, - ) - .unwrap() - } - - pub fn internal_fused_moving_avg_obs_fq_helper_out( - &self, - out0: &Tensor, - out1: &Tensor, - observer_on: &Tensor, - fake_quant_on: &Tensor, - running_min: &Tensor, - running_max: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - averaging_const: f64, - quant_min: i64, - quant_max: i64, - ch_axis: i64, - per_row_fake_quant: bool, - symmetric_quant: bool, - ) -> (Tensor, Tensor) { - self.f_internal_fused_moving_avg_obs_fq_helper_out( - out0, - out1, - observer_on, - fake_quant_on, - running_min, - running_max, - scale, - zero_point, - averaging_const, - quant_min, - quant_max, - ch_axis, - per_row_fake_quant, - symmetric_quant, - ) - .unwrap() - } - - pub fn internal_fused_rms_norm>( - &self, - normalized_shape: impl IntList, - weight: Option, - eps: impl Into>, - ) -> (Tensor, Tensor) { - self.f_internal_fused_rms_norm(normalized_shape, weight, eps).unwrap() - } - - pub fn internal_fused_sdp_choice>( - query: &Tensor, - key: &Tensor, - value: &Tensor, - attn_mask: Option, - dropout_p: f64, - is_causal: bool, - scale: impl Into>, - enable_gqa: bool, - ) -> i64 { - Tensor::f_internal_fused_sdp_choice( - query, key, value, attn_mask, dropout_p, is_causal, scale, enable_gqa, - ) - .unwrap() - } - - pub fn internal_fw_primal(&self, level: i64) -> Tensor { - self.f_internal_fw_primal(level).unwrap() - } - - pub fn internal_fw_primal_copy(&self, level: i64) -> Tensor { - self.f_internal_fw_primal_copy(level).unwrap() - } - - pub fn internal_fw_primal_copy_out(&self, out: &Tensor, level: i64) -> Tensor { - self.f_internal_fw_primal_copy_out(out, level).unwrap() - } - - pub fn internal_gather_sparse_backward( - &self, - dim: i64, - index: &Tensor, - grad: &Tensor, - ) -> Tensor { - self.f_internal_gather_sparse_backward(dim, index, grad).unwrap() - } - - pub fn internal_grid_sampler_2d_cpu_fallback( - &self, - grid: &Tensor, - interpolation_mode: i64, - padding_mode: i64, - align_corners: bool, - ) -> Tensor { - self.f_internal_grid_sampler_2d_cpu_fallback( - grid, - interpolation_mode, - padding_mode, - align_corners, - ) - .unwrap() - } - - pub fn internal_grid_sampler_2d_cpu_fallback_backward( - &self, - grad_output: &Tensor, - grid: &Tensor, - interpolation_mode: i64, - padding_mode: i64, - align_corners: bool, - ) -> (Tensor, Tensor) { - self.f_internal_grid_sampler_2d_cpu_fallback_backward( - grad_output, - grid, - interpolation_mode, - padding_mode, - align_corners, - ) - .unwrap() - } - - pub fn internal_grid_sampler_2d_cpu_fallback_out( - &self, - out: &Tensor, - grid: &Tensor, - interpolation_mode: i64, - padding_mode: i64, - align_corners: bool, - ) -> Tensor { - self.f_internal_grid_sampler_2d_cpu_fallback_out( - out, - grid, - interpolation_mode, - padding_mode, - align_corners, - ) - .unwrap() - } - - pub fn internal_grouped_mm>( - &self, - mat2: &Tensor, - offs: Option, - bias: Option, - out_dtype: impl Into>, - ) -> Tensor { - self.f_internal_grouped_mm(mat2, offs, bias, out_dtype).unwrap() - } - - pub fn internal_has_compatible_shallow_copy_type(&self, from: &Tensor) -> bool { - self.f_internal_has_compatible_shallow_copy_type(from).unwrap() - } - - pub fn internal_has_same_storage_numel(&self, other: &Tensor) -> bool { - self.f_internal_has_same_storage_numel(other).unwrap() - } - - pub fn internal_histogramdd_bin_edges>( - &self, - bins: impl IntList, - range: impl DoubleList, - weight: Option, - density: bool, - ) -> Vec { - self.f_internal_histogramdd_bin_edges(bins, range, weight, density).unwrap() - } - - pub fn internal_histogramdd_bin_edges_out>( - &self, - out: &[T], - bins: impl IntList, - range: impl DoubleList, - weight: Option, - density: bool, - ) { - self.f_internal_histogramdd_bin_edges_out(out, bins, range, weight, density).unwrap() - } - - pub fn internal_histogramdd_from_bin_cts>( - &self, - bins: impl IntList, - range: impl DoubleList, - weight: Option, - density: bool, - ) -> Tensor { - self.f_internal_histogramdd_from_bin_cts(bins, range, weight, density).unwrap() - } - - pub fn internal_histogramdd_from_bin_cts_out>( - &self, - out: &Tensor, - bins: impl IntList, - range: impl DoubleList, - weight: Option, - density: bool, - ) -> Tensor { - self.f_internal_histogramdd_from_bin_cts_out(out, bins, range, weight, density).unwrap() - } - - pub fn internal_histogramdd_from_bin_tensors>( - &self, - bins: &[T], - weight: Option, - density: bool, - ) -> Tensor { - self.f_internal_histogramdd_from_bin_tensors(bins, weight, density).unwrap() - } - - pub fn internal_histogramdd_from_bin_tensors_out>( - &self, - out: &Tensor, - bins: &[T], - weight: Option, - density: bool, - ) -> Tensor { - self.f_internal_histogramdd_from_bin_tensors_out(out, bins, weight, density).unwrap() - } - - pub fn internal_index_put_impl>( - &self, - indices: &[Option], - values: &Tensor, - accumulate: bool, - unsafe_: bool, - ) -> Tensor { - self.f_internal_index_put_impl(indices, values, accumulate, unsafe_).unwrap() - } - - pub fn internal_index_put_impl_>( - &mut self, - indices: &[Option], - values: &Tensor, - accumulate: bool, - unsafe_: bool, - ) -> Tensor { - self.f_internal_index_put_impl_(indices, values, accumulate, unsafe_).unwrap() - } - - pub fn internal_index_put_impl_out>( - &self, - out: &Tensor, - indices: &[Option], - values: &Tensor, - accumulate: bool, - unsafe_: bool, - ) -> Tensor { - self.f_internal_index_put_impl_out(out, indices, values, accumulate, unsafe_).unwrap() - } - - pub fn internal_indices(&self) -> Tensor { - self.f_internal_indices().unwrap() - } - - pub fn internal_indices_copy(&self) -> Tensor { - self.f_internal_indices_copy().unwrap() - } - - pub fn internal_indices_copy_out(&self, out: &Tensor) -> Tensor { - self.f_internal_indices_copy_out(out).unwrap() - } - - pub fn internal_int_mm(&self, mat2: &Tensor) -> Tensor { - self.f_internal_int_mm(mat2).unwrap() - } - - pub fn internal_int_mm_out(&self, out: &Tensor, mat2: &Tensor) -> Tensor { - self.f_internal_int_mm_out(out, mat2).unwrap() - } - - pub fn internal_is_all_true(&self) -> Tensor { - self.f_internal_is_all_true().unwrap() - } - - pub fn internal_is_any_true(&self) -> Tensor { - self.f_internal_is_any_true().unwrap() - } - - pub fn internal_is_zerotensor(&self) -> bool { - self.f_internal_is_zerotensor().unwrap() - } - - pub fn internal_lazy_clone(&self) -> Tensor { - self.f_internal_lazy_clone().unwrap() - } - - pub fn internal_linalg_check_errors(info: &Tensor, api_name: &str, is_matrix: bool) { - Tensor::f_internal_linalg_check_errors(info, api_name, is_matrix).unwrap() - } - - pub fn internal_linalg_det(a: &Tensor) -> (Tensor, Tensor, Tensor) { - Tensor::f_internal_linalg_det(a).unwrap() - } - - pub fn internal_linalg_det_result( - result: &Tensor, - lu: &Tensor, - pivots: &Tensor, - a: &Tensor, - ) -> (Tensor, Tensor, Tensor) { - Tensor::f_internal_linalg_det_result(result, lu, pivots, a).unwrap() - } - - pub fn internal_linalg_eigh(a: &Tensor, uplo: &str, compute_v: bool) -> (Tensor, Tensor) { - Tensor::f_internal_linalg_eigh(a, uplo, compute_v).unwrap() - } - - pub fn internal_linalg_eigh_eigenvalues( - eigenvalues: &Tensor, - eigenvectors: &Tensor, - a: &Tensor, - uplo: &str, - compute_v: bool, - ) -> (Tensor, Tensor) { - Tensor::f_internal_linalg_eigh_eigenvalues(eigenvalues, eigenvectors, a, uplo, compute_v) - .unwrap() - } - - pub fn internal_linalg_eigvals(&self) -> Tensor { - self.f_internal_linalg_eigvals().unwrap() - } - - pub fn internal_linalg_slogdet(a: &Tensor) -> (Tensor, Tensor, Tensor, Tensor) { - Tensor::f_internal_linalg_slogdet(a).unwrap() - } - - pub fn internal_linalg_slogdet_sign( - sign: &Tensor, - logabsdet: &Tensor, - lu: &Tensor, - pivots: &Tensor, - a: &Tensor, - ) -> (Tensor, Tensor, Tensor, Tensor) { - Tensor::f_internal_linalg_slogdet_sign(sign, logabsdet, lu, pivots, a).unwrap() - } - - pub fn internal_linalg_solve_ex( - a: &Tensor, - b: &Tensor, - left: bool, - check_errors: bool, - ) -> (Tensor, Tensor, Tensor, Tensor) { - Tensor::f_internal_linalg_solve_ex(a, b, left, check_errors).unwrap() - } - - pub fn internal_linalg_solve_ex_result( - result: &Tensor, - lu: &Tensor, - pivots: &Tensor, - info: &Tensor, - a: &Tensor, - b: &Tensor, - left: bool, - check_errors: bool, - ) -> (Tensor, Tensor, Tensor, Tensor) { - Tensor::f_internal_linalg_solve_ex_result( - result, - lu, - pivots, - info, - a, - b, - left, - check_errors, - ) - .unwrap() - } - - pub fn internal_linalg_svd( - a: &Tensor, - full_matrices: bool, - compute_uv: bool, - driver: &str, - ) -> (Tensor, Tensor, Tensor) { - Tensor::f_internal_linalg_svd(a, full_matrices, compute_uv, driver).unwrap() - } - - pub fn internal_linalg_svd_u( - u: &Tensor, - s: &Tensor, - vh: &Tensor, - a: &Tensor, - full_matrices: bool, - compute_uv: bool, - driver: &str, - ) -> (Tensor, Tensor, Tensor) { - Tensor::f_internal_linalg_svd_u(u, s, vh, a, full_matrices, compute_uv, driver).unwrap() - } - - pub fn internal_log_softmax(&self, dim: i64, half_to_float: bool) -> Tensor { - self.f_internal_log_softmax(dim, half_to_float).unwrap() - } - - pub fn internal_log_softmax_backward_data( - grad_output: &Tensor, - output: &Tensor, - dim: i64, - input_dtype: Kind, - ) -> Tensor { - Tensor::f_internal_log_softmax_backward_data(grad_output, output, dim, input_dtype).unwrap() - } - - pub fn internal_log_softmax_backward_data_out( - out: &Tensor, - grad_output: &Tensor, - output: &Tensor, - dim: i64, - input_dtype: Kind, - ) -> Tensor { - Tensor::f_internal_log_softmax_backward_data_out(out, grad_output, output, dim, input_dtype) - .unwrap() - } - - pub fn internal_log_softmax_out(&self, out: &Tensor, dim: i64, half_to_float: bool) -> Tensor { - self.f_internal_log_softmax_out(out, dim, half_to_float).unwrap() - } - - pub fn internal_logcumsumexp(&self, dim: i64) -> Tensor { - self.f_internal_logcumsumexp(dim).unwrap() - } - - pub fn internal_logcumsumexp_out(&self, out: &Tensor, dim: i64) -> Tensor { - self.f_internal_logcumsumexp_out(out, dim).unwrap() - } - - pub fn internal_lstm_mps>( - &self, - hx: &[T], - params: &[T], - has_biases: bool, - num_layers: i64, - dropout: f64, - train: bool, - bidirectional: bool, - batch_first: bool, - ) -> (Tensor, Tensor, Tensor, Tensor, Tensor, Tensor) { - self.f_internal_lstm_mps( - hx, - params, - has_biases, - num_layers, - dropout, - train, - bidirectional, - batch_first, - ) - .unwrap() - } - - pub fn internal_lstm_mps_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - out3: &Tensor, - out4: &Tensor, - out5: &Tensor, - hx: &[T], - params: &[T], - has_biases: bool, - num_layers: i64, - dropout: f64, - train: bool, - bidirectional: bool, - batch_first: bool, - ) -> (Tensor, Tensor, Tensor, Tensor, Tensor, Tensor) { - self.f_internal_lstm_mps_out( - out0, - out1, - out2, - out3, - out4, - out5, - hx, - params, - has_biases, - num_layers, - dropout, - train, - bidirectional, - batch_first, - ) - .unwrap() - } - - pub fn internal_lu_with_info( - &self, - pivot: bool, - check_errors: bool, - ) -> (Tensor, Tensor, Tensor) { - self.f_internal_lu_with_info(pivot, check_errors).unwrap() - } - - pub fn internal_make_dep_token(options: (Kind, Device)) -> Tensor { - Tensor::f_internal_make_dep_token(options).unwrap() - } - - pub fn internal_make_dual(primal: &Tensor, tangent: &Tensor, level: i64) -> Tensor { - Tensor::f_internal_make_dual(primal, tangent, level).unwrap() - } - - pub fn internal_make_dual_copy(primal: &Tensor, tangent: &Tensor, level: i64) -> Tensor { - Tensor::f_internal_make_dual_copy(primal, tangent, level).unwrap() - } - - pub fn internal_make_dual_copy_out( - out: &Tensor, - primal: &Tensor, - tangent: &Tensor, - level: i64, - ) -> Tensor { - Tensor::f_internal_make_dual_copy_out(out, primal, tangent, level).unwrap() - } - - pub fn internal_make_per_channel_quantized_tensor( - &self, - scale: &Tensor, - zero_point: &Tensor, - axis: i64, - ) -> Tensor { - self.f_internal_make_per_channel_quantized_tensor(scale, zero_point, axis).unwrap() - } - - pub fn internal_make_per_channel_quantized_tensor_out( - &self, - out: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - axis: i64, - ) -> Tensor { - self.f_internal_make_per_channel_quantized_tensor_out(out, scale, zero_point, axis).unwrap() - } - - pub fn internal_make_per_tensor_quantized_tensor(&self, scale: f64, zero_point: i64) -> Tensor { - self.f_internal_make_per_tensor_quantized_tensor(scale, zero_point).unwrap() - } - - pub fn internal_make_per_tensor_quantized_tensor_out( - &self, - out: &Tensor, - scale: f64, - zero_point: i64, - ) -> Tensor { - self.f_internal_make_per_tensor_quantized_tensor_out(out, scale, zero_point).unwrap() - } - - pub fn internal_masked_scale(&self, mask: &Tensor, scale: f64) -> Tensor { - self.f_internal_masked_scale(mask, scale).unwrap() - } - - pub fn internal_masked_scale_out(&self, out: &Tensor, mask: &Tensor, scale: f64) -> Tensor { - self.f_internal_masked_scale_out(out, mask, scale).unwrap() - } - - pub fn internal_masked_softmax( - &self, - mask: &Tensor, - dim: impl Into>, - mask_type: impl Into>, - ) -> Tensor { - self.f_internal_masked_softmax(mask, dim, mask_type).unwrap() - } - - pub fn internal_masked_softmax_backward( - grad_output: &Tensor, - output: &Tensor, - mask: &Tensor, - dim: impl Into>, - ) -> Tensor { - Tensor::f_internal_masked_softmax_backward(grad_output, output, mask, dim).unwrap() - } - - pub fn internal_masked_softmax_backward_out( - out: &Tensor, - grad_output: &Tensor, - output: &Tensor, - mask: &Tensor, - dim: impl Into>, - ) -> Tensor { - Tensor::f_internal_masked_softmax_backward_out(out, grad_output, output, mask, dim).unwrap() - } - - pub fn internal_masked_softmax_out( - &self, - out: &Tensor, - mask: &Tensor, - dim: impl Into>, - mask_type: impl Into>, - ) -> Tensor { - self.f_internal_masked_softmax_out(out, mask, dim, mask_type).unwrap() - } - - pub fn internal_mixed_dtypes_linear>( - &self, - weight: &Tensor, - scale: &Tensor, - bias: Option, - activation: &str, - ) -> Tensor { - self.f_internal_mixed_dtypes_linear(weight, scale, bias, activation).unwrap() - } - - pub fn internal_mkldnn_reshape(&self, shape: impl IntList) -> Tensor { - self.f_internal_mkldnn_reshape(shape).unwrap() - } - - pub fn internal_mkldnn_reshape_out(&self, out: &Tensor, shape: impl IntList) -> Tensor { - self.f_internal_mkldnn_reshape_out(out, shape).unwrap() - } - - pub fn internal_mkldnn_transpose(&self, dim0: i64, dim1: i64) -> Tensor { - self.f_internal_mkldnn_transpose(dim0, dim1).unwrap() - } - - pub fn internal_mkldnn_transpose_(&mut self, dim0: i64, dim1: i64) -> Tensor { - self.f_internal_mkldnn_transpose_(dim0, dim1).unwrap() - } - - pub fn internal_mkldnn_transpose_out(&self, out: &Tensor, dim0: i64, dim1: i64) -> Tensor { - self.f_internal_mkldnn_transpose_out(out, dim0, dim1).unwrap() - } - - pub fn internal_mps_convolution>( - &self, - weight: &Tensor, - bias: Option, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Tensor { - self.f_internal_mps_convolution(weight, bias, padding, stride, dilation, groups).unwrap() - } - - pub fn internal_mps_convolution_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Tensor { - self.f_internal_mps_convolution_out(out, weight, bias, padding, stride, dilation, groups) - .unwrap() - } - - pub fn internal_mps_convolution_transpose( - &self, - weight: &Tensor, - padding: impl IntList, - output_padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Tensor { - self.f_internal_mps_convolution_transpose( - weight, - padding, - output_padding, - stride, - dilation, - groups, - ) - .unwrap() - } - - pub fn internal_mps_convolution_transpose_out( - &self, - out: &Tensor, - weight: &Tensor, - padding: impl IntList, - output_padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Tensor { - self.f_internal_mps_convolution_transpose_out( - out, - weight, - padding, - output_padding, - stride, - dilation, - groups, - ) - .unwrap() - } - - pub fn internal_native_batch_norm_legit>( - &self, - weight: Option, - bias: Option, - running_mean: &Tensor, - running_var: &Tensor, - training: bool, - momentum: f64, - eps: f64, - ) -> (Tensor, Tensor, Tensor) { - self.f_internal_native_batch_norm_legit( - weight, - bias, - running_mean, - running_var, - training, - momentum, - eps, - ) - .unwrap() - } - - pub fn internal_native_batch_norm_legit_functional>( - &self, - weight: Option, - bias: Option, - running_mean: &Tensor, - running_var: &Tensor, - training: bool, - momentum: f64, - eps: f64, - ) -> (Tensor, Tensor, Tensor, Tensor, Tensor) { - self.f_internal_native_batch_norm_legit_functional( - weight, - bias, - running_mean, - running_var, - training, - momentum, - eps, - ) - .unwrap() - } - - pub fn internal_native_batch_norm_legit_no_stats>( - &self, - weight: Option, - bias: Option, - training: bool, - momentum: f64, - eps: f64, - ) -> (Tensor, Tensor, Tensor) { - self.f_internal_native_batch_norm_legit_no_stats(weight, bias, training, momentum, eps) - .unwrap() - } - - pub fn internal_native_batch_norm_legit_no_stats_out>( - &self, - out: &Tensor, - save_mean: &Tensor, - save_invstd: &Tensor, - weight: Option, - bias: Option, - training: bool, - momentum: f64, - eps: f64, - ) -> (Tensor, Tensor, Tensor) { - self.f_internal_native_batch_norm_legit_no_stats_out( - out, - save_mean, - save_invstd, - weight, - bias, - training, - momentum, - eps, - ) - .unwrap() - } - - pub fn internal_native_batch_norm_legit_no_training>( - &self, - weight: Option, - bias: Option, - running_mean: &Tensor, - running_var: &Tensor, - momentum: f64, - eps: f64, - ) -> (Tensor, Tensor, Tensor) { - self.f_internal_native_batch_norm_legit_no_training( - weight, - bias, - running_mean, - running_var, - momentum, - eps, - ) - .unwrap() - } - - pub fn internal_native_batch_norm_legit_no_training_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - weight: Option, - bias: Option, - running_mean: &Tensor, - running_var: &Tensor, - momentum: f64, - eps: f64, - ) -> (Tensor, Tensor, Tensor) { - self.f_internal_native_batch_norm_legit_no_training_out( - out0, - out1, - out2, - weight, - bias, - running_mean, - running_var, - momentum, - eps, - ) - .unwrap() - } - - pub fn internal_native_batch_norm_legit_out>( - &self, - out: &Tensor, - save_mean: &Tensor, - save_invstd: &Tensor, - weight: Option, - bias: Option, - running_mean: &Tensor, - running_var: &Tensor, - training: bool, - momentum: f64, - eps: f64, - ) -> (Tensor, Tensor, Tensor) { - self.f_internal_native_batch_norm_legit_out( - out, - save_mean, - save_invstd, - weight, - bias, - running_mean, - running_var, - training, - momentum, - eps, - ) - .unwrap() - } - - pub fn internal_native_multi_head_attention>( - query: &Tensor, - key: &Tensor, - value: &Tensor, - embed_dim: i64, - num_head: i64, - qkv_weight: &Tensor, - qkv_bias: &Tensor, - proj_weight: &Tensor, - proj_bias: &Tensor, - mask: Option, - need_weights: bool, - average_attn_weights: bool, - mask_type: impl Into>, - ) -> (Tensor, Tensor) { - Tensor::f_internal_native_multi_head_attention( - query, - key, - value, - embed_dim, - num_head, - qkv_weight, - qkv_bias, - proj_weight, - proj_bias, - mask, - need_weights, - average_attn_weights, - mask_type, - ) - .unwrap() - } - - pub fn internal_native_multi_head_attention_out>( - out0: &Tensor, - out1: &Tensor, - query: &Tensor, - key: &Tensor, - value: &Tensor, - embed_dim: i64, - num_head: i64, - qkv_weight: &Tensor, - qkv_bias: &Tensor, - proj_weight: &Tensor, - proj_bias: &Tensor, - mask: Option, - need_weights: bool, - average_attn_weights: bool, - mask_type: impl Into>, - ) -> (Tensor, Tensor) { - Tensor::f_internal_native_multi_head_attention_out( - out0, - out1, - query, - key, - value, - embed_dim, - num_head, - qkv_weight, - qkv_bias, - proj_weight, - proj_bias, - mask, - need_weights, - average_attn_weights, - mask_type, - ) - .unwrap() - } - - pub fn internal_neg_view(&self) -> Tensor { - self.f_internal_neg_view().unwrap() - } - - pub fn internal_neg_view_copy(&self) -> Tensor { - self.f_internal_neg_view_copy().unwrap() - } - - pub fn internal_neg_view_copy_out(&self, out: &Tensor) -> Tensor { - self.f_internal_neg_view_copy_out(out).unwrap() - } - - pub fn internal_nested_compute_contiguous_strides_offsets( - nested_size: &Tensor, - ) -> (Tensor, Tensor) { - Tensor::f_internal_nested_compute_contiguous_strides_offsets(nested_size).unwrap() - } - - pub fn internal_nested_from_padded( - padded: &Tensor, - cpu_nested_shape_example: &Tensor, - fuse_transform_0213: bool, - ) -> Tensor { - Tensor::f_internal_nested_from_padded(padded, cpu_nested_shape_example, fuse_transform_0213) - .unwrap() - } - - pub fn internal_nested_from_padded_and_nested_example( - padded: &Tensor, - nt_example: &Tensor, - ) -> Tensor { - Tensor::f_internal_nested_from_padded_and_nested_example(padded, nt_example).unwrap() - } - - pub fn internal_nested_from_padded_and_nested_example_out( - out: &Tensor, - padded: &Tensor, - nt_example: &Tensor, - ) -> Tensor { - Tensor::f_internal_nested_from_padded_and_nested_example_out(out, padded, nt_example) - .unwrap() - } - - pub fn internal_nested_from_padded_out( - out: &Tensor, - padded: &Tensor, - cpu_nested_shape_example: &Tensor, - fuse_transform_0213: bool, - ) -> Tensor { - Tensor::f_internal_nested_from_padded_out( - out, - padded, - cpu_nested_shape_example, - fuse_transform_0213, - ) - .unwrap() - } - - pub fn internal_nested_from_padded_tensor>( - padded: &Tensor, - offsets: &Tensor, - dummy: &Tensor, - ragged_idx: i64, - min_seqlen: Option, - max_seqlen: Option, - sum_s: impl Into>, - ) -> Tensor { - Tensor::f_internal_nested_from_padded_tensor( - padded, offsets, dummy, ragged_idx, min_seqlen, max_seqlen, sum_s, - ) - .unwrap() - } - - pub fn internal_nested_get_jagged_dummy(any: &Tensor) -> Tensor { - Tensor::f_internal_nested_get_jagged_dummy(any).unwrap() - } - - pub fn internal_nested_get_lengths(&self) -> Tensor { - self.f_internal_nested_get_lengths().unwrap() - } - - pub fn internal_nested_get_max_seqlen(&self) -> Tensor { - self.f_internal_nested_get_max_seqlen().unwrap() - } - - pub fn internal_nested_get_min_seqlen(&self) -> Tensor { - self.f_internal_nested_get_min_seqlen().unwrap() - } - - pub fn internal_nested_get_offsets(&self) -> Tensor { - self.f_internal_nested_get_offsets().unwrap() - } - - pub fn internal_nested_get_ragged_idx(&self) -> i64 { - self.f_internal_nested_get_ragged_idx().unwrap() - } - - pub fn internal_nested_get_values(&self) -> Tensor { - self.f_internal_nested_get_values().unwrap() - } - - pub fn internal_nested_get_values_copy(&self) -> Tensor { - self.f_internal_nested_get_values_copy().unwrap() - } - - pub fn internal_nested_get_values_copy_out(&self, out: &Tensor) -> Tensor { - self.f_internal_nested_get_values_copy_out(out).unwrap() - } - - pub fn internal_nested_select_backward( - &self, - grad_output: &Tensor, - dim: i64, - index: i64, - ) -> Tensor { - self.f_internal_nested_select_backward(grad_output, dim, index).unwrap() - } - - pub fn internal_nested_sum_backward( - &self, - grad: &Tensor, - dim: impl IntListOption, - keepdim: bool, - ) -> Tensor { - self.f_internal_nested_sum_backward(grad, dim, keepdim).unwrap() - } - - pub fn internal_nested_view_from_buffer( - &self, - nested_size: &Tensor, - nested_strides: &Tensor, - offsets: &Tensor, - ) -> Tensor { - self.f_internal_nested_view_from_buffer(nested_size, nested_strides, offsets).unwrap() - } - - pub fn internal_nested_view_from_buffer_copy( - &self, - nested_size: &Tensor, - nested_strides: &Tensor, - offsets: &Tensor, - ) -> Tensor { - self.f_internal_nested_view_from_buffer_copy(nested_size, nested_strides, offsets).unwrap() - } - - pub fn internal_nested_view_from_buffer_copy_out( - &self, - out: &Tensor, - nested_size: &Tensor, - nested_strides: &Tensor, - offsets: &Tensor, - ) -> Tensor { - self.f_internal_nested_view_from_buffer_copy_out(out, nested_size, nested_strides, offsets) - .unwrap() - } - - pub fn internal_nested_view_from_jagged>( - &self, - offsets: &Tensor, - dummy: &Tensor, - lengths: Option, - ragged_idx: i64, - min_seqlen: Option, - max_seqlen: Option, - ) -> Tensor { - self.f_internal_nested_view_from_jagged( - offsets, dummy, lengths, ragged_idx, min_seqlen, max_seqlen, - ) - .unwrap() - } - - pub fn internal_nested_view_from_jagged_copy>( - &self, - offsets: &Tensor, - dummy: &Tensor, - lengths: Option, - ragged_idx: i64, - min_seqlen: Option, - max_seqlen: Option, - ) -> Tensor { - self.f_internal_nested_view_from_jagged_copy( - offsets, dummy, lengths, ragged_idx, min_seqlen, max_seqlen, - ) - .unwrap() - } - - pub fn internal_nested_view_from_jagged_copy_out>( - &self, - out: &Tensor, - offsets: &Tensor, - dummy: &Tensor, - lengths: Option, - ragged_idx: i64, - min_seqlen: Option, - max_seqlen: Option, - ) -> Tensor { - self.f_internal_nested_view_from_jagged_copy_out( - out, offsets, dummy, lengths, ragged_idx, min_seqlen, max_seqlen, - ) - .unwrap() - } - - pub fn internal_new_zeros_with_same_feature_meta( - &self, - other: &Tensor, - self_num_batch_dims: i64, - ) -> Tensor { - self.f_internal_new_zeros_with_same_feature_meta(other, self_num_batch_dims).unwrap() - } - - pub fn internal_new_zeros_with_same_feature_meta_out( - &self, - out: &Tensor, - other: &Tensor, - self_num_batch_dims: i64, - ) -> Tensor { - self.f_internal_new_zeros_with_same_feature_meta_out(out, other, self_num_batch_dims) - .unwrap() - } - - pub fn internal_nnpack_available() -> bool { - Tensor::f_internal_nnpack_available().unwrap() - } - - pub fn internal_nnpack_spatial_convolution>( - &self, - weight: &Tensor, - bias: Option, - padding: impl IntList, - stride: impl IntList, - ) -> Tensor { - self.f_internal_nnpack_spatial_convolution(weight, bias, padding, stride).unwrap() - } - - pub fn internal_nnpack_spatial_convolution_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - padding: impl IntList, - stride: impl IntList, - ) -> Tensor { - self.f_internal_nnpack_spatial_convolution_out(out, weight, bias, padding, stride).unwrap() - } - - pub fn internal_nnz(&self) -> i64 { - self.f_internal_nnz().unwrap() - } - - pub fn internal_pack_padded_sequence( - &self, - lengths: &Tensor, - batch_first: bool, - ) -> (Tensor, Tensor) { - self.f_internal_pack_padded_sequence(lengths, batch_first).unwrap() - } - - pub fn internal_pack_padded_sequence_backward( - grad: &Tensor, - input_size: impl IntList, - batch_sizes: &Tensor, - batch_first: bool, - ) -> Tensor { - Tensor::f_internal_pack_padded_sequence_backward(grad, input_size, batch_sizes, batch_first) - .unwrap() - } - - pub fn internal_pack_padded_sequence_out( - &self, - out0: &Tensor, - out1: &Tensor, - lengths: &Tensor, - batch_first: bool, - ) -> (Tensor, Tensor) { - self.f_internal_pack_padded_sequence_out(out0, out1, lengths, batch_first).unwrap() - } - - pub fn internal_pad_circular(&self, pad: impl IntList) -> Tensor { - self.f_internal_pad_circular(pad).unwrap() - } - - pub fn internal_pad_enum( - &self, - pad: impl IntList, - mode: i64, - value: impl Into>, - ) -> Tensor { - self.f_internal_pad_enum(pad, mode, value).unwrap() - } - - pub fn internal_pad_packed_sequence>( - data: &Tensor, - batch_sizes: &Tensor, - batch_first: bool, - padding_value: S, - total_length: i64, - ) -> (Tensor, Tensor) { - Tensor::f_internal_pad_packed_sequence( - data, - batch_sizes, - batch_first, - padding_value, - total_length, - ) - .unwrap() - } - - pub fn internal_pdist_backward(&self, grad: &Tensor, p: f64, pdist: &Tensor) -> Tensor { - self.f_internal_pdist_backward(grad, p, pdist).unwrap() - } - - pub fn internal_pdist_backward_out( - &self, - out: &Tensor, - grad: &Tensor, - p: f64, - pdist: &Tensor, - ) -> Tensor { - self.f_internal_pdist_backward_out(out, grad, p, pdist).unwrap() - } - - pub fn internal_pin_memory(&self, device: Device) -> Tensor { - self.f_internal_pin_memory(device).unwrap() - } - - pub fn internal_pin_memory_out(&self, out: &Tensor, device: Device) -> Tensor { - self.f_internal_pin_memory_out(out, device).unwrap() - } - - pub fn internal_prelu_kernel(&self, weight: &Tensor) -> Tensor { - self.f_internal_prelu_kernel(weight).unwrap() - } - - pub fn internal_prelu_kernel_backward( - &self, - grad_output: &Tensor, - weight: &Tensor, - ) -> (Tensor, Tensor) { - self.f_internal_prelu_kernel_backward(grad_output, weight).unwrap() - } - - pub fn internal_print(s: &str) { - Tensor::f_internal_print(s).unwrap() - } - - pub fn internal_propagate_xla_data(&self, output: &Tensor) { - self.f_internal_propagate_xla_data(output).unwrap() - } - - pub fn internal_remove_batch_dim(&self, level: i64, batch_size: i64, out_dim: i64) -> Tensor { - self.f_internal_remove_batch_dim(level, batch_size, out_dim).unwrap() - } - - pub fn internal_reshape_alias(&self, size: impl IntList, stride: impl IntList) -> Tensor { - self.f_internal_reshape_alias(size, stride).unwrap() - } - - pub fn internal_reshape_alias_copy(&self, size: impl IntList, stride: impl IntList) -> Tensor { - self.f_internal_reshape_alias_copy(size, stride).unwrap() - } - - pub fn internal_reshape_alias_copy_out( - &self, - out: &Tensor, - size: impl IntList, - stride: impl IntList, - ) -> Tensor { - self.f_internal_reshape_alias_copy_out(out, size, stride).unwrap() - } - - pub fn internal_reshape_copy(&self, size: impl IntList) -> Tensor { - self.f_internal_reshape_copy(size).unwrap() - } - - pub fn internal_reshape_from_tensor(&self, shape: &Tensor) -> Tensor { - self.f_internal_reshape_from_tensor(shape).unwrap() - } - - pub fn internal_resize_output(&self, size: impl IntList, device: Device) -> Tensor { - self.f_internal_resize_output(size, device).unwrap() - } - - pub fn internal_resize_output_(&mut self, size: impl IntList, device: Device) -> Tensor { - self.f_internal_resize_output_(size, device).unwrap() - } - - pub fn internal_resize_output_out( - &self, - out: &Tensor, - size: impl IntList, - device: Device, - ) -> Tensor { - self.f_internal_resize_output_out(out, size, device).unwrap() - } - - pub fn internal_rowwise_prune( - weight: &Tensor, - mask: &Tensor, - compressed_indices_dtype: Kind, - ) -> (Tensor, Tensor) { - Tensor::f_internal_rowwise_prune(weight, mask, compressed_indices_dtype).unwrap() - } - - pub fn internal_safe_softmax(&self, dim: i64, dtype: impl Into>) -> Tensor { - self.f_internal_safe_softmax(dim, dtype).unwrap() - } - - pub fn internal_sample_dirichlet(&self) -> Tensor { - self.f_internal_sample_dirichlet().unwrap() - } - - pub fn internal_sample_dirichlet_out(&self, out: &Tensor) -> Tensor { - self.f_internal_sample_dirichlet_out(out).unwrap() - } - - pub fn internal_saturate_weight_to_fp16(weight: &Tensor) -> Tensor { - Tensor::f_internal_saturate_weight_to_fp16(weight).unwrap() - } - - pub fn internal_scaled_dot_product_attention_math>( - query: &Tensor, - key: &Tensor, - value: &Tensor, - attn_mask: Option, - dropout_p: f64, - is_causal: bool, - dropout_mask: Option, - scale: impl Into>, - enable_gqa: bool, - ) -> (Tensor, Tensor) { - Tensor::f_internal_scaled_dot_product_attention_math( - query, - key, - value, - attn_mask, - dropout_p, - is_causal, - dropout_mask, - scale, - enable_gqa, - ) - .unwrap() - } - - pub fn internal_scaled_dot_product_attention_math_for_mps>( - query: &Tensor, - key: &Tensor, - value: &Tensor, - attn_mask: Option, - dropout_p: f64, - is_causal: bool, - dropout_mask: Option, - scale: impl Into>, - ) -> (Tensor, Tensor) { - Tensor::f_internal_scaled_dot_product_attention_math_for_mps( - query, - key, - value, - attn_mask, - dropout_p, - is_causal, - dropout_mask, - scale, - ) - .unwrap() - } - - pub fn internal_scaled_dot_product_cudnn_attention_backward( - grad_out: &Tensor, - query: &Tensor, - key: &Tensor, - value: &Tensor, - out: &Tensor, - logsumexp: &Tensor, - philox_seed: &Tensor, - philox_offset: &Tensor, - attn_bias: &Tensor, - cum_seq_q: &Tensor, - cum_seq_k: &Tensor, - max_q: i64, - max_k: i64, - dropout_p: f64, - is_causal: bool, - scale: impl Into>, - ) -> (Tensor, Tensor, Tensor) { - Tensor::f_internal_scaled_dot_product_cudnn_attention_backward( - grad_out, - query, - key, - value, - out, - logsumexp, - philox_seed, - philox_offset, - attn_bias, - cum_seq_q, - cum_seq_k, - max_q, - max_k, - dropout_p, - is_causal, - scale, - ) - .unwrap() - } - - pub fn internal_scaled_dot_product_efficient_attention>( - query: &Tensor, - key: &Tensor, - value: &Tensor, - attn_bias: Option, - compute_log_sumexp: bool, - dropout_p: f64, - is_causal: bool, - scale: impl Into>, - ) -> (Tensor, Tensor, Tensor, Tensor) { - Tensor::f_internal_scaled_dot_product_efficient_attention( - query, - key, - value, - attn_bias, - compute_log_sumexp, - dropout_p, - is_causal, - scale, - ) - .unwrap() - } - - pub fn internal_scaled_dot_product_flash_attention_backward( - grad_out: &Tensor, - query: &Tensor, - key: &Tensor, - value: &Tensor, - out: &Tensor, - logsumexp: &Tensor, - cum_seq_q: &Tensor, - cum_seq_k: &Tensor, - max_q: i64, - max_k: i64, - dropout_p: f64, - is_causal: bool, - philox_seed: &Tensor, - philox_offset: &Tensor, - scale: impl Into>, - ) -> (Tensor, Tensor, Tensor) { - Tensor::f_internal_scaled_dot_product_flash_attention_backward( - grad_out, - query, - key, - value, - out, - logsumexp, - cum_seq_q, - cum_seq_k, - max_q, - max_k, - dropout_p, - is_causal, - philox_seed, - philox_offset, - scale, - ) - .unwrap() - } - - pub fn internal_scaled_dot_product_flash_attention_for_cpu>( - query: &Tensor, - key: &Tensor, - value: &Tensor, - dropout_p: f64, - is_causal: bool, - attn_mask: Option, - scale: impl Into>, - ) -> (Tensor, Tensor) { - Tensor::f_internal_scaled_dot_product_flash_attention_for_cpu( - query, key, value, dropout_p, is_causal, attn_mask, scale, - ) - .unwrap() - } - - pub fn internal_scaled_dot_product_flash_attention_for_cpu_backward>( - grad_out: &Tensor, - query: &Tensor, - key: &Tensor, - value: &Tensor, - out: &Tensor, - logsumexp: &Tensor, - dropout_p: f64, - is_causal: bool, - attn_mask: Option, - scale: impl Into>, - ) -> (Tensor, Tensor, Tensor) { - Tensor::f_internal_scaled_dot_product_flash_attention_for_cpu_backward( - grad_out, query, key, value, out, logsumexp, dropout_p, is_causal, attn_mask, scale, - ) - .unwrap() - } - - pub fn internal_scaled_grouped_mm>( - &self, - mat2: &Tensor, - scale_a: &Tensor, - scale_b: &Tensor, - offs: Option, - bias: Option, - scale_result: Option, - out_dtype: impl Into>, - use_fast_accum: bool, - ) -> Tensor { - self.f_internal_scaled_grouped_mm( - mat2, - scale_a, - scale_b, - offs, - bias, - scale_result, - out_dtype, - use_fast_accum, - ) - .unwrap() - } - - pub fn internal_scaled_grouped_mm_v2>( - &self, - mat2: &Tensor, - scale_a: &[T], - recipe_a: impl IntList, - swizzle_a: impl IntList, - scale_b: &[T], - recipe_b: impl IntList, - swizzle_b: impl IntList, - offs: Option, - bias: Option, - out_dtype: impl Into>, - contraction_dim: impl IntList, - use_fast_accum: bool, - ) -> Tensor { - self.f_internal_scaled_grouped_mm_v2( - mat2, - scale_a, - recipe_a, - swizzle_a, - scale_b, - recipe_b, - swizzle_b, - offs, - bias, - out_dtype, - contraction_dim, - use_fast_accum, - ) - .unwrap() - } - - pub fn internal_scaled_mm>( - &self, - mat2: &Tensor, - scale_a: &Tensor, - scale_b: &Tensor, - bias: Option, - scale_result: Option, - out_dtype: impl Into>, - use_fast_accum: bool, - ) -> Tensor { - self.f_internal_scaled_mm( - mat2, - scale_a, - scale_b, - bias, - scale_result, - out_dtype, - use_fast_accum, - ) - .unwrap() - } - - pub fn internal_scaled_mm_out>( - &self, - out: &Tensor, - mat2: &Tensor, - scale_a: &Tensor, - scale_b: &Tensor, - bias: Option, - scale_result: Option, - out_dtype: impl Into>, - use_fast_accum: bool, - ) -> Tensor { - self.f_internal_scaled_mm_out( - out, - mat2, - scale_a, - scale_b, - bias, - scale_result, - out_dtype, - use_fast_accum, - ) - .unwrap() - } - - pub fn internal_scaled_mm_v2>( - &self, - mat2: &Tensor, - scale_a: &[T], - recipe_a: impl IntList, - swizzle_a: impl IntList, - scale_b: &[T], - recipe_b: impl IntList, - swizzle_b: impl IntList, - bias: Option, - out_dtype: impl Into>, - contraction_dim: impl IntList, - use_fast_accum: bool, - ) -> Tensor { - self.f_internal_scaled_mm_v2( - mat2, - scale_a, - recipe_a, - swizzle_a, - scale_b, - recipe_b, - swizzle_b, - bias, - out_dtype, - contraction_dim, - use_fast_accum, - ) - .unwrap() - } - - pub fn internal_scaled_mm_v2_out>( - &self, - out: &Tensor, - mat2: &Tensor, - scale_a: &[T], - recipe_a: impl IntList, - swizzle_a: impl IntList, - scale_b: &[T], - recipe_b: impl IntList, - swizzle_b: impl IntList, - bias: Option, - out_dtype: impl Into>, - contraction_dim: impl IntList, - use_fast_accum: bool, - ) -> Tensor { - self.f_internal_scaled_mm_v2_out( - out, - mat2, - scale_a, - recipe_a, - swizzle_a, - scale_b, - recipe_b, - swizzle_b, - bias, - out_dtype, - contraction_dim, - use_fast_accum, - ) - .unwrap() - } - - pub fn internal_scatter_reduce( - &self, - dim: i64, - index: &Tensor, - src: &Tensor, - reduce: &str, - include_self: bool, - ) -> Tensor { - self.f_internal_scatter_reduce(dim, index, src, reduce, include_self).unwrap() - } - - pub fn internal_scatter_reduce_( - &mut self, - dim: i64, - index: &Tensor, - src: &Tensor, - reduce: &str, - include_self: bool, - ) -> Tensor { - self.f_internal_scatter_reduce_(dim, index, src, reduce, include_self).unwrap() - } - - pub fn internal_scatter_reduce_two_out( - &self, - out: &Tensor, - dim: i64, - index: &Tensor, - src: &Tensor, - reduce: &str, - include_self: bool, - ) -> Tensor { - self.f_internal_scatter_reduce_two_out(out, dim, index, src, reduce, include_self).unwrap() - } - - pub fn internal_segment_reduce_backward, S: Into>( - grad: &Tensor, - output: &Tensor, - data: &Tensor, - reduce: &str, - lengths: Option, - offsets: Option, - axis: i64, - initial: S, - ) -> Tensor { - Tensor::f_internal_segment_reduce_backward( - grad, output, data, reduce, lengths, offsets, axis, initial, - ) - .unwrap() - } - - pub fn internal_segment_reduce_backward_out, S: Into>( - out: &Tensor, - grad: &Tensor, - output: &Tensor, - data: &Tensor, - reduce: &str, - lengths: Option, - offsets: Option, - axis: i64, - initial: S, - ) -> Tensor { - Tensor::f_internal_segment_reduce_backward_out( - out, grad, output, data, reduce, lengths, offsets, axis, initial, - ) - .unwrap() - } - - pub fn internal_shape_as_tensor(&self) -> Tensor { - self.f_internal_shape_as_tensor().unwrap() - } - - pub fn internal_slow_conv2d_backward( - &self, - grad_input: &Tensor, - grad_weight: &Tensor, - grad_bias: &Tensor, - grad_output: &Tensor, - weight: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ) -> (Tensor, Tensor, Tensor) { - self.f_internal_slow_conv2d_backward( - grad_input, - grad_weight, - grad_bias, - grad_output, - weight, - kernel_size, - stride, - padding, - ) - .unwrap() - } - - pub fn internal_sobol_engine_draw( - quasi: &Tensor, - n: i64, - sobolstate: &Tensor, - dimension: i64, - num_generated: i64, - dtype: impl Into>, - ) -> (Tensor, Tensor) { - Tensor::f_internal_sobol_engine_draw(quasi, n, sobolstate, dimension, num_generated, dtype) - .unwrap() - } - - pub fn internal_sobol_engine_ff_( - &mut self, - n: i64, - sobolstate: &Tensor, - dimension: i64, - num_generated: i64, - ) -> Tensor { - self.f_internal_sobol_engine_ff_(n, sobolstate, dimension, num_generated).unwrap() - } - - pub fn internal_sobol_engine_initialize_state_(&mut self, dimension: i64) -> Tensor { - self.f_internal_sobol_engine_initialize_state_(dimension).unwrap() - } - - pub fn internal_sobol_engine_scramble_(&mut self, ltm: &Tensor, dimension: i64) -> Tensor { - self.f_internal_sobol_engine_scramble_(ltm, dimension).unwrap() - } - - pub fn internal_softmax(&self, dim: i64, half_to_float: bool) -> Tensor { - self.f_internal_softmax(dim, half_to_float).unwrap() - } - - pub fn internal_softmax_backward_data( - grad_output: &Tensor, - output: &Tensor, - dim: i64, - input_dtype: Kind, - ) -> Tensor { - Tensor::f_internal_softmax_backward_data(grad_output, output, dim, input_dtype).unwrap() - } - - pub fn internal_softmax_backward_data_out( - grad_input: &Tensor, - grad_output: &Tensor, - output: &Tensor, - dim: i64, - input_dtype: Kind, - ) -> Tensor { - Tensor::f_internal_softmax_backward_data_out( - grad_input, - grad_output, - output, - dim, - input_dtype, - ) - .unwrap() - } - - pub fn internal_softmax_out(&self, out: &Tensor, dim: i64, half_to_float: bool) -> Tensor { - self.f_internal_softmax_out(out, dim, half_to_float).unwrap() - } - - pub fn internal_sparse_addmm(&self, mat1: &Tensor, mat2: &Tensor) -> Tensor { - self.f_internal_sparse_addmm(mat1, mat2).unwrap() - } - - pub fn internal_sparse_addmm_out(&self, out: &Tensor, mat1: &Tensor, mat2: &Tensor) -> Tensor { - self.f_internal_sparse_addmm_out(out, mat1, mat2).unwrap() - } - - pub fn internal_sparse_broadcast_to(&self, size: impl IntList) -> Tensor { - self.f_internal_sparse_broadcast_to(size).unwrap() - } - - pub fn internal_sparse_broadcast_to_copy(&self, size: impl IntList) -> Tensor { - self.f_internal_sparse_broadcast_to_copy(size).unwrap() - } - - pub fn internal_sparse_broadcast_to_copy_out( - &self, - out: &Tensor, - size: impl IntList, - ) -> Tensor { - self.f_internal_sparse_broadcast_to_copy_out(out, size).unwrap() - } - - pub fn internal_sparse_bsc_tensor_unsafe( - ccol_indices: &Tensor, - row_indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_internal_sparse_bsc_tensor_unsafe( - ccol_indices, - row_indices, - values, - size, - options, - ) - .unwrap() - } - - pub fn internal_sparse_bsr_tensor_unsafe( - crow_indices: &Tensor, - col_indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_internal_sparse_bsr_tensor_unsafe( - crow_indices, - col_indices, - values, - size, - options, - ) - .unwrap() - } - - pub fn internal_sparse_compressed_tensor_unsafe( - compressed_indices: &Tensor, - plain_indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_internal_sparse_compressed_tensor_unsafe( - compressed_indices, - plain_indices, - values, - size, - options, - ) - .unwrap() - } - - pub fn internal_sparse_compressed_tensor_with_dims( - nnz: i64, - dense_dim: i64, - size: impl IntList, - blocksize: impl IntList, - index_dtype: Kind, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_internal_sparse_compressed_tensor_with_dims( - nnz, - dense_dim, - size, - blocksize, - index_dtype, - options, - ) - .unwrap() - } - - pub fn internal_sparse_coo_tensor_unsafe( - indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - is_coalesced: bool, - ) -> Tensor { - Tensor::f_internal_sparse_coo_tensor_unsafe(indices, values, size, options, is_coalesced) - .unwrap() - } - - pub fn internal_sparse_coo_tensor_with_dims( - sparse_dim: i64, - dense_dim: i64, - size: impl IntList, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_internal_sparse_coo_tensor_with_dims(sparse_dim, dense_dim, size, options) - .unwrap() - } - - pub fn internal_sparse_coo_tensor_with_dims_and_tensors( - sparse_dim: i64, - dense_dim: i64, - size: impl IntList, - indices: &Tensor, - values: &Tensor, - options: (Kind, Device), - is_coalesced: bool, - ) -> Tensor { - Tensor::f_internal_sparse_coo_tensor_with_dims_and_tensors( - sparse_dim, - dense_dim, - size, - indices, - values, - options, - is_coalesced, - ) - .unwrap() - } - - pub fn internal_sparse_coo_tensor_with_dims_and_tensors_out( - out: &Tensor, - sparse_dim: i64, - dense_dim: i64, - size: impl IntList, - indices: &Tensor, - values: &Tensor, - is_coalesced: bool, - ) -> Tensor { - Tensor::f_internal_sparse_coo_tensor_with_dims_and_tensors_out( - out, - sparse_dim, - dense_dim, - size, - indices, - values, - is_coalesced, - ) - .unwrap() - } - - pub fn internal_sparse_coo_tensor_with_dims_out( - out: &Tensor, - sparse_dim: i64, - dense_dim: i64, - size: impl IntList, - ) -> Tensor { - Tensor::f_internal_sparse_coo_tensor_with_dims_out(out, sparse_dim, dense_dim, size) - .unwrap() - } - - pub fn internal_sparse_csc_tensor_unsafe( - ccol_indices: &Tensor, - row_indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_internal_sparse_csc_tensor_unsafe( - ccol_indices, - row_indices, - values, - size, - options, - ) - .unwrap() - } - - pub fn internal_sparse_csr_prod( - &self, - dim: impl IntList, - keepdim: bool, - dtype: impl Into>, - ) -> Tensor { - self.f_internal_sparse_csr_prod(dim, keepdim, dtype).unwrap() - } - - pub fn internal_sparse_csr_prod_dim_dtype_out( - &self, - out: &Tensor, - dim: impl IntList, - keepdim: bool, - dtype: impl Into>, - ) -> Tensor { - self.f_internal_sparse_csr_prod_dim_dtype_out(out, dim, keepdim, dtype).unwrap() - } - - pub fn internal_sparse_csr_sum( - &self, - dim: impl IntList, - keepdim: bool, - dtype: impl Into>, - ) -> Tensor { - self.f_internal_sparse_csr_sum(dim, keepdim, dtype).unwrap() - } - - pub fn internal_sparse_csr_sum_dim_dtype_out( - &self, - out: &Tensor, - dim: impl IntList, - keepdim: bool, - dtype: impl Into>, - ) -> Tensor { - self.f_internal_sparse_csr_sum_dim_dtype_out(out, dim, keepdim, dtype).unwrap() - } - - pub fn internal_sparse_csr_tensor_unsafe( - crow_indices: &Tensor, - col_indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_internal_sparse_csr_tensor_unsafe( - crow_indices, - col_indices, - values, - size, - options, - ) - .unwrap() - } - - pub fn internal_sparse_log_softmax(&self, dim: i64, half_to_float: bool) -> Tensor { - self.f_internal_sparse_log_softmax(dim, half_to_float).unwrap() - } - - pub fn internal_sparse_log_softmax_backward_data( - &self, - grad_output: &Tensor, - output: &Tensor, - dim: i64, - ) -> Tensor { - self.f_internal_sparse_log_softmax_backward_data(grad_output, output, dim).unwrap() - } - - pub fn internal_sparse_log_softmax_backward_data_out( - &self, - out: &Tensor, - grad_output: &Tensor, - output: &Tensor, - dim: i64, - ) -> Tensor { - self.f_internal_sparse_log_softmax_backward_data_out(out, grad_output, output, dim).unwrap() - } - - pub fn internal_sparse_log_softmax_int( - &self, - dim: i64, - dtype: impl Into>, - ) -> Tensor { - self.f_internal_sparse_log_softmax_int(dim, dtype).unwrap() - } - - pub fn internal_sparse_log_softmax_out( - &self, - out: &Tensor, - dim: i64, - half_to_float: bool, - ) -> Tensor { - self.f_internal_sparse_log_softmax_out(out, dim, half_to_float).unwrap() - } - - pub fn internal_sparse_mask_projection( - &self, - mask: &Tensor, - accumulate_matches: bool, - ) -> Tensor { - self.f_internal_sparse_mask_projection(mask, accumulate_matches).unwrap() - } - - pub fn internal_sparse_mask_projection_out( - &self, - out: &Tensor, - mask: &Tensor, - accumulate_matches: bool, - ) -> Tensor { - self.f_internal_sparse_mask_projection_out(out, mask, accumulate_matches).unwrap() - } - - pub fn internal_sparse_mm(sparse: &Tensor, dense: &Tensor) -> Tensor { - Tensor::f_internal_sparse_mm(sparse, dense).unwrap() - } - - pub fn internal_sparse_mm_reduce(sparse: &Tensor, dense: &Tensor, reduce: &str) -> Tensor { - Tensor::f_internal_sparse_mm_reduce(sparse, dense, reduce).unwrap() - } - - pub fn internal_sparse_mm_reduce_impl(&self, other: &Tensor, reduce: &str) -> (Tensor, Tensor) { - self.f_internal_sparse_mm_reduce_impl(other, reduce).unwrap() - } - - pub fn internal_sparse_semi_structured_apply(&self, thread_masks: &Tensor) -> (Tensor, Tensor) { - self.f_internal_sparse_semi_structured_apply(thread_masks).unwrap() - } - - pub fn internal_sparse_semi_structured_apply_dense(&self, thread_masks: &Tensor) -> Tensor { - self.f_internal_sparse_semi_structured_apply_dense(thread_masks).unwrap() - } - - pub fn internal_sparse_semi_structured_linear>( - &self, - weight: &Tensor, - meta: &Tensor, - bias: Option, - activation: &str, - out_dtype: impl Into>, - ) -> Tensor { - self.f_internal_sparse_semi_structured_linear(weight, meta, bias, activation, out_dtype) - .unwrap() - } - - pub fn internal_sparse_semi_structured_mm( - mat1: &Tensor, - mat1_meta: &Tensor, - mat2: &Tensor, - out_dtype: impl Into>, - ) -> Tensor { - Tensor::f_internal_sparse_semi_structured_mm(mat1, mat1_meta, mat2, out_dtype).unwrap() - } - - pub fn internal_sparse_semi_structured_tile( - &self, - algorithm: &str, - use_cutlass: bool, - ) -> (Tensor, Tensor, Tensor, Tensor, Tensor) { - self.f_internal_sparse_semi_structured_tile(algorithm, use_cutlass).unwrap() - } - - pub fn internal_sparse_softmax(&self, dim: i64, half_to_float: bool) -> Tensor { - self.f_internal_sparse_softmax(dim, half_to_float).unwrap() - } - - pub fn internal_sparse_softmax_backward_data( - &self, - grad_output: &Tensor, - output: &Tensor, - dim: i64, - ) -> Tensor { - self.f_internal_sparse_softmax_backward_data(grad_output, output, dim).unwrap() - } - - pub fn internal_sparse_softmax_backward_data_out( - &self, - out: &Tensor, - grad_output: &Tensor, - output: &Tensor, - dim: i64, - ) -> Tensor { - self.f_internal_sparse_softmax_backward_data_out(out, grad_output, output, dim).unwrap() - } - - pub fn internal_sparse_softmax_int(&self, dim: i64, dtype: impl Into>) -> Tensor { - self.f_internal_sparse_softmax_int(dim, dtype).unwrap() - } - - pub fn internal_sparse_softmax_out( - &self, - out: &Tensor, - dim: i64, - half_to_float: bool, - ) -> Tensor { - self.f_internal_sparse_softmax_out(out, dim, half_to_float).unwrap() - } - - pub fn internal_sparse_sparse_matmul(&self, other: &Tensor) -> Tensor { - self.f_internal_sparse_sparse_matmul(other).unwrap() - } - - pub fn internal_sparse_sparse_matmul_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_internal_sparse_sparse_matmul_out(out, other).unwrap() - } - - pub fn internal_sparse_sum(&self) -> Tensor { - self.f_internal_sparse_sum().unwrap() - } - - pub fn internal_sparse_sum_backward(&self, grad: &Tensor, dim: impl IntList) -> Tensor { - self.f_internal_sparse_sum_backward(grad, dim).unwrap() - } - - pub fn internal_sparse_sum_backward_out( - &self, - out: &Tensor, - grad: &Tensor, - dim: impl IntList, - ) -> Tensor { - self.f_internal_sparse_sum_backward_out(out, grad, dim).unwrap() - } - - pub fn internal_sparse_sum_dim(&self, dim: impl IntList) -> Tensor { - self.f_internal_sparse_sum_dim(dim).unwrap() - } - - pub fn internal_sparse_sum_dim_dtype(&self, dim: impl IntList, dtype: Kind) -> Tensor { - self.f_internal_sparse_sum_dim_dtype(dim, dtype).unwrap() - } - - pub fn internal_sparse_sum_dim_out(&self, out: &Tensor, dim: impl IntList) -> Tensor { - self.f_internal_sparse_sum_dim_out(out, dim).unwrap() - } - - pub fn internal_sparse_sum_dtype(&self, dtype: Kind) -> Tensor { - self.f_internal_sparse_sum_dtype(dtype).unwrap() - } - - pub fn internal_spdiags( - diagonals: &Tensor, - offsets: &Tensor, - shape: impl IntList, - layout: Option, - ) -> Tensor { - Tensor::f_internal_spdiags(diagonals, offsets, shape, layout).unwrap() - } - - pub fn internal_spdiags_out( - out: &Tensor, - diagonals: &Tensor, - offsets: &Tensor, - shape: impl IntList, - layout: Option, - ) -> Tensor { - Tensor::f_internal_spdiags_out(out, diagonals, offsets, shape, layout).unwrap() - } - - pub fn internal_spsolve(a: &Tensor, b: &Tensor, left: bool) -> Tensor { - Tensor::f_internal_spsolve(a, b, left).unwrap() - } - - pub fn internal_stack>(tensors: &[T], dim: i64) -> Tensor { - Tensor::f_internal_stack(tensors, dim).unwrap() - } - - pub fn internal_stack_out>(out: &Tensor, tensors: &[T], dim: i64) -> Tensor { - Tensor::f_internal_stack_out(out, tensors, dim).unwrap() - } - - pub fn internal_standard_gamma(&self) -> Tensor { - self.f_internal_standard_gamma().unwrap() - } - - pub fn internal_standard_gamma_grad(&self, output: &Tensor) -> Tensor { - self.f_internal_standard_gamma_grad(output).unwrap() - } - - pub fn internal_standard_gamma_grad_out(&self, out: &Tensor, output: &Tensor) -> Tensor { - self.f_internal_standard_gamma_grad_out(out, output).unwrap() - } - - pub fn internal_standard_gamma_out(&self, out: &Tensor) -> Tensor { - self.f_internal_standard_gamma_out(out).unwrap() - } - - pub fn internal_test_ambiguous_defaults(dummy: &Tensor, a: i64, b: i64) -> Tensor { - Tensor::f_internal_test_ambiguous_defaults(dummy, a, b).unwrap() - } - - pub fn internal_test_ambiguous_defaults_b(dummy: &Tensor, a: i64, b: &str) -> Tensor { - Tensor::f_internal_test_ambiguous_defaults_b(dummy, a, b).unwrap() - } - - pub fn internal_test_autograd_multiple_dispatch(&self) -> Tensor { - self.f_internal_test_autograd_multiple_dispatch().unwrap() - } - - pub fn internal_test_autograd_multiple_dispatch_fullcoverage_out( - &self, - out: &Tensor, - ) -> Tensor { - self.f_internal_test_autograd_multiple_dispatch_fullcoverage_out(out).unwrap() - } - - pub fn internal_test_autograd_multiple_dispatch_ntonly(&self, b: bool) -> Tensor { - self.f_internal_test_autograd_multiple_dispatch_ntonly(b).unwrap() - } - - pub fn internal_test_autograd_multiple_dispatch_view(&self) -> Tensor { - self.f_internal_test_autograd_multiple_dispatch_view().unwrap() - } - - pub fn internal_test_autograd_multiple_dispatch_view_copy(&self) -> Tensor { - self.f_internal_test_autograd_multiple_dispatch_view_copy().unwrap() - } - - pub fn internal_test_autograd_multiple_dispatch_view_copy_out(&self, out: &Tensor) -> Tensor { - self.f_internal_test_autograd_multiple_dispatch_view_copy_out(out).unwrap() - } - - pub fn internal_test_check_tensor(&self) -> Tensor { - self.f_internal_test_check_tensor().unwrap() - } - - pub fn internal_test_functorch_fallback(&self, other: &Tensor) -> Tensor { - self.f_internal_test_functorch_fallback(other).unwrap() - } - - pub fn internal_test_functorch_fallback_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_internal_test_functorch_fallback_out(out, other).unwrap() - } - - pub fn internal_test_optional_filled_intlist( - values: &Tensor, - addends: impl IntListOption, - ) -> Tensor { - Tensor::f_internal_test_optional_filled_intlist(values, addends).unwrap() - } - - pub fn internal_test_optional_filled_intlist_out( - out: &Tensor, - values: &Tensor, - addends: impl IntListOption, - ) -> Tensor { - Tensor::f_internal_test_optional_filled_intlist_out(out, values, addends).unwrap() - } - - pub fn internal_test_optional_floatlist(values: &Tensor, addends: impl DoubleList) -> Tensor { - Tensor::f_internal_test_optional_floatlist(values, addends).unwrap() - } - - pub fn internal_test_optional_floatlist_out( - out: &Tensor, - values: &Tensor, - addends: impl DoubleList, - ) -> Tensor { - Tensor::f_internal_test_optional_floatlist_out(out, values, addends).unwrap() - } - - pub fn internal_test_optional_intlist(values: &Tensor, addends: impl IntListOption) -> Tensor { - Tensor::f_internal_test_optional_intlist(values, addends).unwrap() - } - - pub fn internal_test_optional_intlist_out( - out: &Tensor, - values: &Tensor, - addends: impl IntListOption, - ) -> Tensor { - Tensor::f_internal_test_optional_intlist_out(out, values, addends).unwrap() - } - - pub fn internal_test_parallel_materialize( - &self, - num_parallel: i64, - skip_first: bool, - ) -> Tensor { - self.f_internal_test_parallel_materialize(num_parallel, skip_first).unwrap() - } - - pub fn internal_test_serialization_subcmul(&self, other: &Tensor) -> Tensor { - self.f_internal_test_serialization_subcmul(other).unwrap() - } - - pub fn internal_test_string_default(dummy: &Tensor, a: &str, b: &str) -> Tensor { - Tensor::f_internal_test_string_default(dummy, a, b).unwrap() - } - - pub fn internal_test_warn_in_autograd(&self) -> Tensor { - self.f_internal_test_warn_in_autograd().unwrap() - } - - pub fn internal_test_warn_in_autograd_out(&self, out: &Tensor) -> Tensor { - self.f_internal_test_warn_in_autograd_out(out).unwrap() - } - - pub fn internal_to_copy(&self, options: (Kind, Device), non_blocking: bool) -> Tensor { - self.f_internal_to_copy(options, non_blocking).unwrap() - } - - pub fn internal_to_copy_out(&self, out: &Tensor, non_blocking: bool) -> Tensor { - self.f_internal_to_copy_out(out, non_blocking).unwrap() - } - - pub fn internal_to_cpu>(tensors: &[T]) -> Vec { - Tensor::f_internal_to_cpu(tensors).unwrap() - } - - pub fn internal_to_dense(&self, dtype: impl Into>, masked_grad: bool) -> Tensor { - self.f_internal_to_dense(dtype, masked_grad).unwrap() - } - - pub fn internal_to_dense_out( - &self, - out: &Tensor, - dtype: impl Into>, - masked_grad: bool, - ) -> Tensor { - self.f_internal_to_dense_out(out, dtype, masked_grad).unwrap() - } - - pub fn internal_to_sparse( - &self, - layout: Option, - blocksize: impl IntListOption, - dense_dim: impl Into>, - ) -> Tensor { - self.f_internal_to_sparse(layout, blocksize, dense_dim).unwrap() - } - - pub fn internal_to_sparse_bsc( - &self, - blocksize: impl IntList, - dense_dim: impl Into>, - ) -> Tensor { - self.f_internal_to_sparse_bsc(blocksize, dense_dim).unwrap() - } - - pub fn internal_to_sparse_bsc_out( - &self, - out: &Tensor, - blocksize: impl IntList, - dense_dim: impl Into>, - ) -> Tensor { - self.f_internal_to_sparse_bsc_out(out, blocksize, dense_dim).unwrap() - } - - pub fn internal_to_sparse_bsr( - &self, - blocksize: impl IntList, - dense_dim: impl Into>, - ) -> Tensor { - self.f_internal_to_sparse_bsr(blocksize, dense_dim).unwrap() - } - - pub fn internal_to_sparse_bsr_out( - &self, - out: &Tensor, - blocksize: impl IntList, - dense_dim: impl Into>, - ) -> Tensor { - self.f_internal_to_sparse_bsr_out(out, blocksize, dense_dim).unwrap() - } - - pub fn internal_to_sparse_csc(&self, dense_dim: impl Into>) -> Tensor { - self.f_internal_to_sparse_csc(dense_dim).unwrap() - } - - pub fn internal_to_sparse_csc_out( - &self, - out: &Tensor, - dense_dim: impl Into>, - ) -> Tensor { - self.f_internal_to_sparse_csc_out(out, dense_dim).unwrap() - } - - pub fn internal_to_sparse_csr(&self, dense_dim: impl Into>) -> Tensor { - self.f_internal_to_sparse_csr(dense_dim).unwrap() - } - - pub fn internal_to_sparse_csr_out( - &self, - out: &Tensor, - dense_dim: impl Into>, - ) -> Tensor { - self.f_internal_to_sparse_csr_out(out, dense_dim).unwrap() - } - - pub fn internal_to_sparse_out( - &self, - out: &Tensor, - layout: Option, - blocksize: impl IntListOption, - dense_dim: impl Into>, - ) -> Tensor { - self.f_internal_to_sparse_out(out, layout, blocksize, dense_dim).unwrap() - } - - pub fn internal_to_sparse_semi_structured(dense: &Tensor) -> (Tensor, Tensor) { - Tensor::f_internal_to_sparse_semi_structured(dense).unwrap() - } - - pub fn internal_to_sparse_sparse_dim(&self, sparse_dim: i64) -> Tensor { - self.f_internal_to_sparse_sparse_dim(sparse_dim).unwrap() - } - - pub fn internal_to_sparse_sparse_dim_out(&self, out: &Tensor, sparse_dim: i64) -> Tensor { - self.f_internal_to_sparse_sparse_dim_out(out, sparse_dim).unwrap() - } - - pub fn internal_transform_bias_rescale_qkv( - qkv: &Tensor, - qkv_bias: &Tensor, - num_heads: i64, - ) -> (Tensor, Tensor, Tensor) { - Tensor::f_internal_transform_bias_rescale_qkv(qkv, qkv_bias, num_heads).unwrap() - } - - pub fn internal_transform_bias_rescale_qkv_out( - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - qkv: &Tensor, - qkv_bias: &Tensor, - num_heads: i64, - ) -> (Tensor, Tensor, Tensor) { - Tensor::f_internal_transform_bias_rescale_qkv_out( - out0, out1, out2, qkv, qkv_bias, num_heads, - ) - .unwrap() - } - - pub fn internal_transformer_encoder_layer_fwd>( - src: &Tensor, - embed_dim: i64, - num_heads: i64, - qkv_weight: &Tensor, - qkv_bias: &Tensor, - proj_weight: &Tensor, - proj_bias: &Tensor, - use_gelu: bool, - norm_first: bool, - eps: f64, - norm_weight_1: &Tensor, - norm_bias_1: &Tensor, - norm_weight_2: &Tensor, - norm_bias_2: &Tensor, - ffn_weight_1: &Tensor, - ffn_bias_1: &Tensor, - ffn_weight_2: &Tensor, - ffn_bias_2: &Tensor, - mask: Option, - mask_type: impl Into>, - ) -> Tensor { - Tensor::f_internal_transformer_encoder_layer_fwd( - src, - embed_dim, - num_heads, - qkv_weight, - qkv_bias, - proj_weight, - proj_bias, - use_gelu, - norm_first, - eps, - norm_weight_1, - norm_bias_1, - norm_weight_2, - norm_bias_2, - ffn_weight_1, - ffn_bias_1, - ffn_weight_2, - ffn_bias_2, - mask, - mask_type, - ) - .unwrap() - } - - pub fn internal_transformer_encoder_layer_fwd_out>( - out: &Tensor, - src: &Tensor, - embed_dim: i64, - num_heads: i64, - qkv_weight: &Tensor, - qkv_bias: &Tensor, - proj_weight: &Tensor, - proj_bias: &Tensor, - use_gelu: bool, - norm_first: bool, - eps: f64, - norm_weight_1: &Tensor, - norm_bias_1: &Tensor, - norm_weight_2: &Tensor, - norm_bias_2: &Tensor, - ffn_weight_1: &Tensor, - ffn_bias_1: &Tensor, - ffn_weight_2: &Tensor, - ffn_bias_2: &Tensor, - mask: Option, - mask_type: impl Into>, - ) -> Tensor { - Tensor::f_internal_transformer_encoder_layer_fwd_out( - out, - src, - embed_dim, - num_heads, - qkv_weight, - qkv_bias, - proj_weight, - proj_bias, - use_gelu, - norm_first, - eps, - norm_weight_1, - norm_bias_1, - norm_weight_2, - norm_bias_2, - ffn_weight_1, - ffn_bias_1, - ffn_weight_2, - ffn_bias_2, - mask, - mask_type, - ) - .unwrap() - } - - pub fn internal_trilinear( - i1: &Tensor, - i2: &Tensor, - i3: &Tensor, - expand1: impl IntList, - expand2: impl IntList, - expand3: impl IntList, - sumdim: impl IntList, - unroll_dim: i64, - ) -> Tensor { - Tensor::f_internal_trilinear(i1, i2, i3, expand1, expand2, expand3, sumdim, unroll_dim) - .unwrap() - } - - pub fn internal_trilinear_out( - out: &Tensor, - i1: &Tensor, - i2: &Tensor, - i3: &Tensor, - expand1: impl IntList, - expand2: impl IntList, - expand3: impl IntList, - sumdim: impl IntList, - unroll_dim: i64, - ) -> Tensor { - Tensor::f_internal_trilinear_out( - out, i1, i2, i3, expand1, expand2, expand3, sumdim, unroll_dim, - ) - .unwrap() - } - - pub fn internal_triton_multi_head_attention>( - query: &Tensor, - key: &Tensor, - value: &Tensor, - embed_dim: i64, - num_head: i64, - qkv_weight: &Tensor, - qkv_bias: &Tensor, - proj_weight: &Tensor, - proj_bias: &Tensor, - mask: Option, - ) -> Tensor { - Tensor::f_internal_triton_multi_head_attention( - query, - key, - value, - embed_dim, - num_head, - qkv_weight, - qkv_bias, - proj_weight, - proj_bias, - mask, - ) - .unwrap() - } - - pub fn internal_triton_multi_head_attention_out>( - out: &Tensor, - query: &Tensor, - key: &Tensor, - value: &Tensor, - embed_dim: i64, - num_head: i64, - qkv_weight: &Tensor, - qkv_bias: &Tensor, - proj_weight: &Tensor, - proj_bias: &Tensor, - mask: Option, - ) -> Tensor { - Tensor::f_internal_triton_multi_head_attention_out( - out, - query, - key, - value, - embed_dim, - num_head, - qkv_weight, - qkv_bias, - proj_weight, - proj_bias, - mask, - ) - .unwrap() - } - - pub fn internal_triton_scaled_dot_attention( - q: &Tensor, - k: &Tensor, - v: &Tensor, - dropout_p: f64, - ) -> Tensor { - Tensor::f_internal_triton_scaled_dot_attention(q, k, v, dropout_p).unwrap() - } - - pub fn internal_triton_scaled_dot_attention_out( - out: &Tensor, - q: &Tensor, - k: &Tensor, - v: &Tensor, - dropout_p: f64, - ) -> Tensor { - Tensor::f_internal_triton_scaled_dot_attention_out(out, q, k, v, dropout_p).unwrap() - } - - pub fn internal_unique(&self, sorted: bool, return_inverse: bool) -> (Tensor, Tensor) { - self.f_internal_unique(sorted, return_inverse).unwrap() - } - - pub fn internal_unique2( - &self, - sorted: bool, - return_inverse: bool, - return_counts: bool, - ) -> (Tensor, Tensor, Tensor) { - self.f_internal_unique2(sorted, return_inverse, return_counts).unwrap() - } - - pub fn internal_unique2_out( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - sorted: bool, - return_inverse: bool, - return_counts: bool, - ) -> (Tensor, Tensor, Tensor) { - self.f_internal_unique2_out(out0, out1, out2, sorted, return_inverse, return_counts) - .unwrap() - } - - pub fn internal_unique_out( - &self, - out0: &Tensor, - out1: &Tensor, - sorted: bool, - return_inverse: bool, - ) -> (Tensor, Tensor) { - self.f_internal_unique_out(out0, out1, sorted, return_inverse).unwrap() - } - - pub fn internal_unpack_dual(dual: &Tensor, level: i64) -> (Tensor, Tensor) { - Tensor::f_internal_unpack_dual(dual, level).unwrap() - } - - pub fn internal_unsafe_index>(&self, indices: &[Option]) -> Tensor { - self.f_internal_unsafe_index(indices).unwrap() - } - - pub fn internal_unsafe_index_put>( - &self, - indices: &[Option], - values: &Tensor, - accumulate: bool, - ) -> Tensor { - self.f_internal_unsafe_index_put(indices, values, accumulate).unwrap() - } - - pub fn internal_unsafe_masked_index, S: Into>( - &self, - mask: &Tensor, - indices: &[Option], - fill: S, - ) -> Tensor { - self.f_internal_unsafe_masked_index(mask, indices, fill).unwrap() - } - - pub fn internal_unsafe_masked_index_put_accumulate>( - &self, - mask: &Tensor, - indices: &[Option], - values: &Tensor, - ) -> Tensor { - self.f_internal_unsafe_masked_index_put_accumulate(mask, indices, values).unwrap() - } - - pub fn internal_unsafe_view(&self, size: impl IntList) -> Tensor { - self.f_internal_unsafe_view(size).unwrap() - } - - pub fn internal_unsafe_view_out(&self, out: &Tensor, size: impl IntList) -> Tensor { - self.f_internal_unsafe_view_out(out, size).unwrap() - } - - pub fn internal_upsample_bicubic2d_aa( - &self, - output_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - self.f_internal_upsample_bicubic2d_aa(output_size, align_corners, scales_h, scales_w) - .unwrap() - } - - pub fn internal_upsample_bicubic2d_aa_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - Tensor::f_internal_upsample_bicubic2d_aa_backward( - grad_output, - output_size, - input_size, - align_corners, - scales_h, - scales_w, - ) - .unwrap() - } - - pub fn internal_upsample_bicubic2d_aa_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - Tensor::f_internal_upsample_bicubic2d_aa_backward_grad_input( - grad_input, - grad_output, - output_size, - input_size, - align_corners, - scales_h, - scales_w, - ) - .unwrap() - } - - pub fn internal_upsample_bicubic2d_aa_out( - &self, - out: &Tensor, - output_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - self.f_internal_upsample_bicubic2d_aa_out( - out, - output_size, - align_corners, - scales_h, - scales_w, - ) - .unwrap() - } - - pub fn internal_upsample_bicubic2d_aa_vec( - &self, - output_size: impl IntListOption, - align_corners: bool, - scale_factors: impl DoubleList, - ) -> Tensor { - self.f_internal_upsample_bicubic2d_aa_vec(output_size, align_corners, scale_factors) - .unwrap() - } - - pub fn internal_upsample_bilinear2d_aa( - &self, - output_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - self.f_internal_upsample_bilinear2d_aa(output_size, align_corners, scales_h, scales_w) - .unwrap() - } - - pub fn internal_upsample_bilinear2d_aa_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - Tensor::f_internal_upsample_bilinear2d_aa_backward( - grad_output, - output_size, - input_size, - align_corners, - scales_h, - scales_w, - ) - .unwrap() - } - - pub fn internal_upsample_bilinear2d_aa_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - Tensor::f_internal_upsample_bilinear2d_aa_backward_grad_input( - grad_input, - grad_output, - output_size, - input_size, - align_corners, - scales_h, - scales_w, - ) - .unwrap() - } - - pub fn internal_upsample_bilinear2d_aa_out( - &self, - out: &Tensor, - output_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - self.f_internal_upsample_bilinear2d_aa_out( - out, - output_size, - align_corners, - scales_h, - scales_w, - ) - .unwrap() - } - - pub fn internal_upsample_bilinear2d_aa_vec( - &self, - output_size: impl IntListOption, - align_corners: bool, - scale_factors: impl DoubleList, - ) -> Tensor { - self.f_internal_upsample_bilinear2d_aa_vec(output_size, align_corners, scale_factors) - .unwrap() - } - - pub fn internal_upsample_nearest_exact1d( - &self, - output_size: impl IntList, - scales: impl Into>, - ) -> Tensor { - self.f_internal_upsample_nearest_exact1d(output_size, scales).unwrap() - } - - pub fn internal_upsample_nearest_exact1d_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales: impl Into>, - ) -> Tensor { - Tensor::f_internal_upsample_nearest_exact1d_backward( - grad_output, - output_size, - input_size, - scales, - ) - .unwrap() - } - - pub fn internal_upsample_nearest_exact1d_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales: impl Into>, - ) -> Tensor { - Tensor::f_internal_upsample_nearest_exact1d_backward_grad_input( - grad_input, - grad_output, - output_size, - input_size, - scales, - ) - .unwrap() - } - - pub fn internal_upsample_nearest_exact1d_out( - &self, - out: &Tensor, - output_size: impl IntList, - scales: impl Into>, - ) -> Tensor { - self.f_internal_upsample_nearest_exact1d_out(out, output_size, scales).unwrap() - } - - pub fn internal_upsample_nearest_exact1d_vec( - &self, - output_size: impl IntListOption, - scale_factors: impl DoubleList, - ) -> Tensor { - self.f_internal_upsample_nearest_exact1d_vec(output_size, scale_factors).unwrap() - } - - pub fn internal_upsample_nearest_exact2d( - &self, - output_size: impl IntList, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - self.f_internal_upsample_nearest_exact2d(output_size, scales_h, scales_w).unwrap() - } - - pub fn internal_upsample_nearest_exact2d_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - Tensor::f_internal_upsample_nearest_exact2d_backward( - grad_output, - output_size, - input_size, - scales_h, - scales_w, - ) - .unwrap() - } - - pub fn internal_upsample_nearest_exact2d_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - Tensor::f_internal_upsample_nearest_exact2d_backward_grad_input( - grad_input, - grad_output, - output_size, - input_size, - scales_h, - scales_w, - ) - .unwrap() - } - - pub fn internal_upsample_nearest_exact2d_out( - &self, - out: &Tensor, - output_size: impl IntList, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - self.f_internal_upsample_nearest_exact2d_out(out, output_size, scales_h, scales_w).unwrap() - } - - pub fn internal_upsample_nearest_exact2d_vec( - &self, - output_size: impl IntListOption, - scale_factors: impl DoubleList, - ) -> Tensor { - self.f_internal_upsample_nearest_exact2d_vec(output_size, scale_factors).unwrap() - } - - pub fn internal_upsample_nearest_exact3d( - &self, - output_size: impl IntList, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - self.f_internal_upsample_nearest_exact3d(output_size, scales_d, scales_h, scales_w).unwrap() - } - - pub fn internal_upsample_nearest_exact3d_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - Tensor::f_internal_upsample_nearest_exact3d_backward( - grad_output, - output_size, - input_size, - scales_d, - scales_h, - scales_w, - ) - .unwrap() - } - - pub fn internal_upsample_nearest_exact3d_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - Tensor::f_internal_upsample_nearest_exact3d_backward_grad_input( - grad_input, - grad_output, - output_size, - input_size, - scales_d, - scales_h, - scales_w, - ) - .unwrap() - } - - pub fn internal_upsample_nearest_exact3d_out( - &self, - out: &Tensor, - output_size: impl IntList, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - self.f_internal_upsample_nearest_exact3d_out(out, output_size, scales_d, scales_h, scales_w) - .unwrap() - } - - pub fn internal_upsample_nearest_exact3d_vec( - &self, - output_size: impl IntListOption, - scale_factors: impl DoubleList, - ) -> Tensor { - self.f_internal_upsample_nearest_exact3d_vec(output_size, scale_factors).unwrap() - } - - pub fn internal_use_cudnn_ctc_loss( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: impl IntList, - target_lengths: impl IntList, - blank: i64, - ) -> bool { - Tensor::f_internal_use_cudnn_ctc_loss( - log_probs, - targets, - input_lengths, - target_lengths, - blank, - ) - .unwrap() - } - - pub fn internal_use_cudnn_ctc_loss_tensor( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: &Tensor, - target_lengths: &Tensor, - blank: i64, - ) -> bool { - Tensor::f_internal_use_cudnn_ctc_loss_tensor( - log_probs, - targets, - input_lengths, - target_lengths, - blank, - ) - .unwrap() - } - - pub fn internal_use_cudnn_rnn_flatten_weight() -> bool { - Tensor::f_internal_use_cudnn_rnn_flatten_weight().unwrap() - } - - pub fn internal_use_miopen_ctc_loss( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: impl IntList, - target_lengths: impl IntList, - blank: i64, - ) -> bool { - Tensor::f_internal_use_miopen_ctc_loss( - log_probs, - targets, - input_lengths, - target_lengths, - blank, - ) - .unwrap() - } - - pub fn internal_use_miopen_ctc_loss_tensor( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: &Tensor, - target_lengths: &Tensor, - blank: i64, - ) -> bool { - Tensor::f_internal_use_miopen_ctc_loss_tensor( - log_probs, - targets, - input_lengths, - target_lengths, - blank, - ) - .unwrap() - } - - pub fn internal_validate_compressed_sparse_indices( - is_crow: bool, - compressed_idx: &Tensor, - plain_idx: &Tensor, - cdim: i64, - dim: i64, - nnz: i64, - ) { - Tensor::f_internal_validate_compressed_sparse_indices( - is_crow, - compressed_idx, - plain_idx, - cdim, - dim, - nnz, - ) - .unwrap() - } - - pub fn internal_validate_sparse_bsc_tensor_args( - ccol_indices: &Tensor, - row_indices: &Tensor, - values: &Tensor, - size: impl IntList, - check_pinning: bool, - ) { - Tensor::f_internal_validate_sparse_bsc_tensor_args( - ccol_indices, - row_indices, - values, - size, - check_pinning, - ) - .unwrap() - } - - pub fn internal_validate_sparse_bsr_tensor_args( - crow_indices: &Tensor, - col_indices: &Tensor, - values: &Tensor, - size: impl IntList, - check_pinning: bool, - ) { - Tensor::f_internal_validate_sparse_bsr_tensor_args( - crow_indices, - col_indices, - values, - size, - check_pinning, - ) - .unwrap() - } - - pub fn internal_validate_sparse_compressed_tensor_args( - compressed_indices: &Tensor, - plain_indices: &Tensor, - values: &Tensor, - size: impl IntList, - layout: Layout, - check_pinning: bool, - ) { - Tensor::f_internal_validate_sparse_compressed_tensor_args( - compressed_indices, - plain_indices, - values, - size, - layout, - check_pinning, - ) - .unwrap() - } - - pub fn internal_validate_sparse_csc_tensor_args( - ccol_indices: &Tensor, - row_indices: &Tensor, - values: &Tensor, - size: impl IntList, - check_pinning: bool, - ) { - Tensor::f_internal_validate_sparse_csc_tensor_args( - ccol_indices, - row_indices, - values, - size, - check_pinning, - ) - .unwrap() - } - - pub fn internal_validate_sparse_csr_tensor_args( - crow_indices: &Tensor, - col_indices: &Tensor, - values: &Tensor, - size: impl IntList, - check_pinning: bool, - ) { - Tensor::f_internal_validate_sparse_csr_tensor_args( - crow_indices, - col_indices, - values, - size, - check_pinning, - ) - .unwrap() - } - - pub fn internal_values(&self) -> Tensor { - self.f_internal_values().unwrap() - } - - pub fn internal_values_copy(&self) -> Tensor { - self.f_internal_values_copy().unwrap() - } - - pub fn internal_values_copy_out(&self, out: &Tensor) -> Tensor { - self.f_internal_values_copy_out(out).unwrap() - } - - pub fn internal_version(&self) -> i64 { - self.f_internal_version().unwrap() - } - - pub fn internal_weight_int4pack_mm( - &self, - mat2: &Tensor, - qgroupsize: i64, - qscaleandzeros: &Tensor, - ) -> Tensor { - self.f_internal_weight_int4pack_mm(mat2, qgroupsize, qscaleandzeros).unwrap() - } - - pub fn internal_weight_int4pack_mm_for_cpu( - &self, - mat2: &Tensor, - qgroupsize: i64, - qscaleandzeros: &Tensor, - ) -> Tensor { - self.f_internal_weight_int4pack_mm_for_cpu(mat2, qgroupsize, qscaleandzeros).unwrap() - } - - pub fn internal_weight_int4pack_mm_with_scales_and_zeros( - &self, - mat2: &Tensor, - qgroupsize: i64, - qscale: &Tensor, - qzeros: &Tensor, - ) -> Tensor { - self.f_internal_weight_int4pack_mm_with_scales_and_zeros(mat2, qgroupsize, qscale, qzeros) - .unwrap() - } - - pub fn internal_weight_int8pack_mm(&self, mat2: &Tensor, scales: &Tensor) -> Tensor { - self.f_internal_weight_int8pack_mm(mat2, scales).unwrap() - } - - pub fn internal_weight_norm(v: &Tensor, g: &Tensor, dim: i64) -> Tensor { - Tensor::f_internal_weight_norm(v, g, dim).unwrap() - } - - pub fn internal_weight_norm_differentiable_backward( - grad_w: &Tensor, - saved_v: &Tensor, - saved_g: &Tensor, - saved_norms: &Tensor, - dim: i64, - ) -> (Tensor, Tensor) { - Tensor::f_internal_weight_norm_differentiable_backward( - grad_w, - saved_v, - saved_g, - saved_norms, - dim, - ) - .unwrap() - } - - pub fn internal_weight_norm_interface(v: &Tensor, g: &Tensor, dim: i64) -> (Tensor, Tensor) { - Tensor::f_internal_weight_norm_interface(v, g, dim).unwrap() - } - - pub fn internal_weight_norm_interface_backward( - grad_w: &Tensor, - saved_v: &Tensor, - saved_g: &Tensor, - saved_norms: &Tensor, - dim: i64, - ) -> (Tensor, Tensor) { - Tensor::f_internal_weight_norm_interface_backward( - grad_w, - saved_v, - saved_g, - saved_norms, - dim, - ) - .unwrap() - } - - pub fn internal_weight_norm_interface_backward_out( - out0: &Tensor, - out1: &Tensor, - grad_w: &Tensor, - saved_v: &Tensor, - saved_g: &Tensor, - saved_norms: &Tensor, - dim: i64, - ) -> (Tensor, Tensor) { - Tensor::f_internal_weight_norm_interface_backward_out( - out0, - out1, - grad_w, - saved_v, - saved_g, - saved_norms, - dim, - ) - .unwrap() - } - - pub fn internal_weight_norm_interface_out( - out0: &Tensor, - out1: &Tensor, - v: &Tensor, - g: &Tensor, - dim: i64, - ) -> (Tensor, Tensor) { - Tensor::f_internal_weight_norm_interface_out(out0, out1, v, g, dim).unwrap() - } - - pub fn internal_wrapped_linear_prepack( - weight: &Tensor, - weight_scale: &Tensor, - weight_zero_point: &Tensor, - bias: &Tensor, - ) -> Tensor { - Tensor::f_internal_wrapped_linear_prepack(weight, weight_scale, weight_zero_point, bias) - .unwrap() - } - - pub fn internal_wrapped_quantized_linear_prepacked( - &self, - input_scale: &Tensor, - input_zero_point: &Tensor, - packed_weight: &Tensor, - output_scale: &Tensor, - output_zero_point: &Tensor, - out_channel: i64, - ) -> Tensor { - self.f_internal_wrapped_quantized_linear_prepacked( - input_scale, - input_zero_point, - packed_weight, - output_scale, - output_zero_point, - out_channel, - ) - .unwrap() - } - - pub fn abs(&self) -> Tensor { - self.f_abs().unwrap() - } - - pub fn abs_(&mut self) -> Tensor { - self.f_abs_().unwrap() - } - - pub fn abs_out(&self, out: &Tensor) -> Tensor { - self.f_abs_out(out).unwrap() - } - - pub fn absolute(&self) -> Tensor { - self.f_absolute().unwrap() - } - - pub fn absolute_(&mut self) -> Tensor { - self.f_absolute_().unwrap() - } - - pub fn absolute_out(&self, out: &Tensor) -> Tensor { - self.f_absolute_out(out).unwrap() - } - - pub fn acos(&self) -> Tensor { - self.f_acos().unwrap() - } - - pub fn acos_(&mut self) -> Tensor { - self.f_acos_().unwrap() - } - - pub fn acos_out(&self, out: &Tensor) -> Tensor { - self.f_acos_out(out).unwrap() - } - - pub fn acosh(&self) -> Tensor { - self.f_acosh().unwrap() - } - - pub fn acosh_(&mut self) -> Tensor { - self.f_acosh_().unwrap() - } - - pub fn acosh_out(&self, out: &Tensor) -> Tensor { - self.f_acosh_out(out).unwrap() - } - - pub fn adaptive_avg_pool1d(&self, output_size: impl IntList) -> Tensor { - self.f_adaptive_avg_pool1d(output_size).unwrap() - } - - pub fn adaptive_avg_pool1d_out(&self, out: &Tensor, output_size: impl IntList) -> Tensor { - self.f_adaptive_avg_pool1d_out(out, output_size).unwrap() - } - - pub fn adaptive_avg_pool2d(&self, output_size: impl IntList) -> Tensor { - self.f_adaptive_avg_pool2d(output_size).unwrap() - } - - pub fn adaptive_avg_pool2d_out(&self, out: &Tensor, output_size: impl IntList) -> Tensor { - self.f_adaptive_avg_pool2d_out(out, output_size).unwrap() - } - - pub fn adaptive_avg_pool3d(&self, output_size: impl IntList) -> Tensor { - self.f_adaptive_avg_pool3d(output_size).unwrap() - } - - pub fn adaptive_avg_pool3d_backward( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - ) -> Tensor { - self.f_adaptive_avg_pool3d_backward(grad_input, grad_output).unwrap() - } - - pub fn adaptive_avg_pool3d_out(&self, out: &Tensor, output_size: impl IntList) -> Tensor { - self.f_adaptive_avg_pool3d_out(out, output_size).unwrap() - } - - pub fn adaptive_max_pool1d(&self, output_size: impl IntList) -> (Tensor, Tensor) { - self.f_adaptive_max_pool1d(output_size).unwrap() - } - - pub fn adaptive_max_pool2d(&self, output_size: impl IntList) -> (Tensor, Tensor) { - self.f_adaptive_max_pool2d(output_size).unwrap() - } - - pub fn adaptive_max_pool2d_backward(&self, grad_output: &Tensor, indices: &Tensor) -> Tensor { - self.f_adaptive_max_pool2d_backward(grad_output, indices).unwrap() - } - - pub fn adaptive_max_pool2d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - indices: &Tensor, - ) -> Tensor { - self.f_adaptive_max_pool2d_backward_grad_input(grad_input, grad_output, indices).unwrap() - } - - pub fn adaptive_max_pool2d_out( - &self, - out: &Tensor, - indices: &Tensor, - output_size: impl IntList, - ) -> (Tensor, Tensor) { - self.f_adaptive_max_pool2d_out(out, indices, output_size).unwrap() - } - - pub fn adaptive_max_pool3d(&self, output_size: impl IntList) -> (Tensor, Tensor) { - self.f_adaptive_max_pool3d(output_size).unwrap() - } - - pub fn adaptive_max_pool3d_backward(&self, grad_output: &Tensor, indices: &Tensor) -> Tensor { - self.f_adaptive_max_pool3d_backward(grad_output, indices).unwrap() - } - - pub fn adaptive_max_pool3d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - indices: &Tensor, - ) -> Tensor { - self.f_adaptive_max_pool3d_backward_grad_input(grad_input, grad_output, indices).unwrap() - } - - pub fn adaptive_max_pool3d_out( - &self, - out: &Tensor, - indices: &Tensor, - output_size: impl IntList, - ) -> (Tensor, Tensor) { - self.f_adaptive_max_pool3d_out(out, indices, output_size).unwrap() - } - - pub fn g_add(&self, other: &Tensor) -> Tensor { - self.f_add(other).unwrap() - } - - pub fn g_add_(&mut self, other: &Tensor) -> Tensor { - self.f_add_(other).unwrap() - } - - pub fn add_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_add_out(out, other).unwrap() - } - - pub fn g_add_scalar>(&self, other: S) -> Tensor { - self.f_add_scalar(other).unwrap() - } - - pub fn g_add_scalar_>(&mut self, other: S) -> Tensor { - self.f_add_scalar_(other).unwrap() - } - - pub fn add_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_add_scalar_out(out, other).unwrap() - } - - pub fn addbmm(&self, batch1: &Tensor, batch2: &Tensor) -> Tensor { - self.f_addbmm(batch1, batch2).unwrap() - } - - pub fn addbmm_(&mut self, batch1: &Tensor, batch2: &Tensor) -> Tensor { - self.f_addbmm_(batch1, batch2).unwrap() - } - - pub fn addbmm_out(&self, out: &Tensor, batch1: &Tensor, batch2: &Tensor) -> Tensor { - self.f_addbmm_out(out, batch1, batch2).unwrap() - } - - pub fn addcdiv(&self, tensor1: &Tensor, tensor2: &Tensor) -> Tensor { - self.f_addcdiv(tensor1, tensor2).unwrap() - } - - pub fn addcdiv_(&mut self, tensor1: &Tensor, tensor2: &Tensor) -> Tensor { - self.f_addcdiv_(tensor1, tensor2).unwrap() - } - - pub fn addcdiv_out(&self, out: &Tensor, tensor1: &Tensor, tensor2: &Tensor) -> Tensor { - self.f_addcdiv_out(out, tensor1, tensor2).unwrap() - } - - pub fn addcmul(&self, tensor1: &Tensor, tensor2: &Tensor) -> Tensor { - self.f_addcmul(tensor1, tensor2).unwrap() - } - - pub fn addcmul_(&mut self, tensor1: &Tensor, tensor2: &Tensor) -> Tensor { - self.f_addcmul_(tensor1, tensor2).unwrap() - } - - pub fn addcmul_out(&self, out: &Tensor, tensor1: &Tensor, tensor2: &Tensor) -> Tensor { - self.f_addcmul_out(out, tensor1, tensor2).unwrap() - } - - pub fn addmm(&self, mat1: &Tensor, mat2: &Tensor) -> Tensor { - self.f_addmm(mat1, mat2).unwrap() - } - - pub fn addmm_(&mut self, mat1: &Tensor, mat2: &Tensor) -> Tensor { - self.f_addmm_(mat1, mat2).unwrap() - } - - pub fn addmm_dtype(&self, mat1: &Tensor, mat2: &Tensor, out_dtype: Kind) -> Tensor { - self.f_addmm_dtype(mat1, mat2, out_dtype).unwrap() - } - - pub fn addmm_dtype_out( - &self, - out: &Tensor, - mat1: &Tensor, - mat2: &Tensor, - out_dtype: Kind, - ) -> Tensor { - self.f_addmm_dtype_out(out, mat1, mat2, out_dtype).unwrap() - } - - pub fn addmm_out(&self, out: &Tensor, mat1: &Tensor, mat2: &Tensor) -> Tensor { - self.f_addmm_out(out, mat1, mat2).unwrap() - } - - pub fn addmv(&self, mat: &Tensor, vec: &Tensor) -> Tensor { - self.f_addmv(mat, vec).unwrap() - } - - pub fn addmv_(&mut self, mat: &Tensor, vec: &Tensor) -> Tensor { - self.f_addmv_(mat, vec).unwrap() - } - - pub fn addmv_out(&self, out: &Tensor, mat: &Tensor, vec: &Tensor) -> Tensor { - self.f_addmv_out(out, mat, vec).unwrap() - } - - pub fn addr(&self, vec1: &Tensor, vec2: &Tensor) -> Tensor { - self.f_addr(vec1, vec2).unwrap() - } - - pub fn addr_(&mut self, vec1: &Tensor, vec2: &Tensor) -> Tensor { - self.f_addr_(vec1, vec2).unwrap() - } - - pub fn addr_out(&self, out: &Tensor, vec1: &Tensor, vec2: &Tensor) -> Tensor { - self.f_addr_out(out, vec1, vec2).unwrap() - } - - pub fn adjoint(&self) -> Tensor { - self.f_adjoint().unwrap() - } - - pub fn affine_grid_generator( - theta: &Tensor, - size: impl IntList, - align_corners: bool, - ) -> Tensor { - Tensor::f_affine_grid_generator(theta, size, align_corners).unwrap() - } - - pub fn affine_grid_generator_backward( - grad: &Tensor, - size: impl IntList, - align_corners: bool, - ) -> Tensor { - Tensor::f_affine_grid_generator_backward(grad, size, align_corners).unwrap() - } - - pub fn affine_grid_generator_out( - out: &Tensor, - theta: &Tensor, - size: impl IntList, - align_corners: bool, - ) -> Tensor { - Tensor::f_affine_grid_generator_out(out, theta, size, align_corners).unwrap() - } - - pub fn alias(&self) -> Tensor { - self.f_alias().unwrap() - } - - pub fn alias_copy(&self) -> Tensor { - self.f_alias_copy().unwrap() - } - - pub fn alias_copy_out(&self, out: &Tensor) -> Tensor { - self.f_alias_copy_out(out).unwrap() - } - - pub fn align_as(&self, other: &Tensor) -> Tensor { - self.f_align_as(other).unwrap() - } - - pub fn align_tensors>(tensors: &[T]) -> Vec { - Tensor::f_align_tensors(tensors).unwrap() - } - - pub fn all(&self) -> Tensor { - self.f_all().unwrap() - } - - pub fn all_all_out(&self, out: &Tensor) -> Tensor { - self.f_all_all_out(out).unwrap() - } - - pub fn all_dim(&self, dim: i64, keepdim: bool) -> Tensor { - self.f_all_dim(dim, keepdim).unwrap() - } - - pub fn all_dims(&self, dim: impl IntListOption, keepdim: bool) -> Tensor { - self.f_all_dims(dim, keepdim).unwrap() - } - - pub fn all_dims_out(&self, out: &Tensor, dim: impl IntListOption, keepdim: bool) -> Tensor { - self.f_all_dims_out(out, dim, keepdim).unwrap() - } - - pub fn all_out(&self, out: &Tensor, dim: i64, keepdim: bool) -> Tensor { - self.f_all_out(out, dim, keepdim).unwrap() - } - - pub fn allclose(&self, other: &Tensor, rtol: f64, atol: f64, equal_nan: bool) -> bool { - self.f_allclose(other, rtol, atol, equal_nan).unwrap() - } - - pub fn alpha_dropout(&self, p: f64, train: bool) -> Tensor { - self.f_alpha_dropout(p, train).unwrap() - } - - pub fn alpha_dropout_(&mut self, p: f64, train: bool) -> Tensor { - self.f_alpha_dropout_(p, train).unwrap() - } - - pub fn amax(&self, dim: impl IntList, keepdim: bool) -> Tensor { - self.f_amax(dim, keepdim).unwrap() - } - - pub fn amax_out(&self, out: &Tensor, dim: impl IntList, keepdim: bool) -> Tensor { - self.f_amax_out(out, dim, keepdim).unwrap() - } - - pub fn amin(&self, dim: impl IntList, keepdim: bool) -> Tensor { - self.f_amin(dim, keepdim).unwrap() - } - - pub fn amin_out(&self, out: &Tensor, dim: impl IntList, keepdim: bool) -> Tensor { - self.f_amin_out(out, dim, keepdim).unwrap() - } - - pub fn aminmax(&self, dim: impl Into>, keepdim: bool) -> (Tensor, Tensor) { - self.f_aminmax(dim, keepdim).unwrap() - } - - pub fn aminmax_out( - &self, - min: &Tensor, - max: &Tensor, - dim: impl Into>, - keepdim: bool, - ) -> (Tensor, Tensor) { - self.f_aminmax_out(min, max, dim, keepdim).unwrap() - } - - pub fn angle(&self) -> Tensor { - self.f_angle().unwrap() - } - - pub fn angle_out(&self, out: &Tensor) -> Tensor { - self.f_angle_out(out).unwrap() - } - - pub fn any(&self) -> Tensor { - self.f_any().unwrap() - } - - pub fn any_all_out(&self, out: &Tensor) -> Tensor { - self.f_any_all_out(out).unwrap() - } - - pub fn any_dim(&self, dim: i64, keepdim: bool) -> Tensor { - self.f_any_dim(dim, keepdim).unwrap() - } - - pub fn any_dims(&self, dim: impl IntListOption, keepdim: bool) -> Tensor { - self.f_any_dims(dim, keepdim).unwrap() - } - - pub fn any_dims_out(&self, out: &Tensor, dim: impl IntListOption, keepdim: bool) -> Tensor { - self.f_any_dims_out(out, dim, keepdim).unwrap() - } - - pub fn any_out(&self, out: &Tensor, dim: i64, keepdim: bool) -> Tensor { - self.f_any_out(out, dim, keepdim).unwrap() - } - - pub fn arange>(end: S, options: (Kind, Device)) -> Tensor { - Tensor::f_arange(end, options).unwrap() - } - - pub fn arange_start>(start: S, end: S, options: (Kind, Device)) -> Tensor { - Tensor::f_arange_start(start, end, options).unwrap() - } - - pub fn arange_start_step>( - start: S, - end: S, - step: S, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_arange_start_step(start, end, step, options).unwrap() - } - - pub fn arccos(&self) -> Tensor { - self.f_arccos().unwrap() - } - - pub fn arccos_(&mut self) -> Tensor { - self.f_arccos_().unwrap() - } - - pub fn arccos_out(&self, out: &Tensor) -> Tensor { - self.f_arccos_out(out).unwrap() - } - - pub fn arccosh(&self) -> Tensor { - self.f_arccosh().unwrap() - } - - pub fn arccosh_(&mut self) -> Tensor { - self.f_arccosh_().unwrap() - } - - pub fn arccosh_out(&self, out: &Tensor) -> Tensor { - self.f_arccosh_out(out).unwrap() - } - - pub fn arcsin(&self) -> Tensor { - self.f_arcsin().unwrap() - } - - pub fn arcsin_(&mut self) -> Tensor { - self.f_arcsin_().unwrap() - } - - pub fn arcsin_out(&self, out: &Tensor) -> Tensor { - self.f_arcsin_out(out).unwrap() - } - - pub fn arcsinh(&self) -> Tensor { - self.f_arcsinh().unwrap() - } - - pub fn arcsinh_(&mut self) -> Tensor { - self.f_arcsinh_().unwrap() - } - - pub fn arcsinh_out(&self, out: &Tensor) -> Tensor { - self.f_arcsinh_out(out).unwrap() - } - - pub fn arctan(&self) -> Tensor { - self.f_arctan().unwrap() - } - - pub fn arctan2(&self, other: &Tensor) -> Tensor { - self.f_arctan2(other).unwrap() - } - - pub fn arctan2_(&mut self, other: &Tensor) -> Tensor { - self.f_arctan2_(other).unwrap() - } - - pub fn arctan2_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_arctan2_out(out, other).unwrap() - } - - pub fn arctan_(&mut self) -> Tensor { - self.f_arctan_().unwrap() - } - - pub fn arctan_out(&self, out: &Tensor) -> Tensor { - self.f_arctan_out(out).unwrap() - } - - pub fn arctanh(&self) -> Tensor { - self.f_arctanh().unwrap() - } - - pub fn arctanh_(&mut self) -> Tensor { - self.f_arctanh_().unwrap() - } - - pub fn arctanh_out(&self, out: &Tensor) -> Tensor { - self.f_arctanh_out(out).unwrap() - } - - pub fn argmax(&self, dim: impl Into>, keepdim: bool) -> Tensor { - self.f_argmax(dim, keepdim).unwrap() - } - - pub fn argmax_out(&self, out: &Tensor, dim: impl Into>, keepdim: bool) -> Tensor { - self.f_argmax_out(out, dim, keepdim).unwrap() - } - - pub fn argmin(&self, dim: impl Into>, keepdim: bool) -> Tensor { - self.f_argmin(dim, keepdim).unwrap() - } - - pub fn argmin_out(&self, out: &Tensor, dim: impl Into>, keepdim: bool) -> Tensor { - self.f_argmin_out(out, dim, keepdim).unwrap() - } - - pub fn argsort(&self, dim: i64, descending: bool) -> Tensor { - self.f_argsort(dim, descending).unwrap() - } - - pub fn argsort_stable(&self, stable: bool, dim: i64, descending: bool) -> Tensor { - self.f_argsort_stable(stable, dim, descending).unwrap() - } - - pub fn argsort_stable_out( - &self, - out: &Tensor, - stable: bool, - dim: i64, - descending: bool, - ) -> Tensor { - self.f_argsort_stable_out(out, stable, dim, descending).unwrap() - } - - pub fn argwhere(&self) -> Tensor { - self.f_argwhere().unwrap() - } - - pub fn as_strided( - &self, - size: impl IntList, - stride: impl IntList, - storage_offset: impl Into>, - ) -> Tensor { - self.f_as_strided(size, stride, storage_offset).unwrap() - } - - pub fn as_strided_( - &mut self, - size: impl IntList, - stride: impl IntList, - storage_offset: impl Into>, - ) -> Tensor { - self.f_as_strided_(size, stride, storage_offset).unwrap() - } - - pub fn as_strided_copy( - &self, - size: impl IntList, - stride: impl IntList, - storage_offset: impl Into>, - ) -> Tensor { - self.f_as_strided_copy(size, stride, storage_offset).unwrap() - } - - pub fn as_strided_copy_out( - &self, - out: &Tensor, - size: impl IntList, - stride: impl IntList, - storage_offset: impl Into>, - ) -> Tensor { - self.f_as_strided_copy_out(out, size, stride, storage_offset).unwrap() - } - - pub fn as_strided_scatter( - &self, - src: &Tensor, - size: impl IntList, - stride: impl IntList, - storage_offset: impl Into>, - ) -> Tensor { - self.f_as_strided_scatter(src, size, stride, storage_offset).unwrap() - } - - pub fn as_strided_scatter_out( - &self, - out: &Tensor, - src: &Tensor, - size: impl IntList, - stride: impl IntList, - storage_offset: impl Into>, - ) -> Tensor { - self.f_as_strided_scatter_out(out, src, size, stride, storage_offset).unwrap() - } - - pub fn asin(&self) -> Tensor { - self.f_asin().unwrap() - } - - pub fn asin_(&mut self) -> Tensor { - self.f_asin_().unwrap() - } - - pub fn asin_out(&self, out: &Tensor) -> Tensor { - self.f_asin_out(out).unwrap() - } - - pub fn asinh(&self) -> Tensor { - self.f_asinh().unwrap() - } - - pub fn asinh_(&mut self) -> Tensor { - self.f_asinh_().unwrap() - } - - pub fn asinh_out(&self, out: &Tensor) -> Tensor { - self.f_asinh_out(out).unwrap() - } - - pub fn atan(&self) -> Tensor { - self.f_atan().unwrap() - } - - pub fn atan2(&self, other: &Tensor) -> Tensor { - self.f_atan2(other).unwrap() - } - - pub fn atan2_(&mut self, other: &Tensor) -> Tensor { - self.f_atan2_(other).unwrap() - } - - pub fn atan2_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_atan2_out(out, other).unwrap() - } - - pub fn atan_(&mut self) -> Tensor { - self.f_atan_().unwrap() - } - - pub fn atan_out(&self, out: &Tensor) -> Tensor { - self.f_atan_out(out).unwrap() - } - - pub fn atanh(&self) -> Tensor { - self.f_atanh().unwrap() - } - - pub fn atanh_(&mut self) -> Tensor { - self.f_atanh_().unwrap() - } - - pub fn atanh_out(&self, out: &Tensor) -> Tensor { - self.f_atanh_out(out).unwrap() - } - - pub fn atleast_1d(&self) -> Tensor { - self.f_atleast_1d().unwrap() - } - - pub fn atleast_1d_sequence>(tensors: &[T]) -> Vec { - Tensor::f_atleast_1d_sequence(tensors).unwrap() - } - - pub fn atleast_2d(&self) -> Tensor { - self.f_atleast_2d().unwrap() - } - - pub fn atleast_2d_sequence>(tensors: &[T]) -> Vec { - Tensor::f_atleast_2d_sequence(tensors).unwrap() - } - - pub fn atleast_3d(&self) -> Tensor { - self.f_atleast_3d().unwrap() - } - - pub fn atleast_3d_sequence>(tensors: &[T]) -> Vec { - Tensor::f_atleast_3d_sequence(tensors).unwrap() - } - - pub fn avg_pool1d( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ceil_mode: bool, - count_include_pad: bool, - ) -> Tensor { - self.f_avg_pool1d(kernel_size, stride, padding, ceil_mode, count_include_pad).unwrap() - } - - pub fn avg_pool1d_out( - &self, - out: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ceil_mode: bool, - count_include_pad: bool, - ) -> Tensor { - self.f_avg_pool1d_out(out, kernel_size, stride, padding, ceil_mode, count_include_pad) - .unwrap() - } - - pub fn avg_pool2d( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ceil_mode: bool, - count_include_pad: bool, - divisor_override: impl Into>, - ) -> Tensor { - self.f_avg_pool2d( - kernel_size, - stride, - padding, - ceil_mode, - count_include_pad, - divisor_override, - ) - .unwrap() - } - - pub fn avg_pool2d_backward( - &self, - grad_output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ceil_mode: bool, - count_include_pad: bool, - divisor_override: impl Into>, - ) -> Tensor { - self.f_avg_pool2d_backward( - grad_output, - kernel_size, - stride, - padding, - ceil_mode, - count_include_pad, - divisor_override, - ) - .unwrap() - } - - pub fn avg_pool2d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ceil_mode: bool, - count_include_pad: bool, - divisor_override: impl Into>, - ) -> Tensor { - self.f_avg_pool2d_backward_grad_input( - grad_input, - grad_output, - kernel_size, - stride, - padding, - ceil_mode, - count_include_pad, - divisor_override, - ) - .unwrap() - } - - pub fn avg_pool2d_out( - &self, - out: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ceil_mode: bool, - count_include_pad: bool, - divisor_override: impl Into>, - ) -> Tensor { - self.f_avg_pool2d_out( - out, - kernel_size, - stride, - padding, - ceil_mode, - count_include_pad, - divisor_override, - ) - .unwrap() - } - - pub fn avg_pool3d( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ceil_mode: bool, - count_include_pad: bool, - divisor_override: impl Into>, - ) -> Tensor { - self.f_avg_pool3d( - kernel_size, - stride, - padding, - ceil_mode, - count_include_pad, - divisor_override, - ) - .unwrap() - } - - pub fn avg_pool3d_backward( - &self, - grad_output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ceil_mode: bool, - count_include_pad: bool, - divisor_override: impl Into>, - ) -> Tensor { - self.f_avg_pool3d_backward( - grad_output, - kernel_size, - stride, - padding, - ceil_mode, - count_include_pad, - divisor_override, - ) - .unwrap() - } - - pub fn avg_pool3d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ceil_mode: bool, - count_include_pad: bool, - divisor_override: impl Into>, - ) -> Tensor { - self.f_avg_pool3d_backward_grad_input( - grad_input, - grad_output, - kernel_size, - stride, - padding, - ceil_mode, - count_include_pad, - divisor_override, - ) - .unwrap() - } - - pub fn avg_pool3d_out( - &self, - out: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ceil_mode: bool, - count_include_pad: bool, - divisor_override: impl Into>, - ) -> Tensor { - self.f_avg_pool3d_out( - out, - kernel_size, - stride, - padding, - ceil_mode, - count_include_pad, - divisor_override, - ) - .unwrap() - } - - pub fn baddbmm>( - &self, - batch1: &Tensor, - batch2: &Tensor, - beta: S, - alpha: S, - ) -> Tensor { - self.f_baddbmm(batch1, batch2, beta, alpha).unwrap() - } - - pub fn baddbmm_(&mut self, batch1: &Tensor, batch2: &Tensor) -> Tensor { - self.f_baddbmm_(batch1, batch2).unwrap() - } - - pub fn baddbmm_dtype>( - &self, - batch1: &Tensor, - batch2: &Tensor, - out_dtype: Kind, - beta: S, - alpha: S, - ) -> Tensor { - self.f_baddbmm_dtype(batch1, batch2, out_dtype, beta, alpha).unwrap() - } - - pub fn baddbmm_dtype_out( - &self, - out: &Tensor, - batch1: &Tensor, - batch2: &Tensor, - out_dtype: Kind, - ) -> Tensor { - self.f_baddbmm_dtype_out(out, batch1, batch2, out_dtype).unwrap() - } - - pub fn baddbmm_out(&self, out: &Tensor, batch1: &Tensor, batch2: &Tensor) -> Tensor { - self.f_baddbmm_out(out, batch1, batch2).unwrap() - } - - pub fn bartlett_window(window_length: i64, options: (Kind, Device)) -> Tensor { - Tensor::f_bartlett_window(window_length, options).unwrap() - } - - pub fn bartlett_window_out(out: &Tensor, window_length: i64) -> Tensor { - Tensor::f_bartlett_window_out(out, window_length).unwrap() - } - - pub fn bartlett_window_periodic( - window_length: i64, - periodic: bool, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_bartlett_window_periodic(window_length, periodic, options).unwrap() - } - - pub fn bartlett_window_periodic_out( - out: &Tensor, - window_length: i64, - periodic: bool, - ) -> Tensor { - Tensor::f_bartlett_window_periodic_out(out, window_length, periodic).unwrap() - } - - pub fn batch_norm>( - &self, - weight: Option, - bias: Option, - running_mean: Option, - running_var: Option, - training: bool, - momentum: f64, - eps: f64, - cudnn_enabled: bool, - ) -> Tensor { - self.f_batch_norm( - weight, - bias, - running_mean, - running_var, - training, - momentum, - eps, - cudnn_enabled, - ) - .unwrap() - } - - pub fn batch_norm_backward_elemt>( - &self, - grad_out: &Tensor, - mean: &Tensor, - invstd: &Tensor, - weight: Option, - sum_dy: &Tensor, - sum_dy_xmu: &Tensor, - count: &Tensor, - ) -> Tensor { - self.f_batch_norm_backward_elemt(grad_out, mean, invstd, weight, sum_dy, sum_dy_xmu, count) - .unwrap() - } - - pub fn batch_norm_backward_elemt_out>( - &self, - out: &Tensor, - grad_out: &Tensor, - mean: &Tensor, - invstd: &Tensor, - weight: Option, - sum_dy: &Tensor, - sum_dy_xmu: &Tensor, - count: &Tensor, - ) -> Tensor { - self.f_batch_norm_backward_elemt_out( - out, grad_out, mean, invstd, weight, sum_dy, sum_dy_xmu, count, - ) - .unwrap() - } - - pub fn batch_norm_backward_reduce>( - &self, - grad_out: &Tensor, - mean: &Tensor, - invstd: &Tensor, - weight: Option, - input_g: bool, - weight_g: bool, - bias_g: bool, - ) -> (Tensor, Tensor, Tensor, Tensor) { - self.f_batch_norm_backward_reduce(grad_out, mean, invstd, weight, input_g, weight_g, bias_g) - .unwrap() - } - - pub fn batch_norm_backward_reduce_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - out3: &Tensor, - grad_out: &Tensor, - mean: &Tensor, - invstd: &Tensor, - weight: Option, - input_g: bool, - weight_g: bool, - bias_g: bool, - ) -> (Tensor, Tensor, Tensor, Tensor) { - self.f_batch_norm_backward_reduce_out( - out0, out1, out2, out3, grad_out, mean, invstd, weight, input_g, weight_g, bias_g, - ) - .unwrap() - } - - pub fn batch_norm_elemt>( - &self, - weight: Option, - bias: Option, - mean: &Tensor, - invstd: &Tensor, - eps: f64, - ) -> Tensor { - self.f_batch_norm_elemt(weight, bias, mean, invstd, eps).unwrap() - } - - pub fn batch_norm_elemt_out>( - &self, - out: &Tensor, - weight: Option, - bias: Option, - mean: &Tensor, - invstd: &Tensor, - eps: f64, - ) -> Tensor { - self.f_batch_norm_elemt_out(out, weight, bias, mean, invstd, eps).unwrap() - } - - pub fn batch_norm_gather_stats>( - &self, - mean: &Tensor, - invstd: &Tensor, - running_mean: Option, - running_var: Option, - momentum: f64, - eps: f64, - count: i64, - ) -> (Tensor, Tensor) { - self.f_batch_norm_gather_stats( - mean, - invstd, - running_mean, - running_var, - momentum, - eps, - count, - ) - .unwrap() - } - - pub fn batch_norm_gather_stats_out>( - &self, - out0: &Tensor, - out1: &Tensor, - mean: &Tensor, - invstd: &Tensor, - running_mean: Option, - running_var: Option, - momentum: f64, - eps: f64, - count: i64, - ) -> (Tensor, Tensor) { - self.f_batch_norm_gather_stats_out( - out0, - out1, - mean, - invstd, - running_mean, - running_var, - momentum, - eps, - count, - ) - .unwrap() - } - - pub fn batch_norm_gather_stats_with_counts>( - &self, - mean: &Tensor, - invstd: &Tensor, - running_mean: Option, - running_var: Option, - momentum: f64, - eps: f64, - counts: &Tensor, - ) -> (Tensor, Tensor) { - self.f_batch_norm_gather_stats_with_counts( - mean, - invstd, - running_mean, - running_var, - momentum, - eps, - counts, - ) - .unwrap() - } - - pub fn batch_norm_gather_stats_with_counts_out>( - &self, - out0: &Tensor, - out1: &Tensor, - mean: &Tensor, - invstd: &Tensor, - running_mean: Option, - running_var: Option, - momentum: f64, - eps: f64, - counts: &Tensor, - ) -> (Tensor, Tensor) { - self.f_batch_norm_gather_stats_with_counts_out( - out0, - out1, - mean, - invstd, - running_mean, - running_var, - momentum, - eps, - counts, - ) - .unwrap() - } - - pub fn batch_norm_stats(&self, eps: f64) -> (Tensor, Tensor) { - self.f_batch_norm_stats(eps).unwrap() - } - - pub fn batch_norm_stats_out(&self, out0: &Tensor, out1: &Tensor, eps: f64) -> (Tensor, Tensor) { - self.f_batch_norm_stats_out(out0, out1, eps).unwrap() - } - - pub fn batch_norm_update_stats>( - &self, - running_mean: Option, - running_var: Option, - momentum: f64, - ) -> (Tensor, Tensor) { - self.f_batch_norm_update_stats(running_mean, running_var, momentum).unwrap() - } - - pub fn batch_norm_update_stats_out>( - &self, - out0: &Tensor, - out1: &Tensor, - running_mean: Option, - running_var: Option, - momentum: f64, - ) -> (Tensor, Tensor) { - self.f_batch_norm_update_stats_out(out0, out1, running_mean, running_var, momentum).unwrap() - } - - pub fn bernoulli(&self) -> Tensor { - self.f_bernoulli().unwrap() - } - - pub fn bernoulli_(&mut self, p: &Tensor) -> Tensor { - self.f_bernoulli_(p).unwrap() - } - - pub fn bernoulli_float_(&mut self, p: f64) -> Tensor { - self.f_bernoulli_float_(p).unwrap() - } - - pub fn bernoulli_p(&self, p: f64) -> Tensor { - self.f_bernoulli_p(p).unwrap() - } - - pub fn bernoulli_tensor(&self, p: &Tensor) -> Tensor { - self.f_bernoulli_tensor(p).unwrap() - } - - pub fn bilinear>( - input1: &Tensor, - input2: &Tensor, - weight: &Tensor, - bias: Option, - ) -> Tensor { - Tensor::f_bilinear(input1, input2, weight, bias).unwrap() - } - - pub fn binary_cross_entropy>( - &self, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ) -> Tensor { - self.f_binary_cross_entropy(target, weight, reduction).unwrap() - } - - pub fn binary_cross_entropy_backward>( - &self, - grad_output: &Tensor, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ) -> Tensor { - self.f_binary_cross_entropy_backward(grad_output, target, weight, reduction).unwrap() - } - - pub fn binary_cross_entropy_backward_grad_input>( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ) -> Tensor { - self.f_binary_cross_entropy_backward_grad_input( - grad_input, - grad_output, - target, - weight, - reduction, - ) - .unwrap() - } - - pub fn binary_cross_entropy_out>( - &self, - out: &Tensor, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ) -> Tensor { - self.f_binary_cross_entropy_out(out, target, weight, reduction).unwrap() - } - - pub fn binary_cross_entropy_with_logits>( - &self, - target: &Tensor, - weight: Option, - pos_weight: Option, - reduction: crate::Reduction, - ) -> Tensor { - self.f_binary_cross_entropy_with_logits(target, weight, pos_weight, reduction).unwrap() - } - - pub fn binary_cross_entropy_with_logits_out>( - &self, - out: &Tensor, - target: &Tensor, - weight: Option, - pos_weight: Option, - reduction: crate::Reduction, - ) -> Tensor { - self.f_binary_cross_entropy_with_logits_out(out, target, weight, pos_weight, reduction) - .unwrap() - } - - pub fn bincount>(&self, weights: Option, minlength: i64) -> Tensor { - self.f_bincount(weights, minlength).unwrap() - } - - pub fn bincount_out>( - &self, - out: &Tensor, - weights: Option, - minlength: i64, - ) -> Tensor { - self.f_bincount_out(out, weights, minlength).unwrap() - } - - pub fn binomial(count: &Tensor, prob: &Tensor) -> Tensor { - Tensor::f_binomial(count, prob).unwrap() - } - - pub fn binomial_out(out: &Tensor, count: &Tensor, prob: &Tensor) -> Tensor { - Tensor::f_binomial_out(out, count, prob).unwrap() - } - - pub fn bitwise_and>(&self, other: S) -> Tensor { - self.f_bitwise_and(other).unwrap() - } - - pub fn bitwise_and_>(&mut self, other: S) -> Tensor { - self.f_bitwise_and_(other).unwrap() - } - - pub fn bitwise_and_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_bitwise_and_scalar_out(out, other).unwrap() - } - - pub fn bitwise_and_scalar_tensor>(self_scalar: S, other: &Tensor) -> Tensor { - Tensor::f_bitwise_and_scalar_tensor(self_scalar, other).unwrap() - } - - pub fn bitwise_and_scalar_tensor_out>( - out: &Tensor, - self_scalar: S, - other: &Tensor, - ) -> Tensor { - Tensor::f_bitwise_and_scalar_tensor_out(out, self_scalar, other).unwrap() - } - - pub fn bitwise_and_tensor(&self, other: &Tensor) -> Tensor { - self.f_bitwise_and_tensor(other).unwrap() - } - - pub fn bitwise_and_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_bitwise_and_tensor_(other).unwrap() - } - - pub fn bitwise_and_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_bitwise_and_tensor_out(out, other).unwrap() - } - - pub fn bitwise_left_shift(&self, other: &Tensor) -> Tensor { - self.f_bitwise_left_shift(other).unwrap() - } - - pub fn bitwise_left_shift_(&mut self, other: &Tensor) -> Tensor { - self.f_bitwise_left_shift_(other).unwrap() - } - - pub fn bitwise_left_shift_scalar_tensor>( - self_scalar: S, - other: &Tensor, - ) -> Tensor { - Tensor::f_bitwise_left_shift_scalar_tensor(self_scalar, other).unwrap() - } - - pub fn bitwise_left_shift_scalar_tensor_out>( - out: &Tensor, - self_scalar: S, - other: &Tensor, - ) -> Tensor { - Tensor::f_bitwise_left_shift_scalar_tensor_out(out, self_scalar, other).unwrap() - } - - pub fn bitwise_left_shift_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_bitwise_left_shift_tensor_out(out, other).unwrap() - } - - pub fn bitwise_left_shift_tensor_scalar>(&self, other: S) -> Tensor { - self.f_bitwise_left_shift_tensor_scalar(other).unwrap() - } - - pub fn bitwise_left_shift_tensor_scalar_>(&mut self, other: S) -> Tensor { - self.f_bitwise_left_shift_tensor_scalar_(other).unwrap() - } - - pub fn bitwise_left_shift_tensor_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Tensor { - self.f_bitwise_left_shift_tensor_scalar_out(out, other).unwrap() - } - - pub fn bitwise_not(&self) -> Tensor { - self.f_bitwise_not().unwrap() - } - - pub fn bitwise_not_(&mut self) -> Tensor { - self.f_bitwise_not_().unwrap() - } - - pub fn bitwise_not_out(&self, out: &Tensor) -> Tensor { - self.f_bitwise_not_out(out).unwrap() - } - - pub fn bitwise_or>(&self, other: S) -> Tensor { - self.f_bitwise_or(other).unwrap() - } - - pub fn bitwise_or_>(&mut self, other: S) -> Tensor { - self.f_bitwise_or_(other).unwrap() - } - - pub fn bitwise_or_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_bitwise_or_scalar_out(out, other).unwrap() - } - - pub fn bitwise_or_scalar_tensor>(self_scalar: S, other: &Tensor) -> Tensor { - Tensor::f_bitwise_or_scalar_tensor(self_scalar, other).unwrap() - } - - pub fn bitwise_or_scalar_tensor_out>( - out: &Tensor, - self_scalar: S, - other: &Tensor, - ) -> Tensor { - Tensor::f_bitwise_or_scalar_tensor_out(out, self_scalar, other).unwrap() - } - - pub fn bitwise_or_tensor(&self, other: &Tensor) -> Tensor { - self.f_bitwise_or_tensor(other).unwrap() - } - - pub fn bitwise_or_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_bitwise_or_tensor_(other).unwrap() - } - - pub fn bitwise_or_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_bitwise_or_tensor_out(out, other).unwrap() - } - - pub fn bitwise_right_shift(&self, other: &Tensor) -> Tensor { - self.f_bitwise_right_shift(other).unwrap() - } - - pub fn bitwise_right_shift_(&mut self, other: &Tensor) -> Tensor { - self.f_bitwise_right_shift_(other).unwrap() - } - - pub fn bitwise_right_shift_scalar_tensor>( - self_scalar: S, - other: &Tensor, - ) -> Tensor { - Tensor::f_bitwise_right_shift_scalar_tensor(self_scalar, other).unwrap() - } - - pub fn bitwise_right_shift_scalar_tensor_out>( - out: &Tensor, - self_scalar: S, - other: &Tensor, - ) -> Tensor { - Tensor::f_bitwise_right_shift_scalar_tensor_out(out, self_scalar, other).unwrap() - } - - pub fn bitwise_right_shift_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_bitwise_right_shift_tensor_out(out, other).unwrap() - } - - pub fn bitwise_right_shift_tensor_scalar>(&self, other: S) -> Tensor { - self.f_bitwise_right_shift_tensor_scalar(other).unwrap() - } - - pub fn bitwise_right_shift_tensor_scalar_>(&mut self, other: S) -> Tensor { - self.f_bitwise_right_shift_tensor_scalar_(other).unwrap() - } - - pub fn bitwise_right_shift_tensor_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Tensor { - self.f_bitwise_right_shift_tensor_scalar_out(out, other).unwrap() - } - - pub fn bitwise_xor>(&self, other: S) -> Tensor { - self.f_bitwise_xor(other).unwrap() - } - - pub fn bitwise_xor_>(&mut self, other: S) -> Tensor { - self.f_bitwise_xor_(other).unwrap() - } - - pub fn bitwise_xor_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_bitwise_xor_scalar_out(out, other).unwrap() - } - - pub fn bitwise_xor_scalar_tensor>(self_scalar: S, other: &Tensor) -> Tensor { - Tensor::f_bitwise_xor_scalar_tensor(self_scalar, other).unwrap() - } - - pub fn bitwise_xor_scalar_tensor_out>( - out: &Tensor, - self_scalar: S, - other: &Tensor, - ) -> Tensor { - Tensor::f_bitwise_xor_scalar_tensor_out(out, self_scalar, other).unwrap() - } - - pub fn bitwise_xor_tensor(&self, other: &Tensor) -> Tensor { - self.f_bitwise_xor_tensor(other).unwrap() - } - - pub fn bitwise_xor_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_bitwise_xor_tensor_(other).unwrap() - } - - pub fn bitwise_xor_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_bitwise_xor_tensor_out(out, other).unwrap() - } - - pub fn blackman_window(window_length: i64, options: (Kind, Device)) -> Tensor { - Tensor::f_blackman_window(window_length, options).unwrap() - } - - pub fn blackman_window_out(out: &Tensor, window_length: i64) -> Tensor { - Tensor::f_blackman_window_out(out, window_length).unwrap() - } - - pub fn blackman_window_periodic( - window_length: i64, - periodic: bool, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_blackman_window_periodic(window_length, periodic, options).unwrap() - } - - pub fn blackman_window_periodic_out( - out: &Tensor, - window_length: i64, - periodic: bool, - ) -> Tensor { - Tensor::f_blackman_window_periodic_out(out, window_length, periodic).unwrap() - } - - pub fn block_diag>(tensors: &[T]) -> Tensor { - Tensor::f_block_diag(tensors).unwrap() - } - - pub fn block_diag_out>(out: &Tensor, tensors: &[T]) -> Tensor { - Tensor::f_block_diag_out(out, tensors).unwrap() - } - - pub fn bmm(&self, mat2: &Tensor) -> Tensor { - self.f_bmm(mat2).unwrap() - } - - pub fn bmm_dtype(&self, mat2: &Tensor, out_dtype: Kind) -> Tensor { - self.f_bmm_dtype(mat2, out_dtype).unwrap() - } - - pub fn bmm_dtype_out(&self, out: &Tensor, mat2: &Tensor, out_dtype: Kind) -> Tensor { - self.f_bmm_dtype_out(out, mat2, out_dtype).unwrap() - } - - pub fn bmm_out(&self, out: &Tensor, mat2: &Tensor) -> Tensor { - self.f_bmm_out(out, mat2).unwrap() - } - - pub fn broadcast_tensors>(tensors: &[T]) -> Vec { - Tensor::f_broadcast_tensors(tensors).unwrap() - } - - pub fn broadcast_to(&self, size: impl IntList) -> Tensor { - self.f_broadcast_to(size).unwrap() - } - - pub fn bucketize(&self, boundaries: &Tensor, out_int32: bool, right: bool) -> Tensor { - self.f_bucketize(boundaries, out_int32, right).unwrap() - } - - pub fn bucketize_scalar>( - self_scalar: S, - boundaries: &Tensor, - out_int32: bool, - right: bool, - ) -> Tensor { - Tensor::f_bucketize_scalar(self_scalar, boundaries, out_int32, right).unwrap() - } - - pub fn bucketize_scalar_out>( - out: &Tensor, - self_scalar: S, - boundaries: &Tensor, - out_int32: bool, - right: bool, - ) -> Tensor { - Tensor::f_bucketize_scalar_out(out, self_scalar, boundaries, out_int32, right).unwrap() - } - - pub fn bucketize_tensor_out( - &self, - out: &Tensor, - boundaries: &Tensor, - out_int32: bool, - right: bool, - ) -> Tensor { - self.f_bucketize_tensor_out(out, boundaries, out_int32, right).unwrap() - } - - pub fn can_cast(from_: Kind, to: Kind) -> bool { - Tensor::f_can_cast(from_, to).unwrap() - } - - pub fn cartesian_prod>(tensors: &[T]) -> Tensor { - Tensor::f_cartesian_prod(tensors).unwrap() - } - - pub fn cat>(tensors: &[T], dim: i64) -> Tensor { - Tensor::f_cat(tensors, dim).unwrap() - } - - pub fn cat_out>(out: &Tensor, tensors: &[T], dim: i64) -> Tensor { - Tensor::f_cat_out(out, tensors, dim).unwrap() - } - - pub fn cauchy(&self, median: f64, sigma: f64) -> Tensor { - self.f_cauchy(median, sigma).unwrap() - } - - pub fn cauchy_(&mut self, median: f64, sigma: f64) -> Tensor { - self.f_cauchy_(median, sigma).unwrap() - } - - pub fn cauchy_out(&self, out: &Tensor, median: f64, sigma: f64) -> Tensor { - self.f_cauchy_out(out, median, sigma).unwrap() - } - - pub fn ccol_indices(&self) -> Tensor { - self.f_ccol_indices().unwrap() - } - - pub fn ccol_indices_copy(&self) -> Tensor { - self.f_ccol_indices_copy().unwrap() - } - - pub fn ccol_indices_copy_out(&self, out: &Tensor) -> Tensor { - self.f_ccol_indices_copy_out(out).unwrap() - } - - pub fn cdist(x1: &Tensor, x2: &Tensor, p: f64, compute_mode: impl Into>) -> Tensor { - Tensor::f_cdist(x1, x2, p, compute_mode).unwrap() - } - - pub fn ceil(&self) -> Tensor { - self.f_ceil().unwrap() - } - - pub fn ceil_(&mut self) -> Tensor { - self.f_ceil_().unwrap() - } - - pub fn ceil_out(&self, out: &Tensor) -> Tensor { - self.f_ceil_out(out).unwrap() - } - - pub fn celu(&self) -> Tensor { - self.f_celu().unwrap() - } - - pub fn celu_(&mut self) -> Tensor { - self.f_celu_().unwrap() - } - - pub fn celu_out(&self, out: &Tensor) -> Tensor { - self.f_celu_out(out).unwrap() - } - - pub fn chain_matmul>(matrices: &[T]) -> Tensor { - Tensor::f_chain_matmul(matrices).unwrap() - } - - pub fn chain_matmul_out>(out: &Tensor, matrices: &[T]) -> Tensor { - Tensor::f_chain_matmul_out(out, matrices).unwrap() - } - - pub fn chalf(&self) -> Tensor { - self.f_chalf().unwrap() - } - - pub fn channel_shuffle(&self, groups: i64) -> Tensor { - self.f_channel_shuffle(groups).unwrap() - } - - pub fn channel_shuffle_out(&self, out: &Tensor, groups: i64) -> Tensor { - self.f_channel_shuffle_out(out, groups).unwrap() - } - - pub fn cholesky(&self, upper: bool) -> Tensor { - self.f_cholesky(upper).unwrap() - } - - pub fn cholesky_inverse(&self, upper: bool) -> Tensor { - self.f_cholesky_inverse(upper).unwrap() - } - - pub fn cholesky_inverse_out(&self, out: &Tensor, upper: bool) -> Tensor { - self.f_cholesky_inverse_out(out, upper).unwrap() - } - - pub fn cholesky_out(&self, out: &Tensor, upper: bool) -> Tensor { - self.f_cholesky_out(out, upper).unwrap() - } - - pub fn cholesky_solve(&self, input2: &Tensor, upper: bool) -> Tensor { - self.f_cholesky_solve(input2, upper).unwrap() - } - - pub fn cholesky_solve_out(&self, out: &Tensor, input2: &Tensor, upper: bool) -> Tensor { - self.f_cholesky_solve_out(out, input2, upper).unwrap() - } - - pub fn choose_qparams_optimized( - &self, - numel: i64, - n_bins: i64, - ratio: f64, - bit_width: i64, - ) -> (Tensor, Tensor) { - self.f_choose_qparams_optimized(numel, n_bins, ratio, bit_width).unwrap() - } - - pub fn chunk(&self, chunks: i64, dim: i64) -> Vec { - self.f_chunk(chunks, dim).unwrap() - } - - pub fn clamp>(&self, min: S, max: S) -> Tensor { - self.f_clamp(min, max).unwrap() - } - - pub fn clamp_>(&mut self, min: S, max: S) -> Tensor { - self.f_clamp_(min, max).unwrap() - } - - pub fn clamp_max>(&self, max: S) -> Tensor { - self.f_clamp_max(max).unwrap() - } - - pub fn clamp_max_>(&mut self, max: S) -> Tensor { - self.f_clamp_max_(max).unwrap() - } - - pub fn clamp_max_out>(&self, out: &Tensor, max: S) -> Tensor { - self.f_clamp_max_out(out, max).unwrap() - } - - pub fn clamp_max_tensor(&self, max: &Tensor) -> Tensor { - self.f_clamp_max_tensor(max).unwrap() - } - - pub fn clamp_max_tensor_(&mut self, max: &Tensor) -> Tensor { - self.f_clamp_max_tensor_(max).unwrap() - } - - pub fn clamp_max_tensor_out(&self, out: &Tensor, max: &Tensor) -> Tensor { - self.f_clamp_max_tensor_out(out, max).unwrap() - } - - pub fn clamp_min>(&self, min: S) -> Tensor { - self.f_clamp_min(min).unwrap() - } - - pub fn clamp_min_>(&mut self, min: S) -> Tensor { - self.f_clamp_min_(min).unwrap() - } - - pub fn clamp_min_out>(&self, out: &Tensor, min: S) -> Tensor { - self.f_clamp_min_out(out, min).unwrap() - } - - pub fn clamp_min_tensor(&self, min: &Tensor) -> Tensor { - self.f_clamp_min_tensor(min).unwrap() - } - - pub fn clamp_min_tensor_(&mut self, min: &Tensor) -> Tensor { - self.f_clamp_min_tensor_(min).unwrap() - } - - pub fn clamp_min_tensor_out(&self, out: &Tensor, min: &Tensor) -> Tensor { - self.f_clamp_min_tensor_out(out, min).unwrap() - } - - pub fn clamp_out>(&self, out: &Tensor, min: S, max: S) -> Tensor { - self.f_clamp_out(out, min, max).unwrap() - } - - pub fn clamp_tensor>(&self, min: Option, max: Option) -> Tensor { - self.f_clamp_tensor(min, max).unwrap() - } - - pub fn clamp_tensor_>(&mut self, min: Option, max: Option) -> Tensor { - self.f_clamp_tensor_(min, max).unwrap() - } - - pub fn clamp_tensor_out>( - &self, - out: &Tensor, - min: Option, - max: Option, - ) -> Tensor { - self.f_clamp_tensor_out(out, min, max).unwrap() - } - - pub fn clip>(&self, min: S, max: S) -> Tensor { - self.f_clip(min, max).unwrap() - } - - pub fn clip_>(&mut self, min: S, max: S) -> Tensor { - self.f_clip_(min, max).unwrap() - } - - pub fn clip_out>(&self, out: &Tensor, min: S, max: S) -> Tensor { - self.f_clip_out(out, min, max).unwrap() - } - - pub fn clip_tensor>(&self, min: Option, max: Option) -> Tensor { - self.f_clip_tensor(min, max).unwrap() - } - - pub fn clip_tensor_>(&mut self, min: Option, max: Option) -> Tensor { - self.f_clip_tensor_(min, max).unwrap() - } - - pub fn clip_tensor_out>( - &self, - out: &Tensor, - min: Option, - max: Option, - ) -> Tensor { - self.f_clip_tensor_out(out, min, max).unwrap() - } - - pub fn clone(&self, out: &Tensor) -> Tensor { - self.f_clone(out).unwrap() - } - - pub fn coalesce(&self) -> Tensor { - self.f_coalesce().unwrap() - } - - pub fn col2im( - &self, - output_size: impl IntList, - kernel_size: impl IntList, - dilation: impl IntList, - padding: impl IntList, - stride: impl IntList, - ) -> Tensor { - self.f_col2im(output_size, kernel_size, dilation, padding, stride).unwrap() - } - - pub fn col2im_out( - &self, - out: &Tensor, - output_size: impl IntList, - kernel_size: impl IntList, - dilation: impl IntList, - padding: impl IntList, - stride: impl IntList, - ) -> Tensor { - self.f_col2im_out(out, output_size, kernel_size, dilation, padding, stride).unwrap() - } - - pub fn col_indices(&self) -> Tensor { - self.f_col_indices().unwrap() - } - - pub fn col_indices_copy(&self) -> Tensor { - self.f_col_indices_copy().unwrap() - } - - pub fn col_indices_copy_out(&self, out: &Tensor) -> Tensor { - self.f_col_indices_copy_out(out).unwrap() - } - - pub fn column_stack>(tensors: &[T]) -> Tensor { - Tensor::f_column_stack(tensors).unwrap() - } - - pub fn column_stack_out>(out: &Tensor, tensors: &[T]) -> Tensor { - Tensor::f_column_stack_out(out, tensors).unwrap() - } - - pub fn combinations(&self, r: i64, with_replacement: bool) -> Tensor { - self.f_combinations(r, with_replacement).unwrap() - } - - pub fn complex(real: &Tensor, imag: &Tensor) -> Tensor { - Tensor::f_complex(real, imag).unwrap() - } - - pub fn complex_out(out: &Tensor, real: &Tensor, imag: &Tensor) -> Tensor { - Tensor::f_complex_out(out, real, imag).unwrap() - } - - pub fn concat>(tensors: &[T], dim: i64) -> Tensor { - Tensor::f_concat(tensors, dim).unwrap() - } - - pub fn concat_out>(out: &Tensor, tensors: &[T], dim: i64) -> Tensor { - Tensor::f_concat_out(out, tensors, dim).unwrap() - } - - pub fn concatenate>(tensors: &[T], dim: i64) -> Tensor { - Tensor::f_concatenate(tensors, dim).unwrap() - } - - pub fn concatenate_out>(out: &Tensor, tensors: &[T], dim: i64) -> Tensor { - Tensor::f_concatenate_out(out, tensors, dim).unwrap() - } - - pub fn conj(&self) -> Tensor { - self.f_conj().unwrap() - } - - pub fn conj_physical(&self) -> Tensor { - self.f_conj_physical().unwrap() - } - - pub fn conj_physical_(&mut self) -> Tensor { - self.f_conj_physical_().unwrap() - } - - pub fn conj_physical_out(&self, out: &Tensor) -> Tensor { - self.f_conj_physical_out(out).unwrap() - } - - pub fn constant_pad_nd(&self, pad: impl IntList) -> Tensor { - self.f_constant_pad_nd(pad).unwrap() - } - - pub fn constant_pad_nd_out(&self, out: &Tensor, pad: impl IntList) -> Tensor { - self.f_constant_pad_nd_out(out, pad).unwrap() - } - - pub fn contiguous(&self) -> Tensor { - self.f_contiguous().unwrap() - } - - pub fn conv1d>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Tensor { - self.f_conv1d(weight, bias, stride, padding, dilation, groups).unwrap() - } - - pub fn conv1d_padding>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: &str, - dilation: impl IntList, - groups: i64, - ) -> Tensor { - self.f_conv1d_padding(weight, bias, stride, padding, dilation, groups).unwrap() - } - - pub fn conv2d>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Tensor { - self.f_conv2d(weight, bias, stride, padding, dilation, groups).unwrap() - } - - pub fn conv2d_padding>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: &str, - dilation: impl IntList, - groups: i64, - ) -> Tensor { - self.f_conv2d_padding(weight, bias, stride, padding, dilation, groups).unwrap() - } - - pub fn conv3d>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Tensor { - self.f_conv3d(weight, bias, stride, padding, dilation, groups).unwrap() - } - - pub fn conv3d_padding>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: &str, - dilation: impl IntList, - groups: i64, - ) -> Tensor { - self.f_conv3d_padding(weight, bias, stride, padding, dilation, groups).unwrap() - } - - pub fn conv_depthwise3d>( - &self, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ) -> Tensor { - self.f_conv_depthwise3d(weight, kernel_size, bias, stride, padding, dilation).unwrap() - } - - pub fn conv_depthwise3d_out>( - &self, - out: &Tensor, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ) -> Tensor { - self.f_conv_depthwise3d_out(out, weight, kernel_size, bias, stride, padding, dilation) - .unwrap() - } - - pub fn conv_tbc(&self, weight: &Tensor, bias: &Tensor, pad: i64) -> Tensor { - self.f_conv_tbc(weight, bias, pad).unwrap() - } - - pub fn conv_tbc_backward( - &self, - input: &Tensor, - weight: &Tensor, - bias: &Tensor, - pad: i64, - ) -> (Tensor, Tensor, Tensor) { - self.f_conv_tbc_backward(input, weight, bias, pad).unwrap() - } - - pub fn conv_tbc_out(&self, out: &Tensor, weight: &Tensor, bias: &Tensor, pad: i64) -> Tensor { - self.f_conv_tbc_out(out, weight, bias, pad).unwrap() - } - - pub fn conv_transpose1d>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - output_padding: impl IntList, - groups: i64, - dilation: impl IntList, - ) -> Tensor { - self.f_conv_transpose1d(weight, bias, stride, padding, output_padding, groups, dilation) - .unwrap() - } - - pub fn conv_transpose2d>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - output_padding: impl IntList, - groups: i64, - dilation: impl IntList, - ) -> Tensor { - self.f_conv_transpose2d(weight, bias, stride, padding, output_padding, groups, dilation) - .unwrap() - } - - pub fn conv_transpose3d>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - output_padding: impl IntList, - groups: i64, - dilation: impl IntList, - ) -> Tensor { - self.f_conv_transpose3d(weight, bias, stride, padding, output_padding, groups, dilation) - .unwrap() - } - - pub fn convolution>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - transposed: bool, - output_padding: impl IntList, - groups: i64, - ) -> Tensor { - self.f_convolution( - weight, - bias, - stride, - padding, - dilation, - transposed, - output_padding, - groups, - ) - .unwrap() - } - - pub fn convolution_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - transposed: bool, - output_padding: impl IntList, - groups: i64, - ) -> Tensor { - self.f_convolution_out( - out, - weight, - bias, - stride, - padding, - dilation, - transposed, - output_padding, - groups, - ) - .unwrap() - } - - pub fn convolution_overrideable>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - transposed: bool, - output_padding: impl IntList, - groups: i64, - ) -> Tensor { - self.f_convolution_overrideable( - weight, - bias, - stride, - padding, - dilation, - transposed, - output_padding, - groups, - ) - .unwrap() - } - - pub fn convolution_overrideable_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - transposed: bool, - output_padding: impl IntList, - groups: i64, - ) -> Tensor { - self.f_convolution_overrideable_out( - out, - weight, - bias, - stride, - padding, - dilation, - transposed, - output_padding, - groups, - ) - .unwrap() - } - - pub fn copy_sparse_to_sparse(&self, src: &Tensor, non_blocking: bool) -> Tensor { - self.f_copy_sparse_to_sparse(src, non_blocking).unwrap() - } - - pub fn copy_sparse_to_sparse_(&mut self, src: &Tensor, non_blocking: bool) -> Tensor { - self.f_copy_sparse_to_sparse_(src, non_blocking).unwrap() - } - - pub fn copy_sparse_to_sparse_out( - &self, - out: &Tensor, - src: &Tensor, - non_blocking: bool, - ) -> Tensor { - self.f_copy_sparse_to_sparse_out(out, src, non_blocking).unwrap() - } - - pub fn copysign(&self, other: &Tensor) -> Tensor { - self.f_copysign(other).unwrap() - } - - pub fn copysign_(&mut self, other: &Tensor) -> Tensor { - self.f_copysign_(other).unwrap() - } - - pub fn copysign_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_copysign_out(out, other).unwrap() - } - - pub fn copysign_scalar>(&self, other: S) -> Tensor { - self.f_copysign_scalar(other).unwrap() - } - - pub fn copysign_scalar_>(&mut self, other: S) -> Tensor { - self.f_copysign_scalar_(other).unwrap() - } - - pub fn copysign_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_copysign_scalar_out(out, other).unwrap() - } - - pub fn corrcoef(&self) -> Tensor { - self.f_corrcoef().unwrap() - } - - pub fn cos(&self) -> Tensor { - self.f_cos().unwrap() - } - - pub fn cos_(&mut self) -> Tensor { - self.f_cos_().unwrap() - } - - pub fn cos_out(&self, out: &Tensor) -> Tensor { - self.f_cos_out(out).unwrap() - } - - pub fn cosh(&self) -> Tensor { - self.f_cosh().unwrap() - } - - pub fn cosh_(&mut self) -> Tensor { - self.f_cosh_().unwrap() - } - - pub fn cosh_out(&self, out: &Tensor) -> Tensor { - self.f_cosh_out(out).unwrap() - } - - pub fn cosine_embedding_loss( - input1: &Tensor, - input2: &Tensor, - target: &Tensor, - margin: f64, - reduction: crate::Reduction, - ) -> Tensor { - Tensor::f_cosine_embedding_loss(input1, input2, target, margin, reduction).unwrap() - } - - pub fn cosine_similarity(x1: &Tensor, x2: &Tensor, dim: i64, eps: f64) -> Tensor { - Tensor::f_cosine_similarity(x1, x2, dim, eps).unwrap() - } - - pub fn count_nonzero(&self, dim: impl Into>) -> Tensor { - self.f_count_nonzero(dim).unwrap() - } - - pub fn count_nonzero_dim_intlist(&self, dim: impl IntList) -> Tensor { - self.f_count_nonzero_dim_intlist(dim).unwrap() - } - - pub fn count_nonzero_dim_intlist_out(&self, out: &Tensor, dim: impl IntList) -> Tensor { - self.f_count_nonzero_dim_intlist_out(out, dim).unwrap() - } - - pub fn count_nonzero_out(&self, out: &Tensor, dim: impl Into>) -> Tensor { - self.f_count_nonzero_out(out, dim).unwrap() - } - - pub fn cov>( - &self, - correction: i64, - fweights: Option, - aweights: Option, - ) -> Tensor { - self.f_cov(correction, fweights, aweights).unwrap() - } - - pub fn cross(&self, other: &Tensor, dim: impl Into>) -> Tensor { - self.f_cross(other, dim).unwrap() - } - - pub fn cross_entropy_loss>( - &self, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ignore_index: i64, - label_smoothing: f64, - ) -> Tensor { - self.f_cross_entropy_loss(target, weight, reduction, ignore_index, label_smoothing).unwrap() - } - - pub fn cross_out(&self, out: &Tensor, other: &Tensor, dim: impl Into>) -> Tensor { - self.f_cross_out(out, other, dim).unwrap() - } - - pub fn crow_indices(&self) -> Tensor { - self.f_crow_indices().unwrap() - } - - pub fn crow_indices_copy(&self) -> Tensor { - self.f_crow_indices_copy().unwrap() - } - - pub fn crow_indices_copy_out(&self, out: &Tensor) -> Tensor { - self.f_crow_indices_copy_out(out).unwrap() - } - - pub fn ctc_loss( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: impl IntList, - target_lengths: impl IntList, - blank: i64, - reduction: crate::Reduction, - zero_infinity: bool, - ) -> Tensor { - Tensor::f_ctc_loss( - log_probs, - targets, - input_lengths, - target_lengths, - blank, - reduction, - zero_infinity, - ) - .unwrap() - } - - pub fn ctc_loss_tensor( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: &Tensor, - target_lengths: &Tensor, - blank: i64, - reduction: crate::Reduction, - zero_infinity: bool, - ) -> Tensor { - Tensor::f_ctc_loss_tensor( - log_probs, - targets, - input_lengths, - target_lengths, - blank, - reduction, - zero_infinity, - ) - .unwrap() - } - - pub fn cudnn_affine_grid_generator(theta: &Tensor, n: i64, c: i64, h: i64, w: i64) -> Tensor { - Tensor::f_cudnn_affine_grid_generator(theta, n, c, h, w).unwrap() - } - - pub fn cudnn_affine_grid_generator_backward( - grad: &Tensor, - n: i64, - c: i64, - h: i64, - w: i64, - ) -> Tensor { - Tensor::f_cudnn_affine_grid_generator_backward(grad, n, c, h, w).unwrap() - } - - pub fn cudnn_affine_grid_generator_backward_out( - out: &Tensor, - grad: &Tensor, - n: i64, - c: i64, - h: i64, - w: i64, - ) -> Tensor { - Tensor::f_cudnn_affine_grid_generator_backward_out(out, grad, n, c, h, w).unwrap() - } - - pub fn cudnn_affine_grid_generator_out( - out: &Tensor, - theta: &Tensor, - n: i64, - c: i64, - h: i64, - w: i64, - ) -> Tensor { - Tensor::f_cudnn_affine_grid_generator_out(out, theta, n, c, h, w).unwrap() - } - - pub fn cudnn_batch_norm>( - &self, - weight: &Tensor, - bias: Option, - running_mean: Option, - running_var: Option, - training: bool, - exponential_average_factor: f64, - epsilon: f64, - ) -> (Tensor, Tensor, Tensor, Tensor) { - self.f_cudnn_batch_norm( - weight, - bias, - running_mean, - running_var, - training, - exponential_average_factor, - epsilon, - ) - .unwrap() - } - - pub fn cudnn_batch_norm_backward>( - &self, - grad_output: &Tensor, - weight: &Tensor, - running_mean: Option, - running_var: Option, - save_mean: Option, - save_var: Option, - epsilon: f64, - reservespace: &Tensor, - ) -> (Tensor, Tensor, Tensor) { - self.f_cudnn_batch_norm_backward( - grad_output, - weight, - running_mean, - running_var, - save_mean, - save_var, - epsilon, - reservespace, - ) - .unwrap() - } - - pub fn cudnn_batch_norm_backward_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - grad_output: &Tensor, - weight: &Tensor, - running_mean: Option, - running_var: Option, - save_mean: Option, - save_var: Option, - epsilon: f64, - reservespace: &Tensor, - ) -> (Tensor, Tensor, Tensor) { - self.f_cudnn_batch_norm_backward_out( - out0, - out1, - out2, - grad_output, - weight, - running_mean, - running_var, - save_mean, - save_var, - epsilon, - reservespace, - ) - .unwrap() - } - - pub fn cudnn_batch_norm_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - out3: &Tensor, - weight: &Tensor, - bias: Option, - running_mean: Option, - running_var: Option, - training: bool, - exponential_average_factor: f64, - epsilon: f64, - ) -> (Tensor, Tensor, Tensor, Tensor) { - self.f_cudnn_batch_norm_out( - out0, - out1, - out2, - out3, - weight, - bias, - running_mean, - running_var, - training, - exponential_average_factor, - epsilon, - ) - .unwrap() - } - - pub fn cudnn_convolution( - &self, - weight: &Tensor, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - allow_tf32: bool, - ) -> Tensor { - self.f_cudnn_convolution( - weight, - padding, - stride, - dilation, - groups, - benchmark, - deterministic, - allow_tf32, - ) - .unwrap() - } - - pub fn cudnn_convolution_add_relu, S: Into>( - &self, - weight: &Tensor, - z: &Tensor, - alpha: S, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Tensor { - self.f_cudnn_convolution_add_relu(weight, z, alpha, bias, stride, padding, dilation, groups) - .unwrap() - } - - pub fn cudnn_convolution_add_relu_out, S: Into>( - &self, - out: &Tensor, - weight: &Tensor, - z: &Tensor, - alpha: S, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Tensor { - self.f_cudnn_convolution_add_relu_out( - out, weight, z, alpha, bias, stride, padding, dilation, groups, - ) - .unwrap() - } - - pub fn cudnn_convolution_out( - &self, - out: &Tensor, - weight: &Tensor, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - allow_tf32: bool, - ) -> Tensor { - self.f_cudnn_convolution_out( - out, - weight, - padding, - stride, - dilation, - groups, - benchmark, - deterministic, - allow_tf32, - ) - .unwrap() - } - - pub fn cudnn_convolution_relu>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Tensor { - self.f_cudnn_convolution_relu(weight, bias, stride, padding, dilation, groups).unwrap() - } - - pub fn cudnn_convolution_relu_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Tensor { - self.f_cudnn_convolution_relu_out(out, weight, bias, stride, padding, dilation, groups) - .unwrap() - } - - pub fn cudnn_convolution_transpose( - &self, - weight: &Tensor, - padding: impl IntList, - output_padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - allow_tf32: bool, - ) -> Tensor { - self.f_cudnn_convolution_transpose( - weight, - padding, - output_padding, - stride, - dilation, - groups, - benchmark, - deterministic, - allow_tf32, - ) - .unwrap() - } - - pub fn cudnn_convolution_transpose_out( - &self, - out: &Tensor, - weight: &Tensor, - padding: impl IntList, - output_padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - allow_tf32: bool, - ) -> Tensor { - self.f_cudnn_convolution_transpose_out( - out, - weight, - padding, - output_padding, - stride, - dilation, - groups, - benchmark, - deterministic, - allow_tf32, - ) - .unwrap() - } - - pub fn cudnn_grid_sampler(&self, grid: &Tensor) -> Tensor { - self.f_cudnn_grid_sampler(grid).unwrap() - } - - pub fn cudnn_grid_sampler_backward( - &self, - grid: &Tensor, - grad_output: &Tensor, - ) -> (Tensor, Tensor) { - self.f_cudnn_grid_sampler_backward(grid, grad_output).unwrap() - } - - pub fn cudnn_grid_sampler_backward_out( - &self, - out0: &Tensor, - out1: &Tensor, - grid: &Tensor, - grad_output: &Tensor, - ) -> (Tensor, Tensor) { - self.f_cudnn_grid_sampler_backward_out(out0, out1, grid, grad_output).unwrap() - } - - pub fn cudnn_grid_sampler_out(&self, out: &Tensor, grid: &Tensor) -> Tensor { - self.f_cudnn_grid_sampler_out(out, grid).unwrap() - } - - pub fn cudnn_is_acceptable(&self) -> bool { - self.f_cudnn_is_acceptable().unwrap() - } - - pub fn cummax(&self, dim: i64) -> (Tensor, Tensor) { - self.f_cummax(dim).unwrap() - } - - pub fn cummax_out(&self, values: &Tensor, indices: &Tensor, dim: i64) -> (Tensor, Tensor) { - self.f_cummax_out(values, indices, dim).unwrap() - } - - pub fn cummaxmin_backward(&self, grad: &Tensor, indices: &Tensor, dim: i64) -> Tensor { - self.f_cummaxmin_backward(grad, indices, dim).unwrap() - } - - pub fn cummin(&self, dim: i64) -> (Tensor, Tensor) { - self.f_cummin(dim).unwrap() - } - - pub fn cummin_out(&self, values: &Tensor, indices: &Tensor, dim: i64) -> (Tensor, Tensor) { - self.f_cummin_out(values, indices, dim).unwrap() - } - - pub fn cumprod(&self, dim: i64, dtype: impl Into>) -> Tensor { - self.f_cumprod(dim, dtype).unwrap() - } - - pub fn cumprod_(&mut self, dim: i64, dtype: impl Into>) -> Tensor { - self.f_cumprod_(dim, dtype).unwrap() - } - - pub fn cumprod_backward(&self, grad: &Tensor, dim: i64, output: &Tensor) -> Tensor { - self.f_cumprod_backward(grad, dim, output).unwrap() - } - - pub fn cumprod_out(&self, out: &Tensor, dim: i64, dtype: impl Into>) -> Tensor { - self.f_cumprod_out(out, dim, dtype).unwrap() - } - - pub fn cumsum(&self, dim: i64, dtype: impl Into>) -> Tensor { - self.f_cumsum(dim, dtype).unwrap() - } - - pub fn cumsum_(&mut self, dim: i64, dtype: impl Into>) -> Tensor { - self.f_cumsum_(dim, dtype).unwrap() - } - - pub fn cumsum_out(&self, out: &Tensor, dim: i64, dtype: impl Into>) -> Tensor { - self.f_cumsum_out(out, dim, dtype).unwrap() - } - - pub fn cumulative_trapezoid(y: &Tensor, dim: i64) -> Tensor { - Tensor::f_cumulative_trapezoid(y, dim).unwrap() - } - - pub fn cumulative_trapezoid_x(y: &Tensor, x: &Tensor, dim: i64) -> Tensor { - Tensor::f_cumulative_trapezoid_x(y, x, dim).unwrap() - } - - pub fn data(&self) -> Tensor { - self.f_data().unwrap() - } - - pub fn deg2rad(&self) -> Tensor { - self.f_deg2rad().unwrap() - } - - pub fn deg2rad_(&mut self) -> Tensor { - self.f_deg2rad_().unwrap() - } - - pub fn deg2rad_out(&self, out: &Tensor) -> Tensor { - self.f_deg2rad_out(out).unwrap() - } - - pub fn dense_dim(&self) -> i64 { - self.f_dense_dim().unwrap() - } - - pub fn dequantize(&self) -> Tensor { - self.f_dequantize().unwrap() - } - - pub fn dequantize_self_out(&self, out: &Tensor) -> Tensor { - self.f_dequantize_self_out(out).unwrap() - } - - pub fn dequantize_tensors>(tensors: &[T]) -> Vec { - Tensor::f_dequantize_tensors(tensors).unwrap() - } - - pub fn dequantize_tensors_out>(out: &[T], tensors: &[T]) { - Tensor::f_dequantize_tensors_out(out, tensors).unwrap() - } - - pub fn det(&self) -> Tensor { - self.f_det().unwrap() - } - - pub fn detach(&self) -> Tensor { - self.f_detach().unwrap() - } - - pub fn detach_(&mut self) -> Tensor { - self.f_detach_().unwrap() - } - - pub fn detach_copy(&self) -> Tensor { - self.f_detach_copy().unwrap() - } - - pub fn detach_copy_out(&self, out: &Tensor) -> Tensor { - self.f_detach_copy_out(out).unwrap() - } - - pub fn diag(&self, diagonal: i64) -> Tensor { - self.f_diag(diagonal).unwrap() - } - - pub fn diag_embed(&self, offset: i64, dim1: i64, dim2: i64) -> Tensor { - self.f_diag_embed(offset, dim1, dim2).unwrap() - } - - pub fn diag_embed_out(&self, out: &Tensor, offset: i64, dim1: i64, dim2: i64) -> Tensor { - self.f_diag_embed_out(out, offset, dim1, dim2).unwrap() - } - - pub fn diag_out(&self, out: &Tensor, diagonal: i64) -> Tensor { - self.f_diag_out(out, diagonal).unwrap() - } - - pub fn diagflat(&self, offset: i64) -> Tensor { - self.f_diagflat(offset).unwrap() - } - - pub fn diagonal(&self, offset: i64, dim1: i64, dim2: i64) -> Tensor { - self.f_diagonal(offset, dim1, dim2).unwrap() - } - - pub fn diagonal_backward( - grad_output: &Tensor, - input_sizes: impl IntList, - offset: i64, - dim1: i64, - dim2: i64, - ) -> Tensor { - Tensor::f_diagonal_backward(grad_output, input_sizes, offset, dim1, dim2).unwrap() - } - - pub fn diagonal_backward_out( - out: &Tensor, - grad_output: &Tensor, - input_sizes: impl IntList, - offset: i64, - dim1: i64, - dim2: i64, - ) -> Tensor { - Tensor::f_diagonal_backward_out(out, grad_output, input_sizes, offset, dim1, dim2).unwrap() - } - - pub fn diagonal_copy(&self, offset: i64, dim1: i64, dim2: i64) -> Tensor { - self.f_diagonal_copy(offset, dim1, dim2).unwrap() - } - - pub fn diagonal_copy_out(&self, out: &Tensor, offset: i64, dim1: i64, dim2: i64) -> Tensor { - self.f_diagonal_copy_out(out, offset, dim1, dim2).unwrap() - } - - pub fn diagonal_scatter(&self, src: &Tensor, offset: i64, dim1: i64, dim2: i64) -> Tensor { - self.f_diagonal_scatter(src, offset, dim1, dim2).unwrap() - } - - pub fn diagonal_scatter_out( - &self, - out: &Tensor, - src: &Tensor, - offset: i64, - dim1: i64, - dim2: i64, - ) -> Tensor { - self.f_diagonal_scatter_out(out, src, offset, dim1, dim2).unwrap() - } - - pub fn diff>( - &self, - n: i64, - dim: i64, - prepend: Option, - append: Option, - ) -> Tensor { - self.f_diff(n, dim, prepend, append).unwrap() - } - - pub fn diff_out>( - &self, - out: &Tensor, - n: i64, - dim: i64, - prepend: Option, - append: Option, - ) -> Tensor { - self.f_diff_out(out, n, dim, prepend, append).unwrap() - } - - pub fn digamma(&self) -> Tensor { - self.f_digamma().unwrap() - } - - pub fn digamma_(&mut self) -> Tensor { - self.f_digamma_().unwrap() - } - - pub fn digamma_out(&self, out: &Tensor) -> Tensor { - self.f_digamma_out(out).unwrap() - } - - pub fn dist(&self, other: &Tensor) -> Tensor { - self.f_dist(other).unwrap() - } - - pub fn dist_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_dist_out(out, other).unwrap() - } - - pub fn g_div(&self, other: &Tensor) -> Tensor { - self.f_div(other).unwrap() - } - - pub fn g_div_(&mut self, other: &Tensor) -> Tensor { - self.f_div_(other).unwrap() - } - - pub fn div_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_div_out(out, other).unwrap() - } - - pub fn div_out_mode(&self, out: &Tensor, other: &Tensor, rounding_mode: &str) -> Tensor { - self.f_div_out_mode(out, other, rounding_mode).unwrap() - } - - pub fn g_div_scalar>(&self, other: S) -> Tensor { - self.f_div_scalar(other).unwrap() - } - - pub fn g_div_scalar_>(&mut self, other: S) -> Tensor { - self.f_div_scalar_(other).unwrap() - } - - pub fn g_div_scalar_mode>(&self, other: S, rounding_mode: &str) -> Tensor { - self.f_div_scalar_mode(other, rounding_mode).unwrap() - } - - pub fn g_div_scalar_mode_>(&mut self, other: S, rounding_mode: &str) -> Tensor { - self.f_div_scalar_mode_(other, rounding_mode).unwrap() - } - - pub fn div_scalar_mode_out>( - &self, - out: &Tensor, - other: S, - rounding_mode: &str, - ) -> Tensor { - self.f_div_scalar_mode_out(out, other, rounding_mode).unwrap() - } - - pub fn div_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_div_scalar_out(out, other).unwrap() - } - - pub fn g_div_tensor_mode(&self, other: &Tensor, rounding_mode: &str) -> Tensor { - self.f_div_tensor_mode(other, rounding_mode).unwrap() - } - - pub fn g_div_tensor_mode_(&mut self, other: &Tensor, rounding_mode: &str) -> Tensor { - self.f_div_tensor_mode_(other, rounding_mode).unwrap() - } - - pub fn divide(&self, other: &Tensor) -> Tensor { - self.f_divide(other).unwrap() - } - - pub fn divide_(&mut self, other: &Tensor) -> Tensor { - self.f_divide_(other).unwrap() - } - - pub fn divide_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_divide_out(out, other).unwrap() - } - - pub fn divide_out_mode(&self, out: &Tensor, other: &Tensor, rounding_mode: &str) -> Tensor { - self.f_divide_out_mode(out, other, rounding_mode).unwrap() - } - - pub fn divide_scalar>(&self, other: S) -> Tensor { - self.f_divide_scalar(other).unwrap() - } - - pub fn divide_scalar_>(&mut self, other: S) -> Tensor { - self.f_divide_scalar_(other).unwrap() - } - - pub fn divide_scalar_mode>(&self, other: S, rounding_mode: &str) -> Tensor { - self.f_divide_scalar_mode(other, rounding_mode).unwrap() - } - - pub fn divide_scalar_mode_>( - &mut self, - other: S, - rounding_mode: &str, - ) -> Tensor { - self.f_divide_scalar_mode_(other, rounding_mode).unwrap() - } - - pub fn divide_tensor_mode(&self, other: &Tensor, rounding_mode: &str) -> Tensor { - self.f_divide_tensor_mode(other, rounding_mode).unwrap() - } - - pub fn divide_tensor_mode_(&mut self, other: &Tensor, rounding_mode: &str) -> Tensor { - self.f_divide_tensor_mode_(other, rounding_mode).unwrap() - } - - pub fn dot(&self, tensor: &Tensor) -> Tensor { - self.f_dot(tensor).unwrap() - } - - pub fn dot_out(&self, out: &Tensor, tensor: &Tensor) -> Tensor { - self.f_dot_out(out, tensor).unwrap() - } - - pub fn dropout(&self, p: f64, train: bool) -> Tensor { - self.f_dropout(p, train).unwrap() - } - - pub fn dropout_(&mut self, p: f64, train: bool) -> Tensor { - self.f_dropout_(p, train).unwrap() - } - - pub fn dsplit(&self, sections: i64) -> Vec { - self.f_dsplit(sections).unwrap() - } - - pub fn dsplit_array(&self, indices: impl IntList) -> Vec { - self.f_dsplit_array(indices).unwrap() - } - - pub fn dstack>(tensors: &[T]) -> Tensor { - Tensor::f_dstack(tensors).unwrap() - } - - pub fn dstack_out>(out: &Tensor, tensors: &[T]) -> Tensor { - Tensor::f_dstack_out(out, tensors).unwrap() - } - - pub fn einsum>( - equation: &str, - tensors: &[T], - path: impl IntListOption, - ) -> Tensor { - Tensor::f_einsum(equation, tensors, path).unwrap() - } - - pub fn elu(&self) -> Tensor { - self.f_elu().unwrap() - } - - pub fn elu_(&mut self) -> Tensor { - self.f_elu_().unwrap() - } - - pub fn elu_backward>( - grad_output: &Tensor, - alpha: S, - scale: S, - input_scale: S, - is_result: bool, - self_or_result: &Tensor, - ) -> Tensor { - Tensor::f_elu_backward(grad_output, alpha, scale, input_scale, is_result, self_or_result) - .unwrap() - } - - pub fn elu_backward_grad_input>( - grad_input: &Tensor, - grad_output: &Tensor, - alpha: S, - scale: S, - input_scale: S, - is_result: bool, - self_or_result: &Tensor, - ) -> Tensor { - Tensor::f_elu_backward_grad_input( - grad_input, - grad_output, - alpha, - scale, - input_scale, - is_result, - self_or_result, - ) - .unwrap() - } - - pub fn elu_out(&self, out: &Tensor) -> Tensor { - self.f_elu_out(out).unwrap() - } - - pub fn embedding( - weight: &Tensor, - indices: &Tensor, - padding_idx: i64, - scale_grad_by_freq: bool, - sparse: bool, - ) -> Tensor { - Tensor::f_embedding(weight, indices, padding_idx, scale_grad_by_freq, sparse).unwrap() - } - - pub fn embedding_backward( - grad: &Tensor, - indices: &Tensor, - num_weights: i64, - padding_idx: i64, - scale_grad_by_freq: bool, - sparse: bool, - ) -> Tensor { - Tensor::f_embedding_backward( - grad, - indices, - num_weights, - padding_idx, - scale_grad_by_freq, - sparse, - ) - .unwrap() - } - - pub fn embedding_bag>( - weight: &Tensor, - indices: &Tensor, - offsets: &Tensor, - scale_grad_by_freq: bool, - mode: i64, - sparse: bool, - per_sample_weights: Option, - include_last_offset: bool, - ) -> (Tensor, Tensor, Tensor, Tensor) { - Tensor::f_embedding_bag( - weight, - indices, - offsets, - scale_grad_by_freq, - mode, - sparse, - per_sample_weights, - include_last_offset, - ) - .unwrap() - } - - pub fn embedding_bag_padding_idx>( - weight: &Tensor, - indices: &Tensor, - offsets: &Tensor, - scale_grad_by_freq: bool, - mode: i64, - sparse: bool, - per_sample_weights: Option, - include_last_offset: bool, - padding_idx: impl Into>, - ) -> (Tensor, Tensor, Tensor, Tensor) { - Tensor::f_embedding_bag_padding_idx( - weight, - indices, - offsets, - scale_grad_by_freq, - mode, - sparse, - per_sample_weights, - include_last_offset, - padding_idx, - ) - .unwrap() - } - - pub fn embedding_dense_backward( - grad_output: &Tensor, - indices: &Tensor, - num_weights: i64, - padding_idx: i64, - scale_grad_by_freq: bool, - ) -> Tensor { - Tensor::f_embedding_dense_backward( - grad_output, - indices, - num_weights, - padding_idx, - scale_grad_by_freq, - ) - .unwrap() - } - - pub fn embedding_dense_backward_out( - out: &Tensor, - grad_output: &Tensor, - indices: &Tensor, - num_weights: i64, - padding_idx: i64, - scale_grad_by_freq: bool, - ) -> Tensor { - Tensor::f_embedding_dense_backward_out( - out, - grad_output, - indices, - num_weights, - padding_idx, - scale_grad_by_freq, - ) - .unwrap() - } - - pub fn embedding_out( - out: &Tensor, - weight: &Tensor, - indices: &Tensor, - padding_idx: i64, - scale_grad_by_freq: bool, - sparse: bool, - ) -> Tensor { - Tensor::f_embedding_out(out, weight, indices, padding_idx, scale_grad_by_freq, sparse) - .unwrap() - } - - pub fn embedding_renorm(&self, indices: &Tensor, max_norm: f64, norm_type: f64) -> Tensor { - self.f_embedding_renorm(indices, max_norm, norm_type).unwrap() - } - - pub fn embedding_renorm_(&mut self, indices: &Tensor, max_norm: f64, norm_type: f64) -> Tensor { - self.f_embedding_renorm_(indices, max_norm, norm_type).unwrap() - } - - pub fn embedding_renorm_out( - &self, - out: &Tensor, - indices: &Tensor, - max_norm: f64, - norm_type: f64, - ) -> Tensor { - self.f_embedding_renorm_out(out, indices, max_norm, norm_type).unwrap() - } - - pub fn embedding_sparse_backward( - grad: &Tensor, - indices: &Tensor, - num_weights: i64, - padding_idx: i64, - scale_grad_by_freq: bool, - ) -> Tensor { - Tensor::f_embedding_sparse_backward( - grad, - indices, - num_weights, - padding_idx, - scale_grad_by_freq, - ) - .unwrap() - } - - pub fn empty(size: impl IntList, options: (Kind, Device)) -> Tensor { - Tensor::f_empty(size, options).unwrap() - } - - pub fn empty_like(&self) -> Tensor { - self.f_empty_like().unwrap() - } - - pub fn empty_like_out(&self, out: &Tensor) -> Tensor { - self.f_empty_like_out(out).unwrap() - } - - pub fn empty_out(out: &Tensor, size: impl IntList) -> Tensor { - Tensor::f_empty_out(out, size).unwrap() - } - - pub fn empty_permuted( - size: impl IntList, - physical_layout: impl IntList, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_empty_permuted(size, physical_layout, options).unwrap() - } - - pub fn empty_permuted_out( - out: &Tensor, - size: impl IntList, - physical_layout: impl IntList, - ) -> Tensor { - Tensor::f_empty_permuted_out(out, size, physical_layout).unwrap() - } - - pub fn empty_quantized( - size: impl IntList, - qtensor: &Tensor, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_empty_quantized(size, qtensor, options).unwrap() - } - - pub fn empty_quantized_out(out: &Tensor, size: impl IntList, qtensor: &Tensor) -> Tensor { - Tensor::f_empty_quantized_out(out, size, qtensor).unwrap() - } - - pub fn empty_strided( - size: impl IntList, - stride: impl IntList, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_empty_strided(size, stride, options).unwrap() - } - - pub fn empty_strided_out(out: &Tensor, size: impl IntList, stride: impl IntList) -> Tensor { - Tensor::f_empty_strided_out(out, size, stride).unwrap() - } - - pub fn eq>(&self, other: S) -> Tensor { - self.f_eq(other).unwrap() - } - - pub fn eq_>(&mut self, other: S) -> Tensor { - self.f_eq_(other).unwrap() - } - - pub fn eq_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_eq_scalar_out(out, other).unwrap() - } - - pub fn eq_tensor(&self, other: &Tensor) -> Tensor { - self.f_eq_tensor(other).unwrap() - } - - pub fn eq_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_eq_tensor_(other).unwrap() - } - - pub fn eq_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_eq_tensor_out(out, other).unwrap() - } - - pub fn equal(&self, other: &Tensor) -> bool { - self.f_equal(other).unwrap() - } - - pub fn erf(&self) -> Tensor { - self.f_erf().unwrap() - } - - pub fn erf_(&mut self) -> Tensor { - self.f_erf_().unwrap() - } - - pub fn erf_out(&self, out: &Tensor) -> Tensor { - self.f_erf_out(out).unwrap() - } - - pub fn erfc(&self) -> Tensor { - self.f_erfc().unwrap() - } - - pub fn erfc_(&mut self) -> Tensor { - self.f_erfc_().unwrap() - } - - pub fn erfc_out(&self, out: &Tensor) -> Tensor { - self.f_erfc_out(out).unwrap() - } - - pub fn erfinv(&self) -> Tensor { - self.f_erfinv().unwrap() - } - - pub fn erfinv_(&mut self) -> Tensor { - self.f_erfinv_().unwrap() - } - - pub fn erfinv_out(&self, out: &Tensor) -> Tensor { - self.f_erfinv_out(out).unwrap() - } - - pub fn exp(&self) -> Tensor { - self.f_exp().unwrap() - } - - pub fn exp2(&self) -> Tensor { - self.f_exp2().unwrap() - } - - pub fn exp2_(&mut self) -> Tensor { - self.f_exp2_().unwrap() - } - - pub fn exp2_out(&self, out: &Tensor) -> Tensor { - self.f_exp2_out(out).unwrap() - } - - pub fn exp_(&mut self) -> Tensor { - self.f_exp_().unwrap() - } - - pub fn exp_out(&self, out: &Tensor) -> Tensor { - self.f_exp_out(out).unwrap() - } - - pub fn expand(&self, size: impl IntList, implicit: bool) -> Tensor { - self.f_expand(size, implicit).unwrap() - } - - pub fn expand_as(&self, other: &Tensor) -> Tensor { - self.f_expand_as(other).unwrap() - } - - pub fn expand_copy(&self, size: impl IntList, implicit: bool) -> Tensor { - self.f_expand_copy(size, implicit).unwrap() - } - - pub fn expand_copy_out(&self, out: &Tensor, size: impl IntList, implicit: bool) -> Tensor { - self.f_expand_copy_out(out, size, implicit).unwrap() - } - - pub fn expm1(&self) -> Tensor { - self.f_expm1().unwrap() - } - - pub fn expm1_(&mut self) -> Tensor { - self.f_expm1_().unwrap() - } - - pub fn expm1_out(&self, out: &Tensor) -> Tensor { - self.f_expm1_out(out).unwrap() - } - - pub fn exponential(&self, lambd: f64) -> Tensor { - self.f_exponential(lambd).unwrap() - } - - pub fn exponential_(&mut self, lambd: f64) -> Tensor { - self.f_exponential_(lambd).unwrap() - } - - pub fn exponential_out(&self, out: &Tensor, lambd: f64) -> Tensor { - self.f_exponential_out(out, lambd).unwrap() - } - - pub fn eye(n: i64, options: (Kind, Device)) -> Tensor { - Tensor::f_eye(n, options).unwrap() - } - - pub fn eye_m(n: i64, m: i64, options: (Kind, Device)) -> Tensor { - Tensor::f_eye_m(n, m, options).unwrap() - } - - pub fn eye_m_out(out: &Tensor, n: i64, m: i64) -> Tensor { - Tensor::f_eye_m_out(out, n, m).unwrap() - } - - pub fn eye_out(out: &Tensor, n: i64) -> Tensor { - Tensor::f_eye_out(out, n).unwrap() - } - - pub fn fake_quantize_per_channel_affine( - &self, - scale: &Tensor, - zero_point: &Tensor, - axis: i64, - quant_min: i64, - quant_max: i64, - ) -> Tensor { - self.f_fake_quantize_per_channel_affine(scale, zero_point, axis, quant_min, quant_max) - .unwrap() - } - - pub fn fake_quantize_per_channel_affine_cachemask( - &self, - scale: &Tensor, - zero_point: &Tensor, - axis: i64, - quant_min: i64, - quant_max: i64, - ) -> (Tensor, Tensor) { - self.f_fake_quantize_per_channel_affine_cachemask( - scale, zero_point, axis, quant_min, quant_max, - ) - .unwrap() - } - - pub fn fake_quantize_per_channel_affine_cachemask_backward( - grad: &Tensor, - mask: &Tensor, - ) -> Tensor { - Tensor::f_fake_quantize_per_channel_affine_cachemask_backward(grad, mask).unwrap() - } - - pub fn fake_quantize_per_channel_affine_cachemask_out( - &self, - out0: &Tensor, - out1: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - axis: i64, - quant_min: i64, - quant_max: i64, - ) -> (Tensor, Tensor) { - self.f_fake_quantize_per_channel_affine_cachemask_out( - out0, out1, scale, zero_point, axis, quant_min, quant_max, - ) - .unwrap() - } - - pub fn fake_quantize_per_tensor_affine( - &self, - scale: f64, - zero_point: i64, - quant_min: i64, - quant_max: i64, - ) -> Tensor { - self.f_fake_quantize_per_tensor_affine(scale, zero_point, quant_min, quant_max).unwrap() - } - - pub fn fake_quantize_per_tensor_affine_cachemask( - &self, - scale: f64, - zero_point: i64, - quant_min: i64, - quant_max: i64, - ) -> (Tensor, Tensor) { - self.f_fake_quantize_per_tensor_affine_cachemask(scale, zero_point, quant_min, quant_max) - .unwrap() - } - - pub fn fake_quantize_per_tensor_affine_cachemask_backward( - grad: &Tensor, - mask: &Tensor, - ) -> Tensor { - Tensor::f_fake_quantize_per_tensor_affine_cachemask_backward(grad, mask).unwrap() - } - - pub fn fake_quantize_per_tensor_affine_cachemask_out( - &self, - out0: &Tensor, - out1: &Tensor, - scale: f64, - zero_point: i64, - quant_min: i64, - quant_max: i64, - ) -> (Tensor, Tensor) { - self.f_fake_quantize_per_tensor_affine_cachemask_out( - out0, out1, scale, zero_point, quant_min, quant_max, - ) - .unwrap() - } - - pub fn fake_quantize_per_tensor_affine_tensor_qparams( - &self, - scale: &Tensor, - zero_point: &Tensor, - quant_min: i64, - quant_max: i64, - ) -> Tensor { - self.f_fake_quantize_per_tensor_affine_tensor_qparams( - scale, zero_point, quant_min, quant_max, - ) - .unwrap() - } - - pub fn feature_alpha_dropout(&self, p: f64, train: bool) -> Tensor { - self.f_feature_alpha_dropout(p, train).unwrap() - } - - pub fn feature_alpha_dropout_(&mut self, p: f64, train: bool) -> Tensor { - self.f_feature_alpha_dropout_(p, train).unwrap() - } - - pub fn feature_dropout(&self, p: f64, train: bool) -> Tensor { - self.f_feature_dropout(p, train).unwrap() - } - - pub fn feature_dropout_(&mut self, p: f64, train: bool) -> Tensor { - self.f_feature_dropout_(p, train).unwrap() - } - - pub fn fft_fft(&self, n: impl Into>, dim: i64, norm: &str) -> Tensor { - self.f_fft_fft(n, dim, norm).unwrap() - } - - pub fn fft_fft2(&self, s: impl IntListOption, dim: impl IntList, norm: &str) -> Tensor { - self.f_fft_fft2(s, dim, norm).unwrap() - } - - pub fn fft_fft2_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntList, - norm: &str, - ) -> Tensor { - self.f_fft_fft2_out(out, s, dim, norm).unwrap() - } - - pub fn fft_fft_out( - &self, - out: &Tensor, - n: impl Into>, - dim: i64, - norm: &str, - ) -> Tensor { - self.f_fft_fft_out(out, n, dim, norm).unwrap() - } - - pub fn fft_fftfreq(n: i64, d: f64, options: (Kind, Device)) -> Tensor { - Tensor::f_fft_fftfreq(n, d, options).unwrap() - } - - pub fn fft_fftfreq_out(out: &Tensor, n: i64, d: f64) -> Tensor { - Tensor::f_fft_fftfreq_out(out, n, d).unwrap() - } - - pub fn fft_fftn(&self, s: impl IntListOption, dim: impl IntListOption, norm: &str) -> Tensor { - self.f_fft_fftn(s, dim, norm).unwrap() - } - - pub fn fft_fftn_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntListOption, - norm: &str, - ) -> Tensor { - self.f_fft_fftn_out(out, s, dim, norm).unwrap() - } - - pub fn fft_fftshift(&self, dim: impl IntListOption) -> Tensor { - self.f_fft_fftshift(dim).unwrap() - } - - pub fn fft_hfft(&self, n: impl Into>, dim: i64, norm: &str) -> Tensor { - self.f_fft_hfft(n, dim, norm).unwrap() - } - - pub fn fft_hfft2(&self, s: impl IntListOption, dim: impl IntList, norm: &str) -> Tensor { - self.f_fft_hfft2(s, dim, norm).unwrap() - } - - pub fn fft_hfft2_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntList, - norm: &str, - ) -> Tensor { - self.f_fft_hfft2_out(out, s, dim, norm).unwrap() - } - - pub fn fft_hfft_out( - &self, - out: &Tensor, - n: impl Into>, - dim: i64, - norm: &str, - ) -> Tensor { - self.f_fft_hfft_out(out, n, dim, norm).unwrap() - } - - pub fn fft_hfftn(&self, s: impl IntListOption, dim: impl IntListOption, norm: &str) -> Tensor { - self.f_fft_hfftn(s, dim, norm).unwrap() - } - - pub fn fft_hfftn_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntListOption, - norm: &str, - ) -> Tensor { - self.f_fft_hfftn_out(out, s, dim, norm).unwrap() - } - - pub fn fft_ifft(&self, n: impl Into>, dim: i64, norm: &str) -> Tensor { - self.f_fft_ifft(n, dim, norm).unwrap() - } - - pub fn fft_ifft2(&self, s: impl IntListOption, dim: impl IntList, norm: &str) -> Tensor { - self.f_fft_ifft2(s, dim, norm).unwrap() - } - - pub fn fft_ifft2_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntList, - norm: &str, - ) -> Tensor { - self.f_fft_ifft2_out(out, s, dim, norm).unwrap() - } - - pub fn fft_ifft_out( - &self, - out: &Tensor, - n: impl Into>, - dim: i64, - norm: &str, - ) -> Tensor { - self.f_fft_ifft_out(out, n, dim, norm).unwrap() - } - - pub fn fft_ifftn(&self, s: impl IntListOption, dim: impl IntListOption, norm: &str) -> Tensor { - self.f_fft_ifftn(s, dim, norm).unwrap() - } - - pub fn fft_ifftn_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntListOption, - norm: &str, - ) -> Tensor { - self.f_fft_ifftn_out(out, s, dim, norm).unwrap() - } - - pub fn fft_ifftshift(&self, dim: impl IntListOption) -> Tensor { - self.f_fft_ifftshift(dim).unwrap() - } - - pub fn fft_ihfft(&self, n: impl Into>, dim: i64, norm: &str) -> Tensor { - self.f_fft_ihfft(n, dim, norm).unwrap() - } - - pub fn fft_ihfft2(&self, s: impl IntListOption, dim: impl IntList, norm: &str) -> Tensor { - self.f_fft_ihfft2(s, dim, norm).unwrap() - } - - pub fn fft_ihfft2_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntList, - norm: &str, - ) -> Tensor { - self.f_fft_ihfft2_out(out, s, dim, norm).unwrap() - } - - pub fn fft_ihfft_out( - &self, - out: &Tensor, - n: impl Into>, - dim: i64, - norm: &str, - ) -> Tensor { - self.f_fft_ihfft_out(out, n, dim, norm).unwrap() - } - - pub fn fft_ihfftn(&self, s: impl IntListOption, dim: impl IntListOption, norm: &str) -> Tensor { - self.f_fft_ihfftn(s, dim, norm).unwrap() - } - - pub fn fft_ihfftn_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntListOption, - norm: &str, - ) -> Tensor { - self.f_fft_ihfftn_out(out, s, dim, norm).unwrap() - } - - pub fn fft_irfft(&self, n: impl Into>, dim: i64, norm: &str) -> Tensor { - self.f_fft_irfft(n, dim, norm).unwrap() - } - - pub fn fft_irfft2(&self, s: impl IntListOption, dim: impl IntList, norm: &str) -> Tensor { - self.f_fft_irfft2(s, dim, norm).unwrap() - } - - pub fn fft_irfft2_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntList, - norm: &str, - ) -> Tensor { - self.f_fft_irfft2_out(out, s, dim, norm).unwrap() - } - - pub fn fft_irfft_out( - &self, - out: &Tensor, - n: impl Into>, - dim: i64, - norm: &str, - ) -> Tensor { - self.f_fft_irfft_out(out, n, dim, norm).unwrap() - } - - pub fn fft_irfftn(&self, s: impl IntListOption, dim: impl IntListOption, norm: &str) -> Tensor { - self.f_fft_irfftn(s, dim, norm).unwrap() - } - - pub fn fft_irfftn_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntListOption, - norm: &str, - ) -> Tensor { - self.f_fft_irfftn_out(out, s, dim, norm).unwrap() - } - - pub fn fft_rfft(&self, n: impl Into>, dim: i64, norm: &str) -> Tensor { - self.f_fft_rfft(n, dim, norm).unwrap() - } - - pub fn fft_rfft2(&self, s: impl IntListOption, dim: impl IntList, norm: &str) -> Tensor { - self.f_fft_rfft2(s, dim, norm).unwrap() - } - - pub fn fft_rfft2_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntList, - norm: &str, - ) -> Tensor { - self.f_fft_rfft2_out(out, s, dim, norm).unwrap() - } - - pub fn fft_rfft_out( - &self, - out: &Tensor, - n: impl Into>, - dim: i64, - norm: &str, - ) -> Tensor { - self.f_fft_rfft_out(out, n, dim, norm).unwrap() - } - - pub fn fft_rfftfreq(n: i64, d: f64, options: (Kind, Device)) -> Tensor { - Tensor::f_fft_rfftfreq(n, d, options).unwrap() - } - - pub fn fft_rfftfreq_out(out: &Tensor, n: i64, d: f64) -> Tensor { - Tensor::f_fft_rfftfreq_out(out, n, d).unwrap() - } - - pub fn fft_rfftn(&self, s: impl IntListOption, dim: impl IntListOption, norm: &str) -> Tensor { - self.f_fft_rfftn(s, dim, norm).unwrap() - } - - pub fn fft_rfftn_out( - &self, - out: &Tensor, - s: impl IntListOption, - dim: impl IntListOption, - norm: &str, - ) -> Tensor { - self.f_fft_rfftn_out(out, s, dim, norm).unwrap() - } - - pub fn fill>(&self, value: S) -> Tensor { - self.f_fill(value).unwrap() - } - - pub fn fill_>(&mut self, value: S) -> Tensor { - self.f_fill_(value).unwrap() - } - - pub fn fill_diagonal_>(&mut self, fill_value: S, wrap: bool) -> Tensor { - self.f_fill_diagonal_(fill_value, wrap).unwrap() - } - - pub fn fill_scalar_out>(&self, out: &Tensor, value: S) -> Tensor { - self.f_fill_scalar_out(out, value).unwrap() - } - - pub fn fill_tensor(&self, value: &Tensor) -> Tensor { - self.f_fill_tensor(value).unwrap() - } - - pub fn fill_tensor_(&mut self, value: &Tensor) -> Tensor { - self.f_fill_tensor_(value).unwrap() - } - - pub fn fill_tensor_out(&self, out: &Tensor, value: &Tensor) -> Tensor { - self.f_fill_tensor_out(out, value).unwrap() - } - - pub fn fix(&self) -> Tensor { - self.f_fix().unwrap() - } - - pub fn fix_(&mut self) -> Tensor { - self.f_fix_().unwrap() - } - - pub fn fix_out(&self, out: &Tensor) -> Tensor { - self.f_fix_out(out).unwrap() - } - - pub fn flatten(&self, start_dim: i64, end_dim: i64) -> Tensor { - self.f_flatten(start_dim, end_dim).unwrap() - } - - pub fn flatten_dense_tensors>(tensors: &[T]) -> Tensor { - Tensor::f_flatten_dense_tensors(tensors).unwrap() - } - - pub fn flip(&self, dims: impl IntList) -> Tensor { - self.f_flip(dims).unwrap() - } - - pub fn flip_out(&self, out: &Tensor, dims: impl IntList) -> Tensor { - self.f_flip_out(out, dims).unwrap() - } - - pub fn fliplr(&self) -> Tensor { - self.f_fliplr().unwrap() - } - - pub fn flipud(&self) -> Tensor { - self.f_flipud().unwrap() - } - - pub fn float_power(&self, exponent: &Tensor) -> Tensor { - self.f_float_power(exponent).unwrap() - } - - pub fn float_power_>(&mut self, exponent: S) -> Tensor { - self.f_float_power_(exponent).unwrap() - } - - pub fn float_power_scalar>(self_scalar: S, exponent: &Tensor) -> Tensor { - Tensor::f_float_power_scalar(self_scalar, exponent).unwrap() - } - - pub fn float_power_scalar_out>( - out: &Tensor, - self_scalar: S, - exponent: &Tensor, - ) -> Tensor { - Tensor::f_float_power_scalar_out(out, self_scalar, exponent).unwrap() - } - - pub fn float_power_tensor_(&mut self, exponent: &Tensor) -> Tensor { - self.f_float_power_tensor_(exponent).unwrap() - } - - pub fn float_power_tensor_scalar>(&self, exponent: S) -> Tensor { - self.f_float_power_tensor_scalar(exponent).unwrap() - } - - pub fn float_power_tensor_scalar_out>( - &self, - out: &Tensor, - exponent: S, - ) -> Tensor { - self.f_float_power_tensor_scalar_out(out, exponent).unwrap() - } - - pub fn float_power_tensor_tensor_out(&self, out: &Tensor, exponent: &Tensor) -> Tensor { - self.f_float_power_tensor_tensor_out(out, exponent).unwrap() - } - - pub fn floor(&self) -> Tensor { - self.f_floor().unwrap() - } - - pub fn floor_(&mut self) -> Tensor { - self.f_floor_().unwrap() - } - - pub fn floor_divide(&self, other: &Tensor) -> Tensor { - self.f_floor_divide(other).unwrap() - } - - pub fn floor_divide_(&mut self, other: &Tensor) -> Tensor { - self.f_floor_divide_(other).unwrap() - } - - pub fn floor_divide_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_floor_divide_out(out, other).unwrap() - } - - pub fn floor_divide_scalar>(&self, other: S) -> Tensor { - self.f_floor_divide_scalar(other).unwrap() - } - - pub fn floor_divide_scalar_>(&mut self, other: S) -> Tensor { - self.f_floor_divide_scalar_(other).unwrap() - } - - pub fn floor_divide_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_floor_divide_scalar_out(out, other).unwrap() - } - - pub fn floor_out(&self, out: &Tensor) -> Tensor { - self.f_floor_out(out).unwrap() - } - - pub fn fmax(&self, other: &Tensor) -> Tensor { - self.f_fmax(other).unwrap() - } - - pub fn fmax_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_fmax_out(out, other).unwrap() - } - - pub fn fmin(&self, other: &Tensor) -> Tensor { - self.f_fmin(other).unwrap() - } - - pub fn fmin_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_fmin_out(out, other).unwrap() - } - - pub fn fmod>(&self, other: S) -> Tensor { - self.f_fmod(other).unwrap() - } - - pub fn fmod_>(&mut self, other: S) -> Tensor { - self.f_fmod_(other).unwrap() - } - - pub fn fmod_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_fmod_scalar_out(out, other).unwrap() - } - - pub fn fmod_tensor(&self, other: &Tensor) -> Tensor { - self.f_fmod_tensor(other).unwrap() - } - - pub fn fmod_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_fmod_tensor_(other).unwrap() - } - - pub fn fmod_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_fmod_tensor_out(out, other).unwrap() - } - - pub fn frac(&self) -> Tensor { - self.f_frac().unwrap() - } - - pub fn frac_(&mut self) -> Tensor { - self.f_frac_().unwrap() - } - - pub fn frac_out(&self, out: &Tensor) -> Tensor { - self.f_frac_out(out).unwrap() - } - - pub fn fractional_max_pool2d( - &self, - kernel_size: impl IntList, - output_size: impl IntList, - random_samples: &Tensor, - ) -> (Tensor, Tensor) { - self.f_fractional_max_pool2d(kernel_size, output_size, random_samples).unwrap() - } - - pub fn fractional_max_pool2d_backward( - &self, - grad_output: &Tensor, - kernel_size: impl IntList, - output_size: impl IntList, - indices: &Tensor, - ) -> Tensor { - self.f_fractional_max_pool2d_backward(grad_output, kernel_size, output_size, indices) - .unwrap() - } - - pub fn fractional_max_pool2d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - kernel_size: impl IntList, - output_size: impl IntList, - indices: &Tensor, - ) -> Tensor { - self.f_fractional_max_pool2d_backward_grad_input( - grad_input, - grad_output, - kernel_size, - output_size, - indices, - ) - .unwrap() - } - - pub fn fractional_max_pool2d_output( - &self, - output: &Tensor, - indices: &Tensor, - kernel_size: impl IntList, - output_size: impl IntList, - random_samples: &Tensor, - ) -> (Tensor, Tensor) { - self.f_fractional_max_pool2d_output( - output, - indices, - kernel_size, - output_size, - random_samples, - ) - .unwrap() - } - - pub fn fractional_max_pool3d( - &self, - kernel_size: impl IntList, - output_size: impl IntList, - random_samples: &Tensor, - ) -> (Tensor, Tensor) { - self.f_fractional_max_pool3d(kernel_size, output_size, random_samples).unwrap() - } - - pub fn fractional_max_pool3d_backward( - &self, - grad_output: &Tensor, - kernel_size: impl IntList, - output_size: impl IntList, - indices: &Tensor, - ) -> Tensor { - self.f_fractional_max_pool3d_backward(grad_output, kernel_size, output_size, indices) - .unwrap() - } - - pub fn fractional_max_pool3d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - kernel_size: impl IntList, - output_size: impl IntList, - indices: &Tensor, - ) -> Tensor { - self.f_fractional_max_pool3d_backward_grad_input( - grad_input, - grad_output, - kernel_size, - output_size, - indices, - ) - .unwrap() - } - - pub fn fractional_max_pool3d_output( - &self, - output: &Tensor, - indices: &Tensor, - kernel_size: impl IntList, - output_size: impl IntList, - random_samples: &Tensor, - ) -> (Tensor, Tensor) { - self.f_fractional_max_pool3d_output( - output, - indices, - kernel_size, - output_size, - random_samples, - ) - .unwrap() - } - - pub fn frexp(&self) -> (Tensor, Tensor) { - self.f_frexp().unwrap() - } - - pub fn frexp_tensor_out(&self, mantissa: &Tensor, exponent: &Tensor) -> (Tensor, Tensor) { - self.f_frexp_tensor_out(mantissa, exponent).unwrap() - } - - pub fn frobenius_norm(&self, dim: impl IntList, keepdim: bool) -> Tensor { - self.f_frobenius_norm(dim, keepdim).unwrap() - } - - pub fn frobenius_norm_out(&self, out: &Tensor, dim: impl IntList, keepdim: bool) -> Tensor { - self.f_frobenius_norm_out(out, dim, keepdim).unwrap() - } - - pub fn from_file( - filename: &str, - shared: bool, - size: impl Into>, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_from_file(filename, shared, size, options).unwrap() - } - - pub fn from_file_out( - out: &Tensor, - filename: &str, - shared: bool, - size: impl Into>, - ) -> Tensor { - Tensor::f_from_file_out(out, filename, shared, size).unwrap() - } - - pub fn full>( - size: impl IntList, - fill_value: S, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_full(size, fill_value, options).unwrap() - } - - pub fn full_like>(&self, fill_value: S) -> Tensor { - self.f_full_like(fill_value).unwrap() - } - - pub fn full_like_out>(&self, out: &Tensor, fill_value: S) -> Tensor { - self.f_full_like_out(out, fill_value).unwrap() - } - - pub fn full_out>(out: &Tensor, size: impl IntList, fill_value: S) -> Tensor { - Tensor::f_full_out(out, size, fill_value).unwrap() - } - - pub fn fused_moving_avg_obs_fake_quant( - &self, - observer_on: &Tensor, - fake_quant_on: &Tensor, - running_min: &Tensor, - running_max: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - averaging_const: f64, - quant_min: i64, - quant_max: i64, - ch_axis: i64, - per_row_fake_quant: bool, - symmetric_quant: bool, - ) -> Tensor { - self.f_fused_moving_avg_obs_fake_quant( - observer_on, - fake_quant_on, - running_min, - running_max, - scale, - zero_point, - averaging_const, - quant_min, - quant_max, - ch_axis, - per_row_fake_quant, - symmetric_quant, - ) - .unwrap() - } - - pub fn gather(&self, dim: i64, index: &Tensor, sparse_grad: bool) -> Tensor { - self.f_gather(dim, index, sparse_grad).unwrap() - } - - pub fn gather_backward( - &self, - grad: &Tensor, - dim: i64, - index: &Tensor, - sparse_grad: bool, - ) -> Tensor { - self.f_gather_backward(grad, dim, index, sparse_grad).unwrap() - } - - pub fn gather_out(&self, out: &Tensor, dim: i64, index: &Tensor, sparse_grad: bool) -> Tensor { - self.f_gather_out(out, dim, index, sparse_grad).unwrap() - } - - pub fn gcd(&self, other: &Tensor) -> Tensor { - self.f_gcd(other).unwrap() - } - - pub fn gcd_(&mut self, other: &Tensor) -> Tensor { - self.f_gcd_(other).unwrap() - } - - pub fn gcd_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_gcd_out(out, other).unwrap() - } - - pub fn ge>(&self, other: S) -> Tensor { - self.f_ge(other).unwrap() - } - - pub fn ge_>(&mut self, other: S) -> Tensor { - self.f_ge_(other).unwrap() - } - - pub fn ge_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_ge_scalar_out(out, other).unwrap() - } - - pub fn ge_tensor(&self, other: &Tensor) -> Tensor { - self.f_ge_tensor(other).unwrap() - } - - pub fn ge_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_ge_tensor_(other).unwrap() - } - - pub fn ge_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_ge_tensor_out(out, other).unwrap() - } - - pub fn gelu(&self, approximate: &str) -> Tensor { - self.f_gelu(approximate).unwrap() - } - - pub fn gelu_(&mut self, approximate: &str) -> Tensor { - self.f_gelu_(approximate).unwrap() - } - - pub fn gelu_backward(&self, grad_output: &Tensor, approximate: &str) -> Tensor { - self.f_gelu_backward(grad_output, approximate).unwrap() - } - - pub fn gelu_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - approximate: &str, - ) -> Tensor { - self.f_gelu_backward_grad_input(grad_input, grad_output, approximate).unwrap() - } - - pub fn gelu_out(&self, out: &Tensor, approximate: &str) -> Tensor { - self.f_gelu_out(out, approximate).unwrap() - } - - pub fn geometric(&self, p: f64) -> Tensor { - self.f_geometric(p).unwrap() - } - - pub fn geometric_(&mut self, p: f64) -> Tensor { - self.f_geometric_(p).unwrap() - } - - pub fn geometric_out(&self, out: &Tensor, p: f64) -> Tensor { - self.f_geometric_out(out, p).unwrap() - } - - pub fn geqrf(&self) -> (Tensor, Tensor) { - self.f_geqrf().unwrap() - } - - pub fn geqrf_a(&self, a: &Tensor, tau: &Tensor) -> (Tensor, Tensor) { - self.f_geqrf_a(a, tau).unwrap() - } - - pub fn ger(&self, vec2: &Tensor) -> Tensor { - self.f_ger(vec2).unwrap() - } - - pub fn ger_out(&self, out: &Tensor, vec2: &Tensor) -> Tensor { - self.f_ger_out(out, vec2).unwrap() - } - - pub fn glu(&self, dim: i64) -> Tensor { - self.f_glu(dim).unwrap() - } - - pub fn glu_backward(&self, grad_output: &Tensor, dim: i64) -> Tensor { - self.f_glu_backward(grad_output, dim).unwrap() - } - - pub fn glu_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - dim: i64, - ) -> Tensor { - self.f_glu_backward_grad_input(grad_input, grad_output, dim).unwrap() - } - - pub fn glu_backward_jvp( - grad_x: &Tensor, - grad_glu: &Tensor, - x: &Tensor, - dgrad_glu: &Tensor, - dx: &Tensor, - dim: i64, - ) -> Tensor { - Tensor::f_glu_backward_jvp(grad_x, grad_glu, x, dgrad_glu, dx, dim).unwrap() - } - - pub fn glu_backward_jvp_out( - out: &Tensor, - grad_x: &Tensor, - grad_glu: &Tensor, - x: &Tensor, - dgrad_glu: &Tensor, - dx: &Tensor, - dim: i64, - ) -> Tensor { - Tensor::f_glu_backward_jvp_out(out, grad_x, grad_glu, x, dgrad_glu, dx, dim).unwrap() - } - - pub fn glu_jvp(glu: &Tensor, x: &Tensor, dx: &Tensor, dim: i64) -> Tensor { - Tensor::f_glu_jvp(glu, x, dx, dim).unwrap() - } - - pub fn glu_jvp_out(out: &Tensor, glu: &Tensor, x: &Tensor, dx: &Tensor, dim: i64) -> Tensor { - Tensor::f_glu_jvp_out(out, glu, x, dx, dim).unwrap() - } - - pub fn glu_out(&self, out: &Tensor, dim: i64) -> Tensor { - self.f_glu_out(out, dim).unwrap() - } - - pub fn grad(&self) -> Tensor { - self.f_grad().unwrap() - } - - pub fn greater>(&self, other: S) -> Tensor { - self.f_greater(other).unwrap() - } - - pub fn greater_>(&mut self, other: S) -> Tensor { - self.f_greater_(other).unwrap() - } - - pub fn greater_equal>(&self, other: S) -> Tensor { - self.f_greater_equal(other).unwrap() - } - - pub fn greater_equal_>(&mut self, other: S) -> Tensor { - self.f_greater_equal_(other).unwrap() - } - - pub fn greater_equal_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_greater_equal_scalar_out(out, other).unwrap() - } - - pub fn greater_equal_tensor(&self, other: &Tensor) -> Tensor { - self.f_greater_equal_tensor(other).unwrap() - } - - pub fn greater_equal_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_greater_equal_tensor_(other).unwrap() - } - - pub fn greater_equal_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_greater_equal_tensor_out(out, other).unwrap() - } - - pub fn greater_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_greater_scalar_out(out, other).unwrap() - } - - pub fn greater_tensor(&self, other: &Tensor) -> Tensor { - self.f_greater_tensor(other).unwrap() - } - - pub fn greater_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_greater_tensor_(other).unwrap() - } - - pub fn greater_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_greater_tensor_out(out, other).unwrap() - } - - pub fn grid_sampler( - &self, - grid: &Tensor, - interpolation_mode: i64, - padding_mode: i64, - align_corners: bool, - ) -> Tensor { - self.f_grid_sampler(grid, interpolation_mode, padding_mode, align_corners).unwrap() - } - - pub fn grid_sampler_2d( - &self, - grid: &Tensor, - interpolation_mode: i64, - padding_mode: i64, - align_corners: bool, - ) -> Tensor { - self.f_grid_sampler_2d(grid, interpolation_mode, padding_mode, align_corners).unwrap() - } - - pub fn grid_sampler_2d_out( - &self, - out: &Tensor, - grid: &Tensor, - interpolation_mode: i64, - padding_mode: i64, - align_corners: bool, - ) -> Tensor { - self.f_grid_sampler_2d_out(out, grid, interpolation_mode, padding_mode, align_corners) - .unwrap() - } - - pub fn grid_sampler_3d( - &self, - grid: &Tensor, - interpolation_mode: i64, - padding_mode: i64, - align_corners: bool, - ) -> Tensor { - self.f_grid_sampler_3d(grid, interpolation_mode, padding_mode, align_corners).unwrap() - } - - pub fn grid_sampler_3d_out( - &self, - out: &Tensor, - grid: &Tensor, - interpolation_mode: i64, - padding_mode: i64, - align_corners: bool, - ) -> Tensor { - self.f_grid_sampler_3d_out(out, grid, interpolation_mode, padding_mode, align_corners) - .unwrap() - } - - pub fn group_norm>( - &self, - num_groups: i64, - weight: Option, - bias: Option, - eps: f64, - cudnn_enabled: bool, - ) -> Tensor { - self.f_group_norm(num_groups, weight, bias, eps, cudnn_enabled).unwrap() - } - - pub fn gru>( - &self, - hx: &Tensor, - params: &[T], - has_biases: bool, - num_layers: i64, - dropout: f64, - train: bool, - bidirectional: bool, - batch_first: bool, - ) -> (Tensor, Tensor) { - self.f_gru(hx, params, has_biases, num_layers, dropout, train, bidirectional, batch_first) - .unwrap() - } - - pub fn gru_cell>( - &self, - hx: &Tensor, - w_ih: &Tensor, - w_hh: &Tensor, - b_ih: Option, - b_hh: Option, - ) -> Tensor { - self.f_gru_cell(hx, w_ih, w_hh, b_ih, b_hh).unwrap() - } - - pub fn gru_data>( - data: &Tensor, - batch_sizes: &Tensor, - hx: &Tensor, - params: &[T], - has_biases: bool, - num_layers: i64, - dropout: f64, - train: bool, - bidirectional: bool, - ) -> (Tensor, Tensor) { - Tensor::f_gru_data( - data, - batch_sizes, - hx, - params, - has_biases, - num_layers, - dropout, - train, - bidirectional, - ) - .unwrap() - } - - pub fn gt>(&self, other: S) -> Tensor { - self.f_gt(other).unwrap() - } - - pub fn gt_>(&mut self, other: S) -> Tensor { - self.f_gt_(other).unwrap() - } - - pub fn gt_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_gt_scalar_out(out, other).unwrap() - } - - pub fn gt_tensor(&self, other: &Tensor) -> Tensor { - self.f_gt_tensor(other).unwrap() - } - - pub fn gt_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_gt_tensor_(other).unwrap() - } - - pub fn gt_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_gt_tensor_out(out, other).unwrap() - } - - pub fn hamming_window(window_length: i64, options: (Kind, Device)) -> Tensor { - Tensor::f_hamming_window(window_length, options).unwrap() - } - - pub fn hamming_window_out(out: &Tensor, window_length: i64) -> Tensor { - Tensor::f_hamming_window_out(out, window_length).unwrap() - } - - pub fn hamming_window_periodic( - window_length: i64, - periodic: bool, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_hamming_window_periodic(window_length, periodic, options).unwrap() - } - - pub fn hamming_window_periodic_alpha( - window_length: i64, - periodic: bool, - alpha: f64, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_hamming_window_periodic_alpha(window_length, periodic, alpha, options).unwrap() - } - - pub fn hamming_window_periodic_alpha_beta( - window_length: i64, - periodic: bool, - alpha: f64, - beta: f64, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_hamming_window_periodic_alpha_beta(window_length, periodic, alpha, beta, options) - .unwrap() - } - - pub fn hamming_window_periodic_alpha_beta_out( - out: &Tensor, - window_length: i64, - periodic: bool, - alpha: f64, - beta: f64, - ) -> Tensor { - Tensor::f_hamming_window_periodic_alpha_beta_out(out, window_length, periodic, alpha, beta) - .unwrap() - } - - pub fn hamming_window_periodic_alpha_out( - out: &Tensor, - window_length: i64, - periodic: bool, - alpha: f64, - ) -> Tensor { - Tensor::f_hamming_window_periodic_alpha_out(out, window_length, periodic, alpha).unwrap() - } - - pub fn hamming_window_periodic_out(out: &Tensor, window_length: i64, periodic: bool) -> Tensor { - Tensor::f_hamming_window_periodic_out(out, window_length, periodic).unwrap() - } - - pub fn hann_window(window_length: i64, options: (Kind, Device)) -> Tensor { - Tensor::f_hann_window(window_length, options).unwrap() - } - - pub fn hann_window_out(out: &Tensor, window_length: i64) -> Tensor { - Tensor::f_hann_window_out(out, window_length).unwrap() - } - - pub fn hann_window_periodic( - window_length: i64, - periodic: bool, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_hann_window_periodic(window_length, periodic, options).unwrap() - } - - pub fn hann_window_periodic_out(out: &Tensor, window_length: i64, periodic: bool) -> Tensor { - Tensor::f_hann_window_periodic_out(out, window_length, periodic).unwrap() - } - - pub fn hardshrink(&self) -> Tensor { - self.f_hardshrink().unwrap() - } - - pub fn hardshrink_backward>(&self, grad_out: &Tensor, lambd: S) -> Tensor { - self.f_hardshrink_backward(grad_out, lambd).unwrap() - } - - pub fn hardshrink_backward_grad_input>( - &self, - grad_input: &Tensor, - grad_out: &Tensor, - lambd: S, - ) -> Tensor { - self.f_hardshrink_backward_grad_input(grad_input, grad_out, lambd).unwrap() - } - - pub fn hardshrink_out(&self, out: &Tensor) -> Tensor { - self.f_hardshrink_out(out).unwrap() - } - - pub fn hardsigmoid(&self) -> Tensor { - self.f_hardsigmoid().unwrap() - } - - pub fn hardsigmoid_(&mut self) -> Tensor { - self.f_hardsigmoid_().unwrap() - } - - pub fn hardsigmoid_backward(&self, grad_output: &Tensor) -> Tensor { - self.f_hardsigmoid_backward(grad_output).unwrap() - } - - pub fn hardsigmoid_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - ) -> Tensor { - self.f_hardsigmoid_backward_grad_input(grad_input, grad_output).unwrap() - } - - pub fn hardsigmoid_out(&self, out: &Tensor) -> Tensor { - self.f_hardsigmoid_out(out).unwrap() - } - - pub fn hardswish(&self) -> Tensor { - self.f_hardswish().unwrap() - } - - pub fn hardswish_(&mut self) -> Tensor { - self.f_hardswish_().unwrap() - } - - pub fn hardswish_backward(&self, grad_output: &Tensor) -> Tensor { - self.f_hardswish_backward(grad_output).unwrap() - } - - pub fn hardswish_backward_out(&self, out: &Tensor, grad_output: &Tensor) -> Tensor { - self.f_hardswish_backward_out(out, grad_output).unwrap() - } - - pub fn hardswish_out(&self, out: &Tensor) -> Tensor { - self.f_hardswish_out(out).unwrap() - } - - pub fn hardtanh(&self) -> Tensor { - self.f_hardtanh().unwrap() - } - - pub fn hardtanh_(&mut self) -> Tensor { - self.f_hardtanh_().unwrap() - } - - pub fn hardtanh_backward>( - &self, - grad_output: &Tensor, - min_val: S, - max_val: S, - ) -> Tensor { - self.f_hardtanh_backward(grad_output, min_val, max_val).unwrap() - } - - pub fn hardtanh_backward_grad_input>( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - min_val: S, - max_val: S, - ) -> Tensor { - self.f_hardtanh_backward_grad_input(grad_input, grad_output, min_val, max_val).unwrap() - } - - pub fn hardtanh_out(&self, out: &Tensor) -> Tensor { - self.f_hardtanh_out(out).unwrap() - } - - pub fn hash_tensor(&self, dim: impl IntList, keepdim: bool, mode: i64) -> Tensor { - self.f_hash_tensor(dim, keepdim, mode).unwrap() - } - - pub fn hash_tensor_out( - &self, - out: &Tensor, - dim: impl IntList, - keepdim: bool, - mode: i64, - ) -> Tensor { - self.f_hash_tensor_out(out, dim, keepdim, mode).unwrap() - } - - pub fn heaviside(&self, values: &Tensor) -> Tensor { - self.f_heaviside(values).unwrap() - } - - pub fn heaviside_(&mut self, values: &Tensor) -> Tensor { - self.f_heaviside_(values).unwrap() - } - - pub fn heaviside_out(&self, out: &Tensor, values: &Tensor) -> Tensor { - self.f_heaviside_out(out, values).unwrap() - } - - pub fn hinge_embedding_loss( - &self, - target: &Tensor, - margin: f64, - reduction: crate::Reduction, - ) -> Tensor { - self.f_hinge_embedding_loss(target, margin, reduction).unwrap() - } - - pub fn histc(&self, bins: i64) -> Tensor { - self.f_histc(bins).unwrap() - } - - pub fn histc_out(&self, out: &Tensor, bins: i64) -> Tensor { - self.f_histc_out(out, bins).unwrap() - } - - pub fn histogram>( - &self, - bins: &Tensor, - weight: Option, - density: bool, - ) -> (Tensor, Tensor) { - self.f_histogram(bins, weight, density).unwrap() - } - - pub fn histogram_bin_ct>( - &self, - bins: i64, - range: impl DoubleList, - weight: Option, - density: bool, - ) -> (Tensor, Tensor) { - self.f_histogram_bin_ct(bins, range, weight, density).unwrap() - } - - pub fn histogram_bin_ct_out>( - &self, - hist: &Tensor, - bin_edges: &Tensor, - bins: i64, - range: impl DoubleList, - weight: Option, - density: bool, - ) -> (Tensor, Tensor) { - self.f_histogram_bin_ct_out(hist, bin_edges, bins, range, weight, density).unwrap() - } - - pub fn histogram_bins_tensor_out>( - &self, - hist: &Tensor, - bin_edges: &Tensor, - bins: &Tensor, - weight: Option, - density: bool, - ) -> (Tensor, Tensor) { - self.f_histogram_bins_tensor_out(hist, bin_edges, bins, weight, density).unwrap() - } - - pub fn hsplit(&self, sections: i64) -> Vec { - self.f_hsplit(sections).unwrap() - } - - pub fn hsplit_array(&self, indices: impl IntList) -> Vec { - self.f_hsplit_array(indices).unwrap() - } - - pub fn hspmm(mat1: &Tensor, mat2: &Tensor) -> Tensor { - Tensor::f_hspmm(mat1, mat2).unwrap() - } - - pub fn hspmm_out(out: &Tensor, mat1: &Tensor, mat2: &Tensor) -> Tensor { - Tensor::f_hspmm_out(out, mat1, mat2).unwrap() - } - - pub fn hstack>(tensors: &[T]) -> Tensor { - Tensor::f_hstack(tensors).unwrap() - } - - pub fn hstack_out>(out: &Tensor, tensors: &[T]) -> Tensor { - Tensor::f_hstack_out(out, tensors).unwrap() - } - - pub fn huber_loss(&self, target: &Tensor, reduction: crate::Reduction, delta: f64) -> Tensor { - self.f_huber_loss(target, reduction, delta).unwrap() - } - - pub fn huber_loss_backward( - &self, - grad_output: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - delta: f64, - ) -> Tensor { - self.f_huber_loss_backward(grad_output, target, reduction, delta).unwrap() - } - - pub fn huber_loss_backward_out( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - delta: f64, - ) -> Tensor { - self.f_huber_loss_backward_out(grad_input, grad_output, target, reduction, delta).unwrap() - } - - pub fn huber_loss_out( - &self, - out: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - delta: f64, - ) -> Tensor { - self.f_huber_loss_out(out, target, reduction, delta).unwrap() - } - - pub fn hypot(&self, other: &Tensor) -> Tensor { - self.f_hypot(other).unwrap() - } - - pub fn hypot_(&mut self, other: &Tensor) -> Tensor { - self.f_hypot_(other).unwrap() - } - - pub fn hypot_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_hypot_out(out, other).unwrap() - } - - pub fn i0(&self) -> Tensor { - self.f_i0().unwrap() - } - - pub fn i0_(&mut self) -> Tensor { - self.f_i0_().unwrap() - } - - pub fn i0_out(&self, out: &Tensor) -> Tensor { - self.f_i0_out(out).unwrap() - } - - pub fn igamma(&self, other: &Tensor) -> Tensor { - self.f_igamma(other).unwrap() - } - - pub fn igamma_(&mut self, other: &Tensor) -> Tensor { - self.f_igamma_(other).unwrap() - } - - pub fn igamma_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_igamma_out(out, other).unwrap() - } - - pub fn igammac(&self, other: &Tensor) -> Tensor { - self.f_igammac(other).unwrap() - } - - pub fn igammac_(&mut self, other: &Tensor) -> Tensor { - self.f_igammac_(other).unwrap() - } - - pub fn igammac_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_igammac_out(out, other).unwrap() - } - - pub fn im2col( - &self, - kernel_size: impl IntList, - dilation: impl IntList, - padding: impl IntList, - stride: impl IntList, - ) -> Tensor { - self.f_im2col(kernel_size, dilation, padding, stride).unwrap() - } - - pub fn im2col_out( - &self, - out: &Tensor, - kernel_size: impl IntList, - dilation: impl IntList, - padding: impl IntList, - stride: impl IntList, - ) -> Tensor { - self.f_im2col_out(out, kernel_size, dilation, padding, stride).unwrap() - } - - pub fn imag(&self) -> Tensor { - self.f_imag().unwrap() - } - - pub fn index>(&self, indices: &[Option]) -> Tensor { - self.f_index(indices).unwrap() - } - - pub fn index_add(&self, dim: i64, index: &Tensor, source: &Tensor) -> Tensor { - self.f_index_add(dim, index, source).unwrap() - } - - pub fn index_add_(&mut self, dim: i64, index: &Tensor, source: &Tensor) -> Tensor { - self.f_index_add_(dim, index, source).unwrap() - } - - pub fn index_add_out(&self, out: &Tensor, dim: i64, index: &Tensor, source: &Tensor) -> Tensor { - self.f_index_add_out(out, dim, index, source).unwrap() - } - - pub fn index_copy(&self, dim: i64, index: &Tensor, source: &Tensor) -> Tensor { - self.f_index_copy(dim, index, source).unwrap() - } - - pub fn index_copy_(&mut self, dim: i64, index: &Tensor, source: &Tensor) -> Tensor { - self.f_index_copy_(dim, index, source).unwrap() - } - - pub fn index_copy_out( - &self, - out: &Tensor, - dim: i64, - index: &Tensor, - source: &Tensor, - ) -> Tensor { - self.f_index_copy_out(out, dim, index, source).unwrap() - } - - pub fn index_fill>(&self, dim: i64, index: &Tensor, value: S) -> Tensor { - self.f_index_fill(dim, index, value).unwrap() - } - - pub fn index_fill_>(&mut self, dim: i64, index: &Tensor, value: S) -> Tensor { - self.f_index_fill_(dim, index, value).unwrap() - } - - pub fn index_fill_int_scalar_out>( - &self, - out: &Tensor, - dim: i64, - index: &Tensor, - value: S, - ) -> Tensor { - self.f_index_fill_int_scalar_out(out, dim, index, value).unwrap() - } - - pub fn index_fill_int_tensor(&self, dim: i64, index: &Tensor, value: &Tensor) -> Tensor { - self.f_index_fill_int_tensor(dim, index, value).unwrap() - } - - pub fn index_fill_int_tensor_(&mut self, dim: i64, index: &Tensor, value: &Tensor) -> Tensor { - self.f_index_fill_int_tensor_(dim, index, value).unwrap() - } - - pub fn index_fill_int_tensor_out( - &self, - out: &Tensor, - dim: i64, - index: &Tensor, - value: &Tensor, - ) -> Tensor { - self.f_index_fill_int_tensor_out(out, dim, index, value).unwrap() - } - - pub fn index_put>( - &self, - indices: &[Option], - values: &Tensor, - accumulate: bool, - ) -> Tensor { - self.f_index_put(indices, values, accumulate).unwrap() - } - - pub fn index_put_>( - &mut self, - indices: &[Option], - values: &Tensor, - accumulate: bool, - ) -> Tensor { - self.f_index_put_(indices, values, accumulate).unwrap() - } - - pub fn index_put_out>( - &self, - out: &Tensor, - indices: &[Option], - values: &Tensor, - accumulate: bool, - ) -> Tensor { - self.f_index_put_out(out, indices, values, accumulate).unwrap() - } - - pub fn index_reduce( - &self, - dim: i64, - index: &Tensor, - source: &Tensor, - reduce: &str, - include_self: bool, - ) -> Tensor { - self.f_index_reduce(dim, index, source, reduce, include_self).unwrap() - } - - pub fn index_reduce_( - &mut self, - dim: i64, - index: &Tensor, - source: &Tensor, - reduce: &str, - include_self: bool, - ) -> Tensor { - self.f_index_reduce_(dim, index, source, reduce, include_self).unwrap() - } - - pub fn index_reduce_out( - &self, - out: &Tensor, - dim: i64, - index: &Tensor, - source: &Tensor, - reduce: &str, - include_self: bool, - ) -> Tensor { - self.f_index_reduce_out(out, dim, index, source, reduce, include_self).unwrap() - } - - pub fn index_select(&self, dim: i64, index: &Tensor) -> Tensor { - self.f_index_select(dim, index).unwrap() - } - - pub fn index_select_backward( - grad: &Tensor, - self_sizes: impl IntList, - dim: i64, - index: &Tensor, - ) -> Tensor { - Tensor::f_index_select_backward(grad, self_sizes, dim, index).unwrap() - } - - pub fn index_select_out(&self, out: &Tensor, dim: i64, index: &Tensor) -> Tensor { - self.f_index_select_out(out, dim, index).unwrap() - } - - pub fn index_tensor_out>( - &self, - out: &Tensor, - indices: &[Option], - ) -> Tensor { - self.f_index_tensor_out(out, indices).unwrap() - } - - pub fn indices(&self) -> Tensor { - self.f_indices().unwrap() - } - - pub fn indices_copy(&self) -> Tensor { - self.f_indices_copy().unwrap() - } - - pub fn indices_copy_out(&self, out: &Tensor) -> Tensor { - self.f_indices_copy_out(out).unwrap() - } - - pub fn infinitely_differentiable_gelu_backward(&self, grad: &Tensor) -> Tensor { - self.f_infinitely_differentiable_gelu_backward(grad).unwrap() - } - - pub fn inner(&self, other: &Tensor) -> Tensor { - self.f_inner(other).unwrap() - } - - pub fn inner_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_inner_out(out, other).unwrap() - } - - pub fn instance_norm>( - &self, - weight: Option, - bias: Option, - running_mean: Option, - running_var: Option, - use_input_stats: bool, - momentum: f64, - eps: f64, - cudnn_enabled: bool, - ) -> Tensor { - self.f_instance_norm( - weight, - bias, - running_mean, - running_var, - use_input_stats, - momentum, - eps, - cudnn_enabled, - ) - .unwrap() - } - - pub fn int_repr(&self) -> Tensor { - self.f_int_repr().unwrap() - } - - pub fn int_repr_out(&self, out: &Tensor) -> Tensor { - self.f_int_repr_out(out).unwrap() - } - - pub fn inverse(&self) -> Tensor { - self.f_inverse().unwrap() - } - - pub fn inverse_out(&self, out: &Tensor) -> Tensor { - self.f_inverse_out(out).unwrap() - } - - pub fn is_coalesced(&self) -> bool { - self.f_is_coalesced().unwrap() - } - - pub fn is_complex(&self) -> bool { - self.f_is_complex().unwrap() - } - - pub fn is_conj(&self) -> bool { - self.f_is_conj().unwrap() - } - - pub fn is_distributed(&self) -> bool { - self.f_is_distributed().unwrap() - } - - pub fn is_floating_point(&self) -> bool { - self.f_is_floating_point().unwrap() - } - - pub fn is_inference(&self) -> bool { - self.f_is_inference().unwrap() - } - - pub fn is_leaf(&self) -> bool { - self.f_is_leaf().unwrap() - } - - pub fn is_neg(&self) -> bool { - self.f_is_neg().unwrap() - } - - pub fn is_nonzero(&self) -> bool { - self.f_is_nonzero().unwrap() - } - - pub fn is_pinned(&self, device: Device) -> bool { - self.f_is_pinned(device).unwrap() - } - - pub fn is_same_size(&self, other: &Tensor) -> bool { - self.f_is_same_size(other).unwrap() - } - - pub fn is_set_to(&self, tensor: &Tensor) -> bool { - self.f_is_set_to(tensor).unwrap() - } - - pub fn is_signed(&self) -> bool { - self.f_is_signed().unwrap() - } - - pub fn is_vulkan_available() -> bool { - Tensor::f_is_vulkan_available().unwrap() - } - - pub fn isclose(&self, other: &Tensor, rtol: f64, atol: f64, equal_nan: bool) -> Tensor { - self.f_isclose(other, rtol, atol, equal_nan).unwrap() - } - - pub fn isfinite(&self) -> Tensor { - self.f_isfinite().unwrap() - } - - pub fn isin( - elements: &Tensor, - test_elements: &Tensor, - assume_unique: bool, - invert: bool, - ) -> Tensor { - Tensor::f_isin(elements, test_elements, assume_unique, invert).unwrap() - } - - pub fn isin_scalar_tensor>( - element: S, - test_elements: &Tensor, - assume_unique: bool, - invert: bool, - ) -> Tensor { - Tensor::f_isin_scalar_tensor(element, test_elements, assume_unique, invert).unwrap() - } - - pub fn isin_scalar_tensor_out>( - out: &Tensor, - element: S, - test_elements: &Tensor, - assume_unique: bool, - invert: bool, - ) -> Tensor { - Tensor::f_isin_scalar_tensor_out(out, element, test_elements, assume_unique, invert) - .unwrap() - } - - pub fn isin_tensor_scalar>( - elements: &Tensor, - test_element: S, - assume_unique: bool, - invert: bool, - ) -> Tensor { - Tensor::f_isin_tensor_scalar(elements, test_element, assume_unique, invert).unwrap() - } - - pub fn isin_tensor_scalar_out>( - out: &Tensor, - elements: &Tensor, - test_element: S, - assume_unique: bool, - invert: bool, - ) -> Tensor { - Tensor::f_isin_tensor_scalar_out(out, elements, test_element, assume_unique, invert) - .unwrap() - } - - pub fn isin_tensor_tensor_out( - out: &Tensor, - elements: &Tensor, - test_elements: &Tensor, - assume_unique: bool, - invert: bool, - ) -> Tensor { - Tensor::f_isin_tensor_tensor_out(out, elements, test_elements, assume_unique, invert) - .unwrap() - } - - pub fn isinf(&self) -> Tensor { - self.f_isinf().unwrap() - } - - pub fn isinf_out(&self, out: &Tensor) -> Tensor { - self.f_isinf_out(out).unwrap() - } - - pub fn isnan(&self) -> Tensor { - self.f_isnan().unwrap() - } - - pub fn isnan_out(&self, out: &Tensor) -> Tensor { - self.f_isnan_out(out).unwrap() - } - - pub fn isneginf(&self) -> Tensor { - self.f_isneginf().unwrap() - } - - pub fn isneginf_out(&self, out: &Tensor) -> Tensor { - self.f_isneginf_out(out).unwrap() - } - - pub fn isposinf(&self) -> Tensor { - self.f_isposinf().unwrap() - } - - pub fn isposinf_out(&self, out: &Tensor) -> Tensor { - self.f_isposinf_out(out).unwrap() - } - - pub fn isreal(&self) -> Tensor { - self.f_isreal().unwrap() - } - - pub fn istft>( - &self, - n_fft: i64, - hop_length: impl Into>, - win_length: impl Into>, - window: Option, - center: bool, - normalized: bool, - onesided: bool, - length: impl Into>, - return_complex: bool, - ) -> Tensor { - self.f_istft( - n_fft, - hop_length, - win_length, - window, - center, - normalized, - onesided, - length, - return_complex, - ) - .unwrap() - } - - pub fn kaiser_window(window_length: i64, options: (Kind, Device)) -> Tensor { - Tensor::f_kaiser_window(window_length, options).unwrap() - } - - pub fn kaiser_window_beta( - window_length: i64, - periodic: bool, - beta: f64, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_kaiser_window_beta(window_length, periodic, beta, options).unwrap() - } - - pub fn kaiser_window_beta_out( - out: &Tensor, - window_length: i64, - periodic: bool, - beta: f64, - ) -> Tensor { - Tensor::f_kaiser_window_beta_out(out, window_length, periodic, beta).unwrap() - } - - pub fn kaiser_window_out(out: &Tensor, window_length: i64) -> Tensor { - Tensor::f_kaiser_window_out(out, window_length).unwrap() - } - - pub fn kaiser_window_periodic( - window_length: i64, - periodic: bool, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_kaiser_window_periodic(window_length, periodic, options).unwrap() - } - - pub fn kaiser_window_periodic_out(out: &Tensor, window_length: i64, periodic: bool) -> Tensor { - Tensor::f_kaiser_window_periodic_out(out, window_length, periodic).unwrap() - } - - pub fn kl_div(&self, target: &Tensor, reduction: crate::Reduction, log_target: bool) -> Tensor { - self.f_kl_div(target, reduction, log_target).unwrap() - } - - pub fn kron(&self, other: &Tensor) -> Tensor { - self.f_kron(other).unwrap() - } - - pub fn kron_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_kron_out(out, other).unwrap() - } - - pub fn kthvalue(&self, k: i64, dim: i64, keepdim: bool) -> (Tensor, Tensor) { - self.f_kthvalue(k, dim, keepdim).unwrap() - } - - pub fn kthvalue_values( - &self, - values: &Tensor, - indices: &Tensor, - k: i64, - dim: i64, - keepdim: bool, - ) -> (Tensor, Tensor) { - self.f_kthvalue_values(values, indices, k, dim, keepdim).unwrap() - } - - pub fn l1_loss(&self, target: &Tensor, reduction: crate::Reduction) -> Tensor { - self.f_l1_loss(target, reduction).unwrap() - } - - pub fn layer_norm>( - &self, - normalized_shape: impl IntList, - weight: Option, - bias: Option, - eps: f64, - cudnn_enable: bool, - ) -> Tensor { - self.f_layer_norm(normalized_shape, weight, bias, eps, cudnn_enable).unwrap() - } - - pub fn lcm(&self, other: &Tensor) -> Tensor { - self.f_lcm(other).unwrap() - } - - pub fn lcm_(&mut self, other: &Tensor) -> Tensor { - self.f_lcm_(other).unwrap() - } - - pub fn lcm_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_lcm_out(out, other).unwrap() - } - - pub fn ldexp(&self, other: &Tensor) -> Tensor { - self.f_ldexp(other).unwrap() - } - - pub fn ldexp_(&mut self, other: &Tensor) -> Tensor { - self.f_ldexp_(other).unwrap() - } - - pub fn ldexp_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_ldexp_out(out, other).unwrap() - } - - pub fn le>(&self, other: S) -> Tensor { - self.f_le(other).unwrap() - } - - pub fn le_>(&mut self, other: S) -> Tensor { - self.f_le_(other).unwrap() - } - - pub fn le_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_le_scalar_out(out, other).unwrap() - } - - pub fn le_tensor(&self, other: &Tensor) -> Tensor { - self.f_le_tensor(other).unwrap() - } - - pub fn le_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_le_tensor_(other).unwrap() - } - - pub fn le_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_le_tensor_out(out, other).unwrap() - } - - pub fn leaky_relu(&self) -> Tensor { - self.f_leaky_relu().unwrap() - } - - pub fn leaky_relu_(&mut self) -> Tensor { - self.f_leaky_relu_().unwrap() - } - - pub fn leaky_relu_backward>( - &self, - grad_output: &Tensor, - negative_slope: S, - self_is_result: bool, - ) -> Tensor { - self.f_leaky_relu_backward(grad_output, negative_slope, self_is_result).unwrap() - } - - pub fn leaky_relu_backward_grad_input>( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - negative_slope: S, - self_is_result: bool, - ) -> Tensor { - self.f_leaky_relu_backward_grad_input( - grad_input, - grad_output, - negative_slope, - self_is_result, - ) - .unwrap() - } - - pub fn leaky_relu_out(&self, out: &Tensor) -> Tensor { - self.f_leaky_relu_out(out).unwrap() - } - - pub fn lerp>(&self, end: &Tensor, weight: S) -> Tensor { - self.f_lerp(end, weight).unwrap() - } - - pub fn lerp_>(&mut self, end: &Tensor, weight: S) -> Tensor { - self.f_lerp_(end, weight).unwrap() - } - - pub fn lerp_scalar_out>( - &self, - out: &Tensor, - end: &Tensor, - weight: S, - ) -> Tensor { - self.f_lerp_scalar_out(out, end, weight).unwrap() - } - - pub fn lerp_tensor(&self, end: &Tensor, weight: &Tensor) -> Tensor { - self.f_lerp_tensor(end, weight).unwrap() - } - - pub fn lerp_tensor_(&mut self, end: &Tensor, weight: &Tensor) -> Tensor { - self.f_lerp_tensor_(end, weight).unwrap() - } - - pub fn lerp_tensor_out(&self, out: &Tensor, end: &Tensor, weight: &Tensor) -> Tensor { - self.f_lerp_tensor_out(out, end, weight).unwrap() - } - - pub fn less>(&self, other: S) -> Tensor { - self.f_less(other).unwrap() - } - - pub fn less_>(&mut self, other: S) -> Tensor { - self.f_less_(other).unwrap() - } - - pub fn less_equal>(&self, other: S) -> Tensor { - self.f_less_equal(other).unwrap() - } - - pub fn less_equal_>(&mut self, other: S) -> Tensor { - self.f_less_equal_(other).unwrap() - } - - pub fn less_equal_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_less_equal_scalar_out(out, other).unwrap() - } - - pub fn less_equal_tensor(&self, other: &Tensor) -> Tensor { - self.f_less_equal_tensor(other).unwrap() - } - - pub fn less_equal_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_less_equal_tensor_(other).unwrap() - } - - pub fn less_equal_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_less_equal_tensor_out(out, other).unwrap() - } - - pub fn less_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_less_scalar_out(out, other).unwrap() - } - - pub fn less_tensor(&self, other: &Tensor) -> Tensor { - self.f_less_tensor(other).unwrap() - } - - pub fn less_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_less_tensor_(other).unwrap() - } - - pub fn less_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_less_tensor_out(out, other).unwrap() - } - - pub fn lgamma(&self) -> Tensor { - self.f_lgamma().unwrap() - } - - pub fn lgamma_(&mut self) -> Tensor { - self.f_lgamma_().unwrap() - } - - pub fn lgamma_out(&self, out: &Tensor) -> Tensor { - self.f_lgamma_out(out).unwrap() - } - - pub fn lift(&self) -> Tensor { - self.f_lift().unwrap() - } - - pub fn lift_fresh(&self) -> Tensor { - self.f_lift_fresh().unwrap() - } - - pub fn lift_fresh_copy(&self) -> Tensor { - self.f_lift_fresh_copy().unwrap() - } - - pub fn lift_fresh_copy_out(&self, out: &Tensor) -> Tensor { - self.f_lift_fresh_copy_out(out).unwrap() - } - - pub fn lift_out(&self, out: &Tensor) -> Tensor { - self.f_lift_out(out).unwrap() - } - - pub fn linalg_cholesky(&self, upper: bool) -> Tensor { - self.f_linalg_cholesky(upper).unwrap() - } - - pub fn linalg_cholesky_ex(&self, upper: bool, check_errors: bool) -> (Tensor, Tensor) { - self.f_linalg_cholesky_ex(upper, check_errors).unwrap() - } - - pub fn linalg_cholesky_ex_l( - &self, - l: &Tensor, - info: &Tensor, - upper: bool, - check_errors: bool, - ) -> (Tensor, Tensor) { - self.f_linalg_cholesky_ex_l(l, info, upper, check_errors).unwrap() - } - - pub fn linalg_cholesky_out(&self, out: &Tensor, upper: bool) -> Tensor { - self.f_linalg_cholesky_out(out, upper).unwrap() - } - - pub fn linalg_cond>(&self, p: S) -> Tensor { - self.f_linalg_cond(p).unwrap() - } - - pub fn linalg_cond_out>(&self, out: &Tensor, p: S) -> Tensor { - self.f_linalg_cond_out(out, p).unwrap() - } - - pub fn linalg_cond_p_str(&self, p: &str) -> Tensor { - self.f_linalg_cond_p_str(p).unwrap() - } - - pub fn linalg_cond_p_str_out(&self, out: &Tensor, p: &str) -> Tensor { - self.f_linalg_cond_p_str_out(out, p).unwrap() - } - - pub fn linalg_cross(&self, other: &Tensor, dim: i64) -> Tensor { - self.f_linalg_cross(other, dim).unwrap() - } - - pub fn linalg_cross_out(&self, out: &Tensor, other: &Tensor, dim: i64) -> Tensor { - self.f_linalg_cross_out(out, other, dim).unwrap() - } - - pub fn linalg_det(a: &Tensor) -> Tensor { - Tensor::f_linalg_det(a).unwrap() - } - - pub fn linalg_det_out(out: &Tensor, a: &Tensor) -> Tensor { - Tensor::f_linalg_det_out(out, a).unwrap() - } - - pub fn linalg_diagonal(a: &Tensor, offset: i64, dim1: i64, dim2: i64) -> Tensor { - Tensor::f_linalg_diagonal(a, offset, dim1, dim2).unwrap() - } - - pub fn linalg_eig(&self) -> (Tensor, Tensor) { - self.f_linalg_eig().unwrap() - } - - pub fn linalg_eig_out(&self, eigenvalues: &Tensor, eigenvectors: &Tensor) -> (Tensor, Tensor) { - self.f_linalg_eig_out(eigenvalues, eigenvectors).unwrap() - } - - pub fn linalg_eigh(&self, uplo: &str) -> (Tensor, Tensor) { - self.f_linalg_eigh(uplo).unwrap() - } - - pub fn linalg_eigh_eigvals( - &self, - eigvals: &Tensor, - eigvecs: &Tensor, - uplo: &str, - ) -> (Tensor, Tensor) { - self.f_linalg_eigh_eigvals(eigvals, eigvecs, uplo).unwrap() - } - - pub fn linalg_eigvals(&self) -> Tensor { - self.f_linalg_eigvals().unwrap() - } - - pub fn linalg_eigvals_out(&self, out: &Tensor) -> Tensor { - self.f_linalg_eigvals_out(out).unwrap() - } - - pub fn linalg_eigvalsh(&self, uplo: &str) -> Tensor { - self.f_linalg_eigvalsh(uplo).unwrap() - } - - pub fn linalg_eigvalsh_out(&self, out: &Tensor, uplo: &str) -> Tensor { - self.f_linalg_eigvalsh_out(out, uplo).unwrap() - } - - pub fn linalg_householder_product(&self, tau: &Tensor) -> Tensor { - self.f_linalg_householder_product(tau).unwrap() - } - - pub fn linalg_householder_product_out(&self, out: &Tensor, tau: &Tensor) -> Tensor { - self.f_linalg_householder_product_out(out, tau).unwrap() - } - - pub fn linalg_inv(a: &Tensor) -> Tensor { - Tensor::f_linalg_inv(a).unwrap() - } - - pub fn linalg_inv_ex(a: &Tensor, check_errors: bool) -> (Tensor, Tensor) { - Tensor::f_linalg_inv_ex(a, check_errors).unwrap() - } - - pub fn linalg_inv_ex_inverse( - inverse: &Tensor, - info: &Tensor, - a: &Tensor, - check_errors: bool, - ) -> (Tensor, Tensor) { - Tensor::f_linalg_inv_ex_inverse(inverse, info, a, check_errors).unwrap() - } - - pub fn linalg_inv_out(out: &Tensor, a: &Tensor) -> Tensor { - Tensor::f_linalg_inv_out(out, a).unwrap() - } - - pub fn linalg_ldl_factor(&self, hermitian: bool) -> (Tensor, Tensor) { - self.f_linalg_ldl_factor(hermitian).unwrap() - } - - pub fn linalg_ldl_factor_ex( - &self, - hermitian: bool, - check_errors: bool, - ) -> (Tensor, Tensor, Tensor) { - self.f_linalg_ldl_factor_ex(hermitian, check_errors).unwrap() - } - - pub fn linalg_ldl_factor_ex_out( - &self, - ld: &Tensor, - pivots: &Tensor, - info: &Tensor, - hermitian: bool, - check_errors: bool, - ) -> (Tensor, Tensor, Tensor) { - self.f_linalg_ldl_factor_ex_out(ld, pivots, info, hermitian, check_errors).unwrap() - } - - pub fn linalg_ldl_factor_out( - &self, - ld: &Tensor, - pivots: &Tensor, - hermitian: bool, - ) -> (Tensor, Tensor) { - self.f_linalg_ldl_factor_out(ld, pivots, hermitian).unwrap() - } - - pub fn linalg_ldl_solve(ld: &Tensor, pivots: &Tensor, b: &Tensor, hermitian: bool) -> Tensor { - Tensor::f_linalg_ldl_solve(ld, pivots, b, hermitian).unwrap() - } - - pub fn linalg_ldl_solve_out( - out: &Tensor, - ld: &Tensor, - pivots: &Tensor, - b: &Tensor, - hermitian: bool, - ) -> Tensor { - Tensor::f_linalg_ldl_solve_out(out, ld, pivots, b, hermitian).unwrap() - } - - pub fn linalg_lstsq( - &self, - b: &Tensor, - rcond: impl Into>, - driver: &str, - ) -> (Tensor, Tensor, Tensor, Tensor) { - self.f_linalg_lstsq(b, rcond, driver).unwrap() - } - - pub fn linalg_lstsq_out( - &self, - solution: &Tensor, - residuals: &Tensor, - rank: &Tensor, - singular_values: &Tensor, - b: &Tensor, - rcond: impl Into>, - driver: &str, - ) -> (Tensor, Tensor, Tensor, Tensor) { - self.f_linalg_lstsq_out(solution, residuals, rank, singular_values, b, rcond, driver) - .unwrap() - } - - pub fn linalg_lu(a: &Tensor, pivot: bool) -> (Tensor, Tensor, Tensor) { - Tensor::f_linalg_lu(a, pivot).unwrap() - } - - pub fn linalg_lu_factor(a: &Tensor, pivot: bool) -> (Tensor, Tensor) { - Tensor::f_linalg_lu_factor(a, pivot).unwrap() - } - - pub fn linalg_lu_factor_ex( - a: &Tensor, - pivot: bool, - check_errors: bool, - ) -> (Tensor, Tensor, Tensor) { - Tensor::f_linalg_lu_factor_ex(a, pivot, check_errors).unwrap() - } - - pub fn linalg_lu_factor_ex_out( - lu: &Tensor, - pivots: &Tensor, - info: &Tensor, - a: &Tensor, - pivot: bool, - check_errors: bool, - ) -> (Tensor, Tensor, Tensor) { - Tensor::f_linalg_lu_factor_ex_out(lu, pivots, info, a, pivot, check_errors).unwrap() - } - - pub fn linalg_lu_factor_out( - lu: &Tensor, - pivots: &Tensor, - a: &Tensor, - pivot: bool, - ) -> (Tensor, Tensor) { - Tensor::f_linalg_lu_factor_out(lu, pivots, a, pivot).unwrap() - } - - pub fn linalg_lu_out( - p: &Tensor, - l: &Tensor, - u: &Tensor, - a: &Tensor, - pivot: bool, - ) -> (Tensor, Tensor, Tensor) { - Tensor::f_linalg_lu_out(p, l, u, a, pivot).unwrap() - } - - pub fn linalg_lu_solve( - lu: &Tensor, - pivots: &Tensor, - b: &Tensor, - left: bool, - adjoint: bool, - ) -> Tensor { - Tensor::f_linalg_lu_solve(lu, pivots, b, left, adjoint).unwrap() - } - - pub fn linalg_lu_solve_out( - out: &Tensor, - lu: &Tensor, - pivots: &Tensor, - b: &Tensor, - left: bool, - adjoint: bool, - ) -> Tensor { - Tensor::f_linalg_lu_solve_out(out, lu, pivots, b, left, adjoint).unwrap() - } - - pub fn linalg_matmul(&self, other: &Tensor) -> Tensor { - self.f_linalg_matmul(other).unwrap() - } - - pub fn linalg_matmul_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_linalg_matmul_out(out, other).unwrap() - } - - pub fn linalg_matrix_exp(&self) -> Tensor { - self.f_linalg_matrix_exp().unwrap() - } - - pub fn linalg_matrix_exp_out(&self, out: &Tensor) -> Tensor { - self.f_linalg_matrix_exp_out(out).unwrap() - } - - pub fn linalg_matrix_power(&self, n: i64) -> Tensor { - self.f_linalg_matrix_power(n).unwrap() - } - - pub fn linalg_matrix_power_out(&self, out: &Tensor, n: i64) -> Tensor { - self.f_linalg_matrix_power_out(out, n).unwrap() - } - - pub fn linalg_matrix_rank(&self, tol: f64, hermitian: bool) -> Tensor { - self.f_linalg_matrix_rank(tol, hermitian).unwrap() - } - - pub fn linalg_matrix_rank_atol_rtol_float( - &self, - atol: impl Into>, - rtol: impl Into>, - hermitian: bool, - ) -> Tensor { - self.f_linalg_matrix_rank_atol_rtol_float(atol, rtol, hermitian).unwrap() - } - - pub fn linalg_matrix_rank_atol_rtol_float_out( - &self, - out: &Tensor, - atol: impl Into>, - rtol: impl Into>, - hermitian: bool, - ) -> Tensor { - self.f_linalg_matrix_rank_atol_rtol_float_out(out, atol, rtol, hermitian).unwrap() - } - - pub fn linalg_matrix_rank_atol_rtol_tensor>( - &self, - atol: Option, - rtol: Option, - hermitian: bool, - ) -> Tensor { - self.f_linalg_matrix_rank_atol_rtol_tensor(atol, rtol, hermitian).unwrap() - } - - pub fn linalg_matrix_rank_atol_rtol_tensor_out>( - &self, - out: &Tensor, - atol: Option, - rtol: Option, - hermitian: bool, - ) -> Tensor { - self.f_linalg_matrix_rank_atol_rtol_tensor_out(out, atol, rtol, hermitian).unwrap() - } - - pub fn linalg_matrix_rank_out(&self, out: &Tensor, tol: f64, hermitian: bool) -> Tensor { - self.f_linalg_matrix_rank_out(out, tol, hermitian).unwrap() - } - - pub fn linalg_matrix_rank_out_tol_tensor( - &self, - out: &Tensor, - tol: &Tensor, - hermitian: bool, - ) -> Tensor { - self.f_linalg_matrix_rank_out_tol_tensor(out, tol, hermitian).unwrap() - } - - pub fn linalg_matrix_rank_tol_tensor(&self, tol: &Tensor, hermitian: bool) -> Tensor { - self.f_linalg_matrix_rank_tol_tensor(tol, hermitian).unwrap() - } - - pub fn linalg_multi_dot>(tensors: &[T]) -> Tensor { - Tensor::f_linalg_multi_dot(tensors).unwrap() - } - - pub fn linalg_multi_dot_out>(out: &Tensor, tensors: &[T]) -> Tensor { - Tensor::f_linalg_multi_dot_out(out, tensors).unwrap() - } - - pub fn linalg_norm>( - &self, - ord: S, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Tensor { - self.f_linalg_norm(ord, dim, keepdim, dtype).unwrap() - } - - pub fn linalg_norm_ord_str( - &self, - ord: &str, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Tensor { - self.f_linalg_norm_ord_str(ord, dim, keepdim, dtype).unwrap() - } - - pub fn linalg_norm_ord_str_out( - &self, - out: &Tensor, - ord: &str, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Tensor { - self.f_linalg_norm_ord_str_out(out, ord, dim, keepdim, dtype).unwrap() - } - - pub fn linalg_norm_out>( - &self, - out: &Tensor, - ord: S, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Tensor { - self.f_linalg_norm_out(out, ord, dim, keepdim, dtype).unwrap() - } - - pub fn linalg_pinv(&self, rcond: f64, hermitian: bool) -> Tensor { - self.f_linalg_pinv(rcond, hermitian).unwrap() - } - - pub fn linalg_pinv_atol_rtol_float( - &self, - atol: impl Into>, - rtol: impl Into>, - hermitian: bool, - ) -> Tensor { - self.f_linalg_pinv_atol_rtol_float(atol, rtol, hermitian).unwrap() - } - - pub fn linalg_pinv_atol_rtol_float_out( - &self, - out: &Tensor, - atol: impl Into>, - rtol: impl Into>, - hermitian: bool, - ) -> Tensor { - self.f_linalg_pinv_atol_rtol_float_out(out, atol, rtol, hermitian).unwrap() - } - - pub fn linalg_pinv_atol_rtol_tensor>( - &self, - atol: Option, - rtol: Option, - hermitian: bool, - ) -> Tensor { - self.f_linalg_pinv_atol_rtol_tensor(atol, rtol, hermitian).unwrap() - } - - pub fn linalg_pinv_atol_rtol_tensor_out>( - &self, - out: &Tensor, - atol: Option, - rtol: Option, - hermitian: bool, - ) -> Tensor { - self.f_linalg_pinv_atol_rtol_tensor_out(out, atol, rtol, hermitian).unwrap() - } - - pub fn linalg_pinv_out(&self, out: &Tensor, rcond: f64, hermitian: bool) -> Tensor { - self.f_linalg_pinv_out(out, rcond, hermitian).unwrap() - } - - pub fn linalg_pinv_out_rcond_tensor( - &self, - out: &Tensor, - rcond: &Tensor, - hermitian: bool, - ) -> Tensor { - self.f_linalg_pinv_out_rcond_tensor(out, rcond, hermitian).unwrap() - } - - pub fn linalg_pinv_rcond_tensor(&self, rcond: &Tensor, hermitian: bool) -> Tensor { - self.f_linalg_pinv_rcond_tensor(rcond, hermitian).unwrap() - } - - pub fn linalg_qr(a: &Tensor, mode: &str) -> (Tensor, Tensor) { - Tensor::f_linalg_qr(a, mode).unwrap() - } - - pub fn linalg_qr_out(q: &Tensor, r: &Tensor, a: &Tensor, mode: &str) -> (Tensor, Tensor) { - Tensor::f_linalg_qr_out(q, r, a, mode).unwrap() - } - - pub fn linalg_slogdet(a: &Tensor) -> (Tensor, Tensor) { - Tensor::f_linalg_slogdet(a).unwrap() - } - - pub fn linalg_slogdet_out(sign: &Tensor, logabsdet: &Tensor, a: &Tensor) -> (Tensor, Tensor) { - Tensor::f_linalg_slogdet_out(sign, logabsdet, a).unwrap() - } - - pub fn linalg_solve(a: &Tensor, b: &Tensor, left: bool) -> Tensor { - Tensor::f_linalg_solve(a, b, left).unwrap() - } - - pub fn linalg_solve_ex( - a: &Tensor, - b: &Tensor, - left: bool, - check_errors: bool, - ) -> (Tensor, Tensor) { - Tensor::f_linalg_solve_ex(a, b, left, check_errors).unwrap() - } - - pub fn linalg_solve_ex_out( - result: &Tensor, - info: &Tensor, - a: &Tensor, - b: &Tensor, - left: bool, - check_errors: bool, - ) -> (Tensor, Tensor) { - Tensor::f_linalg_solve_ex_out(result, info, a, b, left, check_errors).unwrap() - } - - pub fn linalg_solve_out(out: &Tensor, a: &Tensor, b: &Tensor, left: bool) -> Tensor { - Tensor::f_linalg_solve_out(out, a, b, left).unwrap() - } - - pub fn linalg_solve_triangular( - &self, - b: &Tensor, - upper: bool, - left: bool, - unitriangular: bool, - ) -> Tensor { - self.f_linalg_solve_triangular(b, upper, left, unitriangular).unwrap() - } - - pub fn linalg_solve_triangular_out( - &self, - out: &Tensor, - b: &Tensor, - upper: bool, - left: bool, - unitriangular: bool, - ) -> Tensor { - self.f_linalg_solve_triangular_out(out, b, upper, left, unitriangular).unwrap() - } - - pub fn linalg_svd(a: &Tensor, full_matrices: bool, driver: &str) -> (Tensor, Tensor, Tensor) { - Tensor::f_linalg_svd(a, full_matrices, driver).unwrap() - } - - pub fn linalg_svd_u( - u: &Tensor, - s: &Tensor, - vh: &Tensor, - a: &Tensor, - full_matrices: bool, - driver: &str, - ) -> (Tensor, Tensor, Tensor) { - Tensor::f_linalg_svd_u(u, s, vh, a, full_matrices, driver).unwrap() - } - - pub fn linalg_svdvals(a: &Tensor, driver: &str) -> Tensor { - Tensor::f_linalg_svdvals(a, driver).unwrap() - } - - pub fn linalg_svdvals_out(out: &Tensor, a: &Tensor, driver: &str) -> Tensor { - Tensor::f_linalg_svdvals_out(out, a, driver).unwrap() - } - - pub fn linalg_tensorinv(&self, ind: i64) -> Tensor { - self.f_linalg_tensorinv(ind).unwrap() - } - - pub fn linalg_tensorinv_out(&self, out: &Tensor, ind: i64) -> Tensor { - self.f_linalg_tensorinv_out(out, ind).unwrap() - } - - pub fn linalg_tensorsolve(&self, other: &Tensor, dims: impl IntListOption) -> Tensor { - self.f_linalg_tensorsolve(other, dims).unwrap() - } - - pub fn linalg_tensorsolve_out( - &self, - out: &Tensor, - other: &Tensor, - dims: impl IntListOption, - ) -> Tensor { - self.f_linalg_tensorsolve_out(out, other, dims).unwrap() - } - - pub fn linalg_vander(x: &Tensor, n: impl Into>) -> Tensor { - Tensor::f_linalg_vander(x, n).unwrap() - } - - pub fn linalg_vecdot(x: &Tensor, y: &Tensor, dim: i64) -> Tensor { - Tensor::f_linalg_vecdot(x, y, dim).unwrap() - } - - pub fn linalg_vecdot_out(out: &Tensor, x: &Tensor, y: &Tensor, dim: i64) -> Tensor { - Tensor::f_linalg_vecdot_out(out, x, y, dim).unwrap() - } - - pub fn linear>(&self, weight: &Tensor, bias: Option) -> Tensor { - self.f_linear(weight, bias).unwrap() - } - - pub fn linear_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - ) -> Tensor { - self.f_linear_out(out, weight, bias).unwrap() - } - - pub fn linspace>( - start: S, - end: S, - steps: i64, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_linspace(start, end, steps, options).unwrap() - } - - pub fn linspace_out>(out: &Tensor, start: S, end: S, steps: i64) -> Tensor { - Tensor::f_linspace_out(out, start, end, steps).unwrap() - } - - pub fn linspace_scalar_tensor>( - start: S, - end: &Tensor, - steps: i64, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_linspace_scalar_tensor(start, end, steps, options).unwrap() - } - - pub fn linspace_scalar_tensor_out>( - out: &Tensor, - start: S, - end: &Tensor, - steps: i64, - ) -> Tensor { - Tensor::f_linspace_scalar_tensor_out(out, start, end, steps).unwrap() - } - - pub fn linspace_tensor_scalar>( - start: &Tensor, - end: S, - steps: i64, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_linspace_tensor_scalar(start, end, steps, options).unwrap() - } - - pub fn linspace_tensor_scalar_out>( - out: &Tensor, - start: &Tensor, - end: S, - steps: i64, - ) -> Tensor { - Tensor::f_linspace_tensor_scalar_out(out, start, end, steps).unwrap() - } - - pub fn linspace_tensor_tensor( - start: &Tensor, - end: &Tensor, - steps: i64, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_linspace_tensor_tensor(start, end, steps, options).unwrap() - } - - pub fn linspace_tensor_tensor_out( - out: &Tensor, - start: &Tensor, - end: &Tensor, - steps: i64, - ) -> Tensor { - Tensor::f_linspace_tensor_tensor_out(out, start, end, steps).unwrap() - } - - pub fn log(&self) -> Tensor { - self.f_log().unwrap() - } - - pub fn log10(&self) -> Tensor { - self.f_log10().unwrap() - } - - pub fn log10_(&mut self) -> Tensor { - self.f_log10_().unwrap() - } - - pub fn log10_out(&self, out: &Tensor) -> Tensor { - self.f_log10_out(out).unwrap() - } - - pub fn log1p(&self) -> Tensor { - self.f_log1p().unwrap() - } - - pub fn log1p_(&mut self) -> Tensor { - self.f_log1p_().unwrap() - } - - pub fn log1p_out(&self, out: &Tensor) -> Tensor { - self.f_log1p_out(out).unwrap() - } - - pub fn log2(&self) -> Tensor { - self.f_log2().unwrap() - } - - pub fn log2_(&mut self) -> Tensor { - self.f_log2_().unwrap() - } - - pub fn log2_out(&self, out: &Tensor) -> Tensor { - self.f_log2_out(out).unwrap() - } - - pub fn log_(&mut self) -> Tensor { - self.f_log_().unwrap() - } - - pub fn log_normal(&self, mean: f64, std: f64) -> Tensor { - self.f_log_normal(mean, std).unwrap() - } - - pub fn log_normal_(&mut self, mean: f64, std: f64) -> Tensor { - self.f_log_normal_(mean, std).unwrap() - } - - pub fn log_normal_out(&self, out: &Tensor, mean: f64, std: f64) -> Tensor { - self.f_log_normal_out(out, mean, std).unwrap() - } - - pub fn log_out(&self, out: &Tensor) -> Tensor { - self.f_log_out(out).unwrap() - } - - pub fn log_sigmoid(&self) -> Tensor { - self.f_log_sigmoid().unwrap() - } - - pub fn log_sigmoid_backward(&self, grad_output: &Tensor, buffer: &Tensor) -> Tensor { - self.f_log_sigmoid_backward(grad_output, buffer).unwrap() - } - - pub fn log_sigmoid_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - buffer: &Tensor, - ) -> Tensor { - self.f_log_sigmoid_backward_grad_input(grad_input, grad_output, buffer).unwrap() - } - - pub fn log_sigmoid_out(&self, out: &Tensor) -> Tensor { - self.f_log_sigmoid_out(out).unwrap() - } - - pub fn log_softmax(&self, dim: i64, dtype: impl Into>) -> Tensor { - self.f_log_softmax(dim, dtype).unwrap() - } - - pub fn log_softmax_int_out( - &self, - out: &Tensor, - dim: i64, - dtype: impl Into>, - ) -> Tensor { - self.f_log_softmax_int_out(out, dim, dtype).unwrap() - } - - pub fn logaddexp(&self, other: &Tensor) -> Tensor { - self.f_logaddexp(other).unwrap() - } - - pub fn logaddexp2(&self, other: &Tensor) -> Tensor { - self.f_logaddexp2(other).unwrap() - } - - pub fn logaddexp2_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_logaddexp2_out(out, other).unwrap() - } - - pub fn logaddexp_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_logaddexp_out(out, other).unwrap() - } - - pub fn logcumsumexp(&self, dim: i64) -> Tensor { - self.f_logcumsumexp(dim).unwrap() - } - - pub fn logcumsumexp_out(&self, out: &Tensor, dim: i64) -> Tensor { - self.f_logcumsumexp_out(out, dim).unwrap() - } - - pub fn logdet(&self) -> Tensor { - self.f_logdet().unwrap() - } - - pub fn logical_and(&self, other: &Tensor) -> Tensor { - self.f_logical_and(other).unwrap() - } - - pub fn logical_and_(&mut self, other: &Tensor) -> Tensor { - self.f_logical_and_(other).unwrap() - } - - pub fn logical_and_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_logical_and_out(out, other).unwrap() - } - - pub fn logical_not(&self) -> Tensor { - self.f_logical_not().unwrap() - } - - pub fn logical_not_(&mut self) -> Tensor { - self.f_logical_not_().unwrap() - } - - pub fn logical_not_out(&self, out: &Tensor) -> Tensor { - self.f_logical_not_out(out).unwrap() - } - - pub fn logical_or(&self, other: &Tensor) -> Tensor { - self.f_logical_or(other).unwrap() - } - - pub fn logical_or_(&mut self, other: &Tensor) -> Tensor { - self.f_logical_or_(other).unwrap() - } - - pub fn logical_or_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_logical_or_out(out, other).unwrap() - } - - pub fn logical_xor(&self, other: &Tensor) -> Tensor { - self.f_logical_xor(other).unwrap() - } - - pub fn logical_xor_(&mut self, other: &Tensor) -> Tensor { - self.f_logical_xor_(other).unwrap() - } - - pub fn logical_xor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_logical_xor_out(out, other).unwrap() - } - - pub fn logit(&self, eps: impl Into>) -> Tensor { - self.f_logit(eps).unwrap() - } - - pub fn logit_(&mut self, eps: impl Into>) -> Tensor { - self.f_logit_(eps).unwrap() - } - - pub fn logit_backward(&self, grad_output: &Tensor, eps: impl Into>) -> Tensor { - self.f_logit_backward(grad_output, eps).unwrap() - } - - pub fn logit_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - eps: impl Into>, - ) -> Tensor { - self.f_logit_backward_grad_input(grad_input, grad_output, eps).unwrap() - } - - pub fn logit_out(&self, out: &Tensor, eps: impl Into>) -> Tensor { - self.f_logit_out(out, eps).unwrap() - } - - pub fn logspace>( - start: S, - end: S, - steps: i64, - base: f64, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_logspace(start, end, steps, base, options).unwrap() - } - - pub fn logspace_out>( - out: &Tensor, - start: S, - end: S, - steps: i64, - base: f64, - ) -> Tensor { - Tensor::f_logspace_out(out, start, end, steps, base).unwrap() - } - - pub fn logspace_scalar_tensor>( - start: S, - end: &Tensor, - steps: i64, - base: f64, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_logspace_scalar_tensor(start, end, steps, base, options).unwrap() - } - - pub fn logspace_scalar_tensor_out>( - out: &Tensor, - start: S, - end: &Tensor, - steps: i64, - base: f64, - ) -> Tensor { - Tensor::f_logspace_scalar_tensor_out(out, start, end, steps, base).unwrap() - } - - pub fn logspace_tensor_scalar>( - start: &Tensor, - end: S, - steps: i64, - base: f64, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_logspace_tensor_scalar(start, end, steps, base, options).unwrap() - } - - pub fn logspace_tensor_scalar_out>( - out: &Tensor, - start: &Tensor, - end: S, - steps: i64, - base: f64, - ) -> Tensor { - Tensor::f_logspace_tensor_scalar_out(out, start, end, steps, base).unwrap() - } - - pub fn logspace_tensor_tensor( - start: &Tensor, - end: &Tensor, - steps: i64, - base: f64, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_logspace_tensor_tensor(start, end, steps, base, options).unwrap() - } - - pub fn logspace_tensor_tensor_out( - out: &Tensor, - start: &Tensor, - end: &Tensor, - steps: i64, - base: f64, - ) -> Tensor { - Tensor::f_logspace_tensor_tensor_out(out, start, end, steps, base).unwrap() - } - - pub fn logsumexp(&self, dim: impl IntList, keepdim: bool) -> Tensor { - self.f_logsumexp(dim, keepdim).unwrap() - } - - pub fn logsumexp_out(&self, out: &Tensor, dim: impl IntList, keepdim: bool) -> Tensor { - self.f_logsumexp_out(out, dim, keepdim).unwrap() - } - - pub fn lstm>( - &self, - hx: &[T], - params: &[T], - has_biases: bool, - num_layers: i64, - dropout: f64, - train: bool, - bidirectional: bool, - batch_first: bool, - ) -> (Tensor, Tensor, Tensor) { - self.f_lstm(hx, params, has_biases, num_layers, dropout, train, bidirectional, batch_first) - .unwrap() - } - - pub fn lstm_cell>( - &self, - hx: &[T], - w_ih: &Tensor, - w_hh: &Tensor, - b_ih: Option, - b_hh: Option, - ) -> (Tensor, Tensor) { - self.f_lstm_cell(hx, w_ih, w_hh, b_ih, b_hh).unwrap() - } - - pub fn lstm_data>( - data: &Tensor, - batch_sizes: &Tensor, - hx: &[T], - params: &[T], - has_biases: bool, - num_layers: i64, - dropout: f64, - train: bool, - bidirectional: bool, - ) -> (Tensor, Tensor, Tensor) { - Tensor::f_lstm_data( - data, - batch_sizes, - hx, - params, - has_biases, - num_layers, - dropout, - train, - bidirectional, - ) - .unwrap() - } - - pub fn lstm_mps_backward>( - &self, - out0: &Tensor, - out1: &[T], - out2: &[T], - grad_y: Option, - grad_hy: Option, - grad_cy: Option, - z_state: &Tensor, - cell_state_fwd: &Tensor, - layersoutputs: &Tensor, - hx: &[T], - params: &[T], - has_biases: bool, - num_layers: i64, - dropout: f64, - train: bool, - bidirectional: bool, - batch_first: bool, - ) { - self.f_lstm_mps_backward( - out0, - out1, - out2, - grad_y, - grad_hy, - grad_cy, - z_state, - cell_state_fwd, - layersoutputs, - hx, - params, - has_biases, - num_layers, - dropout, - train, - bidirectional, - batch_first, - ) - .unwrap() - } - - pub fn lt>(&self, other: S) -> Tensor { - self.f_lt(other).unwrap() - } - - pub fn lt_>(&mut self, other: S) -> Tensor { - self.f_lt_(other).unwrap() - } - - pub fn lt_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_lt_scalar_out(out, other).unwrap() - } - - pub fn lt_tensor(&self, other: &Tensor) -> Tensor { - self.f_lt_tensor(other).unwrap() - } - - pub fn lt_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_lt_tensor_(other).unwrap() - } - - pub fn lt_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_lt_tensor_out(out, other).unwrap() - } - - pub fn lu_solve(&self, lu_data: &Tensor, lu_pivots: &Tensor) -> Tensor { - self.f_lu_solve(lu_data, lu_pivots).unwrap() - } - - pub fn lu_solve_out(&self, out: &Tensor, lu_data: &Tensor, lu_pivots: &Tensor) -> Tensor { - self.f_lu_solve_out(out, lu_data, lu_pivots).unwrap() - } - - pub fn lu_unpack( - lu_data: &Tensor, - lu_pivots: &Tensor, - unpack_data: bool, - unpack_pivots: bool, - ) -> (Tensor, Tensor, Tensor) { - Tensor::f_lu_unpack(lu_data, lu_pivots, unpack_data, unpack_pivots).unwrap() - } - - pub fn lu_unpack_out( - p: &Tensor, - l: &Tensor, - u: &Tensor, - lu_data: &Tensor, - lu_pivots: &Tensor, - unpack_data: bool, - unpack_pivots: bool, - ) -> (Tensor, Tensor, Tensor) { - Tensor::f_lu_unpack_out(p, l, u, lu_data, lu_pivots, unpack_data, unpack_pivots).unwrap() - } - - pub fn margin_ranking_loss( - input1: &Tensor, - input2: &Tensor, - target: &Tensor, - margin: f64, - reduction: crate::Reduction, - ) -> Tensor { - Tensor::f_margin_ranking_loss(input1, input2, target, margin, reduction).unwrap() - } - - pub fn masked_fill>(&self, mask: &Tensor, value: S) -> Tensor { - self.f_masked_fill(mask, value).unwrap() - } - - pub fn masked_fill_>(&mut self, mask: &Tensor, value: S) -> Tensor { - self.f_masked_fill_(mask, value).unwrap() - } - - pub fn masked_fill_scalar_out>( - &self, - out: &Tensor, - mask: &Tensor, - value: S, - ) -> Tensor { - self.f_masked_fill_scalar_out(out, mask, value).unwrap() - } - - pub fn masked_fill_tensor(&self, mask: &Tensor, value: &Tensor) -> Tensor { - self.f_masked_fill_tensor(mask, value).unwrap() - } - - pub fn masked_fill_tensor_(&mut self, mask: &Tensor, value: &Tensor) -> Tensor { - self.f_masked_fill_tensor_(mask, value).unwrap() - } - - pub fn masked_fill_tensor_out(&self, out: &Tensor, mask: &Tensor, value: &Tensor) -> Tensor { - self.f_masked_fill_tensor_out(out, mask, value).unwrap() - } - - pub fn masked_scatter(&self, mask: &Tensor, source: &Tensor) -> Tensor { - self.f_masked_scatter(mask, source).unwrap() - } - - pub fn masked_scatter_(&mut self, mask: &Tensor, source: &Tensor) -> Tensor { - self.f_masked_scatter_(mask, source).unwrap() - } - - pub fn masked_scatter_backward( - grad_output: &Tensor, - mask: &Tensor, - sizes: impl IntList, - ) -> Tensor { - Tensor::f_masked_scatter_backward(grad_output, mask, sizes).unwrap() - } - - pub fn masked_scatter_out(&self, out: &Tensor, mask: &Tensor, source: &Tensor) -> Tensor { - self.f_masked_scatter_out(out, mask, source).unwrap() - } - - pub fn masked_select(&self, mask: &Tensor) -> Tensor { - self.f_masked_select(mask).unwrap() - } - - pub fn masked_select_backward(&self, grad: &Tensor, mask: &Tensor) -> Tensor { - self.f_masked_select_backward(grad, mask).unwrap() - } - - pub fn masked_select_out(&self, out: &Tensor, mask: &Tensor) -> Tensor { - self.f_masked_select_out(out, mask).unwrap() - } - - pub fn matmul(&self, other: &Tensor) -> Tensor { - self.f_matmul(other).unwrap() - } - - pub fn matmul_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_matmul_out(out, other).unwrap() - } - - pub fn matrix_exp(&self) -> Tensor { - self.f_matrix_exp().unwrap() - } - - pub fn matrix_exp_backward(&self, grad: &Tensor) -> Tensor { - self.f_matrix_exp_backward(grad).unwrap() - } - - pub fn matrix_h(&self) -> Tensor { - self.f_matrix_h().unwrap() - } - - pub fn matrix_power(&self, n: i64) -> Tensor { - self.f_matrix_power(n).unwrap() - } - - pub fn matrix_power_out(&self, out: &Tensor, n: i64) -> Tensor { - self.f_matrix_power_out(out, n).unwrap() - } - - pub fn max(&self) -> Tensor { - self.f_max().unwrap() - } - - pub fn max_dim(&self, dim: i64, keepdim: bool) -> (Tensor, Tensor) { - self.f_max_dim(dim, keepdim).unwrap() - } - - pub fn max_dim_max( - &self, - max: &Tensor, - max_values: &Tensor, - dim: i64, - keepdim: bool, - ) -> (Tensor, Tensor) { - self.f_max_dim_max(max, max_values, dim, keepdim).unwrap() - } - - pub fn max_other(&self, other: &Tensor) -> Tensor { - self.f_max_other(other).unwrap() - } - - pub fn max_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_max_out(out, other).unwrap() - } - - pub fn max_pool1d( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Tensor { - self.f_max_pool1d(kernel_size, stride, padding, dilation, ceil_mode).unwrap() - } - - pub fn max_pool1d_with_indices( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> (Tensor, Tensor) { - self.f_max_pool1d_with_indices(kernel_size, stride, padding, dilation, ceil_mode).unwrap() - } - - pub fn max_pool2d( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Tensor { - self.f_max_pool2d(kernel_size, stride, padding, dilation, ceil_mode).unwrap() - } - - pub fn max_pool2d_backward( - &self, - grad_output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Tensor { - self.f_max_pool2d_backward(grad_output, kernel_size, stride, padding, dilation, ceil_mode) - .unwrap() - } - - pub fn max_pool2d_backward_out( - &self, - out: &Tensor, - grad_output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Tensor { - self.f_max_pool2d_backward_out( - out, - grad_output, - kernel_size, - stride, - padding, - dilation, - ceil_mode, - ) - .unwrap() - } - - pub fn max_pool2d_with_indices( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> (Tensor, Tensor) { - self.f_max_pool2d_with_indices(kernel_size, stride, padding, dilation, ceil_mode).unwrap() - } - - pub fn max_pool2d_with_indices_backward( - &self, - grad_output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - indices: &Tensor, - ) -> Tensor { - self.f_max_pool2d_with_indices_backward( - grad_output, - kernel_size, - stride, - padding, - dilation, - ceil_mode, - indices, - ) - .unwrap() - } - - pub fn max_pool2d_with_indices_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - indices: &Tensor, - ) -> Tensor { - self.f_max_pool2d_with_indices_backward_grad_input( - grad_input, - grad_output, - kernel_size, - stride, - padding, - dilation, - ceil_mode, - indices, - ) - .unwrap() - } - - pub fn max_pool2d_with_indices_out( - &self, - out: &Tensor, - indices: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> (Tensor, Tensor) { - self.f_max_pool2d_with_indices_out( - out, - indices, - kernel_size, - stride, - padding, - dilation, - ceil_mode, - ) - .unwrap() - } - - pub fn max_pool3d( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Tensor { - self.f_max_pool3d(kernel_size, stride, padding, dilation, ceil_mode).unwrap() - } - - pub fn max_pool3d_with_indices( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> (Tensor, Tensor) { - self.f_max_pool3d_with_indices(kernel_size, stride, padding, dilation, ceil_mode).unwrap() - } - - pub fn max_pool3d_with_indices_backward( - &self, - grad_output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - indices: &Tensor, - ) -> Tensor { - self.f_max_pool3d_with_indices_backward( - grad_output, - kernel_size, - stride, - padding, - dilation, - ceil_mode, - indices, - ) - .unwrap() - } - - pub fn max_pool3d_with_indices_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - indices: &Tensor, - ) -> Tensor { - self.f_max_pool3d_with_indices_backward_grad_input( - grad_input, - grad_output, - kernel_size, - stride, - padding, - dilation, - ceil_mode, - indices, - ) - .unwrap() - } - - pub fn max_pool3d_with_indices_out( - &self, - out: &Tensor, - indices: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> (Tensor, Tensor) { - self.f_max_pool3d_with_indices_out( - out, - indices, - kernel_size, - stride, - padding, - dilation, - ceil_mode, - ) - .unwrap() - } - - pub fn max_unary_out(&self, out: &Tensor) -> Tensor { - self.f_max_unary_out(out).unwrap() - } - - pub fn max_unpool2d(&self, indices: &Tensor, output_size: impl IntList) -> Tensor { - self.f_max_unpool2d(indices, output_size).unwrap() - } - - pub fn max_unpool2d_out( - &self, - out: &Tensor, - indices: &Tensor, - output_size: impl IntList, - ) -> Tensor { - self.f_max_unpool2d_out(out, indices, output_size).unwrap() - } - - pub fn max_unpool3d( - &self, - indices: &Tensor, - output_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ) -> Tensor { - self.f_max_unpool3d(indices, output_size, stride, padding).unwrap() - } - - pub fn max_unpool3d_out( - &self, - out: &Tensor, - indices: &Tensor, - output_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - ) -> Tensor { - self.f_max_unpool3d_out(out, indices, output_size, stride, padding).unwrap() - } - - pub fn maximum(&self, other: &Tensor) -> Tensor { - self.f_maximum(other).unwrap() - } - - pub fn maximum_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_maximum_out(out, other).unwrap() - } - - pub fn mean(&self, dtype: impl Into>) -> Tensor { - self.f_mean(dtype).unwrap() - } - - pub fn mean_dim( - &self, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Tensor { - self.f_mean_dim(dim, keepdim, dtype).unwrap() - } - - pub fn mean_dtype_out(&self, out: &Tensor, dtype: impl Into>) -> Tensor { - self.f_mean_dtype_out(out, dtype).unwrap() - } - - pub fn mean_out( - &self, - out: &Tensor, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Tensor { - self.f_mean_out(out, dim, keepdim, dtype).unwrap() - } - - pub fn median(&self) -> Tensor { - self.f_median().unwrap() - } - - pub fn median_dim(&self, dim: i64, keepdim: bool) -> (Tensor, Tensor) { - self.f_median_dim(dim, keepdim).unwrap() - } - - pub fn median_dim_values( - &self, - values: &Tensor, - indices: &Tensor, - dim: i64, - keepdim: bool, - ) -> (Tensor, Tensor) { - self.f_median_dim_values(values, indices, dim, keepdim).unwrap() - } - - pub fn median_out(&self, out: &Tensor) -> Tensor { - self.f_median_out(out).unwrap() - } - - pub fn meshgrid>(tensors: &[T]) -> Vec { - Tensor::f_meshgrid(tensors).unwrap() - } - - pub fn meshgrid_indexing>(tensors: &[T], indexing: &str) -> Vec { - Tensor::f_meshgrid_indexing(tensors, indexing).unwrap() - } - - pub fn mh(&self) -> Tensor { - self.f_mh().unwrap() - } - - pub fn min(&self) -> Tensor { - self.f_min().unwrap() - } - - pub fn min_dim(&self, dim: i64, keepdim: bool) -> (Tensor, Tensor) { - self.f_min_dim(dim, keepdim).unwrap() - } - - pub fn min_dim_min( - &self, - min: &Tensor, - min_indices: &Tensor, - dim: i64, - keepdim: bool, - ) -> (Tensor, Tensor) { - self.f_min_dim_min(min, min_indices, dim, keepdim).unwrap() - } - - pub fn min_other(&self, other: &Tensor) -> Tensor { - self.f_min_other(other).unwrap() - } - - pub fn min_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_min_out(out, other).unwrap() - } - - pub fn min_unary_out(&self, out: &Tensor) -> Tensor { - self.f_min_unary_out(out).unwrap() - } - - pub fn minimum(&self, other: &Tensor) -> Tensor { - self.f_minimum(other).unwrap() - } - - pub fn minimum_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_minimum_out(out, other).unwrap() - } - - pub fn miopen_batch_norm>( - &self, - weight: &Tensor, - bias: Option, - running_mean: Option, - running_var: Option, - training: bool, - exponential_average_factor: f64, - epsilon: f64, - ) -> (Tensor, Tensor, Tensor) { - self.f_miopen_batch_norm( - weight, - bias, - running_mean, - running_var, - training, - exponential_average_factor, - epsilon, - ) - .unwrap() - } - - pub fn miopen_batch_norm_backward>( - &self, - grad_output: &Tensor, - weight: &Tensor, - running_mean: Option, - running_var: Option, - save_mean: Option, - save_var: Option, - epsilon: f64, - ) -> (Tensor, Tensor, Tensor) { - self.f_miopen_batch_norm_backward( - grad_output, - weight, - running_mean, - running_var, - save_mean, - save_var, - epsilon, - ) - .unwrap() - } - - pub fn miopen_batch_norm_backward_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - grad_output: &Tensor, - weight: &Tensor, - running_mean: Option, - running_var: Option, - save_mean: Option, - save_var: Option, - epsilon: f64, - ) -> (Tensor, Tensor, Tensor) { - self.f_miopen_batch_norm_backward_out( - out0, - out1, - out2, - grad_output, - weight, - running_mean, - running_var, - save_mean, - save_var, - epsilon, - ) - .unwrap() - } - - pub fn miopen_batch_norm_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - weight: &Tensor, - bias: Option, - running_mean: Option, - running_var: Option, - training: bool, - exponential_average_factor: f64, - epsilon: f64, - ) -> (Tensor, Tensor, Tensor) { - self.f_miopen_batch_norm_out( - out0, - out1, - out2, - weight, - bias, - running_mean, - running_var, - training, - exponential_average_factor, - epsilon, - ) - .unwrap() - } - - pub fn miopen_convolution>( - &self, - weight: &Tensor, - bias: Option, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - ) -> Tensor { - self.f_miopen_convolution( - weight, - bias, - padding, - stride, - dilation, - groups, - benchmark, - deterministic, - ) - .unwrap() - } - - pub fn miopen_convolution_add_relu, S: Into>( - &self, - weight: &Tensor, - z: &Tensor, - alpha: S, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Tensor { - self.f_miopen_convolution_add_relu( - weight, z, alpha, bias, stride, padding, dilation, groups, - ) - .unwrap() - } - - pub fn miopen_convolution_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - ) -> Tensor { - self.f_miopen_convolution_out( - out, - weight, - bias, - padding, - stride, - dilation, - groups, - benchmark, - deterministic, - ) - .unwrap() - } - - pub fn miopen_convolution_relu>( - &self, - weight: &Tensor, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Tensor { - self.f_miopen_convolution_relu(weight, bias, stride, padding, dilation, groups).unwrap() - } - - pub fn miopen_convolution_transpose>( - &self, - weight: &Tensor, - bias: Option, - padding: impl IntList, - output_padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - ) -> Tensor { - self.f_miopen_convolution_transpose( - weight, - bias, - padding, - output_padding, - stride, - dilation, - groups, - benchmark, - deterministic, - ) - .unwrap() - } - - pub fn miopen_convolution_transpose_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - padding: impl IntList, - output_padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - ) -> Tensor { - self.f_miopen_convolution_transpose_out( - out, - weight, - bias, - padding, - output_padding, - stride, - dilation, - groups, - benchmark, - deterministic, - ) - .unwrap() - } - - pub fn miopen_ctc_loss( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: impl IntList, - target_lengths: impl IntList, - blank: i64, - deterministic: bool, - zero_infinity: bool, - ) -> (Tensor, Tensor) { - Tensor::f_miopen_ctc_loss( - log_probs, - targets, - input_lengths, - target_lengths, - blank, - deterministic, - zero_infinity, - ) - .unwrap() - } - - pub fn miopen_ctc_loss_out( - out0: &Tensor, - out1: &Tensor, - log_probs: &Tensor, - targets: &Tensor, - input_lengths: impl IntList, - target_lengths: impl IntList, - blank: i64, - deterministic: bool, - zero_infinity: bool, - ) -> (Tensor, Tensor) { - Tensor::f_miopen_ctc_loss_out( - out0, - out1, - log_probs, - targets, - input_lengths, - target_lengths, - blank, - deterministic, - zero_infinity, - ) - .unwrap() - } - - pub fn miopen_ctc_loss_tensor( - log_probs: &Tensor, - targets: &Tensor, - input_lengths: &Tensor, - target_lengths: &Tensor, - blank: i64, - deterministic: bool, - zero_infinity: bool, - ) -> (Tensor, Tensor) { - Tensor::f_miopen_ctc_loss_tensor( - log_probs, - targets, - input_lengths, - target_lengths, - blank, - deterministic, - zero_infinity, - ) - .unwrap() - } - - pub fn miopen_depthwise_convolution>( - &self, - weight: &Tensor, - bias: Option, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - ) -> Tensor { - self.f_miopen_depthwise_convolution( - weight, - bias, - padding, - stride, - dilation, - groups, - benchmark, - deterministic, - ) - .unwrap() - } - - pub fn miopen_depthwise_convolution_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - benchmark: bool, - deterministic: bool, - ) -> Tensor { - self.f_miopen_depthwise_convolution_out( - out, - weight, - bias, - padding, - stride, - dilation, - groups, - benchmark, - deterministic, - ) - .unwrap() - } - - pub fn miopen_rnn>( - &self, - weight: &[T], - weight_stride0: i64, - hx: &Tensor, - cx: Option, - mode: i64, - hidden_size: i64, - num_layers: i64, - batch_first: bool, - dropout: f64, - train: bool, - bidirectional: bool, - batch_sizes: impl IntList, - dropout_state: Option, - ) -> (Tensor, Tensor, Tensor, Tensor, Tensor) { - self.f_miopen_rnn( - weight, - weight_stride0, - hx, - cx, - mode, - hidden_size, - num_layers, - batch_first, - dropout, - train, - bidirectional, - batch_sizes, - dropout_state, - ) - .unwrap() - } - - pub fn miopen_rnn_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - out3: &Tensor, - out4: &Tensor, - weight: &[T], - weight_stride0: i64, - hx: &Tensor, - cx: Option, - mode: i64, - hidden_size: i64, - num_layers: i64, - batch_first: bool, - dropout: f64, - train: bool, - bidirectional: bool, - batch_sizes: impl IntList, - dropout_state: Option, - ) -> (Tensor, Tensor, Tensor, Tensor, Tensor) { - self.f_miopen_rnn_out( - out0, - out1, - out2, - out3, - out4, - weight, - weight_stride0, - hx, - cx, - mode, - hidden_size, - num_layers, - batch_first, - dropout, - train, - bidirectional, - batch_sizes, - dropout_state, - ) - .unwrap() - } - - pub fn mish(&self) -> Tensor { - self.f_mish().unwrap() - } - - pub fn mish_(&mut self) -> Tensor { - self.f_mish_().unwrap() - } - - pub fn mish_backward(&self, grad_output: &Tensor) -> Tensor { - self.f_mish_backward(grad_output).unwrap() - } - - pub fn mish_out(&self, out: &Tensor) -> Tensor { - self.f_mish_out(out).unwrap() - } - - pub fn mkldnn_adaptive_avg_pool2d(&self, output_size: impl IntList) -> Tensor { - self.f_mkldnn_adaptive_avg_pool2d(output_size).unwrap() - } - - pub fn mkldnn_adaptive_avg_pool2d_backward(&self, grad_output: &Tensor) -> Tensor { - self.f_mkldnn_adaptive_avg_pool2d_backward(grad_output).unwrap() - } - - pub fn mkldnn_adaptive_avg_pool2d_backward_out( - &self, - out: &Tensor, - grad_output: &Tensor, - ) -> Tensor { - self.f_mkldnn_adaptive_avg_pool2d_backward_out(out, grad_output).unwrap() - } - - pub fn mkldnn_adaptive_avg_pool2d_out( - &self, - out: &Tensor, - output_size: impl IntList, - ) -> Tensor { - self.f_mkldnn_adaptive_avg_pool2d_out(out, output_size).unwrap() - } - - pub fn mkldnn_convolution>( - &self, - weight: &Tensor, - bias: Option, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Tensor { - self.f_mkldnn_convolution(weight, bias, padding, stride, dilation, groups).unwrap() - } - - pub fn mkldnn_convolution_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - ) -> Tensor { - self.f_mkldnn_convolution_out(out, weight, bias, padding, stride, dilation, groups).unwrap() - } - - pub fn mkldnn_linear>(&self, weight: &Tensor, bias: Option) -> Tensor { - self.f_mkldnn_linear(weight, bias).unwrap() - } - - pub fn mkldnn_linear_backward_input( - input_size: impl IntList, - grad_output: &Tensor, - weight: &Tensor, - ) -> Tensor { - Tensor::f_mkldnn_linear_backward_input(input_size, grad_output, weight).unwrap() - } - - pub fn mkldnn_linear_backward_input_out( - out: &Tensor, - input_size: impl IntList, - grad_output: &Tensor, - weight: &Tensor, - ) -> Tensor { - Tensor::f_mkldnn_linear_backward_input_out(out, input_size, grad_output, weight).unwrap() - } - - pub fn mkldnn_linear_backward_weights( - &self, - grad_output: &Tensor, - weight: &Tensor, - bias_defined: bool, - ) -> (Tensor, Tensor) { - self.f_mkldnn_linear_backward_weights(grad_output, weight, bias_defined).unwrap() - } - - pub fn mkldnn_linear_backward_weights_out( - &self, - out0: &Tensor, - out1: &Tensor, - grad_output: &Tensor, - weight: &Tensor, - bias_defined: bool, - ) -> (Tensor, Tensor) { - self.f_mkldnn_linear_backward_weights_out(out0, out1, grad_output, weight, bias_defined) - .unwrap() - } - - pub fn mkldnn_linear_out>( - &self, - out: &Tensor, - weight: &Tensor, - bias: Option, - ) -> Tensor { - self.f_mkldnn_linear_out(out, weight, bias).unwrap() - } - - pub fn mkldnn_max_pool2d( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Tensor { - self.f_mkldnn_max_pool2d(kernel_size, stride, padding, dilation, ceil_mode).unwrap() - } - - pub fn mkldnn_max_pool2d_backward( - &self, - grad_output: &Tensor, - output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Tensor { - self.f_mkldnn_max_pool2d_backward( - grad_output, - output, - kernel_size, - stride, - padding, - dilation, - ceil_mode, - ) - .unwrap() - } - - pub fn mkldnn_max_pool2d_backward_out( - &self, - out: &Tensor, - grad_output: &Tensor, - output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Tensor { - self.f_mkldnn_max_pool2d_backward_out( - out, - grad_output, - output, - kernel_size, - stride, - padding, - dilation, - ceil_mode, - ) - .unwrap() - } - - pub fn mkldnn_max_pool2d_out( - &self, - out: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Tensor { - self.f_mkldnn_max_pool2d_out(out, kernel_size, stride, padding, dilation, ceil_mode) - .unwrap() - } - - pub fn mkldnn_max_pool3d( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Tensor { - self.f_mkldnn_max_pool3d(kernel_size, stride, padding, dilation, ceil_mode).unwrap() - } - - pub fn mkldnn_max_pool3d_backward( - &self, - grad_output: &Tensor, - output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Tensor { - self.f_mkldnn_max_pool3d_backward( - grad_output, - output, - kernel_size, - stride, - padding, - dilation, - ceil_mode, - ) - .unwrap() - } - - pub fn mkldnn_max_pool3d_backward_out( - &self, - out: &Tensor, - grad_output: &Tensor, - output: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Tensor { - self.f_mkldnn_max_pool3d_backward_out( - out, - grad_output, - output, - kernel_size, - stride, - padding, - dilation, - ceil_mode, - ) - .unwrap() - } - - pub fn mkldnn_max_pool3d_out( - &self, - out: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Tensor { - self.f_mkldnn_max_pool3d_out(out, kernel_size, stride, padding, dilation, ceil_mode) - .unwrap() - } - - pub fn mkldnn_reorder_conv2d_weight( - &self, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - input_size: impl IntListOption, - ) -> Tensor { - self.f_mkldnn_reorder_conv2d_weight(padding, stride, dilation, groups, input_size).unwrap() - } - - pub fn mkldnn_reorder_conv2d_weight_out( - &self, - out: &Tensor, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - input_size: impl IntListOption, - ) -> Tensor { - self.f_mkldnn_reorder_conv2d_weight_out(out, padding, stride, dilation, groups, input_size) - .unwrap() - } - - pub fn mkldnn_reorder_conv3d_weight( - &self, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - input_size: impl IntListOption, - ) -> Tensor { - self.f_mkldnn_reorder_conv3d_weight(padding, stride, dilation, groups, input_size).unwrap() - } - - pub fn mkldnn_reorder_conv3d_weight_out( - &self, - out: &Tensor, - padding: impl IntList, - stride: impl IntList, - dilation: impl IntList, - groups: i64, - input_size: impl IntListOption, - ) -> Tensor { - self.f_mkldnn_reorder_conv3d_weight_out(out, padding, stride, dilation, groups, input_size) - .unwrap() - } - - pub fn mkldnn_rnn_layer( - &self, - weight0: &Tensor, - weight1: &Tensor, - weight2: &Tensor, - weight3: &Tensor, - hx_: &Tensor, - cx_: &Tensor, - reverse: bool, - batch_sizes: impl IntList, - mode: i64, - hidden_size: i64, - num_layers: i64, - has_biases: bool, - bidirectional: bool, - batch_first: bool, - train: bool, - ) -> (Tensor, Tensor, Tensor, Tensor) { - self.f_mkldnn_rnn_layer( - weight0, - weight1, - weight2, - weight3, - hx_, - cx_, - reverse, - batch_sizes, - mode, - hidden_size, - num_layers, - has_biases, - bidirectional, - batch_first, - train, - ) - .unwrap() - } - - pub fn mkldnn_rnn_layer_backward>( - &self, - weight1: &Tensor, - weight2: &Tensor, - weight3: &Tensor, - weight4: &Tensor, - hx_: &Tensor, - cx_tmp: &Tensor, - output: &Tensor, - hy_: &Tensor, - cy_: &Tensor, - grad_output: Option, - grad_hy: Option, - grad_cy: Option, - reverse: bool, - mode: i64, - hidden_size: i64, - num_layers: i64, - has_biases: bool, - train: bool, - bidirectional: bool, - batch_sizes: impl IntList, - batch_first: bool, - workspace: &Tensor, - ) -> (Tensor, Tensor, Tensor, Tensor, Tensor, Tensor, Tensor) { - self.f_mkldnn_rnn_layer_backward( - weight1, - weight2, - weight3, - weight4, - hx_, - cx_tmp, - output, - hy_, - cy_, - grad_output, - grad_hy, - grad_cy, - reverse, - mode, - hidden_size, - num_layers, - has_biases, - train, - bidirectional, - batch_sizes, - batch_first, - workspace, - ) - .unwrap() - } - - pub fn mkldnn_rnn_layer_backward_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - out3: &Tensor, - out4: &Tensor, - out5: &Tensor, - out6: &Tensor, - weight1: &Tensor, - weight2: &Tensor, - weight3: &Tensor, - weight4: &Tensor, - hx_: &Tensor, - cx_tmp: &Tensor, - output: &Tensor, - hy_: &Tensor, - cy_: &Tensor, - grad_output: Option, - grad_hy: Option, - grad_cy: Option, - reverse: bool, - mode: i64, - hidden_size: i64, - num_layers: i64, - has_biases: bool, - train: bool, - bidirectional: bool, - batch_sizes: impl IntList, - batch_first: bool, - workspace: &Tensor, - ) -> (Tensor, Tensor, Tensor, Tensor, Tensor, Tensor, Tensor) { - self.f_mkldnn_rnn_layer_backward_out( - out0, - out1, - out2, - out3, - out4, - out5, - out6, - weight1, - weight2, - weight3, - weight4, - hx_, - cx_tmp, - output, - hy_, - cy_, - grad_output, - grad_hy, - grad_cy, - reverse, - mode, - hidden_size, - num_layers, - has_biases, - train, - bidirectional, - batch_sizes, - batch_first, - workspace, - ) - .unwrap() - } - - pub fn mkldnn_rnn_layer_out( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - out3: &Tensor, - weight0: &Tensor, - weight1: &Tensor, - weight2: &Tensor, - weight3: &Tensor, - hx_: &Tensor, - cx_: &Tensor, - reverse: bool, - batch_sizes: impl IntList, - mode: i64, - hidden_size: i64, - num_layers: i64, - has_biases: bool, - bidirectional: bool, - batch_first: bool, - train: bool, - ) -> (Tensor, Tensor, Tensor, Tensor) { - self.f_mkldnn_rnn_layer_out( - out0, - out1, - out2, - out3, - weight0, - weight1, - weight2, - weight3, - hx_, - cx_, - reverse, - batch_sizes, - mode, - hidden_size, - num_layers, - has_biases, - bidirectional, - batch_first, - train, - ) - .unwrap() - } - - pub fn mm(&self, mat2: &Tensor) -> Tensor { - self.f_mm(mat2).unwrap() - } - - pub fn mm_dtype(&self, mat2: &Tensor, out_dtype: Kind) -> Tensor { - self.f_mm_dtype(mat2, out_dtype).unwrap() - } - - pub fn mm_dtype_out(&self, out: &Tensor, mat2: &Tensor, out_dtype: Kind) -> Tensor { - self.f_mm_dtype_out(out, mat2, out_dtype).unwrap() - } - - pub fn mm_out(&self, out: &Tensor, mat2: &Tensor) -> Tensor { - self.f_mm_out(out, mat2).unwrap() - } - - pub fn mode(&self, dim: i64, keepdim: bool) -> (Tensor, Tensor) { - self.f_mode(dim, keepdim).unwrap() - } - - pub fn mode_values( - &self, - values: &Tensor, - indices: &Tensor, - dim: i64, - keepdim: bool, - ) -> (Tensor, Tensor) { - self.f_mode_values(values, indices, dim, keepdim).unwrap() - } - - pub fn moveaxis(&self, source: impl IntList, destination: impl IntList) -> Tensor { - self.f_moveaxis(source, destination).unwrap() - } - - pub fn moveaxis_int(&self, source: i64, destination: i64) -> Tensor { - self.f_moveaxis_int(source, destination).unwrap() - } - - pub fn movedim(&self, source: impl IntList, destination: impl IntList) -> Tensor { - self.f_movedim(source, destination).unwrap() - } - - pub fn movedim_int(&self, source: i64, destination: i64) -> Tensor { - self.f_movedim_int(source, destination).unwrap() - } - - pub fn mse_loss(&self, target: &Tensor, reduction: crate::Reduction) -> Tensor { - self.f_mse_loss(target, reduction).unwrap() - } - - pub fn mse_loss_backward( - &self, - grad_output: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - ) -> Tensor { - self.f_mse_loss_backward(grad_output, target, reduction).unwrap() - } - - pub fn mse_loss_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - ) -> Tensor { - self.f_mse_loss_backward_grad_input(grad_input, grad_output, target, reduction).unwrap() - } - - pub fn mse_loss_out( - &self, - out: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - ) -> Tensor { - self.f_mse_loss_out(out, target, reduction).unwrap() - } - - pub fn msort(&self) -> Tensor { - self.f_msort().unwrap() - } - - pub fn msort_out(&self, out: &Tensor) -> Tensor { - self.f_msort_out(out).unwrap() - } - - pub fn mt(&self) -> Tensor { - self.f_mt().unwrap() - } - - pub fn g_mul(&self, other: &Tensor) -> Tensor { - self.f_mul(other).unwrap() - } - - pub fn g_mul_(&mut self, other: &Tensor) -> Tensor { - self.f_mul_(other).unwrap() - } - - pub fn mul_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_mul_out(out, other).unwrap() - } - - pub fn g_mul_scalar>(&self, other: S) -> Tensor { - self.f_mul_scalar(other).unwrap() - } - - pub fn g_mul_scalar_>(&mut self, other: S) -> Tensor { - self.f_mul_scalar_(other).unwrap() - } - - pub fn mul_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_mul_scalar_out(out, other).unwrap() - } - - pub fn multi_margin_loss_backward, S: Into>( - &self, - grad_output: &Tensor, - target: &Tensor, - p: S, - margin: S, - weight: Option, - reduction: crate::Reduction, - ) -> Tensor { - self.f_multi_margin_loss_backward(grad_output, target, p, margin, weight, reduction) - .unwrap() - } - - pub fn multi_margin_loss_backward_grad_input, S: Into>( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - target: &Tensor, - p: S, - margin: S, - weight: Option, - reduction: crate::Reduction, - ) -> Tensor { - self.f_multi_margin_loss_backward_grad_input( - grad_input, - grad_output, - target, - p, - margin, - weight, - reduction, - ) - .unwrap() - } - - pub fn multilabel_margin_loss(&self, target: &Tensor, reduction: crate::Reduction) -> Tensor { - self.f_multilabel_margin_loss(target, reduction).unwrap() - } - - pub fn multilabel_margin_loss_backward( - &self, - grad_output: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - is_target: &Tensor, - ) -> Tensor { - self.f_multilabel_margin_loss_backward(grad_output, target, reduction, is_target).unwrap() - } - - pub fn multilabel_margin_loss_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - is_target: &Tensor, - ) -> Tensor { - self.f_multilabel_margin_loss_backward_grad_input( - grad_input, - grad_output, - target, - reduction, - is_target, - ) - .unwrap() - } - - pub fn multilabel_margin_loss_out( - &self, - out: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - ) -> Tensor { - self.f_multilabel_margin_loss_out(out, target, reduction).unwrap() - } - - pub fn multinomial(&self, num_samples: i64, replacement: bool) -> Tensor { - self.f_multinomial(num_samples, replacement).unwrap() - } - - pub fn multinomial_out(&self, out: &Tensor, num_samples: i64, replacement: bool) -> Tensor { - self.f_multinomial_out(out, num_samples, replacement).unwrap() - } - - pub fn multiply(&self, other: &Tensor) -> Tensor { - self.f_multiply(other).unwrap() - } - - pub fn multiply_(&mut self, other: &Tensor) -> Tensor { - self.f_multiply_(other).unwrap() - } - - pub fn multiply_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_multiply_out(out, other).unwrap() - } - - pub fn multiply_scalar>(&self, other: S) -> Tensor { - self.f_multiply_scalar(other).unwrap() - } - - pub fn multiply_scalar_>(&mut self, other: S) -> Tensor { - self.f_multiply_scalar_(other).unwrap() - } - - pub fn mv(&self, vec: &Tensor) -> Tensor { - self.f_mv(vec).unwrap() - } - - pub fn mv_out(&self, out: &Tensor, vec: &Tensor) -> Tensor { - self.f_mv_out(out, vec).unwrap() - } - - pub fn mvlgamma(&self, p: i64) -> Tensor { - self.f_mvlgamma(p).unwrap() - } - - pub fn mvlgamma_(&mut self, p: i64) -> Tensor { - self.f_mvlgamma_(p).unwrap() - } - - pub fn mvlgamma_out(&self, out: &Tensor, p: i64) -> Tensor { - self.f_mvlgamma_out(out, p).unwrap() - } - - pub fn nan_to_num( - &self, - nan: impl Into>, - posinf: impl Into>, - neginf: impl Into>, - ) -> Tensor { - self.f_nan_to_num(nan, posinf, neginf).unwrap() - } - - pub fn nan_to_num_( - &mut self, - nan: impl Into>, - posinf: impl Into>, - neginf: impl Into>, - ) -> Tensor { - self.f_nan_to_num_(nan, posinf, neginf).unwrap() - } - - pub fn nan_to_num_out( - &self, - out: &Tensor, - nan: impl Into>, - posinf: impl Into>, - neginf: impl Into>, - ) -> Tensor { - self.f_nan_to_num_out(out, nan, posinf, neginf).unwrap() - } - - pub fn nanmean( - &self, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Tensor { - self.f_nanmean(dim, keepdim, dtype).unwrap() - } - - pub fn nanmean_out( - &self, - out: &Tensor, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Tensor { - self.f_nanmean_out(out, dim, keepdim, dtype).unwrap() - } - - pub fn nanmedian(&self) -> Tensor { - self.f_nanmedian().unwrap() - } - - pub fn nanmedian_dim(&self, dim: i64, keepdim: bool) -> (Tensor, Tensor) { - self.f_nanmedian_dim(dim, keepdim).unwrap() - } - - pub fn nanmedian_dim_values( - &self, - values: &Tensor, - indices: &Tensor, - dim: i64, - keepdim: bool, - ) -> (Tensor, Tensor) { - self.f_nanmedian_dim_values(values, indices, dim, keepdim).unwrap() - } - - pub fn nanmedian_out(&self, out: &Tensor) -> Tensor { - self.f_nanmedian_out(out).unwrap() - } - - pub fn nanquantile( - &self, - q: &Tensor, - dim: impl Into>, - keepdim: bool, - interpolation: &str, - ) -> Tensor { - self.f_nanquantile(q, dim, keepdim, interpolation).unwrap() - } - - pub fn nanquantile_out( - &self, - out: &Tensor, - q: &Tensor, - dim: impl Into>, - keepdim: bool, - interpolation: &str, - ) -> Tensor { - self.f_nanquantile_out(out, q, dim, keepdim, interpolation).unwrap() - } - - pub fn nanquantile_scalar( - &self, - q: f64, - dim: impl Into>, - keepdim: bool, - interpolation: &str, - ) -> Tensor { - self.f_nanquantile_scalar(q, dim, keepdim, interpolation).unwrap() - } - - pub fn nanquantile_scalar_out( - &self, - out: &Tensor, - q: f64, - dim: impl Into>, - keepdim: bool, - interpolation: &str, - ) -> Tensor { - self.f_nanquantile_scalar_out(out, q, dim, keepdim, interpolation).unwrap() - } - - pub fn nansum( - &self, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Tensor { - self.f_nansum(dim, keepdim, dtype).unwrap() - } - - pub fn nansum_out( - &self, - out: &Tensor, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Tensor { - self.f_nansum_out(out, dim, keepdim, dtype).unwrap() - } - - pub fn narrow(&self, dim: i64, start: i64, length: i64) -> Tensor { - self.f_narrow(dim, start, length).unwrap() - } - - pub fn narrow_copy(&self, dim: i64, start: i64, length: i64) -> Tensor { - self.f_narrow_copy(dim, start, length).unwrap() - } - - pub fn narrow_copy_out(&self, out: &Tensor, dim: i64, start: i64, length: i64) -> Tensor { - self.f_narrow_copy_out(out, dim, start, length).unwrap() - } - - pub fn narrow_tensor(&self, dim: i64, start: &Tensor, length: i64) -> Tensor { - self.f_narrow_tensor(dim, start, length).unwrap() - } - - pub fn native_batch_norm>( - &self, - weight: Option, - bias: Option, - running_mean: Option, - running_var: Option, - training: bool, - momentum: f64, - eps: f64, - ) -> (Tensor, Tensor, Tensor) { - self.f_native_batch_norm(weight, bias, running_mean, running_var, training, momentum, eps) - .unwrap() - } - - pub fn native_batch_norm_out>( - &self, - out: &Tensor, - save_mean: &Tensor, - save_invstd: &Tensor, - weight: Option, - bias: Option, - running_mean: Option, - running_var: Option, - training: bool, - momentum: f64, - eps: f64, - ) -> (Tensor, Tensor, Tensor) { - self.f_native_batch_norm_out( - out, - save_mean, - save_invstd, - weight, - bias, - running_mean, - running_var, - training, - momentum, - eps, - ) - .unwrap() - } - - pub fn native_channel_shuffle(&self, groups: i64) -> Tensor { - self.f_native_channel_shuffle(groups).unwrap() - } - - pub fn native_dropout(&self, p: f64, train: bool) -> (Tensor, Tensor) { - self.f_native_dropout(p, train).unwrap() - } - - pub fn native_dropout_backward(grad_output: &Tensor, mask: &Tensor, scale: f64) -> Tensor { - Tensor::f_native_dropout_backward(grad_output, mask, scale).unwrap() - } - - pub fn native_dropout_backward_out( - out: &Tensor, - grad_output: &Tensor, - mask: &Tensor, - scale: f64, - ) -> Tensor { - Tensor::f_native_dropout_backward_out(out, grad_output, mask, scale).unwrap() - } - - pub fn native_dropout_out( - &self, - out0: &Tensor, - out1: &Tensor, - p: f64, - train: bool, - ) -> (Tensor, Tensor) { - self.f_native_dropout_out(out0, out1, p, train).unwrap() - } - - pub fn native_group_norm>( - &self, - weight: Option, - bias: Option, - n: i64, - c: i64, - hxw: i64, - group: i64, - eps: f64, - ) -> (Tensor, Tensor, Tensor) { - self.f_native_group_norm(weight, bias, n, c, hxw, group, eps).unwrap() - } - - pub fn native_group_norm_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - weight: Option, - bias: Option, - n: i64, - c: i64, - hxw: i64, - group: i64, - eps: f64, - ) -> (Tensor, Tensor, Tensor) { - self.f_native_group_norm_out(out0, out1, out2, weight, bias, n, c, hxw, group, eps).unwrap() - } - - pub fn native_layer_norm>( - &self, - normalized_shape: impl IntList, - weight: Option, - bias: Option, - eps: f64, - ) -> (Tensor, Tensor, Tensor) { - self.f_native_layer_norm(normalized_shape, weight, bias, eps).unwrap() - } - - pub fn native_layer_norm_out>( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - normalized_shape: impl IntList, - weight: Option, - bias: Option, - eps: f64, - ) -> (Tensor, Tensor, Tensor) { - self.f_native_layer_norm_out(out0, out1, out2, normalized_shape, weight, bias, eps).unwrap() - } - - pub fn native_norm(&self) -> Tensor { - self.f_native_norm().unwrap() - } - - pub fn native_norm_out(&self, out: &Tensor) -> Tensor { - self.f_native_norm_out(out).unwrap() - } - - pub fn native_norm_scalaropt_dim_dtype>( - &self, - p: S, - dim: impl IntList, - keepdim: bool, - dtype: impl Into>, - ) -> Tensor { - self.f_native_norm_scalaropt_dim_dtype(p, dim, keepdim, dtype).unwrap() - } - - pub fn native_norm_scalaropt_dim_dtype_out>( - &self, - out: &Tensor, - p: S, - dim: impl IntList, - keepdim: bool, - dtype: impl Into>, - ) -> Tensor { - self.f_native_norm_scalaropt_dim_dtype_out(out, p, dim, keepdim, dtype).unwrap() - } - - pub fn ne>(&self, other: S) -> Tensor { - self.f_ne(other).unwrap() - } - - pub fn ne_>(&mut self, other: S) -> Tensor { - self.f_ne_(other).unwrap() - } - - pub fn ne_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_ne_scalar_out(out, other).unwrap() - } - - pub fn ne_tensor(&self, other: &Tensor) -> Tensor { - self.f_ne_tensor(other).unwrap() - } - - pub fn ne_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_ne_tensor_(other).unwrap() - } - - pub fn ne_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_ne_tensor_out(out, other).unwrap() - } - - pub fn neg(&self) -> Tensor { - self.f_neg().unwrap() - } - - pub fn neg_(&mut self) -> Tensor { - self.f_neg_().unwrap() - } - - pub fn neg_out(&self, out: &Tensor) -> Tensor { - self.f_neg_out(out).unwrap() - } - - pub fn negative(&self) -> Tensor { - self.f_negative().unwrap() - } - - pub fn negative_(&mut self) -> Tensor { - self.f_negative_().unwrap() - } - - pub fn negative_out(&self, out: &Tensor) -> Tensor { - self.f_negative_out(out).unwrap() - } - - pub fn nested_to_padded_tensor(&self, padding: f64, output_size: impl IntListOption) -> Tensor { - self.f_nested_to_padded_tensor(padding, output_size).unwrap() - } - - pub fn new_empty(&self, size: impl IntList, options: (Kind, Device)) -> Tensor { - self.f_new_empty(size, options).unwrap() - } - - pub fn new_empty_out(&self, out: &Tensor, size: impl IntList) -> Tensor { - self.f_new_empty_out(out, size).unwrap() - } - - pub fn new_empty_strided( - &self, - size: impl IntList, - stride: impl IntList, - options: (Kind, Device), - ) -> Tensor { - self.f_new_empty_strided(size, stride, options).unwrap() - } - - pub fn new_empty_strided_out( - &self, - out: &Tensor, - size: impl IntList, - stride: impl IntList, - ) -> Tensor { - self.f_new_empty_strided_out(out, size, stride).unwrap() - } - - pub fn new_full>( - &self, - size: impl IntList, - fill_value: S, - options: (Kind, Device), - ) -> Tensor { - self.f_new_full(size, fill_value, options).unwrap() - } - - pub fn new_full_out>( - &self, - out: &Tensor, - size: impl IntList, - fill_value: S, - ) -> Tensor { - self.f_new_full_out(out, size, fill_value).unwrap() - } - - pub fn new_ones(&self, size: impl IntList, options: (Kind, Device)) -> Tensor { - self.f_new_ones(size, options).unwrap() - } - - pub fn new_ones_out(&self, out: &Tensor, size: impl IntList) -> Tensor { - self.f_new_ones_out(out, size).unwrap() - } - - pub fn new_zeros(&self, size: impl IntList, options: (Kind, Device)) -> Tensor { - self.f_new_zeros(size, options).unwrap() - } - - pub fn new_zeros_out(&self, out: &Tensor, size: impl IntList) -> Tensor { - self.f_new_zeros_out(out, size).unwrap() - } - - pub fn nextafter(&self, other: &Tensor) -> Tensor { - self.f_nextafter(other).unwrap() - } - - pub fn nextafter_(&mut self, other: &Tensor) -> Tensor { - self.f_nextafter_(other).unwrap() - } - - pub fn nextafter_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_nextafter_out(out, other).unwrap() - } - - pub fn g_nll_loss>( - &self, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ignore_index: i64, - ) -> Tensor { - self.f_nll_loss(target, weight, reduction, ignore_index).unwrap() - } - - pub fn nll_loss2d>( - &self, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ignore_index: i64, - ) -> Tensor { - self.f_nll_loss2d(target, weight, reduction, ignore_index).unwrap() - } - - pub fn nll_loss2d_backward>( - &self, - grad_output: &Tensor, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ignore_index: i64, - total_weight: &Tensor, - ) -> Tensor { - self.f_nll_loss2d_backward( - grad_output, - target, - weight, - reduction, - ignore_index, - total_weight, - ) - .unwrap() - } - - pub fn nll_loss2d_backward_grad_input>( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ignore_index: i64, - total_weight: &Tensor, - ) -> Tensor { - self.f_nll_loss2d_backward_grad_input( - grad_input, - grad_output, - target, - weight, - reduction, - ignore_index, - total_weight, - ) - .unwrap() - } - - pub fn nll_loss2d_out>( - &self, - out: &Tensor, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ignore_index: i64, - ) -> Tensor { - self.f_nll_loss2d_out(out, target, weight, reduction, ignore_index).unwrap() - } - - pub fn nll_loss_backward>( - &self, - grad_output: &Tensor, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ignore_index: i64, - total_weight: &Tensor, - ) -> Tensor { - self.f_nll_loss_backward(grad_output, target, weight, reduction, ignore_index, total_weight) - .unwrap() - } - - pub fn nll_loss_backward_grad_input>( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ignore_index: i64, - total_weight: &Tensor, - ) -> Tensor { - self.f_nll_loss_backward_grad_input( - grad_input, - grad_output, - target, - weight, - reduction, - ignore_index, - total_weight, - ) - .unwrap() - } - - pub fn nll_loss_nd>( - &self, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ignore_index: i64, - ) -> Tensor { - self.f_nll_loss_nd(target, weight, reduction, ignore_index).unwrap() - } - - pub fn nll_loss_out>( - &self, - out: &Tensor, - target: &Tensor, - weight: Option, - reduction: crate::Reduction, - ignore_index: i64, - ) -> Tensor { - self.f_nll_loss_out(out, target, weight, reduction, ignore_index).unwrap() - } - - pub fn nonzero(&self) -> Tensor { - self.f_nonzero().unwrap() - } - - pub fn nonzero_numpy(&self) -> Vec { - self.f_nonzero_numpy().unwrap() - } - - pub fn nonzero_out(&self, out: &Tensor) -> Tensor { - self.f_nonzero_out(out).unwrap() - } - - pub fn nonzero_static(&self, size: i64, fill_value: i64) -> Tensor { - self.f_nonzero_static(size, fill_value).unwrap() - } - - pub fn nonzero_static_out(&self, out: &Tensor, size: i64, fill_value: i64) -> Tensor { - self.f_nonzero_static_out(out, size, fill_value).unwrap() - } - - pub fn norm(&self) -> Tensor { - self.f_norm().unwrap() - } - - pub fn norm_dtype_out>( - &self, - out: &Tensor, - p: S, - dim: impl IntList, - keepdim: bool, - dtype: Kind, - ) -> Tensor { - self.f_norm_dtype_out(out, p, dim, keepdim, dtype).unwrap() - } - - pub fn norm_except_dim(v: &Tensor, pow: i64, dim: i64) -> Tensor { - Tensor::f_norm_except_dim(v, pow, dim).unwrap() - } - - pub fn norm_out>( - &self, - out: &Tensor, - p: S, - dim: impl IntList, - keepdim: bool, - ) -> Tensor { - self.f_norm_out(out, p, dim, keepdim).unwrap() - } - - pub fn norm_scalar_out(&self, out: &Tensor) -> Tensor { - self.f_norm_scalar_out(out).unwrap() - } - - pub fn norm_scalaropt_dim>( - &self, - p: S, - dim: impl IntList, - keepdim: bool, - ) -> Tensor { - self.f_norm_scalaropt_dim(p, dim, keepdim).unwrap() - } - - pub fn norm_scalaropt_dim_dtype>( - &self, - p: S, - dim: impl IntList, - keepdim: bool, - dtype: Kind, - ) -> Tensor { - self.f_norm_scalaropt_dim_dtype(p, dim, keepdim, dtype).unwrap() - } - - pub fn norm_scalaropt_dtype>(&self, p: S, dtype: Kind) -> Tensor { - self.f_norm_scalaropt_dtype(p, dtype).unwrap() - } - - pub fn norm_scalaropt_dtype_out>( - &self, - out: &Tensor, - p: S, - dtype: Kind, - ) -> Tensor { - self.f_norm_scalaropt_dtype_out(out, p, dtype).unwrap() - } - - pub fn normal_(&mut self, mean: f64, std: f64) -> Tensor { - self.f_normal_(mean, std).unwrap() - } - - pub fn normal_functional(&self, mean: f64, std: f64) -> Tensor { - self.f_normal_functional(mean, std).unwrap() - } - - pub fn not_equal>(&self, other: S) -> Tensor { - self.f_not_equal(other).unwrap() - } - - pub fn not_equal_>(&mut self, other: S) -> Tensor { - self.f_not_equal_(other).unwrap() - } - - pub fn not_equal_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_not_equal_scalar_out(out, other).unwrap() - } - - pub fn not_equal_tensor(&self, other: &Tensor) -> Tensor { - self.f_not_equal_tensor(other).unwrap() - } - - pub fn not_equal_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_not_equal_tensor_(other).unwrap() - } - - pub fn not_equal_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_not_equal_tensor_out(out, other).unwrap() - } - - pub fn nuclear_norm(&self, keepdim: bool) -> Tensor { - self.f_nuclear_norm(keepdim).unwrap() - } - - pub fn nuclear_norm_dim(&self, dim: impl IntList, keepdim: bool) -> Tensor { - self.f_nuclear_norm_dim(dim, keepdim).unwrap() - } - - pub fn nuclear_norm_dim_out(&self, out: &Tensor, dim: impl IntList, keepdim: bool) -> Tensor { - self.f_nuclear_norm_dim_out(out, dim, keepdim).unwrap() - } - - pub fn nuclear_norm_out(&self, out: &Tensor, keepdim: bool) -> Tensor { - self.f_nuclear_norm_out(out, keepdim).unwrap() - } - - pub fn numpy_t(&self) -> Tensor { - self.f_numpy_t().unwrap() - } - - pub fn one_hot(&self, num_classes: i64) -> Tensor { - self.f_one_hot(num_classes).unwrap() - } - - pub fn ones(size: impl IntList, options: (Kind, Device)) -> Tensor { - Tensor::f_ones(size, options).unwrap() - } - - pub fn ones_like(&self) -> Tensor { - self.f_ones_like().unwrap() - } - - pub fn ones_like_out(&self, out: &Tensor) -> Tensor { - self.f_ones_like_out(out).unwrap() - } - - pub fn ones_out(out: &Tensor, size: impl IntList) -> Tensor { - Tensor::f_ones_out(out, size).unwrap() - } - - pub fn orgqr(&self, input2: &Tensor) -> Tensor { - self.f_orgqr(input2).unwrap() - } - - pub fn orgqr_out(&self, out: &Tensor, input2: &Tensor) -> Tensor { - self.f_orgqr_out(out, input2).unwrap() - } - - pub fn ormqr(&self, input2: &Tensor, input3: &Tensor, left: bool, transpose: bool) -> Tensor { - self.f_ormqr(input2, input3, left, transpose).unwrap() - } - - pub fn ormqr_out( - &self, - out: &Tensor, - input2: &Tensor, - input3: &Tensor, - left: bool, - transpose: bool, - ) -> Tensor { - self.f_ormqr_out(out, input2, input3, left, transpose).unwrap() - } - - pub fn outer(&self, vec2: &Tensor) -> Tensor { - self.f_outer(vec2).unwrap() - } - - pub fn outer_out(&self, out: &Tensor, vec2: &Tensor) -> Tensor { - self.f_outer_out(out, vec2).unwrap() - } - - pub fn output_nr(&self) -> i64 { - self.f_output_nr().unwrap() - } - - pub fn pad(&self, pad: impl IntList, mode: &str, value: impl Into>) -> Tensor { - self.f_pad(pad, mode, value).unwrap() - } - - pub fn pad_sequence>( - sequences: &[T], - batch_first: bool, - padding_value: f64, - padding_side: &str, - ) -> Tensor { - Tensor::f_pad_sequence(sequences, batch_first, padding_value, padding_side).unwrap() - } - - pub fn pairwise_distance(x1: &Tensor, x2: &Tensor, p: f64, eps: f64, keepdim: bool) -> Tensor { - Tensor::f_pairwise_distance(x1, x2, p, eps, keepdim).unwrap() - } - - pub fn pdist(&self, p: f64) -> Tensor { - self.f_pdist(p).unwrap() - } - - pub fn permute(&self, dims: impl IntList) -> Tensor { - self.f_permute(dims).unwrap() - } - - pub fn permute_copy(&self, dims: impl IntList) -> Tensor { - self.f_permute_copy(dims).unwrap() - } - - pub fn permute_copy_out(&self, out: &Tensor, dims: impl IntList) -> Tensor { - self.f_permute_copy_out(out, dims).unwrap() - } - - pub fn pin_memory(&self, device: Device) -> Tensor { - self.f_pin_memory(device).unwrap() - } - - pub fn pinverse(&self, rcond: f64) -> Tensor { - self.f_pinverse(rcond).unwrap() - } - - pub fn pixel_shuffle(&self, upscale_factor: i64) -> Tensor { - self.f_pixel_shuffle(upscale_factor).unwrap() - } - - pub fn pixel_shuffle_out(&self, out: &Tensor, upscale_factor: i64) -> Tensor { - self.f_pixel_shuffle_out(out, upscale_factor).unwrap() - } - - pub fn pixel_unshuffle(&self, downscale_factor: i64) -> Tensor { - self.f_pixel_unshuffle(downscale_factor).unwrap() - } - - pub fn pixel_unshuffle_out(&self, out: &Tensor, downscale_factor: i64) -> Tensor { - self.f_pixel_unshuffle_out(out, downscale_factor).unwrap() - } - - pub fn poisson(&self) -> Tensor { - self.f_poisson().unwrap() - } - - pub fn poisson_nll_loss( - &self, - target: &Tensor, - log_input: bool, - full: bool, - eps: f64, - reduction: crate::Reduction, - ) -> Tensor { - self.f_poisson_nll_loss(target, log_input, full, eps, reduction).unwrap() - } - - pub fn poisson_out(&self, out: &Tensor) -> Tensor { - self.f_poisson_out(out).unwrap() - } - - pub fn polar(abs: &Tensor, angle: &Tensor) -> Tensor { - Tensor::f_polar(abs, angle).unwrap() - } - - pub fn polar_out(out: &Tensor, abs: &Tensor, angle: &Tensor) -> Tensor { - Tensor::f_polar_out(out, abs, angle).unwrap() - } - - pub fn polygamma(&self, n: i64) -> Tensor { - self.f_polygamma(n).unwrap() - } - - pub fn polygamma_(&mut self, n: i64) -> Tensor { - self.f_polygamma_(n).unwrap() - } - - pub fn polygamma_out(&self, out: &Tensor, n: i64) -> Tensor { - self.f_polygamma_out(out, n).unwrap() - } - - pub fn positive(&self) -> Tensor { - self.f_positive().unwrap() - } - - pub fn pow(&self, exponent: &Tensor) -> Tensor { - self.f_pow(exponent).unwrap() - } - - pub fn pow_>(&mut self, exponent: S) -> Tensor { - self.f_pow_(exponent).unwrap() - } - - pub fn pow_scalar>(self_scalar: S, exponent: &Tensor) -> Tensor { - Tensor::f_pow_scalar(self_scalar, exponent).unwrap() - } - - pub fn pow_scalar_out>( - out: &Tensor, - self_scalar: S, - exponent: &Tensor, - ) -> Tensor { - Tensor::f_pow_scalar_out(out, self_scalar, exponent).unwrap() - } - - pub fn pow_tensor_(&mut self, exponent: &Tensor) -> Tensor { - self.f_pow_tensor_(exponent).unwrap() - } - - pub fn pow_tensor_scalar>(&self, exponent: S) -> Tensor { - self.f_pow_tensor_scalar(exponent).unwrap() - } - - pub fn pow_tensor_scalar_out>(&self, out: &Tensor, exponent: S) -> Tensor { - self.f_pow_tensor_scalar_out(out, exponent).unwrap() - } - - pub fn pow_tensor_tensor_out(&self, out: &Tensor, exponent: &Tensor) -> Tensor { - self.f_pow_tensor_tensor_out(out, exponent).unwrap() - } - - pub fn prelu(&self, weight: &Tensor) -> Tensor { - self.f_prelu(weight).unwrap() - } - - pub fn prod(&self, dtype: impl Into>) -> Tensor { - self.f_prod(dtype).unwrap() - } - - pub fn prod_dim_int(&self, dim: i64, keepdim: bool, dtype: impl Into>) -> Tensor { - self.f_prod_dim_int(dim, keepdim, dtype).unwrap() - } - - pub fn prod_int_out( - &self, - out: &Tensor, - dim: i64, - keepdim: bool, - dtype: impl Into>, - ) -> Tensor { - self.f_prod_int_out(out, dim, keepdim, dtype).unwrap() - } - - pub fn prod_out(&self, out: &Tensor, dtype: impl Into>) -> Tensor { - self.f_prod_out(out, dtype).unwrap() - } - - pub fn put(&self, index: &Tensor, source: &Tensor, accumulate: bool) -> Tensor { - self.f_put(index, source, accumulate).unwrap() - } - - pub fn put_(&mut self, index: &Tensor, source: &Tensor, accumulate: bool) -> Tensor { - self.f_put_(index, source, accumulate).unwrap() - } - - pub fn put_out( - &self, - out: &Tensor, - index: &Tensor, - source: &Tensor, - accumulate: bool, - ) -> Tensor { - self.f_put_out(out, index, source, accumulate).unwrap() - } - - pub fn q_per_channel_axis(&self) -> i64 { - self.f_q_per_channel_axis().unwrap() - } - - pub fn q_per_channel_scales(&self) -> Tensor { - self.f_q_per_channel_scales().unwrap() - } - - pub fn q_per_channel_scales_out(&self, out: &Tensor) -> Tensor { - self.f_q_per_channel_scales_out(out).unwrap() - } - - pub fn q_per_channel_zero_points(&self) -> Tensor { - self.f_q_per_channel_zero_points().unwrap() - } - - pub fn q_per_channel_zero_points_out(&self, out: &Tensor) -> Tensor { - self.f_q_per_channel_zero_points_out(out).unwrap() - } - - pub fn q_scale(&self) -> f64 { - self.f_q_scale().unwrap() - } - - pub fn q_zero_point(&self) -> i64 { - self.f_q_zero_point().unwrap() - } - - pub fn qr(&self, some: bool) -> (Tensor, Tensor) { - self.f_qr(some).unwrap() - } - - pub fn qr_q(&self, q: &Tensor, r: &Tensor, some: bool) -> (Tensor, Tensor) { - self.f_qr_q(q, r, some).unwrap() - } - - pub fn quantile( - &self, - q: &Tensor, - dim: impl Into>, - keepdim: bool, - interpolation: &str, - ) -> Tensor { - self.f_quantile(q, dim, keepdim, interpolation).unwrap() - } - - pub fn quantile_out( - &self, - out: &Tensor, - q: &Tensor, - dim: impl Into>, - keepdim: bool, - interpolation: &str, - ) -> Tensor { - self.f_quantile_out(out, q, dim, keepdim, interpolation).unwrap() - } - - pub fn quantile_scalar( - &self, - q: f64, - dim: impl Into>, - keepdim: bool, - interpolation: &str, - ) -> Tensor { - self.f_quantile_scalar(q, dim, keepdim, interpolation).unwrap() - } - - pub fn quantile_scalar_out( - &self, - out: &Tensor, - q: f64, - dim: impl Into>, - keepdim: bool, - interpolation: &str, - ) -> Tensor { - self.f_quantile_scalar_out(out, q, dim, keepdim, interpolation).unwrap() - } - - pub fn quantize_per_channel( - &self, - scales: &Tensor, - zero_points: &Tensor, - axis: i64, - dtype: Kind, - ) -> Tensor { - self.f_quantize_per_channel(scales, zero_points, axis, dtype).unwrap() - } - - pub fn quantize_per_channel_out( - &self, - out: &Tensor, - scales: &Tensor, - zero_points: &Tensor, - axis: i64, - dtype: Kind, - ) -> Tensor { - self.f_quantize_per_channel_out(out, scales, zero_points, axis, dtype).unwrap() - } - - pub fn quantize_per_tensor(&self, scale: f64, zero_point: i64, dtype: Kind) -> Tensor { - self.f_quantize_per_tensor(scale, zero_point, dtype).unwrap() - } - - pub fn quantize_per_tensor_dynamic(&self, dtype: Kind, reduce_range: bool) -> Tensor { - self.f_quantize_per_tensor_dynamic(dtype, reduce_range).unwrap() - } - - pub fn quantize_per_tensor_dynamic_out( - &self, - out: &Tensor, - dtype: Kind, - reduce_range: bool, - ) -> Tensor { - self.f_quantize_per_tensor_dynamic_out(out, dtype, reduce_range).unwrap() - } - - pub fn quantize_per_tensor_out( - &self, - out: &Tensor, - scale: f64, - zero_point: i64, - dtype: Kind, - ) -> Tensor { - self.f_quantize_per_tensor_out(out, scale, zero_point, dtype).unwrap() - } - - pub fn quantize_per_tensor_tensor_qparams( - &self, - scale: &Tensor, - zero_point: &Tensor, - dtype: Kind, - ) -> Tensor { - self.f_quantize_per_tensor_tensor_qparams(scale, zero_point, dtype).unwrap() - } - - pub fn quantize_per_tensor_tensor_qparams_out( - &self, - out: &Tensor, - scale: &Tensor, - zero_point: &Tensor, - dtype: Kind, - ) -> Tensor { - self.f_quantize_per_tensor_tensor_qparams_out(out, scale, zero_point, dtype).unwrap() - } - - pub fn quantize_per_tensor_tensors>( - tensors: &[T], - scales: &Tensor, - zero_points: &Tensor, - dtype: Kind, - ) -> Vec { - Tensor::f_quantize_per_tensor_tensors(tensors, scales, zero_points, dtype).unwrap() - } - - pub fn quantize_per_tensor_tensors_out>( - out: &[T], - tensors: &[T], - scales: &Tensor, - zero_points: &Tensor, - dtype: Kind, - ) { - Tensor::f_quantize_per_tensor_tensors_out(out, tensors, scales, zero_points, dtype).unwrap() - } - - pub fn quantized_batch_norm>( - &self, - weight: Option, - bias: Option, - mean: &Tensor, - var: &Tensor, - eps: f64, - output_scale: f64, - output_zero_point: i64, - ) -> Tensor { - self.f_quantized_batch_norm(weight, bias, mean, var, eps, output_scale, output_zero_point) - .unwrap() - } - - pub fn quantized_batch_norm_out>( - &self, - out: &Tensor, - weight: Option, - bias: Option, - mean: &Tensor, - var: &Tensor, - eps: f64, - output_scale: f64, - output_zero_point: i64, - ) -> Tensor { - self.f_quantized_batch_norm_out( - out, - weight, - bias, - mean, - var, - eps, - output_scale, - output_zero_point, - ) - .unwrap() - } - - pub fn quantized_gru_cell>( - &self, - hx: &Tensor, - w_ih: &Tensor, - w_hh: &Tensor, - b_ih: &Tensor, - b_hh: &Tensor, - packed_ih: &Tensor, - packed_hh: &Tensor, - col_offsets_ih: &Tensor, - col_offsets_hh: &Tensor, - scale_ih: S, - scale_hh: S, - zero_point_ih: S, - zero_point_hh: S, - ) -> Tensor { - self.f_quantized_gru_cell( - hx, - w_ih, - w_hh, - b_ih, - b_hh, - packed_ih, - packed_hh, - col_offsets_ih, - col_offsets_hh, - scale_ih, - scale_hh, - zero_point_ih, - zero_point_hh, - ) - .unwrap() - } - - pub fn quantized_lstm_cell, S: Into>( - &self, - hx: &[T], - w_ih: &Tensor, - w_hh: &Tensor, - b_ih: &Tensor, - b_hh: &Tensor, - packed_ih: &Tensor, - packed_hh: &Tensor, - col_offsets_ih: &Tensor, - col_offsets_hh: &Tensor, - scale_ih: S, - scale_hh: S, - zero_point_ih: S, - zero_point_hh: S, - ) -> (Tensor, Tensor) { - self.f_quantized_lstm_cell( - hx, - w_ih, - w_hh, - b_ih, - b_hh, - packed_ih, - packed_hh, - col_offsets_ih, - col_offsets_hh, - scale_ih, - scale_hh, - zero_point_ih, - zero_point_hh, - ) - .unwrap() - } - - pub fn quantized_max_pool1d( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Tensor { - self.f_quantized_max_pool1d(kernel_size, stride, padding, dilation, ceil_mode).unwrap() - } - - pub fn quantized_max_pool1d_out( - &self, - out: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Tensor { - self.f_quantized_max_pool1d_out(out, kernel_size, stride, padding, dilation, ceil_mode) - .unwrap() - } - - pub fn quantized_max_pool2d( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Tensor { - self.f_quantized_max_pool2d(kernel_size, stride, padding, dilation, ceil_mode).unwrap() - } - - pub fn quantized_max_pool2d_out( - &self, - out: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Tensor { - self.f_quantized_max_pool2d_out(out, kernel_size, stride, padding, dilation, ceil_mode) - .unwrap() - } - - pub fn quantized_max_pool3d( - &self, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Tensor { - self.f_quantized_max_pool3d(kernel_size, stride, padding, dilation, ceil_mode).unwrap() - } - - pub fn quantized_max_pool3d_out( - &self, - out: &Tensor, - kernel_size: impl IntList, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ceil_mode: bool, - ) -> Tensor { - self.f_quantized_max_pool3d_out(out, kernel_size, stride, padding, dilation, ceil_mode) - .unwrap() - } - - pub fn quantized_rnn_relu_cell>( - &self, - hx: &Tensor, - w_ih: &Tensor, - w_hh: &Tensor, - b_ih: &Tensor, - b_hh: &Tensor, - packed_ih: &Tensor, - packed_hh: &Tensor, - col_offsets_ih: &Tensor, - col_offsets_hh: &Tensor, - scale_ih: S, - scale_hh: S, - zero_point_ih: S, - zero_point_hh: S, - ) -> Tensor { - self.f_quantized_rnn_relu_cell( - hx, - w_ih, - w_hh, - b_ih, - b_hh, - packed_ih, - packed_hh, - col_offsets_ih, - col_offsets_hh, - scale_ih, - scale_hh, - zero_point_ih, - zero_point_hh, - ) - .unwrap() - } - - pub fn quantized_rnn_tanh_cell>( - &self, - hx: &Tensor, - w_ih: &Tensor, - w_hh: &Tensor, - b_ih: &Tensor, - b_hh: &Tensor, - packed_ih: &Tensor, - packed_hh: &Tensor, - col_offsets_ih: &Tensor, - col_offsets_hh: &Tensor, - scale_ih: S, - scale_hh: S, - zero_point_ih: S, - zero_point_hh: S, - ) -> Tensor { - self.f_quantized_rnn_tanh_cell( - hx, - w_ih, - w_hh, - b_ih, - b_hh, - packed_ih, - packed_hh, - col_offsets_ih, - col_offsets_hh, - scale_ih, - scale_hh, - zero_point_ih, - zero_point_hh, - ) - .unwrap() - } - - pub fn rad2deg(&self) -> Tensor { - self.f_rad2deg().unwrap() - } - - pub fn rad2deg_(&mut self) -> Tensor { - self.f_rad2deg_().unwrap() - } - - pub fn rad2deg_out(&self, out: &Tensor) -> Tensor { - self.f_rad2deg_out(out).unwrap() - } - - pub fn rand(size: impl IntList, options: (Kind, Device)) -> Tensor { - Tensor::f_rand(size, options).unwrap() - } - - pub fn rand_like(&self) -> Tensor { - self.f_rand_like().unwrap() - } - - pub fn rand_like_out(&self, out: &Tensor) -> Tensor { - self.f_rand_like_out(out).unwrap() - } - - pub fn rand_out(out: &Tensor, size: impl IntList) -> Tensor { - Tensor::f_rand_out(out, size).unwrap() - } - - pub fn randint(high: i64, size: impl IntList, options: (Kind, Device)) -> Tensor { - Tensor::f_randint(high, size, options).unwrap() - } - - pub fn randint_like(&self, high: i64) -> Tensor { - self.f_randint_like(high).unwrap() - } - - pub fn randint_like_low_dtype(&self, low: i64, high: i64) -> Tensor { - self.f_randint_like_low_dtype(low, high).unwrap() - } - - pub fn randint_like_low_dtype_out(&self, out: &Tensor, low: i64, high: i64) -> Tensor { - self.f_randint_like_low_dtype_out(out, low, high).unwrap() - } - - pub fn randint_like_out(&self, out: &Tensor, high: i64) -> Tensor { - self.f_randint_like_out(out, high).unwrap() - } - - pub fn randint_like_tensor(&self, high: &Tensor) -> Tensor { - self.f_randint_like_tensor(high).unwrap() - } - - pub fn randint_like_tensor_out(&self, out: &Tensor, high: &Tensor) -> Tensor { - self.f_randint_like_tensor_out(out, high).unwrap() - } - - pub fn randint_low(low: i64, high: i64, size: impl IntList, options: (Kind, Device)) -> Tensor { - Tensor::f_randint_low(low, high, size, options).unwrap() - } - - pub fn randint_low_out(out: &Tensor, low: i64, high: i64, size: impl IntList) -> Tensor { - Tensor::f_randint_low_out(out, low, high, size).unwrap() - } - - pub fn randint_out(out: &Tensor, high: i64, size: impl IntList) -> Tensor { - Tensor::f_randint_out(out, high, size).unwrap() - } - - pub fn randn(size: impl IntList, options: (Kind, Device)) -> Tensor { - Tensor::f_randn(size, options).unwrap() - } - - pub fn randn_like(&self) -> Tensor { - self.f_randn_like().unwrap() - } - - pub fn randn_like_out(&self, out: &Tensor) -> Tensor { - self.f_randn_like_out(out).unwrap() - } - - pub fn randn_out(out: &Tensor, size: impl IntList) -> Tensor { - Tensor::f_randn_out(out, size).unwrap() - } - - pub fn random(&self) -> Tensor { - self.f_random().unwrap() - } - - pub fn random_(&mut self) -> Tensor { - self.f_random_().unwrap() - } - - pub fn random_from(&self, from: i64, to: impl Into>) -> Tensor { - self.f_random_from(from, to).unwrap() - } - - pub fn random_from_(&mut self, from: i64, to: impl Into>) -> Tensor { - self.f_random_from_(from, to).unwrap() - } - - pub fn random_from_out(&self, out: &Tensor, from: i64, to: impl Into>) -> Tensor { - self.f_random_from_out(out, from, to).unwrap() - } - - pub fn random_out(&self, out: &Tensor) -> Tensor { - self.f_random_out(out).unwrap() - } - - pub fn random_to(&self, to: i64) -> Tensor { - self.f_random_to(to).unwrap() - } - - pub fn random_to_(&mut self, to: i64) -> Tensor { - self.f_random_to_(to).unwrap() - } - - pub fn random_to_out(&self, out: &Tensor, to: i64) -> Tensor { - self.f_random_to_out(out, to).unwrap() - } - - pub fn randperm(n: i64, options: (Kind, Device)) -> Tensor { - Tensor::f_randperm(n, options).unwrap() - } - - pub fn randperm_out(out: &Tensor, n: i64) -> Tensor { - Tensor::f_randperm_out(out, n).unwrap() - } - - pub fn range>(start: S, end: S, options: (Kind, Device)) -> Tensor { - Tensor::f_range(start, end, options).unwrap() - } - - pub fn range_out>(out: &Tensor, start: S, end: S) -> Tensor { - Tensor::f_range_out(out, start, end).unwrap() - } - - pub fn range_out_>(out: &Tensor, start: S, end: S) -> Tensor { - Tensor::f_range_out_(out, start, end).unwrap() - } - - pub fn range_step>(start: S, end: S, options: (Kind, Device)) -> Tensor { - Tensor::f_range_step(start, end, options).unwrap() - } - - pub fn ravel(&self) -> Tensor { - self.f_ravel().unwrap() - } - - pub fn real(&self) -> Tensor { - self.f_real().unwrap() - } - - pub fn reciprocal(&self) -> Tensor { - self.f_reciprocal().unwrap() - } - - pub fn reciprocal_(&mut self) -> Tensor { - self.f_reciprocal_().unwrap() - } - - pub fn reciprocal_out(&self, out: &Tensor) -> Tensor { - self.f_reciprocal_out(out).unwrap() - } - - pub fn reflection_pad1d(&self, padding: impl IntList) -> Tensor { - self.f_reflection_pad1d(padding).unwrap() - } - - pub fn reflection_pad1d_backward(&self, grad_output: &Tensor, padding: impl IntList) -> Tensor { - self.f_reflection_pad1d_backward(grad_output, padding).unwrap() - } - - pub fn reflection_pad1d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - padding: impl IntList, - ) -> Tensor { - self.f_reflection_pad1d_backward_grad_input(grad_input, grad_output, padding).unwrap() - } - - pub fn reflection_pad1d_out(&self, out: &Tensor, padding: impl IntList) -> Tensor { - self.f_reflection_pad1d_out(out, padding).unwrap() - } - - pub fn reflection_pad2d(&self, padding: impl IntList) -> Tensor { - self.f_reflection_pad2d(padding).unwrap() - } - - pub fn reflection_pad2d_backward(&self, grad_output: &Tensor, padding: impl IntList) -> Tensor { - self.f_reflection_pad2d_backward(grad_output, padding).unwrap() - } - - pub fn reflection_pad2d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - padding: impl IntList, - ) -> Tensor { - self.f_reflection_pad2d_backward_grad_input(grad_input, grad_output, padding).unwrap() - } - - pub fn reflection_pad2d_out(&self, out: &Tensor, padding: impl IntList) -> Tensor { - self.f_reflection_pad2d_out(out, padding).unwrap() - } - - pub fn reflection_pad3d(&self, padding: impl IntList) -> Tensor { - self.f_reflection_pad3d(padding).unwrap() - } - - pub fn reflection_pad3d_backward(&self, grad_output: &Tensor, padding: impl IntList) -> Tensor { - self.f_reflection_pad3d_backward(grad_output, padding).unwrap() - } - - pub fn reflection_pad3d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - padding: impl IntList, - ) -> Tensor { - self.f_reflection_pad3d_backward_grad_input(grad_input, grad_output, padding).unwrap() - } - - pub fn reflection_pad3d_out(&self, out: &Tensor, padding: impl IntList) -> Tensor { - self.f_reflection_pad3d_out(out, padding).unwrap() - } - - pub fn relu(&self) -> Tensor { - self.f_relu().unwrap() - } - - pub fn relu6(&self) -> Tensor { - self.f_relu6().unwrap() - } - - pub fn relu6_(&mut self) -> Tensor { - self.f_relu6_().unwrap() - } - - pub fn relu_(&mut self) -> Tensor { - self.f_relu_().unwrap() - } - - pub fn relu_out(&self, out: &Tensor) -> Tensor { - self.f_relu_out(out).unwrap() - } - - pub fn remainder>(&self, other: S) -> Tensor { - self.f_remainder(other).unwrap() - } - - pub fn remainder_>(&mut self, other: S) -> Tensor { - self.f_remainder_(other).unwrap() - } - - pub fn remainder_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_remainder_scalar_out(out, other).unwrap() - } - - pub fn remainder_scalar_tensor>(self_scalar: S, other: &Tensor) -> Tensor { - Tensor::f_remainder_scalar_tensor(self_scalar, other).unwrap() - } - - pub fn remainder_scalar_tensor_out>( - out: &Tensor, - self_scalar: S, - other: &Tensor, - ) -> Tensor { - Tensor::f_remainder_scalar_tensor_out(out, self_scalar, other).unwrap() - } - - pub fn remainder_tensor(&self, other: &Tensor) -> Tensor { - self.f_remainder_tensor(other).unwrap() - } - - pub fn remainder_tensor_(&mut self, other: &Tensor) -> Tensor { - self.f_remainder_tensor_(other).unwrap() - } - - pub fn remainder_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_remainder_tensor_out(out, other).unwrap() - } - - pub fn renorm>(&self, p: S, dim: i64, maxnorm: S) -> Tensor { - self.f_renorm(p, dim, maxnorm).unwrap() - } - - pub fn renorm_>(&mut self, p: S, dim: i64, maxnorm: S) -> Tensor { - self.f_renorm_(p, dim, maxnorm).unwrap() - } - - pub fn renorm_out>(&self, out: &Tensor, p: S, dim: i64, maxnorm: S) -> Tensor { - self.f_renorm_out(out, p, dim, maxnorm).unwrap() - } - - pub fn repeat(&self, repeats: impl IntList) -> Tensor { - self.f_repeat(repeats).unwrap() - } - - pub fn repeat_interleave(repeats: &Tensor, output_size: impl Into>) -> Tensor { - Tensor::f_repeat_interleave(repeats, output_size).unwrap() - } - - pub fn repeat_interleave_self_int( - &self, - repeats: i64, - dim: impl Into>, - output_size: impl Into>, - ) -> Tensor { - self.f_repeat_interleave_self_int(repeats, dim, output_size).unwrap() - } - - pub fn repeat_interleave_self_tensor( - &self, - repeats: &Tensor, - dim: impl Into>, - output_size: impl Into>, - ) -> Tensor { - self.f_repeat_interleave_self_tensor(repeats, dim, output_size).unwrap() - } - - pub fn repeat_interleave_tensor_out( - out: &Tensor, - repeats: &Tensor, - output_size: impl Into>, - ) -> Tensor { - Tensor::f_repeat_interleave_tensor_out(out, repeats, output_size).unwrap() - } - - pub fn repeat_out(&self, out: &Tensor, repeats: impl IntList) -> Tensor { - self.f_repeat_out(out, repeats).unwrap() - } - - pub fn replication_pad1d(&self, padding: impl IntList) -> Tensor { - self.f_replication_pad1d(padding).unwrap() - } - - pub fn replication_pad1d_backward( - &self, - grad_output: &Tensor, - padding: impl IntList, - ) -> Tensor { - self.f_replication_pad1d_backward(grad_output, padding).unwrap() - } - - pub fn replication_pad1d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - padding: impl IntList, - ) -> Tensor { - self.f_replication_pad1d_backward_grad_input(grad_input, grad_output, padding).unwrap() - } - - pub fn replication_pad1d_out(&self, out: &Tensor, padding: impl IntList) -> Tensor { - self.f_replication_pad1d_out(out, padding).unwrap() - } - - pub fn replication_pad2d(&self, padding: impl IntList) -> Tensor { - self.f_replication_pad2d(padding).unwrap() - } - - pub fn replication_pad2d_backward( - &self, - grad_output: &Tensor, - padding: impl IntList, - ) -> Tensor { - self.f_replication_pad2d_backward(grad_output, padding).unwrap() - } - - pub fn replication_pad2d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - padding: impl IntList, - ) -> Tensor { - self.f_replication_pad2d_backward_grad_input(grad_input, grad_output, padding).unwrap() - } - - pub fn replication_pad2d_out(&self, out: &Tensor, padding: impl IntList) -> Tensor { - self.f_replication_pad2d_out(out, padding).unwrap() - } - - pub fn replication_pad3d(&self, padding: impl IntList) -> Tensor { - self.f_replication_pad3d(padding).unwrap() - } - - pub fn replication_pad3d_backward( - &self, - grad_output: &Tensor, - padding: impl IntList, - ) -> Tensor { - self.f_replication_pad3d_backward(grad_output, padding).unwrap() - } - - pub fn replication_pad3d_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - padding: impl IntList, - ) -> Tensor { - self.f_replication_pad3d_backward_grad_input(grad_input, grad_output, padding).unwrap() - } - - pub fn replication_pad3d_out(&self, out: &Tensor, padding: impl IntList) -> Tensor { - self.f_replication_pad3d_out(out, padding).unwrap() - } - - pub fn requires_grad_(&mut self, requires_grad: bool) -> Tensor { - self.f_requires_grad_(requires_grad).unwrap() - } - - pub fn reshape(&self, shape: impl IntList) -> Tensor { - self.f_reshape(shape).unwrap() - } - - pub fn reshape_as(&self, other: &Tensor) -> Tensor { - self.f_reshape_as(other).unwrap() - } - - pub fn resize(&self, size: impl IntList) -> Tensor { - self.f_resize(size).unwrap() - } - - pub fn resize_(&mut self, size: impl IntList) -> Tensor { - self.f_resize_(size).unwrap() - } - - pub fn resize_as(&self, the_template: &Tensor) -> Tensor { - self.f_resize_as(the_template).unwrap() - } - - pub fn resize_as_(&mut self, the_template: &Tensor) -> Tensor { - self.f_resize_as_(the_template).unwrap() - } - - pub fn resize_as_out(&self, out: &Tensor, the_template: &Tensor) -> Tensor { - self.f_resize_as_out(out, the_template).unwrap() - } - - pub fn resize_as_sparse(&self, the_template: &Tensor) -> Tensor { - self.f_resize_as_sparse(the_template).unwrap() - } - - pub fn resize_as_sparse_(&mut self, the_template: &Tensor) -> Tensor { - self.f_resize_as_sparse_(the_template).unwrap() - } - - pub fn resize_as_sparse_out(&self, out: &Tensor, the_template: &Tensor) -> Tensor { - self.f_resize_as_sparse_out(out, the_template).unwrap() - } - - pub fn resize_out(&self, out: &Tensor, size: impl IntList) -> Tensor { - self.f_resize_out(out, size).unwrap() - } - - pub fn resolve_conj(&self) -> Tensor { - self.f_resolve_conj().unwrap() - } - - pub fn resolve_neg(&self) -> Tensor { - self.f_resolve_neg().unwrap() - } - - pub fn retains_grad(&self) -> bool { - self.f_retains_grad().unwrap() - } - - pub fn rms_norm>( - &self, - normalized_shape: impl IntList, - weight: Option, - eps: impl Into>, - ) -> Tensor { - self.f_rms_norm(normalized_shape, weight, eps).unwrap() - } - - pub fn rnn_relu>( - &self, - hx: &Tensor, - params: &[T], - has_biases: bool, - num_layers: i64, - dropout: f64, - train: bool, - bidirectional: bool, - batch_first: bool, - ) -> (Tensor, Tensor) { - self.f_rnn_relu( - hx, - params, - has_biases, - num_layers, - dropout, - train, - bidirectional, - batch_first, - ) - .unwrap() - } - - pub fn rnn_relu_cell>( - &self, - hx: &Tensor, - w_ih: &Tensor, - w_hh: &Tensor, - b_ih: Option, - b_hh: Option, - ) -> Tensor { - self.f_rnn_relu_cell(hx, w_ih, w_hh, b_ih, b_hh).unwrap() - } - - pub fn rnn_relu_data>( - data: &Tensor, - batch_sizes: &Tensor, - hx: &Tensor, - params: &[T], - has_biases: bool, - num_layers: i64, - dropout: f64, - train: bool, - bidirectional: bool, - ) -> (Tensor, Tensor) { - Tensor::f_rnn_relu_data( - data, - batch_sizes, - hx, - params, - has_biases, - num_layers, - dropout, - train, - bidirectional, - ) - .unwrap() - } - - pub fn rnn_tanh>( - &self, - hx: &Tensor, - params: &[T], - has_biases: bool, - num_layers: i64, - dropout: f64, - train: bool, - bidirectional: bool, - batch_first: bool, - ) -> (Tensor, Tensor) { - self.f_rnn_tanh( - hx, - params, - has_biases, - num_layers, - dropout, - train, - bidirectional, - batch_first, - ) - .unwrap() - } - - pub fn rnn_tanh_cell>( - &self, - hx: &Tensor, - w_ih: &Tensor, - w_hh: &Tensor, - b_ih: Option, - b_hh: Option, - ) -> Tensor { - self.f_rnn_tanh_cell(hx, w_ih, w_hh, b_ih, b_hh).unwrap() - } - - pub fn rnn_tanh_data>( - data: &Tensor, - batch_sizes: &Tensor, - hx: &Tensor, - params: &[T], - has_biases: bool, - num_layers: i64, - dropout: f64, - train: bool, - bidirectional: bool, - ) -> (Tensor, Tensor) { - Tensor::f_rnn_tanh_data( - data, - batch_sizes, - hx, - params, - has_biases, - num_layers, - dropout, - train, - bidirectional, - ) - .unwrap() - } - - pub fn roll(&self, shifts: impl IntList, dims: impl IntList) -> Tensor { - self.f_roll(shifts, dims).unwrap() - } - - pub fn roll_out(&self, out: &Tensor, shifts: impl IntList, dims: impl IntList) -> Tensor { - self.f_roll_out(out, shifts, dims).unwrap() - } - - pub fn rot90(&self, k: i64, dims: impl IntList) -> Tensor { - self.f_rot90(k, dims).unwrap() - } - - pub fn rot90_out(&self, out: &Tensor, k: i64, dims: impl IntList) -> Tensor { - self.f_rot90_out(out, k, dims).unwrap() - } - - pub fn round(&self) -> Tensor { - self.f_round().unwrap() - } - - pub fn round_(&mut self) -> Tensor { - self.f_round_().unwrap() - } - - pub fn round_decimals(&self, decimals: i64) -> Tensor { - self.f_round_decimals(decimals).unwrap() - } - - pub fn round_decimals_(&mut self, decimals: i64) -> Tensor { - self.f_round_decimals_(decimals).unwrap() - } - - pub fn round_decimals_out(&self, out: &Tensor, decimals: i64) -> Tensor { - self.f_round_decimals_out(out, decimals).unwrap() - } - - pub fn round_out(&self, out: &Tensor) -> Tensor { - self.f_round_out(out).unwrap() - } - - pub fn row_indices(&self) -> Tensor { - self.f_row_indices().unwrap() - } - - pub fn row_indices_copy(&self) -> Tensor { - self.f_row_indices_copy().unwrap() - } - - pub fn row_indices_copy_out(&self, out: &Tensor) -> Tensor { - self.f_row_indices_copy_out(out).unwrap() - } - - pub fn row_stack>(tensors: &[T]) -> Tensor { - Tensor::f_row_stack(tensors).unwrap() - } - - pub fn row_stack_out>(out: &Tensor, tensors: &[T]) -> Tensor { - Tensor::f_row_stack_out(out, tensors).unwrap() - } - - pub fn rrelu(&self, training: bool) -> Tensor { - self.f_rrelu(training).unwrap() - } - - pub fn rrelu_(&mut self, training: bool) -> Tensor { - self.f_rrelu_(training).unwrap() - } - - pub fn rrelu_with_noise(&self, noise: &Tensor, training: bool) -> Tensor { - self.f_rrelu_with_noise(noise, training).unwrap() - } - - pub fn rrelu_with_noise_(&mut self, noise: &Tensor, training: bool) -> Tensor { - self.f_rrelu_with_noise_(noise, training).unwrap() - } - - pub fn rrelu_with_noise_backward>( - &self, - grad_output: &Tensor, - noise: &Tensor, - lower: S, - upper: S, - training: bool, - self_is_result: bool, - ) -> Tensor { - self.f_rrelu_with_noise_backward(grad_output, noise, lower, upper, training, self_is_result) - .unwrap() - } - - pub fn rrelu_with_noise_backward_out>( - &self, - out: &Tensor, - grad_output: &Tensor, - noise: &Tensor, - lower: S, - upper: S, - training: bool, - self_is_result: bool, - ) -> Tensor { - self.f_rrelu_with_noise_backward_out( - out, - grad_output, - noise, - lower, - upper, - training, - self_is_result, - ) - .unwrap() - } - - pub fn rrelu_with_noise_functional(&self, noise: &Tensor, training: bool) -> (Tensor, Tensor) { - self.f_rrelu_with_noise_functional(noise, training).unwrap() - } - - pub fn rrelu_with_noise_out(&self, out: &Tensor, noise: &Tensor, training: bool) -> Tensor { - self.f_rrelu_with_noise_out(out, noise, training).unwrap() - } - - pub fn rsqrt(&self) -> Tensor { - self.f_rsqrt().unwrap() - } - - pub fn rsqrt_(&mut self) -> Tensor { - self.f_rsqrt_().unwrap() - } - - pub fn rsqrt_out(&self, out: &Tensor) -> Tensor { - self.f_rsqrt_out(out).unwrap() - } - - pub fn rsub(&self, other: &Tensor) -> Tensor { - self.f_rsub(other).unwrap() - } - - pub fn rsub_scalar>(&self, other: S) -> Tensor { - self.f_rsub_scalar(other).unwrap() - } - - pub fn rsub_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_rsub_scalar_out(out, other).unwrap() - } - - pub fn rsub_tensor_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_rsub_tensor_out(out, other).unwrap() - } - - pub fn scalar_tensor>(s: S, options: (Kind, Device)) -> Tensor { - Tensor::f_scalar_tensor(s, options).unwrap() - } - - pub fn scalar_tensor_out>(out: &Tensor, s: S) -> Tensor { - Tensor::f_scalar_tensor_out(out, s).unwrap() - } - - pub fn scaled_dot_product_attention>( - query: &Tensor, - key: &Tensor, - value: &Tensor, - attn_mask: Option, - dropout_p: f64, - is_causal: bool, - scale: impl Into>, - enable_gqa: bool, - ) -> Tensor { - Tensor::f_scaled_dot_product_attention( - query, key, value, attn_mask, dropout_p, is_causal, scale, enable_gqa, - ) - .unwrap() - } - - pub fn scatter(&self, dim: i64, index: &Tensor, src: &Tensor) -> Tensor { - self.f_scatter(dim, index, src).unwrap() - } - - pub fn scatter_(&mut self, dim: i64, index: &Tensor, src: &Tensor) -> Tensor { - self.f_scatter_(dim, index, src).unwrap() - } - - pub fn scatter_add(&self, dim: i64, index: &Tensor, src: &Tensor) -> Tensor { - self.f_scatter_add(dim, index, src).unwrap() - } - - pub fn scatter_add_(&mut self, dim: i64, index: &Tensor, src: &Tensor) -> Tensor { - self.f_scatter_add_(dim, index, src).unwrap() - } - - pub fn scatter_add_out(&self, out: &Tensor, dim: i64, index: &Tensor, src: &Tensor) -> Tensor { - self.f_scatter_add_out(out, dim, index, src).unwrap() - } - - pub fn scatter_reduce(&self, dim: i64, index: &Tensor, src: &Tensor, reduce: &str) -> Tensor { - self.f_scatter_reduce(dim, index, src, reduce).unwrap() - } - - pub fn scatter_reduce_( - &mut self, - dim: i64, - index: &Tensor, - src: &Tensor, - reduce: &str, - ) -> Tensor { - self.f_scatter_reduce_(dim, index, src, reduce).unwrap() - } - - pub fn scatter_reduce_out( - &self, - out: &Tensor, - dim: i64, - index: &Tensor, - src: &Tensor, - reduce: &str, - ) -> Tensor { - self.f_scatter_reduce_out(out, dim, index, src, reduce).unwrap() - } - - pub fn scatter_src_out(&self, out: &Tensor, dim: i64, index: &Tensor, src: &Tensor) -> Tensor { - self.f_scatter_src_out(out, dim, index, src).unwrap() - } - - pub fn scatter_value>(&self, dim: i64, index: &Tensor, value: S) -> Tensor { - self.f_scatter_value(dim, index, value).unwrap() - } - - pub fn scatter_value_>( - &mut self, - dim: i64, - index: &Tensor, - value: S, - ) -> Tensor { - self.f_scatter_value_(dim, index, value).unwrap() - } - - pub fn scatter_value_out>( - &self, - out: &Tensor, - dim: i64, - index: &Tensor, - value: S, - ) -> Tensor { - self.f_scatter_value_out(out, dim, index, value).unwrap() - } - - pub fn scatter_value_reduce>( - &self, - dim: i64, - index: &Tensor, - value: S, - reduce: &str, - ) -> Tensor { - self.f_scatter_value_reduce(dim, index, value, reduce).unwrap() - } - - pub fn scatter_value_reduce_>( - &mut self, - dim: i64, - index: &Tensor, - value: S, - reduce: &str, - ) -> Tensor { - self.f_scatter_value_reduce_(dim, index, value, reduce).unwrap() - } - - pub fn scatter_value_reduce_out>( - &self, - out: &Tensor, - dim: i64, - index: &Tensor, - value: S, - reduce: &str, - ) -> Tensor { - self.f_scatter_value_reduce_out(out, dim, index, value, reduce).unwrap() - } - - pub fn searchsorted>( - &self, - sorted_sequence: &Tensor, - out_int32: bool, - right: bool, - side: &str, - sorter: Option, - ) -> Tensor { - self.f_searchsorted(sorted_sequence, out_int32, right, side, sorter).unwrap() - } - - pub fn searchsorted_scalar, S: Into>( - sorted_sequence: &Tensor, - self_scalar: S, - out_int32: bool, - right: bool, - side: &str, - sorter: Option, - ) -> Tensor { - Tensor::f_searchsorted_scalar(sorted_sequence, self_scalar, out_int32, right, side, sorter) - .unwrap() - } - - pub fn searchsorted_scalar_out, S: Into>( - out: &Tensor, - sorted_sequence: &Tensor, - self_scalar: S, - out_int32: bool, - right: bool, - side: &str, - sorter: Option, - ) -> Tensor { - Tensor::f_searchsorted_scalar_out( - out, - sorted_sequence, - self_scalar, - out_int32, - right, - side, - sorter, - ) - .unwrap() - } - - pub fn searchsorted_tensor_out>( - &self, - out: &Tensor, - sorted_sequence: &Tensor, - out_int32: bool, - right: bool, - side: &str, - sorter: Option, - ) -> Tensor { - self.f_searchsorted_tensor_out(out, sorted_sequence, out_int32, right, side, sorter) - .unwrap() - } - - pub fn segment_reduce, S: Into>( - data: &Tensor, - reduce: &str, - lengths: Option, - indices: Option, - offsets: Option, - axis: i64, - unsafe_: bool, - initial: S, - ) -> Tensor { - Tensor::f_segment_reduce(data, reduce, lengths, indices, offsets, axis, unsafe_, initial) - .unwrap() - } - - pub fn segment_reduce_out, S: Into>( - out: &Tensor, - data: &Tensor, - reduce: &str, - lengths: Option, - indices: Option, - offsets: Option, - axis: i64, - unsafe_: bool, - initial: S, - ) -> Tensor { - Tensor::f_segment_reduce_out( - out, data, reduce, lengths, indices, offsets, axis, unsafe_, initial, - ) - .unwrap() - } - - pub fn select(&self, dim: i64, index: i64) -> Tensor { - self.f_select(dim, index).unwrap() - } - - pub fn select_backward( - grad_output: &Tensor, - input_sizes: impl IntList, - dim: i64, - index: i64, - ) -> Tensor { - Tensor::f_select_backward(grad_output, input_sizes, dim, index).unwrap() - } - - pub fn select_backward_out( - out: &Tensor, - grad_output: &Tensor, - input_sizes: impl IntList, - dim: i64, - index: i64, - ) -> Tensor { - Tensor::f_select_backward_out(out, grad_output, input_sizes, dim, index).unwrap() - } - - pub fn select_copy(&self, dim: i64, index: i64) -> Tensor { - self.f_select_copy(dim, index).unwrap() - } - - pub fn select_copy_int_out(&self, out: &Tensor, dim: i64, index: i64) -> Tensor { - self.f_select_copy_int_out(out, dim, index).unwrap() - } - - pub fn select_scatter(&self, src: &Tensor, dim: i64, index: i64) -> Tensor { - self.f_select_scatter(src, dim, index).unwrap() - } - - pub fn select_scatter_out(&self, out: &Tensor, src: &Tensor, dim: i64, index: i64) -> Tensor { - self.f_select_scatter_out(out, src, dim, index).unwrap() - } - - pub fn selu(&self) -> Tensor { - self.f_selu().unwrap() - } - - pub fn selu_(&mut self) -> Tensor { - self.f_selu_().unwrap() - } - - pub fn set(&self) -> Tensor { - self.f_set().unwrap() - } - - pub fn set_(&mut self) -> Tensor { - self.f_set_().unwrap() - } - - pub fn set_data(&mut self, new_data: &Tensor) { - self.f_set_data(new_data).unwrap() - } - - pub fn set_out(&self, out: &Tensor) -> Tensor { - self.f_set_out(out).unwrap() - } - - pub fn set_requires_grad(&self, r: bool) -> Tensor { - self.f_set_requires_grad(r).unwrap() - } - - pub fn set_source_tensor(&self, source: &Tensor) -> Tensor { - self.f_set_source_tensor(source).unwrap() - } - - pub fn set_source_tensor_(&mut self, source: &Tensor) -> Tensor { - self.f_set_source_tensor_(source).unwrap() - } - - pub fn set_source_tensor_out(&self, out: &Tensor, source: &Tensor) -> Tensor { - self.f_set_source_tensor_out(out, source).unwrap() - } - - pub fn set_source_tensor_storage_offset_( - &mut self, - source: &Tensor, - storage_offset: i64, - size: impl IntList, - stride: impl IntList, - ) -> Tensor { - self.f_set_source_tensor_storage_offset_(source, storage_offset, size, stride).unwrap() - } - - pub fn sgn(&self) -> Tensor { - self.f_sgn().unwrap() - } - - pub fn sgn_(&mut self) -> Tensor { - self.f_sgn_().unwrap() - } - - pub fn sgn_out(&self, out: &Tensor) -> Tensor { - self.f_sgn_out(out).unwrap() - } - - pub fn sigmoid(&self) -> Tensor { - self.f_sigmoid().unwrap() - } - - pub fn sigmoid_(&mut self) -> Tensor { - self.f_sigmoid_().unwrap() - } - - pub fn sigmoid_backward(grad_output: &Tensor, output: &Tensor) -> Tensor { - Tensor::f_sigmoid_backward(grad_output, output).unwrap() - } - - pub fn sigmoid_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output: &Tensor, - ) -> Tensor { - Tensor::f_sigmoid_backward_grad_input(grad_input, grad_output, output).unwrap() - } - - pub fn sigmoid_out(&self, out: &Tensor) -> Tensor { - self.f_sigmoid_out(out).unwrap() - } - - pub fn sign(&self) -> Tensor { - self.f_sign().unwrap() - } - - pub fn sign_(&mut self) -> Tensor { - self.f_sign_().unwrap() - } - - pub fn sign_out(&self, out: &Tensor) -> Tensor { - self.f_sign_out(out).unwrap() - } - - pub fn signbit(&self) -> Tensor { - self.f_signbit().unwrap() - } - - pub fn signbit_out(&self, out: &Tensor) -> Tensor { - self.f_signbit_out(out).unwrap() - } - - pub fn silu(&self) -> Tensor { - self.f_silu().unwrap() - } - - pub fn silu_(&mut self) -> Tensor { - self.f_silu_().unwrap() - } - - pub fn silu_backward(&self, grad_output: &Tensor) -> Tensor { - self.f_silu_backward(grad_output).unwrap() - } - - pub fn silu_backward_grad_input(&self, grad_input: &Tensor, grad_output: &Tensor) -> Tensor { - self.f_silu_backward_grad_input(grad_input, grad_output).unwrap() - } - - pub fn silu_out(&self, out: &Tensor) -> Tensor { - self.f_silu_out(out).unwrap() - } - - pub fn sin(&self) -> Tensor { - self.f_sin().unwrap() - } - - pub fn sin_(&mut self) -> Tensor { - self.f_sin_().unwrap() - } - - pub fn sin_out(&self, out: &Tensor) -> Tensor { - self.f_sin_out(out).unwrap() - } - - pub fn sinc(&self) -> Tensor { - self.f_sinc().unwrap() - } - - pub fn sinc_(&mut self) -> Tensor { - self.f_sinc_().unwrap() - } - - pub fn sinc_out(&self, out: &Tensor) -> Tensor { - self.f_sinc_out(out).unwrap() - } - - pub fn sinh(&self) -> Tensor { - self.f_sinh().unwrap() - } - - pub fn sinh_(&mut self) -> Tensor { - self.f_sinh_().unwrap() - } - - pub fn sinh_out(&self, out: &Tensor) -> Tensor { - self.f_sinh_out(out).unwrap() - } - - pub fn slice( - &self, - dim: i64, - start: impl Into>, - end: impl Into>, - step: i64, - ) -> Tensor { - self.f_slice(dim, start, end, step).unwrap() - } - - pub fn slice_backward( - grad_output: &Tensor, - input_sizes: impl IntList, - dim: i64, - start: i64, - end: i64, - step: i64, - ) -> Tensor { - Tensor::f_slice_backward(grad_output, input_sizes, dim, start, end, step).unwrap() - } - - pub fn slice_backward_out( - out: &Tensor, - grad_output: &Tensor, - input_sizes: impl IntList, - dim: i64, - start: i64, - end: i64, - step: i64, - ) -> Tensor { - Tensor::f_slice_backward_out(out, grad_output, input_sizes, dim, start, end, step).unwrap() - } - - pub fn slice_copy( - &self, - dim: i64, - start: impl Into>, - end: impl Into>, - step: i64, - ) -> Tensor { - self.f_slice_copy(dim, start, end, step).unwrap() - } - - pub fn slice_copy_tensor_out( - &self, - out: &Tensor, - dim: i64, - start: impl Into>, - end: impl Into>, - step: i64, - ) -> Tensor { - self.f_slice_copy_tensor_out(out, dim, start, end, step).unwrap() - } - - pub fn slice_inverse( - &self, - src: &Tensor, - dim: i64, - start: impl Into>, - end: impl Into>, - step: i64, - ) -> Tensor { - self.f_slice_inverse(src, dim, start, end, step).unwrap() - } - - pub fn slice_scatter( - &self, - src: &Tensor, - dim: i64, - start: impl Into>, - end: impl Into>, - step: i64, - ) -> Tensor { - self.f_slice_scatter(src, dim, start, end, step).unwrap() - } - - pub fn slice_scatter_out( - &self, - out: &Tensor, - src: &Tensor, - dim: i64, - start: impl Into>, - end: impl Into>, - step: i64, - ) -> Tensor { - self.f_slice_scatter_out(out, src, dim, start, end, step).unwrap() - } - - pub fn slogdet(&self) -> (Tensor, Tensor) { - self.f_slogdet().unwrap() - } - - pub fn slogdet_out(&self, sign: &Tensor, logabsdet: &Tensor) -> (Tensor, Tensor) { - self.f_slogdet_out(sign, logabsdet).unwrap() - } - - pub fn slow_conv3d>( - &self, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - ) -> Tensor { - self.f_slow_conv3d(weight, kernel_size, bias, stride, padding).unwrap() - } - - pub fn slow_conv3d_out>( - &self, - out: &Tensor, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - ) -> Tensor { - self.f_slow_conv3d_out(out, weight, kernel_size, bias, stride, padding).unwrap() - } - - pub fn slow_conv_dilated2d>( - &self, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ) -> Tensor { - self.f_slow_conv_dilated2d(weight, kernel_size, bias, stride, padding, dilation).unwrap() - } - - pub fn slow_conv_dilated2d_out>( - &self, - out: &Tensor, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ) -> Tensor { - self.f_slow_conv_dilated2d_out(out, weight, kernel_size, bias, stride, padding, dilation) - .unwrap() - } - - pub fn slow_conv_dilated3d>( - &self, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ) -> Tensor { - self.f_slow_conv_dilated3d(weight, kernel_size, bias, stride, padding, dilation).unwrap() - } - - pub fn slow_conv_dilated3d_out>( - &self, - out: &Tensor, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - dilation: impl IntList, - ) -> Tensor { - self.f_slow_conv_dilated3d_out(out, weight, kernel_size, bias, stride, padding, dilation) - .unwrap() - } - - pub fn slow_conv_transpose2d>( - &self, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - output_padding: impl IntList, - dilation: impl IntList, - ) -> Tensor { - self.f_slow_conv_transpose2d( - weight, - kernel_size, - bias, - stride, - padding, - output_padding, - dilation, - ) - .unwrap() - } - - pub fn slow_conv_transpose2d_out>( - &self, - out: &Tensor, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - output_padding: impl IntList, - dilation: impl IntList, - ) -> Tensor { - self.f_slow_conv_transpose2d_out( - out, - weight, - kernel_size, - bias, - stride, - padding, - output_padding, - dilation, - ) - .unwrap() - } - - pub fn slow_conv_transpose3d>( - &self, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - output_padding: impl IntList, - dilation: impl IntList, - ) -> Tensor { - self.f_slow_conv_transpose3d( - weight, - kernel_size, - bias, - stride, - padding, - output_padding, - dilation, - ) - .unwrap() - } - - pub fn slow_conv_transpose3d_out>( - &self, - out: &Tensor, - weight: &Tensor, - kernel_size: impl IntList, - bias: Option, - stride: impl IntList, - padding: impl IntList, - output_padding: impl IntList, - dilation: impl IntList, - ) -> Tensor { - self.f_slow_conv_transpose3d_out( - out, - weight, - kernel_size, - bias, - stride, - padding, - output_padding, - dilation, - ) - .unwrap() - } - - pub fn smm(&self, mat2: &Tensor) -> Tensor { - self.f_smm(mat2).unwrap() - } - - pub fn smooth_l1_loss( - &self, - target: &Tensor, - reduction: crate::Reduction, - beta: f64, - ) -> Tensor { - self.f_smooth_l1_loss(target, reduction, beta).unwrap() - } - - pub fn smooth_l1_loss_backward( - &self, - grad_output: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - beta: f64, - ) -> Tensor { - self.f_smooth_l1_loss_backward(grad_output, target, reduction, beta).unwrap() - } - - pub fn smooth_l1_loss_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - beta: f64, - ) -> Tensor { - self.f_smooth_l1_loss_backward_grad_input(grad_input, grad_output, target, reduction, beta) - .unwrap() - } - - pub fn smooth_l1_loss_out( - &self, - out: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - beta: f64, - ) -> Tensor { - self.f_smooth_l1_loss_out(out, target, reduction, beta).unwrap() - } - - pub fn soft_margin_loss(&self, target: &Tensor, reduction: crate::Reduction) -> Tensor { - self.f_soft_margin_loss(target, reduction).unwrap() - } - - pub fn soft_margin_loss_backward( - &self, - grad_output: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - ) -> Tensor { - self.f_soft_margin_loss_backward(grad_output, target, reduction).unwrap() - } - - pub fn soft_margin_loss_backward_grad_input( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - ) -> Tensor { - self.f_soft_margin_loss_backward_grad_input(grad_input, grad_output, target, reduction) - .unwrap() - } - - pub fn soft_margin_loss_out( - &self, - out: &Tensor, - target: &Tensor, - reduction: crate::Reduction, - ) -> Tensor { - self.f_soft_margin_loss_out(out, target, reduction).unwrap() - } - - pub fn softmax(&self, dim: i64, dtype: impl Into>) -> Tensor { - self.f_softmax(dim, dtype).unwrap() - } - - pub fn softmax_int_out( - &self, - out: &Tensor, - dim: i64, - dtype: impl Into>, - ) -> Tensor { - self.f_softmax_int_out(out, dim, dtype).unwrap() - } - - pub fn softplus(&self) -> Tensor { - self.f_softplus().unwrap() - } - - pub fn softplus_backward>( - &self, - grad_output: &Tensor, - beta: S, - threshold: S, - ) -> Tensor { - self.f_softplus_backward(grad_output, beta, threshold).unwrap() - } - - pub fn softplus_backward_grad_input>( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - beta: S, - threshold: S, - ) -> Tensor { - self.f_softplus_backward_grad_input(grad_input, grad_output, beta, threshold).unwrap() - } - - pub fn softplus_out(&self, out: &Tensor) -> Tensor { - self.f_softplus_out(out).unwrap() - } - - pub fn softshrink(&self) -> Tensor { - self.f_softshrink().unwrap() - } - - pub fn softshrink_backward>(&self, grad_output: &Tensor, lambd: S) -> Tensor { - self.f_softshrink_backward(grad_output, lambd).unwrap() - } - - pub fn softshrink_backward_grad_input>( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - lambd: S, - ) -> Tensor { - self.f_softshrink_backward_grad_input(grad_input, grad_output, lambd).unwrap() - } - - pub fn softshrink_out(&self, out: &Tensor) -> Tensor { - self.f_softshrink_out(out).unwrap() - } - - pub fn sort(&self, dim: i64, descending: bool) -> (Tensor, Tensor) { - self.f_sort(dim, descending).unwrap() - } - - pub fn sort_stable(&self, stable: bool, dim: i64, descending: bool) -> (Tensor, Tensor) { - self.f_sort_stable(stable, dim, descending).unwrap() - } - - pub fn sort_values( - &self, - values: &Tensor, - indices: &Tensor, - dim: i64, - descending: bool, - ) -> (Tensor, Tensor) { - self.f_sort_values(values, indices, dim, descending).unwrap() - } - - pub fn sort_values_stable( - &self, - values: &Tensor, - indices: &Tensor, - stable: bool, - dim: i64, - descending: bool, - ) -> (Tensor, Tensor) { - self.f_sort_values_stable(values, indices, stable, dim, descending).unwrap() - } - - pub fn sparse_bsc_tensor( - ccol_indices: &Tensor, - row_indices: &Tensor, - values: &Tensor, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_sparse_bsc_tensor(ccol_indices, row_indices, values, options).unwrap() - } - - pub fn sparse_bsc_tensor_ccol_row_value_size( - ccol_indices: &Tensor, - row_indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_sparse_bsc_tensor_ccol_row_value_size( - ccol_indices, - row_indices, - values, - size, - options, - ) - .unwrap() - } - - pub fn sparse_bsr_tensor( - crow_indices: &Tensor, - col_indices: &Tensor, - values: &Tensor, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_sparse_bsr_tensor(crow_indices, col_indices, values, options).unwrap() - } - - pub fn sparse_bsr_tensor_crow_col_value_size( - crow_indices: &Tensor, - col_indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_sparse_bsr_tensor_crow_col_value_size( - crow_indices, - col_indices, - values, - size, - options, - ) - .unwrap() - } - - pub fn sparse_compressed_tensor( - compressed_indices: &Tensor, - plain_indices: &Tensor, - values: &Tensor, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_sparse_compressed_tensor(compressed_indices, plain_indices, values, options) - .unwrap() - } - - pub fn sparse_compressed_tensor_comp_plain_value_size( - compressed_indices: &Tensor, - plain_indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_sparse_compressed_tensor_comp_plain_value_size( - compressed_indices, - plain_indices, - values, - size, - options, - ) - .unwrap() - } - - pub fn sparse_coo_tensor(size: impl IntList, options: (Kind, Device)) -> Tensor { - Tensor::f_sparse_coo_tensor(size, options).unwrap() - } - - pub fn sparse_coo_tensor_indices( - indices: &Tensor, - values: &Tensor, - options: (Kind, Device), - is_coalesced: bool, - ) -> Tensor { - Tensor::f_sparse_coo_tensor_indices(indices, values, options, is_coalesced).unwrap() - } - - pub fn sparse_coo_tensor_indices_size( - indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - is_coalesced: bool, - ) -> Tensor { - Tensor::f_sparse_coo_tensor_indices_size(indices, values, size, options, is_coalesced) - .unwrap() - } - - pub fn sparse_coo_tensor_size_out(out: &Tensor, size: impl IntList) -> Tensor { - Tensor::f_sparse_coo_tensor_size_out(out, size).unwrap() - } - - pub fn sparse_csc_tensor( - ccol_indices: &Tensor, - row_indices: &Tensor, - values: &Tensor, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_sparse_csc_tensor(ccol_indices, row_indices, values, options).unwrap() - } - - pub fn sparse_csc_tensor_ccol_row_value_size( - ccol_indices: &Tensor, - row_indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_sparse_csc_tensor_ccol_row_value_size( - ccol_indices, - row_indices, - values, - size, - options, - ) - .unwrap() - } - - pub fn sparse_csr_tensor( - crow_indices: &Tensor, - col_indices: &Tensor, - values: &Tensor, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_sparse_csr_tensor(crow_indices, col_indices, values, options).unwrap() - } - - pub fn sparse_csr_tensor_crow_col_value_size( - crow_indices: &Tensor, - col_indices: &Tensor, - values: &Tensor, - size: impl IntList, - options: (Kind, Device), - ) -> Tensor { - Tensor::f_sparse_csr_tensor_crow_col_value_size( - crow_indices, - col_indices, - values, - size, - options, - ) - .unwrap() - } - - pub fn sparse_dim(&self) -> i64 { - self.f_sparse_dim().unwrap() - } - - pub fn sparse_mask(&self, mask: &Tensor) -> Tensor { - self.f_sparse_mask(mask).unwrap() - } - - pub fn sparse_mask_out(&self, out: &Tensor, mask: &Tensor) -> Tensor { - self.f_sparse_mask_out(out, mask).unwrap() - } - - pub fn sparse_resize(&self, size: impl IntList, sparse_dim: i64, dense_dim: i64) -> Tensor { - self.f_sparse_resize(size, sparse_dim, dense_dim).unwrap() - } - - pub fn sparse_resize_( - &mut self, - size: impl IntList, - sparse_dim: i64, - dense_dim: i64, - ) -> Tensor { - self.f_sparse_resize_(size, sparse_dim, dense_dim).unwrap() - } - - pub fn sparse_resize_and_clear( - &self, - size: impl IntList, - sparse_dim: i64, - dense_dim: i64, - ) -> Tensor { - self.f_sparse_resize_and_clear(size, sparse_dim, dense_dim).unwrap() - } - - pub fn sparse_resize_and_clear_( - &mut self, - size: impl IntList, - sparse_dim: i64, - dense_dim: i64, - ) -> Tensor { - self.f_sparse_resize_and_clear_(size, sparse_dim, dense_dim).unwrap() - } - - pub fn sparse_resize_and_clear_out( - &self, - out: &Tensor, - size: impl IntList, - sparse_dim: i64, - dense_dim: i64, - ) -> Tensor { - self.f_sparse_resize_and_clear_out(out, size, sparse_dim, dense_dim).unwrap() - } - - pub fn sparse_resize_out( - &self, - out: &Tensor, - size: impl IntList, - sparse_dim: i64, - dense_dim: i64, - ) -> Tensor { - self.f_sparse_resize_out(out, size, sparse_dim, dense_dim).unwrap() - } - - pub fn sparse_sampled_addmm(&self, mat1: &Tensor, mat2: &Tensor) -> Tensor { - self.f_sparse_sampled_addmm(mat1, mat2).unwrap() - } - - pub fn sparse_sampled_addmm_out(&self, out: &Tensor, mat1: &Tensor, mat2: &Tensor) -> Tensor { - self.f_sparse_sampled_addmm_out(out, mat1, mat2).unwrap() - } - - pub fn special_airy_ai(x: &Tensor) -> Tensor { - Tensor::f_special_airy_ai(x).unwrap() - } - - pub fn special_airy_ai_out(out: &Tensor, x: &Tensor) -> Tensor { - Tensor::f_special_airy_ai_out(out, x).unwrap() - } - - pub fn special_bessel_j0(&self) -> Tensor { - self.f_special_bessel_j0().unwrap() - } - - pub fn special_bessel_j0_out(&self, out: &Tensor) -> Tensor { - self.f_special_bessel_j0_out(out).unwrap() - } - - pub fn special_bessel_j1(&self) -> Tensor { - self.f_special_bessel_j1().unwrap() - } - - pub fn special_bessel_j1_out(&self, out: &Tensor) -> Tensor { - self.f_special_bessel_j1_out(out).unwrap() - } - - pub fn special_bessel_y0(&self) -> Tensor { - self.f_special_bessel_y0().unwrap() - } - - pub fn special_bessel_y0_out(&self, out: &Tensor) -> Tensor { - self.f_special_bessel_y0_out(out).unwrap() - } - - pub fn special_bessel_y1(&self) -> Tensor { - self.f_special_bessel_y1().unwrap() - } - - pub fn special_bessel_y1_out(&self, out: &Tensor) -> Tensor { - self.f_special_bessel_y1_out(out).unwrap() - } - - pub fn special_chebyshev_polynomial_t(x: &Tensor, n: &Tensor) -> Tensor { - Tensor::f_special_chebyshev_polynomial_t(x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_t_n_scalar>(x: &Tensor, n: S) -> Tensor { - Tensor::f_special_chebyshev_polynomial_t_n_scalar(x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_t_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Tensor { - Tensor::f_special_chebyshev_polynomial_t_n_scalar_out(out, x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_t_out(out: &Tensor, x: &Tensor, n: &Tensor) -> Tensor { - Tensor::f_special_chebyshev_polynomial_t_out(out, x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_t_x_scalar>(x: S, n: &Tensor) -> Tensor { - Tensor::f_special_chebyshev_polynomial_t_x_scalar(x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_t_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Tensor { - Tensor::f_special_chebyshev_polynomial_t_x_scalar_out(out, x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_u(x: &Tensor, n: &Tensor) -> Tensor { - Tensor::f_special_chebyshev_polynomial_u(x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_u_n_scalar>(x: &Tensor, n: S) -> Tensor { - Tensor::f_special_chebyshev_polynomial_u_n_scalar(x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_u_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Tensor { - Tensor::f_special_chebyshev_polynomial_u_n_scalar_out(out, x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_u_out(out: &Tensor, x: &Tensor, n: &Tensor) -> Tensor { - Tensor::f_special_chebyshev_polynomial_u_out(out, x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_u_x_scalar>(x: S, n: &Tensor) -> Tensor { - Tensor::f_special_chebyshev_polynomial_u_x_scalar(x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_u_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Tensor { - Tensor::f_special_chebyshev_polynomial_u_x_scalar_out(out, x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_v(x: &Tensor, n: &Tensor) -> Tensor { - Tensor::f_special_chebyshev_polynomial_v(x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_v_n_scalar>(x: &Tensor, n: S) -> Tensor { - Tensor::f_special_chebyshev_polynomial_v_n_scalar(x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_v_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Tensor { - Tensor::f_special_chebyshev_polynomial_v_n_scalar_out(out, x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_v_out(out: &Tensor, x: &Tensor, n: &Tensor) -> Tensor { - Tensor::f_special_chebyshev_polynomial_v_out(out, x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_v_x_scalar>(x: S, n: &Tensor) -> Tensor { - Tensor::f_special_chebyshev_polynomial_v_x_scalar(x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_v_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Tensor { - Tensor::f_special_chebyshev_polynomial_v_x_scalar_out(out, x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_w(x: &Tensor, n: &Tensor) -> Tensor { - Tensor::f_special_chebyshev_polynomial_w(x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_w_n_scalar>(x: &Tensor, n: S) -> Tensor { - Tensor::f_special_chebyshev_polynomial_w_n_scalar(x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_w_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Tensor { - Tensor::f_special_chebyshev_polynomial_w_n_scalar_out(out, x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_w_out(out: &Tensor, x: &Tensor, n: &Tensor) -> Tensor { - Tensor::f_special_chebyshev_polynomial_w_out(out, x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_w_x_scalar>(x: S, n: &Tensor) -> Tensor { - Tensor::f_special_chebyshev_polynomial_w_x_scalar(x, n).unwrap() - } - - pub fn special_chebyshev_polynomial_w_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Tensor { - Tensor::f_special_chebyshev_polynomial_w_x_scalar_out(out, x, n).unwrap() - } - - pub fn special_digamma(&self) -> Tensor { - self.f_special_digamma().unwrap() - } - - pub fn special_digamma_out(&self, out: &Tensor) -> Tensor { - self.f_special_digamma_out(out).unwrap() - } - - pub fn special_entr(&self) -> Tensor { - self.f_special_entr().unwrap() - } - - pub fn special_entr_out(&self, out: &Tensor) -> Tensor { - self.f_special_entr_out(out).unwrap() - } - - pub fn special_erf(&self) -> Tensor { - self.f_special_erf().unwrap() - } - - pub fn special_erf_out(&self, out: &Tensor) -> Tensor { - self.f_special_erf_out(out).unwrap() - } - - pub fn special_erfc(&self) -> Tensor { - self.f_special_erfc().unwrap() - } - - pub fn special_erfc_out(&self, out: &Tensor) -> Tensor { - self.f_special_erfc_out(out).unwrap() - } - - pub fn special_erfcx(&self) -> Tensor { - self.f_special_erfcx().unwrap() - } - - pub fn special_erfcx_out(&self, out: &Tensor) -> Tensor { - self.f_special_erfcx_out(out).unwrap() - } - - pub fn special_erfinv(&self) -> Tensor { - self.f_special_erfinv().unwrap() - } - - pub fn special_erfinv_out(&self, out: &Tensor) -> Tensor { - self.f_special_erfinv_out(out).unwrap() - } - - pub fn special_exp2(&self) -> Tensor { - self.f_special_exp2().unwrap() - } - - pub fn special_exp2_out(&self, out: &Tensor) -> Tensor { - self.f_special_exp2_out(out).unwrap() - } - - pub fn special_expit(&self) -> Tensor { - self.f_special_expit().unwrap() - } - - pub fn special_expit_out(&self, out: &Tensor) -> Tensor { - self.f_special_expit_out(out).unwrap() - } - - pub fn special_expm1(&self) -> Tensor { - self.f_special_expm1().unwrap() - } - - pub fn special_expm1_out(&self, out: &Tensor) -> Tensor { - self.f_special_expm1_out(out).unwrap() - } - - pub fn special_gammainc(&self, other: &Tensor) -> Tensor { - self.f_special_gammainc(other).unwrap() - } - - pub fn special_gammainc_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_special_gammainc_out(out, other).unwrap() - } - - pub fn special_gammaincc(&self, other: &Tensor) -> Tensor { - self.f_special_gammaincc(other).unwrap() - } - - pub fn special_gammaincc_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_special_gammaincc_out(out, other).unwrap() - } - - pub fn special_gammaln(&self) -> Tensor { - self.f_special_gammaln().unwrap() - } - - pub fn special_gammaln_out(&self, out: &Tensor) -> Tensor { - self.f_special_gammaln_out(out).unwrap() - } - - pub fn special_hermite_polynomial_h(x: &Tensor, n: &Tensor) -> Tensor { - Tensor::f_special_hermite_polynomial_h(x, n).unwrap() - } - - pub fn special_hermite_polynomial_h_n_scalar>(x: &Tensor, n: S) -> Tensor { - Tensor::f_special_hermite_polynomial_h_n_scalar(x, n).unwrap() - } - - pub fn special_hermite_polynomial_h_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Tensor { - Tensor::f_special_hermite_polynomial_h_n_scalar_out(out, x, n).unwrap() - } - - pub fn special_hermite_polynomial_h_out(out: &Tensor, x: &Tensor, n: &Tensor) -> Tensor { - Tensor::f_special_hermite_polynomial_h_out(out, x, n).unwrap() - } - - pub fn special_hermite_polynomial_h_x_scalar>(x: S, n: &Tensor) -> Tensor { - Tensor::f_special_hermite_polynomial_h_x_scalar(x, n).unwrap() - } - - pub fn special_hermite_polynomial_h_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Tensor { - Tensor::f_special_hermite_polynomial_h_x_scalar_out(out, x, n).unwrap() - } - - pub fn special_hermite_polynomial_he(x: &Tensor, n: &Tensor) -> Tensor { - Tensor::f_special_hermite_polynomial_he(x, n).unwrap() - } - - pub fn special_hermite_polynomial_he_n_scalar>(x: &Tensor, n: S) -> Tensor { - Tensor::f_special_hermite_polynomial_he_n_scalar(x, n).unwrap() - } - - pub fn special_hermite_polynomial_he_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Tensor { - Tensor::f_special_hermite_polynomial_he_n_scalar_out(out, x, n).unwrap() - } - - pub fn special_hermite_polynomial_he_out(out: &Tensor, x: &Tensor, n: &Tensor) -> Tensor { - Tensor::f_special_hermite_polynomial_he_out(out, x, n).unwrap() - } - - pub fn special_hermite_polynomial_he_x_scalar>(x: S, n: &Tensor) -> Tensor { - Tensor::f_special_hermite_polynomial_he_x_scalar(x, n).unwrap() - } - - pub fn special_hermite_polynomial_he_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Tensor { - Tensor::f_special_hermite_polynomial_he_x_scalar_out(out, x, n).unwrap() - } - - pub fn special_i0(&self) -> Tensor { - self.f_special_i0().unwrap() - } - - pub fn special_i0_out(&self, out: &Tensor) -> Tensor { - self.f_special_i0_out(out).unwrap() - } - - pub fn special_i0e(&self) -> Tensor { - self.f_special_i0e().unwrap() - } - - pub fn special_i0e_out(&self, out: &Tensor) -> Tensor { - self.f_special_i0e_out(out).unwrap() - } - - pub fn special_i1(&self) -> Tensor { - self.f_special_i1().unwrap() - } - - pub fn special_i1_out(&self, out: &Tensor) -> Tensor { - self.f_special_i1_out(out).unwrap() - } - - pub fn special_i1e(&self) -> Tensor { - self.f_special_i1e().unwrap() - } - - pub fn special_i1e_out(&self, out: &Tensor) -> Tensor { - self.f_special_i1e_out(out).unwrap() - } - - pub fn special_laguerre_polynomial_l(x: &Tensor, n: &Tensor) -> Tensor { - Tensor::f_special_laguerre_polynomial_l(x, n).unwrap() - } - - pub fn special_laguerre_polynomial_l_n_scalar>(x: &Tensor, n: S) -> Tensor { - Tensor::f_special_laguerre_polynomial_l_n_scalar(x, n).unwrap() - } - - pub fn special_laguerre_polynomial_l_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Tensor { - Tensor::f_special_laguerre_polynomial_l_n_scalar_out(out, x, n).unwrap() - } - - pub fn special_laguerre_polynomial_l_out(out: &Tensor, x: &Tensor, n: &Tensor) -> Tensor { - Tensor::f_special_laguerre_polynomial_l_out(out, x, n).unwrap() - } - - pub fn special_laguerre_polynomial_l_x_scalar>(x: S, n: &Tensor) -> Tensor { - Tensor::f_special_laguerre_polynomial_l_x_scalar(x, n).unwrap() - } - - pub fn special_laguerre_polynomial_l_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Tensor { - Tensor::f_special_laguerre_polynomial_l_x_scalar_out(out, x, n).unwrap() - } - - pub fn special_legendre_polynomial_p(x: &Tensor, n: &Tensor) -> Tensor { - Tensor::f_special_legendre_polynomial_p(x, n).unwrap() - } - - pub fn special_legendre_polynomial_p_n_scalar>(x: &Tensor, n: S) -> Tensor { - Tensor::f_special_legendre_polynomial_p_n_scalar(x, n).unwrap() - } - - pub fn special_legendre_polynomial_p_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Tensor { - Tensor::f_special_legendre_polynomial_p_n_scalar_out(out, x, n).unwrap() - } - - pub fn special_legendre_polynomial_p_out(out: &Tensor, x: &Tensor, n: &Tensor) -> Tensor { - Tensor::f_special_legendre_polynomial_p_out(out, x, n).unwrap() - } - - pub fn special_legendre_polynomial_p_x_scalar>(x: S, n: &Tensor) -> Tensor { - Tensor::f_special_legendre_polynomial_p_x_scalar(x, n).unwrap() - } - - pub fn special_legendre_polynomial_p_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Tensor { - Tensor::f_special_legendre_polynomial_p_x_scalar_out(out, x, n).unwrap() - } - - pub fn special_log1p(&self) -> Tensor { - self.f_special_log1p().unwrap() - } - - pub fn special_log1p_out(&self, out: &Tensor) -> Tensor { - self.f_special_log1p_out(out).unwrap() - } - - pub fn special_log_ndtr(&self) -> Tensor { - self.f_special_log_ndtr().unwrap() - } - - pub fn special_log_ndtr_out(&self, out: &Tensor) -> Tensor { - self.f_special_log_ndtr_out(out).unwrap() - } - - pub fn special_log_softmax(&self, dim: i64, dtype: impl Into>) -> Tensor { - self.f_special_log_softmax(dim, dtype).unwrap() - } - - pub fn special_logit(&self, eps: impl Into>) -> Tensor { - self.f_special_logit(eps).unwrap() - } - - pub fn special_logit_out(&self, out: &Tensor, eps: impl Into>) -> Tensor { - self.f_special_logit_out(out, eps).unwrap() - } - - pub fn special_logsumexp(&self, dim: impl IntList, keepdim: bool) -> Tensor { - self.f_special_logsumexp(dim, keepdim).unwrap() - } - - pub fn special_logsumexp_out(&self, out: &Tensor, dim: impl IntList, keepdim: bool) -> Tensor { - self.f_special_logsumexp_out(out, dim, keepdim).unwrap() - } - - pub fn special_modified_bessel_i0(&self) -> Tensor { - self.f_special_modified_bessel_i0().unwrap() - } - - pub fn special_modified_bessel_i0_out(&self, out: &Tensor) -> Tensor { - self.f_special_modified_bessel_i0_out(out).unwrap() - } - - pub fn special_modified_bessel_i1(&self) -> Tensor { - self.f_special_modified_bessel_i1().unwrap() - } - - pub fn special_modified_bessel_i1_out(&self, out: &Tensor) -> Tensor { - self.f_special_modified_bessel_i1_out(out).unwrap() - } - - pub fn special_modified_bessel_k0(&self) -> Tensor { - self.f_special_modified_bessel_k0().unwrap() - } - - pub fn special_modified_bessel_k0_out(&self, out: &Tensor) -> Tensor { - self.f_special_modified_bessel_k0_out(out).unwrap() - } - - pub fn special_modified_bessel_k1(&self) -> Tensor { - self.f_special_modified_bessel_k1().unwrap() - } - - pub fn special_modified_bessel_k1_out(&self, out: &Tensor) -> Tensor { - self.f_special_modified_bessel_k1_out(out).unwrap() - } - - pub fn special_multigammaln(&self, p: i64) -> Tensor { - self.f_special_multigammaln(p).unwrap() - } - - pub fn special_multigammaln_out(&self, out: &Tensor, p: i64) -> Tensor { - self.f_special_multigammaln_out(out, p).unwrap() - } - - pub fn special_ndtr(&self) -> Tensor { - self.f_special_ndtr().unwrap() - } - - pub fn special_ndtr_out(&self, out: &Tensor) -> Tensor { - self.f_special_ndtr_out(out).unwrap() - } - - pub fn special_ndtri(&self) -> Tensor { - self.f_special_ndtri().unwrap() - } - - pub fn special_ndtri_out(&self, out: &Tensor) -> Tensor { - self.f_special_ndtri_out(out).unwrap() - } - - pub fn special_polygamma(&self, n: i64) -> Tensor { - self.f_special_polygamma(n).unwrap() - } - - pub fn special_polygamma_out(&self, out: &Tensor, n: i64) -> Tensor { - self.f_special_polygamma_out(out, n).unwrap() - } - - pub fn special_psi(&self) -> Tensor { - self.f_special_psi().unwrap() - } - - pub fn special_psi_out(&self, out: &Tensor) -> Tensor { - self.f_special_psi_out(out).unwrap() - } - - pub fn special_round(&self, decimals: i64) -> Tensor { - self.f_special_round(decimals).unwrap() - } - - pub fn special_round_out(&self, out: &Tensor, decimals: i64) -> Tensor { - self.f_special_round_out(out, decimals).unwrap() - } - - pub fn special_scaled_modified_bessel_k0(x: &Tensor) -> Tensor { - Tensor::f_special_scaled_modified_bessel_k0(x).unwrap() - } - - pub fn special_scaled_modified_bessel_k0_out(out: &Tensor, x: &Tensor) -> Tensor { - Tensor::f_special_scaled_modified_bessel_k0_out(out, x).unwrap() - } - - pub fn special_scaled_modified_bessel_k1(x: &Tensor) -> Tensor { - Tensor::f_special_scaled_modified_bessel_k1(x).unwrap() - } - - pub fn special_scaled_modified_bessel_k1_out(out: &Tensor, x: &Tensor) -> Tensor { - Tensor::f_special_scaled_modified_bessel_k1_out(out, x).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_t(x: &Tensor, n: &Tensor) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_t(x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_t_n_scalar>( - x: &Tensor, - n: S, - ) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_t_n_scalar(x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_t_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_t_n_scalar_out(out, x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_t_out( - out: &Tensor, - x: &Tensor, - n: &Tensor, - ) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_t_out(out, x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_t_x_scalar>( - x: S, - n: &Tensor, - ) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_t_x_scalar(x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_t_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_t_x_scalar_out(out, x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_u(x: &Tensor, n: &Tensor) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_u(x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_u_n_scalar>( - x: &Tensor, - n: S, - ) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_u_n_scalar(x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_u_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_u_n_scalar_out(out, x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_u_out( - out: &Tensor, - x: &Tensor, - n: &Tensor, - ) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_u_out(out, x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_u_x_scalar>( - x: S, - n: &Tensor, - ) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_u_x_scalar(x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_u_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_u_x_scalar_out(out, x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_v(x: &Tensor, n: &Tensor) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_v(x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_v_n_scalar>( - x: &Tensor, - n: S, - ) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_v_n_scalar(x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_v_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_v_n_scalar_out(out, x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_v_out( - out: &Tensor, - x: &Tensor, - n: &Tensor, - ) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_v_out(out, x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_v_x_scalar>( - x: S, - n: &Tensor, - ) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_v_x_scalar(x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_v_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_v_x_scalar_out(out, x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_w(x: &Tensor, n: &Tensor) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_w(x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_w_n_scalar>( - x: &Tensor, - n: S, - ) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_w_n_scalar(x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_w_n_scalar_out>( - out: &Tensor, - x: &Tensor, - n: S, - ) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_w_n_scalar_out(out, x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_w_out( - out: &Tensor, - x: &Tensor, - n: &Tensor, - ) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_w_out(out, x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_w_x_scalar>( - x: S, - n: &Tensor, - ) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_w_x_scalar(x, n).unwrap() - } - - pub fn special_shifted_chebyshev_polynomial_w_x_scalar_out>( - out: &Tensor, - x: S, - n: &Tensor, - ) -> Tensor { - Tensor::f_special_shifted_chebyshev_polynomial_w_x_scalar_out(out, x, n).unwrap() - } - - pub fn special_sinc(&self) -> Tensor { - self.f_special_sinc().unwrap() - } - - pub fn special_sinc_out(&self, out: &Tensor) -> Tensor { - self.f_special_sinc_out(out).unwrap() - } - - pub fn special_softmax(&self, dim: i64, dtype: impl Into>) -> Tensor { - self.f_special_softmax(dim, dtype).unwrap() - } - - pub fn special_spherical_bessel_j0(x: &Tensor) -> Tensor { - Tensor::f_special_spherical_bessel_j0(x).unwrap() - } - - pub fn special_spherical_bessel_j0_out(out: &Tensor, x: &Tensor) -> Tensor { - Tensor::f_special_spherical_bessel_j0_out(out, x).unwrap() - } - - pub fn special_xlog1py(&self, other: &Tensor) -> Tensor { - self.f_special_xlog1py(other).unwrap() - } - - pub fn special_xlog1py_other_scalar>(&self, other: S) -> Tensor { - self.f_special_xlog1py_other_scalar(other).unwrap() - } - - pub fn special_xlog1py_other_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Tensor { - self.f_special_xlog1py_other_scalar_out(out, other).unwrap() - } - - pub fn special_xlog1py_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_special_xlog1py_out(out, other).unwrap() - } - - pub fn special_xlog1py_self_scalar>(self_scalar: S, other: &Tensor) -> Tensor { - Tensor::f_special_xlog1py_self_scalar(self_scalar, other).unwrap() - } - - pub fn special_xlog1py_self_scalar_out>( - out: &Tensor, - self_scalar: S, - other: &Tensor, - ) -> Tensor { - Tensor::f_special_xlog1py_self_scalar_out(out, self_scalar, other).unwrap() - } - - pub fn special_xlogy(&self, other: &Tensor) -> Tensor { - self.f_special_xlogy(other).unwrap() - } - - pub fn special_xlogy_other_scalar>(&self, other: S) -> Tensor { - self.f_special_xlogy_other_scalar(other).unwrap() - } - - pub fn special_xlogy_other_scalar_out>( - &self, - out: &Tensor, - other: S, - ) -> Tensor { - self.f_special_xlogy_other_scalar_out(out, other).unwrap() - } - - pub fn special_xlogy_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_special_xlogy_out(out, other).unwrap() - } - - pub fn special_xlogy_self_scalar>(self_scalar: S, other: &Tensor) -> Tensor { - Tensor::f_special_xlogy_self_scalar(self_scalar, other).unwrap() - } - - pub fn special_xlogy_self_scalar_out>( - out: &Tensor, - self_scalar: S, - other: &Tensor, - ) -> Tensor { - Tensor::f_special_xlogy_self_scalar_out(out, self_scalar, other).unwrap() - } - - pub fn special_zeta(&self, other: &Tensor) -> Tensor { - self.f_special_zeta(other).unwrap() - } - - pub fn special_zeta_other_scalar>(&self, other: S) -> Tensor { - self.f_special_zeta_other_scalar(other).unwrap() - } - - pub fn special_zeta_other_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_special_zeta_other_scalar_out(out, other).unwrap() - } - - pub fn special_zeta_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_special_zeta_out(out, other).unwrap() - } - - pub fn special_zeta_self_scalar>(self_scalar: S, other: &Tensor) -> Tensor { - Tensor::f_special_zeta_self_scalar(self_scalar, other).unwrap() - } - - pub fn special_zeta_self_scalar_out>( - out: &Tensor, - self_scalar: S, - other: &Tensor, - ) -> Tensor { - Tensor::f_special_zeta_self_scalar_out(out, self_scalar, other).unwrap() - } - - pub fn split(&self, split_size: i64, dim: i64) -> Vec { - self.f_split(split_size, dim).unwrap() - } - - pub fn split_copy(&self, split_size: i64, dim: i64) -> Vec { - self.f_split_copy(split_size, dim).unwrap() - } - - pub fn split_copy_tensor_out>(&self, out: &[T], split_size: i64, dim: i64) { - self.f_split_copy_tensor_out(out, split_size, dim).unwrap() - } - - pub fn split_sizes(&self, split_size: impl IntList, dim: i64) -> Vec { - self.f_split_sizes(split_size, dim).unwrap() - } - - pub fn split_with_sizes(&self, split_sizes: impl IntList, dim: i64) -> Vec { - self.f_split_with_sizes(split_sizes, dim).unwrap() - } - - pub fn split_with_sizes_copy(&self, split_sizes: impl IntList, dim: i64) -> Vec { - self.f_split_with_sizes_copy(split_sizes, dim).unwrap() - } - - pub fn split_with_sizes_copy_out>( - &self, - out: &[T], - split_sizes: impl IntList, - dim: i64, - ) { - self.f_split_with_sizes_copy_out(out, split_sizes, dim).unwrap() - } - - pub fn sqrt(&self) -> Tensor { - self.f_sqrt().unwrap() - } - - pub fn sqrt_(&mut self) -> Tensor { - self.f_sqrt_().unwrap() - } - - pub fn sqrt_out(&self, out: &Tensor) -> Tensor { - self.f_sqrt_out(out).unwrap() - } - - pub fn square(&self) -> Tensor { - self.f_square().unwrap() - } - - pub fn square_(&mut self) -> Tensor { - self.f_square_().unwrap() - } - - pub fn square_out(&self, out: &Tensor) -> Tensor { - self.f_square_out(out).unwrap() - } - - pub fn squeeze(&self) -> Tensor { - self.f_squeeze().unwrap() - } - - pub fn squeeze_(&mut self) -> Tensor { - self.f_squeeze_().unwrap() - } - - pub fn squeeze_copy(&self) -> Tensor { - self.f_squeeze_copy().unwrap() - } - - pub fn squeeze_copy_dim(&self, dim: i64) -> Tensor { - self.f_squeeze_copy_dim(dim).unwrap() - } - - pub fn squeeze_copy_dim_out(&self, out: &Tensor, dim: i64) -> Tensor { - self.f_squeeze_copy_dim_out(out, dim).unwrap() - } - - pub fn squeeze_copy_dims(&self, dim: impl IntList) -> Tensor { - self.f_squeeze_copy_dims(dim).unwrap() - } - - pub fn squeeze_copy_dims_out(&self, out: &Tensor, dim: impl IntList) -> Tensor { - self.f_squeeze_copy_dims_out(out, dim).unwrap() - } - - pub fn squeeze_copy_out(&self, out: &Tensor) -> Tensor { - self.f_squeeze_copy_out(out).unwrap() - } - - pub fn squeeze_dim(&self, dim: i64) -> Tensor { - self.f_squeeze_dim(dim).unwrap() - } - - pub fn squeeze_dim_(&mut self, dim: i64) -> Tensor { - self.f_squeeze_dim_(dim).unwrap() - } - - pub fn squeeze_dims(&self, dim: impl IntList) -> Tensor { - self.f_squeeze_dims(dim).unwrap() - } - - pub fn squeeze_dims_(&mut self, dim: impl IntList) -> Tensor { - self.f_squeeze_dims_(dim).unwrap() - } - - pub fn sspaddmm(&self, mat1: &Tensor, mat2: &Tensor) -> Tensor { - self.f_sspaddmm(mat1, mat2).unwrap() - } - - pub fn sspaddmm_out(&self, out: &Tensor, mat1: &Tensor, mat2: &Tensor) -> Tensor { - self.f_sspaddmm_out(out, mat1, mat2).unwrap() - } - - pub fn stack>(tensors: &[T], dim: i64) -> Tensor { - Tensor::f_stack(tensors, dim).unwrap() - } - - pub fn stack_out>(out: &Tensor, tensors: &[T], dim: i64) -> Tensor { - Tensor::f_stack_out(out, tensors, dim).unwrap() - } - - pub fn std(&self, unbiased: bool) -> Tensor { - self.f_std(unbiased).unwrap() - } - - pub fn std_correction>( - &self, - dim: impl IntListOption, - correction: S, - keepdim: bool, - ) -> Tensor { - self.f_std_correction(dim, correction, keepdim).unwrap() - } - - pub fn std_correction_out>( - &self, - out: &Tensor, - dim: impl IntListOption, - correction: S, - keepdim: bool, - ) -> Tensor { - self.f_std_correction_out(out, dim, correction, keepdim).unwrap() - } - - pub fn std_dim(&self, dim: impl IntListOption, unbiased: bool, keepdim: bool) -> Tensor { - self.f_std_dim(dim, unbiased, keepdim).unwrap() - } - - pub fn std_mean(&self, unbiased: bool) -> (Tensor, Tensor) { - self.f_std_mean(unbiased).unwrap() - } - - pub fn std_mean_correction>( - &self, - dim: impl IntListOption, - correction: S, - keepdim: bool, - ) -> (Tensor, Tensor) { - self.f_std_mean_correction(dim, correction, keepdim).unwrap() - } - - pub fn std_mean_correction_out>( - &self, - out0: &Tensor, - out1: &Tensor, - dim: impl IntListOption, - correction: S, - keepdim: bool, - ) -> (Tensor, Tensor) { - self.f_std_mean_correction_out(out0, out1, dim, correction, keepdim).unwrap() - } - - pub fn std_mean_dim( - &self, - dim: impl IntListOption, - unbiased: bool, - keepdim: bool, - ) -> (Tensor, Tensor) { - self.f_std_mean_dim(dim, unbiased, keepdim).unwrap() - } - - pub fn std_out( - &self, - out: &Tensor, - dim: impl IntListOption, - unbiased: bool, - keepdim: bool, - ) -> Tensor { - self.f_std_out(out, dim, unbiased, keepdim).unwrap() - } - - pub fn stft>( - &self, - n_fft: i64, - hop_length: impl Into>, - win_length: impl Into>, - window: Option, - normalized: bool, - onesided: bool, - return_complex: bool, - align_to_window: bool, - ) -> Tensor { - self.f_stft( - n_fft, - hop_length, - win_length, - window, - normalized, - onesided, - return_complex, - align_to_window, - ) - .unwrap() - } - - pub fn stft_center>( - &self, - n_fft: i64, - hop_length: impl Into>, - win_length: impl Into>, - window: Option, - center: bool, - pad_mode: &str, - normalized: bool, - onesided: bool, - return_complex: bool, - align_to_window: bool, - ) -> Tensor { - self.f_stft_center( - n_fft, - hop_length, - win_length, - window, - center, - pad_mode, - normalized, - onesided, - return_complex, - align_to_window, - ) - .unwrap() - } - - pub fn g_sub(&self, other: &Tensor) -> Tensor { - self.f_sub(other).unwrap() - } - - pub fn g_sub_(&mut self, other: &Tensor) -> Tensor { - self.f_sub_(other).unwrap() - } - - pub fn sub_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_sub_out(out, other).unwrap() - } - - pub fn g_sub_scalar>(&self, other: S) -> Tensor { - self.f_sub_scalar(other).unwrap() - } - - pub fn g_sub_scalar_>(&mut self, other: S) -> Tensor { - self.f_sub_scalar_(other).unwrap() - } - - pub fn sub_scalar_out>(&self, out: &Tensor, other: S) -> Tensor { - self.f_sub_scalar_out(out, other).unwrap() - } - - pub fn subtract(&self, other: &Tensor) -> Tensor { - self.f_subtract(other).unwrap() - } - - pub fn subtract_(&mut self, other: &Tensor) -> Tensor { - self.f_subtract_(other).unwrap() - } - - pub fn subtract_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_subtract_out(out, other).unwrap() - } - - pub fn subtract_scalar>(&self, other: S) -> Tensor { - self.f_subtract_scalar(other).unwrap() - } - - pub fn subtract_scalar_>(&mut self, other: S) -> Tensor { - self.f_subtract_scalar_(other).unwrap() - } - - pub fn sum(&self, dtype: impl Into>) -> Tensor { - self.f_sum(dtype).unwrap() - } - - pub fn sum_dim_intlist( - &self, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Tensor { - self.f_sum_dim_intlist(dim, keepdim, dtype).unwrap() - } - - pub fn sum_intlist_out( - &self, - out: &Tensor, - dim: impl IntListOption, - keepdim: bool, - dtype: impl Into>, - ) -> Tensor { - self.f_sum_intlist_out(out, dim, keepdim, dtype).unwrap() - } - - pub fn sum_out(&self, out: &Tensor, dtype: impl Into>) -> Tensor { - self.f_sum_out(out, dtype).unwrap() - } - - pub fn sum_to_size(&self, size: impl IntList) -> Tensor { - self.f_sum_to_size(size).unwrap() - } - - pub fn svd(&self, some: bool, compute_uv: bool) -> (Tensor, Tensor, Tensor) { - self.f_svd(some, compute_uv).unwrap() - } - - pub fn svd_u( - &self, - u: &Tensor, - s: &Tensor, - v: &Tensor, - some: bool, - compute_uv: bool, - ) -> (Tensor, Tensor, Tensor) { - self.f_svd_u(u, s, v, some, compute_uv).unwrap() - } - - pub fn swapaxes(&self, axis0: i64, axis1: i64) -> Tensor { - self.f_swapaxes(axis0, axis1).unwrap() - } - - pub fn swapaxes_(&mut self, axis0: i64, axis1: i64) -> Tensor { - self.f_swapaxes_(axis0, axis1).unwrap() - } - - pub fn swapdims(&self, dim0: i64, dim1: i64) -> Tensor { - self.f_swapdims(dim0, dim1).unwrap() - } - - pub fn swapdims_(&mut self, dim0: i64, dim1: i64) -> Tensor { - self.f_swapdims_(dim0, dim1).unwrap() - } - - pub fn tr(&self) -> Tensor { - self.f_tr().unwrap() - } - - pub fn t_(&mut self) -> Tensor { - self.f_t_().unwrap() - } - - pub fn t_copy(&self) -> Tensor { - self.f_t_copy().unwrap() - } - - pub fn t_copy_out(&self, out: &Tensor) -> Tensor { - self.f_t_copy_out(out).unwrap() - } - - pub fn take(&self, index: &Tensor) -> Tensor { - self.f_take(index).unwrap() - } - - pub fn take_along_dim(&self, indices: &Tensor, dim: impl Into>) -> Tensor { - self.f_take_along_dim(indices, dim).unwrap() - } - - pub fn take_along_dim_out( - &self, - out: &Tensor, - indices: &Tensor, - dim: impl Into>, - ) -> Tensor { - self.f_take_along_dim_out(out, indices, dim).unwrap() - } - - pub fn take_out(&self, out: &Tensor, index: &Tensor) -> Tensor { - self.f_take_out(out, index).unwrap() - } - - pub fn tan(&self) -> Tensor { - self.f_tan().unwrap() - } - - pub fn tan_(&mut self) -> Tensor { - self.f_tan_().unwrap() - } - - pub fn tan_out(&self, out: &Tensor) -> Tensor { - self.f_tan_out(out).unwrap() - } - - pub fn tanh(&self) -> Tensor { - self.f_tanh().unwrap() - } - - pub fn tanh_(&mut self) -> Tensor { - self.f_tanh_().unwrap() - } - - pub fn tanh_backward(grad_output: &Tensor, output: &Tensor) -> Tensor { - Tensor::f_tanh_backward(grad_output, output).unwrap() - } - - pub fn tanh_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output: &Tensor, - ) -> Tensor { - Tensor::f_tanh_backward_grad_input(grad_input, grad_output, output).unwrap() - } - - pub fn tanh_out(&self, out: &Tensor) -> Tensor { - self.f_tanh_out(out).unwrap() - } - - pub fn tensor_split(&self, sections: i64, dim: i64) -> Vec { - self.f_tensor_split(sections, dim).unwrap() - } - - pub fn tensor_split_indices(&self, indices: impl IntList, dim: i64) -> Vec { - self.f_tensor_split_indices(indices, dim).unwrap() - } - - pub fn tensor_split_tensor_indices_or_sections( - &self, - tensor_indices_or_sections: &Tensor, - dim: i64, - ) -> Vec { - self.f_tensor_split_tensor_indices_or_sections(tensor_indices_or_sections, dim).unwrap() - } - - pub fn tensordot( - &self, - other: &Tensor, - dims_self: impl IntList, - dims_other: impl IntList, - ) -> Tensor { - self.f_tensordot(other, dims_self, dims_other).unwrap() - } - - pub fn tensordot_out( - &self, - out: &Tensor, - other: &Tensor, - dims_self: impl IntList, - dims_other: impl IntList, - ) -> Tensor { - self.f_tensordot_out(out, other, dims_self, dims_other).unwrap() - } - - pub fn threshold>(&self, threshold: S, value: S) -> Tensor { - self.f_threshold(threshold, value).unwrap() - } - - pub fn threshold_>(&mut self, threshold: S, value: S) -> Tensor { - self.f_threshold_(threshold, value).unwrap() - } - - pub fn threshold_backward>( - &self, - grad_output: &Tensor, - threshold: S, - ) -> Tensor { - self.f_threshold_backward(grad_output, threshold).unwrap() - } - - pub fn threshold_backward_grad_input>( - &self, - grad_input: &Tensor, - grad_output: &Tensor, - threshold: S, - ) -> Tensor { - self.f_threshold_backward_grad_input(grad_input, grad_output, threshold).unwrap() - } - - pub fn threshold_out>(&self, out: &Tensor, threshold: S, value: S) -> Tensor { - self.f_threshold_out(out, threshold, value).unwrap() - } - - pub fn tile(&self, dims: impl IntList) -> Tensor { - self.f_tile(dims).unwrap() - } - - pub fn to(&self, device: Device) -> Tensor { - self.f_to(device).unwrap() - } - - pub fn to_dense(&self, dtype: impl Into>, masked_grad: bool) -> Tensor { - self.f_to_dense(dtype, masked_grad).unwrap() - } - - pub fn to_dense_backward(&self, grad: &Tensor, masked_grad: bool) -> Tensor { - self.f_to_dense_backward(grad, masked_grad).unwrap() - } - - pub fn to_device_( - &self, - device: Device, - dtype: Kind, - non_blocking: bool, - copy: bool, - ) -> Tensor { - self.f_to_device_(device, dtype, non_blocking, copy).unwrap() - } - - pub fn to_dtype(&self, dtype: Kind, non_blocking: bool, copy: bool) -> Tensor { - self.f_to_dtype(dtype, non_blocking, copy).unwrap() - } - - pub fn to_dtype_layout( - &self, - options: (Kind, Device), - non_blocking: bool, - copy: bool, - ) -> Tensor { - self.f_to_dtype_layout(options, non_blocking, copy).unwrap() - } - - pub fn g_to_mkldnn(&self, dtype: impl Into>) -> Tensor { - self.f_to_mkldnn(dtype).unwrap() - } - - pub fn to_mkldnn_backward(&self, grad: &Tensor) -> Tensor { - self.f_to_mkldnn_backward(grad).unwrap() - } - - pub fn to_mkldnn_out(&self, out: &Tensor, dtype: impl Into>) -> Tensor { - self.f_to_mkldnn_out(out, dtype).unwrap() - } - - pub fn to_other(&self, other: &Tensor, non_blocking: bool, copy: bool) -> Tensor { - self.f_to_other(other, non_blocking, copy).unwrap() - } - - pub fn to_padded_tensor(&self, padding: f64, output_size: impl IntListOption) -> Tensor { - self.f_to_padded_tensor(padding, output_size).unwrap() - } - - pub fn to_padded_tensor_out( - &self, - out: &Tensor, - padding: f64, - output_size: impl IntListOption, - ) -> Tensor { - self.f_to_padded_tensor_out(out, padding, output_size).unwrap() - } - - pub fn to_sparse( - &self, - layout: Option, - blocksize: impl IntListOption, - dense_dim: impl Into>, - ) -> Tensor { - self.f_to_sparse(layout, blocksize, dense_dim).unwrap() - } - - pub fn to_sparse_bsc( - &self, - blocksize: impl IntList, - dense_dim: impl Into>, - ) -> Tensor { - self.f_to_sparse_bsc(blocksize, dense_dim).unwrap() - } - - pub fn to_sparse_bsr( - &self, - blocksize: impl IntList, - dense_dim: impl Into>, - ) -> Tensor { - self.f_to_sparse_bsr(blocksize, dense_dim).unwrap() - } - - pub fn to_sparse_csc(&self, dense_dim: impl Into>) -> Tensor { - self.f_to_sparse_csc(dense_dim).unwrap() - } - - pub fn to_sparse_csr(&self, dense_dim: impl Into>) -> Tensor { - self.f_to_sparse_csr(dense_dim).unwrap() - } - - pub fn to_sparse_sparse_dim(&self, sparse_dim: i64) -> Tensor { - self.f_to_sparse_sparse_dim(sparse_dim).unwrap() - } - - pub fn topk(&self, k: i64, dim: i64, largest: bool, sorted: bool) -> (Tensor, Tensor) { - self.f_topk(k, dim, largest, sorted).unwrap() - } - - pub fn topk_values( - &self, - values: &Tensor, - indices: &Tensor, - k: i64, - dim: i64, - largest: bool, - sorted: bool, - ) -> (Tensor, Tensor) { - self.f_topk_values(values, indices, k, dim, largest, sorted).unwrap() - } - - pub fn totype(&self, scalar_type: Kind) -> Tensor { - self.f_totype(scalar_type).unwrap() - } - - pub fn trace(&self) -> Tensor { - self.f_trace().unwrap() - } - - pub fn trace_backward(grad: &Tensor, sizes: impl IntList) -> Tensor { - Tensor::f_trace_backward(grad, sizes).unwrap() - } - - pub fn trace_out(&self, out: &Tensor) -> Tensor { - self.f_trace_out(out).unwrap() - } - - pub fn transpose(&self, dim0: i64, dim1: i64) -> Tensor { - self.f_transpose(dim0, dim1).unwrap() - } - - pub fn transpose_(&mut self, dim0: i64, dim1: i64) -> Tensor { - self.f_transpose_(dim0, dim1).unwrap() - } - - pub fn transpose_copy(&self, dim0: i64, dim1: i64) -> Tensor { - self.f_transpose_copy(dim0, dim1).unwrap() - } - - pub fn transpose_copy_int_out(&self, out: &Tensor, dim0: i64, dim1: i64) -> Tensor { - self.f_transpose_copy_int_out(out, dim0, dim1).unwrap() - } - - pub fn trapezoid(y: &Tensor, dim: i64) -> Tensor { - Tensor::f_trapezoid(y, dim).unwrap() - } - - pub fn trapezoid_x(y: &Tensor, x: &Tensor, dim: i64) -> Tensor { - Tensor::f_trapezoid_x(y, x, dim).unwrap() - } - - pub fn trapz(y: &Tensor, x: &Tensor, dim: i64) -> Tensor { - Tensor::f_trapz(y, x, dim).unwrap() - } - - pub fn trapz_dx(y: &Tensor, dx: f64, dim: i64) -> Tensor { - Tensor::f_trapz_dx(y, dx, dim).unwrap() - } - - pub fn triangular_solve( - &self, - a: &Tensor, - upper: bool, - transpose: bool, - unitriangular: bool, - ) -> (Tensor, Tensor) { - self.f_triangular_solve(a, upper, transpose, unitriangular).unwrap() - } - - pub fn triangular_solve_x( - &self, - x: &Tensor, - m: &Tensor, - a: &Tensor, - upper: bool, - transpose: bool, - unitriangular: bool, - ) -> (Tensor, Tensor) { - self.f_triangular_solve_x(x, m, a, upper, transpose, unitriangular).unwrap() - } - - pub fn tril(&self, diagonal: i64) -> Tensor { - self.f_tril(diagonal).unwrap() - } - - pub fn tril_(&mut self, diagonal: i64) -> Tensor { - self.f_tril_(diagonal).unwrap() - } - - pub fn tril_indices(row: i64, col: i64, offset: i64, options: (Kind, Device)) -> Tensor { - Tensor::f_tril_indices(row, col, offset, options).unwrap() - } - - pub fn tril_indices_out(out: &Tensor, row: i64, col: i64, offset: i64) -> Tensor { - Tensor::f_tril_indices_out(out, row, col, offset).unwrap() - } - - pub fn tril_out(&self, out: &Tensor, diagonal: i64) -> Tensor { - self.f_tril_out(out, diagonal).unwrap() - } - - pub fn triplet_margin_loss( - anchor: &Tensor, - positive: &Tensor, - negative: &Tensor, - margin: f64, - p: f64, - eps: f64, - swap: bool, - reduction: crate::Reduction, - ) -> Tensor { - Tensor::f_triplet_margin_loss(anchor, positive, negative, margin, p, eps, swap, reduction) - .unwrap() - } - - pub fn triu(&self, diagonal: i64) -> Tensor { - self.f_triu(diagonal).unwrap() - } - - pub fn triu_(&mut self, diagonal: i64) -> Tensor { - self.f_triu_(diagonal).unwrap() - } - - pub fn triu_indices(row: i64, col: i64, offset: i64, options: (Kind, Device)) -> Tensor { - Tensor::f_triu_indices(row, col, offset, options).unwrap() - } - - pub fn triu_indices_out(out: &Tensor, row: i64, col: i64, offset: i64) -> Tensor { - Tensor::f_triu_indices_out(out, row, col, offset).unwrap() - } - - pub fn triu_out(&self, out: &Tensor, diagonal: i64) -> Tensor { - self.f_triu_out(out, diagonal).unwrap() - } - - pub fn true_divide(&self, other: &Tensor) -> Tensor { - self.f_true_divide(other).unwrap() - } - - pub fn true_divide_(&mut self, other: &Tensor) -> Tensor { - self.f_true_divide_(other).unwrap() - } - - pub fn true_divide_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_true_divide_out(out, other).unwrap() - } - - pub fn true_divide_scalar>(&self, other: S) -> Tensor { - self.f_true_divide_scalar(other).unwrap() - } - - pub fn true_divide_scalar_>(&mut self, other: S) -> Tensor { - self.f_true_divide_scalar_(other).unwrap() - } - - pub fn trunc(&self) -> Tensor { - self.f_trunc().unwrap() - } - - pub fn trunc_(&mut self) -> Tensor { - self.f_trunc_().unwrap() - } - - pub fn trunc_out(&self, out: &Tensor) -> Tensor { - self.f_trunc_out(out).unwrap() - } - - pub fn type_as(&self, other: &Tensor) -> Tensor { - self.f_type_as(other).unwrap() - } - - pub fn unbind(&self, dim: i64) -> Vec { - self.f_unbind(dim).unwrap() - } - - pub fn unbind_copy(&self, dim: i64) -> Vec { - self.f_unbind_copy(dim).unwrap() - } - - pub fn unbind_copy_int_out>(&self, out: &[T], dim: i64) { - self.f_unbind_copy_int_out(out, dim).unwrap() - } - - pub fn unflatten(&self, dim: i64, sizes: impl IntList) -> Tensor { - self.f_unflatten(dim, sizes).unwrap() - } - - pub fn unflatten_dense_tensors>(flat: &Tensor, tensors: &[T]) -> Vec { - Tensor::f_unflatten_dense_tensors(flat, tensors).unwrap() - } - - pub fn unfold(&self, dimension: i64, size: i64, step: i64) -> Tensor { - self.f_unfold(dimension, size, step).unwrap() - } - - pub fn unfold_backward( - grad_in: &Tensor, - input_sizes: impl IntList, - dim: i64, - size: i64, - step: i64, - ) -> Tensor { - Tensor::f_unfold_backward(grad_in, input_sizes, dim, size, step).unwrap() - } - - pub fn unfold_backward_out( - out: &Tensor, - grad_in: &Tensor, - input_sizes: impl IntList, - dim: i64, - size: i64, - step: i64, - ) -> Tensor { - Tensor::f_unfold_backward_out(out, grad_in, input_sizes, dim, size, step).unwrap() - } - - pub fn unfold_copy(&self, dimension: i64, size: i64, step: i64) -> Tensor { - self.f_unfold_copy(dimension, size, step).unwrap() - } - - pub fn unfold_copy_out(&self, out: &Tensor, dimension: i64, size: i64, step: i64) -> Tensor { - self.f_unfold_copy_out(out, dimension, size, step).unwrap() - } - - pub fn uniform(&self, from: f64, to: f64) -> Tensor { - self.f_uniform(from, to).unwrap() - } - - pub fn uniform_(&mut self, from: f64, to: f64) -> Tensor { - self.f_uniform_(from, to).unwrap() - } - - pub fn uniform_out(&self, out: &Tensor, from: f64, to: f64) -> Tensor { - self.f_uniform_out(out, from, to).unwrap() - } - - pub fn unique_consecutive( - &self, - return_inverse: bool, - return_counts: bool, - dim: impl Into>, - ) -> (Tensor, Tensor, Tensor) { - self.f_unique_consecutive(return_inverse, return_counts, dim).unwrap() - } - - pub fn unique_consecutive_out( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - return_inverse: bool, - return_counts: bool, - dim: impl Into>, - ) -> (Tensor, Tensor, Tensor) { - self.f_unique_consecutive_out(out0, out1, out2, return_inverse, return_counts, dim).unwrap() - } - - pub fn unique_dim( - &self, - dim: i64, - sorted: bool, - return_inverse: bool, - return_counts: bool, - ) -> (Tensor, Tensor, Tensor) { - self.f_unique_dim(dim, sorted, return_inverse, return_counts).unwrap() - } - - pub fn unique_dim_consecutive( - &self, - dim: i64, - return_inverse: bool, - return_counts: bool, - ) -> (Tensor, Tensor, Tensor) { - self.f_unique_dim_consecutive(dim, return_inverse, return_counts).unwrap() - } - - pub fn unique_dim_consecutive_out( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - dim: i64, - return_inverse: bool, - return_counts: bool, - ) -> (Tensor, Tensor, Tensor) { - self.f_unique_dim_consecutive_out(out0, out1, out2, dim, return_inverse, return_counts) - .unwrap() - } - - pub fn unique_dim_out( - &self, - out0: &Tensor, - out1: &Tensor, - out2: &Tensor, - dim: i64, - sorted: bool, - return_inverse: bool, - return_counts: bool, - ) -> (Tensor, Tensor, Tensor) { - self.f_unique_dim_out(out0, out1, out2, dim, sorted, return_inverse, return_counts).unwrap() - } - - pub fn unsafe_chunk(&self, chunks: i64, dim: i64) -> Vec { - self.f_unsafe_chunk(chunks, dim).unwrap() - } - - pub fn unsafe_split(&self, split_size: i64, dim: i64) -> Vec { - self.f_unsafe_split(split_size, dim).unwrap() - } - - pub fn unsafe_split_tensor_out>(&self, out: &[T], split_size: i64, dim: i64) { - self.f_unsafe_split_tensor_out(out, split_size, dim).unwrap() - } - - pub fn unsafe_split_with_sizes(&self, split_sizes: impl IntList, dim: i64) -> Vec { - self.f_unsafe_split_with_sizes(split_sizes, dim).unwrap() - } - - pub fn unsafe_split_with_sizes_out>( - &self, - out: &[T], - split_sizes: impl IntList, - dim: i64, - ) { - self.f_unsafe_split_with_sizes_out(out, split_sizes, dim).unwrap() - } - - pub fn unsqueeze(&self, dim: i64) -> Tensor { - self.f_unsqueeze(dim).unwrap() - } - - pub fn unsqueeze_(&mut self, dim: i64) -> Tensor { - self.f_unsqueeze_(dim).unwrap() - } - - pub fn unsqueeze_copy(&self, dim: i64) -> Tensor { - self.f_unsqueeze_copy(dim).unwrap() - } - - pub fn unsqueeze_copy_out(&self, out: &Tensor, dim: i64) -> Tensor { - self.f_unsqueeze_copy_out(out, dim).unwrap() - } - - pub fn upsample_bicubic2d( - &self, - output_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - self.f_upsample_bicubic2d(output_size, align_corners, scales_h, scales_w).unwrap() - } - - pub fn upsample_bicubic2d_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - Tensor::f_upsample_bicubic2d_backward( - grad_output, - output_size, - input_size, - align_corners, - scales_h, - scales_w, - ) - .unwrap() - } - - pub fn upsample_bicubic2d_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - Tensor::f_upsample_bicubic2d_backward_grad_input( - grad_input, - grad_output, - output_size, - input_size, - align_corners, - scales_h, - scales_w, - ) - .unwrap() - } - - pub fn upsample_bicubic2d_out( - &self, - out: &Tensor, - output_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - self.f_upsample_bicubic2d_out(out, output_size, align_corners, scales_h, scales_w).unwrap() - } - - pub fn upsample_bicubic2d_vec( - &self, - output_size: impl IntListOption, - align_corners: bool, - scale_factors: impl DoubleList, - ) -> Tensor { - self.f_upsample_bicubic2d_vec(output_size, align_corners, scale_factors).unwrap() - } - - pub fn upsample_bilinear2d( - &self, - output_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - self.f_upsample_bilinear2d(output_size, align_corners, scales_h, scales_w).unwrap() - } - - pub fn upsample_bilinear2d_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - Tensor::f_upsample_bilinear2d_backward( - grad_output, - output_size, - input_size, - align_corners, - scales_h, - scales_w, - ) - .unwrap() - } - - pub fn upsample_bilinear2d_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - Tensor::f_upsample_bilinear2d_backward_grad_input( - grad_input, - grad_output, - output_size, - input_size, - align_corners, - scales_h, - scales_w, - ) - .unwrap() - } - - pub fn upsample_bilinear2d_out( - &self, - out: &Tensor, - output_size: impl IntList, - align_corners: bool, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - self.f_upsample_bilinear2d_out(out, output_size, align_corners, scales_h, scales_w).unwrap() - } - - pub fn upsample_bilinear2d_vec( - &self, - output_size: impl IntListOption, - align_corners: bool, - scale_factors: impl DoubleList, - ) -> Tensor { - self.f_upsample_bilinear2d_vec(output_size, align_corners, scale_factors).unwrap() - } - - pub fn upsample_bilinear2d_vec_out( - &self, - out: &Tensor, - output_size: impl IntListOption, - align_corners: bool, - scale_factors: impl DoubleList, - ) -> Tensor { - self.f_upsample_bilinear2d_vec_out(out, output_size, align_corners, scale_factors).unwrap() - } - - pub fn upsample_linear1d( - &self, - output_size: impl IntList, - align_corners: bool, - scales: impl Into>, - ) -> Tensor { - self.f_upsample_linear1d(output_size, align_corners, scales).unwrap() - } - - pub fn upsample_linear1d_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales: impl Into>, - ) -> Tensor { - Tensor::f_upsample_linear1d_backward( - grad_output, - output_size, - input_size, - align_corners, - scales, - ) - .unwrap() - } - - pub fn upsample_linear1d_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales: impl Into>, - ) -> Tensor { - Tensor::f_upsample_linear1d_backward_grad_input( - grad_input, - grad_output, - output_size, - input_size, - align_corners, - scales, - ) - .unwrap() - } - - pub fn upsample_linear1d_out( - &self, - out: &Tensor, - output_size: impl IntList, - align_corners: bool, - scales: impl Into>, - ) -> Tensor { - self.f_upsample_linear1d_out(out, output_size, align_corners, scales).unwrap() - } - - pub fn upsample_linear1d_vec( - &self, - output_size: impl IntListOption, - align_corners: bool, - scale_factors: impl DoubleList, - ) -> Tensor { - self.f_upsample_linear1d_vec(output_size, align_corners, scale_factors).unwrap() - } - - pub fn upsample_nearest1d( - &self, - output_size: impl IntList, - scales: impl Into>, - ) -> Tensor { - self.f_upsample_nearest1d(output_size, scales).unwrap() - } - - pub fn upsample_nearest1d_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales: impl Into>, - ) -> Tensor { - Tensor::f_upsample_nearest1d_backward(grad_output, output_size, input_size, scales).unwrap() - } - - pub fn upsample_nearest1d_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales: impl Into>, - ) -> Tensor { - Tensor::f_upsample_nearest1d_backward_grad_input( - grad_input, - grad_output, - output_size, - input_size, - scales, - ) - .unwrap() - } - - pub fn upsample_nearest1d_out( - &self, - out: &Tensor, - output_size: impl IntList, - scales: impl Into>, - ) -> Tensor { - self.f_upsample_nearest1d_out(out, output_size, scales).unwrap() - } - - pub fn upsample_nearest1d_vec( - &self, - output_size: impl IntListOption, - scale_factors: impl DoubleList, - ) -> Tensor { - self.f_upsample_nearest1d_vec(output_size, scale_factors).unwrap() - } - - pub fn upsample_nearest2d( - &self, - output_size: impl IntList, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - self.f_upsample_nearest2d(output_size, scales_h, scales_w).unwrap() - } - - pub fn upsample_nearest2d_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - Tensor::f_upsample_nearest2d_backward( - grad_output, - output_size, - input_size, - scales_h, - scales_w, - ) - .unwrap() - } - - pub fn upsample_nearest2d_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - Tensor::f_upsample_nearest2d_backward_grad_input( - grad_input, - grad_output, - output_size, - input_size, - scales_h, - scales_w, - ) - .unwrap() - } - - pub fn upsample_nearest2d_out( - &self, - out: &Tensor, - output_size: impl IntList, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - self.f_upsample_nearest2d_out(out, output_size, scales_h, scales_w).unwrap() - } - - pub fn upsample_nearest2d_vec( - &self, - output_size: impl IntListOption, - scale_factors: impl DoubleList, - ) -> Tensor { - self.f_upsample_nearest2d_vec(output_size, scale_factors).unwrap() - } - - pub fn upsample_nearest2d_vec_out( - &self, - out: &Tensor, - output_size: impl IntListOption, - scale_factors: impl DoubleList, - ) -> Tensor { - self.f_upsample_nearest2d_vec_out(out, output_size, scale_factors).unwrap() - } - - pub fn upsample_nearest3d( - &self, - output_size: impl IntList, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - self.f_upsample_nearest3d(output_size, scales_d, scales_h, scales_w).unwrap() - } - - pub fn upsample_nearest3d_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - Tensor::f_upsample_nearest3d_backward( - grad_output, - output_size, - input_size, - scales_d, - scales_h, - scales_w, - ) - .unwrap() - } - - pub fn upsample_nearest3d_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - Tensor::f_upsample_nearest3d_backward_grad_input( - grad_input, - grad_output, - output_size, - input_size, - scales_d, - scales_h, - scales_w, - ) - .unwrap() - } - - pub fn upsample_nearest3d_out( - &self, - out: &Tensor, - output_size: impl IntList, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - self.f_upsample_nearest3d_out(out, output_size, scales_d, scales_h, scales_w).unwrap() - } - - pub fn upsample_nearest3d_vec( - &self, - output_size: impl IntListOption, - scale_factors: impl DoubleList, - ) -> Tensor { - self.f_upsample_nearest3d_vec(output_size, scale_factors).unwrap() - } - - pub fn upsample_trilinear3d( - &self, - output_size: impl IntList, - align_corners: bool, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - self.f_upsample_trilinear3d(output_size, align_corners, scales_d, scales_h, scales_w) - .unwrap() - } - - pub fn upsample_trilinear3d_backward( - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - Tensor::f_upsample_trilinear3d_backward( - grad_output, - output_size, - input_size, - align_corners, - scales_d, - scales_h, - scales_w, - ) - .unwrap() - } - - pub fn upsample_trilinear3d_backward_grad_input( - grad_input: &Tensor, - grad_output: &Tensor, - output_size: impl IntList, - input_size: impl IntList, - align_corners: bool, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - Tensor::f_upsample_trilinear3d_backward_grad_input( - grad_input, - grad_output, - output_size, - input_size, - align_corners, - scales_d, - scales_h, - scales_w, - ) - .unwrap() - } - - pub fn upsample_trilinear3d_out( - &self, - out: &Tensor, - output_size: impl IntList, - align_corners: bool, - scales_d: impl Into>, - scales_h: impl Into>, - scales_w: impl Into>, - ) -> Tensor { - self.f_upsample_trilinear3d_out( - out, - output_size, - align_corners, - scales_d, - scales_h, - scales_w, - ) - .unwrap() - } - - pub fn upsample_trilinear3d_vec( - &self, - output_size: impl IntListOption, - align_corners: bool, - scale_factors: impl DoubleList, - ) -> Tensor { - self.f_upsample_trilinear3d_vec(output_size, align_corners, scale_factors).unwrap() - } - - pub fn value_selecting_reduction_backward( - grad: &Tensor, - dim: i64, - indices: &Tensor, - sizes: impl IntList, - keepdim: bool, - ) -> Tensor { - Tensor::f_value_selecting_reduction_backward(grad, dim, indices, sizes, keepdim).unwrap() - } - - pub fn values(&self) -> Tensor { - self.f_values().unwrap() - } - - pub fn values_copy(&self) -> Tensor { - self.f_values_copy().unwrap() - } - - pub fn values_copy_out(&self, out: &Tensor) -> Tensor { - self.f_values_copy_out(out).unwrap() - } - - pub fn vander(x: &Tensor, n: impl Into>, increasing: bool) -> Tensor { - Tensor::f_vander(x, n, increasing).unwrap() - } - - pub fn var(&self, unbiased: bool) -> Tensor { - self.f_var(unbiased).unwrap() - } - - pub fn var_correction>( - &self, - dim: impl IntListOption, - correction: S, - keepdim: bool, - ) -> Tensor { - self.f_var_correction(dim, correction, keepdim).unwrap() - } - - pub fn var_correction_out>( - &self, - out: &Tensor, - dim: impl IntListOption, - correction: S, - keepdim: bool, - ) -> Tensor { - self.f_var_correction_out(out, dim, correction, keepdim).unwrap() - } - - pub fn var_dim(&self, dim: impl IntListOption, unbiased: bool, keepdim: bool) -> Tensor { - self.f_var_dim(dim, unbiased, keepdim).unwrap() - } - - pub fn var_mean(&self, unbiased: bool) -> (Tensor, Tensor) { - self.f_var_mean(unbiased).unwrap() - } - - pub fn var_mean_correction>( - &self, - dim: impl IntListOption, - correction: S, - keepdim: bool, - ) -> (Tensor, Tensor) { - self.f_var_mean_correction(dim, correction, keepdim).unwrap() - } - - pub fn var_mean_correction_out>( - &self, - out0: &Tensor, - out1: &Tensor, - dim: impl IntListOption, - correction: S, - keepdim: bool, - ) -> (Tensor, Tensor) { - self.f_var_mean_correction_out(out0, out1, dim, correction, keepdim).unwrap() - } - - pub fn var_mean_dim( - &self, - dim: impl IntListOption, - unbiased: bool, - keepdim: bool, - ) -> (Tensor, Tensor) { - self.f_var_mean_dim(dim, unbiased, keepdim).unwrap() - } - - pub fn var_out( - &self, - out: &Tensor, - dim: impl IntListOption, - unbiased: bool, - keepdim: bool, - ) -> Tensor { - self.f_var_out(out, dim, unbiased, keepdim).unwrap() - } - - pub fn vdot(&self, other: &Tensor) -> Tensor { - self.f_vdot(other).unwrap() - } - - pub fn vdot_out(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_vdot_out(out, other).unwrap() - } - - pub fn view_(&self, size: impl IntList) -> Tensor { - self.f_view_(size).unwrap() - } - - pub fn view_as(&self, other: &Tensor) -> Tensor { - self.f_view_as(other).unwrap() - } - - pub fn view_as_complex(&self) -> Tensor { - self.f_view_as_complex().unwrap() - } - - pub fn view_as_complex_copy(&self) -> Tensor { - self.f_view_as_complex_copy().unwrap() - } - - pub fn view_as_complex_copy_out(&self, out: &Tensor) -> Tensor { - self.f_view_as_complex_copy_out(out).unwrap() - } - - pub fn view_as_real(&self) -> Tensor { - self.f_view_as_real().unwrap() - } - - pub fn view_as_real_copy(&self) -> Tensor { - self.f_view_as_real_copy().unwrap() - } - - pub fn view_as_real_copy_out(&self, out: &Tensor) -> Tensor { - self.f_view_as_real_copy_out(out).unwrap() - } - - pub fn view_copy(&self, size: impl IntList) -> Tensor { - self.f_view_copy(size).unwrap() - } - - pub fn view_copy_dtype(&self, dtype: Kind) -> Tensor { - self.f_view_copy_dtype(dtype).unwrap() - } - - pub fn view_copy_dtype_out(&self, out: &Tensor, dtype: Kind) -> Tensor { - self.f_view_copy_dtype_out(out, dtype).unwrap() - } - - pub fn view_copy_out(&self, out: &Tensor, size: impl IntList) -> Tensor { - self.f_view_copy_out(out, size).unwrap() - } - - pub fn view_dtype(&self, dtype: Kind) -> Tensor { - self.f_view_dtype(dtype).unwrap() - } - - pub fn vsplit(&self, sections: i64) -> Vec { - self.f_vsplit(sections).unwrap() - } - - pub fn vsplit_array(&self, indices: impl IntList) -> Vec { - self.f_vsplit_array(indices).unwrap() - } - - pub fn vstack>(tensors: &[T]) -> Tensor { - Tensor::f_vstack(tensors).unwrap() - } - - pub fn vstack_out>(out: &Tensor, tensors: &[T]) -> Tensor { - Tensor::f_vstack_out(out, tensors).unwrap() - } - - pub fn where_(condition: &Tensor) -> Vec { - Tensor::f_where_(condition).unwrap() - } - - pub fn where_scalar>(condition: &Tensor, self_scalar: S, other: S) -> Tensor { - Tensor::f_where_scalar(condition, self_scalar, other).unwrap() - } - - pub fn where_scalarother>(&self, condition: &Tensor, other: S) -> Tensor { - self.f_where_scalarother(condition, other).unwrap() - } - - pub fn where_scalarself>( - condition: &Tensor, - self_scalar: S, - other: &Tensor, - ) -> Tensor { - Tensor::f_where_scalarself(condition, self_scalar, other).unwrap() - } - - pub fn where_self(&self, condition: &Tensor, other: &Tensor) -> Tensor { - self.f_where_self(condition, other).unwrap() - } - - pub fn where_self_out(&self, out: &Tensor, condition: &Tensor, other: &Tensor) -> Tensor { - self.f_where_self_out(out, condition, other).unwrap() - } - - pub fn xlogy(&self, other: &Tensor) -> Tensor { - self.f_xlogy(other).unwrap() - } - - pub fn xlogy_(&mut self, other: &Tensor) -> Tensor { - self.f_xlogy_(other).unwrap() - } - - pub fn xlogy_outscalar_other>(&self, out: &Tensor, other: S) -> Tensor { - self.f_xlogy_outscalar_other(out, other).unwrap() - } - - pub fn xlogy_outscalar_self>( - out: &Tensor, - self_scalar: S, - other: &Tensor, - ) -> Tensor { - Tensor::f_xlogy_outscalar_self(out, self_scalar, other).unwrap() - } - - pub fn xlogy_outtensor(&self, out: &Tensor, other: &Tensor) -> Tensor { - self.f_xlogy_outtensor(out, other).unwrap() - } - - pub fn xlogy_scalar_other>(&self, other: S) -> Tensor { - self.f_xlogy_scalar_other(other).unwrap() - } - - pub fn xlogy_scalar_other_>(&mut self, other: S) -> Tensor { - self.f_xlogy_scalar_other_(other).unwrap() - } - - pub fn xlogy_scalar_self>(self_scalar: S, other: &Tensor) -> Tensor { - Tensor::f_xlogy_scalar_self(self_scalar, other).unwrap() - } - - pub fn zero(&self) -> Tensor { - self.f_zero().unwrap() - } - - pub fn zero_(&mut self) -> Tensor { - self.f_zero_().unwrap() - } - - pub fn zero_out(&self, out: &Tensor) -> Tensor { - self.f_zero_out(out).unwrap() - } - - pub fn zeros(size: impl IntList, options: (Kind, Device)) -> Tensor { - Tensor::f_zeros(size, options).unwrap() - } - - pub fn zeros_like(&self) -> Tensor { - self.f_zeros_like().unwrap() - } - - pub fn zeros_like_out(&self, out: &Tensor) -> Tensor { - self.f_zeros_like_out(out).unwrap() - } - - pub fn zeros_out(out: &Tensor, size: impl IntList) -> Tensor { - Tensor::f_zeros_out(out, size).unwrap() - } -} diff --git a/v2/dependency-patches/tch-0.24.0/src/wrappers/utils.rs b/v2/dependency-patches/tch-0.24.0/src/wrappers/utils.rs deleted file mode 100644 index c96ea0cb2b..0000000000 --- a/v2/dependency-patches/tch-0.24.0/src/wrappers/utils.rs +++ /dev/null @@ -1,157 +0,0 @@ -use crate::TchError; -use libc::c_char; -use std::io; - -// This returns None on the null pointer. If not null, the pointer gets -// freed. -pub(super) unsafe fn ptr_to_string(ptr: *mut c_char) -> Option { - if !ptr.is_null() { - let str = std::ffi::CStr::from_ptr(ptr).to_string_lossy().into_owned(); - libc::free(ptr as *mut libc::c_void); - Some(str) - } else { - None - } -} - -pub(super) fn read_and_clean_error() -> Result<(), TchError> { - unsafe { - match ptr_to_string(torch_sys::get_and_reset_last_err()) { - None => Ok(()), - Some(c_error) => Err(TchError::Torch(c_error)), - } - } -} - -macro_rules! unsafe_torch { - ($e:expr) => {{ - let v = unsafe { $e }; - crate::wrappers::utils::read_and_clean_error().unwrap(); - v - }}; -} - -macro_rules! unsafe_torch_err { - ($e:expr) => {{ - let v = unsafe { $e }; - crate::wrappers::utils::read_and_clean_error()?; - v - }}; -} - -// Be cautious when using this function as the returned CString should be stored -// in a variable when using as_ptr. Otherwise dangling pointer issues are likely -// to happen. -pub(super) fn path_to_cstring>( - path: T, -) -> Result { - let path = path.as_ref(); - match path.to_str() { - Some(path) => Ok(std::ffi::CString::new(path)?), - None => Err(TchError::Io(io::Error::other(format!( - "path {path:?} cannot be converted to UTF-8" - )))), - } -} - -/// Sets the random seed used by torch. -pub fn manual_seed(seed: i64) { - unsafe_torch!(torch_sys::at_manual_seed(seed)) -} - -/// Get the number of threads used by torch for inter-op parallelism. -pub fn get_num_interop_threads() -> i32 { - unsafe_torch!(torch_sys::at_get_num_interop_threads()) -} - -/// Get the number of threads used by torch in parallel regions. -pub fn get_num_threads() -> i32 { - unsafe_torch!(torch_sys::at_get_num_threads()) -} - -/// Set the number of threads used by torch for inter-op parallelism. -pub fn set_num_interop_threads(n_threads: i32) { - unsafe_torch!(torch_sys::at_set_num_interop_threads(n_threads)) -} - -/// Set the number of threads used by torch in parallel regions. -pub fn set_num_threads(n_threads: i32) { - unsafe_torch!(torch_sys::at_set_num_threads(n_threads)) -} - -pub fn has_openmp() -> bool { - unsafe_torch!(torch_sys::at_context_has_openmp()) -} - -pub fn has_mkl() -> bool { - unsafe_torch!(torch_sys::at_context_has_mkl()) -} -pub fn has_lapack() -> bool { - unsafe_torch!(torch_sys::at_context_has_lapack()) -} -pub fn has_mkldnn() -> bool { - unsafe_torch!(torch_sys::at_context_has_mkldnn()) -} -pub fn has_magma() -> bool { - unsafe_torch!(torch_sys::at_context_has_magma()) -} -pub fn has_cuda() -> bool { - unsafe_torch!(torch_sys::at_context_has_cuda()) -} -pub fn has_cudart() -> bool { - unsafe_torch!(torch_sys::at_context_has_cudart()) -} -pub fn has_cusolver() -> bool { - unsafe_torch!(torch_sys::at_context_has_cusolver()) -} -pub fn has_hip() -> bool { - unsafe_torch!(torch_sys::at_context_has_hip()) -} -pub fn has_ipu() -> bool { - unsafe_torch!(torch_sys::at_context_has_ipu()) -} -pub fn has_xla() -> bool { - unsafe_torch!(torch_sys::at_context_has_xla()) -} -pub fn has_lazy() -> bool { - unsafe_torch!(torch_sys::at_context_has_lazy()) -} -pub fn has_mps() -> bool { - unsafe_torch!(torch_sys::at_context_has_mps()) -} -pub fn version_cudnn() -> i64 { - unsafe_torch!(torch_sys::at_context_version_cudnn()) -} -pub fn version_cudart() -> i64 { - unsafe_torch!(torch_sys::at_context_version_cudart()) -} - -/// Check whether the vulkan backend is available. None that this -/// backend is not included by default as of PyTorch 2.0.0. -/// https://pytorch.org/tutorials/prototype/vulkan_workflow.html#building-pytorch-with-vulkan-backend -pub fn has_vulkan() -> bool { - crate::Tensor::is_vulkan_available() -} - -/// Quantization engines -#[allow(clippy::upper_case_acronyms)] -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub enum QEngine { - NoQEngine, - FBGEMM, - QNNPACK, -} - -impl QEngine { - fn to_cint(self) -> i32 { - match self { - QEngine::NoQEngine => 0, - QEngine::FBGEMM => 1, - QEngine::QNNPACK => 2, - } - } - pub fn set(self) -> Result<(), TchError> { - unsafe_torch_err!(torch_sys::at_set_qengine(self.to_cint())); - Ok(()) - } -} diff --git a/v2/dependency-patches/tch-0.24.0/tests/autocast.rs b/v2/dependency-patches/tch-0.24.0/tests/autocast.rs deleted file mode 100644 index f0eb41808c..0000000000 --- a/v2/dependency-patches/tch-0.24.0/tests/autocast.rs +++ /dev/null @@ -1,22 +0,0 @@ -#[cfg(test)] -#[cfg(feature = "cuda-tests")] -mod tests { - use tch::{autocast, Device, Kind, Tensor}; - - #[test] - fn autocast_narrows_type() { - let device = Device::Cuda(0); - - let linear = Tensor::rand([10, 10], (Kind::Float, device)); - let input = Tensor::rand([10], (Kind::Float, device)); - - autocast(true, || { - let output1 = autocast(false, || linear.matmul(&input)); - assert_eq!(output1.kind(), Kind::Float); - let output2 = linear.matmul(&output1); - assert_eq!(output2.kind(), Kind::Half); - let output3 = autocast(false, || linear.matmul(&output1)); - assert_eq!(output3.kind(), Kind::Float); - }); - } -} diff --git a/v2/dependency-patches/tch-0.24.0/tests/create_jit_models.py b/v2/dependency-patches/tch-0.24.0/tests/create_jit_models.py deleted file mode 100644 index 376a76ae56..0000000000 --- a/v2/dependency-patches/tch-0.24.0/tests/create_jit_models.py +++ /dev/null @@ -1,119 +0,0 @@ -import torch -from typing import Dict, Tuple - -class Foo(torch.jit.ScriptModule): - def __init__(self, v): - super(Foo, self).__init__() - self.register_buffer('value', v) - - @torch.jit.script_method - def forward(self, x, y): - return 2 * x + y + self.value - -foo = Foo(torch.Tensor([42.0])) -foo.save('foo.pt') - -class Foo1(torch.jit.ScriptModule): - def __init__(self): - super(Foo1, self).__init__() - - @torch.jit.script_method - def forward(self, x, y): - return 2 * x + y - -foo = Foo1() -script_foo = torch.jit.script(foo) -script_foo.save('foo1.pt') - -class Foo2(torch.jit.ScriptModule): - def __init__(self): - super(Foo2, self).__init__() - - @torch.jit.script_method - def forward(self, x, y): - return (2 * x + y, x - y) - -foo = Foo2() -traced_foo = torch.jit.trace(foo, (torch.rand(3), torch.rand(3))) -traced_foo.save('foo2.pt') - -class Foo3(torch.jit.ScriptModule): - def __init__(self): - super(Foo3, self).__init__() - - @torch.jit.script_method - def forward(self, x): - result = x[0] - for i in range(x.size(0)): - if i: result = result * x[i] - return result - -foo = Foo3() -foo.save('foo3.pt') - -from typing import Tuple, List - -class Foo4(torch.jit.ScriptModule): - def __init__(self): - super(Foo4, self).__init__() - - @torch.jit.script_method - def forward(self, x: Tuple[float, float, int]): - return x[0] + x[1] * x[2] - -foo = Foo4() -foo.save('foo4.pt') - -class Foo5(torch.jit.ScriptModule): - def __init__(self): - super(Foo5, self).__init__() - - @torch.jit.script_method - def forward(self, xs: List[str]): - return [x[:-1] for x in xs] - -foo = Foo5() -foo.save('foo5.pt') - -@torch.jit.script -class TorchScriptClass: - def __init__(self, x: torch.Tensor): - self.x = x - - def y(self) -> torch.Tensor: - return self.x * 2 - -@torch.jit.script -def foo_6(x: torch.Tensor): - return TorchScriptClass(x) - - -foo_6.save("foo6.pt") - -# https://github.com/LaurentMazare/tch-rs/issues/475 -@torch.jit.script -class InputObject: - def __init__(self, foo, bar): - self.foo = foo - self.bar = bar - -class TorchScriptExample(torch.jit.ScriptModule): - @torch.jit.script_method - def add_them(self, data: InputObject) -> torch.Tensor: - return data.foo + data.bar - - @torch.jit.script_method - def make_input_object(self, foo, bar): - return InputObject(foo, bar) - -foo_7 = TorchScriptExample() -foo_7.save("foo7.pt") - -# https://github.com/LaurentMazare/tch-rs/issues/597 -class DictExample(torch.jit.ScriptModule): - @torch.jit.script_method - def generate(self, batch: Dict[str, torch.Tensor]) -> Tuple[torch.Tensor, torch.Tensor]: - return batch["foo"], batch["bar"] - -foo_8 = DictExample() -foo_8.save("foo8.pt") diff --git a/v2/dependency-patches/tch-0.24.0/tests/data_tests.rs b/v2/dependency-patches/tch-0.24.0/tests/data_tests.rs deleted file mode 100644 index f56796495d..0000000000 --- a/v2/dependency-patches/tch-0.24.0/tests/data_tests.rs +++ /dev/null @@ -1,60 +0,0 @@ -use std::io::Write; -use tch::{data, IndexOp, Tensor}; - -mod test_utils; -use test_utils::*; - -#[test] -fn iter2() { - let bsize: usize = 4; - let vs: Vec = (0..1337).collect(); - let xs = Tensor::from_slice(&vs); - let ys = Tensor::from_slice(&vs.iter().map(|x| x * 2).collect::>()); - for (batch_xs, batch_ys) in data::Iter2::new(&xs, &ys, bsize as i64) { - let xs = vec_i64_from(&batch_xs); - let ys = vec_i64_from(&batch_ys); - assert_eq!(xs.len(), bsize); - assert_eq!(ys.len(), bsize); - for i in 0..bsize { - assert_eq!(ys[i], 2 * xs[i]); - if i > 0 { - assert_eq!(xs[i - 1] + 1, xs[i]) - } - } - } - let mut all_in_order = true; - for (batch_xs, batch_ys) in data::Iter2::new(&xs, &ys, bsize as i64).shuffle() { - let xs = vec_i64_from(&batch_xs); - let ys = vec_i64_from(&batch_ys); - assert_eq!(xs.len(), bsize); - assert_eq!(ys.len(), bsize); - for i in 0..bsize { - assert_eq!(ys[i], 2 * xs[i]); - if i > 0 && xs[i - 1] + 1 != xs[i] { - all_in_order = false - } - } - } - assert!(!all_in_order) -} - -#[test] -fn text() { - let filename = std::env::temp_dir().join(format!("tch-{}.txt", std::process::id())); - { - let mut file = std::fs::File::create(&filename).unwrap(); - file.write_all(b"01234567890123456789").unwrap(); - } - let text_data = data::TextData::new(&filename).unwrap(); - for i in 0..10 { - assert_eq!(text_data.label_to_char(i), i.to_string().chars().next().unwrap()); - } - for xs in text_data.iter_shuffle(2, 5) { - let first_column_plus_one = (xs.i((.., ..1)) + 1).fmod(10); - let second_column = xs.i((.., 1..=1)); - let err: i64 = from( - &(first_column_plus_one - second_column).pow_tensor_scalar(2).sum(tch::Kind::Float), - ); - assert_eq!(err, 0) - } -} diff --git a/v2/dependency-patches/tch-0.24.0/tests/device_tests.rs b/v2/dependency-patches/tch-0.24.0/tests/device_tests.rs deleted file mode 100644 index 213b70ea0d..0000000000 --- a/v2/dependency-patches/tch-0.24.0/tests/device_tests.rs +++ /dev/null @@ -1,7 +0,0 @@ -use tch::{Device, Tensor}; - -#[test] -fn tensor_device() { - let t = Tensor::from_slice(&[3, 1, 4]); - assert_eq!(t.device(), Device::Cpu) -} diff --git a/v2/dependency-patches/tch-0.24.0/tests/display_tests.rs b/v2/dependency-patches/tch-0.24.0/tests/display_tests.rs deleted file mode 100644 index 12bd9f0709..0000000000 --- a/v2/dependency-patches/tch-0.24.0/tests/display_tests.rs +++ /dev/null @@ -1,79 +0,0 @@ -use tch::{kind, Tensor}; - -#[test] -fn display_scalar() { - let t = Tensor::ones([], kind::INT64_CPU) * (-1234); - let s = format!("{t}"); - assert_eq!(&s, "[-1234]\nTensor[[], Int64]"); - let s = format!("{}", &t / 10.0); - assert_eq!(&s, "[-123.4000]\nTensor[[], Float]"); - let s = format!("{}", &t / 1e8); - assert_eq!(&s, "[-1.2340e-5]\nTensor[[], Float]"); - let s = format!("{}", &t * 1e8); - assert_eq!(&s, "[-1.2340e11]\nTensor[[], Float]"); - let s = format!("{}", &t * 0.); - assert_eq!(&s, "[-0.]\nTensor[[], Float]"); - let s = format!("{}", &t * (-0.)); - assert_eq!(&s, "[0.]\nTensor[[], Float]"); - let s = format!("{}", &t.eq_tensor(&t)); - assert_eq!(&s, "[true]\nTensor[[], Bool]"); - let s = format!("{}", &t.not_equal_tensor(&t)); - assert_eq!(&s, "[false]\nTensor[[], Bool]"); -} - -#[test] -fn display_vector() { - let t = Tensor::from_slice::(&[]); - let s = format!("{t}"); - assert_eq!(&s, "[]\nTensor[[0], Int64]"); - let t = Tensor::from_slice(&[0.1234567, 1.0, -1.2, 4.1, f64::NAN]); - let s = format!("{t}"); - assert_eq!(&s, "[ 0.1235, 1.0000, -1.2000, 4.1000, NaN]\nTensor[[5], Double]"); - let t = Tensor::ones([50], kind::FLOAT_CPU) * 42; - let s = format!("\n{t}"); - let expected = r#" -[42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., - 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., - 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., 42., - 42., 42.] -Tensor[[50], Float]"#; - assert_eq!(&s, expected); - let t = Tensor::ones([11000], kind::FLOAT_CPU) * 42; - let s = format!("{t}"); - assert_eq!(&s, "[42., 42., 42., ..., 42., 42., 42.]\nTensor[[11000], Float]"); -} - -#[test] -fn display_multi_dim() { - let t = Tensor::ones([200, 100], kind::FLOAT_CPU) * 42; - let s = format!("\n{t}"); - let expected = r#" -[[42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - ... - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.]] -Tensor[[200, 100], Float]"#; - assert_eq!(&s, expected); - let t = t.reshape([2, 1, 1, 100, 100]); - let t = format!("\n{t}"); - let expected = r#" -[[[[[42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - ... - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.]]]], - [[[[42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - ... - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.], - [42., 42., 42., ..., 42., 42., 42.]]]]] -Tensor[[2, 1, 1, 100, 100], Float]"#; - assert_eq!(&t, expected); -} diff --git a/v2/dependency-patches/tch-0.24.0/tests/foo.pt b/v2/dependency-patches/tch-0.24.0/tests/foo.pt deleted file mode 100644 index ae7b294023efae893057fb496fcafb0907c15743..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2030 zcmWIWW@cev;NW1u09*_b3~BlK`YDMeiTVa^P9h9ZSO5cqhEsqyI|o2A! zF8`vG)S}cBm(1i6u0jsYppu{nP&k(qC1&Pj=A{>Mc0`B)`SF>>@oA+wIq^w}$=T(J zMJe$a`T5y}Tz;iVh1`0ez?D77o$(N;5`;m4D*}u#%or18kO2p7a(+sxK0HwL3M$>4 zByfhZum9e>Lk0r(D!YV(b#@BAlIZNayGrkFKxt9ZZWkH5l5O03Y_~CYy#4*PeBOy8 z9P`iI6~FD{$E(e7WU^p!$w^O_7Tq^>v#rwJ&%gBRen43^@6~JxwjYIxn?w{lXC9XM z*JWlDy!O0ct=gfBKJA+YyQ`l$f86h%`79wLy!or)YnzH`vtBMzwx9Cyh|ycY-$g6+ z1RG8#=PbO=9{S;1VY_p2LGZfVulFKu>%N$=a(6~WxYW8;N2kf>EadyvYZ4iAN+|Bv zj*H%5Yx6+CG|}~f_X%JS*#P4ScQDB_Xk!m1y%b=|NRJ06OLd$9hL#RnCq?I9HV`>J zIc}TT?k&>W8(W_2-7uxJa|`6%P*FC;>>A|U4>T7pOy$umC=Xj-f_w|IsOp6{ii>fGo?cu%ifBwe$CA-_1a50oRXEu3H<}USDX`%hRp(UF;orxGQJt)eWB#Wv58GJ9tN&UzcLBSHEXJD3vA0oDY8o zOktCNr2y`dPLM$un#%HuOA_-+iXp`vKYF@qVA2F<@mzh*aKhy$H-j_S)Uwo~Vqg*D z=46j<9$NAw$Tj8y`RU@k^iU>X&H>>7Z$=OWuM?0j$dM$EV#E!g=dc=q$o}X)L5>?i6vOO*)eOWi zq*zARjvTn$DB7n$wL_ybz?+Rt2dY4hSr@L94a$PiRlraH0U@XWP#>uFV(0;r3=AOY R0B=^XEE@wG14unYEdYpla1{Um diff --git a/v2/dependency-patches/tch-0.24.0/tests/foo1.pt b/v2/dependency-patches/tch-0.24.0/tests/foo1.pt deleted file mode 100644 index 0a6cc41e738bb2548be475e95b311c92f84c3250..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1659 zcmWIWW@cev;NW1u0K5#M3~BlKhWaUqC5d_k**R`bf(%jUpn)klKE5QsC^;iOp34oW zoU4#Qv$0kq0%T%IQDSCZW?p(BV@HG-kRP8}9G_O2lM|nmn4DdnSdQ`EC)DzR8z=lFNEg;@kJZ zFHU+!s~eov$~_>UuJ-+w)J8duBf0GnOWRLPWt_okv*q)V-LZAs8V*0zK6-5f-$dCQ zX|0tZ`&Jv?s?mEGGG&9vO?FV&7rKF_obi(u1>S&6=X{8m;((lSOZ3g$IJw8UveRCsu&i2lp z-*2qXAXCAXHt9rT(T5Y4dj8aY{#>6^zxFZn=}q(Y%I?|s<zzGz#J4GCLdZ_#Wf^k2v=*s5 z?dtMSz3VZ-K`?&#hvShxLN{mo{M1_a@~m0>;l6pk9fk8O7_FRjLSscwD+lbqg!=6JebKalJY5 z?&s1^_*KfxY`^Noeg9|l%=*t$dgg2}wvXKXCF$GFTepw8|F3Rx$x+&*;lX2F{knR` z%HMLHsu5PZ9+->VFF(*V?f)G6#EZ*YZX}+YbmRJYpBW)$`(M@+bVxO@h8@g5t5#gQ zco~D~3TI}W%Nv+}v+n1ul@JNge(-xwqUf~**;yMH&2kQjxbv`@AVf||ID7IDPpfvRS(QR>>OKP{S&^;$iR@v49QQ}v$_z2 zDlE<96_+IDl@vqryCN)=Amn+a diff --git a/v2/dependency-patches/tch-0.24.0/tests/foo2.pt b/v2/dependency-patches/tch-0.24.0/tests/foo2.pt deleted file mode 100644 index 5c611c168db1f38c60a91bb272991c69c04016af..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1723 zcmWIWW@cev;NW1u0K5#M3~BlKM*1m+pkwp>E@)1=|*%nBlAQ1 z4Eqlm@U)iKwH|kSwNN;D$?l1~8#jcW@S4q{KACclLBY@M%w1J=LBW3uto5_z@BKe%(hS3%hfGpcX{# zKGFXcOD8rjT-Q{3-HLzSQX}7zC#97WBHFYcZDY&gX8v%cph$7kp5s4_wSN3;51x0T zboG9z^t}x~qK(W>?_ImG?Cramv(mY(uO5ndrF-m7cwt!9O_AxFT8a`rGl2q4x88R3 z0bqd1GU5v~1qLG=fu@&|npBz|4~}9z(t6^``nI=!JxZ*5sYw}u7YD?dHotZTTN=DNZJ~P~P z7SE7BE7I|+%Or#Oe)s~>Wx`2E^Nx8attnJFbY^pl{5Q$%EDxKa?UdIk^mRSl@amRA;xgT*&f#z3EAY-c7B(!^Z1xT{^b? z-!`4PI;UgHuE)+<6a1s;Mex>q>Gyx%Cm-owrt|f##r1RhfAQsMy;Bs|@j7<=OV@&M z-wS^}eX_iK;=hFO;nRyO3a(f`?%-~YGrm9Dg~yTE_tNXnlkDRU?_spMvVbp&&xFCV ziaGAUPL|k1+gUd-t=_=q8^O+OmLMFpfi?HVfp1Bd`e1mQhK@K$yDaO$CBL@@@ivB4u{m`fh@MdGv xfvS*W)`hEOg|c9@JJ1&(AOsZv>SG7-BiTSC0|Q7pz?&7!V`X4v0I7$l1px3$B^3Yw diff --git a/v2/dependency-patches/tch-0.24.0/tests/foo3.pt b/v2/dependency-patches/tch-0.24.0/tests/foo3.pt deleted file mode 100644 index b82ffbf335ef71b613ff166e5217230f191196c5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1723 zcmWIWW@cev;NW1u0K5#M3~BlK#`-CVC5d_k**R`bf(%jUpn)klKE5QsC^;iOp34oW zoU4#Qv$0kq0%T%IQDSCZW?p(BV@HG-kRP8}9G_O2lM|nmn4DdnSdkY51AtBw0XhxSZDI_D5T_;Qr=;q`?bj=)baT?hbR)W(k@=0=1f=Wt>KxrF`a8~axI zNhE(;X|i%zX=G!v&~(0EpPuku5`XqQvm83Gdd+`JIZtcQz>Pw%QBjRrM?DN>|s<-;~L|>`bDgJtHWL|Hj7aya$Hw zIYxY8tia%mBaHP@Qj<#4F$#24YMMDBy0@z&6c==X6 zb#>g(b?aM>$Td;sT`eXv-};WgV!8bx?*DC6I|8w2y5OU;)&-ngdH z?@+Cl^TcAS$uC8`gT(DVPnz95i^>1Ol4tV9aV$*JC!3nQ&;PMzYD~Pyj0X`WOI`Xe z{JHzNr+~v9SAu1xMjkFyU2Lo6xmo(;9GeG9lOj9>pUR{QE&p}%k#ASPw{}|&m=Yj))Zyvo&yC$E+u;YOv_pJ+! zdKSypuxEFzJtW2>vV-g2#Lydz0S`2P_)q)ktX-e-wEyc*-ai2sf;K+S3Oul{XikUY6j@P2PZ;K%CU zY{|1z%KY=r|ICd3zv*QD>_4R^Z=Uxm^K4ZIr6sxT6Qp(n)6pF!NLs>Pln61f!qQS+ zaY8=DSPg&eak rTrDe<1*6@8z5oFsr~ptOJBT021|k_4K+*x;tY981Fdc!^L(~EQYi%fc diff --git a/v2/dependency-patches/tch-0.24.0/tests/foo4.pt b/v2/dependency-patches/tch-0.24.0/tests/foo4.pt deleted file mode 100644 index 420a3d2d374e6f5ab8123626bb551227491cfe5e..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1595 zcmWIWW@cev;NW1u0K5#M3~BlKCi*FfC5d_k**R`bf(%jUpn)klKE5QsC^;iOp34oW zoU4#Qv$0kq0%T%IQDSCZW?p(BV@HG-kRP8}9G_O2lM|nmn4DdnSdsaYZ=gyL4gfk$1n4wOw}~+rLY$VIpOUH%w_mTI(#=U1(~anEM&^h1 z*=92v^0a;b!_}LkJ+{z5eNS@qy=) znCJD;--J4rt8H7ek#EkCXFMl2UHAFTIr1q|)?waO@hB7IKhK^zA?Gb=X0qZMnGqS>YXmO{-UI4HOhV+A6wd z%46270}h-UeV5I>vgM?Y--dU0&aVE+rxy~lJD~6(YoSu>GlLKE`#$p)Kiig}FW|&? zE=ryE^xljp_jfz5X3dk^6_)(IV!rU zo*F5&tv}k%E6g`-sJJY@qwqm&Ysl+mjxTm+%{K4|cwF3;9FW7vsPywseaiE`RAvna zK`#fRg*?`i!XF5=v^2F;n@yFEVBPh=Q^Tv`%tv*zuk%CipZ_(fW%``Yt11|nIa4|F zE&t8?Y+v*z_@l&*7D>){%YSn}*B8D2=O_DQ)|$VJplIGP{n(`>U=*`4L82LZjum2% zgGF;*aY>^`3=VQ_-jP?lO$oSC2J=46M* zY~*}~i*3vW^54aI>7hWcF@SJ@HzSCGmr2M)0xw7c1)!8r5Uq$Z137nq${z%n4P=6~ zVw520W*`T>0>S_w>ohPzu$zI%Rp_2U4rU<~(>5`~O+$)2bp6Ny#e1Q=yt*Fd8t zz?+Rt2dYAjSr@LB70QCq?m%CFz$8%20|8JUJBYpsRREF>@MdKLiLnA9NIgU?0Ki`9 AtpET3 diff --git a/v2/dependency-patches/tch-0.24.0/tests/foo5.pt b/v2/dependency-patches/tch-0.24.0/tests/foo5.pt deleted file mode 100644 index 83787825f30c42443261a683c9b894796d88028c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1531 zcmWIWW@cev;NW1u0K5#M3~BlKrur#~C5d_k**R`bf(%jUpn)klKE5QsC^;iOp34oW zoU4#Qv$0kq0%T%IQDSCZW?p(BV@HG-kRP8}9G_O2lM|nmn4DdnSd+pkwp>E@)1=|*%nBlAQ1 ze6tT5@U*@ED{6IX!nCvv-U-(4w)D#jG!+Q#H2u0H!BWS*-Y>|k>dI>Os(Xr7p%ucF zX7WboCgv0~ILwJDx<5rzQ)!RG?c7ZJ>i+^ypD?lSb70=<5wmc?i{}y>%MZUjzEp6% z()LaNXB7#QZHztmwM>OS?$RlVr7vbHotV_7{U}p6M3gPzBHO*@XJN&!bY$ZHFHd9s z5Oc~R?WEr1pV47U7u;=l#IQ!uK;^i&(d@?}oo5$ad*|P^ye4wX(OtXQe}qh$TYA~W zM#?7qyF;?X2_4ZIu3*;Vx8ALCsq~A{lC5I|1*?gesOBSJkVY`#3swaN0UW`qmy()P znjQ~M1Ux7K?CW>h?~sGYT(S5s+;?X9#LSrGv07R@M}(J&?`vbPiQ-w7TL&d|b&ZRq z3!mrNU5?*qy5i2ACr_T(6xqGakXI}zw&$`^l;uBUSIgzzyjzafI{ffDm%NaZ9#Iz^ zS$FL6)(APYH=J>v-;@ZZSc3LA>mT|mYuZ`=4_@0p24u|02>4id zdt&d4g4|;fh8m%L>Z?6>mx^8Y+;d22z0*eJ?TbF2dht2w;Gdi*t!d5wl^UiS%M0BL zy2ZZC@porZaKe^buHvqzjNa=tL+2+i?3%*5}a+qv5Gf(NqWj;n+Zn*PO~f*%b(PghD(O`%t2AoG*yMq6c`=zp;3Z86ACeK!=fawxFj*Jq!^MD z*QTAAUdA;lEBe&m4SLD7E&rXLzL0p4tEI#3mI%(`&3tWXw= kb_e@4+KDc>>zq3Q~^jjz?+o~B*qGaAoUQn05i+DP5=M^ diff --git a/v2/dependency-patches/tch-0.24.0/tests/foo6.pt b/v2/dependency-patches/tch-0.24.0/tests/foo6.pt deleted file mode 100644 index a87986635012200634c08edf013b3e441042cdfc..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1659 zcmWIWW@cev;NW1u0K5#M3~BlKX8I|KC5d_k**R`bf(%jUpn)klKE5QsC^;iOo+}_H zF*!9OKPM%%$TvTwG$)m-kU_JtRwDvrd`VGaW?p7qdLd&+X;LARUVt|{hXq68Q!Ahx z2nPUNA_8;?rc=ZioFFbq&QD3zhg+vtQ0eAmhv_7AC*tL|PB@!&*g&A=eO;IBtqHTX zxI0(B+_^O1zE`A3`#zT%YnK*%(s47MCI2EJd2QeHMiGtE)d!v5daTyFwI_9_ zYmUBA_(cW*s~0CNPV?IA(Ror^Wv|*lx!!l~?ls3sdG_mO3G%)be0_V;vI05f<;O~e zWec18r)_BZ)O+%LPEv^K=kvR+sT_!Z`}kG;nV)Zpzi++z{#ro&j1@UPEoMjR5+l_! z`y$(SE6$8!;xc~7Reti!w=(nBCjVBhtYZX)dESSr#CO2(mSw^h<_ZifIKo^nB{iuu zJszAO7#MOVczYjq5NTDrWXi?w{Eaz?S6i;aoHV%Oat?E+k&r6d6%tn_*=JjbAh?&x)diabC1YS zqgBQi+=7xf7DUC`IdV0NMhbZ?UN&u|ZbZ!ega2J-sc19Xt>!7V`XN!SW9Pc<*qmLq z-AwmveSe5^sCHH+p*&M3&r(&zUD}1Nk2Q`_DS{WA7HLw=ddiB zC)2>lz~BIlQSA9yh`|9Cqj|+8iFqZ(kW6huQ4VTg(gf$^Tz$@PU_k;*HIP(=-3>gj zoLrV#RGgWgM|u7=<^qMl#d+zWK+iLPaDX=>h=P}C$Yl&KNCE|*lyDHOh(ZWCtB9d! z>jLt?S}}?wbTg16M*+o*f50fiZU!QYp?d~7n1xVGyTA-L4Jmrj^&SFB<2?6q;eH9Xg1brM1agMDN4-D%gjqJWbB9#1M=fDi{sNub8_O75|gvb z6N^&fGxGDZ3z_^%lM0#j0=(Hd>bIt_L;_WUZ~)MuB0z^?x>Sro8RF37{FGFExNGzZ zD&3sqF7pqQ zw>^1k`}?Qm)+3c-OU+8Zht00r^(@mwE0r=Kb^4ZfCcCdeh-|J3xCw=7BPkW5y@9 zJ>a|clj+32-`42u@yoT#&u#-dD!A(sy&FX^V!tXJ*JVy#H3vch%-e&X*6i%mF2g z9sR)(SHG|6Q}my4hI`W)zT~A!H_!1_hWwV_Sp47+cidvr9cQmT{S;j(e|*`cUdugM zu1~+ayneARZkwL|o3qZZKb2J%UwXj~ikmm$rdps)P_o>?|3ry@od>w8C2ruDEc1pa|wd&l4oIg+HcOLxC z?zsAgjO#^K;nMjzb8qz66r9~)E1WZV|32xQ4L#+|)Wp;2U=eBiM`)vRD?!cDI zzfJF#Pf#v@w`$q>6@UNnon)8W@a3EFkxd@wy-Ox-(=axka%z#HyWN+%N)O|N16yjY zh!-D@)qXFs;Xs?j)3}VEEPe_X6J8hO@PFX0 zZp}Yv_9o`>{;f4xr$P>Uv`=TA^ThaVJR>NPZ0ykModrxJozO&ry`&LhP=h6syyB9? zypm!_aic^;vS?t^1Q!*#`kdjwf{X!#K}7|2n|K(UAU2hy78Pga=earA5n%^%dP8R$ zbAkMMab9{T(2EQp9N^6eqTppJav{bGl0X3{B`icMqLf3(+`cB0B<%n9jFR9 uW?i^iRwxTby91LK2n0a|fcn@${F!VZl7Rsv9pKFh=CLxcGJw=W)B*r!f<@E- diff --git a/v2/dependency-patches/tch-0.24.0/tests/foo8.pt b/v2/dependency-patches/tch-0.24.0/tests/foo8.pt deleted file mode 100644 index e6bee10da87fbd9e15e8c1abae6798d1b7a1be35..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1595 zcmWIWW@cev;NW1u0K5#M3~BlK7WyfPC5d_k**R`bf(%jUpn)klKE5QsC^;iOp35aO zxx}?1F}ENmm8+0Jv$0kq0%UMWQDSCZW?p(BV@HG-kRP8}9G_O2lM|nmn4DdnSd|5Emxrr=;q`ouOAy>E@({ z=~Q&5gZZI-j#)#^;MsGjPUb6Iz&e#9d zFWP&y9*XAr^{;okY4HDp#ivbX2sadKbKUdsTvhymsmv)&WSv9YGs-F%gb`s+vZIzM|VEoo%r||Q^l1H zGmeDKJ>#2qwLh)TGk%cyQdB5*c7wVZk;O>^Mi+{4lwOk0Ea_f%G z7lp7tTFIhW2OKyT`fk6QF)O89ojw1an7^EeP34977bm-w@^VyB(M zO(i*>x}NKsBwKAIP4cabIGtbReSeS?Gxd-3W-09Cb&NI^*I|Tcl5D>mqDM( zteUgma=U6397N*(>$>jS+{D1F6~G$Ak)=s%DB4`Nj{5Z9P?IP;|Im>zZw03k-)gU1&v?q8C8fu9TvZN z#U+V(CB=~3s!U9bHZW;|b6l=IXE?ARU;trIj>B#h4}%lLs`R!Hzzw{EI~^% z$XsJCkVh}hOAloN+6=-0-i#m$UM3-z1-u{$6o67fL9`+Y3*`I(DtHiJ5s(Siicxf+ zn}Hk$3J3#$tdqc4!EOd3U!i*jIhch|Ok2qeHw`Hk(e)z-6c37i5nz;oT?37p0B<%n w9jFR9W?i^iRwxTby90dz0+T>74+KDc>>&CkQ~^jjz?+o~B*qGaAoUQn0LF3To&W#< diff --git a/v2/dependency-patches/tch-0.24.0/tests/jit_tests.rs b/v2/dependency-patches/tch-0.24.0/tests/jit_tests.rs deleted file mode 100644 index 08d3c36e9e..0000000000 --- a/v2/dependency-patches/tch-0.24.0/tests/jit_tests.rs +++ /dev/null @@ -1,190 +0,0 @@ -use std::convert::{TryFrom, TryInto}; -use tch::{IValue, Kind, Tensor}; - -mod test_utils; -use test_utils::*; - -#[test] -fn jit() { - let x = Tensor::from_slice(&[3, 1, 4, 1, 5]).to_kind(Kind::Float); - let y = Tensor::from_slice(&[7]).to_kind(Kind::Float); - // The JIT module is created in create_jit_models.py - let mod_ = tch::CModule::load("tests/foo.pt").unwrap(); - let result = mod_.forward_ts(&[&x, &y]).unwrap(); - let expected = x * 2.0 + y + 42.0; - assert_eq!(vec_f64_from(&result), vec_f64_from(&expected)); -} - -#[test] -fn jit_data() { - let x = Tensor::from_slice(&[3, 1, 4, 1, 5]).to_kind(Kind::Float); - let y = Tensor::from_slice(&[7]).to_kind(Kind::Float); - let mut file = std::fs::File::open("tests/foo.pt").unwrap(); - let mod_ = tch::CModule::load_data(&mut file).unwrap(); - let result = mod_.forward_ts(&[&x, &y]).unwrap(); - let expected = x * 2.0 + y + 42.0; - assert_eq!(vec_f64_from(&result), vec_f64_from(&expected)); -} - -#[test] -fn jit1() { - let mod_ = tch::CModule::load("tests/foo1.pt").unwrap(); - let result = mod_.forward_ts(&[Tensor::from(42), Tensor::from(1337)]).unwrap(); - assert_eq!(from::(&result), 1421); - let result = mod_.method_ts("forward", &[Tensor::from(42), Tensor::from(1337)]).unwrap(); - assert_eq!(from::(&result), 1421); -} - -#[test] -fn jit2() { - let mod_ = tch::CModule::load("tests/foo2.pt").unwrap(); - let result = mod_ - .forward_is(&[IValue::from(Tensor::from(42)), IValue::from(Tensor::from(1337))]) - .unwrap(); - let expected1 = Tensor::from(1421); - let expected2 = Tensor::from(-1295); - assert_eq!(result, IValue::from((expected1, expected2))); - // Destructure the tuple, using an option. - let (v1, v2) = <(Tensor, Option)>::try_from(result).unwrap(); - assert_eq!(from::(&v1), 1421); - assert_eq!(from::(&v2.unwrap()), -1295); - let result = mod_ - .method_is("forward", &[IValue::from(Tensor::from(42)), IValue::from(Tensor::from(1337))]) - .unwrap(); - let expected1 = Tensor::from(1421); - let expected2 = Tensor::from(-1295); - assert_eq!(result, IValue::from((expected1, expected2))); - let (v1, v2) = <(Tensor, Tensor)>::try_from(result).unwrap(); - assert_eq!(from::(&v1), 1421); - assert_eq!(from::(&v2), -1295); -} - -#[test] -fn jit3() { - let mod_ = tch::CModule::load("tests/foo3.pt").unwrap(); - let xs = Tensor::from_slice(&[1.0, 2.0, 3.0, 4.0, 5.0]); - let result = mod_.forward_ts(&[xs]).unwrap(); - assert_eq!(from::(&result), 120.0); -} - -#[test] -fn jit4() { - let mod_ = tch::CModule::load("tests/foo4.pt").unwrap(); - let result = mod_.forward_is(&[IValue::from((2.0, 3.0, 4))]).unwrap(); - assert_eq!(result, 14.0.into()); - let v = f64::try_from(result).unwrap(); - assert_eq!(v, 14.0); - let named_parameters = mod_.named_parameters().unwrap(); - assert_eq!(named_parameters, vec![]); -} - -#[test] -fn profiling_mode() { - assert!(tch::jit::get_profiling_mode()); - tch::jit::set_profiling_mode(false); - assert!(!tch::jit::get_profiling_mode()); - tch::jit::set_profiling_mode(true); - assert!(tch::jit::get_profiling_mode()); -} - -#[test] -fn tensor_expr_fuser() { - tch::jit::set_tensor_expr_fuser_enabled(true); - assert!(tch::jit::get_tensor_expr_fuser_enabled()); - tch::jit::set_tensor_expr_fuser_enabled(false); - assert!(!tch::jit::get_tensor_expr_fuser_enabled()); -} - -#[test] -fn jit5() { - let mod_ = tch::CModule::load("tests/foo5.pt").unwrap(); - let result = mod_ - .forward_is(&[IValue::StringList(vec![ - "foo".to_string(), - "bar".to_string(), - "foobar".to_string(), - ])]) - .unwrap(); - assert_eq!( - result, - IValue::from(vec![IValue::from("fo"), IValue::from("ba"), IValue::from("fooba")]) - ); - // Destructuring of ivalue. - let (v1, v2, v3) = <(String, String, String)>::try_from(result).unwrap(); - assert_eq!(v1, "fo"); - assert_eq!(v2, "ba"); - assert_eq!(v3, "fooba"); -} - -#[test] -fn jit6() { - let mod_ = tch::CModule::load("tests/foo6.pt").unwrap(); - let xs = Tensor::from_slice(&[3.0, 4.0, 5.0]); - let result = mod_.forward_is(&[IValue::Tensor(xs)]).unwrap(); - - let obj = tch::jit::Object::try_from(result).unwrap(); - let result = obj.method_is::("y", &[]).unwrap(); - assert_eq!(result, IValue::Tensor(Tensor::from_slice(&[6.0, 8.0, 10.0]))); -} - -#[test] -fn create_traced() { - let mut closure = |inputs: &[Tensor]| { - let v1 = inputs[0].shallow_clone(); - let v2 = inputs[1].shallow_clone(); - vec![v1 + v2] - }; - let modl = tch::CModule::create_by_tracing( - "MyModule", - "MyFn", - &[Tensor::from(0.0), Tensor::from(1.0)], - &mut closure, - ) - .unwrap(); - let filename = std::env::temp_dir().join(format!("tch-modl-{}", std::process::id())); - modl.save(&filename).unwrap(); - let modl = tch::CModule::load(&filename).unwrap(); - let xs = Tensor::from_slice(&[1.0, 2.0, 3.0, 4.0, 5.0]); - let ys = Tensor::from_slice(&[41.0, 1335.0, std::f64::consts::PI - 3., 4.0, 5.0]); - let result = modl.method_ts("MyFn", &[xs, ys]).unwrap(); - assert_eq!( - Vec::::try_from(&result).unwrap(), - [42.0, 1337.0, std::f64::consts::PI, 8.0, 10.0] - ) -} - -// https://github.com/LaurentMazare/tch-rs/issues/475 -#[test] -fn jit_double_free() { - let mod_ = tch::CModule::load("tests/foo7.pt").unwrap(); - let input = mod_.method_is( - "make_input_object", - &[ - &Tensor::from_slice(&[1_f32, 2_f32, 3_f32]).into(), - &Tensor::from_slice(&[4_f32, 5_f32, 6_f32]).into(), - ], - ); - let result = mod_.method_is("add_them", &[&input.unwrap()]); - let result = match result.unwrap() { - IValue::Tensor(tensor) => tensor, - result => panic!("expected a tensor got {result:?}"), - }; - assert_eq!(Vec::::try_from(&result).unwrap(), [5.0, 7.0, 9.0]) -} - -// https://github.com/LaurentMazare/tch-rs/issues/597 -#[test] -fn specialized_dict() { - let mod_ = tch::CModule::load("tests/foo8.pt").unwrap(); - let input = IValue::GenericDict(vec![ - (IValue::String("bar".to_owned()), IValue::Tensor(Tensor::from_slice(&[1_f32, 7_f32]))), - ( - IValue::String("foo".to_owned()), - IValue::Tensor(Tensor::from_slice(&[1_f32, 2_f32, 3_f32])), - ), - ]); - let result = mod_.method_is("generate", &[input]).unwrap(); - let result: (Tensor, Tensor) = result.try_into().unwrap(); - assert_eq!(Vec::::try_from(&result.0).unwrap(), [1.0, 2.0, 3.0]); - assert_eq!(Vec::::try_from(&result.1).unwrap(), [1.0, 7.0]) -} diff --git a/v2/dependency-patches/tch-0.24.0/tests/linear/bs.npy b/v2/dependency-patches/tch-0.24.0/tests/linear/bs.npy deleted file mode 100644 index 0c41779d5fa65cfc5af4dc343840765e3b8007a0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 144 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlV+l>qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= mXCxM+0{I#yI+{8PwF(pfE{iO8+h=~yY^v5B*<(=HY6AdA10vx7 diff --git a/v2/dependency-patches/tch-0.24.0/tests/linear/gen-test-data.py b/v2/dependency-patches/tch-0.24.0/tests/linear/gen-test-data.py deleted file mode 100644 index 1a8b86094b..0000000000 --- a/v2/dependency-patches/tch-0.24.0/tests/linear/gen-test-data.py +++ /dev/null @@ -1,16 +0,0 @@ -# This generates some test weights for a linear layer, as well as a sample input/output pair. This -# is used to demonstrate that the implementing a linear layer with `Tensor::matmul()` produces -# outputs that are close to, but not exactly equal to, the outputs produced by `Tensor::linear()`. -# -# The tensor sizes chosen are as small as I could make them while still reproducing this issue. - -import numpy as np -import torch - -linear = torch.nn.Linear(257, 4) -input = torch.rand(2, 257) - -np.save('ws.npy', linear.weight.data) -np.save('bs.npy', linear.bias.data) -np.save('in.npy', input.data) -np.save('out.npy', linear(input).data) diff --git a/v2/dependency-patches/tch-0.24.0/tests/linear/in.npy b/v2/dependency-patches/tch-0.24.0/tests/linear/in.npy deleted file mode 100644 index b88392af3cbda25e078d4969c2fe26da6d5a7f13..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2184 zcmbW2`CAS60>)2C9c7X&k;zeUibhFU>U`h#luD8oA}TFfs8m!qa)!esbY*S27)v*X zWVw+jLfIzNJQ>NZatWU|~z zCRfBb$Zcfun55*?7;fc8s7q{)kfX;E8Gb9PJ_Sg7gwt|{1gISQV0}|b-&d2&Z zg=&)z+#KDhZVkYpJuY;ti{%X?V?Nd#h1Rn^w7Fo3+vBHmU9J|(?(Id|+#G1{+6Rv> z4tx`+N53Q&mP@UL*P$QzWupa5ZaE_TWC3ym22yF1g8Ys~G#pK(4l6T@&q>Lsl4qOwP zz?!E+Q0w&^8;U-QjsMo>fFviBD~x$*@+>Ngtm*OfAwo)}9K23Ox4sRyYi&Wx?B7s% zU6051g<|}3JuZG1CajL6@K8`ZeboKAR#Jewg=3k0C7hj=oiJ|y8CUwx48 zTgPP_d!m7{YbXP6&f=zBm(i?mfIS&ysL)%@@Qvx>y2(ovOt7NEHwBd+UyAS|eK5WU zr>N#KFL3~k;!JU`#*{T2hT4+RjDJ0ieSgtqrD+U?e^asgnlD#0c~WW7lkacVBCW3( zb6j3vSVsbfJKOV{!F1+uG0QJK5K`y;qG5bG{2r^}^)ETMR&5cUC(nwb^KAJ@$Ax~{ zS7E*6BSzVl$Mr?)&;df=Z8Vux9%%?+X`rQb}2qRege0LXQ4SVjcTPiYJ5K zyJ*PS-F)p9DTTpd1Ih-hg{1M0v(w-vT%Gg*9&_3;uiBpuox{1iQO(vr97VyWtx)YP zLQCI?%-pQO_TrbwZ{I0=EtfL;R5Q*B#w)2okr-G zDDG-?WNe;0)B7EVU$crO#Y6a=+Mb$Q)y^~QLYNhr$fo#bs7wF~2MuMreF%+b+=6A~ zX|z}5;B4GH=1z%#^SBfi)o3xm&!66j%ebo2pZfmw$glWGScax>W=1fZ27W?w?PS=8 zE~1^4D@wdd#THd8v)&l+M3*H!MFiICB=g3~;k>XxLcNUd*{<~fFC-Iq%)%2@2d+Wl zW8r)-el;4km2@$Q;+hsC_TIUg64y0~uaOUMqk1)`)C93`%@t9jFT`=}NH)AERy0kH z$A7+;vVZX~u6Ur}CO;ciPEF+fRUd_FTa$2|pU6nz#hHU_|M z=_>@k@M77lW;EoT73KvGakcOWLdtqUy5s{+be=(%j~^EsW{Q?oo~-#;D)PI|!8u>T z!UiX-EsNvB@K?~hTrceWC*nv|FDjqev8=@esk2gO!CCCz1M>66J$ND+)jOAgfVHpQ|UJ*{4R(=AW?g#XPF&E3vFYLDLgb=KVUB zI|fZ=!Ix`)6WN+(#8>BTqx^6W(et-p zx*VNE<;ax+!A&TyYZ1rPj_~l^f`eLiln=Dz^{3a69d5#Z)=g%$n>Hmw1#?SpC<;3c zqqeRUW=AGcvOYwp_h_T!ahniJeRuM5M}MfN;5D2(CTcfq8a z<qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= zXCxM+0{I$7ItnJ5ItsN4WCJc6MmxKgg|2o&EC1SQ*d5+iAkS?lG0oNPe(D_C3GbHd F0|1FtDCPhF diff --git a/v2/dependency-patches/tch-0.24.0/tests/linear/ws.npy b/v2/dependency-patches/tch-0.24.0/tests/linear/ws.npy deleted file mode 100644 index f8c17d3d88b911ea8c918ac61f04fe312722caa9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4240 zcmbW3_dnN-`^AZ}NvP~uR0zF=_v^aKCKN?RiDZ_dj3T3@ke2SKv^7*(qQdLC+C$oV z*WJ*fTe`~~r9OTCh41<8oFAW$$2pJl*SLvegCb;Q=E~$)#>J<{rddi&EFnQ+X>DSe zkeZf}7LyX4nid!Tf4O(e%=Gv_d3sXJ?D#+4N@8u|;OI2O+9c29|0!j2F`b*9*TZx- zKThzC<}1CU_`fg<9+t6_3R+7*`9nF)?UjYE702)`w1z2gXMIR1Cux+^x zS_{6AacLmGJ`+RP@{hz95>?EvNaXUj)46x@I(lLjgl;n7?6<|O!6ITVT8x*ll*I7|}J5H8w9Lz2)iH_}va0vs_uNL7kkR9Txg8>7l*a4y+nl zD+q0u$l{(US2`qf)sRGAl46cJuD|;Z zj2)G5r@jtfOf|*7kNl$WA$E9D;x1;X*ivf1GJ3yt9(?X@f{x`iu*BGsbyCgfTHndI zpu&R>v2V)u+?0(q zD>8AIqaJ&XQQ%dw_h7`YS*X8xFt&_NLd1=fwJ42G-!($Nw_)u5IvMlssbgpGSpIla ziRJC0!N4F0Kh+$j*&AZ`c8NY}Whdi`i(SI~{h9pV@paVDzC?NofMuF-4 zwnm@H7En+c4-a-uWTo%Q_&lQ*YFZ7VFTW0wdM{u0iwfsYt>#$Q6v9*bjlu%A33%Z5 z7D^tn05klf_-Ifdo=#p3b*pW8jfOR54p&5Q20_7V3@9lZa!S;@2DxiZ&^_oqd0dUd zr#tnz?N%D!Nt%zDI{<6{(WIT*Rrx^l6po(ujFLi!Vb0yXlo>3Cj(=rhX5R!HFvAT~ zuNA}4{3d9+b(XemmZILWFutR8iOzrYhHpPYxgw>AESDMK=RqMD^Jbm2J0lv71Vmx= zgAw9o>n@>bVkjS%pNdN?1a^3#3ZK=|F-~I&kKq%d5W5HJG$*3(Ve3a)PozG^jj$rU^_95C#T3~bu#i&1kg!G(8;IMQ97jg~GGt3Omw)}Br9p}jZX zGEspU`q^Nbl!S7PDL6H@jk>3ra7Tg+w+Bju9WOsog41Z;vb$RdS6NLXJHA0o;c3ds zcVWLQ1I8L3T(Hmz4U|T3aQ;eCOnd}es|{(!-q9SH@|AS2W{ErWkAr1o3FzDDi0AiR zkcMcb(1AA-*grKCH%R2Ucuh9Wy1WvWbzXs)&Z+1*Vg~nV8_yRn`tbdQTC8LCiH^QD zM2o$Kth`}7pU)|QRW3=m$UGc;zU84`Uo~3jQ7nehdFs115ob+GLG2Uj7~U`*EfnQQ z-#8Rimkb5V6$4R!haP$F_HJxmyqWBUV*rWy&=c&3cI{hWuCum~os~rSvvvzDcLVrf zPY!zZOo3k>zZ)-&8OCLu3me0q9v1ht`LayhP4I6o2lL8YR18YNNCh?QsP{(u#x`1p zPMFkFCps8cQ@?;*a2E|(bCd@9|L8-%Vk3E9?QNPryPv3JE5hl<7O_Zg8PrDRaQgb& zkYka=mSuW4>c>;iF+NQn>}T@JAy?_a@)z`}*$B;=8|iI7Kc3vj1d~ThjWF&ODejo$+rg|q|ljm*r2kW;^jBc)M$SUUeKTSzYOEC zGFjL)REy=T&j|r4Hh5uo4wejZVJj^~LGxUH#Opi2F7qn|8Kgqh_|cs5vZL`tzf%yn zwcRCTzZ6SUs-WzM4z9aZ31zvuJTEDUWouVc*?>W8GiL}iJBA}=7_)nQJN)4%%@S5Y zxzPoRZC(sF4ZaDjDXthT?@L`nO35xZ0FGBR)1625_~=pw%XQk2-Xtj*{*qxM%VKI% z&PFW*C#+dpN*~{NU|PB#+AnrPStA=}FA)?!7|{(C3w--M3iB4W!bVkRoccwFea?uq zYNQ{eSM<=z&k`Oq?$7?YkLB(44BO`RBgnZf49YOaUWrzMn@nGs3NZL#3iVtOJJ0eN-6w?Ug=?87&3|3oVlm~4ZBL*L2fWf7b+vuDNi z135c|sQTF_xOOg!dal~=6@?^(x>2~Z!VV?p4mDOR=?1?Yf#Q>HQ;z&{9)!{TF+8JG zm{GqP{6f2E z?`Bi(iv0##JS?$#$b6y3Hj2N*TfxZ{y-?wOCT=T@z@Ht{dG-dtqLpjms3Z+r-@Cvy zu|JMD^9%+W%JA@#Lh$(hO?+@-14!(*!&Zf%_;2hUAvrcyC~hjG(WmY~O6q=C;8a6J zA-SkgvjOr~9}-_Ar*VXV4z5;9g@Q{B!lAM%(6$L7U;O~QV*3v*?PwLkG;{F!$q3FL zrt5qqZ3OBnyYf)mP)y5P3cWA&;!{nb6!1bHzcecG?u)^!H3_gQv{JmhGZZ|A-;?gu zPbFRBJ>VNNn+J?^<*OCbG2U)I2Ta>dHQ__?Qmhizs~v|?uC-uac^wXIPZMA6Hm1Zi zbz@=W+B_O-(Ji)go`k}K27D;c1V4P+L9Th}usU%7IPC8Nd+iWb zD_%t(gbc3t)B^MN{+1l4QDalh}YNcm46Y`3kVZqsjMmKi5Fua02t3uWM| zp~~xY?t$)}eC#>g0x@ahab=MobjqI+tux)wt5X%bj}?$n;23uQF%Pdy)uShO6L?^Y zCWnOrKT)*gb3U3`t+R{2BAyJ9lVekkZP?WSh!Go+iO zh?(h5V5PsGwm)|QrQh-xDqAB|wYCV;UVf$92S!1ZOg62Ena!J z*6$(NUzm(bGOMAe>JMY>`}Dj25!k#X1Gm@2@%_%}_{7i}Cn=@F(>!fl6BZ#Jo-$Ls z|7r$n5BKA1XXDuaK^(qaIGk>{{SXJtZ6_bIT(J2j3bTI3WAlYQ@Nw`eiY_*WJwLM9 zVuT($oSBAWe$av#G19ETwbEi$Cap^G|^xZWYosMhZ0Ly!%{L35Fwd!5wrwtH{|F%N*ze|TTHBqLo z2^*v&p#85+q+kZ((au5&bLuZX+vv)Hep_kh#x9r_na!(Bf4IyzQcC(;-LY@;66%%m zo&u}QAlLg5D2+Hu*8=qE=B#z}G+&MPy)l58b@5mjEYAkV75T;)q}J31I`exkocyfL z?~|gbHEF(!yUHfoX8V~9h}?b$%83>!ldQM(DqSTBIjPfz5!d9!)y zC@UW45`?*nE#drOb(EGnVyN{C*fM=IZfkeIzhw2$>VgK`7^uou7X1yP;Q@+=<4||f zmlt`~gM*|mk3Xu40aMXAGrYBa_4`KWz@_apTunNx0L~ksa^Mg*9M|1y=%j?ay3TtCx%!U+w9t zQ8zSxN7k~_<;?IxIC3MG+O=Asxn&nLdUMBGizd;zZ|b3CgvA#~d@m#AF zj@VQQ`OkIOW{j$+Bp1q-cG+A}JCw`%{o8PQX#)OSHwxp-C!tiTf;}yBcyRDZs+nlS z$)y7s4wSm+&oV&YLto+c#<}8S`vJUkuO|gm<%xez8?XA>3;GnBbIK|uJTR()B6Iuk zvR6`}DIlFr_Z~pgpD6HbsW+dL$)NXZWI_DLozFYT;KCwxtS)&)S0~nr*F5#{$MXK{ zzdanR=XQ~v!35MP&x1P?{{ipi-^r?GB+u+tG=j4X%N~ zRn>GO1USxiBx+6#X1RoM_%lqTy)c0<^$dghpKf^Pau6?B*BkRcBy!WKlVpFfQuMnq z8RaYbh(YQj!9*g%OB>Zed;BiCxiF1QU)O?ks0u%<{Rpc^$)f7j@uZ;Pg>H{a>9g=j zP;G4ktcclQ$WYLFQ8DUI&_wnlxwP2h9nKj++6 z?}WCJXuM=KlN*0LqYlehl2#svijLm6)o=oy)r{rhNL`F6IS$G;ui?TW4;+2aiq<@L Gr2hf`tBwEw diff --git a/v2/dependency-patches/tch-0.24.0/tests/nn_tests.rs b/v2/dependency-patches/tch-0.24.0/tests/nn_tests.rs deleted file mode 100644 index 4fe9a27f60..0000000000 --- a/v2/dependency-patches/tch-0.24.0/tests/nn_tests.rs +++ /dev/null @@ -1,412 +0,0 @@ -use tch::nn::{group_norm, layer_norm}; -use tch::nn::{Module, OptimizerConfig}; -use tch::{kind, nn, Device, Kind, Reduction, Tensor}; - -mod test_utils; -use test_utils::*; - -#[test] -fn optimizer_test() { - tch::manual_seed(42); - // Create some linear data. - let xs = Tensor::from_slice(&(1..15).collect::>()).to_kind(Kind::Float).view([-1, 1]); - let ys = &xs * 0.42 + 1.337; - - // Fit a linear model (with deterministic initialization) on the data. - let vs = nn::VarStore::new(Device::Cpu); - let mut opt = nn::Sgd::default().build(&vs, 0.).unwrap(); - let cfg = nn::LinearConfig { - ws_init: nn::Init::Const(0.), - bs_init: Some(nn::Init::Const(0.)), - bias: true, - }; - let mut linear = nn::linear(vs.root(), 1, 1, cfg); - - let loss = xs.apply(&linear).mse_loss(&ys, Reduction::Mean); - let initial_loss: f64 = from(&loss); - assert!(initial_loss > 1.0, "{}", "initial loss {initial_loss}"); - - opt.set_lr(1e-2); - // Optimization loop. - for _idx in 1..50 { - let loss = xs.apply(&linear).mse_loss(&ys, Reduction::Mean); - opt.backward_step(&loss); - } - let loss = xs.apply(&linear).mse_loss(&ys, Reduction::Mean); - let final_loss: f64 = from(&loss); - assert!(final_loss < 0.25, "{}", "final loss {final_loss}"); - - // Reset the weights to their initial values. - tch::no_grad(|| { - linear.ws.init(nn::Init::Const(0.)); - linear.bs.as_mut().unwrap().init(nn::Init::Const(0.)); - }); - let initial_loss2: f64 = from(&xs.apply(&linear).mse_loss(&ys, Reduction::Mean)); - assert_eq!(initial_loss, initial_loss2); - - // Set the learning-rate to be very small and check that the loss does not change - // much. - opt.set_lr(1e-10); - for _idx in 1..50 { - let loss = xs.apply(&linear).mse_loss(&ys, Reduction::Mean); - opt.backward_step(&loss); - } - let loss = xs.apply(&linear).mse_loss(&ys, Reduction::Mean); - let final_loss: f64 = from(&loss); - assert!((final_loss - initial_loss) < 1e-5, "{}", "final loss {final_loss}") -} - -fn my_module(p: nn::Path, dim: i64) -> impl nn::Module { - let x1 = p.zeros("x1", &[dim]); - let x2 = p.zeros("x2", &[dim]); - nn::func(move |xs| xs * &x1 + xs.exp() * &x2) -} - -#[test] -fn gradient_descent_test() { - let vs = nn::VarStore::new(Device::Cpu); - let my_module = my_module(vs.root(), 7); - let mut opt = nn::Sgd::default().build(&vs, 1e-2).unwrap(); - for _idx in 1..50 { - let xs = Tensor::zeros([7], kind::FLOAT_CPU); - let ys = Tensor::zeros([7], kind::FLOAT_CPU); - let loss = (my_module.forward(&xs) - ys).pow_tensor_scalar(2).sum(Kind::Float); - opt.backward_step(&loss); - } -} - -#[test] -fn gradient_descent_test_clip_norm() { - let vs = nn::VarStore::new(Device::Cpu); - let my_module = my_module(vs.root(), 7); - let mut opt = nn::Sgd::default().build(&vs, 1e-2).unwrap(); - for _idx in 1..50 { - let xs = Tensor::zeros([7], kind::FLOAT_CPU); - let ys = Tensor::ones([7], kind::FLOAT_CPU); - let loss = (my_module.forward(&xs) - ys).pow_tensor_scalar(2).sum(Kind::Float); - opt.backward_step_clip_norm(&loss, 0.1); - } -} - -fn round4(t: Tensor) -> Vec { - let v = vec_f64_from(&t); - v.iter().map(|x| (10000. * x).round() / 10000.).collect() -} - -#[test] -fn gradient_clip_test() { - let vs = nn::VarStore::new(Device::Cpu); - let mut opt = nn::Sgd::default().build(&vs, 1e-2).unwrap(); - let root = vs.root(); - let var1 = root.ones("v1", &[2]); - let var2 = root.ones("v2", &[1]); - let mut var3 = root.ones("v3", &[2]); - tch::no_grad(|| { - var3 *= -1; - }); - let all = Tensor::cat(&[&var1, &(2 * &var2), &(2 * &var3)], 0); - let loss = all.pow_tensor_scalar(2).sum(Kind::Float); - opt.zero_grad(); - loss.backward(); - let g1 = var1.grad(); - let g2 = var2.grad(); - let g3 = var3.grad(); - assert_eq!(vec_f64_from(&g1), [2.0, 2.0]); - assert_eq!(vec_f64_from(&g2), [8.0]); - assert_eq!(vec_f64_from(&g3), [-8.0, -8.0]); - // Test clipping the gradient by value. - let loss = all.pow_tensor_scalar(2).sum(Kind::Float); - opt.zero_grad(); - loss.backward(); - opt.clip_grad_value(4.0); - let g1 = var1.grad(); - let g2 = var2.grad(); - let g3 = var3.grad(); - assert_eq!(vec_f64_from(&g1), [2.0, 2.0]); - assert_eq!(vec_f64_from(&g2), [4.0]); - assert_eq!(vec_f64_from(&g3), [-4.0, -4.0]); - // Test clipping the gradient norm. - let loss = all.pow_tensor_scalar(2).sum(Kind::Float); - opt.zero_grad(); - loss.backward(); - opt.clip_grad_norm(1.0); - let g1 = var1.grad(); - let g2 = var2.grad(); - let g3 = var3.grad(); - assert_eq!(round4(g1), [0.1414, 0.1414]); - assert_eq!(round4(g2), [0.5657]); - assert_eq!(round4(g3), [-0.5657, -0.5657]); -} - -#[test] -fn bn_test() { - let opts = (tch::Kind::Float, tch::Device::Cpu); - let vs = nn::VarStore::new(tch::Device::Cpu); - let bn = nn::batch_norm1d(vs.root(), 40, Default::default()); - let x = Tensor::randn([10, 40], opts); - let _y = x.apply_t(&bn, true); - assert_eq!(vs.len(), 4); -} - -#[test] -fn bn_test_no_affine() { - let opts = (tch::Kind::Float, tch::Device::Cpu); - let vs = nn::VarStore::new(tch::Device::Cpu); - let bn_cfg = nn::BatchNormConfig { affine: false, ..Default::default() }; - let bn = nn::batch_norm1d(vs.root(), 40, bn_cfg); - let x = Tensor::randn([10, 40], opts); - let _y = x.apply_t(&bn, true); - assert_eq!(vs.len(), 2); -} - -#[test] -fn layer_norm_test() { - let opts = (tch::Kind::Float, tch::Device::Cpu); - let vs = nn::VarStore::new(tch::Device::Cpu); - let ln = layer_norm(vs.root(), vec![5, 10, 10], Default::default()); - let x = Tensor::randn([20, 5, 10, 10], opts); - let _y = x.apply(&ln); -} - -#[test] -fn group_norm_test() { - let opts = (tch::Kind::Float, tch::Device::Cpu); - let vs = nn::VarStore::new(tch::Device::Cpu); - let ln = group_norm(vs.root(), 2, 10, Default::default()); - let x = Tensor::randn([1, 10, 7], opts); - let _y = x.apply(&ln); -} - -#[test] -fn layer_norm_parameters_test() { - tch::manual_seed(42); - // Create some linear data. - let xs = Tensor::from_slice(&[42.0, 42.0, 42.0, 24.0]).to_kind(Kind::Float).view([-1, 2]); - let ys = &xs * 0.42 + 1.337; - - // Fit a layer normalization layer (with deterministic initialization) on the data. - let vs = nn::VarStore::new(Device::Cpu); - let mut opt = nn::Sgd::default().build(&vs, 1.0).unwrap(); - let mut ln = layer_norm(vs.root(), vec![2], Default::default()); - - let loss = xs.apply(&ln).mse_loss(&ys, Reduction::Mean); - let initial_loss: f64 = from(&loss); - assert!(initial_loss > 1.0, "{}", "initial loss {initial_loss}"); - - // Optimization loop. - for _idx in 1..50 { - let loss = xs.apply(&ln).mse_loss(&ys, Reduction::Mean); - opt.backward_step(&loss); - } - let loss = xs.apply(&ln).mse_loss(&ys, Reduction::Mean); - let final_loss: f64 = from(&loss); - assert!(final_loss < 0.25, "{}", "final loss {final_loss:?}"); - - // Reset the weights to their initial values. - tch::no_grad(|| { - if let Some(ws) = &mut ln.ws { - ws.init(nn::Init::Const(1.)); - } - if let Some(bs) = &mut ln.bs { - bs.init(nn::Init::Const(0.)); - } - }); - let initial_loss2: f64 = from(&xs.apply(&ln).mse_loss(&ys, Reduction::Mean)); - assert_eq!(initial_loss, initial_loss2) -} - -fn gru_test(rnn_config: nn::RNNConfig) { - use nn::RNN; - let batch_dim = 5; - let seq_len = 3; - let input_dim = 2; - let output_dim = 4; - let vs = nn::VarStore::new(tch::Device::Cpu); - let gru = nn::gru(vs.root(), input_dim, output_dim, rnn_config); - - let num_directions = if rnn_config.bidirectional { 2 } else { 1 }; - let layer_dim = rnn_config.num_layers * num_directions; - // - // step test - let input = Tensor::randn([batch_dim, input_dim], kind::FLOAT_CPU); - let nn::GRUState(output) = gru.step(&input, &gru.zero_state(batch_dim)); - assert_eq!(output.size(), [layer_dim, batch_dim, output_dim]); - - // seq test - let input = Tensor::randn([batch_dim, seq_len, input_dim], kind::FLOAT_CPU); - let (output, _) = gru.seq(&input); - assert_eq!(output.size(), [batch_dim, seq_len, output_dim * num_directions]); -} - -#[test] -fn gru() { - gru_test(Default::default()); - gru_test(nn::RNNConfig { has_biases: false, ..Default::default() }); - gru_test(nn::RNNConfig { bidirectional: true, ..Default::default() }); - gru_test(nn::RNNConfig { num_layers: 2, ..Default::default() }); - gru_test(nn::RNNConfig { num_layers: 2, bidirectional: true, ..Default::default() }); -} - -fn lstm_test(rnn_config: nn::RNNConfig) { - use nn::RNN; - let batch_dim = 5; - let seq_len = 3; - let input_dim = 2; - let output_dim = 4; - let vs = nn::VarStore::new(tch::Device::Cpu); - let lstm = nn::lstm(vs.root(), input_dim, output_dim, rnn_config); - - let num_directions = if rnn_config.bidirectional { 2 } else { 1 }; - let layer_dim = rnn_config.num_layers * num_directions; - // - // step test - let input = Tensor::randn([batch_dim, input_dim], kind::FLOAT_CPU); - let nn::LSTMState((h, c)) = lstm.step(&input, &lstm.zero_state(batch_dim)); - assert_eq!(h.size(), [layer_dim, batch_dim, output_dim]); - assert_eq!(c.size(), [layer_dim, batch_dim, output_dim]); - - // seq test - let input = Tensor::randn([batch_dim, seq_len, input_dim], kind::FLOAT_CPU); - let (output, _) = lstm.seq(&input); - assert_eq!(output.size(), [batch_dim, seq_len, output_dim * num_directions]); -} - -#[test] -fn lstm() { - lstm_test(Default::default()); - lstm_test(nn::RNNConfig { has_biases: false, ..Default::default() }); - lstm_test(nn::RNNConfig { bidirectional: true, ..Default::default() }); - lstm_test(nn::RNNConfig { num_layers: 2, ..Default::default() }); - lstm_test(nn::RNNConfig { num_layers: 2, bidirectional: true, ..Default::default() }); -} - -fn embedding_test(embedding_config: nn::EmbeddingConfig) { - let batch_dim = 5; - let seq_len = 7; - let input_dim = 10; - let output_dim = 4; - let vs = nn::VarStore::new(tch::Device::Cpu); - let embeddings = nn::embedding(vs.root(), input_dim, output_dim, embedding_config); - - // forward test - let input = Tensor::randint(10, [batch_dim, seq_len], kind::INT64_CPU); - let output = embeddings.forward(&input); - assert_eq!(output.size(), [batch_dim, seq_len, output_dim]); - - // padding test - let padding_idx = if embedding_config.padding_idx < 0 { - input_dim + embedding_config.padding_idx - } else { - embedding_config.padding_idx - }; - let input = Tensor::from_slice(&[padding_idx]); - let output = embeddings.forward(&input); - assert_eq!(output.size(), [1, output_dim]); - assert_eq!(output.get(0), embeddings.ws.get(padding_idx)); -} - -#[test] -fn embedding_default() { - embedding_test(Default::default()); -} - -#[test] -fn embedding_neg_padding() { - embedding_test(nn::EmbeddingConfig { padding_idx: -1, ..Default::default() }); -} - -#[test] -fn embedding_zero_padding() { - embedding_test(nn::EmbeddingConfig { padding_idx: 0, ..Default::default() }); -} - -fn linear_test(linear_config: nn::LinearConfig) { - let batch_dim = 5; - let input_dim = 10; - let output_dim = 4; - let vs = nn::VarStore::new(tch::Device::Cpu); - let linear = nn::linear(vs.root(), input_dim, output_dim, linear_config); - - // forward test - let input = Tensor::randint(10, [batch_dim, input_dim], kind::FLOAT_CPU); - let expected_var_store_size = if linear_config.bias { 2 } else { 1 }; - - let output = linear.forward(&input); - assert_eq!(output.size(), [batch_dim, output_dim]); - - assert_eq!(output.size(), [batch_dim, output_dim]); - assert_eq!(vs.variables().len(), expected_var_store_size); - assert!(vs.variables().contains_key("weight")); - assert_eq!(vs.variables().contains_key("bias"), linear_config.bias); -} - -#[test] -fn linear() { - linear_test(Default::default()); - linear_test(nn::LinearConfig { bias: true, ..Default::default() }); - linear_test(nn::LinearConfig { bias: false, ..Default::default() }); -} - -#[test] -fn pad() { - let xs = Tensor::from_slice(&[1., 2., 3.]); - let padded = nn::PaddingMode::Zeros.pad(&xs, &[1, 1]); - assert_eq!(vec_f32_from(&padded), [0., 1., 2., 3., 0.]); - - let xs = Tensor::from_slice(&[1., 2., 3.]).view([1, 3]); - let padded = nn::PaddingMode::Zeros.pad(&xs, &[1, 1]); - assert_eq!(vec_f32_from(&padded.reshape(-1)), [0., 1., 2., 3., 0.]); - - let xs = Tensor::from_slice(&[1., 2., 3., 4.]).view([1, 2, 2]); - let padded = nn::PaddingMode::Reflect.pad(&xs, &[1, 1, 1, 1]); - assert_eq!( - vec_f32_from(&padded.reshape(-1)), - &[4.0, 3.0, 4.0, 3.0, 2.0, 1.0, 2.0, 1.0, 4.0, 3.0, 4.0, 3.0, 2.0, 1.0, 2.0, 1.0] - ); - let padded = nn::PaddingMode::Reflect.pad(&xs, &[1, 1, 1, 1]); - assert_eq!( - vec_f32_from(&padded.reshape(-1)), - &[4.0, 3.0, 4.0, 3.0, 2.0, 1.0, 2.0, 1.0, 4.0, 3.0, 4.0, 3.0, 2.0, 1.0, 2.0, 1.0] - ); - let padded = nn::PaddingMode::Reflect.pad(&xs, &[1, 1, 1, 1]); - assert_eq!( - vec_f32_from(&padded.reshape(-1)), - &[4.0, 3.0, 4.0, 3.0, 2.0, 1.0, 2.0, 1.0, 4.0, 3.0, 4.0, 3.0, 2.0, 1.0, 2.0, 1.0] - ); -} - -fn apply_conv(xs: &Tensor, padding_mode: nn::PaddingMode) -> Tensor { - let vs = nn::VarStore::new(Device::Cpu); - let conv_cfg = nn::ConvConfig { padding: 1, bias: false, padding_mode, ..Default::default() }; - let mut conv = nn::conv2d(vs.root(), 1, 1, 3, conv_cfg); - tch::no_grad(|| { - _ = conv.ws.fill_(1.); - }); - conv.forward(xs) -} - -#[test] -fn conv() { - let xs = Tensor::from_slice(&[1f32, 2., 3., 4.]).view([1, 1, 2, 2]); // NCHW - - let conved = apply_conv(&xs, nn::PaddingMode::Zeros); - assert_eq!(vec_f32_from(&conved.reshape(-1)), &[10.0, 10.0, 10.0, 10.0]); - - let conved = apply_conv(&xs, nn::PaddingMode::Reflect); - assert_eq!(vec_f32_from(&conved.reshape(-1)), &[27.0, 24.0, 21.0, 18.0]); - - let conved = apply_conv(&xs, nn::PaddingMode::Circular); - assert_eq!(vec_f32_from(&conved.reshape(-1)), &[27.0, 24.0, 21.0, 18.0]); - - let conved = apply_conv(&xs, nn::PaddingMode::Replicate); - assert_eq!(vec_f32_from(&conved.reshape(-1)), &[18.0, 21.0, 24.0, 27.0]); -} - -#[test] -fn seq() { - let s = nn::seq().add_fn(|xs| xs.shallow_clone().relu_()); - let xs = Tensor::from_slice(&[1.0, -1.0, 2.0]); - let ys = xs.apply(&s); - assert_eq!(vec_f32_from(&xs), [1., 0., 2.]); - assert_eq!(vec_f32_from(&ys), [1., 0., 2.]); -} diff --git a/v2/dependency-patches/tch-0.24.0/tests/serialization_tests.rs b/v2/dependency-patches/tch-0.24.0/tests/serialization_tests.rs deleted file mode 100644 index 0f9ac2e1dc..0000000000 --- a/v2/dependency-patches/tch-0.24.0/tests/serialization_tests.rs +++ /dev/null @@ -1,195 +0,0 @@ -use tch::{Kind, Tensor}; - -mod test_utils; -use test_utils::*; - -struct TmpFile(std::path::PathBuf); - -impl TmpFile { - fn create(base: &str) -> TmpFile { - let filename = std::env::temp_dir().join(format!( - "tch-{}-{}-{:?}", - base, - std::process::id(), - std::thread::current().id(), - )); - TmpFile(filename) - } -} - -impl std::convert::AsRef for TmpFile { - fn as_ref(&self) -> &std::path::Path { - self.0.as_path() - } -} - -impl Drop for TmpFile { - fn drop(&mut self) { - std::fs::remove_file(&self.0).unwrap() - } -} - -#[test] -fn save_and_load() { - let tmp_file = TmpFile::create("save-and-load"); - let vec = [3.0, 1.0, 4.0, 1.0, 5.0].to_vec(); - let t1 = Tensor::from_slice(&vec); - t1.save(&tmp_file).unwrap(); - let t2 = Tensor::load(&tmp_file).unwrap(); - assert_eq!(vec_f64_from(&t2), vec) -} - -#[test] -fn save_to_stream_and_load() { - let tmp_file = TmpFile::create("write-stream"); - let vec = [3.0, 1.0, 4.0, 1.0, 5.0].to_vec(); - let t1 = Tensor::from_slice(&vec); - t1.save_to_stream(std::fs::File::create(&tmp_file).unwrap()).unwrap(); - let t2 = Tensor::load(&tmp_file).unwrap(); - assert_eq!(vec_f64_from(&t2), vec) -} - -#[test] -fn save_and_load_from_stream() { - let tmp_file = TmpFile::create("read-stream"); - let vec = [3.0, 1.0, 4.0, 1.0, 5.0].to_vec(); - let t1 = Tensor::from_slice(&vec); - t1.save(&tmp_file).unwrap(); - let reader = std::io::BufReader::new(std::fs::File::open(&tmp_file).unwrap()); - let t2 = Tensor::load_from_stream(reader).unwrap(); - assert_eq!(vec_f64_from(&t2), vec) -} - -#[test] -fn save_and_load_multi() { - let tmp_file = TmpFile::create("save-and-load-multi"); - let pi = Tensor::from_slice(&[3.0, 1.0, 4.0, 1.0, 5.0]); - let e = Tensor::from_slice(&[2, 7, 1, 8, 2, 8, 1, 8, 2, 8, 4, 6]); - Tensor::save_multi(&[(&"pi", &pi), (&"e", &e)], &tmp_file).unwrap(); - let named_tensors = Tensor::load_multi(&tmp_file).unwrap(); - assert_eq!(named_tensors.len(), 2); - assert_eq!(named_tensors[0].0, "pi"); - assert_eq!(named_tensors[1].0, "e"); - assert_eq!(from::(&named_tensors[1].1.sum(tch::Kind::Float)), 57); -} - -#[test] -fn save_to_stream_and_load_multi() { - let tmp_file = TmpFile::create("save-to-stream-and-load-multi"); - let pi = Tensor::from_slice(&[3.0, 1.0, 4.0, 1.0, 5.0]); - let e = Tensor::from_slice(&[2, 7, 1, 8, 2, 8, 1, 8, 2, 8, 4, 6]); - Tensor::save_multi_to_stream( - &[(&"pi", &pi), (&"e", &e)], - std::fs::File::create(&tmp_file).unwrap(), - ) - .unwrap(); - let named_tensors = Tensor::load_multi(&tmp_file).unwrap(); - assert_eq!(named_tensors.len(), 2); - assert_eq!(named_tensors[0].0, "pi"); - assert_eq!(named_tensors[1].0, "e"); - assert_eq!(from::(&named_tensors[1].1.sum(tch::Kind::Float)), 57); -} - -#[test] -fn save_and_load_multi_from_stream() { - let tmp_file = TmpFile::create("save-and-load-multi-from-stream"); - let pi = Tensor::from_slice(&[3.0, 1.0, 4.0, 1.0, 5.0]); - let e = Tensor::from_slice(&[2, 7, 1, 8, 2, 8, 1, 8, 2, 8, 4, 6]); - Tensor::save_multi(&[(&"pi", &pi), (&"e", &e)], &tmp_file).unwrap(); - let reader = std::io::BufReader::new(std::fs::File::open(&tmp_file).unwrap()); - let named_tensors = Tensor::load_multi_from_stream(reader).unwrap(); - assert_eq!(named_tensors.len(), 2); - assert_eq!(named_tensors[0].0, "pi"); - assert_eq!(named_tensors[1].0, "e"); - assert_eq!(from::(&named_tensors[1].1.sum(tch::Kind::Float)), 57); -} - -#[test] -fn save_and_load_npz() { - let tmp_file = TmpFile::create("save-and-load-npz"); - let pi = Tensor::from_slice(&[3.0, 1.0, 4.0, 1.0, 5.0]); - let e = Tensor::from_slice(&[2, 7, 1, 8, 2, 8, 1, 8, 2, 8, 4, 6]); - Tensor::write_npz(&[(&"pi", &pi), (&"e", &e)], &tmp_file).unwrap(); - let named_tensors = Tensor::read_npz(&tmp_file).unwrap(); - assert_eq!(named_tensors.len(), 2); - assert_eq!(named_tensors[0].0, "pi"); - assert_eq!(named_tensors[1].0, "e"); - assert_eq!(from::(&named_tensors[1].1.sum(tch::Kind::Float)), 57); -} - -#[test] -fn save_and_load_npz_half() { - let tmp_file = TmpFile::create("save-and-load-npz-half"); - let pi = Tensor::from_slice(&[3.0, 1.0, 4.0, 1.0, 5.0]).to_dtype(Kind::Half, true, false); - let e = - Tensor::from_slice(&[2, 7, 1, 8, 2, 8, 1, 8, 2, 8, 4, 6]).to_dtype(Kind::Half, true, false); - Tensor::write_npz(&[(&"pi", &pi), (&"e", &e)], &tmp_file).unwrap(); - let named_tensors = Tensor::read_npz(&tmp_file).unwrap(); - assert_eq!(named_tensors.len(), 2); - assert_eq!(named_tensors[0].0, "pi"); - assert_eq!(named_tensors[1].0, "e"); - assert_eq!(from::(&named_tensors[1].1.sum(tch::Kind::Float)), 57); -} - -#[test] -fn save_and_load_npz_byte() { - let tmp_file = TmpFile::create("save-and-load-npz-byte"); - let pi = Tensor::from_slice(&[3.0, 1.0, 4.0, 1.0, 5.0]).to_dtype(Kind::Int8, true, false); - let e = - Tensor::from_slice(&[2, 7, 1, 8, 2, 8, 1, 8, 2, 8, 4, 6]).to_dtype(Kind::Int8, true, false); - Tensor::write_npz(&[(&"pi", &pi), (&"e", &e)], &tmp_file).unwrap(); - let named_tensors = Tensor::read_npz(&tmp_file).unwrap(); - assert_eq!(named_tensors.len(), 2); - assert_eq!(named_tensors[0].0, "pi"); - assert_eq!(named_tensors[1].0, "e"); - assert_eq!(from::(&named_tensors[1].1.sum(tch::Kind::Int8)), 57); -} - -#[test] -fn save_and_load_npy() { - let tmp_file = TmpFile::create("save-and-load-npy"); - let pi = Tensor::from_slice(&[3.0, 1.0, 4.0, 1.0, 5.0, 9.0]); - pi.write_npy(&tmp_file).unwrap(); - let pi = Tensor::read_npy(&tmp_file).unwrap(); - assert_eq!(vec_f64_from(&pi), [3.0, 1.0, 4.0, 1.0, 5.0, 9.0]); - let pi = pi.reshape([3, 1, 2]); - pi.write_npy(&tmp_file).unwrap(); - let pi = Tensor::read_npy(&tmp_file).unwrap(); - assert_eq!(pi.size(), [3, 1, 2]); - assert_eq!(vec_f64_from(&pi.flatten(0, -1)), [3.0, 1.0, 4.0, 1.0, 5.0, 9.0]); -} - -#[test] -fn save_and_load_safetensors() { - let tmp_file = TmpFile::create("save-and-load-safetensors"); - let pi = Tensor::from_slice(&[3.0, 1.0, 4.0, 1.0, 5.0]); - let e = Tensor::from_slice(&[2, 7, 1, 8, 2, 8, 1, 8, 2, 8, 4, 6]); - Tensor::write_safetensors(&[(&"pi", &pi), (&"e", &e)], &tmp_file).unwrap(); - let named_tensors = Tensor::read_safetensors(&tmp_file).unwrap(); - assert_eq!(named_tensors.len(), 2); - for (name, tensor) in named_tensors { - match name.as_str() { - "pi" => assert_eq!(from::(&tensor.sum(tch::Kind::Float)), 14), - "e" => assert_eq!(from::(&tensor.sum(tch::Kind::Float)), 57), - _ => panic!("unknow name tensors"), - } - } -} - -#[test] -fn save_and_load_safetensors_half() { - let tmp_file = TmpFile::create("save-and-load-safetensors-half"); - let pi = Tensor::from_slice(&[3.0, 1.0, 4.0, 1.0, 5.0]).to_dtype(Kind::Half, true, false); - let e = - Tensor::from_slice(&[2, 7, 1, 8, 2, 8, 1, 8, 2, 8, 4, 6]).to_dtype(Kind::Half, true, false); - Tensor::write_safetensors(&[(&"pi", &pi), (&"e", &e)], &tmp_file).unwrap(); - let named_tensors = Tensor::read_safetensors(&tmp_file).unwrap(); - assert_eq!(named_tensors.len(), 2); - for (name, tensor) in named_tensors { - match name.as_str() { - "pi" => assert_eq!(from::(&tensor.sum(tch::Kind::Float)), 14), - "e" => assert_eq!(from::(&tensor.sum(tch::Kind::Float)), 57), - _ => panic!("unknow name tensors"), - } - } -} diff --git a/v2/dependency-patches/tch-0.24.0/tests/tensor_indexing.rs b/v2/dependency-patches/tch-0.24.0/tests/tensor_indexing.rs deleted file mode 100644 index 7b21f022bb..0000000000 --- a/v2/dependency-patches/tch-0.24.0/tests/tensor_indexing.rs +++ /dev/null @@ -1,229 +0,0 @@ -use tch::{Device, Kind, Tensor}; -use tch::{IndexOp, NewAxis}; - -mod test_utils; -use test_utils::*; - -#[test] -fn integer_index() { - let opt = (Kind::Float, Device::Cpu); - - let tensor = Tensor::arange_start(0, 2 * 3, opt).view([2, 3]); - let result = tensor.i(1); - assert_eq!(result.size(), &[3]); - assert_eq!(vec_i64_from(&result), &[3, 4, 5]); - - let tensor = Tensor::arange_start(0, 2 * 3, opt).view([2, 3]); - let result = tensor.i((.., 2)); - assert_eq!(result.size(), &[2]); - assert_eq!(vec_i64_from(&result), &[2, 5]); - - let result = tensor.i((.., -2)); - assert_eq!(result.size(), &[2]); - assert_eq!(vec_i64_from(&result), &[1, 4]); -} - -#[test] -fn range_index() { - let opt = (Kind::Float, Device::Cpu); - - // Range - let tensor = Tensor::arange_start(0, 4 * 3, opt).view([4, 3]); - let result = tensor.i(1..3); - assert_eq!(result.size(), &[2, 3]); - assert_eq!(vec_i64_from(&result), &[3, 4, 5, 6, 7, 8]); - - // RangeFull - let tensor = Tensor::arange_start(0, 2 * 3, opt).view([2, 3]); - let result = tensor.i(..); - assert_eq!(result.size(), &[2, 3]); - assert_eq!(vec_i64_from(&result), &[0, 1, 2, 3, 4, 5]); - - // RangeFrom - let tensor = Tensor::arange_start(0, 4 * 3, opt).view([4, 3]); - let result = tensor.i(2..); - assert_eq!(result.size(), &[2, 3]); - assert_eq!(vec_i64_from(&result), &[6, 7, 8, 9, 10, 11]); - - // RangeTo - let tensor = Tensor::arange_start(0, 4 * 3, opt).view([4, 3]); - let result = tensor.i(..2); - assert_eq!(result.size(), &[2, 3]); - assert_eq!(vec_i64_from(&result), &[0, 1, 2, 3, 4, 5]); - - // RangeInclusive - let tensor = Tensor::arange_start(0, 4 * 3, opt).view([4, 3]); - let result = tensor.i(1..=2); - assert_eq!(result.size(), &[2, 3]); - assert_eq!(vec_i64_from(&result), &[3, 4, 5, 6, 7, 8]); - - // RangeTo - let tensor = Tensor::arange_start(0, 4 * 3, opt).view([4, 3]); - let result = tensor.i(..1); - assert_eq!(result.size(), &[1, 3]); - assert_eq!(vec_i64_from(&result), &[0, 1, 2]); - - // RangeToInclusive - let tensor = Tensor::arange_start(0, 4 * 3, opt).view([4, 3]); - let result = tensor.i(..=1); - assert_eq!(result.size(), &[2, 3]); - assert_eq!(vec_i64_from(&result), &[0, 1, 2, 3, 4, 5]); - - // Negative Cases - // - // Rust considers both `0..0` and `1..0` to be valid, empty ranges. - let tensor = Tensor::arange_start(0, 4 * 3, opt).view([4, 3]); - let result = tensor.i(0..0); - assert_eq!(result.size(), &[0, 3]); - assert_eq!(vec_i64_from(&result), &[] as &[i64]); - - // Clippy doesn't like explicitly reversed ranges, so let's pretend the bounds were computed. - let (start, end) = (1, 0); - let tensor = Tensor::arange_start(0, 4 * 3, opt).view([4, 3]); - let result = tensor.i(start..end); - assert_eq!(result.size(), &[0, 3]); - assert_eq!(vec_i64_from(&result), &[] as &[i64]); -} - -#[test] -fn slice_index() { - let opt = (Kind::Float, Device::Cpu); - - let tensor = Tensor::arange_start(0, 6 * 2, opt).view([6, 2]); - let index: &[_] = &[1, 3, 5]; - let result = tensor.i(index); - assert_eq!(result.size(), &[3, 2]); - assert_eq!(vec_i64_from(&result), &[2, 3, 6, 7, 10, 11]); - - let tensor = Tensor::arange_start(0, 3 * 4, opt).view([3, 4]); - let index: &[_] = &[3, 0]; - let result = tensor.i((.., index)); - assert_eq!(result.size(), &[3, 2]); - assert_eq!(vec_i64_from(&result), &[3, 0, 7, 4, 11, 8]); -} - -#[test] -fn new_index() { - let opt = (Kind::Float, Device::Cpu); - - let tensor = Tensor::arange_start(0, 2 * 3, opt).view([2, 3]); - let result = tensor.i((NewAxis,)); - assert_eq!(result.size(), &[1, 2, 3]); - assert_eq!(vec_i64_from(&result), &[0, 1, 2, 3, 4, 5]); - - let tensor = Tensor::arange_start(0, 2 * 3, opt).view([2, 3]); - let result = tensor.i((.., NewAxis)); - assert_eq!(result.size(), &[2, 1, 3]); - assert_eq!(vec_i64_from(&result), &[0, 1, 2, 3, 4, 5]); - - let tensor = Tensor::arange_start(0, 2 * 3, opt).view([2, 3]); - let result = tensor.i((.., .., NewAxis)); - assert_eq!(result.size(), &[2, 3, 1]); - assert_eq!(vec_i64_from(&result), &[0, 1, 2, 3, 4, 5]); -} - -#[cfg(target_os = "linux")] -#[test] -fn complex_index() { - let opt = (Kind::Float, Device::Cpu); - - let tensor = Tensor::arange_start(0, 2 * 3 * 5 * 7, opt).view([2, 3, 5, 7]); - let result = tensor.i((1, 1..2, vec![2, 3, 0].as_slice(), NewAxis, 3..)); - assert_eq!(result.size(), &[1, 3, 1, 4]); - assert_eq!( - vec_i64_from(&result), - &[157, 158, 159, 160, 164, 165, 166, 167, 143, 144, 145, 146] - ); -} - -#[test] -fn index_3d() { - let values: Vec = (0..24).collect(); - let tensor = tch::Tensor::from_slice(&values).view((2, 3, 4)); - assert_eq!(vec_i64_from(&tensor.i((0, 0, 0))), &[0]); - assert_eq!(vec_i64_from(&tensor.i((1, 0, 0))), &[12]); - assert_eq!(vec_i64_from(&tensor.i((0..2, 0, 0))), &[0, 12]); -} - -#[test] -fn tensor_index() { - let t = Tensor::arange(6, (Kind::Int64, Device::Cpu)).view((2, 3)); - let rows_select = Tensor::from_slice(&[0i64, 1, 0]); - let column_select = Tensor::from_slice(&[1i64, 2, 2]); - - let selected = t.index(&[Some(rows_select), Some(column_select)]); - assert_eq!(selected.size(), &[3]); - assert_eq!(vec_i64_from(&selected), &[1, 5, 2]); -} - -#[test] -fn tensor_index2() { - let t = Tensor::arange(400, (Kind::Int64, Device::Cpu)).view((2, 2, 10, 10)); - - let selected = t.index(&[ - Some(Tensor::from(0i64)), - Some(Tensor::from(1i64)), - Some(Tensor::from_slice(&[1i64, 2, 3])), - Some(Tensor::from_slice(&[5i64, 6, 7])), - ]); - assert_eq!(selected.size(), &[3]); - // 115 = 0 * 200 + 1 * 100 + 1 * 10 + 5 - // 126 = 0 * 200 + 1 * 100 + 2 * 10 + 6 - // 137 = 0 * 200 + 1 * 100 + 3 * 10 + 7 - assert_eq!(vec_i64_from(&selected), &[115, 126, 137]); -} - -#[test] -fn tensor_multi_index() { - let t = Tensor::arange(6, (Kind::Int64, Device::Cpu)).view((2, 3)); - - let select_1 = Tensor::from_slice(&[0i64, 1, 0]); - let select_2 = Tensor::from_slice(&[1i64, 0, 0]); - let select_final = Tensor::stack(&[select_1, select_2], 0); - assert_eq!(select_final.size(), &[2, 3]); - - let selected = t.index(&[Some(select_final)]); // index only rows - assert_eq!(selected.size(), &[2, 3, 3]); - assert_eq!(vec_i64_from(&selected), &[0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 0, 1, 2]); - // after flattening -} - -#[test] -fn tensor_put() { - let t = Tensor::arange(6, (Kind::Int64, Device::Cpu)).view((2, 3)); - let rows_select = Tensor::from_slice(&[0i64, 1, 0]); - let column_select = Tensor::from_slice(&[1i64, 2, 2]); - let values = Tensor::from_slice(&[10i64, 12, 24]); - - let updated = t.index_put(&[Some(rows_select), Some(column_select)], &values, false); - assert_eq!(vec_i64_from(&updated), &[0i64, 10, 24, 3, 4, 12]); // after flattening -} - -#[test] -fn indexing_doc() { - let tensor = Tensor::from_slice(&[1, 2, 3, 4, 5, 6]).view((2, 3)); - let t = tensor.i(1); - assert_eq!(vec_i64_from(&t), [4, 5, 6]); - let t = tensor.i((.., -2)); - assert_eq!(vec_i64_from(&t), [2, 5]); - - let tensor = Tensor::from_slice(&[1, 2, 3, 4, 5, 6]).view((2, 3)); - let t = tensor.i((.., 1..)); - assert_eq!(t.size(), [2, 2]); - assert_eq!(vec_i64_from(&t.contiguous().view(-1)), [2, 3, 5, 6]); - let t = tensor.i((..1, ..)); - assert_eq!(t.size(), [1, 3]); - assert_eq!(vec_i64_from(&t.contiguous().view(-1)), [1, 2, 3]); - let t = tensor.i((.., 1..2)); - assert_eq!(t.size(), [2, 1]); - assert_eq!(vec_i64_from(&t.contiguous().view(-1)), [2, 5]); - let t = tensor.i((.., 1..=2)); - assert_eq!(t.size(), [2, 2]); - assert_eq!(vec_i64_from(&t.contiguous().view(-1)), [2, 3, 5, 6]); - - let tensor = Tensor::from_slice(&[1, 2, 3, 4, 5, 6]).view((2, 3)); - let t = tensor.i((NewAxis,)); - assert_eq!(t.size(), &[1, 2, 3]); - let t = tensor.i((.., .., NewAxis)); - assert_eq!(t.size(), &[2, 3, 1]); -} diff --git a/v2/dependency-patches/tch-0.24.0/tests/tensor_tests.rs b/v2/dependency-patches/tch-0.24.0/tests/tensor_tests.rs deleted file mode 100644 index 0fdee88431..0000000000 --- a/v2/dependency-patches/tch-0.24.0/tests/tensor_tests.rs +++ /dev/null @@ -1,508 +0,0 @@ -#![allow(clippy::unnecessary_fallible_conversions)] -use anyhow::Result; -use half::f16; -use std::convert::{TryFrom, TryInto}; -use std::f32; -use tch::{Device, TchError, Tensor}; - -mod test_utils; -use test_utils::*; - -#[test] -#[cfg(feature = "cuda-tests")] -fn amp_non_finite_check_and_unscale() { - let mut u = Tensor::from_slice(&[10f32, 20f32]).to_device(Device::Cuda(0)); - let mut found_inf = Tensor::from_slice(&[0f32]).to_device(Device::Cuda(0)); - let inv_scale = Tensor::from_slice(&[0.1f32]).to_device(Device::Cuda(0)); - u.internal_amp_non_finite_check_and_unscale(&mut found_inf, &inv_scale); - assert_eq!(vec_f32_from(&u), &[1f32, 2f32]); - assert_eq!(vec_f32_from(&found_inf), [0f32]); - - let mut v = Tensor::from_slice(&[1f32, f32::INFINITY]).to_device(Device::Cuda(0)); - v.internal_amp_non_finite_check_and_unscale(&mut found_inf, &inv_scale); - assert_eq!(vec_f32_from(&v), &[0.1, f32::INFINITY]); - assert_eq!(vec_f32_from(&found_inf), [1f32]); - - u.internal_amp_non_finite_check_and_unscale(&mut found_inf, &inv_scale); - assert_eq!(vec_f32_from(&u), &[0.1, 0.2]); - // found_inf is sticky - assert_eq!(vec_f32_from(&found_inf), [1f32]); -} - -#[test] -fn assign_ops() { - let mut t = Tensor::from_slice(&[3, 1, 4, 1, 5]); - t += 1; - t *= 2; - t -= 1; - assert_eq!(vec_i64_from(&t), [7, 3, 9, 3, 11]); -} - -#[test] -fn constant_ops() { - let mut t = Tensor::from_slice(&[7i64, 3, 9, 3, 11]); - t = -t; - assert_eq!(vec_i64_from(&t), [-7, -3, -9, -3, -11]); - t = 1 - t; - assert_eq!(vec_i64_from(&t), [8, 4, 10, 4, 12]); - t = 2 * t; - assert_eq!(vec_i64_from(&t), [16, 8, 20, 8, 24]); - - let mut t = Tensor::from_slice(&[0.2f64, 0.1]); - t = 2 / t; - assert_eq!(vec_f64_from(&t), [10.0, 20.0]); -} - -#[test] -fn iter() { - let t = Tensor::from_slice(&[7i64, 3, 9, 3, 11]); - let v = t.iter::().unwrap().collect::>(); - assert_eq!(v, [7, 3, 9, 3, 11]); - let t = Tensor::from_slice(&[std::f64::consts::PI, 15.926, 5.3589, 79.0]); - let v = t.iter::().unwrap().collect::>(); - assert_eq!(v, [std::f64::consts::PI, 15.926, 5.3589, 79.0]); -} - -#[test] -fn array_conversion() { - let vec: Vec<_> = (0..6).map(|x| (x * x) as f64).collect(); - let t = Tensor::from_slice(&vec); - assert_eq!(vec_f64_from(&t), [0.0, 1.0, 4.0, 9.0, 16.0, 25.0]); - let t = t.view([3, 2]); - assert_eq!(from::>>(&t), [[0.0, 1.0], [4.0, 9.0], [16.0, 25.0]]); - let t = t.view([2, 3]); - assert_eq!(from::>>(&t), [[0.0, 1.0, 4.0], [9.0, 16.0, 25.0]]) -} - -#[test] -fn binary_ops() { - let t = Tensor::from_slice(&[3.0, 1.0, 4.0, 1.0, 5.0]); - let t = (&t * &t) + &t - 1.5; - assert_eq!(vec_f64_from(&t), [10.5, 0.5, 18.5, 0.5, 28.5]); -} - -#[test] -fn grad() { - let mut x = Tensor::from(2.0).set_requires_grad(true); - let y = &x * &x + &x + 36; - x.zero_grad(); - y.backward(); - let dy_over_dx = x.grad(); - assert_eq!(vec_f64_from(&dy_over_dx), [5.0]); -} - -#[test] -fn grad_grad() { - // Compute a second order derivative using run_backward. - let mut x = Tensor::from(42.0).set_requires_grad(true); - let y = &x * &x * &x + &x + &x * &x; - x.zero_grad(); - let dy_over_dx = Tensor::run_backward(&[y], &[&x], true, true); - assert_eq!(dy_over_dx.len(), 1); - let dy_over_dx = &dy_over_dx[0]; - dy_over_dx.backward(); - let dy_over_dx2 = x.grad(); - assert_eq!(from::(&dy_over_dx2), 254.0); -} - -#[test] -#[should_panic(expected = "one of the input tensor does not use set_requires_grad")] -fn grad_without_requires() { - let x = Tensor::from(2.0); - let y = &x * &x + &x + 36; - let _dy_over_dx = Tensor::run_backward(&[y], &[&x], true, true); -} - -#[test] -fn cat_and_stack() { - let t = Tensor::from_slice(&[13.0, 37.0]); - let t = Tensor::cat(&[&t, &t, &t], 0); - assert_eq!(t.size(), [6]); - assert_eq!(vec_f64_from(&t), [13.0, 37.0, 13.0, 37.0, 13.0, 37.0]); - - let t = Tensor::from_slice(&[13.0, 37.0]); - let t = Tensor::stack(&[&t, &t, &t], 0); - assert_eq!(t.size(), [3, 2]); - assert_eq!(vec_f64_from(&t), [13.0, 37.0, 13.0, 37.0, 13.0, 37.0]); - - let t = Tensor::from_slice(&[13.0, 37.0]); - let t = Tensor::stack(&[&t, &t, &t], 1); - assert_eq!(t.size(), [2, 3]); - assert_eq!(vec_f64_from(&t), [13.0, 13.0, 13.0, 37.0, 37.0, 37.0]); -} - -#[test] -fn onehot() { - let xs = Tensor::from_slice(&[0, 1, 2, 3]); - let onehot = xs.onehot(4); - assert_eq!( - vec_f64_from(&onehot), - vec![1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 1.0] - ); - assert_eq!(onehot.device(), xs.device()); - assert_eq!(onehot.size(), vec![4, 4]) -} - -#[test] -fn fallible() { - // Try to compare two tensors with incompatible dimensions and check that this returns an - // error. - let xs = Tensor::from_slice(&[0, 1, 2, 3]); - let ys = Tensor::from_slice(&[0, 1, 2, 3, 4]); - assert!(xs.f_eq_tensor(&ys).is_err()) -} - -#[test] -fn chunk() { - let xs = Tensor::from_slice(&[0, 1, 2, 3, 4, 5, 6, 7, 8, 9]); - let tensors = xs.chunk(3, 0); - assert_eq!(tensors.len(), 3); - assert_eq!(vec_i64_from(&tensors[0]), vec![0, 1, 2, 3]); - assert_eq!(vec_i64_from(&tensors[1]), vec![4, 5, 6, 7]); - assert_eq!(vec_i64_from(&tensors[2]), vec![8, 9]); -} - -#[test] -fn broadcast() { - let xs = Tensor::from_slice(&[4, 5, 3]); - let ys = Tensor::from(42); - let tensors = Tensor::broadcast_tensors(&[xs, ys]); - assert_eq!(tensors.len(), 2); - assert_eq!(vec_i64_from(&tensors[0]), vec![4, 5, 3]); - assert_eq!(vec_i64_from(&tensors[1]), vec![42, 42, 42]); -} - -#[test] -fn eq() { - let t = Tensor::from_slice(&[3, 1, 4, 1, 5]); - let u = &t + 1 - 1; - assert_eq!(t, u); - assert!(t == u); - assert!(t != u - 1); - - let t = Tensor::from_slice(&[std::f64::consts::PI]); - let u = Tensor::from(std::f64::consts::PI); - // The tensor shape is important for equality. - assert!(t != u); - assert!(t.size() != u.size()); - - let u = u.reshape([1]); - assert_eq!(t, u); - assert!(t == u); - assert!(t != u - 1) -} - -#[test] -fn values_at_index() { - let t = Tensor::from(42); - assert_eq!(t.int64_value(&[]), 42); - assert_eq!(t.double_value(&[]), 42.0); - assert!(t.f_int64_value(&[0]).is_err()); - assert!(t.f_double_value(&[0]).is_err()); -} - -#[test] -fn into_ndarray_f64() { - let tensor = Tensor::from_slice(&[1., 2., 3., 4.]); - let nd: ndarray::ArrayD = (&tensor).try_into().unwrap(); - assert_eq!(vec_f64_from(&tensor).as_slice(), nd.as_slice().unwrap()); -} - -#[test] -fn into_ndarray_i64() { - let tensor = Tensor::from_slice(&[1, 2, 3, 4]); - let nd: ndarray::ArrayD = (&tensor).try_into().unwrap(); - assert_eq!(vec_i64_from(&tensor).as_slice(), nd.as_slice().unwrap()); -} - -#[test] -fn from_ndarray_f64() { - let nd = ndarray::arr2(&[[1f64, 2.], [3., 4.]]); - let tensor = Tensor::try_from(nd.clone()).unwrap(); - assert_eq!(vec_f64_from(&tensor).as_slice(), nd.as_slice().unwrap()); -} - -#[test] -fn from_ndarray_i64() { - let nd = ndarray::arr2(&[[1i64, 2], [3, 4]]); - let tensor = Tensor::try_from(nd.clone()).unwrap(); - assert_eq!(vec_i64_from(&tensor).as_slice(), nd.as_slice().unwrap()); -} - -#[test] -fn from_ndarray_bool() { - let nd = ndarray::arr2(&[[true, false], [true, true]]); - let tensor = Tensor::try_from(nd.clone()).unwrap(); - assert_eq!(vec_bool_from(&tensor).as_slice(), nd.as_slice().unwrap()); -} - -#[test] -fn from_primitive() -> Result<()> { - assert_eq!(vec_i32_from(&Tensor::try_from(1_i32)?), vec![1]); - assert_eq!(vec_i64_from(&Tensor::try_from(1_i64)?), vec![1]); - assert_eq!(vec_f16_from(&Tensor::try_from(f16::from_f64(1.0))?), vec![f16::from_f64(1.0)]); - assert_eq!(vec_f32_from(&Tensor::try_from(1_f32)?), vec![1.0]); - assert_eq!(vec_f64_from(&Tensor::try_from(1_f64)?), vec![1.0]); - assert_eq!(vec_bool_from(&Tensor::try_from(true)?), vec![true]); - Ok(()) -} - -#[test] -fn from_vec() -> Result<()> { - assert_eq!(vec_i32_from(&Tensor::try_from(vec![-1_i32, 0, 1])?), vec![-1, 0, 1]); - assert_eq!(vec_i64_from(&Tensor::try_from(vec![-1_i64, 0, 1])?), vec![-1, 0, 1]); - assert_eq!( - from::>(&Tensor::try_from(vec![ - f16::from_f64(-1.0), - f16::from_f64(0.0), - f16::from_f64(1.0) - ])?), - vec![f16::from_f64(-1.0), f16::from_f64(0.0), f16::from_f64(1.0)] - ); - assert_eq!(vec_f32_from(&Tensor::try_from(vec![-1_f32, 0.0, 1.0])?), vec![-1.0, 0.0, 1.0]); - assert_eq!(vec_f64_from(&Tensor::try_from(vec![-1_f64, 0.0, 1.0])?), vec![-1.0, 0.0, 1.0]); - assert_eq!(vec_bool_from(&Tensor::try_from(vec![true, false])?), vec![true, false]); - Ok(()) -} - -#[test] -fn from_slice() -> Result<()> { - assert_eq!(vec_i32_from(&Tensor::try_from(&[-1_i32, 0, 1] as &[_])?), vec![-1, 0, 1]); - assert_eq!(vec_i64_from(&Tensor::try_from(&[-1_i64, 0, 1] as &[_])?), vec![-1, 0, 1]); - assert_eq!( - vec_f16_from(&Tensor::try_from(&[ - f16::from_f64(-1.0), - f16::from_f64(0.0), - f16::from_f64(1.0) - ] as &[_])?), - vec![f16::from_f64(-1.0), f16::from_f64(0.0), f16::from_f64(1.0)] - ); - assert_eq!(vec_f32_from(&Tensor::try_from(&[-1_f32, 0.0, 1.0] as &[_])?), vec![-1.0, 0.0, 1.0]); - assert_eq!(vec_f64_from(&Tensor::try_from(&[-1_f64, 0.0, 1.0] as &[_])?), vec![-1.0, 0.0, 1.0]); - assert_eq!(vec_bool_from(&Tensor::try_from(&[true, false] as &[_])?), vec![true, false]); - Ok(()) -} - -#[test] -fn test_device() { - let x = Tensor::from(1); - assert_eq!(x.device(), Device::Cpu); - let x = Tensor::from(1).to_device(Device::Cpu); - assert_eq!(x.device(), Device::Cpu); - if tch::Cuda::device_count() > 0 { - let x = Tensor::from(1).to_device(Device::Cuda(0)); - assert_eq!(x.device(), Device::Cuda(0)); - let x = Tensor::from(1).to_device(Device::Cuda(0)).to_device(Device::Cpu); - assert_eq!(x.device(), Device::Cpu); - } -} - -#[test] -fn where_() { - let t1 = Tensor::from_slice(&[3, 1, 4, 1, 5, 9]); - let t2 = Tensor::from_slice(&[2, 7, 1, 8, 2, 8]); - let t = t1.where_self(&t1.lt(4), &t2); - assert_eq!(vec_i64_from(&t), [3, 1, 1, 1, 2, 8]); -} - -#[test] -fn bool_tensor() { - let t1 = Tensor::from_slice(&[true, true, false]); - assert_eq!(vec_i64_from(&t1), [1, 1, 0]); - assert_eq!(vec_bool_from(&t1), [true, true, false]); - let t1 = Tensor::from_slice(&[0, 1, 0]).to_kind(tch::Kind::Bool); - let t2 = Tensor::from_slice(&[1, 1, 1]).to_kind(tch::Kind::Bool); - let t1_any = t1.any(); - let t2_any = t2.any(); - let t1_all = t1.all(); - let t2_all = t2.all(); - assert!(from::(&t1_any)); - assert!(!from::(&t1_all)); - assert!(from::(&t2_any)); - assert!(from::(&t2_all)); -} - -#[test] -fn copy_overflow() { - let mut s = [f32::consts::PI]; - let r = Tensor::zeros([1], (tch::Kind::Int64, Device::Cpu)).f_copy_data(&mut s, 1); - assert!(r.is_err()); - - let mut s: [i8; 0] = []; - let r = Tensor::zeros([10000], (tch::Kind::Int8, Device::Cpu)).f_copy_data(&mut s, 10000); - assert!(r.is_err()); -} - -#[test] -fn mkldnn() { - if tch::utils::has_mkldnn() { - let t = Tensor::randn([5, 5, 5], (tch::Kind::Float, Device::Cpu)); - assert!(!t.is_mkldnn()); - assert!(t.to_mkldnn().is_mkldnn()); - } -} - -#[test] -fn sparse() { - let t = Tensor::from_slice(&[1, 2, 3]); - assert!(!t.is_sparse()); -} - -#[test] -fn einsum() { - // Element-wise squaring of a vector. - let t = Tensor::from_slice(&[1.0, 2.0, 3.0]); - let t = Tensor::einsum("i, i -> i", &[&t, &t], None::); - assert_eq!(vec_f64_from(&t), [1.0, 4.0, 9.0]); - // Matrix transpose - let t = Tensor::from_slice(&[1.0, 2.0, 3.0, 4.0, 5.0, 6.0]).reshape([2, 3]); - let t = Tensor::einsum("ij -> ji", &[t], None::); - assert_eq!(vec_f64_from(&t), [1.0, 4.0, 2.0, 5.0, 3.0, 6.0]); - // Sum all elements - let t = Tensor::einsum("ij -> ", &[t], None::); - assert_eq!(vec_f64_from(&t), [21.0]); -} - -#[test] -fn vec2() { - let tensor = Tensor::from_slice(&[1., 2., 3., 4., 5., 6.]).reshape([2, 3]); - assert_eq!(Vec::>::try_from(tensor).unwrap(), [[1., 2., 3.], [4., 5., 6.]]) -} - -#[test] -fn upsample1d() { - let tensor = Tensor::from_slice(&[1., 2., 3., 4., 5., 6.]).reshape([2, 3, 1]); - let up1 = tensor.upsample_linear1d([2], false, 1.); - assert_eq!( - // Exclude the last element because of some numerical instability. - vec_f64_from(&up1)[0..11], - [1.0, 1.0, 2.0, 2.0, 3.0, 3.0, 4.0, 4.0, 5.0, 5.0, 6.0] - ); - let up1 = tensor.upsample_linear1d([2], false, None); - assert_eq!(vec_f64_from(&up1), [1.0, 1.0, 2.0, 2.0, 3.0, 3.0, 4.0, 4.0, 5.0, 5.0, 6.0, 6.0]); -} - -#[test] -fn argmax() { - let tensor = Tensor::from_slice(&[7., 2., 3., 4., 5., 6.]).reshape([2, 3]); - let argmax = tensor.argmax(None, false); - assert_eq!(vec_i64_from(&argmax), [0],); - let argmax = tensor.argmax(0, false); - assert_eq!(vec_i64_from(&argmax), [0, 1, 1],); - let argmax = tensor.argmax(-1, false); - assert_eq!(vec_i64_from(&argmax), [0, 2],); -} - -#[test] -fn strides() { - fn check_stride(t: &Tensor) { - let shape = t.size(); - let ndim = shape.len(); - let mut c = 1; - let mut strides = vec![0i64; ndim]; - strides[ndim - 1] = c; - for i in (1..ndim).rev() { - c *= shape[i]; - strides[i - 1] = c; - } - - assert_eq!(t.stride(), strides); - } - - let tensor = Tensor::zeros([2, 3, 4], tch::kind::FLOAT_CPU); - check_stride(&tensor); - - let tensor: Tensor = Tensor::ones([3, 4, 5, 6, 7, 8], tch::kind::FLOAT_CPU); - check_stride(&tensor); -} - -#[test] -fn nested_tensor() { - let vec: Vec> = vec![vec![1, 2], vec![1, 2], vec![4, 5]]; - let t = Tensor::from_slice2(&vec); - assert_eq!(t.size(), [3, 2]); - assert_eq!(vec_i32_from(&t.view([-1])), [1, 2, 1, 2, 4, 5]); -} - -#[test] -fn quantized() { - let t = Tensor::from_slice(&[-1f32, 0., 1., 2., 120., 0.42]); - let t = t.quantize_per_tensor(0.1, 10, tch::Kind::QUInt8); - let t = t.dequantize(); - assert_eq!(vec_f32_from(&t), [-1f32, 0., 1., 2., 24.5, 0.4]); -} - -#[test] -fn nll_loss() { - let input = Tensor::randn([3, 5], (tch::Kind::Float, Device::Cpu)).set_requires_grad(true); - let target = Tensor::from_slice(&[1i64, 0, 4]); - let output = input.nll_loss(&target); - output.backward(); - - let weights = Tensor::from_slice(&[1f32, 2.0, 2.0, 1.0, 1.0]); - // This used to segfault, see https://github.com/LaurentMazare/tch-rs/issues/366 - let _output = input.g_nll_loss(&target, Some(weights), tch::Reduction::Mean, -100); -} - -#[test] -fn allclose() { - let t = Tensor::from_slice(&[-1f32, 0., 1., 2., 120., 0.42]); - let t = t.quantize_per_tensor(0.1, 10, tch::Kind::QUInt8); - let t = t.dequantize(); - assert!(!t.allclose(&(&t + 0.1), 1e-5, 1e-8, false)); - assert!(t.allclose(&(&t + 1e-9), 1e-5, 1e-8, false)); -} - -#[test] -fn set_data() { - let mut t = Tensor::from_slice(&[-1f32, 0., 1., 2., 120., 0.42]); - t.set_data(&t.to_kind(tch::Kind::BFloat16)); - assert_eq!(t.kind(), tch::Kind::BFloat16); -} - -#[test] -fn convert_vec() { - let t_1d = Tensor::from_slice(&[0, 1, 2, 3, 4, 5]); - let vec: Vec = Vec::try_from(t_1d).unwrap(); - assert_eq!(vec, vec![0, 1, 2, 3, 4, 5]); - - let t_2d = Tensor::from_slice(&[0, 1, 2, 3, 4, 5]).view((2, 3)); - let vec: Result, TchError> = Vec::try_from(t_2d); - assert!(matches!(vec, Err(TchError::Convert(msg)) if msg== - "Attempting to convert a Tensor with 2 dimensions to flat vector")); - - let t_2d = Tensor::from_slice(&[0, 1, 2, 3, 4, 5]).view((2, 3)); - let vec: Vec> = Vec::try_from(t_2d).unwrap(); - assert_eq!(vec, vec![vec![0, 1, 2], vec![3, 4, 5]]); -} - -#[test] -fn convert_ndarray() { - let t_1d = Tensor::from_slice(&[0, 1, 2, 3, 4, 5]); - let array_1d: ndarray::ArrayD = t_1d.as_ref().try_into().unwrap(); - assert_eq!(array_1d.as_slice(), ndarray::array![0, 1, 2, 3, 4, 5].as_slice()); - - let t_2d = Tensor::from_slice(&[0, 1, 2, 3, 4, 5]).view((2, 3)); - let array_2d: ndarray::ArrayD = t_2d.as_ref().try_into().unwrap(); - assert_eq!(array_2d.as_slice(), ndarray::array![[0, 1, 2], [3, 4, 5]].as_slice()); - - let t_3d = Tensor::from_slice(&[0, 1, 2, 3, 4, 5, 6, 7]).view((2, 2, 2)); - let array_3d: ndarray::ArrayD = t_3d.as_ref().try_into().unwrap(); - assert_eq!(array_3d.as_slice(), ndarray::array![[[0, 1], [2, 3]], [[4, 5], [6, 7]]].as_slice()); -} - -#[test] -fn fp8_tensor() { - let float8_types = [ - tch::Kind::Float8e5m2, - tch::Kind::Float8e4m3fn, - tch::Kind::Float8e5m2fnuz, - tch::Kind::Float8e4m3fnuz, - ]; - - for &kind in &float8_types { - let t = Tensor::from_slice(&[1.0f32, 2.0, 3.0, 4.0]).to_kind(kind); - assert_eq!(t.kind(), kind); - assert_eq!(t.kind().elt_size_in_bytes(), 1); - } -} diff --git a/v2/dependency-patches/tch-0.24.0/tests/test_utils.rs b/v2/dependency-patches/tch-0.24.0/tests/test_utils.rs deleted file mode 100644 index 29f7fe2c7b..0000000000 --- a/v2/dependency-patches/tch-0.24.0/tests/test_utils.rs +++ /dev/null @@ -1,44 +0,0 @@ -use tch::Tensor; - -pub fn from<'a, T>(t: &'a Tensor) -> T -where - >::Error: std::fmt::Debug, - T: TryFrom<&'a Tensor>, -{ - T::try_from(t).unwrap() -} - -#[allow(dead_code)] -pub fn f64_from(t: &Tensor) -> f64 { - from::(t) -} - -#[allow(dead_code)] -pub fn vec_f64_from(t: &Tensor) -> Vec { - from::>(&t.reshape(-1)) -} - -#[allow(dead_code)] -pub fn vec_f32_from(t: &Tensor) -> Vec { - from::>(&t.reshape(-1)) -} - -#[allow(dead_code)] -pub fn vec_f16_from(t: &Tensor) -> Vec { - from::>(&t.reshape(-1)) -} - -#[allow(dead_code)] -pub fn vec_i64_from(t: &Tensor) -> Vec { - from::>(&t.reshape(-1)) -} - -#[allow(dead_code)] -pub fn vec_i32_from(t: &Tensor) -> Vec { - from::>(&t.reshape(-1)) -} - -#[allow(dead_code)] -pub fn vec_bool_from(t: &Tensor) -> Vec { - from::>(&t.reshape(-1)) -} diff --git a/v2/dependency-patches/tch-0.24.0/tests/var_store.rs b/v2/dependency-patches/tch-0.24.0/tests/var_store.rs deleted file mode 100644 index bfc4882abf..0000000000 --- a/v2/dependency-patches/tch-0.24.0/tests/var_store.rs +++ /dev/null @@ -1,634 +0,0 @@ -use std::fs; -use tch::nn::OptimizerConfig; -use tch::{nn, nn::linear, nn::Init, nn::VarStore, Device, Kind, TchError, Tensor}; - -mod test_utils; -use test_utils::*; - -#[test] -fn path_components() { - let vs = VarStore::new(Device::Cpu); - let root = vs.root(); - let path = root.sub("a"); - let path = path.sub("test"); - assert_eq!(path.components().collect::>(), vec!["a", "test"]); -} - -#[test] -fn var_store_entry() { - let vs = VarStore::new(Device::Cpu); - let root = vs.root(); - - let t1 = root.entry("key").or_zeros(&[3, 1, 4]); - let t2 = root.entry("key").or_zeros(&[1, 5, 9]); - - assert_eq!(t1.size(), &[3, 1, 4]); - assert_eq!(t2.size(), &[3, 1, 4]); -} - -#[test] -fn save_and_load_var_store() { - let filename = - std::env::temp_dir().join(format!("tch-vs-load-complete-{}", std::process::id())); - let add = |vs: &tch::nn::Path| { - let v = vs.sub("a").sub("b").ones("t2", &[3]); - let u = vs.zeros("t1", &[4]); - let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); - let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); - (u, v) - }; - let mut vs1 = VarStore::new(Device::Cpu); - let (mut u1, mut v1) = add(&vs1.root()); - let mut vs2 = VarStore::new(Device::Cpu); - let (u2, v2) = add(&vs2.root()); - tch::no_grad(|| { - u1 += 42.0; - v1 *= 2.0; - }); - assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); - assert_eq!(f64_from(&v1.mean(Kind::Float)), 2.0); - assert_eq!(f64_from(&u2.mean(Kind::Float)), 0.0); - assert_eq!(f64_from(&v2.mean(Kind::Float)), 1.0); - vs1.save(&filename).unwrap(); - vs2.load(&filename).unwrap(); - assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); - assert_eq!(f64_from(&u2.mean(Kind::Float)), 42.0); - assert_eq!(f64_from(&v2.mean(Kind::Float)), 2.0); - fs::remove_file(&filename).unwrap(); - - // Save and reload in half-precision - vs1.half(); - let mut vs2 = VarStore::new(Device::Cpu); - let (u2, v2) = add(&vs2.root()); - vs1.save(&filename).unwrap(); - vs2.load(&filename).unwrap(); - assert_eq!(u2.kind(), Kind::Half); - assert_eq!(v2.kind(), Kind::Half); - assert_eq!(f64_from(&u2.mean(Kind::Half)), 42.0); - assert_eq!(f64_from(&v2.mean(Kind::Half)), 2.0); - fs::remove_file(filename).unwrap(); -} - -#[test] -fn save_to_stream_and_load_var_store() { - let filename = - std::env::temp_dir().join(format!("tch-vs-load-stream-complete-{}", std::process::id())); - let add = |vs: &tch::nn::Path| { - let v = vs.sub("a").sub("b").ones("t2", &[3]); - let u = vs.zeros("t1", &[4]); - let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); - let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); - (u, v) - }; - let mut vs1 = VarStore::new(Device::Cpu); - let mut vs2 = VarStore::new(Device::Cpu); - let (mut u1, mut v1) = add(&vs1.root()); - let (u2, v2) = add(&vs2.root()); - tch::no_grad(|| { - u1 += 42.0; - v1 *= 2.0; - }); - assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); - assert_eq!(f64_from(&v1.mean(Kind::Float)), 2.0); - assert_eq!(f64_from(&u2.mean(Kind::Float)), 0.0); - assert_eq!(f64_from(&v2.mean(Kind::Float)), 1.0); - vs1.save_to_stream(std::fs::File::create(&filename).unwrap()).unwrap(); - vs2.load(&filename).unwrap(); - assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); - assert_eq!(f64_from(&u2.mean(Kind::Float)), 42.0); - assert_eq!(f64_from(&v2.mean(Kind::Float)), 2.0); - fs::remove_file(&filename).unwrap(); - - // Save and reload in half-precision - vs1.half(); - let mut vs2 = VarStore::new(Device::Cpu); - let (u2, v2) = add(&vs2.root()); - vs1.save_to_stream(std::fs::File::create(&filename).unwrap()).unwrap(); - vs2.load(&filename).unwrap(); - assert_eq!(u2.kind(), Kind::Half); - assert_eq!(v2.kind(), Kind::Half); - assert_eq!(f64_from(&u2.mean(Kind::Half)), 42.0); - assert_eq!(f64_from(&v2.mean(Kind::Half)), 2.0); - fs::remove_file(filename).unwrap(); -} - -#[test] -fn save_and_load_from_stream_var_store() { - let filename = - std::env::temp_dir().join(format!("tch-vs-load-stream-complete2-{}", std::process::id())); - let add = |vs: &tch::nn::Path| { - let v = vs.sub("a").sub("b").ones("t2", &[3]); - let u = vs.zeros("t1", &[4]); - let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); - let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); - (u, v) - }; - let mut vs1 = VarStore::new(Device::Cpu); - let mut vs2 = VarStore::new(Device::Cpu); - let (mut u1, mut v1) = add(&vs1.root()); - let (u2, v2) = add(&vs2.root()); - tch::no_grad(|| { - u1 += 42.0; - v1 *= 2.0; - }); - assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); - assert_eq!(f64_from(&v1.mean(Kind::Float)), 2.0); - assert_eq!(f64_from(&u2.mean(Kind::Float)), 0.0); - assert_eq!(f64_from(&v2.mean(Kind::Float)), 1.0); - vs1.save(&filename).unwrap(); - vs2.load_from_stream(std::fs::File::open(&filename).unwrap()).unwrap(); - assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); - assert_eq!(f64_from(&u2.mean(Kind::Float)), 42.0); - assert_eq!(f64_from(&v2.mean(Kind::Float)), 2.0); - fs::remove_file(&filename).unwrap(); - - // Save and reload in half-precision - vs1.half(); - let mut vs2 = VarStore::new(Device::Cpu); - let (u2, v2) = add(&vs2.root()); - vs1.save(&filename).unwrap(); - vs2.load_from_stream(std::fs::File::open(&filename).unwrap()).unwrap(); - assert_eq!(u2.kind(), Kind::Half); - assert_eq!(v2.kind(), Kind::Half); - assert_eq!(f64_from(&u2.mean(Kind::Half)), 42.0); - assert_eq!(f64_from(&v2.mean(Kind::Half)), 2.0); - fs::remove_file(filename).unwrap(); -} - -#[test] -fn save_and_load_partial_var_store() { - let filename = - std::env::temp_dir().join(format!("tch-vs-partial-load-complete-{}", std::process::id())); - let add = |vs: &tch::nn::Path| { - let v = vs.sub("a").sub("b").ones("t2", &[3]); - let u = vs.zeros("t1", &[4]); - let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); - let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); - (u, v) - }; - let mut vs1 = VarStore::new(Device::Cpu); - let mut vs2 = VarStore::new(Device::Cpu); - let (mut u1, mut v1) = add(&vs1.root()); - let (u2, v2) = add(&vs2.root()); - tch::no_grad(|| { - u1 += 42.0; - v1 *= 2.0; - }); - assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); - assert_eq!(f64_from(&v1.mean(Kind::Float)), 2.0); - assert_eq!(f64_from(&u2.mean(Kind::Float)), 0.0); - assert_eq!(f64_from(&v2.mean(Kind::Float)), 1.0); - vs1.save(&filename).unwrap(); - let missing_variables = vs2.load_partial(&filename).unwrap(); - assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); - assert_eq!(f64_from(&u2.mean(Kind::Float)), 42.0); - assert_eq!(f64_from(&v2.mean(Kind::Float)), 2.0); - assert!(missing_variables.is_empty()); - fs::remove_file(&filename).unwrap(); - - // Save and reload in half-precision - vs1.half(); - let mut vs2 = VarStore::new(Device::Cpu); - let (u2, v2) = add(&vs2.root()); - vs1.save(&filename).unwrap(); - let missing_variables = vs2.load_partial(&filename).unwrap(); - assert_eq!(u2.kind(), Kind::Half); - assert_eq!(v2.kind(), Kind::Half); - assert_eq!(f64_from(&u2.mean(Kind::Half)), 42.0); - assert_eq!(f64_from(&v2.mean(Kind::Half)), 2.0); - assert!(missing_variables.is_empty()); - fs::remove_file(filename).unwrap(); -} - -#[test] -#[should_panic] -fn save_and_load_var_store_incomplete_file() { - let filename = - std::env::temp_dir().join(format!("tch-vs-load-incomplete-{}", std::process::id())); - let add = |vs: &tch::nn::Path| { - let u = vs.zeros("t1", &[4]); - let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); - u - }; - let add_partial = |vs: &tch::nn::Path| { - let v = vs.sub("a").sub("b").ones("t2", &[3]); - let u = vs.zeros("t1", &[4]); - let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); - (u, v) - }; - let vs1 = VarStore::new(Device::Cpu); - let mut vs2 = VarStore::new(Device::Cpu); - let mut u1 = add(&vs1.root()); - let (u2, v2) = add_partial(&vs2.root()); - tch::no_grad(|| { - u1 += 42.0; - }); - assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); - assert_eq!(f64_from(&u2.mean(Kind::Float)), 0.0); - assert_eq!(f64_from(&v2.mean(Kind::Float)), 1.0); - vs1.save(&filename).unwrap(); - vs2.load(&filename).unwrap(); - assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); - assert_eq!(f64_from(&u2.mean(Kind::Float)), 42.0); - assert_eq!(f64_from(&v2.mean(Kind::Float)), 1.0); - fs::remove_file(filename).unwrap(); -} - -#[test] -fn save_and_load_partial_var_store_incomplete_file() { - let filename = - std::env::temp_dir().join(format!("tch-vs-partial-load-incomplete-{}", std::process::id())); - let add = |vs: &tch::nn::Path| { - let u = vs.zeros("t1", &[4]); - let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); - u - }; - let add_partial = |vs: &tch::nn::Path| { - let v = vs.sub("a").sub("b").ones("t2", &[3]); - let u = vs.zeros("t1", &[4]); - let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); - (u, v) - }; - let mut vs1 = VarStore::new(Device::Cpu); - let mut vs2 = VarStore::new(Device::Cpu); - let mut u1 = add(&vs1.root()); - let (u2, v2) = add_partial(&vs2.root()); - tch::no_grad(|| { - u1 += 42.0; - }); - assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); - assert_eq!(f64_from(&u2.mean(Kind::Float)), 0.0); - assert_eq!(f64_from(&v2.mean(Kind::Float)), 1.0); - vs1.save(&filename).unwrap(); - let missing_variables = vs2.load_partial(&filename).unwrap(); - assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); - assert_eq!(f64_from(&u2.mean(Kind::Float)), 42.0); - assert_eq!(f64_from(&v2.mean(Kind::Float)), 1.0); - assert_eq!(missing_variables, vec!(String::from("a.b.t2"))); - fs::remove_file(&filename).unwrap(); - - // Save and reload in half-precision - vs1.half(); - let mut vs2 = VarStore::new(Device::Cpu); - let (u2, v2) = add_partial(&vs2.root()); - vs1.save(&filename).unwrap(); - let missing_variables = vs2.load_partial(&filename).unwrap(); - assert_eq!(u2.kind(), Kind::Half); - assert_eq!(v2.kind(), Kind::Float); - assert_eq!(f64_from(&u2.mean(Kind::Half)), 42.0); - assert_eq!(f64_from(&v2.mean(Kind::Float)), 1.0); - assert_eq!(missing_variables, vec!(String::from("a.b.t2"))); - fs::remove_file(&filename).unwrap(); -} - -#[test] -fn init_test() { - tch::manual_seed(42); - let vs = VarStore::new(Device::Cpu); - let zeros = vs.root().zeros("t1", &[3]); - assert_eq!(vec_f64_from(&zeros), [0., 0., 0.]); - let zeros = vs.root().var("t2", &[3], Init::Const(0.)); - assert_eq!(vec_f64_from(&zeros), [0., 0., 0.]); - let ones = vs.root().var("t3", &[3], Init::Const(1.)); - assert_eq!(vec_f64_from(&ones), [1., 1., 1.]); - let ones = vs.root().var("t4", &[3], Init::Const(0.5)); - assert_eq!(vec_f64_from(&ones), [0.5, 0.5, 0.5]); - let forty_two = vs.root().var("t4", &[2], Init::Const(42.)); - assert_eq!(vec_f64_from(&forty_two), [42., 42.]); - let uniform = vs.root().var("t5", &[100], Init::Uniform { lo: 1.0, up: 2.0 }); - let uniform_min = f64_from(&uniform.min()); - let uniform_max = f64_from(&uniform.max()); - assert!(uniform_min >= 1., "{}", "min {uniform_min}"); - assert!(uniform_max <= 2., "{}", "max {uniform_max}"); - let uniform_std = f64_from(&uniform.std(true)); - assert!(uniform_std > 0.15 && uniform_std < 0.35, "{}", "std {uniform_std}"); - let normal = vs.root().var("normal", &[100], Init::Randn { mean: 0., stdev: 0.02 }); - let normal_std = f64_from(&normal.std(true)); - assert!(normal_std <= 0.03, "{}", "std {normal_std}"); - let mut vs2 = VarStore::new(Device::Cpu); - let ones = vs2.root().ones("t1", &[3]); - assert_eq!(vec_f64_from(&ones), [1., 1., 1.]); - vs2.copy(&vs).unwrap(); - assert_eq!(vec_f64_from(&ones), [0., 0., 0.]); - let ortho = vs.root().var("orthogonal", &[100, 100], Init::Orthogonal { gain: 2.0 }); - let ortho_norm = f64_from(&ortho.linalg_norm_ord_str("fro", None::, true, Kind::Float)); - assert!( - f64::abs(ortho_norm - 20.) < 1e-5, - "{}", - "ortho_norm initialization failed {ortho_norm}" - ); - let ortho_shape_fail = - tch::nn::f_init(Init::Orthogonal { gain: 1.0 }, &[10], Device::Cpu, tch::Kind::Float); - assert!(ortho_shape_fail.is_err()); - let kaiming_u = vs.root().var("kaiming_u", &[20, 100], nn::init::DEFAULT_KAIMING_UNIFORM); - assert!(f64::abs(f64_from(&kaiming_u.mean(Kind::Float))) < 5e-3); - // The expected stdev is sqrt(2 / 100) - assert!(f64::abs(f64_from(&kaiming_u.std(true)) - (0.02f64).sqrt()) < 2e-3); - let kaiming_n = vs.root().var("kaiming_n", &[20, 100], nn::init::DEFAULT_KAIMING_NORMAL); - assert!(f64::abs(f64_from(&kaiming_n.mean(Kind::Float))) < 5e-3); - assert!(f64::abs(f64_from(&kaiming_n.std(true)) - (0.02f64).sqrt()) < 3e-3); -} - -fn check_param_group(mut opt: tch::nn::Optimizer, var_foo: Tensor, var_bar: Tensor) { - opt.set_lr(0.1); - opt.set_lr_group(0, 0.); - for _idx in 1..100 { - let loss = (&var_foo + &var_bar).mse_loss(&Tensor::from(0.42f32), tch::Reduction::Mean); - opt.backward_step(&loss); - } - assert_eq!(format!("{:.2}", f64_from(&var_foo)), "0.00"); - assert_eq!(format!("{:.2}", f64_from(&var_bar)), "0.42"); - opt.set_lr_group(0, 0.1); - for _idx in 1..100 { - let loss = (&var_foo + &var_bar).mse_loss(&Tensor::from(0f32), tch::Reduction::Mean); - opt.backward_step(&loss); - } - assert_eq!(format!("{:.2}", f64_from(&var_foo)), "-0.21"); - assert_eq!(format!("{:.2}", f64_from(&var_bar)), "0.21"); - opt.set_lr_group(7, 0.); - for _idx in 1..100 { - let loss = (&var_foo + &var_bar).mse_loss(&Tensor::from(0.22f32), tch::Reduction::Mean); - opt.backward_step(&loss); - } - assert_eq!(format!("{:.2}", f64_from(&var_foo)), "0.01"); - assert_eq!(format!("{:.2}", f64_from(&var_bar)), "0.21"); - // The following sets the learning rate for both groups. - opt.set_lr(0.); - for _idx in 1..100 { - let loss = (&var_foo + &var_bar).mse_loss(&Tensor::from(0.42f32), tch::Reduction::Mean); - opt.backward_step(&loss); - } - assert_eq!(format!("{:.2}", f64_from(&var_foo)), "0.01"); - assert_eq!(format!("{:.2}", f64_from(&var_bar)), "0.21"); - opt.set_lr(0.1); - for _idx in 1..100 { - let loss = (&var_foo + &var_bar).mse_loss(&Tensor::from(0.42f32), tch::Reduction::Mean); - opt.backward_step(&loss); - } - assert_eq!(format!("{:.2}", f64_from(&var_foo)), "0.11"); - assert_eq!(format!("{:.2}", f64_from(&var_bar)), "0.31"); -} - -#[test] -fn param_group() { - tch::manual_seed(42); - let vs = VarStore::new(Device::Cpu); - let opt = tch::nn::Sgd::default().build(&vs, 1.0).unwrap(); - let root = vs.root(); - let var_foo = root.set_group(0).zeros("var_foo", &[]); - let var_bar = root.set_group(7).zeros("var_bar", &[]); - check_param_group(opt, var_foo, var_bar); -} - -#[test] -fn save_and_load_with_group() { - let filename = std::env::temp_dir().join(format!("tch-vs-load-grp-{}", std::process::id())); - let add = |vs: &tch::nn::Path| { - let v = vs.set_group(1).sub("a").sub("b").ones("t2", &[3]); - let u = vs.zeros("t1", &[4]); - let _w = vs.sub("a").sub("b").sub("ccc").ones("t123", &[3]); - let _w = vs.sub("a").set_group(4).sub("b").sub("ccc").ones("t123", &[3]); - (u, v) - }; - let vs1 = VarStore::new(Device::Cpu); - let mut vs2 = VarStore::new(Device::Cpu); - let (mut u1, mut v1) = add(&vs1.root()); - let (u2, v2) = add(&vs2.root()); - tch::no_grad(|| { - u1 += 42.0; - v1 *= 2.0; - }); - assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); - assert_eq!(f64_from(&v1.mean(Kind::Float)), 2.0); - assert_eq!(f64_from(&u2.mean(Kind::Float)), 0.0); - assert_eq!(f64_from(&v2.mean(Kind::Float)), 1.0); - vs1.save(&filename).unwrap(); - vs2.load(&filename).unwrap(); - assert_eq!(f64_from(&u1.mean(Kind::Float)), 42.0); - assert_eq!(f64_from(&u2.mean(Kind::Float)), 42.0); - assert_eq!(f64_from(&v2.mean(Kind::Float)), 2.0); - fs::remove_file(filename).unwrap(); -} - -#[test] -fn param_group_weight_decay() { - tch::manual_seed(42); - let vs = VarStore::new(Device::Cpu); - let mut opt = tch::nn::Sgd::default().build(&vs, 0.0).unwrap(); - opt.set_lr(0.1); - let root = vs.root(); - let var_foo = root.set_group(0).zeros("var_foo", &[]); - let var_bar = root.set_group(7).zeros("var_bar", &[]); - for _idx in 1..100 { - let loss = (&var_foo + &var_bar).mse_loss(&Tensor::from(1f32), tch::Reduction::Mean); - opt.backward_step(&loss); - } - assert_eq!(format!("{:.2}", f64_from(&var_foo)), "0.50"); - assert_eq!(format!("{:.2}", f64_from(&var_bar)), "0.50"); - opt.set_weight_decay(0.1); - for _idx in 1..100 { - let loss = (&var_foo + &var_bar).mse_loss(&Tensor::from(1f32), tch::Reduction::Mean); - opt.backward_step(&loss); - } - assert_eq!(format!("{:.2}", f64_from(&var_foo)), "0.49"); - assert_eq!(format!("{:.2}", f64_from(&var_bar)), "0.49"); - opt.set_weight_decay_group(7, 0.); - for _idx in 1..100 { - let loss = (&var_foo + &var_bar).mse_loss(&Tensor::from(1f32), tch::Reduction::Mean); - opt.backward_step(&loss); - } - assert_eq!(format!("{:.2}", f64_from(&var_foo)), "0.30"); - assert_eq!(format!("{:.2}", f64_from(&var_bar)), "0.69"); -} - -#[test] -fn half_precision_conversion_entire_varstore() { - let mut vs = VarStore::new(Device::Cpu); - - let _ = vs.root().var("zeros", &[1], Init::Const(0.)); - let _ = vs.root().var("ones", &[1], Init::Const(1.)); - let _ = vs.root().var("forty_two", &[1], Init::Const(42.)); - - assert_eq!(vs.root().get("zeros").unwrap().kind(), Kind::Float); - assert_eq!(vs.root().get("ones").unwrap().kind(), Kind::Float); - assert_eq!(vs.root().get("forty_two").unwrap().kind(), Kind::Float); - - vs.half(); - - assert_eq!(vs.root().get("zeros").unwrap().kind(), Kind::Half); - assert_eq!(vs.root().get("ones").unwrap().kind(), Kind::Half); - assert_eq!(vs.root().get("forty_two").unwrap().kind(), Kind::Half); - - vs.bfloat16(); - - assert_eq!(vs.root().get("zeros").unwrap().kind(), Kind::BFloat16); - assert_eq!(vs.root().get("ones").unwrap().kind(), Kind::BFloat16); - assert_eq!(vs.root().get("forty_two").unwrap().kind(), Kind::BFloat16); - - vs.double(); - - assert_eq!(vs.root().get("zeros").unwrap().kind(), Kind::Double); - assert_eq!(vs.root().get("ones").unwrap().kind(), Kind::Double); - assert_eq!(vs.root().get("forty_two").unwrap().kind(), Kind::Double); - - vs.float(); - - assert_eq!(vs.root().get("zeros").unwrap().kind(), Kind::Float); - assert_eq!(vs.root().get("ones").unwrap().kind(), Kind::Float); - assert_eq!(vs.root().get("forty_two").unwrap().kind(), Kind::Float); - assert_eq!(format!("{:.2}", f64_from(&vs.root().get("zeros").unwrap())), "0.00"); - assert_eq!(format!("{:.2}", f64_from(&vs.root().get("ones").unwrap())), "1.00"); - assert_eq!(format!("{:.2}", f64_from(&vs.root().get("forty_two").unwrap())), "42.00"); -} - -#[test] -fn path_half_precision_conversion() { - let vs = VarStore::new(Device::Cpu); - - // Define a VarStore with 3 variables. 2 of them are in a sub-path named "convert" and - // will be cast to half-precision. The other variables in the VarStore will be unaffected - let _ = vs.root().sub("ignore").var("zeros", &[1], Init::Const(0.)); - let _ = vs.root().sub("convert").sub("group_1").var("ones", &[1], Init::Const(1.)); - let linear_layer = - linear(vs.root().sub("convert").sub("group_2").sub("linear"), 10, 42, Default::default()); - - assert_eq!(vs.root().sub("ignore").get("zeros").unwrap().kind(), Kind::Float); - assert_eq!(vs.root().sub("convert").sub("group_1").get("ones").unwrap().kind(), Kind::Float); - assert_eq!(linear_layer.ws.kind(), Kind::Float); - assert_eq!(linear_layer.bs.as_ref().unwrap().kind(), Kind::Float); - - vs.root().sub("convert").half(); - - assert_eq!(vs.root().sub("ignore").get("zeros").unwrap().kind(), Kind::Float); - assert_eq!(vs.root().sub("convert").sub("group_1").get("ones").unwrap().kind(), Kind::Half); - assert_eq!(linear_layer.ws.kind(), Kind::Half); - assert_eq!(linear_layer.bs.as_ref().unwrap().kind(), Kind::Half); - - vs.root().sub("convert").float(); - - assert_eq!(vs.root().sub("ignore").get("zeros").unwrap().kind(), Kind::Float); - assert_eq!(vs.root().sub("convert").sub("group_1").get("ones").unwrap().kind(), Kind::Float); - assert_eq!(linear_layer.ws.kind(), Kind::Float); - assert_eq!(linear_layer.bs.as_ref().unwrap().kind(), Kind::Float); -} - -#[test] -fn path_free_type_conversion() { - let mut vs = VarStore::new(Device::Cpu); - - // Define a VarStore with 3 variables. 2 of them are in a sub-path named "convert" and - // will be cast to half-precision. The other variables in the VarStore will be unaffected - let _ = vs.root().sub("ignore").var("zeros", &[1], Init::Const(0.)); - let _ = vs.root().sub("convert").sub("group_1").var("ones", &[1], Init::Const(1.)); - let _ = vs.root().sub("convert").sub("group_2").var("zeros", &[1], Init::Const(0.)); - - assert_eq!(vs.root().sub("ignore").get("zeros").unwrap().kind(), Kind::Float); - assert_eq!(vs.root().sub("convert").sub("group_1").get("ones").unwrap().kind(), Kind::Float); - assert_eq!(vs.root().sub("convert").sub("group_2").get("zeros").unwrap().kind(), Kind::Float); - - // Disable gradient tracking as this would raise an error when converting to a - // non-float type like bool. - vs.freeze(); - vs.root().sub("convert").set_kind(Kind::Bool); - - assert_eq!(vs.root().sub("ignore").get("zeros").unwrap().kind(), Kind::Float); - assert_eq!(vs.root().sub("convert").sub("group_1").get("ones").unwrap().kind(), Kind::Bool); - assert_eq!(vs.root().sub("convert").sub("group_2").get("zeros").unwrap().kind(), Kind::Bool); - - vs.root().sub("convert").set_kind(Kind::Float); - vs.unfreeze(); - - assert_eq!(vs.root().sub("ignore").get("zeros").unwrap().kind(), Kind::Float); - assert_eq!(vs.root().sub("convert").sub("group_1").get("ones").unwrap().kind(), Kind::Float); - assert_eq!(vs.root().sub("convert").sub("group_2").get("zeros").unwrap().kind(), Kind::Float); - - assert_eq!(format!("{:.2}", f64_from(&vs.root().sub("ignore").get("zeros").unwrap())), "0.00"); - assert_eq!( - format!("{:.2}", f64_from(&vs.root().sub("convert").sub("group_1").get("ones").unwrap())), - "1.00" - ); - assert_eq!( - format!("{:.2}", f64_from(&vs.root().sub("convert").sub("group_2").get("zeros").unwrap())), - "0.00" - ); -} - -#[test] -fn device_migration() { - if tch::Cuda::is_available() { - let mut vs = VarStore::new(Device::Cpu); - - let _ = vs.root().var("zeros", &[3], Init::Const(0.)); - let _ = vs.root().var("ones", &[3], Init::Const(1.)); - let linear_layer = linear(vs.root().sub("linear"), 10, 42, Default::default()); - - vs.set_device(Device::Cuda(0)); - - assert_eq!(vs.root().get("zeros").unwrap().device(), Device::Cuda(0)); - assert_eq!(vs.root().get("ones").unwrap().device(), Device::Cuda(0)); - assert_eq!(linear_layer.ws.device(), Device::Cuda(0)); - assert_eq!(linear_layer.bs.as_ref().unwrap().device(), Device::Cuda(0)); - assert_eq!(vs.device(), Device::Cuda(0)); - - vs.set_device(Device::Cpu); - - assert_eq!(vs.root().get("zeros").unwrap().device(), Device::Cpu); - assert_eq!(vs.root().get("ones").unwrap().device(), Device::Cpu); - assert_eq!(linear_layer.ws.device(), Device::Cpu); - assert_eq!(linear_layer.bs.as_ref().unwrap().device(), Device::Cpu); - assert_eq!(vs.device(), Device::Cpu); - } -} - -#[test] -fn merge_var_stores_no_prefixes() { - let vs_1 = VarStore::new(Device::Cpu); - let _ = vs_1.root().entry("key_1").or_zeros(&[3, 1, 4]); - let _ = vs_1.root().entry("key_2").or_zeros(&[1, 5, 9]); - - let vs_2 = VarStore::new(Device::Cpu); - let _ = vs_2.root().entry("key_3").or_zeros(&[2, 4, 4]); - let _ = vs_2.root().entry("key_4").or_zeros(&[5, 2, 3]); - - let merged_vs = VarStore::merge(vec![(vs_1, None), (vs_2, None)]).unwrap(); - assert_eq!(merged_vs.variables().len(), 4); - assert_eq!(merged_vs.trainable_variables().len(), 4); - assert!(merged_vs.variables().contains_key("key_1")); - assert!(merged_vs.variables().contains_key("key_2")); - assert!(merged_vs.variables().contains_key("key_3")); - assert!(merged_vs.variables().contains_key("key_4")); -} - -#[test] -fn merge_var_stores_conflicting_keys() { - let vs_1 = VarStore::new(Device::Cpu); - let _ = vs_1.root().entry("duplicate_key_1").or_zeros(&[3, 1, 4]); - let _ = vs_1.root().entry("key_2").or_zeros(&[1, 5, 9]); - - let vs_2 = VarStore::new(Device::Cpu); - let _ = vs_2.root().entry("duplicate_key_1").or_zeros(&[2, 4, 4]); - let _ = vs_2.root().entry("key_3").or_zeros(&[5, 2, 3]); - - let merged_vs = VarStore::merge(vec![(vs_1, None), (vs_2, None)]); - assert!(matches!(merged_vs, Err(TchError::Torch(t)) if t== - "Duplicate variable name found: duplicate_key_1. Provide a unique prefix to allow merge operation")); -} - -#[test] -fn merge_var_stores_with_prefixes() { - let vs_1 = VarStore::new(Device::Cpu); - let _ = vs_1.root().entry("key_1").or_zeros(&[3, 1, 4]); - let _ = vs_1.root().entry("key_2").or_zeros(&[1, 5, 9]); - - let vs_2 = VarStore::new(Device::Cpu); - let _ = vs_2.root().entry("key_3").or_zeros(&[2, 4, 4]); - let _ = vs_2.root().entry("key_4").or_zeros(&[5, 2, 3]); - - let merged_vs = VarStore::merge(vec![(vs_1, Some("vs_1.")), (vs_2, Some("vs_2."))]).unwrap(); - assert_eq!(merged_vs.variables().len(), 4); - assert_eq!(merged_vs.trainable_variables().len(), 4); - assert!(merged_vs.variables().contains_key("vs_1.key_1")); - assert!(merged_vs.variables().contains_key("vs_1.key_2")); - assert!(merged_vs.variables().contains_key("vs_2.key_3")); - assert!(merged_vs.variables().contains_key("vs_2.key_4")); -} diff --git a/v2/dependency-patches/tch-0.24.0/tests/vision_tests.rs b/v2/dependency-patches/tch-0.24.0/tests/vision_tests.rs deleted file mode 100644 index ad92e214bd..0000000000 --- a/v2/dependency-patches/tch-0.24.0/tests/vision_tests.rs +++ /dev/null @@ -1,34 +0,0 @@ -use tch::{nn, vision, Tensor}; - -#[test] -fn mobilenet() { - let vs = nn::VarStore::new(tch::Device::Cpu); - let net = vision::mobilenet::v2(&vs.root(), 1000); - let img = Tensor::zeros([1, 3, 224, 224], tch::kind::FLOAT_CPU); - let logits = img.apply_t(&net, false); - assert_eq!(logits.size(), [1, 1000]); -} - -#[test] -fn resize() { - // Check that resizing returns a tensor with the appropriate dimensions. - let img = Tensor::zeros([3, 8, 12], tch::kind::FLOAT_CPU); - let resized_img = vision::image::resize(&img, 16, 16).unwrap(); - assert_eq!(resized_img.size(), [3, 16, 16]); - let resized_img = vision::image::resize(&img, 4, 16).unwrap(); - assert_eq!(resized_img.size(), [3, 16, 4]); - let resized_img = vision::image::resize(&img, 32, 8).unwrap(); - assert_eq!(resized_img.size(), [3, 8, 32]); -} - -#[test] -fn resize_preserve_aspect_ratio() { - // Check that resizing returns a tensor with the appropriate dimensions. - let img = Tensor::zeros([3, 8, 12], tch::kind::FLOAT_CPU); - let resized_img = vision::image::resize_preserve_aspect_ratio(&img, 16, 16).unwrap(); - assert_eq!(resized_img.size(), [3, 16, 16]); - let resized_img = vision::image::resize(&img, 4, 16).unwrap(); - assert_eq!(resized_img.size(), [3, 16, 4]); - let resized_img = vision::image::resize(&img, 32, 8).unwrap(); - assert_eq!(resized_img.size(), [3, 8, 32]); -} diff --git a/v2/dependency-patches/tch-0.24.0/uv.lock b/v2/dependency-patches/tch-0.24.0/uv.lock deleted file mode 100644 index b0a9edbba1..0000000000 --- a/v2/dependency-patches/tch-0.24.0/uv.lock +++ /dev/null @@ -1,433 +0,0 @@ -version = 1 -revision = 3 -requires-python = ">=3.12" - -[[package]] -name = "cuda-bindings" -version = "13.2.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-pathfinder" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/52/c8/b2589d68acf7e3d63e2be330b84bc25712e97ed799affbca7edd7eae25d6/cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e865447abfb83d6a98ad5130ed3c70b1fc295ae3eeee39fd07b4ddb0671b6788", size = 5722404, upload-time = "2026-03-11T00:12:44.041Z" }, - { url = "https://files.pythonhosted.org/packages/1f/92/f899f7bbb5617bb65ec52a6eac1e9a1447a86b916c4194f8a5001b8cde0c/cuda_bindings-13.2.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46d8776a55d6d5da9dd6e9858fba2efcda2abe6743871dee47dd06eb8cb6d955", size = 6320619, upload-time = "2026-03-11T00:12:45.939Z" }, - { url = "https://files.pythonhosted.org/packages/df/93/eef988860a3ca985f82c4f3174fc0cdd94e07331ba9a92e8e064c260337f/cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6629ca2df6f795b784752409bcaedbd22a7a651b74b56a165ebc0c9dcbd504d0", size = 5614610, upload-time = "2026-03-11T00:12:50.337Z" }, - { url = "https://files.pythonhosted.org/packages/18/23/6db3aba46864aee357ab2415135b3fe3da7e9f1fa0221fa2a86a5968099c/cuda_bindings-13.2.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:7dca0da053d3b4cc4869eff49c61c03f3c5dbaa0bcd712317a358d5b8f3f385d", size = 6149914, upload-time = "2026-03-11T00:12:52.374Z" }, - { url = "https://files.pythonhosted.org/packages/c0/87/87a014f045b77c6de5c8527b0757fe644417b184e5367db977236a141602/cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a6464b30f46692d6c7f65d4a0e0450d81dd29de3afc1bb515653973d01c2cd6e", size = 5685673, upload-time = "2026-03-11T00:12:56.371Z" }, - { url = "https://files.pythonhosted.org/packages/ee/5e/c0fe77a73aaefd3fff25ffaccaac69c5a63eafdf8b9a4c476626ef0ac703/cuda_bindings-13.2.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f4af9f3e1be603fa12d5ad6cfca7844c9d230befa9792b5abdf7dd79979c3626", size = 6191386, upload-time = "2026-03-11T00:12:58.965Z" }, - { url = "https://files.pythonhosted.org/packages/5f/58/ed2c3b39c8dd5f96aa7a4abef0d47a73932c7a988e30f5fa428f00ed0da1/cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:df850a1ff8ce1b3385257b08e47b70e959932f5f432d0a4e46a355962b4e4771", size = 5507469, upload-time = "2026-03-11T00:13:04.063Z" }, - { url = "https://files.pythonhosted.org/packages/1f/01/0c941b112ceeb21439b05895eace78ca1aa2eaaf695c8521a068fd9b4c00/cuda_bindings-13.2.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e8a16384c6494e5485f39314b0b4afb04bee48d49edb16d5d8593fd35bbd231b", size = 6059693, upload-time = "2026-03-11T00:13:06.003Z" }, -] - -[[package]] -name = "cuda-pathfinder" -version = "1.3.3" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0b/02/4dbe7568a42e46582248942f54dc64ad094769532adbe21e525e4edf7bc4/cuda_pathfinder-1.3.3-py3-none-any.whl", hash = "sha256:9984b664e404f7c134954a771be8775dfd6180ea1e1aef4a5a37d4be05d9bbb1", size = 27154, upload-time = "2025-12-04T22:35:08.996Z" }, -] - -[[package]] -name = "cuda-toolkit" -version = "13.0.2" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/57/b2/453099f5f3b698d7d0eab38916aac44c7f76229f451709e2eb9db6615dcd/cuda_toolkit-13.0.2-py2.py3-none-any.whl", hash = "sha256:b198824cf2f54003f50d64ada3a0f184b42ca0846c1c94192fa269ecd97a66eb", size = 2364, upload-time = "2025-12-19T23:24:07.328Z" }, -] - -[package.optional-dependencies] -cublas = [ - { name = "nvidia-cublas", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -cudart = [ - { name = "nvidia-cuda-runtime", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -cufft = [ - { name = "nvidia-cufft", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -cufile = [ - { name = "nvidia-cufile", marker = "sys_platform == 'linux'" }, -] -cupti = [ - { name = "nvidia-cuda-cupti", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -curand = [ - { name = "nvidia-curand", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -cusolver = [ - { name = "nvidia-cusolver", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -cusparse = [ - { name = "nvidia-cusparse", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] -nvtx = [ - { name = "nvidia-nvtx", marker = "sys_platform == 'linux' or sys_platform == 'win32'" }, -] - -[[package]] -name = "filelock" -version = "3.20.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/1d/65/ce7f1b70157833bf3cb851b556a37d4547ceafc158aa9b34b36782f23696/filelock-3.20.3.tar.gz", hash = "sha256:18c57ee915c7ec61cff0ecf7f0f869936c7c30191bb0cf406f1341778d0834e1", size = 19485, upload-time = "2026-01-09T17:55:05.421Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b5/36/7fb70f04bf00bc646cd5bb45aa9eddb15e19437a28b8fb2b4a5249fac770/filelock-3.20.3-py3-none-any.whl", hash = "sha256:4b0dda527ee31078689fc205ec4f1c1bf7d56cf88b6dc9426c4f230e46c2dce1", size = 16701, upload-time = "2026-01-09T17:55:04.334Z" }, -] - -[[package]] -name = "fsspec" -version = "2025.10.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/24/7f/2747c0d332b9acfa75dc84447a066fdf812b5a6b8d30472b74d309bfe8cb/fsspec-2025.10.0.tar.gz", hash = "sha256:b6789427626f068f9a83ca4e8a3cc050850b6c0f71f99ddb4f542b8266a26a59", size = 309285, upload-time = "2025-10-30T14:58:44.036Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/02/a6b21098b1d5d6249b7c5ab69dde30108a71e4e819d4a9778f1de1d5b70d/fsspec-2025.10.0-py3-none-any.whl", hash = "sha256:7c7712353ae7d875407f97715f0e1ffcc21e33d5b24556cb1e090ae9409ec61d", size = 200966, upload-time = "2025-10-30T14:58:42.53Z" }, -] - -[[package]] -name = "jinja2" -version = "3.1.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markupsafe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, -] - -[[package]] -name = "markupsafe" -version = "3.0.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7e/99/7690b6d4034fffd95959cbe0c02de8deb3098cc577c67bb6a24fe5d7caa7/markupsafe-3.0.3.tar.gz", hash = "sha256:722695808f4b6457b320fdc131280796bdceb04ab50fe1795cd540799ebe1698", size = 80313, upload-time = "2025-09-27T18:37:40.426Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/72/147da192e38635ada20e0a2e1a51cf8823d2119ce8883f7053879c2199b5/markupsafe-3.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:d53197da72cc091b024dd97249dfc7794d6a56530370992a5e1a08983ad9230e", size = 11615, upload-time = "2025-09-27T18:36:30.854Z" }, - { url = "https://files.pythonhosted.org/packages/9a/81/7e4e08678a1f98521201c3079f77db69fb552acd56067661f8c2f534a718/markupsafe-3.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:1872df69a4de6aead3491198eaf13810b565bdbeec3ae2dc8780f14458ec73ce", size = 12020, upload-time = "2025-09-27T18:36:31.971Z" }, - { url = "https://files.pythonhosted.org/packages/1e/2c/799f4742efc39633a1b54a92eec4082e4f815314869865d876824c257c1e/markupsafe-3.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3a7e8ae81ae39e62a41ec302f972ba6ae23a5c5396c8e60113e9066ef893da0d", size = 24332, upload-time = "2025-09-27T18:36:32.813Z" }, - { url = "https://files.pythonhosted.org/packages/3c/2e/8d0c2ab90a8c1d9a24f0399058ab8519a3279d1bd4289511d74e909f060e/markupsafe-3.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d6dd0be5b5b189d31db7cda48b91d7e0a9795f31430b7f271219ab30f1d3ac9d", size = 22947, upload-time = "2025-09-27T18:36:33.86Z" }, - { url = "https://files.pythonhosted.org/packages/2c/54/887f3092a85238093a0b2154bd629c89444f395618842e8b0c41783898ea/markupsafe-3.0.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:94c6f0bb423f739146aec64595853541634bde58b2135f27f61c1ffd1cd4d16a", size = 21962, upload-time = "2025-09-27T18:36:35.099Z" }, - { url = "https://files.pythonhosted.org/packages/c9/2f/336b8c7b6f4a4d95e91119dc8521402461b74a485558d8f238a68312f11c/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:be8813b57049a7dc738189df53d69395eba14fb99345e0a5994914a3864c8a4b", size = 23760, upload-time = "2025-09-27T18:36:36.001Z" }, - { url = "https://files.pythonhosted.org/packages/32/43/67935f2b7e4982ffb50a4d169b724d74b62a3964bc1a9a527f5ac4f1ee2b/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:83891d0e9fb81a825d9a6d61e3f07550ca70a076484292a70fde82c4b807286f", size = 21529, upload-time = "2025-09-27T18:36:36.906Z" }, - { url = "https://files.pythonhosted.org/packages/89/e0/4486f11e51bbba8b0c041098859e869e304d1c261e59244baa3d295d47b7/markupsafe-3.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:77f0643abe7495da77fb436f50f8dab76dbc6e5fd25d39589a0f1fe6548bfa2b", size = 23015, upload-time = "2025-09-27T18:36:37.868Z" }, - { url = "https://files.pythonhosted.org/packages/2f/e1/78ee7a023dac597a5825441ebd17170785a9dab23de95d2c7508ade94e0e/markupsafe-3.0.3-cp312-cp312-win32.whl", hash = "sha256:d88b440e37a16e651bda4c7c2b930eb586fd15ca7406cb39e211fcff3bf3017d", size = 14540, upload-time = "2025-09-27T18:36:38.761Z" }, - { url = "https://files.pythonhosted.org/packages/aa/5b/bec5aa9bbbb2c946ca2733ef9c4ca91c91b6a24580193e891b5f7dbe8e1e/markupsafe-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:26a5784ded40c9e318cfc2bdb30fe164bdb8665ded9cd64d500a34fb42067b1c", size = 15105, upload-time = "2025-09-27T18:36:39.701Z" }, - { url = "https://files.pythonhosted.org/packages/e5/f1/216fc1bbfd74011693a4fd837e7026152e89c4bcf3e77b6692fba9923123/markupsafe-3.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:35add3b638a5d900e807944a078b51922212fb3dedb01633a8defc4b01a3c85f", size = 13906, upload-time = "2025-09-27T18:36:40.689Z" }, - { url = "https://files.pythonhosted.org/packages/38/2f/907b9c7bbba283e68f20259574b13d005c121a0fa4c175f9bed27c4597ff/markupsafe-3.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e1cf1972137e83c5d4c136c43ced9ac51d0e124706ee1c8aa8532c1287fa8795", size = 11622, upload-time = "2025-09-27T18:36:41.777Z" }, - { url = "https://files.pythonhosted.org/packages/9c/d9/5f7756922cdd676869eca1c4e3c0cd0df60ed30199ffd775e319089cb3ed/markupsafe-3.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:116bb52f642a37c115f517494ea5feb03889e04df47eeff5b130b1808ce7c219", size = 12029, upload-time = "2025-09-27T18:36:43.257Z" }, - { url = "https://files.pythonhosted.org/packages/00/07/575a68c754943058c78f30db02ee03a64b3c638586fba6a6dd56830b30a3/markupsafe-3.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:133a43e73a802c5562be9bbcd03d090aa5a1fe899db609c29e8c8d815c5f6de6", size = 24374, upload-time = "2025-09-27T18:36:44.508Z" }, - { url = "https://files.pythonhosted.org/packages/a9/21/9b05698b46f218fc0e118e1f8168395c65c8a2c750ae2bab54fc4bd4e0e8/markupsafe-3.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ccfcd093f13f0f0b7fdd0f198b90053bf7b2f02a3927a30e63f3ccc9df56b676", size = 22980, upload-time = "2025-09-27T18:36:45.385Z" }, - { url = "https://files.pythonhosted.org/packages/7f/71/544260864f893f18b6827315b988c146b559391e6e7e8f7252839b1b846a/markupsafe-3.0.3-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:509fa21c6deb7a7a273d629cf5ec029bc209d1a51178615ddf718f5918992ab9", size = 21990, upload-time = "2025-09-27T18:36:46.916Z" }, - { url = "https://files.pythonhosted.org/packages/c2/28/b50fc2f74d1ad761af2f5dcce7492648b983d00a65b8c0e0cb457c82ebbe/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a4afe79fb3de0b7097d81da19090f4df4f8d3a2b3adaa8764138aac2e44f3af1", size = 23784, upload-time = "2025-09-27T18:36:47.884Z" }, - { url = "https://files.pythonhosted.org/packages/ed/76/104b2aa106a208da8b17a2fb72e033a5a9d7073c68f7e508b94916ed47a9/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:795e7751525cae078558e679d646ae45574b47ed6e7771863fcc079a6171a0fc", size = 21588, upload-time = "2025-09-27T18:36:48.82Z" }, - { url = "https://files.pythonhosted.org/packages/b5/99/16a5eb2d140087ebd97180d95249b00a03aa87e29cc224056274f2e45fd6/markupsafe-3.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8485f406a96febb5140bfeca44a73e3ce5116b2501ac54fe953e488fb1d03b12", size = 23041, upload-time = "2025-09-27T18:36:49.797Z" }, - { url = "https://files.pythonhosted.org/packages/19/bc/e7140ed90c5d61d77cea142eed9f9c303f4c4806f60a1044c13e3f1471d0/markupsafe-3.0.3-cp313-cp313-win32.whl", hash = "sha256:bdd37121970bfd8be76c5fb069c7751683bdf373db1ed6c010162b2a130248ed", size = 14543, upload-time = "2025-09-27T18:36:51.584Z" }, - { url = "https://files.pythonhosted.org/packages/05/73/c4abe620b841b6b791f2edc248f556900667a5a1cf023a6646967ae98335/markupsafe-3.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:9a1abfdc021a164803f4d485104931fb8f8c1efd55bc6b748d2f5774e78b62c5", size = 15113, upload-time = "2025-09-27T18:36:52.537Z" }, - { url = "https://files.pythonhosted.org/packages/f0/3a/fa34a0f7cfef23cf9500d68cb7c32dd64ffd58a12b09225fb03dd37d5b80/markupsafe-3.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:7e68f88e5b8799aa49c85cd116c932a1ac15caaa3f5db09087854d218359e485", size = 13911, upload-time = "2025-09-27T18:36:53.513Z" }, - { url = "https://files.pythonhosted.org/packages/e4/d7/e05cd7efe43a88a17a37b3ae96e79a19e846f3f456fe79c57ca61356ef01/markupsafe-3.0.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:218551f6df4868a8d527e3062d0fb968682fe92054e89978594c28e642c43a73", size = 11658, upload-time = "2025-09-27T18:36:54.819Z" }, - { url = "https://files.pythonhosted.org/packages/99/9e/e412117548182ce2148bdeacdda3bb494260c0b0184360fe0d56389b523b/markupsafe-3.0.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:3524b778fe5cfb3452a09d31e7b5adefeea8c5be1d43c4f810ba09f2ceb29d37", size = 12066, upload-time = "2025-09-27T18:36:55.714Z" }, - { url = "https://files.pythonhosted.org/packages/bc/e6/fa0ffcda717ef64a5108eaa7b4f5ed28d56122c9a6d70ab8b72f9f715c80/markupsafe-3.0.3-cp313-cp313t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e885a3d1efa2eadc93c894a21770e4bc67899e3543680313b09f139e149ab19", size = 25639, upload-time = "2025-09-27T18:36:56.908Z" }, - { url = "https://files.pythonhosted.org/packages/96/ec/2102e881fe9d25fc16cb4b25d5f5cde50970967ffa5dddafdb771237062d/markupsafe-3.0.3-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8709b08f4a89aa7586de0aadc8da56180242ee0ada3999749b183aa23df95025", size = 23569, upload-time = "2025-09-27T18:36:57.913Z" }, - { url = "https://files.pythonhosted.org/packages/4b/30/6f2fce1f1f205fc9323255b216ca8a235b15860c34b6798f810f05828e32/markupsafe-3.0.3-cp313-cp313t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:b8512a91625c9b3da6f127803b166b629725e68af71f8184ae7e7d54686a56d6", size = 23284, upload-time = "2025-09-27T18:36:58.833Z" }, - { url = "https://files.pythonhosted.org/packages/58/47/4a0ccea4ab9f5dcb6f79c0236d954acb382202721e704223a8aafa38b5c8/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9b79b7a16f7fedff2495d684f2b59b0457c3b493778c9eed31111be64d58279f", size = 24801, upload-time = "2025-09-27T18:36:59.739Z" }, - { url = "https://files.pythonhosted.org/packages/6a/70/3780e9b72180b6fecb83a4814d84c3bf4b4ae4bf0b19c27196104149734c/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:12c63dfb4a98206f045aa9563db46507995f7ef6d83b2f68eda65c307c6829eb", size = 22769, upload-time = "2025-09-27T18:37:00.719Z" }, - { url = "https://files.pythonhosted.org/packages/98/c5/c03c7f4125180fc215220c035beac6b9cb684bc7a067c84fc69414d315f5/markupsafe-3.0.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8f71bc33915be5186016f675cd83a1e08523649b0e33efdb898db577ef5bb009", size = 23642, upload-time = "2025-09-27T18:37:01.673Z" }, - { url = "https://files.pythonhosted.org/packages/80/d6/2d1b89f6ca4bff1036499b1e29a1d02d282259f3681540e16563f27ebc23/markupsafe-3.0.3-cp313-cp313t-win32.whl", hash = "sha256:69c0b73548bc525c8cb9a251cddf1931d1db4d2258e9599c28c07ef3580ef354", size = 14612, upload-time = "2025-09-27T18:37:02.639Z" }, - { url = "https://files.pythonhosted.org/packages/2b/98/e48a4bfba0a0ffcf9925fe2d69240bfaa19c6f7507b8cd09c70684a53c1e/markupsafe-3.0.3-cp313-cp313t-win_amd64.whl", hash = "sha256:1b4b79e8ebf6b55351f0d91fe80f893b4743f104bff22e90697db1590e47a218", size = 15200, upload-time = "2025-09-27T18:37:03.582Z" }, - { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, - { url = "https://files.pythonhosted.org/packages/33/8a/8e42d4838cd89b7dde187011e97fe6c3af66d8c044997d2183fbd6d31352/markupsafe-3.0.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:eaa9599de571d72e2daf60164784109f19978b327a3910d3e9de8c97b5b70cfe", size = 11619, upload-time = "2025-09-27T18:37:06.342Z" }, - { url = "https://files.pythonhosted.org/packages/b5/64/7660f8a4a8e53c924d0fa05dc3a55c9cee10bbd82b11c5afb27d44b096ce/markupsafe-3.0.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:c47a551199eb8eb2121d4f0f15ae0f923d31350ab9280078d1e5f12b249e0026", size = 12029, upload-time = "2025-09-27T18:37:07.213Z" }, - { url = "https://files.pythonhosted.org/packages/da/ef/e648bfd021127bef5fa12e1720ffed0c6cbb8310c8d9bea7266337ff06de/markupsafe-3.0.3-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f34c41761022dd093b4b6896d4810782ffbabe30f2d443ff5f083e0cbbb8c737", size = 24408, upload-time = "2025-09-27T18:37:09.572Z" }, - { url = "https://files.pythonhosted.org/packages/41/3c/a36c2450754618e62008bf7435ccb0f88053e07592e6028a34776213d877/markupsafe-3.0.3-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:457a69a9577064c05a97c41f4e65148652db078a3a509039e64d3467b9e7ef97", size = 23005, upload-time = "2025-09-27T18:37:10.58Z" }, - { url = "https://files.pythonhosted.org/packages/bc/20/b7fdf89a8456b099837cd1dc21974632a02a999ec9bf7ca3e490aacd98e7/markupsafe-3.0.3-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:e8afc3f2ccfa24215f8cb28dcf43f0113ac3c37c2f0f0806d8c70e4228c5cf4d", size = 22048, upload-time = "2025-09-27T18:37:11.547Z" }, - { url = "https://files.pythonhosted.org/packages/9a/a7/591f592afdc734f47db08a75793a55d7fbcc6902a723ae4cfbab61010cc5/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ec15a59cf5af7be74194f7ab02d0f59a62bdcf1a537677ce67a2537c9b87fcda", size = 23821, upload-time = "2025-09-27T18:37:12.48Z" }, - { url = "https://files.pythonhosted.org/packages/7d/33/45b24e4f44195b26521bc6f1a82197118f74df348556594bd2262bda1038/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:0eb9ff8191e8498cca014656ae6b8d61f39da5f95b488805da4bb029cccbfbaf", size = 21606, upload-time = "2025-09-27T18:37:13.485Z" }, - { url = "https://files.pythonhosted.org/packages/ff/0e/53dfaca23a69fbfbbf17a4b64072090e70717344c52eaaaa9c5ddff1e5f0/markupsafe-3.0.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2713baf880df847f2bece4230d4d094280f4e67b1e813eec43b4c0e144a34ffe", size = 23043, upload-time = "2025-09-27T18:37:14.408Z" }, - { url = "https://files.pythonhosted.org/packages/46/11/f333a06fc16236d5238bfe74daccbca41459dcd8d1fa952e8fbd5dccfb70/markupsafe-3.0.3-cp314-cp314-win32.whl", hash = "sha256:729586769a26dbceff69f7a7dbbf59ab6572b99d94576a5592625d5b411576b9", size = 14747, upload-time = "2025-09-27T18:37:15.36Z" }, - { url = "https://files.pythonhosted.org/packages/28/52/182836104b33b444e400b14f797212f720cbc9ed6ba34c800639d154e821/markupsafe-3.0.3-cp314-cp314-win_amd64.whl", hash = "sha256:bdc919ead48f234740ad807933cdf545180bfbe9342c2bb451556db2ed958581", size = 15341, upload-time = "2025-09-27T18:37:16.496Z" }, - { url = "https://files.pythonhosted.org/packages/6f/18/acf23e91bd94fd7b3031558b1f013adfa21a8e407a3fdb32745538730382/markupsafe-3.0.3-cp314-cp314-win_arm64.whl", hash = "sha256:5a7d5dc5140555cf21a6fefbdbf8723f06fcd2f63ef108f2854de715e4422cb4", size = 14073, upload-time = "2025-09-27T18:37:17.476Z" }, - { url = "https://files.pythonhosted.org/packages/3c/f0/57689aa4076e1b43b15fdfa646b04653969d50cf30c32a102762be2485da/markupsafe-3.0.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:1353ef0c1b138e1907ae78e2f6c63ff67501122006b0f9abad68fda5f4ffc6ab", size = 11661, upload-time = "2025-09-27T18:37:18.453Z" }, - { url = "https://files.pythonhosted.org/packages/89/c3/2e67a7ca217c6912985ec766c6393b636fb0c2344443ff9d91404dc4c79f/markupsafe-3.0.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:1085e7fbddd3be5f89cc898938f42c0b3c711fdcb37d75221de2666af647c175", size = 12069, upload-time = "2025-09-27T18:37:19.332Z" }, - { url = "https://files.pythonhosted.org/packages/f0/00/be561dce4e6ca66b15276e184ce4b8aec61fe83662cce2f7d72bd3249d28/markupsafe-3.0.3-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b52b4fb9df4eb9ae465f8d0c228a00624de2334f216f178a995ccdcf82c4634", size = 25670, upload-time = "2025-09-27T18:37:20.245Z" }, - { url = "https://files.pythonhosted.org/packages/50/09/c419f6f5a92e5fadde27efd190eca90f05e1261b10dbd8cbcb39cd8ea1dc/markupsafe-3.0.3-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fed51ac40f757d41b7c48425901843666a6677e3e8eb0abcff09e4ba6e664f50", size = 23598, upload-time = "2025-09-27T18:37:21.177Z" }, - { url = "https://files.pythonhosted.org/packages/22/44/a0681611106e0b2921b3033fc19bc53323e0b50bc70cffdd19f7d679bb66/markupsafe-3.0.3-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:f190daf01f13c72eac4efd5c430a8de82489d9cff23c364c3ea822545032993e", size = 23261, upload-time = "2025-09-27T18:37:22.167Z" }, - { url = "https://files.pythonhosted.org/packages/5f/57/1b0b3f100259dc9fffe780cfb60d4be71375510e435efec3d116b6436d43/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:e56b7d45a839a697b5eb268c82a71bd8c7f6c94d6fd50c3d577fa39a9f1409f5", size = 24835, upload-time = "2025-09-27T18:37:23.296Z" }, - { url = "https://files.pythonhosted.org/packages/26/6a/4bf6d0c97c4920f1597cc14dd720705eca0bf7c787aebc6bb4d1bead5388/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:f3e98bb3798ead92273dc0e5fd0f31ade220f59a266ffd8a4f6065e0a3ce0523", size = 22733, upload-time = "2025-09-27T18:37:24.237Z" }, - { url = "https://files.pythonhosted.org/packages/14/c7/ca723101509b518797fedc2fdf79ba57f886b4aca8a7d31857ba3ee8281f/markupsafe-3.0.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:5678211cb9333a6468fb8d8be0305520aa073f50d17f089b5b4b477ea6e67fdc", size = 23672, upload-time = "2025-09-27T18:37:25.271Z" }, - { url = "https://files.pythonhosted.org/packages/fb/df/5bd7a48c256faecd1d36edc13133e51397e41b73bb77e1a69deab746ebac/markupsafe-3.0.3-cp314-cp314t-win32.whl", hash = "sha256:915c04ba3851909ce68ccc2b8e2cd691618c4dc4c4232fb7982bca3f41fd8c3d", size = 14819, upload-time = "2025-09-27T18:37:26.285Z" }, - { url = "https://files.pythonhosted.org/packages/1a/8a/0402ba61a2f16038b48b39bccca271134be00c5c9f0f623208399333c448/markupsafe-3.0.3-cp314-cp314t-win_amd64.whl", hash = "sha256:4faffd047e07c38848ce017e8725090413cd80cbc23d86e55c587bf979e579c9", size = 15426, upload-time = "2025-09-27T18:37:27.316Z" }, - { url = "https://files.pythonhosted.org/packages/70/bc/6f1c2f612465f5fa89b95bead1f44dcb607670fd42891d8fdcd5d039f4f4/markupsafe-3.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:32001d6a8fc98c8cb5c947787c5d08b0a50663d139f1305bac5885d98d9b40fa", size = 14146, upload-time = "2025-09-27T18:37:28.327Z" }, -] - -[[package]] -name = "mpmath" -version = "1.3.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e0/47/dd32fa426cc72114383ac549964eecb20ecfd886d1e5ccf5340b55b02f57/mpmath-1.3.0.tar.gz", hash = "sha256:7a28eb2a9774d00c7bc92411c19a89209d5da7c4c9a9e227be8330a23a25b91f", size = 508106, upload-time = "2023-03-07T16:47:11.061Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/43/e3/7d92a15f894aa0c9c4b49b8ee9ac9850d6e63b03c9c32c0367a13ae62209/mpmath-1.3.0-py3-none-any.whl", hash = "sha256:a0b2b9fe80bbcd81a6647ff13108738cfb482d481d826cc0e02f5b35e5c88d2c", size = 536198, upload-time = "2023-03-07T16:47:09.197Z" }, -] - -[[package]] -name = "networkx" -version = "3.5" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6c/4f/ccdb8ad3a38e583f214547fd2f7ff1fc160c43a75af88e6aec213404b96a/networkx-3.5.tar.gz", hash = "sha256:d4c6f9cf81f52d69230866796b82afbccdec3db7ae4fbd1b65ea750feed50037", size = 2471065, upload-time = "2025-05-29T11:35:07.804Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/eb/8d/776adee7bbf76365fdd7f2552710282c79a4ead5d2a46408c9043a2b70ba/networkx-3.5-py3-none-any.whl", hash = "sha256:0030d386a9a06dee3565298b4a734b68589749a544acbb6c412dc9e2489ec6ec", size = 2034406, upload-time = "2025-05-29T11:35:04.961Z" }, -] - -[[package]] -name = "nvidia-cublas" -version = "13.1.0.3" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e1/a5/fce49e2ae977e0ccc084e5adafceb4f0ac0c8333cb6863501618a7277f67/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:c86fc7f7ae36d7528288c5d88098edcb7b02c633d262e7ddbb86b0ad91be5df2", size = 542851226, upload-time = "2025-10-09T08:59:04.818Z" }, - { url = "https://files.pythonhosted.org/packages/e7/44/423ac00af4dd95a5aeb27207e2c0d9b7118702149bf4704c3ddb55bb7429/nvidia_cublas-13.1.0.3-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:ee8722c1f0145ab246bccb9e452153b5e0515fd094c3678df50b2a0888b8b171", size = 423133236, upload-time = "2025-10-09T08:59:32.536Z" }, -] - -[[package]] -name = "nvidia-cuda-cupti" -version = "13.0.85" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2a/2a/80353b103fc20ce05ef51e928daed4b6015db4aaa9162ed0997090fe2250/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_aarch64.whl", hash = "sha256:796bd679890ee55fb14a94629b698b6db54bcfd833d391d5e94017dd9d7d3151", size = 10310827, upload-time = "2025-09-04T08:26:42.012Z" }, - { url = "https://files.pythonhosted.org/packages/33/6d/737d164b4837a9bbd202f5ae3078975f0525a55730fe871d8ed4e3b952b0/nvidia_cuda_cupti-13.0.85-py3-none-manylinux_2_25_x86_64.whl", hash = "sha256:4eb01c08e859bf924d222250d2e8f8b8ff6d3db4721288cf35d14252a4d933c8", size = 10715597, upload-time = "2025-09-04T08:26:51.312Z" }, -] - -[[package]] -name = "nvidia-cuda-nvrtc" -version = "13.0.88" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c3/68/483a78f5e8f31b08fb1bb671559968c0ca3a065ac7acabfc7cee55214fd6/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:ad9b6d2ead2435f11cbb6868809d2adeeee302e9bb94bcf0539c7a40d80e8575", size = 90215200, upload-time = "2025-09-04T08:28:44.204Z" }, - { url = "https://files.pythonhosted.org/packages/b7/dc/6bb80850e0b7edd6588d560758f17e0550893a1feaf436807d64d2da040f/nvidia_cuda_nvrtc-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d27f20a0ca67a4bb34268a5e951033496c5b74870b868bacd046b1b8e0c3267b", size = 43015449, upload-time = "2025-09-04T08:28:20.239Z" }, -] - -[[package]] -name = "nvidia-cuda-runtime" -version = "13.0.96" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/87/4f/17d7b9b8e285199c58ce28e31b5c5bbaa4d8271af06a89b6405258245de2/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ef9bcbe90493a2b9d810e43d249adb3d02e98dd30200d86607d8d02687c43f55", size = 2261060, upload-time = "2025-10-09T08:55:15.78Z" }, - { url = "https://files.pythonhosted.org/packages/2e/24/d1558f3b68b1d26e706813b1d10aa1d785e4698c425af8db8edc3dced472/nvidia_cuda_runtime-13.0.96-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7f82250d7782aa23b6cfe765ecc7db554bd3c2870c43f3d1821f1d18aebf0548", size = 2243632, upload-time = "2025-10-09T08:55:36.117Z" }, -] - -[[package]] -name = "nvidia-cudnn-cu13" -version = "9.19.0.56" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cublas" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/f1/84/26025437c1e6b61a707442184fa0c03d083b661adf3a3eecfd6d21677740/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:6ed29ffaee1176c612daf442e4dd6cfeb6a0caa43ddcbeb59da94953030b1be4", size = 433781201, upload-time = "2026-02-03T20:40:53.805Z" }, - { url = "https://files.pythonhosted.org/packages/a3/22/0b4b932655d17a6da1b92fa92ab12844b053bb2ac2475e179ba6f043da1e/nvidia_cudnn_cu13-9.19.0.56-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:d20e1734305e9d68889a96e3f35094d733ff1f83932ebe462753973e53a572bf", size = 366066321, upload-time = "2026-02-03T20:44:52.837Z" }, -] - -[[package]] -name = "nvidia-cufft" -version = "12.0.0.61" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-nvjitlink" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, - { url = "https://files.pythonhosted.org/packages/a8/2f/7b57e29836ea8714f81e9898409196f47d772d5ddedddf1592eadb8ab743/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6c44f692dce8fd5ffd3e3df134b6cdb9c2f72d99cf40b62c32dde45eea9ddad3", size = 214085489, upload-time = "2025-09-04T08:31:56.044Z" }, -] - -[[package]] -name = "nvidia-cufile" -version = "1.15.1.6" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3f/70/4f193de89a48b71714e74602ee14d04e4019ad36a5a9f20c425776e72cd6/nvidia_cufile-1.15.1.6-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:08a3ecefae5a01c7f5117351c64f17c7c62efa5fffdbe24fc7d298da19cd0b44", size = 1223672, upload-time = "2025-09-04T08:32:22.779Z" }, - { url = "https://files.pythonhosted.org/packages/ab/73/cc4a14c9813a8a0d509417cf5f4bdaba76e924d58beb9864f5a7baceefbf/nvidia_cufile-1.15.1.6-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:bdc0deedc61f548bddf7733bdc216456c2fdb101d020e1ab4b88d232d5e2f6d1", size = 1136992, upload-time = "2025-09-04T08:32:14.119Z" }, -] - -[[package]] -name = "nvidia-curand" -version = "10.4.0.35" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/72/7c2ae24fb6b63a32e6ae5d241cc65263ea18d08802aaae087d9f013335a2/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:133df5a7509c3e292aaa2b477afd0194f06ce4ea24d714d616ff36439cee349a", size = 61962106, upload-time = "2025-08-04T10:21:41.128Z" }, - { url = "https://files.pythonhosted.org/packages/a5/9f/be0a41ca4a4917abf5cb9ae0daff1a6060cc5de950aec0396de9f3b52bc5/nvidia_curand-10.4.0.35-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:1aee33a5da6e1db083fe2b90082def8915f30f3248d5896bcec36a579d941bfc", size = 59544258, upload-time = "2025-08-04T10:22:03.992Z" }, -] - -[[package]] -name = "nvidia-cusolver" -version = "12.0.4.66" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-cublas" }, - { name = "nvidia-cusparse" }, - { name = "nvidia-nvjitlink" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, - { url = "https://files.pythonhosted.org/packages/5f/67/cba3777620cdacb99102da4042883709c41c709f4b6323c10781a9c3aa34/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_x86_64.whl", hash = "sha256:0a759da5dea5c0ea10fd307de75cdeb59e7ea4fcb8add0924859b944babf1112", size = 200941980, upload-time = "2025-09-04T08:33:22.767Z" }, -] - -[[package]] -name = "nvidia-cusparse" -version = "12.6.3.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "nvidia-nvjitlink" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, - { url = "https://files.pythonhosted.org/packages/fa/18/623c77619c31d62efd55302939756966f3ecc8d724a14dab2b75f1508850/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b3c89c88d01ee0e477cb7f82ef60a11a4bcd57b6b87c33f789350b59759360b", size = 145942937, upload-time = "2025-09-04T08:33:58.029Z" }, -] - -[[package]] -name = "nvidia-cusparselt-cu13" -version = "0.8.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/46/10/8dcd1175260706a2fc92a16a52e306b71d4c1ea0b0cc4a9484183399818a/nvidia_cusparselt_cu13-0.8.0-py3-none-manylinux2014_aarch64.whl", hash = "sha256:400c6ed1cf6780fc6efedd64ec9f1345871767e6a1a0a552a1ea0578117ea77c", size = 220791277, upload-time = "2025-08-13T19:22:40.982Z" }, - { url = "https://files.pythonhosted.org/packages/fd/53/43b0d71f4e702fa9733f8b4571fdca50a8813f1e450b656c239beff12315/nvidia_cusparselt_cu13-0.8.0-py3-none-manylinux2014_x86_64.whl", hash = "sha256:25e30a8a7323935d4ad0340b95a0b69926eee755767e8e0b1cf8dd85b197d3fd", size = 169884119, upload-time = "2025-08-13T19:23:41.967Z" }, -] - -[[package]] -name = "nvidia-nccl-cu13" -version = "2.28.9" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/39/55/1920646a2e43ffd4fc958536b276197ed740e9e0c54105b4bb3521591fc7/nvidia_nccl_cu13-2.28.9-py3-none-manylinux_2_18_aarch64.whl", hash = "sha256:01c873ba1626b54caa12272ed228dc5b2781545e0ae8ba3f432a8ef1c6d78643", size = 196561677, upload-time = "2025-11-18T05:49:03.45Z" }, - { url = "https://files.pythonhosted.org/packages/b0/b4/878fefaad5b2bcc6fcf8d474a25e3e3774bc5133e4b58adff4d0bca238bc/nvidia_nccl_cu13-2.28.9-py3-none-manylinux_2_18_x86_64.whl", hash = "sha256:e4553a30f34195f3fa1da02a6da3d6337d28f2003943aa0a3d247bbc25fefc42", size = 196493177, upload-time = "2025-11-18T05:49:17.677Z" }, -] - -[[package]] -name = "nvidia-nvjitlink" -version = "13.0.88" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/7a/123e033aaff487c77107195fa5a2b8686795ca537935a24efae476c41f05/nvidia_nvjitlink-13.0.88-py3-none-manylinux2010_x86_64.manylinux_2_12_x86_64.whl", hash = "sha256:13a74f429e23b921c1109976abefacc69835f2f433ebd323d3946e11d804e47b", size = 40713933, upload-time = "2025-09-04T08:35:43.553Z" }, - { url = "https://files.pythonhosted.org/packages/ab/2c/93c5250e64df4f894f1cbb397c6fd71f79813f9fd79d7cd61de3f97b3c2d/nvidia_nvjitlink-13.0.88-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:e931536ccc7d467a98ba1d8b89ff7fa7f1fa3b13f2b0069118cd7f47bff07d0c", size = 38768748, upload-time = "2025-09-04T08:35:20.008Z" }, -] - -[[package]] -name = "nvidia-nvshmem-cu13" -version = "3.4.5" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/0f/05cc9c720236dcd2db9c1ab97fff629e96821be2e63103569da0c9b72f19/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:6dc2a197f38e5d0376ad52cd1a2a3617d3cdc150fd5966f4aee9bcebb1d68fe9", size = 60215947, upload-time = "2025-09-06T00:32:20.022Z" }, - { url = "https://files.pythonhosted.org/packages/3c/35/a9bf80a609e74e3b000fef598933235c908fcefcef9026042b8e6dfde2a9/nvidia_nvshmem_cu13-3.4.5-py3-none-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:290f0a2ee94c9f3687a02502f3b9299a9f9fe826e6d0287ee18482e78d495b80", size = 60412546, upload-time = "2025-09-06T00:32:41.564Z" }, -] - -[[package]] -name = "nvidia-nvtx" -version = "13.0.85" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c2/f3/d86c845465a2723ad7e1e5c36dcd75ddb82898b3f53be47ebd429fb2fa5d/nvidia_nvtx-13.0.85-py3-none-manylinux1_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4936d1d6780fbe68db454f5e72a42ff64d1fd6397df9f363ae786930fd5c1cd4", size = 148047, upload-time = "2025-09-04T08:29:01.761Z" }, - { url = "https://files.pythonhosted.org/packages/a8/64/3708a90d1ebe202ffdeb7185f878a3c84d15c2b2c31858da2ce0583e2def/nvidia_nvtx-13.0.85-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:cb7780edb6b14107373c835bf8b72e7a178bac7367e23da7acb108f973f157a6", size = 148878, upload-time = "2025-09-04T08:28:53.627Z" }, -] - -[[package]] -name = "setuptools" -version = "80.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/18/5d/3bf57dcd21979b887f014ea83c24ae194cfcd12b9e0fda66b957c69d1fca/setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c", size = 1319958, upload-time = "2025-05-27T00:56:51.443Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/dc/17031897dae0efacfea57dfd3a82fdd2a2aeb58e0ff71b77b87e44edc772/setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922", size = 1201486, upload-time = "2025-05-27T00:56:49.664Z" }, -] - -[[package]] -name = "sympy" -version = "1.14.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "mpmath" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/83/d3/803453b36afefb7c2bb238361cd4ae6125a569b4db67cd9e79846ba2d68c/sympy-1.14.0.tar.gz", hash = "sha256:d3d3fe8df1e5a0b42f0e7bdf50541697dbe7d23746e894990c030e2b05e72517", size = 7793921, upload-time = "2025-04-27T18:05:01.611Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a2/09/77d55d46fd61b4a135c444fc97158ef34a095e5681d0a6c10b75bf356191/sympy-1.14.0-py3-none-any.whl", hash = "sha256:e091cc3e99d2141a0ba2847328f5479b05d94a6635cb96148ccb3f34671bd8f5", size = 6299353, upload-time = "2025-04-27T18:04:59.103Z" }, -] - -[[package]] -name = "tch-rs" -version = "0.1.0" -source = { virtual = "." } -dependencies = [ - { name = "torch" }, -] - -[package.metadata] -requires-dist = [{ name = "torch", specifier = "==2.11.0" }] - -[[package]] -name = "torch" -version = "2.11.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cuda-bindings", marker = "sys_platform == 'linux'" }, - { name = "cuda-toolkit", extra = ["cublas", "cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, - { name = "filelock" }, - { name = "fsspec" }, - { name = "jinja2" }, - { name = "networkx" }, - { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, - { name = "setuptools" }, - { name = "sympy" }, - { name = "triton", marker = "sys_platform == 'linux'" }, - { name = "typing-extensions" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/6f/8b/69e3008d78e5cee2b30183340cc425081b78afc5eff3d080daab0adda9aa/torch-2.11.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:4b5866312ee6e52ea625cd211dcb97d6a2cdc1131a5f15cc0d87eec948f6dd34", size = 80606338, upload-time = "2026-03-23T18:11:34.781Z" }, - { url = "https://files.pythonhosted.org/packages/13/16/42e5915ebe4868caa6bac83a8ed59db57f12e9a61b7d749d584776ed53d5/torch-2.11.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:f99924682ef0aa6a4ab3b1b76f40dc6e273fca09f367d15a524266db100a723f", size = 419731115, upload-time = "2026-03-23T18:11:06.944Z" }, - { url = "https://files.pythonhosted.org/packages/1a/c9/82638ef24d7877510f83baf821f5619a61b45568ce21c0a87a91576510aa/torch-2.11.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:0f68f4ac6d95d12e896c3b7a912b5871619542ec54d3649cf48cc1edd4dd2756", size = 530712279, upload-time = "2026-03-23T18:10:31.481Z" }, - { url = "https://files.pythonhosted.org/packages/1c/ff/6756f1c7ee302f6d202120e0f4f05b432b839908f9071157302cedfc5232/torch-2.11.0-cp312-cp312-win_amd64.whl", hash = "sha256:fbf39280699d1b869f55eac536deceaa1b60bd6788ba74f399cc67e60a5fab10", size = 114556047, upload-time = "2026-03-23T18:10:55.931Z" }, - { url = "https://files.pythonhosted.org/packages/87/89/5ea6722763acee56b045435fb84258db7375c48165ec8be7880ab2b281c5/torch-2.11.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:1e6debd97ccd3205bbb37eb806a9d8219e1139d15419982c09e23ef7d4369d18", size = 80606801, upload-time = "2026-03-23T18:10:18.649Z" }, - { url = "https://files.pythonhosted.org/packages/32/d1/8ed2173589cbfe744ed54e5a73efc107c0085ba5777ee93a5f4c1ab90553/torch-2.11.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:63a68fa59de8f87acc7e85a5478bb2dddbb3392b7593ec3e78827c793c4b73fd", size = 419732382, upload-time = "2026-03-23T18:08:30.835Z" }, - { url = "https://files.pythonhosted.org/packages/3d/e1/b73f7c575a4b8f87a5928f50a1e35416b5e27295d8be9397d5293e7e8d4c/torch-2.11.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:cc89b9b173d9adfab59fd227f0ab5e5516d9a52b658ae41d64e59d2e55a418db", size = 530711509, upload-time = "2026-03-23T18:08:47.213Z" }, - { url = "https://files.pythonhosted.org/packages/66/82/3e3fcdd388fbe54e29fd3f991f36846ff4ac90b0d0181e9c8f7236565f82/torch-2.11.0-cp313-cp313-win_amd64.whl", hash = "sha256:4dda3b3f52d121063a731ddb835f010dc137b920d7fec2778e52f60d8e4bf0cd", size = 114555842, upload-time = "2026-03-23T18:09:52.111Z" }, - { url = "https://files.pythonhosted.org/packages/db/38/8ac78069621b8c2b4979c2f96dc8409ef5e9c4189f6aac629189a78677ca/torch-2.11.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:8b394322f49af4362d4f80e424bcaca7efcd049619af03a4cf4501520bdf0fb4", size = 80959574, upload-time = "2026-03-23T18:10:14.214Z" }, - { url = "https://files.pythonhosted.org/packages/6d/6c/56bfb37073e7136e6dd86bfc6af7339946dd684e0ecf2155ac0eee687ae1/torch-2.11.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:2658f34ce7e2dabf4ec73b45e2ca68aedad7a5be87ea756ad656eaf32bf1e1ea", size = 419732324, upload-time = "2026-03-23T18:09:36.604Z" }, - { url = "https://files.pythonhosted.org/packages/07/f4/1b666b6d61d3394cca306ea543ed03a64aad0a201b6cd159f1d41010aeb1/torch-2.11.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:98bb213c3084cfe176302949bdc360074b18a9da7ab59ef2edc9d9f742504778", size = 530596026, upload-time = "2026-03-23T18:09:20.842Z" }, - { url = "https://files.pythonhosted.org/packages/48/6b/30d1459fa7e4b67e9e3fe1685ca1d8bb4ce7c62ef436c3a615963c6c866c/torch-2.11.0-cp313-cp313t-win_amd64.whl", hash = "sha256:a97b94bbf62992949b4730c6cd2cc9aee7b335921ee8dc207d930f2ed09ae2db", size = 114793702, upload-time = "2026-03-23T18:09:47.304Z" }, - { url = "https://files.pythonhosted.org/packages/26/0d/8603382f61abd0db35841148ddc1ffd607bf3100b11c6e1dab6d2fc44e72/torch-2.11.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:01018087326984a33b64e04c8cb5c2795f9120e0d775ada1f6638840227b04d7", size = 80573442, upload-time = "2026-03-23T18:09:10.117Z" }, - { url = "https://files.pythonhosted.org/packages/c7/86/7cd7c66cb9cec6be330fff36db5bd0eef386d80c031b581ec81be1d4b26c/torch-2.11.0-cp314-cp314-manylinux_2_28_aarch64.whl", hash = "sha256:2bb3cc54bd0dea126b0060bb1ec9de0f9c7f7342d93d436646516b0330cd5be7", size = 419749385, upload-time = "2026-03-23T18:07:33.77Z" }, - { url = "https://files.pythonhosted.org/packages/47/e8/b98ca2d39b2e0e4730c0ee52537e488e7008025bc77ca89552ff91021f7c/torch-2.11.0-cp314-cp314-manylinux_2_28_x86_64.whl", hash = "sha256:4dc8b3809469b6c30b411bb8c4cad3828efd26236153d9beb6a3ec500f211a60", size = 530716756, upload-time = "2026-03-23T18:07:50.02Z" }, - { url = "https://files.pythonhosted.org/packages/78/88/d4a4cda8362f8a30d1ed428564878c3cafb0d87971fbd3947d4c84552095/torch-2.11.0-cp314-cp314-win_amd64.whl", hash = "sha256:2b4e811728bd0cc58fb2b0948fe939a1ee2bf1422f6025be2fca4c7bd9d79718", size = 114552300, upload-time = "2026-03-23T18:09:05.617Z" }, - { url = "https://files.pythonhosted.org/packages/bf/46/4419098ed6d801750f26567b478fc185c3432e11e2cad712bc6b4c2ab0d0/torch-2.11.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:8245477871c3700d4370352ffec94b103cfcb737229445cf9946cddb7b2ca7cd", size = 80959460, upload-time = "2026-03-23T18:09:00.818Z" }, - { url = "https://files.pythonhosted.org/packages/fd/66/54a56a4a6ceaffb567231994a9745821d3af922a854ed33b0b3a278e0a99/torch-2.11.0-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:ab9a8482f475f9ba20e12db84b0e55e2f58784bdca43a854a6ccd3fd4b9f75e6", size = 419735835, upload-time = "2026-03-23T18:07:18.974Z" }, - { url = "https://files.pythonhosted.org/packages/b1/e7/0b6665f533aa9e337662dc190425abc0af1fe3234088f4454c52393ded61/torch-2.11.0-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:563ed3d25542d7e7bbc5b235ccfacfeb97fb470c7fee257eae599adb8005c8a2", size = 530613405, upload-time = "2026-03-23T18:08:07.014Z" }, - { url = "https://files.pythonhosted.org/packages/cf/bf/c8d12a2c86dbfd7f40fb2f56fbf5a505ccf2d9ce131eb559dfc7c51e1a04/torch-2.11.0-cp314-cp314t-win_amd64.whl", hash = "sha256:b2a43985ff5ef6ddd923bbcf99943e5f58059805787c5c9a2622bf05ca2965b0", size = 114792991, upload-time = "2026-03-23T18:08:19.216Z" }, -] - -[[package]] -name = "triton" -version = "3.6.0" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/17/5d/08201db32823bdf77a0e2b9039540080b2e5c23a20706ddba942924ebcd6/triton-3.6.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:374f52c11a711fd062b4bfbb201fd9ac0a5febd28a96fb41b4a0f51dde3157f4", size = 176128243, upload-time = "2026-01-20T16:16:07.857Z" }, - { url = "https://files.pythonhosted.org/packages/ab/a8/cdf8b3e4c98132f965f88c2313a4b493266832ad47fb52f23d14d4f86bb5/triton-3.6.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:74caf5e34b66d9f3a429af689c1c7128daba1d8208df60e81106b115c00d6fca", size = 188266850, upload-time = "2026-01-20T16:00:43.041Z" }, - { url = "https://files.pythonhosted.org/packages/3c/12/34d71b350e89a204c2c7777a9bba0dcf2f19a5bfdd70b57c4dbc5ffd7154/triton-3.6.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:448e02fe6dc898e9e5aa89cf0ee5c371e99df5aa5e8ad976a80b93334f3494fd", size = 176133521, upload-time = "2026-01-20T16:16:13.321Z" }, - { url = "https://files.pythonhosted.org/packages/f9/0b/37d991d8c130ce81a8728ae3c25b6e60935838e9be1b58791f5997b24a54/triton-3.6.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10c7f76c6e72d2ef08df639e3d0d30729112f47a56b0c81672edc05ee5116ac9", size = 188289450, upload-time = "2026-01-20T16:00:49.136Z" }, - { url = "https://files.pythonhosted.org/packages/ce/4e/41b0c8033b503fd3cfcd12392cdd256945026a91ff02452bef40ec34bee7/triton-3.6.0-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1722e172d34e32abc3eb7711d0025bb69d7959ebea84e3b7f7a341cd7ed694d6", size = 176276087, upload-time = "2026-01-20T16:16:18.989Z" }, - { url = "https://files.pythonhosted.org/packages/35/f8/9c66bfc55361ec6d0e4040a0337fb5924ceb23de4648b8a81ae9d33b2b38/triton-3.6.0-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:d002e07d7180fd65e622134fbd980c9a3d4211fb85224b56a0a0efbd422ab72f", size = 188400296, upload-time = "2026-01-20T16:00:56.042Z" }, - { url = "https://files.pythonhosted.org/packages/49/55/5ecf0dcaa0f2fbbd4420f7ef227ee3cb172e91e5fede9d0ecaddc43363b4/triton-3.6.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef5523241e7d1abca00f1d240949eebdd7c673b005edbbce0aca95b8191f1d43", size = 176138577, upload-time = "2026-01-20T16:16:25.426Z" }, - { url = "https://files.pythonhosted.org/packages/df/3d/9e7eee57b37c80cec63322c0231bb6da3cfe535a91d7a4d64896fcb89357/triton-3.6.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a17a5d5985f0ac494ed8a8e54568f092f7057ef60e1b0fa09d3fd1512064e803", size = 188273063, upload-time = "2026-01-20T16:01:07.278Z" }, - { url = "https://files.pythonhosted.org/packages/48/db/56ee649cab5eaff4757541325aca81f52d02d4a7cd3506776cad2451e060/triton-3.6.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0b3a97e8ed304dfa9bd23bb41ca04cdf6b2e617d5e782a8653d616037a5d537d", size = 176274804, upload-time = "2026-01-20T16:16:31.528Z" }, - { url = "https://files.pythonhosted.org/packages/f6/56/6113c23ff46c00aae423333eb58b3e60bdfe9179d542781955a5e1514cb3/triton-3.6.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:46bd1c1af4b6704e554cad2eeb3b0a6513a980d470ccfa63189737340c7746a7", size = 188397994, upload-time = "2026-01-20T16:01:14.236Z" }, -] - -[[package]] -name = "typing-extensions" -version = "4.15.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, -] diff --git a/v2/dependency-patches/torch-sys-0.24.0/Cargo.toml b/v2/dependency-patches/torch-sys-0.24.0/Cargo.toml deleted file mode 100644 index 248d98a17a..0000000000 --- a/v2/dependency-patches/torch-sys-0.24.0/Cargo.toml +++ /dev/null @@ -1,79 +0,0 @@ -# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO -# -# When uploading crates to the registry Cargo will automatically -# "normalize" Cargo.toml files for maximal compatibility -# with all versions of Cargo and also rewrite `path` dependencies -# to registry (e.g., crates.io) dependencies. -# -# If you are reading this file be aware that the original Cargo.toml -# will likely look very different (and much more reasonable). -# See Cargo.toml.orig for the original contents. - -[package] -edition = "2021" -name = "torch-sys" -version = "0.24.0" -authors = ["Laurent Mazare "] -build = "build.rs" -links = "tch" -autolib = false -autobins = false -autoexamples = false -autotests = false -autobenches = false -description = "Low-level FFI bindings for the PyTorch C++ api (libtorch)." -readme = false -keywords = [ - "pytorch", - "ffi", - "deep-learning", - "machine-learning", -] -categories = [ - "external-ffi-bindings", - "science", -] -license = "MIT/Apache-2.0" -repository = "https://github.com/LaurentMazare/tch-rs" - -[package.metadata.docs.rs] -features = ["doc-only"] - -[features] -doc-only = [] -download-libtorch = [ - "ureq", - "serde", - "serde_json", -] -python-extension = [] - -[lib] -name = "torch_sys" -path = "src/lib.rs" - -[dependencies.libc] -version = "0.2.0" - -[build-dependencies.anyhow] -version = "^1.0.60" - -[build-dependencies.cc] -version = "1.2" - -[build-dependencies.serde] -version = "1.0" -features = ["derive"] -optional = true - -[build-dependencies.serde_json] -version = "1.0" -optional = true - -[build-dependencies.ureq] -version = "2.6" -features = ["json"] -optional = true - -[build-dependencies.zip] -version = "9.0.1" diff --git a/v2/dependency-patches/torch-sys-0.24.0/Cargo.toml.orig b/v2/dependency-patches/torch-sys-0.24.0/Cargo.toml.orig deleted file mode 100644 index db835c217c..0000000000 --- a/v2/dependency-patches/torch-sys-0.24.0/Cargo.toml.orig +++ /dev/null @@ -1,32 +0,0 @@ -[package] -name = "torch-sys" -version = "0.24.0" -authors = ["Laurent Mazare "] -edition = "2021" -build = "build.rs" -links = "tch" - -description = "Low-level FFI bindings for the PyTorch C++ api (libtorch)." -repository = "https://github.com/LaurentMazare/tch-rs" -keywords = ["pytorch", "ffi", "deep-learning", "machine-learning"] -categories = ["external-ffi-bindings", "science"] -license = "MIT/Apache-2.0" - -[dependencies] -libc = "0.2.0" - -[build-dependencies] -anyhow = "^1.0.60" -cc = "1.2" -ureq = { version = "2.6", optional = true, features = ["json"] } -serde_json = { version = "1.0", optional = true } -serde = { version = "1.0", optional = true, features = ["derive"] } -zip = "0.6" - -[features] -download-libtorch = ["ureq", "serde", "serde_json"] -doc-only = [] -python-extension = [] - -[package.metadata.docs.rs] -features = [ "doc-only" ] diff --git a/v2/dependency-patches/torch-sys-0.24.0/build.rs b/v2/dependency-patches/torch-sys-0.24.0/build.rs deleted file mode 100644 index 9cb2e59859..0000000000 --- a/v2/dependency-patches/torch-sys-0.24.0/build.rs +++ /dev/null @@ -1,493 +0,0 @@ -// The LIBTORCH environment variable can be used to specify the directory -// where libtorch has been installed. -// When not specified this script downloads the cpu version for libtorch -// and extracts it in OUT_DIR. -// -// On Linux, the TORCH_CUDA_VERSION environment variable can be used, -// like 9.0, 90, or cu90 to specify the version of CUDA to use for libtorch. - -use anyhow::{Context, Result}; -use std::path::{Path, PathBuf}; -use std::{env, fs, io}; - -const TORCH_VERSION: &str = "2.11.0"; -const PYTHON_PRINT_PYTORCH_DETAILS: &str = r" -import torch -from torch.utils import cpp_extension -print('LIBTORCH_VERSION:', torch.__version__.split('+')[0]) -print('LIBTORCH_CXX11:', torch._C._GLIBCXX_USE_CXX11_ABI) -for include_path in cpp_extension.include_paths(): - print('LIBTORCH_INCLUDE:', include_path) -for library_path in cpp_extension.library_paths(): - print('LIBTORCH_LIB:', library_path) -"; - -const PYTHON_PRINT_INCLUDE_PATH: &str = r" -import sysconfig -print('PYTHON_INCLUDE:', sysconfig.get_path('include')) -"; - -const NO_DOWNLOAD_ERROR_MESSAGE: &str = r" -Cannot find a libtorch install, you can either: -- Install libtorch manually and set the LIBTORCH environment variable to appropriate path. -- Use a system wide install in /usr/lib/libtorch.so. -- Use a Python environment with PyTorch installed by setting LIBTORCH_USE_PYTORCH=1 - -See the readme for more details: -https://github.com/LaurentMazare/tch-rs/blob/main/README.md -"; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum LinkType { - Dynamic, - Static, -} - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum Os { - Linux, - Macos, - Windows, -} - -#[allow(dead_code)] -#[derive(Debug, Clone)] -struct SystemInfo { - os: Os, - python_interpreter: PathBuf, - cxx11_abi: String, - libtorch_include_dirs: Vec, - libtorch_lib_dir: PathBuf, - link_type: LinkType, -} - -#[cfg(feature = "ureq")] -fn download>(source_url: &str, target_file: P) -> anyhow::Result<()> { - let f = fs::File::create(&target_file)?; - let mut writer = io::BufWriter::new(f); - let response = ureq::get(source_url).call()?; - let response_code = response.status(); - if response_code != 200 { - anyhow::bail!("Unexpected response code {} for {}", response_code, source_url) - } - let mut reader = response.into_reader(); - std::io::copy(&mut reader, &mut writer)?; - Ok(()) -} - -#[cfg(not(feature = "ureq"))] -fn download>(_source_url: &str, _target_file: P) -> anyhow::Result<()> { - anyhow::bail!("cannot use download without the ureq feature") -} - -#[cfg(not(feature = "download-libtorch"))] -fn get_pypi_wheel_url_for_aarch64_macosx() -> anyhow::Result { - anyhow::bail!("cannot get pypi wheel url without the ureq feature") -} - -#[cfg(feature = "download-libtorch")] -#[derive(serde::Deserialize, Debug)] -struct PyPiPackageUrl { - url: String, - filename: String, -} -#[cfg(feature = "download-libtorch")] -#[derive(serde::Deserialize, Debug)] -struct PyPiPackage { - urls: Vec, -} -#[cfg(feature = "download-libtorch")] -fn get_pypi_wheel_url_for_aarch64_macosx() -> anyhow::Result { - let pypi_url = format!("https://pypi.org/pypi/torch/{TORCH_VERSION}/json"); - let response = ureq::get(pypi_url.as_str()).call()?; - let response_code = response.status(); - if response_code != 200 { - anyhow::bail!("Unexpected response code {} for {}", response_code, pypi_url) - } - let pypi_package: PyPiPackage = response.into_json()?; - let urls = pypi_package.urls; - let expected_filename = format!("torch-{TORCH_VERSION}-cp311-none-macosx_11_0_arm64.whl"); - let url = urls.iter().find_map(|pypi_url: &PyPiPackageUrl| { - if pypi_url.filename == expected_filename { - Some(pypi_url.url.clone()) - } else { - None - } - }); - url.context("Failed to find arm64 macosx wheel from pypi") -} - -fn extract>(filename: P, outpath: P) -> anyhow::Result<()> { - let file = fs::File::open(&filename)?; - let buf = io::BufReader::new(file); - let mut archive = zip::ZipArchive::new(buf)?; - // ZIP 9's checked extraction rejects path/symlink escapes and unsafe permissions. - archive.extract(outpath.as_ref())?; - - // This is if we're unzipping a python wheel. - if outpath.as_ref().join("torch").exists() { - fs::rename(outpath.as_ref().join("torch"), outpath.as_ref().join("libtorch"))?; - } - Ok(()) -} - -fn env_var_rerun(name: &str) -> Result { - println!("cargo:rerun-if-env-changed={name}"); - env::var(name) -} - -fn version_check(version: &str) -> Result<()> { - if env_var_rerun("LIBTORCH_BYPASS_VERSION_CHECK").is_ok() { - return Ok(()); - } - let version = version.trim(); - // Typical version number is 2.11.0+cpu or 2.11.0+cu129 - let version = match version.split_once('+') { - None => version, - Some((version, _)) => version, - }; - if version != TORCH_VERSION { - anyhow::bail!("this tch version expects PyTorch {TORCH_VERSION}, got {version}, this check can be bypassed by setting the LIBTORCH_BYPASS_VERSION_CHECK environment variable") - } - Ok(()) -} - -impl SystemInfo { - fn new() -> Result { - let os = match env::var("CARGO_CFG_TARGET_OS").expect("Unable to get TARGET_OS").as_str() { - "linux" => Os::Linux, - "windows" => Os::Windows, - "macos" => Os::Macos, - os => anyhow::bail!("unsupported TARGET_OS '{os}'"), - }; - // Locate the currently active Python binary, similar to: - // https://github.com/PyO3/maturin/blob/243b8ec91d07113f97a6fe74d9b2dcb88086e0eb/src/target.rs#L547 - let python_interpreter = match os { - Os::Windows => PathBuf::from("python.exe"), - Os::Linux | Os::Macos => { - if env::var_os("VIRTUAL_ENV").is_some() { - PathBuf::from("python") - } else { - PathBuf::from("python3") - } - } - }; - let mut libtorch_include_dirs = vec![]; - if cfg!(feature = "python-extension") { - let output = std::process::Command::new(&python_interpreter) - .arg("-c") - .arg(PYTHON_PRINT_INCLUDE_PATH) - .output() - .with_context(|| format!("error running {python_interpreter:?}"))?; - for line in String::from_utf8_lossy(&output.stdout).lines() { - if let Some(path) = line.strip_prefix("PYTHON_INCLUDE: ") { - libtorch_include_dirs.push(PathBuf::from(path)) - } - } - } - let mut libtorch_lib_dir = None; - let cxx11_abi = if env_var_rerun("LIBTORCH_USE_PYTORCH").is_ok() { - let output = std::process::Command::new(&python_interpreter) - .arg("-c") - .arg(PYTHON_PRINT_PYTORCH_DETAILS) - .output() - .with_context(|| format!("error running {python_interpreter:?}"))?; - let mut cxx11_abi = None; - for line in String::from_utf8_lossy(&output.stdout).lines() { - if let Some(version) = line.strip_prefix("LIBTORCH_VERSION: ") { - version_check(version)? - } - match line.strip_prefix("LIBTORCH_CXX11: ") { - Some("True") => cxx11_abi = Some("1".to_owned()), - Some("False") => cxx11_abi = Some("0".to_owned()), - _ => {} - } - if let Some(path) = line.strip_prefix("LIBTORCH_INCLUDE: ") { - libtorch_include_dirs.push(PathBuf::from(path)) - } - if let Some(path) = line.strip_prefix("LIBTORCH_LIB: ") { - libtorch_lib_dir = Some(PathBuf::from(path)) - } - } - match cxx11_abi { - Some(cxx11_abi) => cxx11_abi, - None => anyhow::bail!("no cxx11 abi returned by python {output:?}"), - } - } else { - let libtorch = Self::prepare_libtorch_dir(os)?; - let includes = env_var_rerun("LIBTORCH_INCLUDE") - .map(PathBuf::from) - .unwrap_or_else(|_| libtorch.clone()); - let lib = env_var_rerun("LIBTORCH_LIB") - .map(PathBuf::from) - .unwrap_or_else(|_| libtorch.clone()); - let mut version_file = libtorch; - version_file.push("build-version"); - if version_file.exists() { - if let Ok(version) = std::fs::read_to_string(&version_file) { - version_check(&version)? - } - } - libtorch_include_dirs.push(includes.join("include")); - libtorch_include_dirs.push(includes.join("include/torch/csrc/api/include")); - libtorch_lib_dir = Some(lib.join("lib")); - env_var_rerun("LIBTORCH_CXX11_ABI").unwrap_or_else(|_| "1".to_owned()) - }; - let libtorch_lib_dir = libtorch_lib_dir.expect("no libtorch lib dir found"); - let link_type = match env_var_rerun("LIBTORCH_STATIC").as_deref() { - Err(_) | Ok("0") | Ok("false") | Ok("FALSE") => LinkType::Dynamic, - Ok(_) => LinkType::Static, - }; - Ok(Self { - os, - python_interpreter, - cxx11_abi, - libtorch_include_dirs, - libtorch_lib_dir, - link_type, - }) - } - - fn check_system_location(os: Os) -> Option { - match os { - Os::Linux => Path::new("/usr/lib/libtorch.so").exists().then(|| PathBuf::from("/usr")), - _ => None, - } - } - - fn prepare_libtorch_dir(os: Os) -> Result { - if let Ok(libtorch) = env_var_rerun("LIBTORCH") { - Ok(PathBuf::from(libtorch)) - } else if let Some(pathbuf) = Self::check_system_location(os) { - Ok(pathbuf) - } else { - if !cfg!(feature = "download-libtorch") { - anyhow::bail!(NO_DOWNLOAD_ERROR_MESSAGE) - } - - let device = match env_var_rerun("TORCH_CUDA_VERSION") { - Ok(cuda_env) => match os { - Os::Linux | Os::Windows => cuda_env - .trim() - .to_lowercase() - .trim_start_matches("cu") - .split('.') - .take(2) - .fold("cu".to_owned(), |mut acc, curr| { - acc += curr; - acc - }), - os => anyhow::bail!( - "CUDA was specified with `TORCH_CUDA_VERSION`, but pre-built \ - binaries with CUDA are only available for Linux and Windows, not: {os:?}.", - ), - }, - Err(_) => "cpu".to_owned(), - }; - - let libtorch_dir = - PathBuf::from(env::var("OUT_DIR").context("OUT_DIR variable not set")?) - .join("libtorch"); - if !libtorch_dir.exists() { - fs::create_dir(&libtorch_dir).unwrap_or_default(); - let libtorch_url = match os { - Os::Linux => format!( - "https://download.pytorch.org/libtorch/{}/libtorch-shared-with-deps-{}{}.zip", - device, TORCH_VERSION, match device.as_ref() { - "cpu" => "%2Bcpu", - "cu118" => "%2Bcu118", - "cu121" => "%2Bcu121", - "cu124" => "%2Bcu124", - "cu126" => "%2Bcu126", - "cu128" => "%2Bcu128", - _ => anyhow::bail!("unsupported device {device}, TORCH_CUDA_VERSION may be set incorrectly?"), - } - ), - Os::Macos => { - if env::var("CARGO_CFG_TARGET_ARCH") == Ok(String::from("aarch64")) { - get_pypi_wheel_url_for_aarch64_macosx().expect( - "Failed to retrieve torch from pypi. Pre-built version of libtorch for apple silicon are not available. - You can install torch manually following the indications from https://github.com/LaurentMazare/tch-rs/issues/629 - pip3 install torch=={TORCH_VERSION} - Then update the following environment variables: - export LIBTORCH=$(python3 -c 'import torch; from pathlib import Path; print(Path(torch.__file__).parent)') - export DYLD_LIBRARY_PATH=${{LIBTORCH}}/lib - ") - } else { - format!("https://download.pytorch.org/libtorch/cpu/libtorch-macos-x86_64-{TORCH_VERSION}.zip") - } - }, - Os::Windows => format!( - "https://download.pytorch.org/libtorch/{}/libtorch-win-shared-with-deps-{}{}.zip", - device, TORCH_VERSION, match device.as_ref() { - "cpu" => "%2Bcpu", - "cu118" => "%2Bcu118", - "cu121" => "%2Bcu121", - "cu124" => "%2Bcu124", - "cu126" => "%2Bcu126", - "cu128" => "%2Bcu128", - _ => "" - }), - }; - - let filename = libtorch_dir.join(format!("v{TORCH_VERSION}.zip")); - download(&libtorch_url, &filename)?; - extract(&filename, &libtorch_dir)?; - } - Ok(libtorch_dir.join("libtorch")) - } - } - - fn make(&self) { - println!("cargo:rerun-if-changed=libtch/torch_python.cpp"); - println!("cargo:rerun-if-changed=libtch/torch_python.h"); - println!("cargo:rerun-if-changed=libtch/torch_api_generated.cpp"); - println!("cargo:rerun-if-changed=libtch/torch_api_generated.h"); - println!("cargo:rerun-if-changed=libtch/torch_api.cpp"); - println!("cargo:rerun-if-changed=libtch/torch_api.h"); - println!("cargo:rerun-if-changed=libtch/stb_image_write.h"); - println!("cargo:rerun-if-changed=libtch/stb_image_resize.h"); - println!("cargo:rerun-if-changed=libtch/stb_image.h"); - let mut c_files = vec!["libtch/torch_api.cpp", "libtch/torch_api_generated.cpp"]; - if cfg!(feature = "python-extension") { - c_files.push("libtch/torch_python.cpp") - } - - match self.os { - Os::Linux | Os::Macos => { - // Pass the libtorch lib dir to crates that use torch-sys. This will be available - // as DEP_TORCH_SYS_LIBTORCH_LIB, see: - // https://doc.rust-lang.org/cargo/reference/build-scripts.html#the-links-manifest-key - println!("cargo:libtorch_lib={}", self.libtorch_lib_dir.display()); - cc::Build::new() - .cpp(true) - .pic(true) - .warnings(false) - .includes(&self.libtorch_include_dirs) - .flag(format!("-Wl,-rpath={}", self.libtorch_lib_dir.display())) - .flag("-std=c++17") - .flag(format!("-D_GLIBCXX_USE_CXX11_ABI={}", self.cxx11_abi)) - .flag("-DGLOG_USE_GLOG_EXPORT") - .files(&c_files) - .compile("tch"); - } - Os::Windows => { - // TODO: Pass "/link" "LIBPATH:{}" to cl.exe in order to emulate rpath. - // Not yet supported by cc=rs. - // https://github.com/alexcrichton/cc-rs/issues/323 - cc::Build::new() - .cpp(true) - .pic(true) - .warnings(false) - .includes(&self.libtorch_include_dirs) - .flag("/std:c++17") - .flag("/p:DefineConstants=GLOG_USE_GLOG_EXPORT") - .files(&c_files) - .compile("tch"); - } - }; - } - - fn link(&self, lib_name: &str) { - match self.link_type { - LinkType::Dynamic => println!("cargo:rustc-link-lib={lib_name}"), - LinkType::Static => { - // TODO: whole-archive might only be necessary for libtorch_cpu? - println!("cargo:rustc-link-lib=static:+whole-archive,-bundle={lib_name}") - } - } - } -} - -fn main() -> anyhow::Result<()> { - if !cfg!(feature = "doc-only") { - let system_info = SystemInfo::new()?; - // use_cuda is a hacky way to detect whether cuda is available and - // if it's the case link to it by explicitly depending on a symbol - // from the torch_cuda library. - // It would be better to use -Wl,--no-as-needed but there is no way - // to specify arbitrary linker flags at the moment. - // - // Once https://github.com/rust-lang/cargo/pull/8441 is available - // we should switch to using rustc-link-arg instead e.g. with the - // following flags: - // -Wl,--no-as-needed -Wl,--copy-dt-needed-entries -ltorch - // - // This will be available starting from cargo 1.50 but will be a nightly - // only option to start with. - // https://github.com/rust-lang/cargo/blob/master/CHANGELOG.md - // - // Update: The above doesn't seem to propagate to the downstream binaries - // so doesn't really help, the comment has been kept though to keep track - // if this issue. - // TODO: Try out the as-needed native link modifier when it lands. - // https://github.com/rust-lang/rust/issues/99424 - // - // Update: it seems that the dummy dependency is not necessary anymore, so just - // removing it and keeping this comment around for legacy. - let si_lib = &system_info.libtorch_lib_dir; - let use_cuda = - si_lib.join("libtorch_cuda.so").exists() || si_lib.join("torch_cuda.dll").exists(); - let use_cuda_cu = si_lib.join("libtorch_cuda_cu.so").exists() - || si_lib.join("torch_cuda_cu.dll").exists(); - let use_cuda_cpp = si_lib.join("libtorch_cuda_cpp.so").exists() - || si_lib.join("torch_cuda_cpp.dll").exists(); - let use_hip = - si_lib.join("libtorch_hip.so").exists() || si_lib.join("torch_hip.dll").exists(); - println!("cargo:rustc-link-search=native={}", si_lib.display()); - - system_info.make(); - - println!("cargo:rustc-link-lib=static=tch"); - if use_cuda { - system_info.link("torch_cuda") - } - if use_cuda_cu { - system_info.link("torch_cuda_cu") - } - if use_cuda_cpp { - system_info.link("torch_cuda_cpp") - } - if use_hip { - system_info.link("torch_hip") - } - if cfg!(feature = "python-extension") { - system_info.link("torch_python") - } - if system_info.link_type == LinkType::Static { - // TODO: this has only be tried out on the cpu version. Check that it works - // with cuda too and maybe just try linking all available files? - system_info.link("asmjit"); - system_info.link("clog"); - system_info.link("cpuinfo"); - system_info.link("dnnl"); - system_info.link("dnnl_graph"); - system_info.link("fbgemm"); - system_info.link("gloo"); - system_info.link("kineto"); - system_info.link("nnpack"); - system_info.link("onnx"); - system_info.link("onnx_proto"); - system_info.link("protobuf"); - system_info.link("pthreadpool"); - system_info.link("pytorch_qnnpack"); - system_info.link("sleef"); - system_info.link("tensorpipe"); - system_info.link("tensorpipe_uv"); - system_info.link("XNNPACK"); - } - system_info.link("torch_cpu"); - system_info.link("torch"); - system_info.link("c10"); - if use_hip { - system_info.link("c10_hip"); - } - - let target = env::var("TARGET").context("TARGET variable not set")?; - - if !target.contains("msvc") && !target.contains("apple") { - println!("cargo:rustc-link-lib=gomp"); - } - } - Ok(()) -} diff --git a/v2/dependency-patches/torch-sys-0.24.0/libtch/CMakeLists.txt b/v2/dependency-patches/torch-sys-0.24.0/libtch/CMakeLists.txt deleted file mode 100644 index bba57bbf18..0000000000 --- a/v2/dependency-patches/torch-sys-0.24.0/libtch/CMakeLists.txt +++ /dev/null @@ -1,9 +0,0 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) -project(tch) - -find_package(Torch REQUIRED) - -add_library(tch STATIC torch_api.cpp torch_api_generated.cpp) -target_link_libraries(tch "${TORCH_LIBRARIES}") -set_property(TARGET tch PROPERTY CXX_STANDARD 11) -install(TARGETS tch DESTINATION .) diff --git a/v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image.h b/v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image.h deleted file mode 100644 index 8b710609e3..0000000000 --- a/v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image.h +++ /dev/null @@ -1,7530 +0,0 @@ -/* stb_image - v2.21 - public domain image loader - http://nothings.org/stb - no warranty implied; use at your own risk - - Do this: - #define STB_IMAGE_IMPLEMENTATION - before you include this file in *one* C or C++ file to create the implementation. - - // i.e. it should look like this: - #include ... - #include ... - #include ... - #define STB_IMAGE_IMPLEMENTATION - #include "stb_image.h" - - You can #define STBI_ASSERT(x) before the #include to avoid using assert.h. - And #define STBI_MALLOC, STBI_REALLOC, and STBI_FREE to avoid using malloc,realloc,free - - - QUICK NOTES: - Primarily of interest to game developers and other people who can - avoid problematic images and only need the trivial interface - - JPEG baseline & progressive (12 bpc/arithmetic not supported, same as stock IJG lib) - PNG 1/2/4/8/16-bit-per-channel - - TGA (not sure what subset, if a subset) - BMP non-1bpp, non-RLE - PSD (composited view only, no extra channels, 8/16 bit-per-channel) - - GIF (*comp always reports as 4-channel) - HDR (radiance rgbE format) - PIC (Softimage PIC) - PNM (PPM and PGM binary only) - - Animated GIF still needs a proper API, but here's one way to do it: - http://gist.github.com/urraka/685d9a6340b26b830d49 - - - decode from memory or through FILE (define STBI_NO_STDIO to remove code) - - decode from arbitrary I/O callbacks - - SIMD acceleration on x86/x64 (SSE2) and ARM (NEON) - - Full documentation under "DOCUMENTATION" below. - - -LICENSE - - See end of file for license information. - -RECENT REVISION HISTORY: - - 2.21 (2019-02-25) fix typo in comment - 2.20 (2019-02-07) support utf8 filenames in Windows; fix warnings and platform ifdefs - 2.19 (2018-02-11) fix warning - 2.18 (2018-01-30) fix warnings - 2.17 (2018-01-29) bugfix, 1-bit BMP, 16-bitness query, fix warnings - 2.16 (2017-07-23) all functions have 16-bit variants; optimizations; bugfixes - 2.15 (2017-03-18) fix png-1,2,4; all Imagenet JPGs; no runtime SSE detection on GCC - 2.14 (2017-03-03) remove deprecated STBI_JPEG_OLD; fixes for Imagenet JPGs - 2.13 (2016-12-04) experimental 16-bit API, only for PNG so far; fixes - 2.12 (2016-04-02) fix typo in 2.11 PSD fix that caused crashes - 2.11 (2016-04-02) 16-bit PNGS; enable SSE2 in non-gcc x64 - RGB-format JPEG; remove white matting in PSD; - allocate large structures on the stack; - correct channel count for PNG & BMP - 2.10 (2016-01-22) avoid warning introduced in 2.09 - 2.09 (2016-01-16) 16-bit TGA; comments in PNM files; STBI_REALLOC_SIZED - - See end of file for full revision history. - - - ============================ Contributors ========================= - - Image formats Extensions, features - Sean Barrett (jpeg, png, bmp) Jetro Lauha (stbi_info) - Nicolas Schulz (hdr, psd) Martin "SpartanJ" Golini (stbi_info) - Jonathan Dummer (tga) James "moose2000" Brown (iPhone PNG) - Jean-Marc Lienher (gif) Ben "Disch" Wenger (io callbacks) - Tom Seddon (pic) Omar Cornut (1/2/4-bit PNG) - Thatcher Ulrich (psd) Nicolas Guillemot (vertical flip) - Ken Miller (pgm, ppm) Richard Mitton (16-bit PSD) - github:urraka (animated gif) Junggon Kim (PNM comments) - Christopher Forseth (animated gif) Daniel Gibson (16-bit TGA) - socks-the-fox (16-bit PNG) - Jeremy Sawicki (handle all ImageNet JPGs) - Optimizations & bugfixes Mikhail Morozov (1-bit BMP) - Fabian "ryg" Giesen Anael Seghezzi (is-16-bit query) - Arseny Kapoulkine - John-Mark Allen - Carmelo J Fdez-Aguera - - Bug & warning fixes - Marc LeBlanc David Woo Guillaume George Martins Mozeiko - Christpher Lloyd Jerry Jansson Joseph Thomson Phil Jordan - Dave Moore Roy Eltham Hayaki Saito Nathan Reed - Won Chun Luke Graham Johan Duparc Nick Verigakis - the Horde3D community Thomas Ruf Ronny Chevalier github:rlyeh - Janez Zemva John Bartholomew Michal Cichon github:romigrou - Jonathan Blow Ken Hamada Tero Hanninen github:svdijk - Laurent Gomila Cort Stratton Sergio Gonzalez github:snagar - Aruelien Pocheville Thibault Reuille Cass Everitt github:Zelex - Ryamond Barbiero Paul Du Bois Engin Manap github:grim210 - Aldo Culquicondor Philipp Wiesemann Dale Weiler github:sammyhw - Oriol Ferrer Mesia Josh Tobin Matthew Gregan github:phprus - Julian Raschke Gregory Mullen Baldur Karlsson github:poppolopoppo - Christian Floisand Kevin Schmidt JR Smith github:darealshinji - Blazej Dariusz Roszkowski github:Michaelangel007 -*/ - -#ifndef STBI_INCLUDE_STB_IMAGE_H -#define STBI_INCLUDE_STB_IMAGE_H - -// DOCUMENTATION -// -// Limitations: -// - no 12-bit-per-channel JPEG -// - no JPEGs with arithmetic coding -// - GIF always returns *comp=4 -// -// Basic usage (see HDR discussion below for HDR usage): -// int x,y,n; -// unsigned char *data = stbi_load(filename, &x, &y, &n, 0); -// // ... process data if not NULL ... -// // ... x = width, y = height, n = # 8-bit components per pixel ... -// // ... replace '0' with '1'..'4' to force that many components per pixel -// // ... but 'n' will always be the number that it would have been if you said 0 -// stbi_image_free(data) -// -// Standard parameters: -// int *x -- outputs image width in pixels -// int *y -- outputs image height in pixels -// int *channels_in_file -- outputs # of image components in image file -// int desired_channels -- if non-zero, # of image components requested in result -// -// The return value from an image loader is an 'unsigned char *' which points -// to the pixel data, or NULL on an allocation failure or if the image is -// corrupt or invalid. The pixel data consists of *y scanlines of *x pixels, -// with each pixel consisting of N interleaved 8-bit components; the first -// pixel pointed to is top-left-most in the image. There is no padding between -// image scanlines or between pixels, regardless of format. The number of -// components N is 'desired_channels' if desired_channels is non-zero, or -// *channels_in_file otherwise. If desired_channels is non-zero, -// *channels_in_file has the number of components that _would_ have been -// output otherwise. E.g. if you set desired_channels to 4, you will always -// get RGBA output, but you can check *channels_in_file to see if it's trivially -// opaque because e.g. there were only 3 channels in the source image. -// -// An output image with N components has the following components interleaved -// in this order in each pixel: -// -// N=#comp components -// 1 grey -// 2 grey, alpha -// 3 red, green, blue -// 4 red, green, blue, alpha -// -// If image loading fails for any reason, the return value will be NULL, -// and *x, *y, *channels_in_file will be unchanged. The function -// stbi_failure_reason() can be queried for an extremely brief, end-user -// unfriendly explanation of why the load failed. Define STBI_NO_FAILURE_STRINGS -// to avoid compiling these strings at all, and STBI_FAILURE_USERMSG to get slightly -// more user-friendly ones. -// -// Paletted PNG, BMP, GIF, and PIC images are automatically depalettized. -// -// =========================================================================== -// -// UNICODE: -// -// If compiling for Windows and you wish to use Unicode filenames, compile -// with -// #define STBI_WINDOWS_UTF8 -// and pass utf8-encoded filenames. Call stbi_convert_wchar_to_utf8 to convert -// Windows wchar_t filenames to utf8. -// -// =========================================================================== -// -// Philosophy -// -// stb libraries are designed with the following priorities: -// -// 1. easy to use -// 2. easy to maintain -// 3. good performance -// -// Sometimes I let "good performance" creep up in priority over "easy to maintain", -// and for best performance I may provide less-easy-to-use APIs that give higher -// performance, in addition to the easy-to-use ones. Nevertheless, it's important -// to keep in mind that from the standpoint of you, a client of this library, -// all you care about is #1 and #3, and stb libraries DO NOT emphasize #3 above all. -// -// Some secondary priorities arise directly from the first two, some of which -// provide more explicit reasons why performance can't be emphasized. -// -// - Portable ("ease of use") -// - Small source code footprint ("easy to maintain") -// - No dependencies ("ease of use") -// -// =========================================================================== -// -// I/O callbacks -// -// I/O callbacks allow you to read from arbitrary sources, like packaged -// files or some other source. Data read from callbacks are processed -// through a small internal buffer (currently 128 bytes) to try to reduce -// overhead. -// -// The three functions you must define are "read" (reads some bytes of data), -// "skip" (skips some bytes of data), "eof" (reports if the stream is at the end). -// -// =========================================================================== -// -// SIMD support -// -// The JPEG decoder will try to automatically use SIMD kernels on x86 when -// supported by the compiler. For ARM Neon support, you must explicitly -// request it. -// -// (The old do-it-yourself SIMD API is no longer supported in the current -// code.) -// -// On x86, SSE2 will automatically be used when available based on a run-time -// test; if not, the generic C versions are used as a fall-back. On ARM targets, -// the typical path is to have separate builds for NEON and non-NEON devices -// (at least this is true for iOS and Android). Therefore, the NEON support is -// toggled by a build flag: define STBI_NEON to get NEON loops. -// -// If for some reason you do not want to use any of SIMD code, or if -// you have issues compiling it, you can disable it entirely by -// defining STBI_NO_SIMD. -// -// =========================================================================== -// -// HDR image support (disable by defining STBI_NO_HDR) -// -// stb_image supports loading HDR images in general, and currently the Radiance -// .HDR file format specifically. You can still load any file through the existing -// interface; if you attempt to load an HDR file, it will be automatically remapped -// to LDR, assuming gamma 2.2 and an arbitrary scale factor defaulting to 1; -// both of these constants can be reconfigured through this interface: -// -// stbi_hdr_to_ldr_gamma(2.2f); -// stbi_hdr_to_ldr_scale(1.0f); -// -// (note, do not use _inverse_ constants; stbi_image will invert them -// appropriately). -// -// Additionally, there is a new, parallel interface for loading files as -// (linear) floats to preserve the full dynamic range: -// -// float *data = stbi_loadf(filename, &x, &y, &n, 0); -// -// If you load LDR images through this interface, those images will -// be promoted to floating point values, run through the inverse of -// constants corresponding to the above: -// -// stbi_ldr_to_hdr_scale(1.0f); -// stbi_ldr_to_hdr_gamma(2.2f); -// -// Finally, given a filename (or an open file or memory block--see header -// file for details) containing image data, you can query for the "most -// appropriate" interface to use (that is, whether the image is HDR or -// not), using: -// -// stbi_is_hdr(char *filename); -// -// =========================================================================== -// -// iPhone PNG support: -// -// By default we convert iphone-formatted PNGs back to RGB, even though -// they are internally encoded differently. You can disable this conversion -// by calling stbi_convert_iphone_png_to_rgb(0), in which case -// you will always just get the native iphone "format" through (which -// is BGR stored in RGB). -// -// Call stbi_set_unpremultiply_on_load(1) as well to force a divide per -// pixel to remove any premultiplied alpha *only* if the image file explicitly -// says there's premultiplied data (currently only happens in iPhone images, -// and only if iPhone convert-to-rgb processing is on). -// -// =========================================================================== -// -// ADDITIONAL CONFIGURATION -// -// - You can suppress implementation of any of the decoders to reduce -// your code footprint by #defining one or more of the following -// symbols before creating the implementation. -// -// STBI_NO_JPEG -// STBI_NO_PNG -// STBI_NO_BMP -// STBI_NO_PSD -// STBI_NO_TGA -// STBI_NO_GIF -// STBI_NO_HDR -// STBI_NO_PIC -// STBI_NO_PNM (.ppm and .pgm) -// -// - You can request *only* certain decoders and suppress all other ones -// (this will be more forward-compatible, as addition of new decoders -// doesn't require you to disable them explicitly): -// -// STBI_ONLY_JPEG -// STBI_ONLY_PNG -// STBI_ONLY_BMP -// STBI_ONLY_PSD -// STBI_ONLY_TGA -// STBI_ONLY_GIF -// STBI_ONLY_HDR -// STBI_ONLY_PIC -// STBI_ONLY_PNM (.ppm and .pgm) -// -// - If you use STBI_NO_PNG (or _ONLY_ without PNG), and you still -// want the zlib decoder to be available, #define STBI_SUPPORT_ZLIB -// - - -#ifndef STBI_NO_STDIO -#include -#endif // STBI_NO_STDIO - -#define STBI_VERSION 1 - -enum -{ - STBI_default = 0, // only used for desired_channels - - STBI_grey = 1, - STBI_grey_alpha = 2, - STBI_rgb = 3, - STBI_rgb_alpha = 4 -}; - -#include -typedef unsigned char stbi_uc; -typedef unsigned short stbi_us; - -#ifdef __cplusplus -extern "C" { -#endif - -#ifdef STB_IMAGE_STATIC -#define STBIDEF static -#else -#define STBIDEF extern -#endif - -////////////////////////////////////////////////////////////////////////////// -// -// PRIMARY API - works on images of any type -// - -// -// load image by filename, open file, or memory buffer -// - -typedef struct -{ - int (*read) (void *user,char *data,int size); // fill 'data' with 'size' bytes. return number of bytes actually read - void (*skip) (void *user,int n); // skip the next 'n' bytes, or 'unget' the last -n bytes if negative - int (*eof) (void *user); // returns nonzero if we are at end of file/data -} stbi_io_callbacks; - -//////////////////////////////////// -// -// 8-bits-per-channel interface -// - -STBIDEF stbi_uc *stbi_load_from_memory (stbi_uc const *buffer, int len , int *x, int *y, int *channels_in_file, int desired_channels); -STBIDEF stbi_uc *stbi_load_from_callbacks(stbi_io_callbacks const *clbk , void *user, int *x, int *y, int *channels_in_file, int desired_channels); - -#ifndef STBI_NO_STDIO -STBIDEF stbi_uc *stbi_load (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels); -STBIDEF stbi_uc *stbi_load_from_file (FILE *f, int *x, int *y, int *channels_in_file, int desired_channels); -// for stbi_load_from_file, file pointer is left pointing immediately after image -#endif - -#ifndef STBI_NO_GIF -STBIDEF stbi_uc *stbi_load_gif_from_memory(stbi_uc const *buffer, int len, int **delays, int *x, int *y, int *z, int *comp, int req_comp); -#endif - -#ifdef STBI_WINDOWS_UTF8 -STBIDEF int stbi_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input); -#endif - -//////////////////////////////////// -// -// 16-bits-per-channel interface -// - -STBIDEF stbi_us *stbi_load_16_from_memory (stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels); -STBIDEF stbi_us *stbi_load_16_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels); - -#ifndef STBI_NO_STDIO -STBIDEF stbi_us *stbi_load_16 (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels); -STBIDEF stbi_us *stbi_load_from_file_16(FILE *f, int *x, int *y, int *channels_in_file, int desired_channels); -#endif - -//////////////////////////////////// -// -// float-per-channel interface -// -#ifndef STBI_NO_LINEAR - STBIDEF float *stbi_loadf_from_memory (stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels); - STBIDEF float *stbi_loadf_from_callbacks (stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels); - - #ifndef STBI_NO_STDIO - STBIDEF float *stbi_loadf (char const *filename, int *x, int *y, int *channels_in_file, int desired_channels); - STBIDEF float *stbi_loadf_from_file (FILE *f, int *x, int *y, int *channels_in_file, int desired_channels); - #endif -#endif - -#ifndef STBI_NO_HDR - STBIDEF void stbi_hdr_to_ldr_gamma(float gamma); - STBIDEF void stbi_hdr_to_ldr_scale(float scale); -#endif // STBI_NO_HDR - -#ifndef STBI_NO_LINEAR - STBIDEF void stbi_ldr_to_hdr_gamma(float gamma); - STBIDEF void stbi_ldr_to_hdr_scale(float scale); -#endif // STBI_NO_LINEAR - -// stbi_is_hdr is always defined, but always returns false if STBI_NO_HDR -STBIDEF int stbi_is_hdr_from_callbacks(stbi_io_callbacks const *clbk, void *user); -STBIDEF int stbi_is_hdr_from_memory(stbi_uc const *buffer, int len); -#ifndef STBI_NO_STDIO -STBIDEF int stbi_is_hdr (char const *filename); -STBIDEF int stbi_is_hdr_from_file(FILE *f); -#endif // STBI_NO_STDIO - - -// get a VERY brief reason for failure -// NOT THREADSAFE -STBIDEF const char *stbi_failure_reason (void); - -// free the loaded image -- this is just free() -STBIDEF void stbi_image_free (void *retval_from_stbi_load); - -// get image dimensions & components without fully decoding -STBIDEF int stbi_info_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp); -STBIDEF int stbi_info_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp); -STBIDEF int stbi_is_16_bit_from_memory(stbi_uc const *buffer, int len); -STBIDEF int stbi_is_16_bit_from_callbacks(stbi_io_callbacks const *clbk, void *user); - -#ifndef STBI_NO_STDIO -STBIDEF int stbi_info (char const *filename, int *x, int *y, int *comp); -STBIDEF int stbi_info_from_file (FILE *f, int *x, int *y, int *comp); -STBIDEF int stbi_is_16_bit (char const *filename); -STBIDEF int stbi_is_16_bit_from_file(FILE *f); -#endif - - - -// for image formats that explicitly notate that they have premultiplied alpha, -// we just return the colors as stored in the file. set this flag to force -// unpremultiplication. results are undefined if the unpremultiply overflow. -STBIDEF void stbi_set_unpremultiply_on_load(int flag_true_if_should_unpremultiply); - -// indicate whether we should process iphone images back to canonical format, -// or just pass them through "as-is" -STBIDEF void stbi_convert_iphone_png_to_rgb(int flag_true_if_should_convert); - -// flip the image vertically, so the first pixel in the output array is the bottom left -STBIDEF void stbi_set_flip_vertically_on_load(int flag_true_if_should_flip); - -// ZLIB client - used by PNG, available for other purposes - -STBIDEF char *stbi_zlib_decode_malloc_guesssize(const char *buffer, int len, int initial_size, int *outlen); -STBIDEF char *stbi_zlib_decode_malloc_guesssize_headerflag(const char *buffer, int len, int initial_size, int *outlen, int parse_header); -STBIDEF char *stbi_zlib_decode_malloc(const char *buffer, int len, int *outlen); -STBIDEF int stbi_zlib_decode_buffer(char *obuffer, int olen, const char *ibuffer, int ilen); - -STBIDEF char *stbi_zlib_decode_noheader_malloc(const char *buffer, int len, int *outlen); -STBIDEF int stbi_zlib_decode_noheader_buffer(char *obuffer, int olen, const char *ibuffer, int ilen); - - -#ifdef __cplusplus -} -#endif - -// -// -//// end header file ///////////////////////////////////////////////////// -#endif // STBI_INCLUDE_STB_IMAGE_H - -#ifdef STB_IMAGE_IMPLEMENTATION - -#if defined(STBI_ONLY_JPEG) || defined(STBI_ONLY_PNG) || defined(STBI_ONLY_BMP) \ - || defined(STBI_ONLY_TGA) || defined(STBI_ONLY_GIF) || defined(STBI_ONLY_PSD) \ - || defined(STBI_ONLY_HDR) || defined(STBI_ONLY_PIC) || defined(STBI_ONLY_PNM) \ - || defined(STBI_ONLY_ZLIB) - #ifndef STBI_ONLY_JPEG - #define STBI_NO_JPEG - #endif - #ifndef STBI_ONLY_PNG - #define STBI_NO_PNG - #endif - #ifndef STBI_ONLY_BMP - #define STBI_NO_BMP - #endif - #ifndef STBI_ONLY_PSD - #define STBI_NO_PSD - #endif - #ifndef STBI_ONLY_TGA - #define STBI_NO_TGA - #endif - #ifndef STBI_ONLY_GIF - #define STBI_NO_GIF - #endif - #ifndef STBI_ONLY_HDR - #define STBI_NO_HDR - #endif - #ifndef STBI_ONLY_PIC - #define STBI_NO_PIC - #endif - #ifndef STBI_ONLY_PNM - #define STBI_NO_PNM - #endif -#endif - -#if defined(STBI_NO_PNG) && !defined(STBI_SUPPORT_ZLIB) && !defined(STBI_NO_ZLIB) -#define STBI_NO_ZLIB -#endif - - -#include -#include // ptrdiff_t on osx -#include -#include -#include - -#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR) -#include // ldexp, pow -#endif - -#ifndef STBI_NO_STDIO -#include -#endif - -#ifndef STBI_ASSERT -#include -#define STBI_ASSERT(x) assert(x) -#endif - -#ifdef __cplusplus -#define STBI_EXTERN extern "C" -#else -#define STBI_EXTERN extern -#endif - - -#ifndef _MSC_VER - #ifdef __cplusplus - #define stbi_inline inline - #else - #define stbi_inline - #endif -#else - #define stbi_inline __forceinline -#endif - - -#ifdef _MSC_VER -typedef unsigned short stbi__uint16; -typedef signed short stbi__int16; -typedef unsigned int stbi__uint32; -typedef signed int stbi__int32; -#else -#include -typedef uint16_t stbi__uint16; -typedef int16_t stbi__int16; -typedef uint32_t stbi__uint32; -typedef int32_t stbi__int32; -#endif - -// should produce compiler error if size is wrong -typedef unsigned char validate_uint32[sizeof(stbi__uint32)==4 ? 1 : -1]; - -#ifdef _MSC_VER -#define STBI_NOTUSED(v) (void)(v) -#else -#define STBI_NOTUSED(v) (void)sizeof(v) -#endif - -#ifdef _MSC_VER -#define STBI_HAS_LROTL -#endif - -#ifdef STBI_HAS_LROTL - #define stbi_lrot(x,y) _lrotl(x,y) -#else - #define stbi_lrot(x,y) (((x) << (y)) | ((x) >> (32 - (y)))) -#endif - -#if defined(STBI_MALLOC) && defined(STBI_FREE) && (defined(STBI_REALLOC) || defined(STBI_REALLOC_SIZED)) -// ok -#elif !defined(STBI_MALLOC) && !defined(STBI_FREE) && !defined(STBI_REALLOC) && !defined(STBI_REALLOC_SIZED) -// ok -#else -#error "Must define all or none of STBI_MALLOC, STBI_FREE, and STBI_REALLOC (or STBI_REALLOC_SIZED)." -#endif - -#ifndef STBI_MALLOC -#define STBI_MALLOC(sz) malloc(sz) -#define STBI_REALLOC(p,newsz) realloc(p,newsz) -#define STBI_FREE(p) free(p) -#endif - -#ifndef STBI_REALLOC_SIZED -#define STBI_REALLOC_SIZED(p,oldsz,newsz) STBI_REALLOC(p,newsz) -#endif - -// x86/x64 detection -#if defined(__x86_64__) || defined(_M_X64) -#define STBI__X64_TARGET -#elif defined(__i386) || defined(_M_IX86) -#define STBI__X86_TARGET -#endif - -#if defined(__GNUC__) && defined(STBI__X86_TARGET) && !defined(__SSE2__) && !defined(STBI_NO_SIMD) -// gcc doesn't support sse2 intrinsics unless you compile with -msse2, -// which in turn means it gets to use SSE2 everywhere. This is unfortunate, -// but previous attempts to provide the SSE2 functions with runtime -// detection caused numerous issues. The way architecture extensions are -// exposed in GCC/Clang is, sadly, not really suited for one-file libs. -// New behavior: if compiled with -msse2, we use SSE2 without any -// detection; if not, we don't use it at all. -#define STBI_NO_SIMD -#endif - -#if defined(__MINGW32__) && defined(STBI__X86_TARGET) && !defined(STBI_MINGW_ENABLE_SSE2) && !defined(STBI_NO_SIMD) -// Note that __MINGW32__ doesn't actually mean 32-bit, so we have to avoid STBI__X64_TARGET -// -// 32-bit MinGW wants ESP to be 16-byte aligned, but this is not in the -// Windows ABI and VC++ as well as Windows DLLs don't maintain that invariant. -// As a result, enabling SSE2 on 32-bit MinGW is dangerous when not -// simultaneously enabling "-mstackrealign". -// -// See https://github.com/nothings/stb/issues/81 for more information. -// -// So default to no SSE2 on 32-bit MinGW. If you've read this far and added -// -mstackrealign to your build settings, feel free to #define STBI_MINGW_ENABLE_SSE2. -#define STBI_NO_SIMD -#endif - -#if !defined(STBI_NO_SIMD) && (defined(STBI__X86_TARGET) || defined(STBI__X64_TARGET)) -#define STBI_SSE2 -#include - -#ifdef _MSC_VER - -#if _MSC_VER >= 1400 // not VC6 -#include // __cpuid -static int stbi__cpuid3(void) -{ - int info[4]; - __cpuid(info,1); - return info[3]; -} -#else -static int stbi__cpuid3(void) -{ - int res; - __asm { - mov eax,1 - cpuid - mov res,edx - } - return res; -} -#endif - -#define STBI_SIMD_ALIGN(type, name) __declspec(align(16)) type name - -#if !defined(STBI_NO_JPEG) && defined(STBI_SSE2) -static int stbi__sse2_available(void) -{ - int info3 = stbi__cpuid3(); - return ((info3 >> 26) & 1) != 0; -} -#endif - -#else // assume GCC-style if not VC++ -#define STBI_SIMD_ALIGN(type, name) type name __attribute__((aligned(16))) - -#if !defined(STBI_NO_JPEG) && defined(STBI_SSE2) -static int stbi__sse2_available(void) -{ - // If we're even attempting to compile this on GCC/Clang, that means - // -msse2 is on, which means the compiler is allowed to use SSE2 - // instructions at will, and so are we. - return 1; -} -#endif - -#endif -#endif - -// ARM NEON -#if defined(STBI_NO_SIMD) && defined(STBI_NEON) -#undef STBI_NEON -#endif - -#ifdef STBI_NEON -#include -// assume GCC or Clang on ARM targets -#define STBI_SIMD_ALIGN(type, name) type name __attribute__((aligned(16))) -#endif - -#ifndef STBI_SIMD_ALIGN -#define STBI_SIMD_ALIGN(type, name) type name -#endif - -/////////////////////////////////////////////// -// -// stbi__context struct and start_xxx functions - -// stbi__context structure is our basic context used by all images, so it -// contains all the IO context, plus some basic image information -typedef struct -{ - stbi__uint32 img_x, img_y; - int img_n, img_out_n; - - stbi_io_callbacks io; - void *io_user_data; - - int read_from_callbacks; - int buflen; - stbi_uc buffer_start[128]; - - stbi_uc *img_buffer, *img_buffer_end; - stbi_uc *img_buffer_original, *img_buffer_original_end; -} stbi__context; - - -static void stbi__refill_buffer(stbi__context *s); - -// initialize a memory-decode context -static void stbi__start_mem(stbi__context *s, stbi_uc const *buffer, int len) -{ - s->io.read = NULL; - s->read_from_callbacks = 0; - s->img_buffer = s->img_buffer_original = (stbi_uc *) buffer; - s->img_buffer_end = s->img_buffer_original_end = (stbi_uc *) buffer+len; -} - -// initialize a callback-based context -static void stbi__start_callbacks(stbi__context *s, stbi_io_callbacks *c, void *user) -{ - s->io = *c; - s->io_user_data = user; - s->buflen = sizeof(s->buffer_start); - s->read_from_callbacks = 1; - s->img_buffer_original = s->buffer_start; - stbi__refill_buffer(s); - s->img_buffer_original_end = s->img_buffer_end; -} - -#ifndef STBI_NO_STDIO - -static int stbi__stdio_read(void *user, char *data, int size) -{ - return (int) fread(data,1,size,(FILE*) user); -} - -static void stbi__stdio_skip(void *user, int n) -{ - fseek((FILE*) user, n, SEEK_CUR); -} - -static int stbi__stdio_eof(void *user) -{ - return feof((FILE*) user); -} - -static stbi_io_callbacks stbi__stdio_callbacks = -{ - stbi__stdio_read, - stbi__stdio_skip, - stbi__stdio_eof, -}; - -static void stbi__start_file(stbi__context *s, FILE *f) -{ - stbi__start_callbacks(s, &stbi__stdio_callbacks, (void *) f); -} - -//static void stop_file(stbi__context *s) { } - -#endif // !STBI_NO_STDIO - -static void stbi__rewind(stbi__context *s) -{ - // conceptually rewind SHOULD rewind to the beginning of the stream, - // but we just rewind to the beginning of the initial buffer, because - // we only use it after doing 'test', which only ever looks at at most 92 bytes - s->img_buffer = s->img_buffer_original; - s->img_buffer_end = s->img_buffer_original_end; -} - -enum -{ - STBI_ORDER_RGB, - STBI_ORDER_BGR -}; - -typedef struct -{ - int bits_per_channel; - int num_channels; - int channel_order; -} stbi__result_info; - -#ifndef STBI_NO_JPEG -static int stbi__jpeg_test(stbi__context *s); -static void *stbi__jpeg_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); -static int stbi__jpeg_info(stbi__context *s, int *x, int *y, int *comp); -#endif - -#ifndef STBI_NO_PNG -static int stbi__png_test(stbi__context *s); -static void *stbi__png_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); -static int stbi__png_info(stbi__context *s, int *x, int *y, int *comp); -static int stbi__png_is16(stbi__context *s); -#endif - -#ifndef STBI_NO_BMP -static int stbi__bmp_test(stbi__context *s); -static void *stbi__bmp_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); -static int stbi__bmp_info(stbi__context *s, int *x, int *y, int *comp); -#endif - -#ifndef STBI_NO_TGA -static int stbi__tga_test(stbi__context *s); -static void *stbi__tga_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); -static int stbi__tga_info(stbi__context *s, int *x, int *y, int *comp); -#endif - -#ifndef STBI_NO_PSD -static int stbi__psd_test(stbi__context *s); -static void *stbi__psd_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc); -static int stbi__psd_info(stbi__context *s, int *x, int *y, int *comp); -static int stbi__psd_is16(stbi__context *s); -#endif - -#ifndef STBI_NO_HDR -static int stbi__hdr_test(stbi__context *s); -static float *stbi__hdr_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); -static int stbi__hdr_info(stbi__context *s, int *x, int *y, int *comp); -#endif - -#ifndef STBI_NO_PIC -static int stbi__pic_test(stbi__context *s); -static void *stbi__pic_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); -static int stbi__pic_info(stbi__context *s, int *x, int *y, int *comp); -#endif - -#ifndef STBI_NO_GIF -static int stbi__gif_test(stbi__context *s); -static void *stbi__gif_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); -static void *stbi__load_gif_main(stbi__context *s, int **delays, int *x, int *y, int *z, int *comp, int req_comp); -static int stbi__gif_info(stbi__context *s, int *x, int *y, int *comp); -#endif - -#ifndef STBI_NO_PNM -static int stbi__pnm_test(stbi__context *s); -static void *stbi__pnm_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri); -static int stbi__pnm_info(stbi__context *s, int *x, int *y, int *comp); -#endif - -// this is not threadsafe -static const char *stbi__g_failure_reason; - -STBIDEF const char *stbi_failure_reason(void) -{ - return stbi__g_failure_reason; -} - -static int stbi__err(const char *str) -{ - stbi__g_failure_reason = str; - return 0; -} - -static void *stbi__malloc(size_t size) -{ - return STBI_MALLOC(size); -} - -// stb_image uses ints pervasively, including for offset calculations. -// therefore the largest decoded image size we can support with the -// current code, even on 64-bit targets, is INT_MAX. this is not a -// significant limitation for the intended use case. -// -// we do, however, need to make sure our size calculations don't -// overflow. hence a few helper functions for size calculations that -// multiply integers together, making sure that they're non-negative -// and no overflow occurs. - -// return 1 if the sum is valid, 0 on overflow. -// negative terms are considered invalid. -static int stbi__addsizes_valid(int a, int b) -{ - if (b < 0) return 0; - // now 0 <= b <= INT_MAX, hence also - // 0 <= INT_MAX - b <= INTMAX. - // And "a + b <= INT_MAX" (which might overflow) is the - // same as a <= INT_MAX - b (no overflow) - return a <= INT_MAX - b; -} - -// returns 1 if the product is valid, 0 on overflow. -// negative factors are considered invalid. -static int stbi__mul2sizes_valid(int a, int b) -{ - if (a < 0 || b < 0) return 0; - if (b == 0) return 1; // mul-by-0 is always safe - // portable way to check for no overflows in a*b - return a <= INT_MAX/b; -} - -// returns 1 if "a*b + add" has no negative terms/factors and doesn't overflow -static int stbi__mad2sizes_valid(int a, int b, int add) -{ - return stbi__mul2sizes_valid(a, b) && stbi__addsizes_valid(a*b, add); -} - -// returns 1 if "a*b*c + add" has no negative terms/factors and doesn't overflow -static int stbi__mad3sizes_valid(int a, int b, int c, int add) -{ - return stbi__mul2sizes_valid(a, b) && stbi__mul2sizes_valid(a*b, c) && - stbi__addsizes_valid(a*b*c, add); -} - -// returns 1 if "a*b*c*d + add" has no negative terms/factors and doesn't overflow -#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR) -static int stbi__mad4sizes_valid(int a, int b, int c, int d, int add) -{ - return stbi__mul2sizes_valid(a, b) && stbi__mul2sizes_valid(a*b, c) && - stbi__mul2sizes_valid(a*b*c, d) && stbi__addsizes_valid(a*b*c*d, add); -} -#endif - -// mallocs with size overflow checking -static void *stbi__malloc_mad2(int a, int b, int add) -{ - if (!stbi__mad2sizes_valid(a, b, add)) return NULL; - return stbi__malloc(a*b + add); -} - -static void *stbi__malloc_mad3(int a, int b, int c, int add) -{ - if (!stbi__mad3sizes_valid(a, b, c, add)) return NULL; - return stbi__malloc(a*b*c + add); -} - -#if !defined(STBI_NO_LINEAR) || !defined(STBI_NO_HDR) -static void *stbi__malloc_mad4(int a, int b, int c, int d, int add) -{ - if (!stbi__mad4sizes_valid(a, b, c, d, add)) return NULL; - return stbi__malloc(a*b*c*d + add); -} -#endif - -// stbi__err - error -// stbi__errpf - error returning pointer to float -// stbi__errpuc - error returning pointer to unsigned char - -#ifdef STBI_NO_FAILURE_STRINGS - #define stbi__err(x,y) 0 -#elif defined(STBI_FAILURE_USERMSG) - #define stbi__err(x,y) stbi__err(y) -#else - #define stbi__err(x,y) stbi__err(x) -#endif - -#define stbi__errpf(x,y) ((float *)(size_t) (stbi__err(x,y)?NULL:NULL)) -#define stbi__errpuc(x,y) ((unsigned char *)(size_t) (stbi__err(x,y)?NULL:NULL)) - -STBIDEF void stbi_image_free(void *retval_from_stbi_load) -{ - STBI_FREE(retval_from_stbi_load); -} - -#ifndef STBI_NO_LINEAR -static float *stbi__ldr_to_hdr(stbi_uc *data, int x, int y, int comp); -#endif - -#ifndef STBI_NO_HDR -static stbi_uc *stbi__hdr_to_ldr(float *data, int x, int y, int comp); -#endif - -static int stbi__vertically_flip_on_load = 0; - -STBIDEF void stbi_set_flip_vertically_on_load(int flag_true_if_should_flip) -{ - stbi__vertically_flip_on_load = flag_true_if_should_flip; -} - -static void *stbi__load_main(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc) -{ - memset(ri, 0, sizeof(*ri)); // make sure it's initialized if we add new fields - ri->bits_per_channel = 8; // default is 8 so most paths don't have to be changed - ri->channel_order = STBI_ORDER_RGB; // all current input & output are this, but this is here so we can add BGR order - ri->num_channels = 0; - - #ifndef STBI_NO_JPEG - if (stbi__jpeg_test(s)) return stbi__jpeg_load(s,x,y,comp,req_comp, ri); - #endif - #ifndef STBI_NO_PNG - if (stbi__png_test(s)) return stbi__png_load(s,x,y,comp,req_comp, ri); - #endif - #ifndef STBI_NO_BMP - if (stbi__bmp_test(s)) return stbi__bmp_load(s,x,y,comp,req_comp, ri); - #endif - #ifndef STBI_NO_GIF - if (stbi__gif_test(s)) return stbi__gif_load(s,x,y,comp,req_comp, ri); - #endif - #ifndef STBI_NO_PSD - if (stbi__psd_test(s)) return stbi__psd_load(s,x,y,comp,req_comp, ri, bpc); - #endif - #ifndef STBI_NO_PIC - if (stbi__pic_test(s)) return stbi__pic_load(s,x,y,comp,req_comp, ri); - #endif - #ifndef STBI_NO_PNM - if (stbi__pnm_test(s)) return stbi__pnm_load(s,x,y,comp,req_comp, ri); - #endif - - #ifndef STBI_NO_HDR - if (stbi__hdr_test(s)) { - float *hdr = stbi__hdr_load(s, x,y,comp,req_comp, ri); - return stbi__hdr_to_ldr(hdr, *x, *y, req_comp ? req_comp : *comp); - } - #endif - - #ifndef STBI_NO_TGA - // test tga last because it's a crappy test! - if (stbi__tga_test(s)) - return stbi__tga_load(s,x,y,comp,req_comp, ri); - #endif - - return stbi__errpuc("unknown image type", "Image not of any known type, or corrupt"); -} - -static stbi_uc *stbi__convert_16_to_8(stbi__uint16 *orig, int w, int h, int channels) -{ - int i; - int img_len = w * h * channels; - stbi_uc *reduced; - - reduced = (stbi_uc *) stbi__malloc(img_len); - if (reduced == NULL) return stbi__errpuc("outofmem", "Out of memory"); - - for (i = 0; i < img_len; ++i) - reduced[i] = (stbi_uc)((orig[i] >> 8) & 0xFF); // top half of each byte is sufficient approx of 16->8 bit scaling - - STBI_FREE(orig); - return reduced; -} - -static stbi__uint16 *stbi__convert_8_to_16(stbi_uc *orig, int w, int h, int channels) -{ - int i; - int img_len = w * h * channels; - stbi__uint16 *enlarged; - - enlarged = (stbi__uint16 *) stbi__malloc(img_len*2); - if (enlarged == NULL) return (stbi__uint16 *) stbi__errpuc("outofmem", "Out of memory"); - - for (i = 0; i < img_len; ++i) - enlarged[i] = (stbi__uint16)((orig[i] << 8) + orig[i]); // replicate to high and low byte, maps 0->0, 255->0xffff - - STBI_FREE(orig); - return enlarged; -} - -static void stbi__vertical_flip(void *image, int w, int h, int bytes_per_pixel) -{ - int row; - size_t bytes_per_row = (size_t)w * bytes_per_pixel; - stbi_uc temp[2048]; - stbi_uc *bytes = (stbi_uc *)image; - - for (row = 0; row < (h>>1); row++) { - stbi_uc *row0 = bytes + row*bytes_per_row; - stbi_uc *row1 = bytes + (h - row - 1)*bytes_per_row; - // swap row0 with row1 - size_t bytes_left = bytes_per_row; - while (bytes_left) { - size_t bytes_copy = (bytes_left < sizeof(temp)) ? bytes_left : sizeof(temp); - memcpy(temp, row0, bytes_copy); - memcpy(row0, row1, bytes_copy); - memcpy(row1, temp, bytes_copy); - row0 += bytes_copy; - row1 += bytes_copy; - bytes_left -= bytes_copy; - } - } -} - -#ifndef STBI_NO_GIF -static void stbi__vertical_flip_slices(void *image, int w, int h, int z, int bytes_per_pixel) -{ - int slice; - int slice_size = w * h * bytes_per_pixel; - - stbi_uc *bytes = (stbi_uc *)image; - for (slice = 0; slice < z; ++slice) { - stbi__vertical_flip(bytes, w, h, bytes_per_pixel); - bytes += slice_size; - } -} -#endif - -static unsigned char *stbi__load_and_postprocess_8bit(stbi__context *s, int *x, int *y, int *comp, int req_comp) -{ - stbi__result_info ri; - void *result = stbi__load_main(s, x, y, comp, req_comp, &ri, 8); - - if (result == NULL) - return NULL; - - if (ri.bits_per_channel != 8) { - STBI_ASSERT(ri.bits_per_channel == 16); - result = stbi__convert_16_to_8((stbi__uint16 *) result, *x, *y, req_comp == 0 ? *comp : req_comp); - ri.bits_per_channel = 8; - } - - // @TODO: move stbi__convert_format to here - - if (stbi__vertically_flip_on_load) { - int channels = req_comp ? req_comp : *comp; - stbi__vertical_flip(result, *x, *y, channels * sizeof(stbi_uc)); - } - - return (unsigned char *) result; -} - -static stbi__uint16 *stbi__load_and_postprocess_16bit(stbi__context *s, int *x, int *y, int *comp, int req_comp) -{ - stbi__result_info ri; - void *result = stbi__load_main(s, x, y, comp, req_comp, &ri, 16); - - if (result == NULL) - return NULL; - - if (ri.bits_per_channel != 16) { - STBI_ASSERT(ri.bits_per_channel == 8); - result = stbi__convert_8_to_16((stbi_uc *) result, *x, *y, req_comp == 0 ? *comp : req_comp); - ri.bits_per_channel = 16; - } - - // @TODO: move stbi__convert_format16 to here - // @TODO: special case RGB-to-Y (and RGBA-to-YA) for 8-bit-to-16-bit case to keep more precision - - if (stbi__vertically_flip_on_load) { - int channels = req_comp ? req_comp : *comp; - stbi__vertical_flip(result, *x, *y, channels * sizeof(stbi__uint16)); - } - - return (stbi__uint16 *) result; -} - -#if !defined(STBI_NO_HDR) && !defined(STBI_NO_LINEAR) -static void stbi__float_postprocess(float *result, int *x, int *y, int *comp, int req_comp) -{ - if (stbi__vertically_flip_on_load && result != NULL) { - int channels = req_comp ? req_comp : *comp; - stbi__vertical_flip(result, *x, *y, channels * sizeof(float)); - } -} -#endif - -#ifndef STBI_NO_STDIO - -#if defined(_MSC_VER) && defined(STBI_WINDOWS_UTF8) -STBI_EXTERN __declspec(dllimport) int __stdcall MultiByteToWideChar(unsigned int cp, unsigned long flags, const char *str, int cbmb, wchar_t *widestr, int cchwide); -STBI_EXTERN __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int cp, unsigned long flags, const wchar_t *widestr, int cchwide, char *str, int cbmb, const char *defchar, int *used_default); -#endif - -#if defined(_MSC_VER) && defined(STBI_WINDOWS_UTF8) -STBIDEF int stbi_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input) -{ - return WideCharToMultiByte(65001 /* UTF8 */, 0, input, -1, buffer, bufferlen, NULL, NULL); -} -#endif - -static FILE *stbi__fopen(char const *filename, char const *mode) -{ - FILE *f; -#if defined(_MSC_VER) && defined(STBI_WINDOWS_UTF8) - wchar_t wMode[64]; - wchar_t wFilename[1024]; - if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, filename, -1, wFilename, sizeof(wFilename))) - return 0; - - if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, mode, -1, wMode, sizeof(wMode))) - return 0; - -#if _MSC_VER >= 1400 - if (0 != _wfopen_s(&f, wFilename, wMode)) - f = 0; -#else - f = _wfopen(wFilename, wMode); -#endif - -#elif defined(_MSC_VER) && _MSC_VER >= 1400 - if (0 != fopen_s(&f, filename, mode)) - f=0; -#else - f = fopen(filename, mode); -#endif - return f; -} - - -STBIDEF stbi_uc *stbi_load(char const *filename, int *x, int *y, int *comp, int req_comp) -{ - FILE *f = stbi__fopen(filename, "rb"); - unsigned char *result; - if (!f) return stbi__errpuc("can't fopen", "Unable to open file"); - result = stbi_load_from_file(f,x,y,comp,req_comp); - fclose(f); - return result; -} - -STBIDEF stbi_uc *stbi_load_from_file(FILE *f, int *x, int *y, int *comp, int req_comp) -{ - unsigned char *result; - stbi__context s; - stbi__start_file(&s,f); - result = stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp); - if (result) { - // need to 'unget' all the characters in the IO buffer - fseek(f, - (int) (s.img_buffer_end - s.img_buffer), SEEK_CUR); - } - return result; -} - -STBIDEF stbi__uint16 *stbi_load_from_file_16(FILE *f, int *x, int *y, int *comp, int req_comp) -{ - stbi__uint16 *result; - stbi__context s; - stbi__start_file(&s,f); - result = stbi__load_and_postprocess_16bit(&s,x,y,comp,req_comp); - if (result) { - // need to 'unget' all the characters in the IO buffer - fseek(f, - (int) (s.img_buffer_end - s.img_buffer), SEEK_CUR); - } - return result; -} - -STBIDEF stbi_us *stbi_load_16(char const *filename, int *x, int *y, int *comp, int req_comp) -{ - FILE *f = stbi__fopen(filename, "rb"); - stbi__uint16 *result; - if (!f) return (stbi_us *) stbi__errpuc("can't fopen", "Unable to open file"); - result = stbi_load_from_file_16(f,x,y,comp,req_comp); - fclose(f); - return result; -} - - -#endif //!STBI_NO_STDIO - -STBIDEF stbi_us *stbi_load_16_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *channels_in_file, int desired_channels) -{ - stbi__context s; - stbi__start_mem(&s,buffer,len); - return stbi__load_and_postprocess_16bit(&s,x,y,channels_in_file,desired_channels); -} - -STBIDEF stbi_us *stbi_load_16_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *channels_in_file, int desired_channels) -{ - stbi__context s; - stbi__start_callbacks(&s, (stbi_io_callbacks *)clbk, user); - return stbi__load_and_postprocess_16bit(&s,x,y,channels_in_file,desired_channels); -} - -STBIDEF stbi_uc *stbi_load_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp, int req_comp) -{ - stbi__context s; - stbi__start_mem(&s,buffer,len); - return stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp); -} - -STBIDEF stbi_uc *stbi_load_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp, int req_comp) -{ - stbi__context s; - stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user); - return stbi__load_and_postprocess_8bit(&s,x,y,comp,req_comp); -} - -#ifndef STBI_NO_GIF -STBIDEF stbi_uc *stbi_load_gif_from_memory(stbi_uc const *buffer, int len, int **delays, int *x, int *y, int *z, int *comp, int req_comp) -{ - unsigned char *result; - stbi__context s; - stbi__start_mem(&s,buffer,len); - - result = (unsigned char*) stbi__load_gif_main(&s, delays, x, y, z, comp, req_comp); - if (stbi__vertically_flip_on_load) { - stbi__vertical_flip_slices( result, *x, *y, *z, *comp ); - } - - return result; -} -#endif - -#ifndef STBI_NO_LINEAR -static float *stbi__loadf_main(stbi__context *s, int *x, int *y, int *comp, int req_comp) -{ - unsigned char *data; - #ifndef STBI_NO_HDR - if (stbi__hdr_test(s)) { - stbi__result_info ri; - float *hdr_data = stbi__hdr_load(s,x,y,comp,req_comp, &ri); - if (hdr_data) - stbi__float_postprocess(hdr_data,x,y,comp,req_comp); - return hdr_data; - } - #endif - data = stbi__load_and_postprocess_8bit(s, x, y, comp, req_comp); - if (data) - return stbi__ldr_to_hdr(data, *x, *y, req_comp ? req_comp : *comp); - return stbi__errpf("unknown image type", "Image not of any known type, or corrupt"); -} - -STBIDEF float *stbi_loadf_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp, int req_comp) -{ - stbi__context s; - stbi__start_mem(&s,buffer,len); - return stbi__loadf_main(&s,x,y,comp,req_comp); -} - -STBIDEF float *stbi_loadf_from_callbacks(stbi_io_callbacks const *clbk, void *user, int *x, int *y, int *comp, int req_comp) -{ - stbi__context s; - stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user); - return stbi__loadf_main(&s,x,y,comp,req_comp); -} - -#ifndef STBI_NO_STDIO -STBIDEF float *stbi_loadf(char const *filename, int *x, int *y, int *comp, int req_comp) -{ - float *result; - FILE *f = stbi__fopen(filename, "rb"); - if (!f) return stbi__errpf("can't fopen", "Unable to open file"); - result = stbi_loadf_from_file(f,x,y,comp,req_comp); - fclose(f); - return result; -} - -STBIDEF float *stbi_loadf_from_file(FILE *f, int *x, int *y, int *comp, int req_comp) -{ - stbi__context s; - stbi__start_file(&s,f); - return stbi__loadf_main(&s,x,y,comp,req_comp); -} -#endif // !STBI_NO_STDIO - -#endif // !STBI_NO_LINEAR - -// these is-hdr-or-not is defined independent of whether STBI_NO_LINEAR is -// defined, for API simplicity; if STBI_NO_LINEAR is defined, it always -// reports false! - -STBIDEF int stbi_is_hdr_from_memory(stbi_uc const *buffer, int len) -{ - #ifndef STBI_NO_HDR - stbi__context s; - stbi__start_mem(&s,buffer,len); - return stbi__hdr_test(&s); - #else - STBI_NOTUSED(buffer); - STBI_NOTUSED(len); - return 0; - #endif -} - -#ifndef STBI_NO_STDIO -STBIDEF int stbi_is_hdr (char const *filename) -{ - FILE *f = stbi__fopen(filename, "rb"); - int result=0; - if (f) { - result = stbi_is_hdr_from_file(f); - fclose(f); - } - return result; -} - -STBIDEF int stbi_is_hdr_from_file(FILE *f) -{ - #ifndef STBI_NO_HDR - long pos = ftell(f); - int res; - stbi__context s; - stbi__start_file(&s,f); - res = stbi__hdr_test(&s); - fseek(f, pos, SEEK_SET); - return res; - #else - STBI_NOTUSED(f); - return 0; - #endif -} -#endif // !STBI_NO_STDIO - -STBIDEF int stbi_is_hdr_from_callbacks(stbi_io_callbacks const *clbk, void *user) -{ - #ifndef STBI_NO_HDR - stbi__context s; - stbi__start_callbacks(&s, (stbi_io_callbacks *) clbk, user); - return stbi__hdr_test(&s); - #else - STBI_NOTUSED(clbk); - STBI_NOTUSED(user); - return 0; - #endif -} - -#ifndef STBI_NO_LINEAR -static float stbi__l2h_gamma=2.2f, stbi__l2h_scale=1.0f; - -STBIDEF void stbi_ldr_to_hdr_gamma(float gamma) { stbi__l2h_gamma = gamma; } -STBIDEF void stbi_ldr_to_hdr_scale(float scale) { stbi__l2h_scale = scale; } -#endif - -static float stbi__h2l_gamma_i=1.0f/2.2f, stbi__h2l_scale_i=1.0f; - -STBIDEF void stbi_hdr_to_ldr_gamma(float gamma) { stbi__h2l_gamma_i = 1/gamma; } -STBIDEF void stbi_hdr_to_ldr_scale(float scale) { stbi__h2l_scale_i = 1/scale; } - - -////////////////////////////////////////////////////////////////////////////// -// -// Common code used by all image loaders -// - -enum -{ - STBI__SCAN_load=0, - STBI__SCAN_type, - STBI__SCAN_header -}; - -static void stbi__refill_buffer(stbi__context *s) -{ - int n = (s->io.read)(s->io_user_data,(char*)s->buffer_start,s->buflen); - if (n == 0) { - // at end of file, treat same as if from memory, but need to handle case - // where s->img_buffer isn't pointing to safe memory, e.g. 0-byte file - s->read_from_callbacks = 0; - s->img_buffer = s->buffer_start; - s->img_buffer_end = s->buffer_start+1; - *s->img_buffer = 0; - } else { - s->img_buffer = s->buffer_start; - s->img_buffer_end = s->buffer_start + n; - } -} - -stbi_inline static stbi_uc stbi__get8(stbi__context *s) -{ - if (s->img_buffer < s->img_buffer_end) - return *s->img_buffer++; - if (s->read_from_callbacks) { - stbi__refill_buffer(s); - return *s->img_buffer++; - } - return 0; -} - -stbi_inline static int stbi__at_eof(stbi__context *s) -{ - if (s->io.read) { - if (!(s->io.eof)(s->io_user_data)) return 0; - // if feof() is true, check if buffer = end - // special case: we've only got the special 0 character at the end - if (s->read_from_callbacks == 0) return 1; - } - - return s->img_buffer >= s->img_buffer_end; -} - -static void stbi__skip(stbi__context *s, int n) -{ - if (n < 0) { - s->img_buffer = s->img_buffer_end; - return; - } - if (s->io.read) { - int blen = (int) (s->img_buffer_end - s->img_buffer); - if (blen < n) { - s->img_buffer = s->img_buffer_end; - (s->io.skip)(s->io_user_data, n - blen); - return; - } - } - s->img_buffer += n; -} - -static int stbi__getn(stbi__context *s, stbi_uc *buffer, int n) -{ - if (s->io.read) { - int blen = (int) (s->img_buffer_end - s->img_buffer); - if (blen < n) { - int res, count; - - memcpy(buffer, s->img_buffer, blen); - - count = (s->io.read)(s->io_user_data, (char*) buffer + blen, n - blen); - res = (count == (n-blen)); - s->img_buffer = s->img_buffer_end; - return res; - } - } - - if (s->img_buffer+n <= s->img_buffer_end) { - memcpy(buffer, s->img_buffer, n); - s->img_buffer += n; - return 1; - } else - return 0; -} - -static int stbi__get16be(stbi__context *s) -{ - int z = stbi__get8(s); - return (z << 8) + stbi__get8(s); -} - -static stbi__uint32 stbi__get32be(stbi__context *s) -{ - stbi__uint32 z = stbi__get16be(s); - return (z << 16) + stbi__get16be(s); -} - -#if defined(STBI_NO_BMP) && defined(STBI_NO_TGA) && defined(STBI_NO_GIF) -// nothing -#else -static int stbi__get16le(stbi__context *s) -{ - int z = stbi__get8(s); - return z + (stbi__get8(s) << 8); -} -#endif - -#ifndef STBI_NO_BMP -static stbi__uint32 stbi__get32le(stbi__context *s) -{ - stbi__uint32 z = stbi__get16le(s); - return z + (stbi__get16le(s) << 16); -} -#endif - -#define STBI__BYTECAST(x) ((stbi_uc) ((x) & 255)) // truncate int to byte without warnings - - -////////////////////////////////////////////////////////////////////////////// -// -// generic converter from built-in img_n to req_comp -// individual types do this automatically as much as possible (e.g. jpeg -// does all cases internally since it needs to colorspace convert anyway, -// and it never has alpha, so very few cases ). png can automatically -// interleave an alpha=255 channel, but falls back to this for other cases -// -// assume data buffer is malloced, so malloc a new one and free that one -// only failure mode is malloc failing - -static stbi_uc stbi__compute_y(int r, int g, int b) -{ - return (stbi_uc) (((r*77) + (g*150) + (29*b)) >> 8); -} - -static unsigned char *stbi__convert_format(unsigned char *data, int img_n, int req_comp, unsigned int x, unsigned int y) -{ - int i,j; - unsigned char *good; - - if (req_comp == img_n) return data; - STBI_ASSERT(req_comp >= 1 && req_comp <= 4); - - good = (unsigned char *) stbi__malloc_mad3(req_comp, x, y, 0); - if (good == NULL) { - STBI_FREE(data); - return stbi__errpuc("outofmem", "Out of memory"); - } - - for (j=0; j < (int) y; ++j) { - unsigned char *src = data + j * x * img_n ; - unsigned char *dest = good + j * x * req_comp; - - #define STBI__COMBO(a,b) ((a)*8+(b)) - #define STBI__CASE(a,b) case STBI__COMBO(a,b): for(i=x-1; i >= 0; --i, src += a, dest += b) - // convert source image with img_n components to one with req_comp components; - // avoid switch per pixel, so use switch per scanline and massive macros - switch (STBI__COMBO(img_n, req_comp)) { - STBI__CASE(1,2) { dest[0]=src[0]; dest[1]=255; } break; - STBI__CASE(1,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; - STBI__CASE(1,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=255; } break; - STBI__CASE(2,1) { dest[0]=src[0]; } break; - STBI__CASE(2,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; - STBI__CASE(2,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=src[1]; } break; - STBI__CASE(3,4) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2];dest[3]=255; } break; - STBI__CASE(3,1) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); } break; - STBI__CASE(3,2) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); dest[1] = 255; } break; - STBI__CASE(4,1) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); } break; - STBI__CASE(4,2) { dest[0]=stbi__compute_y(src[0],src[1],src[2]); dest[1] = src[3]; } break; - STBI__CASE(4,3) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2]; } break; - default: STBI_ASSERT(0); - } - #undef STBI__CASE - } - - STBI_FREE(data); - return good; -} - -static stbi__uint16 stbi__compute_y_16(int r, int g, int b) -{ - return (stbi__uint16) (((r*77) + (g*150) + (29*b)) >> 8); -} - -static stbi__uint16 *stbi__convert_format16(stbi__uint16 *data, int img_n, int req_comp, unsigned int x, unsigned int y) -{ - int i,j; - stbi__uint16 *good; - - if (req_comp == img_n) return data; - STBI_ASSERT(req_comp >= 1 && req_comp <= 4); - - good = (stbi__uint16 *) stbi__malloc(req_comp * x * y * 2); - if (good == NULL) { - STBI_FREE(data); - return (stbi__uint16 *) stbi__errpuc("outofmem", "Out of memory"); - } - - for (j=0; j < (int) y; ++j) { - stbi__uint16 *src = data + j * x * img_n ; - stbi__uint16 *dest = good + j * x * req_comp; - - #define STBI__COMBO(a,b) ((a)*8+(b)) - #define STBI__CASE(a,b) case STBI__COMBO(a,b): for(i=x-1; i >= 0; --i, src += a, dest += b) - // convert source image with img_n components to one with req_comp components; - // avoid switch per pixel, so use switch per scanline and massive macros - switch (STBI__COMBO(img_n, req_comp)) { - STBI__CASE(1,2) { dest[0]=src[0]; dest[1]=0xffff; } break; - STBI__CASE(1,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; - STBI__CASE(1,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=0xffff; } break; - STBI__CASE(2,1) { dest[0]=src[0]; } break; - STBI__CASE(2,3) { dest[0]=dest[1]=dest[2]=src[0]; } break; - STBI__CASE(2,4) { dest[0]=dest[1]=dest[2]=src[0]; dest[3]=src[1]; } break; - STBI__CASE(3,4) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2];dest[3]=0xffff; } break; - STBI__CASE(3,1) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); } break; - STBI__CASE(3,2) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); dest[1] = 0xffff; } break; - STBI__CASE(4,1) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); } break; - STBI__CASE(4,2) { dest[0]=stbi__compute_y_16(src[0],src[1],src[2]); dest[1] = src[3]; } break; - STBI__CASE(4,3) { dest[0]=src[0];dest[1]=src[1];dest[2]=src[2]; } break; - default: STBI_ASSERT(0); - } - #undef STBI__CASE - } - - STBI_FREE(data); - return good; -} - -#ifndef STBI_NO_LINEAR -static float *stbi__ldr_to_hdr(stbi_uc *data, int x, int y, int comp) -{ - int i,k,n; - float *output; - if (!data) return NULL; - output = (float *) stbi__malloc_mad4(x, y, comp, sizeof(float), 0); - if (output == NULL) { STBI_FREE(data); return stbi__errpf("outofmem", "Out of memory"); } - // compute number of non-alpha components - if (comp & 1) n = comp; else n = comp-1; - for (i=0; i < x*y; ++i) { - for (k=0; k < n; ++k) { - output[i*comp + k] = (float) (pow(data[i*comp+k]/255.0f, stbi__l2h_gamma) * stbi__l2h_scale); - } - } - if (n < comp) { - for (i=0; i < x*y; ++i) { - output[i*comp + n] = data[i*comp + n]/255.0f; - } - } - STBI_FREE(data); - return output; -} -#endif - -#ifndef STBI_NO_HDR -#define stbi__float2int(x) ((int) (x)) -static stbi_uc *stbi__hdr_to_ldr(float *data, int x, int y, int comp) -{ - int i,k,n; - stbi_uc *output; - if (!data) return NULL; - output = (stbi_uc *) stbi__malloc_mad3(x, y, comp, 0); - if (output == NULL) { STBI_FREE(data); return stbi__errpuc("outofmem", "Out of memory"); } - // compute number of non-alpha components - if (comp & 1) n = comp; else n = comp-1; - for (i=0; i < x*y; ++i) { - for (k=0; k < n; ++k) { - float z = (float) pow(data[i*comp+k]*stbi__h2l_scale_i, stbi__h2l_gamma_i) * 255 + 0.5f; - if (z < 0) z = 0; - if (z > 255) z = 255; - output[i*comp + k] = (stbi_uc) stbi__float2int(z); - } - if (k < comp) { - float z = data[i*comp+k] * 255 + 0.5f; - if (z < 0) z = 0; - if (z > 255) z = 255; - output[i*comp + k] = (stbi_uc) stbi__float2int(z); - } - } - STBI_FREE(data); - return output; -} -#endif - -////////////////////////////////////////////////////////////////////////////// -// -// "baseline" JPEG/JFIF decoder -// -// simple implementation -// - doesn't support delayed output of y-dimension -// - simple interface (only one output format: 8-bit interleaved RGB) -// - doesn't try to recover corrupt jpegs -// - doesn't allow partial loading, loading multiple at once -// - still fast on x86 (copying globals into locals doesn't help x86) -// - allocates lots of intermediate memory (full size of all components) -// - non-interleaved case requires this anyway -// - allows good upsampling (see next) -// high-quality -// - upsampled channels are bilinearly interpolated, even across blocks -// - quality integer IDCT derived from IJG's 'slow' -// performance -// - fast huffman; reasonable integer IDCT -// - some SIMD kernels for common paths on targets with SSE2/NEON -// - uses a lot of intermediate memory, could cache poorly - -#ifndef STBI_NO_JPEG - -// huffman decoding acceleration -#define FAST_BITS 9 // larger handles more cases; smaller stomps less cache - -typedef struct -{ - stbi_uc fast[1 << FAST_BITS]; - // weirdly, repacking this into AoS is a 10% speed loss, instead of a win - stbi__uint16 code[256]; - stbi_uc values[256]; - stbi_uc size[257]; - unsigned int maxcode[18]; - int delta[17]; // old 'firstsymbol' - old 'firstcode' -} stbi__huffman; - -typedef struct -{ - stbi__context *s; - stbi__huffman huff_dc[4]; - stbi__huffman huff_ac[4]; - stbi__uint16 dequant[4][64]; - stbi__int16 fast_ac[4][1 << FAST_BITS]; - -// sizes for components, interleaved MCUs - int img_h_max, img_v_max; - int img_mcu_x, img_mcu_y; - int img_mcu_w, img_mcu_h; - -// definition of jpeg image component - struct - { - int id; - int h,v; - int tq; - int hd,ha; - int dc_pred; - - int x,y,w2,h2; - stbi_uc *data; - void *raw_data, *raw_coeff; - stbi_uc *linebuf; - short *coeff; // progressive only - int coeff_w, coeff_h; // number of 8x8 coefficient blocks - } img_comp[4]; - - stbi__uint32 code_buffer; // jpeg entropy-coded buffer - int code_bits; // number of valid bits - unsigned char marker; // marker seen while filling entropy buffer - int nomore; // flag if we saw a marker so must stop - - int progressive; - int spec_start; - int spec_end; - int succ_high; - int succ_low; - int eob_run; - int jfif; - int app14_color_transform; // Adobe APP14 tag - int rgb; - - int scan_n, order[4]; - int restart_interval, todo; - -// kernels - void (*idct_block_kernel)(stbi_uc *out, int out_stride, short data[64]); - void (*YCbCr_to_RGB_kernel)(stbi_uc *out, const stbi_uc *y, const stbi_uc *pcb, const stbi_uc *pcr, int count, int step); - stbi_uc *(*resample_row_hv_2_kernel)(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs); -} stbi__jpeg; - -static int stbi__build_huffman(stbi__huffman *h, int *count) -{ - int i,j,k=0; - unsigned int code; - // build size list for each symbol (from JPEG spec) - for (i=0; i < 16; ++i) - for (j=0; j < count[i]; ++j) - h->size[k++] = (stbi_uc) (i+1); - h->size[k] = 0; - - // compute actual symbols (from jpeg spec) - code = 0; - k = 0; - for(j=1; j <= 16; ++j) { - // compute delta to add to code to compute symbol id - h->delta[j] = k - code; - if (h->size[k] == j) { - while (h->size[k] == j) - h->code[k++] = (stbi__uint16) (code++); - if (code-1 >= (1u << j)) return stbi__err("bad code lengths","Corrupt JPEG"); - } - // compute largest code + 1 for this size, preshifted as needed later - h->maxcode[j] = code << (16-j); - code <<= 1; - } - h->maxcode[j] = 0xffffffff; - - // build non-spec acceleration table; 255 is flag for not-accelerated - memset(h->fast, 255, 1 << FAST_BITS); - for (i=0; i < k; ++i) { - int s = h->size[i]; - if (s <= FAST_BITS) { - int c = h->code[i] << (FAST_BITS-s); - int m = 1 << (FAST_BITS-s); - for (j=0; j < m; ++j) { - h->fast[c+j] = (stbi_uc) i; - } - } - } - return 1; -} - -// build a table that decodes both magnitude and value of small ACs in -// one go. -static void stbi__build_fast_ac(stbi__int16 *fast_ac, stbi__huffman *h) -{ - int i; - for (i=0; i < (1 << FAST_BITS); ++i) { - stbi_uc fast = h->fast[i]; - fast_ac[i] = 0; - if (fast < 255) { - int rs = h->values[fast]; - int run = (rs >> 4) & 15; - int magbits = rs & 15; - int len = h->size[fast]; - - if (magbits && len + magbits <= FAST_BITS) { - // magnitude code followed by receive_extend code - int k = ((i << len) & ((1 << FAST_BITS) - 1)) >> (FAST_BITS - magbits); - int m = 1 << (magbits - 1); - if (k < m) k += (~0U << magbits) + 1; - // if the result is small enough, we can fit it in fast_ac table - if (k >= -128 && k <= 127) - fast_ac[i] = (stbi__int16) ((k * 256) + (run * 16) + (len + magbits)); - } - } - } -} - -static void stbi__grow_buffer_unsafe(stbi__jpeg *j) -{ - do { - unsigned int b = j->nomore ? 0 : stbi__get8(j->s); - if (b == 0xff) { - int c = stbi__get8(j->s); - while (c == 0xff) c = stbi__get8(j->s); // consume fill bytes - if (c != 0) { - j->marker = (unsigned char) c; - j->nomore = 1; - return; - } - } - j->code_buffer |= b << (24 - j->code_bits); - j->code_bits += 8; - } while (j->code_bits <= 24); -} - -// (1 << n) - 1 -static const stbi__uint32 stbi__bmask[17]={0,1,3,7,15,31,63,127,255,511,1023,2047,4095,8191,16383,32767,65535}; - -// decode a jpeg huffman value from the bitstream -stbi_inline static int stbi__jpeg_huff_decode(stbi__jpeg *j, stbi__huffman *h) -{ - unsigned int temp; - int c,k; - - if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); - - // look at the top FAST_BITS and determine what symbol ID it is, - // if the code is <= FAST_BITS - c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1); - k = h->fast[c]; - if (k < 255) { - int s = h->size[k]; - if (s > j->code_bits) - return -1; - j->code_buffer <<= s; - j->code_bits -= s; - return h->values[k]; - } - - // naive test is to shift the code_buffer down so k bits are - // valid, then test against maxcode. To speed this up, we've - // preshifted maxcode left so that it has (16-k) 0s at the - // end; in other words, regardless of the number of bits, it - // wants to be compared against something shifted to have 16; - // that way we don't need to shift inside the loop. - temp = j->code_buffer >> 16; - for (k=FAST_BITS+1 ; ; ++k) - if (temp < h->maxcode[k]) - break; - if (k == 17) { - // error! code not found - j->code_bits -= 16; - return -1; - } - - if (k > j->code_bits) - return -1; - - // convert the huffman code to the symbol id - c = ((j->code_buffer >> (32 - k)) & stbi__bmask[k]) + h->delta[k]; - STBI_ASSERT((((j->code_buffer) >> (32 - h->size[c])) & stbi__bmask[h->size[c]]) == h->code[c]); - - // convert the id to a symbol - j->code_bits -= k; - j->code_buffer <<= k; - return h->values[c]; -} - -// bias[n] = (-1<code_bits < n) stbi__grow_buffer_unsafe(j); - - sgn = (stbi__int32)j->code_buffer >> 31; // sign bit is always in MSB - k = stbi_lrot(j->code_buffer, n); - STBI_ASSERT(n >= 0 && n < (int) (sizeof(stbi__bmask)/sizeof(*stbi__bmask))); - j->code_buffer = k & ~stbi__bmask[n]; - k &= stbi__bmask[n]; - j->code_bits -= n; - return k + (stbi__jbias[n] & ~sgn); -} - -// get some unsigned bits -stbi_inline static int stbi__jpeg_get_bits(stbi__jpeg *j, int n) -{ - unsigned int k; - if (j->code_bits < n) stbi__grow_buffer_unsafe(j); - k = stbi_lrot(j->code_buffer, n); - j->code_buffer = k & ~stbi__bmask[n]; - k &= stbi__bmask[n]; - j->code_bits -= n; - return k; -} - -stbi_inline static int stbi__jpeg_get_bit(stbi__jpeg *j) -{ - unsigned int k; - if (j->code_bits < 1) stbi__grow_buffer_unsafe(j); - k = j->code_buffer; - j->code_buffer <<= 1; - --j->code_bits; - return k & 0x80000000; -} - -// given a value that's at position X in the zigzag stream, -// where does it appear in the 8x8 matrix coded as row-major? -static const stbi_uc stbi__jpeg_dezigzag[64+15] = -{ - 0, 1, 8, 16, 9, 2, 3, 10, - 17, 24, 32, 25, 18, 11, 4, 5, - 12, 19, 26, 33, 40, 48, 41, 34, - 27, 20, 13, 6, 7, 14, 21, 28, - 35, 42, 49, 56, 57, 50, 43, 36, - 29, 22, 15, 23, 30, 37, 44, 51, - 58, 59, 52, 45, 38, 31, 39, 46, - 53, 60, 61, 54, 47, 55, 62, 63, - // let corrupt input sample past end - 63, 63, 63, 63, 63, 63, 63, 63, - 63, 63, 63, 63, 63, 63, 63 -}; - -// decode one 64-entry block-- -static int stbi__jpeg_decode_block(stbi__jpeg *j, short data[64], stbi__huffman *hdc, stbi__huffman *hac, stbi__int16 *fac, int b, stbi__uint16 *dequant) -{ - int diff,dc,k; - int t; - - if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); - t = stbi__jpeg_huff_decode(j, hdc); - if (t < 0) return stbi__err("bad huffman code","Corrupt JPEG"); - - // 0 all the ac values now so we can do it 32-bits at a time - memset(data,0,64*sizeof(data[0])); - - diff = t ? stbi__extend_receive(j, t) : 0; - dc = j->img_comp[b].dc_pred + diff; - j->img_comp[b].dc_pred = dc; - data[0] = (short) (dc * dequant[0]); - - // decode AC components, see JPEG spec - k = 1; - do { - unsigned int zig; - int c,r,s; - if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); - c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1); - r = fac[c]; - if (r) { // fast-AC path - k += (r >> 4) & 15; // run - s = r & 15; // combined length - j->code_buffer <<= s; - j->code_bits -= s; - // decode into unzigzag'd location - zig = stbi__jpeg_dezigzag[k++]; - data[zig] = (short) ((r >> 8) * dequant[zig]); - } else { - int rs = stbi__jpeg_huff_decode(j, hac); - if (rs < 0) return stbi__err("bad huffman code","Corrupt JPEG"); - s = rs & 15; - r = rs >> 4; - if (s == 0) { - if (rs != 0xf0) break; // end block - k += 16; - } else { - k += r; - // decode into unzigzag'd location - zig = stbi__jpeg_dezigzag[k++]; - data[zig] = (short) (stbi__extend_receive(j,s) * dequant[zig]); - } - } - } while (k < 64); - return 1; -} - -static int stbi__jpeg_decode_block_prog_dc(stbi__jpeg *j, short data[64], stbi__huffman *hdc, int b) -{ - int diff,dc; - int t; - if (j->spec_end != 0) return stbi__err("can't merge dc and ac", "Corrupt JPEG"); - - if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); - - if (j->succ_high == 0) { - // first scan for DC coefficient, must be first - memset(data,0,64*sizeof(data[0])); // 0 all the ac values now - t = stbi__jpeg_huff_decode(j, hdc); - diff = t ? stbi__extend_receive(j, t) : 0; - - dc = j->img_comp[b].dc_pred + diff; - j->img_comp[b].dc_pred = dc; - data[0] = (short) (dc << j->succ_low); - } else { - // refinement scan for DC coefficient - if (stbi__jpeg_get_bit(j)) - data[0] += (short) (1 << j->succ_low); - } - return 1; -} - -// @OPTIMIZE: store non-zigzagged during the decode passes, -// and only de-zigzag when dequantizing -static int stbi__jpeg_decode_block_prog_ac(stbi__jpeg *j, short data[64], stbi__huffman *hac, stbi__int16 *fac) -{ - int k; - if (j->spec_start == 0) return stbi__err("can't merge dc and ac", "Corrupt JPEG"); - - if (j->succ_high == 0) { - int shift = j->succ_low; - - if (j->eob_run) { - --j->eob_run; - return 1; - } - - k = j->spec_start; - do { - unsigned int zig; - int c,r,s; - if (j->code_bits < 16) stbi__grow_buffer_unsafe(j); - c = (j->code_buffer >> (32 - FAST_BITS)) & ((1 << FAST_BITS)-1); - r = fac[c]; - if (r) { // fast-AC path - k += (r >> 4) & 15; // run - s = r & 15; // combined length - j->code_buffer <<= s; - j->code_bits -= s; - zig = stbi__jpeg_dezigzag[k++]; - data[zig] = (short) ((r >> 8) << shift); - } else { - int rs = stbi__jpeg_huff_decode(j, hac); - if (rs < 0) return stbi__err("bad huffman code","Corrupt JPEG"); - s = rs & 15; - r = rs >> 4; - if (s == 0) { - if (r < 15) { - j->eob_run = (1 << r); - if (r) - j->eob_run += stbi__jpeg_get_bits(j, r); - --j->eob_run; - break; - } - k += 16; - } else { - k += r; - zig = stbi__jpeg_dezigzag[k++]; - data[zig] = (short) (stbi__extend_receive(j,s) << shift); - } - } - } while (k <= j->spec_end); - } else { - // refinement scan for these AC coefficients - - short bit = (short) (1 << j->succ_low); - - if (j->eob_run) { - --j->eob_run; - for (k = j->spec_start; k <= j->spec_end; ++k) { - short *p = &data[stbi__jpeg_dezigzag[k]]; - if (*p != 0) - if (stbi__jpeg_get_bit(j)) - if ((*p & bit)==0) { - if (*p > 0) - *p += bit; - else - *p -= bit; - } - } - } else { - k = j->spec_start; - do { - int r,s; - int rs = stbi__jpeg_huff_decode(j, hac); // @OPTIMIZE see if we can use the fast path here, advance-by-r is so slow, eh - if (rs < 0) return stbi__err("bad huffman code","Corrupt JPEG"); - s = rs & 15; - r = rs >> 4; - if (s == 0) { - if (r < 15) { - j->eob_run = (1 << r) - 1; - if (r) - j->eob_run += stbi__jpeg_get_bits(j, r); - r = 64; // force end of block - } else { - // r=15 s=0 should write 16 0s, so we just do - // a run of 15 0s and then write s (which is 0), - // so we don't have to do anything special here - } - } else { - if (s != 1) return stbi__err("bad huffman code", "Corrupt JPEG"); - // sign bit - if (stbi__jpeg_get_bit(j)) - s = bit; - else - s = -bit; - } - - // advance by r - while (k <= j->spec_end) { - short *p = &data[stbi__jpeg_dezigzag[k++]]; - if (*p != 0) { - if (stbi__jpeg_get_bit(j)) - if ((*p & bit)==0) { - if (*p > 0) - *p += bit; - else - *p -= bit; - } - } else { - if (r == 0) { - *p = (short) s; - break; - } - --r; - } - } - } while (k <= j->spec_end); - } - } - return 1; -} - -// take a -128..127 value and stbi__clamp it and convert to 0..255 -stbi_inline static stbi_uc stbi__clamp(int x) -{ - // trick to use a single test to catch both cases - if ((unsigned int) x > 255) { - if (x < 0) return 0; - if (x > 255) return 255; - } - return (stbi_uc) x; -} - -#define stbi__f2f(x) ((int) (((x) * 4096 + 0.5))) -#define stbi__fsh(x) ((x) * 4096) - -// derived from jidctint -- DCT_ISLOW -#define STBI__IDCT_1D(s0,s1,s2,s3,s4,s5,s6,s7) \ - int t0,t1,t2,t3,p1,p2,p3,p4,p5,x0,x1,x2,x3; \ - p2 = s2; \ - p3 = s6; \ - p1 = (p2+p3) * stbi__f2f(0.5411961f); \ - t2 = p1 + p3*stbi__f2f(-1.847759065f); \ - t3 = p1 + p2*stbi__f2f( 0.765366865f); \ - p2 = s0; \ - p3 = s4; \ - t0 = stbi__fsh(p2+p3); \ - t1 = stbi__fsh(p2-p3); \ - x0 = t0+t3; \ - x3 = t0-t3; \ - x1 = t1+t2; \ - x2 = t1-t2; \ - t0 = s7; \ - t1 = s5; \ - t2 = s3; \ - t3 = s1; \ - p3 = t0+t2; \ - p4 = t1+t3; \ - p1 = t0+t3; \ - p2 = t1+t2; \ - p5 = (p3+p4)*stbi__f2f( 1.175875602f); \ - t0 = t0*stbi__f2f( 0.298631336f); \ - t1 = t1*stbi__f2f( 2.053119869f); \ - t2 = t2*stbi__f2f( 3.072711026f); \ - t3 = t3*stbi__f2f( 1.501321110f); \ - p1 = p5 + p1*stbi__f2f(-0.899976223f); \ - p2 = p5 + p2*stbi__f2f(-2.562915447f); \ - p3 = p3*stbi__f2f(-1.961570560f); \ - p4 = p4*stbi__f2f(-0.390180644f); \ - t3 += p1+p4; \ - t2 += p2+p3; \ - t1 += p2+p4; \ - t0 += p1+p3; - -static void stbi__idct_block(stbi_uc *out, int out_stride, short data[64]) -{ - int i,val[64],*v=val; - stbi_uc *o; - short *d = data; - - // columns - for (i=0; i < 8; ++i,++d, ++v) { - // if all zeroes, shortcut -- this avoids dequantizing 0s and IDCTing - if (d[ 8]==0 && d[16]==0 && d[24]==0 && d[32]==0 - && d[40]==0 && d[48]==0 && d[56]==0) { - // no shortcut 0 seconds - // (1|2|3|4|5|6|7)==0 0 seconds - // all separate -0.047 seconds - // 1 && 2|3 && 4|5 && 6|7: -0.047 seconds - int dcterm = d[0]*4; - v[0] = v[8] = v[16] = v[24] = v[32] = v[40] = v[48] = v[56] = dcterm; - } else { - STBI__IDCT_1D(d[ 0],d[ 8],d[16],d[24],d[32],d[40],d[48],d[56]) - // constants scaled things up by 1<<12; let's bring them back - // down, but keep 2 extra bits of precision - x0 += 512; x1 += 512; x2 += 512; x3 += 512; - v[ 0] = (x0+t3) >> 10; - v[56] = (x0-t3) >> 10; - v[ 8] = (x1+t2) >> 10; - v[48] = (x1-t2) >> 10; - v[16] = (x2+t1) >> 10; - v[40] = (x2-t1) >> 10; - v[24] = (x3+t0) >> 10; - v[32] = (x3-t0) >> 10; - } - } - - for (i=0, v=val, o=out; i < 8; ++i,v+=8,o+=out_stride) { - // no fast case since the first 1D IDCT spread components out - STBI__IDCT_1D(v[0],v[1],v[2],v[3],v[4],v[5],v[6],v[7]) - // constants scaled things up by 1<<12, plus we had 1<<2 from first - // loop, plus horizontal and vertical each scale by sqrt(8) so together - // we've got an extra 1<<3, so 1<<17 total we need to remove. - // so we want to round that, which means adding 0.5 * 1<<17, - // aka 65536. Also, we'll end up with -128 to 127 that we want - // to encode as 0..255 by adding 128, so we'll add that before the shift - x0 += 65536 + (128<<17); - x1 += 65536 + (128<<17); - x2 += 65536 + (128<<17); - x3 += 65536 + (128<<17); - // tried computing the shifts into temps, or'ing the temps to see - // if any were out of range, but that was slower - o[0] = stbi__clamp((x0+t3) >> 17); - o[7] = stbi__clamp((x0-t3) >> 17); - o[1] = stbi__clamp((x1+t2) >> 17); - o[6] = stbi__clamp((x1-t2) >> 17); - o[2] = stbi__clamp((x2+t1) >> 17); - o[5] = stbi__clamp((x2-t1) >> 17); - o[3] = stbi__clamp((x3+t0) >> 17); - o[4] = stbi__clamp((x3-t0) >> 17); - } -} - -#ifdef STBI_SSE2 -// sse2 integer IDCT. not the fastest possible implementation but it -// produces bit-identical results to the generic C version so it's -// fully "transparent". -static void stbi__idct_simd(stbi_uc *out, int out_stride, short data[64]) -{ - // This is constructed to match our regular (generic) integer IDCT exactly. - __m128i row0, row1, row2, row3, row4, row5, row6, row7; - __m128i tmp; - - // dot product constant: even elems=x, odd elems=y - #define dct_const(x,y) _mm_setr_epi16((x),(y),(x),(y),(x),(y),(x),(y)) - - // out(0) = c0[even]*x + c0[odd]*y (c0, x, y 16-bit, out 32-bit) - // out(1) = c1[even]*x + c1[odd]*y - #define dct_rot(out0,out1, x,y,c0,c1) \ - __m128i c0##lo = _mm_unpacklo_epi16((x),(y)); \ - __m128i c0##hi = _mm_unpackhi_epi16((x),(y)); \ - __m128i out0##_l = _mm_madd_epi16(c0##lo, c0); \ - __m128i out0##_h = _mm_madd_epi16(c0##hi, c0); \ - __m128i out1##_l = _mm_madd_epi16(c0##lo, c1); \ - __m128i out1##_h = _mm_madd_epi16(c0##hi, c1) - - // out = in << 12 (in 16-bit, out 32-bit) - #define dct_widen(out, in) \ - __m128i out##_l = _mm_srai_epi32(_mm_unpacklo_epi16(_mm_setzero_si128(), (in)), 4); \ - __m128i out##_h = _mm_srai_epi32(_mm_unpackhi_epi16(_mm_setzero_si128(), (in)), 4) - - // wide add - #define dct_wadd(out, a, b) \ - __m128i out##_l = _mm_add_epi32(a##_l, b##_l); \ - __m128i out##_h = _mm_add_epi32(a##_h, b##_h) - - // wide sub - #define dct_wsub(out, a, b) \ - __m128i out##_l = _mm_sub_epi32(a##_l, b##_l); \ - __m128i out##_h = _mm_sub_epi32(a##_h, b##_h) - - // butterfly a/b, add bias, then shift by "s" and pack - #define dct_bfly32o(out0, out1, a,b,bias,s) \ - { \ - __m128i abiased_l = _mm_add_epi32(a##_l, bias); \ - __m128i abiased_h = _mm_add_epi32(a##_h, bias); \ - dct_wadd(sum, abiased, b); \ - dct_wsub(dif, abiased, b); \ - out0 = _mm_packs_epi32(_mm_srai_epi32(sum_l, s), _mm_srai_epi32(sum_h, s)); \ - out1 = _mm_packs_epi32(_mm_srai_epi32(dif_l, s), _mm_srai_epi32(dif_h, s)); \ - } - - // 8-bit interleave step (for transposes) - #define dct_interleave8(a, b) \ - tmp = a; \ - a = _mm_unpacklo_epi8(a, b); \ - b = _mm_unpackhi_epi8(tmp, b) - - // 16-bit interleave step (for transposes) - #define dct_interleave16(a, b) \ - tmp = a; \ - a = _mm_unpacklo_epi16(a, b); \ - b = _mm_unpackhi_epi16(tmp, b) - - #define dct_pass(bias,shift) \ - { \ - /* even part */ \ - dct_rot(t2e,t3e, row2,row6, rot0_0,rot0_1); \ - __m128i sum04 = _mm_add_epi16(row0, row4); \ - __m128i dif04 = _mm_sub_epi16(row0, row4); \ - dct_widen(t0e, sum04); \ - dct_widen(t1e, dif04); \ - dct_wadd(x0, t0e, t3e); \ - dct_wsub(x3, t0e, t3e); \ - dct_wadd(x1, t1e, t2e); \ - dct_wsub(x2, t1e, t2e); \ - /* odd part */ \ - dct_rot(y0o,y2o, row7,row3, rot2_0,rot2_1); \ - dct_rot(y1o,y3o, row5,row1, rot3_0,rot3_1); \ - __m128i sum17 = _mm_add_epi16(row1, row7); \ - __m128i sum35 = _mm_add_epi16(row3, row5); \ - dct_rot(y4o,y5o, sum17,sum35, rot1_0,rot1_1); \ - dct_wadd(x4, y0o, y4o); \ - dct_wadd(x5, y1o, y5o); \ - dct_wadd(x6, y2o, y5o); \ - dct_wadd(x7, y3o, y4o); \ - dct_bfly32o(row0,row7, x0,x7,bias,shift); \ - dct_bfly32o(row1,row6, x1,x6,bias,shift); \ - dct_bfly32o(row2,row5, x2,x5,bias,shift); \ - dct_bfly32o(row3,row4, x3,x4,bias,shift); \ - } - - __m128i rot0_0 = dct_const(stbi__f2f(0.5411961f), stbi__f2f(0.5411961f) + stbi__f2f(-1.847759065f)); - __m128i rot0_1 = dct_const(stbi__f2f(0.5411961f) + stbi__f2f( 0.765366865f), stbi__f2f(0.5411961f)); - __m128i rot1_0 = dct_const(stbi__f2f(1.175875602f) + stbi__f2f(-0.899976223f), stbi__f2f(1.175875602f)); - __m128i rot1_1 = dct_const(stbi__f2f(1.175875602f), stbi__f2f(1.175875602f) + stbi__f2f(-2.562915447f)); - __m128i rot2_0 = dct_const(stbi__f2f(-1.961570560f) + stbi__f2f( 0.298631336f), stbi__f2f(-1.961570560f)); - __m128i rot2_1 = dct_const(stbi__f2f(-1.961570560f), stbi__f2f(-1.961570560f) + stbi__f2f( 3.072711026f)); - __m128i rot3_0 = dct_const(stbi__f2f(-0.390180644f) + stbi__f2f( 2.053119869f), stbi__f2f(-0.390180644f)); - __m128i rot3_1 = dct_const(stbi__f2f(-0.390180644f), stbi__f2f(-0.390180644f) + stbi__f2f( 1.501321110f)); - - // rounding biases in column/row passes, see stbi__idct_block for explanation. - __m128i bias_0 = _mm_set1_epi32(512); - __m128i bias_1 = _mm_set1_epi32(65536 + (128<<17)); - - // load - row0 = _mm_load_si128((const __m128i *) (data + 0*8)); - row1 = _mm_load_si128((const __m128i *) (data + 1*8)); - row2 = _mm_load_si128((const __m128i *) (data + 2*8)); - row3 = _mm_load_si128((const __m128i *) (data + 3*8)); - row4 = _mm_load_si128((const __m128i *) (data + 4*8)); - row5 = _mm_load_si128((const __m128i *) (data + 5*8)); - row6 = _mm_load_si128((const __m128i *) (data + 6*8)); - row7 = _mm_load_si128((const __m128i *) (data + 7*8)); - - // column pass - dct_pass(bias_0, 10); - - { - // 16bit 8x8 transpose pass 1 - dct_interleave16(row0, row4); - dct_interleave16(row1, row5); - dct_interleave16(row2, row6); - dct_interleave16(row3, row7); - - // transpose pass 2 - dct_interleave16(row0, row2); - dct_interleave16(row1, row3); - dct_interleave16(row4, row6); - dct_interleave16(row5, row7); - - // transpose pass 3 - dct_interleave16(row0, row1); - dct_interleave16(row2, row3); - dct_interleave16(row4, row5); - dct_interleave16(row6, row7); - } - - // row pass - dct_pass(bias_1, 17); - - { - // pack - __m128i p0 = _mm_packus_epi16(row0, row1); // a0a1a2a3...a7b0b1b2b3...b7 - __m128i p1 = _mm_packus_epi16(row2, row3); - __m128i p2 = _mm_packus_epi16(row4, row5); - __m128i p3 = _mm_packus_epi16(row6, row7); - - // 8bit 8x8 transpose pass 1 - dct_interleave8(p0, p2); // a0e0a1e1... - dct_interleave8(p1, p3); // c0g0c1g1... - - // transpose pass 2 - dct_interleave8(p0, p1); // a0c0e0g0... - dct_interleave8(p2, p3); // b0d0f0h0... - - // transpose pass 3 - dct_interleave8(p0, p2); // a0b0c0d0... - dct_interleave8(p1, p3); // a4b4c4d4... - - // store - _mm_storel_epi64((__m128i *) out, p0); out += out_stride; - _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p0, 0x4e)); out += out_stride; - _mm_storel_epi64((__m128i *) out, p2); out += out_stride; - _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p2, 0x4e)); out += out_stride; - _mm_storel_epi64((__m128i *) out, p1); out += out_stride; - _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p1, 0x4e)); out += out_stride; - _mm_storel_epi64((__m128i *) out, p3); out += out_stride; - _mm_storel_epi64((__m128i *) out, _mm_shuffle_epi32(p3, 0x4e)); - } - -#undef dct_const -#undef dct_rot -#undef dct_widen -#undef dct_wadd -#undef dct_wsub -#undef dct_bfly32o -#undef dct_interleave8 -#undef dct_interleave16 -#undef dct_pass -} - -#endif // STBI_SSE2 - -#ifdef STBI_NEON - -// NEON integer IDCT. should produce bit-identical -// results to the generic C version. -static void stbi__idct_simd(stbi_uc *out, int out_stride, short data[64]) -{ - int16x8_t row0, row1, row2, row3, row4, row5, row6, row7; - - int16x4_t rot0_0 = vdup_n_s16(stbi__f2f(0.5411961f)); - int16x4_t rot0_1 = vdup_n_s16(stbi__f2f(-1.847759065f)); - int16x4_t rot0_2 = vdup_n_s16(stbi__f2f( 0.765366865f)); - int16x4_t rot1_0 = vdup_n_s16(stbi__f2f( 1.175875602f)); - int16x4_t rot1_1 = vdup_n_s16(stbi__f2f(-0.899976223f)); - int16x4_t rot1_2 = vdup_n_s16(stbi__f2f(-2.562915447f)); - int16x4_t rot2_0 = vdup_n_s16(stbi__f2f(-1.961570560f)); - int16x4_t rot2_1 = vdup_n_s16(stbi__f2f(-0.390180644f)); - int16x4_t rot3_0 = vdup_n_s16(stbi__f2f( 0.298631336f)); - int16x4_t rot3_1 = vdup_n_s16(stbi__f2f( 2.053119869f)); - int16x4_t rot3_2 = vdup_n_s16(stbi__f2f( 3.072711026f)); - int16x4_t rot3_3 = vdup_n_s16(stbi__f2f( 1.501321110f)); - -#define dct_long_mul(out, inq, coeff) \ - int32x4_t out##_l = vmull_s16(vget_low_s16(inq), coeff); \ - int32x4_t out##_h = vmull_s16(vget_high_s16(inq), coeff) - -#define dct_long_mac(out, acc, inq, coeff) \ - int32x4_t out##_l = vmlal_s16(acc##_l, vget_low_s16(inq), coeff); \ - int32x4_t out##_h = vmlal_s16(acc##_h, vget_high_s16(inq), coeff) - -#define dct_widen(out, inq) \ - int32x4_t out##_l = vshll_n_s16(vget_low_s16(inq), 12); \ - int32x4_t out##_h = vshll_n_s16(vget_high_s16(inq), 12) - -// wide add -#define dct_wadd(out, a, b) \ - int32x4_t out##_l = vaddq_s32(a##_l, b##_l); \ - int32x4_t out##_h = vaddq_s32(a##_h, b##_h) - -// wide sub -#define dct_wsub(out, a, b) \ - int32x4_t out##_l = vsubq_s32(a##_l, b##_l); \ - int32x4_t out##_h = vsubq_s32(a##_h, b##_h) - -// butterfly a/b, then shift using "shiftop" by "s" and pack -#define dct_bfly32o(out0,out1, a,b,shiftop,s) \ - { \ - dct_wadd(sum, a, b); \ - dct_wsub(dif, a, b); \ - out0 = vcombine_s16(shiftop(sum_l, s), shiftop(sum_h, s)); \ - out1 = vcombine_s16(shiftop(dif_l, s), shiftop(dif_h, s)); \ - } - -#define dct_pass(shiftop, shift) \ - { \ - /* even part */ \ - int16x8_t sum26 = vaddq_s16(row2, row6); \ - dct_long_mul(p1e, sum26, rot0_0); \ - dct_long_mac(t2e, p1e, row6, rot0_1); \ - dct_long_mac(t3e, p1e, row2, rot0_2); \ - int16x8_t sum04 = vaddq_s16(row0, row4); \ - int16x8_t dif04 = vsubq_s16(row0, row4); \ - dct_widen(t0e, sum04); \ - dct_widen(t1e, dif04); \ - dct_wadd(x0, t0e, t3e); \ - dct_wsub(x3, t0e, t3e); \ - dct_wadd(x1, t1e, t2e); \ - dct_wsub(x2, t1e, t2e); \ - /* odd part */ \ - int16x8_t sum15 = vaddq_s16(row1, row5); \ - int16x8_t sum17 = vaddq_s16(row1, row7); \ - int16x8_t sum35 = vaddq_s16(row3, row5); \ - int16x8_t sum37 = vaddq_s16(row3, row7); \ - int16x8_t sumodd = vaddq_s16(sum17, sum35); \ - dct_long_mul(p5o, sumodd, rot1_0); \ - dct_long_mac(p1o, p5o, sum17, rot1_1); \ - dct_long_mac(p2o, p5o, sum35, rot1_2); \ - dct_long_mul(p3o, sum37, rot2_0); \ - dct_long_mul(p4o, sum15, rot2_1); \ - dct_wadd(sump13o, p1o, p3o); \ - dct_wadd(sump24o, p2o, p4o); \ - dct_wadd(sump23o, p2o, p3o); \ - dct_wadd(sump14o, p1o, p4o); \ - dct_long_mac(x4, sump13o, row7, rot3_0); \ - dct_long_mac(x5, sump24o, row5, rot3_1); \ - dct_long_mac(x6, sump23o, row3, rot3_2); \ - dct_long_mac(x7, sump14o, row1, rot3_3); \ - dct_bfly32o(row0,row7, x0,x7,shiftop,shift); \ - dct_bfly32o(row1,row6, x1,x6,shiftop,shift); \ - dct_bfly32o(row2,row5, x2,x5,shiftop,shift); \ - dct_bfly32o(row3,row4, x3,x4,shiftop,shift); \ - } - - // load - row0 = vld1q_s16(data + 0*8); - row1 = vld1q_s16(data + 1*8); - row2 = vld1q_s16(data + 2*8); - row3 = vld1q_s16(data + 3*8); - row4 = vld1q_s16(data + 4*8); - row5 = vld1q_s16(data + 5*8); - row6 = vld1q_s16(data + 6*8); - row7 = vld1q_s16(data + 7*8); - - // add DC bias - row0 = vaddq_s16(row0, vsetq_lane_s16(1024, vdupq_n_s16(0), 0)); - - // column pass - dct_pass(vrshrn_n_s32, 10); - - // 16bit 8x8 transpose - { -// these three map to a single VTRN.16, VTRN.32, and VSWP, respectively. -// whether compilers actually get this is another story, sadly. -#define dct_trn16(x, y) { int16x8x2_t t = vtrnq_s16(x, y); x = t.val[0]; y = t.val[1]; } -#define dct_trn32(x, y) { int32x4x2_t t = vtrnq_s32(vreinterpretq_s32_s16(x), vreinterpretq_s32_s16(y)); x = vreinterpretq_s16_s32(t.val[0]); y = vreinterpretq_s16_s32(t.val[1]); } -#define dct_trn64(x, y) { int16x8_t x0 = x; int16x8_t y0 = y; x = vcombine_s16(vget_low_s16(x0), vget_low_s16(y0)); y = vcombine_s16(vget_high_s16(x0), vget_high_s16(y0)); } - - // pass 1 - dct_trn16(row0, row1); // a0b0a2b2a4b4a6b6 - dct_trn16(row2, row3); - dct_trn16(row4, row5); - dct_trn16(row6, row7); - - // pass 2 - dct_trn32(row0, row2); // a0b0c0d0a4b4c4d4 - dct_trn32(row1, row3); - dct_trn32(row4, row6); - dct_trn32(row5, row7); - - // pass 3 - dct_trn64(row0, row4); // a0b0c0d0e0f0g0h0 - dct_trn64(row1, row5); - dct_trn64(row2, row6); - dct_trn64(row3, row7); - -#undef dct_trn16 -#undef dct_trn32 -#undef dct_trn64 - } - - // row pass - // vrshrn_n_s32 only supports shifts up to 16, we need - // 17. so do a non-rounding shift of 16 first then follow - // up with a rounding shift by 1. - dct_pass(vshrn_n_s32, 16); - - { - // pack and round - uint8x8_t p0 = vqrshrun_n_s16(row0, 1); - uint8x8_t p1 = vqrshrun_n_s16(row1, 1); - uint8x8_t p2 = vqrshrun_n_s16(row2, 1); - uint8x8_t p3 = vqrshrun_n_s16(row3, 1); - uint8x8_t p4 = vqrshrun_n_s16(row4, 1); - uint8x8_t p5 = vqrshrun_n_s16(row5, 1); - uint8x8_t p6 = vqrshrun_n_s16(row6, 1); - uint8x8_t p7 = vqrshrun_n_s16(row7, 1); - - // again, these can translate into one instruction, but often don't. -#define dct_trn8_8(x, y) { uint8x8x2_t t = vtrn_u8(x, y); x = t.val[0]; y = t.val[1]; } -#define dct_trn8_16(x, y) { uint16x4x2_t t = vtrn_u16(vreinterpret_u16_u8(x), vreinterpret_u16_u8(y)); x = vreinterpret_u8_u16(t.val[0]); y = vreinterpret_u8_u16(t.val[1]); } -#define dct_trn8_32(x, y) { uint32x2x2_t t = vtrn_u32(vreinterpret_u32_u8(x), vreinterpret_u32_u8(y)); x = vreinterpret_u8_u32(t.val[0]); y = vreinterpret_u8_u32(t.val[1]); } - - // sadly can't use interleaved stores here since we only write - // 8 bytes to each scan line! - - // 8x8 8-bit transpose pass 1 - dct_trn8_8(p0, p1); - dct_trn8_8(p2, p3); - dct_trn8_8(p4, p5); - dct_trn8_8(p6, p7); - - // pass 2 - dct_trn8_16(p0, p2); - dct_trn8_16(p1, p3); - dct_trn8_16(p4, p6); - dct_trn8_16(p5, p7); - - // pass 3 - dct_trn8_32(p0, p4); - dct_trn8_32(p1, p5); - dct_trn8_32(p2, p6); - dct_trn8_32(p3, p7); - - // store - vst1_u8(out, p0); out += out_stride; - vst1_u8(out, p1); out += out_stride; - vst1_u8(out, p2); out += out_stride; - vst1_u8(out, p3); out += out_stride; - vst1_u8(out, p4); out += out_stride; - vst1_u8(out, p5); out += out_stride; - vst1_u8(out, p6); out += out_stride; - vst1_u8(out, p7); - -#undef dct_trn8_8 -#undef dct_trn8_16 -#undef dct_trn8_32 - } - -#undef dct_long_mul -#undef dct_long_mac -#undef dct_widen -#undef dct_wadd -#undef dct_wsub -#undef dct_bfly32o -#undef dct_pass -} - -#endif // STBI_NEON - -#define STBI__MARKER_none 0xff -// if there's a pending marker from the entropy stream, return that -// otherwise, fetch from the stream and get a marker. if there's no -// marker, return 0xff, which is never a valid marker value -static stbi_uc stbi__get_marker(stbi__jpeg *j) -{ - stbi_uc x; - if (j->marker != STBI__MARKER_none) { x = j->marker; j->marker = STBI__MARKER_none; return x; } - x = stbi__get8(j->s); - if (x != 0xff) return STBI__MARKER_none; - while (x == 0xff) - x = stbi__get8(j->s); // consume repeated 0xff fill bytes - return x; -} - -// in each scan, we'll have scan_n components, and the order -// of the components is specified by order[] -#define STBI__RESTART(x) ((x) >= 0xd0 && (x) <= 0xd7) - -// after a restart interval, stbi__jpeg_reset the entropy decoder and -// the dc prediction -static void stbi__jpeg_reset(stbi__jpeg *j) -{ - j->code_bits = 0; - j->code_buffer = 0; - j->nomore = 0; - j->img_comp[0].dc_pred = j->img_comp[1].dc_pred = j->img_comp[2].dc_pred = j->img_comp[3].dc_pred = 0; - j->marker = STBI__MARKER_none; - j->todo = j->restart_interval ? j->restart_interval : 0x7fffffff; - j->eob_run = 0; - // no more than 1<<31 MCUs if no restart_interal? that's plenty safe, - // since we don't even allow 1<<30 pixels -} - -static int stbi__parse_entropy_coded_data(stbi__jpeg *z) -{ - stbi__jpeg_reset(z); - if (!z->progressive) { - if (z->scan_n == 1) { - int i,j; - STBI_SIMD_ALIGN(short, data[64]); - int n = z->order[0]; - // non-interleaved data, we just need to process one block at a time, - // in trivial scanline order - // number of blocks to do just depends on how many actual "pixels" this - // component has, independent of interleaved MCU blocking and such - int w = (z->img_comp[n].x+7) >> 3; - int h = (z->img_comp[n].y+7) >> 3; - for (j=0; j < h; ++j) { - for (i=0; i < w; ++i) { - int ha = z->img_comp[n].ha; - if (!stbi__jpeg_decode_block(z, data, z->huff_dc+z->img_comp[n].hd, z->huff_ac+ha, z->fast_ac[ha], n, z->dequant[z->img_comp[n].tq])) return 0; - z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*j*8+i*8, z->img_comp[n].w2, data); - // every data block is an MCU, so countdown the restart interval - if (--z->todo <= 0) { - if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); - // if it's NOT a restart, then just bail, so we get corrupt data - // rather than no data - if (!STBI__RESTART(z->marker)) return 1; - stbi__jpeg_reset(z); - } - } - } - return 1; - } else { // interleaved - int i,j,k,x,y; - STBI_SIMD_ALIGN(short, data[64]); - for (j=0; j < z->img_mcu_y; ++j) { - for (i=0; i < z->img_mcu_x; ++i) { - // scan an interleaved mcu... process scan_n components in order - for (k=0; k < z->scan_n; ++k) { - int n = z->order[k]; - // scan out an mcu's worth of this component; that's just determined - // by the basic H and V specified for the component - for (y=0; y < z->img_comp[n].v; ++y) { - for (x=0; x < z->img_comp[n].h; ++x) { - int x2 = (i*z->img_comp[n].h + x)*8; - int y2 = (j*z->img_comp[n].v + y)*8; - int ha = z->img_comp[n].ha; - if (!stbi__jpeg_decode_block(z, data, z->huff_dc+z->img_comp[n].hd, z->huff_ac+ha, z->fast_ac[ha], n, z->dequant[z->img_comp[n].tq])) return 0; - z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*y2+x2, z->img_comp[n].w2, data); - } - } - } - // after all interleaved components, that's an interleaved MCU, - // so now count down the restart interval - if (--z->todo <= 0) { - if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); - if (!STBI__RESTART(z->marker)) return 1; - stbi__jpeg_reset(z); - } - } - } - return 1; - } - } else { - if (z->scan_n == 1) { - int i,j; - int n = z->order[0]; - // non-interleaved data, we just need to process one block at a time, - // in trivial scanline order - // number of blocks to do just depends on how many actual "pixels" this - // component has, independent of interleaved MCU blocking and such - int w = (z->img_comp[n].x+7) >> 3; - int h = (z->img_comp[n].y+7) >> 3; - for (j=0; j < h; ++j) { - for (i=0; i < w; ++i) { - short *data = z->img_comp[n].coeff + 64 * (i + j * z->img_comp[n].coeff_w); - if (z->spec_start == 0) { - if (!stbi__jpeg_decode_block_prog_dc(z, data, &z->huff_dc[z->img_comp[n].hd], n)) - return 0; - } else { - int ha = z->img_comp[n].ha; - if (!stbi__jpeg_decode_block_prog_ac(z, data, &z->huff_ac[ha], z->fast_ac[ha])) - return 0; - } - // every data block is an MCU, so countdown the restart interval - if (--z->todo <= 0) { - if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); - if (!STBI__RESTART(z->marker)) return 1; - stbi__jpeg_reset(z); - } - } - } - return 1; - } else { // interleaved - int i,j,k,x,y; - for (j=0; j < z->img_mcu_y; ++j) { - for (i=0; i < z->img_mcu_x; ++i) { - // scan an interleaved mcu... process scan_n components in order - for (k=0; k < z->scan_n; ++k) { - int n = z->order[k]; - // scan out an mcu's worth of this component; that's just determined - // by the basic H and V specified for the component - for (y=0; y < z->img_comp[n].v; ++y) { - for (x=0; x < z->img_comp[n].h; ++x) { - int x2 = (i*z->img_comp[n].h + x); - int y2 = (j*z->img_comp[n].v + y); - short *data = z->img_comp[n].coeff + 64 * (x2 + y2 * z->img_comp[n].coeff_w); - if (!stbi__jpeg_decode_block_prog_dc(z, data, &z->huff_dc[z->img_comp[n].hd], n)) - return 0; - } - } - } - // after all interleaved components, that's an interleaved MCU, - // so now count down the restart interval - if (--z->todo <= 0) { - if (z->code_bits < 24) stbi__grow_buffer_unsafe(z); - if (!STBI__RESTART(z->marker)) return 1; - stbi__jpeg_reset(z); - } - } - } - return 1; - } - } -} - -static void stbi__jpeg_dequantize(short *data, stbi__uint16 *dequant) -{ - int i; - for (i=0; i < 64; ++i) - data[i] *= dequant[i]; -} - -static void stbi__jpeg_finish(stbi__jpeg *z) -{ - if (z->progressive) { - // dequantize and idct the data - int i,j,n; - for (n=0; n < z->s->img_n; ++n) { - int w = (z->img_comp[n].x+7) >> 3; - int h = (z->img_comp[n].y+7) >> 3; - for (j=0; j < h; ++j) { - for (i=0; i < w; ++i) { - short *data = z->img_comp[n].coeff + 64 * (i + j * z->img_comp[n].coeff_w); - stbi__jpeg_dequantize(data, z->dequant[z->img_comp[n].tq]); - z->idct_block_kernel(z->img_comp[n].data+z->img_comp[n].w2*j*8+i*8, z->img_comp[n].w2, data); - } - } - } - } -} - -static int stbi__process_marker(stbi__jpeg *z, int m) -{ - int L; - switch (m) { - case STBI__MARKER_none: // no marker found - return stbi__err("expected marker","Corrupt JPEG"); - - case 0xDD: // DRI - specify restart interval - if (stbi__get16be(z->s) != 4) return stbi__err("bad DRI len","Corrupt JPEG"); - z->restart_interval = stbi__get16be(z->s); - return 1; - - case 0xDB: // DQT - define quantization table - L = stbi__get16be(z->s)-2; - while (L > 0) { - int q = stbi__get8(z->s); - int p = q >> 4, sixteen = (p != 0); - int t = q & 15,i; - if (p != 0 && p != 1) return stbi__err("bad DQT type","Corrupt JPEG"); - if (t > 3) return stbi__err("bad DQT table","Corrupt JPEG"); - - for (i=0; i < 64; ++i) - z->dequant[t][stbi__jpeg_dezigzag[i]] = (stbi__uint16)(sixteen ? stbi__get16be(z->s) : stbi__get8(z->s)); - L -= (sixteen ? 129 : 65); - } - return L==0; - - case 0xC4: // DHT - define huffman table - L = stbi__get16be(z->s)-2; - while (L > 0) { - stbi_uc *v; - int sizes[16],i,n=0; - int q = stbi__get8(z->s); - int tc = q >> 4; - int th = q & 15; - if (tc > 1 || th > 3) return stbi__err("bad DHT header","Corrupt JPEG"); - for (i=0; i < 16; ++i) { - sizes[i] = stbi__get8(z->s); - n += sizes[i]; - } - L -= 17; - if (tc == 0) { - if (!stbi__build_huffman(z->huff_dc+th, sizes)) return 0; - v = z->huff_dc[th].values; - } else { - if (!stbi__build_huffman(z->huff_ac+th, sizes)) return 0; - v = z->huff_ac[th].values; - } - for (i=0; i < n; ++i) - v[i] = stbi__get8(z->s); - if (tc != 0) - stbi__build_fast_ac(z->fast_ac[th], z->huff_ac + th); - L -= n; - } - return L==0; - } - - // check for comment block or APP blocks - if ((m >= 0xE0 && m <= 0xEF) || m == 0xFE) { - L = stbi__get16be(z->s); - if (L < 2) { - if (m == 0xFE) - return stbi__err("bad COM len","Corrupt JPEG"); - else - return stbi__err("bad APP len","Corrupt JPEG"); - } - L -= 2; - - if (m == 0xE0 && L >= 5) { // JFIF APP0 segment - static const unsigned char tag[5] = {'J','F','I','F','\0'}; - int ok = 1; - int i; - for (i=0; i < 5; ++i) - if (stbi__get8(z->s) != tag[i]) - ok = 0; - L -= 5; - if (ok) - z->jfif = 1; - } else if (m == 0xEE && L >= 12) { // Adobe APP14 segment - static const unsigned char tag[6] = {'A','d','o','b','e','\0'}; - int ok = 1; - int i; - for (i=0; i < 6; ++i) - if (stbi__get8(z->s) != tag[i]) - ok = 0; - L -= 6; - if (ok) { - stbi__get8(z->s); // version - stbi__get16be(z->s); // flags0 - stbi__get16be(z->s); // flags1 - z->app14_color_transform = stbi__get8(z->s); // color transform - L -= 6; - } - } - - stbi__skip(z->s, L); - return 1; - } - - return stbi__err("unknown marker","Corrupt JPEG"); -} - -// after we see SOS -static int stbi__process_scan_header(stbi__jpeg *z) -{ - int i; - int Ls = stbi__get16be(z->s); - z->scan_n = stbi__get8(z->s); - if (z->scan_n < 1 || z->scan_n > 4 || z->scan_n > (int) z->s->img_n) return stbi__err("bad SOS component count","Corrupt JPEG"); - if (Ls != 6+2*z->scan_n) return stbi__err("bad SOS len","Corrupt JPEG"); - for (i=0; i < z->scan_n; ++i) { - int id = stbi__get8(z->s), which; - int q = stbi__get8(z->s); - for (which = 0; which < z->s->img_n; ++which) - if (z->img_comp[which].id == id) - break; - if (which == z->s->img_n) return 0; // no match - z->img_comp[which].hd = q >> 4; if (z->img_comp[which].hd > 3) return stbi__err("bad DC huff","Corrupt JPEG"); - z->img_comp[which].ha = q & 15; if (z->img_comp[which].ha > 3) return stbi__err("bad AC huff","Corrupt JPEG"); - z->order[i] = which; - } - - { - int aa; - z->spec_start = stbi__get8(z->s); - z->spec_end = stbi__get8(z->s); // should be 63, but might be 0 - aa = stbi__get8(z->s); - z->succ_high = (aa >> 4); - z->succ_low = (aa & 15); - if (z->progressive) { - if (z->spec_start > 63 || z->spec_end > 63 || z->spec_start > z->spec_end || z->succ_high > 13 || z->succ_low > 13) - return stbi__err("bad SOS", "Corrupt JPEG"); - } else { - if (z->spec_start != 0) return stbi__err("bad SOS","Corrupt JPEG"); - if (z->succ_high != 0 || z->succ_low != 0) return stbi__err("bad SOS","Corrupt JPEG"); - z->spec_end = 63; - } - } - - return 1; -} - -static int stbi__free_jpeg_components(stbi__jpeg *z, int ncomp, int why) -{ - int i; - for (i=0; i < ncomp; ++i) { - if (z->img_comp[i].raw_data) { - STBI_FREE(z->img_comp[i].raw_data); - z->img_comp[i].raw_data = NULL; - z->img_comp[i].data = NULL; - } - if (z->img_comp[i].raw_coeff) { - STBI_FREE(z->img_comp[i].raw_coeff); - z->img_comp[i].raw_coeff = 0; - z->img_comp[i].coeff = 0; - } - if (z->img_comp[i].linebuf) { - STBI_FREE(z->img_comp[i].linebuf); - z->img_comp[i].linebuf = NULL; - } - } - return why; -} - -static int stbi__process_frame_header(stbi__jpeg *z, int scan) -{ - stbi__context *s = z->s; - int Lf,p,i,q, h_max=1,v_max=1,c; - Lf = stbi__get16be(s); if (Lf < 11) return stbi__err("bad SOF len","Corrupt JPEG"); // JPEG - p = stbi__get8(s); if (p != 8) return stbi__err("only 8-bit","JPEG format not supported: 8-bit only"); // JPEG baseline - s->img_y = stbi__get16be(s); if (s->img_y == 0) return stbi__err("no header height", "JPEG format not supported: delayed height"); // Legal, but we don't handle it--but neither does IJG - s->img_x = stbi__get16be(s); if (s->img_x == 0) return stbi__err("0 width","Corrupt JPEG"); // JPEG requires - c = stbi__get8(s); - if (c != 3 && c != 1 && c != 4) return stbi__err("bad component count","Corrupt JPEG"); - s->img_n = c; - for (i=0; i < c; ++i) { - z->img_comp[i].data = NULL; - z->img_comp[i].linebuf = NULL; - } - - if (Lf != 8+3*s->img_n) return stbi__err("bad SOF len","Corrupt JPEG"); - - z->rgb = 0; - for (i=0; i < s->img_n; ++i) { - static const unsigned char rgb[3] = { 'R', 'G', 'B' }; - z->img_comp[i].id = stbi__get8(s); - if (s->img_n == 3 && z->img_comp[i].id == rgb[i]) - ++z->rgb; - q = stbi__get8(s); - z->img_comp[i].h = (q >> 4); if (!z->img_comp[i].h || z->img_comp[i].h > 4) return stbi__err("bad H","Corrupt JPEG"); - z->img_comp[i].v = q & 15; if (!z->img_comp[i].v || z->img_comp[i].v > 4) return stbi__err("bad V","Corrupt JPEG"); - z->img_comp[i].tq = stbi__get8(s); if (z->img_comp[i].tq > 3) return stbi__err("bad TQ","Corrupt JPEG"); - } - - if (scan != STBI__SCAN_load) return 1; - - if (!stbi__mad3sizes_valid(s->img_x, s->img_y, s->img_n, 0)) return stbi__err("too large", "Image too large to decode"); - - for (i=0; i < s->img_n; ++i) { - if (z->img_comp[i].h > h_max) h_max = z->img_comp[i].h; - if (z->img_comp[i].v > v_max) v_max = z->img_comp[i].v; - } - - // compute interleaved mcu info - z->img_h_max = h_max; - z->img_v_max = v_max; - z->img_mcu_w = h_max * 8; - z->img_mcu_h = v_max * 8; - // these sizes can't be more than 17 bits - z->img_mcu_x = (s->img_x + z->img_mcu_w-1) / z->img_mcu_w; - z->img_mcu_y = (s->img_y + z->img_mcu_h-1) / z->img_mcu_h; - - for (i=0; i < s->img_n; ++i) { - // number of effective pixels (e.g. for non-interleaved MCU) - z->img_comp[i].x = (s->img_x * z->img_comp[i].h + h_max-1) / h_max; - z->img_comp[i].y = (s->img_y * z->img_comp[i].v + v_max-1) / v_max; - // to simplify generation, we'll allocate enough memory to decode - // the bogus oversized data from using interleaved MCUs and their - // big blocks (e.g. a 16x16 iMCU on an image of width 33); we won't - // discard the extra data until colorspace conversion - // - // img_mcu_x, img_mcu_y: <=17 bits; comp[i].h and .v are <=4 (checked earlier) - // so these muls can't overflow with 32-bit ints (which we require) - z->img_comp[i].w2 = z->img_mcu_x * z->img_comp[i].h * 8; - z->img_comp[i].h2 = z->img_mcu_y * z->img_comp[i].v * 8; - z->img_comp[i].coeff = 0; - z->img_comp[i].raw_coeff = 0; - z->img_comp[i].linebuf = NULL; - z->img_comp[i].raw_data = stbi__malloc_mad2(z->img_comp[i].w2, z->img_comp[i].h2, 15); - if (z->img_comp[i].raw_data == NULL) - return stbi__free_jpeg_components(z, i+1, stbi__err("outofmem", "Out of memory")); - // align blocks for idct using mmx/sse - z->img_comp[i].data = (stbi_uc*) (((size_t) z->img_comp[i].raw_data + 15) & ~15); - if (z->progressive) { - // w2, h2 are multiples of 8 (see above) - z->img_comp[i].coeff_w = z->img_comp[i].w2 / 8; - z->img_comp[i].coeff_h = z->img_comp[i].h2 / 8; - z->img_comp[i].raw_coeff = stbi__malloc_mad3(z->img_comp[i].w2, z->img_comp[i].h2, sizeof(short), 15); - if (z->img_comp[i].raw_coeff == NULL) - return stbi__free_jpeg_components(z, i+1, stbi__err("outofmem", "Out of memory")); - z->img_comp[i].coeff = (short*) (((size_t) z->img_comp[i].raw_coeff + 15) & ~15); - } - } - - return 1; -} - -// use comparisons since in some cases we handle more than one case (e.g. SOF) -#define stbi__DNL(x) ((x) == 0xdc) -#define stbi__SOI(x) ((x) == 0xd8) -#define stbi__EOI(x) ((x) == 0xd9) -#define stbi__SOF(x) ((x) == 0xc0 || (x) == 0xc1 || (x) == 0xc2) -#define stbi__SOS(x) ((x) == 0xda) - -#define stbi__SOF_progressive(x) ((x) == 0xc2) - -static int stbi__decode_jpeg_header(stbi__jpeg *z, int scan) -{ - int m; - z->jfif = 0; - z->app14_color_transform = -1; // valid values are 0,1,2 - z->marker = STBI__MARKER_none; // initialize cached marker to empty - m = stbi__get_marker(z); - if (!stbi__SOI(m)) return stbi__err("no SOI","Corrupt JPEG"); - if (scan == STBI__SCAN_type) return 1; - m = stbi__get_marker(z); - while (!stbi__SOF(m)) { - if (!stbi__process_marker(z,m)) return 0; - m = stbi__get_marker(z); - while (m == STBI__MARKER_none) { - // some files have extra padding after their blocks, so ok, we'll scan - if (stbi__at_eof(z->s)) return stbi__err("no SOF", "Corrupt JPEG"); - m = stbi__get_marker(z); - } - } - z->progressive = stbi__SOF_progressive(m); - if (!stbi__process_frame_header(z, scan)) return 0; - return 1; -} - -// decode image to YCbCr format -static int stbi__decode_jpeg_image(stbi__jpeg *j) -{ - int m; - for (m = 0; m < 4; m++) { - j->img_comp[m].raw_data = NULL; - j->img_comp[m].raw_coeff = NULL; - } - j->restart_interval = 0; - if (!stbi__decode_jpeg_header(j, STBI__SCAN_load)) return 0; - m = stbi__get_marker(j); - while (!stbi__EOI(m)) { - if (stbi__SOS(m)) { - if (!stbi__process_scan_header(j)) return 0; - if (!stbi__parse_entropy_coded_data(j)) return 0; - if (j->marker == STBI__MARKER_none ) { - // handle 0s at the end of image data from IP Kamera 9060 - while (!stbi__at_eof(j->s)) { - int x = stbi__get8(j->s); - if (x == 255) { - j->marker = stbi__get8(j->s); - break; - } - } - // if we reach eof without hitting a marker, stbi__get_marker() below will fail and we'll eventually return 0 - } - } else if (stbi__DNL(m)) { - int Ld = stbi__get16be(j->s); - stbi__uint32 NL = stbi__get16be(j->s); - if (Ld != 4) return stbi__err("bad DNL len", "Corrupt JPEG"); - if (NL != j->s->img_y) return stbi__err("bad DNL height", "Corrupt JPEG"); - } else { - if (!stbi__process_marker(j, m)) return 0; - } - m = stbi__get_marker(j); - } - if (j->progressive) - stbi__jpeg_finish(j); - return 1; -} - -// static jfif-centered resampling (across block boundaries) - -typedef stbi_uc *(*resample_row_func)(stbi_uc *out, stbi_uc *in0, stbi_uc *in1, - int w, int hs); - -#define stbi__div4(x) ((stbi_uc) ((x) >> 2)) - -static stbi_uc *resample_row_1(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) -{ - STBI_NOTUSED(out); - STBI_NOTUSED(in_far); - STBI_NOTUSED(w); - STBI_NOTUSED(hs); - return in_near; -} - -static stbi_uc* stbi__resample_row_v_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) -{ - // need to generate two samples vertically for every one in input - int i; - STBI_NOTUSED(hs); - for (i=0; i < w; ++i) - out[i] = stbi__div4(3*in_near[i] + in_far[i] + 2); - return out; -} - -static stbi_uc* stbi__resample_row_h_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) -{ - // need to generate two samples horizontally for every one in input - int i; - stbi_uc *input = in_near; - - if (w == 1) { - // if only one sample, can't do any interpolation - out[0] = out[1] = input[0]; - return out; - } - - out[0] = input[0]; - out[1] = stbi__div4(input[0]*3 + input[1] + 2); - for (i=1; i < w-1; ++i) { - int n = 3*input[i]+2; - out[i*2+0] = stbi__div4(n+input[i-1]); - out[i*2+1] = stbi__div4(n+input[i+1]); - } - out[i*2+0] = stbi__div4(input[w-2]*3 + input[w-1] + 2); - out[i*2+1] = input[w-1]; - - STBI_NOTUSED(in_far); - STBI_NOTUSED(hs); - - return out; -} - -#define stbi__div16(x) ((stbi_uc) ((x) >> 4)) - -static stbi_uc *stbi__resample_row_hv_2(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) -{ - // need to generate 2x2 samples for every one in input - int i,t0,t1; - if (w == 1) { - out[0] = out[1] = stbi__div4(3*in_near[0] + in_far[0] + 2); - return out; - } - - t1 = 3*in_near[0] + in_far[0]; - out[0] = stbi__div4(t1+2); - for (i=1; i < w; ++i) { - t0 = t1; - t1 = 3*in_near[i]+in_far[i]; - out[i*2-1] = stbi__div16(3*t0 + t1 + 8); - out[i*2 ] = stbi__div16(3*t1 + t0 + 8); - } - out[w*2-1] = stbi__div4(t1+2); - - STBI_NOTUSED(hs); - - return out; -} - -#if defined(STBI_SSE2) || defined(STBI_NEON) -static stbi_uc *stbi__resample_row_hv_2_simd(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) -{ - // need to generate 2x2 samples for every one in input - int i=0,t0,t1; - - if (w == 1) { - out[0] = out[1] = stbi__div4(3*in_near[0] + in_far[0] + 2); - return out; - } - - t1 = 3*in_near[0] + in_far[0]; - // process groups of 8 pixels for as long as we can. - // note we can't handle the last pixel in a row in this loop - // because we need to handle the filter boundary conditions. - for (; i < ((w-1) & ~7); i += 8) { -#if defined(STBI_SSE2) - // load and perform the vertical filtering pass - // this uses 3*x + y = 4*x + (y - x) - __m128i zero = _mm_setzero_si128(); - __m128i farb = _mm_loadl_epi64((__m128i *) (in_far + i)); - __m128i nearb = _mm_loadl_epi64((__m128i *) (in_near + i)); - __m128i farw = _mm_unpacklo_epi8(farb, zero); - __m128i nearw = _mm_unpacklo_epi8(nearb, zero); - __m128i diff = _mm_sub_epi16(farw, nearw); - __m128i nears = _mm_slli_epi16(nearw, 2); - __m128i curr = _mm_add_epi16(nears, diff); // current row - - // horizontal filter works the same based on shifted vers of current - // row. "prev" is current row shifted right by 1 pixel; we need to - // insert the previous pixel value (from t1). - // "next" is current row shifted left by 1 pixel, with first pixel - // of next block of 8 pixels added in. - __m128i prv0 = _mm_slli_si128(curr, 2); - __m128i nxt0 = _mm_srli_si128(curr, 2); - __m128i prev = _mm_insert_epi16(prv0, t1, 0); - __m128i next = _mm_insert_epi16(nxt0, 3*in_near[i+8] + in_far[i+8], 7); - - // horizontal filter, polyphase implementation since it's convenient: - // even pixels = 3*cur + prev = cur*4 + (prev - cur) - // odd pixels = 3*cur + next = cur*4 + (next - cur) - // note the shared term. - __m128i bias = _mm_set1_epi16(8); - __m128i curs = _mm_slli_epi16(curr, 2); - __m128i prvd = _mm_sub_epi16(prev, curr); - __m128i nxtd = _mm_sub_epi16(next, curr); - __m128i curb = _mm_add_epi16(curs, bias); - __m128i even = _mm_add_epi16(prvd, curb); - __m128i odd = _mm_add_epi16(nxtd, curb); - - // interleave even and odd pixels, then undo scaling. - __m128i int0 = _mm_unpacklo_epi16(even, odd); - __m128i int1 = _mm_unpackhi_epi16(even, odd); - __m128i de0 = _mm_srli_epi16(int0, 4); - __m128i de1 = _mm_srli_epi16(int1, 4); - - // pack and write output - __m128i outv = _mm_packus_epi16(de0, de1); - _mm_storeu_si128((__m128i *) (out + i*2), outv); -#elif defined(STBI_NEON) - // load and perform the vertical filtering pass - // this uses 3*x + y = 4*x + (y - x) - uint8x8_t farb = vld1_u8(in_far + i); - uint8x8_t nearb = vld1_u8(in_near + i); - int16x8_t diff = vreinterpretq_s16_u16(vsubl_u8(farb, nearb)); - int16x8_t nears = vreinterpretq_s16_u16(vshll_n_u8(nearb, 2)); - int16x8_t curr = vaddq_s16(nears, diff); // current row - - // horizontal filter works the same based on shifted vers of current - // row. "prev" is current row shifted right by 1 pixel; we need to - // insert the previous pixel value (from t1). - // "next" is current row shifted left by 1 pixel, with first pixel - // of next block of 8 pixels added in. - int16x8_t prv0 = vextq_s16(curr, curr, 7); - int16x8_t nxt0 = vextq_s16(curr, curr, 1); - int16x8_t prev = vsetq_lane_s16(t1, prv0, 0); - int16x8_t next = vsetq_lane_s16(3*in_near[i+8] + in_far[i+8], nxt0, 7); - - // horizontal filter, polyphase implementation since it's convenient: - // even pixels = 3*cur + prev = cur*4 + (prev - cur) - // odd pixels = 3*cur + next = cur*4 + (next - cur) - // note the shared term. - int16x8_t curs = vshlq_n_s16(curr, 2); - int16x8_t prvd = vsubq_s16(prev, curr); - int16x8_t nxtd = vsubq_s16(next, curr); - int16x8_t even = vaddq_s16(curs, prvd); - int16x8_t odd = vaddq_s16(curs, nxtd); - - // undo scaling and round, then store with even/odd phases interleaved - uint8x8x2_t o; - o.val[0] = vqrshrun_n_s16(even, 4); - o.val[1] = vqrshrun_n_s16(odd, 4); - vst2_u8(out + i*2, o); -#endif - - // "previous" value for next iter - t1 = 3*in_near[i+7] + in_far[i+7]; - } - - t0 = t1; - t1 = 3*in_near[i] + in_far[i]; - out[i*2] = stbi__div16(3*t1 + t0 + 8); - - for (++i; i < w; ++i) { - t0 = t1; - t1 = 3*in_near[i]+in_far[i]; - out[i*2-1] = stbi__div16(3*t0 + t1 + 8); - out[i*2 ] = stbi__div16(3*t1 + t0 + 8); - } - out[w*2-1] = stbi__div4(t1+2); - - STBI_NOTUSED(hs); - - return out; -} -#endif - -static stbi_uc *stbi__resample_row_generic(stbi_uc *out, stbi_uc *in_near, stbi_uc *in_far, int w, int hs) -{ - // resample with nearest-neighbor - int i,j; - STBI_NOTUSED(in_far); - for (i=0; i < w; ++i) - for (j=0; j < hs; ++j) - out[i*hs+j] = in_near[i]; - return out; -} - -// this is a reduced-precision calculation of YCbCr-to-RGB introduced -// to make sure the code produces the same results in both SIMD and scalar -#define stbi__float2fixed(x) (((int) ((x) * 4096.0f + 0.5f)) << 8) -static void stbi__YCbCr_to_RGB_row(stbi_uc *out, const stbi_uc *y, const stbi_uc *pcb, const stbi_uc *pcr, int count, int step) -{ - int i; - for (i=0; i < count; ++i) { - int y_fixed = (y[i] << 20) + (1<<19); // rounding - int r,g,b; - int cr = pcr[i] - 128; - int cb = pcb[i] - 128; - r = y_fixed + cr* stbi__float2fixed(1.40200f); - g = y_fixed + (cr*-stbi__float2fixed(0.71414f)) + ((cb*-stbi__float2fixed(0.34414f)) & 0xffff0000); - b = y_fixed + cb* stbi__float2fixed(1.77200f); - r >>= 20; - g >>= 20; - b >>= 20; - if ((unsigned) r > 255) { if (r < 0) r = 0; else r = 255; } - if ((unsigned) g > 255) { if (g < 0) g = 0; else g = 255; } - if ((unsigned) b > 255) { if (b < 0) b = 0; else b = 255; } - out[0] = (stbi_uc)r; - out[1] = (stbi_uc)g; - out[2] = (stbi_uc)b; - out[3] = 255; - out += step; - } -} - -#if defined(STBI_SSE2) || defined(STBI_NEON) -static void stbi__YCbCr_to_RGB_simd(stbi_uc *out, stbi_uc const *y, stbi_uc const *pcb, stbi_uc const *pcr, int count, int step) -{ - int i = 0; - -#ifdef STBI_SSE2 - // step == 3 is pretty ugly on the final interleave, and i'm not convinced - // it's useful in practice (you wouldn't use it for textures, for example). - // so just accelerate step == 4 case. - if (step == 4) { - // this is a fairly straightforward implementation and not super-optimized. - __m128i signflip = _mm_set1_epi8(-0x80); - __m128i cr_const0 = _mm_set1_epi16( (short) ( 1.40200f*4096.0f+0.5f)); - __m128i cr_const1 = _mm_set1_epi16( - (short) ( 0.71414f*4096.0f+0.5f)); - __m128i cb_const0 = _mm_set1_epi16( - (short) ( 0.34414f*4096.0f+0.5f)); - __m128i cb_const1 = _mm_set1_epi16( (short) ( 1.77200f*4096.0f+0.5f)); - __m128i y_bias = _mm_set1_epi8((char) (unsigned char) 128); - __m128i xw = _mm_set1_epi16(255); // alpha channel - - for (; i+7 < count; i += 8) { - // load - __m128i y_bytes = _mm_loadl_epi64((__m128i *) (y+i)); - __m128i cr_bytes = _mm_loadl_epi64((__m128i *) (pcr+i)); - __m128i cb_bytes = _mm_loadl_epi64((__m128i *) (pcb+i)); - __m128i cr_biased = _mm_xor_si128(cr_bytes, signflip); // -128 - __m128i cb_biased = _mm_xor_si128(cb_bytes, signflip); // -128 - - // unpack to short (and left-shift cr, cb by 8) - __m128i yw = _mm_unpacklo_epi8(y_bias, y_bytes); - __m128i crw = _mm_unpacklo_epi8(_mm_setzero_si128(), cr_biased); - __m128i cbw = _mm_unpacklo_epi8(_mm_setzero_si128(), cb_biased); - - // color transform - __m128i yws = _mm_srli_epi16(yw, 4); - __m128i cr0 = _mm_mulhi_epi16(cr_const0, crw); - __m128i cb0 = _mm_mulhi_epi16(cb_const0, cbw); - __m128i cb1 = _mm_mulhi_epi16(cbw, cb_const1); - __m128i cr1 = _mm_mulhi_epi16(crw, cr_const1); - __m128i rws = _mm_add_epi16(cr0, yws); - __m128i gwt = _mm_add_epi16(cb0, yws); - __m128i bws = _mm_add_epi16(yws, cb1); - __m128i gws = _mm_add_epi16(gwt, cr1); - - // descale - __m128i rw = _mm_srai_epi16(rws, 4); - __m128i bw = _mm_srai_epi16(bws, 4); - __m128i gw = _mm_srai_epi16(gws, 4); - - // back to byte, set up for transpose - __m128i brb = _mm_packus_epi16(rw, bw); - __m128i gxb = _mm_packus_epi16(gw, xw); - - // transpose to interleave channels - __m128i t0 = _mm_unpacklo_epi8(brb, gxb); - __m128i t1 = _mm_unpackhi_epi8(brb, gxb); - __m128i o0 = _mm_unpacklo_epi16(t0, t1); - __m128i o1 = _mm_unpackhi_epi16(t0, t1); - - // store - _mm_storeu_si128((__m128i *) (out + 0), o0); - _mm_storeu_si128((__m128i *) (out + 16), o1); - out += 32; - } - } -#endif - -#ifdef STBI_NEON - // in this version, step=3 support would be easy to add. but is there demand? - if (step == 4) { - // this is a fairly straightforward implementation and not super-optimized. - uint8x8_t signflip = vdup_n_u8(0x80); - int16x8_t cr_const0 = vdupq_n_s16( (short) ( 1.40200f*4096.0f+0.5f)); - int16x8_t cr_const1 = vdupq_n_s16( - (short) ( 0.71414f*4096.0f+0.5f)); - int16x8_t cb_const0 = vdupq_n_s16( - (short) ( 0.34414f*4096.0f+0.5f)); - int16x8_t cb_const1 = vdupq_n_s16( (short) ( 1.77200f*4096.0f+0.5f)); - - for (; i+7 < count; i += 8) { - // load - uint8x8_t y_bytes = vld1_u8(y + i); - uint8x8_t cr_bytes = vld1_u8(pcr + i); - uint8x8_t cb_bytes = vld1_u8(pcb + i); - int8x8_t cr_biased = vreinterpret_s8_u8(vsub_u8(cr_bytes, signflip)); - int8x8_t cb_biased = vreinterpret_s8_u8(vsub_u8(cb_bytes, signflip)); - - // expand to s16 - int16x8_t yws = vreinterpretq_s16_u16(vshll_n_u8(y_bytes, 4)); - int16x8_t crw = vshll_n_s8(cr_biased, 7); - int16x8_t cbw = vshll_n_s8(cb_biased, 7); - - // color transform - int16x8_t cr0 = vqdmulhq_s16(crw, cr_const0); - int16x8_t cb0 = vqdmulhq_s16(cbw, cb_const0); - int16x8_t cr1 = vqdmulhq_s16(crw, cr_const1); - int16x8_t cb1 = vqdmulhq_s16(cbw, cb_const1); - int16x8_t rws = vaddq_s16(yws, cr0); - int16x8_t gws = vaddq_s16(vaddq_s16(yws, cb0), cr1); - int16x8_t bws = vaddq_s16(yws, cb1); - - // undo scaling, round, convert to byte - uint8x8x4_t o; - o.val[0] = vqrshrun_n_s16(rws, 4); - o.val[1] = vqrshrun_n_s16(gws, 4); - o.val[2] = vqrshrun_n_s16(bws, 4); - o.val[3] = vdup_n_u8(255); - - // store, interleaving r/g/b/a - vst4_u8(out, o); - out += 8*4; - } - } -#endif - - for (; i < count; ++i) { - int y_fixed = (y[i] << 20) + (1<<19); // rounding - int r,g,b; - int cr = pcr[i] - 128; - int cb = pcb[i] - 128; - r = y_fixed + cr* stbi__float2fixed(1.40200f); - g = y_fixed + cr*-stbi__float2fixed(0.71414f) + ((cb*-stbi__float2fixed(0.34414f)) & 0xffff0000); - b = y_fixed + cb* stbi__float2fixed(1.77200f); - r >>= 20; - g >>= 20; - b >>= 20; - if ((unsigned) r > 255) { if (r < 0) r = 0; else r = 255; } - if ((unsigned) g > 255) { if (g < 0) g = 0; else g = 255; } - if ((unsigned) b > 255) { if (b < 0) b = 0; else b = 255; } - out[0] = (stbi_uc)r; - out[1] = (stbi_uc)g; - out[2] = (stbi_uc)b; - out[3] = 255; - out += step; - } -} -#endif - -// set up the kernels -static void stbi__setup_jpeg(stbi__jpeg *j) -{ - j->idct_block_kernel = stbi__idct_block; - j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_row; - j->resample_row_hv_2_kernel = stbi__resample_row_hv_2; - -#ifdef STBI_SSE2 - if (stbi__sse2_available()) { - j->idct_block_kernel = stbi__idct_simd; - j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_simd; - j->resample_row_hv_2_kernel = stbi__resample_row_hv_2_simd; - } -#endif - -#ifdef STBI_NEON - j->idct_block_kernel = stbi__idct_simd; - j->YCbCr_to_RGB_kernel = stbi__YCbCr_to_RGB_simd; - j->resample_row_hv_2_kernel = stbi__resample_row_hv_2_simd; -#endif -} - -// clean up the temporary component buffers -static void stbi__cleanup_jpeg(stbi__jpeg *j) -{ - stbi__free_jpeg_components(j, j->s->img_n, 0); -} - -typedef struct -{ - resample_row_func resample; - stbi_uc *line0,*line1; - int hs,vs; // expansion factor in each axis - int w_lores; // horizontal pixels pre-expansion - int ystep; // how far through vertical expansion we are - int ypos; // which pre-expansion row we're on -} stbi__resample; - -// fast 0..255 * 0..255 => 0..255 rounded multiplication -static stbi_uc stbi__blinn_8x8(stbi_uc x, stbi_uc y) -{ - unsigned int t = x*y + 128; - return (stbi_uc) ((t + (t >>8)) >> 8); -} - -static stbi_uc *load_jpeg_image(stbi__jpeg *z, int *out_x, int *out_y, int *comp, int req_comp) -{ - int n, decode_n, is_rgb; - z->s->img_n = 0; // make stbi__cleanup_jpeg safe - - // validate req_comp - if (req_comp < 0 || req_comp > 4) return stbi__errpuc("bad req_comp", "Internal error"); - - // load a jpeg image from whichever source, but leave in YCbCr format - if (!stbi__decode_jpeg_image(z)) { stbi__cleanup_jpeg(z); return NULL; } - - // determine actual number of components to generate - n = req_comp ? req_comp : z->s->img_n >= 3 ? 3 : 1; - - is_rgb = z->s->img_n == 3 && (z->rgb == 3 || (z->app14_color_transform == 0 && !z->jfif)); - - if (z->s->img_n == 3 && n < 3 && !is_rgb) - decode_n = 1; - else - decode_n = z->s->img_n; - - // resample and color-convert - { - int k; - unsigned int i,j; - stbi_uc *output; - stbi_uc *coutput[4]; - - stbi__resample res_comp[4]; - - for (k=0; k < decode_n; ++k) { - stbi__resample *r = &res_comp[k]; - - // allocate line buffer big enough for upsampling off the edges - // with upsample factor of 4 - z->img_comp[k].linebuf = (stbi_uc *) stbi__malloc(z->s->img_x + 3); - if (!z->img_comp[k].linebuf) { stbi__cleanup_jpeg(z); return stbi__errpuc("outofmem", "Out of memory"); } - - r->hs = z->img_h_max / z->img_comp[k].h; - r->vs = z->img_v_max / z->img_comp[k].v; - r->ystep = r->vs >> 1; - r->w_lores = (z->s->img_x + r->hs-1) / r->hs; - r->ypos = 0; - r->line0 = r->line1 = z->img_comp[k].data; - - if (r->hs == 1 && r->vs == 1) r->resample = resample_row_1; - else if (r->hs == 1 && r->vs == 2) r->resample = stbi__resample_row_v_2; - else if (r->hs == 2 && r->vs == 1) r->resample = stbi__resample_row_h_2; - else if (r->hs == 2 && r->vs == 2) r->resample = z->resample_row_hv_2_kernel; - else r->resample = stbi__resample_row_generic; - } - - // can't error after this so, this is safe - output = (stbi_uc *) stbi__malloc_mad3(n, z->s->img_x, z->s->img_y, 1); - if (!output) { stbi__cleanup_jpeg(z); return stbi__errpuc("outofmem", "Out of memory"); } - - // now go ahead and resample - for (j=0; j < z->s->img_y; ++j) { - stbi_uc *out = output + n * z->s->img_x * j; - for (k=0; k < decode_n; ++k) { - stbi__resample *r = &res_comp[k]; - int y_bot = r->ystep >= (r->vs >> 1); - coutput[k] = r->resample(z->img_comp[k].linebuf, - y_bot ? r->line1 : r->line0, - y_bot ? r->line0 : r->line1, - r->w_lores, r->hs); - if (++r->ystep >= r->vs) { - r->ystep = 0; - r->line0 = r->line1; - if (++r->ypos < z->img_comp[k].y) - r->line1 += z->img_comp[k].w2; - } - } - if (n >= 3) { - stbi_uc *y = coutput[0]; - if (z->s->img_n == 3) { - if (is_rgb) { - for (i=0; i < z->s->img_x; ++i) { - out[0] = y[i]; - out[1] = coutput[1][i]; - out[2] = coutput[2][i]; - out[3] = 255; - out += n; - } - } else { - z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n); - } - } else if (z->s->img_n == 4) { - if (z->app14_color_transform == 0) { // CMYK - for (i=0; i < z->s->img_x; ++i) { - stbi_uc m = coutput[3][i]; - out[0] = stbi__blinn_8x8(coutput[0][i], m); - out[1] = stbi__blinn_8x8(coutput[1][i], m); - out[2] = stbi__blinn_8x8(coutput[2][i], m); - out[3] = 255; - out += n; - } - } else if (z->app14_color_transform == 2) { // YCCK - z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n); - for (i=0; i < z->s->img_x; ++i) { - stbi_uc m = coutput[3][i]; - out[0] = stbi__blinn_8x8(255 - out[0], m); - out[1] = stbi__blinn_8x8(255 - out[1], m); - out[2] = stbi__blinn_8x8(255 - out[2], m); - out += n; - } - } else { // YCbCr + alpha? Ignore the fourth channel for now - z->YCbCr_to_RGB_kernel(out, y, coutput[1], coutput[2], z->s->img_x, n); - } - } else - for (i=0; i < z->s->img_x; ++i) { - out[0] = out[1] = out[2] = y[i]; - out[3] = 255; // not used if n==3 - out += n; - } - } else { - if (is_rgb) { - if (n == 1) - for (i=0; i < z->s->img_x; ++i) - *out++ = stbi__compute_y(coutput[0][i], coutput[1][i], coutput[2][i]); - else { - for (i=0; i < z->s->img_x; ++i, out += 2) { - out[0] = stbi__compute_y(coutput[0][i], coutput[1][i], coutput[2][i]); - out[1] = 255; - } - } - } else if (z->s->img_n == 4 && z->app14_color_transform == 0) { - for (i=0; i < z->s->img_x; ++i) { - stbi_uc m = coutput[3][i]; - stbi_uc r = stbi__blinn_8x8(coutput[0][i], m); - stbi_uc g = stbi__blinn_8x8(coutput[1][i], m); - stbi_uc b = stbi__blinn_8x8(coutput[2][i], m); - out[0] = stbi__compute_y(r, g, b); - out[1] = 255; - out += n; - } - } else if (z->s->img_n == 4 && z->app14_color_transform == 2) { - for (i=0; i < z->s->img_x; ++i) { - out[0] = stbi__blinn_8x8(255 - coutput[0][i], coutput[3][i]); - out[1] = 255; - out += n; - } - } else { - stbi_uc *y = coutput[0]; - if (n == 1) - for (i=0; i < z->s->img_x; ++i) out[i] = y[i]; - else - for (i=0; i < z->s->img_x; ++i) { *out++ = y[i]; *out++ = 255; } - } - } - } - stbi__cleanup_jpeg(z); - *out_x = z->s->img_x; - *out_y = z->s->img_y; - if (comp) *comp = z->s->img_n >= 3 ? 3 : 1; // report original components, not output - return output; - } -} - -static void *stbi__jpeg_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) -{ - unsigned char* result; - stbi__jpeg* j = (stbi__jpeg*) stbi__malloc(sizeof(stbi__jpeg)); - STBI_NOTUSED(ri); - j->s = s; - stbi__setup_jpeg(j); - result = load_jpeg_image(j, x,y,comp,req_comp); - STBI_FREE(j); - return result; -} - -static int stbi__jpeg_test(stbi__context *s) -{ - int r; - stbi__jpeg* j = (stbi__jpeg*)stbi__malloc(sizeof(stbi__jpeg)); - j->s = s; - stbi__setup_jpeg(j); - r = stbi__decode_jpeg_header(j, STBI__SCAN_type); - stbi__rewind(s); - STBI_FREE(j); - return r; -} - -static int stbi__jpeg_info_raw(stbi__jpeg *j, int *x, int *y, int *comp) -{ - if (!stbi__decode_jpeg_header(j, STBI__SCAN_header)) { - stbi__rewind( j->s ); - return 0; - } - if (x) *x = j->s->img_x; - if (y) *y = j->s->img_y; - if (comp) *comp = j->s->img_n >= 3 ? 3 : 1; - return 1; -} - -static int stbi__jpeg_info(stbi__context *s, int *x, int *y, int *comp) -{ - int result; - stbi__jpeg* j = (stbi__jpeg*) (stbi__malloc(sizeof(stbi__jpeg))); - j->s = s; - result = stbi__jpeg_info_raw(j, x, y, comp); - STBI_FREE(j); - return result; -} -#endif - -// public domain zlib decode v0.2 Sean Barrett 2006-11-18 -// simple implementation -// - all input must be provided in an upfront buffer -// - all output is written to a single output buffer (can malloc/realloc) -// performance -// - fast huffman - -#ifndef STBI_NO_ZLIB - -// fast-way is faster to check than jpeg huffman, but slow way is slower -#define STBI__ZFAST_BITS 9 // accelerate all cases in default tables -#define STBI__ZFAST_MASK ((1 << STBI__ZFAST_BITS) - 1) - -// zlib-style huffman encoding -// (jpegs packs from left, zlib from right, so can't share code) -typedef struct -{ - stbi__uint16 fast[1 << STBI__ZFAST_BITS]; - stbi__uint16 firstcode[16]; - int maxcode[17]; - stbi__uint16 firstsymbol[16]; - stbi_uc size[288]; - stbi__uint16 value[288]; -} stbi__zhuffman; - -stbi_inline static int stbi__bitreverse16(int n) -{ - n = ((n & 0xAAAA) >> 1) | ((n & 0x5555) << 1); - n = ((n & 0xCCCC) >> 2) | ((n & 0x3333) << 2); - n = ((n & 0xF0F0) >> 4) | ((n & 0x0F0F) << 4); - n = ((n & 0xFF00) >> 8) | ((n & 0x00FF) << 8); - return n; -} - -stbi_inline static int stbi__bit_reverse(int v, int bits) -{ - STBI_ASSERT(bits <= 16); - // to bit reverse n bits, reverse 16 and shift - // e.g. 11 bits, bit reverse and shift away 5 - return stbi__bitreverse16(v) >> (16-bits); -} - -static int stbi__zbuild_huffman(stbi__zhuffman *z, const stbi_uc *sizelist, int num) -{ - int i,k=0; - int code, next_code[16], sizes[17]; - - // DEFLATE spec for generating codes - memset(sizes, 0, sizeof(sizes)); - memset(z->fast, 0, sizeof(z->fast)); - for (i=0; i < num; ++i) - ++sizes[sizelist[i]]; - sizes[0] = 0; - for (i=1; i < 16; ++i) - if (sizes[i] > (1 << i)) - return stbi__err("bad sizes", "Corrupt PNG"); - code = 0; - for (i=1; i < 16; ++i) { - next_code[i] = code; - z->firstcode[i] = (stbi__uint16) code; - z->firstsymbol[i] = (stbi__uint16) k; - code = (code + sizes[i]); - if (sizes[i]) - if (code-1 >= (1 << i)) return stbi__err("bad codelengths","Corrupt PNG"); - z->maxcode[i] = code << (16-i); // preshift for inner loop - code <<= 1; - k += sizes[i]; - } - z->maxcode[16] = 0x10000; // sentinel - for (i=0; i < num; ++i) { - int s = sizelist[i]; - if (s) { - int c = next_code[s] - z->firstcode[s] + z->firstsymbol[s]; - stbi__uint16 fastv = (stbi__uint16) ((s << 9) | i); - z->size [c] = (stbi_uc ) s; - z->value[c] = (stbi__uint16) i; - if (s <= STBI__ZFAST_BITS) { - int j = stbi__bit_reverse(next_code[s],s); - while (j < (1 << STBI__ZFAST_BITS)) { - z->fast[j] = fastv; - j += (1 << s); - } - } - ++next_code[s]; - } - } - return 1; -} - -// zlib-from-memory implementation for PNG reading -// because PNG allows splitting the zlib stream arbitrarily, -// and it's annoying structurally to have PNG call ZLIB call PNG, -// we require PNG read all the IDATs and combine them into a single -// memory buffer - -typedef struct -{ - stbi_uc *zbuffer, *zbuffer_end; - int num_bits; - stbi__uint32 code_buffer; - - char *zout; - char *zout_start; - char *zout_end; - int z_expandable; - - stbi__zhuffman z_length, z_distance; -} stbi__zbuf; - -stbi_inline static stbi_uc stbi__zget8(stbi__zbuf *z) -{ - if (z->zbuffer >= z->zbuffer_end) return 0; - return *z->zbuffer++; -} - -static void stbi__fill_bits(stbi__zbuf *z) -{ - do { - STBI_ASSERT(z->code_buffer < (1U << z->num_bits)); - z->code_buffer |= (unsigned int) stbi__zget8(z) << z->num_bits; - z->num_bits += 8; - } while (z->num_bits <= 24); -} - -stbi_inline static unsigned int stbi__zreceive(stbi__zbuf *z, int n) -{ - unsigned int k; - if (z->num_bits < n) stbi__fill_bits(z); - k = z->code_buffer & ((1 << n) - 1); - z->code_buffer >>= n; - z->num_bits -= n; - return k; -} - -static int stbi__zhuffman_decode_slowpath(stbi__zbuf *a, stbi__zhuffman *z) -{ - int b,s,k; - // not resolved by fast table, so compute it the slow way - // use jpeg approach, which requires MSbits at top - k = stbi__bit_reverse(a->code_buffer, 16); - for (s=STBI__ZFAST_BITS+1; ; ++s) - if (k < z->maxcode[s]) - break; - if (s == 16) return -1; // invalid code! - // code size is s, so: - b = (k >> (16-s)) - z->firstcode[s] + z->firstsymbol[s]; - STBI_ASSERT(z->size[b] == s); - a->code_buffer >>= s; - a->num_bits -= s; - return z->value[b]; -} - -stbi_inline static int stbi__zhuffman_decode(stbi__zbuf *a, stbi__zhuffman *z) -{ - int b,s; - if (a->num_bits < 16) stbi__fill_bits(a); - b = z->fast[a->code_buffer & STBI__ZFAST_MASK]; - if (b) { - s = b >> 9; - a->code_buffer >>= s; - a->num_bits -= s; - return b & 511; - } - return stbi__zhuffman_decode_slowpath(a, z); -} - -static int stbi__zexpand(stbi__zbuf *z, char *zout, int n) // need to make room for n bytes -{ - char *q; - int cur, limit, old_limit; - z->zout = zout; - if (!z->z_expandable) return stbi__err("output buffer limit","Corrupt PNG"); - cur = (int) (z->zout - z->zout_start); - limit = old_limit = (int) (z->zout_end - z->zout_start); - while (cur + n > limit) - limit *= 2; - q = (char *) STBI_REALLOC_SIZED(z->zout_start, old_limit, limit); - STBI_NOTUSED(old_limit); - if (q == NULL) return stbi__err("outofmem", "Out of memory"); - z->zout_start = q; - z->zout = q + cur; - z->zout_end = q + limit; - return 1; -} - -static const int stbi__zlength_base[31] = { - 3,4,5,6,7,8,9,10,11,13, - 15,17,19,23,27,31,35,43,51,59, - 67,83,99,115,131,163,195,227,258,0,0 }; - -static const int stbi__zlength_extra[31]= -{ 0,0,0,0,0,0,0,0,1,1,1,1,2,2,2,2,3,3,3,3,4,4,4,4,5,5,5,5,0,0,0 }; - -static const int stbi__zdist_base[32] = { 1,2,3,4,5,7,9,13,17,25,33,49,65,97,129,193, -257,385,513,769,1025,1537,2049,3073,4097,6145,8193,12289,16385,24577,0,0}; - -static const int stbi__zdist_extra[32] = -{ 0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13}; - -static int stbi__parse_huffman_block(stbi__zbuf *a) -{ - char *zout = a->zout; - for(;;) { - int z = stbi__zhuffman_decode(a, &a->z_length); - if (z < 256) { - if (z < 0) return stbi__err("bad huffman code","Corrupt PNG"); // error in huffman codes - if (zout >= a->zout_end) { - if (!stbi__zexpand(a, zout, 1)) return 0; - zout = a->zout; - } - *zout++ = (char) z; - } else { - stbi_uc *p; - int len,dist; - if (z == 256) { - a->zout = zout; - return 1; - } - z -= 257; - len = stbi__zlength_base[z]; - if (stbi__zlength_extra[z]) len += stbi__zreceive(a, stbi__zlength_extra[z]); - z = stbi__zhuffman_decode(a, &a->z_distance); - if (z < 0) return stbi__err("bad huffman code","Corrupt PNG"); - dist = stbi__zdist_base[z]; - if (stbi__zdist_extra[z]) dist += stbi__zreceive(a, stbi__zdist_extra[z]); - if (zout - a->zout_start < dist) return stbi__err("bad dist","Corrupt PNG"); - if (zout + len > a->zout_end) { - if (!stbi__zexpand(a, zout, len)) return 0; - zout = a->zout; - } - p = (stbi_uc *) (zout - dist); - if (dist == 1) { // run of one byte; common in images. - stbi_uc v = *p; - if (len) { do *zout++ = v; while (--len); } - } else { - if (len) { do *zout++ = *p++; while (--len); } - } - } - } -} - -static int stbi__compute_huffman_codes(stbi__zbuf *a) -{ - static const stbi_uc length_dezigzag[19] = { 16,17,18,0,8,7,9,6,10,5,11,4,12,3,13,2,14,1,15 }; - stbi__zhuffman z_codelength; - stbi_uc lencodes[286+32+137];//padding for maximum single op - stbi_uc codelength_sizes[19]; - int i,n; - - int hlit = stbi__zreceive(a,5) + 257; - int hdist = stbi__zreceive(a,5) + 1; - int hclen = stbi__zreceive(a,4) + 4; - int ntot = hlit + hdist; - - memset(codelength_sizes, 0, sizeof(codelength_sizes)); - for (i=0; i < hclen; ++i) { - int s = stbi__zreceive(a,3); - codelength_sizes[length_dezigzag[i]] = (stbi_uc) s; - } - if (!stbi__zbuild_huffman(&z_codelength, codelength_sizes, 19)) return 0; - - n = 0; - while (n < ntot) { - int c = stbi__zhuffman_decode(a, &z_codelength); - if (c < 0 || c >= 19) return stbi__err("bad codelengths", "Corrupt PNG"); - if (c < 16) - lencodes[n++] = (stbi_uc) c; - else { - stbi_uc fill = 0; - if (c == 16) { - c = stbi__zreceive(a,2)+3; - if (n == 0) return stbi__err("bad codelengths", "Corrupt PNG"); - fill = lencodes[n-1]; - } else if (c == 17) - c = stbi__zreceive(a,3)+3; - else { - STBI_ASSERT(c == 18); - c = stbi__zreceive(a,7)+11; - } - if (ntot - n < c) return stbi__err("bad codelengths", "Corrupt PNG"); - memset(lencodes+n, fill, c); - n += c; - } - } - if (n != ntot) return stbi__err("bad codelengths","Corrupt PNG"); - if (!stbi__zbuild_huffman(&a->z_length, lencodes, hlit)) return 0; - if (!stbi__zbuild_huffman(&a->z_distance, lencodes+hlit, hdist)) return 0; - return 1; -} - -static int stbi__parse_uncompressed_block(stbi__zbuf *a) -{ - stbi_uc header[4]; - int len,nlen,k; - if (a->num_bits & 7) - stbi__zreceive(a, a->num_bits & 7); // discard - // drain the bit-packed data into header - k = 0; - while (a->num_bits > 0) { - header[k++] = (stbi_uc) (a->code_buffer & 255); // suppress MSVC run-time check - a->code_buffer >>= 8; - a->num_bits -= 8; - } - STBI_ASSERT(a->num_bits == 0); - // now fill header the normal way - while (k < 4) - header[k++] = stbi__zget8(a); - len = header[1] * 256 + header[0]; - nlen = header[3] * 256 + header[2]; - if (nlen != (len ^ 0xffff)) return stbi__err("zlib corrupt","Corrupt PNG"); - if (a->zbuffer + len > a->zbuffer_end) return stbi__err("read past buffer","Corrupt PNG"); - if (a->zout + len > a->zout_end) - if (!stbi__zexpand(a, a->zout, len)) return 0; - memcpy(a->zout, a->zbuffer, len); - a->zbuffer += len; - a->zout += len; - return 1; -} - -static int stbi__parse_zlib_header(stbi__zbuf *a) -{ - int cmf = stbi__zget8(a); - int cm = cmf & 15; - /* int cinfo = cmf >> 4; */ - int flg = stbi__zget8(a); - if ((cmf*256+flg) % 31 != 0) return stbi__err("bad zlib header","Corrupt PNG"); // zlib spec - if (flg & 32) return stbi__err("no preset dict","Corrupt PNG"); // preset dictionary not allowed in png - if (cm != 8) return stbi__err("bad compression","Corrupt PNG"); // DEFLATE required for png - // window = 1 << (8 + cinfo)... but who cares, we fully buffer output - return 1; -} - -static const stbi_uc stbi__zdefault_length[288] = -{ - 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, - 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, - 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, - 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, - 8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, - 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, - 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, - 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, 9,9,9,9,9,9,9,9,9,9,9,9,9,9,9,9, - 7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7, 7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8 -}; -static const stbi_uc stbi__zdefault_distance[32] = -{ - 5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5 -}; -/* -Init algorithm: -{ - int i; // use <= to match clearly with spec - for (i=0; i <= 143; ++i) stbi__zdefault_length[i] = 8; - for ( ; i <= 255; ++i) stbi__zdefault_length[i] = 9; - for ( ; i <= 279; ++i) stbi__zdefault_length[i] = 7; - for ( ; i <= 287; ++i) stbi__zdefault_length[i] = 8; - - for (i=0; i <= 31; ++i) stbi__zdefault_distance[i] = 5; -} -*/ - -static int stbi__parse_zlib(stbi__zbuf *a, int parse_header) -{ - int final, type; - if (parse_header) - if (!stbi__parse_zlib_header(a)) return 0; - a->num_bits = 0; - a->code_buffer = 0; - do { - final = stbi__zreceive(a,1); - type = stbi__zreceive(a,2); - if (type == 0) { - if (!stbi__parse_uncompressed_block(a)) return 0; - } else if (type == 3) { - return 0; - } else { - if (type == 1) { - // use fixed code lengths - if (!stbi__zbuild_huffman(&a->z_length , stbi__zdefault_length , 288)) return 0; - if (!stbi__zbuild_huffman(&a->z_distance, stbi__zdefault_distance, 32)) return 0; - } else { - if (!stbi__compute_huffman_codes(a)) return 0; - } - if (!stbi__parse_huffman_block(a)) return 0; - } - } while (!final); - return 1; -} - -static int stbi__do_zlib(stbi__zbuf *a, char *obuf, int olen, int exp, int parse_header) -{ - a->zout_start = obuf; - a->zout = obuf; - a->zout_end = obuf + olen; - a->z_expandable = exp; - - return stbi__parse_zlib(a, parse_header); -} - -STBIDEF char *stbi_zlib_decode_malloc_guesssize(const char *buffer, int len, int initial_size, int *outlen) -{ - stbi__zbuf a; - char *p = (char *) stbi__malloc(initial_size); - if (p == NULL) return NULL; - a.zbuffer = (stbi_uc *) buffer; - a.zbuffer_end = (stbi_uc *) buffer + len; - if (stbi__do_zlib(&a, p, initial_size, 1, 1)) { - if (outlen) *outlen = (int) (a.zout - a.zout_start); - return a.zout_start; - } else { - STBI_FREE(a.zout_start); - return NULL; - } -} - -STBIDEF char *stbi_zlib_decode_malloc(char const *buffer, int len, int *outlen) -{ - return stbi_zlib_decode_malloc_guesssize(buffer, len, 16384, outlen); -} - -STBIDEF char *stbi_zlib_decode_malloc_guesssize_headerflag(const char *buffer, int len, int initial_size, int *outlen, int parse_header) -{ - stbi__zbuf a; - char *p = (char *) stbi__malloc(initial_size); - if (p == NULL) return NULL; - a.zbuffer = (stbi_uc *) buffer; - a.zbuffer_end = (stbi_uc *) buffer + len; - if (stbi__do_zlib(&a, p, initial_size, 1, parse_header)) { - if (outlen) *outlen = (int) (a.zout - a.zout_start); - return a.zout_start; - } else { - STBI_FREE(a.zout_start); - return NULL; - } -} - -STBIDEF int stbi_zlib_decode_buffer(char *obuffer, int olen, char const *ibuffer, int ilen) -{ - stbi__zbuf a; - a.zbuffer = (stbi_uc *) ibuffer; - a.zbuffer_end = (stbi_uc *) ibuffer + ilen; - if (stbi__do_zlib(&a, obuffer, olen, 0, 1)) - return (int) (a.zout - a.zout_start); - else - return -1; -} - -STBIDEF char *stbi_zlib_decode_noheader_malloc(char const *buffer, int len, int *outlen) -{ - stbi__zbuf a; - char *p = (char *) stbi__malloc(16384); - if (p == NULL) return NULL; - a.zbuffer = (stbi_uc *) buffer; - a.zbuffer_end = (stbi_uc *) buffer+len; - if (stbi__do_zlib(&a, p, 16384, 1, 0)) { - if (outlen) *outlen = (int) (a.zout - a.zout_start); - return a.zout_start; - } else { - STBI_FREE(a.zout_start); - return NULL; - } -} - -STBIDEF int stbi_zlib_decode_noheader_buffer(char *obuffer, int olen, const char *ibuffer, int ilen) -{ - stbi__zbuf a; - a.zbuffer = (stbi_uc *) ibuffer; - a.zbuffer_end = (stbi_uc *) ibuffer + ilen; - if (stbi__do_zlib(&a, obuffer, olen, 0, 0)) - return (int) (a.zout - a.zout_start); - else - return -1; -} -#endif - -// public domain "baseline" PNG decoder v0.10 Sean Barrett 2006-11-18 -// simple implementation -// - only 8-bit samples -// - no CRC checking -// - allocates lots of intermediate memory -// - avoids problem of streaming data between subsystems -// - avoids explicit window management -// performance -// - uses stb_zlib, a PD zlib implementation with fast huffman decoding - -#ifndef STBI_NO_PNG -typedef struct -{ - stbi__uint32 length; - stbi__uint32 type; -} stbi__pngchunk; - -static stbi__pngchunk stbi__get_chunk_header(stbi__context *s) -{ - stbi__pngchunk c; - c.length = stbi__get32be(s); - c.type = stbi__get32be(s); - return c; -} - -static int stbi__check_png_header(stbi__context *s) -{ - static const stbi_uc png_sig[8] = { 137,80,78,71,13,10,26,10 }; - int i; - for (i=0; i < 8; ++i) - if (stbi__get8(s) != png_sig[i]) return stbi__err("bad png sig","Not a PNG"); - return 1; -} - -typedef struct -{ - stbi__context *s; - stbi_uc *idata, *expanded, *out; - int depth; -} stbi__png; - - -enum { - STBI__F_none=0, - STBI__F_sub=1, - STBI__F_up=2, - STBI__F_avg=3, - STBI__F_paeth=4, - // synthetic filters used for first scanline to avoid needing a dummy row of 0s - STBI__F_avg_first, - STBI__F_paeth_first -}; - -static stbi_uc first_row_filter[5] = -{ - STBI__F_none, - STBI__F_sub, - STBI__F_none, - STBI__F_avg_first, - STBI__F_paeth_first -}; - -static int stbi__paeth(int a, int b, int c) -{ - int p = a + b - c; - int pa = abs(p-a); - int pb = abs(p-b); - int pc = abs(p-c); - if (pa <= pb && pa <= pc) return a; - if (pb <= pc) return b; - return c; -} - -static const stbi_uc stbi__depth_scale_table[9] = { 0, 0xff, 0x55, 0, 0x11, 0,0,0, 0x01 }; - -// create the png data from post-deflated data -static int stbi__create_png_image_raw(stbi__png *a, stbi_uc *raw, stbi__uint32 raw_len, int out_n, stbi__uint32 x, stbi__uint32 y, int depth, int color) -{ - int bytes = (depth == 16? 2 : 1); - stbi__context *s = a->s; - stbi__uint32 i,j,stride = x*out_n*bytes; - stbi__uint32 img_len, img_width_bytes; - int k; - int img_n = s->img_n; // copy it into a local for later - - int output_bytes = out_n*bytes; - int filter_bytes = img_n*bytes; - int width = x; - - STBI_ASSERT(out_n == s->img_n || out_n == s->img_n+1); - a->out = (stbi_uc *) stbi__malloc_mad3(x, y, output_bytes, 0); // extra bytes to write off the end into - if (!a->out) return stbi__err("outofmem", "Out of memory"); - - if (!stbi__mad3sizes_valid(img_n, x, depth, 7)) return stbi__err("too large", "Corrupt PNG"); - img_width_bytes = (((img_n * x * depth) + 7) >> 3); - img_len = (img_width_bytes + 1) * y; - - // we used to check for exact match between raw_len and img_len on non-interlaced PNGs, - // but issue #276 reported a PNG in the wild that had extra data at the end (all zeros), - // so just check for raw_len < img_len always. - if (raw_len < img_len) return stbi__err("not enough pixels","Corrupt PNG"); - - for (j=0; j < y; ++j) { - stbi_uc *cur = a->out + stride*j; - stbi_uc *prior; - int filter = *raw++; - - if (filter > 4) - return stbi__err("invalid filter","Corrupt PNG"); - - if (depth < 8) { - STBI_ASSERT(img_width_bytes <= x); - cur += x*out_n - img_width_bytes; // store output to the rightmost img_len bytes, so we can decode in place - filter_bytes = 1; - width = img_width_bytes; - } - prior = cur - stride; // bugfix: need to compute this after 'cur +=' computation above - - // if first row, use special filter that doesn't sample previous row - if (j == 0) filter = first_row_filter[filter]; - - // handle first byte explicitly - for (k=0; k < filter_bytes; ++k) { - switch (filter) { - case STBI__F_none : cur[k] = raw[k]; break; - case STBI__F_sub : cur[k] = raw[k]; break; - case STBI__F_up : cur[k] = STBI__BYTECAST(raw[k] + prior[k]); break; - case STBI__F_avg : cur[k] = STBI__BYTECAST(raw[k] + (prior[k]>>1)); break; - case STBI__F_paeth : cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(0,prior[k],0)); break; - case STBI__F_avg_first : cur[k] = raw[k]; break; - case STBI__F_paeth_first: cur[k] = raw[k]; break; - } - } - - if (depth == 8) { - if (img_n != out_n) - cur[img_n] = 255; // first pixel - raw += img_n; - cur += out_n; - prior += out_n; - } else if (depth == 16) { - if (img_n != out_n) { - cur[filter_bytes] = 255; // first pixel top byte - cur[filter_bytes+1] = 255; // first pixel bottom byte - } - raw += filter_bytes; - cur += output_bytes; - prior += output_bytes; - } else { - raw += 1; - cur += 1; - prior += 1; - } - - // this is a little gross, so that we don't switch per-pixel or per-component - if (depth < 8 || img_n == out_n) { - int nk = (width - 1)*filter_bytes; - #define STBI__CASE(f) \ - case f: \ - for (k=0; k < nk; ++k) - switch (filter) { - // "none" filter turns into a memcpy here; make that explicit. - case STBI__F_none: memcpy(cur, raw, nk); break; - STBI__CASE(STBI__F_sub) { cur[k] = STBI__BYTECAST(raw[k] + cur[k-filter_bytes]); } break; - STBI__CASE(STBI__F_up) { cur[k] = STBI__BYTECAST(raw[k] + prior[k]); } break; - STBI__CASE(STBI__F_avg) { cur[k] = STBI__BYTECAST(raw[k] + ((prior[k] + cur[k-filter_bytes])>>1)); } break; - STBI__CASE(STBI__F_paeth) { cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k-filter_bytes],prior[k],prior[k-filter_bytes])); } break; - STBI__CASE(STBI__F_avg_first) { cur[k] = STBI__BYTECAST(raw[k] + (cur[k-filter_bytes] >> 1)); } break; - STBI__CASE(STBI__F_paeth_first) { cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k-filter_bytes],0,0)); } break; - } - #undef STBI__CASE - raw += nk; - } else { - STBI_ASSERT(img_n+1 == out_n); - #define STBI__CASE(f) \ - case f: \ - for (i=x-1; i >= 1; --i, cur[filter_bytes]=255,raw+=filter_bytes,cur+=output_bytes,prior+=output_bytes) \ - for (k=0; k < filter_bytes; ++k) - switch (filter) { - STBI__CASE(STBI__F_none) { cur[k] = raw[k]; } break; - STBI__CASE(STBI__F_sub) { cur[k] = STBI__BYTECAST(raw[k] + cur[k- output_bytes]); } break; - STBI__CASE(STBI__F_up) { cur[k] = STBI__BYTECAST(raw[k] + prior[k]); } break; - STBI__CASE(STBI__F_avg) { cur[k] = STBI__BYTECAST(raw[k] + ((prior[k] + cur[k- output_bytes])>>1)); } break; - STBI__CASE(STBI__F_paeth) { cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k- output_bytes],prior[k],prior[k- output_bytes])); } break; - STBI__CASE(STBI__F_avg_first) { cur[k] = STBI__BYTECAST(raw[k] + (cur[k- output_bytes] >> 1)); } break; - STBI__CASE(STBI__F_paeth_first) { cur[k] = STBI__BYTECAST(raw[k] + stbi__paeth(cur[k- output_bytes],0,0)); } break; - } - #undef STBI__CASE - - // the loop above sets the high byte of the pixels' alpha, but for - // 16 bit png files we also need the low byte set. we'll do that here. - if (depth == 16) { - cur = a->out + stride*j; // start at the beginning of the row again - for (i=0; i < x; ++i,cur+=output_bytes) { - cur[filter_bytes+1] = 255; - } - } - } - } - - // we make a separate pass to expand bits to pixels; for performance, - // this could run two scanlines behind the above code, so it won't - // intefere with filtering but will still be in the cache. - if (depth < 8) { - for (j=0; j < y; ++j) { - stbi_uc *cur = a->out + stride*j; - stbi_uc *in = a->out + stride*j + x*out_n - img_width_bytes; - // unpack 1/2/4-bit into a 8-bit buffer. allows us to keep the common 8-bit path optimal at minimal cost for 1/2/4-bit - // png guarante byte alignment, if width is not multiple of 8/4/2 we'll decode dummy trailing data that will be skipped in the later loop - stbi_uc scale = (color == 0) ? stbi__depth_scale_table[depth] : 1; // scale grayscale values to 0..255 range - - // note that the final byte might overshoot and write more data than desired. - // we can allocate enough data that this never writes out of memory, but it - // could also overwrite the next scanline. can it overwrite non-empty data - // on the next scanline? yes, consider 1-pixel-wide scanlines with 1-bit-per-pixel. - // so we need to explicitly clamp the final ones - - if (depth == 4) { - for (k=x*img_n; k >= 2; k-=2, ++in) { - *cur++ = scale * ((*in >> 4) ); - *cur++ = scale * ((*in ) & 0x0f); - } - if (k > 0) *cur++ = scale * ((*in >> 4) ); - } else if (depth == 2) { - for (k=x*img_n; k >= 4; k-=4, ++in) { - *cur++ = scale * ((*in >> 6) ); - *cur++ = scale * ((*in >> 4) & 0x03); - *cur++ = scale * ((*in >> 2) & 0x03); - *cur++ = scale * ((*in ) & 0x03); - } - if (k > 0) *cur++ = scale * ((*in >> 6) ); - if (k > 1) *cur++ = scale * ((*in >> 4) & 0x03); - if (k > 2) *cur++ = scale * ((*in >> 2) & 0x03); - } else if (depth == 1) { - for (k=x*img_n; k >= 8; k-=8, ++in) { - *cur++ = scale * ((*in >> 7) ); - *cur++ = scale * ((*in >> 6) & 0x01); - *cur++ = scale * ((*in >> 5) & 0x01); - *cur++ = scale * ((*in >> 4) & 0x01); - *cur++ = scale * ((*in >> 3) & 0x01); - *cur++ = scale * ((*in >> 2) & 0x01); - *cur++ = scale * ((*in >> 1) & 0x01); - *cur++ = scale * ((*in ) & 0x01); - } - if (k > 0) *cur++ = scale * ((*in >> 7) ); - if (k > 1) *cur++ = scale * ((*in >> 6) & 0x01); - if (k > 2) *cur++ = scale * ((*in >> 5) & 0x01); - if (k > 3) *cur++ = scale * ((*in >> 4) & 0x01); - if (k > 4) *cur++ = scale * ((*in >> 3) & 0x01); - if (k > 5) *cur++ = scale * ((*in >> 2) & 0x01); - if (k > 6) *cur++ = scale * ((*in >> 1) & 0x01); - } - if (img_n != out_n) { - int q; - // insert alpha = 255 - cur = a->out + stride*j; - if (img_n == 1) { - for (q=x-1; q >= 0; --q) { - cur[q*2+1] = 255; - cur[q*2+0] = cur[q]; - } - } else { - STBI_ASSERT(img_n == 3); - for (q=x-1; q >= 0; --q) { - cur[q*4+3] = 255; - cur[q*4+2] = cur[q*3+2]; - cur[q*4+1] = cur[q*3+1]; - cur[q*4+0] = cur[q*3+0]; - } - } - } - } - } else if (depth == 16) { - // force the image data from big-endian to platform-native. - // this is done in a separate pass due to the decoding relying - // on the data being untouched, but could probably be done - // per-line during decode if care is taken. - stbi_uc *cur = a->out; - stbi__uint16 *cur16 = (stbi__uint16*)cur; - - for(i=0; i < x*y*out_n; ++i,cur16++,cur+=2) { - *cur16 = (cur[0] << 8) | cur[1]; - } - } - - return 1; -} - -static int stbi__create_png_image(stbi__png *a, stbi_uc *image_data, stbi__uint32 image_data_len, int out_n, int depth, int color, int interlaced) -{ - int bytes = (depth == 16 ? 2 : 1); - int out_bytes = out_n * bytes; - stbi_uc *final; - int p; - if (!interlaced) - return stbi__create_png_image_raw(a, image_data, image_data_len, out_n, a->s->img_x, a->s->img_y, depth, color); - - // de-interlacing - final = (stbi_uc *) stbi__malloc_mad3(a->s->img_x, a->s->img_y, out_bytes, 0); - for (p=0; p < 7; ++p) { - int xorig[] = { 0,4,0,2,0,1,0 }; - int yorig[] = { 0,0,4,0,2,0,1 }; - int xspc[] = { 8,8,4,4,2,2,1 }; - int yspc[] = { 8,8,8,4,4,2,2 }; - int i,j,x,y; - // pass1_x[4] = 0, pass1_x[5] = 1, pass1_x[12] = 1 - x = (a->s->img_x - xorig[p] + xspc[p]-1) / xspc[p]; - y = (a->s->img_y - yorig[p] + yspc[p]-1) / yspc[p]; - if (x && y) { - stbi__uint32 img_len = ((((a->s->img_n * x * depth) + 7) >> 3) + 1) * y; - if (!stbi__create_png_image_raw(a, image_data, image_data_len, out_n, x, y, depth, color)) { - STBI_FREE(final); - return 0; - } - for (j=0; j < y; ++j) { - for (i=0; i < x; ++i) { - int out_y = j*yspc[p]+yorig[p]; - int out_x = i*xspc[p]+xorig[p]; - memcpy(final + out_y*a->s->img_x*out_bytes + out_x*out_bytes, - a->out + (j*x+i)*out_bytes, out_bytes); - } - } - STBI_FREE(a->out); - image_data += img_len; - image_data_len -= img_len; - } - } - a->out = final; - - return 1; -} - -static int stbi__compute_transparency(stbi__png *z, stbi_uc tc[3], int out_n) -{ - stbi__context *s = z->s; - stbi__uint32 i, pixel_count = s->img_x * s->img_y; - stbi_uc *p = z->out; - - // compute color-based transparency, assuming we've - // already got 255 as the alpha value in the output - STBI_ASSERT(out_n == 2 || out_n == 4); - - if (out_n == 2) { - for (i=0; i < pixel_count; ++i) { - p[1] = (p[0] == tc[0] ? 0 : 255); - p += 2; - } - } else { - for (i=0; i < pixel_count; ++i) { - if (p[0] == tc[0] && p[1] == tc[1] && p[2] == tc[2]) - p[3] = 0; - p += 4; - } - } - return 1; -} - -static int stbi__compute_transparency16(stbi__png *z, stbi__uint16 tc[3], int out_n) -{ - stbi__context *s = z->s; - stbi__uint32 i, pixel_count = s->img_x * s->img_y; - stbi__uint16 *p = (stbi__uint16*) z->out; - - // compute color-based transparency, assuming we've - // already got 65535 as the alpha value in the output - STBI_ASSERT(out_n == 2 || out_n == 4); - - if (out_n == 2) { - for (i = 0; i < pixel_count; ++i) { - p[1] = (p[0] == tc[0] ? 0 : 65535); - p += 2; - } - } else { - for (i = 0; i < pixel_count; ++i) { - if (p[0] == tc[0] && p[1] == tc[1] && p[2] == tc[2]) - p[3] = 0; - p += 4; - } - } - return 1; -} - -static int stbi__expand_png_palette(stbi__png *a, stbi_uc *palette, int len, int pal_img_n) -{ - stbi__uint32 i, pixel_count = a->s->img_x * a->s->img_y; - stbi_uc *p, *temp_out, *orig = a->out; - - p = (stbi_uc *) stbi__malloc_mad2(pixel_count, pal_img_n, 0); - if (p == NULL) return stbi__err("outofmem", "Out of memory"); - - // between here and free(out) below, exitting would leak - temp_out = p; - - if (pal_img_n == 3) { - for (i=0; i < pixel_count; ++i) { - int n = orig[i]*4; - p[0] = palette[n ]; - p[1] = palette[n+1]; - p[2] = palette[n+2]; - p += 3; - } - } else { - for (i=0; i < pixel_count; ++i) { - int n = orig[i]*4; - p[0] = palette[n ]; - p[1] = palette[n+1]; - p[2] = palette[n+2]; - p[3] = palette[n+3]; - p += 4; - } - } - STBI_FREE(a->out); - a->out = temp_out; - - STBI_NOTUSED(len); - - return 1; -} - -static int stbi__unpremultiply_on_load = 0; -static int stbi__de_iphone_flag = 0; - -STBIDEF void stbi_set_unpremultiply_on_load(int flag_true_if_should_unpremultiply) -{ - stbi__unpremultiply_on_load = flag_true_if_should_unpremultiply; -} - -STBIDEF void stbi_convert_iphone_png_to_rgb(int flag_true_if_should_convert) -{ - stbi__de_iphone_flag = flag_true_if_should_convert; -} - -static void stbi__de_iphone(stbi__png *z) -{ - stbi__context *s = z->s; - stbi__uint32 i, pixel_count = s->img_x * s->img_y; - stbi_uc *p = z->out; - - if (s->img_out_n == 3) { // convert bgr to rgb - for (i=0; i < pixel_count; ++i) { - stbi_uc t = p[0]; - p[0] = p[2]; - p[2] = t; - p += 3; - } - } else { - STBI_ASSERT(s->img_out_n == 4); - if (stbi__unpremultiply_on_load) { - // convert bgr to rgb and unpremultiply - for (i=0; i < pixel_count; ++i) { - stbi_uc a = p[3]; - stbi_uc t = p[0]; - if (a) { - stbi_uc half = a / 2; - p[0] = (p[2] * 255 + half) / a; - p[1] = (p[1] * 255 + half) / a; - p[2] = ( t * 255 + half) / a; - } else { - p[0] = p[2]; - p[2] = t; - } - p += 4; - } - } else { - // convert bgr to rgb - for (i=0; i < pixel_count; ++i) { - stbi_uc t = p[0]; - p[0] = p[2]; - p[2] = t; - p += 4; - } - } - } -} - -#define STBI__PNG_TYPE(a,b,c,d) (((unsigned) (a) << 24) + ((unsigned) (b) << 16) + ((unsigned) (c) << 8) + (unsigned) (d)) - -static int stbi__parse_png_file(stbi__png *z, int scan, int req_comp) -{ - stbi_uc palette[1024], pal_img_n=0; - stbi_uc has_trans=0, tc[3]={0}; - stbi__uint16 tc16[3]; - stbi__uint32 ioff=0, idata_limit=0, i, pal_len=0; - int first=1,k,interlace=0, color=0, is_iphone=0; - stbi__context *s = z->s; - - z->expanded = NULL; - z->idata = NULL; - z->out = NULL; - - if (!stbi__check_png_header(s)) return 0; - - if (scan == STBI__SCAN_type) return 1; - - for (;;) { - stbi__pngchunk c = stbi__get_chunk_header(s); - switch (c.type) { - case STBI__PNG_TYPE('C','g','B','I'): - is_iphone = 1; - stbi__skip(s, c.length); - break; - case STBI__PNG_TYPE('I','H','D','R'): { - int comp,filter; - if (!first) return stbi__err("multiple IHDR","Corrupt PNG"); - first = 0; - if (c.length != 13) return stbi__err("bad IHDR len","Corrupt PNG"); - s->img_x = stbi__get32be(s); if (s->img_x > (1 << 24)) return stbi__err("too large","Very large image (corrupt?)"); - s->img_y = stbi__get32be(s); if (s->img_y > (1 << 24)) return stbi__err("too large","Very large image (corrupt?)"); - z->depth = stbi__get8(s); if (z->depth != 1 && z->depth != 2 && z->depth != 4 && z->depth != 8 && z->depth != 16) return stbi__err("1/2/4/8/16-bit only","PNG not supported: 1/2/4/8/16-bit only"); - color = stbi__get8(s); if (color > 6) return stbi__err("bad ctype","Corrupt PNG"); - if (color == 3 && z->depth == 16) return stbi__err("bad ctype","Corrupt PNG"); - if (color == 3) pal_img_n = 3; else if (color & 1) return stbi__err("bad ctype","Corrupt PNG"); - comp = stbi__get8(s); if (comp) return stbi__err("bad comp method","Corrupt PNG"); - filter= stbi__get8(s); if (filter) return stbi__err("bad filter method","Corrupt PNG"); - interlace = stbi__get8(s); if (interlace>1) return stbi__err("bad interlace method","Corrupt PNG"); - if (!s->img_x || !s->img_y) return stbi__err("0-pixel image","Corrupt PNG"); - if (!pal_img_n) { - s->img_n = (color & 2 ? 3 : 1) + (color & 4 ? 1 : 0); - if ((1 << 30) / s->img_x / s->img_n < s->img_y) return stbi__err("too large", "Image too large to decode"); - if (scan == STBI__SCAN_header) return 1; - } else { - // if paletted, then pal_n is our final components, and - // img_n is # components to decompress/filter. - s->img_n = 1; - if ((1 << 30) / s->img_x / 4 < s->img_y) return stbi__err("too large","Corrupt PNG"); - // if SCAN_header, have to scan to see if we have a tRNS - } - break; - } - - case STBI__PNG_TYPE('P','L','T','E'): { - if (first) return stbi__err("first not IHDR", "Corrupt PNG"); - if (c.length > 256*3) return stbi__err("invalid PLTE","Corrupt PNG"); - pal_len = c.length / 3; - if (pal_len * 3 != c.length) return stbi__err("invalid PLTE","Corrupt PNG"); - for (i=0; i < pal_len; ++i) { - palette[i*4+0] = stbi__get8(s); - palette[i*4+1] = stbi__get8(s); - palette[i*4+2] = stbi__get8(s); - palette[i*4+3] = 255; - } - break; - } - - case STBI__PNG_TYPE('t','R','N','S'): { - if (first) return stbi__err("first not IHDR", "Corrupt PNG"); - if (z->idata) return stbi__err("tRNS after IDAT","Corrupt PNG"); - if (pal_img_n) { - if (scan == STBI__SCAN_header) { s->img_n = 4; return 1; } - if (pal_len == 0) return stbi__err("tRNS before PLTE","Corrupt PNG"); - if (c.length > pal_len) return stbi__err("bad tRNS len","Corrupt PNG"); - pal_img_n = 4; - for (i=0; i < c.length; ++i) - palette[i*4+3] = stbi__get8(s); - } else { - if (!(s->img_n & 1)) return stbi__err("tRNS with alpha","Corrupt PNG"); - if (c.length != (stbi__uint32) s->img_n*2) return stbi__err("bad tRNS len","Corrupt PNG"); - has_trans = 1; - if (z->depth == 16) { - for (k = 0; k < s->img_n; ++k) tc16[k] = (stbi__uint16)stbi__get16be(s); // copy the values as-is - } else { - for (k = 0; k < s->img_n; ++k) tc[k] = (stbi_uc)(stbi__get16be(s) & 255) * stbi__depth_scale_table[z->depth]; // non 8-bit images will be larger - } - } - break; - } - - case STBI__PNG_TYPE('I','D','A','T'): { - if (first) return stbi__err("first not IHDR", "Corrupt PNG"); - if (pal_img_n && !pal_len) return stbi__err("no PLTE","Corrupt PNG"); - if (scan == STBI__SCAN_header) { s->img_n = pal_img_n; return 1; } - if ((int)(ioff + c.length) < (int)ioff) return 0; - if (ioff + c.length > idata_limit) { - stbi__uint32 idata_limit_old = idata_limit; - stbi_uc *p; - if (idata_limit == 0) idata_limit = c.length > 4096 ? c.length : 4096; - while (ioff + c.length > idata_limit) - idata_limit *= 2; - STBI_NOTUSED(idata_limit_old); - p = (stbi_uc *) STBI_REALLOC_SIZED(z->idata, idata_limit_old, idata_limit); if (p == NULL) return stbi__err("outofmem", "Out of memory"); - z->idata = p; - } - if (!stbi__getn(s, z->idata+ioff,c.length)) return stbi__err("outofdata","Corrupt PNG"); - ioff += c.length; - break; - } - - case STBI__PNG_TYPE('I','E','N','D'): { - stbi__uint32 raw_len, bpl; - if (first) return stbi__err("first not IHDR", "Corrupt PNG"); - if (scan != STBI__SCAN_load) return 1; - if (z->idata == NULL) return stbi__err("no IDAT","Corrupt PNG"); - // initial guess for decoded data size to avoid unnecessary reallocs - bpl = (s->img_x * z->depth + 7) / 8; // bytes per line, per component - raw_len = bpl * s->img_y * s->img_n /* pixels */ + s->img_y /* filter mode per row */; - z->expanded = (stbi_uc *) stbi_zlib_decode_malloc_guesssize_headerflag((char *) z->idata, ioff, raw_len, (int *) &raw_len, !is_iphone); - if (z->expanded == NULL) return 0; // zlib should set error - STBI_FREE(z->idata); z->idata = NULL; - if ((req_comp == s->img_n+1 && req_comp != 3 && !pal_img_n) || has_trans) - s->img_out_n = s->img_n+1; - else - s->img_out_n = s->img_n; - if (!stbi__create_png_image(z, z->expanded, raw_len, s->img_out_n, z->depth, color, interlace)) return 0; - if (has_trans) { - if (z->depth == 16) { - if (!stbi__compute_transparency16(z, tc16, s->img_out_n)) return 0; - } else { - if (!stbi__compute_transparency(z, tc, s->img_out_n)) return 0; - } - } - if (is_iphone && stbi__de_iphone_flag && s->img_out_n > 2) - stbi__de_iphone(z); - if (pal_img_n) { - // pal_img_n == 3 or 4 - s->img_n = pal_img_n; // record the actual colors we had - s->img_out_n = pal_img_n; - if (req_comp >= 3) s->img_out_n = req_comp; - if (!stbi__expand_png_palette(z, palette, pal_len, s->img_out_n)) - return 0; - } else if (has_trans) { - // non-paletted image with tRNS -> source image has (constant) alpha - ++s->img_n; - } - STBI_FREE(z->expanded); z->expanded = NULL; - return 1; - } - - default: - // if critical, fail - if (first) return stbi__err("first not IHDR", "Corrupt PNG"); - if ((c.type & (1 << 29)) == 0) { - #ifndef STBI_NO_FAILURE_STRINGS - // not threadsafe - static char invalid_chunk[] = "XXXX PNG chunk not known"; - invalid_chunk[0] = STBI__BYTECAST(c.type >> 24); - invalid_chunk[1] = STBI__BYTECAST(c.type >> 16); - invalid_chunk[2] = STBI__BYTECAST(c.type >> 8); - invalid_chunk[3] = STBI__BYTECAST(c.type >> 0); - #endif - return stbi__err(invalid_chunk, "PNG not supported: unknown PNG chunk type"); - } - stbi__skip(s, c.length); - break; - } - // end of PNG chunk, read and skip CRC - stbi__get32be(s); - } -} - -static void *stbi__do_png(stbi__png *p, int *x, int *y, int *n, int req_comp, stbi__result_info *ri) -{ - void *result=NULL; - if (req_comp < 0 || req_comp > 4) return stbi__errpuc("bad req_comp", "Internal error"); - if (stbi__parse_png_file(p, STBI__SCAN_load, req_comp)) { - if (p->depth < 8) - ri->bits_per_channel = 8; - else - ri->bits_per_channel = p->depth; - result = p->out; - p->out = NULL; - if (req_comp && req_comp != p->s->img_out_n) { - if (ri->bits_per_channel == 8) - result = stbi__convert_format((unsigned char *) result, p->s->img_out_n, req_comp, p->s->img_x, p->s->img_y); - else - result = stbi__convert_format16((stbi__uint16 *) result, p->s->img_out_n, req_comp, p->s->img_x, p->s->img_y); - p->s->img_out_n = req_comp; - if (result == NULL) return result; - } - *x = p->s->img_x; - *y = p->s->img_y; - if (n) *n = p->s->img_n; - } - STBI_FREE(p->out); p->out = NULL; - STBI_FREE(p->expanded); p->expanded = NULL; - STBI_FREE(p->idata); p->idata = NULL; - - return result; -} - -static void *stbi__png_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) -{ - stbi__png p; - p.s = s; - return stbi__do_png(&p, x,y,comp,req_comp, ri); -} - -static int stbi__png_test(stbi__context *s) -{ - int r; - r = stbi__check_png_header(s); - stbi__rewind(s); - return r; -} - -static int stbi__png_info_raw(stbi__png *p, int *x, int *y, int *comp) -{ - if (!stbi__parse_png_file(p, STBI__SCAN_header, 0)) { - stbi__rewind( p->s ); - return 0; - } - if (x) *x = p->s->img_x; - if (y) *y = p->s->img_y; - if (comp) *comp = p->s->img_n; - return 1; -} - -static int stbi__png_info(stbi__context *s, int *x, int *y, int *comp) -{ - stbi__png p; - p.s = s; - return stbi__png_info_raw(&p, x, y, comp); -} - -static int stbi__png_is16(stbi__context *s) -{ - stbi__png p; - p.s = s; - if (!stbi__png_info_raw(&p, NULL, NULL, NULL)) - return 0; - if (p.depth != 16) { - stbi__rewind(p.s); - return 0; - } - return 1; -} -#endif - -// Microsoft/Windows BMP image - -#ifndef STBI_NO_BMP -static int stbi__bmp_test_raw(stbi__context *s) -{ - int r; - int sz; - if (stbi__get8(s) != 'B') return 0; - if (stbi__get8(s) != 'M') return 0; - stbi__get32le(s); // discard filesize - stbi__get16le(s); // discard reserved - stbi__get16le(s); // discard reserved - stbi__get32le(s); // discard data offset - sz = stbi__get32le(s); - r = (sz == 12 || sz == 40 || sz == 56 || sz == 108 || sz == 124); - return r; -} - -static int stbi__bmp_test(stbi__context *s) -{ - int r = stbi__bmp_test_raw(s); - stbi__rewind(s); - return r; -} - - -// returns 0..31 for the highest set bit -static int stbi__high_bit(unsigned int z) -{ - int n=0; - if (z == 0) return -1; - if (z >= 0x10000) { n += 16; z >>= 16; } - if (z >= 0x00100) { n += 8; z >>= 8; } - if (z >= 0x00010) { n += 4; z >>= 4; } - if (z >= 0x00004) { n += 2; z >>= 2; } - if (z >= 0x00002) { n += 1; z >>= 1; } - return n; -} - -static int stbi__bitcount(unsigned int a) -{ - a = (a & 0x55555555) + ((a >> 1) & 0x55555555); // max 2 - a = (a & 0x33333333) + ((a >> 2) & 0x33333333); // max 4 - a = (a + (a >> 4)) & 0x0f0f0f0f; // max 8 per 4, now 8 bits - a = (a + (a >> 8)); // max 16 per 8 bits - a = (a + (a >> 16)); // max 32 per 8 bits - return a & 0xff; -} - -// extract an arbitrarily-aligned N-bit value (N=bits) -// from v, and then make it 8-bits long and fractionally -// extend it to full full range. -static int stbi__shiftsigned(unsigned int v, int shift, int bits) -{ - static unsigned int mul_table[9] = { - 0, - 0xff/*0b11111111*/, 0x55/*0b01010101*/, 0x49/*0b01001001*/, 0x11/*0b00010001*/, - 0x21/*0b00100001*/, 0x41/*0b01000001*/, 0x81/*0b10000001*/, 0x01/*0b00000001*/, - }; - static unsigned int shift_table[9] = { - 0, 0,0,1,0,2,4,6,0, - }; - if (shift < 0) - v <<= -shift; - else - v >>= shift; - STBI_ASSERT(v >= 0 && v < 256); - v >>= (8-bits); - STBI_ASSERT(bits >= 0 && bits <= 8); - return (int) ((unsigned) v * mul_table[bits]) >> shift_table[bits]; -} - -typedef struct -{ - int bpp, offset, hsz; - unsigned int mr,mg,mb,ma, all_a; -} stbi__bmp_data; - -static void *stbi__bmp_parse_header(stbi__context *s, stbi__bmp_data *info) -{ - int hsz; - if (stbi__get8(s) != 'B' || stbi__get8(s) != 'M') return stbi__errpuc("not BMP", "Corrupt BMP"); - stbi__get32le(s); // discard filesize - stbi__get16le(s); // discard reserved - stbi__get16le(s); // discard reserved - info->offset = stbi__get32le(s); - info->hsz = hsz = stbi__get32le(s); - info->mr = info->mg = info->mb = info->ma = 0; - - if (hsz != 12 && hsz != 40 && hsz != 56 && hsz != 108 && hsz != 124) return stbi__errpuc("unknown BMP", "BMP type not supported: unknown"); - if (hsz == 12) { - s->img_x = stbi__get16le(s); - s->img_y = stbi__get16le(s); - } else { - s->img_x = stbi__get32le(s); - s->img_y = stbi__get32le(s); - } - if (stbi__get16le(s) != 1) return stbi__errpuc("bad BMP", "bad BMP"); - info->bpp = stbi__get16le(s); - if (hsz != 12) { - int compress = stbi__get32le(s); - if (compress == 1 || compress == 2) return stbi__errpuc("BMP RLE", "BMP type not supported: RLE"); - stbi__get32le(s); // discard sizeof - stbi__get32le(s); // discard hres - stbi__get32le(s); // discard vres - stbi__get32le(s); // discard colorsused - stbi__get32le(s); // discard max important - if (hsz == 40 || hsz == 56) { - if (hsz == 56) { - stbi__get32le(s); - stbi__get32le(s); - stbi__get32le(s); - stbi__get32le(s); - } - if (info->bpp == 16 || info->bpp == 32) { - if (compress == 0) { - if (info->bpp == 32) { - info->mr = 0xffu << 16; - info->mg = 0xffu << 8; - info->mb = 0xffu << 0; - info->ma = 0xffu << 24; - info->all_a = 0; // if all_a is 0 at end, then we loaded alpha channel but it was all 0 - } else { - info->mr = 31u << 10; - info->mg = 31u << 5; - info->mb = 31u << 0; - } - } else if (compress == 3) { - info->mr = stbi__get32le(s); - info->mg = stbi__get32le(s); - info->mb = stbi__get32le(s); - // not documented, but generated by photoshop and handled by mspaint - if (info->mr == info->mg && info->mg == info->mb) { - // ?!?!? - return stbi__errpuc("bad BMP", "bad BMP"); - } - } else - return stbi__errpuc("bad BMP", "bad BMP"); - } - } else { - int i; - if (hsz != 108 && hsz != 124) - return stbi__errpuc("bad BMP", "bad BMP"); - info->mr = stbi__get32le(s); - info->mg = stbi__get32le(s); - info->mb = stbi__get32le(s); - info->ma = stbi__get32le(s); - stbi__get32le(s); // discard color space - for (i=0; i < 12; ++i) - stbi__get32le(s); // discard color space parameters - if (hsz == 124) { - stbi__get32le(s); // discard rendering intent - stbi__get32le(s); // discard offset of profile data - stbi__get32le(s); // discard size of profile data - stbi__get32le(s); // discard reserved - } - } - } - return (void *) 1; -} - - -static void *stbi__bmp_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) -{ - stbi_uc *out; - unsigned int mr=0,mg=0,mb=0,ma=0, all_a; - stbi_uc pal[256][4]; - int psize=0,i,j,width; - int flip_vertically, pad, target; - stbi__bmp_data info; - STBI_NOTUSED(ri); - - info.all_a = 255; - if (stbi__bmp_parse_header(s, &info) == NULL) - return NULL; // error code already set - - flip_vertically = ((int) s->img_y) > 0; - s->img_y = abs((int) s->img_y); - - mr = info.mr; - mg = info.mg; - mb = info.mb; - ma = info.ma; - all_a = info.all_a; - - if (info.hsz == 12) { - if (info.bpp < 24) - psize = (info.offset - 14 - 24) / 3; - } else { - if (info.bpp < 16) - psize = (info.offset - 14 - info.hsz) >> 2; - } - - s->img_n = ma ? 4 : 3; - if (req_comp && req_comp >= 3) // we can directly decode 3 or 4 - target = req_comp; - else - target = s->img_n; // if they want monochrome, we'll post-convert - - // sanity-check size - if (!stbi__mad3sizes_valid(target, s->img_x, s->img_y, 0)) - return stbi__errpuc("too large", "Corrupt BMP"); - - out = (stbi_uc *) stbi__malloc_mad3(target, s->img_x, s->img_y, 0); - if (!out) return stbi__errpuc("outofmem", "Out of memory"); - if (info.bpp < 16) { - int z=0; - if (psize == 0 || psize > 256) { STBI_FREE(out); return stbi__errpuc("invalid", "Corrupt BMP"); } - for (i=0; i < psize; ++i) { - pal[i][2] = stbi__get8(s); - pal[i][1] = stbi__get8(s); - pal[i][0] = stbi__get8(s); - if (info.hsz != 12) stbi__get8(s); - pal[i][3] = 255; - } - stbi__skip(s, info.offset - 14 - info.hsz - psize * (info.hsz == 12 ? 3 : 4)); - if (info.bpp == 1) width = (s->img_x + 7) >> 3; - else if (info.bpp == 4) width = (s->img_x + 1) >> 1; - else if (info.bpp == 8) width = s->img_x; - else { STBI_FREE(out); return stbi__errpuc("bad bpp", "Corrupt BMP"); } - pad = (-width)&3; - if (info.bpp == 1) { - for (j=0; j < (int) s->img_y; ++j) { - int bit_offset = 7, v = stbi__get8(s); - for (i=0; i < (int) s->img_x; ++i) { - int color = (v>>bit_offset)&0x1; - out[z++] = pal[color][0]; - out[z++] = pal[color][1]; - out[z++] = pal[color][2]; - if (target == 4) out[z++] = 255; - if (i+1 == (int) s->img_x) break; - if((--bit_offset) < 0) { - bit_offset = 7; - v = stbi__get8(s); - } - } - stbi__skip(s, pad); - } - } else { - for (j=0; j < (int) s->img_y; ++j) { - for (i=0; i < (int) s->img_x; i += 2) { - int v=stbi__get8(s),v2=0; - if (info.bpp == 4) { - v2 = v & 15; - v >>= 4; - } - out[z++] = pal[v][0]; - out[z++] = pal[v][1]; - out[z++] = pal[v][2]; - if (target == 4) out[z++] = 255; - if (i+1 == (int) s->img_x) break; - v = (info.bpp == 8) ? stbi__get8(s) : v2; - out[z++] = pal[v][0]; - out[z++] = pal[v][1]; - out[z++] = pal[v][2]; - if (target == 4) out[z++] = 255; - } - stbi__skip(s, pad); - } - } - } else { - int rshift=0,gshift=0,bshift=0,ashift=0,rcount=0,gcount=0,bcount=0,acount=0; - int z = 0; - int easy=0; - stbi__skip(s, info.offset - 14 - info.hsz); - if (info.bpp == 24) width = 3 * s->img_x; - else if (info.bpp == 16) width = 2*s->img_x; - else /* bpp = 32 and pad = 0 */ width=0; - pad = (-width) & 3; - if (info.bpp == 24) { - easy = 1; - } else if (info.bpp == 32) { - if (mb == 0xff && mg == 0xff00 && mr == 0x00ff0000 && ma == 0xff000000) - easy = 2; - } - if (!easy) { - if (!mr || !mg || !mb) { STBI_FREE(out); return stbi__errpuc("bad masks", "Corrupt BMP"); } - // right shift amt to put high bit in position #7 - rshift = stbi__high_bit(mr)-7; rcount = stbi__bitcount(mr); - gshift = stbi__high_bit(mg)-7; gcount = stbi__bitcount(mg); - bshift = stbi__high_bit(mb)-7; bcount = stbi__bitcount(mb); - ashift = stbi__high_bit(ma)-7; acount = stbi__bitcount(ma); - } - for (j=0; j < (int) s->img_y; ++j) { - if (easy) { - for (i=0; i < (int) s->img_x; ++i) { - unsigned char a; - out[z+2] = stbi__get8(s); - out[z+1] = stbi__get8(s); - out[z+0] = stbi__get8(s); - z += 3; - a = (easy == 2 ? stbi__get8(s) : 255); - all_a |= a; - if (target == 4) out[z++] = a; - } - } else { - int bpp = info.bpp; - for (i=0; i < (int) s->img_x; ++i) { - stbi__uint32 v = (bpp == 16 ? (stbi__uint32) stbi__get16le(s) : stbi__get32le(s)); - unsigned int a; - out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mr, rshift, rcount)); - out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mg, gshift, gcount)); - out[z++] = STBI__BYTECAST(stbi__shiftsigned(v & mb, bshift, bcount)); - a = (ma ? stbi__shiftsigned(v & ma, ashift, acount) : 255); - all_a |= a; - if (target == 4) out[z++] = STBI__BYTECAST(a); - } - } - stbi__skip(s, pad); - } - } - - // if alpha channel is all 0s, replace with all 255s - if (target == 4 && all_a == 0) - for (i=4*s->img_x*s->img_y-1; i >= 0; i -= 4) - out[i] = 255; - - if (flip_vertically) { - stbi_uc t; - for (j=0; j < (int) s->img_y>>1; ++j) { - stbi_uc *p1 = out + j *s->img_x*target; - stbi_uc *p2 = out + (s->img_y-1-j)*s->img_x*target; - for (i=0; i < (int) s->img_x*target; ++i) { - t = p1[i]; p1[i] = p2[i]; p2[i] = t; - } - } - } - - if (req_comp && req_comp != target) { - out = stbi__convert_format(out, target, req_comp, s->img_x, s->img_y); - if (out == NULL) return out; // stbi__convert_format frees input on failure - } - - *x = s->img_x; - *y = s->img_y; - if (comp) *comp = s->img_n; - return out; -} -#endif - -// Targa Truevision - TGA -// by Jonathan Dummer -#ifndef STBI_NO_TGA -// returns STBI_rgb or whatever, 0 on error -static int stbi__tga_get_comp(int bits_per_pixel, int is_grey, int* is_rgb16) -{ - // only RGB or RGBA (incl. 16bit) or grey allowed - if (is_rgb16) *is_rgb16 = 0; - switch(bits_per_pixel) { - case 8: return STBI_grey; - case 16: if(is_grey) return STBI_grey_alpha; - // fallthrough - case 15: if(is_rgb16) *is_rgb16 = 1; - return STBI_rgb; - case 24: // fallthrough - case 32: return bits_per_pixel/8; - default: return 0; - } -} - -static int stbi__tga_info(stbi__context *s, int *x, int *y, int *comp) -{ - int tga_w, tga_h, tga_comp, tga_image_type, tga_bits_per_pixel, tga_colormap_bpp; - int sz, tga_colormap_type; - stbi__get8(s); // discard Offset - tga_colormap_type = stbi__get8(s); // colormap type - if( tga_colormap_type > 1 ) { - stbi__rewind(s); - return 0; // only RGB or indexed allowed - } - tga_image_type = stbi__get8(s); // image type - if ( tga_colormap_type == 1 ) { // colormapped (paletted) image - if (tga_image_type != 1 && tga_image_type != 9) { - stbi__rewind(s); - return 0; - } - stbi__skip(s,4); // skip index of first colormap entry and number of entries - sz = stbi__get8(s); // check bits per palette color entry - if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) { - stbi__rewind(s); - return 0; - } - stbi__skip(s,4); // skip image x and y origin - tga_colormap_bpp = sz; - } else { // "normal" image w/o colormap - only RGB or grey allowed, +/- RLE - if ( (tga_image_type != 2) && (tga_image_type != 3) && (tga_image_type != 10) && (tga_image_type != 11) ) { - stbi__rewind(s); - return 0; // only RGB or grey allowed, +/- RLE - } - stbi__skip(s,9); // skip colormap specification and image x/y origin - tga_colormap_bpp = 0; - } - tga_w = stbi__get16le(s); - if( tga_w < 1 ) { - stbi__rewind(s); - return 0; // test width - } - tga_h = stbi__get16le(s); - if( tga_h < 1 ) { - stbi__rewind(s); - return 0; // test height - } - tga_bits_per_pixel = stbi__get8(s); // bits per pixel - stbi__get8(s); // ignore alpha bits - if (tga_colormap_bpp != 0) { - if((tga_bits_per_pixel != 8) && (tga_bits_per_pixel != 16)) { - // when using a colormap, tga_bits_per_pixel is the size of the indexes - // I don't think anything but 8 or 16bit indexes makes sense - stbi__rewind(s); - return 0; - } - tga_comp = stbi__tga_get_comp(tga_colormap_bpp, 0, NULL); - } else { - tga_comp = stbi__tga_get_comp(tga_bits_per_pixel, (tga_image_type == 3) || (tga_image_type == 11), NULL); - } - if(!tga_comp) { - stbi__rewind(s); - return 0; - } - if (x) *x = tga_w; - if (y) *y = tga_h; - if (comp) *comp = tga_comp; - return 1; // seems to have passed everything -} - -static int stbi__tga_test(stbi__context *s) -{ - int res = 0; - int sz, tga_color_type; - stbi__get8(s); // discard Offset - tga_color_type = stbi__get8(s); // color type - if ( tga_color_type > 1 ) goto errorEnd; // only RGB or indexed allowed - sz = stbi__get8(s); // image type - if ( tga_color_type == 1 ) { // colormapped (paletted) image - if (sz != 1 && sz != 9) goto errorEnd; // colortype 1 demands image type 1 or 9 - stbi__skip(s,4); // skip index of first colormap entry and number of entries - sz = stbi__get8(s); // check bits per palette color entry - if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) goto errorEnd; - stbi__skip(s,4); // skip image x and y origin - } else { // "normal" image w/o colormap - if ( (sz != 2) && (sz != 3) && (sz != 10) && (sz != 11) ) goto errorEnd; // only RGB or grey allowed, +/- RLE - stbi__skip(s,9); // skip colormap specification and image x/y origin - } - if ( stbi__get16le(s) < 1 ) goto errorEnd; // test width - if ( stbi__get16le(s) < 1 ) goto errorEnd; // test height - sz = stbi__get8(s); // bits per pixel - if ( (tga_color_type == 1) && (sz != 8) && (sz != 16) ) goto errorEnd; // for colormapped images, bpp is size of an index - if ( (sz != 8) && (sz != 15) && (sz != 16) && (sz != 24) && (sz != 32) ) goto errorEnd; - - res = 1; // if we got this far, everything's good and we can return 1 instead of 0 - -errorEnd: - stbi__rewind(s); - return res; -} - -// read 16bit value and convert to 24bit RGB -static void stbi__tga_read_rgb16(stbi__context *s, stbi_uc* out) -{ - stbi__uint16 px = (stbi__uint16)stbi__get16le(s); - stbi__uint16 fiveBitMask = 31; - // we have 3 channels with 5bits each - int r = (px >> 10) & fiveBitMask; - int g = (px >> 5) & fiveBitMask; - int b = px & fiveBitMask; - // Note that this saves the data in RGB(A) order, so it doesn't need to be swapped later - out[0] = (stbi_uc)((r * 255)/31); - out[1] = (stbi_uc)((g * 255)/31); - out[2] = (stbi_uc)((b * 255)/31); - - // some people claim that the most significant bit might be used for alpha - // (possibly if an alpha-bit is set in the "image descriptor byte") - // but that only made 16bit test images completely translucent.. - // so let's treat all 15 and 16bit TGAs as RGB with no alpha. -} - -static void *stbi__tga_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) -{ - // read in the TGA header stuff - int tga_offset = stbi__get8(s); - int tga_indexed = stbi__get8(s); - int tga_image_type = stbi__get8(s); - int tga_is_RLE = 0; - int tga_palette_start = stbi__get16le(s); - int tga_palette_len = stbi__get16le(s); - int tga_palette_bits = stbi__get8(s); - int tga_x_origin = stbi__get16le(s); - int tga_y_origin = stbi__get16le(s); - int tga_width = stbi__get16le(s); - int tga_height = stbi__get16le(s); - int tga_bits_per_pixel = stbi__get8(s); - int tga_comp, tga_rgb16=0; - int tga_inverted = stbi__get8(s); - // int tga_alpha_bits = tga_inverted & 15; // the 4 lowest bits - unused (useless?) - // image data - unsigned char *tga_data; - unsigned char *tga_palette = NULL; - int i, j; - unsigned char raw_data[4] = {0}; - int RLE_count = 0; - int RLE_repeating = 0; - int read_next_pixel = 1; - STBI_NOTUSED(ri); - - // do a tiny bit of precessing - if ( tga_image_type >= 8 ) - { - tga_image_type -= 8; - tga_is_RLE = 1; - } - tga_inverted = 1 - ((tga_inverted >> 5) & 1); - - // If I'm paletted, then I'll use the number of bits from the palette - if ( tga_indexed ) tga_comp = stbi__tga_get_comp(tga_palette_bits, 0, &tga_rgb16); - else tga_comp = stbi__tga_get_comp(tga_bits_per_pixel, (tga_image_type == 3), &tga_rgb16); - - if(!tga_comp) // shouldn't really happen, stbi__tga_test() should have ensured basic consistency - return stbi__errpuc("bad format", "Can't find out TGA pixelformat"); - - // tga info - *x = tga_width; - *y = tga_height; - if (comp) *comp = tga_comp; - - if (!stbi__mad3sizes_valid(tga_width, tga_height, tga_comp, 0)) - return stbi__errpuc("too large", "Corrupt TGA"); - - tga_data = (unsigned char*)stbi__malloc_mad3(tga_width, tga_height, tga_comp, 0); - if (!tga_data) return stbi__errpuc("outofmem", "Out of memory"); - - // skip to the data's starting position (offset usually = 0) - stbi__skip(s, tga_offset ); - - if ( !tga_indexed && !tga_is_RLE && !tga_rgb16 ) { - for (i=0; i < tga_height; ++i) { - int row = tga_inverted ? tga_height -i - 1 : i; - stbi_uc *tga_row = tga_data + row*tga_width*tga_comp; - stbi__getn(s, tga_row, tga_width * tga_comp); - } - } else { - // do I need to load a palette? - if ( tga_indexed) - { - // any data to skip? (offset usually = 0) - stbi__skip(s, tga_palette_start ); - // load the palette - tga_palette = (unsigned char*)stbi__malloc_mad2(tga_palette_len, tga_comp, 0); - if (!tga_palette) { - STBI_FREE(tga_data); - return stbi__errpuc("outofmem", "Out of memory"); - } - if (tga_rgb16) { - stbi_uc *pal_entry = tga_palette; - STBI_ASSERT(tga_comp == STBI_rgb); - for (i=0; i < tga_palette_len; ++i) { - stbi__tga_read_rgb16(s, pal_entry); - pal_entry += tga_comp; - } - } else if (!stbi__getn(s, tga_palette, tga_palette_len * tga_comp)) { - STBI_FREE(tga_data); - STBI_FREE(tga_palette); - return stbi__errpuc("bad palette", "Corrupt TGA"); - } - } - // load the data - for (i=0; i < tga_width * tga_height; ++i) - { - // if I'm in RLE mode, do I need to get a RLE stbi__pngchunk? - if ( tga_is_RLE ) - { - if ( RLE_count == 0 ) - { - // yep, get the next byte as a RLE command - int RLE_cmd = stbi__get8(s); - RLE_count = 1 + (RLE_cmd & 127); - RLE_repeating = RLE_cmd >> 7; - read_next_pixel = 1; - } else if ( !RLE_repeating ) - { - read_next_pixel = 1; - } - } else - { - read_next_pixel = 1; - } - // OK, if I need to read a pixel, do it now - if ( read_next_pixel ) - { - // load however much data we did have - if ( tga_indexed ) - { - // read in index, then perform the lookup - int pal_idx = (tga_bits_per_pixel == 8) ? stbi__get8(s) : stbi__get16le(s); - if ( pal_idx >= tga_palette_len ) { - // invalid index - pal_idx = 0; - } - pal_idx *= tga_comp; - for (j = 0; j < tga_comp; ++j) { - raw_data[j] = tga_palette[pal_idx+j]; - } - } else if(tga_rgb16) { - STBI_ASSERT(tga_comp == STBI_rgb); - stbi__tga_read_rgb16(s, raw_data); - } else { - // read in the data raw - for (j = 0; j < tga_comp; ++j) { - raw_data[j] = stbi__get8(s); - } - } - // clear the reading flag for the next pixel - read_next_pixel = 0; - } // end of reading a pixel - - // copy data - for (j = 0; j < tga_comp; ++j) - tga_data[i*tga_comp+j] = raw_data[j]; - - // in case we're in RLE mode, keep counting down - --RLE_count; - } - // do I need to invert the image? - if ( tga_inverted ) - { - for (j = 0; j*2 < tga_height; ++j) - { - int index1 = j * tga_width * tga_comp; - int index2 = (tga_height - 1 - j) * tga_width * tga_comp; - for (i = tga_width * tga_comp; i > 0; --i) - { - unsigned char temp = tga_data[index1]; - tga_data[index1] = tga_data[index2]; - tga_data[index2] = temp; - ++index1; - ++index2; - } - } - } - // clear my palette, if I had one - if ( tga_palette != NULL ) - { - STBI_FREE( tga_palette ); - } - } - - // swap RGB - if the source data was RGB16, it already is in the right order - if (tga_comp >= 3 && !tga_rgb16) - { - unsigned char* tga_pixel = tga_data; - for (i=0; i < tga_width * tga_height; ++i) - { - unsigned char temp = tga_pixel[0]; - tga_pixel[0] = tga_pixel[2]; - tga_pixel[2] = temp; - tga_pixel += tga_comp; - } - } - - // convert to target component count - if (req_comp && req_comp != tga_comp) - tga_data = stbi__convert_format(tga_data, tga_comp, req_comp, tga_width, tga_height); - - // the things I do to get rid of an error message, and yet keep - // Microsoft's C compilers happy... [8^( - tga_palette_start = tga_palette_len = tga_palette_bits = - tga_x_origin = tga_y_origin = 0; - // OK, done - return tga_data; -} -#endif - -// ************************************************************************************************* -// Photoshop PSD loader -- PD by Thatcher Ulrich, integration by Nicolas Schulz, tweaked by STB - -#ifndef STBI_NO_PSD -static int stbi__psd_test(stbi__context *s) -{ - int r = (stbi__get32be(s) == 0x38425053); - stbi__rewind(s); - return r; -} - -static int stbi__psd_decode_rle(stbi__context *s, stbi_uc *p, int pixelCount) -{ - int count, nleft, len; - - count = 0; - while ((nleft = pixelCount - count) > 0) { - len = stbi__get8(s); - if (len == 128) { - // No-op. - } else if (len < 128) { - // Copy next len+1 bytes literally. - len++; - if (len > nleft) return 0; // corrupt data - count += len; - while (len) { - *p = stbi__get8(s); - p += 4; - len--; - } - } else if (len > 128) { - stbi_uc val; - // Next -len+1 bytes in the dest are replicated from next source byte. - // (Interpret len as a negative 8-bit int.) - len = 257 - len; - if (len > nleft) return 0; // corrupt data - val = stbi__get8(s); - count += len; - while (len) { - *p = val; - p += 4; - len--; - } - } - } - - return 1; -} - -static void *stbi__psd_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri, int bpc) -{ - int pixelCount; - int channelCount, compression; - int channel, i; - int bitdepth; - int w,h; - stbi_uc *out; - STBI_NOTUSED(ri); - - // Check identifier - if (stbi__get32be(s) != 0x38425053) // "8BPS" - return stbi__errpuc("not PSD", "Corrupt PSD image"); - - // Check file type version. - if (stbi__get16be(s) != 1) - return stbi__errpuc("wrong version", "Unsupported version of PSD image"); - - // Skip 6 reserved bytes. - stbi__skip(s, 6 ); - - // Read the number of channels (R, G, B, A, etc). - channelCount = stbi__get16be(s); - if (channelCount < 0 || channelCount > 16) - return stbi__errpuc("wrong channel count", "Unsupported number of channels in PSD image"); - - // Read the rows and columns of the image. - h = stbi__get32be(s); - w = stbi__get32be(s); - - // Make sure the depth is 8 bits. - bitdepth = stbi__get16be(s); - if (bitdepth != 8 && bitdepth != 16) - return stbi__errpuc("unsupported bit depth", "PSD bit depth is not 8 or 16 bit"); - - // Make sure the color mode is RGB. - // Valid options are: - // 0: Bitmap - // 1: Grayscale - // 2: Indexed color - // 3: RGB color - // 4: CMYK color - // 7: Multichannel - // 8: Duotone - // 9: Lab color - if (stbi__get16be(s) != 3) - return stbi__errpuc("wrong color format", "PSD is not in RGB color format"); - - // Skip the Mode Data. (It's the palette for indexed color; other info for other modes.) - stbi__skip(s,stbi__get32be(s) ); - - // Skip the image resources. (resolution, pen tool paths, etc) - stbi__skip(s, stbi__get32be(s) ); - - // Skip the reserved data. - stbi__skip(s, stbi__get32be(s) ); - - // Find out if the data is compressed. - // Known values: - // 0: no compression - // 1: RLE compressed - compression = stbi__get16be(s); - if (compression > 1) - return stbi__errpuc("bad compression", "PSD has an unknown compression format"); - - // Check size - if (!stbi__mad3sizes_valid(4, w, h, 0)) - return stbi__errpuc("too large", "Corrupt PSD"); - - // Create the destination image. - - if (!compression && bitdepth == 16 && bpc == 16) { - out = (stbi_uc *) stbi__malloc_mad3(8, w, h, 0); - ri->bits_per_channel = 16; - } else - out = (stbi_uc *) stbi__malloc(4 * w*h); - - if (!out) return stbi__errpuc("outofmem", "Out of memory"); - pixelCount = w*h; - - // Initialize the data to zero. - //memset( out, 0, pixelCount * 4 ); - - // Finally, the image data. - if (compression) { - // RLE as used by .PSD and .TIFF - // Loop until you get the number of unpacked bytes you are expecting: - // Read the next source byte into n. - // If n is between 0 and 127 inclusive, copy the next n+1 bytes literally. - // Else if n is between -127 and -1 inclusive, copy the next byte -n+1 times. - // Else if n is 128, noop. - // Endloop - - // The RLE-compressed data is preceded by a 2-byte data count for each row in the data, - // which we're going to just skip. - stbi__skip(s, h * channelCount * 2 ); - - // Read the RLE data by channel. - for (channel = 0; channel < 4; channel++) { - stbi_uc *p; - - p = out+channel; - if (channel >= channelCount) { - // Fill this channel with default data. - for (i = 0; i < pixelCount; i++, p += 4) - *p = (channel == 3 ? 255 : 0); - } else { - // Read the RLE data. - if (!stbi__psd_decode_rle(s, p, pixelCount)) { - STBI_FREE(out); - return stbi__errpuc("corrupt", "bad RLE data"); - } - } - } - - } else { - // We're at the raw image data. It's each channel in order (Red, Green, Blue, Alpha, ...) - // where each channel consists of an 8-bit (or 16-bit) value for each pixel in the image. - - // Read the data by channel. - for (channel = 0; channel < 4; channel++) { - if (channel >= channelCount) { - // Fill this channel with default data. - if (bitdepth == 16 && bpc == 16) { - stbi__uint16 *q = ((stbi__uint16 *) out) + channel; - stbi__uint16 val = channel == 3 ? 65535 : 0; - for (i = 0; i < pixelCount; i++, q += 4) - *q = val; - } else { - stbi_uc *p = out+channel; - stbi_uc val = channel == 3 ? 255 : 0; - for (i = 0; i < pixelCount; i++, p += 4) - *p = val; - } - } else { - if (ri->bits_per_channel == 16) { // output bpc - stbi__uint16 *q = ((stbi__uint16 *) out) + channel; - for (i = 0; i < pixelCount; i++, q += 4) - *q = (stbi__uint16) stbi__get16be(s); - } else { - stbi_uc *p = out+channel; - if (bitdepth == 16) { // input bpc - for (i = 0; i < pixelCount; i++, p += 4) - *p = (stbi_uc) (stbi__get16be(s) >> 8); - } else { - for (i = 0; i < pixelCount; i++, p += 4) - *p = stbi__get8(s); - } - } - } - } - } - - // remove weird white matte from PSD - if (channelCount >= 4) { - if (ri->bits_per_channel == 16) { - for (i=0; i < w*h; ++i) { - stbi__uint16 *pixel = (stbi__uint16 *) out + 4*i; - if (pixel[3] != 0 && pixel[3] != 65535) { - float a = pixel[3] / 65535.0f; - float ra = 1.0f / a; - float inv_a = 65535.0f * (1 - ra); - pixel[0] = (stbi__uint16) (pixel[0]*ra + inv_a); - pixel[1] = (stbi__uint16) (pixel[1]*ra + inv_a); - pixel[2] = (stbi__uint16) (pixel[2]*ra + inv_a); - } - } - } else { - for (i=0; i < w*h; ++i) { - unsigned char *pixel = out + 4*i; - if (pixel[3] != 0 && pixel[3] != 255) { - float a = pixel[3] / 255.0f; - float ra = 1.0f / a; - float inv_a = 255.0f * (1 - ra); - pixel[0] = (unsigned char) (pixel[0]*ra + inv_a); - pixel[1] = (unsigned char) (pixel[1]*ra + inv_a); - pixel[2] = (unsigned char) (pixel[2]*ra + inv_a); - } - } - } - } - - // convert to desired output format - if (req_comp && req_comp != 4) { - if (ri->bits_per_channel == 16) - out = (stbi_uc *) stbi__convert_format16((stbi__uint16 *) out, 4, req_comp, w, h); - else - out = stbi__convert_format(out, 4, req_comp, w, h); - if (out == NULL) return out; // stbi__convert_format frees input on failure - } - - if (comp) *comp = 4; - *y = h; - *x = w; - - return out; -} -#endif - -// ************************************************************************************************* -// Softimage PIC loader -// by Tom Seddon -// -// See http://softimage.wiki.softimage.com/index.php/INFO:_PIC_file_format -// See http://ozviz.wasp.uwa.edu.au/~pbourke/dataformats/softimagepic/ - -#ifndef STBI_NO_PIC -static int stbi__pic_is4(stbi__context *s,const char *str) -{ - int i; - for (i=0; i<4; ++i) - if (stbi__get8(s) != (stbi_uc)str[i]) - return 0; - - return 1; -} - -static int stbi__pic_test_core(stbi__context *s) -{ - int i; - - if (!stbi__pic_is4(s,"\x53\x80\xF6\x34")) - return 0; - - for(i=0;i<84;++i) - stbi__get8(s); - - if (!stbi__pic_is4(s,"PICT")) - return 0; - - return 1; -} - -typedef struct -{ - stbi_uc size,type,channel; -} stbi__pic_packet; - -static stbi_uc *stbi__readval(stbi__context *s, int channel, stbi_uc *dest) -{ - int mask=0x80, i; - - for (i=0; i<4; ++i, mask>>=1) { - if (channel & mask) { - if (stbi__at_eof(s)) return stbi__errpuc("bad file","PIC file too short"); - dest[i]=stbi__get8(s); - } - } - - return dest; -} - -static void stbi__copyval(int channel,stbi_uc *dest,const stbi_uc *src) -{ - int mask=0x80,i; - - for (i=0;i<4; ++i, mask>>=1) - if (channel&mask) - dest[i]=src[i]; -} - -static stbi_uc *stbi__pic_load_core(stbi__context *s,int width,int height,int *comp, stbi_uc *result) -{ - int act_comp=0,num_packets=0,y,chained; - stbi__pic_packet packets[10]; - - // this will (should...) cater for even some bizarre stuff like having data - // for the same channel in multiple packets. - do { - stbi__pic_packet *packet; - - if (num_packets==sizeof(packets)/sizeof(packets[0])) - return stbi__errpuc("bad format","too many packets"); - - packet = &packets[num_packets++]; - - chained = stbi__get8(s); - packet->size = stbi__get8(s); - packet->type = stbi__get8(s); - packet->channel = stbi__get8(s); - - act_comp |= packet->channel; - - if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (reading packets)"); - if (packet->size != 8) return stbi__errpuc("bad format","packet isn't 8bpp"); - } while (chained); - - *comp = (act_comp & 0x10 ? 4 : 3); // has alpha channel? - - for(y=0; ytype) { - default: - return stbi__errpuc("bad format","packet has bad compression type"); - - case 0: {//uncompressed - int x; - - for(x=0;xchannel,dest)) - return 0; - break; - } - - case 1://Pure RLE - { - int left=width, i; - - while (left>0) { - stbi_uc count,value[4]; - - count=stbi__get8(s); - if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (pure read count)"); - - if (count > left) - count = (stbi_uc) left; - - if (!stbi__readval(s,packet->channel,value)) return 0; - - for(i=0; ichannel,dest,value); - left -= count; - } - } - break; - - case 2: {//Mixed RLE - int left=width; - while (left>0) { - int count = stbi__get8(s), i; - if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (mixed read count)"); - - if (count >= 128) { // Repeated - stbi_uc value[4]; - - if (count==128) - count = stbi__get16be(s); - else - count -= 127; - if (count > left) - return stbi__errpuc("bad file","scanline overrun"); - - if (!stbi__readval(s,packet->channel,value)) - return 0; - - for(i=0;ichannel,dest,value); - } else { // Raw - ++count; - if (count>left) return stbi__errpuc("bad file","scanline overrun"); - - for(i=0;ichannel,dest)) - return 0; - } - left-=count; - } - break; - } - } - } - } - - return result; -} - -static void *stbi__pic_load(stbi__context *s,int *px,int *py,int *comp,int req_comp, stbi__result_info *ri) -{ - stbi_uc *result; - int i, x,y, internal_comp; - STBI_NOTUSED(ri); - - if (!comp) comp = &internal_comp; - - for (i=0; i<92; ++i) - stbi__get8(s); - - x = stbi__get16be(s); - y = stbi__get16be(s); - if (stbi__at_eof(s)) return stbi__errpuc("bad file","file too short (pic header)"); - if (!stbi__mad3sizes_valid(x, y, 4, 0)) return stbi__errpuc("too large", "PIC image too large to decode"); - - stbi__get32be(s); //skip `ratio' - stbi__get16be(s); //skip `fields' - stbi__get16be(s); //skip `pad' - - // intermediate buffer is RGBA - result = (stbi_uc *) stbi__malloc_mad3(x, y, 4, 0); - memset(result, 0xff, x*y*4); - - if (!stbi__pic_load_core(s,x,y,comp, result)) { - STBI_FREE(result); - result=0; - } - *px = x; - *py = y; - if (req_comp == 0) req_comp = *comp; - result=stbi__convert_format(result,4,req_comp,x,y); - - return result; -} - -static int stbi__pic_test(stbi__context *s) -{ - int r = stbi__pic_test_core(s); - stbi__rewind(s); - return r; -} -#endif - -// ************************************************************************************************* -// GIF loader -- public domain by Jean-Marc Lienher -- simplified/shrunk by stb - -#ifndef STBI_NO_GIF -typedef struct -{ - stbi__int16 prefix; - stbi_uc first; - stbi_uc suffix; -} stbi__gif_lzw; - -typedef struct -{ - int w,h; - stbi_uc *out; // output buffer (always 4 components) - stbi_uc *background; // The current "background" as far as a gif is concerned - stbi_uc *history; - int flags, bgindex, ratio, transparent, eflags; - stbi_uc pal[256][4]; - stbi_uc lpal[256][4]; - stbi__gif_lzw codes[8192]; - stbi_uc *color_table; - int parse, step; - int lflags; - int start_x, start_y; - int max_x, max_y; - int cur_x, cur_y; - int line_size; - int delay; -} stbi__gif; - -static int stbi__gif_test_raw(stbi__context *s) -{ - int sz; - if (stbi__get8(s) != 'G' || stbi__get8(s) != 'I' || stbi__get8(s) != 'F' || stbi__get8(s) != '8') return 0; - sz = stbi__get8(s); - if (sz != '9' && sz != '7') return 0; - if (stbi__get8(s) != 'a') return 0; - return 1; -} - -static int stbi__gif_test(stbi__context *s) -{ - int r = stbi__gif_test_raw(s); - stbi__rewind(s); - return r; -} - -static void stbi__gif_parse_colortable(stbi__context *s, stbi_uc pal[256][4], int num_entries, int transp) -{ - int i; - for (i=0; i < num_entries; ++i) { - pal[i][2] = stbi__get8(s); - pal[i][1] = stbi__get8(s); - pal[i][0] = stbi__get8(s); - pal[i][3] = transp == i ? 0 : 255; - } -} - -static int stbi__gif_header(stbi__context *s, stbi__gif *g, int *comp, int is_info) -{ - stbi_uc version; - if (stbi__get8(s) != 'G' || stbi__get8(s) != 'I' || stbi__get8(s) != 'F' || stbi__get8(s) != '8') - return stbi__err("not GIF", "Corrupt GIF"); - - version = stbi__get8(s); - if (version != '7' && version != '9') return stbi__err("not GIF", "Corrupt GIF"); - if (stbi__get8(s) != 'a') return stbi__err("not GIF", "Corrupt GIF"); - - stbi__g_failure_reason = ""; - g->w = stbi__get16le(s); - g->h = stbi__get16le(s); - g->flags = stbi__get8(s); - g->bgindex = stbi__get8(s); - g->ratio = stbi__get8(s); - g->transparent = -1; - - if (comp != 0) *comp = 4; // can't actually tell whether it's 3 or 4 until we parse the comments - - if (is_info) return 1; - - if (g->flags & 0x80) - stbi__gif_parse_colortable(s,g->pal, 2 << (g->flags & 7), -1); - - return 1; -} - -static int stbi__gif_info_raw(stbi__context *s, int *x, int *y, int *comp) -{ - stbi__gif* g = (stbi__gif*) stbi__malloc(sizeof(stbi__gif)); - if (!stbi__gif_header(s, g, comp, 1)) { - STBI_FREE(g); - stbi__rewind( s ); - return 0; - } - if (x) *x = g->w; - if (y) *y = g->h; - STBI_FREE(g); - return 1; -} - -static void stbi__out_gif_code(stbi__gif *g, stbi__uint16 code) -{ - stbi_uc *p, *c; - int idx; - - // recurse to decode the prefixes, since the linked-list is backwards, - // and working backwards through an interleaved image would be nasty - if (g->codes[code].prefix >= 0) - stbi__out_gif_code(g, g->codes[code].prefix); - - if (g->cur_y >= g->max_y) return; - - idx = g->cur_x + g->cur_y; - p = &g->out[idx]; - g->history[idx / 4] = 1; - - c = &g->color_table[g->codes[code].suffix * 4]; - if (c[3] > 128) { // don't render transparent pixels; - p[0] = c[2]; - p[1] = c[1]; - p[2] = c[0]; - p[3] = c[3]; - } - g->cur_x += 4; - - if (g->cur_x >= g->max_x) { - g->cur_x = g->start_x; - g->cur_y += g->step; - - while (g->cur_y >= g->max_y && g->parse > 0) { - g->step = (1 << g->parse) * g->line_size; - g->cur_y = g->start_y + (g->step >> 1); - --g->parse; - } - } -} - -static stbi_uc *stbi__process_gif_raster(stbi__context *s, stbi__gif *g) -{ - stbi_uc lzw_cs; - stbi__int32 len, init_code; - stbi__uint32 first; - stbi__int32 codesize, codemask, avail, oldcode, bits, valid_bits, clear; - stbi__gif_lzw *p; - - lzw_cs = stbi__get8(s); - if (lzw_cs > 12) return NULL; - clear = 1 << lzw_cs; - first = 1; - codesize = lzw_cs + 1; - codemask = (1 << codesize) - 1; - bits = 0; - valid_bits = 0; - for (init_code = 0; init_code < clear; init_code++) { - g->codes[init_code].prefix = -1; - g->codes[init_code].first = (stbi_uc) init_code; - g->codes[init_code].suffix = (stbi_uc) init_code; - } - - // support no starting clear code - avail = clear+2; - oldcode = -1; - - len = 0; - for(;;) { - if (valid_bits < codesize) { - if (len == 0) { - len = stbi__get8(s); // start new block - if (len == 0) - return g->out; - } - --len; - bits |= (stbi__int32) stbi__get8(s) << valid_bits; - valid_bits += 8; - } else { - stbi__int32 code = bits & codemask; - bits >>= codesize; - valid_bits -= codesize; - // @OPTIMIZE: is there some way we can accelerate the non-clear path? - if (code == clear) { // clear code - codesize = lzw_cs + 1; - codemask = (1 << codesize) - 1; - avail = clear + 2; - oldcode = -1; - first = 0; - } else if (code == clear + 1) { // end of stream code - stbi__skip(s, len); - while ((len = stbi__get8(s)) > 0) - stbi__skip(s,len); - return g->out; - } else if (code <= avail) { - if (first) { - return stbi__errpuc("no clear code", "Corrupt GIF"); - } - - if (oldcode >= 0) { - p = &g->codes[avail++]; - if (avail > 8192) { - return stbi__errpuc("too many codes", "Corrupt GIF"); - } - - p->prefix = (stbi__int16) oldcode; - p->first = g->codes[oldcode].first; - p->suffix = (code == avail) ? p->first : g->codes[code].first; - } else if (code == avail) - return stbi__errpuc("illegal code in raster", "Corrupt GIF"); - - stbi__out_gif_code(g, (stbi__uint16) code); - - if ((avail & codemask) == 0 && avail <= 0x0FFF) { - codesize++; - codemask = (1 << codesize) - 1; - } - - oldcode = code; - } else { - return stbi__errpuc("illegal code in raster", "Corrupt GIF"); - } - } - } -} - -// this function is designed to support animated gifs, although stb_image doesn't support it -// two back is the image from two frames ago, used for a very specific disposal format -static stbi_uc *stbi__gif_load_next(stbi__context *s, stbi__gif *g, int *comp, int req_comp, stbi_uc *two_back) -{ - int dispose; - int first_frame; - int pi; - int pcount; - STBI_NOTUSED(req_comp); - - // on first frame, any non-written pixels get the background colour (non-transparent) - first_frame = 0; - if (g->out == 0) { - if (!stbi__gif_header(s, g, comp,0)) return 0; // stbi__g_failure_reason set by stbi__gif_header - g->out = (stbi_uc *) stbi__malloc(4 * g->w * g->h); - g->background = (stbi_uc *) stbi__malloc(4 * g->w * g->h); - g->history = (stbi_uc *) stbi__malloc(g->w * g->h); - if (g->out == 0) return stbi__errpuc("outofmem", "Out of memory"); - - // image is treated as "transparent" at the start - ie, nothing overwrites the current background; - // background colour is only used for pixels that are not rendered first frame, after that "background" - // color refers to the color that was there the previous frame. - memset( g->out, 0x00, 4 * g->w * g->h ); - memset( g->background, 0x00, 4 * g->w * g->h ); // state of the background (starts transparent) - memset( g->history, 0x00, g->w * g->h ); // pixels that were affected previous frame - first_frame = 1; - } else { - // second frame - how do we dispoase of the previous one? - dispose = (g->eflags & 0x1C) >> 2; - pcount = g->w * g->h; - - if ((dispose == 3) && (two_back == 0)) { - dispose = 2; // if I don't have an image to revert back to, default to the old background - } - - if (dispose == 3) { // use previous graphic - for (pi = 0; pi < pcount; ++pi) { - if (g->history[pi]) { - memcpy( &g->out[pi * 4], &two_back[pi * 4], 4 ); - } - } - } else if (dispose == 2) { - // restore what was changed last frame to background before that frame; - for (pi = 0; pi < pcount; ++pi) { - if (g->history[pi]) { - memcpy( &g->out[pi * 4], &g->background[pi * 4], 4 ); - } - } - } else { - // This is a non-disposal case eithe way, so just - // leave the pixels as is, and they will become the new background - // 1: do not dispose - // 0: not specified. - } - - // background is what out is after the undoing of the previou frame; - memcpy( g->background, g->out, 4 * g->w * g->h ); - } - - // clear my history; - memset( g->history, 0x00, g->w * g->h ); // pixels that were affected previous frame - - for (;;) { - int tag = stbi__get8(s); - switch (tag) { - case 0x2C: /* Image Descriptor */ - { - stbi__int32 x, y, w, h; - stbi_uc *o; - - x = stbi__get16le(s); - y = stbi__get16le(s); - w = stbi__get16le(s); - h = stbi__get16le(s); - if (((x + w) > (g->w)) || ((y + h) > (g->h))) - return stbi__errpuc("bad Image Descriptor", "Corrupt GIF"); - - g->line_size = g->w * 4; - g->start_x = x * 4; - g->start_y = y * g->line_size; - g->max_x = g->start_x + w * 4; - g->max_y = g->start_y + h * g->line_size; - g->cur_x = g->start_x; - g->cur_y = g->start_y; - - g->lflags = stbi__get8(s); - - if (g->lflags & 0x40) { - g->step = 8 * g->line_size; // first interlaced spacing - g->parse = 3; - } else { - g->step = g->line_size; - g->parse = 0; - } - - if (g->lflags & 0x80) { - stbi__gif_parse_colortable(s,g->lpal, 2 << (g->lflags & 7), g->eflags & 0x01 ? g->transparent : -1); - g->color_table = (stbi_uc *) g->lpal; - } else if (g->flags & 0x80) { - g->color_table = (stbi_uc *) g->pal; - } else - return stbi__errpuc("missing color table", "Corrupt GIF"); - - o = stbi__process_gif_raster(s, g); - if (o == NULL) return NULL; - - // if this was the first frame, - pcount = g->w * g->h; - if (first_frame && (g->bgindex > 0)) { - // if first frame, any pixel not drawn to gets the background color - for (pi = 0; pi < pcount; ++pi) { - if (g->history[pi] == 0) { - g->pal[g->bgindex][3] = 255; // just in case it was made transparent, undo that; It will be reset next frame if need be; - memcpy( &g->out[pi * 4], &g->pal[g->bgindex], 4 ); - } - } - } - - return o; - } - - case 0x21: // Comment Extension. - { - int len; - int ext = stbi__get8(s); - if (ext == 0xF9) { // Graphic Control Extension. - len = stbi__get8(s); - if (len == 4) { - g->eflags = stbi__get8(s); - g->delay = 10 * stbi__get16le(s); // delay - 1/100th of a second, saving as 1/1000ths. - - // unset old transparent - if (g->transparent >= 0) { - g->pal[g->transparent][3] = 255; - } - if (g->eflags & 0x01) { - g->transparent = stbi__get8(s); - if (g->transparent >= 0) { - g->pal[g->transparent][3] = 0; - } - } else { - // don't need transparent - stbi__skip(s, 1); - g->transparent = -1; - } - } else { - stbi__skip(s, len); - break; - } - } - while ((len = stbi__get8(s)) != 0) { - stbi__skip(s, len); - } - break; - } - - case 0x3B: // gif stream termination code - return (stbi_uc *) s; // using '1' causes warning on some compilers - - default: - return stbi__errpuc("unknown code", "Corrupt GIF"); - } - } -} - -static void *stbi__load_gif_main(stbi__context *s, int **delays, int *x, int *y, int *z, int *comp, int req_comp) -{ - if (stbi__gif_test(s)) { - int layers = 0; - stbi_uc *u = 0; - stbi_uc *out = 0; - stbi_uc *two_back = 0; - stbi__gif g; - int stride; - memset(&g, 0, sizeof(g)); - if (delays) { - *delays = 0; - } - - do { - u = stbi__gif_load_next(s, &g, comp, req_comp, two_back); - if (u == (stbi_uc *) s) u = 0; // end of animated gif marker - - if (u) { - *x = g.w; - *y = g.h; - ++layers; - stride = g.w * g.h * 4; - - if (out) { - out = (stbi_uc*) STBI_REALLOC( out, layers * stride ); - if (delays) { - *delays = (int*) STBI_REALLOC( *delays, sizeof(int) * layers ); - } - } else { - out = (stbi_uc*)stbi__malloc( layers * stride ); - if (delays) { - *delays = (int*) stbi__malloc( layers * sizeof(int) ); - } - } - memcpy( out + ((layers - 1) * stride), u, stride ); - if (layers >= 2) { - two_back = out - 2 * stride; - } - - if (delays) { - (*delays)[layers - 1U] = g.delay; - } - } - } while (u != 0); - - // free temp buffer; - STBI_FREE(g.out); - STBI_FREE(g.history); - STBI_FREE(g.background); - - // do the final conversion after loading everything; - if (req_comp && req_comp != 4) - out = stbi__convert_format(out, 4, req_comp, layers * g.w, g.h); - - *z = layers; - return out; - } else { - return stbi__errpuc("not GIF", "Image was not as a gif type."); - } -} - -static void *stbi__gif_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) -{ - stbi_uc *u = 0; - stbi__gif g; - memset(&g, 0, sizeof(g)); - STBI_NOTUSED(ri); - - u = stbi__gif_load_next(s, &g, comp, req_comp, 0); - if (u == (stbi_uc *) s) u = 0; // end of animated gif marker - if (u) { - *x = g.w; - *y = g.h; - - // moved conversion to after successful load so that the same - // can be done for multiple frames. - if (req_comp && req_comp != 4) - u = stbi__convert_format(u, 4, req_comp, g.w, g.h); - } - - // free buffers needed for multiple frame loading; - STBI_FREE(g.history); - STBI_FREE(g.background); - - return u; -} - -static int stbi__gif_info(stbi__context *s, int *x, int *y, int *comp) -{ - return stbi__gif_info_raw(s,x,y,comp); -} -#endif - -// ************************************************************************************************* -// Radiance RGBE HDR loader -// originally by Nicolas Schulz -#ifndef STBI_NO_HDR -static int stbi__hdr_test_core(stbi__context *s, const char *signature) -{ - int i; - for (i=0; signature[i]; ++i) - if (stbi__get8(s) != signature[i]) - return 0; - stbi__rewind(s); - return 1; -} - -static int stbi__hdr_test(stbi__context* s) -{ - int r = stbi__hdr_test_core(s, "#?RADIANCE\n"); - stbi__rewind(s); - if(!r) { - r = stbi__hdr_test_core(s, "#?RGBE\n"); - stbi__rewind(s); - } - return r; -} - -#define STBI__HDR_BUFLEN 1024 -static char *stbi__hdr_gettoken(stbi__context *z, char *buffer) -{ - int len=0; - char c = '\0'; - - c = (char) stbi__get8(z); - - while (!stbi__at_eof(z) && c != '\n') { - buffer[len++] = c; - if (len == STBI__HDR_BUFLEN-1) { - // flush to end of line - while (!stbi__at_eof(z) && stbi__get8(z) != '\n') - ; - break; - } - c = (char) stbi__get8(z); - } - - buffer[len] = 0; - return buffer; -} - -static void stbi__hdr_convert(float *output, stbi_uc *input, int req_comp) -{ - if ( input[3] != 0 ) { - float f1; - // Exponent - f1 = (float) ldexp(1.0f, input[3] - (int)(128 + 8)); - if (req_comp <= 2) - output[0] = (input[0] + input[1] + input[2]) * f1 / 3; - else { - output[0] = input[0] * f1; - output[1] = input[1] * f1; - output[2] = input[2] * f1; - } - if (req_comp == 2) output[1] = 1; - if (req_comp == 4) output[3] = 1; - } else { - switch (req_comp) { - case 4: output[3] = 1; /* fallthrough */ - case 3: output[0] = output[1] = output[2] = 0; - break; - case 2: output[1] = 1; /* fallthrough */ - case 1: output[0] = 0; - break; - } - } -} - -static float *stbi__hdr_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) -{ - char buffer[STBI__HDR_BUFLEN]; - char *token; - int valid = 0; - int width, height; - stbi_uc *scanline; - float *hdr_data; - int len; - unsigned char count, value; - int i, j, k, c1,c2, z; - const char *headerToken; - STBI_NOTUSED(ri); - - // Check identifier - headerToken = stbi__hdr_gettoken(s,buffer); - if (strcmp(headerToken, "#?RADIANCE") != 0 && strcmp(headerToken, "#?RGBE") != 0) - return stbi__errpf("not HDR", "Corrupt HDR image"); - - // Parse header - for(;;) { - token = stbi__hdr_gettoken(s,buffer); - if (token[0] == 0) break; - if (strcmp(token, "FORMAT=32-bit_rle_rgbe") == 0) valid = 1; - } - - if (!valid) return stbi__errpf("unsupported format", "Unsupported HDR format"); - - // Parse width and height - // can't use sscanf() if we're not using stdio! - token = stbi__hdr_gettoken(s,buffer); - if (strncmp(token, "-Y ", 3)) return stbi__errpf("unsupported data layout", "Unsupported HDR format"); - token += 3; - height = (int) strtol(token, &token, 10); - while (*token == ' ') ++token; - if (strncmp(token, "+X ", 3)) return stbi__errpf("unsupported data layout", "Unsupported HDR format"); - token += 3; - width = (int) strtol(token, NULL, 10); - - *x = width; - *y = height; - - if (comp) *comp = 3; - if (req_comp == 0) req_comp = 3; - - if (!stbi__mad4sizes_valid(width, height, req_comp, sizeof(float), 0)) - return stbi__errpf("too large", "HDR image is too large"); - - // Read data - hdr_data = (float *) stbi__malloc_mad4(width, height, req_comp, sizeof(float), 0); - if (!hdr_data) - return stbi__errpf("outofmem", "Out of memory"); - - // Load image data - // image data is stored as some number of sca - if ( width < 8 || width >= 32768) { - // Read flat data - for (j=0; j < height; ++j) { - for (i=0; i < width; ++i) { - stbi_uc rgbe[4]; - main_decode_loop: - stbi__getn(s, rgbe, 4); - stbi__hdr_convert(hdr_data + j * width * req_comp + i * req_comp, rgbe, req_comp); - } - } - } else { - // Read RLE-encoded data - scanline = NULL; - - for (j = 0; j < height; ++j) { - c1 = stbi__get8(s); - c2 = stbi__get8(s); - len = stbi__get8(s); - if (c1 != 2 || c2 != 2 || (len & 0x80)) { - // not run-length encoded, so we have to actually use THIS data as a decoded - // pixel (note this can't be a valid pixel--one of RGB must be >= 128) - stbi_uc rgbe[4]; - rgbe[0] = (stbi_uc) c1; - rgbe[1] = (stbi_uc) c2; - rgbe[2] = (stbi_uc) len; - rgbe[3] = (stbi_uc) stbi__get8(s); - stbi__hdr_convert(hdr_data, rgbe, req_comp); - i = 1; - j = 0; - STBI_FREE(scanline); - goto main_decode_loop; // yes, this makes no sense - } - len <<= 8; - len |= stbi__get8(s); - if (len != width) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf("invalid decoded scanline length", "corrupt HDR"); } - if (scanline == NULL) { - scanline = (stbi_uc *) stbi__malloc_mad2(width, 4, 0); - if (!scanline) { - STBI_FREE(hdr_data); - return stbi__errpf("outofmem", "Out of memory"); - } - } - - for (k = 0; k < 4; ++k) { - int nleft; - i = 0; - while ((nleft = width - i) > 0) { - count = stbi__get8(s); - if (count > 128) { - // Run - value = stbi__get8(s); - count -= 128; - if (count > nleft) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf("corrupt", "bad RLE data in HDR"); } - for (z = 0; z < count; ++z) - scanline[i++ * 4 + k] = value; - } else { - // Dump - if (count > nleft) { STBI_FREE(hdr_data); STBI_FREE(scanline); return stbi__errpf("corrupt", "bad RLE data in HDR"); } - for (z = 0; z < count; ++z) - scanline[i++ * 4 + k] = stbi__get8(s); - } - } - } - for (i=0; i < width; ++i) - stbi__hdr_convert(hdr_data+(j*width + i)*req_comp, scanline + i*4, req_comp); - } - if (scanline) - STBI_FREE(scanline); - } - - return hdr_data; -} - -static int stbi__hdr_info(stbi__context *s, int *x, int *y, int *comp) -{ - char buffer[STBI__HDR_BUFLEN]; - char *token; - int valid = 0; - int dummy; - - if (!x) x = &dummy; - if (!y) y = &dummy; - if (!comp) comp = &dummy; - - if (stbi__hdr_test(s) == 0) { - stbi__rewind( s ); - return 0; - } - - for(;;) { - token = stbi__hdr_gettoken(s,buffer); - if (token[0] == 0) break; - if (strcmp(token, "FORMAT=32-bit_rle_rgbe") == 0) valid = 1; - } - - if (!valid) { - stbi__rewind( s ); - return 0; - } - token = stbi__hdr_gettoken(s,buffer); - if (strncmp(token, "-Y ", 3)) { - stbi__rewind( s ); - return 0; - } - token += 3; - *y = (int) strtol(token, &token, 10); - while (*token == ' ') ++token; - if (strncmp(token, "+X ", 3)) { - stbi__rewind( s ); - return 0; - } - token += 3; - *x = (int) strtol(token, NULL, 10); - *comp = 3; - return 1; -} -#endif // STBI_NO_HDR - -#ifndef STBI_NO_BMP -static int stbi__bmp_info(stbi__context *s, int *x, int *y, int *comp) -{ - void *p; - stbi__bmp_data info; - - info.all_a = 255; - p = stbi__bmp_parse_header(s, &info); - stbi__rewind( s ); - if (p == NULL) - return 0; - if (x) *x = s->img_x; - if (y) *y = s->img_y; - if (comp) *comp = info.ma ? 4 : 3; - return 1; -} -#endif - -#ifndef STBI_NO_PSD -static int stbi__psd_info(stbi__context *s, int *x, int *y, int *comp) -{ - int channelCount, dummy, depth; - if (!x) x = &dummy; - if (!y) y = &dummy; - if (!comp) comp = &dummy; - if (stbi__get32be(s) != 0x38425053) { - stbi__rewind( s ); - return 0; - } - if (stbi__get16be(s) != 1) { - stbi__rewind( s ); - return 0; - } - stbi__skip(s, 6); - channelCount = stbi__get16be(s); - if (channelCount < 0 || channelCount > 16) { - stbi__rewind( s ); - return 0; - } - *y = stbi__get32be(s); - *x = stbi__get32be(s); - depth = stbi__get16be(s); - if (depth != 8 && depth != 16) { - stbi__rewind( s ); - return 0; - } - if (stbi__get16be(s) != 3) { - stbi__rewind( s ); - return 0; - } - *comp = 4; - return 1; -} - -static int stbi__psd_is16(stbi__context *s) -{ - int channelCount, depth; - if (stbi__get32be(s) != 0x38425053) { - stbi__rewind( s ); - return 0; - } - if (stbi__get16be(s) != 1) { - stbi__rewind( s ); - return 0; - } - stbi__skip(s, 6); - channelCount = stbi__get16be(s); - if (channelCount < 0 || channelCount > 16) { - stbi__rewind( s ); - return 0; - } - (void) stbi__get32be(s); - (void) stbi__get32be(s); - depth = stbi__get16be(s); - if (depth != 16) { - stbi__rewind( s ); - return 0; - } - return 1; -} -#endif - -#ifndef STBI_NO_PIC -static int stbi__pic_info(stbi__context *s, int *x, int *y, int *comp) -{ - int act_comp=0,num_packets=0,chained,dummy; - stbi__pic_packet packets[10]; - - if (!x) x = &dummy; - if (!y) y = &dummy; - if (!comp) comp = &dummy; - - if (!stbi__pic_is4(s,"\x53\x80\xF6\x34")) { - stbi__rewind(s); - return 0; - } - - stbi__skip(s, 88); - - *x = stbi__get16be(s); - *y = stbi__get16be(s); - if (stbi__at_eof(s)) { - stbi__rewind( s); - return 0; - } - if ( (*x) != 0 && (1 << 28) / (*x) < (*y)) { - stbi__rewind( s ); - return 0; - } - - stbi__skip(s, 8); - - do { - stbi__pic_packet *packet; - - if (num_packets==sizeof(packets)/sizeof(packets[0])) - return 0; - - packet = &packets[num_packets++]; - chained = stbi__get8(s); - packet->size = stbi__get8(s); - packet->type = stbi__get8(s); - packet->channel = stbi__get8(s); - act_comp |= packet->channel; - - if (stbi__at_eof(s)) { - stbi__rewind( s ); - return 0; - } - if (packet->size != 8) { - stbi__rewind( s ); - return 0; - } - } while (chained); - - *comp = (act_comp & 0x10 ? 4 : 3); - - return 1; -} -#endif - -// ************************************************************************************************* -// Portable Gray Map and Portable Pixel Map loader -// by Ken Miller -// -// PGM: http://netpbm.sourceforge.net/doc/pgm.html -// PPM: http://netpbm.sourceforge.net/doc/ppm.html -// -// Known limitations: -// Does not support comments in the header section -// Does not support ASCII image data (formats P2 and P3) -// Does not support 16-bit-per-channel - -#ifndef STBI_NO_PNM - -static int stbi__pnm_test(stbi__context *s) -{ - char p, t; - p = (char) stbi__get8(s); - t = (char) stbi__get8(s); - if (p != 'P' || (t != '5' && t != '6')) { - stbi__rewind( s ); - return 0; - } - return 1; -} - -static void *stbi__pnm_load(stbi__context *s, int *x, int *y, int *comp, int req_comp, stbi__result_info *ri) -{ - stbi_uc *out; - STBI_NOTUSED(ri); - - if (!stbi__pnm_info(s, (int *)&s->img_x, (int *)&s->img_y, (int *)&s->img_n)) - return 0; - - *x = s->img_x; - *y = s->img_y; - if (comp) *comp = s->img_n; - - if (!stbi__mad3sizes_valid(s->img_n, s->img_x, s->img_y, 0)) - return stbi__errpuc("too large", "PNM too large"); - - out = (stbi_uc *) stbi__malloc_mad3(s->img_n, s->img_x, s->img_y, 0); - if (!out) return stbi__errpuc("outofmem", "Out of memory"); - stbi__getn(s, out, s->img_n * s->img_x * s->img_y); - - if (req_comp && req_comp != s->img_n) { - out = stbi__convert_format(out, s->img_n, req_comp, s->img_x, s->img_y); - if (out == NULL) return out; // stbi__convert_format frees input on failure - } - return out; -} - -static int stbi__pnm_isspace(char c) -{ - return c == ' ' || c == '\t' || c == '\n' || c == '\v' || c == '\f' || c == '\r'; -} - -static void stbi__pnm_skip_whitespace(stbi__context *s, char *c) -{ - for (;;) { - while (!stbi__at_eof(s) && stbi__pnm_isspace(*c)) - *c = (char) stbi__get8(s); - - if (stbi__at_eof(s) || *c != '#') - break; - - while (!stbi__at_eof(s) && *c != '\n' && *c != '\r' ) - *c = (char) stbi__get8(s); - } -} - -static int stbi__pnm_isdigit(char c) -{ - return c >= '0' && c <= '9'; -} - -static int stbi__pnm_getinteger(stbi__context *s, char *c) -{ - int value = 0; - - while (!stbi__at_eof(s) && stbi__pnm_isdigit(*c)) { - value = value*10 + (*c - '0'); - *c = (char) stbi__get8(s); - } - - return value; -} - -static int stbi__pnm_info(stbi__context *s, int *x, int *y, int *comp) -{ - int maxv, dummy; - char c, p, t; - - if (!x) x = &dummy; - if (!y) y = &dummy; - if (!comp) comp = &dummy; - - stbi__rewind(s); - - // Get identifier - p = (char) stbi__get8(s); - t = (char) stbi__get8(s); - if (p != 'P' || (t != '5' && t != '6')) { - stbi__rewind(s); - return 0; - } - - *comp = (t == '6') ? 3 : 1; // '5' is 1-component .pgm; '6' is 3-component .ppm - - c = (char) stbi__get8(s); - stbi__pnm_skip_whitespace(s, &c); - - *x = stbi__pnm_getinteger(s, &c); // read width - stbi__pnm_skip_whitespace(s, &c); - - *y = stbi__pnm_getinteger(s, &c); // read height - stbi__pnm_skip_whitespace(s, &c); - - maxv = stbi__pnm_getinteger(s, &c); // read max value - - if (maxv > 255) - return stbi__err("max value > 255", "PPM image not 8-bit"); - else - return 1; -} -#endif - -static int stbi__info_main(stbi__context *s, int *x, int *y, int *comp) -{ - #ifndef STBI_NO_JPEG - if (stbi__jpeg_info(s, x, y, comp)) return 1; - #endif - - #ifndef STBI_NO_PNG - if (stbi__png_info(s, x, y, comp)) return 1; - #endif - - #ifndef STBI_NO_GIF - if (stbi__gif_info(s, x, y, comp)) return 1; - #endif - - #ifndef STBI_NO_BMP - if (stbi__bmp_info(s, x, y, comp)) return 1; - #endif - - #ifndef STBI_NO_PSD - if (stbi__psd_info(s, x, y, comp)) return 1; - #endif - - #ifndef STBI_NO_PIC - if (stbi__pic_info(s, x, y, comp)) return 1; - #endif - - #ifndef STBI_NO_PNM - if (stbi__pnm_info(s, x, y, comp)) return 1; - #endif - - #ifndef STBI_NO_HDR - if (stbi__hdr_info(s, x, y, comp)) return 1; - #endif - - // test tga last because it's a crappy test! - #ifndef STBI_NO_TGA - if (stbi__tga_info(s, x, y, comp)) - return 1; - #endif - return stbi__err("unknown image type", "Image not of any known type, or corrupt"); -} - -static int stbi__is_16_main(stbi__context *s) -{ - #ifndef STBI_NO_PNG - if (stbi__png_is16(s)) return 1; - #endif - - #ifndef STBI_NO_PSD - if (stbi__psd_is16(s)) return 1; - #endif - - return 0; -} - -#ifndef STBI_NO_STDIO -STBIDEF int stbi_info(char const *filename, int *x, int *y, int *comp) -{ - FILE *f = stbi__fopen(filename, "rb"); - int result; - if (!f) return stbi__err("can't fopen", "Unable to open file"); - result = stbi_info_from_file(f, x, y, comp); - fclose(f); - return result; -} - -STBIDEF int stbi_info_from_file(FILE *f, int *x, int *y, int *comp) -{ - int r; - stbi__context s; - long pos = ftell(f); - stbi__start_file(&s, f); - r = stbi__info_main(&s,x,y,comp); - fseek(f,pos,SEEK_SET); - return r; -} - -STBIDEF int stbi_is_16_bit(char const *filename) -{ - FILE *f = stbi__fopen(filename, "rb"); - int result; - if (!f) return stbi__err("can't fopen", "Unable to open file"); - result = stbi_is_16_bit_from_file(f); - fclose(f); - return result; -} - -STBIDEF int stbi_is_16_bit_from_file(FILE *f) -{ - int r; - stbi__context s; - long pos = ftell(f); - stbi__start_file(&s, f); - r = stbi__is_16_main(&s); - fseek(f,pos,SEEK_SET); - return r; -} -#endif // !STBI_NO_STDIO - -STBIDEF int stbi_info_from_memory(stbi_uc const *buffer, int len, int *x, int *y, int *comp) -{ - stbi__context s; - stbi__start_mem(&s,buffer,len); - return stbi__info_main(&s,x,y,comp); -} - -STBIDEF int stbi_info_from_callbacks(stbi_io_callbacks const *c, void *user, int *x, int *y, int *comp) -{ - stbi__context s; - stbi__start_callbacks(&s, (stbi_io_callbacks *) c, user); - return stbi__info_main(&s,x,y,comp); -} - -STBIDEF int stbi_is_16_bit_from_memory(stbi_uc const *buffer, int len) -{ - stbi__context s; - stbi__start_mem(&s,buffer,len); - return stbi__is_16_main(&s); -} - -STBIDEF int stbi_is_16_bit_from_callbacks(stbi_io_callbacks const *c, void *user) -{ - stbi__context s; - stbi__start_callbacks(&s, (stbi_io_callbacks *) c, user); - return stbi__is_16_main(&s); -} - -#endif // STB_IMAGE_IMPLEMENTATION - -/* - revision history: - 2.20 (2019-02-07) support utf8 filenames in Windows; fix warnings and platform ifdefs - 2.19 (2018-02-11) fix warning - 2.18 (2018-01-30) fix warnings - 2.17 (2018-01-29) change sbti__shiftsigned to avoid clang -O2 bug - 1-bit BMP - *_is_16_bit api - avoid warnings - 2.16 (2017-07-23) all functions have 16-bit variants; - STBI_NO_STDIO works again; - compilation fixes; - fix rounding in unpremultiply; - optimize vertical flip; - disable raw_len validation; - documentation fixes - 2.15 (2017-03-18) fix png-1,2,4 bug; now all Imagenet JPGs decode; - warning fixes; disable run-time SSE detection on gcc; - uniform handling of optional "return" values; - thread-safe initialization of zlib tables - 2.14 (2017-03-03) remove deprecated STBI_JPEG_OLD; fixes for Imagenet JPGs - 2.13 (2016-11-29) add 16-bit API, only supported for PNG right now - 2.12 (2016-04-02) fix typo in 2.11 PSD fix that caused crashes - 2.11 (2016-04-02) allocate large structures on the stack - remove white matting for transparent PSD - fix reported channel count for PNG & BMP - re-enable SSE2 in non-gcc 64-bit - support RGB-formatted JPEG - read 16-bit PNGs (only as 8-bit) - 2.10 (2016-01-22) avoid warning introduced in 2.09 by STBI_REALLOC_SIZED - 2.09 (2016-01-16) allow comments in PNM files - 16-bit-per-pixel TGA (not bit-per-component) - info() for TGA could break due to .hdr handling - info() for BMP to shares code instead of sloppy parse - can use STBI_REALLOC_SIZED if allocator doesn't support realloc - code cleanup - 2.08 (2015-09-13) fix to 2.07 cleanup, reading RGB PSD as RGBA - 2.07 (2015-09-13) fix compiler warnings - partial animated GIF support - limited 16-bpc PSD support - #ifdef unused functions - bug with < 92 byte PIC,PNM,HDR,TGA - 2.06 (2015-04-19) fix bug where PSD returns wrong '*comp' value - 2.05 (2015-04-19) fix bug in progressive JPEG handling, fix warning - 2.04 (2015-04-15) try to re-enable SIMD on MinGW 64-bit - 2.03 (2015-04-12) extra corruption checking (mmozeiko) - stbi_set_flip_vertically_on_load (nguillemot) - fix NEON support; fix mingw support - 2.02 (2015-01-19) fix incorrect assert, fix warning - 2.01 (2015-01-17) fix various warnings; suppress SIMD on gcc 32-bit without -msse2 - 2.00b (2014-12-25) fix STBI_MALLOC in progressive JPEG - 2.00 (2014-12-25) optimize JPG, including x86 SSE2 & NEON SIMD (ryg) - progressive JPEG (stb) - PGM/PPM support (Ken Miller) - STBI_MALLOC,STBI_REALLOC,STBI_FREE - GIF bugfix -- seemingly never worked - STBI_NO_*, STBI_ONLY_* - 1.48 (2014-12-14) fix incorrectly-named assert() - 1.47 (2014-12-14) 1/2/4-bit PNG support, both direct and paletted (Omar Cornut & stb) - optimize PNG (ryg) - fix bug in interlaced PNG with user-specified channel count (stb) - 1.46 (2014-08-26) - fix broken tRNS chunk (colorkey-style transparency) in non-paletted PNG - 1.45 (2014-08-16) - fix MSVC-ARM internal compiler error by wrapping malloc - 1.44 (2014-08-07) - various warning fixes from Ronny Chevalier - 1.43 (2014-07-15) - fix MSVC-only compiler problem in code changed in 1.42 - 1.42 (2014-07-09) - don't define _CRT_SECURE_NO_WARNINGS (affects user code) - fixes to stbi__cleanup_jpeg path - added STBI_ASSERT to avoid requiring assert.h - 1.41 (2014-06-25) - fix search&replace from 1.36 that messed up comments/error messages - 1.40 (2014-06-22) - fix gcc struct-initialization warning - 1.39 (2014-06-15) - fix to TGA optimization when req_comp != number of components in TGA; - fix to GIF loading because BMP wasn't rewinding (whoops, no GIFs in my test suite) - add support for BMP version 5 (more ignored fields) - 1.38 (2014-06-06) - suppress MSVC warnings on integer casts truncating values - fix accidental rename of 'skip' field of I/O - 1.37 (2014-06-04) - remove duplicate typedef - 1.36 (2014-06-03) - convert to header file single-file library - if de-iphone isn't set, load iphone images color-swapped instead of returning NULL - 1.35 (2014-05-27) - various warnings - fix broken STBI_SIMD path - fix bug where stbi_load_from_file no longer left file pointer in correct place - fix broken non-easy path for 32-bit BMP (possibly never used) - TGA optimization by Arseny Kapoulkine - 1.34 (unknown) - use STBI_NOTUSED in stbi__resample_row_generic(), fix one more leak in tga failure case - 1.33 (2011-07-14) - make stbi_is_hdr work in STBI_NO_HDR (as specified), minor compiler-friendly improvements - 1.32 (2011-07-13) - support for "info" function for all supported filetypes (SpartanJ) - 1.31 (2011-06-20) - a few more leak fixes, bug in PNG handling (SpartanJ) - 1.30 (2011-06-11) - added ability to load files via callbacks to accomidate custom input streams (Ben Wenger) - removed deprecated format-specific test/load functions - removed support for installable file formats (stbi_loader) -- would have been broken for IO callbacks anyway - error cases in bmp and tga give messages and don't leak (Raymond Barbiero, grisha) - fix inefficiency in decoding 32-bit BMP (David Woo) - 1.29 (2010-08-16) - various warning fixes from Aurelien Pocheville - 1.28 (2010-08-01) - fix bug in GIF palette transparency (SpartanJ) - 1.27 (2010-08-01) - cast-to-stbi_uc to fix warnings - 1.26 (2010-07-24) - fix bug in file buffering for PNG reported by SpartanJ - 1.25 (2010-07-17) - refix trans_data warning (Won Chun) - 1.24 (2010-07-12) - perf improvements reading from files on platforms with lock-heavy fgetc() - minor perf improvements for jpeg - deprecated type-specific functions so we'll get feedback if they're needed - attempt to fix trans_data warning (Won Chun) - 1.23 fixed bug in iPhone support - 1.22 (2010-07-10) - removed image *writing* support - stbi_info support from Jetro Lauha - GIF support from Jean-Marc Lienher - iPhone PNG-extensions from James Brown - warning-fixes from Nicolas Schulz and Janez Zemva (i.stbi__err. Janez (U+017D)emva) - 1.21 fix use of 'stbi_uc' in header (reported by jon blow) - 1.20 added support for Softimage PIC, by Tom Seddon - 1.19 bug in interlaced PNG corruption check (found by ryg) - 1.18 (2008-08-02) - fix a threading bug (local mutable static) - 1.17 support interlaced PNG - 1.16 major bugfix - stbi__convert_format converted one too many pixels - 1.15 initialize some fields for thread safety - 1.14 fix threadsafe conversion bug - header-file-only version (#define STBI_HEADER_FILE_ONLY before including) - 1.13 threadsafe - 1.12 const qualifiers in the API - 1.11 Support installable IDCT, colorspace conversion routines - 1.10 Fixes for 64-bit (don't use "unsigned long") - optimized upsampling by Fabian "ryg" Giesen - 1.09 Fix format-conversion for PSD code (bad global variables!) - 1.08 Thatcher Ulrich's PSD code integrated by Nicolas Schulz - 1.07 attempt to fix C++ warning/errors again - 1.06 attempt to fix C++ warning/errors again - 1.05 fix TGA loading to return correct *comp and use good luminance calc - 1.04 default float alpha is 1, not 255; use 'void *' for stbi_image_free - 1.03 bugfixes to STBI_NO_STDIO, STBI_NO_HDR - 1.02 support for (subset of) HDR files, float interface for preferred access to them - 1.01 fix bug: possible bug in handling right-side up bmps... not sure - fix bug: the stbi__bmp_load() and stbi__tga_load() functions didn't work at all - 1.00 interface to zlib that skips zlib header - 0.99 correct handling of alpha in palette - 0.98 TGA loader by lonesock; dynamically add loaders (untested) - 0.97 jpeg errors on too large a file; also catch another malloc failure - 0.96 fix detection of invalid v value - particleman@mollyrocket forum - 0.95 during header scan, seek to markers in case of padding - 0.94 STBI_NO_STDIO to disable stdio usage; rename all #defines the same - 0.93 handle jpegtran output; verbose errors - 0.92 read 4,8,16,24,32-bit BMP files of several formats - 0.91 output 24-bit Windows 3.0 BMP files - 0.90 fix a few more warnings; bump version number to approach 1.0 - 0.61 bugfixes due to Marc LeBlanc, Christopher Lloyd - 0.60 fix compiling as c++ - 0.59 fix warnings: merge Dave Moore's -Wall fixes - 0.58 fix bug: zlib uncompressed mode len/nlen was wrong endian - 0.57 fix bug: jpg last huffman symbol before marker was >9 bits but less than 16 available - 0.56 fix bug: zlib uncompressed mode len vs. nlen - 0.55 fix bug: restart_interval not initialized to 0 - 0.54 allow NULL for 'int *comp' - 0.53 fix bug in png 3->4; speedup png decoding - 0.52 png handles req_comp=3,4 directly; minor cleanup; jpeg comments - 0.51 obey req_comp requests, 1-component jpegs return as 1-component, - on 'test' only check type, not whether we support this variant - 0.50 (2006-11-19) - first released version -*/ - - -/* ------------------------------------------------------------------------------- -This software is available under 2 licenses -- choose whichever you prefer. ------------------------------------------------------------------------------- -ALTERNATIVE A - MIT License -Copyright (c) 2017 Sean Barrett -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. ------------------------------------------------------------------------------- -ALTERNATIVE B - Public Domain (www.unlicense.org) -This is free and unencumbered software released into the public domain. -Anyone is free to copy, modify, publish, use, compile, sell, or distribute this -software, either in source code form or as a compiled binary, for any purpose, -commercial or non-commercial, and by any means. -In jurisdictions that recognize copyright laws, the author or authors of this -software dedicate any and all copyright interest in the software to the public -domain. We make this dedication for the benefit of the public at large and to -the detriment of our heirs and successors. We intend this dedication to be an -overt act of relinquishment in perpetuity of all present and future rights to -this software under copyright law. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------------------------------------------------------------------------------- -*/ diff --git a/v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image_resize.h b/v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image_resize.h deleted file mode 100644 index 031ca99dc3..0000000000 --- a/v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image_resize.h +++ /dev/null @@ -1,2627 +0,0 @@ -/* stb_image_resize - v0.95 - public domain image resizing - by Jorge L Rodriguez (@VinoBS) - 2014 - http://github.com/nothings/stb - - Written with emphasis on usability, portability, and efficiency. (No - SIMD or threads, so it be easily outperformed by libs that use those.) - Only scaling and translation is supported, no rotations or shears. - Easy API downsamples w/Mitchell filter, upsamples w/cubic interpolation. - - COMPILING & LINKING - In one C/C++ file that #includes this file, do this: - #define STB_IMAGE_RESIZE_IMPLEMENTATION - before the #include. That will create the implementation in that file. - - QUICKSTART - stbir_resize_uint8( input_pixels , in_w , in_h , 0, - output_pixels, out_w, out_h, 0, num_channels) - stbir_resize_float(...) - stbir_resize_uint8_srgb( input_pixels , in_w , in_h , 0, - output_pixels, out_w, out_h, 0, - num_channels , alpha_chan , 0) - stbir_resize_uint8_srgb_edgemode( - input_pixels , in_w , in_h , 0, - output_pixels, out_w, out_h, 0, - num_channels , alpha_chan , 0, STBIR_EDGE_CLAMP) - // WRAP/REFLECT/ZERO - - FULL API - See the "header file" section of the source for API documentation. - - ADDITIONAL DOCUMENTATION - - SRGB & FLOATING POINT REPRESENTATION - The sRGB functions presume IEEE floating point. If you do not have - IEEE floating point, define STBIR_NON_IEEE_FLOAT. This will use - a slower implementation. - - MEMORY ALLOCATION - The resize functions here perform a single memory allocation using - malloc. To control the memory allocation, before the #include that - triggers the implementation, do: - - #define STBIR_MALLOC(size,context) ... - #define STBIR_FREE(ptr,context) ... - - Each resize function makes exactly one call to malloc/free, so to use - temp memory, store the temp memory in the context and return that. - - ASSERT - Define STBIR_ASSERT(boolval) to override assert() and not use assert.h - - OPTIMIZATION - Define STBIR_SATURATE_INT to compute clamp values in-range using - integer operations instead of float operations. This may be faster - on some platforms. - - DEFAULT FILTERS - For functions which don't provide explicit control over what filters - to use, you can change the compile-time defaults with - - #define STBIR_DEFAULT_FILTER_UPSAMPLE STBIR_FILTER_something - #define STBIR_DEFAULT_FILTER_DOWNSAMPLE STBIR_FILTER_something - - See stbir_filter in the header-file section for the list of filters. - - NEW FILTERS - A number of 1D filter kernels are used. For a list of - supported filters see the stbir_filter enum. To add a new filter, - write a filter function and add it to stbir__filter_info_table. - - PROGRESS - For interactive use with slow resize operations, you can install - a progress-report callback: - - #define STBIR_PROGRESS_REPORT(val) some_func(val) - - The parameter val is a float which goes from 0 to 1 as progress is made. - - For example: - - static void my_progress_report(float progress); - #define STBIR_PROGRESS_REPORT(val) my_progress_report(val) - - #define STB_IMAGE_RESIZE_IMPLEMENTATION - #include "stb_image_resize.h" - - static void my_progress_report(float progress) - { - printf("Progress: %f%%\n", progress*100); - } - - MAX CHANNELS - If your image has more than 64 channels, define STBIR_MAX_CHANNELS - to the max you'll have. - - ALPHA CHANNEL - Most of the resizing functions provide the ability to control how - the alpha channel of an image is processed. The important things - to know about this: - - 1. The best mathematically-behaved version of alpha to use is - called "premultiplied alpha", in which the other color channels - have had the alpha value multiplied in. If you use premultiplied - alpha, linear filtering (such as image resampling done by this - library, or performed in texture units on GPUs) does the "right - thing". While premultiplied alpha is standard in the movie CGI - industry, it is still uncommon in the videogame/real-time world. - - If you linearly filter non-premultiplied alpha, strange effects - occur. (For example, the 50/50 average of 99% transparent bright green - and 1% transparent black produces 50% transparent dark green when - non-premultiplied, whereas premultiplied it produces 50% - transparent near-black. The former introduces green energy - that doesn't exist in the source image.) - - 2. Artists should not edit premultiplied-alpha images; artists - want non-premultiplied alpha images. Thus, art tools generally output - non-premultiplied alpha images. - - 3. You will get best results in most cases by converting images - to premultiplied alpha before processing them mathematically. - - 4. If you pass the flag STBIR_FLAG_ALPHA_PREMULTIPLIED, the - resizer does not do anything special for the alpha channel; - it is resampled identically to other channels. This produces - the correct results for premultiplied-alpha images, but produces - less-than-ideal results for non-premultiplied-alpha images. - - 5. If you do not pass the flag STBIR_FLAG_ALPHA_PREMULTIPLIED, - then the resizer weights the contribution of input pixels - based on their alpha values, or, equivalently, it multiplies - the alpha value into the color channels, resamples, then divides - by the resultant alpha value. Input pixels which have alpha=0 do - not contribute at all to output pixels unless _all_ of the input - pixels affecting that output pixel have alpha=0, in which case - the result for that pixel is the same as it would be without - STBIR_FLAG_ALPHA_PREMULTIPLIED. However, this is only true for - input images in integer formats. For input images in float format, - input pixels with alpha=0 have no effect, and output pixels - which have alpha=0 will be 0 in all channels. (For float images, - you can manually achieve the same result by adding a tiny epsilon - value to the alpha channel of every image, and then subtracting - or clamping it at the end.) - - 6. You can suppress the behavior described in #5 and make - all-0-alpha pixels have 0 in all channels by #defining - STBIR_NO_ALPHA_EPSILON. - - 7. You can separately control whether the alpha channel is - interpreted as linear or affected by the colorspace. By default - it is linear; you almost never want to apply the colorspace. - (For example, graphics hardware does not apply sRGB conversion - to the alpha channel.) - - CONTRIBUTORS - Jorge L Rodriguez: Implementation - Sean Barrett: API design, optimizations - Aras Pranckevicius: bugfix - Nathan Reed: warning fixes - - REVISIONS - 0.95 (2017-07-23) fixed warnings - 0.94 (2017-03-18) fixed warnings - 0.93 (2017-03-03) fixed bug with certain combinations of heights - 0.92 (2017-01-02) fix integer overflow on large (>2GB) images - 0.91 (2016-04-02) fix warnings; fix handling of subpixel regions - 0.90 (2014-09-17) first released version - - LICENSE - See end of file for license information. - - TODO - Don't decode all of the image data when only processing a partial tile - Don't use full-width decode buffers when only processing a partial tile - When processing wide images, break processing into tiles so data fits in L1 cache - Installable filters? - Resize that respects alpha test coverage - (Reference code: FloatImage::alphaTestCoverage and FloatImage::scaleAlphaToCoverage: - https://code.google.com/p/nvidia-texture-tools/source/browse/trunk/src/nvimage/FloatImage.cpp ) -*/ - -#ifndef STBIR_INCLUDE_STB_IMAGE_RESIZE_H -#define STBIR_INCLUDE_STB_IMAGE_RESIZE_H - -#ifdef _MSC_VER -typedef unsigned char stbir_uint8; -typedef unsigned short stbir_uint16; -typedef unsigned int stbir_uint32; -#else -#include -typedef uint8_t stbir_uint8; -typedef uint16_t stbir_uint16; -typedef uint32_t stbir_uint32; -#endif - -#ifdef STB_IMAGE_RESIZE_STATIC -#define STBIRDEF static -#else -#ifdef __cplusplus -#define STBIRDEF extern "C" -#else -#define STBIRDEF extern -#endif -#endif - - -////////////////////////////////////////////////////////////////////////////// -// -// Easy-to-use API: -// -// * "input pixels" points to an array of image data with 'num_channels' channels (e.g. RGB=3, RGBA=4) -// * input_w is input image width (x-axis), input_h is input image height (y-axis) -// * stride is the offset between successive rows of image data in memory, in bytes. you can -// specify 0 to mean packed continuously in memory -// * alpha channel is treated identically to other channels. -// * colorspace is linear or sRGB as specified by function name -// * returned result is 1 for success or 0 in case of an error. -// #define STBIR_ASSERT() to trigger an assert on parameter validation errors. -// * Memory required grows approximately linearly with input and output size, but with -// discontinuities at input_w == output_w and input_h == output_h. -// * These functions use a "default" resampling filter defined at compile time. To change the filter, -// you can change the compile-time defaults by #defining STBIR_DEFAULT_FILTER_UPSAMPLE -// and STBIR_DEFAULT_FILTER_DOWNSAMPLE, or you can use the medium-complexity API. - -STBIRDEF int stbir_resize_uint8( const unsigned char *input_pixels , int input_w , int input_h , int input_stride_in_bytes, - unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, - int num_channels); - -STBIRDEF int stbir_resize_float( const float *input_pixels , int input_w , int input_h , int input_stride_in_bytes, - float *output_pixels, int output_w, int output_h, int output_stride_in_bytes, - int num_channels); - - -// The following functions interpret image data as gamma-corrected sRGB. -// Specify STBIR_ALPHA_CHANNEL_NONE if you have no alpha channel, -// or otherwise provide the index of the alpha channel. Flags value -// of 0 will probably do the right thing if you're not sure what -// the flags mean. - -#define STBIR_ALPHA_CHANNEL_NONE -1 - -// Set this flag if your texture has premultiplied alpha. Otherwise, stbir will -// use alpha-weighted resampling (effectively premultiplying, resampling, -// then unpremultiplying). -#define STBIR_FLAG_ALPHA_PREMULTIPLIED (1 << 0) -// The specified alpha channel should be handled as gamma-corrected value even -// when doing sRGB operations. -#define STBIR_FLAG_ALPHA_USES_COLORSPACE (1 << 1) - -STBIRDEF int stbir_resize_uint8_srgb(const unsigned char *input_pixels , int input_w , int input_h , int input_stride_in_bytes, - unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, - int num_channels, int alpha_channel, int flags); - - -typedef enum -{ - STBIR_EDGE_CLAMP = 1, - STBIR_EDGE_REFLECT = 2, - STBIR_EDGE_WRAP = 3, - STBIR_EDGE_ZERO = 4, -} stbir_edge; - -// This function adds the ability to specify how requests to sample off the edge of the image are handled. -STBIRDEF int stbir_resize_uint8_srgb_edgemode(const unsigned char *input_pixels , int input_w , int input_h , int input_stride_in_bytes, - unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, - int num_channels, int alpha_channel, int flags, - stbir_edge edge_wrap_mode); - -////////////////////////////////////////////////////////////////////////////// -// -// Medium-complexity API -// -// This extends the easy-to-use API as follows: -// -// * Alpha-channel can be processed separately -// * If alpha_channel is not STBIR_ALPHA_CHANNEL_NONE -// * Alpha channel will not be gamma corrected (unless flags&STBIR_FLAG_GAMMA_CORRECT) -// * Filters will be weighted by alpha channel (unless flags&STBIR_FLAG_ALPHA_PREMULTIPLIED) -// * Filter can be selected explicitly -// * uint16 image type -// * sRGB colorspace available for all types -// * context parameter for passing to STBIR_MALLOC - -typedef enum -{ - STBIR_FILTER_DEFAULT = 0, // use same filter type that easy-to-use API chooses - STBIR_FILTER_BOX = 1, // A trapezoid w/1-pixel wide ramps, same result as box for integer scale ratios - STBIR_FILTER_TRIANGLE = 2, // On upsampling, produces same results as bilinear texture filtering - STBIR_FILTER_CUBICBSPLINE = 3, // The cubic b-spline (aka Mitchell-Netrevalli with B=1,C=0), gaussian-esque - STBIR_FILTER_CATMULLROM = 4, // An interpolating cubic spline - STBIR_FILTER_MITCHELL = 5, // Mitchell-Netrevalli filter with B=1/3, C=1/3 -} stbir_filter; - -typedef enum -{ - STBIR_COLORSPACE_LINEAR, - STBIR_COLORSPACE_SRGB, - - STBIR_MAX_COLORSPACES, -} stbir_colorspace; - -// The following functions are all identical except for the type of the image data - -STBIRDEF int stbir_resize_uint8_generic( const unsigned char *input_pixels , int input_w , int input_h , int input_stride_in_bytes, - unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, - int num_channels, int alpha_channel, int flags, - stbir_edge edge_wrap_mode, stbir_filter filter, stbir_colorspace space, - void *alloc_context); - -STBIRDEF int stbir_resize_uint16_generic(const stbir_uint16 *input_pixels , int input_w , int input_h , int input_stride_in_bytes, - stbir_uint16 *output_pixels , int output_w, int output_h, int output_stride_in_bytes, - int num_channels, int alpha_channel, int flags, - stbir_edge edge_wrap_mode, stbir_filter filter, stbir_colorspace space, - void *alloc_context); - -STBIRDEF int stbir_resize_float_generic( const float *input_pixels , int input_w , int input_h , int input_stride_in_bytes, - float *output_pixels , int output_w, int output_h, int output_stride_in_bytes, - int num_channels, int alpha_channel, int flags, - stbir_edge edge_wrap_mode, stbir_filter filter, stbir_colorspace space, - void *alloc_context); - - - -////////////////////////////////////////////////////////////////////////////// -// -// Full-complexity API -// -// This extends the medium API as follows: -// -// * uint32 image type -// * not typesafe -// * separate filter types for each axis -// * separate edge modes for each axis -// * can specify scale explicitly for subpixel correctness -// * can specify image source tile using texture coordinates - -typedef enum -{ - STBIR_TYPE_UINT8 , - STBIR_TYPE_UINT16, - STBIR_TYPE_UINT32, - STBIR_TYPE_FLOAT , - - STBIR_MAX_TYPES -} stbir_datatype; - -STBIRDEF int stbir_resize( const void *input_pixels , int input_w , int input_h , int input_stride_in_bytes, - void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, - stbir_datatype datatype, - int num_channels, int alpha_channel, int flags, - stbir_edge edge_mode_horizontal, stbir_edge edge_mode_vertical, - stbir_filter filter_horizontal, stbir_filter filter_vertical, - stbir_colorspace space, void *alloc_context); - -STBIRDEF int stbir_resize_subpixel(const void *input_pixels , int input_w , int input_h , int input_stride_in_bytes, - void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, - stbir_datatype datatype, - int num_channels, int alpha_channel, int flags, - stbir_edge edge_mode_horizontal, stbir_edge edge_mode_vertical, - stbir_filter filter_horizontal, stbir_filter filter_vertical, - stbir_colorspace space, void *alloc_context, - float x_scale, float y_scale, - float x_offset, float y_offset); - -STBIRDEF int stbir_resize_region( const void *input_pixels , int input_w , int input_h , int input_stride_in_bytes, - void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, - stbir_datatype datatype, - int num_channels, int alpha_channel, int flags, - stbir_edge edge_mode_horizontal, stbir_edge edge_mode_vertical, - stbir_filter filter_horizontal, stbir_filter filter_vertical, - stbir_colorspace space, void *alloc_context, - float s0, float t0, float s1, float t1); -// (s0, t0) & (s1, t1) are the top-left and bottom right corner (uv addressing style: [0, 1]x[0, 1]) of a region of the input image to use. - -// -// -//// end header file ///////////////////////////////////////////////////// -#endif // STBIR_INCLUDE_STB_IMAGE_RESIZE_H - - - - - -#ifdef STB_IMAGE_RESIZE_IMPLEMENTATION - -#ifndef STBIR_ASSERT -#include -#define STBIR_ASSERT(x) assert(x) -#endif - -// For memset -#include - -#include - -#ifndef STBIR_MALLOC -#include -// use comma operator to evaluate c, to avoid "unused parameter" warnings -#define STBIR_MALLOC(size,c) ((void)(c), malloc(size)) -#define STBIR_FREE(ptr,c) ((void)(c), free(ptr)) -#endif - -#ifndef _MSC_VER -#ifdef __cplusplus -#define stbir__inline inline -#else -#define stbir__inline -#endif -#else -#define stbir__inline __forceinline -#endif - - -// should produce compiler error if size is wrong -typedef unsigned char stbir__validate_uint32[sizeof(stbir_uint32) == 4 ? 1 : -1]; - -#ifdef _MSC_VER -#define STBIR__NOTUSED(v) (void)(v) -#else -#define STBIR__NOTUSED(v) (void)sizeof(v) -#endif - -#define STBIR__ARRAY_SIZE(a) (sizeof((a))/sizeof((a)[0])) - -#ifndef STBIR_DEFAULT_FILTER_UPSAMPLE -#define STBIR_DEFAULT_FILTER_UPSAMPLE STBIR_FILTER_CATMULLROM -#endif - -#ifndef STBIR_DEFAULT_FILTER_DOWNSAMPLE -#define STBIR_DEFAULT_FILTER_DOWNSAMPLE STBIR_FILTER_MITCHELL -#endif - -#ifndef STBIR_PROGRESS_REPORT -#define STBIR_PROGRESS_REPORT(float_0_to_1) -#endif - -#ifndef STBIR_MAX_CHANNELS -#define STBIR_MAX_CHANNELS 64 -#endif - -#if STBIR_MAX_CHANNELS > 65536 -#error "Too many channels; STBIR_MAX_CHANNELS must be no more than 65536." -// because we store the indices in 16-bit variables -#endif - -// This value is added to alpha just before premultiplication to avoid -// zeroing out color values. It is equivalent to 2^-80. If you don't want -// that behavior (it may interfere if you have floating point images with -// very small alpha values) then you can define STBIR_NO_ALPHA_EPSILON to -// disable it. -#ifndef STBIR_ALPHA_EPSILON -#define STBIR_ALPHA_EPSILON ((float)1 / (1 << 20) / (1 << 20) / (1 << 20) / (1 << 20)) -#endif - - - -#ifdef _MSC_VER -#define STBIR__UNUSED_PARAM(v) (void)(v) -#else -#define STBIR__UNUSED_PARAM(v) (void)sizeof(v) -#endif - -// must match stbir_datatype -static unsigned char stbir__type_size[] = { - 1, // STBIR_TYPE_UINT8 - 2, // STBIR_TYPE_UINT16 - 4, // STBIR_TYPE_UINT32 - 4, // STBIR_TYPE_FLOAT -}; - -// Kernel function centered at 0 -typedef float (stbir__kernel_fn)(float x, float scale); -typedef float (stbir__support_fn)(float scale); - -typedef struct -{ - stbir__kernel_fn* kernel; - stbir__support_fn* support; -} stbir__filter_info; - -// When upsampling, the contributors are which source pixels contribute. -// When downsampling, the contributors are which destination pixels are contributed to. -typedef struct -{ - int n0; // First contributing pixel - int n1; // Last contributing pixel -} stbir__contributors; - -typedef struct -{ - const void* input_data; - int input_w; - int input_h; - int input_stride_bytes; - - void* output_data; - int output_w; - int output_h; - int output_stride_bytes; - - float s0, t0, s1, t1; - - float horizontal_shift; // Units: output pixels - float vertical_shift; // Units: output pixels - float horizontal_scale; - float vertical_scale; - - int channels; - int alpha_channel; - stbir_uint32 flags; - stbir_datatype type; - stbir_filter horizontal_filter; - stbir_filter vertical_filter; - stbir_edge edge_horizontal; - stbir_edge edge_vertical; - stbir_colorspace colorspace; - - stbir__contributors* horizontal_contributors; - float* horizontal_coefficients; - - stbir__contributors* vertical_contributors; - float* vertical_coefficients; - - int decode_buffer_pixels; - float* decode_buffer; - - float* horizontal_buffer; - - // cache these because ceil/floor are inexplicably showing up in profile - int horizontal_coefficient_width; - int vertical_coefficient_width; - int horizontal_filter_pixel_width; - int vertical_filter_pixel_width; - int horizontal_filter_pixel_margin; - int vertical_filter_pixel_margin; - int horizontal_num_contributors; - int vertical_num_contributors; - - int ring_buffer_length_bytes; // The length of an individual entry in the ring buffer. The total number of ring buffers is stbir__get_filter_pixel_width(filter) - int ring_buffer_num_entries; // Total number of entries in the ring buffer. - int ring_buffer_first_scanline; - int ring_buffer_last_scanline; - int ring_buffer_begin_index; // first_scanline is at this index in the ring buffer - float* ring_buffer; - - float* encode_buffer; // A temporary buffer to store floats so we don't lose precision while we do multiply-adds. - - int horizontal_contributors_size; - int horizontal_coefficients_size; - int vertical_contributors_size; - int vertical_coefficients_size; - int decode_buffer_size; - int horizontal_buffer_size; - int ring_buffer_size; - int encode_buffer_size; -} stbir__info; - - -static const float stbir__max_uint8_as_float = 255.0f; -static const float stbir__max_uint16_as_float = 65535.0f; -static const double stbir__max_uint32_as_float = 4294967295.0; - - -static stbir__inline int stbir__min(int a, int b) -{ - return a < b ? a : b; -} - -static stbir__inline float stbir__saturate(float x) -{ - if (x < 0) - return 0; - - if (x > 1) - return 1; - - return x; -} - -#ifdef STBIR_SATURATE_INT -static stbir__inline stbir_uint8 stbir__saturate8(int x) -{ - if ((unsigned int) x <= 255) - return x; - - if (x < 0) - return 0; - - return 255; -} - -static stbir__inline stbir_uint16 stbir__saturate16(int x) -{ - if ((unsigned int) x <= 65535) - return x; - - if (x < 0) - return 0; - - return 65535; -} -#endif - -static float stbir__srgb_uchar_to_linear_float[256] = { - 0.000000f, 0.000304f, 0.000607f, 0.000911f, 0.001214f, 0.001518f, 0.001821f, 0.002125f, 0.002428f, 0.002732f, 0.003035f, - 0.003347f, 0.003677f, 0.004025f, 0.004391f, 0.004777f, 0.005182f, 0.005605f, 0.006049f, 0.006512f, 0.006995f, 0.007499f, - 0.008023f, 0.008568f, 0.009134f, 0.009721f, 0.010330f, 0.010960f, 0.011612f, 0.012286f, 0.012983f, 0.013702f, 0.014444f, - 0.015209f, 0.015996f, 0.016807f, 0.017642f, 0.018500f, 0.019382f, 0.020289f, 0.021219f, 0.022174f, 0.023153f, 0.024158f, - 0.025187f, 0.026241f, 0.027321f, 0.028426f, 0.029557f, 0.030713f, 0.031896f, 0.033105f, 0.034340f, 0.035601f, 0.036889f, - 0.038204f, 0.039546f, 0.040915f, 0.042311f, 0.043735f, 0.045186f, 0.046665f, 0.048172f, 0.049707f, 0.051269f, 0.052861f, - 0.054480f, 0.056128f, 0.057805f, 0.059511f, 0.061246f, 0.063010f, 0.064803f, 0.066626f, 0.068478f, 0.070360f, 0.072272f, - 0.074214f, 0.076185f, 0.078187f, 0.080220f, 0.082283f, 0.084376f, 0.086500f, 0.088656f, 0.090842f, 0.093059f, 0.095307f, - 0.097587f, 0.099899f, 0.102242f, 0.104616f, 0.107023f, 0.109462f, 0.111932f, 0.114435f, 0.116971f, 0.119538f, 0.122139f, - 0.124772f, 0.127438f, 0.130136f, 0.132868f, 0.135633f, 0.138432f, 0.141263f, 0.144128f, 0.147027f, 0.149960f, 0.152926f, - 0.155926f, 0.158961f, 0.162029f, 0.165132f, 0.168269f, 0.171441f, 0.174647f, 0.177888f, 0.181164f, 0.184475f, 0.187821f, - 0.191202f, 0.194618f, 0.198069f, 0.201556f, 0.205079f, 0.208637f, 0.212231f, 0.215861f, 0.219526f, 0.223228f, 0.226966f, - 0.230740f, 0.234551f, 0.238398f, 0.242281f, 0.246201f, 0.250158f, 0.254152f, 0.258183f, 0.262251f, 0.266356f, 0.270498f, - 0.274677f, 0.278894f, 0.283149f, 0.287441f, 0.291771f, 0.296138f, 0.300544f, 0.304987f, 0.309469f, 0.313989f, 0.318547f, - 0.323143f, 0.327778f, 0.332452f, 0.337164f, 0.341914f, 0.346704f, 0.351533f, 0.356400f, 0.361307f, 0.366253f, 0.371238f, - 0.376262f, 0.381326f, 0.386430f, 0.391573f, 0.396755f, 0.401978f, 0.407240f, 0.412543f, 0.417885f, 0.423268f, 0.428691f, - 0.434154f, 0.439657f, 0.445201f, 0.450786f, 0.456411f, 0.462077f, 0.467784f, 0.473532f, 0.479320f, 0.485150f, 0.491021f, - 0.496933f, 0.502887f, 0.508881f, 0.514918f, 0.520996f, 0.527115f, 0.533276f, 0.539480f, 0.545725f, 0.552011f, 0.558340f, - 0.564712f, 0.571125f, 0.577581f, 0.584078f, 0.590619f, 0.597202f, 0.603827f, 0.610496f, 0.617207f, 0.623960f, 0.630757f, - 0.637597f, 0.644480f, 0.651406f, 0.658375f, 0.665387f, 0.672443f, 0.679543f, 0.686685f, 0.693872f, 0.701102f, 0.708376f, - 0.715694f, 0.723055f, 0.730461f, 0.737911f, 0.745404f, 0.752942f, 0.760525f, 0.768151f, 0.775822f, 0.783538f, 0.791298f, - 0.799103f, 0.806952f, 0.814847f, 0.822786f, 0.830770f, 0.838799f, 0.846873f, 0.854993f, 0.863157f, 0.871367f, 0.879622f, - 0.887923f, 0.896269f, 0.904661f, 0.913099f, 0.921582f, 0.930111f, 0.938686f, 0.947307f, 0.955974f, 0.964686f, 0.973445f, - 0.982251f, 0.991102f, 1.0f -}; - -static float stbir__srgb_to_linear(float f) -{ - if (f <= 0.04045f) - return f / 12.92f; - else - return (float)pow((f + 0.055f) / 1.055f, 2.4f); -} - -static float stbir__linear_to_srgb(float f) -{ - if (f <= 0.0031308f) - return f * 12.92f; - else - return 1.055f * (float)pow(f, 1 / 2.4f) - 0.055f; -} - -#ifndef STBIR_NON_IEEE_FLOAT -// From https://gist.github.com/rygorous/2203834 - -typedef union -{ - stbir_uint32 u; - float f; -} stbir__FP32; - -static const stbir_uint32 fp32_to_srgb8_tab4[104] = { - 0x0073000d, 0x007a000d, 0x0080000d, 0x0087000d, 0x008d000d, 0x0094000d, 0x009a000d, 0x00a1000d, - 0x00a7001a, 0x00b4001a, 0x00c1001a, 0x00ce001a, 0x00da001a, 0x00e7001a, 0x00f4001a, 0x0101001a, - 0x010e0033, 0x01280033, 0x01410033, 0x015b0033, 0x01750033, 0x018f0033, 0x01a80033, 0x01c20033, - 0x01dc0067, 0x020f0067, 0x02430067, 0x02760067, 0x02aa0067, 0x02dd0067, 0x03110067, 0x03440067, - 0x037800ce, 0x03df00ce, 0x044600ce, 0x04ad00ce, 0x051400ce, 0x057b00c5, 0x05dd00bc, 0x063b00b5, - 0x06970158, 0x07420142, 0x07e30130, 0x087b0120, 0x090b0112, 0x09940106, 0x0a1700fc, 0x0a9500f2, - 0x0b0f01cb, 0x0bf401ae, 0x0ccb0195, 0x0d950180, 0x0e56016e, 0x0f0d015e, 0x0fbc0150, 0x10630143, - 0x11070264, 0x1238023e, 0x1357021d, 0x14660201, 0x156601e9, 0x165a01d3, 0x174401c0, 0x182401af, - 0x18fe0331, 0x1a9602fe, 0x1c1502d2, 0x1d7e02ad, 0x1ed4028d, 0x201a0270, 0x21520256, 0x227d0240, - 0x239f0443, 0x25c003fe, 0x27bf03c4, 0x29a10392, 0x2b6a0367, 0x2d1d0341, 0x2ebe031f, 0x304d0300, - 0x31d105b0, 0x34a80555, 0x37520507, 0x39d504c5, 0x3c37048b, 0x3e7c0458, 0x40a8042a, 0x42bd0401, - 0x44c20798, 0x488e071e, 0x4c1c06b6, 0x4f76065d, 0x52a50610, 0x55ac05cc, 0x5892058f, 0x5b590559, - 0x5e0c0a23, 0x631c0980, 0x67db08f6, 0x6c55087f, 0x70940818, 0x74a007bd, 0x787d076c, 0x7c330723, -}; - -static stbir_uint8 stbir__linear_to_srgb_uchar(float in) -{ - static const stbir__FP32 almostone = { 0x3f7fffff }; // 1-eps - static const stbir__FP32 minval = { (127-13) << 23 }; - stbir_uint32 tab,bias,scale,t; - stbir__FP32 f; - - // Clamp to [2^(-13), 1-eps]; these two values map to 0 and 1, respectively. - // The tests are carefully written so that NaNs map to 0, same as in the reference - // implementation. - if (!(in > minval.f)) // written this way to catch NaNs - in = minval.f; - if (in > almostone.f) - in = almostone.f; - - // Do the table lookup and unpack bias, scale - f.f = in; - tab = fp32_to_srgb8_tab4[(f.u - minval.u) >> 20]; - bias = (tab >> 16) << 9; - scale = tab & 0xffff; - - // Grab next-highest mantissa bits and perform linear interpolation - t = (f.u >> 12) & 0xff; - return (unsigned char) ((bias + scale*t) >> 16); -} - -#else -// sRGB transition values, scaled by 1<<28 -static int stbir__srgb_offset_to_linear_scaled[256] = -{ - 0, 40738, 122216, 203693, 285170, 366648, 448125, 529603, - 611080, 692557, 774035, 855852, 942009, 1033024, 1128971, 1229926, - 1335959, 1447142, 1563542, 1685229, 1812268, 1944725, 2082664, 2226148, - 2375238, 2529996, 2690481, 2856753, 3028870, 3206888, 3390865, 3580856, - 3776916, 3979100, 4187460, 4402049, 4622919, 4850123, 5083710, 5323731, - 5570236, 5823273, 6082892, 6349140, 6622065, 6901714, 7188133, 7481369, - 7781466, 8088471, 8402427, 8723380, 9051372, 9386448, 9728650, 10078021, - 10434603, 10798439, 11169569, 11548036, 11933879, 12327139, 12727857, 13136073, - 13551826, 13975156, 14406100, 14844697, 15290987, 15745007, 16206795, 16676389, - 17153826, 17639142, 18132374, 18633560, 19142734, 19659934, 20185196, 20718552, - 21260042, 21809696, 22367554, 22933648, 23508010, 24090680, 24681686, 25281066, - 25888850, 26505076, 27129772, 27762974, 28404716, 29055026, 29713942, 30381490, - 31057708, 31742624, 32436272, 33138682, 33849884, 34569912, 35298800, 36036568, - 36783260, 37538896, 38303512, 39077136, 39859796, 40651528, 41452360, 42262316, - 43081432, 43909732, 44747252, 45594016, 46450052, 47315392, 48190064, 49074096, - 49967516, 50870356, 51782636, 52704392, 53635648, 54576432, 55526772, 56486700, - 57456236, 58435408, 59424248, 60422780, 61431036, 62449032, 63476804, 64514376, - 65561776, 66619028, 67686160, 68763192, 69850160, 70947088, 72053992, 73170912, - 74297864, 75434880, 76581976, 77739184, 78906536, 80084040, 81271736, 82469648, - 83677792, 84896192, 86124888, 87363888, 88613232, 89872928, 91143016, 92423512, - 93714432, 95015816, 96327688, 97650056, 98982952, 100326408, 101680440, 103045072, - 104420320, 105806224, 107202800, 108610064, 110028048, 111456776, 112896264, 114346544, - 115807632, 117279552, 118762328, 120255976, 121760536, 123276016, 124802440, 126339832, - 127888216, 129447616, 131018048, 132599544, 134192112, 135795792, 137410592, 139036528, - 140673648, 142321952, 143981456, 145652208, 147334208, 149027488, 150732064, 152447968, - 154175200, 155913792, 157663776, 159425168, 161197984, 162982240, 164777968, 166585184, - 168403904, 170234160, 172075968, 173929344, 175794320, 177670896, 179559120, 181458992, - 183370528, 185293776, 187228736, 189175424, 191133888, 193104112, 195086128, 197079968, - 199085648, 201103184, 203132592, 205173888, 207227120, 209292272, 211369392, 213458480, - 215559568, 217672656, 219797792, 221934976, 224084240, 226245600, 228419056, 230604656, - 232802400, 235012320, 237234432, 239468736, 241715280, 243974080, 246245120, 248528464, - 250824112, 253132064, 255452368, 257785040, 260130080, 262487520, 264857376, 267239664, -}; - -static stbir_uint8 stbir__linear_to_srgb_uchar(float f) -{ - int x = (int) (f * (1 << 28)); // has headroom so you don't need to clamp - int v = 0; - int i; - - // Refine the guess with a short binary search. - i = v + 128; if (x >= stbir__srgb_offset_to_linear_scaled[i]) v = i; - i = v + 64; if (x >= stbir__srgb_offset_to_linear_scaled[i]) v = i; - i = v + 32; if (x >= stbir__srgb_offset_to_linear_scaled[i]) v = i; - i = v + 16; if (x >= stbir__srgb_offset_to_linear_scaled[i]) v = i; - i = v + 8; if (x >= stbir__srgb_offset_to_linear_scaled[i]) v = i; - i = v + 4; if (x >= stbir__srgb_offset_to_linear_scaled[i]) v = i; - i = v + 2; if (x >= stbir__srgb_offset_to_linear_scaled[i]) v = i; - i = v + 1; if (x >= stbir__srgb_offset_to_linear_scaled[i]) v = i; - - return (stbir_uint8) v; -} -#endif - -static float stbir__filter_trapezoid(float x, float scale) -{ - float halfscale = scale / 2; - float t = 0.5f + halfscale; - STBIR_ASSERT(scale <= 1); - - x = (float)fabs(x); - - if (x >= t) - return 0; - else - { - float r = 0.5f - halfscale; - if (x <= r) - return 1; - else - return (t - x) / scale; - } -} - -static float stbir__support_trapezoid(float scale) -{ - STBIR_ASSERT(scale <= 1); - return 0.5f + scale / 2; -} - -static float stbir__filter_triangle(float x, float s) -{ - STBIR__UNUSED_PARAM(s); - - x = (float)fabs(x); - - if (x <= 1.0f) - return 1 - x; - else - return 0; -} - -static float stbir__filter_cubic(float x, float s) -{ - STBIR__UNUSED_PARAM(s); - - x = (float)fabs(x); - - if (x < 1.0f) - return (4 + x*x*(3*x - 6))/6; - else if (x < 2.0f) - return (8 + x*(-12 + x*(6 - x)))/6; - - return (0.0f); -} - -static float stbir__filter_catmullrom(float x, float s) -{ - STBIR__UNUSED_PARAM(s); - - x = (float)fabs(x); - - if (x < 1.0f) - return 1 - x*x*(2.5f - 1.5f*x); - else if (x < 2.0f) - return 2 - x*(4 + x*(0.5f*x - 2.5f)); - - return (0.0f); -} - -static float stbir__filter_mitchell(float x, float s) -{ - STBIR__UNUSED_PARAM(s); - - x = (float)fabs(x); - - if (x < 1.0f) - return (16 + x*x*(21 * x - 36))/18; - else if (x < 2.0f) - return (32 + x*(-60 + x*(36 - 7*x)))/18; - - return (0.0f); -} - -static float stbir__support_zero(float s) -{ - STBIR__UNUSED_PARAM(s); - return 0; -} - -static float stbir__support_one(float s) -{ - STBIR__UNUSED_PARAM(s); - return 1; -} - -static float stbir__support_two(float s) -{ - STBIR__UNUSED_PARAM(s); - return 2; -} - -static stbir__filter_info stbir__filter_info_table[] = { - { NULL, stbir__support_zero }, - { stbir__filter_trapezoid, stbir__support_trapezoid }, - { stbir__filter_triangle, stbir__support_one }, - { stbir__filter_cubic, stbir__support_two }, - { stbir__filter_catmullrom, stbir__support_two }, - { stbir__filter_mitchell, stbir__support_two }, -}; - -stbir__inline static int stbir__use_upsampling(float ratio) -{ - return ratio > 1; -} - -stbir__inline static int stbir__use_width_upsampling(stbir__info* stbir_info) -{ - return stbir__use_upsampling(stbir_info->horizontal_scale); -} - -stbir__inline static int stbir__use_height_upsampling(stbir__info* stbir_info) -{ - return stbir__use_upsampling(stbir_info->vertical_scale); -} - -// This is the maximum number of input samples that can affect an output sample -// with the given filter -static int stbir__get_filter_pixel_width(stbir_filter filter, float scale) -{ - STBIR_ASSERT(filter != 0); - STBIR_ASSERT(filter < STBIR__ARRAY_SIZE(stbir__filter_info_table)); - - if (stbir__use_upsampling(scale)) - return (int)ceil(stbir__filter_info_table[filter].support(1/scale) * 2); - else - return (int)ceil(stbir__filter_info_table[filter].support(scale) * 2 / scale); -} - -// This is how much to expand buffers to account for filters seeking outside -// the image boundaries. -static int stbir__get_filter_pixel_margin(stbir_filter filter, float scale) -{ - return stbir__get_filter_pixel_width(filter, scale) / 2; -} - -static int stbir__get_coefficient_width(stbir_filter filter, float scale) -{ - if (stbir__use_upsampling(scale)) - return (int)ceil(stbir__filter_info_table[filter].support(1 / scale) * 2); - else - return (int)ceil(stbir__filter_info_table[filter].support(scale) * 2); -} - -static int stbir__get_contributors(float scale, stbir_filter filter, int input_size, int output_size) -{ - if (stbir__use_upsampling(scale)) - return output_size; - else - return (input_size + stbir__get_filter_pixel_margin(filter, scale) * 2); -} - -static int stbir__get_total_horizontal_coefficients(stbir__info* info) -{ - return info->horizontal_num_contributors - * stbir__get_coefficient_width (info->horizontal_filter, info->horizontal_scale); -} - -static int stbir__get_total_vertical_coefficients(stbir__info* info) -{ - return info->vertical_num_contributors - * stbir__get_coefficient_width (info->vertical_filter, info->vertical_scale); -} - -static stbir__contributors* stbir__get_contributor(stbir__contributors* contributors, int n) -{ - return &contributors[n]; -} - -// For perf reasons this code is duplicated in stbir__resample_horizontal_upsample/downsample, -// if you change it here change it there too. -static float* stbir__get_coefficient(float* coefficients, stbir_filter filter, float scale, int n, int c) -{ - int width = stbir__get_coefficient_width(filter, scale); - return &coefficients[width*n + c]; -} - -static int stbir__edge_wrap_slow(stbir_edge edge, int n, int max) -{ - switch (edge) - { - case STBIR_EDGE_ZERO: - return 0; // we'll decode the wrong pixel here, and then overwrite with 0s later - - case STBIR_EDGE_CLAMP: - if (n < 0) - return 0; - - if (n >= max) - return max - 1; - - return n; // NOTREACHED - - case STBIR_EDGE_REFLECT: - { - if (n < 0) - { - if (n < max) - return -n; - else - return max - 1; - } - - if (n >= max) - { - int max2 = max * 2; - if (n >= max2) - return 0; - else - return max2 - n - 1; - } - - return n; // NOTREACHED - } - - case STBIR_EDGE_WRAP: - if (n >= 0) - return (n % max); - else - { - int m = (-n) % max; - - if (m != 0) - m = max - m; - - return (m); - } - // NOTREACHED - - default: - STBIR_ASSERT(!"Unimplemented edge type"); - return 0; - } -} - -stbir__inline static int stbir__edge_wrap(stbir_edge edge, int n, int max) -{ - // avoid per-pixel switch - if (n >= 0 && n < max) - return n; - return stbir__edge_wrap_slow(edge, n, max); -} - -// What input pixels contribute to this output pixel? -static void stbir__calculate_sample_range_upsample(int n, float out_filter_radius, float scale_ratio, float out_shift, int* in_first_pixel, int* in_last_pixel, float* in_center_of_out) -{ - float out_pixel_center = (float)n + 0.5f; - float out_pixel_influence_lowerbound = out_pixel_center - out_filter_radius; - float out_pixel_influence_upperbound = out_pixel_center + out_filter_radius; - - float in_pixel_influence_lowerbound = (out_pixel_influence_lowerbound + out_shift) / scale_ratio; - float in_pixel_influence_upperbound = (out_pixel_influence_upperbound + out_shift) / scale_ratio; - - *in_center_of_out = (out_pixel_center + out_shift) / scale_ratio; - *in_first_pixel = (int)(floor(in_pixel_influence_lowerbound + 0.5)); - *in_last_pixel = (int)(floor(in_pixel_influence_upperbound - 0.5)); -} - -// What output pixels does this input pixel contribute to? -static void stbir__calculate_sample_range_downsample(int n, float in_pixels_radius, float scale_ratio, float out_shift, int* out_first_pixel, int* out_last_pixel, float* out_center_of_in) -{ - float in_pixel_center = (float)n + 0.5f; - float in_pixel_influence_lowerbound = in_pixel_center - in_pixels_radius; - float in_pixel_influence_upperbound = in_pixel_center + in_pixels_radius; - - float out_pixel_influence_lowerbound = in_pixel_influence_lowerbound * scale_ratio - out_shift; - float out_pixel_influence_upperbound = in_pixel_influence_upperbound * scale_ratio - out_shift; - - *out_center_of_in = in_pixel_center * scale_ratio - out_shift; - *out_first_pixel = (int)(floor(out_pixel_influence_lowerbound + 0.5)); - *out_last_pixel = (int)(floor(out_pixel_influence_upperbound - 0.5)); -} - -static void stbir__calculate_coefficients_upsample(stbir_filter filter, float scale, int in_first_pixel, int in_last_pixel, float in_center_of_out, stbir__contributors* contributor, float* coefficient_group) -{ - int i; - float total_filter = 0; - float filter_scale; - - STBIR_ASSERT(in_last_pixel - in_first_pixel <= (int)ceil(stbir__filter_info_table[filter].support(1/scale) * 2)); // Taken directly from stbir__get_coefficient_width() which we can't call because we don't know if we're horizontal or vertical. - - contributor->n0 = in_first_pixel; - contributor->n1 = in_last_pixel; - - STBIR_ASSERT(contributor->n1 >= contributor->n0); - - for (i = 0; i <= in_last_pixel - in_first_pixel; i++) - { - float in_pixel_center = (float)(i + in_first_pixel) + 0.5f; - coefficient_group[i] = stbir__filter_info_table[filter].kernel(in_center_of_out - in_pixel_center, 1 / scale); - - // If the coefficient is zero, skip it. (Don't do the <0 check here, we want the influence of those outside pixels.) - if (i == 0 && !coefficient_group[i]) - { - contributor->n0 = ++in_first_pixel; - i--; - continue; - } - - total_filter += coefficient_group[i]; - } - - STBIR_ASSERT(stbir__filter_info_table[filter].kernel((float)(in_last_pixel + 1) + 0.5f - in_center_of_out, 1/scale) == 0); - - STBIR_ASSERT(total_filter > 0.9); - STBIR_ASSERT(total_filter < 1.1f); // Make sure it's not way off. - - // Make sure the sum of all coefficients is 1. - filter_scale = 1 / total_filter; - - for (i = 0; i <= in_last_pixel - in_first_pixel; i++) - coefficient_group[i] *= filter_scale; - - for (i = in_last_pixel - in_first_pixel; i >= 0; i--) - { - if (coefficient_group[i]) - break; - - // This line has no weight. We can skip it. - contributor->n1 = contributor->n0 + i - 1; - } -} - -static void stbir__calculate_coefficients_downsample(stbir_filter filter, float scale_ratio, int out_first_pixel, int out_last_pixel, float out_center_of_in, stbir__contributors* contributor, float* coefficient_group) -{ - int i; - - STBIR_ASSERT(out_last_pixel - out_first_pixel <= (int)ceil(stbir__filter_info_table[filter].support(scale_ratio) * 2)); // Taken directly from stbir__get_coefficient_width() which we can't call because we don't know if we're horizontal or vertical. - - contributor->n0 = out_first_pixel; - contributor->n1 = out_last_pixel; - - STBIR_ASSERT(contributor->n1 >= contributor->n0); - - for (i = 0; i <= out_last_pixel - out_first_pixel; i++) - { - float out_pixel_center = (float)(i + out_first_pixel) + 0.5f; - float x = out_pixel_center - out_center_of_in; - coefficient_group[i] = stbir__filter_info_table[filter].kernel(x, scale_ratio) * scale_ratio; - } - - STBIR_ASSERT(stbir__filter_info_table[filter].kernel((float)(out_last_pixel + 1) + 0.5f - out_center_of_in, scale_ratio) == 0); - - for (i = out_last_pixel - out_first_pixel; i >= 0; i--) - { - if (coefficient_group[i]) - break; - - // This line has no weight. We can skip it. - contributor->n1 = contributor->n0 + i - 1; - } -} - -static void stbir__normalize_downsample_coefficients(stbir__contributors* contributors, float* coefficients, stbir_filter filter, float scale_ratio, int input_size, int output_size) -{ - int num_contributors = stbir__get_contributors(scale_ratio, filter, input_size, output_size); - int num_coefficients = stbir__get_coefficient_width(filter, scale_ratio); - int i, j; - int skip; - - for (i = 0; i < output_size; i++) - { - float scale; - float total = 0; - - for (j = 0; j < num_contributors; j++) - { - if (i >= contributors[j].n0 && i <= contributors[j].n1) - { - float coefficient = *stbir__get_coefficient(coefficients, filter, scale_ratio, j, i - contributors[j].n0); - total += coefficient; - } - else if (i < contributors[j].n0) - break; - } - - STBIR_ASSERT(total > 0.9f); - STBIR_ASSERT(total < 1.1f); - - scale = 1 / total; - - for (j = 0; j < num_contributors; j++) - { - if (i >= contributors[j].n0 && i <= contributors[j].n1) - *stbir__get_coefficient(coefficients, filter, scale_ratio, j, i - contributors[j].n0) *= scale; - else if (i < contributors[j].n0) - break; - } - } - - // Optimize: Skip zero coefficients and contributions outside of image bounds. - // Do this after normalizing because normalization depends on the n0/n1 values. - for (j = 0; j < num_contributors; j++) - { - int range, max, width; - - skip = 0; - while (*stbir__get_coefficient(coefficients, filter, scale_ratio, j, skip) == 0) - skip++; - - contributors[j].n0 += skip; - - while (contributors[j].n0 < 0) - { - contributors[j].n0++; - skip++; - } - - range = contributors[j].n1 - contributors[j].n0 + 1; - max = stbir__min(num_coefficients, range); - - width = stbir__get_coefficient_width(filter, scale_ratio); - for (i = 0; i < max; i++) - { - if (i + skip >= width) - break; - - *stbir__get_coefficient(coefficients, filter, scale_ratio, j, i) = *stbir__get_coefficient(coefficients, filter, scale_ratio, j, i + skip); - } - - continue; - } - - // Using min to avoid writing into invalid pixels. - for (i = 0; i < num_contributors; i++) - contributors[i].n1 = stbir__min(contributors[i].n1, output_size - 1); -} - -// Each scan line uses the same kernel values so we should calculate the kernel -// values once and then we can use them for every scan line. -static void stbir__calculate_filters(stbir__contributors* contributors, float* coefficients, stbir_filter filter, float scale_ratio, float shift, int input_size, int output_size) -{ - int n; - int total_contributors = stbir__get_contributors(scale_ratio, filter, input_size, output_size); - - if (stbir__use_upsampling(scale_ratio)) - { - float out_pixels_radius = stbir__filter_info_table[filter].support(1 / scale_ratio) * scale_ratio; - - // Looping through out pixels - for (n = 0; n < total_contributors; n++) - { - float in_center_of_out; // Center of the current out pixel in the in pixel space - int in_first_pixel, in_last_pixel; - - stbir__calculate_sample_range_upsample(n, out_pixels_radius, scale_ratio, shift, &in_first_pixel, &in_last_pixel, &in_center_of_out); - - stbir__calculate_coefficients_upsample(filter, scale_ratio, in_first_pixel, in_last_pixel, in_center_of_out, stbir__get_contributor(contributors, n), stbir__get_coefficient(coefficients, filter, scale_ratio, n, 0)); - } - } - else - { - float in_pixels_radius = stbir__filter_info_table[filter].support(scale_ratio) / scale_ratio; - - // Looping through in pixels - for (n = 0; n < total_contributors; n++) - { - float out_center_of_in; // Center of the current out pixel in the in pixel space - int out_first_pixel, out_last_pixel; - int n_adjusted = n - stbir__get_filter_pixel_margin(filter, scale_ratio); - - stbir__calculate_sample_range_downsample(n_adjusted, in_pixels_radius, scale_ratio, shift, &out_first_pixel, &out_last_pixel, &out_center_of_in); - - stbir__calculate_coefficients_downsample(filter, scale_ratio, out_first_pixel, out_last_pixel, out_center_of_in, stbir__get_contributor(contributors, n), stbir__get_coefficient(coefficients, filter, scale_ratio, n, 0)); - } - - stbir__normalize_downsample_coefficients(contributors, coefficients, filter, scale_ratio, input_size, output_size); - } -} - -static float* stbir__get_decode_buffer(stbir__info* stbir_info) -{ - // The 0 index of the decode buffer starts after the margin. This makes - // it okay to use negative indexes on the decode buffer. - return &stbir_info->decode_buffer[stbir_info->horizontal_filter_pixel_margin * stbir_info->channels]; -} - -#define STBIR__DECODE(type, colorspace) ((type) * (STBIR_MAX_COLORSPACES) + (colorspace)) - -static void stbir__decode_scanline(stbir__info* stbir_info, int n) -{ - int c; - int channels = stbir_info->channels; - int alpha_channel = stbir_info->alpha_channel; - int type = stbir_info->type; - int colorspace = stbir_info->colorspace; - int input_w = stbir_info->input_w; - size_t input_stride_bytes = stbir_info->input_stride_bytes; - float* decode_buffer = stbir__get_decode_buffer(stbir_info); - stbir_edge edge_horizontal = stbir_info->edge_horizontal; - stbir_edge edge_vertical = stbir_info->edge_vertical; - size_t in_buffer_row_offset = stbir__edge_wrap(edge_vertical, n, stbir_info->input_h) * input_stride_bytes; - const void* input_data = (char *) stbir_info->input_data + in_buffer_row_offset; - int max_x = input_w + stbir_info->horizontal_filter_pixel_margin; - int decode = STBIR__DECODE(type, colorspace); - - int x = -stbir_info->horizontal_filter_pixel_margin; - - // special handling for STBIR_EDGE_ZERO because it needs to return an item that doesn't appear in the input, - // and we want to avoid paying overhead on every pixel if not STBIR_EDGE_ZERO - if (edge_vertical == STBIR_EDGE_ZERO && (n < 0 || n >= stbir_info->input_h)) - { - for (; x < max_x; x++) - for (c = 0; c < channels; c++) - decode_buffer[x*channels + c] = 0; - return; - } - - switch (decode) - { - case STBIR__DECODE(STBIR_TYPE_UINT8, STBIR_COLORSPACE_LINEAR): - for (; x < max_x; x++) - { - int decode_pixel_index = x * channels; - int input_pixel_index = stbir__edge_wrap(edge_horizontal, x, input_w) * channels; - for (c = 0; c < channels; c++) - decode_buffer[decode_pixel_index + c] = ((float)((const unsigned char*)input_data)[input_pixel_index + c]) / stbir__max_uint8_as_float; - } - break; - - case STBIR__DECODE(STBIR_TYPE_UINT8, STBIR_COLORSPACE_SRGB): - for (; x < max_x; x++) - { - int decode_pixel_index = x * channels; - int input_pixel_index = stbir__edge_wrap(edge_horizontal, x, input_w) * channels; - for (c = 0; c < channels; c++) - decode_buffer[decode_pixel_index + c] = stbir__srgb_uchar_to_linear_float[((const unsigned char*)input_data)[input_pixel_index + c]]; - - if (!(stbir_info->flags&STBIR_FLAG_ALPHA_USES_COLORSPACE)) - decode_buffer[decode_pixel_index + alpha_channel] = ((float)((const unsigned char*)input_data)[input_pixel_index + alpha_channel]) / stbir__max_uint8_as_float; - } - break; - - case STBIR__DECODE(STBIR_TYPE_UINT16, STBIR_COLORSPACE_LINEAR): - for (; x < max_x; x++) - { - int decode_pixel_index = x * channels; - int input_pixel_index = stbir__edge_wrap(edge_horizontal, x, input_w) * channels; - for (c = 0; c < channels; c++) - decode_buffer[decode_pixel_index + c] = ((float)((const unsigned short*)input_data)[input_pixel_index + c]) / stbir__max_uint16_as_float; - } - break; - - case STBIR__DECODE(STBIR_TYPE_UINT16, STBIR_COLORSPACE_SRGB): - for (; x < max_x; x++) - { - int decode_pixel_index = x * channels; - int input_pixel_index = stbir__edge_wrap(edge_horizontal, x, input_w) * channels; - for (c = 0; c < channels; c++) - decode_buffer[decode_pixel_index + c] = stbir__srgb_to_linear(((float)((const unsigned short*)input_data)[input_pixel_index + c]) / stbir__max_uint16_as_float); - - if (!(stbir_info->flags&STBIR_FLAG_ALPHA_USES_COLORSPACE)) - decode_buffer[decode_pixel_index + alpha_channel] = ((float)((const unsigned short*)input_data)[input_pixel_index + alpha_channel]) / stbir__max_uint16_as_float; - } - break; - - case STBIR__DECODE(STBIR_TYPE_UINT32, STBIR_COLORSPACE_LINEAR): - for (; x < max_x; x++) - { - int decode_pixel_index = x * channels; - int input_pixel_index = stbir__edge_wrap(edge_horizontal, x, input_w) * channels; - for (c = 0; c < channels; c++) - decode_buffer[decode_pixel_index + c] = (float)(((double)((const unsigned int*)input_data)[input_pixel_index + c]) / stbir__max_uint32_as_float); - } - break; - - case STBIR__DECODE(STBIR_TYPE_UINT32, STBIR_COLORSPACE_SRGB): - for (; x < max_x; x++) - { - int decode_pixel_index = x * channels; - int input_pixel_index = stbir__edge_wrap(edge_horizontal, x, input_w) * channels; - for (c = 0; c < channels; c++) - decode_buffer[decode_pixel_index + c] = stbir__srgb_to_linear((float)(((double)((const unsigned int*)input_data)[input_pixel_index + c]) / stbir__max_uint32_as_float)); - - if (!(stbir_info->flags&STBIR_FLAG_ALPHA_USES_COLORSPACE)) - decode_buffer[decode_pixel_index + alpha_channel] = (float)(((double)((const unsigned int*)input_data)[input_pixel_index + alpha_channel]) / stbir__max_uint32_as_float); - } - break; - - case STBIR__DECODE(STBIR_TYPE_FLOAT, STBIR_COLORSPACE_LINEAR): - for (; x < max_x; x++) - { - int decode_pixel_index = x * channels; - int input_pixel_index = stbir__edge_wrap(edge_horizontal, x, input_w) * channels; - for (c = 0; c < channels; c++) - decode_buffer[decode_pixel_index + c] = ((const float*)input_data)[input_pixel_index + c]; - } - break; - - case STBIR__DECODE(STBIR_TYPE_FLOAT, STBIR_COLORSPACE_SRGB): - for (; x < max_x; x++) - { - int decode_pixel_index = x * channels; - int input_pixel_index = stbir__edge_wrap(edge_horizontal, x, input_w) * channels; - for (c = 0; c < channels; c++) - decode_buffer[decode_pixel_index + c] = stbir__srgb_to_linear(((const float*)input_data)[input_pixel_index + c]); - - if (!(stbir_info->flags&STBIR_FLAG_ALPHA_USES_COLORSPACE)) - decode_buffer[decode_pixel_index + alpha_channel] = ((const float*)input_data)[input_pixel_index + alpha_channel]; - } - - break; - - default: - STBIR_ASSERT(!"Unknown type/colorspace/channels combination."); - break; - } - - if (!(stbir_info->flags & STBIR_FLAG_ALPHA_PREMULTIPLIED)) - { - for (x = -stbir_info->horizontal_filter_pixel_margin; x < max_x; x++) - { - int decode_pixel_index = x * channels; - - // If the alpha value is 0 it will clobber the color values. Make sure it's not. - float alpha = decode_buffer[decode_pixel_index + alpha_channel]; -#ifndef STBIR_NO_ALPHA_EPSILON - if (stbir_info->type != STBIR_TYPE_FLOAT) { - alpha += STBIR_ALPHA_EPSILON; - decode_buffer[decode_pixel_index + alpha_channel] = alpha; - } -#endif - for (c = 0; c < channels; c++) - { - if (c == alpha_channel) - continue; - - decode_buffer[decode_pixel_index + c] *= alpha; - } - } - } - - if (edge_horizontal == STBIR_EDGE_ZERO) - { - for (x = -stbir_info->horizontal_filter_pixel_margin; x < 0; x++) - { - for (c = 0; c < channels; c++) - decode_buffer[x*channels + c] = 0; - } - for (x = input_w; x < max_x; x++) - { - for (c = 0; c < channels; c++) - decode_buffer[x*channels + c] = 0; - } - } -} - -static float* stbir__get_ring_buffer_entry(float* ring_buffer, int index, int ring_buffer_length) -{ - return &ring_buffer[index * ring_buffer_length]; -} - -static float* stbir__add_empty_ring_buffer_entry(stbir__info* stbir_info, int n) -{ - int ring_buffer_index; - float* ring_buffer; - - stbir_info->ring_buffer_last_scanline = n; - - if (stbir_info->ring_buffer_begin_index < 0) - { - ring_buffer_index = stbir_info->ring_buffer_begin_index = 0; - stbir_info->ring_buffer_first_scanline = n; - } - else - { - ring_buffer_index = (stbir_info->ring_buffer_begin_index + (stbir_info->ring_buffer_last_scanline - stbir_info->ring_buffer_first_scanline)) % stbir_info->ring_buffer_num_entries; - STBIR_ASSERT(ring_buffer_index != stbir_info->ring_buffer_begin_index); - } - - ring_buffer = stbir__get_ring_buffer_entry(stbir_info->ring_buffer, ring_buffer_index, stbir_info->ring_buffer_length_bytes / sizeof(float)); - memset(ring_buffer, 0, stbir_info->ring_buffer_length_bytes); - - return ring_buffer; -} - - -static void stbir__resample_horizontal_upsample(stbir__info* stbir_info, float* output_buffer) -{ - int x, k; - int output_w = stbir_info->output_w; - int channels = stbir_info->channels; - float* decode_buffer = stbir__get_decode_buffer(stbir_info); - stbir__contributors* horizontal_contributors = stbir_info->horizontal_contributors; - float* horizontal_coefficients = stbir_info->horizontal_coefficients; - int coefficient_width = stbir_info->horizontal_coefficient_width; - - for (x = 0; x < output_w; x++) - { - int n0 = horizontal_contributors[x].n0; - int n1 = horizontal_contributors[x].n1; - - int out_pixel_index = x * channels; - int coefficient_group = coefficient_width * x; - int coefficient_counter = 0; - - STBIR_ASSERT(n1 >= n0); - STBIR_ASSERT(n0 >= -stbir_info->horizontal_filter_pixel_margin); - STBIR_ASSERT(n1 >= -stbir_info->horizontal_filter_pixel_margin); - STBIR_ASSERT(n0 < stbir_info->input_w + stbir_info->horizontal_filter_pixel_margin); - STBIR_ASSERT(n1 < stbir_info->input_w + stbir_info->horizontal_filter_pixel_margin); - - switch (channels) { - case 1: - for (k = n0; k <= n1; k++) - { - int in_pixel_index = k * 1; - float coefficient = horizontal_coefficients[coefficient_group + coefficient_counter++]; - STBIR_ASSERT(coefficient != 0); - output_buffer[out_pixel_index + 0] += decode_buffer[in_pixel_index + 0] * coefficient; - } - break; - case 2: - for (k = n0; k <= n1; k++) - { - int in_pixel_index = k * 2; - float coefficient = horizontal_coefficients[coefficient_group + coefficient_counter++]; - STBIR_ASSERT(coefficient != 0); - output_buffer[out_pixel_index + 0] += decode_buffer[in_pixel_index + 0] * coefficient; - output_buffer[out_pixel_index + 1] += decode_buffer[in_pixel_index + 1] * coefficient; - } - break; - case 3: - for (k = n0; k <= n1; k++) - { - int in_pixel_index = k * 3; - float coefficient = horizontal_coefficients[coefficient_group + coefficient_counter++]; - STBIR_ASSERT(coefficient != 0); - output_buffer[out_pixel_index + 0] += decode_buffer[in_pixel_index + 0] * coefficient; - output_buffer[out_pixel_index + 1] += decode_buffer[in_pixel_index + 1] * coefficient; - output_buffer[out_pixel_index + 2] += decode_buffer[in_pixel_index + 2] * coefficient; - } - break; - case 4: - for (k = n0; k <= n1; k++) - { - int in_pixel_index = k * 4; - float coefficient = horizontal_coefficients[coefficient_group + coefficient_counter++]; - STBIR_ASSERT(coefficient != 0); - output_buffer[out_pixel_index + 0] += decode_buffer[in_pixel_index + 0] * coefficient; - output_buffer[out_pixel_index + 1] += decode_buffer[in_pixel_index + 1] * coefficient; - output_buffer[out_pixel_index + 2] += decode_buffer[in_pixel_index + 2] * coefficient; - output_buffer[out_pixel_index + 3] += decode_buffer[in_pixel_index + 3] * coefficient; - } - break; - default: - for (k = n0; k <= n1; k++) - { - int in_pixel_index = k * channels; - float coefficient = horizontal_coefficients[coefficient_group + coefficient_counter++]; - int c; - STBIR_ASSERT(coefficient != 0); - for (c = 0; c < channels; c++) - output_buffer[out_pixel_index + c] += decode_buffer[in_pixel_index + c] * coefficient; - } - break; - } - } -} - -static void stbir__resample_horizontal_downsample(stbir__info* stbir_info, float* output_buffer) -{ - int x, k; - int input_w = stbir_info->input_w; - int channels = stbir_info->channels; - float* decode_buffer = stbir__get_decode_buffer(stbir_info); - stbir__contributors* horizontal_contributors = stbir_info->horizontal_contributors; - float* horizontal_coefficients = stbir_info->horizontal_coefficients; - int coefficient_width = stbir_info->horizontal_coefficient_width; - int filter_pixel_margin = stbir_info->horizontal_filter_pixel_margin; - int max_x = input_w + filter_pixel_margin * 2; - - STBIR_ASSERT(!stbir__use_width_upsampling(stbir_info)); - - switch (channels) { - case 1: - for (x = 0; x < max_x; x++) - { - int n0 = horizontal_contributors[x].n0; - int n1 = horizontal_contributors[x].n1; - - int in_x = x - filter_pixel_margin; - int in_pixel_index = in_x * 1; - int max_n = n1; - int coefficient_group = coefficient_width * x; - - for (k = n0; k <= max_n; k++) - { - int out_pixel_index = k * 1; - float coefficient = horizontal_coefficients[coefficient_group + k - n0]; - STBIR_ASSERT(coefficient != 0); - output_buffer[out_pixel_index + 0] += decode_buffer[in_pixel_index + 0] * coefficient; - } - } - break; - - case 2: - for (x = 0; x < max_x; x++) - { - int n0 = horizontal_contributors[x].n0; - int n1 = horizontal_contributors[x].n1; - - int in_x = x - filter_pixel_margin; - int in_pixel_index = in_x * 2; - int max_n = n1; - int coefficient_group = coefficient_width * x; - - for (k = n0; k <= max_n; k++) - { - int out_pixel_index = k * 2; - float coefficient = horizontal_coefficients[coefficient_group + k - n0]; - STBIR_ASSERT(coefficient != 0); - output_buffer[out_pixel_index + 0] += decode_buffer[in_pixel_index + 0] * coefficient; - output_buffer[out_pixel_index + 1] += decode_buffer[in_pixel_index + 1] * coefficient; - } - } - break; - - case 3: - for (x = 0; x < max_x; x++) - { - int n0 = horizontal_contributors[x].n0; - int n1 = horizontal_contributors[x].n1; - - int in_x = x - filter_pixel_margin; - int in_pixel_index = in_x * 3; - int max_n = n1; - int coefficient_group = coefficient_width * x; - - for (k = n0; k <= max_n; k++) - { - int out_pixel_index = k * 3; - float coefficient = horizontal_coefficients[coefficient_group + k - n0]; - STBIR_ASSERT(coefficient != 0); - output_buffer[out_pixel_index + 0] += decode_buffer[in_pixel_index + 0] * coefficient; - output_buffer[out_pixel_index + 1] += decode_buffer[in_pixel_index + 1] * coefficient; - output_buffer[out_pixel_index + 2] += decode_buffer[in_pixel_index + 2] * coefficient; - } - } - break; - - case 4: - for (x = 0; x < max_x; x++) - { - int n0 = horizontal_contributors[x].n0; - int n1 = horizontal_contributors[x].n1; - - int in_x = x - filter_pixel_margin; - int in_pixel_index = in_x * 4; - int max_n = n1; - int coefficient_group = coefficient_width * x; - - for (k = n0; k <= max_n; k++) - { - int out_pixel_index = k * 4; - float coefficient = horizontal_coefficients[coefficient_group + k - n0]; - STBIR_ASSERT(coefficient != 0); - output_buffer[out_pixel_index + 0] += decode_buffer[in_pixel_index + 0] * coefficient; - output_buffer[out_pixel_index + 1] += decode_buffer[in_pixel_index + 1] * coefficient; - output_buffer[out_pixel_index + 2] += decode_buffer[in_pixel_index + 2] * coefficient; - output_buffer[out_pixel_index + 3] += decode_buffer[in_pixel_index + 3] * coefficient; - } - } - break; - - default: - for (x = 0; x < max_x; x++) - { - int n0 = horizontal_contributors[x].n0; - int n1 = horizontal_contributors[x].n1; - - int in_x = x - filter_pixel_margin; - int in_pixel_index = in_x * channels; - int max_n = n1; - int coefficient_group = coefficient_width * x; - - for (k = n0; k <= max_n; k++) - { - int c; - int out_pixel_index = k * channels; - float coefficient = horizontal_coefficients[coefficient_group + k - n0]; - STBIR_ASSERT(coefficient != 0); - for (c = 0; c < channels; c++) - output_buffer[out_pixel_index + c] += decode_buffer[in_pixel_index + c] * coefficient; - } - } - break; - } -} - -static void stbir__decode_and_resample_upsample(stbir__info* stbir_info, int n) -{ - // Decode the nth scanline from the source image into the decode buffer. - stbir__decode_scanline(stbir_info, n); - - // Now resample it into the ring buffer. - if (stbir__use_width_upsampling(stbir_info)) - stbir__resample_horizontal_upsample(stbir_info, stbir__add_empty_ring_buffer_entry(stbir_info, n)); - else - stbir__resample_horizontal_downsample(stbir_info, stbir__add_empty_ring_buffer_entry(stbir_info, n)); - - // Now it's sitting in the ring buffer ready to be used as source for the vertical sampling. -} - -static void stbir__decode_and_resample_downsample(stbir__info* stbir_info, int n) -{ - // Decode the nth scanline from the source image into the decode buffer. - stbir__decode_scanline(stbir_info, n); - - memset(stbir_info->horizontal_buffer, 0, stbir_info->output_w * stbir_info->channels * sizeof(float)); - - // Now resample it into the horizontal buffer. - if (stbir__use_width_upsampling(stbir_info)) - stbir__resample_horizontal_upsample(stbir_info, stbir_info->horizontal_buffer); - else - stbir__resample_horizontal_downsample(stbir_info, stbir_info->horizontal_buffer); - - // Now it's sitting in the horizontal buffer ready to be distributed into the ring buffers. -} - -// Get the specified scan line from the ring buffer. -static float* stbir__get_ring_buffer_scanline(int get_scanline, float* ring_buffer, int begin_index, int first_scanline, int ring_buffer_num_entries, int ring_buffer_length) -{ - int ring_buffer_index = (begin_index + (get_scanline - first_scanline)) % ring_buffer_num_entries; - return stbir__get_ring_buffer_entry(ring_buffer, ring_buffer_index, ring_buffer_length); -} - - -static void stbir__encode_scanline(stbir__info* stbir_info, int num_pixels, void *output_buffer, float *encode_buffer, int channels, int alpha_channel, int decode) -{ - int x; - int n; - int num_nonalpha; - stbir_uint16 nonalpha[STBIR_MAX_CHANNELS]; - - if (!(stbir_info->flags&STBIR_FLAG_ALPHA_PREMULTIPLIED)) - { - for (x=0; x < num_pixels; ++x) - { - int pixel_index = x*channels; - - float alpha = encode_buffer[pixel_index + alpha_channel]; - float reciprocal_alpha = alpha ? 1.0f / alpha : 0; - - // unrolling this produced a 1% slowdown upscaling a large RGBA linear-space image on my machine - stb - for (n = 0; n < channels; n++) - if (n != alpha_channel) - encode_buffer[pixel_index + n] *= reciprocal_alpha; - - // We added in a small epsilon to prevent the color channel from being deleted with zero alpha. - // Because we only add it for integer types, it will automatically be discarded on integer - // conversion, so we don't need to subtract it back out (which would be problematic for - // numeric precision reasons). - } - } - - // build a table of all channels that need colorspace correction, so - // we don't perform colorspace correction on channels that don't need it. - for (x = 0, num_nonalpha = 0; x < channels; ++x) - { - if (x != alpha_channel || (stbir_info->flags & STBIR_FLAG_ALPHA_USES_COLORSPACE)) - { - nonalpha[num_nonalpha++] = (stbir_uint16)x; - } - } - - #define STBIR__ROUND_INT(f) ((int) ((f)+0.5)) - #define STBIR__ROUND_UINT(f) ((stbir_uint32) ((f)+0.5)) - - #ifdef STBIR__SATURATE_INT - #define STBIR__ENCODE_LINEAR8(f) stbir__saturate8 (STBIR__ROUND_INT((f) * stbir__max_uint8_as_float )) - #define STBIR__ENCODE_LINEAR16(f) stbir__saturate16(STBIR__ROUND_INT((f) * stbir__max_uint16_as_float)) - #else - #define STBIR__ENCODE_LINEAR8(f) (unsigned char ) STBIR__ROUND_INT(stbir__saturate(f) * stbir__max_uint8_as_float ) - #define STBIR__ENCODE_LINEAR16(f) (unsigned short) STBIR__ROUND_INT(stbir__saturate(f) * stbir__max_uint16_as_float) - #endif - - switch (decode) - { - case STBIR__DECODE(STBIR_TYPE_UINT8, STBIR_COLORSPACE_LINEAR): - for (x=0; x < num_pixels; ++x) - { - int pixel_index = x*channels; - - for (n = 0; n < channels; n++) - { - int index = pixel_index + n; - ((unsigned char*)output_buffer)[index] = STBIR__ENCODE_LINEAR8(encode_buffer[index]); - } - } - break; - - case STBIR__DECODE(STBIR_TYPE_UINT8, STBIR_COLORSPACE_SRGB): - for (x=0; x < num_pixels; ++x) - { - int pixel_index = x*channels; - - for (n = 0; n < num_nonalpha; n++) - { - int index = pixel_index + nonalpha[n]; - ((unsigned char*)output_buffer)[index] = stbir__linear_to_srgb_uchar(encode_buffer[index]); - } - - if (!(stbir_info->flags & STBIR_FLAG_ALPHA_USES_COLORSPACE)) - ((unsigned char *)output_buffer)[pixel_index + alpha_channel] = STBIR__ENCODE_LINEAR8(encode_buffer[pixel_index+alpha_channel]); - } - break; - - case STBIR__DECODE(STBIR_TYPE_UINT16, STBIR_COLORSPACE_LINEAR): - for (x=0; x < num_pixels; ++x) - { - int pixel_index = x*channels; - - for (n = 0; n < channels; n++) - { - int index = pixel_index + n; - ((unsigned short*)output_buffer)[index] = STBIR__ENCODE_LINEAR16(encode_buffer[index]); - } - } - break; - - case STBIR__DECODE(STBIR_TYPE_UINT16, STBIR_COLORSPACE_SRGB): - for (x=0; x < num_pixels; ++x) - { - int pixel_index = x*channels; - - for (n = 0; n < num_nonalpha; n++) - { - int index = pixel_index + nonalpha[n]; - ((unsigned short*)output_buffer)[index] = (unsigned short)STBIR__ROUND_INT(stbir__linear_to_srgb(stbir__saturate(encode_buffer[index])) * stbir__max_uint16_as_float); - } - - if (!(stbir_info->flags&STBIR_FLAG_ALPHA_USES_COLORSPACE)) - ((unsigned short*)output_buffer)[pixel_index + alpha_channel] = STBIR__ENCODE_LINEAR16(encode_buffer[pixel_index + alpha_channel]); - } - - break; - - case STBIR__DECODE(STBIR_TYPE_UINT32, STBIR_COLORSPACE_LINEAR): - for (x=0; x < num_pixels; ++x) - { - int pixel_index = x*channels; - - for (n = 0; n < channels; n++) - { - int index = pixel_index + n; - ((unsigned int*)output_buffer)[index] = (unsigned int)STBIR__ROUND_UINT(((double)stbir__saturate(encode_buffer[index])) * stbir__max_uint32_as_float); - } - } - break; - - case STBIR__DECODE(STBIR_TYPE_UINT32, STBIR_COLORSPACE_SRGB): - for (x=0; x < num_pixels; ++x) - { - int pixel_index = x*channels; - - for (n = 0; n < num_nonalpha; n++) - { - int index = pixel_index + nonalpha[n]; - ((unsigned int*)output_buffer)[index] = (unsigned int)STBIR__ROUND_UINT(((double)stbir__linear_to_srgb(stbir__saturate(encode_buffer[index]))) * stbir__max_uint32_as_float); - } - - if (!(stbir_info->flags&STBIR_FLAG_ALPHA_USES_COLORSPACE)) - ((unsigned int*)output_buffer)[pixel_index + alpha_channel] = (unsigned int)STBIR__ROUND_INT(((double)stbir__saturate(encode_buffer[pixel_index + alpha_channel])) * stbir__max_uint32_as_float); - } - break; - - case STBIR__DECODE(STBIR_TYPE_FLOAT, STBIR_COLORSPACE_LINEAR): - for (x=0; x < num_pixels; ++x) - { - int pixel_index = x*channels; - - for (n = 0; n < channels; n++) - { - int index = pixel_index + n; - ((float*)output_buffer)[index] = encode_buffer[index]; - } - } - break; - - case STBIR__DECODE(STBIR_TYPE_FLOAT, STBIR_COLORSPACE_SRGB): - for (x=0; x < num_pixels; ++x) - { - int pixel_index = x*channels; - - for (n = 0; n < num_nonalpha; n++) - { - int index = pixel_index + nonalpha[n]; - ((float*)output_buffer)[index] = stbir__linear_to_srgb(encode_buffer[index]); - } - - if (!(stbir_info->flags&STBIR_FLAG_ALPHA_USES_COLORSPACE)) - ((float*)output_buffer)[pixel_index + alpha_channel] = encode_buffer[pixel_index + alpha_channel]; - } - break; - - default: - STBIR_ASSERT(!"Unknown type/colorspace/channels combination."); - break; - } -} - -static void stbir__resample_vertical_upsample(stbir__info* stbir_info, int n) -{ - int x, k; - int output_w = stbir_info->output_w; - stbir__contributors* vertical_contributors = stbir_info->vertical_contributors; - float* vertical_coefficients = stbir_info->vertical_coefficients; - int channels = stbir_info->channels; - int alpha_channel = stbir_info->alpha_channel; - int type = stbir_info->type; - int colorspace = stbir_info->colorspace; - int ring_buffer_entries = stbir_info->ring_buffer_num_entries; - void* output_data = stbir_info->output_data; - float* encode_buffer = stbir_info->encode_buffer; - int decode = STBIR__DECODE(type, colorspace); - int coefficient_width = stbir_info->vertical_coefficient_width; - int coefficient_counter; - int contributor = n; - - float* ring_buffer = stbir_info->ring_buffer; - int ring_buffer_begin_index = stbir_info->ring_buffer_begin_index; - int ring_buffer_first_scanline = stbir_info->ring_buffer_first_scanline; - int ring_buffer_length = stbir_info->ring_buffer_length_bytes/sizeof(float); - - int n0,n1, output_row_start; - int coefficient_group = coefficient_width * contributor; - - n0 = vertical_contributors[contributor].n0; - n1 = vertical_contributors[contributor].n1; - - output_row_start = n * stbir_info->output_stride_bytes; - - STBIR_ASSERT(stbir__use_height_upsampling(stbir_info)); - - memset(encode_buffer, 0, output_w * sizeof(float) * channels); - - // I tried reblocking this for better cache usage of encode_buffer - // (using x_outer, k, x_inner), but it lost speed. -- stb - - coefficient_counter = 0; - switch (channels) { - case 1: - for (k = n0; k <= n1; k++) - { - int coefficient_index = coefficient_counter++; - float* ring_buffer_entry = stbir__get_ring_buffer_scanline(k, ring_buffer, ring_buffer_begin_index, ring_buffer_first_scanline, ring_buffer_entries, ring_buffer_length); - float coefficient = vertical_coefficients[coefficient_group + coefficient_index]; - for (x = 0; x < output_w; ++x) - { - int in_pixel_index = x * 1; - encode_buffer[in_pixel_index + 0] += ring_buffer_entry[in_pixel_index + 0] * coefficient; - } - } - break; - case 2: - for (k = n0; k <= n1; k++) - { - int coefficient_index = coefficient_counter++; - float* ring_buffer_entry = stbir__get_ring_buffer_scanline(k, ring_buffer, ring_buffer_begin_index, ring_buffer_first_scanline, ring_buffer_entries, ring_buffer_length); - float coefficient = vertical_coefficients[coefficient_group + coefficient_index]; - for (x = 0; x < output_w; ++x) - { - int in_pixel_index = x * 2; - encode_buffer[in_pixel_index + 0] += ring_buffer_entry[in_pixel_index + 0] * coefficient; - encode_buffer[in_pixel_index + 1] += ring_buffer_entry[in_pixel_index + 1] * coefficient; - } - } - break; - case 3: - for (k = n0; k <= n1; k++) - { - int coefficient_index = coefficient_counter++; - float* ring_buffer_entry = stbir__get_ring_buffer_scanline(k, ring_buffer, ring_buffer_begin_index, ring_buffer_first_scanline, ring_buffer_entries, ring_buffer_length); - float coefficient = vertical_coefficients[coefficient_group + coefficient_index]; - for (x = 0; x < output_w; ++x) - { - int in_pixel_index = x * 3; - encode_buffer[in_pixel_index + 0] += ring_buffer_entry[in_pixel_index + 0] * coefficient; - encode_buffer[in_pixel_index + 1] += ring_buffer_entry[in_pixel_index + 1] * coefficient; - encode_buffer[in_pixel_index + 2] += ring_buffer_entry[in_pixel_index + 2] * coefficient; - } - } - break; - case 4: - for (k = n0; k <= n1; k++) - { - int coefficient_index = coefficient_counter++; - float* ring_buffer_entry = stbir__get_ring_buffer_scanline(k, ring_buffer, ring_buffer_begin_index, ring_buffer_first_scanline, ring_buffer_entries, ring_buffer_length); - float coefficient = vertical_coefficients[coefficient_group + coefficient_index]; - for (x = 0; x < output_w; ++x) - { - int in_pixel_index = x * 4; - encode_buffer[in_pixel_index + 0] += ring_buffer_entry[in_pixel_index + 0] * coefficient; - encode_buffer[in_pixel_index + 1] += ring_buffer_entry[in_pixel_index + 1] * coefficient; - encode_buffer[in_pixel_index + 2] += ring_buffer_entry[in_pixel_index + 2] * coefficient; - encode_buffer[in_pixel_index + 3] += ring_buffer_entry[in_pixel_index + 3] * coefficient; - } - } - break; - default: - for (k = n0; k <= n1; k++) - { - int coefficient_index = coefficient_counter++; - float* ring_buffer_entry = stbir__get_ring_buffer_scanline(k, ring_buffer, ring_buffer_begin_index, ring_buffer_first_scanline, ring_buffer_entries, ring_buffer_length); - float coefficient = vertical_coefficients[coefficient_group + coefficient_index]; - for (x = 0; x < output_w; ++x) - { - int in_pixel_index = x * channels; - int c; - for (c = 0; c < channels; c++) - encode_buffer[in_pixel_index + c] += ring_buffer_entry[in_pixel_index + c] * coefficient; - } - } - break; - } - stbir__encode_scanline(stbir_info, output_w, (char *) output_data + output_row_start, encode_buffer, channels, alpha_channel, decode); -} - -static void stbir__resample_vertical_downsample(stbir__info* stbir_info, int n) -{ - int x, k; - int output_w = stbir_info->output_w; - stbir__contributors* vertical_contributors = stbir_info->vertical_contributors; - float* vertical_coefficients = stbir_info->vertical_coefficients; - int channels = stbir_info->channels; - int ring_buffer_entries = stbir_info->ring_buffer_num_entries; - float* horizontal_buffer = stbir_info->horizontal_buffer; - int coefficient_width = stbir_info->vertical_coefficient_width; - int contributor = n + stbir_info->vertical_filter_pixel_margin; - - float* ring_buffer = stbir_info->ring_buffer; - int ring_buffer_begin_index = stbir_info->ring_buffer_begin_index; - int ring_buffer_first_scanline = stbir_info->ring_buffer_first_scanline; - int ring_buffer_length = stbir_info->ring_buffer_length_bytes/sizeof(float); - int n0,n1; - - n0 = vertical_contributors[contributor].n0; - n1 = vertical_contributors[contributor].n1; - - STBIR_ASSERT(!stbir__use_height_upsampling(stbir_info)); - - for (k = n0; k <= n1; k++) - { - int coefficient_index = k - n0; - int coefficient_group = coefficient_width * contributor; - float coefficient = vertical_coefficients[coefficient_group + coefficient_index]; - - float* ring_buffer_entry = stbir__get_ring_buffer_scanline(k, ring_buffer, ring_buffer_begin_index, ring_buffer_first_scanline, ring_buffer_entries, ring_buffer_length); - - switch (channels) { - case 1: - for (x = 0; x < output_w; x++) - { - int in_pixel_index = x * 1; - ring_buffer_entry[in_pixel_index + 0] += horizontal_buffer[in_pixel_index + 0] * coefficient; - } - break; - case 2: - for (x = 0; x < output_w; x++) - { - int in_pixel_index = x * 2; - ring_buffer_entry[in_pixel_index + 0] += horizontal_buffer[in_pixel_index + 0] * coefficient; - ring_buffer_entry[in_pixel_index + 1] += horizontal_buffer[in_pixel_index + 1] * coefficient; - } - break; - case 3: - for (x = 0; x < output_w; x++) - { - int in_pixel_index = x * 3; - ring_buffer_entry[in_pixel_index + 0] += horizontal_buffer[in_pixel_index + 0] * coefficient; - ring_buffer_entry[in_pixel_index + 1] += horizontal_buffer[in_pixel_index + 1] * coefficient; - ring_buffer_entry[in_pixel_index + 2] += horizontal_buffer[in_pixel_index + 2] * coefficient; - } - break; - case 4: - for (x = 0; x < output_w; x++) - { - int in_pixel_index = x * 4; - ring_buffer_entry[in_pixel_index + 0] += horizontal_buffer[in_pixel_index + 0] * coefficient; - ring_buffer_entry[in_pixel_index + 1] += horizontal_buffer[in_pixel_index + 1] * coefficient; - ring_buffer_entry[in_pixel_index + 2] += horizontal_buffer[in_pixel_index + 2] * coefficient; - ring_buffer_entry[in_pixel_index + 3] += horizontal_buffer[in_pixel_index + 3] * coefficient; - } - break; - default: - for (x = 0; x < output_w; x++) - { - int in_pixel_index = x * channels; - - int c; - for (c = 0; c < channels; c++) - ring_buffer_entry[in_pixel_index + c] += horizontal_buffer[in_pixel_index + c] * coefficient; - } - break; - } - } -} - -static void stbir__buffer_loop_upsample(stbir__info* stbir_info) -{ - int y; - float scale_ratio = stbir_info->vertical_scale; - float out_scanlines_radius = stbir__filter_info_table[stbir_info->vertical_filter].support(1/scale_ratio) * scale_ratio; - - STBIR_ASSERT(stbir__use_height_upsampling(stbir_info)); - - for (y = 0; y < stbir_info->output_h; y++) - { - float in_center_of_out = 0; // Center of the current out scanline in the in scanline space - int in_first_scanline = 0, in_last_scanline = 0; - - stbir__calculate_sample_range_upsample(y, out_scanlines_radius, scale_ratio, stbir_info->vertical_shift, &in_first_scanline, &in_last_scanline, &in_center_of_out); - - STBIR_ASSERT(in_last_scanline - in_first_scanline + 1 <= stbir_info->ring_buffer_num_entries); - - if (stbir_info->ring_buffer_begin_index >= 0) - { - // Get rid of whatever we don't need anymore. - while (in_first_scanline > stbir_info->ring_buffer_first_scanline) - { - if (stbir_info->ring_buffer_first_scanline == stbir_info->ring_buffer_last_scanline) - { - // We just popped the last scanline off the ring buffer. - // Reset it to the empty state. - stbir_info->ring_buffer_begin_index = -1; - stbir_info->ring_buffer_first_scanline = 0; - stbir_info->ring_buffer_last_scanline = 0; - break; - } - else - { - stbir_info->ring_buffer_first_scanline++; - stbir_info->ring_buffer_begin_index = (stbir_info->ring_buffer_begin_index + 1) % stbir_info->ring_buffer_num_entries; - } - } - } - - // Load in new ones. - if (stbir_info->ring_buffer_begin_index < 0) - stbir__decode_and_resample_upsample(stbir_info, in_first_scanline); - - while (in_last_scanline > stbir_info->ring_buffer_last_scanline) - stbir__decode_and_resample_upsample(stbir_info, stbir_info->ring_buffer_last_scanline + 1); - - // Now all buffers should be ready to write a row of vertical sampling. - stbir__resample_vertical_upsample(stbir_info, y); - - STBIR_PROGRESS_REPORT((float)y / stbir_info->output_h); - } -} - -static void stbir__empty_ring_buffer(stbir__info* stbir_info, int first_necessary_scanline) -{ - int output_stride_bytes = stbir_info->output_stride_bytes; - int channels = stbir_info->channels; - int alpha_channel = stbir_info->alpha_channel; - int type = stbir_info->type; - int colorspace = stbir_info->colorspace; - int output_w = stbir_info->output_w; - void* output_data = stbir_info->output_data; - int decode = STBIR__DECODE(type, colorspace); - - float* ring_buffer = stbir_info->ring_buffer; - int ring_buffer_length = stbir_info->ring_buffer_length_bytes/sizeof(float); - - if (stbir_info->ring_buffer_begin_index >= 0) - { - // Get rid of whatever we don't need anymore. - while (first_necessary_scanline > stbir_info->ring_buffer_first_scanline) - { - if (stbir_info->ring_buffer_first_scanline >= 0 && stbir_info->ring_buffer_first_scanline < stbir_info->output_h) - { - int output_row_start = stbir_info->ring_buffer_first_scanline * output_stride_bytes; - float* ring_buffer_entry = stbir__get_ring_buffer_entry(ring_buffer, stbir_info->ring_buffer_begin_index, ring_buffer_length); - stbir__encode_scanline(stbir_info, output_w, (char *) output_data + output_row_start, ring_buffer_entry, channels, alpha_channel, decode); - STBIR_PROGRESS_REPORT((float)stbir_info->ring_buffer_first_scanline / stbir_info->output_h); - } - - if (stbir_info->ring_buffer_first_scanline == stbir_info->ring_buffer_last_scanline) - { - // We just popped the last scanline off the ring buffer. - // Reset it to the empty state. - stbir_info->ring_buffer_begin_index = -1; - stbir_info->ring_buffer_first_scanline = 0; - stbir_info->ring_buffer_last_scanline = 0; - break; - } - else - { - stbir_info->ring_buffer_first_scanline++; - stbir_info->ring_buffer_begin_index = (stbir_info->ring_buffer_begin_index + 1) % stbir_info->ring_buffer_num_entries; - } - } - } -} - -static void stbir__buffer_loop_downsample(stbir__info* stbir_info) -{ - int y; - float scale_ratio = stbir_info->vertical_scale; - int output_h = stbir_info->output_h; - float in_pixels_radius = stbir__filter_info_table[stbir_info->vertical_filter].support(scale_ratio) / scale_ratio; - int pixel_margin = stbir_info->vertical_filter_pixel_margin; - int max_y = stbir_info->input_h + pixel_margin; - - STBIR_ASSERT(!stbir__use_height_upsampling(stbir_info)); - - for (y = -pixel_margin; y < max_y; y++) - { - float out_center_of_in; // Center of the current out scanline in the in scanline space - int out_first_scanline, out_last_scanline; - - stbir__calculate_sample_range_downsample(y, in_pixels_radius, scale_ratio, stbir_info->vertical_shift, &out_first_scanline, &out_last_scanline, &out_center_of_in); - - STBIR_ASSERT(out_last_scanline - out_first_scanline + 1 <= stbir_info->ring_buffer_num_entries); - - if (out_last_scanline < 0 || out_first_scanline >= output_h) - continue; - - stbir__empty_ring_buffer(stbir_info, out_first_scanline); - - stbir__decode_and_resample_downsample(stbir_info, y); - - // Load in new ones. - if (stbir_info->ring_buffer_begin_index < 0) - stbir__add_empty_ring_buffer_entry(stbir_info, out_first_scanline); - - while (out_last_scanline > stbir_info->ring_buffer_last_scanline) - stbir__add_empty_ring_buffer_entry(stbir_info, stbir_info->ring_buffer_last_scanline + 1); - - // Now the horizontal buffer is ready to write to all ring buffer rows. - stbir__resample_vertical_downsample(stbir_info, y); - } - - stbir__empty_ring_buffer(stbir_info, stbir_info->output_h); -} - -static void stbir__setup(stbir__info *info, int input_w, int input_h, int output_w, int output_h, int channels) -{ - info->input_w = input_w; - info->input_h = input_h; - info->output_w = output_w; - info->output_h = output_h; - info->channels = channels; -} - -static void stbir__calculate_transform(stbir__info *info, float s0, float t0, float s1, float t1, float *transform) -{ - info->s0 = s0; - info->t0 = t0; - info->s1 = s1; - info->t1 = t1; - - if (transform) - { - info->horizontal_scale = transform[0]; - info->vertical_scale = transform[1]; - info->horizontal_shift = transform[2]; - info->vertical_shift = transform[3]; - } - else - { - info->horizontal_scale = ((float)info->output_w / info->input_w) / (s1 - s0); - info->vertical_scale = ((float)info->output_h / info->input_h) / (t1 - t0); - - info->horizontal_shift = s0 * info->output_w / (s1 - s0); - info->vertical_shift = t0 * info->output_h / (t1 - t0); - } -} - -static void stbir__choose_filter(stbir__info *info, stbir_filter h_filter, stbir_filter v_filter) -{ - if (h_filter == 0) - h_filter = stbir__use_upsampling(info->horizontal_scale) ? STBIR_DEFAULT_FILTER_UPSAMPLE : STBIR_DEFAULT_FILTER_DOWNSAMPLE; - if (v_filter == 0) - v_filter = stbir__use_upsampling(info->vertical_scale) ? STBIR_DEFAULT_FILTER_UPSAMPLE : STBIR_DEFAULT_FILTER_DOWNSAMPLE; - info->horizontal_filter = h_filter; - info->vertical_filter = v_filter; -} - -static stbir_uint32 stbir__calculate_memory(stbir__info *info) -{ - int pixel_margin = stbir__get_filter_pixel_margin(info->horizontal_filter, info->horizontal_scale); - int filter_height = stbir__get_filter_pixel_width(info->vertical_filter, info->vertical_scale); - - info->horizontal_num_contributors = stbir__get_contributors(info->horizontal_scale, info->horizontal_filter, info->input_w, info->output_w); - info->vertical_num_contributors = stbir__get_contributors(info->vertical_scale , info->vertical_filter , info->input_h, info->output_h); - - // One extra entry because floating point precision problems sometimes cause an extra to be necessary. - info->ring_buffer_num_entries = filter_height + 1; - - info->horizontal_contributors_size = info->horizontal_num_contributors * sizeof(stbir__contributors); - info->horizontal_coefficients_size = stbir__get_total_horizontal_coefficients(info) * sizeof(float); - info->vertical_contributors_size = info->vertical_num_contributors * sizeof(stbir__contributors); - info->vertical_coefficients_size = stbir__get_total_vertical_coefficients(info) * sizeof(float); - info->decode_buffer_size = (info->input_w + pixel_margin * 2) * info->channels * sizeof(float); - info->horizontal_buffer_size = info->output_w * info->channels * sizeof(float); - info->ring_buffer_size = info->output_w * info->channels * info->ring_buffer_num_entries * sizeof(float); - info->encode_buffer_size = info->output_w * info->channels * sizeof(float); - - STBIR_ASSERT(info->horizontal_filter != 0); - STBIR_ASSERT(info->horizontal_filter < STBIR__ARRAY_SIZE(stbir__filter_info_table)); // this now happens too late - STBIR_ASSERT(info->vertical_filter != 0); - STBIR_ASSERT(info->vertical_filter < STBIR__ARRAY_SIZE(stbir__filter_info_table)); // this now happens too late - - if (stbir__use_height_upsampling(info)) - // The horizontal buffer is for when we're downsampling the height and we - // can't output the result of sampling the decode buffer directly into the - // ring buffers. - info->horizontal_buffer_size = 0; - else - // The encode buffer is to retain precision in the height upsampling method - // and isn't used when height downsampling. - info->encode_buffer_size = 0; - - return info->horizontal_contributors_size + info->horizontal_coefficients_size - + info->vertical_contributors_size + info->vertical_coefficients_size - + info->decode_buffer_size + info->horizontal_buffer_size - + info->ring_buffer_size + info->encode_buffer_size; -} - -static int stbir__resize_allocated(stbir__info *info, - const void* input_data, int input_stride_in_bytes, - void* output_data, int output_stride_in_bytes, - int alpha_channel, stbir_uint32 flags, stbir_datatype type, - stbir_edge edge_horizontal, stbir_edge edge_vertical, stbir_colorspace colorspace, - void* tempmem, size_t tempmem_size_in_bytes) -{ - size_t memory_required = stbir__calculate_memory(info); - - int width_stride_input = input_stride_in_bytes ? input_stride_in_bytes : info->channels * info->input_w * stbir__type_size[type]; - int width_stride_output = output_stride_in_bytes ? output_stride_in_bytes : info->channels * info->output_w * stbir__type_size[type]; - -#ifdef STBIR_DEBUG_OVERWRITE_TEST -#define OVERWRITE_ARRAY_SIZE 8 - unsigned char overwrite_output_before_pre[OVERWRITE_ARRAY_SIZE]; - unsigned char overwrite_tempmem_before_pre[OVERWRITE_ARRAY_SIZE]; - unsigned char overwrite_output_after_pre[OVERWRITE_ARRAY_SIZE]; - unsigned char overwrite_tempmem_after_pre[OVERWRITE_ARRAY_SIZE]; - - size_t begin_forbidden = width_stride_output * (info->output_h - 1) + info->output_w * info->channels * stbir__type_size[type]; - memcpy(overwrite_output_before_pre, &((unsigned char*)output_data)[-OVERWRITE_ARRAY_SIZE], OVERWRITE_ARRAY_SIZE); - memcpy(overwrite_output_after_pre, &((unsigned char*)output_data)[begin_forbidden], OVERWRITE_ARRAY_SIZE); - memcpy(overwrite_tempmem_before_pre, &((unsigned char*)tempmem)[-OVERWRITE_ARRAY_SIZE], OVERWRITE_ARRAY_SIZE); - memcpy(overwrite_tempmem_after_pre, &((unsigned char*)tempmem)[tempmem_size_in_bytes], OVERWRITE_ARRAY_SIZE); -#endif - - STBIR_ASSERT(info->channels >= 0); - STBIR_ASSERT(info->channels <= STBIR_MAX_CHANNELS); - - if (info->channels < 0 || info->channels > STBIR_MAX_CHANNELS) - return 0; - - STBIR_ASSERT(info->horizontal_filter < STBIR__ARRAY_SIZE(stbir__filter_info_table)); - STBIR_ASSERT(info->vertical_filter < STBIR__ARRAY_SIZE(stbir__filter_info_table)); - - if (info->horizontal_filter >= STBIR__ARRAY_SIZE(stbir__filter_info_table)) - return 0; - if (info->vertical_filter >= STBIR__ARRAY_SIZE(stbir__filter_info_table)) - return 0; - - if (alpha_channel < 0) - flags |= STBIR_FLAG_ALPHA_USES_COLORSPACE | STBIR_FLAG_ALPHA_PREMULTIPLIED; - - if (!(flags&STBIR_FLAG_ALPHA_USES_COLORSPACE) || !(flags&STBIR_FLAG_ALPHA_PREMULTIPLIED)) - STBIR_ASSERT(alpha_channel >= 0 && alpha_channel < info->channels); - - if (alpha_channel >= info->channels) - return 0; - - STBIR_ASSERT(tempmem); - - if (!tempmem) - return 0; - - STBIR_ASSERT(tempmem_size_in_bytes >= memory_required); - - if (tempmem_size_in_bytes < memory_required) - return 0; - - memset(tempmem, 0, tempmem_size_in_bytes); - - info->input_data = input_data; - info->input_stride_bytes = width_stride_input; - - info->output_data = output_data; - info->output_stride_bytes = width_stride_output; - - info->alpha_channel = alpha_channel; - info->flags = flags; - info->type = type; - info->edge_horizontal = edge_horizontal; - info->edge_vertical = edge_vertical; - info->colorspace = colorspace; - - info->horizontal_coefficient_width = stbir__get_coefficient_width (info->horizontal_filter, info->horizontal_scale); - info->vertical_coefficient_width = stbir__get_coefficient_width (info->vertical_filter , info->vertical_scale ); - info->horizontal_filter_pixel_width = stbir__get_filter_pixel_width (info->horizontal_filter, info->horizontal_scale); - info->vertical_filter_pixel_width = stbir__get_filter_pixel_width (info->vertical_filter , info->vertical_scale ); - info->horizontal_filter_pixel_margin = stbir__get_filter_pixel_margin(info->horizontal_filter, info->horizontal_scale); - info->vertical_filter_pixel_margin = stbir__get_filter_pixel_margin(info->vertical_filter , info->vertical_scale ); - - info->ring_buffer_length_bytes = info->output_w * info->channels * sizeof(float); - info->decode_buffer_pixels = info->input_w + info->horizontal_filter_pixel_margin * 2; - -#define STBIR__NEXT_MEMPTR(current, newtype) (newtype*)(((unsigned char*)current) + current##_size) - - info->horizontal_contributors = (stbir__contributors *) tempmem; - info->horizontal_coefficients = STBIR__NEXT_MEMPTR(info->horizontal_contributors, float); - info->vertical_contributors = STBIR__NEXT_MEMPTR(info->horizontal_coefficients, stbir__contributors); - info->vertical_coefficients = STBIR__NEXT_MEMPTR(info->vertical_contributors, float); - info->decode_buffer = STBIR__NEXT_MEMPTR(info->vertical_coefficients, float); - - if (stbir__use_height_upsampling(info)) - { - info->horizontal_buffer = NULL; - info->ring_buffer = STBIR__NEXT_MEMPTR(info->decode_buffer, float); - info->encode_buffer = STBIR__NEXT_MEMPTR(info->ring_buffer, float); - - STBIR_ASSERT((size_t)STBIR__NEXT_MEMPTR(info->encode_buffer, unsigned char) == (size_t)tempmem + tempmem_size_in_bytes); - } - else - { - info->horizontal_buffer = STBIR__NEXT_MEMPTR(info->decode_buffer, float); - info->ring_buffer = STBIR__NEXT_MEMPTR(info->horizontal_buffer, float); - info->encode_buffer = NULL; - - STBIR_ASSERT((size_t)STBIR__NEXT_MEMPTR(info->ring_buffer, unsigned char) == (size_t)tempmem + tempmem_size_in_bytes); - } - -#undef STBIR__NEXT_MEMPTR - - // This signals that the ring buffer is empty - info->ring_buffer_begin_index = -1; - - stbir__calculate_filters(info->horizontal_contributors, info->horizontal_coefficients, info->horizontal_filter, info->horizontal_scale, info->horizontal_shift, info->input_w, info->output_w); - stbir__calculate_filters(info->vertical_contributors, info->vertical_coefficients, info->vertical_filter, info->vertical_scale, info->vertical_shift, info->input_h, info->output_h); - - STBIR_PROGRESS_REPORT(0); - - if (stbir__use_height_upsampling(info)) - stbir__buffer_loop_upsample(info); - else - stbir__buffer_loop_downsample(info); - - STBIR_PROGRESS_REPORT(1); - -#ifdef STBIR_DEBUG_OVERWRITE_TEST - STBIR_ASSERT(memcmp(overwrite_output_before_pre, &((unsigned char*)output_data)[-OVERWRITE_ARRAY_SIZE], OVERWRITE_ARRAY_SIZE) == 0); - STBIR_ASSERT(memcmp(overwrite_output_after_pre, &((unsigned char*)output_data)[begin_forbidden], OVERWRITE_ARRAY_SIZE) == 0); - STBIR_ASSERT(memcmp(overwrite_tempmem_before_pre, &((unsigned char*)tempmem)[-OVERWRITE_ARRAY_SIZE], OVERWRITE_ARRAY_SIZE) == 0); - STBIR_ASSERT(memcmp(overwrite_tempmem_after_pre, &((unsigned char*)tempmem)[tempmem_size_in_bytes], OVERWRITE_ARRAY_SIZE) == 0); -#endif - - return 1; -} - - -static int stbir__resize_arbitrary( - void *alloc_context, - const void* input_data, int input_w, int input_h, int input_stride_in_bytes, - void* output_data, int output_w, int output_h, int output_stride_in_bytes, - float s0, float t0, float s1, float t1, float *transform, - int channels, int alpha_channel, stbir_uint32 flags, stbir_datatype type, - stbir_filter h_filter, stbir_filter v_filter, - stbir_edge edge_horizontal, stbir_edge edge_vertical, stbir_colorspace colorspace) -{ - stbir__info info; - int result; - size_t memory_required; - void* extra_memory; - - stbir__setup(&info, input_w, input_h, output_w, output_h, channels); - stbir__calculate_transform(&info, s0,t0,s1,t1,transform); - stbir__choose_filter(&info, h_filter, v_filter); - memory_required = stbir__calculate_memory(&info); - extra_memory = STBIR_MALLOC(memory_required, alloc_context); - - if (!extra_memory) - return 0; - - result = stbir__resize_allocated(&info, input_data, input_stride_in_bytes, - output_data, output_stride_in_bytes, - alpha_channel, flags, type, - edge_horizontal, edge_vertical, - colorspace, extra_memory, memory_required); - - STBIR_FREE(extra_memory, alloc_context); - - return result; -} - -STBIRDEF int stbir_resize_uint8( const unsigned char *input_pixels , int input_w , int input_h , int input_stride_in_bytes, - unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, - int num_channels) -{ - return stbir__resize_arbitrary(NULL, input_pixels, input_w, input_h, input_stride_in_bytes, - output_pixels, output_w, output_h, output_stride_in_bytes, - 0,0,1,1,NULL,num_channels,-1,0, STBIR_TYPE_UINT8, STBIR_FILTER_DEFAULT, STBIR_FILTER_DEFAULT, - STBIR_EDGE_CLAMP, STBIR_EDGE_CLAMP, STBIR_COLORSPACE_LINEAR); -} - -STBIRDEF int stbir_resize_float( const float *input_pixels , int input_w , int input_h , int input_stride_in_bytes, - float *output_pixels, int output_w, int output_h, int output_stride_in_bytes, - int num_channels) -{ - return stbir__resize_arbitrary(NULL, input_pixels, input_w, input_h, input_stride_in_bytes, - output_pixels, output_w, output_h, output_stride_in_bytes, - 0,0,1,1,NULL,num_channels,-1,0, STBIR_TYPE_FLOAT, STBIR_FILTER_DEFAULT, STBIR_FILTER_DEFAULT, - STBIR_EDGE_CLAMP, STBIR_EDGE_CLAMP, STBIR_COLORSPACE_LINEAR); -} - -STBIRDEF int stbir_resize_uint8_srgb(const unsigned char *input_pixels , int input_w , int input_h , int input_stride_in_bytes, - unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, - int num_channels, int alpha_channel, int flags) -{ - return stbir__resize_arbitrary(NULL, input_pixels, input_w, input_h, input_stride_in_bytes, - output_pixels, output_w, output_h, output_stride_in_bytes, - 0,0,1,1,NULL,num_channels,alpha_channel,flags, STBIR_TYPE_UINT8, STBIR_FILTER_DEFAULT, STBIR_FILTER_DEFAULT, - STBIR_EDGE_CLAMP, STBIR_EDGE_CLAMP, STBIR_COLORSPACE_SRGB); -} - -STBIRDEF int stbir_resize_uint8_srgb_edgemode(const unsigned char *input_pixels , int input_w , int input_h , int input_stride_in_bytes, - unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, - int num_channels, int alpha_channel, int flags, - stbir_edge edge_wrap_mode) -{ - return stbir__resize_arbitrary(NULL, input_pixels, input_w, input_h, input_stride_in_bytes, - output_pixels, output_w, output_h, output_stride_in_bytes, - 0,0,1,1,NULL,num_channels,alpha_channel,flags, STBIR_TYPE_UINT8, STBIR_FILTER_DEFAULT, STBIR_FILTER_DEFAULT, - edge_wrap_mode, edge_wrap_mode, STBIR_COLORSPACE_SRGB); -} - -STBIRDEF int stbir_resize_uint8_generic( const unsigned char *input_pixels , int input_w , int input_h , int input_stride_in_bytes, - unsigned char *output_pixels, int output_w, int output_h, int output_stride_in_bytes, - int num_channels, int alpha_channel, int flags, - stbir_edge edge_wrap_mode, stbir_filter filter, stbir_colorspace space, - void *alloc_context) -{ - return stbir__resize_arbitrary(alloc_context, input_pixels, input_w, input_h, input_stride_in_bytes, - output_pixels, output_w, output_h, output_stride_in_bytes, - 0,0,1,1,NULL,num_channels,alpha_channel,flags, STBIR_TYPE_UINT8, filter, filter, - edge_wrap_mode, edge_wrap_mode, space); -} - -STBIRDEF int stbir_resize_uint16_generic(const stbir_uint16 *input_pixels , int input_w , int input_h , int input_stride_in_bytes, - stbir_uint16 *output_pixels , int output_w, int output_h, int output_stride_in_bytes, - int num_channels, int alpha_channel, int flags, - stbir_edge edge_wrap_mode, stbir_filter filter, stbir_colorspace space, - void *alloc_context) -{ - return stbir__resize_arbitrary(alloc_context, input_pixels, input_w, input_h, input_stride_in_bytes, - output_pixels, output_w, output_h, output_stride_in_bytes, - 0,0,1,1,NULL,num_channels,alpha_channel,flags, STBIR_TYPE_UINT16, filter, filter, - edge_wrap_mode, edge_wrap_mode, space); -} - - -STBIRDEF int stbir_resize_float_generic( const float *input_pixels , int input_w , int input_h , int input_stride_in_bytes, - float *output_pixels , int output_w, int output_h, int output_stride_in_bytes, - int num_channels, int alpha_channel, int flags, - stbir_edge edge_wrap_mode, stbir_filter filter, stbir_colorspace space, - void *alloc_context) -{ - return stbir__resize_arbitrary(alloc_context, input_pixels, input_w, input_h, input_stride_in_bytes, - output_pixels, output_w, output_h, output_stride_in_bytes, - 0,0,1,1,NULL,num_channels,alpha_channel,flags, STBIR_TYPE_FLOAT, filter, filter, - edge_wrap_mode, edge_wrap_mode, space); -} - - -STBIRDEF int stbir_resize( const void *input_pixels , int input_w , int input_h , int input_stride_in_bytes, - void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, - stbir_datatype datatype, - int num_channels, int alpha_channel, int flags, - stbir_edge edge_mode_horizontal, stbir_edge edge_mode_vertical, - stbir_filter filter_horizontal, stbir_filter filter_vertical, - stbir_colorspace space, void *alloc_context) -{ - return stbir__resize_arbitrary(alloc_context, input_pixels, input_w, input_h, input_stride_in_bytes, - output_pixels, output_w, output_h, output_stride_in_bytes, - 0,0,1,1,NULL,num_channels,alpha_channel,flags, datatype, filter_horizontal, filter_vertical, - edge_mode_horizontal, edge_mode_vertical, space); -} - - -STBIRDEF int stbir_resize_subpixel(const void *input_pixels , int input_w , int input_h , int input_stride_in_bytes, - void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, - stbir_datatype datatype, - int num_channels, int alpha_channel, int flags, - stbir_edge edge_mode_horizontal, stbir_edge edge_mode_vertical, - stbir_filter filter_horizontal, stbir_filter filter_vertical, - stbir_colorspace space, void *alloc_context, - float x_scale, float y_scale, - float x_offset, float y_offset) -{ - float transform[4]; - transform[0] = x_scale; - transform[1] = y_scale; - transform[2] = x_offset; - transform[3] = y_offset; - return stbir__resize_arbitrary(alloc_context, input_pixels, input_w, input_h, input_stride_in_bytes, - output_pixels, output_w, output_h, output_stride_in_bytes, - 0,0,1,1,transform,num_channels,alpha_channel,flags, datatype, filter_horizontal, filter_vertical, - edge_mode_horizontal, edge_mode_vertical, space); -} - -STBIRDEF int stbir_resize_region( const void *input_pixels , int input_w , int input_h , int input_stride_in_bytes, - void *output_pixels, int output_w, int output_h, int output_stride_in_bytes, - stbir_datatype datatype, - int num_channels, int alpha_channel, int flags, - stbir_edge edge_mode_horizontal, stbir_edge edge_mode_vertical, - stbir_filter filter_horizontal, stbir_filter filter_vertical, - stbir_colorspace space, void *alloc_context, - float s0, float t0, float s1, float t1) -{ - return stbir__resize_arbitrary(alloc_context, input_pixels, input_w, input_h, input_stride_in_bytes, - output_pixels, output_w, output_h, output_stride_in_bytes, - s0,t0,s1,t1,NULL,num_channels,alpha_channel,flags, datatype, filter_horizontal, filter_vertical, - edge_mode_horizontal, edge_mode_vertical, space); -} - -#endif // STB_IMAGE_RESIZE_IMPLEMENTATION - -/* ------------------------------------------------------------------------------- -This software is available under 2 licenses -- choose whichever you prefer. ------------------------------------------------------------------------------- -ALTERNATIVE A - MIT License -Copyright (c) 2017 Sean Barrett -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. ------------------------------------------------------------------------------- -ALTERNATIVE B - Public Domain (www.unlicense.org) -This is free and unencumbered software released into the public domain. -Anyone is free to copy, modify, publish, use, compile, sell, or distribute this -software, either in source code form or as a compiled binary, for any purpose, -commercial or non-commercial, and by any means. -In jurisdictions that recognize copyright laws, the author or authors of this -software dedicate any and all copyright interest in the software to the public -domain. We make this dedication for the benefit of the public at large and to -the detriment of our heirs and successors. We intend this dedication to be an -overt act of relinquishment in perpetuity of all present and future rights to -this software under copyright law. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------------------------------------------------------------------------------- -*/ diff --git a/v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image_write.h b/v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image_write.h deleted file mode 100644 index 6f36938278..0000000000 --- a/v2/dependency-patches/torch-sys-0.24.0/libtch/stb_image_write.h +++ /dev/null @@ -1,1621 +0,0 @@ -/* stb_image_write - v1.11 - public domain - http://nothings.org/stb/stb_image_write.h - writes out PNG/BMP/TGA/JPEG/HDR images to C stdio - Sean Barrett 2010-2015 - no warranty implied; use at your own risk - - Before #including, - - #define STB_IMAGE_WRITE_IMPLEMENTATION - - in the file that you want to have the implementation. - - Will probably not work correctly with strict-aliasing optimizations. - - If using a modern Microsoft Compiler, non-safe versions of CRT calls may cause - compilation warnings or even errors. To avoid this, also before #including, - - #define STBI_MSC_SECURE_CRT - -ABOUT: - - This header file is a library for writing images to C stdio or a callback. - - The PNG output is not optimal; it is 20-50% larger than the file - written by a decent optimizing implementation; though providing a custom - zlib compress function (see STBIW_ZLIB_COMPRESS) can mitigate that. - This library is designed for source code compactness and simplicity, - not optimal image file size or run-time performance. - -BUILDING: - - You can #define STBIW_ASSERT(x) before the #include to avoid using assert.h. - You can #define STBIW_MALLOC(), STBIW_REALLOC(), and STBIW_FREE() to replace - malloc,realloc,free. - You can #define STBIW_MEMMOVE() to replace memmove() - You can #define STBIW_ZLIB_COMPRESS to use a custom zlib-style compress function - for PNG compression (instead of the builtin one), it must have the following signature: - unsigned char * my_compress(unsigned char *data, int data_len, int *out_len, int quality); - The returned data will be freed with STBIW_FREE() (free() by default), - so it must be heap allocated with STBIW_MALLOC() (malloc() by default), - -UNICODE: - - If compiling for Windows and you wish to use Unicode filenames, compile - with - #define STBIW_WINDOWS_UTF8 - and pass utf8-encoded filenames. Call stbiw_convert_wchar_to_utf8 to convert - Windows wchar_t filenames to utf8. - -USAGE: - - There are five functions, one for each image file format: - - int stbi_write_png(char const *filename, int w, int h, int comp, const void *data, int stride_in_bytes); - int stbi_write_bmp(char const *filename, int w, int h, int comp, const void *data); - int stbi_write_tga(char const *filename, int w, int h, int comp, const void *data); - int stbi_write_jpg(char const *filename, int w, int h, int comp, const void *data, int quality); - int stbi_write_hdr(char const *filename, int w, int h, int comp, const float *data); - - void stbi_flip_vertically_on_write(int flag); // flag is non-zero to flip data vertically - - There are also five equivalent functions that use an arbitrary write function. You are - expected to open/close your file-equivalent before and after calling these: - - int stbi_write_png_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data, int stride_in_bytes); - int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); - int stbi_write_tga_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); - int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const float *data); - int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int quality); - - where the callback is: - void stbi_write_func(void *context, void *data, int size); - - You can configure it with these global variables: - int stbi_write_tga_with_rle; // defaults to true; set to 0 to disable RLE - int stbi_write_png_compression_level; // defaults to 8; set to higher for more compression - int stbi_write_force_png_filter; // defaults to -1; set to 0..5 to force a filter mode - - - You can define STBI_WRITE_NO_STDIO to disable the file variant of these - functions, so the library will not use stdio.h at all. However, this will - also disable HDR writing, because it requires stdio for formatted output. - - Each function returns 0 on failure and non-0 on success. - - The functions create an image file defined by the parameters. The image - is a rectangle of pixels stored from left-to-right, top-to-bottom. - Each pixel contains 'comp' channels of data stored interleaved with 8-bits - per channel, in the following order: 1=Y, 2=YA, 3=RGB, 4=RGBA. (Y is - monochrome color.) The rectangle is 'w' pixels wide and 'h' pixels tall. - The *data pointer points to the first byte of the top-left-most pixel. - For PNG, "stride_in_bytes" is the distance in bytes from the first byte of - a row of pixels to the first byte of the next row of pixels. - - PNG creates output files with the same number of components as the input. - The BMP format expands Y to RGB in the file format and does not - output alpha. - - PNG supports writing rectangles of data even when the bytes storing rows of - data are not consecutive in memory (e.g. sub-rectangles of a larger image), - by supplying the stride between the beginning of adjacent rows. The other - formats do not. (Thus you cannot write a native-format BMP through the BMP - writer, both because it is in BGR order and because it may have padding - at the end of the line.) - - PNG allows you to set the deflate compression level by setting the global - variable 'stbi_write_png_compression_level' (it defaults to 8). - - HDR expects linear float data. Since the format is always 32-bit rgb(e) - data, alpha (if provided) is discarded, and for monochrome data it is - replicated across all three channels. - - TGA supports RLE or non-RLE compressed data. To use non-RLE-compressed - data, set the global variable 'stbi_write_tga_with_rle' to 0. - - JPEG does ignore alpha channels in input data; quality is between 1 and 100. - Higher quality looks better but results in a bigger image. - JPEG baseline (no JPEG progressive). - -CREDITS: - - - Sean Barrett - PNG/BMP/TGA - Baldur Karlsson - HDR - Jean-Sebastien Guay - TGA monochrome - Tim Kelsey - misc enhancements - Alan Hickman - TGA RLE - Emmanuel Julien - initial file IO callback implementation - Jon Olick - original jo_jpeg.cpp code - Daniel Gibson - integrate JPEG, allow external zlib - Aarni Koskela - allow choosing PNG filter - - bugfixes: - github:Chribba - Guillaume Chereau - github:jry2 - github:romigrou - Sergio Gonzalez - Jonas Karlsson - Filip Wasil - Thatcher Ulrich - github:poppolopoppo - Patrick Boettcher - github:xeekworx - Cap Petschulat - Simon Rodriguez - Ivan Tikhonov - github:ignotion - Adam Schackart - -LICENSE - - See end of file for license information. - -*/ - -#ifndef INCLUDE_STB_IMAGE_WRITE_H -#define INCLUDE_STB_IMAGE_WRITE_H - -#include - -// if STB_IMAGE_WRITE_STATIC causes problems, try defining STBIWDEF to 'inline' or 'static inline' -#ifndef STBIWDEF -#ifdef STB_IMAGE_WRITE_STATIC -#define STBIWDEF static -#else -#ifdef __cplusplus -#define STBIWDEF extern "C" -#else -#define STBIWDEF extern -#endif -#endif -#endif - -#ifndef STB_IMAGE_WRITE_STATIC // C++ forbids static forward declarations -extern int stbi_write_tga_with_rle; -extern int stbi_write_png_compression_level; -extern int stbi_write_force_png_filter; -#endif - -#ifndef STBI_WRITE_NO_STDIO -STBIWDEF int stbi_write_png(char const *filename, int w, int h, int comp, const void *data, int stride_in_bytes); -STBIWDEF int stbi_write_bmp(char const *filename, int w, int h, int comp, const void *data); -STBIWDEF int stbi_write_tga(char const *filename, int w, int h, int comp, const void *data); -STBIWDEF int stbi_write_hdr(char const *filename, int w, int h, int comp, const float *data); -STBIWDEF int stbi_write_jpg(char const *filename, int x, int y, int comp, const void *data, int quality); - -#ifdef STBI_WINDOWS_UTF8 -STBIWDEF int stbiw_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input); -#endif -#endif - -typedef void stbi_write_func(void *context, void *data, int size); - -STBIWDEF int stbi_write_png_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data, int stride_in_bytes); -STBIWDEF int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); -STBIWDEF int stbi_write_tga_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const void *data); -STBIWDEF int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int w, int h, int comp, const float *data); -STBIWDEF int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int quality); - -STBIWDEF void stbi_flip_vertically_on_write(int flip_boolean); - -#endif//INCLUDE_STB_IMAGE_WRITE_H - -#ifdef STB_IMAGE_WRITE_IMPLEMENTATION - -#ifdef _WIN32 - #ifndef _CRT_SECURE_NO_WARNINGS - #define _CRT_SECURE_NO_WARNINGS - #endif - #ifndef _CRT_NONSTDC_NO_DEPRECATE - #define _CRT_NONSTDC_NO_DEPRECATE - #endif -#endif - -#ifndef STBI_WRITE_NO_STDIO -#include -#endif // STBI_WRITE_NO_STDIO - -#include -#include -#include -#include - -#if defined(STBIW_MALLOC) && defined(STBIW_FREE) && (defined(STBIW_REALLOC) || defined(STBIW_REALLOC_SIZED)) -// ok -#elif !defined(STBIW_MALLOC) && !defined(STBIW_FREE) && !defined(STBIW_REALLOC) && !defined(STBIW_REALLOC_SIZED) -// ok -#else -#error "Must define all or none of STBIW_MALLOC, STBIW_FREE, and STBIW_REALLOC (or STBIW_REALLOC_SIZED)." -#endif - -#ifndef STBIW_MALLOC -#define STBIW_MALLOC(sz) malloc(sz) -#define STBIW_REALLOC(p,newsz) realloc(p,newsz) -#define STBIW_FREE(p) free(p) -#endif - -#ifndef STBIW_REALLOC_SIZED -#define STBIW_REALLOC_SIZED(p,oldsz,newsz) STBIW_REALLOC(p,newsz) -#endif - - -#ifndef STBIW_MEMMOVE -#define STBIW_MEMMOVE(a,b,sz) memmove(a,b,sz) -#endif - - -#ifndef STBIW_ASSERT -#include -#define STBIW_ASSERT(x) assert(x) -#endif - -#define STBIW_UCHAR(x) (unsigned char) ((x) & 0xff) - -#ifdef STB_IMAGE_WRITE_STATIC -static int stbi__flip_vertically_on_write=0; -static int stbi_write_png_compression_level = 8; -static int stbi_write_tga_with_rle = 1; -static int stbi_write_force_png_filter = -1; -#else -int stbi_write_png_compression_level = 8; -int stbi__flip_vertically_on_write=0; -int stbi_write_tga_with_rle = 1; -int stbi_write_force_png_filter = -1; -#endif - -STBIWDEF void stbi_flip_vertically_on_write(int flag) -{ - stbi__flip_vertically_on_write = flag; -} - -typedef struct -{ - stbi_write_func *func; - void *context; -} stbi__write_context; - -// initialize a callback-based context -static void stbi__start_write_callbacks(stbi__write_context *s, stbi_write_func *c, void *context) -{ - s->func = c; - s->context = context; -} - -#ifndef STBI_WRITE_NO_STDIO - -static void stbi__stdio_write(void *context, void *data, int size) -{ - fwrite(data,1,size,(FILE*) context); -} - -#if defined(_MSC_VER) && defined(STBI_WINDOWS_UTF8) -#ifdef __cplusplus -#define STBIW_EXTERN extern "C" -#else -#define STBIW_EXTERN extern -#endif -STBIW_EXTERN __declspec(dllimport) int __stdcall MultiByteToWideChar(unsigned int cp, unsigned long flags, const char *str, int cbmb, wchar_t *widestr, int cchwide); -STBIW_EXTERN __declspec(dllimport) int __stdcall WideCharToMultiByte(unsigned int cp, unsigned long flags, const wchar_t *widestr, int cchwide, char *str, int cbmb, const char *defchar, int *used_default); - -STBIWDEF int stbiw_convert_wchar_to_utf8(char *buffer, size_t bufferlen, const wchar_t* input) -{ - return WideCharToMultiByte(65001 /* UTF8 */, 0, input, -1, buffer, bufferlen, NULL, NULL); -} -#endif - -static FILE *stbiw__fopen(char const *filename, char const *mode) -{ - FILE *f; -#if defined(_MSC_VER) && defined(STBI_WINDOWS_UTF8) - wchar_t wMode[64]; - wchar_t wFilename[1024]; - if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, filename, -1, wFilename, sizeof(wFilename))) - return 0; - - if (0 == MultiByteToWideChar(65001 /* UTF8 */, 0, mode, -1, wMode, sizeof(wMode))) - return 0; - -#if _MSC_VER >= 1400 - if (0 != _wfopen_s(&f, wFilename, wMode)) - f = 0; -#else - f = _wfopen(wFilename, wMode); -#endif - -#elif defined(_MSC_VER) && _MSC_VER >= 1400 - if (0 != fopen_s(&f, filename, mode)) - f=0; -#else - f = fopen(filename, mode); -#endif - return f; -} - -static int stbi__start_write_file(stbi__write_context *s, const char *filename) -{ - FILE *f = stbiw__fopen(filename, "wb"); - stbi__start_write_callbacks(s, stbi__stdio_write, (void *) f); - return f != NULL; -} - -static void stbi__end_write_file(stbi__write_context *s) -{ - fclose((FILE *)s->context); -} - -#endif // !STBI_WRITE_NO_STDIO - -typedef unsigned int stbiw_uint32; -typedef int stb_image_write_test[sizeof(stbiw_uint32)==4 ? 1 : -1]; - -static void stbiw__writefv(stbi__write_context *s, const char *fmt, va_list v) -{ - while (*fmt) { - switch (*fmt++) { - case ' ': break; - case '1': { unsigned char x = STBIW_UCHAR(va_arg(v, int)); - s->func(s->context,&x,1); - break; } - case '2': { int x = va_arg(v,int); - unsigned char b[2]; - b[0] = STBIW_UCHAR(x); - b[1] = STBIW_UCHAR(x>>8); - s->func(s->context,b,2); - break; } - case '4': { stbiw_uint32 x = va_arg(v,int); - unsigned char b[4]; - b[0]=STBIW_UCHAR(x); - b[1]=STBIW_UCHAR(x>>8); - b[2]=STBIW_UCHAR(x>>16); - b[3]=STBIW_UCHAR(x>>24); - s->func(s->context,b,4); - break; } - default: - STBIW_ASSERT(0); - return; - } - } -} - -static void stbiw__writef(stbi__write_context *s, const char *fmt, ...) -{ - va_list v; - va_start(v, fmt); - stbiw__writefv(s, fmt, v); - va_end(v); -} - -static void stbiw__putc(stbi__write_context *s, unsigned char c) -{ - s->func(s->context, &c, 1); -} - -static void stbiw__write3(stbi__write_context *s, unsigned char a, unsigned char b, unsigned char c) -{ - unsigned char arr[3]; - arr[0] = a, arr[1] = b, arr[2] = c; - s->func(s->context, arr, 3); -} - -static void stbiw__write_pixel(stbi__write_context *s, int rgb_dir, int comp, int write_alpha, int expand_mono, unsigned char *d) -{ - unsigned char bg[3] = { 255, 0, 255}, px[3]; - int k; - - if (write_alpha < 0) - s->func(s->context, &d[comp - 1], 1); - - switch (comp) { - case 2: // 2 pixels = mono + alpha, alpha is written separately, so same as 1-channel case - case 1: - if (expand_mono) - stbiw__write3(s, d[0], d[0], d[0]); // monochrome bmp - else - s->func(s->context, d, 1); // monochrome TGA - break; - case 4: - if (!write_alpha) { - // composite against pink background - for (k = 0; k < 3; ++k) - px[k] = bg[k] + ((d[k] - bg[k]) * d[3]) / 255; - stbiw__write3(s, px[1 - rgb_dir], px[1], px[1 + rgb_dir]); - break; - } - /* FALLTHROUGH */ - case 3: - stbiw__write3(s, d[1 - rgb_dir], d[1], d[1 + rgb_dir]); - break; - } - if (write_alpha > 0) - s->func(s->context, &d[comp - 1], 1); -} - -static void stbiw__write_pixels(stbi__write_context *s, int rgb_dir, int vdir, int x, int y, int comp, void *data, int write_alpha, int scanline_pad, int expand_mono) -{ - stbiw_uint32 zero = 0; - int i,j, j_end; - - if (y <= 0) - return; - - if (stbi__flip_vertically_on_write) - vdir *= -1; - - if (vdir < 0) - j_end = -1, j = y-1; - else - j_end = y, j = 0; - - for (; j != j_end; j += vdir) { - for (i=0; i < x; ++i) { - unsigned char *d = (unsigned char *) data + (j*x+i)*comp; - stbiw__write_pixel(s, rgb_dir, comp, write_alpha, expand_mono, d); - } - s->func(s->context, &zero, scanline_pad); - } -} - -static int stbiw__outfile(stbi__write_context *s, int rgb_dir, int vdir, int x, int y, int comp, int expand_mono, void *data, int alpha, int pad, const char *fmt, ...) -{ - if (y < 0 || x < 0) { - return 0; - } else { - va_list v; - va_start(v, fmt); - stbiw__writefv(s, fmt, v); - va_end(v); - stbiw__write_pixels(s,rgb_dir,vdir,x,y,comp,data,alpha,pad, expand_mono); - return 1; - } -} - -static int stbi_write_bmp_core(stbi__write_context *s, int x, int y, int comp, const void *data) -{ - int pad = (-x*3) & 3; - return stbiw__outfile(s,-1,-1,x,y,comp,1,(void *) data,0,pad, - "11 4 22 4" "4 44 22 444444", - 'B', 'M', 14+40+(x*3+pad)*y, 0,0, 14+40, // file header - 40, x,y, 1,24, 0,0,0,0,0,0); // bitmap header -} - -STBIWDEF int stbi_write_bmp_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data) -{ - stbi__write_context s; - stbi__start_write_callbacks(&s, func, context); - return stbi_write_bmp_core(&s, x, y, comp, data); -} - -#ifndef STBI_WRITE_NO_STDIO -STBIWDEF int stbi_write_bmp(char const *filename, int x, int y, int comp, const void *data) -{ - stbi__write_context s; - if (stbi__start_write_file(&s,filename)) { - int r = stbi_write_bmp_core(&s, x, y, comp, data); - stbi__end_write_file(&s); - return r; - } else - return 0; -} -#endif //!STBI_WRITE_NO_STDIO - -static int stbi_write_tga_core(stbi__write_context *s, int x, int y, int comp, void *data) -{ - int has_alpha = (comp == 2 || comp == 4); - int colorbytes = has_alpha ? comp-1 : comp; - int format = colorbytes < 2 ? 3 : 2; // 3 color channels (RGB/RGBA) = 2, 1 color channel (Y/YA) = 3 - - if (y < 0 || x < 0) - return 0; - - if (!stbi_write_tga_with_rle) { - return stbiw__outfile(s, -1, -1, x, y, comp, 0, (void *) data, has_alpha, 0, - "111 221 2222 11", 0, 0, format, 0, 0, 0, 0, 0, x, y, (colorbytes + has_alpha) * 8, has_alpha * 8); - } else { - int i,j,k; - int jend, jdir; - - stbiw__writef(s, "111 221 2222 11", 0,0,format+8, 0,0,0, 0,0,x,y, (colorbytes + has_alpha) * 8, has_alpha * 8); - - if (stbi__flip_vertically_on_write) { - j = 0; - jend = y; - jdir = 1; - } else { - j = y-1; - jend = -1; - jdir = -1; - } - for (; j != jend; j += jdir) { - unsigned char *row = (unsigned char *) data + j * x * comp; - int len; - - for (i = 0; i < x; i += len) { - unsigned char *begin = row + i * comp; - int diff = 1; - len = 1; - - if (i < x - 1) { - ++len; - diff = memcmp(begin, row + (i + 1) * comp, comp); - if (diff) { - const unsigned char *prev = begin; - for (k = i + 2; k < x && len < 128; ++k) { - if (memcmp(prev, row + k * comp, comp)) { - prev += comp; - ++len; - } else { - --len; - break; - } - } - } else { - for (k = i + 2; k < x && len < 128; ++k) { - if (!memcmp(begin, row + k * comp, comp)) { - ++len; - } else { - break; - } - } - } - } - - if (diff) { - unsigned char header = STBIW_UCHAR(len - 1); - s->func(s->context, &header, 1); - for (k = 0; k < len; ++k) { - stbiw__write_pixel(s, -1, comp, has_alpha, 0, begin + k * comp); - } - } else { - unsigned char header = STBIW_UCHAR(len - 129); - s->func(s->context, &header, 1); - stbiw__write_pixel(s, -1, comp, has_alpha, 0, begin); - } - } - } - } - return 1; -} - -STBIWDEF int stbi_write_tga_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data) -{ - stbi__write_context s; - stbi__start_write_callbacks(&s, func, context); - return stbi_write_tga_core(&s, x, y, comp, (void *) data); -} - -#ifndef STBI_WRITE_NO_STDIO -STBIWDEF int stbi_write_tga(char const *filename, int x, int y, int comp, const void *data) -{ - stbi__write_context s; - if (stbi__start_write_file(&s,filename)) { - int r = stbi_write_tga_core(&s, x, y, comp, (void *) data); - stbi__end_write_file(&s); - return r; - } else - return 0; -} -#endif - -// ************************************************************************************************* -// Radiance RGBE HDR writer -// by Baldur Karlsson - -#define stbiw__max(a, b) ((a) > (b) ? (a) : (b)) - -static void stbiw__linear_to_rgbe(unsigned char *rgbe, float *linear) -{ - int exponent; - float maxcomp = stbiw__max(linear[0], stbiw__max(linear[1], linear[2])); - - if (maxcomp < 1e-32f) { - rgbe[0] = rgbe[1] = rgbe[2] = rgbe[3] = 0; - } else { - float normalize = (float) frexp(maxcomp, &exponent) * 256.0f/maxcomp; - - rgbe[0] = (unsigned char)(linear[0] * normalize); - rgbe[1] = (unsigned char)(linear[1] * normalize); - rgbe[2] = (unsigned char)(linear[2] * normalize); - rgbe[3] = (unsigned char)(exponent + 128); - } -} - -static void stbiw__write_run_data(stbi__write_context *s, int length, unsigned char databyte) -{ - unsigned char lengthbyte = STBIW_UCHAR(length+128); - STBIW_ASSERT(length+128 <= 255); - s->func(s->context, &lengthbyte, 1); - s->func(s->context, &databyte, 1); -} - -static void stbiw__write_dump_data(stbi__write_context *s, int length, unsigned char *data) -{ - unsigned char lengthbyte = STBIW_UCHAR(length); - STBIW_ASSERT(length <= 128); // inconsistent with spec but consistent with official code - s->func(s->context, &lengthbyte, 1); - s->func(s->context, data, length); -} - -static void stbiw__write_hdr_scanline(stbi__write_context *s, int width, int ncomp, unsigned char *scratch, float *scanline) -{ - unsigned char scanlineheader[4] = { 2, 2, 0, 0 }; - unsigned char rgbe[4]; - float linear[3]; - int x; - - scanlineheader[2] = (width&0xff00)>>8; - scanlineheader[3] = (width&0x00ff); - - /* skip RLE for images too small or large */ - if (width < 8 || width >= 32768) { - for (x=0; x < width; x++) { - switch (ncomp) { - case 4: /* fallthrough */ - case 3: linear[2] = scanline[x*ncomp + 2]; - linear[1] = scanline[x*ncomp + 1]; - linear[0] = scanline[x*ncomp + 0]; - break; - default: - linear[0] = linear[1] = linear[2] = scanline[x*ncomp + 0]; - break; - } - stbiw__linear_to_rgbe(rgbe, linear); - s->func(s->context, rgbe, 4); - } - } else { - int c,r; - /* encode into scratch buffer */ - for (x=0; x < width; x++) { - switch(ncomp) { - case 4: /* fallthrough */ - case 3: linear[2] = scanline[x*ncomp + 2]; - linear[1] = scanline[x*ncomp + 1]; - linear[0] = scanline[x*ncomp + 0]; - break; - default: - linear[0] = linear[1] = linear[2] = scanline[x*ncomp + 0]; - break; - } - stbiw__linear_to_rgbe(rgbe, linear); - scratch[x + width*0] = rgbe[0]; - scratch[x + width*1] = rgbe[1]; - scratch[x + width*2] = rgbe[2]; - scratch[x + width*3] = rgbe[3]; - } - - s->func(s->context, scanlineheader, 4); - - /* RLE each component separately */ - for (c=0; c < 4; c++) { - unsigned char *comp = &scratch[width*c]; - - x = 0; - while (x < width) { - // find first run - r = x; - while (r+2 < width) { - if (comp[r] == comp[r+1] && comp[r] == comp[r+2]) - break; - ++r; - } - if (r+2 >= width) - r = width; - // dump up to first run - while (x < r) { - int len = r-x; - if (len > 128) len = 128; - stbiw__write_dump_data(s, len, &comp[x]); - x += len; - } - // if there's a run, output it - if (r+2 < width) { // same test as what we break out of in search loop, so only true if we break'd - // find next byte after run - while (r < width && comp[r] == comp[x]) - ++r; - // output run up to r - while (x < r) { - int len = r-x; - if (len > 127) len = 127; - stbiw__write_run_data(s, len, comp[x]); - x += len; - } - } - } - } - } -} - -static int stbi_write_hdr_core(stbi__write_context *s, int x, int y, int comp, float *data) -{ - if (y <= 0 || x <= 0 || data == NULL) - return 0; - else { - // Each component is stored separately. Allocate scratch space for full output scanline. - unsigned char *scratch = (unsigned char *) STBIW_MALLOC(x*4); - int i, len; - char buffer[128]; - char header[] = "#?RADIANCE\n# Written by stb_image_write.h\nFORMAT=32-bit_rle_rgbe\n"; - s->func(s->context, header, sizeof(header)-1); - -#ifdef STBI_MSC_SECURE_CRT - len = sprintf_s(buffer, "EXPOSURE= 1.0000000000000\n\n-Y %d +X %d\n", y, x); -#else - len = snprintf(buffer, 127, "EXPOSURE= 1.0000000000000\n\n-Y %d +X %d\n", y, x); -#endif - s->func(s->context, buffer, len); - - for(i=0; i < y; i++) - stbiw__write_hdr_scanline(s, x, comp, scratch, data + comp*x*(stbi__flip_vertically_on_write ? y-1-i : i)); - STBIW_FREE(scratch); - return 1; - } -} - -STBIWDEF int stbi_write_hdr_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const float *data) -{ - stbi__write_context s; - stbi__start_write_callbacks(&s, func, context); - return stbi_write_hdr_core(&s, x, y, comp, (float *) data); -} - -#ifndef STBI_WRITE_NO_STDIO -STBIWDEF int stbi_write_hdr(char const *filename, int x, int y, int comp, const float *data) -{ - stbi__write_context s; - if (stbi__start_write_file(&s,filename)) { - int r = stbi_write_hdr_core(&s, x, y, comp, (float *) data); - stbi__end_write_file(&s); - return r; - } else - return 0; -} -#endif // STBI_WRITE_NO_STDIO - - -////////////////////////////////////////////////////////////////////////////// -// -// PNG writer -// - -#ifndef STBIW_ZLIB_COMPRESS -// stretchy buffer; stbiw__sbpush() == vector<>::push_back() -- stbiw__sbcount() == vector<>::size() -#define stbiw__sbraw(a) ((int *) (a) - 2) -#define stbiw__sbm(a) stbiw__sbraw(a)[0] -#define stbiw__sbn(a) stbiw__sbraw(a)[1] - -#define stbiw__sbneedgrow(a,n) ((a)==0 || stbiw__sbn(a)+n >= stbiw__sbm(a)) -#define stbiw__sbmaybegrow(a,n) (stbiw__sbneedgrow(a,(n)) ? stbiw__sbgrow(a,n) : 0) -#define stbiw__sbgrow(a,n) stbiw__sbgrowf((void **) &(a), (n), sizeof(*(a))) - -#define stbiw__sbpush(a, v) (stbiw__sbmaybegrow(a,1), (a)[stbiw__sbn(a)++] = (v)) -#define stbiw__sbcount(a) ((a) ? stbiw__sbn(a) : 0) -#define stbiw__sbfree(a) ((a) ? STBIW_FREE(stbiw__sbraw(a)),0 : 0) - -static void *stbiw__sbgrowf(void **arr, int increment, int itemsize) -{ - int m = *arr ? 2*stbiw__sbm(*arr)+increment : increment+1; - void *p = STBIW_REALLOC_SIZED(*arr ? stbiw__sbraw(*arr) : 0, *arr ? (stbiw__sbm(*arr)*itemsize + sizeof(int)*2) : 0, itemsize * m + sizeof(int)*2); - STBIW_ASSERT(p); - if (p) { - if (!*arr) ((int *) p)[1] = 0; - *arr = (void *) ((int *) p + 2); - stbiw__sbm(*arr) = m; - } - return *arr; -} - -static unsigned char *stbiw__zlib_flushf(unsigned char *data, unsigned int *bitbuffer, int *bitcount) -{ - while (*bitcount >= 8) { - stbiw__sbpush(data, STBIW_UCHAR(*bitbuffer)); - *bitbuffer >>= 8; - *bitcount -= 8; - } - return data; -} - -static int stbiw__zlib_bitrev(int code, int codebits) -{ - int res=0; - while (codebits--) { - res = (res << 1) | (code & 1); - code >>= 1; - } - return res; -} - -static unsigned int stbiw__zlib_countm(unsigned char *a, unsigned char *b, int limit) -{ - int i; - for (i=0; i < limit && i < 258; ++i) - if (a[i] != b[i]) break; - return i; -} - -static unsigned int stbiw__zhash(unsigned char *data) -{ - stbiw_uint32 hash = data[0] + (data[1] << 8) + (data[2] << 16); - hash ^= hash << 3; - hash += hash >> 5; - hash ^= hash << 4; - hash += hash >> 17; - hash ^= hash << 25; - hash += hash >> 6; - return hash; -} - -#define stbiw__zlib_flush() (out = stbiw__zlib_flushf(out, &bitbuf, &bitcount)) -#define stbiw__zlib_add(code,codebits) \ - (bitbuf |= (code) << bitcount, bitcount += (codebits), stbiw__zlib_flush()) -#define stbiw__zlib_huffa(b,c) stbiw__zlib_add(stbiw__zlib_bitrev(b,c),c) -// default huffman tables -#define stbiw__zlib_huff1(n) stbiw__zlib_huffa(0x30 + (n), 8) -#define stbiw__zlib_huff2(n) stbiw__zlib_huffa(0x190 + (n)-144, 9) -#define stbiw__zlib_huff3(n) stbiw__zlib_huffa(0 + (n)-256,7) -#define stbiw__zlib_huff4(n) stbiw__zlib_huffa(0xc0 + (n)-280,8) -#define stbiw__zlib_huff(n) ((n) <= 143 ? stbiw__zlib_huff1(n) : (n) <= 255 ? stbiw__zlib_huff2(n) : (n) <= 279 ? stbiw__zlib_huff3(n) : stbiw__zlib_huff4(n)) -#define stbiw__zlib_huffb(n) ((n) <= 143 ? stbiw__zlib_huff1(n) : stbiw__zlib_huff2(n)) - -#define stbiw__ZHASH 16384 - -#endif // STBIW_ZLIB_COMPRESS - -STBIWDEF unsigned char * stbi_zlib_compress(unsigned char *data, int data_len, int *out_len, int quality) -{ -#ifdef STBIW_ZLIB_COMPRESS - // user provided a zlib compress implementation, use that - return STBIW_ZLIB_COMPRESS(data, data_len, out_len, quality); -#else // use builtin - static unsigned short lengthc[] = { 3,4,5,6,7,8,9,10,11,13,15,17,19,23,27,31,35,43,51,59,67,83,99,115,131,163,195,227,258, 259 }; - static unsigned char lengtheb[]= { 0,0,0,0,0,0,0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 0 }; - static unsigned short distc[] = { 1,2,3,4,5,7,9,13,17,25,33,49,65,97,129,193,257,385,513,769,1025,1537,2049,3073,4097,6145,8193,12289,16385,24577, 32768 }; - static unsigned char disteb[] = { 0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,12,12,13,13 }; - unsigned int bitbuf=0; - int i,j, bitcount=0; - unsigned char *out = NULL; - unsigned char ***hash_table = (unsigned char***) STBIW_MALLOC(stbiw__ZHASH * sizeof(char**)); - if (hash_table == NULL) - return NULL; - if (quality < 5) quality = 5; - - stbiw__sbpush(out, 0x78); // DEFLATE 32K window - stbiw__sbpush(out, 0x5e); // FLEVEL = 1 - stbiw__zlib_add(1,1); // BFINAL = 1 - stbiw__zlib_add(1,2); // BTYPE = 1 -- fixed huffman - - for (i=0; i < stbiw__ZHASH; ++i) - hash_table[i] = NULL; - - i=0; - while (i < data_len-3) { - // hash next 3 bytes of data to be compressed - int h = stbiw__zhash(data+i)&(stbiw__ZHASH-1), best=3; - unsigned char *bestloc = 0; - unsigned char **hlist = hash_table[h]; - int n = stbiw__sbcount(hlist); - for (j=0; j < n; ++j) { - if (hlist[j]-data > i-32768) { // if entry lies within window - int d = stbiw__zlib_countm(hlist[j], data+i, data_len-i); - if (d >= best) best=d,bestloc=hlist[j]; - } - } - // when hash table entry is too long, delete half the entries - if (hash_table[h] && stbiw__sbn(hash_table[h]) == 2*quality) { - STBIW_MEMMOVE(hash_table[h], hash_table[h]+quality, sizeof(hash_table[h][0])*quality); - stbiw__sbn(hash_table[h]) = quality; - } - stbiw__sbpush(hash_table[h],data+i); - - if (bestloc) { - // "lazy matching" - check match at *next* byte, and if it's better, do cur byte as literal - h = stbiw__zhash(data+i+1)&(stbiw__ZHASH-1); - hlist = hash_table[h]; - n = stbiw__sbcount(hlist); - for (j=0; j < n; ++j) { - if (hlist[j]-data > i-32767) { - int e = stbiw__zlib_countm(hlist[j], data+i+1, data_len-i-1); - if (e > best) { // if next match is better, bail on current match - bestloc = NULL; - break; - } - } - } - } - - if (bestloc) { - int d = (int) (data+i - bestloc); // distance back - STBIW_ASSERT(d <= 32767 && best <= 258); - for (j=0; best > lengthc[j+1]-1; ++j); - stbiw__zlib_huff(j+257); - if (lengtheb[j]) stbiw__zlib_add(best - lengthc[j], lengtheb[j]); - for (j=0; d > distc[j+1]-1; ++j); - stbiw__zlib_add(stbiw__zlib_bitrev(j,5),5); - if (disteb[j]) stbiw__zlib_add(d - distc[j], disteb[j]); - i += best; - } else { - stbiw__zlib_huffb(data[i]); - ++i; - } - } - // write out final bytes - for (;i < data_len; ++i) - stbiw__zlib_huffb(data[i]); - stbiw__zlib_huff(256); // end of block - // pad with 0 bits to byte boundary - while (bitcount) - stbiw__zlib_add(0,1); - - for (i=0; i < stbiw__ZHASH; ++i) - (void) stbiw__sbfree(hash_table[i]); - STBIW_FREE(hash_table); - - { - // compute adler32 on input - unsigned int s1=1, s2=0; - int blocklen = (int) (data_len % 5552); - j=0; - while (j < data_len) { - for (i=0; i < blocklen; ++i) s1 += data[j+i], s2 += s1; - s1 %= 65521, s2 %= 65521; - j += blocklen; - blocklen = 5552; - } - stbiw__sbpush(out, STBIW_UCHAR(s2 >> 8)); - stbiw__sbpush(out, STBIW_UCHAR(s2)); - stbiw__sbpush(out, STBIW_UCHAR(s1 >> 8)); - stbiw__sbpush(out, STBIW_UCHAR(s1)); - } - *out_len = stbiw__sbn(out); - // make returned pointer freeable - STBIW_MEMMOVE(stbiw__sbraw(out), out, *out_len); - return (unsigned char *) stbiw__sbraw(out); -#endif // STBIW_ZLIB_COMPRESS -} - -static unsigned int stbiw__crc32(unsigned char *buffer, int len) -{ -#ifdef STBIW_CRC32 - return STBIW_CRC32(buffer, len); -#else - static unsigned int crc_table[256] = - { - 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, 0x076DC419, 0x706AF48F, 0xE963A535, 0x9E6495A3, - 0x0eDB8832, 0x79DCB8A4, 0xE0D5E91E, 0x97D2D988, 0x09B64C2B, 0x7EB17CBD, 0xE7B82D07, 0x90BF1D91, - 0x1DB71064, 0x6AB020F2, 0xF3B97148, 0x84BE41DE, 0x1ADAD47D, 0x6DDDE4EB, 0xF4D4B551, 0x83D385C7, - 0x136C9856, 0x646BA8C0, 0xFD62F97A, 0x8A65C9EC, 0x14015C4F, 0x63066CD9, 0xFA0F3D63, 0x8D080DF5, - 0x3B6E20C8, 0x4C69105E, 0xD56041E4, 0xA2677172, 0x3C03E4D1, 0x4B04D447, 0xD20D85FD, 0xA50AB56B, - 0x35B5A8FA, 0x42B2986C, 0xDBBBC9D6, 0xACBCF940, 0x32D86CE3, 0x45DF5C75, 0xDCD60DCF, 0xABD13D59, - 0x26D930AC, 0x51DE003A, 0xC8D75180, 0xBFD06116, 0x21B4F4B5, 0x56B3C423, 0xCFBA9599, 0xB8BDA50F, - 0x2802B89E, 0x5F058808, 0xC60CD9B2, 0xB10BE924, 0x2F6F7C87, 0x58684C11, 0xC1611DAB, 0xB6662D3D, - 0x76DC4190, 0x01DB7106, 0x98D220BC, 0xEFD5102A, 0x71B18589, 0x06B6B51F, 0x9FBFE4A5, 0xE8B8D433, - 0x7807C9A2, 0x0F00F934, 0x9609A88E, 0xE10E9818, 0x7F6A0DBB, 0x086D3D2D, 0x91646C97, 0xE6635C01, - 0x6B6B51F4, 0x1C6C6162, 0x856530D8, 0xF262004E, 0x6C0695ED, 0x1B01A57B, 0x8208F4C1, 0xF50FC457, - 0x65B0D9C6, 0x12B7E950, 0x8BBEB8EA, 0xFCB9887C, 0x62DD1DDF, 0x15DA2D49, 0x8CD37CF3, 0xFBD44C65, - 0x4DB26158, 0x3AB551CE, 0xA3BC0074, 0xD4BB30E2, 0x4ADFA541, 0x3DD895D7, 0xA4D1C46D, 0xD3D6F4FB, - 0x4369E96A, 0x346ED9FC, 0xAD678846, 0xDA60B8D0, 0x44042D73, 0x33031DE5, 0xAA0A4C5F, 0xDD0D7CC9, - 0x5005713C, 0x270241AA, 0xBE0B1010, 0xC90C2086, 0x5768B525, 0x206F85B3, 0xB966D409, 0xCE61E49F, - 0x5EDEF90E, 0x29D9C998, 0xB0D09822, 0xC7D7A8B4, 0x59B33D17, 0x2EB40D81, 0xB7BD5C3B, 0xC0BA6CAD, - 0xEDB88320, 0x9ABFB3B6, 0x03B6E20C, 0x74B1D29A, 0xEAD54739, 0x9DD277AF, 0x04DB2615, 0x73DC1683, - 0xE3630B12, 0x94643B84, 0x0D6D6A3E, 0x7A6A5AA8, 0xE40ECF0B, 0x9309FF9D, 0x0A00AE27, 0x7D079EB1, - 0xF00F9344, 0x8708A3D2, 0x1E01F268, 0x6906C2FE, 0xF762575D, 0x806567CB, 0x196C3671, 0x6E6B06E7, - 0xFED41B76, 0x89D32BE0, 0x10DA7A5A, 0x67DD4ACC, 0xF9B9DF6F, 0x8EBEEFF9, 0x17B7BE43, 0x60B08ED5, - 0xD6D6A3E8, 0xA1D1937E, 0x38D8C2C4, 0x4FDFF252, 0xD1BB67F1, 0xA6BC5767, 0x3FB506DD, 0x48B2364B, - 0xD80D2BDA, 0xAF0A1B4C, 0x36034AF6, 0x41047A60, 0xDF60EFC3, 0xA867DF55, 0x316E8EEF, 0x4669BE79, - 0xCB61B38C, 0xBC66831A, 0x256FD2A0, 0x5268E236, 0xCC0C7795, 0xBB0B4703, 0x220216B9, 0x5505262F, - 0xC5BA3BBE, 0xB2BD0B28, 0x2BB45A92, 0x5CB36A04, 0xC2D7FFA7, 0xB5D0CF31, 0x2CD99E8B, 0x5BDEAE1D, - 0x9B64C2B0, 0xEC63F226, 0x756AA39C, 0x026D930A, 0x9C0906A9, 0xEB0E363F, 0x72076785, 0x05005713, - 0x95BF4A82, 0xE2B87A14, 0x7BB12BAE, 0x0CB61B38, 0x92D28E9B, 0xE5D5BE0D, 0x7CDCEFB7, 0x0BDBDF21, - 0x86D3D2D4, 0xF1D4E242, 0x68DDB3F8, 0x1FDA836E, 0x81BE16CD, 0xF6B9265B, 0x6FB077E1, 0x18B74777, - 0x88085AE6, 0xFF0F6A70, 0x66063BCA, 0x11010B5C, 0x8F659EFF, 0xF862AE69, 0x616BFFD3, 0x166CCF45, - 0xA00AE278, 0xD70DD2EE, 0x4E048354, 0x3903B3C2, 0xA7672661, 0xD06016F7, 0x4969474D, 0x3E6E77DB, - 0xAED16A4A, 0xD9D65ADC, 0x40DF0B66, 0x37D83BF0, 0xA9BCAE53, 0xDEBB9EC5, 0x47B2CF7F, 0x30B5FFE9, - 0xBDBDF21C, 0xCABAC28A, 0x53B39330, 0x24B4A3A6, 0xBAD03605, 0xCDD70693, 0x54DE5729, 0x23D967BF, - 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D - }; - - unsigned int crc = ~0u; - int i; - for (i=0; i < len; ++i) - crc = (crc >> 8) ^ crc_table[buffer[i] ^ (crc & 0xff)]; - return ~crc; -#endif -} - -#define stbiw__wpng4(o,a,b,c,d) ((o)[0]=STBIW_UCHAR(a),(o)[1]=STBIW_UCHAR(b),(o)[2]=STBIW_UCHAR(c),(o)[3]=STBIW_UCHAR(d),(o)+=4) -#define stbiw__wp32(data,v) stbiw__wpng4(data, (v)>>24,(v)>>16,(v)>>8,(v)); -#define stbiw__wptag(data,s) stbiw__wpng4(data, s[0],s[1],s[2],s[3]) - -static void stbiw__wpcrc(unsigned char **data, int len) -{ - unsigned int crc = stbiw__crc32(*data - len - 4, len+4); - stbiw__wp32(*data, crc); -} - -static unsigned char stbiw__paeth(int a, int b, int c) -{ - int p = a + b - c, pa = abs(p-a), pb = abs(p-b), pc = abs(p-c); - if (pa <= pb && pa <= pc) return STBIW_UCHAR(a); - if (pb <= pc) return STBIW_UCHAR(b); - return STBIW_UCHAR(c); -} - -// @OPTIMIZE: provide an option that always forces left-predict or paeth predict -static void stbiw__encode_png_line(unsigned char *pixels, int stride_bytes, int width, int height, int y, int n, int filter_type, signed char *line_buffer) -{ - static int mapping[] = { 0,1,2,3,4 }; - static int firstmap[] = { 0,1,0,5,6 }; - int *mymap = (y != 0) ? mapping : firstmap; - int i; - int type = mymap[filter_type]; - unsigned char *z = pixels + stride_bytes * (stbi__flip_vertically_on_write ? height-1-y : y); - int signed_stride = stbi__flip_vertically_on_write ? -stride_bytes : stride_bytes; - - if (type==0) { - memcpy(line_buffer, z, width*n); - return; - } - - // first loop isn't optimized since it's just one pixel - for (i = 0; i < n; ++i) { - switch (type) { - case 1: line_buffer[i] = z[i]; break; - case 2: line_buffer[i] = z[i] - z[i-signed_stride]; break; - case 3: line_buffer[i] = z[i] - (z[i-signed_stride]>>1); break; - case 4: line_buffer[i] = (signed char) (z[i] - stbiw__paeth(0,z[i-signed_stride],0)); break; - case 5: line_buffer[i] = z[i]; break; - case 6: line_buffer[i] = z[i]; break; - } - } - switch (type) { - case 1: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - z[i-n]; break; - case 2: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - z[i-signed_stride]; break; - case 3: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - ((z[i-n] + z[i-signed_stride])>>1); break; - case 4: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - stbiw__paeth(z[i-n], z[i-signed_stride], z[i-signed_stride-n]); break; - case 5: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - (z[i-n]>>1); break; - case 6: for (i=n; i < width*n; ++i) line_buffer[i] = z[i] - stbiw__paeth(z[i-n], 0,0); break; - } -} - -STBIWDEF unsigned char *stbi_write_png_to_mem(const unsigned char *pixels, int stride_bytes, int x, int y, int n, int *out_len) -{ - int force_filter = stbi_write_force_png_filter; - int ctype[5] = { -1, 0, 4, 2, 6 }; - unsigned char sig[8] = { 137,80,78,71,13,10,26,10 }; - unsigned char *out,*o, *filt, *zlib; - signed char *line_buffer; - int j,zlen; - - if (stride_bytes == 0) - stride_bytes = x * n; - - if (force_filter >= 5) { - force_filter = -1; - } - - filt = (unsigned char *) STBIW_MALLOC((x*n+1) * y); if (!filt) return 0; - line_buffer = (signed char *) STBIW_MALLOC(x * n); if (!line_buffer) { STBIW_FREE(filt); return 0; } - for (j=0; j < y; ++j) { - int filter_type; - if (force_filter > -1) { - filter_type = force_filter; - stbiw__encode_png_line((unsigned char*)(pixels), stride_bytes, x, y, j, n, force_filter, line_buffer); - } else { // Estimate the best filter by running through all of them: - int best_filter = 0, best_filter_val = 0x7fffffff, est, i; - for (filter_type = 0; filter_type < 5; filter_type++) { - stbiw__encode_png_line((unsigned char*)(pixels), stride_bytes, x, y, j, n, filter_type, line_buffer); - - // Estimate the entropy of the line using this filter; the less, the better. - est = 0; - for (i = 0; i < x*n; ++i) { - est += abs((signed char) line_buffer[i]); - } - if (est < best_filter_val) { - best_filter_val = est; - best_filter = filter_type; - } - } - if (filter_type != best_filter) { // If the last iteration already got us the best filter, don't redo it - stbiw__encode_png_line((unsigned char*)(pixels), stride_bytes, x, y, j, n, best_filter, line_buffer); - filter_type = best_filter; - } - } - // when we get here, filter_type contains the filter type, and line_buffer contains the data - filt[j*(x*n+1)] = (unsigned char) filter_type; - STBIW_MEMMOVE(filt+j*(x*n+1)+1, line_buffer, x*n); - } - STBIW_FREE(line_buffer); - zlib = stbi_zlib_compress(filt, y*( x*n+1), &zlen, stbi_write_png_compression_level); - STBIW_FREE(filt); - if (!zlib) return 0; - - // each tag requires 12 bytes of overhead - out = (unsigned char *) STBIW_MALLOC(8 + 12+13 + 12+zlen + 12); - if (!out) return 0; - *out_len = 8 + 12+13 + 12+zlen + 12; - - o=out; - STBIW_MEMMOVE(o,sig,8); o+= 8; - stbiw__wp32(o, 13); // header length - stbiw__wptag(o, "IHDR"); - stbiw__wp32(o, x); - stbiw__wp32(o, y); - *o++ = 8; - *o++ = STBIW_UCHAR(ctype[n]); - *o++ = 0; - *o++ = 0; - *o++ = 0; - stbiw__wpcrc(&o,13); - - stbiw__wp32(o, zlen); - stbiw__wptag(o, "IDAT"); - STBIW_MEMMOVE(o, zlib, zlen); - o += zlen; - STBIW_FREE(zlib); - stbiw__wpcrc(&o, zlen); - - stbiw__wp32(o,0); - stbiw__wptag(o, "IEND"); - stbiw__wpcrc(&o,0); - - STBIW_ASSERT(o == out + *out_len); - - return out; -} - -#ifndef STBI_WRITE_NO_STDIO -STBIWDEF int stbi_write_png(char const *filename, int x, int y, int comp, const void *data, int stride_bytes) -{ - FILE *f; - int len; - unsigned char *png = stbi_write_png_to_mem((const unsigned char *) data, stride_bytes, x, y, comp, &len); - if (png == NULL) return 0; - - f = stbiw__fopen(filename, "wb"); - if (!f) { STBIW_FREE(png); return 0; } - fwrite(png, 1, len, f); - fclose(f); - STBIW_FREE(png); - return 1; -} -#endif - -STBIWDEF int stbi_write_png_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int stride_bytes) -{ - int len; - unsigned char *png = stbi_write_png_to_mem((const unsigned char *) data, stride_bytes, x, y, comp, &len); - if (png == NULL) return 0; - func(context, png, len); - STBIW_FREE(png); - return 1; -} - - -/* *************************************************************************** - * - * JPEG writer - * - * This is based on Jon Olick's jo_jpeg.cpp: - * public domain Simple, Minimalistic JPEG writer - http://www.jonolick.com/code.html - */ - -static const unsigned char stbiw__jpg_ZigZag[] = { 0,1,5,6,14,15,27,28,2,4,7,13,16,26,29,42,3,8,12,17,25,30,41,43,9,11,18, - 24,31,40,44,53,10,19,23,32,39,45,52,54,20,22,33,38,46,51,55,60,21,34,37,47,50,56,59,61,35,36,48,49,57,58,62,63 }; - -static void stbiw__jpg_writeBits(stbi__write_context *s, int *bitBufP, int *bitCntP, const unsigned short *bs) { - int bitBuf = *bitBufP, bitCnt = *bitCntP; - bitCnt += bs[1]; - bitBuf |= bs[0] << (24 - bitCnt); - while(bitCnt >= 8) { - unsigned char c = (bitBuf >> 16) & 255; - stbiw__putc(s, c); - if(c == 255) { - stbiw__putc(s, 0); - } - bitBuf <<= 8; - bitCnt -= 8; - } - *bitBufP = bitBuf; - *bitCntP = bitCnt; -} - -static void stbiw__jpg_DCT(float *d0p, float *d1p, float *d2p, float *d3p, float *d4p, float *d5p, float *d6p, float *d7p) { - float d0 = *d0p, d1 = *d1p, d2 = *d2p, d3 = *d3p, d4 = *d4p, d5 = *d5p, d6 = *d6p, d7 = *d7p; - float z1, z2, z3, z4, z5, z11, z13; - - float tmp0 = d0 + d7; - float tmp7 = d0 - d7; - float tmp1 = d1 + d6; - float tmp6 = d1 - d6; - float tmp2 = d2 + d5; - float tmp5 = d2 - d5; - float tmp3 = d3 + d4; - float tmp4 = d3 - d4; - - // Even part - float tmp10 = tmp0 + tmp3; // phase 2 - float tmp13 = tmp0 - tmp3; - float tmp11 = tmp1 + tmp2; - float tmp12 = tmp1 - tmp2; - - d0 = tmp10 + tmp11; // phase 3 - d4 = tmp10 - tmp11; - - z1 = (tmp12 + tmp13) * 0.707106781f; // c4 - d2 = tmp13 + z1; // phase 5 - d6 = tmp13 - z1; - - // Odd part - tmp10 = tmp4 + tmp5; // phase 2 - tmp11 = tmp5 + tmp6; - tmp12 = tmp6 + tmp7; - - // The rotator is modified from fig 4-8 to avoid extra negations. - z5 = (tmp10 - tmp12) * 0.382683433f; // c6 - z2 = tmp10 * 0.541196100f + z5; // c2-c6 - z4 = tmp12 * 1.306562965f + z5; // c2+c6 - z3 = tmp11 * 0.707106781f; // c4 - - z11 = tmp7 + z3; // phase 5 - z13 = tmp7 - z3; - - *d5p = z13 + z2; // phase 6 - *d3p = z13 - z2; - *d1p = z11 + z4; - *d7p = z11 - z4; - - *d0p = d0; *d2p = d2; *d4p = d4; *d6p = d6; -} - -static void stbiw__jpg_calcBits(int val, unsigned short bits[2]) { - int tmp1 = val < 0 ? -val : val; - val = val < 0 ? val-1 : val; - bits[1] = 1; - while(tmp1 >>= 1) { - ++bits[1]; - } - bits[0] = val & ((1<0)&&(DU[end0pos]==0); --end0pos) { - } - // end0pos = first element in reverse order !=0 - if(end0pos == 0) { - stbiw__jpg_writeBits(s, bitBuf, bitCnt, EOB); - return DU[0]; - } - for(i = 1; i <= end0pos; ++i) { - int startpos = i; - int nrzeroes; - unsigned short bits[2]; - for (; DU[i]==0 && i<=end0pos; ++i) { - } - nrzeroes = i-startpos; - if ( nrzeroes >= 16 ) { - int lng = nrzeroes>>4; - int nrmarker; - for (nrmarker=1; nrmarker <= lng; ++nrmarker) - stbiw__jpg_writeBits(s, bitBuf, bitCnt, M16zeroes); - nrzeroes &= 15; - } - stbiw__jpg_calcBits(DU[i], bits); - stbiw__jpg_writeBits(s, bitBuf, bitCnt, HTAC[(nrzeroes<<4)+bits[1]]); - stbiw__jpg_writeBits(s, bitBuf, bitCnt, bits); - } - if(end0pos != 63) { - stbiw__jpg_writeBits(s, bitBuf, bitCnt, EOB); - } - return DU[0]; -} - -static int stbi_write_jpg_core(stbi__write_context *s, int width, int height, int comp, const void* data, int quality) { - // Constants that don't pollute global namespace - static const unsigned char std_dc_luminance_nrcodes[] = {0,0,1,5,1,1,1,1,1,1,0,0,0,0,0,0,0}; - static const unsigned char std_dc_luminance_values[] = {0,1,2,3,4,5,6,7,8,9,10,11}; - static const unsigned char std_ac_luminance_nrcodes[] = {0,0,2,1,3,3,2,4,3,5,5,4,4,0,0,1,0x7d}; - static const unsigned char std_ac_luminance_values[] = { - 0x01,0x02,0x03,0x00,0x04,0x11,0x05,0x12,0x21,0x31,0x41,0x06,0x13,0x51,0x61,0x07,0x22,0x71,0x14,0x32,0x81,0x91,0xa1,0x08, - 0x23,0x42,0xb1,0xc1,0x15,0x52,0xd1,0xf0,0x24,0x33,0x62,0x72,0x82,0x09,0x0a,0x16,0x17,0x18,0x19,0x1a,0x25,0x26,0x27,0x28, - 0x29,0x2a,0x34,0x35,0x36,0x37,0x38,0x39,0x3a,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0x4a,0x53,0x54,0x55,0x56,0x57,0x58,0x59, - 0x5a,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0x6a,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7a,0x83,0x84,0x85,0x86,0x87,0x88,0x89, - 0x8a,0x92,0x93,0x94,0x95,0x96,0x97,0x98,0x99,0x9a,0xa2,0xa3,0xa4,0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xb2,0xb3,0xb4,0xb5,0xb6, - 0xb7,0xb8,0xb9,0xba,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xd2,0xd3,0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda,0xe1,0xe2, - 0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xf1,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0xf8,0xf9,0xfa - }; - static const unsigned char std_dc_chrominance_nrcodes[] = {0,0,3,1,1,1,1,1,1,1,1,1,0,0,0,0,0}; - static const unsigned char std_dc_chrominance_values[] = {0,1,2,3,4,5,6,7,8,9,10,11}; - static const unsigned char std_ac_chrominance_nrcodes[] = {0,0,2,1,2,4,4,3,4,7,5,4,4,0,1,2,0x77}; - static const unsigned char std_ac_chrominance_values[] = { - 0x00,0x01,0x02,0x03,0x11,0x04,0x05,0x21,0x31,0x06,0x12,0x41,0x51,0x07,0x61,0x71,0x13,0x22,0x32,0x81,0x08,0x14,0x42,0x91, - 0xa1,0xb1,0xc1,0x09,0x23,0x33,0x52,0xf0,0x15,0x62,0x72,0xd1,0x0a,0x16,0x24,0x34,0xe1,0x25,0xf1,0x17,0x18,0x19,0x1a,0x26, - 0x27,0x28,0x29,0x2a,0x35,0x36,0x37,0x38,0x39,0x3a,0x43,0x44,0x45,0x46,0x47,0x48,0x49,0x4a,0x53,0x54,0x55,0x56,0x57,0x58, - 0x59,0x5a,0x63,0x64,0x65,0x66,0x67,0x68,0x69,0x6a,0x73,0x74,0x75,0x76,0x77,0x78,0x79,0x7a,0x82,0x83,0x84,0x85,0x86,0x87, - 0x88,0x89,0x8a,0x92,0x93,0x94,0x95,0x96,0x97,0x98,0x99,0x9a,0xa2,0xa3,0xa4,0xa5,0xa6,0xa7,0xa8,0xa9,0xaa,0xb2,0xb3,0xb4, - 0xb5,0xb6,0xb7,0xb8,0xb9,0xba,0xc2,0xc3,0xc4,0xc5,0xc6,0xc7,0xc8,0xc9,0xca,0xd2,0xd3,0xd4,0xd5,0xd6,0xd7,0xd8,0xd9,0xda, - 0xe2,0xe3,0xe4,0xe5,0xe6,0xe7,0xe8,0xe9,0xea,0xf2,0xf3,0xf4,0xf5,0xf6,0xf7,0xf8,0xf9,0xfa - }; - // Huffman tables - static const unsigned short YDC_HT[256][2] = { {0,2},{2,3},{3,3},{4,3},{5,3},{6,3},{14,4},{30,5},{62,6},{126,7},{254,8},{510,9}}; - static const unsigned short UVDC_HT[256][2] = { {0,2},{1,2},{2,2},{6,3},{14,4},{30,5},{62,6},{126,7},{254,8},{510,9},{1022,10},{2046,11}}; - static const unsigned short YAC_HT[256][2] = { - {10,4},{0,2},{1,2},{4,3},{11,4},{26,5},{120,7},{248,8},{1014,10},{65410,16},{65411,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {12,4},{27,5},{121,7},{502,9},{2038,11},{65412,16},{65413,16},{65414,16},{65415,16},{65416,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {28,5},{249,8},{1015,10},{4084,12},{65417,16},{65418,16},{65419,16},{65420,16},{65421,16},{65422,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {58,6},{503,9},{4085,12},{65423,16},{65424,16},{65425,16},{65426,16},{65427,16},{65428,16},{65429,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {59,6},{1016,10},{65430,16},{65431,16},{65432,16},{65433,16},{65434,16},{65435,16},{65436,16},{65437,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {122,7},{2039,11},{65438,16},{65439,16},{65440,16},{65441,16},{65442,16},{65443,16},{65444,16},{65445,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {123,7},{4086,12},{65446,16},{65447,16},{65448,16},{65449,16},{65450,16},{65451,16},{65452,16},{65453,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {250,8},{4087,12},{65454,16},{65455,16},{65456,16},{65457,16},{65458,16},{65459,16},{65460,16},{65461,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {504,9},{32704,15},{65462,16},{65463,16},{65464,16},{65465,16},{65466,16},{65467,16},{65468,16},{65469,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {505,9},{65470,16},{65471,16},{65472,16},{65473,16},{65474,16},{65475,16},{65476,16},{65477,16},{65478,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {506,9},{65479,16},{65480,16},{65481,16},{65482,16},{65483,16},{65484,16},{65485,16},{65486,16},{65487,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {1017,10},{65488,16},{65489,16},{65490,16},{65491,16},{65492,16},{65493,16},{65494,16},{65495,16},{65496,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {1018,10},{65497,16},{65498,16},{65499,16},{65500,16},{65501,16},{65502,16},{65503,16},{65504,16},{65505,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {2040,11},{65506,16},{65507,16},{65508,16},{65509,16},{65510,16},{65511,16},{65512,16},{65513,16},{65514,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {65515,16},{65516,16},{65517,16},{65518,16},{65519,16},{65520,16},{65521,16},{65522,16},{65523,16},{65524,16},{0,0},{0,0},{0,0},{0,0},{0,0}, - {2041,11},{65525,16},{65526,16},{65527,16},{65528,16},{65529,16},{65530,16},{65531,16},{65532,16},{65533,16},{65534,16},{0,0},{0,0},{0,0},{0,0},{0,0} - }; - static const unsigned short UVAC_HT[256][2] = { - {0,2},{1,2},{4,3},{10,4},{24,5},{25,5},{56,6},{120,7},{500,9},{1014,10},{4084,12},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {11,4},{57,6},{246,8},{501,9},{2038,11},{4085,12},{65416,16},{65417,16},{65418,16},{65419,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {26,5},{247,8},{1015,10},{4086,12},{32706,15},{65420,16},{65421,16},{65422,16},{65423,16},{65424,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {27,5},{248,8},{1016,10},{4087,12},{65425,16},{65426,16},{65427,16},{65428,16},{65429,16},{65430,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {58,6},{502,9},{65431,16},{65432,16},{65433,16},{65434,16},{65435,16},{65436,16},{65437,16},{65438,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {59,6},{1017,10},{65439,16},{65440,16},{65441,16},{65442,16},{65443,16},{65444,16},{65445,16},{65446,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {121,7},{2039,11},{65447,16},{65448,16},{65449,16},{65450,16},{65451,16},{65452,16},{65453,16},{65454,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {122,7},{2040,11},{65455,16},{65456,16},{65457,16},{65458,16},{65459,16},{65460,16},{65461,16},{65462,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {249,8},{65463,16},{65464,16},{65465,16},{65466,16},{65467,16},{65468,16},{65469,16},{65470,16},{65471,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {503,9},{65472,16},{65473,16},{65474,16},{65475,16},{65476,16},{65477,16},{65478,16},{65479,16},{65480,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {504,9},{65481,16},{65482,16},{65483,16},{65484,16},{65485,16},{65486,16},{65487,16},{65488,16},{65489,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {505,9},{65490,16},{65491,16},{65492,16},{65493,16},{65494,16},{65495,16},{65496,16},{65497,16},{65498,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {506,9},{65499,16},{65500,16},{65501,16},{65502,16},{65503,16},{65504,16},{65505,16},{65506,16},{65507,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {2041,11},{65508,16},{65509,16},{65510,16},{65511,16},{65512,16},{65513,16},{65514,16},{65515,16},{65516,16},{0,0},{0,0},{0,0},{0,0},{0,0},{0,0}, - {16352,14},{65517,16},{65518,16},{65519,16},{65520,16},{65521,16},{65522,16},{65523,16},{65524,16},{65525,16},{0,0},{0,0},{0,0},{0,0},{0,0}, - {1018,10},{32707,15},{65526,16},{65527,16},{65528,16},{65529,16},{65530,16},{65531,16},{65532,16},{65533,16},{65534,16},{0,0},{0,0},{0,0},{0,0},{0,0} - }; - static const int YQT[] = {16,11,10,16,24,40,51,61,12,12,14,19,26,58,60,55,14,13,16,24,40,57,69,56,14,17,22,29,51,87,80,62,18,22, - 37,56,68,109,103,77,24,35,55,64,81,104,113,92,49,64,78,87,103,121,120,101,72,92,95,98,112,100,103,99}; - static const int UVQT[] = {17,18,24,47,99,99,99,99,18,21,26,66,99,99,99,99,24,26,56,99,99,99,99,99,47,66,99,99,99,99,99,99, - 99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99,99}; - static const float aasf[] = { 1.0f * 2.828427125f, 1.387039845f * 2.828427125f, 1.306562965f * 2.828427125f, 1.175875602f * 2.828427125f, - 1.0f * 2.828427125f, 0.785694958f * 2.828427125f, 0.541196100f * 2.828427125f, 0.275899379f * 2.828427125f }; - - int row, col, i, k; - float fdtbl_Y[64], fdtbl_UV[64]; - unsigned char YTable[64], UVTable[64]; - - if(!data || !width || !height || comp > 4 || comp < 1) { - return 0; - } - - quality = quality ? quality : 90; - quality = quality < 1 ? 1 : quality > 100 ? 100 : quality; - quality = quality < 50 ? 5000 / quality : 200 - quality * 2; - - for(i = 0; i < 64; ++i) { - int uvti, yti = (YQT[i]*quality+50)/100; - YTable[stbiw__jpg_ZigZag[i]] = (unsigned char) (yti < 1 ? 1 : yti > 255 ? 255 : yti); - uvti = (UVQT[i]*quality+50)/100; - UVTable[stbiw__jpg_ZigZag[i]] = (unsigned char) (uvti < 1 ? 1 : uvti > 255 ? 255 : uvti); - } - - for(row = 0, k = 0; row < 8; ++row) { - for(col = 0; col < 8; ++col, ++k) { - fdtbl_Y[k] = 1 / (YTable [stbiw__jpg_ZigZag[k]] * aasf[row] * aasf[col]); - fdtbl_UV[k] = 1 / (UVTable[stbiw__jpg_ZigZag[k]] * aasf[row] * aasf[col]); - } - } - - // Write Headers - { - static const unsigned char head0[] = { 0xFF,0xD8,0xFF,0xE0,0,0x10,'J','F','I','F',0,1,1,0,0,1,0,1,0,0,0xFF,0xDB,0,0x84,0 }; - static const unsigned char head2[] = { 0xFF,0xDA,0,0xC,3,1,0,2,0x11,3,0x11,0,0x3F,0 }; - const unsigned char head1[] = { 0xFF,0xC0,0,0x11,8,(unsigned char)(height>>8),STBIW_UCHAR(height),(unsigned char)(width>>8),STBIW_UCHAR(width), - 3,1,0x11,0,2,0x11,1,3,0x11,1,0xFF,0xC4,0x01,0xA2,0 }; - s->func(s->context, (void*)head0, sizeof(head0)); - s->func(s->context, (void*)YTable, sizeof(YTable)); - stbiw__putc(s, 1); - s->func(s->context, UVTable, sizeof(UVTable)); - s->func(s->context, (void*)head1, sizeof(head1)); - s->func(s->context, (void*)(std_dc_luminance_nrcodes+1), sizeof(std_dc_luminance_nrcodes)-1); - s->func(s->context, (void*)std_dc_luminance_values, sizeof(std_dc_luminance_values)); - stbiw__putc(s, 0x10); // HTYACinfo - s->func(s->context, (void*)(std_ac_luminance_nrcodes+1), sizeof(std_ac_luminance_nrcodes)-1); - s->func(s->context, (void*)std_ac_luminance_values, sizeof(std_ac_luminance_values)); - stbiw__putc(s, 1); // HTUDCinfo - s->func(s->context, (void*)(std_dc_chrominance_nrcodes+1), sizeof(std_dc_chrominance_nrcodes)-1); - s->func(s->context, (void*)std_dc_chrominance_values, sizeof(std_dc_chrominance_values)); - stbiw__putc(s, 0x11); // HTUACinfo - s->func(s->context, (void*)(std_ac_chrominance_nrcodes+1), sizeof(std_ac_chrominance_nrcodes)-1); - s->func(s->context, (void*)std_ac_chrominance_values, sizeof(std_ac_chrominance_values)); - s->func(s->context, (void*)head2, sizeof(head2)); - } - - // Encode 8x8 macroblocks - { - static const unsigned short fillBits[] = {0x7F, 7}; - const unsigned char *imageData = (const unsigned char *)data; - int DCY=0, DCU=0, DCV=0; - int bitBuf=0, bitCnt=0; - // comp == 2 is grey+alpha (alpha is ignored) - int ofsG = comp > 2 ? 1 : 0, ofsB = comp > 2 ? 2 : 0; - int x, y, pos; - for(y = 0; y < height; y += 8) { - for(x = 0; x < width; x += 8) { - float YDU[64], UDU[64], VDU[64]; - for(row = y, pos = 0; row < y+8; ++row) { - // row >= height => use last input row - int clamped_row = (row < height) ? row : height - 1; - int base_p = (stbi__flip_vertically_on_write ? (height-1-clamped_row) : clamped_row)*width*comp; - for(col = x; col < x+8; ++col, ++pos) { - float r, g, b; - // if col >= width => use pixel from last input column - int p = base_p + ((col < width) ? col : (width-1))*comp; - - r = imageData[p+0]; - g = imageData[p+ofsG]; - b = imageData[p+ofsB]; - YDU[pos]=+0.29900f*r+0.58700f*g+0.11400f*b-128; - UDU[pos]=-0.16874f*r-0.33126f*g+0.50000f*b; - VDU[pos]=+0.50000f*r-0.41869f*g-0.08131f*b; - } - } - - DCY = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, YDU, fdtbl_Y, DCY, YDC_HT, YAC_HT); - DCU = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, UDU, fdtbl_UV, DCU, UVDC_HT, UVAC_HT); - DCV = stbiw__jpg_processDU(s, &bitBuf, &bitCnt, VDU, fdtbl_UV, DCV, UVDC_HT, UVAC_HT); - } - } - - // Do the bit alignment of the EOI marker - stbiw__jpg_writeBits(s, &bitBuf, &bitCnt, fillBits); - } - - // EOI - stbiw__putc(s, 0xFF); - stbiw__putc(s, 0xD9); - - return 1; -} - -STBIWDEF int stbi_write_jpg_to_func(stbi_write_func *func, void *context, int x, int y, int comp, const void *data, int quality) -{ - stbi__write_context s; - stbi__start_write_callbacks(&s, func, context); - return stbi_write_jpg_core(&s, x, y, comp, (void *) data, quality); -} - - -#ifndef STBI_WRITE_NO_STDIO -STBIWDEF int stbi_write_jpg(char const *filename, int x, int y, int comp, const void *data, int quality) -{ - stbi__write_context s; - if (stbi__start_write_file(&s,filename)) { - int r = stbi_write_jpg_core(&s, x, y, comp, data, quality); - stbi__end_write_file(&s); - return r; - } else - return 0; -} -#endif - -#endif // STB_IMAGE_WRITE_IMPLEMENTATION - -/* Revision history - 1.10 (2019-02-07) - support utf8 filenames in Windows; fix warnings and platform ifdefs - 1.09 (2018-02-11) - fix typo in zlib quality API, improve STB_I_W_STATIC in C++ - 1.08 (2018-01-29) - add stbi__flip_vertically_on_write, external zlib, zlib quality, choose PNG filter - 1.07 (2017-07-24) - doc fix - 1.06 (2017-07-23) - writing JPEG (using Jon Olick's code) - 1.05 ??? - 1.04 (2017-03-03) - monochrome BMP expansion - 1.03 ??? - 1.02 (2016-04-02) - avoid allocating large structures on the stack - 1.01 (2016-01-16) - STBIW_REALLOC_SIZED: support allocators with no realloc support - avoid race-condition in crc initialization - minor compile issues - 1.00 (2015-09-14) - installable file IO function - 0.99 (2015-09-13) - warning fixes; TGA rle support - 0.98 (2015-04-08) - added STBIW_MALLOC, STBIW_ASSERT etc - 0.97 (2015-01-18) - fixed HDR asserts, rewrote HDR rle logic - 0.96 (2015-01-17) - add HDR output - fix monochrome BMP - 0.95 (2014-08-17) - add monochrome TGA output - 0.94 (2014-05-31) - rename private functions to avoid conflicts with stb_image.h - 0.93 (2014-05-27) - warning fixes - 0.92 (2010-08-01) - casts to unsigned char to fix warnings - 0.91 (2010-07-17) - first public release - 0.90 first internal release -*/ - -/* ------------------------------------------------------------------------------- -This software is available under 2 licenses -- choose whichever you prefer. ------------------------------------------------------------------------------- -ALTERNATIVE A - MIT License -Copyright (c) 2017 Sean Barrett -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies -of the Software, and to permit persons to whom the Software is furnished to do -so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. ------------------------------------------------------------------------------- -ALTERNATIVE B - Public Domain (www.unlicense.org) -This is free and unencumbered software released into the public domain. -Anyone is free to copy, modify, publish, use, compile, sell, or distribute this -software, either in source code form or as a compiled binary, for any purpose, -commercial or non-commercial, and by any means. -In jurisdictions that recognize copyright laws, the author or authors of this -software dedicate any and all copyright interest in the software to the public -domain. We make this dedication for the benefit of the public at large and to -the detriment of our heirs and successors. We intend this dedication to be an -overt act of relinquishment in perpetuity of all present and future rights to -this software under copyright law. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. ------------------------------------------------------------------------------- -*/ diff --git a/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api.cpp b/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api.cpp deleted file mode 100644 index b966279eb5..0000000000 --- a/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api.cpp +++ /dev/null @@ -1,1356 +0,0 @@ -#include "torch_api.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#define STB_IMAGE_IMPLEMENTATION -#include "stb_image.h" - -#define STB_IMAGE_WRITE_IMPLEMENTATION -#include "stb_image_write.h" - -#define STB_IMAGE_RESIZE_IMPLEMENTATION -#include "stb_image_resize.h" - -thread_local char *torch_last_err = nullptr; - -char *get_and_reset_last_err() { - char *tmp = torch_last_err; - torch_last_err = nullptr; - return tmp; -} - -void at_manual_seed(int64_t seed) { torch::manual_seed(seed); } - -vector of_carray_tensor(torch::Tensor **vs, int len) { - vector result; - for (int i = 0; i < len; ++i) - result.push_back(*(vs[i])); - return result; -} - -c10::List> of_carray_tensor_opt(torch::Tensor **vs, - int len) { - vector> result; - for (int i = 0; i < len; ++i) { - result.push_back(vs[i] != nullptr ? c10::optional(*(vs[i])) - : c10::nullopt); - } - return c10::List>(result); -} - -at::Device device_of_int(int d) { - if (d == -3) - return at::Device(at::kVulkan); - if (d == -2) - return at::Device(at::kMPS); - if (d < 0) - return at::Device(at::kCPU); - return at::Device(at::kCUDA, /*index=*/d); -} -tensor at_new_tensor() { - PROTECT(return new torch::Tensor();) - return nullptr; -} - -tensor at_tensor_of_blob(void *data, int64_t *dims, size_t ndims, - int64_t *strides, size_t nstrides, int type, - int device) { - PROTECT( - at::TensorOptions blobOptions = at::TensorOptions() - .device(device_of_int(device)) - .dtype(torch::ScalarType(type)); - if (nstrides == 0) { - return new torch::Tensor( - torch::for_blob(data, torch::IntArrayRef(dims, ndims)) - .options(blobOptions) - .make_tensor()); - } else { - return new torch::Tensor(torch::from_blob( - data, torch::IntArrayRef(dims, ndims), - torch::IntArrayRef(strides, nstrides), blobOptions)); - }) - - return nullptr; -} - -tensor at_tensor_of_data(void *vs, int64_t *dims, size_t ndims, - size_t element_size_in_bytes, int type) { - PROTECT( - torch::Tensor tensor = torch::zeros(torch::IntArrayRef(dims, ndims), - torch::ScalarType(type)); - if ((int64_t)element_size_in_bytes != tensor.element_size()) throw std:: - invalid_argument("incoherent element sizes in bytes"); - void *tensor_data = tensor.data_ptr(); - memcpy(tensor_data, vs, tensor.numel() * element_size_in_bytes); - return new torch::Tensor(tensor);) - return nullptr; -} - -void at_copy_data(tensor tensor, void *vs, size_t numel, - size_t elt_size_in_bytes) { - PROTECT( - if ((int64_t)elt_size_in_bytes != tensor->element_size()) throw std:: - invalid_argument("incoherent element sizes in bytes"); - if ((int64_t)numel > tensor->numel()) throw std::invalid_argument( - "target numel is larger than tensor numel"); - if (tensor->device().type() != at::kCPU) { - torch::Tensor tmp_tensor = tensor->to(at::kCPU).contiguous(); - void *tensor_data = tmp_tensor.data_ptr(); - memcpy(vs, tensor_data, numel * elt_size_in_bytes); - } else { - auto tmp_tensor = tensor->contiguous(); - void *tensor_data = tmp_tensor.data_ptr(); - memcpy(vs, tensor_data, numel * elt_size_in_bytes); - }) -} - -tensor at_shallow_clone(tensor t) { - PROTECT(return new torch::Tensor(*t);) - return nullptr; -} - -void *at_data_ptr(tensor t) { - PROTECT(return t->data_ptr();) - return nullptr; -} - -int at_defined(tensor t) { - PROTECT(return t->defined();) - return -1; -} - -int at_is_mkldnn(tensor t) { - PROTECT(return t->is_mkldnn();) - return -1; -} - -int at_is_sparse(tensor t) { - PROTECT(return t->is_sparse();) - return -1; -} - -int at_is_contiguous(tensor t) { - PROTECT(return t->is_contiguous();) - return -1; -} - -size_t at_dim(tensor t) { - PROTECT(return t->dim();) - return -1; -} - -void at_shape(tensor t, int64_t *dims) { - PROTECT(int i = 0; for (int64_t dim : t->sizes()) dims[i++] = dim;) -} - -void at_stride(tensor t, int64_t *dims) { - PROTECT(int i = 0; for (int64_t dim : t->strides()) dims[i++] = dim;) -} - -int at_scalar_type(tensor t) { - PROTECT(return static_cast(t->scalar_type());) - return -1; -} - -void at__amp_non_finite_check_and_unscale(tensor t, tensor found_inf, - tensor inf_scale) { - PROTECT(at::_amp_foreach_non_finite_check_and_unscale_(*t, *found_inf, - *inf_scale);) -} - -void at_autocast_clear_cache() { at::autocast::clear_cache(); } - -int at_autocast_decrement_nesting() { - PROTECT(return at::autocast::decrement_nesting();) - return -1; -} - -int at_autocast_increment_nesting() { - PROTECT(return at::autocast::increment_nesting();) - return -1; -} - -bool at_autocast_is_enabled() { - PROTECT(return at::autocast::is_enabled();) - return -1; -} - -bool at_autocast_set_enabled(bool b) { - PROTECT(bool is_enabled = at::autocast::is_enabled(); - at::autocast::set_enabled(b); return is_enabled;) - return -1; -} - -int at_device(tensor t) { - PROTECT(auto device = t->device(); if (device.type() == at::kCPU) return -1; - if (device.type() == at::kMPS) return -2; - if (device.type() == at::kVulkan) return -3; - if (device.type() == at::kCUDA) return device.index();) - return -2; -} - -void at_backward(tensor t, int keep_graph, int create_graph) { - PROTECT(t->backward({}, keep_graph, create_graph);) -} - -int at_requires_grad(tensor t) { - PROTECT(return t->requires_grad();) - return -1; -} - -int at_grad_set_enabled(int b) { - PROTECT(bool is_enabled = torch::autograd::GradMode::is_enabled(); - torch::autograd::GradMode::set_enabled(b); return is_enabled;) - return -1; -} - -tensor at_get(tensor t, int index) { - PROTECT(return new torch::Tensor((*t)[index]);) - return nullptr; -} - -template -T at_value_at_indexes(tensor t, int64_t *indexes, int indexes_len) { - PROTECT(torch::Tensor tensor = *t; for (int i = 0; i < indexes_len; ++i) { - tensor = tensor[indexes[i]]; - } return tensor.item();) - return T(); -} - -double at_double_value_at_indexes(tensor t, int64_t *indexes, int indexes_len) { - return at_value_at_indexes(t, indexes, indexes_len); -} - -int64_t at_int64_value_at_indexes(tensor t, int64_t *indexes, int indexes_len) { - return at_value_at_indexes(t, indexes, indexes_len); -} - -template -void at_set_value_at_indexes(tensor t, int *indexes, int indexes_len, T v) { - PROTECT(torch::Tensor tensor = *t; for (int i = 0; i < indexes_len; ++i) { - tensor = tensor[indexes[i]]; - } tensor.fill_(v);) -} - -void at_set_double_value_at_indexes(tensor t, int *indexes, int indexes_len, - double v) { - at_set_value_at_indexes(t, indexes, indexes_len, v); -} - -void at_set_int64_value_at_indexes(tensor t, int *indexes, int indexes_len, - int64_t v) { - at_set_value_at_indexes(t, indexes, indexes_len, v); -} - -void at_fill_double(tensor t, double v) { PROTECT(t->fill_(v);) } - -void at_fill_int64(tensor t, int64_t v) { PROTECT(t->fill_(v);) } - -void at_print(tensor t) { - PROTECT(torch::Tensor *tensor = (torch::Tensor *)t; cout << *tensor << endl;) -} - -char *at_to_string(tensor t, int line_size) { - PROTECT(std::ostringstream oss; torch::print(oss, *t, line_size); - return strdup(oss.str().c_str());) - return nullptr; -} - -void at_copy_(tensor dst, tensor src) { PROTECT(dst->copy_(*src);) } - -namespace { -class WriteStreamAdapter { -private: - void *_stream_ptr; - -public: - WriteStreamAdapter(void *stream_ptr) : _stream_ptr(stream_ptr) {} - - ~WriteStreamAdapter() { tch_write_stream_destructor(_stream_ptr); } - - size_t save(const void *data, size_t size) { - size_t out_size = 0; - if (!tch_write_stream_write(_stream_ptr, static_cast(data), - size, &out_size)) { - throw std::ios_base::failure("tch_write_stream_write has returned error"); - } - return out_size; - } -}; - -class ReadStreamAdapter : public caffe2::serialize::ReadAdapterInterface { -private: - void *_stream_ptr; - -public: - ReadStreamAdapter(void *stream_ptr) : _stream_ptr(stream_ptr) {} - - virtual ~ReadStreamAdapter() { tch_read_stream_destructor(_stream_ptr); } - - virtual size_t size() const { - uint64_t current = 0; - if (!tch_read_stream_stream_position(_stream_ptr, ¤t)) { - throw std::ios_base::failure( - "tch_read_stream_stream_position has returned error"); - } - - uint64_t size = 0; - if (!tch_read_stream_seek_end(_stream_ptr, 0, &size)) { - throw std::ios_base::failure( - "tch_read_stream_seek_end has returned error"); - } - - uint64_t dummy = 0; - if (!tch_read_stream_seek_start(_stream_ptr, current, &dummy)) { - throw std::ios_base::failure( - "tch_read_stream_seek_start has returned error"); - } - - return size; - } - - virtual size_t read(uint64_t pos, void *buf, size_t n, - const char *what) const { - uint64_t dummy = 0; - if (!tch_read_stream_seek_start(_stream_ptr, pos, &dummy)) { - throw std::ios_base::failure( - "tch_read_stream_seek_start has returned error"); - } - - size_t size = 0; - if (!tch_read_stream_read(_stream_ptr, static_cast(buf), n, - &size)) { - throw std::ios_base::failure("tch_read_stream_read has returned error"); - } - - return size; - } -}; -} // namespace - -void at_save(tensor t, char *filename) { PROTECT(torch::save(*t, filename);) } - -void at_save_to_stream(tensor t, void *stream_ptr) { - PROTECT(auto adapter = std::shared_ptr( - new WriteStreamAdapter(stream_ptr)); - torch::save(*t, [adapter](const void *data, size_t size) { - return adapter->save(data, size); - });) -} - -void at_save_multi(tensor *tensors, char **tensor_names, int ntensors, - char *filename) { - PROTECT(torch::serialize::OutputArchive archive; - for (int i = 0; i < ntensors; ++i) archive.write( - std::string(tensor_names[i]), *(tensors[i]), /* buffer=*/false); - archive.save_to(filename);) -} - -void at_save_multi_to_stream(tensor *tensors, char **tensor_names, int ntensors, - void *stream_ptr) { - PROTECT(auto adapter = std::shared_ptr( - new WriteStreamAdapter(stream_ptr)); - torch::serialize::OutputArchive archive; - for (int i = 0; i < ntensors; ++i) archive.write( - std::string(tensor_names[i]), *(tensors[i]), /* buffer=*/false); - archive.save_to([adapter](const void *data, size_t size) { - return adapter->save(data, size); - });) -} - -void at_load_multi(tensor *tensors, char **tensor_names, int ntensors, - char *filename) { - PROTECT(torch::serialize::InputArchive archive; - archive.load_from(std::string(filename)); - vector ts(ntensors); - for (int i = 0; i < ntensors; ++i) - archive.read(std::string(tensor_names[i]), ts[i]); - // Only allocate the new tensor now so that if there is an exception - // raised during [read], no memory has to be freed. - for (int i = 0; i < ntensors; ++i) tensors[i] = - new torch::Tensor(ts[i]);) -} - -void at_loadz_callback(char *filename, void *data, - void (*f)(void *, char *, tensor)) { - PROTECT(auto params = torch::jit::_load_parameters(filename); - for (const auto &p : params) { - f(data, (char *)p.first.c_str(), new torch::Tensor(p.second)); - }) -} - -void at_loadz_callback_with_device(char *filename, void *data, - void (*f)(void *, char *, tensor), - int device_id) { - PROTECT(auto params = - torch::jit::_load_parameters(filename, device_of_int(device_id)); - for (const auto &p : params) { - f(data, (char *)p.first.c_str(), new torch::Tensor(p.second)); - }) -} - -void at_load_callback(char *filename, void *data, - void (*f)(void *, char *, tensor)) { - PROTECT(auto module = torch::jit::load(filename); - for (const auto &p : module.named_parameters()) { - auto v = p.value; - f(data, (char *)p.name.c_str(), new torch::Tensor(v)); - }) -} - -void at_load_callback_with_device(char *filename, void *data, - void (*f)(void *, char *, tensor), - int device_id) { - PROTECT(auto module = torch::jit::load(filename, device_of_int(device_id)); - for (const auto &p : module.named_parameters()) { - auto v = p.value; - f(data, (char *)p.name.c_str(), new torch::Tensor(v)); - }) -} - -void at_load_from_stream_callback(void *stream_ptr, void *data, - void (*f)(void *, char *, tensor), - bool enable_device_id, int device_id) { - PROTECT( - auto adapter = std::shared_ptr( - new ReadStreamAdapter(stream_ptr)); - auto module = enable_device_id - ? torch::jit::load(adapter, device_of_int(device_id)) - : torch::jit::load(adapter); - for (const auto &p : module.named_parameters()) { - auto v = p.value; - f(data, (char *)p.name.c_str(), new torch::Tensor(v)); - }) -} - -void at_load_multi_(tensor *tensors, char **tensor_names, int ntensors, - char *filename) { - PROTECT(torch::NoGradGuard no_grad; torch::serialize::InputArchive archive; - archive.load_from(std::string(filename)); - for (int i = 0; i < ntensors; ++i) { - if (tensors[i]->device().type() == at::kCPU) - archive.read(std::string(tensor_names[i]), *(tensors[i])); - else { - torch::Tensor tmp_tensor = - torch::empty_like(*(tensors[i]), at::device(at::kCPU)); - archive.read(std::string(tensor_names[i]), tmp_tensor); - tensors[i]->copy_(tmp_tensor); - } - }) -} - -tensor at_load(char *filename) { - PROTECT(torch::Tensor tensor; torch::load(tensor, filename); - return new torch::Tensor(tensor);) - return nullptr; -} - -tensor at_load_from_stream(void *stream_ptr) { - PROTECT(torch::NoGradGuard no_grad; torch::Tensor tensor; - auto adapter = - std::shared_ptr( - new ReadStreamAdapter(stream_ptr)); - torch::load( - tensor, - [adapter](uint64_t pos, void *buf, size_t nbytes) { - return adapter->read(pos, buf, nbytes, "tensor"); - }, - [adapter]() { return adapter->size(); }); - return new torch::Tensor(tensor);) - return nullptr; -} - -tensor at_load_image(char *filename) { - PROTECT( - int w = -1; int h = -1; int c = -1; - void *data = stbi_load(filename, &w, &h, &c, 3); - if (data == nullptr) throw std::invalid_argument(stbi_failure_reason()); - torch::Tensor tensor = torch::zeros({h, w, 3}, at::ScalarType::Byte); - memcpy(tensor.data_ptr(), data, h * w * 3); free(data); - return new torch::Tensor(tensor);) - return nullptr; -} - -tensor at_load_image_from_memory(unsigned char *img_data, size_t img_size) { - PROTECT( - int w = -1; int h = -1; int c = -1; - void *data = stbi_load_from_memory(img_data, img_size, &w, &h, &c, 3); - if (data == nullptr) throw std::invalid_argument(stbi_failure_reason()); - torch::Tensor tensor = torch::zeros({h, w, 3}, at::ScalarType::Byte); - memcpy(tensor.data_ptr(), data, h * w * 3); free(data); - return new torch::Tensor(tensor);) - return nullptr; -} - -bool ends_with(const char *str, const char *suffix) { - int suffix_len = strlen(suffix); - int str_len = strlen(str); - if (str_len < suffix_len) - return false; - for (int i = 1; i <= suffix_len; ++i) - if (str[str_len - i] != suffix[suffix_len - i]) - return false; - return true; -} - -int at_save_image(tensor tensor, char *filename) { - PROTECT(auto sizes = tensor->sizes(); - if (tensor->device().type() != at::kCPU) throw std::invalid_argument( - "the input tensor has to be on cpu"); - if (sizes.size() != 3) throw std::invalid_argument( - "invalid number of dimensions, should be 3"); - int h = sizes[0]; int w = sizes[1]; int c = sizes[2]; - auto tmp_tensor = tensor->contiguous(); - void *tensor_data = tmp_tensor.data_ptr(); - if (ends_with(filename, ".jpg")) return stbi_write_jpg( - filename, w, h, c, tensor_data, 90); - if (ends_with(filename, ".bmp")) return stbi_write_bmp( - filename, w, h, c, tensor_data); - if (ends_with(filename, ".tga")) return stbi_write_tga( - filename, w, h, c, tensor_data); - return stbi_write_png(filename, w, h, c, tensor_data, 0);) - return -1; -} - -int at_get_num_interop_threads() { - PROTECT(return at::get_num_interop_threads();) - return -1; -} - -int at_get_num_threads() { - PROTECT(return at::get_num_threads();) - return -1; -} - -void at_set_num_interop_threads(int n_threads) { - PROTECT(at::set_num_interop_threads(n_threads);) -} - -void at_set_num_threads(int n_threads) { - PROTECT(at::set_num_threads(n_threads);) -} - -void at_set_qengine(int qengine_id) { - PROTECT( - at::QEngine qengine = at::QEngine::NoQEngine; switch (qengine_id) { - case 0: - break; - case 1: - qengine = at::QEngine::FBGEMM; - break; - case 2: - qengine = at::QEngine::QNNPACK; - break; - } auto qengines = at::globalContext().supportedQEngines(); - if (std::find(qengines.begin(), qengines.end(), qengine) != - qengines.end()) { - at::globalContext().setQEngine(qengine); - } else throw std::invalid_argument("unsupported qengine");) -} - -tensor at_resize_image(tensor tensor, int out_w, int out_h) { - PROTECT( - auto sizes = tensor->sizes(); - if (tensor->device().type() != at::kCPU) throw std::invalid_argument( - "the input tensor has to be on cpu"); - if (sizes.size() != 3) throw std::invalid_argument( - "invalid number of dimensions, should be 3"); - int h = sizes[0]; int w = sizes[1]; int c = sizes[2]; - auto tmp_tensor = tensor->contiguous(); - const unsigned char *tensor_data = (unsigned char *)tmp_tensor.data_ptr(); - torch::Tensor out = torch::zeros({out_h, out_w, c}, at::ScalarType::Byte); - stbir_resize_uint8(tensor_data, w, h, 0, (unsigned char *)out.data_ptr(), - out_w, out_h, 0, c); - return new torch::Tensor(out);) - return nullptr; -} - -void at_free(tensor t) { delete (t); } - -void at_run_backward(tensor *tensors, int ntensors, tensor *inputs, int ninputs, - tensor *outputs, int keep_graph, int create_graph) { - PROTECT( - vector roots; - for (int i = 0; i < ntensors; ++i) - roots.push_back(torch::autograd::impl::gradient_edge(*tensors[i])); - - vector inputs_; for (int i = 0; i < ninputs; ++i) { - if (!inputs[i]->requires_grad()) - throw std::invalid_argument( - "one of the input tensor does not use set_requires_grad"); - inputs_.push_back(torch::autograd::impl::gradient_edge(*inputs[i])); - } - - vector - grads; - for (int i = 0; i < ntensors; ++i) - grads.push_back(torch::ones_like(*tensors[i])); - - auto vl = torch::autograd::Engine::get_default_engine().execute( - roots, grads, keep_graph, create_graph, false, inputs_); - for (int i = 0; i < ninputs; ++i) { - outputs[i] = static_cast(new torch::autograd::Variable(vl[i])); - }) -} - -optimizer ato_adam(double learning_rate, double beta1, double beta2, - double weight_decay, double eps, bool amsgrad) { - PROTECT(auto options = torch::optim::AdamOptions(learning_rate) - .betas(std::tuple(beta1, beta2)) - .weight_decay(weight_decay) - .eps(eps) - .amsgrad(amsgrad); - return new torch::optim::Adam(vector(), options);) - return nullptr; -} - -optimizer ato_adamw(double learning_rate, double beta1, double beta2, - double weight_decay, double eps, bool amsgrad) { - PROTECT(auto options = torch::optim::AdamWOptions(learning_rate) - .betas(std::tuple(beta1, beta2)) - .weight_decay(weight_decay) - .eps(eps) - .amsgrad(amsgrad); - return new torch::optim::AdamW(vector(), options);) - return nullptr; -} - -optimizer ato_rms_prop(double learning_rate, double alpha, double eps, - double weight_decay, double momentum, int centered) { - PROTECT(auto options = torch::optim::RMSpropOptions(learning_rate) - .alpha(alpha) - .eps(eps) - .weight_decay(weight_decay) - .momentum(momentum) - .centered(centered != 0); - return new torch::optim::RMSprop(vector(), options);) - return nullptr; -} - -optimizer ato_sgd(double learning_rate, double momentum, double dampening, - double weight_decay, int nesterov) { - PROTECT(auto options = torch::optim::SGDOptions(learning_rate) - .momentum(momentum) - .dampening(dampening) - .weight_decay(weight_decay) - .nesterov(nesterov); - return new torch::optim::SGD(vector(), options);) - return nullptr; -} - -void ato_add_parameters(optimizer t, tensor tensor, size_t group) { - PROTECT(auto &groups = t->param_groups(); while (groups.size() <= group) { - groups.push_back( - torch::optim::OptimizerParamGroup({}, t->defaults().clone())); - } groups[group] - .params() - .push_back(*tensor);) -} - -template void set_lr(optimizer t, double learning_rate) { - torch::optim::OptimizerOptions *d = &(t->defaults()); - if (auto p = dynamic_cast(d)) { - p->lr(learning_rate); - for (auto ¶m_group : t->param_groups()) { - torch::optim::OptimizerOptions *d = &(param_group.options()); - if (auto p2 = dynamic_cast(d)) { - p2->lr(learning_rate); - } else - throw std::invalid_argument("unexpected param group type"); - } - } -} - -void ato_set_learning_rate(optimizer t, double learning_rate) { - PROTECT(set_lr(t, learning_rate); - set_lr(t, learning_rate); - set_lr(t, learning_rate); - set_lr(t, learning_rate);) -} - -template -void set_lr_group(optimizer t, size_t group, double learning_rate) { - auto ¶m_group = t->param_groups().at(group); - torch::optim::OptimizerOptions *d = &(param_group.options()); - if (auto p = dynamic_cast(d)) { - p->lr(learning_rate); - } -} - -void ato_set_learning_rate_group(optimizer t, size_t group, - double learning_rate) { - PROTECT(set_lr_group(t, group, learning_rate); - set_lr_group(t, group, learning_rate); - set_lr_group(t, group, learning_rate); - set_lr_group(t, group, learning_rate);) -} - -void ato_set_momentum(optimizer t, double momentum) { - PROTECT( - torch::optim::OptimizerOptions *d = &(t->defaults()); - if (auto adam = dynamic_cast(d)) { - auto betas = adam->betas(); - adam->betas(std::tuple(momentum, get<1>(betas))); - for (auto ¶m_group : t->param_groups()) { - torch::optim::OptimizerOptions *d = &(param_group.options()); - if (auto adam2 = dynamic_cast(d)) { - adam2->betas(std::tuple(momentum, get<1>(betas))); - } else - throw std::invalid_argument("unexpected param group type"); - } - } else if (auto adamw = dynamic_cast(d)) { - auto betas = adamw->betas(); - adamw->betas(std::tuple(momentum, get<1>(betas))); - for (auto ¶m_group : t->param_groups()) { - torch::optim::OptimizerOptions *d = &(param_group.options()); - if (auto adamw2 = dynamic_cast(d)) { - adamw2->betas(std::tuple(momentum, get<1>(betas))); - } else - throw std::invalid_argument("unexpected param group type"); - } - } else if (auto rms = dynamic_cast(d)) { - rms->momentum(momentum); - for (auto ¶m_group : t->param_groups()) { - torch::optim::OptimizerOptions *d = &(param_group.options()); - if (auto rms2 = dynamic_cast(d)) { - rms2->momentum(momentum); - } else - throw std::invalid_argument("unexpected param group type"); - } - } else if (auto sgd = dynamic_cast(d)) { - sgd->momentum(momentum); - for (auto ¶m_group : t->param_groups()) { - torch::optim::OptimizerOptions *d = &(param_group.options()); - if (auto sgd2 = dynamic_cast(d)) { - sgd2->momentum(momentum); - } else - throw std::invalid_argument("unexpected param group type"); - } - } else throw std::invalid_argument("unexpected optimizer");) -} - -void ato_set_momentum_group(optimizer t, size_t group, double momentum) { - PROTECT( - auto ¶m_group = t->param_groups().at(group); - torch::optim::OptimizerOptions *d = &(param_group.options()); - - if (auto adam = dynamic_cast(d)) { - auto betas = adam->betas(); - adam->betas(std::tuple(momentum, get<1>(betas))); - } else if (auto adamw = dynamic_cast(d)) { - auto betas = adamw->betas(); - adamw->betas(std::tuple(momentum, get<1>(betas))); - } else if (auto rms = dynamic_cast(d)) { - rms->momentum(momentum); - } if (auto sgd = dynamic_cast(d)) { - sgd->momentum(momentum); - } else throw std::invalid_argument("unexpected optimizer");) -} - -template void set_weight_decay(optimizer t, double weight_decay) { - torch::optim::OptimizerOptions *d = &(t->defaults()); - if (auto p = dynamic_cast(d)) { - p->weight_decay(weight_decay); - for (auto ¶m_group : t->param_groups()) { - torch::optim::OptimizerOptions *d = &(param_group.options()); - if (auto p2 = dynamic_cast(d)) { - p2->weight_decay(weight_decay); - } else - throw std::invalid_argument("unexpected param group type"); - } - } -} - -void ato_set_weight_decay(optimizer t, double weight_decay) { - PROTECT(set_weight_decay(t, weight_decay); - set_weight_decay(t, weight_decay); - set_weight_decay(t, weight_decay); - set_weight_decay(t, weight_decay);) -} - -template -void set_weight_decay_group(optimizer t, size_t group, double weight_decay) { - auto ¶m_group = t->param_groups().at(group); - torch::optim::OptimizerOptions *d = &(param_group.options()); - if (auto p = dynamic_cast(d)) { - p->weight_decay(weight_decay); - } -} - -void ato_set_weight_decay_group(optimizer t, size_t group, - double weight_decay) { - PROTECT( - set_weight_decay_group(t, group, weight_decay); - set_weight_decay_group(t, group, - weight_decay); - set_weight_decay_group(t, group, - weight_decay); - set_weight_decay_group(t, group, weight_decay);) -} - -void ato_zero_grad(optimizer t) { PROTECT(t->zero_grad();) } - -void ato_step(optimizer t) { PROTECT(t->step();) } - -void ato_free(optimizer t) { delete (t); } - -scalar ats_int(int64_t v) { - PROTECT(return new torch::Scalar(v);) - return nullptr; -} - -scalar ats_float(double v) { - PROTECT(return new torch::Scalar(v);) - return nullptr; -} - -int64_t ats_to_int(scalar s) { - PROTECT(return s->toLong();) - return -1; -} - -double ats_to_float(scalar s) { - PROTECT(return s->toDouble();) - return 0.; -} - -char *ats_to_string(scalar s) { - PROTECT(using namespace at; std::ostringstream oss; oss << (*s); - return strdup(oss.str().c_str());) - return nullptr; -} - -void ats_free(scalar s) { delete (s); } - -int atc_cuda_device_count() { - PROTECT(return torch::cuda::device_count();) - return -1; -} - -int atc_cuda_is_available() { - PROTECT(return torch::cuda::is_available();) - return -1; -} - -int atc_cudnn_is_available() { - PROTECT(return torch::cuda::cudnn_is_available();) - return -1; -} - -void atc_manual_seed(uint64_t seed) { - PROTECT(return torch::cuda::manual_seed(seed);) -} - -void atc_manual_seed_all(uint64_t seed) { - PROTECT(return torch::cuda::manual_seed_all(seed);) -} - -void atc_synchronize(int64_t device_index) { - PROTECT(return torch::cuda::synchronize(device_index);) -} - -int atc_user_enabled_cudnn() { - PROTECT(return at::globalContext().userEnabledCuDNN();) - return -1; -} - -void atc_set_user_enabled_cudnn(int b) { - PROTECT(at::globalContext().setUserEnabledCuDNN(b);) -} - -void atc_set_benchmark_cudnn(int b) { - PROTECT(at::globalContext().setBenchmarkCuDNN(b);) -} - -bool at_context_has_openmp() { - PROTECT(return at::globalContext().hasOpenMP();) - return 0; -} - -bool at_context_has_mkl() { - PROTECT(return at::globalContext().hasMKL();) - return 0; -} - -bool at_context_has_lapack() { - PROTECT(return at::globalContext().hasLAPACK();) - return 0; -} - -bool at_context_has_mkldnn() { - PROTECT(return at::globalContext().hasMKLDNN();) - return 0; -} - -bool at_context_has_magma() { - PROTECT(return at::globalContext().hasMAGMA();) - return 0; -} - -bool at_context_has_cuda() { - PROTECT(return at::globalContext().hasCUDA();) - return 0; -} - -bool at_context_has_cudart() { - PROTECT(return at::globalContext().hasCUDART();) - return 0; -} - -bool at_context_has_cudnn() { - PROTECT(return at::globalContext().hasCuDNN();) - return 0; -} - -int64_t at_context_version_cudnn() { - PROTECT(return at::globalContext().versionCuDNN();) - return 0; -} - -int64_t at_context_version_cudart() { - PROTECT(return at::globalContext().versionCUDART();) - return 0; -} - -bool at_context_has_cusolver() { - PROTECT(return at::globalContext().hasCuSOLVER();) - return 0; -} - -bool at_context_has_hip() { - PROTECT(return at::globalContext().hasHIP();) - return 0; -} - -bool at_context_has_ipu() { - PROTECT(return at::globalContext().hasIPU();) - return 0; -} - -bool at_context_has_xla() { - PROTECT(return at::globalContext().hasXLA();) - return 0; -} - -bool at_context_has_lazy() { - PROTECT(return at::globalContext().hasLazy();) - return 0; -} - -bool at_context_has_mps() { - PROTECT(return at::globalContext().hasMPS();) - return 0; -} - -module atm_load(char *filename) { - PROTECT(return new torch::jit::script::Module(torch::jit::load(filename));) - return nullptr; -} - -module atm_load_on_device(char *filename, int device) { - PROTECT(return new torch::jit::script::Module( - torch::jit::load(filename, device_of_int(device)));) - return nullptr; -} - -module atm_load_str(char *data, size_t sz) { - PROTECT(std::istringstream stream(std::string(data, sz)); - return new torch::jit::script::Module(torch::jit::load(stream));) - return nullptr; -} - -module atm_load_str_on_device(char *data, size_t sz, int device) { - PROTECT(std::istringstream stream(std::string(data, sz)); - return new torch::jit::script::Module( - torch::jit::load(stream, device_of_int(device)));) - return nullptr; -} - -tensor atm_forward(module m, tensor *tensors, int ntensors) { - PROTECT(std::vector inputs; - for (int i = 0; i < ntensors; ++i) inputs.push_back(*(tensors[i])); - torch::jit::IValue output = m->forward(std::move(inputs)); - if (!output.isTensor()) throw std::invalid_argument( - "forward did not return a tensor"); - return new torch::Tensor(output.toTensor());) - return nullptr; -} - -ivalue atm_forward_(module m, ivalue *ivalues, int nivalues) { - PROTECT(std::vector inputs; - for (int i = 0; i < nivalues; ++i) inputs.push_back(*(ivalues[i])); - torch::jit::IValue output = m->forward(std::move(inputs)); - return new torch::jit::IValue(output);) - return nullptr; -} - -tensor atm_method(module m, char *method_name, tensor *tensors, int ntensors) { - PROTECT(std::vector inputs; - for (int i = 0; i < ntensors; ++i) inputs.push_back(*(tensors[i])); - torch::jit::IValue output = - m->get_method(method_name)(std::move(inputs)); - if (!output.isTensor()) throw std::invalid_argument( - "method did not return a tensor"); - return new torch::Tensor(output.toTensor());) - return nullptr; -} - -ivalue atm_method_(module m, char *method_name, ivalue *ivalues, int nivalues) { - PROTECT(std::vector inputs; - for (int i = 0; i < nivalues; ++i) inputs.push_back(*(ivalues[i])); - torch::jit::IValue output = - m->get_method(method_name)(std::move(inputs)); - return new torch::jit::IValue(output);) - return nullptr; -} - -ivalue atm_create_class_(module m, char *clz_name, ivalue *ivalues, - int nivalues) { - PROTECT(std::vector inputs; - for (int i = 0; i < nivalues; ++i) inputs.push_back(*(ivalues[i])); - - c10::QualifiedName base(clz_name); - torch::jit::IValue obj = - m->create_class(c10::QualifiedName(clz_name), std::move(inputs)); - return new torch::jit::IValue(obj);) - return nullptr; -} - -void atm_eval(module m) { PROTECT(m->eval();) } - -void atm_train(module m) { PROTECT(m->train();) } - -void atm_free(module m) { delete (m); } - -void atm_save(module m, char *filename) { PROTECT(m->save(filename);) } - -void atm_to(module m, int device, int dtype, bool non_blocking) { - PROTECT(m->to(device_of_int(device), at::ScalarType(dtype), non_blocking);) -} - -int atm_get_profiling_mode() { - PROTECT(return torch::jit::getProfilingMode();) - return 0; -} - -void atm_set_tensor_expr_fuser_enabled(int enabled) { - PROTECT(torch::jit::setTensorExprFuserEnabled((bool)enabled);) -} - -bool atm_get_tensor_expr_fuser_enabled() { - PROTECT(return torch::jit::tensorExprFuserEnabled();) - return false; -} - -void atm_set_profiling_mode(int b) { - PROTECT(torch::jit::getProfilingMode() = (bool)b;) -} - -void atm_fuser_cuda_set_enabled(bool b) { - PROTECT(torch::jit::fuser::cuda::setEnabled(b);) -} - -bool atm_fuser_cuda_is_enabled() { - PROTECT(return torch::jit::fuser::cuda::isEnabled();) - return false; -} - -module atm_create_for_tracing(char *modl_name, tensor *inputs, int ninputs) { - PROTECT(torch::jit::script::Module modl(modl_name); - if (torch::jit::tracer::isTracing()) throw std::invalid_argument( - "cannot nest tracing calls"); - auto state = std::make_shared(); - torch::jit::tracer::setTracingState(state); - auto *_modl_value = state->graph->insertInput(0, "self")->setType( - modl._ivalue()->type()); - for (int i = 0; i < ninputs; ++i) { - auto value = state->graph->addInput(); - value->setType(torch::jit::TensorType::get()); - state->setValue(*inputs[i], value); - } return new torch::jit::script::Module(modl);) - torch::jit::tracer::abandon(); - return nullptr; -} - -void atm_end_tracing(module m, char *fn_name, tensor *outputs, int noutputs) { - PROTECT( - auto state = torch::jit::tracer::getTracingState(); - if (state == nullptr) throw std::invalid_argument("not in tracing mode"); - for (int i = 0; i < noutputs; ++i) { - state->graph->registerOutput(state->getOutput(*outputs[i], i)); - } torch::jit::FixupTraceScopeBlocks(state->graph, m); - torch::jit::NormalizeOps(state->graph); - torch::jit::tracer::setTracingState(nullptr); - auto fn = m->_ivalue()->compilation_unit()->create_function(fn_name, - state->graph); - m->type()->addMethod(fn);) -} - -void atm_named_parameters(module m, void *data, - void (*f)(void *, char *, tensor)) { - PROTECT(for (const auto &p : m->named_parameters()) { - auto v = p.value; - f(data, (char *)p.name.c_str(), new torch::Tensor(v)); - }) -} - -ivalue ati_tensor(tensor t) { - PROTECT(return new torch::jit::IValue(*t);) - return nullptr; -} - -ivalue ati_int(int64_t i) { - PROTECT(return new torch::jit::IValue(i);) - return nullptr; -} - -ivalue ati_double(double d) { - PROTECT(return new torch::jit::IValue(d);) - return nullptr; -} - -ivalue ati_bool(int i) { - PROTECT(return new torch::jit::IValue((bool)i);) - return nullptr; -} - -ivalue ati_string(char *s) { - PROTECT(string str(s); return new torch::jit::IValue(str);) - return nullptr; -} - -ivalue ati_none() { - PROTECT(return new torch::jit::IValue();) - return nullptr; -} - -ivalue ati_tuple(ivalue *is, int nvalues) { - PROTECT(vector vec; - for (int i = 0; i < nvalues; ++i) vec.push_back(*(is[i])); - return new torch::jit::IValue(torch::ivalue::Tuple::create(vec));) - return nullptr; -} - -ivalue ati_generic_list(ivalue *is, int nvalues) { - PROTECT(c10::List vec(c10::AnyType::get()); - for (int i = 0; i < nvalues; ++i) vec.push_back(*(is[i])); - return new torch::jit::IValue(c10::List(vec));) - return nullptr; -} - -using generic_dict = c10::Dict; - -ivalue ati_generic_dict(ivalue *is, int nvalues) { - PROTECT( - bool all_keys_are_str = true; for (int i = 0; i < nvalues; ++i) { - if (!is[2 * i]->isString()) - all_keys_are_str = false; - } bool all_values_are_tensor = true; - for (int i = 0; i < nvalues; ++i) { - if (!is[2 * i + 1]->isTensor()) - all_values_are_tensor = false; - } if (all_keys_are_str && all_values_are_tensor) { - generic_dict dict(c10::StringType::get(), c10::TensorType::get()); - for (int i = 0; i < nvalues; ++i) - dict.insert(is[2 * i]->toString(), is[2 * i + 1]->toTensor()); - return new torch::jit::IValue(dict); - } else if (all_keys_are_str) { - generic_dict dict(c10::StringType::get(), c10::AnyType::get()); - for (int i = 0; i < nvalues; ++i) - dict.insert(is[2 * i]->toString(), *(is[2 * i + 1])); - return new torch::jit::IValue(dict); - } else { - generic_dict dict(c10::AnyType::get(), c10::AnyType::get()); - for (int i = 0; i < nvalues; ++i) - dict.insert(*(is[2 * i]), *(is[2 * i + 1])); - return new torch::jit::IValue(dict); - }) - return nullptr; -} - -ivalue ati_int_list(int64_t *is, int nvalues) { - PROTECT(c10::List vec; - for (int i = 0; i < nvalues; ++i) vec.push_back(is[i]); - return new torch::jit::IValue(vec);) - return nullptr; -} - -ivalue ati_double_list(double *is, int nvalues) { - PROTECT(c10::List vec; - for (int i = 0; i < nvalues; ++i) vec.push_back(is[i]); - return new torch::jit::IValue(vec);) - return nullptr; -} - -ivalue ati_bool_list(char *is, int nvalues) { - PROTECT(c10::List vec; - for (int i = 0; i < nvalues; ++i) vec.push_back(is[i] != 0); - return new torch::jit::IValue(vec);) - return nullptr; -} - -ivalue ati_string_list(char **is, int nvalues) { - PROTECT(c10::List vec; - for (int i = 0; i < nvalues; ++i) vec.push_back(string(is[i])); - return new torch::jit::IValue(vec);) - return nullptr; -} - -ivalue ati_tensor_list(tensor *is, int nvalues) { - PROTECT(c10::List vec; - for (int i = 0; i < nvalues; ++i) vec.push_back(*(is[i])); - return new torch::jit::IValue(vec);) - return nullptr; -} - -ivalue ati_device(int device_idx) { - PROTECT(return new torch::jit::IValue(device_of_int(device_idx));) - return nullptr; -} - -int ati_tag(ivalue i) { - PROTECT( - if (i->isNone()) return 0; else if (i->isTensor()) return 1; - else if (i->isDouble()) return 2; else if (i->isInt()) return 3; - else if (i->isBool()) return 4; else if (i->isTuple()) return 5; - else if (i->isIntList()) return 6; else if (i->isDoubleList()) return 7; - else if (i->isBoolList()) return 8; else if (i->isString()) return 9; - else if (i->isTensorList()) return 10; else if (i->isList()) return 12; - else if (i->isGenericDict()) return 13; else if (i->isObject()) return 14; - throw std::invalid_argument(("unsupported tag " + i->tagKind()).c_str()); - return -1;) - return -1; -} - -int64_t ati_to_int(ivalue i) { - PROTECT(return i->toInt();) - return -1; -} - -double ati_to_double(ivalue i) { - PROTECT(return i->toDouble();) - return 0; -} - -int ati_to_bool(ivalue i) { - PROTECT(return i->toBool();) - return -1; -} - -char *ati_to_string(ivalue i) { - PROTECT(auto str = i->toStringRef(); return strdup(str.c_str());) - return nullptr; -} - -tensor ati_to_tensor(ivalue i) { - PROTECT(return new torch::Tensor(i->toTensor());) - return nullptr; -} - -int ati_length(ivalue i) { - PROTECT(if (i->isTuple()) return i->toTuple()->elements().size(); - else if (i->isIntList()) return i->toIntList().size(); - else if (i->isDoubleList()) return i->toDoubleList().size(); - else if (i->isBoolList()) return i->toBoolList().size(); - else if (i->isString()) return i->toStringRef().size(); - else if (i->isTensorList()) return i->toTensorList().size(); - else if (i->isList()) return i->toList().size(); - else if (i->isGenericDict()) return i->toGenericDict().size(); - throw std::invalid_argument( - ("unsupported tag for length " + i->tagKind()).c_str()); - return -1;) - return -1; -} - -int ati_tuple_length(ivalue i) { - PROTECT(return i->toTuple()->elements().size();) - return -1; -} - -void ati_to_tuple(ivalue i, ivalue *outputs, int noutputs) { - PROTECT(auto vec = i->toTuple()->elements(); if (vec.size() != noutputs) { - throw std::invalid_argument("unexpected tuple size"); - } for (int i = 0; i < noutputs; - ++i) outputs[i] = new torch::jit::IValue(vec[i]);) -} - -void ati_to_generic_list(ivalue i, ivalue *outputs, int noutputs) { - PROTECT(auto vec = i->toList(); if (vec.size() != noutputs) { - throw std::invalid_argument("unexpected list size"); - } for (int i = 0; i < noutputs; ++i) outputs[i] = - new torch::jit::IValue(vec[i]);) -} - -void ati_to_generic_dict(ivalue i, ivalue *outputs, int noutputs) { - PROTECT( - auto dict = i->toGenericDict(); if (dict.size() != noutputs) { - throw std::invalid_argument("unexpected dict size"); - } int k = 0; - for (auto it = dict.begin(); it != dict.end(); ++it) { - outputs[k++] = new torch::jit::IValue(it->key()); - outputs[k++] = new torch::jit::IValue(it->value()); - }) -} - -void ati_to_int_list(ivalue i, int64_t *outputs, int noutputs) { - PROTECT(auto vec = i->toIntList(); if (vec.size() != noutputs) { - throw std::invalid_argument("unexpected list size"); - } for (int i = 0; i < noutputs; ++i) outputs[i] = vec[i];) -} - -void ati_to_double_list(ivalue i, double *outputs, int noutputs) { - PROTECT(auto vec = i->toDoubleList(); if (vec.size() != noutputs) { - throw std::invalid_argument("unexpected list size"); - } for (int i = 0; i < noutputs; ++i) outputs[i] = vec[i];) -} - -void ati_to_bool_list(ivalue i, char *outputs, int noutputs) { - PROTECT(auto vec = i->toBoolList(); if (vec.size() != noutputs) { - throw std::invalid_argument("unexpected list size"); - } for (int i = 0; i < noutputs; ++i) outputs[i] = vec[i];) -} - -void ati_to_tensor_list(ivalue i, tensor *outputs, int noutputs) { - PROTECT(auto vec = i->toTensorList(); if (vec.size() != noutputs) { - throw std::invalid_argument("unexpected list size"); - } for (int i = 0; i < noutputs; ++i) outputs[i] = new torch::Tensor(vec[i]);) -} - -ivalue ati_object_method_(ivalue i, char *method_name, ivalue *ivalues, - int nivalues) { - PROTECT(std::vector inputs; - inputs.push_back(*i); // self parameter - for (int j = 0; j < nivalues; ++j) inputs.push_back(*(ivalues[j])); - torch::jit::IValue output = i->toObjectRef().type()->getMethod( - method_name)(std::move(inputs)); - return new torch::jit::IValue(output);) - return nullptr; -} - -ivalue ati_object_getattr_(ivalue i, char *attr_name) { - PROTECT(torch::jit::IValue output = i->toObjectRef().getAttr(attr_name); - return new torch::jit::IValue(output);) - return nullptr; -} - -ivalue ati_clone(ivalue i) { - PROTECT(return new torch::jit::IValue(*i);) - return nullptr; -} - -void ati_free(ivalue i) { delete (i); } - -void at_set_graph_executor_optimize(bool o) { - torch::jit::setGraphExecutorOptimize(o); -} diff --git a/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api.h b/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api.h deleted file mode 100644 index 5628963889..0000000000 --- a/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api.h +++ /dev/null @@ -1,295 +0,0 @@ -#ifndef __TORCH_API_H__ -#define __TORCH_API_H__ -#include - -#ifdef __cplusplus -#include -#include -using namespace std; -extern thread_local char *torch_last_err; - -extern "C" { -typedef torch::Tensor *tensor; -typedef torch::Scalar *scalar; -typedef torch::optim::Optimizer *optimizer; -typedef torch::jit::script::Module *module; -typedef torch::jit::IValue *ivalue; -#define PROTECT(x) \ - try { \ - x \ - } catch (const exception& e) { \ - torch_last_err = strdup(e.what()); \ - } -#else -typedef void *tensor; -typedef void *optimizer; -typedef void *scalar; -typedef void *module; -typedef void *ivalue; -#endif - -char *get_and_reset_last_err(); // thread-local -void at_manual_seed(int64_t); -tensor at_new_tensor(); -tensor at_tensor_of_blob(void *data, int64_t *dims, size_t ndims, int64_t *strides, size_t nstrides, int type, int device); -tensor at_tensor_of_data(void *vs, int64_t *dims, size_t ndims, size_t element_size_in_bytes, int type); -void at_copy_data(tensor tensor, void *vs, size_t numel, size_t element_size_in_bytes); -tensor at_shallow_clone(tensor); - -void *at_data_ptr(tensor); -int at_defined(tensor); -int at_is_mkldnn(tensor); -int at_is_sparse(tensor); -int at_is_contiguous(tensor); -int at_device(tensor); -size_t at_dim(tensor); -void at_shape(tensor, int64_t *); -void at_stride(tensor, int64_t *); -int at_scalar_type(tensor); - -void at__amp_non_finite_check_and_unscale(tensor, tensor, tensor); - -void at_autocast_clear_cache(); -int at_autocast_decrement_nesting(); -int at_autocast_increment_nesting(); -bool at_autocast_is_enabled(); -bool at_autocast_set_enabled(bool b); - -void at_backward(tensor, int, int); -int at_requires_grad(tensor); -int at_grad_set_enabled(int); - -tensor at_get(tensor, int index); -void at_fill_double(tensor, double); -void at_fill_int64(tensor, int64_t); - -double at_double_value_at_indexes(tensor, int64_t *indexes, int indexes_len); -int64_t at_int64_value_at_indexes(tensor, int64_t *indexes, int indexes_len); -void at_set_double_value_at_indexes(tensor, int *indexes, int indexes_len, double v); -void at_set_int64_value_at_indexes(tensor, int *indexes, int indexes_len, int64_t v); - -void at_copy_(tensor dst, tensor src); - -void at_print(tensor); -char *at_to_string(tensor, int line_size); -void at_save(tensor, char *filename); -void at_save_to_stream(tensor t, void *stream_ptr); -tensor at_load(char *filename); -tensor at_load_from_stream(void *stream_ptr); -tensor at_load_image(char *filename); -tensor at_load_image_from_memory(unsigned char *img_data, size_t img_size); -int at_save_image(tensor, char *filename); -tensor at_resize_image(tensor, int w, int h); - -void at_save_multi(tensor *tensors, char **tensor_names, int ntensors, char *filename); -void at_save_multi_to_stream(tensor *tensors, char **tensor_names, int ntensors, void *stream_ptr); -/* [at_load_multi] takes as input an array of nullptr for [tensors]. */ -void at_load_multi(tensor *tensors, char **tensor_names, int ntensors, char *filename); -/* [at_load_multi_] takes as input an array of allocation [tensors]. */ -void at_load_multi_(tensor *tensors, char **tensor_names, int ntensors, char *filename); - -void at_loadz_callback(char *filename, void *data, void (*f)(void *, char *, tensor)); -void at_loadz_callback_with_device(char *filename, void *data, void (*f)(void *, char *, tensor), int device_id); -void at_load_callback(char *filename, void *data, void (*f)(void *, char *, tensor)); -void at_load_callback_with_device(char *filename, void *data, void (*f)(void *, char *, tensor), int device_id); -void at_load_from_stream_callback(void *stream_ptr, void *data, void (*f)(void *, char *, tensor), bool enable_device_id, int device_id); - -int at_get_num_interop_threads(); - -int at_get_num_threads(); - -void at_set_num_interop_threads(int n_threads); - -void at_set_num_threads(int n_threads); - -void at_set_qengine(int qengine); - -void at_free(tensor); - -void at_run_backward(tensor *tensors, - int ntensors, - tensor *inputs, - int ninputs, - tensor *outputs, - int keep_graph, - int create_graph); - -optimizer ato_adam(double learning_rate, - double beta1, - double beta2, - double weight_decay, - double eps, - bool amsgrad); -optimizer ato_adamw(double learning_rate, - double beta1, - double beta2, - double weight_decay, - double eps, - bool amsgrad); -optimizer ato_rms_prop(double learning_rate, - double alpha, - double eps, - double weight_decay, - double momentum, - int centered); -optimizer ato_sgd(double learning_rate, - double momentum, - double dampening, - double weight_decay, - int nesterov); -void ato_add_parameters(optimizer, tensor, size_t group); -void ato_set_learning_rate(optimizer, double learning_rate); -void ato_set_momentum(optimizer, double momentum); -void ato_set_learning_rate_group(optimizer, size_t group, double learning_rate); -void ato_set_momentum_group(optimizer, size_t group, double momentum); -void ato_set_weight_decay(optimizer t, double weight_decay); -void ato_set_weight_decay_group(optimizer t, size_t group, double weight_decay); -void ato_zero_grad(optimizer); -void ato_step(optimizer); -void ato_free(optimizer); - -scalar ats_int(int64_t); -scalar ats_float(double); -int64_t ats_to_int(scalar); -double ats_to_float(scalar); -char *ats_to_string(scalar); -void ats_free(scalar); - -bool at_context_has_openmp(); -bool at_context_has_mkl(); -bool at_context_has_lapack(); -bool at_context_has_mkldnn(); -bool at_context_has_magma(); -bool at_context_has_cuda(); -bool at_context_has_cudart(); -bool at_context_has_cudnn(); -int64_t at_context_version_cudnn(); -int64_t at_context_version_cudart(); -bool at_context_has_cusolver(); -bool at_context_has_hip(); -bool at_context_has_ipu(); -bool at_context_has_xla(); -bool at_context_has_lazy(); -bool at_context_has_mps(); - - -/// Returns the number of CUDA devices available. -int atc_cuda_device_count(); - -/// Returns true if at least one CUDA device is available. -int atc_cuda_is_available(); - -/// Returns true if CUDA is available, and CuDNN is available. -int atc_cudnn_is_available(); - -/// Sets the seed for the current GPU. -void atc_manual_seed(uint64_t seed); - -/// Sets the seed for all available GPUs. -void atc_manual_seed_all(uint64_t seed); - -/// Waits for all kernels in all streams on a CUDA device to complete. -void atc_synchronize(int64_t device_index); - - -int atc_user_enabled_cudnn(); -void atc_set_user_enabled_cudnn(int b); -void atc_set_benchmark_cudnn(int b); - -module atm_load(char *); -module atm_load_on_device(char *, int device); -module atm_load_str(char *, size_t sz); -module atm_load_str_on_device(char *, size_t sz, int device); -tensor atm_forward(module, tensor *tensors, int ntensors); -ivalue atm_forward_(module, - ivalue *ivalues, - int nivalues); -tensor atm_method(module, - char *method_name, - tensor *tensors, - int ntensors); -ivalue atm_method_(module, - char *method_name, - ivalue *ivalues, - int nivalues); -ivalue atm_create_class_(module, - char *clz_name, - ivalue *ivalues, - int nivalues); -void atm_eval(module); -void atm_train(module); -void atm_free(module); -void atm_to(module m, int device, int dtype, bool non_blocking); -void atm_save(module m, char*); -int atm_get_profiling_mode(); -void atm_set_profiling_mode(int); -void atm_fuser_cuda_set_enabled(bool); -bool atm_fuser_cuda_is_enabled(); -void atm_named_parameters(module, void *data, void (*f)(void *, char *, tensor)); - -// This function has to be followed by a call to atm_end_tracing. -module atm_create_for_tracing(char *modl_name, tensor *inputs, int ninputs); -void atm_end_tracing(module m, char *fn_name, tensor *outputs, int noutputs); - -ivalue ati_none(); -ivalue ati_tensor(tensor); -ivalue ati_int(int64_t); -ivalue ati_double(double); -ivalue ati_bool(int); -ivalue ati_string(char *); -ivalue ati_tuple(ivalue *, int); -ivalue ati_generic_list(ivalue *, int); -ivalue ati_generic_dict(ivalue *, int); -ivalue ati_int_list(int64_t *, int); -ivalue ati_double_list(double *, int); -ivalue ati_bool_list(char *, int); -ivalue ati_string_list(char **, int); -ivalue ati_tensor_list(tensor *, int); -ivalue ati_device(int); - -tensor ati_to_tensor(ivalue); -int64_t ati_to_int(ivalue); -double ati_to_double(ivalue); -char *ati_to_string(ivalue); -int ati_to_bool(ivalue); -int ati_length(ivalue); -int ati_tuple_length(ivalue); -void ati_to_tuple(ivalue, ivalue *, int); -void ati_to_generic_list(ivalue, ivalue *, int); -void ati_to_generic_dict(ivalue, ivalue *, int); -void ati_to_int_list(ivalue, int64_t *, int); -void ati_to_double_list(ivalue, double *, int); -void ati_to_bool_list(ivalue, char *, int); -void ati_to_tensor_list(ivalue, tensor *, int); - -void atm_set_tensor_expr_fuser_enabled(int); -bool atm_get_tensor_expr_fuser_enabled(); - -int ati_tag(ivalue); - -ivalue ati_object_method_(ivalue i, char *method_name, ivalue *ivalues, int nivalues); -ivalue ati_object_getattr_(ivalue i, char *attr_name); - -ivalue ati_clone(ivalue); -void ati_free(ivalue); - -/// Enables or disables the graph executor optimizer for the current thread. -void at_set_graph_executor_optimize(bool); - -// for internal use -bool tch_write_stream_destructor(void *stream_ptr); -bool tch_write_stream_write(void *stream_ptr, const uint8_t *buf, size_t size, size_t *out_size); -bool tch_read_stream_destructor(void *stream_ptr); -bool tch_read_stream_stream_position(void *stream_ptr, uint64_t *pos); -bool tch_read_stream_seek_start(void *stream_ptr, uint64_t pos, uint64_t *new_pos); -bool tch_read_stream_seek_end(void *stream_ptr, int64_t pos, uint64_t *new_pos); -bool tch_read_stream_read(void *stream_ptr, uint8_t *buf, size_t size, size_t *new_pos); - -#ifdef __cplusplus -}; - -std::vector of_carray_tensor(torch::Tensor **vs, int len); -at::Device device_of_int(int d); -c10::List> of_carray_tensor_opt(torch::Tensor **vs, int len); -#endif -#endif diff --git a/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api_generated.cpp b/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api_generated.cpp deleted file mode 100644 index acd34c8355..0000000000 --- a/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api_generated.cpp +++ /dev/null @@ -1,19081 +0,0 @@ -// THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT BY HAND! -#include "torch_api_generated.h" - -void atg___and__(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::__and__(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___and__tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::__and__(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___iand__(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->__iand__(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___iand__tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->__iand__(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___ilshift__(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->__ilshift__(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___ilshift__tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->__ilshift__(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___ior__(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->__ior__(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___ior__tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->__ior__(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___irshift__(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->__irshift__(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___irshift__tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->__irshift__(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___ixor__(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->__ixor__(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___ixor__tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->__ixor__(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___lshift__(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::__lshift__(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___lshift__scalar_out_(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::__lshift___out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___lshift__tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::__lshift__(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___lshift__tensor_out_(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::__lshift___out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___or__(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::__or__(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___or__tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::__or__(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___rshift__(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::__rshift__(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___rshift__scalar_out_(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::__rshift___out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___rshift__tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::__rshift__(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___rshift__tensor_out_(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::__rshift___out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___xor__(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::__xor__(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg___xor__tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::__xor__(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__adaptive_avg_pool2d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len) { - PROTECT( - auto outputs__ = torch::_adaptive_avg_pool2d(*self, torch::IntArrayRef(output_size_data, output_size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__adaptive_avg_pool2d_backward(tensor *out__, tensor grad_output, tensor self) { - PROTECT( - auto outputs__ = torch::_adaptive_avg_pool2d_backward(*grad_output, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__adaptive_avg_pool2d_backward_out(tensor *out__, tensor out, tensor grad_output, tensor self) { - PROTECT( - auto outputs__ = torch::_adaptive_avg_pool2d_backward_out(*out, *grad_output, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__adaptive_avg_pool2d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len) { - PROTECT( - auto outputs__ = torch::_adaptive_avg_pool2d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__adaptive_avg_pool3d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len) { - PROTECT( - auto outputs__ = torch::_adaptive_avg_pool3d(*self, torch::IntArrayRef(output_size_data, output_size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__adaptive_avg_pool3d_backward(tensor *out__, tensor grad_output, tensor self) { - PROTECT( - auto outputs__ = torch::_adaptive_avg_pool3d_backward(*grad_output, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__adaptive_avg_pool3d_backward_out(tensor *out__, tensor out, tensor grad_output, tensor self) { - PROTECT( - auto outputs__ = torch::_adaptive_avg_pool3d_backward_out(*out, *grad_output, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__adaptive_avg_pool3d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len) { - PROTECT( - auto outputs__ = torch::_adaptive_avg_pool3d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__add_batch_dim(tensor *out__, tensor self, int64_t batch_dim, int64_t level) { - PROTECT( - auto outputs__ = torch::_add_batch_dim(*self, batch_dim, level); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__add_relu(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::_add_relu(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__add_relu_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::_add_relu_(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__add_relu_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::_add_relu_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__add_relu_scalar(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::_add_relu(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__add_relu_scalar_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::_add_relu_(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__add_relu_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::_add_relu_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__addmm_activation(tensor *out__, tensor self, tensor mat1, tensor mat2, int use_gelu) { - PROTECT( - auto outputs__ = torch::_addmm_activation(*self, *mat1, *mat2, (bool)use_gelu); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__addmm_activation_out(tensor *out__, tensor out, tensor self, tensor mat1, tensor mat2, int use_gelu) { - PROTECT( - auto outputs__ = torch::_addmm_activation_out(*out, *self, *mat1, *mat2, (bool)use_gelu); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__aminmax(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_aminmax(*self); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__aminmax_dim(tensor *out__, tensor self, int64_t dim, int keepdim) { - PROTECT( - auto outputs__ = torch::_aminmax(*self, dim, (bool)keepdim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__aminmax_dim_out(tensor *out__, tensor out0, tensor out1, tensor self, int64_t dim, int keepdim) { - PROTECT( - auto outputs__ = torch::_aminmax_out(*out0, *out1, *self, dim, (bool)keepdim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__aminmax_out(tensor *out__, tensor out0, tensor out1, tensor self) { - PROTECT( - auto outputs__ = torch::_aminmax_out(*out0, *out1, *self); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__amp_update_scale(tensor *out__, tensor self, tensor growth_tracker, tensor found_inf, double scale_growth_factor, double scale_backoff_factor, int64_t growth_interval) { - PROTECT( - auto outputs__ = torch::_amp_update_scale(*self, *growth_tracker, *found_inf, scale_growth_factor, scale_backoff_factor, growth_interval); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__amp_update_scale_(tensor *out__, tensor self, tensor growth_tracker, tensor found_inf, double scale_growth_factor, double scale_backoff_factor, int64_t growth_interval) { - PROTECT( - auto outputs__ = torch::_amp_update_scale_(*self, *growth_tracker, *found_inf, scale_growth_factor, scale_backoff_factor, growth_interval); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__amp_update_scale_out(tensor *out__, tensor out, tensor self, tensor growth_tracker, tensor found_inf, double scale_growth_factor, double scale_backoff_factor, int64_t growth_interval) { - PROTECT( - auto outputs__ = torch::_amp_update_scale_out(*out, *self, *growth_tracker, *found_inf, scale_growth_factor, scale_backoff_factor, growth_interval); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__assert_scalar(scalar self_scalar, char* assert_msg_ptr, int assert_msg_len) { - PROTECT( - torch::_assert_scalar(*self_scalar, std::string(assert_msg_ptr, assert_msg_len)); - ) -} - -void atg__assert_tensor_metadata(tensor a, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int dtype, int device, int8_t layout) { - PROTECT( - torch::_assert_tensor_metadata(*a, size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(size_data, size_len)), stride_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(stride_data, stride_len)), dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype)), device_of_int(device), (layout == -1 ? c10::nullopt : c10::optional(static_cast(layout)))); - ) -} - -void atg__autocast_to_full_precision(tensor *out__, tensor self, int cuda_enabled, int cpu_enabled) { - PROTECT( - auto outputs__ = self->_autocast_to_full_precision((bool)cuda_enabled, (bool)cpu_enabled); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__autocast_to_reduced_precision(tensor *out__, tensor self, int cuda_enabled, int cpu_enabled, int cuda_dtype, int cpu_dtype) { - PROTECT( - auto outputs__ = self->_autocast_to_reduced_precision((bool)cuda_enabled, (bool)cpu_enabled, at::ScalarType(cuda_dtype), at::ScalarType(cpu_dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__batch_norm_no_update(tensor *out__, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps) { - PROTECT( - auto outputs__ = torch::_batch_norm_no_update(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), momentum, eps); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg__batch_norm_no_update_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor out3, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps) { - PROTECT( - auto outputs__ = torch::_batch_norm_no_update_out(*out0, *out1, *out2, *out3, *input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), momentum, eps); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg__batch_norm_with_update(tensor *out__, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps) { - PROTECT( - auto outputs__ = torch::_batch_norm_with_update(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *running_mean, *running_var, momentum, eps); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg__batch_norm_with_update_functional(tensor *out__, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps) { - PROTECT( - auto outputs__ = torch::_batch_norm_with_update_functional(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *running_mean, *running_var, momentum, eps); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - out__[4] = new torch::Tensor(std::get<4>(outputs__)); - out__[5] = new torch::Tensor(std::get<5>(outputs__)); - ) -} - -void atg__batch_norm_with_update_out(tensor *out__, tensor out, tensor save_mean, tensor save_invstd, tensor reserve, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps) { - PROTECT( - auto outputs__ = torch::_batch_norm_with_update_out(*out, *save_mean, *save_invstd, *reserve, *input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *running_mean, *running_var, momentum, eps); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg__cast_byte(tensor *out__, tensor self, int non_blocking) { - PROTECT( - auto outputs__ = torch::_cast_Byte(*self, (bool)non_blocking); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__cast_char(tensor *out__, tensor self, int non_blocking) { - PROTECT( - auto outputs__ = torch::_cast_Char(*self, (bool)non_blocking); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__cast_double(tensor *out__, tensor self, int non_blocking) { - PROTECT( - auto outputs__ = torch::_cast_Double(*self, (bool)non_blocking); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__cast_float(tensor *out__, tensor self, int non_blocking) { - PROTECT( - auto outputs__ = torch::_cast_Float(*self, (bool)non_blocking); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__cast_half(tensor *out__, tensor self, int non_blocking) { - PROTECT( - auto outputs__ = torch::_cast_Half(*self, (bool)non_blocking); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__cast_int(tensor *out__, tensor self, int non_blocking) { - PROTECT( - auto outputs__ = torch::_cast_Int(*self, (bool)non_blocking); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__cast_long(tensor *out__, tensor self, int non_blocking) { - PROTECT( - auto outputs__ = torch::_cast_Long(*self, (bool)non_blocking); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__cast_short(tensor *out__, tensor self, int non_blocking) { - PROTECT( - auto outputs__ = torch::_cast_Short(*self, (bool)non_blocking); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__cdist_backward(tensor *out__, tensor grad, tensor x1, tensor x2, double p, tensor cdist) { - PROTECT( - auto outputs__ = torch::_cdist_backward(*grad, *x1, *x2, p, *cdist); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__cdist_backward_out(tensor *out__, tensor out, tensor grad, tensor x1, tensor x2, double p, tensor cdist) { - PROTECT( - auto outputs__ = torch::_cdist_backward_out(*out, *grad, *x1, *x2, p, *cdist); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__cholesky_solve_helper(tensor *out__, tensor self, tensor A, int upper) { - PROTECT( - auto outputs__ = torch::_cholesky_solve_helper(*self, *A, (bool)upper); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__cholesky_solve_helper_out(tensor *out__, tensor out, tensor self, tensor A, int upper) { - PROTECT( - auto outputs__ = torch::_cholesky_solve_helper_out(*out, *self, *A, (bool)upper); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__chunk_cat(tensor *out__, tensor *tensors_data, int tensors_len, int64_t dim, int64_t num_chunks) { - PROTECT( - auto outputs__ = torch::_chunk_cat(of_carray_tensor(tensors_data, tensors_len), dim, num_chunks); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__chunk_cat_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len, int64_t dim, int64_t num_chunks) { - PROTECT( - auto outputs__ = torch::_chunk_cat_out(*out, of_carray_tensor(tensors_data, tensors_len), dim, num_chunks); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__coalesce(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_coalesce(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__coalesce_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::_coalesce_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__coalesced(tensor *out__, tensor self, int coalesced) { - PROTECT( - auto outputs__ = torch::_coalesced(*self, (bool)coalesced); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__coalesced_(tensor *out__, tensor self, int coalesced) { - PROTECT( - auto outputs__ = self->_coalesced_((bool)coalesced); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__coalesced_out(tensor *out__, tensor out, tensor self, int coalesced) { - PROTECT( - auto outputs__ = torch::_coalesced_out(*out, *self, (bool)coalesced); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__compute_linear_combination(tensor *out__, tensor input, tensor coefficients) { - PROTECT( - auto outputs__ = torch::_compute_linear_combination(*input, *coefficients); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__compute_linear_combination_out(tensor *out__, tensor out, tensor input, tensor coefficients) { - PROTECT( - auto outputs__ = torch::_compute_linear_combination_out(*out, *input, *coefficients); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__conj(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_conj(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__conj_copy(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_conj_copy(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__conj_copy_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::_conj_copy_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__conj_physical(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_conj_physical(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__conj_physical_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::_conj_physical_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__conv_depthwise2d(tensor *out__, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len) { - PROTECT( - auto outputs__ = torch::_conv_depthwise2d(*self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__conv_depthwise2d_out(tensor *out__, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len) { - PROTECT( - auto outputs__ = torch::_conv_depthwise2d_out(*out, *self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__convert_indices_from_coo_to_csr(tensor *out__, tensor self, int64_t size, int out_int32) { - PROTECT( - auto outputs__ = torch::_convert_indices_from_coo_to_csr(*self, size, (bool)out_int32); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__convert_indices_from_coo_to_csr_out(tensor *out__, tensor out, tensor self, int64_t size, int out_int32) { - PROTECT( - auto outputs__ = torch::_convert_indices_from_coo_to_csr_out(*out, *self, size, (bool)out_int32); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__convert_indices_from_csr_to_coo(tensor *out__, tensor crow_indices, tensor col_indices, int out_int32, int transpose) { - PROTECT( - auto outputs__ = torch::_convert_indices_from_csr_to_coo(*crow_indices, *col_indices, (bool)out_int32, (bool)transpose); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__convert_indices_from_csr_to_coo_out(tensor *out__, tensor out, tensor crow_indices, tensor col_indices, int out_int32, int transpose) { - PROTECT( - auto outputs__ = torch::_convert_indices_from_csr_to_coo_out(*out, *crow_indices, *col_indices, (bool)out_int32, (bool)transpose); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__convert_weight_to_int4pack(tensor *out__, tensor self, int64_t innerKTiles) { - PROTECT( - auto outputs__ = torch::_convert_weight_to_int4pack(*self, innerKTiles); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__convert_weight_to_int4pack_for_cpu(tensor *out__, tensor self, int64_t innerKTiles) { - PROTECT( - auto outputs__ = torch::_convert_weight_to_int4pack_for_cpu(*self, innerKTiles); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__convolution(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups, int benchmark, int deterministic, int cudnn_enabled, int allow_tf32) { - PROTECT( - auto outputs__ = torch::_convolution(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)transposed, torch::IntArrayRef(output_padding_data, output_padding_len), groups, (bool)benchmark, (bool)deterministic, (bool)cudnn_enabled, (bool)allow_tf32); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__convolution_deprecated(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups, int benchmark, int deterministic, int cudnn_enabled) { - PROTECT( - auto outputs__ = torch::_convolution(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)transposed, torch::IntArrayRef(output_padding_data, output_padding_len), groups, (bool)benchmark, (bool)deterministic, (bool)cudnn_enabled); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__convolution_mode(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, char* padding_ptr, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::_convolution_mode(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), std::string(padding_ptr, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__convolution_out(tensor *out__, tensor out, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups, int benchmark, int deterministic, int cudnn_enabled, int allow_tf32) { - PROTECT( - auto outputs__ = torch::_convolution_out(*out, *input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)transposed, torch::IntArrayRef(output_padding_data, output_padding_len), groups, (bool)benchmark, (bool)deterministic, (bool)cudnn_enabled, (bool)allow_tf32); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__copy_from(tensor *out__, tensor self, tensor dst, int non_blocking) { - PROTECT( - auto outputs__ = torch::_copy_from(*self, *dst, (bool)non_blocking); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__copy_from_and_resize(tensor *out__, tensor self, tensor dst) { - PROTECT( - auto outputs__ = torch::_copy_from_and_resize(*self, *dst); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__copy_from_and_resize_out(tensor *out__, tensor out, tensor self, tensor dst) { - PROTECT( - auto outputs__ = torch::_copy_from_and_resize_out(*out, *self, *dst); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__copy_from_out(tensor *out__, tensor out, tensor self, tensor dst, int non_blocking) { - PROTECT( - auto outputs__ = torch::_copy_from_out(*out, *self, *dst, (bool)non_blocking); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__cslt_compress(tensor *out__, tensor input) { - PROTECT( - auto outputs__ = torch::_cslt_compress(*input); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__cslt_sparse_mm(tensor *out__, tensor compressed_A, tensor dense_B, tensor bias, tensor alpha, int out_dtype, int transpose_result, int64_t alg_id, int64_t split_k, int64_t split_k_mode) { - PROTECT( - auto outputs__ = torch::_cslt_sparse_mm(*compressed_A, *dense_B, (bias ? ::std::optional(*bias) : ::std::nullopt), (alpha ? ::std::optional(*alpha) : ::std::nullopt), out_dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(out_dtype)), (bool)transpose_result, alg_id, split_k, split_k_mode); - out__[0] = new torch::Tensor(outputs__); - ) -} - -int64_t atg__cslt_sparse_mm_search(tensor compressed_A, tensor dense_B, tensor bias, tensor alpha, int out_dtype, int transpose_result) { - PROTECT( - return torch::_cslt_sparse_mm_search(*compressed_A, *dense_B, (bias ? ::std::optional(*bias) : ::std::nullopt), (alpha ? ::std::optional(*alpha) : ::std::nullopt), out_dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(out_dtype)), (bool)transpose_result); - ) - return 0; -} - -void atg__ctc_loss(tensor *out__, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int zero_infinity) { - PROTECT( - auto outputs__ = torch::_ctc_loss(*log_probs, *targets, torch::IntArrayRef(input_lengths_data, input_lengths_len), torch::IntArrayRef(target_lengths_data, target_lengths_len), blank, (bool)zero_infinity); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__ctc_loss_backward(tensor *out__, tensor grad, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, tensor neg_log_likelihood, tensor log_alpha, int64_t blank, int zero_infinity) { - PROTECT( - auto outputs__ = torch::_ctc_loss_backward(*grad, *log_probs, *targets, torch::IntArrayRef(input_lengths_data, input_lengths_len), torch::IntArrayRef(target_lengths_data, target_lengths_len), *neg_log_likelihood, *log_alpha, blank, (bool)zero_infinity); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__ctc_loss_backward_out(tensor *out__, tensor out, tensor grad, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, tensor neg_log_likelihood, tensor log_alpha, int64_t blank, int zero_infinity) { - PROTECT( - auto outputs__ = torch::_ctc_loss_backward_out(*out, *grad, *log_probs, *targets, torch::IntArrayRef(input_lengths_data, input_lengths_len), torch::IntArrayRef(target_lengths_data, target_lengths_len), *neg_log_likelihood, *log_alpha, blank, (bool)zero_infinity); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__ctc_loss_backward_tensor(tensor *out__, tensor grad, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, tensor neg_log_likelihood, tensor log_alpha, int64_t blank, int zero_infinity) { - PROTECT( - auto outputs__ = torch::_ctc_loss_backward(*grad, *log_probs, *targets, *input_lengths, *target_lengths, *neg_log_likelihood, *log_alpha, blank, (bool)zero_infinity); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__ctc_loss_out(tensor *out__, tensor out0, tensor out1, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int zero_infinity) { - PROTECT( - auto outputs__ = torch::_ctc_loss_out(*out0, *out1, *log_probs, *targets, torch::IntArrayRef(input_lengths_data, input_lengths_len), torch::IntArrayRef(target_lengths_data, target_lengths_len), blank, (bool)zero_infinity); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__ctc_loss_tensor(tensor *out__, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank, int zero_infinity) { - PROTECT( - auto outputs__ = torch::_ctc_loss(*log_probs, *targets, *input_lengths, *target_lengths, blank, (bool)zero_infinity); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__ctc_loss_tensor_out(tensor *out__, tensor out0, tensor out1, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank, int zero_infinity) { - PROTECT( - auto outputs__ = torch::_ctc_loss_out(*out0, *out1, *log_probs, *targets, *input_lengths, *target_lengths, blank, (bool)zero_infinity); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__cudnn_attention_backward(tensor *out__, tensor grad_out, tensor query, tensor key, tensor value, tensor out, tensor logsumexp, tensor philox_seed, tensor philox_offset, tensor attn_bias, tensor cum_seq_q, tensor cum_seq_k, int64_t max_q, int64_t max_k, double dropout_p, int is_causal, double scale_v, uint8_t scale_null) { - PROTECT( - auto outputs__ = torch::_cudnn_attention_backward(*grad_out, *query, *key, *value, *out, *logsumexp, *philox_seed, *philox_offset, *attn_bias, *cum_seq_q, *cum_seq_k, max_q, max_k, dropout_p, (bool)is_causal, scale_null ? c10::nullopt : c10::optional(scale_v)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__cudnn_ctc_loss(tensor *out__, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int deterministic, int zero_infinity) { - PROTECT( - auto outputs__ = torch::_cudnn_ctc_loss(*log_probs, *targets, torch::IntArrayRef(input_lengths_data, input_lengths_len), torch::IntArrayRef(target_lengths_data, target_lengths_len), blank, (bool)deterministic, (bool)zero_infinity); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__cudnn_ctc_loss_out(tensor *out__, tensor out0, tensor out1, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int deterministic, int zero_infinity) { - PROTECT( - auto outputs__ = torch::_cudnn_ctc_loss_out(*out0, *out1, *log_probs, *targets, torch::IntArrayRef(input_lengths_data, input_lengths_len), torch::IntArrayRef(target_lengths_data, target_lengths_len), blank, (bool)deterministic, (bool)zero_infinity); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__cudnn_ctc_loss_tensor(tensor *out__, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank, int deterministic, int zero_infinity) { - PROTECT( - auto outputs__ = torch::_cudnn_ctc_loss(*log_probs, *targets, *input_lengths, *target_lengths, blank, (bool)deterministic, (bool)zero_infinity); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__cudnn_init_dropout_state(tensor *out__, double dropout, int train, int64_t dropout_seed, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::_cudnn_init_dropout_state(dropout, (bool)train, dropout_seed, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__cudnn_init_dropout_state_out(tensor *out__, tensor out, double dropout, int train, int64_t dropout_seed) { - PROTECT( - auto outputs__ = torch::_cudnn_init_dropout_state_out(*out, dropout, (bool)train, dropout_seed); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__cudnn_rnn(tensor *out__, tensor input, tensor *weight_data, int weight_len, int64_t weight_stride0, tensor weight_buf, tensor hx, tensor cx, int64_t mode, int64_t hidden_size, int64_t proj_size, int64_t num_layers, int batch_first, double dropout, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, tensor dropout_state) { - PROTECT( - auto outputs__ = torch::_cudnn_rnn(*input, of_carray_tensor(weight_data, weight_len), weight_stride0, (weight_buf ? ::std::optional(*weight_buf) : ::std::nullopt), *hx, (cx ? ::std::optional(*cx) : ::std::nullopt), mode, hidden_size, proj_size, num_layers, (bool)batch_first, dropout, (bool)train, (bool)bidirectional, torch::IntArrayRef(batch_sizes_data, batch_sizes_len), (dropout_state ? ::std::optional(*dropout_state) : ::std::nullopt)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - out__[4] = new torch::Tensor(std::get<4>(outputs__)); - ) -} - -void atg__cudnn_rnn_flatten_weight(tensor *out__, tensor *weight_arr_data, int weight_arr_len, int64_t weight_stride0, int64_t input_size, int64_t mode, int64_t hidden_size, int64_t proj_size, int64_t num_layers, int batch_first, int bidirectional) { - PROTECT( - auto outputs__ = torch::_cudnn_rnn_flatten_weight(of_carray_tensor(weight_arr_data, weight_arr_len), weight_stride0, input_size, mode, hidden_size, proj_size, num_layers, (bool)batch_first, (bool)bidirectional); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__cudnn_rnn_flatten_weight_out(tensor *out__, tensor out, tensor *weight_arr_data, int weight_arr_len, int64_t weight_stride0, int64_t input_size, int64_t mode, int64_t hidden_size, int64_t proj_size, int64_t num_layers, int batch_first, int bidirectional) { - PROTECT( - auto outputs__ = torch::_cudnn_rnn_flatten_weight_out(*out, of_carray_tensor(weight_arr_data, weight_arr_len), weight_stride0, input_size, mode, hidden_size, proj_size, num_layers, (bool)batch_first, (bool)bidirectional); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__cudnn_rnn_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor out3, tensor out4, tensor input, tensor *weight_data, int weight_len, int64_t weight_stride0, tensor weight_buf, tensor hx, tensor cx, int64_t mode, int64_t hidden_size, int64_t proj_size, int64_t num_layers, int batch_first, double dropout, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, tensor dropout_state) { - PROTECT( - auto outputs__ = torch::_cudnn_rnn_out(*out0, *out1, *out2, *out3, *out4, *input, of_carray_tensor(weight_data, weight_len), weight_stride0, (weight_buf ? ::std::optional(*weight_buf) : ::std::nullopt), *hx, (cx ? ::std::optional(*cx) : ::std::nullopt), mode, hidden_size, proj_size, num_layers, (bool)batch_first, dropout, (bool)train, (bool)bidirectional, torch::IntArrayRef(batch_sizes_data, batch_sizes_len), (dropout_state ? ::std::optional(*dropout_state) : ::std::nullopt)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - out__[4] = new torch::Tensor(std::get<4>(outputs__)); - ) -} - -int64_t atg__debug_has_internal_overlap(tensor self) { - PROTECT( - return torch::_debug_has_internal_overlap(*self); - ) - return 0; -} - -void atg__dim_arange(tensor *out__, tensor like, int64_t dim) { - PROTECT( - auto outputs__ = torch::_dim_arange(*like, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -int64_t atg__dimi(tensor self) { - PROTECT( - return self->_dimI(); - ) - return 0; -} - -int64_t atg__dimv(tensor self) { - PROTECT( - return self->_dimV(); - ) - return 0; -} - -void atg__dirichlet_grad(tensor *out__, tensor x, tensor alpha, tensor total) { - PROTECT( - auto outputs__ = torch::_dirichlet_grad(*x, *alpha, *total); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__dirichlet_grad_out(tensor *out__, tensor out, tensor x, tensor alpha, tensor total) { - PROTECT( - auto outputs__ = torch::_dirichlet_grad_out(*out, *x, *alpha, *total); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__dyn_quant_matmul_4bit(tensor *out__, tensor inp, tensor packed_weights, int64_t block_size, int64_t in_features, int64_t out_features) { - PROTECT( - auto outputs__ = torch::_dyn_quant_matmul_4bit(*inp, *packed_weights, block_size, in_features, out_features); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__dyn_quant_pack_4bit_weight(tensor *out__, tensor weights, tensor scales_zeros, tensor bias, int64_t block_size, int64_t in_features, int64_t out_features) { - PROTECT( - auto outputs__ = torch::_dyn_quant_pack_4bit_weight(*weights, *scales_zeros, (bias ? ::std::optional(*bias) : ::std::nullopt), block_size, in_features, out_features); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__efficient_attention_backward(tensor *out__, tensor grad_out_, tensor query, tensor key, tensor value, tensor bias, tensor out, tensor cu_seqlens_q, tensor cu_seqlens_k, int64_t max_seqlen_q, int64_t max_seqlen_k, tensor logsumexp, double dropout_p, tensor philox_seed, tensor philox_offset, int64_t custom_mask_type, int bias_requires_grad, double scale_v, uint8_t scale_null, int64_t num_splits_key_v, uint8_t num_splits_key_null, int64_t window_size_v, uint8_t window_size_null, int shared_storage_dqdkdv) { - PROTECT( - auto outputs__ = torch::_efficient_attention_backward(*grad_out_, *query, *key, *value, (bias ? ::std::optional(*bias) : ::std::nullopt), *out, (cu_seqlens_q ? ::std::optional(*cu_seqlens_q) : ::std::nullopt), (cu_seqlens_k ? ::std::optional(*cu_seqlens_k) : ::std::nullopt), max_seqlen_q, max_seqlen_k, *logsumexp, dropout_p, *philox_seed, *philox_offset, custom_mask_type, (bool)bias_requires_grad, scale_null ? c10::nullopt : c10::optional(scale_v), num_splits_key_null ? c10::nullopt : c10::optional(num_splits_key_v), window_size_null ? c10::nullopt : c10::optional(window_size_v), (bool)shared_storage_dqdkdv); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg__efficientzerotensor(tensor *out__, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::_efficientzerotensor(torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__efficientzerotensor_out(tensor *out__, tensor out, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::_efficientzerotensor_out(*out, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__embedding_bag(tensor *out__, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset, int64_t padding_idx) { - PROTECT( - auto outputs__ = torch::_embedding_bag(*weight, *indices, *offsets, (bool)scale_grad_by_freq, mode, (bool)sparse, (per_sample_weights ? ::std::optional(*per_sample_weights) : ::std::nullopt), (bool)include_last_offset, padding_idx); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg__embedding_bag_backward(tensor *out__, tensor grad, tensor indices, tensor offsets, tensor offset2bag, tensor bag_size, tensor maximum_indices, int64_t num_weights, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int64_t padding_idx) { - PROTECT( - auto outputs__ = torch::_embedding_bag_backward(*grad, *indices, *offsets, *offset2bag, *bag_size, *maximum_indices, num_weights, (bool)scale_grad_by_freq, mode, (bool)sparse, (per_sample_weights ? ::std::optional(*per_sample_weights) : ::std::nullopt), padding_idx); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__embedding_bag_dense_backward(tensor *out__, tensor grad, tensor indices, tensor offset2bag, tensor bag_size, tensor maximum_indices, int64_t num_weights, int scale_grad_by_freq, int64_t mode, tensor per_sample_weights, int64_t padding_idx) { - PROTECT( - auto outputs__ = torch::_embedding_bag_dense_backward(*grad, *indices, *offset2bag, *bag_size, *maximum_indices, num_weights, (bool)scale_grad_by_freq, mode, (per_sample_weights ? ::std::optional(*per_sample_weights) : ::std::nullopt), padding_idx); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__embedding_bag_dense_backward_out(tensor *out__, tensor out, tensor grad, tensor indices, tensor offset2bag, tensor bag_size, tensor maximum_indices, int64_t num_weights, int scale_grad_by_freq, int64_t mode, tensor per_sample_weights, int64_t padding_idx) { - PROTECT( - auto outputs__ = torch::_embedding_bag_dense_backward_out(*out, *grad, *indices, *offset2bag, *bag_size, *maximum_indices, num_weights, (bool)scale_grad_by_freq, mode, (per_sample_weights ? ::std::optional(*per_sample_weights) : ::std::nullopt), padding_idx); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__embedding_bag_forward_only(tensor *out__, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset, int64_t padding_idx) { - PROTECT( - auto outputs__ = torch::_embedding_bag_forward_only(*weight, *indices, *offsets, (bool)scale_grad_by_freq, mode, (bool)sparse, (per_sample_weights ? ::std::optional(*per_sample_weights) : ::std::nullopt), (bool)include_last_offset, padding_idx); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg__embedding_bag_forward_only_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor out3, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset, int64_t padding_idx) { - PROTECT( - auto outputs__ = torch::_embedding_bag_forward_only_out(*out0, *out1, *out2, *out3, *weight, *indices, *offsets, (bool)scale_grad_by_freq, mode, (bool)sparse, (per_sample_weights ? ::std::optional(*per_sample_weights) : ::std::nullopt), (bool)include_last_offset, padding_idx); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg__embedding_bag_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor out3, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset, int64_t padding_idx) { - PROTECT( - auto outputs__ = torch::_embedding_bag_out(*out0, *out1, *out2, *out3, *weight, *indices, *offsets, (bool)scale_grad_by_freq, mode, (bool)sparse, (per_sample_weights ? ::std::optional(*per_sample_weights) : ::std::nullopt), (bool)include_last_offset, padding_idx); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg__embedding_bag_per_sample_weights_backward(tensor *out__, tensor grad, tensor weight, tensor indices, tensor offsets, tensor offset2bag, int64_t mode, int64_t padding_idx) { - PROTECT( - auto outputs__ = torch::_embedding_bag_per_sample_weights_backward(*grad, *weight, *indices, *offsets, *offset2bag, mode, padding_idx); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__embedding_bag_per_sample_weights_backward_out(tensor *out__, tensor out, tensor grad, tensor weight, tensor indices, tensor offsets, tensor offset2bag, int64_t mode, int64_t padding_idx) { - PROTECT( - auto outputs__ = torch::_embedding_bag_per_sample_weights_backward_out(*out, *grad, *weight, *indices, *offsets, *offset2bag, mode, padding_idx); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__embedding_bag_sparse_backward(tensor *out__, tensor grad, tensor indices, tensor offsets, tensor offset2bag, tensor bag_size, int64_t num_weights, int scale_grad_by_freq, int64_t mode, tensor per_sample_weights, int64_t padding_idx) { - PROTECT( - auto outputs__ = torch::_embedding_bag_sparse_backward(*grad, *indices, *offsets, *offset2bag, *bag_size, num_weights, (bool)scale_grad_by_freq, mode, (per_sample_weights ? ::std::optional(*per_sample_weights) : ::std::nullopt), padding_idx); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__empty_affine_quantized(tensor *out__, int64_t *size_data, int size_len, int options_kind, int options_device, double scale, int64_t zero_point) { - PROTECT( - auto outputs__ = torch::_empty_affine_quantized(torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind)), scale, zero_point); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__empty_affine_quantized_out(tensor *out__, tensor out, int64_t *size_data, int size_len, double scale, int64_t zero_point) { - PROTECT( - auto outputs__ = torch::_empty_affine_quantized_out(*out, torch::IntArrayRef(size_data, size_len), scale, zero_point); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__empty_per_channel_affine_quantized(tensor *out__, int64_t *size_data, int size_len, tensor scales, tensor zero_points, int64_t axis, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::_empty_per_channel_affine_quantized(torch::IntArrayRef(size_data, size_len), *scales, *zero_points, axis, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__empty_per_channel_affine_quantized_out(tensor *out__, tensor out, int64_t *size_data, int size_len, tensor scales, tensor zero_points, int64_t axis) { - PROTECT( - auto outputs__ = torch::_empty_per_channel_affine_quantized_out(*out, torch::IntArrayRef(size_data, size_len), *scales, *zero_points, axis); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__euclidean_dist(tensor *out__, tensor x1, tensor x2) { - PROTECT( - auto outputs__ = torch::_euclidean_dist(*x1, *x2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__euclidean_dist_out(tensor *out__, tensor out, tensor x1, tensor x2) { - PROTECT( - auto outputs__ = torch::_euclidean_dist_out(*out, *x1, *x2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__fake_quantize_learnable_per_channel_affine(tensor *out__, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max, double grad_factor) { - PROTECT( - auto outputs__ = torch::_fake_quantize_learnable_per_channel_affine(*self, *scale, *zero_point, axis, quant_min, quant_max, grad_factor); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__fake_quantize_learnable_per_channel_affine_backward(tensor *out__, tensor grad, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max, double grad_factor) { - PROTECT( - auto outputs__ = torch::_fake_quantize_learnable_per_channel_affine_backward(*grad, *self, *scale, *zero_point, axis, quant_min, quant_max, grad_factor); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__fake_quantize_learnable_per_channel_affine_out(tensor *out__, tensor out, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max, double grad_factor) { - PROTECT( - auto outputs__ = torch::_fake_quantize_learnable_per_channel_affine_out(*out, *self, *scale, *zero_point, axis, quant_min, quant_max, grad_factor); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__fake_quantize_learnable_per_tensor_affine(tensor *out__, tensor self, tensor scale, tensor zero_point, int64_t quant_min, int64_t quant_max, double grad_factor) { - PROTECT( - auto outputs__ = torch::_fake_quantize_learnable_per_tensor_affine(*self, *scale, *zero_point, quant_min, quant_max, grad_factor); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__fake_quantize_learnable_per_tensor_affine_backward(tensor *out__, tensor grad, tensor self, tensor scale, tensor zero_point, int64_t quant_min, int64_t quant_max, double grad_factor) { - PROTECT( - auto outputs__ = torch::_fake_quantize_learnable_per_tensor_affine_backward(*grad, *self, *scale, *zero_point, quant_min, quant_max, grad_factor); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__fake_quantize_learnable_per_tensor_affine_out(tensor *out__, tensor out, tensor self, tensor scale, tensor zero_point, int64_t quant_min, int64_t quant_max, double grad_factor) { - PROTECT( - auto outputs__ = torch::_fake_quantize_learnable_per_tensor_affine_out(*out, *self, *scale, *zero_point, quant_min, quant_max, grad_factor); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__fake_quantize_per_tensor_affine_cachemask_tensor_qparams(tensor *out__, tensor self, tensor scale, tensor zero_point, tensor fake_quant_enabled, int64_t quant_min, int64_t quant_max) { - PROTECT( - auto outputs__ = torch::_fake_quantize_per_tensor_affine_cachemask_tensor_qparams(*self, *scale, *zero_point, *fake_quant_enabled, quant_min, quant_max); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__fake_quantize_per_tensor_affine_cachemask_tensor_qparams_out(tensor *out__, tensor out0, tensor out1, tensor self, tensor scale, tensor zero_point, tensor fake_quant_enabled, int64_t quant_min, int64_t quant_max) { - PROTECT( - auto outputs__ = torch::_fake_quantize_per_tensor_affine_cachemask_tensor_qparams_out(*out0, *out1, *self, *scale, *zero_point, *fake_quant_enabled, quant_min, quant_max); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__fft_c2c(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int forward) { - PROTECT( - auto outputs__ = torch::_fft_c2c(*self, torch::IntArrayRef(dim_data, dim_len), normalization, (bool)forward); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__fft_c2c_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int forward) { - PROTECT( - auto outputs__ = torch::_fft_c2c_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), normalization, (bool)forward); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__fft_c2r(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int64_t last_dim_size) { - PROTECT( - auto outputs__ = torch::_fft_c2r(*self, torch::IntArrayRef(dim_data, dim_len), normalization, last_dim_size); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__fft_c2r_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int64_t last_dim_size) { - PROTECT( - auto outputs__ = torch::_fft_c2r_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), normalization, last_dim_size); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__fft_r2c(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int onesided) { - PROTECT( - auto outputs__ = torch::_fft_r2c(*self, torch::IntArrayRef(dim_data, dim_len), normalization, (bool)onesided); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__fft_r2c_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int onesided) { - PROTECT( - auto outputs__ = torch::_fft_r2c_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), normalization, (bool)onesided); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__fill_mem_eff_dropout_mask_(tensor *out__, tensor self, double dropout_p, int64_t seed, int64_t offset) { - PROTECT( - auto outputs__ = torch::_fill_mem_eff_dropout_mask_(*self, dropout_p, seed, offset); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__flash_attention_backward(tensor *out__, tensor grad_out, tensor query, tensor key, tensor value, tensor out, tensor logsumexp, tensor cum_seq_q, tensor cum_seq_k, int64_t max_q, int64_t max_k, double dropout_p, int is_causal, tensor rng_state, tensor unused, double scale_v, uint8_t scale_null, int64_t window_size_left_v, uint8_t window_size_left_null, int64_t window_size_right_v, uint8_t window_size_right_null) { - PROTECT( - auto outputs__ = torch::_flash_attention_backward(*grad_out, *query, *key, *value, *out, *logsumexp, *cum_seq_q, *cum_seq_k, max_q, max_k, dropout_p, (bool)is_causal, *rng_state, *unused, scale_null ? c10::nullopt : c10::optional(scale_v), window_size_left_null ? c10::nullopt : c10::optional(window_size_left_v), window_size_right_null ? c10::nullopt : c10::optional(window_size_right_v)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__foobar(tensor *out__, tensor self, int arg1, int arg2, int arg3) { - PROTECT( - auto outputs__ = torch::_foobar(*self, (bool)arg1, (bool)arg2, (bool)arg3); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__foobar_out(tensor *out__, tensor out, tensor self, int arg1, int arg2, int arg3) { - PROTECT( - auto outputs__ = torch::_foobar_out(*out, *self, (bool)arg1, (bool)arg2, (bool)arg3); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__functional_assert_async(tensor *out__, tensor self, char* assert_msg_ptr, int assert_msg_len, tensor dep_token) { - PROTECT( - auto outputs__ = torch::_functional_assert_async(*self, std::string(assert_msg_ptr, assert_msg_len), *dep_token); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__functional_assert_scalar(tensor *out__, scalar self_scalar, char* assert_msg_ptr, int assert_msg_len, tensor dep_token) { - PROTECT( - auto outputs__ = torch::_functional_assert_scalar(*self_scalar, std::string(assert_msg_ptr, assert_msg_len), *dep_token); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__functional_sym_constrain_range(tensor *out__, scalar size, int64_t min_v, uint8_t min_null, int64_t max_v, uint8_t max_null, tensor dep_token) { - PROTECT( - auto outputs__ = torch::_functional_sym_constrain_range(*size, min_null ? c10::nullopt : c10::optional(min_v), max_null ? c10::nullopt : c10::optional(max_v), *dep_token); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__functional_sym_constrain_range_for_size(tensor *out__, scalar size, int64_t min_v, uint8_t min_null, int64_t max_v, uint8_t max_null, tensor dep_token) { - PROTECT( - auto outputs__ = torch::_functional_sym_constrain_range_for_size(*size, min_null ? c10::nullopt : c10::optional(min_v), max_null ? c10::nullopt : c10::optional(max_v), *dep_token); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__fused_dropout(tensor *out__, tensor self, double p) { - PROTECT( - auto outputs__ = torch::_fused_dropout(*self, p); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__fused_dropout_out(tensor *out__, tensor out0, tensor out1, tensor self, double p) { - PROTECT( - auto outputs__ = torch::_fused_dropout_out(*out0, *out1, *self, p); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__fused_moving_avg_obs_fq_helper(tensor *out__, tensor self, tensor observer_on, tensor fake_quant_on, tensor running_min, tensor running_max, tensor scale, tensor zero_point, double averaging_const, int64_t quant_min, int64_t quant_max, int64_t ch_axis, int per_row_fake_quant, int symmetric_quant) { - PROTECT( - auto outputs__ = torch::_fused_moving_avg_obs_fq_helper(*self, *observer_on, *fake_quant_on, *running_min, *running_max, *scale, *zero_point, averaging_const, quant_min, quant_max, ch_axis, (bool)per_row_fake_quant, (bool)symmetric_quant); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__fused_moving_avg_obs_fq_helper_functional(tensor *out__, tensor self, tensor observer_on, tensor fake_quant_on, tensor running_min, tensor running_max, tensor scale, tensor zero_point, double averaging_const, int64_t quant_min, int64_t quant_max, int64_t ch_axis, int per_row_fake_quant, int symmetric_quant) { - PROTECT( - auto outputs__ = torch::_fused_moving_avg_obs_fq_helper_functional(*self, *observer_on, *fake_quant_on, *running_min, *running_max, *scale, *zero_point, averaging_const, quant_min, quant_max, ch_axis, (bool)per_row_fake_quant, (bool)symmetric_quant); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - out__[4] = new torch::Tensor(std::get<4>(outputs__)); - out__[5] = new torch::Tensor(std::get<5>(outputs__)); - ) -} - -void atg__fused_moving_avg_obs_fq_helper_out(tensor *out__, tensor out0, tensor out1, tensor self, tensor observer_on, tensor fake_quant_on, tensor running_min, tensor running_max, tensor scale, tensor zero_point, double averaging_const, int64_t quant_min, int64_t quant_max, int64_t ch_axis, int per_row_fake_quant, int symmetric_quant) { - PROTECT( - auto outputs__ = torch::_fused_moving_avg_obs_fq_helper_out(*out0, *out1, *self, *observer_on, *fake_quant_on, *running_min, *running_max, *scale, *zero_point, averaging_const, quant_min, quant_max, ch_axis, (bool)per_row_fake_quant, (bool)symmetric_quant); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__fused_rms_norm(tensor *out__, tensor input, int64_t *normalized_shape_data, int normalized_shape_len, tensor weight, double eps_v, uint8_t eps_null) { - PROTECT( - auto outputs__ = torch::_fused_rms_norm(*input, torch::IntArrayRef(normalized_shape_data, normalized_shape_len), (weight ? ::std::optional(*weight) : ::std::nullopt), eps_null ? c10::nullopt : c10::optional(eps_v)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -int64_t atg__fused_sdp_choice(tensor query, tensor key, tensor value, tensor attn_mask, double dropout_p, int is_causal, double scale_v, uint8_t scale_null, int enable_gqa) { - PROTECT( - return torch::_fused_sdp_choice(*query, *key, *value, (attn_mask ? ::std::optional(*attn_mask) : ::std::nullopt), dropout_p, (bool)is_causal, scale_null ? c10::nullopt : c10::optional(scale_v), (bool)enable_gqa); - ) - return 0; -} - -void atg__fw_primal(tensor *out__, tensor self, int64_t level) { - PROTECT( - auto outputs__ = self->_fw_primal(level); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__fw_primal_copy(tensor *out__, tensor self, int64_t level) { - PROTECT( - auto outputs__ = torch::_fw_primal_copy(*self, level); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__fw_primal_copy_out(tensor *out__, tensor out, tensor self, int64_t level) { - PROTECT( - auto outputs__ = torch::_fw_primal_copy_out(*out, *self, level); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__gather_sparse_backward(tensor *out__, tensor self, int64_t dim, tensor index, tensor grad) { - PROTECT( - auto outputs__ = torch::_gather_sparse_backward(*self, dim, *index, *grad); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__grid_sampler_2d_cpu_fallback(tensor *out__, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners) { - PROTECT( - auto outputs__ = torch::_grid_sampler_2d_cpu_fallback(*input, *grid, interpolation_mode, padding_mode, (bool)align_corners); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__grid_sampler_2d_cpu_fallback_backward(tensor *out__, tensor grad_output, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners) { - PROTECT( - auto outputs__ = torch::_grid_sampler_2d_cpu_fallback_backward(*grad_output, *input, *grid, interpolation_mode, padding_mode, (bool)align_corners); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__grid_sampler_2d_cpu_fallback_out(tensor *out__, tensor out, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners) { - PROTECT( - auto outputs__ = torch::_grid_sampler_2d_cpu_fallback_out(*out, *input, *grid, interpolation_mode, padding_mode, (bool)align_corners); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__grouped_mm(tensor *out__, tensor self, tensor mat2, tensor offs, tensor bias, int out_dtype) { - PROTECT( - auto outputs__ = torch::_grouped_mm(*self, *mat2, (offs ? ::std::optional(*offs) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), out_dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(out_dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -int atg__has_compatible_shallow_copy_type(tensor self, tensor from) { - PROTECT( - return torch::_has_compatible_shallow_copy_type(*self, *from); - ) - return 0; -} - -int atg__has_same_storage_numel(tensor self, tensor other) { - PROTECT( - return torch::_has_same_storage_numel(*self, *other); - ) - return 0; -} - -tensor *atg__histogramdd_bin_edges(tensor self, int64_t *bins_data, int bins_len, double *range_data, int range_len, tensor weight, int density) { - PROTECT( - auto outputs__ = torch::_histogramdd_bin_edges(*self, torch::IntArrayRef(bins_data, bins_len), at::ArrayRef(range_data, range_len), (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)density); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg__histogramdd_bin_edges_out(tensor *out_data, int out_len, tensor self, int64_t *bins_data, int bins_len, double *range_data, int range_len, tensor weight, int density) { - PROTECT( - torch::_histogramdd_bin_edges_out(of_carray_tensor(out_data, out_len), *self, torch::IntArrayRef(bins_data, bins_len), at::ArrayRef(range_data, range_len), (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)density); - ) -} - -void atg__histogramdd_from_bin_cts(tensor *out__, tensor self, int64_t *bins_data, int bins_len, double *range_data, int range_len, tensor weight, int density) { - PROTECT( - auto outputs__ = torch::_histogramdd_from_bin_cts(*self, torch::IntArrayRef(bins_data, bins_len), at::ArrayRef(range_data, range_len), (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)density); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__histogramdd_from_bin_cts_out(tensor *out__, tensor out, tensor self, int64_t *bins_data, int bins_len, double *range_data, int range_len, tensor weight, int density) { - PROTECT( - auto outputs__ = torch::_histogramdd_from_bin_cts_out(*out, *self, torch::IntArrayRef(bins_data, bins_len), at::ArrayRef(range_data, range_len), (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)density); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__histogramdd_from_bin_tensors(tensor *out__, tensor self, tensor *bins_data, int bins_len, tensor weight, int density) { - PROTECT( - auto outputs__ = torch::_histogramdd_from_bin_tensors(*self, of_carray_tensor(bins_data, bins_len), (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)density); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__histogramdd_from_bin_tensors_out(tensor *out__, tensor out, tensor self, tensor *bins_data, int bins_len, tensor weight, int density) { - PROTECT( - auto outputs__ = torch::_histogramdd_from_bin_tensors_out(*out, *self, of_carray_tensor(bins_data, bins_len), (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)density); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__index_put_impl(tensor *out__, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate, int unsafe) { - PROTECT( - auto outputs__ = torch::_index_put_impl(*self, of_carray_tensor_opt(indices_data, indices_len), *values, (bool)accumulate, (bool)unsafe); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__index_put_impl_(tensor *out__, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate, int unsafe) { - PROTECT( - auto outputs__ = torch::_index_put_impl_(*self, of_carray_tensor_opt(indices_data, indices_len), *values, (bool)accumulate, (bool)unsafe); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__index_put_impl_out(tensor *out__, tensor out, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate, int unsafe) { - PROTECT( - auto outputs__ = torch::_index_put_impl_out(*out, *self, of_carray_tensor_opt(indices_data, indices_len), *values, (bool)accumulate, (bool)unsafe); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__indices(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->_indices(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__indices_copy(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_indices_copy(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__indices_copy_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::_indices_copy_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__int_mm(tensor *out__, tensor self, tensor mat2) { - PROTECT( - auto outputs__ = torch::_int_mm(*self, *mat2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__int_mm_out(tensor *out__, tensor out, tensor self, tensor mat2) { - PROTECT( - auto outputs__ = torch::_int_mm_out(*out, *self, *mat2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__is_all_true(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_is_all_true(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__is_any_true(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_is_any_true(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -int atg__is_zerotensor(tensor self) { - PROTECT( - return torch::_is_zerotensor(*self); - ) - return 0; -} - -void atg__lazy_clone(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_lazy_clone(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__linalg_check_errors(tensor info, char* api_name_ptr, int api_name_len, int is_matrix) { - PROTECT( - torch::_linalg_check_errors(*info, std::string(api_name_ptr, api_name_len), (bool)is_matrix); - ) -} - -void atg__linalg_det(tensor *out__, tensor A) { - PROTECT( - auto outputs__ = torch::_linalg_det(*A); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__linalg_det_result(tensor *out__, tensor result, tensor LU, tensor pivots, tensor A) { - PROTECT( - auto outputs__ = torch::_linalg_det_out(*result, *LU, *pivots, *A); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__linalg_eigh(tensor *out__, tensor A, char* UPLO_ptr, int UPLO_len, int compute_v) { - PROTECT( - auto outputs__ = torch::_linalg_eigh(*A, std::string(UPLO_ptr, UPLO_len), (bool)compute_v); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__linalg_eigh_eigenvalues(tensor *out__, tensor eigenvalues, tensor eigenvectors, tensor A, char* UPLO_ptr, int UPLO_len, int compute_v) { - PROTECT( - auto outputs__ = torch::_linalg_eigh_out(*eigenvalues, *eigenvectors, *A, std::string(UPLO_ptr, UPLO_len), (bool)compute_v); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__linalg_eigvals(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_linalg_eigvals(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__linalg_slogdet(tensor *out__, tensor A) { - PROTECT( - auto outputs__ = torch::_linalg_slogdet(*A); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg__linalg_slogdet_sign(tensor *out__, tensor sign, tensor logabsdet, tensor LU, tensor pivots, tensor A) { - PROTECT( - auto outputs__ = torch::_linalg_slogdet_out(*sign, *logabsdet, *LU, *pivots, *A); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg__linalg_solve_ex(tensor *out__, tensor A, tensor B, int left, int check_errors) { - PROTECT( - auto outputs__ = torch::_linalg_solve_ex(*A, *B, (bool)left, (bool)check_errors); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg__linalg_solve_ex_result(tensor *out__, tensor result, tensor LU, tensor pivots, tensor info, tensor A, tensor B, int left, int check_errors) { - PROTECT( - auto outputs__ = torch::_linalg_solve_ex_out(*result, *LU, *pivots, *info, *A, *B, (bool)left, (bool)check_errors); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg__linalg_svd(tensor *out__, tensor A, int full_matrices, int compute_uv, char* driver_ptr, int driver_len) { - PROTECT( - auto outputs__ = torch::_linalg_svd(*A, (bool)full_matrices, (bool)compute_uv, std::string(driver_ptr, driver_len)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__linalg_svd_u(tensor *out__, tensor U, tensor S, tensor Vh, tensor A, int full_matrices, int compute_uv, char* driver_ptr, int driver_len) { - PROTECT( - auto outputs__ = torch::_linalg_svd_out(*U, *S, *Vh, *A, (bool)full_matrices, (bool)compute_uv, std::string(driver_ptr, driver_len)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__log_softmax(tensor *out__, tensor self, int64_t dim, int half_to_float) { - PROTECT( - auto outputs__ = torch::_log_softmax(*self, dim, (bool)half_to_float); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__log_softmax_backward_data(tensor *out__, tensor grad_output, tensor output, int64_t dim, int input_dtype) { - PROTECT( - auto outputs__ = torch::_log_softmax_backward_data(*grad_output, *output, dim, at::ScalarType(input_dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__log_softmax_backward_data_out(tensor *out__, tensor out, tensor grad_output, tensor output, int64_t dim, int input_dtype) { - PROTECT( - auto outputs__ = torch::_log_softmax_backward_data_out(*out, *grad_output, *output, dim, at::ScalarType(input_dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__log_softmax_out(tensor *out__, tensor out, tensor self, int64_t dim, int half_to_float) { - PROTECT( - auto outputs__ = torch::_log_softmax_out(*out, *self, dim, (bool)half_to_float); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__logcumsumexp(tensor *out__, tensor self, int64_t dim) { - PROTECT( - auto outputs__ = torch::_logcumsumexp(*self, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__logcumsumexp_out(tensor *out__, tensor out, tensor self, int64_t dim) { - PROTECT( - auto outputs__ = torch::_logcumsumexp_out(*out, *self, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__lstm_mps(tensor *out__, tensor input, tensor *hx_data, int hx_len, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first) { - PROTECT( - auto outputs__ = torch::_lstm_mps(*input, of_carray_tensor(hx_data, hx_len), of_carray_tensor(params_data, params_len), (bool)has_biases, num_layers, dropout, (bool)train, (bool)bidirectional, (bool)batch_first); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - out__[4] = new torch::Tensor(std::get<4>(outputs__)); - out__[5] = new torch::Tensor(std::get<5>(outputs__)); - ) -} - -void atg__lstm_mps_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor out3, tensor out4, tensor out5, tensor input, tensor *hx_data, int hx_len, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first) { - PROTECT( - auto outputs__ = torch::_lstm_mps_out(*out0, *out1, *out2, *out3, *out4, *out5, *input, of_carray_tensor(hx_data, hx_len), of_carray_tensor(params_data, params_len), (bool)has_biases, num_layers, dropout, (bool)train, (bool)bidirectional, (bool)batch_first); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - out__[4] = new torch::Tensor(std::get<4>(outputs__)); - out__[5] = new torch::Tensor(std::get<5>(outputs__)); - ) -} - -void atg__lu_with_info(tensor *out__, tensor self, int pivot, int check_errors) { - PROTECT( - auto outputs__ = torch::_lu_with_info(*self, (bool)pivot, (bool)check_errors); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__make_dep_token(tensor *out__, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::_make_dep_token(at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__make_dual(tensor *out__, tensor primal, tensor tangent, int64_t level) { - PROTECT( - auto outputs__ = torch::_make_dual(*primal, *tangent, level); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__make_dual_copy(tensor *out__, tensor primal, tensor tangent, int64_t level) { - PROTECT( - auto outputs__ = torch::_make_dual_copy(*primal, *tangent, level); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__make_dual_copy_out(tensor *out__, tensor out, tensor primal, tensor tangent, int64_t level) { - PROTECT( - auto outputs__ = torch::_make_dual_copy_out(*out, *primal, *tangent, level); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__make_per_channel_quantized_tensor(tensor *out__, tensor self, tensor scale, tensor zero_point, int64_t axis) { - PROTECT( - auto outputs__ = torch::_make_per_channel_quantized_tensor(*self, *scale, *zero_point, axis); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__make_per_channel_quantized_tensor_out(tensor *out__, tensor out, tensor self, tensor scale, tensor zero_point, int64_t axis) { - PROTECT( - auto outputs__ = torch::_make_per_channel_quantized_tensor_out(*out, *self, *scale, *zero_point, axis); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__make_per_tensor_quantized_tensor(tensor *out__, tensor self, double scale, int64_t zero_point) { - PROTECT( - auto outputs__ = torch::_make_per_tensor_quantized_tensor(*self, scale, zero_point); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__make_per_tensor_quantized_tensor_out(tensor *out__, tensor out, tensor self, double scale, int64_t zero_point) { - PROTECT( - auto outputs__ = torch::_make_per_tensor_quantized_tensor_out(*out, *self, scale, zero_point); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__masked_scale(tensor *out__, tensor self, tensor mask, double scale) { - PROTECT( - auto outputs__ = torch::_masked_scale(*self, *mask, scale); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__masked_scale_out(tensor *out__, tensor out, tensor self, tensor mask, double scale) { - PROTECT( - auto outputs__ = torch::_masked_scale_out(*out, *self, *mask, scale); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__masked_softmax(tensor *out__, tensor self, tensor mask, int64_t dim_v, uint8_t dim_null, int64_t mask_type_v, uint8_t mask_type_null) { - PROTECT( - auto outputs__ = torch::_masked_softmax(*self, *mask, dim_null ? c10::nullopt : c10::optional(dim_v), mask_type_null ? c10::nullopt : c10::optional(mask_type_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__masked_softmax_backward(tensor *out__, tensor grad_output, tensor output, tensor mask, int64_t dim_v, uint8_t dim_null) { - PROTECT( - auto outputs__ = torch::_masked_softmax_backward(*grad_output, *output, *mask, dim_null ? c10::nullopt : c10::optional(dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__masked_softmax_backward_out(tensor *out__, tensor out, tensor grad_output, tensor output, tensor mask, int64_t dim_v, uint8_t dim_null) { - PROTECT( - auto outputs__ = torch::_masked_softmax_backward_out(*out, *grad_output, *output, *mask, dim_null ? c10::nullopt : c10::optional(dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__masked_softmax_out(tensor *out__, tensor out, tensor self, tensor mask, int64_t dim_v, uint8_t dim_null, int64_t mask_type_v, uint8_t mask_type_null) { - PROTECT( - auto outputs__ = torch::_masked_softmax_out(*out, *self, *mask, dim_null ? c10::nullopt : c10::optional(dim_v), mask_type_null ? c10::nullopt : c10::optional(mask_type_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__mixed_dtypes_linear(tensor *out__, tensor input, tensor weight, tensor scale, tensor bias, char* activation_ptr, int activation_len) { - PROTECT( - auto outputs__ = torch::_mixed_dtypes_linear(*input, *weight, *scale, (bias ? ::std::optional(*bias) : ::std::nullopt), std::string(activation_ptr, activation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__mkldnn_reshape(tensor *out__, tensor self, int64_t *shape_data, int shape_len) { - PROTECT( - auto outputs__ = torch::_mkldnn_reshape(*self, torch::IntArrayRef(shape_data, shape_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__mkldnn_reshape_out(tensor *out__, tensor out, tensor self, int64_t *shape_data, int shape_len) { - PROTECT( - auto outputs__ = torch::_mkldnn_reshape_out(*out, *self, torch::IntArrayRef(shape_data, shape_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__mkldnn_transpose(tensor *out__, tensor self, int64_t dim0, int64_t dim1) { - PROTECT( - auto outputs__ = torch::_mkldnn_transpose(*self, dim0, dim1); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__mkldnn_transpose_(tensor *out__, tensor self, int64_t dim0, int64_t dim1) { - PROTECT( - auto outputs__ = torch::_mkldnn_transpose_(*self, dim0, dim1); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__mkldnn_transpose_out(tensor *out__, tensor out, tensor self, int64_t dim0, int64_t dim1) { - PROTECT( - auto outputs__ = torch::_mkldnn_transpose_out(*out, *self, dim0, dim1); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__mps_convolution(tensor *out__, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::_mps_convolution(*self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__mps_convolution_out(tensor *out__, tensor out, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::_mps_convolution_out(*out, *self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__mps_convolution_transpose(tensor *out__, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::_mps_convolution_transpose(*self, *weight, torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__mps_convolution_transpose_out(tensor *out__, tensor out, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::_mps_convolution_transpose_out(*out, *self, *weight, torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__native_batch_norm_legit(tensor *out__, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps) { - PROTECT( - auto outputs__ = torch::_native_batch_norm_legit(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *running_mean, *running_var, (bool)training, momentum, eps); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__native_batch_norm_legit_functional(tensor *out__, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps) { - PROTECT( - auto outputs__ = torch::_native_batch_norm_legit_functional(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *running_mean, *running_var, (bool)training, momentum, eps); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - out__[4] = new torch::Tensor(std::get<4>(outputs__)); - ) -} - -void atg__native_batch_norm_legit_no_stats(tensor *out__, tensor input, tensor weight, tensor bias, int training, double momentum, double eps) { - PROTECT( - auto outputs__ = torch::_native_batch_norm_legit(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), (bool)training, momentum, eps); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__native_batch_norm_legit_no_stats_out(tensor *out__, tensor out, tensor save_mean, tensor save_invstd, tensor input, tensor weight, tensor bias, int training, double momentum, double eps) { - PROTECT( - auto outputs__ = torch::_native_batch_norm_legit_out(*out, *save_mean, *save_invstd, *input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), (bool)training, momentum, eps); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__native_batch_norm_legit_no_training(tensor *out__, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps) { - PROTECT( - auto outputs__ = torch::_native_batch_norm_legit_no_training(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *running_mean, *running_var, momentum, eps); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__native_batch_norm_legit_no_training_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps) { - PROTECT( - auto outputs__ = torch::_native_batch_norm_legit_no_training_out(*out0, *out1, *out2, *input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *running_mean, *running_var, momentum, eps); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__native_batch_norm_legit_out(tensor *out__, tensor out, tensor save_mean, tensor save_invstd, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps) { - PROTECT( - auto outputs__ = torch::_native_batch_norm_legit_out(*out, *save_mean, *save_invstd, *input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *running_mean, *running_var, (bool)training, momentum, eps); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__native_multi_head_attention(tensor *out__, tensor query, tensor key, tensor value, int64_t embed_dim, int64_t num_head, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, tensor mask, int need_weights, int average_attn_weights, int64_t mask_type_v, uint8_t mask_type_null) { - PROTECT( - auto outputs__ = torch::_native_multi_head_attention(*query, *key, *value, embed_dim, num_head, *qkv_weight, *qkv_bias, *proj_weight, *proj_bias, (mask ? ::std::optional(*mask) : ::std::nullopt), (bool)need_weights, (bool)average_attn_weights, mask_type_null ? c10::nullopt : c10::optional(mask_type_v)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__native_multi_head_attention_out(tensor *out__, tensor out0, tensor out1, tensor query, tensor key, tensor value, int64_t embed_dim, int64_t num_head, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, tensor mask, int need_weights, int average_attn_weights, int64_t mask_type_v, uint8_t mask_type_null) { - PROTECT( - auto outputs__ = torch::_native_multi_head_attention_out(*out0, *out1, *query, *key, *value, embed_dim, num_head, *qkv_weight, *qkv_bias, *proj_weight, *proj_bias, (mask ? ::std::optional(*mask) : ::std::nullopt), (bool)need_weights, (bool)average_attn_weights, mask_type_null ? c10::nullopt : c10::optional(mask_type_v)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__neg_view(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_neg_view(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__neg_view_copy(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_neg_view_copy(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__neg_view_copy_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::_neg_view_copy_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__nested_compute_contiguous_strides_offsets(tensor *out__, tensor nested_size) { - PROTECT( - auto outputs__ = torch::_nested_compute_contiguous_strides_offsets(*nested_size); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__nested_from_padded(tensor *out__, tensor padded, tensor cpu_nested_shape_example, int fuse_transform_0213) { - PROTECT( - auto outputs__ = torch::_nested_from_padded(*padded, *cpu_nested_shape_example, (bool)fuse_transform_0213); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__nested_from_padded_and_nested_example(tensor *out__, tensor padded, tensor nt_example) { - PROTECT( - auto outputs__ = torch::_nested_from_padded_and_nested_example(*padded, *nt_example); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__nested_from_padded_and_nested_example_out(tensor *out__, tensor out, tensor padded, tensor nt_example) { - PROTECT( - auto outputs__ = torch::_nested_from_padded_and_nested_example_out(*out, *padded, *nt_example); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__nested_from_padded_out(tensor *out__, tensor out, tensor padded, tensor cpu_nested_shape_example, int fuse_transform_0213) { - PROTECT( - auto outputs__ = torch::_nested_from_padded_out(*out, *padded, *cpu_nested_shape_example, (bool)fuse_transform_0213); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__nested_from_padded_tensor(tensor *out__, tensor padded, tensor offsets, tensor dummy, int64_t ragged_idx, tensor min_seqlen, tensor max_seqlen, int64_t sum_S_v, uint8_t sum_S_null) { - PROTECT( - auto outputs__ = torch::_nested_from_padded_tensor(*padded, *offsets, *dummy, ragged_idx, (min_seqlen ? ::std::optional(*min_seqlen) : ::std::nullopt), (max_seqlen ? ::std::optional(*max_seqlen) : ::std::nullopt), sum_S_null ? c10::nullopt : c10::optional(sum_S_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__nested_get_jagged_dummy(tensor *out__, tensor any) { - PROTECT( - auto outputs__ = torch::_nested_get_jagged_dummy(*any); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__nested_get_lengths(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_nested_get_lengths(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__nested_get_max_seqlen(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_nested_get_max_seqlen(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__nested_get_min_seqlen(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_nested_get_min_seqlen(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__nested_get_offsets(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_nested_get_offsets(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -int64_t atg__nested_get_ragged_idx(tensor self) { - PROTECT( - return torch::_nested_get_ragged_idx(*self); - ) - return 0; -} - -void atg__nested_get_values(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_nested_get_values(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__nested_get_values_copy(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_nested_get_values_copy(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__nested_get_values_copy_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::_nested_get_values_copy_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__nested_select_backward(tensor *out__, tensor grad_output, tensor self, int64_t dim, int64_t index) { - PROTECT( - auto outputs__ = torch::_nested_select_backward(*grad_output, *self, dim, index); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__nested_sum_backward(tensor *out__, tensor grad, tensor self, int64_t *dim_data, int dim_len, int keepdim) { - PROTECT( - auto outputs__ = torch::_nested_sum_backward(*grad, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__nested_view_from_buffer(tensor *out__, tensor self, tensor nested_size, tensor nested_strides, tensor offsets) { - PROTECT( - auto outputs__ = torch::_nested_view_from_buffer(*self, *nested_size, *nested_strides, *offsets); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__nested_view_from_buffer_copy(tensor *out__, tensor self, tensor nested_size, tensor nested_strides, tensor offsets) { - PROTECT( - auto outputs__ = torch::_nested_view_from_buffer_copy(*self, *nested_size, *nested_strides, *offsets); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__nested_view_from_buffer_copy_out(tensor *out__, tensor out, tensor self, tensor nested_size, tensor nested_strides, tensor offsets) { - PROTECT( - auto outputs__ = torch::_nested_view_from_buffer_copy_out(*out, *self, *nested_size, *nested_strides, *offsets); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__nested_view_from_jagged(tensor *out__, tensor self, tensor offsets, tensor dummy, tensor lengths, int64_t ragged_idx, tensor min_seqlen, tensor max_seqlen) { - PROTECT( - auto outputs__ = torch::_nested_view_from_jagged(*self, *offsets, *dummy, (lengths ? ::std::optional(*lengths) : ::std::nullopt), ragged_idx, (min_seqlen ? ::std::optional(*min_seqlen) : ::std::nullopt), (max_seqlen ? ::std::optional(*max_seqlen) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__nested_view_from_jagged_copy(tensor *out__, tensor self, tensor offsets, tensor dummy, tensor lengths, int64_t ragged_idx, tensor min_seqlen, tensor max_seqlen) { - PROTECT( - auto outputs__ = torch::_nested_view_from_jagged_copy(*self, *offsets, *dummy, (lengths ? ::std::optional(*lengths) : ::std::nullopt), ragged_idx, (min_seqlen ? ::std::optional(*min_seqlen) : ::std::nullopt), (max_seqlen ? ::std::optional(*max_seqlen) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__nested_view_from_jagged_copy_out(tensor *out__, tensor out, tensor self, tensor offsets, tensor dummy, tensor lengths, int64_t ragged_idx, tensor min_seqlen, tensor max_seqlen) { - PROTECT( - auto outputs__ = torch::_nested_view_from_jagged_copy_out(*out, *self, *offsets, *dummy, (lengths ? ::std::optional(*lengths) : ::std::nullopt), ragged_idx, (min_seqlen ? ::std::optional(*min_seqlen) : ::std::nullopt), (max_seqlen ? ::std::optional(*max_seqlen) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__new_zeros_with_same_feature_meta(tensor *out__, tensor self, tensor other, int64_t self_num_batch_dims) { - PROTECT( - auto outputs__ = torch::_new_zeros_with_same_feature_meta(*self, *other, self_num_batch_dims); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__new_zeros_with_same_feature_meta_out(tensor *out__, tensor out, tensor self, tensor other, int64_t self_num_batch_dims) { - PROTECT( - auto outputs__ = torch::_new_zeros_with_same_feature_meta_out(*out, *self, *other, self_num_batch_dims); - out__[0] = new torch::Tensor(outputs__); - ) -} - -int atg__nnpack_available() { - PROTECT( - return torch::_nnpack_available(); - ) - return 0; -} - -void atg__nnpack_spatial_convolution(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len) { - PROTECT( - auto outputs__ = torch::_nnpack_spatial_convolution(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__nnpack_spatial_convolution_out(tensor *out__, tensor out, tensor input, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len) { - PROTECT( - auto outputs__ = torch::_nnpack_spatial_convolution_out(*out, *input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -int64_t atg__nnz(tensor self) { - PROTECT( - return self->_nnz(); - ) - return 0; -} - -void atg__pack_padded_sequence(tensor *out__, tensor input, tensor lengths, int batch_first) { - PROTECT( - auto outputs__ = torch::_pack_padded_sequence(*input, *lengths, (bool)batch_first); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__pack_padded_sequence_backward(tensor *out__, tensor grad, int64_t *input_size_data, int input_size_len, tensor batch_sizes, int batch_first) { - PROTECT( - auto outputs__ = torch::_pack_padded_sequence_backward(*grad, torch::IntArrayRef(input_size_data, input_size_len), *batch_sizes, (bool)batch_first); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__pack_padded_sequence_out(tensor *out__, tensor out0, tensor out1, tensor input, tensor lengths, int batch_first) { - PROTECT( - auto outputs__ = torch::_pack_padded_sequence_out(*out0, *out1, *input, *lengths, (bool)batch_first); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__pad_circular(tensor *out__, tensor self, int64_t *pad_data, int pad_len) { - PROTECT( - auto outputs__ = torch::_pad_circular(*self, torch::IntArrayRef(pad_data, pad_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__pad_enum(tensor *out__, tensor self, int64_t *pad_data, int pad_len, int64_t mode, double value_v, uint8_t value_null) { - PROTECT( - auto outputs__ = torch::_pad_enum(*self, torch::IntArrayRef(pad_data, pad_len), mode, value_null ? c10::nullopt : c10::optional(value_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__pad_packed_sequence(tensor *out__, tensor data, tensor batch_sizes, int batch_first, scalar padding_value, int64_t total_length) { - PROTECT( - auto outputs__ = torch::_pad_packed_sequence(*data, *batch_sizes, (bool)batch_first, *padding_value, total_length); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__pdist_backward(tensor *out__, tensor grad, tensor self, double p, tensor pdist) { - PROTECT( - auto outputs__ = torch::_pdist_backward(*grad, *self, p, *pdist); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__pdist_backward_out(tensor *out__, tensor out, tensor grad, tensor self, double p, tensor pdist) { - PROTECT( - auto outputs__ = torch::_pdist_backward_out(*out, *grad, *self, p, *pdist); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__pin_memory(tensor *out__, tensor self, int device) { - PROTECT( - auto outputs__ = torch::_pin_memory(*self, device_of_int(device)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__pin_memory_out(tensor *out__, tensor out, tensor self, int device) { - PROTECT( - auto outputs__ = torch::_pin_memory_out(*out, *self, device_of_int(device)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__prelu_kernel(tensor *out__, tensor self, tensor weight) { - PROTECT( - auto outputs__ = torch::_prelu_kernel(*self, *weight); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__prelu_kernel_backward(tensor *out__, tensor grad_output, tensor self, tensor weight) { - PROTECT( - auto outputs__ = torch::_prelu_kernel_backward(*grad_output, *self, *weight); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__print(char* s_ptr, int s_len) { - PROTECT( - torch::_print(std::string(s_ptr, s_len)); - ) -} - -void atg__propagate_xla_data(tensor input, tensor output) { - PROTECT( - torch::_propagate_xla_data(*input, *output); - ) -} - -void atg__remove_batch_dim(tensor *out__, tensor self, int64_t level, int64_t batch_size, int64_t out_dim) { - PROTECT( - auto outputs__ = torch::_remove_batch_dim(*self, level, batch_size, out_dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__reshape_alias(tensor *out__, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len) { - PROTECT( - auto outputs__ = torch::_reshape_alias(*self, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__reshape_alias_copy(tensor *out__, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len) { - PROTECT( - auto outputs__ = torch::_reshape_alias_copy(*self, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__reshape_alias_copy_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len) { - PROTECT( - auto outputs__ = torch::_reshape_alias_copy_out(*out, *self, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__reshape_copy(tensor *out__, tensor self, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::_reshape_copy(*self, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__reshape_from_tensor(tensor *out__, tensor self, tensor shape) { - PROTECT( - auto outputs__ = torch::_reshape_from_tensor(*self, *shape); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__resize_output(tensor *out__, tensor self, int64_t *size_data, int size_len, int device) { - PROTECT( - auto outputs__ = torch::_resize_output(*self, torch::IntArrayRef(size_data, size_len), device_of_int(device)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__resize_output_(tensor *out__, tensor self, int64_t *size_data, int size_len, int device) { - PROTECT( - auto outputs__ = torch::_resize_output_(*self, torch::IntArrayRef(size_data, size_len), device_of_int(device)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__resize_output_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len, int device) { - PROTECT( - auto outputs__ = torch::_resize_output_out(*out, *self, torch::IntArrayRef(size_data, size_len), device_of_int(device)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__rowwise_prune(tensor *out__, tensor weight, tensor mask, int compressed_indices_dtype) { - PROTECT( - auto outputs__ = torch::_rowwise_prune(*weight, *mask, at::ScalarType(compressed_indices_dtype)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__safe_softmax(tensor *out__, tensor self, int64_t dim, int dtype) { - PROTECT( - auto outputs__ = torch::_safe_softmax(*self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sample_dirichlet(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_sample_dirichlet(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sample_dirichlet_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::_sample_dirichlet_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__saturate_weight_to_fp16(tensor *out__, tensor weight) { - PROTECT( - auto outputs__ = torch::_saturate_weight_to_fp16(*weight); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__scaled_dot_product_attention_math(tensor *out__, tensor query, tensor key, tensor value, tensor attn_mask, double dropout_p, int is_causal, tensor dropout_mask, double scale_v, uint8_t scale_null, int enable_gqa) { - PROTECT( - auto outputs__ = torch::_scaled_dot_product_attention_math(*query, *key, *value, (attn_mask ? ::std::optional(*attn_mask) : ::std::nullopt), dropout_p, (bool)is_causal, (dropout_mask ? ::std::optional(*dropout_mask) : ::std::nullopt), scale_null ? c10::nullopt : c10::optional(scale_v), (bool)enable_gqa); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__scaled_dot_product_attention_math_for_mps(tensor *out__, tensor query, tensor key, tensor value, tensor attn_mask, double dropout_p, int is_causal, tensor dropout_mask, double scale_v, uint8_t scale_null) { - PROTECT( - auto outputs__ = torch::_scaled_dot_product_attention_math_for_mps(*query, *key, *value, (attn_mask ? ::std::optional(*attn_mask) : ::std::nullopt), dropout_p, (bool)is_causal, (dropout_mask ? ::std::optional(*dropout_mask) : ::std::nullopt), scale_null ? c10::nullopt : c10::optional(scale_v)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__scaled_dot_product_cudnn_attention_backward(tensor *out__, tensor grad_out, tensor query, tensor key, tensor value, tensor out, tensor logsumexp, tensor philox_seed, tensor philox_offset, tensor attn_bias, tensor cum_seq_q, tensor cum_seq_k, int64_t max_q, int64_t max_k, double dropout_p, int is_causal, double scale_v, uint8_t scale_null) { - PROTECT( - auto outputs__ = torch::_scaled_dot_product_cudnn_attention_backward(*grad_out, *query, *key, *value, *out, *logsumexp, *philox_seed, *philox_offset, *attn_bias, *cum_seq_q, *cum_seq_k, max_q, max_k, dropout_p, (bool)is_causal, scale_null ? c10::nullopt : c10::optional(scale_v)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__scaled_dot_product_efficient_attention(tensor *out__, tensor query, tensor key, tensor value, tensor attn_bias, int compute_log_sumexp, double dropout_p, int is_causal, double scale_v, uint8_t scale_null) { - PROTECT( - auto outputs__ = torch::_scaled_dot_product_efficient_attention(*query, *key, *value, (attn_bias ? ::std::optional(*attn_bias) : ::std::nullopt), (bool)compute_log_sumexp, dropout_p, (bool)is_causal, scale_null ? c10::nullopt : c10::optional(scale_v)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg__scaled_dot_product_flash_attention_backward(tensor *out__, tensor grad_out, tensor query, tensor key, tensor value, tensor out, tensor logsumexp, tensor cum_seq_q, tensor cum_seq_k, int64_t max_q, int64_t max_k, double dropout_p, int is_causal, tensor philox_seed, tensor philox_offset, double scale_v, uint8_t scale_null) { - PROTECT( - auto outputs__ = torch::_scaled_dot_product_flash_attention_backward(*grad_out, *query, *key, *value, *out, *logsumexp, *cum_seq_q, *cum_seq_k, max_q, max_k, dropout_p, (bool)is_causal, *philox_seed, *philox_offset, scale_null ? c10::nullopt : c10::optional(scale_v)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__scaled_dot_product_flash_attention_for_cpu(tensor *out__, tensor query, tensor key, tensor value, double dropout_p, int is_causal, tensor attn_mask, double scale_v, uint8_t scale_null) { - PROTECT( - auto outputs__ = torch::_scaled_dot_product_flash_attention_for_cpu(*query, *key, *value, dropout_p, (bool)is_causal, (attn_mask ? ::std::optional(*attn_mask) : ::std::nullopt), scale_null ? c10::nullopt : c10::optional(scale_v)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__scaled_dot_product_flash_attention_for_cpu_backward(tensor *out__, tensor grad_out, tensor query, tensor key, tensor value, tensor out, tensor logsumexp, double dropout_p, int is_causal, tensor attn_mask, double scale_v, uint8_t scale_null) { - PROTECT( - auto outputs__ = torch::_scaled_dot_product_flash_attention_for_cpu_backward(*grad_out, *query, *key, *value, *out, *logsumexp, dropout_p, (bool)is_causal, (attn_mask ? ::std::optional(*attn_mask) : ::std::nullopt), scale_null ? c10::nullopt : c10::optional(scale_v)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__scaled_grouped_mm(tensor *out__, tensor self, tensor mat2, tensor scale_a, tensor scale_b, tensor offs, tensor bias, tensor scale_result, int out_dtype, int use_fast_accum) { - PROTECT( - auto outputs__ = torch::_scaled_grouped_mm(*self, *mat2, *scale_a, *scale_b, (offs ? ::std::optional(*offs) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), (scale_result ? ::std::optional(*scale_result) : ::std::nullopt), out_dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(out_dtype)), (bool)use_fast_accum); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__scaled_grouped_mm_v2(tensor *out__, tensor self, tensor mat2, tensor *scale_a_data, int scale_a_len, int64_t *recipe_a_data, int recipe_a_len, int64_t *swizzle_a_data, int swizzle_a_len, tensor *scale_b_data, int scale_b_len, int64_t *recipe_b_data, int recipe_b_len, int64_t *swizzle_b_data, int swizzle_b_len, tensor offs, tensor bias, int out_dtype, int64_t *contraction_dim_data, int contraction_dim_len, int use_fast_accum) { - PROTECT( - auto outputs__ = torch::_scaled_grouped_mm_v2(*self, *mat2, of_carray_tensor(scale_a_data, scale_a_len), torch::IntArrayRef(recipe_a_data, recipe_a_len), torch::IntArrayRef(swizzle_a_data, swizzle_a_len), of_carray_tensor(scale_b_data, scale_b_len), torch::IntArrayRef(recipe_b_data, recipe_b_len), torch::IntArrayRef(swizzle_b_data, swizzle_b_len), (offs ? ::std::optional(*offs) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), out_dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(out_dtype)), torch::IntArrayRef(contraction_dim_data, contraction_dim_len), (bool)use_fast_accum); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__scaled_mm(tensor *out__, tensor self, tensor mat2, tensor scale_a, tensor scale_b, tensor bias, tensor scale_result, int out_dtype, int use_fast_accum) { - PROTECT( - auto outputs__ = torch::_scaled_mm(*self, *mat2, *scale_a, *scale_b, (bias ? ::std::optional(*bias) : ::std::nullopt), (scale_result ? ::std::optional(*scale_result) : ::std::nullopt), out_dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(out_dtype)), (bool)use_fast_accum); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__scaled_mm_out(tensor *out__, tensor out, tensor self, tensor mat2, tensor scale_a, tensor scale_b, tensor bias, tensor scale_result, int out_dtype, int use_fast_accum) { - PROTECT( - auto outputs__ = torch::_scaled_mm_out(*out, *self, *mat2, *scale_a, *scale_b, (bias ? ::std::optional(*bias) : ::std::nullopt), (scale_result ? ::std::optional(*scale_result) : ::std::nullopt), out_dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(out_dtype)), (bool)use_fast_accum); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__scaled_mm_v2(tensor *out__, tensor self, tensor mat2, tensor *scale_a_data, int scale_a_len, int64_t *recipe_a_data, int recipe_a_len, int64_t *swizzle_a_data, int swizzle_a_len, tensor *scale_b_data, int scale_b_len, int64_t *recipe_b_data, int recipe_b_len, int64_t *swizzle_b_data, int swizzle_b_len, tensor bias, int out_dtype, int64_t *contraction_dim_data, int contraction_dim_len, int use_fast_accum) { - PROTECT( - auto outputs__ = torch::_scaled_mm_v2(*self, *mat2, of_carray_tensor(scale_a_data, scale_a_len), torch::IntArrayRef(recipe_a_data, recipe_a_len), torch::IntArrayRef(swizzle_a_data, swizzle_a_len), of_carray_tensor(scale_b_data, scale_b_len), torch::IntArrayRef(recipe_b_data, recipe_b_len), torch::IntArrayRef(swizzle_b_data, swizzle_b_len), (bias ? ::std::optional(*bias) : ::std::nullopt), out_dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(out_dtype)), torch::IntArrayRef(contraction_dim_data, contraction_dim_len), (bool)use_fast_accum); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__scaled_mm_v2_out(tensor *out__, tensor out, tensor self, tensor mat2, tensor *scale_a_data, int scale_a_len, int64_t *recipe_a_data, int recipe_a_len, int64_t *swizzle_a_data, int swizzle_a_len, tensor *scale_b_data, int scale_b_len, int64_t *recipe_b_data, int recipe_b_len, int64_t *swizzle_b_data, int swizzle_b_len, tensor bias, int out_dtype, int64_t *contraction_dim_data, int contraction_dim_len, int use_fast_accum) { - PROTECT( - auto outputs__ = torch::_scaled_mm_v2_out(*out, *self, *mat2, of_carray_tensor(scale_a_data, scale_a_len), torch::IntArrayRef(recipe_a_data, recipe_a_len), torch::IntArrayRef(swizzle_a_data, swizzle_a_len), of_carray_tensor(scale_b_data, scale_b_len), torch::IntArrayRef(recipe_b_data, recipe_b_len), torch::IntArrayRef(swizzle_b_data, swizzle_b_len), (bias ? ::std::optional(*bias) : ::std::nullopt), out_dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(out_dtype)), torch::IntArrayRef(contraction_dim_data, contraction_dim_len), (bool)use_fast_accum); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__scatter_reduce(tensor *out__, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len, int include_self) { - PROTECT( - auto outputs__ = torch::scatter_reduce(*self, dim, *index, *src, std::string(reduce_ptr, reduce_len), (bool)include_self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__scatter_reduce_(tensor *out__, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len, int include_self) { - PROTECT( - auto outputs__ = self->scatter_reduce_(dim, *index, *src, std::string(reduce_ptr, reduce_len), (bool)include_self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__scatter_reduce_two_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len, int include_self) { - PROTECT( - auto outputs__ = torch::scatter_reduce_out(*out, *self, dim, *index, *src, std::string(reduce_ptr, reduce_len), (bool)include_self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__segment_reduce_backward(tensor *out__, tensor grad, tensor output, tensor data, char* reduce_ptr, int reduce_len, tensor lengths, tensor offsets, int64_t axis, scalar initial) { - PROTECT( - auto outputs__ = torch::_segment_reduce_backward(*grad, *output, *data, std::string(reduce_ptr, reduce_len), (lengths ? ::std::optional(*lengths) : ::std::nullopt), (offsets ? ::std::optional(*offsets) : ::std::nullopt), axis, *initial); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__segment_reduce_backward_out(tensor *out__, tensor out, tensor grad, tensor output, tensor data, char* reduce_ptr, int reduce_len, tensor lengths, tensor offsets, int64_t axis, scalar initial) { - PROTECT( - auto outputs__ = torch::_segment_reduce_backward_out(*out, *grad, *output, *data, std::string(reduce_ptr, reduce_len), (lengths ? ::std::optional(*lengths) : ::std::nullopt), (offsets ? ::std::optional(*offsets) : ::std::nullopt), axis, *initial); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__shape_as_tensor(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_shape_as_tensor(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__slow_conv2d_backward(tensor *out__, tensor grad_input, tensor grad_weight, tensor grad_bias, tensor grad_output, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::_slow_conv2d_backward_out(*grad_input, *grad_weight, *grad_bias, *grad_output, *self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__sobol_engine_draw(tensor *out__, tensor quasi, int64_t n, tensor sobolstate, int64_t dimension, int64_t num_generated, int dtype) { - PROTECT( - auto outputs__ = torch::_sobol_engine_draw(*quasi, n, *sobolstate, dimension, num_generated, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__sobol_engine_ff_(tensor *out__, tensor self, int64_t n, tensor sobolstate, int64_t dimension, int64_t num_generated) { - PROTECT( - auto outputs__ = torch::_sobol_engine_ff_(*self, n, *sobolstate, dimension, num_generated); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sobol_engine_initialize_state_(tensor *out__, tensor self, int64_t dimension) { - PROTECT( - auto outputs__ = torch::_sobol_engine_initialize_state_(*self, dimension); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sobol_engine_scramble_(tensor *out__, tensor self, tensor ltm, int64_t dimension) { - PROTECT( - auto outputs__ = torch::_sobol_engine_scramble_(*self, *ltm, dimension); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__softmax(tensor *out__, tensor self, int64_t dim, int half_to_float) { - PROTECT( - auto outputs__ = torch::_softmax(*self, dim, (bool)half_to_float); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__softmax_backward_data(tensor *out__, tensor grad_output, tensor output, int64_t dim, int input_dtype) { - PROTECT( - auto outputs__ = torch::_softmax_backward_data(*grad_output, *output, dim, at::ScalarType(input_dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__softmax_backward_data_out(tensor *out__, tensor grad_input, tensor grad_output, tensor output, int64_t dim, int input_dtype) { - PROTECT( - auto outputs__ = torch::_softmax_backward_data_out(*grad_input, *grad_output, *output, dim, at::ScalarType(input_dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__softmax_out(tensor *out__, tensor out, tensor self, int64_t dim, int half_to_float) { - PROTECT( - auto outputs__ = torch::_softmax_out(*out, *self, dim, (bool)half_to_float); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_addmm(tensor *out__, tensor self, tensor mat1, tensor mat2) { - PROTECT( - auto outputs__ = torch::_sparse_addmm(*self, *mat1, *mat2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_addmm_out(tensor *out__, tensor out, tensor self, tensor mat1, tensor mat2) { - PROTECT( - auto outputs__ = torch::_sparse_addmm_out(*out, *self, *mat1, *mat2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_broadcast_to(tensor *out__, tensor self, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::_sparse_broadcast_to(*self, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_broadcast_to_copy(tensor *out__, tensor self, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::_sparse_broadcast_to_copy(*self, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_broadcast_to_copy_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::_sparse_broadcast_to_copy_out(*out, *self, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_bsc_tensor_unsafe(tensor *out__, tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::_sparse_bsc_tensor_unsafe(*ccol_indices, *row_indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_bsr_tensor_unsafe(tensor *out__, tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::_sparse_bsr_tensor_unsafe(*crow_indices, *col_indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_compressed_tensor_unsafe(tensor *out__, tensor compressed_indices, tensor plain_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::_sparse_compressed_tensor_unsafe(*compressed_indices, *plain_indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_compressed_tensor_with_dims(tensor *out__, int64_t nnz, int64_t dense_dim, int64_t *size_data, int size_len, int64_t *blocksize_data, int blocksize_len, int index_dtype, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::_sparse_compressed_tensor_with_dims(nnz, dense_dim, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(blocksize_data, blocksize_len), at::ScalarType(index_dtype), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_coo_tensor_unsafe(tensor *out__, tensor indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device, int is_coalesced) { - PROTECT( - auto outputs__ = torch::_sparse_coo_tensor_unsafe(*indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind)), (bool)is_coalesced); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_coo_tensor_with_dims(tensor *out__, int64_t sparse_dim, int64_t dense_dim, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::_sparse_coo_tensor_with_dims(sparse_dim, dense_dim, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_coo_tensor_with_dims_and_tensors(tensor *out__, int64_t sparse_dim, int64_t dense_dim, int64_t *size_data, int size_len, tensor indices, tensor values, int options_kind, int options_device, int is_coalesced) { - PROTECT( - auto outputs__ = torch::_sparse_coo_tensor_with_dims_and_tensors(sparse_dim, dense_dim, torch::IntArrayRef(size_data, size_len), *indices, *values, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind)), (bool)is_coalesced); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_coo_tensor_with_dims_and_tensors_out(tensor *out__, tensor out, int64_t sparse_dim, int64_t dense_dim, int64_t *size_data, int size_len, tensor indices, tensor values, int is_coalesced) { - PROTECT( - auto outputs__ = torch::_sparse_coo_tensor_with_dims_and_tensors_out(*out, sparse_dim, dense_dim, torch::IntArrayRef(size_data, size_len), *indices, *values, (bool)is_coalesced); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_coo_tensor_with_dims_out(tensor *out__, tensor out, int64_t sparse_dim, int64_t dense_dim, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::_sparse_coo_tensor_with_dims_out(*out, sparse_dim, dense_dim, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_csc_tensor_unsafe(tensor *out__, tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::_sparse_csc_tensor_unsafe(*ccol_indices, *row_indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_csr_prod(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { - PROTECT( - auto outputs__ = torch::_sparse_csr_prod(*self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_csr_prod_dim_dtype_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { - PROTECT( - auto outputs__ = torch::_sparse_csr_prod_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_csr_sum(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { - PROTECT( - auto outputs__ = torch::_sparse_csr_sum(*self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_csr_sum_dim_dtype_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { - PROTECT( - auto outputs__ = torch::_sparse_csr_sum_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_csr_tensor_unsafe(tensor *out__, tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::_sparse_csr_tensor_unsafe(*crow_indices, *col_indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_log_softmax(tensor *out__, tensor self, int64_t dim, int half_to_float) { - PROTECT( - auto outputs__ = torch::_sparse_log_softmax(*self, dim, (bool)half_to_float); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_log_softmax_backward_data(tensor *out__, tensor grad_output, tensor output, int64_t dim, tensor self) { - PROTECT( - auto outputs__ = torch::_sparse_log_softmax_backward_data(*grad_output, *output, dim, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_log_softmax_backward_data_out(tensor *out__, tensor out, tensor grad_output, tensor output, int64_t dim, tensor self) { - PROTECT( - auto outputs__ = torch::_sparse_log_softmax_backward_data_out(*out, *grad_output, *output, dim, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_log_softmax_int(tensor *out__, tensor self, int64_t dim, int dtype) { - PROTECT( - auto outputs__ = torch::_sparse_log_softmax(*self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_log_softmax_out(tensor *out__, tensor out, tensor self, int64_t dim, int half_to_float) { - PROTECT( - auto outputs__ = torch::_sparse_log_softmax_out(*out, *self, dim, (bool)half_to_float); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_mask_projection(tensor *out__, tensor self, tensor mask, int accumulate_matches) { - PROTECT( - auto outputs__ = self->_sparse_mask_projection(*mask, (bool)accumulate_matches); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_mask_projection_out(tensor *out__, tensor out, tensor self, tensor mask, int accumulate_matches) { - PROTECT( - auto outputs__ = torch::_sparse_mask_projection_out(*out, *self, *mask, (bool)accumulate_matches); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_mm(tensor *out__, tensor sparse, tensor dense) { - PROTECT( - auto outputs__ = torch::_sparse_mm(*sparse, *dense); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_mm_reduce(tensor *out__, tensor sparse, tensor dense, char* reduce_ptr, int reduce_len) { - PROTECT( - auto outputs__ = torch::_sparse_mm(*sparse, *dense, std::string(reduce_ptr, reduce_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_mm_reduce_impl(tensor *out__, tensor self, tensor other, char* reduce_ptr, int reduce_len) { - PROTECT( - auto outputs__ = torch::_sparse_mm_reduce_impl(*self, *other, std::string(reduce_ptr, reduce_len)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__sparse_semi_structured_apply(tensor *out__, tensor input, tensor thread_masks) { - PROTECT( - auto outputs__ = torch::_sparse_semi_structured_apply(*input, *thread_masks); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__sparse_semi_structured_apply_dense(tensor *out__, tensor input, tensor thread_masks) { - PROTECT( - auto outputs__ = torch::_sparse_semi_structured_apply_dense(*input, *thread_masks); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_semi_structured_linear(tensor *out__, tensor input, tensor weight, tensor meta, tensor bias, char* activation_ptr, int activation_len, int out_dtype) { - PROTECT( - auto outputs__ = torch::_sparse_semi_structured_linear(*input, *weight, *meta, (bias ? ::std::optional(*bias) : ::std::nullopt), std::string(activation_ptr, activation_len), out_dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(out_dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_semi_structured_mm(tensor *out__, tensor mat1, tensor mat1_meta, tensor mat2, int out_dtype) { - PROTECT( - auto outputs__ = torch::_sparse_semi_structured_mm(*mat1, *mat1_meta, *mat2, out_dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(out_dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_semi_structured_tile(tensor *out__, tensor input, char* algorithm_ptr, int algorithm_len, int use_cutlass) { - PROTECT( - auto outputs__ = torch::_sparse_semi_structured_tile(*input, std::string(algorithm_ptr, algorithm_len), (bool)use_cutlass); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - out__[4] = new torch::Tensor(std::get<4>(outputs__)); - ) -} - -void atg__sparse_softmax(tensor *out__, tensor self, int64_t dim, int half_to_float) { - PROTECT( - auto outputs__ = torch::_sparse_softmax(*self, dim, (bool)half_to_float); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_softmax_backward_data(tensor *out__, tensor grad_output, tensor output, int64_t dim, tensor self) { - PROTECT( - auto outputs__ = torch::_sparse_softmax_backward_data(*grad_output, *output, dim, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_softmax_backward_data_out(tensor *out__, tensor out, tensor grad_output, tensor output, int64_t dim, tensor self) { - PROTECT( - auto outputs__ = torch::_sparse_softmax_backward_data_out(*out, *grad_output, *output, dim, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_softmax_int(tensor *out__, tensor self, int64_t dim, int dtype) { - PROTECT( - auto outputs__ = torch::_sparse_softmax(*self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_softmax_out(tensor *out__, tensor out, tensor self, int64_t dim, int half_to_float) { - PROTECT( - auto outputs__ = torch::_sparse_softmax_out(*out, *self, dim, (bool)half_to_float); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_sparse_matmul(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::_sparse_sparse_matmul(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_sparse_matmul_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::_sparse_sparse_matmul_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_sum(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_sparse_sum(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_sum_backward(tensor *out__, tensor grad, tensor self, int64_t *dim_data, int dim_len) { - PROTECT( - auto outputs__ = torch::_sparse_sum_backward(*grad, *self, torch::IntArrayRef(dim_data, dim_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_sum_backward_out(tensor *out__, tensor out, tensor grad, tensor self, int64_t *dim_data, int dim_len) { - PROTECT( - auto outputs__ = torch::_sparse_sum_backward_out(*out, *grad, *self, torch::IntArrayRef(dim_data, dim_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_sum_dim(tensor *out__, tensor self, int64_t *dim_data, int dim_len) { - PROTECT( - auto outputs__ = torch::_sparse_sum(*self, torch::IntArrayRef(dim_data, dim_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_sum_dim_dtype(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int dtype) { - PROTECT( - auto outputs__ = torch::_sparse_sum(*self, torch::IntArrayRef(dim_data, dim_len), at::ScalarType(dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_sum_dim_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len) { - PROTECT( - auto outputs__ = torch::_sparse_sum_out(*out, *self, torch::IntArrayRef(dim_data, dim_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__sparse_sum_dtype(tensor *out__, tensor self, int dtype) { - PROTECT( - auto outputs__ = torch::_sparse_sum(*self, at::ScalarType(dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__spdiags(tensor *out__, tensor diagonals, tensor offsets, int64_t *shape_data, int shape_len, int8_t layout) { - PROTECT( - auto outputs__ = torch::_spdiags(*diagonals, *offsets, torch::IntArrayRef(shape_data, shape_len), (layout == -1 ? c10::nullopt : c10::optional(static_cast(layout)))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__spdiags_out(tensor *out__, tensor out, tensor diagonals, tensor offsets, int64_t *shape_data, int shape_len, int8_t layout) { - PROTECT( - auto outputs__ = torch::_spdiags_out(*out, *diagonals, *offsets, torch::IntArrayRef(shape_data, shape_len), (layout == -1 ? c10::nullopt : c10::optional(static_cast(layout)))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__spsolve(tensor *out__, tensor A, tensor B, int left) { - PROTECT( - auto outputs__ = torch::_spsolve(*A, *B, (bool)left); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__stack(tensor *out__, tensor *tensors_data, int tensors_len, int64_t dim) { - PROTECT( - auto outputs__ = torch::_stack(of_carray_tensor(tensors_data, tensors_len), dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__stack_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len, int64_t dim) { - PROTECT( - auto outputs__ = torch::_stack_out(*out, of_carray_tensor(tensors_data, tensors_len), dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__standard_gamma(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_standard_gamma(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__standard_gamma_grad(tensor *out__, tensor self, tensor output) { - PROTECT( - auto outputs__ = torch::_standard_gamma_grad(*self, *output); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__standard_gamma_grad_out(tensor *out__, tensor out, tensor self, tensor output) { - PROTECT( - auto outputs__ = torch::_standard_gamma_grad_out(*out, *self, *output); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__standard_gamma_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::_standard_gamma_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__test_ambiguous_defaults(tensor *out__, tensor dummy, int64_t a, int64_t b) { - PROTECT( - auto outputs__ = torch::_test_ambiguous_defaults(*dummy, a, b); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__test_ambiguous_defaults_b(tensor *out__, tensor dummy, int64_t a, char* b_ptr, int b_len) { - PROTECT( - auto outputs__ = torch::_test_ambiguous_defaults(*dummy, a, std::string(b_ptr, b_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__test_autograd_multiple_dispatch(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_test_autograd_multiple_dispatch(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__test_autograd_multiple_dispatch_fullcoverage_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::_test_autograd_multiple_dispatch_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__test_autograd_multiple_dispatch_ntonly(tensor *out__, tensor self, int b) { - PROTECT( - auto outputs__ = torch::_test_autograd_multiple_dispatch(*self, (bool)b); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__test_autograd_multiple_dispatch_view(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_test_autograd_multiple_dispatch_view(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__test_autograd_multiple_dispatch_view_copy(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_test_autograd_multiple_dispatch_view_copy(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__test_autograd_multiple_dispatch_view_copy_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::_test_autograd_multiple_dispatch_view_copy_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__test_check_tensor(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_test_check_tensor(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__test_functorch_fallback(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::_test_functorch_fallback(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__test_functorch_fallback_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::_test_functorch_fallback_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__test_optional_filled_intlist(tensor *out__, tensor values, int64_t *addends_data, int addends_len) { - PROTECT( - auto outputs__ = torch::_test_optional_filled_intlist(*values, addends_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(addends_data, addends_len))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__test_optional_filled_intlist_out(tensor *out__, tensor out, tensor values, int64_t *addends_data, int addends_len) { - PROTECT( - auto outputs__ = torch::_test_optional_filled_intlist_out(*out, *values, addends_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(addends_data, addends_len))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__test_optional_floatlist(tensor *out__, tensor values, double *addends_data, int addends_len) { - PROTECT( - auto outputs__ = torch::_test_optional_floatlist(*values, at::ArrayRef(addends_data, addends_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__test_optional_floatlist_out(tensor *out__, tensor out, tensor values, double *addends_data, int addends_len) { - PROTECT( - auto outputs__ = torch::_test_optional_floatlist_out(*out, *values, at::ArrayRef(addends_data, addends_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__test_optional_intlist(tensor *out__, tensor values, int64_t *addends_data, int addends_len) { - PROTECT( - auto outputs__ = torch::_test_optional_intlist(*values, addends_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(addends_data, addends_len))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__test_optional_intlist_out(tensor *out__, tensor out, tensor values, int64_t *addends_data, int addends_len) { - PROTECT( - auto outputs__ = torch::_test_optional_intlist_out(*out, *values, addends_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(addends_data, addends_len))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__test_parallel_materialize(tensor *out__, tensor self, int64_t num_parallel, int skip_first) { - PROTECT( - auto outputs__ = torch::_test_parallel_materialize(*self, num_parallel, (bool)skip_first); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__test_serialization_subcmul(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::_test_serialization_subcmul(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__test_string_default(tensor *out__, tensor dummy, char* a_ptr, int a_len, char* b_ptr, int b_len) { - PROTECT( - auto outputs__ = torch::_test_string_default(*dummy, std::string(a_ptr, a_len), std::string(b_ptr, b_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__test_warn_in_autograd(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_test_warn_in_autograd(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__test_warn_in_autograd_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::_test_warn_in_autograd_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__to_copy(tensor *out__, tensor self, int options_kind, int options_device, int non_blocking) { - PROTECT( - auto outputs__ = torch::_to_copy(*self, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind)), (bool)non_blocking); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__to_copy_out(tensor *out__, tensor out, tensor self, int non_blocking) { - PROTECT( - auto outputs__ = torch::_to_copy_out(*out, *self, (bool)non_blocking); - out__[0] = new torch::Tensor(outputs__); - ) -} - -tensor *atg__to_cpu(tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::_to_cpu(of_carray_tensor(tensors_data, tensors_len)); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg__to_dense(tensor *out__, tensor self, int dtype, int masked_grad) { - PROTECT( - auto outputs__ = self->_to_dense(dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype)), (bool)masked_grad); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__to_dense_out(tensor *out__, tensor out, tensor self, int dtype, int masked_grad) { - PROTECT( - auto outputs__ = torch::_to_dense_out(*out, *self, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype)), (bool)masked_grad); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__to_sparse(tensor *out__, tensor self, int8_t layout, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null) { - PROTECT( - auto outputs__ = self->_to_sparse((layout == -1 ? c10::nullopt : c10::optional(static_cast(layout))), blocksize_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(blocksize_data, blocksize_len)), dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__to_sparse_bsc(tensor *out__, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null) { - PROTECT( - auto outputs__ = self->_to_sparse_bsc(torch::IntArrayRef(blocksize_data, blocksize_len), dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__to_sparse_bsc_out(tensor *out__, tensor out, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null) { - PROTECT( - auto outputs__ = torch::_to_sparse_bsc_out(*out, *self, torch::IntArrayRef(blocksize_data, blocksize_len), dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__to_sparse_bsr(tensor *out__, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null) { - PROTECT( - auto outputs__ = self->_to_sparse_bsr(torch::IntArrayRef(blocksize_data, blocksize_len), dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__to_sparse_bsr_out(tensor *out__, tensor out, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null) { - PROTECT( - auto outputs__ = torch::_to_sparse_bsr_out(*out, *self, torch::IntArrayRef(blocksize_data, blocksize_len), dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__to_sparse_csc(tensor *out__, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null) { - PROTECT( - auto outputs__ = self->_to_sparse_csc(dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__to_sparse_csc_out(tensor *out__, tensor out, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null) { - PROTECT( - auto outputs__ = torch::_to_sparse_csc_out(*out, *self, dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__to_sparse_csr(tensor *out__, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null) { - PROTECT( - auto outputs__ = self->_to_sparse_csr(dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__to_sparse_csr_out(tensor *out__, tensor out, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null) { - PROTECT( - auto outputs__ = torch::_to_sparse_csr_out(*out, *self, dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__to_sparse_out(tensor *out__, tensor out, tensor self, int8_t layout, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null) { - PROTECT( - auto outputs__ = torch::_to_sparse_out(*out, *self, (layout == -1 ? c10::nullopt : c10::optional(static_cast(layout))), blocksize_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(blocksize_data, blocksize_len)), dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__to_sparse_semi_structured(tensor *out__, tensor dense) { - PROTECT( - auto outputs__ = torch::_to_sparse_semi_structured(*dense); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__to_sparse_sparse_dim(tensor *out__, tensor self, int64_t sparse_dim) { - PROTECT( - auto outputs__ = self->_to_sparse(sparse_dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__to_sparse_sparse_dim_out(tensor *out__, tensor out, tensor self, int64_t sparse_dim) { - PROTECT( - auto outputs__ = torch::_to_sparse_out(*out, *self, sparse_dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__transform_bias_rescale_qkv(tensor *out__, tensor qkv, tensor qkv_bias, int64_t num_heads) { - PROTECT( - auto outputs__ = torch::_transform_bias_rescale_qkv(*qkv, *qkv_bias, num_heads); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__transform_bias_rescale_qkv_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor qkv, tensor qkv_bias, int64_t num_heads) { - PROTECT( - auto outputs__ = torch::_transform_bias_rescale_qkv_out(*out0, *out1, *out2, *qkv, *qkv_bias, num_heads); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__transformer_encoder_layer_fwd(tensor *out__, tensor src, int64_t embed_dim, int64_t num_heads, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, int use_gelu, int norm_first, double eps, tensor norm_weight_1, tensor norm_bias_1, tensor norm_weight_2, tensor norm_bias_2, tensor ffn_weight_1, tensor ffn_bias_1, tensor ffn_weight_2, tensor ffn_bias_2, tensor mask, int64_t mask_type_v, uint8_t mask_type_null) { - PROTECT( - auto outputs__ = torch::_transformer_encoder_layer_fwd(*src, embed_dim, num_heads, *qkv_weight, *qkv_bias, *proj_weight, *proj_bias, (bool)use_gelu, (bool)norm_first, eps, *norm_weight_1, *norm_bias_1, *norm_weight_2, *norm_bias_2, *ffn_weight_1, *ffn_bias_1, *ffn_weight_2, *ffn_bias_2, (mask ? ::std::optional(*mask) : ::std::nullopt), mask_type_null ? c10::nullopt : c10::optional(mask_type_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__transformer_encoder_layer_fwd_out(tensor *out__, tensor out, tensor src, int64_t embed_dim, int64_t num_heads, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, int use_gelu, int norm_first, double eps, tensor norm_weight_1, tensor norm_bias_1, tensor norm_weight_2, tensor norm_bias_2, tensor ffn_weight_1, tensor ffn_bias_1, tensor ffn_weight_2, tensor ffn_bias_2, tensor mask, int64_t mask_type_v, uint8_t mask_type_null) { - PROTECT( - auto outputs__ = torch::_transformer_encoder_layer_fwd_out(*out, *src, embed_dim, num_heads, *qkv_weight, *qkv_bias, *proj_weight, *proj_bias, (bool)use_gelu, (bool)norm_first, eps, *norm_weight_1, *norm_bias_1, *norm_weight_2, *norm_bias_2, *ffn_weight_1, *ffn_bias_1, *ffn_weight_2, *ffn_bias_2, (mask ? ::std::optional(*mask) : ::std::nullopt), mask_type_null ? c10::nullopt : c10::optional(mask_type_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__trilinear(tensor *out__, tensor i1, tensor i2, tensor i3, int64_t *expand1_data, int expand1_len, int64_t *expand2_data, int expand2_len, int64_t *expand3_data, int expand3_len, int64_t *sumdim_data, int sumdim_len, int64_t unroll_dim) { - PROTECT( - auto outputs__ = torch::_trilinear(*i1, *i2, *i3, torch::IntArrayRef(expand1_data, expand1_len), torch::IntArrayRef(expand2_data, expand2_len), torch::IntArrayRef(expand3_data, expand3_len), torch::IntArrayRef(sumdim_data, sumdim_len), unroll_dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__trilinear_out(tensor *out__, tensor out, tensor i1, tensor i2, tensor i3, int64_t *expand1_data, int expand1_len, int64_t *expand2_data, int expand2_len, int64_t *expand3_data, int expand3_len, int64_t *sumdim_data, int sumdim_len, int64_t unroll_dim) { - PROTECT( - auto outputs__ = torch::_trilinear_out(*out, *i1, *i2, *i3, torch::IntArrayRef(expand1_data, expand1_len), torch::IntArrayRef(expand2_data, expand2_len), torch::IntArrayRef(expand3_data, expand3_len), torch::IntArrayRef(sumdim_data, sumdim_len), unroll_dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__triton_multi_head_attention(tensor *out__, tensor query, tensor key, tensor value, int64_t embed_dim, int64_t num_head, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, tensor mask) { - PROTECT( - auto outputs__ = torch::_triton_multi_head_attention(*query, *key, *value, embed_dim, num_head, *qkv_weight, *qkv_bias, *proj_weight, *proj_bias, (mask ? ::std::optional(*mask) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__triton_multi_head_attention_out(tensor *out__, tensor out, tensor query, tensor key, tensor value, int64_t embed_dim, int64_t num_head, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, tensor mask) { - PROTECT( - auto outputs__ = torch::_triton_multi_head_attention_out(*out, *query, *key, *value, embed_dim, num_head, *qkv_weight, *qkv_bias, *proj_weight, *proj_bias, (mask ? ::std::optional(*mask) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__triton_scaled_dot_attention(tensor *out__, tensor q, tensor k, tensor v, double dropout_p) { - PROTECT( - auto outputs__ = torch::_triton_scaled_dot_attention(*q, *k, *v, dropout_p); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__triton_scaled_dot_attention_out(tensor *out__, tensor out, tensor q, tensor k, tensor v, double dropout_p) { - PROTECT( - auto outputs__ = torch::_triton_scaled_dot_attention_out(*out, *q, *k, *v, dropout_p); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__unique(tensor *out__, tensor self, int sorted, int return_inverse) { - PROTECT( - auto outputs__ = torch::_unique(*self, (bool)sorted, (bool)return_inverse); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__unique2(tensor *out__, tensor self, int sorted, int return_inverse, int return_counts) { - PROTECT( - auto outputs__ = torch::_unique2(*self, (bool)sorted, (bool)return_inverse, (bool)return_counts); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__unique2_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor self, int sorted, int return_inverse, int return_counts) { - PROTECT( - auto outputs__ = torch::_unique2_out(*out0, *out1, *out2, *self, (bool)sorted, (bool)return_inverse, (bool)return_counts); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg__unique_out(tensor *out__, tensor out0, tensor out1, tensor self, int sorted, int return_inverse) { - PROTECT( - auto outputs__ = torch::_unique_out(*out0, *out1, *self, (bool)sorted, (bool)return_inverse); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__unpack_dual(tensor *out__, tensor dual, int64_t level) { - PROTECT( - auto outputs__ = torch::_unpack_dual(*dual, level); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__unsafe_index(tensor *out__, tensor self, tensor *indices_data, int indices_len) { - PROTECT( - auto outputs__ = torch::_unsafe_index(*self, of_carray_tensor_opt(indices_data, indices_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__unsafe_index_put(tensor *out__, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate) { - PROTECT( - auto outputs__ = torch::_unsafe_index_put(*self, of_carray_tensor_opt(indices_data, indices_len), *values, (bool)accumulate); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__unsafe_masked_index(tensor *out__, tensor self, tensor mask, tensor *indices_data, int indices_len, scalar fill) { - PROTECT( - auto outputs__ = torch::_unsafe_masked_index(*self, *mask, of_carray_tensor_opt(indices_data, indices_len), *fill); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__unsafe_masked_index_put_accumulate(tensor *out__, tensor self, tensor mask, tensor *indices_data, int indices_len, tensor values) { - PROTECT( - auto outputs__ = torch::_unsafe_masked_index_put_accumulate(*self, *mask, of_carray_tensor_opt(indices_data, indices_len), *values); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__unsafe_view(tensor *out__, tensor self, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::_unsafe_view(*self, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__unsafe_view_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::_unsafe_view_out(*out, *self, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_bicubic2d_aa(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::_upsample_bicubic2d_aa(*self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_bicubic2d_aa_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::_upsample_bicubic2d_aa_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_bicubic2d_aa_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::_upsample_bicubic2d_aa_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_bicubic2d_aa_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::_upsample_bicubic2d_aa_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_bicubic2d_aa_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len) { - PROTECT( - auto outputs__ = torch::_upsample_bicubic2d_aa(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), (bool)align_corners, at::ArrayRef(scale_factors_data, scale_factors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_bilinear2d_aa(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::_upsample_bilinear2d_aa(*self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_bilinear2d_aa_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::_upsample_bilinear2d_aa_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_bilinear2d_aa_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::_upsample_bilinear2d_aa_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_bilinear2d_aa_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::_upsample_bilinear2d_aa_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_bilinear2d_aa_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len) { - PROTECT( - auto outputs__ = torch::_upsample_bilinear2d_aa(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), (bool)align_corners, at::ArrayRef(scale_factors_data, scale_factors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_nearest_exact1d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, double scales_v, uint8_t scales_null) { - PROTECT( - auto outputs__ = torch::_upsample_nearest_exact1d(*self, torch::IntArrayRef(output_size_data, output_size_len), scales_null ? c10::nullopt : c10::optional(scales_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_nearest_exact1d_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_v, uint8_t scales_null) { - PROTECT( - auto outputs__ = torch::_upsample_nearest_exact1d_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_null ? c10::nullopt : c10::optional(scales_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_nearest_exact1d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_v, uint8_t scales_null) { - PROTECT( - auto outputs__ = torch::_upsample_nearest_exact1d_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_null ? c10::nullopt : c10::optional(scales_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_nearest_exact1d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_v, uint8_t scales_null) { - PROTECT( - auto outputs__ = torch::_upsample_nearest_exact1d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), scales_null ? c10::nullopt : c10::optional(scales_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_nearest_exact1d_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len) { - PROTECT( - auto outputs__ = torch::_upsample_nearest_exact1d(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), at::ArrayRef(scale_factors_data, scale_factors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_nearest_exact2d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::_upsample_nearest_exact2d(*self, torch::IntArrayRef(output_size_data, output_size_len), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_nearest_exact2d_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::_upsample_nearest_exact2d_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_nearest_exact2d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::_upsample_nearest_exact2d_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_nearest_exact2d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::_upsample_nearest_exact2d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_nearest_exact2d_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len) { - PROTECT( - auto outputs__ = torch::_upsample_nearest_exact2d(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), at::ArrayRef(scale_factors_data, scale_factors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_nearest_exact3d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::_upsample_nearest_exact3d(*self, torch::IntArrayRef(output_size_data, output_size_len), scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_nearest_exact3d_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::_upsample_nearest_exact3d_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_nearest_exact3d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::_upsample_nearest_exact3d_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_nearest_exact3d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::_upsample_nearest_exact3d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__upsample_nearest_exact3d_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len) { - PROTECT( - auto outputs__ = torch::_upsample_nearest_exact3d(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), at::ArrayRef(scale_factors_data, scale_factors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -int atg__use_cudnn_ctc_loss(tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank) { - PROTECT( - return torch::_use_cudnn_ctc_loss(*log_probs, *targets, torch::IntArrayRef(input_lengths_data, input_lengths_len), torch::IntArrayRef(target_lengths_data, target_lengths_len), blank); - ) - return 0; -} - -int atg__use_cudnn_ctc_loss_tensor(tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank) { - PROTECT( - return torch::_use_cudnn_ctc_loss(*log_probs, *targets, *input_lengths, *target_lengths, blank); - ) - return 0; -} - -int atg__use_cudnn_rnn_flatten_weight() { - PROTECT( - return torch::_use_cudnn_rnn_flatten_weight(); - ) - return 0; -} - -int atg__use_miopen_ctc_loss(tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank) { - PROTECT( - return torch::_use_miopen_ctc_loss(*log_probs, *targets, torch::IntArrayRef(input_lengths_data, input_lengths_len), torch::IntArrayRef(target_lengths_data, target_lengths_len), blank); - ) - return 0; -} - -int atg__use_miopen_ctc_loss_tensor(tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank) { - PROTECT( - return torch::_use_miopen_ctc_loss(*log_probs, *targets, *input_lengths, *target_lengths, blank); - ) - return 0; -} - -void atg__validate_compressed_sparse_indices(int is_crow, tensor compressed_idx, tensor plain_idx, int64_t cdim, int64_t dim, int64_t nnz) { - PROTECT( - torch::_validate_compressed_sparse_indices((bool)is_crow, *compressed_idx, *plain_idx, cdim, dim, nnz); - ) -} - -void atg__validate_sparse_bsc_tensor_args(tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int check_pinning) { - PROTECT( - torch::_validate_sparse_bsc_tensor_args(*ccol_indices, *row_indices, *values, torch::IntArrayRef(size_data, size_len), (bool)check_pinning); - ) -} - -void atg__validate_sparse_bsr_tensor_args(tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int check_pinning) { - PROTECT( - torch::_validate_sparse_bsr_tensor_args(*crow_indices, *col_indices, *values, torch::IntArrayRef(size_data, size_len), (bool)check_pinning); - ) -} - -void atg__validate_sparse_compressed_tensor_args(tensor compressed_indices, tensor plain_indices, tensor values, int64_t *size_data, int size_len, int8_t layout, int check_pinning) { - PROTECT( - torch::_validate_sparse_compressed_tensor_args(*compressed_indices, *plain_indices, *values, torch::IntArrayRef(size_data, size_len), static_cast(layout), (bool)check_pinning); - ) -} - -void atg__validate_sparse_csc_tensor_args(tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int check_pinning) { - PROTECT( - torch::_validate_sparse_csc_tensor_args(*ccol_indices, *row_indices, *values, torch::IntArrayRef(size_data, size_len), (bool)check_pinning); - ) -} - -void atg__validate_sparse_csr_tensor_args(tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int check_pinning) { - PROTECT( - torch::_validate_sparse_csr_tensor_args(*crow_indices, *col_indices, *values, torch::IntArrayRef(size_data, size_len), (bool)check_pinning); - ) -} - -void atg__values(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->_values(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__values_copy(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::_values_copy(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__values_copy_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::_values_copy_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -int64_t atg__version(tensor self) { - PROTECT( - return self->_version(); - ) - return 0; -} - -void atg__weight_int4pack_mm(tensor *out__, tensor self, tensor mat2, int64_t qGroupSize, tensor qScaleAndZeros) { - PROTECT( - auto outputs__ = torch::_weight_int4pack_mm(*self, *mat2, qGroupSize, *qScaleAndZeros); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__weight_int4pack_mm_for_cpu(tensor *out__, tensor self, tensor mat2, int64_t qGroupSize, tensor qScaleAndZeros) { - PROTECT( - auto outputs__ = torch::_weight_int4pack_mm_for_cpu(*self, *mat2, qGroupSize, *qScaleAndZeros); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__weight_int4pack_mm_with_scales_and_zeros(tensor *out__, tensor self, tensor mat2, int64_t qGroupSize, tensor qScale, tensor qZeros) { - PROTECT( - auto outputs__ = torch::_weight_int4pack_mm_with_scales_and_zeros(*self, *mat2, qGroupSize, *qScale, *qZeros); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__weight_int8pack_mm(tensor *out__, tensor self, tensor mat2, tensor scales) { - PROTECT( - auto outputs__ = torch::_weight_int8pack_mm(*self, *mat2, *scales); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__weight_norm(tensor *out__, tensor v, tensor g, int64_t dim) { - PROTECT( - auto outputs__ = torch::_weight_norm(*v, *g, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__weight_norm_differentiable_backward(tensor *out__, tensor grad_w, tensor saved_v, tensor saved_g, tensor saved_norms, int64_t dim) { - PROTECT( - auto outputs__ = torch::_weight_norm_differentiable_backward(*grad_w, *saved_v, *saved_g, *saved_norms, dim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__weight_norm_interface(tensor *out__, tensor v, tensor g, int64_t dim) { - PROTECT( - auto outputs__ = torch::_weight_norm_interface(*v, *g, dim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__weight_norm_interface_backward(tensor *out__, tensor grad_w, tensor saved_v, tensor saved_g, tensor saved_norms, int64_t dim) { - PROTECT( - auto outputs__ = torch::_weight_norm_interface_backward(*grad_w, *saved_v, *saved_g, *saved_norms, dim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__weight_norm_interface_backward_out(tensor *out__, tensor out0, tensor out1, tensor grad_w, tensor saved_v, tensor saved_g, tensor saved_norms, int64_t dim) { - PROTECT( - auto outputs__ = torch::_weight_norm_interface_backward_out(*out0, *out1, *grad_w, *saved_v, *saved_g, *saved_norms, dim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__weight_norm_interface_out(tensor *out__, tensor out0, tensor out1, tensor v, tensor g, int64_t dim) { - PROTECT( - auto outputs__ = torch::_weight_norm_interface_out(*out0, *out1, *v, *g, dim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg__wrapped_linear_prepack(tensor *out__, tensor weight, tensor weight_scale, tensor weight_zero_point, tensor bias) { - PROTECT( - auto outputs__ = torch::_wrapped_linear_prepack(*weight, *weight_scale, *weight_zero_point, *bias); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg__wrapped_quantized_linear_prepacked(tensor *out__, tensor input, tensor input_scale, tensor input_zero_point, tensor packed_weight, tensor output_scale, tensor output_zero_point, int64_t out_channel) { - PROTECT( - auto outputs__ = torch::_wrapped_quantized_linear_prepacked(*input, *input_scale, *input_zero_point, *packed_weight, *output_scale, *output_zero_point, out_channel); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_abs(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::abs(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_abs_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::abs_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_abs_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::abs_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_absolute(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::absolute(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_absolute_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->absolute_(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_absolute_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::absolute_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_acos(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::acos(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_acos_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::acos_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_acos_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::acos_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_acosh(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::acosh(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_acosh_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::acosh_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_acosh_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::acosh_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_adaptive_avg_pool1d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len) { - PROTECT( - auto outputs__ = torch::adaptive_avg_pool1d(*self, torch::IntArrayRef(output_size_data, output_size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_adaptive_avg_pool1d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len) { - PROTECT( - auto outputs__ = torch::adaptive_avg_pool1d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_adaptive_avg_pool2d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len) { - PROTECT( - auto outputs__ = torch::adaptive_avg_pool2d(*self, torch::IntArrayRef(output_size_data, output_size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_adaptive_avg_pool2d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len) { - PROTECT( - auto outputs__ = torch::adaptive_avg_pool2d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_adaptive_avg_pool3d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len) { - PROTECT( - auto outputs__ = torch::adaptive_avg_pool3d(*self, torch::IntArrayRef(output_size_data, output_size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_adaptive_avg_pool3d_backward(tensor *out__, tensor grad_input, tensor grad_output, tensor self) { - PROTECT( - auto outputs__ = torch::adaptive_avg_pool3d_backward_out(*grad_input, *grad_output, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_adaptive_avg_pool3d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len) { - PROTECT( - auto outputs__ = torch::adaptive_avg_pool3d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_adaptive_max_pool1d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len) { - PROTECT( - auto outputs__ = torch::adaptive_max_pool1d(*self, torch::IntArrayRef(output_size_data, output_size_len)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_adaptive_max_pool2d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len) { - PROTECT( - auto outputs__ = torch::adaptive_max_pool2d(*self, torch::IntArrayRef(output_size_data, output_size_len)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_adaptive_max_pool2d_backward(tensor *out__, tensor grad_output, tensor self, tensor indices) { - PROTECT( - auto outputs__ = torch::adaptive_max_pool2d_backward(*grad_output, *self, *indices); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_adaptive_max_pool2d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor indices) { - PROTECT( - auto outputs__ = torch::adaptive_max_pool2d_backward_out(*grad_input, *grad_output, *self, *indices); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_adaptive_max_pool2d_out(tensor *out__, tensor out, tensor indices, tensor self, int64_t *output_size_data, int output_size_len) { - PROTECT( - auto outputs__ = torch::adaptive_max_pool2d_out(*out, *indices, *self, torch::IntArrayRef(output_size_data, output_size_len)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_adaptive_max_pool3d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len) { - PROTECT( - auto outputs__ = torch::adaptive_max_pool3d(*self, torch::IntArrayRef(output_size_data, output_size_len)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_adaptive_max_pool3d_backward(tensor *out__, tensor grad_output, tensor self, tensor indices) { - PROTECT( - auto outputs__ = torch::adaptive_max_pool3d_backward(*grad_output, *self, *indices); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_adaptive_max_pool3d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor indices) { - PROTECT( - auto outputs__ = torch::adaptive_max_pool3d_backward_out(*grad_input, *grad_output, *self, *indices); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_adaptive_max_pool3d_out(tensor *out__, tensor out, tensor indices, tensor self, int64_t *output_size_data, int output_size_len) { - PROTECT( - auto outputs__ = torch::adaptive_max_pool3d_out(*out, *indices, *self, torch::IntArrayRef(output_size_data, output_size_len)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_add(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::add(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_add_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->add_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_add_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::add_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_add_scalar(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::add(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_add_scalar_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->add_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_add_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::add_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_addbmm(tensor *out__, tensor self, tensor batch1, tensor batch2) { - PROTECT( - auto outputs__ = torch::addbmm(*self, *batch1, *batch2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_addbmm_(tensor *out__, tensor self, tensor batch1, tensor batch2) { - PROTECT( - auto outputs__ = self->addbmm_(*batch1, *batch2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_addbmm_out(tensor *out__, tensor out, tensor self, tensor batch1, tensor batch2) { - PROTECT( - auto outputs__ = torch::addbmm_out(*out, *self, *batch1, *batch2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_addcdiv(tensor *out__, tensor self, tensor tensor1, tensor tensor2) { - PROTECT( - auto outputs__ = torch::addcdiv(*self, *tensor1, *tensor2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_addcdiv_(tensor *out__, tensor self, tensor tensor1, tensor tensor2) { - PROTECT( - auto outputs__ = self->addcdiv_(*tensor1, *tensor2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_addcdiv_out(tensor *out__, tensor out, tensor self, tensor tensor1, tensor tensor2) { - PROTECT( - auto outputs__ = torch::addcdiv_out(*out, *self, *tensor1, *tensor2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_addcmul(tensor *out__, tensor self, tensor tensor1, tensor tensor2) { - PROTECT( - auto outputs__ = torch::addcmul(*self, *tensor1, *tensor2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_addcmul_(tensor *out__, tensor self, tensor tensor1, tensor tensor2) { - PROTECT( - auto outputs__ = self->addcmul_(*tensor1, *tensor2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_addcmul_out(tensor *out__, tensor out, tensor self, tensor tensor1, tensor tensor2) { - PROTECT( - auto outputs__ = torch::addcmul_out(*out, *self, *tensor1, *tensor2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_addmm(tensor *out__, tensor self, tensor mat1, tensor mat2) { - PROTECT( - auto outputs__ = torch::addmm(*self, *mat1, *mat2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_addmm_(tensor *out__, tensor self, tensor mat1, tensor mat2) { - PROTECT( - auto outputs__ = self->addmm_(*mat1, *mat2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_addmm_dtype(tensor *out__, tensor self, tensor mat1, tensor mat2, int out_dtype) { - PROTECT( - auto outputs__ = torch::addmm(*self, *mat1, *mat2, at::ScalarType(out_dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_addmm_dtype_out(tensor *out__, tensor out, tensor self, tensor mat1, tensor mat2, int out_dtype) { - PROTECT( - auto outputs__ = torch::addmm_out(*out, *self, *mat1, *mat2, at::ScalarType(out_dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_addmm_out(tensor *out__, tensor out, tensor self, tensor mat1, tensor mat2) { - PROTECT( - auto outputs__ = torch::addmm_out(*out, *self, *mat1, *mat2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_addmv(tensor *out__, tensor self, tensor mat, tensor vec) { - PROTECT( - auto outputs__ = torch::addmv(*self, *mat, *vec); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_addmv_(tensor *out__, tensor self, tensor mat, tensor vec) { - PROTECT( - auto outputs__ = torch::addmv_(*self, *mat, *vec); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_addmv_out(tensor *out__, tensor out, tensor self, tensor mat, tensor vec) { - PROTECT( - auto outputs__ = torch::addmv_out(*out, *self, *mat, *vec); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_addr(tensor *out__, tensor self, tensor vec1, tensor vec2) { - PROTECT( - auto outputs__ = torch::addr(*self, *vec1, *vec2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_addr_(tensor *out__, tensor self, tensor vec1, tensor vec2) { - PROTECT( - auto outputs__ = self->addr_(*vec1, *vec2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_addr_out(tensor *out__, tensor out, tensor self, tensor vec1, tensor vec2) { - PROTECT( - auto outputs__ = torch::addr_out(*out, *self, *vec1, *vec2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_adjoint(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::adjoint(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_affine_grid_generator(tensor *out__, tensor theta, int64_t *size_data, int size_len, int align_corners) { - PROTECT( - auto outputs__ = torch::affine_grid_generator(*theta, torch::IntArrayRef(size_data, size_len), (bool)align_corners); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_affine_grid_generator_backward(tensor *out__, tensor grad, int64_t *size_data, int size_len, int align_corners) { - PROTECT( - auto outputs__ = torch::affine_grid_generator_backward(*grad, torch::IntArrayRef(size_data, size_len), (bool)align_corners); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_affine_grid_generator_out(tensor *out__, tensor out, tensor theta, int64_t *size_data, int size_len, int align_corners) { - PROTECT( - auto outputs__ = torch::affine_grid_generator_out(*out, *theta, torch::IntArrayRef(size_data, size_len), (bool)align_corners); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_alias(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::alias(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_alias_copy(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::alias_copy(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_alias_copy_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::alias_copy_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_align_as(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->align_as(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -tensor *atg_align_tensors(tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::align_tensors(of_carray_tensor(tensors_data, tensors_len)); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg_all(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::all(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_all_all_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::all_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_all_dim(tensor *out__, tensor self, int64_t dim, int keepdim) { - PROTECT( - auto outputs__ = torch::all(*self, dim, (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_all_dims(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim) { - PROTECT( - auto outputs__ = torch::all(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_all_dims_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim) { - PROTECT( - auto outputs__ = torch::all_out(*out, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_all_out(tensor *out__, tensor out, tensor self, int64_t dim, int keepdim) { - PROTECT( - auto outputs__ = torch::all_out(*out, *self, dim, (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -int atg_allclose(tensor self, tensor other, double rtol, double atol, int equal_nan) { - PROTECT( - return torch::allclose(*self, *other, rtol, atol, (bool)equal_nan); - ) - return 0; -} - -void atg_alpha_dropout(tensor *out__, tensor input, double p, int train) { - PROTECT( - auto outputs__ = torch::alpha_dropout(*input, p, (bool)train); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_alpha_dropout_(tensor *out__, tensor self, double p, int train) { - PROTECT( - auto outputs__ = torch::alpha_dropout_(*self, p, (bool)train); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_amax(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim) { - PROTECT( - auto outputs__ = torch::amax(*self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_amax_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim) { - PROTECT( - auto outputs__ = torch::amax_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_amin(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim) { - PROTECT( - auto outputs__ = torch::amin(*self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_amin_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim) { - PROTECT( - auto outputs__ = torch::amin_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_aminmax(tensor *out__, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim) { - PROTECT( - auto outputs__ = torch::aminmax(*self, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_aminmax_out(tensor *out__, tensor min, tensor max, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim) { - PROTECT( - auto outputs__ = torch::aminmax_out(*min, *max, *self, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_angle(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::angle(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_angle_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::angle_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_any(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::any(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_any_all_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::any_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_any_dim(tensor *out__, tensor self, int64_t dim, int keepdim) { - PROTECT( - auto outputs__ = torch::any(*self, dim, (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_any_dims(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim) { - PROTECT( - auto outputs__ = torch::any(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_any_dims_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim) { - PROTECT( - auto outputs__ = torch::any_out(*out, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_any_out(tensor *out__, tensor out, tensor self, int64_t dim, int keepdim) { - PROTECT( - auto outputs__ = torch::any_out(*out, *self, dim, (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arange(tensor *out__, scalar end, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::arange(*end, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arange_start(tensor *out__, scalar start, scalar end, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::arange(*start, *end, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arange_start_step(tensor *out__, scalar start, scalar end, scalar step, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::arange(*start, *end, *step, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arccos(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::arccos(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arccos_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::arccos_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arccos_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::arccos_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arccosh(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::arccosh(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arccosh_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::arccosh_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arccosh_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::arccosh_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arcsin(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::arcsin(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arcsin_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::arcsin_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arcsin_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::arcsin_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arcsinh(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::arcsinh(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arcsinh_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::arcsinh_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arcsinh_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::arcsinh_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arctan(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::arctan(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arctan2(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::arctan2(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arctan2_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->arctan2_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arctan2_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::arctan2_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arctan_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::arctan_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arctan_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::arctan_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arctanh(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::arctanh(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arctanh_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::arctanh_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_arctanh_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::arctanh_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_argmax(tensor *out__, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim) { - PROTECT( - auto outputs__ = torch::argmax(*self, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_argmax_out(tensor *out__, tensor out, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim) { - PROTECT( - auto outputs__ = torch::argmax_out(*out, *self, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_argmin(tensor *out__, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim) { - PROTECT( - auto outputs__ = torch::argmin(*self, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_argmin_out(tensor *out__, tensor out, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim) { - PROTECT( - auto outputs__ = torch::argmin_out(*out, *self, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_argsort(tensor *out__, tensor self, int64_t dim, int descending) { - PROTECT( - auto outputs__ = torch::argsort(*self, dim, (bool)descending); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_argsort_stable(tensor *out__, tensor self, int stable, int64_t dim, int descending) { - PROTECT( - auto outputs__ = torch::argsort(*self, (bool)stable, dim, (bool)descending); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_argsort_stable_out(tensor *out__, tensor out, tensor self, int stable, int64_t dim, int descending) { - PROTECT( - auto outputs__ = torch::argsort_out(*out, *self, (bool)stable, dim, (bool)descending); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_argwhere(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::argwhere(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_as_strided(tensor *out__, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null) { - PROTECT( - auto outputs__ = torch::as_strided(*self, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len), storage_offset_null ? c10::nullopt : c10::optional(storage_offset_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_as_strided_(tensor *out__, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null) { - PROTECT( - auto outputs__ = torch::as_strided_(*self, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len), storage_offset_null ? c10::nullopt : c10::optional(storage_offset_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_as_strided_copy(tensor *out__, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null) { - PROTECT( - auto outputs__ = torch::as_strided_copy(*self, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len), storage_offset_null ? c10::nullopt : c10::optional(storage_offset_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_as_strided_copy_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null) { - PROTECT( - auto outputs__ = torch::as_strided_copy_out(*out, *self, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len), storage_offset_null ? c10::nullopt : c10::optional(storage_offset_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_as_strided_scatter(tensor *out__, tensor self, tensor src, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null) { - PROTECT( - auto outputs__ = torch::as_strided_scatter(*self, *src, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len), storage_offset_null ? c10::nullopt : c10::optional(storage_offset_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_as_strided_scatter_out(tensor *out__, tensor out, tensor self, tensor src, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null) { - PROTECT( - auto outputs__ = torch::as_strided_scatter_out(*out, *self, *src, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len), storage_offset_null ? c10::nullopt : c10::optional(storage_offset_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_asin(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::asin(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_asin_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::asin_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_asin_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::asin_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_asinh(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::asinh(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_asinh_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::asinh_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_asinh_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::asinh_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_atan(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::atan(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_atan2(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::atan2(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_atan2_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->atan2_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_atan2_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::atan2_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_atan_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::atan_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_atan_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::atan_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_atanh(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::atanh(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_atanh_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::atanh_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_atanh_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::atanh_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_atleast_1d(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::atleast_1d(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -tensor *atg_atleast_1d_sequence(tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::atleast_1d(of_carray_tensor(tensors_data, tensors_len)); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg_atleast_2d(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::atleast_2d(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -tensor *atg_atleast_2d_sequence(tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::atleast_2d(of_carray_tensor(tensors_data, tensors_len)); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg_atleast_3d(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::atleast_3d(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -tensor *atg_atleast_3d_sequence(tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::atleast_3d(of_carray_tensor(tensors_data, tensors_len)); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg_avg_pool1d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad) { - PROTECT( - auto outputs__ = torch::avg_pool1d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), (bool)ceil_mode, (bool)count_include_pad); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_avg_pool1d_out(tensor *out__, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad) { - PROTECT( - auto outputs__ = torch::avg_pool1d_out(*out, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), (bool)ceil_mode, (bool)count_include_pad); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_avg_pool2d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null) { - PROTECT( - auto outputs__ = torch::avg_pool2d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), (bool)ceil_mode, (bool)count_include_pad, divisor_override_null ? c10::nullopt : c10::optional(divisor_override_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_avg_pool2d_backward(tensor *out__, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null) { - PROTECT( - auto outputs__ = torch::avg_pool2d_backward(*grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), (bool)ceil_mode, (bool)count_include_pad, divisor_override_null ? c10::nullopt : c10::optional(divisor_override_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_avg_pool2d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null) { - PROTECT( - auto outputs__ = torch::avg_pool2d_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), (bool)ceil_mode, (bool)count_include_pad, divisor_override_null ? c10::nullopt : c10::optional(divisor_override_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_avg_pool2d_out(tensor *out__, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null) { - PROTECT( - auto outputs__ = torch::avg_pool2d_out(*out, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), (bool)ceil_mode, (bool)count_include_pad, divisor_override_null ? c10::nullopt : c10::optional(divisor_override_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_avg_pool3d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null) { - PROTECT( - auto outputs__ = torch::avg_pool3d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), (bool)ceil_mode, (bool)count_include_pad, divisor_override_null ? c10::nullopt : c10::optional(divisor_override_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_avg_pool3d_backward(tensor *out__, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null) { - PROTECT( - auto outputs__ = torch::avg_pool3d_backward(*grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), (bool)ceil_mode, (bool)count_include_pad, divisor_override_null ? c10::nullopt : c10::optional(divisor_override_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_avg_pool3d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null) { - PROTECT( - auto outputs__ = torch::avg_pool3d_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), (bool)ceil_mode, (bool)count_include_pad, divisor_override_null ? c10::nullopt : c10::optional(divisor_override_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_avg_pool3d_out(tensor *out__, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null) { - PROTECT( - auto outputs__ = torch::avg_pool3d_out(*out, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), (bool)ceil_mode, (bool)count_include_pad, divisor_override_null ? c10::nullopt : c10::optional(divisor_override_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_baddbmm(tensor *out__, tensor self, tensor batch1, tensor batch2, scalar beta, scalar alpha) { - PROTECT( - auto outputs__ = torch::baddbmm(*self, *batch1, *batch2, *beta, *alpha); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_baddbmm_(tensor *out__, tensor self, tensor batch1, tensor batch2) { - PROTECT( - auto outputs__ = self->baddbmm_(*batch1, *batch2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_baddbmm_dtype(tensor *out__, tensor self, tensor batch1, tensor batch2, int out_dtype, scalar beta, scalar alpha) { - PROTECT( - auto outputs__ = torch::baddbmm(*self, *batch1, *batch2, at::ScalarType(out_dtype), *beta, *alpha); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_baddbmm_dtype_out(tensor *out__, tensor out, tensor self, tensor batch1, tensor batch2, int out_dtype) { - PROTECT( - auto outputs__ = torch::baddbmm_out(*out, *self, *batch1, *batch2, at::ScalarType(out_dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_baddbmm_out(tensor *out__, tensor out, tensor self, tensor batch1, tensor batch2) { - PROTECT( - auto outputs__ = torch::baddbmm_out(*out, *self, *batch1, *batch2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bartlett_window(tensor *out__, int64_t window_length, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::bartlett_window(window_length, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bartlett_window_out(tensor *out__, tensor out, int64_t window_length) { - PROTECT( - auto outputs__ = torch::bartlett_window_out(*out, window_length); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bartlett_window_periodic(tensor *out__, int64_t window_length, int periodic, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::bartlett_window(window_length, (bool)periodic, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bartlett_window_periodic_out(tensor *out__, tensor out, int64_t window_length, int periodic) { - PROTECT( - auto outputs__ = torch::bartlett_window_out(*out, window_length, (bool)periodic); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_batch_norm(tensor *out__, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps, int cudnn_enabled) { - PROTECT( - auto outputs__ = torch::batch_norm(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (bool)training, momentum, eps, (bool)cudnn_enabled); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_batch_norm_backward_elemt(tensor *out__, tensor grad_out, tensor input, tensor mean, tensor invstd, tensor weight, tensor sum_dy, tensor sum_dy_xmu, tensor count) { - PROTECT( - auto outputs__ = torch::batch_norm_backward_elemt(*grad_out, *input, *mean, *invstd, (weight ? ::std::optional(*weight) : ::std::nullopt), *sum_dy, *sum_dy_xmu, *count); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_batch_norm_backward_elemt_out(tensor *out__, tensor out, tensor grad_out, tensor input, tensor mean, tensor invstd, tensor weight, tensor sum_dy, tensor sum_dy_xmu, tensor count) { - PROTECT( - auto outputs__ = torch::batch_norm_backward_elemt_out(*out, *grad_out, *input, *mean, *invstd, (weight ? ::std::optional(*weight) : ::std::nullopt), *sum_dy, *sum_dy_xmu, *count); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_batch_norm_backward_reduce(tensor *out__, tensor grad_out, tensor input, tensor mean, tensor invstd, tensor weight, int input_g, int weight_g, int bias_g) { - PROTECT( - auto outputs__ = torch::batch_norm_backward_reduce(*grad_out, *input, *mean, *invstd, (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)input_g, (bool)weight_g, (bool)bias_g); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg_batch_norm_backward_reduce_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor out3, tensor grad_out, tensor input, tensor mean, tensor invstd, tensor weight, int input_g, int weight_g, int bias_g) { - PROTECT( - auto outputs__ = torch::batch_norm_backward_reduce_out(*out0, *out1, *out2, *out3, *grad_out, *input, *mean, *invstd, (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)input_g, (bool)weight_g, (bool)bias_g); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg_batch_norm_elemt(tensor *out__, tensor input, tensor weight, tensor bias, tensor mean, tensor invstd, double eps) { - PROTECT( - auto outputs__ = torch::batch_norm_elemt(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *mean, *invstd, eps); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_batch_norm_elemt_out(tensor *out__, tensor out, tensor input, tensor weight, tensor bias, tensor mean, tensor invstd, double eps) { - PROTECT( - auto outputs__ = torch::batch_norm_elemt_out(*out, *input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *mean, *invstd, eps); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_batch_norm_gather_stats(tensor *out__, tensor input, tensor mean, tensor invstd, tensor running_mean, tensor running_var, double momentum, double eps, int64_t count) { - PROTECT( - auto outputs__ = torch::batch_norm_gather_stats(*input, *mean, *invstd, (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), momentum, eps, count); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_batch_norm_gather_stats_out(tensor *out__, tensor out0, tensor out1, tensor input, tensor mean, tensor invstd, tensor running_mean, tensor running_var, double momentum, double eps, int64_t count) { - PROTECT( - auto outputs__ = torch::batch_norm_gather_stats_out(*out0, *out1, *input, *mean, *invstd, (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), momentum, eps, count); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_batch_norm_gather_stats_with_counts(tensor *out__, tensor input, tensor mean, tensor invstd, tensor running_mean, tensor running_var, double momentum, double eps, tensor counts) { - PROTECT( - auto outputs__ = torch::batch_norm_gather_stats_with_counts(*input, *mean, *invstd, (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), momentum, eps, *counts); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_batch_norm_gather_stats_with_counts_out(tensor *out__, tensor out0, tensor out1, tensor input, tensor mean, tensor invstd, tensor running_mean, tensor running_var, double momentum, double eps, tensor counts) { - PROTECT( - auto outputs__ = torch::batch_norm_gather_stats_with_counts_out(*out0, *out1, *input, *mean, *invstd, (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), momentum, eps, *counts); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_batch_norm_stats(tensor *out__, tensor input, double eps) { - PROTECT( - auto outputs__ = torch::batch_norm_stats(*input, eps); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_batch_norm_stats_out(tensor *out__, tensor out0, tensor out1, tensor input, double eps) { - PROTECT( - auto outputs__ = torch::batch_norm_stats_out(*out0, *out1, *input, eps); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_batch_norm_update_stats(tensor *out__, tensor input, tensor running_mean, tensor running_var, double momentum) { - PROTECT( - auto outputs__ = torch::batch_norm_update_stats(*input, (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), momentum); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_batch_norm_update_stats_out(tensor *out__, tensor out0, tensor out1, tensor input, tensor running_mean, tensor running_var, double momentum) { - PROTECT( - auto outputs__ = torch::batch_norm_update_stats_out(*out0, *out1, *input, (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), momentum); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_bernoulli(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::bernoulli(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bernoulli_(tensor *out__, tensor self, tensor p) { - PROTECT( - auto outputs__ = self->bernoulli_(*p); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bernoulli_float_(tensor *out__, tensor self, double p) { - PROTECT( - auto outputs__ = self->bernoulli_(p); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bernoulli_p(tensor *out__, tensor self, double p) { - PROTECT( - auto outputs__ = torch::bernoulli(*self, p); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bernoulli_tensor(tensor *out__, tensor self, tensor p) { - PROTECT( - auto outputs__ = torch::bernoulli(*self, *p); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bilinear(tensor *out__, tensor input1, tensor input2, tensor weight, tensor bias) { - PROTECT( - auto outputs__ = torch::bilinear(*input1, *input2, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_binary_cross_entropy(tensor *out__, tensor self, tensor target, tensor weight, int64_t reduction) { - PROTECT( - auto outputs__ = torch::binary_cross_entropy(*self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_binary_cross_entropy_backward(tensor *out__, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction) { - PROTECT( - auto outputs__ = torch::binary_cross_entropy_backward(*grad_output, *self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_binary_cross_entropy_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction) { - PROTECT( - auto outputs__ = torch::binary_cross_entropy_backward_out(*grad_input, *grad_output, *self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_binary_cross_entropy_out(tensor *out__, tensor out, tensor self, tensor target, tensor weight, int64_t reduction) { - PROTECT( - auto outputs__ = torch::binary_cross_entropy_out(*out, *self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_binary_cross_entropy_with_logits(tensor *out__, tensor self, tensor target, tensor weight, tensor pos_weight, int64_t reduction) { - PROTECT( - auto outputs__ = torch::binary_cross_entropy_with_logits(*self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), (pos_weight ? ::std::optional(*pos_weight) : ::std::nullopt), reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_binary_cross_entropy_with_logits_out(tensor *out__, tensor out, tensor self, tensor target, tensor weight, tensor pos_weight, int64_t reduction) { - PROTECT( - auto outputs__ = torch::binary_cross_entropy_with_logits_out(*out, *self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), (pos_weight ? ::std::optional(*pos_weight) : ::std::nullopt), reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bincount(tensor *out__, tensor self, tensor weights, int64_t minlength) { - PROTECT( - auto outputs__ = torch::bincount(*self, (weights ? ::std::optional(*weights) : ::std::nullopt), minlength); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bincount_out(tensor *out__, tensor out, tensor self, tensor weights, int64_t minlength) { - PROTECT( - auto outputs__ = torch::bincount_out(*out, *self, (weights ? ::std::optional(*weights) : ::std::nullopt), minlength); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_binomial(tensor *out__, tensor count, tensor prob) { - PROTECT( - auto outputs__ = torch::binomial(*count, *prob); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_binomial_out(tensor *out__, tensor out, tensor count, tensor prob) { - PROTECT( - auto outputs__ = torch::binomial_out(*out, *count, *prob); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_and(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::bitwise_and(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_and_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->bitwise_and_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_and_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::bitwise_and_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_and_scalar_tensor(tensor *out__, scalar self_scalar, tensor other) { - PROTECT( - auto outputs__ = torch::bitwise_and(*self_scalar, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_and_scalar_tensor_out(tensor *out__, tensor out, scalar self_scalar, tensor other) { - PROTECT( - auto outputs__ = torch::bitwise_and_out(*out, *self_scalar, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_and_tensor(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::bitwise_and(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_and_tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->bitwise_and_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_and_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::bitwise_and_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_left_shift(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::bitwise_left_shift(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_left_shift_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->bitwise_left_shift_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_left_shift_scalar_tensor(tensor *out__, scalar self_scalar, tensor other) { - PROTECT( - auto outputs__ = torch::bitwise_left_shift(*self_scalar, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_left_shift_scalar_tensor_out(tensor *out__, tensor out, scalar self_scalar, tensor other) { - PROTECT( - auto outputs__ = torch::bitwise_left_shift_out(*out, *self_scalar, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_left_shift_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::bitwise_left_shift_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_left_shift_tensor_scalar(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::bitwise_left_shift(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_left_shift_tensor_scalar_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->bitwise_left_shift_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_left_shift_tensor_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::bitwise_left_shift_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_not(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::bitwise_not(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_not_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->bitwise_not_(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_not_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::bitwise_not_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_or(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::bitwise_or(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_or_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->bitwise_or_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_or_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::bitwise_or_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_or_scalar_tensor(tensor *out__, scalar self_scalar, tensor other) { - PROTECT( - auto outputs__ = torch::bitwise_or(*self_scalar, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_or_scalar_tensor_out(tensor *out__, tensor out, scalar self_scalar, tensor other) { - PROTECT( - auto outputs__ = torch::bitwise_or_out(*out, *self_scalar, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_or_tensor(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::bitwise_or(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_or_tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->bitwise_or_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_or_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::bitwise_or_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_right_shift(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::bitwise_right_shift(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_right_shift_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->bitwise_right_shift_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_right_shift_scalar_tensor(tensor *out__, scalar self_scalar, tensor other) { - PROTECT( - auto outputs__ = torch::bitwise_right_shift(*self_scalar, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_right_shift_scalar_tensor_out(tensor *out__, tensor out, scalar self_scalar, tensor other) { - PROTECT( - auto outputs__ = torch::bitwise_right_shift_out(*out, *self_scalar, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_right_shift_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::bitwise_right_shift_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_right_shift_tensor_scalar(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::bitwise_right_shift(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_right_shift_tensor_scalar_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->bitwise_right_shift_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_right_shift_tensor_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::bitwise_right_shift_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_xor(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::bitwise_xor(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_xor_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->bitwise_xor_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_xor_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::bitwise_xor_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_xor_scalar_tensor(tensor *out__, scalar self_scalar, tensor other) { - PROTECT( - auto outputs__ = torch::bitwise_xor(*self_scalar, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_xor_scalar_tensor_out(tensor *out__, tensor out, scalar self_scalar, tensor other) { - PROTECT( - auto outputs__ = torch::bitwise_xor_out(*out, *self_scalar, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_xor_tensor(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::bitwise_xor(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_xor_tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->bitwise_xor_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bitwise_xor_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::bitwise_xor_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_blackman_window(tensor *out__, int64_t window_length, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::blackman_window(window_length, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_blackman_window_out(tensor *out__, tensor out, int64_t window_length) { - PROTECT( - auto outputs__ = torch::blackman_window_out(*out, window_length); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_blackman_window_periodic(tensor *out__, int64_t window_length, int periodic, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::blackman_window(window_length, (bool)periodic, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_blackman_window_periodic_out(tensor *out__, tensor out, int64_t window_length, int periodic) { - PROTECT( - auto outputs__ = torch::blackman_window_out(*out, window_length, (bool)periodic); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_block_diag(tensor *out__, tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::block_diag(of_carray_tensor(tensors_data, tensors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_block_diag_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::block_diag_out(*out, of_carray_tensor(tensors_data, tensors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bmm(tensor *out__, tensor self, tensor mat2) { - PROTECT( - auto outputs__ = torch::bmm(*self, *mat2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bmm_dtype(tensor *out__, tensor self, tensor mat2, int out_dtype) { - PROTECT( - auto outputs__ = torch::bmm(*self, *mat2, at::ScalarType(out_dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bmm_dtype_out(tensor *out__, tensor out, tensor self, tensor mat2, int out_dtype) { - PROTECT( - auto outputs__ = torch::bmm_out(*out, *self, *mat2, at::ScalarType(out_dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bmm_out(tensor *out__, tensor out, tensor self, tensor mat2) { - PROTECT( - auto outputs__ = torch::bmm_out(*out, *self, *mat2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -tensor *atg_broadcast_tensors(tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::broadcast_tensors(of_carray_tensor(tensors_data, tensors_len)); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg_broadcast_to(tensor *out__, tensor self, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::broadcast_to(*self, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bucketize(tensor *out__, tensor self, tensor boundaries, int out_int32, int right) { - PROTECT( - auto outputs__ = torch::bucketize(*self, *boundaries, (bool)out_int32, (bool)right); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bucketize_scalar(tensor *out__, scalar self_scalar, tensor boundaries, int out_int32, int right) { - PROTECT( - auto outputs__ = torch::bucketize(*self_scalar, *boundaries, (bool)out_int32, (bool)right); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bucketize_scalar_out(tensor *out__, tensor out, scalar self_scalar, tensor boundaries, int out_int32, int right) { - PROTECT( - auto outputs__ = torch::bucketize_out(*out, *self_scalar, *boundaries, (bool)out_int32, (bool)right); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_bucketize_tensor_out(tensor *out__, tensor out, tensor self, tensor boundaries, int out_int32, int right) { - PROTECT( - auto outputs__ = torch::bucketize_out(*out, *self, *boundaries, (bool)out_int32, (bool)right); - out__[0] = new torch::Tensor(outputs__); - ) -} - -int atg_can_cast(int from_, int to) { - PROTECT( - return torch::can_cast(at::ScalarType(from_), at::ScalarType(to)); - ) - return 0; -} - -void atg_cartesian_prod(tensor *out__, tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::cartesian_prod(of_carray_tensor(tensors_data, tensors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cat(tensor *out__, tensor *tensors_data, int tensors_len, int64_t dim) { - PROTECT( - auto outputs__ = torch::cat(of_carray_tensor(tensors_data, tensors_len), dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cat_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len, int64_t dim) { - PROTECT( - auto outputs__ = torch::cat_out(*out, of_carray_tensor(tensors_data, tensors_len), dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cauchy(tensor *out__, tensor self, double median, double sigma) { - PROTECT( - auto outputs__ = torch::cauchy(*self, median, sigma); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cauchy_(tensor *out__, tensor self, double median, double sigma) { - PROTECT( - auto outputs__ = self->cauchy_(median, sigma); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cauchy_out(tensor *out__, tensor out, tensor self, double median, double sigma) { - PROTECT( - auto outputs__ = torch::cauchy_out(*out, *self, median, sigma); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ccol_indices(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->ccol_indices(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ccol_indices_copy(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::ccol_indices_copy(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ccol_indices_copy_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::ccol_indices_copy_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cdist(tensor *out__, tensor x1, tensor x2, double p, int64_t compute_mode_v, uint8_t compute_mode_null) { - PROTECT( - auto outputs__ = torch::cdist(*x1, *x2, p, compute_mode_null ? c10::nullopt : c10::optional(compute_mode_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ceil(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::ceil(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ceil_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::ceil_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ceil_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::ceil_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_celu(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::celu(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_celu_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::celu_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_celu_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::celu_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_chain_matmul(tensor *out__, tensor *matrices_data, int matrices_len) { - PROTECT( - auto outputs__ = torch::chain_matmul(of_carray_tensor(matrices_data, matrices_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_chain_matmul_out(tensor *out__, tensor out, tensor *matrices_data, int matrices_len) { - PROTECT( - auto outputs__ = torch::chain_matmul_out(*out, of_carray_tensor(matrices_data, matrices_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_chalf(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->chalf(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_channel_shuffle(tensor *out__, tensor self, int64_t groups) { - PROTECT( - auto outputs__ = torch::channel_shuffle(*self, groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_channel_shuffle_out(tensor *out__, tensor out, tensor self, int64_t groups) { - PROTECT( - auto outputs__ = torch::channel_shuffle_out(*out, *self, groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cholesky(tensor *out__, tensor self, int upper) { - PROTECT( - auto outputs__ = torch::cholesky(*self, (bool)upper); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cholesky_inverse(tensor *out__, tensor self, int upper) { - PROTECT( - auto outputs__ = torch::cholesky_inverse(*self, (bool)upper); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cholesky_inverse_out(tensor *out__, tensor out, tensor self, int upper) { - PROTECT( - auto outputs__ = torch::cholesky_inverse_out(*out, *self, (bool)upper); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cholesky_out(tensor *out__, tensor out, tensor self, int upper) { - PROTECT( - auto outputs__ = torch::cholesky_out(*out, *self, (bool)upper); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cholesky_solve(tensor *out__, tensor self, tensor input2, int upper) { - PROTECT( - auto outputs__ = torch::cholesky_solve(*self, *input2, (bool)upper); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cholesky_solve_out(tensor *out__, tensor out, tensor self, tensor input2, int upper) { - PROTECT( - auto outputs__ = torch::cholesky_solve_out(*out, *self, *input2, (bool)upper); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_choose_qparams_optimized(tensor *out__, tensor input, int64_t numel, int64_t n_bins, double ratio, int64_t bit_width) { - PROTECT( - auto outputs__ = torch::choose_qparams_optimized(*input, numel, n_bins, ratio, bit_width); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -tensor *atg_chunk(tensor self, int64_t chunks, int64_t dim) { - PROTECT( - auto outputs__ = torch::chunk(*self, chunks, dim); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg_clamp(tensor *out__, tensor self, scalar min, scalar max) { - PROTECT( - auto outputs__ = torch::clamp(*self, *min, *max); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clamp_(tensor *out__, tensor self, scalar min, scalar max) { - PROTECT( - auto outputs__ = torch::clamp_(*self, *min, *max); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clamp_max(tensor *out__, tensor self, scalar max) { - PROTECT( - auto outputs__ = torch::clamp_max(*self, *max); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clamp_max_(tensor *out__, tensor self, scalar max) { - PROTECT( - auto outputs__ = torch::clamp_max_(*self, *max); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clamp_max_out(tensor *out__, tensor out, tensor self, scalar max) { - PROTECT( - auto outputs__ = torch::clamp_max_out(*out, *self, *max); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clamp_max_tensor(tensor *out__, tensor self, tensor max) { - PROTECT( - auto outputs__ = torch::clamp_max(*self, *max); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clamp_max_tensor_(tensor *out__, tensor self, tensor max) { - PROTECT( - auto outputs__ = torch::clamp_max_(*self, *max); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clamp_max_tensor_out(tensor *out__, tensor out, tensor self, tensor max) { - PROTECT( - auto outputs__ = torch::clamp_max_out(*out, *self, *max); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clamp_min(tensor *out__, tensor self, scalar min) { - PROTECT( - auto outputs__ = torch::clamp_min(*self, *min); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clamp_min_(tensor *out__, tensor self, scalar min) { - PROTECT( - auto outputs__ = torch::clamp_min_(*self, *min); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clamp_min_out(tensor *out__, tensor out, tensor self, scalar min) { - PROTECT( - auto outputs__ = torch::clamp_min_out(*out, *self, *min); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clamp_min_tensor(tensor *out__, tensor self, tensor min) { - PROTECT( - auto outputs__ = torch::clamp_min(*self, *min); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clamp_min_tensor_(tensor *out__, tensor self, tensor min) { - PROTECT( - auto outputs__ = torch::clamp_min_(*self, *min); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clamp_min_tensor_out(tensor *out__, tensor out, tensor self, tensor min) { - PROTECT( - auto outputs__ = torch::clamp_min_out(*out, *self, *min); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clamp_out(tensor *out__, tensor out, tensor self, scalar min, scalar max) { - PROTECT( - auto outputs__ = torch::clamp_out(*out, *self, *min, *max); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clamp_tensor(tensor *out__, tensor self, tensor min, tensor max) { - PROTECT( - auto outputs__ = torch::clamp(*self, (min ? ::std::optional(*min) : ::std::nullopt), (max ? ::std::optional(*max) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clamp_tensor_(tensor *out__, tensor self, tensor min, tensor max) { - PROTECT( - auto outputs__ = torch::clamp_(*self, (min ? ::std::optional(*min) : ::std::nullopt), (max ? ::std::optional(*max) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clamp_tensor_out(tensor *out__, tensor out, tensor self, tensor min, tensor max) { - PROTECT( - auto outputs__ = torch::clamp_out(*out, *self, (min ? ::std::optional(*min) : ::std::nullopt), (max ? ::std::optional(*max) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clip(tensor *out__, tensor self, scalar min, scalar max) { - PROTECT( - auto outputs__ = torch::clip(*self, *min, *max); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clip_(tensor *out__, tensor self, scalar min, scalar max) { - PROTECT( - auto outputs__ = torch::clip_(*self, *min, *max); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clip_out(tensor *out__, tensor out, tensor self, scalar min, scalar max) { - PROTECT( - auto outputs__ = torch::clip_out(*out, *self, *min, *max); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clip_tensor(tensor *out__, tensor self, tensor min, tensor max) { - PROTECT( - auto outputs__ = torch::clip(*self, (min ? ::std::optional(*min) : ::std::nullopt), (max ? ::std::optional(*max) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clip_tensor_(tensor *out__, tensor self, tensor min, tensor max) { - PROTECT( - auto outputs__ = torch::clip_(*self, (min ? ::std::optional(*min) : ::std::nullopt), (max ? ::std::optional(*max) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clip_tensor_out(tensor *out__, tensor out, tensor self, tensor min, tensor max) { - PROTECT( - auto outputs__ = torch::clip_out(*out, *self, (min ? ::std::optional(*min) : ::std::nullopt), (max ? ::std::optional(*max) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_clone(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::clone_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_coalesce(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->coalesce(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_col2im(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, int64_t *kernel_size_data, int kernel_size_len, int64_t *dilation_data, int dilation_len, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len) { - PROTECT( - auto outputs__ = torch::col2im(*self, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(dilation_data, dilation_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_col2im_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int64_t *kernel_size_data, int kernel_size_len, int64_t *dilation_data, int dilation_len, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len) { - PROTECT( - auto outputs__ = torch::col2im_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(dilation_data, dilation_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_col_indices(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->col_indices(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_col_indices_copy(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::col_indices_copy(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_col_indices_copy_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::col_indices_copy_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_column_stack(tensor *out__, tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::column_stack(of_carray_tensor(tensors_data, tensors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_column_stack_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::column_stack_out(*out, of_carray_tensor(tensors_data, tensors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_combinations(tensor *out__, tensor self, int64_t r, int with_replacement) { - PROTECT( - auto outputs__ = torch::combinations(*self, r, (bool)with_replacement); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_complex(tensor *out__, tensor real, tensor imag) { - PROTECT( - auto outputs__ = torch::complex(*real, *imag); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_complex_out(tensor *out__, tensor out, tensor real, tensor imag) { - PROTECT( - auto outputs__ = torch::complex_out(*out, *real, *imag); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_concat(tensor *out__, tensor *tensors_data, int tensors_len, int64_t dim) { - PROTECT( - auto outputs__ = torch::concat(of_carray_tensor(tensors_data, tensors_len), dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_concat_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len, int64_t dim) { - PROTECT( - auto outputs__ = torch::concat_out(*out, of_carray_tensor(tensors_data, tensors_len), dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_concatenate(tensor *out__, tensor *tensors_data, int tensors_len, int64_t dim) { - PROTECT( - auto outputs__ = torch::concatenate(of_carray_tensor(tensors_data, tensors_len), dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_concatenate_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len, int64_t dim) { - PROTECT( - auto outputs__ = torch::concatenate_out(*out, of_carray_tensor(tensors_data, tensors_len), dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_conj(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::conj(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_conj_physical(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::conj_physical(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_conj_physical_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::conj_physical_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_conj_physical_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::conj_physical_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_constant_pad_nd(tensor *out__, tensor self, int64_t *pad_data, int pad_len) { - PROTECT( - auto outputs__ = torch::constant_pad_nd(*self, torch::IntArrayRef(pad_data, pad_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_constant_pad_nd_out(tensor *out__, tensor out, tensor self, int64_t *pad_data, int pad_len) { - PROTECT( - auto outputs__ = torch::constant_pad_nd_out(*out, *self, torch::IntArrayRef(pad_data, pad_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_contiguous(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->contiguous(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_conv1d(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::conv1d(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_conv1d_padding(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, char* padding_ptr, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::conv1d(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), std::string(padding_ptr, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_conv2d(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::conv2d(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_conv2d_padding(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, char* padding_ptr, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::conv2d(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), std::string(padding_ptr, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_conv3d(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::conv3d(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_conv3d_padding(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, char* padding_ptr, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::conv3d(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), std::string(padding_ptr, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_conv_depthwise3d(tensor *out__, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len) { - PROTECT( - auto outputs__ = torch::conv_depthwise3d(*self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_conv_depthwise3d_out(tensor *out__, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len) { - PROTECT( - auto outputs__ = torch::conv_depthwise3d_out(*out, *self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_conv_tbc(tensor *out__, tensor self, tensor weight, tensor bias, int64_t pad) { - PROTECT( - auto outputs__ = torch::conv_tbc(*self, *weight, *bias, pad); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_conv_tbc_backward(tensor *out__, tensor self, tensor input, tensor weight, tensor bias, int64_t pad) { - PROTECT( - auto outputs__ = torch::conv_tbc_backward(*self, *input, *weight, *bias, pad); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_conv_tbc_out(tensor *out__, tensor out, tensor self, tensor weight, tensor bias, int64_t pad) { - PROTECT( - auto outputs__ = torch::conv_tbc_out(*out, *self, *weight, *bias, pad); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_conv_transpose1d(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t groups, int64_t *dilation_data, int dilation_len) { - PROTECT( - auto outputs__ = torch::conv_transpose1d(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), groups, torch::IntArrayRef(dilation_data, dilation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_conv_transpose2d(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t groups, int64_t *dilation_data, int dilation_len) { - PROTECT( - auto outputs__ = torch::conv_transpose2d(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), groups, torch::IntArrayRef(dilation_data, dilation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_conv_transpose3d(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t groups, int64_t *dilation_data, int dilation_len) { - PROTECT( - auto outputs__ = torch::conv_transpose3d(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), groups, torch::IntArrayRef(dilation_data, dilation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_convolution(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::convolution(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)transposed, torch::IntArrayRef(output_padding_data, output_padding_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_convolution_out(tensor *out__, tensor out, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::convolution_out(*out, *input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)transposed, torch::IntArrayRef(output_padding_data, output_padding_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_convolution_overrideable(tensor *out__, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::convolution_overrideable(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)transposed, torch::IntArrayRef(output_padding_data, output_padding_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_convolution_overrideable_out(tensor *out__, tensor out, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::convolution_overrideable_out(*out, *input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)transposed, torch::IntArrayRef(output_padding_data, output_padding_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_copy_sparse_to_sparse(tensor *out__, tensor self, tensor src, int non_blocking) { - PROTECT( - auto outputs__ = torch::copy_sparse_to_sparse(*self, *src, (bool)non_blocking); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_copy_sparse_to_sparse_(tensor *out__, tensor self, tensor src, int non_blocking) { - PROTECT( - auto outputs__ = torch::copy_sparse_to_sparse_(*self, *src, (bool)non_blocking); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_copy_sparse_to_sparse_out(tensor *out__, tensor out, tensor self, tensor src, int non_blocking) { - PROTECT( - auto outputs__ = torch::copy_sparse_to_sparse_out(*out, *self, *src, (bool)non_blocking); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_copysign(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::copysign(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_copysign_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->copysign_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_copysign_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::copysign_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_copysign_scalar(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::copysign(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_copysign_scalar_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->copysign_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_copysign_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::copysign_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_corrcoef(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::corrcoef(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cos(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::cos(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cos_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::cos_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cos_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::cos_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cosh(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::cosh(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cosh_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::cosh_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cosh_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::cosh_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cosine_embedding_loss(tensor *out__, tensor input1, tensor input2, tensor target, double margin, int64_t reduction) { - PROTECT( - auto outputs__ = torch::cosine_embedding_loss(*input1, *input2, *target, margin, reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cosine_similarity(tensor *out__, tensor x1, tensor x2, int64_t dim, double eps) { - PROTECT( - auto outputs__ = torch::cosine_similarity(*x1, *x2, dim, eps); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_count_nonzero(tensor *out__, tensor self, int64_t dim_v, uint8_t dim_null) { - PROTECT( - auto outputs__ = torch::count_nonzero(*self, dim_null ? c10::nullopt : c10::optional(dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_count_nonzero_dim_intlist(tensor *out__, tensor self, int64_t *dim_data, int dim_len) { - PROTECT( - auto outputs__ = torch::count_nonzero(*self, torch::IntArrayRef(dim_data, dim_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_count_nonzero_dim_intlist_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len) { - PROTECT( - auto outputs__ = torch::count_nonzero_out(*out, *self, torch::IntArrayRef(dim_data, dim_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_count_nonzero_out(tensor *out__, tensor out, tensor self, int64_t dim_v, uint8_t dim_null) { - PROTECT( - auto outputs__ = torch::count_nonzero_out(*out, *self, dim_null ? c10::nullopt : c10::optional(dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cov(tensor *out__, tensor self, int64_t correction, tensor fweights, tensor aweights) { - PROTECT( - auto outputs__ = torch::cov(*self, correction, (fweights ? ::std::optional(*fweights) : ::std::nullopt), (aweights ? ::std::optional(*aweights) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cross(tensor *out__, tensor self, tensor other, int64_t dim_v, uint8_t dim_null) { - PROTECT( - auto outputs__ = torch::cross(*self, *other, dim_null ? c10::nullopt : c10::optional(dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cross_entropy_loss(tensor *out__, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index, double label_smoothing) { - PROTECT( - auto outputs__ = torch::cross_entropy_loss(*self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction, ignore_index, label_smoothing); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cross_out(tensor *out__, tensor out, tensor self, tensor other, int64_t dim_v, uint8_t dim_null) { - PROTECT( - auto outputs__ = torch::cross_out(*out, *self, *other, dim_null ? c10::nullopt : c10::optional(dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_crow_indices(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->crow_indices(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_crow_indices_copy(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::crow_indices_copy(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_crow_indices_copy_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::crow_indices_copy_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ctc_loss(tensor *out__, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int64_t reduction, int zero_infinity) { - PROTECT( - auto outputs__ = torch::ctc_loss(*log_probs, *targets, torch::IntArrayRef(input_lengths_data, input_lengths_len), torch::IntArrayRef(target_lengths_data, target_lengths_len), blank, reduction, (bool)zero_infinity); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ctc_loss_tensor(tensor *out__, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank, int64_t reduction, int zero_infinity) { - PROTECT( - auto outputs__ = torch::ctc_loss(*log_probs, *targets, *input_lengths, *target_lengths, blank, reduction, (bool)zero_infinity); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cudnn_affine_grid_generator(tensor *out__, tensor theta, int64_t n, int64_t C, int64_t H, int64_t W) { - PROTECT( - auto outputs__ = torch::cudnn_affine_grid_generator(*theta, n, C, H, W); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cudnn_affine_grid_generator_backward(tensor *out__, tensor grad, int64_t n, int64_t C, int64_t H, int64_t W) { - PROTECT( - auto outputs__ = torch::cudnn_affine_grid_generator_backward(*grad, n, C, H, W); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cudnn_affine_grid_generator_backward_out(tensor *out__, tensor out, tensor grad, int64_t n, int64_t C, int64_t H, int64_t W) { - PROTECT( - auto outputs__ = torch::cudnn_affine_grid_generator_backward_out(*out, *grad, n, C, H, W); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cudnn_affine_grid_generator_out(tensor *out__, tensor out, tensor theta, int64_t n, int64_t C, int64_t H, int64_t W) { - PROTECT( - auto outputs__ = torch::cudnn_affine_grid_generator_out(*out, *theta, n, C, H, W); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cudnn_batch_norm(tensor *out__, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double exponential_average_factor, double epsilon) { - PROTECT( - auto outputs__ = torch::cudnn_batch_norm(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (bool)training, exponential_average_factor, epsilon); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg_cudnn_batch_norm_backward(tensor *out__, tensor input, tensor grad_output, tensor weight, tensor running_mean, tensor running_var, tensor save_mean, tensor save_var, double epsilon, tensor reserveSpace) { - PROTECT( - auto outputs__ = torch::cudnn_batch_norm_backward(*input, *grad_output, *weight, (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (save_mean ? ::std::optional(*save_mean) : ::std::nullopt), (save_var ? ::std::optional(*save_var) : ::std::nullopt), epsilon, *reserveSpace); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_cudnn_batch_norm_backward_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor input, tensor grad_output, tensor weight, tensor running_mean, tensor running_var, tensor save_mean, tensor save_var, double epsilon, tensor reserveSpace) { - PROTECT( - auto outputs__ = torch::cudnn_batch_norm_backward_out(*out0, *out1, *out2, *input, *grad_output, *weight, (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (save_mean ? ::std::optional(*save_mean) : ::std::nullopt), (save_var ? ::std::optional(*save_var) : ::std::nullopt), epsilon, *reserveSpace); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_cudnn_batch_norm_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor out3, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double exponential_average_factor, double epsilon) { - PROTECT( - auto outputs__ = torch::cudnn_batch_norm_out(*out0, *out1, *out2, *out3, *input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (bool)training, exponential_average_factor, epsilon); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg_cudnn_convolution(tensor *out__, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic, int allow_tf32) { - PROTECT( - auto outputs__ = torch::cudnn_convolution(*self, *weight, torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, (bool)benchmark, (bool)deterministic, (bool)allow_tf32); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cudnn_convolution_add_relu(tensor *out__, tensor self, tensor weight, tensor z, scalar alpha, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::cudnn_convolution_add_relu(*self, *weight, *z, *alpha, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cudnn_convolution_add_relu_out(tensor *out__, tensor out, tensor self, tensor weight, tensor z, scalar alpha, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::cudnn_convolution_add_relu_out(*out, *self, *weight, *z, *alpha, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cudnn_convolution_out(tensor *out__, tensor out, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic, int allow_tf32) { - PROTECT( - auto outputs__ = torch::cudnn_convolution_out(*out, *self, *weight, torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, (bool)benchmark, (bool)deterministic, (bool)allow_tf32); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cudnn_convolution_relu(tensor *out__, tensor self, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::cudnn_convolution_relu(*self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cudnn_convolution_relu_out(tensor *out__, tensor out, tensor self, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::cudnn_convolution_relu_out(*out, *self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cudnn_convolution_transpose(tensor *out__, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic, int allow_tf32) { - PROTECT( - auto outputs__ = torch::cudnn_convolution_transpose(*self, *weight, torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, (bool)benchmark, (bool)deterministic, (bool)allow_tf32); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cudnn_convolution_transpose_out(tensor *out__, tensor out, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic, int allow_tf32) { - PROTECT( - auto outputs__ = torch::cudnn_convolution_transpose_out(*out, *self, *weight, torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, (bool)benchmark, (bool)deterministic, (bool)allow_tf32); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cudnn_grid_sampler(tensor *out__, tensor self, tensor grid) { - PROTECT( - auto outputs__ = torch::cudnn_grid_sampler(*self, *grid); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cudnn_grid_sampler_backward(tensor *out__, tensor self, tensor grid, tensor grad_output) { - PROTECT( - auto outputs__ = torch::cudnn_grid_sampler_backward(*self, *grid, *grad_output); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_cudnn_grid_sampler_backward_out(tensor *out__, tensor out0, tensor out1, tensor self, tensor grid, tensor grad_output) { - PROTECT( - auto outputs__ = torch::cudnn_grid_sampler_backward_out(*out0, *out1, *self, *grid, *grad_output); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_cudnn_grid_sampler_out(tensor *out__, tensor out, tensor self, tensor grid) { - PROTECT( - auto outputs__ = torch::cudnn_grid_sampler_out(*out, *self, *grid); - out__[0] = new torch::Tensor(outputs__); - ) -} - -int atg_cudnn_is_acceptable(tensor self) { - PROTECT( - return torch::cudnn_is_acceptable(*self); - ) - return 0; -} - -void atg_cummax(tensor *out__, tensor self, int64_t dim) { - PROTECT( - auto outputs__ = torch::cummax(*self, dim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_cummax_out(tensor *out__, tensor values, tensor indices, tensor self, int64_t dim) { - PROTECT( - auto outputs__ = torch::cummax_out(*values, *indices, *self, dim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_cummaxmin_backward(tensor *out__, tensor grad, tensor input, tensor indices, int64_t dim) { - PROTECT( - auto outputs__ = torch::cummaxmin_backward(*grad, *input, *indices, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cummin(tensor *out__, tensor self, int64_t dim) { - PROTECT( - auto outputs__ = torch::cummin(*self, dim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_cummin_out(tensor *out__, tensor values, tensor indices, tensor self, int64_t dim) { - PROTECT( - auto outputs__ = torch::cummin_out(*values, *indices, *self, dim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_cumprod(tensor *out__, tensor self, int64_t dim, int dtype) { - PROTECT( - auto outputs__ = torch::cumprod(*self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cumprod_(tensor *out__, tensor self, int64_t dim, int dtype) { - PROTECT( - auto outputs__ = self->cumprod_(dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cumprod_backward(tensor *out__, tensor grad, tensor input, int64_t dim, tensor output) { - PROTECT( - auto outputs__ = torch::cumprod_backward(*grad, *input, dim, *output); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cumprod_out(tensor *out__, tensor out, tensor self, int64_t dim, int dtype) { - PROTECT( - auto outputs__ = torch::cumprod_out(*out, *self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cumsum(tensor *out__, tensor self, int64_t dim, int dtype) { - PROTECT( - auto outputs__ = torch::cumsum(*self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cumsum_(tensor *out__, tensor self, int64_t dim, int dtype) { - PROTECT( - auto outputs__ = self->cumsum_(dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cumsum_out(tensor *out__, tensor out, tensor self, int64_t dim, int dtype) { - PROTECT( - auto outputs__ = torch::cumsum_out(*out, *self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cumulative_trapezoid(tensor *out__, tensor y, int64_t dim) { - PROTECT( - auto outputs__ = torch::cumulative_trapezoid(*y, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_cumulative_trapezoid_x(tensor *out__, tensor y, tensor x, int64_t dim) { - PROTECT( - auto outputs__ = torch::cumulative_trapezoid(*y, *x, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_data(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->data(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_deg2rad(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::deg2rad(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_deg2rad_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::deg2rad_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_deg2rad_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::deg2rad_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -int64_t atg_dense_dim(tensor self) { - PROTECT( - return self->dense_dim(); - ) - return 0; -} - -void atg_dequantize(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::dequantize(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_dequantize_self_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::dequantize_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -tensor *atg_dequantize_tensors(tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::dequantize(of_carray_tensor(tensors_data, tensors_len)); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg_dequantize_tensors_out(tensor *out_data, int out_len, tensor *tensors_data, int tensors_len) { - PROTECT( - torch::dequantize_out(of_carray_tensor(out_data, out_len), of_carray_tensor(tensors_data, tensors_len)); - ) -} - -void atg_det(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::det(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_detach(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::detach(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_detach_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::detach_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_detach_copy(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::detach_copy(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_detach_copy_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::detach_copy_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_diag(tensor *out__, tensor self, int64_t diagonal) { - PROTECT( - auto outputs__ = torch::diag(*self, diagonal); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_diag_embed(tensor *out__, tensor self, int64_t offset, int64_t dim1, int64_t dim2) { - PROTECT( - auto outputs__ = torch::diag_embed(*self, offset, dim1, dim2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_diag_embed_out(tensor *out__, tensor out, tensor self, int64_t offset, int64_t dim1, int64_t dim2) { - PROTECT( - auto outputs__ = torch::diag_embed_out(*out, *self, offset, dim1, dim2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_diag_out(tensor *out__, tensor out, tensor self, int64_t diagonal) { - PROTECT( - auto outputs__ = torch::diag_out(*out, *self, diagonal); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_diagflat(tensor *out__, tensor self, int64_t offset) { - PROTECT( - auto outputs__ = torch::diagflat(*self, offset); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_diagonal(tensor *out__, tensor self, int64_t offset, int64_t dim1, int64_t dim2) { - PROTECT( - auto outputs__ = torch::diagonal(*self, offset, dim1, dim2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_diagonal_backward(tensor *out__, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t offset, int64_t dim1, int64_t dim2) { - PROTECT( - auto outputs__ = torch::diagonal_backward(*grad_output, torch::IntArrayRef(input_sizes_data, input_sizes_len), offset, dim1, dim2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_diagonal_backward_out(tensor *out__, tensor out, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t offset, int64_t dim1, int64_t dim2) { - PROTECT( - auto outputs__ = torch::diagonal_backward_out(*out, *grad_output, torch::IntArrayRef(input_sizes_data, input_sizes_len), offset, dim1, dim2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_diagonal_copy(tensor *out__, tensor self, int64_t offset, int64_t dim1, int64_t dim2) { - PROTECT( - auto outputs__ = torch::diagonal_copy(*self, offset, dim1, dim2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_diagonal_copy_out(tensor *out__, tensor out, tensor self, int64_t offset, int64_t dim1, int64_t dim2) { - PROTECT( - auto outputs__ = torch::diagonal_copy_out(*out, *self, offset, dim1, dim2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_diagonal_scatter(tensor *out__, tensor self, tensor src, int64_t offset, int64_t dim1, int64_t dim2) { - PROTECT( - auto outputs__ = torch::diagonal_scatter(*self, *src, offset, dim1, dim2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_diagonal_scatter_out(tensor *out__, tensor out, tensor self, tensor src, int64_t offset, int64_t dim1, int64_t dim2) { - PROTECT( - auto outputs__ = torch::diagonal_scatter_out(*out, *self, *src, offset, dim1, dim2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_diff(tensor *out__, tensor self, int64_t n, int64_t dim, tensor prepend, tensor append) { - PROTECT( - auto outputs__ = torch::diff(*self, n, dim, (prepend ? ::std::optional(*prepend) : ::std::nullopt), (append ? ::std::optional(*append) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_diff_out(tensor *out__, tensor out, tensor self, int64_t n, int64_t dim, tensor prepend, tensor append) { - PROTECT( - auto outputs__ = torch::diff_out(*out, *self, n, dim, (prepend ? ::std::optional(*prepend) : ::std::nullopt), (append ? ::std::optional(*append) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_digamma(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::digamma(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_digamma_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->digamma_(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_digamma_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::digamma_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_dist(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::dist(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_dist_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::dist_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_div(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::div(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_div_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->div_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_div_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::div_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_div_out_mode(tensor *out__, tensor out, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len) { - PROTECT( - auto outputs__ = torch::div_out(*out, *self, *other, std::string(rounding_mode_ptr, rounding_mode_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_div_scalar(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::div(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_div_scalar_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->div_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_div_scalar_mode(tensor *out__, tensor self, scalar other, char* rounding_mode_ptr, int rounding_mode_len) { - PROTECT( - auto outputs__ = torch::div(*self, *other, std::string(rounding_mode_ptr, rounding_mode_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_div_scalar_mode_(tensor *out__, tensor self, scalar other, char* rounding_mode_ptr, int rounding_mode_len) { - PROTECT( - auto outputs__ = self->div_(*other, std::string(rounding_mode_ptr, rounding_mode_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_div_scalar_mode_out(tensor *out__, tensor out, tensor self, scalar other, char* rounding_mode_ptr, int rounding_mode_len) { - PROTECT( - auto outputs__ = torch::div_out(*out, *self, *other, std::string(rounding_mode_ptr, rounding_mode_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_div_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::div_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_div_tensor_mode(tensor *out__, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len) { - PROTECT( - auto outputs__ = torch::div(*self, *other, std::string(rounding_mode_ptr, rounding_mode_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_div_tensor_mode_(tensor *out__, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len) { - PROTECT( - auto outputs__ = self->div_(*other, std::string(rounding_mode_ptr, rounding_mode_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_divide(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::divide(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_divide_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->divide_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_divide_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::divide_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_divide_out_mode(tensor *out__, tensor out, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len) { - PROTECT( - auto outputs__ = torch::divide_out(*out, *self, *other, std::string(rounding_mode_ptr, rounding_mode_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_divide_scalar(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::divide(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_divide_scalar_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->divide_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_divide_scalar_mode(tensor *out__, tensor self, scalar other, char* rounding_mode_ptr, int rounding_mode_len) { - PROTECT( - auto outputs__ = torch::divide(*self, *other, std::string(rounding_mode_ptr, rounding_mode_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_divide_scalar_mode_(tensor *out__, tensor self, scalar other, char* rounding_mode_ptr, int rounding_mode_len) { - PROTECT( - auto outputs__ = self->divide_(*other, std::string(rounding_mode_ptr, rounding_mode_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_divide_tensor_mode(tensor *out__, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len) { - PROTECT( - auto outputs__ = torch::divide(*self, *other, std::string(rounding_mode_ptr, rounding_mode_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_divide_tensor_mode_(tensor *out__, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len) { - PROTECT( - auto outputs__ = self->divide_(*other, std::string(rounding_mode_ptr, rounding_mode_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_dot(tensor *out__, tensor self, tensor tensor) { - PROTECT( - auto outputs__ = torch::dot(*self, *tensor); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_dot_out(tensor *out__, tensor out, tensor self, tensor tensor) { - PROTECT( - auto outputs__ = torch::dot_out(*out, *self, *tensor); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_dropout(tensor *out__, tensor input, double p, int train) { - PROTECT( - auto outputs__ = torch::dropout(*input, p, (bool)train); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_dropout_(tensor *out__, tensor self, double p, int train) { - PROTECT( - auto outputs__ = torch::dropout_(*self, p, (bool)train); - out__[0] = new torch::Tensor(outputs__); - ) -} - -tensor *atg_dsplit(tensor self, int64_t sections) { - PROTECT( - auto outputs__ = torch::dsplit(*self, sections); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -tensor *atg_dsplit_array(tensor self, int64_t *indices_data, int indices_len) { - PROTECT( - auto outputs__ = torch::dsplit(*self, torch::IntArrayRef(indices_data, indices_len)); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg_dstack(tensor *out__, tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::dstack(of_carray_tensor(tensors_data, tensors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_dstack_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::dstack_out(*out, of_carray_tensor(tensors_data, tensors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_einsum(tensor *out__, char* equation_ptr, int equation_len, tensor *tensors_data, int tensors_len, int64_t *path_data, int path_len) { - PROTECT( - auto outputs__ = torch::einsum(std::string(equation_ptr, equation_len), of_carray_tensor(tensors_data, tensors_len), path_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(path_data, path_len))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_elu(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::elu(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_elu_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::elu_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_elu_backward(tensor *out__, tensor grad_output, scalar alpha, scalar scale, scalar input_scale, int is_result, tensor self_or_result) { - PROTECT( - auto outputs__ = torch::elu_backward(*grad_output, *alpha, *scale, *input_scale, (bool)is_result, *self_or_result); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_elu_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, scalar alpha, scalar scale, scalar input_scale, int is_result, tensor self_or_result) { - PROTECT( - auto outputs__ = torch::elu_backward_out(*grad_input, *grad_output, *alpha, *scale, *input_scale, (bool)is_result, *self_or_result); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_elu_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::elu_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_embedding(tensor *out__, tensor weight, tensor indices, int64_t padding_idx, int scale_grad_by_freq, int sparse) { - PROTECT( - auto outputs__ = torch::embedding(*weight, *indices, padding_idx, (bool)scale_grad_by_freq, (bool)sparse); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_embedding_backward(tensor *out__, tensor grad, tensor indices, int64_t num_weights, int64_t padding_idx, int scale_grad_by_freq, int sparse) { - PROTECT( - auto outputs__ = torch::embedding_backward(*grad, *indices, num_weights, padding_idx, (bool)scale_grad_by_freq, (bool)sparse); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_embedding_bag(tensor *out__, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset) { - PROTECT( - auto outputs__ = torch::embedding_bag(*weight, *indices, *offsets, (bool)scale_grad_by_freq, mode, (bool)sparse, (per_sample_weights ? ::std::optional(*per_sample_weights) : ::std::nullopt), (bool)include_last_offset); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg_embedding_bag_padding_idx(tensor *out__, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset, int64_t padding_idx_v, uint8_t padding_idx_null) { - PROTECT( - auto outputs__ = torch::embedding_bag(*weight, *indices, *offsets, (bool)scale_grad_by_freq, mode, (bool)sparse, (per_sample_weights ? ::std::optional(*per_sample_weights) : ::std::nullopt), (bool)include_last_offset, padding_idx_null ? c10::nullopt : c10::optional(padding_idx_v)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg_embedding_dense_backward(tensor *out__, tensor grad_output, tensor indices, int64_t num_weights, int64_t padding_idx, int scale_grad_by_freq) { - PROTECT( - auto outputs__ = torch::embedding_dense_backward(*grad_output, *indices, num_weights, padding_idx, (bool)scale_grad_by_freq); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_embedding_dense_backward_out(tensor *out__, tensor out, tensor grad_output, tensor indices, int64_t num_weights, int64_t padding_idx, int scale_grad_by_freq) { - PROTECT( - auto outputs__ = torch::embedding_dense_backward_out(*out, *grad_output, *indices, num_weights, padding_idx, (bool)scale_grad_by_freq); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_embedding_out(tensor *out__, tensor out, tensor weight, tensor indices, int64_t padding_idx, int scale_grad_by_freq, int sparse) { - PROTECT( - auto outputs__ = torch::embedding_out(*out, *weight, *indices, padding_idx, (bool)scale_grad_by_freq, (bool)sparse); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_embedding_renorm(tensor *out__, tensor self, tensor indices, double max_norm, double norm_type) { - PROTECT( - auto outputs__ = torch::embedding_renorm(*self, *indices, max_norm, norm_type); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_embedding_renorm_(tensor *out__, tensor self, tensor indices, double max_norm, double norm_type) { - PROTECT( - auto outputs__ = torch::embedding_renorm_(*self, *indices, max_norm, norm_type); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_embedding_renorm_out(tensor *out__, tensor out, tensor self, tensor indices, double max_norm, double norm_type) { - PROTECT( - auto outputs__ = torch::embedding_renorm_out(*out, *self, *indices, max_norm, norm_type); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_embedding_sparse_backward(tensor *out__, tensor grad, tensor indices, int64_t num_weights, int64_t padding_idx, int scale_grad_by_freq) { - PROTECT( - auto outputs__ = torch::embedding_sparse_backward(*grad, *indices, num_weights, padding_idx, (bool)scale_grad_by_freq); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_empty(tensor *out__, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::empty(torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_empty_like(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::empty_like(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_empty_like_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::empty_like_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_empty_out(tensor *out__, tensor out, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::empty_out(*out, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_empty_permuted(tensor *out__, int64_t *size_data, int size_len, int64_t *physical_layout_data, int physical_layout_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::empty_permuted(torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(physical_layout_data, physical_layout_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_empty_permuted_out(tensor *out__, tensor out, int64_t *size_data, int size_len, int64_t *physical_layout_data, int physical_layout_len) { - PROTECT( - auto outputs__ = torch::empty_permuted_out(*out, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(physical_layout_data, physical_layout_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_empty_quantized(tensor *out__, int64_t *size_data, int size_len, tensor qtensor, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::empty_quantized(torch::IntArrayRef(size_data, size_len), *qtensor, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_empty_quantized_out(tensor *out__, tensor out, int64_t *size_data, int size_len, tensor qtensor) { - PROTECT( - auto outputs__ = torch::empty_quantized_out(*out, torch::IntArrayRef(size_data, size_len), *qtensor); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_empty_strided(tensor *out__, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::empty_strided(torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_empty_strided_out(tensor *out__, tensor out, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len) { - PROTECT( - auto outputs__ = torch::empty_strided_out(*out, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_eq(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::eq(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_eq_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->eq_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_eq_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::eq_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_eq_tensor(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::eq(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_eq_tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->eq_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_eq_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::eq_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -int atg_equal(tensor self, tensor other) { - PROTECT( - return torch::equal(*self, *other); - ) - return 0; -} - -void atg_erf(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::erf(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_erf_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::erf_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_erf_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::erf_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_erfc(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::erfc(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_erfc_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::erfc_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_erfc_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::erfc_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_erfinv(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::erfinv(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_erfinv_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->erfinv_(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_erfinv_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::erfinv_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_exp(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::exp(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_exp2(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::exp2(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_exp2_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::exp2_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_exp2_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::exp2_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_exp_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::exp_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_exp_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::exp_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_expand(tensor *out__, tensor self, int64_t *size_data, int size_len, int implicit) { - PROTECT( - auto outputs__ = self->expand(torch::IntArrayRef(size_data, size_len), (bool)implicit); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_expand_as(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->expand_as(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_expand_copy(tensor *out__, tensor self, int64_t *size_data, int size_len, int implicit) { - PROTECT( - auto outputs__ = torch::expand_copy(*self, torch::IntArrayRef(size_data, size_len), (bool)implicit); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_expand_copy_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len, int implicit) { - PROTECT( - auto outputs__ = torch::expand_copy_out(*out, *self, torch::IntArrayRef(size_data, size_len), (bool)implicit); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_expm1(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::expm1(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_expm1_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::expm1_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_expm1_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::expm1_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_exponential(tensor *out__, tensor self, double lambd) { - PROTECT( - auto outputs__ = torch::exponential(*self, lambd); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_exponential_(tensor *out__, tensor self, double lambd) { - PROTECT( - auto outputs__ = self->exponential_(lambd); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_exponential_out(tensor *out__, tensor out, tensor self, double lambd) { - PROTECT( - auto outputs__ = torch::exponential_out(*out, *self, lambd); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_eye(tensor *out__, int64_t n, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::eye(n, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_eye_m(tensor *out__, int64_t n, int64_t m, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::eye(n, m, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_eye_m_out(tensor *out__, tensor out, int64_t n, int64_t m) { - PROTECT( - auto outputs__ = torch::eye_out(*out, n, m); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_eye_out(tensor *out__, tensor out, int64_t n) { - PROTECT( - auto outputs__ = torch::eye_out(*out, n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fake_quantize_per_channel_affine(tensor *out__, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max) { - PROTECT( - auto outputs__ = torch::fake_quantize_per_channel_affine(*self, *scale, *zero_point, axis, quant_min, quant_max); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fake_quantize_per_channel_affine_cachemask(tensor *out__, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max) { - PROTECT( - auto outputs__ = torch::fake_quantize_per_channel_affine_cachemask(*self, *scale, *zero_point, axis, quant_min, quant_max); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_fake_quantize_per_channel_affine_cachemask_backward(tensor *out__, tensor grad, tensor mask) { - PROTECT( - auto outputs__ = torch::fake_quantize_per_channel_affine_cachemask_backward(*grad, *mask); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fake_quantize_per_channel_affine_cachemask_out(tensor *out__, tensor out0, tensor out1, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max) { - PROTECT( - auto outputs__ = torch::fake_quantize_per_channel_affine_cachemask_out(*out0, *out1, *self, *scale, *zero_point, axis, quant_min, quant_max); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_fake_quantize_per_tensor_affine(tensor *out__, tensor self, double scale, int64_t zero_point, int64_t quant_min, int64_t quant_max) { - PROTECT( - auto outputs__ = torch::fake_quantize_per_tensor_affine(*self, scale, zero_point, quant_min, quant_max); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fake_quantize_per_tensor_affine_cachemask(tensor *out__, tensor self, double scale, int64_t zero_point, int64_t quant_min, int64_t quant_max) { - PROTECT( - auto outputs__ = torch::fake_quantize_per_tensor_affine_cachemask(*self, scale, zero_point, quant_min, quant_max); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_fake_quantize_per_tensor_affine_cachemask_backward(tensor *out__, tensor grad, tensor mask) { - PROTECT( - auto outputs__ = torch::fake_quantize_per_tensor_affine_cachemask_backward(*grad, *mask); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fake_quantize_per_tensor_affine_cachemask_out(tensor *out__, tensor out0, tensor out1, tensor self, double scale, int64_t zero_point, int64_t quant_min, int64_t quant_max) { - PROTECT( - auto outputs__ = torch::fake_quantize_per_tensor_affine_cachemask_out(*out0, *out1, *self, scale, zero_point, quant_min, quant_max); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_fake_quantize_per_tensor_affine_tensor_qparams(tensor *out__, tensor self, tensor scale, tensor zero_point, int64_t quant_min, int64_t quant_max) { - PROTECT( - auto outputs__ = torch::fake_quantize_per_tensor_affine(*self, *scale, *zero_point, quant_min, quant_max); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_feature_alpha_dropout(tensor *out__, tensor input, double p, int train) { - PROTECT( - auto outputs__ = torch::feature_alpha_dropout(*input, p, (bool)train); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_feature_alpha_dropout_(tensor *out__, tensor self, double p, int train) { - PROTECT( - auto outputs__ = torch::feature_alpha_dropout_(*self, p, (bool)train); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_feature_dropout(tensor *out__, tensor input, double p, int train) { - PROTECT( - auto outputs__ = torch::feature_dropout(*input, p, (bool)train); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_feature_dropout_(tensor *out__, tensor self, double p, int train) { - PROTECT( - auto outputs__ = torch::feature_dropout_(*self, p, (bool)train); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_fft(tensor *out__, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_fft(*self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_fft2(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_fft2(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_fft2_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_fft2_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_fft_out(tensor *out__, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_fft_out(*out, *self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_fftfreq(tensor *out__, int64_t n, double d, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::fft_fftfreq(n, d, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_fftfreq_out(tensor *out__, tensor out, int64_t n, double d) { - PROTECT( - auto outputs__ = torch::fft_fftfreq_out(*out, n, d); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_fftn(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_fftn(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_fftn_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_fftn_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_fftshift(tensor *out__, tensor self, int64_t *dim_data, int dim_len) { - PROTECT( - auto outputs__ = torch::fft_fftshift(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_hfft(tensor *out__, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_hfft(*self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_hfft2(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_hfft2(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_hfft2_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_hfft2_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_hfft_out(tensor *out__, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_hfft_out(*out, *self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_hfftn(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_hfftn(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_hfftn_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_hfftn_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_ifft(tensor *out__, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_ifft(*self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_ifft2(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_ifft2(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_ifft2_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_ifft2_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_ifft_out(tensor *out__, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_ifft_out(*out, *self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_ifftn(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_ifftn(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_ifftn_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_ifftn_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_ifftshift(tensor *out__, tensor self, int64_t *dim_data, int dim_len) { - PROTECT( - auto outputs__ = torch::fft_ifftshift(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_ihfft(tensor *out__, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_ihfft(*self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_ihfft2(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_ihfft2(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_ihfft2_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_ihfft2_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_ihfft_out(tensor *out__, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_ihfft_out(*out, *self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_ihfftn(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_ihfftn(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_ihfftn_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_ihfftn_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_irfft(tensor *out__, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_irfft(*self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_irfft2(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_irfft2(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_irfft2_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_irfft2_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_irfft_out(tensor *out__, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_irfft_out(*out, *self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_irfftn(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_irfftn(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_irfftn_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_irfftn_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_rfft(tensor *out__, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_rfft(*self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_rfft2(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_rfft2(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_rfft2_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_rfft2_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), torch::IntArrayRef(dim_data, dim_len), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_rfft_out(tensor *out__, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_rfft_out(*out, *self, n_null ? c10::nullopt : c10::optional(n_v), dim, std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_rfftfreq(tensor *out__, int64_t n, double d, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::fft_rfftfreq(n, d, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_rfftfreq_out(tensor *out__, tensor out, int64_t n, double d) { - PROTECT( - auto outputs__ = torch::fft_rfftfreq_out(*out, n, d); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_rfftn(tensor *out__, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_rfftn(*self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fft_rfftn_out(tensor *out__, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len) { - PROTECT( - auto outputs__ = torch::fft_rfftn_out(*out, *self, s_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(s_data, s_len)), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), std::string(norm_ptr, norm_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fill(tensor *out__, tensor self, scalar value) { - PROTECT( - auto outputs__ = torch::fill(*self, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fill_(tensor *out__, tensor self, scalar value) { - PROTECT( - auto outputs__ = torch::fill_(*self, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fill_diagonal_(tensor *out__, tensor self, scalar fill_value, int wrap) { - PROTECT( - auto outputs__ = self->fill_diagonal_(*fill_value, (bool)wrap); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fill_scalar_out(tensor *out__, tensor out, tensor self, scalar value) { - PROTECT( - auto outputs__ = torch::fill_out(*out, *self, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fill_tensor(tensor *out__, tensor self, tensor value) { - PROTECT( - auto outputs__ = torch::fill(*self, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fill_tensor_(tensor *out__, tensor self, tensor value) { - PROTECT( - auto outputs__ = torch::fill_(*self, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fill_tensor_out(tensor *out__, tensor out, tensor self, tensor value) { - PROTECT( - auto outputs__ = torch::fill_out(*out, *self, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fix(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::fix(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fix_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::fix_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fix_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::fix_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_flatten(tensor *out__, tensor self, int64_t start_dim, int64_t end_dim) { - PROTECT( - auto outputs__ = torch::flatten(*self, start_dim, end_dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_flatten_dense_tensors(tensor *out__, tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::flatten_dense_tensors(of_carray_tensor(tensors_data, tensors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_flip(tensor *out__, tensor self, int64_t *dims_data, int dims_len) { - PROTECT( - auto outputs__ = torch::flip(*self, torch::IntArrayRef(dims_data, dims_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_flip_out(tensor *out__, tensor out, tensor self, int64_t *dims_data, int dims_len) { - PROTECT( - auto outputs__ = torch::flip_out(*out, *self, torch::IntArrayRef(dims_data, dims_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fliplr(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::fliplr(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_flipud(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::flipud(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_float_power(tensor *out__, tensor self, tensor exponent) { - PROTECT( - auto outputs__ = torch::float_power(*self, *exponent); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_float_power_(tensor *out__, tensor self, scalar exponent) { - PROTECT( - auto outputs__ = self->float_power_(*exponent); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_float_power_scalar(tensor *out__, scalar self_scalar, tensor exponent) { - PROTECT( - auto outputs__ = torch::float_power(*self_scalar, *exponent); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_float_power_scalar_out(tensor *out__, tensor out, scalar self_scalar, tensor exponent) { - PROTECT( - auto outputs__ = torch::float_power_out(*out, *self_scalar, *exponent); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_float_power_tensor_(tensor *out__, tensor self, tensor exponent) { - PROTECT( - auto outputs__ = self->float_power_(*exponent); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_float_power_tensor_scalar(tensor *out__, tensor self, scalar exponent) { - PROTECT( - auto outputs__ = torch::float_power(*self, *exponent); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_float_power_tensor_scalar_out(tensor *out__, tensor out, tensor self, scalar exponent) { - PROTECT( - auto outputs__ = torch::float_power_out(*out, *self, *exponent); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_float_power_tensor_tensor_out(tensor *out__, tensor out, tensor self, tensor exponent) { - PROTECT( - auto outputs__ = torch::float_power_out(*out, *self, *exponent); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_floor(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::floor(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_floor_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::floor_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_floor_divide(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::floor_divide(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_floor_divide_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->floor_divide_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_floor_divide_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::floor_divide_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_floor_divide_scalar(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::floor_divide(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_floor_divide_scalar_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->floor_divide_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_floor_divide_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::floor_divide_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_floor_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::floor_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fmax(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::fmax(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fmax_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::fmax_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fmin(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::fmin(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fmin_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::fmin_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fmod(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::fmod(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fmod_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->fmod_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fmod_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::fmod_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fmod_tensor(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::fmod(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fmod_tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->fmod_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fmod_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::fmod_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_frac(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::frac(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_frac_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::frac_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_frac_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::frac_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fractional_max_pool2d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor random_samples) { - PROTECT( - auto outputs__ = torch::fractional_max_pool2d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(output_size_data, output_size_len), *random_samples); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_fractional_max_pool2d_backward(tensor *out__, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor indices) { - PROTECT( - auto outputs__ = torch::fractional_max_pool2d_backward(*grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(output_size_data, output_size_len), *indices); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fractional_max_pool2d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor indices) { - PROTECT( - auto outputs__ = torch::fractional_max_pool2d_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(output_size_data, output_size_len), *indices); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fractional_max_pool2d_output(tensor *out__, tensor output, tensor indices, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor random_samples) { - PROTECT( - auto outputs__ = torch::fractional_max_pool2d_out(*output, *indices, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(output_size_data, output_size_len), *random_samples); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_fractional_max_pool3d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor random_samples) { - PROTECT( - auto outputs__ = torch::fractional_max_pool3d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(output_size_data, output_size_len), *random_samples); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_fractional_max_pool3d_backward(tensor *out__, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor indices) { - PROTECT( - auto outputs__ = torch::fractional_max_pool3d_backward(*grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(output_size_data, output_size_len), *indices); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fractional_max_pool3d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor indices) { - PROTECT( - auto outputs__ = torch::fractional_max_pool3d_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(output_size_data, output_size_len), *indices); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fractional_max_pool3d_output(tensor *out__, tensor output, tensor indices, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor random_samples) { - PROTECT( - auto outputs__ = torch::fractional_max_pool3d_out(*output, *indices, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(output_size_data, output_size_len), *random_samples); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_frexp(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::frexp(*self); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_frexp_tensor_out(tensor *out__, tensor mantissa, tensor exponent, tensor self) { - PROTECT( - auto outputs__ = torch::frexp_out(*mantissa, *exponent, *self); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_frobenius_norm(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim) { - PROTECT( - auto outputs__ = torch::frobenius_norm(*self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_frobenius_norm_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim) { - PROTECT( - auto outputs__ = torch::frobenius_norm_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_from_file(tensor *out__, char* filename_ptr, int filename_len, int shared, int64_t size_v, uint8_t size_null, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::from_file(std::string(filename_ptr, filename_len), (bool)shared, size_null ? c10::nullopt : c10::optional(size_v), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_from_file_out(tensor *out__, tensor out, char* filename_ptr, int filename_len, int shared, int64_t size_v, uint8_t size_null) { - PROTECT( - auto outputs__ = torch::from_file_out(*out, std::string(filename_ptr, filename_len), (bool)shared, size_null ? c10::nullopt : c10::optional(size_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_full(tensor *out__, int64_t *size_data, int size_len, scalar fill_value, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::full(torch::IntArrayRef(size_data, size_len), *fill_value, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_full_like(tensor *out__, tensor self, scalar fill_value) { - PROTECT( - auto outputs__ = torch::full_like(*self, *fill_value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_full_like_out(tensor *out__, tensor out, tensor self, scalar fill_value) { - PROTECT( - auto outputs__ = torch::full_like_out(*out, *self, *fill_value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_full_out(tensor *out__, tensor out, int64_t *size_data, int size_len, scalar fill_value) { - PROTECT( - auto outputs__ = torch::full_out(*out, torch::IntArrayRef(size_data, size_len), *fill_value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_fused_moving_avg_obs_fake_quant(tensor *out__, tensor self, tensor observer_on, tensor fake_quant_on, tensor running_min, tensor running_max, tensor scale, tensor zero_point, double averaging_const, int64_t quant_min, int64_t quant_max, int64_t ch_axis, int per_row_fake_quant, int symmetric_quant) { - PROTECT( - auto outputs__ = torch::fused_moving_avg_obs_fake_quant(*self, *observer_on, *fake_quant_on, *running_min, *running_max, *scale, *zero_point, averaging_const, quant_min, quant_max, ch_axis, (bool)per_row_fake_quant, (bool)symmetric_quant); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_gather(tensor *out__, tensor self, int64_t dim, tensor index, int sparse_grad) { - PROTECT( - auto outputs__ = torch::gather(*self, dim, *index, (bool)sparse_grad); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_gather_backward(tensor *out__, tensor grad, tensor self, int64_t dim, tensor index, int sparse_grad) { - PROTECT( - auto outputs__ = torch::gather_backward(*grad, *self, dim, *index, (bool)sparse_grad); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_gather_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, int sparse_grad) { - PROTECT( - auto outputs__ = torch::gather_out(*out, *self, dim, *index, (bool)sparse_grad); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_gcd(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::gcd(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_gcd_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::gcd_(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_gcd_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::gcd_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ge(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::ge(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ge_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->ge_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ge_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::ge_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ge_tensor(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::ge(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ge_tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->ge_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ge_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::ge_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_gelu(tensor *out__, tensor self, char* approximate_ptr, int approximate_len) { - PROTECT( - auto outputs__ = torch::gelu(*self, std::string(approximate_ptr, approximate_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_gelu_(tensor *out__, tensor self, char* approximate_ptr, int approximate_len) { - PROTECT( - auto outputs__ = torch::gelu_(*self, std::string(approximate_ptr, approximate_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_gelu_backward(tensor *out__, tensor grad_output, tensor self, char* approximate_ptr, int approximate_len) { - PROTECT( - auto outputs__ = torch::gelu_backward(*grad_output, *self, std::string(approximate_ptr, approximate_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_gelu_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, char* approximate_ptr, int approximate_len) { - PROTECT( - auto outputs__ = torch::gelu_backward_out(*grad_input, *grad_output, *self, std::string(approximate_ptr, approximate_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_gelu_out(tensor *out__, tensor out, tensor self, char* approximate_ptr, int approximate_len) { - PROTECT( - auto outputs__ = torch::gelu_out(*out, *self, std::string(approximate_ptr, approximate_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_geometric(tensor *out__, tensor self, double p) { - PROTECT( - auto outputs__ = torch::geometric(*self, p); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_geometric_(tensor *out__, tensor self, double p) { - PROTECT( - auto outputs__ = self->geometric_(p); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_geometric_out(tensor *out__, tensor out, tensor self, double p) { - PROTECT( - auto outputs__ = torch::geometric_out(*out, *self, p); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_geqrf(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::geqrf(*self); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_geqrf_a(tensor *out__, tensor a, tensor tau, tensor self) { - PROTECT( - auto outputs__ = torch::geqrf_out(*a, *tau, *self); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_ger(tensor *out__, tensor self, tensor vec2) { - PROTECT( - auto outputs__ = torch::ger(*self, *vec2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ger_out(tensor *out__, tensor out, tensor self, tensor vec2) { - PROTECT( - auto outputs__ = torch::ger_out(*out, *self, *vec2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_glu(tensor *out__, tensor self, int64_t dim) { - PROTECT( - auto outputs__ = torch::glu(*self, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_glu_backward(tensor *out__, tensor grad_output, tensor self, int64_t dim) { - PROTECT( - auto outputs__ = torch::glu_backward(*grad_output, *self, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_glu_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t dim) { - PROTECT( - auto outputs__ = torch::glu_backward_out(*grad_input, *grad_output, *self, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_glu_backward_jvp(tensor *out__, tensor grad_x, tensor grad_glu, tensor x, tensor dgrad_glu, tensor dx, int64_t dim) { - PROTECT( - auto outputs__ = torch::glu_backward_jvp(*grad_x, *grad_glu, *x, *dgrad_glu, *dx, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_glu_backward_jvp_out(tensor *out__, tensor out, tensor grad_x, tensor grad_glu, tensor x, tensor dgrad_glu, tensor dx, int64_t dim) { - PROTECT( - auto outputs__ = torch::glu_backward_jvp_out(*out, *grad_x, *grad_glu, *x, *dgrad_glu, *dx, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_glu_jvp(tensor *out__, tensor glu, tensor x, tensor dx, int64_t dim) { - PROTECT( - auto outputs__ = torch::glu_jvp(*glu, *x, *dx, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_glu_jvp_out(tensor *out__, tensor out, tensor glu, tensor x, tensor dx, int64_t dim) { - PROTECT( - auto outputs__ = torch::glu_jvp_out(*out, *glu, *x, *dx, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_glu_out(tensor *out__, tensor out, tensor self, int64_t dim) { - PROTECT( - auto outputs__ = torch::glu_out(*out, *self, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_grad(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->grad(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_greater(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::greater(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_greater_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->greater_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_greater_equal(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::greater_equal(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_greater_equal_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->greater_equal_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_greater_equal_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::greater_equal_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_greater_equal_tensor(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::greater_equal(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_greater_equal_tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->greater_equal_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_greater_equal_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::greater_equal_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_greater_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::greater_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_greater_tensor(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::greater(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_greater_tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->greater_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_greater_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::greater_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_grid_sampler(tensor *out__, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners) { - PROTECT( - auto outputs__ = torch::grid_sampler(*input, *grid, interpolation_mode, padding_mode, (bool)align_corners); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_grid_sampler_2d(tensor *out__, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners) { - PROTECT( - auto outputs__ = torch::grid_sampler_2d(*input, *grid, interpolation_mode, padding_mode, (bool)align_corners); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_grid_sampler_2d_out(tensor *out__, tensor out, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners) { - PROTECT( - auto outputs__ = torch::grid_sampler_2d_out(*out, *input, *grid, interpolation_mode, padding_mode, (bool)align_corners); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_grid_sampler_3d(tensor *out__, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners) { - PROTECT( - auto outputs__ = torch::grid_sampler_3d(*input, *grid, interpolation_mode, padding_mode, (bool)align_corners); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_grid_sampler_3d_out(tensor *out__, tensor out, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners) { - PROTECT( - auto outputs__ = torch::grid_sampler_3d_out(*out, *input, *grid, interpolation_mode, padding_mode, (bool)align_corners); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_group_norm(tensor *out__, tensor input, int64_t num_groups, tensor weight, tensor bias, double eps, int cudnn_enabled) { - PROTECT( - auto outputs__ = torch::group_norm(*input, num_groups, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), eps, (bool)cudnn_enabled); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_gru(tensor *out__, tensor input, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first) { - PROTECT( - auto outputs__ = torch::gru(*input, *hx, of_carray_tensor(params_data, params_len), (bool)has_biases, num_layers, dropout, (bool)train, (bool)bidirectional, (bool)batch_first); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_gru_cell(tensor *out__, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh) { - PROTECT( - auto outputs__ = torch::gru_cell(*input, *hx, *w_ih, *w_hh, (b_ih ? ::std::optional(*b_ih) : ::std::nullopt), (b_hh ? ::std::optional(*b_hh) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_gru_data(tensor *out__, tensor data, tensor batch_sizes, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional) { - PROTECT( - auto outputs__ = torch::gru(*data, *batch_sizes, *hx, of_carray_tensor(params_data, params_len), (bool)has_biases, num_layers, dropout, (bool)train, (bool)bidirectional); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_gt(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::gt(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_gt_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->gt_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_gt_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::gt_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_gt_tensor(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::gt(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_gt_tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->gt_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_gt_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::gt_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hamming_window(tensor *out__, int64_t window_length, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::hamming_window(window_length, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hamming_window_out(tensor *out__, tensor out, int64_t window_length) { - PROTECT( - auto outputs__ = torch::hamming_window_out(*out, window_length); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hamming_window_periodic(tensor *out__, int64_t window_length, int periodic, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::hamming_window(window_length, (bool)periodic, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hamming_window_periodic_alpha(tensor *out__, int64_t window_length, int periodic, double alpha, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::hamming_window(window_length, (bool)periodic, alpha, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hamming_window_periodic_alpha_beta(tensor *out__, int64_t window_length, int periodic, double alpha, double beta, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::hamming_window(window_length, (bool)periodic, alpha, beta, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hamming_window_periodic_alpha_beta_out(tensor *out__, tensor out, int64_t window_length, int periodic, double alpha, double beta) { - PROTECT( - auto outputs__ = torch::hamming_window_out(*out, window_length, (bool)periodic, alpha, beta); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hamming_window_periodic_alpha_out(tensor *out__, tensor out, int64_t window_length, int periodic, double alpha) { - PROTECT( - auto outputs__ = torch::hamming_window_out(*out, window_length, (bool)periodic, alpha); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hamming_window_periodic_out(tensor *out__, tensor out, int64_t window_length, int periodic) { - PROTECT( - auto outputs__ = torch::hamming_window_out(*out, window_length, (bool)periodic); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hann_window(tensor *out__, int64_t window_length, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::hann_window(window_length, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hann_window_out(tensor *out__, tensor out, int64_t window_length) { - PROTECT( - auto outputs__ = torch::hann_window_out(*out, window_length); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hann_window_periodic(tensor *out__, int64_t window_length, int periodic, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::hann_window(window_length, (bool)periodic, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hann_window_periodic_out(tensor *out__, tensor out, int64_t window_length, int periodic) { - PROTECT( - auto outputs__ = torch::hann_window_out(*out, window_length, (bool)periodic); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hardshrink(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::hardshrink(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hardshrink_backward(tensor *out__, tensor grad_out, tensor self, scalar lambd) { - PROTECT( - auto outputs__ = torch::hardshrink_backward(*grad_out, *self, *lambd); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hardshrink_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_out, tensor self, scalar lambd) { - PROTECT( - auto outputs__ = torch::hardshrink_backward_out(*grad_input, *grad_out, *self, *lambd); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hardshrink_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::hardshrink_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hardsigmoid(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::hardsigmoid(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hardsigmoid_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::hardsigmoid_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hardsigmoid_backward(tensor *out__, tensor grad_output, tensor self) { - PROTECT( - auto outputs__ = torch::hardsigmoid_backward(*grad_output, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hardsigmoid_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self) { - PROTECT( - auto outputs__ = torch::hardsigmoid_backward_out(*grad_input, *grad_output, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hardsigmoid_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::hardsigmoid_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hardswish(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::hardswish(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hardswish_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::hardswish_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hardswish_backward(tensor *out__, tensor grad_output, tensor self) { - PROTECT( - auto outputs__ = torch::hardswish_backward(*grad_output, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hardswish_backward_out(tensor *out__, tensor out, tensor grad_output, tensor self) { - PROTECT( - auto outputs__ = torch::hardswish_backward_out(*out, *grad_output, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hardswish_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::hardswish_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hardtanh(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::hardtanh(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hardtanh_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::hardtanh_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hardtanh_backward(tensor *out__, tensor grad_output, tensor self, scalar min_val, scalar max_val) { - PROTECT( - auto outputs__ = torch::hardtanh_backward(*grad_output, *self, *min_val, *max_val); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hardtanh_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, scalar min_val, scalar max_val) { - PROTECT( - auto outputs__ = torch::hardtanh_backward_out(*grad_input, *grad_output, *self, *min_val, *max_val); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hardtanh_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::hardtanh_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hash_tensor(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim, int64_t mode) { - PROTECT( - auto outputs__ = torch::hash_tensor(*self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim, mode); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hash_tensor_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int64_t mode) { - PROTECT( - auto outputs__ = torch::hash_tensor_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim, mode); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_heaviside(tensor *out__, tensor self, tensor values) { - PROTECT( - auto outputs__ = torch::heaviside(*self, *values); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_heaviside_(tensor *out__, tensor self, tensor values) { - PROTECT( - auto outputs__ = self->heaviside_(*values); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_heaviside_out(tensor *out__, tensor out, tensor self, tensor values) { - PROTECT( - auto outputs__ = torch::heaviside_out(*out, *self, *values); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hinge_embedding_loss(tensor *out__, tensor self, tensor target, double margin, int64_t reduction) { - PROTECT( - auto outputs__ = torch::hinge_embedding_loss(*self, *target, margin, reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_histc(tensor *out__, tensor self, int64_t bins) { - PROTECT( - auto outputs__ = torch::histc(*self, bins); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_histc_out(tensor *out__, tensor out, tensor self, int64_t bins) { - PROTECT( - auto outputs__ = torch::histc_out(*out, *self, bins); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_histogram(tensor *out__, tensor self, tensor bins, tensor weight, int density) { - PROTECT( - auto outputs__ = torch::histogram(*self, *bins, (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)density); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_histogram_bin_ct(tensor *out__, tensor self, int64_t bins, double *range_data, int range_len, tensor weight, int density) { - PROTECT( - auto outputs__ = torch::histogram(*self, bins, at::ArrayRef(range_data, range_len), (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)density); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_histogram_bin_ct_out(tensor *out__, tensor hist, tensor bin_edges, tensor self, int64_t bins, double *range_data, int range_len, tensor weight, int density) { - PROTECT( - auto outputs__ = torch::histogram_out(*hist, *bin_edges, *self, bins, at::ArrayRef(range_data, range_len), (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)density); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_histogram_bins_tensor_out(tensor *out__, tensor hist, tensor bin_edges, tensor self, tensor bins, tensor weight, int density) { - PROTECT( - auto outputs__ = torch::histogram_out(*hist, *bin_edges, *self, *bins, (weight ? ::std::optional(*weight) : ::std::nullopt), (bool)density); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -tensor *atg_hsplit(tensor self, int64_t sections) { - PROTECT( - auto outputs__ = torch::hsplit(*self, sections); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -tensor *atg_hsplit_array(tensor self, int64_t *indices_data, int indices_len) { - PROTECT( - auto outputs__ = torch::hsplit(*self, torch::IntArrayRef(indices_data, indices_len)); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg_hspmm(tensor *out__, tensor mat1, tensor mat2) { - PROTECT( - auto outputs__ = torch::hspmm(*mat1, *mat2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hspmm_out(tensor *out__, tensor out, tensor mat1, tensor mat2) { - PROTECT( - auto outputs__ = torch::hspmm_out(*out, *mat1, *mat2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hstack(tensor *out__, tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::hstack(of_carray_tensor(tensors_data, tensors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hstack_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::hstack_out(*out, of_carray_tensor(tensors_data, tensors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_huber_loss(tensor *out__, tensor self, tensor target, int64_t reduction, double delta) { - PROTECT( - auto outputs__ = torch::huber_loss(*self, *target, reduction, delta); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_huber_loss_backward(tensor *out__, tensor grad_output, tensor self, tensor target, int64_t reduction, double delta) { - PROTECT( - auto outputs__ = torch::huber_loss_backward(*grad_output, *self, *target, reduction, delta); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_huber_loss_backward_out(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor target, int64_t reduction, double delta) { - PROTECT( - auto outputs__ = torch::huber_loss_backward_out(*grad_input, *grad_output, *self, *target, reduction, delta); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_huber_loss_out(tensor *out__, tensor out, tensor self, tensor target, int64_t reduction, double delta) { - PROTECT( - auto outputs__ = torch::huber_loss_out(*out, *self, *target, reduction, delta); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hypot(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::hypot(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hypot_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->hypot_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_hypot_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::hypot_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_i0(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::i0(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_i0_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::i0_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_i0_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::i0_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_igamma(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::igamma(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_igamma_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->igamma_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_igamma_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::igamma_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_igammac(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::igammac(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_igammac_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->igammac_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_igammac_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::igammac_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_im2col(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *dilation_data, int dilation_len, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len) { - PROTECT( - auto outputs__ = torch::im2col(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(dilation_data, dilation_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_im2col_out(tensor *out__, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *dilation_data, int dilation_len, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len) { - PROTECT( - auto outputs__ = torch::im2col_out(*out, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(dilation_data, dilation_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_imag(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::imag(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index(tensor *out__, tensor self, tensor *indices_data, int indices_len) { - PROTECT( - auto outputs__ = torch::index(*self, of_carray_tensor_opt(indices_data, indices_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index_add(tensor *out__, tensor self, int64_t dim, tensor index, tensor source) { - PROTECT( - auto outputs__ = torch::index_add(*self, dim, *index, *source); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index_add_(tensor *out__, tensor self, int64_t dim, tensor index, tensor source) { - PROTECT( - auto outputs__ = self->index_add_(dim, *index, *source); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index_add_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, tensor source) { - PROTECT( - auto outputs__ = torch::index_add_out(*out, *self, dim, *index, *source); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index_copy(tensor *out__, tensor self, int64_t dim, tensor index, tensor source) { - PROTECT( - auto outputs__ = torch::index_copy(*self, dim, *index, *source); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index_copy_(tensor *out__, tensor self, int64_t dim, tensor index, tensor source) { - PROTECT( - auto outputs__ = self->index_copy_(dim, *index, *source); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index_copy_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, tensor source) { - PROTECT( - auto outputs__ = torch::index_copy_out(*out, *self, dim, *index, *source); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index_fill(tensor *out__, tensor self, int64_t dim, tensor index, scalar value) { - PROTECT( - auto outputs__ = torch::index_fill(*self, dim, *index, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index_fill_(tensor *out__, tensor self, int64_t dim, tensor index, scalar value) { - PROTECT( - auto outputs__ = self->index_fill_(dim, *index, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index_fill_int_scalar_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, scalar value) { - PROTECT( - auto outputs__ = torch::index_fill_out(*out, *self, dim, *index, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index_fill_int_tensor(tensor *out__, tensor self, int64_t dim, tensor index, tensor value) { - PROTECT( - auto outputs__ = torch::index_fill(*self, dim, *index, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index_fill_int_tensor_(tensor *out__, tensor self, int64_t dim, tensor index, tensor value) { - PROTECT( - auto outputs__ = self->index_fill_(dim, *index, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index_fill_int_tensor_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, tensor value) { - PROTECT( - auto outputs__ = torch::index_fill_out(*out, *self, dim, *index, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index_put(tensor *out__, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate) { - PROTECT( - auto outputs__ = torch::index_put(*self, of_carray_tensor_opt(indices_data, indices_len), *values, (bool)accumulate); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index_put_(tensor *out__, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate) { - PROTECT( - auto outputs__ = torch::index_put_(*self, of_carray_tensor_opt(indices_data, indices_len), *values, (bool)accumulate); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index_put_out(tensor *out__, tensor out, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate) { - PROTECT( - auto outputs__ = torch::index_put_out(*out, *self, of_carray_tensor_opt(indices_data, indices_len), *values, (bool)accumulate); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index_reduce(tensor *out__, tensor self, int64_t dim, tensor index, tensor source, char* reduce_ptr, int reduce_len, int include_self) { - PROTECT( - auto outputs__ = torch::index_reduce(*self, dim, *index, *source, std::string(reduce_ptr, reduce_len), (bool)include_self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index_reduce_(tensor *out__, tensor self, int64_t dim, tensor index, tensor source, char* reduce_ptr, int reduce_len, int include_self) { - PROTECT( - auto outputs__ = self->index_reduce_(dim, *index, *source, std::string(reduce_ptr, reduce_len), (bool)include_self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index_reduce_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, tensor source, char* reduce_ptr, int reduce_len, int include_self) { - PROTECT( - auto outputs__ = torch::index_reduce_out(*out, *self, dim, *index, *source, std::string(reduce_ptr, reduce_len), (bool)include_self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index_select(tensor *out__, tensor self, int64_t dim, tensor index) { - PROTECT( - auto outputs__ = torch::index_select(*self, dim, *index); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index_select_backward(tensor *out__, tensor grad, int64_t *self_sizes_data, int self_sizes_len, int64_t dim, tensor index) { - PROTECT( - auto outputs__ = torch::index_select_backward(*grad, torch::IntArrayRef(self_sizes_data, self_sizes_len), dim, *index); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index_select_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index) { - PROTECT( - auto outputs__ = torch::index_select_out(*out, *self, dim, *index); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_index_tensor_out(tensor *out__, tensor out, tensor self, tensor *indices_data, int indices_len) { - PROTECT( - auto outputs__ = torch::index_out(*out, *self, of_carray_tensor_opt(indices_data, indices_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_indices(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->indices(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_indices_copy(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::indices_copy(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_indices_copy_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::indices_copy_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_infinitely_differentiable_gelu_backward(tensor *out__, tensor grad, tensor self) { - PROTECT( - auto outputs__ = torch::infinitely_differentiable_gelu_backward(*grad, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_inner(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::inner(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_inner_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::inner_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_instance_norm(tensor *out__, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int use_input_stats, double momentum, double eps, int cudnn_enabled) { - PROTECT( - auto outputs__ = torch::instance_norm(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (bool)use_input_stats, momentum, eps, (bool)cudnn_enabled); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_int_repr(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::int_repr(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_int_repr_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::int_repr_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_inverse(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::inverse(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_inverse_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::inverse_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -int atg_is_coalesced(tensor self) { - PROTECT( - return self->is_coalesced(); - ) - return 0; -} - -int atg_is_complex(tensor self) { - PROTECT( - return torch::is_complex(*self); - ) - return 0; -} - -int atg_is_conj(tensor self) { - PROTECT( - return torch::is_conj(*self); - ) - return 0; -} - -int atg_is_distributed(tensor self) { - PROTECT( - return torch::is_distributed(*self); - ) - return 0; -} - -int atg_is_floating_point(tensor self) { - PROTECT( - return torch::is_floating_point(*self); - ) - return 0; -} - -int atg_is_inference(tensor self) { - PROTECT( - return torch::is_inference(*self); - ) - return 0; -} - -int atg_is_leaf(tensor self) { - PROTECT( - return self->is_leaf(); - ) - return 0; -} - -int atg_is_neg(tensor self) { - PROTECT( - return torch::is_neg(*self); - ) - return 0; -} - -int atg_is_nonzero(tensor self) { - PROTECT( - return torch::is_nonzero(*self); - ) - return 0; -} - -int atg_is_pinned(tensor self, int device) { - PROTECT( - return self->is_pinned(device_of_int(device)); - ) - return 0; -} - -int atg_is_same_size(tensor self, tensor other) { - PROTECT( - return torch::is_same_size(*self, *other); - ) - return 0; -} - -int atg_is_set_to(tensor self, tensor tensor) { - PROTECT( - return self->is_set_to(*tensor); - ) - return 0; -} - -int atg_is_signed(tensor self) { - PROTECT( - return torch::is_signed(*self); - ) - return 0; -} - -int atg_is_vulkan_available() { - PROTECT( - return torch::is_vulkan_available(); - ) - return 0; -} - -void atg_isclose(tensor *out__, tensor self, tensor other, double rtol, double atol, int equal_nan) { - PROTECT( - auto outputs__ = torch::isclose(*self, *other, rtol, atol, (bool)equal_nan); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_isfinite(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::isfinite(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_isin(tensor *out__, tensor elements, tensor test_elements, int assume_unique, int invert) { - PROTECT( - auto outputs__ = torch::isin(*elements, *test_elements, (bool)assume_unique, (bool)invert); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_isin_scalar_tensor(tensor *out__, scalar element, tensor test_elements, int assume_unique, int invert) { - PROTECT( - auto outputs__ = torch::isin(*element, *test_elements, (bool)assume_unique, (bool)invert); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_isin_scalar_tensor_out(tensor *out__, tensor out, scalar element, tensor test_elements, int assume_unique, int invert) { - PROTECT( - auto outputs__ = torch::isin_out(*out, *element, *test_elements, (bool)assume_unique, (bool)invert); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_isin_tensor_scalar(tensor *out__, tensor elements, scalar test_element, int assume_unique, int invert) { - PROTECT( - auto outputs__ = torch::isin(*elements, *test_element, (bool)assume_unique, (bool)invert); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_isin_tensor_scalar_out(tensor *out__, tensor out, tensor elements, scalar test_element, int assume_unique, int invert) { - PROTECT( - auto outputs__ = torch::isin_out(*out, *elements, *test_element, (bool)assume_unique, (bool)invert); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_isin_tensor_tensor_out(tensor *out__, tensor out, tensor elements, tensor test_elements, int assume_unique, int invert) { - PROTECT( - auto outputs__ = torch::isin_out(*out, *elements, *test_elements, (bool)assume_unique, (bool)invert); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_isinf(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::isinf(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_isinf_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::isinf_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_isnan(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::isnan(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_isnan_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::isnan_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_isneginf(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::isneginf(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_isneginf_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::isneginf_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_isposinf(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::isposinf(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_isposinf_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::isposinf_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_isreal(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::isreal(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_istft(tensor *out__, tensor self, int64_t n_fft, int64_t hop_length_v, uint8_t hop_length_null, int64_t win_length_v, uint8_t win_length_null, tensor window, int center, int normalized, int onesided, int64_t length_v, uint8_t length_null, int return_complex) { - PROTECT( - auto outputs__ = torch::istft(*self, n_fft, hop_length_null ? c10::nullopt : c10::optional(hop_length_v), win_length_null ? c10::nullopt : c10::optional(win_length_v), (window ? ::std::optional(*window) : ::std::nullopt), (bool)center, (bool)normalized, (bool)onesided, length_null ? c10::nullopt : c10::optional(length_v), (bool)return_complex); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_kaiser_window(tensor *out__, int64_t window_length, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::kaiser_window(window_length, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_kaiser_window_beta(tensor *out__, int64_t window_length, int periodic, double beta, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::kaiser_window(window_length, (bool)periodic, beta, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_kaiser_window_beta_out(tensor *out__, tensor out, int64_t window_length, int periodic, double beta) { - PROTECT( - auto outputs__ = torch::kaiser_window_out(*out, window_length, (bool)periodic, beta); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_kaiser_window_out(tensor *out__, tensor out, int64_t window_length) { - PROTECT( - auto outputs__ = torch::kaiser_window_out(*out, window_length); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_kaiser_window_periodic(tensor *out__, int64_t window_length, int periodic, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::kaiser_window(window_length, (bool)periodic, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_kaiser_window_periodic_out(tensor *out__, tensor out, int64_t window_length, int periodic) { - PROTECT( - auto outputs__ = torch::kaiser_window_out(*out, window_length, (bool)periodic); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_kl_div(tensor *out__, tensor self, tensor target, int64_t reduction, int log_target) { - PROTECT( - auto outputs__ = torch::kl_div(*self, *target, reduction, (bool)log_target); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_kron(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::kron(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_kron_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::kron_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_kthvalue(tensor *out__, tensor self, int64_t k, int64_t dim, int keepdim) { - PROTECT( - auto outputs__ = torch::kthvalue(*self, k, dim, (bool)keepdim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_kthvalue_values(tensor *out__, tensor values, tensor indices, tensor self, int64_t k, int64_t dim, int keepdim) { - PROTECT( - auto outputs__ = torch::kthvalue_out(*values, *indices, *self, k, dim, (bool)keepdim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_l1_loss(tensor *out__, tensor self, tensor target, int64_t reduction) { - PROTECT( - auto outputs__ = torch::l1_loss(*self, *target, reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_layer_norm(tensor *out__, tensor input, int64_t *normalized_shape_data, int normalized_shape_len, tensor weight, tensor bias, double eps, int cudnn_enable) { - PROTECT( - auto outputs__ = torch::layer_norm(*input, torch::IntArrayRef(normalized_shape_data, normalized_shape_len), (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), eps, (bool)cudnn_enable); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lcm(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::lcm(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lcm_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::lcm_(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lcm_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::lcm_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ldexp(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::ldexp(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ldexp_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::ldexp_(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ldexp_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::ldexp_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_le(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::le(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_le_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->le_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_le_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::le_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_le_tensor(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::le(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_le_tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->le_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_le_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::le_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_leaky_relu(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::leaky_relu(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_leaky_relu_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::leaky_relu_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_leaky_relu_backward(tensor *out__, tensor grad_output, tensor self, scalar negative_slope, int self_is_result) { - PROTECT( - auto outputs__ = torch::leaky_relu_backward(*grad_output, *self, *negative_slope, (bool)self_is_result); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_leaky_relu_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, scalar negative_slope, int self_is_result) { - PROTECT( - auto outputs__ = torch::leaky_relu_backward_out(*grad_input, *grad_output, *self, *negative_slope, (bool)self_is_result); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_leaky_relu_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::leaky_relu_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lerp(tensor *out__, tensor self, tensor end, scalar weight) { - PROTECT( - auto outputs__ = torch::lerp(*self, *end, *weight); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lerp_(tensor *out__, tensor self, tensor end, scalar weight) { - PROTECT( - auto outputs__ = self->lerp_(*end, *weight); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lerp_scalar_out(tensor *out__, tensor out, tensor self, tensor end, scalar weight) { - PROTECT( - auto outputs__ = torch::lerp_out(*out, *self, *end, *weight); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lerp_tensor(tensor *out__, tensor self, tensor end, tensor weight) { - PROTECT( - auto outputs__ = torch::lerp(*self, *end, *weight); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lerp_tensor_(tensor *out__, tensor self, tensor end, tensor weight) { - PROTECT( - auto outputs__ = self->lerp_(*end, *weight); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lerp_tensor_out(tensor *out__, tensor out, tensor self, tensor end, tensor weight) { - PROTECT( - auto outputs__ = torch::lerp_out(*out, *self, *end, *weight); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_less(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::less(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_less_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->less_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_less_equal(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::less_equal(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_less_equal_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->less_equal_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_less_equal_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::less_equal_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_less_equal_tensor(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::less_equal(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_less_equal_tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->less_equal_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_less_equal_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::less_equal_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_less_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::less_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_less_tensor(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::less(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_less_tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->less_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_less_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::less_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lgamma(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::lgamma(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lgamma_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->lgamma_(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lgamma_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::lgamma_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lift(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::lift(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lift_fresh(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::lift_fresh(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lift_fresh_copy(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::lift_fresh_copy(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lift_fresh_copy_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::lift_fresh_copy_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lift_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::lift_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_cholesky(tensor *out__, tensor self, int upper) { - PROTECT( - auto outputs__ = torch::linalg_cholesky(*self, (bool)upper); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_cholesky_ex(tensor *out__, tensor self, int upper, int check_errors) { - PROTECT( - auto outputs__ = torch::linalg_cholesky_ex(*self, (bool)upper, (bool)check_errors); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_linalg_cholesky_ex_l(tensor *out__, tensor L, tensor info, tensor self, int upper, int check_errors) { - PROTECT( - auto outputs__ = torch::linalg_cholesky_ex_out(*L, *info, *self, (bool)upper, (bool)check_errors); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_linalg_cholesky_out(tensor *out__, tensor out, tensor self, int upper) { - PROTECT( - auto outputs__ = torch::linalg_cholesky_out(*out, *self, (bool)upper); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_cond(tensor *out__, tensor self, scalar p) { - PROTECT( - auto outputs__ = torch::linalg_cond(*self, *p); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_cond_out(tensor *out__, tensor out, tensor self, scalar p) { - PROTECT( - auto outputs__ = torch::linalg_cond_out(*out, *self, *p); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_cond_p_str(tensor *out__, tensor self, char* p_ptr, int p_len) { - PROTECT( - auto outputs__ = torch::linalg_cond(*self, std::string(p_ptr, p_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_cond_p_str_out(tensor *out__, tensor out, tensor self, char* p_ptr, int p_len) { - PROTECT( - auto outputs__ = torch::linalg_cond_out(*out, *self, std::string(p_ptr, p_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_cross(tensor *out__, tensor self, tensor other, int64_t dim) { - PROTECT( - auto outputs__ = torch::linalg_cross(*self, *other, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_cross_out(tensor *out__, tensor out, tensor self, tensor other, int64_t dim) { - PROTECT( - auto outputs__ = torch::linalg_cross_out(*out, *self, *other, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_det(tensor *out__, tensor A) { - PROTECT( - auto outputs__ = torch::linalg_det(*A); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_det_out(tensor *out__, tensor out, tensor A) { - PROTECT( - auto outputs__ = torch::linalg_det_out(*out, *A); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_diagonal(tensor *out__, tensor A, int64_t offset, int64_t dim1, int64_t dim2) { - PROTECT( - auto outputs__ = torch::linalg_diagonal(*A, offset, dim1, dim2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_eig(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::linalg_eig(*self); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_linalg_eig_out(tensor *out__, tensor eigenvalues, tensor eigenvectors, tensor self) { - PROTECT( - auto outputs__ = torch::linalg_eig_out(*eigenvalues, *eigenvectors, *self); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_linalg_eigh(tensor *out__, tensor self, char* UPLO_ptr, int UPLO_len) { - PROTECT( - auto outputs__ = torch::linalg_eigh(*self, std::string(UPLO_ptr, UPLO_len)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_linalg_eigh_eigvals(tensor *out__, tensor eigvals, tensor eigvecs, tensor self, char* UPLO_ptr, int UPLO_len) { - PROTECT( - auto outputs__ = torch::linalg_eigh_out(*eigvals, *eigvecs, *self, std::string(UPLO_ptr, UPLO_len)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_linalg_eigvals(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::linalg_eigvals(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_eigvals_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::linalg_eigvals_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_eigvalsh(tensor *out__, tensor self, char* UPLO_ptr, int UPLO_len) { - PROTECT( - auto outputs__ = torch::linalg_eigvalsh(*self, std::string(UPLO_ptr, UPLO_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_eigvalsh_out(tensor *out__, tensor out, tensor self, char* UPLO_ptr, int UPLO_len) { - PROTECT( - auto outputs__ = torch::linalg_eigvalsh_out(*out, *self, std::string(UPLO_ptr, UPLO_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_householder_product(tensor *out__, tensor input, tensor tau) { - PROTECT( - auto outputs__ = torch::linalg_householder_product(*input, *tau); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_householder_product_out(tensor *out__, tensor out, tensor input, tensor tau) { - PROTECT( - auto outputs__ = torch::linalg_householder_product_out(*out, *input, *tau); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_inv(tensor *out__, tensor A) { - PROTECT( - auto outputs__ = torch::linalg_inv(*A); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_inv_ex(tensor *out__, tensor A, int check_errors) { - PROTECT( - auto outputs__ = torch::linalg_inv_ex(*A, (bool)check_errors); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_linalg_inv_ex_inverse(tensor *out__, tensor inverse, tensor info, tensor A, int check_errors) { - PROTECT( - auto outputs__ = torch::linalg_inv_ex_out(*inverse, *info, *A, (bool)check_errors); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_linalg_inv_out(tensor *out__, tensor out, tensor A) { - PROTECT( - auto outputs__ = torch::linalg_inv_out(*out, *A); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_ldl_factor(tensor *out__, tensor self, int hermitian) { - PROTECT( - auto outputs__ = torch::linalg_ldl_factor(*self, (bool)hermitian); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_linalg_ldl_factor_ex(tensor *out__, tensor self, int hermitian, int check_errors) { - PROTECT( - auto outputs__ = torch::linalg_ldl_factor_ex(*self, (bool)hermitian, (bool)check_errors); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_linalg_ldl_factor_ex_out(tensor *out__, tensor LD, tensor pivots, tensor info, tensor self, int hermitian, int check_errors) { - PROTECT( - auto outputs__ = torch::linalg_ldl_factor_ex_out(*LD, *pivots, *info, *self, (bool)hermitian, (bool)check_errors); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_linalg_ldl_factor_out(tensor *out__, tensor LD, tensor pivots, tensor self, int hermitian) { - PROTECT( - auto outputs__ = torch::linalg_ldl_factor_out(*LD, *pivots, *self, (bool)hermitian); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_linalg_ldl_solve(tensor *out__, tensor LD, tensor pivots, tensor B, int hermitian) { - PROTECT( - auto outputs__ = torch::linalg_ldl_solve(*LD, *pivots, *B, (bool)hermitian); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_ldl_solve_out(tensor *out__, tensor out, tensor LD, tensor pivots, tensor B, int hermitian) { - PROTECT( - auto outputs__ = torch::linalg_ldl_solve_out(*out, *LD, *pivots, *B, (bool)hermitian); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_lstsq(tensor *out__, tensor self, tensor b, double rcond_v, uint8_t rcond_null, char* driver_ptr, int driver_len) { - PROTECT( - auto outputs__ = torch::linalg_lstsq(*self, *b, rcond_null ? c10::nullopt : c10::optional(rcond_v), std::string(driver_ptr, driver_len)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg_linalg_lstsq_out(tensor *out__, tensor solution, tensor residuals, tensor rank, tensor singular_values, tensor self, tensor b, double rcond_v, uint8_t rcond_null, char* driver_ptr, int driver_len) { - PROTECT( - auto outputs__ = torch::linalg_lstsq_out(*solution, *residuals, *rank, *singular_values, *self, *b, rcond_null ? c10::nullopt : c10::optional(rcond_v), std::string(driver_ptr, driver_len)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg_linalg_lu(tensor *out__, tensor A, int pivot) { - PROTECT( - auto outputs__ = torch::linalg_lu(*A, (bool)pivot); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_linalg_lu_factor(tensor *out__, tensor A, int pivot) { - PROTECT( - auto outputs__ = torch::linalg_lu_factor(*A, (bool)pivot); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_linalg_lu_factor_ex(tensor *out__, tensor A, int pivot, int check_errors) { - PROTECT( - auto outputs__ = torch::linalg_lu_factor_ex(*A, (bool)pivot, (bool)check_errors); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_linalg_lu_factor_ex_out(tensor *out__, tensor LU, tensor pivots, tensor info, tensor A, int pivot, int check_errors) { - PROTECT( - auto outputs__ = torch::linalg_lu_factor_ex_out(*LU, *pivots, *info, *A, (bool)pivot, (bool)check_errors); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_linalg_lu_factor_out(tensor *out__, tensor LU, tensor pivots, tensor A, int pivot) { - PROTECT( - auto outputs__ = torch::linalg_lu_factor_out(*LU, *pivots, *A, (bool)pivot); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_linalg_lu_out(tensor *out__, tensor P, tensor L, tensor U, tensor A, int pivot) { - PROTECT( - auto outputs__ = torch::linalg_lu_out(*P, *L, *U, *A, (bool)pivot); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_linalg_lu_solve(tensor *out__, tensor LU, tensor pivots, tensor B, int left, int adjoint) { - PROTECT( - auto outputs__ = torch::linalg_lu_solve(*LU, *pivots, *B, (bool)left, (bool)adjoint); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_lu_solve_out(tensor *out__, tensor out, tensor LU, tensor pivots, tensor B, int left, int adjoint) { - PROTECT( - auto outputs__ = torch::linalg_lu_solve_out(*out, *LU, *pivots, *B, (bool)left, (bool)adjoint); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_matmul(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::linalg_matmul(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_matmul_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::linalg_matmul_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_matrix_exp(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::linalg_matrix_exp(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_matrix_exp_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::linalg_matrix_exp_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_matrix_power(tensor *out__, tensor self, int64_t n) { - PROTECT( - auto outputs__ = torch::linalg_matrix_power(*self, n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_matrix_power_out(tensor *out__, tensor out, tensor self, int64_t n) { - PROTECT( - auto outputs__ = torch::linalg_matrix_power_out(*out, *self, n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_matrix_rank(tensor *out__, tensor self, double tol, int hermitian) { - PROTECT( - auto outputs__ = torch::linalg_matrix_rank(*self, tol, (bool)hermitian); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_matrix_rank_atol_rtol_float(tensor *out__, tensor self, double atol_v, uint8_t atol_null, double rtol_v, uint8_t rtol_null, int hermitian) { - PROTECT( - auto outputs__ = torch::linalg_matrix_rank(*self, atol_null ? c10::nullopt : c10::optional(atol_v), rtol_null ? c10::nullopt : c10::optional(rtol_v), (bool)hermitian); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_matrix_rank_atol_rtol_float_out(tensor *out__, tensor out, tensor self, double atol_v, uint8_t atol_null, double rtol_v, uint8_t rtol_null, int hermitian) { - PROTECT( - auto outputs__ = torch::linalg_matrix_rank_out(*out, *self, atol_null ? c10::nullopt : c10::optional(atol_v), rtol_null ? c10::nullopt : c10::optional(rtol_v), (bool)hermitian); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_matrix_rank_atol_rtol_tensor(tensor *out__, tensor input, tensor atol, tensor rtol, int hermitian) { - PROTECT( - auto outputs__ = torch::linalg_matrix_rank(*input, (atol ? ::std::optional(*atol) : ::std::nullopt), (rtol ? ::std::optional(*rtol) : ::std::nullopt), (bool)hermitian); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_matrix_rank_atol_rtol_tensor_out(tensor *out__, tensor out, tensor input, tensor atol, tensor rtol, int hermitian) { - PROTECT( - auto outputs__ = torch::linalg_matrix_rank_out(*out, *input, (atol ? ::std::optional(*atol) : ::std::nullopt), (rtol ? ::std::optional(*rtol) : ::std::nullopt), (bool)hermitian); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_matrix_rank_out(tensor *out__, tensor out, tensor self, double tol, int hermitian) { - PROTECT( - auto outputs__ = torch::linalg_matrix_rank_out(*out, *self, tol, (bool)hermitian); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_matrix_rank_out_tol_tensor(tensor *out__, tensor out, tensor input, tensor tol, int hermitian) { - PROTECT( - auto outputs__ = torch::linalg_matrix_rank_out(*out, *input, *tol, (bool)hermitian); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_matrix_rank_tol_tensor(tensor *out__, tensor input, tensor tol, int hermitian) { - PROTECT( - auto outputs__ = torch::linalg_matrix_rank(*input, *tol, (bool)hermitian); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_multi_dot(tensor *out__, tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::linalg_multi_dot(of_carray_tensor(tensors_data, tensors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_multi_dot_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::linalg_multi_dot_out(*out, of_carray_tensor(tensors_data, tensors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_norm(tensor *out__, tensor self, scalar ord, int64_t *dim_data, int dim_len, int keepdim, int dtype) { - PROTECT( - auto outputs__ = torch::linalg_norm(*self, *ord, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_norm_ord_str(tensor *out__, tensor self, char* ord_ptr, int ord_len, int64_t *dim_data, int dim_len, int keepdim, int dtype) { - PROTECT( - auto outputs__ = torch::linalg_norm(*self, std::string(ord_ptr, ord_len), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_norm_ord_str_out(tensor *out__, tensor out, tensor self, char* ord_ptr, int ord_len, int64_t *dim_data, int dim_len, int keepdim, int dtype) { - PROTECT( - auto outputs__ = torch::linalg_norm_out(*out, *self, std::string(ord_ptr, ord_len), dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_norm_out(tensor *out__, tensor out, tensor self, scalar ord, int64_t *dim_data, int dim_len, int keepdim, int dtype) { - PROTECT( - auto outputs__ = torch::linalg_norm_out(*out, *self, *ord, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_pinv(tensor *out__, tensor self, double rcond, int hermitian) { - PROTECT( - auto outputs__ = torch::linalg_pinv(*self, rcond, (bool)hermitian); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_pinv_atol_rtol_float(tensor *out__, tensor self, double atol_v, uint8_t atol_null, double rtol_v, uint8_t rtol_null, int hermitian) { - PROTECT( - auto outputs__ = torch::linalg_pinv(*self, atol_null ? c10::nullopt : c10::optional(atol_v), rtol_null ? c10::nullopt : c10::optional(rtol_v), (bool)hermitian); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_pinv_atol_rtol_float_out(tensor *out__, tensor out, tensor self, double atol_v, uint8_t atol_null, double rtol_v, uint8_t rtol_null, int hermitian) { - PROTECT( - auto outputs__ = torch::linalg_pinv_out(*out, *self, atol_null ? c10::nullopt : c10::optional(atol_v), rtol_null ? c10::nullopt : c10::optional(rtol_v), (bool)hermitian); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_pinv_atol_rtol_tensor(tensor *out__, tensor self, tensor atol, tensor rtol, int hermitian) { - PROTECT( - auto outputs__ = torch::linalg_pinv(*self, (atol ? ::std::optional(*atol) : ::std::nullopt), (rtol ? ::std::optional(*rtol) : ::std::nullopt), (bool)hermitian); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_pinv_atol_rtol_tensor_out(tensor *out__, tensor out, tensor self, tensor atol, tensor rtol, int hermitian) { - PROTECT( - auto outputs__ = torch::linalg_pinv_out(*out, *self, (atol ? ::std::optional(*atol) : ::std::nullopt), (rtol ? ::std::optional(*rtol) : ::std::nullopt), (bool)hermitian); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_pinv_out(tensor *out__, tensor out, tensor self, double rcond, int hermitian) { - PROTECT( - auto outputs__ = torch::linalg_pinv_out(*out, *self, rcond, (bool)hermitian); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_pinv_out_rcond_tensor(tensor *out__, tensor out, tensor self, tensor rcond, int hermitian) { - PROTECT( - auto outputs__ = torch::linalg_pinv_out(*out, *self, *rcond, (bool)hermitian); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_pinv_rcond_tensor(tensor *out__, tensor self, tensor rcond, int hermitian) { - PROTECT( - auto outputs__ = torch::linalg_pinv(*self, *rcond, (bool)hermitian); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_qr(tensor *out__, tensor A, char* mode_ptr, int mode_len) { - PROTECT( - auto outputs__ = torch::linalg_qr(*A, std::string(mode_ptr, mode_len)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_linalg_qr_out(tensor *out__, tensor Q, tensor R, tensor A, char* mode_ptr, int mode_len) { - PROTECT( - auto outputs__ = torch::linalg_qr_out(*Q, *R, *A, std::string(mode_ptr, mode_len)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_linalg_slogdet(tensor *out__, tensor A) { - PROTECT( - auto outputs__ = torch::linalg_slogdet(*A); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_linalg_slogdet_out(tensor *out__, tensor sign, tensor logabsdet, tensor A) { - PROTECT( - auto outputs__ = torch::linalg_slogdet_out(*sign, *logabsdet, *A); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_linalg_solve(tensor *out__, tensor A, tensor B, int left) { - PROTECT( - auto outputs__ = torch::linalg_solve(*A, *B, (bool)left); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_solve_ex(tensor *out__, tensor A, tensor B, int left, int check_errors) { - PROTECT( - auto outputs__ = torch::linalg_solve_ex(*A, *B, (bool)left, (bool)check_errors); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_linalg_solve_ex_out(tensor *out__, tensor result, tensor info, tensor A, tensor B, int left, int check_errors) { - PROTECT( - auto outputs__ = torch::linalg_solve_ex_out(*result, *info, *A, *B, (bool)left, (bool)check_errors); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_linalg_solve_out(tensor *out__, tensor out, tensor A, tensor B, int left) { - PROTECT( - auto outputs__ = torch::linalg_solve_out(*out, *A, *B, (bool)left); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_solve_triangular(tensor *out__, tensor self, tensor B, int upper, int left, int unitriangular) { - PROTECT( - auto outputs__ = torch::linalg_solve_triangular(*self, *B, (bool)upper, (bool)left, (bool)unitriangular); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_solve_triangular_out(tensor *out__, tensor out, tensor self, tensor B, int upper, int left, int unitriangular) { - PROTECT( - auto outputs__ = torch::linalg_solve_triangular_out(*out, *self, *B, (bool)upper, (bool)left, (bool)unitriangular); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_svd(tensor *out__, tensor A, int full_matrices, char* driver_ptr, int driver_len) { - PROTECT( - auto outputs__ = torch::linalg_svd(*A, (bool)full_matrices, std::string(driver_ptr, driver_len)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_linalg_svd_u(tensor *out__, tensor U, tensor S, tensor Vh, tensor A, int full_matrices, char* driver_ptr, int driver_len) { - PROTECT( - auto outputs__ = torch::linalg_svd_out(*U, *S, *Vh, *A, (bool)full_matrices, std::string(driver_ptr, driver_len)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_linalg_svdvals(tensor *out__, tensor A, char* driver_ptr, int driver_len) { - PROTECT( - auto outputs__ = torch::linalg_svdvals(*A, std::string(driver_ptr, driver_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_svdvals_out(tensor *out__, tensor out, tensor A, char* driver_ptr, int driver_len) { - PROTECT( - auto outputs__ = torch::linalg_svdvals_out(*out, *A, std::string(driver_ptr, driver_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_tensorinv(tensor *out__, tensor self, int64_t ind) { - PROTECT( - auto outputs__ = torch::linalg_tensorinv(*self, ind); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_tensorinv_out(tensor *out__, tensor out, tensor self, int64_t ind) { - PROTECT( - auto outputs__ = torch::linalg_tensorinv_out(*out, *self, ind); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_tensorsolve(tensor *out__, tensor self, tensor other, int64_t *dims_data, int dims_len) { - PROTECT( - auto outputs__ = torch::linalg_tensorsolve(*self, *other, dims_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dims_data, dims_len))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_tensorsolve_out(tensor *out__, tensor out, tensor self, tensor other, int64_t *dims_data, int dims_len) { - PROTECT( - auto outputs__ = torch::linalg_tensorsolve_out(*out, *self, *other, dims_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dims_data, dims_len))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_vander(tensor *out__, tensor x, int64_t n_v, uint8_t n_null) { - PROTECT( - auto outputs__ = torch::linalg_vander(*x, n_null ? c10::nullopt : c10::optional(n_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_vecdot(tensor *out__, tensor x, tensor y, int64_t dim) { - PROTECT( - auto outputs__ = torch::linalg_vecdot(*x, *y, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linalg_vecdot_out(tensor *out__, tensor out, tensor x, tensor y, int64_t dim) { - PROTECT( - auto outputs__ = torch::linalg_vecdot_out(*out, *x, *y, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linear(tensor *out__, tensor input, tensor weight, tensor bias) { - PROTECT( - auto outputs__ = torch::linear(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linear_out(tensor *out__, tensor out, tensor input, tensor weight, tensor bias) { - PROTECT( - auto outputs__ = torch::linear_out(*out, *input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linspace(tensor *out__, scalar start, scalar end, int64_t steps, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::linspace(*start, *end, steps, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linspace_out(tensor *out__, tensor out, scalar start, scalar end, int64_t steps) { - PROTECT( - auto outputs__ = torch::linspace_out(*out, *start, *end, steps); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linspace_scalar_tensor(tensor *out__, scalar start, tensor end, int64_t steps, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::linspace(*start, *end, steps, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linspace_scalar_tensor_out(tensor *out__, tensor out, scalar start, tensor end, int64_t steps) { - PROTECT( - auto outputs__ = torch::linspace_out(*out, *start, *end, steps); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linspace_tensor_scalar(tensor *out__, tensor start, scalar end, int64_t steps, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::linspace(*start, *end, steps, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linspace_tensor_scalar_out(tensor *out__, tensor out, tensor start, scalar end, int64_t steps) { - PROTECT( - auto outputs__ = torch::linspace_out(*out, *start, *end, steps); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linspace_tensor_tensor(tensor *out__, tensor start, tensor end, int64_t steps, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::linspace(*start, *end, steps, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_linspace_tensor_tensor_out(tensor *out__, tensor out, tensor start, tensor end, int64_t steps) { - PROTECT( - auto outputs__ = torch::linspace_out(*out, *start, *end, steps); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_log(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::log(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_log10(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::log10(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_log10_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::log10_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_log10_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::log10_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_log1p(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::log1p(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_log1p_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::log1p_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_log1p_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::log1p_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_log2(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::log2(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_log2_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::log2_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_log2_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::log2_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_log_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::log_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_log_normal(tensor *out__, tensor self, double mean, double std) { - PROTECT( - auto outputs__ = torch::log_normal(*self, mean, std); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_log_normal_(tensor *out__, tensor self, double mean, double std) { - PROTECT( - auto outputs__ = self->log_normal_(mean, std); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_log_normal_out(tensor *out__, tensor out, tensor self, double mean, double std) { - PROTECT( - auto outputs__ = torch::log_normal_out(*out, *self, mean, std); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_log_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::log_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_log_sigmoid(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::log_sigmoid(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_log_sigmoid_backward(tensor *out__, tensor grad_output, tensor self, tensor buffer) { - PROTECT( - auto outputs__ = torch::log_sigmoid_backward(*grad_output, *self, *buffer); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_log_sigmoid_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor buffer) { - PROTECT( - auto outputs__ = torch::log_sigmoid_backward_out(*grad_input, *grad_output, *self, *buffer); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_log_sigmoid_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::log_sigmoid_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_log_softmax(tensor *out__, tensor self, int64_t dim, int dtype) { - PROTECT( - auto outputs__ = torch::log_softmax(*self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_log_softmax_int_out(tensor *out__, tensor out, tensor self, int64_t dim, int dtype) { - PROTECT( - auto outputs__ = torch::log_softmax_out(*out, *self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logaddexp(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::logaddexp(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logaddexp2(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::logaddexp2(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logaddexp2_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::logaddexp2_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logaddexp_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::logaddexp_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logcumsumexp(tensor *out__, tensor self, int64_t dim) { - PROTECT( - auto outputs__ = torch::logcumsumexp(*self, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logcumsumexp_out(tensor *out__, tensor out, tensor self, int64_t dim) { - PROTECT( - auto outputs__ = torch::logcumsumexp_out(*out, *self, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logdet(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::logdet(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logical_and(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::logical_and(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logical_and_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->logical_and_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logical_and_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::logical_and_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logical_not(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::logical_not(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logical_not_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->logical_not_(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logical_not_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::logical_not_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logical_or(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::logical_or(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logical_or_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->logical_or_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logical_or_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::logical_or_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logical_xor(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::logical_xor(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logical_xor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->logical_xor_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logical_xor_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::logical_xor_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logit(tensor *out__, tensor self, double eps_v, uint8_t eps_null) { - PROTECT( - auto outputs__ = torch::logit(*self, eps_null ? c10::nullopt : c10::optional(eps_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logit_(tensor *out__, tensor self, double eps_v, uint8_t eps_null) { - PROTECT( - auto outputs__ = torch::logit_(*self, eps_null ? c10::nullopt : c10::optional(eps_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logit_backward(tensor *out__, tensor grad_output, tensor self, double eps_v, uint8_t eps_null) { - PROTECT( - auto outputs__ = torch::logit_backward(*grad_output, *self, eps_null ? c10::nullopt : c10::optional(eps_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logit_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, double eps_v, uint8_t eps_null) { - PROTECT( - auto outputs__ = torch::logit_backward_out(*grad_input, *grad_output, *self, eps_null ? c10::nullopt : c10::optional(eps_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logit_out(tensor *out__, tensor out, tensor self, double eps_v, uint8_t eps_null) { - PROTECT( - auto outputs__ = torch::logit_out(*out, *self, eps_null ? c10::nullopt : c10::optional(eps_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logspace(tensor *out__, scalar start, scalar end, int64_t steps, double base, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::logspace(*start, *end, steps, base, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logspace_out(tensor *out__, tensor out, scalar start, scalar end, int64_t steps, double base) { - PROTECT( - auto outputs__ = torch::logspace_out(*out, *start, *end, steps, base); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logspace_scalar_tensor(tensor *out__, scalar start, tensor end, int64_t steps, double base, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::logspace(*start, *end, steps, base, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logspace_scalar_tensor_out(tensor *out__, tensor out, scalar start, tensor end, int64_t steps, double base) { - PROTECT( - auto outputs__ = torch::logspace_out(*out, *start, *end, steps, base); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logspace_tensor_scalar(tensor *out__, tensor start, scalar end, int64_t steps, double base, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::logspace(*start, *end, steps, base, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logspace_tensor_scalar_out(tensor *out__, tensor out, tensor start, scalar end, int64_t steps, double base) { - PROTECT( - auto outputs__ = torch::logspace_out(*out, *start, *end, steps, base); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logspace_tensor_tensor(tensor *out__, tensor start, tensor end, int64_t steps, double base, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::logspace(*start, *end, steps, base, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logspace_tensor_tensor_out(tensor *out__, tensor out, tensor start, tensor end, int64_t steps, double base) { - PROTECT( - auto outputs__ = torch::logspace_out(*out, *start, *end, steps, base); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logsumexp(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim) { - PROTECT( - auto outputs__ = torch::logsumexp(*self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_logsumexp_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim) { - PROTECT( - auto outputs__ = torch::logsumexp_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lstm(tensor *out__, tensor input, tensor *hx_data, int hx_len, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first) { - PROTECT( - auto outputs__ = torch::lstm(*input, of_carray_tensor(hx_data, hx_len), of_carray_tensor(params_data, params_len), (bool)has_biases, num_layers, dropout, (bool)train, (bool)bidirectional, (bool)batch_first); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_lstm_cell(tensor *out__, tensor input, tensor *hx_data, int hx_len, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh) { - PROTECT( - auto outputs__ = torch::lstm_cell(*input, of_carray_tensor(hx_data, hx_len), *w_ih, *w_hh, (b_ih ? ::std::optional(*b_ih) : ::std::nullopt), (b_hh ? ::std::optional(*b_hh) : ::std::nullopt)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_lstm_data(tensor *out__, tensor data, tensor batch_sizes, tensor *hx_data, int hx_len, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional) { - PROTECT( - auto outputs__ = torch::lstm(*data, *batch_sizes, of_carray_tensor(hx_data, hx_len), of_carray_tensor(params_data, params_len), (bool)has_biases, num_layers, dropout, (bool)train, (bool)bidirectional); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_lstm_mps_backward(tensor out0, tensor *out1_data, int out1_len, tensor *out2_data, int out2_len, tensor grad_y, tensor grad_hy, tensor grad_cy, tensor z_state, tensor cell_state_fwd, tensor input, tensor layersOutputs, tensor *hx_data, int hx_len, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first) { - PROTECT( - torch::lstm_mps_backward_out(*out0, of_carray_tensor(out1_data, out1_len), of_carray_tensor(out2_data, out2_len), (grad_y ? ::std::optional(*grad_y) : ::std::nullopt), (grad_hy ? ::std::optional(*grad_hy) : ::std::nullopt), (grad_cy ? ::std::optional(*grad_cy) : ::std::nullopt), *z_state, *cell_state_fwd, *input, *layersOutputs, of_carray_tensor(hx_data, hx_len), of_carray_tensor(params_data, params_len), (bool)has_biases, num_layers, dropout, (bool)train, (bool)bidirectional, (bool)batch_first); - ) -} - -void atg_lt(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::lt(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lt_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->lt_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lt_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::lt_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lt_tensor(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::lt(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lt_tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->lt_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lt_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::lt_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lu_solve(tensor *out__, tensor self, tensor LU_data, tensor LU_pivots) { - PROTECT( - auto outputs__ = torch::lu_solve(*self, *LU_data, *LU_pivots); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lu_solve_out(tensor *out__, tensor out, tensor self, tensor LU_data, tensor LU_pivots) { - PROTECT( - auto outputs__ = torch::lu_solve_out(*out, *self, *LU_data, *LU_pivots); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_lu_unpack(tensor *out__, tensor LU_data, tensor LU_pivots, int unpack_data, int unpack_pivots) { - PROTECT( - auto outputs__ = torch::lu_unpack(*LU_data, *LU_pivots, (bool)unpack_data, (bool)unpack_pivots); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_lu_unpack_out(tensor *out__, tensor P, tensor L, tensor U, tensor LU_data, tensor LU_pivots, int unpack_data, int unpack_pivots) { - PROTECT( - auto outputs__ = torch::lu_unpack_out(*P, *L, *U, *LU_data, *LU_pivots, (bool)unpack_data, (bool)unpack_pivots); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_margin_ranking_loss(tensor *out__, tensor input1, tensor input2, tensor target, double margin, int64_t reduction) { - PROTECT( - auto outputs__ = torch::margin_ranking_loss(*input1, *input2, *target, margin, reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_masked_fill(tensor *out__, tensor self, tensor mask, scalar value) { - PROTECT( - auto outputs__ = torch::masked_fill(*self, *mask, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_masked_fill_(tensor *out__, tensor self, tensor mask, scalar value) { - PROTECT( - auto outputs__ = self->masked_fill_(*mask, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_masked_fill_scalar_out(tensor *out__, tensor out, tensor self, tensor mask, scalar value) { - PROTECT( - auto outputs__ = torch::masked_fill_out(*out, *self, *mask, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_masked_fill_tensor(tensor *out__, tensor self, tensor mask, tensor value) { - PROTECT( - auto outputs__ = torch::masked_fill(*self, *mask, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_masked_fill_tensor_(tensor *out__, tensor self, tensor mask, tensor value) { - PROTECT( - auto outputs__ = self->masked_fill_(*mask, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_masked_fill_tensor_out(tensor *out__, tensor out, tensor self, tensor mask, tensor value) { - PROTECT( - auto outputs__ = torch::masked_fill_out(*out, *self, *mask, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_masked_scatter(tensor *out__, tensor self, tensor mask, tensor source) { - PROTECT( - auto outputs__ = torch::masked_scatter(*self, *mask, *source); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_masked_scatter_(tensor *out__, tensor self, tensor mask, tensor source) { - PROTECT( - auto outputs__ = self->masked_scatter_(*mask, *source); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_masked_scatter_backward(tensor *out__, tensor grad_output, tensor mask, int64_t *sizes_data, int sizes_len) { - PROTECT( - auto outputs__ = torch::masked_scatter_backward(*grad_output, *mask, torch::IntArrayRef(sizes_data, sizes_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_masked_scatter_out(tensor *out__, tensor out, tensor self, tensor mask, tensor source) { - PROTECT( - auto outputs__ = torch::masked_scatter_out(*out, *self, *mask, *source); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_masked_select(tensor *out__, tensor self, tensor mask) { - PROTECT( - auto outputs__ = torch::masked_select(*self, *mask); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_masked_select_backward(tensor *out__, tensor grad, tensor input, tensor mask) { - PROTECT( - auto outputs__ = torch::masked_select_backward(*grad, *input, *mask); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_masked_select_out(tensor *out__, tensor out, tensor self, tensor mask) { - PROTECT( - auto outputs__ = torch::masked_select_out(*out, *self, *mask); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_matmul(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::matmul(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_matmul_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::matmul_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_matrix_exp(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::matrix_exp(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_matrix_exp_backward(tensor *out__, tensor self, tensor grad) { - PROTECT( - auto outputs__ = torch::matrix_exp_backward(*self, *grad); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_matrix_h(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->matrix_H(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_matrix_power(tensor *out__, tensor self, int64_t n) { - PROTECT( - auto outputs__ = torch::matrix_power(*self, n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_matrix_power_out(tensor *out__, tensor out, tensor self, int64_t n) { - PROTECT( - auto outputs__ = torch::matrix_power_out(*out, *self, n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_max(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::max(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_max_dim(tensor *out__, tensor self, int64_t dim, int keepdim) { - PROTECT( - auto outputs__ = torch::max(*self, dim, (bool)keepdim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_max_dim_max(tensor *out__, tensor max, tensor max_values, tensor self, int64_t dim, int keepdim) { - PROTECT( - auto outputs__ = torch::max_out(*max, *max_values, *self, dim, (bool)keepdim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_max_other(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::max(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_max_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::max_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_max_pool1d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::max_pool1d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_max_pool1d_with_indices(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::max_pool1d_with_indices(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_max_pool2d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::max_pool2d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_max_pool2d_backward(tensor *out__, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::max_pool2d_backward(*grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_max_pool2d_backward_out(tensor *out__, tensor out, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::max_pool2d_backward_out(*out, *grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_max_pool2d_with_indices(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::max_pool2d_with_indices(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_max_pool2d_with_indices_backward(tensor *out__, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode, tensor indices) { - PROTECT( - auto outputs__ = torch::max_pool2d_with_indices_backward(*grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode, *indices); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_max_pool2d_with_indices_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode, tensor indices) { - PROTECT( - auto outputs__ = torch::max_pool2d_with_indices_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode, *indices); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_max_pool2d_with_indices_out(tensor *out__, tensor out, tensor indices, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::max_pool2d_with_indices_out(*out, *indices, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_max_pool3d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::max_pool3d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_max_pool3d_with_indices(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::max_pool3d_with_indices(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_max_pool3d_with_indices_backward(tensor *out__, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode, tensor indices) { - PROTECT( - auto outputs__ = torch::max_pool3d_with_indices_backward(*grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode, *indices); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_max_pool3d_with_indices_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode, tensor indices) { - PROTECT( - auto outputs__ = torch::max_pool3d_with_indices_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode, *indices); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_max_pool3d_with_indices_out(tensor *out__, tensor out, tensor indices, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::max_pool3d_with_indices_out(*out, *indices, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_max_unary_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::max_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_max_unpool2d(tensor *out__, tensor self, tensor indices, int64_t *output_size_data, int output_size_len) { - PROTECT( - auto outputs__ = torch::max_unpool2d(*self, *indices, torch::IntArrayRef(output_size_data, output_size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_max_unpool2d_out(tensor *out__, tensor out, tensor self, tensor indices, int64_t *output_size_data, int output_size_len) { - PROTECT( - auto outputs__ = torch::max_unpool2d_out(*out, *self, *indices, torch::IntArrayRef(output_size_data, output_size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_max_unpool3d(tensor *out__, tensor self, tensor indices, int64_t *output_size_data, int output_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::max_unpool3d(*self, *indices, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_max_unpool3d_out(tensor *out__, tensor out, tensor self, tensor indices, int64_t *output_size_data, int output_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::max_unpool3d_out(*out, *self, *indices, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_maximum(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::maximum(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_maximum_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::maximum_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mean(tensor *out__, tensor self, int dtype) { - PROTECT( - auto outputs__ = torch::mean(*self, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mean_dim(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { - PROTECT( - auto outputs__ = torch::mean(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mean_dtype_out(tensor *out__, tensor out, tensor self, int dtype) { - PROTECT( - auto outputs__ = torch::mean_out(*out, *self, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mean_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { - PROTECT( - auto outputs__ = torch::mean_out(*out, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_median(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::median(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_median_dim(tensor *out__, tensor self, int64_t dim, int keepdim) { - PROTECT( - auto outputs__ = torch::median(*self, dim, (bool)keepdim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_median_dim_values(tensor *out__, tensor values, tensor indices, tensor self, int64_t dim, int keepdim) { - PROTECT( - auto outputs__ = torch::median_out(*values, *indices, *self, dim, (bool)keepdim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_median_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::median_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -tensor *atg_meshgrid(tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::meshgrid(of_carray_tensor(tensors_data, tensors_len)); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -tensor *atg_meshgrid_indexing(tensor *tensors_data, int tensors_len, char* indexing_ptr, int indexing_len) { - PROTECT( - auto outputs__ = torch::meshgrid(of_carray_tensor(tensors_data, tensors_len), std::string(indexing_ptr, indexing_len)); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg_mh(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->mH(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_min(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::min(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_min_dim(tensor *out__, tensor self, int64_t dim, int keepdim) { - PROTECT( - auto outputs__ = torch::min(*self, dim, (bool)keepdim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_min_dim_min(tensor *out__, tensor min, tensor min_indices, tensor self, int64_t dim, int keepdim) { - PROTECT( - auto outputs__ = torch::min_out(*min, *min_indices, *self, dim, (bool)keepdim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_min_other(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::min(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_min_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::min_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_min_unary_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::min_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_minimum(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::minimum(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_minimum_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::minimum_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_miopen_batch_norm(tensor *out__, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double exponential_average_factor, double epsilon) { - PROTECT( - auto outputs__ = torch::miopen_batch_norm(*input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (bool)training, exponential_average_factor, epsilon); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_miopen_batch_norm_backward(tensor *out__, tensor input, tensor grad_output, tensor weight, tensor running_mean, tensor running_var, tensor save_mean, tensor save_var, double epsilon) { - PROTECT( - auto outputs__ = torch::miopen_batch_norm_backward(*input, *grad_output, *weight, (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (save_mean ? ::std::optional(*save_mean) : ::std::nullopt), (save_var ? ::std::optional(*save_var) : ::std::nullopt), epsilon); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_miopen_batch_norm_backward_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor input, tensor grad_output, tensor weight, tensor running_mean, tensor running_var, tensor save_mean, tensor save_var, double epsilon) { - PROTECT( - auto outputs__ = torch::miopen_batch_norm_backward_out(*out0, *out1, *out2, *input, *grad_output, *weight, (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (save_mean ? ::std::optional(*save_mean) : ::std::nullopt), (save_var ? ::std::optional(*save_var) : ::std::nullopt), epsilon); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_miopen_batch_norm_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double exponential_average_factor, double epsilon) { - PROTECT( - auto outputs__ = torch::miopen_batch_norm_out(*out0, *out1, *out2, *input, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (bool)training, exponential_average_factor, epsilon); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_miopen_convolution(tensor *out__, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic) { - PROTECT( - auto outputs__ = torch::miopen_convolution(*self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, (bool)benchmark, (bool)deterministic); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_miopen_convolution_add_relu(tensor *out__, tensor self, tensor weight, tensor z, scalar alpha, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::miopen_convolution_add_relu(*self, *weight, *z, *alpha, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_miopen_convolution_out(tensor *out__, tensor out, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic) { - PROTECT( - auto outputs__ = torch::miopen_convolution_out(*out, *self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, (bool)benchmark, (bool)deterministic); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_miopen_convolution_relu(tensor *out__, tensor self, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::miopen_convolution_relu(*self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_miopen_convolution_transpose(tensor *out__, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic) { - PROTECT( - auto outputs__ = torch::miopen_convolution_transpose(*self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, (bool)benchmark, (bool)deterministic); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_miopen_convolution_transpose_out(tensor *out__, tensor out, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic) { - PROTECT( - auto outputs__ = torch::miopen_convolution_transpose_out(*out, *self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, (bool)benchmark, (bool)deterministic); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_miopen_ctc_loss(tensor *out__, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int deterministic, int zero_infinity) { - PROTECT( - auto outputs__ = torch::miopen_ctc_loss(*log_probs, *targets, torch::IntArrayRef(input_lengths_data, input_lengths_len), torch::IntArrayRef(target_lengths_data, target_lengths_len), blank, (bool)deterministic, (bool)zero_infinity); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_miopen_ctc_loss_out(tensor *out__, tensor out0, tensor out1, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int deterministic, int zero_infinity) { - PROTECT( - auto outputs__ = torch::miopen_ctc_loss_out(*out0, *out1, *log_probs, *targets, torch::IntArrayRef(input_lengths_data, input_lengths_len), torch::IntArrayRef(target_lengths_data, target_lengths_len), blank, (bool)deterministic, (bool)zero_infinity); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_miopen_ctc_loss_tensor(tensor *out__, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank, int deterministic, int zero_infinity) { - PROTECT( - auto outputs__ = torch::miopen_ctc_loss(*log_probs, *targets, *input_lengths, *target_lengths, blank, (bool)deterministic, (bool)zero_infinity); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_miopen_depthwise_convolution(tensor *out__, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic) { - PROTECT( - auto outputs__ = torch::miopen_depthwise_convolution(*self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, (bool)benchmark, (bool)deterministic); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_miopen_depthwise_convolution_out(tensor *out__, tensor out, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic) { - PROTECT( - auto outputs__ = torch::miopen_depthwise_convolution_out(*out, *self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, (bool)benchmark, (bool)deterministic); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_miopen_rnn(tensor *out__, tensor input, tensor *weight_data, int weight_len, int64_t weight_stride0, tensor hx, tensor cx, int64_t mode, int64_t hidden_size, int64_t num_layers, int batch_first, double dropout, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, tensor dropout_state) { - PROTECT( - auto outputs__ = torch::miopen_rnn(*input, of_carray_tensor(weight_data, weight_len), weight_stride0, *hx, (cx ? ::std::optional(*cx) : ::std::nullopt), mode, hidden_size, num_layers, (bool)batch_first, dropout, (bool)train, (bool)bidirectional, torch::IntArrayRef(batch_sizes_data, batch_sizes_len), (dropout_state ? ::std::optional(*dropout_state) : ::std::nullopt)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - out__[4] = new torch::Tensor(std::get<4>(outputs__)); - ) -} - -void atg_miopen_rnn_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor out3, tensor out4, tensor input, tensor *weight_data, int weight_len, int64_t weight_stride0, tensor hx, tensor cx, int64_t mode, int64_t hidden_size, int64_t num_layers, int batch_first, double dropout, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, tensor dropout_state) { - PROTECT( - auto outputs__ = torch::miopen_rnn_out(*out0, *out1, *out2, *out3, *out4, *input, of_carray_tensor(weight_data, weight_len), weight_stride0, *hx, (cx ? ::std::optional(*cx) : ::std::nullopt), mode, hidden_size, num_layers, (bool)batch_first, dropout, (bool)train, (bool)bidirectional, torch::IntArrayRef(batch_sizes_data, batch_sizes_len), (dropout_state ? ::std::optional(*dropout_state) : ::std::nullopt)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - out__[4] = new torch::Tensor(std::get<4>(outputs__)); - ) -} - -void atg_mish(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::mish(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mish_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::mish_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mish_backward(tensor *out__, tensor grad_output, tensor self) { - PROTECT( - auto outputs__ = torch::mish_backward(*grad_output, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mish_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::mish_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_adaptive_avg_pool2d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len) { - PROTECT( - auto outputs__ = torch::mkldnn_adaptive_avg_pool2d(*self, torch::IntArrayRef(output_size_data, output_size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_adaptive_avg_pool2d_backward(tensor *out__, tensor grad_output, tensor self) { - PROTECT( - auto outputs__ = torch::mkldnn_adaptive_avg_pool2d_backward(*grad_output, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_adaptive_avg_pool2d_backward_out(tensor *out__, tensor out, tensor grad_output, tensor self) { - PROTECT( - auto outputs__ = torch::mkldnn_adaptive_avg_pool2d_backward_out(*out, *grad_output, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_adaptive_avg_pool2d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len) { - PROTECT( - auto outputs__ = torch::mkldnn_adaptive_avg_pool2d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_convolution(tensor *out__, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::mkldnn_convolution(*self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_convolution_out(tensor *out__, tensor out, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups) { - PROTECT( - auto outputs__ = torch::mkldnn_convolution_out(*out, *self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_linear(tensor *out__, tensor self, tensor weight, tensor bias) { - PROTECT( - auto outputs__ = torch::mkldnn_linear(*self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_linear_backward_input(tensor *out__, int64_t *input_size_data, int input_size_len, tensor grad_output, tensor weight) { - PROTECT( - auto outputs__ = torch::mkldnn_linear_backward_input(torch::IntArrayRef(input_size_data, input_size_len), *grad_output, *weight); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_linear_backward_input_out(tensor *out__, tensor out, int64_t *input_size_data, int input_size_len, tensor grad_output, tensor weight) { - PROTECT( - auto outputs__ = torch::mkldnn_linear_backward_input_out(*out, torch::IntArrayRef(input_size_data, input_size_len), *grad_output, *weight); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_linear_backward_weights(tensor *out__, tensor grad_output, tensor input, tensor weight, int bias_defined) { - PROTECT( - auto outputs__ = torch::mkldnn_linear_backward_weights(*grad_output, *input, *weight, (bool)bias_defined); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_mkldnn_linear_backward_weights_out(tensor *out__, tensor out0, tensor out1, tensor grad_output, tensor input, tensor weight, int bias_defined) { - PROTECT( - auto outputs__ = torch::mkldnn_linear_backward_weights_out(*out0, *out1, *grad_output, *input, *weight, (bool)bias_defined); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_mkldnn_linear_out(tensor *out__, tensor out, tensor self, tensor weight, tensor bias) { - PROTECT( - auto outputs__ = torch::mkldnn_linear_out(*out, *self, *weight, (bias ? ::std::optional(*bias) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_max_pool2d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::mkldnn_max_pool2d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_max_pool2d_backward(tensor *out__, tensor grad_output, tensor output, tensor input, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::mkldnn_max_pool2d_backward(*grad_output, *output, *input, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_max_pool2d_backward_out(tensor *out__, tensor out, tensor grad_output, tensor output, tensor input, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::mkldnn_max_pool2d_backward_out(*out, *grad_output, *output, *input, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_max_pool2d_out(tensor *out__, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::mkldnn_max_pool2d_out(*out, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_max_pool3d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::mkldnn_max_pool3d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_max_pool3d_backward(tensor *out__, tensor grad_output, tensor output, tensor input, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::mkldnn_max_pool3d_backward(*grad_output, *output, *input, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_max_pool3d_backward_out(tensor *out__, tensor out, tensor grad_output, tensor output, tensor input, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::mkldnn_max_pool3d_backward_out(*out, *grad_output, *output, *input, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_max_pool3d_out(tensor *out__, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::mkldnn_max_pool3d_out(*out, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_reorder_conv2d_weight(tensor *out__, tensor self, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int64_t *input_size_data, int input_size_len) { - PROTECT( - auto outputs__ = torch::mkldnn_reorder_conv2d_weight(*self, torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, input_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(input_size_data, input_size_len))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_reorder_conv2d_weight_out(tensor *out__, tensor out, tensor self, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int64_t *input_size_data, int input_size_len) { - PROTECT( - auto outputs__ = torch::mkldnn_reorder_conv2d_weight_out(*out, *self, torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, input_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(input_size_data, input_size_len))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_reorder_conv3d_weight(tensor *out__, tensor self, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int64_t *input_size_data, int input_size_len) { - PROTECT( - auto outputs__ = torch::mkldnn_reorder_conv3d_weight(*self, torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, input_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(input_size_data, input_size_len))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_reorder_conv3d_weight_out(tensor *out__, tensor out, tensor self, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int64_t *input_size_data, int input_size_len) { - PROTECT( - auto outputs__ = torch::mkldnn_reorder_conv3d_weight_out(*out, *self, torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(dilation_data, dilation_len), groups, input_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(input_size_data, input_size_len))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mkldnn_rnn_layer(tensor *out__, tensor input, tensor weight0, tensor weight1, tensor weight2, tensor weight3, tensor hx_, tensor cx_, int reverse, int64_t *batch_sizes_data, int batch_sizes_len, int64_t mode, int64_t hidden_size, int64_t num_layers, int has_biases, int bidirectional, int batch_first, int train) { - PROTECT( - auto outputs__ = torch::mkldnn_rnn_layer(*input, *weight0, *weight1, *weight2, *weight3, *hx_, *cx_, (bool)reverse, torch::IntArrayRef(batch_sizes_data, batch_sizes_len), mode, hidden_size, num_layers, (bool)has_biases, (bool)bidirectional, (bool)batch_first, (bool)train); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg_mkldnn_rnn_layer_backward(tensor *out__, tensor input, tensor weight1, tensor weight2, tensor weight3, tensor weight4, tensor hx_, tensor cx_tmp, tensor output, tensor hy_, tensor cy_, tensor grad_output, tensor grad_hy, tensor grad_cy, int reverse, int64_t mode, int64_t hidden_size, int64_t num_layers, int has_biases, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, int batch_first, tensor workspace) { - PROTECT( - auto outputs__ = torch::mkldnn_rnn_layer_backward(*input, *weight1, *weight2, *weight3, *weight4, *hx_, *cx_tmp, *output, *hy_, *cy_, (grad_output ? ::std::optional(*grad_output) : ::std::nullopt), (grad_hy ? ::std::optional(*grad_hy) : ::std::nullopt), (grad_cy ? ::std::optional(*grad_cy) : ::std::nullopt), (bool)reverse, mode, hidden_size, num_layers, (bool)has_biases, (bool)train, (bool)bidirectional, torch::IntArrayRef(batch_sizes_data, batch_sizes_len), (bool)batch_first, *workspace); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - out__[4] = new torch::Tensor(std::get<4>(outputs__)); - out__[5] = new torch::Tensor(std::get<5>(outputs__)); - out__[6] = new torch::Tensor(std::get<6>(outputs__)); - ) -} - -void atg_mkldnn_rnn_layer_backward_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor out3, tensor out4, tensor out5, tensor out6, tensor input, tensor weight1, tensor weight2, tensor weight3, tensor weight4, tensor hx_, tensor cx_tmp, tensor output, tensor hy_, tensor cy_, tensor grad_output, tensor grad_hy, tensor grad_cy, int reverse, int64_t mode, int64_t hidden_size, int64_t num_layers, int has_biases, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, int batch_first, tensor workspace) { - PROTECT( - auto outputs__ = torch::mkldnn_rnn_layer_backward_out(*out0, *out1, *out2, *out3, *out4, *out5, *out6, *input, *weight1, *weight2, *weight3, *weight4, *hx_, *cx_tmp, *output, *hy_, *cy_, (grad_output ? ::std::optional(*grad_output) : ::std::nullopt), (grad_hy ? ::std::optional(*grad_hy) : ::std::nullopt), (grad_cy ? ::std::optional(*grad_cy) : ::std::nullopt), (bool)reverse, mode, hidden_size, num_layers, (bool)has_biases, (bool)train, (bool)bidirectional, torch::IntArrayRef(batch_sizes_data, batch_sizes_len), (bool)batch_first, *workspace); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - out__[4] = new torch::Tensor(std::get<4>(outputs__)); - out__[5] = new torch::Tensor(std::get<5>(outputs__)); - out__[6] = new torch::Tensor(std::get<6>(outputs__)); - ) -} - -void atg_mkldnn_rnn_layer_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor out3, tensor input, tensor weight0, tensor weight1, tensor weight2, tensor weight3, tensor hx_, tensor cx_, int reverse, int64_t *batch_sizes_data, int batch_sizes_len, int64_t mode, int64_t hidden_size, int64_t num_layers, int has_biases, int bidirectional, int batch_first, int train) { - PROTECT( - auto outputs__ = torch::mkldnn_rnn_layer_out(*out0, *out1, *out2, *out3, *input, *weight0, *weight1, *weight2, *weight3, *hx_, *cx_, (bool)reverse, torch::IntArrayRef(batch_sizes_data, batch_sizes_len), mode, hidden_size, num_layers, (bool)has_biases, (bool)bidirectional, (bool)batch_first, (bool)train); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - out__[3] = new torch::Tensor(std::get<3>(outputs__)); - ) -} - -void atg_mm(tensor *out__, tensor self, tensor mat2) { - PROTECT( - auto outputs__ = torch::mm(*self, *mat2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mm_dtype(tensor *out__, tensor self, tensor mat2, int out_dtype) { - PROTECT( - auto outputs__ = torch::mm(*self, *mat2, at::ScalarType(out_dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mm_dtype_out(tensor *out__, tensor out, tensor self, tensor mat2, int out_dtype) { - PROTECT( - auto outputs__ = torch::mm_out(*out, *self, *mat2, at::ScalarType(out_dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mm_out(tensor *out__, tensor out, tensor self, tensor mat2) { - PROTECT( - auto outputs__ = torch::mm_out(*out, *self, *mat2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mode(tensor *out__, tensor self, int64_t dim, int keepdim) { - PROTECT( - auto outputs__ = torch::mode(*self, dim, (bool)keepdim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_mode_values(tensor *out__, tensor values, tensor indices, tensor self, int64_t dim, int keepdim) { - PROTECT( - auto outputs__ = torch::mode_out(*values, *indices, *self, dim, (bool)keepdim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_moveaxis(tensor *out__, tensor self, int64_t *source_data, int source_len, int64_t *destination_data, int destination_len) { - PROTECT( - auto outputs__ = torch::moveaxis(*self, torch::IntArrayRef(source_data, source_len), torch::IntArrayRef(destination_data, destination_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_moveaxis_int(tensor *out__, tensor self, int64_t source, int64_t destination) { - PROTECT( - auto outputs__ = torch::moveaxis(*self, source, destination); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_movedim(tensor *out__, tensor self, int64_t *source_data, int source_len, int64_t *destination_data, int destination_len) { - PROTECT( - auto outputs__ = torch::movedim(*self, torch::IntArrayRef(source_data, source_len), torch::IntArrayRef(destination_data, destination_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_movedim_int(tensor *out__, tensor self, int64_t source, int64_t destination) { - PROTECT( - auto outputs__ = torch::movedim(*self, source, destination); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mse_loss(tensor *out__, tensor self, tensor target, int64_t reduction) { - PROTECT( - auto outputs__ = torch::mse_loss(*self, *target, reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mse_loss_backward(tensor *out__, tensor grad_output, tensor self, tensor target, int64_t reduction) { - PROTECT( - auto outputs__ = torch::mse_loss_backward(*grad_output, *self, *target, reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mse_loss_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor target, int64_t reduction) { - PROTECT( - auto outputs__ = torch::mse_loss_backward_out(*grad_input, *grad_output, *self, *target, reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mse_loss_out(tensor *out__, tensor out, tensor self, tensor target, int64_t reduction) { - PROTECT( - auto outputs__ = torch::mse_loss_out(*out, *self, *target, reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_msort(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::msort(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_msort_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::msort_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mt(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->mT(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mul(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::mul(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mul_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->mul_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mul_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::mul_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mul_scalar(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::mul(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mul_scalar_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->mul_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mul_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::mul_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_multi_margin_loss_backward(tensor *out__, tensor grad_output, tensor self, tensor target, scalar p, scalar margin, tensor weight, int64_t reduction) { - PROTECT( - auto outputs__ = torch::multi_margin_loss_backward(*grad_output, *self, *target, *p, *margin, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_multi_margin_loss_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor target, scalar p, scalar margin, tensor weight, int64_t reduction) { - PROTECT( - auto outputs__ = torch::multi_margin_loss_backward_out(*grad_input, *grad_output, *self, *target, *p, *margin, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_multilabel_margin_loss(tensor *out__, tensor self, tensor target, int64_t reduction) { - PROTECT( - auto outputs__ = torch::multilabel_margin_loss(*self, *target, reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_multilabel_margin_loss_backward(tensor *out__, tensor grad_output, tensor self, tensor target, int64_t reduction, tensor is_target) { - PROTECT( - auto outputs__ = torch::multilabel_margin_loss_backward(*grad_output, *self, *target, reduction, *is_target); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_multilabel_margin_loss_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor target, int64_t reduction, tensor is_target) { - PROTECT( - auto outputs__ = torch::multilabel_margin_loss_backward_out(*grad_input, *grad_output, *self, *target, reduction, *is_target); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_multilabel_margin_loss_out(tensor *out__, tensor out, tensor self, tensor target, int64_t reduction) { - PROTECT( - auto outputs__ = torch::multilabel_margin_loss_out(*out, *self, *target, reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_multinomial(tensor *out__, tensor self, int64_t num_samples, int replacement) { - PROTECT( - auto outputs__ = torch::multinomial(*self, num_samples, (bool)replacement); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_multinomial_out(tensor *out__, tensor out, tensor self, int64_t num_samples, int replacement) { - PROTECT( - auto outputs__ = torch::multinomial_out(*out, *self, num_samples, (bool)replacement); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_multiply(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::multiply(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_multiply_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->multiply_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_multiply_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::multiply_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_multiply_scalar(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::multiply(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_multiply_scalar_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->multiply_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mv(tensor *out__, tensor self, tensor vec) { - PROTECT( - auto outputs__ = torch::mv(*self, *vec); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mv_out(tensor *out__, tensor out, tensor self, tensor vec) { - PROTECT( - auto outputs__ = torch::mv_out(*out, *self, *vec); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mvlgamma(tensor *out__, tensor self, int64_t p) { - PROTECT( - auto outputs__ = torch::mvlgamma(*self, p); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mvlgamma_(tensor *out__, tensor self, int64_t p) { - PROTECT( - auto outputs__ = self->mvlgamma_(p); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_mvlgamma_out(tensor *out__, tensor out, tensor self, int64_t p) { - PROTECT( - auto outputs__ = torch::mvlgamma_out(*out, *self, p); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nan_to_num(tensor *out__, tensor self, double nan_v, uint8_t nan_null, double posinf_v, uint8_t posinf_null, double neginf_v, uint8_t neginf_null) { - PROTECT( - auto outputs__ = torch::nan_to_num(*self, nan_null ? c10::nullopt : c10::optional(nan_v), posinf_null ? c10::nullopt : c10::optional(posinf_v), neginf_null ? c10::nullopt : c10::optional(neginf_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nan_to_num_(tensor *out__, tensor self, double nan_v, uint8_t nan_null, double posinf_v, uint8_t posinf_null, double neginf_v, uint8_t neginf_null) { - PROTECT( - auto outputs__ = torch::nan_to_num_(*self, nan_null ? c10::nullopt : c10::optional(nan_v), posinf_null ? c10::nullopt : c10::optional(posinf_v), neginf_null ? c10::nullopt : c10::optional(neginf_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nan_to_num_out(tensor *out__, tensor out, tensor self, double nan_v, uint8_t nan_null, double posinf_v, uint8_t posinf_null, double neginf_v, uint8_t neginf_null) { - PROTECT( - auto outputs__ = torch::nan_to_num_out(*out, *self, nan_null ? c10::nullopt : c10::optional(nan_v), posinf_null ? c10::nullopt : c10::optional(posinf_v), neginf_null ? c10::nullopt : c10::optional(neginf_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nanmean(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { - PROTECT( - auto outputs__ = torch::nanmean(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nanmean_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { - PROTECT( - auto outputs__ = torch::nanmean_out(*out, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nanmedian(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::nanmedian(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nanmedian_dim(tensor *out__, tensor self, int64_t dim, int keepdim) { - PROTECT( - auto outputs__ = torch::nanmedian(*self, dim, (bool)keepdim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_nanmedian_dim_values(tensor *out__, tensor values, tensor indices, tensor self, int64_t dim, int keepdim) { - PROTECT( - auto outputs__ = torch::nanmedian_out(*values, *indices, *self, dim, (bool)keepdim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_nanmedian_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::nanmedian_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nanquantile(tensor *out__, tensor self, tensor q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len) { - PROTECT( - auto outputs__ = torch::nanquantile(*self, *q, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim, std::string(interpolation_ptr, interpolation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nanquantile_out(tensor *out__, tensor out, tensor self, tensor q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len) { - PROTECT( - auto outputs__ = torch::nanquantile_out(*out, *self, *q, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim, std::string(interpolation_ptr, interpolation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nanquantile_scalar(tensor *out__, tensor self, double q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len) { - PROTECT( - auto outputs__ = torch::nanquantile(*self, q, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim, std::string(interpolation_ptr, interpolation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nanquantile_scalar_out(tensor *out__, tensor out, tensor self, double q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len) { - PROTECT( - auto outputs__ = torch::nanquantile_out(*out, *self, q, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim, std::string(interpolation_ptr, interpolation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nansum(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { - PROTECT( - auto outputs__ = torch::nansum(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nansum_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { - PROTECT( - auto outputs__ = torch::nansum_out(*out, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_narrow(tensor *out__, tensor self, int64_t dim, int64_t start, int64_t length) { - PROTECT( - auto outputs__ = torch::narrow(*self, dim, start, length); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_narrow_copy(tensor *out__, tensor self, int64_t dim, int64_t start, int64_t length) { - PROTECT( - auto outputs__ = torch::narrow_copy(*self, dim, start, length); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_narrow_copy_out(tensor *out__, tensor out, tensor self, int64_t dim, int64_t start, int64_t length) { - PROTECT( - auto outputs__ = torch::narrow_copy_out(*out, *self, dim, start, length); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_narrow_tensor(tensor *out__, tensor self, int64_t dim, tensor start, int64_t length) { - PROTECT( - auto outputs__ = torch::narrow(*self, dim, *start, length); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_native_batch_norm(tensor *out__, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps) { - PROTECT( - auto outputs__ = torch::native_batch_norm(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (bool)training, momentum, eps); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_native_batch_norm_out(tensor *out__, tensor out, tensor save_mean, tensor save_invstd, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps) { - PROTECT( - auto outputs__ = torch::native_batch_norm_out(*out, *save_mean, *save_invstd, *input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), (running_mean ? ::std::optional(*running_mean) : ::std::nullopt), (running_var ? ::std::optional(*running_var) : ::std::nullopt), (bool)training, momentum, eps); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_native_channel_shuffle(tensor *out__, tensor self, int64_t groups) { - PROTECT( - auto outputs__ = torch::native_channel_shuffle(*self, groups); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_native_dropout(tensor *out__, tensor input, double p, int train) { - PROTECT( - auto outputs__ = torch::native_dropout(*input, p, (bool)train); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_native_dropout_backward(tensor *out__, tensor grad_output, tensor mask, double scale) { - PROTECT( - auto outputs__ = torch::native_dropout_backward(*grad_output, *mask, scale); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_native_dropout_backward_out(tensor *out__, tensor out, tensor grad_output, tensor mask, double scale) { - PROTECT( - auto outputs__ = torch::native_dropout_backward_out(*out, *grad_output, *mask, scale); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_native_dropout_out(tensor *out__, tensor out0, tensor out1, tensor input, double p, int train) { - PROTECT( - auto outputs__ = torch::native_dropout_out(*out0, *out1, *input, p, (bool)train); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_native_group_norm(tensor *out__, tensor input, tensor weight, tensor bias, int64_t n, int64_t C, int64_t HxW, int64_t group, double eps) { - PROTECT( - auto outputs__ = torch::native_group_norm(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), n, C, HxW, group, eps); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_native_group_norm_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor input, tensor weight, tensor bias, int64_t n, int64_t C, int64_t HxW, int64_t group, double eps) { - PROTECT( - auto outputs__ = torch::native_group_norm_out(*out0, *out1, *out2, *input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), n, C, HxW, group, eps); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_native_layer_norm(tensor *out__, tensor input, int64_t *normalized_shape_data, int normalized_shape_len, tensor weight, tensor bias, double eps) { - PROTECT( - auto outputs__ = torch::native_layer_norm(*input, torch::IntArrayRef(normalized_shape_data, normalized_shape_len), (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), eps); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_native_layer_norm_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor input, int64_t *normalized_shape_data, int normalized_shape_len, tensor weight, tensor bias, double eps) { - PROTECT( - auto outputs__ = torch::native_layer_norm_out(*out0, *out1, *out2, *input, torch::IntArrayRef(normalized_shape_data, normalized_shape_len), (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), eps); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_native_norm(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::native_norm(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_native_norm_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::native_norm_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_native_norm_scalaropt_dim_dtype(tensor *out__, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim, int dtype) { - PROTECT( - auto outputs__ = torch::native_norm(*self, *p, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_native_norm_scalaropt_dim_dtype_out(tensor *out__, tensor out, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim, int dtype) { - PROTECT( - auto outputs__ = torch::native_norm_out(*out, *self, *p, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ne(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::ne(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ne_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->ne_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ne_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::ne_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ne_tensor(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::ne(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ne_tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->ne_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ne_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::ne_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_neg(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::neg(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_neg_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::neg_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_neg_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::neg_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_negative(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::negative(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_negative_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::negative_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_negative_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::negative_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nested_to_padded_tensor(tensor *out__, tensor self, double padding, int64_t *output_size_data, int output_size_len) { - PROTECT( - auto outputs__ = torch::nested_to_padded_tensor(*self, padding, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_new_empty(tensor *out__, tensor self, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = self->new_empty(torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_new_empty_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::new_empty_out(*out, *self, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_new_empty_strided(tensor *out__, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = self->new_empty_strided(torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_new_empty_strided_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len) { - PROTECT( - auto outputs__ = torch::new_empty_strided_out(*out, *self, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_new_full(tensor *out__, tensor self, int64_t *size_data, int size_len, scalar fill_value, int options_kind, int options_device) { - PROTECT( - auto outputs__ = self->new_full(torch::IntArrayRef(size_data, size_len), *fill_value, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_new_full_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len, scalar fill_value) { - PROTECT( - auto outputs__ = torch::new_full_out(*out, *self, torch::IntArrayRef(size_data, size_len), *fill_value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_new_ones(tensor *out__, tensor self, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = self->new_ones(torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_new_ones_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::new_ones_out(*out, *self, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_new_zeros(tensor *out__, tensor self, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = self->new_zeros(torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_new_zeros_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::new_zeros_out(*out, *self, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nextafter(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::nextafter(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nextafter_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->nextafter_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nextafter_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::nextafter_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nll_loss(tensor *out__, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index) { - PROTECT( - auto outputs__ = torch::nll_loss(*self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction, ignore_index); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nll_loss2d(tensor *out__, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index) { - PROTECT( - auto outputs__ = torch::nll_loss2d(*self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction, ignore_index); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nll_loss2d_backward(tensor *out__, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index, tensor total_weight) { - PROTECT( - auto outputs__ = torch::nll_loss2d_backward(*grad_output, *self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction, ignore_index, *total_weight); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nll_loss2d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index, tensor total_weight) { - PROTECT( - auto outputs__ = torch::nll_loss2d_backward_out(*grad_input, *grad_output, *self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction, ignore_index, *total_weight); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nll_loss2d_out(tensor *out__, tensor out, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index) { - PROTECT( - auto outputs__ = torch::nll_loss2d_out(*out, *self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction, ignore_index); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nll_loss_backward(tensor *out__, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index, tensor total_weight) { - PROTECT( - auto outputs__ = torch::nll_loss_backward(*grad_output, *self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction, ignore_index, *total_weight); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nll_loss_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index, tensor total_weight) { - PROTECT( - auto outputs__ = torch::nll_loss_backward_out(*grad_input, *grad_output, *self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction, ignore_index, *total_weight); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nll_loss_nd(tensor *out__, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index) { - PROTECT( - auto outputs__ = torch::nll_loss_nd(*self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction, ignore_index); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nll_loss_out(tensor *out__, tensor out, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index) { - PROTECT( - auto outputs__ = torch::nll_loss_out(*out, *self, *target, (weight ? ::std::optional(*weight) : ::std::nullopt), reduction, ignore_index); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nonzero(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::nonzero(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -tensor *atg_nonzero_numpy(tensor self) { - PROTECT( - auto outputs__ = torch::nonzero_numpy(*self); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg_nonzero_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::nonzero_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nonzero_static(tensor *out__, tensor self, int64_t size, int64_t fill_value) { - PROTECT( - auto outputs__ = torch::nonzero_static(*self, size, fill_value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nonzero_static_out(tensor *out__, tensor out, tensor self, int64_t size, int64_t fill_value) { - PROTECT( - auto outputs__ = torch::nonzero_static_out(*out, *self, size, fill_value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_norm(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::norm(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_norm_dtype_out(tensor *out__, tensor out, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim, int dtype) { - PROTECT( - auto outputs__ = torch::norm_out(*out, *self, *p, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim, at::ScalarType(dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_norm_except_dim(tensor *out__, tensor v, int64_t pow, int64_t dim) { - PROTECT( - auto outputs__ = torch::norm_except_dim(*v, pow, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_norm_out(tensor *out__, tensor out, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim) { - PROTECT( - auto outputs__ = torch::norm_out(*out, *self, *p, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_norm_scalar_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::norm_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_norm_scalaropt_dim(tensor *out__, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim) { - PROTECT( - auto outputs__ = torch::norm(*self, *p, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_norm_scalaropt_dim_dtype(tensor *out__, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim, int dtype) { - PROTECT( - auto outputs__ = torch::norm(*self, *p, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim, at::ScalarType(dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_norm_scalaropt_dtype(tensor *out__, tensor self, scalar p, int dtype) { - PROTECT( - auto outputs__ = torch::norm(*self, *p, at::ScalarType(dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_norm_scalaropt_dtype_out(tensor *out__, tensor out, tensor self, scalar p, int dtype) { - PROTECT( - auto outputs__ = torch::norm_out(*out, *self, *p, at::ScalarType(dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_normal_(tensor *out__, tensor self, double mean, double std) { - PROTECT( - auto outputs__ = self->normal_(mean, std); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_normal_functional(tensor *out__, tensor self, double mean, double std) { - PROTECT( - auto outputs__ = torch::normal_functional(*self, mean, std); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_not_equal(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::not_equal(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_not_equal_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->not_equal_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_not_equal_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::not_equal_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_not_equal_tensor(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::not_equal(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_not_equal_tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->not_equal_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_not_equal_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::not_equal_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nuclear_norm(tensor *out__, tensor self, int keepdim) { - PROTECT( - auto outputs__ = torch::nuclear_norm(*self, (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nuclear_norm_dim(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim) { - PROTECT( - auto outputs__ = torch::nuclear_norm(*self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nuclear_norm_dim_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim) { - PROTECT( - auto outputs__ = torch::nuclear_norm_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_nuclear_norm_out(tensor *out__, tensor out, tensor self, int keepdim) { - PROTECT( - auto outputs__ = torch::nuclear_norm_out(*out, *self, (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_numpy_t(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->numpy_T(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_one_hot(tensor *out__, tensor self, int64_t num_classes) { - PROTECT( - auto outputs__ = torch::one_hot(*self, num_classes); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ones(tensor *out__, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::ones(torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ones_like(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::ones_like(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ones_like_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::ones_like_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ones_out(tensor *out__, tensor out, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::ones_out(*out, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_orgqr(tensor *out__, tensor self, tensor input2) { - PROTECT( - auto outputs__ = torch::orgqr(*self, *input2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_orgqr_out(tensor *out__, tensor out, tensor self, tensor input2) { - PROTECT( - auto outputs__ = torch::orgqr_out(*out, *self, *input2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ormqr(tensor *out__, tensor self, tensor input2, tensor input3, int left, int transpose) { - PROTECT( - auto outputs__ = torch::ormqr(*self, *input2, *input3, (bool)left, (bool)transpose); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ormqr_out(tensor *out__, tensor out, tensor self, tensor input2, tensor input3, int left, int transpose) { - PROTECT( - auto outputs__ = torch::ormqr_out(*out, *self, *input2, *input3, (bool)left, (bool)transpose); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_outer(tensor *out__, tensor self, tensor vec2) { - PROTECT( - auto outputs__ = torch::outer(*self, *vec2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_outer_out(tensor *out__, tensor out, tensor self, tensor vec2) { - PROTECT( - auto outputs__ = torch::outer_out(*out, *self, *vec2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -int64_t atg_output_nr(tensor self) { - PROTECT( - return self->output_nr(); - ) - return 0; -} - -void atg_pad(tensor *out__, tensor self, int64_t *pad_data, int pad_len, char* mode_ptr, int mode_len, double value_v, uint8_t value_null) { - PROTECT( - auto outputs__ = torch::pad(*self, torch::IntArrayRef(pad_data, pad_len), std::string(mode_ptr, mode_len), value_null ? c10::nullopt : c10::optional(value_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_pad_sequence(tensor *out__, tensor *sequences_data, int sequences_len, int batch_first, double padding_value, char* padding_side_ptr, int padding_side_len) { - PROTECT( - auto outputs__ = torch::pad_sequence(of_carray_tensor(sequences_data, sequences_len), (bool)batch_first, padding_value, std::string(padding_side_ptr, padding_side_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_pairwise_distance(tensor *out__, tensor x1, tensor x2, double p, double eps, int keepdim) { - PROTECT( - auto outputs__ = torch::pairwise_distance(*x1, *x2, p, eps, (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_pdist(tensor *out__, tensor self, double p) { - PROTECT( - auto outputs__ = torch::pdist(*self, p); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_permute(tensor *out__, tensor self, int64_t *dims_data, int dims_len) { - PROTECT( - auto outputs__ = torch::permute(*self, torch::IntArrayRef(dims_data, dims_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_permute_copy(tensor *out__, tensor self, int64_t *dims_data, int dims_len) { - PROTECT( - auto outputs__ = torch::permute_copy(*self, torch::IntArrayRef(dims_data, dims_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_permute_copy_out(tensor *out__, tensor out, tensor self, int64_t *dims_data, int dims_len) { - PROTECT( - auto outputs__ = torch::permute_copy_out(*out, *self, torch::IntArrayRef(dims_data, dims_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_pin_memory(tensor *out__, tensor self, int device) { - PROTECT( - auto outputs__ = self->pin_memory(device_of_int(device)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_pinverse(tensor *out__, tensor self, double rcond) { - PROTECT( - auto outputs__ = torch::pinverse(*self, rcond); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_pixel_shuffle(tensor *out__, tensor self, int64_t upscale_factor) { - PROTECT( - auto outputs__ = torch::pixel_shuffle(*self, upscale_factor); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_pixel_shuffle_out(tensor *out__, tensor out, tensor self, int64_t upscale_factor) { - PROTECT( - auto outputs__ = torch::pixel_shuffle_out(*out, *self, upscale_factor); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_pixel_unshuffle(tensor *out__, tensor self, int64_t downscale_factor) { - PROTECT( - auto outputs__ = torch::pixel_unshuffle(*self, downscale_factor); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_pixel_unshuffle_out(tensor *out__, tensor out, tensor self, int64_t downscale_factor) { - PROTECT( - auto outputs__ = torch::pixel_unshuffle_out(*out, *self, downscale_factor); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_poisson(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::poisson(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_poisson_nll_loss(tensor *out__, tensor input, tensor target, int log_input, int full, double eps, int64_t reduction) { - PROTECT( - auto outputs__ = torch::poisson_nll_loss(*input, *target, (bool)log_input, (bool)full, eps, reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_poisson_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::poisson_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_polar(tensor *out__, tensor abs, tensor angle) { - PROTECT( - auto outputs__ = torch::polar(*abs, *angle); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_polar_out(tensor *out__, tensor out, tensor abs, tensor angle) { - PROTECT( - auto outputs__ = torch::polar_out(*out, *abs, *angle); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_polygamma(tensor *out__, int64_t n, tensor self) { - PROTECT( - auto outputs__ = torch::polygamma(n, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_polygamma_(tensor *out__, tensor self, int64_t n) { - PROTECT( - auto outputs__ = self->polygamma_(n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_polygamma_out(tensor *out__, tensor out, int64_t n, tensor self) { - PROTECT( - auto outputs__ = torch::polygamma_out(*out, n, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_positive(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::positive(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_pow(tensor *out__, tensor self, tensor exponent) { - PROTECT( - auto outputs__ = torch::pow(*self, *exponent); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_pow_(tensor *out__, tensor self, scalar exponent) { - PROTECT( - auto outputs__ = self->pow_(*exponent); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_pow_scalar(tensor *out__, scalar self_scalar, tensor exponent) { - PROTECT( - auto outputs__ = torch::pow(*self_scalar, *exponent); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_pow_scalar_out(tensor *out__, tensor out, scalar self_scalar, tensor exponent) { - PROTECT( - auto outputs__ = torch::pow_out(*out, *self_scalar, *exponent); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_pow_tensor_(tensor *out__, tensor self, tensor exponent) { - PROTECT( - auto outputs__ = self->pow_(*exponent); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_pow_tensor_scalar(tensor *out__, tensor self, scalar exponent) { - PROTECT( - auto outputs__ = torch::pow(*self, *exponent); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_pow_tensor_scalar_out(tensor *out__, tensor out, tensor self, scalar exponent) { - PROTECT( - auto outputs__ = torch::pow_out(*out, *self, *exponent); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_pow_tensor_tensor_out(tensor *out__, tensor out, tensor self, tensor exponent) { - PROTECT( - auto outputs__ = torch::pow_out(*out, *self, *exponent); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_prelu(tensor *out__, tensor self, tensor weight) { - PROTECT( - auto outputs__ = torch::prelu(*self, *weight); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_prod(tensor *out__, tensor self, int dtype) { - PROTECT( - auto outputs__ = torch::prod(*self, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_prod_dim_int(tensor *out__, tensor self, int64_t dim, int keepdim, int dtype) { - PROTECT( - auto outputs__ = torch::prod(*self, dim, (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_prod_int_out(tensor *out__, tensor out, tensor self, int64_t dim, int keepdim, int dtype) { - PROTECT( - auto outputs__ = torch::prod_out(*out, *self, dim, (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_prod_out(tensor *out__, tensor out, tensor self, int dtype) { - PROTECT( - auto outputs__ = torch::prod_out(*out, *self, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_put(tensor *out__, tensor self, tensor index, tensor source, int accumulate) { - PROTECT( - auto outputs__ = torch::put(*self, *index, *source, (bool)accumulate); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_put_(tensor *out__, tensor self, tensor index, tensor source, int accumulate) { - PROTECT( - auto outputs__ = self->put_(*index, *source, (bool)accumulate); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_put_out(tensor *out__, tensor out, tensor self, tensor index, tensor source, int accumulate) { - PROTECT( - auto outputs__ = torch::put_out(*out, *self, *index, *source, (bool)accumulate); - out__[0] = new torch::Tensor(outputs__); - ) -} - -int64_t atg_q_per_channel_axis(tensor self) { - PROTECT( - return torch::q_per_channel_axis(*self); - ) - return 0; -} - -void atg_q_per_channel_scales(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::q_per_channel_scales(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_q_per_channel_scales_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::q_per_channel_scales_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_q_per_channel_zero_points(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::q_per_channel_zero_points(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_q_per_channel_zero_points_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::q_per_channel_zero_points_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -double atg_q_scale(tensor self) { - PROTECT( - return torch::q_scale(*self); - ) - return 0; -} - -int64_t atg_q_zero_point(tensor self) { - PROTECT( - return torch::q_zero_point(*self); - ) - return 0; -} - -void atg_qr(tensor *out__, tensor self, int some) { - PROTECT( - auto outputs__ = torch::qr(*self, (bool)some); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_qr_q(tensor *out__, tensor Q, tensor R, tensor self, int some) { - PROTECT( - auto outputs__ = torch::qr_out(*Q, *R, *self, (bool)some); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_quantile(tensor *out__, tensor self, tensor q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len) { - PROTECT( - auto outputs__ = torch::quantile(*self, *q, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim, std::string(interpolation_ptr, interpolation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_quantile_out(tensor *out__, tensor out, tensor self, tensor q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len) { - PROTECT( - auto outputs__ = torch::quantile_out(*out, *self, *q, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim, std::string(interpolation_ptr, interpolation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_quantile_scalar(tensor *out__, tensor self, double q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len) { - PROTECT( - auto outputs__ = torch::quantile(*self, q, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim, std::string(interpolation_ptr, interpolation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_quantile_scalar_out(tensor *out__, tensor out, tensor self, double q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len) { - PROTECT( - auto outputs__ = torch::quantile_out(*out, *self, q, dim_null ? c10::nullopt : c10::optional(dim_v), (bool)keepdim, std::string(interpolation_ptr, interpolation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_quantize_per_channel(tensor *out__, tensor self, tensor scales, tensor zero_points, int64_t axis, int dtype) { - PROTECT( - auto outputs__ = torch::quantize_per_channel(*self, *scales, *zero_points, axis, at::ScalarType(dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_quantize_per_channel_out(tensor *out__, tensor out, tensor self, tensor scales, tensor zero_points, int64_t axis, int dtype) { - PROTECT( - auto outputs__ = torch::quantize_per_channel_out(*out, *self, *scales, *zero_points, axis, at::ScalarType(dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_quantize_per_tensor(tensor *out__, tensor self, double scale, int64_t zero_point, int dtype) { - PROTECT( - auto outputs__ = torch::quantize_per_tensor(*self, scale, zero_point, at::ScalarType(dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_quantize_per_tensor_dynamic(tensor *out__, tensor self, int dtype, int reduce_range) { - PROTECT( - auto outputs__ = torch::quantize_per_tensor_dynamic(*self, at::ScalarType(dtype), (bool)reduce_range); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_quantize_per_tensor_dynamic_out(tensor *out__, tensor out, tensor self, int dtype, int reduce_range) { - PROTECT( - auto outputs__ = torch::quantize_per_tensor_dynamic_out(*out, *self, at::ScalarType(dtype), (bool)reduce_range); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_quantize_per_tensor_out(tensor *out__, tensor out, tensor self, double scale, int64_t zero_point, int dtype) { - PROTECT( - auto outputs__ = torch::quantize_per_tensor_out(*out, *self, scale, zero_point, at::ScalarType(dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_quantize_per_tensor_tensor_qparams(tensor *out__, tensor self, tensor scale, tensor zero_point, int dtype) { - PROTECT( - auto outputs__ = torch::quantize_per_tensor(*self, *scale, *zero_point, at::ScalarType(dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_quantize_per_tensor_tensor_qparams_out(tensor *out__, tensor out, tensor self, tensor scale, tensor zero_point, int dtype) { - PROTECT( - auto outputs__ = torch::quantize_per_tensor_out(*out, *self, *scale, *zero_point, at::ScalarType(dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -tensor *atg_quantize_per_tensor_tensors(tensor *tensors_data, int tensors_len, tensor scales, tensor zero_points, int dtype) { - PROTECT( - auto outputs__ = torch::quantize_per_tensor(of_carray_tensor(tensors_data, tensors_len), *scales, *zero_points, at::ScalarType(dtype)); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg_quantize_per_tensor_tensors_out(tensor *out_data, int out_len, tensor *tensors_data, int tensors_len, tensor scales, tensor zero_points, int dtype) { - PROTECT( - torch::quantize_per_tensor_out(of_carray_tensor(out_data, out_len), of_carray_tensor(tensors_data, tensors_len), *scales, *zero_points, at::ScalarType(dtype)); - ) -} - -void atg_quantized_batch_norm(tensor *out__, tensor input, tensor weight, tensor bias, tensor mean, tensor var, double eps, double output_scale, int64_t output_zero_point) { - PROTECT( - auto outputs__ = torch::quantized_batch_norm(*input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *mean, *var, eps, output_scale, output_zero_point); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_quantized_batch_norm_out(tensor *out__, tensor out, tensor input, tensor weight, tensor bias, tensor mean, tensor var, double eps, double output_scale, int64_t output_zero_point) { - PROTECT( - auto outputs__ = torch::quantized_batch_norm_out(*out, *input, (weight ? ::std::optional(*weight) : ::std::nullopt), (bias ? ::std::optional(*bias) : ::std::nullopt), *mean, *var, eps, output_scale, output_zero_point); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_quantized_gru_cell(tensor *out__, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh, tensor packed_ih, tensor packed_hh, tensor col_offsets_ih, tensor col_offsets_hh, scalar scale_ih, scalar scale_hh, scalar zero_point_ih, scalar zero_point_hh) { - PROTECT( - auto outputs__ = torch::quantized_gru_cell(*input, *hx, *w_ih, *w_hh, *b_ih, *b_hh, *packed_ih, *packed_hh, *col_offsets_ih, *col_offsets_hh, *scale_ih, *scale_hh, *zero_point_ih, *zero_point_hh); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_quantized_lstm_cell(tensor *out__, tensor input, tensor *hx_data, int hx_len, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh, tensor packed_ih, tensor packed_hh, tensor col_offsets_ih, tensor col_offsets_hh, scalar scale_ih, scalar scale_hh, scalar zero_point_ih, scalar zero_point_hh) { - PROTECT( - auto outputs__ = torch::quantized_lstm_cell(*input, of_carray_tensor(hx_data, hx_len), *w_ih, *w_hh, *b_ih, *b_hh, *packed_ih, *packed_hh, *col_offsets_ih, *col_offsets_hh, *scale_ih, *scale_hh, *zero_point_ih, *zero_point_hh); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_quantized_max_pool1d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::quantized_max_pool1d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_quantized_max_pool1d_out(tensor *out__, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::quantized_max_pool1d_out(*out, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_quantized_max_pool2d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::quantized_max_pool2d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_quantized_max_pool2d_out(tensor *out__, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::quantized_max_pool2d_out(*out, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_quantized_max_pool3d(tensor *out__, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::quantized_max_pool3d(*self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_quantized_max_pool3d_out(tensor *out__, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode) { - PROTECT( - auto outputs__ = torch::quantized_max_pool3d_out(*out, *self, torch::IntArrayRef(kernel_size_data, kernel_size_len), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len), (bool)ceil_mode); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_quantized_rnn_relu_cell(tensor *out__, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh, tensor packed_ih, tensor packed_hh, tensor col_offsets_ih, tensor col_offsets_hh, scalar scale_ih, scalar scale_hh, scalar zero_point_ih, scalar zero_point_hh) { - PROTECT( - auto outputs__ = torch::quantized_rnn_relu_cell(*input, *hx, *w_ih, *w_hh, *b_ih, *b_hh, *packed_ih, *packed_hh, *col_offsets_ih, *col_offsets_hh, *scale_ih, *scale_hh, *zero_point_ih, *zero_point_hh); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_quantized_rnn_tanh_cell(tensor *out__, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh, tensor packed_ih, tensor packed_hh, tensor col_offsets_ih, tensor col_offsets_hh, scalar scale_ih, scalar scale_hh, scalar zero_point_ih, scalar zero_point_hh) { - PROTECT( - auto outputs__ = torch::quantized_rnn_tanh_cell(*input, *hx, *w_ih, *w_hh, *b_ih, *b_hh, *packed_ih, *packed_hh, *col_offsets_ih, *col_offsets_hh, *scale_ih, *scale_hh, *zero_point_ih, *zero_point_hh); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rad2deg(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::rad2deg(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rad2deg_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::rad2deg_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rad2deg_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::rad2deg_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rand(tensor *out__, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::rand(torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rand_like(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::rand_like(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rand_like_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::rand_like_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rand_out(tensor *out__, tensor out, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::rand_out(*out, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_randint(tensor *out__, int64_t high, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::randint(high, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_randint_like(tensor *out__, tensor self, int64_t high) { - PROTECT( - auto outputs__ = torch::randint_like(*self, high); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_randint_like_low_dtype(tensor *out__, tensor self, int64_t low, int64_t high) { - PROTECT( - auto outputs__ = torch::randint_like(*self, low, high); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_randint_like_low_dtype_out(tensor *out__, tensor out, tensor self, int64_t low, int64_t high) { - PROTECT( - auto outputs__ = torch::randint_like_out(*out, *self, low, high); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_randint_like_out(tensor *out__, tensor out, tensor self, int64_t high) { - PROTECT( - auto outputs__ = torch::randint_like_out(*out, *self, high); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_randint_like_tensor(tensor *out__, tensor self, tensor high) { - PROTECT( - auto outputs__ = torch::randint_like(*self, *high); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_randint_like_tensor_out(tensor *out__, tensor out, tensor self, tensor high) { - PROTECT( - auto outputs__ = torch::randint_like_out(*out, *self, *high); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_randint_low(tensor *out__, int64_t low, int64_t high, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::randint(low, high, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_randint_low_out(tensor *out__, tensor out, int64_t low, int64_t high, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::randint_out(*out, low, high, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_randint_out(tensor *out__, tensor out, int64_t high, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::randint_out(*out, high, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_randn(tensor *out__, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::randn(torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_randn_like(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::randn_like(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_randn_like_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::randn_like_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_randn_out(tensor *out__, tensor out, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::randn_out(*out, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_random(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::random(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_random_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->random_(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_random_from(tensor *out__, tensor self, int64_t from, int64_t to_v, uint8_t to_null) { - PROTECT( - auto outputs__ = torch::random(*self, from, to_null ? c10::nullopt : c10::optional(to_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_random_from_(tensor *out__, tensor self, int64_t from, int64_t to_v, uint8_t to_null) { - PROTECT( - auto outputs__ = self->random_(from, to_null ? c10::nullopt : c10::optional(to_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_random_from_out(tensor *out__, tensor out, tensor self, int64_t from, int64_t to_v, uint8_t to_null) { - PROTECT( - auto outputs__ = torch::random_out(*out, *self, from, to_null ? c10::nullopt : c10::optional(to_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_random_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::random_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_random_to(tensor *out__, tensor self, int64_t to) { - PROTECT( - auto outputs__ = torch::random(*self, to); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_random_to_(tensor *out__, tensor self, int64_t to) { - PROTECT( - auto outputs__ = self->random_(to); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_random_to_out(tensor *out__, tensor out, tensor self, int64_t to) { - PROTECT( - auto outputs__ = torch::random_out(*out, *self, to); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_randperm(tensor *out__, int64_t n, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::randperm(n, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_randperm_out(tensor *out__, tensor out, int64_t n) { - PROTECT( - auto outputs__ = torch::randperm_out(*out, n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_range(tensor *out__, scalar start, scalar end, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::range(*start, *end, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_range_out(tensor *out__, tensor out, scalar start, scalar end) { - PROTECT( - auto outputs__ = torch::range_out(*out, *start, *end); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_range_out_(tensor *out__, tensor out, scalar start, scalar end) { - PROTECT( - auto outputs__ = torch::range_out(*out, *start, *end); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_range_step(tensor *out__, scalar start, scalar end, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::range(*start, *end, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_ravel(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::ravel(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_real(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::real(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_reciprocal(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::reciprocal(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_reciprocal_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::reciprocal_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_reciprocal_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::reciprocal_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_reflection_pad1d(tensor *out__, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::reflection_pad1d(*self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_reflection_pad1d_backward(tensor *out__, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::reflection_pad1d_backward(*grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_reflection_pad1d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::reflection_pad1d_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_reflection_pad1d_out(tensor *out__, tensor out, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::reflection_pad1d_out(*out, *self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_reflection_pad2d(tensor *out__, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::reflection_pad2d(*self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_reflection_pad2d_backward(tensor *out__, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::reflection_pad2d_backward(*grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_reflection_pad2d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::reflection_pad2d_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_reflection_pad2d_out(tensor *out__, tensor out, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::reflection_pad2d_out(*out, *self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_reflection_pad3d(tensor *out__, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::reflection_pad3d(*self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_reflection_pad3d_backward(tensor *out__, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::reflection_pad3d_backward(*grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_reflection_pad3d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::reflection_pad3d_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_reflection_pad3d_out(tensor *out__, tensor out, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::reflection_pad3d_out(*out, *self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_relu(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::relu(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_relu6(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::relu6(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_relu6_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::relu6_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_relu_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::relu_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_relu_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::relu_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_remainder(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::remainder(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_remainder_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->remainder_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_remainder_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::remainder_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_remainder_scalar_tensor(tensor *out__, scalar self_scalar, tensor other) { - PROTECT( - auto outputs__ = torch::remainder(*self_scalar, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_remainder_scalar_tensor_out(tensor *out__, tensor out, scalar self_scalar, tensor other) { - PROTECT( - auto outputs__ = torch::remainder_out(*out, *self_scalar, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_remainder_tensor(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::remainder(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_remainder_tensor_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->remainder_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_remainder_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::remainder_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_renorm(tensor *out__, tensor self, scalar p, int64_t dim, scalar maxnorm) { - PROTECT( - auto outputs__ = torch::renorm(*self, *p, dim, *maxnorm); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_renorm_(tensor *out__, tensor self, scalar p, int64_t dim, scalar maxnorm) { - PROTECT( - auto outputs__ = self->renorm_(*p, dim, *maxnorm); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_renorm_out(tensor *out__, tensor out, tensor self, scalar p, int64_t dim, scalar maxnorm) { - PROTECT( - auto outputs__ = torch::renorm_out(*out, *self, *p, dim, *maxnorm); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_repeat(tensor *out__, tensor self, int64_t *repeats_data, int repeats_len) { - PROTECT( - auto outputs__ = self->repeat(torch::IntArrayRef(repeats_data, repeats_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_repeat_interleave(tensor *out__, tensor repeats, int64_t output_size_v, uint8_t output_size_null) { - PROTECT( - auto outputs__ = torch::repeat_interleave(*repeats, output_size_null ? c10::nullopt : c10::optional(output_size_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_repeat_interleave_self_int(tensor *out__, tensor self, int64_t repeats, int64_t dim_v, uint8_t dim_null, int64_t output_size_v, uint8_t output_size_null) { - PROTECT( - auto outputs__ = torch::repeat_interleave(*self, repeats, dim_null ? c10::nullopt : c10::optional(dim_v), output_size_null ? c10::nullopt : c10::optional(output_size_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_repeat_interleave_self_tensor(tensor *out__, tensor self, tensor repeats, int64_t dim_v, uint8_t dim_null, int64_t output_size_v, uint8_t output_size_null) { - PROTECT( - auto outputs__ = torch::repeat_interleave(*self, *repeats, dim_null ? c10::nullopt : c10::optional(dim_v), output_size_null ? c10::nullopt : c10::optional(output_size_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_repeat_interleave_tensor_out(tensor *out__, tensor out, tensor repeats, int64_t output_size_v, uint8_t output_size_null) { - PROTECT( - auto outputs__ = torch::repeat_interleave_out(*out, *repeats, output_size_null ? c10::nullopt : c10::optional(output_size_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_repeat_out(tensor *out__, tensor out, tensor self, int64_t *repeats_data, int repeats_len) { - PROTECT( - auto outputs__ = torch::repeat_out(*out, *self, torch::IntArrayRef(repeats_data, repeats_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_replication_pad1d(tensor *out__, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::replication_pad1d(*self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_replication_pad1d_backward(tensor *out__, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::replication_pad1d_backward(*grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_replication_pad1d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::replication_pad1d_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_replication_pad1d_out(tensor *out__, tensor out, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::replication_pad1d_out(*out, *self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_replication_pad2d(tensor *out__, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::replication_pad2d(*self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_replication_pad2d_backward(tensor *out__, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::replication_pad2d_backward(*grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_replication_pad2d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::replication_pad2d_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_replication_pad2d_out(tensor *out__, tensor out, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::replication_pad2d_out(*out, *self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_replication_pad3d(tensor *out__, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::replication_pad3d(*self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_replication_pad3d_backward(tensor *out__, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::replication_pad3d_backward(*grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_replication_pad3d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::replication_pad3d_backward_out(*grad_input, *grad_output, *self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_replication_pad3d_out(tensor *out__, tensor out, tensor self, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::replication_pad3d_out(*out, *self, torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_requires_grad_(tensor *out__, tensor self, int requires_grad) { - PROTECT( - auto outputs__ = self->requires_grad_((bool)requires_grad); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_reshape(tensor *out__, tensor self, int64_t *shape_data, int shape_len) { - PROTECT( - auto outputs__ = torch::reshape(*self, torch::IntArrayRef(shape_data, shape_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_reshape_as(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->reshape_as(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_resize(tensor *out__, tensor self, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::resize(*self, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_resize_(tensor *out__, tensor self, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = self->resize_(torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_resize_as(tensor *out__, tensor self, tensor the_template) { - PROTECT( - auto outputs__ = torch::resize_as(*self, *the_template); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_resize_as_(tensor *out__, tensor self, tensor the_template) { - PROTECT( - auto outputs__ = torch::resize_as_(*self, *the_template); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_resize_as_out(tensor *out__, tensor out, tensor self, tensor the_template) { - PROTECT( - auto outputs__ = torch::resize_as_out(*out, *self, *the_template); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_resize_as_sparse(tensor *out__, tensor self, tensor the_template) { - PROTECT( - auto outputs__ = torch::resize_as_sparse(*self, *the_template); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_resize_as_sparse_(tensor *out__, tensor self, tensor the_template) { - PROTECT( - auto outputs__ = torch::resize_as_sparse_(*self, *the_template); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_resize_as_sparse_out(tensor *out__, tensor out, tensor self, tensor the_template) { - PROTECT( - auto outputs__ = torch::resize_as_sparse_out(*out, *self, *the_template); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_resize_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::resize_out(*out, *self, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_resolve_conj(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::resolve_conj(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_resolve_neg(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::resolve_neg(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -int atg_retains_grad(tensor self) { - PROTECT( - return self->retains_grad(); - ) - return 0; -} - -void atg_rms_norm(tensor *out__, tensor input, int64_t *normalized_shape_data, int normalized_shape_len, tensor weight, double eps_v, uint8_t eps_null) { - PROTECT( - auto outputs__ = torch::rms_norm(*input, torch::IntArrayRef(normalized_shape_data, normalized_shape_len), (weight ? ::std::optional(*weight) : ::std::nullopt), eps_null ? c10::nullopt : c10::optional(eps_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rnn_relu(tensor *out__, tensor input, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first) { - PROTECT( - auto outputs__ = torch::rnn_relu(*input, *hx, of_carray_tensor(params_data, params_len), (bool)has_biases, num_layers, dropout, (bool)train, (bool)bidirectional, (bool)batch_first); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_rnn_relu_cell(tensor *out__, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh) { - PROTECT( - auto outputs__ = torch::rnn_relu_cell(*input, *hx, *w_ih, *w_hh, (b_ih ? ::std::optional(*b_ih) : ::std::nullopt), (b_hh ? ::std::optional(*b_hh) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rnn_relu_data(tensor *out__, tensor data, tensor batch_sizes, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional) { - PROTECT( - auto outputs__ = torch::rnn_relu(*data, *batch_sizes, *hx, of_carray_tensor(params_data, params_len), (bool)has_biases, num_layers, dropout, (bool)train, (bool)bidirectional); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_rnn_tanh(tensor *out__, tensor input, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first) { - PROTECT( - auto outputs__ = torch::rnn_tanh(*input, *hx, of_carray_tensor(params_data, params_len), (bool)has_biases, num_layers, dropout, (bool)train, (bool)bidirectional, (bool)batch_first); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_rnn_tanh_cell(tensor *out__, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh) { - PROTECT( - auto outputs__ = torch::rnn_tanh_cell(*input, *hx, *w_ih, *w_hh, (b_ih ? ::std::optional(*b_ih) : ::std::nullopt), (b_hh ? ::std::optional(*b_hh) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rnn_tanh_data(tensor *out__, tensor data, tensor batch_sizes, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional) { - PROTECT( - auto outputs__ = torch::rnn_tanh(*data, *batch_sizes, *hx, of_carray_tensor(params_data, params_len), (bool)has_biases, num_layers, dropout, (bool)train, (bool)bidirectional); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_roll(tensor *out__, tensor self, int64_t *shifts_data, int shifts_len, int64_t *dims_data, int dims_len) { - PROTECT( - auto outputs__ = torch::roll(*self, torch::IntArrayRef(shifts_data, shifts_len), torch::IntArrayRef(dims_data, dims_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_roll_out(tensor *out__, tensor out, tensor self, int64_t *shifts_data, int shifts_len, int64_t *dims_data, int dims_len) { - PROTECT( - auto outputs__ = torch::roll_out(*out, *self, torch::IntArrayRef(shifts_data, shifts_len), torch::IntArrayRef(dims_data, dims_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rot90(tensor *out__, tensor self, int64_t k, int64_t *dims_data, int dims_len) { - PROTECT( - auto outputs__ = torch::rot90(*self, k, torch::IntArrayRef(dims_data, dims_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rot90_out(tensor *out__, tensor out, tensor self, int64_t k, int64_t *dims_data, int dims_len) { - PROTECT( - auto outputs__ = torch::rot90_out(*out, *self, k, torch::IntArrayRef(dims_data, dims_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_round(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::round(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_round_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::round_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_round_decimals(tensor *out__, tensor self, int64_t decimals) { - PROTECT( - auto outputs__ = torch::round(*self, decimals); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_round_decimals_(tensor *out__, tensor self, int64_t decimals) { - PROTECT( - auto outputs__ = torch::round_(*self, decimals); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_round_decimals_out(tensor *out__, tensor out, tensor self, int64_t decimals) { - PROTECT( - auto outputs__ = torch::round_out(*out, *self, decimals); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_round_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::round_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_row_indices(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->row_indices(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_row_indices_copy(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::row_indices_copy(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_row_indices_copy_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::row_indices_copy_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_row_stack(tensor *out__, tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::row_stack(of_carray_tensor(tensors_data, tensors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_row_stack_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::row_stack_out(*out, of_carray_tensor(tensors_data, tensors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rrelu(tensor *out__, tensor self, int training) { - PROTECT( - auto outputs__ = torch::rrelu(*self, (bool)training); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rrelu_(tensor *out__, tensor self, int training) { - PROTECT( - auto outputs__ = torch::rrelu_(*self, (bool)training); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rrelu_with_noise(tensor *out__, tensor self, tensor noise, int training) { - PROTECT( - auto outputs__ = torch::rrelu_with_noise(*self, *noise, (bool)training); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rrelu_with_noise_(tensor *out__, tensor self, tensor noise, int training) { - PROTECT( - auto outputs__ = torch::rrelu_with_noise_(*self, *noise, (bool)training); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rrelu_with_noise_backward(tensor *out__, tensor grad_output, tensor self, tensor noise, scalar lower, scalar upper, int training, int self_is_result) { - PROTECT( - auto outputs__ = torch::rrelu_with_noise_backward(*grad_output, *self, *noise, *lower, *upper, (bool)training, (bool)self_is_result); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rrelu_with_noise_backward_out(tensor *out__, tensor out, tensor grad_output, tensor self, tensor noise, scalar lower, scalar upper, int training, int self_is_result) { - PROTECT( - auto outputs__ = torch::rrelu_with_noise_backward_out(*out, *grad_output, *self, *noise, *lower, *upper, (bool)training, (bool)self_is_result); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rrelu_with_noise_functional(tensor *out__, tensor self, tensor noise, int training) { - PROTECT( - auto outputs__ = torch::rrelu_with_noise_functional(*self, *noise, (bool)training); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_rrelu_with_noise_out(tensor *out__, tensor out, tensor self, tensor noise, int training) { - PROTECT( - auto outputs__ = torch::rrelu_with_noise_out(*out, *self, *noise, (bool)training); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rsqrt(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::rsqrt(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rsqrt_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::rsqrt_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rsqrt_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::rsqrt_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rsub(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::rsub(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rsub_scalar(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::rsub(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rsub_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::rsub_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_rsub_tensor_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::rsub_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_scalar_tensor(tensor *out__, scalar s, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::scalar_tensor(*s, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_scalar_tensor_out(tensor *out__, tensor out, scalar s) { - PROTECT( - auto outputs__ = torch::scalar_tensor_out(*out, *s); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_scaled_dot_product_attention(tensor *out__, tensor query, tensor key, tensor value, tensor attn_mask, double dropout_p, int is_causal, double scale_v, uint8_t scale_null, int enable_gqa) { - PROTECT( - auto outputs__ = torch::scaled_dot_product_attention(*query, *key, *value, (attn_mask ? ::std::optional(*attn_mask) : ::std::nullopt), dropout_p, (bool)is_causal, scale_null ? c10::nullopt : c10::optional(scale_v), (bool)enable_gqa); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_scatter(tensor *out__, tensor self, int64_t dim, tensor index, tensor src) { - PROTECT( - auto outputs__ = torch::scatter(*self, dim, *index, *src); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_scatter_(tensor *out__, tensor self, int64_t dim, tensor index, tensor src) { - PROTECT( - auto outputs__ = self->scatter_(dim, *index, *src); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_scatter_add(tensor *out__, tensor self, int64_t dim, tensor index, tensor src) { - PROTECT( - auto outputs__ = torch::scatter_add(*self, dim, *index, *src); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_scatter_add_(tensor *out__, tensor self, int64_t dim, tensor index, tensor src) { - PROTECT( - auto outputs__ = self->scatter_add_(dim, *index, *src); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_scatter_add_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, tensor src) { - PROTECT( - auto outputs__ = torch::scatter_add_out(*out, *self, dim, *index, *src); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_scatter_reduce(tensor *out__, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len) { - PROTECT( - auto outputs__ = torch::scatter(*self, dim, *index, *src, std::string(reduce_ptr, reduce_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_scatter_reduce_(tensor *out__, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len) { - PROTECT( - auto outputs__ = self->scatter_(dim, *index, *src, std::string(reduce_ptr, reduce_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_scatter_reduce_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len) { - PROTECT( - auto outputs__ = torch::scatter_out(*out, *self, dim, *index, *src, std::string(reduce_ptr, reduce_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_scatter_src_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, tensor src) { - PROTECT( - auto outputs__ = torch::scatter_out(*out, *self, dim, *index, *src); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_scatter_value(tensor *out__, tensor self, int64_t dim, tensor index, scalar value) { - PROTECT( - auto outputs__ = torch::scatter(*self, dim, *index, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_scatter_value_(tensor *out__, tensor self, int64_t dim, tensor index, scalar value) { - PROTECT( - auto outputs__ = self->scatter_(dim, *index, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_scatter_value_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, scalar value) { - PROTECT( - auto outputs__ = torch::scatter_out(*out, *self, dim, *index, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_scatter_value_reduce(tensor *out__, tensor self, int64_t dim, tensor index, scalar value, char* reduce_ptr, int reduce_len) { - PROTECT( - auto outputs__ = torch::scatter(*self, dim, *index, *value, std::string(reduce_ptr, reduce_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_scatter_value_reduce_(tensor *out__, tensor self, int64_t dim, tensor index, scalar value, char* reduce_ptr, int reduce_len) { - PROTECT( - auto outputs__ = self->scatter_(dim, *index, *value, std::string(reduce_ptr, reduce_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_scatter_value_reduce_out(tensor *out__, tensor out, tensor self, int64_t dim, tensor index, scalar value, char* reduce_ptr, int reduce_len) { - PROTECT( - auto outputs__ = torch::scatter_out(*out, *self, dim, *index, *value, std::string(reduce_ptr, reduce_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_searchsorted(tensor *out__, tensor sorted_sequence, tensor self, int out_int32, int right, char* side_ptr, int side_len, tensor sorter) { - PROTECT( - auto outputs__ = torch::searchsorted(*sorted_sequence, *self, (bool)out_int32, (bool)right, std::string(side_ptr, side_len), (sorter ? ::std::optional(*sorter) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_searchsorted_scalar(tensor *out__, tensor sorted_sequence, scalar self_scalar, int out_int32, int right, char* side_ptr, int side_len, tensor sorter) { - PROTECT( - auto outputs__ = torch::searchsorted(*sorted_sequence, *self_scalar, (bool)out_int32, (bool)right, std::string(side_ptr, side_len), (sorter ? ::std::optional(*sorter) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_searchsorted_scalar_out(tensor *out__, tensor out, tensor sorted_sequence, scalar self_scalar, int out_int32, int right, char* side_ptr, int side_len, tensor sorter) { - PROTECT( - auto outputs__ = torch::searchsorted_out(*out, *sorted_sequence, *self_scalar, (bool)out_int32, (bool)right, std::string(side_ptr, side_len), (sorter ? ::std::optional(*sorter) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_searchsorted_tensor_out(tensor *out__, tensor out, tensor sorted_sequence, tensor self, int out_int32, int right, char* side_ptr, int side_len, tensor sorter) { - PROTECT( - auto outputs__ = torch::searchsorted_out(*out, *sorted_sequence, *self, (bool)out_int32, (bool)right, std::string(side_ptr, side_len), (sorter ? ::std::optional(*sorter) : ::std::nullopt)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_segment_reduce(tensor *out__, tensor data, char* reduce_ptr, int reduce_len, tensor lengths, tensor indices, tensor offsets, int64_t axis, int unsafe, scalar initial) { - PROTECT( - auto outputs__ = torch::segment_reduce(*data, std::string(reduce_ptr, reduce_len), (lengths ? ::std::optional(*lengths) : ::std::nullopt), (indices ? ::std::optional(*indices) : ::std::nullopt), (offsets ? ::std::optional(*offsets) : ::std::nullopt), axis, (bool)unsafe, *initial); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_segment_reduce_out(tensor *out__, tensor out, tensor data, char* reduce_ptr, int reduce_len, tensor lengths, tensor indices, tensor offsets, int64_t axis, int unsafe, scalar initial) { - PROTECT( - auto outputs__ = torch::segment_reduce_out(*out, *data, std::string(reduce_ptr, reduce_len), (lengths ? ::std::optional(*lengths) : ::std::nullopt), (indices ? ::std::optional(*indices) : ::std::nullopt), (offsets ? ::std::optional(*offsets) : ::std::nullopt), axis, (bool)unsafe, *initial); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_select(tensor *out__, tensor self, int64_t dim, int64_t index) { - PROTECT( - auto outputs__ = torch::select(*self, dim, index); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_select_backward(tensor *out__, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t index) { - PROTECT( - auto outputs__ = torch::select_backward(*grad_output, torch::IntArrayRef(input_sizes_data, input_sizes_len), dim, index); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_select_backward_out(tensor *out__, tensor out, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t index) { - PROTECT( - auto outputs__ = torch::select_backward_out(*out, *grad_output, torch::IntArrayRef(input_sizes_data, input_sizes_len), dim, index); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_select_copy(tensor *out__, tensor self, int64_t dim, int64_t index) { - PROTECT( - auto outputs__ = torch::select_copy(*self, dim, index); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_select_copy_int_out(tensor *out__, tensor out, tensor self, int64_t dim, int64_t index) { - PROTECT( - auto outputs__ = torch::select_copy_out(*out, *self, dim, index); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_select_scatter(tensor *out__, tensor self, tensor src, int64_t dim, int64_t index) { - PROTECT( - auto outputs__ = torch::select_scatter(*self, *src, dim, index); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_select_scatter_out(tensor *out__, tensor out, tensor self, tensor src, int64_t dim, int64_t index) { - PROTECT( - auto outputs__ = torch::select_scatter_out(*out, *self, *src, dim, index); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_selu(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::selu(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_selu_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::selu_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_set(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::set(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_set_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->set_(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_set_data(tensor self, tensor new_data) { - PROTECT( - self->set_data(*new_data); - ) -} - -void atg_set_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::set_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_set_requires_grad(tensor *out__, tensor self, int r) { - PROTECT( - auto outputs__ = self->set_requires_grad((bool)r); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_set_source_tensor(tensor *out__, tensor self, tensor source) { - PROTECT( - auto outputs__ = torch::set(*self, *source); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_set_source_tensor_(tensor *out__, tensor self, tensor source) { - PROTECT( - auto outputs__ = self->set_(*source); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_set_source_tensor_out(tensor *out__, tensor out, tensor self, tensor source) { - PROTECT( - auto outputs__ = torch::set_out(*out, *self, *source); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_set_source_tensor_storage_offset_(tensor *out__, tensor self, tensor source, int64_t storage_offset, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len) { - PROTECT( - auto outputs__ = self->set_(*source, storage_offset, torch::IntArrayRef(size_data, size_len), torch::IntArrayRef(stride_data, stride_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sgn(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::sgn(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sgn_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->sgn_(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sgn_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::sgn_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sigmoid(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::sigmoid(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sigmoid_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::sigmoid_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sigmoid_backward(tensor *out__, tensor grad_output, tensor output) { - PROTECT( - auto outputs__ = torch::sigmoid_backward(*grad_output, *output); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sigmoid_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor output) { - PROTECT( - auto outputs__ = torch::sigmoid_backward_out(*grad_input, *grad_output, *output); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sigmoid_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::sigmoid_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sign(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::sign(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sign_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->sign_(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sign_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::sign_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_signbit(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::signbit(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_signbit_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::signbit_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_silu(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::silu(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_silu_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::silu_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_silu_backward(tensor *out__, tensor grad_output, tensor self) { - PROTECT( - auto outputs__ = torch::silu_backward(*grad_output, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_silu_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self) { - PROTECT( - auto outputs__ = torch::silu_backward_out(*grad_input, *grad_output, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_silu_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::silu_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sin(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::sin(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sin_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::sin_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sin_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::sin_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sinc(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::sinc(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sinc_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::sinc_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sinc_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::sinc_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sinh(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::sinh(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sinh_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::sinh_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sinh_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::sinh_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_slice(tensor *out__, tensor self, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step) { - PROTECT( - auto outputs__ = torch::slice(*self, dim, start_null ? c10::nullopt : c10::optional(start_v), end_null ? c10::nullopt : c10::optional(end_v), step); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_slice_backward(tensor *out__, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t start, int64_t end, int64_t step) { - PROTECT( - auto outputs__ = torch::slice_backward(*grad_output, torch::IntArrayRef(input_sizes_data, input_sizes_len), dim, start, end, step); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_slice_backward_out(tensor *out__, tensor out, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t start, int64_t end, int64_t step) { - PROTECT( - auto outputs__ = torch::slice_backward_out(*out, *grad_output, torch::IntArrayRef(input_sizes_data, input_sizes_len), dim, start, end, step); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_slice_copy(tensor *out__, tensor self, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step) { - PROTECT( - auto outputs__ = torch::slice_copy(*self, dim, start_null ? c10::nullopt : c10::optional(start_v), end_null ? c10::nullopt : c10::optional(end_v), step); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_slice_copy_tensor_out(tensor *out__, tensor out, tensor self, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step) { - PROTECT( - auto outputs__ = torch::slice_copy_out(*out, *self, dim, start_null ? c10::nullopt : c10::optional(start_v), end_null ? c10::nullopt : c10::optional(end_v), step); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_slice_inverse(tensor *out__, tensor self, tensor src, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step) { - PROTECT( - auto outputs__ = torch::slice_inverse(*self, *src, dim, start_null ? c10::nullopt : c10::optional(start_v), end_null ? c10::nullopt : c10::optional(end_v), step); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_slice_scatter(tensor *out__, tensor self, tensor src, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step) { - PROTECT( - auto outputs__ = torch::slice_scatter(*self, *src, dim, start_null ? c10::nullopt : c10::optional(start_v), end_null ? c10::nullopt : c10::optional(end_v), step); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_slice_scatter_out(tensor *out__, tensor out, tensor self, tensor src, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step) { - PROTECT( - auto outputs__ = torch::slice_scatter_out(*out, *self, *src, dim, start_null ? c10::nullopt : c10::optional(start_v), end_null ? c10::nullopt : c10::optional(end_v), step); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_slogdet(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::slogdet(*self); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_slogdet_out(tensor *out__, tensor sign, tensor logabsdet, tensor self) { - PROTECT( - auto outputs__ = torch::slogdet_out(*sign, *logabsdet, *self); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_slow_conv3d(tensor *out__, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::slow_conv3d(*self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_slow_conv3d_out(tensor *out__, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len) { - PROTECT( - auto outputs__ = torch::slow_conv3d_out(*out, *self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_slow_conv_dilated2d(tensor *out__, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len) { - PROTECT( - auto outputs__ = torch::slow_conv_dilated2d(*self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_slow_conv_dilated2d_out(tensor *out__, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len) { - PROTECT( - auto outputs__ = torch::slow_conv_dilated2d_out(*out, *self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_slow_conv_dilated3d(tensor *out__, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len) { - PROTECT( - auto outputs__ = torch::slow_conv_dilated3d(*self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_slow_conv_dilated3d_out(tensor *out__, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len) { - PROTECT( - auto outputs__ = torch::slow_conv_dilated3d_out(*out, *self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(dilation_data, dilation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_slow_conv_transpose2d(tensor *out__, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *dilation_data, int dilation_len) { - PROTECT( - auto outputs__ = torch::slow_conv_transpose2d(*self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), torch::IntArrayRef(dilation_data, dilation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_slow_conv_transpose2d_out(tensor *out__, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *dilation_data, int dilation_len) { - PROTECT( - auto outputs__ = torch::slow_conv_transpose2d_out(*out, *self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), torch::IntArrayRef(dilation_data, dilation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_slow_conv_transpose3d(tensor *out__, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *dilation_data, int dilation_len) { - PROTECT( - auto outputs__ = torch::slow_conv_transpose3d(*self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), torch::IntArrayRef(dilation_data, dilation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_slow_conv_transpose3d_out(tensor *out__, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *dilation_data, int dilation_len) { - PROTECT( - auto outputs__ = torch::slow_conv_transpose3d_out(*out, *self, *weight, torch::IntArrayRef(kernel_size_data, kernel_size_len), (bias ? ::std::optional(*bias) : ::std::nullopt), torch::IntArrayRef(stride_data, stride_len), torch::IntArrayRef(padding_data, padding_len), torch::IntArrayRef(output_padding_data, output_padding_len), torch::IntArrayRef(dilation_data, dilation_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_smm(tensor *out__, tensor self, tensor mat2) { - PROTECT( - auto outputs__ = torch::smm(*self, *mat2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_smooth_l1_loss(tensor *out__, tensor self, tensor target, int64_t reduction, double beta) { - PROTECT( - auto outputs__ = torch::smooth_l1_loss(*self, *target, reduction, beta); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_smooth_l1_loss_backward(tensor *out__, tensor grad_output, tensor self, tensor target, int64_t reduction, double beta) { - PROTECT( - auto outputs__ = torch::smooth_l1_loss_backward(*grad_output, *self, *target, reduction, beta); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_smooth_l1_loss_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor target, int64_t reduction, double beta) { - PROTECT( - auto outputs__ = torch::smooth_l1_loss_backward_out(*grad_input, *grad_output, *self, *target, reduction, beta); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_smooth_l1_loss_out(tensor *out__, tensor out, tensor self, tensor target, int64_t reduction, double beta) { - PROTECT( - auto outputs__ = torch::smooth_l1_loss_out(*out, *self, *target, reduction, beta); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_soft_margin_loss(tensor *out__, tensor self, tensor target, int64_t reduction) { - PROTECT( - auto outputs__ = torch::soft_margin_loss(*self, *target, reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_soft_margin_loss_backward(tensor *out__, tensor grad_output, tensor self, tensor target, int64_t reduction) { - PROTECT( - auto outputs__ = torch::soft_margin_loss_backward(*grad_output, *self, *target, reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_soft_margin_loss_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, tensor target, int64_t reduction) { - PROTECT( - auto outputs__ = torch::soft_margin_loss_backward_out(*grad_input, *grad_output, *self, *target, reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_soft_margin_loss_out(tensor *out__, tensor out, tensor self, tensor target, int64_t reduction) { - PROTECT( - auto outputs__ = torch::soft_margin_loss_out(*out, *self, *target, reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_softmax(tensor *out__, tensor self, int64_t dim, int dtype) { - PROTECT( - auto outputs__ = torch::softmax(*self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_softmax_int_out(tensor *out__, tensor out, tensor self, int64_t dim, int dtype) { - PROTECT( - auto outputs__ = torch::softmax_out(*out, *self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_softplus(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::softplus(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_softplus_backward(tensor *out__, tensor grad_output, tensor self, scalar beta, scalar threshold) { - PROTECT( - auto outputs__ = torch::softplus_backward(*grad_output, *self, *beta, *threshold); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_softplus_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, scalar beta, scalar threshold) { - PROTECT( - auto outputs__ = torch::softplus_backward_out(*grad_input, *grad_output, *self, *beta, *threshold); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_softplus_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::softplus_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_softshrink(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::softshrink(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_softshrink_backward(tensor *out__, tensor grad_output, tensor self, scalar lambd) { - PROTECT( - auto outputs__ = torch::softshrink_backward(*grad_output, *self, *lambd); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_softshrink_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, scalar lambd) { - PROTECT( - auto outputs__ = torch::softshrink_backward_out(*grad_input, *grad_output, *self, *lambd); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_softshrink_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::softshrink_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sort(tensor *out__, tensor self, int64_t dim, int descending) { - PROTECT( - auto outputs__ = torch::sort(*self, dim, (bool)descending); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_sort_stable(tensor *out__, tensor self, int stable, int64_t dim, int descending) { - PROTECT( - auto outputs__ = torch::sort(*self, (bool)stable, dim, (bool)descending); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_sort_values(tensor *out__, tensor values, tensor indices, tensor self, int64_t dim, int descending) { - PROTECT( - auto outputs__ = torch::sort_out(*values, *indices, *self, dim, (bool)descending); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_sort_values_stable(tensor *out__, tensor values, tensor indices, tensor self, int stable, int64_t dim, int descending) { - PROTECT( - auto outputs__ = torch::sort_out(*values, *indices, *self, (bool)stable, dim, (bool)descending); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_sparse_bsc_tensor(tensor *out__, tensor ccol_indices, tensor row_indices, tensor values, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::sparse_bsc_tensor(*ccol_indices, *row_indices, *values, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sparse_bsc_tensor_ccol_row_value_size(tensor *out__, tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::sparse_bsc_tensor(*ccol_indices, *row_indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sparse_bsr_tensor(tensor *out__, tensor crow_indices, tensor col_indices, tensor values, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::sparse_bsr_tensor(*crow_indices, *col_indices, *values, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sparse_bsr_tensor_crow_col_value_size(tensor *out__, tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::sparse_bsr_tensor(*crow_indices, *col_indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sparse_compressed_tensor(tensor *out__, tensor compressed_indices, tensor plain_indices, tensor values, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::sparse_compressed_tensor(*compressed_indices, *plain_indices, *values, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sparse_compressed_tensor_comp_plain_value_size(tensor *out__, tensor compressed_indices, tensor plain_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::sparse_compressed_tensor(*compressed_indices, *plain_indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sparse_coo_tensor(tensor *out__, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::sparse_coo_tensor(torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sparse_coo_tensor_indices(tensor *out__, tensor indices, tensor values, int options_kind, int options_device, int is_coalesced) { - PROTECT( - auto outputs__ = torch::sparse_coo_tensor(*indices, *values, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind)), (bool)is_coalesced); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sparse_coo_tensor_indices_size(tensor *out__, tensor indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device, int is_coalesced) { - PROTECT( - auto outputs__ = torch::sparse_coo_tensor(*indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind)), (bool)is_coalesced); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sparse_coo_tensor_size_out(tensor *out__, tensor out, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::sparse_coo_tensor_out(*out, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sparse_csc_tensor(tensor *out__, tensor ccol_indices, tensor row_indices, tensor values, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::sparse_csc_tensor(*ccol_indices, *row_indices, *values, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sparse_csc_tensor_ccol_row_value_size(tensor *out__, tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::sparse_csc_tensor(*ccol_indices, *row_indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sparse_csr_tensor(tensor *out__, tensor crow_indices, tensor col_indices, tensor values, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::sparse_csr_tensor(*crow_indices, *col_indices, *values, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sparse_csr_tensor_crow_col_value_size(tensor *out__, tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::sparse_csr_tensor(*crow_indices, *col_indices, *values, torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -int64_t atg_sparse_dim(tensor self) { - PROTECT( - return self->sparse_dim(); - ) - return 0; -} - -void atg_sparse_mask(tensor *out__, tensor self, tensor mask) { - PROTECT( - auto outputs__ = self->sparse_mask(*mask); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sparse_mask_out(tensor *out__, tensor out, tensor self, tensor mask) { - PROTECT( - auto outputs__ = torch::sparse_mask_out(*out, *self, *mask); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sparse_resize(tensor *out__, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim) { - PROTECT( - auto outputs__ = torch::sparse_resize(*self, torch::IntArrayRef(size_data, size_len), sparse_dim, dense_dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sparse_resize_(tensor *out__, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim) { - PROTECT( - auto outputs__ = self->sparse_resize_(torch::IntArrayRef(size_data, size_len), sparse_dim, dense_dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sparse_resize_and_clear(tensor *out__, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim) { - PROTECT( - auto outputs__ = torch::sparse_resize_and_clear(*self, torch::IntArrayRef(size_data, size_len), sparse_dim, dense_dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sparse_resize_and_clear_(tensor *out__, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim) { - PROTECT( - auto outputs__ = self->sparse_resize_and_clear_(torch::IntArrayRef(size_data, size_len), sparse_dim, dense_dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sparse_resize_and_clear_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim) { - PROTECT( - auto outputs__ = torch::sparse_resize_and_clear_out(*out, *self, torch::IntArrayRef(size_data, size_len), sparse_dim, dense_dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sparse_resize_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim) { - PROTECT( - auto outputs__ = torch::sparse_resize_out(*out, *self, torch::IntArrayRef(size_data, size_len), sparse_dim, dense_dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sparse_sampled_addmm(tensor *out__, tensor self, tensor mat1, tensor mat2) { - PROTECT( - auto outputs__ = torch::sparse_sampled_addmm(*self, *mat1, *mat2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sparse_sampled_addmm_out(tensor *out__, tensor out, tensor self, tensor mat1, tensor mat2) { - PROTECT( - auto outputs__ = torch::sparse_sampled_addmm_out(*out, *self, *mat1, *mat2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_airy_ai(tensor *out__, tensor x) { - PROTECT( - auto outputs__ = torch::special_airy_ai(*x); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_airy_ai_out(tensor *out__, tensor out, tensor x) { - PROTECT( - auto outputs__ = torch::special_airy_ai_out(*out, *x); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_bessel_j0(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_bessel_j0(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_bessel_j0_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_bessel_j0_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_bessel_j1(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_bessel_j1(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_bessel_j1_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_bessel_j1_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_bessel_y0(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_bessel_y0(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_bessel_y0_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_bessel_y0_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_bessel_y1(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_bessel_y1(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_bessel_y1_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_bessel_y1_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_t(tensor *out__, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_t(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_t_n_scalar(tensor *out__, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_t(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_t_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_t_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_t_out(tensor *out__, tensor out, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_t_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_t_x_scalar(tensor *out__, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_t(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_t_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_t_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_u(tensor *out__, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_u(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_u_n_scalar(tensor *out__, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_u(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_u_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_u_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_u_out(tensor *out__, tensor out, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_u_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_u_x_scalar(tensor *out__, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_u(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_u_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_u_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_v(tensor *out__, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_v(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_v_n_scalar(tensor *out__, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_v(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_v_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_v_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_v_out(tensor *out__, tensor out, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_v_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_v_x_scalar(tensor *out__, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_v(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_v_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_v_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_w(tensor *out__, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_w(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_w_n_scalar(tensor *out__, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_w(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_w_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_w_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_w_out(tensor *out__, tensor out, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_w_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_w_x_scalar(tensor *out__, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_w(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_chebyshev_polynomial_w_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_chebyshev_polynomial_w_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_digamma(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_digamma(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_digamma_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_digamma_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_entr(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_entr(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_entr_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_entr_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_erf(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_erf(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_erf_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_erf_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_erfc(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_erfc(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_erfc_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_erfc_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_erfcx(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_erfcx(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_erfcx_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_erfcx_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_erfinv(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_erfinv(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_erfinv_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_erfinv_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_exp2(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_exp2(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_exp2_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_exp2_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_expit(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_expit(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_expit_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_expit_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_expm1(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_expm1(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_expm1_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_expm1_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_gammainc(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::special_gammainc(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_gammainc_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::special_gammainc_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_gammaincc(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::special_gammaincc(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_gammaincc_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::special_gammaincc_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_gammaln(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_gammaln(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_gammaln_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_gammaln_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_hermite_polynomial_h(tensor *out__, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_hermite_polynomial_h(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_hermite_polynomial_h_n_scalar(tensor *out__, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_hermite_polynomial_h(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_hermite_polynomial_h_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_hermite_polynomial_h_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_hermite_polynomial_h_out(tensor *out__, tensor out, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_hermite_polynomial_h_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_hermite_polynomial_h_x_scalar(tensor *out__, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_hermite_polynomial_h(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_hermite_polynomial_h_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_hermite_polynomial_h_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_hermite_polynomial_he(tensor *out__, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_hermite_polynomial_he(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_hermite_polynomial_he_n_scalar(tensor *out__, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_hermite_polynomial_he(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_hermite_polynomial_he_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_hermite_polynomial_he_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_hermite_polynomial_he_out(tensor *out__, tensor out, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_hermite_polynomial_he_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_hermite_polynomial_he_x_scalar(tensor *out__, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_hermite_polynomial_he(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_hermite_polynomial_he_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_hermite_polynomial_he_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_i0(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_i0(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_i0_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_i0_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_i0e(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_i0e(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_i0e_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_i0e_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_i1(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_i1(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_i1_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_i1_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_i1e(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_i1e(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_i1e_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_i1e_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_laguerre_polynomial_l(tensor *out__, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_laguerre_polynomial_l(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_laguerre_polynomial_l_n_scalar(tensor *out__, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_laguerre_polynomial_l(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_laguerre_polynomial_l_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_laguerre_polynomial_l_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_laguerre_polynomial_l_out(tensor *out__, tensor out, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_laguerre_polynomial_l_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_laguerre_polynomial_l_x_scalar(tensor *out__, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_laguerre_polynomial_l(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_laguerre_polynomial_l_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_laguerre_polynomial_l_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_legendre_polynomial_p(tensor *out__, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_legendre_polynomial_p(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_legendre_polynomial_p_n_scalar(tensor *out__, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_legendre_polynomial_p(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_legendre_polynomial_p_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_legendre_polynomial_p_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_legendre_polynomial_p_out(tensor *out__, tensor out, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_legendre_polynomial_p_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_legendre_polynomial_p_x_scalar(tensor *out__, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_legendre_polynomial_p(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_legendre_polynomial_p_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_legendre_polynomial_p_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_log1p(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_log1p(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_log1p_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_log1p_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_log_ndtr(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_log_ndtr(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_log_ndtr_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_log_ndtr_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_log_softmax(tensor *out__, tensor self, int64_t dim, int dtype) { - PROTECT( - auto outputs__ = torch::special_log_softmax(*self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_logit(tensor *out__, tensor self, double eps_v, uint8_t eps_null) { - PROTECT( - auto outputs__ = torch::special_logit(*self, eps_null ? c10::nullopt : c10::optional(eps_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_logit_out(tensor *out__, tensor out, tensor self, double eps_v, uint8_t eps_null) { - PROTECT( - auto outputs__ = torch::special_logit_out(*out, *self, eps_null ? c10::nullopt : c10::optional(eps_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_logsumexp(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim) { - PROTECT( - auto outputs__ = torch::special_logsumexp(*self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_logsumexp_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim) { - PROTECT( - auto outputs__ = torch::special_logsumexp_out(*out, *self, torch::IntArrayRef(dim_data, dim_len), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_modified_bessel_i0(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_modified_bessel_i0(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_modified_bessel_i0_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_modified_bessel_i0_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_modified_bessel_i1(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_modified_bessel_i1(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_modified_bessel_i1_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_modified_bessel_i1_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_modified_bessel_k0(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_modified_bessel_k0(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_modified_bessel_k0_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_modified_bessel_k0_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_modified_bessel_k1(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_modified_bessel_k1(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_modified_bessel_k1_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_modified_bessel_k1_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_multigammaln(tensor *out__, tensor self, int64_t p) { - PROTECT( - auto outputs__ = torch::special_multigammaln(*self, p); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_multigammaln_out(tensor *out__, tensor out, tensor self, int64_t p) { - PROTECT( - auto outputs__ = torch::special_multigammaln_out(*out, *self, p); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_ndtr(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_ndtr(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_ndtr_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_ndtr_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_ndtri(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_ndtri(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_ndtri_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_ndtri_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_polygamma(tensor *out__, int64_t n, tensor self) { - PROTECT( - auto outputs__ = torch::special_polygamma(n, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_polygamma_out(tensor *out__, tensor out, int64_t n, tensor self) { - PROTECT( - auto outputs__ = torch::special_polygamma_out(*out, n, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_psi(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_psi(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_psi_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_psi_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_round(tensor *out__, tensor self, int64_t decimals) { - PROTECT( - auto outputs__ = torch::special_round(*self, decimals); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_round_out(tensor *out__, tensor out, tensor self, int64_t decimals) { - PROTECT( - auto outputs__ = torch::special_round_out(*out, *self, decimals); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_scaled_modified_bessel_k0(tensor *out__, tensor x) { - PROTECT( - auto outputs__ = torch::special_scaled_modified_bessel_k0(*x); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_scaled_modified_bessel_k0_out(tensor *out__, tensor out, tensor x) { - PROTECT( - auto outputs__ = torch::special_scaled_modified_bessel_k0_out(*out, *x); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_scaled_modified_bessel_k1(tensor *out__, tensor x) { - PROTECT( - auto outputs__ = torch::special_scaled_modified_bessel_k1(*x); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_scaled_modified_bessel_k1_out(tensor *out__, tensor out, tensor x) { - PROTECT( - auto outputs__ = torch::special_scaled_modified_bessel_k1_out(*out, *x); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_t(tensor *out__, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_t(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_t_n_scalar(tensor *out__, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_t(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_t_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_t_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_t_out(tensor *out__, tensor out, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_t_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_t_x_scalar(tensor *out__, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_t(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_t_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_t_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_u(tensor *out__, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_u(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_u_n_scalar(tensor *out__, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_u(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_u_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_u_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_u_out(tensor *out__, tensor out, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_u_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_u_x_scalar(tensor *out__, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_u(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_u_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_u_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_v(tensor *out__, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_v(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_v_n_scalar(tensor *out__, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_v(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_v_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_v_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_v_out(tensor *out__, tensor out, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_v_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_v_x_scalar(tensor *out__, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_v(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_v_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_v_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_w(tensor *out__, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_w(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_w_n_scalar(tensor *out__, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_w(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_w_n_scalar_out(tensor *out__, tensor out, tensor x, scalar n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_w_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_w_out(tensor *out__, tensor out, tensor x, tensor n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_w_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_w_x_scalar(tensor *out__, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_w(*x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_shifted_chebyshev_polynomial_w_x_scalar_out(tensor *out__, tensor out, scalar x, tensor n) { - PROTECT( - auto outputs__ = torch::special_shifted_chebyshev_polynomial_w_out(*out, *x, *n); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_sinc(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::special_sinc(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_sinc_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::special_sinc_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_softmax(tensor *out__, tensor self, int64_t dim, int dtype) { - PROTECT( - auto outputs__ = torch::special_softmax(*self, dim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_spherical_bessel_j0(tensor *out__, tensor x) { - PROTECT( - auto outputs__ = torch::special_spherical_bessel_j0(*x); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_spherical_bessel_j0_out(tensor *out__, tensor out, tensor x) { - PROTECT( - auto outputs__ = torch::special_spherical_bessel_j0_out(*out, *x); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_xlog1py(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::special_xlog1py(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_xlog1py_other_scalar(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::special_xlog1py(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_xlog1py_other_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::special_xlog1py_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_xlog1py_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::special_xlog1py_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_xlog1py_self_scalar(tensor *out__, scalar self_scalar, tensor other) { - PROTECT( - auto outputs__ = torch::special_xlog1py(*self_scalar, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_xlog1py_self_scalar_out(tensor *out__, tensor out, scalar self_scalar, tensor other) { - PROTECT( - auto outputs__ = torch::special_xlog1py_out(*out, *self_scalar, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_xlogy(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::special_xlogy(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_xlogy_other_scalar(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::special_xlogy(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_xlogy_other_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::special_xlogy_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_xlogy_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::special_xlogy_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_xlogy_self_scalar(tensor *out__, scalar self_scalar, tensor other) { - PROTECT( - auto outputs__ = torch::special_xlogy(*self_scalar, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_xlogy_self_scalar_out(tensor *out__, tensor out, scalar self_scalar, tensor other) { - PROTECT( - auto outputs__ = torch::special_xlogy_out(*out, *self_scalar, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_zeta(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::special_zeta(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_zeta_other_scalar(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::special_zeta(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_zeta_other_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::special_zeta_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_zeta_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::special_zeta_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_zeta_self_scalar(tensor *out__, scalar self_scalar, tensor other) { - PROTECT( - auto outputs__ = torch::special_zeta(*self_scalar, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_special_zeta_self_scalar_out(tensor *out__, tensor out, scalar self_scalar, tensor other) { - PROTECT( - auto outputs__ = torch::special_zeta_out(*out, *self_scalar, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -tensor *atg_split(tensor self, int64_t split_size, int64_t dim) { - PROTECT( - auto outputs__ = torch::split(*self, split_size, dim); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -tensor *atg_split_copy(tensor self, int64_t split_size, int64_t dim) { - PROTECT( - auto outputs__ = torch::split_copy(*self, split_size, dim); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg_split_copy_tensor_out(tensor *out_data, int out_len, tensor self, int64_t split_size, int64_t dim) { - PROTECT( - torch::split_copy_out(of_carray_tensor(out_data, out_len), *self, split_size, dim); - ) -} - -tensor *atg_split_sizes(tensor self, int64_t *split_size_data, int split_size_len, int64_t dim) { - PROTECT( - auto outputs__ = torch::split(*self, torch::IntArrayRef(split_size_data, split_size_len), dim); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -tensor *atg_split_with_sizes(tensor self, int64_t *split_sizes_data, int split_sizes_len, int64_t dim) { - PROTECT( - auto outputs__ = torch::split_with_sizes(*self, torch::IntArrayRef(split_sizes_data, split_sizes_len), dim); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -tensor *atg_split_with_sizes_copy(tensor self, int64_t *split_sizes_data, int split_sizes_len, int64_t dim) { - PROTECT( - auto outputs__ = torch::split_with_sizes_copy(*self, torch::IntArrayRef(split_sizes_data, split_sizes_len), dim); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg_split_with_sizes_copy_out(tensor *out_data, int out_len, tensor self, int64_t *split_sizes_data, int split_sizes_len, int64_t dim) { - PROTECT( - torch::split_with_sizes_copy_out(of_carray_tensor(out_data, out_len), *self, torch::IntArrayRef(split_sizes_data, split_sizes_len), dim); - ) -} - -void atg_sqrt(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::sqrt(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sqrt_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::sqrt_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sqrt_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::sqrt_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_square(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::square(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_square_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::square_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_square_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::square_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_squeeze(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::squeeze(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_squeeze_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->squeeze_(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_squeeze_copy(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::squeeze_copy(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_squeeze_copy_dim(tensor *out__, tensor self, int64_t dim) { - PROTECT( - auto outputs__ = torch::squeeze_copy(*self, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_squeeze_copy_dim_out(tensor *out__, tensor out, tensor self, int64_t dim) { - PROTECT( - auto outputs__ = torch::squeeze_copy_out(*out, *self, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_squeeze_copy_dims(tensor *out__, tensor self, int64_t *dim_data, int dim_len) { - PROTECT( - auto outputs__ = torch::squeeze_copy(*self, torch::IntArrayRef(dim_data, dim_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_squeeze_copy_dims_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len) { - PROTECT( - auto outputs__ = torch::squeeze_copy_out(*out, *self, torch::IntArrayRef(dim_data, dim_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_squeeze_copy_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::squeeze_copy_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_squeeze_dim(tensor *out__, tensor self, int64_t dim) { - PROTECT( - auto outputs__ = torch::squeeze(*self, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_squeeze_dim_(tensor *out__, tensor self, int64_t dim) { - PROTECT( - auto outputs__ = self->squeeze_(dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_squeeze_dims(tensor *out__, tensor self, int64_t *dim_data, int dim_len) { - PROTECT( - auto outputs__ = torch::squeeze(*self, torch::IntArrayRef(dim_data, dim_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_squeeze_dims_(tensor *out__, tensor self, int64_t *dim_data, int dim_len) { - PROTECT( - auto outputs__ = self->squeeze_(torch::IntArrayRef(dim_data, dim_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sspaddmm(tensor *out__, tensor self, tensor mat1, tensor mat2) { - PROTECT( - auto outputs__ = torch::sspaddmm(*self, *mat1, *mat2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sspaddmm_out(tensor *out__, tensor out, tensor self, tensor mat1, tensor mat2) { - PROTECT( - auto outputs__ = torch::sspaddmm_out(*out, *self, *mat1, *mat2); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_stack(tensor *out__, tensor *tensors_data, int tensors_len, int64_t dim) { - PROTECT( - auto outputs__ = torch::stack(of_carray_tensor(tensors_data, tensors_len), dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_stack_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len, int64_t dim) { - PROTECT( - auto outputs__ = torch::stack_out(*out, of_carray_tensor(tensors_data, tensors_len), dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_std(tensor *out__, tensor self, int unbiased) { - PROTECT( - auto outputs__ = torch::std(*self, (bool)unbiased); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_std_correction(tensor *out__, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim) { - PROTECT( - auto outputs__ = torch::std(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), *correction, (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_std_correction_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim) { - PROTECT( - auto outputs__ = torch::std_out(*out, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), *correction, (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_std_dim(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim) { - PROTECT( - auto outputs__ = torch::std(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)unbiased, (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_std_mean(tensor *out__, tensor self, int unbiased) { - PROTECT( - auto outputs__ = torch::std_mean(*self, (bool)unbiased); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_std_mean_correction(tensor *out__, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim) { - PROTECT( - auto outputs__ = torch::std_mean(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), *correction, (bool)keepdim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_std_mean_correction_out(tensor *out__, tensor out0, tensor out1, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim) { - PROTECT( - auto outputs__ = torch::std_mean_out(*out0, *out1, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), *correction, (bool)keepdim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_std_mean_dim(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim) { - PROTECT( - auto outputs__ = torch::std_mean(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)unbiased, (bool)keepdim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_std_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim) { - PROTECT( - auto outputs__ = torch::std_out(*out, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)unbiased, (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_stft(tensor *out__, tensor self, int64_t n_fft, int64_t hop_length_v, uint8_t hop_length_null, int64_t win_length_v, uint8_t win_length_null, tensor window, int normalized, int onesided, int return_complex, int align_to_window) { - PROTECT( - auto outputs__ = torch::stft(*self, n_fft, hop_length_null ? c10::nullopt : c10::optional(hop_length_v), win_length_null ? c10::nullopt : c10::optional(win_length_v), (window ? ::std::optional(*window) : ::std::nullopt), (bool)normalized, (bool)onesided, (bool)return_complex, (bool)align_to_window); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_stft_center(tensor *out__, tensor self, int64_t n_fft, int64_t hop_length_v, uint8_t hop_length_null, int64_t win_length_v, uint8_t win_length_null, tensor window, int center, char* pad_mode_ptr, int pad_mode_len, int normalized, int onesided, int return_complex, int align_to_window) { - PROTECT( - auto outputs__ = torch::stft(*self, n_fft, hop_length_null ? c10::nullopt : c10::optional(hop_length_v), win_length_null ? c10::nullopt : c10::optional(win_length_v), (window ? ::std::optional(*window) : ::std::nullopt), (bool)center, std::string(pad_mode_ptr, pad_mode_len), (bool)normalized, (bool)onesided, (bool)return_complex, (bool)align_to_window); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sub(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::sub(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sub_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->sub_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sub_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::sub_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sub_scalar(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::sub(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sub_scalar_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->sub_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sub_scalar_out(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::sub_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_subtract(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::subtract(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_subtract_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->subtract_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_subtract_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::subtract_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_subtract_scalar(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::subtract(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_subtract_scalar_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->subtract_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sum(tensor *out__, tensor self, int dtype) { - PROTECT( - auto outputs__ = torch::sum(*self, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sum_dim_intlist(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { - PROTECT( - auto outputs__ = torch::sum(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sum_intlist_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype) { - PROTECT( - auto outputs__ = torch::sum_out(*out, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)keepdim, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sum_out(tensor *out__, tensor out, tensor self, int dtype) { - PROTECT( - auto outputs__ = torch::sum_out(*out, *self, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_sum_to_size(tensor *out__, tensor self, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = self->sum_to_size(torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_svd(tensor *out__, tensor self, int some, int compute_uv) { - PROTECT( - auto outputs__ = torch::svd(*self, (bool)some, (bool)compute_uv); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_svd_u(tensor *out__, tensor U, tensor S, tensor V, tensor self, int some, int compute_uv) { - PROTECT( - auto outputs__ = torch::svd_out(*U, *S, *V, *self, (bool)some, (bool)compute_uv); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_swapaxes(tensor *out__, tensor self, int64_t axis0, int64_t axis1) { - PROTECT( - auto outputs__ = torch::swapaxes(*self, axis0, axis1); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_swapaxes_(tensor *out__, tensor self, int64_t axis0, int64_t axis1) { - PROTECT( - auto outputs__ = self->swapaxes_(axis0, axis1); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_swapdims(tensor *out__, tensor self, int64_t dim0, int64_t dim1) { - PROTECT( - auto outputs__ = torch::swapdims(*self, dim0, dim1); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_swapdims_(tensor *out__, tensor self, int64_t dim0, int64_t dim1) { - PROTECT( - auto outputs__ = self->swapdims_(dim0, dim1); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_t(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::t(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_t_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->t_(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_t_copy(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::t_copy(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_t_copy_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::t_copy_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_take(tensor *out__, tensor self, tensor index) { - PROTECT( - auto outputs__ = torch::take(*self, *index); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_take_along_dim(tensor *out__, tensor self, tensor indices, int64_t dim_v, uint8_t dim_null) { - PROTECT( - auto outputs__ = torch::take_along_dim(*self, *indices, dim_null ? c10::nullopt : c10::optional(dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_take_along_dim_out(tensor *out__, tensor out, tensor self, tensor indices, int64_t dim_v, uint8_t dim_null) { - PROTECT( - auto outputs__ = torch::take_along_dim_out(*out, *self, *indices, dim_null ? c10::nullopt : c10::optional(dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_take_out(tensor *out__, tensor out, tensor self, tensor index) { - PROTECT( - auto outputs__ = torch::take_out(*out, *self, *index); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_tan(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::tan(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_tan_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::tan_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_tan_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::tan_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_tanh(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::tanh(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_tanh_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::tanh_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_tanh_backward(tensor *out__, tensor grad_output, tensor output) { - PROTECT( - auto outputs__ = torch::tanh_backward(*grad_output, *output); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_tanh_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor output) { - PROTECT( - auto outputs__ = torch::tanh_backward_out(*grad_input, *grad_output, *output); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_tanh_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::tanh_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -tensor *atg_tensor_split(tensor self, int64_t sections, int64_t dim) { - PROTECT( - auto outputs__ = torch::tensor_split(*self, sections, dim); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -tensor *atg_tensor_split_indices(tensor self, int64_t *indices_data, int indices_len, int64_t dim) { - PROTECT( - auto outputs__ = torch::tensor_split(*self, torch::IntArrayRef(indices_data, indices_len), dim); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -tensor *atg_tensor_split_tensor_indices_or_sections(tensor self, tensor tensor_indices_or_sections, int64_t dim) { - PROTECT( - auto outputs__ = torch::tensor_split(*self, *tensor_indices_or_sections, dim); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg_tensordot(tensor *out__, tensor self, tensor other, int64_t *dims_self_data, int dims_self_len, int64_t *dims_other_data, int dims_other_len) { - PROTECT( - auto outputs__ = torch::tensordot(*self, *other, torch::IntArrayRef(dims_self_data, dims_self_len), torch::IntArrayRef(dims_other_data, dims_other_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_tensordot_out(tensor *out__, tensor out, tensor self, tensor other, int64_t *dims_self_data, int dims_self_len, int64_t *dims_other_data, int dims_other_len) { - PROTECT( - auto outputs__ = torch::tensordot_out(*out, *self, *other, torch::IntArrayRef(dims_self_data, dims_self_len), torch::IntArrayRef(dims_other_data, dims_other_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_threshold(tensor *out__, tensor self, scalar threshold, scalar value) { - PROTECT( - auto outputs__ = torch::threshold(*self, *threshold, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_threshold_(tensor *out__, tensor self, scalar threshold, scalar value) { - PROTECT( - auto outputs__ = torch::threshold_(*self, *threshold, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_threshold_backward(tensor *out__, tensor grad_output, tensor self, scalar threshold) { - PROTECT( - auto outputs__ = torch::threshold_backward(*grad_output, *self, *threshold); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_threshold_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, tensor self, scalar threshold) { - PROTECT( - auto outputs__ = torch::threshold_backward_out(*grad_input, *grad_output, *self, *threshold); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_threshold_out(tensor *out__, tensor out, tensor self, scalar threshold, scalar value) { - PROTECT( - auto outputs__ = torch::threshold_out(*out, *self, *threshold, *value); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_tile(tensor *out__, tensor self, int64_t *dims_data, int dims_len) { - PROTECT( - auto outputs__ = torch::tile(*self, torch::IntArrayRef(dims_data, dims_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_to(tensor *out__, tensor self, int device) { - PROTECT( - auto outputs__ = self->to(device_of_int(device)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_to_dense(tensor *out__, tensor self, int dtype, int masked_grad) { - PROTECT( - auto outputs__ = self->to_dense(dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype)), (bool)masked_grad); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_to_dense_backward(tensor *out__, tensor grad, tensor input, int masked_grad) { - PROTECT( - auto outputs__ = torch::to_dense_backward(*grad, *input, (bool)masked_grad); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_to_device(tensor *out__, tensor self, int device, int dtype, int non_blocking, int copy) { - PROTECT( - auto outputs__ = self->to(device_of_int(device), at::ScalarType(dtype), (bool)non_blocking, (bool)copy); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_to_dtype(tensor *out__, tensor self, int dtype, int non_blocking, int copy) { - PROTECT( - auto outputs__ = self->to(at::ScalarType(dtype), (bool)non_blocking, (bool)copy); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_to_dtype_layout(tensor *out__, tensor self, int options_kind, int options_device, int non_blocking, int copy) { - PROTECT( - auto outputs__ = self->to(at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind)), (bool)non_blocking, (bool)copy); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_to_mkldnn(tensor *out__, tensor self, int dtype) { - PROTECT( - auto outputs__ = self->to_mkldnn(dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_to_mkldnn_backward(tensor *out__, tensor grad, tensor input) { - PROTECT( - auto outputs__ = torch::to_mkldnn_backward(*grad, *input); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_to_mkldnn_out(tensor *out__, tensor out, tensor self, int dtype) { - PROTECT( - auto outputs__ = torch::to_mkldnn_out(*out, *self, dtype < 0 ? c10::nullopt : c10::optional(at::ScalarType(dtype))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_to_other(tensor *out__, tensor self, tensor other, int non_blocking, int copy) { - PROTECT( - auto outputs__ = self->to(*other, (bool)non_blocking, (bool)copy); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_to_padded_tensor(tensor *out__, tensor self, double padding, int64_t *output_size_data, int output_size_len) { - PROTECT( - auto outputs__ = self->to_padded_tensor(padding, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_to_padded_tensor_out(tensor *out__, tensor out, tensor self, double padding, int64_t *output_size_data, int output_size_len) { - PROTECT( - auto outputs__ = torch::to_padded_tensor_out(*out, *self, padding, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_to_sparse(tensor *out__, tensor self, int8_t layout, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null) { - PROTECT( - auto outputs__ = self->to_sparse((layout == -1 ? c10::nullopt : c10::optional(static_cast(layout))), blocksize_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(blocksize_data, blocksize_len)), dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_to_sparse_bsc(tensor *out__, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null) { - PROTECT( - auto outputs__ = self->to_sparse_bsc(torch::IntArrayRef(blocksize_data, blocksize_len), dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_to_sparse_bsr(tensor *out__, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null) { - PROTECT( - auto outputs__ = self->to_sparse_bsr(torch::IntArrayRef(blocksize_data, blocksize_len), dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_to_sparse_csc(tensor *out__, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null) { - PROTECT( - auto outputs__ = self->to_sparse_csc(dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_to_sparse_csr(tensor *out__, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null) { - PROTECT( - auto outputs__ = self->to_sparse_csr(dense_dim_null ? c10::nullopt : c10::optional(dense_dim_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_to_sparse_sparse_dim(tensor *out__, tensor self, int64_t sparse_dim) { - PROTECT( - auto outputs__ = self->to_sparse(sparse_dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_topk(tensor *out__, tensor self, int64_t k, int64_t dim, int largest, int sorted) { - PROTECT( - auto outputs__ = torch::topk(*self, k, dim, (bool)largest, (bool)sorted); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_topk_values(tensor *out__, tensor values, tensor indices, tensor self, int64_t k, int64_t dim, int largest, int sorted) { - PROTECT( - auto outputs__ = torch::topk_out(*values, *indices, *self, k, dim, (bool)largest, (bool)sorted); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_totype(tensor *out__, tensor self, int scalar_type) { - PROTECT( - auto outputs__ = self->toType(at::ScalarType(scalar_type)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_trace(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::trace(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_trace_backward(tensor *out__, tensor grad, int64_t *sizes_data, int sizes_len) { - PROTECT( - auto outputs__ = torch::trace_backward(*grad, torch::IntArrayRef(sizes_data, sizes_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_trace_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::trace_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_transpose(tensor *out__, tensor self, int64_t dim0, int64_t dim1) { - PROTECT( - auto outputs__ = torch::transpose(*self, dim0, dim1); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_transpose_(tensor *out__, tensor self, int64_t dim0, int64_t dim1) { - PROTECT( - auto outputs__ = self->transpose_(dim0, dim1); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_transpose_copy(tensor *out__, tensor self, int64_t dim0, int64_t dim1) { - PROTECT( - auto outputs__ = torch::transpose_copy(*self, dim0, dim1); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_transpose_copy_int_out(tensor *out__, tensor out, tensor self, int64_t dim0, int64_t dim1) { - PROTECT( - auto outputs__ = torch::transpose_copy_out(*out, *self, dim0, dim1); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_trapezoid(tensor *out__, tensor y, int64_t dim) { - PROTECT( - auto outputs__ = torch::trapezoid(*y, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_trapezoid_x(tensor *out__, tensor y, tensor x, int64_t dim) { - PROTECT( - auto outputs__ = torch::trapezoid(*y, *x, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_trapz(tensor *out__, tensor y, tensor x, int64_t dim) { - PROTECT( - auto outputs__ = torch::trapz(*y, *x, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_trapz_dx(tensor *out__, tensor y, double dx, int64_t dim) { - PROTECT( - auto outputs__ = torch::trapz(*y, dx, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_triangular_solve(tensor *out__, tensor self, tensor A, int upper, int transpose, int unitriangular) { - PROTECT( - auto outputs__ = torch::triangular_solve(*self, *A, (bool)upper, (bool)transpose, (bool)unitriangular); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_triangular_solve_x(tensor *out__, tensor X, tensor M, tensor self, tensor A, int upper, int transpose, int unitriangular) { - PROTECT( - auto outputs__ = torch::triangular_solve_out(*X, *M, *self, *A, (bool)upper, (bool)transpose, (bool)unitriangular); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_tril(tensor *out__, tensor self, int64_t diagonal) { - PROTECT( - auto outputs__ = torch::tril(*self, diagonal); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_tril_(tensor *out__, tensor self, int64_t diagonal) { - PROTECT( - auto outputs__ = self->tril_(diagonal); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_tril_indices(tensor *out__, int64_t row, int64_t col, int64_t offset, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::tril_indices(row, col, offset, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_tril_indices_out(tensor *out__, tensor out, int64_t row, int64_t col, int64_t offset) { - PROTECT( - auto outputs__ = torch::tril_indices_out(*out, row, col, offset); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_tril_out(tensor *out__, tensor out, tensor self, int64_t diagonal) { - PROTECT( - auto outputs__ = torch::tril_out(*out, *self, diagonal); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_triplet_margin_loss(tensor *out__, tensor anchor, tensor positive, tensor negative, double margin, double p, double eps, int swap, int64_t reduction) { - PROTECT( - auto outputs__ = torch::triplet_margin_loss(*anchor, *positive, *negative, margin, p, eps, (bool)swap, reduction); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_triu(tensor *out__, tensor self, int64_t diagonal) { - PROTECT( - auto outputs__ = torch::triu(*self, diagonal); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_triu_(tensor *out__, tensor self, int64_t diagonal) { - PROTECT( - auto outputs__ = self->triu_(diagonal); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_triu_indices(tensor *out__, int64_t row, int64_t col, int64_t offset, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::triu_indices(row, col, offset, at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_triu_indices_out(tensor *out__, tensor out, int64_t row, int64_t col, int64_t offset) { - PROTECT( - auto outputs__ = torch::triu_indices_out(*out, row, col, offset); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_triu_out(tensor *out__, tensor out, tensor self, int64_t diagonal) { - PROTECT( - auto outputs__ = torch::triu_out(*out, *self, diagonal); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_true_divide(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::true_divide(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_true_divide_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->true_divide_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_true_divide_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::true_divide_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_true_divide_scalar(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::true_divide(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_true_divide_scalar_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = self->true_divide_(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_trunc(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::trunc(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_trunc_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::trunc_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_trunc_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::trunc_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_type_as(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->type_as(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -tensor *atg_unbind(tensor self, int64_t dim) { - PROTECT( - auto outputs__ = torch::unbind(*self, dim); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -tensor *atg_unbind_copy(tensor self, int64_t dim) { - PROTECT( - auto outputs__ = torch::unbind_copy(*self, dim); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg_unbind_copy_int_out(tensor *out_data, int out_len, tensor self, int64_t dim) { - PROTECT( - torch::unbind_copy_out(of_carray_tensor(out_data, out_len), *self, dim); - ) -} - -void atg_unflatten(tensor *out__, tensor self, int64_t dim, int64_t *sizes_data, int sizes_len) { - PROTECT( - auto outputs__ = torch::unflatten(*self, dim, torch::IntArrayRef(sizes_data, sizes_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -tensor *atg_unflatten_dense_tensors(tensor flat, tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::unflatten_dense_tensors(*flat, of_carray_tensor(tensors_data, tensors_len)); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg_unfold(tensor *out__, tensor self, int64_t dimension, int64_t size, int64_t step) { - PROTECT( - auto outputs__ = self->unfold(dimension, size, step); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_unfold_backward(tensor *out__, tensor grad_in, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t size, int64_t step) { - PROTECT( - auto outputs__ = torch::unfold_backward(*grad_in, torch::IntArrayRef(input_sizes_data, input_sizes_len), dim, size, step); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_unfold_backward_out(tensor *out__, tensor out, tensor grad_in, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t size, int64_t step) { - PROTECT( - auto outputs__ = torch::unfold_backward_out(*out, *grad_in, torch::IntArrayRef(input_sizes_data, input_sizes_len), dim, size, step); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_unfold_copy(tensor *out__, tensor self, int64_t dimension, int64_t size, int64_t step) { - PROTECT( - auto outputs__ = torch::unfold_copy(*self, dimension, size, step); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_unfold_copy_out(tensor *out__, tensor out, tensor self, int64_t dimension, int64_t size, int64_t step) { - PROTECT( - auto outputs__ = torch::unfold_copy_out(*out, *self, dimension, size, step); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_uniform(tensor *out__, tensor self, double from, double to) { - PROTECT( - auto outputs__ = torch::uniform(*self, from, to); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_uniform_(tensor *out__, tensor self, double from, double to) { - PROTECT( - auto outputs__ = self->uniform_(from, to); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_uniform_out(tensor *out__, tensor out, tensor self, double from, double to) { - PROTECT( - auto outputs__ = torch::uniform_out(*out, *self, from, to); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_unique_consecutive(tensor *out__, tensor self, int return_inverse, int return_counts, int64_t dim_v, uint8_t dim_null) { - PROTECT( - auto outputs__ = torch::unique_consecutive(*self, (bool)return_inverse, (bool)return_counts, dim_null ? c10::nullopt : c10::optional(dim_v)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_unique_consecutive_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor self, int return_inverse, int return_counts, int64_t dim_v, uint8_t dim_null) { - PROTECT( - auto outputs__ = torch::unique_consecutive_out(*out0, *out1, *out2, *self, (bool)return_inverse, (bool)return_counts, dim_null ? c10::nullopt : c10::optional(dim_v)); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_unique_dim(tensor *out__, tensor self, int64_t dim, int sorted, int return_inverse, int return_counts) { - PROTECT( - auto outputs__ = torch::unique_dim(*self, dim, (bool)sorted, (bool)return_inverse, (bool)return_counts); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_unique_dim_consecutive(tensor *out__, tensor self, int64_t dim, int return_inverse, int return_counts) { - PROTECT( - auto outputs__ = torch::unique_dim_consecutive(*self, dim, (bool)return_inverse, (bool)return_counts); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_unique_dim_consecutive_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor self, int64_t dim, int return_inverse, int return_counts) { - PROTECT( - auto outputs__ = torch::unique_dim_consecutive_out(*out0, *out1, *out2, *self, dim, (bool)return_inverse, (bool)return_counts); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -void atg_unique_dim_out(tensor *out__, tensor out0, tensor out1, tensor out2, tensor self, int64_t dim, int sorted, int return_inverse, int return_counts) { - PROTECT( - auto outputs__ = torch::unique_dim_out(*out0, *out1, *out2, *self, dim, (bool)sorted, (bool)return_inverse, (bool)return_counts); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - out__[2] = new torch::Tensor(std::get<2>(outputs__)); - ) -} - -tensor *atg_unsafe_chunk(tensor self, int64_t chunks, int64_t dim) { - PROTECT( - auto outputs__ = torch::unsafe_chunk(*self, chunks, dim); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -tensor *atg_unsafe_split(tensor self, int64_t split_size, int64_t dim) { - PROTECT( - auto outputs__ = torch::unsafe_split(*self, split_size, dim); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg_unsafe_split_tensor_out(tensor *out_data, int out_len, tensor self, int64_t split_size, int64_t dim) { - PROTECT( - torch::unsafe_split_out(of_carray_tensor(out_data, out_len), *self, split_size, dim); - ) -} - -tensor *atg_unsafe_split_with_sizes(tensor self, int64_t *split_sizes_data, int split_sizes_len, int64_t dim) { - PROTECT( - auto outputs__ = torch::unsafe_split_with_sizes(*self, torch::IntArrayRef(split_sizes_data, split_sizes_len), dim); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg_unsafe_split_with_sizes_out(tensor *out_data, int out_len, tensor self, int64_t *split_sizes_data, int split_sizes_len, int64_t dim) { - PROTECT( - torch::unsafe_split_with_sizes_out(of_carray_tensor(out_data, out_len), *self, torch::IntArrayRef(split_sizes_data, split_sizes_len), dim); - ) -} - -void atg_unsqueeze(tensor *out__, tensor self, int64_t dim) { - PROTECT( - auto outputs__ = torch::unsqueeze(*self, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_unsqueeze_(tensor *out__, tensor self, int64_t dim) { - PROTECT( - auto outputs__ = self->unsqueeze_(dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_unsqueeze_copy(tensor *out__, tensor self, int64_t dim) { - PROTECT( - auto outputs__ = torch::unsqueeze_copy(*self, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_unsqueeze_copy_out(tensor *out__, tensor out, tensor self, int64_t dim) { - PROTECT( - auto outputs__ = torch::unsqueeze_copy_out(*out, *self, dim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_bicubic2d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::upsample_bicubic2d(*self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_bicubic2d_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::upsample_bicubic2d_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_bicubic2d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::upsample_bicubic2d_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_bicubic2d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::upsample_bicubic2d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_bicubic2d_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len) { - PROTECT( - auto outputs__ = torch::upsample_bicubic2d(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), (bool)align_corners, at::ArrayRef(scale_factors_data, scale_factors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_bilinear2d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::upsample_bilinear2d(*self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_bilinear2d_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::upsample_bilinear2d_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_bilinear2d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::upsample_bilinear2d_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_bilinear2d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::upsample_bilinear2d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_bilinear2d_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len) { - PROTECT( - auto outputs__ = torch::upsample_bilinear2d(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), (bool)align_corners, at::ArrayRef(scale_factors_data, scale_factors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_bilinear2d_vec_out(tensor *out__, tensor out, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len) { - PROTECT( - auto outputs__ = torch::upsample_bilinear2d_out(*out, *input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), (bool)align_corners, at::ArrayRef(scale_factors_data, scale_factors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_linear1d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_v, uint8_t scales_null) { - PROTECT( - auto outputs__ = torch::upsample_linear1d(*self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_null ? c10::nullopt : c10::optional(scales_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_linear1d_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_v, uint8_t scales_null) { - PROTECT( - auto outputs__ = torch::upsample_linear1d_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_null ? c10::nullopt : c10::optional(scales_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_linear1d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_v, uint8_t scales_null) { - PROTECT( - auto outputs__ = torch::upsample_linear1d_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_null ? c10::nullopt : c10::optional(scales_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_linear1d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_v, uint8_t scales_null) { - PROTECT( - auto outputs__ = torch::upsample_linear1d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_null ? c10::nullopt : c10::optional(scales_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_linear1d_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len) { - PROTECT( - auto outputs__ = torch::upsample_linear1d(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), (bool)align_corners, at::ArrayRef(scale_factors_data, scale_factors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_nearest1d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, double scales_v, uint8_t scales_null) { - PROTECT( - auto outputs__ = torch::upsample_nearest1d(*self, torch::IntArrayRef(output_size_data, output_size_len), scales_null ? c10::nullopt : c10::optional(scales_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_nearest1d_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_v, uint8_t scales_null) { - PROTECT( - auto outputs__ = torch::upsample_nearest1d_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_null ? c10::nullopt : c10::optional(scales_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_nearest1d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_v, uint8_t scales_null) { - PROTECT( - auto outputs__ = torch::upsample_nearest1d_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_null ? c10::nullopt : c10::optional(scales_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_nearest1d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_v, uint8_t scales_null) { - PROTECT( - auto outputs__ = torch::upsample_nearest1d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), scales_null ? c10::nullopt : c10::optional(scales_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_nearest1d_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len) { - PROTECT( - auto outputs__ = torch::upsample_nearest1d(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), at::ArrayRef(scale_factors_data, scale_factors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_nearest2d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::upsample_nearest2d(*self, torch::IntArrayRef(output_size_data, output_size_len), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_nearest2d_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::upsample_nearest2d_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_nearest2d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::upsample_nearest2d_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_nearest2d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::upsample_nearest2d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_nearest2d_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len) { - PROTECT( - auto outputs__ = torch::upsample_nearest2d(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), at::ArrayRef(scale_factors_data, scale_factors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_nearest2d_vec_out(tensor *out__, tensor out, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len) { - PROTECT( - auto outputs__ = torch::upsample_nearest2d_out(*out, *input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), at::ArrayRef(scale_factors_data, scale_factors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_nearest3d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::upsample_nearest3d(*self, torch::IntArrayRef(output_size_data, output_size_len), scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_nearest3d_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::upsample_nearest3d_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_nearest3d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::upsample_nearest3d_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_nearest3d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::upsample_nearest3d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_nearest3d_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len) { - PROTECT( - auto outputs__ = torch::upsample_nearest3d(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), at::ArrayRef(scale_factors_data, scale_factors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_trilinear3d(tensor *out__, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::upsample_trilinear3d(*self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_trilinear3d_backward(tensor *out__, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::upsample_trilinear3d_backward(*grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_trilinear3d_backward_grad_input(tensor *out__, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::upsample_trilinear3d_backward_out(*grad_input, *grad_output, torch::IntArrayRef(output_size_data, output_size_len), torch::IntArrayRef(input_size_data, input_size_len), (bool)align_corners, scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_trilinear3d_out(tensor *out__, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null) { - PROTECT( - auto outputs__ = torch::upsample_trilinear3d_out(*out, *self, torch::IntArrayRef(output_size_data, output_size_len), (bool)align_corners, scales_d_null ? c10::nullopt : c10::optional(scales_d_v), scales_h_null ? c10::nullopt : c10::optional(scales_h_v), scales_w_null ? c10::nullopt : c10::optional(scales_w_v)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_upsample_trilinear3d_vec(tensor *out__, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len) { - PROTECT( - auto outputs__ = torch::upsample_trilinear3d(*input, output_size_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(output_size_data, output_size_len)), (bool)align_corners, at::ArrayRef(scale_factors_data, scale_factors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_value_selecting_reduction_backward(tensor *out__, tensor grad, int64_t dim, tensor indices, int64_t *sizes_data, int sizes_len, int keepdim) { - PROTECT( - auto outputs__ = torch::value_selecting_reduction_backward(*grad, dim, *indices, torch::IntArrayRef(sizes_data, sizes_len), (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_values(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = self->values(); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_values_copy(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::values_copy(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_values_copy_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::values_copy_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_vander(tensor *out__, tensor x, int64_t n_v, uint8_t n_null, int increasing) { - PROTECT( - auto outputs__ = torch::vander(*x, n_null ? c10::nullopt : c10::optional(n_v), (bool)increasing); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_var(tensor *out__, tensor self, int unbiased) { - PROTECT( - auto outputs__ = torch::var(*self, (bool)unbiased); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_var_correction(tensor *out__, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim) { - PROTECT( - auto outputs__ = torch::var(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), *correction, (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_var_correction_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim) { - PROTECT( - auto outputs__ = torch::var_out(*out, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), *correction, (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_var_dim(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim) { - PROTECT( - auto outputs__ = torch::var(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)unbiased, (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_var_mean(tensor *out__, tensor self, int unbiased) { - PROTECT( - auto outputs__ = torch::var_mean(*self, (bool)unbiased); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_var_mean_correction(tensor *out__, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim) { - PROTECT( - auto outputs__ = torch::var_mean(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), *correction, (bool)keepdim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_var_mean_correction_out(tensor *out__, tensor out0, tensor out1, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim) { - PROTECT( - auto outputs__ = torch::var_mean_out(*out0, *out1, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), *correction, (bool)keepdim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_var_mean_dim(tensor *out__, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim) { - PROTECT( - auto outputs__ = torch::var_mean(*self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)unbiased, (bool)keepdim); - out__[0] = new torch::Tensor(std::get<0>(outputs__)); - out__[1] = new torch::Tensor(std::get<1>(outputs__)); - ) -} - -void atg_var_out(tensor *out__, tensor out, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim) { - PROTECT( - auto outputs__ = torch::var_out(*out, *self, dim_data == nullptr ? c10::nullopt : c10::optional(torch::IntArrayRef(dim_data, dim_len)), (bool)unbiased, (bool)keepdim); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_vdot(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::vdot(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_vdot_out(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::vdot_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_view(tensor *out__, tensor self, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = self->view(torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_view_as(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = self->view_as(*other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_view_as_complex(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::view_as_complex(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_view_as_complex_copy(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::view_as_complex_copy(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_view_as_complex_copy_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::view_as_complex_copy_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_view_as_real(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::view_as_real(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_view_as_real_copy(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::view_as_real_copy(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_view_as_real_copy_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::view_as_real_copy_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_view_copy(tensor *out__, tensor self, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::view_copy(*self, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_view_copy_dtype(tensor *out__, tensor self, int dtype) { - PROTECT( - auto outputs__ = torch::view_copy(*self, at::ScalarType(dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_view_copy_dtype_out(tensor *out__, tensor out, tensor self, int dtype) { - PROTECT( - auto outputs__ = torch::view_copy_out(*out, *self, at::ScalarType(dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_view_copy_out(tensor *out__, tensor out, tensor self, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::view_copy_out(*out, *self, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_view_dtype(tensor *out__, tensor self, int dtype) { - PROTECT( - auto outputs__ = self->view(at::ScalarType(dtype)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -tensor *atg_vsplit(tensor self, int64_t sections) { - PROTECT( - auto outputs__ = torch::vsplit(*self, sections); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -tensor *atg_vsplit_array(tensor self, int64_t *indices_data, int indices_len) { - PROTECT( - auto outputs__ = torch::vsplit(*self, torch::IntArrayRef(indices_data, indices_len)); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg_vstack(tensor *out__, tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::vstack(of_carray_tensor(tensors_data, tensors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_vstack_out(tensor *out__, tensor out, tensor *tensors_data, int tensors_len) { - PROTECT( - auto outputs__ = torch::vstack_out(*out, of_carray_tensor(tensors_data, tensors_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - -tensor *atg_where(tensor condition) { - PROTECT( - auto outputs__ = torch::where(*condition); - int sz = outputs__.size(); - torch::Tensor **out__ = (torch::Tensor**)malloc((sz + 1) * sizeof(torch::Tensor*)); - for (int i = 0; i < sz; ++i) - out__[i] = new torch::Tensor(outputs__[i]); - out__[sz] = nullptr; - return out__; - ) - return nullptr; -} - -void atg_where_scalar(tensor *out__, tensor condition, scalar self_scalar, scalar other) { - PROTECT( - auto outputs__ = torch::where(*condition, *self_scalar, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_where_scalarother(tensor *out__, tensor condition, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::where(*condition, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_where_scalarself(tensor *out__, tensor condition, scalar self_scalar, tensor other) { - PROTECT( - auto outputs__ = torch::where(*condition, *self_scalar, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_where_self(tensor *out__, tensor condition, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::where(*condition, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_where_self_out(tensor *out__, tensor out, tensor condition, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::where_out(*out, *condition, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_xlogy(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::xlogy(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_xlogy_(tensor *out__, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::xlogy_(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_xlogy_outscalar_other(tensor *out__, tensor out, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::xlogy_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_xlogy_outscalar_self(tensor *out__, tensor out, scalar self_scalar, tensor other) { - PROTECT( - auto outputs__ = torch::xlogy_out(*out, *self_scalar, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_xlogy_outtensor(tensor *out__, tensor out, tensor self, tensor other) { - PROTECT( - auto outputs__ = torch::xlogy_out(*out, *self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_xlogy_scalar_other(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::xlogy(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_xlogy_scalar_other_(tensor *out__, tensor self, scalar other) { - PROTECT( - auto outputs__ = torch::xlogy_(*self, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_xlogy_scalar_self(tensor *out__, scalar self_scalar, tensor other) { - PROTECT( - auto outputs__ = torch::xlogy(*self_scalar, *other); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_zero(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::zero(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_zero_(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::zero_(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_zero_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::zero_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_zeros(tensor *out__, int64_t *size_data, int size_len, int options_kind, int options_device) { - PROTECT( - auto outputs__ = torch::zeros(torch::IntArrayRef(size_data, size_len), at::device(device_of_int(options_device)).dtype(at::ScalarType(options_kind))); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_zeros_like(tensor *out__, tensor self) { - PROTECT( - auto outputs__ = torch::zeros_like(*self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_zeros_like_out(tensor *out__, tensor out, tensor self) { - PROTECT( - auto outputs__ = torch::zeros_like_out(*out, *self); - out__[0] = new torch::Tensor(outputs__); - ) -} - -void atg_zeros_out(tensor *out__, tensor out, int64_t *size_data, int size_len) { - PROTECT( - auto outputs__ = torch::zeros_out(*out, torch::IntArrayRef(size_data, size_len)); - out__[0] = new torch::Tensor(outputs__); - ) -} - diff --git a/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api_generated.h b/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api_generated.h deleted file mode 100644 index 1c35bbdfe3..0000000000 --- a/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_api_generated.h +++ /dev/null @@ -1,2648 +0,0 @@ -// THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT BY HAND! -#include "torch_api.h" - -extern "C" { -void atg___and__(tensor *, tensor self, scalar other); -void atg___and__tensor_(tensor *, tensor self, tensor other); -void atg___iand__(tensor *, tensor self, scalar other); -void atg___iand__tensor_(tensor *, tensor self, tensor other); -void atg___ilshift__(tensor *, tensor self, scalar other); -void atg___ilshift__tensor_(tensor *, tensor self, tensor other); -void atg___ior__(tensor *, tensor self, scalar other); -void atg___ior__tensor_(tensor *, tensor self, tensor other); -void atg___irshift__(tensor *, tensor self, scalar other); -void atg___irshift__tensor_(tensor *, tensor self, tensor other); -void atg___ixor__(tensor *, tensor self, scalar other); -void atg___ixor__tensor_(tensor *, tensor self, tensor other); -void atg___lshift__(tensor *, tensor self, scalar other); -void atg___lshift__scalar_out_(tensor *, tensor out, tensor self, scalar other); -void atg___lshift__tensor_(tensor *, tensor self, tensor other); -void atg___lshift__tensor_out_(tensor *, tensor out, tensor self, tensor other); -void atg___or__(tensor *, tensor self, scalar other); -void atg___or__tensor_(tensor *, tensor self, tensor other); -void atg___rshift__(tensor *, tensor self, scalar other); -void atg___rshift__scalar_out_(tensor *, tensor out, tensor self, scalar other); -void atg___rshift__tensor_(tensor *, tensor self, tensor other); -void atg___rshift__tensor_out_(tensor *, tensor out, tensor self, tensor other); -void atg___xor__(tensor *, tensor self, scalar other); -void atg___xor__tensor_(tensor *, tensor self, tensor other); -void atg__adaptive_avg_pool2d(tensor *, tensor self, int64_t *output_size_data, int output_size_len); -void atg__adaptive_avg_pool2d_backward(tensor *, tensor grad_output, tensor self); -void atg__adaptive_avg_pool2d_backward_out(tensor *, tensor out, tensor grad_output, tensor self); -void atg__adaptive_avg_pool2d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len); -void atg__adaptive_avg_pool3d(tensor *, tensor self, int64_t *output_size_data, int output_size_len); -void atg__adaptive_avg_pool3d_backward(tensor *, tensor grad_output, tensor self); -void atg__adaptive_avg_pool3d_backward_out(tensor *, tensor out, tensor grad_output, tensor self); -void atg__adaptive_avg_pool3d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len); -void atg__add_batch_dim(tensor *, tensor self, int64_t batch_dim, int64_t level); -void atg__add_relu(tensor *, tensor self, tensor other); -void atg__add_relu_(tensor *, tensor self, tensor other); -void atg__add_relu_out(tensor *, tensor out, tensor self, tensor other); -void atg__add_relu_scalar(tensor *, tensor self, scalar other); -void atg__add_relu_scalar_(tensor *, tensor self, scalar other); -void atg__add_relu_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg__addmm_activation(tensor *, tensor self, tensor mat1, tensor mat2, int use_gelu); -void atg__addmm_activation_out(tensor *, tensor out, tensor self, tensor mat1, tensor mat2, int use_gelu); -void atg__aminmax(tensor *, tensor self); -void atg__aminmax_dim(tensor *, tensor self, int64_t dim, int keepdim); -void atg__aminmax_dim_out(tensor *, tensor out0, tensor out1, tensor self, int64_t dim, int keepdim); -void atg__aminmax_out(tensor *, tensor out0, tensor out1, tensor self); -void atg__amp_update_scale(tensor *, tensor self, tensor growth_tracker, tensor found_inf, double scale_growth_factor, double scale_backoff_factor, int64_t growth_interval); -void atg__amp_update_scale_(tensor *, tensor self, tensor growth_tracker, tensor found_inf, double scale_growth_factor, double scale_backoff_factor, int64_t growth_interval); -void atg__amp_update_scale_out(tensor *, tensor out, tensor self, tensor growth_tracker, tensor found_inf, double scale_growth_factor, double scale_backoff_factor, int64_t growth_interval); -void atg__assert_scalar(scalar self_scalar, char* assert_msg_ptr, int assert_msg_len); -void atg__assert_tensor_metadata(tensor a, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int dtype, int device, int8_t layout); -void atg__autocast_to_full_precision(tensor *, tensor self, int cuda_enabled, int cpu_enabled); -void atg__autocast_to_reduced_precision(tensor *, tensor self, int cuda_enabled, int cpu_enabled, int cuda_dtype, int cpu_dtype); -void atg__batch_norm_no_update(tensor *, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps); -void atg__batch_norm_no_update_out(tensor *, tensor out0, tensor out1, tensor out2, tensor out3, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps); -void atg__batch_norm_with_update(tensor *, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps); -void atg__batch_norm_with_update_functional(tensor *, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps); -void atg__batch_norm_with_update_out(tensor *, tensor out, tensor save_mean, tensor save_invstd, tensor reserve, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps); -void atg__cast_byte(tensor *, tensor self, int non_blocking); -void atg__cast_char(tensor *, tensor self, int non_blocking); -void atg__cast_double(tensor *, tensor self, int non_blocking); -void atg__cast_float(tensor *, tensor self, int non_blocking); -void atg__cast_half(tensor *, tensor self, int non_blocking); -void atg__cast_int(tensor *, tensor self, int non_blocking); -void atg__cast_long(tensor *, tensor self, int non_blocking); -void atg__cast_short(tensor *, tensor self, int non_blocking); -void atg__cdist_backward(tensor *, tensor grad, tensor x1, tensor x2, double p, tensor cdist); -void atg__cdist_backward_out(tensor *, tensor out, tensor grad, tensor x1, tensor x2, double p, tensor cdist); -void atg__cholesky_solve_helper(tensor *, tensor self, tensor A, int upper); -void atg__cholesky_solve_helper_out(tensor *, tensor out, tensor self, tensor A, int upper); -void atg__chunk_cat(tensor *, tensor *tensors_data, int tensors_len, int64_t dim, int64_t num_chunks); -void atg__chunk_cat_out(tensor *, tensor out, tensor *tensors_data, int tensors_len, int64_t dim, int64_t num_chunks); -void atg__coalesce(tensor *, tensor self); -void atg__coalesce_out(tensor *, tensor out, tensor self); -void atg__coalesced(tensor *, tensor self, int coalesced); -void atg__coalesced_(tensor *, tensor self, int coalesced); -void atg__coalesced_out(tensor *, tensor out, tensor self, int coalesced); -void atg__compute_linear_combination(tensor *, tensor input, tensor coefficients); -void atg__compute_linear_combination_out(tensor *, tensor out, tensor input, tensor coefficients); -void atg__conj(tensor *, tensor self); -void atg__conj_copy(tensor *, tensor self); -void atg__conj_copy_out(tensor *, tensor out, tensor self); -void atg__conj_physical(tensor *, tensor self); -void atg__conj_physical_out(tensor *, tensor out, tensor self); -void atg__conv_depthwise2d(tensor *, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len); -void atg__conv_depthwise2d_out(tensor *, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len); -void atg__convert_indices_from_coo_to_csr(tensor *, tensor self, int64_t size, int out_int32); -void atg__convert_indices_from_coo_to_csr_out(tensor *, tensor out, tensor self, int64_t size, int out_int32); -void atg__convert_indices_from_csr_to_coo(tensor *, tensor crow_indices, tensor col_indices, int out_int32, int transpose); -void atg__convert_indices_from_csr_to_coo_out(tensor *, tensor out, tensor crow_indices, tensor col_indices, int out_int32, int transpose); -void atg__convert_weight_to_int4pack(tensor *, tensor self, int64_t innerKTiles); -void atg__convert_weight_to_int4pack_for_cpu(tensor *, tensor self, int64_t innerKTiles); -void atg__convolution(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups, int benchmark, int deterministic, int cudnn_enabled, int allow_tf32); -void atg__convolution_deprecated(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups, int benchmark, int deterministic, int cudnn_enabled); -void atg__convolution_mode(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, char* padding_ptr, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); -void atg__convolution_out(tensor *, tensor out, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups, int benchmark, int deterministic, int cudnn_enabled, int allow_tf32); -void atg__copy_from(tensor *, tensor self, tensor dst, int non_blocking); -void atg__copy_from_and_resize(tensor *, tensor self, tensor dst); -void atg__copy_from_and_resize_out(tensor *, tensor out, tensor self, tensor dst); -void atg__copy_from_out(tensor *, tensor out, tensor self, tensor dst, int non_blocking); -void atg__cslt_compress(tensor *, tensor input); -void atg__cslt_sparse_mm(tensor *, tensor compressed_A, tensor dense_B, tensor bias, tensor alpha, int out_dtype, int transpose_result, int64_t alg_id, int64_t split_k, int64_t split_k_mode); -int64_t atg__cslt_sparse_mm_search(tensor compressed_A, tensor dense_B, tensor bias, tensor alpha, int out_dtype, int transpose_result); -void atg__ctc_loss(tensor *, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int zero_infinity); -void atg__ctc_loss_backward(tensor *, tensor grad, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, tensor neg_log_likelihood, tensor log_alpha, int64_t blank, int zero_infinity); -void atg__ctc_loss_backward_out(tensor *, tensor out, tensor grad, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, tensor neg_log_likelihood, tensor log_alpha, int64_t blank, int zero_infinity); -void atg__ctc_loss_backward_tensor(tensor *, tensor grad, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, tensor neg_log_likelihood, tensor log_alpha, int64_t blank, int zero_infinity); -void atg__ctc_loss_out(tensor *, tensor out0, tensor out1, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int zero_infinity); -void atg__ctc_loss_tensor(tensor *, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank, int zero_infinity); -void atg__ctc_loss_tensor_out(tensor *, tensor out0, tensor out1, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank, int zero_infinity); -void atg__cudnn_attention_backward(tensor *, tensor grad_out, tensor query, tensor key, tensor value, tensor out, tensor logsumexp, tensor philox_seed, tensor philox_offset, tensor attn_bias, tensor cum_seq_q, tensor cum_seq_k, int64_t max_q, int64_t max_k, double dropout_p, int is_causal, double scale_v, uint8_t scale_null); -void atg__cudnn_ctc_loss(tensor *, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int deterministic, int zero_infinity); -void atg__cudnn_ctc_loss_out(tensor *, tensor out0, tensor out1, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int deterministic, int zero_infinity); -void atg__cudnn_ctc_loss_tensor(tensor *, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank, int deterministic, int zero_infinity); -void atg__cudnn_init_dropout_state(tensor *, double dropout, int train, int64_t dropout_seed, int options_kind, int options_device); -void atg__cudnn_init_dropout_state_out(tensor *, tensor out, double dropout, int train, int64_t dropout_seed); -void atg__cudnn_rnn(tensor *, tensor input, tensor *weight_data, int weight_len, int64_t weight_stride0, tensor weight_buf, tensor hx, tensor cx, int64_t mode, int64_t hidden_size, int64_t proj_size, int64_t num_layers, int batch_first, double dropout, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, tensor dropout_state); -void atg__cudnn_rnn_flatten_weight(tensor *, tensor *weight_arr_data, int weight_arr_len, int64_t weight_stride0, int64_t input_size, int64_t mode, int64_t hidden_size, int64_t proj_size, int64_t num_layers, int batch_first, int bidirectional); -void atg__cudnn_rnn_flatten_weight_out(tensor *, tensor out, tensor *weight_arr_data, int weight_arr_len, int64_t weight_stride0, int64_t input_size, int64_t mode, int64_t hidden_size, int64_t proj_size, int64_t num_layers, int batch_first, int bidirectional); -void atg__cudnn_rnn_out(tensor *, tensor out0, tensor out1, tensor out2, tensor out3, tensor out4, tensor input, tensor *weight_data, int weight_len, int64_t weight_stride0, tensor weight_buf, tensor hx, tensor cx, int64_t mode, int64_t hidden_size, int64_t proj_size, int64_t num_layers, int batch_first, double dropout, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, tensor dropout_state); -int64_t atg__debug_has_internal_overlap(tensor self); -void atg__dim_arange(tensor *, tensor like, int64_t dim); -int64_t atg__dimi(tensor self); -int64_t atg__dimv(tensor self); -void atg__dirichlet_grad(tensor *, tensor x, tensor alpha, tensor total); -void atg__dirichlet_grad_out(tensor *, tensor out, tensor x, tensor alpha, tensor total); -void atg__dyn_quant_matmul_4bit(tensor *, tensor inp, tensor packed_weights, int64_t block_size, int64_t in_features, int64_t out_features); -void atg__dyn_quant_pack_4bit_weight(tensor *, tensor weights, tensor scales_zeros, tensor bias, int64_t block_size, int64_t in_features, int64_t out_features); -void atg__efficient_attention_backward(tensor *, tensor grad_out_, tensor query, tensor key, tensor value, tensor bias, tensor out, tensor cu_seqlens_q, tensor cu_seqlens_k, int64_t max_seqlen_q, int64_t max_seqlen_k, tensor logsumexp, double dropout_p, tensor philox_seed, tensor philox_offset, int64_t custom_mask_type, int bias_requires_grad, double scale_v, uint8_t scale_null, int64_t num_splits_key_v, uint8_t num_splits_key_null, int64_t window_size_v, uint8_t window_size_null, int shared_storage_dqdkdv); -void atg__efficientzerotensor(tensor *, int64_t *size_data, int size_len, int options_kind, int options_device); -void atg__efficientzerotensor_out(tensor *, tensor out, int64_t *size_data, int size_len); -void atg__embedding_bag(tensor *, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset, int64_t padding_idx); -void atg__embedding_bag_backward(tensor *, tensor grad, tensor indices, tensor offsets, tensor offset2bag, tensor bag_size, tensor maximum_indices, int64_t num_weights, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int64_t padding_idx); -void atg__embedding_bag_dense_backward(tensor *, tensor grad, tensor indices, tensor offset2bag, tensor bag_size, tensor maximum_indices, int64_t num_weights, int scale_grad_by_freq, int64_t mode, tensor per_sample_weights, int64_t padding_idx); -void atg__embedding_bag_dense_backward_out(tensor *, tensor out, tensor grad, tensor indices, tensor offset2bag, tensor bag_size, tensor maximum_indices, int64_t num_weights, int scale_grad_by_freq, int64_t mode, tensor per_sample_weights, int64_t padding_idx); -void atg__embedding_bag_forward_only(tensor *, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset, int64_t padding_idx); -void atg__embedding_bag_forward_only_out(tensor *, tensor out0, tensor out1, tensor out2, tensor out3, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset, int64_t padding_idx); -void atg__embedding_bag_out(tensor *, tensor out0, tensor out1, tensor out2, tensor out3, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset, int64_t padding_idx); -void atg__embedding_bag_per_sample_weights_backward(tensor *, tensor grad, tensor weight, tensor indices, tensor offsets, tensor offset2bag, int64_t mode, int64_t padding_idx); -void atg__embedding_bag_per_sample_weights_backward_out(tensor *, tensor out, tensor grad, tensor weight, tensor indices, tensor offsets, tensor offset2bag, int64_t mode, int64_t padding_idx); -void atg__embedding_bag_sparse_backward(tensor *, tensor grad, tensor indices, tensor offsets, tensor offset2bag, tensor bag_size, int64_t num_weights, int scale_grad_by_freq, int64_t mode, tensor per_sample_weights, int64_t padding_idx); -void atg__empty_affine_quantized(tensor *, int64_t *size_data, int size_len, int options_kind, int options_device, double scale, int64_t zero_point); -void atg__empty_affine_quantized_out(tensor *, tensor out, int64_t *size_data, int size_len, double scale, int64_t zero_point); -void atg__empty_per_channel_affine_quantized(tensor *, int64_t *size_data, int size_len, tensor scales, tensor zero_points, int64_t axis, int options_kind, int options_device); -void atg__empty_per_channel_affine_quantized_out(tensor *, tensor out, int64_t *size_data, int size_len, tensor scales, tensor zero_points, int64_t axis); -void atg__euclidean_dist(tensor *, tensor x1, tensor x2); -void atg__euclidean_dist_out(tensor *, tensor out, tensor x1, tensor x2); -void atg__fake_quantize_learnable_per_channel_affine(tensor *, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max, double grad_factor); -void atg__fake_quantize_learnable_per_channel_affine_backward(tensor *, tensor grad, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max, double grad_factor); -void atg__fake_quantize_learnable_per_channel_affine_out(tensor *, tensor out, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max, double grad_factor); -void atg__fake_quantize_learnable_per_tensor_affine(tensor *, tensor self, tensor scale, tensor zero_point, int64_t quant_min, int64_t quant_max, double grad_factor); -void atg__fake_quantize_learnable_per_tensor_affine_backward(tensor *, tensor grad, tensor self, tensor scale, tensor zero_point, int64_t quant_min, int64_t quant_max, double grad_factor); -void atg__fake_quantize_learnable_per_tensor_affine_out(tensor *, tensor out, tensor self, tensor scale, tensor zero_point, int64_t quant_min, int64_t quant_max, double grad_factor); -void atg__fake_quantize_per_tensor_affine_cachemask_tensor_qparams(tensor *, tensor self, tensor scale, tensor zero_point, tensor fake_quant_enabled, int64_t quant_min, int64_t quant_max); -void atg__fake_quantize_per_tensor_affine_cachemask_tensor_qparams_out(tensor *, tensor out0, tensor out1, tensor self, tensor scale, tensor zero_point, tensor fake_quant_enabled, int64_t quant_min, int64_t quant_max); -void atg__fft_c2c(tensor *, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int forward); -void atg__fft_c2c_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int forward); -void atg__fft_c2r(tensor *, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int64_t last_dim_size); -void atg__fft_c2r_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int64_t last_dim_size); -void atg__fft_r2c(tensor *, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int onesided); -void atg__fft_r2c_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int64_t normalization, int onesided); -void atg__fill_mem_eff_dropout_mask_(tensor *, tensor self, double dropout_p, int64_t seed, int64_t offset); -void atg__flash_attention_backward(tensor *, tensor grad_out, tensor query, tensor key, tensor value, tensor out, tensor logsumexp, tensor cum_seq_q, tensor cum_seq_k, int64_t max_q, int64_t max_k, double dropout_p, int is_causal, tensor rng_state, tensor unused, double scale_v, uint8_t scale_null, int64_t window_size_left_v, uint8_t window_size_left_null, int64_t window_size_right_v, uint8_t window_size_right_null); -void atg__foobar(tensor *, tensor self, int arg1, int arg2, int arg3); -void atg__foobar_out(tensor *, tensor out, tensor self, int arg1, int arg2, int arg3); -void atg__functional_assert_async(tensor *, tensor self, char* assert_msg_ptr, int assert_msg_len, tensor dep_token); -void atg__functional_assert_scalar(tensor *, scalar self_scalar, char* assert_msg_ptr, int assert_msg_len, tensor dep_token); -void atg__functional_sym_constrain_range(tensor *, scalar size, int64_t min_v, uint8_t min_null, int64_t max_v, uint8_t max_null, tensor dep_token); -void atg__functional_sym_constrain_range_for_size(tensor *, scalar size, int64_t min_v, uint8_t min_null, int64_t max_v, uint8_t max_null, tensor dep_token); -void atg__fused_dropout(tensor *, tensor self, double p); -void atg__fused_dropout_out(tensor *, tensor out0, tensor out1, tensor self, double p); -void atg__fused_moving_avg_obs_fq_helper(tensor *, tensor self, tensor observer_on, tensor fake_quant_on, tensor running_min, tensor running_max, tensor scale, tensor zero_point, double averaging_const, int64_t quant_min, int64_t quant_max, int64_t ch_axis, int per_row_fake_quant, int symmetric_quant); -void atg__fused_moving_avg_obs_fq_helper_functional(tensor *, tensor self, tensor observer_on, tensor fake_quant_on, tensor running_min, tensor running_max, tensor scale, tensor zero_point, double averaging_const, int64_t quant_min, int64_t quant_max, int64_t ch_axis, int per_row_fake_quant, int symmetric_quant); -void atg__fused_moving_avg_obs_fq_helper_out(tensor *, tensor out0, tensor out1, tensor self, tensor observer_on, tensor fake_quant_on, tensor running_min, tensor running_max, tensor scale, tensor zero_point, double averaging_const, int64_t quant_min, int64_t quant_max, int64_t ch_axis, int per_row_fake_quant, int symmetric_quant); -void atg__fused_rms_norm(tensor *, tensor input, int64_t *normalized_shape_data, int normalized_shape_len, tensor weight, double eps_v, uint8_t eps_null); -int64_t atg__fused_sdp_choice(tensor query, tensor key, tensor value, tensor attn_mask, double dropout_p, int is_causal, double scale_v, uint8_t scale_null, int enable_gqa); -void atg__fw_primal(tensor *, tensor self, int64_t level); -void atg__fw_primal_copy(tensor *, tensor self, int64_t level); -void atg__fw_primal_copy_out(tensor *, tensor out, tensor self, int64_t level); -void atg__gather_sparse_backward(tensor *, tensor self, int64_t dim, tensor index, tensor grad); -void atg__grid_sampler_2d_cpu_fallback(tensor *, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners); -void atg__grid_sampler_2d_cpu_fallback_backward(tensor *, tensor grad_output, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners); -void atg__grid_sampler_2d_cpu_fallback_out(tensor *, tensor out, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners); -void atg__grouped_mm(tensor *, tensor self, tensor mat2, tensor offs, tensor bias, int out_dtype); -int atg__has_compatible_shallow_copy_type(tensor self, tensor from); -int atg__has_same_storage_numel(tensor self, tensor other); -tensor *atg__histogramdd_bin_edges(tensor self, int64_t *bins_data, int bins_len, double *range_data, int range_len, tensor weight, int density); -void atg__histogramdd_bin_edges_out(tensor *out_data, int out_len, tensor self, int64_t *bins_data, int bins_len, double *range_data, int range_len, tensor weight, int density); -void atg__histogramdd_from_bin_cts(tensor *, tensor self, int64_t *bins_data, int bins_len, double *range_data, int range_len, tensor weight, int density); -void atg__histogramdd_from_bin_cts_out(tensor *, tensor out, tensor self, int64_t *bins_data, int bins_len, double *range_data, int range_len, tensor weight, int density); -void atg__histogramdd_from_bin_tensors(tensor *, tensor self, tensor *bins_data, int bins_len, tensor weight, int density); -void atg__histogramdd_from_bin_tensors_out(tensor *, tensor out, tensor self, tensor *bins_data, int bins_len, tensor weight, int density); -void atg__index_put_impl(tensor *, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate, int unsafe); -void atg__index_put_impl_(tensor *, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate, int unsafe); -void atg__index_put_impl_out(tensor *, tensor out, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate, int unsafe); -void atg__indices(tensor *, tensor self); -void atg__indices_copy(tensor *, tensor self); -void atg__indices_copy_out(tensor *, tensor out, tensor self); -void atg__int_mm(tensor *, tensor self, tensor mat2); -void atg__int_mm_out(tensor *, tensor out, tensor self, tensor mat2); -void atg__is_all_true(tensor *, tensor self); -void atg__is_any_true(tensor *, tensor self); -int atg__is_zerotensor(tensor self); -void atg__lazy_clone(tensor *, tensor self); -void atg__linalg_check_errors(tensor info, char* api_name_ptr, int api_name_len, int is_matrix); -void atg__linalg_det(tensor *, tensor A); -void atg__linalg_det_result(tensor *, tensor result, tensor LU, tensor pivots, tensor A); -void atg__linalg_eigh(tensor *, tensor A, char* UPLO_ptr, int UPLO_len, int compute_v); -void atg__linalg_eigh_eigenvalues(tensor *, tensor eigenvalues, tensor eigenvectors, tensor A, char* UPLO_ptr, int UPLO_len, int compute_v); -void atg__linalg_eigvals(tensor *, tensor self); -void atg__linalg_slogdet(tensor *, tensor A); -void atg__linalg_slogdet_sign(tensor *, tensor sign, tensor logabsdet, tensor LU, tensor pivots, tensor A); -void atg__linalg_solve_ex(tensor *, tensor A, tensor B, int left, int check_errors); -void atg__linalg_solve_ex_result(tensor *, tensor result, tensor LU, tensor pivots, tensor info, tensor A, tensor B, int left, int check_errors); -void atg__linalg_svd(tensor *, tensor A, int full_matrices, int compute_uv, char* driver_ptr, int driver_len); -void atg__linalg_svd_u(tensor *, tensor U, tensor S, tensor Vh, tensor A, int full_matrices, int compute_uv, char* driver_ptr, int driver_len); -void atg__log_softmax(tensor *, tensor self, int64_t dim, int half_to_float); -void atg__log_softmax_backward_data(tensor *, tensor grad_output, tensor output, int64_t dim, int input_dtype); -void atg__log_softmax_backward_data_out(tensor *, tensor out, tensor grad_output, tensor output, int64_t dim, int input_dtype); -void atg__log_softmax_out(tensor *, tensor out, tensor self, int64_t dim, int half_to_float); -void atg__logcumsumexp(tensor *, tensor self, int64_t dim); -void atg__logcumsumexp_out(tensor *, tensor out, tensor self, int64_t dim); -void atg__lstm_mps(tensor *, tensor input, tensor *hx_data, int hx_len, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first); -void atg__lstm_mps_out(tensor *, tensor out0, tensor out1, tensor out2, tensor out3, tensor out4, tensor out5, tensor input, tensor *hx_data, int hx_len, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first); -void atg__lu_with_info(tensor *, tensor self, int pivot, int check_errors); -void atg__make_dep_token(tensor *, int options_kind, int options_device); -void atg__make_dual(tensor *, tensor primal, tensor tangent, int64_t level); -void atg__make_dual_copy(tensor *, tensor primal, tensor tangent, int64_t level); -void atg__make_dual_copy_out(tensor *, tensor out, tensor primal, tensor tangent, int64_t level); -void atg__make_per_channel_quantized_tensor(tensor *, tensor self, tensor scale, tensor zero_point, int64_t axis); -void atg__make_per_channel_quantized_tensor_out(tensor *, tensor out, tensor self, tensor scale, tensor zero_point, int64_t axis); -void atg__make_per_tensor_quantized_tensor(tensor *, tensor self, double scale, int64_t zero_point); -void atg__make_per_tensor_quantized_tensor_out(tensor *, tensor out, tensor self, double scale, int64_t zero_point); -void atg__masked_scale(tensor *, tensor self, tensor mask, double scale); -void atg__masked_scale_out(tensor *, tensor out, tensor self, tensor mask, double scale); -void atg__masked_softmax(tensor *, tensor self, tensor mask, int64_t dim_v, uint8_t dim_null, int64_t mask_type_v, uint8_t mask_type_null); -void atg__masked_softmax_backward(tensor *, tensor grad_output, tensor output, tensor mask, int64_t dim_v, uint8_t dim_null); -void atg__masked_softmax_backward_out(tensor *, tensor out, tensor grad_output, tensor output, tensor mask, int64_t dim_v, uint8_t dim_null); -void atg__masked_softmax_out(tensor *, tensor out, tensor self, tensor mask, int64_t dim_v, uint8_t dim_null, int64_t mask_type_v, uint8_t mask_type_null); -void atg__mixed_dtypes_linear(tensor *, tensor input, tensor weight, tensor scale, tensor bias, char* activation_ptr, int activation_len); -void atg__mkldnn_reshape(tensor *, tensor self, int64_t *shape_data, int shape_len); -void atg__mkldnn_reshape_out(tensor *, tensor out, tensor self, int64_t *shape_data, int shape_len); -void atg__mkldnn_transpose(tensor *, tensor self, int64_t dim0, int64_t dim1); -void atg__mkldnn_transpose_(tensor *, tensor self, int64_t dim0, int64_t dim1); -void atg__mkldnn_transpose_out(tensor *, tensor out, tensor self, int64_t dim0, int64_t dim1); -void atg__mps_convolution(tensor *, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups); -void atg__mps_convolution_out(tensor *, tensor out, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups); -void atg__mps_convolution_transpose(tensor *, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups); -void atg__mps_convolution_transpose_out(tensor *, tensor out, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups); -void atg__native_batch_norm_legit(tensor *, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps); -void atg__native_batch_norm_legit_functional(tensor *, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps); -void atg__native_batch_norm_legit_no_stats(tensor *, tensor input, tensor weight, tensor bias, int training, double momentum, double eps); -void atg__native_batch_norm_legit_no_stats_out(tensor *, tensor out, tensor save_mean, tensor save_invstd, tensor input, tensor weight, tensor bias, int training, double momentum, double eps); -void atg__native_batch_norm_legit_no_training(tensor *, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps); -void atg__native_batch_norm_legit_no_training_out(tensor *, tensor out0, tensor out1, tensor out2, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, double momentum, double eps); -void atg__native_batch_norm_legit_out(tensor *, tensor out, tensor save_mean, tensor save_invstd, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps); -void atg__native_multi_head_attention(tensor *, tensor query, tensor key, tensor value, int64_t embed_dim, int64_t num_head, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, tensor mask, int need_weights, int average_attn_weights, int64_t mask_type_v, uint8_t mask_type_null); -void atg__native_multi_head_attention_out(tensor *, tensor out0, tensor out1, tensor query, tensor key, tensor value, int64_t embed_dim, int64_t num_head, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, tensor mask, int need_weights, int average_attn_weights, int64_t mask_type_v, uint8_t mask_type_null); -void atg__neg_view(tensor *, tensor self); -void atg__neg_view_copy(tensor *, tensor self); -void atg__neg_view_copy_out(tensor *, tensor out, tensor self); -void atg__nested_compute_contiguous_strides_offsets(tensor *, tensor nested_size); -void atg__nested_from_padded(tensor *, tensor padded, tensor cpu_nested_shape_example, int fuse_transform_0213); -void atg__nested_from_padded_and_nested_example(tensor *, tensor padded, tensor nt_example); -void atg__nested_from_padded_and_nested_example_out(tensor *, tensor out, tensor padded, tensor nt_example); -void atg__nested_from_padded_out(tensor *, tensor out, tensor padded, tensor cpu_nested_shape_example, int fuse_transform_0213); -void atg__nested_from_padded_tensor(tensor *, tensor padded, tensor offsets, tensor dummy, int64_t ragged_idx, tensor min_seqlen, tensor max_seqlen, int64_t sum_S_v, uint8_t sum_S_null); -void atg__nested_get_jagged_dummy(tensor *, tensor any); -void atg__nested_get_lengths(tensor *, tensor self); -void atg__nested_get_max_seqlen(tensor *, tensor self); -void atg__nested_get_min_seqlen(tensor *, tensor self); -void atg__nested_get_offsets(tensor *, tensor self); -int64_t atg__nested_get_ragged_idx(tensor self); -void atg__nested_get_values(tensor *, tensor self); -void atg__nested_get_values_copy(tensor *, tensor self); -void atg__nested_get_values_copy_out(tensor *, tensor out, tensor self); -void atg__nested_select_backward(tensor *, tensor grad_output, tensor self, int64_t dim, int64_t index); -void atg__nested_sum_backward(tensor *, tensor grad, tensor self, int64_t *dim_data, int dim_len, int keepdim); -void atg__nested_view_from_buffer(tensor *, tensor self, tensor nested_size, tensor nested_strides, tensor offsets); -void atg__nested_view_from_buffer_copy(tensor *, tensor self, tensor nested_size, tensor nested_strides, tensor offsets); -void atg__nested_view_from_buffer_copy_out(tensor *, tensor out, tensor self, tensor nested_size, tensor nested_strides, tensor offsets); -void atg__nested_view_from_jagged(tensor *, tensor self, tensor offsets, tensor dummy, tensor lengths, int64_t ragged_idx, tensor min_seqlen, tensor max_seqlen); -void atg__nested_view_from_jagged_copy(tensor *, tensor self, tensor offsets, tensor dummy, tensor lengths, int64_t ragged_idx, tensor min_seqlen, tensor max_seqlen); -void atg__nested_view_from_jagged_copy_out(tensor *, tensor out, tensor self, tensor offsets, tensor dummy, tensor lengths, int64_t ragged_idx, tensor min_seqlen, tensor max_seqlen); -void atg__new_zeros_with_same_feature_meta(tensor *, tensor self, tensor other, int64_t self_num_batch_dims); -void atg__new_zeros_with_same_feature_meta_out(tensor *, tensor out, tensor self, tensor other, int64_t self_num_batch_dims); -int atg__nnpack_available(); -void atg__nnpack_spatial_convolution(tensor *, tensor input, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len); -void atg__nnpack_spatial_convolution_out(tensor *, tensor out, tensor input, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len); -int64_t atg__nnz(tensor self); -void atg__pack_padded_sequence(tensor *, tensor input, tensor lengths, int batch_first); -void atg__pack_padded_sequence_backward(tensor *, tensor grad, int64_t *input_size_data, int input_size_len, tensor batch_sizes, int batch_first); -void atg__pack_padded_sequence_out(tensor *, tensor out0, tensor out1, tensor input, tensor lengths, int batch_first); -void atg__pad_circular(tensor *, tensor self, int64_t *pad_data, int pad_len); -void atg__pad_enum(tensor *, tensor self, int64_t *pad_data, int pad_len, int64_t mode, double value_v, uint8_t value_null); -void atg__pad_packed_sequence(tensor *, tensor data, tensor batch_sizes, int batch_first, scalar padding_value, int64_t total_length); -void atg__pdist_backward(tensor *, tensor grad, tensor self, double p, tensor pdist); -void atg__pdist_backward_out(tensor *, tensor out, tensor grad, tensor self, double p, tensor pdist); -void atg__pin_memory(tensor *, tensor self, int device); -void atg__pin_memory_out(tensor *, tensor out, tensor self, int device); -void atg__prelu_kernel(tensor *, tensor self, tensor weight); -void atg__prelu_kernel_backward(tensor *, tensor grad_output, tensor self, tensor weight); -void atg__print(char* s_ptr, int s_len); -void atg__propagate_xla_data(tensor input, tensor output); -void atg__remove_batch_dim(tensor *, tensor self, int64_t level, int64_t batch_size, int64_t out_dim); -void atg__reshape_alias(tensor *, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len); -void atg__reshape_alias_copy(tensor *, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len); -void atg__reshape_alias_copy_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len); -void atg__reshape_copy(tensor *, tensor self, int64_t *size_data, int size_len); -void atg__reshape_from_tensor(tensor *, tensor self, tensor shape); -void atg__resize_output(tensor *, tensor self, int64_t *size_data, int size_len, int device); -void atg__resize_output_(tensor *, tensor self, int64_t *size_data, int size_len, int device); -void atg__resize_output_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len, int device); -void atg__rowwise_prune(tensor *, tensor weight, tensor mask, int compressed_indices_dtype); -void atg__safe_softmax(tensor *, tensor self, int64_t dim, int dtype); -void atg__sample_dirichlet(tensor *, tensor self); -void atg__sample_dirichlet_out(tensor *, tensor out, tensor self); -void atg__saturate_weight_to_fp16(tensor *, tensor weight); -void atg__scaled_dot_product_attention_math(tensor *, tensor query, tensor key, tensor value, tensor attn_mask, double dropout_p, int is_causal, tensor dropout_mask, double scale_v, uint8_t scale_null, int enable_gqa); -void atg__scaled_dot_product_attention_math_for_mps(tensor *, tensor query, tensor key, tensor value, tensor attn_mask, double dropout_p, int is_causal, tensor dropout_mask, double scale_v, uint8_t scale_null); -void atg__scaled_dot_product_cudnn_attention_backward(tensor *, tensor grad_out, tensor query, tensor key, tensor value, tensor out, tensor logsumexp, tensor philox_seed, tensor philox_offset, tensor attn_bias, tensor cum_seq_q, tensor cum_seq_k, int64_t max_q, int64_t max_k, double dropout_p, int is_causal, double scale_v, uint8_t scale_null); -void atg__scaled_dot_product_efficient_attention(tensor *, tensor query, tensor key, tensor value, tensor attn_bias, int compute_log_sumexp, double dropout_p, int is_causal, double scale_v, uint8_t scale_null); -void atg__scaled_dot_product_flash_attention_backward(tensor *, tensor grad_out, tensor query, tensor key, tensor value, tensor out, tensor logsumexp, tensor cum_seq_q, tensor cum_seq_k, int64_t max_q, int64_t max_k, double dropout_p, int is_causal, tensor philox_seed, tensor philox_offset, double scale_v, uint8_t scale_null); -void atg__scaled_dot_product_flash_attention_for_cpu(tensor *, tensor query, tensor key, tensor value, double dropout_p, int is_causal, tensor attn_mask, double scale_v, uint8_t scale_null); -void atg__scaled_dot_product_flash_attention_for_cpu_backward(tensor *, tensor grad_out, tensor query, tensor key, tensor value, tensor out, tensor logsumexp, double dropout_p, int is_causal, tensor attn_mask, double scale_v, uint8_t scale_null); -void atg__scaled_grouped_mm(tensor *, tensor self, tensor mat2, tensor scale_a, tensor scale_b, tensor offs, tensor bias, tensor scale_result, int out_dtype, int use_fast_accum); -void atg__scaled_grouped_mm_v2(tensor *, tensor self, tensor mat2, tensor *scale_a_data, int scale_a_len, int64_t *recipe_a_data, int recipe_a_len, int64_t *swizzle_a_data, int swizzle_a_len, tensor *scale_b_data, int scale_b_len, int64_t *recipe_b_data, int recipe_b_len, int64_t *swizzle_b_data, int swizzle_b_len, tensor offs, tensor bias, int out_dtype, int64_t *contraction_dim_data, int contraction_dim_len, int use_fast_accum); -void atg__scaled_mm(tensor *, tensor self, tensor mat2, tensor scale_a, tensor scale_b, tensor bias, tensor scale_result, int out_dtype, int use_fast_accum); -void atg__scaled_mm_out(tensor *, tensor out, tensor self, tensor mat2, tensor scale_a, tensor scale_b, tensor bias, tensor scale_result, int out_dtype, int use_fast_accum); -void atg__scaled_mm_v2(tensor *, tensor self, tensor mat2, tensor *scale_a_data, int scale_a_len, int64_t *recipe_a_data, int recipe_a_len, int64_t *swizzle_a_data, int swizzle_a_len, tensor *scale_b_data, int scale_b_len, int64_t *recipe_b_data, int recipe_b_len, int64_t *swizzle_b_data, int swizzle_b_len, tensor bias, int out_dtype, int64_t *contraction_dim_data, int contraction_dim_len, int use_fast_accum); -void atg__scaled_mm_v2_out(tensor *, tensor out, tensor self, tensor mat2, tensor *scale_a_data, int scale_a_len, int64_t *recipe_a_data, int recipe_a_len, int64_t *swizzle_a_data, int swizzle_a_len, tensor *scale_b_data, int scale_b_len, int64_t *recipe_b_data, int recipe_b_len, int64_t *swizzle_b_data, int swizzle_b_len, tensor bias, int out_dtype, int64_t *contraction_dim_data, int contraction_dim_len, int use_fast_accum); -void atg__scatter_reduce(tensor *, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len, int include_self); -void atg__scatter_reduce_(tensor *, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len, int include_self); -void atg__scatter_reduce_two_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len, int include_self); -void atg__segment_reduce_backward(tensor *, tensor grad, tensor output, tensor data, char* reduce_ptr, int reduce_len, tensor lengths, tensor offsets, int64_t axis, scalar initial); -void atg__segment_reduce_backward_out(tensor *, tensor out, tensor grad, tensor output, tensor data, char* reduce_ptr, int reduce_len, tensor lengths, tensor offsets, int64_t axis, scalar initial); -void atg__shape_as_tensor(tensor *, tensor self); -void atg__slow_conv2d_backward(tensor *, tensor grad_input, tensor grad_weight, tensor grad_bias, tensor grad_output, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len); -void atg__sobol_engine_draw(tensor *, tensor quasi, int64_t n, tensor sobolstate, int64_t dimension, int64_t num_generated, int dtype); -void atg__sobol_engine_ff_(tensor *, tensor self, int64_t n, tensor sobolstate, int64_t dimension, int64_t num_generated); -void atg__sobol_engine_initialize_state_(tensor *, tensor self, int64_t dimension); -void atg__sobol_engine_scramble_(tensor *, tensor self, tensor ltm, int64_t dimension); -void atg__softmax(tensor *, tensor self, int64_t dim, int half_to_float); -void atg__softmax_backward_data(tensor *, tensor grad_output, tensor output, int64_t dim, int input_dtype); -void atg__softmax_backward_data_out(tensor *, tensor grad_input, tensor grad_output, tensor output, int64_t dim, int input_dtype); -void atg__softmax_out(tensor *, tensor out, tensor self, int64_t dim, int half_to_float); -void atg__sparse_addmm(tensor *, tensor self, tensor mat1, tensor mat2); -void atg__sparse_addmm_out(tensor *, tensor out, tensor self, tensor mat1, tensor mat2); -void atg__sparse_broadcast_to(tensor *, tensor self, int64_t *size_data, int size_len); -void atg__sparse_broadcast_to_copy(tensor *, tensor self, int64_t *size_data, int size_len); -void atg__sparse_broadcast_to_copy_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len); -void atg__sparse_bsc_tensor_unsafe(tensor *, tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device); -void atg__sparse_bsr_tensor_unsafe(tensor *, tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device); -void atg__sparse_compressed_tensor_unsafe(tensor *, tensor compressed_indices, tensor plain_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device); -void atg__sparse_compressed_tensor_with_dims(tensor *, int64_t nnz, int64_t dense_dim, int64_t *size_data, int size_len, int64_t *blocksize_data, int blocksize_len, int index_dtype, int options_kind, int options_device); -void atg__sparse_coo_tensor_unsafe(tensor *, tensor indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device, int is_coalesced); -void atg__sparse_coo_tensor_with_dims(tensor *, int64_t sparse_dim, int64_t dense_dim, int64_t *size_data, int size_len, int options_kind, int options_device); -void atg__sparse_coo_tensor_with_dims_and_tensors(tensor *, int64_t sparse_dim, int64_t dense_dim, int64_t *size_data, int size_len, tensor indices, tensor values, int options_kind, int options_device, int is_coalesced); -void atg__sparse_coo_tensor_with_dims_and_tensors_out(tensor *, tensor out, int64_t sparse_dim, int64_t dense_dim, int64_t *size_data, int size_len, tensor indices, tensor values, int is_coalesced); -void atg__sparse_coo_tensor_with_dims_out(tensor *, tensor out, int64_t sparse_dim, int64_t dense_dim, int64_t *size_data, int size_len); -void atg__sparse_csc_tensor_unsafe(tensor *, tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device); -void atg__sparse_csr_prod(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); -void atg__sparse_csr_prod_dim_dtype_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); -void atg__sparse_csr_sum(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); -void atg__sparse_csr_sum_dim_dtype_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); -void atg__sparse_csr_tensor_unsafe(tensor *, tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device); -void atg__sparse_log_softmax(tensor *, tensor self, int64_t dim, int half_to_float); -void atg__sparse_log_softmax_backward_data(tensor *, tensor grad_output, tensor output, int64_t dim, tensor self); -void atg__sparse_log_softmax_backward_data_out(tensor *, tensor out, tensor grad_output, tensor output, int64_t dim, tensor self); -void atg__sparse_log_softmax_int(tensor *, tensor self, int64_t dim, int dtype); -void atg__sparse_log_softmax_out(tensor *, tensor out, tensor self, int64_t dim, int half_to_float); -void atg__sparse_mask_projection(tensor *, tensor self, tensor mask, int accumulate_matches); -void atg__sparse_mask_projection_out(tensor *, tensor out, tensor self, tensor mask, int accumulate_matches); -void atg__sparse_mm(tensor *, tensor sparse, tensor dense); -void atg__sparse_mm_reduce(tensor *, tensor sparse, tensor dense, char* reduce_ptr, int reduce_len); -void atg__sparse_mm_reduce_impl(tensor *, tensor self, tensor other, char* reduce_ptr, int reduce_len); -void atg__sparse_semi_structured_apply(tensor *, tensor input, tensor thread_masks); -void atg__sparse_semi_structured_apply_dense(tensor *, tensor input, tensor thread_masks); -void atg__sparse_semi_structured_linear(tensor *, tensor input, tensor weight, tensor meta, tensor bias, char* activation_ptr, int activation_len, int out_dtype); -void atg__sparse_semi_structured_mm(tensor *, tensor mat1, tensor mat1_meta, tensor mat2, int out_dtype); -void atg__sparse_semi_structured_tile(tensor *, tensor input, char* algorithm_ptr, int algorithm_len, int use_cutlass); -void atg__sparse_softmax(tensor *, tensor self, int64_t dim, int half_to_float); -void atg__sparse_softmax_backward_data(tensor *, tensor grad_output, tensor output, int64_t dim, tensor self); -void atg__sparse_softmax_backward_data_out(tensor *, tensor out, tensor grad_output, tensor output, int64_t dim, tensor self); -void atg__sparse_softmax_int(tensor *, tensor self, int64_t dim, int dtype); -void atg__sparse_softmax_out(tensor *, tensor out, tensor self, int64_t dim, int half_to_float); -void atg__sparse_sparse_matmul(tensor *, tensor self, tensor other); -void atg__sparse_sparse_matmul_out(tensor *, tensor out, tensor self, tensor other); -void atg__sparse_sum(tensor *, tensor self); -void atg__sparse_sum_backward(tensor *, tensor grad, tensor self, int64_t *dim_data, int dim_len); -void atg__sparse_sum_backward_out(tensor *, tensor out, tensor grad, tensor self, int64_t *dim_data, int dim_len); -void atg__sparse_sum_dim(tensor *, tensor self, int64_t *dim_data, int dim_len); -void atg__sparse_sum_dim_dtype(tensor *, tensor self, int64_t *dim_data, int dim_len, int dtype); -void atg__sparse_sum_dim_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len); -void atg__sparse_sum_dtype(tensor *, tensor self, int dtype); -void atg__spdiags(tensor *, tensor diagonals, tensor offsets, int64_t *shape_data, int shape_len, int8_t layout); -void atg__spdiags_out(tensor *, tensor out, tensor diagonals, tensor offsets, int64_t *shape_data, int shape_len, int8_t layout); -void atg__spsolve(tensor *, tensor A, tensor B, int left); -void atg__stack(tensor *, tensor *tensors_data, int tensors_len, int64_t dim); -void atg__stack_out(tensor *, tensor out, tensor *tensors_data, int tensors_len, int64_t dim); -void atg__standard_gamma(tensor *, tensor self); -void atg__standard_gamma_grad(tensor *, tensor self, tensor output); -void atg__standard_gamma_grad_out(tensor *, tensor out, tensor self, tensor output); -void atg__standard_gamma_out(tensor *, tensor out, tensor self); -void atg__test_ambiguous_defaults(tensor *, tensor dummy, int64_t a, int64_t b); -void atg__test_ambiguous_defaults_b(tensor *, tensor dummy, int64_t a, char* b_ptr, int b_len); -void atg__test_autograd_multiple_dispatch(tensor *, tensor self); -void atg__test_autograd_multiple_dispatch_fullcoverage_out(tensor *, tensor out, tensor self); -void atg__test_autograd_multiple_dispatch_ntonly(tensor *, tensor self, int b); -void atg__test_autograd_multiple_dispatch_view(tensor *, tensor self); -void atg__test_autograd_multiple_dispatch_view_copy(tensor *, tensor self); -void atg__test_autograd_multiple_dispatch_view_copy_out(tensor *, tensor out, tensor self); -void atg__test_check_tensor(tensor *, tensor self); -void atg__test_functorch_fallback(tensor *, tensor self, tensor other); -void atg__test_functorch_fallback_out(tensor *, tensor out, tensor self, tensor other); -void atg__test_optional_filled_intlist(tensor *, tensor values, int64_t *addends_data, int addends_len); -void atg__test_optional_filled_intlist_out(tensor *, tensor out, tensor values, int64_t *addends_data, int addends_len); -void atg__test_optional_floatlist(tensor *, tensor values, double *addends_data, int addends_len); -void atg__test_optional_floatlist_out(tensor *, tensor out, tensor values, double *addends_data, int addends_len); -void atg__test_optional_intlist(tensor *, tensor values, int64_t *addends_data, int addends_len); -void atg__test_optional_intlist_out(tensor *, tensor out, tensor values, int64_t *addends_data, int addends_len); -void atg__test_parallel_materialize(tensor *, tensor self, int64_t num_parallel, int skip_first); -void atg__test_serialization_subcmul(tensor *, tensor self, tensor other); -void atg__test_string_default(tensor *, tensor dummy, char* a_ptr, int a_len, char* b_ptr, int b_len); -void atg__test_warn_in_autograd(tensor *, tensor self); -void atg__test_warn_in_autograd_out(tensor *, tensor out, tensor self); -void atg__to_copy(tensor *, tensor self, int options_kind, int options_device, int non_blocking); -void atg__to_copy_out(tensor *, tensor out, tensor self, int non_blocking); -tensor *atg__to_cpu(tensor *tensors_data, int tensors_len); -void atg__to_dense(tensor *, tensor self, int dtype, int masked_grad); -void atg__to_dense_out(tensor *, tensor out, tensor self, int dtype, int masked_grad); -void atg__to_sparse(tensor *, tensor self, int8_t layout, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null); -void atg__to_sparse_bsc(tensor *, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null); -void atg__to_sparse_bsc_out(tensor *, tensor out, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null); -void atg__to_sparse_bsr(tensor *, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null); -void atg__to_sparse_bsr_out(tensor *, tensor out, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null); -void atg__to_sparse_csc(tensor *, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null); -void atg__to_sparse_csc_out(tensor *, tensor out, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null); -void atg__to_sparse_csr(tensor *, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null); -void atg__to_sparse_csr_out(tensor *, tensor out, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null); -void atg__to_sparse_out(tensor *, tensor out, tensor self, int8_t layout, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null); -void atg__to_sparse_semi_structured(tensor *, tensor dense); -void atg__to_sparse_sparse_dim(tensor *, tensor self, int64_t sparse_dim); -void atg__to_sparse_sparse_dim_out(tensor *, tensor out, tensor self, int64_t sparse_dim); -void atg__transform_bias_rescale_qkv(tensor *, tensor qkv, tensor qkv_bias, int64_t num_heads); -void atg__transform_bias_rescale_qkv_out(tensor *, tensor out0, tensor out1, tensor out2, tensor qkv, tensor qkv_bias, int64_t num_heads); -void atg__transformer_encoder_layer_fwd(tensor *, tensor src, int64_t embed_dim, int64_t num_heads, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, int use_gelu, int norm_first, double eps, tensor norm_weight_1, tensor norm_bias_1, tensor norm_weight_2, tensor norm_bias_2, tensor ffn_weight_1, tensor ffn_bias_1, tensor ffn_weight_2, tensor ffn_bias_2, tensor mask, int64_t mask_type_v, uint8_t mask_type_null); -void atg__transformer_encoder_layer_fwd_out(tensor *, tensor out, tensor src, int64_t embed_dim, int64_t num_heads, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, int use_gelu, int norm_first, double eps, tensor norm_weight_1, tensor norm_bias_1, tensor norm_weight_2, tensor norm_bias_2, tensor ffn_weight_1, tensor ffn_bias_1, tensor ffn_weight_2, tensor ffn_bias_2, tensor mask, int64_t mask_type_v, uint8_t mask_type_null); -void atg__trilinear(tensor *, tensor i1, tensor i2, tensor i3, int64_t *expand1_data, int expand1_len, int64_t *expand2_data, int expand2_len, int64_t *expand3_data, int expand3_len, int64_t *sumdim_data, int sumdim_len, int64_t unroll_dim); -void atg__trilinear_out(tensor *, tensor out, tensor i1, tensor i2, tensor i3, int64_t *expand1_data, int expand1_len, int64_t *expand2_data, int expand2_len, int64_t *expand3_data, int expand3_len, int64_t *sumdim_data, int sumdim_len, int64_t unroll_dim); -void atg__triton_multi_head_attention(tensor *, tensor query, tensor key, tensor value, int64_t embed_dim, int64_t num_head, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, tensor mask); -void atg__triton_multi_head_attention_out(tensor *, tensor out, tensor query, tensor key, tensor value, int64_t embed_dim, int64_t num_head, tensor qkv_weight, tensor qkv_bias, tensor proj_weight, tensor proj_bias, tensor mask); -void atg__triton_scaled_dot_attention(tensor *, tensor q, tensor k, tensor v, double dropout_p); -void atg__triton_scaled_dot_attention_out(tensor *, tensor out, tensor q, tensor k, tensor v, double dropout_p); -void atg__unique(tensor *, tensor self, int sorted, int return_inverse); -void atg__unique2(tensor *, tensor self, int sorted, int return_inverse, int return_counts); -void atg__unique2_out(tensor *, tensor out0, tensor out1, tensor out2, tensor self, int sorted, int return_inverse, int return_counts); -void atg__unique_out(tensor *, tensor out0, tensor out1, tensor self, int sorted, int return_inverse); -void atg__unpack_dual(tensor *, tensor dual, int64_t level); -void atg__unsafe_index(tensor *, tensor self, tensor *indices_data, int indices_len); -void atg__unsafe_index_put(tensor *, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate); -void atg__unsafe_masked_index(tensor *, tensor self, tensor mask, tensor *indices_data, int indices_len, scalar fill); -void atg__unsafe_masked_index_put_accumulate(tensor *, tensor self, tensor mask, tensor *indices_data, int indices_len, tensor values); -void atg__unsafe_view(tensor *, tensor self, int64_t *size_data, int size_len); -void atg__unsafe_view_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len); -void atg__upsample_bicubic2d_aa(tensor *, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg__upsample_bicubic2d_aa_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg__upsample_bicubic2d_aa_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg__upsample_bicubic2d_aa_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg__upsample_bicubic2d_aa_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len); -void atg__upsample_bilinear2d_aa(tensor *, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg__upsample_bilinear2d_aa_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg__upsample_bilinear2d_aa_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg__upsample_bilinear2d_aa_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg__upsample_bilinear2d_aa_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len); -void atg__upsample_nearest_exact1d(tensor *, tensor self, int64_t *output_size_data, int output_size_len, double scales_v, uint8_t scales_null); -void atg__upsample_nearest_exact1d_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_v, uint8_t scales_null); -void atg__upsample_nearest_exact1d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_v, uint8_t scales_null); -void atg__upsample_nearest_exact1d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_v, uint8_t scales_null); -void atg__upsample_nearest_exact1d_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len); -void atg__upsample_nearest_exact2d(tensor *, tensor self, int64_t *output_size_data, int output_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg__upsample_nearest_exact2d_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg__upsample_nearest_exact2d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg__upsample_nearest_exact2d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg__upsample_nearest_exact2d_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len); -void atg__upsample_nearest_exact3d(tensor *, tensor self, int64_t *output_size_data, int output_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg__upsample_nearest_exact3d_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg__upsample_nearest_exact3d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg__upsample_nearest_exact3d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg__upsample_nearest_exact3d_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len); -int atg__use_cudnn_ctc_loss(tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank); -int atg__use_cudnn_ctc_loss_tensor(tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank); -int atg__use_cudnn_rnn_flatten_weight(); -int atg__use_miopen_ctc_loss(tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank); -int atg__use_miopen_ctc_loss_tensor(tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank); -void atg__validate_compressed_sparse_indices(int is_crow, tensor compressed_idx, tensor plain_idx, int64_t cdim, int64_t dim, int64_t nnz); -void atg__validate_sparse_bsc_tensor_args(tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int check_pinning); -void atg__validate_sparse_bsr_tensor_args(tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int check_pinning); -void atg__validate_sparse_compressed_tensor_args(tensor compressed_indices, tensor plain_indices, tensor values, int64_t *size_data, int size_len, int8_t layout, int check_pinning); -void atg__validate_sparse_csc_tensor_args(tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int check_pinning); -void atg__validate_sparse_csr_tensor_args(tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int check_pinning); -void atg__values(tensor *, tensor self); -void atg__values_copy(tensor *, tensor self); -void atg__values_copy_out(tensor *, tensor out, tensor self); -int64_t atg__version(tensor self); -void atg__weight_int4pack_mm(tensor *, tensor self, tensor mat2, int64_t qGroupSize, tensor qScaleAndZeros); -void atg__weight_int4pack_mm_for_cpu(tensor *, tensor self, tensor mat2, int64_t qGroupSize, tensor qScaleAndZeros); -void atg__weight_int4pack_mm_with_scales_and_zeros(tensor *, tensor self, tensor mat2, int64_t qGroupSize, tensor qScale, tensor qZeros); -void atg__weight_int8pack_mm(tensor *, tensor self, tensor mat2, tensor scales); -void atg__weight_norm(tensor *, tensor v, tensor g, int64_t dim); -void atg__weight_norm_differentiable_backward(tensor *, tensor grad_w, tensor saved_v, tensor saved_g, tensor saved_norms, int64_t dim); -void atg__weight_norm_interface(tensor *, tensor v, tensor g, int64_t dim); -void atg__weight_norm_interface_backward(tensor *, tensor grad_w, tensor saved_v, tensor saved_g, tensor saved_norms, int64_t dim); -void atg__weight_norm_interface_backward_out(tensor *, tensor out0, tensor out1, tensor grad_w, tensor saved_v, tensor saved_g, tensor saved_norms, int64_t dim); -void atg__weight_norm_interface_out(tensor *, tensor out0, tensor out1, tensor v, tensor g, int64_t dim); -void atg__wrapped_linear_prepack(tensor *, tensor weight, tensor weight_scale, tensor weight_zero_point, tensor bias); -void atg__wrapped_quantized_linear_prepacked(tensor *, tensor input, tensor input_scale, tensor input_zero_point, tensor packed_weight, tensor output_scale, tensor output_zero_point, int64_t out_channel); -void atg_abs(tensor *, tensor self); -void atg_abs_(tensor *, tensor self); -void atg_abs_out(tensor *, tensor out, tensor self); -void atg_absolute(tensor *, tensor self); -void atg_absolute_(tensor *, tensor self); -void atg_absolute_out(tensor *, tensor out, tensor self); -void atg_acos(tensor *, tensor self); -void atg_acos_(tensor *, tensor self); -void atg_acos_out(tensor *, tensor out, tensor self); -void atg_acosh(tensor *, tensor self); -void atg_acosh_(tensor *, tensor self); -void atg_acosh_out(tensor *, tensor out, tensor self); -void atg_adaptive_avg_pool1d(tensor *, tensor self, int64_t *output_size_data, int output_size_len); -void atg_adaptive_avg_pool1d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len); -void atg_adaptive_avg_pool2d(tensor *, tensor self, int64_t *output_size_data, int output_size_len); -void atg_adaptive_avg_pool2d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len); -void atg_adaptive_avg_pool3d(tensor *, tensor self, int64_t *output_size_data, int output_size_len); -void atg_adaptive_avg_pool3d_backward(tensor *, tensor grad_input, tensor grad_output, tensor self); -void atg_adaptive_avg_pool3d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len); -void atg_adaptive_max_pool1d(tensor *, tensor self, int64_t *output_size_data, int output_size_len); -void atg_adaptive_max_pool2d(tensor *, tensor self, int64_t *output_size_data, int output_size_len); -void atg_adaptive_max_pool2d_backward(tensor *, tensor grad_output, tensor self, tensor indices); -void atg_adaptive_max_pool2d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor indices); -void atg_adaptive_max_pool2d_out(tensor *, tensor out, tensor indices, tensor self, int64_t *output_size_data, int output_size_len); -void atg_adaptive_max_pool3d(tensor *, tensor self, int64_t *output_size_data, int output_size_len); -void atg_adaptive_max_pool3d_backward(tensor *, tensor grad_output, tensor self, tensor indices); -void atg_adaptive_max_pool3d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor indices); -void atg_adaptive_max_pool3d_out(tensor *, tensor out, tensor indices, tensor self, int64_t *output_size_data, int output_size_len); -void atg_add(tensor *, tensor self, tensor other); -void atg_add_(tensor *, tensor self, tensor other); -void atg_add_out(tensor *, tensor out, tensor self, tensor other); -void atg_add_scalar(tensor *, tensor self, scalar other); -void atg_add_scalar_(tensor *, tensor self, scalar other); -void atg_add_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_addbmm(tensor *, tensor self, tensor batch1, tensor batch2); -void atg_addbmm_(tensor *, tensor self, tensor batch1, tensor batch2); -void atg_addbmm_out(tensor *, tensor out, tensor self, tensor batch1, tensor batch2); -void atg_addcdiv(tensor *, tensor self, tensor tensor1, tensor tensor2); -void atg_addcdiv_(tensor *, tensor self, tensor tensor1, tensor tensor2); -void atg_addcdiv_out(tensor *, tensor out, tensor self, tensor tensor1, tensor tensor2); -void atg_addcmul(tensor *, tensor self, tensor tensor1, tensor tensor2); -void atg_addcmul_(tensor *, tensor self, tensor tensor1, tensor tensor2); -void atg_addcmul_out(tensor *, tensor out, tensor self, tensor tensor1, tensor tensor2); -void atg_addmm(tensor *, tensor self, tensor mat1, tensor mat2); -void atg_addmm_(tensor *, tensor self, tensor mat1, tensor mat2); -void atg_addmm_dtype(tensor *, tensor self, tensor mat1, tensor mat2, int out_dtype); -void atg_addmm_dtype_out(tensor *, tensor out, tensor self, tensor mat1, tensor mat2, int out_dtype); -void atg_addmm_out(tensor *, tensor out, tensor self, tensor mat1, tensor mat2); -void atg_addmv(tensor *, tensor self, tensor mat, tensor vec); -void atg_addmv_(tensor *, tensor self, tensor mat, tensor vec); -void atg_addmv_out(tensor *, tensor out, tensor self, tensor mat, tensor vec); -void atg_addr(tensor *, tensor self, tensor vec1, tensor vec2); -void atg_addr_(tensor *, tensor self, tensor vec1, tensor vec2); -void atg_addr_out(tensor *, tensor out, tensor self, tensor vec1, tensor vec2); -void atg_adjoint(tensor *, tensor self); -void atg_affine_grid_generator(tensor *, tensor theta, int64_t *size_data, int size_len, int align_corners); -void atg_affine_grid_generator_backward(tensor *, tensor grad, int64_t *size_data, int size_len, int align_corners); -void atg_affine_grid_generator_out(tensor *, tensor out, tensor theta, int64_t *size_data, int size_len, int align_corners); -void atg_alias(tensor *, tensor self); -void atg_alias_copy(tensor *, tensor self); -void atg_alias_copy_out(tensor *, tensor out, tensor self); -void atg_align_as(tensor *, tensor self, tensor other); -tensor *atg_align_tensors(tensor *tensors_data, int tensors_len); -void atg_all(tensor *, tensor self); -void atg_all_all_out(tensor *, tensor out, tensor self); -void atg_all_dim(tensor *, tensor self, int64_t dim, int keepdim); -void atg_all_dims(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim); -void atg_all_dims_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim); -void atg_all_out(tensor *, tensor out, tensor self, int64_t dim, int keepdim); -int atg_allclose(tensor self, tensor other, double rtol, double atol, int equal_nan); -void atg_alpha_dropout(tensor *, tensor input, double p, int train); -void atg_alpha_dropout_(tensor *, tensor self, double p, int train); -void atg_amax(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim); -void atg_amax_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim); -void atg_amin(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim); -void atg_amin_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim); -void atg_aminmax(tensor *, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim); -void atg_aminmax_out(tensor *, tensor min, tensor max, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim); -void atg_angle(tensor *, tensor self); -void atg_angle_out(tensor *, tensor out, tensor self); -void atg_any(tensor *, tensor self); -void atg_any_all_out(tensor *, tensor out, tensor self); -void atg_any_dim(tensor *, tensor self, int64_t dim, int keepdim); -void atg_any_dims(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim); -void atg_any_dims_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim); -void atg_any_out(tensor *, tensor out, tensor self, int64_t dim, int keepdim); -void atg_arange(tensor *, scalar end, int options_kind, int options_device); -void atg_arange_start(tensor *, scalar start, scalar end, int options_kind, int options_device); -void atg_arange_start_step(tensor *, scalar start, scalar end, scalar step, int options_kind, int options_device); -void atg_arccos(tensor *, tensor self); -void atg_arccos_(tensor *, tensor self); -void atg_arccos_out(tensor *, tensor out, tensor self); -void atg_arccosh(tensor *, tensor self); -void atg_arccosh_(tensor *, tensor self); -void atg_arccosh_out(tensor *, tensor out, tensor self); -void atg_arcsin(tensor *, tensor self); -void atg_arcsin_(tensor *, tensor self); -void atg_arcsin_out(tensor *, tensor out, tensor self); -void atg_arcsinh(tensor *, tensor self); -void atg_arcsinh_(tensor *, tensor self); -void atg_arcsinh_out(tensor *, tensor out, tensor self); -void atg_arctan(tensor *, tensor self); -void atg_arctan2(tensor *, tensor self, tensor other); -void atg_arctan2_(tensor *, tensor self, tensor other); -void atg_arctan2_out(tensor *, tensor out, tensor self, tensor other); -void atg_arctan_(tensor *, tensor self); -void atg_arctan_out(tensor *, tensor out, tensor self); -void atg_arctanh(tensor *, tensor self); -void atg_arctanh_(tensor *, tensor self); -void atg_arctanh_out(tensor *, tensor out, tensor self); -void atg_argmax(tensor *, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim); -void atg_argmax_out(tensor *, tensor out, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim); -void atg_argmin(tensor *, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim); -void atg_argmin_out(tensor *, tensor out, tensor self, int64_t dim_v, uint8_t dim_null, int keepdim); -void atg_argsort(tensor *, tensor self, int64_t dim, int descending); -void atg_argsort_stable(tensor *, tensor self, int stable, int64_t dim, int descending); -void atg_argsort_stable_out(tensor *, tensor out, tensor self, int stable, int64_t dim, int descending); -void atg_argwhere(tensor *, tensor self); -void atg_as_strided(tensor *, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null); -void atg_as_strided_(tensor *, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null); -void atg_as_strided_copy(tensor *, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null); -void atg_as_strided_copy_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null); -void atg_as_strided_scatter(tensor *, tensor self, tensor src, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null); -void atg_as_strided_scatter_out(tensor *, tensor out, tensor self, tensor src, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int64_t storage_offset_v, uint8_t storage_offset_null); -void atg_asin(tensor *, tensor self); -void atg_asin_(tensor *, tensor self); -void atg_asin_out(tensor *, tensor out, tensor self); -void atg_asinh(tensor *, tensor self); -void atg_asinh_(tensor *, tensor self); -void atg_asinh_out(tensor *, tensor out, tensor self); -void atg_atan(tensor *, tensor self); -void atg_atan2(tensor *, tensor self, tensor other); -void atg_atan2_(tensor *, tensor self, tensor other); -void atg_atan2_out(tensor *, tensor out, tensor self, tensor other); -void atg_atan_(tensor *, tensor self); -void atg_atan_out(tensor *, tensor out, tensor self); -void atg_atanh(tensor *, tensor self); -void atg_atanh_(tensor *, tensor self); -void atg_atanh_out(tensor *, tensor out, tensor self); -void atg_atleast_1d(tensor *, tensor self); -tensor *atg_atleast_1d_sequence(tensor *tensors_data, int tensors_len); -void atg_atleast_2d(tensor *, tensor self); -tensor *atg_atleast_2d_sequence(tensor *tensors_data, int tensors_len); -void atg_atleast_3d(tensor *, tensor self); -tensor *atg_atleast_3d_sequence(tensor *tensors_data, int tensors_len); -void atg_avg_pool1d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad); -void atg_avg_pool1d_out(tensor *, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad); -void atg_avg_pool2d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null); -void atg_avg_pool2d_backward(tensor *, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null); -void atg_avg_pool2d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null); -void atg_avg_pool2d_out(tensor *, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null); -void atg_avg_pool3d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null); -void atg_avg_pool3d_backward(tensor *, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null); -void atg_avg_pool3d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null); -void atg_avg_pool3d_out(tensor *, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int ceil_mode, int count_include_pad, int64_t divisor_override_v, uint8_t divisor_override_null); -void atg_baddbmm(tensor *, tensor self, tensor batch1, tensor batch2, scalar beta, scalar alpha); -void atg_baddbmm_(tensor *, tensor self, tensor batch1, tensor batch2); -void atg_baddbmm_dtype(tensor *, tensor self, tensor batch1, tensor batch2, int out_dtype, scalar beta, scalar alpha); -void atg_baddbmm_dtype_out(tensor *, tensor out, tensor self, tensor batch1, tensor batch2, int out_dtype); -void atg_baddbmm_out(tensor *, tensor out, tensor self, tensor batch1, tensor batch2); -void atg_bartlett_window(tensor *, int64_t window_length, int options_kind, int options_device); -void atg_bartlett_window_out(tensor *, tensor out, int64_t window_length); -void atg_bartlett_window_periodic(tensor *, int64_t window_length, int periodic, int options_kind, int options_device); -void atg_bartlett_window_periodic_out(tensor *, tensor out, int64_t window_length, int periodic); -void atg_batch_norm(tensor *, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps, int cudnn_enabled); -void atg_batch_norm_backward_elemt(tensor *, tensor grad_out, tensor input, tensor mean, tensor invstd, tensor weight, tensor sum_dy, tensor sum_dy_xmu, tensor count); -void atg_batch_norm_backward_elemt_out(tensor *, tensor out, tensor grad_out, tensor input, tensor mean, tensor invstd, tensor weight, tensor sum_dy, tensor sum_dy_xmu, tensor count); -void atg_batch_norm_backward_reduce(tensor *, tensor grad_out, tensor input, tensor mean, tensor invstd, tensor weight, int input_g, int weight_g, int bias_g); -void atg_batch_norm_backward_reduce_out(tensor *, tensor out0, tensor out1, tensor out2, tensor out3, tensor grad_out, tensor input, tensor mean, tensor invstd, tensor weight, int input_g, int weight_g, int bias_g); -void atg_batch_norm_elemt(tensor *, tensor input, tensor weight, tensor bias, tensor mean, tensor invstd, double eps); -void atg_batch_norm_elemt_out(tensor *, tensor out, tensor input, tensor weight, tensor bias, tensor mean, tensor invstd, double eps); -void atg_batch_norm_gather_stats(tensor *, tensor input, tensor mean, tensor invstd, tensor running_mean, tensor running_var, double momentum, double eps, int64_t count); -void atg_batch_norm_gather_stats_out(tensor *, tensor out0, tensor out1, tensor input, tensor mean, tensor invstd, tensor running_mean, tensor running_var, double momentum, double eps, int64_t count); -void atg_batch_norm_gather_stats_with_counts(tensor *, tensor input, tensor mean, tensor invstd, tensor running_mean, tensor running_var, double momentum, double eps, tensor counts); -void atg_batch_norm_gather_stats_with_counts_out(tensor *, tensor out0, tensor out1, tensor input, tensor mean, tensor invstd, tensor running_mean, tensor running_var, double momentum, double eps, tensor counts); -void atg_batch_norm_stats(tensor *, tensor input, double eps); -void atg_batch_norm_stats_out(tensor *, tensor out0, tensor out1, tensor input, double eps); -void atg_batch_norm_update_stats(tensor *, tensor input, tensor running_mean, tensor running_var, double momentum); -void atg_batch_norm_update_stats_out(tensor *, tensor out0, tensor out1, tensor input, tensor running_mean, tensor running_var, double momentum); -void atg_bernoulli(tensor *, tensor self); -void atg_bernoulli_(tensor *, tensor self, tensor p); -void atg_bernoulli_float_(tensor *, tensor self, double p); -void atg_bernoulli_p(tensor *, tensor self, double p); -void atg_bernoulli_tensor(tensor *, tensor self, tensor p); -void atg_bilinear(tensor *, tensor input1, tensor input2, tensor weight, tensor bias); -void atg_binary_cross_entropy(tensor *, tensor self, tensor target, tensor weight, int64_t reduction); -void atg_binary_cross_entropy_backward(tensor *, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction); -void atg_binary_cross_entropy_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction); -void atg_binary_cross_entropy_out(tensor *, tensor out, tensor self, tensor target, tensor weight, int64_t reduction); -void atg_binary_cross_entropy_with_logits(tensor *, tensor self, tensor target, tensor weight, tensor pos_weight, int64_t reduction); -void atg_binary_cross_entropy_with_logits_out(tensor *, tensor out, tensor self, tensor target, tensor weight, tensor pos_weight, int64_t reduction); -void atg_bincount(tensor *, tensor self, tensor weights, int64_t minlength); -void atg_bincount_out(tensor *, tensor out, tensor self, tensor weights, int64_t minlength); -void atg_binomial(tensor *, tensor count, tensor prob); -void atg_binomial_out(tensor *, tensor out, tensor count, tensor prob); -void atg_bitwise_and(tensor *, tensor self, scalar other); -void atg_bitwise_and_(tensor *, tensor self, scalar other); -void atg_bitwise_and_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_bitwise_and_scalar_tensor(tensor *, scalar self_scalar, tensor other); -void atg_bitwise_and_scalar_tensor_out(tensor *, tensor out, scalar self_scalar, tensor other); -void atg_bitwise_and_tensor(tensor *, tensor self, tensor other); -void atg_bitwise_and_tensor_(tensor *, tensor self, tensor other); -void atg_bitwise_and_tensor_out(tensor *, tensor out, tensor self, tensor other); -void atg_bitwise_left_shift(tensor *, tensor self, tensor other); -void atg_bitwise_left_shift_(tensor *, tensor self, tensor other); -void atg_bitwise_left_shift_scalar_tensor(tensor *, scalar self_scalar, tensor other); -void atg_bitwise_left_shift_scalar_tensor_out(tensor *, tensor out, scalar self_scalar, tensor other); -void atg_bitwise_left_shift_tensor_out(tensor *, tensor out, tensor self, tensor other); -void atg_bitwise_left_shift_tensor_scalar(tensor *, tensor self, scalar other); -void atg_bitwise_left_shift_tensor_scalar_(tensor *, tensor self, scalar other); -void atg_bitwise_left_shift_tensor_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_bitwise_not(tensor *, tensor self); -void atg_bitwise_not_(tensor *, tensor self); -void atg_bitwise_not_out(tensor *, tensor out, tensor self); -void atg_bitwise_or(tensor *, tensor self, scalar other); -void atg_bitwise_or_(tensor *, tensor self, scalar other); -void atg_bitwise_or_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_bitwise_or_scalar_tensor(tensor *, scalar self_scalar, tensor other); -void atg_bitwise_or_scalar_tensor_out(tensor *, tensor out, scalar self_scalar, tensor other); -void atg_bitwise_or_tensor(tensor *, tensor self, tensor other); -void atg_bitwise_or_tensor_(tensor *, tensor self, tensor other); -void atg_bitwise_or_tensor_out(tensor *, tensor out, tensor self, tensor other); -void atg_bitwise_right_shift(tensor *, tensor self, tensor other); -void atg_bitwise_right_shift_(tensor *, tensor self, tensor other); -void atg_bitwise_right_shift_scalar_tensor(tensor *, scalar self_scalar, tensor other); -void atg_bitwise_right_shift_scalar_tensor_out(tensor *, tensor out, scalar self_scalar, tensor other); -void atg_bitwise_right_shift_tensor_out(tensor *, tensor out, tensor self, tensor other); -void atg_bitwise_right_shift_tensor_scalar(tensor *, tensor self, scalar other); -void atg_bitwise_right_shift_tensor_scalar_(tensor *, tensor self, scalar other); -void atg_bitwise_right_shift_tensor_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_bitwise_xor(tensor *, tensor self, scalar other); -void atg_bitwise_xor_(tensor *, tensor self, scalar other); -void atg_bitwise_xor_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_bitwise_xor_scalar_tensor(tensor *, scalar self_scalar, tensor other); -void atg_bitwise_xor_scalar_tensor_out(tensor *, tensor out, scalar self_scalar, tensor other); -void atg_bitwise_xor_tensor(tensor *, tensor self, tensor other); -void atg_bitwise_xor_tensor_(tensor *, tensor self, tensor other); -void atg_bitwise_xor_tensor_out(tensor *, tensor out, tensor self, tensor other); -void atg_blackman_window(tensor *, int64_t window_length, int options_kind, int options_device); -void atg_blackman_window_out(tensor *, tensor out, int64_t window_length); -void atg_blackman_window_periodic(tensor *, int64_t window_length, int periodic, int options_kind, int options_device); -void atg_blackman_window_periodic_out(tensor *, tensor out, int64_t window_length, int periodic); -void atg_block_diag(tensor *, tensor *tensors_data, int tensors_len); -void atg_block_diag_out(tensor *, tensor out, tensor *tensors_data, int tensors_len); -void atg_bmm(tensor *, tensor self, tensor mat2); -void atg_bmm_dtype(tensor *, tensor self, tensor mat2, int out_dtype); -void atg_bmm_dtype_out(tensor *, tensor out, tensor self, tensor mat2, int out_dtype); -void atg_bmm_out(tensor *, tensor out, tensor self, tensor mat2); -tensor *atg_broadcast_tensors(tensor *tensors_data, int tensors_len); -void atg_broadcast_to(tensor *, tensor self, int64_t *size_data, int size_len); -void atg_bucketize(tensor *, tensor self, tensor boundaries, int out_int32, int right); -void atg_bucketize_scalar(tensor *, scalar self_scalar, tensor boundaries, int out_int32, int right); -void atg_bucketize_scalar_out(tensor *, tensor out, scalar self_scalar, tensor boundaries, int out_int32, int right); -void atg_bucketize_tensor_out(tensor *, tensor out, tensor self, tensor boundaries, int out_int32, int right); -int atg_can_cast(int from_, int to); -void atg_cartesian_prod(tensor *, tensor *tensors_data, int tensors_len); -void atg_cat(tensor *, tensor *tensors_data, int tensors_len, int64_t dim); -void atg_cat_out(tensor *, tensor out, tensor *tensors_data, int tensors_len, int64_t dim); -void atg_cauchy(tensor *, tensor self, double median, double sigma); -void atg_cauchy_(tensor *, tensor self, double median, double sigma); -void atg_cauchy_out(tensor *, tensor out, tensor self, double median, double sigma); -void atg_ccol_indices(tensor *, tensor self); -void atg_ccol_indices_copy(tensor *, tensor self); -void atg_ccol_indices_copy_out(tensor *, tensor out, tensor self); -void atg_cdist(tensor *, tensor x1, tensor x2, double p, int64_t compute_mode_v, uint8_t compute_mode_null); -void atg_ceil(tensor *, tensor self); -void atg_ceil_(tensor *, tensor self); -void atg_ceil_out(tensor *, tensor out, tensor self); -void atg_celu(tensor *, tensor self); -void atg_celu_(tensor *, tensor self); -void atg_celu_out(tensor *, tensor out, tensor self); -void atg_chain_matmul(tensor *, tensor *matrices_data, int matrices_len); -void atg_chain_matmul_out(tensor *, tensor out, tensor *matrices_data, int matrices_len); -void atg_chalf(tensor *, tensor self); -void atg_channel_shuffle(tensor *, tensor self, int64_t groups); -void atg_channel_shuffle_out(tensor *, tensor out, tensor self, int64_t groups); -void atg_cholesky(tensor *, tensor self, int upper); -void atg_cholesky_inverse(tensor *, tensor self, int upper); -void atg_cholesky_inverse_out(tensor *, tensor out, tensor self, int upper); -void atg_cholesky_out(tensor *, tensor out, tensor self, int upper); -void atg_cholesky_solve(tensor *, tensor self, tensor input2, int upper); -void atg_cholesky_solve_out(tensor *, tensor out, tensor self, tensor input2, int upper); -void atg_choose_qparams_optimized(tensor *, tensor input, int64_t numel, int64_t n_bins, double ratio, int64_t bit_width); -tensor *atg_chunk(tensor self, int64_t chunks, int64_t dim); -void atg_clamp(tensor *, tensor self, scalar min, scalar max); -void atg_clamp_(tensor *, tensor self, scalar min, scalar max); -void atg_clamp_max(tensor *, tensor self, scalar max); -void atg_clamp_max_(tensor *, tensor self, scalar max); -void atg_clamp_max_out(tensor *, tensor out, tensor self, scalar max); -void atg_clamp_max_tensor(tensor *, tensor self, tensor max); -void atg_clamp_max_tensor_(tensor *, tensor self, tensor max); -void atg_clamp_max_tensor_out(tensor *, tensor out, tensor self, tensor max); -void atg_clamp_min(tensor *, tensor self, scalar min); -void atg_clamp_min_(tensor *, tensor self, scalar min); -void atg_clamp_min_out(tensor *, tensor out, tensor self, scalar min); -void atg_clamp_min_tensor(tensor *, tensor self, tensor min); -void atg_clamp_min_tensor_(tensor *, tensor self, tensor min); -void atg_clamp_min_tensor_out(tensor *, tensor out, tensor self, tensor min); -void atg_clamp_out(tensor *, tensor out, tensor self, scalar min, scalar max); -void atg_clamp_tensor(tensor *, tensor self, tensor min, tensor max); -void atg_clamp_tensor_(tensor *, tensor self, tensor min, tensor max); -void atg_clamp_tensor_out(tensor *, tensor out, tensor self, tensor min, tensor max); -void atg_clip(tensor *, tensor self, scalar min, scalar max); -void atg_clip_(tensor *, tensor self, scalar min, scalar max); -void atg_clip_out(tensor *, tensor out, tensor self, scalar min, scalar max); -void atg_clip_tensor(tensor *, tensor self, tensor min, tensor max); -void atg_clip_tensor_(tensor *, tensor self, tensor min, tensor max); -void atg_clip_tensor_out(tensor *, tensor out, tensor self, tensor min, tensor max); -void atg_clone(tensor *, tensor out, tensor self); -void atg_coalesce(tensor *, tensor self); -void atg_col2im(tensor *, tensor self, int64_t *output_size_data, int output_size_len, int64_t *kernel_size_data, int kernel_size_len, int64_t *dilation_data, int dilation_len, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len); -void atg_col2im_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int64_t *kernel_size_data, int kernel_size_len, int64_t *dilation_data, int dilation_len, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len); -void atg_col_indices(tensor *, tensor self); -void atg_col_indices_copy(tensor *, tensor self); -void atg_col_indices_copy_out(tensor *, tensor out, tensor self); -void atg_column_stack(tensor *, tensor *tensors_data, int tensors_len); -void atg_column_stack_out(tensor *, tensor out, tensor *tensors_data, int tensors_len); -void atg_combinations(tensor *, tensor self, int64_t r, int with_replacement); -void atg_complex(tensor *, tensor real, tensor imag); -void atg_complex_out(tensor *, tensor out, tensor real, tensor imag); -void atg_concat(tensor *, tensor *tensors_data, int tensors_len, int64_t dim); -void atg_concat_out(tensor *, tensor out, tensor *tensors_data, int tensors_len, int64_t dim); -void atg_concatenate(tensor *, tensor *tensors_data, int tensors_len, int64_t dim); -void atg_concatenate_out(tensor *, tensor out, tensor *tensors_data, int tensors_len, int64_t dim); -void atg_conj(tensor *, tensor self); -void atg_conj_physical(tensor *, tensor self); -void atg_conj_physical_(tensor *, tensor self); -void atg_conj_physical_out(tensor *, tensor out, tensor self); -void atg_constant_pad_nd(tensor *, tensor self, int64_t *pad_data, int pad_len); -void atg_constant_pad_nd_out(tensor *, tensor out, tensor self, int64_t *pad_data, int pad_len); -void atg_contiguous(tensor *, tensor self); -void atg_conv1d(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); -void atg_conv1d_padding(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, char* padding_ptr, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); -void atg_conv2d(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); -void atg_conv2d_padding(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, char* padding_ptr, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); -void atg_conv3d(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); -void atg_conv3d_padding(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, char* padding_ptr, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); -void atg_conv_depthwise3d(tensor *, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len); -void atg_conv_depthwise3d_out(tensor *, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len); -void atg_conv_tbc(tensor *, tensor self, tensor weight, tensor bias, int64_t pad); -void atg_conv_tbc_backward(tensor *, tensor self, tensor input, tensor weight, tensor bias, int64_t pad); -void atg_conv_tbc_out(tensor *, tensor out, tensor self, tensor weight, tensor bias, int64_t pad); -void atg_conv_transpose1d(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t groups, int64_t *dilation_data, int dilation_len); -void atg_conv_transpose2d(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t groups, int64_t *dilation_data, int dilation_len); -void atg_conv_transpose3d(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t groups, int64_t *dilation_data, int dilation_len); -void atg_convolution(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups); -void atg_convolution_out(tensor *, tensor out, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups); -void atg_convolution_overrideable(tensor *, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups); -void atg_convolution_overrideable_out(tensor *, tensor out, tensor input, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int transposed, int64_t *output_padding_data, int output_padding_len, int64_t groups); -void atg_copy_sparse_to_sparse(tensor *, tensor self, tensor src, int non_blocking); -void atg_copy_sparse_to_sparse_(tensor *, tensor self, tensor src, int non_blocking); -void atg_copy_sparse_to_sparse_out(tensor *, tensor out, tensor self, tensor src, int non_blocking); -void atg_copysign(tensor *, tensor self, tensor other); -void atg_copysign_(tensor *, tensor self, tensor other); -void atg_copysign_out(tensor *, tensor out, tensor self, tensor other); -void atg_copysign_scalar(tensor *, tensor self, scalar other); -void atg_copysign_scalar_(tensor *, tensor self, scalar other); -void atg_copysign_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_corrcoef(tensor *, tensor self); -void atg_cos(tensor *, tensor self); -void atg_cos_(tensor *, tensor self); -void atg_cos_out(tensor *, tensor out, tensor self); -void atg_cosh(tensor *, tensor self); -void atg_cosh_(tensor *, tensor self); -void atg_cosh_out(tensor *, tensor out, tensor self); -void atg_cosine_embedding_loss(tensor *, tensor input1, tensor input2, tensor target, double margin, int64_t reduction); -void atg_cosine_similarity(tensor *, tensor x1, tensor x2, int64_t dim, double eps); -void atg_count_nonzero(tensor *, tensor self, int64_t dim_v, uint8_t dim_null); -void atg_count_nonzero_dim_intlist(tensor *, tensor self, int64_t *dim_data, int dim_len); -void atg_count_nonzero_dim_intlist_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len); -void atg_count_nonzero_out(tensor *, tensor out, tensor self, int64_t dim_v, uint8_t dim_null); -void atg_cov(tensor *, tensor self, int64_t correction, tensor fweights, tensor aweights); -void atg_cross(tensor *, tensor self, tensor other, int64_t dim_v, uint8_t dim_null); -void atg_cross_entropy_loss(tensor *, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index, double label_smoothing); -void atg_cross_out(tensor *, tensor out, tensor self, tensor other, int64_t dim_v, uint8_t dim_null); -void atg_crow_indices(tensor *, tensor self); -void atg_crow_indices_copy(tensor *, tensor self); -void atg_crow_indices_copy_out(tensor *, tensor out, tensor self); -void atg_ctc_loss(tensor *, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int64_t reduction, int zero_infinity); -void atg_ctc_loss_tensor(tensor *, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank, int64_t reduction, int zero_infinity); -void atg_cudnn_affine_grid_generator(tensor *, tensor theta, int64_t n, int64_t C, int64_t H, int64_t W); -void atg_cudnn_affine_grid_generator_backward(tensor *, tensor grad, int64_t n, int64_t C, int64_t H, int64_t W); -void atg_cudnn_affine_grid_generator_backward_out(tensor *, tensor out, tensor grad, int64_t n, int64_t C, int64_t H, int64_t W); -void atg_cudnn_affine_grid_generator_out(tensor *, tensor out, tensor theta, int64_t n, int64_t C, int64_t H, int64_t W); -void atg_cudnn_batch_norm(tensor *, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double exponential_average_factor, double epsilon); -void atg_cudnn_batch_norm_backward(tensor *, tensor input, tensor grad_output, tensor weight, tensor running_mean, tensor running_var, tensor save_mean, tensor save_var, double epsilon, tensor reserveSpace); -void atg_cudnn_batch_norm_backward_out(tensor *, tensor out0, tensor out1, tensor out2, tensor input, tensor grad_output, tensor weight, tensor running_mean, tensor running_var, tensor save_mean, tensor save_var, double epsilon, tensor reserveSpace); -void atg_cudnn_batch_norm_out(tensor *, tensor out0, tensor out1, tensor out2, tensor out3, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double exponential_average_factor, double epsilon); -void atg_cudnn_convolution(tensor *, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic, int allow_tf32); -void atg_cudnn_convolution_add_relu(tensor *, tensor self, tensor weight, tensor z, scalar alpha, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); -void atg_cudnn_convolution_add_relu_out(tensor *, tensor out, tensor self, tensor weight, tensor z, scalar alpha, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); -void atg_cudnn_convolution_out(tensor *, tensor out, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic, int allow_tf32); -void atg_cudnn_convolution_relu(tensor *, tensor self, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); -void atg_cudnn_convolution_relu_out(tensor *, tensor out, tensor self, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); -void atg_cudnn_convolution_transpose(tensor *, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic, int allow_tf32); -void atg_cudnn_convolution_transpose_out(tensor *, tensor out, tensor self, tensor weight, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic, int allow_tf32); -void atg_cudnn_grid_sampler(tensor *, tensor self, tensor grid); -void atg_cudnn_grid_sampler_backward(tensor *, tensor self, tensor grid, tensor grad_output); -void atg_cudnn_grid_sampler_backward_out(tensor *, tensor out0, tensor out1, tensor self, tensor grid, tensor grad_output); -void atg_cudnn_grid_sampler_out(tensor *, tensor out, tensor self, tensor grid); -int atg_cudnn_is_acceptable(tensor self); -void atg_cummax(tensor *, tensor self, int64_t dim); -void atg_cummax_out(tensor *, tensor values, tensor indices, tensor self, int64_t dim); -void atg_cummaxmin_backward(tensor *, tensor grad, tensor input, tensor indices, int64_t dim); -void atg_cummin(tensor *, tensor self, int64_t dim); -void atg_cummin_out(tensor *, tensor values, tensor indices, tensor self, int64_t dim); -void atg_cumprod(tensor *, tensor self, int64_t dim, int dtype); -void atg_cumprod_(tensor *, tensor self, int64_t dim, int dtype); -void atg_cumprod_backward(tensor *, tensor grad, tensor input, int64_t dim, tensor output); -void atg_cumprod_out(tensor *, tensor out, tensor self, int64_t dim, int dtype); -void atg_cumsum(tensor *, tensor self, int64_t dim, int dtype); -void atg_cumsum_(tensor *, tensor self, int64_t dim, int dtype); -void atg_cumsum_out(tensor *, tensor out, tensor self, int64_t dim, int dtype); -void atg_cumulative_trapezoid(tensor *, tensor y, int64_t dim); -void atg_cumulative_trapezoid_x(tensor *, tensor y, tensor x, int64_t dim); -void atg_data(tensor *, tensor self); -void atg_deg2rad(tensor *, tensor self); -void atg_deg2rad_(tensor *, tensor self); -void atg_deg2rad_out(tensor *, tensor out, tensor self); -int64_t atg_dense_dim(tensor self); -void atg_dequantize(tensor *, tensor self); -void atg_dequantize_self_out(tensor *, tensor out, tensor self); -tensor *atg_dequantize_tensors(tensor *tensors_data, int tensors_len); -void atg_dequantize_tensors_out(tensor *out_data, int out_len, tensor *tensors_data, int tensors_len); -void atg_det(tensor *, tensor self); -void atg_detach(tensor *, tensor self); -void atg_detach_(tensor *, tensor self); -void atg_detach_copy(tensor *, tensor self); -void atg_detach_copy_out(tensor *, tensor out, tensor self); -void atg_diag(tensor *, tensor self, int64_t diagonal); -void atg_diag_embed(tensor *, tensor self, int64_t offset, int64_t dim1, int64_t dim2); -void atg_diag_embed_out(tensor *, tensor out, tensor self, int64_t offset, int64_t dim1, int64_t dim2); -void atg_diag_out(tensor *, tensor out, tensor self, int64_t diagonal); -void atg_diagflat(tensor *, tensor self, int64_t offset); -void atg_diagonal(tensor *, tensor self, int64_t offset, int64_t dim1, int64_t dim2); -void atg_diagonal_backward(tensor *, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t offset, int64_t dim1, int64_t dim2); -void atg_diagonal_backward_out(tensor *, tensor out, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t offset, int64_t dim1, int64_t dim2); -void atg_diagonal_copy(tensor *, tensor self, int64_t offset, int64_t dim1, int64_t dim2); -void atg_diagonal_copy_out(tensor *, tensor out, tensor self, int64_t offset, int64_t dim1, int64_t dim2); -void atg_diagonal_scatter(tensor *, tensor self, tensor src, int64_t offset, int64_t dim1, int64_t dim2); -void atg_diagonal_scatter_out(tensor *, tensor out, tensor self, tensor src, int64_t offset, int64_t dim1, int64_t dim2); -void atg_diff(tensor *, tensor self, int64_t n, int64_t dim, tensor prepend, tensor append); -void atg_diff_out(tensor *, tensor out, tensor self, int64_t n, int64_t dim, tensor prepend, tensor append); -void atg_digamma(tensor *, tensor self); -void atg_digamma_(tensor *, tensor self); -void atg_digamma_out(tensor *, tensor out, tensor self); -void atg_dist(tensor *, tensor self, tensor other); -void atg_dist_out(tensor *, tensor out, tensor self, tensor other); -void atg_div(tensor *, tensor self, tensor other); -void atg_div_(tensor *, tensor self, tensor other); -void atg_div_out(tensor *, tensor out, tensor self, tensor other); -void atg_div_out_mode(tensor *, tensor out, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len); -void atg_div_scalar(tensor *, tensor self, scalar other); -void atg_div_scalar_(tensor *, tensor self, scalar other); -void atg_div_scalar_mode(tensor *, tensor self, scalar other, char* rounding_mode_ptr, int rounding_mode_len); -void atg_div_scalar_mode_(tensor *, tensor self, scalar other, char* rounding_mode_ptr, int rounding_mode_len); -void atg_div_scalar_mode_out(tensor *, tensor out, tensor self, scalar other, char* rounding_mode_ptr, int rounding_mode_len); -void atg_div_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_div_tensor_mode(tensor *, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len); -void atg_div_tensor_mode_(tensor *, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len); -void atg_divide(tensor *, tensor self, tensor other); -void atg_divide_(tensor *, tensor self, tensor other); -void atg_divide_out(tensor *, tensor out, tensor self, tensor other); -void atg_divide_out_mode(tensor *, tensor out, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len); -void atg_divide_scalar(tensor *, tensor self, scalar other); -void atg_divide_scalar_(tensor *, tensor self, scalar other); -void atg_divide_scalar_mode(tensor *, tensor self, scalar other, char* rounding_mode_ptr, int rounding_mode_len); -void atg_divide_scalar_mode_(tensor *, tensor self, scalar other, char* rounding_mode_ptr, int rounding_mode_len); -void atg_divide_tensor_mode(tensor *, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len); -void atg_divide_tensor_mode_(tensor *, tensor self, tensor other, char* rounding_mode_ptr, int rounding_mode_len); -void atg_dot(tensor *, tensor self, tensor tensor); -void atg_dot_out(tensor *, tensor out, tensor self, tensor tensor); -void atg_dropout(tensor *, tensor input, double p, int train); -void atg_dropout_(tensor *, tensor self, double p, int train); -tensor *atg_dsplit(tensor self, int64_t sections); -tensor *atg_dsplit_array(tensor self, int64_t *indices_data, int indices_len); -void atg_dstack(tensor *, tensor *tensors_data, int tensors_len); -void atg_dstack_out(tensor *, tensor out, tensor *tensors_data, int tensors_len); -void atg_einsum(tensor *, char* equation_ptr, int equation_len, tensor *tensors_data, int tensors_len, int64_t *path_data, int path_len); -void atg_elu(tensor *, tensor self); -void atg_elu_(tensor *, tensor self); -void atg_elu_backward(tensor *, tensor grad_output, scalar alpha, scalar scale, scalar input_scale, int is_result, tensor self_or_result); -void atg_elu_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, scalar alpha, scalar scale, scalar input_scale, int is_result, tensor self_or_result); -void atg_elu_out(tensor *, tensor out, tensor self); -void atg_embedding(tensor *, tensor weight, tensor indices, int64_t padding_idx, int scale_grad_by_freq, int sparse); -void atg_embedding_backward(tensor *, tensor grad, tensor indices, int64_t num_weights, int64_t padding_idx, int scale_grad_by_freq, int sparse); -void atg_embedding_bag(tensor *, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset); -void atg_embedding_bag_padding_idx(tensor *, tensor weight, tensor indices, tensor offsets, int scale_grad_by_freq, int64_t mode, int sparse, tensor per_sample_weights, int include_last_offset, int64_t padding_idx_v, uint8_t padding_idx_null); -void atg_embedding_dense_backward(tensor *, tensor grad_output, tensor indices, int64_t num_weights, int64_t padding_idx, int scale_grad_by_freq); -void atg_embedding_dense_backward_out(tensor *, tensor out, tensor grad_output, tensor indices, int64_t num_weights, int64_t padding_idx, int scale_grad_by_freq); -void atg_embedding_out(tensor *, tensor out, tensor weight, tensor indices, int64_t padding_idx, int scale_grad_by_freq, int sparse); -void atg_embedding_renorm(tensor *, tensor self, tensor indices, double max_norm, double norm_type); -void atg_embedding_renorm_(tensor *, tensor self, tensor indices, double max_norm, double norm_type); -void atg_embedding_renorm_out(tensor *, tensor out, tensor self, tensor indices, double max_norm, double norm_type); -void atg_embedding_sparse_backward(tensor *, tensor grad, tensor indices, int64_t num_weights, int64_t padding_idx, int scale_grad_by_freq); -void atg_empty(tensor *, int64_t *size_data, int size_len, int options_kind, int options_device); -void atg_empty_like(tensor *, tensor self); -void atg_empty_like_out(tensor *, tensor out, tensor self); -void atg_empty_out(tensor *, tensor out, int64_t *size_data, int size_len); -void atg_empty_permuted(tensor *, int64_t *size_data, int size_len, int64_t *physical_layout_data, int physical_layout_len, int options_kind, int options_device); -void atg_empty_permuted_out(tensor *, tensor out, int64_t *size_data, int size_len, int64_t *physical_layout_data, int physical_layout_len); -void atg_empty_quantized(tensor *, int64_t *size_data, int size_len, tensor qtensor, int options_kind, int options_device); -void atg_empty_quantized_out(tensor *, tensor out, int64_t *size_data, int size_len, tensor qtensor); -void atg_empty_strided(tensor *, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int options_kind, int options_device); -void atg_empty_strided_out(tensor *, tensor out, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len); -void atg_eq(tensor *, tensor self, scalar other); -void atg_eq_(tensor *, tensor self, scalar other); -void atg_eq_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_eq_tensor(tensor *, tensor self, tensor other); -void atg_eq_tensor_(tensor *, tensor self, tensor other); -void atg_eq_tensor_out(tensor *, tensor out, tensor self, tensor other); -int atg_equal(tensor self, tensor other); -void atg_erf(tensor *, tensor self); -void atg_erf_(tensor *, tensor self); -void atg_erf_out(tensor *, tensor out, tensor self); -void atg_erfc(tensor *, tensor self); -void atg_erfc_(tensor *, tensor self); -void atg_erfc_out(tensor *, tensor out, tensor self); -void atg_erfinv(tensor *, tensor self); -void atg_erfinv_(tensor *, tensor self); -void atg_erfinv_out(tensor *, tensor out, tensor self); -void atg_exp(tensor *, tensor self); -void atg_exp2(tensor *, tensor self); -void atg_exp2_(tensor *, tensor self); -void atg_exp2_out(tensor *, tensor out, tensor self); -void atg_exp_(tensor *, tensor self); -void atg_exp_out(tensor *, tensor out, tensor self); -void atg_expand(tensor *, tensor self, int64_t *size_data, int size_len, int implicit); -void atg_expand_as(tensor *, tensor self, tensor other); -void atg_expand_copy(tensor *, tensor self, int64_t *size_data, int size_len, int implicit); -void atg_expand_copy_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len, int implicit); -void atg_expm1(tensor *, tensor self); -void atg_expm1_(tensor *, tensor self); -void atg_expm1_out(tensor *, tensor out, tensor self); -void atg_exponential(tensor *, tensor self, double lambd); -void atg_exponential_(tensor *, tensor self, double lambd); -void atg_exponential_out(tensor *, tensor out, tensor self, double lambd); -void atg_eye(tensor *, int64_t n, int options_kind, int options_device); -void atg_eye_m(tensor *, int64_t n, int64_t m, int options_kind, int options_device); -void atg_eye_m_out(tensor *, tensor out, int64_t n, int64_t m); -void atg_eye_out(tensor *, tensor out, int64_t n); -void atg_fake_quantize_per_channel_affine(tensor *, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max); -void atg_fake_quantize_per_channel_affine_cachemask(tensor *, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max); -void atg_fake_quantize_per_channel_affine_cachemask_backward(tensor *, tensor grad, tensor mask); -void atg_fake_quantize_per_channel_affine_cachemask_out(tensor *, tensor out0, tensor out1, tensor self, tensor scale, tensor zero_point, int64_t axis, int64_t quant_min, int64_t quant_max); -void atg_fake_quantize_per_tensor_affine(tensor *, tensor self, double scale, int64_t zero_point, int64_t quant_min, int64_t quant_max); -void atg_fake_quantize_per_tensor_affine_cachemask(tensor *, tensor self, double scale, int64_t zero_point, int64_t quant_min, int64_t quant_max); -void atg_fake_quantize_per_tensor_affine_cachemask_backward(tensor *, tensor grad, tensor mask); -void atg_fake_quantize_per_tensor_affine_cachemask_out(tensor *, tensor out0, tensor out1, tensor self, double scale, int64_t zero_point, int64_t quant_min, int64_t quant_max); -void atg_fake_quantize_per_tensor_affine_tensor_qparams(tensor *, tensor self, tensor scale, tensor zero_point, int64_t quant_min, int64_t quant_max); -void atg_feature_alpha_dropout(tensor *, tensor input, double p, int train); -void atg_feature_alpha_dropout_(tensor *, tensor self, double p, int train); -void atg_feature_dropout(tensor *, tensor input, double p, int train); -void atg_feature_dropout_(tensor *, tensor self, double p, int train); -void atg_fft_fft(tensor *, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); -void atg_fft_fft2(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_fft2_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_fft_out(tensor *, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); -void atg_fft_fftfreq(tensor *, int64_t n, double d, int options_kind, int options_device); -void atg_fft_fftfreq_out(tensor *, tensor out, int64_t n, double d); -void atg_fft_fftn(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_fftn_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_fftshift(tensor *, tensor self, int64_t *dim_data, int dim_len); -void atg_fft_hfft(tensor *, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); -void atg_fft_hfft2(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_hfft2_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_hfft_out(tensor *, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); -void atg_fft_hfftn(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_hfftn_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_ifft(tensor *, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); -void atg_fft_ifft2(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_ifft2_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_ifft_out(tensor *, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); -void atg_fft_ifftn(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_ifftn_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_ifftshift(tensor *, tensor self, int64_t *dim_data, int dim_len); -void atg_fft_ihfft(tensor *, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); -void atg_fft_ihfft2(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_ihfft2_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_ihfft_out(tensor *, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); -void atg_fft_ihfftn(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_ihfftn_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_irfft(tensor *, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); -void atg_fft_irfft2(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_irfft2_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_irfft_out(tensor *, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); -void atg_fft_irfftn(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_irfftn_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_rfft(tensor *, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); -void atg_fft_rfft2(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_rfft2_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_rfft_out(tensor *, tensor out, tensor self, int64_t n_v, uint8_t n_null, int64_t dim, char* norm_ptr, int norm_len); -void atg_fft_rfftfreq(tensor *, int64_t n, double d, int options_kind, int options_device); -void atg_fft_rfftfreq_out(tensor *, tensor out, int64_t n, double d); -void atg_fft_rfftn(tensor *, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fft_rfftn_out(tensor *, tensor out, tensor self, int64_t *s_data, int s_len, int64_t *dim_data, int dim_len, char* norm_ptr, int norm_len); -void atg_fill(tensor *, tensor self, scalar value); -void atg_fill_(tensor *, tensor self, scalar value); -void atg_fill_diagonal_(tensor *, tensor self, scalar fill_value, int wrap); -void atg_fill_scalar_out(tensor *, tensor out, tensor self, scalar value); -void atg_fill_tensor(tensor *, tensor self, tensor value); -void atg_fill_tensor_(tensor *, tensor self, tensor value); -void atg_fill_tensor_out(tensor *, tensor out, tensor self, tensor value); -void atg_fix(tensor *, tensor self); -void atg_fix_(tensor *, tensor self); -void atg_fix_out(tensor *, tensor out, tensor self); -void atg_flatten(tensor *, tensor self, int64_t start_dim, int64_t end_dim); -void atg_flatten_dense_tensors(tensor *, tensor *tensors_data, int tensors_len); -void atg_flip(tensor *, tensor self, int64_t *dims_data, int dims_len); -void atg_flip_out(tensor *, tensor out, tensor self, int64_t *dims_data, int dims_len); -void atg_fliplr(tensor *, tensor self); -void atg_flipud(tensor *, tensor self); -void atg_float_power(tensor *, tensor self, tensor exponent); -void atg_float_power_(tensor *, tensor self, scalar exponent); -void atg_float_power_scalar(tensor *, scalar self_scalar, tensor exponent); -void atg_float_power_scalar_out(tensor *, tensor out, scalar self_scalar, tensor exponent); -void atg_float_power_tensor_(tensor *, tensor self, tensor exponent); -void atg_float_power_tensor_scalar(tensor *, tensor self, scalar exponent); -void atg_float_power_tensor_scalar_out(tensor *, tensor out, tensor self, scalar exponent); -void atg_float_power_tensor_tensor_out(tensor *, tensor out, tensor self, tensor exponent); -void atg_floor(tensor *, tensor self); -void atg_floor_(tensor *, tensor self); -void atg_floor_divide(tensor *, tensor self, tensor other); -void atg_floor_divide_(tensor *, tensor self, tensor other); -void atg_floor_divide_out(tensor *, tensor out, tensor self, tensor other); -void atg_floor_divide_scalar(tensor *, tensor self, scalar other); -void atg_floor_divide_scalar_(tensor *, tensor self, scalar other); -void atg_floor_divide_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_floor_out(tensor *, tensor out, tensor self); -void atg_fmax(tensor *, tensor self, tensor other); -void atg_fmax_out(tensor *, tensor out, tensor self, tensor other); -void atg_fmin(tensor *, tensor self, tensor other); -void atg_fmin_out(tensor *, tensor out, tensor self, tensor other); -void atg_fmod(tensor *, tensor self, scalar other); -void atg_fmod_(tensor *, tensor self, scalar other); -void atg_fmod_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_fmod_tensor(tensor *, tensor self, tensor other); -void atg_fmod_tensor_(tensor *, tensor self, tensor other); -void atg_fmod_tensor_out(tensor *, tensor out, tensor self, tensor other); -void atg_frac(tensor *, tensor self); -void atg_frac_(tensor *, tensor self); -void atg_frac_out(tensor *, tensor out, tensor self); -void atg_fractional_max_pool2d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor random_samples); -void atg_fractional_max_pool2d_backward(tensor *, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor indices); -void atg_fractional_max_pool2d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor indices); -void atg_fractional_max_pool2d_output(tensor *, tensor output, tensor indices, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor random_samples); -void atg_fractional_max_pool3d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor random_samples); -void atg_fractional_max_pool3d_backward(tensor *, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor indices); -void atg_fractional_max_pool3d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor indices); -void atg_fractional_max_pool3d_output(tensor *, tensor output, tensor indices, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *output_size_data, int output_size_len, tensor random_samples); -void atg_frexp(tensor *, tensor self); -void atg_frexp_tensor_out(tensor *, tensor mantissa, tensor exponent, tensor self); -void atg_frobenius_norm(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim); -void atg_frobenius_norm_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim); -void atg_from_file(tensor *, char* filename_ptr, int filename_len, int shared, int64_t size_v, uint8_t size_null, int options_kind, int options_device); -void atg_from_file_out(tensor *, tensor out, char* filename_ptr, int filename_len, int shared, int64_t size_v, uint8_t size_null); -void atg_full(tensor *, int64_t *size_data, int size_len, scalar fill_value, int options_kind, int options_device); -void atg_full_like(tensor *, tensor self, scalar fill_value); -void atg_full_like_out(tensor *, tensor out, tensor self, scalar fill_value); -void atg_full_out(tensor *, tensor out, int64_t *size_data, int size_len, scalar fill_value); -void atg_fused_moving_avg_obs_fake_quant(tensor *, tensor self, tensor observer_on, tensor fake_quant_on, tensor running_min, tensor running_max, tensor scale, tensor zero_point, double averaging_const, int64_t quant_min, int64_t quant_max, int64_t ch_axis, int per_row_fake_quant, int symmetric_quant); -void atg_gather(tensor *, tensor self, int64_t dim, tensor index, int sparse_grad); -void atg_gather_backward(tensor *, tensor grad, tensor self, int64_t dim, tensor index, int sparse_grad); -void atg_gather_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, int sparse_grad); -void atg_gcd(tensor *, tensor self, tensor other); -void atg_gcd_(tensor *, tensor self, tensor other); -void atg_gcd_out(tensor *, tensor out, tensor self, tensor other); -void atg_ge(tensor *, tensor self, scalar other); -void atg_ge_(tensor *, tensor self, scalar other); -void atg_ge_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_ge_tensor(tensor *, tensor self, tensor other); -void atg_ge_tensor_(tensor *, tensor self, tensor other); -void atg_ge_tensor_out(tensor *, tensor out, tensor self, tensor other); -void atg_gelu(tensor *, tensor self, char* approximate_ptr, int approximate_len); -void atg_gelu_(tensor *, tensor self, char* approximate_ptr, int approximate_len); -void atg_gelu_backward(tensor *, tensor grad_output, tensor self, char* approximate_ptr, int approximate_len); -void atg_gelu_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, char* approximate_ptr, int approximate_len); -void atg_gelu_out(tensor *, tensor out, tensor self, char* approximate_ptr, int approximate_len); -void atg_geometric(tensor *, tensor self, double p); -void atg_geometric_(tensor *, tensor self, double p); -void atg_geometric_out(tensor *, tensor out, tensor self, double p); -void atg_geqrf(tensor *, tensor self); -void atg_geqrf_a(tensor *, tensor a, tensor tau, tensor self); -void atg_ger(tensor *, tensor self, tensor vec2); -void atg_ger_out(tensor *, tensor out, tensor self, tensor vec2); -void atg_glu(tensor *, tensor self, int64_t dim); -void atg_glu_backward(tensor *, tensor grad_output, tensor self, int64_t dim); -void atg_glu_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t dim); -void atg_glu_backward_jvp(tensor *, tensor grad_x, tensor grad_glu, tensor x, tensor dgrad_glu, tensor dx, int64_t dim); -void atg_glu_backward_jvp_out(tensor *, tensor out, tensor grad_x, tensor grad_glu, tensor x, tensor dgrad_glu, tensor dx, int64_t dim); -void atg_glu_jvp(tensor *, tensor glu, tensor x, tensor dx, int64_t dim); -void atg_glu_jvp_out(tensor *, tensor out, tensor glu, tensor x, tensor dx, int64_t dim); -void atg_glu_out(tensor *, tensor out, tensor self, int64_t dim); -void atg_grad(tensor *, tensor self); -void atg_greater(tensor *, tensor self, scalar other); -void atg_greater_(tensor *, tensor self, scalar other); -void atg_greater_equal(tensor *, tensor self, scalar other); -void atg_greater_equal_(tensor *, tensor self, scalar other); -void atg_greater_equal_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_greater_equal_tensor(tensor *, tensor self, tensor other); -void atg_greater_equal_tensor_(tensor *, tensor self, tensor other); -void atg_greater_equal_tensor_out(tensor *, tensor out, tensor self, tensor other); -void atg_greater_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_greater_tensor(tensor *, tensor self, tensor other); -void atg_greater_tensor_(tensor *, tensor self, tensor other); -void atg_greater_tensor_out(tensor *, tensor out, tensor self, tensor other); -void atg_grid_sampler(tensor *, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners); -void atg_grid_sampler_2d(tensor *, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners); -void atg_grid_sampler_2d_out(tensor *, tensor out, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners); -void atg_grid_sampler_3d(tensor *, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners); -void atg_grid_sampler_3d_out(tensor *, tensor out, tensor input, tensor grid, int64_t interpolation_mode, int64_t padding_mode, int align_corners); -void atg_group_norm(tensor *, tensor input, int64_t num_groups, tensor weight, tensor bias, double eps, int cudnn_enabled); -void atg_gru(tensor *, tensor input, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first); -void atg_gru_cell(tensor *, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh); -void atg_gru_data(tensor *, tensor data, tensor batch_sizes, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional); -void atg_gt(tensor *, tensor self, scalar other); -void atg_gt_(tensor *, tensor self, scalar other); -void atg_gt_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_gt_tensor(tensor *, tensor self, tensor other); -void atg_gt_tensor_(tensor *, tensor self, tensor other); -void atg_gt_tensor_out(tensor *, tensor out, tensor self, tensor other); -void atg_hamming_window(tensor *, int64_t window_length, int options_kind, int options_device); -void atg_hamming_window_out(tensor *, tensor out, int64_t window_length); -void atg_hamming_window_periodic(tensor *, int64_t window_length, int periodic, int options_kind, int options_device); -void atg_hamming_window_periodic_alpha(tensor *, int64_t window_length, int periodic, double alpha, int options_kind, int options_device); -void atg_hamming_window_periodic_alpha_beta(tensor *, int64_t window_length, int periodic, double alpha, double beta, int options_kind, int options_device); -void atg_hamming_window_periodic_alpha_beta_out(tensor *, tensor out, int64_t window_length, int periodic, double alpha, double beta); -void atg_hamming_window_periodic_alpha_out(tensor *, tensor out, int64_t window_length, int periodic, double alpha); -void atg_hamming_window_periodic_out(tensor *, tensor out, int64_t window_length, int periodic); -void atg_hann_window(tensor *, int64_t window_length, int options_kind, int options_device); -void atg_hann_window_out(tensor *, tensor out, int64_t window_length); -void atg_hann_window_periodic(tensor *, int64_t window_length, int periodic, int options_kind, int options_device); -void atg_hann_window_periodic_out(tensor *, tensor out, int64_t window_length, int periodic); -void atg_hardshrink(tensor *, tensor self); -void atg_hardshrink_backward(tensor *, tensor grad_out, tensor self, scalar lambd); -void atg_hardshrink_backward_grad_input(tensor *, tensor grad_input, tensor grad_out, tensor self, scalar lambd); -void atg_hardshrink_out(tensor *, tensor out, tensor self); -void atg_hardsigmoid(tensor *, tensor self); -void atg_hardsigmoid_(tensor *, tensor self); -void atg_hardsigmoid_backward(tensor *, tensor grad_output, tensor self); -void atg_hardsigmoid_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self); -void atg_hardsigmoid_out(tensor *, tensor out, tensor self); -void atg_hardswish(tensor *, tensor self); -void atg_hardswish_(tensor *, tensor self); -void atg_hardswish_backward(tensor *, tensor grad_output, tensor self); -void atg_hardswish_backward_out(tensor *, tensor out, tensor grad_output, tensor self); -void atg_hardswish_out(tensor *, tensor out, tensor self); -void atg_hardtanh(tensor *, tensor self); -void atg_hardtanh_(tensor *, tensor self); -void atg_hardtanh_backward(tensor *, tensor grad_output, tensor self, scalar min_val, scalar max_val); -void atg_hardtanh_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, scalar min_val, scalar max_val); -void atg_hardtanh_out(tensor *, tensor out, tensor self); -void atg_hash_tensor(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim, int64_t mode); -void atg_hash_tensor_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int64_t mode); -void atg_heaviside(tensor *, tensor self, tensor values); -void atg_heaviside_(tensor *, tensor self, tensor values); -void atg_heaviside_out(tensor *, tensor out, tensor self, tensor values); -void atg_hinge_embedding_loss(tensor *, tensor self, tensor target, double margin, int64_t reduction); -void atg_histc(tensor *, tensor self, int64_t bins); -void atg_histc_out(tensor *, tensor out, tensor self, int64_t bins); -void atg_histogram(tensor *, tensor self, tensor bins, tensor weight, int density); -void atg_histogram_bin_ct(tensor *, tensor self, int64_t bins, double *range_data, int range_len, tensor weight, int density); -void atg_histogram_bin_ct_out(tensor *, tensor hist, tensor bin_edges, tensor self, int64_t bins, double *range_data, int range_len, tensor weight, int density); -void atg_histogram_bins_tensor_out(tensor *, tensor hist, tensor bin_edges, tensor self, tensor bins, tensor weight, int density); -tensor *atg_hsplit(tensor self, int64_t sections); -tensor *atg_hsplit_array(tensor self, int64_t *indices_data, int indices_len); -void atg_hspmm(tensor *, tensor mat1, tensor mat2); -void atg_hspmm_out(tensor *, tensor out, tensor mat1, tensor mat2); -void atg_hstack(tensor *, tensor *tensors_data, int tensors_len); -void atg_hstack_out(tensor *, tensor out, tensor *tensors_data, int tensors_len); -void atg_huber_loss(tensor *, tensor self, tensor target, int64_t reduction, double delta); -void atg_huber_loss_backward(tensor *, tensor grad_output, tensor self, tensor target, int64_t reduction, double delta); -void atg_huber_loss_backward_out(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor target, int64_t reduction, double delta); -void atg_huber_loss_out(tensor *, tensor out, tensor self, tensor target, int64_t reduction, double delta); -void atg_hypot(tensor *, tensor self, tensor other); -void atg_hypot_(tensor *, tensor self, tensor other); -void atg_hypot_out(tensor *, tensor out, tensor self, tensor other); -void atg_i0(tensor *, tensor self); -void atg_i0_(tensor *, tensor self); -void atg_i0_out(tensor *, tensor out, tensor self); -void atg_igamma(tensor *, tensor self, tensor other); -void atg_igamma_(tensor *, tensor self, tensor other); -void atg_igamma_out(tensor *, tensor out, tensor self, tensor other); -void atg_igammac(tensor *, tensor self, tensor other); -void atg_igammac_(tensor *, tensor self, tensor other); -void atg_igammac_out(tensor *, tensor out, tensor self, tensor other); -void atg_im2col(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *dilation_data, int dilation_len, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len); -void atg_im2col_out(tensor *, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *dilation_data, int dilation_len, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len); -void atg_imag(tensor *, tensor self); -void atg_index(tensor *, tensor self, tensor *indices_data, int indices_len); -void atg_index_add(tensor *, tensor self, int64_t dim, tensor index, tensor source); -void atg_index_add_(tensor *, tensor self, int64_t dim, tensor index, tensor source); -void atg_index_add_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, tensor source); -void atg_index_copy(tensor *, tensor self, int64_t dim, tensor index, tensor source); -void atg_index_copy_(tensor *, tensor self, int64_t dim, tensor index, tensor source); -void atg_index_copy_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, tensor source); -void atg_index_fill(tensor *, tensor self, int64_t dim, tensor index, scalar value); -void atg_index_fill_(tensor *, tensor self, int64_t dim, tensor index, scalar value); -void atg_index_fill_int_scalar_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, scalar value); -void atg_index_fill_int_tensor(tensor *, tensor self, int64_t dim, tensor index, tensor value); -void atg_index_fill_int_tensor_(tensor *, tensor self, int64_t dim, tensor index, tensor value); -void atg_index_fill_int_tensor_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, tensor value); -void atg_index_put(tensor *, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate); -void atg_index_put_(tensor *, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate); -void atg_index_put_out(tensor *, tensor out, tensor self, tensor *indices_data, int indices_len, tensor values, int accumulate); -void atg_index_reduce(tensor *, tensor self, int64_t dim, tensor index, tensor source, char* reduce_ptr, int reduce_len, int include_self); -void atg_index_reduce_(tensor *, tensor self, int64_t dim, tensor index, tensor source, char* reduce_ptr, int reduce_len, int include_self); -void atg_index_reduce_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, tensor source, char* reduce_ptr, int reduce_len, int include_self); -void atg_index_select(tensor *, tensor self, int64_t dim, tensor index); -void atg_index_select_backward(tensor *, tensor grad, int64_t *self_sizes_data, int self_sizes_len, int64_t dim, tensor index); -void atg_index_select_out(tensor *, tensor out, tensor self, int64_t dim, tensor index); -void atg_index_tensor_out(tensor *, tensor out, tensor self, tensor *indices_data, int indices_len); -void atg_indices(tensor *, tensor self); -void atg_indices_copy(tensor *, tensor self); -void atg_indices_copy_out(tensor *, tensor out, tensor self); -void atg_infinitely_differentiable_gelu_backward(tensor *, tensor grad, tensor self); -void atg_inner(tensor *, tensor self, tensor other); -void atg_inner_out(tensor *, tensor out, tensor self, tensor other); -void atg_instance_norm(tensor *, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int use_input_stats, double momentum, double eps, int cudnn_enabled); -void atg_int_repr(tensor *, tensor self); -void atg_int_repr_out(tensor *, tensor out, tensor self); -void atg_inverse(tensor *, tensor self); -void atg_inverse_out(tensor *, tensor out, tensor self); -int atg_is_coalesced(tensor self); -int atg_is_complex(tensor self); -int atg_is_conj(tensor self); -int atg_is_distributed(tensor self); -int atg_is_floating_point(tensor self); -int atg_is_inference(tensor self); -int atg_is_leaf(tensor self); -int atg_is_neg(tensor self); -int atg_is_nonzero(tensor self); -int atg_is_pinned(tensor self, int device); -int atg_is_same_size(tensor self, tensor other); -int atg_is_set_to(tensor self, tensor tensor); -int atg_is_signed(tensor self); -int atg_is_vulkan_available(); -void atg_isclose(tensor *, tensor self, tensor other, double rtol, double atol, int equal_nan); -void atg_isfinite(tensor *, tensor self); -void atg_isin(tensor *, tensor elements, tensor test_elements, int assume_unique, int invert); -void atg_isin_scalar_tensor(tensor *, scalar element, tensor test_elements, int assume_unique, int invert); -void atg_isin_scalar_tensor_out(tensor *, tensor out, scalar element, tensor test_elements, int assume_unique, int invert); -void atg_isin_tensor_scalar(tensor *, tensor elements, scalar test_element, int assume_unique, int invert); -void atg_isin_tensor_scalar_out(tensor *, tensor out, tensor elements, scalar test_element, int assume_unique, int invert); -void atg_isin_tensor_tensor_out(tensor *, tensor out, tensor elements, tensor test_elements, int assume_unique, int invert); -void atg_isinf(tensor *, tensor self); -void atg_isinf_out(tensor *, tensor out, tensor self); -void atg_isnan(tensor *, tensor self); -void atg_isnan_out(tensor *, tensor out, tensor self); -void atg_isneginf(tensor *, tensor self); -void atg_isneginf_out(tensor *, tensor out, tensor self); -void atg_isposinf(tensor *, tensor self); -void atg_isposinf_out(tensor *, tensor out, tensor self); -void atg_isreal(tensor *, tensor self); -void atg_istft(tensor *, tensor self, int64_t n_fft, int64_t hop_length_v, uint8_t hop_length_null, int64_t win_length_v, uint8_t win_length_null, tensor window, int center, int normalized, int onesided, int64_t length_v, uint8_t length_null, int return_complex); -void atg_kaiser_window(tensor *, int64_t window_length, int options_kind, int options_device); -void atg_kaiser_window_beta(tensor *, int64_t window_length, int periodic, double beta, int options_kind, int options_device); -void atg_kaiser_window_beta_out(tensor *, tensor out, int64_t window_length, int periodic, double beta); -void atg_kaiser_window_out(tensor *, tensor out, int64_t window_length); -void atg_kaiser_window_periodic(tensor *, int64_t window_length, int periodic, int options_kind, int options_device); -void atg_kaiser_window_periodic_out(tensor *, tensor out, int64_t window_length, int periodic); -void atg_kl_div(tensor *, tensor self, tensor target, int64_t reduction, int log_target); -void atg_kron(tensor *, tensor self, tensor other); -void atg_kron_out(tensor *, tensor out, tensor self, tensor other); -void atg_kthvalue(tensor *, tensor self, int64_t k, int64_t dim, int keepdim); -void atg_kthvalue_values(tensor *, tensor values, tensor indices, tensor self, int64_t k, int64_t dim, int keepdim); -void atg_l1_loss(tensor *, tensor self, tensor target, int64_t reduction); -void atg_layer_norm(tensor *, tensor input, int64_t *normalized_shape_data, int normalized_shape_len, tensor weight, tensor bias, double eps, int cudnn_enable); -void atg_lcm(tensor *, tensor self, tensor other); -void atg_lcm_(tensor *, tensor self, tensor other); -void atg_lcm_out(tensor *, tensor out, tensor self, tensor other); -void atg_ldexp(tensor *, tensor self, tensor other); -void atg_ldexp_(tensor *, tensor self, tensor other); -void atg_ldexp_out(tensor *, tensor out, tensor self, tensor other); -void atg_le(tensor *, tensor self, scalar other); -void atg_le_(tensor *, tensor self, scalar other); -void atg_le_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_le_tensor(tensor *, tensor self, tensor other); -void atg_le_tensor_(tensor *, tensor self, tensor other); -void atg_le_tensor_out(tensor *, tensor out, tensor self, tensor other); -void atg_leaky_relu(tensor *, tensor self); -void atg_leaky_relu_(tensor *, tensor self); -void atg_leaky_relu_backward(tensor *, tensor grad_output, tensor self, scalar negative_slope, int self_is_result); -void atg_leaky_relu_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, scalar negative_slope, int self_is_result); -void atg_leaky_relu_out(tensor *, tensor out, tensor self); -void atg_lerp(tensor *, tensor self, tensor end, scalar weight); -void atg_lerp_(tensor *, tensor self, tensor end, scalar weight); -void atg_lerp_scalar_out(tensor *, tensor out, tensor self, tensor end, scalar weight); -void atg_lerp_tensor(tensor *, tensor self, tensor end, tensor weight); -void atg_lerp_tensor_(tensor *, tensor self, tensor end, tensor weight); -void atg_lerp_tensor_out(tensor *, tensor out, tensor self, tensor end, tensor weight); -void atg_less(tensor *, tensor self, scalar other); -void atg_less_(tensor *, tensor self, scalar other); -void atg_less_equal(tensor *, tensor self, scalar other); -void atg_less_equal_(tensor *, tensor self, scalar other); -void atg_less_equal_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_less_equal_tensor(tensor *, tensor self, tensor other); -void atg_less_equal_tensor_(tensor *, tensor self, tensor other); -void atg_less_equal_tensor_out(tensor *, tensor out, tensor self, tensor other); -void atg_less_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_less_tensor(tensor *, tensor self, tensor other); -void atg_less_tensor_(tensor *, tensor self, tensor other); -void atg_less_tensor_out(tensor *, tensor out, tensor self, tensor other); -void atg_lgamma(tensor *, tensor self); -void atg_lgamma_(tensor *, tensor self); -void atg_lgamma_out(tensor *, tensor out, tensor self); -void atg_lift(tensor *, tensor self); -void atg_lift_fresh(tensor *, tensor self); -void atg_lift_fresh_copy(tensor *, tensor self); -void atg_lift_fresh_copy_out(tensor *, tensor out, tensor self); -void atg_lift_out(tensor *, tensor out, tensor self); -void atg_linalg_cholesky(tensor *, tensor self, int upper); -void atg_linalg_cholesky_ex(tensor *, tensor self, int upper, int check_errors); -void atg_linalg_cholesky_ex_l(tensor *, tensor L, tensor info, tensor self, int upper, int check_errors); -void atg_linalg_cholesky_out(tensor *, tensor out, tensor self, int upper); -void atg_linalg_cond(tensor *, tensor self, scalar p); -void atg_linalg_cond_out(tensor *, tensor out, tensor self, scalar p); -void atg_linalg_cond_p_str(tensor *, tensor self, char* p_ptr, int p_len); -void atg_linalg_cond_p_str_out(tensor *, tensor out, tensor self, char* p_ptr, int p_len); -void atg_linalg_cross(tensor *, tensor self, tensor other, int64_t dim); -void atg_linalg_cross_out(tensor *, tensor out, tensor self, tensor other, int64_t dim); -void atg_linalg_det(tensor *, tensor A); -void atg_linalg_det_out(tensor *, tensor out, tensor A); -void atg_linalg_diagonal(tensor *, tensor A, int64_t offset, int64_t dim1, int64_t dim2); -void atg_linalg_eig(tensor *, tensor self); -void atg_linalg_eig_out(tensor *, tensor eigenvalues, tensor eigenvectors, tensor self); -void atg_linalg_eigh(tensor *, tensor self, char* UPLO_ptr, int UPLO_len); -void atg_linalg_eigh_eigvals(tensor *, tensor eigvals, tensor eigvecs, tensor self, char* UPLO_ptr, int UPLO_len); -void atg_linalg_eigvals(tensor *, tensor self); -void atg_linalg_eigvals_out(tensor *, tensor out, tensor self); -void atg_linalg_eigvalsh(tensor *, tensor self, char* UPLO_ptr, int UPLO_len); -void atg_linalg_eigvalsh_out(tensor *, tensor out, tensor self, char* UPLO_ptr, int UPLO_len); -void atg_linalg_householder_product(tensor *, tensor input, tensor tau); -void atg_linalg_householder_product_out(tensor *, tensor out, tensor input, tensor tau); -void atg_linalg_inv(tensor *, tensor A); -void atg_linalg_inv_ex(tensor *, tensor A, int check_errors); -void atg_linalg_inv_ex_inverse(tensor *, tensor inverse, tensor info, tensor A, int check_errors); -void atg_linalg_inv_out(tensor *, tensor out, tensor A); -void atg_linalg_ldl_factor(tensor *, tensor self, int hermitian); -void atg_linalg_ldl_factor_ex(tensor *, tensor self, int hermitian, int check_errors); -void atg_linalg_ldl_factor_ex_out(tensor *, tensor LD, tensor pivots, tensor info, tensor self, int hermitian, int check_errors); -void atg_linalg_ldl_factor_out(tensor *, tensor LD, tensor pivots, tensor self, int hermitian); -void atg_linalg_ldl_solve(tensor *, tensor LD, tensor pivots, tensor B, int hermitian); -void atg_linalg_ldl_solve_out(tensor *, tensor out, tensor LD, tensor pivots, tensor B, int hermitian); -void atg_linalg_lstsq(tensor *, tensor self, tensor b, double rcond_v, uint8_t rcond_null, char* driver_ptr, int driver_len); -void atg_linalg_lstsq_out(tensor *, tensor solution, tensor residuals, tensor rank, tensor singular_values, tensor self, tensor b, double rcond_v, uint8_t rcond_null, char* driver_ptr, int driver_len); -void atg_linalg_lu(tensor *, tensor A, int pivot); -void atg_linalg_lu_factor(tensor *, tensor A, int pivot); -void atg_linalg_lu_factor_ex(tensor *, tensor A, int pivot, int check_errors); -void atg_linalg_lu_factor_ex_out(tensor *, tensor LU, tensor pivots, tensor info, tensor A, int pivot, int check_errors); -void atg_linalg_lu_factor_out(tensor *, tensor LU, tensor pivots, tensor A, int pivot); -void atg_linalg_lu_out(tensor *, tensor P, tensor L, tensor U, tensor A, int pivot); -void atg_linalg_lu_solve(tensor *, tensor LU, tensor pivots, tensor B, int left, int adjoint); -void atg_linalg_lu_solve_out(tensor *, tensor out, tensor LU, tensor pivots, tensor B, int left, int adjoint); -void atg_linalg_matmul(tensor *, tensor self, tensor other); -void atg_linalg_matmul_out(tensor *, tensor out, tensor self, tensor other); -void atg_linalg_matrix_exp(tensor *, tensor self); -void atg_linalg_matrix_exp_out(tensor *, tensor out, tensor self); -void atg_linalg_matrix_power(tensor *, tensor self, int64_t n); -void atg_linalg_matrix_power_out(tensor *, tensor out, tensor self, int64_t n); -void atg_linalg_matrix_rank(tensor *, tensor self, double tol, int hermitian); -void atg_linalg_matrix_rank_atol_rtol_float(tensor *, tensor self, double atol_v, uint8_t atol_null, double rtol_v, uint8_t rtol_null, int hermitian); -void atg_linalg_matrix_rank_atol_rtol_float_out(tensor *, tensor out, tensor self, double atol_v, uint8_t atol_null, double rtol_v, uint8_t rtol_null, int hermitian); -void atg_linalg_matrix_rank_atol_rtol_tensor(tensor *, tensor input, tensor atol, tensor rtol, int hermitian); -void atg_linalg_matrix_rank_atol_rtol_tensor_out(tensor *, tensor out, tensor input, tensor atol, tensor rtol, int hermitian); -void atg_linalg_matrix_rank_out(tensor *, tensor out, tensor self, double tol, int hermitian); -void atg_linalg_matrix_rank_out_tol_tensor(tensor *, tensor out, tensor input, tensor tol, int hermitian); -void atg_linalg_matrix_rank_tol_tensor(tensor *, tensor input, tensor tol, int hermitian); -void atg_linalg_multi_dot(tensor *, tensor *tensors_data, int tensors_len); -void atg_linalg_multi_dot_out(tensor *, tensor out, tensor *tensors_data, int tensors_len); -void atg_linalg_norm(tensor *, tensor self, scalar ord, int64_t *dim_data, int dim_len, int keepdim, int dtype); -void atg_linalg_norm_ord_str(tensor *, tensor self, char* ord_ptr, int ord_len, int64_t *dim_data, int dim_len, int keepdim, int dtype); -void atg_linalg_norm_ord_str_out(tensor *, tensor out, tensor self, char* ord_ptr, int ord_len, int64_t *dim_data, int dim_len, int keepdim, int dtype); -void atg_linalg_norm_out(tensor *, tensor out, tensor self, scalar ord, int64_t *dim_data, int dim_len, int keepdim, int dtype); -void atg_linalg_pinv(tensor *, tensor self, double rcond, int hermitian); -void atg_linalg_pinv_atol_rtol_float(tensor *, tensor self, double atol_v, uint8_t atol_null, double rtol_v, uint8_t rtol_null, int hermitian); -void atg_linalg_pinv_atol_rtol_float_out(tensor *, tensor out, tensor self, double atol_v, uint8_t atol_null, double rtol_v, uint8_t rtol_null, int hermitian); -void atg_linalg_pinv_atol_rtol_tensor(tensor *, tensor self, tensor atol, tensor rtol, int hermitian); -void atg_linalg_pinv_atol_rtol_tensor_out(tensor *, tensor out, tensor self, tensor atol, tensor rtol, int hermitian); -void atg_linalg_pinv_out(tensor *, tensor out, tensor self, double rcond, int hermitian); -void atg_linalg_pinv_out_rcond_tensor(tensor *, tensor out, tensor self, tensor rcond, int hermitian); -void atg_linalg_pinv_rcond_tensor(tensor *, tensor self, tensor rcond, int hermitian); -void atg_linalg_qr(tensor *, tensor A, char* mode_ptr, int mode_len); -void atg_linalg_qr_out(tensor *, tensor Q, tensor R, tensor A, char* mode_ptr, int mode_len); -void atg_linalg_slogdet(tensor *, tensor A); -void atg_linalg_slogdet_out(tensor *, tensor sign, tensor logabsdet, tensor A); -void atg_linalg_solve(tensor *, tensor A, tensor B, int left); -void atg_linalg_solve_ex(tensor *, tensor A, tensor B, int left, int check_errors); -void atg_linalg_solve_ex_out(tensor *, tensor result, tensor info, tensor A, tensor B, int left, int check_errors); -void atg_linalg_solve_out(tensor *, tensor out, tensor A, tensor B, int left); -void atg_linalg_solve_triangular(tensor *, tensor self, tensor B, int upper, int left, int unitriangular); -void atg_linalg_solve_triangular_out(tensor *, tensor out, tensor self, tensor B, int upper, int left, int unitriangular); -void atg_linalg_svd(tensor *, tensor A, int full_matrices, char* driver_ptr, int driver_len); -void atg_linalg_svd_u(tensor *, tensor U, tensor S, tensor Vh, tensor A, int full_matrices, char* driver_ptr, int driver_len); -void atg_linalg_svdvals(tensor *, tensor A, char* driver_ptr, int driver_len); -void atg_linalg_svdvals_out(tensor *, tensor out, tensor A, char* driver_ptr, int driver_len); -void atg_linalg_tensorinv(tensor *, tensor self, int64_t ind); -void atg_linalg_tensorinv_out(tensor *, tensor out, tensor self, int64_t ind); -void atg_linalg_tensorsolve(tensor *, tensor self, tensor other, int64_t *dims_data, int dims_len); -void atg_linalg_tensorsolve_out(tensor *, tensor out, tensor self, tensor other, int64_t *dims_data, int dims_len); -void atg_linalg_vander(tensor *, tensor x, int64_t n_v, uint8_t n_null); -void atg_linalg_vecdot(tensor *, tensor x, tensor y, int64_t dim); -void atg_linalg_vecdot_out(tensor *, tensor out, tensor x, tensor y, int64_t dim); -void atg_linear(tensor *, tensor input, tensor weight, tensor bias); -void atg_linear_out(tensor *, tensor out, tensor input, tensor weight, tensor bias); -void atg_linspace(tensor *, scalar start, scalar end, int64_t steps, int options_kind, int options_device); -void atg_linspace_out(tensor *, tensor out, scalar start, scalar end, int64_t steps); -void atg_linspace_scalar_tensor(tensor *, scalar start, tensor end, int64_t steps, int options_kind, int options_device); -void atg_linspace_scalar_tensor_out(tensor *, tensor out, scalar start, tensor end, int64_t steps); -void atg_linspace_tensor_scalar(tensor *, tensor start, scalar end, int64_t steps, int options_kind, int options_device); -void atg_linspace_tensor_scalar_out(tensor *, tensor out, tensor start, scalar end, int64_t steps); -void atg_linspace_tensor_tensor(tensor *, tensor start, tensor end, int64_t steps, int options_kind, int options_device); -void atg_linspace_tensor_tensor_out(tensor *, tensor out, tensor start, tensor end, int64_t steps); -void atg_log(tensor *, tensor self); -void atg_log10(tensor *, tensor self); -void atg_log10_(tensor *, tensor self); -void atg_log10_out(tensor *, tensor out, tensor self); -void atg_log1p(tensor *, tensor self); -void atg_log1p_(tensor *, tensor self); -void atg_log1p_out(tensor *, tensor out, tensor self); -void atg_log2(tensor *, tensor self); -void atg_log2_(tensor *, tensor self); -void atg_log2_out(tensor *, tensor out, tensor self); -void atg_log_(tensor *, tensor self); -void atg_log_normal(tensor *, tensor self, double mean, double std); -void atg_log_normal_(tensor *, tensor self, double mean, double std); -void atg_log_normal_out(tensor *, tensor out, tensor self, double mean, double std); -void atg_log_out(tensor *, tensor out, tensor self); -void atg_log_sigmoid(tensor *, tensor self); -void atg_log_sigmoid_backward(tensor *, tensor grad_output, tensor self, tensor buffer); -void atg_log_sigmoid_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor buffer); -void atg_log_sigmoid_out(tensor *, tensor out, tensor self); -void atg_log_softmax(tensor *, tensor self, int64_t dim, int dtype); -void atg_log_softmax_int_out(tensor *, tensor out, tensor self, int64_t dim, int dtype); -void atg_logaddexp(tensor *, tensor self, tensor other); -void atg_logaddexp2(tensor *, tensor self, tensor other); -void atg_logaddexp2_out(tensor *, tensor out, tensor self, tensor other); -void atg_logaddexp_out(tensor *, tensor out, tensor self, tensor other); -void atg_logcumsumexp(tensor *, tensor self, int64_t dim); -void atg_logcumsumexp_out(tensor *, tensor out, tensor self, int64_t dim); -void atg_logdet(tensor *, tensor self); -void atg_logical_and(tensor *, tensor self, tensor other); -void atg_logical_and_(tensor *, tensor self, tensor other); -void atg_logical_and_out(tensor *, tensor out, tensor self, tensor other); -void atg_logical_not(tensor *, tensor self); -void atg_logical_not_(tensor *, tensor self); -void atg_logical_not_out(tensor *, tensor out, tensor self); -void atg_logical_or(tensor *, tensor self, tensor other); -void atg_logical_or_(tensor *, tensor self, tensor other); -void atg_logical_or_out(tensor *, tensor out, tensor self, tensor other); -void atg_logical_xor(tensor *, tensor self, tensor other); -void atg_logical_xor_(tensor *, tensor self, tensor other); -void atg_logical_xor_out(tensor *, tensor out, tensor self, tensor other); -void atg_logit(tensor *, tensor self, double eps_v, uint8_t eps_null); -void atg_logit_(tensor *, tensor self, double eps_v, uint8_t eps_null); -void atg_logit_backward(tensor *, tensor grad_output, tensor self, double eps_v, uint8_t eps_null); -void atg_logit_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, double eps_v, uint8_t eps_null); -void atg_logit_out(tensor *, tensor out, tensor self, double eps_v, uint8_t eps_null); -void atg_logspace(tensor *, scalar start, scalar end, int64_t steps, double base, int options_kind, int options_device); -void atg_logspace_out(tensor *, tensor out, scalar start, scalar end, int64_t steps, double base); -void atg_logspace_scalar_tensor(tensor *, scalar start, tensor end, int64_t steps, double base, int options_kind, int options_device); -void atg_logspace_scalar_tensor_out(tensor *, tensor out, scalar start, tensor end, int64_t steps, double base); -void atg_logspace_tensor_scalar(tensor *, tensor start, scalar end, int64_t steps, double base, int options_kind, int options_device); -void atg_logspace_tensor_scalar_out(tensor *, tensor out, tensor start, scalar end, int64_t steps, double base); -void atg_logspace_tensor_tensor(tensor *, tensor start, tensor end, int64_t steps, double base, int options_kind, int options_device); -void atg_logspace_tensor_tensor_out(tensor *, tensor out, tensor start, tensor end, int64_t steps, double base); -void atg_logsumexp(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim); -void atg_logsumexp_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim); -void atg_lstm(tensor *, tensor input, tensor *hx_data, int hx_len, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first); -void atg_lstm_cell(tensor *, tensor input, tensor *hx_data, int hx_len, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh); -void atg_lstm_data(tensor *, tensor data, tensor batch_sizes, tensor *hx_data, int hx_len, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional); -void atg_lstm_mps_backward(tensor out0, tensor *out1_data, int out1_len, tensor *out2_data, int out2_len, tensor grad_y, tensor grad_hy, tensor grad_cy, tensor z_state, tensor cell_state_fwd, tensor input, tensor layersOutputs, tensor *hx_data, int hx_len, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first); -void atg_lt(tensor *, tensor self, scalar other); -void atg_lt_(tensor *, tensor self, scalar other); -void atg_lt_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_lt_tensor(tensor *, tensor self, tensor other); -void atg_lt_tensor_(tensor *, tensor self, tensor other); -void atg_lt_tensor_out(tensor *, tensor out, tensor self, tensor other); -void atg_lu_solve(tensor *, tensor self, tensor LU_data, tensor LU_pivots); -void atg_lu_solve_out(tensor *, tensor out, tensor self, tensor LU_data, tensor LU_pivots); -void atg_lu_unpack(tensor *, tensor LU_data, tensor LU_pivots, int unpack_data, int unpack_pivots); -void atg_lu_unpack_out(tensor *, tensor P, tensor L, tensor U, tensor LU_data, tensor LU_pivots, int unpack_data, int unpack_pivots); -void atg_margin_ranking_loss(tensor *, tensor input1, tensor input2, tensor target, double margin, int64_t reduction); -void atg_masked_fill(tensor *, tensor self, tensor mask, scalar value); -void atg_masked_fill_(tensor *, tensor self, tensor mask, scalar value); -void atg_masked_fill_scalar_out(tensor *, tensor out, tensor self, tensor mask, scalar value); -void atg_masked_fill_tensor(tensor *, tensor self, tensor mask, tensor value); -void atg_masked_fill_tensor_(tensor *, tensor self, tensor mask, tensor value); -void atg_masked_fill_tensor_out(tensor *, tensor out, tensor self, tensor mask, tensor value); -void atg_masked_scatter(tensor *, tensor self, tensor mask, tensor source); -void atg_masked_scatter_(tensor *, tensor self, tensor mask, tensor source); -void atg_masked_scatter_backward(tensor *, tensor grad_output, tensor mask, int64_t *sizes_data, int sizes_len); -void atg_masked_scatter_out(tensor *, tensor out, tensor self, tensor mask, tensor source); -void atg_masked_select(tensor *, tensor self, tensor mask); -void atg_masked_select_backward(tensor *, tensor grad, tensor input, tensor mask); -void atg_masked_select_out(tensor *, tensor out, tensor self, tensor mask); -void atg_matmul(tensor *, tensor self, tensor other); -void atg_matmul_out(tensor *, tensor out, tensor self, tensor other); -void atg_matrix_exp(tensor *, tensor self); -void atg_matrix_exp_backward(tensor *, tensor self, tensor grad); -void atg_matrix_h(tensor *, tensor self); -void atg_matrix_power(tensor *, tensor self, int64_t n); -void atg_matrix_power_out(tensor *, tensor out, tensor self, int64_t n); -void atg_max(tensor *, tensor self); -void atg_max_dim(tensor *, tensor self, int64_t dim, int keepdim); -void atg_max_dim_max(tensor *, tensor max, tensor max_values, tensor self, int64_t dim, int keepdim); -void atg_max_other(tensor *, tensor self, tensor other); -void atg_max_out(tensor *, tensor out, tensor self, tensor other); -void atg_max_pool1d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_max_pool1d_with_indices(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_max_pool2d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_max_pool2d_backward(tensor *, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_max_pool2d_backward_out(tensor *, tensor out, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_max_pool2d_with_indices(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_max_pool2d_with_indices_backward(tensor *, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode, tensor indices); -void atg_max_pool2d_with_indices_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode, tensor indices); -void atg_max_pool2d_with_indices_out(tensor *, tensor out, tensor indices, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_max_pool3d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_max_pool3d_with_indices(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_max_pool3d_with_indices_backward(tensor *, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode, tensor indices); -void atg_max_pool3d_with_indices_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode, tensor indices); -void atg_max_pool3d_with_indices_out(tensor *, tensor out, tensor indices, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_max_unary_out(tensor *, tensor out, tensor self); -void atg_max_unpool2d(tensor *, tensor self, tensor indices, int64_t *output_size_data, int output_size_len); -void atg_max_unpool2d_out(tensor *, tensor out, tensor self, tensor indices, int64_t *output_size_data, int output_size_len); -void atg_max_unpool3d(tensor *, tensor self, tensor indices, int64_t *output_size_data, int output_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len); -void atg_max_unpool3d_out(tensor *, tensor out, tensor self, tensor indices, int64_t *output_size_data, int output_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len); -void atg_maximum(tensor *, tensor self, tensor other); -void atg_maximum_out(tensor *, tensor out, tensor self, tensor other); -void atg_mean(tensor *, tensor self, int dtype); -void atg_mean_dim(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); -void atg_mean_dtype_out(tensor *, tensor out, tensor self, int dtype); -void atg_mean_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); -void atg_median(tensor *, tensor self); -void atg_median_dim(tensor *, tensor self, int64_t dim, int keepdim); -void atg_median_dim_values(tensor *, tensor values, tensor indices, tensor self, int64_t dim, int keepdim); -void atg_median_out(tensor *, tensor out, tensor self); -tensor *atg_meshgrid(tensor *tensors_data, int tensors_len); -tensor *atg_meshgrid_indexing(tensor *tensors_data, int tensors_len, char* indexing_ptr, int indexing_len); -void atg_mh(tensor *, tensor self); -void atg_min(tensor *, tensor self); -void atg_min_dim(tensor *, tensor self, int64_t dim, int keepdim); -void atg_min_dim_min(tensor *, tensor min, tensor min_indices, tensor self, int64_t dim, int keepdim); -void atg_min_other(tensor *, tensor self, tensor other); -void atg_min_out(tensor *, tensor out, tensor self, tensor other); -void atg_min_unary_out(tensor *, tensor out, tensor self); -void atg_minimum(tensor *, tensor self, tensor other); -void atg_minimum_out(tensor *, tensor out, tensor self, tensor other); -void atg_miopen_batch_norm(tensor *, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double exponential_average_factor, double epsilon); -void atg_miopen_batch_norm_backward(tensor *, tensor input, tensor grad_output, tensor weight, tensor running_mean, tensor running_var, tensor save_mean, tensor save_var, double epsilon); -void atg_miopen_batch_norm_backward_out(tensor *, tensor out0, tensor out1, tensor out2, tensor input, tensor grad_output, tensor weight, tensor running_mean, tensor running_var, tensor save_mean, tensor save_var, double epsilon); -void atg_miopen_batch_norm_out(tensor *, tensor out0, tensor out1, tensor out2, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double exponential_average_factor, double epsilon); -void atg_miopen_convolution(tensor *, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic); -void atg_miopen_convolution_add_relu(tensor *, tensor self, tensor weight, tensor z, scalar alpha, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); -void atg_miopen_convolution_out(tensor *, tensor out, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic); -void atg_miopen_convolution_relu(tensor *, tensor self, tensor weight, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int64_t groups); -void atg_miopen_convolution_transpose(tensor *, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic); -void atg_miopen_convolution_transpose_out(tensor *, tensor out, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic); -void atg_miopen_ctc_loss(tensor *, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int deterministic, int zero_infinity); -void atg_miopen_ctc_loss_out(tensor *, tensor out0, tensor out1, tensor log_probs, tensor targets, int64_t *input_lengths_data, int input_lengths_len, int64_t *target_lengths_data, int target_lengths_len, int64_t blank, int deterministic, int zero_infinity); -void atg_miopen_ctc_loss_tensor(tensor *, tensor log_probs, tensor targets, tensor input_lengths, tensor target_lengths, int64_t blank, int deterministic, int zero_infinity); -void atg_miopen_depthwise_convolution(tensor *, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic); -void atg_miopen_depthwise_convolution_out(tensor *, tensor out, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int benchmark, int deterministic); -void atg_miopen_rnn(tensor *, tensor input, tensor *weight_data, int weight_len, int64_t weight_stride0, tensor hx, tensor cx, int64_t mode, int64_t hidden_size, int64_t num_layers, int batch_first, double dropout, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, tensor dropout_state); -void atg_miopen_rnn_out(tensor *, tensor out0, tensor out1, tensor out2, tensor out3, tensor out4, tensor input, tensor *weight_data, int weight_len, int64_t weight_stride0, tensor hx, tensor cx, int64_t mode, int64_t hidden_size, int64_t num_layers, int batch_first, double dropout, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, tensor dropout_state); -void atg_mish(tensor *, tensor self); -void atg_mish_(tensor *, tensor self); -void atg_mish_backward(tensor *, tensor grad_output, tensor self); -void atg_mish_out(tensor *, tensor out, tensor self); -void atg_mkldnn_adaptive_avg_pool2d(tensor *, tensor self, int64_t *output_size_data, int output_size_len); -void atg_mkldnn_adaptive_avg_pool2d_backward(tensor *, tensor grad_output, tensor self); -void atg_mkldnn_adaptive_avg_pool2d_backward_out(tensor *, tensor out, tensor grad_output, tensor self); -void atg_mkldnn_adaptive_avg_pool2d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len); -void atg_mkldnn_convolution(tensor *, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups); -void atg_mkldnn_convolution_out(tensor *, tensor out, tensor self, tensor weight, tensor bias, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups); -void atg_mkldnn_linear(tensor *, tensor self, tensor weight, tensor bias); -void atg_mkldnn_linear_backward_input(tensor *, int64_t *input_size_data, int input_size_len, tensor grad_output, tensor weight); -void atg_mkldnn_linear_backward_input_out(tensor *, tensor out, int64_t *input_size_data, int input_size_len, tensor grad_output, tensor weight); -void atg_mkldnn_linear_backward_weights(tensor *, tensor grad_output, tensor input, tensor weight, int bias_defined); -void atg_mkldnn_linear_backward_weights_out(tensor *, tensor out0, tensor out1, tensor grad_output, tensor input, tensor weight, int bias_defined); -void atg_mkldnn_linear_out(tensor *, tensor out, tensor self, tensor weight, tensor bias); -void atg_mkldnn_max_pool2d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_mkldnn_max_pool2d_backward(tensor *, tensor grad_output, tensor output, tensor input, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_mkldnn_max_pool2d_backward_out(tensor *, tensor out, tensor grad_output, tensor output, tensor input, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_mkldnn_max_pool2d_out(tensor *, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_mkldnn_max_pool3d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_mkldnn_max_pool3d_backward(tensor *, tensor grad_output, tensor output, tensor input, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_mkldnn_max_pool3d_backward_out(tensor *, tensor out, tensor grad_output, tensor output, tensor input, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_mkldnn_max_pool3d_out(tensor *, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_mkldnn_reorder_conv2d_weight(tensor *, tensor self, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int64_t *input_size_data, int input_size_len); -void atg_mkldnn_reorder_conv2d_weight_out(tensor *, tensor out, tensor self, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int64_t *input_size_data, int input_size_len); -void atg_mkldnn_reorder_conv3d_weight(tensor *, tensor self, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int64_t *input_size_data, int input_size_len); -void atg_mkldnn_reorder_conv3d_weight_out(tensor *, tensor out, tensor self, int64_t *padding_data, int padding_len, int64_t *stride_data, int stride_len, int64_t *dilation_data, int dilation_len, int64_t groups, int64_t *input_size_data, int input_size_len); -void atg_mkldnn_rnn_layer(tensor *, tensor input, tensor weight0, tensor weight1, tensor weight2, tensor weight3, tensor hx_, tensor cx_, int reverse, int64_t *batch_sizes_data, int batch_sizes_len, int64_t mode, int64_t hidden_size, int64_t num_layers, int has_biases, int bidirectional, int batch_first, int train); -void atg_mkldnn_rnn_layer_backward(tensor *, tensor input, tensor weight1, tensor weight2, tensor weight3, tensor weight4, tensor hx_, tensor cx_tmp, tensor output, tensor hy_, tensor cy_, tensor grad_output, tensor grad_hy, tensor grad_cy, int reverse, int64_t mode, int64_t hidden_size, int64_t num_layers, int has_biases, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, int batch_first, tensor workspace); -void atg_mkldnn_rnn_layer_backward_out(tensor *, tensor out0, tensor out1, tensor out2, tensor out3, tensor out4, tensor out5, tensor out6, tensor input, tensor weight1, tensor weight2, tensor weight3, tensor weight4, tensor hx_, tensor cx_tmp, tensor output, tensor hy_, tensor cy_, tensor grad_output, tensor grad_hy, tensor grad_cy, int reverse, int64_t mode, int64_t hidden_size, int64_t num_layers, int has_biases, int train, int bidirectional, int64_t *batch_sizes_data, int batch_sizes_len, int batch_first, tensor workspace); -void atg_mkldnn_rnn_layer_out(tensor *, tensor out0, tensor out1, tensor out2, tensor out3, tensor input, tensor weight0, tensor weight1, tensor weight2, tensor weight3, tensor hx_, tensor cx_, int reverse, int64_t *batch_sizes_data, int batch_sizes_len, int64_t mode, int64_t hidden_size, int64_t num_layers, int has_biases, int bidirectional, int batch_first, int train); -void atg_mm(tensor *, tensor self, tensor mat2); -void atg_mm_dtype(tensor *, tensor self, tensor mat2, int out_dtype); -void atg_mm_dtype_out(tensor *, tensor out, tensor self, tensor mat2, int out_dtype); -void atg_mm_out(tensor *, tensor out, tensor self, tensor mat2); -void atg_mode(tensor *, tensor self, int64_t dim, int keepdim); -void atg_mode_values(tensor *, tensor values, tensor indices, tensor self, int64_t dim, int keepdim); -void atg_moveaxis(tensor *, tensor self, int64_t *source_data, int source_len, int64_t *destination_data, int destination_len); -void atg_moveaxis_int(tensor *, tensor self, int64_t source, int64_t destination); -void atg_movedim(tensor *, tensor self, int64_t *source_data, int source_len, int64_t *destination_data, int destination_len); -void atg_movedim_int(tensor *, tensor self, int64_t source, int64_t destination); -void atg_mse_loss(tensor *, tensor self, tensor target, int64_t reduction); -void atg_mse_loss_backward(tensor *, tensor grad_output, tensor self, tensor target, int64_t reduction); -void atg_mse_loss_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor target, int64_t reduction); -void atg_mse_loss_out(tensor *, tensor out, tensor self, tensor target, int64_t reduction); -void atg_msort(tensor *, tensor self); -void atg_msort_out(tensor *, tensor out, tensor self); -void atg_mt(tensor *, tensor self); -void atg_mul(tensor *, tensor self, tensor other); -void atg_mul_(tensor *, tensor self, tensor other); -void atg_mul_out(tensor *, tensor out, tensor self, tensor other); -void atg_mul_scalar(tensor *, tensor self, scalar other); -void atg_mul_scalar_(tensor *, tensor self, scalar other); -void atg_mul_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_multi_margin_loss_backward(tensor *, tensor grad_output, tensor self, tensor target, scalar p, scalar margin, tensor weight, int64_t reduction); -void atg_multi_margin_loss_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor target, scalar p, scalar margin, tensor weight, int64_t reduction); -void atg_multilabel_margin_loss(tensor *, tensor self, tensor target, int64_t reduction); -void atg_multilabel_margin_loss_backward(tensor *, tensor grad_output, tensor self, tensor target, int64_t reduction, tensor is_target); -void atg_multilabel_margin_loss_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor target, int64_t reduction, tensor is_target); -void atg_multilabel_margin_loss_out(tensor *, tensor out, tensor self, tensor target, int64_t reduction); -void atg_multinomial(tensor *, tensor self, int64_t num_samples, int replacement); -void atg_multinomial_out(tensor *, tensor out, tensor self, int64_t num_samples, int replacement); -void atg_multiply(tensor *, tensor self, tensor other); -void atg_multiply_(tensor *, tensor self, tensor other); -void atg_multiply_out(tensor *, tensor out, tensor self, tensor other); -void atg_multiply_scalar(tensor *, tensor self, scalar other); -void atg_multiply_scalar_(tensor *, tensor self, scalar other); -void atg_mv(tensor *, tensor self, tensor vec); -void atg_mv_out(tensor *, tensor out, tensor self, tensor vec); -void atg_mvlgamma(tensor *, tensor self, int64_t p); -void atg_mvlgamma_(tensor *, tensor self, int64_t p); -void atg_mvlgamma_out(tensor *, tensor out, tensor self, int64_t p); -void atg_nan_to_num(tensor *, tensor self, double nan_v, uint8_t nan_null, double posinf_v, uint8_t posinf_null, double neginf_v, uint8_t neginf_null); -void atg_nan_to_num_(tensor *, tensor self, double nan_v, uint8_t nan_null, double posinf_v, uint8_t posinf_null, double neginf_v, uint8_t neginf_null); -void atg_nan_to_num_out(tensor *, tensor out, tensor self, double nan_v, uint8_t nan_null, double posinf_v, uint8_t posinf_null, double neginf_v, uint8_t neginf_null); -void atg_nanmean(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); -void atg_nanmean_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); -void atg_nanmedian(tensor *, tensor self); -void atg_nanmedian_dim(tensor *, tensor self, int64_t dim, int keepdim); -void atg_nanmedian_dim_values(tensor *, tensor values, tensor indices, tensor self, int64_t dim, int keepdim); -void atg_nanmedian_out(tensor *, tensor out, tensor self); -void atg_nanquantile(tensor *, tensor self, tensor q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len); -void atg_nanquantile_out(tensor *, tensor out, tensor self, tensor q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len); -void atg_nanquantile_scalar(tensor *, tensor self, double q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len); -void atg_nanquantile_scalar_out(tensor *, tensor out, tensor self, double q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len); -void atg_nansum(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); -void atg_nansum_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); -void atg_narrow(tensor *, tensor self, int64_t dim, int64_t start, int64_t length); -void atg_narrow_copy(tensor *, tensor self, int64_t dim, int64_t start, int64_t length); -void atg_narrow_copy_out(tensor *, tensor out, tensor self, int64_t dim, int64_t start, int64_t length); -void atg_narrow_tensor(tensor *, tensor self, int64_t dim, tensor start, int64_t length); -void atg_native_batch_norm(tensor *, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps); -void atg_native_batch_norm_out(tensor *, tensor out, tensor save_mean, tensor save_invstd, tensor input, tensor weight, tensor bias, tensor running_mean, tensor running_var, int training, double momentum, double eps); -void atg_native_channel_shuffle(tensor *, tensor self, int64_t groups); -void atg_native_dropout(tensor *, tensor input, double p, int train); -void atg_native_dropout_backward(tensor *, tensor grad_output, tensor mask, double scale); -void atg_native_dropout_backward_out(tensor *, tensor out, tensor grad_output, tensor mask, double scale); -void atg_native_dropout_out(tensor *, tensor out0, tensor out1, tensor input, double p, int train); -void atg_native_group_norm(tensor *, tensor input, tensor weight, tensor bias, int64_t n, int64_t C, int64_t HxW, int64_t group, double eps); -void atg_native_group_norm_out(tensor *, tensor out0, tensor out1, tensor out2, tensor input, tensor weight, tensor bias, int64_t n, int64_t C, int64_t HxW, int64_t group, double eps); -void atg_native_layer_norm(tensor *, tensor input, int64_t *normalized_shape_data, int normalized_shape_len, tensor weight, tensor bias, double eps); -void atg_native_layer_norm_out(tensor *, tensor out0, tensor out1, tensor out2, tensor input, int64_t *normalized_shape_data, int normalized_shape_len, tensor weight, tensor bias, double eps); -void atg_native_norm(tensor *, tensor self); -void atg_native_norm_out(tensor *, tensor out, tensor self); -void atg_native_norm_scalaropt_dim_dtype(tensor *, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim, int dtype); -void atg_native_norm_scalaropt_dim_dtype_out(tensor *, tensor out, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim, int dtype); -void atg_ne(tensor *, tensor self, scalar other); -void atg_ne_(tensor *, tensor self, scalar other); -void atg_ne_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_ne_tensor(tensor *, tensor self, tensor other); -void atg_ne_tensor_(tensor *, tensor self, tensor other); -void atg_ne_tensor_out(tensor *, tensor out, tensor self, tensor other); -void atg_neg(tensor *, tensor self); -void atg_neg_(tensor *, tensor self); -void atg_neg_out(tensor *, tensor out, tensor self); -void atg_negative(tensor *, tensor self); -void atg_negative_(tensor *, tensor self); -void atg_negative_out(tensor *, tensor out, tensor self); -void atg_nested_to_padded_tensor(tensor *, tensor self, double padding, int64_t *output_size_data, int output_size_len); -void atg_new_empty(tensor *, tensor self, int64_t *size_data, int size_len, int options_kind, int options_device); -void atg_new_empty_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len); -void atg_new_empty_strided(tensor *, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len, int options_kind, int options_device); -void atg_new_empty_strided_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len); -void atg_new_full(tensor *, tensor self, int64_t *size_data, int size_len, scalar fill_value, int options_kind, int options_device); -void atg_new_full_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len, scalar fill_value); -void atg_new_ones(tensor *, tensor self, int64_t *size_data, int size_len, int options_kind, int options_device); -void atg_new_ones_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len); -void atg_new_zeros(tensor *, tensor self, int64_t *size_data, int size_len, int options_kind, int options_device); -void atg_new_zeros_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len); -void atg_nextafter(tensor *, tensor self, tensor other); -void atg_nextafter_(tensor *, tensor self, tensor other); -void atg_nextafter_out(tensor *, tensor out, tensor self, tensor other); -void atg_nll_loss(tensor *, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index); -void atg_nll_loss2d(tensor *, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index); -void atg_nll_loss2d_backward(tensor *, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index, tensor total_weight); -void atg_nll_loss2d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index, tensor total_weight); -void atg_nll_loss2d_out(tensor *, tensor out, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index); -void atg_nll_loss_backward(tensor *, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index, tensor total_weight); -void atg_nll_loss_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index, tensor total_weight); -void atg_nll_loss_nd(tensor *, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index); -void atg_nll_loss_out(tensor *, tensor out, tensor self, tensor target, tensor weight, int64_t reduction, int64_t ignore_index); -void atg_nonzero(tensor *, tensor self); -tensor *atg_nonzero_numpy(tensor self); -void atg_nonzero_out(tensor *, tensor out, tensor self); -void atg_nonzero_static(tensor *, tensor self, int64_t size, int64_t fill_value); -void atg_nonzero_static_out(tensor *, tensor out, tensor self, int64_t size, int64_t fill_value); -void atg_norm(tensor *, tensor self); -void atg_norm_dtype_out(tensor *, tensor out, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim, int dtype); -void atg_norm_except_dim(tensor *, tensor v, int64_t pow, int64_t dim); -void atg_norm_out(tensor *, tensor out, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim); -void atg_norm_scalar_out(tensor *, tensor out, tensor self); -void atg_norm_scalaropt_dim(tensor *, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim); -void atg_norm_scalaropt_dim_dtype(tensor *, tensor self, scalar p, int64_t *dim_data, int dim_len, int keepdim, int dtype); -void atg_norm_scalaropt_dtype(tensor *, tensor self, scalar p, int dtype); -void atg_norm_scalaropt_dtype_out(tensor *, tensor out, tensor self, scalar p, int dtype); -void atg_normal_(tensor *, tensor self, double mean, double std); -void atg_normal_functional(tensor *, tensor self, double mean, double std); -void atg_not_equal(tensor *, tensor self, scalar other); -void atg_not_equal_(tensor *, tensor self, scalar other); -void atg_not_equal_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_not_equal_tensor(tensor *, tensor self, tensor other); -void atg_not_equal_tensor_(tensor *, tensor self, tensor other); -void atg_not_equal_tensor_out(tensor *, tensor out, tensor self, tensor other); -void atg_nuclear_norm(tensor *, tensor self, int keepdim); -void atg_nuclear_norm_dim(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim); -void atg_nuclear_norm_dim_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim); -void atg_nuclear_norm_out(tensor *, tensor out, tensor self, int keepdim); -void atg_numpy_t(tensor *, tensor self); -void atg_one_hot(tensor *, tensor self, int64_t num_classes); -void atg_ones(tensor *, int64_t *size_data, int size_len, int options_kind, int options_device); -void atg_ones_like(tensor *, tensor self); -void atg_ones_like_out(tensor *, tensor out, tensor self); -void atg_ones_out(tensor *, tensor out, int64_t *size_data, int size_len); -void atg_orgqr(tensor *, tensor self, tensor input2); -void atg_orgqr_out(tensor *, tensor out, tensor self, tensor input2); -void atg_ormqr(tensor *, tensor self, tensor input2, tensor input3, int left, int transpose); -void atg_ormqr_out(tensor *, tensor out, tensor self, tensor input2, tensor input3, int left, int transpose); -void atg_outer(tensor *, tensor self, tensor vec2); -void atg_outer_out(tensor *, tensor out, tensor self, tensor vec2); -int64_t atg_output_nr(tensor self); -void atg_pad(tensor *, tensor self, int64_t *pad_data, int pad_len, char* mode_ptr, int mode_len, double value_v, uint8_t value_null); -void atg_pad_sequence(tensor *, tensor *sequences_data, int sequences_len, int batch_first, double padding_value, char* padding_side_ptr, int padding_side_len); -void atg_pairwise_distance(tensor *, tensor x1, tensor x2, double p, double eps, int keepdim); -void atg_pdist(tensor *, tensor self, double p); -void atg_permute(tensor *, tensor self, int64_t *dims_data, int dims_len); -void atg_permute_copy(tensor *, tensor self, int64_t *dims_data, int dims_len); -void atg_permute_copy_out(tensor *, tensor out, tensor self, int64_t *dims_data, int dims_len); -void atg_pin_memory(tensor *, tensor self, int device); -void atg_pinverse(tensor *, tensor self, double rcond); -void atg_pixel_shuffle(tensor *, tensor self, int64_t upscale_factor); -void atg_pixel_shuffle_out(tensor *, tensor out, tensor self, int64_t upscale_factor); -void atg_pixel_unshuffle(tensor *, tensor self, int64_t downscale_factor); -void atg_pixel_unshuffle_out(tensor *, tensor out, tensor self, int64_t downscale_factor); -void atg_poisson(tensor *, tensor self); -void atg_poisson_nll_loss(tensor *, tensor input, tensor target, int log_input, int full, double eps, int64_t reduction); -void atg_poisson_out(tensor *, tensor out, tensor self); -void atg_polar(tensor *, tensor abs, tensor angle); -void atg_polar_out(tensor *, tensor out, tensor abs, tensor angle); -void atg_polygamma(tensor *, int64_t n, tensor self); -void atg_polygamma_(tensor *, tensor self, int64_t n); -void atg_polygamma_out(tensor *, tensor out, int64_t n, tensor self); -void atg_positive(tensor *, tensor self); -void atg_pow(tensor *, tensor self, tensor exponent); -void atg_pow_(tensor *, tensor self, scalar exponent); -void atg_pow_scalar(tensor *, scalar self_scalar, tensor exponent); -void atg_pow_scalar_out(tensor *, tensor out, scalar self_scalar, tensor exponent); -void atg_pow_tensor_(tensor *, tensor self, tensor exponent); -void atg_pow_tensor_scalar(tensor *, tensor self, scalar exponent); -void atg_pow_tensor_scalar_out(tensor *, tensor out, tensor self, scalar exponent); -void atg_pow_tensor_tensor_out(tensor *, tensor out, tensor self, tensor exponent); -void atg_prelu(tensor *, tensor self, tensor weight); -void atg_prod(tensor *, tensor self, int dtype); -void atg_prod_dim_int(tensor *, tensor self, int64_t dim, int keepdim, int dtype); -void atg_prod_int_out(tensor *, tensor out, tensor self, int64_t dim, int keepdim, int dtype); -void atg_prod_out(tensor *, tensor out, tensor self, int dtype); -void atg_put(tensor *, tensor self, tensor index, tensor source, int accumulate); -void atg_put_(tensor *, tensor self, tensor index, tensor source, int accumulate); -void atg_put_out(tensor *, tensor out, tensor self, tensor index, tensor source, int accumulate); -int64_t atg_q_per_channel_axis(tensor self); -void atg_q_per_channel_scales(tensor *, tensor self); -void atg_q_per_channel_scales_out(tensor *, tensor out, tensor self); -void atg_q_per_channel_zero_points(tensor *, tensor self); -void atg_q_per_channel_zero_points_out(tensor *, tensor out, tensor self); -double atg_q_scale(tensor self); -int64_t atg_q_zero_point(tensor self); -void atg_qr(tensor *, tensor self, int some); -void atg_qr_q(tensor *, tensor Q, tensor R, tensor self, int some); -void atg_quantile(tensor *, tensor self, tensor q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len); -void atg_quantile_out(tensor *, tensor out, tensor self, tensor q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len); -void atg_quantile_scalar(tensor *, tensor self, double q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len); -void atg_quantile_scalar_out(tensor *, tensor out, tensor self, double q, int64_t dim_v, uint8_t dim_null, int keepdim, char* interpolation_ptr, int interpolation_len); -void atg_quantize_per_channel(tensor *, tensor self, tensor scales, tensor zero_points, int64_t axis, int dtype); -void atg_quantize_per_channel_out(tensor *, tensor out, tensor self, tensor scales, tensor zero_points, int64_t axis, int dtype); -void atg_quantize_per_tensor(tensor *, tensor self, double scale, int64_t zero_point, int dtype); -void atg_quantize_per_tensor_dynamic(tensor *, tensor self, int dtype, int reduce_range); -void atg_quantize_per_tensor_dynamic_out(tensor *, tensor out, tensor self, int dtype, int reduce_range); -void atg_quantize_per_tensor_out(tensor *, tensor out, tensor self, double scale, int64_t zero_point, int dtype); -void atg_quantize_per_tensor_tensor_qparams(tensor *, tensor self, tensor scale, tensor zero_point, int dtype); -void atg_quantize_per_tensor_tensor_qparams_out(tensor *, tensor out, tensor self, tensor scale, tensor zero_point, int dtype); -tensor *atg_quantize_per_tensor_tensors(tensor *tensors_data, int tensors_len, tensor scales, tensor zero_points, int dtype); -void atg_quantize_per_tensor_tensors_out(tensor *out_data, int out_len, tensor *tensors_data, int tensors_len, tensor scales, tensor zero_points, int dtype); -void atg_quantized_batch_norm(tensor *, tensor input, tensor weight, tensor bias, tensor mean, tensor var, double eps, double output_scale, int64_t output_zero_point); -void atg_quantized_batch_norm_out(tensor *, tensor out, tensor input, tensor weight, tensor bias, tensor mean, tensor var, double eps, double output_scale, int64_t output_zero_point); -void atg_quantized_gru_cell(tensor *, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh, tensor packed_ih, tensor packed_hh, tensor col_offsets_ih, tensor col_offsets_hh, scalar scale_ih, scalar scale_hh, scalar zero_point_ih, scalar zero_point_hh); -void atg_quantized_lstm_cell(tensor *, tensor input, tensor *hx_data, int hx_len, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh, tensor packed_ih, tensor packed_hh, tensor col_offsets_ih, tensor col_offsets_hh, scalar scale_ih, scalar scale_hh, scalar zero_point_ih, scalar zero_point_hh); -void atg_quantized_max_pool1d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_quantized_max_pool1d_out(tensor *, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_quantized_max_pool2d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_quantized_max_pool2d_out(tensor *, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_quantized_max_pool3d(tensor *, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_quantized_max_pool3d_out(tensor *, tensor out, tensor self, int64_t *kernel_size_data, int kernel_size_len, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len, int ceil_mode); -void atg_quantized_rnn_relu_cell(tensor *, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh, tensor packed_ih, tensor packed_hh, tensor col_offsets_ih, tensor col_offsets_hh, scalar scale_ih, scalar scale_hh, scalar zero_point_ih, scalar zero_point_hh); -void atg_quantized_rnn_tanh_cell(tensor *, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh, tensor packed_ih, tensor packed_hh, tensor col_offsets_ih, tensor col_offsets_hh, scalar scale_ih, scalar scale_hh, scalar zero_point_ih, scalar zero_point_hh); -void atg_rad2deg(tensor *, tensor self); -void atg_rad2deg_(tensor *, tensor self); -void atg_rad2deg_out(tensor *, tensor out, tensor self); -void atg_rand(tensor *, int64_t *size_data, int size_len, int options_kind, int options_device); -void atg_rand_like(tensor *, tensor self); -void atg_rand_like_out(tensor *, tensor out, tensor self); -void atg_rand_out(tensor *, tensor out, int64_t *size_data, int size_len); -void atg_randint(tensor *, int64_t high, int64_t *size_data, int size_len, int options_kind, int options_device); -void atg_randint_like(tensor *, tensor self, int64_t high); -void atg_randint_like_low_dtype(tensor *, tensor self, int64_t low, int64_t high); -void atg_randint_like_low_dtype_out(tensor *, tensor out, tensor self, int64_t low, int64_t high); -void atg_randint_like_out(tensor *, tensor out, tensor self, int64_t high); -void atg_randint_like_tensor(tensor *, tensor self, tensor high); -void atg_randint_like_tensor_out(tensor *, tensor out, tensor self, tensor high); -void atg_randint_low(tensor *, int64_t low, int64_t high, int64_t *size_data, int size_len, int options_kind, int options_device); -void atg_randint_low_out(tensor *, tensor out, int64_t low, int64_t high, int64_t *size_data, int size_len); -void atg_randint_out(tensor *, tensor out, int64_t high, int64_t *size_data, int size_len); -void atg_randn(tensor *, int64_t *size_data, int size_len, int options_kind, int options_device); -void atg_randn_like(tensor *, tensor self); -void atg_randn_like_out(tensor *, tensor out, tensor self); -void atg_randn_out(tensor *, tensor out, int64_t *size_data, int size_len); -void atg_random(tensor *, tensor self); -void atg_random_(tensor *, tensor self); -void atg_random_from(tensor *, tensor self, int64_t from, int64_t to_v, uint8_t to_null); -void atg_random_from_(tensor *, tensor self, int64_t from, int64_t to_v, uint8_t to_null); -void atg_random_from_out(tensor *, tensor out, tensor self, int64_t from, int64_t to_v, uint8_t to_null); -void atg_random_out(tensor *, tensor out, tensor self); -void atg_random_to(tensor *, tensor self, int64_t to); -void atg_random_to_(tensor *, tensor self, int64_t to); -void atg_random_to_out(tensor *, tensor out, tensor self, int64_t to); -void atg_randperm(tensor *, int64_t n, int options_kind, int options_device); -void atg_randperm_out(tensor *, tensor out, int64_t n); -void atg_range(tensor *, scalar start, scalar end, int options_kind, int options_device); -void atg_range_out(tensor *, tensor out, scalar start, scalar end); -void atg_range_out_(tensor *, tensor out, scalar start, scalar end); -void atg_range_step(tensor *, scalar start, scalar end, int options_kind, int options_device); -void atg_ravel(tensor *, tensor self); -void atg_real(tensor *, tensor self); -void atg_reciprocal(tensor *, tensor self); -void atg_reciprocal_(tensor *, tensor self); -void atg_reciprocal_out(tensor *, tensor out, tensor self); -void atg_reflection_pad1d(tensor *, tensor self, int64_t *padding_data, int padding_len); -void atg_reflection_pad1d_backward(tensor *, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); -void atg_reflection_pad1d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); -void atg_reflection_pad1d_out(tensor *, tensor out, tensor self, int64_t *padding_data, int padding_len); -void atg_reflection_pad2d(tensor *, tensor self, int64_t *padding_data, int padding_len); -void atg_reflection_pad2d_backward(tensor *, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); -void atg_reflection_pad2d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); -void atg_reflection_pad2d_out(tensor *, tensor out, tensor self, int64_t *padding_data, int padding_len); -void atg_reflection_pad3d(tensor *, tensor self, int64_t *padding_data, int padding_len); -void atg_reflection_pad3d_backward(tensor *, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); -void atg_reflection_pad3d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); -void atg_reflection_pad3d_out(tensor *, tensor out, tensor self, int64_t *padding_data, int padding_len); -void atg_relu(tensor *, tensor self); -void atg_relu6(tensor *, tensor self); -void atg_relu6_(tensor *, tensor self); -void atg_relu_(tensor *, tensor self); -void atg_relu_out(tensor *, tensor out, tensor self); -void atg_remainder(tensor *, tensor self, scalar other); -void atg_remainder_(tensor *, tensor self, scalar other); -void atg_remainder_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_remainder_scalar_tensor(tensor *, scalar self_scalar, tensor other); -void atg_remainder_scalar_tensor_out(tensor *, tensor out, scalar self_scalar, tensor other); -void atg_remainder_tensor(tensor *, tensor self, tensor other); -void atg_remainder_tensor_(tensor *, tensor self, tensor other); -void atg_remainder_tensor_out(tensor *, tensor out, tensor self, tensor other); -void atg_renorm(tensor *, tensor self, scalar p, int64_t dim, scalar maxnorm); -void atg_renorm_(tensor *, tensor self, scalar p, int64_t dim, scalar maxnorm); -void atg_renorm_out(tensor *, tensor out, tensor self, scalar p, int64_t dim, scalar maxnorm); -void atg_repeat(tensor *, tensor self, int64_t *repeats_data, int repeats_len); -void atg_repeat_interleave(tensor *, tensor repeats, int64_t output_size_v, uint8_t output_size_null); -void atg_repeat_interleave_self_int(tensor *, tensor self, int64_t repeats, int64_t dim_v, uint8_t dim_null, int64_t output_size_v, uint8_t output_size_null); -void atg_repeat_interleave_self_tensor(tensor *, tensor self, tensor repeats, int64_t dim_v, uint8_t dim_null, int64_t output_size_v, uint8_t output_size_null); -void atg_repeat_interleave_tensor_out(tensor *, tensor out, tensor repeats, int64_t output_size_v, uint8_t output_size_null); -void atg_repeat_out(tensor *, tensor out, tensor self, int64_t *repeats_data, int repeats_len); -void atg_replication_pad1d(tensor *, tensor self, int64_t *padding_data, int padding_len); -void atg_replication_pad1d_backward(tensor *, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); -void atg_replication_pad1d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); -void atg_replication_pad1d_out(tensor *, tensor out, tensor self, int64_t *padding_data, int padding_len); -void atg_replication_pad2d(tensor *, tensor self, int64_t *padding_data, int padding_len); -void atg_replication_pad2d_backward(tensor *, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); -void atg_replication_pad2d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); -void atg_replication_pad2d_out(tensor *, tensor out, tensor self, int64_t *padding_data, int padding_len); -void atg_replication_pad3d(tensor *, tensor self, int64_t *padding_data, int padding_len); -void atg_replication_pad3d_backward(tensor *, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); -void atg_replication_pad3d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, int64_t *padding_data, int padding_len); -void atg_replication_pad3d_out(tensor *, tensor out, tensor self, int64_t *padding_data, int padding_len); -void atg_requires_grad_(tensor *, tensor self, int requires_grad); -void atg_reshape(tensor *, tensor self, int64_t *shape_data, int shape_len); -void atg_reshape_as(tensor *, tensor self, tensor other); -void atg_resize(tensor *, tensor self, int64_t *size_data, int size_len); -void atg_resize_(tensor *, tensor self, int64_t *size_data, int size_len); -void atg_resize_as(tensor *, tensor self, tensor the_template); -void atg_resize_as_(tensor *, tensor self, tensor the_template); -void atg_resize_as_out(tensor *, tensor out, tensor self, tensor the_template); -void atg_resize_as_sparse(tensor *, tensor self, tensor the_template); -void atg_resize_as_sparse_(tensor *, tensor self, tensor the_template); -void atg_resize_as_sparse_out(tensor *, tensor out, tensor self, tensor the_template); -void atg_resize_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len); -void atg_resolve_conj(tensor *, tensor self); -void atg_resolve_neg(tensor *, tensor self); -int atg_retains_grad(tensor self); -void atg_rms_norm(tensor *, tensor input, int64_t *normalized_shape_data, int normalized_shape_len, tensor weight, double eps_v, uint8_t eps_null); -void atg_rnn_relu(tensor *, tensor input, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first); -void atg_rnn_relu_cell(tensor *, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh); -void atg_rnn_relu_data(tensor *, tensor data, tensor batch_sizes, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional); -void atg_rnn_tanh(tensor *, tensor input, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional, int batch_first); -void atg_rnn_tanh_cell(tensor *, tensor input, tensor hx, tensor w_ih, tensor w_hh, tensor b_ih, tensor b_hh); -void atg_rnn_tanh_data(tensor *, tensor data, tensor batch_sizes, tensor hx, tensor *params_data, int params_len, int has_biases, int64_t num_layers, double dropout, int train, int bidirectional); -void atg_roll(tensor *, tensor self, int64_t *shifts_data, int shifts_len, int64_t *dims_data, int dims_len); -void atg_roll_out(tensor *, tensor out, tensor self, int64_t *shifts_data, int shifts_len, int64_t *dims_data, int dims_len); -void atg_rot90(tensor *, tensor self, int64_t k, int64_t *dims_data, int dims_len); -void atg_rot90_out(tensor *, tensor out, tensor self, int64_t k, int64_t *dims_data, int dims_len); -void atg_round(tensor *, tensor self); -void atg_round_(tensor *, tensor self); -void atg_round_decimals(tensor *, tensor self, int64_t decimals); -void atg_round_decimals_(tensor *, tensor self, int64_t decimals); -void atg_round_decimals_out(tensor *, tensor out, tensor self, int64_t decimals); -void atg_round_out(tensor *, tensor out, tensor self); -void atg_row_indices(tensor *, tensor self); -void atg_row_indices_copy(tensor *, tensor self); -void atg_row_indices_copy_out(tensor *, tensor out, tensor self); -void atg_row_stack(tensor *, tensor *tensors_data, int tensors_len); -void atg_row_stack_out(tensor *, tensor out, tensor *tensors_data, int tensors_len); -void atg_rrelu(tensor *, tensor self, int training); -void atg_rrelu_(tensor *, tensor self, int training); -void atg_rrelu_with_noise(tensor *, tensor self, tensor noise, int training); -void atg_rrelu_with_noise_(tensor *, tensor self, tensor noise, int training); -void atg_rrelu_with_noise_backward(tensor *, tensor grad_output, tensor self, tensor noise, scalar lower, scalar upper, int training, int self_is_result); -void atg_rrelu_with_noise_backward_out(tensor *, tensor out, tensor grad_output, tensor self, tensor noise, scalar lower, scalar upper, int training, int self_is_result); -void atg_rrelu_with_noise_functional(tensor *, tensor self, tensor noise, int training); -void atg_rrelu_with_noise_out(tensor *, tensor out, tensor self, tensor noise, int training); -void atg_rsqrt(tensor *, tensor self); -void atg_rsqrt_(tensor *, tensor self); -void atg_rsqrt_out(tensor *, tensor out, tensor self); -void atg_rsub(tensor *, tensor self, tensor other); -void atg_rsub_scalar(tensor *, tensor self, scalar other); -void atg_rsub_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_rsub_tensor_out(tensor *, tensor out, tensor self, tensor other); -void atg_scalar_tensor(tensor *, scalar s, int options_kind, int options_device); -void atg_scalar_tensor_out(tensor *, tensor out, scalar s); -void atg_scaled_dot_product_attention(tensor *, tensor query, tensor key, tensor value, tensor attn_mask, double dropout_p, int is_causal, double scale_v, uint8_t scale_null, int enable_gqa); -void atg_scatter(tensor *, tensor self, int64_t dim, tensor index, tensor src); -void atg_scatter_(tensor *, tensor self, int64_t dim, tensor index, tensor src); -void atg_scatter_add(tensor *, tensor self, int64_t dim, tensor index, tensor src); -void atg_scatter_add_(tensor *, tensor self, int64_t dim, tensor index, tensor src); -void atg_scatter_add_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, tensor src); -void atg_scatter_reduce(tensor *, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len); -void atg_scatter_reduce_(tensor *, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len); -void atg_scatter_reduce_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, tensor src, char* reduce_ptr, int reduce_len); -void atg_scatter_src_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, tensor src); -void atg_scatter_value(tensor *, tensor self, int64_t dim, tensor index, scalar value); -void atg_scatter_value_(tensor *, tensor self, int64_t dim, tensor index, scalar value); -void atg_scatter_value_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, scalar value); -void atg_scatter_value_reduce(tensor *, tensor self, int64_t dim, tensor index, scalar value, char* reduce_ptr, int reduce_len); -void atg_scatter_value_reduce_(tensor *, tensor self, int64_t dim, tensor index, scalar value, char* reduce_ptr, int reduce_len); -void atg_scatter_value_reduce_out(tensor *, tensor out, tensor self, int64_t dim, tensor index, scalar value, char* reduce_ptr, int reduce_len); -void atg_searchsorted(tensor *, tensor sorted_sequence, tensor self, int out_int32, int right, char* side_ptr, int side_len, tensor sorter); -void atg_searchsorted_scalar(tensor *, tensor sorted_sequence, scalar self_scalar, int out_int32, int right, char* side_ptr, int side_len, tensor sorter); -void atg_searchsorted_scalar_out(tensor *, tensor out, tensor sorted_sequence, scalar self_scalar, int out_int32, int right, char* side_ptr, int side_len, tensor sorter); -void atg_searchsorted_tensor_out(tensor *, tensor out, tensor sorted_sequence, tensor self, int out_int32, int right, char* side_ptr, int side_len, tensor sorter); -void atg_segment_reduce(tensor *, tensor data, char* reduce_ptr, int reduce_len, tensor lengths, tensor indices, tensor offsets, int64_t axis, int unsafe, scalar initial); -void atg_segment_reduce_out(tensor *, tensor out, tensor data, char* reduce_ptr, int reduce_len, tensor lengths, tensor indices, tensor offsets, int64_t axis, int unsafe, scalar initial); -void atg_select(tensor *, tensor self, int64_t dim, int64_t index); -void atg_select_backward(tensor *, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t index); -void atg_select_backward_out(tensor *, tensor out, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t index); -void atg_select_copy(tensor *, tensor self, int64_t dim, int64_t index); -void atg_select_copy_int_out(tensor *, tensor out, tensor self, int64_t dim, int64_t index); -void atg_select_scatter(tensor *, tensor self, tensor src, int64_t dim, int64_t index); -void atg_select_scatter_out(tensor *, tensor out, tensor self, tensor src, int64_t dim, int64_t index); -void atg_selu(tensor *, tensor self); -void atg_selu_(tensor *, tensor self); -void atg_set(tensor *, tensor self); -void atg_set_(tensor *, tensor self); -void atg_set_data(tensor self, tensor new_data); -void atg_set_out(tensor *, tensor out, tensor self); -void atg_set_requires_grad(tensor *, tensor self, int r); -void atg_set_source_tensor(tensor *, tensor self, tensor source); -void atg_set_source_tensor_(tensor *, tensor self, tensor source); -void atg_set_source_tensor_out(tensor *, tensor out, tensor self, tensor source); -void atg_set_source_tensor_storage_offset_(tensor *, tensor self, tensor source, int64_t storage_offset, int64_t *size_data, int size_len, int64_t *stride_data, int stride_len); -void atg_sgn(tensor *, tensor self); -void atg_sgn_(tensor *, tensor self); -void atg_sgn_out(tensor *, tensor out, tensor self); -void atg_sigmoid(tensor *, tensor self); -void atg_sigmoid_(tensor *, tensor self); -void atg_sigmoid_backward(tensor *, tensor grad_output, tensor output); -void atg_sigmoid_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor output); -void atg_sigmoid_out(tensor *, tensor out, tensor self); -void atg_sign(tensor *, tensor self); -void atg_sign_(tensor *, tensor self); -void atg_sign_out(tensor *, tensor out, tensor self); -void atg_signbit(tensor *, tensor self); -void atg_signbit_out(tensor *, tensor out, tensor self); -void atg_silu(tensor *, tensor self); -void atg_silu_(tensor *, tensor self); -void atg_silu_backward(tensor *, tensor grad_output, tensor self); -void atg_silu_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self); -void atg_silu_out(tensor *, tensor out, tensor self); -void atg_sin(tensor *, tensor self); -void atg_sin_(tensor *, tensor self); -void atg_sin_out(tensor *, tensor out, tensor self); -void atg_sinc(tensor *, tensor self); -void atg_sinc_(tensor *, tensor self); -void atg_sinc_out(tensor *, tensor out, tensor self); -void atg_sinh(tensor *, tensor self); -void atg_sinh_(tensor *, tensor self); -void atg_sinh_out(tensor *, tensor out, tensor self); -void atg_slice(tensor *, tensor self, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step); -void atg_slice_backward(tensor *, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t start, int64_t end, int64_t step); -void atg_slice_backward_out(tensor *, tensor out, tensor grad_output, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t start, int64_t end, int64_t step); -void atg_slice_copy(tensor *, tensor self, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step); -void atg_slice_copy_tensor_out(tensor *, tensor out, tensor self, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step); -void atg_slice_inverse(tensor *, tensor self, tensor src, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step); -void atg_slice_scatter(tensor *, tensor self, tensor src, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step); -void atg_slice_scatter_out(tensor *, tensor out, tensor self, tensor src, int64_t dim, int64_t start_v, uint8_t start_null, int64_t end_v, uint8_t end_null, int64_t step); -void atg_slogdet(tensor *, tensor self); -void atg_slogdet_out(tensor *, tensor sign, tensor logabsdet, tensor self); -void atg_slow_conv3d(tensor *, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len); -void atg_slow_conv3d_out(tensor *, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len); -void atg_slow_conv_dilated2d(tensor *, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len); -void atg_slow_conv_dilated2d_out(tensor *, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len); -void atg_slow_conv_dilated3d(tensor *, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len); -void atg_slow_conv_dilated3d_out(tensor *, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *dilation_data, int dilation_len); -void atg_slow_conv_transpose2d(tensor *, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *dilation_data, int dilation_len); -void atg_slow_conv_transpose2d_out(tensor *, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *dilation_data, int dilation_len); -void atg_slow_conv_transpose3d(tensor *, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *dilation_data, int dilation_len); -void atg_slow_conv_transpose3d_out(tensor *, tensor out, tensor self, tensor weight, int64_t *kernel_size_data, int kernel_size_len, tensor bias, int64_t *stride_data, int stride_len, int64_t *padding_data, int padding_len, int64_t *output_padding_data, int output_padding_len, int64_t *dilation_data, int dilation_len); -void atg_smm(tensor *, tensor self, tensor mat2); -void atg_smooth_l1_loss(tensor *, tensor self, tensor target, int64_t reduction, double beta); -void atg_smooth_l1_loss_backward(tensor *, tensor grad_output, tensor self, tensor target, int64_t reduction, double beta); -void atg_smooth_l1_loss_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor target, int64_t reduction, double beta); -void atg_smooth_l1_loss_out(tensor *, tensor out, tensor self, tensor target, int64_t reduction, double beta); -void atg_soft_margin_loss(tensor *, tensor self, tensor target, int64_t reduction); -void atg_soft_margin_loss_backward(tensor *, tensor grad_output, tensor self, tensor target, int64_t reduction); -void atg_soft_margin_loss_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, tensor target, int64_t reduction); -void atg_soft_margin_loss_out(tensor *, tensor out, tensor self, tensor target, int64_t reduction); -void atg_softmax(tensor *, tensor self, int64_t dim, int dtype); -void atg_softmax_int_out(tensor *, tensor out, tensor self, int64_t dim, int dtype); -void atg_softplus(tensor *, tensor self); -void atg_softplus_backward(tensor *, tensor grad_output, tensor self, scalar beta, scalar threshold); -void atg_softplus_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, scalar beta, scalar threshold); -void atg_softplus_out(tensor *, tensor out, tensor self); -void atg_softshrink(tensor *, tensor self); -void atg_softshrink_backward(tensor *, tensor grad_output, tensor self, scalar lambd); -void atg_softshrink_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, scalar lambd); -void atg_softshrink_out(tensor *, tensor out, tensor self); -void atg_sort(tensor *, tensor self, int64_t dim, int descending); -void atg_sort_stable(tensor *, tensor self, int stable, int64_t dim, int descending); -void atg_sort_values(tensor *, tensor values, tensor indices, tensor self, int64_t dim, int descending); -void atg_sort_values_stable(tensor *, tensor values, tensor indices, tensor self, int stable, int64_t dim, int descending); -void atg_sparse_bsc_tensor(tensor *, tensor ccol_indices, tensor row_indices, tensor values, int options_kind, int options_device); -void atg_sparse_bsc_tensor_ccol_row_value_size(tensor *, tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device); -void atg_sparse_bsr_tensor(tensor *, tensor crow_indices, tensor col_indices, tensor values, int options_kind, int options_device); -void atg_sparse_bsr_tensor_crow_col_value_size(tensor *, tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device); -void atg_sparse_compressed_tensor(tensor *, tensor compressed_indices, tensor plain_indices, tensor values, int options_kind, int options_device); -void atg_sparse_compressed_tensor_comp_plain_value_size(tensor *, tensor compressed_indices, tensor plain_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device); -void atg_sparse_coo_tensor(tensor *, int64_t *size_data, int size_len, int options_kind, int options_device); -void atg_sparse_coo_tensor_indices(tensor *, tensor indices, tensor values, int options_kind, int options_device, int is_coalesced); -void atg_sparse_coo_tensor_indices_size(tensor *, tensor indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device, int is_coalesced); -void atg_sparse_coo_tensor_size_out(tensor *, tensor out, int64_t *size_data, int size_len); -void atg_sparse_csc_tensor(tensor *, tensor ccol_indices, tensor row_indices, tensor values, int options_kind, int options_device); -void atg_sparse_csc_tensor_ccol_row_value_size(tensor *, tensor ccol_indices, tensor row_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device); -void atg_sparse_csr_tensor(tensor *, tensor crow_indices, tensor col_indices, tensor values, int options_kind, int options_device); -void atg_sparse_csr_tensor_crow_col_value_size(tensor *, tensor crow_indices, tensor col_indices, tensor values, int64_t *size_data, int size_len, int options_kind, int options_device); -int64_t atg_sparse_dim(tensor self); -void atg_sparse_mask(tensor *, tensor self, tensor mask); -void atg_sparse_mask_out(tensor *, tensor out, tensor self, tensor mask); -void atg_sparse_resize(tensor *, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim); -void atg_sparse_resize_(tensor *, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim); -void atg_sparse_resize_and_clear(tensor *, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim); -void atg_sparse_resize_and_clear_(tensor *, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim); -void atg_sparse_resize_and_clear_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim); -void atg_sparse_resize_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len, int64_t sparse_dim, int64_t dense_dim); -void atg_sparse_sampled_addmm(tensor *, tensor self, tensor mat1, tensor mat2); -void atg_sparse_sampled_addmm_out(tensor *, tensor out, tensor self, tensor mat1, tensor mat2); -void atg_special_airy_ai(tensor *, tensor x); -void atg_special_airy_ai_out(tensor *, tensor out, tensor x); -void atg_special_bessel_j0(tensor *, tensor self); -void atg_special_bessel_j0_out(tensor *, tensor out, tensor self); -void atg_special_bessel_j1(tensor *, tensor self); -void atg_special_bessel_j1_out(tensor *, tensor out, tensor self); -void atg_special_bessel_y0(tensor *, tensor self); -void atg_special_bessel_y0_out(tensor *, tensor out, tensor self); -void atg_special_bessel_y1(tensor *, tensor self); -void atg_special_bessel_y1_out(tensor *, tensor out, tensor self); -void atg_special_chebyshev_polynomial_t(tensor *, tensor x, tensor n); -void atg_special_chebyshev_polynomial_t_n_scalar(tensor *, tensor x, scalar n); -void atg_special_chebyshev_polynomial_t_n_scalar_out(tensor *, tensor out, tensor x, scalar n); -void atg_special_chebyshev_polynomial_t_out(tensor *, tensor out, tensor x, tensor n); -void atg_special_chebyshev_polynomial_t_x_scalar(tensor *, scalar x, tensor n); -void atg_special_chebyshev_polynomial_t_x_scalar_out(tensor *, tensor out, scalar x, tensor n); -void atg_special_chebyshev_polynomial_u(tensor *, tensor x, tensor n); -void atg_special_chebyshev_polynomial_u_n_scalar(tensor *, tensor x, scalar n); -void atg_special_chebyshev_polynomial_u_n_scalar_out(tensor *, tensor out, tensor x, scalar n); -void atg_special_chebyshev_polynomial_u_out(tensor *, tensor out, tensor x, tensor n); -void atg_special_chebyshev_polynomial_u_x_scalar(tensor *, scalar x, tensor n); -void atg_special_chebyshev_polynomial_u_x_scalar_out(tensor *, tensor out, scalar x, tensor n); -void atg_special_chebyshev_polynomial_v(tensor *, tensor x, tensor n); -void atg_special_chebyshev_polynomial_v_n_scalar(tensor *, tensor x, scalar n); -void atg_special_chebyshev_polynomial_v_n_scalar_out(tensor *, tensor out, tensor x, scalar n); -void atg_special_chebyshev_polynomial_v_out(tensor *, tensor out, tensor x, tensor n); -void atg_special_chebyshev_polynomial_v_x_scalar(tensor *, scalar x, tensor n); -void atg_special_chebyshev_polynomial_v_x_scalar_out(tensor *, tensor out, scalar x, tensor n); -void atg_special_chebyshev_polynomial_w(tensor *, tensor x, tensor n); -void atg_special_chebyshev_polynomial_w_n_scalar(tensor *, tensor x, scalar n); -void atg_special_chebyshev_polynomial_w_n_scalar_out(tensor *, tensor out, tensor x, scalar n); -void atg_special_chebyshev_polynomial_w_out(tensor *, tensor out, tensor x, tensor n); -void atg_special_chebyshev_polynomial_w_x_scalar(tensor *, scalar x, tensor n); -void atg_special_chebyshev_polynomial_w_x_scalar_out(tensor *, tensor out, scalar x, tensor n); -void atg_special_digamma(tensor *, tensor self); -void atg_special_digamma_out(tensor *, tensor out, tensor self); -void atg_special_entr(tensor *, tensor self); -void atg_special_entr_out(tensor *, tensor out, tensor self); -void atg_special_erf(tensor *, tensor self); -void atg_special_erf_out(tensor *, tensor out, tensor self); -void atg_special_erfc(tensor *, tensor self); -void atg_special_erfc_out(tensor *, tensor out, tensor self); -void atg_special_erfcx(tensor *, tensor self); -void atg_special_erfcx_out(tensor *, tensor out, tensor self); -void atg_special_erfinv(tensor *, tensor self); -void atg_special_erfinv_out(tensor *, tensor out, tensor self); -void atg_special_exp2(tensor *, tensor self); -void atg_special_exp2_out(tensor *, tensor out, tensor self); -void atg_special_expit(tensor *, tensor self); -void atg_special_expit_out(tensor *, tensor out, tensor self); -void atg_special_expm1(tensor *, tensor self); -void atg_special_expm1_out(tensor *, tensor out, tensor self); -void atg_special_gammainc(tensor *, tensor self, tensor other); -void atg_special_gammainc_out(tensor *, tensor out, tensor self, tensor other); -void atg_special_gammaincc(tensor *, tensor self, tensor other); -void atg_special_gammaincc_out(tensor *, tensor out, tensor self, tensor other); -void atg_special_gammaln(tensor *, tensor self); -void atg_special_gammaln_out(tensor *, tensor out, tensor self); -void atg_special_hermite_polynomial_h(tensor *, tensor x, tensor n); -void atg_special_hermite_polynomial_h_n_scalar(tensor *, tensor x, scalar n); -void atg_special_hermite_polynomial_h_n_scalar_out(tensor *, tensor out, tensor x, scalar n); -void atg_special_hermite_polynomial_h_out(tensor *, tensor out, tensor x, tensor n); -void atg_special_hermite_polynomial_h_x_scalar(tensor *, scalar x, tensor n); -void atg_special_hermite_polynomial_h_x_scalar_out(tensor *, tensor out, scalar x, tensor n); -void atg_special_hermite_polynomial_he(tensor *, tensor x, tensor n); -void atg_special_hermite_polynomial_he_n_scalar(tensor *, tensor x, scalar n); -void atg_special_hermite_polynomial_he_n_scalar_out(tensor *, tensor out, tensor x, scalar n); -void atg_special_hermite_polynomial_he_out(tensor *, tensor out, tensor x, tensor n); -void atg_special_hermite_polynomial_he_x_scalar(tensor *, scalar x, tensor n); -void atg_special_hermite_polynomial_he_x_scalar_out(tensor *, tensor out, scalar x, tensor n); -void atg_special_i0(tensor *, tensor self); -void atg_special_i0_out(tensor *, tensor out, tensor self); -void atg_special_i0e(tensor *, tensor self); -void atg_special_i0e_out(tensor *, tensor out, tensor self); -void atg_special_i1(tensor *, tensor self); -void atg_special_i1_out(tensor *, tensor out, tensor self); -void atg_special_i1e(tensor *, tensor self); -void atg_special_i1e_out(tensor *, tensor out, tensor self); -void atg_special_laguerre_polynomial_l(tensor *, tensor x, tensor n); -void atg_special_laguerre_polynomial_l_n_scalar(tensor *, tensor x, scalar n); -void atg_special_laguerre_polynomial_l_n_scalar_out(tensor *, tensor out, tensor x, scalar n); -void atg_special_laguerre_polynomial_l_out(tensor *, tensor out, tensor x, tensor n); -void atg_special_laguerre_polynomial_l_x_scalar(tensor *, scalar x, tensor n); -void atg_special_laguerre_polynomial_l_x_scalar_out(tensor *, tensor out, scalar x, tensor n); -void atg_special_legendre_polynomial_p(tensor *, tensor x, tensor n); -void atg_special_legendre_polynomial_p_n_scalar(tensor *, tensor x, scalar n); -void atg_special_legendre_polynomial_p_n_scalar_out(tensor *, tensor out, tensor x, scalar n); -void atg_special_legendre_polynomial_p_out(tensor *, tensor out, tensor x, tensor n); -void atg_special_legendre_polynomial_p_x_scalar(tensor *, scalar x, tensor n); -void atg_special_legendre_polynomial_p_x_scalar_out(tensor *, tensor out, scalar x, tensor n); -void atg_special_log1p(tensor *, tensor self); -void atg_special_log1p_out(tensor *, tensor out, tensor self); -void atg_special_log_ndtr(tensor *, tensor self); -void atg_special_log_ndtr_out(tensor *, tensor out, tensor self); -void atg_special_log_softmax(tensor *, tensor self, int64_t dim, int dtype); -void atg_special_logit(tensor *, tensor self, double eps_v, uint8_t eps_null); -void atg_special_logit_out(tensor *, tensor out, tensor self, double eps_v, uint8_t eps_null); -void atg_special_logsumexp(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim); -void atg_special_logsumexp_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim); -void atg_special_modified_bessel_i0(tensor *, tensor self); -void atg_special_modified_bessel_i0_out(tensor *, tensor out, tensor self); -void atg_special_modified_bessel_i1(tensor *, tensor self); -void atg_special_modified_bessel_i1_out(tensor *, tensor out, tensor self); -void atg_special_modified_bessel_k0(tensor *, tensor self); -void atg_special_modified_bessel_k0_out(tensor *, tensor out, tensor self); -void atg_special_modified_bessel_k1(tensor *, tensor self); -void atg_special_modified_bessel_k1_out(tensor *, tensor out, tensor self); -void atg_special_multigammaln(tensor *, tensor self, int64_t p); -void atg_special_multigammaln_out(tensor *, tensor out, tensor self, int64_t p); -void atg_special_ndtr(tensor *, tensor self); -void atg_special_ndtr_out(tensor *, tensor out, tensor self); -void atg_special_ndtri(tensor *, tensor self); -void atg_special_ndtri_out(tensor *, tensor out, tensor self); -void atg_special_polygamma(tensor *, int64_t n, tensor self); -void atg_special_polygamma_out(tensor *, tensor out, int64_t n, tensor self); -void atg_special_psi(tensor *, tensor self); -void atg_special_psi_out(tensor *, tensor out, tensor self); -void atg_special_round(tensor *, tensor self, int64_t decimals); -void atg_special_round_out(tensor *, tensor out, tensor self, int64_t decimals); -void atg_special_scaled_modified_bessel_k0(tensor *, tensor x); -void atg_special_scaled_modified_bessel_k0_out(tensor *, tensor out, tensor x); -void atg_special_scaled_modified_bessel_k1(tensor *, tensor x); -void atg_special_scaled_modified_bessel_k1_out(tensor *, tensor out, tensor x); -void atg_special_shifted_chebyshev_polynomial_t(tensor *, tensor x, tensor n); -void atg_special_shifted_chebyshev_polynomial_t_n_scalar(tensor *, tensor x, scalar n); -void atg_special_shifted_chebyshev_polynomial_t_n_scalar_out(tensor *, tensor out, tensor x, scalar n); -void atg_special_shifted_chebyshev_polynomial_t_out(tensor *, tensor out, tensor x, tensor n); -void atg_special_shifted_chebyshev_polynomial_t_x_scalar(tensor *, scalar x, tensor n); -void atg_special_shifted_chebyshev_polynomial_t_x_scalar_out(tensor *, tensor out, scalar x, tensor n); -void atg_special_shifted_chebyshev_polynomial_u(tensor *, tensor x, tensor n); -void atg_special_shifted_chebyshev_polynomial_u_n_scalar(tensor *, tensor x, scalar n); -void atg_special_shifted_chebyshev_polynomial_u_n_scalar_out(tensor *, tensor out, tensor x, scalar n); -void atg_special_shifted_chebyshev_polynomial_u_out(tensor *, tensor out, tensor x, tensor n); -void atg_special_shifted_chebyshev_polynomial_u_x_scalar(tensor *, scalar x, tensor n); -void atg_special_shifted_chebyshev_polynomial_u_x_scalar_out(tensor *, tensor out, scalar x, tensor n); -void atg_special_shifted_chebyshev_polynomial_v(tensor *, tensor x, tensor n); -void atg_special_shifted_chebyshev_polynomial_v_n_scalar(tensor *, tensor x, scalar n); -void atg_special_shifted_chebyshev_polynomial_v_n_scalar_out(tensor *, tensor out, tensor x, scalar n); -void atg_special_shifted_chebyshev_polynomial_v_out(tensor *, tensor out, tensor x, tensor n); -void atg_special_shifted_chebyshev_polynomial_v_x_scalar(tensor *, scalar x, tensor n); -void atg_special_shifted_chebyshev_polynomial_v_x_scalar_out(tensor *, tensor out, scalar x, tensor n); -void atg_special_shifted_chebyshev_polynomial_w(tensor *, tensor x, tensor n); -void atg_special_shifted_chebyshev_polynomial_w_n_scalar(tensor *, tensor x, scalar n); -void atg_special_shifted_chebyshev_polynomial_w_n_scalar_out(tensor *, tensor out, tensor x, scalar n); -void atg_special_shifted_chebyshev_polynomial_w_out(tensor *, tensor out, tensor x, tensor n); -void atg_special_shifted_chebyshev_polynomial_w_x_scalar(tensor *, scalar x, tensor n); -void atg_special_shifted_chebyshev_polynomial_w_x_scalar_out(tensor *, tensor out, scalar x, tensor n); -void atg_special_sinc(tensor *, tensor self); -void atg_special_sinc_out(tensor *, tensor out, tensor self); -void atg_special_softmax(tensor *, tensor self, int64_t dim, int dtype); -void atg_special_spherical_bessel_j0(tensor *, tensor x); -void atg_special_spherical_bessel_j0_out(tensor *, tensor out, tensor x); -void atg_special_xlog1py(tensor *, tensor self, tensor other); -void atg_special_xlog1py_other_scalar(tensor *, tensor self, scalar other); -void atg_special_xlog1py_other_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_special_xlog1py_out(tensor *, tensor out, tensor self, tensor other); -void atg_special_xlog1py_self_scalar(tensor *, scalar self_scalar, tensor other); -void atg_special_xlog1py_self_scalar_out(tensor *, tensor out, scalar self_scalar, tensor other); -void atg_special_xlogy(tensor *, tensor self, tensor other); -void atg_special_xlogy_other_scalar(tensor *, tensor self, scalar other); -void atg_special_xlogy_other_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_special_xlogy_out(tensor *, tensor out, tensor self, tensor other); -void atg_special_xlogy_self_scalar(tensor *, scalar self_scalar, tensor other); -void atg_special_xlogy_self_scalar_out(tensor *, tensor out, scalar self_scalar, tensor other); -void atg_special_zeta(tensor *, tensor self, tensor other); -void atg_special_zeta_other_scalar(tensor *, tensor self, scalar other); -void atg_special_zeta_other_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_special_zeta_out(tensor *, tensor out, tensor self, tensor other); -void atg_special_zeta_self_scalar(tensor *, scalar self_scalar, tensor other); -void atg_special_zeta_self_scalar_out(tensor *, tensor out, scalar self_scalar, tensor other); -tensor *atg_split(tensor self, int64_t split_size, int64_t dim); -tensor *atg_split_copy(tensor self, int64_t split_size, int64_t dim); -void atg_split_copy_tensor_out(tensor *out_data, int out_len, tensor self, int64_t split_size, int64_t dim); -tensor *atg_split_sizes(tensor self, int64_t *split_size_data, int split_size_len, int64_t dim); -tensor *atg_split_with_sizes(tensor self, int64_t *split_sizes_data, int split_sizes_len, int64_t dim); -tensor *atg_split_with_sizes_copy(tensor self, int64_t *split_sizes_data, int split_sizes_len, int64_t dim); -void atg_split_with_sizes_copy_out(tensor *out_data, int out_len, tensor self, int64_t *split_sizes_data, int split_sizes_len, int64_t dim); -void atg_sqrt(tensor *, tensor self); -void atg_sqrt_(tensor *, tensor self); -void atg_sqrt_out(tensor *, tensor out, tensor self); -void atg_square(tensor *, tensor self); -void atg_square_(tensor *, tensor self); -void atg_square_out(tensor *, tensor out, tensor self); -void atg_squeeze(tensor *, tensor self); -void atg_squeeze_(tensor *, tensor self); -void atg_squeeze_copy(tensor *, tensor self); -void atg_squeeze_copy_dim(tensor *, tensor self, int64_t dim); -void atg_squeeze_copy_dim_out(tensor *, tensor out, tensor self, int64_t dim); -void atg_squeeze_copy_dims(tensor *, tensor self, int64_t *dim_data, int dim_len); -void atg_squeeze_copy_dims_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len); -void atg_squeeze_copy_out(tensor *, tensor out, tensor self); -void atg_squeeze_dim(tensor *, tensor self, int64_t dim); -void atg_squeeze_dim_(tensor *, tensor self, int64_t dim); -void atg_squeeze_dims(tensor *, tensor self, int64_t *dim_data, int dim_len); -void atg_squeeze_dims_(tensor *, tensor self, int64_t *dim_data, int dim_len); -void atg_sspaddmm(tensor *, tensor self, tensor mat1, tensor mat2); -void atg_sspaddmm_out(tensor *, tensor out, tensor self, tensor mat1, tensor mat2); -void atg_stack(tensor *, tensor *tensors_data, int tensors_len, int64_t dim); -void atg_stack_out(tensor *, tensor out, tensor *tensors_data, int tensors_len, int64_t dim); -void atg_std(tensor *, tensor self, int unbiased); -void atg_std_correction(tensor *, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim); -void atg_std_correction_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim); -void atg_std_dim(tensor *, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim); -void atg_std_mean(tensor *, tensor self, int unbiased); -void atg_std_mean_correction(tensor *, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim); -void atg_std_mean_correction_out(tensor *, tensor out0, tensor out1, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim); -void atg_std_mean_dim(tensor *, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim); -void atg_std_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim); -void atg_stft(tensor *, tensor self, int64_t n_fft, int64_t hop_length_v, uint8_t hop_length_null, int64_t win_length_v, uint8_t win_length_null, tensor window, int normalized, int onesided, int return_complex, int align_to_window); -void atg_stft_center(tensor *, tensor self, int64_t n_fft, int64_t hop_length_v, uint8_t hop_length_null, int64_t win_length_v, uint8_t win_length_null, tensor window, int center, char* pad_mode_ptr, int pad_mode_len, int normalized, int onesided, int return_complex, int align_to_window); -void atg_sub(tensor *, tensor self, tensor other); -void atg_sub_(tensor *, tensor self, tensor other); -void atg_sub_out(tensor *, tensor out, tensor self, tensor other); -void atg_sub_scalar(tensor *, tensor self, scalar other); -void atg_sub_scalar_(tensor *, tensor self, scalar other); -void atg_sub_scalar_out(tensor *, tensor out, tensor self, scalar other); -void atg_subtract(tensor *, tensor self, tensor other); -void atg_subtract_(tensor *, tensor self, tensor other); -void atg_subtract_out(tensor *, tensor out, tensor self, tensor other); -void atg_subtract_scalar(tensor *, tensor self, scalar other); -void atg_subtract_scalar_(tensor *, tensor self, scalar other); -void atg_sum(tensor *, tensor self, int dtype); -void atg_sum_dim_intlist(tensor *, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); -void atg_sum_intlist_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int keepdim, int dtype); -void atg_sum_out(tensor *, tensor out, tensor self, int dtype); -void atg_sum_to_size(tensor *, tensor self, int64_t *size_data, int size_len); -void atg_svd(tensor *, tensor self, int some, int compute_uv); -void atg_svd_u(tensor *, tensor U, tensor S, tensor V, tensor self, int some, int compute_uv); -void atg_swapaxes(tensor *, tensor self, int64_t axis0, int64_t axis1); -void atg_swapaxes_(tensor *, tensor self, int64_t axis0, int64_t axis1); -void atg_swapdims(tensor *, tensor self, int64_t dim0, int64_t dim1); -void atg_swapdims_(tensor *, tensor self, int64_t dim0, int64_t dim1); -void atg_t(tensor *, tensor self); -void atg_t_(tensor *, tensor self); -void atg_t_copy(tensor *, tensor self); -void atg_t_copy_out(tensor *, tensor out, tensor self); -void atg_take(tensor *, tensor self, tensor index); -void atg_take_along_dim(tensor *, tensor self, tensor indices, int64_t dim_v, uint8_t dim_null); -void atg_take_along_dim_out(tensor *, tensor out, tensor self, tensor indices, int64_t dim_v, uint8_t dim_null); -void atg_take_out(tensor *, tensor out, tensor self, tensor index); -void atg_tan(tensor *, tensor self); -void atg_tan_(tensor *, tensor self); -void atg_tan_out(tensor *, tensor out, tensor self); -void atg_tanh(tensor *, tensor self); -void atg_tanh_(tensor *, tensor self); -void atg_tanh_backward(tensor *, tensor grad_output, tensor output); -void atg_tanh_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor output); -void atg_tanh_out(tensor *, tensor out, tensor self); -tensor *atg_tensor_split(tensor self, int64_t sections, int64_t dim); -tensor *atg_tensor_split_indices(tensor self, int64_t *indices_data, int indices_len, int64_t dim); -tensor *atg_tensor_split_tensor_indices_or_sections(tensor self, tensor tensor_indices_or_sections, int64_t dim); -void atg_tensordot(tensor *, tensor self, tensor other, int64_t *dims_self_data, int dims_self_len, int64_t *dims_other_data, int dims_other_len); -void atg_tensordot_out(tensor *, tensor out, tensor self, tensor other, int64_t *dims_self_data, int dims_self_len, int64_t *dims_other_data, int dims_other_len); -void atg_threshold(tensor *, tensor self, scalar threshold, scalar value); -void atg_threshold_(tensor *, tensor self, scalar threshold, scalar value); -void atg_threshold_backward(tensor *, tensor grad_output, tensor self, scalar threshold); -void atg_threshold_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, tensor self, scalar threshold); -void atg_threshold_out(tensor *, tensor out, tensor self, scalar threshold, scalar value); -void atg_tile(tensor *, tensor self, int64_t *dims_data, int dims_len); -void atg_to(tensor *, tensor self, int device); -void atg_to_dense(tensor *, tensor self, int dtype, int masked_grad); -void atg_to_dense_backward(tensor *, tensor grad, tensor input, int masked_grad); -void atg_to_device(tensor *, tensor self, int device, int dtype, int non_blocking, int copy); -void atg_to_dtype(tensor *, tensor self, int dtype, int non_blocking, int copy); -void atg_to_dtype_layout(tensor *, tensor self, int options_kind, int options_device, int non_blocking, int copy); -void atg_to_mkldnn(tensor *, tensor self, int dtype); -void atg_to_mkldnn_backward(tensor *, tensor grad, tensor input); -void atg_to_mkldnn_out(tensor *, tensor out, tensor self, int dtype); -void atg_to_other(tensor *, tensor self, tensor other, int non_blocking, int copy); -void atg_to_padded_tensor(tensor *, tensor self, double padding, int64_t *output_size_data, int output_size_len); -void atg_to_padded_tensor_out(tensor *, tensor out, tensor self, double padding, int64_t *output_size_data, int output_size_len); -void atg_to_sparse(tensor *, tensor self, int8_t layout, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null); -void atg_to_sparse_bsc(tensor *, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null); -void atg_to_sparse_bsr(tensor *, tensor self, int64_t *blocksize_data, int blocksize_len, int64_t dense_dim_v, uint8_t dense_dim_null); -void atg_to_sparse_csc(tensor *, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null); -void atg_to_sparse_csr(tensor *, tensor self, int64_t dense_dim_v, uint8_t dense_dim_null); -void atg_to_sparse_sparse_dim(tensor *, tensor self, int64_t sparse_dim); -void atg_topk(tensor *, tensor self, int64_t k, int64_t dim, int largest, int sorted); -void atg_topk_values(tensor *, tensor values, tensor indices, tensor self, int64_t k, int64_t dim, int largest, int sorted); -void atg_totype(tensor *, tensor self, int scalar_type); -void atg_trace(tensor *, tensor self); -void atg_trace_backward(tensor *, tensor grad, int64_t *sizes_data, int sizes_len); -void atg_trace_out(tensor *, tensor out, tensor self); -void atg_transpose(tensor *, tensor self, int64_t dim0, int64_t dim1); -void atg_transpose_(tensor *, tensor self, int64_t dim0, int64_t dim1); -void atg_transpose_copy(tensor *, tensor self, int64_t dim0, int64_t dim1); -void atg_transpose_copy_int_out(tensor *, tensor out, tensor self, int64_t dim0, int64_t dim1); -void atg_trapezoid(tensor *, tensor y, int64_t dim); -void atg_trapezoid_x(tensor *, tensor y, tensor x, int64_t dim); -void atg_trapz(tensor *, tensor y, tensor x, int64_t dim); -void atg_trapz_dx(tensor *, tensor y, double dx, int64_t dim); -void atg_triangular_solve(tensor *, tensor self, tensor A, int upper, int transpose, int unitriangular); -void atg_triangular_solve_x(tensor *, tensor X, tensor M, tensor self, tensor A, int upper, int transpose, int unitriangular); -void atg_tril(tensor *, tensor self, int64_t diagonal); -void atg_tril_(tensor *, tensor self, int64_t diagonal); -void atg_tril_indices(tensor *, int64_t row, int64_t col, int64_t offset, int options_kind, int options_device); -void atg_tril_indices_out(tensor *, tensor out, int64_t row, int64_t col, int64_t offset); -void atg_tril_out(tensor *, tensor out, tensor self, int64_t diagonal); -void atg_triplet_margin_loss(tensor *, tensor anchor, tensor positive, tensor negative, double margin, double p, double eps, int swap, int64_t reduction); -void atg_triu(tensor *, tensor self, int64_t diagonal); -void atg_triu_(tensor *, tensor self, int64_t diagonal); -void atg_triu_indices(tensor *, int64_t row, int64_t col, int64_t offset, int options_kind, int options_device); -void atg_triu_indices_out(tensor *, tensor out, int64_t row, int64_t col, int64_t offset); -void atg_triu_out(tensor *, tensor out, tensor self, int64_t diagonal); -void atg_true_divide(tensor *, tensor self, tensor other); -void atg_true_divide_(tensor *, tensor self, tensor other); -void atg_true_divide_out(tensor *, tensor out, tensor self, tensor other); -void atg_true_divide_scalar(tensor *, tensor self, scalar other); -void atg_true_divide_scalar_(tensor *, tensor self, scalar other); -void atg_trunc(tensor *, tensor self); -void atg_trunc_(tensor *, tensor self); -void atg_trunc_out(tensor *, tensor out, tensor self); -void atg_type_as(tensor *, tensor self, tensor other); -tensor *atg_unbind(tensor self, int64_t dim); -tensor *atg_unbind_copy(tensor self, int64_t dim); -void atg_unbind_copy_int_out(tensor *out_data, int out_len, tensor self, int64_t dim); -void atg_unflatten(tensor *, tensor self, int64_t dim, int64_t *sizes_data, int sizes_len); -tensor *atg_unflatten_dense_tensors(tensor flat, tensor *tensors_data, int tensors_len); -void atg_unfold(tensor *, tensor self, int64_t dimension, int64_t size, int64_t step); -void atg_unfold_backward(tensor *, tensor grad_in, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t size, int64_t step); -void atg_unfold_backward_out(tensor *, tensor out, tensor grad_in, int64_t *input_sizes_data, int input_sizes_len, int64_t dim, int64_t size, int64_t step); -void atg_unfold_copy(tensor *, tensor self, int64_t dimension, int64_t size, int64_t step); -void atg_unfold_copy_out(tensor *, tensor out, tensor self, int64_t dimension, int64_t size, int64_t step); -void atg_uniform(tensor *, tensor self, double from, double to); -void atg_uniform_(tensor *, tensor self, double from, double to); -void atg_uniform_out(tensor *, tensor out, tensor self, double from, double to); -void atg_unique_consecutive(tensor *, tensor self, int return_inverse, int return_counts, int64_t dim_v, uint8_t dim_null); -void atg_unique_consecutive_out(tensor *, tensor out0, tensor out1, tensor out2, tensor self, int return_inverse, int return_counts, int64_t dim_v, uint8_t dim_null); -void atg_unique_dim(tensor *, tensor self, int64_t dim, int sorted, int return_inverse, int return_counts); -void atg_unique_dim_consecutive(tensor *, tensor self, int64_t dim, int return_inverse, int return_counts); -void atg_unique_dim_consecutive_out(tensor *, tensor out0, tensor out1, tensor out2, tensor self, int64_t dim, int return_inverse, int return_counts); -void atg_unique_dim_out(tensor *, tensor out0, tensor out1, tensor out2, tensor self, int64_t dim, int sorted, int return_inverse, int return_counts); -tensor *atg_unsafe_chunk(tensor self, int64_t chunks, int64_t dim); -tensor *atg_unsafe_split(tensor self, int64_t split_size, int64_t dim); -void atg_unsafe_split_tensor_out(tensor *out_data, int out_len, tensor self, int64_t split_size, int64_t dim); -tensor *atg_unsafe_split_with_sizes(tensor self, int64_t *split_sizes_data, int split_sizes_len, int64_t dim); -void atg_unsafe_split_with_sizes_out(tensor *out_data, int out_len, tensor self, int64_t *split_sizes_data, int split_sizes_len, int64_t dim); -void atg_unsqueeze(tensor *, tensor self, int64_t dim); -void atg_unsqueeze_(tensor *, tensor self, int64_t dim); -void atg_unsqueeze_copy(tensor *, tensor self, int64_t dim); -void atg_unsqueeze_copy_out(tensor *, tensor out, tensor self, int64_t dim); -void atg_upsample_bicubic2d(tensor *, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg_upsample_bicubic2d_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg_upsample_bicubic2d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg_upsample_bicubic2d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg_upsample_bicubic2d_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len); -void atg_upsample_bilinear2d(tensor *, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg_upsample_bilinear2d_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg_upsample_bilinear2d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg_upsample_bilinear2d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg_upsample_bilinear2d_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len); -void atg_upsample_bilinear2d_vec_out(tensor *, tensor out, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len); -void atg_upsample_linear1d(tensor *, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_v, uint8_t scales_null); -void atg_upsample_linear1d_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_v, uint8_t scales_null); -void atg_upsample_linear1d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_v, uint8_t scales_null); -void atg_upsample_linear1d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_v, uint8_t scales_null); -void atg_upsample_linear1d_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len); -void atg_upsample_nearest1d(tensor *, tensor self, int64_t *output_size_data, int output_size_len, double scales_v, uint8_t scales_null); -void atg_upsample_nearest1d_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_v, uint8_t scales_null); -void atg_upsample_nearest1d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_v, uint8_t scales_null); -void atg_upsample_nearest1d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_v, uint8_t scales_null); -void atg_upsample_nearest1d_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len); -void atg_upsample_nearest2d(tensor *, tensor self, int64_t *output_size_data, int output_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg_upsample_nearest2d_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg_upsample_nearest2d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg_upsample_nearest2d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg_upsample_nearest2d_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len); -void atg_upsample_nearest2d_vec_out(tensor *, tensor out, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len); -void atg_upsample_nearest3d(tensor *, tensor self, int64_t *output_size_data, int output_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg_upsample_nearest3d_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg_upsample_nearest3d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg_upsample_nearest3d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg_upsample_nearest3d_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, double *scale_factors_data, int scale_factors_len); -void atg_upsample_trilinear3d(tensor *, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg_upsample_trilinear3d_backward(tensor *, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg_upsample_trilinear3d_backward_grad_input(tensor *, tensor grad_input, tensor grad_output, int64_t *output_size_data, int output_size_len, int64_t *input_size_data, int input_size_len, int align_corners, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg_upsample_trilinear3d_out(tensor *, tensor out, tensor self, int64_t *output_size_data, int output_size_len, int align_corners, double scales_d_v, uint8_t scales_d_null, double scales_h_v, uint8_t scales_h_null, double scales_w_v, uint8_t scales_w_null); -void atg_upsample_trilinear3d_vec(tensor *, tensor input, int64_t *output_size_data, int output_size_len, int align_corners, double *scale_factors_data, int scale_factors_len); -void atg_value_selecting_reduction_backward(tensor *, tensor grad, int64_t dim, tensor indices, int64_t *sizes_data, int sizes_len, int keepdim); -void atg_values(tensor *, tensor self); -void atg_values_copy(tensor *, tensor self); -void atg_values_copy_out(tensor *, tensor out, tensor self); -void atg_vander(tensor *, tensor x, int64_t n_v, uint8_t n_null, int increasing); -void atg_var(tensor *, tensor self, int unbiased); -void atg_var_correction(tensor *, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim); -void atg_var_correction_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim); -void atg_var_dim(tensor *, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim); -void atg_var_mean(tensor *, tensor self, int unbiased); -void atg_var_mean_correction(tensor *, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim); -void atg_var_mean_correction_out(tensor *, tensor out0, tensor out1, tensor self, int64_t *dim_data, int dim_len, scalar correction, int keepdim); -void atg_var_mean_dim(tensor *, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim); -void atg_var_out(tensor *, tensor out, tensor self, int64_t *dim_data, int dim_len, int unbiased, int keepdim); -void atg_vdot(tensor *, tensor self, tensor other); -void atg_vdot_out(tensor *, tensor out, tensor self, tensor other); -void atg_view(tensor *, tensor self, int64_t *size_data, int size_len); -void atg_view_as(tensor *, tensor self, tensor other); -void atg_view_as_complex(tensor *, tensor self); -void atg_view_as_complex_copy(tensor *, tensor self); -void atg_view_as_complex_copy_out(tensor *, tensor out, tensor self); -void atg_view_as_real(tensor *, tensor self); -void atg_view_as_real_copy(tensor *, tensor self); -void atg_view_as_real_copy_out(tensor *, tensor out, tensor self); -void atg_view_copy(tensor *, tensor self, int64_t *size_data, int size_len); -void atg_view_copy_dtype(tensor *, tensor self, int dtype); -void atg_view_copy_dtype_out(tensor *, tensor out, tensor self, int dtype); -void atg_view_copy_out(tensor *, tensor out, tensor self, int64_t *size_data, int size_len); -void atg_view_dtype(tensor *, tensor self, int dtype); -tensor *atg_vsplit(tensor self, int64_t sections); -tensor *atg_vsplit_array(tensor self, int64_t *indices_data, int indices_len); -void atg_vstack(tensor *, tensor *tensors_data, int tensors_len); -void atg_vstack_out(tensor *, tensor out, tensor *tensors_data, int tensors_len); -tensor *atg_where(tensor condition); -void atg_where_scalar(tensor *, tensor condition, scalar self_scalar, scalar other); -void atg_where_scalarother(tensor *, tensor condition, tensor self, scalar other); -void atg_where_scalarself(tensor *, tensor condition, scalar self_scalar, tensor other); -void atg_where_self(tensor *, tensor condition, tensor self, tensor other); -void atg_where_self_out(tensor *, tensor out, tensor condition, tensor self, tensor other); -void atg_xlogy(tensor *, tensor self, tensor other); -void atg_xlogy_(tensor *, tensor self, tensor other); -void atg_xlogy_outscalar_other(tensor *, tensor out, tensor self, scalar other); -void atg_xlogy_outscalar_self(tensor *, tensor out, scalar self_scalar, tensor other); -void atg_xlogy_outtensor(tensor *, tensor out, tensor self, tensor other); -void atg_xlogy_scalar_other(tensor *, tensor self, scalar other); -void atg_xlogy_scalar_other_(tensor *, tensor self, scalar other); -void atg_xlogy_scalar_self(tensor *, scalar self_scalar, tensor other); -void atg_zero(tensor *, tensor self); -void atg_zero_(tensor *, tensor self); -void atg_zero_out(tensor *, tensor out, tensor self); -void atg_zeros(tensor *, int64_t *size_data, int size_len, int options_kind, int options_device); -void atg_zeros_like(tensor *, tensor self); -void atg_zeros_like_out(tensor *, tensor out, tensor self); -void atg_zeros_out(tensor *, tensor out, int64_t *size_data, int size_len); -} diff --git a/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_python.cpp b/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_python.cpp deleted file mode 100644 index 03ce8743e7..0000000000 --- a/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_python.cpp +++ /dev/null @@ -1,22 +0,0 @@ -#include "torch_python.h" - -bool thp_variable_check(PyObject* pyobj) { - PROTECT( - return THPVariable_Check(pyobj); - ) - return false; -} - -PyObject* thp_variable_wrap(tensor t) { - PROTECT( - return THPVariable_Wrap(*t); - ) - return nullptr; -} - -tensor thp_variable_unpack(PyObject* pyobj) { - PROTECT( - return new torch::Tensor(THPVariable_Unpack(pyobj)); - ) - return nullptr; -} diff --git a/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_python.h b/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_python.h deleted file mode 100644 index bd5f17eb6d..0000000000 --- a/v2/dependency-patches/torch-sys-0.24.0/libtch/torch_python.h +++ /dev/null @@ -1,19 +0,0 @@ -#ifndef __TORCH_PYTHON_H__ -#define __TORCH_PYTHON_H__ - -#include "torch_api.h" -#include - -#ifdef __cplusplus -extern "C" { -#endif - -bool thp_variable_check(PyObject*); -PyObject* thp_variable_wrap(tensor); -tensor thp_variable_unpack(PyObject*); - -#ifdef __cplusplus -}; -#endif - -#endif diff --git a/v2/dependency-patches/torch-sys-0.24.0/src/c_generated.rs b/v2/dependency-patches/torch-sys-0.24.0/src/c_generated.rs deleted file mode 100644 index a89dce6b7c..0000000000 --- a/v2/dependency-patches/torch-sys-0.24.0/src/c_generated.rs +++ /dev/null @@ -1,16055 +0,0 @@ -/* THIS FILE IS AUTOMATICALLY GENERATED, DO NOT EDIT BY HAND! */ -#[allow(clippy::all)] -use crate::{C_scalar, C_tensor}; -use libc::c_int; - -extern "C" { - pub fn atg___and__(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg___and__tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg___iand__(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg___iand__tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg___ilshift__(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg___ilshift__tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg___ior__(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg___ior__tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg___irshift__(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg___irshift__tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg___ixor__(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg___ixor__tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg___lshift__(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg___lshift__scalar_out_( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg___lshift__tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg___lshift__tensor_out_( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg___or__(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg___or__tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg___rshift__(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg___rshift__scalar_out_( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg___rshift__tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg___rshift__tensor_out_( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg___xor__(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg___xor__tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg__adaptive_avg_pool2d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - ); - pub fn atg__adaptive_avg_pool2d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg__adaptive_avg_pool2d_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg__adaptive_avg_pool2d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - ); - pub fn atg__adaptive_avg_pool3d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - ); - pub fn atg__adaptive_avg_pool3d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg__adaptive_avg_pool3d_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg__adaptive_avg_pool3d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - ); - pub fn atg__add_batch_dim( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - batch_dim_: i64, - level_: i64, - ); - pub fn atg__add_relu(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg__add_relu_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg__add_relu_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg__add_relu_scalar( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg__add_relu_scalar_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg__add_relu_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg__addmm_activation( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mat1_: *mut C_tensor, - mat2_: *mut C_tensor, - use_gelu_: c_int, - ); - pub fn atg__addmm_activation_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mat1_: *mut C_tensor, - mat2_: *mut C_tensor, - use_gelu_: c_int, - ); - pub fn atg__aminmax(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__aminmax_dim( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - keepdim_: c_int, - ); - pub fn atg__aminmax_dim_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - keepdim_: c_int, - ); - pub fn atg__aminmax_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg__amp_update_scale( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - growth_tracker_: *mut C_tensor, - found_inf_: *mut C_tensor, - scale_growth_factor_: f64, - scale_backoff_factor_: f64, - growth_interval_: i64, - ); - pub fn atg__amp_update_scale_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - growth_tracker_: *mut C_tensor, - found_inf_: *mut C_tensor, - scale_growth_factor_: f64, - scale_backoff_factor_: f64, - growth_interval_: i64, - ); - pub fn atg__amp_update_scale_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - growth_tracker_: *mut C_tensor, - found_inf_: *mut C_tensor, - scale_growth_factor_: f64, - scale_backoff_factor_: f64, - growth_interval_: i64, - ); - pub fn atg__assert_scalar( - self_scalar_: *mut C_scalar, - assert_msg_ptr: *const u8, - assert_msg_len: c_int, - ); - pub fn atg__assert_tensor_metadata( - a_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - dtype_: c_int, - device_: c_int, - layout_: i8, - ); - pub fn atg__autocast_to_full_precision( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - cuda_enabled_: c_int, - cpu_enabled_: c_int, - ); - pub fn atg__autocast_to_reduced_precision( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - cuda_enabled_: c_int, - cpu_enabled_: c_int, - cuda_dtype_: c_int, - cpu_dtype_: c_int, - ); - pub fn atg__batch_norm_no_update( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - momentum_: f64, - eps_: f64, - ); - pub fn atg__batch_norm_no_update_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - out2_: *mut C_tensor, - out3_: *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - momentum_: f64, - eps_: f64, - ); - pub fn atg__batch_norm_with_update( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - momentum_: f64, - eps_: f64, - ); - pub fn atg__batch_norm_with_update_functional( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - momentum_: f64, - eps_: f64, - ); - pub fn atg__batch_norm_with_update_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - save_mean_: *mut C_tensor, - save_invstd_: *mut C_tensor, - reserve_: *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - momentum_: f64, - eps_: f64, - ); - pub fn atg__cast_byte(out__: *mut *mut C_tensor, self_: *mut C_tensor, non_blocking_: c_int); - pub fn atg__cast_char(out__: *mut *mut C_tensor, self_: *mut C_tensor, non_blocking_: c_int); - pub fn atg__cast_double(out__: *mut *mut C_tensor, self_: *mut C_tensor, non_blocking_: c_int); - pub fn atg__cast_float(out__: *mut *mut C_tensor, self_: *mut C_tensor, non_blocking_: c_int); - pub fn atg__cast_half(out__: *mut *mut C_tensor, self_: *mut C_tensor, non_blocking_: c_int); - pub fn atg__cast_int(out__: *mut *mut C_tensor, self_: *mut C_tensor, non_blocking_: c_int); - pub fn atg__cast_long(out__: *mut *mut C_tensor, self_: *mut C_tensor, non_blocking_: c_int); - pub fn atg__cast_short(out__: *mut *mut C_tensor, self_: *mut C_tensor, non_blocking_: c_int); - pub fn atg__cdist_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - x1_: *mut C_tensor, - x2_: *mut C_tensor, - p_: f64, - cdist_: *mut C_tensor, - ); - pub fn atg__cdist_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_: *mut C_tensor, - x1_: *mut C_tensor, - x2_: *mut C_tensor, - p_: f64, - cdist_: *mut C_tensor, - ); - pub fn atg__cholesky_solve_helper( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - A_: *mut C_tensor, - upper_: c_int, - ); - pub fn atg__cholesky_solve_helper_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - A_: *mut C_tensor, - upper_: c_int, - ); - pub fn atg__chunk_cat( - out__: *mut *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - dim_: i64, - num_chunks_: i64, - ); - pub fn atg__chunk_cat_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - dim_: i64, - num_chunks_: i64, - ); - pub fn atg__coalesce(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__coalesce_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg__coalesced(out__: *mut *mut C_tensor, self_: *mut C_tensor, coalesced_: c_int); - pub fn atg__coalesced_(out__: *mut *mut C_tensor, self_: *mut C_tensor, coalesced_: c_int); - pub fn atg__coalesced_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - coalesced_: c_int, - ); - pub fn atg__compute_linear_combination( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - coefficients_: *mut C_tensor, - ); - pub fn atg__compute_linear_combination_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - input_: *mut C_tensor, - coefficients_: *mut C_tensor, - ); - pub fn atg__conj(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__conj_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__conj_copy_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg__conj_physical(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__conj_physical_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg__conv_depthwise2d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ); - pub fn atg__conv_depthwise2d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ); - pub fn atg__convert_indices_from_coo_to_csr( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_: i64, - out_int32_: c_int, - ); - pub fn atg__convert_indices_from_coo_to_csr_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - size_: i64, - out_int32_: c_int, - ); - pub fn atg__convert_indices_from_csr_to_coo( - out__: *mut *mut C_tensor, - crow_indices_: *mut C_tensor, - col_indices_: *mut C_tensor, - out_int32_: c_int, - transpose_: c_int, - ); - pub fn atg__convert_indices_from_csr_to_coo_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - crow_indices_: *mut C_tensor, - col_indices_: *mut C_tensor, - out_int32_: c_int, - transpose_: c_int, - ); - pub fn atg__convert_weight_to_int4pack( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - innerKTiles_: i64, - ); - pub fn atg__convert_weight_to_int4pack_for_cpu( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - innerKTiles_: i64, - ); - pub fn atg__convolution( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - transposed_: c_int, - output_padding_data: *const i64, - output_padding_len: c_int, - groups_: i64, - benchmark_: c_int, - deterministic_: c_int, - cudnn_enabled_: c_int, - allow_tf32_: c_int, - ); - pub fn atg__convolution_deprecated( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - transposed_: c_int, - output_padding_data: *const i64, - output_padding_len: c_int, - groups_: i64, - benchmark_: c_int, - deterministic_: c_int, - cudnn_enabled_: c_int, - ); - pub fn atg__convolution_mode( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_ptr: *const u8, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - ); - pub fn atg__convolution_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - transposed_: c_int, - output_padding_data: *const i64, - output_padding_len: c_int, - groups_: i64, - benchmark_: c_int, - deterministic_: c_int, - cudnn_enabled_: c_int, - allow_tf32_: c_int, - ); - pub fn atg__copy_from( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dst_: *mut C_tensor, - non_blocking_: c_int, - ); - pub fn atg__copy_from_and_resize( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dst_: *mut C_tensor, - ); - pub fn atg__copy_from_and_resize_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dst_: *mut C_tensor, - ); - pub fn atg__copy_from_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dst_: *mut C_tensor, - non_blocking_: c_int, - ); - pub fn atg__cslt_compress(out__: *mut *mut C_tensor, input_: *mut C_tensor); - pub fn atg__cslt_sparse_mm( - out__: *mut *mut C_tensor, - compressed_A_: *mut C_tensor, - dense_B_: *mut C_tensor, - bias_: *mut C_tensor, - alpha_: *mut C_tensor, - out_dtype_: c_int, - transpose_result_: c_int, - alg_id_: i64, - split_k_: i64, - split_k_mode_: i64, - ); - pub fn atg__cslt_sparse_mm_search( - compressed_A_: *mut C_tensor, - dense_B_: *mut C_tensor, - bias_: *mut C_tensor, - alpha_: *mut C_tensor, - out_dtype_: c_int, - transpose_result_: c_int, - ) -> i64; - pub fn atg__ctc_loss( - out__: *mut *mut C_tensor, - log_probs_: *mut C_tensor, - targets_: *mut C_tensor, - input_lengths_data: *const i64, - input_lengths_len: c_int, - target_lengths_data: *const i64, - target_lengths_len: c_int, - blank_: i64, - zero_infinity_: c_int, - ); - pub fn atg__ctc_loss_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - log_probs_: *mut C_tensor, - targets_: *mut C_tensor, - input_lengths_data: *const i64, - input_lengths_len: c_int, - target_lengths_data: *const i64, - target_lengths_len: c_int, - neg_log_likelihood_: *mut C_tensor, - log_alpha_: *mut C_tensor, - blank_: i64, - zero_infinity_: c_int, - ); - pub fn atg__ctc_loss_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_: *mut C_tensor, - log_probs_: *mut C_tensor, - targets_: *mut C_tensor, - input_lengths_data: *const i64, - input_lengths_len: c_int, - target_lengths_data: *const i64, - target_lengths_len: c_int, - neg_log_likelihood_: *mut C_tensor, - log_alpha_: *mut C_tensor, - blank_: i64, - zero_infinity_: c_int, - ); - pub fn atg__ctc_loss_backward_tensor( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - log_probs_: *mut C_tensor, - targets_: *mut C_tensor, - input_lengths_: *mut C_tensor, - target_lengths_: *mut C_tensor, - neg_log_likelihood_: *mut C_tensor, - log_alpha_: *mut C_tensor, - blank_: i64, - zero_infinity_: c_int, - ); - pub fn atg__ctc_loss_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - log_probs_: *mut C_tensor, - targets_: *mut C_tensor, - input_lengths_data: *const i64, - input_lengths_len: c_int, - target_lengths_data: *const i64, - target_lengths_len: c_int, - blank_: i64, - zero_infinity_: c_int, - ); - pub fn atg__ctc_loss_tensor( - out__: *mut *mut C_tensor, - log_probs_: *mut C_tensor, - targets_: *mut C_tensor, - input_lengths_: *mut C_tensor, - target_lengths_: *mut C_tensor, - blank_: i64, - zero_infinity_: c_int, - ); - pub fn atg__ctc_loss_tensor_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - log_probs_: *mut C_tensor, - targets_: *mut C_tensor, - input_lengths_: *mut C_tensor, - target_lengths_: *mut C_tensor, - blank_: i64, - zero_infinity_: c_int, - ); - pub fn atg__cudnn_attention_backward( - out__: *mut *mut C_tensor, - grad_out_: *mut C_tensor, - query_: *mut C_tensor, - key_: *mut C_tensor, - value_: *mut C_tensor, - out_: *mut C_tensor, - logsumexp_: *mut C_tensor, - philox_seed_: *mut C_tensor, - philox_offset_: *mut C_tensor, - attn_bias_: *mut C_tensor, - cum_seq_q_: *mut C_tensor, - cum_seq_k_: *mut C_tensor, - max_q_: i64, - max_k_: i64, - dropout_p_: f64, - is_causal_: c_int, - scale_v: f64, - scale_null: i8, - ); - pub fn atg__cudnn_ctc_loss( - out__: *mut *mut C_tensor, - log_probs_: *mut C_tensor, - targets_: *mut C_tensor, - input_lengths_data: *const i64, - input_lengths_len: c_int, - target_lengths_data: *const i64, - target_lengths_len: c_int, - blank_: i64, - deterministic_: c_int, - zero_infinity_: c_int, - ); - pub fn atg__cudnn_ctc_loss_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - log_probs_: *mut C_tensor, - targets_: *mut C_tensor, - input_lengths_data: *const i64, - input_lengths_len: c_int, - target_lengths_data: *const i64, - target_lengths_len: c_int, - blank_: i64, - deterministic_: c_int, - zero_infinity_: c_int, - ); - pub fn atg__cudnn_ctc_loss_tensor( - out__: *mut *mut C_tensor, - log_probs_: *mut C_tensor, - targets_: *mut C_tensor, - input_lengths_: *mut C_tensor, - target_lengths_: *mut C_tensor, - blank_: i64, - deterministic_: c_int, - zero_infinity_: c_int, - ); - pub fn atg__cudnn_init_dropout_state( - out__: *mut *mut C_tensor, - dropout_: f64, - train_: c_int, - dropout_seed_: i64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg__cudnn_init_dropout_state_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - dropout_: f64, - train_: c_int, - dropout_seed_: i64, - ); - pub fn atg__cudnn_rnn( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_data: *const *mut C_tensor, - weight_len: c_int, - weight_stride0_: i64, - weight_buf_: *mut C_tensor, - hx_: *mut C_tensor, - cx_: *mut C_tensor, - mode_: i64, - hidden_size_: i64, - proj_size_: i64, - num_layers_: i64, - batch_first_: c_int, - dropout_: f64, - train_: c_int, - bidirectional_: c_int, - batch_sizes_data: *const i64, - batch_sizes_len: c_int, - dropout_state_: *mut C_tensor, - ); - pub fn atg__cudnn_rnn_flatten_weight( - out__: *mut *mut C_tensor, - weight_arr_data: *const *mut C_tensor, - weight_arr_len: c_int, - weight_stride0_: i64, - input_size_: i64, - mode_: i64, - hidden_size_: i64, - proj_size_: i64, - num_layers_: i64, - batch_first_: c_int, - bidirectional_: c_int, - ); - pub fn atg__cudnn_rnn_flatten_weight_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - weight_arr_data: *const *mut C_tensor, - weight_arr_len: c_int, - weight_stride0_: i64, - input_size_: i64, - mode_: i64, - hidden_size_: i64, - proj_size_: i64, - num_layers_: i64, - batch_first_: c_int, - bidirectional_: c_int, - ); - pub fn atg__cudnn_rnn_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - out2_: *mut C_tensor, - out3_: *mut C_tensor, - out4_: *mut C_tensor, - input_: *mut C_tensor, - weight_data: *const *mut C_tensor, - weight_len: c_int, - weight_stride0_: i64, - weight_buf_: *mut C_tensor, - hx_: *mut C_tensor, - cx_: *mut C_tensor, - mode_: i64, - hidden_size_: i64, - proj_size_: i64, - num_layers_: i64, - batch_first_: c_int, - dropout_: f64, - train_: c_int, - bidirectional_: c_int, - batch_sizes_data: *const i64, - batch_sizes_len: c_int, - dropout_state_: *mut C_tensor, - ); - pub fn atg__debug_has_internal_overlap(self_: *mut C_tensor) -> i64; - pub fn atg__dim_arange(out__: *mut *mut C_tensor, like_: *mut C_tensor, dim_: i64); - pub fn atg__dimi(self_: *mut C_tensor) -> i64; - pub fn atg__dimv(self_: *mut C_tensor) -> i64; - pub fn atg__dirichlet_grad( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - alpha_: *mut C_tensor, - total_: *mut C_tensor, - ); - pub fn atg__dirichlet_grad_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - alpha_: *mut C_tensor, - total_: *mut C_tensor, - ); - pub fn atg__dyn_quant_matmul_4bit( - out__: *mut *mut C_tensor, - inp_: *mut C_tensor, - packed_weights_: *mut C_tensor, - block_size_: i64, - in_features_: i64, - out_features_: i64, - ); - pub fn atg__dyn_quant_pack_4bit_weight( - out__: *mut *mut C_tensor, - weights_: *mut C_tensor, - scales_zeros_: *mut C_tensor, - bias_: *mut C_tensor, - block_size_: i64, - in_features_: i64, - out_features_: i64, - ); - pub fn atg__efficient_attention_backward( - out__: *mut *mut C_tensor, - grad_out__: *mut C_tensor, - query_: *mut C_tensor, - key_: *mut C_tensor, - value_: *mut C_tensor, - bias_: *mut C_tensor, - out_: *mut C_tensor, - cu_seqlens_q_: *mut C_tensor, - cu_seqlens_k_: *mut C_tensor, - max_seqlen_q_: i64, - max_seqlen_k_: i64, - logsumexp_: *mut C_tensor, - dropout_p_: f64, - philox_seed_: *mut C_tensor, - philox_offset_: *mut C_tensor, - custom_mask_type_: i64, - bias_requires_grad_: c_int, - scale_v: f64, - scale_null: i8, - num_splits_key_v: i64, - num_splits_key_null: i8, - window_size_v: i64, - window_size_null: i8, - shared_storage_dqdkdv_: c_int, - ); - pub fn atg__efficientzerotensor( - out__: *mut *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg__efficientzerotensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg__embedding_bag( - out__: *mut *mut C_tensor, - weight_: *mut C_tensor, - indices_: *mut C_tensor, - offsets_: *mut C_tensor, - scale_grad_by_freq_: c_int, - mode_: i64, - sparse_: c_int, - per_sample_weights_: *mut C_tensor, - include_last_offset_: c_int, - padding_idx_: i64, - ); - pub fn atg__embedding_bag_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - indices_: *mut C_tensor, - offsets_: *mut C_tensor, - offset2bag_: *mut C_tensor, - bag_size_: *mut C_tensor, - maximum_indices_: *mut C_tensor, - num_weights_: i64, - scale_grad_by_freq_: c_int, - mode_: i64, - sparse_: c_int, - per_sample_weights_: *mut C_tensor, - padding_idx_: i64, - ); - pub fn atg__embedding_bag_dense_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - indices_: *mut C_tensor, - offset2bag_: *mut C_tensor, - bag_size_: *mut C_tensor, - maximum_indices_: *mut C_tensor, - num_weights_: i64, - scale_grad_by_freq_: c_int, - mode_: i64, - per_sample_weights_: *mut C_tensor, - padding_idx_: i64, - ); - pub fn atg__embedding_bag_dense_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_: *mut C_tensor, - indices_: *mut C_tensor, - offset2bag_: *mut C_tensor, - bag_size_: *mut C_tensor, - maximum_indices_: *mut C_tensor, - num_weights_: i64, - scale_grad_by_freq_: c_int, - mode_: i64, - per_sample_weights_: *mut C_tensor, - padding_idx_: i64, - ); - pub fn atg__embedding_bag_forward_only( - out__: *mut *mut C_tensor, - weight_: *mut C_tensor, - indices_: *mut C_tensor, - offsets_: *mut C_tensor, - scale_grad_by_freq_: c_int, - mode_: i64, - sparse_: c_int, - per_sample_weights_: *mut C_tensor, - include_last_offset_: c_int, - padding_idx_: i64, - ); - pub fn atg__embedding_bag_forward_only_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - out2_: *mut C_tensor, - out3_: *mut C_tensor, - weight_: *mut C_tensor, - indices_: *mut C_tensor, - offsets_: *mut C_tensor, - scale_grad_by_freq_: c_int, - mode_: i64, - sparse_: c_int, - per_sample_weights_: *mut C_tensor, - include_last_offset_: c_int, - padding_idx_: i64, - ); - pub fn atg__embedding_bag_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - out2_: *mut C_tensor, - out3_: *mut C_tensor, - weight_: *mut C_tensor, - indices_: *mut C_tensor, - offsets_: *mut C_tensor, - scale_grad_by_freq_: c_int, - mode_: i64, - sparse_: c_int, - per_sample_weights_: *mut C_tensor, - include_last_offset_: c_int, - padding_idx_: i64, - ); - pub fn atg__embedding_bag_per_sample_weights_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - weight_: *mut C_tensor, - indices_: *mut C_tensor, - offsets_: *mut C_tensor, - offset2bag_: *mut C_tensor, - mode_: i64, - padding_idx_: i64, - ); - pub fn atg__embedding_bag_per_sample_weights_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_: *mut C_tensor, - weight_: *mut C_tensor, - indices_: *mut C_tensor, - offsets_: *mut C_tensor, - offset2bag_: *mut C_tensor, - mode_: i64, - padding_idx_: i64, - ); - pub fn atg__embedding_bag_sparse_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - indices_: *mut C_tensor, - offsets_: *mut C_tensor, - offset2bag_: *mut C_tensor, - bag_size_: *mut C_tensor, - num_weights_: i64, - scale_grad_by_freq_: c_int, - mode_: i64, - per_sample_weights_: *mut C_tensor, - padding_idx_: i64, - ); - pub fn atg__empty_affine_quantized( - out__: *mut *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - scale_: f64, - zero_point_: i64, - ); - pub fn atg__empty_affine_quantized_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - scale_: f64, - zero_point_: i64, - ); - pub fn atg__empty_per_channel_affine_quantized( - out__: *mut *mut C_tensor, - size_data: *const i64, - size_len: c_int, - scales_: *mut C_tensor, - zero_points_: *mut C_tensor, - axis_: i64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg__empty_per_channel_affine_quantized_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - scales_: *mut C_tensor, - zero_points_: *mut C_tensor, - axis_: i64, - ); - pub fn atg__euclidean_dist(out__: *mut *mut C_tensor, x1_: *mut C_tensor, x2_: *mut C_tensor); - pub fn atg__euclidean_dist_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x1_: *mut C_tensor, - x2_: *mut C_tensor, - ); - pub fn atg__fake_quantize_learnable_per_channel_affine( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - scale_: *mut C_tensor, - zero_point_: *mut C_tensor, - axis_: i64, - quant_min_: i64, - quant_max_: i64, - grad_factor_: f64, - ); - pub fn atg__fake_quantize_learnable_per_channel_affine_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - self_: *mut C_tensor, - scale_: *mut C_tensor, - zero_point_: *mut C_tensor, - axis_: i64, - quant_min_: i64, - quant_max_: i64, - grad_factor_: f64, - ); - pub fn atg__fake_quantize_learnable_per_channel_affine_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - scale_: *mut C_tensor, - zero_point_: *mut C_tensor, - axis_: i64, - quant_min_: i64, - quant_max_: i64, - grad_factor_: f64, - ); - pub fn atg__fake_quantize_learnable_per_tensor_affine( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - scale_: *mut C_tensor, - zero_point_: *mut C_tensor, - quant_min_: i64, - quant_max_: i64, - grad_factor_: f64, - ); - pub fn atg__fake_quantize_learnable_per_tensor_affine_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - self_: *mut C_tensor, - scale_: *mut C_tensor, - zero_point_: *mut C_tensor, - quant_min_: i64, - quant_max_: i64, - grad_factor_: f64, - ); - pub fn atg__fake_quantize_learnable_per_tensor_affine_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - scale_: *mut C_tensor, - zero_point_: *mut C_tensor, - quant_min_: i64, - quant_max_: i64, - grad_factor_: f64, - ); - pub fn atg__fake_quantize_per_tensor_affine_cachemask_tensor_qparams( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - scale_: *mut C_tensor, - zero_point_: *mut C_tensor, - fake_quant_enabled_: *mut C_tensor, - quant_min_: i64, - quant_max_: i64, - ); - pub fn atg__fake_quantize_per_tensor_affine_cachemask_tensor_qparams_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - self_: *mut C_tensor, - scale_: *mut C_tensor, - zero_point_: *mut C_tensor, - fake_quant_enabled_: *mut C_tensor, - quant_min_: i64, - quant_max_: i64, - ); - pub fn atg__fft_c2c( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - normalization_: i64, - forward_: c_int, - ); - pub fn atg__fft_c2c_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - normalization_: i64, - forward_: c_int, - ); - pub fn atg__fft_c2r( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - normalization_: i64, - last_dim_size_: i64, - ); - pub fn atg__fft_c2r_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - normalization_: i64, - last_dim_size_: i64, - ); - pub fn atg__fft_r2c( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - normalization_: i64, - onesided_: c_int, - ); - pub fn atg__fft_r2c_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - normalization_: i64, - onesided_: c_int, - ); - pub fn atg__fill_mem_eff_dropout_mask_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dropout_p_: f64, - seed_: i64, - offset_: i64, - ); - pub fn atg__flash_attention_backward( - out__: *mut *mut C_tensor, - grad_out_: *mut C_tensor, - query_: *mut C_tensor, - key_: *mut C_tensor, - value_: *mut C_tensor, - out_: *mut C_tensor, - logsumexp_: *mut C_tensor, - cum_seq_q_: *mut C_tensor, - cum_seq_k_: *mut C_tensor, - max_q_: i64, - max_k_: i64, - dropout_p_: f64, - is_causal_: c_int, - rng_state_: *mut C_tensor, - unused_: *mut C_tensor, - scale_v: f64, - scale_null: i8, - window_size_left_v: i64, - window_size_left_null: i8, - window_size_right_v: i64, - window_size_right_null: i8, - ); - pub fn atg__foobar( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - arg1_: c_int, - arg2_: c_int, - arg3_: c_int, - ); - pub fn atg__foobar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - arg1_: c_int, - arg2_: c_int, - arg3_: c_int, - ); - pub fn atg__functional_assert_async( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - assert_msg_ptr: *const u8, - assert_msg_len: c_int, - dep_token_: *mut C_tensor, - ); - pub fn atg__functional_assert_scalar( - out__: *mut *mut C_tensor, - self_scalar_: *mut C_scalar, - assert_msg_ptr: *const u8, - assert_msg_len: c_int, - dep_token_: *mut C_tensor, - ); - pub fn atg__functional_sym_constrain_range( - out__: *mut *mut C_tensor, - size_: *mut C_scalar, - min_v: i64, - min_null: i8, - max_v: i64, - max_null: i8, - dep_token_: *mut C_tensor, - ); - pub fn atg__functional_sym_constrain_range_for_size( - out__: *mut *mut C_tensor, - size_: *mut C_scalar, - min_v: i64, - min_null: i8, - max_v: i64, - max_null: i8, - dep_token_: *mut C_tensor, - ); - pub fn atg__fused_dropout(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: f64); - pub fn atg__fused_dropout_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - self_: *mut C_tensor, - p_: f64, - ); - pub fn atg__fused_moving_avg_obs_fq_helper( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - observer_on_: *mut C_tensor, - fake_quant_on_: *mut C_tensor, - running_min_: *mut C_tensor, - running_max_: *mut C_tensor, - scale_: *mut C_tensor, - zero_point_: *mut C_tensor, - averaging_const_: f64, - quant_min_: i64, - quant_max_: i64, - ch_axis_: i64, - per_row_fake_quant_: c_int, - symmetric_quant_: c_int, - ); - pub fn atg__fused_moving_avg_obs_fq_helper_functional( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - observer_on_: *mut C_tensor, - fake_quant_on_: *mut C_tensor, - running_min_: *mut C_tensor, - running_max_: *mut C_tensor, - scale_: *mut C_tensor, - zero_point_: *mut C_tensor, - averaging_const_: f64, - quant_min_: i64, - quant_max_: i64, - ch_axis_: i64, - per_row_fake_quant_: c_int, - symmetric_quant_: c_int, - ); - pub fn atg__fused_moving_avg_obs_fq_helper_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - self_: *mut C_tensor, - observer_on_: *mut C_tensor, - fake_quant_on_: *mut C_tensor, - running_min_: *mut C_tensor, - running_max_: *mut C_tensor, - scale_: *mut C_tensor, - zero_point_: *mut C_tensor, - averaging_const_: f64, - quant_min_: i64, - quant_max_: i64, - ch_axis_: i64, - per_row_fake_quant_: c_int, - symmetric_quant_: c_int, - ); - pub fn atg__fused_rms_norm( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - normalized_shape_data: *const i64, - normalized_shape_len: c_int, - weight_: *mut C_tensor, - eps_v: f64, - eps_null: i8, - ); - pub fn atg__fused_sdp_choice( - query_: *mut C_tensor, - key_: *mut C_tensor, - value_: *mut C_tensor, - attn_mask_: *mut C_tensor, - dropout_p_: f64, - is_causal_: c_int, - scale_v: f64, - scale_null: i8, - enable_gqa_: c_int, - ) -> i64; - pub fn atg__fw_primal(out__: *mut *mut C_tensor, self_: *mut C_tensor, level_: i64); - pub fn atg__fw_primal_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor, level_: i64); - pub fn atg__fw_primal_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - level_: i64, - ); - pub fn atg__gather_sparse_backward( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - grad_: *mut C_tensor, - ); - pub fn atg__grid_sampler_2d_cpu_fallback( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - grid_: *mut C_tensor, - interpolation_mode_: i64, - padding_mode_: i64, - align_corners_: c_int, - ); - pub fn atg__grid_sampler_2d_cpu_fallback_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - input_: *mut C_tensor, - grid_: *mut C_tensor, - interpolation_mode_: i64, - padding_mode_: i64, - align_corners_: c_int, - ); - pub fn atg__grid_sampler_2d_cpu_fallback_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - input_: *mut C_tensor, - grid_: *mut C_tensor, - interpolation_mode_: i64, - padding_mode_: i64, - align_corners_: c_int, - ); - pub fn atg__grouped_mm( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mat2_: *mut C_tensor, - offs_: *mut C_tensor, - bias_: *mut C_tensor, - out_dtype_: c_int, - ); - pub fn atg__has_compatible_shallow_copy_type( - self_: *mut C_tensor, - from_: *mut C_tensor, - ) -> c_int; - pub fn atg__has_same_storage_numel(self_: *mut C_tensor, other_: *mut C_tensor) -> c_int; - pub fn atg__histogramdd_bin_edges( - self_: *mut C_tensor, - bins_data: *const i64, - bins_len: c_int, - range_data: *const f64, - range_len: c_int, - weight_: *mut C_tensor, - density_: c_int, - ) -> *mut *mut C_tensor; - pub fn atg__histogramdd_bin_edges_out( - out_data: *const *mut C_tensor, - out_len: c_int, - self_: *mut C_tensor, - bins_data: *const i64, - bins_len: c_int, - range_data: *const f64, - range_len: c_int, - weight_: *mut C_tensor, - density_: c_int, - ); - pub fn atg__histogramdd_from_bin_cts( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - bins_data: *const i64, - bins_len: c_int, - range_data: *const f64, - range_len: c_int, - weight_: *mut C_tensor, - density_: c_int, - ); - pub fn atg__histogramdd_from_bin_cts_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - bins_data: *const i64, - bins_len: c_int, - range_data: *const f64, - range_len: c_int, - weight_: *mut C_tensor, - density_: c_int, - ); - pub fn atg__histogramdd_from_bin_tensors( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - bins_data: *const *mut C_tensor, - bins_len: c_int, - weight_: *mut C_tensor, - density_: c_int, - ); - pub fn atg__histogramdd_from_bin_tensors_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - bins_data: *const *mut C_tensor, - bins_len: c_int, - weight_: *mut C_tensor, - density_: c_int, - ); - pub fn atg__index_put_impl( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - indices_data: *const *mut C_tensor, - indices_len: c_int, - values_: *mut C_tensor, - accumulate_: c_int, - unsafe_: c_int, - ); - pub fn atg__index_put_impl_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - indices_data: *const *mut C_tensor, - indices_len: c_int, - values_: *mut C_tensor, - accumulate_: c_int, - unsafe_: c_int, - ); - pub fn atg__index_put_impl_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - indices_data: *const *mut C_tensor, - indices_len: c_int, - values_: *mut C_tensor, - accumulate_: c_int, - unsafe_: c_int, - ); - pub fn atg__indices(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__indices_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__indices_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg__int_mm(out__: *mut *mut C_tensor, self_: *mut C_tensor, mat2_: *mut C_tensor); - pub fn atg__int_mm_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mat2_: *mut C_tensor, - ); - pub fn atg__is_all_true(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__is_any_true(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__is_zerotensor(self_: *mut C_tensor) -> c_int; - pub fn atg__lazy_clone(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__linalg_check_errors( - info_: *mut C_tensor, - api_name_ptr: *const u8, - api_name_len: c_int, - is_matrix_: c_int, - ); - pub fn atg__linalg_det(out__: *mut *mut C_tensor, A_: *mut C_tensor); - pub fn atg__linalg_det_result( - out__: *mut *mut C_tensor, - result_: *mut C_tensor, - LU_: *mut C_tensor, - pivots_: *mut C_tensor, - A_: *mut C_tensor, - ); - pub fn atg__linalg_eigh( - out__: *mut *mut C_tensor, - A_: *mut C_tensor, - UPLO_ptr: *const u8, - UPLO_len: c_int, - compute_v_: c_int, - ); - pub fn atg__linalg_eigh_eigenvalues( - out__: *mut *mut C_tensor, - eigenvalues_: *mut C_tensor, - eigenvectors_: *mut C_tensor, - A_: *mut C_tensor, - UPLO_ptr: *const u8, - UPLO_len: c_int, - compute_v_: c_int, - ); - pub fn atg__linalg_eigvals(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__linalg_slogdet(out__: *mut *mut C_tensor, A_: *mut C_tensor); - pub fn atg__linalg_slogdet_sign( - out__: *mut *mut C_tensor, - sign_: *mut C_tensor, - logabsdet_: *mut C_tensor, - LU_: *mut C_tensor, - pivots_: *mut C_tensor, - A_: *mut C_tensor, - ); - pub fn atg__linalg_solve_ex( - out__: *mut *mut C_tensor, - A_: *mut C_tensor, - B_: *mut C_tensor, - left_: c_int, - check_errors_: c_int, - ); - pub fn atg__linalg_solve_ex_result( - out__: *mut *mut C_tensor, - result_: *mut C_tensor, - LU_: *mut C_tensor, - pivots_: *mut C_tensor, - info_: *mut C_tensor, - A_: *mut C_tensor, - B_: *mut C_tensor, - left_: c_int, - check_errors_: c_int, - ); - pub fn atg__linalg_svd( - out__: *mut *mut C_tensor, - A_: *mut C_tensor, - full_matrices_: c_int, - compute_uv_: c_int, - driver_ptr: *const u8, - driver_len: c_int, - ); - pub fn atg__linalg_svd_u( - out__: *mut *mut C_tensor, - U_: *mut C_tensor, - S_: *mut C_tensor, - Vh_: *mut C_tensor, - A_: *mut C_tensor, - full_matrices_: c_int, - compute_uv_: c_int, - driver_ptr: *const u8, - driver_len: c_int, - ); - pub fn atg__log_softmax( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - half_to_float_: c_int, - ); - pub fn atg__log_softmax_backward_data( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - output_: *mut C_tensor, - dim_: i64, - input_dtype_: c_int, - ); - pub fn atg__log_softmax_backward_data_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_output_: *mut C_tensor, - output_: *mut C_tensor, - dim_: i64, - input_dtype_: c_int, - ); - pub fn atg__log_softmax_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - half_to_float_: c_int, - ); - pub fn atg__logcumsumexp(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64); - pub fn atg__logcumsumexp_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - ); - pub fn atg__lstm_mps( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - hx_data: *const *mut C_tensor, - hx_len: c_int, - params_data: *const *mut C_tensor, - params_len: c_int, - has_biases_: c_int, - num_layers_: i64, - dropout_: f64, - train_: c_int, - bidirectional_: c_int, - batch_first_: c_int, - ); - pub fn atg__lstm_mps_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - out2_: *mut C_tensor, - out3_: *mut C_tensor, - out4_: *mut C_tensor, - out5_: *mut C_tensor, - input_: *mut C_tensor, - hx_data: *const *mut C_tensor, - hx_len: c_int, - params_data: *const *mut C_tensor, - params_len: c_int, - has_biases_: c_int, - num_layers_: i64, - dropout_: f64, - train_: c_int, - bidirectional_: c_int, - batch_first_: c_int, - ); - pub fn atg__lu_with_info( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - pivot_: c_int, - check_errors_: c_int, - ); - pub fn atg__make_dep_token( - out__: *mut *mut C_tensor, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg__make_dual( - out__: *mut *mut C_tensor, - primal_: *mut C_tensor, - tangent_: *mut C_tensor, - level_: i64, - ); - pub fn atg__make_dual_copy( - out__: *mut *mut C_tensor, - primal_: *mut C_tensor, - tangent_: *mut C_tensor, - level_: i64, - ); - pub fn atg__make_dual_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - primal_: *mut C_tensor, - tangent_: *mut C_tensor, - level_: i64, - ); - pub fn atg__make_per_channel_quantized_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - scale_: *mut C_tensor, - zero_point_: *mut C_tensor, - axis_: i64, - ); - pub fn atg__make_per_channel_quantized_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - scale_: *mut C_tensor, - zero_point_: *mut C_tensor, - axis_: i64, - ); - pub fn atg__make_per_tensor_quantized_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - scale_: f64, - zero_point_: i64, - ); - pub fn atg__make_per_tensor_quantized_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - scale_: f64, - zero_point_: i64, - ); - pub fn atg__masked_scale( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mask_: *mut C_tensor, - scale_: f64, - ); - pub fn atg__masked_scale_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mask_: *mut C_tensor, - scale_: f64, - ); - pub fn atg__masked_softmax( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mask_: *mut C_tensor, - dim_v: i64, - dim_null: i8, - mask_type_v: i64, - mask_type_null: i8, - ); - pub fn atg__masked_softmax_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - output_: *mut C_tensor, - mask_: *mut C_tensor, - dim_v: i64, - dim_null: i8, - ); - pub fn atg__masked_softmax_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_output_: *mut C_tensor, - output_: *mut C_tensor, - mask_: *mut C_tensor, - dim_v: i64, - dim_null: i8, - ); - pub fn atg__masked_softmax_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mask_: *mut C_tensor, - dim_v: i64, - dim_null: i8, - mask_type_v: i64, - mask_type_null: i8, - ); - pub fn atg__mixed_dtypes_linear( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - scale_: *mut C_tensor, - bias_: *mut C_tensor, - activation_ptr: *const u8, - activation_len: c_int, - ); - pub fn atg__mkldnn_reshape( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - shape_data: *const i64, - shape_len: c_int, - ); - pub fn atg__mkldnn_reshape_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - shape_data: *const i64, - shape_len: c_int, - ); - pub fn atg__mkldnn_transpose( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim0_: i64, - dim1_: i64, - ); - pub fn atg__mkldnn_transpose_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim0_: i64, - dim1_: i64, - ); - pub fn atg__mkldnn_transpose_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim0_: i64, - dim1_: i64, - ); - pub fn atg__mps_convolution( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - ); - pub fn atg__mps_convolution_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - ); - pub fn atg__mps_convolution_transpose( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - output_padding_data: *const i64, - output_padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - ); - pub fn atg__mps_convolution_transpose_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - output_padding_data: *const i64, - output_padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - ); - pub fn atg__native_batch_norm_legit( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - training_: c_int, - momentum_: f64, - eps_: f64, - ); - pub fn atg__native_batch_norm_legit_functional( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - training_: c_int, - momentum_: f64, - eps_: f64, - ); - pub fn atg__native_batch_norm_legit_no_stats( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - training_: c_int, - momentum_: f64, - eps_: f64, - ); - pub fn atg__native_batch_norm_legit_no_stats_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - save_mean_: *mut C_tensor, - save_invstd_: *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - training_: c_int, - momentum_: f64, - eps_: f64, - ); - pub fn atg__native_batch_norm_legit_no_training( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - momentum_: f64, - eps_: f64, - ); - pub fn atg__native_batch_norm_legit_no_training_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - out2_: *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - momentum_: f64, - eps_: f64, - ); - pub fn atg__native_batch_norm_legit_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - save_mean_: *mut C_tensor, - save_invstd_: *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - training_: c_int, - momentum_: f64, - eps_: f64, - ); - pub fn atg__native_multi_head_attention( - out__: *mut *mut C_tensor, - query_: *mut C_tensor, - key_: *mut C_tensor, - value_: *mut C_tensor, - embed_dim_: i64, - num_head_: i64, - qkv_weight_: *mut C_tensor, - qkv_bias_: *mut C_tensor, - proj_weight_: *mut C_tensor, - proj_bias_: *mut C_tensor, - mask_: *mut C_tensor, - need_weights_: c_int, - average_attn_weights_: c_int, - mask_type_v: i64, - mask_type_null: i8, - ); - pub fn atg__native_multi_head_attention_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - query_: *mut C_tensor, - key_: *mut C_tensor, - value_: *mut C_tensor, - embed_dim_: i64, - num_head_: i64, - qkv_weight_: *mut C_tensor, - qkv_bias_: *mut C_tensor, - proj_weight_: *mut C_tensor, - proj_bias_: *mut C_tensor, - mask_: *mut C_tensor, - need_weights_: c_int, - average_attn_weights_: c_int, - mask_type_v: i64, - mask_type_null: i8, - ); - pub fn atg__neg_view(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__neg_view_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__neg_view_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg__nested_compute_contiguous_strides_offsets( - out__: *mut *mut C_tensor, - nested_size_: *mut C_tensor, - ); - pub fn atg__nested_from_padded( - out__: *mut *mut C_tensor, - padded_: *mut C_tensor, - cpu_nested_shape_example_: *mut C_tensor, - fuse_transform_0213_: c_int, - ); - pub fn atg__nested_from_padded_and_nested_example( - out__: *mut *mut C_tensor, - padded_: *mut C_tensor, - nt_example_: *mut C_tensor, - ); - pub fn atg__nested_from_padded_and_nested_example_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - padded_: *mut C_tensor, - nt_example_: *mut C_tensor, - ); - pub fn atg__nested_from_padded_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - padded_: *mut C_tensor, - cpu_nested_shape_example_: *mut C_tensor, - fuse_transform_0213_: c_int, - ); - pub fn atg__nested_from_padded_tensor( - out__: *mut *mut C_tensor, - padded_: *mut C_tensor, - offsets_: *mut C_tensor, - dummy_: *mut C_tensor, - ragged_idx_: i64, - min_seqlen_: *mut C_tensor, - max_seqlen_: *mut C_tensor, - sum_S_v: i64, - sum_S_null: i8, - ); - pub fn atg__nested_get_jagged_dummy(out__: *mut *mut C_tensor, any_: *mut C_tensor); - pub fn atg__nested_get_lengths(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__nested_get_max_seqlen(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__nested_get_min_seqlen(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__nested_get_offsets(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__nested_get_ragged_idx(self_: *mut C_tensor) -> i64; - pub fn atg__nested_get_values(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__nested_get_values_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__nested_get_values_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg__nested_select_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: i64, - ); - pub fn atg__nested_sum_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - ); - pub fn atg__nested_view_from_buffer( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - nested_size_: *mut C_tensor, - nested_strides_: *mut C_tensor, - offsets_: *mut C_tensor, - ); - pub fn atg__nested_view_from_buffer_copy( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - nested_size_: *mut C_tensor, - nested_strides_: *mut C_tensor, - offsets_: *mut C_tensor, - ); - pub fn atg__nested_view_from_buffer_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - nested_size_: *mut C_tensor, - nested_strides_: *mut C_tensor, - offsets_: *mut C_tensor, - ); - pub fn atg__nested_view_from_jagged( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - offsets_: *mut C_tensor, - dummy_: *mut C_tensor, - lengths_: *mut C_tensor, - ragged_idx_: i64, - min_seqlen_: *mut C_tensor, - max_seqlen_: *mut C_tensor, - ); - pub fn atg__nested_view_from_jagged_copy( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - offsets_: *mut C_tensor, - dummy_: *mut C_tensor, - lengths_: *mut C_tensor, - ragged_idx_: i64, - min_seqlen_: *mut C_tensor, - max_seqlen_: *mut C_tensor, - ); - pub fn atg__nested_view_from_jagged_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - offsets_: *mut C_tensor, - dummy_: *mut C_tensor, - lengths_: *mut C_tensor, - ragged_idx_: i64, - min_seqlen_: *mut C_tensor, - max_seqlen_: *mut C_tensor, - ); - pub fn atg__new_zeros_with_same_feature_meta( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - self_num_batch_dims_: i64, - ); - pub fn atg__new_zeros_with_same_feature_meta_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - self_num_batch_dims_: i64, - ); - pub fn atg__nnpack_available() -> c_int; - pub fn atg__nnpack_spatial_convolution( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - ); - pub fn atg__nnpack_spatial_convolution_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - ); - pub fn atg__nnz(self_: *mut C_tensor) -> i64; - pub fn atg__pack_padded_sequence( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - lengths_: *mut C_tensor, - batch_first_: c_int, - ); - pub fn atg__pack_padded_sequence_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - input_size_data: *const i64, - input_size_len: c_int, - batch_sizes_: *mut C_tensor, - batch_first_: c_int, - ); - pub fn atg__pack_padded_sequence_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - input_: *mut C_tensor, - lengths_: *mut C_tensor, - batch_first_: c_int, - ); - pub fn atg__pad_circular( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - pad_data: *const i64, - pad_len: c_int, - ); - pub fn atg__pad_enum( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - pad_data: *const i64, - pad_len: c_int, - mode_: i64, - value_v: f64, - value_null: i8, - ); - pub fn atg__pad_packed_sequence( - out__: *mut *mut C_tensor, - data_: *mut C_tensor, - batch_sizes_: *mut C_tensor, - batch_first_: c_int, - padding_value_: *mut C_scalar, - total_length_: i64, - ); - pub fn atg__pdist_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - self_: *mut C_tensor, - p_: f64, - pdist_: *mut C_tensor, - ); - pub fn atg__pdist_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_: *mut C_tensor, - self_: *mut C_tensor, - p_: f64, - pdist_: *mut C_tensor, - ); - pub fn atg__pin_memory(out__: *mut *mut C_tensor, self_: *mut C_tensor, device_: c_int); - pub fn atg__pin_memory_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - device_: c_int, - ); - pub fn atg__prelu_kernel( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - ); - pub fn atg__prelu_kernel_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - ); - pub fn atg__print(s_ptr: *const u8, s_len: c_int); - pub fn atg__propagate_xla_data(input_: *mut C_tensor, output_: *mut C_tensor); - pub fn atg__remove_batch_dim( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - level_: i64, - batch_size_: i64, - out_dim_: i64, - ); - pub fn atg__reshape_alias( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - ); - pub fn atg__reshape_alias_copy( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - ); - pub fn atg__reshape_alias_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - ); - pub fn atg__reshape_copy( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg__reshape_from_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - shape_: *mut C_tensor, - ); - pub fn atg__resize_output( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - device_: c_int, - ); - pub fn atg__resize_output_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - device_: c_int, - ); - pub fn atg__resize_output_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - device_: c_int, - ); - pub fn atg__rowwise_prune( - out__: *mut *mut C_tensor, - weight_: *mut C_tensor, - mask_: *mut C_tensor, - compressed_indices_dtype_: c_int, - ); - pub fn atg__safe_softmax( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - dtype_: c_int, - ); - pub fn atg__sample_dirichlet(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__sample_dirichlet_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg__saturate_weight_to_fp16(out__: *mut *mut C_tensor, weight_: *mut C_tensor); - pub fn atg__scaled_dot_product_attention_math( - out__: *mut *mut C_tensor, - query_: *mut C_tensor, - key_: *mut C_tensor, - value_: *mut C_tensor, - attn_mask_: *mut C_tensor, - dropout_p_: f64, - is_causal_: c_int, - dropout_mask_: *mut C_tensor, - scale_v: f64, - scale_null: i8, - enable_gqa_: c_int, - ); - pub fn atg__scaled_dot_product_attention_math_for_mps( - out__: *mut *mut C_tensor, - query_: *mut C_tensor, - key_: *mut C_tensor, - value_: *mut C_tensor, - attn_mask_: *mut C_tensor, - dropout_p_: f64, - is_causal_: c_int, - dropout_mask_: *mut C_tensor, - scale_v: f64, - scale_null: i8, - ); - pub fn atg__scaled_dot_product_cudnn_attention_backward( - out__: *mut *mut C_tensor, - grad_out_: *mut C_tensor, - query_: *mut C_tensor, - key_: *mut C_tensor, - value_: *mut C_tensor, - out_: *mut C_tensor, - logsumexp_: *mut C_tensor, - philox_seed_: *mut C_tensor, - philox_offset_: *mut C_tensor, - attn_bias_: *mut C_tensor, - cum_seq_q_: *mut C_tensor, - cum_seq_k_: *mut C_tensor, - max_q_: i64, - max_k_: i64, - dropout_p_: f64, - is_causal_: c_int, - scale_v: f64, - scale_null: i8, - ); - pub fn atg__scaled_dot_product_efficient_attention( - out__: *mut *mut C_tensor, - query_: *mut C_tensor, - key_: *mut C_tensor, - value_: *mut C_tensor, - attn_bias_: *mut C_tensor, - compute_log_sumexp_: c_int, - dropout_p_: f64, - is_causal_: c_int, - scale_v: f64, - scale_null: i8, - ); - pub fn atg__scaled_dot_product_flash_attention_backward( - out__: *mut *mut C_tensor, - grad_out_: *mut C_tensor, - query_: *mut C_tensor, - key_: *mut C_tensor, - value_: *mut C_tensor, - out_: *mut C_tensor, - logsumexp_: *mut C_tensor, - cum_seq_q_: *mut C_tensor, - cum_seq_k_: *mut C_tensor, - max_q_: i64, - max_k_: i64, - dropout_p_: f64, - is_causal_: c_int, - philox_seed_: *mut C_tensor, - philox_offset_: *mut C_tensor, - scale_v: f64, - scale_null: i8, - ); - pub fn atg__scaled_dot_product_flash_attention_for_cpu( - out__: *mut *mut C_tensor, - query_: *mut C_tensor, - key_: *mut C_tensor, - value_: *mut C_tensor, - dropout_p_: f64, - is_causal_: c_int, - attn_mask_: *mut C_tensor, - scale_v: f64, - scale_null: i8, - ); - pub fn atg__scaled_dot_product_flash_attention_for_cpu_backward( - out__: *mut *mut C_tensor, - grad_out_: *mut C_tensor, - query_: *mut C_tensor, - key_: *mut C_tensor, - value_: *mut C_tensor, - out_: *mut C_tensor, - logsumexp_: *mut C_tensor, - dropout_p_: f64, - is_causal_: c_int, - attn_mask_: *mut C_tensor, - scale_v: f64, - scale_null: i8, - ); - pub fn atg__scaled_grouped_mm( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mat2_: *mut C_tensor, - scale_a_: *mut C_tensor, - scale_b_: *mut C_tensor, - offs_: *mut C_tensor, - bias_: *mut C_tensor, - scale_result_: *mut C_tensor, - out_dtype_: c_int, - use_fast_accum_: c_int, - ); - pub fn atg__scaled_grouped_mm_v2( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mat2_: *mut C_tensor, - scale_a_data: *const *mut C_tensor, - scale_a_len: c_int, - recipe_a_data: *const i64, - recipe_a_len: c_int, - swizzle_a_data: *const i64, - swizzle_a_len: c_int, - scale_b_data: *const *mut C_tensor, - scale_b_len: c_int, - recipe_b_data: *const i64, - recipe_b_len: c_int, - swizzle_b_data: *const i64, - swizzle_b_len: c_int, - offs_: *mut C_tensor, - bias_: *mut C_tensor, - out_dtype_: c_int, - contraction_dim_data: *const i64, - contraction_dim_len: c_int, - use_fast_accum_: c_int, - ); - pub fn atg__scaled_mm( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mat2_: *mut C_tensor, - scale_a_: *mut C_tensor, - scale_b_: *mut C_tensor, - bias_: *mut C_tensor, - scale_result_: *mut C_tensor, - out_dtype_: c_int, - use_fast_accum_: c_int, - ); - pub fn atg__scaled_mm_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mat2_: *mut C_tensor, - scale_a_: *mut C_tensor, - scale_b_: *mut C_tensor, - bias_: *mut C_tensor, - scale_result_: *mut C_tensor, - out_dtype_: c_int, - use_fast_accum_: c_int, - ); - pub fn atg__scaled_mm_v2( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mat2_: *mut C_tensor, - scale_a_data: *const *mut C_tensor, - scale_a_len: c_int, - recipe_a_data: *const i64, - recipe_a_len: c_int, - swizzle_a_data: *const i64, - swizzle_a_len: c_int, - scale_b_data: *const *mut C_tensor, - scale_b_len: c_int, - recipe_b_data: *const i64, - recipe_b_len: c_int, - swizzle_b_data: *const i64, - swizzle_b_len: c_int, - bias_: *mut C_tensor, - out_dtype_: c_int, - contraction_dim_data: *const i64, - contraction_dim_len: c_int, - use_fast_accum_: c_int, - ); - pub fn atg__scaled_mm_v2_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mat2_: *mut C_tensor, - scale_a_data: *const *mut C_tensor, - scale_a_len: c_int, - recipe_a_data: *const i64, - recipe_a_len: c_int, - swizzle_a_data: *const i64, - swizzle_a_len: c_int, - scale_b_data: *const *mut C_tensor, - scale_b_len: c_int, - recipe_b_data: *const i64, - recipe_b_len: c_int, - swizzle_b_data: *const i64, - swizzle_b_len: c_int, - bias_: *mut C_tensor, - out_dtype_: c_int, - contraction_dim_data: *const i64, - contraction_dim_len: c_int, - use_fast_accum_: c_int, - ); - pub fn atg__scatter_reduce( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - src_: *mut C_tensor, - reduce_ptr: *const u8, - reduce_len: c_int, - include_self_: c_int, - ); - pub fn atg__scatter_reduce_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - src_: *mut C_tensor, - reduce_ptr: *const u8, - reduce_len: c_int, - include_self_: c_int, - ); - pub fn atg__scatter_reduce_two_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - src_: *mut C_tensor, - reduce_ptr: *const u8, - reduce_len: c_int, - include_self_: c_int, - ); - pub fn atg__segment_reduce_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - output_: *mut C_tensor, - data_: *mut C_tensor, - reduce_ptr: *const u8, - reduce_len: c_int, - lengths_: *mut C_tensor, - offsets_: *mut C_tensor, - axis_: i64, - initial_: *mut C_scalar, - ); - pub fn atg__segment_reduce_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_: *mut C_tensor, - output_: *mut C_tensor, - data_: *mut C_tensor, - reduce_ptr: *const u8, - reduce_len: c_int, - lengths_: *mut C_tensor, - offsets_: *mut C_tensor, - axis_: i64, - initial_: *mut C_scalar, - ); - pub fn atg__shape_as_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__slow_conv2d_backward( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_weight_: *mut C_tensor, - grad_bias_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg__sobol_engine_draw( - out__: *mut *mut C_tensor, - quasi_: *mut C_tensor, - n_: i64, - sobolstate_: *mut C_tensor, - dimension_: i64, - num_generated_: i64, - dtype_: c_int, - ); - pub fn atg__sobol_engine_ff_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - n_: i64, - sobolstate_: *mut C_tensor, - dimension_: i64, - num_generated_: i64, - ); - pub fn atg__sobol_engine_initialize_state_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dimension_: i64, - ); - pub fn atg__sobol_engine_scramble_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - ltm_: *mut C_tensor, - dimension_: i64, - ); - pub fn atg__softmax( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - half_to_float_: c_int, - ); - pub fn atg__softmax_backward_data( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - output_: *mut C_tensor, - dim_: i64, - input_dtype_: c_int, - ); - pub fn atg__softmax_backward_data_out( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - output_: *mut C_tensor, - dim_: i64, - input_dtype_: c_int, - ); - pub fn atg__softmax_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - half_to_float_: c_int, - ); - pub fn atg__sparse_addmm( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mat1_: *mut C_tensor, - mat2_: *mut C_tensor, - ); - pub fn atg__sparse_addmm_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mat1_: *mut C_tensor, - mat2_: *mut C_tensor, - ); - pub fn atg__sparse_broadcast_to( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg__sparse_broadcast_to_copy( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg__sparse_broadcast_to_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg__sparse_bsc_tensor_unsafe( - out__: *mut *mut C_tensor, - ccol_indices_: *mut C_tensor, - row_indices_: *mut C_tensor, - values_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg__sparse_bsr_tensor_unsafe( - out__: *mut *mut C_tensor, - crow_indices_: *mut C_tensor, - col_indices_: *mut C_tensor, - values_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg__sparse_compressed_tensor_unsafe( - out__: *mut *mut C_tensor, - compressed_indices_: *mut C_tensor, - plain_indices_: *mut C_tensor, - values_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg__sparse_compressed_tensor_with_dims( - out__: *mut *mut C_tensor, - nnz_: i64, - dense_dim_: i64, - size_data: *const i64, - size_len: c_int, - blocksize_data: *const i64, - blocksize_len: c_int, - index_dtype_: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg__sparse_coo_tensor_unsafe( - out__: *mut *mut C_tensor, - indices_: *mut C_tensor, - values_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - is_coalesced_: c_int, - ); - pub fn atg__sparse_coo_tensor_with_dims( - out__: *mut *mut C_tensor, - sparse_dim_: i64, - dense_dim_: i64, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg__sparse_coo_tensor_with_dims_and_tensors( - out__: *mut *mut C_tensor, - sparse_dim_: i64, - dense_dim_: i64, - size_data: *const i64, - size_len: c_int, - indices_: *mut C_tensor, - values_: *mut C_tensor, - options_kind: c_int, - options_device: c_int, - is_coalesced_: c_int, - ); - pub fn atg__sparse_coo_tensor_with_dims_and_tensors_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - sparse_dim_: i64, - dense_dim_: i64, - size_data: *const i64, - size_len: c_int, - indices_: *mut C_tensor, - values_: *mut C_tensor, - is_coalesced_: c_int, - ); - pub fn atg__sparse_coo_tensor_with_dims_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - sparse_dim_: i64, - dense_dim_: i64, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg__sparse_csc_tensor_unsafe( - out__: *mut *mut C_tensor, - ccol_indices_: *mut C_tensor, - row_indices_: *mut C_tensor, - values_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg__sparse_csr_prod( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - dtype_: c_int, - ); - pub fn atg__sparse_csr_prod_dim_dtype_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - dtype_: c_int, - ); - pub fn atg__sparse_csr_sum( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - dtype_: c_int, - ); - pub fn atg__sparse_csr_sum_dim_dtype_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - dtype_: c_int, - ); - pub fn atg__sparse_csr_tensor_unsafe( - out__: *mut *mut C_tensor, - crow_indices_: *mut C_tensor, - col_indices_: *mut C_tensor, - values_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg__sparse_log_softmax( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - half_to_float_: c_int, - ); - pub fn atg__sparse_log_softmax_backward_data( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - output_: *mut C_tensor, - dim_: i64, - self_: *mut C_tensor, - ); - pub fn atg__sparse_log_softmax_backward_data_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_output_: *mut C_tensor, - output_: *mut C_tensor, - dim_: i64, - self_: *mut C_tensor, - ); - pub fn atg__sparse_log_softmax_int( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - dtype_: c_int, - ); - pub fn atg__sparse_log_softmax_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - half_to_float_: c_int, - ); - pub fn atg__sparse_mask_projection( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mask_: *mut C_tensor, - accumulate_matches_: c_int, - ); - pub fn atg__sparse_mask_projection_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mask_: *mut C_tensor, - accumulate_matches_: c_int, - ); - pub fn atg__sparse_mm(out__: *mut *mut C_tensor, sparse_: *mut C_tensor, dense_: *mut C_tensor); - pub fn atg__sparse_mm_reduce( - out__: *mut *mut C_tensor, - sparse_: *mut C_tensor, - dense_: *mut C_tensor, - reduce_ptr: *const u8, - reduce_len: c_int, - ); - pub fn atg__sparse_mm_reduce_impl( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - reduce_ptr: *const u8, - reduce_len: c_int, - ); - pub fn atg__sparse_semi_structured_apply( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - thread_masks_: *mut C_tensor, - ); - pub fn atg__sparse_semi_structured_apply_dense( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - thread_masks_: *mut C_tensor, - ); - pub fn atg__sparse_semi_structured_linear( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - meta_: *mut C_tensor, - bias_: *mut C_tensor, - activation_ptr: *const u8, - activation_len: c_int, - out_dtype_: c_int, - ); - pub fn atg__sparse_semi_structured_mm( - out__: *mut *mut C_tensor, - mat1_: *mut C_tensor, - mat1_meta_: *mut C_tensor, - mat2_: *mut C_tensor, - out_dtype_: c_int, - ); - pub fn atg__sparse_semi_structured_tile( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - algorithm_ptr: *const u8, - algorithm_len: c_int, - use_cutlass_: c_int, - ); - pub fn atg__sparse_softmax( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - half_to_float_: c_int, - ); - pub fn atg__sparse_softmax_backward_data( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - output_: *mut C_tensor, - dim_: i64, - self_: *mut C_tensor, - ); - pub fn atg__sparse_softmax_backward_data_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_output_: *mut C_tensor, - output_: *mut C_tensor, - dim_: i64, - self_: *mut C_tensor, - ); - pub fn atg__sparse_softmax_int( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - dtype_: c_int, - ); - pub fn atg__sparse_softmax_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - half_to_float_: c_int, - ); - pub fn atg__sparse_sparse_matmul( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg__sparse_sparse_matmul_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg__sparse_sum(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__sparse_sum_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - ); - pub fn atg__sparse_sum_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - ); - pub fn atg__sparse_sum_dim( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - ); - pub fn atg__sparse_sum_dim_dtype( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - dtype_: c_int, - ); - pub fn atg__sparse_sum_dim_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - ); - pub fn atg__sparse_sum_dtype(out__: *mut *mut C_tensor, self_: *mut C_tensor, dtype_: c_int); - pub fn atg__spdiags( - out__: *mut *mut C_tensor, - diagonals_: *mut C_tensor, - offsets_: *mut C_tensor, - shape_data: *const i64, - shape_len: c_int, - layout_: i8, - ); - pub fn atg__spdiags_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - diagonals_: *mut C_tensor, - offsets_: *mut C_tensor, - shape_data: *const i64, - shape_len: c_int, - layout_: i8, - ); - pub fn atg__spsolve( - out__: *mut *mut C_tensor, - A_: *mut C_tensor, - B_: *mut C_tensor, - left_: c_int, - ); - pub fn atg__stack( - out__: *mut *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - dim_: i64, - ); - pub fn atg__stack_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - dim_: i64, - ); - pub fn atg__standard_gamma(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__standard_gamma_grad( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_: *mut C_tensor, - ); - pub fn atg__standard_gamma_grad_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - output_: *mut C_tensor, - ); - pub fn atg__standard_gamma_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg__test_ambiguous_defaults( - out__: *mut *mut C_tensor, - dummy_: *mut C_tensor, - a_: i64, - b_: i64, - ); - pub fn atg__test_ambiguous_defaults_b( - out__: *mut *mut C_tensor, - dummy_: *mut C_tensor, - a_: i64, - b_ptr: *const u8, - b_len: c_int, - ); - pub fn atg__test_autograd_multiple_dispatch(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__test_autograd_multiple_dispatch_fullcoverage_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg__test_autograd_multiple_dispatch_ntonly( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - b_: c_int, - ); - pub fn atg__test_autograd_multiple_dispatch_view( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg__test_autograd_multiple_dispatch_view_copy( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg__test_autograd_multiple_dispatch_view_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg__test_check_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__test_functorch_fallback( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg__test_functorch_fallback_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg__test_optional_filled_intlist( - out__: *mut *mut C_tensor, - values_: *mut C_tensor, - addends_data: *const i64, - addends_len: c_int, - ); - pub fn atg__test_optional_filled_intlist_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - values_: *mut C_tensor, - addends_data: *const i64, - addends_len: c_int, - ); - pub fn atg__test_optional_floatlist( - out__: *mut *mut C_tensor, - values_: *mut C_tensor, - addends_data: *const f64, - addends_len: c_int, - ); - pub fn atg__test_optional_floatlist_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - values_: *mut C_tensor, - addends_data: *const f64, - addends_len: c_int, - ); - pub fn atg__test_optional_intlist( - out__: *mut *mut C_tensor, - values_: *mut C_tensor, - addends_data: *const i64, - addends_len: c_int, - ); - pub fn atg__test_optional_intlist_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - values_: *mut C_tensor, - addends_data: *const i64, - addends_len: c_int, - ); - pub fn atg__test_parallel_materialize( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - num_parallel_: i64, - skip_first_: c_int, - ); - pub fn atg__test_serialization_subcmul( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg__test_string_default( - out__: *mut *mut C_tensor, - dummy_: *mut C_tensor, - a_ptr: *const u8, - a_len: c_int, - b_ptr: *const u8, - b_len: c_int, - ); - pub fn atg__test_warn_in_autograd(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__test_warn_in_autograd_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg__to_copy( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - options_kind: c_int, - options_device: c_int, - non_blocking_: c_int, - ); - pub fn atg__to_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - non_blocking_: c_int, - ); - pub fn atg__to_cpu( - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ) -> *mut *mut C_tensor; - pub fn atg__to_dense( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dtype_: c_int, - masked_grad_: c_int, - ); - pub fn atg__to_dense_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dtype_: c_int, - masked_grad_: c_int, - ); - pub fn atg__to_sparse( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - layout_: i8, - blocksize_data: *const i64, - blocksize_len: c_int, - dense_dim_v: i64, - dense_dim_null: i8, - ); - pub fn atg__to_sparse_bsc( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - blocksize_data: *const i64, - blocksize_len: c_int, - dense_dim_v: i64, - dense_dim_null: i8, - ); - pub fn atg__to_sparse_bsc_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - blocksize_data: *const i64, - blocksize_len: c_int, - dense_dim_v: i64, - dense_dim_null: i8, - ); - pub fn atg__to_sparse_bsr( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - blocksize_data: *const i64, - blocksize_len: c_int, - dense_dim_v: i64, - dense_dim_null: i8, - ); - pub fn atg__to_sparse_bsr_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - blocksize_data: *const i64, - blocksize_len: c_int, - dense_dim_v: i64, - dense_dim_null: i8, - ); - pub fn atg__to_sparse_csc( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dense_dim_v: i64, - dense_dim_null: i8, - ); - pub fn atg__to_sparse_csc_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dense_dim_v: i64, - dense_dim_null: i8, - ); - pub fn atg__to_sparse_csr( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dense_dim_v: i64, - dense_dim_null: i8, - ); - pub fn atg__to_sparse_csr_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dense_dim_v: i64, - dense_dim_null: i8, - ); - pub fn atg__to_sparse_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - layout_: i8, - blocksize_data: *const i64, - blocksize_len: c_int, - dense_dim_v: i64, - dense_dim_null: i8, - ); - pub fn atg__to_sparse_semi_structured(out__: *mut *mut C_tensor, dense_: *mut C_tensor); - pub fn atg__to_sparse_sparse_dim( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - sparse_dim_: i64, - ); - pub fn atg__to_sparse_sparse_dim_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - sparse_dim_: i64, - ); - pub fn atg__transform_bias_rescale_qkv( - out__: *mut *mut C_tensor, - qkv_: *mut C_tensor, - qkv_bias_: *mut C_tensor, - num_heads_: i64, - ); - pub fn atg__transform_bias_rescale_qkv_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - out2_: *mut C_tensor, - qkv_: *mut C_tensor, - qkv_bias_: *mut C_tensor, - num_heads_: i64, - ); - pub fn atg__transformer_encoder_layer_fwd( - out__: *mut *mut C_tensor, - src_: *mut C_tensor, - embed_dim_: i64, - num_heads_: i64, - qkv_weight_: *mut C_tensor, - qkv_bias_: *mut C_tensor, - proj_weight_: *mut C_tensor, - proj_bias_: *mut C_tensor, - use_gelu_: c_int, - norm_first_: c_int, - eps_: f64, - norm_weight_1_: *mut C_tensor, - norm_bias_1_: *mut C_tensor, - norm_weight_2_: *mut C_tensor, - norm_bias_2_: *mut C_tensor, - ffn_weight_1_: *mut C_tensor, - ffn_bias_1_: *mut C_tensor, - ffn_weight_2_: *mut C_tensor, - ffn_bias_2_: *mut C_tensor, - mask_: *mut C_tensor, - mask_type_v: i64, - mask_type_null: i8, - ); - pub fn atg__transformer_encoder_layer_fwd_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - src_: *mut C_tensor, - embed_dim_: i64, - num_heads_: i64, - qkv_weight_: *mut C_tensor, - qkv_bias_: *mut C_tensor, - proj_weight_: *mut C_tensor, - proj_bias_: *mut C_tensor, - use_gelu_: c_int, - norm_first_: c_int, - eps_: f64, - norm_weight_1_: *mut C_tensor, - norm_bias_1_: *mut C_tensor, - norm_weight_2_: *mut C_tensor, - norm_bias_2_: *mut C_tensor, - ffn_weight_1_: *mut C_tensor, - ffn_bias_1_: *mut C_tensor, - ffn_weight_2_: *mut C_tensor, - ffn_bias_2_: *mut C_tensor, - mask_: *mut C_tensor, - mask_type_v: i64, - mask_type_null: i8, - ); - pub fn atg__trilinear( - out__: *mut *mut C_tensor, - i1_: *mut C_tensor, - i2_: *mut C_tensor, - i3_: *mut C_tensor, - expand1_data: *const i64, - expand1_len: c_int, - expand2_data: *const i64, - expand2_len: c_int, - expand3_data: *const i64, - expand3_len: c_int, - sumdim_data: *const i64, - sumdim_len: c_int, - unroll_dim_: i64, - ); - pub fn atg__trilinear_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - i1_: *mut C_tensor, - i2_: *mut C_tensor, - i3_: *mut C_tensor, - expand1_data: *const i64, - expand1_len: c_int, - expand2_data: *const i64, - expand2_len: c_int, - expand3_data: *const i64, - expand3_len: c_int, - sumdim_data: *const i64, - sumdim_len: c_int, - unroll_dim_: i64, - ); - pub fn atg__triton_multi_head_attention( - out__: *mut *mut C_tensor, - query_: *mut C_tensor, - key_: *mut C_tensor, - value_: *mut C_tensor, - embed_dim_: i64, - num_head_: i64, - qkv_weight_: *mut C_tensor, - qkv_bias_: *mut C_tensor, - proj_weight_: *mut C_tensor, - proj_bias_: *mut C_tensor, - mask_: *mut C_tensor, - ); - pub fn atg__triton_multi_head_attention_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - query_: *mut C_tensor, - key_: *mut C_tensor, - value_: *mut C_tensor, - embed_dim_: i64, - num_head_: i64, - qkv_weight_: *mut C_tensor, - qkv_bias_: *mut C_tensor, - proj_weight_: *mut C_tensor, - proj_bias_: *mut C_tensor, - mask_: *mut C_tensor, - ); - pub fn atg__triton_scaled_dot_attention( - out__: *mut *mut C_tensor, - q_: *mut C_tensor, - k_: *mut C_tensor, - v_: *mut C_tensor, - dropout_p_: f64, - ); - pub fn atg__triton_scaled_dot_attention_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - q_: *mut C_tensor, - k_: *mut C_tensor, - v_: *mut C_tensor, - dropout_p_: f64, - ); - pub fn atg__unique( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - sorted_: c_int, - return_inverse_: c_int, - ); - pub fn atg__unique2( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - sorted_: c_int, - return_inverse_: c_int, - return_counts_: c_int, - ); - pub fn atg__unique2_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - out2_: *mut C_tensor, - self_: *mut C_tensor, - sorted_: c_int, - return_inverse_: c_int, - return_counts_: c_int, - ); - pub fn atg__unique_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - self_: *mut C_tensor, - sorted_: c_int, - return_inverse_: c_int, - ); - pub fn atg__unpack_dual(out__: *mut *mut C_tensor, dual_: *mut C_tensor, level_: i64); - pub fn atg__unsafe_index( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - indices_data: *const *mut C_tensor, - indices_len: c_int, - ); - pub fn atg__unsafe_index_put( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - indices_data: *const *mut C_tensor, - indices_len: c_int, - values_: *mut C_tensor, - accumulate_: c_int, - ); - pub fn atg__unsafe_masked_index( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mask_: *mut C_tensor, - indices_data: *const *mut C_tensor, - indices_len: c_int, - fill_: *mut C_scalar, - ); - pub fn atg__unsafe_masked_index_put_accumulate( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mask_: *mut C_tensor, - indices_data: *const *mut C_tensor, - indices_len: c_int, - values_: *mut C_tensor, - ); - pub fn atg__unsafe_view( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg__unsafe_view_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg__upsample_bicubic2d_aa( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - align_corners_: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg__upsample_bicubic2d_aa_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - align_corners_: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg__upsample_bicubic2d_aa_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - align_corners_: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg__upsample_bicubic2d_aa_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - align_corners_: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg__upsample_bicubic2d_aa_vec( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - align_corners_: c_int, - scale_factors_data: *const f64, - scale_factors_len: c_int, - ); - pub fn atg__upsample_bilinear2d_aa( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - align_corners_: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg__upsample_bilinear2d_aa_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - align_corners_: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg__upsample_bilinear2d_aa_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - align_corners_: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg__upsample_bilinear2d_aa_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - align_corners_: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg__upsample_bilinear2d_aa_vec( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - align_corners_: c_int, - scale_factors_data: *const f64, - scale_factors_len: c_int, - ); - pub fn atg__upsample_nearest_exact1d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - scales_v: f64, - scales_null: i8, - ); - pub fn atg__upsample_nearest_exact1d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - scales_v: f64, - scales_null: i8, - ); - pub fn atg__upsample_nearest_exact1d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - scales_v: f64, - scales_null: i8, - ); - pub fn atg__upsample_nearest_exact1d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - scales_v: f64, - scales_null: i8, - ); - pub fn atg__upsample_nearest_exact1d_vec( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - scale_factors_data: *const f64, - scale_factors_len: c_int, - ); - pub fn atg__upsample_nearest_exact2d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg__upsample_nearest_exact2d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg__upsample_nearest_exact2d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg__upsample_nearest_exact2d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg__upsample_nearest_exact2d_vec( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - scale_factors_data: *const f64, - scale_factors_len: c_int, - ); - pub fn atg__upsample_nearest_exact3d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - scales_d_v: f64, - scales_d_null: i8, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg__upsample_nearest_exact3d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - scales_d_v: f64, - scales_d_null: i8, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg__upsample_nearest_exact3d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - scales_d_v: f64, - scales_d_null: i8, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg__upsample_nearest_exact3d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - scales_d_v: f64, - scales_d_null: i8, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg__upsample_nearest_exact3d_vec( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - scale_factors_data: *const f64, - scale_factors_len: c_int, - ); - pub fn atg__use_cudnn_ctc_loss( - log_probs_: *mut C_tensor, - targets_: *mut C_tensor, - input_lengths_data: *const i64, - input_lengths_len: c_int, - target_lengths_data: *const i64, - target_lengths_len: c_int, - blank_: i64, - ) -> c_int; - pub fn atg__use_cudnn_ctc_loss_tensor( - log_probs_: *mut C_tensor, - targets_: *mut C_tensor, - input_lengths_: *mut C_tensor, - target_lengths_: *mut C_tensor, - blank_: i64, - ) -> c_int; - pub fn atg__use_cudnn_rnn_flatten_weight() -> c_int; - pub fn atg__use_miopen_ctc_loss( - log_probs_: *mut C_tensor, - targets_: *mut C_tensor, - input_lengths_data: *const i64, - input_lengths_len: c_int, - target_lengths_data: *const i64, - target_lengths_len: c_int, - blank_: i64, - ) -> c_int; - pub fn atg__use_miopen_ctc_loss_tensor( - log_probs_: *mut C_tensor, - targets_: *mut C_tensor, - input_lengths_: *mut C_tensor, - target_lengths_: *mut C_tensor, - blank_: i64, - ) -> c_int; - pub fn atg__validate_compressed_sparse_indices( - is_crow_: c_int, - compressed_idx_: *mut C_tensor, - plain_idx_: *mut C_tensor, - cdim_: i64, - dim_: i64, - nnz_: i64, - ); - pub fn atg__validate_sparse_bsc_tensor_args( - ccol_indices_: *mut C_tensor, - row_indices_: *mut C_tensor, - values_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - check_pinning_: c_int, - ); - pub fn atg__validate_sparse_bsr_tensor_args( - crow_indices_: *mut C_tensor, - col_indices_: *mut C_tensor, - values_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - check_pinning_: c_int, - ); - pub fn atg__validate_sparse_compressed_tensor_args( - compressed_indices_: *mut C_tensor, - plain_indices_: *mut C_tensor, - values_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - layout_: i8, - check_pinning_: c_int, - ); - pub fn atg__validate_sparse_csc_tensor_args( - ccol_indices_: *mut C_tensor, - row_indices_: *mut C_tensor, - values_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - check_pinning_: c_int, - ); - pub fn atg__validate_sparse_csr_tensor_args( - crow_indices_: *mut C_tensor, - col_indices_: *mut C_tensor, - values_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - check_pinning_: c_int, - ); - pub fn atg__values(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__values_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg__values_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg__version(self_: *mut C_tensor) -> i64; - pub fn atg__weight_int4pack_mm( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mat2_: *mut C_tensor, - qGroupSize_: i64, - qScaleAndZeros_: *mut C_tensor, - ); - pub fn atg__weight_int4pack_mm_for_cpu( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mat2_: *mut C_tensor, - qGroupSize_: i64, - qScaleAndZeros_: *mut C_tensor, - ); - pub fn atg__weight_int4pack_mm_with_scales_and_zeros( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mat2_: *mut C_tensor, - qGroupSize_: i64, - qScale_: *mut C_tensor, - qZeros_: *mut C_tensor, - ); - pub fn atg__weight_int8pack_mm( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mat2_: *mut C_tensor, - scales_: *mut C_tensor, - ); - pub fn atg__weight_norm( - out__: *mut *mut C_tensor, - v_: *mut C_tensor, - g_: *mut C_tensor, - dim_: i64, - ); - pub fn atg__weight_norm_differentiable_backward( - out__: *mut *mut C_tensor, - grad_w_: *mut C_tensor, - saved_v_: *mut C_tensor, - saved_g_: *mut C_tensor, - saved_norms_: *mut C_tensor, - dim_: i64, - ); - pub fn atg__weight_norm_interface( - out__: *mut *mut C_tensor, - v_: *mut C_tensor, - g_: *mut C_tensor, - dim_: i64, - ); - pub fn atg__weight_norm_interface_backward( - out__: *mut *mut C_tensor, - grad_w_: *mut C_tensor, - saved_v_: *mut C_tensor, - saved_g_: *mut C_tensor, - saved_norms_: *mut C_tensor, - dim_: i64, - ); - pub fn atg__weight_norm_interface_backward_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - grad_w_: *mut C_tensor, - saved_v_: *mut C_tensor, - saved_g_: *mut C_tensor, - saved_norms_: *mut C_tensor, - dim_: i64, - ); - pub fn atg__weight_norm_interface_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - v_: *mut C_tensor, - g_: *mut C_tensor, - dim_: i64, - ); - pub fn atg__wrapped_linear_prepack( - out__: *mut *mut C_tensor, - weight_: *mut C_tensor, - weight_scale_: *mut C_tensor, - weight_zero_point_: *mut C_tensor, - bias_: *mut C_tensor, - ); - pub fn atg__wrapped_quantized_linear_prepacked( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - input_scale_: *mut C_tensor, - input_zero_point_: *mut C_tensor, - packed_weight_: *mut C_tensor, - output_scale_: *mut C_tensor, - output_zero_point_: *mut C_tensor, - out_channel_: i64, - ); - pub fn atg_abs(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_abs_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_abs_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_absolute(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_absolute_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_absolute_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_acos(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_acos_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_acos_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_acosh(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_acosh_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_acosh_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_adaptive_avg_pool1d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - ); - pub fn atg_adaptive_avg_pool1d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - ); - pub fn atg_adaptive_avg_pool2d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - ); - pub fn atg_adaptive_avg_pool2d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - ); - pub fn atg_adaptive_avg_pool3d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - ); - pub fn atg_adaptive_avg_pool3d_backward( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_adaptive_avg_pool3d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - ); - pub fn atg_adaptive_max_pool1d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - ); - pub fn atg_adaptive_max_pool2d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - ); - pub fn atg_adaptive_max_pool2d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - indices_: *mut C_tensor, - ); - pub fn atg_adaptive_max_pool2d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - indices_: *mut C_tensor, - ); - pub fn atg_adaptive_max_pool2d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - indices_: *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - ); - pub fn atg_adaptive_max_pool3d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - ); - pub fn atg_adaptive_max_pool3d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - indices_: *mut C_tensor, - ); - pub fn atg_adaptive_max_pool3d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - indices_: *mut C_tensor, - ); - pub fn atg_adaptive_max_pool3d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - indices_: *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - ); - pub fn atg_add(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_add_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_add_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_add_scalar(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_add_scalar_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_add_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_addbmm( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - batch1_: *mut C_tensor, - batch2_: *mut C_tensor, - ); - pub fn atg_addbmm_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - batch1_: *mut C_tensor, - batch2_: *mut C_tensor, - ); - pub fn atg_addbmm_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - batch1_: *mut C_tensor, - batch2_: *mut C_tensor, - ); - pub fn atg_addcdiv( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - tensor1_: *mut C_tensor, - tensor2_: *mut C_tensor, - ); - pub fn atg_addcdiv_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - tensor1_: *mut C_tensor, - tensor2_: *mut C_tensor, - ); - pub fn atg_addcdiv_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - tensor1_: *mut C_tensor, - tensor2_: *mut C_tensor, - ); - pub fn atg_addcmul( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - tensor1_: *mut C_tensor, - tensor2_: *mut C_tensor, - ); - pub fn atg_addcmul_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - tensor1_: *mut C_tensor, - tensor2_: *mut C_tensor, - ); - pub fn atg_addcmul_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - tensor1_: *mut C_tensor, - tensor2_: *mut C_tensor, - ); - pub fn atg_addmm( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mat1_: *mut C_tensor, - mat2_: *mut C_tensor, - ); - pub fn atg_addmm_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mat1_: *mut C_tensor, - mat2_: *mut C_tensor, - ); - pub fn atg_addmm_dtype( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mat1_: *mut C_tensor, - mat2_: *mut C_tensor, - out_dtype_: c_int, - ); - pub fn atg_addmm_dtype_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mat1_: *mut C_tensor, - mat2_: *mut C_tensor, - out_dtype_: c_int, - ); - pub fn atg_addmm_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mat1_: *mut C_tensor, - mat2_: *mut C_tensor, - ); - pub fn atg_addmv( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mat_: *mut C_tensor, - vec_: *mut C_tensor, - ); - pub fn atg_addmv_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mat_: *mut C_tensor, - vec_: *mut C_tensor, - ); - pub fn atg_addmv_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mat_: *mut C_tensor, - vec_: *mut C_tensor, - ); - pub fn atg_addr( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - vec1_: *mut C_tensor, - vec2_: *mut C_tensor, - ); - pub fn atg_addr_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - vec1_: *mut C_tensor, - vec2_: *mut C_tensor, - ); - pub fn atg_addr_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - vec1_: *mut C_tensor, - vec2_: *mut C_tensor, - ); - pub fn atg_adjoint(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_affine_grid_generator( - out__: *mut *mut C_tensor, - theta_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - align_corners_: c_int, - ); - pub fn atg_affine_grid_generator_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - align_corners_: c_int, - ); - pub fn atg_affine_grid_generator_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - theta_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - align_corners_: c_int, - ); - pub fn atg_alias(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_alias_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_alias_copy_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_align_as(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_align_tensors( - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ) -> *mut *mut C_tensor; - pub fn atg_all(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_all_all_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_all_dim(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, keepdim_: c_int); - pub fn atg_all_dims( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - ); - pub fn atg_all_dims_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - ); - pub fn atg_all_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - keepdim_: c_int, - ); - pub fn atg_allclose( - self_: *mut C_tensor, - other_: *mut C_tensor, - rtol_: f64, - atol_: f64, - equal_nan_: c_int, - ) -> c_int; - pub fn atg_alpha_dropout( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - p_: f64, - train_: c_int, - ); - pub fn atg_alpha_dropout_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - p_: f64, - train_: c_int, - ); - pub fn atg_amax( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - ); - pub fn atg_amax_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - ); - pub fn atg_amin( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - ); - pub fn atg_amin_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - ); - pub fn atg_aminmax( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_v: i64, - dim_null: i8, - keepdim_: c_int, - ); - pub fn atg_aminmax_out( - out__: *mut *mut C_tensor, - min_: *mut C_tensor, - max_: *mut C_tensor, - self_: *mut C_tensor, - dim_v: i64, - dim_null: i8, - keepdim_: c_int, - ); - pub fn atg_angle(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_angle_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_any(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_any_all_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_any_dim(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, keepdim_: c_int); - pub fn atg_any_dims( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - ); - pub fn atg_any_dims_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - ); - pub fn atg_any_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - keepdim_: c_int, - ); - pub fn atg_arange( - out__: *mut *mut C_tensor, - end_: *mut C_scalar, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_arange_start( - out__: *mut *mut C_tensor, - start_: *mut C_scalar, - end_: *mut C_scalar, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_arange_start_step( - out__: *mut *mut C_tensor, - start_: *mut C_scalar, - end_: *mut C_scalar, - step_: *mut C_scalar, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_arccos(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_arccos_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_arccos_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_arccosh(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_arccosh_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_arccosh_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_arcsin(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_arcsin_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_arcsin_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_arcsinh(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_arcsinh_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_arcsinh_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_arctan(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_arctan2(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_arctan2_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_arctan2_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_arctan_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_arctan_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_arctanh(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_arctanh_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_arctanh_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_argmax( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_v: i64, - dim_null: i8, - keepdim_: c_int, - ); - pub fn atg_argmax_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_v: i64, - dim_null: i8, - keepdim_: c_int, - ); - pub fn atg_argmin( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_v: i64, - dim_null: i8, - keepdim_: c_int, - ); - pub fn atg_argmin_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_v: i64, - dim_null: i8, - keepdim_: c_int, - ); - pub fn atg_argsort( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - descending_: c_int, - ); - pub fn atg_argsort_stable( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - stable_: c_int, - dim_: i64, - descending_: c_int, - ); - pub fn atg_argsort_stable_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - stable_: c_int, - dim_: i64, - descending_: c_int, - ); - pub fn atg_argwhere(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_as_strided( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - storage_offset_v: i64, - storage_offset_null: i8, - ); - pub fn atg_as_strided_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - storage_offset_v: i64, - storage_offset_null: i8, - ); - pub fn atg_as_strided_copy( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - storage_offset_v: i64, - storage_offset_null: i8, - ); - pub fn atg_as_strided_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - storage_offset_v: i64, - storage_offset_null: i8, - ); - pub fn atg_as_strided_scatter( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - src_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - storage_offset_v: i64, - storage_offset_null: i8, - ); - pub fn atg_as_strided_scatter_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - src_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - storage_offset_v: i64, - storage_offset_null: i8, - ); - pub fn atg_asin(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_asin_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_asin_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_asinh(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_asinh_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_asinh_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_atan(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_atan2(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_atan2_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_atan2_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_atan_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_atan_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_atanh(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_atanh_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_atanh_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_atleast_1d(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_atleast_1d_sequence( - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ) -> *mut *mut C_tensor; - pub fn atg_atleast_2d(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_atleast_2d_sequence( - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ) -> *mut *mut C_tensor; - pub fn atg_atleast_3d(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_atleast_3d_sequence( - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ) -> *mut *mut C_tensor; - pub fn atg_avg_pool1d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - ceil_mode_: c_int, - count_include_pad_: c_int, - ); - pub fn atg_avg_pool1d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - ceil_mode_: c_int, - count_include_pad_: c_int, - ); - pub fn atg_avg_pool2d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - ceil_mode_: c_int, - count_include_pad_: c_int, - divisor_override_v: i64, - divisor_override_null: i8, - ); - pub fn atg_avg_pool2d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - ceil_mode_: c_int, - count_include_pad_: c_int, - divisor_override_v: i64, - divisor_override_null: i8, - ); - pub fn atg_avg_pool2d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - ceil_mode_: c_int, - count_include_pad_: c_int, - divisor_override_v: i64, - divisor_override_null: i8, - ); - pub fn atg_avg_pool2d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - ceil_mode_: c_int, - count_include_pad_: c_int, - divisor_override_v: i64, - divisor_override_null: i8, - ); - pub fn atg_avg_pool3d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - ceil_mode_: c_int, - count_include_pad_: c_int, - divisor_override_v: i64, - divisor_override_null: i8, - ); - pub fn atg_avg_pool3d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - ceil_mode_: c_int, - count_include_pad_: c_int, - divisor_override_v: i64, - divisor_override_null: i8, - ); - pub fn atg_avg_pool3d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - ceil_mode_: c_int, - count_include_pad_: c_int, - divisor_override_v: i64, - divisor_override_null: i8, - ); - pub fn atg_avg_pool3d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - ceil_mode_: c_int, - count_include_pad_: c_int, - divisor_override_v: i64, - divisor_override_null: i8, - ); - pub fn atg_baddbmm( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - batch1_: *mut C_tensor, - batch2_: *mut C_tensor, - beta_: *mut C_scalar, - alpha_: *mut C_scalar, - ); - pub fn atg_baddbmm_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - batch1_: *mut C_tensor, - batch2_: *mut C_tensor, - ); - pub fn atg_baddbmm_dtype( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - batch1_: *mut C_tensor, - batch2_: *mut C_tensor, - out_dtype_: c_int, - beta_: *mut C_scalar, - alpha_: *mut C_scalar, - ); - pub fn atg_baddbmm_dtype_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - batch1_: *mut C_tensor, - batch2_: *mut C_tensor, - out_dtype_: c_int, - ); - pub fn atg_baddbmm_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - batch1_: *mut C_tensor, - batch2_: *mut C_tensor, - ); - pub fn atg_bartlett_window( - out__: *mut *mut C_tensor, - window_length_: i64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_bartlett_window_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - window_length_: i64, - ); - pub fn atg_bartlett_window_periodic( - out__: *mut *mut C_tensor, - window_length_: i64, - periodic_: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_bartlett_window_periodic_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - window_length_: i64, - periodic_: c_int, - ); - pub fn atg_batch_norm( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - training_: c_int, - momentum_: f64, - eps_: f64, - cudnn_enabled_: c_int, - ); - pub fn atg_batch_norm_backward_elemt( - out__: *mut *mut C_tensor, - grad_out_: *mut C_tensor, - input_: *mut C_tensor, - mean_: *mut C_tensor, - invstd_: *mut C_tensor, - weight_: *mut C_tensor, - sum_dy_: *mut C_tensor, - sum_dy_xmu_: *mut C_tensor, - count_: *mut C_tensor, - ); - pub fn atg_batch_norm_backward_elemt_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_out_: *mut C_tensor, - input_: *mut C_tensor, - mean_: *mut C_tensor, - invstd_: *mut C_tensor, - weight_: *mut C_tensor, - sum_dy_: *mut C_tensor, - sum_dy_xmu_: *mut C_tensor, - count_: *mut C_tensor, - ); - pub fn atg_batch_norm_backward_reduce( - out__: *mut *mut C_tensor, - grad_out_: *mut C_tensor, - input_: *mut C_tensor, - mean_: *mut C_tensor, - invstd_: *mut C_tensor, - weight_: *mut C_tensor, - input_g_: c_int, - weight_g_: c_int, - bias_g_: c_int, - ); - pub fn atg_batch_norm_backward_reduce_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - out2_: *mut C_tensor, - out3_: *mut C_tensor, - grad_out_: *mut C_tensor, - input_: *mut C_tensor, - mean_: *mut C_tensor, - invstd_: *mut C_tensor, - weight_: *mut C_tensor, - input_g_: c_int, - weight_g_: c_int, - bias_g_: c_int, - ); - pub fn atg_batch_norm_elemt( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - mean_: *mut C_tensor, - invstd_: *mut C_tensor, - eps_: f64, - ); - pub fn atg_batch_norm_elemt_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - mean_: *mut C_tensor, - invstd_: *mut C_tensor, - eps_: f64, - ); - pub fn atg_batch_norm_gather_stats( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - mean_: *mut C_tensor, - invstd_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - momentum_: f64, - eps_: f64, - count_: i64, - ); - pub fn atg_batch_norm_gather_stats_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - input_: *mut C_tensor, - mean_: *mut C_tensor, - invstd_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - momentum_: f64, - eps_: f64, - count_: i64, - ); - pub fn atg_batch_norm_gather_stats_with_counts( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - mean_: *mut C_tensor, - invstd_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - momentum_: f64, - eps_: f64, - counts_: *mut C_tensor, - ); - pub fn atg_batch_norm_gather_stats_with_counts_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - input_: *mut C_tensor, - mean_: *mut C_tensor, - invstd_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - momentum_: f64, - eps_: f64, - counts_: *mut C_tensor, - ); - pub fn atg_batch_norm_stats(out__: *mut *mut C_tensor, input_: *mut C_tensor, eps_: f64); - pub fn atg_batch_norm_stats_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - input_: *mut C_tensor, - eps_: f64, - ); - pub fn atg_batch_norm_update_stats( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - momentum_: f64, - ); - pub fn atg_batch_norm_update_stats_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - input_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - momentum_: f64, - ); - pub fn atg_bernoulli(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_bernoulli_(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: *mut C_tensor); - pub fn atg_bernoulli_float_(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: f64); - pub fn atg_bernoulli_p(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: f64); - pub fn atg_bernoulli_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: *mut C_tensor); - pub fn atg_bilinear( - out__: *mut *mut C_tensor, - input1_: *mut C_tensor, - input2_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - ); - pub fn atg_binary_cross_entropy( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - weight_: *mut C_tensor, - reduction_: i64, - ); - pub fn atg_binary_cross_entropy_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - weight_: *mut C_tensor, - reduction_: i64, - ); - pub fn atg_binary_cross_entropy_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - weight_: *mut C_tensor, - reduction_: i64, - ); - pub fn atg_binary_cross_entropy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - weight_: *mut C_tensor, - reduction_: i64, - ); - pub fn atg_binary_cross_entropy_with_logits( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - weight_: *mut C_tensor, - pos_weight_: *mut C_tensor, - reduction_: i64, - ); - pub fn atg_binary_cross_entropy_with_logits_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - weight_: *mut C_tensor, - pos_weight_: *mut C_tensor, - reduction_: i64, - ); - pub fn atg_bincount( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weights_: *mut C_tensor, - minlength_: i64, - ); - pub fn atg_bincount_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - weights_: *mut C_tensor, - minlength_: i64, - ); - pub fn atg_binomial(out__: *mut *mut C_tensor, count_: *mut C_tensor, prob_: *mut C_tensor); - pub fn atg_binomial_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - count_: *mut C_tensor, - prob_: *mut C_tensor, - ); - pub fn atg_bitwise_and(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_bitwise_and_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_bitwise_and_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_bitwise_and_scalar_tensor( - out__: *mut *mut C_tensor, - self_scalar_: *mut C_scalar, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_and_scalar_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_scalar_: *mut C_scalar, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_and_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_and_tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_and_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_left_shift( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_left_shift_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_left_shift_scalar_tensor( - out__: *mut *mut C_tensor, - self_scalar_: *mut C_scalar, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_left_shift_scalar_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_scalar_: *mut C_scalar, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_left_shift_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_left_shift_tensor_scalar( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_bitwise_left_shift_tensor_scalar_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_bitwise_left_shift_tensor_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_bitwise_not(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_bitwise_not_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_bitwise_not_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_bitwise_or(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_bitwise_or_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_bitwise_or_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_bitwise_or_scalar_tensor( - out__: *mut *mut C_tensor, - self_scalar_: *mut C_scalar, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_or_scalar_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_scalar_: *mut C_scalar, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_or_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_or_tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_or_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_right_shift( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_right_shift_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_right_shift_scalar_tensor( - out__: *mut *mut C_tensor, - self_scalar_: *mut C_scalar, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_right_shift_scalar_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_scalar_: *mut C_scalar, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_right_shift_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_right_shift_tensor_scalar( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_bitwise_right_shift_tensor_scalar_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_bitwise_right_shift_tensor_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_bitwise_xor(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_bitwise_xor_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_bitwise_xor_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_bitwise_xor_scalar_tensor( - out__: *mut *mut C_tensor, - self_scalar_: *mut C_scalar, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_xor_scalar_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_scalar_: *mut C_scalar, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_xor_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_xor_tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_bitwise_xor_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_blackman_window( - out__: *mut *mut C_tensor, - window_length_: i64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_blackman_window_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - window_length_: i64, - ); - pub fn atg_blackman_window_periodic( - out__: *mut *mut C_tensor, - window_length_: i64, - periodic_: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_blackman_window_periodic_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - window_length_: i64, - periodic_: c_int, - ); - pub fn atg_block_diag( - out__: *mut *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ); - pub fn atg_block_diag_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ); - pub fn atg_bmm(out__: *mut *mut C_tensor, self_: *mut C_tensor, mat2_: *mut C_tensor); - pub fn atg_bmm_dtype( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mat2_: *mut C_tensor, - out_dtype_: c_int, - ); - pub fn atg_bmm_dtype_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mat2_: *mut C_tensor, - out_dtype_: c_int, - ); - pub fn atg_bmm_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mat2_: *mut C_tensor, - ); - pub fn atg_broadcast_tensors( - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ) -> *mut *mut C_tensor; - pub fn atg_broadcast_to( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg_bucketize( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - boundaries_: *mut C_tensor, - out_int32_: c_int, - right_: c_int, - ); - pub fn atg_bucketize_scalar( - out__: *mut *mut C_tensor, - self_scalar_: *mut C_scalar, - boundaries_: *mut C_tensor, - out_int32_: c_int, - right_: c_int, - ); - pub fn atg_bucketize_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_scalar_: *mut C_scalar, - boundaries_: *mut C_tensor, - out_int32_: c_int, - right_: c_int, - ); - pub fn atg_bucketize_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - boundaries_: *mut C_tensor, - out_int32_: c_int, - right_: c_int, - ); - pub fn atg_can_cast(from__: c_int, to_: c_int) -> c_int; - pub fn atg_cartesian_prod( - out__: *mut *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ); - pub fn atg_cat( - out__: *mut *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - dim_: i64, - ); - pub fn atg_cat_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - dim_: i64, - ); - pub fn atg_cauchy(out__: *mut *mut C_tensor, self_: *mut C_tensor, median_: f64, sigma_: f64); - pub fn atg_cauchy_(out__: *mut *mut C_tensor, self_: *mut C_tensor, median_: f64, sigma_: f64); - pub fn atg_cauchy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - median_: f64, - sigma_: f64, - ); - pub fn atg_ccol_indices(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_ccol_indices_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_ccol_indices_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_cdist( - out__: *mut *mut C_tensor, - x1_: *mut C_tensor, - x2_: *mut C_tensor, - p_: f64, - compute_mode_v: i64, - compute_mode_null: i8, - ); - pub fn atg_ceil(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_ceil_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_ceil_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_celu(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_celu_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_celu_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_chain_matmul( - out__: *mut *mut C_tensor, - matrices_data: *const *mut C_tensor, - matrices_len: c_int, - ); - pub fn atg_chain_matmul_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - matrices_data: *const *mut C_tensor, - matrices_len: c_int, - ); - pub fn atg_chalf(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_channel_shuffle(out__: *mut *mut C_tensor, self_: *mut C_tensor, groups_: i64); - pub fn atg_channel_shuffle_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - groups_: i64, - ); - pub fn atg_cholesky(out__: *mut *mut C_tensor, self_: *mut C_tensor, upper_: c_int); - pub fn atg_cholesky_inverse(out__: *mut *mut C_tensor, self_: *mut C_tensor, upper_: c_int); - pub fn atg_cholesky_inverse_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - upper_: c_int, - ); - pub fn atg_cholesky_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - upper_: c_int, - ); - pub fn atg_cholesky_solve( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - input2_: *mut C_tensor, - upper_: c_int, - ); - pub fn atg_cholesky_solve_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - input2_: *mut C_tensor, - upper_: c_int, - ); - pub fn atg_choose_qparams_optimized( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - numel_: i64, - n_bins_: i64, - ratio_: f64, - bit_width_: i64, - ); - pub fn atg_chunk(self_: *mut C_tensor, chunks_: i64, dim_: i64) -> *mut *mut C_tensor; - pub fn atg_clamp( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - min_: *mut C_scalar, - max_: *mut C_scalar, - ); - pub fn atg_clamp_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - min_: *mut C_scalar, - max_: *mut C_scalar, - ); - pub fn atg_clamp_max(out__: *mut *mut C_tensor, self_: *mut C_tensor, max_: *mut C_scalar); - pub fn atg_clamp_max_(out__: *mut *mut C_tensor, self_: *mut C_tensor, max_: *mut C_scalar); - pub fn atg_clamp_max_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - max_: *mut C_scalar, - ); - pub fn atg_clamp_max_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - max_: *mut C_tensor, - ); - pub fn atg_clamp_max_tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - max_: *mut C_tensor, - ); - pub fn atg_clamp_max_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - max_: *mut C_tensor, - ); - pub fn atg_clamp_min(out__: *mut *mut C_tensor, self_: *mut C_tensor, min_: *mut C_scalar); - pub fn atg_clamp_min_(out__: *mut *mut C_tensor, self_: *mut C_tensor, min_: *mut C_scalar); - pub fn atg_clamp_min_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - min_: *mut C_scalar, - ); - pub fn atg_clamp_min_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - min_: *mut C_tensor, - ); - pub fn atg_clamp_min_tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - min_: *mut C_tensor, - ); - pub fn atg_clamp_min_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - min_: *mut C_tensor, - ); - pub fn atg_clamp_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - min_: *mut C_scalar, - max_: *mut C_scalar, - ); - pub fn atg_clamp_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - min_: *mut C_tensor, - max_: *mut C_tensor, - ); - pub fn atg_clamp_tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - min_: *mut C_tensor, - max_: *mut C_tensor, - ); - pub fn atg_clamp_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - min_: *mut C_tensor, - max_: *mut C_tensor, - ); - pub fn atg_clip( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - min_: *mut C_scalar, - max_: *mut C_scalar, - ); - pub fn atg_clip_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - min_: *mut C_scalar, - max_: *mut C_scalar, - ); - pub fn atg_clip_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - min_: *mut C_scalar, - max_: *mut C_scalar, - ); - pub fn atg_clip_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - min_: *mut C_tensor, - max_: *mut C_tensor, - ); - pub fn atg_clip_tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - min_: *mut C_tensor, - max_: *mut C_tensor, - ); - pub fn atg_clip_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - min_: *mut C_tensor, - max_: *mut C_tensor, - ); - pub fn atg_clone(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_coalesce(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_col2im( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - kernel_size_data: *const i64, - kernel_size_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - padding_data: *const i64, - padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - ); - pub fn atg_col2im_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - kernel_size_data: *const i64, - kernel_size_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - padding_data: *const i64, - padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - ); - pub fn atg_col_indices(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_col_indices_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_col_indices_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_column_stack( - out__: *mut *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ); - pub fn atg_column_stack_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ); - pub fn atg_combinations( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - r_: i64, - with_replacement_: c_int, - ); - pub fn atg_complex(out__: *mut *mut C_tensor, real_: *mut C_tensor, imag_: *mut C_tensor); - pub fn atg_complex_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - real_: *mut C_tensor, - imag_: *mut C_tensor, - ); - pub fn atg_concat( - out__: *mut *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - dim_: i64, - ); - pub fn atg_concat_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - dim_: i64, - ); - pub fn atg_concatenate( - out__: *mut *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - dim_: i64, - ); - pub fn atg_concatenate_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - dim_: i64, - ); - pub fn atg_conj(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_conj_physical(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_conj_physical_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_conj_physical_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_constant_pad_nd( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - pad_data: *const i64, - pad_len: c_int, - ); - pub fn atg_constant_pad_nd_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - pad_data: *const i64, - pad_len: c_int, - ); - pub fn atg_contiguous(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_conv1d( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - ); - pub fn atg_conv1d_padding( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_ptr: *const u8, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - ); - pub fn atg_conv2d( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - ); - pub fn atg_conv2d_padding( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_ptr: *const u8, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - ); - pub fn atg_conv3d( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - ); - pub fn atg_conv3d_padding( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_ptr: *const u8, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - ); - pub fn atg_conv_depthwise3d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ); - pub fn atg_conv_depthwise3d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ); - pub fn atg_conv_tbc( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - pad_: i64, - ); - pub fn atg_conv_tbc_backward( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - pad_: i64, - ); - pub fn atg_conv_tbc_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - pad_: i64, - ); - pub fn atg_conv_transpose1d( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - output_padding_data: *const i64, - output_padding_len: c_int, - groups_: i64, - dilation_data: *const i64, - dilation_len: c_int, - ); - pub fn atg_conv_transpose2d( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - output_padding_data: *const i64, - output_padding_len: c_int, - groups_: i64, - dilation_data: *const i64, - dilation_len: c_int, - ); - pub fn atg_conv_transpose3d( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - output_padding_data: *const i64, - output_padding_len: c_int, - groups_: i64, - dilation_data: *const i64, - dilation_len: c_int, - ); - pub fn atg_convolution( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - transposed_: c_int, - output_padding_data: *const i64, - output_padding_len: c_int, - groups_: i64, - ); - pub fn atg_convolution_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - transposed_: c_int, - output_padding_data: *const i64, - output_padding_len: c_int, - groups_: i64, - ); - pub fn atg_convolution_overrideable( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - transposed_: c_int, - output_padding_data: *const i64, - output_padding_len: c_int, - groups_: i64, - ); - pub fn atg_convolution_overrideable_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - transposed_: c_int, - output_padding_data: *const i64, - output_padding_len: c_int, - groups_: i64, - ); - pub fn atg_copy_sparse_to_sparse( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - src_: *mut C_tensor, - non_blocking_: c_int, - ); - pub fn atg_copy_sparse_to_sparse_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - src_: *mut C_tensor, - non_blocking_: c_int, - ); - pub fn atg_copy_sparse_to_sparse_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - src_: *mut C_tensor, - non_blocking_: c_int, - ); - pub fn atg_copysign(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_copysign_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_copysign_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_copysign_scalar( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_copysign_scalar_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_copysign_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_corrcoef(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_cos(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_cos_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_cos_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_cosh(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_cosh_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_cosh_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_cosine_embedding_loss( - out__: *mut *mut C_tensor, - input1_: *mut C_tensor, - input2_: *mut C_tensor, - target_: *mut C_tensor, - margin_: f64, - reduction_: i64, - ); - pub fn atg_cosine_similarity( - out__: *mut *mut C_tensor, - x1_: *mut C_tensor, - x2_: *mut C_tensor, - dim_: i64, - eps_: f64, - ); - pub fn atg_count_nonzero( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_v: i64, - dim_null: i8, - ); - pub fn atg_count_nonzero_dim_intlist( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - ); - pub fn atg_count_nonzero_dim_intlist_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - ); - pub fn atg_count_nonzero_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_v: i64, - dim_null: i8, - ); - pub fn atg_cov( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - correction_: i64, - fweights_: *mut C_tensor, - aweights_: *mut C_tensor, - ); - pub fn atg_cross( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - dim_v: i64, - dim_null: i8, - ); - pub fn atg_cross_entropy_loss( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - weight_: *mut C_tensor, - reduction_: i64, - ignore_index_: i64, - label_smoothing_: f64, - ); - pub fn atg_cross_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - dim_v: i64, - dim_null: i8, - ); - pub fn atg_crow_indices(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_crow_indices_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_crow_indices_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_ctc_loss( - out__: *mut *mut C_tensor, - log_probs_: *mut C_tensor, - targets_: *mut C_tensor, - input_lengths_data: *const i64, - input_lengths_len: c_int, - target_lengths_data: *const i64, - target_lengths_len: c_int, - blank_: i64, - reduction_: i64, - zero_infinity_: c_int, - ); - pub fn atg_ctc_loss_tensor( - out__: *mut *mut C_tensor, - log_probs_: *mut C_tensor, - targets_: *mut C_tensor, - input_lengths_: *mut C_tensor, - target_lengths_: *mut C_tensor, - blank_: i64, - reduction_: i64, - zero_infinity_: c_int, - ); - pub fn atg_cudnn_affine_grid_generator( - out__: *mut *mut C_tensor, - theta_: *mut C_tensor, - n_: i64, - C_: i64, - H_: i64, - W_: i64, - ); - pub fn atg_cudnn_affine_grid_generator_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - n_: i64, - C_: i64, - H_: i64, - W_: i64, - ); - pub fn atg_cudnn_affine_grid_generator_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_: *mut C_tensor, - n_: i64, - C_: i64, - H_: i64, - W_: i64, - ); - pub fn atg_cudnn_affine_grid_generator_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - theta_: *mut C_tensor, - n_: i64, - C_: i64, - H_: i64, - W_: i64, - ); - pub fn atg_cudnn_batch_norm( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - training_: c_int, - exponential_average_factor_: f64, - epsilon_: f64, - ); - pub fn atg_cudnn_batch_norm_backward( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - grad_output_: *mut C_tensor, - weight_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - save_mean_: *mut C_tensor, - save_var_: *mut C_tensor, - epsilon_: f64, - reserveSpace_: *mut C_tensor, - ); - pub fn atg_cudnn_batch_norm_backward_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - out2_: *mut C_tensor, - input_: *mut C_tensor, - grad_output_: *mut C_tensor, - weight_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - save_mean_: *mut C_tensor, - save_var_: *mut C_tensor, - epsilon_: f64, - reserveSpace_: *mut C_tensor, - ); - pub fn atg_cudnn_batch_norm_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - out2_: *mut C_tensor, - out3_: *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - training_: c_int, - exponential_average_factor_: f64, - epsilon_: f64, - ); - pub fn atg_cudnn_convolution( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - benchmark_: c_int, - deterministic_: c_int, - allow_tf32_: c_int, - ); - pub fn atg_cudnn_convolution_add_relu( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - z_: *mut C_tensor, - alpha_: *mut C_scalar, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - ); - pub fn atg_cudnn_convolution_add_relu_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - z_: *mut C_tensor, - alpha_: *mut C_scalar, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - ); - pub fn atg_cudnn_convolution_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - benchmark_: c_int, - deterministic_: c_int, - allow_tf32_: c_int, - ); - pub fn atg_cudnn_convolution_relu( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - ); - pub fn atg_cudnn_convolution_relu_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - ); - pub fn atg_cudnn_convolution_transpose( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - output_padding_data: *const i64, - output_padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - benchmark_: c_int, - deterministic_: c_int, - allow_tf32_: c_int, - ); - pub fn atg_cudnn_convolution_transpose_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - output_padding_data: *const i64, - output_padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - benchmark_: c_int, - deterministic_: c_int, - allow_tf32_: c_int, - ); - pub fn atg_cudnn_grid_sampler( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - grid_: *mut C_tensor, - ); - pub fn atg_cudnn_grid_sampler_backward( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - grid_: *mut C_tensor, - grad_output_: *mut C_tensor, - ); - pub fn atg_cudnn_grid_sampler_backward_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - self_: *mut C_tensor, - grid_: *mut C_tensor, - grad_output_: *mut C_tensor, - ); - pub fn atg_cudnn_grid_sampler_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - grid_: *mut C_tensor, - ); - pub fn atg_cudnn_is_acceptable(self_: *mut C_tensor) -> c_int; - pub fn atg_cummax(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64); - pub fn atg_cummax_out( - out__: *mut *mut C_tensor, - values_: *mut C_tensor, - indices_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - ); - pub fn atg_cummaxmin_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - input_: *mut C_tensor, - indices_: *mut C_tensor, - dim_: i64, - ); - pub fn atg_cummin(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64); - pub fn atg_cummin_out( - out__: *mut *mut C_tensor, - values_: *mut C_tensor, - indices_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - ); - pub fn atg_cumprod(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, dtype_: c_int); - pub fn atg_cumprod_(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, dtype_: c_int); - pub fn atg_cumprod_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - input_: *mut C_tensor, - dim_: i64, - output_: *mut C_tensor, - ); - pub fn atg_cumprod_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - dtype_: c_int, - ); - pub fn atg_cumsum(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, dtype_: c_int); - pub fn atg_cumsum_(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, dtype_: c_int); - pub fn atg_cumsum_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - dtype_: c_int, - ); - pub fn atg_cumulative_trapezoid(out__: *mut *mut C_tensor, y_: *mut C_tensor, dim_: i64); - pub fn atg_cumulative_trapezoid_x( - out__: *mut *mut C_tensor, - y_: *mut C_tensor, - x_: *mut C_tensor, - dim_: i64, - ); - pub fn atg_data(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_deg2rad(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_deg2rad_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_deg2rad_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_dense_dim(self_: *mut C_tensor) -> i64; - pub fn atg_dequantize(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_dequantize_self_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_dequantize_tensors( - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ) -> *mut *mut C_tensor; - pub fn atg_dequantize_tensors_out( - out_data: *const *mut C_tensor, - out_len: c_int, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ); - pub fn atg_det(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_detach(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_detach_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_detach_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_detach_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_diag(out__: *mut *mut C_tensor, self_: *mut C_tensor, diagonal_: i64); - pub fn atg_diag_embed( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - offset_: i64, - dim1_: i64, - dim2_: i64, - ); - pub fn atg_diag_embed_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - offset_: i64, - dim1_: i64, - dim2_: i64, - ); - pub fn atg_diag_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - diagonal_: i64, - ); - pub fn atg_diagflat(out__: *mut *mut C_tensor, self_: *mut C_tensor, offset_: i64); - pub fn atg_diagonal( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - offset_: i64, - dim1_: i64, - dim2_: i64, - ); - pub fn atg_diagonal_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - input_sizes_data: *const i64, - input_sizes_len: c_int, - offset_: i64, - dim1_: i64, - dim2_: i64, - ); - pub fn atg_diagonal_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_output_: *mut C_tensor, - input_sizes_data: *const i64, - input_sizes_len: c_int, - offset_: i64, - dim1_: i64, - dim2_: i64, - ); - pub fn atg_diagonal_copy( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - offset_: i64, - dim1_: i64, - dim2_: i64, - ); - pub fn atg_diagonal_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - offset_: i64, - dim1_: i64, - dim2_: i64, - ); - pub fn atg_diagonal_scatter( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - src_: *mut C_tensor, - offset_: i64, - dim1_: i64, - dim2_: i64, - ); - pub fn atg_diagonal_scatter_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - src_: *mut C_tensor, - offset_: i64, - dim1_: i64, - dim2_: i64, - ); - pub fn atg_diff( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - n_: i64, - dim_: i64, - prepend_: *mut C_tensor, - append_: *mut C_tensor, - ); - pub fn atg_diff_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - n_: i64, - dim_: i64, - prepend_: *mut C_tensor, - append_: *mut C_tensor, - ); - pub fn atg_digamma(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_digamma_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_digamma_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_dist(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_dist_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_div(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_div_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_div_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_div_out_mode( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - rounding_mode_ptr: *const u8, - rounding_mode_len: c_int, - ); - pub fn atg_div_scalar(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_div_scalar_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_div_scalar_mode( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - rounding_mode_ptr: *const u8, - rounding_mode_len: c_int, - ); - pub fn atg_div_scalar_mode_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - rounding_mode_ptr: *const u8, - rounding_mode_len: c_int, - ); - pub fn atg_div_scalar_mode_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - rounding_mode_ptr: *const u8, - rounding_mode_len: c_int, - ); - pub fn atg_div_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_div_tensor_mode( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - rounding_mode_ptr: *const u8, - rounding_mode_len: c_int, - ); - pub fn atg_div_tensor_mode_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - rounding_mode_ptr: *const u8, - rounding_mode_len: c_int, - ); - pub fn atg_divide(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_divide_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_divide_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_divide_out_mode( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - rounding_mode_ptr: *const u8, - rounding_mode_len: c_int, - ); - pub fn atg_divide_scalar( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_divide_scalar_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_divide_scalar_mode( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - rounding_mode_ptr: *const u8, - rounding_mode_len: c_int, - ); - pub fn atg_divide_scalar_mode_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - rounding_mode_ptr: *const u8, - rounding_mode_len: c_int, - ); - pub fn atg_divide_tensor_mode( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - rounding_mode_ptr: *const u8, - rounding_mode_len: c_int, - ); - pub fn atg_divide_tensor_mode_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - rounding_mode_ptr: *const u8, - rounding_mode_len: c_int, - ); - pub fn atg_dot(out__: *mut *mut C_tensor, self_: *mut C_tensor, tensor_: *mut C_tensor); - pub fn atg_dot_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - tensor_: *mut C_tensor, - ); - pub fn atg_dropout(out__: *mut *mut C_tensor, input_: *mut C_tensor, p_: f64, train_: c_int); - pub fn atg_dropout_(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: f64, train_: c_int); - pub fn atg_dsplit(self_: *mut C_tensor, sections_: i64) -> *mut *mut C_tensor; - pub fn atg_dsplit_array( - self_: *mut C_tensor, - indices_data: *const i64, - indices_len: c_int, - ) -> *mut *mut C_tensor; - pub fn atg_dstack( - out__: *mut *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ); - pub fn atg_dstack_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ); - pub fn atg_einsum( - out__: *mut *mut C_tensor, - equation_ptr: *const u8, - equation_len: c_int, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - path_data: *const i64, - path_len: c_int, - ); - pub fn atg_elu(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_elu_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_elu_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - alpha_: *mut C_scalar, - scale_: *mut C_scalar, - input_scale_: *mut C_scalar, - is_result_: c_int, - self_or_result_: *mut C_tensor, - ); - pub fn atg_elu_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - alpha_: *mut C_scalar, - scale_: *mut C_scalar, - input_scale_: *mut C_scalar, - is_result_: c_int, - self_or_result_: *mut C_tensor, - ); - pub fn atg_elu_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_embedding( - out__: *mut *mut C_tensor, - weight_: *mut C_tensor, - indices_: *mut C_tensor, - padding_idx_: i64, - scale_grad_by_freq_: c_int, - sparse_: c_int, - ); - pub fn atg_embedding_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - indices_: *mut C_tensor, - num_weights_: i64, - padding_idx_: i64, - scale_grad_by_freq_: c_int, - sparse_: c_int, - ); - pub fn atg_embedding_bag( - out__: *mut *mut C_tensor, - weight_: *mut C_tensor, - indices_: *mut C_tensor, - offsets_: *mut C_tensor, - scale_grad_by_freq_: c_int, - mode_: i64, - sparse_: c_int, - per_sample_weights_: *mut C_tensor, - include_last_offset_: c_int, - ); - pub fn atg_embedding_bag_padding_idx( - out__: *mut *mut C_tensor, - weight_: *mut C_tensor, - indices_: *mut C_tensor, - offsets_: *mut C_tensor, - scale_grad_by_freq_: c_int, - mode_: i64, - sparse_: c_int, - per_sample_weights_: *mut C_tensor, - include_last_offset_: c_int, - padding_idx_v: i64, - padding_idx_null: i8, - ); - pub fn atg_embedding_dense_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - indices_: *mut C_tensor, - num_weights_: i64, - padding_idx_: i64, - scale_grad_by_freq_: c_int, - ); - pub fn atg_embedding_dense_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_output_: *mut C_tensor, - indices_: *mut C_tensor, - num_weights_: i64, - padding_idx_: i64, - scale_grad_by_freq_: c_int, - ); - pub fn atg_embedding_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - weight_: *mut C_tensor, - indices_: *mut C_tensor, - padding_idx_: i64, - scale_grad_by_freq_: c_int, - sparse_: c_int, - ); - pub fn atg_embedding_renorm( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - indices_: *mut C_tensor, - max_norm_: f64, - norm_type_: f64, - ); - pub fn atg_embedding_renorm_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - indices_: *mut C_tensor, - max_norm_: f64, - norm_type_: f64, - ); - pub fn atg_embedding_renorm_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - indices_: *mut C_tensor, - max_norm_: f64, - norm_type_: f64, - ); - pub fn atg_embedding_sparse_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - indices_: *mut C_tensor, - num_weights_: i64, - padding_idx_: i64, - scale_grad_by_freq_: c_int, - ); - pub fn atg_empty( - out__: *mut *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_empty_like(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_empty_like_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_empty_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg_empty_permuted( - out__: *mut *mut C_tensor, - size_data: *const i64, - size_len: c_int, - physical_layout_data: *const i64, - physical_layout_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_empty_permuted_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - physical_layout_data: *const i64, - physical_layout_len: c_int, - ); - pub fn atg_empty_quantized( - out__: *mut *mut C_tensor, - size_data: *const i64, - size_len: c_int, - qtensor_: *mut C_tensor, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_empty_quantized_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - qtensor_: *mut C_tensor, - ); - pub fn atg_empty_strided( - out__: *mut *mut C_tensor, - size_data: *const i64, - size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_empty_strided_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - ); - pub fn atg_eq(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_eq_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_eq_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_eq_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_eq_tensor_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_eq_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_equal(self_: *mut C_tensor, other_: *mut C_tensor) -> c_int; - pub fn atg_erf(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_erf_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_erf_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_erfc(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_erfc_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_erfc_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_erfinv(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_erfinv_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_erfinv_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_exp(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_exp2(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_exp2_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_exp2_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_exp_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_exp_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_expand( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - implicit_: c_int, - ); - pub fn atg_expand_as(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_expand_copy( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - implicit_: c_int, - ); - pub fn atg_expand_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - implicit_: c_int, - ); - pub fn atg_expm1(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_expm1_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_expm1_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_exponential(out__: *mut *mut C_tensor, self_: *mut C_tensor, lambd_: f64); - pub fn atg_exponential_(out__: *mut *mut C_tensor, self_: *mut C_tensor, lambd_: f64); - pub fn atg_exponential_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - lambd_: f64, - ); - pub fn atg_eye(out__: *mut *mut C_tensor, n_: i64, options_kind: c_int, options_device: c_int); - pub fn atg_eye_m( - out__: *mut *mut C_tensor, - n_: i64, - m_: i64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_eye_m_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, n_: i64, m_: i64); - pub fn atg_eye_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, n_: i64); - pub fn atg_fake_quantize_per_channel_affine( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - scale_: *mut C_tensor, - zero_point_: *mut C_tensor, - axis_: i64, - quant_min_: i64, - quant_max_: i64, - ); - pub fn atg_fake_quantize_per_channel_affine_cachemask( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - scale_: *mut C_tensor, - zero_point_: *mut C_tensor, - axis_: i64, - quant_min_: i64, - quant_max_: i64, - ); - pub fn atg_fake_quantize_per_channel_affine_cachemask_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - mask_: *mut C_tensor, - ); - pub fn atg_fake_quantize_per_channel_affine_cachemask_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - self_: *mut C_tensor, - scale_: *mut C_tensor, - zero_point_: *mut C_tensor, - axis_: i64, - quant_min_: i64, - quant_max_: i64, - ); - pub fn atg_fake_quantize_per_tensor_affine( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - scale_: f64, - zero_point_: i64, - quant_min_: i64, - quant_max_: i64, - ); - pub fn atg_fake_quantize_per_tensor_affine_cachemask( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - scale_: f64, - zero_point_: i64, - quant_min_: i64, - quant_max_: i64, - ); - pub fn atg_fake_quantize_per_tensor_affine_cachemask_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - mask_: *mut C_tensor, - ); - pub fn atg_fake_quantize_per_tensor_affine_cachemask_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - self_: *mut C_tensor, - scale_: f64, - zero_point_: i64, - quant_min_: i64, - quant_max_: i64, - ); - pub fn atg_fake_quantize_per_tensor_affine_tensor_qparams( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - scale_: *mut C_tensor, - zero_point_: *mut C_tensor, - quant_min_: i64, - quant_max_: i64, - ); - pub fn atg_feature_alpha_dropout( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - p_: f64, - train_: c_int, - ); - pub fn atg_feature_alpha_dropout_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - p_: f64, - train_: c_int, - ); - pub fn atg_feature_dropout( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - p_: f64, - train_: c_int, - ); - pub fn atg_feature_dropout_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - p_: f64, - train_: c_int, - ); - pub fn atg_fft_fft( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - n_v: i64, - n_null: i8, - dim_: i64, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_fft2( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_fft2_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_fft_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - n_v: i64, - n_null: i8, - dim_: i64, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_fftfreq( - out__: *mut *mut C_tensor, - n_: i64, - d_: f64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_fft_fftfreq_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, n_: i64, d_: f64); - pub fn atg_fft_fftn( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_fftn_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_fftshift( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - ); - pub fn atg_fft_hfft( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - n_v: i64, - n_null: i8, - dim_: i64, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_hfft2( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_hfft2_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_hfft_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - n_v: i64, - n_null: i8, - dim_: i64, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_hfftn( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_hfftn_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_ifft( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - n_v: i64, - n_null: i8, - dim_: i64, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_ifft2( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_ifft2_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_ifft_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - n_v: i64, - n_null: i8, - dim_: i64, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_ifftn( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_ifftn_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_ifftshift( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - ); - pub fn atg_fft_ihfft( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - n_v: i64, - n_null: i8, - dim_: i64, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_ihfft2( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_ihfft2_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_ihfft_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - n_v: i64, - n_null: i8, - dim_: i64, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_ihfftn( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_ihfftn_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_irfft( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - n_v: i64, - n_null: i8, - dim_: i64, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_irfft2( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_irfft2_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_irfft_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - n_v: i64, - n_null: i8, - dim_: i64, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_irfftn( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_irfftn_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_rfft( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - n_v: i64, - n_null: i8, - dim_: i64, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_rfft2( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_rfft2_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_rfft_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - n_v: i64, - n_null: i8, - dim_: i64, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_rfftfreq( - out__: *mut *mut C_tensor, - n_: i64, - d_: f64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_fft_rfftfreq_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, n_: i64, d_: f64); - pub fn atg_fft_rfftn( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fft_rfftn_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - s_data: *const i64, - s_len: c_int, - dim_data: *const i64, - dim_len: c_int, - norm_ptr: *const u8, - norm_len: c_int, - ); - pub fn atg_fill(out__: *mut *mut C_tensor, self_: *mut C_tensor, value_: *mut C_scalar); - pub fn atg_fill_(out__: *mut *mut C_tensor, self_: *mut C_tensor, value_: *mut C_scalar); - pub fn atg_fill_diagonal_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - fill_value_: *mut C_scalar, - wrap_: c_int, - ); - pub fn atg_fill_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - value_: *mut C_scalar, - ); - pub fn atg_fill_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor, value_: *mut C_tensor); - pub fn atg_fill_tensor_(out__: *mut *mut C_tensor, self_: *mut C_tensor, value_: *mut C_tensor); - pub fn atg_fill_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - value_: *mut C_tensor, - ); - pub fn atg_fix(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_fix_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_fix_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_flatten( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - start_dim_: i64, - end_dim_: i64, - ); - pub fn atg_flatten_dense_tensors( - out__: *mut *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ); - pub fn atg_flip( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dims_data: *const i64, - dims_len: c_int, - ); - pub fn atg_flip_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dims_data: *const i64, - dims_len: c_int, - ); - pub fn atg_fliplr(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_flipud(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_float_power( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - exponent_: *mut C_tensor, - ); - pub fn atg_float_power_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - exponent_: *mut C_scalar, - ); - pub fn atg_float_power_scalar( - out__: *mut *mut C_tensor, - self_scalar_: *mut C_scalar, - exponent_: *mut C_tensor, - ); - pub fn atg_float_power_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_scalar_: *mut C_scalar, - exponent_: *mut C_tensor, - ); - pub fn atg_float_power_tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - exponent_: *mut C_tensor, - ); - pub fn atg_float_power_tensor_scalar( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - exponent_: *mut C_scalar, - ); - pub fn atg_float_power_tensor_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - exponent_: *mut C_scalar, - ); - pub fn atg_float_power_tensor_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - exponent_: *mut C_tensor, - ); - pub fn atg_floor(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_floor_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_floor_divide(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_floor_divide_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_floor_divide_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_floor_divide_scalar( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_floor_divide_scalar_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_floor_divide_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_floor_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_fmax(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_fmax_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_fmin(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_fmin_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_fmod(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_fmod_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_fmod_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_fmod_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_fmod_tensor_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_fmod_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_frac(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_frac_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_frac_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_fractional_max_pool2d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - output_size_data: *const i64, - output_size_len: c_int, - random_samples_: *mut C_tensor, - ); - pub fn atg_fractional_max_pool2d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - output_size_data: *const i64, - output_size_len: c_int, - indices_: *mut C_tensor, - ); - pub fn atg_fractional_max_pool2d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - output_size_data: *const i64, - output_size_len: c_int, - indices_: *mut C_tensor, - ); - pub fn atg_fractional_max_pool2d_output( - out__: *mut *mut C_tensor, - output_: *mut C_tensor, - indices_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - output_size_data: *const i64, - output_size_len: c_int, - random_samples_: *mut C_tensor, - ); - pub fn atg_fractional_max_pool3d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - output_size_data: *const i64, - output_size_len: c_int, - random_samples_: *mut C_tensor, - ); - pub fn atg_fractional_max_pool3d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - output_size_data: *const i64, - output_size_len: c_int, - indices_: *mut C_tensor, - ); - pub fn atg_fractional_max_pool3d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - output_size_data: *const i64, - output_size_len: c_int, - indices_: *mut C_tensor, - ); - pub fn atg_fractional_max_pool3d_output( - out__: *mut *mut C_tensor, - output_: *mut C_tensor, - indices_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - output_size_data: *const i64, - output_size_len: c_int, - random_samples_: *mut C_tensor, - ); - pub fn atg_frexp(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_frexp_tensor_out( - out__: *mut *mut C_tensor, - mantissa_: *mut C_tensor, - exponent_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_frobenius_norm( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - ); - pub fn atg_frobenius_norm_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - ); - pub fn atg_from_file( - out__: *mut *mut C_tensor, - filename_ptr: *const u8, - filename_len: c_int, - shared_: c_int, - size_v: i64, - size_null: i8, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_from_file_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - filename_ptr: *const u8, - filename_len: c_int, - shared_: c_int, - size_v: i64, - size_null: i8, - ); - pub fn atg_full( - out__: *mut *mut C_tensor, - size_data: *const i64, - size_len: c_int, - fill_value_: *mut C_scalar, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_full_like( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - fill_value_: *mut C_scalar, - ); - pub fn atg_full_like_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - fill_value_: *mut C_scalar, - ); - pub fn atg_full_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - fill_value_: *mut C_scalar, - ); - pub fn atg_fused_moving_avg_obs_fake_quant( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - observer_on_: *mut C_tensor, - fake_quant_on_: *mut C_tensor, - running_min_: *mut C_tensor, - running_max_: *mut C_tensor, - scale_: *mut C_tensor, - zero_point_: *mut C_tensor, - averaging_const_: f64, - quant_min_: i64, - quant_max_: i64, - ch_axis_: i64, - per_row_fake_quant_: c_int, - symmetric_quant_: c_int, - ); - pub fn atg_gather( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - sparse_grad_: c_int, - ); - pub fn atg_gather_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - sparse_grad_: c_int, - ); - pub fn atg_gather_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - sparse_grad_: c_int, - ); - pub fn atg_gcd(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_gcd_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_gcd_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_ge(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_ge_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_ge_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_ge_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_ge_tensor_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_ge_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_gelu( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - approximate_ptr: *const u8, - approximate_len: c_int, - ); - pub fn atg_gelu_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - approximate_ptr: *const u8, - approximate_len: c_int, - ); - pub fn atg_gelu_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - approximate_ptr: *const u8, - approximate_len: c_int, - ); - pub fn atg_gelu_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - approximate_ptr: *const u8, - approximate_len: c_int, - ); - pub fn atg_gelu_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - approximate_ptr: *const u8, - approximate_len: c_int, - ); - pub fn atg_geometric(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: f64); - pub fn atg_geometric_(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: f64); - pub fn atg_geometric_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - p_: f64, - ); - pub fn atg_geqrf(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_geqrf_a( - out__: *mut *mut C_tensor, - a_: *mut C_tensor, - tau_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_ger(out__: *mut *mut C_tensor, self_: *mut C_tensor, vec2_: *mut C_tensor); - pub fn atg_ger_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - vec2_: *mut C_tensor, - ); - pub fn atg_glu(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64); - pub fn atg_glu_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - ); - pub fn atg_glu_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - ); - pub fn atg_glu_backward_jvp( - out__: *mut *mut C_tensor, - grad_x_: *mut C_tensor, - grad_glu_: *mut C_tensor, - x_: *mut C_tensor, - dgrad_glu_: *mut C_tensor, - dx_: *mut C_tensor, - dim_: i64, - ); - pub fn atg_glu_backward_jvp_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_x_: *mut C_tensor, - grad_glu_: *mut C_tensor, - x_: *mut C_tensor, - dgrad_glu_: *mut C_tensor, - dx_: *mut C_tensor, - dim_: i64, - ); - pub fn atg_glu_jvp( - out__: *mut *mut C_tensor, - glu_: *mut C_tensor, - x_: *mut C_tensor, - dx_: *mut C_tensor, - dim_: i64, - ); - pub fn atg_glu_jvp_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - glu_: *mut C_tensor, - x_: *mut C_tensor, - dx_: *mut C_tensor, - dim_: i64, - ); - pub fn atg_glu_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - ); - pub fn atg_grad(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_greater(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_greater_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_greater_equal( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_greater_equal_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_greater_equal_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_greater_equal_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_greater_equal_tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_greater_equal_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_greater_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_greater_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_greater_tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_greater_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_grid_sampler( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - grid_: *mut C_tensor, - interpolation_mode_: i64, - padding_mode_: i64, - align_corners_: c_int, - ); - pub fn atg_grid_sampler_2d( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - grid_: *mut C_tensor, - interpolation_mode_: i64, - padding_mode_: i64, - align_corners_: c_int, - ); - pub fn atg_grid_sampler_2d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - input_: *mut C_tensor, - grid_: *mut C_tensor, - interpolation_mode_: i64, - padding_mode_: i64, - align_corners_: c_int, - ); - pub fn atg_grid_sampler_3d( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - grid_: *mut C_tensor, - interpolation_mode_: i64, - padding_mode_: i64, - align_corners_: c_int, - ); - pub fn atg_grid_sampler_3d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - input_: *mut C_tensor, - grid_: *mut C_tensor, - interpolation_mode_: i64, - padding_mode_: i64, - align_corners_: c_int, - ); - pub fn atg_group_norm( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - num_groups_: i64, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - eps_: f64, - cudnn_enabled_: c_int, - ); - pub fn atg_gru( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - hx_: *mut C_tensor, - params_data: *const *mut C_tensor, - params_len: c_int, - has_biases_: c_int, - num_layers_: i64, - dropout_: f64, - train_: c_int, - bidirectional_: c_int, - batch_first_: c_int, - ); - pub fn atg_gru_cell( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - hx_: *mut C_tensor, - w_ih_: *mut C_tensor, - w_hh_: *mut C_tensor, - b_ih_: *mut C_tensor, - b_hh_: *mut C_tensor, - ); - pub fn atg_gru_data( - out__: *mut *mut C_tensor, - data_: *mut C_tensor, - batch_sizes_: *mut C_tensor, - hx_: *mut C_tensor, - params_data: *const *mut C_tensor, - params_len: c_int, - has_biases_: c_int, - num_layers_: i64, - dropout_: f64, - train_: c_int, - bidirectional_: c_int, - ); - pub fn atg_gt(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_gt_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_gt_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_gt_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_gt_tensor_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_gt_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_hamming_window( - out__: *mut *mut C_tensor, - window_length_: i64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_hamming_window_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - window_length_: i64, - ); - pub fn atg_hamming_window_periodic( - out__: *mut *mut C_tensor, - window_length_: i64, - periodic_: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_hamming_window_periodic_alpha( - out__: *mut *mut C_tensor, - window_length_: i64, - periodic_: c_int, - alpha_: f64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_hamming_window_periodic_alpha_beta( - out__: *mut *mut C_tensor, - window_length_: i64, - periodic_: c_int, - alpha_: f64, - beta_: f64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_hamming_window_periodic_alpha_beta_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - window_length_: i64, - periodic_: c_int, - alpha_: f64, - beta_: f64, - ); - pub fn atg_hamming_window_periodic_alpha_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - window_length_: i64, - periodic_: c_int, - alpha_: f64, - ); - pub fn atg_hamming_window_periodic_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - window_length_: i64, - periodic_: c_int, - ); - pub fn atg_hann_window( - out__: *mut *mut C_tensor, - window_length_: i64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_hann_window_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, window_length_: i64); - pub fn atg_hann_window_periodic( - out__: *mut *mut C_tensor, - window_length_: i64, - periodic_: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_hann_window_periodic_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - window_length_: i64, - periodic_: c_int, - ); - pub fn atg_hardshrink(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_hardshrink_backward( - out__: *mut *mut C_tensor, - grad_out_: *mut C_tensor, - self_: *mut C_tensor, - lambd_: *mut C_scalar, - ); - pub fn atg_hardshrink_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_out_: *mut C_tensor, - self_: *mut C_tensor, - lambd_: *mut C_scalar, - ); - pub fn atg_hardshrink_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_hardsigmoid(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_hardsigmoid_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_hardsigmoid_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_hardsigmoid_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_hardsigmoid_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_hardswish(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_hardswish_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_hardswish_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_hardswish_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_hardswish_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_hardtanh(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_hardtanh_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_hardtanh_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - min_val_: *mut C_scalar, - max_val_: *mut C_scalar, - ); - pub fn atg_hardtanh_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - min_val_: *mut C_scalar, - max_val_: *mut C_scalar, - ); - pub fn atg_hardtanh_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_hash_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - mode_: i64, - ); - pub fn atg_hash_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - mode_: i64, - ); - pub fn atg_heaviside(out__: *mut *mut C_tensor, self_: *mut C_tensor, values_: *mut C_tensor); - pub fn atg_heaviside_(out__: *mut *mut C_tensor, self_: *mut C_tensor, values_: *mut C_tensor); - pub fn atg_heaviside_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - values_: *mut C_tensor, - ); - pub fn atg_hinge_embedding_loss( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - margin_: f64, - reduction_: i64, - ); - pub fn atg_histc(out__: *mut *mut C_tensor, self_: *mut C_tensor, bins_: i64); - pub fn atg_histc_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - bins_: i64, - ); - pub fn atg_histogram( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - bins_: *mut C_tensor, - weight_: *mut C_tensor, - density_: c_int, - ); - pub fn atg_histogram_bin_ct( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - bins_: i64, - range_data: *const f64, - range_len: c_int, - weight_: *mut C_tensor, - density_: c_int, - ); - pub fn atg_histogram_bin_ct_out( - out__: *mut *mut C_tensor, - hist_: *mut C_tensor, - bin_edges_: *mut C_tensor, - self_: *mut C_tensor, - bins_: i64, - range_data: *const f64, - range_len: c_int, - weight_: *mut C_tensor, - density_: c_int, - ); - pub fn atg_histogram_bins_tensor_out( - out__: *mut *mut C_tensor, - hist_: *mut C_tensor, - bin_edges_: *mut C_tensor, - self_: *mut C_tensor, - bins_: *mut C_tensor, - weight_: *mut C_tensor, - density_: c_int, - ); - pub fn atg_hsplit(self_: *mut C_tensor, sections_: i64) -> *mut *mut C_tensor; - pub fn atg_hsplit_array( - self_: *mut C_tensor, - indices_data: *const i64, - indices_len: c_int, - ) -> *mut *mut C_tensor; - pub fn atg_hspmm(out__: *mut *mut C_tensor, mat1_: *mut C_tensor, mat2_: *mut C_tensor); - pub fn atg_hspmm_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - mat1_: *mut C_tensor, - mat2_: *mut C_tensor, - ); - pub fn atg_hstack( - out__: *mut *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ); - pub fn atg_hstack_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ); - pub fn atg_huber_loss( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - reduction_: i64, - delta_: f64, - ); - pub fn atg_huber_loss_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - reduction_: i64, - delta_: f64, - ); - pub fn atg_huber_loss_backward_out( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - reduction_: i64, - delta_: f64, - ); - pub fn atg_huber_loss_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - reduction_: i64, - delta_: f64, - ); - pub fn atg_hypot(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_hypot_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_hypot_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_i0(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_i0_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_i0_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_igamma(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_igamma_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_igamma_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_igammac(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_igammac_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_igammac_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_im2col( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - padding_data: *const i64, - padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - ); - pub fn atg_im2col_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - padding_data: *const i64, - padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - ); - pub fn atg_imag(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_index( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - indices_data: *const *mut C_tensor, - indices_len: c_int, - ); - pub fn atg_index_add( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - source_: *mut C_tensor, - ); - pub fn atg_index_add_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - source_: *mut C_tensor, - ); - pub fn atg_index_add_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - source_: *mut C_tensor, - ); - pub fn atg_index_copy( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - source_: *mut C_tensor, - ); - pub fn atg_index_copy_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - source_: *mut C_tensor, - ); - pub fn atg_index_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - source_: *mut C_tensor, - ); - pub fn atg_index_fill( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - value_: *mut C_scalar, - ); - pub fn atg_index_fill_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - value_: *mut C_scalar, - ); - pub fn atg_index_fill_int_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - value_: *mut C_scalar, - ); - pub fn atg_index_fill_int_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - value_: *mut C_tensor, - ); - pub fn atg_index_fill_int_tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - value_: *mut C_tensor, - ); - pub fn atg_index_fill_int_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - value_: *mut C_tensor, - ); - pub fn atg_index_put( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - indices_data: *const *mut C_tensor, - indices_len: c_int, - values_: *mut C_tensor, - accumulate_: c_int, - ); - pub fn atg_index_put_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - indices_data: *const *mut C_tensor, - indices_len: c_int, - values_: *mut C_tensor, - accumulate_: c_int, - ); - pub fn atg_index_put_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - indices_data: *const *mut C_tensor, - indices_len: c_int, - values_: *mut C_tensor, - accumulate_: c_int, - ); - pub fn atg_index_reduce( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - source_: *mut C_tensor, - reduce_ptr: *const u8, - reduce_len: c_int, - include_self_: c_int, - ); - pub fn atg_index_reduce_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - source_: *mut C_tensor, - reduce_ptr: *const u8, - reduce_len: c_int, - include_self_: c_int, - ); - pub fn atg_index_reduce_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - source_: *mut C_tensor, - reduce_ptr: *const u8, - reduce_len: c_int, - include_self_: c_int, - ); - pub fn atg_index_select( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - ); - pub fn atg_index_select_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - self_sizes_data: *const i64, - self_sizes_len: c_int, - dim_: i64, - index_: *mut C_tensor, - ); - pub fn atg_index_select_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - ); - pub fn atg_index_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - indices_data: *const *mut C_tensor, - indices_len: c_int, - ); - pub fn atg_indices(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_indices_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_indices_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_infinitely_differentiable_gelu_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_inner(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_inner_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_instance_norm( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - use_input_stats_: c_int, - momentum_: f64, - eps_: f64, - cudnn_enabled_: c_int, - ); - pub fn atg_int_repr(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_int_repr_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_inverse(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_inverse_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_is_coalesced(self_: *mut C_tensor) -> c_int; - pub fn atg_is_complex(self_: *mut C_tensor) -> c_int; - pub fn atg_is_conj(self_: *mut C_tensor) -> c_int; - pub fn atg_is_distributed(self_: *mut C_tensor) -> c_int; - pub fn atg_is_floating_point(self_: *mut C_tensor) -> c_int; - pub fn atg_is_inference(self_: *mut C_tensor) -> c_int; - pub fn atg_is_leaf(self_: *mut C_tensor) -> c_int; - pub fn atg_is_neg(self_: *mut C_tensor) -> c_int; - pub fn atg_is_nonzero(self_: *mut C_tensor) -> c_int; - pub fn atg_is_pinned(self_: *mut C_tensor, device_: c_int) -> c_int; - pub fn atg_is_same_size(self_: *mut C_tensor, other_: *mut C_tensor) -> c_int; - pub fn atg_is_set_to(self_: *mut C_tensor, tensor_: *mut C_tensor) -> c_int; - pub fn atg_is_signed(self_: *mut C_tensor) -> c_int; - pub fn atg_is_vulkan_available() -> c_int; - pub fn atg_isclose( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - rtol_: f64, - atol_: f64, - equal_nan_: c_int, - ); - pub fn atg_isfinite(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_isin( - out__: *mut *mut C_tensor, - elements_: *mut C_tensor, - test_elements_: *mut C_tensor, - assume_unique_: c_int, - invert_: c_int, - ); - pub fn atg_isin_scalar_tensor( - out__: *mut *mut C_tensor, - element_: *mut C_scalar, - test_elements_: *mut C_tensor, - assume_unique_: c_int, - invert_: c_int, - ); - pub fn atg_isin_scalar_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - element_: *mut C_scalar, - test_elements_: *mut C_tensor, - assume_unique_: c_int, - invert_: c_int, - ); - pub fn atg_isin_tensor_scalar( - out__: *mut *mut C_tensor, - elements_: *mut C_tensor, - test_element_: *mut C_scalar, - assume_unique_: c_int, - invert_: c_int, - ); - pub fn atg_isin_tensor_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - elements_: *mut C_tensor, - test_element_: *mut C_scalar, - assume_unique_: c_int, - invert_: c_int, - ); - pub fn atg_isin_tensor_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - elements_: *mut C_tensor, - test_elements_: *mut C_tensor, - assume_unique_: c_int, - invert_: c_int, - ); - pub fn atg_isinf(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_isinf_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_isnan(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_isnan_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_isneginf(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_isneginf_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_isposinf(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_isposinf_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_isreal(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_istft( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - n_fft_: i64, - hop_length_v: i64, - hop_length_null: i8, - win_length_v: i64, - win_length_null: i8, - window_: *mut C_tensor, - center_: c_int, - normalized_: c_int, - onesided_: c_int, - length_v: i64, - length_null: i8, - return_complex_: c_int, - ); - pub fn atg_kaiser_window( - out__: *mut *mut C_tensor, - window_length_: i64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_kaiser_window_beta( - out__: *mut *mut C_tensor, - window_length_: i64, - periodic_: c_int, - beta_: f64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_kaiser_window_beta_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - window_length_: i64, - periodic_: c_int, - beta_: f64, - ); - pub fn atg_kaiser_window_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - window_length_: i64, - ); - pub fn atg_kaiser_window_periodic( - out__: *mut *mut C_tensor, - window_length_: i64, - periodic_: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_kaiser_window_periodic_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - window_length_: i64, - periodic_: c_int, - ); - pub fn atg_kl_div( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - reduction_: i64, - log_target_: c_int, - ); - pub fn atg_kron(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_kron_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_kthvalue( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - k_: i64, - dim_: i64, - keepdim_: c_int, - ); - pub fn atg_kthvalue_values( - out__: *mut *mut C_tensor, - values_: *mut C_tensor, - indices_: *mut C_tensor, - self_: *mut C_tensor, - k_: i64, - dim_: i64, - keepdim_: c_int, - ); - pub fn atg_l1_loss( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - reduction_: i64, - ); - pub fn atg_layer_norm( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - normalized_shape_data: *const i64, - normalized_shape_len: c_int, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - eps_: f64, - cudnn_enable_: c_int, - ); - pub fn atg_lcm(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_lcm_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_lcm_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_ldexp(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_ldexp_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_ldexp_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_le(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_le_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_le_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_le_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_le_tensor_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_le_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_leaky_relu(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_leaky_relu_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_leaky_relu_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - negative_slope_: *mut C_scalar, - self_is_result_: c_int, - ); - pub fn atg_leaky_relu_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - negative_slope_: *mut C_scalar, - self_is_result_: c_int, - ); - pub fn atg_leaky_relu_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_lerp( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - end_: *mut C_tensor, - weight_: *mut C_scalar, - ); - pub fn atg_lerp_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - end_: *mut C_tensor, - weight_: *mut C_scalar, - ); - pub fn atg_lerp_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - end_: *mut C_tensor, - weight_: *mut C_scalar, - ); - pub fn atg_lerp_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - end_: *mut C_tensor, - weight_: *mut C_tensor, - ); - pub fn atg_lerp_tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - end_: *mut C_tensor, - weight_: *mut C_tensor, - ); - pub fn atg_lerp_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - end_: *mut C_tensor, - weight_: *mut C_tensor, - ); - pub fn atg_less(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_less_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_less_equal(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_less_equal_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_less_equal_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_less_equal_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_less_equal_tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_less_equal_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_less_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_less_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_less_tensor_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_less_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_lgamma(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_lgamma_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_lgamma_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_lift(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_lift_fresh(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_lift_fresh_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_lift_fresh_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_lift_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_linalg_cholesky(out__: *mut *mut C_tensor, self_: *mut C_tensor, upper_: c_int); - pub fn atg_linalg_cholesky_ex( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - upper_: c_int, - check_errors_: c_int, - ); - pub fn atg_linalg_cholesky_ex_l( - out__: *mut *mut C_tensor, - L_: *mut C_tensor, - info_: *mut C_tensor, - self_: *mut C_tensor, - upper_: c_int, - check_errors_: c_int, - ); - pub fn atg_linalg_cholesky_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - upper_: c_int, - ); - pub fn atg_linalg_cond(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: *mut C_scalar); - pub fn atg_linalg_cond_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - p_: *mut C_scalar, - ); - pub fn atg_linalg_cond_p_str( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - p_ptr: *const u8, - p_len: c_int, - ); - pub fn atg_linalg_cond_p_str_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - p_ptr: *const u8, - p_len: c_int, - ); - pub fn atg_linalg_cross( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - dim_: i64, - ); - pub fn atg_linalg_cross_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - dim_: i64, - ); - pub fn atg_linalg_det(out__: *mut *mut C_tensor, A_: *mut C_tensor); - pub fn atg_linalg_det_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, A_: *mut C_tensor); - pub fn atg_linalg_diagonal( - out__: *mut *mut C_tensor, - A_: *mut C_tensor, - offset_: i64, - dim1_: i64, - dim2_: i64, - ); - pub fn atg_linalg_eig(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_linalg_eig_out( - out__: *mut *mut C_tensor, - eigenvalues_: *mut C_tensor, - eigenvectors_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_linalg_eigh( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - UPLO_ptr: *const u8, - UPLO_len: c_int, - ); - pub fn atg_linalg_eigh_eigvals( - out__: *mut *mut C_tensor, - eigvals_: *mut C_tensor, - eigvecs_: *mut C_tensor, - self_: *mut C_tensor, - UPLO_ptr: *const u8, - UPLO_len: c_int, - ); - pub fn atg_linalg_eigvals(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_linalg_eigvals_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_linalg_eigvalsh( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - UPLO_ptr: *const u8, - UPLO_len: c_int, - ); - pub fn atg_linalg_eigvalsh_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - UPLO_ptr: *const u8, - UPLO_len: c_int, - ); - pub fn atg_linalg_householder_product( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - tau_: *mut C_tensor, - ); - pub fn atg_linalg_householder_product_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - input_: *mut C_tensor, - tau_: *mut C_tensor, - ); - pub fn atg_linalg_inv(out__: *mut *mut C_tensor, A_: *mut C_tensor); - pub fn atg_linalg_inv_ex(out__: *mut *mut C_tensor, A_: *mut C_tensor, check_errors_: c_int); - pub fn atg_linalg_inv_ex_inverse( - out__: *mut *mut C_tensor, - inverse_: *mut C_tensor, - info_: *mut C_tensor, - A_: *mut C_tensor, - check_errors_: c_int, - ); - pub fn atg_linalg_inv_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, A_: *mut C_tensor); - pub fn atg_linalg_ldl_factor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - hermitian_: c_int, - ); - pub fn atg_linalg_ldl_factor_ex( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - hermitian_: c_int, - check_errors_: c_int, - ); - pub fn atg_linalg_ldl_factor_ex_out( - out__: *mut *mut C_tensor, - LD_: *mut C_tensor, - pivots_: *mut C_tensor, - info_: *mut C_tensor, - self_: *mut C_tensor, - hermitian_: c_int, - check_errors_: c_int, - ); - pub fn atg_linalg_ldl_factor_out( - out__: *mut *mut C_tensor, - LD_: *mut C_tensor, - pivots_: *mut C_tensor, - self_: *mut C_tensor, - hermitian_: c_int, - ); - pub fn atg_linalg_ldl_solve( - out__: *mut *mut C_tensor, - LD_: *mut C_tensor, - pivots_: *mut C_tensor, - B_: *mut C_tensor, - hermitian_: c_int, - ); - pub fn atg_linalg_ldl_solve_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - LD_: *mut C_tensor, - pivots_: *mut C_tensor, - B_: *mut C_tensor, - hermitian_: c_int, - ); - pub fn atg_linalg_lstsq( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - b_: *mut C_tensor, - rcond_v: f64, - rcond_null: i8, - driver_ptr: *const u8, - driver_len: c_int, - ); - pub fn atg_linalg_lstsq_out( - out__: *mut *mut C_tensor, - solution_: *mut C_tensor, - residuals_: *mut C_tensor, - rank_: *mut C_tensor, - singular_values_: *mut C_tensor, - self_: *mut C_tensor, - b_: *mut C_tensor, - rcond_v: f64, - rcond_null: i8, - driver_ptr: *const u8, - driver_len: c_int, - ); - pub fn atg_linalg_lu(out__: *mut *mut C_tensor, A_: *mut C_tensor, pivot_: c_int); - pub fn atg_linalg_lu_factor(out__: *mut *mut C_tensor, A_: *mut C_tensor, pivot_: c_int); - pub fn atg_linalg_lu_factor_ex( - out__: *mut *mut C_tensor, - A_: *mut C_tensor, - pivot_: c_int, - check_errors_: c_int, - ); - pub fn atg_linalg_lu_factor_ex_out( - out__: *mut *mut C_tensor, - LU_: *mut C_tensor, - pivots_: *mut C_tensor, - info_: *mut C_tensor, - A_: *mut C_tensor, - pivot_: c_int, - check_errors_: c_int, - ); - pub fn atg_linalg_lu_factor_out( - out__: *mut *mut C_tensor, - LU_: *mut C_tensor, - pivots_: *mut C_tensor, - A_: *mut C_tensor, - pivot_: c_int, - ); - pub fn atg_linalg_lu_out( - out__: *mut *mut C_tensor, - P_: *mut C_tensor, - L_: *mut C_tensor, - U_: *mut C_tensor, - A_: *mut C_tensor, - pivot_: c_int, - ); - pub fn atg_linalg_lu_solve( - out__: *mut *mut C_tensor, - LU_: *mut C_tensor, - pivots_: *mut C_tensor, - B_: *mut C_tensor, - left_: c_int, - adjoint_: c_int, - ); - pub fn atg_linalg_lu_solve_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - LU_: *mut C_tensor, - pivots_: *mut C_tensor, - B_: *mut C_tensor, - left_: c_int, - adjoint_: c_int, - ); - pub fn atg_linalg_matmul( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_linalg_matmul_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_linalg_matrix_exp(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_linalg_matrix_exp_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_linalg_matrix_power(out__: *mut *mut C_tensor, self_: *mut C_tensor, n_: i64); - pub fn atg_linalg_matrix_power_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - n_: i64, - ); - pub fn atg_linalg_matrix_rank( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - tol_: f64, - hermitian_: c_int, - ); - pub fn atg_linalg_matrix_rank_atol_rtol_float( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - atol_v: f64, - atol_null: i8, - rtol_v: f64, - rtol_null: i8, - hermitian_: c_int, - ); - pub fn atg_linalg_matrix_rank_atol_rtol_float_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - atol_v: f64, - atol_null: i8, - rtol_v: f64, - rtol_null: i8, - hermitian_: c_int, - ); - pub fn atg_linalg_matrix_rank_atol_rtol_tensor( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - atol_: *mut C_tensor, - rtol_: *mut C_tensor, - hermitian_: c_int, - ); - pub fn atg_linalg_matrix_rank_atol_rtol_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - input_: *mut C_tensor, - atol_: *mut C_tensor, - rtol_: *mut C_tensor, - hermitian_: c_int, - ); - pub fn atg_linalg_matrix_rank_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - tol_: f64, - hermitian_: c_int, - ); - pub fn atg_linalg_matrix_rank_out_tol_tensor( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - input_: *mut C_tensor, - tol_: *mut C_tensor, - hermitian_: c_int, - ); - pub fn atg_linalg_matrix_rank_tol_tensor( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - tol_: *mut C_tensor, - hermitian_: c_int, - ); - pub fn atg_linalg_multi_dot( - out__: *mut *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ); - pub fn atg_linalg_multi_dot_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ); - pub fn atg_linalg_norm( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - ord_: *mut C_scalar, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - dtype_: c_int, - ); - pub fn atg_linalg_norm_ord_str( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - ord_ptr: *const u8, - ord_len: c_int, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - dtype_: c_int, - ); - pub fn atg_linalg_norm_ord_str_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ord_ptr: *const u8, - ord_len: c_int, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - dtype_: c_int, - ); - pub fn atg_linalg_norm_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ord_: *mut C_scalar, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - dtype_: c_int, - ); - pub fn atg_linalg_pinv( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - rcond_: f64, - hermitian_: c_int, - ); - pub fn atg_linalg_pinv_atol_rtol_float( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - atol_v: f64, - atol_null: i8, - rtol_v: f64, - rtol_null: i8, - hermitian_: c_int, - ); - pub fn atg_linalg_pinv_atol_rtol_float_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - atol_v: f64, - atol_null: i8, - rtol_v: f64, - rtol_null: i8, - hermitian_: c_int, - ); - pub fn atg_linalg_pinv_atol_rtol_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - atol_: *mut C_tensor, - rtol_: *mut C_tensor, - hermitian_: c_int, - ); - pub fn atg_linalg_pinv_atol_rtol_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - atol_: *mut C_tensor, - rtol_: *mut C_tensor, - hermitian_: c_int, - ); - pub fn atg_linalg_pinv_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - rcond_: f64, - hermitian_: c_int, - ); - pub fn atg_linalg_pinv_out_rcond_tensor( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - rcond_: *mut C_tensor, - hermitian_: c_int, - ); - pub fn atg_linalg_pinv_rcond_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - rcond_: *mut C_tensor, - hermitian_: c_int, - ); - pub fn atg_linalg_qr( - out__: *mut *mut C_tensor, - A_: *mut C_tensor, - mode_ptr: *const u8, - mode_len: c_int, - ); - pub fn atg_linalg_qr_out( - out__: *mut *mut C_tensor, - Q_: *mut C_tensor, - R_: *mut C_tensor, - A_: *mut C_tensor, - mode_ptr: *const u8, - mode_len: c_int, - ); - pub fn atg_linalg_slogdet(out__: *mut *mut C_tensor, A_: *mut C_tensor); - pub fn atg_linalg_slogdet_out( - out__: *mut *mut C_tensor, - sign_: *mut C_tensor, - logabsdet_: *mut C_tensor, - A_: *mut C_tensor, - ); - pub fn atg_linalg_solve( - out__: *mut *mut C_tensor, - A_: *mut C_tensor, - B_: *mut C_tensor, - left_: c_int, - ); - pub fn atg_linalg_solve_ex( - out__: *mut *mut C_tensor, - A_: *mut C_tensor, - B_: *mut C_tensor, - left_: c_int, - check_errors_: c_int, - ); - pub fn atg_linalg_solve_ex_out( - out__: *mut *mut C_tensor, - result_: *mut C_tensor, - info_: *mut C_tensor, - A_: *mut C_tensor, - B_: *mut C_tensor, - left_: c_int, - check_errors_: c_int, - ); - pub fn atg_linalg_solve_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - A_: *mut C_tensor, - B_: *mut C_tensor, - left_: c_int, - ); - pub fn atg_linalg_solve_triangular( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - B_: *mut C_tensor, - upper_: c_int, - left_: c_int, - unitriangular_: c_int, - ); - pub fn atg_linalg_solve_triangular_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - B_: *mut C_tensor, - upper_: c_int, - left_: c_int, - unitriangular_: c_int, - ); - pub fn atg_linalg_svd( - out__: *mut *mut C_tensor, - A_: *mut C_tensor, - full_matrices_: c_int, - driver_ptr: *const u8, - driver_len: c_int, - ); - pub fn atg_linalg_svd_u( - out__: *mut *mut C_tensor, - U_: *mut C_tensor, - S_: *mut C_tensor, - Vh_: *mut C_tensor, - A_: *mut C_tensor, - full_matrices_: c_int, - driver_ptr: *const u8, - driver_len: c_int, - ); - pub fn atg_linalg_svdvals( - out__: *mut *mut C_tensor, - A_: *mut C_tensor, - driver_ptr: *const u8, - driver_len: c_int, - ); - pub fn atg_linalg_svdvals_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - A_: *mut C_tensor, - driver_ptr: *const u8, - driver_len: c_int, - ); - pub fn atg_linalg_tensorinv(out__: *mut *mut C_tensor, self_: *mut C_tensor, ind_: i64); - pub fn atg_linalg_tensorinv_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ind_: i64, - ); - pub fn atg_linalg_tensorsolve( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - dims_data: *const i64, - dims_len: c_int, - ); - pub fn atg_linalg_tensorsolve_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - dims_data: *const i64, - dims_len: c_int, - ); - pub fn atg_linalg_vander(out__: *mut *mut C_tensor, x_: *mut C_tensor, n_v: i64, n_null: i8); - pub fn atg_linalg_vecdot( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - y_: *mut C_tensor, - dim_: i64, - ); - pub fn atg_linalg_vecdot_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - y_: *mut C_tensor, - dim_: i64, - ); - pub fn atg_linear( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - ); - pub fn atg_linear_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - ); - pub fn atg_linspace( - out__: *mut *mut C_tensor, - start_: *mut C_scalar, - end_: *mut C_scalar, - steps_: i64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_linspace_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - start_: *mut C_scalar, - end_: *mut C_scalar, - steps_: i64, - ); - pub fn atg_linspace_scalar_tensor( - out__: *mut *mut C_tensor, - start_: *mut C_scalar, - end_: *mut C_tensor, - steps_: i64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_linspace_scalar_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - start_: *mut C_scalar, - end_: *mut C_tensor, - steps_: i64, - ); - pub fn atg_linspace_tensor_scalar( - out__: *mut *mut C_tensor, - start_: *mut C_tensor, - end_: *mut C_scalar, - steps_: i64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_linspace_tensor_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - start_: *mut C_tensor, - end_: *mut C_scalar, - steps_: i64, - ); - pub fn atg_linspace_tensor_tensor( - out__: *mut *mut C_tensor, - start_: *mut C_tensor, - end_: *mut C_tensor, - steps_: i64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_linspace_tensor_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - start_: *mut C_tensor, - end_: *mut C_tensor, - steps_: i64, - ); - pub fn atg_log(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_log10(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_log10_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_log10_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_log1p(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_log1p_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_log1p_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_log2(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_log2_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_log2_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_log_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_log_normal(out__: *mut *mut C_tensor, self_: *mut C_tensor, mean_: f64, std_: f64); - pub fn atg_log_normal_(out__: *mut *mut C_tensor, self_: *mut C_tensor, mean_: f64, std_: f64); - pub fn atg_log_normal_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mean_: f64, - std_: f64, - ); - pub fn atg_log_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_log_sigmoid(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_log_sigmoid_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - buffer_: *mut C_tensor, - ); - pub fn atg_log_sigmoid_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - buffer_: *mut C_tensor, - ); - pub fn atg_log_sigmoid_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_log_softmax( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - dtype_: c_int, - ); - pub fn atg_log_softmax_int_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - dtype_: c_int, - ); - pub fn atg_logaddexp(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_logaddexp2(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_logaddexp2_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_logaddexp_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_logcumsumexp(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64); - pub fn atg_logcumsumexp_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - ); - pub fn atg_logdet(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_logical_and(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_logical_and_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_logical_and_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_logical_not(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_logical_not_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_logical_not_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_logical_or(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_logical_or_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_logical_or_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_logical_xor(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_logical_xor_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_logical_xor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_logit(out__: *mut *mut C_tensor, self_: *mut C_tensor, eps_v: f64, eps_null: i8); - pub fn atg_logit_(out__: *mut *mut C_tensor, self_: *mut C_tensor, eps_v: f64, eps_null: i8); - pub fn atg_logit_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - eps_v: f64, - eps_null: i8, - ); - pub fn atg_logit_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - eps_v: f64, - eps_null: i8, - ); - pub fn atg_logit_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - eps_v: f64, - eps_null: i8, - ); - pub fn atg_logspace( - out__: *mut *mut C_tensor, - start_: *mut C_scalar, - end_: *mut C_scalar, - steps_: i64, - base_: f64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_logspace_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - start_: *mut C_scalar, - end_: *mut C_scalar, - steps_: i64, - base_: f64, - ); - pub fn atg_logspace_scalar_tensor( - out__: *mut *mut C_tensor, - start_: *mut C_scalar, - end_: *mut C_tensor, - steps_: i64, - base_: f64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_logspace_scalar_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - start_: *mut C_scalar, - end_: *mut C_tensor, - steps_: i64, - base_: f64, - ); - pub fn atg_logspace_tensor_scalar( - out__: *mut *mut C_tensor, - start_: *mut C_tensor, - end_: *mut C_scalar, - steps_: i64, - base_: f64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_logspace_tensor_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - start_: *mut C_tensor, - end_: *mut C_scalar, - steps_: i64, - base_: f64, - ); - pub fn atg_logspace_tensor_tensor( - out__: *mut *mut C_tensor, - start_: *mut C_tensor, - end_: *mut C_tensor, - steps_: i64, - base_: f64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_logspace_tensor_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - start_: *mut C_tensor, - end_: *mut C_tensor, - steps_: i64, - base_: f64, - ); - pub fn atg_logsumexp( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - ); - pub fn atg_logsumexp_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - ); - pub fn atg_lstm( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - hx_data: *const *mut C_tensor, - hx_len: c_int, - params_data: *const *mut C_tensor, - params_len: c_int, - has_biases_: c_int, - num_layers_: i64, - dropout_: f64, - train_: c_int, - bidirectional_: c_int, - batch_first_: c_int, - ); - pub fn atg_lstm_cell( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - hx_data: *const *mut C_tensor, - hx_len: c_int, - w_ih_: *mut C_tensor, - w_hh_: *mut C_tensor, - b_ih_: *mut C_tensor, - b_hh_: *mut C_tensor, - ); - pub fn atg_lstm_data( - out__: *mut *mut C_tensor, - data_: *mut C_tensor, - batch_sizes_: *mut C_tensor, - hx_data: *const *mut C_tensor, - hx_len: c_int, - params_data: *const *mut C_tensor, - params_len: c_int, - has_biases_: c_int, - num_layers_: i64, - dropout_: f64, - train_: c_int, - bidirectional_: c_int, - ); - pub fn atg_lstm_mps_backward( - out0_: *mut C_tensor, - out1_data: *const *mut C_tensor, - out1_len: c_int, - out2_data: *const *mut C_tensor, - out2_len: c_int, - grad_y_: *mut C_tensor, - grad_hy_: *mut C_tensor, - grad_cy_: *mut C_tensor, - z_state_: *mut C_tensor, - cell_state_fwd_: *mut C_tensor, - input_: *mut C_tensor, - layersOutputs_: *mut C_tensor, - hx_data: *const *mut C_tensor, - hx_len: c_int, - params_data: *const *mut C_tensor, - params_len: c_int, - has_biases_: c_int, - num_layers_: i64, - dropout_: f64, - train_: c_int, - bidirectional_: c_int, - batch_first_: c_int, - ); - pub fn atg_lt(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_lt_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_lt_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_lt_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_lt_tensor_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_lt_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_lu_solve( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - LU_data_: *mut C_tensor, - LU_pivots_: *mut C_tensor, - ); - pub fn atg_lu_solve_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - LU_data_: *mut C_tensor, - LU_pivots_: *mut C_tensor, - ); - pub fn atg_lu_unpack( - out__: *mut *mut C_tensor, - LU_data_: *mut C_tensor, - LU_pivots_: *mut C_tensor, - unpack_data_: c_int, - unpack_pivots_: c_int, - ); - pub fn atg_lu_unpack_out( - out__: *mut *mut C_tensor, - P_: *mut C_tensor, - L_: *mut C_tensor, - U_: *mut C_tensor, - LU_data_: *mut C_tensor, - LU_pivots_: *mut C_tensor, - unpack_data_: c_int, - unpack_pivots_: c_int, - ); - pub fn atg_margin_ranking_loss( - out__: *mut *mut C_tensor, - input1_: *mut C_tensor, - input2_: *mut C_tensor, - target_: *mut C_tensor, - margin_: f64, - reduction_: i64, - ); - pub fn atg_masked_fill( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mask_: *mut C_tensor, - value_: *mut C_scalar, - ); - pub fn atg_masked_fill_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mask_: *mut C_tensor, - value_: *mut C_scalar, - ); - pub fn atg_masked_fill_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mask_: *mut C_tensor, - value_: *mut C_scalar, - ); - pub fn atg_masked_fill_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mask_: *mut C_tensor, - value_: *mut C_tensor, - ); - pub fn atg_masked_fill_tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mask_: *mut C_tensor, - value_: *mut C_tensor, - ); - pub fn atg_masked_fill_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mask_: *mut C_tensor, - value_: *mut C_tensor, - ); - pub fn atg_masked_scatter( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mask_: *mut C_tensor, - source_: *mut C_tensor, - ); - pub fn atg_masked_scatter_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mask_: *mut C_tensor, - source_: *mut C_tensor, - ); - pub fn atg_masked_scatter_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - mask_: *mut C_tensor, - sizes_data: *const i64, - sizes_len: c_int, - ); - pub fn atg_masked_scatter_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mask_: *mut C_tensor, - source_: *mut C_tensor, - ); - pub fn atg_masked_select(out__: *mut *mut C_tensor, self_: *mut C_tensor, mask_: *mut C_tensor); - pub fn atg_masked_select_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - input_: *mut C_tensor, - mask_: *mut C_tensor, - ); - pub fn atg_masked_select_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mask_: *mut C_tensor, - ); - pub fn atg_matmul(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_matmul_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_matrix_exp(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_matrix_exp_backward( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - grad_: *mut C_tensor, - ); - pub fn atg_matrix_h(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_matrix_power(out__: *mut *mut C_tensor, self_: *mut C_tensor, n_: i64); - pub fn atg_matrix_power_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - n_: i64, - ); - pub fn atg_max(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_max_dim(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, keepdim_: c_int); - pub fn atg_max_dim_max( - out__: *mut *mut C_tensor, - max_: *mut C_tensor, - max_values_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - keepdim_: c_int, - ); - pub fn atg_max_other(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_max_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_max_pool1d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_max_pool1d_with_indices( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_max_pool2d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_max_pool2d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_max_pool2d_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_max_pool2d_with_indices( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_max_pool2d_with_indices_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - indices_: *mut C_tensor, - ); - pub fn atg_max_pool2d_with_indices_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - indices_: *mut C_tensor, - ); - pub fn atg_max_pool2d_with_indices_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - indices_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_max_pool3d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_max_pool3d_with_indices( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_max_pool3d_with_indices_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - indices_: *mut C_tensor, - ); - pub fn atg_max_pool3d_with_indices_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - indices_: *mut C_tensor, - ); - pub fn atg_max_pool3d_with_indices_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - indices_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_max_unary_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_max_unpool2d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - indices_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - ); - pub fn atg_max_unpool2d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - indices_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - ); - pub fn atg_max_unpool3d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - indices_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_max_unpool3d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - indices_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_maximum(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_maximum_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_mean(out__: *mut *mut C_tensor, self_: *mut C_tensor, dtype_: c_int); - pub fn atg_mean_dim( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - dtype_: c_int, - ); - pub fn atg_mean_dtype_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dtype_: c_int, - ); - pub fn atg_mean_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - dtype_: c_int, - ); - pub fn atg_median(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_median_dim( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - keepdim_: c_int, - ); - pub fn atg_median_dim_values( - out__: *mut *mut C_tensor, - values_: *mut C_tensor, - indices_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - keepdim_: c_int, - ); - pub fn atg_median_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_meshgrid( - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ) -> *mut *mut C_tensor; - pub fn atg_meshgrid_indexing( - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - indexing_ptr: *const u8, - indexing_len: c_int, - ) -> *mut *mut C_tensor; - pub fn atg_mh(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_min(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_min_dim(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, keepdim_: c_int); - pub fn atg_min_dim_min( - out__: *mut *mut C_tensor, - min_: *mut C_tensor, - min_indices_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - keepdim_: c_int, - ); - pub fn atg_min_other(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_min_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_min_unary_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_minimum(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_minimum_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_miopen_batch_norm( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - training_: c_int, - exponential_average_factor_: f64, - epsilon_: f64, - ); - pub fn atg_miopen_batch_norm_backward( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - grad_output_: *mut C_tensor, - weight_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - save_mean_: *mut C_tensor, - save_var_: *mut C_tensor, - epsilon_: f64, - ); - pub fn atg_miopen_batch_norm_backward_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - out2_: *mut C_tensor, - input_: *mut C_tensor, - grad_output_: *mut C_tensor, - weight_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - save_mean_: *mut C_tensor, - save_var_: *mut C_tensor, - epsilon_: f64, - ); - pub fn atg_miopen_batch_norm_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - out2_: *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - training_: c_int, - exponential_average_factor_: f64, - epsilon_: f64, - ); - pub fn atg_miopen_convolution( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - benchmark_: c_int, - deterministic_: c_int, - ); - pub fn atg_miopen_convolution_add_relu( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - z_: *mut C_tensor, - alpha_: *mut C_scalar, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - ); - pub fn atg_miopen_convolution_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - benchmark_: c_int, - deterministic_: c_int, - ); - pub fn atg_miopen_convolution_relu( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - ); - pub fn atg_miopen_convolution_transpose( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - output_padding_data: *const i64, - output_padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - benchmark_: c_int, - deterministic_: c_int, - ); - pub fn atg_miopen_convolution_transpose_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - output_padding_data: *const i64, - output_padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - benchmark_: c_int, - deterministic_: c_int, - ); - pub fn atg_miopen_ctc_loss( - out__: *mut *mut C_tensor, - log_probs_: *mut C_tensor, - targets_: *mut C_tensor, - input_lengths_data: *const i64, - input_lengths_len: c_int, - target_lengths_data: *const i64, - target_lengths_len: c_int, - blank_: i64, - deterministic_: c_int, - zero_infinity_: c_int, - ); - pub fn atg_miopen_ctc_loss_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - log_probs_: *mut C_tensor, - targets_: *mut C_tensor, - input_lengths_data: *const i64, - input_lengths_len: c_int, - target_lengths_data: *const i64, - target_lengths_len: c_int, - blank_: i64, - deterministic_: c_int, - zero_infinity_: c_int, - ); - pub fn atg_miopen_ctc_loss_tensor( - out__: *mut *mut C_tensor, - log_probs_: *mut C_tensor, - targets_: *mut C_tensor, - input_lengths_: *mut C_tensor, - target_lengths_: *mut C_tensor, - blank_: i64, - deterministic_: c_int, - zero_infinity_: c_int, - ); - pub fn atg_miopen_depthwise_convolution( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - benchmark_: c_int, - deterministic_: c_int, - ); - pub fn atg_miopen_depthwise_convolution_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - benchmark_: c_int, - deterministic_: c_int, - ); - pub fn atg_miopen_rnn( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_data: *const *mut C_tensor, - weight_len: c_int, - weight_stride0_: i64, - hx_: *mut C_tensor, - cx_: *mut C_tensor, - mode_: i64, - hidden_size_: i64, - num_layers_: i64, - batch_first_: c_int, - dropout_: f64, - train_: c_int, - bidirectional_: c_int, - batch_sizes_data: *const i64, - batch_sizes_len: c_int, - dropout_state_: *mut C_tensor, - ); - pub fn atg_miopen_rnn_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - out2_: *mut C_tensor, - out3_: *mut C_tensor, - out4_: *mut C_tensor, - input_: *mut C_tensor, - weight_data: *const *mut C_tensor, - weight_len: c_int, - weight_stride0_: i64, - hx_: *mut C_tensor, - cx_: *mut C_tensor, - mode_: i64, - hidden_size_: i64, - num_layers_: i64, - batch_first_: c_int, - dropout_: f64, - train_: c_int, - bidirectional_: c_int, - batch_sizes_data: *const i64, - batch_sizes_len: c_int, - dropout_state_: *mut C_tensor, - ); - pub fn atg_mish(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_mish_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_mish_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_mish_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_mkldnn_adaptive_avg_pool2d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - ); - pub fn atg_mkldnn_adaptive_avg_pool2d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_mkldnn_adaptive_avg_pool2d_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_mkldnn_adaptive_avg_pool2d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - ); - pub fn atg_mkldnn_convolution( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - ); - pub fn atg_mkldnn_convolution_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - ); - pub fn atg_mkldnn_linear( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - ); - pub fn atg_mkldnn_linear_backward_input( - out__: *mut *mut C_tensor, - input_size_data: *const i64, - input_size_len: c_int, - grad_output_: *mut C_tensor, - weight_: *mut C_tensor, - ); - pub fn atg_mkldnn_linear_backward_input_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - input_size_data: *const i64, - input_size_len: c_int, - grad_output_: *mut C_tensor, - weight_: *mut C_tensor, - ); - pub fn atg_mkldnn_linear_backward_weights( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_defined_: c_int, - ); - pub fn atg_mkldnn_linear_backward_weights_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - grad_output_: *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_defined_: c_int, - ); - pub fn atg_mkldnn_linear_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - ); - pub fn atg_mkldnn_max_pool2d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_mkldnn_max_pool2d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - output_: *mut C_tensor, - input_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_mkldnn_max_pool2d_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_output_: *mut C_tensor, - output_: *mut C_tensor, - input_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_mkldnn_max_pool2d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_mkldnn_max_pool3d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_mkldnn_max_pool3d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - output_: *mut C_tensor, - input_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_mkldnn_max_pool3d_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_output_: *mut C_tensor, - output_: *mut C_tensor, - input_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_mkldnn_max_pool3d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_mkldnn_reorder_conv2d_weight( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - input_size_data: *const i64, - input_size_len: c_int, - ); - pub fn atg_mkldnn_reorder_conv2d_weight_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - input_size_data: *const i64, - input_size_len: c_int, - ); - pub fn atg_mkldnn_reorder_conv3d_weight( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - input_size_data: *const i64, - input_size_len: c_int, - ); - pub fn atg_mkldnn_reorder_conv3d_weight_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - stride_data: *const i64, - stride_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - groups_: i64, - input_size_data: *const i64, - input_size_len: c_int, - ); - pub fn atg_mkldnn_rnn_layer( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight0_: *mut C_tensor, - weight1_: *mut C_tensor, - weight2_: *mut C_tensor, - weight3_: *mut C_tensor, - hx__: *mut C_tensor, - cx__: *mut C_tensor, - reverse_: c_int, - batch_sizes_data: *const i64, - batch_sizes_len: c_int, - mode_: i64, - hidden_size_: i64, - num_layers_: i64, - has_biases_: c_int, - bidirectional_: c_int, - batch_first_: c_int, - train_: c_int, - ); - pub fn atg_mkldnn_rnn_layer_backward( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight1_: *mut C_tensor, - weight2_: *mut C_tensor, - weight3_: *mut C_tensor, - weight4_: *mut C_tensor, - hx__: *mut C_tensor, - cx_tmp_: *mut C_tensor, - output_: *mut C_tensor, - hy__: *mut C_tensor, - cy__: *mut C_tensor, - grad_output_: *mut C_tensor, - grad_hy_: *mut C_tensor, - grad_cy_: *mut C_tensor, - reverse_: c_int, - mode_: i64, - hidden_size_: i64, - num_layers_: i64, - has_biases_: c_int, - train_: c_int, - bidirectional_: c_int, - batch_sizes_data: *const i64, - batch_sizes_len: c_int, - batch_first_: c_int, - workspace_: *mut C_tensor, - ); - pub fn atg_mkldnn_rnn_layer_backward_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - out2_: *mut C_tensor, - out3_: *mut C_tensor, - out4_: *mut C_tensor, - out5_: *mut C_tensor, - out6_: *mut C_tensor, - input_: *mut C_tensor, - weight1_: *mut C_tensor, - weight2_: *mut C_tensor, - weight3_: *mut C_tensor, - weight4_: *mut C_tensor, - hx__: *mut C_tensor, - cx_tmp_: *mut C_tensor, - output_: *mut C_tensor, - hy__: *mut C_tensor, - cy__: *mut C_tensor, - grad_output_: *mut C_tensor, - grad_hy_: *mut C_tensor, - grad_cy_: *mut C_tensor, - reverse_: c_int, - mode_: i64, - hidden_size_: i64, - num_layers_: i64, - has_biases_: c_int, - train_: c_int, - bidirectional_: c_int, - batch_sizes_data: *const i64, - batch_sizes_len: c_int, - batch_first_: c_int, - workspace_: *mut C_tensor, - ); - pub fn atg_mkldnn_rnn_layer_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - out2_: *mut C_tensor, - out3_: *mut C_tensor, - input_: *mut C_tensor, - weight0_: *mut C_tensor, - weight1_: *mut C_tensor, - weight2_: *mut C_tensor, - weight3_: *mut C_tensor, - hx__: *mut C_tensor, - cx__: *mut C_tensor, - reverse_: c_int, - batch_sizes_data: *const i64, - batch_sizes_len: c_int, - mode_: i64, - hidden_size_: i64, - num_layers_: i64, - has_biases_: c_int, - bidirectional_: c_int, - batch_first_: c_int, - train_: c_int, - ); - pub fn atg_mm(out__: *mut *mut C_tensor, self_: *mut C_tensor, mat2_: *mut C_tensor); - pub fn atg_mm_dtype( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mat2_: *mut C_tensor, - out_dtype_: c_int, - ); - pub fn atg_mm_dtype_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mat2_: *mut C_tensor, - out_dtype_: c_int, - ); - pub fn atg_mm_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mat2_: *mut C_tensor, - ); - pub fn atg_mode(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, keepdim_: c_int); - pub fn atg_mode_values( - out__: *mut *mut C_tensor, - values_: *mut C_tensor, - indices_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - keepdim_: c_int, - ); - pub fn atg_moveaxis( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - source_data: *const i64, - source_len: c_int, - destination_data: *const i64, - destination_len: c_int, - ); - pub fn atg_moveaxis_int( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - source_: i64, - destination_: i64, - ); - pub fn atg_movedim( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - source_data: *const i64, - source_len: c_int, - destination_data: *const i64, - destination_len: c_int, - ); - pub fn atg_movedim_int( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - source_: i64, - destination_: i64, - ); - pub fn atg_mse_loss( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - reduction_: i64, - ); - pub fn atg_mse_loss_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - reduction_: i64, - ); - pub fn atg_mse_loss_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - reduction_: i64, - ); - pub fn atg_mse_loss_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - reduction_: i64, - ); - pub fn atg_msort(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_msort_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_mt(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_mul(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_mul_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_mul_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_mul_scalar(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_mul_scalar_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_mul_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_multi_margin_loss_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - p_: *mut C_scalar, - margin_: *mut C_scalar, - weight_: *mut C_tensor, - reduction_: i64, - ); - pub fn atg_multi_margin_loss_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - p_: *mut C_scalar, - margin_: *mut C_scalar, - weight_: *mut C_tensor, - reduction_: i64, - ); - pub fn atg_multilabel_margin_loss( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - reduction_: i64, - ); - pub fn atg_multilabel_margin_loss_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - reduction_: i64, - is_target_: *mut C_tensor, - ); - pub fn atg_multilabel_margin_loss_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - reduction_: i64, - is_target_: *mut C_tensor, - ); - pub fn atg_multilabel_margin_loss_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - reduction_: i64, - ); - pub fn atg_multinomial( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - num_samples_: i64, - replacement_: c_int, - ); - pub fn atg_multinomial_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - num_samples_: i64, - replacement_: c_int, - ); - pub fn atg_multiply(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_multiply_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_multiply_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_multiply_scalar( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_multiply_scalar_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_mv(out__: *mut *mut C_tensor, self_: *mut C_tensor, vec_: *mut C_tensor); - pub fn atg_mv_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - vec_: *mut C_tensor, - ); - pub fn atg_mvlgamma(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: i64); - pub fn atg_mvlgamma_(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: i64); - pub fn atg_mvlgamma_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - p_: i64, - ); - pub fn atg_nan_to_num( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - nan_v: f64, - nan_null: i8, - posinf_v: f64, - posinf_null: i8, - neginf_v: f64, - neginf_null: i8, - ); - pub fn atg_nan_to_num_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - nan_v: f64, - nan_null: i8, - posinf_v: f64, - posinf_null: i8, - neginf_v: f64, - neginf_null: i8, - ); - pub fn atg_nan_to_num_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - nan_v: f64, - nan_null: i8, - posinf_v: f64, - posinf_null: i8, - neginf_v: f64, - neginf_null: i8, - ); - pub fn atg_nanmean( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - dtype_: c_int, - ); - pub fn atg_nanmean_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - dtype_: c_int, - ); - pub fn atg_nanmedian(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_nanmedian_dim( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - keepdim_: c_int, - ); - pub fn atg_nanmedian_dim_values( - out__: *mut *mut C_tensor, - values_: *mut C_tensor, - indices_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - keepdim_: c_int, - ); - pub fn atg_nanmedian_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_nanquantile( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - q_: *mut C_tensor, - dim_v: i64, - dim_null: i8, - keepdim_: c_int, - interpolation_ptr: *const u8, - interpolation_len: c_int, - ); - pub fn atg_nanquantile_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - q_: *mut C_tensor, - dim_v: i64, - dim_null: i8, - keepdim_: c_int, - interpolation_ptr: *const u8, - interpolation_len: c_int, - ); - pub fn atg_nanquantile_scalar( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - q_: f64, - dim_v: i64, - dim_null: i8, - keepdim_: c_int, - interpolation_ptr: *const u8, - interpolation_len: c_int, - ); - pub fn atg_nanquantile_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - q_: f64, - dim_v: i64, - dim_null: i8, - keepdim_: c_int, - interpolation_ptr: *const u8, - interpolation_len: c_int, - ); - pub fn atg_nansum( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - dtype_: c_int, - ); - pub fn atg_nansum_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - dtype_: c_int, - ); - pub fn atg_narrow( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - start_: i64, - length_: i64, - ); - pub fn atg_narrow_copy( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - start_: i64, - length_: i64, - ); - pub fn atg_narrow_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - start_: i64, - length_: i64, - ); - pub fn atg_narrow_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - start_: *mut C_tensor, - length_: i64, - ); - pub fn atg_native_batch_norm( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - training_: c_int, - momentum_: f64, - eps_: f64, - ); - pub fn atg_native_batch_norm_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - save_mean_: *mut C_tensor, - save_invstd_: *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - running_mean_: *mut C_tensor, - running_var_: *mut C_tensor, - training_: c_int, - momentum_: f64, - eps_: f64, - ); - pub fn atg_native_channel_shuffle( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - groups_: i64, - ); - pub fn atg_native_dropout( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - p_: f64, - train_: c_int, - ); - pub fn atg_native_dropout_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - mask_: *mut C_tensor, - scale_: f64, - ); - pub fn atg_native_dropout_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_output_: *mut C_tensor, - mask_: *mut C_tensor, - scale_: f64, - ); - pub fn atg_native_dropout_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - input_: *mut C_tensor, - p_: f64, - train_: c_int, - ); - pub fn atg_native_group_norm( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - n_: i64, - C_: i64, - HxW_: i64, - group_: i64, - eps_: f64, - ); - pub fn atg_native_group_norm_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - out2_: *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - n_: i64, - C_: i64, - HxW_: i64, - group_: i64, - eps_: f64, - ); - pub fn atg_native_layer_norm( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - normalized_shape_data: *const i64, - normalized_shape_len: c_int, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - eps_: f64, - ); - pub fn atg_native_layer_norm_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - out2_: *mut C_tensor, - input_: *mut C_tensor, - normalized_shape_data: *const i64, - normalized_shape_len: c_int, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - eps_: f64, - ); - pub fn atg_native_norm(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_native_norm_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_native_norm_scalaropt_dim_dtype( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - p_: *mut C_scalar, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - dtype_: c_int, - ); - pub fn atg_native_norm_scalaropt_dim_dtype_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - p_: *mut C_scalar, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - dtype_: c_int, - ); - pub fn atg_ne(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_ne_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_ne_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_ne_tensor(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_ne_tensor_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_ne_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_neg(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_neg_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_neg_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_negative(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_negative_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_negative_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_nested_to_padded_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - padding_: f64, - output_size_data: *const i64, - output_size_len: c_int, - ); - pub fn atg_new_empty( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_new_empty_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg_new_empty_strided( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_new_empty_strided_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - ); - pub fn atg_new_full( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - fill_value_: *mut C_scalar, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_new_full_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - fill_value_: *mut C_scalar, - ); - pub fn atg_new_ones( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_new_ones_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg_new_zeros( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_new_zeros_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg_nextafter(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_nextafter_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_nextafter_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_nll_loss( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - weight_: *mut C_tensor, - reduction_: i64, - ignore_index_: i64, - ); - pub fn atg_nll_loss2d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - weight_: *mut C_tensor, - reduction_: i64, - ignore_index_: i64, - ); - pub fn atg_nll_loss2d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - weight_: *mut C_tensor, - reduction_: i64, - ignore_index_: i64, - total_weight_: *mut C_tensor, - ); - pub fn atg_nll_loss2d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - weight_: *mut C_tensor, - reduction_: i64, - ignore_index_: i64, - total_weight_: *mut C_tensor, - ); - pub fn atg_nll_loss2d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - weight_: *mut C_tensor, - reduction_: i64, - ignore_index_: i64, - ); - pub fn atg_nll_loss_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - weight_: *mut C_tensor, - reduction_: i64, - ignore_index_: i64, - total_weight_: *mut C_tensor, - ); - pub fn atg_nll_loss_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - weight_: *mut C_tensor, - reduction_: i64, - ignore_index_: i64, - total_weight_: *mut C_tensor, - ); - pub fn atg_nll_loss_nd( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - weight_: *mut C_tensor, - reduction_: i64, - ignore_index_: i64, - ); - pub fn atg_nll_loss_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - weight_: *mut C_tensor, - reduction_: i64, - ignore_index_: i64, - ); - pub fn atg_nonzero(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_nonzero_numpy(self_: *mut C_tensor) -> *mut *mut C_tensor; - pub fn atg_nonzero_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_nonzero_static( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_: i64, - fill_value_: i64, - ); - pub fn atg_nonzero_static_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - size_: i64, - fill_value_: i64, - ); - pub fn atg_norm(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_norm_dtype_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - p_: *mut C_scalar, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - dtype_: c_int, - ); - pub fn atg_norm_except_dim(out__: *mut *mut C_tensor, v_: *mut C_tensor, pow_: i64, dim_: i64); - pub fn atg_norm_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - p_: *mut C_scalar, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - ); - pub fn atg_norm_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_norm_scalaropt_dim( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - p_: *mut C_scalar, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - ); - pub fn atg_norm_scalaropt_dim_dtype( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - p_: *mut C_scalar, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - dtype_: c_int, - ); - pub fn atg_norm_scalaropt_dtype( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - p_: *mut C_scalar, - dtype_: c_int, - ); - pub fn atg_norm_scalaropt_dtype_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - p_: *mut C_scalar, - dtype_: c_int, - ); - pub fn atg_normal_(out__: *mut *mut C_tensor, self_: *mut C_tensor, mean_: f64, std_: f64); - pub fn atg_normal_functional( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mean_: f64, - std_: f64, - ); - pub fn atg_not_equal(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_not_equal_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_not_equal_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_not_equal_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_not_equal_tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_not_equal_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_nuclear_norm(out__: *mut *mut C_tensor, self_: *mut C_tensor, keepdim_: c_int); - pub fn atg_nuclear_norm_dim( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - ); - pub fn atg_nuclear_norm_dim_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - ); - pub fn atg_nuclear_norm_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - keepdim_: c_int, - ); - pub fn atg_numpy_t(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_one_hot(out__: *mut *mut C_tensor, self_: *mut C_tensor, num_classes_: i64); - pub fn atg_ones( - out__: *mut *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_ones_like(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_ones_like_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_ones_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg_orgqr(out__: *mut *mut C_tensor, self_: *mut C_tensor, input2_: *mut C_tensor); - pub fn atg_orgqr_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - input2_: *mut C_tensor, - ); - pub fn atg_ormqr( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - input2_: *mut C_tensor, - input3_: *mut C_tensor, - left_: c_int, - transpose_: c_int, - ); - pub fn atg_ormqr_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - input2_: *mut C_tensor, - input3_: *mut C_tensor, - left_: c_int, - transpose_: c_int, - ); - pub fn atg_outer(out__: *mut *mut C_tensor, self_: *mut C_tensor, vec2_: *mut C_tensor); - pub fn atg_outer_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - vec2_: *mut C_tensor, - ); - pub fn atg_output_nr(self_: *mut C_tensor) -> i64; - pub fn atg_pad( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - pad_data: *const i64, - pad_len: c_int, - mode_ptr: *const u8, - mode_len: c_int, - value_v: f64, - value_null: i8, - ); - pub fn atg_pad_sequence( - out__: *mut *mut C_tensor, - sequences_data: *const *mut C_tensor, - sequences_len: c_int, - batch_first_: c_int, - padding_value_: f64, - padding_side_ptr: *const u8, - padding_side_len: c_int, - ); - pub fn atg_pairwise_distance( - out__: *mut *mut C_tensor, - x1_: *mut C_tensor, - x2_: *mut C_tensor, - p_: f64, - eps_: f64, - keepdim_: c_int, - ); - pub fn atg_pdist(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: f64); - pub fn atg_permute( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dims_data: *const i64, - dims_len: c_int, - ); - pub fn atg_permute_copy( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dims_data: *const i64, - dims_len: c_int, - ); - pub fn atg_permute_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dims_data: *const i64, - dims_len: c_int, - ); - pub fn atg_pin_memory(out__: *mut *mut C_tensor, self_: *mut C_tensor, device_: c_int); - pub fn atg_pinverse(out__: *mut *mut C_tensor, self_: *mut C_tensor, rcond_: f64); - pub fn atg_pixel_shuffle(out__: *mut *mut C_tensor, self_: *mut C_tensor, upscale_factor_: i64); - pub fn atg_pixel_shuffle_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - upscale_factor_: i64, - ); - pub fn atg_pixel_unshuffle( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - downscale_factor_: i64, - ); - pub fn atg_pixel_unshuffle_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - downscale_factor_: i64, - ); - pub fn atg_poisson(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_poisson_nll_loss( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - target_: *mut C_tensor, - log_input_: c_int, - full_: c_int, - eps_: f64, - reduction_: i64, - ); - pub fn atg_poisson_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_polar(out__: *mut *mut C_tensor, abs_: *mut C_tensor, angle_: *mut C_tensor); - pub fn atg_polar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - abs_: *mut C_tensor, - angle_: *mut C_tensor, - ); - pub fn atg_polygamma(out__: *mut *mut C_tensor, n_: i64, self_: *mut C_tensor); - pub fn atg_polygamma_(out__: *mut *mut C_tensor, self_: *mut C_tensor, n_: i64); - pub fn atg_polygamma_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - n_: i64, - self_: *mut C_tensor, - ); - pub fn atg_positive(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_pow(out__: *mut *mut C_tensor, self_: *mut C_tensor, exponent_: *mut C_tensor); - pub fn atg_pow_(out__: *mut *mut C_tensor, self_: *mut C_tensor, exponent_: *mut C_scalar); - pub fn atg_pow_scalar( - out__: *mut *mut C_tensor, - self_scalar_: *mut C_scalar, - exponent_: *mut C_tensor, - ); - pub fn atg_pow_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_scalar_: *mut C_scalar, - exponent_: *mut C_tensor, - ); - pub fn atg_pow_tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - exponent_: *mut C_tensor, - ); - pub fn atg_pow_tensor_scalar( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - exponent_: *mut C_scalar, - ); - pub fn atg_pow_tensor_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - exponent_: *mut C_scalar, - ); - pub fn atg_pow_tensor_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - exponent_: *mut C_tensor, - ); - pub fn atg_prelu(out__: *mut *mut C_tensor, self_: *mut C_tensor, weight_: *mut C_tensor); - pub fn atg_prod(out__: *mut *mut C_tensor, self_: *mut C_tensor, dtype_: c_int); - pub fn atg_prod_dim_int( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - keepdim_: c_int, - dtype_: c_int, - ); - pub fn atg_prod_int_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - keepdim_: c_int, - dtype_: c_int, - ); - pub fn atg_prod_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dtype_: c_int, - ); - pub fn atg_put( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - index_: *mut C_tensor, - source_: *mut C_tensor, - accumulate_: c_int, - ); - pub fn atg_put_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - index_: *mut C_tensor, - source_: *mut C_tensor, - accumulate_: c_int, - ); - pub fn atg_put_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - index_: *mut C_tensor, - source_: *mut C_tensor, - accumulate_: c_int, - ); - pub fn atg_q_per_channel_axis(self_: *mut C_tensor) -> i64; - pub fn atg_q_per_channel_scales(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_q_per_channel_scales_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_q_per_channel_zero_points(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_q_per_channel_zero_points_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_q_scale(self_: *mut C_tensor) -> f64; - pub fn atg_q_zero_point(self_: *mut C_tensor) -> i64; - pub fn atg_qr(out__: *mut *mut C_tensor, self_: *mut C_tensor, some_: c_int); - pub fn atg_qr_q( - out__: *mut *mut C_tensor, - Q_: *mut C_tensor, - R_: *mut C_tensor, - self_: *mut C_tensor, - some_: c_int, - ); - pub fn atg_quantile( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - q_: *mut C_tensor, - dim_v: i64, - dim_null: i8, - keepdim_: c_int, - interpolation_ptr: *const u8, - interpolation_len: c_int, - ); - pub fn atg_quantile_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - q_: *mut C_tensor, - dim_v: i64, - dim_null: i8, - keepdim_: c_int, - interpolation_ptr: *const u8, - interpolation_len: c_int, - ); - pub fn atg_quantile_scalar( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - q_: f64, - dim_v: i64, - dim_null: i8, - keepdim_: c_int, - interpolation_ptr: *const u8, - interpolation_len: c_int, - ); - pub fn atg_quantile_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - q_: f64, - dim_v: i64, - dim_null: i8, - keepdim_: c_int, - interpolation_ptr: *const u8, - interpolation_len: c_int, - ); - pub fn atg_quantize_per_channel( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - scales_: *mut C_tensor, - zero_points_: *mut C_tensor, - axis_: i64, - dtype_: c_int, - ); - pub fn atg_quantize_per_channel_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - scales_: *mut C_tensor, - zero_points_: *mut C_tensor, - axis_: i64, - dtype_: c_int, - ); - pub fn atg_quantize_per_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - scale_: f64, - zero_point_: i64, - dtype_: c_int, - ); - pub fn atg_quantize_per_tensor_dynamic( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dtype_: c_int, - reduce_range_: c_int, - ); - pub fn atg_quantize_per_tensor_dynamic_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dtype_: c_int, - reduce_range_: c_int, - ); - pub fn atg_quantize_per_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - scale_: f64, - zero_point_: i64, - dtype_: c_int, - ); - pub fn atg_quantize_per_tensor_tensor_qparams( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - scale_: *mut C_tensor, - zero_point_: *mut C_tensor, - dtype_: c_int, - ); - pub fn atg_quantize_per_tensor_tensor_qparams_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - scale_: *mut C_tensor, - zero_point_: *mut C_tensor, - dtype_: c_int, - ); - pub fn atg_quantize_per_tensor_tensors( - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - scales_: *mut C_tensor, - zero_points_: *mut C_tensor, - dtype_: c_int, - ) -> *mut *mut C_tensor; - pub fn atg_quantize_per_tensor_tensors_out( - out_data: *const *mut C_tensor, - out_len: c_int, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - scales_: *mut C_tensor, - zero_points_: *mut C_tensor, - dtype_: c_int, - ); - pub fn atg_quantized_batch_norm( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - mean_: *mut C_tensor, - var_: *mut C_tensor, - eps_: f64, - output_scale_: f64, - output_zero_point_: i64, - ); - pub fn atg_quantized_batch_norm_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - input_: *mut C_tensor, - weight_: *mut C_tensor, - bias_: *mut C_tensor, - mean_: *mut C_tensor, - var_: *mut C_tensor, - eps_: f64, - output_scale_: f64, - output_zero_point_: i64, - ); - pub fn atg_quantized_gru_cell( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - hx_: *mut C_tensor, - w_ih_: *mut C_tensor, - w_hh_: *mut C_tensor, - b_ih_: *mut C_tensor, - b_hh_: *mut C_tensor, - packed_ih_: *mut C_tensor, - packed_hh_: *mut C_tensor, - col_offsets_ih_: *mut C_tensor, - col_offsets_hh_: *mut C_tensor, - scale_ih_: *mut C_scalar, - scale_hh_: *mut C_scalar, - zero_point_ih_: *mut C_scalar, - zero_point_hh_: *mut C_scalar, - ); - pub fn atg_quantized_lstm_cell( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - hx_data: *const *mut C_tensor, - hx_len: c_int, - w_ih_: *mut C_tensor, - w_hh_: *mut C_tensor, - b_ih_: *mut C_tensor, - b_hh_: *mut C_tensor, - packed_ih_: *mut C_tensor, - packed_hh_: *mut C_tensor, - col_offsets_ih_: *mut C_tensor, - col_offsets_hh_: *mut C_tensor, - scale_ih_: *mut C_scalar, - scale_hh_: *mut C_scalar, - zero_point_ih_: *mut C_scalar, - zero_point_hh_: *mut C_scalar, - ); - pub fn atg_quantized_max_pool1d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_quantized_max_pool1d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_quantized_max_pool2d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_quantized_max_pool2d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_quantized_max_pool3d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_quantized_max_pool3d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ceil_mode_: c_int, - ); - pub fn atg_quantized_rnn_relu_cell( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - hx_: *mut C_tensor, - w_ih_: *mut C_tensor, - w_hh_: *mut C_tensor, - b_ih_: *mut C_tensor, - b_hh_: *mut C_tensor, - packed_ih_: *mut C_tensor, - packed_hh_: *mut C_tensor, - col_offsets_ih_: *mut C_tensor, - col_offsets_hh_: *mut C_tensor, - scale_ih_: *mut C_scalar, - scale_hh_: *mut C_scalar, - zero_point_ih_: *mut C_scalar, - zero_point_hh_: *mut C_scalar, - ); - pub fn atg_quantized_rnn_tanh_cell( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - hx_: *mut C_tensor, - w_ih_: *mut C_tensor, - w_hh_: *mut C_tensor, - b_ih_: *mut C_tensor, - b_hh_: *mut C_tensor, - packed_ih_: *mut C_tensor, - packed_hh_: *mut C_tensor, - col_offsets_ih_: *mut C_tensor, - col_offsets_hh_: *mut C_tensor, - scale_ih_: *mut C_scalar, - scale_hh_: *mut C_scalar, - zero_point_ih_: *mut C_scalar, - zero_point_hh_: *mut C_scalar, - ); - pub fn atg_rad2deg(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_rad2deg_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_rad2deg_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_rand( - out__: *mut *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_rand_like(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_rand_like_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_rand_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg_randint( - out__: *mut *mut C_tensor, - high_: i64, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_randint_like(out__: *mut *mut C_tensor, self_: *mut C_tensor, high_: i64); - pub fn atg_randint_like_low_dtype( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - low_: i64, - high_: i64, - ); - pub fn atg_randint_like_low_dtype_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - low_: i64, - high_: i64, - ); - pub fn atg_randint_like_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - high_: i64, - ); - pub fn atg_randint_like_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - high_: *mut C_tensor, - ); - pub fn atg_randint_like_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - high_: *mut C_tensor, - ); - pub fn atg_randint_low( - out__: *mut *mut C_tensor, - low_: i64, - high_: i64, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_randint_low_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - low_: i64, - high_: i64, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg_randint_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - high_: i64, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg_randn( - out__: *mut *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_randn_like(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_randn_like_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_randn_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg_random(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_random_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_random_from( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - from_: i64, - to_v: i64, - to_null: i8, - ); - pub fn atg_random_from_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - from_: i64, - to_v: i64, - to_null: i8, - ); - pub fn atg_random_from_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - from_: i64, - to_v: i64, - to_null: i8, - ); - pub fn atg_random_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_random_to(out__: *mut *mut C_tensor, self_: *mut C_tensor, to_: i64); - pub fn atg_random_to_(out__: *mut *mut C_tensor, self_: *mut C_tensor, to_: i64); - pub fn atg_random_to_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - to_: i64, - ); - pub fn atg_randperm( - out__: *mut *mut C_tensor, - n_: i64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_randperm_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, n_: i64); - pub fn atg_range( - out__: *mut *mut C_tensor, - start_: *mut C_scalar, - end_: *mut C_scalar, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_range_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - start_: *mut C_scalar, - end_: *mut C_scalar, - ); - pub fn atg_range_out_( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - start_: *mut C_scalar, - end_: *mut C_scalar, - ); - pub fn atg_range_step( - out__: *mut *mut C_tensor, - start_: *mut C_scalar, - end_: *mut C_scalar, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_ravel(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_real(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_reciprocal(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_reciprocal_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_reciprocal_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_reflection_pad1d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_reflection_pad1d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_reflection_pad1d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_reflection_pad1d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_reflection_pad2d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_reflection_pad2d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_reflection_pad2d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_reflection_pad2d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_reflection_pad3d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_reflection_pad3d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_reflection_pad3d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_reflection_pad3d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_relu(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_relu6(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_relu6_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_relu_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_relu_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_remainder(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_remainder_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_remainder_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_remainder_scalar_tensor( - out__: *mut *mut C_tensor, - self_scalar_: *mut C_scalar, - other_: *mut C_tensor, - ); - pub fn atg_remainder_scalar_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_scalar_: *mut C_scalar, - other_: *mut C_tensor, - ); - pub fn atg_remainder_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_remainder_tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_remainder_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_renorm( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - p_: *mut C_scalar, - dim_: i64, - maxnorm_: *mut C_scalar, - ); - pub fn atg_renorm_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - p_: *mut C_scalar, - dim_: i64, - maxnorm_: *mut C_scalar, - ); - pub fn atg_renorm_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - p_: *mut C_scalar, - dim_: i64, - maxnorm_: *mut C_scalar, - ); - pub fn atg_repeat( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - repeats_data: *const i64, - repeats_len: c_int, - ); - pub fn atg_repeat_interleave( - out__: *mut *mut C_tensor, - repeats_: *mut C_tensor, - output_size_v: i64, - output_size_null: i8, - ); - pub fn atg_repeat_interleave_self_int( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - repeats_: i64, - dim_v: i64, - dim_null: i8, - output_size_v: i64, - output_size_null: i8, - ); - pub fn atg_repeat_interleave_self_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - repeats_: *mut C_tensor, - dim_v: i64, - dim_null: i8, - output_size_v: i64, - output_size_null: i8, - ); - pub fn atg_repeat_interleave_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - repeats_: *mut C_tensor, - output_size_v: i64, - output_size_null: i8, - ); - pub fn atg_repeat_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - repeats_data: *const i64, - repeats_len: c_int, - ); - pub fn atg_replication_pad1d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_replication_pad1d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_replication_pad1d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_replication_pad1d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_replication_pad2d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_replication_pad2d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_replication_pad2d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_replication_pad2d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_replication_pad3d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_replication_pad3d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_replication_pad3d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_replication_pad3d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_requires_grad_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - requires_grad_: c_int, - ); - pub fn atg_reshape( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - shape_data: *const i64, - shape_len: c_int, - ); - pub fn atg_reshape_as(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_resize( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg_resize_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg_resize_as( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - the_template_: *mut C_tensor, - ); - pub fn atg_resize_as_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - the_template_: *mut C_tensor, - ); - pub fn atg_resize_as_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - the_template_: *mut C_tensor, - ); - pub fn atg_resize_as_sparse( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - the_template_: *mut C_tensor, - ); - pub fn atg_resize_as_sparse_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - the_template_: *mut C_tensor, - ); - pub fn atg_resize_as_sparse_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - the_template_: *mut C_tensor, - ); - pub fn atg_resize_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg_resolve_conj(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_resolve_neg(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_retains_grad(self_: *mut C_tensor) -> c_int; - pub fn atg_rms_norm( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - normalized_shape_data: *const i64, - normalized_shape_len: c_int, - weight_: *mut C_tensor, - eps_v: f64, - eps_null: i8, - ); - pub fn atg_rnn_relu( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - hx_: *mut C_tensor, - params_data: *const *mut C_tensor, - params_len: c_int, - has_biases_: c_int, - num_layers_: i64, - dropout_: f64, - train_: c_int, - bidirectional_: c_int, - batch_first_: c_int, - ); - pub fn atg_rnn_relu_cell( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - hx_: *mut C_tensor, - w_ih_: *mut C_tensor, - w_hh_: *mut C_tensor, - b_ih_: *mut C_tensor, - b_hh_: *mut C_tensor, - ); - pub fn atg_rnn_relu_data( - out__: *mut *mut C_tensor, - data_: *mut C_tensor, - batch_sizes_: *mut C_tensor, - hx_: *mut C_tensor, - params_data: *const *mut C_tensor, - params_len: c_int, - has_biases_: c_int, - num_layers_: i64, - dropout_: f64, - train_: c_int, - bidirectional_: c_int, - ); - pub fn atg_rnn_tanh( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - hx_: *mut C_tensor, - params_data: *const *mut C_tensor, - params_len: c_int, - has_biases_: c_int, - num_layers_: i64, - dropout_: f64, - train_: c_int, - bidirectional_: c_int, - batch_first_: c_int, - ); - pub fn atg_rnn_tanh_cell( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - hx_: *mut C_tensor, - w_ih_: *mut C_tensor, - w_hh_: *mut C_tensor, - b_ih_: *mut C_tensor, - b_hh_: *mut C_tensor, - ); - pub fn atg_rnn_tanh_data( - out__: *mut *mut C_tensor, - data_: *mut C_tensor, - batch_sizes_: *mut C_tensor, - hx_: *mut C_tensor, - params_data: *const *mut C_tensor, - params_len: c_int, - has_biases_: c_int, - num_layers_: i64, - dropout_: f64, - train_: c_int, - bidirectional_: c_int, - ); - pub fn atg_roll( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - shifts_data: *const i64, - shifts_len: c_int, - dims_data: *const i64, - dims_len: c_int, - ); - pub fn atg_roll_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - shifts_data: *const i64, - shifts_len: c_int, - dims_data: *const i64, - dims_len: c_int, - ); - pub fn atg_rot90( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - k_: i64, - dims_data: *const i64, - dims_len: c_int, - ); - pub fn atg_rot90_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - k_: i64, - dims_data: *const i64, - dims_len: c_int, - ); - pub fn atg_round(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_round_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_round_decimals(out__: *mut *mut C_tensor, self_: *mut C_tensor, decimals_: i64); - pub fn atg_round_decimals_(out__: *mut *mut C_tensor, self_: *mut C_tensor, decimals_: i64); - pub fn atg_round_decimals_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - decimals_: i64, - ); - pub fn atg_round_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_row_indices(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_row_indices_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_row_indices_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_row_stack( - out__: *mut *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ); - pub fn atg_row_stack_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ); - pub fn atg_rrelu(out__: *mut *mut C_tensor, self_: *mut C_tensor, training_: c_int); - pub fn atg_rrelu_(out__: *mut *mut C_tensor, self_: *mut C_tensor, training_: c_int); - pub fn atg_rrelu_with_noise( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - noise_: *mut C_tensor, - training_: c_int, - ); - pub fn atg_rrelu_with_noise_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - noise_: *mut C_tensor, - training_: c_int, - ); - pub fn atg_rrelu_with_noise_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - noise_: *mut C_tensor, - lower_: *mut C_scalar, - upper_: *mut C_scalar, - training_: c_int, - self_is_result_: c_int, - ); - pub fn atg_rrelu_with_noise_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - noise_: *mut C_tensor, - lower_: *mut C_scalar, - upper_: *mut C_scalar, - training_: c_int, - self_is_result_: c_int, - ); - pub fn atg_rrelu_with_noise_functional( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - noise_: *mut C_tensor, - training_: c_int, - ); - pub fn atg_rrelu_with_noise_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - noise_: *mut C_tensor, - training_: c_int, - ); - pub fn atg_rsqrt(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_rsqrt_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_rsqrt_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_rsub(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_rsub_scalar(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_rsub_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_rsub_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_scalar_tensor( - out__: *mut *mut C_tensor, - s_: *mut C_scalar, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_scalar_tensor_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, s_: *mut C_scalar); - pub fn atg_scaled_dot_product_attention( - out__: *mut *mut C_tensor, - query_: *mut C_tensor, - key_: *mut C_tensor, - value_: *mut C_tensor, - attn_mask_: *mut C_tensor, - dropout_p_: f64, - is_causal_: c_int, - scale_v: f64, - scale_null: i8, - enable_gqa_: c_int, - ); - pub fn atg_scatter( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - src_: *mut C_tensor, - ); - pub fn atg_scatter_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - src_: *mut C_tensor, - ); - pub fn atg_scatter_add( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - src_: *mut C_tensor, - ); - pub fn atg_scatter_add_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - src_: *mut C_tensor, - ); - pub fn atg_scatter_add_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - src_: *mut C_tensor, - ); - pub fn atg_scatter_reduce( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - src_: *mut C_tensor, - reduce_ptr: *const u8, - reduce_len: c_int, - ); - pub fn atg_scatter_reduce_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - src_: *mut C_tensor, - reduce_ptr: *const u8, - reduce_len: c_int, - ); - pub fn atg_scatter_reduce_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - src_: *mut C_tensor, - reduce_ptr: *const u8, - reduce_len: c_int, - ); - pub fn atg_scatter_src_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - src_: *mut C_tensor, - ); - pub fn atg_scatter_value( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - value_: *mut C_scalar, - ); - pub fn atg_scatter_value_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - value_: *mut C_scalar, - ); - pub fn atg_scatter_value_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - value_: *mut C_scalar, - ); - pub fn atg_scatter_value_reduce( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - value_: *mut C_scalar, - reduce_ptr: *const u8, - reduce_len: c_int, - ); - pub fn atg_scatter_value_reduce_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - value_: *mut C_scalar, - reduce_ptr: *const u8, - reduce_len: c_int, - ); - pub fn atg_scatter_value_reduce_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: *mut C_tensor, - value_: *mut C_scalar, - reduce_ptr: *const u8, - reduce_len: c_int, - ); - pub fn atg_searchsorted( - out__: *mut *mut C_tensor, - sorted_sequence_: *mut C_tensor, - self_: *mut C_tensor, - out_int32_: c_int, - right_: c_int, - side_ptr: *const u8, - side_len: c_int, - sorter_: *mut C_tensor, - ); - pub fn atg_searchsorted_scalar( - out__: *mut *mut C_tensor, - sorted_sequence_: *mut C_tensor, - self_scalar_: *mut C_scalar, - out_int32_: c_int, - right_: c_int, - side_ptr: *const u8, - side_len: c_int, - sorter_: *mut C_tensor, - ); - pub fn atg_searchsorted_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - sorted_sequence_: *mut C_tensor, - self_scalar_: *mut C_scalar, - out_int32_: c_int, - right_: c_int, - side_ptr: *const u8, - side_len: c_int, - sorter_: *mut C_tensor, - ); - pub fn atg_searchsorted_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - sorted_sequence_: *mut C_tensor, - self_: *mut C_tensor, - out_int32_: c_int, - right_: c_int, - side_ptr: *const u8, - side_len: c_int, - sorter_: *mut C_tensor, - ); - pub fn atg_segment_reduce( - out__: *mut *mut C_tensor, - data_: *mut C_tensor, - reduce_ptr: *const u8, - reduce_len: c_int, - lengths_: *mut C_tensor, - indices_: *mut C_tensor, - offsets_: *mut C_tensor, - axis_: i64, - unsafe_: c_int, - initial_: *mut C_scalar, - ); - pub fn atg_segment_reduce_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - data_: *mut C_tensor, - reduce_ptr: *const u8, - reduce_len: c_int, - lengths_: *mut C_tensor, - indices_: *mut C_tensor, - offsets_: *mut C_tensor, - axis_: i64, - unsafe_: c_int, - initial_: *mut C_scalar, - ); - pub fn atg_select(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, index_: i64); - pub fn atg_select_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - input_sizes_data: *const i64, - input_sizes_len: c_int, - dim_: i64, - index_: i64, - ); - pub fn atg_select_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_output_: *mut C_tensor, - input_sizes_data: *const i64, - input_sizes_len: c_int, - dim_: i64, - index_: i64, - ); - pub fn atg_select_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, index_: i64); - pub fn atg_select_copy_int_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - index_: i64, - ); - pub fn atg_select_scatter( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - src_: *mut C_tensor, - dim_: i64, - index_: i64, - ); - pub fn atg_select_scatter_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - src_: *mut C_tensor, - dim_: i64, - index_: i64, - ); - pub fn atg_selu(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_selu_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_set(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_set_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_set_data(self_: *mut C_tensor, new_data_: *mut C_tensor); - pub fn atg_set_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_set_requires_grad(out__: *mut *mut C_tensor, self_: *mut C_tensor, r_: c_int); - pub fn atg_set_source_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - source_: *mut C_tensor, - ); - pub fn atg_set_source_tensor_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - source_: *mut C_tensor, - ); - pub fn atg_set_source_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - source_: *mut C_tensor, - ); - pub fn atg_set_source_tensor_storage_offset_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - source_: *mut C_tensor, - storage_offset_: i64, - size_data: *const i64, - size_len: c_int, - stride_data: *const i64, - stride_len: c_int, - ); - pub fn atg_sgn(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_sgn_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_sgn_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_sigmoid(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_sigmoid_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_sigmoid_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - output_: *mut C_tensor, - ); - pub fn atg_sigmoid_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - output_: *mut C_tensor, - ); - pub fn atg_sigmoid_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_sign(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_sign_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_sign_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_signbit(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_signbit_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_silu(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_silu_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_silu_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_silu_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_silu_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_sin(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_sin_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_sin_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_sinc(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_sinc_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_sinc_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_sinh(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_sinh_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_sinh_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_slice( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - start_v: i64, - start_null: i8, - end_v: i64, - end_null: i8, - step_: i64, - ); - pub fn atg_slice_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - input_sizes_data: *const i64, - input_sizes_len: c_int, - dim_: i64, - start_: i64, - end_: i64, - step_: i64, - ); - pub fn atg_slice_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_output_: *mut C_tensor, - input_sizes_data: *const i64, - input_sizes_len: c_int, - dim_: i64, - start_: i64, - end_: i64, - step_: i64, - ); - pub fn atg_slice_copy( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - start_v: i64, - start_null: i8, - end_v: i64, - end_null: i8, - step_: i64, - ); - pub fn atg_slice_copy_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - start_v: i64, - start_null: i8, - end_v: i64, - end_null: i8, - step_: i64, - ); - pub fn atg_slice_inverse( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - src_: *mut C_tensor, - dim_: i64, - start_v: i64, - start_null: i8, - end_v: i64, - end_null: i8, - step_: i64, - ); - pub fn atg_slice_scatter( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - src_: *mut C_tensor, - dim_: i64, - start_v: i64, - start_null: i8, - end_v: i64, - end_null: i8, - step_: i64, - ); - pub fn atg_slice_scatter_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - src_: *mut C_tensor, - dim_: i64, - start_v: i64, - start_null: i8, - end_v: i64, - end_null: i8, - step_: i64, - ); - pub fn atg_slogdet(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_slogdet_out( - out__: *mut *mut C_tensor, - sign_: *mut C_tensor, - logabsdet_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_slow_conv3d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_slow_conv3d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - ); - pub fn atg_slow_conv_dilated2d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ); - pub fn atg_slow_conv_dilated2d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ); - pub fn atg_slow_conv_dilated3d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ); - pub fn atg_slow_conv_dilated3d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ); - pub fn atg_slow_conv_transpose2d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - output_padding_data: *const i64, - output_padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ); - pub fn atg_slow_conv_transpose2d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - output_padding_data: *const i64, - output_padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ); - pub fn atg_slow_conv_transpose3d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - output_padding_data: *const i64, - output_padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ); - pub fn atg_slow_conv_transpose3d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - weight_: *mut C_tensor, - kernel_size_data: *const i64, - kernel_size_len: c_int, - bias_: *mut C_tensor, - stride_data: *const i64, - stride_len: c_int, - padding_data: *const i64, - padding_len: c_int, - output_padding_data: *const i64, - output_padding_len: c_int, - dilation_data: *const i64, - dilation_len: c_int, - ); - pub fn atg_smm(out__: *mut *mut C_tensor, self_: *mut C_tensor, mat2_: *mut C_tensor); - pub fn atg_smooth_l1_loss( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - reduction_: i64, - beta_: f64, - ); - pub fn atg_smooth_l1_loss_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - reduction_: i64, - beta_: f64, - ); - pub fn atg_smooth_l1_loss_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - reduction_: i64, - beta_: f64, - ); - pub fn atg_smooth_l1_loss_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - reduction_: i64, - beta_: f64, - ); - pub fn atg_soft_margin_loss( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - reduction_: i64, - ); - pub fn atg_soft_margin_loss_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - reduction_: i64, - ); - pub fn atg_soft_margin_loss_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - reduction_: i64, - ); - pub fn atg_soft_margin_loss_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - target_: *mut C_tensor, - reduction_: i64, - ); - pub fn atg_softmax(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, dtype_: c_int); - pub fn atg_softmax_int_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - dtype_: c_int, - ); - pub fn atg_softplus(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_softplus_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - beta_: *mut C_scalar, - threshold_: *mut C_scalar, - ); - pub fn atg_softplus_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - beta_: *mut C_scalar, - threshold_: *mut C_scalar, - ); - pub fn atg_softplus_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_softshrink(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_softshrink_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - lambd_: *mut C_scalar, - ); - pub fn atg_softshrink_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - lambd_: *mut C_scalar, - ); - pub fn atg_softshrink_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_sort(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64, descending_: c_int); - pub fn atg_sort_stable( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - stable_: c_int, - dim_: i64, - descending_: c_int, - ); - pub fn atg_sort_values( - out__: *mut *mut C_tensor, - values_: *mut C_tensor, - indices_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - descending_: c_int, - ); - pub fn atg_sort_values_stable( - out__: *mut *mut C_tensor, - values_: *mut C_tensor, - indices_: *mut C_tensor, - self_: *mut C_tensor, - stable_: c_int, - dim_: i64, - descending_: c_int, - ); - pub fn atg_sparse_bsc_tensor( - out__: *mut *mut C_tensor, - ccol_indices_: *mut C_tensor, - row_indices_: *mut C_tensor, - values_: *mut C_tensor, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_sparse_bsc_tensor_ccol_row_value_size( - out__: *mut *mut C_tensor, - ccol_indices_: *mut C_tensor, - row_indices_: *mut C_tensor, - values_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_sparse_bsr_tensor( - out__: *mut *mut C_tensor, - crow_indices_: *mut C_tensor, - col_indices_: *mut C_tensor, - values_: *mut C_tensor, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_sparse_bsr_tensor_crow_col_value_size( - out__: *mut *mut C_tensor, - crow_indices_: *mut C_tensor, - col_indices_: *mut C_tensor, - values_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_sparse_compressed_tensor( - out__: *mut *mut C_tensor, - compressed_indices_: *mut C_tensor, - plain_indices_: *mut C_tensor, - values_: *mut C_tensor, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_sparse_compressed_tensor_comp_plain_value_size( - out__: *mut *mut C_tensor, - compressed_indices_: *mut C_tensor, - plain_indices_: *mut C_tensor, - values_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_sparse_coo_tensor( - out__: *mut *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_sparse_coo_tensor_indices( - out__: *mut *mut C_tensor, - indices_: *mut C_tensor, - values_: *mut C_tensor, - options_kind: c_int, - options_device: c_int, - is_coalesced_: c_int, - ); - pub fn atg_sparse_coo_tensor_indices_size( - out__: *mut *mut C_tensor, - indices_: *mut C_tensor, - values_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - is_coalesced_: c_int, - ); - pub fn atg_sparse_coo_tensor_size_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg_sparse_csc_tensor( - out__: *mut *mut C_tensor, - ccol_indices_: *mut C_tensor, - row_indices_: *mut C_tensor, - values_: *mut C_tensor, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_sparse_csc_tensor_ccol_row_value_size( - out__: *mut *mut C_tensor, - ccol_indices_: *mut C_tensor, - row_indices_: *mut C_tensor, - values_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_sparse_csr_tensor( - out__: *mut *mut C_tensor, - crow_indices_: *mut C_tensor, - col_indices_: *mut C_tensor, - values_: *mut C_tensor, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_sparse_csr_tensor_crow_col_value_size( - out__: *mut *mut C_tensor, - crow_indices_: *mut C_tensor, - col_indices_: *mut C_tensor, - values_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_sparse_dim(self_: *mut C_tensor) -> i64; - pub fn atg_sparse_mask(out__: *mut *mut C_tensor, self_: *mut C_tensor, mask_: *mut C_tensor); - pub fn atg_sparse_mask_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mask_: *mut C_tensor, - ); - pub fn atg_sparse_resize( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - sparse_dim_: i64, - dense_dim_: i64, - ); - pub fn atg_sparse_resize_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - sparse_dim_: i64, - dense_dim_: i64, - ); - pub fn atg_sparse_resize_and_clear( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - sparse_dim_: i64, - dense_dim_: i64, - ); - pub fn atg_sparse_resize_and_clear_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - sparse_dim_: i64, - dense_dim_: i64, - ); - pub fn atg_sparse_resize_and_clear_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - sparse_dim_: i64, - dense_dim_: i64, - ); - pub fn atg_sparse_resize_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - sparse_dim_: i64, - dense_dim_: i64, - ); - pub fn atg_sparse_sampled_addmm( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mat1_: *mut C_tensor, - mat2_: *mut C_tensor, - ); - pub fn atg_sparse_sampled_addmm_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mat1_: *mut C_tensor, - mat2_: *mut C_tensor, - ); - pub fn atg_special_airy_ai(out__: *mut *mut C_tensor, x_: *mut C_tensor); - pub fn atg_special_airy_ai_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - ); - pub fn atg_special_bessel_j0(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_bessel_j0_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_bessel_j1(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_bessel_j1_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_bessel_y0(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_bessel_y0_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_bessel_y1(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_bessel_y1_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_chebyshev_polynomial_t( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_chebyshev_polynomial_t_n_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_chebyshev_polynomial_t_n_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_chebyshev_polynomial_t_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_chebyshev_polynomial_t_x_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_chebyshev_polynomial_t_x_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_chebyshev_polynomial_u( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_chebyshev_polynomial_u_n_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_chebyshev_polynomial_u_n_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_chebyshev_polynomial_u_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_chebyshev_polynomial_u_x_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_chebyshev_polynomial_u_x_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_chebyshev_polynomial_v( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_chebyshev_polynomial_v_n_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_chebyshev_polynomial_v_n_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_chebyshev_polynomial_v_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_chebyshev_polynomial_v_x_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_chebyshev_polynomial_v_x_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_chebyshev_polynomial_w( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_chebyshev_polynomial_w_n_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_chebyshev_polynomial_w_n_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_chebyshev_polynomial_w_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_chebyshev_polynomial_w_x_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_chebyshev_polynomial_w_x_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_digamma(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_digamma_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_entr(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_entr_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_erf(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_erf_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_erfc(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_erfc_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_erfcx(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_erfcx_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_erfinv(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_erfinv_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_exp2(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_exp2_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_expit(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_expit_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_expm1(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_expm1_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_gammainc( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_special_gammainc_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_special_gammaincc( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_special_gammaincc_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_special_gammaln(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_gammaln_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_hermite_polynomial_h( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_hermite_polynomial_h_n_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_hermite_polynomial_h_n_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_hermite_polynomial_h_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_hermite_polynomial_h_x_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_hermite_polynomial_h_x_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_hermite_polynomial_he( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_hermite_polynomial_he_n_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_hermite_polynomial_he_n_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_hermite_polynomial_he_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_hermite_polynomial_he_x_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_hermite_polynomial_he_x_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_i0(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_i0_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_i0e(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_i0e_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_i1(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_i1_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_i1e(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_i1e_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_laguerre_polynomial_l( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_laguerre_polynomial_l_n_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_laguerre_polynomial_l_n_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_laguerre_polynomial_l_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_laguerre_polynomial_l_x_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_laguerre_polynomial_l_x_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_legendre_polynomial_p( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_legendre_polynomial_p_n_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_legendre_polynomial_p_n_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_legendre_polynomial_p_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_legendre_polynomial_p_x_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_legendre_polynomial_p_x_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_log1p(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_log1p_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_log_ndtr(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_log_ndtr_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_log_softmax( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - dtype_: c_int, - ); - pub fn atg_special_logit( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - eps_v: f64, - eps_null: i8, - ); - pub fn atg_special_logit_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - eps_v: f64, - eps_null: i8, - ); - pub fn atg_special_logsumexp( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - ); - pub fn atg_special_logsumexp_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - ); - pub fn atg_special_modified_bessel_i0(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_modified_bessel_i0_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_modified_bessel_i1(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_modified_bessel_i1_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_modified_bessel_k0(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_modified_bessel_k0_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_modified_bessel_k1(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_modified_bessel_k1_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_multigammaln(out__: *mut *mut C_tensor, self_: *mut C_tensor, p_: i64); - pub fn atg_special_multigammaln_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - p_: i64, - ); - pub fn atg_special_ndtr(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_ndtr_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_ndtri(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_ndtri_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_polygamma(out__: *mut *mut C_tensor, n_: i64, self_: *mut C_tensor); - pub fn atg_special_polygamma_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - n_: i64, - self_: *mut C_tensor, - ); - pub fn atg_special_psi(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_psi_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_round(out__: *mut *mut C_tensor, self_: *mut C_tensor, decimals_: i64); - pub fn atg_special_round_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - decimals_: i64, - ); - pub fn atg_special_scaled_modified_bessel_k0(out__: *mut *mut C_tensor, x_: *mut C_tensor); - pub fn atg_special_scaled_modified_bessel_k0_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - ); - pub fn atg_special_scaled_modified_bessel_k1(out__: *mut *mut C_tensor, x_: *mut C_tensor); - pub fn atg_special_scaled_modified_bessel_k1_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - ); - pub fn atg_special_shifted_chebyshev_polynomial_t( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_shifted_chebyshev_polynomial_t_n_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_shifted_chebyshev_polynomial_t_n_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_shifted_chebyshev_polynomial_t_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_shifted_chebyshev_polynomial_t_x_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_shifted_chebyshev_polynomial_t_x_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_shifted_chebyshev_polynomial_u( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_shifted_chebyshev_polynomial_u_n_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_shifted_chebyshev_polynomial_u_n_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_shifted_chebyshev_polynomial_u_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_shifted_chebyshev_polynomial_u_x_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_shifted_chebyshev_polynomial_u_x_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_shifted_chebyshev_polynomial_v( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_shifted_chebyshev_polynomial_v_n_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_shifted_chebyshev_polynomial_v_n_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_shifted_chebyshev_polynomial_v_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_shifted_chebyshev_polynomial_v_x_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_shifted_chebyshev_polynomial_v_x_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_shifted_chebyshev_polynomial_w( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_shifted_chebyshev_polynomial_w_n_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_shifted_chebyshev_polynomial_w_n_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_scalar, - ); - pub fn atg_special_shifted_chebyshev_polynomial_w_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - n_: *mut C_tensor, - ); - pub fn atg_special_shifted_chebyshev_polynomial_w_x_scalar( - out__: *mut *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_shifted_chebyshev_polynomial_w_x_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_scalar, - n_: *mut C_tensor, - ); - pub fn atg_special_sinc(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_special_sinc_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_special_softmax( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - dtype_: c_int, - ); - pub fn atg_special_spherical_bessel_j0(out__: *mut *mut C_tensor, x_: *mut C_tensor); - pub fn atg_special_spherical_bessel_j0_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - x_: *mut C_tensor, - ); - pub fn atg_special_xlog1py( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_special_xlog1py_other_scalar( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_special_xlog1py_other_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_special_xlog1py_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_special_xlog1py_self_scalar( - out__: *mut *mut C_tensor, - self_scalar_: *mut C_scalar, - other_: *mut C_tensor, - ); - pub fn atg_special_xlog1py_self_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_scalar_: *mut C_scalar, - other_: *mut C_tensor, - ); - pub fn atg_special_xlogy( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_special_xlogy_other_scalar( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_special_xlogy_other_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_special_xlogy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_special_xlogy_self_scalar( - out__: *mut *mut C_tensor, - self_scalar_: *mut C_scalar, - other_: *mut C_tensor, - ); - pub fn atg_special_xlogy_self_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_scalar_: *mut C_scalar, - other_: *mut C_tensor, - ); - pub fn atg_special_zeta(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_special_zeta_other_scalar( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_special_zeta_other_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_special_zeta_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_special_zeta_self_scalar( - out__: *mut *mut C_tensor, - self_scalar_: *mut C_scalar, - other_: *mut C_tensor, - ); - pub fn atg_special_zeta_self_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_scalar_: *mut C_scalar, - other_: *mut C_tensor, - ); - pub fn atg_split(self_: *mut C_tensor, split_size_: i64, dim_: i64) -> *mut *mut C_tensor; - pub fn atg_split_copy(self_: *mut C_tensor, split_size_: i64, dim_: i64) -> *mut *mut C_tensor; - pub fn atg_split_copy_tensor_out( - out_data: *const *mut C_tensor, - out_len: c_int, - self_: *mut C_tensor, - split_size_: i64, - dim_: i64, - ); - pub fn atg_split_sizes( - self_: *mut C_tensor, - split_size_data: *const i64, - split_size_len: c_int, - dim_: i64, - ) -> *mut *mut C_tensor; - pub fn atg_split_with_sizes( - self_: *mut C_tensor, - split_sizes_data: *const i64, - split_sizes_len: c_int, - dim_: i64, - ) -> *mut *mut C_tensor; - pub fn atg_split_with_sizes_copy( - self_: *mut C_tensor, - split_sizes_data: *const i64, - split_sizes_len: c_int, - dim_: i64, - ) -> *mut *mut C_tensor; - pub fn atg_split_with_sizes_copy_out( - out_data: *const *mut C_tensor, - out_len: c_int, - self_: *mut C_tensor, - split_sizes_data: *const i64, - split_sizes_len: c_int, - dim_: i64, - ); - pub fn atg_sqrt(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_sqrt_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_sqrt_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_square(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_square_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_square_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_squeeze(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_squeeze_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_squeeze_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_squeeze_copy_dim(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64); - pub fn atg_squeeze_copy_dim_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - ); - pub fn atg_squeeze_copy_dims( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - ); - pub fn atg_squeeze_copy_dims_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - ); - pub fn atg_squeeze_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_squeeze_dim(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64); - pub fn atg_squeeze_dim_(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64); - pub fn atg_squeeze_dims( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - ); - pub fn atg_squeeze_dims_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - ); - pub fn atg_sspaddmm( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - mat1_: *mut C_tensor, - mat2_: *mut C_tensor, - ); - pub fn atg_sspaddmm_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - mat1_: *mut C_tensor, - mat2_: *mut C_tensor, - ); - pub fn atg_stack( - out__: *mut *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - dim_: i64, - ); - pub fn atg_stack_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - dim_: i64, - ); - pub fn atg_std(out__: *mut *mut C_tensor, self_: *mut C_tensor, unbiased_: c_int); - pub fn atg_std_correction( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - correction_: *mut C_scalar, - keepdim_: c_int, - ); - pub fn atg_std_correction_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - correction_: *mut C_scalar, - keepdim_: c_int, - ); - pub fn atg_std_dim( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - unbiased_: c_int, - keepdim_: c_int, - ); - pub fn atg_std_mean(out__: *mut *mut C_tensor, self_: *mut C_tensor, unbiased_: c_int); - pub fn atg_std_mean_correction( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - correction_: *mut C_scalar, - keepdim_: c_int, - ); - pub fn atg_std_mean_correction_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - correction_: *mut C_scalar, - keepdim_: c_int, - ); - pub fn atg_std_mean_dim( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - unbiased_: c_int, - keepdim_: c_int, - ); - pub fn atg_std_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - unbiased_: c_int, - keepdim_: c_int, - ); - pub fn atg_stft( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - n_fft_: i64, - hop_length_v: i64, - hop_length_null: i8, - win_length_v: i64, - win_length_null: i8, - window_: *mut C_tensor, - normalized_: c_int, - onesided_: c_int, - return_complex_: c_int, - align_to_window_: c_int, - ); - pub fn atg_stft_center( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - n_fft_: i64, - hop_length_v: i64, - hop_length_null: i8, - win_length_v: i64, - win_length_null: i8, - window_: *mut C_tensor, - center_: c_int, - pad_mode_ptr: *const u8, - pad_mode_len: c_int, - normalized_: c_int, - onesided_: c_int, - return_complex_: c_int, - align_to_window_: c_int, - ); - pub fn atg_sub(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_sub_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_sub_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_sub_scalar(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_sub_scalar_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_scalar); - pub fn atg_sub_scalar_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_subtract(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_subtract_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_subtract_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_subtract_scalar( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_subtract_scalar_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_sum(out__: *mut *mut C_tensor, self_: *mut C_tensor, dtype_: c_int); - pub fn atg_sum_dim_intlist( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - dtype_: c_int, - ); - pub fn atg_sum_intlist_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - keepdim_: c_int, - dtype_: c_int, - ); - pub fn atg_sum_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dtype_: c_int, - ); - pub fn atg_sum_to_size( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg_svd( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - some_: c_int, - compute_uv_: c_int, - ); - pub fn atg_svd_u( - out__: *mut *mut C_tensor, - U_: *mut C_tensor, - S_: *mut C_tensor, - V_: *mut C_tensor, - self_: *mut C_tensor, - some_: c_int, - compute_uv_: c_int, - ); - pub fn atg_swapaxes(out__: *mut *mut C_tensor, self_: *mut C_tensor, axis0_: i64, axis1_: i64); - pub fn atg_swapaxes_(out__: *mut *mut C_tensor, self_: *mut C_tensor, axis0_: i64, axis1_: i64); - pub fn atg_swapdims(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim0_: i64, dim1_: i64); - pub fn atg_swapdims_(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim0_: i64, dim1_: i64); - pub fn atg_t(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_t_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_t_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_t_copy_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_take(out__: *mut *mut C_tensor, self_: *mut C_tensor, index_: *mut C_tensor); - pub fn atg_take_along_dim( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - indices_: *mut C_tensor, - dim_v: i64, - dim_null: i8, - ); - pub fn atg_take_along_dim_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - indices_: *mut C_tensor, - dim_v: i64, - dim_null: i8, - ); - pub fn atg_take_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - index_: *mut C_tensor, - ); - pub fn atg_tan(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_tan_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_tan_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_tanh(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_tanh_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_tanh_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - output_: *mut C_tensor, - ); - pub fn atg_tanh_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - output_: *mut C_tensor, - ); - pub fn atg_tanh_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_tensor_split(self_: *mut C_tensor, sections_: i64, dim_: i64) -> *mut *mut C_tensor; - pub fn atg_tensor_split_indices( - self_: *mut C_tensor, - indices_data: *const i64, - indices_len: c_int, - dim_: i64, - ) -> *mut *mut C_tensor; - pub fn atg_tensor_split_tensor_indices_or_sections( - self_: *mut C_tensor, - tensor_indices_or_sections_: *mut C_tensor, - dim_: i64, - ) -> *mut *mut C_tensor; - pub fn atg_tensordot( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - dims_self_data: *const i64, - dims_self_len: c_int, - dims_other_data: *const i64, - dims_other_len: c_int, - ); - pub fn atg_tensordot_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - dims_self_data: *const i64, - dims_self_len: c_int, - dims_other_data: *const i64, - dims_other_len: c_int, - ); - pub fn atg_threshold( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - threshold_: *mut C_scalar, - value_: *mut C_scalar, - ); - pub fn atg_threshold_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - threshold_: *mut C_scalar, - value_: *mut C_scalar, - ); - pub fn atg_threshold_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - threshold_: *mut C_scalar, - ); - pub fn atg_threshold_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - self_: *mut C_tensor, - threshold_: *mut C_scalar, - ); - pub fn atg_threshold_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - threshold_: *mut C_scalar, - value_: *mut C_scalar, - ); - pub fn atg_tile( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dims_data: *const i64, - dims_len: c_int, - ); - pub fn atg_to(out__: *mut *mut C_tensor, self_: *mut C_tensor, device_: c_int); - pub fn atg_to_dense( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dtype_: c_int, - masked_grad_: c_int, - ); - pub fn atg_to_dense_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - input_: *mut C_tensor, - masked_grad_: c_int, - ); - pub fn atg_to_device( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - device_: c_int, - dtype_: c_int, - non_blocking_: c_int, - copy_: c_int, - ); - pub fn atg_to_dtype( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dtype_: c_int, - non_blocking_: c_int, - copy_: c_int, - ); - pub fn atg_to_dtype_layout( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - options_kind: c_int, - options_device: c_int, - non_blocking_: c_int, - copy_: c_int, - ); - pub fn atg_to_mkldnn(out__: *mut *mut C_tensor, self_: *mut C_tensor, dtype_: c_int); - pub fn atg_to_mkldnn_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - input_: *mut C_tensor, - ); - pub fn atg_to_mkldnn_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dtype_: c_int, - ); - pub fn atg_to_other( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - non_blocking_: c_int, - copy_: c_int, - ); - pub fn atg_to_padded_tensor( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - padding_: f64, - output_size_data: *const i64, - output_size_len: c_int, - ); - pub fn atg_to_padded_tensor_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - padding_: f64, - output_size_data: *const i64, - output_size_len: c_int, - ); - pub fn atg_to_sparse( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - layout_: i8, - blocksize_data: *const i64, - blocksize_len: c_int, - dense_dim_v: i64, - dense_dim_null: i8, - ); - pub fn atg_to_sparse_bsc( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - blocksize_data: *const i64, - blocksize_len: c_int, - dense_dim_v: i64, - dense_dim_null: i8, - ); - pub fn atg_to_sparse_bsr( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - blocksize_data: *const i64, - blocksize_len: c_int, - dense_dim_v: i64, - dense_dim_null: i8, - ); - pub fn atg_to_sparse_csc( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dense_dim_v: i64, - dense_dim_null: i8, - ); - pub fn atg_to_sparse_csr( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dense_dim_v: i64, - dense_dim_null: i8, - ); - pub fn atg_to_sparse_sparse_dim( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - sparse_dim_: i64, - ); - pub fn atg_topk( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - k_: i64, - dim_: i64, - largest_: c_int, - sorted_: c_int, - ); - pub fn atg_topk_values( - out__: *mut *mut C_tensor, - values_: *mut C_tensor, - indices_: *mut C_tensor, - self_: *mut C_tensor, - k_: i64, - dim_: i64, - largest_: c_int, - sorted_: c_int, - ); - pub fn atg_totype(out__: *mut *mut C_tensor, self_: *mut C_tensor, scalar_type_: c_int); - pub fn atg_trace(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_trace_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - sizes_data: *const i64, - sizes_len: c_int, - ); - pub fn atg_trace_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_transpose(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim0_: i64, dim1_: i64); - pub fn atg_transpose_(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim0_: i64, dim1_: i64); - pub fn atg_transpose_copy( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim0_: i64, - dim1_: i64, - ); - pub fn atg_transpose_copy_int_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim0_: i64, - dim1_: i64, - ); - pub fn atg_trapezoid(out__: *mut *mut C_tensor, y_: *mut C_tensor, dim_: i64); - pub fn atg_trapezoid_x( - out__: *mut *mut C_tensor, - y_: *mut C_tensor, - x_: *mut C_tensor, - dim_: i64, - ); - pub fn atg_trapz(out__: *mut *mut C_tensor, y_: *mut C_tensor, x_: *mut C_tensor, dim_: i64); - pub fn atg_trapz_dx(out__: *mut *mut C_tensor, y_: *mut C_tensor, dx_: f64, dim_: i64); - pub fn atg_triangular_solve( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - A_: *mut C_tensor, - upper_: c_int, - transpose_: c_int, - unitriangular_: c_int, - ); - pub fn atg_triangular_solve_x( - out__: *mut *mut C_tensor, - X_: *mut C_tensor, - M_: *mut C_tensor, - self_: *mut C_tensor, - A_: *mut C_tensor, - upper_: c_int, - transpose_: c_int, - unitriangular_: c_int, - ); - pub fn atg_tril(out__: *mut *mut C_tensor, self_: *mut C_tensor, diagonal_: i64); - pub fn atg_tril_(out__: *mut *mut C_tensor, self_: *mut C_tensor, diagonal_: i64); - pub fn atg_tril_indices( - out__: *mut *mut C_tensor, - row_: i64, - col_: i64, - offset_: i64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_tril_indices_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - row_: i64, - col_: i64, - offset_: i64, - ); - pub fn atg_tril_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - diagonal_: i64, - ); - pub fn atg_triplet_margin_loss( - out__: *mut *mut C_tensor, - anchor_: *mut C_tensor, - positive_: *mut C_tensor, - negative_: *mut C_tensor, - margin_: f64, - p_: f64, - eps_: f64, - swap_: c_int, - reduction_: i64, - ); - pub fn atg_triu(out__: *mut *mut C_tensor, self_: *mut C_tensor, diagonal_: i64); - pub fn atg_triu_(out__: *mut *mut C_tensor, self_: *mut C_tensor, diagonal_: i64); - pub fn atg_triu_indices( - out__: *mut *mut C_tensor, - row_: i64, - col_: i64, - offset_: i64, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_triu_indices_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - row_: i64, - col_: i64, - offset_: i64, - ); - pub fn atg_triu_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - diagonal_: i64, - ); - pub fn atg_true_divide(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_true_divide_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_true_divide_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_true_divide_scalar( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_true_divide_scalar_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_trunc(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_trunc_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_trunc_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_type_as(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_unbind(self_: *mut C_tensor, dim_: i64) -> *mut *mut C_tensor; - pub fn atg_unbind_copy(self_: *mut C_tensor, dim_: i64) -> *mut *mut C_tensor; - pub fn atg_unbind_copy_int_out( - out_data: *const *mut C_tensor, - out_len: c_int, - self_: *mut C_tensor, - dim_: i64, - ); - pub fn atg_unflatten( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - sizes_data: *const i64, - sizes_len: c_int, - ); - pub fn atg_unflatten_dense_tensors( - flat_: *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ) -> *mut *mut C_tensor; - pub fn atg_unfold( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dimension_: i64, - size_: i64, - step_: i64, - ); - pub fn atg_unfold_backward( - out__: *mut *mut C_tensor, - grad_in_: *mut C_tensor, - input_sizes_data: *const i64, - input_sizes_len: c_int, - dim_: i64, - size_: i64, - step_: i64, - ); - pub fn atg_unfold_backward_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - grad_in_: *mut C_tensor, - input_sizes_data: *const i64, - input_sizes_len: c_int, - dim_: i64, - size_: i64, - step_: i64, - ); - pub fn atg_unfold_copy( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dimension_: i64, - size_: i64, - step_: i64, - ); - pub fn atg_unfold_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dimension_: i64, - size_: i64, - step_: i64, - ); - pub fn atg_uniform(out__: *mut *mut C_tensor, self_: *mut C_tensor, from_: f64, to_: f64); - pub fn atg_uniform_(out__: *mut *mut C_tensor, self_: *mut C_tensor, from_: f64, to_: f64); - pub fn atg_uniform_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - from_: f64, - to_: f64, - ); - pub fn atg_unique_consecutive( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - return_inverse_: c_int, - return_counts_: c_int, - dim_v: i64, - dim_null: i8, - ); - pub fn atg_unique_consecutive_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - out2_: *mut C_tensor, - self_: *mut C_tensor, - return_inverse_: c_int, - return_counts_: c_int, - dim_v: i64, - dim_null: i8, - ); - pub fn atg_unique_dim( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - sorted_: c_int, - return_inverse_: c_int, - return_counts_: c_int, - ); - pub fn atg_unique_dim_consecutive( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - return_inverse_: c_int, - return_counts_: c_int, - ); - pub fn atg_unique_dim_consecutive_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - out2_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - return_inverse_: c_int, - return_counts_: c_int, - ); - pub fn atg_unique_dim_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - out2_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - sorted_: c_int, - return_inverse_: c_int, - return_counts_: c_int, - ); - pub fn atg_unsafe_chunk(self_: *mut C_tensor, chunks_: i64, dim_: i64) -> *mut *mut C_tensor; - pub fn atg_unsafe_split( - self_: *mut C_tensor, - split_size_: i64, - dim_: i64, - ) -> *mut *mut C_tensor; - pub fn atg_unsafe_split_tensor_out( - out_data: *const *mut C_tensor, - out_len: c_int, - self_: *mut C_tensor, - split_size_: i64, - dim_: i64, - ); - pub fn atg_unsafe_split_with_sizes( - self_: *mut C_tensor, - split_sizes_data: *const i64, - split_sizes_len: c_int, - dim_: i64, - ) -> *mut *mut C_tensor; - pub fn atg_unsafe_split_with_sizes_out( - out_data: *const *mut C_tensor, - out_len: c_int, - self_: *mut C_tensor, - split_sizes_data: *const i64, - split_sizes_len: c_int, - dim_: i64, - ); - pub fn atg_unsqueeze(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64); - pub fn atg_unsqueeze_(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64); - pub fn atg_unsqueeze_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor, dim_: i64); - pub fn atg_unsqueeze_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_: i64, - ); - pub fn atg_upsample_bicubic2d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - align_corners_: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg_upsample_bicubic2d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - align_corners_: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg_upsample_bicubic2d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - align_corners_: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg_upsample_bicubic2d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - align_corners_: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg_upsample_bicubic2d_vec( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - align_corners_: c_int, - scale_factors_data: *const f64, - scale_factors_len: c_int, - ); - pub fn atg_upsample_bilinear2d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - align_corners_: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg_upsample_bilinear2d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - align_corners_: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg_upsample_bilinear2d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - align_corners_: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg_upsample_bilinear2d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - align_corners_: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg_upsample_bilinear2d_vec( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - align_corners_: c_int, - scale_factors_data: *const f64, - scale_factors_len: c_int, - ); - pub fn atg_upsample_bilinear2d_vec_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - input_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - align_corners_: c_int, - scale_factors_data: *const f64, - scale_factors_len: c_int, - ); - pub fn atg_upsample_linear1d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - align_corners_: c_int, - scales_v: f64, - scales_null: i8, - ); - pub fn atg_upsample_linear1d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - align_corners_: c_int, - scales_v: f64, - scales_null: i8, - ); - pub fn atg_upsample_linear1d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - align_corners_: c_int, - scales_v: f64, - scales_null: i8, - ); - pub fn atg_upsample_linear1d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - align_corners_: c_int, - scales_v: f64, - scales_null: i8, - ); - pub fn atg_upsample_linear1d_vec( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - align_corners_: c_int, - scale_factors_data: *const f64, - scale_factors_len: c_int, - ); - pub fn atg_upsample_nearest1d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - scales_v: f64, - scales_null: i8, - ); - pub fn atg_upsample_nearest1d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - scales_v: f64, - scales_null: i8, - ); - pub fn atg_upsample_nearest1d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - scales_v: f64, - scales_null: i8, - ); - pub fn atg_upsample_nearest1d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - scales_v: f64, - scales_null: i8, - ); - pub fn atg_upsample_nearest1d_vec( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - scale_factors_data: *const f64, - scale_factors_len: c_int, - ); - pub fn atg_upsample_nearest2d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg_upsample_nearest2d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg_upsample_nearest2d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg_upsample_nearest2d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg_upsample_nearest2d_vec( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - scale_factors_data: *const f64, - scale_factors_len: c_int, - ); - pub fn atg_upsample_nearest2d_vec_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - input_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - scale_factors_data: *const f64, - scale_factors_len: c_int, - ); - pub fn atg_upsample_nearest3d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - scales_d_v: f64, - scales_d_null: i8, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg_upsample_nearest3d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - scales_d_v: f64, - scales_d_null: i8, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg_upsample_nearest3d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - scales_d_v: f64, - scales_d_null: i8, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg_upsample_nearest3d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - scales_d_v: f64, - scales_d_null: i8, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg_upsample_nearest3d_vec( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - scale_factors_data: *const f64, - scale_factors_len: c_int, - ); - pub fn atg_upsample_trilinear3d( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - align_corners_: c_int, - scales_d_v: f64, - scales_d_null: i8, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg_upsample_trilinear3d_backward( - out__: *mut *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - align_corners_: c_int, - scales_d_v: f64, - scales_d_null: i8, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg_upsample_trilinear3d_backward_grad_input( - out__: *mut *mut C_tensor, - grad_input_: *mut C_tensor, - grad_output_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - input_size_data: *const i64, - input_size_len: c_int, - align_corners_: c_int, - scales_d_v: f64, - scales_d_null: i8, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg_upsample_trilinear3d_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - align_corners_: c_int, - scales_d_v: f64, - scales_d_null: i8, - scales_h_v: f64, - scales_h_null: i8, - scales_w_v: f64, - scales_w_null: i8, - ); - pub fn atg_upsample_trilinear3d_vec( - out__: *mut *mut C_tensor, - input_: *mut C_tensor, - output_size_data: *const i64, - output_size_len: c_int, - align_corners_: c_int, - scale_factors_data: *const f64, - scale_factors_len: c_int, - ); - pub fn atg_value_selecting_reduction_backward( - out__: *mut *mut C_tensor, - grad_: *mut C_tensor, - dim_: i64, - indices_: *mut C_tensor, - sizes_data: *const i64, - sizes_len: c_int, - keepdim_: c_int, - ); - pub fn atg_values(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_values_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_values_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_vander( - out__: *mut *mut C_tensor, - x_: *mut C_tensor, - n_v: i64, - n_null: i8, - increasing_: c_int, - ); - pub fn atg_var(out__: *mut *mut C_tensor, self_: *mut C_tensor, unbiased_: c_int); - pub fn atg_var_correction( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - correction_: *mut C_scalar, - keepdim_: c_int, - ); - pub fn atg_var_correction_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - correction_: *mut C_scalar, - keepdim_: c_int, - ); - pub fn atg_var_dim( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - unbiased_: c_int, - keepdim_: c_int, - ); - pub fn atg_var_mean(out__: *mut *mut C_tensor, self_: *mut C_tensor, unbiased_: c_int); - pub fn atg_var_mean_correction( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - correction_: *mut C_scalar, - keepdim_: c_int, - ); - pub fn atg_var_mean_correction_out( - out__: *mut *mut C_tensor, - out0_: *mut C_tensor, - out1_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - correction_: *mut C_scalar, - keepdim_: c_int, - ); - pub fn atg_var_mean_dim( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - unbiased_: c_int, - keepdim_: c_int, - ); - pub fn atg_var_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dim_data: *const i64, - dim_len: c_int, - unbiased_: c_int, - keepdim_: c_int, - ); - pub fn atg_vdot(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_vdot_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_view( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg_view_as(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_view_as_complex(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_view_as_complex_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_view_as_complex_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_view_as_real(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_view_as_real_copy(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_view_as_real_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - ); - pub fn atg_view_copy( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg_view_copy_dtype(out__: *mut *mut C_tensor, self_: *mut C_tensor, dtype_: c_int); - pub fn atg_view_copy_dtype_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - dtype_: c_int, - ); - pub fn atg_view_copy_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); - pub fn atg_view_dtype(out__: *mut *mut C_tensor, self_: *mut C_tensor, dtype_: c_int); - pub fn atg_vsplit(self_: *mut C_tensor, sections_: i64) -> *mut *mut C_tensor; - pub fn atg_vsplit_array( - self_: *mut C_tensor, - indices_data: *const i64, - indices_len: c_int, - ) -> *mut *mut C_tensor; - pub fn atg_vstack( - out__: *mut *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ); - pub fn atg_vstack_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - tensors_data: *const *mut C_tensor, - tensors_len: c_int, - ); - pub fn atg_where(condition_: *mut C_tensor) -> *mut *mut C_tensor; - pub fn atg_where_scalar( - out__: *mut *mut C_tensor, - condition_: *mut C_tensor, - self_scalar_: *mut C_scalar, - other_: *mut C_scalar, - ); - pub fn atg_where_scalarother( - out__: *mut *mut C_tensor, - condition_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_where_scalarself( - out__: *mut *mut C_tensor, - condition_: *mut C_tensor, - self_scalar_: *mut C_scalar, - other_: *mut C_tensor, - ); - pub fn atg_where_self( - out__: *mut *mut C_tensor, - condition_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_where_self_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - condition_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_xlogy(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_xlogy_(out__: *mut *mut C_tensor, self_: *mut C_tensor, other_: *mut C_tensor); - pub fn atg_xlogy_outscalar_other( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_xlogy_outscalar_self( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_scalar_: *mut C_scalar, - other_: *mut C_tensor, - ); - pub fn atg_xlogy_outtensor( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_tensor, - ); - pub fn atg_xlogy_scalar_other( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_xlogy_scalar_other_( - out__: *mut *mut C_tensor, - self_: *mut C_tensor, - other_: *mut C_scalar, - ); - pub fn atg_xlogy_scalar_self( - out__: *mut *mut C_tensor, - self_scalar_: *mut C_scalar, - other_: *mut C_tensor, - ); - pub fn atg_zero(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_zero_(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_zero_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_zeros( - out__: *mut *mut C_tensor, - size_data: *const i64, - size_len: c_int, - options_kind: c_int, - options_device: c_int, - ); - pub fn atg_zeros_like(out__: *mut *mut C_tensor, self_: *mut C_tensor); - pub fn atg_zeros_like_out(out__: *mut *mut C_tensor, out_: *mut C_tensor, self_: *mut C_tensor); - pub fn atg_zeros_out( - out__: *mut *mut C_tensor, - out_: *mut C_tensor, - size_data: *const i64, - size_len: c_int, - ); -} diff --git a/v2/dependency-patches/torch-sys-0.24.0/src/cuda.rs b/v2/dependency-patches/torch-sys-0.24.0/src/cuda.rs deleted file mode 100644 index 7ea4bbeff3..0000000000 --- a/v2/dependency-patches/torch-sys-0.24.0/src/cuda.rs +++ /dev/null @@ -1,30 +0,0 @@ -use libc::c_int; - -extern "C" { - /// Returns the number of CUDA devices available. - pub fn atc_cuda_device_count() -> c_int; - - /// Returns true if at least one CUDA device is available. - pub fn atc_cuda_is_available() -> c_int; - - /// Returns true if CUDA is available, and CuDNN is available. - pub fn atc_cudnn_is_available() -> c_int; - - /// Sets the seed for the current GPU. - pub fn atc_manual_seed(seed: u64); - - /// Sets the seed for all available GPUs. - pub fn atc_manual_seed_all(seed: u64); - - /// Waits for all kernels in all streams on a CUDA device to complete. - pub fn atc_synchronize(device_index: i64); - - /// Returns true if CUDNN is enabled by the user. - pub fn atc_user_enabled_cudnn() -> c_int; - - /// Enable or disable CUDNN. - pub fn atc_set_user_enabled_cudnn(b: c_int); - - /// Sets CUDNN benchmark mode. - pub fn atc_set_benchmark_cudnn(b: c_int); -} diff --git a/v2/dependency-patches/torch-sys-0.24.0/src/io.rs b/v2/dependency-patches/torch-sys-0.24.0/src/io.rs deleted file mode 100644 index 0e242eb698..0000000000 --- a/v2/dependency-patches/torch-sys-0.24.0/src/io.rs +++ /dev/null @@ -1,111 +0,0 @@ -use libc::{c_void, size_t}; -use std::io::{Read, Seek, SeekFrom, Write}; - -pub trait ReadStream: Read + Seek {} - -#[no_mangle] -extern "C" fn tch_write_stream_destructor(stream_ptr: *mut c_void) { - unsafe { - let boxed_stream: Box> = Box::from_raw(stream_ptr as *mut Box); - drop(boxed_stream) - } -} - -#[no_mangle] -extern "C" fn tch_write_stream_write( - stream_ptr: *mut c_void, - buf: *const u8, - size: size_t, - out_size: *mut size_t, -) -> bool { - unsafe { - let boxed_stream = stream_ptr as *mut Box; - let buffer = std::slice::from_raw_parts(buf, size); - match (*boxed_stream).write(buffer) { - Ok(x) => { - *out_size = x; - true - } - Err(_) => false, - } - } -} - -#[no_mangle] -extern "C" fn tch_read_stream_destructor(stream_ptr: *mut c_void) { - unsafe { - let boxed_stream: Box> = - Box::from_raw(stream_ptr as *mut Box); - drop(boxed_stream) - } -} - -#[no_mangle] -extern "C" fn tch_read_stream_stream_position(stream_ptr: *mut c_void, current: *mut u64) -> bool { - unsafe { - let boxed_stream = stream_ptr as *mut Box; - match (*boxed_stream).stream_position() { - Ok(ret) => { - *current = ret; - true - } - Err(_) => false, - } - } -} - -#[no_mangle] -extern "C" fn tch_read_stream_seek_start( - stream_ptr: *mut c_void, - pos: u64, - new_pos: *mut u64, -) -> bool { - unsafe { - let boxed_stream = stream_ptr as *mut Box; - match (*boxed_stream).seek(SeekFrom::Start(pos)) { - Ok(ret) => { - *new_pos = ret; - true - } - Err(_) => false, - } - } -} - -#[no_mangle] -extern "C" fn tch_read_stream_seek_end( - stream_ptr: *mut c_void, - pos: i64, - new_pos: *mut u64, -) -> bool { - unsafe { - let boxed_stream = stream_ptr as *mut Box; - match (*boxed_stream).seek(SeekFrom::End(pos)) { - Ok(ret) => { - *new_pos = ret; - true - } - Err(_) => false, - } - } -} - -#[no_mangle] -extern "C" fn tch_read_stream_read( - stream_ptr: *mut c_void, - buf: *mut u8, - size: size_t, - read_size: *mut size_t, -) -> bool { - unsafe { - let boxed_stream = stream_ptr as *mut Box; - let buffer = std::slice::from_raw_parts_mut(buf, size); - match (*boxed_stream).read(buffer) { - Ok(ret) => { - *read_size = ret; - true - } - Err(_) => false, - } - } -} diff --git a/v2/dependency-patches/torch-sys-0.24.0/src/lib.rs b/v2/dependency-patches/torch-sys-0.24.0/src/lib.rs deleted file mode 100644 index 2ae76e1aff..0000000000 --- a/v2/dependency-patches/torch-sys-0.24.0/src/lib.rs +++ /dev/null @@ -1,336 +0,0 @@ -pub mod cuda; -pub mod io; -#[cfg(feature = "python-extension")] -pub mod python; -mod traits; - -use libc::{c_char, c_int, c_uchar, c_void, size_t}; -pub use traits::{DoubleList, IntList, IntListOption}; - -#[repr(C)] -pub struct C_scalar { - _private: [u8; 0], -} - -extern "C" { - pub fn ats_int(v: i64) -> *mut C_scalar; - pub fn ats_float(v: f64) -> *mut C_scalar; - pub fn ats_to_int(arg: *mut C_scalar) -> i64; - pub fn ats_to_float(arg: *mut C_scalar) -> f64; - pub fn ats_to_string(arg: *mut C_scalar) -> *mut c_char; - pub fn ats_free(arg: *mut C_scalar); -} - -#[repr(C)] -pub struct C_tensor { - _private: [u8; 0], -} - -extern "C" { - pub fn at_new_tensor() -> *mut C_tensor; - pub fn at_shallow_clone(arg: *mut C_tensor) -> *mut C_tensor; - pub fn at_copy_(dst: *mut C_tensor, src: *mut C_tensor); - pub fn at_data_ptr(arg: *mut C_tensor) -> *mut c_void; - pub fn at_defined(arg: *mut C_tensor) -> c_int; - pub fn at_is_sparse(arg: *mut C_tensor) -> c_int; - pub fn at_is_mkldnn(arg: *mut C_tensor) -> c_int; - pub fn at_is_contiguous(args: *mut C_tensor) -> c_int; - pub fn at_backward(arg: *mut C_tensor, keep_graph: c_int, create_graph: c_int); - pub fn at_print(arg: *mut C_tensor); - pub fn at_to_string(arg: *mut C_tensor, line_size: c_int) -> *mut c_char; - pub fn at_dim(arg: *mut C_tensor) -> size_t; - pub fn at_get(arg: *mut C_tensor, index: c_int) -> *mut C_tensor; - pub fn at_requires_grad(arg: *mut C_tensor) -> c_int; - pub fn at_shape(arg: *mut C_tensor, sz: *mut i64); - pub fn at_stride(arg: *mut C_tensor, sz: *mut i64); - pub fn at_double_value_at_indexes(arg: *mut C_tensor, idx: *const i64, idx_len: c_int) -> f64; - pub fn at_int64_value_at_indexes(arg: *mut C_tensor, idx: *const i64, idx_len: c_int) -> i64; - pub fn at_get_num_interop_threads() -> c_int; - pub fn at_get_num_threads() -> c_int; - pub fn at_set_num_interop_threads(n_threads: c_int); - pub fn at_set_num_threads(n_threads: c_int); - pub fn at_set_qengine(qengine: c_int); - pub fn at_free(arg: *mut C_tensor); - pub fn at_run_backward( - arg: *const *mut C_tensor, - ntensors: c_int, - inputs: *const *mut C_tensor, - ninputs: c_int, - outputs: *mut *mut C_tensor, - keep_graph: c_int, - create_graph: c_int, - ); - pub fn at_copy_data( - arg: *mut C_tensor, - vs: *const c_void, - numel: size_t, - elt_size_in_bytes: size_t, - ); - pub fn at_scalar_type(arg: *mut C_tensor) -> c_int; - pub fn at__amp_non_finite_check_and_unscale( - t: *mut C_tensor, - found_inf: *mut C_tensor, - inf_scale: *mut C_tensor, - ); - pub fn at_autocast_clear_cache(); - pub fn at_autocast_decrement_nesting() -> c_int; - pub fn at_autocast_increment_nesting() -> c_int; - pub fn at_autocast_is_enabled() -> c_int; - pub fn at_autocast_set_enabled(b: c_int) -> c_int; - pub fn at_device(arg: *mut C_tensor) -> c_int; - pub fn at_tensor_of_data( - vs: *const c_void, - dims: *const i64, - ndims: size_t, - elt_size_in_bytes: size_t, - kind: c_int, - ) -> *mut C_tensor; - pub fn at_tensor_of_blob( - vs: *const c_void, - dims: *const i64, - ndims: size_t, - strides: *const i64, - nstrides: size_t, - kind: c_int, - device: c_int, - ) -> *mut C_tensor; - pub fn at_grad_set_enabled(b: c_int) -> c_int; - pub fn at_save(arg: *mut C_tensor, filename: *const c_char); - pub fn at_save_to_stream(arg: *mut C_tensor, stream_ptr: *mut c_void); - pub fn at_load(filename: *const c_char) -> *mut C_tensor; - pub fn at_load_from_stream(stream_ptr: *mut c_void) -> *mut C_tensor; - pub fn at_save_multi( - args: *const *mut C_tensor, - names: *const *const c_char, - n: c_int, - filename: *const c_char, - ); - pub fn at_save_multi_to_stream( - args: *const *mut C_tensor, - names: *const *const c_char, - n: c_int, - stream_ptr: *mut c_void, - ); - pub fn at_loadz_callback( - filename: *const c_char, - data: *mut c_void, - f: extern "C" fn(*mut c_void, name: *const c_char, t: *mut C_tensor), - ); - pub fn at_loadz_callback_with_device( - filename: *const c_char, - data: *mut c_void, - f: extern "C" fn(*mut c_void, name: *const c_char, t: *mut C_tensor), - device_id: c_int, - ); - pub fn at_load_callback( - filename: *const c_char, - data: *mut c_void, - f: extern "C" fn(*mut c_void, name: *const c_char, t: *mut C_tensor), - ); - pub fn at_load_callback_with_device( - filename: *const c_char, - data: *mut c_void, - f: extern "C" fn(*mut c_void, name: *const c_char, t: *mut C_tensor), - device_id: c_int, - ); - pub fn at_load_from_stream_callback( - stream_ptr: *mut c_void, - data: *mut c_void, - f: extern "C" fn(*mut c_void, name: *const c_char, t: *mut C_tensor), - enable_device_id: bool, - device_id: c_int, - ); - - pub fn at_manual_seed(seed: i64); - pub fn at_set_graph_executor_optimize(b: bool); - pub fn at_context_has_openmp() -> bool; - pub fn at_context_has_mkl() -> bool; - pub fn at_context_has_lapack() -> bool; - pub fn at_context_has_mkldnn() -> bool; - pub fn at_context_has_magma() -> bool; - pub fn at_context_has_cuda() -> bool; - pub fn at_context_has_cudart() -> bool; - pub fn at_context_has_cusolver() -> bool; - pub fn at_context_has_hip() -> bool; - pub fn at_context_has_ipu() -> bool; - pub fn at_context_has_xla() -> bool; - pub fn at_context_has_lazy() -> bool; - pub fn at_context_has_mps() -> bool; - pub fn at_context_version_cudnn() -> i64; - pub fn at_context_version_cudart() -> i64; -} - -pub mod c_generated; - -extern "C" { - pub fn get_and_reset_last_err() -> *mut c_char; -} - -#[repr(C)] -pub struct C_optimizer { - _private: [u8; 0], -} - -extern "C" { - pub fn ato_adam( - lr: f64, - beta1: f64, - beta2: f64, - wd: f64, - eps: f64, - amsgrad: bool, - ) -> *mut C_optimizer; - pub fn ato_adamw( - lr: f64, - beta1: f64, - beta2: f64, - wd: f64, - eps: f64, - amsgrad: bool, - ) -> *mut C_optimizer; - pub fn ato_rms_prop( - lr: f64, - alpha: f64, - eps: f64, - wd: f64, - momentum: f64, - centered: c_int, - ) -> *mut C_optimizer; - pub fn ato_sgd( - lr: f64, - momentum: f64, - dampening: f64, - wd: f64, - nesterov: c_int, - ) -> *mut C_optimizer; - pub fn ato_add_parameters(arg: *mut C_optimizer, ts: *mut C_tensor, group: size_t); - pub fn ato_set_learning_rate(arg: *mut C_optimizer, lr: f64); - pub fn ato_set_learning_rate_group(arg: *mut C_optimizer, group: size_t, lr: f64); - pub fn ato_set_momentum(arg: *mut C_optimizer, momentum: f64); - pub fn ato_set_momentum_group(arg: *mut C_optimizer, group: size_t, momentum: f64); - pub fn ato_set_weight_decay(arg: *mut C_optimizer, weight_decay: f64); - pub fn ato_set_weight_decay_group(arg: *mut C_optimizer, group: size_t, weight_decay: f64); - pub fn ato_zero_grad(arg: *mut C_optimizer); - pub fn ato_step(arg: *mut C_optimizer); - pub fn ato_free(arg: *mut C_optimizer); - pub fn at_save_image(arg: *mut C_tensor, filename: *const c_char) -> c_int; - pub fn at_load_image(filename: *const c_char) -> *mut C_tensor; - pub fn at_load_image_from_memory( - img_data: *const c_uchar, - img_data_len: size_t, - ) -> *mut C_tensor; - pub fn at_resize_image(arg: *mut C_tensor, out_w: c_int, out_h: c_int) -> *mut C_tensor; -} - -#[allow(clippy::upper_case_acronyms)] -#[repr(C)] -pub struct CIValue { - _private: [u8; 0], -} - -#[repr(C)] -pub struct CModule_ { - _private: [u8; 0], -} - -extern "C" { - // Constructors - pub fn ati_none() -> *mut CIValue; - pub fn ati_bool(b: c_int) -> *mut CIValue; - pub fn ati_int(v: i64) -> *mut CIValue; - pub fn ati_double(v: f64) -> *mut CIValue; - pub fn ati_tensor(v: *mut C_tensor) -> *mut CIValue; - pub fn ati_string(s: *const c_char) -> *mut CIValue; - pub fn ati_tuple(v: *const *mut CIValue, n: c_int) -> *mut CIValue; - pub fn ati_generic_list(v: *const *mut CIValue, n: c_int) -> *mut CIValue; - pub fn ati_generic_dict(v: *const *mut CIValue, n: c_int) -> *mut CIValue; - pub fn ati_int_list(v: *const i64, n: c_int) -> *mut CIValue; - pub fn ati_double_list(v: *const f64, n: c_int) -> *mut CIValue; - pub fn ati_bool_list(v: *const c_char, n: c_int) -> *mut CIValue; - pub fn ati_string_list(v: *const *const c_char, n: c_int) -> *mut CIValue; - pub fn ati_tensor_list(v: *const *mut C_tensor, n: c_int) -> *mut CIValue; - pub fn ati_device(device_idx: c_int) -> *mut CIValue; - - // Type query - pub fn ati_tag(arg: *mut CIValue) -> c_int; - - // Getters - pub fn ati_to_int(arg: *mut CIValue) -> i64; - pub fn ati_to_bool(arg: *mut CIValue) -> c_int; - pub fn ati_to_double(arg: *mut CIValue) -> f64; - pub fn ati_to_tensor(arg: *mut CIValue) -> *mut C_tensor; - pub fn ati_length(arg: *mut CIValue) -> c_int; - pub fn ati_tuple_length(arg: *mut CIValue) -> c_int; - pub fn ati_to_tuple(arg: *mut CIValue, outputs: *mut *mut CIValue, n: c_int); - pub fn ati_to_generic_list(arg: *mut CIValue, outputs: *mut *mut CIValue, n: c_int); - pub fn ati_to_generic_dict(arg: *mut CIValue, outputs: *mut *mut CIValue, n: c_int); - pub fn ati_to_int_list(arg: *mut CIValue, outputs: *mut i64, n: c_int); - pub fn ati_to_double_list(arg: *mut CIValue, outputs: *mut f64, n: c_int); - pub fn ati_to_bool_list(arg: *mut CIValue, outputs: *mut c_char, n: c_int); - pub fn ati_to_tensor_list(arg: *mut CIValue, outputs: *mut *mut C_tensor, n: c_int); - pub fn ati_to_string(arg: *mut CIValue) -> *mut c_char; - - pub fn ati_clone(arg: *mut CIValue) -> *mut CIValue; - pub fn ati_free(arg: *mut CIValue); - - pub fn ati_object_method_( - arg: *mut CIValue, - method_name: *const c_char, - args: *const *mut CIValue, - n: c_int, - ) -> *mut CIValue; - - pub fn ati_object_getattr_(arg: *mut CIValue, attr_name: *const c_char) -> *mut CIValue; - - pub fn atm_load(filename: *const c_char) -> *mut CModule_; - pub fn atm_load_on_device(filename: *const c_char, device: c_int) -> *mut CModule_; - pub fn atm_load_str(data: *const c_char, sz: size_t) -> *mut CModule_; - pub fn atm_load_str_on_device(data: *const c_char, sz: size_t, device: c_int) -> *mut CModule_; - pub fn atm_forward(m: *mut CModule_, args: *const *mut C_tensor, n: c_int) -> *mut C_tensor; - pub fn atm_forward_(m: *mut CModule_, args: *const *mut CIValue, n: c_int) -> *mut CIValue; - pub fn atm_method( - m: *mut CModule_, - method_name: *const c_char, - args: *const *mut C_tensor, - n: c_int, - ) -> *mut C_tensor; - pub fn atm_method_( - m: *mut CModule_, - method_name: *const c_char, - args: *const *mut CIValue, - n: c_int, - ) -> *mut CIValue; - pub fn atm_create_class_( - m: *mut CModule_, - clz_name: *const c_char, - args: *const *mut CIValue, - n: c_int, - ) -> *mut CIValue; - pub fn atm_eval(m: *mut CModule_); - pub fn atm_train(m: *mut CModule_); - pub fn atm_free(m: *mut CModule_); - pub fn atm_to(m: *mut CModule_, device: c_int, kind: c_int, non_blocking: bool); - pub fn atm_save(m: *mut CModule_, filename: *const c_char); - pub fn atm_get_profiling_mode() -> c_int; - pub fn atm_set_profiling_mode(profiling_mode: c_int); - pub fn atm_fuser_cuda_set_enabled(enabled: bool); - pub fn atm_fuser_cuda_is_enabled() -> bool; - pub fn atm_named_parameters( - m: *mut CModule_, - data: *mut c_void, - f: extern "C" fn(*mut c_void, name: *const c_char, t: *mut C_tensor), - ); - pub fn atm_create_for_tracing( - modl_name: *const c_char, - inputs: *const *mut C_tensor, - ninputs: c_int, - ) -> *mut CModule_; - pub fn atm_end_tracing( - m: *mut CModule_, - fn_name: *const c_char, - outputs: *const *mut C_tensor, - noutputs: c_int, - ); - pub fn atm_set_tensor_expr_fuser_enabled(enabled: c_int); - pub fn atm_get_tensor_expr_fuser_enabled() -> bool; -} diff --git a/v2/dependency-patches/torch-sys-0.24.0/src/python.rs b/v2/dependency-patches/torch-sys-0.24.0/src/python.rs deleted file mode 100644 index 64c4b8c649..0000000000 --- a/v2/dependency-patches/torch-sys-0.24.0/src/python.rs +++ /dev/null @@ -1,12 +0,0 @@ -use super::C_tensor; - -#[repr(C)] -pub struct C_pyobject { - _private: [u8; 0], -} - -extern "C" { - pub fn thp_variable_check(obj: *mut C_pyobject) -> bool; - pub fn thp_variable_wrap(var: *mut C_tensor) -> *mut C_pyobject; - pub fn thp_variable_unpack(obj: *mut C_pyobject) -> *mut C_tensor; -} diff --git a/v2/dependency-patches/torch-sys-0.24.0/src/traits.rs b/v2/dependency-patches/torch-sys-0.24.0/src/traits.rs deleted file mode 100644 index e40190d8d9..0000000000 --- a/v2/dependency-patches/torch-sys-0.24.0/src/traits.rs +++ /dev/null @@ -1,119 +0,0 @@ -macro_rules! list_trait { - ($trait: ident, $ty: ident) => { - pub trait $trait { - fn as_ptr(&self) -> *const $ty; - fn len_i32(&self) -> i32; - } - - impl $trait for $ty { - fn as_ptr(&self) -> *const $ty { - self - } - fn len_i32(&self) -> i32 { - 1 - } - } - - impl $trait for &[$ty] { - fn as_ptr(&self) -> *const $ty { - (*self).as_ptr() - } - fn len_i32(&self) -> i32 { - self.len() as i32 - } - } - - impl $trait for Vec<$ty> { - fn as_ptr(&self) -> *const $ty { - self.as_slice().as_ptr() - } - fn len_i32(&self) -> i32 { - self.len() as i32 - } - } - - impl $trait for &Vec<$ty> { - fn as_ptr(&self) -> *const $ty { - self.as_slice().as_ptr() - } - fn len_i32(&self) -> i32 { - self.len() as i32 - } - } - - impl $trait for &Box<[$ty]> { - fn as_ptr(&self) -> *const $ty { - (**self).as_ptr() - } - fn len_i32(&self) -> i32 { - self.len() as i32 - } - } - - impl $trait for [$ty] { - fn as_ptr(&self) -> *const $ty { - self.as_ptr() - } - fn len_i32(&self) -> i32 { - self.len() as i32 - } - } - - impl $trait for &[$ty; N] { - fn as_ptr(&self) -> *const $ty { - self.as_slice().as_ptr() - } - fn len_i32(&self) -> i32 { - self.len() as i32 - } - } - - impl $trait for [$ty; N] { - fn as_ptr(&self) -> *const $ty { - self.as_slice().as_ptr() - } - fn len_i32(&self) -> i32 { - self.len() as i32 - } - } - }; -} - -list_trait!(DoubleList, f64); -list_trait!(IntList, i64); - -pub trait IntListOption { - fn as_ptr(&self) -> *const i64; - fn len_i32(&self) -> i32; -} - -trait IntListOption_: IntList {} - -impl IntListOption for T { - fn as_ptr(&self) -> *const i64 { - self.as_ptr() - } - fn len_i32(&self) -> i32 { - self.len_i32() - } -} - -impl IntListOption for Option { - fn as_ptr(&self) -> *const i64 { - match self { - Some(v) => v.as_ptr(), - None => std::ptr::null(), - } - } - - fn len_i32(&self) -> i32 { - self.as_ref().map_or(-1, |v| v.len_i32()) - } -} - -impl IntListOption_ for i64 {} -impl IntListOption_ for [i64] {} -impl IntListOption_ for &[i64] {} -impl IntListOption_ for Vec {} -impl IntListOption_ for &Vec {} -impl IntListOption_ for &Box<[i64]> {} From e57d4f33250f78cb95c0ee9f12a7689bdb00ba66 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Thu, 22 Oct 2026 10:35:44 +0200 Subject: [PATCH 011/100] fix(mat): apply host privacy policy to stream and alert prose --- .../wifi-densepose-mat/src/api/websocket.rs | 28 +- .../wifi-densepose-sensing-server/src/main.rs | 2 +- .../src/mat_surface.rs | 306 +++++++++++++++++- .../src/privacy_filter.rs | 28 +- 4 files changed, 352 insertions(+), 12 deletions(-) diff --git a/v2/crates/wifi-densepose-mat/src/api/websocket.rs b/v2/crates/wifi-densepose-mat/src/api/websocket.rs index 708488be25..0bdc1cee89 100644 --- a/v2/crates/wifi-densepose-mat/src/api/websocket.rs +++ b/v2/crates/wifi-densepose-mat/src/api/websocket.rs @@ -37,6 +37,7 @@ use axum::{ State, }, response::Response, + Extension, }; use futures_util::{SinkExt, StreamExt}; use parking_lot::Mutex; @@ -46,6 +47,14 @@ use uuid::Uuid; use super::dto::{WebSocketMessage, WebSocketRequest}; use super::state::AppState; +/// Optional host-owned output policy for serialized WebSocket frames. +/// +/// HTTP response middleware cannot filter frames after the upgrade. A host +/// applying a privacy policy must install this extension on the MAT router. +/// Returning `None` withholds a frame (fail closed). +#[derive(Clone, Copy)] +pub struct StreamOutputFilter(pub fn(&str) -> Option); + /// WebSocket connection handler. /// /// # OpenAPI Specification @@ -75,13 +84,17 @@ use super::state::AppState; /// 101: /// description: WebSocket connection established /// ``` -#[tracing::instrument(skip(state, ws))] -pub async fn ws_handler(State(state): State, ws: WebSocketUpgrade) -> Response { - ws.on_upgrade(move |socket| handle_socket(socket, state)) +#[tracing::instrument(skip(state, ws, filter))] +pub async fn ws_handler( + State(state): State, + filter: Option>, + ws: WebSocketUpgrade, +) -> Response { + ws.on_upgrade(move |socket| handle_socket(socket, state, filter.map(|Extension(f)| f))) } /// Handle an established WebSocket connection. -async fn handle_socket(socket: WebSocket, state: AppState) { +async fn handle_socket(socket: WebSocket, state: AppState, filter: Option) { let (mut sender, mut receiver) = socket.split(); // Subscription state for this connection @@ -102,7 +115,12 @@ async fn handle_socket(socket: WebSocket, state: AppState) { Ok(msg) => { // Check if this message matches subscription filter if subs_clone.lock().should_receive(&msg) { - if let Ok(json) = serde_json::to_string(&msg) { + if let Some(json) = serde_json::to_string(&msg).ok().and_then(|json| { + match filter { + Some(filter) => (filter.0)(&json), + None => Some(json), + } + }) { if sender.send(Message::Text(json)).await.is_err() { break; } diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index 72b76899de..f478ab367f 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -15502,7 +15502,7 @@ async fn main() { .merge(rufield_surface::router(field_surface.clone())) // MAT REST and WebSocket share one incident state on the HTTP origin // used by the app, inside privacy/auth/host and companion protection. - .merge(wifi_densepose_sensing_server::mat_surface::router()) + .merge(wifi_densepose_sensing_server::mat_surface::router(args.privacy_mode)) // #2094: strip biometrics from every JSON response in privacy mode // (no-op otherwise). Applied after the merge so it covers every route. .layer(axum::middleware::from_fn_with_state( diff --git a/v2/crates/wifi-densepose-sensing-server/src/mat_surface.rs b/v2/crates/wifi-densepose-sensing-server/src/mat_surface.rs index f731ec499f..914a9b897b 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/mat_surface.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/mat_surface.rs @@ -2,8 +2,21 @@ //! Incident state is in-memory; CSI ingest must be configured explicitly. /// Attach before host, bearer, companion and privacy middleware are applied. -pub fn router() -> axum::Router { - wifi_densepose_mat::api::create_router(wifi_densepose_mat::api::AppState::new()) +pub fn router(privacy_mode: bool) -> axum::Router { + router_with_state(wifi_densepose_mat::api::AppState::new(), privacy_mode) +} + +fn router_with_state(state: wifi_densepose_mat::api::AppState, privacy_mode: bool) -> axum::Router { + let router = wifi_densepose_mat::api::create_router(state); + if privacy_mode { + router.layer(axum::Extension( + wifi_densepose_mat::api::websocket::StreamOutputFilter( + crate::privacy_filter::redact_json_str, + ), + )) + } else { + router + } } #[cfg(test)] @@ -16,7 +29,7 @@ mod tests { #[tokio::test] async fn real_mat_router_starts_empty_and_reports_optional_ml_truthfully() { - let app = super::router(); + let app = super::router(false); for path in [ "/api/v1/mat/events?page=0&page_size=100", "/api/v1/mat/scan/status", @@ -42,7 +55,7 @@ mod tests { #[tokio::test] async fn mat_reads_mutations_and_stream_remain_behind_host_auth_layers() { - let app = super::router().layer(axum::middleware::from_fn_with_state( + let app = super::router(false).layer(axum::middleware::from_fn_with_state( crate::bearer_auth::AuthState::from_token("mat-test-fixture-token"), crate::bearer_auth::require_bearer, )); @@ -83,7 +96,7 @@ mod tests { #[tokio::test] async fn real_incident_create_and_read_use_only_explicit_test_state() { - let app = super::router(); + let app = super::router(false); let body = r#"{"event_type":"Earthquake","latitude":0,"longitude":0,"description":"EXPLICIT ISOLATED TEST FIXTURE"}"#; let response = app .clone() @@ -116,4 +129,287 @@ mod tests { "EXPLICIT ISOLATED TEST FIXTURE" ); } + + // SYNTHETIC values in isolated in-memory incident state. This exercises the + // real MAT handlers, upgrade, broadcaster and host output/auth policies; + // it makes no claim about sensor detection or the packaged app. + #[tokio::test] + async fn authenticated_nonempty_mat_stream_and_snapshot_obey_privacy() { + use futures_util::StreamExt; + use tokio_tungstenite::{connect_async, tungstenite::client::IntoClientRequest}; + use wifi_densepose_mat::{api::dto::*, domain::*}; + + for privacy in [false, true] { + let state = wifi_densepose_mat::api::AppState::new(); + let app = super::router_with_state(state.clone(), privacy) + .layer(axum::middleware::from_fn_with_state( + crate::privacy_filter::PrivacyFilter::new(privacy), + crate::privacy_filter::redact_json_responses, + )) + .layer(axum::middleware::from_fn_with_state( + crate::bearer_auth::AuthState::from_token("mat-test-fixture-token"), + crate::bearer_auth::require_bearer, + )); + let response = app.clone().oneshot(Request::builder() + .method("POST").uri("/api/v1/mat/events") + .header("authorization", "Bearer mat-test-fixture-token") + .header("content-type", "application/json") + .body(Body::from(r#"{"event_type":"Earthquake","latitude":0,"longitude":0,"description":"SYNTHETIC PRIVACY REGRESSION"}"#)).unwrap() + ).await.unwrap(); + assert_eq!(response.status(), StatusCode::CREATED); + let bytes = to_bytes(response.into_body(), 128 * 1024).await.unwrap(); + let event: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + let event_id = event["id"].as_str().unwrap().parse().unwrap(); + let zone = ScanZone::new("SYNTHETIC", ZoneBounds::rectangle(0., 0., 10., 10.)); + let zone_id = zone.id().clone(); + state + .update_event(event_id, |event| { + event.add_zone(zone); + event + .record_detection( + zone_id.clone(), + VitalSignsReading { + breathing: Some(BreathingPattern { + rate_bpm: 16., + amplitude: 0.8, + regularity: 0.9, + pattern_type: BreathingType::Normal, + }), + heartbeat: None, + movement: Default::default(), + timestamp: chrono::Utc::now(), + confidence: ConfidenceScore::new(0.9), + }, + None, + ) + .unwrap(); + }) + .unwrap(); + let domain_survivor = state.get_event(event_id).unwrap().survivors()[0].clone(); + let alert = wifi_densepose_mat::alerting::AlertGenerator::new() + .generate(&domain_survivor) + .unwrap(); + assert!(alert.payload().message.contains("Breathing: 16.0 BPM")); + state.store_alert(alert.clone(), event_id); + let response = app + .clone() + .oneshot( + Request::builder() + .uri(format!("/api/v1/mat/events/{event_id}/alerts")) + .header("authorization", "Bearer mat-test-fixture-token") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let bytes = to_bytes(response.into_body(), 128 * 1024).await.unwrap(); + let alerts: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(alerts["total"], 1); + assert_eq!(alerts["alerts"][0].get("message").is_none(), privacy); + if !privacy { + assert!(alerts["alerts"][0]["message"] + .as_str() + .unwrap() + .contains("Breathing: 16.0 BPM")); + } + // Non-empty REST snapshot, including an array-nested vital_signs + // object: policy parity with the socket is the security invariant. + let response = app + .clone() + .oneshot( + Request::builder() + .uri(format!("/api/v1/mat/events/{event_id}/survivors")) + .header("authorization", "Bearer mat-test-fixture-token") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let bytes = to_bytes(response.into_body(), 128 * 1024).await.unwrap(); + let snapshot: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(snapshot["total"], 1); + assert_eq!( + snapshot["survivors"][0].get("vital_signs").is_none(), + privacy + ); + if !privacy { + assert_eq!( + snapshot["survivors"][0]["vital_signs"]["breathing_rate"], + 16. + ); + } + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let served_app = app.clone(); + let server = + tokio::spawn(async move { axum::serve(listener, served_app).await.unwrap() }); + let url = format!("ws://{addr}/ws/mat/stream"); + match connect_async(&url).await { + Err(tokio_tungstenite::tungstenite::Error::Http(response)) => { + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + } + other => panic!("unauthenticated MAT upgrade was not denied: {other:?}"), + } + let mut request = url.into_client_request().unwrap(); + request.headers_mut().insert( + "authorization", + "Bearer mat-test-fixture-token".parse().unwrap(), + ); + let (mut socket, response) = connect_async(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::SWITCHING_PROTOCOLS); + tokio::time::timeout(std::time::Duration::from_secs(5), async { + while state.subscriber_count() == 0 { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + let survivor = SurvivorResponse { + id: *state.get_event(event_id).unwrap().survivors()[0] + .id() + .as_uuid(), + zone_id: *zone_id.as_uuid(), + status: SurvivorStatusDto::Active, + triage_status: TriageStatusDto::Minor, + location: None, + vital_signs: VitalSignsSummaryDto { + breathing_rate: Some(16.), + breathing_type: Some("Normal".into()), + heart_rate: Some(72.), + has_heartbeat: true, + has_movement: false, + movement_type: None, + timestamp: chrono::Utc::now(), + }, + confidence: 0.9, + first_detected: chrono::Utc::now(), + last_updated: chrono::Utc::now(), + is_deteriorating: false, + metadata: None, + }; + for (kind, message) in [ + ( + "survivor_detected", + WebSocketMessage::SurvivorDetected { + event_id, + survivor: survivor.clone(), + }, + ), + ( + "survivor_updated", + WebSocketMessage::SurvivorUpdated { event_id, survivor }, + ), + ( + "event_status_changed", + WebSocketMessage::EventStatusChanged { + event_id, + old_status: EventStatusDto::Initializing, + new_status: EventStatusDto::Active, + }, + ), + ] { + state.broadcast(message); + let frame = tokio::time::timeout(std::time::Duration::from_secs(5), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + let value: serde_json::Value = + serde_json::from_str(frame.to_text().unwrap()).unwrap(); + assert_eq!(value["type"], kind); + assert_eq!(value["event_id"], event_id.to_string()); + if kind.starts_with("survivor_") { + assert_eq!(value["survivor"]["confidence"], 0.9); + assert_eq!( + value["survivor"].get("vital_signs").is_none(), + privacy, + "{value}" + ); + if !privacy { + assert_eq!(value["survivor"]["vital_signs"]["heart_rate"], 72.); + assert_eq!(value["survivor"]["vital_signs"]["breathing_rate"], 16.); + } + } else { + assert_eq!(value["new_status"], "Active"); + } + } + let alert_response = AlertResponse { + id: *alert.id().as_uuid(), + survivor_id: *alert.survivor_id().as_uuid(), + priority: alert.priority().into(), + status: alert.status().clone().into(), + title: alert.payload().title.clone(), + message: alert.payload().message.clone(), + triage_status: alert.payload().triage_status.clone().into(), + location: None, + recommended_action: Some(alert.payload().recommended_action.clone()), + created_at: *alert.created_at(), + acknowledged_at: None, + acknowledged_by: None, + escalation_count: 0, + }; + for message in [ + WebSocketMessage::AlertCreated { + event_id, + alert: alert_response.clone(), + }, + WebSocketMessage::AlertUpdated { + event_id, + alert: alert_response, + }, + ] { + state.broadcast(message); + let frame = tokio::time::timeout(std::time::Duration::from_secs(5), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + let value: serde_json::Value = + serde_json::from_str(frame.to_text().unwrap()).unwrap(); + assert_eq!(value["alert"]["id"], alert.id().to_string()); + assert_eq!(value["alert"].get("message").is_none(), privacy, "{value}"); + assert!(value["alert"].get("recommended_action").is_some()); + if !privacy { + assert!(value["alert"]["message"] + .as_str() + .unwrap() + .contains("Breathing: 16.0 BPM")); + } + } + // Actual REST mutation -> generated AlertUpdated -> authenticated + // socket, not only a manually constructed broadcast fixture. + let response = app + .oneshot( + Request::builder() + .method("POST") + .uri(format!("/api/v1/mat/alerts/{}/acknowledge", alert.id())) + .header("authorization", "Bearer mat-test-fixture-token") + .header("content-type", "application/json") + .body(Body::from( + r#"{"acknowledged_by":"SYNTHETIC PRIVACY TEST"}"#, + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let bytes = to_bytes(response.into_body(), 128 * 1024).await.unwrap(); + let acknowledgement: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(acknowledgement["alert"].get("message").is_none(), privacy); + let frame = tokio::time::timeout(std::time::Duration::from_secs(5), socket.next()) + .await + .unwrap() + .unwrap() + .unwrap(); + let value: serde_json::Value = serde_json::from_str(frame.to_text().unwrap()).unwrap(); + assert_eq!(value["type"], "alert_updated"); + assert_eq!(value["alert"]["status"], "Acknowledged"); + assert_eq!(value["alert"].get("message").is_none(), privacy); + socket.close(None).await.unwrap(); + server.abort(); + } + } } diff --git a/v2/crates/wifi-densepose-sensing-server/src/privacy_filter.rs b/v2/crates/wifi-densepose-sensing-server/src/privacy_filter.rs index ad51370db9..e02c302837 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/privacy_filter.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/privacy_filter.rs @@ -14,7 +14,9 @@ //! breathing rate and pose keypoints. Presence, motion, person count, zone //! and the coarse posture label are kept, as they are on MQTT. Keys are //! removed, not nulled, so a client can't tell a suppressed value from one -//! the server never had. +//! the server never had. MAT alert messages are also removed because their +//! generated prose embeds literal breathing and heart rates. Structured alert +//! identity, status, triage and recommended action remain available. //! //! The filter fails closed: a frame or JSON body that can't be parsed is //! dropped (WebSocket, recording) or replaced with a 500 (REST) rather than @@ -72,6 +74,15 @@ impl PrivacyFilter { pub fn redact_value(value: &mut serde_json::Value) { match value { serde_json::Value::Object(map) => { + // MAT's generated alert prose embeds literal HR/BR values. It is + // not safe merely to remove the structured vital_signs object. + // Match the alert DTO shape so ordinary error messages survive. + if map.contains_key("survivor_id") + && map.contains_key("priority") + && map.contains_key("triage_status") + { + map.remove("message"); + } map.retain(|key, _| !BIOMETRIC_KEYS.contains(&key.as_str())); for child in map.values_mut() { redact_value(child); @@ -197,6 +208,21 @@ mod tests { assert_eq!(v["mmwave"]["present"], true); } + #[test] + fn redact_mat_alert_prose_at_any_depth_without_hiding_errors() { + let mut value = json!({ + "alerts": [{"survivor_id": "fixture", "priority": "High", + "triage_status": "Delayed", "message": "Breathing: 16.0 BPM", + "title": "Survivor Detected", "status": "Pending"}], + "error": {"code": "fixture", "message": "ordinary error"} + }); + redact_value(&mut value); + assert!(value["alerts"][0].get("message").is_none()); + assert_eq!(value["alerts"][0]["status"], "Pending"); + assert_eq!(value["alerts"][0]["title"], "Survivor Detected"); + assert_eq!(value["error"]["message"], "ordinary error"); + } + #[test] fn redact_json_str_fails_closed_on_invalid_json() { assert!(redact_json_str("not json").is_none()); From 383ea4d5648701d9b3e7311ec1d2cf5af700c50f Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Thu, 22 Oct 2026 10:39:07 +0200 Subject: [PATCH 012/100] fix(mat): withhold biometric domain-event debug snapshots --- .../src/mat_surface.rs | 35 +++++++++++++++++++ .../src/privacy_filter.rs | 8 ++++- 2 files changed, 42 insertions(+), 1 deletion(-) diff --git a/v2/crates/wifi-densepose-sensing-server/src/mat_surface.rs b/v2/crates/wifi-densepose-sensing-server/src/mat_surface.rs index 914a9b897b..8df3b81cca 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/mat_surface.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/mat_surface.rs @@ -186,6 +186,41 @@ mod tests { }) .unwrap(); let domain_survivor = state.get_event(event_id).unwrap().survivors()[0].clone(); + state + .event_store() + .append(events::DomainEvent::Detection( + events::DetectionEvent::SurvivorDetected { + survivor_id: domain_survivor.id().clone(), + zone_id: zone_id.clone(), + vital_signs: domain_survivor.vital_signs().latest().unwrap().clone(), + location: None, + timestamp: chrono::Utc::now(), + }, + )) + .unwrap(); + let response = app + .clone() + .oneshot( + Request::builder() + .uri("/api/v1/mat/events/domain") + .header("authorization", "Bearer mat-test-fixture-token") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let bytes = to_bytes(response.into_body(), 128 * 1024).await.unwrap(); + let domain_events: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(domain_events["total"], 1); + assert_eq!(domain_events["events"][0]["event_type"], "SurvivorDetected"); + assert_eq!(domain_events["events"][0].get("details").is_none(), privacy); + if !privacy { + assert!(domain_events["events"][0]["details"] + .as_str() + .unwrap() + .contains("rate_bpm: 16.0")); + } let alert = wifi_densepose_mat::alerting::AlertGenerator::new() .generate(&domain_survivor) .unwrap(); diff --git a/v2/crates/wifi-densepose-sensing-server/src/privacy_filter.rs b/v2/crates/wifi-densepose-sensing-server/src/privacy_filter.rs index e02c302837..244bfc6190 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/privacy_filter.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/privacy_filter.rs @@ -16,7 +16,8 @@ //! removed, not nulled, so a client can't tell a suppressed value from one //! the server never had. MAT alert messages are also removed because their //! generated prose embeds literal breathing and heart rates. Structured alert -//! identity, status, triage and recommended action remain available. +//! identity, status, triage and recommended action remain available. MAT +//! domain-event Debug details are withheld for the same reason; type/time stay. //! //! The filter fails closed: a frame or JSON body that can't be parsed is //! dropped (WebSocket, recording) or replaced with a 500 (REST) rather than @@ -83,6 +84,11 @@ pub fn redact_value(value: &mut serde_json::Value) { { map.remove("message"); } + // MAT's domain-event DTO is a Debug snapshot, including the full + // vital reading as text. Keep its event type/time, withhold prose. + if map.contains_key("event_type") && map.contains_key("timestamp") { + map.remove("details"); + } map.retain(|key, _| !BIOMETRIC_KEYS.contains(&key.as_str())); for child in map.values_mut() { redact_value(child); From d05aa59ccafc76e7339dc5003fb0ed5c3bbc6d7d Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Thu, 22 Oct 2026 13:44:52 +0200 Subject: [PATCH 013/100] docs(adr): clarify MAT privacy and companion state lifetimes --- .../ADR-001-wifi-mat-disaster-detection.md | 23 +++++++++++++++++++ ...DR-355-local-empty-room-bootstrap-prior.md | 13 +++++++++++ .../ADR-419-secure-local-companion-pairing.md | 21 +++++++++++++++++ docs/adr/README.md | 1 + 4 files changed, 58 insertions(+) diff --git a/docs/adr/ADR-001-wifi-mat-disaster-detection.md b/docs/adr/ADR-001-wifi-mat-disaster-detection.md index f03c55346b..cb997aee4f 100644 --- a/docs/adr/ADR-001-wifi-mat-disaster-detection.md +++ b/docs/adr/ADR-001-wifi-mat-disaster-detection.md @@ -165,6 +165,29 @@ The module integrates with existing crates through adapters: | False positives from animals | Size/pattern classification | | Power constraints in field | Low-power modes, solar charging | +## Hosted API and privacy boundary amendment (2026-10-22) + +PR #2249 composes the MAT API into the sensing-server host with one in-memory +MAT `AppState`. It remains inside host validation and authentication; optional +ML unavailability and empty incident state must be reported explicitly. This +composition does not implement a durable incident database or establish the +physical performance targets above. + +The host's `--privacy-mode` applies to MAT REST output and WebSocket broadcasts. +HTTP response middleware cannot inspect messages after an upgrade, so the host +installs the same recursive serialized-output filter in the MAT stream path. +Besides structured vital signs, this removes generated alert prose and domain +event Debug details that can contain literal heart/breathing rates. Permitted +status, identity and coarse metadata remain; ordinary error messages and +privacy-off serialization retain their existing behavior. Standalone MAT hosts +must install their own output policy; the extension hook alone is not a privacy +guarantee. + +Authenticated nonempty REST and real-socket stream regressions cover the output +boundary in both privacy modes. They use synthetic incidents and do not qualify +survivor detection, clinical accuracy, phone transport, or restart persistence. +Native companion stream support remains governed by ADR-419's allowlist. + ## References - [WiFi-based Vital Signs Monitoring](https://dl.acm.org/doi/10.1145/3130944) diff --git a/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md b/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md index b47082dbf3..641beed67c 100644 --- a/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md +++ b/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md @@ -140,6 +140,19 @@ This is measured negative evidence for one installation. No occupied positive holdout was recorded, so it does not prove person recall, person counting, or cross home accuracy improvement. +## Full room receipt versus restart prior (2026-10-22) + +An explicit room calibration can name multiple contributing radios, but this +persisted field model remains bound to exactly one model-writing radio and its +CSI grid. On helper restart, the full explicit session, room binding and receipt +are not restored. Client history cannot recreate that live server authority. + +Verify all three radios' fresh reception independently of the restored prior's +single source. A one-radio `bootstrap_only` source set is neither proof that +other radios stopped receiving nor proof of a restored three-radio calibration. +Original expiry and negative-only authority remain unchanged. Retaining a room +or scan on an app is a separate lifecycle from restarting its paired helper. + ## Acceptance test With a physically empty room, complete calibration on stable nodes, promote diff --git a/docs/adr/ADR-419-secure-local-companion-pairing.md b/docs/adr/ADR-419-secure-local-companion-pairing.md index aeaa5ef994..844d02017a 100644 --- a/docs/adr/ADR-419-secure-local-companion-pairing.md +++ b/docs/adr/ADR-419-secure-local-companion-pairing.md @@ -116,3 +116,24 @@ The consumer independently opts in to foreground sharing and renders remote geometry separately from local sensing/calibration authority. Software protocol, real socket and owner/companion middleware checks do not establish physical phone-to-Mac capture accuracy or calibration acceptance. + +## Retained replica boundaries (2026-10-22) + +The helper's durable `companion-room.json` is one canonical geometry envelope, +not a replica of either app's scan, models or local evidence history. The app +containers own those records separately; Cognitum sign-in and local pairing do +not synchronize them. Expiry or revocation withdraws transport access and does +not establish deletion of already retained client data. + +Every accepted room PUT increments the server revision. A client may avoid an +unnecessary new binding by validating and reusing the exact unchanged published +configuration instead of issuing another PUT. Importing that identical envelope +must preserve legitimate local evidence; a deliberate changed binding withdraws +current authority while local records remain historical. The client retention +repair is proposed in ruview-mobile ADR-047; it is not present in TestFlight +build 2026102213 and requires a subsequent delivered-build restart check. + +The sensor preview remains transient and currently carries no room-binding +digest. A displayed phone mesh is not a saved canonical room, durable draft, +live AR alignment or RF calibration receipt. MAT HTTP composition likewise does +not expand native companion WebSocket paths; see ADR-001's privacy amendment. diff --git a/docs/adr/README.md b/docs/adr/README.md index d4b5167dff..b4c0226cb0 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -135,6 +135,7 @@ Statuses: **Proposed** (under discussion), **Accepted** (approved and/or impleme | [ADR-346](ADR-346-fail-closed-edge-occupancy-evidence.md) | Fail closed ESP32 occupancy evidence | Accepted (C6 occupancy integrity qualified) | | [ADR-347](ADR-347-rate-aware-esp32-temporal-sensing.md) | Rate aware ESP32 temporal sensing | Accepted (C6 timing and transport qualified) | | [ADR-363](ADR-363-native-macos-usb-node-onboarding.md) | Native macOS USB node onboarding with nonce bound discovery and fresh server verification | Accepted and physically qualified on one ESP32 S3; C6 and signed distribution pending | +| [ADR-419](ADR-419-secure-local-companion-pairing.md) | Secure local Mac–iPhone pairing, canonical room publication and transient sensor preview | Accepted decision; software verified, physical qualification pending | | [ADR-115](ADR-115-home-assistant-integration.md) | Home Assistant integration via MQTT auto-discovery + Matter bridge (HA-DISCO + HA-FABRIC + HA-MIND) | Accepted (MQTT track) / Proposed (Matter SDK P8b) | | [ADR-169](ADR-169-adam-mode-light-theme.md) | adam-mode — light theme toggle for the three.js realtime demo | Proposed | | [ADR-170](ADR-170-yoga-mode-pose-system.md) | yoga-mode — yoga pose detection, classification, and scoring for the three.js realtime demo | Proposed | From a909f8baf04c1c434c188ebffe5ee60f72572157 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Fri, 23 Oct 2026 20:22:28 +0200 Subject: [PATCH 014/100] fix(companion): isolate account-owned helpers and session authority --- v2/crates/ruview-auth/src/lib.rs | 2 +- v2/crates/ruview-auth/src/verify.rs | 75 ++++++++- .../ruview-auth/tests/verifier_matrix.rs | 46 +++++- .../src/account_profile.rs | 152 ++++++++++++++++++ .../src/bearer_auth.rs | 86 ++++++++++ .../src/browser_session.rs | 12 +- .../src/companion.rs | 36 ++++- .../src/companion_integration.rs | 9 +- .../wifi-densepose-sensing-server/src/lib.rs | 1 + .../wifi-densepose-sensing-server/src/main.rs | 35 ++-- .../src/model_manager.rs | 3 + 11 files changed, 431 insertions(+), 26 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/account_profile.rs diff --git a/v2/crates/ruview-auth/src/lib.rs b/v2/crates/ruview-auth/src/lib.rs index 821abed8d1..50ce1acacd 100644 --- a/v2/crates/ruview-auth/src/lib.rs +++ b/v2/crates/ruview-auth/src/lib.rs @@ -86,4 +86,4 @@ pub use jwks::{JwksCache, JwksError, JwksFetcher}; #[cfg(feature = "ureq-transport")] pub use jwks::UreqFetcher; pub use principal::{scope, Principal}; -pub use verify::{extract_bearer, verify_access_token, VerifierConfig, VerifyError}; +pub use verify::{extract_bearer, verify_access_token, verify_account_identity, AccountIdentity, VerifierConfig, VerifyError}; diff --git a/v2/crates/ruview-auth/src/verify.rs b/v2/crates/ruview-auth/src/verify.rs index e5bd5c7b88..a512815c8c 100644 --- a/v2/crates/ruview-auth/src/verify.rs +++ b/v2/crates/ruview-auth/src/verify.rs @@ -95,6 +95,8 @@ pub enum VerifyError { LongLivedCredential, #[error("token carries no account_id and cannot be attributed")] MissingAccountId, + #[error("token does not contain a bounded interactive user ownership context")] + InvalidOwnership, #[error("token does not carry the required scope {required:?}")] MissingScope { required: String }, /// Minted for a different Cognitum product. `client_id` is the platform's @@ -126,6 +128,12 @@ struct AccessTokenClaims { #[serde(default)] jti: String, exp: i64, + #[serde(default)] + iat: i64, + #[serde(default)] + iss: Option, + #[serde(default)] + aud: Option, /// Long-lived, non-rotating setup credential. Absent on older tokens. #[serde(default)] setup: bool, @@ -166,11 +174,11 @@ pub struct VerifierConfig { /// /// Every rejection path returns a typed error; none of them return a partially /// trusted principal. -pub fn verify_access_token( +fn verified_interactive_claims( token: &str, jwks: &JwksCache, config: &VerifierConfig, -) -> Result { +) -> Result { let header = decode_header(token).map_err(|e| VerifyError::Malformed(e.to_string()))?; // Compared, never selected. `decode` below independently enforces the same @@ -217,6 +225,15 @@ pub fn verify_access_token( return Err(VerifyError::LongLivedCredential); } + Ok(claims) +} + +pub fn verify_access_token( + token: &str, + jwks: &JwksCache, + config: &VerifierConfig, +) -> Result { + let claims = verified_interactive_claims(token, jwks, config)?; let account_id = claims.account_id.filter(|a| !a.is_empty()); let Some(account_id) = account_id else { // meta-llm requires this so a token cannot bill an account it doesn't @@ -246,6 +263,60 @@ pub fn verify_access_token( Ok(principal) } +/// Local-data ownership is a user/tenant/workspace identity, not a billing +/// principal. This separate type cannot be used as a metering Principal. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct AccountIdentity { + pub issuer: String, + pub subject: String, + pub org_id: String, + pub workspace_id: String, + pub expires_at: i64, + _verified: (), +} + +fn canonical_uuid(value: &str) -> Option { + (value.len() == 36 && value.bytes().enumerate().all(|(index, byte)| { + if [8, 13, 18, 23].contains(&index) { byte == b'-' } else { byte.is_ascii_hexdigit() } + })).then(|| value.to_ascii_lowercase()) +} + +/// Verify account ownership with the existing fixed ES256/JWKS chain. Unlike +/// verify_access_token, this does not require the optional Firebase billing ID. +/// Existing billing/attribution acceptance is unchanged. +pub fn verify_account_identity( + token: &str, + jwks: &JwksCache, + config: &VerifierConfig, +) -> Result { + if token.len() > 16384 { return Err(VerifyError::InvalidOwnership); } + let claims = verified_interactive_claims(token, jwks, config)?; + let now = std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH) + .map_err(|_| VerifyError::InvalidOwnership)?.as_secs() as i64; + let issuer = config.issuer.trim_end_matches('/'); + // Older Identity tokens omit iss/aud. The configured trusted JWKS binds + // their issuer, and exact client_id binds their audience. Present values + // must match; no token field can choose a key URL or broaden the client. + if claims.client_id != "ruview" + || claims.iss.as_deref().is_some_and(|value| value != issuer) + || claims.aud.as_deref().is_some_and(|value| value != "ruview") + || claims.iat > now + 30 || claims.exp <= now || claims.exp <= claims.iat + || claims.exp.checked_sub(claims.iat).is_none_or(|ttl| ttl > 3600) + { return Err(VerifyError::InvalidOwnership); } + let scopes: Vec<&str> = claims.scope.split_whitespace().collect(); + if !scopes.contains(&config.required_scope.as_str()) { + return Err(VerifyError::MissingScope { required: config.required_scope.clone() }); + } + Ok(AccountIdentity { + issuer: issuer.into(), + subject: canonical_uuid(&claims.sub).ok_or(VerifyError::InvalidOwnership)?, + org_id: canonical_uuid(&claims.org_id).ok_or(VerifyError::InvalidOwnership)?, + workspace_id: canonical_uuid(&claims.workspace_id).ok_or(VerifyError::InvalidOwnership)?, + expires_at: claims.exp, + _verified: (), + }) +} + /// Extract a bearer token from an `Authorization` header value. /// /// The scheme is matched **case-insensitively** per RFC 7235 §2.1, and leading diff --git a/v2/crates/ruview-auth/tests/verifier_matrix.rs b/v2/crates/ruview-auth/tests/verifier_matrix.rs index e376d83722..065a37c410 100644 --- a/v2/crates/ruview-auth/tests/verifier_matrix.rs +++ b/v2/crates/ruview-auth/tests/verifier_matrix.rs @@ -20,7 +20,7 @@ use p256::ecdsa::SigningKey; use p256::pkcs8::{EncodePrivateKey, LineEnding}; use ruview_auth::{ jwks::{JwksError, JwksFetcher}, - scope, verify_access_token, JwksCache, VerifierConfig, VerifyError, + scope, verify_access_token, verify_account_identity, JwksCache, VerifierConfig, VerifyError, }; use serde_json::json; @@ -142,6 +142,50 @@ fn verify(token: &str, required_scope: &str) -> Result serde_json::Value { + let mut claims = valid_claims(); + claims["org_id"] = json!("20000000-0000-4000-8000-000000000001"); + claims["workspace_id"] = json!("30000000-0000-4000-8000-000000000001"); + claims["scope"] = json!("sensing:read spaces:read"); + claims.as_object_mut().unwrap().remove("account_id"); + claims +} + +#[test] +fn local_ownership_is_a_user_identity_without_requiring_firebase_billing() { + let token = sign(&ownership_claims()); + let identity = verify_account_identity(&token, &jwks_serving_test_key(), &config_for(scope::SENSING_READ)).unwrap(); + assert_eq!(identity.subject, "0f8fad5b-d9cb-469f-a165-70867728950e"); + assert_eq!(identity.org_id, "20000000-0000-4000-8000-000000000001"); + // This new verifier must not weaken the existing billing/attribution path. + assert!(matches!(verify(&token, scope::SENSING_READ), Err(VerifyError::MissingAccountId))); +} + +#[test] +fn local_ownership_rejects_wrong_or_unbounded_interactive_identity() { + for (field, value) in [ + ("sub", json!("")), ("sub", json!("../other-user")), + ("org_id", json!("org-1")), ("workspace_id", json!("")), + ("client_id", json!("other")), ("typ", json!("inference")), + ("setup", json!(true)), ("workload", json!(true)), + ("scope", json!("spaces:read")), ("iat", json!(now() + 300)), + ("exp", json!(now() + 7200)), ("exp", json!(now() - 1)), + ("iss", json!("https://attacker.example")), ("aud", json!("other")), + ] { + let mut claims = ownership_claims(); claims[field] = value; + assert!(verify_account_identity(&sign(&claims), &jwks_serving_test_key(), &config_for(scope::SENSING_READ)).is_err(), "accepted field {field}"); + } +} + +#[test] +fn local_ownership_verifies_signature_and_supports_current_issuer_claims() { + let mut claims = ownership_claims(); + claims["iss"] = json!(TEST_ISSUER); claims["aud"] = json!("ruview"); + assert!(verify_account_identity(&sign(&claims), &jwks_serving_test_key(), &config_for(scope::SENSING_READ)).is_ok()); + assert!(verify_account_identity(&sign_with(&claims, other_key()), &jwks_serving_test_key(), &config_for(scope::SENSING_READ)).is_err()); + assert!(verify_account_identity(ALG_NONE_TOKEN, &jwks_serving_test_key(), &config_for(scope::SENSING_READ)).is_err()); +} + // ─────────────────────────── accept ─────────────────────────── #[test] diff --git a/v2/crates/wifi-densepose-sensing-server/src/account_profile.rs b/v2/crates/wifi-densepose-sensing-server/src/account_profile.rs new file mode 100644 index 0000000000..44b263bacd --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/account_profile.rs @@ -0,0 +1,152 @@ +//! Private helper working directory and durable account ownership witness. +//! Profile selection comes from the authenticated native app, never an HTTP +//! request. Unknown/nonempty legacy data cannot become a new account's data. +use std::{ + io, + path::{Path, PathBuf}, +}; + +const OWNER_FILE: &str = "account-owner-v1.json"; +const SCHEMA: &str = "ruview.account-owner.v1"; + +/// A desktop-owned helper cannot inherit a device-wide model mount. Manual +/// unowned server deployments retain their existing MODELS_DIR contract. +pub fn models_directory( + owned: bool, + data_dir: &Path, + override_dir: Option<&std::ffi::OsStr>, +) -> PathBuf { + if owned { + data_dir.join("models") + } else { + override_dir + .map(PathBuf::from) + .unwrap_or_else(|| data_dir.join("models")) + } +} + +pub fn prepare_directory(scope: &str, directory: &Path) -> io::Result { + if scope.len() != 64 + || !scope + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + || !directory.is_absolute() + || directory.file_name().and_then(|v| v.to_str()) != Some("SensingServer") + || directory + .parent() + .and_then(Path::file_name) + .and_then(|v| v.to_str()) + != Some(scope) + { + return Err(io::Error::other("invalid owned helper directory")); + } + if let Some(parent) = directory.parent().and_then(|path| path.canonicalize().ok()) { + if parent.file_name().and_then(|v| v.to_str()) != Some(scope) { + return Err(io::Error::other("account directory alias rejected")); + } + } + crate::companion::private_dir(directory)?; + let canonical = directory.canonicalize()?; + if canonical + .parent() + .and_then(Path::file_name) + .and_then(|v| v.to_str()) + != Some(scope) + { + return Err(io::Error::other( + "helper directory belongs to another account namespace", + )); + } + let witness = canonical.join(OWNER_FILE); + match std::fs::symlink_metadata(&witness) { + Ok(_) => { + let value: serde_json::Value = + serde_json::from_slice(&crate::companion::private_read(&witness)?) + .map_err(io::Error::other)?; + if value.get("schema").and_then(|v| v.as_str()) != Some(SCHEMA) + || value.get("scope_key").and_then(|v| v.as_str()) != Some(scope) + { + return Err(io::Error::other( + "helper account ownership witness mismatch", + )); + } + } + Err(error) if error.kind() == io::ErrorKind::NotFound => { + if std::fs::read_dir(&canonical)?.next().is_some() { + return Err(io::Error::other( + "existing helper data needs explicit owner migration", + )); + } + let value = + serde_json::to_vec(&serde_json::json!({"schema": SCHEMA, "scope_key": scope})) + .map_err(io::Error::other)?; + crate::companion::atomic_private_write(&witness, &value)?; + if crate::companion::private_read(&witness)? != value { + return Err(io::Error::other("helper ownership readback failed")); + } + } + Err(error) => return Err(error), + } + Ok(canonical) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn owned_models_ignore_a_device_wide_directory_override() { + let a = Path::new("/fixture/account-A/models"); + let b = Path::new("/fixture/account-B/SensingServer"); + assert_eq!( + models_directory(true, b, Some(a.as_os_str())), + b.join("models") + ); + assert_eq!(models_directory(false, b, Some(a.as_os_str())), a); + } + #[test] + fn independent_profiles_keep_their_own_witness_and_legacy_is_not_adopted() { + let root = tempfile::tempdir().unwrap(); + let a = "a".repeat(64); + let b = "b".repeat(64); + let a_dir = root.path().join(&a).join("SensingServer"); + let b_dir = root.path().join(&b).join("SensingServer"); + let a_path = prepare_directory(&a, &a_dir).unwrap(); + let a_before = std::fs::read(a_path.join(OWNER_FILE)).unwrap(); + assert_eq!(prepare_directory(&a, &a_dir).unwrap(), a_path); + prepare_directory(&b, &b_dir).unwrap(); + assert_eq!(std::fs::read(a_path.join(OWNER_FILE)).unwrap(), a_before); + assert!(prepare_directory(&b, &a_dir).is_err()); + let legacy = root.path().join("c".repeat(64)).join("SensingServer"); + std::fs::create_dir_all(&legacy).unwrap(); + std::fs::write(legacy.join("old-model.json"), "legacy bytes").unwrap(); + assert!(prepare_directory(&"c".repeat(64), &legacy).is_err()); + assert_eq!( + std::fs::read_to_string(legacy.join("old-model.json")).unwrap(), + "legacy bytes" + ); + assert!(!legacy.join(OWNER_FILE).exists()); + } + #[test] + fn mismatched_or_future_witness_is_preserved_and_rejected() { + let root = tempfile::tempdir().unwrap(); + let key = "a".repeat(64); + let dir = prepare_directory(&key, &root.path().join(&key).join("SensingServer")).unwrap(); + let future = br#"{"schema":"unknown.future","scope_key":"unreadable"}"#; + crate::companion::atomic_private_write(&dir.join(OWNER_FILE), future).unwrap(); + assert!(prepare_directory(&key, &dir).is_err()); + assert_eq!(std::fs::read(dir.join(OWNER_FILE)).unwrap(), future); + assert!(prepare_directory("../legacy", &dir).is_err()); + assert!(prepare_directory(&key, Path::new("relative/SensingServer")).is_err()); + } + #[cfg(unix)] + #[test] + fn symlink_to_another_account_profile_is_rejected() { + let root = tempfile::tempdir().unwrap(); + let a = "a".repeat(64); + let b = "b".repeat(64); + let a_dir = prepare_directory(&a, &root.path().join(&a).join("SensingServer")).unwrap(); + std::fs::create_dir(root.path().join(&b)).unwrap(); + std::os::unix::fs::symlink(&a_dir, root.path().join(&b).join("SensingServer")).unwrap(); + assert!(prepare_directory(&b, &root.path().join(&b).join("SensingServer")).is_err()); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs b/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs index 49667f3f18..33e27bcda1 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs @@ -55,9 +55,18 @@ use axum::{ response::{IntoResponse, Response}, }; use ruview_auth::{scope, verify_access_token, JwksCache, UreqFetcher, VerifierConfig}; +use sha2::{Digest, Sha256}; /// Environment variable that gates the middleware. Unset / empty ⇒ auth off. pub const API_TOKEN_ENV: &str = "RUVIEW_API_TOKEN"; +pub const OWNER_ACCOUNT_SCOPE_ENV: &str = "RUVIEW_OWNER_ACCOUNT_SCOPE"; + +fn account_scope_key(identity: &ruview_auth::AccountIdentity) -> String { + let encoded = serde_json::to_vec(&["ruview.account-data.v1", identity.issuer.as_str(), + identity.subject.as_str(), identity.org_id.as_str(), identity.workspace_id.as_str()]) + .expect("string array serializes"); + format!("{:x}", Sha256::digest(encoded)) +} /// Issuer origin of the Cognitum authorization server. Setting this enables /// OAuth verification; unset ⇒ OAuth off and behaviour is unchanged. @@ -239,6 +248,8 @@ impl std::fmt::Debug for OAuthState { /// [`AuthState::from_env`]'s contract about failing closed. #[derive(Debug, thiserror::Error)] pub enum OAuthConfigError { + #[error("owned account scope requires a bounded namespace, owner credential, fixed Cognitum issuer/client and authenticated streams")] + InvalidOwnerContext, #[error("{OAUTH_ISSUER_ENV} is set but empty")] EmptyIssuer, #[error("JWKS at {url} is unreachable, so no token could ever be verified: {source}")] @@ -258,6 +269,7 @@ pub enum OAuthConfigError { /// should not be the one that does not. #[derive(Clone, Default)] pub struct AuthState { + owner_account_scope: Option, /// The expected static bearer token, if any. token: Option>, /// Cognitum OAuth verification, if enabled. @@ -288,6 +300,7 @@ impl AuthState { AuthState::default() } else { AuthState { + owner_account_scope: None, token: Some(Arc::new(s)), oauth: None, tickets: crate::ws_ticket::TicketStore::new(), @@ -362,7 +375,16 @@ impl AuthState { WS_PATHS.join(", ") ); } + let owner_account_scope = match std::env::var(OWNER_ACCOUNT_SCOPE_ENV) { + Err(std::env::VarError::NotPresent) => None, + Ok(value) if value.len() == 64 && value.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + && token.is_some() && !legacy_ws + && std::env::var(OAUTH_JWKS_URL_ENV).ok().as_deref().is_none_or(|url| url == "https://auth.cognitum.one/.well-known/jwks.json") + && oauth.as_ref().is_some_and(|o| o.issuer == COGNITUM_ISSUER && o.allowed_client_ids == [DEFAULT_CLIENT_ID]) => Some(value), + _ => return Err(OAuthConfigError::InvalidOwnerContext), + }; Ok(AuthState { + owner_account_scope, token, oauth, tickets: crate::ws_ticket::TicketStore::new(), @@ -394,6 +416,7 @@ impl AuthState { &self, token: &str, ) -> Result { + self.verify_account_ownership(token).map_err(|_| ruview_auth::VerifyError::InvalidOwnership)?; let oauth = self .oauth .as_ref() @@ -409,6 +432,22 @@ impl AuthState { ) } + pub fn owner_account_scope(&self) -> Option<&str> { self.owner_account_scope.as_deref() } + + /// The companion proof alone is not Cognitum account ownership. Require + /// the same verified user/tenant/workspace for every owned TLS request. + /// Unscoped manually configured servers retain their existing contract. + pub fn verify_account_ownership(&self, token: &str) -> Result, StatusCode> { + let Some(expected) = self.owner_account_scope.as_deref() else { return Ok(None); }; + let oauth = self.oauth.as_ref().ok_or(StatusCode::UNAUTHORIZED)?; + let identity = ruview_auth::verify_account_identity(token, &oauth.jwks, &VerifierConfig { + issuer: oauth.issuer.clone(), required_scope: scope::SENSING_READ.into(), + allowed_client_ids: vec![DEFAULT_CLIENT_ID.into()], + }).map_err(|_| StatusCode::UNAUTHORIZED)?; + if !ct_eq(expected.as_bytes(), account_scope_key(&identity).as_bytes()) { return Err(StatusCode::FORBIDDEN); } + Ok(Some(identity.expires_at as u64)) + } + /// Whether the legacy unauthenticated-WebSocket escape hatch is active. pub fn legacy_ws_enabled(&self) -> bool { self.legacy_ws @@ -612,9 +651,11 @@ pub async fn require_bearer( // as "the same mistake this codebase had to fix in jwks.rs". The hot // verification path had not been given the same treatment. let token = supplied.to_string(); + let owner_auth = auth.clone(); let oauth = Arc::clone(oauth); let required_owned = required.to_string(); let verified = tokio::task::spawn_blocking(move || { + owner_auth.verify_account_ownership(&token).map_err(|_| ruview_auth::VerifyError::InvalidOwnership)?; let config = VerifierConfig { issuer: oauth.issuer.clone(), required_scope: required_owned, @@ -670,6 +711,9 @@ pub async fn require_bearer( .and_then(|v| v.to_str().ok()) { if let Some(session) = crate::browser_session::from_cookie_header(cookie_header) { + if !session_matches_owner(&auth, &session) { + return unauthorized(&auth); + } let required = if is_ws_path(&path) { scope::SENSING_READ } else { @@ -718,6 +762,25 @@ pub async fn require_bearer( unauthorized(&auth) } +fn session_matches_owner(auth: &AuthState, session: &crate::browser_session::BrowserSession) -> bool { + auth.owner_account_scope().is_none() || auth.owner_account_scope() == session.owner_account_scope.as_deref() +} + +/// The public browser status must agree with the protected cookie boundary. +/// A legacy cookie may still verify against a migrated signing key without +/// belonging to this account-owned helper. +pub fn browser_status(auth: &AuthState, cookie: Option<&str>) -> serde_json::Value { + let session = cookie.and_then(crate::browser_session::from_cookie_header) + .filter(|session| session_matches_owner(auth, session)); + serde_json::json!({ + "auth_required": auth.is_enabled(), "oauth_enabled": auth.oauth_enabled(), + "browser_signin": auth.oauth_enabled() && crate::browser_session::is_configured(), + "signed_in": session.is_some(), + "account": session.as_ref().map(|value| value.account_id.clone()), + "scope": session.as_ref().map(|value| value.scope.clone()), + }) +} + /// The no-bearer path: try a browser session cookie, else 401. async fn session_or_unauthorized(auth: &AuthState, request: Request, next: Next) -> Response { let path = request.uri().path().to_string(); @@ -728,6 +791,7 @@ async fn session_or_unauthorized(auth: &AuthState, request: Request, next: Next) .and_then(|v| v.to_str().ok()) { if let Some(session) = crate::browser_session::from_cookie_header(h) { + if !session_matches_owner(auth, &session) { return unauthorized(auth); } let required = if is_ws_path(&path) { scope::SENSING_READ } else { @@ -1282,6 +1346,25 @@ mod oauth_tests { ) } + #[tokio::test] + async fn owned_helper_cookie_path_rejects_legacy_and_other_owner_but_accepts_its_owner() { + let owner = "a".repeat(64); + for candidate in [None, Some("b".repeat(64)), Some(owner.clone())] { + let mut auth = oauth_only(); auth.owner_account_scope = Some(owner.clone()); + let mut claims = serde_json::json!({"subject":"synthetic-user","account_id":"synthetic-billing", + "scope":scope::SENSING_READ,"exp":chrono::Utc::now().timestamp()+300}); + if let Some(key) = &candidate { claims["owner_account_scope"] = serde_json::json!(key); } + let raw = crate::browser_session::test_sign_for_tests(&serde_json::to_vec(&claims).unwrap()); + let cookie = format!("ruview_session={raw}"); + let status = browser_status(&auth, Some(&cookie)); + let actual = call_with_session(auth, "GET", "/api/v1/models", &cookie).await; + let expected = if candidate.as_deref() == Some(owner.as_str()) { StatusCode::OK } else { StatusCode::UNAUTHORIZED }; + assert_eq!(actual, expected, "cookie ownership mismatch on normal no-bearer request"); + assert_eq!(status["signed_in"], expected == StatusCode::OK, "public status disagrees with cookie authority"); + assert_eq!(status["account"].is_null(), expected != StatusCode::OK); + } + } + // ── browser session cookie as a credential ──────────────────────── // // The session cookie authorizes /api/v1/* and WebSocket upgrades, and had @@ -1534,6 +1617,7 @@ mod oauth_tests { fn oauth_only() -> AuthState { AuthState { + owner_account_scope: None, token: None, oauth: Some(oauth_state()), tickets: crate::ws_ticket::TicketStore::new(), @@ -1671,6 +1755,7 @@ mod oauth_tests { fn both() -> AuthState { AuthState { + owner_account_scope: None, token: Some(Arc::new("legacy-secret".to_string())), oauth: Some(oauth_state()), tickets: crate::ws_ticket::TicketStore::new(), @@ -1799,6 +1884,7 @@ mod ws_gate_tests { fn static_auth() -> AuthState { AuthState { + owner_account_scope: None, token: Some(Arc::new("secret".into())), oauth: None, tickets: crate::ws_ticket::TicketStore::new(), diff --git a/v2/crates/wifi-densepose-sensing-server/src/browser_session.rs b/v2/crates/wifi-densepose-sensing-server/src/browser_session.rs index 4a0080413b..e2531c7b18 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/browser_session.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/browser_session.rs @@ -163,6 +163,8 @@ struct Transaction { /// server already verified one — nothing replayable elsewhere. #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub struct BrowserSession { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub owner_account_scope: Option, pub subject: String, pub account_id: String, pub scope: String, @@ -476,14 +478,19 @@ pub fn verifier_for_callback(cookie_header: &str, state: &str) -> Result Result { + issue_for_account(principal, secure, None) +} + +pub fn issue_for_account(principal: &ruview_auth::Principal, secure: bool, owner_account_scope: Option<&str>) -> Result { let secret = secret()?; let session = BrowserSession { + owner_account_scope: owner_account_scope.map(str::to_owned), subject: principal.subject.clone(), account_id: principal.account_id.clone(), scope: principal.scopes().collect::>().join(" "), // Never outlive our own ceiling, and never inherit the access token's // 15 minutes either — this is a browser session, not the token. - exp: now() + SESSION_TTL_SECS, + exp: if owner_account_scope.is_some() { (now() + SESSION_TTL_SECS).min(principal.expires_at) } else { now() + SESSION_TTL_SECS }, // Stamped at issue, which is the moment a Cognitum token was actually // verified. Never refreshed by activity: it answers "when did you last // prove who you are", not "when were you last here". @@ -550,6 +557,7 @@ pub(crate) fn test_cookie_value_aged( init_secret_for_tests(); let secret = secret().expect("secret installed above"); let session = BrowserSession { + owner_account_scope: None, subject: subject.to_string(), account_id: account_id.to_string(), scope: scope.to_string(), @@ -720,6 +728,7 @@ mod tests { // so only the MAC stands between it and a valid session. let forged = sign( &serde_json::to_vec(&BrowserSession { + owner_account_scope: None, subject: "attacker".into(), account_id: "acct-attacker".into(), scope: "sensing:admin".into(), @@ -869,6 +878,7 @@ mod tests { fn session(exp: i64) -> BrowserSession { BrowserSession { + owner_account_scope: None, subject: "user-1".into(), account_id: "acct-1".into(), scope: "sensing:read".into(), diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion.rs b/v2/crates/wifi-densepose-sensing-server/src/companion.rs index 0ae02c75bf..a43801a777 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/companion.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/companion.rs @@ -1,9 +1,9 @@ //! Owner-approved, QR-pinned companion pairing; durable secrets are hashed. use axum::{ - extract::{DefaultBodyLimit, Path as RoutePath, State}, + extract::{DefaultBodyLimit, Path as RoutePath, Request, State}, http::StatusCode, routing::{delete, get, post}, - Json, Router, + middleware::Next, response::{IntoResponse, Response}, Json, Router, }; use base64::{engine::general_purpose::URL_SAFE_NO_PAD, Engine}; use rand::{rngs::OsRng, RngCore}; @@ -21,6 +21,26 @@ use std::{ time::{SystemTime, UNIX_EPOCH}, }; use tokio::sync::watch; +use crate::bearer_auth::AuthState; + +#[derive(Clone, Copy)] +pub struct AccountOwnershipExpiry(pub u64); + +pub async fn require_account_ownership(State(auth): State, mut request: Request, next: Next) -> Response { + if auth.owner_account_scope().is_none() { return next.run(request).await; } + let Some(token) = request.headers().get("x-ruview-account-authorization").and_then(|value| value.to_str().ok()) + .filter(|value| value.len() <= 16_384).map(str::to_owned) else { return StatusCode::UNAUTHORIZED.into_response(); }; + // A cold/rotated JWKS cache may fetch. Never block a Tokio worker on it. + let verified = tokio::task::spawn_blocking(move || auth.verify_account_ownership(&token)).await; + match verified { + Ok(Ok(Some(expiry))) => { request.extensions_mut().insert(AccountOwnershipExpiry(expiry)); }, + Ok(Err(status)) => return status.into_response(), + _ => return StatusCode::UNAUTHORIZED.into_response(), + } + next.run(request).await +} + + pub fn secret() -> String { let mut b = [0; 32]; OsRng.fill_bytes(&mut b); @@ -124,6 +144,7 @@ pub struct Companion { pub installation_id: String, pub cert_sha256: String, pub origin: String, + owner_account_scope: Option, path: PathBuf, fault: AtomicBool, inner: Mutex, @@ -137,6 +158,14 @@ pub struct Grant { pub revoked: watch::Receiver, } impl Companion { + pub fn load_for_account(path: &Path, installation_id: String, cert_sha256: String, origin: String, owner_account_scope: Option) -> io::Result { + if owner_account_scope.as_ref().is_some_and(|key| key.len() != 64 || !key.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))) { + return Err(io::Error::other("invalid owner account namespace")); + } + let mut service = Self::load(path, installation_id, cert_sha256, origin)?; + Arc::get_mut(&mut service).ok_or_else(|| io::Error::other("shared companion initialization"))?.owner_account_scope = owner_account_scope; + Ok(service) + } pub fn load( dir: &Path, installation_id: String, @@ -174,6 +203,7 @@ impl Companion { installation_id, cert_sha256, origin, + owner_account_scope: None, path, fault: AtomicBool::new(false), inner: Mutex::new(Inner { @@ -329,7 +359,7 @@ async fn invite(State(s): State) -> ApiResult { }, ); Ok(Json( - json!({"schema":"ruview.companion.invite.v1","installation_id":s.installation_id,"cert_sha256":s.cert_sha256,"origin":s.origin,"invite_id":id,"invite_secret":token,"expires_at":expires}), + json!({"schema":"ruview.companion.invite.v1","installation_id":s.installation_id,"cert_sha256":s.cert_sha256,"origin":s.origin,"invite_id":id,"invite_secret":token,"expires_at":expires,"owner_account_scope":s.owner_account_scope}), )) } async fn enroll(State(s): State, Json(r): Json) -> ApiResult { diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs index 949b6eeda3..d2ad378a9f 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs @@ -69,11 +69,12 @@ impl Integration { wifi_densepose_sensing_server::discovery::discovery_hostname(&identity.installation_id); let authority = companion_authority(&identity, port); let origin = format!("https://{authority}"); - let service = Companion::load( + let service = Companion::load_for_account( &dir, identity.installation_id.clone(), identity.cert_sha256, origin, + auth.owner_account_scope().map(str::to_owned), )?; state.write().await.installation_id = Some(identity.installation_id.clone()); let ip: std::net::IpAddr = args.bind_addr.parse().map_err(std::io::Error::other)?; @@ -141,6 +142,7 @@ impl Integration { require_companion, )) .merge(companion::public_router(self.service)) + .layer(axum::middleware::from_fn_with_state(self.auth, companion::require_account_ownership)) .layer(axum::extract::DefaultBodyLimit::max(64 * 1024)); tokio::spawn(async move { let _advertiser = self.advertiser; @@ -276,10 +278,13 @@ async fn require_companion( .get("x-ruview-installation-id") .and_then(|v| v.to_str().ok()) .unwrap_or(""); - let grant = match service.authorize(installation, bearer(&request).unwrap_or(""), calibration) { + let mut grant = match service.authorize(installation, bearer(&request).unwrap_or(""), calibration) { Ok(grant) => grant, Err(status) => return status.into_response(), }; + if let Some(companion::AccountOwnershipExpiry(expiry)) = request.extensions().get::() { + grant.expires_at = grant.expires_at.min(*expiry); + } request.extensions_mut().insert(grant); next.run(request).await } diff --git a/v2/crates/wifi-densepose-sensing-server/src/lib.rs b/v2/crates/wifi-densepose-sensing-server/src/lib.rs index ee8583d95e..8ee14aa2a0 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/lib.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/lib.rs @@ -73,6 +73,7 @@ pub mod vendor_rf; pub use wifi_densepose_aether::{embedding, graph_transformer, sona, sparse_inference}; pub mod companion; +pub mod account_profile; pub mod companion_tls; pub mod companion_sensors; diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index f478ab367f..cb2ab0d5a5 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -8193,9 +8193,9 @@ async fn activate_lora_profile(Json(body): Json) -> Json/models` (issue #2088). pub(crate) fn models_dir(data_dir: &std::path::Path) -> PathBuf { - std::env::var_os("MODELS_DIR") - .map(PathBuf::from) - .unwrap_or_else(|| data_dir.join("models")) + wifi_densepose_sensing_server::account_profile::models_directory( + std::env::var_os(wifi_densepose_sensing_server::bearer_auth::OWNER_ACCOUNT_SCOPE_ENV).is_some(), + data_dir, std::env::var_os("MODELS_DIR").as_deref()) } /// Return the directory CSI recordings are written to and read from: @@ -14040,6 +14040,18 @@ async fn main() { let _telemetry = telemetry::init(); let mut args = Args::parse(); + if let Ok(scope) = std::env::var(wifi_densepose_sensing_server::bearer_auth::OWNER_ACCOUNT_SCOPE_ENV) { + let directory = match wifi_densepose_sensing_server::account_profile::prepare_directory(&scope, &args.data_dir) { + Ok(directory) => directory, + Err(error) => { eprintln!("Owned helper startup refused: {error}"); std::process::exit(2); }, + }; + // Child-process cwd only: relative model, recording and cache paths + // must not reopen a different account's directory or the app's cwd. + if let Err(error) = std::env::set_current_dir(&directory) { + eprintln!("Owned helper working directory could not be selected: {error}"); std::process::exit(2); + } + args.data_dir = directory; + } args.ui_path = coalesce_ui_path(args.ui_path); if let Err(error) = validate_startup_calibration_request(args.calibrate) { @@ -15218,8 +15230,8 @@ async fn main() { Ok(s) => s, Err(e) => { error!( - "API auth: OAuth was requested but cannot be initialised: {e}. \ - Refusing to start — unset RUVIEW_OAUTH_ISSUER to run without it." + "API auth could not be initialised: {e}. Correct the configuration before restarting; \ + an owned helper profile must remain account-bound." ); std::process::exit(1); } @@ -16684,7 +16696,7 @@ async fn oauth_callback( Err(e) => return bad(axum::http::StatusCode::UNAUTHORIZED, format!("{e}\n")), }; - let session_cookie = match bs::issue(&principal, secure) { + let session_cookie = match bs::issue_for_account(&principal, secure, auth.owner_account_scope()) { Ok(c) => c, Err(e) => return bad(axum::http::StatusCode::SERVICE_UNAVAILABLE, format!("{e}\n")), }; @@ -16742,19 +16754,10 @@ async fn oauth_status( axum::Extension(auth): axum::Extension, headers: axum::http::HeaderMap, ) -> axum::Json { - use wifi_densepose_sensing_server::browser_session as bs; let raw = headers .get(axum::http::header::COOKIE) .and_then(|v| v.to_str().ok()); - let session = raw.and_then(bs::from_cookie_header); - axum::Json(serde_json::json!({ - "auth_required": auth.is_enabled(), - "oauth_enabled": auth.oauth_enabled(), - "browser_signin": auth.oauth_enabled() && bs::is_configured(), - "signed_in": session.is_some(), - "account": session.as_ref().map(|s| s.account_id.clone()), - "scope": session.as_ref().map(|s| s.scope.clone()), - })) + axum::Json(wifi_densepose_sensing_server::bearer_auth::browser_status(&auth, raw)) } #[cfg(test)] mod adr186_http_tests { diff --git a/v2/crates/wifi-densepose-sensing-server/src/model_manager.rs b/v2/crates/wifi-densepose-sensing-server/src/model_manager.rs index 6cbeebef9f..b982748611 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/model_manager.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/model_manager.rs @@ -39,6 +39,9 @@ pub const MODELS_DIR_DEFAULT: &str = "data/models"; /// Return the effective models directory, respecting `MODELS_DIR` env var. pub fn models_dir() -> PathBuf { + if std::env::var_os(crate::bearer_auth::OWNER_ACCOUNT_SCOPE_ENV).is_some() { + return std::env::current_dir().unwrap_or_else(|_| PathBuf::from(".")).join("models"); + } PathBuf::from( std::env::var("MODELS_DIR").unwrap_or_else(|_| MODELS_DIR_DEFAULT.to_string()), ) From 827e2f0a25bfd14aa32de719ef048341774b7270 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Fri, 23 Oct 2026 20:39:15 +0200 Subject: [PATCH 015/100] test(companion): exercise account ownership through real HTTP middleware --- .../src/bearer_auth.rs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs b/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs index 33e27bcda1..f1b1ba7685 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs @@ -1346,6 +1346,34 @@ mod oauth_tests { ) } + #[tokio::test] + async fn companion_owner_http_guard_matches_the_native_namespace_without_a_billing_id() { + let mut auth = oauth_only(); + Arc::get_mut(auth.oauth.as_mut().unwrap()).unwrap().issuer = "https://auth.cognitum.one".into(); + auth.token = Some(Arc::new("synthetic-local-owner-capability".into())); + // Same fixed user/tenant/workspace vector as the JS and Swift checks. + auth.owner_account_scope = Some("40b085eb7fdbb89941aa63cd909813c2b9a718e1b5b2b052f17a600d7bb372b4".into()); + let now = chrono::Utc::now().timestamp(); + let make_token = |subject: &str, workspace: &str| { + let claims = serde_json::json!({"typ":"access", "iss":"https://auth.cognitum.one", "aud":"ruview", "client_id":"ruview", + "sub":subject, "org_id":"20000000-0000-4000-8000-000000000001", "workspace_id":workspace, + "scope":"sensing:read spaces:read", "iat":now, "exp":now+300, "setup":false, "workload":false}); + let mut header = Header::new(jsonwebtoken::Algorithm::ES256); header.kid = Some(KID.into()); + encode(&header, &claims, &EncodingKey::from_ec_pem(key().pem.as_bytes()).unwrap()).unwrap() + }; + let a = make_token("10000000-0000-4000-8000-000000000001", "30000000-0000-4000-8000-000000000001"); + let b = make_token("10000000-0000-4000-8000-000000000002", "30000000-0000-4000-8000-000000000001"); + let workspace = make_token("10000000-0000-4000-8000-000000000001", "30000000-0000-4000-8000-000000000002"); + for (credential, expected) in [(None, StatusCode::UNAUTHORIZED), (Some(b), StatusCode::FORBIDDEN), + (Some(workspace), StatusCode::FORBIDDEN), (Some(a.clone()), StatusCode::OK), (Some(format!("{a}x")), StatusCode::UNAUTHORIZED)] { + let router = Router::new().route("/companion/v1/enroll", post(|| async { "accepted" })) + .layer(axum::middleware::from_fn_with_state(auth.clone(), crate::companion::require_account_ownership)); + let mut request = Request::builder().method("POST").uri("/companion/v1/enroll"); + if let Some(value) = credential { request = request.header("X-RuView-Account-Authorization", value); } + assert_eq!(router.oneshot(request.body(Body::empty()).unwrap()).await.unwrap().status(), expected); + } + } + #[tokio::test] async fn owned_helper_cookie_path_rejects_legacy_and_other_owner_but_accepts_its_owner() { let owner = "a".repeat(64); From 415de146b313f161d0102e48921ed58f608916d5 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Sun, 25 Oct 2026 13:32:22 +0100 Subject: [PATCH 016/100] fix(deps): refresh compatible companion runtime security patches --- v2/Cargo.lock | 349 ++++++++++++++++++++++++++++++++------------------ 1 file changed, 224 insertions(+), 125 deletions(-) diff --git a/v2/Cargo.lock b/v2/Cargo.lock index 821e45e2c3..27ff69715e 100644 --- a/v2/Cargo.lock +++ b/v2/Cargo.lock @@ -1544,7 +1544,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" dependencies = [ "lazy_static", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -1553,7 +1553,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -1791,36 +1791,36 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4353b283b6c76205abc8eab20b41838e47510c9669ed83c27f7bca8cd1c262f" +checksum = "b3741b9ad77e89c710586b2fcdd619baf28482d92b70d88cecb2a081dd912ae6" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61f6d6eb342f6c48d97278c7549c5b91756f03db80bce9cdeddaa536c356c937" +checksum = "8243e149085051b23faebd352ef7031aeebc5f45e63aaefd63ee7f810c22c7df" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c746ef625773869efaffda6918a2b73f973c73e419e1de20833f3a1a9d614006" +checksum = "c538af3aee13eb6aa8a85d2ef85b4d0aa41623dae8b19f59bbacfd5d6c114920" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-bitset" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5447cd11203bcb5b71bb91e1e8b8470f350870ca837847bf82ec2e25f55fcd7b" +checksum = "2292ea3964d7d751031da1a1261d64feb09e89bd4fbebf3c78f43f9c34c20a11" dependencies = [ "serde", "serde_derive", @@ -1828,9 +1828,9 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ac26ac02abded3bb8223f0cf8a0055c6284be55ca9a6c380f78494bdf686adc" +checksum = "8967a699a4982d02c32246a00f2aa4cee5bccfaaa042608d5e151cb99cba210e" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -1855,9 +1855,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c4a9197c03f7b75c228a66ecaf0b7469e0f9b86fe3cf2cf7ebfbcaacfce0a9b" +checksum = "e258eeeb6903b4e91e314a3667ab2a1e2924738ef2df08bdb50355e38decc749" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -1868,24 +1868,24 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3279a717ec8773811c148650b77cb7c554c6898c858cb547d3da477a8bca21b3" +checksum = "e1d36ff0de30460c01dad152ca2097bcabef2f16c659b9ffbca2f67e9bbdb321" [[package]] name = "cranelift-control" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e2a7ad0219d6f4a4899e5e93d5a4944a79dd6cffe2f6d6fba4d6b2ac165a154" +checksum = "acc880d91877efb6199031bda872726ca3287dc1e18ccf3f61c07bfdab62930b" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "554aaa52367f0396570d031aa0bdedfff4f63794022dc418d0cdcf8f6d321b60" +checksum = "5f7af321c4b6e5be4729e4a5b1ed0fc78fc31695c1ecd3c9b45b91908bf3d0a3" dependencies = [ "cranelift-bitset", "serde", @@ -1894,9 +1894,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72db97f8a7ea1626ecfcac1adf664cba1f1d68448abbb36b7dbaa0a6b78bc11f" +checksum = "3463f1b26faac3491008cede8e9cdf7ceb83139143bd92ba86c14b0882aaf1bb" dependencies = [ "cranelift-codegen", "log", @@ -1906,15 +1906,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17a237c16305f95ff07031be4a5226a4c0753761f40bbad2105c758a33aee692" +checksum = "90b76554b329634c4241c5670a96058affca99ac3b33daa14fb221bd68c0038e" [[package]] name = "cranelift-native" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd6f62b06973b3e5fd813167b01958b09f634bfa59a8f5db2cd49b930d93aa6f" +checksum = "36e7c67bcfd30a8160187a6d52b2ec0cbe45698a9d61f12d1292bd24701f7ed2" dependencies = [ "cranelift-codegen", "libc", @@ -1923,9 +1923,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.123.16" +version = "0.123.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd76efe2edc25d9f062aaf0a16c7528a30c8ba5782afe6296237ed48d8d6127c" +checksum = "1ccca83c45ddfe0412b1b2e4fe509d28d1bdbc014e306574a848519741bebc36" [[package]] name = "crc" @@ -2069,9 +2069,9 @@ dependencies = [ [[package]] name = "crypto-bigint" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a52aa3fcda4e6302a9f48734f234d35d4721b96f8fe07d073f07ce9df4f0271" +checksum = "6bb08299a6046833a879ace115697f12fd2bd6c8d3e93c909ffc9e0738e7cae0" dependencies = [ "cpubits", "ctutils", @@ -2163,9 +2163,9 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.2" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +checksum = "38d28fb6aa7dd680b3ffd1a3e64e115c177bdba63c0ae7f9cd48129ff05c4fa0" dependencies = [ "cmov", ] @@ -3333,7 +3333,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3364,13 +3364,13 @@ checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" [[package]] name = "fastbloom" -version = "0.14.1" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7f34442dbe69c60fe8eaf58a8cafff81a1f278816d8ab4db255b3bef4ac3c4" +checksum = "ef975e30683b2d965054bb0a836f8973857c4ebf6acf274fe46617cd285060d8" dependencies = [ - "getrandom 0.3.4", + "foldhash 0.2.0", "libm", - "rand 0.9.2", + "portable-atomic", "siphasher 1.0.2", ] @@ -4304,11 +4304,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", - "js-sys", "libc", "r-efi", "wasip2", - "wasm-bindgen", ] [[package]] @@ -4318,11 +4316,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "139ef39800118c7683f2fd3c98c1b23c09ae076556b435f8e9064ae108aaeeec" dependencies = [ "cfg-if", + "js-sys", "libc", "r-efi", "rand_core 0.10.1", "wasip2", "wasip3", + "wasm-bindgen", ] [[package]] @@ -5231,7 +5231,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.7", + "webpki-roots 1.0.10", ] [[package]] @@ -5549,7 +5549,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5677,18 +5677,70 @@ dependencies = [ "cesu8", "cfg-if", "combine", - "jni-sys", + "jni-sys 0.3.1", "log", "thiserror 1.0.69", "walkdir", "windows-sys 0.45.0", ] +[[package]] +name = "jni" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498" +dependencies = [ + "cfg-if", + "combine", + "jni-macros", + "jni-sys 0.4.1", + "log", + "simd_cesu8", + "thiserror 2.0.18", + "walkdir", + "windows-link 0.2.1", +] + +[[package]] +name = "jni-macros" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3" +dependencies = [ + "proc-macro2", + "quote", + "rustc_version", + "simd_cesu8", + "syn 2.0.117", +] + [[package]] name = "jni-sys" -version = "0.3.0" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41a652e1f9b6e0275df1f15b32661cf0d4b78d4d87ddec5e0c3c20f097433258" +dependencies = [ + "jni-sys 0.4.1", +] + +[[package]] +name = "jni-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6377a88cb3910bee9b0fa88d4f42e1d2da8e79915598f65fb0c7ee14c878af2" +dependencies = [ + "jni-sys-macros", +] + +[[package]] +name = "jni-sys-macros" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" +checksum = "38c0b942f458fe50cdac086d2f946512305e5631e720728f2a61aabcd47a6264" +dependencies = [ + "quote", + "syn 2.0.117", +] [[package]] name = "jobserver" @@ -5901,14 +5953,14 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.14" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" dependencies = [ "bitflags 2.11.0", "libc", "plain", - "redox_syscall 0.7.4", + "redox_syscall 0.9.4", ] [[package]] @@ -6122,9 +6174,9 @@ checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" [[package]] name = "memfd" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad38eb12aea514a0466ea40a80fd8cc83637065948eb4a426e4aa46261175227" +checksum = "57804b2c9b69967f1536a56f86297e367a33b19e98852ed624b84551cdbc0d90" dependencies = [ "rustix", ] @@ -6178,7 +6230,7 @@ dependencies = [ "js-sys", "quinn", "rcgen", - "rustls-platform-verifier", + "rustls-platform-verifier 0.6.2", "serde", "thiserror 2.0.18", "tokio", @@ -6609,7 +6661,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3f42e7bbe13d351b6bead8286a43aac9534b82bd3cc43e47037f012ebfd62d4" dependencies = [ "bitflags 2.11.0", - "jni-sys", + "jni-sys 0.3.1", "log", "ndk-sys", "num_enum", @@ -6629,7 +6681,7 @@ version = "0.6.0+11769913" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee6cda3051665f1fb8d9e08fc35c96d5a244fb1be711a03b71118828afc9a873" dependencies = [ - "jni-sys", + "jni-sys 0.3.1", ] [[package]] @@ -7272,7 +7324,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.45.0", ] [[package]] @@ -8051,7 +8103,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.117", @@ -8088,9 +8140,9 @@ dependencies = [ [[package]] name = "pulley-interpreter" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f09f3fdea932cc327d051b2aaccfc82fddcd49997e8579f3d91ae3468a954b94" +checksum = "c956c7140270b72e3bf03381a51f2eb79c70fd6881fc17cd027fa7cd3f56ae5a" dependencies = [ "cranelift-bitset", "log", @@ -8100,9 +8152,9 @@ dependencies = [ [[package]] name = "pulley-macros" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96da696c8995a3033eb1287ee418895592da41a5d549b8615b5f1e5851318b48" +checksum = "74db420d7ce2bb03eed5b81c1f778bda5771b3fd818e2b496dfc3a2783ceb60e" dependencies = [ "proc-macro2", "quote", @@ -8208,20 +8260,21 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.15" +version = "0.11.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fcb935c5bec503c2f0e306bdd3e58bb9029dcb14fa8d9ac76e3a5256ac0763e" +checksum = "a9746dbde176634f4f2f1faf2404e30a31b2bc1e9cafb5329c95d8177a18c9fc" dependencies = [ "bytes", "fastbloom", - "getrandom 0.3.4", + "getrandom 0.4.1", "lru-slab", - "rand 0.9.2", + "rand 0.10.2", + "rand_pcg 0.10.2", "ring", "rustc-hash 2.1.1", "rustls", "rustls-pki-types", - "rustls-platform-verifier", + "rustls-platform-verifier 0.7.1", "slab", "thiserror 2.0.18", "tinyvec", @@ -8240,7 +8293,7 @@ dependencies = [ "once_cell", "socket2 0.6.2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -8278,7 +8331,7 @@ dependencies = [ "rand_chacha 0.2.2", "rand_core 0.5.1", "rand_hc", - "rand_pcg", + "rand_pcg 0.2.1", ] [[package]] @@ -8424,6 +8477,15 @@ dependencies = [ "rand_core 0.5.1", ] +[[package]] +name = "rand_pcg" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caa0f4137e1c0a72f4c651489402276c8e8e1cf081f3b0ba156d2cbeef09e86a" +dependencies = [ + "rand_core 0.10.1", +] + [[package]] name = "rand_xorshift" version = "0.4.0" @@ -8560,9 +8622,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.7.4" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f450ad9c3b1da563fb6948a8e0fb0fb9269711c9c73d9ea1de5058c79c8d643a" +checksum = "737970939a87c6fa31e7acad13307bccbb017a073b695b6089a2c484f929e20e" dependencies = [ "bitflags 2.11.0", ] @@ -8717,7 +8779,7 @@ dependencies = [ "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "webpki-roots 1.0.7", + "webpki-roots 1.0.10", ] [[package]] @@ -9104,14 +9166,14 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "log", "once_cell", @@ -9152,17 +9214,38 @@ checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" dependencies = [ "core-foundation 0.10.1", "core-foundation-sys", - "jni", + "jni 0.21.1", "log", "once_cell", "rustls", "rustls-native-certs", - "rustls-platform-verifier-android", + "rustls-platform-verifier-android 0.1.1", "rustls-webpki", "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni 0.22.4", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android 0.2.0", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.52.0", ] [[package]] @@ -9171,11 +9254,17 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" +[[package]] +name = "rustls-platform-verifier-android" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" + [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "ring", "rustls-pki-types", @@ -9736,9 +9825,9 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.28" +version = "0.1.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" dependencies = [ "windows-sys 0.61.2", ] @@ -10247,6 +10336,16 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" +[[package]] +name = "simd_cesu8" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11031e251abf8611c80f460e19dbdeb54a66db918e49c65a7065b46ac7aec520" +dependencies = [ + "rustc_version", + "simdutf8", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -10502,7 +10601,7 @@ version = "0.7.0-rc.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b5f622c2d21b826b3501f4c620ccc4696e4a09a6b51727fcb21036dec87c101" dependencies = [ - "crypto-bigint 0.7.5", + "crypto-bigint 0.7.6", "crypto-common 0.2.2", "digest 0.11.3", "subtle", @@ -10768,7 +10867,7 @@ dependencies = [ "gdkwayland-sys", "gdkx11-sys", "gtk", - "jni", + "jni 0.21.1", "libc", "log", "ndk", @@ -10840,7 +10939,7 @@ dependencies = [ "gtk", "heck 0.5.0", "http", - "jni", + "jni 0.21.1", "libc", "log", "mime", @@ -11027,7 +11126,7 @@ dependencies = [ "dpi", "gtk", "http", - "jni", + "jni 0.21.1", "objc2", "objc2-ui-kit", "objc2-web-kit", @@ -11050,7 +11149,7 @@ checksum = "e11ea2e6f801d275fdd890d6c9603736012742a1c33b96d0db788c9cdebf7f9e" dependencies = [ "gtk", "http", - "jni", + "jni 0.21.1", "log", "objc2", "objc2-app-kit", @@ -11144,7 +11243,7 @@ dependencies = [ "getrandom 0.4.1", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -12510,9 +12609,9 @@ dependencies = [ [[package]] name = "wasmtime" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc1374519a6d4c3eb99e0598f2ce069e6feb63de65bed7d0240a5f84b58c3b" +checksum = "60e5f244a2c9d40afa9e9a607ee7f048c0648bb1c22cc28b130c5fc71e23d71d" dependencies = [ "addr2line", "anyhow", @@ -12565,9 +12664,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7e05a9ebefb914d4df7928652e27aa1e89e1ca23c5ab73c647316783f3bcbc2" +checksum = "3e246cd5107aa406cfa4acf6942e864bd189ae8136bab853365147da131c9ca3" dependencies = [ "anyhow", "cpp_demangle", @@ -12592,18 +12691,18 @@ dependencies = [ [[package]] name = "wasmtime-internal-asm-macros" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fcf4b431fb653aefd2cc3c3a65ea78988acf5a7626cb825c54b5aab5aa1a8e" +checksum = "5380a8106fea8bfedb7208bb11ced9e7488ee44394ca889ede73acd3bdfd9d0b" dependencies = [ "cfg-if", ] [[package]] name = "wasmtime-internal-cache" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65b514e39cd914f34a135ce7696373287ccb9611b083cbe244a48c4903380fb5" +checksum = "4108e1964b3a02b4b5604ea1a2918a0cbcab512e5de0ccf4c062fc6c06b2eb8e" dependencies = [ "anyhow", "base64 0.22.1", @@ -12621,9 +12720,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-macro" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92fe9cf6ca766adc68b5ed4474155e88cc97e79d13c63450fe91779de9c41a3" +checksum = "8278aa2bb237788ac9e3a70c5bea3ea3de6b4f95fb003df303858941871b8c4e" dependencies = [ "anyhow", "proc-macro2", @@ -12636,15 +12735,15 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-util" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36b8553647a6ae550ec3bca45759565204af763c37b1b51d5e45d6c5a26a1e32" +checksum = "838c652b6cafd82a432a3a2c7af5bea4c6957d218aafbe50018e0297486ab7cc" [[package]] name = "wasmtime-internal-cranelift" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9bd306f478b4a68d94a80e64d1a92871b697188eafe8bfc387f865085c818a04" +checksum = "7339fdd64a3fde574505adea23244e065d5b1de0e72c8dd87e150f5d4ab63a63" dependencies = [ "anyhow", "cfg-if", @@ -12669,9 +12768,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-fiber" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f9bb63081293450d0dfb439f246339e91e34d51c09b1310bdd297b67be5933" +checksum = "6f3afc72d3c72f71d4ebc72e83f479ba8ca24409c41d43e5e0228f0bf82b873e" dependencies = [ "anyhow", "cc", @@ -12685,9 +12784,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-debug" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c04edbaf79fb10460ee0867a829f831b7afa8d5d8c9d32287d88b3f1aaef79" +checksum = "21d33adf2d1eae36df9b6b033a1d549c7553ea607f4f9a81f75d51627725c354" dependencies = [ "cc", "object", @@ -12697,9 +12796,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e2b3b397afafb7e6b3603dd3f867cc626d3bde726bd65ce01443a70fe66e1c0" +checksum = "5bde2fcf14cc76084242c6fe792e49778b5a9008c7df28ca4fe21c35c69d5f55" dependencies = [ "anyhow", "cfg-if", @@ -12709,24 +12808,24 @@ dependencies = [ [[package]] name = "wasmtime-internal-math" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2509a3a45fbdc79eab62680df8f8d5ffe5420c3bf7f0c2083d49110c09c61b34" +checksum = "4bd982249e933e730859f84c17254c7e95f38c3b0cdf81695bfc6a270e237852" dependencies = [ "libm", ] [[package]] name = "wasmtime-internal-slab" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0985eb32142c197f1aa5d50e94498a769f55095e8ed2838eff386c735f9da72f" +checksum = "8280b359384cc412bddf0023193eeaffadcb0d7fbbf860f2b7e0580e8806a344" [[package]] name = "wasmtime-internal-unwinder" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b88fa7f08080eeb4926d6062c2ae8c38c19a2b1a6d0f01c569b827d987f4909" +checksum = "bb43477c41769fec0d031d32cb3236195acfb1e6aa4efdf463e5fd001c23f6cc" dependencies = [ "anyhow", "cfg-if", @@ -12737,9 +12836,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-versioned-export-macros" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfff26e96f2539205ff7c049e4dff8343a479722bbbae88bc4da54adde12c1fa" +checksum = "b33afac17819f2f767ea280b4b87a6c95458ef1ae43da410cfc5b26813ecb78f" dependencies = [ "proc-macro2", "quote", @@ -12748,9 +12847,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-winch" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "176d188460e507807f9f8b7521d7c1dc62b356fa221b4e2ce566e88652294d9d" +checksum = "1757c8b0626385c93994c7a3c7a23ee5b469061465b601c1fdce71ff61b11c82" dependencies = [ "anyhow", "cranelift-codegen", @@ -12765,9 +12864,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-wit-bindgen" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d3c06e67956fabd2fa3b1aa94200779cc0df8100dc45c6a9b80fda67f565697" +checksum = "061dd5bae2e618b32a8ed2acda5953d07dd308ac0e3f7960afc62ccdf6df7c44" dependencies = [ "anyhow", "bitflags 2.11.0", @@ -12876,9 +12975,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.6" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "804f18a4ac2676ffb4e8b5b5fa9ae38af06df08162314f96a68d2a363e21a8ca" +checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb" dependencies = [ "rustls-pki-types", ] @@ -12889,14 +12988,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.7", + "webpki-roots 1.0.10", ] [[package]] name = "webpki-roots" -version = "1.0.7" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" dependencies = [ "rustls-pki-types", ] @@ -13670,7 +13769,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -13681,9 +13780,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winch-codegen" -version = "36.0.16" +version = "36.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cde193a101057fce046a3fff773932419b7861f6bb95632d01bca90843c9ce51" +checksum = "69940a29fc6b01b0e6b141912b700c17eaf956d58d63bc0c6e733435caf020e4" dependencies = [ "anyhow", "cranelift-assembler-x64", @@ -14456,7 +14555,7 @@ dependencies = [ "html5ever", "http", "javascriptcore-rs", - "jni", + "jni 0.21.1", "kuchikiki", "libc", "ndk", From e42fe809bbf311d5b275ece219104933261d177c Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Tue, 27 Oct 2026 10:37:41 +0100 Subject: [PATCH 017/100] fix(deps): patch companion Tokio I/O-safety constructors Resolve Tokio 1.51.6 with its compatible Mio, socket2 and macro dependencies. Preserve the scoped companion stack and existing audit policy; no upstream vendor or framework adoption. Existing LRU unsoundness and yanked-package warnings remain separately recorded. --- v2/Cargo.lock | 67 ++++++++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 28 deletions(-) diff --git a/v2/Cargo.lock b/v2/Cargo.lock index 27ff69715e..497b9848a1 100644 --- a/v2/Cargo.lock +++ b/v2/Cargo.lock @@ -1544,7 +1544,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" dependencies = [ "lazy_static", - "windows-sys 0.52.0", + "windows-sys 0.48.0", ] [[package]] @@ -1553,7 +1553,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.48.0", ] [[package]] @@ -2900,7 +2900,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.2", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3333,7 +3333,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5280,7 +5280,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.2", + "socket2 0.5.10", "system-configuration", "tokio", "tower-service", @@ -5549,7 +5549,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5911,9 +5911,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.182" +version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" [[package]] name = "libloading" @@ -6320,9 +6320,9 @@ dependencies = [ [[package]] name = "mio" -version = "1.1.1" +version = "1.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69bcab0ad47271a0234d9422b131806bf3968021e5dc9328caf2d4cd58557fc" +checksum = "1788edb87fdc09c7e26304471e2f5be8cdefb1b6930d6e3985fc02ff53bf86ee" dependencies = [ "libc", "log", @@ -6771,7 +6771,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -8251,7 +8251,7 @@ dependencies = [ "quinn-udp", "rustc-hash 2.1.1", "rustls", - "socket2 0.6.2", + "socket2 0.5.10", "thiserror 2.0.18", "tokio", "tracing", @@ -8291,9 +8291,9 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.2", + "socket2 0.5.10", "tracing", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -9166,7 +9166,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -9224,7 +9224,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -9245,7 +9245,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -10421,12 +10421,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.2" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86f4aa3ad99f2088c990dfa82d367e19cb29268ed67c574d10d0a4bfe71f07e0" +checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -10728,6 +10728,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "sync_wrapper" version = "1.0.2" @@ -11243,7 +11254,7 @@ dependencies = [ "getrandom 0.4.1", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -11419,9 +11430,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.49.0" +version = "1.51.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72a2903cd7736441aac9df9d7688bd0ce48edccaadf181c3b90be801e81d3d86" +checksum = "a13e32eacd0222f9658be2ccbd83b718bfd9bfb0d54b5921913c184cb86124a5" dependencies = [ "bytes", "libc", @@ -11429,20 +11440,20 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.6.2", + "socket2 0.6.5", "tokio-macros", "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.6.0" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" +checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 3.0.9", ] [[package]] @@ -13769,7 +13780,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.48.0", ] [[package]] From 77ceba732d7ca63e5e6bf4db300df877325ada1f Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Tue, 27 Oct 2026 15:00:04 +0100 Subject: [PATCH 018/100] fix(vitals): bind publication decisions to source and calibration --- ...DR-355-local-empty-room-bootstrap-prior.md | 39 + .../wifi-densepose-sensing-server/src/main.rs | 773 ++++++++++++++++-- .../src/privacy_filter.rs | 1 + .../src/vital_publication.rs | 141 ++++ .../tests/auth_wiring.rs | 19 +- 5 files changed, 911 insertions(+), 62 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/vital_publication.rs diff --git a/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md b/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md index 641beed67c..db9a579ad5 100644 --- a/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md +++ b/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md @@ -127,6 +127,45 @@ occupied sequences. Empty data is never relabeled as human training. performs the bounded one time held out refinement. It requires administrator authority when transport authentication is enabled. +## Publication and diagnostics clarification, 2026-10-27 + +The REST vital endpoint and both raw-CSI/edge sensing publishers share the same +room aggregate and bound-model publication gates. REST must not substitute the +single-link model's count for the room aggregate: a fresh second node can still +make occupant attribution ambiguous. A cached candidate is bound to its producing +node, packet tick/time and full immutable calibration receipt identity. A REST +read may withdraw a previously published candidate as gates expire or change; +it cannot promote a sample withheld at its original publication decision. A new +qualifying packet is required. Candidate freshness is bounded independently of +other nodes' arrival clocks. + +Raw CSI estimates carry the existing exact `ruview.vital-evidence.v1` envelope +with capability `esp32_csi_spectral_vitals_v1`. `source_node_ids` names only the +radio producing the estimate, not the active-node inventory or occupancy-model +source. The legacy five-field mirror may remain for dashboard compatibility, +but must match the exact envelope. Firmware edge estimates use the distinct +`esp32_firmware_edge_vitals_v1` capability; this does not qualify them for a +consumer that only accepts raw CSI spectral estimates. Calibrated presence tags +the radio whose history the model actually scored, rather than the radio whose +packet happened to trigger publication. Complementary vital/occupancy sensors +are not silently claimed as joint vital inference. + +`GET /api/v1/vital-signs` retains its existing managed owner/account/paired-read +authorization. Its bounded `ruview.vital-publication-diagnostics.v1` object +separates the last packet's publication decision from current reevaluation for +each node. It reports candidate kind/tick/time/age, occupancy source, gate +outcomes, actual per-node detector buffers, reset count, and distinct detector +and observed sample clocks. It contains no raw CSI, suppressed BPM, credentials, +room names or user identity. It is not sensing authority or clinical evidence. +No such diagnostic is added to public health. Privacy mode withholds node/buffer +details and both vital envelopes. Diagnostic snapshots explain only their own +observation time; they do not extend rate/model freshness. + +The single-link calibration algorithm and negative-only restart prior are +unchanged by this clarification. Multiple radios in a room receipt do not imply +separate trained baselines. A per-source model bank and fused presence require a +separately reviewed capture, coverage, persistence and holdout contract. + ## Measured installation evidence One physical empty home run on source Node 5 collected 1,762 accepted frames diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index cb2ab0d5a5..30f593cc0b 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -40,6 +40,7 @@ mod rvf_pipeline; mod tracker_bridge; pub mod types; mod vital_signs; +mod vital_publication; // Training pipeline modules (exposed via lib.rs) use wifi_densepose_sensing_server::{ @@ -510,6 +511,8 @@ struct SensingUpdate { /// Vital sign estimates (breathing rate, heart rate, confidence). #[serde(skip_serializing_if = "Option::is_none")] vital_signs: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + vital_evidence: Option, /// Strict calibrated occupancy evidence for this frame, bound to the /// active model receipt. Omitted when no calibrated result is available. #[serde(default, skip_serializing_if = "Option::is_none")] @@ -1154,6 +1157,9 @@ struct NodeState { rssi_history: VecDeque, vital_detector: VitalSignDetector, latest_vitals: VitalSigns, + vital_candidate: Option, + last_vital_decision: Option, + vital_sample_rate_resets: u64, pub(crate) last_frame_time: Option, edge_vitals: Option, /// ADR-110 §A0.12: Latest sync packet received from this node. When a @@ -1557,6 +1563,9 @@ impl NodeState { rssi_history: VecDeque::new(), vital_detector: VitalSignDetector::new(20.0), latest_vitals: VitalSigns::default(), + vital_candidate: None, + last_vital_decision: None, + vital_sample_rate_resets: 0, last_frame_time: None, edge_vitals: None, latest_sync: None, @@ -2792,7 +2801,7 @@ impl AppStateInner { /// heuristic, so a consumer may treat a present value as calibrated. fn calibrated_presence_evidence( &self, - inference_node_id: u8, + _inference_node_id: u8, source_tick: u64, observed_at_unix_ms: u64, ) -> Option { @@ -2813,7 +2822,7 @@ impl AppStateInner { binding_digest: receipt.binding_digest.clone(), source_node_ids: receipt.source_node_ids.clone(), model_completed_at_unix_ms: receipt.completed_at_unix_ms, - inference_node_id, + inference_node_id: self.bound_source_node_id()?, source_tick, observed_at_unix_ms, inference_method: occupancy.method.wire_name().to_string(), @@ -3263,6 +3272,58 @@ mod calibration_expiry_tests { } } + // Investigation fixture only: no production gate or publication change. + #[test] + fn investigation_vitals_candidate_source_is_not_checked_by_bound_model_gate() { + let mut state = state_with_runtime_window(); + let baseline = state.field_model.as_ref().unwrap().modes().unwrap().baseline[0].clone(); + state.node_states.get_mut(&5).unwrap().field_model_history = + (0..50).map(|_| baseline.iter().map(|value| value + 0.3).collect()).collect(); + assert_eq!(state.bound_source_node_id(), Some(5)); + assert_eq!(state.person_count_at(1_500), 1); + let mut foreign = NodeState::new(); + foreign.latest_vitals = VitalSigns { heart_rate_bpm: Some(101.0), ..qualified_vital_candidates() }; + state.node_states.insert(6, foreign); + let result = calibrated_vitals_for_publication( + &state, &state.node_states[&6].latest_vitals, 1, 1_500, + ).expect("baseline accepts a qualified candidate without checking its node identity"); + assert_eq!(result.heart_rate_bpm, Some(101.0)); + println!("INVESTIGATION: candidate node6 accepted using model bound to node5; publication API carries no candidate-node identity"); + } + + #[tokio::test] + async fn investigation_vitals_rest_preserves_websocket_multi_person_refusal() { + let observed = chrono::Utc::now().timestamp_millis().max(0) as u64; + let mut state = state_with_runtime_window(); + let mut snapshot = finalized_model(observed).export_snapshot().unwrap(); + snapshot.modes.baseline_runtime_window_size = Some(50); + state.field_model = Some(FieldModel::from_snapshot(snapshot, observed * 1_000).unwrap()); + state.calibration_model_receipt.as_mut().unwrap().completed_at_unix_ms = observed; + let baseline = state.field_model.as_ref().unwrap().modes().unwrap().baseline[0].clone(); + let node = state.node_states.get_mut(&5).unwrap(); + node.field_model_history = (0..50).map(|_| baseline.iter().map(|value| value + 0.3).collect()).collect(); + node.last_frame_time = Some(std::time::Instant::now()); + node.prev_person_count = 1; + let mut second = NodeState::new(); + second.last_frame_time = Some(std::time::Instant::now()); + second.prev_person_count = 2; + state.node_states.insert(6, second); + let room_count = update_room_person_count(&mut state, &ClassificationInfo { + motion_level: "present_still".into(), presence: true, confidence: 0.9, + }, std::time::Instant::now(), observed); + assert_eq!(room_count, 2); + assert_eq!(state.person_count_at(observed), 1); + let (ws_rates, _) = record_node_vital_decision(&mut state, 5, vital_publication::NodeVitalCandidate { + kind: vital_publication::CandidateKind::RawCsiSpectral, tick: 77, + observed_at_unix_ms: observed, received_at: std::time::Instant::now(), rates: qualified_vital_candidates(), calibration_identity: None, + }, room_count); + assert!(ws_rates.is_none()); + assert!(node_vital_decision(&state, 5, 1, observed).0.is_some(), "same candidate must qualify with a single room occupant"); + let Json(body) = vital_signs_endpoint(State(Arc::new(RwLock::new(state)))).await; + println!("INVESTIGATION: fused room count2 / bound count1: websocket withheld; REST authority={} heart={}", body["authority"], body["vital_signs"]["heart_rate_bpm"]); + assert_eq!(body["authority"], "abstained", "REST must not bypass the existing multi-person refusal used by WebSocket publication"); + } + fn state_with_runtime_window() -> AppStateInner { let mut state = state_with_receipt(); let model = state.field_model.as_mut().unwrap(); @@ -3272,6 +3333,288 @@ mod calibration_expiry_tests { state } + fn qualified_vital_state(observed: u64) -> AppStateInner { + let mut state = state_with_runtime_window(); + let mut snapshot = finalized_model(observed).export_snapshot().unwrap(); + snapshot.modes.baseline_runtime_window_size = Some(50); + state.field_model = Some(FieldModel::from_snapshot(snapshot, observed * 1_000).unwrap()); + state + .calibration_model_receipt + .as_mut() + .unwrap() + .completed_at_unix_ms = observed; + let baseline = state + .field_model + .as_ref() + .unwrap() + .modes() + .unwrap() + .baseline[0] + .clone(); + let node = state.node_states.get_mut(&5).unwrap(); + node.field_model_history = (0..50) + .map(|_| baseline.iter().map(|value| value + 0.3).collect()) + .collect(); + node.last_frame_time = Some(std::time::Instant::now()); + state.prev_person_count = 1; + state.tick = 77; + state + } + + fn record_qualified_vitals( + state: &mut AppStateInner, + observed: u64, + ) -> (Option, Option) { + record_node_vital_decision( + state, + 5, + vital_publication::NodeVitalCandidate { + kind: vital_publication::CandidateKind::RawCsiSpectral, + tick: 77, + observed_at_unix_ms: observed, + received_at: std::time::Instant::now(), + rates: qualified_vital_candidates(), + calibration_identity: None, + }, + 1, + ) + } + + #[tokio::test] + async fn vital_publication_rest_matches_recorded_single_person_decision() { + let observed = chrono::Utc::now().timestamp_millis().max(0) as u64; + let mut state = qualified_vital_state(observed); + let (ws, proof) = record_qualified_vitals(&mut state, observed); + assert_eq!(proof.unwrap().source_node_ids, vec![5]); + let Json(rest) = vital_signs_endpoint(State(Arc::new(RwLock::new(state)))).await; + assert_eq!( + rest["vital_signs"], + serde_json::to_value(ws.unwrap()).unwrap() + ); + assert_eq!( + rest["diagnostics"]["nodes"][0]["last_publication_decision"]["reason"], + "published" + ); + } + + #[test] + fn vital_publication_diagnostics_identify_gates_without_suppressed_rates() { + use vital_publication::Reason; + let mut state = qualified_vital_state(1_500); + record_qualified_vitals(&mut state, 1_500); + let inspect = |state: &AppStateInner| node_vital_decision(state, 5, 1, 1_500); + let node = state.node_states.get_mut(&5).unwrap(); + node.vital_candidate.as_mut().unwrap().rates.signal_quality = 0.1; + let (rates, diagnostic) = inspect(&state); + assert!(rates.is_none()); + assert_eq!(diagnostic.reason, Reason::LowSignalQuality); + assert!(!serde_json::to_string(&diagnostic) + .unwrap() + .contains("rate_bpm")); + state + .node_states + .get_mut(&5) + .unwrap() + .vital_candidate + .as_mut() + .unwrap() + .rates = VitalSigns { + heart_rate_bpm: Some(70.0), + heartbeat_confidence: 0.1, + signal_quality: 0.9, + ..VitalSigns::default() + }; + assert_eq!(inspect(&state).1.reason, Reason::LowRateConfidence); + state.node_states.get_mut(&5).unwrap().vital_candidate.as_mut().unwrap().rates = VitalSigns { + heart_rate_bpm: Some(f64::NAN), heartbeat_confidence: 0.9, signal_quality: 0.9, ..VitalSigns::default() + }; + assert_eq!(inspect(&state).1.reason, Reason::InvalidCandidateRate); + state + .node_states + .get_mut(&5) + .unwrap() + .vital_candidate + .as_mut() + .unwrap() + .rates = VitalSigns { + signal_quality: 0.9, + ..VitalSigns::default() + }; + assert_eq!(inspect(&state).1.reason, Reason::NoCandidateRate); + state.calibration_sequence_fault_node_ids.insert(5); + assert_eq!(inspect(&state).1.reason, Reason::SequenceFault); + } + + #[tokio::test] + async fn vital_publication_stale_candidate_cannot_be_revived_by_another_nodes_freshness() { + let observed = chrono::Utc::now().timestamp_millis().max(0) as u64; + let mut state = qualified_vital_state(observed); + record_qualified_vitals(&mut state, observed); + state + .node_states + .get_mut(&5) + .unwrap() + .vital_candidate + .as_mut() + .unwrap() + .received_at -= Duration::from_secs(4); + let Json(rest) = vital_signs_endpoint(State(Arc::new(RwLock::new(state)))).await; + assert_eq!(rest["authority"], "abstained"); + assert_eq!(rest["abstention_reason"], "candidate_stale"); + } + + #[test] + fn vital_publication_rest_cannot_promote_a_previously_withheld_packet() { + let mut state = qualified_vital_state(1_500); + let (rates, _) = record_node_vital_decision( + &mut state, + 5, + vital_publication::NodeVitalCandidate { + kind: vital_publication::CandidateKind::RawCsiSpectral, + tick: 77, + observed_at_unix_ms: 1_500, + received_at: std::time::Instant::now(), + rates: qualified_vital_candidates(), + calibration_identity: None, + }, + 2, + ); + assert!(rates.is_none()); + // Synthetic state transition without a new candidate: a read cannot + // grant publication authority to the packet withheld at capture time. + let (rates, decision) = reevaluate_node_vitals(&state, 5, 1, 1_500); + assert!(rates.is_none()); + assert_eq!( + decision.reason, + vital_publication::Reason::NotPublishedForSourceTick + ); + } + + #[test] + fn vital_publication_candidate_is_bound_to_full_calibration_identity() { + let mut state = qualified_vital_state(1_500); + record_qualified_vitals(&mut state, 1_500); + // Keep the same model ID, replacing only the session identity. + state.calibration_model_receipt.as_mut().unwrap().session_id = "other-session".into(); + let (rates, decision) = reevaluate_node_vitals(&state, 5, 1, 1_500); + assert!(rates.is_none()); + assert_eq!( + decision.reason, + vital_publication::Reason::CandidateCalibrationChanged + ); + } + + #[test] + fn vital_publication_rejected_feature_grid_cannot_promote_old_candidate() { + use wifi_densepose_hardware::PpduType; + let observed = chrono::Utc::now().timestamp_millis().max(0) as u64; + let mut state = qualified_vital_state(observed); + state + .node_states + .get_mut(&5) + .unwrap() + .field_model_history + .clear(); + let (rates, _) = record_qualified_vitals(&mut state, observed); + assert!( + rates.is_none(), + "candidate withheld while bound window is incomplete" + ); + let baseline = state + .field_model + .as_ref() + .unwrap() + .modes() + .unwrap() + .baseline[0] + .clone(); + let amplitudes: Vec<_> = (0..64) + .map(|i| { + let x = i as f64 * 55.0 / 63.0; + let lo = x.floor() as usize; + baseline[lo] * (1.0 - x.fract()) + baseline[(lo + 1).min(55)] * x.fract() + 0.3 + }) + .collect(); + for sequence in 100..150 { + // Mirror receive ordering: field model accepts its frozen grid + // before the independent feature grid refuses this same packet. + for _ in 0..5 { + state + .node_states + .get_mut(&5) + .unwrap() + .accept_grid((256, PpduType::HeSu)); + } + assert!(state.maybe_feed_calibration_frame( + 5, + CsiGridKey { + n_subcarriers: 64, + ppdu_type: 0 + }, + sequence, + &litudes, + std::time::Instant::now() + )); + assert!(!state + .node_states + .get_mut(&5) + .unwrap() + .accept_grid((64, PpduType::HtLegacy))); + } + assert!( + node_vital_decision(&state, 5, 1, observed).0.is_some(), + "updated model alone now permits the cached candidate" + ); + let (rates, decision) = reevaluate_node_vitals(&state, 5, 1, observed); + assert!(rates.is_none()); + assert_eq!( + decision.reason, + vital_publication::Reason::NotPublishedForSourceTick + ); + } + + #[test] + fn vital_publication_diagnostics_separate_observed_and_detector_clocks() { + let mut state = qualified_vital_state(1_500); + let node = state.node_states.get_mut(&5).unwrap(); + node.csi_fps_ema = 23.0; + node.csi_fps_samples = 10; + assert!(!node.vital_detector.reconfigure_sample_rate(23.0)); + let (_, diagnostic) = node_vital_decision(&state, 5, 1, 1_500); + assert_eq!(diagnostic.sample_rate_hz, 20.0); + assert_eq!(diagnostic.observed_sample_rate_hz, 23.0); + } + + #[tokio::test] + async fn vital_publication_privacy_hides_candidate_diagnostics_and_rates() { + let observed = chrono::Utc::now().timestamp_millis().max(0) as u64; + let mut state = qualified_vital_state(observed); + record_qualified_vitals(&mut state, observed); + state.privacy_mode = true; + let Json(rest) = vital_signs_endpoint(State(Arc::new(RwLock::new(state)))).await; + assert_eq!(rest["abstention_reason"], "privacy_redacted"); + assert_eq!(rest["diagnostics"]["nodes"], serde_json::json!([])); + assert!(rest["buffer_status"].is_null()); + assert!(rest["vital_signs"]["heart_rate_bpm"].is_null()); + let redacted = privacy_filter::redact_json_str( + &serde_json::json!({ + "vital_evidence": {"heart_rate_bpm": 70, "source_node_ids": [5]}, + "classification": {"presence": true} + }) + .to_string(), + ) + .unwrap(); + assert!(!redacted.contains("vital_evidence")); + } + + #[test] + fn vital_publication_presence_tags_scored_node_not_packet_arrival() { + let state = qualified_vital_state(1_500); + let proof = state.calibrated_presence_evidence(6, 77, 1_500).unwrap(); + assert_eq!(proof.inference_node_id, 5); + assert!(proof.source_node_ids.contains(&proof.inference_node_id)); + } + #[test] fn diagnostics_stay_outside_the_strict_calibration_authority_envelope() { let state = state_with_runtime_window(); @@ -5109,6 +5452,7 @@ async fn wifi_task(state: SharedState, tick_ms: u64) { &sub_variances, ), vital_signs: None, + vital_evidence: None, calibrated_presence_evidence: None, detector_diagnostics: None, enhanced_motion, @@ -5279,6 +5623,7 @@ async fn windows_wifi_fallback_tick(state: &SharedState, seq: u32) { &sub_variances, ), vital_signs: None, + vital_evidence: None, calibrated_presence_evidence: None, detector_diagnostics: None, enhanced_motion: None, @@ -6173,6 +6518,7 @@ mod latest_update_staleness_tests { values: vec![], }, vital_signs: None, + vital_evidence: None, classifier: None, activity: None, calibrated_presence_evidence: None, @@ -8598,26 +8944,50 @@ fn vitals_for_publication( if !explicit_calibration_fresh || person_count != 1 { return None; } + if !candidates.signal_quality.is_finite() { + return None; + } let signal_quality = candidates.signal_quality.clamp(0.0, 1.0); if signal_quality < VITAL_PUBLICATION_MIN_SIGNAL_QUALITY { return None; } - let breathing_rate_bpm = (candidates.breathing_confidence - >= VITAL_PUBLICATION_MIN_CONFIDENCE) - .then_some(candidates.breathing_rate_bpm) - .flatten(); - let heart_rate_bpm = - (candidates.heartbeat_confidence >= VITAL_PUBLICATION_MIN_CONFIDENCE) - .then_some(candidates.heart_rate_bpm) - .flatten(); + let qualified = |rate: Option, confidence: f64, minimum, maximum| { + rate.filter(|rate| { + rate.is_finite() + && *rate >= minimum + && *rate <= maximum + && confidence.is_finite() + && confidence >= VITAL_PUBLICATION_MIN_CONFIDENCE + }) + }; + let breathing_rate_bpm = qualified( + candidates.breathing_rate_bpm, + candidates.breathing_confidence, + 4.0, + 40.0, + ); + let heart_rate_bpm = qualified( + candidates.heart_rate_bpm, + candidates.heartbeat_confidence, + 30.0, + 240.0, + ); if breathing_rate_bpm.is_none() && heart_rate_bpm.is_none() { return None; } Some(VitalSigns { breathing_rate_bpm, heart_rate_bpm, - breathing_confidence: candidates.breathing_confidence.clamp(0.0, 1.0), - heartbeat_confidence: candidates.heartbeat_confidence.clamp(0.0, 1.0), + breathing_confidence: if breathing_rate_bpm.is_some() { + candidates.breathing_confidence.clamp(0.0, 1.0) + } else { + 0.0 + }, + heartbeat_confidence: if heart_rate_bpm.is_some() { + candidates.heartbeat_confidence.clamp(0.0, 1.0) + } else { + 0.0 + }, signal_quality, }) } @@ -8626,26 +8996,222 @@ fn vitals_for_publication( /// cannot authorize numeric rates when the bound field model says absent. /// Require both the existing room gate and a fresh, complete model window; /// never use a heuristic fallback as calibrated single-occupant evidence. +struct VitalGateAssessment { + published: Option, + reason: vital_publication::Reason, + bound_window_ready: bool, + calibrated_person_count: Option, +} + +fn assess_calibrated_vitals( + state: &AppStateInner, + candidates: &VitalSigns, + room_person_count: usize, + observed_at_unix_ms: u64, +) -> VitalGateAssessment { + use vital_publication::Reason; + let binding = state.calibration_grid_binding; + let window = state + .field_model + .as_ref() + .and_then(|m| m.modes()) + .and_then(|m| m.baseline_runtime_window_size); + let ready = binding.zip(window).is_some_and(|(binding, window)| { + state.field_model_holdout_ready(binding, window, std::time::Instant::now()) + }); + let evidence = binding.filter(|_| ready).and_then(|binding| { + state.calibrated_presence_evidence(binding.source_node_id, state.tick, observed_at_unix_ms) + }); + let count = evidence.as_ref().map(|e| e.person_count); + let reason = if state.privacy_mode { + Reason::PrivacyRedacted + } else if room_person_count != 1 { + Reason::RoomNotSingleOccupant + } else if !state.calibration_sequence_fault_node_ids.is_empty() { + Reason::SequenceFault + } else if !state.explicit_calibration_fresh_at(observed_at_unix_ms) { + Reason::ExplicitCalibrationRequired + } else if binding.is_none() || window.is_none() { + Reason::BoundModelUnavailable + } else if !ready { + Reason::BoundWindowNotReady + } else if count.is_none() { + Reason::CalibratedPresenceUnavailable + } else if count != Some(1) { + Reason::CalibratedNotSingleOccupant + } else if !candidates.signal_quality.is_finite() + || candidates.signal_quality < VITAL_PUBLICATION_MIN_SIGNAL_QUALITY + { + Reason::LowSignalQuality + } else if candidates.breathing_rate_bpm.is_none() && candidates.heart_rate_bpm.is_none() { + Reason::NoCandidateRate + } else if !candidates.breathing_rate_bpm.is_some_and(|rate| rate.is_finite() && (4.0..=40.0).contains(&rate)) + && !candidates.heart_rate_bpm.is_some_and(|rate| rate.is_finite() && (30.0..=240.0).contains(&rate)) { + Reason::InvalidCandidateRate + } else { + Reason::LowRateConfidence + }; + let published = if matches!(reason, Reason::LowRateConfidence) { + vitals_for_publication(candidates, true, 1) + } else { + None + }; + VitalGateAssessment { + reason: if published.is_some() { + Reason::Published + } else { + reason + }, + published, + bound_window_ready: ready, + calibrated_person_count: count, + } +} + fn calibrated_vitals_for_publication( state: &AppStateInner, candidates: &VitalSigns, room_person_count: usize, observed_at_unix_ms: u64, ) -> Option { - if room_person_count != 1 || !state.calibration_sequence_fault_node_ids.is_empty() { - return None; + assess_calibrated_vitals(state, candidates, room_person_count, observed_at_unix_ms).published +} + +/// REST reuses the last atomic room aggregate, never the single-link field +/// count in isolation. A stale candidate is independently refused below. +fn current_vital_room_count(state: &AppStateInner, now: std::time::Instant) -> usize { + if state.node_states.values().any(|n| node_is_fresh(n, now)) { + state.prev_person_count + } else { + 0 } - let binding = state.calibration_grid_binding?; - let window_size = state.field_model.as_ref()?.modes()?.baseline_runtime_window_size?; - if !state.field_model_holdout_ready(binding, window_size, std::time::Instant::now()) { - return None; +} + +fn vital_calibration_identity( + state: &AppStateInner, +) -> Option { + let receipt = state.calibration_model_receipt.as_ref()?; + Some(vital_publication::CalibrationIdentity { + boot_epoch: receipt.boot_epoch.clone(), + session_id: receipt.session_id.clone(), + model_id: receipt.model_id.clone(), + binding_digest: receipt.binding_digest.clone(), + source_node_ids: receipt.source_node_ids.clone(), + completed_at_unix_ms: receipt.completed_at_unix_ms, + }) +} + +fn node_vital_decision( + state: &AppStateInner, + node_id: u8, + room_count: usize, + observed_at_unix_ms: u64, +) -> (Option, vital_publication::Decision) { + use vital_publication::{Decision, Reason, CANDIDATE_MAX_AGE_MS}; + let node = &state.node_states[&node_id]; + let candidate = node.vital_candidate.as_ref(); + let rates = candidate.map(|c| c.rates.clone()).unwrap_or_default(); + let age = candidate.map(|c| c.received_at.elapsed().as_millis().min(u64::MAX as u128) as u64); + let mut gate = assess_calibrated_vitals(state, &rates, room_count, observed_at_unix_ms); + if !state.privacy_mode { + if candidate.is_none() { + gate.reason = Reason::NoCandidate; + gate.published = None; + } else if age.is_none_or(|age| age > CANDIDATE_MAX_AGE_MS) + || candidate.is_some_and(|c| { + observed_at_unix_ms.abs_diff(c.observed_at_unix_ms) > CANDIDATE_MAX_AGE_MS + }) + { + gate.reason = Reason::CandidateStale; + gate.published = None; + } + } + if gate.published.is_some() + && candidate.is_some_and(|c| { + c.calibration_identity.is_none() + || c.calibration_identity != vital_calibration_identity(state) + }) + { + gate.reason = Reason::CandidateCalibrationChanged; + gate.published = None; + } + let diagnostic = Decision { + node_id, + candidate_kind: candidate.map(|c| c.kind), + source_tick: candidate.map(|c| c.tick), + observed_at_unix_ms: candidate.map(|c| c.observed_at_unix_ms), + candidate_age_ms: age, + occupancy_source_node_id: state.bound_source_node_id(), + room_person_count: room_count, + calibrated_person_count: gate.calibrated_person_count, + explicit_calibration_fresh: state.explicit_calibration_fresh_at(observed_at_unix_ms), + sequence_continuity_ok: state.calibration_sequence_fault_node_ids.is_empty(), + bound_window_ready: gate.bound_window_ready, + breathing_rate_present: rates.breathing_rate_bpm.is_some(), + heart_rate_present: rates.heart_rate_bpm.is_some(), + signal_quality_passed: rates.signal_quality.is_finite() + && rates.signal_quality >= VITAL_PUBLICATION_MIN_SIGNAL_QUALITY, + breathing_confidence_passed: rates.breathing_confidence.is_finite() + && rates.breathing_confidence >= VITAL_PUBLICATION_MIN_CONFIDENCE, + heartbeat_confidence_passed: rates.heartbeat_confidence.is_finite() + && rates.heartbeat_confidence >= VITAL_PUBLICATION_MIN_CONFIDENCE, + buffers: node.vital_detector.buffer_status().into(), + sample_rate_hz: node.vital_detector.sample_rate_hz(), + observed_sample_rate_hz: node.effective_sample_rate_hz(), + sample_rate_resets: node.vital_sample_rate_resets, + reason: gate.reason, + }; + (gate.published, diagnostic) +} + +/// Called under the same write lock as the room aggregate and outgoing frame. +fn record_node_vital_decision( + state: &mut AppStateInner, + node_id: u8, + mut candidate: vital_publication::NodeVitalCandidate, + room_count: usize, +) -> (Option, Option) { + let observed = candidate.observed_at_unix_ms; + candidate.calibration_identity = vital_calibration_identity(state); + state + .node_states + .get_mut(&node_id) + .expect("active node") + .vital_candidate = Some(candidate); + let (published, diagnostic) = node_vital_decision(state, node_id, room_count, observed); + let node = state.node_states.get_mut(&node_id).expect("active node"); + let evidence = published.as_ref().and_then(|rates| { + vital_publication::evidence( + node_id, + node.vital_candidate.as_ref().expect("recorded candidate"), + rates, + ) + }); + node.last_vital_decision = Some(diagnostic); + (published, evidence) +} + +/// A read may withdraw a previously published candidate, never promote an old +/// withheld sample under a new room/privacy/calibration context. +fn reevaluate_node_vitals( + state: &AppStateInner, + node_id: u8, + room_count: usize, + observed: u64, +) -> (Option, vital_publication::Decision) { + let (mut rates, mut diagnostic) = node_vital_decision(state, node_id, room_count, observed); + let recorded = state.node_states[&node_id].last_vital_decision.as_ref(); + if rates.is_some() + && !recorded.is_some_and(|last| { + last.reason == vital_publication::Reason::Published + && last.source_tick == diagnostic.source_tick + && last.observed_at_unix_ms == diagnostic.observed_at_unix_ms + }) + { + rates = None; + diagnostic.reason = vital_publication::Reason::NotPublishedForSourceTick; } - let evidence = state.calibrated_presence_evidence( - binding.source_node_id, - state.tick, - observed_at_unix_ms, - )?; - vitals_for_publication(candidates, true, evidence.person_count) + (rates, diagnostic) } fn edge_vitals_message_for_publication( @@ -10780,33 +11346,72 @@ fn chrono_timestamp() -> u64 { async fn vital_signs_endpoint(State(state): State) -> Json { let s = state.read().await; - let observed_at_unix_ms = chrono::Utc::now().timestamp_millis().max(0) as u64; - let person_count = s.person_count_at(observed_at_unix_ms); - let published = calibrated_vitals_for_publication( - &s, - &s.latest_vitals, - person_count, - observed_at_unix_ms, - ); - let (br_len, br_cap, hb_len, hb_cap) = s.vital_detector.buffer_status(); + let observed = chrono::Utc::now().timestamp_millis().max(0) as u64; + let room_count = current_vital_room_count(&s, std::time::Instant::now()); + // Candidate, room aggregate, model and diagnostic snapshot share one lock. + let selected = s + .node_states + .iter() + .filter_map(|(&id, node)| node.vital_candidate.as_ref().map(|c| (id, c.tick))) + .max_by_key(|&(id, tick)| (tick, id)) + .map(|(id, _)| id); + let (published, reason) = selected + .map(|id| { + let (published, diagnostic) = reevaluate_node_vitals(&s, id, room_count, observed); + (published, diagnostic.reason) + }) + .unwrap_or(( + None, + if s.privacy_mode { + vital_publication::Reason::PrivacyRedacted + } else { + vital_publication::Reason::NoCandidate + }, + )); + let mut ids: Vec<_> = s.node_states.keys().copied().collect(); + ids.sort_unstable(); + // This existing endpoint retains its owner/paired-read/account middleware. + // Diagnostics are never added to public health or authority envelopes. + let diagnostics: Vec<_> = if s.privacy_mode { + vec![] + } else { + ids.into_iter() + .take(256) + .map(|id| { + let (_, current) = reevaluate_node_vitals(&s, id, room_count, observed); + serde_json::json!({"current": current, + "last_publication_decision": s.node_states[&id].last_vital_decision}) + }) + .collect() + }; + let buffer_source = if s.privacy_mode { + None + } else { + selected.or_else(|| { + s.node_states + .iter() + .max_by_key(|(&id, n)| (n.last_frame_time, id)) + .map(|(&id, _)| id) + }) + }; + let buffers = buffer_source.map(|id| { + vital_publication::BufferStatus::from(s.node_states[&id].vital_detector.buffer_status()) + }); Json(serde_json::json!({ "vital_signs": { - "breathing_rate_bpm": published.as_ref().and_then(|value| value.breathing_rate_bpm), - "heart_rate_bpm": published.as_ref().and_then(|value| value.heart_rate_bpm), - "breathing_confidence": published.as_ref().map(|value| value.breathing_confidence), - "heartbeat_confidence": published.as_ref().map(|value| value.heartbeat_confidence), - "signal_quality": published.as_ref().map(|value| value.signal_quality), + "breathing_rate_bpm": published.as_ref().and_then(|v| v.breathing_rate_bpm), + "heart_rate_bpm": published.as_ref().and_then(|v| v.heart_rate_bpm), + "breathing_confidence": published.as_ref().map(|v| v.breathing_confidence), + "heartbeat_confidence": published.as_ref().map(|v| v.heartbeat_confidence), + "signal_quality": published.as_ref().map(|v| v.signal_quality), }, "authority": if published.is_some() { "explicit_calibration" } else { "abstained" }, - "abstention_reason": if published.is_some() { serde_json::Value::Null } else { serde_json::json!("fresh explicit calibration with exactly one occupant and qualified evidence required") }, - "buffer_status": { - "breathing_samples": br_len, - "breathing_capacity": br_cap, - "heartbeat_samples": hb_len, - "heartbeat_capacity": hb_cap, - }, - "source": s.effective_source(), - "tick": s.tick, + "abstention_reason": if published.is_some() { serde_json::Value::Null } else { serde_json::json!(reason) }, + "buffer_status": buffers, + "buffer_source_node_id": buffer_source, + "diagnostics": {"schema": vital_publication::DIAGNOSTICS_SCHEMA, + "privacy_redacted": s.privacy_mode, "nodes": diagnostics}, + "source": s.effective_source(), "tick": s.tick, })) } @@ -12003,6 +12608,7 @@ fn mediatek_room_update( values: vec![], }, vital_signs: None, + vital_evidence: None, calibrated_presence_evidence: None, detector_diagnostics: None, enhanced_motion: None, @@ -12888,11 +13494,11 @@ async fn udp_receiver_task( }; let explicit_calibration_fresh = s.explicit_calibration_fresh_at(observed_at_unix_ms); - let published_vitals = calibrated_vitals_for_publication( - &s, - &vital_candidates, - total_persons, - observed_at_unix_ms, + let (published_vitals, vital_evidence) = record_node_vital_decision( + &mut s, node_id, vital_publication::NodeVitalCandidate { + kind: vital_publication::CandidateKind::FirmwareEdge, + tick, observed_at_unix_ms, received_at: now, rates: vital_candidates, calibration_identity: None, + }, total_persons, ); // Keep the legacy edge_vitals stream aligned with the @@ -12913,11 +13519,11 @@ async fn udp_receiver_task( let calibrated_presence_evidence = s.calibrated_presence_evidence( node_id, tick, - chrono::Utc::now().timestamp_millis().max(0) as u64, + observed_at_unix_ms, ); let mut update = SensingUpdate { msg_type: "sensing_update".to_string(), - timestamp: chrono::Utc::now().timestamp_millis() as f64 / 1000.0, + timestamp: observed_at_unix_ms as f64 / 1000.0, source: "esp32".to_string(), tick, nodes: active_nodes, @@ -12925,6 +13531,7 @@ async fn udp_receiver_task( classification, signal_field, vital_signs: published_vitals, + vital_evidence, detector_diagnostics: calibrated_presence_evidence.as_ref().and_then(|e| e.diagnostics.clone()), calibrated_presence_evidence, enhanced_motion: None, @@ -13236,6 +13843,7 @@ async fn udp_receiver_task( if ns.csi_fps_samples >= 5 && ns.vital_detector.reconfigure_sample_rate(sample_rate_hz) { + ns.vital_sample_rate_resets = ns.vital_sample_rate_resets.saturating_add(1); // Never smooth estimates computed against two clocks. ns.smoothed_hr = 0.0; ns.smoothed_br = 0.0; @@ -13389,21 +13997,21 @@ async fn udp_receiver_task( now, observed_at_unix_ms, ); - let published_vitals = calibrated_vitals_for_publication( - &s, - &vitals, - total_persons, - observed_at_unix_ms, + let (published_vitals, vital_evidence) = record_node_vital_decision( + &mut s, node_id, vital_publication::NodeVitalCandidate { + kind: vital_publication::CandidateKind::RawCsiSpectral, + tick, observed_at_unix_ms, received_at: now, rates: vitals, calibration_identity: None, + }, total_persons, ); let calibrated_presence_evidence = s.calibrated_presence_evidence( node_id, tick, - chrono::Utc::now().timestamp_millis().max(0) as u64, + observed_at_unix_ms, ); let mut update = SensingUpdate { msg_type: "sensing_update".to_string(), - timestamp: chrono::Utc::now().timestamp_millis() as f64 / 1000.0, + timestamp: observed_at_unix_ms as f64 / 1000.0, source: "esp32".to_string(), tick, nodes: active_nodes, @@ -13421,6 +14029,7 @@ async fn udp_receiver_task( &sub_variances, ), vital_signs: published_vitals, + vital_evidence, detector_diagnostics: calibrated_presence_evidence.as_ref().and_then(|e| e.diagnostics.clone()), calibrated_presence_evidence, enhanced_motion: None, @@ -13661,6 +14270,7 @@ async fn simulated_data_task(state: SharedState, tick_ms: u64) { &sub_variances, ), vital_signs: None, + vital_evidence: None, calibrated_presence_evidence: None, detector_diagnostics: None, enhanced_motion: None, @@ -16386,6 +16996,7 @@ mod observatory_persons_field_position_tests { }, signal_field, vital_signs: None, + vital_evidence: None, calibrated_presence_evidence: None, detector_diagnostics: None, enhanced_motion: None, @@ -17207,3 +17818,43 @@ mod issue_2088_data_dir_tests { assert!(!models_dir.join("dd_model_2088.rvf").exists()); } } + + +#[cfg(test)] +mod investigation_vitals_endpoint_tests { + use super::*; + + #[tokio::test] + async fn investigation_vitals_rest_buffers_follow_active_esp32_detector() { + let mut state = AppStateInner::minimal(); + state.source = "esp32".to_string(); + state.last_esp32_frame = Some(std::time::Instant::now()); + let mut node = NodeState::new(); + for _ in 0..12 { + node.vital_detector.process_frame(&[10.0, 11.0, 12.0, 13.0], &[0.0, 0.1, 0.2, 0.3]); + } + let (expected, _, expected_heart, _) = node.vital_detector.buffer_status(); + assert_eq!(expected, 12); + state.node_states.insert(15, node); + let Json(body) = vital_signs_endpoint(State(Arc::new(RwLock::new(state)))).await; + println!("INVESTIGATION: ESP32 node15 has {} breathing/{} heartbeat samples; endpoint reports {} / {}", expected, expected_heart, body["buffer_status"]["breathing_samples"], body["buffer_status"]["heartbeat_samples"]); + assert_eq!(body["buffer_status"]["breathing_samples"], serde_json::json!(expected), "REST diagnostic must describe the detector populated by the active ESP32 path"); + } + + #[test] + fn investigation_vitals_regular_bursts_do_not_continuously_reset_buffers() { + let mut detector = VitalSignDetector::new(20.0); + let mut fps = 20.0; + let mut resets = 0; + for index in 0..300 { + let dt = [0.001, 0.001, 0.094][index % 3]; + fps = update_csi_fps_ema(fps, dt).unwrap().clamp(1.0, 50.0); + if index >= 5 && detector.reconfigure_sample_rate(fps) { resets += 1; } + detector.process_frame(&[10.0, 11.0, 12.0, 13.0], &[0.0, 0.1, 0.2, 0.3]); + } + let buffers = detector.buffer_status(); + println!("INVESTIGATION: synthetic31.25Hz3-packet bursts: resets={} buffers={:?} detector_rate={}", resets, buffers, detector.sample_rate_hz()); + assert!(resets < 6); + assert!(buffers.0 > 40 && buffers.2 > 30); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/privacy_filter.rs b/v2/crates/wifi-densepose-sensing-server/src/privacy_filter.rs index 244bfc6190..7459860674 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/privacy_filter.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/privacy_filter.rs @@ -40,6 +40,7 @@ pub const BIOMETRIC_KEYS: &[&str] = &[ // confidences; the leaf keys cover edge vitals, fused vitals and the // mmWave block. "vital_signs", + "vital_evidence", "breathing_rate_bpm", "heart_rate_bpm", "heartrate_bpm", diff --git a/v2/crates/wifi-densepose-sensing-server/src/vital_publication.rs b/v2/crates/wifi-densepose-sensing-server/src/vital_publication.rs new file mode 100644 index 0000000000..e94ca6dce5 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/vital_publication.rs @@ -0,0 +1,141 @@ +//! Bounded vital publication provenance and read-only diagnostic DTOs. +//! Candidate rates stay private; diagnostics contain gate outcomes only. + +use crate::vital_signs::VitalSigns; +use serde::{Deserialize, Serialize}; + +pub const DIAGNOSTICS_SCHEMA: &str = "ruview.vital-publication-diagnostics.v1"; +pub const CANDIDATE_MAX_AGE_MS: u64 = 3_000; + +#[derive(Debug, Clone, Copy, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum CandidateKind { + RawCsiSpectral, + FirmwareEdge, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CalibrationIdentity { + pub boot_epoch: String, + pub session_id: String, + pub model_id: String, + pub binding_digest: String, + pub source_node_ids: Vec, + pub completed_at_unix_ms: u64, +} + +#[derive(Debug, Clone)] +pub struct NodeVitalCandidate { + pub kind: CandidateKind, + pub tick: u64, + pub observed_at_unix_ms: u64, + pub received_at: std::time::Instant, + pub rates: VitalSigns, + pub calibration_identity: Option, +} + +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum Reason { + Published, + PrivacyRedacted, + NoCandidate, + CandidateStale, + CandidateCalibrationChanged, + NotPublishedForSourceTick, + RoomNotSingleOccupant, + SequenceFault, + ExplicitCalibrationRequired, + BoundModelUnavailable, + BoundWindowNotReady, + CalibratedPresenceUnavailable, + CalibratedNotSingleOccupant, + LowSignalQuality, + NoCandidateRate, + InvalidCandidateRate, + LowRateConfidence, +} + +#[derive(Debug, Clone, Serialize)] +pub struct BufferStatus { + pub breathing_samples: usize, + pub breathing_capacity: usize, + pub heartbeat_samples: usize, + pub heartbeat_capacity: usize, +} +impl From<(usize, usize, usize, usize)> for BufferStatus { + fn from( + (breathing_samples, breathing_capacity, heartbeat_samples, heartbeat_capacity): ( + usize, + usize, + usize, + usize, + ), + ) -> Self { + Self { + breathing_samples, + breathing_capacity, + heartbeat_samples, + heartbeat_capacity, + } + } +} + +#[derive(Debug, Clone, Serialize)] +pub struct Decision { + pub node_id: u8, + pub candidate_kind: Option, + pub source_tick: Option, + pub observed_at_unix_ms: Option, + pub candidate_age_ms: Option, + pub occupancy_source_node_id: Option, + pub room_person_count: usize, + pub calibrated_person_count: Option, + pub explicit_calibration_fresh: bool, + pub sequence_continuity_ok: bool, + pub bound_window_ready: bool, + pub breathing_rate_present: bool, + pub heart_rate_present: bool, + pub signal_quality_passed: bool, + pub breathing_confidence_passed: bool, + pub heartbeat_confidence_passed: bool, + pub buffers: BufferStatus, + pub sample_rate_hz: f64, + pub observed_sample_rate_hz: f64, + pub sample_rate_resets: u64, + pub reason: Reason, +} + +/// Exact v1 envelope already consumed by the native clients. Inventory and +/// occupancy-source IDs are deliberately not used as rate contributors. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VitalEvidence { + pub schema: String, + pub capability: String, + pub source: String, + pub source_tick: u64, + pub source_node_ids: Vec, + pub observed_at_unix_ms: u64, + #[serde(flatten)] + pub rates: VitalSigns, +} + +pub fn evidence( + node_id: u8, + candidate: &NodeVitalCandidate, + published: &VitalSigns, +) -> Option { + Some(VitalEvidence { + schema: "ruview.vital-evidence.v1".into(), + capability: match candidate.kind { + CandidateKind::RawCsiSpectral => "esp32_csi_spectral_vitals_v1", + CandidateKind::FirmwareEdge => "esp32_firmware_edge_vitals_v1", + } + .into(), + source: "esp32".into(), + source_tick: candidate.tick, + source_node_ids: vec![node_id], + observed_at_unix_ms: candidate.observed_at_unix_ms, + rates: published.clone(), + }) +} diff --git a/v2/crates/wifi-densepose-sensing-server/tests/auth_wiring.rs b/v2/crates/wifi-densepose-sensing-server/tests/auth_wiring.rs index 79e8eb2e45..533008f116 100644 --- a/v2/crates/wifi-densepose-sensing-server/tests/auth_wiring.rs +++ b/v2/crates/wifi-densepose-sensing-server/tests/auth_wiring.rs @@ -203,7 +203,24 @@ const WS_PATHS: &[&str] = &["/ws/sensing", "/ws/introspection", "/api/v1/stream/ /// protected `/api/v1/*` by prefix. `/ws/field` was gated in this PR and its /// REST twin, one path segment over, returned 200 to an anonymous caller on /// both listeners. That is why the gate is now deny-by-default. -const PROTECTED_REST_PATHS: &[&str] = &["/api/field", "/api/v1/models"]; +const PROTECTED_REST_PATHS: &[&str] = &["/api/field", "/api/v1/models", "/api/v1/vital-signs"]; + +#[test] +fn actual_vital_diagnostics_require_the_existing_bearer_boundary() { + let server = Server::start(&[("RUVIEW_API_TOKEN", TOKEN)]); + assert_eq!(status(server.http, "GET", "/api/v1/vital-signs", &[]), 401); + let mut connection = TcpStream::connect(("127.0.0.1", server.http)).unwrap(); + connection.set_read_timeout(Some(Duration::from_secs(10))).unwrap(); + write!(connection, "GET /api/v1/vital-signs HTTP/1.1\r\nHost: 127.0.0.1:{}\r\nAuthorization: Bearer {}\r\nConnection: close\r\n\r\n", server.http, TOKEN).unwrap(); + let mut response = String::new(); + connection.read_to_string(&mut response).unwrap(); + assert!(response.starts_with("HTTP/1.1 200")); + let (_, body) = response.split_once("\r\n\r\n").unwrap(); + let body: serde_json::Value = serde_json::from_str(body).unwrap(); + assert_eq!(body["diagnostics"]["schema"], "ruview.vital-publication-diagnostics.v1"); + assert_eq!(body["authority"], "abstained"); + assert!(body["vital_signs"]["heart_rate_bpm"].is_null()); +} #[test] fn with_auth_on_no_listener_serves_sensing_data_anonymously() { From fc235343b1a76e62c5b047f11d1b3c1ddeb31c69 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Wed, 28 Oct 2026 15:24:31 +0100 Subject: [PATCH 019/100] fix(stream): stop cached-frame rebroadcast and withhold stale pose --- ...ADR-295-source-provenance-state-machine.md | 35 +++ scripts/verify-sensing-stream-freshness.mjs | 101 +++++++ .../wifi-densepose-sensing-server/src/main.rs | 250 ++++++++++++++++-- 3 files changed, 363 insertions(+), 23 deletions(-) create mode 100644 scripts/verify-sensing-stream-freshness.mjs diff --git a/docs/adr/ADR-295-source-provenance-state-machine.md b/docs/adr/ADR-295-source-provenance-state-machine.md index c101d9e608..8bc4fddd6c 100644 --- a/docs/adr/ADR-295-source-provenance-state-machine.md +++ b/docs/adr/ADR-295-source-provenance-state-machine.md @@ -62,3 +62,38 @@ simulator status, sensing client source label). Broader UI adoption follows. simulator → never live without a verified frame; freshness expiry → `Stale`; watermark present on synthetic export. - `cargo test -p wifi-densepose-sensing-server`. + +## October 28 amendment: cached observations and stream keepalive + +A timer must not repeatedly publish the same sensing observation as transport +keepalive. Raw, edge and synthetic producers continue to publish their new +observations directly. The ESP32 timer now publishes a cached snapshot only on +an effective-source transition, including the first offline transition and a +later recovery. It preserves the observation's original tick and wall-clock +measurement timestamp rather than making old evidence fresh. + +A new raw or legacy-pose subscriber receives one local initial snapshot through +that connection's existing authorization and privacy filtering. This initial +snapshot is not sent to the shared broadcast bus. Both transports have an +independent protocol Ping keepalive, so quiet sensing requires no synthetic +measurement traffic. Late subscribers can inspect the historical timestamp and +offline source without amplifying traffic to every existing client. + +Cached snapshots refresh ESP32 receive-clock node age/stale metadata. Optional +expired vital, calibrated-presence, detector, room-inference and pose-proof blocks +are withheld; the historical classification is not rewritten as an observed +absence. Receive-rate estimates describe their original sample history: an +unchanging accepted CSI sequence alongside a receive rate can reflect feature-grid +rejection, and is not relabelled as packet loss or an invented accepted rate. + +Validation uses the real raw/legacy handlers and an isolated actual-helper +UDP-stop/late-subscribe/resume probe, including privacy-on behavior: +`node scripts/verify-sensing-stream-freshness.mjs /path/to/sensing-server`. +The probe uses synthetic loopback data and private temporary storage; it does +not establish real-hardware sensing accuracy or room-selective occupancy. + +The legacy pose conversion abstains before any model/signal reconstruction or +physics assessment for an offline or expired cached observation. It retains the +original timestamp/source but emits `pose_source`/`activity: unavailable`, null +confidence and null estimated-person count, with no produced person poses. The +empty pose array means no available pose candidates, not a measured empty room. diff --git a/scripts/verify-sensing-stream-freshness.mjs b/scripts/verify-sensing-stream-freshness.mjs new file mode 100644 index 0000000000..3ac62c3bd5 --- /dev/null +++ b/scripts/verify-sensing-stream-freshness.mjs @@ -0,0 +1,101 @@ +#!/usr/bin/env node +// SYNTHETIC loopback-only regression against an actual helper. Node22+. +// Usage: node scripts/verify-sensing-stream-freshness.mjs /path/to/sensing-server +import assert from 'node:assert/strict'; +import { createHash } from 'node:crypto'; +import { mkdtempSync, readFileSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { spawn } from 'node:child_process'; +import net from 'node:net'; +import dgram from 'node:dgram'; +import { runInNewContext } from 'node:vm'; +const binary = resolve(process.argv[2] ?? 'v2/target/release/sensing-server'); +const binarySha256 = createHash('sha256').update(readFileSync(binary)).digest('hex'); +const evidence = mkdtempSync(join(tmpdir(), 'ruview-stream-freshness-')); +const delay = ms => new Promise(resolve => setTimeout(resolve, ms)); +const legacySource = readFileSync(new URL('../ui/services/pose.service.js', import.meta.url), 'utf8') + .replace(/^import[^\n]*\n/gm, '').replace('export class PoseService', 'class PoseService') + .replace('export const poseService', 'const poseService'); +const PoseService = runInNewContext(`${legacySource};PoseService;`, { + console: {log() {}, warn() {}, error() {}, debug() {}}, API_CONFIG: {}, +}); + +const freePort = () => new Promise(resolve => { + const socket = net.createServer(); + socket.listen(0, '127.0.0.1', () => { const port = socket.address().port; socket.close(() => resolve(port)); }); +}); +const connect = async url => { + const socket = new WebSocket(url), frames = []; + socket.onmessage = event => { if (typeof event.data === 'string') frames.push(JSON.parse(event.data)); }; + await new Promise((resolve, reject) => { socket.onopen = resolve; socket.onerror = reject; }); + return { socket, frames }; +}; +async function scenario(privacy) { + const root = join(evidence, privacy ? 'privacy' : 'normal'); + const [http, ws, udpPort] = [await freePort(), await freePort(), await freePort()]; + const args = ['--source', 'esp32', '--bind-addr', '127.0.0.1', '--udp-bind', '127.0.0.1', + '--http-port', String(http), '--ws-port', String(ws), '--udp-port', String(udpPort), + '--data-dir', root, '--no-mdns', ...(privacy ? ['--privacy-mode'] : [])]; + const child = spawn(binary, args, {env: {PATH: process.env.PATH, RUST_LOG: 'warn'}, stdio: ['ignore', 'pipe', 'pipe']}); + let logs = ''; child.stdout.on('data', b => logs += b); child.stderr.on('data', b => logs += b); + const sockets = [], udp = dgram.createSocket('udp4'); + try { + let ready = false; + for (let i = 0; i < 100; i++) { try { await fetch(`http://127.0.0.1:${http}/health`); ready = true; break; } catch { await delay(50); } } + assert(ready, 'isolated helper did not start'); + const first = await connect(`ws://127.0.0.1:${ws}/ws/sensing`); sockets.push(first.socket); + async function send(sequence) { + const tones = 56, packet = Buffer.alloc(20 + tones * 2); + packet.writeUInt32LE(0xC5110001, 0); packet[4] = 7; packet[5] = 1; + packet.writeUInt16LE(tones, 6); packet.writeUInt32LE(2437, 8); packet.writeUInt32LE(sequence, 12); + packet.writeInt8(-50, 16); packet.writeInt8(-90, 17); + for (let i = 0; i < tones; i++) { packet[20 + i * 2] = 5 + i % 40; packet[21 + i * 2] = i % 30; } + await new Promise((resolve, reject) => udp.send(packet, udpPort, '127.0.0.1', e => e ? reject(e) : resolve())); + await delay(33); + } + for (let sequence = 1; sequence <= 60; sequence++) await send(sequence); + await delay(100); const beforeQuiet = first.frames.length; + const last = first.frames.filter(f => f.type === 'sensing_update').at(-1); + assert(last, 'live producer did not publish'); + await delay(6200); + const quietFrames = first.frames.slice(beforeQuiet).filter(f => f.type === 'sensing_update'); + assert.equal(quietFrames.length, 1, 'quiet period must emit exactly one offline transition, not frozen snapshots'); + const offline = quietFrames[0]; + assert.equal(offline.source, 'esp32:offline'); assert.equal(offline.tick, last.tick); assert.equal(offline.timestamp, last.timestamp); + if (!privacy) { assert(offline.node_features[0].stale); assert(offline.node_features[0].last_seen_ms >= 5000); } + const count = first.frames.length; + const late = await connect(`ws://127.0.0.1:${ws}/ws/sensing`); sockets.push(late.socket); + const legacy = await connect(`ws://127.0.0.1:${http}/api/v1/stream/pose`); sockets.push(legacy.socket); + await delay(250); + assert.equal(first.frames.length, count, 'new subscriber must not broadcast its snapshot to existing clients'); + const initial = late.frames.filter(f => f.type === 'sensing_update'); + assert.equal(initial.length, 1); assert.equal(initial[0].tick, last.tick); assert.equal(initial[0].timestamp, last.timestamp); + assert.equal(initial[0].source, 'esp32:offline'); + const pose = legacy.frames.filter(f => f.type === 'pose_data'); assert.equal(pose.length, 1); + assert.equal(pose[0].timestamp, last.timestamp); assert.equal(pose[0].payload.metadata.source, 'esp32:offline'); + assert.equal(pose[0].payload.pose_source, 'unavailable'); assert.equal(pose[0].payload.confidence, null); + assert.equal(pose[0].payload.metadata.estimated_persons, null); assert.deepEqual(pose[0].payload.pose.persons, []); + assert.equal(pose[0].payload.metadata.evidence_state, 'unavailable'); + const consumer = new PoseService(); assert(consumer.validatePoseData(pose[0].payload).valid); + const converted = consumer.convertZoneDataToRestFormat(pose[0].payload, 'zone_1', pose[0]); + assert.equal(converted.pose_source, 'unavailable'); assert.equal(converted.metadata.confidence, null); + assert.equal(JSON.stringify(converted.zone_summary), '{}'); + if (privacy) for (const frame of [offline, initial[0], pose[0]]) { + const serialized = JSON.stringify(frame); assert(!serialized.includes('pose_keypoints')); assert(!serialized.includes('keypoints')); assert(!serialized.includes('vital_signs')); + } + for (let sequence = 61; sequence <= 70; sequence++) await send(sequence); + await delay(100); + assert(first.frames.some(f => f.type === 'sensing_update' && f.source === 'esp32' && f.tick > last.tick), 'fresh input must resume publication'); + return {privacy, quietFrames: quietFrames.length, lateInitialFrames: initial.length, legacyInitialFrames: pose.length, resumed: true}; + } finally { + sockets.forEach(socket => socket.close()); udp.close(); child.kill('SIGTERM'); + await Promise.race([new Promise(resolve => child.once('exit', resolve)), delay(3000)]); + if (child.exitCode === null) child.kill('SIGKILL'); + writeFileSync(join(evidence, privacy ? 'privacy.log' : 'normal.log'), logs); + } +} +const results = [await scenario(false), await scenario(true)]; +const receipt = {evidence: 'SYNTHETIC isolated actual-binary UDP/WS', binarySha256, results}; +writeFileSync(join(evidence, 'result.json'), JSON.stringify(receipt, null, 2)); +console.log(JSON.stringify({evidencePath: join(evidence, 'result.json'), ...receipt}, null, 2)); diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index 30f593cc0b..0df44f6f3f 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -6151,6 +6151,34 @@ fn generate_simulated_frame(tick: u64) -> Esp32Frame { } } +/// A cached observation is not a new measurement. Keep its original identity +/// and timestamp while refreshing receive-clock metadata for a new subscriber +/// or source-state transition. Never rebroadcast this snapshot per subscriber. +fn current_sensing_snapshot(state: &AppStateInner) -> Option { + let mut snapshot = state.latest_update.clone()?; + snapshot.source = state.effective_source(); + if snapshot.source.starts_with("esp32") { + let now = std::time::Instant::now(); + snapshot.node_features = build_node_features(&state.node_states, now); + for node in &mut snapshot.nodes { + node.node_inference = state.node_states.get(&node.node_id) + .map(|state| node_inference_for(state, now)); + } + } + let age_ms = chrono::Utc::now().timestamp_millis() as f64 - snapshot.timestamp * 1000.0; + if snapshot.source.ends_with(":offline") || age_ms > LATEST_UPDATE_STALE_AFTER.as_millis() as f64 { + // Preserve historical classification, not a fabricated absence. + snapshot.vital_signs = None; + snapshot.vital_evidence = None; + snapshot.calibrated_presence_evidence = None; + snapshot.detector_diagnostics = None; + snapshot.room_inference = None; + snapshot.persons = None; + snapshot.pose_keypoints = None; + } + Some(snapshot) +} + // ── WebSocket handler ──────────────────────────────────────────────────────── async fn ws_sensing_handler( @@ -6166,9 +6194,10 @@ async fn ws_sensing_handler( } async fn handle_ws_client(mut socket: WebSocket, state: SharedState) { - let (mut rx, privacy_mode, _client) = { + let (mut rx, privacy_mode, _client, mut initial_snapshot) = { let s = state.read().await; - (s.tx.subscribe(), s.privacy_mode, s.ws_clients.enter()) + (s.tx.subscribe(), s.privacy_mode, s.ws_clients.enter(), + current_sensing_snapshot(&s).and_then(|update| serde_json::to_string(&update).ok())) }; info!("WebSocket client connected (sensing)"); @@ -6179,7 +6208,12 @@ async fn handle_ws_client(mut socket: WebSocket, state: SharedState) { loop { tokio::select! { - msg = rx.recv() => { + msg = async { + match initial_snapshot.take() { + Some(snapshot) => Ok(snapshot), + None => rx.recv().await, + } + } => { match msg { Ok(json) => { let json = if privacy_mode { @@ -6283,9 +6317,10 @@ async fn ws_pose_handler( } async fn handle_ws_pose_client(mut socket: WebSocket, state: SharedState) { - let (mut rx, privacy_mode, _client) = { + let (mut rx, privacy_mode, _client, mut initial_snapshot) = { let s = state.read().await; - (s.tx.subscribe(), s.privacy_mode, s.ws_clients.enter()) + (s.tx.subscribe(), s.privacy_mode, s.ws_clients.enter(), + current_sensing_snapshot(&s).and_then(|update| serde_json::to_string(&update).ok())) }; info!("WebSocket client connected (pose)"); @@ -6297,14 +6332,41 @@ async fn handle_ws_pose_client(mut socket: WebSocket, state: SharedState) { }); let _ = socket.send(Message::Text(conn_msg.to_string())).await; + let mut ping_interval = tokio::time::interval(Duration::from_secs(30)); + ping_interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + loop { tokio::select! { - msg = rx.recv() => { + msg = async { + match initial_snapshot.take() { + Some(snapshot) => Ok(snapshot), + None => rx.recv().await, + } + } => { match msg { Ok(json) => { // Parse the sensing update and convert to pose format if let Ok(sensing) = serde_json::from_str::(&json) { if sensing.msg_type == "sensing_update" { + let age_ms = chrono::Utc::now().timestamp_millis() as f64 - sensing.timestamp * 1000.0; + if sensing.source.ends_with(":offline") || age_ms > LATEST_UPDATE_STALE_AFTER.as_millis() as f64 { + // Unavailable is not measured absence. Never reconstruct + // a skeleton from a deliberately expired cached snapshot. + let mut unavailable = serde_json::json!({ + "type": "pose_data", "zone_id": "zone_1", + "timestamp": sensing.timestamp, "physics": null, + "payload": { + "pose": {"persons": []}, "confidence": null, + "activity": "unavailable", "pose_source": "unavailable", + "metadata": {"frame_id": format!("rust_frame_{}", sensing.tick), + "source": sensing.source, "tick": sensing.tick, + "estimated_persons": null, "evidence_state": "unavailable"} + } + }); + if privacy_mode { privacy_filter::redact_value(&mut unavailable); } + if socket.send(Message::Text(unavailable.to_string())).await.is_err() { break; } + continue; + } // Determine pose estimation mode for the UI indicator. // "model_inference" — a trained RVF model is loaded. // "signal_derived" — keypoints estimated from raw CSI features. @@ -6413,6 +6475,9 @@ async fn handle_ws_pose_client(mut socket: WebSocket, state: SharedState) { Err(_) => break, // channel closed } } + _ = ping_interval.tick() => { + if socket.send(Message::Ping(vec![])).await.is_err() { break; } + } msg = socket.recv() => { match msg { Some(Ok(Message::Text(text))) => { @@ -14334,26 +14399,19 @@ async fn simulated_data_task(state: SharedState, tick_ms: u64) { async fn broadcast_tick_task(state: SharedState, tick_ms: u64) { let mut interval = tokio::time::interval(Duration::from_millis(tick_ms)); - + interval.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Skip); + let mut previous_source = None; loop { interval.tick().await; let s = state.read().await; - if let Some(ref update) = s.latest_update { - if s.tx.receiver_count() > 0 { - // Re-broadcast the latest sensing_update so pose WS clients - // always get data even when ESP32 pauses between frames. - // - // Issue #618: overwrite `source` with `effective_source()` - // before each broadcast so a stale latest_update (frozen - // payload from a now-offline ESP32) is emitted with - // `source: "esp32:offline"` instead of `source: "esp32"`. - // The REST `/health` endpoint already does this; before - // this fix the WS path was the only consumer that didn't, - // so the UI's "LIVE — ESP32 HARDWARE Connected" banner - // stayed green long after the hardware went away. - let mut tagged = update.clone(); - tagged.source = s.effective_source(); - if let Ok(json) = serde_json::to_string(&tagged) { + let source = s.effective_source(); + let changed = previous_source.as_ref().is_some_and(|previous| previous != &source); + previous_source = Some(source); + // Producers publish observations directly. Only announce source-state + // transitions here; WebSocket Ping provides keepalive without cached CSI. + if changed && s.tx.receiver_count() > 0 { + if let Some(snapshot) = current_sensing_snapshot(&s) { + if let Ok(json) = serde_json::to_string(&snapshot) { let _ = s.tx.send(json); } } @@ -17858,3 +17916,149 @@ mod investigation_vitals_endpoint_tests { assert!(buffers.0 > 40 && buffers.2 > 30); } } + +#[cfg(test)] +mod cached_stream_freshness_tests { + use super::*; + + fn fixture() -> AppStateInner { + let mut state = AppStateInner::minimal(); + state.source = "esp32".into(); + state.last_esp32_frame = Some(std::time::Instant::now()); + state.latest_update = Some(serde_json::from_value(serde_json::json!({ + "type":"sensing_update", "tick":9, + "timestamp":chrono::Utc::now().timestamp_millis() as f64 / 1000.0, + "source":"esp32", "nodes":[], + "features":{"mean_rssi":-50.0,"variance":1.0,"motion_band_power":0.5, + "breathing_band_power":0.2,"dominant_freq_hz":0.25,"change_points":0,"spectral_power":1.0}, + "classification":{"motion_level":"present_still","presence":true,"confidence":0.9}, + "signal_field":{"grid_size":[1,1,1],"values":[0.0]}, + "vital_signs":{"breathing_rate_bpm":14.0,"heart_rate_bpm":62.0, + "breathing_confidence":0.8,"heartbeat_confidence":0.7,"signal_quality":0.9}, + "pose_keypoints":[[0.1,0.2,0.0,0.9]] + })).unwrap()); + state + } + + #[test] + fn cached_snapshot_retains_observation_identity_and_ages_node_metadata() { + let mut state = fixture(); + let old = std::time::Instant::now() - Duration::from_secs(6); + state.last_esp32_frame = Some(old); + let mut node = NodeState::new(); + node.last_frame_time = Some(old); + state.node_states.insert(7, node); + let previous = serde_json::to_string(state.latest_update.as_ref().unwrap()).unwrap(); + let snapshot = current_sensing_snapshot(&state).unwrap(); + let original = state.latest_update.as_ref().unwrap(); + assert_eq!(snapshot.tick, original.tick); + assert_eq!(snapshot.timestamp, original.timestamp); + assert_eq!(snapshot.source, "esp32:offline"); + assert!(snapshot.classification.presence, "history must not become a fabricated absence"); + assert!(snapshot.vital_signs.is_none()); + assert!(snapshot.pose_keypoints.is_none()); + let node = &snapshot.node_features.as_ref().unwrap()[0]; + assert!(node.stale); + assert!(node.last_seen_ms >= 6000); + assert_eq!(serde_json::to_string(original).unwrap(), previous, "snapshot must not rewrite stored observations"); + } + + #[tokio::test] + async fn cached_snapshots_use_each_clients_filter_and_protocol_keepalive() { + use futures_util::StreamExt; + for privacy in [false, true] { + let mut inner = fixture(); + inner.privacy_mode = privacy; + let state = Arc::new(RwLock::new(inner)); + let app = Router::new() + .route("/ws/sensing", get(ws_sensing_handler)) + .route("/api/v1/stream/pose", get(ws_pose_handler)) + .with_state(state); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + for (path, kind) in [("/ws/sensing", "sensing_update"), ("/api/v1/stream/pose", "pose_data")] { + let (mut socket, _) = tokio_tungstenite::connect_async(format!("ws://{address}{path}")).await.unwrap(); + let mut snapshot = None; + let mut ping = false; + for _ in 0..4 { + let message = tokio::time::timeout(Duration::from_secs(1), socket.next()).await.unwrap().unwrap().unwrap(); + match message { + tokio_tungstenite::tungstenite::Message::Ping(_) => ping = true, + tokio_tungstenite::tungstenite::Message::Text(text) => { + let value: serde_json::Value = serde_json::from_str(&text).unwrap(); + if value["type"] == kind { snapshot = Some(text); } + } + _ => {} + } + if snapshot.is_some() && ping { break; } + } + assert!(ping, "{path} needs keepalive independent of sensor snapshots"); + let snapshot = snapshot.expect("initial snapshot without timer or producer"); + if privacy { + assert!(!snapshot.contains("vital_signs")); + assert!(!snapshot.contains("keypoints")); + } else if kind == "sensing_update" { + assert!(snapshot.contains("heart_rate_bpm")); + } + socket.close(None).await.unwrap(); + } + server.abort(); let _ = server.await; + } + } + + #[tokio::test] + async fn offline_legacy_snapshot_abstains_instead_of_deriving_a_pose_or_absence() { + use futures_util::StreamExt; + for model_loaded in [false, true] { + let mut inner = fixture(); + inner.model_loaded = model_loaded; + inner.last_esp32_frame = Some(std::time::Instant::now() - Duration::from_secs(6)); + let original_timestamp = inner.latest_update.as_ref().unwrap().timestamp; + let app = Router::new().route("/pose", get(ws_pose_handler)) + .with_state(Arc::new(RwLock::new(inner))); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let (mut socket, _) = tokio_tungstenite::connect_async(format!("ws://{address}/pose")).await.unwrap(); + let value = loop { + let message = tokio::time::timeout(Duration::from_secs(1), socket.next()).await.unwrap().unwrap().unwrap(); + if let tokio_tungstenite::tungstenite::Message::Text(text) = message { + let value: serde_json::Value = serde_json::from_str(&text).unwrap(); + if value["type"] == "pose_data" { break value; } + } + }; + assert_eq!(value["timestamp"], original_timestamp); + assert_eq!(value["payload"]["pose_source"], "unavailable"); + assert_eq!(value["payload"]["activity"], "unavailable"); + assert!(value["payload"]["confidence"].is_null()); + assert!(value["payload"]["metadata"]["estimated_persons"].is_null()); + assert_eq!(value["payload"]["metadata"]["source"], "esp32:offline"); + assert_eq!(value["payload"]["pose"]["persons"], serde_json::json!([])); + assert!(value["physics"].is_null()); + socket.close(None).await.unwrap(); + server.abort(); let _ = server.await; + } + } + + #[tokio::test] + async fn timer_emits_only_source_transitions_and_remains_quiet_between_them() { + let state = Arc::new(RwLock::new(fixture())); + let mut rx = state.read().await.tx.subscribe(); + let task = tokio::spawn(broadcast_tick_task(state.clone(), 2)); + tokio::time::sleep(Duration::from_millis(15)).await; + assert!(matches!(rx.try_recv(), Err(tokio::sync::broadcast::error::TryRecvError::Empty))); + state.write().await.last_esp32_frame = Some(std::time::Instant::now() - Duration::from_secs(6)); + let offline = tokio::time::timeout(Duration::from_secs(1), rx.recv()).await.unwrap().unwrap(); + let offline: SensingUpdate = serde_json::from_str(&offline).unwrap(); + assert_eq!(offline.source, "esp32:offline"); + assert_eq!(offline.tick, 9); + tokio::time::sleep(Duration::from_millis(15)).await; + assert!(matches!(rx.try_recv(), Err(tokio::sync::broadcast::error::TryRecvError::Empty))); + state.write().await.last_esp32_frame = Some(std::time::Instant::now()); + let resumed = tokio::time::timeout(Duration::from_secs(1), rx.recv()).await.unwrap().unwrap(); + assert_eq!(serde_json::from_str::(&resumed).unwrap().source, "esp32"); + task.abort(); + let _ = task.await; + } +} From f7fd237a376fae13470bb03ad0d47e430f01c4f6 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Thu, 29 Oct 2026 12:54:36 +0100 Subject: [PATCH 020/100] fix(server): bound managed helper lifetime to spawning app --- .../ADR-419-secure-local-companion-pairing.md | 17 ++ .../wifi-densepose-sensing-server/src/lib.rs | 1 + .../wifi-densepose-sensing-server/src/main.rs | 23 +- .../src/parent_lifetime.rs | 89 ++++++ .../tests/managed_parent.rs | 284 ++++++++++++++++++ 5 files changed, 410 insertions(+), 4 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/parent_lifetime.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/tests/managed_parent.rs diff --git a/docs/adr/ADR-419-secure-local-companion-pairing.md b/docs/adr/ADR-419-secure-local-companion-pairing.md index 844d02017a..ad94536c86 100644 --- a/docs/adr/ADR-419-secure-local-companion-pairing.md +++ b/docs/adr/ADR-419-secure-local-companion-pairing.md @@ -137,3 +137,20 @@ The sensor preview remains transient and currently carries no room-binding digest. A displayed phone mesh is not a saved canonical room, durable draft, live AR alignment or RF calibration receipt. MAT HTTP composition likewise does not expand native companion WebSocket paths; see ADR-001's privacy amendment. + +## App-owned helper lifetime (2026-10-29) + +An embedded Mac helper may opt into `--parent-pid`. The value must be greater +than1 and match the OS spawning parent before account state or listeners open. +The helper watches reparenting from a native thread, requests graceful HTTP +shutdown when its app ends, and exits within roughly2.1seconds even if async +shutdown or an open socket cannot finish. The guard stays armed through final +persistence. Reparenting is checked rather than merely testing whether an old +PID exists, so PID reuse cannot transfer lifetime authority. + +Standalone servers omit this flag and retain their independent lifetime. This +does not grant control over another process or authorize port-based PID killing. +The Mac launcher still refuses an occupied foreign endpoint, with a bounded +vacancy wait for a previous managed helper to finish. Existing durable room, +pairing and account records are not removed by parent loss. Physical capture and +TestFlight acceptance must name the newly embedded helper, not an older build. diff --git a/v2/crates/wifi-densepose-sensing-server/src/lib.rs b/v2/crates/wifi-densepose-sensing-server/src/lib.rs index 8ee14aa2a0..05e7b40501 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/lib.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/lib.rs @@ -29,6 +29,7 @@ pub mod mat_surface; pub mod model_format; pub mod mqtt; pub mod path_safety; +pub mod parent_lifetime; /// ADR-323: fail-closed pose physics integration and raw/refined view selection. pub mod pose_physics; /// #2094: server-wide `--privacy-mode` filter for REST, WebSocket and recordings. diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index 0df44f6f3f..da6b7544b8 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -105,6 +105,10 @@ use wifi_densepose_signal::ruvsense::pose_tracker::PoseTracker; #[derive(Parser, Debug)] #[command(name = "sensing-server", about = "WiFi-DensePose sensing server")] struct Args { + /// App-managed helper only: exit when this verified spawning parent ends. + #[arg(long)] + parent_pid: Option, + /// HTTP port for UI and REST API #[arg(long, default_value = "8080")] http_port: u16, @@ -14708,6 +14712,14 @@ async fn main() { let _telemetry = telemetry::init(); let mut args = Args::parse(); + let (_parent_lifetime_guard, parent_exit) = + match wifi_densepose_sensing_server::parent_lifetime::arm(args.parent_pid) { + Ok(lifetime) => lifetime, + Err(error) => { + eprintln!("Managed helper startup refused: {error}"); + std::process::exit(2); + } + }; if let Ok(scope) = std::env::var(wifi_densepose_sensing_server::bearer_auth::OWNER_ACCOUNT_SCOPE_ENV) { let directory = match wifi_densepose_sensing_server::account_profile::prepare_directory(&scope, &args.data_dir) { Ok(directory) => directory, @@ -16258,10 +16270,13 @@ async fn main() { // Run the HTTP server with graceful shutdown support let shutdown_state = state.clone(); - let server = axum::serve(http_listener, http_app.into_make_service_with_connect_info::()).with_graceful_shutdown(async { - tokio::signal::ctrl_c() - .await - .expect("failed to install CTRL+C handler"); + let server = axum::serve(http_listener, http_app.into_make_service_with_connect_info::()).with_graceful_shutdown(async move { + tokio::select! { + result = tokio::signal::ctrl_c() => result.expect("failed to install CTRL+C handler"), + () = wifi_densepose_sensing_server::parent_lifetime::wait_for_exit(parent_exit) => { + info!("Managed parent ended; closing helper listeners"); + } + } info!("Shutdown signal received"); }); diff --git a/v2/crates/wifi-densepose-sensing-server/src/parent_lifetime.rs b/v2/crates/wifi-densepose-sensing-server/src/parent_lifetime.rs new file mode 100644 index 0000000000..1daff9fa00 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/parent_lifetime.rs @@ -0,0 +1,89 @@ +//! Opt-in lifetime boundary for an app-spawned helper, never port/PID discovery. +use std::{ + io, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::{Duration, Instant}, +}; +use tokio::sync::oneshot; + +/// Keep this guard until server shutdown and final persistence have completed. +pub struct ParentLifetimeGuard(Arc); + +impl Drop for ParentLifetimeGuard { + fn drop(&mut self) { + self.0.store(true, Ordering::Release); + } +} + +#[cfg(unix)] +fn current_parent() -> u32 { + extern "C" { + fn getppid() -> std::os::raw::c_int; + } + // getppid has no arguments, allocation, or borrowed memory. + unsafe { getppid() as u32 } +} + +/// Refuse a wrong/already-detached parent before opening listeners or records. +/// A native thread still observes parent death if the async runtime is blocked. +pub fn arm( + expected_parent: Option, +) -> io::Result<(ParentLifetimeGuard, Option>)> { + let stopped = Arc::new(AtomicBool::new(false)); + let guard = ParentLifetimeGuard(stopped.clone()); + let Some(expected_parent) = expected_parent else { + return Ok((guard, None)); + }; + #[cfg(not(unix))] + { + let _ = expected_parent; + Err(io::Error::new( + io::ErrorKind::Unsupported, + "managed parent lifetime is unsupported on this platform", + )) + } + #[cfg(unix)] + { + if expected_parent <= 1 || current_parent() != expected_parent { + return Err(io::Error::new( + io::ErrorKind::PermissionDenied, + "managed parent does not match the spawning process", + )); + } + let (sender, receiver) = oneshot::channel(); + std::thread::Builder::new() + .name("managed-parent".into()) + .spawn(move || { + while !stopped.load(Ordering::Acquire) { + // Reparenting changes getppid even if the old PID is reused. + if current_parent() != expected_parent { + let _ = sender.send(()); + let deadline = Instant::now() + Duration::from_secs(2); + while !stopped.load(Ordering::Acquire) && Instant::now() < deadline { + std::thread::sleep(Duration::from_millis(25)); + } + if !stopped.load(Ordering::Acquire) { + // Long-lived WebSockets must not keep former account + // authority listening after its owning app disappeared. + eprintln!("Managed parent ended; helper shutdown deadline reached."); + std::process::exit(0); + } + return; + } + std::thread::sleep(Duration::from_millis(100)); + } + })?; + Ok((guard, Some(receiver))) + } +} + +pub async fn wait_for_exit(receiver: Option>) { + if let Some(receiver) = receiver { + let _ = receiver.await; + } else { + std::future::pending::<()>().await; + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/tests/managed_parent.rs b/v2/crates/wifi-densepose-sensing-server/tests/managed_parent.rs new file mode 100644 index 0000000000..48e99d569c --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/tests/managed_parent.rs @@ -0,0 +1,284 @@ +//! Real isolated child processes and sockets; no physical sensing claims. +#![cfg(unix)] + +use std::{ + io::{BufRead, BufReader, Read, Write}, + net::{TcpListener, TcpStream, UdpSocket}, + process::{Child, Command, Stdio}, + thread::sleep, + time::{Duration, Instant}, +}; + +struct Fixture { + parent: Child, + helper: u32, + http: u16, + ws: u16, + directory: tempfile::TempDir, +} + +fn signal(pid: u32, signal: &str) { + let _ = Command::new("kill") + .args([signal, &pid.to_string()]) + .status(); +} + +fn running(pid: u32) -> bool { + let output = Command::new("ps") + .args(["-p", &pid.to_string(), "-o", "stat="]) + .output() + .unwrap(); + let state = String::from_utf8_lossy(&output.stdout); + !state.trim().is_empty() && !state.trim().starts_with('Z') +} + +impl Fixture { + fn start(guarded: bool) -> Self { + let a = TcpListener::bind("127.0.0.1:0").unwrap(); + let b = TcpListener::bind("127.0.0.1:0").unwrap(); + let udp = UdpSocket::bind("127.0.0.1:0").unwrap(); + let (http, ws, udp_port) = ( + a.local_addr().unwrap().port(), + b.local_addr().unwrap().port(), + udp.local_addr().unwrap().port(), + ); + let directory = tempfile::tempdir().unwrap(); + std::fs::write( + directory.path().join("retained-record.json"), + b"synthetic-retained-record", + ) + .unwrap(); + let script = if guarded { + "\"$@\" --parent-pid \"$$\" >/dev/null & child=$!; printf '%s\\n' \"$child\"; wait \"$child\"" + } else { + "\"$@\" >/dev/null & child=$!; printf '%s\\n' \"$child\"; wait \"$child\"" + }; + drop((a, b, udp)); + let mut parent = Command::new("/bin/sh") + .args([ + "-c", + script, + "managed-parent-test", + env!("CARGO_BIN_EXE_sensing-server"), + ]) + .args([ + "--source", + "esp32", + "--bind-addr", + "127.0.0.1", + "--udp-bind", + "127.0.0.1", + "--no-mdns", + "--http-port", + &http.to_string(), + "--ws-port", + &ws.to_string(), + "--udp-port", + &udp_port.to_string(), + "--data-dir", + directory.path().to_str().unwrap(), + ]) + .env_clear() + .env("PATH", "/usr/bin:/bin") + .env("RUST_LOG", "warn") + .stdout(Stdio::piped()) + .stderr(std::fs::File::create(directory.path().join("helper.log")).unwrap()) + .spawn() + .unwrap(); + let mut line = String::new(); + BufReader::new(parent.stdout.take().unwrap()) + .read_line(&mut line) + .unwrap(); + let fixture = Self { + parent, + helper: line.trim().parse().unwrap(), + http, + ws, + directory, + }; + let deadline = Instant::now() + Duration::from_secs(8); + while !fixture.healthy() { + assert!( + Instant::now() < deadline, + "helper did not start: {}", + std::fs::read_to_string(fixture.directory.path().join("helper.log")).unwrap() + ); + sleep(Duration::from_millis(25)); + } + fixture + } + + fn healthy(&self) -> bool { + let Ok(mut stream) = TcpStream::connect(("127.0.0.1", self.http)) else { + return false; + }; + stream + .set_read_timeout(Some(Duration::from_millis(250))) + .unwrap(); + if stream + .write_all(b"GET /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + .is_err() + { + return false; + } + let mut response = [0; 64]; + stream + .read(&mut response) + .is_ok_and(|count| response[..count].starts_with(b"HTTP/1.1 200")) + } + + fn websocket(&self, port: u16) -> TcpStream { + let mut stream = TcpStream::connect(("127.0.0.1", port)).unwrap(); + stream + .set_read_timeout(Some(Duration::from_secs(2))) + .unwrap(); + stream.write_all(b"GET /ws/sensing HTTP/1.1\r\nHost: localhost\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\r\n").unwrap(); + let mut response = [0; 1024]; + let count = stream.read(&mut response).unwrap(); + assert!(response[..count].starts_with(b"HTTP/1.1 101")); + stream + } + + fn end_parent(&mut self, how: &str) { + signal(self.parent.id(), how); + self.parent.wait().unwrap(); + } + + fn assert_stopped(&self) { + let deadline = Instant::now() + Duration::from_secs(4); + while running(self.helper) { + assert!( + Instant::now() < deadline, + "managed helper survived its parent" + ); + sleep(Duration::from_millis(25)); + } + assert!(TcpStream::connect(("127.0.0.1", self.http)).is_err()); + assert!(TcpStream::connect(("127.0.0.1", self.ws)).is_err()); + assert_eq!( + std::fs::read(self.directory.path().join("retained-record.json")).unwrap(), + b"synthetic-retained-record" + ); + } +} + +impl Drop for Fixture { + fn drop(&mut self) { + // Only these fixture-created processes; never enumerate a user's ports. + if running(self.helper) { + signal(self.helper, "-KILL"); + } + if self.parent.try_wait().unwrap().is_none() { + let _ = self.parent.kill(); + } + let _ = self.parent.wait(); + } +} + +#[test] +fn managed_helper_stops_after_parent_force_quit_even_with_an_open_websocket() { + let mut fixture = Fixture::start(true); + let _open_socket = fixture.websocket(fixture.ws); + let _open_http_socket = fixture.websocket(fixture.http); + fixture.end_parent("-KILL"); + fixture.assert_stopped(); +} + +#[test] +fn managed_helper_stops_after_parent_termination() { + let mut fixture = Fixture::start(true); + fixture.end_parent("-TERM"); + fixture.assert_stopped(); +} + +#[test] +fn unmanaged_server_survives_parent_exit() { + let mut fixture = Fixture::start(false); + fixture.end_parent("-KILL"); + sleep(Duration::from_millis(500)); + assert!(running(fixture.helper)); + assert!(fixture.healthy()); +} + +#[test] +fn managed_helper_remains_available_while_its_parent_lives() { + let fixture = Fixture::start(true); + sleep(Duration::from_millis(500)); + assert!(running(fixture.helper)); + assert!(fixture.healthy()); +} + +#[test] +fn wrong_parent_identity_is_rejected_before_creating_state() { + let directory = tempfile::tempdir().unwrap(); + let state = directory.path().join("must-not-be-created"); + for parent in ["0", "1", "4294967295"] { + let result = Command::new(env!("CARGO_BIN_EXE_sensing-server")) + .args([ + "--parent-pid", + parent, + "--data-dir", + state.to_str().unwrap(), + ]) + .env_clear() + .env("PATH", "/usr/bin:/bin") + .output() + .unwrap(); + assert_eq!(result.status.code(), Some(2)); + assert!(!state.exists()); + } +} + +// Subprocess fixture for the production watchdog when its async receiver is +// never polled. Ordinary test execution does not arm or park this fixture. +#[test] +fn watchdog_fixture() { + let Ok(parent) = std::env::var("RUVIEW_TEST_PARENT_PID") else { + return; + }; + let (_guard, _unpolled_receiver) = + wifi_densepose_sensing_server::parent_lifetime::arm(Some(parent.parse().unwrap())).unwrap(); + std::fs::write(std::env::var("RUVIEW_TEST_READY_FILE").unwrap(), b"ready").unwrap(); + loop { + std::thread::park(); + } +} + +#[test] +fn watchdog_bounds_exit_when_graceful_shutdown_cannot_run() { + let directory = tempfile::tempdir().unwrap(); + std::fs::write( + directory.path().join("retained-record.json"), + b"synthetic-retained-record", + ) + .unwrap(); + let ready = directory.path().join("ready"); + let mut parent = Command::new("/bin/sh") + .args(["-c", "RUVIEW_TEST_PARENT_PID=\"$$\" \"$@\" --exact watchdog_fixture --nocapture >/dev/null & child=$!; printf '%s\\n' \"$child\"; wait \"$child\"", "watchdog-parent"]) + .arg(std::env::current_exe().unwrap()) + .env_clear().env("PATH", "/usr/bin:/bin").env("RUVIEW_TEST_READY_FILE", &ready) + .stdout(Stdio::piped()).stderr(Stdio::null()).spawn().unwrap(); + let mut line = String::new(); + BufReader::new(parent.stdout.take().unwrap()) + .read_line(&mut line) + .unwrap(); + let mut fixture = Fixture { + parent, + helper: line.trim().parse().unwrap(), + http: 0, + ws: 0, + directory, + }; + let deadline = Instant::now() + Duration::from_secs(4); + while !ready.exists() { + assert!(Instant::now() < deadline, "watchdog fixture did not arm"); + sleep(Duration::from_millis(25)); + } + let started = Instant::now(); + fixture.end_parent("-KILL"); + fixture.assert_stopped(); + assert!( + started.elapsed() >= Duration::from_secs(2), + "fixture must exercise the hard deadline, not async shutdown" + ); +} From fbd38c3823fe4e7db48f06db6ee27ff0fd7117a1 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Fri, 30 Oct 2026 14:18:32 +0100 Subject: [PATCH 021/100] feat(companion): explicitly review and retain bounded spatial artifacts --- .../ADR-419-secure-local-companion-pairing.md | 26 + .../src/companion_integration.rs | 16 +- .../src/companion_room.rs | 1 + .../src/companion_spatial.rs | 772 ++++++++++++++++++ .../wifi-densepose-sensing-server/src/main.rs | 1 + 5 files changed, 815 insertions(+), 1 deletion(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/companion_spatial.rs diff --git a/docs/adr/ADR-419-secure-local-companion-pairing.md b/docs/adr/ADR-419-secure-local-companion-pairing.md index ad94536c86..45dcad5e31 100644 --- a/docs/adr/ADR-419-secure-local-companion-pairing.md +++ b/docs/adr/ADR-419-secure-local-companion-pairing.md @@ -154,3 +154,29 @@ The Mac launcher still refuses an occupied foreign endpoint, with a bounded vacancy wait for a previous managed helper to finish. Existing durable room, pairing and account records are not removed by parent loss. Physical capture and TestFlight acceptance must name the newly embedded helper, not an older build. + +## Explicit saved spatial artifacts (mobile issue102, 2026-10-30) + +A separate owner-only receive action opens one15-minute artifact slot for an +already paired phone. The paired TLS route accepts a single bounded geometry +payload with a one-use nonce and SHA-256 digest. Publisher identity comes from +the authenticated grant. Upload and acceptance both match the current canonical +installation, room revision and binding; they serialize with room publication. +Pairing revocation and permission expiry withdraw pending review authority. + +Only an explicit owner acceptance retains the artifact. It is historical geometry, +not a calibration/model import: no points enter the detector, no room dimensions +are overwritten and no cold-restored scan establishes current AR alignment. +The payload has at most128 surfaces and64 anchors,48KiB encoded content, capture +and coordinate provenance, and an optional source calibration digest. Raw camera, +depth and CSI are excluded. Accepted history is capped at32 records without eviction. + +Pending permissions/artifacts are memory-only. Accepted history is atomically +written0600 in the account-owned private companion directory. Startup checks +account/installation ownership, geometry bounds and content digests; corrupt, +non-private or symlinked files fail closed. A persistence error freezes the inbox +until restart so an uncertain rename/fsync result cannot be reported as accepted +or followed by further writes. Pairing expiry does not delete already accepted +history. Consumers must implement review and distinguish this retained history +from current alignment. Rollout requires a rebuilt helper and mobile consumer; +source tests are not delivered-device or measured accuracy evidence. diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs index d2ad378a9f..afd277acc3 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs @@ -42,6 +42,7 @@ pub struct Integration { room: Arc, control: Control, sensors: companion_sensors::Shared, + spatial: Option>, advertiser: Option, } impl Integration { @@ -98,6 +99,7 @@ impl Integration { serial: Arc::new(tokio::sync::Mutex::new(())), }; let sensors = companion_sensors::SensorStore::new(service.clone()); + let spatial = auth.owner_account_scope().map(|scope| companion_spatial::SpatialStore::load(&dir, scope, &identity.installation_id, service.clone(), room.clone())).transpose()?; Ok(Some(Self { service, authority, @@ -108,6 +110,7 @@ impl Integration { room, control, sensors, + spatial, })) } pub fn authority(&self) -> &str { @@ -117,6 +120,7 @@ impl Integration { companion::owner_router(self.service.clone()) .merge(companion_sensors::owner_router(self.sensors.clone())) .merge(companion_room::owner_router(self.room.clone())) + .merge(self.spatial.as_ref().map(|s| companion_spatial::owner_router(s.clone())).unwrap_or_default()) .layer(axum::middleware::from_fn_with_state( self.control.clone(), serialize_control, @@ -137,6 +141,8 @@ impl Integration { let app = app .merge(companion_sensors::companion_router(self.sensors.clone())) .merge(companion_room::read_router(self.room.clone())) + .merge(self.spatial.as_ref().map(|s| companion_spatial::peer_router(s.clone())).unwrap_or_default()) + .layer(axum::middleware::from_fn_with_state(self.control.clone(), serialize_control)) .layer(axum::middleware::from_fn_with_state( self.service.clone(), require_companion, @@ -169,7 +175,8 @@ async fn serialize_control( request.uri().path(), "/api/v1/calibration/start" | "/api/v1/pose/calibrate" ); - if !publish && !start { + let artifact = request.method() == axum::http::Method::POST && matches!(request.uri().path(), "/api/v1/companion/spatial" | "/companion/v1/spatial"); + if !publish && !start && !artifact { return next.run(request).await; } let _guard = control.serial.lock().await; @@ -236,10 +243,12 @@ fn policy(method: &axum::http::Method, path: &str) -> Option { | "/ws/sensing" | "/companion/v1/room" | "/companion/v1/sensors" + | "/companion/v1/spatial" ) { return Some(false); } } + if *method == Method::POST && path == "/companion/v1/spatial" { return Some(false); } if *method == Method::POST && matches!( path, @@ -370,6 +379,11 @@ mod tests { assert_eq!(owner.clone().oneshot(req).await.unwrap().status().as_u16(), expected); } assert_eq!(policy(&Method::GET,"/companion/v1/sensors"),Some(false)); + assert_eq!(policy(&Method::GET,"/companion/v1/spatial"),Some(false)); + assert_eq!(policy(&Method::POST,"/companion/v1/spatial"),Some(false)); + assert_eq!(policy(&Method::PUT,"/companion/v1/spatial"),None); + assert_eq!(policy(&Method::GET,"/api/v1/companion/spatial"),None); + assert_eq!(policy(&Method::POST,"/api/v1/companion/spatial"),None); for path in ["/api/v1/companion/sensors", "/api/v1/companion/devices/phone/sensors"] { assert_eq!(policy(&Method::GET,path),None); assert_eq!(policy(&Method::POST,path),None); assert_eq!(policy(&Method::DELETE,path),None); } diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_room.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_room.rs index f91fde656d..aa9a62d24a 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/companion_room.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_room.rs @@ -242,6 +242,7 @@ impl SharedRoomStore { current: Mutex::new(current), })) } + pub fn snapshot(&self) -> Option { self.current.lock().ok().map(|v| v.clone()) } pub fn matches_binding(&self, binding: &str, node_ids: &[u8]) -> bool { let Ok(e) = self.current.lock() else { return false; diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_spatial.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_spatial.rs new file mode 100644 index 0000000000..53a63bff59 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_spatial.rs @@ -0,0 +1,772 @@ +//! Explicit, bounded artifact handoff. Imported geometry never grants RF/AR authority. +use super::companion_room::SharedRoomStore; +use axum::{extract::State, http::StatusCode, routing::get, Extension, Json, Router}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +#[cfg(unix)] +use std::os::unix::fs::OpenOptionsExt; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::{ + collections::HashMap, + fs, + io::Write, + path::{Path, PathBuf}, + sync::{Arc, Mutex}, +}; +use wifi_densepose_sensing_server::companion::{Companion, Grant}; +const MAX_PAYLOAD: usize = 48 * 1024; +fn now() -> u64 { + chrono::Utc::now().timestamp_millis().max(0) as u64 +} +fn id(s: &str) -> bool { + !s.is_empty() + && s.len() <= 128 + && s.bytes() + .all(|b| b.is_ascii_alphanumeric() || b"-_.:".contains(&b)) +} +fn hash(s: &str) -> String { + format!("{:x}", Sha256::digest(s.as_bytes())) +} +fn hex(s: &str) -> bool { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct Surface { + id: String, + kind: String, + dimensions_m: [f64; 3], + transform: [f64; 16], +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct Node { + id: String, + role: String, + position_m: [f64; 3], +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct Geometry { + schema: String, + installation_id: String, + room_revision: u64, + room_binding_digest: String, + coordinate_frame_id: String, + source_scan_id: String, + captured_at_unix_ms: u64, + exported_at_unix_ms: u64, + provenance: String, + calibration_digest: Option, + surfaces: Vec, + nodes: Vec, +} +impl Geometry { + fn valid(&self, time: u64) -> bool { + self.schema == "ruview.companion.spatial-artifact.v1" + && id(&self.installation_id) + && self.room_revision > 0 + && self.room_revision <= 9_007_199_254_740_991 + && hex(&self.room_binding_digest) + && id(&self.coordinate_frame_id) + && id(&self.source_scan_id) + && self.captured_at_unix_ms > 0 + && self.captured_at_unix_ms <= self.exported_at_unix_ms + && self.exported_at_unix_ms <= time.saturating_add(5000) + && time <= self.exported_at_unix_ms.saturating_add(900_000) + && self.provenance == "historical_geometry_only" + && self.calibration_digest.as_deref().is_none_or(hex) + && !self.surfaces.is_empty() + && self.surfaces.len() <= 128 + && self.nodes.len() <= 64 + && self.surfaces.iter().all(|s| { + id(&s.id) + && ["wall", "door", "window", "opening", "floor"].contains(&s.kind.as_str()) + && s.dimensions_m + .iter() + .all(|v| v.is_finite() && *v >= 0. && *v <= 1000.) + && s.transform + .iter() + .all(|v| v.is_finite() && v.abs() <= 1000.) + && rigid(&s.transform) + }) + && self.nodes.iter().all(|n| { + id(&n.id) + && ["esp32_csi_receiver", "wifi_transmitter", "reference"] + .contains(&n.role.as_str()) + && n.position_m + .iter() + .all(|v| v.is_finite() && v.abs() <= 1000.) + }) + && self + .surfaces + .iter() + .map(|s| &s.id) + .collect::>() + .len() + == self.surfaces.len() + && self + .nodes + .iter() + .map(|s| &s.id) + .collect::>() + .len() + == self.nodes.len() + } +} +fn rigid(m: &[f64; 16]) -> bool { + if m[3].abs() > 1e-4 || m[7].abs() > 1e-4 || m[11].abs() > 1e-4 || (m[15] - 1.).abs() > 1e-4 { + return false; + } + for a in 0..3 { + for b in 0..3 { + let dot = (0..3).map(|r| m[a * 4 + r] * m[b * 4 + r]).sum::(); + if (dot - if a == b { 1. } else { 0. }).abs() > 0.015 { + return false; + } + } + } + if m[12..15].iter().any(|v| v.abs() > 30.) { + return false; + } + let det = m[0] * (m[5] * m[10] - m[9] * m[6]) - m[4] * (m[1] * m[10] - m[9] * m[2]) + + m[8] * (m[1] * m[6] - m[5] * m[2]); + (det - 1.).abs() < 0.02 +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct Artifact { + digest: String, + payload: String, + publisher_device_id: String, + received_at_unix_ms: u64, + accepted_at_unix_ms: Option, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct Saved { + schema: String, + owner_scope: String, + installation_id: String, + accepted: Vec, +} +struct Receipt { + nonce: String, + until: u64, + pending: Option, + revoked: tokio::sync::watch::Receiver, +} +struct Inner { + saved: Saved, + receipts: HashMap, +} +pub struct SpatialStore { + fault: AtomicBool, + path: PathBuf, + service: Arc, + room: Arc, + inner: Mutex, +} +type Shared = Arc; +impl SpatialStore { + pub fn load( + dir: &Path, + scope: &str, + installation: &str, + service: Arc, + room: Arc, + ) -> std::io::Result { + if !hex(scope) { + return Err(std::io::Error::other( + "spatial handoff requires account ownership", + )); + } + let path = dir.join("accepted-spatial-artifacts.json"); + let saved = match (|| -> std::io::Result> { + let meta = fs::symlink_metadata(&path)?; + if !meta.is_file() || meta.file_type().is_symlink() || meta.len() > 2 * 1024 * 1024 { + return Err(std::io::Error::other("invalid spatial history file")); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if meta.permissions().mode() & 0o077 != 0 { + return Err(std::io::Error::other( + "spatial history permissions are not private", + )); + } + } + fs::read(&path) + })() { + Ok(bytes) => { + if bytes.len() > 2 * 1024 * 1024 { + return Err(std::io::Error::other("spatial history too large")); + } + let s: Saved = serde_json::from_slice(&bytes)?; + if s.schema != "ruview.companion.spatial-history.v1" + || s.owner_scope != scope + || s.installation_id != installation + || s.accepted.len() > 32 + || s.accepted.iter().any(|a| { + a.payload.len() > MAX_PAYLOAD + || hash(&a.payload) != a.digest + || a.accepted_at_unix_ms.is_none() + }) + { + return Err(std::io::Error::other( + "invalid spatial history ownership or digest", + )); + } + for a in &s.accepted { + let g: Geometry = serde_json::from_str(&a.payload)?; + if !g.valid(g.exported_at_unix_ms) || g.installation_id != installation { + return Err(std::io::Error::other("invalid spatial geometry")); + } + } + s + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Saved { + schema: "ruview.companion.spatial-history.v1".into(), + owner_scope: scope.into(), + installation_id: installation.into(), + accepted: vec![], + }, + Err(e) => return Err(e), + }; + Ok(Arc::new(Self { + fault: AtomicBool::new(false), + path, + service, + room, + inner: Mutex::new(Inner { + saved, + receipts: HashMap::new(), + }), + })) + } + fn persist(&self, s: &Saved) -> std::io::Result<()> { + let tmp = self + .path + .with_extension(format!("{}.tmp", rand::random::())); + let mut options = fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + options.mode(0o600); + let mut file = options.open(&tmp)?; + file.write_all(&serde_json::to_vec(s)?)?; + file.sync_all()?; + fs::rename(tmp, &self.path)?; + fs::File::open(self.path.parent().unwrap())?.sync_all()?; + Ok(()) + } + fn matches(&self, g: &Geometry) -> bool { + let r = self.room.snapshot(); + r.is_some_and(|r| { + r.installation_id == g.installation_id + && r.revision == g.room_revision + && r.binding_digest.as_deref() == Some(&g.room_binding_digest) + }) + } +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct Upload { + nonce: String, + digest: String, + payload: String, +} +#[derive(Deserialize)] +#[serde(tag = "action", deny_unknown_fields, rename_all = "camelCase")] +enum Action { + Receive { + #[serde(rename = "deviceId")] + device_id: String, + }, + Accept { + digest: String, + }, + Reject { + digest: String, + }, +} +fn active(r: &Receipt, t: u64) -> bool { + r.until > t && !*r.revoked.borrow() +} +async fn snapshot(State(s): State) -> Result, StatusCode> { + if s.fault.load(Ordering::SeqCst) { + return Err(StatusCode::SERVICE_UNAVAILABLE); + } + let inner = s + .inner + .lock() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let t = now(); + let pending: Vec<_> = inner + .receipts + .values() + .filter(|r| active(r, t)) + .filter_map(|r| r.pending.as_ref()) + .collect(); + Ok(Json( + serde_json::json!({"schema":"ruview.companion.spatial-inbox.v1","pending":pending,"accepted":inner.saved.accepted}), + )) +} +async fn ticket( + State(s): State, + Extension(g): Extension, +) -> Result, StatusCode> { + if s.fault.load(Ordering::SeqCst) { + return Err(StatusCode::SERVICE_UNAVAILABLE); + } + let inner = s + .inner + .lock() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let t = now(); + if *g.revoked.borrow() || g.expires_at * 1000 <= t { + return Err(StatusCode::FORBIDDEN); + } + let r = inner + .receipts + .get(&g.device_id) + .filter(|r| active(r, t) && r.pending.is_none()) + .ok_or(StatusCode::FORBIDDEN)?; + Ok(Json(serde_json::json!({"nonce":r.nonce,"until":r.until}))) +} +async fn upload( + State(s): State, + Extension(g): Extension, + Json(u): Json, +) -> Result, StatusCode> { + if s.fault.load(Ordering::SeqCst) { + return Err(StatusCode::SERVICE_UNAVAILABLE); + } + let t = now(); + if *g.revoked.borrow() || g.expires_at * 1000 <= t { + return Err(StatusCode::FORBIDDEN); + } + if u.payload.len() > MAX_PAYLOAD || !hex(&u.digest) || hash(&u.payload) != u.digest { + return Err(StatusCode::BAD_REQUEST); + } + let raw: serde_json::Value = + serde_json::from_str(&u.payload).map_err(|_| StatusCode::BAD_REQUEST)?; + if raw.get("calibrationDigest").is_none() { + return Err(StatusCode::BAD_REQUEST); + } + let geometry: Geometry = + serde_json::from_str(&u.payload).map_err(|_| StatusCode::BAD_REQUEST)?; + if !geometry.valid(t) || !s.matches(&geometry) { + return Err(StatusCode::CONFLICT); + } + let mut i = s + .inner + .lock() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + if i.saved.accepted.iter().any(|a| a.digest == u.digest) { + return Err(StatusCode::CONFLICT); + } + let r = i + .receipts + .get_mut(&g.device_id) + .filter(|r| active(r, t) && r.nonce == u.nonce && r.pending.is_none()) + .ok_or(StatusCode::FORBIDDEN)?; + r.pending = Some(Artifact { + digest: u.digest.clone(), + payload: u.payload, + publisher_device_id: g.device_id, + received_at_unix_ms: t, + accepted_at_unix_ms: None, + }); + Ok(Json( + serde_json::json!({"digest":u.digest,"state":"pending_review"}), + )) +} +async fn action( + State(s): State, + Json(a): Json, +) -> Result, StatusCode> { + if s.fault.load(Ordering::SeqCst) { + return Err(StatusCode::SERVICE_UNAVAILABLE); + } + let t = now(); + let mut i = s + .inner + .lock() + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + let accept = matches!(&a, Action::Accept { .. }); + match a { + Action::Receive { device_id } => { + let (_, expiry) = s + .service + .paired_device(&device_id) + .ok_or(StatusCode::FORBIDDEN)?; + let revoked = s + .service + .device_revocation(&device_id) + .ok_or(StatusCode::FORBIDDEN)?; + i.receipts.retain(|_, r| active(r, t)); + if i.receipts.len() >= 256 || i.saved.accepted.len() >= 32 { + return Err(StatusCode::CONFLICT); + } + let until = (t + 900_000).min(expiry * 1000); + if until <= t { + return Err(StatusCode::FORBIDDEN); + } + i.receipts.insert( + device_id, + Receipt { + nonce: format!("{:032x}", rand::random::()), + until, + pending: None, + revoked, + }, + ); + } + Action::Accept { digest } | Action::Reject { digest } => { + let (device, r) = i + .receipts + .iter() + .find(|(_, r)| { + active(r, t) && r.pending.as_ref().is_some_and(|p| p.digest == digest) + }) + .ok_or(StatusCode::CONFLICT)?; + let device = device.clone(); + let mut artifact = r.pending.clone().unwrap(); + if accept { + let geometry: Geometry = + serde_json::from_str(&artifact.payload).map_err(|_| StatusCode::BAD_REQUEST)?; + if !s.matches(&geometry) || i.saved.accepted.len() >= 32 { + return Err(StatusCode::CONFLICT); + } + artifact.accepted_at_unix_ms = Some(t); + i.saved.accepted.push(artifact); + if s.persist(&i.saved).is_err() { + s.fault.store(true, Ordering::SeqCst); + i.saved.accepted.pop(); + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } + } + i.receipts.remove(&device); + } + } + Ok(Json(serde_json::json!({"state":"ok"}))) +} +pub fn owner_router(s: Shared) -> Router { + Router::new() + .route("/api/v1/companion/spatial", get(snapshot).post(action)) + .with_state(s) +} +pub fn peer_router(s: Shared) -> Router { + Router::new() + .route("/companion/v1/spatial", get(ticket).post(upload)) + .with_state(s) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use tower::ServiceExt; + async fn setup() -> (tempfile::TempDir, Shared, Grant) { + let d = tempfile::tempdir().unwrap(); + fs::write(d.path().join("companion-grants.json"),serde_json::to_vec(&json!({"version":1,"installation_id":"test","devices":[{"device_id":"phone","device_name":"Phone","pairing_hash":"a".repeat(64),"expires_at":now()/1000+3600}]})).unwrap()).unwrap(); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions( + d.path().join("companion-grants.json"), + fs::Permissions::from_mode(0o600), + ) + .unwrap(); + } + let c = Companion::load( + d.path(), + "test".into(), + "a".repeat(64), + "https://test.local:3443".into(), + ) + .unwrap(); + let room = SharedRoomStore::load(d.path(), "test").unwrap(); + let body = json!({"expected_revision":0,"room":{"schema":"ruview.companion.room.v1","id":"room-1","name":"Room","widthM":2.5,"lengthM":2.5,"heightM":2.4,"nodes":[{"nodeId":15,"slot":"north_west","xM":0,"yM":0,"role":"receiver","headingDeg":90,"orientationConfirmed":true,"confirmedAtUnixMs":1}],"accessPoint":{"slot":"south_east","xM":2.5,"yM":2.5,"provenance":"operator_entered","confirmedAtUnixMs":1}}}); + let r = super::super::companion_room::owner_router(room.clone()) + .oneshot( + axum::http::Request::builder() + .method("PUT") + .uri("/api/v1/companion/room") + .header("content-type", "application/json") + .body(axum::body::Body::from(body.to_string())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(r.status(), StatusCode::OK); + let grant = Grant { + device_id: "phone".into(), + expires_at: now() / 1000 + 3600, + revoked: c.device_revocation("phone").unwrap(), + }; + let s = SpatialStore::load(d.path(), &"a".repeat(64), "test", c, room).unwrap(); + (d, s, grant) + } + fn payload(s: &Shared) -> String { + json!({"schema":"ruview.companion.spatial-artifact.v1","installationId":"test","roomRevision":1,"roomBindingDigest":s.room.snapshot().unwrap().binding_digest,"coordinateFrameId":"frame","sourceScanId":"scan","capturedAtUnixMs":1,"exportedAtUnixMs":now(),"provenance":"historical_geometry_only","calibrationDigest":null,"surfaces":[{"id":"wall-1","kind":"wall","dimensionsM":[2.5,2.4,0.1],"transform":[1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1]}],"nodes":[]}).to_string() + } + async fn allow(s: &Shared, g: &Grant) -> String { + let _ = action( + State(s.clone()), + Json(Action::Receive { + device_id: g.device_id.clone(), + }), + ) + .await + .unwrap(); + ticket(State(s.clone()), Extension(g.clone())) + .await + .unwrap() + .0["nonce"] + .as_str() + .unwrap() + .into() + } + #[tokio::test] + async fn explicit_review_retains_across_restart_and_rejects_other_owner() { + let (d, s, g) = setup().await; + let nonce = allow(&s, &g).await; + let p = payload(&s); + let digest = hash(&p); + let _ = upload( + State(s.clone()), + Extension(g.clone()), + Json(Upload { + nonce: nonce.clone(), + digest: digest.clone(), + payload: p.clone(), + }), + ) + .await + .unwrap(); + assert!(!s.path.exists()); + assert_eq!( + snapshot(State(s.clone())).await.unwrap().0["pending"] + .as_array() + .unwrap() + .len(), + 1 + ); + assert!(upload( + State(s.clone()), + Extension(g), + Json(Upload { + nonce, + digest: digest.clone(), + payload: p + }) + ) + .await + .is_err()); + let _ = action( + State(s.clone()), + Json(Action::Accept { + digest: digest.clone(), + }), + ) + .await + .unwrap(); + let restored = SpatialStore::load( + d.path(), + &"a".repeat(64), + "test", + s.service.clone(), + s.room.clone(), + ) + .unwrap(); + assert_eq!( + snapshot(State(restored)).await.unwrap().0["accepted"][0]["digest"], + digest + ); + assert!(SpatialStore::load( + d.path(), + &"b".repeat(64), + "test", + s.service.clone(), + s.room.clone() + ) + .is_err()); + assert!(action(State(s), Json(Action::Accept { digest })) + .await + .is_err()); + } + #[tokio::test] + async fn rejects_expiry_wrong_device_nonce_binding_digest_and_authority() { + let (_d, s, g) = setup().await; + let nonce = allow(&s, &g).await; + let p = payload(&s); + for mode in [ + "wrong-device", + "nonce", + "binding", + "integrity", + "authority", + "reflection", + "unknown", + "expired", + "missing-null", + ] { + let mut g = g.clone(); + let mut v: serde_json::Value = serde_json::from_str(&p).unwrap(); + let mut n = nonce.clone(); + match mode { + "wrong-device" => g.device_id = "other".into(), + "nonce" => n = "wrong".into(), + "binding" => v["roomRevision"] = json!(2), + "authority" => v["provenance"] = json!("current_alignment"), + "reflection" => v["surfaces"][0]["transform"][0] = json!(-1), + "unknown" => v["cameraImage"] = json!("raw"), + "expired" => g.expires_at = 0, + "missing-null" => { + v.as_object_mut().unwrap().remove("calibrationDigest"); + } + _ => {} + } + let p = v.to_string(); + let digest = if mode == "integrity" { + "0".repeat(64) + } else { + hash(&p) + }; + assert!( + upload( + State(s.clone()), + Extension(g), + Json(Upload { + nonce: n, + digest, + payload: p + }) + ) + .await + .is_err(), + "{mode}" + ); + } + s.inner + .lock() + .unwrap() + .receipts + .get_mut("phone") + .unwrap() + .until = 0; + assert!(ticket(State(s), Extension(g)).await.is_err()); + } + #[tokio::test] + async fn revocation_during_review_removes_pending_authority() { + let (_d, s, g) = setup().await; + let nonce = allow(&s, &g).await; + let p = payload(&s); + let digest = hash(&p); + let _ = upload( + State(s.clone()), + Extension(g.clone()), + Json(Upload { + nonce, + digest: digest.clone(), + payload: p, + }), + ) + .await + .unwrap(); + let response = wifi_densepose_sensing_server::companion::owner_router(s.service.clone()) + .oneshot( + axum::http::Request::builder() + .method("DELETE") + .uri("/api/v1/companion/devices/phone") + .body(axum::body::Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert!(action(State(s.clone()), Json(Action::Accept { digest })) + .await + .is_err()); + assert_eq!(snapshot(State(s)).await.unwrap().0["pending"], json!([])); + } + #[tokio::test] + async fn uncertain_disk_failure_never_claims_acceptance_or_allows_more_writes() { + let (_d, s, g) = setup().await; + let nonce = allow(&s, &g).await; + let p = payload(&s); + let digest = hash(&p); + let _ = upload( + State(s.clone()), + Extension(g), + Json(Upload { + nonce, + digest: digest.clone(), + payload: p, + }), + ) + .await + .unwrap(); + fs::create_dir(&s.path).unwrap(); + assert!(action(State(s.clone()), Json(Action::Accept { digest })) + .await + .is_err()); + assert_eq!( + snapshot(State(s.clone())).await.unwrap_err(), + StatusCode::SERVICE_UNAVAILABLE + ); + assert!(s.inner.lock().unwrap().saved.accepted.is_empty()); + } + #[tokio::test] + async fn unknown_actions_are_rejected_by_real_router() { + let (_d, s, _g) = setup().await; + for body in [ + json!({"action":"receive","deviceId":"unknown"}).to_string(), + json!({"action":"accept","digest":"a".repeat(64),"promote":true}).to_string(), + ] { + let response = owner_router(s.clone()) + .oneshot( + axum::http::Request::builder() + .method("POST") + .uri("/api/v1/companion/spatial") + .header("content-type", "application/json") + .body(axum::body::Body::from(body)) + .unwrap(), + ) + .await + .unwrap(); + assert!(!response.status().is_success()); + } + } +} + +#[cfg(test)] +mod hostile_storage_tests { + use super::*; + #[test] + fn rejects_non_private_and_symlinked_history_before_reading() { + let d = tempfile::tempdir().unwrap(); + let c = Companion::load( + d.path(), + "test".into(), + "a".repeat(64), + "https://test.local:3443".into(), + ) + .unwrap(); + let room = SharedRoomStore::load(d.path(), "test").unwrap(); + let path = d.path().join("accepted-spatial-artifacts.json"); + fs::write(&path, b"{}").unwrap(); + assert!( + SpatialStore::load(d.path(), &"a".repeat(64), "test", c.clone(), room.clone()).is_err() + ); + #[cfg(unix)] + { + fs::remove_file(&path).unwrap(); + std::os::unix::fs::symlink("missing", &path).unwrap(); + assert!(SpatialStore::load(d.path(), &"a".repeat(64), "test", c, room).is_err()); + } + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index da6b7544b8..6eb538a969 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -12,6 +12,7 @@ mod adaptive_classifier; mod companion_integration; mod companion_room; +mod companion_spatial; pub mod cli; pub mod csi; mod engine_bridge; From 89dd87f2cb8fc9c4a3753c0e0848de90cc0ab682 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Fri, 30 Oct 2026 14:22:26 +0100 Subject: [PATCH 022/100] fix(companion): bound upload reads before taking calibration lock --- .../src/companion_integration.rs | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs index afd277acc3..b05ab62ebc 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs @@ -179,6 +179,17 @@ async fn serialize_control( if !publish && !start && !artifact { return next.run(request).await; } + // Do not let a permitted phone hold the shared calibration lock while + // streaming an unfinished body. Bound and time-limit it before locking. + let request = if artifact { + let (parts, body) = request.into_parts(); + let bytes = match tokio::time::timeout(std::time::Duration::from_secs(5), axum::body::to_bytes(body, 64 * 1024)).await { + Ok(Ok(bytes)) => bytes, + Ok(Err(_)) => return StatusCode::PAYLOAD_TOO_LARGE.into_response(), + Err(_) => return StatusCode::REQUEST_TIMEOUT.into_response(), + }; + Request::from_parts(parts, axum::body::Body::from(bytes)) + } else { request }; let _guard = control.serial.lock().await; if publish { let state = control.state.read().await; @@ -366,6 +377,23 @@ mod tests { assert_eq!(policy(&Method::GET, "/ws/train/progress"), None); } #[tokio::test] + async fn unfinished_spatial_upload_does_not_hold_calibration_lock() { + use tower::ServiceExt; + use axum::body::Body; + let serial=Arc::new(tokio::sync::Mutex::new(())); + let control=Control{state:Arc::new(RwLock::new(AppStateInner::minimal())),serial:serial.clone()}; + let router=Router::new().route("/companion/v1/spatial",post(||async{StatusCode::OK})) + .layer(axum::middleware::from_fn_with_state(control,serialize_control)); + let stream=futures_util::stream::pending::>(); + let request=Request::builder().method("POST").uri("/companion/v1/spatial").body(Body::from_stream(stream)).unwrap(); + let pending=tokio::spawn(router.clone().oneshot(request)); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + assert!(serial.try_lock().is_ok()); + assert_eq!(pending.await.unwrap().unwrap().status(),StatusCode::REQUEST_TIMEOUT); + let huge=Request::builder().method("POST").uri("/companion/v1/spatial").body(Body::from(vec![b'x';65537])).unwrap(); + assert_eq!(router.oneshot(huge).await.unwrap().status(),StatusCode::PAYLOAD_TOO_LARGE); + } + #[tokio::test] async fn sensor_routes_preserve_owner_peer_and_companion_isolation() { use tower::ServiceExt; use axum::{body::Body, http::{Request, Method}}; From 157fc35cc8d55235753f0bbf9b89f89548b90ab8 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Fri, 30 Oct 2026 14:38:07 +0100 Subject: [PATCH 023/100] fix(deps): patch Wasmtime and libz audit findings for helper stack --- v2/Cargo.lock | 156 +++++++++--------- v2/Cargo.toml | 8 +- v2/crates/homecore-plugins/src/discovery.rs | 7 +- .../RUVIEW-PROVENANCE.md | 14 -- .../Cargo.toml | 2 +- .../Cargo.toml.orig | 0 .../LICENSE | 0 .../RUVIEW-PROVENANCE.md | 11 ++ .../build.rs | 0 .../src/config.rs | 0 .../src/config/tests.rs | 0 .../src/lib.rs | 0 .../src/tests.rs | 0 .../src/worker.rs | 0 .../src/worker/tests.rs | 0 .../src/worker/tests/system_time_stub.rs | 0 .../tests/cache_write_default_config.rs | 0 17 files changed, 99 insertions(+), 99 deletions(-) delete mode 100644 v2/vendor/security/wasmtime-internal-cache-36.0.19/RUVIEW-PROVENANCE.md rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/Cargo.toml (99%) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/Cargo.toml.orig (100%) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/LICENSE (100%) create mode 100644 v2/vendor/security/wasmtime-internal-cache-36.0.20/RUVIEW-PROVENANCE.md rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/build.rs (100%) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/src/config.rs (100%) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/src/config/tests.rs (100%) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/src/lib.rs (100%) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/src/tests.rs (100%) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/src/worker.rs (100%) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/src/worker/tests.rs (100%) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/src/worker/tests/system_time_stub.rs (100%) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/tests/cache_write_default_config.rs (100%) diff --git a/v2/Cargo.lock b/v2/Cargo.lock index 78d957deb6..8b33844379 100644 --- a/v2/Cargo.lock +++ b/v2/Cargo.lock @@ -1553,7 +1553,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -1791,36 +1791,36 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3741b9ad77e89c710586b2fcdd619baf28482d92b70d88cecb2a081dd912ae6" +checksum = "40d4018626c25bb153a72738b261bdfed97ad6b7f21b100d556c0b8457fa9b3b" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8243e149085051b23faebd352ef7031aeebc5f45e63aaefd63ee7f810c22c7df" +checksum = "c69b47b7ffbd42f8d162ee958886e685b374b102b3bd116b0470c911153428b4" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c538af3aee13eb6aa8a85d2ef85b4d0aa41623dae8b19f59bbacfd5d6c114920" +checksum = "6d1994514a2e498d42a7173804fb0618472c94176b72c54ef8d64acdada05d0f" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-bitset" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2292ea3964d7d751031da1a1261d64feb09e89bd4fbebf3c78f43f9c34c20a11" +checksum = "019bfe57e970cf56fd9ad9ea45e4290457eb1ceb9bea954646abbe9d657a4a61" dependencies = [ "serde", "serde_derive", @@ -1828,9 +1828,9 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8967a699a4982d02c32246a00f2aa4cee5bccfaaa042608d5e151cb99cba210e" +checksum = "397c0a3cc8a67f9f85dbe47fbba2c1e88430293b130f91f3c98411f6ae4e0e80" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -1855,9 +1855,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e258eeeb6903b4e91e314a3667ab2a1e2924738ef2df08bdb50355e38decc749" +checksum = "a5bc504c51980615d594a685367584d1fa95d36c32c14a15b0eb4b1d9cc0dc2e" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -1868,24 +1868,24 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d36ff0de30460c01dad152ca2097bcabef2f16c659b9ffbca2f67e9bbdb321" +checksum = "bc8ae2c52ca44f49719cd78e48480c076e115a490b4e51ae28a89535ec4d76df" [[package]] name = "cranelift-control" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acc880d91877efb6199031bda872726ca3287dc1e18ccf3f61c07bfdab62930b" +checksum = "d94159e930a391bb589c8c4832769c3e2f9c81d8f1cd1266083a33a1787d41a9" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f7af321c4b6e5be4729e4a5b1ed0fc78fc31695c1ecd3c9b45b91908bf3d0a3" +checksum = "5cc0257dc68c7d282dabc1b3c9effec3f5d00fff56a964a80e7c08af7a3aa4dc" dependencies = [ "cranelift-bitset", "serde", @@ -1894,9 +1894,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3463f1b26faac3491008cede8e9cdf7ceb83139143bd92ba86c14b0882aaf1bb" +checksum = "5a60aa092dbffc0b8e501845afc3f67a3c987d780d22dd302829aa9f0255738e" dependencies = [ "cranelift-codegen", "log", @@ -1906,15 +1906,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90b76554b329634c4241c5670a96058affca99ac3b33daa14fb221bd68c0038e" +checksum = "e62bea43dd7767887c6b7861021846b0f0a03d9cc72bafddd0827e79b5f5deb5" [[package]] name = "cranelift-native" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36e7c67bcfd30a8160187a6d52b2ec0cbe45698a9d61f12d1292bd24701f7ed2" +checksum = "f410a5e75d4758d0564106931a1809d50d17681385ea1686518d029593a06b2a" dependencies = [ "cranelift-codegen", "libc", @@ -1923,9 +1923,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ccca83c45ddfe0412b1b2e4fe509d28d1bdbc014e306574a848519741bebc36" +checksum = "d5f98d0e96c17d2654a95820c77403050754561f2b2cd737dbf59462c64896f3" [[package]] name = "crc" @@ -2887,7 +2887,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.2", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3286,7 +3286,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5481,7 +5481,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -5925,9 +5925,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.30" +version = "1.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f710a23e6dbf193214fd46ca56a9d6864e550abe86202184532ae7275e46de19" +checksum = "ff20308ecabd635a2dc1dd5834a86c462cca20efba1d5d3ddab1f8aea3455cb2" dependencies = [ "cc", "pkg-config", @@ -6711,7 +6711,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7264,7 +7264,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.45.0", ] [[package]] @@ -8022,7 +8022,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.117", @@ -8059,9 +8059,9 @@ dependencies = [ [[package]] name = "pulley-interpreter" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c956c7140270b72e3bf03381a51f2eb79c70fd6881fc17cd027fa7cd3f56ae5a" +checksum = "4f6a7699ba3c24456c4235f6578a5f182386a1a9230653e8783f165ea1085bbe" dependencies = [ "cranelift-bitset", "log", @@ -8071,9 +8071,9 @@ dependencies = [ [[package]] name = "pulley-macros" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74db420d7ce2bb03eed5b81c1f778bda5771b3fd818e2b496dfc3a2783ceb60e" +checksum = "1efdd17eb6ed2d844c29350fffb7230c550ddb0cbe93e1fd2b9cfcc345d77450" dependencies = [ "proc-macro2", "quote", @@ -8212,7 +8212,7 @@ dependencies = [ "once_cell", "socket2 0.6.5", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.52.0", ] [[package]] @@ -9075,7 +9075,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -9133,7 +9133,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -9154,7 +9154,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -11148,7 +11148,7 @@ dependencies = [ "getrandom 0.4.1", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -12514,9 +12514,9 @@ dependencies = [ [[package]] name = "wasmtime" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60e5f244a2c9d40afa9e9a607ee7f048c0648bb1c22cc28b130c5fc71e23d71d" +checksum = "a32143e651d97f1b8d749713d242f70a72c9a0d7e16fd498a7e5a3f5574ed10b" dependencies = [ "addr2line", "anyhow", @@ -12569,9 +12569,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e246cd5107aa406cfa4acf6942e864bd189ae8136bab853365147da131c9ca3" +checksum = "f0501ad47df09ae57fbb76e3bdc6513b1f6d9dd0eadf3a115bb39360798890c7" dependencies = [ "anyhow", "cpp_demangle", @@ -12596,16 +12596,16 @@ dependencies = [ [[package]] name = "wasmtime-internal-asm-macros" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5380a8106fea8bfedb7208bb11ced9e7488ee44394ca889ede73acd3bdfd9d0b" +checksum = "de4c70c3fb098b9b28b73f7fc2b0147c5624621fb017a5354cdedb660f273e07" dependencies = [ "cfg-if", ] [[package]] name = "wasmtime-internal-cache" -version = "36.0.19" +version = "36.0.20" dependencies = [ "anyhow", "base64 0.22.1", @@ -12626,9 +12626,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-macro" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8278aa2bb237788ac9e3a70c5bea3ea3de6b4f95fb003df303858941871b8c4e" +checksum = "28729d33f5731506bcfe774a4b9eaf9e7e713449e1e3d848748f7de9978bea53" dependencies = [ "anyhow", "proc-macro2", @@ -12641,15 +12641,15 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-util" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "838c652b6cafd82a432a3a2c7af5bea4c6957d218aafbe50018e0297486ab7cc" +checksum = "91abf6b751d0f135cab1e78448f402af00a0f3e0638be858d18cc93170ab1009" [[package]] name = "wasmtime-internal-cranelift" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7339fdd64a3fde574505adea23244e065d5b1de0e72c8dd87e150f5d4ab63a63" +checksum = "c3840e7bb2d2593f3178b13a6717f215ad63e233ef54b61892f3cee80582f098" dependencies = [ "anyhow", "cfg-if", @@ -12674,9 +12674,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-fiber" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f3afc72d3c72f71d4ebc72e83f479ba8ca24409c41d43e5e0228f0bf82b873e" +checksum = "458d24ab006050e9b5fcf79f2d83ccc236f69846685fb27a7bf4c50a1583f1ea" dependencies = [ "anyhow", "cc", @@ -12690,9 +12690,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-debug" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21d33adf2d1eae36df9b6b033a1d549c7553ea607f4f9a81f75d51627725c354" +checksum = "eca1ce583227e82e6370939f48395ae902acdcd4c91ca32ea1ae7abbc86232a7" dependencies = [ "cc", "object", @@ -12702,9 +12702,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bde2fcf14cc76084242c6fe792e49778b5a9008c7df28ca4fe21c35c69d5f55" +checksum = "5c110f07b222a8eb29d957eabf90dce38f41461f883099ab28dc50861913ba17" dependencies = [ "anyhow", "cfg-if", @@ -12714,24 +12714,24 @@ dependencies = [ [[package]] name = "wasmtime-internal-math" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd982249e933e730859f84c17254c7e95f38c3b0cdf81695bfc6a270e237852" +checksum = "a273dd4ec37457d435538c00b30f3ec9ae99e04a4f6ba89a737048e8de6095df" dependencies = [ "libm", ] [[package]] name = "wasmtime-internal-slab" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8280b359384cc412bddf0023193eeaffadcb0d7fbbf860f2b7e0580e8806a344" +checksum = "3d12c7b3ae65639ab90d8d29135348dfd16de6d96eb36ab5b63861f3e12cf8e0" [[package]] name = "wasmtime-internal-unwinder" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb43477c41769fec0d031d32cb3236195acfb1e6aa4efdf463e5fd001c23f6cc" +checksum = "6da6131cb87700e06ffb7a83cc933658f521176f04d6d73511f1ee60db58c515" dependencies = [ "anyhow", "cfg-if", @@ -12742,9 +12742,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-versioned-export-macros" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b33afac17819f2f767ea280b4b87a6c95458ef1ae43da410cfc5b26813ecb78f" +checksum = "71efc44d34bb86408d7f620d308dad19457f34543e5c44ace680bdbe2d819917" dependencies = [ "proc-macro2", "quote", @@ -12753,9 +12753,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-winch" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1757c8b0626385c93994c7a3c7a23ee5b469061465b601c1fdce71ff61b11c82" +checksum = "af4da8c46ddb215374f023c0e3ec8917185b878b3d9874fbfaaefdc2d2df38e0" dependencies = [ "anyhow", "cranelift-codegen", @@ -12770,9 +12770,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-wit-bindgen" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061dd5bae2e618b32a8ed2acda5953d07dd308ac0e3f7960afc62ccdf6df7c44" +checksum = "356a1ae3cfc0534a84df35d9df214a241ca454983c0d311f8169b1674b4979a3" dependencies = [ "anyhow", "bitflags 2.11.0", @@ -13675,7 +13675,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] @@ -13686,9 +13686,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winch-codegen" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69940a29fc6b01b0e6b141912b700c17eaf956d58d63bc0c6e733435caf020e4" +checksum = "09aff830750e48ca9828e9df5ba4589688f49f2748e4a1d43fd543d6a673d272" dependencies = [ "anyhow", "cranelift-assembler-x64", diff --git a/v2/Cargo.toml b/v2/Cargo.toml index 591942bdf4..cad9e28487 100644 --- a/v2/Cargo.toml +++ b/v2/Cargo.toml @@ -3,7 +3,7 @@ resolver = "2" members = [ # Checksum-verified dependency patches; keep their upstream tests in CI. "vendor/security/zip-0.6.6", - "vendor/security/wasmtime-internal-cache-36.0.19", + "vendor/security/wasmtime-internal-cache-36.0.20", "crates/wifi-densepose-core", "crates/wifi-densepose-physics", # ADR-323 bounded pose assessment/correction "crates/wifi-densepose-signal", @@ -275,8 +275,8 @@ wiremock = "0.5" # The transitive security floors live in Cargo.lock, which the gating # Security Scanning job audits: quinn-proto >=0.11.18 # (RUSTSEC-2026-0338/0339/0340/0341/0342), rustls >=0.23.46 -# (RUSTSEC-2026-0346/0347/0348/0349) and wasmtime >=36.0.19 -# (RUSTSEC-2026-0363/0379/0380). A plain `cargo update` can walk them backwards; +# (RUSTSEC-2026-0346/0347/0348/0349) and wasmtime >=36.0.20 +# (RUSTSEC-2026-0363/0379/0380/0424). A plain `cargo update` can walk them backwards; # keep the lockfile at or above these until the direct requirements # enforce them. # @@ -346,4 +346,4 @@ debug = true # See each directory's RUVIEW-PROVENANCE.md for archive digests and scope. [patch.crates-io] zip-legacy = { package = "zip", path = "vendor/security/zip-0.6.6" } -wasmtime-internal-cache = { path = "vendor/security/wasmtime-internal-cache-36.0.19" } +wasmtime-internal-cache = { path = "vendor/security/wasmtime-internal-cache-36.0.20" } diff --git a/v2/crates/homecore-plugins/src/discovery.rs b/v2/crates/homecore-plugins/src/discovery.rs index f8d134d4ae..558339a6a1 100644 --- a/v2/crates/homecore-plugins/src/discovery.rs +++ b/v2/crates/homecore-plugins/src/discovery.rs @@ -198,15 +198,18 @@ fn bounded_regular_file(path: &Path, maximum: u64, kind: &str) -> Result mod tests { use super::*; use std::time::{SystemTime, UNIX_EPOCH}; + use std::sync::atomic::{AtomicU64, Ordering}; + static NEXT_ROOT: AtomicU64 = AtomicU64::new(0); fn root() -> PathBuf { let path = std::env::temp_dir().join(format!( - "homecore-plugin-discovery-{}-{}", + "homecore-plugin-discovery-{}-{}-{}", std::process::id(), SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() - .as_nanos() + .as_nanos(), + NEXT_ROOT.fetch_add(1, Ordering::Relaxed) )); fs::create_dir(&path).unwrap(); path diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/RUVIEW-PROVENANCE.md b/v2/vendor/security/wasmtime-internal-cache-36.0.19/RUVIEW-PROVENANCE.md deleted file mode 100644 index 4d8e8ae6e1..0000000000 --- a/v2/vendor/security/wasmtime-internal-cache-36.0.19/RUVIEW-PROVENANCE.md +++ /dev/null @@ -1,14 +0,0 @@ -# Dependency compatibility patch - -Source: crates.io `wasmtime-internal-cache` 36.0.19. -Published archive SHA-256: `4108e1964b3a02b4b5604ea1a2918a0cbcab512e5de0ccf4c062fc6c06b2eb8e` (verified against the pre-patch workspace lockfile at `fefb0f918`). - -Upstream executable source code and tests are unchanged. Trailing whitespace -in upstream documentation/comments is normalized for clean repository diffs. -The normalized Cargo.toml changes only the -zstd dependency requirement from 0.13.0 to 0.14.3, retaining existing features. -Registry bookkeeping, CI files, and the nested lockfile are omitted. -The upstream license and tests are retained. This preserves the parent API -while using the fixed Zstandard implementation without vendoring ML frameworks. - -Remove this patch when the parent releases a compatible dependency update. diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/Cargo.toml b/v2/vendor/security/wasmtime-internal-cache-36.0.20/Cargo.toml similarity index 99% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/Cargo.toml rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/Cargo.toml index e3175bfe0f..9b86a5661b 100644 --- a/v2/vendor/security/wasmtime-internal-cache-36.0.19/Cargo.toml +++ b/v2/vendor/security/wasmtime-internal-cache-36.0.20/Cargo.toml @@ -13,7 +13,7 @@ edition = "2024" rust-version = "1.86.0" name = "wasmtime-internal-cache" -version = "36.0.19" +version = "36.0.20" authors = ["The Wasmtime Project Developers"] build = "build.rs" autolib = false diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/Cargo.toml.orig b/v2/vendor/security/wasmtime-internal-cache-36.0.20/Cargo.toml.orig similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/Cargo.toml.orig rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/Cargo.toml.orig diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/LICENSE b/v2/vendor/security/wasmtime-internal-cache-36.0.20/LICENSE similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/LICENSE rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/LICENSE diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.20/RUVIEW-PROVENANCE.md b/v2/vendor/security/wasmtime-internal-cache-36.0.20/RUVIEW-PROVENANCE.md new file mode 100644 index 0000000000..09c4526f4e --- /dev/null +++ b/v2/vendor/security/wasmtime-internal-cache-36.0.20/RUVIEW-PROVENANCE.md @@ -0,0 +1,11 @@ +# Dependency compatibility patch + +Source: crates.io `wasmtime-internal-cache`36.0.20. +Published archive SHA-256: `65f09c789d1bb2e7ab1618ac8336d783c1ff0babc38430c7f1940d01c546722b` (verified against the local crates.io index checksum). + +Executable source and tests are byte-identical to the36.0.20 archive; its source +also matches the previous36.0.19 patch. The normalized manifest changes only +zstd0.13.0 to0.14.3, preserving the existing fixed-codec patch. Registry metadata, +CI files and nested lockfile remain omitted. License/tests are retained. + +Remove this patch when upstream accepts the fixed Zstandard dependency. diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/build.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.20/build.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/build.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/build.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/src/config.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.20/src/config.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/src/config.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/src/config.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/src/config/tests.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.20/src/config/tests.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/src/config/tests.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/src/config/tests.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/src/lib.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.20/src/lib.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/src/lib.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/src/lib.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/src/tests.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.20/src/tests.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/src/tests.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/src/tests.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/src/worker.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.20/src/worker.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/src/worker.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/src/worker.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/src/worker/tests.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.20/src/worker/tests.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/src/worker/tests.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/src/worker/tests.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/src/worker/tests/system_time_stub.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.20/src/worker/tests/system_time_stub.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/src/worker/tests/system_time_stub.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/src/worker/tests/system_time_stub.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/tests/cache_write_default_config.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.20/tests/cache_write_default_config.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/tests/cache_write_default_config.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/tests/cache_write_default_config.rs From f965bb26e37f081eb86c921a504bce8621c36aba Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Fri, 30 Oct 2026 14:47:02 +0100 Subject: [PATCH 024/100] chore(deps): leave discovery fixture repair to existing PRs 2171 and 2172 --- v2/crates/homecore-plugins/src/discovery.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/v2/crates/homecore-plugins/src/discovery.rs b/v2/crates/homecore-plugins/src/discovery.rs index 558339a6a1..f8d134d4ae 100644 --- a/v2/crates/homecore-plugins/src/discovery.rs +++ b/v2/crates/homecore-plugins/src/discovery.rs @@ -198,18 +198,15 @@ fn bounded_regular_file(path: &Path, maximum: u64, kind: &str) -> Result mod tests { use super::*; use std::time::{SystemTime, UNIX_EPOCH}; - use std::sync::atomic::{AtomicU64, Ordering}; - static NEXT_ROOT: AtomicU64 = AtomicU64::new(0); fn root() -> PathBuf { let path = std::env::temp_dir().join(format!( - "homecore-plugin-discovery-{}-{}-{}", + "homecore-plugin-discovery-{}-{}", std::process::id(), SystemTime::now() .duration_since(UNIX_EPOCH) .unwrap() - .as_nanos(), - NEXT_ROOT.fetch_add(1, Ordering::Relaxed) + .as_nanos() )); fs::create_dir(&path).unwrap(); path From e63e6aced4f75b802762e35c0ed6e5434710373e Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Fri, 30 Oct 2026 15:13:43 +0100 Subject: [PATCH 025/100] fix(companion): avoid nested calibration locks in peer router composition --- .../src/companion_integration.rs | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs index b05ab62ebc..7ef55af8e1 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs @@ -138,11 +138,12 @@ impl Integration { )) } pub async fn start(self, app: Router) { - let app = app - .merge(companion_sensors::companion_router(self.sensors.clone())) - .merge(companion_room::read_router(self.room.clone())) - .merge(self.spatial.as_ref().map(|s| companion_spatial::peer_router(s.clone())).unwrap_or_default()) - .layer(axum::middleware::from_fn_with_state(self.control.clone(), serialize_control)) + let app = attach_peer_artifacts( + app.merge(companion_sensors::companion_router(self.sensors.clone())) + .merge(companion_room::read_router(self.room.clone())), + self.spatial.as_ref().map(|s| companion_spatial::peer_router(s.clone())).unwrap_or_default(), + self.control.clone(), + ) .layer(axum::middleware::from_fn_with_state( self.service.clone(), require_companion, @@ -158,6 +159,12 @@ impl Integration { }); } } +// Existing calibration routes already use protect_control. Layering their +// router a second time would reacquire the same non-reentrant mutex. +fn attach_peer_artifacts(app: Router, artifacts: Router, control: Control) -> Router { + app.merge(artifacts.layer(axum::middleware::from_fn_with_state(control, serialize_control))) +} + #[derive(Clone)] struct Control { state: SharedState, @@ -377,6 +384,21 @@ mod tests { assert_eq!(policy(&Method::GET, "/ws/train/progress"), None); } #[tokio::test] + async fn peer_composition_never_takes_the_calibration_lock_twice() { + use tower::ServiceExt; + use axum::body::Body; + let control=Control{state:Arc::new(RwLock::new(AppStateInner::minimal())),serial:Arc::new(tokio::sync::Mutex::new(()))}; + let existing=Router::new().route("/api/v1/calibration/start",post(||async{StatusCode::OK})) + .layer(axum::middleware::from_fn_with_state(control.clone(),serialize_control)); + let artifacts=Router::new().route("/companion/v1/spatial",post(||async{StatusCode::OK})); + let app=attach_peer_artifacts(existing,artifacts,control); + for path in ["/api/v1/calibration/start","/companion/v1/spatial"] { + let request=Request::builder().method("POST").uri(path).body(Body::from("{}")).unwrap(); + let response=tokio::time::timeout(std::time::Duration::from_secs(1),app.clone().oneshot(request)).await.expect("nested calibration lock").unwrap(); + assert_eq!(response.status(),StatusCode::OK); + } + } + #[tokio::test] async fn unfinished_spatial_upload_does_not_hold_calibration_lock() { use tower::ServiceExt; use axum::body::Body; From 9fd56ea2e2234c1d36e0d65d4a20c7745ee30767 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Sat, 31 Oct 2026 11:46:54 +0100 Subject: [PATCH 026/100] fix(calibration): restore validated full-room models across helper restarts --- ...DR-355-local-empty-room-bootstrap-prior.md | 11 + .../src/bootstrap_baseline.rs | 4 +- .../wifi-densepose-sensing-server/src/lib.rs | 1 + .../wifi-densepose-sensing-server/src/main.rs | 158 +++++++++++-- .../src/room_calibration.rs | 216 ++++++++++++++++++ 5 files changed, 371 insertions(+), 19 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_calibration.rs diff --git a/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md b/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md index db9a579ad5..bb2f8e2c18 100644 --- a/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md +++ b/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md @@ -205,3 +205,14 @@ the source set for the entire capture. After restart, a confirmed empty holdout may refine the restored boundary once. The stored snapshot must reject a second refinement and must continue to deny calibrated evidence and numeric vital authority. + + +## Proposed amendment: bounded full-room restoration (2026-10-31) + +The bootstrap-only contract above remains the fallback for existing or incomplete saved images. A separately integrity-checked `room-receipt-v1.json` companion may now preserve the completed full-room receipt, but only after the server's independent held-out promotion succeeds. It binds the exact bootstrap payload digest, installation identity, room digest, original model ID, sorted contributor set, original frame count and completion time. It contains no raw CSI, room names, geometry, credentials or live history; its size is capped at 64 KiB and writes are private/atomic. + +On restart, both files must validate together under the current account-owned data directory and installation ID. No original lifetime is renewed. The process generates a new boot/session receipt for the same saved model and original completion time. Every recorded contributor must deliver fresh current-boot CSI on the frozen grid, the scored source must rebuild a full runtime window, and sequence faults or missing/stale sources withhold calibrated publication. Clients must explicitly adopt the new receipt and perform their independent empty check again; historical client verification is never restored. Numeric Vitals still require their independent existing qualification gates. + +Reset, replacement capture and explicit downgrade to a bootstrap prior retire the companion receipt. A crash between the two writes leaves a digest mismatch and falls back to bootstrap-only, never a mismatched full model. Account erasure removes both through the owned helper data directory. Older saved priors cannot be upgraded from client history and require one new complete capture after installation of this change. Expiry/staleness policy (24/12 hours) is unchanged. + +Existing upstream PR #1529 was assessed before this amendment: it proposes aggregate FieldModel persistence but does not carry this installation/room/session authority boundary. The implementation reuses the reviewed snapshot/atomic-write infrastructure already present in ADR-355 rather than introducing a competing field-model format. Physical logout/relaunch acceptance remains required before claiming delivered restoration. diff --git a/v2/crates/wifi-densepose-sensing-server/src/bootstrap_baseline.rs b/v2/crates/wifi-densepose-sensing-server/src/bootstrap_baseline.rs index 326e2f5fa0..ec0450aba9 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/bootstrap_baseline.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/bootstrap_baseline.rs @@ -409,12 +409,12 @@ fn metadata(image: &BootstrapImageV2) -> BootstrapBaselineMetadata { } } -fn hex_sha256(bytes: &[u8]) -> String { +pub(crate) fn hex_sha256(bytes: &[u8]) -> String { let digest = Sha256::digest(bytes); digest.iter().map(|byte| format!("{byte:02x}")).collect() } -fn atomic_write_private(path: &Path, bytes: &[u8]) -> Result<(), BootstrapBaselineError> { +pub(crate) fn atomic_write_private(path: &Path, bytes: &[u8]) -> Result<(), BootstrapBaselineError> { let parent = path.parent().ok_or(BootstrapBaselineError::InvalidPath)?; fs::create_dir_all(parent)?; let file_name = path diff --git a/v2/crates/wifi-densepose-sensing-server/src/lib.rs b/v2/crates/wifi-densepose-sensing-server/src/lib.rs index 05e7b40501..ba062de795 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/lib.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/lib.rs @@ -10,6 +10,7 @@ pub mod bearer_auth; pub mod bootstrap_baseline; +pub mod room_calibration; pub mod browser_session; pub mod ws_ticket; pub mod cli; diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index 6eb538a969..aa36aba0a9 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -453,19 +453,7 @@ struct CalibrationGridBinding { evidence: Option, } -#[derive(Debug, Clone, Serialize)] -struct CalibrationModelReceipt { - schema: &'static str, - boot_epoch: String, - session_id: String, - model_id: String, - binding_digest: String, - source_node_ids: Vec, - frame_count: u64, - variance_explained: f64, - baseline_eigenvalue_count: usize, - completed_at_unix_ms: u64, -} +use wifi_densepose_sensing_server::room_calibration::{self, RoomReceipt as CalibrationModelReceipt}; /// Per-frame occupancy result bound to one immutable field-model calibration /// receipt. Absent whenever the model is stale, unavailable, or cannot score @@ -2233,6 +2221,8 @@ struct AppStateInner { calibration_observed_source_node_ids: std::collections::BTreeSet, /// Immutable evidence created when the active model is finalized. calibration_model_receipt: Option, + /// Requires current-boot evidence from every saved contributor before publication. + calibration_restored: bool, /// Immutable source and CSI symbol grid selected from recent header-only /// evidence for the current field model. calibration_grid_binding: Option, @@ -2569,6 +2559,37 @@ fn calibration_sequence_order(previous: Option, sequence: u32) -> Calibrati } impl AppStateInner { + fn restore_saved_room(&mut self) -> Result { + let Some((installation, metadata)) = self.installation_id.as_deref().zip(self.bootstrap_baseline.as_ref()) else { return Ok(false); }; + match room_calibration::load(&room_calibration::path_in(&self.data_dir), installation, metadata) { + Ok(receipt) => { + let [source_node_id] = metadata.source_node_ids.as_slice() else { return Ok(false); }; + self.calibration_grid_binding = Some(CalibrationGridBinding { + source_node_id: *source_node_id, grid: CsiGridKey::from_bootstrap_grid(metadata.source_grid), evidence: None, + }); + self.install_restored_room(receipt); + Ok(true) + } + Err(bootstrap_baseline::BootstrapBaselineError::Io(error)) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } + } + + fn install_restored_room(&mut self, mut receipt: CalibrationModelReceipt) { + receipt.boot_epoch = self.calibration_boot_epoch.clone(); + receipt.session_id = opaque_calibration_id("cal-session"); + self.calibration_model_id = Some(receipt.model_id.clone()); + self.calibration_session_id = Some(receipt.session_id.clone()); + self.calibration_binding_digest = Some(receipt.binding_digest.clone()); + self.calibration_source_node_ids = receipt.source_node_ids.iter().copied().collect(); + self.calibration_observed_source_node_ids.clear(); + self.clear_calibration_sequence_state(); + for node in self.node_states.values_mut() { node.clear_field_model_history(); } + self.calibration_model_receipt = Some(receipt); + self.calibration_restored = true; + self.bootstrap_baseline_active = false; + } + /// Advance the sensing tick and record the update for the measured /// stream rate (#2087). Returns the new tick. fn advance_tick(&mut self) -> u64 { @@ -2751,7 +2772,16 @@ impl AppStateInner { } fn explicit_calibration_fresh_at(&self, observed_at_unix_ms: u64) -> bool { - !self.bootstrap_baseline_active + (!self.calibration_restored || (!self.calibration_source_node_ids.is_empty() + && self.calibration_source_node_ids.iter().all(|id| self.node_states.get(id).is_some_and(|node| + self.calibration_observed_source_node_ids.contains(id) + && !self.calibration_sequence_fault_node_ids.contains(id) + && node.field_model_latest_seen.is_some_and(|seen| seen.elapsed() < ESP32_OFFLINE_TIMEOUT))) + && self.calibration_grid_binding.is_some_and(|binding| self.field_model.as_ref() + .and_then(|model| model.modes()).and_then(|modes| modes.baseline_runtime_window_size) + .filter(|size| *size > 0 && *size <= FRAME_HISTORY_CAPACITY) + .is_some_and(|size| self.field_model_holdout_ready(binding, size, std::time::Instant::now()))))) + && !self.bootstrap_baseline_active && self.field_model_status_at(observed_at_unix_ms) == Some(CalibrationStatus::Fresh) } @@ -2851,7 +2881,7 @@ impl AppStateInner { // A persisted bootstrap model has negative-only authority. Its only // allowed occupancy effect is the explicit empty-background suppression // in `bootstrap_empty_prior_applies`; it must never infer positive count. - if self.bootstrap_baseline_active { + if self.bootstrap_baseline_active || (self.calibration_restored && !self.explicit_calibration_fresh_at(observed_at_unix_ms)) { return score_to_person_count(self.smoothed_person_score, self.prev_person_count); } match self.field_model.as_ref() { @@ -3110,6 +3140,7 @@ impl AppStateInner { calibration_source_node_ids: std::collections::BTreeSet::new(), calibration_observed_source_node_ids: std::collections::BTreeSet::new(), calibration_model_receipt: None, + calibration_restored: false, calibration_grid_binding: None, calibration_last_sequences: HashMap::new(), calibration_reordered_packets: HashMap::new(), @@ -3203,7 +3234,7 @@ mod calibration_expiry_tests { state.calibration_model_id = Some("cal-model-test".to_string()); state.calibration_binding_digest = Some("ab".repeat(32)); state.calibration_model_receipt = Some(CalibrationModelReceipt { - schema: field_bridge::CALIBRATION_MODEL_RECEIPT_SCHEMA, + schema: field_bridge::CALIBRATION_MODEL_RECEIPT_SCHEMA.to_string(), boot_epoch: state.calibration_boot_epoch.clone(), session_id: "cal-session-test".to_string(), model_id: "cal-model-test".to_string(), @@ -3338,6 +3369,61 @@ mod calibration_expiry_tests { state } + #[test] + fn saved_room_roundtrips_through_real_restart_loader_with_new_identity() { + let dir = tempfile::tempdir().unwrap(); + let now = chrono::Utc::now().timestamp_millis().max(0) as u64; + let source = qualified_vital_state(now); + let mut receipt = source.calibration_model_receipt.clone().unwrap(); + receipt.frame_count = 18942; + let metadata = bootstrap_baseline::store(&bootstrap_baseline::path_in(dir.path()), "account-A-installation", &[5], + source.calibration_grid_binding.unwrap().grid.as_bootstrap_grid(), &receipt.model_id, now, + source.field_model.as_ref().unwrap()).unwrap(); + room_calibration::store(&room_calibration::path_in(dir.path()), "account-A-installation", &metadata, &receipt).unwrap(); + let (model, metadata) = bootstrap_baseline::load(&bootstrap_baseline::path_in(dir.path()), "account-A-installation", now + 1).unwrap(); + let mut restarted = AppStateInner::minimal(); + restarted.data_dir = dir.path().to_path_buf(); + restarted.installation_id = Some("account-A-installation".into()); + restarted.field_model = Some(model); + restarted.bootstrap_baseline = Some(metadata.clone()); + restarted.bootstrap_baseline_active = true; + assert!(restarted.restore_saved_room().unwrap()); + let current = restarted.calibration_model_receipt.as_ref().unwrap(); + assert_eq!(current.frame_count, 18942); + assert_eq!(current.completed_at_unix_ms, receipt.completed_at_unix_ms); + assert_ne!(current.boot_epoch, receipt.boot_epoch); + assert_ne!(current.session_id, receipt.session_id); + assert!(!restarted.explicit_calibration_fresh_at(now)); + restarted.installation_id = Some("account-B-installation".into()); + assert!(restarted.restore_saved_room().is_err()); + room_calibration::remove(dir.path()).unwrap(); + assert!(!restarted.restore_saved_room().unwrap()); + } + + #[test] + fn restored_room_requires_new_boot_source_evidence_and_never_renews_completion() { + let mut state = state_with_runtime_window(); + let old = state.calibration_model_receipt.clone().unwrap(); + state.install_restored_room(old.clone()); + let current = state.calibration_model_receipt.as_ref().unwrap(); + assert_eq!(current.completed_at_unix_ms, old.completed_at_unix_ms); + assert_eq!(current.model_id, old.model_id); + assert_ne!(current.session_id, old.session_id); + assert!(!state.explicit_calibration_fresh_at(1_500)); + state.calibration_observed_source_node_ids.insert(5); + assert!(!state.explicit_calibration_fresh_at(1_500)); + let baseline = state.field_model.as_ref().unwrap().modes().unwrap().baseline[0].clone(); + let node = state.node_states.get_mut(&5).unwrap(); + node.field_model_history = (0..50).map(|_| baseline.clone()).collect(); + node.field_model_latest_seen = Some(std::time::Instant::now()); + node.field_model_latest_sequence = Some(60); + assert!(state.explicit_calibration_fresh_at(1_500)); + state.node_states.get_mut(&5).unwrap().field_model_history.pop_front(); + assert!(!state.explicit_calibration_fresh_at(1_500)); + state.calibration_source_node_ids.insert(17); + assert!(!state.explicit_calibration_fresh_at(1_500)); + } + fn qualified_vital_state(observed: u64) -> AppStateInner { let mut state = state_with_runtime_window(); let mut snapshot = finalized_model(observed).export_snapshot().unwrap(); @@ -10223,6 +10309,9 @@ async fn calibration_recover_bootstrap( })); }; let binding = CalibrationGridBinding { source_node_id: *source_node_id, grid, evidence: Some(evidence) }; + if let Err(error) = room_calibration::remove(&s.data_dir) { + return error_response::internal_error_json("room calibration downgrade", error); + } s.clear_field_model_binding(); s.clear_calibration_sequence_state(); s.calibration_source_node_ids.clear(); @@ -10232,6 +10321,7 @@ async fn calibration_recover_bootstrap( s.calibration_session_id = None; s.calibration_binding_digest = None; s.calibration_model_receipt = None; + s.calibration_restored = false; s.field_model = Some(model); s.calibration_grid_binding = Some(binding); s.bootstrap_baseline = Some(metadata.clone()); @@ -10462,6 +10552,9 @@ async fn calibration_start( _ => {} // Stale/Expired/Uncalibrated — ok to recalibrate } } + if let Err(error) = room_calibration::remove(&s.data_dir) { + return error_response::internal_error_json("room calibration replacement", error); + } match FieldModel::new(field_bridge::single_link_config()) { Ok(fm) => { s.field_model = Some(fm); @@ -10471,6 +10564,7 @@ async fn calibration_start( s.calibration_source_node_ids.clear(); s.calibration_observed_source_node_ids.clear(); s.calibration_model_receipt = None; + s.calibration_restored = false; s.clear_field_model_binding(); s.clear_calibration_sequence_state(); s.calibration_source_node_ids @@ -10604,7 +10698,7 @@ async fn calibration_stop( let holdout_window_size = modes.baseline_runtime_window_size; let final_frame_count = fm.calibration_frame_count(); let receipt = CalibrationModelReceipt { - schema: field_bridge::CALIBRATION_MODEL_RECEIPT_SCHEMA, + schema: field_bridge::CALIBRATION_MODEL_RECEIPT_SCHEMA.to_string(), boot_epoch: s.calibration_boot_epoch.clone(), session_id: s.calibration_session_id.clone().expect("validated identity"), model_id: model_id.clone().expect("bound calibration model identity"), @@ -10615,6 +10709,7 @@ async fn calibration_stop( baseline_eigenvalue_count: baseline, completed_at_unix_ms: chrono::Utc::now().timestamp_millis().max(0) as u64, }; + s.calibration_restored = false; s.calibration_model_receipt = Some(receipt.clone()); s.begin_field_model_holdout(binding); info!("Field model calibrated: baseline_eigenvalues={baseline}, variance_explained={variance_explained:.2}"); @@ -10760,6 +10855,11 @@ async fn calibration_status(State(state): State) -> Json>() } else { Vec::new() }); response["observed_source_node_ids"] = serde_json::json!(if bound { s.calibration_observed_source_node_ids.iter().copied().collect::>() } else { Vec::new() }); @@ -11018,6 +11118,12 @@ async fn calibration_promote_bootstrap( return error_response::internal_error_json("bootstrap baseline store", error) } }; + let Some(receipt) = s.calibration_model_receipt.as_ref() else { + return error_response::internal_error_json("room calibration receipt", "missing receipt"); + }; + if let Err(error) = room_calibration::store(&room_calibration::path_in(&s.data_dir), &installation_id, &stored, receipt) { + return error_response::internal_error_json("room calibration store", error); + } s.bootstrap_baseline = Some(stored.clone()); info!( model_id = %stored.source_model_id, @@ -11308,6 +11414,9 @@ async fn calibration_cancel( })); } let cancelled_session_id = s.calibration_session_id.clone(); + if let Err(error) = room_calibration::remove(&s.data_dir) { + return error_response::internal_error_json("room calibration removal", error); + } s.field_model = None; s.calibration_model_id = None; s.calibration_session_id = None; @@ -11315,6 +11424,7 @@ async fn calibration_cancel( s.calibration_source_node_ids.clear(); s.calibration_observed_source_node_ids.clear(); s.calibration_model_receipt = None; + s.calibration_restored = false; s.clear_field_model_binding(); s.clear_calibration_sequence_state(); if let Some(installation_id) = s.installation_id.clone() { @@ -11365,6 +11475,9 @@ async fn calibration_reset( if !boot_only && !calibration_identity_matches(&s, &request) { return calibration_identity_error(); } + if let Err(error) = room_calibration::remove(&s.data_dir) { + return error_response::internal_error_json("room calibration removal", error); + } s.field_model = None; s.calibration_model_id = None; s.calibration_session_id = None; @@ -11372,6 +11485,7 @@ async fn calibration_reset( s.calibration_source_node_ids.clear(); s.calibration_observed_source_node_ids.clear(); s.calibration_model_receipt = None; + s.calibration_restored = false; s.clear_field_model_binding(); s.clear_calibration_sequence_state(); s.bootstrap_baseline_active = false; @@ -15771,6 +15885,7 @@ async fn main() { .unwrap_or_default(), calibration_observed_source_node_ids: std::collections::BTreeSet::new(), calibration_model_receipt: None, + calibration_restored: false, calibration_grid_binding: bootstrap_metadata.as_ref().and_then(|metadata| { let [source_node_id] = metadata.source_node_ids.as_slice() else { return None; @@ -15799,6 +15914,15 @@ async fn main() { .expect("default pose physics configuration is valid"), })); + { + let mut s = state.write().await; + match s.restore_saved_room() { + Ok(true) => info!("Restored saved room model; waiting for fresh current-boot source windows"), + Ok(false) => {}, + Err(error) => warn!(%error, "Ignored invalid full-room receipt; retaining bootstrap-only authority"), + } + } + // Start background tasks from the resolved plan (issue #1004). // // In `auto` mode with no boot source, `bind_udp` AND `run_simulator` are diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_calibration.rs b/v2/crates/wifi-densepose-sensing-server/src/room_calibration.rs new file mode 100644 index 0000000000..24e80d2937 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_calibration.rs @@ -0,0 +1,216 @@ +//! Full room receipt companion to a separately validated aggregate model. +//! No CSI/history is stored. A bootstrap image alone never gains this authority. +use crate::bootstrap_baseline::{self, BootstrapBaselineError, BootstrapBaselineMetadata}; +use serde::{Deserialize, Serialize}; +use serde_json::value::RawValue; +use std::{ + fs, + io::Read, + path::{Path, PathBuf}, +}; + +const MAX_BYTES: u64 = 65_536; +const SCHEMA: &str = "ruview.saved-room-calibration.v1"; + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RoomReceipt { + pub schema: String, + pub boot_epoch: String, + pub session_id: String, + pub model_id: String, + pub binding_digest: String, + pub source_node_ids: Vec, + pub frame_count: u64, + pub variance_explained: f64, + pub baseline_eigenvalue_count: usize, + pub completed_at_unix_ms: u64, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Payload { + schema: String, + installation_binding_sha256: String, + bootstrap_content_sha256: String, + receipt: RoomReceipt, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Envelope { + payload: Box, + content_sha256: String, +} +pub fn path_in(data_dir: &Path) -> PathBuf { + data_dir.join("calibration").join("room-receipt-v1.json") +} +fn validate( + payload: &Payload, + installation: &str, + model: &BootstrapBaselineMetadata, +) -> Result<(), BootstrapBaselineError> { + let r = &payload.receipt; + let valid_id = |id: &str, prefix: &str| { + id.starts_with(prefix) + && id.len() <= 80 + && id.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-') + }; + let digest = |value: &str| { + value.len() == 64 + && value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }; + if payload.schema != SCHEMA + || payload.installation_binding_sha256 + != bootstrap_baseline::installation_binding(installation)? + || payload.bootstrap_content_sha256 != model.content_sha256 + || r.schema != "ruview.calibration.field-model-receipt.v1" + || !valid_id(&r.boot_epoch, "cal-boot-") + || !valid_id(&r.session_id, "cal-session-") + || !valid_id(&r.model_id, "cal-model-") + || r.model_id != model.source_model_id + || !digest(&r.binding_digest) + || r.frame_count < 1000 + || !r.variance_explained.is_finite() + || !(0.0..=1.0).contains(&r.variance_explained) + || r.baseline_eigenvalue_count > 2048 + || r.source_node_ids.is_empty() + || r.source_node_ids.len() > 16 + || !r.source_node_ids.windows(2).all(|pair| pair[0] < pair[1]) + || model.source_node_ids.len() != 1 + || !r.source_node_ids.contains(&model.source_node_ids[0]) + || r.completed_at_unix_ms == 0 + || r.completed_at_unix_ms > model.created_at_unix_ms + || r.completed_at_unix_ms >= model.expires_at_unix_ms + { + return Err(BootstrapBaselineError::InvalidPath); + } + Ok(()) +} +pub fn store( + path: &Path, + installation: &str, + model: &BootstrapBaselineMetadata, + receipt: &RoomReceipt, +) -> Result<(), BootstrapBaselineError> { + let payload = Payload { + schema: SCHEMA.into(), + installation_binding_sha256: bootstrap_baseline::installation_binding(installation)?, + bootstrap_content_sha256: model.content_sha256.clone(), + receipt: receipt.clone(), + }; + validate(&payload, installation, model)?; + let raw = serde_json::value::to_raw_value(&payload)?; + let content_sha256 = bootstrap_baseline::hex_sha256(raw.get().as_bytes()); + let bytes = serde_json::to_vec(&Envelope { + payload: raw, + content_sha256, + })?; + if bytes.len() as u64 > MAX_BYTES { + return Err(BootstrapBaselineError::FileTooLarge); + } + bootstrap_baseline::atomic_write_private(path, &bytes) +} +pub fn load( + path: &Path, + installation: &str, + model: &BootstrapBaselineMetadata, +) -> Result { + let metadata = fs::symlink_metadata(path)?; + if metadata.file_type().is_symlink() || !metadata.is_file() { + return Err(BootstrapBaselineError::InvalidPath); + } + if metadata.len() > MAX_BYTES { + return Err(BootstrapBaselineError::FileTooLarge); + } + let mut bytes = Vec::new(); + fs::File::open(path)? + .take(MAX_BYTES + 1) + .read_to_end(&mut bytes)?; + if bytes.len() as u64 > MAX_BYTES { + return Err(BootstrapBaselineError::FileTooLarge); + } + let envelope: Envelope = serde_json::from_slice(&bytes)?; + if bootstrap_baseline::hex_sha256(envelope.payload.get().as_bytes()) != envelope.content_sha256 + { + return Err(BootstrapBaselineError::InvalidPath); + } + let payload: Payload = serde_json::from_str(envelope.payload.get())?; + validate(&payload, installation, model)?; + Ok(payload.receipt) +} +pub fn remove(data_dir: &Path) -> Result<(), BootstrapBaselineError> { + match fs::remove_file(path_in(data_dir)) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(e.into()), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::bootstrap_baseline::BootstrapCsiGrid; + fn fixture() -> (BootstrapBaselineMetadata, RoomReceipt) { + ( + BootstrapBaselineMetadata { + authority: "bootstrap_only", + source_node_ids: vec![15], + source_grid: BootstrapCsiGrid { + n_subcarriers: 56, + ppdu_type: 1, + }, + source_model_id: "cal-model-abc".into(), + created_at_unix_ms: 2000, + expires_at_unix_ms: 9000, + content_sha256: "ab".repeat(32), + }, + RoomReceipt { + schema: "ruview.calibration.field-model-receipt.v1".into(), + boot_epoch: "cal-boot-abc".into(), + session_id: "cal-session-abc".into(), + model_id: "cal-model-abc".into(), + binding_digest: "cd".repeat(32), + source_node_ids: vec![15, 16, 17], + frame_count: 18942, + variance_explained: 0.9800000000000001, + baseline_eigenvalue_count: 5, + completed_at_unix_ms: 1900, + }, + ) + } + #[test] + fn exact_roundtrip_and_foreign_owner_or_replaced_model_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = path_in(dir.path()); + let (metadata, receipt) = fixture(); + store(&path, "A", &metadata, &receipt).unwrap(); + let read = load(&path, "A", &metadata).unwrap(); + assert_eq!(read.source_node_ids, vec![15, 16, 17]); + assert_eq!(read.frame_count, 18942); + assert!(load(&path, "B", &metadata).is_err()); + let mut replaced = metadata.clone(); + replaced.content_sha256 = "ef".repeat(32); + assert!(load(&path, "A", &replaced).is_err()); + let mut changed = receipt; + changed.source_node_ids = vec![16, 17]; + assert!(store(&path, "A", &metadata, &changed).is_err()); + remove(dir.path()).unwrap(); + assert!(!path.exists()); + } + #[test] + fn corrupt_oversized_and_unfinished_receipts_refused() { + let dir = tempfile::tempdir().unwrap(); + let path = path_in(dir.path()); + let (metadata, mut receipt) = fixture(); + store(&path, "A", &metadata, &receipt).unwrap(); + let mut bytes = fs::read(&path).unwrap(); + bytes[20] ^= 1; + fs::write(&path, bytes).unwrap(); + assert!(load(&path, "A", &metadata).is_err()); + fs::write(&path, vec![b' '; MAX_BYTES as usize + 1]).unwrap(); + assert!(load(&path, "A", &metadata).is_err()); + receipt.frame_count = 999; + assert!(store(&path, "A", &metadata, &receipt).is_err()); + } +} From cda38f58d74100d4615019445f8c7f05c4c331f2 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Sat, 31 Oct 2026 17:04:59 +0100 Subject: [PATCH 027/100] docs(calibration): specify bounded same-model client reconnect --- docs/adr/ADR-355-local-empty-room-bootstrap-prior.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md b/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md index bb2f8e2c18..2cf45f6fc1 100644 --- a/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md +++ b/docs/adr/ADR-355-local-empty-room-bootstrap-prior.md @@ -211,7 +211,7 @@ calibrated evidence and numeric vital authority. The bootstrap-only contract above remains the fallback for existing or incomplete saved images. A separately integrity-checked `room-receipt-v1.json` companion may now preserve the completed full-room receipt, but only after the server's independent held-out promotion succeeds. It binds the exact bootstrap payload digest, installation identity, room digest, original model ID, sorted contributor set, original frame count and completion time. It contains no raw CSI, room names, geometry, credentials or live history; its size is capped at 64 KiB and writes are private/atomic. -On restart, both files must validate together under the current account-owned data directory and installation ID. No original lifetime is renewed. The process generates a new boot/session receipt for the same saved model and original completion time. Every recorded contributor must deliver fresh current-boot CSI on the frozen grid, the scored source must rebuild a full runtime window, and sequence faults or missing/stale sources withhold calibrated publication. Clients must explicitly adopt the new receipt and perform their independent empty check again; historical client verification is never restored. Numeric Vitals still require their independent existing qualification gates. +On restart, both files must validate together under the current account-owned data directory and installation ID. No original lifetime is renewed. The process generates a new boot/session receipt for the same saved model and original completion time. Every recorded contributor must deliver fresh current-boot CSI on the frozen grid, the scored source must rebuild a full runtime window, and sequence faults or missing/stale sources withhold calibrated publication. A client with its own previously saved and passed check may reconnect the unchanged model only after account/origin/room binding, every immutable model field, current contributor readiness and a fresh matching calibrated stream are verified. It retains the original check timestamp and original model lifetime; it does not claim a new empty observation. A different or previously unverified model still requires explicit adoption and a new independent check. Bootstrap-only images cannot gain full authority from client history. Numeric Vitals still require their independent existing qualification gates. Reset, replacement capture and explicit downgrade to a bootstrap prior retire the companion receipt. A crash between the two writes leaves a digest mismatch and falls back to bootstrap-only, never a mismatched full model. Account erasure removes both through the owned helper data directory. Older saved priors cannot be upgraded from client history and require one new complete capture after installation of this change. Expiry/staleness policy (24/12 hours) is unchanged. From 3dc368b2a62574ad295a6208a73c3d7ca2de0119 Mon Sep 17 00:00:00 2001 From: Joe Date: Sat, 31 Oct 2026 22:10:50 -0400 Subject: [PATCH 028/100] Bump wasmtime 36.0.19 -> 36.0.20 (RUSTSEC-2026-0424), re-vendor cache crate wasmtime 36.0.20 is the patched release (GHSA-6fr9-xmw9-5352). The workspace patches wasmtime-internal-cache to a vendored copy that raises zstd to 0.14.3, so a lock-only bump would have dropped that patch and pulled zstd 0.13.3 back in. Re-vendor wasmtime-internal-cache at 36.0.20 (archive checksum matches Cargo.lock; only the version and zstd requirement differ from upstream, and 36.0.20 source is otherwise identical to 36.0.19). cargo audit: 0 vulnerabilities. homecore-plugins --features wasmtime tests pass. Co-Authored-By: claude-flow --- v2/Cargo.lock | 152 +++++++++--------- v2/Cargo.toml | 4 +- .../Cargo.toml | 2 +- .../Cargo.toml.orig | 0 .../LICENSE | 1 + .../RUVIEW-PROVENANCE.md | 4 +- .../build.rs | 0 .../src/config.rs | 0 .../src/config/tests.rs | 0 .../src/lib.rs | 0 .../src/tests.rs | 0 .../src/worker.rs | 0 .../src/worker/tests.rs | 0 .../src/worker/tests/system_time_stub.rs | 0 .../tests/cache_write_default_config.rs | 0 15 files changed, 82 insertions(+), 81 deletions(-) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/Cargo.toml (99%) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/Cargo.toml.orig (100%) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/LICENSE (99%) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/RUVIEW-PROVENANCE.md (73%) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/build.rs (100%) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/src/config.rs (100%) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/src/config/tests.rs (100%) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/src/lib.rs (100%) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/src/tests.rs (100%) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/src/worker.rs (100%) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/src/worker/tests.rs (100%) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/src/worker/tests/system_time_stub.rs (100%) rename v2/vendor/security/{wasmtime-internal-cache-36.0.19 => wasmtime-internal-cache-36.0.20}/tests/cache_write_default_config.rs (100%) diff --git a/v2/Cargo.lock b/v2/Cargo.lock index 554a09fd17..39888ea77a 100644 --- a/v2/Cargo.lock +++ b/v2/Cargo.lock @@ -1553,7 +1553,7 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -1791,36 +1791,36 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3741b9ad77e89c710586b2fcdd619baf28482d92b70d88cecb2a081dd912ae6" +checksum = "40d4018626c25bb153a72738b261bdfed97ad6b7f21b100d556c0b8457fa9b3b" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8243e149085051b23faebd352ef7031aeebc5f45e63aaefd63ee7f810c22c7df" +checksum = "c69b47b7ffbd42f8d162ee958886e685b374b102b3bd116b0470c911153428b4" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c538af3aee13eb6aa8a85d2ef85b4d0aa41623dae8b19f59bbacfd5d6c114920" +checksum = "6d1994514a2e498d42a7173804fb0618472c94176b72c54ef8d64acdada05d0f" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-bitset" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2292ea3964d7d751031da1a1261d64feb09e89bd4fbebf3c78f43f9c34c20a11" +checksum = "019bfe57e970cf56fd9ad9ea45e4290457eb1ceb9bea954646abbe9d657a4a61" dependencies = [ "serde", "serde_derive", @@ -1828,9 +1828,9 @@ dependencies = [ [[package]] name = "cranelift-codegen" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8967a699a4982d02c32246a00f2aa4cee5bccfaaa042608d5e151cb99cba210e" +checksum = "397c0a3cc8a67f9f85dbe47fbba2c1e88430293b130f91f3c98411f6ae4e0e80" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -1855,9 +1855,9 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e258eeeb6903b4e91e314a3667ab2a1e2924738ef2df08bdb50355e38decc749" +checksum = "a5bc504c51980615d594a685367584d1fa95d36c32c14a15b0eb4b1d9cc0dc2e" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", @@ -1868,24 +1868,24 @@ dependencies = [ [[package]] name = "cranelift-codegen-shared" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1d36ff0de30460c01dad152ca2097bcabef2f16c659b9ffbca2f67e9bbdb321" +checksum = "bc8ae2c52ca44f49719cd78e48480c076e115a490b4e51ae28a89535ec4d76df" [[package]] name = "cranelift-control" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acc880d91877efb6199031bda872726ca3287dc1e18ccf3f61c07bfdab62930b" +checksum = "d94159e930a391bb589c8c4832769c3e2f9c81d8f1cd1266083a33a1787d41a9" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f7af321c4b6e5be4729e4a5b1ed0fc78fc31695c1ecd3c9b45b91908bf3d0a3" +checksum = "5cc0257dc68c7d282dabc1b3c9effec3f5d00fff56a964a80e7c08af7a3aa4dc" dependencies = [ "cranelift-bitset", "serde", @@ -1894,9 +1894,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3463f1b26faac3491008cede8e9cdf7ceb83139143bd92ba86c14b0882aaf1bb" +checksum = "5a60aa092dbffc0b8e501845afc3f67a3c987d780d22dd302829aa9f0255738e" dependencies = [ "cranelift-codegen", "log", @@ -1906,15 +1906,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90b76554b329634c4241c5670a96058affca99ac3b33daa14fb221bd68c0038e" +checksum = "e62bea43dd7767887c6b7861021846b0f0a03d9cc72bafddd0827e79b5f5deb5" [[package]] name = "cranelift-native" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36e7c67bcfd30a8160187a6d52b2ec0cbe45698a9d61f12d1292bd24701f7ed2" +checksum = "f410a5e75d4758d0564106931a1809d50d17681385ea1686518d029593a06b2a" dependencies = [ "cranelift-codegen", "libc", @@ -1923,9 +1923,9 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.123.19" +version = "0.123.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ccca83c45ddfe0412b1b2e4fe509d28d1bdbc014e306574a848519741bebc36" +checksum = "d5f98d0e96c17d2654a95820c77403050754561f2b2cd737dbf59462c64896f3" [[package]] name = "crc" @@ -2887,7 +2887,7 @@ dependencies = [ "libc", "option-ext", "redox_users 0.5.2", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3286,7 +3286,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5481,7 +5481,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6711,7 +6711,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7264,7 +7264,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7d8fae84b431384b68627d0f9b3b1245fcf9f46f6c0e3dc902e9dce64edd1967" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.45.0", ] [[package]] @@ -8022,7 +8022,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b570b25f7617e43d59005d0990ccb79e950a423952cea19671b7a876da390adf" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.10.5", "proc-macro2", "quote", "syn 2.0.117", @@ -8059,9 +8059,9 @@ dependencies = [ [[package]] name = "pulley-interpreter" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c956c7140270b72e3bf03381a51f2eb79c70fd6881fc17cd027fa7cd3f56ae5a" +checksum = "4f6a7699ba3c24456c4235f6578a5f182386a1a9230653e8783f165ea1085bbe" dependencies = [ "cranelift-bitset", "log", @@ -8071,9 +8071,9 @@ dependencies = [ [[package]] name = "pulley-macros" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74db420d7ce2bb03eed5b81c1f778bda5771b3fd818e2b496dfc3a2783ceb60e" +checksum = "1efdd17eb6ed2d844c29350fffb7230c550ddb0cbe93e1fd2b9cfcc345d77450" dependencies = [ "proc-macro2", "quote", @@ -8212,7 +8212,7 @@ dependencies = [ "once_cell", "socket2 0.6.5", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -9075,7 +9075,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -9133,7 +9133,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -9154,7 +9154,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -11148,7 +11148,7 @@ dependencies = [ "getrandom 0.4.1", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -12514,9 +12514,9 @@ dependencies = [ [[package]] name = "wasmtime" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60e5f244a2c9d40afa9e9a607ee7f048c0648bb1c22cc28b130c5fc71e23d71d" +checksum = "a32143e651d97f1b8d749713d242f70a72c9a0d7e16fd498a7e5a3f5574ed10b" dependencies = [ "addr2line", "anyhow", @@ -12569,9 +12569,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e246cd5107aa406cfa4acf6942e864bd189ae8136bab853365147da131c9ca3" +checksum = "f0501ad47df09ae57fbb76e3bdc6513b1f6d9dd0eadf3a115bb39360798890c7" dependencies = [ "anyhow", "cpp_demangle", @@ -12596,16 +12596,16 @@ dependencies = [ [[package]] name = "wasmtime-internal-asm-macros" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5380a8106fea8bfedb7208bb11ced9e7488ee44394ca889ede73acd3bdfd9d0b" +checksum = "de4c70c3fb098b9b28b73f7fc2b0147c5624621fb017a5354cdedb660f273e07" dependencies = [ "cfg-if", ] [[package]] name = "wasmtime-internal-cache" -version = "36.0.19" +version = "36.0.20" dependencies = [ "anyhow", "base64 0.22.1", @@ -12626,9 +12626,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-macro" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8278aa2bb237788ac9e3a70c5bea3ea3de6b4f95fb003df303858941871b8c4e" +checksum = "28729d33f5731506bcfe774a4b9eaf9e7e713449e1e3d848748f7de9978bea53" dependencies = [ "anyhow", "proc-macro2", @@ -12641,15 +12641,15 @@ dependencies = [ [[package]] name = "wasmtime-internal-component-util" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "838c652b6cafd82a432a3a2c7af5bea4c6957d218aafbe50018e0297486ab7cc" +checksum = "91abf6b751d0f135cab1e78448f402af00a0f3e0638be858d18cc93170ab1009" [[package]] name = "wasmtime-internal-cranelift" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7339fdd64a3fde574505adea23244e065d5b1de0e72c8dd87e150f5d4ab63a63" +checksum = "c3840e7bb2d2593f3178b13a6717f215ad63e233ef54b61892f3cee80582f098" dependencies = [ "anyhow", "cfg-if", @@ -12674,9 +12674,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-fiber" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f3afc72d3c72f71d4ebc72e83f479ba8ca24409c41d43e5e0228f0bf82b873e" +checksum = "458d24ab006050e9b5fcf79f2d83ccc236f69846685fb27a7bf4c50a1583f1ea" dependencies = [ "anyhow", "cc", @@ -12690,9 +12690,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-debug" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21d33adf2d1eae36df9b6b033a1d549c7553ea607f4f9a81f75d51627725c354" +checksum = "eca1ce583227e82e6370939f48395ae902acdcd4c91ca32ea1ae7abbc86232a7" dependencies = [ "cc", "object", @@ -12702,9 +12702,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-jit-icache-coherence" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bde2fcf14cc76084242c6fe792e49778b5a9008c7df28ca4fe21c35c69d5f55" +checksum = "5c110f07b222a8eb29d957eabf90dce38f41461f883099ab28dc50861913ba17" dependencies = [ "anyhow", "cfg-if", @@ -12714,24 +12714,24 @@ dependencies = [ [[package]] name = "wasmtime-internal-math" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bd982249e933e730859f84c17254c7e95f38c3b0cdf81695bfc6a270e237852" +checksum = "a273dd4ec37457d435538c00b30f3ec9ae99e04a4f6ba89a737048e8de6095df" dependencies = [ "libm", ] [[package]] name = "wasmtime-internal-slab" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8280b359384cc412bddf0023193eeaffadcb0d7fbbf860f2b7e0580e8806a344" +checksum = "3d12c7b3ae65639ab90d8d29135348dfd16de6d96eb36ab5b63861f3e12cf8e0" [[package]] name = "wasmtime-internal-unwinder" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb43477c41769fec0d031d32cb3236195acfb1e6aa4efdf463e5fd001c23f6cc" +checksum = "6da6131cb87700e06ffb7a83cc933658f521176f04d6d73511f1ee60db58c515" dependencies = [ "anyhow", "cfg-if", @@ -12742,9 +12742,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-versioned-export-macros" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b33afac17819f2f767ea280b4b87a6c95458ef1ae43da410cfc5b26813ecb78f" +checksum = "71efc44d34bb86408d7f620d308dad19457f34543e5c44ace680bdbe2d819917" dependencies = [ "proc-macro2", "quote", @@ -12753,9 +12753,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-winch" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1757c8b0626385c93994c7a3c7a23ee5b469061465b601c1fdce71ff61b11c82" +checksum = "af4da8c46ddb215374f023c0e3ec8917185b878b3d9874fbfaaefdc2d2df38e0" dependencies = [ "anyhow", "cranelift-codegen", @@ -12770,9 +12770,9 @@ dependencies = [ [[package]] name = "wasmtime-internal-wit-bindgen" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "061dd5bae2e618b32a8ed2acda5953d07dd308ac0e3f7960afc62ccdf6df7c44" +checksum = "356a1ae3cfc0534a84df35d9df214a241ca454983c0d311f8169b1674b4979a3" dependencies = [ "anyhow", "bitflags 2.11.0", @@ -13669,7 +13669,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -13680,9 +13680,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winch-codegen" -version = "36.0.19" +version = "36.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69940a29fc6b01b0e6b141912b700c17eaf956d58d63bc0c6e733435caf020e4" +checksum = "09aff830750e48ca9828e9df5ba4589688f49f2748e4a1d43fd543d6a673d272" dependencies = [ "anyhow", "cranelift-assembler-x64", diff --git a/v2/Cargo.toml b/v2/Cargo.toml index 591942bdf4..e994169c5c 100644 --- a/v2/Cargo.toml +++ b/v2/Cargo.toml @@ -3,7 +3,7 @@ resolver = "2" members = [ # Checksum-verified dependency patches; keep their upstream tests in CI. "vendor/security/zip-0.6.6", - "vendor/security/wasmtime-internal-cache-36.0.19", + "vendor/security/wasmtime-internal-cache-36.0.20", "crates/wifi-densepose-core", "crates/wifi-densepose-physics", # ADR-323 bounded pose assessment/correction "crates/wifi-densepose-signal", @@ -346,4 +346,4 @@ debug = true # See each directory's RUVIEW-PROVENANCE.md for archive digests and scope. [patch.crates-io] zip-legacy = { package = "zip", path = "vendor/security/zip-0.6.6" } -wasmtime-internal-cache = { path = "vendor/security/wasmtime-internal-cache-36.0.19" } +wasmtime-internal-cache = { path = "vendor/security/wasmtime-internal-cache-36.0.20" } diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/Cargo.toml b/v2/vendor/security/wasmtime-internal-cache-36.0.20/Cargo.toml similarity index 99% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/Cargo.toml rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/Cargo.toml index e3175bfe0f..9b86a5661b 100644 --- a/v2/vendor/security/wasmtime-internal-cache-36.0.19/Cargo.toml +++ b/v2/vendor/security/wasmtime-internal-cache-36.0.20/Cargo.toml @@ -13,7 +13,7 @@ edition = "2024" rust-version = "1.86.0" name = "wasmtime-internal-cache" -version = "36.0.19" +version = "36.0.20" authors = ["The Wasmtime Project Developers"] build = "build.rs" autolib = false diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/Cargo.toml.orig b/v2/vendor/security/wasmtime-internal-cache-36.0.20/Cargo.toml.orig similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/Cargo.toml.orig rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/Cargo.toml.orig diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/LICENSE b/v2/vendor/security/wasmtime-internal-cache-36.0.20/LICENSE similarity index 99% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/LICENSE rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/LICENSE index be1d7c438a..f9d81955f4 100644 --- a/v2/vendor/security/wasmtime-internal-cache-36.0.19/LICENSE +++ b/v2/vendor/security/wasmtime-internal-cache-36.0.20/LICENSE @@ -217,3 +217,4 @@ conflicts with the conditions of the GPLv2, you may retroactively and prospectively choose to deem waived or otherwise exclude such Section(s) of the License, but only in their entirety and only with respect to the Combined Software. + diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/RUVIEW-PROVENANCE.md b/v2/vendor/security/wasmtime-internal-cache-36.0.20/RUVIEW-PROVENANCE.md similarity index 73% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/RUVIEW-PROVENANCE.md rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/RUVIEW-PROVENANCE.md index 4d8e8ae6e1..aec9ef6dd1 100644 --- a/v2/vendor/security/wasmtime-internal-cache-36.0.19/RUVIEW-PROVENANCE.md +++ b/v2/vendor/security/wasmtime-internal-cache-36.0.20/RUVIEW-PROVENANCE.md @@ -1,7 +1,7 @@ # Dependency compatibility patch -Source: crates.io `wasmtime-internal-cache` 36.0.19. -Published archive SHA-256: `4108e1964b3a02b4b5604ea1a2918a0cbcab512e5de0ccf4c062fc6c06b2eb8e` (verified against the pre-patch workspace lockfile at `fefb0f918`). +Source: crates.io `wasmtime-internal-cache` 36.0.20. +Published archive SHA-256: `65f09c789d1bb2e7ab1618ac8336d783c1ff0babc38430c7f1940d01c546722b` (verified against the crates.io checksum in the workspace lockfile). Upstream executable source code and tests are unchanged. Trailing whitespace in upstream documentation/comments is normalized for clean repository diffs. diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/build.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.20/build.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/build.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/build.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/src/config.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.20/src/config.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/src/config.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/src/config.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/src/config/tests.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.20/src/config/tests.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/src/config/tests.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/src/config/tests.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/src/lib.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.20/src/lib.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/src/lib.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/src/lib.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/src/tests.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.20/src/tests.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/src/tests.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/src/tests.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/src/worker.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.20/src/worker.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/src/worker.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/src/worker.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/src/worker/tests.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.20/src/worker/tests.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/src/worker/tests.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/src/worker/tests.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/src/worker/tests/system_time_stub.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.20/src/worker/tests/system_time_stub.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/src/worker/tests/system_time_stub.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/src/worker/tests/system_time_stub.rs diff --git a/v2/vendor/security/wasmtime-internal-cache-36.0.19/tests/cache_write_default_config.rs b/v2/vendor/security/wasmtime-internal-cache-36.0.20/tests/cache_write_default_config.rs similarity index 100% rename from v2/vendor/security/wasmtime-internal-cache-36.0.19/tests/cache_write_default_config.rs rename to v2/vendor/security/wasmtime-internal-cache-36.0.20/tests/cache_write_default_config.rs From 6a9aa03b289bda26204b45f010dbad73204135d7 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Sun, 1 Nov 2026 16:16:18 +0100 Subject: [PATCH 029/100] feat(calibration): add opt-in per-source field bank shadow --- .../wifi-densepose-sensing-server/README.md | 21 + .../wifi-densepose-sensing-server/src/main.rs | 27 ++ .../src/source_bank_shadow.rs | 375 ++++++++++++++++++ .../wifi-densepose-signal/src/ruvsense/mod.rs | 1 + .../src/ruvsense/source_field_bank.rs | 315 +++++++++++++++ .../src/ruvsense/source_field_bank_tests.rs | 201 ++++++++++ 6 files changed, 940 insertions(+) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/source_bank_shadow.rs create mode 100644 v2/crates/wifi-densepose-signal/src/ruvsense/source_field_bank.rs create mode 100644 v2/crates/wifi-densepose-signal/src/ruvsense/source_field_bank_tests.rs diff --git a/v2/crates/wifi-densepose-sensing-server/README.md b/v2/crates/wifi-densepose-sensing-server/README.md index 0fae86a5fc..5749868bcf 100644 --- a/v2/crates/wifi-densepose-sensing-server/README.md +++ b/v2/crates/wifi-densepose-sensing-server/README.md @@ -76,6 +76,27 @@ cargo run -p wifi-densepose-sensing-server -- \ --ui-path ../ui ``` +### Experimental per-source shadow diagnostics + +`--source-bank-shadow` is off by default. When explicitly enabled, the next normal, +operator-confirmed room calibration also trains one independent, canonicalized +FieldModel per bound source (at least two). Each source must satisfy its own frame, +duration, grid and continuity requirements. Primary single-source calibration and +publication gates remain unchanged. This first adapter uses the existing frozen +raw grid for all sources; independent per-node grid selection is not implemented. +A radio missing that grid cannot complete its shadow calibration. + +After normal calibration completion, protected `GET /api/v1/calibration/status` +includes `source_bank_shadow`: per-source frame counts and empty-background +comparisons only after complete, fresh, post-training windows arrive. Privacy mode +omits the diagnostics. Cancel/reset clears the bank; normal `/calibration/stop` +completion freezes it for shadow comparison. Restart does not restore it. +No bank data is persisted or promoted into person count, presence or Vitals. +The existing UDP source/grid/sequence admission is not cryptographic sensor +authentication; diagnostics must retain that transport limitation. +This developer prototype needs independent physical empty/occupied/adjacent +qualification before any detection benefit can be claimed. + ### Empty-room startup baseline Provide a stable installation ID and an application-owned private state directory: diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index aa36aba0a9..6db68f6852 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -17,6 +17,7 @@ pub mod cli; pub mod csi; mod engine_bridge; mod field_bridge; +mod source_bank_shadow; mod field_localize; mod model_format; mod multistatic_bridge; @@ -316,6 +317,10 @@ struct Args { #[arg(long)] calibrate: bool, + /// Collect a memory-only per-source shadow during explicit room calibration; never promote it. + #[arg(long)] + source_bank_shadow: bool, + // --------------------------------------------------------------- // ADR-102: Edge Module Registry — surface the canonical Cognitum // cog catalog via `GET /api/v1/edge/registry`. @@ -2203,6 +2208,8 @@ struct AppStateInner { engine_bridge: engine_bridge::EngineBridge, /// SVD-based room field model for eigenvalue person counting (None until calibration). field_model: Option, + source_bank_shadow_enabled: bool, + source_bank_shadow: Option, /// Stable installation identity used only to bind local persisted state. installation_id: Option, /// Metadata for the privacy reduced empty room image, when available. @@ -2645,6 +2652,7 @@ impl AppStateInner { } CalibrationSequenceOrder::Discontinuity => { self.calibration_sequence_fault_node_ids.insert(node_id); + if let Some(shadow) = self.source_bank_shadow.as_mut() { shadow.invalidate(); } warn!( node_id, previous_sequence, @@ -2700,6 +2708,9 @@ impl AppStateInner { if let Some(node) = self.node_states.get_mut(&node_id) { node.push_field_model_frame(sequence, amplitudes, observed_at); } + if let Some(shadow) = self.source_bank_shadow.as_mut() { + shadow.observe(node_id, chrono::Utc::now().timestamp_micros().max(0) as u64, amplitudes); + } } accepted } @@ -2712,6 +2723,7 @@ impl AppStateInner { } fn clear_field_model_binding(&mut self) { + self.source_bank_shadow = None; self.calibration_grid_binding = None; for node in self.node_states.values_mut() { node.clear_field_model_history(); @@ -3130,6 +3142,8 @@ impl AppStateInner { None, ), field_model: None, + source_bank_shadow_enabled: false, + source_bank_shadow: None, installation_id: None, bootstrap_baseline: None, bootstrap_baseline_active: false, @@ -10570,6 +10584,11 @@ async fn calibration_start( s.calibration_source_node_ids .extend(request.source_node_ids.iter().copied()); s.calibration_grid_binding = Some(selected_binding); + if s.source_bank_shadow_enabled { + s.source_bank_shadow = source_bank_shadow::ShadowCapture::new( + &request.source_node_ids, selected_binding.grid.n_subcarriers as usize, + format!("csi-{}-{}", selected_binding.grid.n_subcarriers, selected_binding.grid.ppdu_type)); + } Json(serde_json::json!({ "success": true, "message": "Calibration started — keep room empty while frames accumulate.", @@ -10711,6 +10730,7 @@ async fn calibration_stop( }; s.calibration_restored = false; s.calibration_model_receipt = Some(receipt.clone()); + if let Some(shadow) = s.source_bank_shadow.as_mut() { shadow.freeze(ts); } s.begin_field_model_holdout(binding); info!("Field model calibrated: baseline_eigenvalues={baseline}, variance_explained={variance_explained:.2}"); Json(serde_json::json!({ @@ -10864,6 +10884,11 @@ async fn calibration_status(State(state): State) -> Json>() } else { Vec::new() }); response["observed_source_node_ids"] = serde_json::json!(if bound { s.calibration_observed_source_node_ids.iter().copied().collect::>() } else { Vec::new() }); response["missing_source_node_ids"] = serde_json::json!(if bound { s.calibration_source_node_ids.difference(&s.calibration_observed_source_node_ids).copied().collect::>() } else { Vec::new() }); + if !s.privacy_mode { + if let Some(shadow) = s.source_bank_shadow.as_ref() { + response["source_bank_shadow"] = shadow.diagnostics(observed_at_unix_ms.saturating_mul(1_000)); + } + } Json(response) } @@ -15872,6 +15897,8 @@ async fn main() { ) .with_node_positions(engine_bridge_node_positions), field_model: bootstrap_field_model, + source_bank_shadow_enabled: args.source_bank_shadow, + source_bank_shadow: None, installation_id: args.installation_id.clone(), bootstrap_baseline: bootstrap_metadata.clone(), bootstrap_baseline_active, diff --git a/v2/crates/wifi-densepose-sensing-server/src/source_bank_shadow.rs b/v2/crates/wifi-densepose-sensing-server/src/source_bank_shadow.rs new file mode 100644 index 0000000000..e1e42ac33f --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/source_bank_shadow.rs @@ -0,0 +1,375 @@ +//! Opt-in, memory-only per-source calibration diagnostics. Never used by the +//! authoritative model, receipts, persistence, person count or vital publication. +use serde_json::{json, Value}; +use std::collections::BTreeMap; +use wifi_densepose_signal::hardware_norm::HardwareNormalizer; +use wifi_densepose_signal::ruvsense::{ + field_model::FieldModelConfig, + source_field_bank::{SourceBankPhase, SourceFieldBank, SourceFieldSpec, SourceFieldWindow}, +}; + +pub(crate) struct ShadowCapture { + bank: SourceFieldBank, + grid: String, + ordinals: BTreeMap, + latest: BTreeMap, + windows: BTreeMap>>, + failed: bool, + input_width: usize, + canonical_width: usize, + normalizer: HardwareNormalizer, +} + +impl ShadowCapture { + pub(crate) fn new(ids: &[u8], width: usize, grid: String) -> Option { + let mut capture = Self::with_config(ids, grid, super::field_bridge::single_link_config())?; + capture.input_width = width; + Some(capture) + } + + fn with_config(ids: &[u8], grid: String, config: FieldModelConfig) -> Option { + let specs = ids + .iter() + .map(|&source_id| SourceFieldSpec { + source_id, + grid_id: grid.clone(), + config: config.clone(), + }) + .collect(); + let bank = SourceFieldBank::new(specs, 5_000_000).ok()?; + Some(Self { + bank, + grid, + ordinals: ids.iter().map(|&id| (id, 0)).collect(), + latest: BTreeMap::new(), + windows: BTreeMap::new(), + failed: false, + input_width: config.n_subcarriers, + canonical_width: config.n_subcarriers, + normalizer: HardwareNormalizer::new(), + }) + } + + /// Only called after the existing source/grid/sequence admission succeeds. + /// Receiver timestamps and validated ordinals never come from an HTTP client. + pub(crate) fn observe(&mut self, id: u8, timestamp_us: u64, frame: &[f64]) { + if self.failed { + return; + } + let Some(ordinal) = self.ordinals.get_mut(&id) else { + self.failed = true; + return; + }; + if frame.len() != self.input_width + || frame.iter().any(|v| !v.is_finite() || v.abs() > 1_000_000.) + || timestamp_us == 0 + || self + .latest + .get(&id) + .is_some_and(|at| timestamp_us <= *at || timestamp_us - *at > 5_000_000) + { + self.failed = true; + return; + } + let canonical = if self.canonical_width == 56 { + self.normalizer.resample_to_canonical(frame) + } else { + frame.to_vec() + }; + let frame = canonical.as_slice(); + *ordinal = ordinal.saturating_add(1); + if self.bank.phase() == SourceBankPhase::Collecting { + if self + .bank + .feed(id, &self.grid, *ordinal, timestamp_us, frame) + .is_err() + { + self.failed = true; + return; + } + } else if self.bank.phase() != SourceBankPhase::Frozen { + self.failed = true; + return; + } + // Keep at most one runtime-sized window per configured source. + self.latest.insert(id, timestamp_us); + let window = self.windows.entry(id).or_default(); + window.push(frame.to_vec()); + if window.len() > 50 { + window.remove(0); + } + } + + pub(crate) fn invalidate(&mut self) { + self.failed = true; + } + + pub(crate) fn freeze(&mut self, timestamp_us: u64) { + if self.failed || self.bank.freeze(timestamp_us, 0).is_err() { + self.failed = true; + } + // Held-out windows must contain no training frames. + self.windows.clear(); + } + + pub(crate) fn diagnostics(&self, timestamp_us: u64) -> Value { + let phase = if self.failed { + "invalid" + } else { + match self.bank.phase() { + SourceBankPhase::Collecting => "collecting", + SourceBankPhase::Frozen => "frozen", + SourceBankPhase::Invalid => "invalid", + } + }; + let comparisons = if !self.failed + && self.bank.phase() == SourceBankPhase::Frozen + && self.windows.values().all(|frames| frames.len() == 50) + { + let windows: Vec<_> = self + .windows + .iter() + .map(|(&id, frames)| SourceFieldWindow { + source_id: id, + grid_id: &self.grid, + observed_at_us: self.latest[&id], + frames, + }) + .collect(); + self.bank + .compare_empty_windows(timestamp_us, &windows) + .ok() + .map(|result| { + result + .into_iter() + .map(|(id, score)| { + ( + id.to_string(), + json!({ + "matches_empty": score.matches_empty, + "residual_energy": score.residual_energy, + "threshold": score.residual_energy_threshold, + "reliable": score.reliable, + "maturity": score.maturity, + "reference_window_count": score.reference_window_count, + "window_size": score.window_size, + }), + ) + }) + .collect::>() + }) + } else { + None + }; + json!({ "schema": "ruview.calibration.source-bank-shadow.v1", "phase": phase, + "frame_counts": self.bank.frame_counts(), "comparisons": comparisons, + "authority": "diagnostic_only", "persisted": false, + "calibrated_presence_authorized": false, "numeric_vitals_authorized": false }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn capture() -> ShadowCapture { + ShadowCapture::with_config( + &[15, 16, 17], + "ht64".into(), + FieldModelConfig { + n_links: 1, + n_subcarriers: 4, + n_modes: 1, + min_calibration_frames: 1_000, + min_calibration_duration_s: 0.0, + ..Default::default() + }, + ) + .unwrap() + } + #[test] + fn incomplete_capture_never_freezes_or_exposes_comparisons() { + let mut s = capture(); + for i in 1..=1_000 { + s.observe(15, i * 1_000, &[1., 2., 3., 4.]); + } + s.freeze(1_000_001); + let d = s.diagnostics(1_000_002); + assert_eq!(d["phase"], "invalid"); + assert!(d["comparisons"].is_null()); + assert_eq!(d["numeric_vitals_authorized"], false); + } + #[test] + fn freeze_requires_new_complete_held_out_windows_and_never_exports_raw_frames() { + let mut s = capture(); + for i in 1..=1_000 { + for id in [15, 16, 17] { + s.observe(id, i * 1_000, &[id as f64, 2., 3., 4.]); + } + } + s.freeze(1_000_001); + assert_eq!(s.diagnostics(1_000_002)["phase"], "frozen"); + assert!(s.diagnostics(1_000_002)["comparisons"].is_null()); + for i in 1..=50 { + for id in [15, 16, 17] { + s.observe(id, 1_001_000 + i * 1_000, &[id as f64, 2., 3., 4.]); + } + } + let d = s.diagnostics(1_052_000); + assert_eq!(d["comparisons"].as_object().unwrap().len(), 3); + assert_eq!(d["frame_counts"]["15"], 1_000); + assert_eq!(d["calibrated_presence_authorized"], false); + assert!(d.get("windows").is_none()); + assert!(s.diagnostics(7_000_000)["comparisons"].is_null()); + } + #[test] + fn production_resampling_matches_the_single_source_control() { + use wifi_densepose_signal::ruvsense::field_model::FieldModel; + let config = FieldModelConfig { + min_calibration_frames: 1_000, + min_calibration_duration_s: 0.0, + ..super::super::field_bridge::single_link_config() + }; + let mut control = FieldModel::new(config.clone()).unwrap(); + let mut shadow = ShadowCapture::with_config(&[15, 16, 17], "ht64".into(), config).unwrap(); + shadow.input_width = 64; + for i in 1..=1_000 { + for id in [15, 16, 17] { + let frame: Vec<_> = (0..64) + .map(|tone| id as f64 + tone as f64 + (i % 13) as f64 * 0.01) + .collect(); + if id == 15 { + assert!(super::super::field_bridge::maybe_feed_calibration( + &mut control, + &frame + )); + } + shadow.observe(id, i * 1_000, &frame); + } + } + control.finalize_calibration(1_000_001, 0).unwrap(); + shadow.freeze(1_000_001); + let models = shadow.bank.frozen_models().unwrap(); + assert_eq!( + models[0].snapshot.modes.baseline, + control.export_snapshot().unwrap().modes.baseline + ); + assert_ne!( + models[0].snapshot.modes.baseline, + models[1].snapshot.modes.baseline + ); + } + #[test] + fn unknown_or_changed_runtime_sources_invalidate_only_the_shadow() { + let mut s = capture(); + s.observe(99, 1_000, &[1., 2., 3., 4.]); + assert_eq!(s.diagnostics(1_001)["phase"], "invalid"); + assert_eq!(s.bank.frame_counts().len(), 3); + let mut s = capture(); + s.observe(15, 1_000, &[1., 2., 3., 4.]); + s.observe(15, 1_000, &[1., 2., 3., 4.]); + assert_eq!(s.diagnostics(1_001)["phase"], "invalid"); + } + #[tokio::test] + async fn status_is_opt_in_privacy_redacted_and_cleared_with_binding() { + use super::super::{calibration_status, AppStateInner}; + use axum::extract::State; + use std::sync::Arc; + use tokio::sync::RwLock; + let state = Arc::new(RwLock::new(AppStateInner::minimal())); + let axum::Json(status) = calibration_status(State(state.clone())).await; + assert!(status.get("source_bank_shadow").is_none()); + state.write().await.source_bank_shadow = Some(capture()); + let axum::Json(status) = calibration_status(State(state.clone())).await; + assert_eq!(status["source_bank_shadow"]["authority"], "diagnostic_only"); + state.write().await.privacy_mode = true; + let axum::Json(status) = calibration_status(State(state.clone())).await; + assert!(status.get("source_bank_shadow").is_none()); + state.write().await.clear_field_model_binding(); + assert!(state.read().await.source_bank_shadow.is_none()); + } + #[test] + fn opt_in_admission_trains_each_shadow_without_pooling_the_primary() { + use super::super::{AppStateInner, CalibrationGridBinding, CsiGridKey}; + use wifi_densepose_signal::ruvsense::field_model::FieldModel; + let mut state = AppStateInner::minimal(); + let grid = CsiGridKey { + n_subcarriers: 64, + ppdu_type: 0, + }; + state.calibration_grid_binding = Some(CalibrationGridBinding { + source_node_id: 15, + grid, + evidence: None, + }); + state.calibration_source_node_ids.extend([15, 16, 17]); + state.field_model = + Some(FieldModel::new(super::super::field_bridge::single_link_config()).unwrap()); + state.source_bank_shadow = ShadowCapture::new(&[15, 16, 17], 64, "ht64".into()); + for sequence in 1..=100 { + for id in [15, 16, 17] { + assert!(state.maybe_feed_calibration_frame( + id, + grid, + sequence, + &vec![id as f64; 64], + std::time::Instant::now() + )); + } + } + assert_eq!( + state + .field_model + .as_ref() + .unwrap() + .calibration_frame_count(), + 100 + ); + let shadow = state.source_bank_shadow.as_ref().unwrap(); + assert_eq!( + shadow.bank.frame_counts(), + BTreeMap::from([(15, 100), (16, 100), (17, 100)]) + ); + assert!(state.calibration_model_receipt.is_none()); + assert_eq!( + shadow.diagnostics(u64::MAX)["numeric_vitals_authorized"], + false + ); + assert!(!state.maybe_feed_calibration_frame( + 15, + grid, + 0, + &vec![15.; 64], + std::time::Instant::now() + )); + assert_eq!( + state + .source_bank_shadow + .as_ref() + .unwrap() + .diagnostics(u64::MAX)["phase"], + "invalid" + ); + assert_eq!( + state + .field_model + .as_ref() + .unwrap() + .calibration_frame_count(), + 100 + ); + } + #[test] + fn command_line_shadow_is_disabled_by_default() { + use clap::Parser; + assert!( + !super::super::Args::try_parse_from(["sensing-server"]) + .unwrap() + .source_bank_shadow + ); + assert!( + super::super::Args::try_parse_from(["sensing-server", "--source-bank-shadow"]) + .unwrap() + .source_bank_shadow + ); + } +} diff --git a/v2/crates/wifi-densepose-signal/src/ruvsense/mod.rs b/v2/crates/wifi-densepose-signal/src/ruvsense/mod.rs index e2f56af434..0c75bde21b 100644 --- a/v2/crates/wifi-densepose-signal/src/ruvsense/mod.rs +++ b/v2/crates/wifi-densepose-signal/src/ruvsense/mod.rs @@ -38,6 +38,7 @@ pub mod adversarial; pub mod cross_room; pub mod field_model; +pub mod source_field_bank; pub mod gesture; pub mod intention; pub mod longitudinal; diff --git a/v2/crates/wifi-densepose-signal/src/ruvsense/source_field_bank.rs b/v2/crates/wifi-densepose-signal/src/ruvsense/source_field_bank.rs new file mode 100644 index 0000000000..57d1ddbf08 --- /dev/null +++ b/v2/crates/wifi-densepose-signal/src/ruvsense/source_field_bank.rs @@ -0,0 +1,315 @@ +//! Experimental calibration-only bank: one existing FieldModel per source. +//! +//! No sensing-server caller or detection promotion is introduced here. A frozen +//! bank is learned statistics, not an owner/room receipt or occupied acceptance. +//! The caller must enforce the production capture duration, room binding and +//! independent physical qualification. Tests use explicitly synthetic inputs. + +use std::collections::BTreeMap; +use thiserror::Error; + +use super::field_model::{ + CalibrationStatus, EmptyRoomMatch, FieldModel, FieldModelConfig, FieldModelError, + FieldModelSnapshotV1, +}; + +const MAX_SOURCES: usize = 32; +const MAX_SUBCARRIERS: usize = 256; +const MAX_SOURCE_AGE_US: u64 = 5_000_000; +const MAX_AMPLITUDE: f64 = 1_000_000.0; + +pub struct SourceFieldSpec { + pub source_id: u8, + pub grid_id: String, + pub config: FieldModelConfig, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SourceBankPhase { + Collecting, + Frozen, + Invalid, +} + +#[derive(Debug, Error)] +pub enum SourceBankError { + #[error("invalid source-bank configuration: {0}")] + Configuration(&'static str), + #[error("duplicate source {0}")] + DuplicateSource(u8), + #[error("unknown source {0}")] + UnknownSource(u8), + #[error("source {0} changed its grid, sample order, timing or frame shape")] + SourceChanged(u8), + #[error("source {0} has not completed its independent calibration")] + IncompleteSource(u8), + #[error("source {0} is missing, stale or future-dated")] + StaleSource(u8), + #[error("bank is not in the required phase")] + Phase, + #[error("field-model calibration failed: {0}")] + Model(#[from] FieldModelError), +} + +pub struct FrozenSourceField { + pub source_id: u8, + pub grid_id: String, + pub snapshot: FieldModelSnapshotV1, +} + +/// A bounded, memory-only window from the caller's admitted/epoch-bound source +/// recorder. The caller must describe its actual transport trust; this type does +/// not authenticate a radio. Comparisons are empty-background tests, not person labels. +pub struct SourceFieldWindow<'a> { + pub source_id: u8, + pub grid_id: &'a str, + pub observed_at_us: u64, + pub frames: &'a [Vec], +} + +struct SourceTraining { + grid_id: String, + width: usize, + model: FieldModel, + last_sample: Option<(u64, u64)>, +} + +pub struct SourceFieldBank { + sources: BTreeMap, + phase: SourceBankPhase, + source_age_us: u64, + frozen_at_us: Option, +} + +impl SourceFieldBank { + pub fn new(specs: Vec, source_age_us: u64) -> Result { + if !(2..=MAX_SOURCES).contains(&specs.len()) + || source_age_us == 0 + || source_age_us > MAX_SOURCE_AGE_US + { + return Err(SourceBankError::Configuration( + "bounded source set and freshness window required", + )); + } + // Validate every allocation shape before constructing any model. + for spec in &specs { + let c = &spec.config; + if c.n_links != 1 + || !(1..=MAX_SUBCARRIERS).contains(&c.n_subcarriers) + || !(1..=5).contains(&c.n_modes) + || !(2..=1_000_000).contains(&c.min_calibration_frames) + || !c.min_calibration_duration_s.is_finite() + || !(0.0..=86_400.0).contains(&c.min_calibration_duration_s) + || !c.baseline_expiry_s.is_finite() + || c.baseline_expiry_s <= 0.0 + || c.baseline_expiry_s > 86_400.0 + || spec.grid_id.is_empty() + || spec.grid_id.len() > 128 + || !spec + .grid_id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || b"-_.".contains(&b)) + { + return Err(SourceBankError::Configuration( + "one bounded finite single-link model and grid per source required", + )); + } + } + let mut sources = BTreeMap::new(); + for spec in specs { + if sources.contains_key(&spec.source_id) { + return Err(SourceBankError::DuplicateSource(spec.source_id)); + } + sources.insert( + spec.source_id, + SourceTraining { + grid_id: spec.grid_id, + width: spec.config.n_subcarriers, + model: FieldModel::new(spec.config)?, + last_sample: None, + }, + ); + } + Ok(Self { + sources, + phase: SourceBankPhase::Collecting, + source_age_us, + frozen_at_us: None, + }) + } + + pub fn phase(&self) -> SourceBankPhase { + self.phase + } + + /// `ordinal` is the server's validated per-source ordinal, not an unchecked + /// raw device counter. Device wrap/transport authentication remain upstream. + pub fn feed( + &mut self, + source_id: u8, + grid_id: &str, + ordinal: u64, + observed_at_us: u64, + amplitudes: &[f64], + ) -> Result<(), SourceBankError> { + if self.phase != SourceBankPhase::Collecting { + return Err(SourceBankError::Phase); + } + let source = self + .sources + .get_mut(&source_id) + .ok_or(SourceBankError::UnknownSource(source_id))?; + let changed = grid_id != source.grid_id + || observed_at_us == 0 + || amplitudes.len() != source.width + || amplitudes + .iter() + .any(|x| !x.is_finite() || x.abs() > MAX_AMPLITUDE) + || source.last_sample.is_some_and(|(last_ordinal, last_at)| { + ordinal <= last_ordinal + || observed_at_us <= last_at + || observed_at_us.saturating_sub(last_at) > self.source_age_us + }); + if changed { + self.phase = SourceBankPhase::Invalid; + return Err(SourceBankError::SourceChanged(source_id)); + } + if let Err(error) = source.model.feed_calibration(&[amplitudes.to_vec()]) { + self.phase = SourceBankPhase::Invalid; + return Err(error.into()); + } + source.last_sample = Some((ordinal, observed_at_us)); + Ok(()) + } + + pub fn frame_counts(&self) -> BTreeMap { + self.sources + .iter() + .map(|(id, s)| (*id, s.model.calibration_frame_count())) + .collect() + } + + /// Check all members before mutating any model. A model-level failure + /// invalidates the whole generation; partially frozen members cannot escape. + pub fn freeze(&mut self, timestamp_us: u64, geometry_hash: u64) -> Result<(), SourceBankError> { + if self.phase != SourceBankPhase::Collecting { + return Err(SourceBankError::Phase); + } + for (id, source) in &self.sources { + if source.model.calibration_frame_count() < source.model.min_calibration_frames() as u64 + || source.model.calibration_elapsed_s() < source.model.min_calibration_duration_s() + { + return Err(SourceBankError::IncompleteSource(*id)); + } + if source + .last_sample + .is_none_or(|(_, at)| timestamp_us < at || timestamp_us - at > self.source_age_us) + { + return Err(SourceBankError::StaleSource(*id)); + } + } + for source in self.sources.values_mut() { + if let Err(error) = source + .model + .finalize_calibration(timestamp_us, geometry_hash) + { + self.phase = SourceBankPhase::Invalid; + return Err(error.into()); + } + } + self.phase = SourceBankPhase::Frozen; + self.frozen_at_us = Some(timestamp_us); + Ok(()) + } + + /// Statistics only. The server must wrap these in a new versioned, + /// account/installation/room-bound artifact before persistence or adoption. + pub fn frozen_models(&self) -> Result, SourceBankError> { + if self.phase != SourceBankPhase::Frozen { + return Err(SourceBankError::Phase); + } + self.sources + .iter() + .map(|(id, source)| { + Ok(FrozenSourceField { + source_id: *id, + grid_id: source.grid_id.clone(), + snapshot: source.model.export_snapshot()?, + }) + }) + .collect() + } + + /// Shadow diagnostics only: require every configured source and return its + /// individual empty-background conformance. No aggregate presence is emitted. + pub fn compare_empty_windows( + &self, + timestamp_us: u64, + windows: &[SourceFieldWindow<'_>], + ) -> Result, SourceBankError> { + let freshness = self.freshness(timestamp_us)?; + if windows.len() != self.sources.len() { + return Err(SourceBankError::Configuration( + "complete runtime window set required", + )); + } + let mut result = BTreeMap::new(); + for window in windows { + let id = window.source_id; + let source = self + .sources + .get(&id) + .ok_or(SourceBankError::UnknownSource(id))?; + if result.contains_key(&id) + || window.grid_id != source.grid_id + || window.frames.is_empty() + || window.frames.len() > 200 + || window.frames.iter().any(|frame| { + frame.len() != source.width + || frame + .iter() + .any(|v| !v.is_finite() || v.abs() > MAX_AMPLITUDE) + }) + { + return Err(SourceBankError::SourceChanged(id)); + } + if freshness[&id] != CalibrationStatus::Fresh + || timestamp_us < window.observed_at_us + || timestamp_us - window.observed_at_us > self.source_age_us + { + return Err(SourceBankError::StaleSource(id)); + } + let compared = source + .model + .empty_room_match(window.frames) + .ok_or(SourceBankError::IncompleteSource(id))?; + result.insert(id, compared); + } + if result.len() != self.sources.len() { + return Err(SourceBankError::Configuration( + "required source window missing", + )); + } + Ok(result) + } + + pub fn freshness( + &self, + timestamp_us: u64, + ) -> Result, SourceBankError> { + if self.phase != SourceBankPhase::Frozen + || self.frozen_at_us.is_none_or(|at| timestamp_us < at) + { + return Err(SourceBankError::Phase); + } + Ok(self + .sources + .iter() + .map(|(id, s)| (*id, s.model.check_freshness(timestamp_us))) + .collect()) + } +} + +#[cfg(test)] +#[path = "source_field_bank_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-signal/src/ruvsense/source_field_bank_tests.rs b/v2/crates/wifi-densepose-signal/src/ruvsense/source_field_bank_tests.rs new file mode 100644 index 0000000000..947f4fa9c4 --- /dev/null +++ b/v2/crates/wifi-densepose-signal/src/ruvsense/source_field_bank_tests.rs @@ -0,0 +1,201 @@ +use super::*; + +// SYNTHETIC software fixtures, not measured RF or a physical qualification. +fn config() -> FieldModelConfig { + FieldModelConfig { + n_links: 1, + n_subcarriers: 4, + n_modes: 3, + min_calibration_frames: 16, + min_calibration_duration_s: 0.0, + baseline_expiry_s: 10.0, + } +} +fn spec(id: u8) -> SourceFieldSpec { + SourceFieldSpec { + source_id: id, + grid_id: format!("grid-{id}"), + config: config(), + } +} +fn bank() -> SourceFieldBank { + SourceFieldBank::new(vec![spec(15), spec(16)], 1000).unwrap() +} +fn frame(base: f64, i: u64) -> Vec { + (0..4) + .map(|j| base + j as f64 * 0.1 + i as f64 * 0.01) + .collect() +} +fn train(bank: &mut SourceFieldBank, id: u8, base: f64) { + for i in 0..16 { + bank.feed(id, &format!("grid-{id}"), i, i + 1, &frame(base, i)) + .unwrap(); + } +} + +#[test] +fn source_scale_does_not_change_the_other_sources_model() { + let mut bank = bank(); + let mut control = FieldModel::new(config()).unwrap(); + for i in 0..16 { + let f = frame(10.0, i); + control.feed_calibration(&[f.clone()]).unwrap(); + bank.feed(15, "grid-15", i, i + 1, &f).unwrap(); + bank.feed(16, "grid-16", i, i + 1, &frame(10000.0, i)) + .unwrap(); + } + control.finalize_calibration(100, 42).unwrap(); + bank.freeze(100, 42).unwrap(); + let frozen = bank.frozen_models().unwrap(); + assert_eq!( + frozen.iter().map(|s| s.source_id).collect::>(), + vec![15, 16] + ); + assert_eq!( + frozen[0].snapshot.modes, + control.export_snapshot().unwrap().modes + ); + assert!(frozen[1].snapshot.modes.baseline[0][0] > 9000.0); +} + +#[test] +fn observed_or_missing_members_cannot_borrow_another_members_frames() { + let mut b = bank(); + train(&mut b, 15, 10.0); + b.feed(16, "grid-16", 0, 1, &frame(50.0, 0)).unwrap(); + assert!(matches!( + b.freeze(100, 42), + Err(SourceBankError::IncompleteSource(16)) + )); + assert_eq!(b.phase(), SourceBankPhase::Collecting); + assert!(b.frozen_models().is_err()); + assert_eq!(b.frame_counts()[&16], 1); +} + +#[test] +fn invalid_shapes_and_nonfinite_frames_invalidate_the_generation() { + for bad in [ + vec![1.0], + vec![f64::NAN; 4], + vec![f64::INFINITY; 4], + vec![f64::MAX; 4], + ] { + let mut b = bank(); + assert!(matches!( + b.feed(15, "grid-15", 0, 1, &bad), + Err(SourceBankError::SourceChanged(15)) + )); + assert_eq!(b.phase(), SourceBankPhase::Invalid); + assert!(b.freeze(2, 42).is_err()); + } +} + +#[test] +fn grid_sequence_clock_and_continuity_changes_are_not_partial_success() { + for (grid, ordinal, time) in [ + ("other", 1, 2), + ("grid-15", 0, 2), + ("grid-15", 1, 1), + ("grid-15", 1, 1002), + ] { + let mut b = bank(); + b.feed(15, "grid-15", 0, 1, &frame(10.0, 0)).unwrap(); + assert!(b.feed(15, grid, ordinal, time, &frame(10.0, 1)).is_err()); + assert_eq!(b.phase(), SourceBankPhase::Invalid); + assert!(b.frozen_models().is_err()); + } +} + +#[test] +fn stale_and_future_source_windows_cannot_freeze() { + let mut b = bank(); + train(&mut b, 15, 10.0); + train(&mut b, 16, 20.0); + assert!(matches!( + b.freeze(1017, 42), + Err(SourceBankError::StaleSource(15)) + )); + assert!(matches!( + b.freeze(15, 42), + Err(SourceBankError::StaleSource(15)) + )); + assert_eq!(b.phase(), SourceBankPhase::Collecting); + assert!(b.frozen_models().is_err()); +} + +#[test] +fn freeze_is_immutable_and_does_not_renew_model_age() { + let mut b = bank(); + train(&mut b, 15, 10.0); + train(&mut b, 16, 20.0); + b.freeze(100, 42).unwrap(); + assert!(b.feed(15, "grid-15", 16, 101, &frame(10.0, 16)).is_err()); + assert!(b.freeze(200, 42).is_err()); + assert!(b.freshness(99).is_err()); + assert!(b + .freshness(6_000_100) + .unwrap() + .values() + .all(|x| *x == CalibrationStatus::Stale)); + assert!(b + .freshness(11_000_100) + .unwrap() + .values() + .all(|x| *x == CalibrationStatus::Expired)); + assert!(b + .frozen_models() + .unwrap() + .iter() + .all(|s| s.snapshot.modes.calibrated_at_us == 100)); +} + +#[test] +fn configuration_is_bounded_and_unknown_nodes_do_not_enter_training() { + assert!(SourceFieldBank::new(vec![spec(15)], 1000).is_err()); + assert!(SourceFieldBank::new(vec![spec(15), spec(15)], 1000).is_err()); + assert!(SourceFieldBank::new(vec![spec(15), spec(16)], 0).is_err()); + let mut wide = spec(16); + wide.config.n_subcarriers = 10000; + assert!(SourceFieldBank::new(vec![spec(15), wide], 1000).is_err()); + let mut b = bank(); + assert!(matches!( + b.feed(99, "grid-99", 0, 1, &frame(1.0, 0)), + Err(SourceBankError::UnknownSource(99)) + )); + assert_eq!(b.phase(), SourceBankPhase::Collecting); + assert!(b.frame_counts().values().all(|n| *n == 0)); +} + +#[test] +fn shadow_windows_compare_individually_and_abstain_on_incomplete_or_stale_sets() { + let mut b = bank(); + for i in 0..1000 { + b.feed(15, "grid-15", i, i + 1, &frame(10.0, i)).unwrap(); + b.feed(16, "grid-16", i, i + 1, &frame(10000.0, i)).unwrap(); + } + b.freeze(1500, 42).unwrap(); + let quiet: Vec> = (450..500).map(|i| frame(10.0, i)).collect(); + let changed: Vec> = (450..500).map(|i| frame(11000.0, i)).collect(); + let windows = [ + SourceFieldWindow { + source_id: 15, + grid_id: "grid-15", + observed_at_us: 1990, + frames: &quiet, + }, + SourceFieldWindow { + source_id: 16, + grid_id: "grid-16", + observed_at_us: 1990, + frames: &changed, + }, + ]; + let result = b.compare_empty_windows(2000, &windows).unwrap(); + assert!(result[&15].matches_empty); + assert!(!result[&16].matches_empty); + assert!(b.compare_empty_windows(2000, &windows[..1]).is_err()); + assert!(b.compare_empty_windows(4000, &windows).is_err()); + assert!(b.compare_empty_windows(1980, &windows).is_err()); + assert!(b.compare_empty_windows(6_001_500, &windows).is_err()); + assert_eq!(b.phase(), SourceBankPhase::Frozen); +} From e5afe12fbbd3bb1e77d1f8ae64ae8f2d0ecd21bf Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Sun, 1 Nov 2026 20:29:46 +0100 Subject: [PATCH 030/100] feat(calibration): preserve source-aware native observations and capture roles --- v2/Cargo.lock | 2 + .../wifi-densepose-sensing-server/Cargo.toml | 2 + .../wifi-densepose-sensing-server/src/lib.rs | 3 + .../src/room_learner/capture.rs | 335 ++++++++++++++ .../src/room_learner/capture_tests.rs | 195 ++++++++ .../src/room_learner/mod.rs | 4 + .../src/room_learner/native.rs | 161 +++++++ .../src/room_learner/native_tests.rs | 148 ++++++ .../src/room_learner/observation.rs | 421 ++++++++++++++++++ .../src/room_learner/observation_tests.rs | 247 ++++++++++ 10 files changed, 1518 insertions(+) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/native.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/native_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/observation.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/observation_tests.rs diff --git a/v2/Cargo.lock b/v2/Cargo.lock index 8b33844379..468b53d44c 100644 --- a/v2/Cargo.lock +++ b/v2/Cargo.lock @@ -13492,6 +13492,7 @@ dependencies = [ "mdns-sd", "midstreamer-attractor", "midstreamer-temporal-compare", + "num-complex", "opentelemetry-appender-tracing", "opentelemetry-otlp", "opentelemetry_sdk", @@ -13503,6 +13504,7 @@ dependencies = [ "rustls", "ruvector-mincut", "ruview-auth", + "ruview-unified", "serde", "serde_json", "sha2 0.10.9", diff --git a/v2/crates/wifi-densepose-sensing-server/Cargo.toml b/v2/crates/wifi-densepose-sensing-server/Cargo.toml index 2b727dc121..a9c7ce35ff 100644 --- a/v2/crates/wifi-densepose-sensing-server/Cargo.toml +++ b/v2/crates/wifi-densepose-sensing-server/Cargo.toml @@ -52,6 +52,8 @@ clap = { workspace = true } # from `lib.rs` so the Python `[aether]` wheel can bind it without this server's # Axum/tokio/worldgraph/ruvector tree. wifi-densepose-aether = { version = "0.3.0", path = "../wifi-densepose-aether" } +ruview-unified = { path = "../ruview-unified" } +num-complex = { workspace = true } wifi-densepose-core = { workspace = true, features = ["serde"] } wifi-densepose-physics = { workspace = true } wifi-densepose-mat = { version = "0.3.2", path = "../wifi-densepose-mat", default-features = false, features = ["std", "api"] } diff --git a/v2/crates/wifi-densepose-sensing-server/src/lib.rs b/v2/crates/wifi-densepose-sensing-server/src/lib.rs index ba062de795..d92d579dfb 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/lib.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/lib.rs @@ -79,3 +79,6 @@ pub mod account_profile; pub mod companion_tls; pub mod companion_sensors; + +/// Account-bound multi-source room learning and managed lifecycle. +pub mod room_learner; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs new file mode 100644 index 0000000000..f8cdd2ffb0 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs @@ -0,0 +1,335 @@ +//! Bounded receiver-observed capture. Operator phase labels are experimental +//! claims, never independently verified empty-room or source-clock evidence. +use super::observation::*; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::{HashMap, VecDeque}; + +pub const WINDOW_PACKETS: usize = 8; +pub const COHORTS_PER_PHASE: usize = 4; +pub const MIN_PHASE_MS: u64 = 12_000; +pub const MAX_PHASE_MS: u64 = 600_000; +pub const MAX_CAPTURE_BINS: usize = 128; +const MAX_PHASE_HISTORY: usize = 128; + +/// Borrowed actual packet metadata. The caller must reject unknown bandwidth +/// and retain the known raw grid identity. Amplitudes may be length-only +/// interpolated to the declared feature_count; no gain/mean normalization. +/// This adapter never guesses PHY or original bin count from feature length. +pub struct RawSourceFrame<'a> { + pub source_id: u8, + pub grid_id: &'a str, + pub frequency_hz: u64, + pub bandwidth_hz: u64, + pub sequence: u32, + pub amplitudes: &'a [f64], +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PhaseRequest { + pub trial_id: String, + pub session_id: String, + pub role: DataRole, + pub label: RoomLabel, + pub duration_ms: u64, + #[serde(default)] + pub lead_in_ms: u64, + pub conditions_confirmed: bool, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum PhaseStatus { Sealed, Cancelled, Failed } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PhaseReceipt { + pub trial_id: String, + pub session_id: String, + pub role: DataRole, + pub label: RoomLabel, + pub status: PhaseStatus, + pub cohort_count: usize, + pub reason: Option, +} + +#[derive(Debug, Clone)] +struct Packet { values: Vec, frame_id: u128, ordinal: u64, received_ns: u64 } +#[derive(Debug, Default)] +struct SourceBuffer { + sequence: Option, + ordinal: u64, + last_time_ns: u64, + admitted_packets: u64, + live: VecDeque, +} +#[derive(Debug)] +struct Phase { + request: PhaseRequest, + started_ns: u64, + bucket: usize, + pending: HashMap>, + representatives: Vec, + admitted: HashMap, +} + +/// Counts and receiver freshness only. No learned/qualified state is implied. +#[derive(Debug, Clone, Serialize)] +pub struct CaptureProgress { + pub phase: Option, + pub sources: Vec, + pub sealed_cohorts: usize, + pub last_failure: Option, +} +#[derive(Debug, Clone, Serialize)] +pub struct PhaseProgress { + pub trial_id: String, + pub session_id: String, + pub role: DataRole, + pub label: RoomLabel, + pub stage: &'static str, + pub lead_in_remaining_ms: u64, + pub capture_elapsed_ms: u64, + pub elapsed_ms: u64, + pub target_ms: u64, + pub representative_cohorts: usize, + pub required_cohorts: usize, +} +#[derive(Debug, Clone, Serialize)] +pub struct SourceProgress { + pub source_id: u8, + pub grid_id: String, + pub admitted_packets: u64, + pub phase_admitted_packets: u64, + pub live_buffer_frames: usize, + pub phase_buffer_frames: usize, + pub required_frames: usize, + pub latest_age_ms: Option, + pub window_eligible: bool, + pub reason: Option, +} + +#[derive(Debug)] +pub struct CaptureSession { + context: CaptureContext, + run_id: String, + prefix: u64, + last_seen_ns: u64, + sources: HashMap, + phase: Option, + cohorts: Vec, + history: Vec, + trials: HashMap, + sessions: HashMap, +} + +impl CaptureSession { + pub fn new(context: CaptureContext, run_id: String) -> Result { + context.validate()?; + identifier(&run_id)?; + if context.sources.iter().any(|s| s.feature_count > MAX_CAPTURE_BINS) { + return Err("capture supports at most 128 declared feature bins per source".into()); + } + let hash = Sha256::digest(format!("{}:{run_id}", context.digest()).as_bytes()); + let prefix = u64::from_be_bytes(hash[..8].try_into().map_err(|_| "run digest")?); + let sources = context.sources.iter().map(|s| (s.source_id, SourceBuffer::default())).collect(); + Ok(Self { context, run_id, prefix, last_seen_ns: 0, sources, phase: None, cohorts: vec![], history: vec![], + trials: HashMap::new(), sessions: HashMap::new() }) + } + + pub fn context(&self) -> &CaptureContext { &self.context } + pub fn cohorts(&self) -> &[CaptureCohort] { &self.cohorts } + pub fn history(&self) -> &[PhaseReceipt] { &self.history } + pub fn phase_active(&self) -> bool { self.phase.is_some() } + pub fn validated_dataset(&self) -> Result { + ValidatedDataset::new(self.context.clone(), self.cohorts.clone()) + } + + pub fn progress(&self, now_ns: u64) -> CaptureProgress { + let phase = self.phase.as_ref().map(|p| PhaseProgress { + trial_id: p.request.trial_id.clone(), session_id: p.request.session_id.clone(), + role: p.request.role, label: p.request.label, + stage: if now_ns < p.started_ns + p.request.lead_in_ms * 1_000_000 { "lead_in" } else { "collecting" }, + lead_in_remaining_ms: (p.started_ns + p.request.lead_in_ms * 1_000_000).saturating_sub(now_ns) / 1_000_000, + capture_elapsed_ms: now_ns.saturating_sub(p.started_ns + p.request.lead_in_ms * 1_000_000) / 1_000_000, + elapsed_ms: now_ns.saturating_sub(p.started_ns + p.request.lead_in_ms * 1_000_000) / 1_000_000, + target_ms: p.request.duration_ms, representative_cohorts: p.representatives.len(), + required_cohorts: COHORTS_PER_PHASE, + }); + let sources = self.context.sources.iter().map(|spec| { + let source = &self.sources[&spec.source_id]; + let latest_age = source.live.back().and_then(|p| now_ns.checked_sub(p.received_ns)); + let oldest_age = source.live.front().and_then(|p| now_ns.checked_sub(p.received_ns)); + let reason = if source.live.is_empty() { Some("missing_source") } + else if latest_age.is_none() || oldest_age.is_none() { Some("receiver_clock_regressed") } + else if oldest_age.is_some_and(|age| age > MAX_AGE_NS) { Some("stale_source") } + else if source.live.len() != WINDOW_PACKETS { Some("incomplete_window") } else { None }; + SourceProgress { source_id: spec.source_id, grid_id: spec.grid_id.clone(), + admitted_packets: source.admitted_packets, + phase_admitted_packets: self.phase.as_ref().and_then(|p| p.admitted.get(&spec.source_id)).copied().unwrap_or(0), + live_buffer_frames: source.live.len(), + phase_buffer_frames: self.phase.as_ref().and_then(|p| p.pending.get(&spec.source_id)).map_or(0, VecDeque::len), + required_frames: WINDOW_PACKETS, latest_age_ms: latest_age.map(|age| age / 1_000_000), + window_eligible: reason.is_none(), reason: reason.map(str::to_string) } + }).collect(); + CaptureProgress { phase, sources, sealed_cohorts: self.cohorts.len(), + last_failure: self.history.iter().rev().find(|r| r.status == PhaseStatus::Failed).and_then(|r| r.reason.clone()) } + } + + pub fn begin_phase(&mut self, request: PhaseRequest, now_ns: u64) -> Result<(), String> { + identifier(&request.trial_id)?; + identifier(&request.session_id)?; + if self.phase.is_some() { return Err("a phase is already active".into()); } + if !request.conditions_confirmed || request.role == DataRole::Inference || request.label == RoomLabel::Unknown + || !(MIN_PHASE_MS..=MAX_PHASE_MS).contains(&request.duration_ms) || request.lead_in_ms > 120_000 || now_ns == 0 + || now_ns < self.last_seen_ns || now_ns.checked_add((request.duration_ms + request.lead_in_ms) * 1_000_000).is_none() + || self.history.len() >= MAX_PHASE_HISTORY || self.cohorts.len() + COHORTS_PER_PHASE > MAX_COHORTS { + return Err("phase requires confirmed labelled conditions, bounded duration and capacity".into()); + } + for (roles, key) in [(&self.trials, &request.trial_id), (&self.sessions, &request.session_id)] { + if roles.get(key).is_some_and(|role| *role != request.role) { + return Err("trial/session role is frozen".into()); + } + } + self.trials.insert(request.trial_id.clone(), request.role); + self.sessions.insert(request.session_id.clone(), request.role); + self.phase = Some(Phase { request, started_ns: now_ns, bucket: 0, pending: HashMap::new(), representatives: vec![], admitted: HashMap::new() }); + Ok(()) + } + + pub fn ingest(&mut self, frame: RawSourceFrame<'_>, now_ns: u64) -> Result<(), String> { + let spec = match self.context.sources.iter().find(|s| s.source_id == frame.source_id) { + Some(s) => s, + None => return self.invalidate("unknown source changed capture context"), + }; + if frame.grid_id != spec.grid_id || frame.frequency_hz != spec.center_frequency_hz + || frame.bandwidth_hz != spec.bandwidth_hz || frame.amplitudes.len() != spec.feature_count + || frame.amplitudes.iter().any(|x| !x.is_finite() || *x < 0.0 || *x > f32::MAX as f64) { + return self.invalidate("source PHY/grid/shape or finite magnitude contract changed"); + } + let source = self.sources.get_mut(&frame.source_id).ok_or("source buffer unavailable")?; + if now_ns == 0 || now_ns < self.last_seen_ns || now_ns <= source.last_time_ns { return Err("nonmonotonic receiver time".into()); } + let delta = match source.sequence { + Some(last) => { + let delta = frame.sequence.wrapping_sub(last); + if delta == 0 || delta >= (1u32 << 31) { return Err("replayed or reversed source sequence".into()); } + u64::from(delta) + } + None => 1, + }; + let ordinal = source.ordinal.checked_add(delta).filter(|n| *n < (1u64 << 56)).ok_or("sequence ordinal exhausted")?; + let packet = Packet { values: frame.amplitudes.iter().map(|x| *x as f32).collect(), + frame_id: (u128::from(self.prefix) << 64) | (u128::from(frame.source_id) << 56) | u128::from(ordinal), + ordinal, received_ns: now_ns }; + source.sequence = Some(frame.sequence); + source.ordinal = ordinal; + source.last_time_ns = now_ns; + source.admitted_packets = source.admitted_packets.saturating_add(1); + self.last_seen_ns = now_ns; + push_bounded(&mut source.live, packet.clone()); + let Some(phase) = self.phase.as_mut() else { return Ok(()); }; + if now_ns < phase.started_ns { return self.invalidate("receiver time precedes phase start"); } + let capture_start = phase.started_ns + phase.request.lead_in_ms * 1_000_000; + if now_ns < capture_start { return Ok(()); } + let elapsed = now_ns - capture_start; + let duration_ns = phase.request.duration_ms * 1_000_000; + if elapsed >= duration_ns { return Ok(()); } + *phase.admitted.entry(frame.source_id).or_default() += 1; + let bucket = ((elapsed * COHORTS_PER_PHASE as u64) / duration_ns) as usize; + if bucket != phase.bucket { phase.pending.clear(); phase.bucket = bucket; } + // Exactly one complete nonoverlapping representative per quarter. + if phase.representatives.iter().any(|c| c.id.ends_with(&format!("-bucket-{bucket}"))) { return Ok(()); } + push_bounded(phase.pending.entry(frame.source_id).or_default(), packet); + if phase.pending.len() != self.context.sources.len() + || phase.pending.values().any(|v| v.len() != WINDOW_PACKETS) { return Ok(()); } + let id = format!("{}-phase-{}-bucket-{bucket}", self.run_id, self.history.len()); + // Disjoint streams remain incomplete until a complete aligned window + // arrives. No pooling, padding, or fabricated source-clock alignment. + if let Ok(cohort) = assemble(&self.context, &phase.pending, now_ns, id, &phase.request) { + phase.representatives.push(cohort); phase.pending.clear(); + } + Ok(()) + } + + pub fn finish_phase(&mut self, now_ns: u64) -> Result { + let phase = self.phase.as_ref().ok_or("no active capture phase")?; + if now_ns < self.last_seen_ns || now_ns < phase.started_ns || now_ns - phase.started_ns < (phase.request.duration_ms + phase.request.lead_in_ms) * 1_000_000 { + return Err("phase duration has not elapsed".into()); + } + if phase.representatives.len() != COHORTS_PER_PHASE { + return self.invalidate("phase lacks four complete aligned representative windows"); + } + let phase = self.phase.take().ok_or("no active capture phase")?; + let receipt = receipt(&phase, PhaseStatus::Sealed, None); + let mut combined = self.cohorts.clone(); + combined.extend(phase.representatives); + if let Err(error) = ValidatedDataset::new(self.context.clone(), combined.clone()) { + let failed = PhaseReceipt { status: PhaseStatus::Failed, cohort_count: 0, reason: Some(error.clone()), ..receipt }; + self.history.push(failed); + return Err(error); + } + self.cohorts = combined; + self.history.push(receipt.clone()); + Ok(receipt) + } + + pub fn cancel_phase(&mut self) { + if let Some(phase) = self.phase.take() { + self.history.push(receipt(&phase, PhaseStatus::Cancelled, Some("operator cancelled phase".into()))); + } + } + + pub fn live_cohort(&self, now_ns: u64) -> Result { + let buffers = self.sources.iter().map(|(id,s)| (*id,s.live.clone())).collect(); + let request = PhaseRequest { trial_id: "live-unlabelled".into(), session_id: self.run_id.clone(), + role: DataRole::Inference, label: RoomLabel::Unknown, duration_ms: MIN_PHASE_MS, lead_in_ms: 0, conditions_confirmed: false }; + assemble(&self.context, &buffers, now_ns, format!("{}-live-{now_ns}", self.run_id), &request) + } + + fn invalidate(&mut self, reason: &str) -> Result { + if let Some(phase) = self.phase.take() { + self.history.push(receipt(&phase, PhaseStatus::Failed, Some(reason.into()))); + } + for source in self.sources.values_mut() { source.live.clear(); } + Err(reason.into()) + } +} + +fn assemble(context: &CaptureContext, buffers: &HashMap>, now_ns: u64, + id: String, request: &PhaseRequest) -> Result { + let mut views = Vec::with_capacity(context.sources.len()); + for spec in &context.sources { + let rows = buffers.get(&spec.source_id).filter(|v| v.len() == WINDOW_PACKETS).ok_or("incomplete source window")?; + let first = rows.front().ok_or("empty source window")?.received_ns; + let last = rows.back().ok_or("empty source window")?.received_ns; + let age = now_ns.checked_sub(first).filter(|age| *age <= MAX_AGE_NS).ok_or("stale or future source window")?; + views.push(SourceView { source_id: spec.source_id, device_id: spec.device_id.clone(), grid_id: spec.grid_id.clone(), + signal_kind: spec.signal_kind, center_frequency_hz: spec.center_frequency_hz, bandwidth_hz: spec.bandwidth_hz, + link_id: spec.link_id.clone(), link_index: spec.link_index, geometry: spec.geometry, + samples: rows.iter().map(|p| p.values.clone()).collect(), frame_ids: rows.iter().map(|p| p.frame_id).collect(), + sequence_ids: rows.iter().map(|p| p.ordinal).collect(), first_time_ns: first, last_time_ns: last, + clock: ClockInfo { kind: ClockKind::ReceiverOnly, uncertainty_ns: u64::MAX }, sample_age_ns: age, + phase: PhaseAvailability::MagnitudeOnly, provenance: CaptureProvenance::LiveReceiverObserved }); + } + let cohort = CaptureCohort { id, context_digest: context.digest(), trial_id: request.trial_id.clone(), + session_id: request.session_id.clone(), role: request.role, label: request.label, views, teacher: None, + provenance: CaptureProvenance::LiveReceiverObserved }; + validate_cohort(context, &cohort)?; + Ok(cohort) +} +fn push_bounded(buffer: &mut VecDeque, packet: Packet) { + if buffer.len() == WINDOW_PACKETS { buffer.pop_front(); } + buffer.push_back(packet); +} +fn receipt(phase: &Phase, status: PhaseStatus, reason: Option) -> PhaseReceipt { + PhaseReceipt { trial_id: phase.request.trial_id.clone(), session_id: phase.request.session_id.clone(), + role: phase.request.role, label: phase.request.label, status, + cohort_count: if status == PhaseStatus::Sealed { phase.representatives.len() } else { 0 }, reason } +} +fn identifier(s: &str) -> Result<(), String> { + if s.is_empty() || s.len() > 128 || !s.is_ascii() || s.trim() != s || s.chars().any(char::is_control) { + Err("invalid bounded capture identifier".into()) + } else { Ok(()) } +} + +#[cfg(test)] +#[path = "capture_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_tests.rs new file mode 100644 index 0000000000..6d6b31f93d --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_tests.rs @@ -0,0 +1,195 @@ +//! SYNTHETIC packets exercising production public APIs; no hardware evidence. +use super::*; +use ruview_unified::RfModality; + +const START: u64 = 1_000_000_000; +fn context() -> CaptureContext { + CaptureContext { owner_scope:"synthetic-owner".into(),installation_id:"synthetic-install".into(), + room_digest:"synthetic-room".into(),revision:1,sources:(1..=2).map(|id| SourceSpec { + source_id:id,device_id:format!("synthetic-{id}"),grid_id:"ht20-64-raw".into(), + feature_count:64,signal_kind:RfModality::WifiCsi,center_frequency_hz:2_437_000_000, + bandwidth_hz:20_000_000,link_id:format!("tx-rx-{id}"),link_index:0,geometry:None }).collect() } +} +fn session() -> CaptureSession { CaptureSession::new(context(),"synthetic-run".into()).unwrap() } +fn request(role:DataRole,label:RoomLabel,id:&str) -> PhaseRequest { + PhaseRequest {trial_id:format!("synthetic-trial-{id}"),session_id:format!("synthetic-session-{id}"), + role,label,duration_ms:MIN_PHASE_MS, lead_in_ms: 0,conditions_confirmed:true} +} +fn packet(source_id:u8,sequence:u32,values:&[f64]) -> RawSourceFrame<'_> { + RawSourceFrame {source_id,grid_id:"ht20-64-raw",frequency_hz:2_437_000_000, + bandwidth_hz:20_000_000,sequence,amplitudes:values} +} +fn window(s:&mut CaptureSession,at:u64,start_sequence:u32,gain:f64) { + let values=vec![gain;64]; + for t in 0..WINDOW_PACKETS { + for source in 1..=2 { + s.ingest(packet(source,start_sequence.wrapping_add(t as u32),&values), + at+t as u64*10_000_000+u64::from(source)).unwrap(); + } + } +} +fn phase(s:&mut CaptureSession,at:u64,seq:u32,role:DataRole,label:RoomLabel,id:&str,gain:f64) { + s.begin_phase(request(role,label,id),at).unwrap(); + for quarter in 0..COHORTS_PER_PHASE { + window(s,at+quarter as u64*3_000_000_000+1_000_000,seq+quarter as u32*8,gain); + } + s.finish_phase(at+MIN_PHASE_MS*1_000_000).unwrap(); +} + +#[test] +fn four_confirmed_phase_quarters_seal_complete_source_cohorts_without_raw_receipt() { + let mut s=session(); + phase(&mut s,START,1,DataRole::Train,RoomLabel::Empty,"empty",2.0); + assert_eq!(s.cohorts().len(),4); + assert!(s.cohorts().iter().all(|c|c.views.len()==2 && c.views.iter().all(|v|v.samples.len()==8))); + assert_eq!(s.history()[0].status,PhaseStatus::Sealed); + let json=serde_json::to_string(&s.history()[0]).unwrap(); + assert!(!json.contains("samples") && !json.contains("amplitudes")); + assert!(!s.validated_dataset().unwrap().measurement_clock_available()); +} +#[test] +fn live_prediction_is_unknown_and_receiver_only_never_an_empty_reference() { + let mut s=session();window(&mut s,START,1,7.0); + let c=s.live_cohort(START+80_000_000).unwrap(); + assert_eq!((c.role,c.label),(DataRole::Inference,RoomLabel::Unknown)); + assert!(c.teacher.is_none()); + assert!(c.views.iter().all(|v|v.clock.kind==ClockKind::ReceiverOnly + && v.phase==PhaseAvailability::MagnitudeOnly && v.provenance==CaptureProvenance::LiveReceiverObserved)); + assert!(ValidatedDataset::new(context(),vec![c]).is_err()); +} +#[test] +fn gain_and_mean_shift_survive_distinct_operator_phase_labels() { + let mut s=session(); + phase(&mut s,START,1,DataRole::Train,RoomLabel::Empty,"empty",2.0); + phase(&mut s,START+13_000_000_000,33,DataRole::Train,RoomLabel::OccupiedStatic,"static",8.0); + assert!(s.cohorts()[0].views[0].samples.iter().flatten().all(|v|*v==2.0)); + assert!(s.cohorts()[4].views[0].samples.iter().flatten().all(|v|*v==8.0)); +} +#[test] +fn missing_source_dropout_never_pads_or_seals_partial_phase() { + let mut s=session();s.begin_phase(request(DataRole::HeldOut,RoomLabel::OccupiedMoving,"dropout"),START).unwrap(); + let values=vec![3.0;64]; + for t in 0..64 {s.ingest(packet(1,t,&values),START+1+t as u64*10_000_000).unwrap();} + assert!(s.finish_phase(START+MIN_PHASE_MS*1_000_000).is_err()); + assert!(s.cohorts().is_empty()); + assert_eq!(s.history()[0].status,PhaseStatus::Failed); + assert!(s.history()[0].reason.as_ref().unwrap().contains("four complete")); +} +#[test] +fn cancel_drops_only_inflight_phase_and_keeps_sealed_role_history() { + let mut s=session();phase(&mut s,START,1,DataRole::Train,RoomLabel::Empty,"sealed",2.0); + s.begin_phase(request(DataRole::Validation,RoomLabel::Empty,"cancel"),START+13_000_000_000).unwrap(); + window(&mut s,START+13_001_000_000,33,5.0);s.cancel_phase(); + assert_eq!(s.cohorts().len(),4);assert_eq!(s.history()[1].status,PhaseStatus::Cancelled); + assert!(!s.phase_active()); +} +#[test] +fn source_phy_grid_unknown_source_and_nonfinite_changes_fail_active_phase() { + for change in 0..5 { + let mut s=session();s.begin_phase(request(DataRole::Train,RoomLabel::Empty,"bad"),START).unwrap(); + let mut values=vec![2.0;64];if change==4{values[0]=f64::NAN;} + let mut f=packet(1,1,&values); + match change {0=>f.source_id=3,1=>f.grid_id="different-phy",2=>f.frequency_hz+=5_000_000, + 3=>f.bandwidth_hz*=2,_=>{}} + assert!(s.ingest(f,START+1).is_err());assert!(!s.phase_active()); + assert_eq!(s.history()[0].status,PhaseStatus::Failed); + } +} +#[test] +fn forward_sequence_wrap_extends_ordinals_and_replay_is_rejected() { + let mut s=session();window(&mut s,START,u32::MAX-3,2.0); + let c=s.live_cohort(START+80_000_000).unwrap(); + assert!(c.views.iter().all(|v|v.sequence_ids.windows(2).all(|w|w[1]>w[0]))); + let values=vec![2.0;64]; + assert!(s.ingest(packet(1,3,&values),START+90_000_000).unwrap_err().contains("replayed")); + assert!(s.ingest(packet(1,2,&values),START+91_000_000).is_err()); +} +#[test] +fn phase_boundary_never_reuses_live_window_or_previous_phase_frames() { + let mut s=session();window(&mut s,START,1,99.0); + let begin=START+100_000_000;s.begin_phase(request(DataRole::Train,RoomLabel::Empty,"first"),begin).unwrap(); + let values=vec![2.0;64]; + for t in 0..7 {for source in 1..=2 {s.ingest(packet(source,9+t,&values), + begin+1+t as u64*10_000_000+u64::from(source)).unwrap();}} + assert!(s.finish_phase(begin+MIN_PHASE_MS*1_000_000).is_err()); + assert!(s.cohorts().is_empty()); +} +#[test] +fn interleaved_train_validation_holdout_keep_distinct_frames_and_frozen_roles() { + let mut s=session(); + for (i,role) in [DataRole::Train,DataRole::HeldOut,DataRole::Validation].into_iter().enumerate() { + phase(&mut s,START+i as u64*13_000_000_000,1+i as u32*32,role,RoomLabel::Empty,&format!("{i}"),2.0); + } + let dataset=s.validated_dataset().unwrap(); + assert_eq!((dataset.train().len(),dataset.validation().len(),dataset.held_out().len()),(4,4,4)); + let mut wrong=request(DataRole::Validation,RoomLabel::Empty,"0");wrong.session_id="new-session".into(); + assert!(s.begin_phase(wrong,START+40_000_000_000).unwrap_err().contains("frozen")); +} +#[test] +fn preconditions_duration_and_unlabelled_training_requests_are_rejected() { + for change in 0..5 { + let mut s=session();let mut p=request(DataRole::Train,RoomLabel::Empty,"invalid"); + match change {0=>p.conditions_confirmed=false,1=>p.duration_ms=MIN_PHASE_MS-1, + 2=>p.duration_ms=MAX_PHASE_MS+1,3=>p.label=RoomLabel::Unknown,_=>p.role=DataRole::Inference} + assert!(s.begin_phase(p,START).is_err()); + } +} +#[test] +fn early_finish_leaves_phase_active_and_live_staleness_is_checked() { + let mut s=session();s.begin_phase(request(DataRole::Train,RoomLabel::Empty,"early"),START).unwrap(); + window(&mut s,START+1_000_000,1,2.0); + assert!(s.finish_phase(START+1_000_000_000).is_err());assert!(s.phase_active()); + assert!(s.live_cohort(START+MAX_AGE_NS+100_000_000).is_err()); +} +#[test] +fn quarter_windows_are_nonoverlapping_even_under_high_packet_rate() { + let mut s=session();s.begin_phase(request(DataRole::Train,RoomLabel::Empty,"rate"),START).unwrap(); + for w in 0..10 {window(&mut s,START+1_000_000+w as u64*100_000_000,1+w*8,2.0);} + for q in 1..4 {window(&mut s,START+q as u64*3_000_000_000+1_000_000,81+(q-1)*8,2.0);} + let receipt=s.finish_phase(START+MIN_PHASE_MS*1_000_000).unwrap(); + assert_eq!(receipt.cohort_count,4); + for pair in s.cohorts().windows(2) {assert!(pair[0].views[0].last_time_ns { + pub node_id: u8, + pub n_antennas: usize, + pub raw_bins: usize, + pub frequency_hz: u64, + pub ppdu_type: PpduType, + pub sequence: u32, + pub received_at_ns: u64, + /// Actual measured receiver/grid cadence, never a source sample-rate claim. + pub receiver_sample_rate_hz: f64, + pub iq_bytes: &'a [u8], + pub flags: Adr018Flags, + pub provenance: CaptureProvenance, +} + +/// Contains raw native samples and deliberately has no Serialize implementation. +/// Only `summary()` is appropriate for public/status export. +/// +/// ```compile_fail +/// use wifi_densepose_sensing_server::room_learner::native::CanonicalPacket; +/// fn export(packet: &CanonicalPacket) { let _ = serde_json::to_string(packet); } +/// ``` +pub struct CanonicalPacket { + pub native_frame: RfFrameV2, + pub source_id: u8, + pub grid_id: String, + pub sequence: u32, + pub frequency_hz: u64, + pub bandwidth_hz: u64, + pub amplitudes: Vec, + pub clock: ClockInfo, + pub phase_consumed: bool, + pub receiver_sample_rate_hz: f64, + pub temporal_sample_rate_known: bool, + pub signal_quality_known: bool, + pub provenance: CaptureProvenance, +} + +/// Bounded counts/metadata, never native IQ or derived amplitude rows. +#[derive(Debug, Serialize)] +pub struct NativePacketSummary<'a> { + pub source_id: u8, + pub grid_id: &'a str, + pub raw_bins: usize, + pub canonical_bins: usize, + pub frequency_hz: u64, + pub bandwidth_hz: u64, + pub clock: ClockInfo, + pub phase_consumed: bool, + pub receiver_sample_rate_hz: f64, + pub temporal_sample_rate_known: bool, + pub signal_quality_known: bool, + pub provenance: CaptureProvenance, +} +impl CanonicalPacket { + pub fn summary(&self) -> NativePacketSummary<'_> { + NativePacketSummary { source_id: self.source_id, grid_id: &self.grid_id, + raw_bins: self.native_frame.native_shape[1], canonical_bins: self.amplitudes.len(), + frequency_hz: self.frequency_hz, bandwidth_hz: self.bandwidth_hz, clock: self.clock, + phase_consumed: self.phase_consumed, receiver_sample_rate_hz: self.receiver_sample_rate_hz, + temporal_sample_rate_known: self.temporal_sample_rate_known, + signal_quality_known: self.signal_quality_known, provenance: self.provenance } + } +} +impl std::fmt::Debug for CanonicalPacket { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.summary().fmt(f) + } +} + +/// Fixed projection identity binds original hardware grid plus excluded region. +pub fn grid_id(n_antennas: usize, raw_bins: usize, ppdu_type: PpduType) -> Result { + let supported = n_antennas == 1 && match ppdu_type { + PpduType::HtLegacy => raw_bins == 64, + PpduType::HeSu | PpduType::HeMu | PpduType::HeTb => raw_bins == 256, + PpduType::Unknown => false, + }; + if !supported { return Err("unsupported ESP32 antenna/grid/PPDU".into()); } + Ok(format!("ant{n_antennas}-bins{raw_bins}-ppdu{}-usable-drop2-v1", ppdu_type.to_byte())) +} + +/// Preserves original signed int8 complex samples; uses only the existing +/// length-only normalizer for the derived magnitude features. Never calls +/// `normalize()` or `RfFrameV2::to_canonical()` (both remove amplitude gain). +pub fn canonicalize_esp32(input: NativePacketInput<'_>) -> Result { + let grid_id = grid_id(input.n_antennas, input.raw_bins, input.ppdu_type)?; + if input.flags.bw40 || input.received_at_ns == 0 || !input.receiver_sample_rate_hz.is_finite() + || input.receiver_sample_rate_hz <= 0.0 { + return Err("unsupported bandwidth, receiver time or measured receiver cadence".into()); + } + let mhz = input.frequency_hz / 1_000_000; + if input.frequency_hz % 1_000_000 != 0 + || !((2412..=2484).contains(&mhz) || (4900..=5900).contains(&mhz) || (5925..=7125).contains(&mhz)) { + return Err("unknown or unsupported ESP32 centre frequency".into()); + } + let expected = input.raw_bins * 2; + if input.iq_bytes.len() != expected { + return Err("ESP32 native IQ payload must exactly match bounded declared grid".into()); + } + // ADR-356 / firmware csi_collector.c: the invalid first word is FOUR + // bytes = TWO complex int8 I/Q samples. Keep those bytes unchanged and + // record their invalidity; every packet uses the same drop2 projection. + let mut valid_mask = vec![true; input.raw_bins]; + if input.flags.first_word_sanitized { valid_mask[..2].fill(false); } + let iq: Vec<_> = input.iq_bytes.chunks_exact(2).map(|pair| + Complex64::new(f64::from(pair[0] as i8), f64::from(pair[1] as i8))).collect(); + let frame_id = (u128::from(input.received_at_ns) << 40) + | (u128::from(input.node_id) << 32) | u128::from(input.sequence); + let (class, evidence) = match input.provenance { + CaptureProvenance::Synthetic => (ProvenanceClass::Synthetic, EvidenceLevel::L0Simulation), + CaptureProvenance::CapturedReplay | CaptureProvenance::LiveReceiverObserved => + (ProvenanceClass::Measured, EvidenceLevel::L1CapturedReplay), + }; + let native_frame = RfFrameV2::new(frame_id, input.received_at_ns, RfModality::WifiCsi, + vec![FieldAxis::Antenna, FieldAxis::Frequency, FieldAxis::Time], + input.frequency_hz as f64, 20_000_000.0, + // This complete frame is receiver-observed. The time axis and rate + // describe measured host reception, never a native/source clock. + input.receiver_sample_rate_hz, vec![1, input.raw_bins, 1], iq, valid_mask, None, None, vec![], 0, + CalibrationState { phase_state: PhaseState::Raw, gain_calibrated: false, + clock_ppm: f64::MAX, baseline_id: None, confidence: 0.0 }, + // No RSSI/noise/loss metadata in this narrow input: neutral constructor + // placeholders, explicitly unavailable via signal_quality_known=false. + SignalQuality { rssi_dbm: 0.0, noise_floor_dbm: 0.0, packet_loss: 0.0, interference: 0.0 }, + FrameProvenance { class, evidence, device_id: format!("esp32-node-{}", input.node_id), + firmware: "unreported".into(), receipt_id: frame_id }).map_err(|e| e.to_string())?; + if native_frame.valid_mask[2..].iter().any(|valid| !valid) { + return Err("retained native region contains an invalid sample".into()); + } + let magnitudes: Vec<_> = native_frame.native_iq[2..].iter().map(|z| z.norm()).collect(); + let amplitudes = HardwareNormalizer::new().resample_to_canonical(&magnitudes); + if amplitudes.len() != CANONICAL_PACKET_BINS || amplitudes.iter().any(|v| !v.is_finite() || *v < 0.0) { + return Err("length-only magnitude projection produced an invalid feature vector".into()); + } + Ok(CanonicalPacket { native_frame, source_id: input.node_id, + grid_id, + sequence: input.sequence, frequency_hz: input.frequency_hz, bandwidth_hz: 20_000_000, + amplitudes, clock: ClockInfo { kind: ClockKind::ReceiverOnly, uncertainty_ns: u64::MAX }, + phase_consumed: false, receiver_sample_rate_hz: input.receiver_sample_rate_hz, temporal_sample_rate_known: false, signal_quality_known: false, + provenance: input.provenance }) +} + +#[cfg(test)] +#[path = "native_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/native_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/native_tests.rs new file mode 100644 index 0000000000..0854d5fe68 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/native_tests.rs @@ -0,0 +1,148 @@ +//! SYNTHETIC wire payloads through the real native and signal contracts. +//! These test contract behavior, never sensing accuracy or real hardware. +use super::*; + +fn bytes(bins:usize,i:i8,q:i8)->Vec { + (0..bins).flat_map(|_|[i as u8,q as u8]).collect() +} +fn input(payload:&[u8],bins:usize)->NativePacketInput<'_> { + NativePacketInput {node_id:7,n_antennas:1,raw_bins:bins,frequency_hz:2_437_000_000, + ppdu_type:if bins==64 {PpduType::HtLegacy} else {PpduType::HeSu},sequence:23, + received_at_ns:1_234_000_000,receiver_sample_rate_hz:42.0,iq_bytes:payload,flags:Adr018Flags::default(), + provenance:CaptureProvenance::Synthetic} +} + +#[test] +fn ht64_packet_retains_original_signed_iq_phase_mask_and_unknown_geometry() { + let payload=bytes(64,-3,4); + let packet=canonicalize_esp32(input(&payload,64)).unwrap(); + assert_eq!(packet.native_frame.native_shape,vec![1,64,1]); + assert_eq!(packet.native_frame.axes,vec![FieldAxis::Antenna,FieldAxis::Frequency,FieldAxis::Time]); + assert!(packet.native_frame.native_iq.iter().all(|z|z.re==-3.0 && z.im==4.0)); + assert_eq!(packet.native_frame.native_iq[0].arg(),4.0f64.atan2(-3.0)); + assert!(packet.native_frame.valid_mask.iter().all(|v|*v)); + assert!(packet.native_frame.transmitter_pose.is_none() && packet.native_frame.receiver_pose.is_none() + && packet.native_frame.antenna_geometry.is_empty()); + assert_eq!(packet.grid_id,"ant1-bins64-ppdu0-usable-drop2-v1"); +} +#[test] +fn he256_packet_uses_actual_frame_contract_and_existing_56_bin_projection() { + let payload=bytes(256,6,8); + let packet=canonicalize_esp32(input(&payload,256)).unwrap(); + let direct=HardwareNormalizer::new().resample_to_canonical(&vec![10.0;254]); + assert_eq!(packet.amplitudes,direct); + assert_eq!(packet.native_frame.native_shape,vec![1,256,1]); + assert_eq!(packet.grid_id,"ant1-bins256-ppdu1-usable-drop2-v1"); + assert_eq!(packet.amplitudes.len(),56); +} +#[test] +fn gain_and_dc_levels_survive_canonicalization_without_median_or_zscore() { + let low=bytes(64,3,4);let high=bytes(64,9,12); + let a=canonicalize_esp32(input(&low,64)).unwrap(); + let b=canonicalize_esp32(input(&high,64)).unwrap(); + assert!(a.amplitudes.iter().all(|v|*v==5.0)); + assert!(b.amplitudes.iter().all(|v|*v==15.0)); + assert_eq!(a.native_frame.native_iq[0],Complex64::new(3.0,4.0)); +} +#[test] +fn varying_native_iq_is_not_reconstructed_from_magnitude_or_overwritten() { + let payload:Vec<_>=(0..64).flat_map(|n|[(n as i8-32) as u8,(-(n as i8)/2) as u8]).collect(); + let packet=canonicalize_esp32(input(&payload,64)).unwrap(); + for (sample,pair) in packet.native_frame.native_iq.iter().zip(payload.chunks_exact(2)) { + assert_eq!((sample.re,sample.im),(f64::from(pair[0] as i8),f64::from(pair[1] as i8))); + } +} +#[test] +fn raw_phase_is_retained_but_amplitude_learner_consumes_no_phase_or_source_clock() { + let payload=bytes(64,3,4);let mut input=input(&payload,64); + input.flags.ieee802154_sync_valid=true; + let packet=canonicalize_esp32(input).unwrap(); + assert_eq!(packet.native_frame.calibration.phase_state,PhaseState::Raw); + assert!(!packet.phase_consumed && !packet.temporal_sample_rate_known && !packet.signal_quality_known); + assert_eq!(packet.clock,ClockInfo {kind:ClockKind::ReceiverOnly,uncertainty_ns:u64::MAX}); + assert_eq!(packet.native_frame.timestamp_ns,1_234_000_000); + assert_eq!(packet.native_frame.calibration.confidence,0.0); + assert!(!packet.native_frame.calibration.gain_calibrated); +} +#[test] +fn provenance_never_grants_lab_or_physical_qualification() { + for provenance in [CaptureProvenance::Synthetic,CaptureProvenance::CapturedReplay,CaptureProvenance::LiveReceiverObserved] { + let payload=bytes(64,3,4);let mut input=input(&payload,64);input.provenance=provenance; + let packet=canonicalize_esp32(input).unwrap(); + let expected=if provenance==CaptureProvenance::Synthetic {(ProvenanceClass::Synthetic,EvidenceLevel::L0Simulation)} + else {(ProvenanceClass::Measured,EvidenceLevel::L1CapturedReplay)}; + assert_eq!((packet.native_frame.provenance.class,packet.native_frame.provenance.evidence),expected); + assert_eq!(packet.provenance,provenance); + assert_eq!(packet.clock.kind,ClockKind::ReceiverOnly); + } +} +#[test] +fn public_summary_and_debug_never_export_native_samples_amplitudes_or_frame_ids() { + let payload=bytes(64,-3,4);let packet=canonicalize_esp32(input(&payload,64)).unwrap(); + let public=serde_json::to_string(&packet.summary()).unwrap();let debug=format!("{packet:?}"); + for sensitive in ["native_iq","amplitudes","samples","frame_id","valid_mask"] { + assert!(!public.contains(sensitive));assert!(!debug.contains(sensitive)); + } + assert!(public.contains("ReceiverOnly") && public.contains("\"phase_consumed\":false")); +} +#[test] +fn malformed_grid_payload_bandwidth_frequency_and_validity_fail_closed() { + for change in 0..11 { + let payload=bytes(256,3,4);let mut i=input(&payload[..128],64); + match change {0=>i.n_antennas=0,1=>i.n_antennas=2,2=>i.raw_bins=usize::MAX, + 3=>i.raw_bins=128,4=>i.ppdu_type=PpduType::Unknown,5=>i.ppdu_type=PpduType::HeSu, + 6=>i.flags.bw40=true,7=>i.frequency_hz=0,8=>i.frequency_hz=2_437_000_001, + 9=>i.received_at_ns=0,_=>i.iq_bytes=&payload} + assert!(canonicalize_esp32(i).is_err(),"survived input mutant {change}"); + } +} +#[test] +fn iq_wire_integer_domain_cannot_produce_nan_or_inf_native_samples() { + let payload:Vec<_>=(0u8..=255).cycle().take(512).collect(); + let packet=canonicalize_esp32(input(&payload,256)).unwrap(); + assert!(packet.native_frame.native_iq.iter().all(|z|z.re.is_finite() && z.im.is_finite())); + assert!(packet.amplitudes.iter().all(|x|x.is_finite())); +} +#[test] +fn exact_declared_packet_size_is_required_without_padding_or_trailing_data() { + for size in [0,1,127,129,1024] { + let payload=vec![1;size]; + assert!(canonicalize_esp32(input(&payload,64)).is_err()); + } +} +#[test] +fn actual_raw_grid_identity_survives_both_supported_families() { + for ppdu in [PpduType::HtLegacy,PpduType::HeSu,PpduType::HeMu,PpduType::HeTb] { + let bins=if ppdu==PpduType::HtLegacy {64} else {256};let payload=bytes(bins,3,4); + let mut i=input(&payload,bins);i.ppdu_type=ppdu; + let packet=canonicalize_esp32(i).unwrap(); + assert_eq!(packet.grid_id,format!("ant1-bins{bins}-ppdu{}-usable-drop2-v1",ppdu.to_byte())); + assert_eq!((packet.frequency_hz,packet.bandwidth_hz),(2_437_000_000,20_000_000)); + } +} + +#[test] +fn sanitized_word_is_retained_with_two_invalid_samples_and_fixed_projection() { + let original=bytes(256,3,4);let mut sanitized=original.clone();sanitized[..4].fill(0); + let normal=canonicalize_esp32(input(&original,256)).unwrap(); + let mut i=input(&sanitized,256);i.flags.first_word_sanitized=true; + let masked=canonicalize_esp32(i).unwrap(); + assert_eq!(normal.amplitudes,masked.amplitudes); + assert_eq!(normal.grid_id,masked.grid_id); + assert_eq!(&masked.native_frame.valid_mask[..2],&[false,false]); + assert!(masked.native_frame.valid_mask[2..].iter().all(|v|*v)); + assert_eq!(masked.native_frame.native_iq[0],Complex64::new(0.0,0.0)); + assert_eq!(normal.native_frame.native_iq[0],Complex64::new(3.0,4.0)); +} +#[test] +fn source_record_requires_actual_receiver_rate_and_never_invents_one_hz() { + let payload=bytes(64,3,4); + for rate in [0.0,-1.0,f64::NAN,f64::INFINITY,f64::NEG_INFINITY] { + let mut i=input(&payload,64);i.receiver_sample_rate_hz=rate; + assert!(canonicalize_esp32(i).is_err()); + } + let packet=canonicalize_esp32(input(&payload,64)).unwrap(); + assert_eq!(packet.native_frame.sample_rate_hz,42.0); + assert_eq!(packet.receiver_sample_rate_hz,42.0); + assert!(!packet.temporal_sample_rate_known); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/observation.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/observation.rs new file mode 100644 index 0000000000..2e77b75bdc --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/observation.rs @@ -0,0 +1,421 @@ +//! Source-preserving RF views and frozen, leakage-free capture cohorts. +//! All declarations are input claims: validation never grants physical authority. +use ruview_unified::{LinkGeometry, RfModality, RfTensor}; +use ruview_unified::frame::{FieldAxis, PhaseState, ProvenanceClass, RfFrameV2}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::{HashMap, HashSet}; + +pub const MAX_SOURCES: usize = 16; +pub const MAX_FEATURES: usize = 512; +pub const MAX_ROWS: usize = 256; +pub const MAX_COHORTS: usize = 128; +pub const MAX_TOTAL_VALUES: usize = 2_000_000; +pub const MAX_AGE_NS: u64 = 5_000_000_000; +pub const MAX_WINDOW_NS: u64 = 60_000_000_000; +pub const MAX_CLOCK_UNCERTAINTY_NS: u64 = 100_000_000; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct SourceSpec { + pub source_id: u8, + pub device_id: String, + pub grid_id: String, + pub feature_count: usize, + pub signal_kind: RfModality, + pub center_frequency_hz: u64, + pub bandwidth_hz: u64, + pub link_id: String, + pub link_index: usize, + pub geometry: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct CaptureContext { + pub owner_scope: String, + pub installation_id: String, + pub room_digest: String, + pub revision: u64, + pub sources: Vec, +} + +impl CaptureContext { + /// SHA-256 of the full ordered source/configuration contract, not RF data. + pub fn digest(&self) -> String { + let bytes = serde_json::to_vec(self).unwrap_or_default(); + format!("{:x}", Sha256::digest(bytes)) + } + + pub fn validate(&self) -> Result<(), String> { + for s in [&self.owner_scope, &self.installation_id, &self.room_digest] { identifier(s)?; } + if self.revision == 0 || self.sources.is_empty() || self.sources.len() > MAX_SOURCES { + return Err("invalid context revision or source count".into()); + } + if self.sources.windows(2).any(|w| w[0].source_id >= w[1].source_id) { + return Err("source specs must be sorted uniquely by source id".into()); + } + let mut ids = HashSet::new(); + let mut links = HashSet::new(); + for s in &self.sources { + for id in [&s.device_id, &s.grid_id, &s.link_id] { identifier(id)?; } + if !ids.insert(s.source_id) || !links.insert((&s.device_id, &s.link_id)) + || s.feature_count == 0 || s.feature_count > MAX_FEATURES || s.link_index >= MAX_SOURCES + || s.center_frequency_hz == 0 || s.bandwidth_hz == 0 { + return Err("duplicate source/link or invalid feature count".into()); + } + if let Some(g) = s.geometry { geometry(g)?; } + } + Ok(()) + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum ClockKind { SourceMeasurement, ReceiverOnly, Unknown } +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct ClockInfo { pub kind: ClockKind, pub uncertainty_ns: u64 } +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum CaptureProvenance { Synthetic, CapturedReplay, LiveReceiverObserved } +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum PhaseAvailability { Raw, Sanitized, Calibrated, MagnitudeOnly, Unknown } + +/// Explicit per-row identities/times supplied by ingestion; tensor timestamp alone +/// does not establish a source clock (#1710). +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ViewMetadata { + pub frame_ids: Vec, + pub sequence_ids: Vec, + pub first_time_ns: u64, + pub last_time_ns: u64, + pub clock: ClockInfo, + pub phase: PhaseAvailability, + pub link_index: usize, + pub provenance: CaptureProvenance, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceView { + pub source_id: u8, + pub device_id: String, + pub grid_id: String, + pub signal_kind: RfModality, + pub center_frequency_hz: u64, + pub bandwidth_hz: u64, + pub link_id: String, + pub link_index: usize, + pub geometry: Option, + pub samples: Vec>, + pub frame_ids: Vec, + pub sequence_ids: Vec, + pub first_time_ns: u64, + pub last_time_ns: u64, + pub clock: ClockInfo, + pub sample_age_ns: u64, + pub phase: PhaseAvailability, + pub provenance: CaptureProvenance, +} + +impl SourceView { + /// Reads magnitudes without per-window centering, scaling or gain removal. + /// Unified adapter normalization averages away per-link gain: it is not + /// invertible. Such tensors must use the native-frame entry point instead. + pub fn from_tensor(source: &SourceSpec, tensor: &RfTensor, meta: ViewMetadata) -> Result { + let (links, bins, rows) = tensor.dims(); + if tensor.calibration.gain_offset_db != 0.0 || tensor.calibration.baseline_id.is_some() { + return Err("gain-normalized/baseline-subtracted tensor cannot preserve source amplitude".into()); + } + if links != 1 || meta.link_index != source.link_index || meta.link_index >= links || tensor.links.len() != links || bins != source.feature_count + || tensor.device_id != source.device_id || tensor.modality != source.signal_kind + || tensor.center_freq_hz != source.center_frequency_hz as f64 + || tensor.bandwidth_hz != source.bandwidth_hz as f64 + || !tensor.sample_age_s.is_finite() || tensor.sample_age_s < 0.0 + || tensor.sample_age_s > MAX_AGE_NS as f64 / 1e9 + || rows == 0 || rows > MAX_ROWS || bins > MAX_FEATURES + || tensor.timestamp_ns != meta.last_time_ns { + return Err("tensor/source/window contract mismatch".into()); + } + if meta.phase == PhaseAvailability::Raw || meta.phase == PhaseAvailability::Sanitized + || (meta.phase == PhaseAvailability::Calibrated && !tensor.calibration.phase_calibrated) { + return Err("tensor does not establish requested phase state".into()); + } + let actual_geometry = tensor.links[meta.link_index]; + if source.geometry.is_some_and(|g| g != actual_geometry) { + return Err("tensor link geometry changed".into()); + } + let samples = (0..rows).map(|t| (0..bins) + .map(|b| tensor.data[[meta.link_index, b, t]].norm() as f32).collect()).collect(); + let view = Self::build(source, samples, meta, (tensor.sample_age_s * 1e9) as u64, + tensor.modality == RfModality::Synthetic); + view.validate(source)?; + Ok(view) + } + + /// Native IQ avoids the existing canonical adapter's median normalization. + /// Only an explicit `(link, bin, time)` layout with complete valid samples + /// is accepted; missing bins cannot silently become measurements. + pub fn from_frame(source: &SourceSpec, frame: &RfFrameV2, meta: ViewMetadata) -> Result { + if frame.schema_version != 2 || frame.native_shape.len() != 3 + || frame.axes.len() != 3 || frame.axes[0] != FieldAxis::Antenna + || !matches!(frame.axes[1], FieldAxis::Frequency | FieldAxis::Delay | FieldAxis::Range) + || frame.axes[2] != FieldAxis::Time || frame.sample_age_ns > MAX_AGE_NS + || frame.modality != source.signal_kind || frame.provenance.device_id != source.device_id + || frame.centre_frequency_hz != source.center_frequency_hz as f64 + || frame.bandwidth_hz != source.bandwidth_hz as f64 + || frame.calibration.baseline_id.is_some() || frame.timestamp_ns != meta.last_time_ns { + return Err("native frame/source/layout contract mismatch".into()); + } + let (links, bins, rows) = (frame.native_shape[0], frame.native_shape[1], frame.native_shape[2]); + if links == 0 || links > MAX_SOURCES || bins != source.feature_count || bins > MAX_FEATURES + || rows == 0 || rows > MAX_ROWS || meta.link_index != source.link_index || meta.link_index >= links + || frame.native_iq.len() != links * bins * rows || frame.valid_mask.len() != links * bins * rows + || meta.frame_ids.iter().any(|id| *id != frame.frame_id) { + return Err("native shape/frame identity mismatch".into()); + } + let expected_phase = match frame.calibration.phase_state { + PhaseState::Raw => PhaseAvailability::Raw, + PhaseState::Sanitized => PhaseAvailability::Sanitized, + PhaseState::Calibrated => PhaseAvailability::Calibrated, + PhaseState::Unavailable => PhaseAvailability::MagnitudeOnly, + }; + if meta.phase != expected_phase { return Err("native phase declaration mismatch".into()); } + if let Some(g) = source.geometry { + if frame.transmitter_pose.map(|p| p.position_m) != Some(g.tx_pos) + || frame.receiver_pose.map(|p| p.position_m) != Some(g.rx_pos) { + return Err("native geometry changed".into()); + } + } + let mut samples = Vec::with_capacity(rows); + for t in 0..rows { + let mut row = Vec::with_capacity(bins); + for b in 0..bins { + let i = meta.link_index * bins * rows + b * rows + t; + if !frame.valid_mask[i] { return Err("native sample is invalid".into()); } + row.push(frame.native_iq[i].norm() as f32); + } + samples.push(row); + } + let view = Self::build(source, samples, meta, frame.sample_age_ns, + frame.provenance.class == ProvenanceClass::Synthetic); + view.validate(source)?; + Ok(view) + } + + fn build(s: &SourceSpec, samples: Vec>, m: ViewMetadata, sample_age_ns: u64, synthetic: bool) -> Self { + Self { source_id: s.source_id, device_id: s.device_id.clone(), grid_id: s.grid_id.clone(), + signal_kind: s.signal_kind, center_frequency_hz: s.center_frequency_hz, bandwidth_hz: s.bandwidth_hz, + link_id: s.link_id.clone(), link_index: s.link_index, geometry: s.geometry, samples, + frame_ids: m.frame_ids, sequence_ids: m.sequence_ids, first_time_ns: m.first_time_ns, + last_time_ns: m.last_time_ns, clock: m.clock, sample_age_ns, phase: m.phase, + provenance: if synthetic { CaptureProvenance::Synthetic } else { m.provenance } } + } + + pub fn validate(&self, s: &SourceSpec) -> Result<(), String> { + if self.source_id != s.source_id || self.grid_id != s.grid_id || self.device_id != s.device_id + || self.link_id != s.link_id || self.link_index != s.link_index || self.signal_kind != s.signal_kind || self.geometry != s.geometry + || self.center_frequency_hz != s.center_frequency_hz || self.bandwidth_hz != s.bandwidth_hz { + return Err("source identity, PHY grid, modality or geometry changed".into()); + } + if (self.signal_kind == RfModality::Synthetic && self.provenance != CaptureProvenance::Synthetic) + || self.samples.len() < 2 || self.samples.len() > MAX_ROWS + || self.frame_ids.len() != self.samples.len() || self.sequence_ids.len() != self.samples.len() + || self.first_time_ns == 0 || self.last_time_ns <= self.first_time_ns + || self.last_time_ns - self.first_time_ns > MAX_WINDOW_NS || self.sample_age_ns > MAX_AGE_NS + || self.frame_ids.contains(&0) || self.sequence_ids.windows(2).any(|w| w[1] <= w[0]) + || (self.clock.kind == ClockKind::SourceMeasurement + && self.clock.uncertainty_ns > MAX_CLOCK_UNCERTAINTY_NS) { + return Err("invalid bounded source window, identities, age or clock".into()); + } + if self.samples.iter().any(|row| row.len() != s.feature_count + || row.iter().any(|v| !v.is_finite() || *v < 0.0)) { + return Err("invalid magnitude sample/feature grid".into()); + } + Ok(()) + } +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub enum DataRole { Train, Validation, HeldOut, Inference } +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, Hash)] +pub enum RoomLabel { Empty, OccupiedStatic, OccupiedMoving, AdjacentOnly, Unknown } +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum TeacherKind { Camera, Radar } +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TeacherEvidence { + pub kind: TeacherKind, + pub source_id: String, + /// Unverified receipt claim; only an external paired registry can resolve it. + pub validation_receipt: String, + pub context_digest: String, + pub room_digest: String, + pub trial_id: String, + pub label: RoomLabel, + pub frame_ids: Vec, + pub first_time_ns: u64, + pub last_time_ns: u64, + pub clock: ClockInfo, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CaptureCohort { + pub id: String, + pub context_digest: String, + pub trial_id: String, + pub session_id: String, + pub role: DataRole, + pub label: RoomLabel, + pub views: Vec, + pub teacher: Option, + pub provenance: CaptureProvenance, +} + +/// Read-only after construction: held-out rows cannot accidentally be fitted. +#[derive(Debug, Clone)] +pub struct ValidatedDataset { + context: CaptureContext, + train: Vec, + validation: Vec, + held_out: Vec, + measurement_clock_available: bool, +} +type SourceRoleIntervals = HashMap<(u8, String), Vec<(u64, u64, DataRole)>>; +impl ValidatedDataset { + pub fn new(context: CaptureContext, cohorts: Vec) -> Result { + context.validate()?; + if cohorts.is_empty() || cohorts.len() > MAX_COHORTS { return Err("invalid cohort count".into()); } + let digest = context.digest(); + let mut result = Self { context, train: vec![], validation: vec![], held_out: vec![], measurement_clock_available: true }; + let mut total_values = 0usize; + let mut ids = HashSet::new(); + let mut trials = HashMap::new(); + let mut sessions = HashMap::new(); + let mut frames = HashMap::new(); + let mut sequences = HashMap::new(); + let mut intervals: SourceRoleIntervals = HashMap::new(); + for c in cohorts { + if c.role == DataRole::Inference || c.label == RoomLabel::Unknown { + return Err("inference/unlabelled input cannot enter a labelled dataset".into()); + } + validate_cohort(&result.context, &c)?; + for id in [&c.id, &c.trial_id, &c.session_id] { identifier(id)?; } + if !ids.insert(c.id.clone()) || c.context_digest != digest || c.views.len() != result.context.sources.len() { + return Err("duplicate cohort, wrong context or incomplete selected sources".into()); + } + for (map, key) in [(&mut trials, &c.trial_id), (&mut sessions, &c.session_id)] { + if map.insert(key.clone(), c.role).is_some_and(|role| role != c.role) { + return Err("trial/session crosses frozen roles".into()); + } + } + let mut selected = HashSet::new(); + let mut first = 0; + let mut last = u64::MAX; + for v in &c.views { + let spec = result.context.sources.iter().find(|s| s.source_id == v.source_id) + .ok_or("unknown source")?; + v.validate(spec)?; + total_values += v.samples.len() * spec.feature_count; + if total_values > MAX_TOTAL_VALUES { return Err("dataset value budget exceeded".into()); } + if !selected.insert(v.source_id) { return Err("duplicate source view".into()); } + result.measurement_clock_available &= v.provenance != CaptureProvenance::Synthetic + && v.clock.kind == ClockKind::SourceMeasurement; + first = first.max(v.first_time_ns); + last = last.min(v.last_time_ns); + for id in &v.frame_ids { + if frames.insert((v.source_id, *id), c.role).is_some_and(|r| r != c.role) { + return Err("frame overlap crosses roles".into()); + } + } + for seq in &v.sequence_ids { + if sequences.insert((v.source_id, c.session_id.clone(), *seq), c.id.clone()) + .is_some_and(|id| id != c.id) { return Err("reused source sequence".into()); } + } + let ranges = intervals.entry((v.source_id, c.session_id.clone())).or_default(); + if ranges.iter().any(|(a,b,r)| *r != c.role && v.first_time_ns <= *b && v.last_time_ns >= *a) { + return Err("source time window overlaps roles".into()); + } + ranges.push((v.first_time_ns, v.last_time_ns, c.role)); + } + if first >= last { return Err("selected source windows are not aligned".into()); } + if let Some(t) = &c.teacher { validate_teacher(t, &c, &result.context, first, last)?; } + // Labels without an independently validated teacher are still useful + // experimental/manual cohorts, never physical empty-room evidence. + + match c.role { DataRole::Train => result.train.push(c), DataRole::Validation => result.validation.push(c), DataRole::HeldOut => result.held_out.push(c), DataRole::Inference => return Err("inference dataset rejected".into()) } + } + Ok(result) + } + pub fn context(&self) -> &CaptureContext { &self.context } + pub fn train(&self) -> &[CaptureCohort] { &self.train } + pub fn validation(&self) -> &[CaptureCohort] { &self.validation } + pub fn held_out(&self) -> &[CaptureCohort] { &self.held_out } + /// Clock/provenance input claims only; never an activation or qualification receipt. + pub fn measurement_clock_available(&self) -> bool { self.measurement_clock_available } +} + +/// Validates one inference/capture boundary without assigning or changing splits. +pub fn validate_cohort(context: &CaptureContext, c: &CaptureCohort) -> Result<(), String> { + context.validate()?; + if (c.role == DataRole::Inference) != (c.label == RoomLabel::Unknown) + || (c.role == DataRole::Inference && c.teacher.is_some()) { + return Err("inference must be unlabelled and cannot carry teacher labels".into()); + } + for id in [&c.id, &c.trial_id, &c.session_id] { identifier(id)?; } + if c.context_digest != context.digest() || c.views.len() != context.sources.len() { + return Err("cohort context or selected source count mismatch".into()); + } + let mut ids = HashSet::new(); + let (mut first, mut last) = (0, u64::MAX); + for v in &c.views { + let spec = context.sources.iter().find(|s| s.source_id == v.source_id).ok_or("unknown source")?; + v.validate(spec)?; + if !ids.insert(v.source_id) || v.provenance != c.provenance { + return Err("duplicate view or inconsistent provenance".into()); + } + first = first.max(v.first_time_ns); + last = last.min(v.last_time_ns); + } + if first >= last { return Err("selected source windows are not aligned".into()); } + if let Some(t) = &c.teacher { validate_teacher(t, c, context, first, last)?; } + Ok(()) +} + +fn validate_teacher(t: &TeacherEvidence, c: &CaptureCohort, context: &CaptureContext, first: u64, last: u64) -> Result<(), String> { + // A client-supplied receipt string is never proof of a validated source. + // Until a paired-source resolver exists, only synthetic fixtures can attach + // teacher claims. Real capture uses explicit operator labels. + if c.provenance != CaptureProvenance::Synthetic { + return Err("teacher registry resolver unavailable for measured capture".into()); + } + identifier(&t.source_id)?; + identifier(&t.validation_receipt)?; + if t.context_digest != c.context_digest || t.room_digest != context.room_digest || t.trial_id != c.trial_id + || t.label != c.label || t.first_time_ns > first || t.last_time_ns < last + || c.views.iter().any(|v| t.first_time_ns > v.first_time_ns || t.last_time_ns < v.last_time_ns) + || t.clock.kind != ClockKind::SourceMeasurement || t.clock.uncertainty_ns > MAX_CLOCK_UNCERTAINTY_NS + || t.frame_ids.is_empty() || t.frame_ids.len() > MAX_ROWS || t.frame_ids.contains(&0) { + return Err("teacher is not validated and bound to this room/phase/time/frame".into()); + } + let rf_frames: HashSet<_> = c.views.iter().flat_map(|v| v.frame_ids.iter()).collect(); + if !t.frame_ids.iter().all(|id| rf_frames.contains(id)) + || !rf_frames.iter().all(|id| t.frame_ids.contains(id)) { + return Err("teacher references an unknown RF frame".into()); + } + Ok(()) +} +fn identifier(s: &str) -> Result<(), String> { + if s.is_empty() || s.len() > 256 || s.trim() != s || !s.is_ascii() || s.chars().any(char::is_control) { + Err("invalid bounded identifier".into()) + } else { Ok(()) } +} +fn geometry(g: LinkGeometry) -> Result<(), String> { + if g.tx_pos.iter().chain(g.rx_pos.iter()).any(|x| !x.is_finite()) { + Err("non-finite link geometry".into()) + } else { Ok(()) } +} + +#[cfg(test)] +#[path = "observation_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/observation_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/observation_tests.rs new file mode 100644 index 0000000000..298ae12b96 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/observation_tests.rs @@ -0,0 +1,247 @@ +//! SYNTHETIC contract fixtures only; these tests make no sensing-accuracy claim. +use super::*; +use ruview_unified::frame::{CalibrationState, EvidenceLevel, FrameProvenance, SignalQuality}; + +fn context() -> CaptureContext { + CaptureContext { owner_scope: "synthetic-owner".into(), installation_id: "synthetic-install".into(), + room_digest: "synthetic-room".into(), revision: 1, sources: vec![SourceSpec { + source_id: 1, device_id: "synthetic-device".into(), grid_id: "phy-20mhz-2bins".into(), + feature_count: 2, signal_kind: RfModality::Synthetic, center_frequency_hz: 2_400_000_000, bandwidth_hz: 20_000_000, link_id: "tx-a-rx-b".into(), link_index: 0, geometry: None }] } +} +fn cohort(context: &CaptureContext, id: u64, role: DataRole) -> CaptureCohort { + CaptureCohort { id: format!("synthetic-{id}"), context_digest: context.digest(), + trial_id: format!("synthetic-trial-{id}"), session_id: format!("synthetic-session-{id}"), role, + label: RoomLabel::Empty, teacher: None, provenance: CaptureProvenance::Synthetic, + views: vec![SourceView { source_id: 1, device_id: "synthetic-device".into(), + grid_id: "phy-20mhz-2bins".into(), signal_kind: RfModality::Synthetic, center_frequency_hz: 2_400_000_000, bandwidth_hz: 20_000_000, + link_id: "tx-a-rx-b".into(), link_index: 0, geometry: None, samples: vec![vec![2.0,3.0], vec![4.0,5.0]], + frame_ids: vec![u128::from(id)*2+1,u128::from(id)*2+2], sequence_ids: vec![1,2], + first_time_ns: 100+id*10, last_time_ns: 102+id*10, + clock: ClockInfo {kind: ClockKind::Unknown, uncertainty_ns: u64::MAX}, sample_age_ns: 0, + phase: PhaseAvailability::MagnitudeOnly, provenance: CaptureProvenance::Synthetic }] } +} +fn native_frame() -> RfFrameV2 { + // Get the real complex element type through the actual native contract; + // no lookalike tensor type or direct ndarray dependency is introduced. + let iq = [2.0,4.0,3.0,5.0].into_iter().map(Into::into).collect(); + RfFrameV2::new(7, 102, RfModality::Synthetic, + vec![FieldAxis::Antenna,FieldAxis::Frequency,FieldAxis::Time], + 2.4e9,20e6,100.0,vec![1,2,2],iq,vec![true;4],None,None,vec![],0, + CalibrationState {phase_state: PhaseState::Unavailable,gain_calibrated:false,clock_ppm:20.0, + baseline_id:None,confidence:0.0}, + SignalQuality {rssi_dbm:-50.0,noise_floor_dbm:-90.0,packet_loss:0.0,interference:0.0}, + FrameProvenance {class:ProvenanceClass::Synthetic,evidence:EvidenceLevel::L0Simulation, + device_id:"synthetic-device".into(),firmware:"SYNTHETIC".into(),receipt_id:7}).unwrap() +} +fn metadata() -> ViewMetadata { + ViewMetadata {frame_ids:vec![7,7],sequence_ids:vec![1,2],first_time_ns:100,last_time_ns:102, + clock:ClockInfo {kind:ClockKind::Unknown,uncertainty_ns:u64::MAX}, + phase:PhaseAvailability::MagnitudeOnly,link_index:0,provenance:CaptureProvenance::Synthetic} +} + +#[test] +fn synthetic_dataset_exposes_frozen_roles_and_no_physical_clock_claim() { + let c = context(); + let ds = ValidatedDataset::new(c.clone(),vec![cohort(&c,1,DataRole::Train), + cohort(&c,2,DataRole::Validation),cohort(&c,3,DataRole::HeldOut)]).unwrap(); + assert_eq!((ds.train().len(),ds.validation().len(),ds.held_out().len()),(1,1,1)); + assert!(!ds.measurement_clock_available()); +} +#[test] +fn native_adapter_preserves_raw_gain_and_static_mean_shift() { + let c = context(); + let f = native_frame(); + let first = SourceView::from_frame(&c.sources[0],&f,metadata()).unwrap(); + let mut stronger = f.clone(); + for z in &mut stronger.native_iq { *z *= 3.0; } + let second = SourceView::from_frame(&c.sources[0],&stronger,metadata()).unwrap(); + assert_eq!(first.samples,vec![vec![2.0,3.0],vec![4.0,5.0]]); + assert_eq!(second.samples,vec![vec![6.0,9.0],vec![12.0,15.0]]); + assert_eq!(f.native_iq,native_frame().native_iq); +} +#[test] +fn actual_canonical_adapter_gain_normalization_is_refused() { + let c = context(); + let t = native_frame().to_canonical(vec![LinkGeometry{tx_pos:[0.0;3],rx_pos:[1.0,0.0,0.0]}]).unwrap(); + assert!(SourceView::from_tensor(&c.sources[0],&t,metadata()).unwrap_err().contains("gain-normalized")); +} +#[test] +fn tensor_adapter_reads_real_tensor_without_centering() { + let mut c = context(); + c.sources[0].feature_count = 56; + let mut t = native_frame().to_canonical(vec![LinkGeometry{tx_pos:[0.0;3],rx_pos:[1.0,0.0,0.0]}]).unwrap(); + // Explicitly synthetic raw-amplitude tensor: remove fixture adapter scale + // and overwrite every sample. No assertion is made about measured gain. + t.calibration.gain_offset_db = 0.0; + for z in &mut t.data { z.re = 7.0; z.im = 0.0; } + let mut m = metadata(); m.frame_ids = vec![7;8]; m.sequence_ids = (1..=8).collect(); + let view = SourceView::from_tensor(&c.sources[0],&t,m).unwrap(); + assert!(view.samples.iter().flatten().all(|x| *x == 7.0)); +} +#[test] +fn tensor_mutant_nonfinite_samples_are_rejected_despite_public_tensor_fields() { + let mut c = context(); c.sources[0].feature_count = 56; + let mut t = native_frame().to_canonical(vec![LinkGeometry{tx_pos:[0.0;3],rx_pos:[1.0,0.0,0.0]}]).unwrap(); + t.calibration.gain_offset_db = 0.0; + t.data[[0,0,0]].re = f64::NAN; + let mut m = metadata(); m.frame_ids = vec![7;8]; m.sequence_ids = (1..=8).collect(); + assert!(SourceView::from_tensor(&c.sources[0],&t,m).is_err()); +} +#[test] +fn native_invalid_mask_and_phase_are_not_invented_as_measurements() { + let c = context(); let mut f = native_frame(); f.valid_mask[0] = false; + assert!(SourceView::from_frame(&c.sources[0],&f,metadata()).is_err()); + f.valid_mask[0] = true; + let mut m = metadata(); m.phase = PhaseAvailability::Calibrated; + assert!(SourceView::from_frame(&c.sources[0],&f,m).is_err()); +} +#[test] +fn synthetic_boundary_mutants_are_rejected() { + let c = context(); + let original = cohort(&c,1,DataRole::Train); + type Mutant = fn(&mut CaptureCohort); + let mutants: &[(&str,Mutant)] = &[ + ("context",|x|x.context_digest="wrong-room-owner-revision".into()), + ("missing source",|x|x.views.clear()), + ("unknown source",|x|x.views[0].source_id=2), + ("duplicate source",|x|x.views.push(x.views[0].clone())), + ("changed PHY #1440",|x|x.views[0].grid_id="phy-40mhz".into()), + ("changed frequency #1440",|x|x.views[0].center_frequency_hz+=5_000_000), + ("changed bandwidth #1440",|x|x.views[0].bandwidth_hz*=2), + ("changed link",|x|x.views[0].link_id="other-link".into()), + ("changed device",|x|x.views[0].device_id="other-device".into()), + ("NaN",|x|x.views[0].samples[0][0]=f32::NAN), + ("oversized rows",|x|x.views[0].samples.resize(MAX_ROWS+1,vec![1.0,2.0])), + ("wrong feature count",|x|x.views[0].samples[0].push(1.0)), + ("stale",|x|x.views[0].sample_age_ns=MAX_AGE_NS+1), + ("sequence reversal",|x|x.views[0].sequence_ids=vec![2,1]), + ("source clock uncertainty #1710",|x|x.views[0].clock=ClockInfo{kind:ClockKind::SourceMeasurement,uncertainty_ns:u64::MAX}), + ("nonmonotonic time",|x|x.views[0].last_time_ns=1), + ("long window",|x|x.views[0].last_time_ns=MAX_WINDOW_NS+200), + ("forged provenance",|x|x.provenance=CaptureProvenance::LiveReceiverObserved), + ]; + for (name,mutate) in mutants { + let mut candidate = original.clone(); mutate(&mut candidate); + assert!(ValidatedDataset::new(c.clone(),vec![candidate]).is_err(),"survived mutant: {name}"); + } +} +#[test] +fn frame_reuse_cannot_cross_roles_even_when_trial_and_session_are_renamed() { + let c = context(); let a = cohort(&c,1,DataRole::Train); + let mut b = cohort(&c,2,DataRole::HeldOut); b.views[0].frame_ids = a.views[0].frame_ids.clone(); + assert!(ValidatedDataset::new(c,vec![a,b]).unwrap_err().contains("frame overlap")); +} +#[test] +fn trial_or_session_cannot_cross_roles() { + for use_trial in [true,false] { + let c = context(); let a = cohort(&c,1,DataRole::Train); let mut b = cohort(&c,2,DataRole::Validation); + if use_trial {b.trial_id=a.trial_id.clone();} else {b.session_id=a.session_id.clone();} + assert!(ValidatedDataset::new(c,vec![a,b]).unwrap_err().contains("frozen roles")); + } +} +#[test] +fn source_clock_is_not_inferred_from_receive_time_or_time_of_day() { + let mut c = context(); c.sources[0].signal_kind=RfModality::WifiCsi; + let mut a = cohort(&c,1,DataRole::Train); a.views[0].signal_kind=RfModality::WifiCsi; + a.provenance=CaptureProvenance::LiveReceiverObserved; + a.views[0].provenance=a.provenance; + a.views[0].clock.kind=ClockKind::ReceiverOnly; + let dataset=ValidatedDataset::new(c,vec![a]).unwrap(); + assert!(!dataset.measurement_clock_available()); +} +#[test] +fn context_digest_binds_owner_room_revision_and_phy_not_time_of_day() { + let c=context(); + for change in 0..4 { + let mut other=c.clone(); + match change {0=>other.owner_scope="other-owner".into(),1=>other.room_digest="other-room".into(), + 2=>other.revision+=1,_=>other.sources[0].grid_id="other-PHY".into()} + assert_ne!(other.digest(),c.digest()); + } +} +#[test] +fn missing_teacher_never_yields_a_camera_derived_empty_label() { + let c=context(); let a=cohort(&c,1,DataRole::Train); + let ds=ValidatedDataset::new(c,vec![a]).unwrap(); + assert!(ds.train()[0].teacher.is_none()); +} +fn teacher(c:&CaptureContext,a:&CaptureCohort)->TeacherEvidence { + TeacherEvidence {kind:TeacherKind::Radar,source_id:"synthetic-teacher".into(), + validation_receipt:"synthetic-receipt-NOT-qualification".into(),context_digest:c.digest(), + room_digest:c.room_digest.clone(),trial_id:a.trial_id.clone(),label:a.label, + frame_ids:a.views[0].frame_ids.clone(),first_time_ns:a.views[0].first_time_ns, + last_time_ns:a.views[0].last_time_ns,clock:ClockInfo{kind:ClockKind::SourceMeasurement,uncertainty_ns:1}} +} +#[test] +fn receive_time_only_radar_cannot_supply_empty_teacher_evidence() { + let c=context(); let mut a=cohort(&c,1,DataRole::Train); let mut t=teacher(&c,&a); + t.clock.kind=ClockKind::ReceiverOnly; a.teacher=Some(t); + assert!(ValidatedDataset::new(c,vec![a]).is_err()); +} +#[test] +fn teacher_room_phase_time_frame_and_validated_source_are_bound() { + let c=context(); let a=cohort(&c,1,DataRole::Train); let t=teacher(&c,&a); + let mutants: &[fn(&mut TeacherEvidence)] = &[ + |x|x.room_digest="elsewhere".into(),|x|x.label=RoomLabel::OccupiedMoving, + |x|x.first_time_ns=x.last_time_ns+1,|x|x.frame_ids=vec![999],|x|x.validation_receipt.clear(), + |x|x.source_id.clear(),|x|x.trial_id="different-trial".into()]; + for mutate in mutants { + let mut candidate=a.clone(); let mut evidence=t.clone(); mutate(&mut evidence); + candidate.teacher=Some(evidence); + assert!(ValidatedDataset::new(c.clone(),vec![candidate]).is_err()); + } +} +#[test] +fn context_rejects_unicode_unbounded_ids_and_unsorted_sources() { + let mut c=context(); c.owner_scope="untrusted-\u{202e}".into(); assert!(c.validate().is_err()); + c=context();c.owner_scope="x".repeat(257);assert!(c.validate().is_err()); + c=context();let mut second=c.sources[0].clone();second.source_id=0;second.link_id="other".into(); + c.sources.push(second);assert!(c.validate().is_err()); +} + +#[test] +fn nonempty_teacher_receipt_claim_never_replaces_validated_registry() { + let mut c=context(); c.sources[0].signal_kind=RfModality::WifiCsi; + let mut a=cohort(&c,1,DataRole::Train); a.views[0].signal_kind=RfModality::WifiCsi; + a.provenance=CaptureProvenance::CapturedReplay; a.views[0].provenance=a.provenance; + a.teacher=Some(teacher(&c,&a)); + assert!(ValidatedDataset::new(c,vec![a]).unwrap_err().contains("registry resolver unavailable")); +} + +#[test] +fn selected_sources_require_complete_aligned_windows_and_bound_link_indices() { + let mut c=context(); let mut second=c.sources[0].clone(); + second.source_id=2; second.link_id="second-link".into(); c.sources.push(second); + let mut a=cohort(&c,1,DataRole::Train); + let mut view=a.views[0].clone();view.source_id=2;view.link_id="second-link".into(); + a.views.push(view); + assert!(validate_cohort(&c,&a).is_ok()); + a.views[1].first_time_ns=1000;a.views[1].last_time_ns=1002; + assert!(validate_cohort(&c,&a).is_err()); + a.views[1].first_time_ns=a.views[0].first_time_ns;a.views[1].last_time_ns=a.views[0].last_time_ns; + a.views[1].link_index=1;assert!(validate_cohort(&c,&a).is_err()); +} +#[test] +fn unrecognized_fields_cannot_silently_claim_validated_sources() { + let c=context();let a=cohort(&c,1,DataRole::Train); + let mut json=serde_json::to_value(a).unwrap();json["validated"]=true.into(); + assert!(serde_json::from_value::(json).is_err()); +} +#[test] +fn tensor_cannot_promote_unknown_phase_to_calibrated() { + let mut c=context();c.sources[0].feature_count=56; + let mut t=native_frame().to_canonical(vec![LinkGeometry{tx_pos:[0.0;3],rx_pos:[1.0,0.0,0.0]}]).unwrap(); + t.calibration.gain_offset_db=0.0;t.calibration.phase_calibrated=false; + let mut m=metadata();m.phase=PhaseAvailability::Calibrated; + m.frame_ids=vec![7;8];m.sequence_ids=(1..=8).collect(); + assert!(SourceView::from_tensor(&c.sources[0],&t,m).unwrap_err().contains("phase state")); +} + +#[test] +fn inference_is_unlabelled_and_cannot_enter_training_or_evaluation_dataset() { + let c=context();let mut a=cohort(&c,1,DataRole::Inference); + assert!(validate_cohort(&c,&a).is_err()); + a.label=RoomLabel::Unknown;assert!(validate_cohort(&c,&a).is_ok()); + assert!(ValidatedDataset::new(c.clone(),vec![a.clone()]).is_err()); + a.role=DataRole::Train;assert!(validate_cohort(&c,&a).is_err()); +} From b29ab4525ab09c82ff07ca7c9209bbc2aa5fbd56 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Sun, 1 Nov 2026 20:29:46 +0100 Subject: [PATCH 031/100] feat(learning): train source-conditioned AETHER room candidates --- .../src/room_learner/artifact.rs | 273 ++++++++++ .../src/room_learner/artifact_catalog.rs | 114 +++++ .../src/room_learner/artifact_tests.rs | 452 +++++++++++++++++ .../src/room_learner/fit_config.rs | 101 ++++ .../src/room_learner/learning_tests.rs | 468 ++++++++++++++++++ .../src/room_learner/manifest.rs | 131 +++++ .../src/room_learner/manifest_tests.rs | 142 ++++++ .../src/room_learner/mod.rs | 3 + .../src/room_learner/model.rs | 431 ++++++++++++++++ .../src/room_learner/optimization.rs | 142 ++++++ .../src/room_learner/projection.rs | 110 ++++ .../room_learner/source_conditioning_tests.rs | 230 +++++++++ .../src/room_learner/training.rs | 408 +++++++++++++++ 13 files changed, 3005 insertions(+) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact_catalog.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/fit_config.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/learning_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/manifest.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/manifest_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/model.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/optimization.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/projection.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/source_conditioning_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/training.rs diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs new file mode 100644 index 0000000000..c6da41acc8 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs @@ -0,0 +1,273 @@ +//! Private, owner-bound candidate persistence. This store grants no qualification. +use super::{model::RoomModel, observation::CaptureContext}; +use crate::companion::{atomic_private_write, private_dir, private_read}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::{ + io, + path::{Path, PathBuf}, +}; + +const POINTER_SCHEMA: &str = "ruview.room-learner.candidate.v1"; +const MODEL_SCHEMA: &str = "ruview.room-learner.model.v1"; +const LIMIT: usize = 65_536; // Same bound as the actual private_read primitive. +const POINTER: &str = "candidate-v1.json"; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct CandidateReceipt { + pub schema: String, + pub owner_scope: String, + pub installation_id: String, + pub room_digest: String, + pub context_digest: String, + pub model_sha256: String, + pub model_version: u32, + pub parameter_count: usize, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Envelope { + schema: String, + receipt: CandidateReceipt, + /// Exact RoomModel::export bytes, escaped as JSON without losing float bits. + learned_model_json: String, +} +#[derive(Debug)] +pub struct ArtifactStore { + data_dir: PathBuf, + root: PathBuf, + owner_scope: String, + installation_id: String, +} +fn digest(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} +fn hex_digest(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(&c)) +} +fn bounded_id(value: &str) -> bool { + !value.is_empty() + && value.len() <= 256 + && value.is_ascii() + && value.trim() == value + && !value.bytes().any(|c| c.is_ascii_control()) +} +fn encode(value: &T) -> Result, String> { + let bytes = serde_json::to_vec(value).map_err(|e| e.to_string())?; + if bytes.len() > LIMIT { + return Err("candidate artifact exceeds private storage bound".into()); + } + Ok(bytes) +} +fn regular_or_missing(path: &Path) -> Result { + match std::fs::symlink_metadata(path) { + Ok(meta) if meta.is_file() && !meta.file_type().is_symlink() => Ok(true), + Ok(_) => Err("candidate path is not a private regular file".into()), + Err(e) if e.kind() == io::ErrorKind::NotFound => Ok(false), + Err(e) => Err(e.to_string()), + } +} +impl ArtifactStore { + pub fn new( + data_dir: &Path, + expected_owner_scope: &str, + expected_installation: &str, + ) -> Result { + if !bounded_id(expected_installation) { + return Err("invalid expected installation".into()); + } + let data_dir = crate::account_profile::prepare_directory(expected_owner_scope, data_dir) + .map_err(|e| e.to_string())?; + let root = data_dir.join("room-learner"); + private_dir(&root).map_err(|e| e.to_string())?; + Ok(Self { + data_dir, + root, + owner_scope: expected_owner_scope.into(), + installation_id: expected_installation.into(), + }) + } + fn verify_owner(&self) -> Result<(), String> { + let actual = crate::account_profile::prepare_directory(&self.owner_scope, &self.data_dir) + .map_err(|e| e.to_string())?; + if actual != self.data_dir { + return Err("owner directory changed".into()); + } + private_dir(&self.root).map_err(|e| e.to_string()) + } + fn validate_context(&self, context: &CaptureContext) -> Result<(), String> { + context.validate()?; + if context.owner_scope != self.owner_scope + || context.installation_id != self.installation_id + { + return Err("candidate owner or installation mismatch".into()); + } + Ok(()) + } + fn context_dir( + &self, + context: &CaptureContext, + create: bool, + ) -> Result, String> { + self.verify_owner()?; + self.validate_context(context)?; + let key = context.digest(); + if !hex_digest(&key) { + return Err("invalid context digest".into()); + } + let dir = self.root.join(key); + if !create { + match std::fs::symlink_metadata(&dir) { + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(e) => return Err(e.to_string()), + Ok(meta) if !meta.is_dir() || meta.file_type().is_symlink() => { + return Err("invalid candidate context directory".into()) + } + _ => {} + } + } + private_dir(&dir).map_err(|e| e.to_string())?; + Ok(Some(dir)) + } + fn receipt_for(&self, model: &RoomModel, bytes: &[u8]) -> CandidateReceipt { + CandidateReceipt { + schema: POINTER_SCHEMA.into(), + owner_scope: self.owner_scope.clone(), + installation_id: self.installation_id.clone(), + room_digest: model.context().room_digest.clone(), + context_digest: model.context().digest(), + model_sha256: digest(bytes), + model_version: model.version, + parameter_count: model.parameter_count(), + } + } + fn validate_receipt( + &self, + receipt: &CandidateReceipt, + context: &CaptureContext, + ) -> Result<(), String> { + if receipt.schema != POINTER_SCHEMA + || receipt.owner_scope != self.owner_scope + || receipt.installation_id != self.installation_id + || receipt.room_digest != context.room_digest + || receipt.context_digest != context.digest() + || !hex_digest(&receipt.model_sha256) + || receipt.model_version != 1 + || !(1..=512).contains(&receipt.parameter_count) + { + return Err("candidate receipt/context mismatch".into()); + } + Ok(()) + } + fn read_model( + &self, + dir: &Path, + receipt: &CandidateReceipt, + context: &CaptureContext, + ) -> Result { + self.validate_receipt(receipt, context)?; + let bytes = private_read(&dir.join(format!("{}.json", receipt.model_sha256))) + .map_err(|e| e.to_string())?; + let envelope: Envelope = serde_json::from_slice(&bytes).map_err(|e| e.to_string())?; + if envelope.schema != MODEL_SCHEMA + || &envelope.receipt != receipt + || digest(envelope.learned_model_json.as_bytes()) != receipt.model_sha256 + { + return Err("candidate immutable artifact integrity mismatch".into()); + } + let model = RoomModel::import(envelope.learned_model_json.as_bytes())?; + if model.context() != context || self.receipt_for(&model, &model.export()?) != *receipt { + return Err("candidate learned model/context mismatch".into()); + } + Ok(model) + } + fn read_pointer( + &self, + dir: &Path, + context: &CaptureContext, + ) -> Result, String> { + let path = dir.join(POINTER); + if !regular_or_missing(&path)? { + return Ok(None); + } + let receipt: CandidateReceipt = + serde_json::from_slice(&private_read(&path).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + self.validate_receipt(&receipt, context)?; + // A corrupt existing pointer/artifact is never silently overwritten. + self.read_model(dir, &receipt, context)?; + Ok(Some(receipt)) + } + pub fn save_candidate(&self, model: &RoomModel) -> Result { + self.validate_context(model.context())?; + let bytes = model.export()?; + let receipt = self.receipt_for(model, &bytes); + let envelope = Envelope { + schema: MODEL_SCHEMA.into(), + receipt: receipt.clone(), + learned_model_json: String::from_utf8(bytes).map_err(|e| e.to_string())?, + }; + let encoded = encode(&envelope)?; + let pointer = encode(&receipt)?; + let dir = self + .context_dir(model.context(), true)? + .ok_or("missing context directory")?; + self.read_pointer(&dir, model.context())?; + let immutable = dir.join(format!("{}.json", receipt.model_sha256)); + if regular_or_missing(&immutable)? { + if private_read(&immutable).map_err(|e| e.to_string())? != encoded { + return Err("existing immutable candidate differs".into()); + } + } else { + // Actual private writer supplies 0600 + fsync. An exclusive link + // installs the complete inode without replacing any previous model. + let staging = dir.join(format!("model-stage-{}", crate::companion::secret())); + atomic_private_write(&staging, &encoded).map_err(|e| e.to_string())?; + let installed = std::fs::hard_link(&staging, &immutable); + let cleanup = std::fs::remove_file(&staging); + match installed { + Ok(()) => {} + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => { + if private_read(&immutable).map_err(|e| e.to_string())? != encoded { + return Err("immutable candidate collision".into()); + } + } + Err(e) => return Err(e.to_string()), + } + cleanup.map_err(|e| e.to_string())?; + std::fs::File::open(&dir) + .and_then(|f| f.sync_all()) + .map_err(|e| e.to_string())?; + } + self.read_model(&dir, &receipt, model.context())?; + atomic_private_write(&dir.join(POINTER), &pointer).map_err(|e| e.to_string())?; + if private_read(&dir.join(POINTER)).map_err(|e| e.to_string())? != pointer { + return Err("candidate pointer readback mismatch".into()); + } + Ok(receipt) + } + pub fn load_candidate( + &self, + expected_context: &CaptureContext, + ) -> Result, String> { + let Some(dir) = self.context_dir(expected_context, false)? else { + return Ok(None); + }; + let Some(receipt) = self.read_pointer(&dir, expected_context)? else { + return Ok(None); + }; + self.read_model(&dir, &receipt, expected_context).map(Some) + } +} + +#[cfg(test)] +#[path = "artifact_tests.rs"] +mod tests; + +#[path="artifact_catalog.rs"] +mod catalog; +pub use catalog::{CandidateCatalog,SavedCandidate}; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact_catalog.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact_catalog.rs new file mode 100644 index 0000000000..1f511590b9 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact_catalog.rs @@ -0,0 +1,114 @@ +//! Bounded owner-private metadata discovery. It does not select/adopt authority. +use super::*; +#[derive(Debug, Clone, Serialize)] +pub struct SavedCandidate { + pub receipt: CandidateReceipt, + pub context: CaptureContext, + /// Selection convenience only; never evidence freshness or qualification. + pub stored_at_unix_ms: u64, +} +#[derive(Debug, Clone, Serialize)] +pub struct CandidateCatalog { + pub candidates: Vec, + pub unreadable_contexts: usize, + pub truncated: bool, +} +impl ArtifactStore { + pub fn catalog(&self) -> Result { + self.catalog_filtered(None) + } + pub fn catalog_for_room(&self, room_digest: &str) -> Result { + if !bounded_id(room_digest) { + return Err("invalid catalog room".into()); + } + self.catalog_filtered(Some(room_digest)) + } + fn catalog_filtered(&self, room_digest: Option<&str>) -> Result { + self.verify_owner()?; + let mut contexts = Vec::new(); + let mut truncated = false; + for (index, entry) in std::fs::read_dir(&self.root) + .map_err(|e| e.to_string())? + .take(257) + .enumerate() + { + if index == 256 { + truncated = true; + break; + } + let entry = entry.map_err(|e| e.to_string())?; + if contexts.len() >= 256 { + truncated = true; + break; + } + let Some(key) = entry.file_name().to_str().map(str::to_string) else { + continue; + }; + if hex_digest(&key) { + contexts.push((key, entry.path())); + } + } + let mut result = CandidateCatalog { + candidates: Vec::new(), + unreadable_contexts: 0, + truncated, + }; + contexts.sort_by(|a, b| a.0.cmp(&b.0)); + for (key, dir) in contexts { + let candidate = (|| { + let meta = std::fs::symlink_metadata(&dir).map_err(|e| e.to_string())?; + if !meta.is_dir() || meta.file_type().is_symlink() { + return Err("invalid catalog context directory".to_string()); + } + private_dir(&dir).map_err(|e| e.to_string())?; + let pointer = dir.join(POINTER); + let receipt: CandidateReceipt = + serde_json::from_slice(&private_read(&pointer).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + if receipt.owner_scope != self.owner_scope + || receipt.installation_id != self.installation_id + || receipt.context_digest != key + || !hex_digest(&receipt.model_sha256) + { + return Err("catalog owner/context mismatch".into()); + } + let envelope: Envelope = serde_json::from_slice( + &private_read(&dir.join(format!("{}.json", receipt.model_sha256))) + .map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + let model = RoomModel::import(envelope.learned_model_json.as_bytes())?; + self.validate_context(model.context())?; + self.read_model(&dir, &receipt, model.context())?; + let stored_at_unix_ms = std::fs::symlink_metadata(&pointer) + .ok() + .and_then(|m| m.modified().ok()) + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| d.as_millis().min(u64::MAX as u128) as u64) + .unwrap_or(0); + Ok::<_, String>(SavedCandidate { + receipt, + context: model.context().clone(), + stored_at_unix_ms, + }) + })(); + match candidate { + Ok(c) if room_digest.is_none_or(|room| c.context.room_digest == room) => { + result.candidates.push(c) + } + Ok(_) => {} + Err(_) => result.unreadable_contexts += 1, + } + } + result.candidates.sort_by(|a, b| { + b.stored_at_unix_ms + .cmp(&a.stored_at_unix_ms) + .then(a.receipt.context_digest.cmp(&b.receipt.context_digest)) + }); + if result.candidates.len() > 32 { + result.candidates.truncate(32); + result.truncated = true; + } + Ok(result) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact_tests.rs new file mode 100644 index 0000000000..73d16a8c2e --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact_tests.rs @@ -0,0 +1,452 @@ +//! Actual trained model + real private filesystem primitives; RF is SYNTHETIC. +use super::*; +use crate::room_learner::{observation::*, training::FitConfig}; +use ruview_unified::RfModality; +use std::sync::OnceLock; + +fn owner() -> String { + "a".repeat(64) +} +fn fitted(room: &str, revision: u64, epochs: usize) -> RoomModel { + let context = CaptureContext { + owner_scope: owner(), + installation_id: "synthetic-install".into(), + room_digest: room.into(), + revision, + sources: (1..=2) + .map(|id| SourceSpec { + source_id: id, + device_id: format!("synthetic-device-{id}"), + grid_id: "synthetic-4-bin".into(), + feature_count: 4, + signal_kind: RfModality::Synthetic, + link_id: format!("link-{id}"), + link_index: 0, + center_frequency_hz: 2_400_000_000, + bandwidth_hz: 20_000_000, + geometry: None, + }) + .collect(), + }; + let labels = [ + RoomLabel::Empty, + RoomLabel::AdjacentOnly, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + ]; + let rows = (1..=8u64) + .map(|id| { + let label = labels[(id as usize - 1) % 4]; + let views = context + .sources + .iter() + .map(|s| { + let shift = match label { + RoomLabel::Empty => 0.0, + RoomLabel::AdjacentOnly => { + if s.source_id == 1 { + 1.5 + } else { + -1.5 + } + } + _ => 1.5, + }; + SourceView { + source_id: s.source_id, + device_id: s.device_id.clone(), + grid_id: s.grid_id.clone(), + signal_kind: s.signal_kind, + link_id: s.link_id.clone(), + link_index: 0, + center_frequency_hz: s.center_frequency_hz, + bandwidth_hz: s.bandwidth_hz, + geometry: None, + samples: (0..4) + .map(|t| { + (0..4) + .map(|b| { + 10.0 + shift + + t as f32 * 0.02 + + b as f32 * 0.01 + + id as f32 * 0.002 + }) + .collect() + }) + .collect(), + frame_ids: (0..4).map(|t| u128::from(id) * 10 + t + 1).collect(), + sequence_ids: vec![1, 2, 3, 4], + first_time_ns: id * 1000 + 1, + last_time_ns: id * 1000 + 100, + clock: ClockInfo { + kind: ClockKind::Unknown, + uncertainty_ns: u64::MAX, + }, + sample_age_ns: 0, + phase: PhaseAvailability::MagnitudeOnly, + provenance: CaptureProvenance::Synthetic, + } + }) + .collect(); + CaptureCohort { + id: format!("event-{id}"), + context_digest: context.digest(), + trial_id: format!("trial-{id}"), + session_id: format!("session-{id}"), + role: DataRole::Train, + label, + views, + teacher: None, + provenance: CaptureProvenance::Synthetic, + } + }) + .collect(); + RoomModel::fit( + &ValidatedDataset::new(context, rows).unwrap(), + FitConfig { + epochs, + ..FitConfig::default() + }, + ) + .unwrap() + .0 +} +fn model() -> RoomModel { + static MODEL: OnceLock = OnceLock::new(); + MODEL + .get_or_init(|| fitted("synthetic-room-a", 1, 2)) + .clone() +} +fn store(root: &Path) -> ArtifactStore { + ArtifactStore::new( + &root.join(owner()).join("SensingServer"), + &owner(), + "synthetic-install", + ) + .unwrap() +} +fn immutable(store: &ArtifactStore, receipt: &CandidateReceipt) -> PathBuf { + store + .root + .join(&receipt.context_digest) + .join(format!("{}.json", receipt.model_sha256)) +} +fn pointer(store: &ArtifactStore, model: &RoomModel) -> PathBuf { + store.root.join(model.context().digest()).join(POINTER) +} +#[test] +fn trained_model_private_save_cold_reload_inference_round_trip_has_no_raw_data() { + let temp = tempfile::tempdir().unwrap(); + let store = store(temp.path()); + let model = model(); + assert!(store.load_candidate(model.context()).unwrap().is_none()); + let receipt = store.save_candidate(&model).unwrap(); + let envelope = private_read(&immutable(&store, &receipt)).unwrap(); + let text = String::from_utf8(envelope).unwrap(); + for raw in ["samples", "frame_ids", "sequence_ids", "trial_id"] { + assert!(!text.contains(raw)); + } + let cold = ArtifactStore::new(&store.data_dir, &owner(), "synthetic-install").unwrap(); + let restored = cold.load_candidate(model.context()).unwrap().unwrap(); + assert_eq!(model.parameters(), restored.parameters()); + assert_eq!( + restored.training_manifest().training_provenance, + [CaptureProvenance::Synthetic] + ); + assert!(!restored.training_manifest().measurement_clock_available); + assert_eq!(model.export().unwrap(), restored.export().unwrap()); + let context = model.context(); + let live = CaptureCohort { + id: "live-input".into(), + context_digest: context.digest(), + trial_id: "live-trial".into(), + session_id: "live-session".into(), + role: DataRole::Inference, + label: RoomLabel::Unknown, + teacher: None, + provenance: CaptureProvenance::Synthetic, + views: context + .sources + .iter() + .map(|s| SourceView { + source_id: s.source_id, + device_id: s.device_id.clone(), + grid_id: s.grid_id.clone(), + signal_kind: s.signal_kind, + link_id: s.link_id.clone(), + link_index: 0, + center_frequency_hz: s.center_frequency_hz, + bandwidth_hz: s.bandwidth_hz, + geometry: None, + samples: vec![vec![11.5; 4]; 4], + frame_ids: vec![101, 102, 103, 104], + sequence_ids: vec![1, 2, 3, 4], + first_time_ns: 1000, + last_time_ns: 1100, + clock: ClockInfo { + kind: ClockKind::Unknown, + uncertainty_ns: u64::MAX, + }, + sample_age_ns: 0, + phase: PhaseAvailability::MagnitudeOnly, + provenance: CaptureProvenance::Synthetic, + }) + .collect(), + }; + assert_eq!( + model.infer(&live).occupied_probability, + restored.infer(&live).occupied_probability + ); + assert!(restored.infer(&live).occupied_probability.is_some()); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + std::fs::metadata(immutable(&store, &receipt)) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 + ); + assert_eq!( + std::fs::metadata(pointer(&store, &model)) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 + ); + assert_eq!( + std::fs::metadata(&store.root).unwrap().permissions().mode() & 0o777, + 0o700 + ); + } +} +#[test] +fn independent_rooms_revisions_and_replaced_candidates_retain_all_prior_artifacts() { + let temp = tempfile::tempdir().unwrap(); + let store = store(temp.path()); + let a = model(); + let b = fitted("synthetic-room-b", 1, 2); + let a2 = fitted("synthetic-room-a", 2, 2); + let ar = store.save_candidate(&a).unwrap(); + let before = private_read(&immutable(&store, &ar)).unwrap(); + store.save_candidate(&b).unwrap(); + store.save_candidate(&a2).unwrap(); + for expected in [&a, &b, &a2] { + assert_eq!( + store + .load_candidate(expected.context()) + .unwrap() + .unwrap() + .parameters(), + expected.parameters() + ); + } + let replacement = fitted("synthetic-room-a", 1, 3); + let next = store.save_candidate(&replacement).unwrap(); + assert_ne!(ar.model_sha256, next.model_sha256); + assert_eq!(private_read(&immutable(&store, &ar)).unwrap(), before); + assert_eq!( + store + .load_candidate(a.context()) + .unwrap() + .unwrap() + .parameters(), + replacement.parameters() + ); + assert!(immutable(&store, &next).is_file()); +} +#[test] +fn corrupt_pointer_or_immutable_bytes_fail_closed_and_preserve_existing_candidate() { + for pointer_corruption in [true, false] { + let temp = tempfile::tempdir().unwrap(); + let store = store(temp.path()); + let model = model(); + let receipt = store.save_candidate(&model).unwrap(); + let path = if pointer_corruption { + pointer(&store, &model) + } else { + immutable(&store, &receipt) + }; + atomic_private_write(&path, b"corrupt candidate bytes").unwrap(); + let before = private_read(&path).unwrap(); + assert!(store.load_candidate(model.context()).is_err()); + assert!(store.save_candidate(&model).is_err()); + assert_eq!(private_read(&path).unwrap(), before); + } +} +#[test] +fn model_hash_tampering_wrong_context_and_installation_cannot_be_adopted() { + let temp = tempfile::tempdir().unwrap(); + let store = store(temp.path()); + let model = model(); + let receipt = store.save_candidate(&model).unwrap(); + let mut envelope: serde_json::Value = + serde_json::from_slice(&private_read(&immutable(&store, &receipt)).unwrap()).unwrap(); + envelope["learned_model_json"] = serde_json::Value::String("{}".into()); + atomic_private_write( + &immutable(&store, &receipt), + &serde_json::to_vec(&envelope).unwrap(), + ) + .unwrap(); + assert!(store + .load_candidate(model.context()) + .unwrap_err() + .contains("integrity")); + let mut other = model.context().clone(); + other.installation_id = "other-install".into(); + assert!(store.load_candidate(&other).is_err()); + let wrong = ArtifactStore::new(&store.data_dir, &owner(), "other-install").unwrap(); + assert!(wrong.save_candidate(&model).is_err()); + other = model.context().clone(); + other.owner_scope = "b".repeat(64); + assert!(store.load_candidate(&other).is_err()); + assert!(ArtifactStore::new(&store.data_dir, &"b".repeat(64), "synthetic-install").is_err()); +} +#[test] +fn existing_account_witness_and_accepted_control_models_are_preserved() { + let temp = tempfile::tempdir().unwrap(); + let store = store(temp.path()); + let witness = store.data_dir.join("account-owner-v1.json"); + let before = private_read(&witness).unwrap(); + let accepted = store.data_dir.join("accepted-model.json"); + let control = store.data_dir.join("control-model.json"); + atomic_private_write(&accepted, b"accepted bytes").unwrap(); + atomic_private_write(&control, b"control bytes").unwrap(); + store.save_candidate(&model()).unwrap(); + assert_eq!(private_read(&witness).unwrap(), before); + assert_eq!(private_read(&accepted).unwrap(), b"accepted bytes"); + assert_eq!(private_read(&control).unwrap(), b"control bytes"); + atomic_private_write(&witness, b"changed owner witness").unwrap(); + assert!(store.save_candidate(&model()).is_err()); + assert_eq!(private_read(&witness).unwrap(), b"changed owner witness"); +} +#[cfg(unix)] +#[test] +fn symlink_candidate_pointer_context_and_root_are_rejected_without_touching_victims() { + use std::os::unix::fs::symlink; + for level in ["pointer", "immutable", "context", "root"] { + let temp = tempfile::tempdir().unwrap(); + let store = store(temp.path()); + let model = model(); + let receipt = store.save_candidate(&model).unwrap(); + let victim = temp.path().join("victim"); + atomic_private_write(&victim, b"victim private bytes").unwrap(); + let path = match level { + "pointer" => pointer(&store, &model), + "immutable" => immutable(&store, &receipt), + "context" => store.root.join(model.context().digest()), + _ => store.root.clone(), + }; + let preserved = path.with_extension("preserved"); + std::fs::rename(&path, &preserved).unwrap(); + symlink(&victim, &path).unwrap(); + assert!( + store.load_candidate(model.context()).is_err(), + "accepted symlink {level}" + ); + assert!( + store.save_candidate(&model).is_err(), + "overwrote symlink {level}" + ); + assert_eq!(private_read(&victim).unwrap(), b"victim private bytes"); + assert!(std::fs::symlink_metadata(path) + .unwrap() + .file_type() + .is_symlink()); + } +} +#[cfg(unix)] +#[test] +fn insecure_permissions_and_disk_path_failure_leave_previous_model_unchanged() { + use std::os::unix::fs::PermissionsExt; + let temp = tempfile::tempdir().unwrap(); + let store = store(temp.path()); + let model = model(); + let receipt = store.save_candidate(&model).unwrap(); + let path = pointer(&store, &model); + let before = private_read(&path).unwrap(); + let immutable_before = private_read(&immutable(&store, &receipt)).unwrap(); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + assert!(store.load_candidate(model.context()).is_err()); + assert!(store.save_candidate(&model).is_err()); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600)).unwrap(); + assert_eq!(private_read(&path).unwrap(), before); + // A filesystem obstacle at the destination refuses publication and retains + // the old pointer (moved aside by the fixture) and immutable learned bytes. + let old = path.with_extension("old"); + std::fs::rename(&path, &old).unwrap(); + std::fs::create_dir(&path).unwrap(); + assert!(store.save_candidate(&model).is_err()); + assert_eq!(private_read(&old).unwrap(), before); + assert_eq!( + private_read(&immutable(&store, &receipt)).unwrap(), + immutable_before + ); +} +#[test] +fn claimed_qualification_and_unbounded_or_traversing_pointers_are_rejected() { + let temp = tempfile::tempdir().unwrap(); + let store = store(temp.path()); + let model = model(); + let receipt = store.save_candidate(&model).unwrap(); + let path = pointer(&store, &model); + let original = private_read(&path).unwrap(); + let artifact_before = private_read(&immutable(&store, &receipt)).unwrap(); + let mut forged: serde_json::Value = serde_json::from_slice(&original).unwrap(); + forged["physically_validated"] = serde_json::json!(true); + let mut traversing: serde_json::Value = serde_json::from_slice(&original).unwrap(); + traversing["model_sha256"] = serde_json::json!("../../accepted-model"); + for bytes in [ + serde_json::to_vec(&forged).unwrap(), + serde_json::to_vec(&traversing).unwrap(), + vec![b' '; LIMIT + 1], + ] { + atomic_private_write(&path, &bytes).unwrap(); + assert!(store.load_candidate(model.context()).is_err()); + assert!(store.save_candidate(&model).is_err()); + assert_eq!(std::fs::read(&path).unwrap(), bytes); + assert_eq!( + private_read(&immutable(&store, &receipt)).unwrap(), + artifact_before + ); + } +} +#[test] +fn unowned_legacy_data_and_blocked_context_paths_are_preserved() { + let temp = tempfile::tempdir().unwrap(); + let legacy = temp.path().join(owner()).join("SensingServer"); + std::fs::create_dir_all(&legacy).unwrap(); + std::fs::write(legacy.join("legacy-model"), b"unowned bytes").unwrap(); + assert!(ArtifactStore::new(&legacy, &owner(), "synthetic-install").is_err()); + assert_eq!( + std::fs::read(legacy.join("legacy-model")).unwrap(), + b"unowned bytes" + ); + assert!(!legacy.join("account-owner-v1.json").exists()); + let other = tempfile::tempdir().unwrap(); + let store = store(other.path()); + let model = model(); + let blocked = store.root.join(model.context().digest()); + atomic_private_write(&blocked, b"existing obstacle").unwrap(); + assert!(store.save_candidate(&model).is_err()); + assert_eq!(private_read(&blocked).unwrap(), b"existing obstacle"); +} + +#[test] +fn catalog_retains_exact_saved_source_contexts_and_quarantines_other_corrupt_entries() { + let temp=tempfile::tempdir().unwrap();let store=store(temp.path());let a=model(); + let receipt=store.save_candidate(&a).unwrap(); + let unknown=store.root.join("f".repeat(64));std::fs::create_dir(&unknown).unwrap(); + atomic_private_write(&unknown.join(POINTER),b"bad pointer").unwrap(); + let listed=store.catalog().unwrap(); + assert_eq!(listed.candidates.len(),1);assert_eq!(listed.unreadable_contexts,1); + assert_eq!(listed.candidates[0].context,*a.context()); + assert_eq!(listed.candidates[0].receipt,receipt); + assert!(store.load_candidate(a.context()).unwrap().is_some()); + assert_eq!(private_read(&unknown.join(POINTER)).unwrap(),b"bad pointer"); + let encoded=serde_json::to_string(&listed).unwrap(); + assert!(!encoded.contains("encoder_weights") && !encoded.contains("samples")); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/fit_config.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/fit_config.rs new file mode 100644 index 0000000000..8adec18220 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/fit_config.rs @@ -0,0 +1,101 @@ +//! Bounded fit settings and actual architecture preflight. +use super::*; +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FitConfig { + pub epochs: usize, + pub learning_rate: f32, + pub contrastive_weight: f32, + pub supervised_weight: f32, + pub gradient_epsilon: f32, + pub max_parameters: usize, + pub max_duration_ms: u64, + pub max_objective_evaluations: usize, + pub hidden_dim: usize, + pub embedding_dim: usize, + pub temporal_tokens: usize, + pub temperature: f32, +} +impl Default for FitConfig { + fn default() -> Self { + Self { + epochs: 48, + learning_rate: 0.2, + contrastive_weight: 0.1, + supervised_weight: 1.0, + gradient_epsilon: 0.002, + max_parameters: 512, + max_duration_ms: 60_000, + max_objective_evaluations: 100_000, + hidden_dim: 3, + embedding_dim: 3, + temporal_tokens: 4, + temperature: 0.3, + } + } +} +impl FitConfig { + /// Keep every advertised source count within the bounded optimizer budget. + pub fn for_source_count(source_count: usize) -> Self { + let mut config = Self::default(); + if source_count >= 7 { + config.hidden_dim = 2; + config.embedding_dim = 2; + } + config + } + pub fn validate_context(&self, context: &CaptureContext) -> Result { + self.validate()?; + context.validate()?; + let config = EncoderConfig { + source_count: context.sources.len(), + bins: context + .sources + .iter() + .map(|s| s.feature_count) + .max() + .unwrap_or(0), + hidden_dim: self.hidden_dim, + embedding_dim: self.embedding_dim, + temporal_tokens: self.temporal_tokens, + temperature: self.temperature, + }; + let count = initial_encoder(&config)? + .transformer + .flatten_weights() + .len() + + initial_projection_weights(&config) + .iter() + .map(Vec::len) + .sum::() + + context.sources.len() * (self.embedding_dim + 2) + + 1; + if count > self.max_parameters { + return Err("model exceeds parameter budget".into()); + } + Ok(count) + } + pub fn validate(&self) -> Result<(), String> { + if !(1..=128).contains(&self.epochs) + || !(1..=512).contains(&self.max_parameters) + || !(1..=120_000).contains(&self.max_duration_ms) + || !(1..=200_000).contains(&self.max_objective_evaluations) + || !(2..=4).contains(&self.hidden_dim) + || !(2..=4).contains(&self.embedding_dim) + || !(2..=16).contains(&self.temporal_tokens) + || !self.learning_rate.is_finite() + || !(0.0001..=1.0).contains(&self.learning_rate) + || !self.contrastive_weight.is_finite() + || !(0.001..=10.0).contains(&self.contrastive_weight) + || !self.supervised_weight.is_finite() + || !(0.001..=10.0).contains(&self.supervised_weight) + || !self.gradient_epsilon.is_finite() + || !(0.0001..=0.05).contains(&self.gradient_epsilon) + || !self.temperature.is_finite() + || !(0.1..=2.0).contains(&self.temperature) + { + return Err("invalid bounded training configuration".into()); + } + Ok(()) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/learning_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/learning_tests.rs new file mode 100644 index 0000000000..03415db175 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/learning_tests.rs @@ -0,0 +1,468 @@ +//! SYNTHETIC leakage-free RF fixtures: learning mechanics, not room accuracy. +use crate::room_learner::model::*; +use crate::room_learner::observation::*; +use crate::room_learner::training::*; +use ruview_unified::RfModality; +use std::sync::atomic::AtomicBool; + +fn context(width: usize) -> CaptureContext { + CaptureContext { + owner_scope: "synthetic-owner".into(), + installation_id: "synthetic-install".into(), + room_digest: "synthetic-room".into(), + revision: 1, + sources: (1..=2) + .map(|id| SourceSpec { + source_id: id, + device_id: format!("synthetic-device-{id}"), + grid_id: format!("phy-{width}-bins"), + feature_count: width, + link_index: 0, + center_frequency_hz: 2_400_000_000, + bandwidth_hz: 20_000_000, + signal_kind: RfModality::Synthetic, + link_id: format!("link-{id}"), + geometry: None, + }) + .collect(), + } +} +fn cohort(c: &CaptureContext, id: u64, label: RoomLabel, role: DataRole) -> CaptureCohort { + let views = c + .sources + .iter() + .map(|source| { + let shift = match label { + RoomLabel::Empty | RoomLabel::Unknown => 0.0, + RoomLabel::AdjacentOnly => { + if source.source_id == 1 { + 1.7 + } else { + -1.7 + } + } + RoomLabel::OccupiedStatic | RoomLabel::OccupiedMoving => 1.7, + }; + let variation = (id % 5) as f32 * 0.015; + let samples = (0..4) + .map(|t| { + (0..source.feature_count) + .map(|b| { + let motion = if label == RoomLabel::OccupiedMoving { + (t as f32 * 1.3).sin() * 0.12 + } else { + 0.0 + }; + 10.0 + b as f32 * 0.003 + + (t as f32 - 1.5) * 0.03 + + shift + + variation + + motion + }) + .collect() + }) + .collect(); + SourceView { + source_id: source.source_id, + device_id: source.device_id.clone(), + grid_id: source.grid_id.clone(), + signal_kind: source.signal_kind, + link_index: 0, + center_frequency_hz: 2_400_000_000, + bandwidth_hz: 20_000_000, + link_id: source.link_id.clone(), + geometry: None, + samples, + frame_ids: (0..4).map(|t| u128::from(id) * 10 + t + 1).collect(), + sequence_ids: vec![1, 2, 3, 4], + first_time_ns: id * 1000 + 1, + last_time_ns: id * 1000 + 101, + clock: ClockInfo { + kind: ClockKind::Unknown, + uncertainty_ns: u64::MAX, + }, + sample_age_ns: 0, + phase: PhaseAvailability::MagnitudeOnly, + provenance: CaptureProvenance::Synthetic, + } + }) + .collect(); + CaptureCohort { + id: format!("synthetic-{id}"), + context_digest: c.digest(), + trial_id: format!("trial-{id}"), + session_id: format!("session-{id}"), + role, + label, + views, + teacher: None, + provenance: CaptureProvenance::Synthetic, + } +} +pub(crate) fn dataset(width: usize) -> ValidatedDataset { + let c = context(width); + let labels = [ + RoomLabel::Empty, + RoomLabel::AdjacentOnly, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + ]; + let mut rows = Vec::new(); + for (i, label) in labels.iter().cycle().take(8).enumerate() { + rows.push(cohort(&c, i as u64 + 1, *label, DataRole::Train)); + } + for (i, label) in labels.iter().enumerate() { + rows.push(cohort(&c, 100 + i as u64, *label, DataRole::Validation)); + rows.push(cohort(&c, 200 + i as u64, *label, DataRole::HeldOut)); + } + ValidatedDataset::new(c, rows).unwrap() +} +fn config() -> FitConfig { + FitConfig { + epochs: 32, + ..FitConfig::default() + } +} + +#[test] +fn actual_encoder_and_head_weights_reduce_declared_objective_and_learn_controls() { + let ds = dataset(8); + let (model, report) = RoomModel::fit(&ds, config()).unwrap(); + eprintln!("SYNTHETIC report: {report:?}"); + assert!(report.completed_epochs > 0); + assert!( + report.encoder_delta_l2 > 1.0e-5, + "zero-gradient/skipped-encoder mutant survived" + ); + assert!( + report.head_delta_l2 > 1.0e-3, + "skipped task update mutant survived" + ); + assert!(report.final_objective.total < report.initial.total * 0.8); + assert!(report.final_objective.supervised < report.initial.supervised * 0.5); + let held = model.evaluate(ds.held_out()).unwrap(); + assert_eq!( + held.correct, 4, + "SYNTHETIC disjoint occupancy/adjacent controls: {held:?}" + ); + assert_eq!(held.abstained, 0); + assert!(!model.infer(&ds.held_out()[0]).physically_validated); +} +#[test] +fn held_out_and_validation_changes_cannot_update_weights_or_training_profiles() { + let ds = dataset(8); + let mut rows = ds.train().to_vec(); + for c in ds.validation().iter().chain(ds.held_out()) { + let mut other = c.clone(); + other.label = RoomLabel::OccupiedMoving; + for v in &mut other.views { + for frame in &mut v.samples { + for value in frame { + *value += 0.05; + } + } + } + rows.push(other); + } + let changed = ValidatedDataset::new(ds.context().clone(), rows).unwrap(); + let cfg = FitConfig { + epochs: 8, + ..config() + }; + let (a, ar) = RoomModel::fit(&ds, cfg.clone()).unwrap(); + let (b, br) = RoomModel::fit(&changed, cfg).unwrap(); + assert_eq!(a.parameters(), b.parameters()); + assert_eq!(a.profiles(), b.profiles()); + assert_eq!(a.training_manifest(), b.training_manifest()); + assert_eq!(a.export().unwrap(), b.export().unwrap()); + assert_eq!(ar.final_objective.total, br.final_objective.total); +} +#[test] +fn source_order_dc_and_full_56_bin_grid_reach_real_aether_encoder() { + let ds = dataset(56); + let (model, report) = RoomModel::fit( + &ds, + FitConfig { + epochs: 2, + ..config() + }, + ) + .unwrap(); + assert_eq!(model.encoder_config().bins, 56); + assert!(report.parameter_count <= 512); + assert!(report.encoder_delta_l2 > 0.0); + let row = &ds.held_out()[1]; + let tokens = model.tokens(row, false).unwrap(); + assert_eq!(tokens[0][0].len(), 58); + assert_eq!(&tokens[0][0][56..], &[1.0, 0.0]); + assert_eq!(&tokens[1][0][56..], &[0.0, 1.0]); + // Individual native frequency bins survive; no fixed8-bin summary is used. + let mut changed = row.clone(); + changed.views[0].samples[0][55] += 0.05; + let changed_tokens = model.tokens(&changed, false).unwrap(); + assert_eq!(tokens[0][0][54], changed_tokens[0][0][54]); + assert_ne!(tokens[0][0][55], changed_tokens[0][0][55]); + let (_, a) = model.represent(&tokens, &model.parameters()).unwrap(); + let (_, b) = model + .represent(&changed_tokens, &model.parameters()) + .unwrap(); + let backbone = |input: &[Vec]| { + let mut encoder = extractor(model.encoder_config()); + encoder + .transformer + .unflatten_weights(&model.encoder_weights) + .unwrap(); + let mut input = input.to_vec(); + for row in &mut input { + for x in &mut row[..56] { + *x /= model.profiles()[0].max_normalized; + } + } + encoder.transformer.embed(&input) + }; + assert_ne!( + backbone(&tokens[0]), + backbone(&changed_tokens[0]), + "canonical bin must reach shared RF backbone" + ); + assert_eq!(a[1], b[1], "source0 cannot erase distinct source1 encoding"); + let empty = model.tokens(&ds.held_out()[0], false).unwrap(); + let occupied = model.tokens(&ds.held_out()[2], false).unwrap(); + assert!( + occupied[0][0][0] - empty[0][0][0] > 1.0, + "per-window normalization erased DC" + ); +} +#[test] +fn contrastive_objective_updates_encoder_and_is_not_a_logged_task_loss() { + let ds = dataset(8); + let (contrast, cr) = RoomModel::fit( + &ds, + FitConfig { + epochs: 16, + contrastive_weight: 5.0, + supervised_weight: 0.001, + ..config() + }, + ) + .unwrap(); + let (task, _) = RoomModel::fit( + &ds, + FitConfig { + epochs: 16, + contrastive_weight: 0.001, + ..config() + }, + ) + .unwrap(); + eprintln!( + "SYNTHETIC contrastive objective {} -> {}", + cr.initial.contrastive, cr.final_objective.contrastive + ); + assert!(cr.final_objective.contrastive < cr.initial.contrastive - 0.001); + assert!(cr.encoder_delta_l2 > 1.0e-4); + assert_ne!(contrast.encoder_weights, task.encoder_weights); +} +#[test] +fn model_round_trip_has_identical_predictions_and_no_observation_payload() { + let ds = dataset(8); + let (model, _) = RoomModel::fit( + &ds, + FitConfig { + epochs: 3, + ..config() + }, + ) + .unwrap(); + let bytes = model.export().unwrap(); + let json = String::from_utf8(bytes.clone()).unwrap(); + for raw in [ + "samples", + "frame_ids", + "sequence_ids", + "trial_id", + "synthetic-200", + ] { + assert!(!json.contains(raw)); + } + let imported = RoomModel::import(&bytes).unwrap(); + assert_eq!(model.parameters(), imported.parameters()); + assert_eq!( + model.infer(&ds.held_out()[2]).occupied_probability, + imported.infer(&ds.held_out()[2]).occupied_probability + ); + let original: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + for mutant in [ + "version", + "encoder_weights", + "head_weights", + "profiles", + "config", + "algorithm", + ] { + let mut value = original.clone(); + value[mutant] = serde_json::json!(null); + assert!( + RoomModel::import(&serde_json::to_vec(&value).unwrap()).is_err(), + "accepted malformed {mutant}" + ); + } + let mut invalid = model.clone(); + invalid.encoder_weights[0] = f32::NAN; + assert!(invalid.export().is_err()); + assert!(RoomModel::import(&vec![b' '; 1_048_577]).is_err()); +} +#[test] +fn missing_changed_stale_nonfinite_and_out_of_support_sources_abstain() { + let ds = dataset(8); + let (model, _) = RoomModel::fit( + &ds, + FitConfig { + epochs: 2, + ..config() + }, + ) + .unwrap(); + type Mutant = fn(&mut CaptureCohort); + let mutants: &[Mutant] = &[ + |c| { + c.views.pop(); + }, + |c| c.context_digest = "different-installation".into(), + |c| c.views[0].device_id = "different-device".into(), + |c| c.views[0].grid_id = "other-phy".into(), + |c| c.views[0].samples[0][0] = f32::NAN, + |c| c.views[0].samples[0][0] = 1.0e8, + |c| c.views[0].sample_age_ns = MAX_AGE_NS + 1, + ]; + for mutate in mutants { + let mut row = ds.held_out()[0].clone(); + mutate(&mut row); + let prediction = model.infer(&row); + assert_eq!(prediction.decision, PresenceDecision::Abstain); + assert!(prediction.occupied_probability.is_none()); + } +} +#[test] +fn parameter_time_evaluation_budgets_and_cancellation_fail_closed() { + let ds = dataset(8); + assert!(RoomModel::fit( + &ds, + FitConfig { + max_parameters: 1, + ..config() + } + ) + .unwrap_err() + .contains("parameter budget")); + assert!(RoomModel::fit( + &ds, + FitConfig { + max_objective_evaluations: 2, + ..config() + } + ) + .unwrap_err() + .contains("evaluation budget")); + assert!(RoomModel::fit( + &ds, + FitConfig { + max_duration_ms: 1, + ..config() + } + ) + .unwrap_err() + .contains("time budget")); + assert!( + RoomModel::fit_with_cancel(&ds, config(), &AtomicBool::new(true)) + .unwrap_err() + .contains("cancelled") + ); + for cfg in [ + FitConfig { + learning_rate: f32::NAN, + ..config() + }, + FitConfig { + epochs: 129, + ..config() + }, + FitConfig { + hidden_dim: 5000, + ..config() + }, + ] { + assert!(RoomModel::fit(&ds, cfg).is_err()); + } +} + +#[test] +fn sibling_trials_shared_frames_and_overlapping_windows_never_become_negatives() { + let ds = dataset(8); + let mut rows = ds.train().to_vec(); + rows[1].trial_id = rows[0].trial_id.clone(); + for i in 0..2 { + rows[2].views[i].first_time_ns = rows[0].views[i].first_time_ns; + rows[2].views[i].last_time_ns = rows[0].views[i].last_time_ns; + rows[3].views[i].frame_ids[0] = rows[0].views[i].frame_ids[0]; + } + let ds = ValidatedDataset::new(ds.context().clone(), rows).unwrap(); + let (_, report) = RoomModel::fit( + &ds, + FitConfig { + epochs: 1, + ..config() + }, + ) + .unwrap(); + assert_eq!(report.training_cohorts, 8); + assert_eq!(report.contrastive_trials, 5); +} +#[test] +fn running_fit_observes_cancellation_without_publishing_partial_model() { + use std::{ + sync::{atomic::Ordering, Arc}, + thread, + time::Duration, + }; + let ds = dataset(8); + let cancel = Arc::new(AtomicBool::new(false)); + let trigger = cancel.clone(); + let handle = thread::spawn(move || { + thread::sleep(Duration::from_millis(2)); + trigger.store(true, Ordering::SeqCst); + }); + let result = RoomModel::fit_with_cancel( + &ds, + FitConfig { + epochs: 128, + ..config() + }, + &cancel, + ); + handle.join().unwrap(); + assert!(result.unwrap_err().contains("cancelled")); +} + +#[test] +fn inference_does_not_invent_empty_ground_truth_or_enter_evaluation() { + let ds = dataset(8); + let (model, _) = RoomModel::fit( + &ds, + FitConfig { + epochs: 2, + ..config() + }, + ) + .unwrap(); + let mut live = ds.held_out()[2].clone(); + let expected = model.infer(&live).occupied_probability; + live.role = DataRole::Inference; + live.label = RoomLabel::Unknown; + assert_eq!(model.infer(&live).occupied_probability, expected); + assert!(model + .evaluate(&[live.clone()]) + .unwrap_err() + .contains("labeled")); + assert!(ValidatedDataset::new(ds.context().clone(), vec![live]).is_err()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/manifest.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/manifest.rs new file mode 100644 index 0000000000..5587f18c73 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/manifest.rs @@ -0,0 +1,131 @@ +//! Immutable input claims and hash from Train rows only; no raw rows retained. +use crate::room_learner::observation::{ + CaptureCohort, CaptureProvenance, ClockKind, DataRole, RoomLabel, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::io::{self, Write}; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ConsumedChannel { + Magnitude, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TrainingManifest { + pub schema: String, + pub training_provenance: Vec, + pub training_data_sha256: String, + pub consumed_channels: Vec, + /// Validated input clock declarations, never an independent hardware receipt. + pub clock_claims: Vec, + pub measurement_clock_available: bool, + pub training_cohorts: usize, +} +fn provenance_rank(value: &CaptureProvenance) -> u8 { + match value { + CaptureProvenance::Synthetic => 0, + CaptureProvenance::CapturedReplay => 1, + CaptureProvenance::LiveReceiverObserved => 2, + } +} +fn clock_rank(value: &ClockKind) -> u8 { + match value { + ClockKind::Unknown => 0, + ClockKind::ReceiverOnly => 1, + ClockKind::SourceMeasurement => 2, + } +} +struct HashWriter(Sha256); +impl Write for HashWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.update(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +impl TrainingManifest { + pub(crate) fn from_training(rows: &[CaptureCohort]) -> Result { + if !(4..=64).contains(&rows.len()) + || rows + .iter() + .any(|c| c.role != DataRole::Train || c.label == RoomLabel::Unknown) + { + return Err("training manifest requires bounded labeled Train rows".into()); + } + if rows.iter().any(|c| { + c.views.iter().any(|v| { + v.provenance != c.provenance + || (v.signal_kind == ruview_unified::RfModality::Synthetic + && v.provenance != CaptureProvenance::Synthetic) + }) + }) { + return Err( + "synthetic RF or view provenance cannot claim captured/live training".into(), + ); + } + let mut provenance: Vec<_> = rows.iter().map(|r| r.provenance).collect(); + provenance.sort_by_key(provenance_rank); + provenance.dedup(); + let mut clocks: Vec<_> = rows + .iter() + .flat_map(|r| r.views.iter().map(|v| v.clock.kind)) + .collect(); + clocks.sort_by_key(clock_rank); + clocks.dedup(); + let available = !provenance.contains(&CaptureProvenance::Synthetic) + && clocks == [ClockKind::SourceMeasurement]; + let mut hash = HashWriter(Sha256::new()); + serde_json::to_writer(&mut hash, rows).map_err(|e| e.to_string())?; + let manifest = Self { + schema: "ruview.room-learner.train-manifest.v1".into(), + training_provenance: provenance, + training_data_sha256: format!("{:x}", hash.0.finalize()), + consumed_channels: vec![ConsumedChannel::Magnitude], + clock_claims: clocks, + measurement_clock_available: available, + training_cohorts: rows.len(), + }; + manifest.validate()?; + Ok(manifest) + } + pub(crate) fn validate(&self) -> Result<(), String> { + let expected_clock = !self + .training_provenance + .contains(&CaptureProvenance::Synthetic) + && self.clock_claims == [ClockKind::SourceMeasurement]; + if self.schema != "ruview.room-learner.train-manifest.v1" + || !(4..=64).contains(&self.training_cohorts) + || self.training_data_sha256.len() != 64 + || !self + .training_data_sha256 + .bytes() + .all(|c| c.is_ascii_digit() || (b'a'..=b'f').contains(&c)) + || self.training_provenance.is_empty() + || self.training_provenance.len() > 3 + || self + .training_provenance + .windows(2) + .any(|w| provenance_rank(&w[0]) >= provenance_rank(&w[1])) + || self.clock_claims.is_empty() + || self.clock_claims.len() > 3 + || self + .clock_claims + .windows(2) + .any(|w| clock_rank(&w[0]) >= clock_rank(&w[1])) + || self.consumed_channels != [ConsumedChannel::Magnitude] + || self.measurement_clock_available != expected_clock + { + return Err("invalid immutable Train provenance manifest".into()); + } + Ok(()) + } +} + +#[cfg(test)] +#[path = "manifest_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/manifest_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/manifest_tests.rs new file mode 100644 index 0000000000..c7b5585e27 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/manifest_tests.rs @@ -0,0 +1,142 @@ +//! SYNTHETIC manifest integrity; provenance remains an input claim, not a receipt. +use super::*; +use crate::room_learner::{ + model::RoomModel, + training::{learning_tests::dataset, FitConfig}, +}; +use std::sync::OnceLock; +fn model() -> RoomModel { + static MODEL: OnceLock = OnceLock::new(); + MODEL + .get_or_init(|| { + RoomModel::fit( + &dataset(8), + FitConfig { + epochs: 2, + ..FitConfig::default() + }, + ) + .unwrap() + .0 + }) + .clone() +} +#[test] +fn streamed_digest_covers_actual_train_bytes_and_consumed_channels_are_explicit() { + let ds = dataset(8); + let manifest = TrainingManifest::from_training(ds.train()).unwrap(); + assert_eq!( + manifest.training_data_sha256, + format!( + "{:x}", + Sha256::digest(serde_json::to_vec(ds.train()).unwrap()) + ) + ); + assert_eq!(manifest.training_provenance, [CaptureProvenance::Synthetic]); + assert_eq!(manifest.consumed_channels, [ConsumedChannel::Magnitude]); + assert_eq!(manifest.clock_claims, [ClockKind::Unknown]); + assert!(!manifest.measurement_clock_available); + assert_eq!(manifest.training_cohorts, 8); + let mut altered = ds.train().to_vec(); + altered[0].views[0].samples[0][0] += 0.01; + assert_ne!( + manifest.training_data_sha256, + TrainingManifest::from_training(&altered) + .unwrap() + .training_data_sha256 + ); + assert!(TrainingManifest::from_training(ds.held_out()).is_err()); + assert!(TrainingManifest::from_training(ds.validation()).is_err()); +} +#[test] +fn model_round_trip_retains_synthetic_origin_without_raw_rows_or_phone_channels() { + let trained = model(); + let bytes = trained.export().unwrap(); + let imported = RoomModel::import(&bytes).unwrap(); + assert_eq!(trained.training_manifest(), imported.training_manifest()); + assert_eq!( + imported.training_manifest().training_provenance, + [CaptureProvenance::Synthetic] + ); + assert!(!imported.training_manifest().measurement_clock_available); + let json = String::from_utf8(bytes).unwrap(); + for raw in [ + "samples", + "frame_ids", + "trial_id", + "phase", + "camera", + "phone", + ] { + assert!(!json.contains(raw)); + } +} +#[test] +fn missing_unknown_duplicated_forged_live_and_nonfinite_manifest_fields_are_rejected() { + let bytes = model().export().unwrap(); + let original: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + let mut missing = original.clone(); + missing.as_object_mut().unwrap().remove("training_manifest"); + assert!(RoomModel::import(&serde_json::to_vec(&missing).unwrap()).is_err()); + for (field, value) in [ + ("training_provenance", serde_json::json!([])), + ( + "training_provenance", + serde_json::json!(["UnknownPhysicalSource"]), + ), + ( + "training_provenance", + serde_json::json!(["Synthetic", "Synthetic"]), + ), + ( + "training_provenance", + serde_json::json!(["LiveReceiverObserved"]), + ), + ( + "training_data_sha256", + serde_json::json!("not-a-training-data-hash"), + ), + ("training_data_sha256", serde_json::json!("A".repeat(64))), + ("consumed_channels", serde_json::json!(["phase"])), + ("consumed_channels", serde_json::json!([])), + ("clock_claims", serde_json::json!(["Unknown", "Unknown"])), + ("measurement_clock_available", serde_json::json!(true)), + ("training_cohorts", serde_json::json!(1.5)), + ("training_cohorts", serde_json::Value::Null), + ] { + let mut candidate = original.clone(); + candidate["training_manifest"][field] = value; + assert!( + RoomModel::import(&serde_json::to_vec(&candidate).unwrap()).is_err(), + "accepted malformed {field}" + ); + } + let nonfinite = String::from_utf8(bytes).unwrap().replace( + "\"measurement_clock_available\":false", + "\"measurement_clock_available\":NaN", + ); + assert!(RoomModel::import(nonfinite.as_bytes()).is_err()); +} +#[test] +fn source_clock_claims_never_promote_synthetic_to_measured_or_live() { + let ds = dataset(8); + let mut rows = ds.train().to_vec(); + for c in &mut rows { + for v in &mut c.views { + v.clock.kind = ClockKind::SourceMeasurement; + v.clock.uncertainty_ns = 0; + } + } + let manifest = TrainingManifest::from_training(&rows).unwrap(); + assert_eq!(manifest.clock_claims, [ClockKind::SourceMeasurement]); + assert!(!manifest.measurement_clock_available); + for c in &mut rows { + c.provenance = CaptureProvenance::LiveReceiverObserved; + for v in &mut c.views { + v.provenance = CaptureProvenance::LiveReceiverObserved; + } + } + assert!(TrainingManifest::from_training(&rows) + .unwrap_err() + .contains("synthetic RF")); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs index c470e96931..3e015e2801 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs @@ -2,3 +2,6 @@ pub mod observation; pub mod capture; pub mod native; +pub mod model; +pub mod training; +pub mod artifact; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/model.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/model.rs new file mode 100644 index 0000000000..814ca2c995 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/model.rs @@ -0,0 +1,431 @@ +//! Portable RF-only room presence learner. No raw observations are persisted. +use super::observation::{CaptureCohort, CaptureContext, SourceSpec}; +use crate::embedding::{EmbeddingConfig, EmbeddingExtractor}; +use crate::graph_transformer::{CrossAttention, Linear, TransformerConfig}; +use serde::{Deserialize, Serialize}; + +#[path = "manifest.rs"] +mod manifest; +pub use manifest::{ConsumedChannel, TrainingManifest}; +#[path = "projection.rs"] +mod projection; +pub(crate) use projection::{center_empty_heads, initial_weights as initial_projection_weights}; + +const MAX_ARTIFACT_BYTES: usize = 1_048_576; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct SourceProfile { + pub source: SourceSpec, + pub mean: Vec, + pub scale: Vec, + pub max_normalized: f32, +} + +/// Versioned, bounded architecture. Values are checked before allocating a network. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct EncoderConfig { + pub bins: usize, + /// Stable ordered context sources; metadata tags are not measured RF bins. + pub source_count: usize, + pub temporal_tokens: usize, + pub hidden_dim: usize, + pub embedding_dim: usize, + pub temperature: f32, +} +impl Default for EncoderConfig { + fn default() -> Self { + Self { + bins: 56, + source_count: 2, + temporal_tokens: 4, + hidden_dim: 3, + embedding_dim: 3, + temperature: 0.3, + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RoomModel { + pub(crate) version: u32, + pub(crate) algorithm: String, + pub(crate) context: CaptureContext, + pub(crate) training_manifest: TrainingManifest, + pub(crate) config: EncoderConfig, + pub(crate) profiles: Vec, + pub(crate) encoder_weights: Vec, + pub(crate) source_projection_weights: Vec>, + pub(crate) head_weights: Vec, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum PresenceDecision { + Empty, + Occupied, + Abstain, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct RoomPrediction { + pub decision: PresenceDecision, + pub occupied_probability: Option, + pub reason: Option, + /// Always false: a trained candidate is not physical qualification evidence. + pub physically_validated: bool, +} +impl RoomPrediction { + fn abstain(reason: String) -> Self { + Self { + decision: PresenceDecision::Abstain, + occupied_probability: None, + reason: Some(reason), + physically_validated: false, + } + } +} + +pub(crate) fn extractor(config: &EncoderConfig) -> EmbeddingExtractor { + EmbeddingExtractor::new( + TransformerConfig { + n_subcarriers: config.bins + config.source_count, + n_keypoints: 17, + d_model: config.hidden_dim, + n_heads: 1, + n_gnn_layers: 1, + }, + EmbeddingConfig { + d_model: config.hidden_dim, + d_proj: config.embedding_dim, + temperature: config.temperature, + normalize: false, + }, + ) +} + +/// Keep the tiny reused ReLU graph active around the calibrated empty baseline. +/// Layout follows CsiToPoseTransformer::flatten_weights' documented ordering. +pub(crate) fn initial_encoder(config: &EncoderConfig) -> Result { + let mut encoder = extractor(config); + let d = config.hidden_dim; + let mut weights = encoder.transformer.flatten_weights(); + // A separate fan-in for source metadata keeps its scale independent of + // how many canonical RF bins are present. This assumes no response polarity. + let input_width = config.bins + config.source_count; + let tags = Linear::with_seed(config.source_count, d, 501); + for row in 0..d { + for source in 0..config.source_count { + weights[row * input_width + config.bins + source] = tags.weights()[row][source]; + } + } + let gnn = (config.bins + config.source_count + 1) * d + + 17 * d + + CrossAttention::new(d, 1).param_count(); + for r in 0..d { + for c in 0..d { + weights[gnn + r * d + c] = if r == c { 1.0 } else { 0.0 }; + } + } + weights[gnn + d * d..gnn + d * d + d].fill(1.0); + encoder.transformer.unflatten_weights(&weights)?; + Ok(encoder) +} + +impl RoomModel { + pub fn context(&self) -> &CaptureContext { + &self.context + } + pub fn training_manifest(&self) -> &TrainingManifest { + &self.training_manifest + } + pub fn profiles(&self) -> &[SourceProfile] { + &self.profiles + } + pub fn encoder_config(&self) -> &EncoderConfig { + &self.config + } + pub fn parameter_count(&self) -> usize { + self.representation_parameter_count() + self.head_weights.len() + } + pub fn representation_parameter_count(&self) -> usize { + self.encoder_weights.len() + + self + .source_projection_weights + .iter() + .map(Vec::len) + .sum::() + } + /// Read-only learned parameters for independent audits and regression oracles. + pub fn parameters(&self) -> Vec { + self.encoder_weights + .iter() + .chain(self.source_projection_weights.iter().flatten()) + .chain(&self.head_weights) + .copied() + .collect() + } + pub(crate) fn set_parameters(&mut self, values: &[f32]) -> Result<(), String> { + if values.len() != self.parameter_count() || !values.iter().all(|x| x.is_finite()) { + return Err("invalid model parameter vector".into()); + } + let mut offset = self.encoder_weights.len(); + self.encoder_weights.copy_from_slice(&values[..offset]); + for weights in &mut self.source_projection_weights { + let end = offset + weights.len(); + weights.copy_from_slice(&values[offset..end]); + offset = end; + } + self.head_weights.copy_from_slice(&values[offset..]); + Ok(()) + } + pub fn export(&self) -> Result, String> { + self.validate()?; + let bytes = serde_json::to_vec(self).map_err(|e| e.to_string())?; + if bytes.len() > MAX_ARTIFACT_BYTES { + return Err("model artifact too large".into()); + } + Ok(bytes) + } + pub fn import(bytes: &[u8]) -> Result { + if bytes.len() > MAX_ARTIFACT_BYTES { + return Err("model artifact too large".into()); + } + let model: Self = serde_json::from_slice(bytes).map_err(|e| e.to_string())?; + model.validate()?; + Ok(model) + } + pub(crate) fn validate(&self) -> Result<(), String> { + self.training_manifest.validate()?; + if self + .context + .sources + .iter() + .any(|s| s.signal_kind == ruview_unified::RfModality::Synthetic) + && self.training_manifest.training_provenance + != [super::observation::CaptureProvenance::Synthetic] + { + return Err("synthetic modality cannot have live training provenance".into()); + } + if self.version != 1 + || self.algorithm != "aether_multiview_infonce_room_bce_v1" + || !(1..=128).contains(&self.config.bins) + || !(2..=8).contains(&self.config.source_count) + || self.config.source_count != self.context.sources.len() + || !(2..=16).contains(&self.config.temporal_tokens) + || !(2..=4).contains(&self.config.hidden_dim) + || !(2..=4).contains(&self.config.embedding_dim) + || !self.config.temperature.is_finite() + || !(0.1..=2.0).contains(&self.config.temperature) + { + return Err("unsupported room model version or architecture".into()); + } + if !(2..=8).contains(&self.context.sources.len()) + || self.profiles.len() != self.context.sources.len() + { + return Err("invalid source profile count".into()); + } + // Validate the context through the same trusted contract as capture. + self.context.validate()?; + if self + .context + .sources + .iter() + .any(|s| s.feature_count != self.config.bins) + { + return Err("encoder width does not match source grids".into()); + } + for (profile, source) in self.profiles.iter().zip(&self.context.sources) { + if &profile.source != source + || profile.mean.len() != source.feature_count + || profile.scale.len() != source.feature_count + || !profile.mean.iter().all(|x| x.is_finite()) + || !profile.scale.iter().all(|x| x.is_finite() && *x >= 0.01) + || !profile.max_normalized.is_finite() + || !(1.0..=1.0e6).contains(&profile.max_normalized) + { + return Err("invalid normalization profile".into()); + } + } + if self.encoder_weights.len() != extractor(&self.config).transformer.param_count() + || self.source_projection_weights.len() != self.config.source_count + || self.source_projection_weights.iter().any(|w| { + w.len() + != crate::embedding::ProjectionHead::new(projection::config(&self.config)) + .param_count() + }) + || self.head_weights.len() != self.profiles.len() * (self.config.embedding_dim + 2) + 1 + || self.parameter_count() > 512 + || !self + .parameters() + .iter() + .all(|x| x.is_finite() && x.abs() <= 1.0e4) + { + return Err("invalid learned weights".into()); + } + Ok(()) + } + pub fn infer(&self, cohort: &CaptureCohort) -> RoomPrediction { + let result = (|| { + self.validate()?; + let tokens = self.tokens(cohort, true)?; + let (features, _) = self.represent(&tokens, &self.parameters())?; + let probability = sigmoid(self.logit(&features, &self.head_weights)); + if !probability.is_finite() { + return Err("non-finite prediction".into()); + } + Ok(probability) + })(); + match result { + Err(e) => RoomPrediction::abstain(e), + Ok(p) => RoomPrediction { + occupied_probability: Some(p), + decision: if (0.45..=0.55).contains(&p) { + PresenceDecision::Abstain + } else if p > 0.5 { + PresenceDecision::Occupied + } else { + PresenceDecision::Empty + }, + reason: if (0.45..=0.55).contains(&p) { + Some("uncertain task probability".into()) + } else { + None + }, + physically_validated: false, + }, + } + } + pub(crate) fn tokens( + &self, + cohort: &CaptureCohort, + check_range: bool, + ) -> Result>>, String> { + super::observation::validate_cohort(&self.context, cohort)?; + let mut result = Vec::with_capacity(self.profiles.len()); + for (source_index, profile) in self.profiles.iter().enumerate() { + let view = cohort + .views + .iter() + .find(|v| v.source_id == profile.source.source_id) + .ok_or("missing trained source")?; + if view.samples.len() < self.config.temporal_tokens { + return Err("insufficient measured temporal rows".into()); + } + let bins = self.config.bins; + let time = self.config.temporal_tokens; + let mut tokens = vec![vec![0.0; bins]; time]; + let mut counts = vec![vec![0usize; bins]; time]; + for (t, frame) in view.samples.iter().enumerate() { + let ti = t * time / view.samples.len(); + for (k, value) in frame.iter().enumerate() { + let value = (*value - profile.mean[k]) / profile.scale[k]; + if !value.is_finite() + || (check_range && value.abs() > profile.max_normalized * 2.0 + 4.0) + { + return Err("RF input outside trained support".into()); + } + let bi = k; + tokens[ti][bi] += value; + counts[ti][bi] += 1; + } + } + for t in 0..time { + for b in 0..bins { + if counts[t][b] > 0 { + tokens[t][b] /= counts[t][b] as f32; + } + } + } + if !tokens.iter().flatten().all(|x| x.is_finite()) { + return Err("invalid RF tokens".into()); + } + // Source identity is a conditioning channel, appended only after + // RF baseline normalization; metadata never enters RF statistics. + for token in &mut tokens { + token.extend((0..self.config.source_count).map(|i| { + if i == source_index { + 1.0 + } else { + 0.0 + } + })); + } + result.push(tokens); + } + Ok(result) + } + pub(crate) fn represent( + &self, + tokens: &[Vec>], + params: &[f32], + ) -> Result<(Vec, Vec>), String> { + let mut encoder = extractor(&self.config); + encoder + .transformer + .unflatten_weights(¶ms[..self.encoder_weights.len()])?; + let projectors = self.source_projectors(params)?; + self.represent_with_encoder(tokens, &mut encoder, &projectors) + } + pub(crate) fn represent_with_encoder( + &self, + tokens: &[Vec>], + encoder: &mut EmbeddingExtractor, + projectors: &[crate::embedding::ProjectionHead], + ) -> Result<(Vec, Vec>), String> { + let mut features = + Vec::with_capacity(self.profiles.len() * (self.config.embedding_dim + 2)); + let mut embeddings = Vec::with_capacity(tokens.len()); + for (index, source) in tokens.iter().enumerate() { + // A frozen Train-only support scale bounds encoder RF values while + // retaining their signed baseline-relative DC. The task skip below + // uses the original calibrated values; metadata is never rescaled. + let mut conditioned = source.clone(); + for row in &mut conditioned { + for x in &mut row[..self.config.bins] { + *x /= self.profiles[index].max_normalized; + } + } + encoder.projection = projectors[index].clone(); + let embedding = encoder.extract(&conditioned); + features.extend_from_slice(&embedding); + // Calibrated DC and temporal energy skip preserve static occupancy + // evidence that per-window centering or L2 projection would erase. + let mean = source + .iter() + .flat_map(|row| row[..self.config.bins].iter()) + .sum::() + / (self.config.temporal_tokens * self.config.bins) as f32; + let temporal = source + .windows(2) + .flat_map(|w| { + w[0][..self.config.bins] + .iter() + .zip(&w[1][..self.config.bins]) + }) + .map(|(a, b)| (a - b).abs()) + .sum::() + / ((self.config.temporal_tokens - 1) * self.config.bins) as f32; + features.extend_from_slice(&[mean, temporal]); + embeddings.push(embedding); + } + if !features.iter().all(|x| x.is_finite()) { + return Err("non-finite representation".into()); + } + Ok((features, embeddings)) + } + pub(crate) fn logit(&self, features: &[f32], head: &[f32]) -> f32 { + features.iter().zip(head).map(|(x, w)| x * w).sum::() + head[head.len() - 1] + } +} + +pub(crate) fn sigmoid(x: f32) -> f32 { + if x >= 0.0 { + 1.0 / (1.0 + (-x).exp()) + } else { + let e = x.exp(); + e / (1.0 + e) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/optimization.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/optimization.rs new file mode 100644 index 0000000000..c27fa78868 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/optimization.rs @@ -0,0 +1,142 @@ +//! Bounded RF descent followed by exact linear-task block descent. +//! Both blocks minimize the same declared joint objective; no validation tuning. +use super::{objective, FitConfig, ObjectiveValues, Row}; +use crate::{ + room_learner::model::{extractor, sigmoid, RoomModel}, + trainer::{clip_gradients, estimate_gradient, SgdOptimizer}, +}; +use std::cell::Cell; +pub(super) fn head_step( + model: &RoomModel, + rows: &[Row], + params: &[f32], + config: &FitConfig, + current: ObjectiveValues, + check: &impl Fn() -> Result<(), String>, + evaluations: &Cell, +) -> Result, ObjectiveValues)>, String> { + let offset = model.representation_parameter_count(); + let mut gradient = vec![0.0; params.len()]; + let mut encoder = extractor(&model.config); + encoder + .transformer + .unflatten_weights(¶ms[..model.encoder_weights.len()])?; + let projectors = model.source_projectors(params)?; + for row in rows { + check()?; + let (features, _) = model.represent_with_encoder(&row.tokens, &mut encoder, &projectors)?; + let residual = (sigmoid(model.logit(&features, ¶ms[offset..])) - row.occupied) + * config.supervised_weight + / rows.len() as f32; + for (g, x) in gradient[offset..].iter_mut().zip(features) { + *g += residual * x; + } + *gradient.last_mut().unwrap() += residual; + } + clip_gradients(&mut gradient, 5.0); + for backtrack in 0..8 { + check()?; + let mut next = params.to_vec(); + SgdOptimizer::new(config.learning_rate * 0.5f32.powi(backtrack), 0.0, 0.0) + .step(&mut next, &gradient); + evaluations.set(evaluations.get() + 1); + let loss = objective(model, rows, &next, config)?; + if loss.total < current.total { + return Ok(Some((next, loss))); + } + } + Ok(None) +} + +/// A second, smaller finite-difference radius handles ReLU boundaries without +/// changing the objective. Every probe remains inside the caller's hard budgets. +pub(super) fn joint_step( + model: &RoomModel, + rows: &[Row], + params: &[f32], + config: &FitConfig, + current: ObjectiveValues, + check: &impl Fn() -> Result<(), String>, + evaluations: &Cell, +) -> Result, ObjectiveValues)>, String> { + for attempt in 0..2 { + check()?; + let epsilon = (config.gradient_epsilon * 0.25f32.powi(attempt)).max(0.0001); + if attempt > 0 && epsilon == config.gradient_epsilon { + break; + } + let mut gradient = estimate_gradient( + |p| { + if check().is_err() { + return f32::NAN; + } + evaluations.set(evaluations.get() + 1); + objective(model, rows, p, config) + .map(|l| l.total) + .unwrap_or(f32::NAN) + }, + params, + epsilon, + ); + check()?; + if !gradient.iter().all(|g| g.is_finite()) { + return Err("non-finite training gradient".into()); + } + clip_gradients(&mut gradient, 5.0); + for backtrack in 0..16 { + check()?; + let mut next = params.to_vec(); + SgdOptimizer::new(config.learning_rate * 0.5f32.powi(backtrack), 0.0, 0.0) + .step(&mut next, &gradient); + evaluations.set(evaluations.get() + 1); + let loss = objective(model, rows, &next, config)?; + if loss.total < current.total { + return Ok(Some((next, loss))); + } + } + } + Ok(None) +} +pub(super) fn require_rf_update( + model: &RoomModel, + params: &[f32], + initial: &[f32], +) -> Result<(), String> { + let backbone = model.encoder_weights.len(); + let representation = model.representation_parameter_count(); + if params[..backbone] == initial[..backbone] + || params[backbone..representation] == initial[backbone..representation] + { + return Err( + "no improving backbone/projection update; refusing a head-only RF candidate".into(), + ); + } + Ok(()) +} + +/// Task gradients must not be suppressed by a clipped RF gradient or its small +/// accepted step. Optimize both blocks against the same Train-only objective. +pub(super) fn epoch_step( + model: &RoomModel, + rows: &[Row], + params: &[f32], + config: &FitConfig, + current: ObjectiveValues, + check: &impl Fn() -> Result<(), String>, + evaluations: &Cell, +) -> Result, ObjectiveValues)>, String> { + let mut next = params.to_vec(); + let mut loss = current; + let mut changed = false; + if let Some((p, l)) = joint_step(model, rows, params, config, current, check, evaluations)? { + next = p; + loss = l; + changed = true; + } + if let Some((p, l)) = head_step(model, rows, &next, config, loss, check, evaluations)? { + next = p; + loss = l; + changed = true; + } + Ok(if changed { Some((next, loss)) } else { None }) +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/projection.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/projection.rs new file mode 100644 index 0000000000..01452f115c --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/projection.rs @@ -0,0 +1,110 @@ +//! Existing AETHER projection heads routed by frozen source identity. +//! Opposite RF responses can swap across room states. Source-routed coordinates +//! preserve receiver identity during alignment while the transformer is shared. +use super::{EncoderConfig, RoomModel}; +use crate::{ + embedding::{EmbeddingConfig, ProjectionHead}, + graph_transformer::Linear, +}; +pub(crate) fn config(c: &EncoderConfig) -> EmbeddingConfig { + EmbeddingConfig { + d_model: c.hidden_dim, + d_proj: c.embedding_dim, + temperature: c.temperature, + normalize: false, + } +} +pub(crate) fn initial_weights(c: &EncoderConfig) -> Vec> { + (0..c.source_count) + .map(|source| { + let mut head = ProjectionHead::new(config(c)); + // Source-specific deterministic Xavier seeds break routing symmetry; + // no label, amplitude polarity or fixture value enters initialization. + head.proj_1 = + Linear::with_seed(c.hidden_dim, c.embedding_dim, 20_240 + source as u64 * 2); + head.proj_1.set_bias(vec![0.1; c.embedding_dim]); + head.proj_2 = + Linear::with_seed(c.embedding_dim, c.embedding_dim, 20_241 + source as u64 * 2); + let mut weights = Vec::new(); + head.flatten_into(&mut weights); + weights + }) + .collect() +} +impl RoomModel { + pub(crate) fn source_projectors(&self, params: &[f32]) -> Result, String> { + if params.len() != self.parameter_count() { + return Err("invalid conditional parameter count".into()); + } + let mut offset = self.encoder_weights.len(); + let mut heads = Vec::new(); + for weights in &self.source_projection_weights { + let (head, n) = ProjectionHead::unflatten_from( + ¶ms[offset..offset + weights.len()], + &config(&self.config), + ); + offset += n; + heads.push(head); + } + Ok(heads) + } +} + +/// Center each initial first-layer activation on Train Empty only. This keeps +/// every small ReLU head trainable without assuming any source's signal polarity. +pub(crate) fn center_empty_heads( + model: &mut RoomModel, + rows: &[crate::room_learner::observation::CaptureCohort], +) -> Result<(), String> { + use crate::room_learner::observation::RoomLabel; + if rows + .iter() + .any(|r| r.role != crate::room_learner::observation::DataRole::Train) + { + return Err("projection centering only accepts Train".into()); + } + let mut encoder = super::extractor(&model.config); + encoder + .transformer + .unflatten_weights(&model.encoder_weights)?; + let mut baseline = vec![vec![0.0; model.config.hidden_dim]; model.config.source_count]; + let mut count = 0usize; + for row in rows.iter().filter(|r| r.label == RoomLabel::Empty) { + count += 1; + for (source, mut tokens) in model.tokens(row, false)?.into_iter().enumerate() { + for frame in &mut tokens { + for x in &mut frame[..model.config.bins] { + *x /= model.profiles[source].max_normalized; + } + } + let features = encoder.transformer.embed(&tokens); + for vector in &features { + for (mean, x) in baseline[source].iter_mut().zip(vector) { + *mean += x / features.len() as f32; + } + } + } + } + if count == 0 { + return Err("projection centering requires Train Empty".into()); + } + for (source, mean) in baseline.iter_mut().enumerate() { + mean.iter_mut().for_each(|x| *x /= count as f32); + let (mut head, _) = ProjectionHead::unflatten_from( + &model.source_projection_weights[source], + &config(&model.config), + ); + // Initial bias is 0.1; retain a 0.1 positive activation at the baseline. + let bias = head + .proj_1 + .forward(mean) + .iter() + .map(|value| 0.2 - value) + .collect(); + head.proj_1.set_bias(bias); + let mut weights = Vec::new(); + head.flatten_into(&mut weights); + model.source_projection_weights[source] = weights; + } + Ok(()) +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/source_conditioning_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/source_conditioning_tests.rs new file mode 100644 index 0000000000..2641665a1c --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/source_conditioning_tests.rs @@ -0,0 +1,230 @@ +//! SYNTHETIC exact sign-inversion oracle: pooled views cannot identify the room. +use crate::room_learner::{ + model::*, + observation::*, + training::{learning_tests::dataset, FitConfig}, +}; + +fn inverted_dataset(reverse: bool) -> ValidatedDataset { + let base = dataset(8); + let mut rows = base + .train() + .iter() + .chain(base.validation()) + .chain(base.held_out()) + .cloned() + .collect::>(); + for c in &mut rows { + for v in &mut c.views { + for (t, frame) in v.samples.iter_mut().enumerate() { + for (b, x) in frame.iter_mut().enumerate() { + let empty = 10.0 + t as f32 * 0.02 + b as f32 * 0.001; + let state = match c.label { + RoomLabel::Empty => 0.0, + RoomLabel::AdjacentOnly => -1.0, + _ => 1.0, + }; + let source = if (v.source_id == 1) != reverse { + 1.0 + } else { + -1.0 + }; + *x = empty + source * state * 0.25; + } + } + } + } + ValidatedDataset::new(base.context().clone(), rows).unwrap() +} +fn cosine(a: &[f32], b: &[f32]) -> f32 { + a.iter().zip(b).map(|(x, y)| x * y).sum::() + / (a.iter().map(|x| x * x).sum::().sqrt() + * b.iter().map(|x| x * x).sum::().sqrt()) + .max(1e-10) +} +fn margin( + model: &RoomModel, + occupied: &CaptureCohort, + adjacent: &CaptureCohort, + condition: bool, +) -> f32 { + let mut stripped = model.clone(); + if !condition { + for i in 1..stripped.source_projection_weights.len() { + stripped.source_projection_weights[i] = stripped.source_projection_weights[0].clone(); + } + } + let model = if condition { model } else { &stripped }; + let mut a = model.tokens(occupied, false).unwrap(); + let mut b = model.tokens(adjacent, false).unwrap(); + if !condition { + for tokens in a.iter_mut().chain(b.iter_mut()) { + for row in tokens { + row[model.encoder_config().bins..].fill(0.0); + } + } + } + let (_, a) = model.represent(&a, &model.parameters()).unwrap(); + let (_, b) = model.represent(&b, &model.parameters()).unwrap(); + (cosine(&a[0], &a[1]) + cosine(&b[0], &b[1]) - cosine(&a[0], &b[1]) - cosine(&b[0], &a[1])) + * 0.5 +} +fn check_inverted_case(reverse: bool) { + let ds = inverted_dataset(reverse); + let occupied = &ds.held_out()[2]; + let adjacent = &ds.held_out()[1]; + let pooled = |c: &CaptureCohort| { + c.views + .iter() + .flat_map(|v| v.samples.iter().flatten()) + .sum::() + }; + assert!((pooled(occupied) - pooled(adjacent)).abs() < 0.001); + assert_eq!(occupied.views[0].samples, adjacent.views[1].samples); + assert_eq!(occupied.views[1].samples, adjacent.views[0].samples); + let (model, report) = RoomModel::fit( + &ds, + FitConfig { + epochs: 48, + max_duration_ms: 120_000, + learning_rate: 0.5, + hidden_dim: 4, + embedding_dim: 4, + contrastive_weight: 5.0, + supervised_weight: 0.5, + ..FitConfig::default() + }, + ) + .unwrap(); + let conditional = margin(&model, occupied, adjacent, true); + let unconditioned = margin(&model, occupied, adjacent, false); + eprintln!("SYNTHETIC inverted RF InfoNCE {}->{}; positive-vs-opposite margin conditioned={conditional}, removed-tags={unconditioned}; report={report:?}",report.initial.contrastive,report.final_objective.contrastive); + assert!(report.backbone_delta_l2 > 1e-4); + assert!(report.projection_delta_l2 > 1e-4); + assert!(report.final_objective.contrastive < report.initial.contrastive - 0.02); + assert!( + conditional > 0.1, + "shared encoder must align same-event opposite-sign views" + ); + assert!( + unconditioned <= 1e-5, + "removed conditioning must recover the impossible symmetric-view case" + ); + let evaluation = model.evaluate(ds.held_out()).unwrap(); + assert_eq!(evaluation.correct, 4, "{evaluation:?}"); + assert_eq!(evaluation.abstained, 0); + assert!(!model.infer(occupied).physically_validated); +} +#[test] +fn opposite_source_responses_learn_real_alignment_and_disjoint_room_controls() { + check_inverted_case(false); +} +#[test] +fn reversed_response_polarities_learn_with_the_same_generic_initialization() { + check_inverted_case(true); +} +#[test] +fn source_count_and_metadata_cannot_alias_canonical_rf_or_evade_import_bounds() { + let ds = dataset(56); + let (model, report) = RoomModel::fit( + &ds, + FitConfig { + epochs: 1, + ..FitConfig::default() + }, + ) + .unwrap(); + assert_eq!(model.encoder_config().bins, 56); + assert_eq!(model.encoder_config().source_count, 2); + assert!(report.parameter_count <= 512); + let original: serde_json::Value = serde_json::from_slice(&model.export().unwrap()).unwrap(); + for count in [0, 1, 3, 5000] { + let mut bad = original.clone(); + bad["config"]["source_count"] = serde_json::json!(count); + assert!(RoomModel::import(&serde_json::to_vec(&bad).unwrap()).is_err()); + } + let mut missing = original; + missing["config"] + .as_object_mut() + .unwrap() + .remove("source_count"); + assert!(RoomModel::import(&serde_json::to_vec(&missing).unwrap()).is_err()); + for malformed in [ + serde_json::Value::Null, + serde_json::json!([]), + serde_json::json!([[], []]), + ] { + let mut invalid: serde_json::Value = + serde_json::from_slice(&model.export().unwrap()).unwrap(); + invalid["source_projection_weights"] = malformed; + assert!(RoomModel::import(&serde_json::to_vec(&invalid).unwrap()).is_err()); + } + let tokens = model.tokens(&ds.held_out()[0], false).unwrap(); + let (features, _) = model.represent(&tokens, &model.parameters()).unwrap(); + let mut zeroed = tokens; + for source in &mut zeroed { + for row in source { + row[56..].fill(0.0); + } + } + let (without_tags, _) = model.represent(&zeroed, &model.parameters()).unwrap(); + let width = model.encoder_config().embedding_dim + 2; + for source in 0..2 { + assert_eq!( + &features[source * width + width - 2..source * width + width], + &without_tags[source * width + width - 2..source * width + width] + ); + } +} + +#[test] +fn three_native_56_bin_sources_train_every_projection_within_actual_budget() { + let base = dataset(56); + let mut context = base.context().clone(); + let mut source = context.sources[0].clone(); + source.source_id = 3; + source.device_id = "synthetic-device-3".into(); + source.link_id = "link-3".into(); + context.sources.push(source.clone()); + let mut rows = base + .train() + .iter() + .chain(base.validation()) + .chain(base.held_out()) + .cloned() + .collect::>(); + for row in &mut rows { + let mut view = row.views[0].clone(); + view.source_id = 3; + view.device_id = source.device_id.clone(); + view.link_id = source.link_id.clone(); + row.views.push(view); + row.context_digest = context.digest(); + } + let ds = ValidatedDataset::new(context, rows).unwrap(); + let (model, report) = RoomModel::fit( + &ds, + FitConfig { + epochs: 1, + ..FitConfig::default() + }, + ) + .unwrap(); + assert_eq!(model.encoder_config().source_count, 3); + assert_eq!(report.parameter_count, 395); + assert!(report.backbone_delta_l2 > 0.0); + let mut initial_model = model.clone(); + initial_model.encoder_weights = initial_encoder(model.encoder_config()) + .unwrap() + .transformer + .flatten_weights(); + initial_model.source_projection_weights = initial_projection_weights(model.encoder_config()); + center_empty_heads(&mut initial_model, ds.train()).unwrap(); + let initial = initial_model.source_projection_weights; + for (before, after) in initial.iter().zip(&model.source_projection_weights) { + assert!(before.iter().zip(after).any(|(a, b)| (a - b).abs() > 1e-6)); + } + let tokens = model.tokens(&ds.held_out()[0], false).unwrap(); + assert_eq!(tokens[2][0].len(), 59); + assert_eq!(&tokens[2][0][56..], &[0.0, 0.0, 1.0]); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/training.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/training.rs new file mode 100644 index 0000000000..e1b1a0fcbd --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/training.rs @@ -0,0 +1,408 @@ +//! Bounded numerical optimization using the production trainer machinery. +use super::model::{ + center_empty_heads, extractor, initial_encoder, initial_projection_weights, EncoderConfig, + RoomModel, SourceProfile, TrainingManifest, +}; +use super::observation::{CaptureCohort, CaptureContext, DataRole, RoomLabel, ValidatedDataset}; +use crate::embedding::info_nce_loss; +use serde::{Deserialize, Serialize}; +use std::cell::Cell; +use std::collections::HashSet; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::time::Instant; + +#[path = "optimization.rs"] +mod optimization; + +#[path = "fit_config.rs"] +mod fit_config; +pub use fit_config::FitConfig; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize)] +pub struct ObjectiveValues { + pub total: f32, + pub contrastive: f32, + pub supervised: f32, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct EvaluationReport { + pub samples: usize, + pub correct: usize, + pub abstained: usize, + pub binary_cross_entropy: f32, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TrainingReport { + pub initial: ObjectiveValues, + pub final_objective: ObjectiveValues, + pub completed_epochs: usize, + pub parameter_count: usize, + pub parameter_delta_l2: f32, + pub encoder_delta_l2: f32, + pub backbone_delta_l2: f32, + pub projection_delta_l2: f32, + pub head_delta_l2: f32, + pub objective_evaluations: usize, + pub training_cohorts: usize, + pub contrastive_trials: usize, + pub validation: Option, + pub evidence: String, +} +struct Row { + tokens: Vec>>, + occupied: f32, + contrastive: bool, +} +fn occupied(label: RoomLabel) -> f32 { + if matches!(label, RoomLabel::OccupiedStatic | RoomLabel::OccupiedMoving) { + 1.0 + } else { + 0.0 + } +} +fn bce(logit: f32, target: f32) -> f32 { + logit.max(0.0) - logit * target + (-logit.abs()).exp().ln_1p() +} +fn normalized(mut embedding: Vec) -> Vec { + let norm = embedding + .iter() + .map(|x| x * x) + .sum::() + .sqrt() + .max(1.0e-8); + embedding.iter_mut().for_each(|x| *x /= norm); + embedding +} +fn objective( + model: &RoomModel, + rows: &[Row], + params: &[f32], + config: &FitConfig, +) -> Result { + let mut supervised = 0.0; + let mut by_source = vec![Vec::new(); model.profiles.len()]; + let head = ¶ms[model.representation_parameter_count()..]; + let projectors = model.source_projectors(params)?; + let mut encoder = extractor(&model.config); + encoder + .transformer + .unflatten_weights(¶ms[..model.encoder_weights.len()])?; + for row in rows { + let (features, embeddings) = + model.represent_with_encoder(&row.tokens, &mut encoder, &projectors)?; + supervised += bce(model.logit(&features, head), row.occupied); + // One event per independent trial per batch. Sibling/overlapping windows + // in the same trial cannot accidentally become contrastive negatives. + if row.contrastive { + for (source, embedding) in by_source.iter_mut().zip(embeddings) { + source.push(normalized(embedding)); + } + } + } + supervised /= rows.len() as f32; + let mut contrastive = 0.0; + // A source-zero star aligns every source without O(source_count^2) pairs. + // Both directions are optimized; each row's other source is its positive. + for other in by_source.iter().skip(1) { + contrastive += (info_nce_loss(&by_source[0], other, config.temperature) + + info_nce_loss(other, &by_source[0], config.temperature)) + * 0.5; + } + contrastive /= (by_source.len() - 1) as f32; + let total = supervised * config.supervised_weight + contrastive * config.contrastive_weight; + if !total.is_finite() { + return Err("non-finite training objective".into()); + } + Ok(ObjectiveValues { + total, + supervised, + contrastive, + }) +} +fn profiles(dataset: &ValidatedDataset) -> Result, String> { + let mut profiles = Vec::new(); + for source in &dataset.context().sources { + let width = source.feature_count; + let mut mean = vec![0.0f64; width]; + let mut square = vec![0.0f64; width]; + let mut count = 0usize; + for event in dataset + .train() + .iter() + .filter(|c| c.label == RoomLabel::Empty) + { + let view = event + .views + .iter() + .find(|v| v.source_id == source.source_id) + .ok_or("missing baseline source")?; + for frame in &view.samples { + count += 1; + for (k, &v) in frame.iter().enumerate() { + mean[k] += v as f64; + square[k] += (v as f64).powi(2); + } + } + } + if count < 2 { + return Err("each source needs Train Empty baseline frames".into()); + } + let scale: Vec = mean + .iter() + .zip(&square) + .map(|(m, s)| { + (s / count as f64 - (m / count as f64).powi(2)) + .max(0.0) + .sqrt() + .max(0.1) as f32 + }) + .collect(); + let mean: Vec = mean.iter().map(|m| (m / count as f64) as f32).collect(); + let mut max_normalized = 1.0f32; + for event in dataset.train() { + let view = event + .views + .iter() + .find(|v| v.source_id == source.source_id) + .ok_or("missing training source")?; + for frame in &view.samples { + for (k, v) in frame.iter().enumerate() { + max_normalized = max_normalized.max(((v - mean[k]) / scale[k]).abs()); + } + } + } + profiles.push(SourceProfile { + source: source.clone(), + mean, + scale, + max_normalized, + }); + } + Ok(profiles) +} +impl RoomModel { + pub fn fit( + dataset: &ValidatedDataset, + config: FitConfig, + ) -> Result<(Self, TrainingReport), String> { + Self::fit_with_cancel(dataset, config, &AtomicBool::new(false)) + } + pub fn fit_with_cancel( + dataset: &ValidatedDataset, + config: FitConfig, + cancel: &AtomicBool, + ) -> Result<(Self, TrainingReport), String> { + let started = Instant::now(); + config.validate()?; + if cancel.load(Ordering::Relaxed) { + return Err("training cancelled".into()); + } + if !(4..=64).contains(&dataset.train().len()) { + return Err("training requires 4..64 cohorts".into()); + } + let trial_count = dataset + .train() + .iter() + .map(|c| &c.trial_id) + .collect::>() + .len(); + if trial_count < 2 + || !dataset.train().iter().any(|c| occupied(c.label) == 1.0) + || !dataset + .train() + .iter() + .any(|c| c.label == RoomLabel::AdjacentOnly) + { + return Err( + "training needs independent trials, occupied and adjacent-only controls".into(), + ); + } + let encoder_config = EncoderConfig { + source_count: dataset.context().sources.len(), + bins: dataset + .context() + .sources + .iter() + .map(|s| s.feature_count) + .max() + .unwrap_or(0), + hidden_dim: config.hidden_dim, + embedding_dim: config.embedding_dim, + temporal_tokens: config.temporal_tokens, + temperature: config.temperature, + }; + if !(1..=128).contains(&encoder_config.bins) { + return Err("source width exceeds bounded encoder".into()); + } + let encoder = initial_encoder(&encoder_config)?; + let mut model = Self { + version: 1, + algorithm: "aether_multiview_infonce_room_bce_v1".into(), + context: dataset.context().clone(), + training_manifest: TrainingManifest::from_training(dataset.train())?, + source_projection_weights: initial_projection_weights(&encoder_config), + config: encoder_config, + profiles: profiles(dataset)?, + encoder_weights: encoder.transformer.flatten_weights(), + head_weights: vec![ + 0.0; + dataset.context().sources.len() * (config.embedding_dim + 2) + 1 + ], + }; + model.validate()?; + if model.parameter_count() > config.max_parameters { + return Err("model exceeds parameter budget".into()); + } + center_empty_heads(&mut model, dataset.train())?; + model.validate()?; + let mut independent: Vec<&CaptureCohort> = Vec::new(); + let mut rows = Vec::new(); + for c in dataset.train() { + let contrastive = !independent.iter().any(|other| { + other.trial_id == c.trial_id + || c.views.iter().any(|v| { + other.views.iter().any(|w| { + v.source_id == w.source_id + && (v.frame_ids.iter().any(|id| w.frame_ids.contains(id)) + || (v.first_time_ns <= w.last_time_ns + && v.last_time_ns >= w.first_time_ns)) + }) + }) + }); + if contrastive { + independent.push(c); + } + rows.push(Row { + tokens: model.tokens(c, false)?, + occupied: occupied(c.label), + contrastive, + }); + } + let contrastive_trials = independent.len(); + if contrastive_trials < 2 { + return Err("contrastive objective needs non-overlapping independent trials".into()); + } + let initial_params = model.parameters(); + let mut params = initial_params.clone(); + let initial = objective(&model, &rows, ¶ms, &config)?; + let mut current = initial; + let evaluations = Cell::new(1usize); + let check = || -> Result<(), String> { + if cancel.load(Ordering::Relaxed) { + Err("training cancelled".into()) + } else if started.elapsed().as_millis() >= config.max_duration_ms as u128 { + Err("training time budget exhausted".into()) + } else if evaluations.get() >= config.max_objective_evaluations { + Err("objective evaluation budget exhausted".into()) + } else { + Ok(()) + } + }; + let mut epochs = 0; + for _ in 0..config.epochs { + check()?; + let next = optimization::epoch_step( + &model, + &rows, + ¶ms, + &config, + current, + &check, + &evaluations, + )?; + if let Some((next, loss)) = next { + params = next; + current = loss; + } else { + break; + } + epochs += 1; + } + check()?; + if epochs == 0 || params == initial_params { + return Err("optimizer made no improving parameter update".into()); + } + optimization::require_rf_update(&model, ¶ms, &initial_params)?; + model.set_parameters(¶ms)?; + model.validate()?; + let delta = |a: &[f32], b: &[f32]| { + a.iter() + .zip(b) + .map(|(x, y)| (x - y).powi(2)) + .sum::() + .sqrt() + }; + let split = model.representation_parameter_count(); + let report = TrainingReport { + initial, + final_objective: current, + completed_epochs: epochs, + parameter_count: params.len(), + parameter_delta_l2: delta(¶ms, &initial_params), + encoder_delta_l2: delta(¶ms[..split], &initial_params[..split]), + backbone_delta_l2: delta( + ¶ms[..model.encoder_weights.len()], + &initial_params[..model.encoder_weights.len()], + ), + projection_delta_l2: delta( + ¶ms[model.encoder_weights.len()..split], + &initial_params[model.encoder_weights.len()..split], + ), + head_delta_l2: delta(¶ms[split..], &initial_params[split..]), + objective_evaluations: evaluations.get(), + training_cohorts: rows.len(), + contrastive_trials, + validation: if dataset.validation().is_empty() { + None + } else { + Some(model.evaluate(dataset.validation())?) + }, + evidence: "EXPERIMENTAL_RF_ONLY; physical accuracy and promotion are unqualified" + .into(), + }; + check()?; + Ok((model, report)) + } + /// Explicit evaluation only; fit never reads HeldOut observations. + pub fn evaluate(&self, cohorts: &[CaptureCohort]) -> Result { + if cohorts.is_empty() || cohorts.len() > 4096 { + return Err("invalid evaluation cohort count".into()); + } + let mut result = EvaluationReport { + samples: cohorts.len(), + correct: 0, + abstained: 0, + binary_cross_entropy: 0.0, + }; + for cohort in cohorts { + if cohort.role == DataRole::Inference || cohort.label == RoomLabel::Unknown { + return Err("evaluation requires a labeled non-inference cohort".into()); + } + let prediction = self.infer(cohort); + if prediction.decision == super::model::PresenceDecision::Abstain { + result.abstained += 1; + } + let probability = prediction + .occupied_probability + .ok_or_else(|| prediction.reason.unwrap_or("evaluation abstained".into()))?; + let target = occupied(cohort.label); + if prediction.decision != super::model::PresenceDecision::Abstain + && (probability > 0.5) == (target > 0.5) + { + result.correct += 1; + } + let p = probability.clamp(1.0e-6, 1.0 - 1.0e-6); + result.binary_cross_entropy -= target * p.ln() + (1.0 - target) * (1.0 - p).ln(); + } + result.binary_cross_entropy /= cohorts.len() as f32; + Ok(result) + } +} + +#[cfg(test)] +#[path = "learning_tests.rs"] +pub(crate) mod learning_tests; + +#[cfg(test)] +#[path = "source_conditioning_tests.rs"] +mod source_conditioning_tests; From 7e76c6572fa97f1d49e7ac1f214ca6fa1241523f Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Sun, 1 Nov 2026 20:29:47 +0100 Subject: [PATCH 032/100] feat(server): manage guarded room learner lifecycle and private persistence --- .../wifi-densepose-sensing-server/src/main.rs | 37 ++ .../src/room_learner/command.rs | 56 ++ .../src/room_learner/lifecycle.rs | 355 ++++++++++++ .../src/room_learner/lifecycle_capture.rs | 164 ++++++ .../src/room_learner/lifecycle_tests.rs | 505 ++++++++++++++++++ .../src/room_learner/mod.rs | 3 +- .../src/room_learner_api.rs | 450 ++++++++++++++++ .../src/room_learner_api_tests.rs | 280 ++++++++++ 8 files changed, 1849 insertions(+), 1 deletion(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/command.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index 6db68f6852..e71d6159ab 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -18,6 +18,7 @@ pub mod csi; mod engine_bridge; mod field_bridge; mod source_bank_shadow; +mod room_learner_api; mod field_localize; mod model_format; mod multistatic_bridge; @@ -321,6 +322,10 @@ struct Args { #[arg(long)] source_bank_shadow: bool, + /// Enable the account-owned managed capture/train/candidate room learner. + #[arg(long)] + room_learner: bool, + // --------------------------------------------------------------- // ADR-102: Edge Module Registry — surface the canonical Cognitum // cog catalog via `GET /api/v1/edge/registry`. @@ -2210,6 +2215,8 @@ struct AppStateInner { field_model: Option, source_bank_shadow_enabled: bool, source_bank_shadow: Option, + room_learning: Option>, + room_learning_sources: HashMap<(u8, u16, u8), room_learner_api::LiveSource>, /// Stable installation identity used only to bind local persisted state. installation_id: Option, /// Metadata for the privacy reduced empty room image, when available. @@ -3144,6 +3151,8 @@ impl AppStateInner { field_model: None, source_bank_shadow_enabled: false, source_bank_shadow: None, + room_learning: None, + room_learning_sources: HashMap::new(), installation_id: None, bootstrap_baseline: None, bootstrap_baseline_active: false, @@ -10472,6 +10481,13 @@ async fn calibration_start( return Json(serde_json::json!({"success":false,"error_code":"shared_room_binding_required","error":"Import the current Mac room revision before starting calibration."})); } let mut s = state.write().await; + if let Some(learner)=s.room_learning.as_ref() { + match learner.capture_active() { + Ok(false)=>{}, + Ok(true)=>return Json(serde_json::json!({"success":false,"error_code":"learner_capture_active","error":"Finish or cancel learner example capture before starting a legacy empty baseline."})), + Err(_)=>return Json(serde_json::json!({"success":false,"error_code":"learner_state_unavailable","error":"Learner capture state is unavailable; retry before changing calibration mode."})), + } + } if !valid_binding_digest(&request.binding_digest) || !valid_source_node_ids(&request.source_node_ids) { @@ -13899,6 +13915,7 @@ async fn udp_receiver_task( .entry(frame.node_id) .or_insert_with(NodeState::new) .observe_raw_grid(grid_key, observed_at); + room_learner_api::observe(&mut s, &frame, &buf[..len], observed_at); // The field model owns a separate, frozen grid path. Feed // it before the global inference grid gate so a stable @@ -15745,6 +15762,14 @@ async fn main() { .as_ref() .map(|p| mediatek_activity::load_activity_state(p)) .unwrap_or_default(); + let room_learning = if args.room_learner { + let owner = std::env::var(wifi_densepose_sensing_server::bearer_auth::OWNER_ACCOUNT_SCOPE_ENV).unwrap_or_default(); + let installation = args.installation_id.as_deref().unwrap_or(""); + match wifi_densepose_sensing_server::room_learner::lifecycle::RoomLearningRuntime::new(&args.data_dir, &owner, installation) { + Ok(runtime) => Some(Arc::new(runtime)), + Err(error) => { error!("Managed room learner could not initialize: {error}"); std::process::exit(2); } + } + } else { None }; let state: SharedState = Arc::new(RwLock::new(AppStateInner { latest_update: None, rssi_history: VecDeque::new(), @@ -15899,6 +15924,8 @@ async fn main() { field_model: bootstrap_field_model, source_bank_shadow_enabled: args.source_bank_shadow, source_bank_shadow: None, + room_learning, + room_learning_sources: HashMap::new(), installation_id: args.installation_id.clone(), bootstrap_baseline: bootstrap_metadata.clone(), bootstrap_baseline_active, @@ -16316,6 +16343,16 @@ async fn main() { ) .route("/api/v1/calibration/cancel", post(calibration_cancel)) .route("/api/v1/calibration/status", get(calibration_status)) + // Mutations remain behind the existing default-admin bearer policy. + .route("/api/v1/room-learner/status", get(room_learner_api::status)) + .route("/api/v1/room-learner/configure", post(room_learner_api::configure)) + .route("/api/v1/room-learner/phase/start", post(room_learner_api::begin_phase)) + .route("/api/v1/room-learner/phase/finish", post(room_learner_api::finish_phase)) + .route("/api/v1/room-learner/phase/cancel", post(room_learner_api::cancel_phase)) + .route("/api/v1/room-learner/train", post(room_learner_api::train)) + .route("/api/v1/room-learner/train/cancel", post(room_learner_api::cancel_training)) + .route("/api/v1/room-learner/evaluate", post(room_learner_api::evaluate)) + .route("/api/v1/room-learner/prediction", get(room_learner_api::prediction)) .route("/api/v1/calibration/reset", post(calibration_reset)) .route("/api/v1/calibration/bootstrap/recover", post(calibration_recover_bootstrap)) // ADR-044 §5.3: runtime-configurable dedup factor diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/command.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/command.rs new file mode 100644 index 0000000000..eb6edb643c --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/command.rs @@ -0,0 +1,56 @@ +//! Optimistic command identity, not authentication or qualification authority. +use super::*; +use serde::Deserialize; +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct PhaseIdentity { + pub trial_id: String, + pub session_id: String, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct MutationGuard { + pub runtime_epoch: String, + pub generation: u64, + pub workflow_revision: u64, + pub context_digest: Option, + pub phase: Option, +} +impl RoomLearningRuntime { + pub(super) fn guard_for(&self, state: &Inner) -> MutationGuard { + MutationGuard { + runtime_epoch: self.runtime_epoch.clone(), + generation: state.generation, + workflow_revision: state.workflow_revision, + context_digest: state.capture.as_ref().map(|c| c.context().digest()), + phase: state + .capture + .as_ref() + .and_then(|c| c.progress(0).phase) + .map(|p| PhaseIdentity { + trial_id: p.trial_id, + session_id: p.session_id, + }), + } + } + pub fn mutation_guard(&self) -> Result { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + Ok(self.guard_for(&state)) + } + /// Checks and consumes one expected workflow revision while holding the same + /// mutex used by the actual mutation. Raw packet arrival does not consume it. + pub(super) fn command_state( + &self, + expected: &MutationGuard, + ) -> Result, String> { + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if *expected != self.guard_for(&state) { + return Err("learner_state_changed".into()); + } + state.workflow_revision = state + .workflow_revision + .checked_add(1) + .ok_or("learner command revision exhausted")?; + Ok(state) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs new file mode 100644 index 0000000000..f6cd607d90 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs @@ -0,0 +1,355 @@ +//! Managed capture -> genuine learning -> private cold-loadable candidate. +//! Candidate predictions never grant physical qualification or replace control. +use super::{ + artifact::{ArtifactStore, CandidateReceipt}, + capture::{CaptureSession, PhaseReceipt, PhaseRequest, RawSourceFrame}, + model::{RoomModel, RoomPrediction}, + observation::{CaptureContext, DataRole, RoomLabel}, + training::{EvaluationReport, FitConfig, TrainingReport}, +}; +use serde::Serialize; +use serde_json::{json, Value}; +use std::{ + path::Path, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, + }, +}; + +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum JobPhase { + Idle, + Running, + Cancelling, + Ready, + Failed, + Cancelled, +} +struct Job { + phase: JobPhase, + cancel: Arc, + report: Option, + error: Option, +} +impl Default for Job { + fn default() -> Self { + Self { + phase: JobPhase::Idle, + cancel: Arc::new(AtomicBool::new(false)), + report: None, + error: None, + } + } +} +struct Inner { + generation: u64, + workflow_revision: u64, + capture: Option, + model: Option>, + receipt: Option, + frozen_training: bool, + job: Job, + last_evaluation: Option, + input_error: Option, +} +pub struct RoomLearningRuntime { + runtime_epoch: String, + owner: String, + installation: String, + store: ArtifactStore, + inner: Mutex, + clock_origin: std::time::Instant, + clock_epoch_ns: u64, +} +fn busy(job: &Job) -> bool { + matches!(job.phase, JobPhase::Running | JobPhase::Cancelling) +} +fn bounded(error: String) -> String { + error + .chars() + .filter(|c| !c.is_control()) + .take(256) + .collect() +} +impl RoomLearningRuntime { + pub fn new(directory: &Path, owner: &str, installation: &str) -> Result { + let clock_origin = std::time::Instant::now(); + let clock_epoch_ns = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|_| "receiver clock unavailable")? + .as_nanos() + .try_into() + .map_err(|_| "receiver clock outside bounds")?; + Ok(Self { + clock_origin, + clock_epoch_ns, + runtime_epoch: crate::companion::secret(), + owner: owner.into(), + installation: installation.into(), + store: ArtifactStore::new(directory, owner, installation)?, + inner: Mutex::new(Inner { + generation: 0, + workflow_revision: 0, + capture: None, + model: None, + receipt: None, + frozen_training: false, + job: Job::default(), + last_evaluation: None, + input_error: None, + }), + }) + } + /// UTC-anchored monotonic receiver time. Wall-clock adjustments cannot + /// shorten an operator capture or turn a countdown into labelled RF data. + pub fn receiver_now_ns(&self) -> u64 { + self.clock_epoch_ns + .saturating_add(self.clock_origin.elapsed().as_nanos().min(u64::MAX as u128) as u64) + } + pub fn saved_candidates( + &self, + room_digest: &str, + ) -> Result { + self.store.catalog_for_room(room_digest) + } + /// Explicit new capture versus resuming an exact persisted candidate. Neither + /// path deletes old models, room receipts, pairing or artifact history. + pub fn configure( + &self, + expected: &MutationGuard, + context: CaptureContext, + run_id: String, + new_training: bool, + ) -> Result<(), String> { + context.validate()?; + if context.owner_scope != self.owner || context.installation_id != self.installation { + return Err("learner context does not belong to this managed profile".into()); + } + FitConfig::for_source_count(context.sources.len()).validate_context(&context)?; + let session = CaptureSession::new(context.clone(), run_id)?; + let mut state = self.command_state(expected)?; + if busy(&state.job) || state.capture.as_ref().is_some_and(|c| c.phase_active()) { + return Err( + "finish/cancel the active capture or training job before changing context".into(), + ); + } + let restored = self.store.load_candidate(&context)?; + if !new_training && restored.is_none() { + return Err( + "no stored candidate for this exact context; start a new training capture".into(), + ); + } + state.generation = state + .generation + .checked_add(1) + .ok_or("learner generation exhausted")?; + state.capture = Some(session); + state.model = if new_training { + None + } else { + restored.map(Arc::new) + }; + state.frozen_training = !new_training; + state.receipt = None; + state.job = Job::default(); + if state.model.is_some() { + state.job.phase = JobPhase::Ready; + } + state.last_evaluation = None; + state.input_error = None; + Ok(()) + } + /// Called from an explicit protected action. Completion commits only a + /// candidate, with cancellation and context generation checked under lock. + pub fn start_training( + self: &Arc, + expected: &MutationGuard, + config: FitConfig, + ) -> Result { + config.validate()?; + let (generation, dataset, cancel) = { + let mut state = self.command_state(expected)?; + if busy(&state.job) + || (state.frozen_training + && !matches!(state.job.phase, JobPhase::Failed | JobPhase::Cancelled)) + { + return Err("this run is already training or frozen".into()); + } + let capture = state + .capture + .as_ref() + .ok_or("no learner capture configured")?; + config.validate_context(capture.context())?; + if capture.phase_active() { + return Err("finish or cancel the capture phase first".into()); + } + let dataset = capture.validated_dataset()?; + for label in [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ] { + if !dataset.train().iter().any(|c| c.label == label) { + return Err("capture Train empty, occupied-static, occupied-moving and adjacent-only trials before training".into()); + } + } + let cancel = Arc::new(AtomicBool::new(false)); + state.job = Job { + phase: JobPhase::Running, + cancel: cancel.clone(), + report: None, + error: None, + }; + state.frozen_training = true; + (state.generation, dataset, cancel) + }; + let runtime = self.clone(); + tokio::task::spawn_blocking(move || { + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + RoomModel::fit_with_cancel(&dataset, config, &cancel) + })) + .unwrap_or_else(|_| Err("learner worker failed safely".into())); + let Ok(mut state) = runtime.inner.lock() else { + return; + }; + if state.generation != generation { + return; + } + state.workflow_revision = state.workflow_revision.saturating_add(1); + if cancel.load(Ordering::Acquire) { + state.job.phase = JobPhase::Cancelled; + state.job.error = + Some("training cancelled; previous stored candidates preserved".into()); + return; + } + match outcome { + Ok((model, report)) => match runtime.store.save_candidate(&model) { + Ok(receipt) => { + state.model = Some(Arc::new(model)); + state.receipt = Some(receipt); + state.job.report = Some(report); + state.job.phase = JobPhase::Ready; + } + Err(error) => { + state.job.phase = JobPhase::Failed; + state.job.error = Some(bounded(error)); + } + }, + Err(error) => { + state.job.phase = JobPhase::Failed; + state.job.error = Some(bounded(error)); + } + } + }); + Ok(generation) + } + pub fn prediction(&self, now_ns: u64) -> Result<(MutationGuard, RoomPrediction), String> { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + let model = state + .model + .as_ref() + .ok_or("no trained/restored candidate for this context")?; + let cohort = state + .capture + .as_ref() + .ok_or("no current capture context")? + .live_cohort(now_ns)?; + Ok((self.guard_for(&state), model.infer(&cohort))) + } + /// Evaluation does not mutate model parameters and never promotes physical + /// authority. The held-out result is a candidate diagnostic, not a certificate. + pub fn evaluate(&self, expected: &MutationGuard, role: DataRole) -> Result { + if !matches!(role, DataRole::Validation | DataRole::HeldOut) { + return Err("evaluate only validation or held-out data".into()); + } + let mut state = self.command_state(expected)?; + if busy(&state.job) { + return Err("wait for the frozen candidate".into()); + } + let model = state.model.as_ref().ok_or("no frozen candidate")?; + let capture = state.capture.as_ref().ok_or("no capture context")?; + if capture.phase_active() { + return Err("finish the current phase before evaluation".into()); + } + let dataset = capture.validated_dataset()?; + let rows = if role == DataRole::HeldOut { + dataset.held_out() + } else { + dataset.validation() + }; + if rows.is_empty() { + return Err("no independent rows for the selected evaluation role".into()); + } + let report: EvaluationReport = model.evaluate(rows)?; + let result = json!({"schema":"ruview.room-learner.evaluation.v1","role":role, + "context_digest":capture.context().digest(),"report":report, + "measurement_clock_available":dataset.measurement_clock_available(), + "physically_validated":false,"adoption_authorized":false}); + state.last_evaluation = Some(result.clone()); + Ok(result) + } + pub fn status(&self, now_ns: u64) -> Value { + let Ok(state) = self.inner.lock() else { + return json!({"enabled":true,"state":"unavailable"}); + }; + let required_ids: Vec<_> = state + .capture + .as_ref() + .map(|c| c.context().sources.iter().map(|s| s.source_id).collect()) + .unwrap_or_default(); + let trained_ids: Vec<_> = state + .model + .as_ref() + .map(|m| m.context().sources.iter().map(|s| s.source_id).collect()) + .unwrap_or_default(); + let prediction = state.model.as_ref().and_then(|m| { + state + .capture + .as_ref() + .and_then(|c| c.live_cohort(now_ns).ok()) + .map(|c| m.infer(&c)) + }); + let scored_ids = if prediction + .as_ref() + .is_some_and(|p| p.occupied_probability.is_some()) + { + trained_ids.clone() + } else { + Vec::new() + }; + json!({"schema":"ruview.room-learner.status.v1","enabled":true,"mutation_guard":self.guard_for(&state), + "required_source_ids":required_ids,"trained_source_ids":trained_ids,"scored_source_ids":scored_ids, + "candidate_prediction":prediction,"training_manifest":state.model.as_ref().map(|m|m.training_manifest()), + "phase_history":state.capture.as_ref().map(|c|c.history()), + "phase_duration_defaults":{"Empty":600000,"OccupiedStatic":12000,"OccupiedMoving":12000,"AdjacentOnly":12000}, + "phase_lead_in_defaults":{"Empty":15000,"OccupiedStatic":5000,"OccupiedMoving":5000,"AdjacentOnly":15000}, + "untrained_capture_progress_is_volatile":true, + "generation":state.generation,"state":state.job.phase,"context_digest":state.capture.as_ref().map(|c|c.context().digest()), + "room_digest":state.capture.as_ref().map(|c| &c.context().room_digest), + "revision":state.capture.as_ref().map(|c|c.context().revision), + "sources":state.capture.as_ref().map(|c| &c.context().sources), + "capture":state.capture.as_ref().map(|c|c.progress(now_ns)), + "training_frozen":state.frozen_training,"training_defaults":FitConfig::for_source_count(state.capture.as_ref().map_or(0, |c| c.context().sources.len())),"training_report":state.job.report, + "error":state.job.error,"input_error":state.input_error, + "candidate":state.receipt,"candidate_loaded":state.model.is_some(), + "last_evaluation":state.last_evaluation,"physically_validated":false, + "control_detector_changed":false,"raw_observations_persisted":false, + "limitations":["receiver-observed magnitude path; no coherent phase claim", + "legacy UDP transmitter identity is unverified","independent teacher registry not configured", + "candidate training is not physical qualification"]}) + } +} + +#[cfg(test)] +#[path = "lifecycle_tests.rs"] +mod tests; + +#[path = "lifecycle_capture.rs"] +mod capture_control; + +#[path = "command.rs"] +mod command; +pub use command::{MutationGuard, PhaseIdentity}; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs new file mode 100644 index 0000000000..7807789f8f --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs @@ -0,0 +1,164 @@ +//! Managed capture controls and source/provenance admission. +use super::*; +impl RoomLearningRuntime { + pub fn bind_context( + &self, + room_digest: String, + revision: u64, + sources: Vec, + ) -> Result { + let context = CaptureContext { + owner_scope: self.owner.clone(), + installation_id: self.installation.clone(), + room_digest, + revision, + sources, + }; + context.validate()?; + Ok(context) + } + pub fn selects_grid(&self, source_id: u8, grid_id: &str) -> bool { + self.inner + .lock() + .ok() + .and_then(|s| { + s.capture.as_ref().map(|c| { + c.context() + .sources + .iter() + .any(|v| v.source_id == source_id && v.grid_id == grid_id) + }) + }) + .unwrap_or(false) + } + pub fn capture_active(&self) -> Result { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + Ok(state.capture.as_ref().is_some_and(|c| c.phase_active())) + } + pub fn context(&self) -> Option { + self.inner + .lock() + .ok()? + .capture + .as_ref() + .map(|c| c.context().clone()) + } + pub fn begin_phase( + &self, + expected: &MutationGuard, + request: PhaseRequest, + now_ns: u64, + ) -> Result<(), String> { + let mut state = self.command_state(expected)?; + if busy(&state.job) { + return Err("training is active".into()); + } + if request.role == DataRole::Train && state.frozen_training { + return Err( + "training data is frozen; explicitly start a new capture run to retrain".into(), + ); + } + if request.role == DataRole::HeldOut && state.model.is_none() { + return Err( + "held-out capture starts only after a candidate is trained or restored".into(), + ); + } + state + .capture + .as_mut() + .ok_or("configure the current room and source set first")? + .begin_phase(request, now_ns)?; + state.input_error = None; + Ok(()) + } + pub fn ingest(&self, frame: RawSourceFrame<'_>, now_ns: u64) -> Result<(), String> { + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + let Some(capture) = state.capture.as_mut() else { + return Ok(()); + }; + let result = capture.ingest(frame, now_ns); + if let Err(error) = &result { + state.input_error = Some(bounded(error.clone())); + } + result + } + pub fn ingest_native( + &self, + packet: &crate::room_learner::native::CanonicalPacket, + now_ns: u64, + ) -> Result<(), String> { + if packet.provenance + != crate::room_learner::observation::CaptureProvenance::LiveReceiverObserved + || packet.native_frame.provenance.class + != ruview_unified::frame::ProvenanceClass::Measured + { + return Err("live managed capture rejects synthetic/replayed native provenance".into()); + } + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + let Some(capture) = state.capture.as_mut() else { + return Ok(()); + }; + let source = capture + .context() + .sources + .iter() + .find(|s| s.source_id == packet.source_id) + .ok_or("unselected native source")?; + if packet.native_frame.provenance.device_id != source.device_id + || packet.native_frame.modality != source.signal_kind + { + return Err("native RF provenance does not match the selected source".into()); + } + let result = capture.ingest( + RawSourceFrame { + source_id: packet.source_id, + grid_id: &packet.grid_id, + frequency_hz: packet.frequency_hz, + bandwidth_hz: packet.bandwidth_hz, + sequence: packet.sequence, + amplitudes: &packet.amplitudes, + }, + now_ns, + ); + if let Err(error) = &result { + state.input_error = Some(bounded(error.clone())); + } + result + } + pub fn finish_phase( + &self, + expected: &MutationGuard, + now_ns: u64, + conditions_held: bool, + ) -> Result { + let mut state = self.command_state(expected)?; + let capture = state + .capture + .as_mut() + .ok_or("no learner capture configured")?; + if !conditions_held { + capture.cancel_phase(); + return Err( + "operator did not confirm conditions held; in-flight examples discarded".into(), + ); + } + capture.finish_phase(now_ns) + } + pub fn cancel_phase(&self, expected: &MutationGuard) -> Result<(), String> { + let mut state = self.command_state(expected)?; + state + .capture + .as_mut() + .ok_or("no learner capture configured")? + .cancel_phase(); + Ok(()) + } + pub fn cancel_training(&self, expected: &MutationGuard) -> Result { + let mut state = self.command_state(expected)?; + if busy(&state.job) { + state.job.cancel.store(true, Ordering::Release); + state.job.phase = JobPhase::Cancelling; + } + Ok(state.job.phase) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs new file mode 100644 index 0000000000..f3cea7067a --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs @@ -0,0 +1,505 @@ +//! @tier durable — SYNTHETIC packets through the actual managed public path. +use super::*; +use crate::room_learner::{ + capture::{MIN_PHASE_MS, WINDOW_PACKETS}, + observation::{RoomLabel, SourceSpec}, +}; +use ruview_unified::RfModality; +use std::time::{Duration, Instant}; +const START: u64 = 1_000_000_000_000; +fn context() -> CaptureContext { + CaptureContext { + owner_scope: "a".repeat(64), + installation_id: "synthetic-install".into(), + room_digest: "synthetic-room".into(), + revision: 1, + sources: (1..=2) + .map(|id| SourceSpec { + source_id: id, + device_id: format!("synthetic-esp32-{id}"), + grid_id: "ht64-canonical8".into(), + feature_count: 8, + signal_kind: RfModality::WifiCsi, + link_id: format!("synthetic-link-{id}"), + link_index: 0, + center_frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + geometry: None, + }) + .collect(), + } +} +fn runtime(root: &Path) -> Arc { + Arc::new( + RoomLearningRuntime::new( + &root.join("a".repeat(64)).join("SensingServer"), + &"a".repeat(64), + "synthetic-install", + ) + .unwrap(), + ) +} +fn phase(rt: &RoomLearningRuntime, role: DataRole, label: RoomLabel, index: u32) -> u64 { + let at = START + u64::from(index) * 13_000_000_000; + rt.begin_phase( + &rt.mutation_guard().unwrap(), + PhaseRequest { + trial_id: format!("synthetic-trial-{index}"), + session_id: format!("synthetic-session-{index}"), + role, + label, + duration_ms: MIN_PHASE_MS, + lead_in_ms: 0, + conditions_confirmed: true, + }, + at, + ) + .unwrap(); + let gain = match label { + RoomLabel::Empty => 2.0, + RoomLabel::OccupiedStatic => 6.0, + RoomLabel::OccupiedMoving => 7.0, + RoomLabel::AdjacentOnly => 2.3, + RoomLabel::Unknown => panic!("not labelled"), + }; + for quarter in 0..4 { + for sample in 0..WINDOW_PACKETS { + for source in 1..=2 { + let t = at + + quarter * 3_000_000_000 + + 1_000_000 + + sample as u64 * 10_000_000 + + u64::from(source); + let values: Vec<_> = (0..8) + .map(|bin| { + gain + f64::from(source) * 0.3 + bin as f64 * 0.04 + sample as f64 * 0.003 + }) + .collect(); + rt.ingest( + RawSourceFrame { + source_id: source, + grid_id: "ht64-canonical8", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: index * 32 + quarter as u32 * 8 + sample as u32 + 1, + amplitudes: &values, + }, + t, + ) + .unwrap(); + } + } + } + rt.finish_phase( + &rt.mutation_guard().unwrap(), + at + MIN_PHASE_MS * 1_000_000, + true, + ) + .unwrap(); + at + MIN_PHASE_MS * 1_000_000 +} +fn train_phases(rt: &RoomLearningRuntime) { + for (index, label) in [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ] + .into_iter() + .enumerate() + { + phase(rt, DataRole::Train, label, index as u32); + } +} +async fn finished(rt: &RoomLearningRuntime) -> Value { + let start = Instant::now(); + loop { + let s = rt.status(START + 60_000_000_000); + if !matches!(s["state"].as_str(), Some("running" | "cancelling")) { + return s; + } + assert!( + start.elapsed() < Duration::from_secs(65), + "bounded synthetic training did not finish: {s}" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } +} +#[tokio::test] +async fn real_capture_train_private_store_cold_load_and_unlabelled_inference_compose() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + let ctx = context(); + rt.configure( + &rt.mutation_guard().unwrap(), + ctx.clone(), + "synthetic-run".into(), + true, + ) + .unwrap(); + train_phases(&rt); + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 32, + ..FitConfig::default() + }, + ) + .unwrap(); + let trained = finished(&rt).await; + assert_eq!(trained["state"], "ready", "{trained}"); + assert!( + trained["training_report"]["encoder_delta_l2"] + .as_f64() + .unwrap() + > 0.0 + ); + assert_eq!(trained["physically_validated"], false); + assert_eq!(trained["control_detector_changed"], false); + assert!(rt + .begin_phase( + &rt.mutation_guard().unwrap(), + PhaseRequest { + trial_id: "late-train".into(), + session_id: "late-session".into(), + role: DataRole::Train, + label: RoomLabel::Empty, + duration_ms: MIN_PHASE_MS, + lead_in_ms: 0, + conditions_confirmed: true + }, + START + 60_000_000_000 + ) + .is_err()); + phase(&rt, DataRole::HeldOut, RoomLabel::Empty, 4); + phase(&rt, DataRole::HeldOut, RoomLabel::OccupiedStatic, 5); + let report = rt + .evaluate(&rt.mutation_guard().unwrap(), DataRole::HeldOut) + .unwrap(); + assert_eq!(report["adoption_authorized"], false); + assert!(rt + .evaluate(&rt.mutation_guard().unwrap(), DataRole::Train) + .is_err()); + let cold = runtime(dir.path()); + cold.configure( + &cold.mutation_guard().unwrap(), + ctx, + "synthetic-cold".into(), + false, + ) + .unwrap(); + assert_eq!(cold.status(START)["candidate_loaded"], true); + for row in 0..8 { + for source in 1..=2 { + let values = vec![6.0 + f64::from(source) * 0.3; 8]; + cold.ingest( + RawSourceFrame { + source_id: source, + grid_id: "ht64-canonical8", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: 100 + row, + amplitudes: &values, + }, + START + 100_000_000_000 + u64::from(row) * 10_000_000 + u64::from(source), + ) + .unwrap(); + } + } + let (_, prediction) = cold.prediction(START + 100_080_000_000).unwrap(); + assert!(prediction.occupied_probability.is_some()); + assert!(!prediction.physically_validated); + assert!(!serde_json::to_string(&cold.status(START)) + .unwrap() + .contains("samples")); +} +#[tokio::test] +async fn cancellation_and_context_replacement_cannot_publish_a_late_model() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + let ctx = context(); + rt.configure( + &rt.mutation_guard().unwrap(), + ctx.clone(), + "synthetic-cancel".into(), + true, + ) + .unwrap(); + train_phases(&rt); + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 128, + ..FitConfig::default() + }, + ) + .unwrap(); + assert!(rt + .configure( + &rt.mutation_guard().unwrap(), + ctx.clone(), + "other-run".into(), + true + ) + .is_err()); + assert_eq!( + rt.cancel_training(&rt.mutation_guard().unwrap(),).unwrap(), + JobPhase::Cancelling + ); + assert_eq!(finished(&rt).await["state"], "cancelled"); + let cold = runtime(dir.path()); + assert!(cold + .configure( + &cold.mutation_guard().unwrap(), + ctx, + "no-late-model".into(), + false + ) + .is_err()); + assert_eq!(cold.status(START)["candidate_loaded"], false); +} +#[tokio::test] +async fn wrong_owner_and_incomplete_training_leave_records_and_model_store_untouched() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + let mut ctx = context(); + ctx.owner_scope = "b".repeat(64); + assert!(rt + .configure(&rt.mutation_guard().unwrap(), ctx, "wrong".into(), true) + .is_err()); + rt.configure( + &rt.mutation_guard().unwrap(), + context(), + "incomplete".into(), + true, + ) + .unwrap(); + phase(&rt, DataRole::Train, RoomLabel::Empty, 0); + assert!(rt + .start_training(&rt.mutation_guard().unwrap(), FitConfig::default()) + .is_err()); + assert_eq!(rt.status(START)["state"], "idle"); + assert_eq!(rt.status(START)["capture"]["sealed_cohorts"], 4); + assert!(rt.prediction(START).is_err()); +} + +#[tokio::test] +async fn operator_refusal_discards_only_unsealed_examples_and_bad_config_does_not_freeze_data() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + rt.configure( + &rt.mutation_guard().unwrap(), + context(), + "synthetic-refusal".into(), + true, + ) + .unwrap(); + phase(&rt, DataRole::Train, RoomLabel::Empty, 0); + rt.begin_phase( + &rt.mutation_guard().unwrap(), + PhaseRequest { + trial_id: "refused".into(), + session_id: "refused-session".into(), + role: DataRole::Train, + label: RoomLabel::OccupiedStatic, + duration_ms: MIN_PHASE_MS, + lead_in_ms: 0, + conditions_confirmed: true, + }, + START + 13_000_000_000, + ) + .unwrap(); + assert!(rt + .finish_phase(&rt.mutation_guard().unwrap(), START + 25_000_000_000, false) + .is_err()); + assert_eq!( + rt.status(START + 25_000_000_000)["capture"]["sealed_cohorts"], + 4 + ); + assert_eq!(rt.status(START + 25_000_000_000)["training_frozen"], false); + assert!(rt + .start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 0, + ..FitConfig::default() + } + ) + .is_err()); + assert_eq!(rt.status(START)["training_frozen"], false); +} +#[tokio::test] +async fn cancelled_training_can_retry_frozen_data_without_recapture_or_reopening_train() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + rt.configure( + &rt.mutation_guard().unwrap(), + context(), + "synthetic-retry".into(), + true, + ) + .unwrap(); + train_phases(&rt); + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 128, + ..FitConfig::default() + }, + ) + .unwrap(); + rt.cancel_training(&rt.mutation_guard().unwrap()).unwrap(); + assert_eq!(finished(&rt).await["state"], "cancelled"); + assert_eq!(rt.status(START)["capture"]["sealed_cohorts"], 16); + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 4, + ..FitConfig::default() + }, + ) + .unwrap(); + assert_eq!(finished(&rt).await["state"], "ready"); + assert_eq!(rt.status(START)["capture"]["sealed_cohorts"], 16); +} +#[test] +fn typed_synthetic_native_input_cannot_be_relabelled_live_by_runtime() { + use crate::room_learner::{ + native::{canonicalize_esp32, NativePacketInput}, + observation::CaptureProvenance, + }; + use wifi_densepose_hardware::{Adr018Flags, PpduType}; + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + let bytes: Vec<_> = (0..64).flat_map(|_| [3u8, 4]).collect(); + for provenance in [ + CaptureProvenance::Synthetic, + CaptureProvenance::CapturedReplay, + ] { + let packet = canonicalize_esp32(NativePacketInput { + node_id: 1, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence: 1, + received_at_ns: START, + receiver_sample_rate_hz: 10.0, + iq_bytes: &bytes, + flags: Adr018Flags::default(), + provenance, + }) + .unwrap(); + assert!(rt.ingest_native(&packet, START).is_err()); + } +} + +#[test] +fn delayed_commands_cannot_cross_room_phase_or_helper_restart() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + let initial = rt.mutation_guard().unwrap(); + rt.configure(&initial, context(), "run-a".into(), true) + .unwrap(); + let room_a = rt.mutation_guard().unwrap(); + let mut b = context(); + b.room_digest = "other-room".into(); + rt.configure(&room_a, b.clone(), "run-b".into(), true) + .unwrap(); + let request = PhaseRequest { + trial_id: "delayed-a".into(), + session_id: "a".into(), + role: DataRole::Train, + label: RoomLabel::Empty, + duration_ms: MIN_PHASE_MS, + lead_in_ms: 0, + conditions_confirmed: true, + }; + assert_eq!( + rt.begin_phase(&room_a, request.clone(), START).unwrap_err(), + "learner_state_changed" + ); + assert_eq!( + rt.configure(&room_a, context(), "late-config".into(), true) + .unwrap_err(), + "learner_state_changed" + ); + let idle_b = rt.mutation_guard().unwrap(); + rt.begin_phase(&idle_b, request.clone(), START).unwrap(); + let phase_one = rt.mutation_guard().unwrap(); + rt.cancel_phase(&phase_one).unwrap(); + let mut next = request; + next.trial_id = "fresh-b".into(); + rt.begin_phase(&rt.mutation_guard().unwrap(), next, START) + .unwrap(); + assert_eq!( + rt.finish_phase(&phase_one, START + 13_000_000_000, true) + .unwrap_err(), + "learner_state_changed" + ); + assert_eq!( + rt.cancel_phase(&phase_one).unwrap_err(), + "learner_state_changed" + ); + assert_eq!( + rt.cancel_training(&phase_one).unwrap_err(), + "learner_state_changed" + ); + assert_eq!( + rt.start_training(&phase_one, FitConfig::default()) + .unwrap_err(), + "learner_state_changed" + ); + assert_eq!( + rt.evaluate(&phase_one, DataRole::HeldOut).unwrap_err(), + "learner_state_changed" + ); + assert_eq!( + rt.mutation_guard().unwrap().phase.unwrap().trial_id, + "fresh-b" + ); + let cold = runtime(dir.path()); + assert_ne!( + initial.runtime_epoch, + cold.mutation_guard().unwrap().runtime_epoch + ); + assert_eq!( + cold.configure(&initial, b, "stale-after-restart".into(), true) + .unwrap_err(), + "learner_state_changed" + ); + assert!(cold.context().is_none()); +} +#[test] +fn advertised_defaults_fit_all_native_source_counts_before_capture() { + for count in 2..=8 { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + let mut ctx = context(); + let template = ctx.sources[0].clone(); + ctx.sources = (1..=count) + .map(|id| { + let mut spec = template.clone(); + spec.source_id = id as u8; + spec.device_id = format!("synthetic-device-{id}"); + spec.link_id = format!("synthetic-link-{id}"); + spec.feature_count = 56; + spec + }) + .collect(); + rt.configure( + &rt.mutation_guard().unwrap(), + ctx.clone(), + "budget".into(), + true, + ) + .unwrap(); + let defaults: FitConfig = + serde_json::from_value(rt.status(START)["training_defaults"].clone()).unwrap(); + let actual = defaults.validate_context(&ctx).unwrap(); + assert!(actual <= 512, "{count} sources require {actual} parameters"); + if count >= 7 { + assert!(FitConfig::default().validate_context(&ctx).is_err()); + } + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs index 3e015e2801..9dcddd4655 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs @@ -1,7 +1,8 @@ //! Account-bound multi-source room calibration and learned inference. pub mod observation; pub mod capture; -pub mod native; pub mod model; pub mod training; pub mod artifact; +pub mod lifecycle; +pub mod native; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs new file mode 100644 index 0000000000..cc1067bbc1 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs @@ -0,0 +1,450 @@ +//! Managed integration; raw RF enters only from the existing receiver, never +//! caller-supplied learning scores or uploaded synthetic examples. +use super::*; +use ruview_unified::RfModality; +use wifi_densepose_sensing_server::room_learner::{ + capture::PhaseRequest, + lifecycle::{MutationGuard, RoomLearningRuntime}, + native::{self, NativePacketInput}, + observation::{CaptureProvenance, DataRole, SourceSpec}, + training::FitConfig, +}; + +#[derive(Debug, Clone)] +pub(super) struct LiveSource { + frequency_hz: u64, + bandwidth_hz: Option, + seen: std::time::Instant, + receiver_rate_hz: f64, +} +fn grid_id(frame: &Esp32Frame) -> Option { + native::grid_id( + frame.n_antennas as usize, + frame.n_subcarriers as usize, + frame.ppdu_type, + ) + .ok() +} +fn bandwidth(frame: &Esp32Frame) -> Option { + if !frame.adr018_flags.bw40 && grid_id(frame).is_some() { + Some(20_000_000) + } else { + None + } +} +pub(super) fn observe( + state: &mut AppStateInner, + frame: &Esp32Frame, + packet: &[u8], + now: std::time::Instant, +) { + let Some(runtime) = state.room_learning.clone() else { + return; + }; + let key = ( + frame.node_id, + frame.n_subcarriers, + frame.ppdu_type.to_byte(), + ); + let rate = state + .room_learning_sources + .get(&key) + .map_or(0.0, |previous| { + let delta = now.saturating_duration_since(previous.seen).as_secs_f64(); + if delta <= 0.0 || delta > 5.0 { + 0.0 + } else if previous.receiver_rate_hz > 0.0 { + previous.receiver_rate_hz * 0.9 + (1.0 / delta) * 0.1 + } else { + 1.0 / delta + } + }); + let meta = LiveSource { + frequency_hz: u64::from(frame.freq_mhz) * 1_000_000, + bandwidth_hz: bandwidth(frame), + seen: now, + receiver_rate_hz: rate, + }; + if state.room_learning_sources.len() < 1_024 || state.room_learning_sources.contains_key(&key) { + state.room_learning_sources.insert(key, meta.clone()); + } + let Some(id) = grid_id(frame) else { + return; + }; + if !runtime.selects_grid(frame.node_id, &id) { + return; + } + let bytes = usize::from(frame.n_antennas) * usize::from(frame.n_subcarriers) * 2; + let Some(iq) = packet.get(20..20 + bytes) else { + return; + }; + let ns = runtime.receiver_now_ns(); + let projected = match native::canonicalize_esp32(NativePacketInput { + node_id: frame.node_id, + n_antennas: frame.n_antennas as usize, + raw_bins: frame.n_subcarriers as usize, + frequency_hz: meta.frequency_hz, + ppdu_type: frame.ppdu_type, + sequence: frame.sequence, + received_at_ns: ns, + receiver_sample_rate_hz: rate, + iq_bytes: iq, + flags: frame.adr018_flags, + provenance: CaptureProvenance::LiveReceiverObserved, + }) { + Ok(p) => p, + Err(_) => return, + }; + // This is the real canonical RF record plus an explicitly derived magnitude + // view. No IQ, source-clock or coherent-phase claim is invented/persisted. + let _ = runtime.ingest_native(&projected, ns); +} +fn failure(code: &str, error: impl ToString) -> Json { + let code = if error.to_string() == "learner_state_changed" { + "learner_state_changed" + } else { + code + }; + Json(serde_json::json!({"success":false,"error_code":code, + "error":error.to_string().chars().filter(|c|!c.is_control()).take(256).collect::()})) +} +async fn runtime(state: &SharedState) -> Result, Json> { + let s = state.read().await; + if s.privacy_mode { + return Err(failure( + "learner_privacy_mode", + "Room learning is unavailable in privacy-redacted mode.", + )); + } + s.room_learning.clone().ok_or_else(|| { + failure( + "learner_disabled", + "This helper does not enable the managed room learner.", + ) + }) +} +fn now_ns(runtime: &RoomLearningRuntime) -> u64 { + runtime.receiver_now_ns() +} +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct ConfigureRequest { + expected: MutationGuard, + binding_digest: String, + revision: u64, + source_node_ids: Vec, + new_training: bool, +} +fn source_ready(state: &AppStateInner, spec: &SourceSpec, now: std::time::Instant) -> bool { + state.node_states.get(&spec.source_id).is_some_and(|node| { + node.calibration_grid_candidates(now) + .iter() + .any(|(grid, evidence)| { + native::grid_id( + 1, + grid.n_subcarriers as usize, + wifi_densepose_hardware::PpduType::from_byte(grid.ppdu_type), + ) + .ok() + .as_deref() + == Some(spec.grid_id.as_str()) + && evidence.span_s >= CALIBRATION_GRID_MIN_SPAN_S + && evidence.rate_hz >= CALIBRATION_GRID_MIN_RATE_HZ + && evidence.max_gap_s < CALIBRATION_GRID_MAX_GAP_S + && evidence.latest_age_s < 2.0 + && state + .room_learning_sources + .get(&(spec.source_id, grid.n_subcarriers, grid.ppdu_type)) + .is_some_and(|meta| { + now.saturating_duration_since(meta.seen).as_secs_f64() < 2.0 + && meta.frequency_hz == spec.center_frequency_hz + && meta.bandwidth_hz == Some(spec.bandwidth_hz) + }) + }) + }) && spec.feature_count == 56 + && spec.signal_kind == RfModality::WifiCsi + && spec.device_id == format!("esp32-node-{}", spec.source_id) +} +pub(super) async fn configure( + State(state): State, + Json(request): Json, +) -> Json { + let rt = match runtime(&state).await { + Ok(r) => r, + Err(e) => return e, + }; + if !valid_binding_digest(&request.binding_digest) + || !valid_source_node_ids(&request.source_node_ids) + || !(2..=8).contains(&request.source_node_ids.len()) + || request.revision == 0 + { + return failure( + "learner_context_invalid", + "Use a current room digest/revision and 2 to 8 sorted unique node IDs.", + ); + } + let now = std::time::Instant::now(); + let s = state.read().await; + if !request.new_training { + let catalog = match rt.saved_candidates(&request.binding_digest) { + Ok(c) => c, + Err(e) => return failure("learner_saved_models_unavailable", e), + }; + let found = catalog.candidates.iter().find(|entry| { + entry.context.revision == request.revision + && entry + .context + .sources + .iter() + .map(|spec| spec.source_id) + .collect::>() + == request.source_node_ids + && entry + .context + .sources + .iter() + .all(|spec| source_ready(&s, spec, now)) + }); + let Some(saved) = found else { + return failure( + "learner_saved_context_unavailable", + if catalog.candidates.is_empty() { + "No readable saved candidate for this exact room. Stored history is preserved; start a new capture or inspect model recovery." + } else { + "Saved candidates are retained, but their trained source/PHY context is not currently receiving. Restore those inputs or explicitly train a new model." + }, + ); + }; + let context = saved.context.clone(); + drop(s); + return match rt.configure( + &request.expected, + context, + opaque_calibration_id("learner-resume"), + false, + ) { + Ok(()) => Json(serde_json::json!({"success":true,"status":rt.status(now_ns(&rt))})), + Err(e) => failure("learner_resume_failed", e), + }; + } + let mut sources = Vec::new(); + for &id in &request.source_node_ids { + let Some(node) = s.node_states.get(&id) else { + return failure( + "learner_source_missing", + format!("Node {id} is not receiving."), + ); + }; + let Some((grid, _)) = node.select_calibration_grid(now) else { + return failure( + "learner_source_unready", + format!("Node {id} needs an eligible stable raw CSI grid."), + ); + }; + let key = (id, grid.n_subcarriers, grid.ppdu_type); + let Some(meta) = s.room_learning_sources.get(&key) else { + return failure( + "learner_phy_missing", + format!("Node {id} lacks current PHY metadata."), + ); + }; + let Some(bandwidth_hz) = meta.bandwidth_hz else { + return failure( + "learner_phy_unsupported", + format!("Node {id} uses an unsupported bandwidth layout."), + ); + }; + if now.saturating_duration_since(meta.seen).as_secs_f64() > 2.0 + || !(2_000_000_000..=8_000_000_000).contains(&meta.frequency_hz) + { + return failure( + "learner_phy_stale", + format!("Node {id} PHY metadata is stale or invalid."), + ); + } + sources.push(SourceSpec { + source_id: id, + device_id: format!("esp32-node-{id}"), + grid_id: match native::grid_id( + 1, + grid.n_subcarriers as usize, + wifi_densepose_hardware::PpduType::from_byte(grid.ppdu_type), + ) { + Ok(id) => id, + Err(error) => return failure("learner_grid_unsupported", error), + }, + feature_count: 56, + signal_kind: RfModality::WifiCsi, + center_frequency_hz: meta.frequency_hz, + bandwidth_hz, + link_id: format!("receiver-{id}-transmitter-unverified"), + link_index: 0, + geometry: None, + }); + } + drop(s); + let context = match rt.bind_context(request.binding_digest, request.revision, sources) { + Ok(c) => c, + Err(e) => return failure("learner_context_invalid", e), + }; + let run = opaque_calibration_id("learner-run"); + match rt.configure(&request.expected, context, run, request.new_training) { + Ok(()) => Json(serde_json::json!({"success":true,"status":rt.status(now_ns(&rt))})), + Err(e) => failure("learner_configure_failed", e), + } +} +pub(super) async fn status(State(state): State) -> Json { + match runtime(&state).await { + Ok(rt) => Json(rt.status(now_ns(&rt))), + Err(e) => e, + } +} +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct BeginRequest { + expected: MutationGuard, + phase: PhaseRequest, +} +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct GuardRequest { + expected: MutationGuard, +} +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct TrainRequest { + expected: MutationGuard, + config: FitConfig, +} +pub(super) async fn begin_phase( + State(state): State, + Json(request): Json, +) -> Json { + let state = state.write().await; + if state.privacy_mode { + return failure( + "learner_privacy_mode", + "Room learning is unavailable in privacy-redacted mode.", + ); + } + if state.field_model.as_ref().is_some_and(|f| { + matches!(f.status(), CalibrationStatus::Collecting) + || (state.calibration_session_id.is_some() + && matches!(f.status(), CalibrationStatus::Uncalibrated)) + }) { + return failure( + "legacy_empty_capture_active", + "Finish or cancel the legacy empty capture before learner example capture.", + ); + } + let Some(rt) = state.room_learning.as_ref() else { + return failure("learner_disabled", "Managed learner is not enabled."); + }; + if rt.context().as_ref().is_none_or(|context| { + context + .sources + .iter() + .any(|spec| !source_ready(&state, spec, std::time::Instant::now())) + }) { + return failure("learner_source_unready","A selected source/grid is no longer fresh. Restore its stream and refresh source readiness before capture."); + } + match rt.begin_phase(&request.expected, request.phase, now_ns(rt)) { + Ok(()) => Json(serde_json::json!({"success":true,"status":rt.status(now_ns(rt))})), + Err(e) => failure("learner_phase_rejected", e), + } +} +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct FinishRequest { + expected: MutationGuard, + conditions_held: bool, +} +pub(super) async fn finish_phase( + State(state): State, + Json(request): Json, +) -> Json { + let rt = match runtime(&state).await { + Ok(r) => r, + Err(e) => return e, + }; + match rt.finish_phase(&request.expected, now_ns(&rt), request.conditions_held) { + Ok(receipt) => Json(serde_json::json!({"success":true,"receipt":receipt})), + Err(e) => failure("learner_phase_incomplete", e), + } +} +pub(super) async fn cancel_phase( + State(state): State, + Json(request): Json, +) -> Json { + let rt = match runtime(&state).await { + Ok(r) => r, + Err(e) => return e, + }; + match rt.cancel_phase(&request.expected) { + Ok(()) => Json(serde_json::json!({"success":true})), + Err(e) => failure("learner_phase_cancel_failed", e), + } +} +pub(super) async fn train( + State(state): State, + Json(request): Json, +) -> Json { + let rt = match runtime(&state).await { + Ok(r) => r, + Err(e) => return e, + }; + match rt.start_training(&request.expected, request.config) { + Ok(generation) => Json( + serde_json::json!({"success":true,"generation":generation,"status":rt.status(now_ns(&rt))}), + ), + Err(e) => failure("learner_train_rejected", e), + } +} +pub(super) async fn cancel_training( + State(state): State, + Json(request): Json, +) -> Json { + let rt = match runtime(&state).await { + Ok(r) => r, + Err(e) => return e, + }; + match rt.cancel_training(&request.expected) { + Ok(phase) => Json(serde_json::json!({"success":true,"state":phase})), + Err(e) => failure("learner_cancel_failed", e), + } +} +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct EvaluationRequest { + expected: MutationGuard, + role: DataRole, +} +pub(super) async fn evaluate( + State(state): State, + Json(request): Json, +) -> Json { + let rt = match runtime(&state).await { + Ok(r) => r, + Err(e) => return e, + }; + match rt.evaluate(&request.expected, request.role) { + Ok(report) => Json(serde_json::json!({"success":true,"evaluation":report})), + Err(e) => failure("learner_evaluation_rejected", e), + } +} +pub(super) async fn prediction(State(state): State) -> Json { + let rt = match runtime(&state).await { + Ok(r) => r, + Err(e) => return e, + }; + match rt.prediction(now_ns(&rt)) { + Ok((mutation_guard, prediction)) => Json( + serde_json::json!({"success":true,"mutation_guard":mutation_guard,"candidate_prediction":prediction,"authority":"experimental_candidate_only"}), + ), + Err(e) => failure("learner_prediction_unavailable", e), + } +} + +#[cfg(test)] +#[path = "room_learner_api_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs new file mode 100644 index 0000000000..511e22c96d --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs @@ -0,0 +1,280 @@ +//! @tier durable — SYNTHETIC datagrams through real configure/parser/observe. +//! Receiver-observed fixtures are not physical hardware or accuracy evidence. +use super::*; +use axum::{ + body::Body, + http::{Method, Request}, +}; +use tower::ServiceExt; +fn wire(node: u8, seq: u32) -> Vec { + let mut bytes = Vec::new(); + bytes.extend(0xC5110001u32.to_le_bytes()); + bytes.extend([node, 1]); + bytes.extend(64u16.to_le_bytes()); + bytes.extend(2437u32.to_le_bytes()); + bytes.extend(seq.to_le_bytes()); + bytes.extend([(-40i8) as u8, (-90i8) as u8, 0, 0x20]); + for _ in 0..64 { + bytes.extend([3, 4]); + } + bytes +} +fn fixture() -> (tempfile::TempDir, SharedState) { + let dir = tempfile::tempdir().unwrap(); + let owner = "a".repeat(64); + let runtime = Arc::new( + RoomLearningRuntime::new( + &dir.path().join(&owner).join("SensingServer"), + &owner, + "synthetic-install", + ) + .unwrap(), + ); + let mut inner = AppStateInner::minimal(); + inner.room_learning = Some(runtime); + let now = std::time::Instant::now(); + for row in 0..120 { + for node in [15, 16, 17] { + let packet = wire(node, row + 1); + let frame = parse_esp32_frame(&packet).unwrap(); + let at = now - std::time::Duration::from_millis((120 - row) as u64 * 100); + inner + .node_states + .entry(node) + .or_insert_with(NodeState::new) + .observe_raw_grid(CsiGridKey::from_frame(&frame), at); + observe(&mut inner, &frame, &packet, at); + } + } + (dir, Arc::new(RwLock::new(inner))) +} +fn app(state: SharedState) -> Router { + Router::new() + .route("/configure", post(configure)) + .route("/status", get(status)) + .route("/phase", post(begin_phase)) + .route("/finish", post(finish_phase)) + .with_state(state) +} +async fn json_request( + app: Router, + path: &str, + method: Method, + mut body: serde_json::Value, +) -> serde_json::Value { + if method == Method::POST && body.get("expected").is_none() { + let current = json_request_status(app.clone()).await; + if path == "/phase" { + body = serde_json::json!({"phase":body}); + } + body["expected"] = if current["mutation_guard"].is_object() { + current["mutation_guard"].clone() + } else { + serde_json::json!({"runtime_epoch":"synthetic-disabled","generation":0,"workflow_revision":0,"context_digest":null,"phase":null}) + }; + } + let response = app + .oneshot( + Request::builder() + .method(method) + .uri(path) + .header("content-type", "application/json") + .body(Body::from(serde_json::to_vec(&body).unwrap())) + .unwrap(), + ) + .await + .unwrap(); + let bytes = axum::body::to_bytes(response.into_body(), 65536) + .await + .unwrap(); + serde_json::from_slice(&bytes).unwrap() +} +async fn json_request_status(app: Router) -> serde_json::Value { + let response = app + .oneshot( + Request::builder() + .uri("/status") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + let bytes = axum::body::to_bytes(response.into_body(), 65536) + .await + .unwrap(); + serde_json::from_slice(&bytes).unwrap() +} +fn config() -> serde_json::Value { + serde_json::json!({"binding_digest":"b".repeat(64),"revision":1,"source_node_ids":[15,16,17],"new_training":true}) +} +#[tokio::test] +async fn configure_and_actual_native_ingress_share_grid_projection_and_device_identity() { + let (_dir, state) = fixture(); + let router = app(state.clone()); + let configured = json_request(router.clone(), "/configure", Method::POST, config()).await; + assert_eq!(configured["success"], true, "{configured}"); + let sources = configured["status"]["sources"].as_array().unwrap(); + assert_eq!(sources.len(), 3); + assert_eq!(sources[0]["device_id"], "esp32-node-15"); + assert_eq!(sources[0]["grid_id"], "ant1-bins64-ppdu0-usable-drop2-v1"); + let begun=json_request(router.clone(),"/phase",Method::POST,serde_json::json!({ + "trial_id":"synthetic-empty","session_id":"synthetic-train-session","role":"Train","label":"Empty", + "duration_ms":12000,"conditions_confirmed":true})).await; + assert_eq!(begun["success"], true, "{begun}"); + for row in 0..8 { + for node in [15, 16, 17] { + let packet = wire(node, 121 + row); + let frame = parse_esp32_frame(&packet).unwrap(); + let mut inner = state.write().await; + observe(&mut inner, &frame, &packet, std::time::Instant::now()); + } + } + let read = json_request( + router.clone(), + "/status", + Method::GET, + serde_json::Value::Null, + ) + .await; + assert_eq!(read["required_source_ids"], serde_json::json!([15, 16, 17])); + assert_eq!(read["trained_source_ids"], serde_json::json!([])); + assert_eq!(read["scored_source_ids"], serde_json::json!([])); + for source in read["capture"]["sources"].as_array().unwrap() { + assert_eq!(source["admitted_packets"], 8, "{source}"); + assert_eq!(source["live_buffer_frames"], 8); + } + assert_eq!(read["physically_validated"], false); + assert!( + state.read().await.field_model.is_none(), + "new learner must not silently create/change control calibration" + ); + let incomplete = json_request( + router, + "/finish", + Method::POST, + serde_json::json!({"conditions_held":true}), + ) + .await; + assert_eq!( + incomplete["success"], false, + "wall-clock duration cannot be bypassed by fast synthetic packets" + ); +} +#[tokio::test] +async fn privacy_disabled_source_drop_and_unconfirmed_conditions_are_explicit() { + let empty = Arc::new(RwLock::new(AppStateInner::minimal())); + let disabled = json_request(app(empty), "/status", Method::GET, serde_json::Value::Null).await; + assert_eq!(disabled["error_code"], "learner_disabled"); + let (_dir, state) = fixture(); + state.write().await.privacy_mode = true; + let hidden = json_request(app(state.clone()), "/configure", Method::POST, config()).await; + assert_eq!(hidden["error_code"], "learner_privacy_mode"); + assert!(!hidden.to_string().contains("owner_scope")); + state.write().await.privacy_mode = false; + state.write().await.node_states.remove(&17); + let missing = json_request(app(state.clone()), "/configure", Method::POST, config()).await; + assert_eq!(missing["error_code"], "learner_source_missing"); + assert!(state + .read() + .await + .room_learning + .as_ref() + .unwrap() + .context() + .is_none()); +} +#[test] +fn every_new_mutation_requires_admin_and_default_cli_does_not_enable_capture() { + use wifi_densepose_sensing_server::bearer_auth::required_scope_for; + for suffix in [ + "configure", + "phase/start", + "phase/finish", + "phase/cancel", + "train", + "train/cancel", + "evaluate", + ] { + assert_eq!( + required_scope_for(&Method::POST, &format!("/api/v1/room-learner/{suffix}")), + "sensing:admin" + ); + } + assert_eq!( + required_scope_for(&Method::GET, "/api/v1/room-learner/status"), + "sensing:read" + ); + assert!( + !Args::try_parse_from(["sensing-server"]) + .unwrap() + .room_learner + ); +} + +#[tokio::test] +async fn learner_examples_and_legacy_empty_capture_cannot_contaminate_each_other() { + let (_dir, state) = fixture(); + let router = app(state.clone()); + assert_eq!( + json_request(router.clone(), "/configure", Method::POST, config()).await["success"], + true + ); + { + let mut s = state.write().await; + s.field_model = Some(FieldModel::new(field_bridge::single_link_config()).unwrap()); + s.calibration_session_id = Some("cal-session-synthetic".into()); + } + let request = serde_json::json!({"trial_id":"occupied","session_id":"train","role":"Train", + "label":"OccupiedStatic","duration_ms":12000,"lead_in_ms":5000,"conditions_confirmed":true}); + let blocked = json_request(router.clone(), "/phase", Method::POST, request.clone()).await; + assert_eq!(blocked["error_code"], "legacy_empty_capture_active"); + { + let mut s = state.write().await; + s.field_model = None; + s.calibration_session_id = None; + } + assert_eq!( + json_request(router, "/phase", Method::POST, request).await["success"], + true + ); + let Json(blocked) = calibration_start( + State(state.clone()), + Query(CalibrationStartQuery { + source_node_id: None, + }), + None, + None, + Json(CalibrationBindingRequest { + binding_digest: "b".repeat(64), + source_node_ids: vec![15, 16, 17], + }), + ) + .await; + assert_eq!(blocked["error_code"], "learner_capture_active"); + assert!(state.read().await.field_model.is_none()); +} + +#[tokio::test] +async fn stale_http_start_is_rejected_after_another_client_changes_room() { + let (_dir, state) = fixture(); + let router = app(state); + let a = json_request(router.clone(), "/configure", Method::POST, config()).await; + assert_eq!(a["success"], true); + let guard_a = a["status"]["mutation_guard"].clone(); + let mut desired_b = config(); + desired_b["binding_digest"] = serde_json::json!("c".repeat(64)); + desired_b["expected"] = guard_a.clone(); + let b = json_request(router.clone(), "/configure", Method::POST, desired_b).await; + assert_eq!(b["success"], true, "{b}"); + let delayed = serde_json::json!({"expected":guard_a,"phase":{ + "trial_id":"delayed-room-a","session_id":"room-a","role":"Train","label":"Empty", + "duration_ms":12000,"conditions_confirmed":true}}); + let rejected = json_request(router.clone(), "/phase", Method::POST, delayed).await; + assert_eq!( + rejected["error_code"], "learner_state_changed", + "{rejected}" + ); + let current = json_request_status(router).await; + assert_eq!(current["mutation_guard"], b["status"]["mutation_guard"]); + assert!(current["capture"]["phase"].is_null()); +} From 9923347bcdeff8b3f8c2164b9801a38b4dd0ac2b Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Sun, 1 Nov 2026 20:53:04 +0100 Subject: [PATCH 033/100] test(learning): bound lifecycle composition fixture workload --- .../src/room_learner/lifecycle_tests.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs index f3cea7067a..2444715a58 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs @@ -141,7 +141,7 @@ async fn real_capture_train_private_store_cold_load_and_unlabelled_inference_com rt.start_training( &rt.mutation_guard().unwrap(), FitConfig { - epochs: 32, + epochs: 4, // Composition needs real updates, not accuracy convergence. ..FitConfig::default() }, ) From cc59662d284fe2870330de6f1c4263811aed66c1 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Sun, 1 Nov 2026 21:04:50 +0100 Subject: [PATCH 034/100] fix(learning): expose bounded native input rejection diagnostics --- .../src/room_learner/lifecycle_capture.rs | 9 +++++++++ .../src/room_learner/lifecycle_tests.rs | 17 +++++++++++++++++ .../src/room_learner/native.rs | 7 ++++--- .../src/room_learner_api.rs | 2 +- .../src/room_learner_api_tests.rs | 15 +++++++++++++++ 5 files changed, 46 insertions(+), 4 deletions(-) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs index 7807789f8f..b0235ae3eb 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs @@ -125,6 +125,15 @@ impl RoomLearningRuntime { } result } + /// Bounded protected diagnostics only; never grants admission or advances a + /// workflow revision. Ignore sources/grids outside the current context. + pub fn record_native_rejection(&self, source_id:u8, grid_id:&str, reason:&str) { + if let Ok(mut state)=self.inner.lock() { + if state.capture.as_ref().is_some_and(|c|c.context().sources.iter().any(|s|s.source_id==source_id && s.grid_id==grid_id)) { + state.input_error=Some(bounded(format!("Node {source_id} native input rejected: {reason}"))); + } + } + } pub fn finish_phase( &self, expected: &MutationGuard, diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs index 2444715a58..9854cec8bc 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs @@ -503,3 +503,20 @@ fn advertised_defaults_fit_all_native_source_counts_before_capture() { } } } + +#[test] +fn native_rejection_diagnostics_are_context_bound_bounded_and_not_admission() { + let dir=tempfile::tempdir().unwrap();let rt=runtime(dir.path()); + rt.configure(&rt.mutation_guard().unwrap(),context(),"native-diagnostic".into(),true).unwrap(); + let observed=rt.mutation_guard().unwrap(); + rt.record_native_rejection(17,"ht64-canonical8","unselected"); + rt.record_native_rejection(1,"other-grid","unselected"); + assert!(rt.status(START)["input_error"].is_null()); + rt.record_native_rejection(1,"ht64-canonical8",&"x".repeat(1000)); + let status=rt.status(START); + assert!(status["input_error"].as_str().unwrap().starts_with("Node 1 native input rejected:")); + assert!(status["input_error"].as_str().unwrap().len()<=256); + assert_eq!(rt.mutation_guard().unwrap(),observed); + assert_eq!(status["capture"]["sources"][0]["admitted_packets"],0); + assert_eq!(status["physically_validated"],false); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/native.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/native.rs index 4440fdf535..7ac721086d 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/native.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/native.rs @@ -100,9 +100,10 @@ pub fn grid_id(n_antennas: usize, raw_bins: usize, ppdu_type: PpduType) -> Resul /// `normalize()` or `RfFrameV2::to_canonical()` (both remove amplitude gain). pub fn canonicalize_esp32(input: NativePacketInput<'_>) -> Result { let grid_id = grid_id(input.n_antennas, input.raw_bins, input.ppdu_type)?; - if input.flags.bw40 || input.received_at_ns == 0 || !input.receiver_sample_rate_hz.is_finite() - || input.receiver_sample_rate_hz <= 0.0 { - return Err("unsupported bandwidth, receiver time or measured receiver cadence".into()); + if input.flags.bw40 { return Err("native capture does not support the reported 40 MHz bandwidth".into()); } + if input.received_at_ns == 0 { return Err("native receiver time is unavailable".into()); } + if !input.receiver_sample_rate_hz.is_finite() || input.receiver_sample_rate_hz <= 0.0 { + return Err("measured receiver cadence is unavailable".into()); } let mhz = input.frequency_hz / 1_000_000; if input.frequency_hz % 1_000_000 != 0 diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs index cc1067bbc1..f895625471 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs @@ -93,7 +93,7 @@ pub(super) fn observe( provenance: CaptureProvenance::LiveReceiverObserved, }) { Ok(p) => p, - Err(_) => return, + Err(reason) => { runtime.record_native_rejection(frame.node_id, &id, &reason); return; } }; // This is the real canonical RF record plus an explicitly derived magnitude // view. No IQ, source-clock or coherent-phase claim is invented/persisted. diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs index 511e22c96d..3dde272369 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs @@ -278,3 +278,18 @@ async fn stale_http_start_is_rejected_after_another_client_changes_room() { assert_eq!(current["mutation_guard"], b["status"]["mutation_guard"]); assert!(current["capture"]["phase"].is_null()); } + +#[tokio::test] +async fn native_adapter_rejection_is_visible_without_admitting_or_labelling_packets() { + let (_dir,state)=fixture();let router=app(state.clone()); + let configured=json_request(router.clone(),"/configure",Method::POST,config()).await; + assert_eq!(configured["success"],true); + let mut packet=wire(15,121);packet[19]|=0x01; // Actual ADR-018 40 MHz bit. + let frame=parse_esp32_frame(&packet).unwrap(); + observe(&mut *state.write().await,&frame,&packet,std::time::Instant::now()); + let status=json_request_status(router).await; + assert_eq!(status["input_error"],"Node 15 native input rejected: native capture does not support the reported 40 MHz bandwidth"); + assert_eq!(status["capture"]["sources"][0]["admitted_packets"],0); + assert_eq!(status["mutation_guard"],configured["status"]["mutation_guard"]); + assert_eq!(status["physically_validated"],false); +} From c6066b24f3306baace161da29dab06e2a526ce88 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Sun, 1 Nov 2026 21:33:17 +0100 Subject: [PATCH 035/100] feat(signal): add gain-preserving nonnegative magnitude projection --- .../src/hardware_norm.rs | 163 ++++++++++++++++++ 1 file changed, 163 insertions(+) diff --git a/v2/crates/wifi-densepose-signal/src/hardware_norm.rs b/v2/crates/wifi-densepose-signal/src/hardware_norm.rs index 188dd76a2b..11d5873e2c 100644 --- a/v2/crates/wifi-densepose-signal/src/hardware_norm.rs +++ b/v2/crates/wifi-densepose-signal/src/hardware_norm.rs @@ -21,6 +21,8 @@ pub enum HardwareNormError { UnknownHardware(usize), #[error("Invalid canonical subcarrier count: {0}")] InvalidCanonical(usize), + #[error("Magnitude must be finite and non-negative at index {index}")] + InvalidMagnitude { index: usize }, } /// Known WiFi chipset families with their subcarrier counts and MIMO configs. @@ -183,6 +185,56 @@ impl HardwareNormalizer { pub fn resample_to_canonical(&self, raw: &[f64]) -> Vec { resample_cubic(raw, self.canonical_subcarriers) } + + /// Length-only magnitude projection using convex linear interpolation. + /// + /// Spectral nulls and guard bands can make the cubic control path overshoot + /// below zero. This opt-in path retains the canonical grid and amplitude + /// units, including constant DC and gain, without phase or z-score changes. + /// Invalid inputs are rejected before identity/singleton handling; they are + /// never clamped, replaced with zeros, or reinterpreted as measurements. + pub fn resample_magnitude_to_canonical( + &self, + raw: &[f64], + ) -> Result, HardwareNormError> { + if raw.is_empty() { + return Err(HardwareNormError::EmptyFrame { amp: 0, phase: 0 }); + } + if let Some(index) = raw.iter().position(|v| !v.is_finite() || *v < 0.0) { + return Err(HardwareNormError::InvalidMagnitude { index }); + } + let dst_len = self.canonical_subcarriers; + if raw.len() == dst_len { + return Ok(raw.to_vec()); + } + if raw.len() == 1 || dst_len == 1 { + return Ok(vec![raw[0]; dst_len]); + } + let last = raw.len() - 1; + let ratio = last as f64 / (dst_len - 1) as f64; + Ok((0..dst_len) + .map(|i| { + if i == dst_len - 1 { + return raw[last]; + } + let position = i as f64 * ratio; + let left = (position.floor() as usize).min(last); + let right = (left + 1).min(last); + let fraction = position - left as f64; + if raw[left] == raw[right] { + raw[left] + } else { + // Same-sign finite endpoints keep the difference finite; + // FMA avoids the overflowing sum of a weighted blend. + let interpolated = (raw[right] - raw[left]).mul_add(fraction, raw[left]); + // Bound rounding to the two valid endpoints, never the input. + interpolated + .max(raw[left].min(raw[right])) + .min(raw[left].max(raw[right])) + } + }) + .collect()) + } } impl Default for HardwareNormalizer { @@ -294,6 +346,117 @@ fn sanitize_phase(phase: &[f64]) -> Vec { mod tests { use super::*; + #[test] + fn magnitude_projection_rejects_invalid_inputs_even_on_identity_grid() { + let norm = HardwareNormalizer::new(); + assert!(matches!( + norm.resample_magnitude_to_canonical(&[]), + Err(HardwareNormError::EmptyFrame { .. }) + )); + for invalid in [-0.01, f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + for len in [1, 56, 62, 254] { + let mut raw = vec![2.0; len]; + raw[len - 1] = invalid; + assert!(matches!( + norm.resample_magnitude_to_canonical(&raw), + Err(HardwareNormError::InvalidMagnitude { index }) if index == len - 1 + )); + } + } + } + + #[test] + fn magnitude_projection_preserves_native_hull_at_sharp_guard_bands() { + let norm = HardwareNormalizer::new(); + for len in [62, 254] { + let raw: Vec = (0..len) + .map(|i| { + if (len / 3..2 * len / 3).contains(&i) { + 40.0 + } else { + 0.0 + } + }) + .collect(); + let cubic = norm.resample_to_canonical(&raw); + assert!( + cubic.iter().any(|v| *v < -1.0), + "fixture must expose cubic undershoot" + ); + let projected = norm.resample_magnitude_to_canonical(&raw).unwrap(); + assert_eq!(projected.len(), 56); + assert!(projected + .iter() + .all(|v| v.is_finite() && (0.0..=40.0).contains(v))); + assert_eq!(projected[0], raw[0]); + assert_eq!(projected[55], raw[len - 1]); + } + // A zero-valued notch also exposes cubic ringing on the 62-bin grid. + let raw: Vec = (0..62) + .map(|i| if (20..41).contains(&i) { 0.0 } else { 40.0 }) + .collect(); + assert!(norm.resample_to_canonical(&raw).iter().any(|v| *v < 0.0)); + assert!(norm + .resample_magnitude_to_canonical(&raw) + .unwrap() + .iter() + .all(|v| (0.0..=40.0).contains(v))); + } + + #[test] + fn magnitude_projection_preserves_dc_gain_and_configured_width() { + for len in [62, 254] { + let norm = HardwareNormalizer::new(); + assert_eq!( + norm.resample_magnitude_to_canonical(&vec![7.25; len]) + .unwrap(), + vec![7.25; 56] + ); + let raw: Vec = (0..len).map(|i| (i % 13) as f64 * 0.25).collect(); + let affine: Vec = raw.iter().map(|v| 3.5 * v + 2.75).collect(); + let base = norm.resample_magnitude_to_canonical(&raw).unwrap(); + let scaled = norm.resample_magnitude_to_canonical(&affine).unwrap(); + for (a, b) in base.iter().zip(scaled) { + assert!((b - (3.5 * a + 2.75)).abs() < 1e-12); + } + } + let custom = HardwareNormalizer::with_canonical_subcarriers(3).unwrap(); + assert_eq!( + custom + .resample_magnitude_to_canonical(&[2.0, 10.0]) + .unwrap(), + vec![2.0, 6.0, 10.0] + ); + assert_eq!( + custom.resample_magnitude_to_canonical(&[3.0]).unwrap(), + vec![3.0; 3] + ); + assert_eq!( + custom + .resample_magnitude_to_canonical(&[2.0, 0.0, 9.0]) + .unwrap(), + vec![2.0, 0.0, 9.0] + ); + let quarters = HardwareNormalizer::with_canonical_subcarriers(5).unwrap(); + assert_eq!( + quarters.resample_magnitude_to_canonical(&[0.0, 8.0]).unwrap(), + vec![0.0, 2.0, 4.0, 6.0, 8.0] + ); + let singleton = HardwareNormalizer::with_canonical_subcarriers(1).unwrap(); + assert_eq!( + singleton + .resample_magnitude_to_canonical(&[2.0, 10.0]) + .unwrap(), + vec![2.0] + ); + let extreme = HardwareNormalizer::new() + .resample_magnitude_to_canonical(&[0.0, f64::MAX]) + .unwrap(); + assert!(extreme + .iter() + .all(|v| v.is_finite() && *v >= 0.0 && *v <= f64::MAX)); + } + #[test] fn detect_hardware_and_properties() { assert_eq!( From e379b3afa84d54f989a0f5071371d6030edee020 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Sun, 1 Nov 2026 21:33:17 +0100 Subject: [PATCH 036/100] fix(learning): admit valid spectral nulls with versioned native projection --- .../src/room_learner/native.rs | 15 +++++--- .../src/room_learner/native_tests.rs | 36 ++++++++++++++++--- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/native.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/native.rs index 7ac721086d..862f2d02c7 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/native.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/native.rs @@ -92,7 +92,14 @@ pub fn grid_id(n_antennas: usize, raw_bins: usize, ppdu_type: PpduType) -> Resul PpduType::Unknown => false, }; if !supported { return Err("unsupported ESP32 antenna/grid/PPDU".into()); } - Ok(format!("ant{n_antennas}-bins{raw_bins}-ppdu{}-usable-drop2-v1", ppdu_type.to_byte())) + Ok(format!("ant{n_antennas}-bins{raw_bins}-ppdu{}-usable-drop2-linear-v2", ppdu_type.to_byte())) +} + +pub fn is_supported_frequency_hz(frequency_hz: u64) -> bool { + let mhz = frequency_hz / 1_000_000; + frequency_hz % 1_000_000 == 0 + && ((2412..=2484).contains(&mhz) || (4900..=5900).contains(&mhz) + || (5925..=7125).contains(&mhz)) } /// Preserves original signed int8 complex samples; uses only the existing @@ -105,9 +112,7 @@ pub fn canonicalize_esp32(input: NativePacketInput<'_>) -> Result) -> Result = native_frame.native_iq[2..].iter().map(|z| z.norm()).collect(); - let amplitudes = HardwareNormalizer::new().resample_to_canonical(&magnitudes); + let amplitudes = HardwareNormalizer::new().resample_magnitude_to_canonical(&magnitudes).map_err(|e|e.to_string())?; if amplitudes.len() != CANONICAL_PACKET_BINS || amplitudes.iter().any(|v| !v.is_finite() || *v < 0.0) { return Err("length-only magnitude projection produced an invalid feature vector".into()); } diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/native_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/native_tests.rs index 0854d5fe68..1a49f5a9c4 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/native_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/native_tests.rs @@ -23,16 +23,16 @@ fn ht64_packet_retains_original_signed_iq_phase_mask_and_unknown_geometry() { assert!(packet.native_frame.valid_mask.iter().all(|v|*v)); assert!(packet.native_frame.transmitter_pose.is_none() && packet.native_frame.receiver_pose.is_none() && packet.native_frame.antenna_geometry.is_empty()); - assert_eq!(packet.grid_id,"ant1-bins64-ppdu0-usable-drop2-v1"); + assert_eq!(packet.grid_id,"ant1-bins64-ppdu0-usable-drop2-linear-v2"); } #[test] fn he256_packet_uses_actual_frame_contract_and_existing_56_bin_projection() { let payload=bytes(256,6,8); let packet=canonicalize_esp32(input(&payload,256)).unwrap(); - let direct=HardwareNormalizer::new().resample_to_canonical(&vec![10.0;254]); + let direct=HardwareNormalizer::new().resample_magnitude_to_canonical(&vec![10.0;254]).unwrap(); assert_eq!(packet.amplitudes,direct); assert_eq!(packet.native_frame.native_shape,vec![1,256,1]); - assert_eq!(packet.grid_id,"ant1-bins256-ppdu1-usable-drop2-v1"); + assert_eq!(packet.grid_id,"ant1-bins256-ppdu1-usable-drop2-linear-v2"); assert_eq!(packet.amplitudes.len(),56); } #[test] @@ -116,7 +116,7 @@ fn actual_raw_grid_identity_survives_both_supported_families() { let bins=if ppdu==PpduType::HtLegacy {64} else {256};let payload=bytes(bins,3,4); let mut i=input(&payload,bins);i.ppdu_type=ppdu; let packet=canonicalize_esp32(i).unwrap(); - assert_eq!(packet.grid_id,format!("ant1-bins{bins}-ppdu{}-usable-drop2-v1",ppdu.to_byte())); + assert_eq!(packet.grid_id,format!("ant1-bins{bins}-ppdu{}-usable-drop2-linear-v2",ppdu.to_byte())); assert_eq!((packet.frequency_hz,packet.bandwidth_hz),(2_437_000_000,20_000_000)); } } @@ -146,3 +146,31 @@ fn source_record_requires_actual_receiver_rate_and_never_invents_one_hz() { assert_eq!(packet.receiver_sample_rate_hz,42.0); assert!(!packet.temporal_sample_rate_known); } + +#[test] +fn valid_native_spectral_nulls_are_admitted_without_negative_projected_magnitudes() { + for bins in [64, 256] { + let payload: Vec = (0..bins) + .flat_map(|bin| [if bin % 8 == 0 { 100 } else { 0 }, 0]) + .collect(); + let result = canonicalize_esp32(input(&payload, bins)); + assert!( + result.is_ok(), + "valid {bins}-bin spectral nulls must not be rejected by interpolation: {result:?}" + ); + let packet = result.unwrap(); + assert!(packet + .amplitudes + .iter() + .all(|v| v.is_finite() && *v >= 0.0 && *v <= 100.0)); + for (actual, original) in packet + .native_frame + .native_iq + .iter() + .zip(payload.chunks_exact(2)) + { + assert_eq!(actual.re, f64::from(original[0] as i8)); + assert_eq!(actual.im, 0.0); + } + } +} From e6fe333640fb1f821135c92a9d68967e27ce7513 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Sun, 1 Nov 2026 21:33:17 +0100 Subject: [PATCH 037/100] fix(learning): bind readiness and admission to exact reported PHY cohorts --- .../wifi-densepose-sensing-server/src/main.rs | 55 +++--- .../src/room_learner/lifecycle_capture.rs | 22 ++- .../src/room_learner_api.rs | 182 ++++-------------- .../src/room_learner_api_tests.rs | 117 +++++++++-- .../src/room_learner_ingress.rs | 164 ++++++++++++++++ 5 files changed, 350 insertions(+), 190 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner_ingress.rs diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index e71d6159ab..0f59f6444d 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -456,6 +456,29 @@ struct CalibrationGridEvidence { latest_age_s: f64, } +impl CalibrationGridEvidence { + fn from_arrivals(arrivals: &[std::time::Instant], now: std::time::Instant) -> Option { + let first = *arrivals.first()?; + let last = *arrivals.last()?; + let span_s = last.saturating_duration_since(first).as_secs_f64(); + let latest_age_s = now.saturating_duration_since(last).as_secs_f64(); + let mut max_gap_s = latest_age_s; + for pair in arrivals.windows(2) { + max_gap_s = max_gap_s.max( + pair[1] + .saturating_duration_since(pair[0]) + .as_secs_f64(), + ); + } + let rate_hz = if span_s > 0.0 { + arrivals.len().saturating_sub(1) as f64 / span_s + } else { + 0.0 + }; + Some(Self {sample_count:arrivals.len(),span_s,rate_hz,max_gap_s,latest_age_s}) + } +} + #[derive(Debug, Clone, Copy)] struct CalibrationGridBinding { source_node_id: u8, @@ -1641,35 +1664,7 @@ impl NodeState { } grouped .into_iter() - .filter_map(|(grid, arrivals)| { - let first = *arrivals.first()?; - let last = *arrivals.last()?; - let span_s = last.saturating_duration_since(first).as_secs_f64(); - let latest_age_s = now.saturating_duration_since(last).as_secs_f64(); - let mut max_gap_s = latest_age_s; - for pair in arrivals.windows(2) { - max_gap_s = max_gap_s.max( - pair[1] - .saturating_duration_since(pair[0]) - .as_secs_f64(), - ); - } - let rate_hz = if span_s > 0.0 { - arrivals.len().saturating_sub(1) as f64 / span_s - } else { - 0.0 - }; - Some(( - grid, - CalibrationGridEvidence { - sample_count: arrivals.len(), - span_s, - rate_hz, - max_gap_s, - latest_age_s, - }, - )) - }) + .filter_map(|(grid, arrivals)| CalibrationGridEvidence::from_arrivals(&arrivals, now).map(|evidence|(grid,evidence))) .collect() } @@ -2216,7 +2211,7 @@ struct AppStateInner { source_bank_shadow_enabled: bool, source_bank_shadow: Option, room_learning: Option>, - room_learning_sources: HashMap<(u8, u16, u8), room_learner_api::LiveSource>, + room_learning_sources: HashMap, /// Stable installation identity used only to bind local persisted state. installation_id: Option, /// Metadata for the privacy reduced empty room image, when available. diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs index b0235ae3eb..c472171923 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs @@ -109,6 +109,13 @@ impl RoomLearningRuntime { { return Err("native RF provenance does not match the selected source".into()); } + // Another PHY class from the same receiver is not this selected source + // view. Ignore it; required-cohort freshness still gates every decision. + if packet.frequency_hz != source.center_frequency_hz + || packet.bandwidth_hz != source.bandwidth_hz + { + return Ok(()); + } let result = capture.ingest( RawSourceFrame { source_id: packet.source_id, @@ -127,10 +134,17 @@ impl RoomLearningRuntime { } /// Bounded protected diagnostics only; never grants admission or advances a /// workflow revision. Ignore sources/grids outside the current context. - pub fn record_native_rejection(&self, source_id:u8, grid_id:&str, reason:&str) { - if let Ok(mut state)=self.inner.lock() { - if state.capture.as_ref().is_some_and(|c|c.context().sources.iter().any(|s|s.source_id==source_id && s.grid_id==grid_id)) { - state.input_error=Some(bounded(format!("Node {source_id} native input rejected: {reason}"))); + pub fn record_native_rejection(&self, source_id: u8, grid_id: &str, reason: &str) { + if let Ok(mut state) = self.inner.lock() { + if state.capture.as_ref().is_some_and(|c| { + c.context() + .sources + .iter() + .any(|s| s.source_id == source_id && s.grid_id == grid_id) + }) { + state.input_error = Some(bounded(format!( + "Node {source_id} native input rejected: {reason}" + ))); } } } diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs index f895625471..3eb175297e 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs @@ -10,95 +10,11 @@ use wifi_densepose_sensing_server::room_learner::{ training::FitConfig, }; -#[derive(Debug, Clone)] -pub(super) struct LiveSource { - frequency_hz: u64, - bandwidth_hz: Option, - seen: std::time::Instant, - receiver_rate_hz: f64, -} -fn grid_id(frame: &Esp32Frame) -> Option { - native::grid_id( - frame.n_antennas as usize, - frame.n_subcarriers as usize, - frame.ppdu_type, - ) - .ok() -} -fn bandwidth(frame: &Esp32Frame) -> Option { - if !frame.adr018_flags.bw40 && grid_id(frame).is_some() { - Some(20_000_000) - } else { - None - } -} -pub(super) fn observe( - state: &mut AppStateInner, - frame: &Esp32Frame, - packet: &[u8], - now: std::time::Instant, -) { - let Some(runtime) = state.room_learning.clone() else { - return; - }; - let key = ( - frame.node_id, - frame.n_subcarriers, - frame.ppdu_type.to_byte(), - ); - let rate = state - .room_learning_sources - .get(&key) - .map_or(0.0, |previous| { - let delta = now.saturating_duration_since(previous.seen).as_secs_f64(); - if delta <= 0.0 || delta > 5.0 { - 0.0 - } else if previous.receiver_rate_hz > 0.0 { - previous.receiver_rate_hz * 0.9 + (1.0 / delta) * 0.1 - } else { - 1.0 / delta - } - }); - let meta = LiveSource { - frequency_hz: u64::from(frame.freq_mhz) * 1_000_000, - bandwidth_hz: bandwidth(frame), - seen: now, - receiver_rate_hz: rate, - }; - if state.room_learning_sources.len() < 1_024 || state.room_learning_sources.contains_key(&key) { - state.room_learning_sources.insert(key, meta.clone()); - } - let Some(id) = grid_id(frame) else { - return; - }; - if !runtime.selects_grid(frame.node_id, &id) { - return; - } - let bytes = usize::from(frame.n_antennas) * usize::from(frame.n_subcarriers) * 2; - let Some(iq) = packet.get(20..20 + bytes) else { - return; - }; - let ns = runtime.receiver_now_ns(); - let projected = match native::canonicalize_esp32(NativePacketInput { - node_id: frame.node_id, - n_antennas: frame.n_antennas as usize, - raw_bins: frame.n_subcarriers as usize, - frequency_hz: meta.frequency_hz, - ppdu_type: frame.ppdu_type, - sequence: frame.sequence, - received_at_ns: ns, - receiver_sample_rate_hz: rate, - iq_bytes: iq, - flags: frame.adr018_flags, - provenance: CaptureProvenance::LiveReceiverObserved, - }) { - Ok(p) => p, - Err(reason) => { runtime.record_native_rejection(frame.node_id, &id, &reason); return; } - }; - // This is the real canonical RF record plus an explicitly derived magnitude - // view. No IQ, source-clock or coherent-phase claim is invented/persisted. - let _ = runtime.ingest_native(&projected, ns); -} +#[path = "room_learner_ingress.rs"] +mod ingress; +use ingress::native_candidates; +pub(super) use ingress::{observe, LiveSource, NativeSourceKey}; + fn failure(code: &str, error: impl ToString) -> Json { let code = if error.to_string() == "learner_state_changed" { "learner_state_changed" @@ -136,32 +52,21 @@ pub(super) struct ConfigureRequest { new_training: bool, } fn source_ready(state: &AppStateInner, spec: &SourceSpec, now: std::time::Instant) -> bool { - state.node_states.get(&spec.source_id).is_some_and(|node| { - node.calibration_grid_candidates(now) - .iter() - .any(|(grid, evidence)| { - native::grid_id( - 1, - grid.n_subcarriers as usize, - wifi_densepose_hardware::PpduType::from_byte(grid.ppdu_type), - ) - .ok() - .as_deref() - == Some(spec.grid_id.as_str()) - && evidence.span_s >= CALIBRATION_GRID_MIN_SPAN_S - && evidence.rate_hz >= CALIBRATION_GRID_MIN_RATE_HZ - && evidence.max_gap_s < CALIBRATION_GRID_MAX_GAP_S - && evidence.latest_age_s < 2.0 - && state - .room_learning_sources - .get(&(spec.source_id, grid.n_subcarriers, grid.ppdu_type)) - .is_some_and(|meta| { - now.saturating_duration_since(meta.seen).as_secs_f64() < 2.0 - && meta.frequency_hz == spec.center_frequency_hz - && meta.bandwidth_hz == Some(spec.bandwidth_hz) - }) - }) - }) && spec.feature_count == 56 + native_candidates(state, spec.source_id, now) + .iter() + .any(|(key, meta)| { + native::grid_id( + key.n_antennas as usize, + key.raw_bins as usize, + wifi_densepose_hardware::PpduType::from_byte(key.ppdu_type), + ) + .ok() + .as_deref() + == Some(spec.grid_id.as_str()) + && meta.frequency_hz == spec.center_frequency_hz + && meta.bandwidth_hz == Some(spec.bandwidth_hz) + }) + && spec.feature_count == 56 && spec.signal_kind == RfModality::WifiCsi && spec.device_id == format!("esp32-node-{}", spec.source_id) } @@ -229,46 +134,33 @@ pub(super) async fn configure( } let mut sources = Vec::new(); for &id in &request.source_node_ids { - let Some(node) = s.node_states.get(&id) else { + let Some(_) = s.node_states.get(&id) else { return failure( "learner_source_missing", format!("Node {id} is not receiving."), ); }; - let Some((grid, _)) = node.select_calibration_grid(now) else { - return failure( - "learner_source_unready", - format!("Node {id} needs an eligible stable raw CSI grid."), - ); + let Some((key, meta)) = + native_candidates(&s, id, now) + .into_iter() + .max_by(|(ka, a), (kb, b)| { + a.receiver_rate_hz + .total_cmp(&b.receiver_rate_hz) + .then(ka.raw_bins.cmp(&kb.raw_bins)) + }) + else { + return failure("learner_source_unready",format!("Node {id} needs a stable, supported exact native PHY cohort and a fresh eight-packet window.")); }; - let key = (id, grid.n_subcarriers, grid.ppdu_type); - let Some(meta) = s.room_learning_sources.get(&key) else { - return failure( - "learner_phy_missing", - format!("Node {id} lacks current PHY metadata."), - ); - }; - let Some(bandwidth_hz) = meta.bandwidth_hz else { - return failure( - "learner_phy_unsupported", - format!("Node {id} uses an unsupported bandwidth layout."), - ); - }; - if now.saturating_duration_since(meta.seen).as_secs_f64() > 2.0 - || !(2_000_000_000..=8_000_000_000).contains(&meta.frequency_hz) - { - return failure( - "learner_phy_stale", - format!("Node {id} PHY metadata is stale or invalid."), - ); - } + let bandwidth_hz = meta + .bandwidth_hz + .expect("eligible native cohort has supported bandwidth"); sources.push(SourceSpec { source_id: id, device_id: format!("esp32-node-{id}"), grid_id: match native::grid_id( - 1, - grid.n_subcarriers as usize, - wifi_densepose_hardware::PpduType::from_byte(grid.ppdu_type), + key.n_antennas as usize, + key.raw_bins as usize, + wifi_densepose_hardware::PpduType::from_byte(key.ppdu_type), ) { Ok(id) => id, Err(error) => return failure("learner_grid_unsupported", error), diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs index 3dde272369..0c56760da2 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs @@ -116,7 +116,10 @@ async fn configure_and_actual_native_ingress_share_grid_projection_and_device_id let sources = configured["status"]["sources"].as_array().unwrap(); assert_eq!(sources.len(), 3); assert_eq!(sources[0]["device_id"], "esp32-node-15"); - assert_eq!(sources[0]["grid_id"], "ant1-bins64-ppdu0-usable-drop2-v1"); + assert_eq!( + sources[0]["grid_id"], + "ant1-bins64-ppdu0-usable-drop2-linear-v2" + ); let begun=json_request(router.clone(),"/phase",Method::POST,serde_json::json!({ "trial_id":"synthetic-empty","session_id":"synthetic-train-session","role":"Train","label":"Empty", "duration_ms":12000,"conditions_confirmed":true})).await; @@ -281,15 +284,107 @@ async fn stale_http_start_is_rejected_after_another_client_changes_room() { #[tokio::test] async fn native_adapter_rejection_is_visible_without_admitting_or_labelling_packets() { - let (_dir,state)=fixture();let router=app(state.clone()); - let configured=json_request(router.clone(),"/configure",Method::POST,config()).await; - assert_eq!(configured["success"],true); - let mut packet=wire(15,121);packet[19]|=0x01; // Actual ADR-018 40 MHz bit. - let frame=parse_esp32_frame(&packet).unwrap(); - observe(&mut *state.write().await,&frame,&packet,std::time::Instant::now()); - let status=json_request_status(router).await; + let (_dir, state) = fixture(); + let router = app(state.clone()); + let configured = json_request(router.clone(), "/configure", Method::POST, config()).await; + assert_eq!(configured["success"], true); + let mut packet = wire(15, 121); + packet[19] |= 0x01; // Actual ADR-018 40 MHz bit. + let frame = parse_esp32_frame(&packet).unwrap(); + observe( + &mut *state.write().await, + &frame, + &packet, + std::time::Instant::now(), + ); + let status = json_request_status(router).await; assert_eq!(status["input_error"],"Node 15 native input rejected: native capture does not support the reported 40 MHz bandwidth"); - assert_eq!(status["capture"]["sources"][0]["admitted_packets"],0); - assert_eq!(status["mutation_guard"],configured["status"]["mutation_guard"]); - assert_eq!(status["physically_validated"],false); + assert_eq!(status["capture"]["sources"][0]["admitted_packets"], 0); + assert_eq!( + status["mutation_guard"], + configured["status"]["mutation_guard"] + ); + assert_eq!(status["physically_validated"], false); +} + +#[tokio::test] +async fn minority_supported_bandwidth_cannot_borrow_wide_grid_eligibility() { + let (_dir, state) = fixture(); + { + let mut inner = state.write().await; + inner.node_states.clear(); + inner.room_learning_sources.clear(); + let now = std::time::Instant::now(); + for row in 0..120 { + for node in [15, 16, 17] { + let mut packet = wire(node, row + 1); + if row < 119 { + packet[19] |= 0x01; + } + let frame = parse_esp32_frame(&packet).unwrap(); + let at = now - std::time::Duration::from_millis((120 - row) as u64 * 100); + inner + .node_states + .entry(node) + .or_insert_with(NodeState::new) + .observe_raw_grid(CsiGridKey::from_frame(&frame), at); + observe(&mut inner, &frame, &packet, at); + } + } + } + let result = json_request(app(state), "/configure", Method::POST, config()).await; + assert_eq!( + result["success"], false, + "one 20MHz packet must not inherit sustained 40MHz cadence: {result}" + ); + assert_eq!(result["error_code"], "learner_source_unready"); +} + +#[tokio::test] +async fn rare_foreign_frequency_neither_relabels_selection_nor_invalidates_a_phase() { + let (_dir, state) = fixture(); + let mut foreign = wire(15, 121); + foreign[8..12].copy_from_slice(&2462u32.to_le_bytes()); + let frame = parse_esp32_frame(&foreign).unwrap(); + observe( + &mut *state.write().await, + &frame, + &foreign, + std::time::Instant::now(), + ); + let router = app(state.clone()); + let configured = json_request(router.clone(), "/configure", Method::POST, config()).await; + assert_eq!(configured["success"], true, "{configured}"); + assert_eq!( + configured["status"]["sources"][0]["center_frequency_hz"], + 2_437_000_000u64 + ); + let begun = json_request( + router.clone(), + "/phase", + Method::POST, + serde_json::json!({ + "trial_id":"synthetic-phy","session_id":"synthetic-phy","role":"Train","label":"Empty", + "duration_ms":12000,"conditions_confirmed":true}), + ) + .await; + assert_eq!(begun["success"], true, "{begun}"); + observe( + &mut *state.write().await, + &frame, + &foreign, + std::time::Instant::now(), + ); + let packet = wire(15, 122); + let frame = parse_esp32_frame(&packet).unwrap(); + observe( + &mut *state.write().await, + &frame, + &packet, + std::time::Instant::now(), + ); + let status = json_request_status(router).await; + assert_eq!(status["capture"]["phase"]["trial_id"], "synthetic-phy"); + assert_eq!(status["capture"]["sources"][0]["admitted_packets"], 1); + assert_eq!(status["mutation_guard"], begun["status"]["mutation_guard"]); } diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_ingress.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_ingress.rs new file mode 100644 index 0000000000..399f9ccd50 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_ingress.rs @@ -0,0 +1,164 @@ +//! Header-only readiness is keyed to the exact PHY class admitted by the learner. +use super::*; +use std::collections::VecDeque; +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] +pub(crate) struct NativeSourceKey { + pub(super) node_id: u8, + pub(super) n_antennas: u8, + pub(super) raw_bins: u16, + pub(super) ppdu_type: u8, + pub(super) frequency_mhz: u16, + pub(super) bw40: bool, +} +impl NativeSourceKey { + fn from_frame(frame: &Esp32Frame) -> Self { + Self { + node_id: frame.node_id, + n_antennas: frame.n_antennas, + raw_bins: frame.n_subcarriers, + ppdu_type: frame.ppdu_type.to_byte(), + frequency_mhz: frame.freq_mhz, + bw40: frame.adr018_flags.bw40, + } + } +} +#[derive(Debug, Clone)] +pub(crate) struct LiveSource { + pub(super) frequency_hz: u64, + pub(super) bandwidth_hz: Option, + pub(super) receiver_rate_hz: f64, + arrivals: VecDeque, +} +impl LiveSource { + fn record(&mut self, now: std::time::Instant) { + while self + .arrivals + .front() + .is_some_and(|t| now.saturating_duration_since(*t) > CALIBRATION_GRID_EVIDENCE_WINDOW) + { + self.arrivals.pop_front(); + } + if self.arrivals.len() == CALIBRATION_GRID_EVIDENCE_CAPACITY { + self.arrivals.pop_front(); + } + if self.arrivals.back().is_none_or(|t| now > *t) { + self.arrivals.push_back(now); + } + self.receiver_rate_hz = self.evidence(now).map_or(0.0, |e| e.rate_hz); + } + fn evidence(&self, now: std::time::Instant) -> Option { + let arrivals: Vec<_> = self + .arrivals + .iter() + .copied() + .filter(|t| now.saturating_duration_since(*t) <= CALIBRATION_GRID_EVIDENCE_WINDOW) + .collect(); + CalibrationGridEvidence::from_arrivals(&arrivals, now) + } + fn eligible(&self, now: std::time::Instant) -> bool { + self.bandwidth_hz == Some(20_000_000) + && native::is_supported_frequency_hz(self.frequency_hz) + && self.evidence(now).is_some_and(|e| { + e.span_s >= CALIBRATION_GRID_MIN_SPAN_S + && e.rate_hz >= CALIBRATION_GRID_MIN_RATE_HZ + && e.max_gap_s < CALIBRATION_GRID_MAX_GAP_S + && e.latest_age_s < 2.0 + }) + && self + .arrivals + .iter() + .rev() + .nth(wifi_densepose_sensing_server::room_learner::capture::WINDOW_PACKETS - 1) + .is_some_and(|t| { + now.saturating_duration_since(*t).as_nanos() + <= u128::from( + wifi_densepose_sensing_server::room_learner::observation::MAX_AGE_NS, + ) + }) + } +} +pub(super) fn native_candidates( + state: &AppStateInner, + node: u8, + now: std::time::Instant, +) -> Vec<(&NativeSourceKey, &LiveSource)> { + state + .room_learning_sources + .iter() + .filter(|(key, meta)| { + key.node_id == node + && meta.eligible(now) + && native::grid_id( + key.n_antennas as usize, + key.raw_bins as usize, + wifi_densepose_hardware::PpduType::from_byte(key.ppdu_type), + ) + .is_ok() + }) + .collect() +} +pub(crate) fn observe( + state: &mut AppStateInner, + frame: &Esp32Frame, + packet: &[u8], + now: std::time::Instant, +) { + let Some(runtime) = state.room_learning.clone() else { + return; + }; + let key = NativeSourceKey::from_frame(frame); + if state.room_learning_sources.len() >= 1024 && !state.room_learning_sources.contains_key(&key) + { + return; + } + let id = native::grid_id( + frame.n_antennas as usize, + frame.n_subcarriers as usize, + frame.ppdu_type, + ) + .ok(); + let meta = state + .room_learning_sources + .entry(key) + .or_insert_with(|| LiveSource { + frequency_hz: u64::from(frame.freq_mhz) * 1_000_000, + bandwidth_hz: if !frame.adr018_flags.bw40 && id.is_some() { + Some(20_000_000) + } else { + None + }, + receiver_rate_hz: 0.0, + arrivals: VecDeque::new(), + }); + meta.record(now); + let (frequency_hz, rate) = (meta.frequency_hz, meta.receiver_rate_hz); + let Some(id) = id else { return }; + if !runtime.selects_grid(frame.node_id, &id) { + return; + } + let bytes = usize::from(frame.n_antennas) * usize::from(frame.n_subcarriers) * 2; + let Some(iq) = packet.get(20..20 + bytes) else { + return; + }; + let ns = runtime.receiver_now_ns(); + let projected = match native::canonicalize_esp32(NativePacketInput { + node_id: frame.node_id, + n_antennas: frame.n_antennas as usize, + raw_bins: frame.n_subcarriers as usize, + frequency_hz, + ppdu_type: frame.ppdu_type, + sequence: frame.sequence, + received_at_ns: ns, + receiver_sample_rate_hz: rate, + iq_bytes: iq, + flags: frame.adr018_flags, + provenance: CaptureProvenance::LiveReceiverObserved, + }) { + Ok(p) => p, + Err(reason) => { + runtime.record_native_rejection(frame.node_id, &id, &reason); + return; + } + }; + let _ = runtime.ingest_native(&projected, ns); +} From e2255e8ea528807e4ee293744f4837a0e8036c2c Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Sun, 1 Nov 2026 23:15:33 +0100 Subject: [PATCH 038/100] feat(learning): report condition metrics and honest cohort coverage --- .../src/room_learner/evaluation_report.rs | 369 ++++++++++++++++++ .../room_learner/evaluation_report_tests.rs | 234 +++++++++++ .../src/room_learner/training.rs | 46 +-- 3 files changed, 610 insertions(+), 39 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_report.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_report_tests.rs diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_report.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_report.rs new file mode 100644 index 0000000000..4c41bebecd --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_report.rs @@ -0,0 +1,369 @@ +//! Candidate cohort diagnostics. Declared trials are not verified independence. +use super::super::{ + model::{PresenceDecision, RoomModel, RoomPrediction}, + observation::{CaptureCohort, DataRole, RoomLabel, MAX_COHORTS}, +}; +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; + +pub(crate) const CONDITIONS: [RoomLabel; 4] = [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, +]; +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct BinaryConfusion { + pub true_empty: usize, + pub false_occupied: usize, + pub false_empty: usize, + pub true_occupied: usize, + pub abstained_empty: usize, + pub abstained_occupied: usize, + /// Missing probabilities are a subset of abstentions, never correct results. + pub missing_empty: usize, + pub missing_occupied: usize, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct EvaluationRates { + pub evaluated_cohort_coverage: Option, + pub unconditional_accuracy: Option, + pub scored_accuracy: Option, + /// Includes every positive cohort, including abstained/missing positives. + pub unconditional_sensitivity: Option, + pub scored_sensitivity: Option, + pub unconditional_specificity: Option, + pub scored_specificity: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ConditionEvaluation { + pub condition: RoomLabel, + pub samples: usize, + pub correct: usize, + pub abstained: usize, + pub scored: usize, + pub missing: usize, + pub binary_cross_entropy: Option, + pub binary_cross_entropy_samples: usize, + pub declared_trial_count: usize, + pub declared_session_count: usize, + pub confusion: BinaryConfusion, + pub rates: EvaluationRates, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct EvaluationReport { + pub samples: usize, + pub correct: usize, + pub abstained: usize, + /// Same numeric mean as before when probabilities exist; otherwise null. + pub binary_cross_entropy: Option, + pub binary_cross_entropy_samples: usize, + pub scored: usize, + pub missing: usize, + pub declared_trial_count: usize, + pub declared_session_count: usize, + pub confusion: BinaryConfusion, + pub rates: EvaluationRates, + pub per_condition: Vec, + pub independence_verified: bool, +} +fn ratio(n: usize, d: usize) -> Option { + if d == 0 { + None + } else { + Some(n as f32 / d as f32) + } +} +impl BinaryConfusion { + fn scored(&self) -> usize { + self.true_empty + self.false_occupied + self.false_empty + self.true_occupied + } + fn correct(&self) -> usize { + self.true_empty + self.true_occupied + } + fn abstained(&self) -> usize { + self.abstained_empty + self.abstained_occupied + } + fn missing(&self) -> usize { + self.missing_empty + self.missing_occupied + } + fn rates(&self) -> EvaluationRates { + let scored = self.scored(); + let samples = scored + self.abstained(); + EvaluationRates { + evaluated_cohort_coverage: ratio(scored, samples), + unconditional_accuracy: ratio(self.correct(), samples), + scored_accuracy: ratio(self.correct(), scored), + unconditional_sensitivity: ratio( + self.true_occupied, + self.true_occupied + self.false_empty + self.abstained_occupied, + ), + scored_sensitivity: ratio(self.true_occupied, self.true_occupied + self.false_empty), + unconditional_specificity: ratio( + self.true_empty, + self.true_empty + self.false_occupied + self.abstained_empty, + ), + scored_specificity: ratio(self.true_empty, self.true_empty + self.false_occupied), + } + } +} +#[derive(Default)] +struct Accumulator<'a> { + confusion: BinaryConfusion, + loss: f64, + probabilities: usize, + trials: HashSet<&'a str>, + sessions: HashSet<&'a str>, +} +impl<'a> Accumulator<'a> { + fn observe(&mut self, row: &'a CaptureCohort, p: &RoomPrediction) { + self.trials.insert(&row.trial_id); + self.sessions.insert(&row.session_id); + let occupied = matches!( + row.label, + RoomLabel::OccupiedStatic | RoomLabel::OccupiedMoving + ); + let probability = p + .occupied_probability + .filter(|v| v.is_finite() && (0.0..=1.0).contains(v)); + if let Some(v) = probability { + let v = v.clamp(1e-6, 1.0 - 1e-6) as f64; + self.loss -= if occupied { v.ln() } else { (1.0 - v).ln() }; + self.probabilities += 1; + } + let c = &mut self.confusion; + if p.decision == PresenceDecision::Abstain || probability.is_none() { + if occupied { + c.abstained_occupied += 1; + if probability.is_none() { + c.missing_occupied += 1; + } + } else { + c.abstained_empty += 1; + if probability.is_none() { + c.missing_empty += 1; + } + } + } else { + match (occupied, p.decision) { + (false, PresenceDecision::Empty) => c.true_empty += 1, + (false, PresenceDecision::Occupied) => c.false_occupied += 1, + (true, PresenceDecision::Empty) => c.false_empty += 1, + (true, PresenceDecision::Occupied) => c.true_occupied += 1, + _ => unreachable!(), + } + } + } + fn loss(&self) -> Option { + if self.probabilities == 0 { + None + } else { + Some((self.loss / self.probabilities as f64) as f32) + } + } + fn condition(&self, condition: RoomLabel) -> ConditionEvaluation { + ConditionEvaluation { + condition, + samples: self.confusion.scored() + self.confusion.abstained(), + correct: self.confusion.correct(), + abstained: self.confusion.abstained(), + scored: self.confusion.scored(), + missing: self.confusion.missing(), + binary_cross_entropy: self.loss(), + binary_cross_entropy_samples: self.probabilities, + declared_trial_count: self.trials.len(), + declared_session_count: self.sessions.len(), + confusion: self.confusion.clone(), + rates: self.confusion.rates(), + } + } +} +impl EvaluationReport { + pub(crate) fn compute(model: &RoomModel, rows: &[CaptureCohort]) -> Result { + model.validate()?; + if rows.is_empty() || rows.len() > MAX_COHORTS { + return Err("invalid bounded evaluation cohort count".into()); + } + let role = rows[0].role; + if !matches!(role, DataRole::Validation | DataRole::HeldOut) { + return Err("evaluation requires labeled validation or held-out cohorts".into()); + } + let mut ids = HashSet::new(); + let mut total = Accumulator::default(); + let mut conditions: [Accumulator; 4] = Default::default(); + let digest = model.context().digest(); + for row in rows { + if row.role != role || row.label == RoomLabel::Unknown { + return Err("evaluation requires one labeled non-inference role".into()); + } + if row.context_digest != digest { + return Err("evaluation context mismatch".into()); + } + if [&row.id, &row.trial_id, &row.session_id].iter().any(|id| { + id.is_empty() + || id.len() > 256 + || !id.is_ascii() + || id.trim() != id.as_str() + || id.chars().any(char::is_control) + }) || !ids.insert(&row.id) + { + return Err("invalid or duplicate evaluation cohort identity".into()); + } + let prediction = model.infer(row); + let condition = CONDITIONS + .iter() + .position(|label| *label == row.label) + .ok_or("unknown evaluation condition")?; + total.observe(row, &prediction); + conditions[condition].observe(row, &prediction); + } + let summary = total.condition(RoomLabel::Unknown); + let report = Self { + samples: summary.samples, + correct: summary.correct, + abstained: summary.abstained, + binary_cross_entropy: summary.binary_cross_entropy, + binary_cross_entropy_samples: summary.binary_cross_entropy_samples, + scored: summary.scored, + missing: summary.missing, + declared_trial_count: summary.declared_trial_count, + declared_session_count: summary.declared_session_count, + confusion: summary.confusion, + rates: summary.rates, + per_condition: conditions + .iter() + .zip(CONDITIONS) + .map(|(acc, label)| acc.condition(label)) + .collect(), + independence_verified: false, + }; + report.validate()?; + Ok(report) + } + pub(crate) fn validate(&self) -> Result<(), String> { + if !(1..=MAX_COHORTS).contains(&self.samples) + || self.independence_verified + || self.per_condition.len() != 4 + { + return Err("invalid evaluation report bounds".into()); + } + let mut sum = BinaryConfusion::default(); + let mut probability_count = 0usize; + let mut loss = 0.0; + for (condition, label) in self.per_condition.iter().zip(CONDITIONS) { + if condition.condition != label { + return Err("invalid evaluation conditions".into()); + } + validate_summary(condition)?; + let c = &condition.confusion; + if (matches!(label, RoomLabel::Empty | RoomLabel::AdjacentOnly) + && c.true_occupied + c.false_empty + c.abstained_occupied != 0) + || (matches!(label, RoomLabel::OccupiedStatic | RoomLabel::OccupiedMoving) + && c.true_empty + c.false_occupied + c.abstained_empty != 0) + { + return Err("condition confusion target mismatch".into()); + } + sum.true_empty += c.true_empty; + sum.false_occupied += c.false_occupied; + sum.false_empty += c.false_empty; + sum.true_occupied += c.true_occupied; + sum.abstained_empty += c.abstained_empty; + sum.abstained_occupied += c.abstained_occupied; + sum.missing_empty += c.missing_empty; + sum.missing_occupied += c.missing_occupied; + probability_count = probability_count + .checked_add(condition.binary_cross_entropy_samples) + .ok_or("evaluation probability count overflow")?; + loss += condition.binary_cross_entropy.unwrap_or(0.0) as f64 + * condition.binary_cross_entropy_samples as f64; + } + let summary = ConditionEvaluation { + condition: RoomLabel::Unknown, + samples: self.samples, + correct: self.correct, + abstained: self.abstained, + scored: self.scored, + missing: self.missing, + binary_cross_entropy: self.binary_cross_entropy, + binary_cross_entropy_samples: self.binary_cross_entropy_samples, + declared_trial_count: self.declared_trial_count, + declared_session_count: self.declared_session_count, + confusion: self.confusion.clone(), + rates: self.rates.clone(), + }; + validate_summary(&summary)?; + if sum != self.confusion + || probability_count != self.binary_cross_entropy_samples + || (probability_count > 0 + && (self.binary_cross_entropy.unwrap_or(0.0) as f64 + - loss / probability_count as f64) + .abs() + > 1e-4) + || !valid_declared_union( + self.declared_trial_count, + self.per_condition.iter().map(|c| c.declared_trial_count), + ) + || !valid_declared_union( + self.declared_session_count, + self.per_condition.iter().map(|c| c.declared_session_count), + ) + { + return Err("evaluation aggregate mismatch".into()); + } + Ok(()) + } +} +fn valid_declared_union(total: usize, mut counts: impl Iterator) -> bool { + let Some((maximum, sum)) = counts.try_fold((0usize, 0usize), |(maximum, sum), count| { + sum.checked_add(count).map(|sum| (maximum.max(count), sum)) + }) else { + return false; + }; + total >= maximum && total <= sum +} +fn validate_summary(s: &ConditionEvaluation) -> Result<(), String> { + let c = &s.confusion; + // Bound components before addition so malformed JSON cannot overflow usize. + if [ + c.true_empty, + c.false_occupied, + c.false_empty, + c.true_occupied, + c.abstained_empty, + c.abstained_occupied, + c.missing_empty, + c.missing_occupied, + ] + .iter() + .any(|n| *n > MAX_COHORTS) + || s.samples > MAX_COHORTS + || c.scored() + c.abstained() != s.samples + || c.correct() != s.correct + || c.scored() != s.scored + || c.abstained() != s.abstained + || c.missing() != s.missing + || c.missing_empty > c.abstained_empty + || c.missing_occupied > c.abstained_occupied + || s.samples.checked_sub(s.missing) != Some(s.binary_cross_entropy_samples) + || (s.samples > 0 && (s.declared_trial_count == 0 || s.declared_session_count == 0)) + || s.declared_trial_count > s.samples + || s.declared_session_count > s.samples + || c.rates() != s.rates + { + return Err("invalid evaluation cohort metrics".into()); + } + match (s.binary_cross_entropy_samples, s.binary_cross_entropy) { + (0, None) => {} + (n, Some(loss)) if n > 0 && loss.is_finite() && (0.0..=32.0).contains(&loss) => {} + _ => return Err("invalid evaluation probability loss".into()), + } + Ok(()) +} + +#[cfg(test)] +#[path = "evaluation_report_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_report_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_report_tests.rs new file mode 100644 index 0000000000..f1f2a0bf3c --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_report_tests.rs @@ -0,0 +1,234 @@ +//! SYNTHETIC metrics and denominator oracles; no sensing accuracy claim. +use super::*; +use crate::room_learner::training::{learning_tests::dataset, FitConfig}; +use std::sync::OnceLock; +fn model() -> RoomModel { + static MODEL: OnceLock = OnceLock::new(); + MODEL + .get_or_init(|| { + RoomModel::fit( + &dataset(8), + FitConfig { + epochs: 2, + ..FitConfig::default() + }, + ) + .unwrap() + .0 + }) + .clone() +} +#[test] +fn actual_evaluation_retains_legacy_fields_and_has_all_declared_conditions() { + let ds = dataset(8); + let report = model().evaluate(ds.held_out()).unwrap(); + assert_eq!(report.samples, 4); + assert_eq!(report.declared_trial_count, 4); + assert_eq!(report.per_condition.len(), 4); + assert_eq!(report.scored + report.abstained, report.samples); + assert_eq!(report.binary_cross_entropy_samples, 4); + assert!(report.binary_cross_entropy.unwrap().is_finite()); + assert!(!report.independence_verified); + report.validate().unwrap(); + let json = serde_json::to_value(&report).unwrap(); + for key in ["samples", "correct", "abstained", "binary_cross_entropy"] { + assert!(json.get(key).is_some()); + } + for condition in report.per_condition { + assert_eq!(condition.samples, 1); + assert_eq!(condition.declared_trial_count, 1); + } +} +#[test] +fn windows_are_not_distinct_declared_trials_and_missing_positives_reduce_unconditional_recall() { + let ds = dataset(8); + let mut windows = vec![ds.held_out()[2].clone(); 4]; + for (i, row) in windows.iter_mut().enumerate() { + row.id = format!("window-{i}"); + } + let report = model().evaluate(&windows).unwrap(); + assert_eq!(report.samples, 4); + assert_eq!(report.declared_trial_count, 1); + assert_eq!(report.declared_session_count, 1); + let mut missing = ds.held_out()[2].clone(); + missing.views.clear(); + let report = model().evaluate(&[missing]).unwrap(); + assert_eq!( + ( + report.samples, + report.correct, + report.abstained, + report.missing + ), + (1, 0, 1, 1) + ); + assert_eq!(report.rates.evaluated_cohort_coverage, Some(0.0)); + assert_eq!(report.rates.unconditional_sensitivity, Some(0.0)); + assert_eq!(report.rates.scored_sensitivity, None); + assert_eq!(report.binary_cross_entropy, None); + assert_eq!(report.binary_cross_entropy_samples, 0); + assert_eq!(report.rates.unconditional_specificity, None); +} +#[test] +fn abstentions_are_in_unconditional_sensitivity_and_absent_conditions_are_null() { + let ds = dataset(8); + let row = &ds.held_out()[2]; + let mut acc = Accumulator::default(); + acc.observe( + row, + &RoomPrediction { + decision: PresenceDecision::Occupied, + occupied_probability: Some(0.9), + reason: None, + physically_validated: false, + }, + ); + acc.observe( + row, + &RoomPrediction { + decision: PresenceDecision::Abstain, + occupied_probability: Some(0.5), + reason: None, + physically_validated: false, + }, + ); + acc.observe( + row, + &RoomPrediction { + decision: PresenceDecision::Abstain, + occupied_probability: None, + reason: None, + physically_validated: false, + }, + ); + let summary = acc.condition(RoomLabel::OccupiedStatic); + assert_eq!(summary.rates.unconditional_sensitivity, Some(1.0 / 3.0)); + assert_eq!(summary.rates.scored_sensitivity, Some(1.0)); + assert_eq!(summary.rates.evaluated_cohort_coverage, Some(1.0 / 3.0)); + assert_eq!(summary.binary_cross_entropy_samples, 2); + assert_eq!(summary.abstained, 2); + assert_eq!(summary.missing, 1); + let empty = Accumulator::default().condition(RoomLabel::Empty); + assert_eq!(empty.rates.evaluated_cohort_coverage, None); + assert_eq!(empty.rates.unconditional_accuracy, None); + assert_eq!(empty.binary_cross_entropy, None); +} +#[test] +fn unknown_train_inference_mixed_roles_wrong_context_and_duplicate_windows_are_rejected() { + let ds = dataset(8); + let model = model(); + assert!(model.evaluate(ds.train()).is_err()); + let mut row = ds.held_out()[0].clone(); + row.label = RoomLabel::Unknown; + assert!(model.evaluate(&[row.clone()]).is_err()); + row.role = DataRole::Inference; + assert!(model.evaluate(&[row]).is_err()); + assert!(model + .evaluate(&[ds.validation()[0].clone(), ds.held_out()[0].clone()]) + .is_err()); + assert!(model + .evaluate(&[ds.held_out()[0].clone(), ds.held_out()[0].clone()]) + .is_err()); + let mut wrong = ds.held_out()[0].clone(); + wrong.context_digest = "another-room".into(); + assert!(model.evaluate(&[wrong]).is_err()); +} +#[test] +fn tampered_metrics_nonfinite_losses_and_inflated_claims_fail_validation() { + let report = model().evaluate(dataset(8).held_out()).unwrap(); + let mut bad = report.clone(); + bad.correct = bad.samples + 1; + assert!(bad.validate().is_err()); + let mut bad = report.clone(); + bad.independence_verified = true; + assert!(bad.validate().is_err()); + let mut bad = report.clone(); + bad.binary_cross_entropy = Some(f32::NAN); + assert!(bad.validate().is_err()); + let mut bad = report.clone(); + bad.per_condition[0].rates.evaluated_cohort_coverage = Some(1.5); + assert!(bad.validate().is_err()); + let mut bad = report; + bad.confusion.true_empty = usize::MAX; + assert!(bad.validate().is_err()); +} + +fn sixteen_declared_windows() -> Vec { + let base = dataset(8); + let mut rows = Vec::new(); + for condition in base.held_out() { + for window in 0..4 { + let mut row = condition.clone(); + row.id = format!("{}-window-{window}", row.id); + rows.push(row); + } + } + rows +} +#[test] +fn impossible_declared_union_totals_are_rejected_without_rejecting_shared_trials() { + let rows = sixteen_declared_windows(); + let report = model().evaluate(&rows).unwrap(); + assert_eq!(report.samples, 16); + assert_eq!(report.declared_trial_count, 4); + assert_eq!( + report + .per_condition + .iter() + .map(|p| p.declared_trial_count) + .sum::(), + 4 + ); + let mut impossible = report.clone(); + impossible.declared_trial_count = 16; + assert!( + impossible.validate().is_err(), + "global trials cannot exceed per-condition union bound" + ); + let mut impossible = report; + impossible.declared_session_count = 16; + assert!( + impossible.validate().is_err(), + "global sessions cannot exceed per-condition union bound" + ); + let shared = rows + .into_iter() + .map(|mut row| { + row.trial_id = "shared-cross-condition-trial".into(); + row.session_id = "shared-cross-condition-session".into(); + row + }) + .collect::>(); + let shared = model().evaluate(&shared).unwrap(); + assert_eq!(shared.declared_trial_count, 1); + assert_eq!(shared.declared_session_count, 1); + shared.validate().unwrap(); +} +#[test] +fn nonmissing_abstention_probabilities_must_all_contribute_to_bce() { + let mut json = serde_json::to_value(model()).unwrap(); + for value in json["head_weights"].as_array_mut().unwrap() { + *value = serde_json::json!(0.0); + } + let uncertain = RoomModel::import(&serde_json::to_vec(&json).unwrap()).unwrap(); + let mut report = uncertain.evaluate(&sixteen_declared_windows()).unwrap(); + assert_eq!( + ( + report.samples, + report.scored, + report.missing, + report.binary_cross_entropy_samples + ), + (16, 0, 0, 16) + ); + report.binary_cross_entropy = None; + report.binary_cross_entropy_samples = 0; + for part in &mut report.per_condition { + part.binary_cross_entropy = None; + part.binary_cross_entropy_samples = 0; + } + assert!( + report.validate().is_err(), + "nonmissing uncertain probabilities cannot disappear from BCE" + ); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/training.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/training.rs index e1b1a0fcbd..1d38a6d22d 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/training.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/training.rs @@ -3,7 +3,7 @@ use super::model::{ center_empty_heads, extractor, initial_encoder, initial_projection_weights, EncoderConfig, RoomModel, SourceProfile, TrainingManifest, }; -use super::observation::{CaptureCohort, CaptureContext, DataRole, RoomLabel, ValidatedDataset}; +use super::observation::{CaptureCohort, CaptureContext, RoomLabel, ValidatedDataset}; use crate::embedding::info_nce_loss; use serde::{Deserialize, Serialize}; use std::cell::Cell; @@ -24,13 +24,11 @@ pub struct ObjectiveValues { pub contrastive: f32, pub supervised: f32, } -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct EvaluationReport { - pub samples: usize, - pub correct: usize, - pub abstained: usize, - pub binary_cross_entropy: f32, -} +#[path = "evaluation_report.rs"] +mod evaluation_report; +pub use evaluation_report::{ + BinaryConfusion, ConditionEvaluation, EvaluationRates, EvaluationReport, +}; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TrainingReport { pub initial: ObjectiveValues, @@ -365,37 +363,7 @@ impl RoomModel { } /// Explicit evaluation only; fit never reads HeldOut observations. pub fn evaluate(&self, cohorts: &[CaptureCohort]) -> Result { - if cohorts.is_empty() || cohorts.len() > 4096 { - return Err("invalid evaluation cohort count".into()); - } - let mut result = EvaluationReport { - samples: cohorts.len(), - correct: 0, - abstained: 0, - binary_cross_entropy: 0.0, - }; - for cohort in cohorts { - if cohort.role == DataRole::Inference || cohort.label == RoomLabel::Unknown { - return Err("evaluation requires a labeled non-inference cohort".into()); - } - let prediction = self.infer(cohort); - if prediction.decision == super::model::PresenceDecision::Abstain { - result.abstained += 1; - } - let probability = prediction - .occupied_probability - .ok_or_else(|| prediction.reason.unwrap_or("evaluation abstained".into()))?; - let target = occupied(cohort.label); - if prediction.decision != super::model::PresenceDecision::Abstain - && (probability > 0.5) == (target > 0.5) - { - result.correct += 1; - } - let p = probability.clamp(1.0e-6, 1.0 - 1.0e-6); - result.binary_cross_entropy -= target * p.ln() + (1.0 - target) * (1.0 - p).ln(); - } - result.binary_cross_entropy /= cohorts.len() as f32; - Ok(result) + EvaluationReport::compute(self, cohorts) } } From f9f6ec98102d4bf63237de2ca3be6d9606cb3acc Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Sun, 1 Nov 2026 23:15:33 +0100 Subject: [PATCH 039/100] feat(learning): persist immutable evaluation evidence without renewal --- .../src/room_learner/artifact.rs | 4 + .../src/room_learner/evaluation_index.rs | 94 ++++ .../room_learner/evaluation_index_tests.rs | 253 ++++++++++ .../src/room_learner/evaluation_receipt.rs | 171 +++++++ .../src/room_learner/evaluation_store.rs | 265 ++++++++++ .../room_learner/evaluation_store_tests.rs | 475 ++++++++++++++++++ 6 files changed, 1262 insertions(+) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_index.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_index_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_receipt.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_store.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_store_tests.rs diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs index c6da41acc8..862e952853 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs @@ -271,3 +271,7 @@ mod tests; #[path="artifact_catalog.rs"] mod catalog; pub use catalog::{CandidateCatalog,SavedCandidate}; + +#[path="evaluation_store.rs"] +mod evaluation_store; +pub use evaluation_store::EvaluationReceipt; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_index.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_index.rs new file mode 100644 index 0000000000..2e83c9208e --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_index.rs @@ -0,0 +1,94 @@ +//! Direct immutable lookup by exact model/role/dataset evidence identity. +use super::*; +const DIRECTORY: &str = "by-dataset"; +const SCHEMA_INDEX: &str = "ruview.room-learner.evaluation-evidence-index.v1"; +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct EvidenceIndex { + schema: String, + owner_scope: String, + installation_id: String, + room_digest: String, + context_digest: String, + model_sha256: String, + model_version: u32, + role: DataRole, + dataset_sha256: String, + receipt_sha256: String, +} +impl ArtifactStore { + pub(super) fn read_evidence_index( + &self, + directory: &Path, + data_sha: &str, + model: &RoomModel, + role: DataRole, + ) -> Result, String> { + if !hex_digest(data_sha) { + return Err("invalid evaluation dataset digest".into()); + } + let Some(index_dir) = private_subdirectory(directory, DIRECTORY, false)? else { + return Ok(None); + }; + let path = index_dir.join(format!("{data_sha}.json")); + if !regular_or_missing(&path)? { + return Ok(None); + } + let index: EvidenceIndex = + serde_json::from_slice(&private_read(&path).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + let context = model.context(); + if index.schema != SCHEMA_INDEX + || index.owner_scope != context.owner_scope + || index.installation_id != context.installation_id + || index.room_digest != context.room_digest + || index.context_digest != context.digest() + || index.model_sha256 != digest(&model.export()?) + || index.model_version != model.version + || index.role != role + || index.dataset_sha256 != data_sha + || !hex_digest(&index.receipt_sha256) + { + return Err("evaluation evidence index binding mismatch".into()); + } + let receipt = self.read_evaluation(directory, &index.receipt_sha256, model, role)?; + if receipt.dataset_sha256 != data_sha { + return Err("evaluation index references another dataset".into()); + } + Ok(Some(receipt)) + } + pub(super) fn save_evidence_index( + &self, + directory: &Path, + receipt: &EvaluationReceipt, + model: &RoomModel, + role: DataRole, + ) -> Result<(), String> { + receipt.validate(model, role)?; + let index_dir = private_subdirectory(directory, DIRECTORY, true)? + .ok_or("missing evaluation evidence index")?; + let index = EvidenceIndex { + schema: SCHEMA_INDEX.into(), + owner_scope: receipt.owner_scope.clone(), + installation_id: receipt.installation_id.clone(), + room_digest: receipt.room_digest.clone(), + context_digest: receipt.context_digest.clone(), + model_sha256: receipt.model_sha256.clone(), + model_version: receipt.model_version, + role, + dataset_sha256: receipt.dataset_sha256.clone(), + receipt_sha256: receipt.sha256()?, + }; + write_immutable( + &index_dir.join(format!("{}.json", receipt.dataset_sha256)), + &encode(&index)?, + )?; + let restored = self + .read_evidence_index(directory, &receipt.dataset_sha256, model, role)? + .ok_or("evaluation evidence index readback missing")?; + if restored.sha256()? != index.receipt_sha256 { + return Err("evaluation evidence index readback mismatch".into()); + } + Ok(()) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_index_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_index_tests.rs new file mode 100644 index 0000000000..d6540ebd8e --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_index_tests.rs @@ -0,0 +1,253 @@ +//! SYNTHETIC evidence identity across intervening evaluation datasets. +use super::*; +fn changed_evidence(ds: &ValidatedDataset) -> ValidatedDataset { + let mut rows = ds + .train() + .iter() + .chain(ds.validation()) + .chain(ds.held_out()) + .cloned() + .collect::>(); + rows.iter_mut() + .find(|r| r.role == DataRole::HeldOut) + .unwrap() + .views[0] + .samples[0][0] += 0.03; + ValidatedDataset::new(ds.context().clone(), rows).unwrap() +} +#[test] +fn alternating_datasets_reuse_original_evidence_identity_and_timestamp() { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let a = dataset(); + let b = changed_evidence(&a); + let first = store + .evaluate_candidate_at(&model, &a, DataRole::HeldOut, AT) + .unwrap(); + let path = receipt_path(&store, &model, &first); + let first_bytes = private_read(&path).unwrap(); + let middle = store + .evaluate_candidate_at(&model, &b, DataRole::HeldOut, AT + 1) + .unwrap(); + assert_ne!(first.dataset_sha256, middle.dataset_sha256); + // A direct evidence key must work beyond catalog-sized history and ignore + // unrelated unreadable entries; it must not scan or adopt another receipt. + let directory = store + .evaluation_directory(&model, DataRole::HeldOut, false) + .unwrap() + .unwrap(); + for n in 0..257 { + std::fs::write( + directory.join(format!("{n:064x}.json")), + b"unrelated unreadable history", + ) + .unwrap(); + } + let repeat = store + .evaluate_candidate_at(&model, &a, DataRole::HeldOut, AT + 2) + .unwrap(); + assert_eq!( + repeat.sha256().unwrap(), + first.sha256().unwrap(), + "A→B→A must not renew evidence A" + ); + assert_eq!(repeat.evaluated_at_unix_ms, AT); + assert_eq!(repeat.train_lineage_checked, first.train_lineage_checked); + assert_eq!(private_read(&path).unwrap(), first_bytes); + assert_eq!( + store + .load_latest_evaluation(&model, DataRole::HeldOut) + .unwrap(), + Some(first.clone()) + ); + let cold_a = ValidatedDataset::new(a.context().clone(), a.held_out().to_vec()).unwrap(); + let cold_repeat = store + .evaluate_candidate_at(&model, &cold_a, DataRole::HeldOut, AT + 3) + .unwrap(); + assert_eq!( + cold_repeat, first, + "historical lineage proof must not be rewritten on reuse" + ); +} + +fn index_path(store: &ArtifactStore, model: &RoomModel, receipt: &EvaluationReceipt) -> PathBuf { + store + .evaluation_directory(model, receipt.role, false) + .unwrap() + .unwrap() + .join("by-dataset") + .join(format!("{}.json", receipt.dataset_sha256)) +} +#[test] +fn corrupt_or_mismatched_index_never_overwrites_another_latest_receipt() { + for field in [ + "corrupt", + "owner_scope", + "installation_id", + "context_digest", + "model_sha256", + "role", + "dataset_sha256", + "receipt_sha256", + ] { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let a = dataset(); + let b = changed_evidence(&a); + let first = store + .evaluate_candidate_at(&model, &a, DataRole::HeldOut, AT) + .unwrap(); + let middle = store + .evaluate_candidate_at(&model, &b, DataRole::HeldOut, AT + 1) + .unwrap(); + let directory = store + .evaluation_directory(&model, DataRole::HeldOut, false) + .unwrap() + .unwrap(); + let pointer = directory.join(LATEST); + let before = private_read(&pointer).unwrap(); + let path = index_path(&store, &model, &first); + let mut bad: serde_json::Value = + serde_json::from_slice(&private_read(&path).unwrap()).unwrap(); + let changed = match field { + "owner_scope" => serde_json::json!("b".repeat(64)), + "installation_id" => serde_json::json!("wrong-install"), + "context_digest" | "model_sha256" => serde_json::json!("0".repeat(64)), + "role" => serde_json::json!("Validation"), + "dataset_sha256" => serde_json::json!(middle.dataset_sha256), + "receipt_sha256" => serde_json::json!(middle.sha256().unwrap()), + _ => serde_json::Value::Null, + }; + let bytes = if field == "corrupt" { + b"corrupt index".to_vec() + } else { + bad[field] = changed; + serde_json::to_vec(&bad).unwrap() + }; + atomic_private_write(&path, &bytes).unwrap(); + assert!( + store + .evaluate_candidate_at(&model, &a, DataRole::HeldOut, AT + 2) + .is_err(), + "index {field}" + ); + assert_eq!(private_read(&pointer).unwrap(), before); + assert_eq!(private_read(&path).unwrap(), bytes); + assert_eq!( + store + .load_latest_evaluation(&model, DataRole::HeldOut) + .unwrap(), + Some(middle) + ); + assert!(store.load_candidate(model.context()).unwrap().is_some()); + } +} +#[cfg(unix)] +#[test] +fn evidence_index_symlink_permission_and_size_errors_preserve_latest_and_victims() { + use std::os::unix::fs::{symlink, PermissionsExt}; + for failure in ["symlink", "permission", "oversized"] { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let a = dataset(); + let b = changed_evidence(&a); + let first = store + .evaluate_candidate_at(&model, &a, DataRole::HeldOut, AT) + .unwrap(); + store + .evaluate_candidate_at(&model, &b, DataRole::HeldOut, AT + 1) + .unwrap(); + let path = index_path(&store, &model, &first); + let directory = store + .evaluation_directory(&model, DataRole::HeldOut, false) + .unwrap() + .unwrap(); + let before = private_read(&directory.join(LATEST)).unwrap(); + let victim = temp.path().join("victim"); + atomic_private_write(&victim, b"preserved private victim").unwrap(); + match failure { + "symlink" => { + std::fs::rename(&path, path.with_extension("preserved")).unwrap(); + symlink(&victim, &path).unwrap(); + } + "permission" => { + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap() + } + _ => atomic_private_write(&path, &vec![b' '; LIMIT + 1]).unwrap(), + } + assert!(store + .evaluate_candidate_at(&model, &a, DataRole::HeldOut, AT + 2) + .is_err()); + assert_eq!(private_read(&directory.join(LATEST)).unwrap(), before); + assert_eq!(private_read(&victim).unwrap(), b"preserved private victim"); + assert!(store.load_candidate(model.context()).unwrap().is_some()); + } +} +#[test] +fn blocked_new_dataset_index_fails_without_publishing_or_replacing_existing_evidence() { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let a = dataset(); + let b = changed_evidence(&a); + let first = store + .evaluate_candidate_at(&model, &a, DataRole::HeldOut, AT) + .unwrap(); + let directory = store + .evaluation_directory(&model, DataRole::HeldOut, false) + .unwrap() + .unwrap(); + let pointer = directory.join(LATEST); + let before = private_read(&pointer).unwrap(); + let old_receipt = private_read(&receipt_path(&store, &model, &first)).unwrap(); + let blocked = directory + .join("by-dataset") + .join(format!("{}.json", data_sha256(b.held_out()).unwrap())); + std::fs::create_dir(&blocked).unwrap(); + assert!(store + .evaluate_candidate_at(&model, &b, DataRole::HeldOut, AT + 1) + .is_err()); + assert_eq!(private_read(&pointer).unwrap(), before); + assert_eq!( + private_read(&receipt_path(&store, &model, &first)).unwrap(), + old_receipt + ); + assert!(blocked.is_dir()); + assert_eq!( + store + .load_latest_evaluation(&model, DataRole::HeldOut) + .unwrap(), + Some(first) + ); +} + +#[test] +fn persisted_evidence_index_recovers_original_receipt_after_missing_latest_pointer() { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let a = dataset(); + let original = store + .evaluate_candidate_at(&model, &a, DataRole::HeldOut, AT) + .unwrap(); + let directory = store + .evaluation_directory(&model, DataRole::HeldOut, false) + .unwrap() + .unwrap(); + let previous_pointer = directory.join("latest-before-interruption.json"); + std::fs::rename(directory.join(LATEST), &previous_pointer).unwrap(); + let bytes = private_read(&previous_pointer).unwrap(); + let cold = open_store(temp.path()); + let loaded = cold.load_candidate(model.context()).unwrap().unwrap(); + let retry = cold + .evaluate_candidate_at(&loaded, &a, DataRole::HeldOut, AT + 50) + .unwrap(); + assert_eq!( + retry, original, + "retry must not renew evidence after a partial publication" + ); + assert_eq!(private_read(&previous_pointer).unwrap(), bytes); + assert_eq!( + cold.load_latest_evaluation(&loaded, DataRole::HeldOut) + .unwrap(), + Some(original) + ); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_receipt.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_receipt.rs new file mode 100644 index 0000000000..04ac7d44f7 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_receipt.rs @@ -0,0 +1,171 @@ +//! Hashed metadata only. Input provenance/declared trials do not grant adoption. +use super::super::{digest, encode, hex_digest}; +use crate::room_learner::{ + model::RoomModel, + observation::{CaptureCohort, CaptureProvenance, ClockKind, DataRole}, + training::EvaluationReport, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::io::{self, Write}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct EvaluationReceipt { + pub schema: String, + pub owner_scope: String, + pub installation_id: String, + pub room_digest: String, + pub context_digest: String, + pub model_sha256: String, + pub model_version: u32, + pub role: DataRole, + pub dataset_sha256: String, + pub evaluated_at_unix_ms: u64, + pub provenance: Vec, + pub clock_claims: Vec, + pub measurement_clock_available: bool, + /// True only when the retained Train set matched this model's manifest and + /// the dataset validator checked role separation. Not statistical independence. + pub train_lineage_checked: bool, + pub training_data_sha256: String, + pub report: EvaluationReport, + pub independence_verified: bool, + pub physically_validated: bool, + pub adoption_authorized: bool, +} +struct HashWriter(Sha256); +impl Write for HashWriter { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.update(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } +} +pub(super) fn data_sha256(rows: &[CaptureCohort]) -> Result { + let mut writer = HashWriter(Sha256::new()); + serde_json::to_writer(&mut writer, rows).map_err(|e| e.to_string())?; + Ok(format!("{:x}", writer.0.finalize())) +} +fn provenance_rank(p: &CaptureProvenance) -> u8 { + match p { + CaptureProvenance::Synthetic => 0, + CaptureProvenance::CapturedReplay => 1, + CaptureProvenance::LiveReceiverObserved => 2, + } +} +fn clock_rank(p: &ClockKind) -> u8 { + match p { + ClockKind::Unknown => 0, + ClockKind::ReceiverOnly => 1, + ClockKind::SourceMeasurement => 2, + } +} +impl EvaluationReceipt { + pub fn sha256(&self) -> Result { + Ok(digest(&encode(self)?)) + } + pub(super) fn new( + model: &RoomModel, + rows: &[CaptureCohort], + role: DataRole, + at: u64, + train_checked: bool, + ) -> Result { + if rows.is_empty() + || rows.iter().any(|r| { + r.role != role + || r.views.iter().any(|v| { + v.provenance != r.provenance + || (v.signal_kind == ruview_unified::RfModality::Synthetic + && v.provenance != CaptureProvenance::Synthetic) + }) + }) + { + return Err("invalid evaluation role/provenance".into()); + } + let mut provenance: Vec<_> = rows.iter().map(|r| r.provenance).collect(); + provenance.sort_by_key(provenance_rank); + provenance.dedup(); + let mut clocks: Vec<_> = rows + .iter() + .flat_map(|r| r.views.iter().map(|v| v.clock.kind)) + .collect(); + clocks.sort_by_key(clock_rank); + clocks.dedup(); + let available = !provenance.contains(&CaptureProvenance::Synthetic) + && clocks == [ClockKind::SourceMeasurement]; + let c = model.context(); + let result = Self { + schema: "ruview.room-learner.evaluation.v1".into(), + owner_scope: c.owner_scope.clone(), + installation_id: c.installation_id.clone(), + room_digest: c.room_digest.clone(), + context_digest: c.digest(), + model_sha256: digest(&model.export()?), + model_version: model.version, + role, + dataset_sha256: data_sha256(rows)?, + evaluated_at_unix_ms: at, + provenance, + clock_claims: clocks, + measurement_clock_available: available, + train_lineage_checked: train_checked, + training_data_sha256: model.training_manifest().training_data_sha256.clone(), + report: model.evaluate(rows)?, + independence_verified: false, + physically_validated: false, + adoption_authorized: false, + }; + result.validate(model, role)?; + Ok(result) + } + pub(super) fn validate(&self, model: &RoomModel, role: DataRole) -> Result<(), String> { + let c = model.context(); + if self.schema != "ruview.room-learner.evaluation.v1" + || self.owner_scope != c.owner_scope + || self.installation_id != c.installation_id + || self.room_digest != c.room_digest + || self.context_digest != c.digest() + || self.model_sha256 != digest(&model.export()?) + || self.model_version != model.version + || self.role != role + || !matches!(role, DataRole::Validation | DataRole::HeldOut) + || !hex_digest(&self.dataset_sha256) + || self.evaluated_at_unix_ms == 0 + || self.evaluated_at_unix_ms > 253_402_300_799_999 + || self.training_data_sha256 != model.training_manifest().training_data_sha256 + || self.independence_verified + || self.physically_validated + || self.adoption_authorized + { + return Err("evaluation receipt/candidate binding mismatch".into()); + } + if self.provenance.is_empty() + || self.provenance.len() > 3 + || self + .provenance + .windows(2) + .any(|w| provenance_rank(&w[0]) >= provenance_rank(&w[1])) + || self.clock_claims.is_empty() + || self.clock_claims.len() > 3 + || self + .clock_claims + .windows(2) + .any(|w| clock_rank(&w[0]) >= clock_rank(&w[1])) + || self.measurement_clock_available + != (!self.provenance.contains(&CaptureProvenance::Synthetic) + && self.clock_claims == [ClockKind::SourceMeasurement]) + || (c + .sources + .iter() + .any(|s| s.signal_kind == ruview_unified::RfModality::Synthetic) + && self.provenance != [CaptureProvenance::Synthetic]) + { + return Err("invalid evaluation provenance/clock claims".into()); + } + self.report.validate() + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_store.rs new file mode 100644 index 0000000000..57173b1275 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_store.rs @@ -0,0 +1,265 @@ +//! Per-model, per-role immutable evaluation history; no activation authority. +use super::*; +use crate::room_learner::observation::{DataRole, ValidatedDataset}; +#[path = "evaluation_receipt.rs"] +mod receipt; +use receipt::data_sha256; +#[path = "evaluation_index.rs"] +mod index; +pub use receipt::EvaluationReceipt; +const SCHEMA: &str = "ruview.room-learner.evaluation-artifact.v1"; +const POINTER_SCHEMA_EVAL: &str = "ruview.room-learner.evaluation-pointer.v1"; +const LATEST: &str = "latest-v1.json"; +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct EvaluationEnvelope { + schema: String, + receipt_sha256: String, + receipt: EvaluationReceipt, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct EvaluationPointer { + schema: String, + receipt_sha256: String, + model_sha256: String, + context_digest: String, + role: DataRole, +} +fn role_directory(role: DataRole) -> Result<&'static str, String> { + match role { + DataRole::Validation => Ok("validation"), + DataRole::HeldOut => Ok("held-out"), + _ => Err("evaluation only accepts validation or held-out data".into()), + } +} +fn private_subdirectory( + parent: &Path, + name: &str, + create: bool, +) -> Result, String> { + let path = parent.join(name); + match std::fs::symlink_metadata(&path) { + Ok(m) if !m.is_dir() || m.file_type().is_symlink() => { + return Err("invalid private evaluation directory".into()) + } + Err(e) if e.kind() == io::ErrorKind::NotFound && !create => return Ok(None), + Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e.to_string()), + _ => {} + } + private_dir(&path).map_err(|e| e.to_string())?; + Ok(Some(path)) +} +fn write_immutable(path: &Path, bytes: &[u8]) -> Result<(), String> { + if regular_or_missing(path)? { + return if private_read(path).map_err(|e| e.to_string())? == bytes { + Ok(()) + } else { + Err("immutable evaluation collision/corruption".into()) + }; + } + let directory = path.parent().ok_or("missing evaluation parent")?; + let staging = directory.join(format!("evaluation-stage-{}", crate::companion::secret())); + atomic_private_write(&staging, bytes).map_err(|e| e.to_string())?; + let installed = std::fs::hard_link(&staging, path); + let cleanup = std::fs::remove_file(&staging); + match installed { + Ok(()) => {} + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => { + if private_read(path).map_err(|e| e.to_string())? != bytes { + return Err("immutable evaluation collision/corruption".into()); + } + } + Err(e) => return Err(e.to_string()), + } + cleanup.map_err(|e| e.to_string())?; + std::fs::File::open(directory) + .and_then(|f| f.sync_all()) + .map_err(|e| e.to_string()) +} +impl ArtifactStore { + fn evaluation_directory( + &self, + model: &RoomModel, + role: DataRole, + create: bool, + ) -> Result, String> { + let role_name = role_directory(role)?; + self.validate_context(model.context())?; + let Some(context) = self.context_dir(model.context(), false)? else { + return Err("persist the frozen candidate before evaluation".into()); + }; + let bytes = model.export()?; + let model_receipt = self.receipt_for(model, &bytes); + self.read_model(&context, &model_receipt, model.context())?; + let Some(evaluations) = private_subdirectory(&context, "evaluations", create)? else { + return Ok(None); + }; + let Some(model_dir) = + private_subdirectory(&evaluations, &model_receipt.model_sha256, create)? + else { + return Ok(None); + }; + private_subdirectory(&model_dir, role_name, create) + } + fn read_evaluation( + &self, + directory: &Path, + key: &str, + model: &RoomModel, + role: DataRole, + ) -> Result { + if !hex_digest(key) { + return Err("invalid evaluation digest".into()); + } + let bytes = + private_read(&directory.join(format!("{key}.json"))).map_err(|e| e.to_string())?; + let envelope: EvaluationEnvelope = + serde_json::from_slice(&bytes).map_err(|e| e.to_string())?; + if envelope.schema != SCHEMA + || envelope.receipt_sha256 != key + || envelope.receipt.sha256()? != key + { + return Err("evaluation integrity mismatch".into()); + } + envelope.receipt.validate(model, role)?; + Ok(envelope.receipt) + } + fn read_latest_evaluation( + &self, + directory: &Path, + model: &RoomModel, + role: DataRole, + ) -> Result, String> { + let path = directory.join(LATEST); + if !regular_or_missing(&path)? { + return Ok(None); + } + let pointer: EvaluationPointer = + serde_json::from_slice(&private_read(&path).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + if pointer.schema != POINTER_SCHEMA_EVAL + || pointer.context_digest != model.context().digest() + || pointer.role != role + || pointer.model_sha256 != digest(&model.export()?) + || !hex_digest(&pointer.receipt_sha256) + { + return Err("evaluation pointer/candidate mismatch".into()); + } + let receipt = self.read_evaluation(directory, &pointer.receipt_sha256, model, role)?; + let indexed = self + .read_evidence_index(directory, &receipt.dataset_sha256, model, role)? + .ok_or("evaluation evidence index missing")?; + if indexed.sha256()? != pointer.receipt_sha256 { + return Err("evaluation latest pointer differs from original evidence".into()); + } + Ok(Some(receipt)) + } + /// Compute metrics from a validated role split, then persist. No supplied + /// report, physical-qualification claim or caller-controlled filename enters. + pub fn evaluate_candidate( + &self, + model: &RoomModel, + dataset: &ValidatedDataset, + role: DataRole, + ) -> Result { + let at = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_err(|_| "evaluation clock unavailable")? + .as_millis() + .try_into() + .map_err(|_| "evaluation clock outside bounds")?; + self.evaluate_candidate_at(model, dataset, role, at) + } + fn evaluate_candidate_at( + &self, + model: &RoomModel, + dataset: &ValidatedDataset, + role: DataRole, + at: u64, + ) -> Result { + role_directory(role)?; + if dataset.context() != model.context() { + return Err("evaluation dataset/candidate context mismatch".into()); + } + let train_checked = !dataset.train().is_empty(); + if train_checked + && data_sha256(dataset.train())? != model.training_manifest().training_data_sha256 + { + return Err("retained Train set differs from the frozen candidate".into()); + } + let rows = if role == DataRole::HeldOut { + dataset.held_out() + } else { + dataset.validation() + }; + if rows.is_empty() { + return Err("no cohorts for the selected evaluation role".into()); + } + let directory = self + .evaluation_directory(model, role, true)? + .ok_or("missing evaluation directory")?; + let previous = self.read_latest_evaluation(&directory, model, role)?; + let data_sha = data_sha256(rows)?; + if let Some(original) = self.read_evidence_index(&directory, &data_sha, model, role)? { + // Selecting old evidence can move latest, but never changes its + // immutable receipt identity, original timestamp or lineage proof. + if previous + .as_ref() + .map(EvaluationReceipt::sha256) + .transpose()? + .as_deref() + != Some(original.sha256()?.as_str()) + { + self.publish_evaluation_pointer(&directory, &original)?; + } + return Ok(original); + } + let receipt = EvaluationReceipt::new(model, rows, role, at, train_checked)?; + let key = receipt.sha256()?; + let envelope = EvaluationEnvelope { + schema: SCHEMA.into(), + receipt_sha256: key.clone(), + receipt: receipt.clone(), + }; + let bytes = encode(&envelope)?; + write_immutable(&directory.join(format!("{key}.json")), &bytes)?; + self.read_evaluation(&directory, &key, model, role)?; + self.save_evidence_index(&directory, &receipt, model, role)?; + self.publish_evaluation_pointer(&directory, &receipt)?; + Ok(receipt) + } + fn publish_evaluation_pointer( + &self, + directory: &Path, + receipt: &EvaluationReceipt, + ) -> Result<(), String> { + let pointer = EvaluationPointer { + schema: POINTER_SCHEMA_EVAL.into(), + receipt_sha256: receipt.sha256()?, + model_sha256: receipt.model_sha256.clone(), + context_digest: receipt.context_digest.clone(), + role: receipt.role, + }; + let bytes = encode(&pointer)?; + atomic_private_write(&directory.join(LATEST), &bytes).map_err(|e| e.to_string())?; + if private_read(&directory.join(LATEST)).map_err(|e| e.to_string())? != bytes { + return Err("evaluation pointer readback failed".into()); + } + Ok(()) + } + pub fn load_latest_evaluation( + &self, + model: &RoomModel, + role: DataRole, + ) -> Result, String> { + let Some(directory) = self.evaluation_directory(model, role, false)? else { + return Ok(None); + }; + self.read_latest_evaluation(&directory, model, role) + } +} + +#[cfg(test)] +#[path = "evaluation_store_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_store_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_store_tests.rs new file mode 100644 index 0000000000..af96f0f2b9 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_store_tests.rs @@ -0,0 +1,475 @@ +//! SYNTHETIC RF through real fitting, private storage and cold-load boundaries. +use super::*; +use crate::room_learner::{ + observation::CaptureCohort, + training::{learning_tests, FitConfig}, +}; +use std::sync::OnceLock; +const AT: u64 = 1_800_000_000_000; +fn dataset() -> ValidatedDataset { + let base = learning_tests::dataset(8); + let mut context = base.context().clone(); + context.owner_scope = "a".repeat(64); + let mut rows = base + .train() + .iter() + .chain(base.validation()) + .chain(base.held_out()) + .cloned() + .collect::>(); + for row in &mut rows { + row.context_digest = context.digest(); + } + ValidatedDataset::new(context, rows).unwrap() +} +fn model() -> RoomModel { + static MODEL: OnceLock = OnceLock::new(); + MODEL + .get_or_init(|| { + RoomModel::fit( + &dataset(), + FitConfig { + epochs: 2, + ..FitConfig::default() + }, + ) + .unwrap() + .0 + }) + .clone() +} +fn open_store(root: &Path) -> ArtifactStore { + ArtifactStore::new( + &root.join("a".repeat(64)).join("SensingServer"), + &"a".repeat(64), + "synthetic-install", + ) + .unwrap() +} +fn saved(root: &Path) -> (ArtifactStore, RoomModel) { + let store = open_store(root); + let model = model(); + store.save_candidate(&model).unwrap(); + (store, model) +} +fn receipt_path(store: &ArtifactStore, model: &RoomModel, r: &EvaluationReceipt) -> PathBuf { + store + .evaluation_directory(model, r.role, false) + .unwrap() + .unwrap() + .join(format!("{}.json", r.sha256().unwrap())) +} +#[test] +fn real_fit_evaluate_private_store_cold_load_preserves_bound_metadata_without_raw_rows() { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let ds = dataset(); + let before = model.export().unwrap(); + assert!(store + .load_latest_evaluation(&model, DataRole::HeldOut) + .unwrap() + .is_none()); + let report = store + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT) + .unwrap(); + assert_eq!(model.export().unwrap(), before); + assert_eq!(report.dataset_sha256, data_sha256(ds.held_out()).unwrap()); + assert_eq!(report.model_sha256, digest(&before)); + assert!(report.train_lineage_checked); + assert!( + !report.adoption_authorized + && !report.physically_validated + && !report.independence_verified + && !report.measurement_clock_available + ); + let cold = open_store(temp.path()); + let restored = cold.load_candidate(model.context()).unwrap().unwrap(); + assert_eq!( + cold.load_latest_evaluation(&restored, DataRole::HeldOut) + .unwrap(), + Some(report.clone()) + ); + let text = + String::from_utf8(private_read(&receipt_path(&cold, &restored, &report)).unwrap()).unwrap(); + for raw in [ + "frame_ids", + "sequence_ids", + "trial_id\"", + "views\"", + "samples\":[", + "amplitudes", + ] { + assert!(!text.contains(raw), "raw field {raw}"); + } + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + std::fs::metadata(receipt_path(&cold, &restored, &report)) + .unwrap() + .permissions() + .mode() + & 0o777, + 0o600 + ); + } +} +#[test] +fn repeated_receipt_identity_is_idempotent_and_new_receipts_retain_history() { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let ds = dataset(); + let first = store + .evaluate_candidate_at(&model, &ds, DataRole::Validation, AT) + .unwrap(); + let path = receipt_path(&store, &model, &first); + let bytes = private_read(&path).unwrap(); + #[cfg(unix)] + let inode = { + use std::os::unix::fs::MetadataExt; + std::fs::metadata(&path).unwrap().ino() + }; + let repeat = store + .evaluate_candidate_at(&model, &ds, DataRole::Validation, AT) + .unwrap(); + assert_eq!(first, repeat); + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + assert_eq!(std::fs::metadata(&path).unwrap().ino(), inode); + } + let unchanged = store + .evaluate_candidate_at(&model, &ds, DataRole::Validation, AT + 1) + .unwrap(); + assert_eq!( + unchanged, first, + "identical evidence must not renew its timestamp" + ); + let mut changed_rows = ds + .train() + .iter() + .chain(ds.validation()) + .chain(ds.held_out()) + .cloned() + .collect::>(); + let changed = changed_rows + .iter_mut() + .find(|r| r.role == DataRole::Validation) + .unwrap(); + changed.views[0].samples[0][0] += 0.01; + let changed = ValidatedDataset::new(ds.context().clone(), changed_rows).unwrap(); + let next = store + .evaluate_candidate_at(&model, &changed, DataRole::Validation, AT + 2) + .unwrap(); + assert_ne!(first.sha256().unwrap(), next.sha256().unwrap()); + assert_eq!(private_read(&path).unwrap(), bytes); + assert_eq!( + store + .load_latest_evaluation(&model, DataRole::Validation) + .unwrap(), + Some(next) + ); +} +#[test] +fn frozen_model_role_and_context_isolation_and_missing_retained_train_are_explicit() { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let ds = dataset(); + for role in [DataRole::Train, DataRole::Inference] { + assert!(store.evaluate_candidate_at(&model, &ds, role, AT).is_err()); + } + let mut rows = ds.train().to_vec(); + rows[0].views[0].samples[0][0] += 0.01; + rows.extend_from_slice(ds.held_out()); + let changed = ValidatedDataset::new(ds.context().clone(), rows).unwrap(); + assert!(store + .evaluate_candidate_at(&model, &changed, DataRole::HeldOut, AT) + .unwrap_err() + .contains("Train set")); + let cold_rows = ValidatedDataset::new(ds.context().clone(), ds.held_out().to_vec()).unwrap(); + let result = store + .evaluate_candidate_at(&model, &cold_rows, DataRole::HeldOut, AT) + .unwrap(); + assert!(!result.train_lineage_checked); + let second = RoomModel::fit( + &ds, + FitConfig { + epochs: 3, + ..FitConfig::default() + }, + ) + .unwrap() + .0; + store.save_candidate(&second).unwrap(); + assert!(store + .load_latest_evaluation(&second, DataRole::HeldOut) + .unwrap() + .is_none()); + assert_eq!( + store + .load_latest_evaluation(&model, DataRole::HeldOut) + .unwrap(), + Some(result) + ); + let wrong = ArtifactStore::new( + &temp.path().join("b".repeat(64)).join("SensingServer"), + &"b".repeat(64), + "synthetic-install", + ) + .unwrap(); + assert!(wrong + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT) + .is_err()); + assert!(wrong + .load_latest_evaluation(&model, DataRole::HeldOut) + .is_err()); + let unsaved = tempfile::tempdir().unwrap(); + assert!(open_store(unsaved.path()) + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT) + .is_err()); +} +#[test] +fn role_leakage_and_repeat_cohort_ids_cannot_reach_receipt_generation() { + let ds = dataset(); + let mut rows = ds.train().to_vec(); + let mut overlap = rows[0].clone(); + overlap.role = DataRole::HeldOut; + overlap.id = "renamed-eval".into(); + overlap.trial_id = "renamed-trial".into(); + overlap.session_id = "renamed-session".into(); + rows.push(overlap); + assert!(ValidatedDataset::new(ds.context().clone(), rows) + .unwrap_err() + .contains("frame overlap")); + let duplicate: Vec = vec![ds.held_out()[0].clone(), ds.held_out()[0].clone()]; + assert!(ValidatedDataset::new(ds.context().clone(), duplicate).is_err()); +} +#[test] +fn corrupt_receipt_or_pointer_is_retained_without_discarding_the_model_or_overwriting_history() { + for pointer in [true, false] { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let ds = dataset(); + let receipt = store + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT) + .unwrap(); + let directory = store + .evaluation_directory(&model, DataRole::HeldOut, false) + .unwrap() + .unwrap(); + let path = if pointer { + directory.join(LATEST) + } else { + receipt_path(&store, &model, &receipt) + }; + atomic_private_write(&path, b"corrupt evaluation").unwrap(); + assert!(store + .load_latest_evaluation(&model, DataRole::HeldOut) + .is_err()); + assert!(store + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT + 1) + .is_err()); + assert_eq!(private_read(&path).unwrap(), b"corrupt evaluation"); + assert_eq!( + store + .load_candidate(model.context()) + .unwrap() + .unwrap() + .export() + .unwrap(), + model.export().unwrap() + ); + } +} +#[test] +fn self_consistent_false_authority_or_wrong_model_role_metrics_are_rejected() { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let ds = dataset(); + let original = store + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT) + .unwrap(); + let directory = store + .evaluation_directory(&model, DataRole::HeldOut, false) + .unwrap() + .unwrap(); + type Mutant = fn(&mut EvaluationReceipt); + let mutants: [Mutant; 8] = [ + |r: &mut EvaluationReceipt| r.adoption_authorized = true, + |r: &mut EvaluationReceipt| r.physically_validated = true, + |r: &mut EvaluationReceipt| r.independence_verified = true, + |r: &mut EvaluationReceipt| r.report.correct = 999, + |r: &mut EvaluationReceipt| r.model_sha256 = "0".repeat(64), + |r: &mut EvaluationReceipt| r.role = DataRole::Train, + |r: &mut EvaluationReceipt| { + r.provenance = + vec![crate::room_learner::observation::CaptureProvenance::LiveReceiverObserved] + }, + |r: &mut EvaluationReceipt| r.training_data_sha256 = "0".repeat(64), + ]; + for mutate in mutants { + let mut bad = original.clone(); + mutate(&mut bad); + let key = bad.sha256().unwrap(); + let envelope = EvaluationEnvelope { + schema: SCHEMA.into(), + receipt_sha256: key.clone(), + receipt: bad, + }; + write_immutable( + &directory.join(format!("{key}.json")), + &encode(&envelope).unwrap(), + ) + .unwrap(); + let pointer = EvaluationPointer { + schema: POINTER_SCHEMA_EVAL.into(), + receipt_sha256: key, + model_sha256: original.model_sha256.clone(), + context_digest: original.context_digest.clone(), + role: DataRole::HeldOut, + }; + atomic_private_write(&directory.join(LATEST), &encode(&pointer).unwrap()).unwrap(); + assert!(store + .load_latest_evaluation(&model, DataRole::HeldOut) + .is_err()); + } +} +#[cfg(unix)] +#[test] +fn symlinks_and_insecure_files_never_replace_private_evidence_or_touch_victims() { + use std::os::unix::fs::{symlink, PermissionsExt}; + for target in ["receipt", "pointer", "role", "model", "evaluations"] { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let ds = dataset(); + let receipt = store + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT) + .unwrap(); + let directory = store + .evaluation_directory(&model, DataRole::HeldOut, false) + .unwrap() + .unwrap(); + let path = match target { + "receipt" => receipt_path(&store, &model, &receipt), + "pointer" => directory.join(LATEST), + "role" => directory.clone(), + "model" => directory.parent().unwrap().to_path_buf(), + _ => directory.parent().unwrap().parent().unwrap().to_path_buf(), + }; + std::fs::rename(&path, path.with_extension("preserved")).unwrap(); + let victim = temp.path().join("victim"); + atomic_private_write(&victim, b"victim bytes").unwrap(); + symlink(&victim, &path).unwrap(); + assert!(store + .load_latest_evaluation(&model, DataRole::HeldOut) + .is_err()); + assert!(store + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT + 1) + .is_err()); + assert_eq!(private_read(&victim).unwrap(), b"victim bytes"); + } + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let ds = dataset(); + let receipt = store + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT) + .unwrap(); + let path = receipt_path(&store, &model, &receipt); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + assert!(store + .load_latest_evaluation(&model, DataRole::HeldOut) + .is_err()); +} +#[test] +fn disk_obstacles_and_oversized_pointers_leave_frozen_model_and_receipts_untouched() { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let ds = dataset(); + let context = store.context_dir(model.context(), false).unwrap().unwrap(); + let obstacle = context.join("evaluations"); + atomic_private_write(&obstacle, b"existing bytes").unwrap(); + assert!(store + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT) + .is_err()); + assert_eq!(private_read(&obstacle).unwrap(), b"existing bytes"); + assert!(store.load_candidate(model.context()).unwrap().is_some()); + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let receipt = store + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT) + .unwrap(); + let path = receipt_path(&store, &model, &receipt); + let before = private_read(&path).unwrap(); + let pointer = store + .evaluation_directory(&model, DataRole::HeldOut, false) + .unwrap() + .unwrap() + .join(LATEST); + let oversized = vec![b' '; LIMIT + 1]; + atomic_private_write(&pointer, &oversized).unwrap(); + assert!(store + .load_latest_evaluation(&model, DataRole::HeldOut) + .is_err()); + assert!(store + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT + 1) + .is_err()); + assert_eq!(private_read(&path).unwrap(), before); + assert_eq!(std::fs::read(pointer).unwrap(), oversized); +} + +#[test] +fn preexisting_immutable_collision_preserves_the_previous_pointer_and_all_evidence() { + let temp = tempfile::tempdir().unwrap(); + let (store, model) = saved(temp.path()); + let ds = dataset(); + let original = store + .evaluate_candidate_at(&model, &ds, DataRole::HeldOut, AT) + .unwrap(); + let directory = store + .evaluation_directory(&model, DataRole::HeldOut, false) + .unwrap() + .unwrap(); + let pointer_before = private_read(&directory.join(LATEST)).unwrap(); + let original_before = private_read(&receipt_path(&store, &model, &original)).unwrap(); + let mut rows = ds + .train() + .iter() + .chain(ds.validation()) + .chain(ds.held_out()) + .cloned() + .collect::>(); + rows.iter_mut() + .find(|r| r.role == DataRole::HeldOut) + .unwrap() + .views[0] + .samples[0][0] += 0.02; + let changed = ValidatedDataset::new(ds.context().clone(), rows).unwrap(); + let next = EvaluationReceipt::new(&model, changed.held_out(), DataRole::HeldOut, AT + 1, true) + .unwrap(); + let collision = directory.join(format!("{}.json", next.sha256().unwrap())); + atomic_private_write(&collision, b"existing immutable evidence").unwrap(); + assert!(store + .evaluate_candidate_at(&model, &changed, DataRole::HeldOut, AT + 1) + .is_err()); + assert_eq!( + private_read(&collision).unwrap(), + b"existing immutable evidence" + ); + assert_eq!( + private_read(&directory.join(LATEST)).unwrap(), + pointer_before + ); + assert_eq!( + private_read(&receipt_path(&store, &model, &original)).unwrap(), + original_before + ); + assert_eq!( + store + .load_latest_evaluation(&model, DataRole::HeldOut) + .unwrap(), + Some(original) + ); +} + +#[path = "evaluation_index_tests.rs"] +mod index_tests; From e6af4ba58d16f5ca1faffa4f264d76d61d8afff7 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Sun, 1 Nov 2026 23:15:33 +0100 Subject: [PATCH 040/100] feat(learning): restore matching historical candidate evaluations --- .../src/room_learner/lifecycle.rs | 56 ++---- .../src/room_learner/lifecycle_evaluation.rs | 80 ++++++++ .../lifecycle_evaluation_tests.rs | 181 ++++++++++++++++++ .../src/room_learner/lifecycle_tests.rs | 16 +- 4 files changed, 289 insertions(+), 44 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_evaluation.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_evaluation_tests.rs diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs index f6cd607d90..9b20eb01a4 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs @@ -5,7 +5,7 @@ use super::{ capture::{CaptureSession, PhaseReceipt, PhaseRequest, RawSourceFrame}, model::{RoomModel, RoomPrediction}, observation::{CaptureContext, DataRole, RoomLabel}, - training::{EvaluationReport, FitConfig, TrainingReport}, + training::{FitConfig, TrainingReport}, }; use serde::Serialize; use serde_json::{json, Value}; @@ -52,6 +52,7 @@ struct Inner { frozen_training: bool, job: Job, last_evaluation: Option, + evaluation_warning: Option, input_error: Option, } pub struct RoomLearningRuntime { @@ -98,6 +99,7 @@ impl RoomLearningRuntime { frozen_training: false, job: Job::default(), last_evaluation: None, + evaluation_warning: None, input_error: None, }), }) @@ -141,6 +143,14 @@ impl RoomLearningRuntime { "no stored candidate for this exact context; start a new training capture".into(), ); } + let (restored_evaluation, evaluation_warning) = if !new_training { + restored + .as_ref() + .map(|model| self.restored_evaluation(model)) + .unwrap_or((None, None)) + } else { + (None, None) + }; state.generation = state .generation .checked_add(1) @@ -157,7 +167,8 @@ impl RoomLearningRuntime { if state.model.is_some() { state.job.phase = JobPhase::Ready; } - state.last_evaluation = None; + state.last_evaluation = restored_evaluation; + state.evaluation_warning = evaluation_warning; state.input_error = None; Ok(()) } @@ -259,38 +270,6 @@ impl RoomLearningRuntime { .live_cohort(now_ns)?; Ok((self.guard_for(&state), model.infer(&cohort))) } - /// Evaluation does not mutate model parameters and never promotes physical - /// authority. The held-out result is a candidate diagnostic, not a certificate. - pub fn evaluate(&self, expected: &MutationGuard, role: DataRole) -> Result { - if !matches!(role, DataRole::Validation | DataRole::HeldOut) { - return Err("evaluate only validation or held-out data".into()); - } - let mut state = self.command_state(expected)?; - if busy(&state.job) { - return Err("wait for the frozen candidate".into()); - } - let model = state.model.as_ref().ok_or("no frozen candidate")?; - let capture = state.capture.as_ref().ok_or("no capture context")?; - if capture.phase_active() { - return Err("finish the current phase before evaluation".into()); - } - let dataset = capture.validated_dataset()?; - let rows = if role == DataRole::HeldOut { - dataset.held_out() - } else { - dataset.validation() - }; - if rows.is_empty() { - return Err("no independent rows for the selected evaluation role".into()); - } - let report: EvaluationReport = model.evaluate(rows)?; - let result = json!({"schema":"ruview.room-learner.evaluation.v1","role":role, - "context_digest":capture.context().digest(),"report":report, - "measurement_clock_available":dataset.measurement_clock_available(), - "physically_validated":false,"adoption_authorized":false}); - state.last_evaluation = Some(result.clone()); - Ok(result) - } pub fn status(&self, now_ns: u64) -> Value { let Ok(state) = self.inner.lock() else { return json!({"enabled":true,"state":"unavailable"}); @@ -335,7 +314,7 @@ impl RoomLearningRuntime { "training_frozen":state.frozen_training,"training_defaults":FitConfig::for_source_count(state.capture.as_ref().map_or(0, |c| c.context().sources.len())),"training_report":state.job.report, "error":state.job.error,"input_error":state.input_error, "candidate":state.receipt,"candidate_loaded":state.model.is_some(), - "last_evaluation":state.last_evaluation,"physically_validated":false, + "last_evaluation":state.last_evaluation,"evaluation_warning":state.evaluation_warning,"physically_validated":false, "control_detector_changed":false,"raw_observations_persisted":false, "limitations":["receiver-observed magnitude path; no coherent phase claim", "legacy UDP transmitter identity is unverified","independent teacher registry not configured", @@ -353,3 +332,10 @@ mod capture_control; #[path = "command.rs"] mod command; pub use command::{MutationGuard, PhaseIdentity}; + +#[path = "lifecycle_evaluation.rs"] +mod evaluation_control; + +#[cfg(test)] +#[path = "lifecycle_evaluation_tests.rs"] +mod evaluation_tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_evaluation.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_evaluation.rs new file mode 100644 index 0000000000..266152cff6 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_evaluation.rs @@ -0,0 +1,80 @@ +//! Durable diagnostics only. Historical lineage checks are not current identity. +use super::*; +use crate::room_learner::artifact::EvaluationReceipt; +fn presentation( + receipt: &EvaluationReceipt, + restored: bool, + current_train_checked: bool, +) -> Result { + Ok( + json!({"schema":"ruview.room-learner.evaluation.v1","role":receipt.role, + "context_digest":receipt.context_digest,"model_sha256":receipt.model_sha256,"report":receipt.report, + "measurement_clock_available":receipt.measurement_clock_available, + "receipt":receipt,"receipt_sha256":receipt.sha256()?,"durable":true,"restored":restored, + "train_lineage_checked":current_train_checked, + "historical_train_lineage_checked":receipt.train_lineage_checked, + "independence_verified":false,"physically_validated":false,"adoption_authorized":false}), + ) +} +impl RoomLearningRuntime { + pub(super) fn restored_evaluation(&self, model: &RoomModel) -> (Option, Option) { + let mut receipts = Vec::new(); + for role in [DataRole::Validation, DataRole::HeldOut] { + match self.store.load_latest_evaluation(model,role) { + Ok(Some(receipt))=>receipts.push(receipt),Ok(None)=>{}, + Err(_)=>return (None,Some("Saved evaluation could not be verified for this candidate; evidence files were retained.".into())), + } + } + let Some(receipt) = receipts.into_iter().max_by_key(|r| { + ( + r.evaluated_at_unix_ms, + usize::from(r.role == DataRole::HeldOut), + ) + }) else { + return (None, None); + }; + match presentation(&receipt, true, false) { + Ok(value) => (Some(value), None), + Err(_) => ( + None, + Some( + "Saved evaluation metadata could not be read; the candidate is still loaded." + .into(), + ), + ), + } + } + /// Only a frozen private candidate and a validated role split can produce a + /// durable diagnostic. This never adopts the candidate or changes control. + pub fn evaluate(&self, expected: &MutationGuard, role: DataRole) -> Result { + if !matches!(role, DataRole::Validation | DataRole::HeldOut) { + return Err("evaluate only validation or held-out data".into()); + } + let mut state = self.command_state(expected)?; + if busy(&state.job) || !state.frozen_training { + return Err("wait for the frozen candidate".into()); + } + let model = state.model.as_ref().ok_or("no frozen candidate")?.clone(); + let capture = state.capture.as_ref().ok_or("no capture context")?; + if capture.phase_active() { + return Err("finish the current phase before evaluation".into()); + } + let dataset = capture.validated_dataset()?; + let result = self + .store + .evaluate_candidate(&model, &dataset, role) + .and_then(|receipt| presentation(&receipt, false, !dataset.train().is_empty())); + match result { + Ok(value) => { + state.last_evaluation = Some(value.clone()); + state.evaluation_warning = None; + Ok(value) + } + Err(error) => { + state.last_evaluation = None; + state.evaluation_warning=Some("Evaluation did not complete durably; previous evidence and the candidate were retained.".into()); + Err(bounded(error)) + } + } + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_evaluation_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_evaluation_tests.rs new file mode 100644 index 0000000000..45da3a35fc --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_evaluation_tests.rs @@ -0,0 +1,181 @@ +//! SYNTHETIC managed lifecycle, private persistence and cold diagnostics. +use super::tests::{context, finished, phase, runtime, train_phases, START}; +use super::*; +async fn fitted(root: &Path) -> Arc { + let rt = runtime(root); + rt.configure( + &rt.mutation_guard().unwrap(), + context(), + "evaluation-fixture".into(), + true, + ) + .unwrap(); + train_phases(&rt); + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 2, + ..FitConfig::default() + }, + ) + .unwrap(); + let status = finished(&rt).await; + assert_eq!(status["state"], "ready", "{status}"); + assert!( + status["training_report"]["backbone_delta_l2"] + .as_f64() + .unwrap() + > 0.0 + ); + rt +} +fn evaluation_directory(root: &Path, report: &Value) -> std::path::PathBuf { + root.join("a".repeat(64)) + .join("SensingServer/room-learner") + .join(report["context_digest"].as_str().unwrap()) + .join("evaluations") + .join(report["model_sha256"].as_str().unwrap()) + .join("held-out") +} +#[tokio::test] +async fn frozen_capture_evaluate_persist_cold_resume_restores_exact_diagnostic_without_current_train_claim( +) { + let dir = tempfile::tempdir().unwrap(); + let rt = fitted(dir.path()).await; + phase(&rt, DataRole::HeldOut, RoomLabel::Empty, 4); + phase(&rt, DataRole::HeldOut, RoomLabel::OccupiedStatic, 5); + let result = rt + .evaluate(&rt.mutation_guard().unwrap(), DataRole::HeldOut) + .unwrap(); + assert_eq!(result["durable"], true); + assert_eq!(result["train_lineage_checked"], true); + assert_eq!(result["report"]["samples"], 8); + assert_eq!(result["report"]["declared_trial_count"], 2); + assert_eq!(result["report"]["independence_verified"], false); + assert_eq!(result["adoption_authorized"], false); + let key = result["receipt_sha256"].as_str().unwrap(); + assert!(evaluation_directory(dir.path(), &result) + .join(format!("{key}.json")) + .is_file()); + let cold = runtime(dir.path()); + cold.configure( + &cold.mutation_guard().unwrap(), + context(), + "cold-evaluation".into(), + false, + ) + .unwrap(); + let status = cold.status(START); + let restored = &status["last_evaluation"]; + assert_eq!(status["candidate_loaded"], true); + assert!(status["evaluation_warning"].is_null()); + assert_eq!(restored["receipt_sha256"], result["receipt_sha256"]); + assert_eq!(restored["report"], result["report"]); + assert_eq!(restored["restored"], true); + assert_eq!(restored["train_lineage_checked"], false); + assert_eq!(restored["historical_train_lineage_checked"], true); + assert_eq!(status["physically_validated"], false); + assert_eq!(status["control_detector_changed"], false); + phase(&cold, DataRole::HeldOut, RoomLabel::Empty, 6); + let fresh = cold + .evaluate(&cold.mutation_guard().unwrap(), DataRole::HeldOut) + .unwrap(); + assert_eq!(fresh["train_lineage_checked"], false); + for raw in ["frame_ids", "sequence_ids", "\"views\"", "\"samples\":["] { + assert!(!serde_json::to_string(&cold.status(START)) + .unwrap() + .contains(raw)); + } +} +#[tokio::test] +async fn corrupted_report_is_retained_and_warned_while_the_valid_candidate_remains_loaded() { + let dir = tempfile::tempdir().unwrap(); + let rt = fitted(dir.path()).await; + phase(&rt, DataRole::HeldOut, RoomLabel::Empty, 4); + let result = rt + .evaluate(&rt.mutation_guard().unwrap(), DataRole::HeldOut) + .unwrap(); + let pointer = evaluation_directory(dir.path(), &result).join("latest-v1.json"); + crate::companion::atomic_private_write(&pointer, b"broken evaluation pointer").unwrap(); + let cold = runtime(dir.path()); + cold.configure( + &cold.mutation_guard().unwrap(), + context(), + "corrupt-eval".into(), + false, + ) + .unwrap(); + let status = cold.status(START); + assert_eq!(status["candidate_loaded"], true); + assert_eq!(status["state"], "ready"); + assert!(status["last_evaluation"].is_null()); + let warning = status["evaluation_warning"].as_str().unwrap(); + assert!(!warning.is_empty() && warning.len() <= 256); + assert_eq!( + std::fs::read(&pointer).unwrap(), + b"broken evaluation pointer" + ); + assert_eq!(status["physically_validated"], false); + phase(&cold, DataRole::HeldOut, RoomLabel::Empty, 6); + assert!(cold + .evaluate(&cold.mutation_guard().unwrap(), DataRole::HeldOut) + .is_err()); + let failed = cold.status(START); + assert!(failed["last_evaluation"].is_null()); + assert!(failed["evaluation_warning"].is_string()); + assert_eq!(failed["candidate_loaded"], true); +} +#[tokio::test] +async fn persistence_failure_never_claims_a_durable_evaluation_or_changes_control() { + let dir = tempfile::tempdir().unwrap(); + let rt = fitted(dir.path()).await; + phase(&rt, DataRole::HeldOut, RoomLabel::Empty, 4); + let root = dir + .path() + .join("a".repeat(64)) + .join("SensingServer/room-learner") + .join(context().digest()); + crate::companion::atomic_private_write(&root.join("evaluations"), b"filesystem obstacle") + .unwrap(); + assert!(rt + .evaluate(&rt.mutation_guard().unwrap(), DataRole::HeldOut) + .is_err()); + let status = rt.status(START); + assert!(status["last_evaluation"].is_null()); + assert!(status["evaluation_warning"].is_string()); + assert_eq!(status["candidate_loaded"], true); + assert_eq!(status["control_detector_changed"], false); + assert!(rt + .evaluate(&rt.mutation_guard().unwrap(), DataRole::Train) + .is_err()); +} + +pub(super) fn assert_no_raw_observation_arrays(value: &Value) { + match value { + Value::Object(object) => { + for (key, child) in object { + assert!( + !([ + "samples", + "views", + "frame_ids", + "sequence_ids", + "amplitudes", + "native_iq", + "valid_mask" + ] + .contains(&key.as_str()) + && child.is_array()), + "raw observation array: {key}" + ); + assert_no_raw_observation_arrays(child); + } + } + Value::Array(items) => { + for item in items { + assert_no_raw_observation_arrays(item); + } + } + _ => {} + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs index 9854cec8bc..9eac2f729c 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs @@ -6,8 +6,8 @@ use crate::room_learner::{ }; use ruview_unified::RfModality; use std::time::{Duration, Instant}; -const START: u64 = 1_000_000_000_000; -fn context() -> CaptureContext { +pub(super) const START: u64 = 1_000_000_000_000; +pub(super) fn context() -> CaptureContext { CaptureContext { owner_scope: "a".repeat(64), installation_id: "synthetic-install".into(), @@ -29,7 +29,7 @@ fn context() -> CaptureContext { .collect(), } } -fn runtime(root: &Path) -> Arc { +pub(super) fn runtime(root: &Path) -> Arc { Arc::new( RoomLearningRuntime::new( &root.join("a".repeat(64)).join("SensingServer"), @@ -39,7 +39,7 @@ fn runtime(root: &Path) -> Arc { .unwrap(), ) } -fn phase(rt: &RoomLearningRuntime, role: DataRole, label: RoomLabel, index: u32) -> u64 { +pub(super) fn phase(rt: &RoomLearningRuntime, role: DataRole, label: RoomLabel, index: u32) -> u64 { let at = START + u64::from(index) * 13_000_000_000; rt.begin_phase( &rt.mutation_guard().unwrap(), @@ -98,7 +98,7 @@ fn phase(rt: &RoomLearningRuntime, role: DataRole, label: RoomLabel, index: u32) .unwrap(); at + MIN_PHASE_MS * 1_000_000 } -fn train_phases(rt: &RoomLearningRuntime) { +pub(super) fn train_phases(rt: &RoomLearningRuntime) { for (index, label) in [ RoomLabel::Empty, RoomLabel::OccupiedStatic, @@ -111,7 +111,7 @@ fn train_phases(rt: &RoomLearningRuntime) { phase(rt, DataRole::Train, label, index as u32); } } -async fn finished(rt: &RoomLearningRuntime) -> Value { +pub(super) async fn finished(rt: &RoomLearningRuntime) -> Value { let start = Instant::now(); loop { let s = rt.status(START + 60_000_000_000); @@ -209,9 +209,7 @@ async fn real_capture_train_private_store_cold_load_and_unlabelled_inference_com let (_, prediction) = cold.prediction(START + 100_080_000_000).unwrap(); assert!(prediction.occupied_probability.is_some()); assert!(!prediction.physically_validated); - assert!(!serde_json::to_string(&cold.status(START)) - .unwrap() - .contains("samples")); + super::evaluation_tests::assert_no_raw_observation_arrays(&cold.status(START)); } #[tokio::test] async fn cancellation_and_context_replacement_cannot_publish_a_late_model() { From 77e43804462bf97c0288df93942ad7868b83b0d3 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Sun, 1 Nov 2026 23:15:33 +0100 Subject: [PATCH 041/100] docs(adr): propose managed source-aware learner and evidence boundaries --- ...R-420-managed-source-aware-room-learner.md | 186 ++++++++++++++++++ docs/adr/README.md | 2 + 2 files changed, 188 insertions(+) create mode 100644 docs/adr/ADR-420-managed-source-aware-room-learner.md diff --git a/docs/adr/ADR-420-managed-source-aware-room-learner.md b/docs/adr/ADR-420-managed-source-aware-room-learner.md new file mode 100644 index 0000000000..7b9ca615f6 --- /dev/null +++ b/docs/adr/ADR-420-managed-source-aware-room-learner.md @@ -0,0 +1,186 @@ +# ADR-420: Managed source-aware room learning and durable candidate evidence + +**Status:** Proposed +**Date:** 2026-11-01 +**Deciders:** RuView maintainers and app delivery owners; privacy and reference-sensor owners for teacher admission + +## Context + +The managed application has an installation-bound selected-source background +detector. Its live feature path also receives multiple nodes. Receiving those +nodes does not establish that a room model was trained on each source. + +Existing implementations supply substantial building blocks: + +- [ADR-024](ADR-024-contrastive-csi-embedding-model.md) describes contrastive CSI + representations; AETHER supplies the Rust transformer and projection stack. +- [ADR-027](ADR-027-cross-environment-domain-generalization.md) covers domain + generalization. Geometry/domain adaptation requires measured inputs and a + composed training path. +- [ADR-151](ADR-151-room-calibration-specialist-training.md) defines room + specialists and statistical multi-node late fusion. It remains a useful + comparative baseline with its own enrollment contract. +- [ADR-008](ADR-008-distributed-consensus-multi-ap.md) concerns distributed + AP/server coordination. Several receivers observing one AP do not establish + that distributed consensus has occurred. +- `ruview-unified` provides native RF records, masks and provenance; + `ruview-groundtruth` and `ruview-fusion` provide distinct reference and claim + fusion mechanisms. Their presence does not establish managed-server callers. + +The missing seam is a managed source-preserving capture, actual learning, +account-owned persistence and evaluation lifecycle. The first target is +room-selective binary presence: empty, occupied-static, occupied-moving and +adjacent-only controls. Pose, person count and numeric Vitals require separate +acceptance. + +Implementation is reviewed in [#2423](https://github.com/ruvnet/RuView/pull/2423), +with the client in +[mobile #122](https://github.com/cognitum-one/ruview-mobile/pull/122). +[Tracker #2418](https://github.com/ruvnet/RuView/issues/2418) separates source, +artifact, device and physical acceptance. This ADR and those draft PRs do not +constitute architectural acceptance or measured physical accuracy. + +## Decision + +### Preserve the admitted RF view + +Use native `RfFrameV2` records with original IQ, validity masks and explicit +provenance. The initial learner consumes derived magnitude features only. +Preserve gain and DC information needed by the task; avoid per-window centering +that removes the evidence the model is expected to learn. + +Use an opt-in convex magnitude projection. The existing cubic control path can +undershoot below zero at valid spectral nulls. Keep that control path unchanged +for comparison; bind the new projection to `usable-drop2-linear-v2` so previous +candidate preprocessing cannot be silently relabelled. + +Key readiness by the reported receiver, antenna count, native bin grid, PPDU, +frequency and bandwidth. A stable arrival stream for one class cannot supply +cadence for another class later rejected by the adapter. Require both the +bounded stability window and the fresh packet window consumed by the learner. +Ignore other PHY classes from the same receiver without admitting them into the +selected view. Missing selected inputs continue to withdraw inference coverage. + +The first adapter supports its explicitly recognized narrow-band layouts. +Primary-channel metadata must not be silently treated as the centre of a bonded +wide channel. Receiver timing remains receiver timing; it does not establish a +measurement clock, coherent phase, authenticated transmitter or calibrated +signal quality. + +### Train a real source-conditioned candidate + +Reuse the AETHER transformer, with context-ordered source conditioning, +source-routed projection heads and a room-presence head. Optimize the actual +symmetric InfoNCE and supervised BCE objectives. Require intended backbone and +projection updates; a task-head improvement or a logged loss alone is +insufficient evidence of representation learning. + +Treat aligned views of the same observation as positives. Exclude sibling trials, +shared frame identities and overlapping windows from contrastive negatives. +Freeze train, validation and held-out roles before fitting. Fit normalization +profiles from Train only. Several receivers on the same AP are correlated views; +their count is not an independent-confidence multiplier. + +The initial bounded optimizer/model is a vertical integration candidate. Physical +comparison determines whether it has adequate capacity, robustness and latency. +This decision does not establish equivalence to a pretrained pose model or +justify scaling capacity without new evidence. + +### Keep lifecycle and authority distinct + +Expose explicit configure, prepare/start, complete/cancel, train, evaluate and +resume operations through the existing protected API. Feature support in a +verified helper manifest permits the runtime flag; it grants no model authority. + +Every mutation carries the observed runtime epoch, context, generation, workflow +revision and active phase identity. Compare and mutate under the same lock. +Return prediction identity and its score atomically. Client read polling and +mutation ownership must be separate so a queued poll cannot abort an action or +leave the UI busy. + +Preparation is excluded from labelled collection. Completion requires the +operator's conditions-held acknowledgement. A completed recording interval can +remain provisional while that acknowledgement is pending. Untrained observations +stay in bounded memory under the current retention policy and can be lost if the +helper or account runtime closes; the UI must disclose that limitation. + +Store trained candidates privately beneath the verified owner/installation +namespace, with immutable model versions, hashes and an exact-context pointer. +Preserve previous rooms/models and original provenance through cold load. +Logout retains owned records; another account cannot load them. Resume must use +the trained view rather than today's preferred, different source class. + +Candidate evaluation is diagnostic. Preserve aggregate confusion, abstention and +condition metrics with exact model/context/role/data hashes and original time. +Coverage describes evaluated cohorts, not wall-time availability. Distinct +declared trial IDs do not prove statistical independence. Missing or abstained +positives cannot disappear from unconditional sensitivity. + +Historical evaluation must remain historical after restart. Missing retained +Train identities cannot be converted into a newly verified lineage claim. +Corrupt evaluation evidence must be retained/quarantined without discarding a +valid candidate or manufacturing a pass. Evaluation receipts authorize neither +physical qualification nor adoption. + +### Admit optional teachers through a validated reference boundary + +The initial RF workflow does not activate radar, camera or LiDAR teaching. +Current receipt strings and historical geometry are insufficient admission +proofs. A server-owned registry must resolve authenticated source/session +identity, measured clock mapping, spatial registration, coverage, expiry and +revocation before constructing trusted derived teacher evidence. + +Visible-region absence and whole-room absence are different capabilities. +Occlusion, missing data and out-of-view observations are UNKNOWN. Adjacent-only +labels require target-room absence and separate adjacent activity evidence. +CSI/radar-fused outputs cannot serve as independent RF ground truth. + +References may teach a frozen RF input model. Phone departure ends future +teaching; RF-only inference must remain usable under its own acceptance gates. +Measured spatial benefit belongs to +[mobile #105](https://github.com/cognitum-one/ruview-mobile/issues/105), including +current anchors/transforms and aligned, whole-sequence comparisons. + +### Preserve the control and require explicit adoption + +Keep the selected-source detector and source-bank shadow as labelled controls. +The candidate does not replace their decisions automatically. Adopting a model +requires a separately reviewed policy and matched fresh evidence; saving a +candidate, reporting accuracy or restoring an old receipt cannot substitute. +Presence acceptance does not grant pose, count or Vitals authority. + +Full local account erasure must stop the helper before deleting its owned tree. +Selective learner erasure needs its own scope, generation/cancellation fence and +writer drain before removing artifacts or RAM. Legacy calibration reset and +phone-local spatial deletion must not imply they remove server learner history. + +## Options considered + +| Option | Complexity and reuse | Evidence and limitation | +|---|---|---| +| Independent background models | Reuses existing detector; useful shadow control | Does not train a joint representation or establish occupied/adjacent discrimination | +| Managed source-conditioned learner | Reuses native records and real AETHER computation; adds lifecycle/evidence seams | Requires physical qualification, reference admission and explicit adoption | +| Pretrained pose backbone first | Potential transfer benefit after input/task adaptation | Cannot repair source identity, preprocessing, ownership or reference-registration gaps by itself | + +Choose the managed composition as the next implementation, retaining the other +options as comparisons or later adapters. Keep new architecture review separate +from source CI, packaged native verification and end-user validation. + +## Consequences + +- Source admission and training attribution become explicit and testable. +- Candidate/model/evidence histories can survive sessions without renewing trust. +- Real-world preprocessing errors are visible before spending a room-capture window. +- Bounded RAM-only capture creates a deliberate interruption/restart limitation. +- Additional sources or modalities require compatible view and reference contracts. +- Model capacity, generalization and uncertainty remain empirical questions. + +## Action items + +- [ ] Review this decision and the managed source/client PRs. +- [ ] Qualify the delivered native artifacts and independent physical conditions. +- [ ] Complete durable evaluation integration and historical UI attribution. +- [ ] Implement fenced selective learner erasure with absence/resurrection tests. +- [ ] Agree an adoption/rollback policy before enabling candidate authority. +- [ ] Validate reference clock/coverage/registration and optional derived teachers. +- [ ] Measure spatial, multi-source and reference benefit against frozen controls. diff --git a/docs/adr/README.md b/docs/adr/README.md index b4c0226cb0..24b3b97793 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -2,6 +2,8 @@ Latest proposed decisions: +- [ADR-420: Managed source-aware room learning and durable candidate evidence](ADR-420-managed-source-aware-room-learner.md) + - [ADR-379: An OTA'd image confirms itself with a bounded health check, or rolls back](ADR-379-ota-first-boot-health-check.md) - [ADR-380: Loopback UDP tee for second consumers of the CSI stream](ADR-380-loopback-udp-tee.md) - [ADR-382: `spatial.evidence.v1` export of RF Gaussians and link observations](ADR-382-spatial-evidence-rf-export.md) From d973e2c9188b9e22b9df628ca443f7c5d46d3686 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Mon, 2 Nov 2026 08:42:14 +0100 Subject: [PATCH 042/100] fix(learning): bound and reclaim exact-source readiness classes --- .../src/room_learner_api_tests.rs | 141 ++++++++++++++++++ .../src/room_learner_ingress.rs | 75 +++++++--- 2 files changed, 197 insertions(+), 19 deletions(-) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs index 0c56760da2..2bcf8c18fe 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs @@ -388,3 +388,144 @@ async fn rare_foreign_frequency_neither_relabels_selection_nor_invalidates_a_pha assert_eq!(status["capture"]["sources"][0]["admitted_packets"], 1); assert_eq!(status["mutation_guard"], begun["status"]["mutation_guard"]); } + +// SYNTHETIC cache lifecycle regressions through the real parser and ingress. +fn observe_wire_at(inner: &mut AppStateInner, packet: &[u8], at: std::time::Instant) { + let frame = parse_esp32_frame(packet).expect("complete synthetic packet"); + inner + .node_states + .entry(frame.node_id) + .or_insert_with(NodeState::new) + .observe_raw_grid(CsiGridKey::from_frame(&frame), at); + observe(inner, &frame, packet, at); +} +#[tokio::test] +async fn unsupported_class_churn_cannot_exhaust_readiness_for_healthy_nodes() { + let (_dir, state) = fixture(); + let now = std::time::Instant::now(); + { + let mut inner = state.write().await; + inner.room_learning_sources.clear(); + for bins in 1u16..=1024 { + let mut packet = wire(200, 1); + packet[6..8].copy_from_slice(&bins.to_le_bytes()); + packet.resize(20 + usize::from(bins) * 2, 3); + observe_wire_at(&mut inner, &packet, now); + } + for frequency in 1u32..=1024 { + let mut packet = wire(200, 1); + packet[8..12].copy_from_slice(&frequency.to_le_bytes()); + observe_wire_at(&mut inner, &packet, now); + } + for row in 0..120 { + for node in [30, 31] { + let at = now - std::time::Duration::from_millis((120 - row) as u64 * 100); + observe_wire_at(&mut inner, &wire(node, row + 1), at); + } + } + } + let mut request = config(); + request["source_node_ids"] = serde_json::json!([30, 31]); + let response = json_request(app(state.clone()), "/configure", Method::POST, request).await; + assert_eq!(response["success"], true, "{response}"); + // Only the supported 64-bin class at 2437 MHz plus the two healthy nodes. + assert_eq!(state.read().await.room_learning_sources.len(), 3); +} +#[tokio::test] +async fn expired_supported_classes_are_reclaimed_without_manufacturing_readiness() { + let (_dir, state) = fixture(); + let now = std::time::Instant::now(); + let mut inner = state.write().await; + inner.room_learning_sources.clear(); + let old = now - CALIBRATION_GRID_EVIDENCE_WINDOW - std::time::Duration::from_secs(1); + // Supported one-antenna HT20 classes, all unique by frequency. + for frequency in 4900u32..5924 { + let mut packet = wire(200, 1); + packet[8..12].copy_from_slice(&frequency.to_le_bytes()); + // 5901..5924 falls outside the supported range; use a different node + // at the beginning of the same supported band for those slots. + if frequency > 5900 { + packet[4] = 201; + packet[8..12].copy_from_slice(&(frequency - 1001).to_le_bytes()); + } + observe_wire_at(&mut inner, &packet, old); + } + assert_eq!(inner.room_learning_sources.len(), 1024); + observe_wire_at(&mut inner, &wire(30, 1), now); + assert!(inner.room_learning_sources.len() <= 1024); + assert!( + native_candidates(&inner, 30, now).is_empty(), + "one fresh packet is not readiness" + ); + for row in 1..120 { + observe_wire_at( + &mut inner, + &wire(30, row + 1), + now + std::time::Duration::from_millis(row as u64 * 100), + ); + } + let later = now + std::time::Duration::from_millis(11900); + assert_eq!( + native_candidates(&inner, 30, later).len(), + 1, + "expired unrelated classes must not permanently block a new supported stream" + ); + assert_eq!(inner.room_learning_sources.len(), 1); +} +#[tokio::test] +async fn cache_pressure_preserves_selected_active_class_and_freshness() { + let (_dir, state) = fixture(); + let configured = json_request(app(state.clone()), "/configure", Method::POST, config()).await; + assert_eq!(configured["success"], true); + let now = std::time::Instant::now(); + let mut inner = state.write().await; + // Fill the remaining capacity with unique supported classes. Existing + // selected classes must retain their measured 120-packet readiness. + for frequency in 4900u32..5921 { + let mut packet = wire(200, 1); + packet[8..12].copy_from_slice(&frequency.to_le_bytes()); + if frequency > 5900 { + packet[4] = 201; + packet[8..12].copy_from_slice(&(frequency - 1001).to_le_bytes()); + } + observe_wire_at(&mut inner, &packet, now); + } + assert_eq!(inner.room_learning_sources.len(), 1024); + observe_wire_at(&mut inner, &wire(30, 1), now); + assert_eq!( + inner.room_learning_sources.len(), + 1024, + "fresh cache stays bounded" + ); + assert!(native_candidates(&inner, 30, now).is_empty()); + for node in [15, 16, 17] { + assert_eq!( + native_candidates(&inner, node, now).len(), + 1, + "pressure must not evict a selected ready source" + ); + } + let later = now + CALIBRATION_GRID_EVIDENCE_WINDOW + std::time::Duration::from_secs(1); + observe_wire_at(&mut inner, &wire(30, 2), later); + assert_eq!( + inner.room_learning_sources.len(), + 4, + "reclaim only unrelated expired classes" + ); + for node in [15, 16, 17] { + assert!( + native_candidates(&inner, node, later).is_empty(), + "retaining a selected class must not preserve expired readiness" + ); + } + assert!(native_candidates(&inner, 30, later).is_empty()); + let status = inner + .room_learning + .as_ref() + .unwrap() + .status(now_ns(inner.room_learning.as_ref().unwrap())); + assert_eq!( + status["mutation_guard"], + configured["status"]["mutation_guard"] + ); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_ingress.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_ingress.rs index 399f9ccd50..f20845fedb 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_ingress.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_ingress.rs @@ -106,40 +106,77 @@ pub(crate) fn observe( let Some(runtime) = state.room_learning.clone() else { return; }; + // Unsupported input must never consume the bounded readiness cache. Keep + // the selected-source rejection diagnostic without admitting any evidence. + let Ok(id) = native::grid_id( + frame.n_antennas as usize, + frame.n_subcarriers as usize, + frame.ppdu_type, + ) else { + return; + }; + let frequency_hz = u64::from(frame.freq_mhz) * 1_000_000; + let rejection = if frame.adr018_flags.bw40 { + Some("native capture does not support the reported 40 MHz bandwidth") + } else if !native::is_supported_frequency_hz(frequency_hz) { + Some("unknown or unsupported ESP32 centre frequency") + } else { + None + }; + if let Some(reason) = rejection { + runtime.record_native_rejection(frame.node_id, &id, reason); + return; + } + let bytes = usize::from(frame.n_antennas) * usize::from(frame.n_subcarriers) * 2; + let Some(iq) = packet.get(20..20 + bytes) else { + return; + }; let key = NativeSourceKey::from_frame(frame); if state.room_learning_sources.len() >= 1024 && !state.room_learning_sources.contains_key(&key) { - return; + // Reclaim only expired unrelated classes. Selected contexts keep their + // identity, but eligible() still expires their readiness by receiver + // time; retaining a key cannot manufacture a fresh source window. + let context = runtime.context(); + state.room_learning_sources.retain(|key, meta| { + let fresh = meta.arrivals.back().is_some_and(|last| { + now.saturating_duration_since(*last) <= CALIBRATION_GRID_EVIDENCE_WINDOW + }); + fresh + || context.as_ref().is_some_and(|context| { + context.sources.iter().any(|source| { + source.source_id == key.node_id + && source.center_frequency_hz == meta.frequency_hz + && Some(source.bandwidth_hz) == meta.bandwidth_hz + && native::grid_id( + key.n_antennas as usize, + key.raw_bins as usize, + wifi_densepose_hardware::PpduType::from_byte(key.ppdu_type), + ) + .ok() + .as_deref() + == Some(source.grid_id.as_str()) + }) + }) + }); + if state.room_learning_sources.len() >= 1024 { + return; + } } - let id = native::grid_id( - frame.n_antennas as usize, - frame.n_subcarriers as usize, - frame.ppdu_type, - ) - .ok(); let meta = state .room_learning_sources .entry(key) .or_insert_with(|| LiveSource { - frequency_hz: u64::from(frame.freq_mhz) * 1_000_000, - bandwidth_hz: if !frame.adr018_flags.bw40 && id.is_some() { - Some(20_000_000) - } else { - None - }, + frequency_hz, + bandwidth_hz: Some(20_000_000), receiver_rate_hz: 0.0, arrivals: VecDeque::new(), }); meta.record(now); - let (frequency_hz, rate) = (meta.frequency_hz, meta.receiver_rate_hz); - let Some(id) = id else { return }; + let rate = meta.receiver_rate_hz; if !runtime.selects_grid(frame.node_id, &id) { return; } - let bytes = usize::from(frame.n_antennas) * usize::from(frame.n_subcarriers) * 2; - let Some(iq) = packet.get(20..20 + bytes) else { - return; - }; let ns = runtime.receiver_now_ns(); let projected = match native::canonicalize_esp32(NativePacketInput { node_id: frame.node_id, From d6d81ca62feab03e1a50ffb00a14646350af06dd Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Mon, 2 Nov 2026 11:40:17 +0100 Subject: [PATCH 043/100] feat(learning): add opt-in encrypted owner-bound capture storage --- .../wifi-densepose-sensing-server/Cargo.toml | 3 +- .../src/room_learner/mod.rs | 1 + .../src/room_learner/recovery_store.rs | 435 ++++++++++++++++++ .../src/room_learner/recovery_store_tests.rs | 360 +++++++++++++++ 4 files changed, 798 insertions(+), 1 deletion(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store_tests.rs diff --git a/v2/crates/wifi-densepose-sensing-server/Cargo.toml b/v2/crates/wifi-densepose-sensing-server/Cargo.toml index a9c7ce35ff..b07e458ed8 100644 --- a/v2/crates/wifi-densepose-sensing-server/Cargo.toml +++ b/v2/crates/wifi-densepose-sensing-server/Cargo.toml @@ -29,6 +29,8 @@ ruvector-mincut = { workspace = true } mdns-sd = "0.11" rcgen = "0.12.1" rustls = { version = "0.23", default-features = false, features = ["ring", "std", "tls12"] } +# Opt-in encrypted capture recovery uses the same already-locked crypto package. +ring = "0.17" tokio-rustls = { version = "0.26", default-features = false, features = ["ring", "tls12"] } hyper = { workspace = true } hyper-util = { version = "0.1", features = ["server-auto", "tokio", "service"] } @@ -168,7 +170,6 @@ proptest = { version = "1.5", default-features = false, features = ["std"] } # end to end (router → middleware → verifier), not just mocked at the seam. # Keys are generated at test runtime; none are committed. jsonwebtoken = "9" -ring = "0.17" base64 = "0.21" [[bench]] diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs index 9dcddd4655..757f4fc665 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs @@ -6,3 +6,4 @@ pub mod training; pub mod artifact; pub mod lifecycle; pub mod native; +pub mod recovery_store; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store.rs new file mode 100644 index 0000000000..c19a9c8687 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store.rs @@ -0,0 +1,435 @@ +//! Explicitly consented encrypted reduced-sample recovery; no plaintext fallback. +use ring::{ + aead, hkdf, + rand::{SecureRandom, SystemRandom}, +}; +use serde::{Deserialize, Serialize}; +use std::{ + fs::{self, File, OpenOptions}, + io::{Read, Write}, + path::{Path, PathBuf}, + sync::Mutex, +}; + +pub const RECOVERY_SCHEMA: &str = "ruview.room-learner.capture-recovery.v1"; +pub const MAX_RECOVERY_BYTES: usize = 64 * 1024 * 1024; +pub const RECOVERY_TTL_MS: u64 = 24 * 60 * 60 * 1000; +const MAX_DESCRIPTOR: usize = 4096; +const MAGIC: &[u8; 8] = b"RVCREC01"; +const FILE: &str = "capture-recovery-v1.bin"; +const MAX_FILE: usize = MAX_RECOVERY_BYTES + MAX_DESCRIPTOR + 40; +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct RecoveryDescriptor { + pub schema: String, + pub id: String, + pub owner_scope: String, + pub installation_id: String, + pub context_digest: String, + pub created_at_ms: u64, + pub saved_at_ms: u64, + pub expires_at_ms: u64, +} +pub struct LoadedRecovery { + pub descriptor: RecoveryDescriptor, + pub plaintext: Vec, +} +pub struct RecoveryStore { + directory: PathBuf, + owner: String, + installation: String, + key: Option, + serial: Mutex<()>, +} +fn id(value: &str) -> bool { + !value.is_empty() + && value.len() <= 256 + && value.is_ascii() + && value.trim() == value + && !value.bytes().any(|b| b.is_ascii_control()) +} +fn hex(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} +#[cfg(unix)] +fn private_metadata(meta: &fs::Metadata, directory: bool) -> Result<(), String> { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + extern "C" { + fn geteuid() -> u32; + } + // SAFETY: geteuid has no arguments or memory side effects. + let uid = unsafe { geteuid() }; + if meta.uid() != uid + || meta.permissions().mode() & 0o7777 != if directory { 0o700 } else { 0o600 } + { + return Err("recovery_private_permissions".into()); + } + Ok(()) +} +#[cfg(not(unix))] +fn private_metadata(_: &fs::Metadata, _: bool) -> Result<(), String> { + Err("recovery_private_permissions_unsupported".into()) +} +fn private_directory(path: &Path) -> Result<(), String> { + match fs::symlink_metadata(path) { + Ok(m) if m.file_type().is_symlink() || !m.is_dir() => { + return Err("recovery_unsafe_directory".into()) + } + Ok(m) => return private_metadata(&m, true), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(_) => return Err("recovery_directory_unavailable".into()), + } + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + let mut b = fs::DirBuilder::new(); + b.mode(0o700); + match b.create(path) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(_) => return Err("recovery_directory_unavailable".into()), + } + } + let m = fs::symlink_metadata(path).map_err(|_| "recovery_directory_unavailable")?; + if m.file_type().is_symlink() || !m.is_dir() { + return Err("recovery_unsafe_directory".into()); + } + private_metadata(&m, true) +} +fn managed_seed(stable: Option<&str>, value: Option<&str>) -> Option<[u8; 32]> { + if stable != Some("1") { + return None; + } + let value = value?; + if value.len() != 64 || !value.bytes().all(|b| b.is_ascii_hexdigit()) { + return None; + } + let mut bytes = [0u8; 32]; + for (i, slot) in bytes.iter_mut().enumerate() { + *slot = u8::from_str_radix(&value[i * 2..i * 2 + 2], 16).ok()?; + } + Some(bytes) +} +impl RecoveryStore { + pub fn new( + data_dir: &Path, + owner: &str, + installation: &str, + seed: Option<&[u8; 32]>, + ) -> Result { + if !hex(owner) || !id(installation) { + return Err("recovery_invalid_identity".into()); + } + // Check existing permissions before the legacy owner preparer can repair them. + for path in [ + data_dir.to_path_buf(), + data_dir.join("account-owner-v1.json"), + ] { + if let Ok(m) = fs::symlink_metadata(&path) { + if m.file_type().is_symlink() { + return Err("recovery_unsafe_owner_path".into()); + } + private_metadata(&m, path == data_dir)?; + } + } + if data_dir + .parent() + .is_some_and(|p| fs::symlink_metadata(p).is_ok_and(|m| m.file_type().is_symlink())) + { + return Err("recovery_unsafe_owner_path".into()); + } + let owned = crate::account_profile::prepare_directory(owner, data_dir) + .map_err(|_| "recovery_owner_mismatch")?; + let directory = owned.join("room-learner-recovery"); + private_directory(&directory)?; + let key = seed + .map(|seed| { + let mut info = Vec::new(); + for item in [ + b"ruview.capture-recovery.aes256gcm.v1".as_slice(), + owner.as_bytes(), + installation.as_bytes(), + ] { + info.extend_from_slice(&(item.len() as u32).to_be_bytes()); + info.extend_from_slice(item); + } + let salt = hkdf::Salt::new(hkdf::HKDF_SHA256, b"ruview.capture-recovery.hkdf.v1"); + let prk = salt.extract(seed); + let infos = [info.as_slice()]; + let okm = prk + .expand(&infos, &aead::AES_256_GCM) + .map_err(|_| "recovery_key_unavailable")?; + let mut material = [0u8; 32]; + okm.fill(&mut material) + .map_err(|_| "recovery_key_unavailable")?; + let result = aead::UnboundKey::new(&aead::AES_256_GCM, &material) + .map(aead::LessSafeKey::new) + .map_err(|_| "recovery_key_unavailable"); + material.fill(0); + result + }) + .transpose()?; + Ok(Self { + directory, + owner: owner.into(), + installation: installation.into(), + key, + serial: Mutex::new(()), + }) + } + pub fn from_managed_env( + data_dir: &Path, + owner: &str, + installation: &str, + ) -> Result { + let stable = std::env::var("RUVIEW_CAPTURE_RECOVERY_KEY_STABLE").ok(); + let value = if stable.as_deref() == Some("1") { + std::env::var("WDP_RUFIELD_SIGNING_SEED").ok() + } else { + None + }; + let mut seed = managed_seed(stable.as_deref(), value.as_deref()); + let result = Self::new(data_dir, owner, installation, seed.as_ref()); + if let Some(ref mut bytes) = seed { + bytes.fill(0); + } + result + } + pub fn available(&self) -> bool { + self.key.is_some() + } + fn verify_directory(&self) -> Result<(), String> { + let owned = self.directory.parent().ok_or("recovery_owner_mismatch")?; + for (path, is_directory) in [ + (owned.to_path_buf(), true), + (owned.join("account-owner-v1.json"), false), + ] { + let m = fs::symlink_metadata(path).map_err(|_| "recovery_owner_mismatch")?; + if m.file_type().is_symlink() + || (is_directory && !m.is_dir()) + || (!is_directory && !m.is_file()) + { + return Err("recovery_unsafe_owner_path".into()); + } + private_metadata(&m, is_directory)?; + } + if owned + .parent() + .is_some_and(|p| fs::symlink_metadata(p).is_ok_and(|m| m.file_type().is_symlink())) + { + return Err("recovery_unsafe_owner_path".into()); + } + let verified = crate::account_profile::prepare_directory(&self.owner, owned) + .map_err(|_| "recovery_owner_mismatch")?; + if verified != owned { + return Err("recovery_owner_mismatch".into()); + } + private_directory(&self.directory) + } + fn descriptor(&self, d: &RecoveryDescriptor, now: Option) -> Result<(), String> { + if d.schema != RECOVERY_SCHEMA + || !id(&d.id) + || d.owner_scope != self.owner + || d.installation_id != self.installation + || !hex(&d.context_digest) + || d.created_at_ms == 0 + || d.saved_at_ms < d.created_at_ms + || d.expires_at_ms <= d.created_at_ms + || d.expires_at_ms - d.created_at_ms > RECOVERY_TTL_MS + || d.saved_at_ms >= d.expires_at_ms + { + return Err("recovery_descriptor_invalid".into()); + } + if let Some(now) = now { + if now < d.created_at_ms || now < d.saved_at_ms { + return Err("recovery_clock_regression".into()); + } + if now >= d.expires_at_ms { + return Err("recovery_expired".into()); + } + } + Ok(()) + } + fn read(&self, now: Option) -> Result, String> { + let key = self.key.as_ref().ok_or("recovery_unavailable")?; + self.verify_directory()?; + let path = self.directory.join(FILE); + let m = match fs::symlink_metadata(&path) { + Ok(m) => m, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(_) => return Err("recovery_read_failed".into()), + }; + if !m.is_file() || m.file_type().is_symlink() || m.len() > MAX_FILE as u64 { + return Err("recovery_unsafe_file".into()); + } + private_metadata(&m, false)?; + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + #[cfg(target_os = "macos")] + options.custom_flags(0x100); // O_NOFOLLOW + #[cfg(target_os = "linux")] + options.custom_flags(0x20000); // O_NOFOLLOW + } + let file = options.open(&path).map_err(|_| "recovery_read_failed")?; + let opened = file.metadata().map_err(|_| "recovery_read_failed")?; + private_metadata(&opened, false)?; + if !opened.is_file() || opened.len() > MAX_FILE as u64 { + return Err("recovery_unsafe_file".into()); + } + let mut bytes = Vec::new(); + file.take((MAX_FILE + 1) as u64) + .read_to_end(&mut bytes) + .map_err(|_| "recovery_read_failed")?; + if bytes.len() > MAX_FILE || bytes.len() < 40 || &bytes[..8] != MAGIC { + return Err("recovery_corrupt".into()); + } + let n = + u32::from_be_bytes(bytes[8..12].try_into().map_err(|_| "recovery_corrupt")?) as usize; + if n == 0 || n > MAX_DESCRIPTOR || bytes.len() < 12 + n + 12 + 16 { + return Err("recovery_corrupt".into()); + } + let aad = bytes[..12 + n].to_vec(); + let d: RecoveryDescriptor = serde_json::from_slice(&bytes[12..12 + n]) + .map_err(|_| "recovery_descriptor_invalid")?; + let nonce = aead::Nonce::try_assume_unique_for_key(&bytes[12 + n..24 + n]) + .map_err(|_| "recovery_corrupt")?; + let mut payload = bytes[24 + n..].to_vec(); + let plain = key + .open_in_place(nonce, aead::Aad::from(&aad), &mut payload) + .map_err(|_| "recovery_authentication_failed")?; + if plain.len() > MAX_RECOVERY_BYTES { + return Err("recovery_budget_exceeded".into()); + } + let plaintext = plain.to_vec(); + self.descriptor(&d, now)?; + Ok(Some(LoadedRecovery { + descriptor: d, + plaintext, + })) + } + /// Authenticate retained metadata for an explicit restore/discard decision. + /// Expiry and current-clock checks belong to load; inspection never renews + /// the original deadline and never exposes decrypted sample bytes. + pub fn inspect(&self) -> Result, String> { + let _lock = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + Ok(self.read(None)?.map(|loaded| loaded.descriptor)) + } + pub fn load(&self, now_ms: u64) -> Result, String> { + let _lock = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + self.read(Some(now_ms)) + } + pub fn save( + &self, + d: &RecoveryDescriptor, + plaintext: &[u8], + now_ms: u64, + ) -> Result<(), String> { + let _lock = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + let key = self.key.as_ref().ok_or("recovery_unavailable")?; + self.descriptor(d, Some(now_ms))?; + if plaintext.len() > MAX_RECOVERY_BYTES { + return Err("recovery_budget_exceeded".into()); + } + if let Some(previous) = self.read(Some(now_ms))? { + let p = previous.descriptor; + if p.id != d.id + || p.context_digest != d.context_digest + || p.created_at_ms != d.created_at_ms + || p.expires_at_ms != d.expires_at_ms + || d.saved_at_ms < p.saved_at_ms + { + return Err("recovery_discard_required".into()); + } + } + let descriptor = serde_json::to_vec(d).map_err(|_| "recovery_descriptor_invalid")?; + if descriptor.len() > MAX_DESCRIPTOR { + return Err("recovery_descriptor_invalid".into()); + } + let mut output = MAGIC.to_vec(); + output.extend_from_slice(&(descriptor.len() as u32).to_be_bytes()); + output.extend_from_slice(&descriptor); + let mut nonce = [0u8; 12]; + SystemRandom::new() + .fill(&mut nonce) + .map_err(|_| "recovery_random_unavailable")?; + let mut ciphertext = plaintext.to_vec(); + key.seal_in_place_append_tag( + aead::Nonce::assume_unique_for_key(nonce), + aead::Aad::from(&output), + &mut ciphertext, + ) + .map_err(|_| "recovery_encrypt_failed")?; + output.extend_from_slice(&nonce); + output.extend_from_slice(&ciphertext); + let mut suffix = [0u8; 16]; + SystemRandom::new() + .fill(&mut suffix) + .map_err(|_| "recovery_random_unavailable")?; + let temp = self + .directory + .join(format!("recovery-stage-{:x}", u128::from_be_bytes(suffix))); + let result = (|| { + let mut options = OpenOptions::new(); + options.create_new(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(&temp).map_err(|_| "recovery_write_failed")?; + file.write_all(&output) + .and_then(|_| file.sync_all()) + .map_err(|_| "recovery_write_failed")?; + fs::rename(&temp, self.directory.join(FILE)).map_err(|_| "recovery_write_failed")?; + File::open(&self.directory) + .and_then(|f| f.sync_all()) + .map_err(|_| "recovery_write_failed")?; + let restored = self.read(Some(now_ms))?.ok_or("recovery_readback_failed")?; + if restored.descriptor != *d || restored.plaintext != plaintext { + return Err("recovery_readback_failed".into()); + } + Ok(()) + })(); + if temp.exists() { + let _ = fs::remove_file(&temp); + } + result + } + pub fn discard(&self, expected_id: &str) -> Result<(), String> { + let _lock = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + if !id(expected_id) { + return Err("recovery_invalid_identity".into()); + } + if let Some(existing) = self.read(None)? { + if existing.descriptor.id != expected_id { + return Err("recovery_identity_mismatch".into()); + } + fs::remove_file(self.directory.join(FILE)).map_err(|_| "recovery_discard_failed")?; + File::open(&self.directory) + .and_then(|f| f.sync_all()) + .map_err(|_| "recovery_discard_failed")?; + } + Ok(()) + } +} +#[cfg(test)] +#[path = "recovery_store_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store_tests.rs new file mode 100644 index 0000000000..db0360f71d --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store_tests.rs @@ -0,0 +1,360 @@ +//! Synthetic keys and temporary owned directories only. +use super::*; +const NOW: u64 = 1_800_000_000_000; +fn fixture(seed: Option<&[u8; 32]>) -> (tempfile::TempDir, RecoveryStore) { + let t = tempfile::tempdir().unwrap(); + let s = RecoveryStore::new( + &t.path().join("a".repeat(64)).join("SensingServer"), + &"a".repeat(64), + "install", + seed, + ) + .unwrap(); + (t, s) +} +fn descriptor() -> RecoveryDescriptor { + RecoveryDescriptor { + schema: RECOVERY_SCHEMA.into(), + id: "original-run".into(), + owner_scope: "a".repeat(64), + installation_id: "install".into(), + context_digest: "b".repeat(64), + created_at_ms: NOW, + saved_at_ms: NOW, + expires_at_ms: NOW + RECOVERY_TTL_MS, + } +} +#[test] +fn binary_roundtrip_fresh_nonce_and_original_deadline() { + let (_t, s) = fixture(Some(&[7; 32])); + let mut d = descriptor(); + let payload = b"opaque u128=340282366920938463463374607431768211455\0\xff"; + s.save(&d, payload, NOW).unwrap(); + let before = fs::read(s.directory.join(FILE)).unwrap(); + let r = s.load(NOW + 1).unwrap().unwrap(); + assert_eq!(r.descriptor, d); + assert_eq!(r.plaintext, payload); + assert!(!before.windows(payload.len()).any(|w| w == payload)); + d.saved_at_ms += 1; + s.save(&d, payload, NOW + 1).unwrap(); + let after = fs::read(s.directory.join(FILE)).unwrap(); + assert_ne!(before, after); + assert_eq!( + s.load(NOW + 2).unwrap().unwrap().descriptor.expires_at_ms, + NOW + RECOVERY_TTL_MS + ); +} +#[test] +fn unavailable_never_writes_plaintext() { + let (_t, s) = fixture(None); + assert!(!s.available()); + assert_eq!( + s.save(&descriptor(), b"secret", NOW).unwrap_err(), + "recovery_unavailable" + ); + assert!(s.load(NOW).is_err()); + assert!(!s.directory.join(FILE).exists()); +} +#[test] +fn identity_context_deadline_and_clock_cannot_be_replaced_or_renewed() { + let (_t, s) = fixture(Some(&[7; 32])); + let d = descriptor(); + s.save(&d, b"old", NOW).unwrap(); + let path = s.directory.join(FILE); + let before = fs::read(&path).unwrap(); + for field in [ + "id", "context", "created", "expires", "saved", "owner", "install", "schema", + ] { + let mut bad = d.clone(); + match field { + "id" => bad.id = "new".into(), + "context" => bad.context_digest = "c".repeat(64), + "created" => bad.created_at_ms += 1, + "expires" => bad.expires_at_ms -= 1, + "saved" => bad.saved_at_ms -= 1, + "owner" => bad.owner_scope = "d".repeat(64), + "install" => bad.installation_id = "other".into(), + _ => bad.schema = "future".into(), + }; + assert!(s.save(&bad, b"new", NOW + 5).is_err(), "{field}"); + assert_eq!(fs::read(&path).unwrap(), before); + } + assert!(s.load(NOW - 1).unwrap_err_string().contains("clock")); + assert!(s.discard("wrong-id").is_err()); + assert_eq!(fs::read(&path).unwrap(), before); +} +// Avoid requiring Debug on a value containing plaintext. +trait ErrorText { + fn unwrap_err_string(self) -> String; +} +impl ErrorText for Result { + fn unwrap_err_string(self) -> String { + match self { + Err(e) => e, + Ok(_) => panic!("unexpected success"), + } + } +} +#[test] +fn expiry_retained_and_explicit_authenticated_discard() { + let (_t, s) = fixture(Some(&[7; 32])); + s.save(&descriptor(), b"bytes", NOW).unwrap(); + let path = s.directory.join(FILE); + let before = fs::read(&path).unwrap(); + assert_eq!( + s.load(NOW + RECOVERY_TTL_MS).unwrap_err_string(), + "recovery_expired" + ); + let mut next = descriptor(); + next.saved_at_ms = NOW + 1; + assert_eq!( + s.save(&next, b"new", NOW + RECOVERY_TTL_MS).unwrap_err(), + "recovery_expired" + ); + assert_eq!(fs::read(&path).unwrap(), before); + s.discard("original-run").unwrap(); + assert!(!path.exists()); + s.save(&descriptor(), b"new", NOW).unwrap(); +} +#[test] +fn corruption_descriptor_nonce_tag_and_ciphertext_fail_closed() { + for position in [0, 8, 12, 50, 350, 400] { + let (_t, s) = fixture(Some(&[7; 32])); + s.save(&descriptor(), &[42; 512], NOW).unwrap(); + let path = s.directory.join(FILE); + let mut bytes = fs::read(&path).unwrap(); + let n = bytes.len(); + bytes[position % n] ^= 1; + fs::write(&path, &bytes).unwrap(); + assert!(s.load(NOW).is_err(), "position {position}"); + assert!(s.save(&descriptor(), b"new", NOW).is_err()); + assert!(s.discard("original-run").is_err()); + assert_eq!(fs::read(&path).unwrap(), bytes); + } +} +#[test] +fn wrong_key_and_installation_cannot_read_or_overwrite() { + let (_t, s) = fixture(Some(&[7; 32])); + s.save(&descriptor(), b"opaque", NOW).unwrap(); + let path = s.directory.join(FILE); + let before = fs::read(&path).unwrap(); + let data = s.directory.parent().unwrap(); + for (seed, installation) in [([8; 32], "install"), ([7; 32], "other-install")] { + let other = RecoveryStore::new(data, &"a".repeat(64), installation, Some(&seed)).unwrap(); + assert!(other.load(NOW).is_err()); + let mut d = descriptor(); + d.installation_id = installation.into(); + assert!(other.save(&d, b"bad", NOW).is_err()); + assert_eq!(fs::read(&path).unwrap(), before); + } + assert!(RecoveryStore::new(data, &"c".repeat(64), "install", Some(&[7; 32])).is_err()); +} +#[test] +fn bounded_large_payload_and_oversize_file_fail_closed() { + let (_t, s) = fixture(Some(&[7; 32])); + assert_eq!( + s.save(&descriptor(), &vec![1; MAX_RECOVERY_BYTES + 1], NOW) + .unwrap_err(), + "recovery_budget_exceeded" + ); + assert!(!s.directory.join(FILE).exists()); + s.save(&descriptor(), b"small", NOW).unwrap(); + let f = OpenOptions::new() + .write(true) + .open(s.directory.join(FILE)) + .unwrap(); + f.set_len((MAX_FILE + 1) as u64).unwrap(); + assert!(s.load(NOW).is_err()); + assert!(s.save(&descriptor(), b"new", NOW).is_err()); +} +#[cfg(unix)] +#[test] +fn private_files_symlinks_and_owner_witness_revalidated() { + use std::os::unix::fs::{symlink, PermissionsExt}; + for target in ["file", "directory", "witness"] { + let (t, s) = fixture(Some(&[7; 32])); + s.save(&descriptor(), b"small", NOW).unwrap(); + let path = match target { + "file" => s.directory.join(FILE), + "directory" => s.directory.clone(), + _ => s.directory.parent().unwrap().join("account-owner-v1.json"), + }; + fs::rename(&path, path.with_extension("preserved")).unwrap(); + let victim = t.path().join("victim"); + fs::write(&victim, b"untouched").unwrap(); + fs::set_permissions(&victim, fs::Permissions::from_mode(0o600)).unwrap(); + symlink(&victim, &path).unwrap(); + assert!(s.load(NOW).is_err()); + assert!(s.save(&descriptor(), b"bad", NOW).is_err()); + assert_eq!(fs::read(victim).unwrap(), b"untouched"); + } + let (_t, s) = fixture(Some(&[7; 32])); + s.save(&descriptor(), b"small", NOW).unwrap(); + let p = s.directory.join(FILE); + fs::set_permissions(&p, fs::Permissions::from_mode(0o644)).unwrap(); + assert!(s.load(NOW).is_err()); + assert!(s.save(&descriptor(), b"bad", NOW).is_err()); +} +#[test] +fn invalid_paths_and_descriptor_bounds_do_not_disclose_key_or_payload() { + let t = tempfile::tempdir().unwrap(); + assert!(RecoveryStore::new( + Path::new("relative"), + &"a".repeat(64), + "install", + Some(&[7; 32]) + ) + .is_err()); + let (_t, s) = fixture(Some(&[7; 32])); + let mut d = descriptor(); + d.id = "../escape".into(); // IDs are opaque authenticated metadata, never paths. + s.save(&d, b"sensitive", NOW).unwrap(); + assert!(!t.path().join("escape").exists()); + d.id = "x".repeat(257); + let e = s.save(&d, b"sensitive", NOW).unwrap_err(); + assert!(!e.contains("sensitive")); + assert!(!e.contains("070707")); +} + +#[test] +fn managed_key_policy_has_no_implicit_dev_or_ephemeral_fallback() { + let value = "07".repeat(32); + for stable in [None, Some(""), Some("true"), Some("0"), Some("1 ")] { + assert!(managed_seed(stable, Some(&value)).is_none()); + } + for invalid in [ + None, + Some("short"), + Some("adr262-ruview-rufield-dev-seed!!"), + ] { + assert!(managed_seed(Some("1"), invalid).is_none()); + } + assert_eq!(managed_seed(Some("1"), Some(&value)), Some([7; 32])); + assert!(managed_seed(Some("1"), Some(&"z".repeat(64))).is_none()); +} +#[test] +fn copied_other_owner_ciphertext_does_not_authenticate() { + let (_a, a) = fixture(Some(&[7; 32])); + a.save(&descriptor(), b"private-owner-a", NOW).unwrap(); + let b_root = tempfile::tempdir().unwrap(); + let b = RecoveryStore::new( + &b_root.path().join("b".repeat(64)).join("SensingServer"), + &"b".repeat(64), + "install", + Some(&[7; 32]), + ) + .unwrap(); + fs::copy(a.directory.join(FILE), b.directory.join(FILE)).unwrap(); + assert!(b.load(NOW).is_err()); + let mut d = descriptor(); + d.owner_scope = "b".repeat(64); + assert!(b.save(&d, b"bad", NOW).is_err()); +} +#[cfg(unix)] +#[test] +fn owner_permissions_drift_is_not_silently_repaired() { + use std::os::unix::fs::PermissionsExt; + for witness in [true, false] { + let (_t, s) = fixture(Some(&[7; 32])); + s.save(&descriptor(), b"original", NOW).unwrap(); + let path = if witness { + s.directory.parent().unwrap().join("account-owner-v1.json") + } else { + s.directory.parent().unwrap().to_path_buf() + }; + fs::set_permissions( + &path, + fs::Permissions::from_mode(if witness { 0o644 } else { 0o755 }), + ) + .unwrap(); + assert!(s.load(NOW).is_err()); + assert!(s.save(&descriptor(), b"bad", NOW).is_err()); + assert_eq!( + fs::metadata(path).unwrap().permissions().mode() & 0o777, + if witness { 0o644 } else { 0o755 } + ); + } +} + +#[test] +fn descriptor_nonce_and_tag_are_all_authenticated_and_truncation_is_retained() { + for part in ["descriptor", "nonce", "ciphertext", "tag", "truncated"] { + let (_t, s) = fixture(Some(&[7; 32])); + s.save(&descriptor(), b"bounded opaque payload", NOW) + .unwrap(); + let path = s.directory.join(FILE); + let mut bytes = fs::read(&path).unwrap(); + let descriptor_len = u32::from_be_bytes(bytes[8..12].try_into().unwrap()) as usize; + match part { + "descriptor" => { + let at = bytes + .windows(12) + .position(|w| w == b"original-run") + .unwrap(); + bytes[at] = b'O'; + } + "nonce" => bytes[12 + descriptor_len] ^= 1, + "ciphertext" => bytes[24 + descriptor_len] ^= 1, + "tag" => { + let end = bytes.len() - 1; + bytes[end] ^= 1; + } + _ => { + bytes.truncate(bytes.len() - 1); + } + } + fs::write(&path, &bytes).unwrap(); + assert!(s.load(NOW).is_err(), "{part}"); + assert!(s.save(&descriptor(), b"replacement", NOW).is_err()); + assert_eq!(fs::read(path).unwrap(), bytes); + } +} +#[test] +fn exact_plaintext_budget_roundtrips() { + let (_t, s) = fixture(Some(&[7; 32])); + let bytes = vec![0x65; MAX_RECOVERY_BYTES]; + s.save(&descriptor(), &bytes, NOW).unwrap(); + assert_eq!(s.load(NOW).unwrap().unwrap().plaintext, bytes); +} + +#[test] +fn expired_authenticated_inspection_preserves_deadline_then_explicit_discard() { + let (_t, s) = fixture(Some(&[7; 32])); + let d = descriptor(); + s.save(&d, b"private reduced samples", NOW).unwrap(); + let path = s.directory.join(FILE); + let before = fs::read(&path).unwrap(); + assert_eq!( + s.load(d.expires_at_ms).unwrap_err_string(), + "recovery_expired" + ); + assert_eq!(s.inspect().unwrap(), Some(d.clone())); + assert!(s.load(NOW - 1).is_err()); + assert_eq!(s.inspect().unwrap(), Some(d)); + assert_eq!(fs::read(&path).unwrap(), before); + s.discard("original-run").unwrap(); + assert_eq!(s.inspect().unwrap(), None); +} +#[test] +fn wrong_key_or_tampered_inspection_fails_without_altering_retained_evidence() { + let (_t, s) = fixture(Some(&[7; 32])); + s.save(&descriptor(), b"private reduced samples", NOW) + .unwrap(); + let path = s.directory.join(FILE); + let before = fs::read(&path).unwrap(); + let other = RecoveryStore::new( + s.directory.parent().unwrap(), + &"a".repeat(64), + "install", + Some(&[8; 32]), + ) + .unwrap(); + assert!(other.inspect().is_err()); + assert_eq!(fs::read(&path).unwrap(), before); + let mut bad = before; + let last = bad.len() - 1; + bad[last] ^= 1; + fs::write(&path, &bad).unwrap(); + assert!(s.inspect().is_err()); + assert_eq!(fs::read(&path).unwrap(), bad); +} From db22bc74df9d56c0fad159db6ade4bbc7d085a19 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Mon, 2 Nov 2026 11:40:18 +0100 Subject: [PATCH 044/100] feat(capture): recover original cohorts without restoring live freshness --- .../src/room_learner/capture.rs | 4 + .../src/room_learner/capture_recovery.rs | 416 ++++++++++++++++++ .../room_learner/capture_recovery_tests.rs | 332 ++++++++++++++ 3 files changed, 752 insertions(+) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery_tests.rs diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs index f8cdd2ffb0..f7b1770da7 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs @@ -333,3 +333,7 @@ fn identifier(s: &str) -> Result<(), String> { #[cfg(test)] #[path = "capture_tests.rs"] mod tests; + +#[path = "capture_recovery.rs"] +mod recovery; +pub use recovery::{CaptureCheckpoint, CaptureCheckpointSummary, CaptureRestore, RecoveryPhaseDisposition, RecoveryPhaseSummary}; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery.rs new file mode 100644 index 0000000000..6a0e9f876d --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery.rs @@ -0,0 +1,416 @@ +//! Typed reduced-sample recovery. Receiver buffers are deliberately never saved. +use super::*; +use std::collections::HashSet; +const SCHEMA: &str = "ruview.room-learner.capture-checkpoint.v1"; +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CaptureCheckpoint { + schema: String, + context: CaptureContext, + run_id: String, + saved_at_ns: u64, + last_seen_ns: u64, + sources: Vec, + cohorts: Vec, + history: Vec, + phase: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct SourceWatermark { + source_id: u8, + sequence: Option, + ordinal: u64, + last_time_ns: u64, + admitted_packets: u64, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct CheckpointPhase { + request: PhaseRequest, + started_ns: u64, + bucket: usize, + representatives: Vec, +} +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum RecoveryPhaseDisposition { + NoActivePhase, + CompletedNeedsConfirmation, + InterruptedNeedsRestart, +} +#[derive(Debug)] +pub struct CaptureRestore { + pub session: CaptureSession, + pub disposition: RecoveryPhaseDisposition, +} +#[derive(Debug, Clone, Serialize)] +pub struct CaptureCheckpointSummary { + pub run_id: String, + pub room_digest: String, + pub revision: u64, + pub source_ids: Vec, + pub sealed_cohorts: usize, + pub phase: Option, +} +#[derive(Debug, Clone, Serialize)] +pub struct RecoveryPhaseSummary { + pub role: DataRole, + pub label: RoomLabel, + pub complete: bool, + pub representative_cohorts: usize, + pub target_ms: u64, + pub capture_elapsed_ms: u64, +} +fn phase_end(p: &CheckpointPhase) -> Result { + if !p.request.conditions_confirmed + || p.request.role == DataRole::Inference + || p.request.label == RoomLabel::Unknown + || !(MIN_PHASE_MS..=MAX_PHASE_MS).contains(&p.request.duration_ms) + || p.request.lead_in_ms > 120_000 + || p.started_ns == 0 + || p.bucket >= COHORTS_PER_PHASE + { + return Err("invalid recovery phase request".into()); + } + identifier(&p.request.trial_id)?; + identifier(&p.request.session_id)?; + p.started_ns + .checked_add((p.request.duration_ms + p.request.lead_in_ms) * 1_000_000) + .ok_or_else(|| "recovery phase time overflow".into()) +} +impl CaptureCheckpoint { + pub fn context(&self) -> &CaptureContext { + &self.context + } + pub fn run_id(&self) -> &str { + &self.run_id + } + pub fn saved_at_ns(&self) -> u64 { + self.saved_at_ns + } + pub fn summary(&self) -> CaptureCheckpointSummary { + CaptureCheckpointSummary { + run_id: self.run_id.clone(), + room_digest: self.context.room_digest.clone(), + revision: self.context.revision, + source_ids: self.context.sources.iter().map(|s| s.source_id).collect(), + sealed_cohorts: self.cohorts.len(), + phase: self.phase.as_ref().map(|p| RecoveryPhaseSummary { + role: p.request.role, + label: p.request.label, + complete: phase_end(p).is_ok_and(|end| self.saved_at_ns >= end) + && p.representatives.len() == COHORTS_PER_PHASE, + representative_cohorts: p.representatives.len(), + target_ms: p.request.duration_ms, + capture_elapsed_ms: self.saved_at_ns.saturating_sub( + p.started_ns + .saturating_add(p.request.lead_in_ms.saturating_mul(1_000_000)), + ) / 1_000_000, + }), + } + } +} +impl CaptureSession { + pub fn checkpoint(&self, now_ns: u64) -> Result { + let snapshot = CaptureCheckpoint { + schema: SCHEMA.into(), + context: self.context.clone(), + run_id: self.run_id.clone(), + saved_at_ns: now_ns, + last_seen_ns: self.last_seen_ns, + sources: self + .context + .sources + .iter() + .map(|s| { + let b = &self.sources[&s.source_id]; + SourceWatermark { + source_id: s.source_id, + sequence: b.sequence, + ordinal: b.ordinal, + last_time_ns: b.last_time_ns, + admitted_packets: b.admitted_packets, + } + }) + .collect(), + cohorts: self.cohorts.clone(), + history: self.history.clone(), + phase: self.phase.as_ref().map(|p| CheckpointPhase { + request: p.request.clone(), + started_ns: p.started_ns, + bucket: p.bucket, + representatives: p.representatives.clone(), + }), + }; + // Apply the same validation to exports and imports; do not roundtrip + // through serde_json::Value, which cannot retain arbitrary u128 IDs. + snapshot.validate(&self.context, now_ns)?; + Ok(snapshot) + } + pub fn restore_checkpoint( + snapshot: CaptureCheckpoint, + expected: &CaptureContext, + now_ns: u64, + ) -> Result { + snapshot.validate(expected, now_ns)?; + let mut session = Self::new(snapshot.context.clone(), snapshot.run_id.clone())?; + session.last_seen_ns = snapshot.last_seen_ns; + session.cohorts = snapshot.cohorts; + session.history = snapshot.history; + for receipt in &session.history { + session + .trials + .insert(receipt.trial_id.clone(), receipt.role); + session + .sessions + .insert(receipt.session_id.clone(), receipt.role); + } + for watermark in snapshot.sources { + session.sources.insert( + watermark.source_id, + SourceBuffer { + sequence: watermark.sequence, + ordinal: watermark.ordinal, + last_time_ns: watermark.last_time_ns, + admitted_packets: watermark.admitted_packets, + live: VecDeque::new(), + }, + ); + } + let disposition = if let Some(p) = snapshot.phase { + session + .trials + .insert(p.request.trial_id.clone(), p.request.role); + session + .sessions + .insert(p.request.session_id.clone(), p.request.role); + let complete = snapshot.saved_at_ns >= phase_end(&p)? + && p.representatives.len() == COHORTS_PER_PHASE; + let phase = Phase { + request: p.request, + started_ns: p.started_ns, + bucket: p.bucket, + representatives: if complete { p.representatives } else { vec![] }, + pending: HashMap::new(), + admitted: HashMap::new(), + }; + if complete { + session.phase = Some(phase); + RecoveryPhaseDisposition::CompletedNeedsConfirmation + } else { + session.history.push(receipt( + &phase, + PhaseStatus::Failed, + Some( + "capture interrupted; restart required; downtime is not observation".into(), + ), + )); + RecoveryPhaseDisposition::InterruptedNeedsRestart + } + } else { + RecoveryPhaseDisposition::NoActivePhase + }; + Ok(CaptureRestore { + session, + disposition, + }) + } + /// Only semantic milestones trigger persistence; packet cadence and receiver + /// counters cannot turn this into a raw-packet write path. + pub fn recovery_marker(&self, now_ns: u64) -> String { + let phase = self.phase.as_ref().map(|p| { + let end = p + .started_ns + .saturating_add((p.request.duration_ms + p.request.lead_in_ms) * 1_000_000); + ( + &p.request, + p.started_ns, + p.representatives.len(), + now_ns >= end, + ) + }); + let bytes = serde_json::to_vec(&( + &self.context, + &self.run_id, + &self.history, + self.cohorts.len(), + phase, + )) + .unwrap_or_default(); + format!("{:x}", Sha256::digest(bytes)) + } +} +impl CaptureCheckpoint { + fn validate(&self, expected: &CaptureContext, now_ns: u64) -> Result<(), String> { + self.context.validate()?; + if self.schema != SCHEMA + || &self.context != expected + || self.saved_at_ns == 0 + || self.saved_at_ns > now_ns + || self.last_seen_ns > self.saved_at_ns + || self.sources.len() != self.context.sources.len() + || self.history.len() > MAX_PHASE_HISTORY + || (self.phase.is_some() && self.history.len() >= MAX_PHASE_HISTORY) + || self.cohorts.len() > MAX_COHORTS + { + return Err("invalid recovery context, time or capacity".into()); + } + let session = CaptureSession::new(self.context.clone(), self.run_id.clone())?; + let mut latest = 0; + for (w, spec) in self.sources.iter().zip(&self.context.sources) { + if w.source_id != spec.source_id + || w.ordinal >= (1u64 << 56) + || w.last_time_ns > self.last_seen_ns + || w.admitted_packets > w.ordinal + || match w.sequence { + None => w.ordinal != 0 || w.last_time_ns != 0 || w.admitted_packets != 0, + Some(_) => w.ordinal == 0 || w.last_time_ns == 0 || w.admitted_packets == 0, + } + { + return Err("invalid source recovery watermark".into()); + } + latest = latest.max(w.last_time_ns); + } + if latest != self.last_seen_ns { + return Err("recovery receiver watermark mismatch".into()); + } + let mut trials = HashMap::new(); + let mut sessions = HashMap::new(); + let mut bind = + |trial: &String, capture_session: &String, role: DataRole| -> Result<(), String> { + identifier(trial)?; + identifier(capture_session)?; + if role == DataRole::Inference + || trials + .insert(trial.clone(), role) + .is_some_and(|r| r != role) + || sessions + .insert(capture_session.clone(), role) + .is_some_and(|r| r != role) + { + return Err("recovery phase crosses frozen roles".into()); + } + Ok(()) + }; + for r in &self.history { + bind(&r.trial_id, &r.session_id, r.role)?; + if r.label == RoomLabel::Unknown + || r.cohort_count + != if r.status == PhaseStatus::Sealed { + COHORTS_PER_PHASE + } else { + 0 + } + || r.reason + .as_ref() + .is_some_and(|s| s.len() > 512 || s.chars().any(char::is_control)) + { + return Err("invalid recovery history".into()); + } + } + let mut all = self.cohorts.clone(); + if let Some(p) = &self.phase { + phase_end(p)?; + bind(&p.request.trial_id, &p.request.session_id, p.request.role)?; + if p.started_ns > self.saved_at_ns + || p.representatives.len() > COHORTS_PER_PHASE + || all.len() + p.representatives.len() > MAX_COHORTS + { + return Err("invalid provisional recovery phase".into()); + } + let mut previous_bucket = None; + let start = p.started_ns + p.request.lead_in_ms * 1_000_000; + let width = p.request.duration_ms * 1_000_000 / COHORTS_PER_PHASE as u64; + for c in &p.representatives { + let bucket = (0..COHORTS_PER_PHASE) + .find(|b| { + c.id == format!("{}-phase-{}-bucket-{b}", self.run_id, self.history.len()) + }) + .ok_or("invalid provisional cohort identity")?; + if previous_bucket.is_some_and(|b| b >= bucket) + || bucket > p.bucket + || c.role != p.request.role + || c.label != p.request.label + || c.trial_id != p.request.trial_id + || c.session_id != p.request.session_id + || c.views.iter().any(|v| { + v.first_time_ns < start + bucket as u64 * width + || v.last_time_ns >= start + (bucket as u64 + 1) * width + }) + { + return Err("invalid provisional phase representative".into()); + } + previous_bucket = Some(bucket); + } + all.extend(p.representatives.clone()); + } + let mut expected_cohorts = HashSet::new(); + for (index, r) in self + .history + .iter() + .enumerate() + .filter(|(_, r)| r.status == PhaseStatus::Sealed) + { + for bucket in 0..COHORTS_PER_PHASE { + let id = format!("{}-phase-{index}-bucket-{bucket}", self.run_id); + let c = self + .cohorts + .iter() + .find(|c| c.id == id) + .ok_or("missing sealed recovery cohort")?; + if c.role != r.role + || c.label != r.label + || c.trial_id != r.trial_id + || c.session_id != r.session_id + { + return Err("sealed recovery history mismatch".into()); + } + expected_cohorts.insert(id); + } + } + if expected_cohorts.len() != self.cohorts.len() { + return Err("unrecorded sealed recovery cohort".into()); + } + let mut frames = HashSet::new(); + for c in &all { + if c.provenance != CaptureProvenance::LiveReceiverObserved || c.teacher.is_some() { + return Err("recovery requires original managed capture provenance".into()); + } + for v in &c.views { + let w = self + .sources + .iter() + .find(|w| w.source_id == v.source_id) + .ok_or("unknown recovery source")?; + if v.samples.len() != WINDOW_PACKETS + || v.last_time_ns > w.last_time_ns + || v.clock.kind != ClockKind::ReceiverOnly + || v.clock.uncertainty_ns != u64::MAX + || v.phase != PhaseAvailability::MagnitudeOnly + { + return Err("invalid reduced recovery source window".into()); + } + for (frame, ordinal) in v.frame_ids.iter().zip(&v.sequence_ids) { + let expected = (u128::from(session.prefix) << 64) + | (u128::from(v.source_id) << 56) + | u128::from(*ordinal); + if *ordinal == 0 + || *ordinal > w.ordinal + || *frame != expected + || !frames.insert(*frame) + { + return Err("reused or invalid recovery frame/sequence watermark".into()); + } + } + } + } + if !all.is_empty() { + ValidatedDataset::new(self.context.clone(), all)?; + } + Ok(()) + } +} +#[cfg(test)] +#[path = "capture_recovery_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery_tests.rs new file mode 100644 index 0000000000..a6b9afb983 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery_tests.rs @@ -0,0 +1,332 @@ +//! Synthetic recovery fixtures, never physical qualification. +use super::*; +use ruview_unified::RfModality; +const START: u64 = 1_000_000_000; +fn context() -> CaptureContext { + CaptureContext { + owner_scope: "synthetic-owner".into(), + installation_id: "synthetic-install".into(), + room_digest: "synthetic-room".into(), + revision: 1, + sources: (1..=2) + .map(|id| SourceSpec { + source_id: id, + device_id: format!("synthetic-{id}"), + grid_id: "ht20-64-raw".into(), + feature_count: 64, + signal_kind: RfModality::WifiCsi, + center_frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + link_id: format!("tx-rx-{id}"), + link_index: 0, + geometry: None, + }) + .collect(), + } +} +fn session() -> CaptureSession { + CaptureSession::new(context(), "synthetic-run".into()).unwrap() +} +fn request(role: DataRole, label: RoomLabel, id: &str) -> PhaseRequest { + PhaseRequest { + trial_id: format!("synthetic-trial-{id}"), + session_id: format!("synthetic-session-{id}"), + role, + label, + duration_ms: MIN_PHASE_MS, + lead_in_ms: 0, + conditions_confirmed: true, + } +} +fn packet(source_id: u8, sequence: u32, values: &[f64]) -> RawSourceFrame<'_> { + RawSourceFrame { + source_id, + grid_id: "ht20-64-raw", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence, + amplitudes: values, + } +} +fn window(s: &mut CaptureSession, at: u64, start_sequence: u32, gain: f64) { + let values = vec![gain; 64]; + for t in 0..WINDOW_PACKETS { + for source in 1..=2 { + s.ingest( + packet(source, start_sequence.wrapping_add(t as u32), &values), + at + t as u64 * 10_000_000 + u64::from(source), + ) + .unwrap(); + } + } +} +fn phase( + s: &mut CaptureSession, + at: u64, + seq: u32, + role: DataRole, + label: RoomLabel, + id: &str, + gain: f64, +) { + s.begin_phase(request(role, label, id), at).unwrap(); + for quarter in 0..COHORTS_PER_PHASE { + window( + s, + at + quarter as u64 * 3_000_000_000 + 1_000_000, + seq + quarter as u32 * 8, + gain, + ); + } + s.finish_phase(at + MIN_PHASE_MS * 1_000_000).unwrap(); +} + +fn reopen(snapshot: &CaptureCheckpoint, now: u64) -> Result { + let bytes = serde_json::to_vec(snapshot).unwrap(); + let decoded: CaptureCheckpoint = serde_json::from_slice(&bytes).unwrap(); + CaptureSession::restore_checkpoint(decoded, &context(), now) +} +#[test] +fn completed_600s_requires_confirmation_after_typed_u128_roundtrip() { + let mut s = session(); + let mut r = request(DataRole::Train, RoomLabel::Empty, "600"); + r.duration_ms = 600_000; + s.begin_phase(r, START).unwrap(); + for bucket in 0..4 { + window( + &mut s, + START + bucket * 150_000_000_000 + 1_000_000, + 1 + bucket as u32 * 8, + 2.0, + ); + } + let end = START + 600_000_000_000; + let snapshot = s.checkpoint(end).unwrap(); + assert!(snapshot.summary().phase.unwrap().complete); + assert!( + snapshot.phase.as_ref().unwrap().representatives[0].views[0].frame_ids[0] + > u128::from(u64::MAX) + ); + let mut restored = reopen(&snapshot, end + 100_000_000_000).unwrap(); + assert_eq!( + restored.disposition, + RecoveryPhaseDisposition::CompletedNeedsConfirmation + ); + assert!(restored.session.cohorts().is_empty()); + assert!(restored.session.live_cohort(end + 100_000_000_000).is_err()); + assert!(restored + .session + .progress(end) + .sources + .iter() + .all(|s| s.live_buffer_frames == 0 && s.phase_buffer_frames == 0)); + let receipt = restored + .session + .finish_phase(end + 100_000_000_000) + .unwrap(); + assert_eq!(receipt.status, PhaseStatus::Sealed); + assert_eq!(restored.session.cohorts().len(), 4); + assert_eq!( + serde_json::to_vec(restored.session.cohorts()).unwrap(), + serde_json::to_vec(&snapshot.phase.unwrap().representatives).unwrap() + ); +} +#[test] +fn interrupted_phase_cannot_count_downtime_or_reuse_provisional_samples() { + let mut s = session(); + s.begin_phase(request(DataRole::Train, RoomLabel::Empty, "partial"), START) + .unwrap(); + window(&mut s, START + 1_000_000, 1, 2.0); + let snapshot = s.checkpoint(START + 2_000_000_000).unwrap(); + let mut recovered = reopen(&snapshot, START + 20_000_000_000).unwrap(); + assert_eq!( + recovered.disposition, + RecoveryPhaseDisposition::InterruptedNeedsRestart + ); + assert!(!recovered.session.phase_active()); + assert!(recovered.session.cohorts().is_empty()); + assert!(recovered + .session + .finish_phase(START + 20_000_000_000) + .is_err()); + assert_eq!(recovered.session.history()[0].status, PhaseStatus::Failed); + assert!(recovered + .session + .begin_phase( + request(DataRole::HeldOut, RoomLabel::Empty, "partial"), + START + 20_000_000_000 + ) + .is_err()); + recovered + .session + .begin_phase( + request(DataRole::Train, RoomLabel::Empty, "partial"), + START + 20_000_000_000, + ) + .unwrap(); + assert!(recovered + .session + .live_cohort(START + 20_000_000_000) + .is_err()); +} +#[test] +fn four_representatives_before_deadline_still_restore_as_interrupted() { + let mut s = session(); + s.begin_phase(request(DataRole::Train, RoomLabel::Empty, "early"), START) + .unwrap(); + for bucket in 0..4 { + window( + &mut s, + START + bucket * 3_000_000_000 + 1_000_000, + 1 + bucket as u32 * 8, + 2.0, + ); + } + let snapshot = s.checkpoint(START + 10_000_000_000).unwrap(); + assert!(!snapshot.summary().phase.unwrap().complete); + assert_eq!( + reopen(&snapshot, START + 20_000_000_000) + .unwrap() + .disposition, + RecoveryPhaseDisposition::InterruptedNeedsRestart + ); +} +#[test] +fn sealed_roles_watermarks_and_replay_guard_survive_without_live_buffers() { + let mut s = session(); + phase( + &mut s, + START, + 1, + DataRole::Train, + RoomLabel::Empty, + "sealed", + 2.0, + ); + let at = START + MIN_PHASE_MS * 1_000_000; + let snapshot = s.checkpoint(at).unwrap(); + let mut restored = reopen(&snapshot, at + 1).unwrap(); + assert_eq!( + restored.disposition, + RecoveryPhaseDisposition::NoActivePhase + ); + assert_eq!( + restored.session.validated_dataset().unwrap().train().len(), + 4 + ); + assert!(restored + .session + .begin_phase( + request(DataRole::Validation, RoomLabel::Empty, "sealed"), + at + 2 + ) + .is_err()); + let values = vec![2.0; 64]; + assert!(restored + .session + .ingest(packet(1, 32, &values), at + 3) + .is_err()); + assert!(restored + .session + .ingest(packet(1, 1, &values), at + 3) + .is_err()); + restored + .session + .ingest(packet(1, 33, &values), at + 3) + .unwrap(); + assert_eq!(restored.session.sources[&1].ordinal, 33); + assert_eq!(restored.session.sources[&1].live.len(), 1); + assert!(restored.session.live_cohort(at + 3).is_err()); +} +#[test] +fn malformed_context_times_sources_roles_and_frame_reuse_fail_closed() { + let mut s = session(); + phase( + &mut s, + START, + 1, + DataRole::Train, + RoomLabel::Empty, + "sealed", + 2.0, + ); + let now = START + MIN_PHASE_MS * 1_000_000; + let original = s.checkpoint(now).unwrap(); + let mut wrong = context(); + wrong.revision += 1; + assert!(CaptureSession::restore_checkpoint(original.clone(), &wrong, now).is_err()); + assert!(reopen(&original, now - 1).is_err()); + for mutant in 0..8 { + let mut c = original.clone(); + match mutant { + 0 => c.sources[0].source_id = 99, + 1 => c.sources[0].last_time_ns = now + 1, + 2 => c.sources[0].ordinal = 1, + 3 => c.history[0].role = DataRole::HeldOut, + 4 => c.cohorts[1].views[0].frame_ids[0] = c.cohorts[0].views[0].frame_ids[0], + 5 => c.cohorts[0].views[0].sequence_ids[0] = 0, + 6 => c.cohorts[0].views[0].samples[0][0] = f32::NAN, + _ => c.cohorts[0].views[0].last_time_ns = now + 1, + } + assert!( + CaptureSession::restore_checkpoint(c, &context(), now).is_err(), + "mutant {mutant}" + ); + } + let mut bytes = serde_json::to_vec(&original).unwrap(); + bytes.pop(); + bytes.extend_from_slice(br#","unknown":1}"#); + assert!(serde_json::from_slice::(&bytes).is_err()); +} +#[test] +fn marker_changes_only_at_capture_milestones_and_original_deadline() { + let mut s = session(); + let initial = s.recovery_marker(START); + let values = vec![2.0; 64]; + s.ingest(packet(1, 1, &values), START).unwrap(); + assert_eq!(initial, s.recovery_marker(START + 1)); + s.begin_phase( + request(DataRole::Train, RoomLabel::Empty, "marker"), + START + 1, + ) + .unwrap(); + let began = s.recovery_marker(START + 2); + assert_ne!(initial, began); + s.ingest(packet(1, 2, &values), START + 3).unwrap(); + assert_eq!(began, s.recovery_marker(START + 4)); + window(&mut s, START + 1_000_000, 3, 2.0); + let representative = s.recovery_marker(START + 100_000_000); + assert_ne!(began, representative); + assert_eq!(representative, s.recovery_marker(START + 1_000_000_000)); + assert_ne!( + representative, + s.recovery_marker(START + MIN_PHASE_MS * 1_000_000 + 1) + ); +} + +#[test] +fn unlabelled_live_windows_and_partial_packet_buffers_are_never_serialized() { + let mut s = session(); + window(&mut s, START, 1, 2.0); + assert!(s.live_cohort(START + 100_000_000).is_ok()); + let saved = s.checkpoint(START + 100_000_000).unwrap(); + let encoded = String::from_utf8(serde_json::to_vec(&saved).unwrap()).unwrap(); + assert!(!encoded.contains("\"samples\"")); + assert!(!encoded.contains("\"live\"")); + assert!(!encoded.contains("\"pending\"")); + let restored = reopen(&saved, START + 100_000_001).unwrap(); + assert!(restored.session.live_cohort(START + 100_000_001).is_err()); + s.begin_phase( + request(DataRole::Train, RoomLabel::Empty, "pending"), + START + 100_000_001, + ) + .unwrap(); + let values = vec![2.0; 64]; + s.ingest(packet(1, 9, &values), START + 100_000_002) + .unwrap(); + let saved = s.checkpoint(START + 100_000_003).unwrap(); + assert!(saved.phase.as_ref().unwrap().representatives.is_empty()); + assert!(!String::from_utf8(serde_json::to_vec(&saved).unwrap()) + .unwrap() + .contains("\"samples\"")); +} From 0c31804dedda9c171a4a07eb70c457ff935d916e Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Mon, 2 Nov 2026 11:40:53 +0100 Subject: [PATCH 045/100] feat(learning): wire guarded provisional capture recovery lifecycle --- .../wifi-densepose-sensing-server/src/main.rs | 9 +- .../src/room_learner/artifact.rs | 22 + .../src/room_learner/lifecycle.rs | 128 +++- .../src/room_learner/lifecycle_capture.rs | 25 +- .../src/room_learner/lifecycle_recovery.rs | 411 ++++++++++++ .../room_learner/lifecycle_recovery_tests.rs | 621 ++++++++++++++++++ .../src/room_learner_api.rs | 109 ++- .../src/room_learner_api_recovery_tests.rs | 160 +++++ .../src/room_learner_api_tests.rs | 6 + 9 files changed, 1474 insertions(+), 17 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner_api_recovery_tests.rs diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index 0f59f6444d..152205c8f5 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -15760,7 +15760,12 @@ async fn main() { let room_learning = if args.room_learner { let owner = std::env::var(wifi_densepose_sensing_server::bearer_auth::OWNER_ACCOUNT_SCOPE_ENV).unwrap_or_default(); let installation = args.installation_id.as_deref().unwrap_or(""); - match wifi_densepose_sensing_server::room_learner::lifecycle::RoomLearningRuntime::new(&args.data_dir, &owner, installation) { + let recovery = wifi_densepose_sensing_server::room_learner::recovery_store::RecoveryStore::from_managed_env(&args.data_dir, &owner, installation); + let runtime = match recovery { + Ok(store) => wifi_densepose_sensing_server::room_learner::lifecycle::RoomLearningRuntime::new_with_recovery(&args.data_dir, &owner, installation, store), + Err(reason) => wifi_densepose_sensing_server::room_learner::lifecycle::RoomLearningRuntime::new_with_recovery_unavailable(&args.data_dir, &owner, installation, reason), + }; + match runtime { Ok(runtime) => Some(Arc::new(runtime)), Err(error) => { error!("Managed room learner could not initialize: {error}"); std::process::exit(2); } } @@ -16348,6 +16353,8 @@ async fn main() { .route("/api/v1/room-learner/train/cancel", post(room_learner_api::cancel_training)) .route("/api/v1/room-learner/evaluate", post(room_learner_api::evaluate)) .route("/api/v1/room-learner/prediction", get(room_learner_api::prediction)) + .route("/api/v1/room-learner/recovery/restore", post(room_learner_api::restore_capture)) + .route("/api/v1/room-learner/recovery/discard", post(room_learner_api::discard_capture)) .route("/api/v1/calibration/reset", post(calibration_reset)) .route("/api/v1/calibration/bootstrap/recover", post(calibration_recover_bootstrap)) // ADR-044 §5.3: runtime-configurable dedup factor diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs index 862e952853..6743e852f6 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs @@ -72,6 +72,28 @@ fn regular_or_missing(path: &Path) -> Result { } } impl ArtifactStore { + /// Load an immutable historical model by exact hash; never use today's pointer. + pub fn load_exact_candidate( + &self, + context: &CaptureContext, + sha: &str, + ) -> Result { + self.validate_context(context)?; + if !hex_digest(sha) { + return Err("invalid exact candidate digest".into()); + } + let dir = self + .context_dir(context, false)? + .ok_or("exact candidate missing")?; + let envelope: Envelope = serde_json::from_slice( + &private_read(&dir.join(format!("{sha}.json"))).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + if envelope.receipt.model_sha256 != sha { + return Err("exact candidate hash mismatch".into()); + } + self.read_model(&dir, &envelope.receipt, context) + } pub fn new( data_dir: &Path, expected_owner_scope: &str, diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs index 9b20eb01a4..04d93dc4dc 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs @@ -7,7 +7,7 @@ use super::{ observation::{CaptureContext, DataRole, RoomLabel}, training::{FitConfig, TrainingReport}, }; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use std::{ path::Path, @@ -17,7 +17,7 @@ use std::{ }, }; -#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub enum JobPhase { Idle, @@ -54,12 +54,14 @@ struct Inner { last_evaluation: Option, evaluation_warning: Option, input_error: Option, + recovery: RecoveryState, } pub struct RoomLearningRuntime { runtime_epoch: String, owner: String, installation: String, store: ArtifactStore, + recovery_store: Option, inner: Mutex, clock_origin: std::time::Instant, clock_epoch_ns: u64, @@ -76,6 +78,63 @@ fn bounded(error: String) -> String { } impl RoomLearningRuntime { pub fn new(directory: &Path, owner: &str, installation: &str) -> Result { + Self::build(directory, owner, installation, None) + } + pub fn new_with_recovery( + directory: &Path, + owner: &str, + installation: &str, + recovery: super::recovery_store::RecoveryStore, + ) -> Result { + Self::build(directory, owner, installation, Some(recovery)) + } + pub fn new_with_recovery_unavailable( + directory: &Path, + owner: &str, + installation: &str, + reason: String, + ) -> Result { + let runtime = Self::new(directory, owner, installation)?; + { + let mut state = runtime + .inner + .lock() + .map_err(|_| "learner state unavailable")?; + state.recovery.supported = true; + state.recovery.error = Some(bounded(reason)); + } + Ok(runtime) + } + /// Explicit time domain for deterministic fixtures; production uses the managed clock. + pub fn new_with_recovery_at( + directory: &Path, + owner: &str, + installation: &str, + recovery: super::recovery_store::RecoveryStore, + now_ns: u64, + ) -> Result { + if now_ns == 0 { + return Err("recovery_clock_invalid".into()); + } + let mut runtime = Self::build(directory, owner, installation, Some(recovery))?; + runtime.clock_epoch_ns = now_ns; + runtime.clock_origin = std::time::Instant::now(); + { + let mut state = runtime + .inner + .lock() + .map_err(|_| "learner state unavailable")?; + state.recovery = RecoveryState::default(); + runtime.inspect_recovery(&mut state, now_ns); + } + Ok(runtime) + } + fn build( + directory: &Path, + owner: &str, + installation: &str, + recovery_store: Option, + ) -> Result { let clock_origin = std::time::Instant::now(); let clock_epoch_ns = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) @@ -83,13 +142,14 @@ impl RoomLearningRuntime { .as_nanos() .try_into() .map_err(|_| "receiver clock outside bounds")?; - Ok(Self { + let runtime = Self { clock_origin, clock_epoch_ns, runtime_epoch: crate::companion::secret(), owner: owner.into(), installation: installation.into(), store: ArtifactStore::new(directory, owner, installation)?, + recovery_store, inner: Mutex::new(Inner { generation: 0, workflow_revision: 0, @@ -101,8 +161,17 @@ impl RoomLearningRuntime { last_evaluation: None, evaluation_warning: None, input_error: None, + recovery: RecoveryState::default(), }), - }) + }; + { + let mut state = runtime + .inner + .lock() + .map_err(|_| "learner state unavailable")?; + runtime.inspect_recovery(&mut state, runtime.receiver_now_ns()); + } + Ok(runtime) } /// UTC-anchored monotonic receiver time. Wall-clock adjustments cannot /// shorten an operator capture or turn a countdown into labelled RF data. @@ -124,6 +193,24 @@ impl RoomLearningRuntime { context: CaptureContext, run_id: String, new_training: bool, + ) -> Result<(), String> { + self.configure_for_recovery( + expected, + context, + run_id, + new_training, + false, + self.receiver_now_ns(), + ) + } + pub fn configure_for_recovery( + &self, + expected: &MutationGuard, + context: CaptureContext, + run_id: String, + new_training: bool, + recovery_consent: bool, + now_ns: u64, ) -> Result<(), String> { context.validate()?; if context.owner_scope != self.owner || context.installation_id != self.installation { @@ -151,6 +238,12 @@ impl RoomLearningRuntime { } else { (None, None) }; + if recovery_consent { + self.prepare_recovery(&mut state, &context, now_ns)?; + } else { + state.recovery.enabled = false; + state.recovery.marker = None; + } state.generation = state .generation .checked_add(1) @@ -170,6 +263,7 @@ impl RoomLearningRuntime { state.last_evaluation = restored_evaluation; state.evaluation_warning = evaluation_warning; state.input_error = None; + self.checkpoint_recovery(&mut state, now_ns, true)?; Ok(()) } /// Called from an explicit protected action. Completion commits only a @@ -182,6 +276,7 @@ impl RoomLearningRuntime { config.validate()?; let (generation, dataset, cancel) = { let mut state = self.command_state(expected)?; + self.require_recovery_ready(&state)?; if busy(&state.job) || (state.frozen_training && !matches!(state.job.phase, JobPhase::Failed | JobPhase::Cancelled)) @@ -215,6 +310,12 @@ impl RoomLearningRuntime { error: None, }; state.frozen_training = true; + let now = self.checkpoint_now(&state); + if let Err(error) = self.checkpoint_recovery(&mut state, now, true) { + state.job.phase = JobPhase::Failed; + state.job.error = Some(bounded(error.clone())); + return Err(error); + } (state.generation, dataset, cancel) }; let runtime = self.clone(); @@ -234,6 +335,8 @@ impl RoomLearningRuntime { state.job.phase = JobPhase::Cancelled; state.job.error = Some("training cancelled; previous stored candidates preserved".into()); + let now = runtime.checkpoint_now(&state); + let _ = runtime.checkpoint_recovery(&mut state, now, true); return; } match outcome { @@ -254,6 +357,8 @@ impl RoomLearningRuntime { state.job.error = Some(bounded(error)); } } + let now = runtime.checkpoint_now(&state); + let _ = runtime.checkpoint_recovery(&mut state, now, true); }); Ok(generation) } @@ -271,9 +376,10 @@ impl RoomLearningRuntime { Ok((self.guard_for(&state), model.infer(&cohort))) } pub fn status(&self, now_ns: u64) -> Value { - let Ok(state) = self.inner.lock() else { + let Ok(mut state) = self.inner.lock() else { return json!({"enabled":true,"state":"unavailable"}); }; + let _ = self.checkpoint_recovery(&mut state, now_ns, false); let required_ids: Vec<_> = state .capture .as_ref() @@ -305,7 +411,7 @@ impl RoomLearningRuntime { "phase_history":state.capture.as_ref().map(|c|c.history()), "phase_duration_defaults":{"Empty":600000,"OccupiedStatic":12000,"OccupiedMoving":12000,"AdjacentOnly":12000}, "phase_lead_in_defaults":{"Empty":15000,"OccupiedStatic":5000,"OccupiedMoving":5000,"AdjacentOnly":15000}, - "untrained_capture_progress_is_volatile":true, + "untrained_capture_progress_is_volatile":!state.recovery.enabled || state.recovery.error.is_some(),"recovery":self.recovery_status(&state, now_ns), "generation":state.generation,"state":state.job.phase,"context_digest":state.capture.as_ref().map(|c|c.context().digest()), "room_digest":state.capture.as_ref().map(|c| &c.context().room_digest), "revision":state.capture.as_ref().map(|c|c.context().revision), @@ -315,7 +421,7 @@ impl RoomLearningRuntime { "error":state.job.error,"input_error":state.input_error, "candidate":state.receipt,"candidate_loaded":state.model.is_some(), "last_evaluation":state.last_evaluation,"evaluation_warning":state.evaluation_warning,"physically_validated":false, - "control_detector_changed":false,"raw_observations_persisted":false, + "control_detector_changed":false,"raw_observations_persisted":state.recovery.derived_persisted(),"derived_samples_persisted":state.recovery.derived_persisted(),"raw_iq_persisted":false, "limitations":["receiver-observed magnitude path; no coherent phase claim", "legacy UDP transmitter identity is unverified","independent teacher registry not configured", "candidate training is not physical qualification"]}) @@ -333,6 +439,14 @@ mod capture_control; mod command; pub use command::{MutationGuard, PhaseIdentity}; +#[path = "lifecycle_recovery.rs"] +mod recovery_control; +use recovery_control::RecoveryState; + +#[cfg(all(test, unix))] +#[path = "lifecycle_recovery_tests.rs"] +mod recovery_tests; + #[path = "lifecycle_evaluation.rs"] mod evaluation_control; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs index c472171923..6896815fff 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs @@ -50,6 +50,7 @@ impl RoomLearningRuntime { now_ns: u64, ) -> Result<(), String> { let mut state = self.command_state(expected)?; + self.require_recovery_ready(&state)?; if busy(&state.job) { return Err("training is active".into()); } @@ -69,6 +70,7 @@ impl RoomLearningRuntime { .ok_or("configure the current room and source set first")? .begin_phase(request, now_ns)?; state.input_error = None; + self.checkpoint_recovery(&mut state, now_ns, true)?; Ok(()) } pub fn ingest(&self, frame: RawSourceFrame<'_>, now_ns: u64) -> Result<(), String> { @@ -80,7 +82,8 @@ impl RoomLearningRuntime { if let Err(error) = &result { state.input_error = Some(bounded(error.clone())); } - result + let saved = self.checkpoint_recovery(&mut state, now_ns, false); + result.and(saved) } pub fn ingest_native( &self, @@ -130,7 +133,8 @@ impl RoomLearningRuntime { if let Err(error) = &result { state.input_error = Some(bounded(error.clone())); } - result + let saved = self.checkpoint_recovery(&mut state, now_ns, false); + result.and(saved) } /// Bounded protected diagnostics only; never grants admission or advances a /// workflow revision. Ignore sources/grids outside the current context. @@ -155,17 +159,28 @@ impl RoomLearningRuntime { conditions_held: bool, ) -> Result { let mut state = self.command_state(expected)?; + let recovered_complete = state.recovery.disposition + == Some( + crate::room_learner::capture::RecoveryPhaseDisposition::CompletedNeedsConfirmation, + ); let capture = state .capture .as_mut() .ok_or("no learner capture configured")?; if !conditions_held { capture.cancel_phase(); + let _ = self.checkpoint_recovery(&mut state, now_ns, true); return Err( "operator did not confirm conditions held; in-flight examples discarded".into(), ); } - capture.finish_phase(now_ns) + if recovered_complete { + capture.live_cohort(now_ns)?; + } + let receipt = capture.finish_phase(now_ns)?; + state.recovery.disposition = None; + self.checkpoint_recovery(&mut state, now_ns, true)?; + Ok(receipt) } pub fn cancel_phase(&self, expected: &MutationGuard) -> Result<(), String> { let mut state = self.command_state(expected)?; @@ -174,6 +189,8 @@ impl RoomLearningRuntime { .as_mut() .ok_or("no learner capture configured")? .cancel_phase(); + let now = self.checkpoint_now(&state); + self.checkpoint_recovery(&mut state, now, true)?; Ok(()) } pub fn cancel_training(&self, expected: &MutationGuard) -> Result { @@ -182,6 +199,8 @@ impl RoomLearningRuntime { state.job.cancel.store(true, Ordering::Release); state.job.phase = JobPhase::Cancelling; } + let now = self.checkpoint_now(&state); + self.checkpoint_recovery(&mut state, now, true)?; Ok(state.job.phase) } } diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs new file mode 100644 index 0000000000..005e18675e --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs @@ -0,0 +1,411 @@ +//! Explicit, encrypted recovery. No receiver buffers or automatic work resume. +use super::*; +use crate::room_learner::{ + capture::{CaptureCheckpoint, CaptureCheckpointSummary, RecoveryPhaseDisposition}, + recovery_store::{RecoveryDescriptor, RECOVERY_SCHEMA, RECOVERY_TTL_MS}, +}; +use sha2::{Digest, Sha256}; + +#[derive(Default)] +pub(super) struct RecoveryState { + pub(super) supported: bool, + pub(super) enabled: bool, + pub(super) descriptor: Option, + summary: Option, + pub(super) marker: Option, + pub(super) error: Option, + pub(super) disposition: Option, + last_now_ns: u64, + last_attempt_ns: u64, +} +impl RecoveryState { + pub(super) fn derived_persisted(&self) -> Option { + // Unreadable retained contents are unknown, never evidence of absence. + // An error also invalidates any older cached summary for this claim. + if self.error.is_some() || (self.descriptor.is_some() && self.summary.is_none()) { + return None; + } + Some(self.summary.as_ref().is_some_and(|s| { + s.sealed_cohorts > 0 + || s.phase + .as_ref() + .is_some_and(|p| p.representative_cohorts > 0) + })) + } +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct RuntimeCheckpoint { + schema: String, + capture: CaptureCheckpoint, + frozen_training: bool, + job_phase: JobPhase, + model_sha256: Option, +} +const SCHEMA: &str = "ruview.room-learner.runtime-recovery.v1"; +pub(super) fn model_sha(model: &RoomModel) -> Result { + Ok(format!("{:x}", Sha256::digest(model.export()?))) +} +fn checkpoint_payload( + bytes: &[u8], + d: &RecoveryDescriptor, + owner: &str, + installation: &str, + now_ns: u64, +) -> Result { + let p: RuntimeCheckpoint = + serde_json::from_slice(bytes).map_err(|_| "recovery_payload_invalid")?; + if p.schema != SCHEMA + || p.capture.context().owner_scope != owner + || p.capture.context().installation_id != installation + || p.capture.context().digest() != d.context_digest + || p.capture.saved_at_ns() / 1_000_000 != d.saved_at_ms + || p.capture.saved_at_ns() < d.created_at_ms.saturating_mul(1_000_000) + || (p.model_sha256.is_some() && (!p.frozen_training || p.job_phase != JobPhase::Ready)) + || (p.job_phase == JobPhase::Ready && p.model_sha256.is_none()) + || (!p.frozen_training && p.job_phase != JobPhase::Idle) + || (p.frozen_training && p.job_phase == JobPhase::Idle) + { + return Err("recovery_payload_binding_mismatch".into()); + } + CaptureSession::restore_checkpoint(p.capture.clone(), p.capture.context(), now_ns)?; + Ok(p) +} +impl RoomLearningRuntime { + pub(super) fn require_recovery_ready(&self, state: &Inner) -> Result<(), String> { + if state.recovery.enabled && state.recovery.error.is_some() { + Err( + "recovery_save_failed; recover, discard, or explicitly configure RAM-only capture" + .into(), + ) + } else { + Ok(()) + } + } + pub(super) fn checkpoint_now(&self, state: &Inner) -> u64 { + self.receiver_now_ns().max(state.recovery.last_now_ns) + } + pub(super) fn inspect_recovery(&self, state: &mut Inner, now_ns: u64) { + let Some(store) = &self.recovery_store else { + return; + }; + match store.inspect() { + Ok(None) => { + state.recovery.descriptor = None; + state.recovery.summary = None; + } + Ok(Some(d)) => { + if d.owner_scope != self.owner || d.installation_id != self.installation { + state.recovery.error = Some("recovery_owner_mismatch".into()); + return; + } + state.recovery.descriptor = Some(d.clone()); + match store + .load(now_ns / 1_000_000) + .and_then(|v| v.ok_or_else(|| "recovery_missing".into())) + .and_then(|loaded| { + checkpoint_payload( + &loaded.plaintext, + &d, + &self.owner, + &self.installation, + now_ns, + ) + }) { + Ok(p) => { + state.recovery.summary = Some(p.capture.summary()); + state.recovery.error = None; + } + Err(error) => state.recovery.error = Some(bounded(error)), + } + } + Err(error) => state.recovery.error = Some(bounded(error)), + } + } + pub(super) fn prepare_recovery( + &self, + state: &mut Inner, + context: &CaptureContext, + now_ns: u64, + ) -> Result<(), String> { + let store = self + .recovery_store + .as_ref() + .filter(|s| s.available()) + .ok_or("recovery_unavailable")?; + match store.inspect() { + Ok(Some(_)) => return Err("learner_recovery_pending".into()), + Ok(None) => {} + Err(error) => { + state.recovery.error = Some(bounded(error.clone())); + return Err(error); + } + } + let created = now_ns / 1_000_000; + let expires = created + .checked_add(RECOVERY_TTL_MS) + .ok_or("recovery_clock_invalid")?; + state.recovery = RecoveryState { + enabled: true, + descriptor: Some(RecoveryDescriptor { + schema: RECOVERY_SCHEMA.into(), + id: crate::companion::secret(), + owner_scope: self.owner.clone(), + installation_id: self.installation.clone(), + context_digest: context.digest(), + created_at_ms: created, + saved_at_ms: created, + expires_at_ms: expires, + }), + ..Default::default() + }; + Ok(()) + } + pub(super) fn checkpoint_recovery( + &self, + state: &mut Inner, + now_ns: u64, + force: bool, + ) -> Result<(), String> { + state.recovery.last_now_ns = state.recovery.last_now_ns.max(now_ns); + if !state.recovery.enabled { + return Ok(()); + } + let result: Result = (|| { + let capture = state.capture.as_ref().ok_or("recovery_capture_missing")?; + let marker = format!( + "{}:{:?}:{}", + capture.recovery_marker(now_ns), + state.job.phase, + state.model.is_some() + ); + if !force + && state.recovery.marker.as_ref() == Some(&marker) + && (state.recovery.error.is_none() + || now_ns.saturating_sub(state.recovery.last_attempt_ns) < 5_000_000_000) + { + return Ok(false); + } + // Record attempts too, so one disk failure cannot cause packet-rate writes. + state.recovery.marker = Some(marker); + state.recovery.last_attempt_ns = now_ns; + let snapshot = capture.checkpoint(now_ns)?; + let summary = snapshot.summary(); + let payload = RuntimeCheckpoint { + schema: SCHEMA.into(), + capture: snapshot, + frozen_training: state.frozen_training, + job_phase: state.job.phase, + model_sha256: state.model.as_ref().map(|m| model_sha(m)).transpose()?, + }; + let mut descriptor = state + .recovery + .descriptor + .clone() + .ok_or("recovery_descriptor_missing")?; + descriptor.saved_at_ms = now_ns / 1_000_000; + let bytes = + serde_json::to_vec(&payload).map_err(|_| "recovery_payload_encode_failed")?; + let store = self.recovery_store.as_ref().ok_or("recovery_unavailable")?; + if let Err(error) = store.save(&descriptor, &bytes, now_ns / 1_000_000) { + // A rename may commit before directory fsync reports failure. + // Read back exactly this attempt; never replay a mutation or + // advertise durable success after the failed durability barrier. + if let Ok(Some(loaded)) = store.load(now_ns / 1_000_000) { + if loaded.descriptor == descriptor && loaded.plaintext == bytes { + state.recovery.descriptor = Some(descriptor); + state.recovery.summary = Some(summary); + } + } + return Err(error); + } + state.recovery.descriptor = Some(descriptor); + state.recovery.summary = Some(summary); + Ok(true) + })(); + match result { + Ok(true) => { + state.recovery.error = None; + Ok(()) + } + Ok(false) => Ok(()), + Err(error) => { + state.recovery.error = Some(bounded(error.clone())); + Err(error) + } + } + } + pub(super) fn recovery_status(&self, state: &Inner, now_ns: u64) -> Value { + let supported = self.recovery_store.is_some() || state.recovery.supported; + let available = self.recovery_store.as_ref().is_some_and(|s| s.available()); + let expired = state + .recovery + .descriptor + .as_ref() + .is_some_and(|d| now_ns / 1_000_000 >= d.expires_at_ms); + let saved = state + .recovery + .descriptor + .as_ref() + .zip(state.recovery.summary.as_ref()) + .filter(|_| !expired) + .map(|(d, s)| { + #[derive(Serialize)] + struct Saved<'a> { + #[serde(flatten)] + descriptor: &'a RecoveryDescriptor, + #[serde(flatten)] + summary: &'a CaptureCheckpointSummary, + } + Saved { + descriptor: d, + summary: s, + } + }); + let status = if !supported { + "disabled" + } else if !available { + "unavailable" + } else if expired { + "expired" + } else if state.recovery.error.is_some() { + "error" + } else if state.recovery.summary.is_some() { + if state.recovery.enabled { + "saved" + } else { + "restore_required" + } + } else { + "disabled" + }; + let error = state.recovery.error.clone().or_else(|| { + (supported && !available).then(|| { + "recovery_unavailable: stable managed encryption key is unavailable".to_string() + }) + }); + json!({"supported":supported,"enabled":state.recovery.enabled,"state":status,"error":error, + "saved":saved,"discard_id":state.recovery.descriptor.as_ref().map(|d| &d.id), + "restored_disposition":state.recovery.disposition,"raw_iq_persisted":false, + "derived_samples_persisted":state.recovery.derived_persisted(),"automatic_resume":false}) + } + pub fn saved_recovery_context( + &self, + recovery_id: &str, + now_ns: u64, + ) -> Result { + let _state = self.inner.lock().map_err(|_| "learner state unavailable")?; + let loaded = self + .recovery_store + .as_ref() + .ok_or("recovery_unavailable")? + .load(now_ns / 1_000_000)? + .ok_or("recovery_missing")?; + if loaded.descriptor.id != recovery_id { + return Err("recovery_identity_mismatch".into()); + } + let payload = checkpoint_payload( + &loaded.plaintext, + &loaded.descriptor, + &self.owner, + &self.installation, + now_ns, + )?; + Ok(payload.capture.context().clone()) + } + pub fn restore_capture_recovery( + &self, + expected: &MutationGuard, + recovery_id: &str, + context: &CaptureContext, + now_ns: u64, + ) -> Result<(), String> { + let mut state = self.command_state(expected)?; + if busy(&state.job) || state.capture.as_ref().is_some_and(|c| c.phase_active()) { + return Err("finish or cancel the active operation first".into()); + } + let loaded = self + .recovery_store + .as_ref() + .ok_or("recovery_unavailable")? + .load(now_ns / 1_000_000)? + .ok_or("recovery_missing")?; + if loaded.descriptor.id != recovery_id { + return Err("recovery_identity_mismatch".into()); + } + let payload = checkpoint_payload( + &loaded.plaintext, + &loaded.descriptor, + &self.owner, + &self.installation, + now_ns, + )?; + if payload.capture.context() != context { + return Err("recovery_context_mismatch".into()); + } + let restored = CaptureSession::restore_checkpoint(payload.capture, context, now_ns)?; + let model = payload + .model_sha256 + .as_ref() + .map(|sha| self.store.load_exact_candidate(context, sha)) + .transpose()?; + state.generation = state + .generation + .checked_add(1) + .ok_or("learner generation exhausted")?; + state.capture = Some(restored.session); + state.model = model.map(Arc::new); + state.receipt = None; + state.frozen_training = payload.frozen_training; + state.job = Job::default(); + state.job.phase = if busy(&Job { + phase: payload.job_phase, + ..Job::default() + }) { + JobPhase::Failed + } else { + payload.job_phase + }; + if state.job.phase == JobPhase::Failed { + state.job.error = Some( + "Saved training was interrupted or failed; retry explicitly using the frozen data." + .into(), + ); + } + state.last_evaluation = None; + state.evaluation_warning = None; + state.input_error = None; + state.recovery = RecoveryState { + enabled: true, + descriptor: Some(loaded.descriptor), + disposition: Some(restored.disposition), + ..Default::default() + }; + self.checkpoint_recovery(&mut state, now_ns, true) + } + pub fn discard_capture_recovery( + &self, + expected: &MutationGuard, + recovery_id: &str, + ) -> Result<(), String> { + let mut state = self.command_state(expected)?; + // Fence every later ingest/status/worker save before touching the file. + state.recovery.enabled = false; + state.recovery.marker = None; + let result = self + .recovery_store + .as_ref() + .ok_or("recovery_unavailable")? + .discard(recovery_id); + match result { + Ok(()) => { + state.recovery = RecoveryState::default(); + Ok(()) + } + Err(error) => { + state.recovery.error = Some(bounded(error.clone())); + Err(error) + } + } + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_tests.rs new file mode 100644 index 0000000000..2b3621a395 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_tests.rs @@ -0,0 +1,621 @@ +//! SYNTHETIC process-loss and disk-failure integration; no real account or clock. +use super::*; +use crate::room_learner::{capture::RawSourceFrame, recovery_store::RecoveryStore}; +const BASE: u64 = 1_800_000_000_000_000_000; +fn runtime(root: &Path, now: u64, key: bool) -> Arc { + let owner = "a".repeat(64); + let path = root.join(&owner).join("SensingServer"); + let seed = [7u8; 32]; + let store = + RecoveryStore::new(&path, &owner, "synthetic-install", key.then_some(&seed)).unwrap(); + Arc::new( + RoomLearningRuntime::new_with_recovery_at(&path, &owner, "synthetic-install", store, now) + .unwrap(), + ) +} +fn context() -> CaptureContext { + super::tests::context() +} +fn request(duration: u64) -> PhaseRequest { + PhaseRequest { + trial_id: "original-empty".into(), + session_id: "original-train".into(), + role: DataRole::Train, + label: RoomLabel::Empty, + duration_ms: duration, + lead_in_ms: 0, + conditions_confirmed: true, + } +} +fn window(rt: &RoomLearningRuntime, at: u64, sequence: u32) { + for row in 0..8 { + for source in 1..=2 { + let values = vec![6.0 + f64::from(source) * 0.3; 8]; + rt.ingest( + RawSourceFrame { + source_id: source, + grid_id: "ht64-canonical8", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: sequence + row, + amplitudes: &values, + }, + at + u64::from(row) * 10_000_000 + u64::from(source), + ) + .unwrap(); + } + } +} +#[test] +fn completed_six_hundred_second_capture_survives_process_loss_without_auto_confirmation() { + let temp = tempfile::tempdir().unwrap(); + let rt = runtime(temp.path(), BASE, true); + let ctx = context(); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + ctx.clone(), + "original-run".into(), + true, + true, + BASE, + ) + .unwrap(); + rt.begin_phase(&rt.mutation_guard().unwrap(), request(600_000), BASE) + .unwrap(); + for quarter in 0..4 { + window( + &rt, + BASE + u64::from(quarter) * 150_000_000_000 + 1_000_000, + quarter * 8 + 1, + ); + } + let complete = rt.status(BASE + 600_000_000_000); + assert_eq!(complete["recovery"]["saved"]["phase"]["complete"], true); + let id = complete["recovery"]["discard_id"] + .as_str() + .unwrap() + .to_owned(); + let original_created = complete["recovery"]["saved"]["created_at_ms"].clone(); + let original_expiry = complete["recovery"]["saved"]["expires_at_ms"].clone(); + drop(rt); // The only process state holding live/pending buffers is gone. + let cold = runtime(temp.path(), BASE + 601_000_000_000, true); + assert!(cold.context().is_none()); + cold.restore_capture_recovery( + &cold.mutation_guard().unwrap(), + &id, + &ctx, + BASE + 601_000_000_000, + ) + .unwrap(); + let restored = cold.status(BASE + 601_000_000_000); + assert_eq!( + restored["recovery"]["restored_disposition"], + "completed_needs_confirmation" + ); + assert_eq!( + restored["recovery"]["saved"]["created_at_ms"], + original_created + ); + assert_eq!( + restored["recovery"]["saved"]["expires_at_ms"], + original_expiry + ); + assert_eq!(restored["capture"]["sealed_cohorts"], 0); + assert!(restored["capture"]["sources"] + .as_array() + .unwrap() + .iter() + .all(|s| s["live_buffer_frames"] == 0)); + assert!(cold + .finish_phase( + &cold.mutation_guard().unwrap(), + BASE + 601_000_000_000, + true + ) + .is_err()); + window(&cold, BASE + 601_000_000_001, 33); + cold.finish_phase( + &cold.mutation_guard().unwrap(), + BASE + 601_100_000_000, + true, + ) + .unwrap(); + assert_eq!( + cold.status(BASE + 601_100_000_000)["capture"]["sealed_cohorts"], + 4 + ); + assert_eq!(restored["raw_iq_persisted"], false); + assert_eq!(restored["derived_samples_persisted"], true); + assert_eq!(restored["physically_validated"], false); +} +#[test] +fn unavailable_key_blocks_opt_in_without_starting_capture_and_ram_remains_explicit() { + let temp = tempfile::tempdir().unwrap(); + let rt = runtime(temp.path(), BASE, false); + assert!(rt + .configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "run".into(), + true, + true, + BASE + ) + .is_err()); + assert!(rt.context().is_none()); + assert_eq!(rt.status(BASE)["recovery"]["state"], "unavailable"); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "ram".into(), + true, + false, + BASE, + ) + .unwrap(); + assert_eq!( + rt.status(BASE)["untrained_capture_progress_is_volatile"], + true + ); +} +#[test] +fn failed_initial_save_is_visible_and_blocks_long_capture_until_explicit_ram_configure() { + let temp = tempfile::tempdir().unwrap(); + let rt = runtime(temp.path(), BASE, true); + let dir = temp + .path() + .join("a".repeat(64)) + .join("SensingServer/room-learner-recovery"); + // A conflicting non-file destination is an owned fixture only. + let file = dir.join("capture-recovery-v1.bin"); + std::fs::create_dir(&file).unwrap(); + assert!(rt + .configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "blocked".into(), + true, + true, + BASE + ) + .is_err()); + assert!(rt + .begin_phase(&rt.mutation_guard().unwrap(), request(600_000), BASE) + .is_err()); + assert_ne!(rt.status(BASE)["recovery"]["state"], "saved"); + assert!(file.is_dir()); +} +#[test] +fn discard_fences_later_status_and_ingress_writes_and_does_not_remove_models() { + let temp = tempfile::tempdir().unwrap(); + let rt = runtime(temp.path(), BASE, true); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "retained".into(), + true, + true, + BASE, + ) + .unwrap(); + let id = rt.status(BASE)["recovery"]["discard_id"] + .as_str() + .unwrap() + .to_owned(); + rt.discard_capture_recovery(&rt.mutation_guard().unwrap(), &id) + .unwrap(); + window(&rt, BASE + 100_000_000, 1); + let status = rt.status(BASE + 200_000_000); + assert_eq!(status["recovery"]["enabled"], false); + assert!(status["recovery"]["saved"].is_null()); + let owner = "a".repeat(64); + let path = temp.path().join(&owner).join("SensingServer"); + assert!( + RecoveryStore::new(&path, &owner, "synthetic-install", Some(&[7; 32])) + .unwrap() + .inspect() + .unwrap() + .is_none() + ); +} + +#[cfg(unix)] +#[test] +fn failed_completion_checkpoint_retries_after_bounded_backoff_with_original_expiry() { + use std::os::unix::fs::PermissionsExt; + let temp = tempfile::tempdir().unwrap(); + let rt = runtime(temp.path(), BASE, true); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "retry-run".into(), + true, + true, + BASE, + ) + .unwrap(); + rt.begin_phase(&rt.mutation_guard().unwrap(), request(600_000), BASE) + .unwrap(); + for quarter in 0..4 { + window( + &rt, + BASE + u64::from(quarter) * 150_000_000_000 + 1_000_000, + quarter * 8 + 1, + ); + } + let expiry = rt.status(BASE + 450_100_000_000)["recovery"]["saved"]["expires_at_ms"].clone(); + let file = temp + .path() + .join("a".repeat(64)) + .join("SensingServer/room-learner-recovery/capture-recovery-v1.bin"); + std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o644)).unwrap(); + let failed = rt.status(BASE + 600_000_000_000); + assert_eq!(failed["recovery"]["state"], "error"); + assert_persisted_flags(&failed, Value::Null); + assert_eq!(failed["recovery"]["saved"]["phase"]["complete"], false); + std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o600)).unwrap(); + assert_eq!( + rt.status(BASE + 601_000_000_000)["recovery"]["state"], + "error" + ); + let retried = rt.status(BASE + 605_000_000_000); + assert_eq!(retried["recovery"]["state"], "saved"); + assert_eq!(retried["recovery"]["saved"]["phase"]["complete"], true); + assert_eq!(retried["recovery"]["saved"]["expires_at_ms"], expiry); + rt.finish_phase(&rt.mutation_guard().unwrap(), BASE + 605_000_000_000, true) + .unwrap(); + assert_eq!( + rt.status(BASE + 605_000_000_000)["capture"]["sealed_cohorts"], + 4 + ); +} +#[test] +fn expired_slot_is_summary_redacted_but_explicitly_discardable() { + let temp = tempfile::tempdir().unwrap(); + let rt = runtime(temp.path(), BASE, true); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "expiry-run".into(), + true, + true, + BASE, + ) + .unwrap(); + drop(rt); + let cold = runtime(temp.path(), BASE + 86_400_000_000_000, true); + let status = cold.status(BASE + 86_400_000_000_000); + assert_eq!(status["recovery"]["state"], "expired"); + assert!(status["recovery"]["saved"].is_null()); + let id = status["recovery"]["discard_id"].as_str().unwrap(); + assert!(cold + .saved_recovery_context(id, BASE + 86_400_000_000_000) + .is_err()); + cold.discard_capture_recovery(&cold.mutation_guard().unwrap(), id) + .unwrap(); + assert!(cold.status(BASE + 86_400_000_000_000)["recovery"]["discard_id"].is_null()); +} + +#[tokio::test] +async fn trained_checkpoint_cold_restores_exact_candidate_without_live_buffers_or_auto_fit() { + let temp = tempfile::tempdir().unwrap(); + let now = super::tests::START; + let rt = runtime(temp.path(), now, true); + let ctx = context(); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + ctx.clone(), + "fit-run".into(), + true, + true, + now, + ) + .unwrap(); + super::tests::train_phases(&rt); + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 4, + ..FitConfig::default() + }, + ) + .unwrap(); + let ready = super::tests::finished(&rt).await; + assert_eq!(ready["state"], "ready", "{ready}"); + let id = ready["recovery"]["discard_id"].as_str().unwrap().to_owned(); + let expected = + super::recovery_control::model_sha(rt.inner.lock().unwrap().model.as_ref().unwrap()) + .unwrap(); + // Move today's candidate pointer after the recovery was saved. Recovery + // must still load the immutable candidate bound by its original hash. + let dataset = rt + .inner + .lock() + .unwrap() + .capture + .as_ref() + .unwrap() + .validated_dataset() + .unwrap(); + let (replacement, _) = RoomModel::fit_with_cancel( + &dataset, + FitConfig { + epochs: 1, + ..FitConfig::default() + }, + &AtomicBool::new(false), + ) + .unwrap(); + assert_ne!( + super::recovery_control::model_sha(&replacement).unwrap(), + expected + ); + rt.store.save_candidate(&replacement).unwrap(); + drop(rt); + let cold = runtime(temp.path(), now + 61_000_000_000, true); + cold.restore_capture_recovery( + &cold.mutation_guard().unwrap(), + &id, + &ctx, + now + 61_000_000_000, + ) + .unwrap(); + let status = cold.status(now + 61_000_000_000); + assert_eq!(status["state"], "ready"); + assert_eq!(status["training_frozen"], true); + assert_eq!(status["capture"]["sealed_cohorts"], 16); + assert_eq!( + super::recovery_control::model_sha(cold.inner.lock().unwrap().model.as_ref().unwrap()) + .unwrap(), + expected + ); + assert!(cold.prediction(now + 61_000_000_000).is_err()); + assert!(cold + .begin_phase( + &cold.mutation_guard().unwrap(), + request(12_000), + now + 61_000_000_000 + ) + .is_err()); + assert_eq!(status["physically_validated"], false); +} +#[test] +fn interrupted_fit_checkpoint_restores_failed_frozen_dataset_without_job() { + let temp = tempfile::tempdir().unwrap(); + let now = super::tests::START; + let rt = runtime(temp.path(), now, true); + let ctx = context(); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + ctx.clone(), + "interrupted-fit".into(), + true, + true, + now, + ) + .unwrap(); + super::tests::train_phases(&rt); + // Exact durable boundary immediately before handing work to the fitter. + { + let mut state = rt.inner.lock().unwrap(); + state.frozen_training = true; + state.job.phase = JobPhase::Running; + rt.checkpoint_recovery(&mut state, now + 60_000_000_000, true) + .unwrap(); + } + let id = rt.status(now + 60_000_000_000)["recovery"]["discard_id"] + .as_str() + .unwrap() + .to_owned(); + drop(rt); + let cold = runtime(temp.path(), now + 61_000_000_000, true); + cold.restore_capture_recovery( + &cold.mutation_guard().unwrap(), + &id, + &ctx, + now + 61_000_000_000, + ) + .unwrap(); + let status = cold.status(now + 61_000_000_000); + assert_eq!(status["state"], "failed"); + assert_eq!(status["training_frozen"], true); + assert_eq!(status["capture"]["sealed_cohorts"], 16); + assert!(cold.inner.lock().unwrap().model.is_none()); +} + +#[test] +fn partial_phase_restores_restart_required_and_ram_run_preserves_saved_slot() { + let temp = tempfile::tempdir().unwrap(); + let rt = runtime(temp.path(), BASE, true); + let ctx = context(); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + ctx.clone(), + "partial-run".into(), + true, + true, + BASE, + ) + .unwrap(); + rt.begin_phase(&rt.mutation_guard().unwrap(), request(600_000), BASE) + .unwrap(); + window(&rt, BASE + 1_000_000, 1); + let id = rt.status(BASE + 100_000_000)["recovery"]["discard_id"] + .as_str() + .unwrap() + .to_owned(); + drop(rt); + let cold = runtime(temp.path(), BASE + 1_000_000_000, true); + cold.configure_for_recovery( + &cold.mutation_guard().unwrap(), + ctx.clone(), + "explicit-ram".into(), + true, + false, + BASE + 1_000_000_000, + ) + .unwrap(); + assert_eq!( + cold.status(BASE + 1_000_000_000)["recovery"]["discard_id"], + id + ); + cold.restore_capture_recovery( + &cold.mutation_guard().unwrap(), + &id, + &ctx, + BASE + 1_000_000_000, + ) + .unwrap(); + let s = cold.status(BASE + 1_000_000_000); + assert_eq!( + s["recovery"]["restored_disposition"], + "interrupted_needs_restart" + ); + assert_eq!(s["capture"]["sealed_cohorts"], 0); + assert!(s["capture"]["phase"].is_null()); + assert_eq!(s["recovery"]["saved"]["phase"], Value::Null); +} +#[test] +fn failed_erase_is_error_and_disables_future_writer_without_claiming_data_removed() { + use std::os::unix::fs::PermissionsExt; + let temp = tempfile::tempdir().unwrap(); + let rt = runtime(temp.path(), BASE, true); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "erase-fail".into(), + true, + true, + BASE, + ) + .unwrap(); + let id = rt.status(BASE)["recovery"]["discard_id"] + .as_str() + .unwrap() + .to_owned(); + let file = temp + .path() + .join("a".repeat(64)) + .join("SensingServer/room-learner-recovery/capture-recovery-v1.bin"); + std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o644)).unwrap(); + assert!(rt + .discard_capture_recovery(&rt.mutation_guard().unwrap(), &id) + .is_err()); + let s = rt.status(BASE + 10_000_000_000); + assert_eq!(s["recovery"]["enabled"], false); + assert_eq!(s["recovery"]["state"], "error"); + assert_eq!(s["recovery"]["discard_id"], id); + assert!(file.exists()); + std::fs::set_permissions(&file, std::fs::Permissions::from_mode(0o600)).unwrap(); + rt.discard_capture_recovery(&rt.mutation_guard().unwrap(), &id) + .unwrap(); + assert!(!file.exists()); +} + +fn assert_persisted_flags(status: &Value, expected: Value) { + assert_eq!(status["derived_samples_persisted"], expected, "{status}"); + assert_eq!(status["raw_observations_persisted"], expected); + assert_eq!(status["recovery"]["derived_samples_persisted"], expected); + assert_eq!(status["raw_iq_persisted"], false); + assert_eq!(status["recovery"]["raw_iq_persisted"], false); +} +fn assert_unverified_retained_contents(case: &str) { + let temp = tempfile::tempdir().unwrap(); + let rt = runtime(temp.path(), BASE, true); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "retained-flags".into(), + true, + true, + BASE, + ) + .unwrap(); + rt.begin_phase(&rt.mutation_guard().unwrap(), request(600_000), BASE) + .unwrap(); + window(&rt, BASE + 1_000_000, 1); + assert_persisted_flags(&rt.status(BASE + 100_000_000), json!(true)); + drop(rt); + let owner = "a".repeat(64); + let path = temp.path().join(&owner).join("SensingServer"); + let file = path.join("room-learner-recovery/capture-recovery-v1.bin"); + if case == "corrupt" { + let mut bytes = std::fs::read(&file).unwrap(); + *bytes.last_mut().unwrap() ^= 1; + std::fs::write(&file, bytes).unwrap(); + } + let retained = std::fs::read(&file).unwrap(); + let now = if case == "expired" { + BASE + 86_400_000_000_000 + } else { + BASE + 1_000_000_000 + }; + let seed = if case == "wrong_key" { + [8; 32] + } else { + [7; 32] + }; + let store = RecoveryStore::new( + &path, + &owner, + "synthetic-install", + (case != "missing_key").then_some(&seed), + ) + .unwrap(); + let cold = + RoomLearningRuntime::new_with_recovery_at(&path, &owner, "synthetic-install", store, now) + .unwrap(); + let status = cold.status(now); + assert_persisted_flags(&status, Value::Null); + assert!(status["recovery"]["error"].is_string()); + assert!(status["recovery"]["saved"].is_null()); + assert_eq!( + std::fs::read(&file).unwrap(), + retained, + "{case} altered retained encrypted bytes" + ); +} +#[test] +fn persisted_flags_expired_contents_are_unknown_and_retained() { + assert_unverified_retained_contents("expired"); +} +#[test] +fn persisted_flags_corrupt_contents_are_unknown_and_retained() { + assert_unverified_retained_contents("corrupt"); +} +#[test] +fn persisted_flags_wrong_key_contents_are_unknown_and_retained() { + assert_unverified_retained_contents("wrong_key"); +} +#[test] +fn persisted_flags_missing_key_contents_are_unknown_and_retained() { + assert_unverified_retained_contents("missing_key"); +} +#[test] +fn persisted_flags_confirmed_empty_discard_and_default_ram_are_false() { + let temp = tempfile::tempdir().unwrap(); + let ram = super::tests::runtime(temp.path()); + assert_persisted_flags(&ram.status(BASE), json!(false)); + drop(ram); + let rt = runtime(temp.path(), BASE, true); + assert_persisted_flags(&rt.status(BASE), json!(false)); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "verified-empty".into(), + true, + true, + BASE, + ) + .unwrap(); + let empty = rt.status(BASE); + assert_persisted_flags(&empty, json!(false)); + let id = empty["recovery"]["discard_id"].as_str().unwrap(); + rt.begin_phase(&rt.mutation_guard().unwrap(), request(600_000), BASE) + .unwrap(); + window(&rt, BASE + 1_000_000, 1); + assert_persisted_flags(&rt.status(BASE + 100_000_000), json!(true)); + rt.discard_capture_recovery(&rt.mutation_guard().unwrap(), id) + .unwrap(); + assert_persisted_flags(&rt.status(BASE), json!(false)); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs index 3eb175297e..7449bf488d 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs @@ -16,10 +16,11 @@ use ingress::native_candidates; pub(super) use ingress::{observe, LiveSource, NativeSourceKey}; fn failure(code: &str, error: impl ToString) -> Json { - let code = if error.to_string() == "learner_state_changed" { - "learner_state_changed" - } else { - code + let message = error.to_string(); + let code = match message.as_str() { + "learner_state_changed" => "learner_state_changed", + "learner_recovery_pending" => "learner_recovery_pending", + _ => code, }; Json(serde_json::json!({"success":false,"error_code":code, "error":error.to_string().chars().filter(|c|!c.is_control()).take(256).collect::()})) @@ -50,6 +51,8 @@ pub(super) struct ConfigureRequest { revision: u64, source_node_ids: Vec, new_training: bool, + #[serde(default)] + recovery_consent: bool, } fn source_ready(state: &AppStateInner, spec: &SourceSpec, now: std::time::Instant) -> bool { native_candidates(state, spec.source_id, now) @@ -122,11 +125,13 @@ pub(super) async fn configure( }; let context = saved.context.clone(); drop(s); - return match rt.configure( + return match rt.configure_for_recovery( &request.expected, context, opaque_calibration_id("learner-resume"), false, + request.recovery_consent, + now_ns(&rt), ) { Ok(()) => Json(serde_json::json!({"success":true,"status":rt.status(now_ns(&rt))})), Err(e) => failure("learner_resume_failed", e), @@ -180,7 +185,14 @@ pub(super) async fn configure( Err(e) => return failure("learner_context_invalid", e), }; let run = opaque_calibration_id("learner-run"); - match rt.configure(&request.expected, context, run, request.new_training) { + match rt.configure_for_recovery( + &request.expected, + context, + run, + request.new_training, + request.recovery_consent, + now_ns(&rt), + ) { Ok(()) => Json(serde_json::json!({"success":true,"status":rt.status(now_ns(&rt))})), Err(e) => failure("learner_configure_failed", e), } @@ -337,6 +349,91 @@ pub(super) async fn prediction(State(state): State) -> Json, +} +pub(super) async fn restore_capture( + State(state): State, + Json(request): Json, +) -> Json { + let rt = match runtime(&state).await { + Ok(r) => r, + Err(e) => return e, + }; + let now = now_ns(&rt); + let context = match rt.saved_recovery_context(&request.recovery_id, now) { + Ok(c) => c, + Err(e) => return failure("learner_recovery_restore_failed", e), + }; + if context.room_digest != request.binding_digest + || context.revision != request.revision + || context + .sources + .iter() + .map(|s| s.source_id) + .collect::>() + != request.source_node_ids + { + return failure( + "learner_recovery_context_mismatch", + "Saved capture does not match the requested room, revision and source set.", + ); + } + let s = state.read().await; + if s.field_model.as_ref().is_some_and(|f| { + matches!(f.status(), CalibrationStatus::Collecting) + || (s.calibration_session_id.is_some() + && matches!(f.status(), CalibrationStatus::Uncalibrated)) + }) { + return failure( + "learner_capture_conflict", + "Finish or cancel the active field calibration first.", + ); + } + if context + .sources + .iter() + .any(|spec| !source_ready(&s, spec, std::time::Instant::now())) + { + return failure( + "learner_recovery_source_unready", + "Restore the exact saved native source/PHY streams before recovering capture.", + ); + } + let restored = + rt.restore_capture_recovery(&request.expected, &request.recovery_id, &context, now); + drop(s); + match restored { + Ok(()) => Json(serde_json::json!({"success":true,"status":rt.status(now_ns(&rt))})), + Err(e) => failure("learner_recovery_restore_failed", e), + } +} +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct DiscardCaptureRequest { + expected: MutationGuard, + recovery_id: String, +} +pub(super) async fn discard_capture( + State(state): State, + Json(request): Json, +) -> Json { + let rt = match runtime(&state).await { + Ok(r) => r, + Err(e) => return e, + }; + match rt.discard_capture_recovery(&request.expected, &request.recovery_id) { + Ok(()) => Json(serde_json::json!({"success":true,"status":rt.status(now_ns(&rt))})), + Err(e) => failure("learner_recovery_discard_failed", e), + } +} + #[cfg(test)] #[path = "room_learner_api_tests.rs"] mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_recovery_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_recovery_tests.rs new file mode 100644 index 0000000000..3688a5a8a3 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_recovery_tests.rs @@ -0,0 +1,160 @@ +//! SYNTHETIC actual HTTP handlers; no listeners, user data or hardware. +use super::*; +use wifi_densepose_sensing_server::room_learner::recovery_store::RecoveryStore; +async fn recovery_fixture() -> (tempfile::TempDir, SharedState) { + let (dir, state) = fixture(); + replace_runtime(dir.path(), &state).await; + (dir, state) +} +async fn replace_runtime(root: &std::path::Path, state: &SharedState) { + let owner = "a".repeat(64); + let path = root.join(&owner).join("SensingServer"); + let store = RecoveryStore::new(&path, &owner, "synthetic-install", Some(&[7; 32])).unwrap(); + state.write().await.room_learning = Some(Arc::new( + RoomLearningRuntime::new_with_recovery(&path, &owner, "synthetic-install", store).unwrap(), + )); +} +fn recovery_app(state: SharedState) -> Router { + Router::new() + .route("/configure", post(configure)) + .route("/status", get(status)) + .route("/phase", post(begin_phase)) + .route("/finish", post(finish_phase)) + .route("/recovery/restore", post(restore_capture)) + .route("/recovery/discard", post(discard_capture)) + .with_state(state) +} +fn recovery_config() -> serde_json::Value { + let mut c = config(); + c["recovery_consent"] = true.into(); + c +} +#[tokio::test] +async fn http_recovery_is_explicit_guarded_exact_and_metadata_only() { + let (dir, state) = recovery_fixture().await; + let router = recovery_app(state.clone()); + let configured = json_request( + router.clone(), + "/configure", + Method::POST, + recovery_config(), + ) + .await; + assert_eq!(configured["success"], true, "{configured}"); + assert_eq!(configured["status"]["derived_samples_persisted"], false); + let id = configured["status"]["recovery"]["discard_id"].clone(); + let oldguard = configured["status"]["mutation_guard"].clone(); + assert_eq!( + json_request( + router.clone(), + "/configure", + Method::POST, + recovery_config() + ) + .await["error_code"], + "learner_recovery_pending" + ); + replace_runtime(dir.path(), &state).await; + let cold = json_request_status(router.clone()).await; + assert!(cold["context_digest"].is_null()); + assert_eq!(cold["recovery"]["state"], "restore_required"); + let mut restore = serde_json::json!({"recovery_id":id,"binding_digest":"b".repeat(64),"revision":1,"source_node_ids":[15,16,17]}); + restore["expected"] = oldguard; + assert_eq!( + json_request( + router.clone(), + "/recovery/restore", + Method::POST, + restore.clone() + ) + .await["error_code"], + "learner_state_changed" + ); + restore["expected"] = cold["mutation_guard"].clone(); + restore["revision"] = 2.into(); + assert_eq!( + json_request( + router.clone(), + "/recovery/restore", + Method::POST, + restore.clone() + ) + .await["error_code"], + "learner_recovery_context_mismatch" + ); + restore["revision"] = 1.into(); + let recovered = json_request(router.clone(), "/recovery/restore", Method::POST, restore).await; + assert_eq!(recovered["success"], true, "{recovered}"); + assert_eq!(recovered["status"]["recovery"]["automatic_resume"], false); + assert!(recovered["status"]["capture"]["sources"] + .as_array() + .unwrap() + .iter() + .all(|s| s["live_buffer_frames"] == 0)); + let encoded = serde_json::to_string(&recovered).unwrap(); + for forbidden in [ + "amplitudes", + "plaintext", + "source_ordinals", + "cohort_id", + "signing_seed", + ] { + assert!(!encoded.contains(forbidden), "{forbidden}"); + } + let guard = recovered["status"]["mutation_guard"].clone(); + assert_eq!( + json_request( + router.clone(), + "/recovery/discard", + Method::POST, + serde_json::json!({"expected":guard,"recovery_id":"wrong"}) + ) + .await["success"], + false + ); + let discarded = json_request( + router.clone(), + "/recovery/discard", + Method::POST, + serde_json::json!({"recovery_id":id}), + ) + .await; + assert_eq!(discarded["success"], true, "{discarded}"); + assert!(discarded["status"]["recovery"]["saved"].is_null()); +} +#[tokio::test] +async fn http_recovery_rejects_unready_native_context_and_privacy_without_deleting_slot() { + let (_dir, state) = recovery_fixture().await; + let router = recovery_app(state.clone()); + let configured = json_request( + router.clone(), + "/configure", + Method::POST, + recovery_config(), + ) + .await; + let id = configured["status"]["recovery"]["discard_id"].clone(); + state.write().await.room_learning_sources.clear(); + let restore = serde_json::json!({"recovery_id":id,"binding_digest":"b".repeat(64),"revision":1,"source_node_ids":[15,16,17]}); + assert_eq!( + json_request(router.clone(), "/recovery/restore", Method::POST, restore).await + ["error_code"], + "learner_recovery_source_unready" + ); + state.write().await.privacy_mode = true; + assert_eq!( + json_request( + router.clone(), + "/recovery/discard", + Method::POST, + serde_json::json!({"recovery_id":id}) + ) + .await["error_code"], + "learner_privacy_mode" + ); + state.write().await.privacy_mode = false; + assert_eq!( + json_request_status(router).await["recovery"]["discard_id"], + id + ); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs index 2bcf8c18fe..ef9f5b0497 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs @@ -197,6 +197,8 @@ fn every_new_mutation_requires_admin_and_default_cli_does_not_enable_capture() { "train", "train/cancel", "evaluate", + "recovery/restore", + "recovery/discard", ] { assert_eq!( required_scope_for(&Method::POST, &format!("/api/v1/room-learner/{suffix}")), @@ -529,3 +531,7 @@ async fn cache_pressure_preserves_selected_active_class_and_freshness() { configured["status"]["mutation_guard"] ); } + +#[cfg(unix)] +#[path = "room_learner_api_recovery_tests.rs"] +mod recovery_tests; From f4ec743e368b29839cee0cded3e52743ddae48af Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Mon, 2 Nov 2026 11:42:21 +0100 Subject: [PATCH 046/100] docs(learning): propose explicit encrypted capture recovery boundary --- ...R-420-managed-source-aware-room-learner.md | 65 +++++++++++++++++-- 1 file changed, 61 insertions(+), 4 deletions(-) diff --git a/docs/adr/ADR-420-managed-source-aware-room-learner.md b/docs/adr/ADR-420-managed-source-aware-room-learner.md index 7b9ca615f6..32b6d7ff31 100644 --- a/docs/adr/ADR-420-managed-source-aware-room-learner.md +++ b/docs/adr/ADR-420-managed-source-aware-room-learner.md @@ -100,9 +100,10 @@ leave the UI busy. Preparation is excluded from labelled collection. Completion requires the operator's conditions-held acknowledgement. A completed recording interval can -remain provisional while that acknowledgement is pending. Untrained observations -stay in bounded memory under the current retention policy and can be lost if the -helper or account runtime closes; the UI must disclose that limitation. +remain provisional while that acknowledgement is pending. By default, untrained +observations stay in bounded memory and can be lost if the helper or account +runtime closes. The proposed explicit-consent recovery extension below is a +narrow local-storage exception; it is not enabled by an app update or helper flag. Store trained candidates privately beneath the verified owner/installation namespace, with immutable model versions, hashes and an exact-context pointer. @@ -122,6 +123,62 @@ Corrupt evaluation evidence must be retained/quarantined without discarding a valid candidate or manufacturing a pass. Evaluation receipts authorize neither physical qualification nor adoption. +### Proposed November 2 extension: opt-in provisional capture recovery + +The operator can explicitly choose encrypted local recovery when configuring a +run. This retains bounded **reduced magnitude training samples** in selected +representative/sealed cohorts, original source/context/role/time identities and +sequence watermarks. It excludes raw IQ/CSI packet files, camera/depth images, +teacher frames, live receiver windows and pending packet buffers. It is not a +blanket relaxation of the default no-observation-retention policy. API metadata +must disclose when derived samples persist rather than keeping a false raw-data +absence flag. + +Use the existing per-owner persistent Keychain signing seed only as input to a +domain-separated HKDF purpose key bound to owner and installation. AES-256-GCM +uses a fresh nonce and authenticates the descriptor. No key enters JavaScript, +HTTP responses or diagnostics; the existing native-to-helper secret mechanism +is unchanged. A nonsecret stability marker is true only for actual persistent +Keychain sources. Missing, ephemeral or unavailable keys cannot silently fall +back to plaintext or claim recoverability. Other platforms without a qualified +private-file/key path keep the RAM-only workflow. + +One owner-bound recovery slot is limited to64MiB of plaintext plus bounded +framing,0700 directories and0600 regular files, with authenticated atomic +write/fsync/rename/directory barriers. Symlinks, unsafe ownership/permissions, +wrong keys, identity/context mismatches and unknown/corrupt formats fail closed. +The original24-hour recovery-availability deadline never renews on save/read/ +resume. Expiry withdraws recovery; it does **not** promise that a stopped app +can automatically erase ciphertext. The UI states that an expired encrypted +copy may remain until explicitly discarded. Authenticated expired records can +be explicitly deleted by their guarded identity; unreadable unknown files remain +blocked rather than being overwritten or silently erased. + +After restart, recovery is explicit and requires the exact current room, +revision, source/PHY contract and verified owner/installation. Restore preserves +original evidence time and role isolation. A completed phase remains pending +operator conditions-held confirmation. An incomplete interrupted phase fails +and must restart; waiting through downtime never creates measured windows. +Restored live/pending buffers are empty, so new measured packets must establish +current freshness before inference. Interrupted fitting becomes an explicit +retry state, never an automatic job. A saved model is restored by its exact +immutable hash, not today's pointer. Neither recovery nor retention grants +physical qualification, statistical independence or adoption. + +Serialize checkpoint writes and disable/delete under the same runtime lock, +with generation/phase guards and bounded failed-write retry. Cancellation or +discard never acknowledges deletion after I/O failure. A late writer cannot +recreate a successfully discarded copy. Saved state is shown only after durable +success; unavailable/unverifiable retained contents report `null` (unknown), not +`false` (absence), in persistence metadata. Confirmed empty/deleted state may +report false. Full account erasure still drains/stops the helper before removing +its tree; narrower checkpoint discard preserves room scans, candidates and +control/evaluation artifacts. + +Software fixtures are not crash/power-loss or signed-device acceptance. The +extension remains Proposed pending privacy/maintainer review and actual Mac/ +iPhone interruption, owner-isolation, expiry and retention verification. + ### Admit optional teachers through a validated reference boundary The initial RF workflow does not activate radar, camera or LiDAR teaching. @@ -171,7 +228,7 @@ from source CI, packaged native verification and end-user validation. - Source admission and training attribution become explicit and testable. - Candidate/model/evidence histories can survive sessions without renewing trust. - Real-world preprocessing errors are visible before spending a room-capture window. -- Bounded RAM-only capture creates a deliberate interruption/restart limitation. +- Default RAM capture has an interruption limitation; explicit recovery adds encrypted sample-retention consent and cold-restore obligations. - Additional sources or modalities require compatible view and reference contracts. - Model capacity, generalization and uncertainty remain empirical questions. From 05ef9e6eaa37c1c1d92c926b5dae31ec3ed645d5 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Tue, 3 Nov 2026 11:25:03 +0100 Subject: [PATCH 047/100] fix(deps): bound ZIP 7 symlink allocation without a framework upgrade --- .github/workflows/security-scan.yml | 3 + v2/Cargo.lock | 50 - v2/Cargo.toml | 6 +- .../zip-7.2.0/.amazonq/rules/rules.md | 14 + .../security/zip-7.2.0/.cargo_vcs_info.json | 6 + v2/vendor/security/zip-7.2.0/.gitattributes | 8 + v2/vendor/security/zip-7.2.0/.gitignore | 6 + v2/vendor/security/zip-7.2.0/.whitesource | 14 + v2/vendor/security/zip-7.2.0/CHANGELOG.md | 860 ++++ .../security/zip-7.2.0/CODE_OF_CONDUCT.md | 76 + v2/vendor/security/zip-7.2.0/CONTRIBUTING.md | 1 + v2/vendor/security/zip-7.2.0/Cargo.lock | 1300 ++++++ v2/vendor/security/zip-7.2.0/Cargo.toml | 330 ++ v2/vendor/security/zip-7.2.0/Cargo.toml.orig | 139 + v2/vendor/security/zip-7.2.0/LICENSE | 24 + v2/vendor/security/zip-7.2.0/README.md | 145 + .../zip-7.2.0/RUSTSEC-2026-0429.patch | 59 + .../security/zip-7.2.0/RUVIEW-BACKPORT.md | 35 + v2/vendor/security/zip-7.2.0/SECURITY.md | 36 + .../zip-7.2.0/benches/merge_archive.rs | 130 + .../security/zip-7.2.0/benches/read_entry.rs | 41 + .../zip-7.2.0/benches/read_metadata.rs | 143 + v2/vendor/security/zip-7.2.0/cliff.toml | 91 + .../zip-7.2.0/pull_request_template.md | 38 + v2/vendor/security/zip-7.2.0/release-plz.toml | 27 + .../security/zip-7.2.0/ruview-provenance.json | 16 + v2/vendor/security/zip-7.2.0/src/aes.rs | 407 ++ v2/vendor/security/zip-7.2.0/src/aes_ctr.rs | 277 ++ .../security/zip-7.2.0/src/compression.rs | 557 +++ v2/vendor/security/zip-7.2.0/src/cp437.rs | 207 + v2/vendor/security/zip-7.2.0/src/crc32.rs | 131 + .../src/extra_fields/extended_timestamp.rs | 111 + .../zip-7.2.0/src/extra_fields/mod.rs | 35 + .../zip-7.2.0/src/extra_fields/ntfs.rs | 97 + .../src/extra_fields/zipinfo_utf8.rs | 40 + .../security/zip-7.2.0/src/legacy/huffman.rs | 260 ++ .../security/zip-7.2.0/src/legacy/implode.rs | 250 + .../security/zip-7.2.0/src/legacy/mod.rs | 4 + .../security/zip-7.2.0/src/legacy/reduce.rs | 313 ++ .../security/zip-7.2.0/src/legacy/shrink.rs | 373 ++ v2/vendor/security/zip-7.2.0/src/lib.rs | 76 + v2/vendor/security/zip-7.2.0/src/macros.rs | 162 + v2/vendor/security/zip-7.2.0/src/path.rs | 110 + v2/vendor/security/zip-7.2.0/src/read.rs | 2650 +++++++++++ .../security/zip-7.2.0/src/read/config.rs | 23 + .../zip-7.2.0/src/read/magic_finder.rs | 291 ++ .../security/zip-7.2.0/src/read/stream.rs | 415 ++ v2/vendor/security/zip-7.2.0/src/result.rs | 143 + v2/vendor/security/zip-7.2.0/src/spec.rs | 873 ++++ v2/vendor/security/zip-7.2.0/src/types.rs | 1768 +++++++ v2/vendor/security/zip-7.2.0/src/unstable.rs | 121 + v2/vendor/security/zip-7.2.0/src/write.rs | 4077 +++++++++++++++++ v2/vendor/security/zip-7.2.0/src/zipcrypto.rs | 280 ++ .../zip-7.2.0/tests/ruview_0429_backport.rs | 143 + 54 files changed, 17741 insertions(+), 51 deletions(-) create mode 100644 v2/vendor/security/zip-7.2.0/.amazonq/rules/rules.md create mode 100644 v2/vendor/security/zip-7.2.0/.cargo_vcs_info.json create mode 100644 v2/vendor/security/zip-7.2.0/.gitattributes create mode 100644 v2/vendor/security/zip-7.2.0/.gitignore create mode 100644 v2/vendor/security/zip-7.2.0/.whitesource create mode 100644 v2/vendor/security/zip-7.2.0/CHANGELOG.md create mode 100644 v2/vendor/security/zip-7.2.0/CODE_OF_CONDUCT.md create mode 100644 v2/vendor/security/zip-7.2.0/CONTRIBUTING.md create mode 100644 v2/vendor/security/zip-7.2.0/Cargo.lock create mode 100644 v2/vendor/security/zip-7.2.0/Cargo.toml create mode 100644 v2/vendor/security/zip-7.2.0/Cargo.toml.orig create mode 100644 v2/vendor/security/zip-7.2.0/LICENSE create mode 100644 v2/vendor/security/zip-7.2.0/README.md create mode 100644 v2/vendor/security/zip-7.2.0/RUSTSEC-2026-0429.patch create mode 100644 v2/vendor/security/zip-7.2.0/RUVIEW-BACKPORT.md create mode 100644 v2/vendor/security/zip-7.2.0/SECURITY.md create mode 100644 v2/vendor/security/zip-7.2.0/benches/merge_archive.rs create mode 100644 v2/vendor/security/zip-7.2.0/benches/read_entry.rs create mode 100644 v2/vendor/security/zip-7.2.0/benches/read_metadata.rs create mode 100644 v2/vendor/security/zip-7.2.0/cliff.toml create mode 100644 v2/vendor/security/zip-7.2.0/pull_request_template.md create mode 100644 v2/vendor/security/zip-7.2.0/release-plz.toml create mode 100644 v2/vendor/security/zip-7.2.0/ruview-provenance.json create mode 100644 v2/vendor/security/zip-7.2.0/src/aes.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/aes_ctr.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/compression.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/cp437.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/crc32.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/extra_fields/extended_timestamp.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/extra_fields/mod.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/extra_fields/ntfs.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/extra_fields/zipinfo_utf8.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/legacy/huffman.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/legacy/implode.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/legacy/mod.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/legacy/reduce.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/legacy/shrink.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/lib.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/macros.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/path.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/read.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/read/config.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/read/magic_finder.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/read/stream.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/result.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/spec.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/types.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/unstable.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/write.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/zipcrypto.rs create mode 100644 v2/vendor/security/zip-7.2.0/tests/ruview_0429_backport.rs diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 4b406ed775..0921727c90 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -40,6 +40,9 @@ jobs: set -o pipefail cargo audit --file Cargo.lock --json | tee cargo-audit.json + - name: Exercise the vendored ZIP symlink allocation guard + run: cargo test --manifest-path v2/vendor/security/zip-7.2.0/Cargo.toml --locked --no-default-features --features deflate --test ruview_0429_backport + - name: Upload Rust advisory report uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 if: always() diff --git a/v2/Cargo.lock b/v2/Cargo.lock index 468b53d44c..7c113923c8 100644 --- a/v2/Cargo.lock +++ b/v2/Cargo.lock @@ -234,9 +234,6 @@ name = "arbitrary" version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" -dependencies = [ - "derive_arbitrary", -] [[package]] name = "arrayref" @@ -2763,17 +2760,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "derive_arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "derive_more" version = "0.99.20" @@ -3396,7 +3382,6 @@ dependencies = [ "crc32fast", "libz-sys", "miniz_oxide", - "zlib-rs", ] [[package]] @@ -6591,7 +6576,6 @@ dependencies = [ "num-complex", "num-traits", "py_literal", - "zip 6.0.0", ] [[package]] @@ -14716,25 +14700,9 @@ dependencies = [ "zstd", ] -[[package]] -name = "zip" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" -dependencies = [ - "arbitrary", - "crc32fast", - "flate2", - "indexmap 2.13.0", - "memchr", - "zopfli", -] - [[package]] name = "zip" version = "7.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0" dependencies = [ "crc32fast", "indexmap 2.13.0", @@ -14742,30 +14710,12 @@ dependencies = [ "typed-path", ] -[[package]] -name = "zlib-rs" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" - [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" -[[package]] -name = "zopfli" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" -dependencies = [ - "bumpalo", - "crc32fast", - "log", - "simd-adler32", -] - [[package]] name = "zstd" version = "0.14.3" diff --git a/v2/Cargo.toml b/v2/Cargo.toml index cad9e28487..fea11579e9 100644 --- a/v2/Cargo.toml +++ b/v2/Cargo.toml @@ -146,6 +146,9 @@ members = [ # excluded for the same reason. Build separately: # cargo build --target wasm32-unknown-unknown --release -p homecore-plugin-example exclude = [ + # ZIP registry archives omit large upstream unit fixtures; run its dedicated + # self-contained security regression target explicitly in CI. + "vendor/security/zip-7.2.0", "crates/wifi-densepose-wasm-edge", "crates/homecore-plugin-example", # ADR-368: napi-rs transport for ruview-kernel. Own workspace so napi never @@ -238,7 +241,7 @@ pcap = "1.1" petgraph = "0.6" # Data loading -ndarray-npy = "0.10" +ndarray-npy = { version = "0.10", default-features = false, features = ["num-complex-0_4"] } walkdir = "2.4" # Hashing (for proof) @@ -345,5 +348,6 @@ debug = true # unchanged; both parents use the fixed zstd 0.14.3 implementation. # See each directory's RUVIEW-PROVENANCE.md for archive digests and scope. [patch.crates-io] +zip-current = { package = "zip", path = "vendor/security/zip-7.2.0" } zip-legacy = { package = "zip", path = "vendor/security/zip-0.6.6" } wasmtime-internal-cache = { path = "vendor/security/wasmtime-internal-cache-36.0.20" } diff --git a/v2/vendor/security/zip-7.2.0/.amazonq/rules/rules.md b/v2/vendor/security/zip-7.2.0/.amazonq/rules/rules.md new file mode 100644 index 0000000000..a96b3eab20 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/.amazonq/rules/rules.md @@ -0,0 +1,14 @@ +- Conform to the ZIP file specification at https://support.pkware.com/pkzip/appnote. +- PR titles must conform to [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) and start + with one of the types specified by the [Angular convention](https://github.com/angular/angular/blob/22b96b9/CONTRIBUTING.md#type). + The type prefix will usually be `fix:`, `feat:`, `perf:`, `docs:` or `chore(deps):`. +- When adding a new compression or decompression method, add a feature flag for it. +- Make sure that if new code has a dependency that's gated on a feature flag, then so is the new code. +- Carefully consider the configurations of feature flags your change will affect, and test accordingly. +- Optimize your code's performance for archives that contain many files *and* archives that contain large files. +- Consider whether your change affects ZIP64 files. If so, test it on 4GiB files both with and without compression. + Generate these files by copying repeatedly to or from the same buffer, so that the test doesn't occupy or allocate + 4GiB of memory. +- Wait and see whether the tests for your PR pass in CI. Fix them if they fail. +- Rebase your PR whenever it has a merge conflict. If you can't fix the rebase conflicts, rewrite it from scratch. +- Always run `cargo fmt --all` and `cargo clippy --all-features --all-targets`. \ No newline at end of file diff --git a/v2/vendor/security/zip-7.2.0/.cargo_vcs_info.json b/v2/vendor/security/zip-7.2.0/.cargo_vcs_info.json new file mode 100644 index 0000000000..509f2182f7 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "46dc29cda6749736ddb1ec502aff06a65ffe54f8" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/v2/vendor/security/zip-7.2.0/.gitattributes b/v2/vendor/security/zip-7.2.0/.gitattributes new file mode 100644 index 0000000000..2b83c1e9d1 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/.gitattributes @@ -0,0 +1,8 @@ +tests/data/** binary=true +tests/data/**/LICENSE.*.txt binary=false +tests/data/folder/** binary=true +tests/data/legacy/** binary=true +fuzz/corpus/** binary=true + +tests/data/folder/first.txt text eol=lf +tests/data/legacy/*.out text eol=crlf \ No newline at end of file diff --git a/v2/vendor/security/zip-7.2.0/.gitignore b/v2/vendor/security/zip-7.2.0/.gitignore new file mode 100644 index 0000000000..eded420b2c --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/.gitignore @@ -0,0 +1,6 @@ +Cargo.lock +target +.DS_Store +\.idea/ +/fuzz_read/out/ +/fuzz_write/out/ diff --git a/v2/vendor/security/zip-7.2.0/.whitesource b/v2/vendor/security/zip-7.2.0/.whitesource new file mode 100644 index 0000000000..9c7ae90b4e --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/.whitesource @@ -0,0 +1,14 @@ +{ + "scanSettings": { + "baseBranches": [] + }, + "checkRunSettings": { + "vulnerableCheckRunConclusionLevel": "failure", + "displayMode": "diff", + "useMendCheckNames": true + }, + "issueSettings": { + "minSeverityLevel": "LOW", + "issueType": "DEPENDENCY" + } +} \ No newline at end of file diff --git a/v2/vendor/security/zip-7.2.0/CHANGELOG.md b/v2/vendor/security/zip-7.2.0/CHANGELOG.md new file mode 100644 index 0000000000..92c6d4d457 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/CHANGELOG.md @@ -0,0 +1,860 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [7.2.0](https://github.com/zip-rs/zip2/compare/v7.1.0...v7.2.0) - 2026-01-20 + +### 🚀 Features + +- add read_zipfile_from_stream_with_compressed_size ([#70](https://github.com/zip-rs/zip2/pull/70)) +- Allow choosing bzip2 rust backend ([#329](https://github.com/zip-rs/zip2/pull/329)) + +### 🐛 Bug Fixes + +- Need to include zip64 extra field in central directory (fix #353) ([#360](https://github.com/zip-rs/zip2/pull/360)) +- Fails to extract file which might or might not be malformed ([#376](https://github.com/zip-rs/zip2/pull/376)) ([#426](https://github.com/zip-rs/zip2/pull/426)) +- *(aes)* Allow AES encryption while streaming ([#463](https://github.com/zip-rs/zip2/pull/463)) +- Default "platform" field in zip files should be set to the local platform, rather than always "Unix" ([#470](https://github.com/zip-rs/zip2/pull/470)) ([#471](https://github.com/zip-rs/zip2/pull/471)) + +### 🚜 Refactor + +- Define cfg_if! and cfg_if_expr! internal macros ([#438](https://github.com/zip-rs/zip2/pull/438)) + +### ⚡ Performance + +- Change an assert to debug_assert when encrypting/decrypting AES, and eliminate a fallible operation ([#521](https://github.com/zip-rs/zip2/pull/521)) +- eliminate a String clone per new file added to archive, and other related refactors ([#522](https://github.com/zip-rs/zip2/pull/522)) + +### ⚙️ Miscellaneous Tasks + +- Fix another merge error, this one affecting only builds with flate2 and not zopfli +- Fix more merge issues +- Fix merge +- Fix write_dir build errors on specific feature configs +- Fix clippy warning +- Fix --all-features build error +- Fix merge + +## [7.1.0](https://github.com/zip-rs/zip2/compare/v7.0.0...v7.1.0) - 2026-01-14 + +### 🚀 Features + +- display the underlying error in Display impl for ZipError::Display ([#483](https://github.com/zip-rs/zip2/pull/483)) +- Enable creation of `ZipArchive` without reparsing ([#485](https://github.com/zip-rs/zip2/pull/485)) + +### 🐛 Bug Fixes + +- Return InvalidPassword rather than panic when AES key is the wrong length ([#457](https://github.com/zip-rs/zip2/pull/457)) +- bench with auto zip64 comment ([#505](https://github.com/zip-rs/zip2/pull/505)) +- add condition for `getrandom` dependency ([#504](https://github.com/zip-rs/zip2/pull/504)) +- *(zipcrypto)* Support streaming ZipCrypto encryption, don't store entire file in memory ([#462](https://github.com/zip-rs/zip2/pull/462)) + +### 🚜 Refactor + +- Clean up imports and move types ([#461](https://github.com/zip-rs/zip2/pull/461)) +- Replace handwritten `Ord` and `PartialOrd` for `DateTime` ([#484](https://github.com/zip-rs/zip2/pull/484)) + +### ⚙️ Miscellaneous Tasks + +- Lock `lzma-rust2` to at least 0.15.5 ([#491](https://github.com/zip-rs/zip2/pull/491)) + +## [7.0.0](https://github.com/zip-rs/zip2/compare/v6.0.0...v7.0.0) - 2025-12-05 + +### ⚠️ Breaking Changes + +- Removed the following features: `getrandom`, `hmac`, `pbkdf2`, `sha1`, `zeroize`. +- Removed `lzma-static` and `xz-static` feature flags, which were deprecated synonyms of `lzma` and `xz`. ([#405](https://github.com/zip-rs/zip2/pull/405), [#425](https://github.com/zip-rs/zip2/pull/425)) + +### 🚀 Features + +- *(`SimpleFileOptions`)* const DEFAULT implementation ([#474](https://github.com/zip-rs/zip2/pull/474)) +- ZipWriter `set_auto_large_file()` method to enable large-file data descriptor when necessary ([#468](https://github.com/zip-rs/zip2/pull/468)) + +### 🐛 Bug Fixes + +- print previous error when failing to search another cde ([#460](https://github.com/zip-rs/zip2/pull/460)) +- cargo doc warnings ([#472](https://github.com/zip-rs/zip2/pull/472)) +- Write ZIP64 data descriptors when large_file option is true ([#467](https://github.com/zip-rs/zip2/pull/467)) +- Pin generic-array to an old version to work around https://github.com/RustCrypto/traits/issues/2036 until next RustCrypto & aes-crypto releases ([#458](https://github.com/zip-rs/zip2/pull/458)) + +### ⚙️ Miscellaneous Tasks + +- Revert version bump so that release-plz will trigger +- expose more flate2 feature flags ([#476](https://github.com/zip-rs/zip2/pull/476)) +- Next release will be 7.0.0 +- release v6.0.0 ([#442](https://github.com/zip-rs/zip2/pull/442)) + +### Deps + +- Bump lzma-rust2 to v0.15 ([#465](https://github.com/zip-rs/zip2/pull/465)) + +## [6.0.0](https://github.com/zip-rs/zip2/compare/v5.1.1...v6.0.0) - 2025-10-09 + +### 🚀 Features + +- Add by_index_with_options(), which can be used to ignore encryption in a file's metadata ([#439](https://github.com/zip-rs/zip2/pull/439)) and may be used for other file-specific overrides in the future. + +### ⚙️ Miscellaneous Tasks + +- [**breaking**] `FileOptions::add_extra_data` is now generic and accepts any `AsRef<[u8]>`. (#435) + +## [5.1.1](https://github.com/zip-rs/zip2/compare/v5.1.0...v5.1.1) - 2025-09-11 + +### 🐛 Bug Fixes + +- panic when reading empty extended-timestamp field ([#404](https://github.com/zip-rs/zip2/pull/404)) ([#422](https://github.com/zip-rs/zip2/pull/422)) +- Restore original file timestamp when unzipping with `chrono` ([#46](https://github.com/zip-rs/zip2/pull/46)) + +### ⚙️ Miscellaneous Tasks + +- Configure Amazon Q rules ([#421](https://github.com/zip-rs/zip2/pull/421)) + +## [5.1.0](https://github.com/zip-rs/zip2/compare/v5.0.1...v5.1.0) - 2025-09-10 + +### 🚀 Features + +- Add legacy shrink/reduce/implode compression ([#303](https://github.com/zip-rs/zip2/pull/303)) + +## [5.0.1](https://github.com/zip-rs/zip2/compare/v5.0.0...v5.0.1) - 2025-09-09 + +### 🐛 Bug Fixes + +- AES metadata was not copied correctly in raw copy methods, which could corrupt the copied file. ([#417](https://github.com/zip-rs/zip2/pull/417)) + +## [5.0.0](https://github.com/zip-rs/zip2/compare/v4.6.1...v5.0.0) - 2025-09-05 + +### 🚀 Features + +- Implement by_path*() methods on ZipArchive ([#382](https://github.com/zip-rs/zip2/pull/382)) + +## [4.6.1](https://github.com/zip-rs/zip2/compare/v4.6.0...v4.6.1) - 2025-09-03 + +### 🐛 Bug Fixes + +- Fixes an issue introduced by the swap from `lzma-rs` to `liblzma` ([#407](https://github.com/zip-rs/zip2/pull/407)) + +## [4.6.0](https://github.com/zip-rs/zip2/compare/v4.5.0...v4.6.0) - 2025-08-30 + +### 🚀 Features + +- Allow to read zip files with unsupported extended timestamps ([#400](https://github.com/zip-rs/zip2/pull/400)) + +### 🐛 Bug Fixes + +- enable clamp_opt for ppmd and xz ([#401](https://github.com/zip-rs/zip2/pull/401)) + +## [4.5.0](https://github.com/zip-rs/zip2/compare/v4.4.0...v4.5.0) - 2025-08-21 + +### 🚀 Features + +- Allow reading ZIP files where the central directory comes *before* the files ([#384](https://github.com/zip-rs/zip2/pull/384)) ([#396](https://github.com/zip-rs/zip2/pull/396)) + +## [4.4.0](https://github.com/zip-rs/zip2/compare/v4.3.0...v4.4.0) - 2025-08-21 + +### 🚀 Features + +- Add `lzma-static` and `xz-static` features that enable `liblzma/static` ([#393](https://github.com/zip-rs/zip2/pull/393)) + +### ⚙️ Miscellaneous Tasks + +- Move deprecated annotations to fix a Clippy warning ([#391](https://github.com/zip-rs/zip2/pull/391)) + +## [4.3.0](https://github.com/zip-rs/zip2/compare/v4.2.0...v4.3.0) - 2025-07-09 + +### 🚀 Features + +- Add support for PPMd ([#370](https://github.com/zip-rs/zip2/pull/370)) + +## [4.2.0](https://github.com/zip-rs/zip2/compare/v4.1.0...v4.2.0) - 2025-06-21 + +### 🚀 Features + +- Write ZIP file to stream ([#246](https://github.com/zip-rs/zip2/pull/246)) + +## [4.1.0](https://github.com/zip-rs/zip2/compare/v4.0.0...v4.1.0) - 2025-06-14 + +### 🚀 Features + +- Add has_overlapping_files method + +## [4.0.0](https://github.com/zip-rs/zip2/compare/v3.0.0...v4.0.0) - 2025-05-21 + +### 🐛 Bug Fixes + +- Allow extraction of Zip64 where "Version needed to extract" is higher than "Version made by" ([#356](https://github.com/zip-rs/zip2/pull/356)) + +### ⚙️ Miscellaneous Tasks + +- Revert nt-time upgrade (would increase MSRV) +- Revert constant_time_eq update (would increase MSRV) +- Update fully-qualified names of liblzma imports + +## [3.0.0](https://github.com/zip-rs/zip2/compare/v2.6.1...v3.0.0) - 2025-05-14 + +### 🐛 Bug Fixes + +- return correct offset in SeekableTake::seek ([#342](https://github.com/zip-rs/zip2/pull/342)) +- When only zopfli is available, decompression of deflate should not be possible ([#348](https://github.com/zip-rs/zip2/pull/348)) +- Specify `flate2` dependency of the `deflate-flate2` feature. ([#345](https://github.com/zip-rs/zip2/pull/345)) + +### ⚙️ Miscellaneous Tasks + +- drop unused crossbeam-utils dependency ([#339](https://github.com/zip-rs/zip2/pull/339)) +- fix typo +- remove `deflate-flate2` dependency on specific backend +- [**breaking**] Drop deprecated `deflate-miniz` feature flag ([#351](https://github.com/zip-rs/zip2/pull/351)) + +## [2.6.1](https://github.com/zip-rs/zip2/compare/v2.6.0...v2.6.1) - 2025-04-03 + +### 🐛 Bug Fixes + +- avoid scanning through all local file headers while opening an archive ([#281](https://github.com/zip-rs/zip2/pull/281)) + +## [2.5.0](https://github.com/zip-rs/zip2/compare/v2.4.2...v2.5.0) - 2025-03-23 + +### 🚀 Features + +- Add support for `time::PrimitiveDateTime` ([#322](https://github.com/zip-rs/zip2/pull/322)) +- Add `jiff` integration ([#323](https://github.com/zip-rs/zip2/pull/323)) + +### 🐛 Bug Fixes + +- improve error message for duplicated file ([#277](https://github.com/zip-rs/zip2/pull/277)) + +## [2.4.2](https://github.com/zip-rs/zip2/compare/v2.4.1...v2.4.2) - 2025-03-18 + +### 🐛 Bug Fixes + +- `deep_copy_file` produced a mangled file header on big-endian platforms (#309) + +## [2.4.1](https://github.com/zip-rs/zip2/compare/v2.4.0...v2.4.1) - 2025-03-17 + +### 🐛 Bug Fixes + +- type issue in test +- double as_ref().canonicalize()? +- CI failures +- Create directory for extraction if necessary ([#314](https://github.com/zip-rs/zip2/pull/314)) + +## [2.4.0](https://github.com/zip-rs/zip2/compare/v2.3.0...v2.4.0) - 2025-03-17 + +### 🚀 Features + +- `ZipArchive::root_dir` and `ZipArchive::extract_unwrapped_root_dir` ([#304](https://github.com/zip-rs/zip2/pull/304)) + +### 🐛 Bug Fixes + +- wasm build failure due to a missing use statement ([#313](https://github.com/zip-rs/zip2/pull/313)) + +## [2.3.0](https://github.com/zip-rs/zip2/compare/v2.2.3...v2.3.0) - 2025-03-16 + +### 🚀 Features + +- Add support for NTFS extra field ([#279](https://github.com/zip-rs/zip2/pull/279)) + +### 🐛 Bug Fixes + +- *(test)* Conditionalize a zip64 doctest ([#308](https://github.com/zip-rs/zip2/pull/308)) +- fix failing tests, remove symlink loop check +- Canonicalize output path to avoid false negatives +- Symlink handling in stream extraction +- Canonicalize output paths and symlink targets, and ensure they descend from the destination + +### ⚙️ Miscellaneous Tasks + +- Fix clippy and cargo fmt warnings ([#310](https://github.com/zip-rs/zip2/pull/310)) + +## [2.2.3](https://github.com/zip-rs/zip2/compare/v2.2.2...v2.2.3) - 2025-02-26 + +### 🚜 Refactor + +- Change the inner structure of `DateTime` (#267) + +### ⚙️ Miscellaneous Tasks + +- cargo fix --edition + +## [2.2.2](https://github.com/zip-rs/zip2/compare/v2.2.1...v2.2.2) - 2024-12-16 + +### 🐛 Bug Fixes + +- rewrite the EOCD/EOCD64 detection to fix extreme performance regression (#247) + +## [2.2.1](https://github.com/zip-rs/zip2/compare/v2.2.0...v2.2.1) - 2024-11-20 + +### 🐛 Bug Fixes + +- remove executable bit ([#238](https://github.com/zip-rs/zip2/pull/238)) +- *(lzma)* fixed panic in case of invalid lzma stream ([#259](https://github.com/zip-rs/zip2/pull/259)) +- resolve new clippy warnings on nightly ([#262](https://github.com/zip-rs/zip2/pull/262)) +- resolve clippy warning in nightly ([#252](https://github.com/zip-rs/zip2/pull/252)) + +### ⚡ Performance + +- Faster cde rejection ([#255](https://github.com/zip-rs/zip2/pull/255)) + +## [2.2.0](https://github.com/zip-rs/zip2/compare/v2.1.6...v2.2.0) - 2024-08-11 + +### 🚀 Features +- Expose `ZipArchive::central_directory_start` ([#232](https://github.com/zip-rs/zip2/pull/232)) + +## [2.1.6](https://github.com/zip-rs/zip2/compare/v2.1.5...v2.1.6) - 2024-07-29 + +### 🐛 Bug Fixes +- ([#33](https://github.com/zip-rs/zip2/pull/33)) Rare combination of settings could lead to writing a corrupt archive with overlength extra data, and data_start locations when reading the archive back were also wrong ([#221](https://github.com/zip-rs/zip2/pull/221)) + +### 🚜 Refactor +- Eliminate some magic numbers and unnecessary path prefixes ([#225](https://github.com/zip-rs/zip2/pull/225)) + +## [2.1.5](https://github.com/zip-rs/zip2/compare/v2.1.4...v2.1.5) - 2024-07-20 + +### 🚜 Refactor +- change invalid_state() return type to io::Result + +## [2.1.4](https://github.com/zip-rs/zip2/compare/v2.1.3...v2.1.4) - 2024-07-18 + +### 🐛 Bug Fixes +- fix([#215](https://github.com/zip-rs/zip2/pull/215)): Upgrade to deflate64 0.1.9 +- Panic when reading a file truncated in the middle of an XZ block header +- Some archives with over u16::MAX files were handled incorrectly or slowly ([#189](https://github.com/zip-rs/zip2/pull/189)) +- Check number of files when deciding whether a CDE is the real one +- Could still select a fake CDE over a real one in some cases +- May have to consider multiple CDEs before filtering for validity +- We now keep searching for a real CDE header after read an invalid one from the file comment +- Always search for data start when opening an archive for append, and reject the header if data appears to start after central directory +- `deep_copy_file` no longer allows overwriting an existing file, to match the behavior of `shallow_copy_file` +- File start position was wrong when extra data was present +- Abort file if central extra data is too large +- Overflow panic when central directory extra data is too large +- ZIP64 header was being written twice when copying a file +- ZIP64 header was being written to central header twice +- Start position was incorrect when file had no extra data +- Allow all reserved headers we can create +- Fix a bug where alignment padding interacts with other extra-data fields +- Fix bugs involving alignment padding and Unicode extra fields +- Incorrect header when adding AES-encrypted files +- Parse the extra field and reject it if invalid +- Incorrect behavior following a rare combination of `merge_archive`, `abort_file` and `deep_copy_file`. As well, we now return an error when a file is being copied to itself. +- path_to_string now properly handles the case of an empty path +- Implement `Debug` for `ZipWriter` even when it's not implemented for the inner writer's type +- Fix an issue where the central directory could be incorrectly detected +- `finish_into_readable()` would corrupt the archive if the central directory had moved + +### 🚜 Refactor +- Verify with debug assertions that no FixedSizeBlock expects a multi-byte alignment ([#198](https://github.com/zip-rs/zip2/pull/198)) +- Use new do_or_abort_file method + +### ⚡ Performance +- Speed up CRC when encrypting small files +- Limit the number of extra fields +- Refactor extra-data validation +- Store extra data in plain vectors until after validation +- Only build one IndexMap after choosing among the possible valid headers +- Simplify validation of empty extra-data fields +- Validate automatic extra-data fields only once, even if several are present +- Remove redundant `validate_extra_data()` call +- Skip searching for the ZIP32 header if a valid ZIP64 header is present ([#189](https://github.com/zip-rs/zip2/pull/189)) + +### ⚙️ Miscellaneous Tasks +- Fix a bug introduced by c934c824 +- Fix a failing unit test +- Fix build errors on older Rust versions +- Fix build +- Fix another fuzz failure +- Switch to `ok_or_abort_file`, and inline when that fails borrow checker +- Switch to `ok_or_abort_file`, and inline when that fails borrow checker +- Fix a build error +- Fix boxed_local warning (can borrow instead) +- Partial debug +- Fix more errors when parsing multiple extra fields +- Fix an error when decoding AES header +- Fix an error caused by not allowing 0xa11e field +- Bug fix: crypto_header was being counted toward extra_data_end +- Bug fix: revert a change where crypto_header was incorrectly treated as an extra field +- Fix a bug where a modulo of 0 was used +- Fix a bug when ZipCrypto, alignment *and* a custom header are used +- Fix a bug when both ZipCrypto and alignment are used +- Fix another bug: header_end vs extra_data_end +- Fix use of a stale value in a `debug_assert_eq!` +- Fix: may still get an incorrect size if opening an invalid file for append +- Fix: may need the absolute start as tiebreaker to ensure deterministic behavior + +## [2.1.3](https://github.com/zip-rs/zip2/compare/v2.1.2...v2.1.3) - 2024-06-04 + +### 🐛 Bug Fixes +- Some date/time filters were previously unreliable (i.e. later-pass filters had no earliest-pass or latest-fail, and vice-versa) +- Decode Zip-Info UTF8 name and comment fields ([#159](https://github.com/zip-rs/zip2/pull/159)) + +### 🚜 Refactor +- Return extended timestamp fields copied rather than borrowed ([#183](https://github.com/zip-rs/zip2/pull/183)) + +### ⚙️ Miscellaneous Tasks +- Fix a new Clippy warning +- Fix a bug and inline `deserialize` for safety +- Add check for wrong-length blocks, and incorporate fixed-size requirement into the trait name +- Fix a fuzz failure by using checked_sub +- Add feature gate for new unit test + +## [2.1.1](https://github.com/zip-rs/zip2/compare/v2.1.0...v2.1.1) - 2024-05-28 + +### 🐛 Bug Fixes +- Derive `Debug` for `ZipWriter` +- lower default version to 4.5 and use the version-needed-to-extract where feasible. + +### 🚜 Refactor +- use a MIN_VERSION constant + +### ⚙️ Miscellaneous Tasks +- Bug fixes for debug implementation +- Bug fixes for debug implementation +- Update unit tests +- Remove unused import + +## [2.1.0](https://github.com/zip-rs/zip2/compare/v2.0.0...v2.1.0) - 2024-05-25 + +### 🚀 Features +- Support mutual conversion between `DateTime` and MS-DOS pair + +### 🐛 Bug Fixes +- version-needed-to-extract was incorrect in central header, and version-made-by could be lower than that ([#100](https://github.com/zip-rs/zip2/pull/100)) +- version-needed-to-extract was incorrect in central header, and version-made-by could be lower than that ([#100](https://github.com/zip-rs/zip2/pull/100)) + +### ⚙️ Miscellaneous Tasks +- Another tweak to ensure `version_needed` is applied +- Tweaks to make `version_needed` and `version_made_by` work with recently-merged changes + +## [2.0.0](https://github.com/zip-rs/zip2/compare/v1.3.1...v2.0.0) - 2024-05-24 + +### 🚀 Features +- Add `fmt::Display` for `DateTime` +- Implement more traits for `DateTime` + +### 🚜 Refactor +- Change type of `last_modified_time` to `Option` +- [**breaking**] Rename `from_msdos` to `from_msdos_unchecked`, make it unsafe, and add `try_from_msdos` ([#145](https://github.com/zip-rs/zip2/pull/145)) + +### ⚙️ Miscellaneous Tasks +- Continue to accept archives with invalid DateTime, and use `now_utc()` as default only when writing, not reading + +## [1.3.1](https://github.com/zip-rs/zip2/compare/v1.3.0...v1.3.1) - 2024-05-21 + +### 🚜 Refactor +- Make `deflate` enable both default implementations +- Merge the hidden deflate-flate2 flag into the public one +- Rename _deflate-non-zopfli to _deflate-flate2 +- Reject encrypted and using_data_descriptor files slightly faster in read_zipfile_from_stream +- Convert `impl TryInto for DateTime` to `impl TryFrom for NaiveDateTime` ([#136](https://github.com/zip-rs/zip2/pull/136)) + +### ⚡ Performance +- Change default compression implementation to `flate2/zlib-ng` + +### ⚙️ Miscellaneous Tasks +- chore([#132](https://github.com/zip-rs/zip2/pull/132)): Attribution for some copied test data +- chore([#133](https://github.com/zip-rs/zip2/pull/133)): chmod -x src/result.rs + +## [1.3.0](https://github.com/zip-rs/zip2/compare/v1.2.3...v1.3.0) - 2024-05-17 + +### 🚀 Features +- Add `is_symlink` method + +### 🐛 Bug Fixes +- Extract symlinks into symlinks on Unix and Windows, and fix a bug that affected making directories writable on MacOS + +### 🚜 Refactor +- Eliminate deprecation warning when `--all-features` implicitly enables the deprecated feature +- Check if archive contains a symlink's target, without borrowing both at the same time +- Eliminate a clone that's no longer necessary +- is_dir only needs to look at the filename +- Remove unnecessary #[cfg] attributes + +### ⚙️ Miscellaneous Tasks +- Fix borrow-of-moved-value +- Box doesn't directly convert to PathBuf, so convert back to String first +- partial revert - only &str has chars(), but Box should auto-deref +- contains_key needs a `Box`, so generify `is_dir` to accept one +- Add missing `ZipFileData::is_dir()` method +- Fix another Windows-specific error +- More bug fixes for Windows-specific symlink code +- More bug fixes for Windows-specific symlink code +- Bug fix: variable name change +- Bug fix: need both internal and output path to determine whether to symlink_dir +- Another bug fix +- Fix another error-type conversion error +- Fix error-type conversion on Windows +- Fix conditionally-unused import +- Fix continued issues, and factor out the Vec-to-OsString conversion (cc: [#125](https://github.com/zip-rs/zip2/pull/125)) +- Fix CI failure involving conversion to OsString for symlinks (see my comments on [#125](https://github.com/zip-rs/zip2/pull/125)) +- Move path join into platform-independent code + +## [1.2.3](https://github.com/zip-rs/zip2/compare/v1.2.2...v1.2.3) - 2024-05-10 + +### 🐛 Bug Fixes +- Remove a window when an extracted directory might be unexpectedly listable and/or `cd`able by non-owners +- Extract directory contents on Unix even if the directory doesn't have write permission (https://github.com/zip-rs/zip-old/issues/423) + +### ⚙️ Miscellaneous Tasks +- More conditionally-unused imports + +## [1.2.2](https://github.com/zip-rs/zip2/compare/v1.2.1...v1.2.2) - 2024-05-09 + +### 🐛 Bug Fixes +- Failed to clear "writing_raw" before finishing a symlink, leading to dropped extra fields + +### ⚡ Performance +- Use boxed slice for archive comment, since it can't be concatenated +- Optimize for the fact that false signatures can't overlap with real ones + +## [1.2.1](https://github.com/zip-rs/zip2/compare/v1.2.0...v1.2.1) - 2024-05-06 + +### 🐛 Bug Fixes +- Prevent panic when trying to read a file with an unsupported compression method +- Prevent panic after reading an invalid LZMA file +- Make `Stored` the default compression method if `Deflated` isn't available, so that zip files are readable by as much software as possible +- version_needed was wrong when e.g. cfg(bzip2) but current file wasn't bzip2 ([#100](https://github.com/zip-rs/zip2/pull/100)) +- file paths shouldn't start with slashes ([#102](https://github.com/zip-rs/zip2/pull/102)) + +### 🚜 Refactor +- Overhaul `impl Arbitrary for FileOptions` +- Remove unused `atomic` module + +## [1.2.0](https://github.com/zip-rs/zip2/compare/v1.1.4...v1.2.0) - 2024-05-06 + +### 🚀 Features +- Add method `decompressed_size()` so non-recursive ZIP bombs can be detected + +### 🚜 Refactor +- Make `ZipWriter::finish()` consume the `ZipWriter` + +### ⚙️ Miscellaneous Tasks +- Use panic! rather than abort to ensure the fuzz harness can process the failure +- Update fuzz_write to use replace_with +- Remove a drop that can no longer be explicit +- Add `#![allow(unexpected_cfgs)]` in nightly + +## [1.1.4](https://github.com/zip-rs/zip2/compare/v1.1.3...v1.1.4) - 2024-05-04 + +### 🐛 Bug Fixes +- Build was failing with bzip2 enabled +- use is_dir in more places where Windows paths might be handled incorrectly + +### ⚡ Performance +- Quick filter for paths that contain "/../" or "/./" or start with "./" or "../" +- Fast handling for separator-free paths +- Speed up logic if main separator isn't '/' +- Drop `normalized_components` slightly sooner when not using it +- Speed up `path_to_string` in cases where the path is already in the proper format + +### ⚙️ Miscellaneous Tasks +- Refactor: can short-circuit handling of paths that start with MAIN_SEPARATOR, no matter what MAIN_SEPARATOR is +- Bug fix: non-canonical path detection when MAIN_SEPARATOR is not slash or occurs twice in a row +- Bug fix: must recreate if . or .. is a path element +- Bug fix + +### ◀️ Revert +- [#58](https://github.com/zip-rs/zip2/pull/58) (partial): `bzip2-rs` can't replace `bzip2` because it's decompress-only + +## [1.1.3](https://github.com/zip-rs/zip2/compare/v1.1.2...v1.1.3) - 2024-04-30 + +### 🐛 Bug Fixes +- Rare bug where find_and_parse would give up prematurely on detecting a false end-of-CDR header + +## [1.1.2](https://github.com/Pr0methean/zip/compare/v1.1.1...v1.1.2) - 2024-04-28 + +### 🐛 Bug Fixes +- Alignment was previously handled incorrectly ([#33](https://github.com/Pr0methean/zip/pull/33)) + +### 🚜 Refactor +- deprecate `deflate-miniz` feature since it's now equivalent to `deflate` ([#35](https://github.com/Pr0methean/zip/pull/35)) + +## [1.1.1] + +### Added + +- `index_for_name`, `index_for_path`, `name_for_index`: get the index of a file given its path or vice-versa, without + initializing metadata from the local-file header or needing to mutably borrow the `ZipArchive`. +- `add_symlink_from_path`, `shallow_copy_file_from_path`, `deep_copy_file_from_path`, `raw_copy_file_to_path`: copy a + file or create a symlink using `AsRef` arguments + +### Changed + +- `add_directory_from_path` and `start_file_from_path` are no longer deprecated, and they now normalize `..` as well as + `.`. + +## [1.1.0] + +### Added + +- Support for decoding LZMA. + +### Changed + +- Eliminated a custom `AtomicU64` type by replacing it with `OnceLock` in the only place it's used. +- `FileOptions` now has the subtype `SimpleFileOptions` which implements `Copy` but has no extra data. + +## [1.0.1] + +### Changed + +- The published package on crates.io no longer includes the tests or examples. + +## [1.0.0] + +### Changed + +- Now uses boxed slices rather than `String` or `Vec` for metadata fields that aren't likely to grow. + +## [0.11.0] + +### Added + +- Support for `DEFLATE64` (decompression only). +- Support for Zopfli compression levels up to `i64::MAX`. + +### Changed + +- `InvalidPassword` is now a kind of `ZipError` to eliminate the need for nested `Result` structs. +- Updated dependencies. + +## [0.10.3] + +### Changed + +- Updated dependencies. +- MSRV increased to `1.67`. + +### Fixed + +- Fixed some rare bugs that could cause panics when trying to read an invalid ZIP file or using an incorrect password. + +## [0.10.2] + +### Changed + +- Where possible, methods are now `const`. This improves performance, especially when reading. + +## [0.10.1] + +### Changed + +- Date and time conversion methods now return `DateTimeRangeError` rather than `()` on error. + +## [0.10.0] + +### Changed + +- Replaces the `flush_on_finish_file` parameter of `ZipWriter::new` and `ZipWriter::Append` with + a `set_flush_on_finish_file` method. + +### Fixed + +- Fixes build errors that occur when all default features are disabled. +- Fixes more cases of a bug when ZIP64 magic bytes occur in filenames. + +## [0.9.2] + +### Added + +- `zlib-ng` for fast Deflate compression. This is now the default for compression levels 0-9. +- `chrono` to convert zip::DateTime to and from chrono::NaiveDateTime + +## [0.9.1] + +### Added + +- Zopfli for aggressive Deflate compression. + +## [0.9.0] + +### Added + + - `flush_on_finish_file` parameter for `ZipWriter`. + +## [0.8.3] + +### Changed + +- Uses the `aes::cipher::KeyInit` trait from `aes` 0.8.2 where appropriate. + +### Fixed + +- Calling `abort_file()` no longer corrupts the archive if called on a + shallow copy of a remaining file, or on an archive whose CDR entries are out + of sequence. However, it may leave an unused entry in the archive. +- Calling `abort_file()` while writing a ZipCrypto-encrypted file no longer + causes a crash. +- Calling `abort_file()` on the last file before `finish()` no longer produces + an invalid ZIP file or garbage in the comment. + +### Added + +- `ZipWriter` methods `get_comment()` and `get_raw_comment()`. + +## [0.8.2] + +### Fixed + +- Fixed an issue where code might spuriously fail during write fuzzing. + +### Added + +- New method `with_alignment` on `FileOptions`. + +## [0.8.1] + +### Fixed + +- `ZipWriter` now once again implements `Send` if the underlying writer does. + +## [0.8.0] + +### Deleted + +- Methods `start_file_aligned`, `start_file_with_extra_data`, `end_local_start_central_extra_data` and + `end_extra_data` (see below). + +### Changed + +- Alignment and extra-data fields are now attributes of [`zip::unstable::write::FileOptions`], allowing them to be + specified for `add_directory` and `add_symlink`. +- Extra-data fields are now formatted by the `FileOptions` method `add_extra_data`. +- Improved performance, especially for `shallow_copy_file` and `deep_copy_file` on files with extra data. + +### Fixed + +- Fixes a rare bug where the size of the extra-data field could overflow when `large_file` was set. +- Fixes more cases of a bug when ZIP64 magic bytes occur in filenames. + +## [0.7.5] + +### Fixed + +- Fixed a bug that occurs when ZIP64 magic bytes occur twice in a filename or across two filenames. + +## [0.7.4] + +### Added + +- Added experimental [`zip::unstable::write::FileOptions::with_deprecated_encryption`] API to enable encrypting + files with PKWARE encryption. + +## [0.7.3] + +### Fixed + +- Fixed a bug that occurs when a filename in a ZIP32 file includes the ZIP64 magic bytes. + +## [0.7.2] + +### Added + +- Method `abort_file` - removes the current or most recently-finished file from the archive. + +### Fixed + +- Fixed a bug where a file could remain open for writing after validations failed. + +## [0.7.1] + +### Changed + +- Bumped the version number in order to upload an updated README to crates.io. + +## [0.7.0] + +### Fixed + +- Calling `start_file` with invalid parameters no longer closes the `ZipWriter`. +- Attempting to write a 4GiB file without calling `FileOptions::large_file(true)` now removes the file from the archive + but does not close the `ZipWriter`. +- Attempting to write a file with an unrepresentable or invalid last-modified date will instead add it with a date of + 1980-01-01 00:00:00. + +### Added + +- Method `is_writing_file` - indicates whether a file is open for writing. + +## [0.6.13] + +### Fixed + +- Fixed a possible bug in deep_copy_file. + +## [0.6.12] + +### Fixed + +- Fixed a Clippy warning that was missed during the last release. + +## [0.6.11] + +### Fixed + +- Fixed a bug that could cause later writes to fail after a `deep_copy_file` call. + +## [0.6.10] + +### Changed + +- Updated dependency versions. + +## [0.6.9] + +### Fixed + +- Fixed an issue that prevented `ZipWriter` from implementing `Send`. + +## [0.6.8] + +### Added + +- Detects duplicate filenames. + +### Fixed + +- `deep_copy_file` could set incorrect Unix permissions. +- `deep_copy_file` could handle files incorrectly if their compressed size was u32::MAX bytes or less but their + uncompressed size was not. +- Documented that `deep_copy_file` does not copy a directory's contents. + +### Changed + +- Improved performance of `deep_copy_file` by using a HashMap and eliminating a redundant search. + +## [0.6.7] + +### Added + +- `deep_copy_file` method: more standards-compliant way to copy a file from within the ZipWriter + +## [0.6.6] + +### Fixed + +- Unused flag `#![feature(read_buf)]` was breaking compatibility with stable compiler. + +### Changed + +- Updated `aes` dependency to `0.8.2` (https://github.com/zip-rs/zip/pull/354) +- Updated other dependency versions. + +## [0.6.5] +### Changed + +- Added experimental [`zip::unstable::write::FileOptions::with_deprecated_encryption`] API to enable encrypting files with PKWARE encryption. + +### Added + +- `shallow_copy_file` method: copy a file from within the ZipWriter + + +## [0.6.4] + +### Changed + +- [#333](https://github.com/zip-rs/zip/pull/333): disabled the default features of the `time` dependency, and also `formatting` and `macros`, as they were enabled by mistake. +- Deprecated [`DateTime::from_time`](https://docs.rs/zip/0.6/zip/struct.DateTime.html#method.from_time) in favor of [`DateTime::try_from`](https://docs.rs/zip/0.6/zip/struct.DateTime.html#impl-TryFrom-for-DateTime) diff --git a/v2/vendor/security/zip-7.2.0/CODE_OF_CONDUCT.md b/v2/vendor/security/zip-7.2.0/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..2290ec2bf8 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to make participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies within all project spaces, and it also applies when +an individual is representing the project or its community in public spaces. +Examples of representing a project or community include using an official +project e-mail address, posting via an official social media account, or acting +as an appointed representative at an online or offline event. Representation of +a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at ryan.levick@gmail.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/v2/vendor/security/zip-7.2.0/CONTRIBUTING.md b/v2/vendor/security/zip-7.2.0/CONTRIBUTING.md new file mode 100644 index 0000000000..8d638d9f7b --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/CONTRIBUTING.md @@ -0,0 +1 @@ +Pull requests are welcome, but they're subject to some requirements that a lot of them don't meet. See https://github.com/zip-rs/zip2/raw/master/pull_request_template.md for details. diff --git a/v2/vendor/security/zip-7.2.0/Cargo.lock b/v2/vendor/security/zip-7.2.0/Cargo.lock new file mode 100644 index 0000000000..342a95aa45 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/Cargo.lock @@ -0,0 +1,1300 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bencher" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dfdb4953a096c551ce9ace855a604d702e6e62d77fac690575ae347571717f5" + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" + +[[package]] +name = "bitstream-io" +version = "4.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60d4bd9d1db2c6bdf285e223a7fa369d5ce98ec767dec949c6ca62863ce61757" +dependencies = [ + "core2", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" + +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "bzip2-sys", + "libbz2-rs-sys", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.2.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "755d2fce177175ffca841e9a06afdb2c4ab0f593d53b4dee48147dfaade85932" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" +dependencies = [ + "num-traits", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "clap" +version = "4.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" + +[[package]] +name = "cmake" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +dependencies = [ + "cc", +] + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + +[[package]] +name = "core2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" +dependencies = [ + "memchr", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "deflate64" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26bf8fc351c5ed29b5c2f0cbbac1b209b74f60ecd62e675a998df72c49af5204" + +[[package]] +name = "deranged" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "find-msvc-tools" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" + +[[package]] +name = "flate2" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b375d6465b98090a5f25b1c7703f3859783755aa9a80433b36e0379a3ec2f369" +dependencies = [ + "crc32fast", + "libz-ng-sys", + "libz-sys", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "jiff" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67e8da4c49d6d9909fe03361f9b620f58898859f5c7aded68351e85e71ecf50" +dependencies = [ + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-sys", +] + +[[package]] +name = "jiff-static" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0c84ee7f197eca9a86c6fd6cb771e55eb991632f15f2bc3ca6ec838929e6e78" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68971ebff725b9e2ca27a601c5eb38a4c5d64422c4cbab0c535f248087eda5c2" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libbz2-rs-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" + +[[package]] +name = "libc" +version = "0.2.180" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" + +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + +[[package]] +name = "libz-ng-sys" +version = "1.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bf914b7dd154ca9193afec311d8e39345c1bd93b48b3faa77329f0db8f553c0" +dependencies = [ + "cmake", + "libc", +] + +[[package]] +name = "libz-sys" +version = "1.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15d118bbf3771060e7311cc7bb0545b01d08a8b4a7de949198dec1fa0ca1c0f7" +dependencies = [ + "cc", + "cmake", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lzma-rust2" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1670343e58806300d87950e3401e820b519b9384281bbabfb15e3636689ffd69" +dependencies = [ + "crc", + "sha2", +] + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "minicov" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4869b6a491569605d66d3952bcdf03df789e5b536e5f0cf7758a7f08a55ae24d" +dependencies = [ + "cc", + "walkdir", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "nt-time" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c367e8edaff1f8a871e56343eb5e03888f6e0d1c2861880ccf4b7dc830899ed" +dependencies = [ + "time", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "portable-atomic" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f89776e4d69bb58bc6993e99ffa1d11f228b839984854c7daeb5d37f87cbe950" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppmd-rust" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d558c559f0450f16f2a27a1f017ef38468c1090c9ce63c8e51366232d53717b4" + +[[package]] +name = "proc-macro2" +version = "1.0.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "535d180e0ecab6268a3e718bb9fd44db66bbbc256257165fc699dadf70d16fe7" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rustix" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "time" +version = "0.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9e442fc33d7fdb45aa9bfeb312c095964abdf596f7567261062b2a7107aaabd" +dependencies = [ + "deranged", + "itoa", + "js-sys", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b36ee98fd31ec7426d599183e8fe26932a8dc1fb76ddb6214d05493377d34ca" + +[[package]] +name = "time-macros" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e552d1249bf61ac2a52db88179fd0673def1e1ad8243a00d9ec9ed71fee3dd" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "typed-path" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7922f2cdc51280d47b491af9eafc41eb0cdab85eabcb390c854412fcbf26dbe8" + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" +dependencies = [ + "cfg-if", + "futures-util", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-bindgen-test" +version = "0.3.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45649196a53b0b7a15101d845d44d2dda7374fc1b5b5e2bbf58b7577ff4b346d" +dependencies = [ + "async-trait", + "cast", + "js-sys", + "libm", + "minicov", + "nu-ansi-term", + "num-traits", + "oorandom", + "serde", + "serde_json", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test-macro", + "wasm-bindgen-test-shared", +] + +[[package]] +name = "wasm-bindgen-test-macro" +version = "0.3.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f579cdd0123ac74b94e1a4a72bd963cf30ebac343f2df347da0b8df24cdebed2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "wasm-bindgen-test-shared" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8145dd1593bf0fb137dbfa85b8be79ec560a447298955877804640e40c2d6ea" + +[[package]] +name = "web-sys" +version = "0.3.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zip" +version = "7.2.0" +dependencies = [ + "aes", + "arbitrary", + "bencher", + "bitstream-io", + "bzip2", + "chrono", + "clap", + "constant_time_eq", + "crc32fast", + "deflate64", + "flate2", + "generic-array", + "getrandom", + "hmac", + "indexmap", + "jiff", + "lzma-rust2", + "memchr", + "nt-time", + "pbkdf2", + "ppmd-rust", + "rayon", + "sha1", + "tempfile", + "time", + "typed-path", + "walkdir", + "wasm-bindgen-test", + "zeroize", + "zopfli", + "zstd", +] + +[[package]] +name = "zlib-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" + +[[package]] +name = "zmij" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfcd145825aace48cff44a8844de64bf75feec3080e0aa5cdbde72961ae51a65" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/v2/vendor/security/zip-7.2.0/Cargo.toml b/v2/vendor/security/zip-7.2.0/Cargo.toml new file mode 100644 index 0000000000..91548c5cf5 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/Cargo.toml @@ -0,0 +1,330 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.83.0" +name = "zip" +version = "7.2.0" +authors = [ + "Mathijs van de Nes ", + "Marli Frost ", + "Ryan Levick ", + "Chris Hennick ", +] +build = false +exclude = [ + "tests/**", + "examples/**", + ".github/**", + "fuzz_read/**", + "fuzz_write/**", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = """ +Library to support the reading and writing of zip files. +""" +readme = "README.md" +keywords = [ + "zip", + "archive", + "compression", +] +categories = [ + "compression", + "filesystem", + "parser-implementations", +] +license = "MIT" +repository = "https://github.com/zip-rs/zip2.git" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = [ + "--cfg", + "docsrs", +] + +[features] +_all-features = [] +_arbitrary = ["dep:arbitrary"] +_bzip2_any = [] +_deflate-any = [] +aes-crypto = [ + "dep:aes", + "dep:constant_time_eq", + "dep:generic-array", + "getrandom/std", + "dep:hmac", + "dep:pbkdf2", + "dep:sha1", + "dep:zeroize", +] +bzip2 = [ + "dep:bzip2", + "bzip2/default", + "_bzip2_any", +] +bzip2-rs = [ + "dep:bzip2", + "bzip2/bzip2-sys", + "_bzip2_any", +] +chrono = ["dep:chrono"] +default = [ + "aes-crypto", + "bzip2", + "deflate64", + "deflate", + "lzma", + "ppmd", + "time", + "zstd", + "xz", +] +deflate = [ + "deflate-zopfli", + "deflate-flate2-zlib-rs", +] +deflate-flate2 = [ + "_deflate-any", + "dep:flate2", +] +deflate-flate2-zlib = [ + "deflate-flate2", + "flate2/zlib", +] +deflate-flate2-zlib-ng = [ + "deflate-flate2", + "flate2/zlib-ng", +] +deflate-flate2-zlib-ng-compat = [ + "deflate-flate2", + "flate2/zlib-ng-compat", +] +deflate-flate2-zlib-rs = [ + "deflate-flate2", + "flate2/zlib-rs", +] +deflate-zopfli = [ + "dep:zopfli", + "_deflate-any", +] +jiff-02 = ["dep:jiff"] +legacy-zip = ["bitstream-io"] +lzma = ["dep:lzma-rust2"] +nt-time = ["dep:nt-time"] +ppmd = ["dep:ppmd-rust"] +unreserved = [] +xz = ["dep:lzma-rust2"] + +[lib] +name = "zip" +path = "src/lib.rs" + +[[bench]] +name = "merge_archive" +path = "benches/merge_archive.rs" +harness = false + +[[bench]] +name = "read_entry" +path = "benches/read_entry.rs" +harness = false + +[[bench]] +name = "read_metadata" +path = "benches/read_metadata.rs" +harness = false + +[dependencies.aes] +version = "0.8" +optional = true + +[dependencies.arbitrary] +version = "1.4" +features = ["derive"] +optional = true + +[dependencies.bitstream-io] +version = "4.5" +optional = true + +[dependencies.bzip2] +version = "0.6" +optional = true +default-features = false + +[dependencies.chrono] +version = "^0.4.27" +features = ["std"] +optional = true +default-features = false + +[dependencies.constant_time_eq] +version = "0.3" +optional = true + +[dependencies.crc32fast] +version = "1.5" + +[dependencies.deflate64] +version = "0.1.10" +optional = true + +[dependencies.flate2] +version = "1.1" +optional = true +default-features = false + +[dependencies.generic-array] +version = "=0.14.7" +optional = true + +[dependencies.hmac] +version = "0.12" +features = ["reset"] +optional = true + +[dependencies.indexmap] +version = "2" + +[dependencies.jiff] +version = "0.2.4" +optional = true + +[dependencies.lzma-rust2] +version = "0.15" +features = [ + "std", + "encoder", + "optimization", + "xz", +] +optional = true +default-features = false + +[dependencies.memchr] +version = "2.7" + +[dependencies.nt-time] +version = "0.10.6" +optional = true +default-features = false + +[dependencies.pbkdf2] +version = "0.12" +optional = true + +[dependencies.ppmd-rust] +version = "1.3" +optional = true + +[dependencies.sha1] +version = "0.10" +optional = true + +[dependencies.time] +version = "0.3" +features = ["std"] +optional = true +default-features = false + +[dependencies.typed-path] +version = "0.12" + +[dependencies.zeroize] +version = "1.8" +features = ["zeroize_derive"] +optional = true + +[dependencies.zopfli] +version = "0.8" +optional = true + +[dependencies.zstd] +version = "^0.13.3" +optional = true +default-features = false + +[dev-dependencies.bencher] +version = "0.1" + +[dev-dependencies.clap] +version = "=4.4.18" +features = ["derive"] + +[dev-dependencies.getrandom] +version = "0.3" +features = [] +default-features = false + +[dev-dependencies.rayon] +version = "1.11" + +[dev-dependencies.tempfile] +version = "3.15" + +[dev-dependencies.time] +version = "0.3" +features = [ + "formatting", + "macros", +] + +[dev-dependencies.walkdir] +version = "2.5" + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies.getrandom] +version = "0.3" +optional = true +default-features = false + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies.time] +version = "0.3" +features = ["wasm-bindgen"] +optional = true +default-features = false + +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies.getrandom] +version = "0.3" +default-features = false + +[target.'cfg(target_arch = "wasm32")'.dependencies.getrandom] +version = "0.3" +features = ["wasm_js"] +optional = true +default-features = false + +[target.'cfg(target_arch = "wasm32")'.dependencies.time] +version = "0.3" +features = [ + "std", + "wasm-bindgen", +] +optional = true +default-features = false + +[target.'cfg(target_arch = "wasm32")'.dev-dependencies.getrandom] +version = "0.3" +features = ["wasm_js"] +default-features = false + +[target.'cfg(target_arch = "wasm32")'.dev-dependencies.wasm-bindgen-test] +version = "^0.3.56" + +[[test]] +name = "ruview_0429_backport" +path = "tests/ruview_0429_backport.rs" diff --git a/v2/vendor/security/zip-7.2.0/Cargo.toml.orig b/v2/vendor/security/zip-7.2.0/Cargo.toml.orig new file mode 100644 index 0000000000..761bf24a7e --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/Cargo.toml.orig @@ -0,0 +1,139 @@ +[package] +name = "zip" +version = "7.2.0" +authors = [ + "Mathijs van de Nes ", + "Marli Frost ", + "Ryan Levick ", + "Chris Hennick ", +] +license = "MIT" +repository = "https://github.com/zip-rs/zip2.git" +keywords = ["zip", "archive", "compression"] +# Any change to rust-version must be reflected also in `README.md` and `.github/workflows/ci.yaml`. +# The MSRV policy is documented in `README.md`. +rust-version = "1.83.0" +categories = ["compression", "filesystem", "parser-implementations"] +description = """ +Library to support the reading and writing of zip files. +""" +edition = "2021" +exclude = ["tests/**", "examples/**", ".github/**", "fuzz_read/**", "fuzz_write/**"] + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + +[dependencies] +arbitrary = { version = "1.4", features = ["derive"], optional = true } +aes = { version = "0.8", optional = true } +bzip2 = { version = "0.6", default-features = false, optional = true } +chrono = { version = "^0.4.27", optional = true, default-features = false, features = ["std"] } +constant_time_eq = { version = "0.3", optional = true } +crc32fast = "1.5" +flate2 = { version = "1.1", default-features = false, optional = true } +generic-array = { version = "=0.14.7", optional = true } +hmac = { version = "0.12", optional = true, features = ["reset"] } +indexmap = "2" +jiff = { version = "0.2.4", optional = true } +memchr = "2.7" +nt-time = { version = "0.10.6", default-features = false, optional = true } +ppmd-rust = { version = "1.3", optional = true } +pbkdf2 = { version = "0.12", optional = true } +sha1 = { version = "0.10", optional = true } +time = { version = "0.3", default-features = false, optional = true, features = [ + "std", +] } +zeroize = { version = "1.8", optional = true, features = ["zeroize_derive"] } +zstd = { version = "^0.13.3", optional = true, default-features = false } +zopfli = { version = "0.8", optional = true } +deflate64 = { version = "0.1.10", optional = true } +lzma-rust2 = { version = "0.15", optional = true, default-features = false, features = ["std", "encoder", "optimization", "xz"] } +bitstream-io = { version = "4.5", optional = true } +typed-path = "0.12" + +# native: +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +getrandom = { version = "0.3", default-features = false, optional = true } +time = { version = "0.3", default-features = false, optional = true, features = [ + "wasm-bindgen" +] } + +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +getrandom = { version = "0.3", default-features = false } + +# wasm32 +[target.'cfg(target_arch = "wasm32")'.dependencies] +getrandom = { version = "0.3", default-features = false, features = ["wasm_js"], optional = true } +time = { version = "0.3", default-features = false, optional = true, features = [ + "std", + "wasm-bindgen" +] } + +[target.'cfg(target_arch = "wasm32")'.dev-dependencies] +getrandom = { version = "0.3", default-features = false, features = ["wasm_js"] } +wasm-bindgen-test = "^0.3.56" + +[dev-dependencies] +bencher = "0.1" +getrandom = { version = "0.3", default-features = false, features = [] } +walkdir = "2.5" +time = { version = "0.3", features = ["formatting", "macros"] } +clap = { version = "=4.4.18", features = ["derive"] } +tempfile = "3.15" +rayon = "1.11" + +[features] +_arbitrary = ["dep:arbitrary"] +aes-crypto = ["dep:aes", "dep:constant_time_eq", "dep:generic-array", "getrandom/std", "dep:hmac", "dep:pbkdf2", "dep:sha1", "dep:zeroize"] +chrono = ["dep:chrono"] +_deflate-any = [] +_all-features = [] # Detect when --all-features is used +deflate = ["deflate-zopfli", "deflate-flate2-zlib-rs"] +# Pull in flate2, but don't choose a backend; useful if you want to choose your own flate2 backend +deflate-flate2 = ["_deflate-any", "dep:flate2"] +# Pull in flate2 and the fast zlib-rs backend; this is what most users will want +deflate-flate2-zlib-rs = ["deflate-flate2", "flate2/zlib-rs"] +# Pull in zopfli (write-only DEFLATE, slower than flate2 with better compression ratios) +deflate-zopfli = ["dep:zopfli", "_deflate-any"] +jiff-02 = ["dep:jiff"] +nt-time = ["dep:nt-time"] +lzma = ["dep:lzma-rust2"] +ppmd = ["dep:ppmd-rust"] +unreserved = [] +xz = ["dep:lzma-rust2"] +_bzip2_any = [] +bzip2 = ["dep:bzip2", "bzip2/default", "_bzip2_any"] +legacy-zip = ["bitstream-io"] +default = [ + "aes-crypto", + "bzip2", + "deflate64", + "deflate", + "lzma", + "ppmd", + "time", + "zstd", + "xz", +] + +# The following features are not supported on wasm +bzip2-rs = ["dep:bzip2", "bzip2/bzip2-sys", "_bzip2_any"] +# Pull in flate2 and the zlib backend; only use this if you need a dynamically linked system zlib +deflate-flate2-zlib = ["deflate-flate2", "flate2/zlib"] +# Pull in flate2 and the zlib-ng backend; a modern fork of zlib +deflate-flate2-zlib-ng = ["deflate-flate2", "flate2/zlib-ng"] +# Pull in flate2 and the zlib-ng backend with a zlib-compatible API +deflate-flate2-zlib-ng-compat = ["deflate-flate2", "flate2/zlib-ng-compat"] + +[[bench]] +name = "read_entry" +harness = false + +[[bench]] +name = "read_metadata" +harness = false + +[[bench]] +name = "merge_archive" +harness = false diff --git a/v2/vendor/security/zip-7.2.0/LICENSE b/v2/vendor/security/zip-7.2.0/LICENSE new file mode 100644 index 0000000000..4a45976f88 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/LICENSE @@ -0,0 +1,24 @@ +The MIT License (MIT) + +Copyright (c) 2014 Mathijs van de Nes + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Some files in the "tests/data" subdirectory of this repository are under other +licences; see files named LICENSE.*.txt for details. \ No newline at end of file diff --git a/v2/vendor/security/zip-7.2.0/README.md b/v2/vendor/security/zip-7.2.0/README.md new file mode 100644 index 0000000000..1459e9b852 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/README.md @@ -0,0 +1,145 @@ +zip +=== + +[![Build Status](https://github.com/zip-rs/zip2/actions/workflows/ci.yaml/badge.svg)](https://github.com/zip-rs/zip2/actions?query=branch%3Amaster+workflow%3ACI) +[![Crates.io version](https://img.shields.io/crates/v/zip.svg)](https://crates.io/crates/zip) + +[Documentation](https://docs.rs/zip/latest/zip/) + +# Info + +A zip library for rust which supports reading and writing of simple ZIP files. Formerly hosted at +https://github.com/zip-rs/zip2. + +Supported compression formats: + +* stored (i.e. none) +* deflate +* deflate64 (decompression only) +* bzip2 +* zstd +* lzma (decompression only) +* xz +* ppmd + +Currently unsupported zip extensions: + +* Multi-disk + +# Features + +The features available are: + +* `aes-crypto`: Enables decryption of files which were encrypted with AES. Supports AE-1 and AE-2 methods. +* `deflate`: Enables compressing and decompressing an unspecified implementation (that may change in future versions) of + the deflate compression algorithm, which is the default for zip files. Supports compression quality 1..=264. +* `deflate-flate2`: Combine this with any `flate2` feature flag that enables a back-end, to support deflate compression + at quality 1..=9. +* `deflate-zopfli`: Enables deflating files with the `zopfli` library (used when compression quality is 10..=264). This + is the most effective `deflate` implementation available, but also among the slowest. If `flate2` isn't also enabled, + only compression will be supported and not decompression. +* `deflate64`: Enables the deflate64 compression algorithm. Only decompression is supported. +* `lzma`: Enables the LZMA compression algorithm. Only decompression is supported. +* `bzip2`: Enables the BZip2 compression algorithm. +* `ppmd`: Enables the PPMd compression algorithm. +* `time`: Enables features using the [time](https://github.com/rust-lang-deprecated/time) crate. +* `chrono`: Enables converting last-modified `zip::DateTime` to and from `chrono::NaiveDateTime`. +* `jiff-02`: Enables converting last-modified `zip::DateTime` to and from `jiff::civil::DateTime`. +* `nt-time`: Enables returning timestamps stored in the NTFS extra field as `nt_time::FileTime`. +* `xz`: Enables the XZ compression algorithm. +* `zstd`: Enables the Zstandard compression algorithm. + +By default `aes-crypto`, `bzip2`, `deflate`, `deflate64`, `lzma`, `ppmd`, `time`, `xz` and `zstd` are enabled. + +# MSRV + +Our current Minimum Supported Rust Version is **1.83**. When adding features, +we will follow these guidelines: + +- We will always support a minor Rust version that has been stable for at least 6 months. +- Any change to the MSRV will be accompanied with a **minor** version bump. + +# Examples + +See the [examples directory](examples) for: + +* How to write a file to a zip. +* How to write a directory of files to a zip (using [walkdir](https://github.com/BurntSushi/walkdir)). +* How to extract a zip file. +* How to extract a single file from a zip. +* How to read a zip from the standard input. +* How to append a directory to an existing archive + +# Fuzzing + +Fuzzing support is through [`cargo afl`](https://rust-fuzz.github.io/book/afl/tutorial.html). To install `cargo afl`: + +```bash +cargo install cargo-afl +``` + +To start fuzzing zip extraction: + +```bash +mkdir -vp fuzz-read-out +cargo afl build --manifest-path=fuzz/Cargo.toml --all-features -p fuzz_read +# Curated input corpus: +cargo afl fuzz -i fuzz/read/in -o fuzz-read-out fuzz/target/debug/fuzz_read +# Test data files: +cargo afl fuzz -i tests/data -e zip -o fuzz-read-out fuzz/target/debug/fuzz_read +``` + +To start fuzzing zip creation: + +```bash +mkdir -vp fuzz-write-out +cargo afl build --manifest-path=fuzz/Cargo.toml --all-features -p fuzz_write +# Curated input corpus and dictionary schema: +cargo afl fuzz -x fuzz/write/fuzz.dict -i fuzz/write/in -o fuzz-write-out fuzz/target/debug/fuzz_write +``` + +## Fuzzing stdio + +The read and write fuzzers can also receive input over stdin for one-off validation. Note here that the fuzzers can be configured to build in support for DEFLATE, or not: +```bash +# Success, no output: +cargo run --manifest-path=fuzz/Cargo.toml --quiet --all-features -p fuzz_read MAX_SYMLINK_TARGET_LEN { ++ return Err(invalid!( ++ "symlink target declares {} bytes, more than the {} byte limit", ++ declared_len, ++ MAX_SYMLINK_TARGET_LEN ++ )); ++ } ++ let mut target = Vec::with_capacity(declared_len as usize); + file.read_to_end(&mut target)?; + drop(file); + make_symlink(&outpath, &target, &self.shared.files)?; +--- a/src/read/stream.rs ++++ b/src/read/stream.rs +@@ -1,4 +1,7 @@ ++use crate::result::invalid; ++ + use super::{ ++ MAX_SYMLINK_TARGET_LEN, + central_header_to_zip_file_inner, make_symlink, read_zipfile_from_stream, ZipCentralEntryBlock, + ZipFile, ZipFileData, ZipResult, + }; +@@ -67,7 +70,15 @@ + file.safe_prepare_path(&self.0, &mut outpath, None::<&(_, fn(&Path) -> bool)>)?; + + if file.is_symlink() { +- let mut target = Vec::with_capacity(file.size() as usize); ++ let declared_len = file.size(); ++ if declared_len > MAX_SYMLINK_TARGET_LEN { ++ return Err(invalid!( ++ "symlink target declares {} bytes, more than the {} byte limit", ++ declared_len, ++ MAX_SYMLINK_TARGET_LEN ++ )); ++ } ++ let mut target = Vec::with_capacity(declared_len as usize); + file.read_to_end(&mut target)?; + make_symlink(&outpath, &target, &self.1)?; + return Ok(()); diff --git a/v2/vendor/security/zip-7.2.0/RUVIEW-BACKPORT.md b/v2/vendor/security/zip-7.2.0/RUVIEW-BACKPORT.md new file mode 100644 index 0000000000..612944437a --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/RUVIEW-BACKPORT.md @@ -0,0 +1,35 @@ +# RuView ZIP 7.2.0 security backport + +This is the official crates.io ZIP 7.2.0 source package with the final upstream +[ZIP #984](https://github.com/zip-rs/zip2/pull/984) fix adapted to its archive and +stream readers. Version and licenses remain unchanged. Source/archive/fix hashes +are recorded in `ruview-provenance.json`; the production diff is +`RUSTSEC-2026-0429.patch` (two files). The only additional manifest entry enables +our self-contained public-API regression target. No Candle/framework snapshot or +RuView submodule revision changes are required. + +The patch rejects symlink target lengths over4096 before allocation. The exact +112-byte RUSTSEC-2026-0429 example aborts pristine7.2.0 and is rejected with +InvalidArchive here. Dedicated tests cover hostile lengths, the stream flag, +ordinary files and valid relative symlinks. A removed-guard mutation fails. + +Run from the repository root: + +```sh +cargo test --manifest-path v2/vendor/security/zip-7.2.0/Cargo.toml --locked --no-default-features --features deflate --test ruview_0429_backport +``` + +The registry package omits large upstream test archives. Therefore this package +is excluded from the parent workspace's unit-test membership; the explicit +regression target above is mandatory in security CI. Full upstream library and +selected integration tests were separately executed with fixtures from package +VCS46dc29cda6749736ddb1ec502aff06a65ffe54f8; they are not claimed as part of this +small committed fixture set. + +cargo-audit0.22.2 excludes path sources from registry advisory matching. A green +audit alone does not establish this patch's correctness; executable regressions, +provenance and review do. No advisory policy or exception was added or broadened. +This addresses0429 only; existing0367/0368/0369 reachability decisions remain +unchanged. Retire this backport when a reviewed compatible upstream upgrade is +available. Current Candle0.9 requires ZIP7, while moving the swarm submodule to +its new upstream workspace would be a separate integration change. diff --git a/v2/vendor/security/zip-7.2.0/SECURITY.md b/v2/vendor/security/zip-7.2.0/SECURITY.md new file mode 100644 index 0000000000..031b4cf473 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/SECURITY.md @@ -0,0 +1,36 @@ +# Security Policy + +## Supported Versions + +Use this section to tell people about which versions of your project are +currently being supported with security updates. + +| Version | Supported | +| ------- | ------------------ | +| 2.4.x | :white_check_mark: | +| 2.3.x | :white_check_mark: | +| < 2.0 | :x: | + +## Reporting a Vulnerability + +To report a vulnerability, please go to https://github.com/zip-rs/zip2/security/advisories/new. We'll attempt to: + +* Close the report within 7 days if it's invalid, or if a fix has already been released but some old versions needed to be yanked. +* Provide progress reports at least every 7 days to the original reporter. +* Fix vulnerabilities within 30 days of the initial report. + +## Disclosure + +A vulnerability will only be publicly disclosed once a fix is released. At that point, the delay before full public disclosure +will be determined as follows: + +* If the proof-of-concept is very simple, or an exploit is already in the wild (whether or not it specifically targets `zip`, + all details will be made public right away. +* If the vulnerability is specific to `zip` and cannot easily be reverse-engineered from the code history, then the + proof-of-concept and most of the details will be withheld until 14 days after the fix is released and all vulnerable + versions are yanked with `cargo yank`. +* If a potential victim requests more time to deploy a fix based on a credible risk, then the withholding of details can + be extended up to 30 days. This may be extended to 90 days if the victim is high-value (e.g. manages over US$1 billion + worth of financial assets or intellectual property, or has evidence that they're a target of nation-state attackers) + and there's a valid reason why they cannot deploy the fix as fast as most users (e.g. heavy reliance on an old version's + interface, or infrastructure damage in a war zone). diff --git a/v2/vendor/security/zip-7.2.0/benches/merge_archive.rs b/v2/vendor/security/zip-7.2.0/benches/merge_archive.rs new file mode 100644 index 0000000000..9e5598f7dd --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/benches/merge_archive.rs @@ -0,0 +1,130 @@ +use bencher::{benchmark_group, benchmark_main}; + +use std::io::{Cursor, Read, Seek, Write}; + +use bencher::Bencher; +use zip::{result::ZipResult, write::SimpleFileOptions, ZipArchive, ZipWriter}; + +fn generate_random_archive( + num_entries: usize, + entry_size: usize, + options: SimpleFileOptions, +) -> ZipResult<(usize, ZipArchive>>)> { + let buf = Cursor::new(Vec::new()); + let mut zip = ZipWriter::new(buf); + + let mut bytes = vec![0u8; entry_size]; + for i in 0..num_entries { + let name = format!("random{i}.dat"); + zip.start_file(name, options)?; + getrandom::fill(&mut bytes).unwrap(); + zip.write_all(&bytes)?; + } + + let buf = zip.finish()?.into_inner(); + let len = buf.len(); + + Ok((len, ZipArchive::new(Cursor::new(buf))?)) +} + +fn perform_merge( + src: ZipArchive, + mut target: ZipWriter, +) -> ZipResult> { + target.merge_archive(src)?; + Ok(target) +} + +fn perform_raw_copy_file( + mut src: ZipArchive, + mut target: ZipWriter, +) -> ZipResult> { + for i in 0..src.len() { + let entry = src.by_index(i)?; + target.raw_copy_file(entry)?; + } + Ok(target) +} + +const NUM_ENTRIES: usize = 100; +const ENTRY_SIZE: usize = 1024; + +fn merge_archive_stored(bench: &mut Bencher) { + let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + let (len, src) = generate_random_archive(NUM_ENTRIES, ENTRY_SIZE, options).unwrap(); + + bench.bytes = len as u64; + + bench.iter(|| { + let buf = Cursor::new(Vec::new()); + let zip = ZipWriter::new(buf); + let zip = perform_merge(src.clone(), zip).unwrap(); + let buf = zip.finish().unwrap().into_inner(); + assert_eq!(buf.len(), len); + }); +} + +#[cfg(feature = "_deflate-any")] +fn merge_archive_compressed(bench: &mut Bencher) { + let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); + let (len, src) = generate_random_archive(NUM_ENTRIES, ENTRY_SIZE, options).unwrap(); + + bench.bytes = len as u64; + + bench.iter(|| { + let buf = Cursor::new(Vec::new()); + let zip = ZipWriter::new(buf); + let zip = perform_merge(src.clone(), zip).unwrap(); + let buf = zip.finish().unwrap().into_inner(); + assert_eq!(buf.len(), len); + }); +} + +fn merge_archive_raw_copy_file_stored(bench: &mut Bencher) { + let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + let (len, src) = generate_random_archive(NUM_ENTRIES, ENTRY_SIZE, options).unwrap(); + + bench.bytes = len as u64; + + bench.iter(|| { + let buf = Cursor::new(Vec::new()); + let zip = ZipWriter::new(buf); + let zip = perform_raw_copy_file(src.clone(), zip).unwrap(); + let buf = zip.finish().unwrap().into_inner(); + assert_eq!(buf.len(), len); + }); +} + +#[cfg(feature = "_deflate-any")] +fn merge_archive_raw_copy_file_compressed(bench: &mut Bencher) { + let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); + let (len, src) = generate_random_archive(NUM_ENTRIES, ENTRY_SIZE, options).unwrap(); + + bench.bytes = len as u64; + + bench.iter(|| { + let buf = Cursor::new(Vec::new()); + let zip = ZipWriter::new(buf); + let zip = perform_raw_copy_file(src.clone(), zip).unwrap(); + let buf = zip.finish().unwrap().into_inner(); + assert_eq!(buf.len(), len); + }); +} + +#[cfg(feature = "_deflate-any")] +benchmark_group!( + benches, + merge_archive_stored, + merge_archive_compressed, + merge_archive_raw_copy_file_stored, + merge_archive_raw_copy_file_compressed, +); + +#[cfg(not(feature = "_deflate-any"))] +benchmark_group!( + benches, + merge_archive_stored, + merge_archive_raw_copy_file_stored, +); + +benchmark_main!(benches); diff --git a/v2/vendor/security/zip-7.2.0/benches/read_entry.rs b/v2/vendor/security/zip-7.2.0/benches/read_entry.rs new file mode 100644 index 0000000000..8fd4ef3a5c --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/benches/read_entry.rs @@ -0,0 +1,41 @@ +use bencher::{benchmark_group, benchmark_main}; + +use std::io::{Cursor, Read, Write}; + +use bencher::Bencher; +use zip::{write::SimpleFileOptions, ZipArchive, ZipWriter}; + +fn generate_random_archive(size: usize) -> Vec { + let data = Vec::new(); + let mut writer = ZipWriter::new(Cursor::new(data)); + let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + + writer.start_file("random.dat", options).unwrap(); + let mut bytes = vec![0u8; size]; + getrandom::fill(&mut bytes).unwrap(); + writer.write_all(&bytes).unwrap(); + + writer.finish().unwrap().into_inner() +} + +fn read_entry(bench: &mut Bencher) { + let size = 1024 * 1024; + let bytes = generate_random_archive(size); + let mut archive = ZipArchive::new(Cursor::new(bytes.as_slice())).unwrap(); + + bench.iter(|| { + let mut file = archive.by_name("random.dat").unwrap(); + let mut buf = [0u8; 1024]; + loop { + let n = file.read(&mut buf).unwrap(); + if n == 0 { + break; + } + } + }); + + bench.bytes = size as u64; +} + +benchmark_group!(benches, read_entry); +benchmark_main!(benches); diff --git a/v2/vendor/security/zip-7.2.0/benches/read_metadata.rs b/v2/vendor/security/zip-7.2.0/benches/read_metadata.rs new file mode 100644 index 0000000000..06b4aa397c --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/benches/read_metadata.rs @@ -0,0 +1,143 @@ +use bencher::{benchmark_group, benchmark_main}; + +use std::fs; +use std::io::{self, Cursor, Write}; + +use bencher::Bencher; +use tempfile::TempDir; +use zip::write::SimpleFileOptions; +use zip::{result::ZipResult, CompressionMethod, ZipArchive, ZipWriter}; + +const FILE_COUNT: usize = 15_000; +const FILE_SIZE: usize = 1024; + +fn generate_random_archive(count_files: usize, file_size: usize) -> ZipResult> { + let data = Vec::new(); + let mut writer = ZipWriter::new(Cursor::new(data)); + let options = SimpleFileOptions::default().compression_method(CompressionMethod::Stored); + + let mut bytes = vec![0u8; file_size]; + + for i in 0..count_files { + let name = format!("file_deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef_{i}.dat"); + writer.start_file(name, options)?; + getrandom::fill(&mut bytes).map_err(io::Error::from)?; + writer.write_all(&bytes)?; + } + + Ok(writer.finish()?.into_inner()) +} + +fn read_metadata(bench: &mut Bencher) { + let bytes = generate_random_archive(FILE_COUNT, FILE_SIZE).unwrap(); + + bench.iter(|| { + let archive = ZipArchive::new(Cursor::new(bytes.as_slice())).unwrap(); + archive.len() + }); + bench.bytes = bytes.len() as u64; +} + +const COMMENT_SIZE: usize = 50_000; + +fn generate_zip32_archive_with_random_comment(comment_length: usize) -> ZipResult> { + let data = Vec::new(); + let mut writer = ZipWriter::new(Cursor::new(data)); + let options = SimpleFileOptions::default().compression_method(CompressionMethod::Stored); + + let mut bytes = vec![0u8; comment_length]; + getrandom::fill(&mut bytes).unwrap(); + writer.set_raw_comment(bytes.into_boxed_slice()); + + writer.start_file("asdf.txt", options)?; + writer.write_all(b"asdf")?; + + Ok(writer.finish()?.into_inner()) +} + +fn parse_archive_with_comment(bench: &mut Bencher) { + let bytes = generate_zip32_archive_with_random_comment(COMMENT_SIZE).unwrap(); + + bench.bench_n(1, |_| { + let archive = ZipArchive::new(Cursor::new(bytes.as_slice())).unwrap(); + let _ = archive.comment().len(); + }); + bench.bytes = bytes.len() as u64; +} + +const COMMENT_SIZE_64: usize = 500_000; + +fn generate_zip64_archive_with_random_comment(comment_length: usize) -> ZipResult> { + let data = Vec::new(); + let mut writer = ZipWriter::new(Cursor::new(data)); + let options = SimpleFileOptions::default() + .compression_method(CompressionMethod::Stored) + .large_file(true); + + let mut bytes = vec![0u8; comment_length]; + getrandom::fill(&mut bytes).unwrap(); + // should use the line below but still works + // writer.set_raw_zip64_comment(Some(comment)); + writer.set_raw_comment(bytes.into_boxed_slice()); + + writer.start_file("asdf.txt", options)?; + writer.write_all(b"asdf")?; + + Ok(writer.finish()?.into_inner()) +} + +fn parse_zip64_archive_with_comment(bench: &mut Bencher) { + let bytes = generate_zip64_archive_with_random_comment(COMMENT_SIZE_64).unwrap(); + + bench.iter(|| { + let archive = ZipArchive::new(Cursor::new(bytes.as_slice())).unwrap(); + archive.comment().len() + }); + bench.bytes = bytes.len() as u64; +} + +fn parse_stream_archive(bench: &mut Bencher) { + const STREAM_ZIP_ENTRIES: usize = 5; + const STREAM_FILE_SIZE: usize = 5; + + let bytes = generate_random_archive(STREAM_ZIP_ENTRIES, STREAM_FILE_SIZE).unwrap(); + + /* Write to a temporary file path to incur some filesystem overhead from repeated reads */ + let dir = TempDir::with_prefix("stream-bench").unwrap(); + let out = dir.path().join("bench-out.zip"); + fs::write(&out, &bytes).unwrap(); + + bench.iter(|| { + let mut f = fs::File::open(&out).unwrap(); + while zip::read::read_zipfile_from_stream(&mut f) + .unwrap() + .is_some() + {} + }); + bench.bytes = bytes.len() as u64; +} + +fn parse_large_non_zip(bench: &mut Bencher) { + const FILE_SIZE: usize = 17_000_000; + + // Create a large file that doesn't have a zip header (generating random data _might_ make a zip magic + // number somewhere which is _not_ what we're trying to test). + let dir = TempDir::with_prefix("large-non-zip-bench").unwrap(); + let file = dir.path().join("zeros"); + let buf = vec![0u8; FILE_SIZE]; + fs::write(&file, &buf).unwrap(); + + bench.iter(|| { + assert!(zip::ZipArchive::new(std::fs::File::open(&file).unwrap()).is_err()); + }) +} + +benchmark_group!( + benches, + read_metadata, + parse_archive_with_comment, + parse_zip64_archive_with_comment, + parse_stream_archive, + parse_large_non_zip, +); +benchmark_main!(benches); diff --git a/v2/vendor/security/zip-7.2.0/cliff.toml b/v2/vendor/security/zip-7.2.0/cliff.toml new file mode 100644 index 0000000000..5c1ceed58a --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/cliff.toml @@ -0,0 +1,91 @@ +# git-cliff ~ default configuration file +# https://git-cliff.org/docs/configuration +# +# Lines starting with "#" are comments. +# Configuration options are organized into tables and keys. +# See documentation for more information on available options. + +[changelog] +# changelog header +header = """ +# Changelog\n +All notable changes to this project will be documented in this file.\n +""" +# template for the changelog body +# https://keats.github.io/tera/docs/#introduction +body = """ +{% if version %}\ + ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }} +{% else %}\ + ## [unreleased] +{% endif %}\ +{% for group, commits in commits | group_by(attribute="group") %} + ### {{ group | striptags | trim | upper_first }} + {% for commit in commits %} + - {% if commit.scope %}*({{ commit.scope }})* {% endif %}\ + {% if commit.breaking %}[**breaking**] {% endif %}\ + {{ commit.message | upper_first }}\ + {% endfor %} +{% endfor %}\n +""" +# template for the changelog footer +footer = """ + +""" +# remove the leading and trailing s +trim = true +# postprocessors +postprocessors = [ + # { pattern = '', replace = "https://github.com/orhun/git-cliff" }, # replace repository URL +] + +[git] +# parse the commits based on https://www.conventionalcommits.org +conventional_commits = true +# filter out the commits that are not conventional +filter_unconventional = true +# process each line of a commit as an individual commit +split_commits = false +# regex for preprocessing the commit messages +commit_preprocessors = [ + # Replace issue numbers + #{ pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](/issues/${2}))"}, + # Check spelling of the commit with https://github.com/crate-ci/typos + # If the spelling is incorrect, it will be automatically fixed. + #{ pattern = '.*', replace_command = 'typos --write-changes -' }, +] +# regex for parsing and grouping commits +commit_parsers = [ + { message = "^feat", group = "🚀 Features" }, + { message = "^fix", group = "🐛 Bug Fixes" }, + { message = "^doc", skip = true }, + { message = "^perf", group = "⚡ Performance" }, + { message = "^refactor", group = "🚜 Refactor" }, + { message = "^style", skip = true }, + { message = "^test", skip = true }, + { message = "^build", skip = true }, + { message = "^ci", skip = true }, + { message = "^chore\\(release\\)", skip = true }, + { message = "^chore\\(deps.*\\)", skip = true }, + { message = "^chore\\(pr\\)", skip = true }, + { message = "^chore\\(pull\\)", skip = true }, + { message = "^chore", group = "⚙️ Miscellaneous Tasks" }, + { body = ".*security", group = "🛡️ Security" }, + { message = "^revert", group = "◀️ Revert" }, +] +# protect breaking changes from being skipped due to matching a skipping commit_parser +protect_breaking_commits = true +# filter out the commits that are not matched by commit parsers +filter_commits = false +# regex for matching git tags +# tag_pattern = "v[0-9].*" +# regex for skipping tags +# skip_tags = "" +# regex for ignoring tags +# ignore_tags = "" +# sort the tags topologically +topo_order = false +# sort the commits inside sections by oldest/newest order +sort_commits = "oldest" +# limit the number of commits included in the changelog. +# limit_commits = 42 diff --git a/v2/vendor/security/zip-7.2.0/pull_request_template.md b/v2/vendor/security/zip-7.2.0/pull_request_template.md new file mode 100644 index 0000000000..9b3e77c179 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/pull_request_template.md @@ -0,0 +1,38 @@ + diff --git a/v2/vendor/security/zip-7.2.0/release-plz.toml b/v2/vendor/security/zip-7.2.0/release-plz.toml new file mode 100644 index 0000000000..58e434cbb1 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/release-plz.toml @@ -0,0 +1,27 @@ +[workspace] +dependencies_update = true # update dependencies with `cargo update` +pr_labels = ["release"] # add the `release` label to the release Pull Request +release_commits = "^(feat|fix|perf|refactor):" # prepare release only if at least one commit matches a regex +git_release_type = "auto" +release_always = false + +[changelog] +protect_breaking_commits = true +commit_parsers = [ + { message = "^feat", group = "🚀 Features" }, + { message = "^fix", group = "🐛 Bug Fixes" }, + { message = "^doc", skip = true }, + { message = "^perf", group = "⚡ Performance" }, + { message = "^refactor", group = "🚜 Refactor" }, + { message = "^style", skip = true }, + { message = "^test", skip = true }, + { message = "^build", skip = true }, + { message = "^ci", skip = true }, + { message = "^chore\\(release\\)", skip = true }, + { message = "^chore\\(deps.*\\)", skip = true }, + { message = "^chore\\(pr\\)", skip = true }, + { message = "^chore\\(pull\\)", skip = true }, + { message = "^chore", group = "⚙️ Miscellaneous Tasks" }, + { body = ".*security", group = "🛡️ Security" }, + { message = "^revert", group = "◀️ Revert" }, +] \ No newline at end of file diff --git a/v2/vendor/security/zip-7.2.0/ruview-provenance.json b/v2/vendor/security/zip-7.2.0/ruview-provenance.json new file mode 100644 index 0000000000..c043d76c61 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/ruview-provenance.json @@ -0,0 +1,16 @@ +{ + "crate": "zip", + "version": "7.2.0", + "registrySHA256": "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0", + "upstreamVCS": "46dc29cda6749736ddb1ec502aff06a65ffe54f8", + "fixPR": "https://github.com/zip-rs/zip2/pull/984", + "fixMerge": "a543a1fc8f3f3eef36f27bb137230295284cdb98", + "fixHead": "4d1c864bb62cc342147bf44c13a0a743fc844c4d", + "hostileSHA256": "0c68c3934f22651a6bd2726f7d58140124b1839196a1d0ae468864cdfda74676", + "licenseSHA256": "04dc52136d82400b11df5f474949863a70a5d05d2278dec45b99c479494450a6", + "securityPatchSHA256": "d7da70fb4223322f0915384589558fa732999eacc62f4eef09ca4cb58719280f", + "upstreamTestSourceCommit": "46dc29cda6749736ddb1ec502aff06a65ffe54f8", + "codeAdaptation": "final PR984 guards, read/mod.rs mapped to7.2 read.rs, stream invalid macro imported", + "tests": "original102lib+1symlink integration; upstream984 hostile test plus4097 boundary and stream-bit regression", + "manifestChanges": "only explicit ruview_0429_backport self-contained integration-test registration; version7.2.0 and licenses unchanged" +} diff --git a/v2/vendor/security/zip-7.2.0/src/aes.rs b/v2/vendor/security/zip-7.2.0/src/aes.rs new file mode 100644 index 0000000000..ec441f705c --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/aes.rs @@ -0,0 +1,407 @@ +//! Implementation of the AES decryption for zip files. +//! +//! This was implemented according to the [WinZip specification](https://www.winzip.com/win/en/aes_info.html). +//! Note that using CRC with AES depends on the used encryption specification, AE-1 or AE-2. +//! If the file is marked as encrypted with AE-2 the CRC field is ignored, even if it isn't set to 0. + +use crate::aes_ctr::AesCipher; +use crate::result::ZipResult; +use crate::types::AesMode; +use crate::{aes_ctr, result::ZipError}; +use constant_time_eq::constant_time_eq; +use hmac::{Hmac, Mac}; +use sha1::Sha1; +use std::io::{self, Error, ErrorKind, Read, Write}; +use zeroize::{Zeroize, Zeroizing}; + +/// The length of the password verification value in bytes +pub const PWD_VERIFY_LENGTH: usize = 2; +/// The length of the authentication code in bytes +const AUTH_CODE_LENGTH: usize = 10; +/// The number of iterations used with PBKDF2 +const ITERATION_COUNT: u32 = 1000; + +enum Cipher { + Aes128(Box>), + Aes192(Box>), + Aes256(Box>), +} + +impl Cipher { + /// Create a `Cipher` depending on the used `AesMode` and the given `key`. + /// + /// Returns `ZipError::InvalidPassword` if `key` doesn't have the correct size for the chosen + /// AES mode. + fn from_mode(aes_mode: AesMode, key: &[u8]) -> ZipResult { + Ok(match aes_mode { + AesMode::Aes128 => Cipher::Aes128(Box::new(aes_ctr::AesCtrZipKeyStream::< + aes_ctr::Aes128, + >::new(key)?)), + AesMode::Aes192 => Cipher::Aes192(Box::new(aes_ctr::AesCtrZipKeyStream::< + aes_ctr::Aes192, + >::new(key)?)), + AesMode::Aes256 => Cipher::Aes256(Box::new(aes_ctr::AesCtrZipKeyStream::< + aes_ctr::Aes256, + >::new(key)?)), + }) + } + + fn crypt_in_place(&mut self, target: &mut [u8]) { + match self { + Self::Aes128(cipher) => cipher.crypt_in_place(target), + Self::Aes192(cipher) => cipher.crypt_in_place(target), + Self::Aes256(cipher) => cipher.crypt_in_place(target), + } + } +} + +// An aes encrypted file starts with a salt, whose length depends on the used aes mode +// followed by a 2 byte password verification value +// then the variable length encrypted data +// and lastly a 10 byte authentication code +pub struct AesReader { + reader: R, + aes_mode: AesMode, + data_length: u64, +} + +impl AesReader { + pub const fn new(reader: R, aes_mode: AesMode, compressed_size: u64) -> AesReader { + let data_length = compressed_size + - (PWD_VERIFY_LENGTH + AUTH_CODE_LENGTH + aes_mode.salt_length()) as u64; + + Self { + reader, + aes_mode, + data_length, + } + } + + /// Read the AES header bytes and validate the password. + /// + /// Even if the validation succeeds, there is still a 1 in 65536 chance that an incorrect + /// password was provided. + /// It isn't possible to check the authentication code in this step. This will be done after + /// reading and decrypting the file. + pub fn validate(mut self, password: &[u8]) -> Result, ZipError> { + let salt_length = self.aes_mode.salt_length(); + let key_length = self.aes_mode.key_length(); + + let mut salt = vec![0; salt_length]; + self.reader.read_exact(&mut salt)?; + + // next are 2 bytes used for password verification + let mut pwd_verification_value = vec![0; PWD_VERIFY_LENGTH]; + self.reader.read_exact(&mut pwd_verification_value)?; + + // derive a key from the password and salt + // the length depends on the aes key length + let derived_key_len = 2 * key_length + PWD_VERIFY_LENGTH; + let mut derived_key: Box<[u8]> = vec![0; derived_key_len].into_boxed_slice(); + + // use PBKDF2 with HMAC-Sha1 to derive the key + pbkdf2::pbkdf2::>(password, &salt, ITERATION_COUNT, &mut derived_key) + .map_err(|e| Error::new(ErrorKind::InvalidInput, e))?; + let decrypt_key = &derived_key[0..key_length]; + let hmac_key = &derived_key[key_length..key_length * 2]; + let pwd_verify = &derived_key[derived_key_len - 2..]; + + // the last 2 bytes should equal the password verification value + if pwd_verification_value != pwd_verify { + // wrong password + return Err(ZipError::InvalidPassword); + } + + let cipher = Cipher::from_mode(self.aes_mode, decrypt_key)?; + let hmac = Hmac::::new_from_slice(hmac_key).unwrap(); + + Ok(AesReaderValid { + reader: self.reader, + data_remaining: self.data_length, + cipher, + hmac, + finalized: false, + }) + } + + /// Read the AES header bytes and returns the verification value and salt. + /// + /// # Returns + /// + /// the verification value and the salt + pub fn get_verification_value_and_salt( + mut self, + ) -> io::Result<([u8; PWD_VERIFY_LENGTH], Vec)> { + let salt_length = self.aes_mode.salt_length(); + + let mut salt = vec![0; salt_length]; + self.reader.read_exact(&mut salt)?; + + // next are 2 bytes used for password verification + let mut pwd_verification_value = [0; PWD_VERIFY_LENGTH]; + self.reader.read_exact(&mut pwd_verification_value)?; + Ok((pwd_verification_value, salt)) + } +} + +/// A reader for aes encrypted files, which has already passed the first password check. +/// +/// There is a 1 in 65536 chance that an invalid password passes that check. +/// After the data has been read and decrypted an HMAC will be checked and provide a final means +/// to check if either the password is invalid or if the data has been changed. +pub struct AesReaderValid { + reader: R, + data_remaining: u64, + cipher: Cipher, + hmac: Hmac, + finalized: bool, +} + +impl Read for AesReaderValid { + /// This implementation does not fulfill all requirements set in the trait documentation. + /// + /// ```txt + /// "If an error is returned then it must be guaranteed that no bytes were read." + /// ``` + /// + /// Whether this applies to errors that occur while reading the encrypted data depends on the + /// underlying reader. If the error occurs while verifying the HMAC, the reader might become + /// practically unusable, since its position after the error is not known. + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if self.data_remaining == 0 { + return Ok(0); + } + + // get the number of bytes to read, compare as u64 to make sure we can read more than + // 2^32 bytes even on 32 bit systems. + let bytes_to_read = self.data_remaining.min(buf.len() as u64) as usize; + let read = self.reader.read(&mut buf[0..bytes_to_read])?; + self.data_remaining -= read as u64; + + // Update the hmac with the encrypted data + self.hmac.update(&buf[0..read]); + + // decrypt the data + self.cipher.crypt_in_place(&mut buf[0..read]); + + // if there is no data left to read, check the integrity of the data + if self.data_remaining == 0 { + assert!( + !self.finalized, + "Tried to use an already finalized HMAC. This is a bug!" + ); + self.finalized = true; + + // Zip uses HMAC-Sha1-80, which only uses the first half of the hash + // see https://www.winzip.com/win/en/aes_info.html#auth-faq + let mut read_auth_code = [0; AUTH_CODE_LENGTH]; + self.reader.read_exact(&mut read_auth_code)?; + let computed_auth_code = &self.hmac.finalize_reset().into_bytes()[0..AUTH_CODE_LENGTH]; + + // use constant time comparison to mitigate timing attacks + if !constant_time_eq(computed_auth_code, &read_auth_code) { + return Err( + Error::new( + ErrorKind::InvalidData, + "Invalid authentication code, this could be due to an invalid password or errors in the data" + ) + ); + } + } + + Ok(read) + } +} + +impl AesReaderValid { + /// Consumes this decoder, returning the underlying reader. + pub fn into_inner(self) -> R { + self.reader + } +} + +pub struct AesWriter { + writer: W, + cipher: Cipher, + hmac: Hmac, + buffer: Zeroizing>, + encrypted_file_header: Option>, +} + +impl AesWriter { + pub fn new(writer: W, aes_mode: AesMode, password: &[u8]) -> ZipResult { + let salt_length = aes_mode.salt_length(); + let key_length = aes_mode.key_length(); + + let mut encrypted_file_header = Vec::with_capacity(salt_length + 2); + + let mut salt = vec![0; salt_length]; + getrandom::fill(&mut salt).map_err(|e| ZipError::Io(e.into()))?; + encrypted_file_header.write_all(&salt)?; + + // Derive a key from the password and salt. The length depends on the aes key length + let derived_key_len = 2 * key_length + PWD_VERIFY_LENGTH; + let mut derived_key: Zeroizing> = Zeroizing::new(vec![0; derived_key_len]); + + // Use PBKDF2 with HMAC-Sha1 to derive the key. + pbkdf2::pbkdf2::>(password, &salt, ITERATION_COUNT, &mut derived_key) + .map_err(|e| Error::new(ErrorKind::InvalidInput, e))?; + let encryption_key = &derived_key[0..key_length]; + let hmac_key = &derived_key[key_length..key_length * 2]; + + let pwd_verify = derived_key[derived_key_len - 2..].to_vec(); + encrypted_file_header.write_all(&pwd_verify)?; + + let cipher = Cipher::from_mode(aes_mode, encryption_key)?; + let hmac = Hmac::::new_from_slice(hmac_key).unwrap(); + + Ok(Self { + writer, + cipher, + hmac, + buffer: Default::default(), + encrypted_file_header: Some(encrypted_file_header), + }) + } + + pub fn finish(mut self) -> io::Result { + self.write_encrypted_file_header()?; + + // Zip uses HMAC-Sha1-80, which only uses the first half of the hash + // see https://www.winzip.com/win/en/aes_info.html#auth-faq + let computed_auth_code = &self.hmac.finalize_reset().into_bytes()[0..AUTH_CODE_LENGTH]; + self.writer.write_all(computed_auth_code)?; + + Ok(self.writer) + } + + /// The AES encryption specification requires some metadata being written at the start of the + /// file data section, but this can only be done once the extra data writing has been finished + /// so we can't do it when the writer is constructed. + fn write_encrypted_file_header(&mut self) -> io::Result<()> { + if let Some(header) = self.encrypted_file_header.take() { + self.writer.write_all(&header)?; + } + + Ok(()) + } +} + +impl Write for AesWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.write_encrypted_file_header()?; + + // Fill the internal buffer and encrypt it in-place. + self.buffer.extend_from_slice(buf); + self.cipher.crypt_in_place(&mut self.buffer[..]); + + // Update the hmac with the encrypted data. + self.hmac.update(&self.buffer[..]); + + // Write the encrypted buffer to the inner writer. We need to use `write_all` here as if + // we only write parts of the data we can't easily reverse the keystream in the cipher + // implementation. + self.writer.write_all(&self.buffer[..])?; + + // Zeroize the backing memory before clearing the buffer to prevent cleartext data from + // being left in memory. + self.buffer.zeroize(); + self.buffer.clear(); + + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.writer.flush() + } +} + +#[cfg(all(test, feature = "aes-crypto"))] +mod tests { + use std::io::{self, Read, Write}; + + use crate::{ + aes::{AesReader, AesWriter}, + result::ZipError, + types::AesMode, + }; + + /// Checks whether `AesReader` can successfully decrypt what `AesWriter` produces. + fn roundtrip(aes_mode: AesMode, password: &[u8], plaintext: &[u8]) -> Result { + let mut buf = io::Cursor::new(vec![]); + let mut read_buffer = vec![]; + + { + let mut writer = AesWriter::new(&mut buf, aes_mode, password)?; + writer.write_all(plaintext)?; + writer.finish()?; + } + + // Reset cursor position to the beginning. + buf.set_position(0); + + { + let compressed_length = buf.get_ref().len() as u64; + let mut reader = + AesReader::new(&mut buf, aes_mode, compressed_length).validate(password)?; + reader.read_to_end(&mut read_buffer)?; + } + + Ok(plaintext == read_buffer) + } + + #[test] + fn crypt_aes_256_0_byte() { + let plaintext = &[]; + let password = b"some super secret password"; + assert!(roundtrip(AesMode::Aes256, password, plaintext).expect("could encrypt and decrypt")); + } + + #[test] + fn crypt_aes_128_5_byte() { + let plaintext = b"asdf\n"; + let password = b"some super secret password"; + + assert!(roundtrip(AesMode::Aes128, password, plaintext).expect("could encrypt and decrypt")); + } + + #[test] + fn crypt_aes_192_5_byte() { + let plaintext = b"asdf\n"; + let password = b"some super secret password"; + + assert!(roundtrip(AesMode::Aes192, password, plaintext).expect("could encrypt and decrypt")); + } + + #[test] + fn crypt_aes_256_5_byte() { + let plaintext = b"asdf\n"; + let password = b"some super secret password"; + + assert!(roundtrip(AesMode::Aes256, password, plaintext).expect("could encrypt and decrypt")); + } + + #[test] + fn crypt_aes_128_40_byte() { + let plaintext = b"Lorem ipsum dolor sit amet, consectetur\n"; + let password = b"some super secret password"; + + assert!(roundtrip(AesMode::Aes128, password, plaintext).expect("could encrypt and decrypt")); + } + + #[test] + fn crypt_aes_192_40_byte() { + let plaintext = b"Lorem ipsum dolor sit amet, consectetur\n"; + let password = b"some super secret password"; + + assert!(roundtrip(AesMode::Aes192, password, plaintext).expect("could encrypt and decrypt")); + } + + #[test] + fn crypt_aes_256_40_byte() { + let plaintext = b"Lorem ipsum dolor sit amet, consectetur\n"; + let password = b"some super secret password"; + + assert!(roundtrip(AesMode::Aes256, password, plaintext).expect("could encrypt and decrypt")); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/aes_ctr.rs b/v2/vendor/security/zip-7.2.0/src/aes_ctr.rs new file mode 100644 index 0000000000..7b3a3c0c4d --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/aes_ctr.rs @@ -0,0 +1,277 @@ +//! A counter mode (CTR) for AES to work with the encryption used in zip files. +//! +//! This was implemented since the zip specification requires the mode to not use a nonce and uses a +//! different byte order (little endian) than NIST (big endian). +//! See [AesCtrZipKeyStream] for more information. + +use crate::result::{ZipError, ZipResult}; +use aes::cipher::{BlockEncrypt, KeyInit}; +use core::{any, fmt}; + +/// Internal block size of an AES cipher. +const AES_BLOCK_SIZE: usize = 16; + +/// AES-128. +#[derive(Debug)] +pub struct Aes128; +/// AES-192. +#[derive(Debug)] +pub struct Aes192; +/// AES-256. +#[derive(Debug)] +pub struct Aes256; + +/// An AES cipher kind. +pub trait AesKind { + /// Key type. + type Key: AsRef<[u8]>; + /// Cipher used to decrypt. + type Cipher: KeyInit; +} + +impl AesKind for Aes128 { + type Key = [u8; 16]; + type Cipher = aes::Aes128; +} + +impl AesKind for Aes192 { + type Key = [u8; 24]; + type Cipher = aes::Aes192; +} + +impl AesKind for Aes256 { + type Key = [u8; 32]; + type Cipher = aes::Aes256; +} + +/// An AES-CTR key stream generator. +/// +/// Implements the slightly non-standard AES-CTR variant used by WinZip AES encryption. +/// +/// Typical AES-CTR implementations combine a nonce with a 64 bit counter. WinZIP AES instead uses +/// no nonce and also uses a different byte order (little endian) than NIST (big endian). +/// +/// The stream implements the `Read` trait; encryption or decryption is performed by XOR-ing the +/// bytes from the key stream with the ciphertext/plaintext. +pub struct AesCtrZipKeyStream { + /// Current AES counter. + counter: u128, + /// AES cipher instance. + cipher: C::Cipher, + /// Stores the currently available keystream bytes. + buffer: [u8; AES_BLOCK_SIZE], + /// Number of bytes already used up from `buffer`. + pos: usize, +} + +impl fmt::Debug for AesCtrZipKeyStream +where + C: AesKind, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "AesCtrZipKeyStream<{}>(counter: {})", + any::type_name::(), + self.counter + ) + } +} + +impl AesCtrZipKeyStream +where + C: AesKind, + C::Cipher: KeyInit, +{ + /// Creates a new zip variant AES-CTR key stream. + /// + /// Returns `ZipError::InvalidPassword` if `key` doesn't have the correct size for cipher `C`. + pub fn new(key: &[u8]) -> ZipResult> { + Ok(AesCtrZipKeyStream { + counter: 1, + cipher: C::Cipher::new_from_slice(key).map_err(|_| ZipError::InvalidPassword)?, + buffer: [0u8; AES_BLOCK_SIZE], + pos: AES_BLOCK_SIZE, + }) + } +} + +impl AesCipher for AesCtrZipKeyStream +where + C: AesKind, + C::Cipher: BlockEncrypt, +{ + /// Decrypt or encrypt `target`. + #[inline] + fn crypt_in_place(&mut self, mut target: &mut [u8]) { + while !target.is_empty() { + if self.pos == AES_BLOCK_SIZE { + self.buffer = self.counter.to_le_bytes(); + self.cipher.encrypt_block(self.buffer.as_mut().into()); + self.counter += 1; + self.pos = 0; + } + + let target_len = target.len().min(AES_BLOCK_SIZE - self.pos); + + xor( + &mut target[0..target_len], + &self.buffer[self.pos..(self.pos + target_len)], + ); + target = &mut target[target_len..]; + self.pos += target_len; + } + } +} + +/// This trait allows using generic AES ciphers with different key sizes. +pub trait AesCipher { + fn crypt_in_place(&mut self, target: &mut [u8]); +} + +/// XORs a slice in place with another slice. +#[inline] +fn xor(dest: &mut [u8], src: &[u8]) { + debug_assert_eq!( + dest.len(), + src.len(), + "XOR operation requires destination and source slices to have equal length" + ); + + for (lhs, rhs) in dest.iter_mut().zip(src.iter()) { + *lhs ^= *rhs; + } +} + +#[cfg(test)] +mod tests { + use super::{Aes128, Aes192, Aes256, AesCipher, AesCtrZipKeyStream, AesKind}; + use aes::cipher::{BlockEncrypt, KeyInit}; + + /// Checks whether `crypt_in_place` produces the correct plaintext after one use and yields the + /// ciphertext again after applying it again. + fn roundtrip(key: &[u8], ciphertext: &[u8], expected_plaintext: &[u8]) + where + Aes: AesKind, + Aes::Cipher: KeyInit + BlockEncrypt, + { + let mut key_stream = AesCtrZipKeyStream::::new(key).unwrap(); + + let mut plaintext = ciphertext.to_vec().into_boxed_slice(); + key_stream.crypt_in_place(&mut plaintext); + assert_eq!(*plaintext, *expected_plaintext); + + // Round-tripping should yield the ciphertext again. + let mut key_stream = AesCtrZipKeyStream::::new(key).unwrap(); + key_stream.crypt_in_place(&mut plaintext); + assert_eq!(*plaintext, *ciphertext); + } + + #[test] + fn new_with_wrong_key_size() { + AesCtrZipKeyStream::::new(&[1, 2, 3, 4, 5]) + .expect_err("Wrong key size should fail"); + } + + // The data used in these tests was generated with p7zip without any compression. + // It's not possible to recreate the exact same data, since a random salt is used for encryption. + // `7z a -phelloworld -mem=AES256 -mx=0 aes256_40byte.zip 40byte_data.txt` + #[test] + fn crypt_aes_256_0_byte() { + let ciphertext = []; + let expected_plaintext = &[]; + let key = [ + 0x0b, 0xec, 0x2e, 0xf2, 0x46, 0xf0, 0x7e, 0x35, 0x16, 0x54, 0xe0, 0x98, 0x10, 0xb3, + 0x18, 0x55, 0x24, 0xa3, 0x9e, 0x0e, 0x40, 0xe7, 0x92, 0xad, 0xb2, 0x8a, 0x48, 0xf4, + 0x5c, 0xd0, 0xc0, 0x54, + ]; + + roundtrip::(&key, &ciphertext, expected_plaintext); + } + + #[test] + fn crypt_aes_128_5_byte() { + let ciphertext = [0x98, 0xa9, 0x8c, 0x26, 0x0e]; + let expected_plaintext = b"asdf\n"; + let key = [ + 0xe0, 0x25, 0x7b, 0x57, 0x97, 0x6a, 0xa4, 0x23, 0xab, 0x94, 0xaa, 0x44, 0xfd, 0x47, + 0x4f, 0xa5, + ]; + + roundtrip::(&key, &ciphertext, expected_plaintext); + } + + #[test] + fn crypt_aes_192_5_byte() { + let ciphertext = [0x36, 0x55, 0x5c, 0x61, 0x3c]; + let expected_plaintext = b"asdf\n"; + let key = [ + 0xe4, 0x4a, 0x88, 0x52, 0x8f, 0xf7, 0x0b, 0x81, 0x7b, 0x75, 0xf1, 0x74, 0x21, 0x37, + 0x8c, 0x90, 0xad, 0xbe, 0x4a, 0x65, 0xa8, 0x96, 0x0e, 0xcc, + ]; + + roundtrip::(&key, &ciphertext, expected_plaintext); + } + + #[test] + fn crypt_aes_256_5_byte() { + let ciphertext = [0xc2, 0x47, 0xc0, 0xdc, 0x56]; + let expected_plaintext = b"asdf\n"; + let key = [ + 0x79, 0x5e, 0x17, 0xf2, 0xc6, 0x3d, 0x28, 0x9b, 0x4b, 0x4b, 0xbb, 0xa9, 0xba, 0xc9, + 0xa5, 0xee, 0x3a, 0x4f, 0x0f, 0x4b, 0x29, 0xbd, 0xe9, 0xb8, 0x41, 0x9c, 0x41, 0xa5, + 0x15, 0xb2, 0x86, 0xab, + ]; + + roundtrip::(&key, &ciphertext, expected_plaintext); + } + + #[test] + fn crypt_aes_128_40_byte() { + let ciphertext = [ + 0xcf, 0x72, 0x6b, 0xa1, 0xb2, 0x0f, 0xdf, 0xaa, 0x10, 0xad, 0x9c, 0x7f, 0x6d, 0x1c, + 0x8d, 0xb5, 0x16, 0x7e, 0xbb, 0x11, 0x69, 0x52, 0x8c, 0x89, 0x80, 0x32, 0xaa, 0x76, + 0xa6, 0x18, 0x31, 0x98, 0xee, 0xdd, 0x22, 0x68, 0xb7, 0xe6, 0x77, 0xd2, + ]; + let expected_plaintext = b"Lorem ipsum dolor sit amet, consectetur\n"; + let key = [ + 0x43, 0x2b, 0x6d, 0xbe, 0x05, 0x76, 0x6c, 0x9e, 0xde, 0xca, 0x3b, 0xf8, 0xaf, 0x5d, + 0x81, 0xb6, + ]; + + roundtrip::(&key, &ciphertext, expected_plaintext); + } + + #[test] + fn crypt_aes_192_40_byte() { + let ciphertext = [ + 0xa6, 0xfc, 0x52, 0x79, 0x2c, 0x6c, 0xfe, 0x68, 0xb1, 0xa8, 0xb3, 0x07, 0x52, 0x8b, + 0x82, 0xa6, 0x87, 0x9c, 0x72, 0x42, 0x3a, 0xf8, 0xc6, 0xa9, 0xc9, 0xfb, 0x61, 0x19, + 0x37, 0xb9, 0x56, 0x62, 0xf4, 0xfc, 0x5e, 0x7a, 0xdd, 0x55, 0x0a, 0x48, + ]; + let expected_plaintext = b"Lorem ipsum dolor sit amet, consectetur\n"; + let key = [ + 0xac, 0x92, 0x41, 0xba, 0xde, 0xd9, 0x02, 0xfe, 0x40, 0x92, 0x20, 0xf6, 0x56, 0x03, + 0xfe, 0xae, 0x1b, 0xba, 0x01, 0x97, 0x97, 0x79, 0xbb, 0xa6, + ]; + + roundtrip::(&key, &ciphertext, expected_plaintext); + } + + #[test] + fn crypt_aes_256_40_byte() { + let ciphertext = [ + 0xa9, 0x99, 0xbd, 0xea, 0x82, 0x9b, 0x8f, 0x2f, 0xb7, 0x52, 0x2f, 0x6b, 0xd8, 0xf6, + 0xab, 0x0e, 0x24, 0x51, 0x9e, 0x18, 0x0f, 0xc0, 0x8f, 0x54, 0x15, 0x80, 0xae, 0xbc, + 0xa0, 0x5c, 0x8a, 0x11, 0x8d, 0x14, 0x7e, 0xc5, 0xb4, 0xae, 0xd3, 0x37, + ]; + let expected_plaintext = b"Lorem ipsum dolor sit amet, consectetur\n"; + let key = [ + 0x64, 0x7c, 0x7a, 0xde, 0xf0, 0xf2, 0x61, 0x49, 0x1c, 0xf1, 0xf1, 0xe3, 0x37, 0xfc, + 0xe1, 0x4d, 0x4a, 0x77, 0xd4, 0xeb, 0x9e, 0x3d, 0x75, 0xce, 0x9a, 0x3e, 0x10, 0x50, + 0xc2, 0x07, 0x36, 0xb6, + ]; + + roundtrip::(&key, &ciphertext, expected_plaintext); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/compression.rs b/v2/vendor/security/zip-7.2.0/src/compression.rs new file mode 100644 index 0000000000..2848f8749a --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/compression.rs @@ -0,0 +1,557 @@ +//! Possible ZIP compression methods. + +use crate::cfg_if_expr; +use core::fmt; +use std::io; + +#[allow(deprecated)] +/// Identifies the storage format used to compress a file within a ZIP archive. +/// +/// Each file's compression method is stored alongside it, allowing the +/// contents to be read without context. +/// +/// When creating ZIP files, you may choose the method to use with +/// [`crate::write::FileOptions::compression_method`] +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +#[cfg_attr(feature = "_arbitrary", derive(arbitrary::Arbitrary))] +#[non_exhaustive] +pub enum CompressionMethod { + /// Store the file as is + Stored, + /// Compress the file using Deflate + #[cfg(feature = "_deflate-any")] + Deflated, + /// Compress the file using Deflate64. + /// Decoding deflate64 is supported but encoding deflate64 is not supported. + #[cfg(feature = "deflate64")] + Deflate64, + /// Compress the file using BZIP2 + #[cfg(feature = "_bzip2_any")] + Bzip2, + /// Encrypted using AES. + /// + /// The actual compression method has to be taken from the AES extra data field + /// or from `ZipFileData`. + #[cfg(feature = "aes-crypto")] + Aes, + /// Compress the file using ZStandard + #[cfg(feature = "zstd")] + Zstd, + /// Compress the file using LZMA + #[cfg(feature = "lzma")] + Lzma, + #[cfg(feature = "legacy-zip")] + /// Method 1 Shrink + Shrink, + #[cfg(feature = "legacy-zip")] + /// Reduce (Method 2-5) + Reduce(u8), + #[cfg(feature = "legacy-zip")] + /// Method 6 Implode/explode + Implode, + /// Compress the file using XZ + #[cfg(feature = "xz")] + Xz, + /// Compress the file using PPMd + #[cfg(feature = "ppmd")] + Ppmd, + /// Unsupported compression method + #[cfg_attr( + not(any(fuzzing, feature = "_arbitrary")), + deprecated(since = "0.5.7", note = "use the constants instead") + )] + Unsupported(u16), +} +#[allow(deprecated, missing_docs)] +/// All compression methods defined for the ZIP format +impl CompressionMethod { + pub const STORE: Self = CompressionMethod::Stored; + #[cfg(feature = "legacy-zip")] + pub const SHRINK: Self = CompressionMethod::Shrink; + #[cfg(not(feature = "legacy-zip"))] + /// Legacy compression method (enable feature `legacy-zip` to get support) + pub const SHRINK: Self = CompressionMethod::Unsupported(1); + #[cfg(feature = "legacy-zip")] + /// Legacy compression method + pub const REDUCE_1: Self = CompressionMethod::Reduce(1); + #[cfg(not(feature = "legacy-zip"))] + /// Legacy compression method (enable feature `legacy-zip` to get support) + pub const REDUCE_1: Self = CompressionMethod::Unsupported(2); + #[cfg(feature = "legacy-zip")] + /// Legacy compression method + pub const REDUCE_2: Self = CompressionMethod::Reduce(2); + #[cfg(not(feature = "legacy-zip"))] + /// Legacy compression method (enable feature `legacy-zip` to get support) + pub const REDUCE_2: Self = CompressionMethod::Unsupported(3); + #[cfg(feature = "legacy-zip")] + /// Legacy compression method + pub const REDUCE_3: Self = CompressionMethod::Reduce(3); + #[cfg(not(feature = "legacy-zip"))] + /// Legacy compression method (enable feature `legacy-zip` to get support) + pub const REDUCE_3: Self = CompressionMethod::Unsupported(4); + #[cfg(feature = "legacy-zip")] + /// Legacy compression method + pub const REDUCE_4: Self = CompressionMethod::Reduce(4); + #[cfg(not(feature = "legacy-zip"))] + /// Legacy compression method (enable feature `legacy-zip` to get support) + pub const REDUCE_4: Self = CompressionMethod::Unsupported(5); + #[cfg(feature = "legacy-zip")] + /// Legacy compression method + pub const IMPLODE: Self = CompressionMethod::Implode; + #[cfg(not(feature = "legacy-zip"))] + /// Legacy compression method (enable feature `legacy-zip` to get support) + pub const IMPLODE: Self = CompressionMethod::Unsupported(6); + #[cfg(feature = "_deflate-any")] + pub const DEFLATE: Self = CompressionMethod::Deflated; + #[cfg(not(feature = "_deflate-any"))] + pub const DEFLATE: Self = CompressionMethod::Unsupported(8); + #[cfg(feature = "deflate64")] + pub const DEFLATE64: Self = CompressionMethod::Deflate64; + #[cfg(not(feature = "deflate64"))] + pub const DEFLATE64: Self = CompressionMethod::Unsupported(9); + pub const PKWARE_IMPLODE: Self = CompressionMethod::Unsupported(10); + #[cfg(feature = "_bzip2_any")] + pub const BZIP2: Self = CompressionMethod::Bzip2; + #[cfg(not(feature = "_bzip2_any"))] + pub const BZIP2: Self = CompressionMethod::Unsupported(12); + #[cfg(not(feature = "lzma"))] + pub const LZMA: Self = CompressionMethod::Unsupported(14); + #[cfg(feature = "lzma")] + pub const LZMA: Self = CompressionMethod::Lzma; + pub const IBM_ZOS_CMPSC: Self = CompressionMethod::Unsupported(16); + pub const IBM_TERSE: Self = CompressionMethod::Unsupported(18); + pub const ZSTD_DEPRECATED: Self = CompressionMethod::Unsupported(20); + #[cfg(feature = "zstd")] + pub const ZSTD: Self = CompressionMethod::Zstd; + #[cfg(not(feature = "zstd"))] + pub const ZSTD: Self = CompressionMethod::Unsupported(93); + pub const MP3: Self = CompressionMethod::Unsupported(94); + #[cfg(feature = "xz")] + pub const XZ: Self = CompressionMethod::Xz; + #[cfg(not(feature = "xz"))] + pub const XZ: Self = CompressionMethod::Unsupported(95); + pub const JPEG: Self = CompressionMethod::Unsupported(96); + pub const WAVPACK: Self = CompressionMethod::Unsupported(97); + #[cfg(feature = "ppmd")] + pub const PPMD: Self = CompressionMethod::Ppmd; + #[cfg(not(feature = "ppmd"))] + pub const PPMD: Self = CompressionMethod::Unsupported(98); + #[cfg(feature = "aes-crypto")] + pub const AES: Self = CompressionMethod::Aes; + #[cfg(not(feature = "aes-crypto"))] + pub const AES: Self = CompressionMethod::Unsupported(99); + + #[cfg(feature = "_deflate-any")] + pub const DEFAULT: Self = CompressionMethod::Deflated; + + #[cfg(all(not(feature = "_deflate-any"), feature = "_bzip2_any"))] + pub const DEFAULT: Self = CompressionMethod::Bzip2; + + #[cfg(all(not(feature = "_deflate-any"), not(feature = "_bzip2_any")))] + pub const DEFAULT: Self = CompressionMethod::Stored; +} +impl CompressionMethod { + pub(crate) const fn parse_from_u16(val: u16) -> Self { + match val { + 0 => CompressionMethod::Stored, + #[cfg(feature = "legacy-zip")] + 1 => CompressionMethod::Shrink, + #[cfg(feature = "legacy-zip")] + 2 => CompressionMethod::Reduce(1), + #[cfg(feature = "legacy-zip")] + 3 => CompressionMethod::Reduce(2), + #[cfg(feature = "legacy-zip")] + 4 => CompressionMethod::Reduce(3), + #[cfg(feature = "legacy-zip")] + 5 => CompressionMethod::Reduce(4), + #[cfg(feature = "legacy-zip")] + 6 => CompressionMethod::Implode, + #[cfg(feature = "_deflate-any")] + 8 => CompressionMethod::Deflated, + #[cfg(feature = "deflate64")] + 9 => CompressionMethod::Deflate64, + #[cfg(feature = "_bzip2_any")] + 12 => CompressionMethod::Bzip2, + #[cfg(feature = "lzma")] + 14 => CompressionMethod::Lzma, + #[cfg(feature = "xz")] + 95 => CompressionMethod::Xz, + #[cfg(feature = "zstd")] + 93 => CompressionMethod::Zstd, + #[cfg(feature = "ppmd")] + 98 => CompressionMethod::Ppmd, + #[cfg(feature = "aes-crypto")] + 99 => CompressionMethod::Aes, + #[allow(deprecated)] + v => CompressionMethod::Unsupported(v), + } + } + + /// Converts a u16 to its corresponding CompressionMethod + #[deprecated( + since = "0.5.7", + note = "use a constant to construct a compression method" + )] + pub const fn from_u16(val: u16) -> CompressionMethod { + Self::parse_from_u16(val) + } + + pub(crate) const fn serialize_to_u16(self) -> u16 { + match self { + CompressionMethod::Stored => 0, + #[cfg(feature = "legacy-zip")] + CompressionMethod::Shrink => 1, + #[cfg(feature = "legacy-zip")] + CompressionMethod::Reduce(n) => 1 + n as u16, + #[cfg(feature = "legacy-zip")] + CompressionMethod::Implode => 6, + + #[cfg(feature = "_deflate-any")] + CompressionMethod::Deflated => 8, + #[cfg(feature = "deflate64")] + CompressionMethod::Deflate64 => 9, + #[cfg(feature = "_bzip2_any")] + CompressionMethod::Bzip2 => 12, + #[cfg(feature = "aes-crypto")] + CompressionMethod::Aes => 99, + #[cfg(feature = "zstd")] + CompressionMethod::Zstd => 93, + #[cfg(feature = "lzma")] + CompressionMethod::Lzma => 14, + #[cfg(feature = "xz")] + CompressionMethod::Xz => 95, + #[cfg(feature = "ppmd")] + CompressionMethod::Ppmd => 98, + #[allow(deprecated)] + CompressionMethod::Unsupported(v) => v, + } + } + + /// Converts a CompressionMethod to a u16 + #[deprecated( + since = "0.5.7", + note = "to match on other compression methods, use a constant" + )] + pub const fn to_u16(self) -> u16 { + self.serialize_to_u16() + } +} + +impl Default for CompressionMethod { + fn default() -> Self { + cfg_if_expr! { + #[cfg(feature = "_deflate-any")] => CompressionMethod::Deflated, + _ => CompressionMethod::Stored + } + } +} + +impl fmt::Display for CompressionMethod { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // Just duplicate what the Debug format looks like, i.e, the enum key: + write!(f, "{self:?}") + } +} + +/// The compression methods which have been implemented. +pub const SUPPORTED_COMPRESSION_METHODS: &[CompressionMethod] = &[ + CompressionMethod::Stored, + #[cfg(feature = "_deflate-any")] + CompressionMethod::Deflated, + #[cfg(feature = "deflate64")] + CompressionMethod::Deflate64, + #[cfg(feature = "_bzip2_any")] + CompressionMethod::Bzip2, + #[cfg(feature = "zstd")] + CompressionMethod::Zstd, + #[cfg(feature = "xz")] + CompressionMethod::Xz, + #[cfg(feature = "ppmd")] + CompressionMethod::Ppmd, +]; + +pub(crate) enum Decompressor { + Stored(R), + #[cfg(feature = "deflate-flate2")] + Deflated(flate2::bufread::DeflateDecoder), + #[cfg(feature = "deflate64")] + Deflate64(deflate64::Deflate64Decoder), + #[cfg(feature = "_bzip2_any")] + Bzip2(bzip2::bufread::BzDecoder), + #[cfg(feature = "zstd")] + Zstd(zstd::Decoder<'static, R>), + #[cfg(feature = "lzma")] + Lzma(Lzma), + #[cfg(feature = "legacy-zip")] + Shrink(crate::legacy::shrink::ShrinkDecoder), + #[cfg(feature = "legacy-zip")] + Reduce(crate::legacy::reduce::ReduceDecoder), + #[cfg(feature = "legacy-zip")] + Implode(crate::legacy::implode::ImplodeDecoder), + #[cfg(feature = "xz")] + Xz(Box>), + #[cfg(feature = "ppmd")] + Ppmd(Ppmd), +} + +#[cfg(feature = "lzma")] +pub(crate) enum Lzma { + Uninitialized { + reader: Option, + uncompressed_size: u64, + }, + Initialized(Box>), +} + +#[cfg(feature = "ppmd")] +pub(crate) enum Ppmd { + Uninitialized(Option), + Initialized(Box>), +} + +impl io::Read for Decompressor { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + Decompressor::Stored(r) => r.read(buf), + #[cfg(feature = "deflate-flate2")] + Decompressor::Deflated(r) => r.read(buf), + #[cfg(feature = "deflate64")] + Decompressor::Deflate64(r) => r.read(buf), + #[cfg(feature = "_bzip2_any")] + Decompressor::Bzip2(r) => r.read(buf), + #[cfg(feature = "zstd")] + Decompressor::Zstd(r) => r.read(buf), + #[cfg(feature = "lzma")] + Decompressor::Lzma(r) => match r { + Lzma::Uninitialized { + reader, + uncompressed_size, + } => { + let mut reader = reader.take().ok_or_else(|| { + io::Error::other("Reader was not set while reading LZMA data") + })?; + + // 5.8.8.1 LZMA Version Information & 5.8.8.2 LZMA Properties Size + let mut header = [0; 4]; + reader.read_exact(&mut header)?; + let _version_information = u16::from_le_bytes(header[0..2].try_into().unwrap()); + let properties_size = u16::from_le_bytes(header[2..4].try_into().unwrap()); + if properties_size != 5 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("unexpected LZMA properties size of {properties_size}"), + )); + } + + let mut props_data = [0; 5]; + reader.read_exact(&mut props_data)?; + let props = props_data[0]; + let dict_size = u32::from_le_bytes(props_data[1..5].try_into().unwrap()); + + // We don't need to handle the end-of-stream marker here, since the LZMA reader + // stops at the end-of-stream marker OR when it has decoded uncompressed_size bytes, whichever comes first. + let mut decompressor = lzma_rust2::LzmaReader::new_with_props( + reader, + *uncompressed_size, + props, + dict_size, + None, + )?; + + let read = decompressor.read(buf)?; + + *r = Lzma::Initialized(Box::new(decompressor)); + + Ok(read) + } + Lzma::Initialized(decompressor) => decompressor.read(buf), + }, + #[cfg(feature = "xz")] + Decompressor::Xz(r) => r.read(buf), + #[cfg(feature = "ppmd")] + Decompressor::Ppmd(r) => match r { + Ppmd::Uninitialized(reader) => { + let mut reader = reader.take().ok_or_else(|| { + io::Error::other("Reader was not set while reading PPMd data") + })?; + + let mut buffer = [0; 2]; + reader.read_exact(&mut buffer)?; + let parameters = u16::from_le_bytes(buffer); + + let order = ((parameters & 0x0F) + 1) as u32; + let memory_size = 1024 * 1024 * (((parameters >> 4) & 0xFF) + 1) as u32; + let restoration_method = (parameters >> 12) & 0x0F; + + let mut decompressor = ppmd_rust::Ppmd8Decoder::new( + reader, + order, + memory_size, + restoration_method.into(), + ) + .map_err(|error| match error { + ppmd_rust::Error::RangeDecoderInitialization => io::Error::new( + io::ErrorKind::InvalidData, + "PPMd range coder initialization failed", + ), + ppmd_rust::Error::InvalidParameter => { + io::Error::new(io::ErrorKind::InvalidInput, "Invalid PPMd parameter") + } + ppmd_rust::Error::IoError(io_error) => io_error, + ppmd_rust::Error::MemoryAllocation => { + io::Error::new(io::ErrorKind::OutOfMemory, "Memory allocation failed") + } + })?; + + let read = decompressor.read(buf)?; + + *r = Ppmd::Initialized(Box::new(decompressor)); + + Ok(read) + } + Ppmd::Initialized(decompressor) => decompressor.read(buf), + }, + #[cfg(feature = "legacy-zip")] + Decompressor::Shrink(r) => r.read(buf), + #[cfg(feature = "legacy-zip")] + Decompressor::Reduce(r) => r.read(buf), + #[cfg(feature = "legacy-zip")] + Decompressor::Implode(r) => r.read(buf), + } + } +} + +impl Decompressor { + pub fn new( + reader: R, + compression_method: CompressionMethod, + #[cfg(any(feature = "lzma", feature = "legacy-zip"))] uncompressed_size: u64, + #[cfg(not(any(feature = "lzma", feature = "legacy-zip")))] _uncompressed_size: u64, + #[cfg(feature = "legacy-zip")] flags: u16, + #[cfg(not(feature = "legacy-zip"))] _flags: u16, + ) -> crate::result::ZipResult { + Ok(match compression_method { + CompressionMethod::Stored => Decompressor::Stored(reader), + #[cfg(feature = "deflate-flate2")] + CompressionMethod::Deflated => { + Decompressor::Deflated(flate2::bufread::DeflateDecoder::new(reader)) + } + #[cfg(feature = "deflate64")] + CompressionMethod::Deflate64 => { + Decompressor::Deflate64(deflate64::Deflate64Decoder::with_buffer(reader)) + } + #[cfg(feature = "_bzip2_any")] + CompressionMethod::Bzip2 => Decompressor::Bzip2(bzip2::bufread::BzDecoder::new(reader)), + #[cfg(feature = "zstd")] + CompressionMethod::Zstd => Decompressor::Zstd(zstd::Decoder::with_buffer(reader)?), + #[cfg(feature = "lzma")] + CompressionMethod::Lzma => Decompressor::Lzma(Lzma::Uninitialized { + reader: Some(reader), + uncompressed_size, + }), + #[cfg(feature = "xz")] + CompressionMethod::Xz => { + Decompressor::Xz(Box::new(lzma_rust2::XzReader::new(reader, false))) + } + #[cfg(feature = "ppmd")] + CompressionMethod::Ppmd => Decompressor::Ppmd(Ppmd::Uninitialized(Some(reader))), + #[cfg(feature = "legacy-zip")] + CompressionMethod::Shrink => Decompressor::Shrink( + crate::legacy::shrink::ShrinkDecoder::new(reader, uncompressed_size), + ), + #[cfg(feature = "legacy-zip")] + CompressionMethod::Reduce(n) => Decompressor::Reduce( + crate::legacy::reduce::ReduceDecoder::new(reader, uncompressed_size, n), + ), + #[cfg(feature = "legacy-zip")] + CompressionMethod::Implode => Decompressor::Implode( + crate::legacy::implode::ImplodeDecoder::new(reader, uncompressed_size, flags), + ), + _ => { + return Err(crate::result::ZipError::UnsupportedArchive( + "Compression method not supported", + )) + } + }) + } + + /// Consumes this decoder, returning the underlying reader. + #[allow(clippy::infallible_destructuring_match)] + pub fn into_inner(self) -> io::Result { + let inner = match self { + Decompressor::Stored(r) => r, + #[cfg(feature = "deflate-flate2")] + Decompressor::Deflated(r) => r.into_inner(), + #[cfg(feature = "deflate64")] + Decompressor::Deflate64(r) => r.into_inner(), + #[cfg(feature = "_bzip2_any")] + Decompressor::Bzip2(r) => r.into_inner(), + #[cfg(feature = "zstd")] + Decompressor::Zstd(r) => r.finish(), + #[cfg(feature = "lzma")] + Decompressor::Lzma(r) => match r { + Lzma::Uninitialized { mut reader, .. } => reader + .take() + .ok_or_else(|| io::Error::other("Reader was not set"))?, + Lzma::Initialized(decoder) => decoder.into_inner(), + }, + #[cfg(feature = "legacy-zip")] + Decompressor::Shrink(r) => r.into_inner(), + #[cfg(feature = "legacy-zip")] + Decompressor::Reduce(r) => r.into_inner(), + #[cfg(feature = "legacy-zip")] + Decompressor::Implode(r) => r.into_inner(), + #[cfg(feature = "xz")] + Decompressor::Xz(r) => r.into_inner(), + #[cfg(feature = "ppmd")] + Decompressor::Ppmd(r) => match r { + Ppmd::Uninitialized(mut reader) => reader + .take() + .ok_or_else(|| io::Error::other("Reader was not set"))?, + Ppmd::Initialized(decoder) => decoder.into_inner(), + }, + }; + Ok(inner) + } +} + +#[cfg(test)] +mod test { + use super::{CompressionMethod, SUPPORTED_COMPRESSION_METHODS}; + + #[test] + fn from_eq_to() { + for v in 0..(u16::MAX as u32 + 1) { + let from = CompressionMethod::parse_from_u16(v as u16); + let to = from.serialize_to_u16() as u32; + assert_eq!(v, to); + } + } + + #[test] + fn to_eq_from() { + fn check_match(method: CompressionMethod) { + let to = method.serialize_to_u16(); + let from = CompressionMethod::parse_from_u16(to); + let back = from.serialize_to_u16(); + assert_eq!(to, back); + } + + for &method in SUPPORTED_COMPRESSION_METHODS { + check_match(method); + } + } + + #[test] + fn to_display_fmt() { + fn check_match(method: CompressionMethod) { + let debug_str = format!("{method:?}"); + let display_str = format!("{method}"); + assert_eq!(debug_str, display_str); + } + + for &method in SUPPORTED_COMPRESSION_METHODS { + check_match(method); + } + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/cp437.rs b/v2/vendor/security/zip-7.2.0/src/cp437.rs new file mode 100644 index 0000000000..2c38063f32 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/cp437.rs @@ -0,0 +1,207 @@ +//! Convert a string in IBM codepage 437 to UTF-8 + +/// Trait to convert IBM codepage 437 to the target type +pub trait FromCp437 { + /// Target type + type Target; + + /// Function that does the conversion from cp437. + /// Generally allocations will be avoided if all data falls into the ASCII range. + #[allow(clippy::wrong_self_convention)] + fn from_cp437(self) -> Self::Target; +} + +impl<'a> FromCp437 for &'a [u8] { + type Target = ::std::borrow::Cow<'a, str>; + + fn from_cp437(self) -> Self::Target { + if self.iter().all(|c| *c < 0x80) { + ::std::str::from_utf8(self).unwrap().into() + } else { + self.iter().map(|c| to_char(*c)).collect::().into() + } + } +} + +impl FromCp437 for Box<[u8]> { + type Target = Box; + + fn from_cp437(self) -> Self::Target { + if self.iter().all(|c| *c < 0x80) { + String::from_utf8(self.into()).unwrap() + } else { + self.iter().copied().map(to_char).collect() + } + .into_boxed_str() + } +} + +fn to_char(input: u8) -> char { + let output = match input { + 0x00..=0x7f => input as u32, + 0x80 => 0x00c7, + 0x81 => 0x00fc, + 0x82 => 0x00e9, + 0x83 => 0x00e2, + 0x84 => 0x00e4, + 0x85 => 0x00e0, + 0x86 => 0x00e5, + 0x87 => 0x00e7, + 0x88 => 0x00ea, + 0x89 => 0x00eb, + 0x8a => 0x00e8, + 0x8b => 0x00ef, + 0x8c => 0x00ee, + 0x8d => 0x00ec, + 0x8e => 0x00c4, + 0x8f => 0x00c5, + 0x90 => 0x00c9, + 0x91 => 0x00e6, + 0x92 => 0x00c6, + 0x93 => 0x00f4, + 0x94 => 0x00f6, + 0x95 => 0x00f2, + 0x96 => 0x00fb, + 0x97 => 0x00f9, + 0x98 => 0x00ff, + 0x99 => 0x00d6, + 0x9a => 0x00dc, + 0x9b => 0x00a2, + 0x9c => 0x00a3, + 0x9d => 0x00a5, + 0x9e => 0x20a7, + 0x9f => 0x0192, + 0xa0 => 0x00e1, + 0xa1 => 0x00ed, + 0xa2 => 0x00f3, + 0xa3 => 0x00fa, + 0xa4 => 0x00f1, + 0xa5 => 0x00d1, + 0xa6 => 0x00aa, + 0xa7 => 0x00ba, + 0xa8 => 0x00bf, + 0xa9 => 0x2310, + 0xaa => 0x00ac, + 0xab => 0x00bd, + 0xac => 0x00bc, + 0xad => 0x00a1, + 0xae => 0x00ab, + 0xaf => 0x00bb, + 0xb0 => 0x2591, + 0xb1 => 0x2592, + 0xb2 => 0x2593, + 0xb3 => 0x2502, + 0xb4 => 0x2524, + 0xb5 => 0x2561, + 0xb6 => 0x2562, + 0xb7 => 0x2556, + 0xb8 => 0x2555, + 0xb9 => 0x2563, + 0xba => 0x2551, + 0xbb => 0x2557, + 0xbc => 0x255d, + 0xbd => 0x255c, + 0xbe => 0x255b, + 0xbf => 0x2510, + 0xc0 => 0x2514, + 0xc1 => 0x2534, + 0xc2 => 0x252c, + 0xc3 => 0x251c, + 0xc4 => 0x2500, + 0xc5 => 0x253c, + 0xc6 => 0x255e, + 0xc7 => 0x255f, + 0xc8 => 0x255a, + 0xc9 => 0x2554, + 0xca => 0x2569, + 0xcb => 0x2566, + 0xcc => 0x2560, + 0xcd => 0x2550, + 0xce => 0x256c, + 0xcf => 0x2567, + 0xd0 => 0x2568, + 0xd1 => 0x2564, + 0xd2 => 0x2565, + 0xd3 => 0x2559, + 0xd4 => 0x2558, + 0xd5 => 0x2552, + 0xd6 => 0x2553, + 0xd7 => 0x256b, + 0xd8 => 0x256a, + 0xd9 => 0x2518, + 0xda => 0x250c, + 0xdb => 0x2588, + 0xdc => 0x2584, + 0xdd => 0x258c, + 0xde => 0x2590, + 0xdf => 0x2580, + 0xe0 => 0x03b1, + 0xe1 => 0x00df, + 0xe2 => 0x0393, + 0xe3 => 0x03c0, + 0xe4 => 0x03a3, + 0xe5 => 0x03c3, + 0xe6 => 0x00b5, + 0xe7 => 0x03c4, + 0xe8 => 0x03a6, + 0xe9 => 0x0398, + 0xea => 0x03a9, + 0xeb => 0x03b4, + 0xec => 0x221e, + 0xed => 0x03c6, + 0xee => 0x03b5, + 0xef => 0x2229, + 0xf0 => 0x2261, + 0xf1 => 0x00b1, + 0xf2 => 0x2265, + 0xf3 => 0x2264, + 0xf4 => 0x2320, + 0xf5 => 0x2321, + 0xf6 => 0x00f7, + 0xf7 => 0x2248, + 0xf8 => 0x00b0, + 0xf9 => 0x2219, + 0xfa => 0x00b7, + 0xfb => 0x221a, + 0xfc => 0x207f, + 0xfd => 0x00b2, + 0xfe => 0x25a0, + 0xff => 0x00a0, + }; + ::std::char::from_u32(output).unwrap() +} + +#[cfg(test)] +mod test { + #[test] + fn to_char_valid() { + for i in 0x00_u32..0x100 { + super::to_char(i as u8); + } + } + + #[test] + fn ascii() { + for i in 0x00..0x80 { + assert_eq!(super::to_char(i), i as char); + } + } + + #[test] + #[allow(unknown_lints)] // invalid_from_utf8 was added in rust 1.72 + #[allow(invalid_from_utf8)] + fn example_slice() { + use super::FromCp437; + let data = b"Cura\x87ao"; + assert!(::std::str::from_utf8(data).is_err()); + assert_eq!(data.from_cp437(), "Curaçao"); + } + + #[test] + fn example_vec() { + use super::FromCp437; + let data = vec![0xCC, 0xCD, 0xCD, 0xB9]; + assert!(String::from_utf8(data.clone()).is_err()); + assert_eq!(&*data.from_cp437(), "╠══╣"); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/crc32.rs b/v2/vendor/security/zip-7.2.0/src/crc32.rs new file mode 100644 index 0000000000..4d1bd8e264 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/crc32.rs @@ -0,0 +1,131 @@ +//! Helper module to compute a CRC32 checksum + +use std::io::{self, Read}; + +use crc32fast::Hasher; + +/// Reader that validates the CRC32 when it reaches the EOF. +pub struct Crc32Reader { + inner: R, + hasher: Hasher, + check: u32, + /// Signals if `inner` stores aes encrypted data. + /// AE-2 encrypted data doesn't use crc and sets the value to 0. + enabled: bool, +} + +impl Crc32Reader { + /// Get a new Crc32Reader which checks the inner reader against checksum. + /// The check is disabled if `ae2_encrypted == true`. + pub(crate) fn new(inner: R, checksum: u32, ae2_encrypted: bool) -> Crc32Reader { + Crc32Reader { + inner, + hasher: Hasher::new(), + check: checksum, + enabled: !ae2_encrypted, + } + } + + fn check_matches(&self) -> bool { + self.check == self.hasher.clone().finalize() + } + + pub fn into_inner(self) -> R { + self.inner + } +} + +#[cold] +fn invalid_checksum() -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, "Invalid checksum") +} + +impl Read for Crc32Reader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let count = self.inner.read(buf)?; + + if self.enabled { + if count == 0 && !buf.is_empty() && !self.check_matches() { + return Err(invalid_checksum()); + } + self.hasher.update(&buf[..count]); + } + Ok(count) + } + + fn read_to_end(&mut self, buf: &mut Vec) -> io::Result { + let start = buf.len(); + let n = self.inner.read_to_end(buf)?; + + if self.enabled { + self.hasher.update(&buf[start..]); + if !self.check_matches() { + return Err(invalid_checksum()); + } + } + + Ok(n) + } + + fn read_to_string(&mut self, buf: &mut String) -> io::Result { + let start = buf.len(); + let n = self.inner.read_to_string(buf)?; + + if self.enabled { + self.hasher.update(&buf.as_bytes()[start..]); + if !self.check_matches() { + return Err(invalid_checksum()); + } + } + + Ok(n) + } +} + +#[cfg(test)] +mod test { + use std::io::Read; + + use super::Crc32Reader; + + #[test] + fn test_empty_reader() { + let data: &[u8] = b""; + let mut buf = [0; 1]; + + let mut reader = Crc32Reader::new(data, 0, false); + assert_eq!(reader.read(&mut buf).unwrap(), 0); + + let mut reader = Crc32Reader::new(data, 1, false); + assert!(reader + .read(&mut buf) + .unwrap_err() + .to_string() + .contains("Invalid checksum")); + } + + #[test] + fn test_byte_by_byte() { + let data: &[u8] = b"1234"; + let mut buf = [0; 1]; + + let mut reader = Crc32Reader::new(data, 0x9be3e0a3, false); + assert_eq!(reader.read(&mut buf).unwrap(), 1); + assert_eq!(reader.read(&mut buf).unwrap(), 1); + assert_eq!(reader.read(&mut buf).unwrap(), 1); + assert_eq!(reader.read(&mut buf).unwrap(), 1); + assert_eq!(reader.read(&mut buf).unwrap(), 0); + // Can keep reading 0 bytes after the end + assert_eq!(reader.read(&mut buf).unwrap(), 0); + } + + #[test] + fn test_zero_read() { + let data: &[u8] = b"1234"; + let mut buf = [0; 5]; + + let mut reader = Crc32Reader::new(data, 0x9be3e0a3, false); + assert_eq!(reader.read(&mut buf[..0]).unwrap(), 0); + assert_eq!(reader.read(&mut buf).unwrap(), 4); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/extra_fields/extended_timestamp.rs b/v2/vendor/security/zip-7.2.0/src/extra_fields/extended_timestamp.rs new file mode 100644 index 0000000000..22ebacf23c --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/extra_fields/extended_timestamp.rs @@ -0,0 +1,111 @@ +use crate::result::invalid; +use crate::result::{ZipError, ZipResult}; +use crate::unstable::LittleEndianReadExt; +use std::io::Read; + +/// extended timestamp, as described in + +#[derive(Debug, Clone)] +pub struct ExtendedTimestamp { + mod_time: Option, + ac_time: Option, + cr_time: Option, +} + +impl ExtendedTimestamp { + /// creates an extended timestamp struct by reading the required bytes from the reader. + /// + /// This method assumes that the length has already been read, therefore + /// it must be passed as an argument + pub fn try_from_reader(reader: &mut R, len: u16) -> ZipResult + where + R: Read, + { + if len == 0 { + return Err(invalid!("Extended timestamp field is empty")); + } + let mut flags = [0u8]; + let mut bytes_to_read = len as usize; + reader.read_exact(&mut flags)?; + bytes_to_read -= flags.len(); + let flags = flags[0]; + + // the `flags` field refers to the local headers and might not correspond + // to the len field. If the length field is 1+4, we assume that only + // the modification time has been set + + // > Those times that are present will appear in the order indicated, but + // > any combination of times may be omitted. (Creation time may be + // > present without access time, for example.) TSize should equal + // > (1 + 4*(number of set bits in Flags)), as the block is currently + // > defined. + if len != 5 && len as u32 != 1 + 4 * flags.count_ones() { + //panic!("found len {len} and flags {flags:08b}"); + return Err(ZipError::UnsupportedArchive( + "flags and len don't match in extended timestamp field", + )); + } + + // allow unsupported/undocumented flags + + let mod_time = if (flags & 0b00000001u8 == 0b00000001u8) || len == 5 { + bytes_to_read -= size_of::(); + Some(reader.read_u32_le()?) + } else { + None + }; + + let ac_time = if flags & 0b00000010u8 == 0b00000010u8 && len > 5 { + bytes_to_read -= size_of::(); + Some(reader.read_u32_le()?) + } else { + None + }; + + let cr_time = if flags & 0b00000100u8 == 0b00000100u8 && len > 5 { + bytes_to_read -= size_of::(); + Some(reader.read_u32_le()?) + } else { + None + }; + + if bytes_to_read > 0 { + // ignore undocumented bytes + reader.read_exact(&mut vec![0; bytes_to_read])? + } + + Ok(Self { + mod_time, + ac_time, + cr_time, + }) + } + + /// returns the last modification timestamp, if defined, as UNIX epoch seconds + pub fn mod_time(&self) -> Option { + self.mod_time + } + + /// returns the last access timestamp, if defined, as UNIX epoch seconds + pub fn ac_time(&self) -> Option { + self.ac_time + } + + /// returns the creation timestamp, if defined, as UNIX epoch seconds + pub fn cr_time(&self) -> Option { + self.cr_time + } +} + +#[test] +/// Ensure we don't panic or read garbage data if the field body is empty +pub fn test_bad_extended_timestamp() -> ZipResult<()> { + use crate::ZipArchive; + use std::io::Cursor; + + assert!(ZipArchive::new(Cursor::new(include_bytes!( + "../../tests/data/extended_timestamp_bad.zip" + ))) + .is_err()); + Ok(()) +} diff --git a/v2/vendor/security/zip-7.2.0/src/extra_fields/mod.rs b/v2/vendor/security/zip-7.2.0/src/extra_fields/mod.rs new file mode 100644 index 0000000000..de6a3b6c72 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/extra_fields/mod.rs @@ -0,0 +1,35 @@ +//! types for extra fields + +/// marker trait to denote the place where this extra field has been stored +pub trait ExtraFieldVersion {} + +/// use this to mark extra fields specified in a local header + +#[derive(Debug, Clone)] +pub struct LocalHeaderVersion; + +/// use this to mark extra fields specified in the central header + +#[derive(Debug, Clone)] +pub struct CentralHeaderVersion; + +impl ExtraFieldVersion for LocalHeaderVersion {} +impl ExtraFieldVersion for CentralHeaderVersion {} + +mod extended_timestamp; +mod ntfs; +mod zipinfo_utf8; + +pub use extended_timestamp::*; +pub use ntfs::Ntfs; +pub use zipinfo_utf8::UnicodeExtraField; + +/// contains one extra field +#[derive(Debug, Clone)] +pub enum ExtraField { + /// NTFS extra field + Ntfs(Ntfs), + + /// extended timestamp, as described in + ExtendedTimestamp(ExtendedTimestamp), +} diff --git a/v2/vendor/security/zip-7.2.0/src/extra_fields/ntfs.rs b/v2/vendor/security/zip-7.2.0/src/extra_fields/ntfs.rs new file mode 100644 index 0000000000..d303735bac --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/extra_fields/ntfs.rs @@ -0,0 +1,97 @@ +use std::io::Read; + +use crate::{ + result::{ZipError, ZipResult}, + unstable::LittleEndianReadExt, +}; + +/// The NTFS extra field as described in [PKWARE's APPNOTE.TXT v6.3.9]. +/// +/// This field stores [Windows file times], which are 64-bit unsigned integer +/// values that represents the number of 100-nanosecond intervals that have +/// elapsed since "1601-01-01 00:00:00 UTC". +/// +/// [PKWARE's APPNOTE.TXT v6.3.9]: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT +/// [Windows file times]: https://docs.microsoft.com/en-us/windows/win32/sysinfo/file-times +#[derive(Clone, Debug)] +pub struct Ntfs { + mtime: u64, + atime: u64, + ctime: u64, +} + +impl Ntfs { + /// Creates a NTFS extra field struct by reading the required bytes from the + /// reader. + /// + /// This method assumes that the length has already been read, therefore it + /// must be passed as an argument. + pub fn try_from_reader(reader: &mut R, len: u16) -> ZipResult + where + R: Read, + { + if len != 32 { + return Err(ZipError::UnsupportedArchive( + "NTFS extra field has an unsupported length", + )); + } + + // Read reserved for future use. + let _ = reader.read_u32_le()?; + + let tag = reader.read_u16_le()?; + if tag != 0x0001 { + return Err(ZipError::UnsupportedArchive( + "NTFS extra field has an unsupported attribute tag", + )); + } + let size = reader.read_u16_le()?; + if size != 24 { + return Err(ZipError::UnsupportedArchive( + "NTFS extra field has an unsupported attribute size", + )); + } + + let mtime = reader.read_u64_le()?; + let atime = reader.read_u64_le()?; + let ctime = reader.read_u64_le()?; + Ok(Self { + mtime, + atime, + ctime, + }) + } + + /// Returns the file last modification time as a file time. + pub fn mtime(&self) -> u64 { + self.mtime + } + + /// Returns the file last modification time as a file time. + #[cfg(feature = "nt-time")] + pub fn modified_file_time(&self) -> nt_time::FileTime { + nt_time::FileTime::new(self.mtime) + } + + /// Returns the file last access time as a file time. + pub fn atime(&self) -> u64 { + self.atime + } + + /// Returns the file last access time as a file time. + #[cfg(feature = "nt-time")] + pub fn accessed_file_time(&self) -> nt_time::FileTime { + nt_time::FileTime::new(self.atime) + } + + /// Returns the file creation time as a file time. + pub fn ctime(&self) -> u64 { + self.ctime + } + + /// Returns the file creation time as a file time. + #[cfg(feature = "nt-time")] + pub fn created_file_time(&self) -> nt_time::FileTime { + nt_time::FileTime::new(self.ctime) + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/extra_fields/zipinfo_utf8.rs b/v2/vendor/security/zip-7.2.0/src/extra_fields/zipinfo_utf8.rs new file mode 100644 index 0000000000..c2c09e50cd --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/extra_fields/zipinfo_utf8.rs @@ -0,0 +1,40 @@ +use crate::result::{invalid, ZipResult}; +use crate::unstable::LittleEndianReadExt; +use core::mem::size_of; +use std::io::Read; + +/// Info-ZIP Unicode Path Extra Field (0x7075) or Unicode Comment Extra Field (0x6375), as +/// specified in APPNOTE 4.6.8 and 4.6.9 +#[derive(Clone, Debug)] +pub struct UnicodeExtraField { + crc32: u32, + content: Box<[u8]>, +} + +impl UnicodeExtraField { + /// Verifies the checksum and returns the content. + pub fn unwrap_valid(self, ascii_field: &[u8]) -> ZipResult> { + let mut crc32 = crc32fast::Hasher::new(); + crc32.update(ascii_field); + let actual_crc32 = crc32.finalize(); + if self.crc32 != actual_crc32 { + return Err(invalid!("CRC32 checksum failed on Unicode extra field")); + } + Ok(self.content) + } +} + +impl UnicodeExtraField { + pub(crate) fn try_from_reader(reader: &mut R, len: u16) -> ZipResult { + // Read and discard version byte + reader.read_exact(&mut [0u8])?; + + let crc32 = reader.read_u32_le()?; + let content_len = (len as usize) + .checked_sub(size_of::() + size_of::()) + .ok_or(invalid!("Unicode extra field is too small"))?; + let mut content = vec![0u8; content_len].into_boxed_slice(); + reader.read_exact(&mut content)?; + Ok(Self { crc32, content }) + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/legacy/huffman.rs b/v2/vendor/security/zip-7.2.0/src/legacy/huffman.rs new file mode 100644 index 0000000000..e7b2d3c77a --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/legacy/huffman.rs @@ -0,0 +1,260 @@ +use std::io::{self, Error, Seek}; + +use bitstream_io::{BitRead, BitReader, Endianness}; + +#[derive(Default, Clone, Copy)] +pub struct TableEntry { + /// Wide enough to fit the max symbol nbr. + pub sym: u16, + /// 0 means no symbol. + pub len: u8, +} + +/// Deflate uses max 288 symbols. +const MAX_HUFFMAN_SYMBOLS: usize = 288; +/// Implode uses max 16-bit codewords. +const MAX_HUFFMAN_BITS: usize = 16; +/// Seems a good trade-off. +const HUFFMAN_LOOKUP_TABLE_BITS: u8 = 8; + +pub struct HuffmanDecoder { + /// Lookup table for fast decoding of short codewords. + pub table: [TableEntry; 1 << HUFFMAN_LOOKUP_TABLE_BITS], + /// "Sentinel bits" value for each codeword length. + pub sentinel_bits: [u32; MAX_HUFFMAN_BITS + 1], + /// First symbol index minus first codeword mod 2**16 for each length. + pub offset_first_sym_idx: [u16; MAX_HUFFMAN_BITS + 1], + /// Map from symbol index to symbol. + pub syms: [u16; MAX_HUFFMAN_SYMBOLS], + // num_syms:usize +} + +impl Default for HuffmanDecoder { + fn default() -> Self { + Self { + table: [TableEntry::default(); 1 << HUFFMAN_LOOKUP_TABLE_BITS], + sentinel_bits: [0; MAX_HUFFMAN_BITS + 1], + offset_first_sym_idx: [0; MAX_HUFFMAN_BITS + 1], + syms: [0; MAX_HUFFMAN_SYMBOLS], + } + } +} + +/// Reverse the n least significant bits of x. +/// The (16 - n) most significant bits of the result will be zero. +pub fn reverse_lsb(x: u16, n: usize) -> u16 { + debug_assert!(n > 0); + debug_assert!(n <= 16); + x.reverse_bits() >> (16 - n) +} + +/// Initialize huffman decoder d for a code defined by the n codeword lengths. +/// Returns `Err` if the codeword lengths do not correspond to a valid prefix +/// code. +impl HuffmanDecoder { + pub fn init(&mut self, lengths: &[u8], n: usize) -> std::io::Result<()> { + let mut count = [0; MAX_HUFFMAN_BITS + 1]; + let mut code = [0; MAX_HUFFMAN_BITS + 1]; + let mut sym_idx = [0u16; MAX_HUFFMAN_BITS + 1]; + + // Count the number of codewords of each length. + for i in 0..n { + debug_assert!(lengths[i] as usize <= MAX_HUFFMAN_BITS); + count[lengths[i] as usize] += 1; + } + count[0] = 0; // Ignore zero-length codewords. + // Compute sentinel_bits and offset_first_sym_idx for each length. + code[0] = 0; + sym_idx[0] = 0; + for l in 1..=MAX_HUFFMAN_BITS { + // First canonical codeword of this length. + code[l] = (code[l - 1] + count[l - 1]) << 1; + + if count[l] != 0 && code[l] as u32 + count[l] as u32 - 1 > (1u32 << l) - 1 { + // The last codeword is longer than l bits. + return Err(Error::new( + io::ErrorKind::InvalidData, + "the last codeword is longer than len bits", + )); + } + + let s = (code[l] as u32 + count[l] as u32) << (MAX_HUFFMAN_BITS - l); + self.sentinel_bits[l] = s; + debug_assert!(self.sentinel_bits[l] >= code[l] as u32, "No overflow!"); + + sym_idx[l] = sym_idx[l - 1] + count[l - 1]; + self.offset_first_sym_idx[l] = sym_idx[l].wrapping_sub(code[l]); + } + + // Zero-initialize the lookup table. + self.table.fill(TableEntry::default()); + + // Build mapping from index to symbol and populate the lookup table. + lengths + .iter() + .enumerate() + .take(n) + .for_each(|(i, code_len)| { + let l = *code_len as usize; + if l == 0 { + return; + } + + self.syms[sym_idx[l] as usize] = i as u16; + sym_idx[l] += 1; + + if l <= HUFFMAN_LOOKUP_TABLE_BITS as usize { + self.table_insert(i, l, code[l]); + code[l] += 1; + } + }); + + Ok(()) + } + + pub fn table_insert(&mut self, sym: usize, len: usize, codeword: u16) { + debug_assert!(len <= HUFFMAN_LOOKUP_TABLE_BITS as usize); + + let codeword = reverse_lsb(codeword, len); // Make it LSB-first. + let pad_len = HUFFMAN_LOOKUP_TABLE_BITS as usize - len; + + // Pad the pad_len upper bits with all bit combinations. + for padding in 0..(1 << pad_len) { + let index = (codeword | (padding << len)) as usize; + debug_assert!(sym <= u16::MAX as usize); + self.table[index].sym = sym as u16; + debug_assert!(len <= u8::MAX as usize); + self.table[index].len = len as u8; + } + } + + /// Use the decoder d to decode a symbol from the LSB-first zero-padded bits. + /// Returns the decoded symbol number or an error if no symbol could be decoded. + /// *num_used_bits will be set to the number of bits used to decode the symbol, + /// or zero if no symbol could be decoded. + pub fn huffman_decode( + &mut self, + length: u64, + is: &mut BitReader, + ) -> std::io::Result { + // First try the lookup table. + let read_bits1 = (HUFFMAN_LOOKUP_TABLE_BITS as u64).min(length - is.position_in_bits()?); + let lookup_bits = !is.read_var::(read_bits1 as u32)? as usize; + debug_assert!(lookup_bits < self.table.len()); + if self.table[lookup_bits].len != 0 { + debug_assert!(self.table[lookup_bits].len <= HUFFMAN_LOOKUP_TABLE_BITS); + is.seek_bits(io::SeekFrom::Current( + -(read_bits1 as i64) + self.table[lookup_bits].len as i64, + ))?; + return Ok(self.table[lookup_bits].sym); + } + + // Then do canonical decoding with the bits in MSB-first order. + let read_bits2 = (HUFFMAN_LOOKUP_TABLE_BITS as u64).min(length - is.position_in_bits()?); + let mut bits = reverse_lsb( + (lookup_bits | ((!is.read_var::(read_bits2 as u32)? as usize) << read_bits1)) + as u16, + MAX_HUFFMAN_BITS, + ); + + for l in (HUFFMAN_LOOKUP_TABLE_BITS as usize + 1)..=MAX_HUFFMAN_BITS { + if (bits as u32) < self.sentinel_bits[l] { + bits >>= MAX_HUFFMAN_BITS - l; + let sym_idx = (self.offset_first_sym_idx[l] as usize + bits as usize) & 0xFFFF; + //assert(sym_idx < self.num_syms); + is.seek_bits(io::SeekFrom::Current( + -(read_bits1 as i64 + read_bits2 as i64) + l as i64, + ))?; + return Ok(self.syms[sym_idx]); + } + } + Err(Error::new( + io::ErrorKind::InvalidData, + "huffman decode failed", + )) + } +} + +#[cfg(test)] +mod tests { + use std::io::Cursor; + + use bitstream_io::{BitReader, LittleEndian}; + + use super::HuffmanDecoder; + #[test] + fn test_huffman_decode_basic() { + let lens = [ + 3, // sym 0: 000 + 3, // sym 1: 001 + 3, // sym 2: 010 + 3, // sym 3: 011 + 3, // sym 4: 100 + 3, // sym 5: 101 + 4, // sym 6: 1100 + 4, // sym 7: 1101 + 0, // sym 8: + 0, // sym 9: + 0, // sym 10: + 0, // sym 11: + 0, // sym 12: + 0, // sym 13: + 0, // sym 14: + 0, // sym 15: + 6, // sym 16: 111110 + 5, // sym 17: 11110 + 4, // sym 18: 1110 + ]; + + let mut d = HuffmanDecoder::default(); + d.init(&lens, lens.len()).unwrap(); + + // 000 (msb-first) -> 000 (lsb-first) + assert_eq!( + d.huffman_decode( + 8, + &mut BitReader::endian(&mut Cursor::new(&[!0x0]), LittleEndian) + ) + .unwrap(), + 0 + ); + + /* 011 (msb-first) -> 110 (lsb-first)*/ + assert_eq!( + d.huffman_decode( + 8, + &mut BitReader::endian(&mut Cursor::new(&[!0b110]), LittleEndian) + ) + .unwrap(), + 0b011 + ); + + /* 11110 (msb-first) -> 01111 (lsb-first)*/ + assert_eq!( + d.huffman_decode( + 8, + &mut BitReader::endian(&mut Cursor::new(&[!0b1111]), LittleEndian) + ) + .unwrap(), + 0b10001 + ); + + /* 111110 (msb-first) -> 011111 (lsb-first)*/ + assert_eq!( + d.huffman_decode( + 8, + &mut BitReader::endian(&mut Cursor::new(&[!0b11111]), LittleEndian) + ) + .unwrap(), + 0b10000 + ); + + /* 1111111 (msb-first) -> 1111111 (lsb-first)*/ + assert!(d + .huffman_decode( + 8, + &mut BitReader::endian(&mut Cursor::new(&[!0x7f]), LittleEndian) + ) + .is_err()); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/legacy/implode.rs b/v2/vendor/security/zip-7.2.0/src/legacy/implode.rs new file mode 100644 index 0000000000..bd35ed1a6f --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/legacy/implode.rs @@ -0,0 +1,250 @@ +use super::huffman::HuffmanDecoder; +use bitstream_io::{BitRead, BitReader, Endianness, LittleEndian}; +use std::io::{self, Cursor, Error, Read, Result}; + +/// Maximum symbol value in the length Huffman table (6 bits) +/// When this value is decoded, an additional byte is read for extended length +const MAX_LEN_SYMBOL: u16 = 63; + +/// Maximum number of symbols in a Huffman table (256 for literals, 64 for lengths/distances) +const MAX_HUFFMAN_SYMBOLS: usize = 1 << 8; // 256 + +/// Maximum code length in bits for Huffman codes (per ZIP specification) +const MAX_CODE_LENGTH: usize = 16; + +/// Size of the code length count array (lengths are actually 1..=16) +const CODE_LENGTH_COUNT_SIZE: usize = MAX_CODE_LENGTH; + +/// Initialize the Huffman decoder d with num_lens codeword lengths read from is. +/// Returns false if the input is invalid. +fn read_huffman_code( + is: &mut BitReader, + num_lens: usize, +) -> std::io::Result> { + let mut lens = [0; MAX_HUFFMAN_SYMBOLS]; + let mut len_count = [0; CODE_LENGTH_COUNT_SIZE]; + + // Number of bytes representing the Huffman code. + let byte = is.read::<8, u8>()?; + let num_bytes = (byte + 1) as usize; + + let mut codeword_idx = 0; + for _byte_idx in 0..num_bytes { + let byte = is.read::<8, u16>()?; + + let codeword_len = (byte & 0xf) + 1; /* Low four bits plus one. */ + let run_length = (byte >> 4) + 1; /* High four bits plus one. */ + + debug_assert!((1..=16).contains(&codeword_len)); + len_count[codeword_len as usize - 1] += run_length; + + if (codeword_idx + run_length) as usize > num_lens { + return Err(Error::new( + io::ErrorKind::InvalidData, + "too many codeword lengths", + )); + } + for _ in 0..run_length { + debug_assert!((codeword_idx as usize) < num_lens); + lens[codeword_idx as usize] = codeword_len as u8; + codeword_idx += 1; + } + } + + debug_assert!(codeword_idx as usize <= num_lens); + if (codeword_idx as usize) < num_lens { + return Err(Error::new( + io::ErrorKind::InvalidData, + "not enough codeword lengths", + )); + } + + // Check that the Huffman tree is full. + let mut avail_codewords = 1; + for i in 1..=16 { + debug_assert!(avail_codewords >= 0); + avail_codewords *= 2; + avail_codewords -= len_count[i - 1] as i32; + if avail_codewords < 0 { + return Err(Error::new( + io::ErrorKind::InvalidData, + "huffman tree is not full", + )); + } + } + if avail_codewords != 0 { + // Not all codewords were used. + return Err(Error::new( + io::ErrorKind::InvalidData, + "not all codewords were used", + )); + } + + let mut d = Box::new(HuffmanDecoder::default()); + d.init(&lens, num_lens)?; + Ok(d) +} + +fn hwexplode( + src: &[u8], + uncomp_len: usize, + large_wnd: bool, + lit_tree: bool, + pk101_bug_compat: bool, + dst: &mut Vec, +) -> std::io::Result<()> { + // Pre-allocate capacity + dst.reserve(uncomp_len); + + let bit_length = src.len() as u64 * 8; + let mut is = BitReader::endian(Cursor::new(&src), LittleEndian); + let mut lit_decoder_opt = if lit_tree { + Some(read_huffman_code(&mut is, 256)?) + } else { + None + }; + let mut len_decoder = read_huffman_code(&mut is, 64)?; + let mut dist_decoder = read_huffman_code(&mut is, 64)?; + let min_len = if (pk101_bug_compat && large_wnd) || (!pk101_bug_compat && lit_tree) { + 3 + } else { + 2 + }; + let dist_low_bits = if large_wnd { 7 } else { 6 }; + + while dst.len() < uncomp_len { + let is_literal = is.read_bit()?; + if is_literal { + // Literal. + let sym; + if let Some(lit_decoder) = &mut lit_decoder_opt { + sym = lit_decoder.huffman_decode(bit_length, &mut is)?; + } else { + sym = is.read::<8, u8>()? as u16; + } + debug_assert!(sym <= u8::MAX as u16); + dst.push(sym as u8); + continue; + } + + // Read the low dist bits. + let mut dist = is.read_var::(dist_low_bits)?; + // Read the Huffman-encoded high dist bits. + let sym = dist_decoder.huffman_decode(bit_length, &mut is)?; + dist |= (sym as u16) << dist_low_bits; + dist += 1; + + // Read the Huffman-encoded len. + let sym = len_decoder.huffman_decode(bit_length, &mut is)?; + let mut len = (sym + min_len) as usize; + + if sym == MAX_LEN_SYMBOL { + // Read an extra len byte. + len += is.read::<8, u16>()? as usize; + } + let len = len.min(uncomp_len - dst.len()); + + // Optimize the copy loop using extend_from_within when possible + if dist as usize <= dst.len() { + if dist as usize >= len { + // No overlap in copy, can use extend_from_within + let start = dst.len() - dist as usize; + dst.extend_from_within(start..start + len); + } else { + // Overlapping copy + for _ in 0..len { + let byte = dst[dst.len() - dist as usize]; + dst.push(byte); + } + } + } else { + // Copy with implicit zeros + for _ in 0..len { + if dist as usize > dst.len() { + dst.push(0); + } else { + let byte = dst[dst.len() - dist as usize]; + dst.push(byte); + } + } + } + } + Ok(()) +} + +#[derive(Debug)] +pub struct ImplodeDecoder { + compressed_reader: R, + uncompressed_size: u64, + stream_read: bool, + large_wnd: bool, + lit_tree: bool, + stream: Vec, + read_pos: usize, // Add read position tracker +} + +impl ImplodeDecoder { + pub fn new(inner: R, uncompressed_size: u64, flags: u16) -> Self { + let large_wnd = (flags & 2) != 0; + let lit_tree = (flags & 4) != 0; + ImplodeDecoder { + compressed_reader: inner, + uncompressed_size, + stream_read: false, + large_wnd, + lit_tree, + stream: Vec::new(), + read_pos: 0, + } + } + + pub fn into_inner(self) -> R { + self.compressed_reader + } +} + +impl Read for ImplodeDecoder { + fn read(&mut self, buf: &mut [u8]) -> Result { + if !self.stream_read { + self.stream_read = true; + let mut compressed_bytes = Vec::new(); + self.compressed_reader.read_to_end(&mut compressed_bytes)?; + + // Pre-allocate stream buffer + self.stream.reserve(self.uncompressed_size as usize); + + hwexplode( + &compressed_bytes, + self.uncompressed_size as usize, + self.large_wnd, + self.lit_tree, + false, + &mut self.stream, + )?; + } + + let available = self.stream.len() - self.read_pos; + let bytes_to_read = available.min(buf.len()); + buf[..bytes_to_read] + .copy_from_slice(&self.stream[self.read_pos..self.read_pos + bytes_to_read]); + self.read_pos += bytes_to_read; + Ok(bytes_to_read) + } +} + +#[cfg(test)] +mod tests { + use super::hwexplode; + + const HAMLET_256: &[u8; 249] = include_bytes!("../../tests/data/legacy/implode_hamlet_256.bin"); + const HAMLET_256_OUT: &[u8; 256] = + include_bytes!("../../tests/data/legacy/implode_hamlet_256.out"); + + #[test] + fn test_explode_hamlet_256() { + let mut dst = Vec::new(); + hwexplode(HAMLET_256, 256, false, false, false, &mut dst).unwrap(); + assert_eq!(dst.len(), 256); + assert_eq!(&dst, &HAMLET_256_OUT); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/legacy/mod.rs b/v2/vendor/security/zip-7.2.0/src/legacy/mod.rs new file mode 100644 index 0000000000..80858e2110 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/legacy/mod.rs @@ -0,0 +1,4 @@ +mod huffman; +pub(crate) mod implode; +pub(crate) mod reduce; +pub(crate) mod shrink; diff --git a/v2/vendor/security/zip-7.2.0/src/legacy/reduce.rs b/v2/vendor/security/zip-7.2.0/src/legacy/reduce.rs new file mode 100644 index 0000000000..e90d06a0ef --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/legacy/reduce.rs @@ -0,0 +1,313 @@ +use std::io::{self, Read, Result}; + +use bitstream_io::{BitRead, BitReader, Endianness, LittleEndian}; + +/// Number of bits used to represent indices in a follower set of size n. +const fn follower_idx_bitlen(n: u8) -> u8 { + debug_assert!(n <= 32); + match n { + 0 => 0, + 1 => 1, + _ => 8 - (n - 1).leading_zeros() as u8, + } +} + +#[derive(Default, Clone, Copy)] +struct FollowerSet { + followers: [u8; 32], + size: u8, + idx_bitlen: u8, +} + +/// Read the follower sets from is into fsets. Returns true on success. +type FollowerSetArray = [FollowerSet; u8::MAX as usize + 1]; + +fn read_follower_sets( + is: &mut BitReader, +) -> io::Result { + let mut fsets = [FollowerSet::default(); u8::MAX as usize + 1]; + for i in (0..=u8::MAX as usize).rev() { + let n = is.read::<6, u8>()?; + if n > 32 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid follower set", + )); + } + fsets[i].size = n; + fsets[i].idx_bitlen = follower_idx_bitlen(n); + + for j in 0..fsets[i].size as usize { + fsets[i].followers[j] = is.read::<8, u8>()?; + } + } + + Ok(fsets) +} + +/// Read the next byte from is, decoded based on prev_byte and the follower sets. +/// The byte is returned in *out_byte. +/// +/// # Returns +/// +/// * `Ok` with the byte if it was successfully read. +/// * `Err(io::Error)` on bad data or end of input. +fn read_next_byte( + is: &mut BitReader, + prev_byte: u8, + fsets: &mut FollowerSetArray, +) -> io::Result { + if fsets[prev_byte as usize].size == 0 // No followers + || is.read::<1, u8>()? == 1 + // Indicates next symbol is a literal byte + { + return is.read::<8, u8>(); + } + + // The bits represent the index of a follower byte. + let idx_bitlen = fsets[prev_byte as usize].idx_bitlen; + let follower_idx = is.read_var::(idx_bitlen as u32)? as usize; + if follower_idx >= fsets[prev_byte as usize].size as usize { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid follower index", + )); + } + Ok(fsets[prev_byte as usize].followers[follower_idx]) +} + +fn max_len(comp_factor: u8) -> usize { + debug_assert!((1..=4).contains(&comp_factor)); + let v_len_bits = (8 - comp_factor) as usize; + // Bits in V + extra len byte + implicit 3. + ((1 << v_len_bits) - 1) + u8::MAX as usize + 3 +} + +fn max_dist(comp_factor: u8) -> usize { + debug_assert!((1..=4).contains(&comp_factor)); + let v_dist_bits = comp_factor as usize; + // Bits in V * 256 + W byte + implicit 1. */ + 1 << (v_dist_bits + 8) +} + +const DLE_BYTE: u8 = 0x90; + +/// Get the n least significant bits of x. +fn lsb(x: u8, n: u8) -> u8 { + if n >= 8 { + return x; + } + x & ((1 << n) - 1) +} + +fn hwexpand(src: &[u8], uncomp_len: usize, comp_factor: u8, dst: &mut Vec) -> io::Result<()> { + debug_assert!((1..=4).contains(&comp_factor)); + + // Pre-allocate to avoid reallocations + dst.reserve(uncomp_len); + + let mut is = BitReader::endian(src, LittleEndian); + let mut fsets = read_follower_sets(&mut is)?; + + // Number of bits in V used for backref length. + let v_len_bits = 8 - comp_factor; + + let mut curr_byte = 0; // The first "previous byte" is implicitly zero. + + while dst.len() < uncomp_len { + // Read a literal byte or DLE marker. + curr_byte = read_next_byte(&mut is, curr_byte, &mut fsets)?; + if curr_byte != DLE_BYTE { + // Output a literal byte. + dst.push(curr_byte); + continue; + } + + // Read the V byte which determines the length. + curr_byte = read_next_byte(&mut is, curr_byte, &mut fsets)?; + if curr_byte == 0 { + // Output a literal DLE byte. + dst.push(DLE_BYTE); + continue; + } + let v = curr_byte; + let mut len = lsb(v, v_len_bits) as usize; + if len == (1 << v_len_bits) - 1 { + // Read an extra length byte. + curr_byte = read_next_byte(&mut is, curr_byte, &mut fsets)?; + len += curr_byte as usize; + } + len += 3; + + // Read the W byte, which together with V gives the distance. + curr_byte = read_next_byte(&mut is, curr_byte, &mut fsets)?; + let dist = (((v as usize) >> v_len_bits) << 8) + curr_byte as usize + 1; + + debug_assert!(len <= max_len(comp_factor)); + debug_assert!(dist <= max_dist(comp_factor)); + + // Output the back reference. + if dist <= dst.len() { + // Optimize for non-overlapping copies + if dist >= len { + // No overlap, can use extend_from_within + let start = dst.len() - dist; + dst.extend_from_within(start..start + len.min(uncomp_len - dst.len())); + } else { + // Overlapping copy + let copy_len = len.min(uncomp_len - dst.len()); + for _ in 0..copy_len { + let byte = dst[dst.len() - dist]; + dst.push(byte); + } + } + } else { + // Copy with implicit zeros + let copy_len = len.min(uncomp_len - dst.len()); + for _ in 0..copy_len { + if dist > dst.len() { + dst.push(0); + } else { + let byte = dst[dst.len() - dist]; + dst.push(byte); + } + } + } + } + Ok(()) +} + +#[derive(Debug)] +pub struct ReduceDecoder { + compressed_reader: R, + uncompressed_size: u64, + stream_read: bool, + comp_factor: u8, + stream: Vec, + read_pos: usize, // Add read position tracker +} + +impl ReduceDecoder { + pub fn new(inner: R, uncompressed_size: u64, comp_factor: u8) -> Self { + ReduceDecoder { + compressed_reader: inner, + uncompressed_size, + stream_read: false, + comp_factor, + stream: Vec::new(), + read_pos: 0, + } + } + + pub fn into_inner(self) -> R { + self.compressed_reader + } +} + +impl Read for ReduceDecoder { + fn read(&mut self, buf: &mut [u8]) -> Result { + if !self.stream_read { + self.stream_read = true; + let mut compressed_bytes = Vec::new(); + self.compressed_reader.read_to_end(&mut compressed_bytes)?; + hwexpand( + &compressed_bytes, + self.uncompressed_size as usize, + self.comp_factor, + &mut self.stream, + )?; + } + + let available = self.stream.len() - self.read_pos; + let bytes_to_read = available.min(buf.len()); + buf[..bytes_to_read] + .copy_from_slice(&self.stream[self.read_pos..self.read_pos + bytes_to_read]); + self.read_pos += bytes_to_read; + Ok(bytes_to_read) + } +} + +#[cfg(test)] +mod tests { + use super::hwexpand; + use crate::legacy::reduce::{follower_idx_bitlen, lsb, max_dist}; + const HAMLET_2048: &[u8; 1285] = + include_bytes!("../../tests/data/legacy/reduce_hamlet_2048.bin"); + const HAMLET_2048_OUT: &[u8; 2048] = + include_bytes!("../../tests/data/legacy/implode_hamlet_2048.out"); + + #[test] + fn test_lsb() { + assert_eq!(lsb(0xFF, 8), 0xFF); + for i in 0..7 { + assert_eq!(lsb(0xFF, i), (1 << i) - 1); + } + } + + #[test] + fn test_expand_hamlet2048() { + let mut dst = Vec::new(); + hwexpand(HAMLET_2048, 2048, 4, &mut dst).unwrap(); + assert_eq!(dst.len(), 2048); + assert_eq!(&dst, &HAMLET_2048_OUT); + } + + /* + Put some text first to make PKZIP actually use Reduce compression. + Target the code path which copies a zero when dist > current position. + + $ curl -O http://cd.textfiles.com/originalsw/25/pkz092.exe + $ dosbox -c "mount c ." -c "c:" -c "pkz092" -c "exit" + $ dd if=hamlet.txt bs=1 count=2048 > a + $ dd if=/dev/zero bs=1 count=1024 >> a + $ dosbox -c "mount c ." -c "c:" -c "pkzip -ea4 a.zip a" -c "exit" + $ xxd -i -s 31 -l $(expr $(find A.ZIP -printf %s) - 100) A.ZIP + */ + const ZEROS_REDUCED: &[u8; 1297] = + include_bytes!("../../tests/data/legacy/reduce_zero_reduced.bin"); + + #[test] + fn test_expand_zeros() { + let mut dst = Vec::new(); + hwexpand(ZEROS_REDUCED, 2048 + 1024, 4, &mut dst).unwrap(); + assert_eq!(dst.len(), 2048 + 1024); + for i in 0..(1 << 10) { + assert_eq!(dst[(1 << 11) + i], 0); + } + } + + fn orig_follower_idx_bitlen(n: u8) -> u8 { + if n > 16 { + return 5; + } + if n > 8 { + return 4; + } + if n > 4 { + return 3; + } + if n > 2 { + return 2; + } + if n > 0 { + return 1; + } + 0 + } + + #[test] + fn test_follower_idx_biten() { + for i in 0..=32 { + assert_eq!(orig_follower_idx_bitlen(i), follower_idx_bitlen(i)); + } + } + + #[test] + fn test_max_dist() { + for i in 1..=4 { + let v_dist_bits = i as usize; + let c = 1 << (v_dist_bits + 8); + assert_eq!(max_dist(i), c); + } + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/legacy/shrink.rs b/v2/vendor/security/zip-7.2.0/src/legacy/shrink.rs new file mode 100644 index 0000000000..c8615a920a --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/legacy/shrink.rs @@ -0,0 +1,373 @@ +use std::io::{self, Error, Read}; + +use bitstream_io::{BitRead, BitReader, Endianness, LittleEndian}; + +const MIN_CODE_SIZE: u8 = 9; +const MAX_CODE_SIZE: u8 = 13; + +const MAX_CODE: usize = (1 << MAX_CODE_SIZE) - 1; +const CONTROL_CODE: usize = 256; +const INC_CODE_SIZE: u16 = 1; +const PARTIAL_CLEAR: u16 = 2; + +/// Number of codes available for the LZW dictionary (excluding control codes) +/// These are the codes from CONTROL_CODE+1 to MAX_CODE +const FREE_CODE_QUEUE_SIZE: usize = MAX_CODE - CONTROL_CODE + 1; + +// const HASH_BITS: usize = MAX_CODE_SIZE + 1; /* For a load factor of 0.5. */ +// const HASHTAB_SIZE: usize = 1 << HASH_BITS; +const UNKNOWN_LEN: u16 = u16::MAX; + +struct CodeQueue { + next_idx: usize, + codes: [Option; FREE_CODE_QUEUE_SIZE], +} + +impl CodeQueue { + fn new() -> Self { + let mut codes = [None; FREE_CODE_QUEUE_SIZE]; + for (i, code) in ((CONTROL_CODE as u16 + 1)..=(MAX_CODE as u16 - 1)).enumerate() { + codes[i] = Some(code); + } + Self { next_idx: 0, codes } + } + + // Return the next code in the queue, or INVALID_CODE if the queue is empty. + fn next(&self) -> Option { + // assert(q->next_idx < sizeof(q->codes) / sizeof(q->codes[0])); + if let Some(Some(next)) = self.codes.get(self.next_idx) { + Some(*next) + } else { + None + } + } + + /// Return and remove the next code from the queue, or return INVALID_CODE if + /// the queue is empty. + fn remove_next(&mut self) -> Option { + let res = self.next(); + if res.is_some() { + self.next_idx += 1; + } + res + } +} + +#[derive(Clone, Debug, Copy, Default)] +struct Codetab { + last_dst_pos: usize, + prefix_code: Option, + len: u16, + ext_byte: u8, +} + +impl Codetab { + pub fn create_new() -> [Self; MAX_CODE + 1] { + let mut codetab = [Codetab::default(); MAX_CODE + 1]; + for (i, code) in codetab.iter_mut().enumerate().take(u8::MAX as usize + 1) { + *code = Codetab { + prefix_code: Some(i as u16), + ext_byte: i as u8, + len: 1, + last_dst_pos: 0, + }; + } + codetab + } +} + +fn unshrink_partial_clear(codetab: &mut [Codetab], queue: &mut CodeQueue) { + let mut is_prefix = [false; MAX_CODE + 1]; + + // Scan for codes that have been used as a prefix. + for code in codetab.iter().take(MAX_CODE + 1).skip(CONTROL_CODE + 1) { + if let Some(prefix_code) = code.prefix_code { + is_prefix[prefix_code as usize] = true; + } + } + + // Clear "non-prefix" codes in the table; populate the code queue. + let mut code_queue_size = 0; + for i in (CONTROL_CODE + 1)..MAX_CODE { + if !is_prefix[i] { + codetab[i].prefix_code = None; + queue.codes[code_queue_size] = Some(i as u16); + code_queue_size += 1; + } + } + queue.codes[code_queue_size] = None; // End-of-queue marker. + queue.next_idx = 0; +} + +/// Read the next code from the input stream and return it in next_code. Returns +/// `Ok(None)` if the end of the stream is reached. If the stream contains invalid +/// data, returns `Err`. +fn read_code( + is: &mut BitReader, + code_size: &mut u8, + codetab: &mut [Codetab], + queue: &mut CodeQueue, +) -> io::Result> { + // assert(sizeof(code) * CHAR_BIT >= *code_size); + let code = is.read_var::(*code_size as u32)?; + + // Handle regular codes (the common case). + if code != CONTROL_CODE as u16 { + return Ok(Some(code)); + } + + // Handle control codes. + if let Ok(control_code) = is.read_var::(*code_size as u32) { + match control_code { + INC_CODE_SIZE => { + if *code_size >= MAX_CODE_SIZE { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "tried to increase code size when already at maximum", + )); + } + *code_size += 1; + } + PARTIAL_CLEAR => { + unshrink_partial_clear(codetab, queue); + } + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("Invalid control code {}", control_code), + )); + } + } + return read_code(is, code_size, codetab, queue); + } + Ok(None) +} + +/// Output the string represented by a code into dst at dst_pos. +/// +/// # Returns +/// new first byte of the string, or io::Error InvalidData on invalid prefix code. +fn output_code( + dst: &mut Vec, + code: u16, + prev_code: u16, + codetab: &mut [Codetab], + queue: &CodeQueue, + first_byte: u8, +) -> io::Result { + debug_assert!(code <= MAX_CODE as u16 && code != CONTROL_CODE as u16); + if code <= u8::MAX as u16 { + dst.push(code as u8); + return Ok(code as u8); + } + if codetab[code as usize].prefix_code.is_none() + || codetab[code as usize].prefix_code == Some(code) + { + // Reject invalid codes. Self-referential codes may exist in + // the table but cannot be used. + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid code")); + } + if codetab[code as usize].len != UNKNOWN_LEN { + // Output string with known length (the common case). + let ct = &codetab[code as usize]; + for i in ct.last_dst_pos..ct.last_dst_pos + ct.len as usize { + dst.push(dst[i]); + } + return Ok(dst[ct.last_dst_pos]); + } + + // Output a string of unknown length. This happens when the prefix + // was invalid (due to partial clearing) when the code was inserted into + // the table. The prefix can then become valid when it's added to the + // table at a later point. + let prefix_code = codetab[code as usize].prefix_code.unwrap(); + if cfg!(debug_assertions) { + let tab_entry = codetab[code as usize]; + assert!(tab_entry.len == UNKNOWN_LEN); + assert!(tab_entry.prefix_code.unwrap() as usize > CONTROL_CODE); + } + + if Some(prefix_code) == queue.next() { + /* The prefix code hasn't been added yet, but we were just + about to: the KwKwK case. Add the previous string extended + with its first byte. */ + codetab[prefix_code as usize] = Codetab { + prefix_code: Some(prev_code), + ext_byte: first_byte, + len: codetab[prev_code as usize].len + 1, + last_dst_pos: codetab[prev_code as usize].last_dst_pos, + }; + dst.push(first_byte); + } else if codetab[prefix_code as usize].prefix_code.is_none() { + // The prefix code is still invalid. + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid prefix code", + )); + } + + // Output the prefix string, then the extension byte. + let ct = codetab[prefix_code as usize]; + let last_dst_pos = dst.len(); + let start = ct.last_dst_pos; + let len = ct.len as usize; + let first = dst[start]; + dst.extend_from_within(start..start + len); + dst.push(codetab[code as usize].ext_byte); + // Update the code table now that the string has a length and pos. + debug_assert!(prev_code != code); + + codetab[code as usize].len = (len + 1) as u16; + codetab[code as usize].last_dst_pos = last_dst_pos; + + Ok(first) +} + +fn hwunshrink(src: &[u8], uncompressed_size: usize, dst: &mut Vec) -> io::Result<()> { + dst.reserve(uncompressed_size); + let mut codetab = Codetab::create_new(); + let mut queue = CodeQueue::new(); + let mut is = BitReader::endian(src, LittleEndian); + let mut code_size = MIN_CODE_SIZE; + + // Handle the first code separately since there is no previous code. + let Some(curr_code) = read_code(&mut is, &mut code_size, &mut codetab, &mut queue)? else { + return Ok(()); + }; + + debug_assert!(curr_code != CONTROL_CODE as u16); + if curr_code > u8::MAX as u16 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "the first code must be a literal", + )); + } + let mut first_byte = curr_code as u8; // make mutable + codetab[curr_code as usize].last_dst_pos = dst.len(); + dst.push(first_byte); + let mut prev_code = curr_code; + while dst.len() < uncompressed_size { + let curr_opt = match read_code(&mut is, &mut code_size, &mut codetab, &mut queue) { + Ok(c) => c, + Err(_) => break, // treat read error as end-of-stream for now; validated after loop + }; + + let Some(curr_code) = curr_opt else { + return Err(Error::new(io::ErrorKind::InvalidData, "invalid code")); + }; + + let dst_pos = dst.len(); + // Handle KwKwK: next code used before being added. + if Some(curr_code) == queue.next() { + if codetab[prev_code as usize].prefix_code.is_none() { + return Err(Error::new( + io::ErrorKind::InvalidData, + "previous code no longer valid", + )); + } + // Extend the previous code with its first byte. + debug_assert!(curr_code != prev_code); + codetab[curr_code as usize] = Codetab { + prefix_code: Some(prev_code), + ext_byte: first_byte, + len: codetab[prev_code as usize].len + 1, + last_dst_pos: codetab[prev_code as usize].last_dst_pos, + }; + } + // Output the string represented by the current code. + first_byte = output_code(dst, curr_code, prev_code, &mut codetab, &queue, first_byte)?; + if let Some(new_code) = queue.remove_next() { + let prev_entry = codetab[prev_code as usize]; + codetab[new_code as usize] = Codetab { + prefix_code: Some(prev_code), + ext_byte: first_byte, + last_dst_pos: prev_entry.last_dst_pos, + len: if prev_entry.prefix_code.is_none() { + // prev_code was invalidated in a partial + // clearing. Until that code is re-used, the + // string represented by new_code is + // indeterminate. + UNKNOWN_LEN + } else { + prev_entry.len + 1 + }, + }; + } + codetab[curr_code as usize].last_dst_pos = dst_pos; + prev_code = curr_code; + } + + if dst.len() != uncompressed_size { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "unexpected end of compressed stream", + )); + } + + Ok(()) +} + +#[derive(Debug)] +pub struct ShrinkDecoder { + compressed_reader: R, + stream_read: bool, + uncompressed_size: u64, + stream: Vec, + read_pos: usize, +} + +impl ShrinkDecoder { + pub fn new(inner: R, uncompressed_size: u64) -> Self { + Self { + compressed_reader: inner, + uncompressed_size, + stream_read: false, + stream: Vec::new(), + read_pos: 0, + } + } + pub fn into_inner(self) -> R { + self.compressed_reader + } +} + +impl Read for ShrinkDecoder { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if !self.stream_read { + self.stream_read = true; + let mut compressed_bytes = Vec::new(); + self.compressed_reader.read_to_end(&mut compressed_bytes)?; + self.stream.reserve(self.uncompressed_size as usize); + hwunshrink( + &compressed_bytes, + self.uncompressed_size as usize, + &mut self.stream, + )?; + } + let available = self.stream.len().saturating_sub(self.read_pos); + if available == 0 { + return Ok(0); + } + let n = available.min(buf.len()); + buf[..n].copy_from_slice(&self.stream[self.read_pos..self.read_pos + n]); + self.read_pos += n; + Ok(n) + } +} + +#[cfg(test)] +mod tests { + use crate::legacy::shrink::hwunshrink; + + const LZW_FIG5: &[u8; 17] = b"ababcbababaaaaaaa"; + const LZW_FIG5_SHRUNK: [u8; 12] = [ + 0x61, 0xc4, 0x04, 0x1c, 0x23, 0xb0, 0x60, 0x98, 0x83, 0x08, 0xc3, 0x00, + ]; + + #[test] + fn test_unshrink_lzw_fig5() { + let mut dst = Vec::with_capacity(LZW_FIG5.len()); + hwunshrink(&LZW_FIG5_SHRUNK, LZW_FIG5.len(), &mut dst).unwrap(); + assert_eq!(dst.as_slice(), LZW_FIG5); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/lib.rs b/v2/vendor/security/zip-7.2.0/src/lib.rs new file mode 100644 index 0000000000..adc69dc1cb --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/lib.rs @@ -0,0 +1,76 @@ +//! A library for reading and writing ZIP archives. +//! ZIP is a format designed for cross-platform file "archiving". +//! That is, storing a collection of files in a single datastream +//! to make them easier to share between computers. +//! Additionally, ZIP is able to compress and encrypt files in its +//! archives. +//! +//! The current implementation is based on [PKWARE's APPNOTE.TXT v6.3.9](https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT) +//! +//! --- +//! +//! [`zip`](`crate`) has support for the most common ZIP archives found in common use. +//! However, in special cases, +//! there are some zip archives that are difficult to read or write. +//! +//! This is a list of supported features: +//! +//! | | Reading | Writing | +//! | ------------------------------- | ------------------------------------- | -------------------------------------------------------- | +//! | Stored | ✅ | ✅ | +//! | Deflate | ✅ [->](`crate::ZipArchive::by_name`) | ✅ [->](`crate::write::FileOptions::compression_method`) | +//! | Deflate64 | ✅ | | +//! | Bzip2 | ✅ | ✅ | +//! | ZStandard | ✅ | ✅ | +//! | LZMA | ✅ | | +//! | XZ | ✅ | ✅ | +//! | PPMd | ✅ | ✅ | +//! | AES encryption | ✅ | ✅ | +//! | ZipCrypto deprecated encryption | ✅ | ✅ | +//! + +#![cfg_attr(docsrs, feature(doc_cfg))] +#![warn(missing_docs)] +#![allow(unexpected_cfgs)] // Needed for cfg(fuzzing) +pub use crate::compression::{CompressionMethod, SUPPORTED_COMPRESSION_METHODS}; +pub use crate::read::HasZipMetadata; +pub use crate::read::{ZipArchive, ZipReadOptions}; +pub use crate::spec::{ZIP64_BYTES_THR, ZIP64_ENTRY_THR}; +pub use crate::types::{AesMode, DateTime}; +pub use crate::write::ZipWriter; + +#[cfg(feature = "aes-crypto")] +mod aes; +#[cfg(feature = "aes-crypto")] +mod aes_ctr; +mod compression; +mod cp437; +mod crc32; +pub mod extra_fields; +mod path; +pub mod read; +pub mod result; +mod spec; +mod types; +pub mod write; +mod zipcrypto; +pub use extra_fields::ExtraField; +#[cfg(feature = "legacy-zip")] +mod legacy; + +#[doc = "Unstable APIs\n\ +\ +All APIs accessible by importing this module are unstable; They may be changed in patch \ +releases. You MUST use an exact version specifier in `Cargo.toml`, to indicate the version of this \ +API you're using:\n\ +\ +```toml\n +[dependencies]\n +zip = \"="] +#[doc=env!("CARGO_PKG_VERSION")] +#[doc = "\"\n\ +```"] +pub mod unstable; + +#[doc(hidden)] +pub mod macros; diff --git a/v2/vendor/security/zip-7.2.0/src/macros.rs b/v2/vendor/security/zip-7.2.0/src/macros.rs new file mode 100644 index 0000000000..3d07efd7b8 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/macros.rs @@ -0,0 +1,162 @@ +//! Macros used internally. +//! +//! These may technically be exported, but that's only to make them available to internal +//! project dependencies. The `#[doc(hidden)]` mark indicates that these are not stable or supported +//! APIs, and should not be relied upon by external dependees. + +/// The single macro export of the [`cfg-if`](https://docs.rs/cfg-if) crate. +/// +/// It is packaged here to avoid pulling in another dependency. The stdlib does the same[^1]. +/// +/// [^1]: https://github.com/rust-lang/rust/blob/a2db9280539229a3b8a084a09886670a57bc7e9c/library/compiler-builtins/libm/src/math/support/macros.rs#L1 +#[doc(hidden)] +#[macro_export] +macro_rules! cfg_if { + // match if/else chains with a final `else` + ( + $( + if #[cfg( $i_meta:meta )] { $( $i_tokens:tt )* } + ) else+ + else { $( $e_tokens:tt )* } + ) => { + $crate::cfg_if! { + @__items () ; + $( + (( $i_meta ) ( $( $i_tokens )* )) , + )+ + (() ( $( $e_tokens )* )) , + }; + }; + + // match if/else chains lacking a final `else` + ( + if #[cfg( $i_meta:meta )] { $( $i_tokens:tt )* } + $( + else if #[cfg( $e_meta:meta )] { $( $e_tokens:tt )* } + )* + ) => { + $crate::cfg_if! { + @__items () ; + (( $i_meta ) ( $( $i_tokens )* )) , + $( + (( $e_meta ) ( $( $e_tokens )* )) , + )* + }; + }; + + // Internal and recursive macro to emit all the items + // + // Collects all the previous cfgs in a list at the beginning, so they can be + // negated. After the semicolon are all the remaining items. + (@__items ( $( $_:meta , )* ) ; ) => {}; + ( + @__items ( $( $no:meta , )* ) ; + (( $( $yes:meta )? ) ( $( $tokens:tt )* )) , + $( $rest:tt , )* + ) => { + // Emit all items within one block, applying an appropriate #[cfg]. The + // #[cfg] will require all `$yes` matchers specified and must also negate + // all previous matchers. + #[cfg(all( + $( $yes , )? + not(any( $( $no ),* )) + ))] + $crate::cfg_if! { @__identity $( $tokens )* } + + // Recurse to emit all other items in `$rest`, and when we do so add all + // our `$yes` matchers to the list of `$no` matchers as future emissions + // will have to negate everything we just matched as well. + $crate::cfg_if! { + @__items ( $( $no , )* $( $yes , )? ) ; + $( $rest , )* + }; + }; + + // Internal macro to make __apply work out right for different match types, + // because of how macros match/expand stuff. + (@__identity $( $tokens:tt )* ) => { + $( $tokens )* + }; +} + +/// Similar to [`cfg_if`](cfg_if), but accepts a list of expressions, and generates an internal +/// closure to return each value. +/// +/// The main reason this is necessary is because attaching `#[cfg(...)]` annotations to certain +/// types of statements requires a nightly feature, or `cfg_if` would be enough on its own. This +/// macro's restricted interface allows it to generate a closure as a circumlocution that is legal +/// on stable rust. +/// +/// Note that any `return` operation within the expressions provided to this macro will apply to the +/// generated closure, not the enclosing scope--it cannot be used to interfere with external +/// control flow. +/// +/// The generated closure is non-[`const`](const@keyword), so cannot be used inside `const` methods. +#[doc(hidden)] +#[macro_export] +macro_rules! cfg_if_expr { + // Match =>, chains, maybe with a final _ => catchall clause. + ( + $( $ret_ty:ty : )? + $( + #[cfg( $i_meta:meta )] => $i_val:expr + ),+ , + _ => $rem_val:expr $(,)? + ) => { + (|| $( -> $ret_ty )? { + $crate::cfg_if_expr! { + @__items (); + $( + (( $i_meta ) ( + #[allow(unreachable_code)] + return $i_val ; + )) , + )+ + (() ( + #[allow(unreachable_code)] + return $rem_val ; + )) , + } + })() + }; + // Match =>, chains *without* any _ => clause. + ( + $( $ret_ty:ty : )? + $( + #[cfg( $i_meta:meta )] => $i_val:expr + ),+ $(,)? + ) => { + (|| $( -> $ret_ty )? { + $crate::cfg_if_expr! { + @__items (); + $( + (( $i_meta ) ( + #[allow(unreachable_code)] + return $i_val ; + )) , + )+ + } + })() + }; + + (@__items ( $( $_:meta , )* ) ; ) => {}; + ( + @__items ( $( $no:meta , )* ) ; + (( $( $yes:meta )? ) ( $( $tokens:tt )* )) , + $( $rest:tt , )* + ) => { + #[cfg(all( + $( $yes , )? + not(any( $( $no ),* )) + ))] + $crate::cfg_if_expr! { @__identity $( $tokens )* } + + $crate::cfg_if_expr! { + @__items ( $( $no , )* $( $yes , )? ) ; + $( $rest , )* + }; + }; + (@__identity $( $tokens:tt )* ) => { + $( $tokens )* + }; +} diff --git a/v2/vendor/security/zip-7.2.0/src/path.rs b/v2/vendor/security/zip-7.2.0/src/path.rs new file mode 100644 index 0000000000..8f0c698a03 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/path.rs @@ -0,0 +1,110 @@ +//! Path manipulation utilities + +use std::{ffi::OsStr, path::Path}; +use typed_path::{Utf8WindowsComponent, Utf8WindowsPath}; + +/// Simplify a path by removing the prefix and parent directories and only return normal components +pub(crate) fn simplified_components(input: &Path) -> Option> { + let mut out = Vec::new(); + let input_str = input.to_str()?; + for component in Utf8WindowsPath::new(input_str).components() { + match component { + // Skip prefix and root directory components instead of rejecting the entire path + // This allows extraction of ZIP files with absolute paths, similar to other ZIP tools + Utf8WindowsComponent::Prefix(_) | Utf8WindowsComponent::RootDir => (), + Utf8WindowsComponent::ParentDir => { + out.pop()?; + } + Utf8WindowsComponent::Normal(s) => out.push(OsStr::new(s)), + Utf8WindowsComponent::CurDir => (), + } + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::Path; + + #[test] + fn test_simplified_components_relative_path() { + let path = Path::new("foo/bar/baz.txt"); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 3); + assert_eq!(components[0], "foo"); + assert_eq!(components[1], "bar"); + assert_eq!(components[2], "baz.txt"); + } + + #[test] + fn test_simplified_components_absolute_unix_path() { + let path = Path::new("/foo/bar/baz.txt"); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 3); + assert_eq!(components[0], "foo"); + assert_eq!(components[1], "bar"); + assert_eq!(components[2], "baz.txt"); + } + + #[test] + fn test_simplified_components_with_parent_dirs() { + let path = Path::new("foo/../bar/baz.txt"); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 2); + assert_eq!(components[0], "bar"); + assert_eq!(components[1], "baz.txt"); + } + + #[test] + fn test_simplified_components_too_many_parent_dirs() { + let path = Path::new("foo/../../bar"); + let result = simplified_components(path); + assert!(result.is_none()); // Should still fail for directory traversal attacks + } + + #[test] + fn test_simplified_components_with_current_dir() { + let path = Path::new("foo/./bar/baz.txt"); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 3); + assert_eq!(components[0], "foo"); + assert_eq!(components[1], "bar"); + assert_eq!(components[2], "baz.txt"); + } + + #[test] + fn test_simplified_components_empty_path() { + let path = Path::new(""); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 0); + } + + #[test] + fn test_simplified_components_root_only() { + let path = Path::new("/"); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 0); + } + + #[cfg(windows)] + #[test] + fn test_simplified_components_windows_absolute_path() { + let path = Path::new(r"C:\foo\bar\baz.txt"); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 3); + assert_eq!(components[0], "foo"); + assert_eq!(components[1], "bar"); + assert_eq!(components[2], "baz.txt"); + } + + #[cfg(windows)] + #[test] + fn test_simplified_components_windows_unc_path() { + let path = Path::new(r"\\server\share\foo\bar.txt"); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 2); + assert_eq!(components[0], "foo"); + assert_eq!(components[1], "bar.txt"); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/read.rs b/v2/vendor/security/zip-7.2.0/src/read.rs new file mode 100644 index 0000000000..be430435c0 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/read.rs @@ -0,0 +1,2650 @@ +//! Types for reading ZIP archives + +/// Upper bound on the uncompressed size of a symlink entry. +/// +/// A symlink target is a filesystem path, so its length is bounded by the +/// platform (`PATH_MAX` is 4096 on Linux) no matter what the archive says. The +/// uncompressed size stored in the central directory is attacker-controlled, +/// so entries above this bound are rejected rather than pre-allocated. +pub(crate) const MAX_SYMLINK_TARGET_LEN: u64 = 4096; + +#[cfg(feature = "aes-crypto")] +use crate::aes::{AesReader, AesReaderValid}; +use crate::cfg_if; +use crate::compression::{CompressionMethod, Decompressor}; +use crate::cp437::FromCp437; +use crate::crc32::Crc32Reader; +use crate::extra_fields::{ExtendedTimestamp, ExtraField, Ntfs}; +use crate::result::{invalid, ZipError, ZipResult}; +use crate::spec::{ + self, CentralDirectoryEndInfo, DataAndPosition, FixedSizeBlock, Pod, ZIP64_BYTES_THR, +}; +use crate::types::{ + AesMode, AesVendorVersion, DateTime, SimpleFileOptions, System, ZipCentralEntryBlock, + ZipFileData, ZipLocalEntryBlock, +}; +use crate::zipcrypto::{ZipCryptoReader, ZipCryptoReaderValid, ZipCryptoValidator}; +use core::mem::{replace, size_of}; +use core::ops::{Deref, Range}; +use indexmap::IndexMap; +use std::borrow::Cow; +use std::ffi::OsStr; +use std::io::{self, copy, sink, Read, Seek, SeekFrom, Write}; +use std::path::{Component, Path, PathBuf}; +use std::sync::{Arc, OnceLock}; + +mod config; + +pub use config::{ArchiveOffset, Config}; + +/// Provides high level API for reading from a stream. +pub(crate) mod stream; + +pub(crate) mod magic_finder; + +/// Immutable metadata about a `ZipArchive`. +#[derive(Debug)] +pub struct ZipArchiveMetadata { + pub(crate) files: IndexMap, ZipFileData>, + pub(crate) offset: u64, + pub(crate) dir_start: u64, + // This isn't yet used anywhere, but it is here for use cases in the future. + #[allow(dead_code)] + pub(crate) config: Config, + pub(crate) comment: Box<[u8]>, + pub(crate) zip64_comment: Option>, +} + +pub(crate) mod zip_archive { + use crate::read::ZipArchiveMetadata; + use indexmap::IndexMap; + use std::sync::Arc; + + #[derive(Debug)] + pub(crate) struct SharedBuilder { + pub(crate) files: Vec, + pub(super) offset: u64, + pub(super) dir_start: u64, + // This isn't yet used anywhere, but it is here for use cases in the future. + #[allow(dead_code)] + pub(super) config: super::Config, + } + + impl SharedBuilder { + pub fn build( + self, + comment: Box<[u8]>, + zip64_comment: Option>, + ) -> ZipArchiveMetadata { + let mut index_map = IndexMap::with_capacity(self.files.len()); + self.files.into_iter().for_each(|file| { + index_map.insert(file.file_name.clone(), file); + }); + ZipArchiveMetadata { + files: index_map, + offset: self.offset, + dir_start: self.dir_start, + config: self.config, + comment, + zip64_comment, + } + } + } + + /// ZIP archive reader + /// + /// At the moment, this type is cheap to clone if this is the case for the + /// reader it uses. However, this is not guaranteed by this crate and it may + /// change in the future. + /// + /// ```no_run + /// use std::io::{Read, Seek}; + /// fn list_zip_contents(reader: impl Read + Seek) -> zip::result::ZipResult<()> { + /// use zip::HasZipMetadata; + /// let mut zip = zip::ZipArchive::new(reader)?; + /// + /// for i in 0..zip.len() { + /// let mut file = zip.by_index(i)?; + /// println!("Filename: {}", file.name()); + /// std::io::copy(&mut file, &mut std::io::stdout())?; + /// } + /// + /// Ok(()) + /// } + /// ``` + #[derive(Clone, Debug)] + pub struct ZipArchive { + pub(super) reader: R, + pub(super) shared: Arc, + } +} + +#[cfg(feature = "aes-crypto")] +use crate::aes::PWD_VERIFY_LENGTH; +use crate::extra_fields::UnicodeExtraField; +use crate::result::ZipError::InvalidPassword; +use crate::spec::is_dir; +use crate::types::ffi::{S_IFLNK, S_IFREG}; +use crate::unstable::{path_to_string, LittleEndianReadExt}; +pub use zip_archive::ZipArchive; + +#[allow(clippy::large_enum_variant)] +pub(crate) enum CryptoReader<'a, R: Read + ?Sized> { + Plaintext(io::Take<&'a mut R>), + ZipCrypto(ZipCryptoReaderValid>), + #[cfg(feature = "aes-crypto")] + Aes { + reader: AesReaderValid>, + vendor_version: AesVendorVersion, + }, +} + +impl Read for CryptoReader<'_, R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + CryptoReader::Plaintext(r) => r.read(buf), + CryptoReader::ZipCrypto(r) => r.read(buf), + #[cfg(feature = "aes-crypto")] + CryptoReader::Aes { reader: r, .. } => r.read(buf), + } + } + + fn read_to_end(&mut self, buf: &mut Vec) -> io::Result { + match self { + CryptoReader::Plaintext(r) => r.read_to_end(buf), + CryptoReader::ZipCrypto(r) => r.read_to_end(buf), + #[cfg(feature = "aes-crypto")] + CryptoReader::Aes { reader: r, .. } => r.read_to_end(buf), + } + } + + fn read_to_string(&mut self, buf: &mut String) -> io::Result { + match self { + CryptoReader::Plaintext(r) => r.read_to_string(buf), + CryptoReader::ZipCrypto(r) => r.read_to_string(buf), + #[cfg(feature = "aes-crypto")] + CryptoReader::Aes { reader: r, .. } => r.read_to_string(buf), + } + } +} + +impl<'a, R: Read + ?Sized> CryptoReader<'a, R> { + /// Consumes this decoder, returning the underlying reader. + pub fn into_inner(self) -> io::Take<&'a mut R> { + match self { + CryptoReader::Plaintext(r) => r, + CryptoReader::ZipCrypto(r) => r.into_inner(), + #[cfg(feature = "aes-crypto")] + CryptoReader::Aes { reader: r, .. } => r.into_inner(), + } + } + + /// Returns `true` if the data is encrypted using AE2. + #[allow(clippy::needless_return)] // can't use cfg_if_expr! because const + pub const fn is_ae2_encrypted(&self) -> bool { + cfg_if! { + if #[cfg(feature = "aes-crypto")] { + return matches!( + self, + CryptoReader::Aes { + vendor_version: AesVendorVersion::Ae2, + .. + } + ); + } else { + return false; + } + } + } +} + +#[cold] +fn invalid_state() -> io::Result { + Err(io::Error::other("ZipFileReader was in an invalid state")) +} + +pub(crate) enum ZipFileReader<'a, R: Read + ?Sized> { + NoReader, + Raw(io::Take<&'a mut R>), + Compressed(Box>>>>), +} + +impl Read for ZipFileReader<'_, R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + ZipFileReader::NoReader => invalid_state(), + ZipFileReader::Raw(r) => r.read(buf), + ZipFileReader::Compressed(r) => r.read(buf), + } + } + + fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> { + match self { + ZipFileReader::NoReader => invalid_state(), + ZipFileReader::Raw(r) => r.read_exact(buf), + ZipFileReader::Compressed(r) => r.read_exact(buf), + } + } + + fn read_to_end(&mut self, buf: &mut Vec) -> io::Result { + match self { + ZipFileReader::NoReader => invalid_state(), + ZipFileReader::Raw(r) => r.read_to_end(buf), + ZipFileReader::Compressed(r) => r.read_to_end(buf), + } + } + + fn read_to_string(&mut self, buf: &mut String) -> io::Result { + match self { + ZipFileReader::NoReader => invalid_state(), + ZipFileReader::Raw(r) => r.read_to_string(buf), + ZipFileReader::Compressed(r) => r.read_to_string(buf), + } + } +} + +impl<'a, R: Read + ?Sized> ZipFileReader<'a, R> { + fn into_inner(self) -> io::Result> { + match self { + ZipFileReader::NoReader => invalid_state(), + ZipFileReader::Raw(r) => Ok(r), + ZipFileReader::Compressed(r) => { + Ok(r.into_inner().into_inner()?.into_inner().into_inner()) + } + } + } +} + +/// A struct for reading a zip file +pub struct ZipFile<'a, R: Read + ?Sized> { + pub(crate) data: Cow<'a, ZipFileData>, + pub(crate) reader: ZipFileReader<'a, R>, +} + +/// A struct for reading and seeking a zip file +pub struct ZipFileSeek<'a, R> { + data: Cow<'a, ZipFileData>, + reader: ZipFileSeekReader<'a, R>, +} + +enum ZipFileSeekReader<'a, R: ?Sized> { + Raw(SeekableTake<'a, R>), +} + +struct SeekableTake<'a, R: ?Sized> { + inner: &'a mut R, + inner_starting_offset: u64, + length: u64, + current_offset: u64, +} + +impl<'a, R: Seek + ?Sized> SeekableTake<'a, R> { + pub fn new(inner: &'a mut R, length: u64) -> io::Result { + let inner_starting_offset = inner.stream_position()?; + Ok(Self { + inner, + inner_starting_offset, + length, + current_offset: 0, + }) + } +} + +impl Seek for SeekableTake<'_, R> { + fn seek(&mut self, pos: SeekFrom) -> io::Result { + let offset = match pos { + SeekFrom::Start(offset) => Some(offset), + SeekFrom::End(offset) => self.length.checked_add_signed(offset), + SeekFrom::Current(offset) => self.current_offset.checked_add_signed(offset), + }; + match offset { + None => Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid seek to a negative or overflowing position", + )), + Some(offset) => { + let clamped_offset = std::cmp::min(self.length, offset); + let new_inner_offset = self + .inner + .seek(SeekFrom::Start(self.inner_starting_offset + clamped_offset))?; + self.current_offset = new_inner_offset - self.inner_starting_offset; + Ok(self.current_offset) + } + } + } +} + +impl Read for SeekableTake<'_, R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let written = self + .inner + .take(self.length - self.current_offset) + .read(buf)?; + self.current_offset += written as u64; + Ok(written) + } +} + +pub(crate) fn make_writable_dir_all>(outpath: T) -> Result<(), ZipError> { + use std::fs; + fs::create_dir_all(outpath.as_ref())?; + #[cfg(unix)] + { + // Dirs must be writable until all normal files are extracted + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions( + outpath.as_ref(), + std::fs::Permissions::from_mode( + 0o700 | std::fs::metadata(outpath.as_ref())?.permissions().mode(), + ), + )?; + } + Ok(()) +} + +pub(crate) fn find_content<'a, R: Read + Seek + ?Sized>( + data: &ZipFileData, + reader: &'a mut R, +) -> ZipResult> { + // TODO: use .get_or_try_init() once stabilized to provide a closure returning a Result! + let data_start = data.data_start(reader)?; + + reader.seek(SeekFrom::Start(data_start))?; + Ok(reader.take(data.compressed_size)) +} + +fn find_content_seek<'a, R: Read + Seek + ?Sized>( + data: &ZipFileData, + reader: &'a mut R, +) -> ZipResult> { + // Parse local header + let data_start = data.data_start(reader)?; + reader.seek(SeekFrom::Start(data_start))?; + + // Explicit Ok and ? are needed to convert io::Error to ZipError + Ok(SeekableTake::new(reader, data.compressed_size)?) +} + +pub(crate) fn find_data_start( + data: &ZipFileData, + reader: &mut (impl Read + Seek + ?Sized), +) -> Result { + // Go to start of data. + reader.seek(SeekFrom::Start(data.header_start))?; + + // Parse static-sized fields and check the magic value. + let block = ZipLocalEntryBlock::parse(reader)?; + + // Calculate the end of the local header from the fields we just parsed. + let variable_fields_len = + // Each of these fields must be converted to u64 before adding, as the result may + // easily overflow a u16. + block.file_name_length as u64 + block.extra_field_length as u64; + let data_start = + data.header_start + size_of::() as u64 + variable_fields_len; + + // Set the value so we don't have to read it again. + match data.data_start.set(data_start) { + Ok(()) => (), + // If the value was already set in the meantime, ensure it matches (this is probably + // unnecessary). + Err(_) => { + debug_assert_eq!(*data.data_start.get().unwrap(), data_start); + } + } + + Ok(data_start) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn make_crypto_reader<'a, R: Read + ?Sized>( + data: &ZipFileData, + reader: io::Take<&'a mut R>, + password: Option<&[u8]>, + aes_info: Option<(AesMode, AesVendorVersion, CompressionMethod)>, +) -> ZipResult> { + #[allow(deprecated)] + { + if let CompressionMethod::Unsupported(_) = data.compression_method { + return unsupported_zip_error("Compression method not supported"); + } + } + + let reader = match (password, aes_info) { + #[cfg(not(feature = "aes-crypto"))] + (Some(_), Some(_)) => { + return Err(ZipError::UnsupportedArchive( + "AES encrypted files cannot be decrypted without the aes-crypto feature.", + )) + } + #[cfg(feature = "aes-crypto")] + (Some(password), Some((aes_mode, vendor_version, _))) => CryptoReader::Aes { + reader: AesReader::new(reader, aes_mode, data.compressed_size).validate(password)?, + vendor_version, + }, + (Some(password), None) => { + let validator = if data.using_data_descriptor { + ZipCryptoValidator::InfoZipMsdosTime( + data.last_modified_time.map_or(0, |x| x.timepart()), + ) + } else { + ZipCryptoValidator::PkzipCrc32(data.crc32) + }; + CryptoReader::ZipCrypto(ZipCryptoReader::new(reader, password).validate(validator)?) + } + (None, Some(_)) => return Err(InvalidPassword), + (None, None) => CryptoReader::Plaintext(reader), + }; + Ok(reader) +} + +pub(crate) fn make_reader<'a, R: Read + ?Sized>( + compression_method: CompressionMethod, + uncompressed_size: u64, + crc32: u32, + reader: CryptoReader<'a, R>, + #[cfg(feature = "legacy-zip")] flags: u16, +) -> ZipResult> { + let ae2_encrypted = reader.is_ae2_encrypted(); + #[cfg(not(feature = "legacy-zip"))] + let flags = 0; + Ok(ZipFileReader::Compressed(Box::new(Crc32Reader::new( + Decompressor::new( + io::BufReader::new(reader), + compression_method, + uncompressed_size, + flags, + )?, + crc32, + ae2_encrypted, + )))) +} + +pub(crate) fn make_symlink( + outpath: &Path, + target: &[u8], + #[cfg_attr(not(windows), allow(unused))] existing_files: &IndexMap, T>, +) -> ZipResult<()> { + #[cfg_attr(not(any(unix, windows)), allow(unused))] + let Ok(target_str) = std::str::from_utf8(target) else { + return Err(invalid!("Invalid UTF-8 as symlink target")); + }; + + cfg_if! { + if #[cfg(unix)] { + std::os::unix::fs::symlink(Path::new(&target_str), outpath)?; + } else if #[cfg(windows)] { + let target = Path::new(OsStr::new(&target_str)); + let target_is_dir_from_archive = + existing_files.contains_key(target_str) && is_dir(target_str); + let target_is_dir = if target_is_dir_from_archive { + true + } else if let Ok(meta) = std::fs::metadata(target) { + meta.is_dir() + } else { + false + }; + if target_is_dir { + std::os::windows::fs::symlink_dir(target, outpath)?; + } else { + std::os::windows::fs::symlink_file(target, outpath)?; + } + } else { + use std::fs::File; + let output = File::create(outpath); + output?.write_all(target)?; + } + } + + Ok(()) +} + +#[derive(Debug)] +pub(crate) struct CentralDirectoryInfo { + pub(crate) archive_offset: u64, + pub(crate) directory_start: u64, + pub(crate) number_of_files: usize, + pub(crate) disk_number: u32, + pub(crate) disk_with_central_directory: u32, +} + +impl<'a> TryFrom<&'a CentralDirectoryEndInfo> for CentralDirectoryInfo { + type Error = ZipError; + + fn try_from(value: &'a CentralDirectoryEndInfo) -> Result { + let (relative_cd_offset, number_of_files, disk_number, disk_with_central_directory) = + match &value.eocd64 { + Some(DataAndPosition { data: eocd64, .. }) => { + if eocd64.number_of_files_on_this_disk > eocd64.number_of_files { + return Err(invalid!("ZIP64 footer indicates more files on this disk than in the whole archive")); + } + ( + eocd64.central_directory_offset, + eocd64.number_of_files as usize, + eocd64.disk_number, + eocd64.disk_with_central_directory, + ) + } + _ => ( + value.eocd.data.central_directory_offset as u64, + value.eocd.data.number_of_files_on_this_disk as usize, + value.eocd.data.disk_number as u32, + value.eocd.data.disk_with_central_directory as u32, + ), + }; + + let directory_start = relative_cd_offset + .checked_add(value.archive_offset) + .ok_or(invalid!("Invalid central directory size or offset"))?; + + Ok(Self { + archive_offset: value.archive_offset, + directory_start, + number_of_files, + disk_number, + disk_with_central_directory, + }) + } +} + +/// Store all entries which specify a numeric "mode" which is familiar to POSIX operating systems. +#[cfg(unix)] +#[derive(Default, Debug)] +struct UnixFileModes { + map: std::collections::BTreeMap, +} + +#[cfg(unix)] +impl UnixFileModes { + #[cfg_attr(not(debug_assertions), allow(unused))] + pub fn add_mode(&mut self, path: PathBuf, mode: u32) { + // We don't print a warning or consider it remotely out of the ordinary to receive two + // separate modes for the same path: just take the later one. + let old_entry = self.map.insert(path, mode); + debug_assert_eq!(old_entry, None); + } + + // Child nodes will be sorted later lexicographically, so reversing the order puts them first. + pub fn all_perms_with_children_first( + self, + ) -> impl IntoIterator { + use std::os::unix::fs::PermissionsExt; + self.map + .into_iter() + .rev() + .map(|(p, m)| (p, std::fs::Permissions::from_mode(m))) + } +} + +impl ZipArchive { + pub(crate) fn from_finalized_writer( + files: IndexMap, ZipFileData>, + comment: Box<[u8]>, + zip64_comment: Option>, + reader: R, + central_start: u64, + ) -> ZipResult { + let initial_offset = match files.first() { + Some((_, file)) => file.header_start, + None => central_start, + }; + let shared = Arc::new(ZipArchiveMetadata { + files, + offset: initial_offset, + dir_start: central_start, + config: Config { + archive_offset: ArchiveOffset::Known(initial_offset), + }, + comment, + zip64_comment, + }); + Ok(Self { reader, shared }) + } + + /// Total size of the files in the archive, if it can be known. Doesn't include directories or + /// metadata. + pub fn decompressed_size(&self) -> Option { + let mut total = 0u128; + for file in self.shared.files.values() { + if file.using_data_descriptor { + return None; + } + total = total.checked_add(file.uncompressed_size as u128)?; + } + Some(total) + } +} + +impl ZipArchive { + pub(crate) fn merge_contents( + &mut self, + mut w: W, + ) -> ZipResult, ZipFileData>> { + if self.shared.files.is_empty() { + return Ok(IndexMap::new()); + } + let mut new_files = self.shared.files.clone(); + /* The first file header will probably start at the beginning of the file, but zip doesn't + * enforce that, and executable zips like PEX files will have a shebang line so will + * definitely be greater than 0. + * + * assert_eq!(0, new_files[0].header_start); // Avoid this. + */ + + let first_new_file_header_start = w.stream_position()?; + + /* Push back file header starts for all entries in the covered files. */ + new_files.values_mut().try_for_each(|f| { + /* This is probably the only really important thing to change. */ + f.header_start = f + .header_start + .checked_add(first_new_file_header_start) + .ok_or(invalid!( + "new header start from merge would have been too large" + ))?; + /* This is only ever used internally to cache metadata lookups (it's not part of the + * zip spec), and 0 is the sentinel value. */ + f.central_header_start = 0; + /* This is an atomic variable so it can be updated from another thread in the + * implementation (which is good!). */ + if let Some(old_data_start) = f.data_start.take() { + let new_data_start = old_data_start + .checked_add(first_new_file_header_start) + .ok_or(invalid!( + "new data start from merge would have been too large" + ))?; + f.data_start.get_or_init(|| new_data_start); + } + Ok::<_, ZipError>(()) + })?; + + /* Rewind to the beginning of the file. + * + * NB: we *could* decide to start copying from new_files[0].header_start instead, which + * would avoid copying over e.g. any pex shebangs or other file contents that start before + * the first zip file entry. However, zip files actually shouldn't care about garbage data + * in *between* real entries, since the central directory header records the correct start + * location of each, and keeping track of that math is more complicated logic that will only + * rarely be used, since most zips that get merged together are likely to be produced + * specifically for that purpose (and therefore are unlikely to have a shebang or other + * preface). Finally, this preserves any data that might actually be useful. + */ + self.reader.rewind()?; + /* Find the end of the file data. */ + let length_to_read = self.shared.dir_start; + /* Produce a Read that reads bytes up until the start of the central directory header. + * This "as &mut dyn Read" trick is used elsewhere to avoid having to clone the underlying + * handle, which it really shouldn't need to anyway. */ + let mut limited_raw = (&mut self.reader as &mut dyn Read).take(length_to_read); + /* Copy over file data from source archive directly. */ + io::copy(&mut limited_raw, &mut w)?; + + /* Return the files we've just written to the data stream. */ + Ok(new_files) + } + + /// Get the directory start offset and number of files. This is done in a + /// separate function to ease the control flow design. + pub(crate) fn get_metadata(config: Config, reader: &mut R) -> ZipResult { + // End of the probed region, initially set to the end of the file + let file_len = reader.seek(io::SeekFrom::End(0))?; + let mut end_exclusive = file_len; + let mut last_err = None; + + loop { + // Find the EOCD and possibly EOCD64 entries and determine the archive offset. + let cde = match spec::find_central_directory( + reader, + config.archive_offset, + end_exclusive, + file_len, + ) { + Ok(cde) => cde, + Err(e) => { + // return the previous error first (if there is) + return Err(last_err.unwrap_or(e)); + } + }; + + // Turn EOCD into internal representation. + match CentralDirectoryInfo::try_from(&cde) + .and_then(|info| Self::read_central_header(info, config, reader)) + { + Ok(shared) => { + return Ok(shared.build( + cde.eocd.data.zip_file_comment, + cde.eocd64.map(|v| v.data.extensible_data_sector), + )); + } + Err(e) => { + last_err = Some(e); + } + }; + // Something went wrong while decoding the cde, try to find a new one + end_exclusive = cde.eocd.position; + continue; + } + } + + fn read_central_header( + dir_info: CentralDirectoryInfo, + config: Config, + reader: &mut R, + ) -> Result { + // If the parsed number of files is greater than the offset then + // something fishy is going on and we shouldn't trust number_of_files. + let file_capacity = if dir_info.number_of_files > dir_info.directory_start as usize { + 0 + } else { + dir_info.number_of_files + }; + + if dir_info.disk_number != dir_info.disk_with_central_directory { + return unsupported_zip_error("Support for multi-disk files is not implemented"); + } + + if file_capacity.saturating_mul(size_of::()) > isize::MAX as usize { + return unsupported_zip_error("Oversized central directory"); + } + + let mut files = Vec::with_capacity(file_capacity); + reader.seek(SeekFrom::Start(dir_info.directory_start))?; + for _ in 0..dir_info.number_of_files { + let file = central_header_to_zip_file(reader, &dir_info)?; + files.push(file); + } + + Ok(zip_archive::SharedBuilder { + files, + offset: dir_info.archive_offset, + dir_start: dir_info.directory_start, + config, + }) + } + + /// Returns the verification value and salt for the AES encryption of the file + /// + /// It fails if the file number is invalid. + /// + /// # Returns + /// + /// - None if the file is not encrypted with AES + #[cfg(feature = "aes-crypto")] + pub fn get_aes_verification_key_and_salt( + &mut self, + file_number: usize, + ) -> ZipResult> { + let (_, data) = self + .shared + .files + .get_index(file_number) + .ok_or(ZipError::FileNotFound)?; + + let limit_reader = find_content(data, &mut self.reader)?; + match data.aes_mode { + None => Ok(None), + Some((aes_mode, _, _)) => { + let (verification_value, salt) = + AesReader::new(limit_reader, aes_mode, data.compressed_size) + .get_verification_value_and_salt()?; + let aes_info = AesInfo { + aes_mode, + verification_value, + salt, + }; + Ok(Some(aes_info)) + } + } + } + + /// Read a ZIP archive, collecting the files it contains. + /// + /// This uses the central directory record of the ZIP file, and ignores local file headers. + /// + /// A default [`Config`] is used. + pub fn new(reader: R) -> ZipResult> { + Self::with_config(Default::default(), reader) + } + + /// Get the metadata associated with the ZIP archive. + /// + /// This can be used with [`Self::unsafe_new_with_metadata`] to create a new reader over the + /// same file without needing to reparse the metadata. + pub fn metadata(&self) -> Arc { + self.shared.clone() + } + + /// Read a ZIP archive using the given `metadata`. + /// + /// This is useful for creating multiple readers over the same file without + /// needing to reparse the metadata. + /// + /// # Safety + /// `unsafe` is used here to indicate that `reader` and `metadata` could + /// potentially be incompatible, and it is left to the user to ensure they are. + /// + /// # Example + /// + /// ```no_run + /// # use std::fs; + /// use rayon::prelude::*; + /// + /// const FILE_NAME: &str = "my_data.zip"; + /// + /// let file = fs::File::open(FILE_NAME).unwrap(); + /// let mut archive = zip::ZipArchive::new(file).unwrap(); + /// + /// let file_names = (0..archive.len()) + /// .into_par_iter() + /// .map_init({ + /// let metadata = archive.metadata().clone(); + /// move || { + /// let file = fs::File::open(FILE_NAME).unwrap(); + /// unsafe { zip::ZipArchive::unsafe_new_with_metadata(file, metadata.clone()) } + /// }}, + /// |archive, i| { + /// let mut file = archive.by_index(i).unwrap(); + /// file.enclosed_name() + /// } + /// ) + /// .filter_map(|name| name) + /// .collect::>(); + /// ``` + pub unsafe fn unsafe_new_with_metadata(reader: R, metadata: Arc) -> Self { + Self { + reader, + shared: metadata, + } + } + + /// Read a ZIP archive providing a read configuration, collecting the files it contains. + /// + /// This uses the central directory record of the ZIP file, and ignores local file headers. + pub fn with_config(config: Config, mut reader: R) -> ZipResult> { + let shared = Self::get_metadata(config, &mut reader)?; + + Ok(ZipArchive { + reader, + shared: shared.into(), + }) + } + + /// Extract a Zip archive into a directory, overwriting files if they + /// already exist. Paths are sanitized with [`ZipFile::enclosed_name`]. Symbolic links are only + /// created and followed if the target is within the destination directory (this is checked + /// conservatively using [`std::fs::canonicalize`]). + /// + /// Extraction is not atomic. If an error is encountered, some of the files + /// may be left on disk. However, on Unix targets, no newly-created directories with part but + /// not all of their contents extracted will be readable, writable or usable as process working + /// directories by any non-root user except you. + /// + /// On Unix and Windows, symbolic links are extracted correctly. On other platforms such as + /// WebAssembly, symbolic links aren't supported, so they're extracted as normal files + /// containing the target path in UTF-8. + pub fn extract>(&mut self, directory: P) -> ZipResult<()> { + self.extract_internal(directory, None:: bool>) + } + + /// Extracts a Zip archive into a directory in the same fashion as + /// [`ZipArchive::extract`], but detects a "root" directory in the archive + /// (a single top-level directory that contains the rest of the archive's + /// entries) and extracts its contents directly. + /// + /// For a sensible default `filter`, you can use [`root_dir_common_filter`]. + /// For a custom `filter`, see [`RootDirFilter`]. + /// + /// See [`ZipArchive::root_dir`] for more information on how the root + /// directory is detected and the meaning of the `filter` parameter. + /// + /// ## Example + /// + /// Imagine a Zip archive with the following structure: + /// + /// ```text + /// root/file1.txt + /// root/file2.txt + /// root/sub/file3.txt + /// root/sub/subsub/file4.txt + /// ``` + /// + /// If the archive is extracted to `foo` using [`ZipArchive::extract`], + /// the resulting directory structure will be: + /// + /// ```text + /// foo/root/file1.txt + /// foo/root/file2.txt + /// foo/root/sub/file3.txt + /// foo/root/sub/subsub/file4.txt + /// ``` + /// + /// If the archive is extracted to `foo` using + /// [`ZipArchive::extract_unwrapped_root_dir`], the resulting directory + /// structure will be: + /// + /// ```text + /// foo/file1.txt + /// foo/file2.txt + /// foo/sub/file3.txt + /// foo/sub/subsub/file4.txt + /// ``` + /// + /// ## Example - No Root Directory + /// + /// Imagine a Zip archive with the following structure: + /// + /// ```text + /// root/file1.txt + /// root/file2.txt + /// root/sub/file3.txt + /// root/sub/subsub/file4.txt + /// other/file5.txt + /// ``` + /// + /// Due to the presence of the `other` directory, + /// [`ZipArchive::extract_unwrapped_root_dir`] will extract this in the same + /// fashion as [`ZipArchive::extract`] as there is now no "root directory." + pub fn extract_unwrapped_root_dir>( + &mut self, + directory: P, + root_dir_filter: impl RootDirFilter, + ) -> ZipResult<()> { + self.extract_internal(directory, Some(root_dir_filter)) + } + + fn extract_internal>( + &mut self, + directory: P, + root_dir_filter: Option, + ) -> ZipResult<()> { + use std::fs; + + fs::create_dir_all(&directory)?; + let directory = directory.as_ref().canonicalize()?; + + let root_dir = root_dir_filter + .and_then(|filter| { + self.root_dir(&filter) + .transpose() + .map(|root_dir| root_dir.map(|root_dir| (root_dir, filter))) + }) + .transpose()?; + + // If we have a root dir, simplify the path components to be more + // appropriate for passing to `safe_prepare_path` + let root_dir = root_dir + .as_ref() + .map(|(root_dir, filter)| { + crate::path::simplified_components(root_dir) + .ok_or_else(|| { + // Should be unreachable + debug_assert!(false, "Invalid root dir path"); + + invalid!("Invalid root dir path") + }) + .map(|root_dir| (root_dir, filter)) + }) + .transpose()?; + + #[cfg(unix)] + let mut files_by_unix_mode = UnixFileModes::default(); + + for i in 0..self.len() { + let mut file = self.by_index(i)?; + + let mut outpath = directory.clone(); + /* TODO: the control flow of this method call and subsequent expectations about the + * values in this loop is extremely difficult to follow. It also appears to + * perform a nested loop upon extracting every single file entry? Why does it + * accept two arguments that point to the same directory path, one mutable? */ + file.safe_prepare_path(directory.as_ref(), &mut outpath, root_dir.as_ref())?; + + #[cfg(any(unix, windows))] + if file.is_symlink() { + let declared_len = file.size(); + if declared_len > MAX_SYMLINK_TARGET_LEN { + return Err(invalid!( + "symlink target declares {} bytes, more than the {} byte limit", + declared_len, + MAX_SYMLINK_TARGET_LEN + )); + } + let mut target = Vec::with_capacity(declared_len as usize); + file.read_to_end(&mut target)?; + drop(file); + make_symlink(&outpath, &target, &self.shared.files)?; + continue; + } else if file.is_dir() { + crate::read::make_writable_dir_all(&outpath)?; + continue; + } + let mut outfile = fs::File::create(&outpath)?; + io::copy(&mut file, &mut outfile)?; + + // Check for real permissions, which we'll set in a second pass. + #[cfg(unix)] + if let Some(mode) = file.unix_mode() { + files_by_unix_mode.add_mode(outpath, mode); + } + + // Set original timestamp. + #[cfg(feature = "chrono")] + if let Some(last_modified) = file.last_modified() { + if let Some(t) = datetime_to_systemtime(&last_modified) { + outfile.set_modified(t)?; + } + } + } + + // Ensure we update children's permissions before making a parent unwritable. + #[cfg(unix)] + for (path, perms) in files_by_unix_mode.all_perms_with_children_first() { + std::fs::set_permissions(path, perms)?; + } + + Ok(()) + } + + /// Number of files contained in this zip. + pub fn len(&self) -> usize { + self.shared.files.len() + } + + /// Get the starting offset of the zip central directory. + pub fn central_directory_start(&self) -> u64 { + self.shared.dir_start + } + + /// Whether this zip archive contains no files + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Get the offset from the beginning of the underlying reader that this zip begins at, in bytes. + /// + /// Normally this value is zero, but if the zip has arbitrary data prepended to it, then this value will be the size + /// of that prepended data. + pub fn offset(&self) -> u64 { + self.shared.offset + } + + /// Get the comment of the zip archive. + pub fn comment(&self) -> &[u8] { + &self.shared.comment + } + + /// Get the ZIP64 comment of the zip archive, if it is ZIP64. + pub fn zip64_comment(&self) -> Option<&[u8]> { + self.shared.zip64_comment.as_deref() + } + + /// Returns an iterator over all the file and directory names in this archive. + pub fn file_names(&self) -> impl Iterator { + self.shared.files.keys().map(|s| s.as_ref()) + } + + /// Returns Ok(true) if any compressed data in this archive belongs to more than one file. This + /// doesn't make the archive invalid, but some programs will refuse to decompress it because the + /// copies would take up space independently in the destination. + pub fn has_overlapping_files(&mut self) -> ZipResult { + let mut ranges = Vec::>::with_capacity(self.shared.files.len()); + for file in self.shared.files.values() { + if file.compressed_size == 0 { + continue; + } + let start = file.data_start(&mut self.reader)?; + let end = start + file.compressed_size; + if ranges + .iter() + .any(|range| range.start <= end && start <= range.end) + { + return Ok(true); + } + ranges.push(start..end); + } + Ok(false) + } + + /// Search for a file entry by name, decrypt with given password + /// + /// # Warning + /// + /// The implementation of the cryptographic algorithms has not + /// gone through a correctness review, and you should assume it is insecure: + /// passwords used with this API may be compromised. + /// + /// This function sometimes accepts wrong password. This is because the ZIP spec only allows us + /// to check for a 1/256 chance that the password is correct. + /// There are many passwords out there that will also pass the validity checks + /// we are able to perform. This is a weakness of the ZipCrypto algorithm, + /// due to its fairly primitive approach to cryptography. + pub fn by_name_decrypt(&mut self, name: &str, password: &[u8]) -> ZipResult> { + self.by_name_with_optional_password(name, Some(password)) + } + + /// Search for a file entry by name + pub fn by_name(&mut self, name: &str) -> ZipResult> { + self.by_name_with_optional_password(name, None) + } + + /// Get the index of a file entry by name, if it's present. + #[inline(always)] + pub fn index_for_name(&self, name: &str) -> Option { + self.shared.files.get_index_of(name) + } + + /// Search for a file entry by path, decrypt with given password + /// + /// # Warning + /// + /// The implementation of the cryptographic algorithms has not + /// gone through a correctness review, and you should assume it is insecure: + /// passwords used with this API may be compromised. + /// + /// This function sometimes accepts wrong password. This is because the ZIP spec only allows us + /// to check for a 1/256 chance that the password is correct. + /// There are many passwords out there that will also pass the validity checks + /// we are able to perform. This is a weakness of the ZipCrypto algorithm, + /// due to its fairly primitive approach to cryptography. + pub fn by_path_decrypt>( + &mut self, + path: T, + password: &[u8], + ) -> ZipResult> { + self.index_for_path(path) + .ok_or(ZipError::FileNotFound) + .and_then(|index| { + self.by_index_with_options(index, ZipReadOptions::new().password(Some(password))) + }) + } + + /// Search for a file entry by path + pub fn by_path>(&mut self, path: T) -> ZipResult> { + self.index_for_path(path) + .ok_or(ZipError::FileNotFound) + .and_then(|index| self.by_index_with_options(index, ZipReadOptions::new())) + } + + /// Get the index of a file entry by path, if it's present. + #[inline(always)] + pub fn index_for_path>(&self, path: T) -> Option { + self.index_for_name(&path_to_string(path)) + } + + /// Get the name of a file entry, if it's present. + #[inline(always)] + pub fn name_for_index(&self, index: usize) -> Option<&str> { + self.shared + .files + .get_index(index) + .map(|(name, _)| name.as_ref()) + } + + /// Search for a file entry by name and return a seekable object. + pub fn by_name_seek(&mut self, name: &str) -> ZipResult> { + self.by_index_seek(self.index_for_name(name).ok_or(ZipError::FileNotFound)?) + } + + /// Search for a file entry by index and return a seekable object. + pub fn by_index_seek(&mut self, index: usize) -> ZipResult> { + let reader = &mut self.reader; + self.shared + .files + .get_index(index) + .ok_or(ZipError::FileNotFound) + .and_then(move |(_, data)| { + let seek_reader = match data.compression_method { + CompressionMethod::Stored => { + ZipFileSeekReader::Raw(find_content_seek(data, reader)?) + } + _ => { + return Err(ZipError::UnsupportedArchive( + "Seekable compressed files are not yet supported", + )) + } + }; + Ok(ZipFileSeek { + reader: seek_reader, + data: Cow::Borrowed(data), + }) + }) + } + + fn by_name_with_optional_password<'a>( + &'a mut self, + name: &str, + password: Option<&[u8]>, + ) -> ZipResult> { + let Some(index) = self.shared.files.get_index_of(name) else { + return Err(ZipError::FileNotFound); + }; + self.by_index_with_options(index, ZipReadOptions::new().password(password)) + } + + /// Get a contained file by index, decrypt with given password + /// + /// # Warning + /// + /// The implementation of the cryptographic algorithms has not + /// gone through a correctness review, and you should assume it is insecure: + /// passwords used with this API may be compromised. + /// + /// This function sometimes accepts wrong password. This is because the ZIP spec only allows us + /// to check for a 1/256 chance that the password is correct. + /// There are many passwords out there that will also pass the validity checks + /// we are able to perform. This is a weakness of the ZipCrypto algorithm, + /// due to its fairly primitive approach to cryptography. + pub fn by_index_decrypt( + &mut self, + file_number: usize, + password: &[u8], + ) -> ZipResult> { + self.by_index_with_options(file_number, ZipReadOptions::new().password(Some(password))) + } + + /// Get a contained file by index + pub fn by_index(&mut self, file_number: usize) -> ZipResult> { + self.by_index_with_options(file_number, ZipReadOptions::new()) + } + + /// Get a contained file by index without decompressing it + pub fn by_index_raw(&mut self, file_number: usize) -> ZipResult> { + let reader = &mut self.reader; + let (_, data) = self + .shared + .files + .get_index(file_number) + .ok_or(ZipError::FileNotFound)?; + Ok(ZipFile { + reader: ZipFileReader::Raw(find_content(data, reader)?), + data: Cow::Borrowed(data), + }) + } + + /// Get a contained file by index with options. + pub fn by_index_with_options( + &mut self, + file_number: usize, + mut options: ZipReadOptions<'_>, + ) -> ZipResult> { + let (_, data) = self + .shared + .files + .get_index(file_number) + .ok_or(ZipError::FileNotFound)?; + + if options.ignore_encryption_flag { + // Always use no password when we're ignoring the encryption flag. + options.password = None; + } else { + // Require and use the password only if the file is encrypted. + match (options.password, data.encrypted) { + (None, true) => { + return Err(ZipError::UnsupportedArchive(ZipError::PASSWORD_REQUIRED)) + } + // Password supplied, but none needed! Discard. + (Some(_), false) => options.password = None, + _ => {} + } + } + let limit_reader = find_content(data, &mut self.reader)?; + + let crypto_reader = + make_crypto_reader(data, limit_reader, options.password, data.aes_mode)?; + + Ok(ZipFile { + data: Cow::Borrowed(data), + reader: make_reader( + data.compression_method, + data.uncompressed_size, + data.crc32, + crypto_reader, + #[cfg(feature = "legacy-zip")] + data.flags, + )?, + }) + } + + /// Find the "root directory" of an archive if it exists, filtering out + /// irrelevant entries when searching. + /// + /// Our definition of a "root directory" is a single top-level directory + /// that contains the rest of the archive's entries. This is useful for + /// extracting archives that contain a single top-level directory that + /// you want to "unwrap" and extract directly. + /// + /// For a sensible default filter, you can use [`root_dir_common_filter`]. + /// For a custom filter, see [`RootDirFilter`]. + pub fn root_dir(&self, filter: impl RootDirFilter) -> ZipResult> { + let mut root_dir: Option = None; + + for i in 0..self.len() { + let (_, file) = self + .shared + .files + .get_index(i) + .ok_or(ZipError::FileNotFound)?; + + let path = match file.enclosed_name() { + Some(path) => path, + None => return Ok(None), + }; + + if !filter(&path) { + continue; + } + + macro_rules! replace_root_dir { + ($path:ident) => { + match &mut root_dir { + Some(root_dir) => { + if *root_dir != $path { + // We've found multiple root directories, + // abort. + return Ok(None); + } else { + continue; + } + } + + None => { + root_dir = Some($path.into()); + continue; + } + } + }; + } + + // If this entry is located at the root of the archive... + if path.components().count() == 1 { + if file.is_dir() { + // If it's a directory, it could be the root directory. + replace_root_dir!(path); + } else { + // If it's anything else, this archive does not have a + // root directory. + return Ok(None); + } + } + + // Find the root directory for this entry. + let mut path = path.as_path(); + while let Some(parent) = path.parent().filter(|path| *path != Path::new("")) { + path = parent; + } + + replace_root_dir!(path); + } + + Ok(root_dir) + } + + /// Unwrap and return the inner reader object + /// + /// The position of the reader is undefined. + pub fn into_inner(self) -> R { + self.reader + } +} + +/// Holds the AES information of a file in the zip archive +#[derive(Debug)] +#[cfg(feature = "aes-crypto")] +pub struct AesInfo { + /// The AES encryption mode + pub aes_mode: AesMode, + /// The verification key + pub verification_value: [u8; PWD_VERIFY_LENGTH], + /// The salt + pub salt: Vec, +} + +const fn unsupported_zip_error(detail: &'static str) -> ZipResult { + Err(ZipError::UnsupportedArchive(detail)) +} + +/// Parse a central directory entry to collect the information for the file. +pub(crate) fn central_header_to_zip_file( + reader: &mut R, + central_directory: &CentralDirectoryInfo, +) -> ZipResult { + let central_header_start = reader.stream_position()?; + + // Parse central header + let block = ZipCentralEntryBlock::parse(reader)?; + + let file = central_header_to_zip_file_inner( + reader, + central_directory.archive_offset, + central_header_start, + block, + )?; + + let central_header_end = reader.stream_position()?; + + reader.seek(SeekFrom::Start(central_header_end))?; + Ok(file) +} + +#[inline] +fn read_variable_length_byte_field(reader: &mut R, len: usize) -> ZipResult> { + let mut data = vec![0; len].into_boxed_slice(); + if let Err(e) = reader.read_exact(&mut data) { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!( + "Variable-length field extends beyond file boundary" + )); + } + return Err(e.into()); + } + Ok(data) +} + +/// Parse a central directory entry to collect the information for the file. +fn central_header_to_zip_file_inner( + reader: &mut R, + archive_offset: u64, + central_header_start: u64, + block: ZipCentralEntryBlock, +) -> ZipResult { + let ZipCentralEntryBlock { + // magic, + version_made_by, + // version_to_extract, + flags, + compression_method, + last_mod_time, + last_mod_date, + crc32, + compressed_size, + uncompressed_size, + file_name_length, + extra_field_length, + file_comment_length, + // disk_number, + // internal_file_attributes, + external_file_attributes, + offset, + .. + } = block; + + let encrypted = flags & 1 == 1; + let is_utf8 = flags & (1 << 11) != 0; + let using_data_descriptor = flags & (1 << 3) != 0; + + let file_name_raw = read_variable_length_byte_field(reader, file_name_length as usize)?; + let extra_field = read_variable_length_byte_field(reader, extra_field_length as usize)?; + let file_comment_raw = read_variable_length_byte_field(reader, file_comment_length as usize)?; + let file_name: Box = match is_utf8 { + true => String::from_utf8_lossy(&file_name_raw).into(), + false => file_name_raw.clone().from_cp437(), + }; + let file_comment: Box = match is_utf8 { + true => String::from_utf8_lossy(&file_comment_raw).into(), + false => file_comment_raw.from_cp437(), + }; + + // Construct the result + let mut result = ZipFileData { + system: System::from((version_made_by >> 8) as u8), + /* NB: this strips the top 8 bits! */ + version_made_by: version_made_by as u8, + encrypted, + using_data_descriptor, + is_utf8, + compression_method: CompressionMethod::parse_from_u16(compression_method), + compression_level: None, + last_modified_time: DateTime::try_from_msdos(last_mod_date, last_mod_time).ok(), + crc32, + compressed_size: compressed_size.into(), + uncompressed_size: uncompressed_size.into(), + flags, + file_name, + file_name_raw, + extra_field: Some(Arc::new(extra_field.to_vec())), + central_extra_field: None, + file_comment, + header_start: offset.into(), + extra_data_start: None, + central_header_start, + data_start: OnceLock::new(), + external_attributes: external_file_attributes, + large_file: false, + aes_mode: None, + aes_extra_data_start: 0, + extra_fields: Vec::new(), + }; + parse_extra_field(&mut result)?; + + let aes_enabled = result.compression_method == CompressionMethod::AES; + if aes_enabled && result.aes_mode.is_none() { + return Err(invalid!("AES encryption without AES extra data field")); + } + + // Account for shifted zip offsets. + result.header_start = result + .header_start + .checked_add(archive_offset) + .ok_or(invalid!("Archive header is too large"))?; + + Ok(result) +} + +pub(crate) fn parse_extra_field(file: &mut ZipFileData) -> ZipResult<()> { + let mut extra_field = file.extra_field.clone(); + let mut central_extra_field = file.central_extra_field.clone(); + for field_group in [&mut extra_field, &mut central_extra_field] { + let Some(extra_field) = field_group else { + continue; + }; + let mut modified = false; + let mut processed_extra_field = vec![]; + let len = extra_field.len(); + let mut reader = io::Cursor::new(&**extra_field); + + let mut position = reader.position(); + while position < len as u64 { + let old_position = position; + let remove = parse_single_extra_field(file, &mut reader, position, false)?; + position = reader.position(); + if remove { + modified = true; + } else { + let field_len = (position - old_position) as usize; + let write_start = processed_extra_field.len(); + reader.seek(SeekFrom::Start(old_position))?; + processed_extra_field.extend_from_slice(&vec![0u8; field_len]); + if let Err(e) = reader + .read_exact(&mut processed_extra_field[write_start..(write_start + field_len)]) + { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!("Extra field content exceeds declared length")); + } + return Err(e.into()); + } + } + } + if modified { + *field_group = Some(Arc::new(processed_extra_field)); + } + } + file.extra_field = extra_field; + file.central_extra_field = central_extra_field; + Ok(()) +} + +pub(crate) fn parse_single_extra_field( + file: &mut ZipFileData, + reader: &mut R, + bytes_already_read: u64, + disallow_zip64: bool, +) -> ZipResult { + let kind = match reader.read_u16_le() { + Ok(kind) => kind, + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(false), + Err(e) => return Err(e.into()), + }; + let len = match reader.read_u16_le() { + Ok(len) => len, + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => { + return Err(invalid!("Extra field header truncated")) + } + Err(e) => return Err(e.into()), + }; + match kind { + // Zip64 extended information extra field + 0x0001 => { + if disallow_zip64 { + return Err(invalid!("Can't write a custom field using the ZIP64 ID")); + } + file.large_file = true; + let mut consumed_len = 0; + if len >= 24 || file.uncompressed_size == spec::ZIP64_BYTES_THR { + file.uncompressed_size = match reader.read_u64_le() { + Ok(v) => v, + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => { + return Err(invalid!("ZIP64 extra field truncated")) + } + Err(e) => return Err(e.into()), + }; + consumed_len += size_of::(); + } + if len >= 24 || file.compressed_size == spec::ZIP64_BYTES_THR { + file.compressed_size = match reader.read_u64_le() { + Ok(v) => v, + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => { + return Err(invalid!("ZIP64 extra field truncated")) + } + Err(e) => return Err(e.into()), + }; + consumed_len += size_of::(); + } + if len >= 24 || file.header_start == spec::ZIP64_BYTES_THR { + file.header_start = match reader.read_u64_le() { + Ok(v) => v, + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => { + return Err(invalid!("ZIP64 extra field truncated")) + } + Err(e) => return Err(e.into()), + }; + consumed_len += size_of::(); + } + let Some(leftover_len) = (len as usize).checked_sub(consumed_len) else { + return Err(invalid!("ZIP64 extra-data field is the wrong length")); + }; + if let Err(e) = reader.read_exact(&mut vec![0u8; leftover_len]) { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!("ZIP64 extra field truncated")); + } + return Err(e.into()); + } + return Ok(true); + } + 0x000a => { + // NTFS extra field + file.extra_fields + .push(ExtraField::Ntfs(Ntfs::try_from_reader(reader, len)?)); + } + 0x9901 => { + // AES + if len != 7 { + return Err(ZipError::UnsupportedArchive( + "AES extra data field has an unsupported length", + )); + } + let vendor_version = reader.read_u16_le()?; + let vendor_id = reader.read_u16_le()?; + let mut out = [0u8]; + if let Err(e) = reader.read_exact(&mut out) { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!("AES extra field truncated")); + } + return Err(e.into()); + } + let aes_mode = out[0]; + let compression_method = CompressionMethod::parse_from_u16(reader.read_u16_le()?); + + if vendor_id != 0x4541 { + return Err(invalid!("Invalid AES vendor")); + } + let vendor_version = match vendor_version { + 0x0001 => AesVendorVersion::Ae1, + 0x0002 => AesVendorVersion::Ae2, + _ => return Err(invalid!("Invalid AES vendor version")), + }; + match aes_mode { + 0x01 => file.aes_mode = Some((AesMode::Aes128, vendor_version, compression_method)), + 0x02 => file.aes_mode = Some((AesMode::Aes192, vendor_version, compression_method)), + 0x03 => file.aes_mode = Some((AesMode::Aes256, vendor_version, compression_method)), + _ => return Err(invalid!("Invalid AES encryption strength")), + }; + file.compression_method = compression_method; + file.aes_extra_data_start = bytes_already_read; + } + 0x5455 => { + // extended timestamp + // https://libzip.org/specifications/extrafld.txt + + file.extra_fields.push(ExtraField::ExtendedTimestamp( + ExtendedTimestamp::try_from_reader(reader, len)?, + )); + } + 0x6375 => { + // Info-ZIP Unicode Comment Extra Field + // APPNOTE 4.6.8 and https://libzip.org/specifications/extrafld.txt + file.file_comment = String::from_utf8( + UnicodeExtraField::try_from_reader(reader, len)? + .unwrap_valid(file.file_comment.as_bytes())? + .into_vec(), + )? + .into(); + } + 0x7075 => { + // Info-ZIP Unicode Path Extra Field + // APPNOTE 4.6.9 and https://libzip.org/specifications/extrafld.txt + file.file_name_raw = UnicodeExtraField::try_from_reader(reader, len)? + .unwrap_valid(&file.file_name_raw)?; + file.file_name = + String::from_utf8(file.file_name_raw.clone().into_vec())?.into_boxed_str(); + file.is_utf8 = true; + } + _ => { + if let Err(e) = reader.read_exact(&mut vec![0u8; len as usize]) { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!("Extra field content truncated")); + } + return Err(e.into()); + } + // Other fields are ignored + } + } + Ok(false) +} + +/// A trait for exposing file metadata inside the zip. +pub trait HasZipMetadata { + /// Get the file metadata + fn get_metadata(&self) -> &ZipFileData; +} + +/// Options for reading a file from an archive. +#[derive(Default)] +pub struct ZipReadOptions<'a> { + /// The password to use when decrypting the file. This is ignored if not required. + password: Option<&'a [u8]>, + + /// Ignore the value of the encryption flag and proceed as if the file were plaintext. + ignore_encryption_flag: bool, +} + +impl<'a> ZipReadOptions<'a> { + /// Create a new set of options with the default values. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Set the password, if any, to use. Return for chaining. + #[must_use] + pub fn password(mut self, password: Option<&'a [u8]>) -> Self { + self.password = password; + self + } + + /// Set the ignore encryption flag. Return for chaining. + #[must_use] + pub fn ignore_encryption_flag(mut self, ignore: bool) -> Self { + self.ignore_encryption_flag = ignore; + self + } +} + +/// Methods for retrieving information on zip files +impl<'a, R: Read + ?Sized> ZipFile<'a, R> { + pub(crate) fn take_raw_reader(&mut self) -> io::Result> { + replace(&mut self.reader, ZipFileReader::NoReader).into_inner() + } + + /// Get the version of the file + pub fn version_made_by(&self) -> (u8, u8) { + ( + self.get_metadata().version_made_by / 10, + self.get_metadata().version_made_by % 10, + ) + } + + /// Get the name of the file + /// + /// # Warnings + /// + /// It is dangerous to use this name directly when extracting an archive. + /// It may contain an absolute path (`/etc/shadow`), or break out of the + /// current directory (`../runtime`). Carelessly writing to these paths + /// allows an attacker to craft a ZIP archive that will overwrite critical + /// files. + /// + /// You can use the [`ZipFile::enclosed_name`] method to validate the name + /// as a safe path. + pub fn name(&self) -> &str { + &self.get_metadata().file_name + } + + /// Get the name of the file, in the raw (internal) byte representation. + /// + /// The encoding of this data is currently undefined. + pub fn name_raw(&self) -> &[u8] { + &self.get_metadata().file_name_raw + } + + /// Get the name of the file in a sanitized form. It truncates the name to the first NULL byte, + /// removes a leading '/' and removes '..' parts. + #[deprecated( + since = "0.5.7", + note = "by stripping `..`s from the path, the meaning of paths can change. + `mangled_name` can be used if this behaviour is desirable" + )] + pub fn sanitized_name(&self) -> PathBuf { + self.mangled_name() + } + + /// Rewrite the path, ignoring any path components with special meaning. + /// + /// - Absolute paths are made relative + /// - [`ParentDir`]s are ignored + /// - Truncates the filename at a NULL byte + /// + /// This is appropriate if you need to be able to extract *something* from + /// any archive, but will easily misrepresent trivial paths like + /// `foo/../bar` as `foo/bar` (instead of `bar`). Because of this, + /// [`ZipFile::enclosed_name`] is the better option in most scenarios. + /// + /// [`ParentDir`]: `Component::ParentDir` + pub fn mangled_name(&self) -> PathBuf { + self.get_metadata().file_name_sanitized() + } + + /// Ensure the file path is safe to use as a [`Path`]. + /// + /// - It can't contain NULL bytes + /// - It can't resolve to a path outside the current directory + /// > `foo/../bar` is fine, `foo/../../bar` is not. + /// - It can't be an absolute path + /// + /// This will read well-formed ZIP files correctly, and is resistant + /// to path-based exploits. It is recommended over + /// [`ZipFile::mangled_name`]. + pub fn enclosed_name(&self) -> Option { + self.get_metadata().enclosed_name() + } + + pub(crate) fn simplified_components(&self) -> Option> { + self.get_metadata().simplified_components() + } + + /// Prepare the path for extraction by creating necessary missing directories and checking for symlinks to be contained within the base path. + /// + /// `base_path` parameter is assumed to be canonicalized. + pub(crate) fn safe_prepare_path( + &self, + base_path: &Path, + outpath: &mut PathBuf, + root_dir: Option<&(Vec<&OsStr>, impl RootDirFilter)>, + ) -> ZipResult<()> { + let components = self + .simplified_components() + .ok_or(invalid!("Invalid file path"))?; + + let components = match root_dir { + Some((root_dir, filter)) => match components.strip_prefix(&**root_dir) { + Some(components) => components, + + // In this case, we expect that the file was not in the root + // directory, but was filtered out when searching for the + // root directory. + None => { + // We could technically find ourselves at this code + // path if the user provides an unstable or + // non-deterministic `filter` function. + // + // If debug assertions are on, we should panic here. + // Otherwise, the safest thing to do here is to just + // extract as-is. + debug_assert!( + !filter(&PathBuf::from_iter(components.iter())), + "Root directory filter should not match at this point" + ); + + // Extract as-is. + &components[..] + } + }, + + None => &components[..], + }; + + let components_len = components.len(); + + for (is_last, component) in components + .iter() + .copied() + .enumerate() + .map(|(i, c)| (i == components_len - 1, c)) + { + // we can skip the target directory itself because the base path is assumed to be "trusted" (if the user say extract to a symlink we can follow it) + outpath.push(component); + + // check if the path is a symlink, the target must be _inherently_ within the directory + for limit in (0..5u8).rev() { + let meta = match std::fs::symlink_metadata(&outpath) { + Ok(meta) => meta, + Err(e) if e.kind() == io::ErrorKind::NotFound => { + if !is_last { + crate::read::make_writable_dir_all(&outpath)?; + } + break; + } + Err(e) => return Err(e.into()), + }; + + if !meta.is_symlink() { + break; + } + + if limit == 0 { + return Err(invalid!("Extraction followed a symlink too deep")); + } + + // note that we cannot accept links that do not inherently resolve to a path inside the directory to prevent: + // - disclosure of unrelated path exists (no check for a path exist and then ../ out) + // - issues with file-system specific path resolution (case sensitivity, etc) + let target = std::fs::read_link(&outpath)?; + + if !crate::path::simplified_components(&target) + .ok_or(invalid!("Invalid symlink target path"))? + .starts_with( + &crate::path::simplified_components(base_path) + .ok_or(invalid!("Invalid base path"))?, + ) + { + let is_absolute_enclosed = base_path + .components() + .map(Some) + .chain(std::iter::once(None)) + .zip(target.components().map(Some).chain(std::iter::repeat(None))) + .all(|(a, b)| match (a, b) { + // both components are normal + (Some(Component::Normal(a)), Some(Component::Normal(b))) => a == b, + // both components consumed fully + (None, None) => true, + // target consumed fully but base path is not + (Some(_), None) => false, + // base path consumed fully but target is not (and normal) + (None, Some(Component::CurDir | Component::Normal(_))) => true, + _ => false, + }); + + if !is_absolute_enclosed { + return Err(invalid!("Symlink is not inherently safe")); + } + } + + outpath.push(target); + } + } + Ok(()) + } + + /// Get the comment of the file + pub fn comment(&self) -> &str { + &self.get_metadata().file_comment + } + + /// Get the compression method used to store the file + pub fn compression(&self) -> CompressionMethod { + self.get_metadata().compression_method + } + + /// Get if the files is encrypted or not + pub fn encrypted(&self) -> bool { + self.data.encrypted + } + + /// Get the size of the file, in bytes, in the archive + pub fn compressed_size(&self) -> u64 { + self.get_metadata().compressed_size + } + + /// Get the size of the file, in bytes, when uncompressed + pub fn size(&self) -> u64 { + self.get_metadata().uncompressed_size + } + + /// Get the time the file was last modified + pub fn last_modified(&self) -> Option { + self.data.last_modified_time + } + /// Returns whether the file is actually a directory + pub fn is_dir(&self) -> bool { + is_dir(self.name()) + } + + /// Returns whether the file is actually a symbolic link + pub fn is_symlink(&self) -> bool { + self.unix_mode() + .is_some_and(|mode| mode & S_IFLNK == S_IFLNK) + } + + /// Returns whether the file is a normal file (i.e. not a directory or symlink) + pub fn is_file(&self) -> bool { + !self.is_dir() && !self.is_symlink() + } + + /// Get unix mode for the file + pub fn unix_mode(&self) -> Option { + self.get_metadata().unix_mode() + } + + /// Get the CRC32 hash of the original file + pub fn crc32(&self) -> u32 { + self.get_metadata().crc32 + } + + /// Get the extra data of the zip header for this file + pub fn extra_data(&self) -> Option<&[u8]> { + self.get_metadata() + .extra_field + .as_ref() + .map(|v| v.deref().deref()) + } + + /// Get the starting offset of the data of the compressed file + pub fn data_start(&self) -> u64 { + *self.data.data_start.get().unwrap() + } + + /// Get the starting offset of the zip header for this file + pub fn header_start(&self) -> u64 { + self.get_metadata().header_start + } + /// Get the starting offset of the zip header in the central directory for this file + pub fn central_header_start(&self) -> u64 { + self.get_metadata().central_header_start + } + + /// Get the [`SimpleFileOptions`] that would be used to write this file to + /// a new zip archive. + pub fn options(&self) -> SimpleFileOptions { + let mut options = SimpleFileOptions::default() + .large_file(self.compressed_size().max(self.size()) > ZIP64_BYTES_THR) + .compression_method(self.compression()) + .unix_permissions(self.unix_mode().unwrap_or(0o644) | S_IFREG) + .last_modified_time( + self.last_modified() + .filter(|m| m.is_valid()) + .unwrap_or_else(DateTime::default_for_write), + ); + + options.normalize(); + #[cfg(feature = "aes-crypto")] + if let Some(aes) = self.get_metadata().aes_mode { + // Preserve AES metadata in options for downstream writers. + // This is metadata-only and does not trigger encryption. + options.aes_mode = Some(aes); + } + options + } +} + +/// Methods for retrieving information on zip files +impl ZipFile<'_, R> { + /// iterate through all extra fields + pub fn extra_data_fields(&self) -> impl Iterator { + self.data.extra_fields.iter() + } +} + +impl HasZipMetadata for ZipFile<'_, R> { + fn get_metadata(&self) -> &ZipFileData { + self.data.as_ref() + } +} + +impl Read for ZipFile<'_, R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.reader.read(buf) + } + + fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> { + self.reader.read_exact(buf) + } + + fn read_to_end(&mut self, buf: &mut Vec) -> io::Result { + self.reader.read_to_end(buf) + } + + fn read_to_string(&mut self, buf: &mut String) -> io::Result { + self.reader.read_to_string(buf) + } +} + +impl Read for ZipFileSeek<'_, R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match &mut self.reader { + ZipFileSeekReader::Raw(r) => r.read(buf), + } + } +} + +impl Seek for ZipFileSeek<'_, R> { + fn seek(&mut self, pos: SeekFrom) -> io::Result { + match &mut self.reader { + ZipFileSeekReader::Raw(r) => r.seek(pos), + } + } +} + +impl HasZipMetadata for ZipFileSeek<'_, R> { + fn get_metadata(&self) -> &ZipFileData { + self.data.as_ref() + } +} + +impl Drop for ZipFile<'_, R> { + fn drop(&mut self) { + // self.data is Owned, this reader is constructed by a streaming reader. + // In this case, we want to exhaust the reader so that the next file is accessible. + if let Cow::Owned(_) = self.data { + // Get the inner `Take` reader so all decryption, decompression and CRC calculation is skipped. + if let Ok(mut inner) = self.take_raw_reader() { + let _ = copy(&mut inner, &mut sink()); + } + } + } +} + +/// Read ZipFile structures from a non-seekable reader. +/// +/// This is an alternative method to read a zip file. If possible, use the ZipArchive functions +/// as some information will be missing when reading this manner. +/// +/// Reads a file header from the start of the stream. Will return `Ok(Some(..))` if a file is +/// present at the start of the stream. Returns `Ok(None)` if the start of the central directory +/// is encountered. No more files should be read after this. +/// +/// The Drop implementation of ZipFile ensures that the reader will be correctly positioned after +/// the structure is done. +/// +/// Missing fields are: +/// * `comment`: set to an empty string +/// * `data_start`: set to 0 +/// * `external_attributes`: `unix_mode()`: will return None +pub fn read_zipfile_from_stream(reader: &mut R) -> ZipResult>> { + // We can't use the typical ::parse() method, as we follow separate code paths depending on the + // "magic" value (since the magic value will be from the central directory header if we've + // finished iterating over all the actual files). + /* TODO: smallvec? */ + + let mut block = ZipLocalEntryBlock::zeroed(); + reader.read_exact(block.as_bytes_mut())?; + + match block.magic().from_le() { + spec::Magic::LOCAL_FILE_HEADER_SIGNATURE => (), + spec::Magic::CENTRAL_DIRECTORY_HEADER_SIGNATURE => return Ok(None), + _ => return Err(ZipLocalEntryBlock::WRONG_MAGIC_ERROR), + } + + let block = block.from_le(); + + let mut result = ZipFileData::from_local_block(block, reader)?; + + match parse_extra_field(&mut result) { + Ok(..) | Err(ZipError::Io(..)) => {} + Err(e) => return Err(e), + } + + let limit_reader = reader.take(result.compressed_size); + let crypto_reader = make_crypto_reader(&result, limit_reader, None, None)?; + let ZipFileData { + crc32, + uncompressed_size, + compression_method, + #[cfg(feature = "legacy-zip")] + flags, + .. + } = result; + + Ok(Some(ZipFile { + data: Cow::Owned(result), + reader: make_reader( + compression_method, + uncompressed_size, + crc32, + crypto_reader, + #[cfg(feature = "legacy-zip")] + flags, + )?, + })) +} + +/// A filter that determines whether an entry should be ignored when searching +/// for the root directory of a Zip archive. +/// +/// Returns `true` if the entry should be considered, and `false` if it should +/// be ignored. +/// +/// See [`root_dir_common_filter`] for a sensible default filter. +pub trait RootDirFilter: Fn(&Path) -> bool {} +impl bool> RootDirFilter for F {} + +/// Common filters when finding the root directory of a Zip archive. +/// +/// This filter is a sensible default for most use cases and filters out common +/// system files that are usually irrelevant to the contents of the archive. +/// +/// Currently, the filter ignores: +/// - `/__MACOSX/` +/// - `/.DS_Store` +/// - `/Thumbs.db` +/// +/// **This function is not guaranteed to be stable and may change in future versions.** +/// +/// # Example +/// +/// ```rust +/// # use std::path::Path; +/// assert!(zip::read::root_dir_common_filter(Path::new("foo.txt"))); +/// assert!(!zip::read::root_dir_common_filter(Path::new(".DS_Store"))); +/// assert!(!zip::read::root_dir_common_filter(Path::new("Thumbs.db"))); +/// assert!(!zip::read::root_dir_common_filter(Path::new("__MACOSX"))); +/// assert!(!zip::read::root_dir_common_filter(Path::new("__MACOSX/foo.txt"))); +/// ``` +pub fn root_dir_common_filter(path: &Path) -> bool { + const COMMON_FILTER_ROOT_FILES: &[&str] = &[".DS_Store", "Thumbs.db"]; + + if path.starts_with("__MACOSX") { + return false; + } + + if path.components().count() == 1 + && path.file_name().is_some_and(|file_name| { + COMMON_FILTER_ROOT_FILES + .iter() + .map(OsStr::new) + .any(|cmp| cmp == file_name) + }) + { + return false; + } + + true +} + +#[cfg(feature = "chrono")] +/// Generate a `SystemTime` from a `DateTime`. +fn datetime_to_systemtime(datetime: &DateTime) -> Option { + if let Some(t) = generate_chrono_datetime(datetime) { + let time = chrono::DateTime::::from_naive_utc_and_offset(t, chrono::Utc); + return Some(time.into()); + } + None +} + +#[cfg(feature = "chrono")] +/// Generate a `NaiveDateTime` from a `DateTime`. +fn generate_chrono_datetime(datetime: &DateTime) -> Option { + if let Some(d) = chrono::NaiveDate::from_ymd_opt( + datetime.year().into(), + datetime.month().into(), + datetime.day().into(), + ) { + if let Some(d) = d.and_hms_opt( + datetime.hour().into(), + datetime.minute().into(), + datetime.second().into(), + ) { + return Some(d); + } + } + None +} + +/// Read ZipFile from a non-seekable reader like [read_zipfile_from_stream] does, but assume the +/// given compressed size and don't read any further ahead than that. +pub fn read_zipfile_from_stream_with_compressed_size( + reader: &mut R, + compressed_size: u64, +) -> ZipResult>> { + let mut block = ZipLocalEntryBlock::zeroed(); + reader.read_exact(block.as_bytes_mut())?; + + match block.magic().from_le() { + spec::Magic::LOCAL_FILE_HEADER_SIGNATURE => (), + spec::Magic::CENTRAL_DIRECTORY_HEADER_SIGNATURE => return Ok(None), + _ => return Err(ZipLocalEntryBlock::WRONG_MAGIC_ERROR), + } + + let block = block.from_le(); + + let mut result = ZipFileData::from_local_block(block, reader)?; + result.compressed_size = compressed_size; + + if result.encrypted { + return unsupported_zip_error("Encrypted files are not supported"); + } + + let limit_reader = reader.take(result.compressed_size); + let crypto_reader = make_crypto_reader(&result, limit_reader, None, None)?; + let ZipFileData { + crc32, + compression_method, + uncompressed_size, + #[cfg(feature = "legacy-zip")] + flags, + .. + } = result; + + Ok(Some(ZipFile { + data: Cow::Owned(result), + reader: make_reader( + compression_method, + uncompressed_size, + crc32, + crypto_reader, + #[cfg(feature = "legacy-zip")] + flags, + )?, + })) +} + +#[cfg(test)] +mod test { + use crate::read::ZipReadOptions; + use crate::result::ZipResult; + use crate::types::SimpleFileOptions; + use crate::CompressionMethod::Stored; + use crate::{ZipArchive, ZipWriter}; + use std::io::{Cursor, Read, Write}; + use tempfile::TempDir; + + #[test] + fn invalid_offset() { + use super::ZipArchive; + + let reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/invalid_offset.zip" + ))); + assert!(reader.is_err()); + } + + #[test] + fn invalid_offset2() { + use super::ZipArchive; + + let reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/invalid_offset2.zip" + ))); + assert!(reader.is_err()); + } + + #[test] + fn zip64_with_leading_junk() { + use super::ZipArchive; + + let reader = + ZipArchive::new(Cursor::new(include_bytes!("../tests/data/zip64_demo.zip"))).unwrap(); + assert_eq!(reader.len(), 1); + } + + #[test] + fn zip_contents() { + use super::ZipArchive; + + let mut reader = + ZipArchive::new(Cursor::new(include_bytes!("../tests/data/mimetype.zip"))).unwrap(); + assert_eq!(reader.comment(), b""); + assert_eq!(reader.by_index(0).unwrap().central_header_start(), 77); + } + + #[test] + fn zip_read_streaming() { + use super::read_zipfile_from_stream; + + let mut reader = Cursor::new(include_bytes!("../tests/data/mimetype.zip")); + loop { + if read_zipfile_from_stream(&mut reader).unwrap().is_none() { + break; + } + } + } + + #[test] + fn zip_clone() { + use super::ZipArchive; + use std::io::Read; + + let mut reader1 = + ZipArchive::new(Cursor::new(include_bytes!("../tests/data/mimetype.zip"))).unwrap(); + let mut reader2 = reader1.clone(); + + let mut file1 = reader1.by_index(0).unwrap(); + let mut file2 = reader2.by_index(0).unwrap(); + + let t = file1.last_modified().unwrap(); + assert_eq!( + ( + t.year(), + t.month(), + t.day(), + t.hour(), + t.minute(), + t.second() + ), + (1980, 1, 1, 0, 0, 0) + ); + + let mut buf1 = [0; 5]; + let mut buf2 = [0; 5]; + let mut buf3 = [0; 5]; + let mut buf4 = [0; 5]; + + file1.read_exact(&mut buf1).unwrap(); + file2.read_exact(&mut buf2).unwrap(); + file1.read_exact(&mut buf3).unwrap(); + file2.read_exact(&mut buf4).unwrap(); + + assert_eq!(buf1, buf2); + assert_eq!(buf3, buf4); + assert_ne!(buf1, buf3); + } + + #[test] + fn file_and_dir_predicates() { + use super::ZipArchive; + + let mut zip = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/files_and_dirs.zip" + ))) + .unwrap(); + + for i in 0..zip.len() { + let zip_file = zip.by_index(i).unwrap(); + let full_name = zip_file.enclosed_name().unwrap(); + let file_name = full_name.file_name().unwrap().to_str().unwrap(); + assert!( + (file_name.starts_with("dir") && zip_file.is_dir()) + || (file_name.starts_with("file") && zip_file.is_file()) + ); + } + } + + #[test] + fn zip64_magic_in_filenames() { + let files = vec![ + include_bytes!("../tests/data/zip64_magic_in_filename_1.zip").to_vec(), + include_bytes!("../tests/data/zip64_magic_in_filename_2.zip").to_vec(), + include_bytes!("../tests/data/zip64_magic_in_filename_3.zip").to_vec(), + include_bytes!("../tests/data/zip64_magic_in_filename_4.zip").to_vec(), + include_bytes!("../tests/data/zip64_magic_in_filename_5.zip").to_vec(), + ]; + // Although we don't allow adding files whose names contain the ZIP64 CDB-end or + // CDB-end-locator signatures, we still read them when they aren't genuinely ambiguous. + for file in files { + ZipArchive::new(Cursor::new(file)).unwrap(); + } + } + + /// test case to ensure we don't preemptively over allocate based on the + /// declared number of files in the CDE of an invalid zip when the number of + /// files declared is more than the alleged offset in the CDE + #[test] + fn invalid_cde_number_of_files_allocation_smaller_offset() { + use super::ZipArchive; + + let reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/invalid_cde_number_of_files_allocation_smaller_offset.zip" + ))); + assert!(reader.is_err() || reader.unwrap().is_empty()); + } + + /// test case to ensure we don't preemptively over allocate based on the + /// declared number of files in the CDE of an invalid zip when the number of + /// files declared is less than the alleged offset in the CDE + #[test] + fn invalid_cde_number_of_files_allocation_greater_offset() { + use super::ZipArchive; + + let reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/invalid_cde_number_of_files_allocation_greater_offset.zip" + ))); + assert!(reader.is_err()); + } + + #[cfg(feature = "deflate64")] + #[test] + fn deflate64_index_out_of_bounds() -> std::io::Result<()> { + let mut reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/raw_deflate64_index_out_of_bounds.zip" + )))?; + std::io::copy(&mut reader.by_index(0)?, &mut std::io::sink()).expect_err("Invalid file"); + Ok(()) + } + + #[cfg(feature = "deflate64")] + #[test] + fn deflate64_not_enough_space() { + ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/deflate64_issue_25.zip" + ))) + .expect_err("Invalid file"); + } + + #[cfg(feature = "deflate-flate2")] + #[test] + fn test_read_with_data_descriptor() { + use std::io::Read; + + let mut reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/data_descriptor.zip" + ))) + .unwrap(); + let mut decompressed = [0u8; 16]; + let mut file = reader.by_index(0).unwrap(); + assert_eq!(file.read(&mut decompressed).unwrap(), 12); + } + + #[test] + fn test_is_symlink() -> std::io::Result<()> { + let mut reader = ZipArchive::new(Cursor::new(include_bytes!("../tests/data/symlink.zip")))?; + assert!(reader.by_index(0)?.is_symlink()); + let tempdir = TempDir::with_prefix("test_is_symlink")?; + reader.extract(&tempdir)?; + assert!(tempdir.path().join("bar").is_symlink()); + Ok(()) + } + + #[test] + #[cfg(feature = "deflate-flate2")] + fn test_utf8_extra_field() { + let mut reader = + ZipArchive::new(Cursor::new(include_bytes!("../tests/data/chinese.zip"))).unwrap(); + reader.by_name("七个房间.txt").unwrap(); + } + + #[test] + fn test_utf8() { + let mut reader = + ZipArchive::new(Cursor::new(include_bytes!("../tests/data/linux-7z.zip"))).unwrap(); + reader.by_name("你好.txt").unwrap(); + } + + #[test] + fn test_utf8_2() { + let mut reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/windows-7zip.zip" + ))) + .unwrap(); + reader.by_name("你好.txt").unwrap(); + } + + #[test] + fn test_64k_files() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = SimpleFileOptions { + compression_method: Stored, + ..Default::default() + }; + for i in 0..=u16::MAX { + let file_name = format!("{i}.txt"); + writer.start_file(&*file_name, options)?; + writer.write_all(i.to_string().as_bytes())?; + } + + let mut reader = ZipArchive::new(writer.finish()?)?; + for i in 0..=u16::MAX { + let expected_name = format!("{i}.txt"); + let expected_contents = i.to_string(); + let expected_contents = expected_contents.as_bytes(); + let mut file = reader.by_name(&expected_name)?; + let mut contents = Vec::with_capacity(expected_contents.len()); + file.read_to_end(&mut contents)?; + assert_eq!(contents, expected_contents); + drop(file); + contents.clear(); + let mut file = reader.by_index(i as usize)?; + file.read_to_end(&mut contents)?; + assert_eq!(contents, expected_contents); + } + Ok(()) + } + + /// Symlinks being extracted shouldn't be followed out of the destination directory. + #[test] + fn test_cannot_symlink_outside_destination() -> ZipResult<()> { + use std::fs::create_dir; + + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.add_symlink("symlink/", "../dest-sibling/", SimpleFileOptions::default())?; + writer.start_file("symlink/dest-file", SimpleFileOptions::default())?; + let mut reader = writer.finish_into_readable()?; + let dest_parent = TempDir::with_prefix("read__test_cannot_symlink_outside_destination")?; + let dest_sibling = dest_parent.path().join("dest-sibling"); + create_dir(&dest_sibling)?; + let dest = dest_parent.path().join("dest"); + create_dir(&dest)?; + assert!(reader.extract(dest).is_err()); + assert!(!dest_sibling.join("dest-file").exists()); + Ok(()) + } + + #[test] + fn test_can_create_destination() -> ZipResult<()> { + let mut reader = + ZipArchive::new(Cursor::new(include_bytes!("../tests/data/mimetype.zip")))?; + let dest = TempDir::with_prefix("read__test_can_create_destination")?; + reader.extract(&dest)?; + assert!(dest.path().join("mimetype").exists()); + Ok(()) + } + + #[test] + fn test_central_directory_not_at_end() -> ZipResult<()> { + let mut reader = ZipArchive::new(Cursor::new(include_bytes!("../tests/data/omni.ja")))?; + let mut file = reader.by_name("chrome.manifest")?; + let mut contents = String::new(); + file.read_to_string(&mut contents)?; // ensures valid UTF-8 + assert!(!contents.is_empty(), "chrome.manifest should not be empty"); + drop(file); + for i in 0..reader.len() { + let mut file = reader.by_index(i)?; + // Attempt to read a small portion or all of each file to ensure it's accessible + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer)?; + assert_eq!( + buffer.len(), + file.size() as usize, + "File size mismatch for {}", + file.name() + ); + } + Ok(()) + } + + #[test] + fn test_ignore_encryption_flag() -> ZipResult<()> { + let mut reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/ignore_encryption_flag.zip" + )))?; + + // Get the file entry by ignoring its encryption flag. + let mut file = + reader.by_index_with_options(0, ZipReadOptions::new().ignore_encryption_flag(true))?; + let mut contents = String::new(); + assert_eq!(file.name(), "plaintext.txt"); + + // The file claims it is encrypted, but it is not. + assert!(file.encrypted()); + file.read_to_string(&mut contents)?; // ensures valid UTF-8 + assert_eq!(contents, "This file is not encrypted.\n"); + Ok(()) + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/read/config.rs b/v2/vendor/security/zip-7.2.0/src/read/config.rs new file mode 100644 index 0000000000..b00c58a49a --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/read/config.rs @@ -0,0 +1,23 @@ +/// Configuration for reading ZIP archives. +#[repr(transparent)] +#[derive(Debug, Default, Clone, Copy)] +pub struct Config { + /// An offset into the reader to use to find the start of the archive. + pub archive_offset: ArchiveOffset, +} + +/// The offset of the start of the archive from the beginning of the reader. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ArchiveOffset { + /// Try to detect the archive offset automatically. + /// + /// This will look at the central directory specified by `FromCentralDirectory` for a header. + /// If missing, this will behave as if `None` were specified. + #[default] + Detect, + /// Use the central directory length and offset to determine the start of the archive. + #[deprecated(since = "2.3.0", note = "use `Detect` instead")] + FromCentralDirectory, + /// Specify a fixed archive offset. + Known(u64), +} diff --git a/v2/vendor/security/zip-7.2.0/src/read/magic_finder.rs b/v2/vendor/security/zip-7.2.0/src/read/magic_finder.rs new file mode 100644 index 0000000000..b6e8c7d8c9 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/read/magic_finder.rs @@ -0,0 +1,291 @@ +use std::io::{Read, Seek, SeekFrom}; + +use memchr::memmem::{Finder, FinderRev}; + +use crate::result::ZipResult; + +pub trait FinderDirection<'a> { + fn new(needle: &'a [u8]) -> Self; + fn reset_cursor(bounds: (u64, u64), window_size: usize) -> u64; + fn scope_window(window: &[u8], mid_window_offset: usize) -> (&[u8], usize); + + fn needle(&self) -> &[u8]; + fn find(&self, haystack: &[u8]) -> Option; + fn move_cursor(&self, cursor: u64, bounds: (u64, u64), window_size: usize) -> Option; + fn move_scope(&self, offset: usize) -> usize; +} + +pub struct Forward<'a>(Finder<'a>); +impl<'a> FinderDirection<'a> for Forward<'a> { + fn new(needle: &'a [u8]) -> Self { + Self(Finder::new(needle)) + } + + fn reset_cursor((start_inclusive, _): (u64, u64), _: usize) -> u64 { + start_inclusive + } + + fn scope_window(window: &[u8], mid_window_offset: usize) -> (&[u8], usize) { + (&window[mid_window_offset..], mid_window_offset) + } + + fn find(&self, haystack: &[u8]) -> Option { + self.0.find(haystack) + } + + fn needle(&self) -> &[u8] { + self.0.needle() + } + + fn move_cursor(&self, cursor: u64, bounds: (u64, u64), window_size: usize) -> Option { + let magic_overlap = self.needle().len().saturating_sub(1) as u64; + let next = cursor.saturating_add(window_size as u64 - magic_overlap); + + if next >= bounds.1 { + None + } else { + Some(next) + } + } + + fn move_scope(&self, offset: usize) -> usize { + offset + self.needle().len() + } +} + +pub struct Backwards<'a>(FinderRev<'a>); +impl<'a> FinderDirection<'a> for Backwards<'a> { + fn new(needle: &'a [u8]) -> Self { + Self(FinderRev::new(needle)) + } + + fn reset_cursor(bounds: (u64, u64), window_size: usize) -> u64 { + bounds + .1 + .saturating_sub(window_size as u64) + .clamp(bounds.0, bounds.1) + } + + fn scope_window(window: &[u8], mid_window_offset: usize) -> (&[u8], usize) { + (&window[..mid_window_offset], 0) + } + + fn find(&self, haystack: &[u8]) -> Option { + self.0.rfind(haystack) + } + + fn needle(&self) -> &[u8] { + self.0.needle() + } + + fn move_cursor(&self, cursor: u64, bounds: (u64, u64), window_size: usize) -> Option { + let magic_overlap = self.needle().len().saturating_sub(1) as u64; + + if cursor <= bounds.0 { + None + } else { + Some( + cursor + .saturating_add(magic_overlap) + .saturating_sub(window_size as u64) + .clamp(bounds.0, bounds.1), + ) + } + } + + fn move_scope(&self, offset: usize) -> usize { + offset + } +} + +/// A utility for finding magic symbols from the end of a seekable reader. +/// +/// Can be repurposed to recycle the internal buffer. +pub struct MagicFinder { + buffer: Box<[u8]>, + pub(self) finder: Direction, + cursor: u64, + mid_buffer_offset: Option, + bounds: (u64, u64), +} + +impl<'a, T: FinderDirection<'a>> MagicFinder { + /// Create a new magic bytes finder to look within specific bounds. + pub fn new(magic_bytes: &'a [u8], start_inclusive: u64, end_exclusive: u64) -> Self { + const BUFFER_SIZE: usize = 2048; + + // Smaller buffer size would be unable to locate bytes. + // Equal buffer size would stall (the window could not be moved). + debug_assert!(BUFFER_SIZE >= magic_bytes.len()); + + Self { + buffer: vec![0; BUFFER_SIZE].into_boxed_slice(), + finder: T::new(magic_bytes), + cursor: T::reset_cursor((start_inclusive, end_exclusive), BUFFER_SIZE), + mid_buffer_offset: None, + bounds: (start_inclusive, end_exclusive), + } + } + + /// Repurpose the finder for different bytes or bounds. + pub fn repurpose(&mut self, magic_bytes: &'a [u8], bounds: (u64, u64)) -> &mut Self { + debug_assert!(self.buffer.len() >= magic_bytes.len()); + + self.finder = T::new(magic_bytes); + self.cursor = T::reset_cursor(bounds, self.buffer.len()); + self.bounds = bounds; + + // Reset the mid-buffer offset, to invalidate buffer content. + self.mid_buffer_offset = None; + + self + } + + /// Find the next magic bytes in the direction specified in the type. + pub fn next(&mut self, reader: &mut R) -> ZipResult> { + loop { + if self.cursor < self.bounds.0 || self.cursor >= self.bounds.1 { + // The finder is consumed + break; + } + + /* Position the window and ensure correct length */ + let window_start = self.cursor; + let window_end = self + .cursor + .saturating_add(self.buffer.len() as u64) + .min(self.bounds.1); + + if window_end <= window_start { + // Short-circuit on zero-sized windows to prevent loop + break; + } + + let window = &mut self.buffer[..(window_end - window_start) as usize]; + + if self.mid_buffer_offset.is_none() { + reader.seek(SeekFrom::Start(window_start))?; + if let Err(e) = reader.read_exact(window) { + if e.kind() == std::io::ErrorKind::UnexpectedEof { + // If we can't read the whole window, it means the bounds or file length are wrong. + break; + } + return Err(e.into()); + } + } + + let (window, window_start_offset) = match self.mid_buffer_offset { + Some(mid_buffer_offset) => T::scope_window(window, mid_buffer_offset), + None => (&*window, 0usize), + }; + + if let Some(offset) = self.finder.find(window) { + let magic_pos = window_start + window_start_offset as u64 + offset as u64; + reader.seek(SeekFrom::Start(magic_pos))?; + + self.mid_buffer_offset = Some(self.finder.move_scope(window_start_offset + offset)); + + return Ok(Some(magic_pos)); + } + + self.mid_buffer_offset = None; + + match self + .finder + .move_cursor(self.cursor, self.bounds, self.buffer.len()) + { + Some(new_cursor) => { + self.cursor = new_cursor; + } + None => { + // Destroy the finder when we've reached the end of the bounds. + self.bounds.0 = self.bounds.1; + break; + } + } + } + + Ok(None) + } +} + +/// A magic bytes finder with an optimistic guess that is tried before +/// the inner finder begins searching from end. This enables much faster +/// lookup in files without appended junk, because the magic bytes will be +/// found directly. +/// +/// The guess can be marked as mandatory to produce an error. This is useful +/// if the ArchiveOffset is known and auto-detection is not desired. +pub struct OptimisticMagicFinder { + inner: MagicFinder, + initial_guess: Option<(u64, bool)>, +} + +/// This is a temporary restriction, to avoid heap allocation in [`Self::next_back`]. +/// +/// We only use magic bytes of size 4 at the moment. +const STACK_BUFFER_SIZE: usize = 8; + +impl<'a, Direction: FinderDirection<'a>> OptimisticMagicFinder { + /// Create a new empty optimistic magic bytes finder. + pub fn new_empty() -> Self { + Self { + inner: MagicFinder::new(&[], 0, 0), + initial_guess: None, + } + } + + /// Repurpose the finder for different bytes, bounds and initial guesses. + pub fn repurpose( + &mut self, + magic_bytes: &'a [u8], + bounds: (u64, u64), + initial_guess: Option<(u64, bool)>, + ) -> &mut Self { + debug_assert!(magic_bytes.len() <= STACK_BUFFER_SIZE); + + self.inner.repurpose(magic_bytes, bounds); + self.initial_guess = initial_guess; + + self + } + + /// Equivalent to `next_back`, with an optional initial guess attempted before + /// proceeding with reading from the back of the reader. + pub fn next(&mut self, reader: &mut R) -> ZipResult> { + if let Some((v, mandatory)) = self.initial_guess { + reader.seek(SeekFrom::Start(v))?; + + let mut buffer = [0; STACK_BUFFER_SIZE]; + let buffer = &mut buffer[..self.inner.finder.needle().len()]; + + // Attempt to match only if there's enough space for the needle + if v.saturating_add(buffer.len() as u64) <= self.inner.bounds.1 { + if let Err(e) = reader.read_exact(buffer) { + if e.kind() == std::io::ErrorKind::UnexpectedEof { + if mandatory { + return Ok(None); + } + } else { + return Err(e.into()); + } + } else if self.inner.finder.needle() == buffer { + // If a match is found, yield it. + self.initial_guess.take(); + reader.seek(SeekFrom::Start(v))?; + return Ok(Some(v)); + } + } + + // If a match is not found, but the initial guess was mandatory, return an error. + if mandatory { + return Ok(None); + } + + // If the initial guess was not mandatory, remove it, as it was not found. + self.initial_guess.take(); + } + + self.inner.next(reader) + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/read/stream.rs b/v2/vendor/security/zip-7.2.0/src/read/stream.rs new file mode 100644 index 0000000000..7b9dfada56 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/read/stream.rs @@ -0,0 +1,415 @@ +use crate::result::invalid; + +use super::{ + MAX_SYMLINK_TARGET_LEN, + central_header_to_zip_file_inner, make_symlink, read_zipfile_from_stream, ZipCentralEntryBlock, + ZipFile, ZipFileData, ZipResult, +}; +use crate::spec::FixedSizeBlock; +use indexmap::IndexMap; +use std::io::{self, Read}; +use std::path::{Path, PathBuf}; + +/// Stream decoder for zip. +#[derive(Debug)] +pub struct ZipStreamReader(R); + +impl ZipStreamReader { + /// Create a new ZipStreamReader + pub const fn new(reader: R) -> Self { + Self(reader) + } +} + +impl ZipStreamReader { + fn parse_central_directory(&mut self) -> ZipResult { + // Give archive_offset and central_header_start dummy value 0, since + // they are not used in the output. + let archive_offset = 0; + let central_header_start = 0; + + // Parse central header + let block = ZipCentralEntryBlock::parse(&mut self.0)?; + let file = central_header_to_zip_file_inner( + &mut self.0, + archive_offset, + central_header_start, + block, + )?; + Ok(ZipStreamFileMetadata(file)) + } + + /// Iterate over the stream and extract all file and their + /// metadata. + pub fn visit(mut self, visitor: &mut V) -> ZipResult<()> { + while let Some(mut file) = read_zipfile_from_stream(&mut self.0)? { + visitor.visit_file(&mut file)?; + } + + while let Ok(metadata) = self.parse_central_directory() { + visitor.visit_additional_metadata(&metadata)?; + } + + Ok(()) + } + + /// Extract a Zip archive into a directory, overwriting files if they + /// already exist. Paths are sanitized with [`ZipFile::enclosed_name`]. + /// + /// Extraction is not atomic; If an error is encountered, some of the files + /// may be left on disk. + pub fn extract>(self, directory: P) -> ZipResult<()> { + use std::fs; + fs::create_dir_all(&directory)?; + let directory = directory.as_ref().canonicalize()?; + struct Extractor(PathBuf, IndexMap, ()>); + impl ZipStreamVisitor for Extractor { + fn visit_file(&mut self, file: &mut ZipFile<'_, R>) -> ZipResult<()> { + self.1.insert(file.name().into(), ()); + let mut outpath = self.0.clone(); + file.safe_prepare_path(&self.0, &mut outpath, None::<&(_, fn(&Path) -> bool)>)?; + + if file.is_symlink() { + let declared_len = file.size(); + if declared_len > MAX_SYMLINK_TARGET_LEN { + return Err(invalid!( + "symlink target declares {} bytes, more than the {} byte limit", + declared_len, + MAX_SYMLINK_TARGET_LEN + )); + } + let mut target = Vec::with_capacity(declared_len as usize); + file.read_to_end(&mut target)?; + make_symlink(&outpath, &target, &self.1)?; + return Ok(()); + } + + if file.is_dir() { + fs::create_dir_all(&outpath)?; + } else { + let mut outfile = fs::File::create(&outpath)?; + io::copy(file, &mut outfile)?; + } + + Ok(()) + } + + #[allow(unused)] + fn visit_additional_metadata( + &mut self, + metadata: &ZipStreamFileMetadata, + ) -> ZipResult<()> { + #[cfg(unix)] + { + use super::ZipError; + let filepath = metadata + .enclosed_name() + .ok_or(crate::result::invalid!("Invalid file path"))?; + + let outpath = self.0.join(filepath); + + use std::os::unix::fs::PermissionsExt; + if let Some(mode) = metadata.unix_mode() { + fs::set_permissions(outpath, fs::Permissions::from_mode(mode))?; + } + } + + Ok(()) + } + } + + self.visit(&mut Extractor(directory, IndexMap::new())) + } +} + +/// Visitor for ZipStreamReader +pub trait ZipStreamVisitor { + /// * `file` - contains the content of the file and most of the metadata, + /// except: + /// - `comment`: set to an empty string + /// - `data_start`: set to 0 + /// - `external_attributes`: `unix_mode()`: will return None + fn visit_file(&mut self, file: &mut ZipFile<'_, R>) -> ZipResult<()>; + + /// This function is guaranteed to be called after all `visit_file`s. + /// + /// * `metadata` - Provides missing metadata in `visit_file`. + fn visit_additional_metadata(&mut self, metadata: &ZipStreamFileMetadata) -> ZipResult<()>; +} + +/// Additional metadata for the file. +#[derive(Debug)] +pub struct ZipStreamFileMetadata(ZipFileData); + +impl ZipStreamFileMetadata { + /// Get the name of the file + /// + /// # Warnings + /// + /// It is dangerous to use this name directly when extracting an archive. + /// It may contain an absolute path (`/etc/shadow`), or break out of the + /// current directory (`../runtime`). Carelessly writing to these paths + /// allows an attacker to craft a ZIP archive that will overwrite critical + /// files. + /// + /// You can use the [`ZipFile::enclosed_name`] method to validate the name + /// as a safe path. + pub fn name(&self) -> &str { + &self.0.file_name + } + + /// Get the name of the file, in the raw (internal) byte representation. + /// + /// The encoding of this data is currently undefined. + pub fn name_raw(&self) -> &[u8] { + &self.0.file_name_raw + } + + /// Rewrite the path, ignoring any path components with special meaning. + /// + /// - Absolute paths are made relative + /// - [std::path::Component::ParentDir]s are ignored + /// - Truncates the filename at a NULL byte + /// + /// This is appropriate if you need to be able to extract *something* from + /// any archive, but will easily misrepresent trivial paths like + /// `foo/../bar` as `foo/bar` (instead of `bar`). Because of this, + /// [`ZipFile::enclosed_name`] is the better option in most scenarios. + pub fn mangled_name(&self) -> PathBuf { + self.0.file_name_sanitized() + } + + /// Ensure the file path is safe to use as a [`Path`]. + /// + /// - It can't contain NULL bytes + /// - It can't resolve to a path outside the current directory + /// > `foo/../bar` is fine, `foo/../../bar` is not. + /// - It can't be an absolute path + /// + /// This will read well-formed ZIP files correctly, and is resistant + /// to path-based exploits. It is recommended over + /// [`ZipFile::mangled_name`]. + pub fn enclosed_name(&self) -> Option { + self.0.enclosed_name() + } + + /// Returns whether the file is actually a directory + pub fn is_dir(&self) -> bool { + self.name() + .chars() + .next_back() + .is_some_and(|c| c == '/' || c == '\\') + } + + /// Returns whether the file is a regular file + pub fn is_file(&self) -> bool { + !self.is_dir() + } + + /// Get the comment of the file + pub fn comment(&self) -> &str { + &self.0.file_comment + } + + /// Get unix mode for the file + pub const fn unix_mode(&self) -> Option { + self.0.unix_mode() + } +} + +#[cfg(test)] +mod test { + use tempfile::TempDir; + + use crate::read::stream::{ZipStreamFileMetadata, ZipStreamReader, ZipStreamVisitor}; + use crate::read::ZipFile; + use crate::result::ZipResult; + use crate::write::SimpleFileOptions; + use crate::ZipWriter; + use std::collections::BTreeSet; + use std::io::{Cursor, Read}; + + struct DummyVisitor; + impl ZipStreamVisitor for DummyVisitor { + fn visit_file(&mut self, _file: &mut ZipFile<'_, R>) -> ZipResult<()> { + Ok(()) + } + + fn visit_additional_metadata( + &mut self, + _metadata: &ZipStreamFileMetadata, + ) -> ZipResult<()> { + Ok(()) + } + } + + #[allow(dead_code)] + #[derive(Default, Debug, Eq, PartialEq)] + struct CounterVisitor(u64, u64); + impl ZipStreamVisitor for CounterVisitor { + fn visit_file(&mut self, _file: &mut ZipFile<'_, R>) -> ZipResult<()> { + self.0 += 1; + Ok(()) + } + + fn visit_additional_metadata( + &mut self, + _metadata: &ZipStreamFileMetadata, + ) -> ZipResult<()> { + self.1 += 1; + Ok(()) + } + } + + #[test] + fn invalid_offset() { + ZipStreamReader::new(Cursor::new(include_bytes!( + "../../tests/data/invalid_offset.zip" + ))) + .visit(&mut DummyVisitor) + .unwrap_err(); + } + + #[test] + fn invalid_offset2() { + ZipStreamReader::new(Cursor::new(include_bytes!( + "../../tests/data/invalid_offset2.zip" + ))) + .visit(&mut DummyVisitor) + .unwrap_err(); + } + + #[test] + fn zip_read_streaming() { + let reader = + ZipStreamReader::new(Cursor::new(include_bytes!("../../tests/data/mimetype.zip"))); + + #[derive(Default)] + struct V { + filenames: BTreeSet>, + } + impl ZipStreamVisitor for V { + fn visit_file(&mut self, file: &mut ZipFile<'_, R>) -> ZipResult<()> { + if file.is_file() { + self.filenames.insert(file.name().into()); + } + + Ok(()) + } + fn visit_additional_metadata( + &mut self, + metadata: &ZipStreamFileMetadata, + ) -> ZipResult<()> { + if metadata.is_file() { + assert!( + self.filenames.contains(metadata.name()), + "{} is missing its file content", + metadata.name() + ); + } + + Ok(()) + } + } + + reader.visit(&mut V::default()).unwrap(); + } + + #[test] + fn file_and_dir_predicates() { + let reader = ZipStreamReader::new(Cursor::new(include_bytes!( + "../../tests/data/files_and_dirs.zip" + ))); + + #[derive(Default)] + struct V { + filenames: BTreeSet>, + } + impl ZipStreamVisitor for V { + fn visit_file(&mut self, file: &mut ZipFile<'_, R>) -> ZipResult<()> { + let full_name = file.enclosed_name().unwrap(); + let file_name = full_name.file_name().unwrap().to_str().unwrap(); + assert!( + (file_name.starts_with("dir") && file.is_dir()) + || (file_name.starts_with("file") && file.is_file()) + ); + + if file.is_file() { + self.filenames.insert(file.name().into()); + } + + Ok(()) + } + fn visit_additional_metadata( + &mut self, + metadata: &ZipStreamFileMetadata, + ) -> ZipResult<()> { + if metadata.is_file() { + assert!( + self.filenames.contains(metadata.name()), + "{} is missing its file content", + metadata.name() + ); + } + + Ok(()) + } + } + + reader.visit(&mut V::default()).unwrap(); + } + + /// test case to ensure we don't preemptively over allocate based on the + /// declared number of files in the CDE of an invalid zip when the number of + /// files declared is more than the alleged offset in the CDE + #[test] + fn invalid_cde_number_of_files_allocation_smaller_offset() { + ZipStreamReader::new(Cursor::new(include_bytes!( + "../../tests/data/invalid_cde_number_of_files_allocation_smaller_offset.zip" + ))) + .visit(&mut DummyVisitor) + .unwrap_err(); + } + + /// test case to ensure we don't preemptively over allocate based on the + /// declared number of files in the CDE of an invalid zip when the number of + /// files declared is less than the alleged offset in the CDE + #[test] + fn invalid_cde_number_of_files_allocation_greater_offset() { + ZipStreamReader::new(Cursor::new(include_bytes!( + "../../tests/data/invalid_cde_number_of_files_allocation_greater_offset.zip" + ))) + .visit(&mut DummyVisitor) + .unwrap_err(); + } + + /// Symlinks being extracted shouldn't be followed out of the destination directory. + #[test] + fn test_cannot_symlink_outside_destination() -> ZipResult<()> { + use std::fs::create_dir; + + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.add_symlink("symlink/", "../dest-sibling/", SimpleFileOptions::default())?; + writer.start_file("symlink/dest-file", SimpleFileOptions::default())?; + let reader = ZipStreamReader::new(writer.finish()?); + let dest_parent = TempDir::with_prefix("stream__cannot_symlink_outside_destination")?; + let dest_sibling = dest_parent.path().join("dest-sibling"); + create_dir(&dest_sibling)?; + let dest = dest_parent.path().join("dest"); + create_dir(&dest)?; + assert!(reader.extract(dest).is_err()); + assert!(!dest_sibling.join("dest-file").exists()); + Ok(()) + } + + #[test] + fn test_can_create_destination() -> ZipResult<()> { + let mut v = Vec::new(); + v.extend_from_slice(include_bytes!("../../tests/data/mimetype.zip")); + let reader = ZipStreamReader::new(v.as_slice()); + let dest = TempDir::with_prefix("stream_test_can_create_destination").unwrap(); + reader.extract(&dest)?; + assert!(dest.path().join("mimetype").exists()); + Ok(()) + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/result.rs b/v2/vendor/security/zip-7.2.0/src/result.rs new file mode 100644 index 0000000000..2217f20eaa --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/result.rs @@ -0,0 +1,143 @@ +#![allow(unknown_lints)] // non_local_definitions isn't in Rust 1.70 +#![allow(non_local_definitions)] +//! Error types that can be emitted from this library + +use core::error::Error; +use core::fmt::{self, Display, Formatter}; +use core::num::TryFromIntError; +use std::borrow::Cow; +use std::io; + +/// Generic result type with ZipError as its error variant +pub type ZipResult = Result; + +/// Error type for Zip +#[derive(Debug)] +#[non_exhaustive] +pub enum ZipError { + /// i/o error + Io(io::Error), + + /// invalid Zip archive + InvalidArchive(Cow<'static, str>), + + /// unsupported Zip archive + UnsupportedArchive(&'static str), + + /// specified file not found in archive + FileNotFound, + + /// provided password is incorrect + InvalidPassword, +} + +impl ZipError { + /// The text used as an error when a password is required and not supplied + /// + /// ```rust,no_run + /// # use zip::result::ZipError; + /// # let mut archive = zip::ZipArchive::new(std::io::Cursor::new(&[])).unwrap(); + /// match archive.by_index(1) { + /// Err(ZipError::UnsupportedArchive(ZipError::PASSWORD_REQUIRED)) => eprintln!("a password is needed to unzip this file"), + /// _ => (), + /// } + /// # () + /// ``` + pub const PASSWORD_REQUIRED: &'static str = "Password required to decrypt file"; +} + +impl Display for ZipError { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + Self::Io(e) => write!(f, "i/o error: {e}"), + Self::InvalidArchive(e) => write!(f, "invalid Zip archive: {e}"), + Self::UnsupportedArchive(e) => write!(f, "unsupported Zip archive: {e}"), + Self::FileNotFound => f.write_str("specified file not found in archive"), + Self::InvalidPassword => f.write_str("provided password is incorrect"), + } + } +} + +impl Error for ZipError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Io(e) => Some(e), + Self::InvalidArchive(_) + | Self::UnsupportedArchive(_) + | Self::FileNotFound + | Self::InvalidPassword => None, + } + } +} + +impl From for io::Error { + fn from(err: ZipError) -> io::Error { + let kind = match &err { + ZipError::Io(err) => err.kind(), + ZipError::InvalidArchive(_) => io::ErrorKind::InvalidData, + ZipError::UnsupportedArchive(_) => io::ErrorKind::Unsupported, + ZipError::FileNotFound => io::ErrorKind::NotFound, + ZipError::InvalidPassword => io::ErrorKind::InvalidInput, + }; + + io::Error::new(kind, err) + } +} + +impl From for ZipError { + fn from(value: io::Error) -> Self { + Self::Io(value) + } +} + +impl From for ZipError { + fn from(_: DateTimeRangeError) -> Self { + invalid!("Invalid date or time") + } +} + +impl From for ZipError { + fn from(_: std::string::FromUtf8Error) -> Self { + invalid!("Invalid UTF-8") + } +} + +/// Error type for time parsing +#[derive(Debug)] +pub struct DateTimeRangeError; + +// TryFromIntError is also an out-of-range error. +impl From for DateTimeRangeError { + fn from(_value: TryFromIntError) -> Self { + DateTimeRangeError + } +} + +impl fmt::Display for DateTimeRangeError { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + write!( + fmt, + "a date could not be represented within the bounds the MS-DOS date range (1980-2107)" + ) + } +} + +impl Error for DateTimeRangeError {} + +pub(crate) fn invalid_archive>>(message: M) -> ZipError { + ZipError::InvalidArchive(message.into()) +} + +pub(crate) const fn invalid_archive_const(message: &'static str) -> ZipError { + ZipError::InvalidArchive(Cow::Borrowed(message)) +} + +macro_rules! invalid { + ($message:literal) => { + crate::result::invalid_archive_const($message) + }; + ($($arg:tt)*) => { + crate::result::invalid_archive(format!($($arg)*)) + }; +} +pub(crate) use invalid; diff --git a/v2/vendor/security/zip-7.2.0/src/spec.rs b/v2/vendor/security/zip-7.2.0/src/spec.rs new file mode 100644 index 0000000000..61107d0f79 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/spec.rs @@ -0,0 +1,873 @@ +#![macro_use] + +use crate::read::magic_finder::{Backwards, Forward, MagicFinder, OptimisticMagicFinder}; +use crate::read::ArchiveOffset; +use crate::result::{invalid, ZipError, ZipResult}; +use core::any::type_name; +use core::mem; +use core::slice; +use std::io::{self, Read, Seek, Write}; + +/// "Magic" header values used in the zip spec to locate metadata records. +/// +/// These values currently always take up a fixed four bytes, so we can parse and wrap them in this +/// struct to enforce some small amount of type safety. +#[derive(Copy, Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash)] +#[repr(transparent)] +pub(crate) struct Magic(u32); + +impl Magic { + pub const fn literal(x: u32) -> Self { + Self(x) + } + + #[inline(always)] + #[allow(dead_code)] + pub const fn from_le_bytes(bytes: [u8; 4]) -> Self { + Self(u32::from_le_bytes(bytes)) + } + + #[inline(always)] + pub const fn to_le_bytes(self) -> [u8; 4] { + self.0.to_le_bytes() + } + + #[allow(clippy::wrong_self_convention)] + #[inline(always)] + pub fn from_le(self) -> Self { + Self(u32::from_le(self.0)) + } + + #[allow(clippy::wrong_self_convention)] + #[inline(always)] + pub fn to_le(self) -> Self { + Self(u32::to_le(self.0)) + } + + pub const LOCAL_FILE_HEADER_SIGNATURE: Self = Self::literal(0x04034b50); + pub const CENTRAL_DIRECTORY_HEADER_SIGNATURE: Self = Self::literal(0x02014b50); + pub const CENTRAL_DIRECTORY_END_SIGNATURE: Self = Self::literal(0x06054b50); + pub const ZIP64_CENTRAL_DIRECTORY_END_SIGNATURE: Self = Self::literal(0x06064b50); + pub const ZIP64_CENTRAL_DIRECTORY_END_LOCATOR_SIGNATURE: Self = Self::literal(0x07064b50); + pub const DATA_DESCRIPTOR_SIGNATURE: Self = Self::literal(0x08074b50); +} + +/// Similar to [`Magic`], but used for extra field tags as per section 4.5.3 of APPNOTE.TXT. +#[derive(Copy, Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash)] +#[repr(transparent)] +pub(crate) struct ExtraFieldMagic(u16); + +/* TODO: maybe try to use this for parsing extra fields as well as writing them? */ +#[allow(dead_code)] +impl ExtraFieldMagic { + pub const fn literal(x: u16) -> Self { + Self(x) + } + + #[inline(always)] + pub const fn from_le_bytes(bytes: [u8; 2]) -> Self { + Self(u16::from_le_bytes(bytes)) + } + + #[inline(always)] + pub const fn to_le_bytes(self) -> [u8; 2] { + self.0.to_le_bytes() + } + + #[allow(clippy::wrong_self_convention)] + #[inline(always)] + pub fn from_le(self) -> Self { + Self(u16::from_le(self.0)) + } + + #[allow(clippy::wrong_self_convention)] + #[inline(always)] + pub fn to_le(self) -> Self { + Self(u16::to_le(self.0)) + } + + pub const ZIP64_EXTRA_FIELD_TAG: Self = Self::literal(0x0001); +} + +/// The file size at which a ZIP64 record becomes necessary. +/// +/// If a file larger than this threshold attempts to be written, compressed or uncompressed, and +/// [`FileOptions::large_file()`](crate::write::FileOptions::large_file) was not true, then [`crate::ZipWriter`] will +/// raise an [`io::Error`] with [`io::ErrorKind::Other`]. +/// +/// If the zip file itself is larger than this value, then a zip64 central directory record will be +/// written to the end of the file. +/// +///``` +/// # fn main() -> Result<(), zip::result::ZipError> { +/// # #[cfg(target_pointer_width = "64")] +/// # { +/// use std::io::{self, Cursor, Write}; +/// use std::error::Error; +/// use zip::{ZipWriter, write::SimpleFileOptions}; +/// +/// let mut zip = ZipWriter::new(Cursor::new(Vec::new())); +/// // Writing an extremely large file for this test is faster without compression. +/// +/// let big_len: usize = (zip::ZIP64_BYTES_THR as usize) + 1; +/// let big_buf = vec![0u8; big_len]; +/// { +/// let options = SimpleFileOptions::default() +/// .compression_method(zip::CompressionMethod::Stored); +/// zip.start_file("zero.dat", options)?; +/// // This is too big! +/// let res = zip.write_all(&big_buf[..]).err().unwrap(); +/// assert_eq!(res.kind(), io::ErrorKind::Other); +/// let description = format!("{}", &res); +/// assert_eq!(description, "Large file option has not been set"); +/// // Attempting to write anything further to the same zip will still succeed, but the previous +/// // failing entry has been removed. +/// zip.start_file("one.dat", options)?; +/// let zip = zip.finish_into_readable()?; +/// let names: Vec<_> = zip.file_names().collect(); +/// assert_eq!(&names, &["one.dat"]); +/// } +/// +/// // Create a new zip output. +/// let mut zip = ZipWriter::new(Cursor::new(Vec::new())); +/// // This time, create a zip64 record for the file. +/// let options = SimpleFileOptions::default() +/// .compression_method(zip::CompressionMethod::Stored) +/// .large_file(true); +/// zip.start_file("zero.dat", options)?; +/// // This succeeds because we specified that it could be a large file. +/// assert!(zip.write_all(&big_buf[..]).is_ok()); +/// # } +/// # Ok(()) +/// # } +///``` +pub const ZIP64_BYTES_THR: u64 = u32::MAX as u64; +/// The number of entries within a single zip necessary to allocate a zip64 central +/// directory record. +/// +/// If more than this number of entries is written to a [`crate::ZipWriter`], then [`crate::ZipWriter::finish()`] +/// will write out extra zip64 data to the end of the zip file. +pub const ZIP64_ENTRY_THR: usize = u16::MAX as usize; + +/// # Safety +/// +/// - No padding/uninit bytes +/// - All bytes patterns must be valid +/// - No cell, pointers +/// +/// See `bytemuck::Pod` for more details. +pub(crate) unsafe trait Pod: Copy + 'static { + #[inline] + fn zeroed() -> Self { + unsafe { mem::zeroed() } + } + + #[inline] + fn as_bytes(&self) -> &[u8] { + unsafe { slice::from_raw_parts(self as *const Self as *const u8, mem::size_of::()) } + } + + #[inline] + fn as_bytes_mut(&mut self) -> &mut [u8] { + unsafe { slice::from_raw_parts_mut(self as *mut Self as *mut u8, mem::size_of::()) } + } +} + +pub(crate) trait FixedSizeBlock: Pod { + const MAGIC: Magic; + + fn magic(self) -> Magic; + + const WRONG_MAGIC_ERROR: ZipError; + + #[allow(clippy::wrong_self_convention)] + fn from_le(self) -> Self; + + fn parse(reader: &mut R) -> ZipResult { + let mut block = Self::zeroed(); + if let Err(e) = reader.read_exact(block.as_bytes_mut()) { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!("Unexpected end of {}", type_name::())); + } + return Err(e.into()); + } + let block = Self::from_le(block); + + if block.magic() != Self::MAGIC { + return Err(Self::WRONG_MAGIC_ERROR); + } + Ok(block) + } + + fn to_le(self) -> Self; + + fn write(self, writer: &mut T) -> ZipResult<()> { + let block = self.to_le(); + writer.write_all(block.as_bytes())?; + Ok(()) + } +} + +/// Convert all the fields of a struct *from* little-endian representations. +macro_rules! from_le { + ($obj:ident, $field:ident, $type:ty) => { + $obj.$field = <$type>::from_le($obj.$field); + }; + ($obj:ident, [($field:ident, $type:ty) $(,)?]) => { + from_le![$obj, $field, $type]; + }; + ($obj:ident, [($field:ident, $type:ty), $($rest:tt),+ $(,)?]) => { + from_le![$obj, $field, $type]; + from_le!($obj, [$($rest),+]); + }; +} + +/// Convert all the fields of a struct *into* little-endian representations. +macro_rules! to_le { + ($obj:ident, $field:ident, $type:ty) => { + $obj.$field = <$type>::to_le($obj.$field); + }; + ($obj:ident, [($field:ident, $type:ty) $(,)?]) => { + to_le![$obj, $field, $type]; + }; + ($obj:ident, [($field:ident, $type:ty), $($rest:tt),+ $(,)?]) => { + to_le![$obj, $field, $type]; + to_le!($obj, [$($rest),+]); + }; +} + +/* TODO: derive macro to generate these fields? */ +/// Implement `from_le()` and `to_le()`, providing the field specification to both macros +/// and methods. +macro_rules! to_and_from_le { + ($($args:tt),+ $(,)?) => { + #[inline(always)] + fn from_le(mut self) -> Self { + from_le![self, [$($args),+]]; + self + } + #[inline(always)] + fn to_le(mut self) -> Self { + to_le![self, [$($args),+]]; + self + } + }; +} + +#[derive(Copy, Clone, Debug)] +#[repr(packed, C)] +pub(crate) struct Zip32CDEBlock { + magic: Magic, + pub disk_number: u16, + pub disk_with_central_directory: u16, + pub number_of_files_on_this_disk: u16, + pub number_of_files: u16, + pub central_directory_size: u32, + pub central_directory_offset: u32, + pub zip_file_comment_length: u16, +} + +unsafe impl Pod for Zip32CDEBlock {} + +impl FixedSizeBlock for Zip32CDEBlock { + const MAGIC: Magic = Magic::CENTRAL_DIRECTORY_END_SIGNATURE; + + #[inline(always)] + fn magic(self) -> Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("Invalid digital signature header"); + + to_and_from_le![ + (magic, Magic), + (disk_number, u16), + (disk_with_central_directory, u16), + (number_of_files_on_this_disk, u16), + (number_of_files, u16), + (central_directory_size, u32), + (central_directory_offset, u32), + (zip_file_comment_length, u16) + ]; +} + +#[derive(Debug)] +pub(crate) struct Zip32CentralDirectoryEnd { + pub disk_number: u16, + pub disk_with_central_directory: u16, + pub number_of_files_on_this_disk: u16, + pub number_of_files: u16, + pub central_directory_size: u32, + pub central_directory_offset: u32, + pub zip_file_comment: Box<[u8]>, +} + +impl Zip32CentralDirectoryEnd { + fn into_block_and_comment(self) -> (Zip32CDEBlock, Box<[u8]>) { + let Self { + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + zip_file_comment, + } = self; + let block = Zip32CDEBlock { + magic: Zip32CDEBlock::MAGIC, + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + zip_file_comment_length: zip_file_comment.len() as u16, + }; + + (block, zip_file_comment) + } + + pub fn parse(reader: &mut T) -> ZipResult { + let Zip32CDEBlock { + // magic, + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + zip_file_comment_length, + .. + } = Zip32CDEBlock::parse(reader)?; + + let mut zip_file_comment = vec![0u8; zip_file_comment_length as usize].into_boxed_slice(); + if let Err(e) = reader.read_exact(&mut zip_file_comment) { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!("EOCD comment exceeds file boundary")); + } + + return Err(e.into()); + } + + Ok(Zip32CentralDirectoryEnd { + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + zip_file_comment, + }) + } + + pub fn write(self, writer: &mut T) -> ZipResult<()> { + let (block, comment) = self.into_block_and_comment(); + + if comment.len() > u16::MAX as usize { + return Err(invalid!("EOCD comment length exceeds u16::MAX")); + } + + block.write(writer)?; + writer.write_all(&comment)?; + Ok(()) + } + + pub fn may_be_zip64(&self) -> bool { + self.number_of_files == u16::MAX || self.central_directory_offset == u32::MAX + } +} + +#[derive(Copy, Clone)] +#[repr(packed, C)] +pub(crate) struct Zip64CDELocatorBlock { + magic: Magic, + pub disk_with_central_directory: u32, + pub end_of_central_directory_offset: u64, + pub number_of_disks: u32, +} + +unsafe impl Pod for Zip64CDELocatorBlock {} + +impl FixedSizeBlock for Zip64CDELocatorBlock { + const MAGIC: Magic = Magic::ZIP64_CENTRAL_DIRECTORY_END_LOCATOR_SIGNATURE; + + #[inline(always)] + fn magic(self) -> Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("Invalid zip64 locator digital signature header"); + + to_and_from_le![ + (magic, Magic), + (disk_with_central_directory, u32), + (end_of_central_directory_offset, u64), + (number_of_disks, u32), + ]; +} + +pub(crate) struct Zip64CentralDirectoryEndLocator { + pub disk_with_central_directory: u32, + pub end_of_central_directory_offset: u64, + pub number_of_disks: u32, +} + +impl Zip64CentralDirectoryEndLocator { + pub fn parse(reader: &mut T) -> ZipResult { + let Zip64CDELocatorBlock { + // magic, + disk_with_central_directory, + end_of_central_directory_offset, + number_of_disks, + .. + } = Zip64CDELocatorBlock::parse(reader)?; + + Ok(Zip64CentralDirectoryEndLocator { + disk_with_central_directory, + end_of_central_directory_offset, + number_of_disks, + }) + } + + pub fn block(self) -> Zip64CDELocatorBlock { + let Self { + disk_with_central_directory, + end_of_central_directory_offset, + number_of_disks, + } = self; + Zip64CDELocatorBlock { + magic: Zip64CDELocatorBlock::MAGIC, + disk_with_central_directory, + end_of_central_directory_offset, + number_of_disks, + } + } + + pub fn write(self, writer: &mut T) -> ZipResult<()> { + self.block().write(writer) + } +} + +#[derive(Copy, Clone)] +#[repr(packed, C)] +pub(crate) struct Zip64CDEBlock { + magic: Magic, + pub record_size: u64, + pub version_made_by: u16, + pub version_needed_to_extract: u16, + pub disk_number: u32, + pub disk_with_central_directory: u32, + pub number_of_files_on_this_disk: u64, + pub number_of_files: u64, + pub central_directory_size: u64, + pub central_directory_offset: u64, +} + +unsafe impl Pod for Zip64CDEBlock {} + +impl FixedSizeBlock for Zip64CDEBlock { + const MAGIC: Magic = Magic::ZIP64_CENTRAL_DIRECTORY_END_SIGNATURE; + + fn magic(self) -> Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("Invalid digital signature header"); + + to_and_from_le![ + (magic, Magic), + (record_size, u64), + (version_made_by, u16), + (version_needed_to_extract, u16), + (disk_number, u32), + (disk_with_central_directory, u32), + (number_of_files_on_this_disk, u64), + (number_of_files, u64), + (central_directory_size, u64), + (central_directory_offset, u64), + ]; +} + +pub(crate) struct Zip64CentralDirectoryEnd { + pub record_size: u64, + pub version_made_by: u16, + pub version_needed_to_extract: u16, + pub disk_number: u32, + pub disk_with_central_directory: u32, + pub number_of_files_on_this_disk: u64, + pub number_of_files: u64, + pub central_directory_size: u64, + pub central_directory_offset: u64, + pub extensible_data_sector: Box<[u8]>, +} + +impl Zip64CentralDirectoryEnd { + pub fn parse( + reader: &mut T, + max_size: u64, + ) -> ZipResult { + let Zip64CDEBlock { + record_size, + version_made_by, + version_needed_to_extract, + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + .. + } = Zip64CDEBlock::parse(reader)?; + + if record_size < 44 { + return Err(invalid!("Low EOCD64 record size")); + } else if record_size.saturating_add(12) > max_size { + return Err(invalid!("EOCD64 extends beyond EOCD64 locator")); + } + + let mut zip_file_comment = vec![0u8; record_size as usize - 44].into_boxed_slice(); + if let Err(e) = reader.read_exact(&mut zip_file_comment) { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!( + "EOCD64 extensible data sector exceeds file boundary" + )); + } + return Err(e.into()); + } + + Ok(Self { + record_size, + version_made_by, + version_needed_to_extract, + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + extensible_data_sector: zip_file_comment, + }) + } + + pub fn into_block_and_comment(self) -> (Zip64CDEBlock, Box<[u8]>) { + let Self { + record_size, + version_made_by, + version_needed_to_extract, + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + extensible_data_sector, + } = self; + + ( + Zip64CDEBlock { + magic: Zip64CDEBlock::MAGIC, + record_size, + version_made_by, + version_needed_to_extract, + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + }, + extensible_data_sector, + ) + } + + pub fn write(self, writer: &mut T) -> ZipResult<()> { + let (block, comment) = self.into_block_and_comment(); + block.write(writer)?; + writer.write_all(&comment)?; + Ok(()) + } +} + +pub(crate) struct DataAndPosition { + pub data: T, + #[allow(dead_code)] + pub position: u64, +} + +impl From<(T, u64)> for DataAndPosition { + fn from(value: (T, u64)) -> Self { + Self { + data: value.0, + position: value.1, + } + } +} + +pub(crate) struct CentralDirectoryEndInfo { + pub eocd: DataAndPosition, + pub eocd64: Option>, + + pub archive_offset: u64, +} + +/// Finds the EOCD and possibly the EOCD64 block and determines the archive offset. +/// +/// In the best case scenario (no prepended junk), this function will not backtrack +/// in the reader. +pub(crate) fn find_central_directory( + reader: &mut R, + archive_offset: ArchiveOffset, + end_exclusive: u64, + file_len: u64, +) -> ZipResult { + const EOCD_SIG_BYTES: [u8; mem::size_of::()] = + Magic::CENTRAL_DIRECTORY_END_SIGNATURE.to_le_bytes(); + + const EOCD64_SIG_BYTES: [u8; mem::size_of::()] = + Magic::ZIP64_CENTRAL_DIRECTORY_END_SIGNATURE.to_le_bytes(); + + const CDFH_SIG_BYTES: [u8; mem::size_of::()] = + Magic::CENTRAL_DIRECTORY_HEADER_SIGNATURE.to_le_bytes(); + + // Instantiate the mandatory finder + let mut eocd_finder = MagicFinder::>::new(&EOCD_SIG_BYTES, 0, end_exclusive); + let mut subfinder: Option>> = None; + + // Keep the last errors for cases of improper EOCD instances. + let mut parsing_error = None; + + while let Some(eocd_offset) = eocd_finder.next(reader)? { + // Attempt to parse the EOCD block + let eocd = match Zip32CentralDirectoryEnd::parse(reader) { + Ok(eocd) => eocd, + Err(e) => { + if parsing_error.is_none() { + parsing_error = Some(e); + } + continue; + } + }; + + // ! Relaxed (inequality) due to garbage-after-comment Python files + // Consistency check: the EOCD comment must terminate before the end of file + if eocd.zip_file_comment.len() as u64 + eocd_offset + 22 > file_len { + parsing_error = Some(invalid!("Invalid EOCD comment length")); + continue; + } + + let zip64_metadata = if eocd.may_be_zip64() { + fn try_read_eocd64_locator( + reader: &mut (impl Read + Seek + ?Sized), + eocd_offset: u64, + ) -> ZipResult<(u64, Zip64CentralDirectoryEndLocator)> { + if eocd_offset < mem::size_of::() as u64 { + return Err(invalid!("EOCD64 Locator does not fit in file")); + } + + let locator64_offset = eocd_offset - mem::size_of::() as u64; + + reader.seek(io::SeekFrom::Start(locator64_offset))?; + let locator64 = Zip64CentralDirectoryEndLocator::parse(reader); + Ok((locator64_offset, locator64?)) + } + + try_read_eocd64_locator(reader, eocd_offset).ok() + } else { + None + }; + + let Some((locator64_offset, locator64)) = zip64_metadata else { + // Branch out for zip32 + let relative_cd_offset = eocd.central_directory_offset as u64; + + // If the archive is empty, there is nothing more to be checked, the archive is correct. + if eocd.number_of_files == 0 { + return Ok(CentralDirectoryEndInfo { + eocd: (eocd, eocd_offset).into(), + eocd64: None, + archive_offset: eocd_offset.saturating_sub(relative_cd_offset), + }); + } + + // Consistency check: the CD relative offset cannot be after the EOCD + if relative_cd_offset >= eocd_offset { + parsing_error = Some(invalid!("Invalid CDFH offset in EOCD")); + continue; + } + + // Attempt to find the first CDFH + let subfinder = subfinder + .get_or_insert_with(OptimisticMagicFinder::new_empty) + .repurpose( + &CDFH_SIG_BYTES, + // The CDFH must be before the EOCD and after the relative offset, + // because prepended junk can only move it forward. + (relative_cd_offset, eocd_offset), + match archive_offset { + ArchiveOffset::Known(n) => { + Some((relative_cd_offset.saturating_add(n).min(eocd_offset), true)) + } + _ => Some((relative_cd_offset, false)), + }, + ); + + // Consistency check: find the first CDFH + if let Some(cd_offset) = subfinder.next(reader)? { + // The first CDFH will define the archive offset + let archive_offset = cd_offset - relative_cd_offset; + + return Ok(CentralDirectoryEndInfo { + eocd: (eocd, eocd_offset).into(), + eocd64: None, + archive_offset, + }); + } + + parsing_error = Some(invalid!("No CDFH found")); + continue; + }; + + // Consistency check: the EOCD64 offset must be before EOCD64 Locator offset */ + if locator64.end_of_central_directory_offset >= locator64_offset { + parsing_error = Some(invalid!("Invalid EOCD64 Locator CD offset")); + continue; + } + + if locator64.number_of_disks > 1 { + parsing_error = Some(invalid!("Multi-disk ZIP files are not supported")); + continue; + } + + // This was hidden inside a function to collect errors in a single place. + // Once try blocks are stabilized, this can go away. + fn try_read_eocd64( + reader: &mut R, + locator64: &Zip64CentralDirectoryEndLocator, + expected_length: u64, + ) -> ZipResult { + let z64 = Zip64CentralDirectoryEnd::parse(reader, expected_length)?; + + // Consistency check: EOCD64 locator should agree with the EOCD64 + if z64.disk_with_central_directory != locator64.disk_with_central_directory { + return Err(invalid!("Invalid EOCD64: inconsistency with Locator data")); + } + + // Consistency check: the EOCD64 must have the expected length + if z64.record_size + 12 != expected_length { + return Err(invalid!("Invalid EOCD64: inconsistent length")); + } + + Ok(z64) + } + + // Attempt to find the EOCD64 with an initial guess + let subfinder = subfinder + .get_or_insert_with(OptimisticMagicFinder::new_empty) + .repurpose( + &EOCD64_SIG_BYTES, + (locator64.end_of_central_directory_offset, locator64_offset), + match archive_offset { + ArchiveOffset::Known(n) => Some(( + locator64 + .end_of_central_directory_offset + .saturating_add(n) + .min(locator64_offset), + true, + )), + _ => Some((locator64.end_of_central_directory_offset, false)), + }, + ); + + // Consistency check: Find the EOCD64 + let mut local_error = None; + while let Some(eocd64_offset) = subfinder.next(reader)? { + let archive_offset = eocd64_offset - locator64.end_of_central_directory_offset; + + match try_read_eocd64( + reader, + &locator64, + locator64_offset.saturating_sub(eocd64_offset), + ) { + Ok(eocd64) => { + if eocd64_offset + < eocd64 + .number_of_files + .saturating_mul( + mem::size_of::() as u64 + ) + .saturating_add(eocd64.central_directory_offset) + { + local_error = + Some(invalid!("Invalid EOCD64: inconsistent number of files")); + continue; + } + + return Ok(CentralDirectoryEndInfo { + eocd: (eocd, eocd_offset).into(), + eocd64: Some((eocd64, eocd64_offset).into()), + archive_offset, + }); + } + Err(e) => { + local_error = Some(e); + } + } + } + + parsing_error = local_error.or(Some(invalid!("Could not find EOCD64"))); + } + + Err(parsing_error.unwrap_or(invalid!("Could not find EOCD"))) +} + +pub(crate) fn is_dir(filename: &str) -> bool { + filename + .chars() + .next_back() + .is_some_and(|c| c == '/' || c == '\\') +} + +#[cfg(test)] +mod test { + use std::io::Cursor; + + use crate::{ + result::{invalid, ZipError}, + spec::{FixedSizeBlock, Magic, Pod}, + }; + + #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] + #[repr(packed, C)] + pub struct TestBlock { + magic: Magic, + pub file_name_length: u16, + } + + unsafe impl Pod for TestBlock {} + + impl FixedSizeBlock for TestBlock { + const MAGIC: Magic = Magic::literal(0x01111); + + fn magic(self) -> Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("unreachable"); + + to_and_from_le![(magic, Magic), (file_name_length, u16)]; + } + + /// Demonstrate that a block object can be safely written to memory and deserialized back out. + #[test] + fn block_serde() { + let block = TestBlock { + magic: TestBlock::MAGIC, + file_name_length: 3, + }; + let mut c = Cursor::new(Vec::new()); + block.write(&mut c).unwrap(); + c.set_position(0); + let block2 = TestBlock::parse(&mut c).unwrap(); + assert_eq!(block, block2); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/types.rs b/v2/vendor/security/zip-7.2.0/src/types.rs new file mode 100644 index 0000000000..cb906c1175 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/types.rs @@ -0,0 +1,1768 @@ +//! Types that specify what is contained in a ZIP. +use crate::cfg_if_expr; +use crate::cp437::FromCp437; +use crate::write::FileOptionExtension; +use crate::zipcrypto::EncryptWith; +use core::fmt::{self, Debug, Formatter}; +use core::mem; +use std::ffi::OsStr; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, OnceLock}; +use typed_path::{Utf8WindowsComponent, Utf8WindowsPath}; + +#[cfg(feature = "chrono")] +use chrono::{Datelike, NaiveDate, NaiveDateTime, NaiveTime, Timelike}; +#[cfg(feature = "jiff-02")] +use jiff::civil; + +use crate::result::{invalid, ZipError, ZipResult}; +use crate::spec::{self, FixedSizeBlock, Pod}; + +pub(crate) mod ffi { + pub const S_IFDIR: u32 = 0o0040000; + pub const S_IFREG: u32 = 0o0100000; + pub const S_IFLNK: u32 = 0o0120000; +} + +use crate::extra_fields::ExtraField; +use crate::read::find_data_start; +use crate::result::DateTimeRangeError; +use crate::spec::is_dir; +use crate::types::ffi::S_IFDIR; +use crate::{CompressionMethod, ZIP64_BYTES_THR}; +use std::io::{Read, Seek}; +#[cfg(feature = "time")] +use time::{error::ComponentRange, Date, Month, OffsetDateTime, PrimitiveDateTime, Time}; + +pub(crate) struct ZipRawValues { + pub(crate) crc32: u32, + pub(crate) compressed_size: u64, + pub(crate) uncompressed_size: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +#[repr(u8)] +pub enum System { + Dos = 0, + Unix = 3, + #[default] + Unknown, +} + +impl From for System { + fn from(system: u8) -> Self { + match system { + 0 => Self::Dos, + 3 => Self::Unix, + _ => Self::Unknown, + } + } +} + +impl From for u8 { + fn from(system: System) -> Self { + match system { + System::Dos => 0, + System::Unix => 3, + System::Unknown => 4, + } + } +} + +/// Metadata for a file to be written +#[derive(Clone, Debug, Copy, Eq, PartialEq)] +pub struct FileOptions<'k, T: FileOptionExtension> { + pub(crate) compression_method: CompressionMethod, + pub(crate) compression_level: Option, + pub(crate) last_modified_time: DateTime, + pub(crate) permissions: Option, + pub(crate) large_file: bool, + pub(crate) encrypt_with: Option>, + pub(crate) extended_options: T, + pub(crate) alignment: u16, + #[cfg(feature = "deflate-zopfli")] + pub(super) zopfli_buffer_size: Option, + #[cfg(feature = "aes-crypto")] + pub(crate) aes_mode: Option<(AesMode, AesVendorVersion, CompressionMethod)>, +} +/// Simple File Options. Can be copied and good for simple writing zip files +pub type SimpleFileOptions = FileOptions<'static, ()>; + +/// Representation of a moment in time. +/// +/// Zip files use an old format from DOS to store timestamps, +/// with its own set of peculiarities. +/// For example, it has a resolution of 2 seconds! +/// +/// A [`DateTime`] can be stored directly in a zipfile with [`FileOptions::last_modified_time`], +/// or read from one with [`ZipFile::last_modified`](crate::read::ZipFile::last_modified). +/// +/// # Warning +/// +/// Because there is no timezone associated with the [`DateTime`], they should ideally only +/// be used for user-facing descriptions. +/// +/// Modern zip files store more precise timestamps; see [`crate::extra_fields::ExtendedTimestamp`] +/// for details. +#[derive(Clone, Copy, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct DateTime { + datepart: u16, + timepart: u16, +} + +impl Debug for DateTime { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + if *self == Self::default() { + return f.write_str("DateTime::default()"); + } + f.write_fmt(format_args!( + "DateTime::from_date_and_time({}, {}, {}, {}, {}, {})?", + self.year(), + self.month(), + self.day(), + self.hour(), + self.minute(), + self.second() + )) + } +} + +impl DateTime { + /// Constructs a default datetime of 1980-01-01 00:00:00. + pub const DEFAULT: Self = DateTime { + datepart: 0b0000000000100001, + timepart: 0, + }; + + /// Returns the current time if possible, otherwise the default of 1980-01-01. + #[cfg(feature = "time")] + pub fn default_for_write() -> Self { + let now = OffsetDateTime::now_utc(); + PrimitiveDateTime::new(now.date(), now.time()) + .try_into() + .unwrap_or_else(|_| DateTime::default()) + } + + /// Returns the current time if possible, otherwise the default of 1980-01-01. + #[cfg(not(feature = "time"))] + pub fn default_for_write() -> Self { + DateTime::default() + } +} + +#[cfg(feature = "_arbitrary")] +impl arbitrary::Arbitrary<'_> for DateTime { + fn arbitrary(u: &mut arbitrary::Unstructured) -> arbitrary::Result { + let year: u16 = u.int_in_range(1980..=2107)?; + let month: u16 = u.int_in_range(1..=12)?; + let day: u16 = u.int_in_range(1..=31)?; + let datepart = day | (month << 5) | ((year - 1980) << 9); + let hour: u16 = u.int_in_range(0..=23)?; + let minute: u16 = u.int_in_range(0..=59)?; + let second: u16 = u.int_in_range(0..=58)?; + let timepart = (second >> 1) | (minute << 5) | (hour << 11); + Ok(DateTime { datepart, timepart }) + } +} + +#[cfg(feature = "chrono")] +impl TryFrom for DateTime { + type Error = DateTimeRangeError; + + fn try_from(value: NaiveDateTime) -> Result { + DateTime::from_date_and_time( + value.year().try_into()?, + value.month().try_into()?, + value.day().try_into()?, + value.hour().try_into()?, + value.minute().try_into()?, + value.second().try_into()?, + ) + } +} + +#[cfg(feature = "chrono")] +impl TryFrom for NaiveDateTime { + type Error = DateTimeRangeError; + + fn try_from(value: DateTime) -> Result { + let date = NaiveDate::from_ymd_opt( + value.year().into(), + value.month().into(), + value.day().into(), + ) + .ok_or(DateTimeRangeError)?; + let time = NaiveTime::from_hms_opt( + value.hour().into(), + value.minute().into(), + value.second().into(), + ) + .ok_or(DateTimeRangeError)?; + Ok(NaiveDateTime::new(date, time)) + } +} + +#[cfg(feature = "jiff-02")] +impl TryFrom for DateTime { + type Error = DateTimeRangeError; + + fn try_from(value: civil::DateTime) -> Result { + Self::from_date_and_time( + value.year().try_into()?, + value.month() as u8, + value.day() as u8, + value.hour() as u8, + value.minute() as u8, + value.second() as u8, + ) + } +} + +#[cfg(feature = "jiff-02")] +impl TryFrom for civil::DateTime { + type Error = jiff::Error; + + fn try_from(value: DateTime) -> Result { + Self::new( + value.year() as i16, + value.month() as i8, + value.day() as i8, + value.hour() as i8, + value.minute() as i8, + value.second() as i8, + 0, + ) + } +} + +impl TryFrom<(u16, u16)> for DateTime { + type Error = DateTimeRangeError; + + #[inline] + fn try_from(values: (u16, u16)) -> Result { + Self::try_from_msdos(values.0, values.1) + } +} + +impl From for (u16, u16) { + #[inline] + fn from(dt: DateTime) -> Self { + (dt.datepart(), dt.timepart()) + } +} + +impl Default for DateTime { + /// Constructs an 'default' datetime of 1980-01-01 00:00:00 + fn default() -> DateTime { + DateTime::DEFAULT + } +} + +impl fmt::Display for DateTime { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{:04}-{:02}-{:02} {:02}:{:02}:{:02}", + self.year(), + self.month(), + self.day(), + self.hour(), + self.minute(), + self.second() + ) + } +} + +impl DateTime { + /// Converts an msdos (u16, u16) pair to a DateTime object + /// + /// # Safety + /// The caller must ensure the date and time are valid. + pub const unsafe fn from_msdos_unchecked(datepart: u16, timepart: u16) -> DateTime { + DateTime { datepart, timepart } + } + + /// Converts an msdos (u16, u16) pair to a DateTime object if it represents a valid date and + /// time. + pub fn try_from_msdos(datepart: u16, timepart: u16) -> Result { + let seconds = (timepart & 0b0000000000011111) << 1; + let minutes = (timepart & 0b0000011111100000) >> 5; + let hours = (timepart & 0b1111100000000000) >> 11; + let days = datepart & 0b0000000000011111; + let months = (datepart & 0b0000000111100000) >> 5; + let years = (datepart & 0b1111111000000000) >> 9; + Self::from_date_and_time( + years.checked_add(1980).ok_or(DateTimeRangeError)?, + months.try_into()?, + days.try_into()?, + hours.try_into()?, + minutes.try_into()?, + seconds.try_into()?, + ) + } + + /// Constructs a DateTime from a specific date and time + /// + /// The bounds are: + /// * year: [1980, 2107] + /// * month: [1, 12] + /// * day: [1, 28..=31] + /// * hour: [0, 23] + /// * minute: [0, 59] + /// * second: [0, 58] + pub fn from_date_and_time( + year: u16, + month: u8, + day: u8, + hour: u8, + minute: u8, + second: u8, + ) -> Result { + fn is_leap_year(year: u16) -> bool { + (year % 4 == 0) && ((year % 25 != 0) || (year % 16 == 0)) + } + + if (1980..=2107).contains(&year) + && (1..=12).contains(&month) + && (1..=31).contains(&day) + && hour <= 23 + && minute <= 59 + && second <= 60 + { + let second = second.min(58); // exFAT can't store leap seconds + let max_day = match month { + 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31, + 4 | 6 | 9 | 11 => 30, + 2 if is_leap_year(year) => 29, + 2 => 28, + _ => unreachable!(), + }; + if day > max_day { + return Err(DateTimeRangeError); + } + let datepart = (day as u16) | ((month as u16) << 5) | ((year - 1980) << 9); + let timepart = ((second as u16) >> 1) | ((minute as u16) << 5) | ((hour as u16) << 11); + Ok(DateTime { datepart, timepart }) + } else { + Err(DateTimeRangeError) + } + } + + /// Indicates whether this date and time can be written to a zip archive. + pub fn is_valid(&self) -> bool { + Self::try_from_msdos(self.datepart, self.timepart).is_ok() + } + + #[cfg(feature = "time")] + /// Converts a OffsetDateTime object to a DateTime + /// + /// Returns `Err` when this object is out of bounds + #[deprecated(since = "0.6.4", note = "use `DateTime::try_from()` instead")] + pub fn from_time(dt: OffsetDateTime) -> Result { + dt.try_into() + } + + /// Gets the time portion of this datetime in the msdos representation + pub const fn timepart(&self) -> u16 { + self.timepart + } + + /// Gets the date portion of this datetime in the msdos representation + pub const fn datepart(&self) -> u16 { + self.datepart + } + + #[cfg(feature = "time")] + /// Converts the DateTime to a OffsetDateTime structure + #[deprecated(since = "1.3.1", note = "use `OffsetDateTime::try_from()` instead")] + pub fn to_time(&self) -> Result { + (*self).try_into() + } + + /// Get the year. There is no epoch, i.e. 2018 will be returned as 2018. + pub const fn year(&self) -> u16 { + (self.datepart >> 9) + 1980 + } + + /// Get the month, where 1 = january and 12 = december + /// + /// # Warning + /// + /// When read from a zip file, this may not be a reasonable value + pub const fn month(&self) -> u8 { + ((self.datepart & 0b0000000111100000) >> 5) as u8 + } + + /// Get the day + /// + /// # Warning + /// + /// When read from a zip file, this may not be a reasonable value + pub const fn day(&self) -> u8 { + (self.datepart & 0b0000000000011111) as u8 + } + + /// Get the hour + /// + /// # Warning + /// + /// When read from a zip file, this may not be a reasonable value + pub const fn hour(&self) -> u8 { + (self.timepart >> 11) as u8 + } + + /// Get the minute + /// + /// # Warning + /// + /// When read from a zip file, this may not be a reasonable value + pub const fn minute(&self) -> u8 { + ((self.timepart & 0b0000011111100000) >> 5) as u8 + } + + /// Get the second + /// + /// # Warning + /// + /// When read from a zip file, this may not be a reasonable value + pub const fn second(&self) -> u8 { + ((self.timepart & 0b0000000000011111) << 1) as u8 + } +} + +#[cfg(feature = "time")] +impl TryFrom for DateTime { + type Error = DateTimeRangeError; + + fn try_from(dt: OffsetDateTime) -> Result { + Self::try_from(PrimitiveDateTime::new(dt.date(), dt.time())) + } +} + +#[cfg(feature = "time")] +impl TryFrom for DateTime { + type Error = DateTimeRangeError; + + fn try_from(dt: PrimitiveDateTime) -> Result { + Self::from_date_and_time( + dt.year().try_into()?, + dt.month().into(), + dt.day(), + dt.hour(), + dt.minute(), + dt.second(), + ) + } +} + +#[cfg(feature = "time")] +impl TryFrom for OffsetDateTime { + type Error = ComponentRange; + + fn try_from(dt: DateTime) -> Result { + PrimitiveDateTime::try_from(dt).map(PrimitiveDateTime::assume_utc) + } +} + +#[cfg(feature = "time")] +impl TryFrom for PrimitiveDateTime { + type Error = ComponentRange; + + fn try_from(dt: DateTime) -> Result { + let date = + Date::from_calendar_date(dt.year() as i32, Month::try_from(dt.month())?, dt.day())?; + let time = Time::from_hms(dt.hour(), dt.minute(), dt.second())?; + Ok(PrimitiveDateTime::new(date, time)) + } +} + +pub const MIN_VERSION: u8 = 10; +pub const DEFAULT_VERSION: u8 = 45; + +/// Structure representing a ZIP file. +#[derive(Debug, Clone, Default)] +pub struct ZipFileData { + /// Compatibility of the file attribute information + pub system: System, + /// Specification version + pub version_made_by: u8, + /// ZIP flags + pub flags: u16, + /// True if the file is encrypted. + pub encrypted: bool, + /// True if file_name and file_comment are UTF8 + pub is_utf8: bool, + /// True if the file uses a data-descriptor section + pub using_data_descriptor: bool, + /// Compression method used to store the file + pub compression_method: crate::compression::CompressionMethod, + /// Compression level to store the file + pub compression_level: Option, + /// Last modified time. This will only have a 2 second precision. + pub last_modified_time: Option, + /// CRC32 checksum + pub crc32: u32, + /// Size of the file in the ZIP + pub compressed_size: u64, + /// Size of the file when extracted + pub uncompressed_size: u64, + /// Name of the file + pub file_name: Box, + /// Raw file name. To be used when file_name was incorrectly decoded. + pub file_name_raw: Box<[u8]>, + /// Extra field usually used for storage expansion + pub extra_field: Option>>, + /// Extra field only written to central directory + pub central_extra_field: Option>>, + /// File comment + pub file_comment: Box, + /// Specifies where the local header of the file starts + pub header_start: u64, + /// Specifies where the extra data of the file starts + pub extra_data_start: Option, + /// Specifies where the central header of the file starts + /// + /// Note that when this is not known, it is set to 0 + pub central_header_start: u64, + /// Specifies where the compressed data of the file starts + pub data_start: OnceLock, + /// External file attributes + pub external_attributes: u32, + /// Reserve local ZIP64 extra field + pub large_file: bool, + /// AES mode if applicable + pub aes_mode: Option<(AesMode, AesVendorVersion, CompressionMethod)>, + /// Specifies where in the extra data the AES metadata starts + pub aes_extra_data_start: u64, + + /// extra fields, see + pub extra_fields: Vec, +} + +impl ZipFileData { + /// Get the starting offset of the data of the compressed file + pub fn data_start(&self, reader: &mut (impl Read + Seek + ?Sized)) -> ZipResult { + match self.data_start.get() { + Some(data_start) => Ok(*data_start), + None => Ok(find_data_start(self, reader)?), + } + } + + #[allow(dead_code)] + pub fn is_dir(&self) -> bool { + is_dir(&self.file_name) + } + + pub fn file_name_sanitized(&self) -> PathBuf { + let no_null_filename = match self.file_name.find('\0') { + Some(index) => &self.file_name[0..index], + None => &self.file_name, + }; + + Utf8WindowsPath::new(no_null_filename) + .components() + .filter(|component| matches!(*component, Utf8WindowsComponent::Normal(..))) + .fold(PathBuf::new(), |mut path, cur| { + if let Utf8WindowsComponent::Normal(s) = cur { + path.push(s); + } + path + }) + } + + /// Simplify the file name by removing the prefix and parent directories and only return normal components + pub(crate) fn simplified_components(&self) -> Option> { + if self.file_name.contains('\0') { + return None; + } + let input = Path::new(OsStr::new(&*self.file_name)); + crate::path::simplified_components(input) + } + + pub(crate) fn enclosed_name(&self) -> Option { + if self.file_name.contains('\0') { + return None; + } + let mut depth = 0usize; + let mut out_path = PathBuf::new(); + for component in Utf8WindowsPath::new(&self.file_name).components() { + match component { + Utf8WindowsComponent::Prefix(_) | Utf8WindowsComponent::RootDir => { + if depth > 0 { + return None; + } + } + Utf8WindowsComponent::ParentDir => { + depth = depth.checked_sub(1)?; + out_path.pop(); + } + Utf8WindowsComponent::Normal(s) => { + depth += 1; + out_path.push(s); + } + Utf8WindowsComponent::CurDir => (), + } + } + Some(out_path) + } + + /// Get unix mode for the file + pub(crate) const fn unix_mode(&self) -> Option { + if self.external_attributes == 0 { + return None; + } + let unix_mode = self.external_attributes >> 16; + if unix_mode != 0 { + // If the high 16 bits are non-zero, they probably contain Unix permissions. + // This happens for archives created on Windows by this crate or other tools, + // and is the only way to identify symlinks in such archives. + return Some(unix_mode); + } + match self.system { + System::Unix => Some(unix_mode), + System::Dos => { + // Interpret MS-DOS directory bit + let mut mode = if 0x10 == (self.external_attributes & 0x10) { + ffi::S_IFDIR | 0o0775 + } else { + ffi::S_IFREG | 0o0664 + }; + if 0x01 == (self.external_attributes & 0x01) { + // Read-only bit; strip write permissions + mode &= !0o222; + } + Some(mode) + } + _ => None, + } + } + + /// PKZIP version needed to open this file (from APPNOTE 4.4.3.2). + pub fn version_needed(&self) -> u16 { + let compression_version: u16 = match self.compression_method { + CompressionMethod::Stored => MIN_VERSION.into(), + #[cfg(feature = "_deflate-any")] + CompressionMethod::Deflated => 20, + #[cfg(feature = "_bzip2_any")] + CompressionMethod::Bzip2 => 46, + #[cfg(feature = "deflate64")] + CompressionMethod::Deflate64 => 21, + #[cfg(feature = "lzma")] + CompressionMethod::Lzma => 63, + #[cfg(feature = "xz")] + CompressionMethod::Xz => 63, + // APPNOTE doesn't specify a version for Zstandard + _ => DEFAULT_VERSION as u16, + }; + let crypto_version: u16 = if self.aes_mode.is_some() { + 51 + } else if self.encrypted { + 20 + } else { + 10 + }; + let misc_feature_version: u16 = if self.large_file { + 45 + } else if self + .unix_mode() + .is_some_and(|mode| mode & S_IFDIR == S_IFDIR) + { + // file is directory + 20 + } else { + 10 + }; + compression_version + .max(crypto_version) + .max(misc_feature_version) + } + #[inline(always)] + pub(crate) fn extra_field_len(&self) -> usize { + self.extra_field + .as_ref() + .map(|v| v.len()) + .unwrap_or_default() + } + #[inline(always)] + pub(crate) fn central_extra_field_len(&self) -> usize { + self.central_extra_field + .as_ref() + .map(|v| v.len()) + .unwrap_or_default() + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn initialize_local_block( + name: S, + options: &FileOptions, + raw_values: ZipRawValues, + header_start: u64, + extra_data_start: Option, + aes_extra_data_start: u64, + compression_method: crate::compression::CompressionMethod, + aes_mode: Option<(AesMode, AesVendorVersion, CompressionMethod)>, + extra_field: &[u8], + ) -> Self + where + S: ToString, + { + let permissions = options.permissions.unwrap_or(0o100644); + let file_name: Box = name.to_string().into_boxed_str(); + let file_name_raw: Box<[u8]> = file_name.bytes().collect(); + let mut external_attributes = permissions << 16; + let system = if (permissions & ffi::S_IFLNK) == ffi::S_IFLNK { + System::Unix + } else if cfg!(windows) { + if is_dir(&file_name) { + // DOS directory bit + external_attributes |= 0x10; + } + if options + .permissions + .is_some_and(|permissions| permissions & 0o444 == 0) + { + // DOS read-only bit + external_attributes |= 0x01; + } + System::Dos + } else { + System::Unix + }; + let mut local_block = ZipFileData { + system, + version_made_by: DEFAULT_VERSION, + flags: 0, + encrypted: options.encrypt_with.is_some() + || cfg_if_expr! { + #[cfg(feature = "aes-crypto")] => options.aes_mode.is_some(), + _ => false + }, + using_data_descriptor: false, + is_utf8: !file_name.is_ascii(), + compression_method, + compression_level: options.compression_level, + last_modified_time: Some(options.last_modified_time), + crc32: raw_values.crc32, + compressed_size: raw_values.compressed_size, + uncompressed_size: raw_values.uncompressed_size, + file_name, // Never used for saving, but used as map key in insert_file_data() + file_name_raw, + extra_field: Some(extra_field.to_vec().into()), + central_extra_field: options.extended_options.central_extra_data().cloned(), + file_comment: String::with_capacity(0).into_boxed_str(), + header_start, + data_start: OnceLock::new(), + central_header_start: 0, + external_attributes, + large_file: options.large_file, + aes_mode, + extra_fields: Vec::new(), + extra_data_start, + aes_extra_data_start, + }; + local_block.version_made_by = local_block.version_needed() as u8; + local_block + } + + pub(crate) fn from_local_block( + block: ZipLocalEntryBlock, + reader: &mut R, + ) -> ZipResult { + let ZipLocalEntryBlock { + // magic, + version_made_by, + flags, + compression_method, + last_mod_time, + last_mod_date, + crc32, + compressed_size, + uncompressed_size, + file_name_length, + extra_field_length, + .. + } = block; + + let encrypted: bool = flags & 1 == 1; + if encrypted { + return Err(ZipError::UnsupportedArchive( + "Encrypted files are not supported", + )); + } + + /* FIXME: these were previously incorrect: add testing! */ + /* flags & (1 << 3) != 0 */ + let using_data_descriptor: bool = flags & (1 << 3) == 1 << 3; + if using_data_descriptor { + return Err(ZipError::UnsupportedArchive( + "The file length is not available in the local header", + )); + } + + /* flags & (1 << 1) != 0 */ + let is_utf8: bool = flags & (1 << 11) != 0; + let compression_method = crate::CompressionMethod::parse_from_u16(compression_method); + let file_name_length: usize = file_name_length.into(); + let extra_field_length: usize = extra_field_length.into(); + + let mut file_name_raw = vec![0u8; file_name_length]; + if let Err(e) = reader.read_exact(&mut file_name_raw) { + if e.kind() == std::io::ErrorKind::UnexpectedEof { + return Err(invalid!("File name extends beyond file boundary")); + } + return Err(e.into()); + } + let mut extra_field = vec![0u8; extra_field_length]; + if let Err(e) = reader.read_exact(&mut extra_field) { + if e.kind() == std::io::ErrorKind::UnexpectedEof { + return Err(invalid!("Extra field extends beyond file boundary")); + } + return Err(e.into()); + } + + let file_name: Box = match is_utf8 { + true => String::from_utf8_lossy(&file_name_raw).into(), + false => file_name_raw.clone().from_cp437().into(), + }; + + let system: u8 = (version_made_by >> 8).try_into().unwrap(); + Ok(ZipFileData { + system: System::from(system), + /* NB: this strips the top 8 bits! */ + version_made_by: version_made_by as u8, + flags, + encrypted, + using_data_descriptor, + is_utf8, + compression_method, + compression_level: None, + last_modified_time: DateTime::try_from_msdos(last_mod_date, last_mod_time).ok(), + crc32, + compressed_size: compressed_size.into(), + uncompressed_size: uncompressed_size.into(), + file_name, + file_name_raw: file_name_raw.into(), + extra_field: Some(Arc::new(extra_field)), + central_extra_field: None, + file_comment: String::with_capacity(0).into_boxed_str(), // file comment is only available in the central directory + // header_start and data start are not available, but also don't matter, since seeking is + // not available. + header_start: 0, + data_start: OnceLock::new(), + central_header_start: 0, + // The external_attributes field is only available in the central directory. + // We set this to zero, which should be valid as the docs state 'If input came + // from standard input, this field is set to zero.' + external_attributes: 0, + large_file: false, + aes_mode: None, + extra_fields: Vec::new(), + extra_data_start: None, + aes_extra_data_start: 0, + }) + } + + fn is_utf8(&self) -> bool { + std::str::from_utf8(&self.file_name_raw).is_ok() + } + + fn is_ascii(&self) -> bool { + self.file_name_raw.is_ascii() + } + + fn flags(&self) -> u16 { + let utf8_bit: u16 = if self.is_utf8() && !self.is_ascii() { + 1u16 << 11 + } else { + 0 + }; + + let using_data_descriptor_bit = if self.using_data_descriptor { + 1u16 << 3 + } else { + 0 + }; + + let encrypted_bit: u16 = if self.encrypted { 1u16 << 0 } else { 0 }; + + utf8_bit | using_data_descriptor_bit | encrypted_bit + } + + fn clamp_size_field(&self, field: u64) -> u32 { + if self.large_file { + spec::ZIP64_BYTES_THR as u32 + } else { + field.min(spec::ZIP64_BYTES_THR).try_into().unwrap() + } + } + + pub(crate) fn local_block(&self) -> ZipResult { + let (compressed_size, uncompressed_size) = if self.using_data_descriptor { + (0, 0) + } else { + ( + self.clamp_size_field(self.compressed_size), + self.clamp_size_field(self.uncompressed_size), + ) + }; + let extra_field_length: u16 = self + .extra_field_len() + .try_into() + .map_err(|_| invalid!("Extra data field is too large"))?; + + let last_modified_time = self + .last_modified_time + .unwrap_or_else(DateTime::default_for_write); + Ok(ZipLocalEntryBlock { + magic: ZipLocalEntryBlock::MAGIC, + version_made_by: self.version_needed(), + flags: self.flags(), + compression_method: self.compression_method.serialize_to_u16(), + last_mod_time: last_modified_time.timepart(), + last_mod_date: last_modified_time.datepart(), + crc32: self.crc32, + compressed_size, + uncompressed_size, + file_name_length: self.file_name_raw.len().try_into().unwrap(), + extra_field_length, + }) + } + + pub(crate) fn block(&self) -> ZipResult { + let extra_field_len: u16 = self.extra_field_len().try_into().unwrap(); + let central_extra_field_len: u16 = self.central_extra_field_len().try_into().unwrap(); + let last_modified_time = self + .last_modified_time + .unwrap_or_else(DateTime::default_for_write); + let version_to_extract = self.version_needed(); + let version_made_by = (self.version_made_by as u16).max(version_to_extract); + Ok(ZipCentralEntryBlock { + magic: ZipCentralEntryBlock::MAGIC, + version_made_by: ((self.system as u16) << 8) | version_made_by, + version_to_extract, + flags: self.flags(), + compression_method: self.compression_method.serialize_to_u16(), + last_mod_time: last_modified_time.timepart(), + last_mod_date: last_modified_time.datepart(), + crc32: self.crc32, + compressed_size: self + .compressed_size + .min(spec::ZIP64_BYTES_THR) + .try_into() + .unwrap(), + uncompressed_size: self + .uncompressed_size + .min(spec::ZIP64_BYTES_THR) + .try_into() + .unwrap(), + file_name_length: self.file_name_raw.len().try_into().unwrap(), + extra_field_length: extra_field_len.checked_add(central_extra_field_len).ok_or( + invalid!("Extra field length in central directory exceeds 64KiB"), + )?, + file_comment_length: self.file_comment.len().try_into().unwrap(), + disk_number: 0, + internal_file_attributes: 0, + external_file_attributes: self.external_attributes, + offset: self + .header_start + .min(spec::ZIP64_BYTES_THR) + .try_into() + .unwrap(), + }) + } + + pub(crate) fn zip64_extra_field_block(&self) -> Option { + Zip64ExtraFieldBlock::maybe_new( + self.large_file, + self.uncompressed_size, + self.compressed_size, + self.header_start, + ) + } + + pub(crate) fn write_data_descriptor( + &self, + writer: &mut W, + auto_large_file: bool, + ) -> Result<(), ZipError> { + if self.large_file { + return self.zip64_data_descriptor_block().write(writer); + } + if self.compressed_size > spec::ZIP64_BYTES_THR + || self.uncompressed_size > spec::ZIP64_BYTES_THR + { + if auto_large_file { + return self.zip64_data_descriptor_block().write(writer); + } + return Err(ZipError::Io(std::io::Error::other( + "Large file option has not been set - use .large_file(true) in options", + ))); + } + self.data_descriptor_block().write(writer) + } + + pub(crate) fn data_descriptor_block(&self) -> ZipDataDescriptorBlock { + ZipDataDescriptorBlock { + magic: ZipDataDescriptorBlock::MAGIC, + crc32: self.crc32, + compressed_size: self.compressed_size as u32, + uncompressed_size: self.uncompressed_size as u32, + } + } + + pub(crate) fn zip64_data_descriptor_block(&self) -> Zip64DataDescriptorBlock { + Zip64DataDescriptorBlock { + magic: Zip64DataDescriptorBlock::MAGIC, + crc32: self.crc32, + compressed_size: self.compressed_size, + uncompressed_size: self.uncompressed_size, + } + } +} + +#[derive(Copy, Clone, Debug)] +#[repr(packed, C)] +pub(crate) struct ZipCentralEntryBlock { + magic: spec::Magic, + pub version_made_by: u16, + pub version_to_extract: u16, + pub flags: u16, + pub compression_method: u16, + pub last_mod_time: u16, + pub last_mod_date: u16, + pub crc32: u32, + pub compressed_size: u32, + pub uncompressed_size: u32, + pub file_name_length: u16, + pub extra_field_length: u16, + pub file_comment_length: u16, + pub disk_number: u16, + pub internal_file_attributes: u16, + pub external_file_attributes: u32, + pub offset: u32, +} + +unsafe impl Pod for ZipCentralEntryBlock {} + +impl FixedSizeBlock for ZipCentralEntryBlock { + const MAGIC: spec::Magic = spec::Magic::CENTRAL_DIRECTORY_HEADER_SIGNATURE; + + #[inline(always)] + fn magic(self) -> spec::Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("Invalid Central Directory header"); + + to_and_from_le![ + (magic, spec::Magic), + (version_made_by, u16), + (version_to_extract, u16), + (flags, u16), + (compression_method, u16), + (last_mod_time, u16), + (last_mod_date, u16), + (crc32, u32), + (compressed_size, u32), + (uncompressed_size, u32), + (file_name_length, u16), + (extra_field_length, u16), + (file_comment_length, u16), + (disk_number, u16), + (internal_file_attributes, u16), + (external_file_attributes, u32), + (offset, u32), + ]; +} + +#[derive(Copy, Clone, Debug)] +#[repr(packed, C)] +pub(crate) struct ZipLocalEntryBlock { + magic: spec::Magic, + pub version_made_by: u16, + pub flags: u16, + pub compression_method: u16, + pub last_mod_time: u16, + pub last_mod_date: u16, + pub crc32: u32, + pub compressed_size: u32, + pub uncompressed_size: u32, + pub file_name_length: u16, + pub extra_field_length: u16, +} + +unsafe impl Pod for ZipLocalEntryBlock {} + +impl FixedSizeBlock for ZipLocalEntryBlock { + const MAGIC: spec::Magic = spec::Magic::LOCAL_FILE_HEADER_SIGNATURE; + + #[inline(always)] + fn magic(self) -> spec::Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("Invalid local file header"); + + to_and_from_le![ + (magic, spec::Magic), + (version_made_by, u16), + (flags, u16), + (compression_method, u16), + (last_mod_time, u16), + (last_mod_date, u16), + (crc32, u32), + (compressed_size, u32), + (uncompressed_size, u32), + (file_name_length, u16), + (extra_field_length, u16), + ]; +} + +#[derive(Copy, Clone, Debug)] +pub(crate) struct Zip64ExtraFieldBlock { + magic: spec::ExtraFieldMagic, + size: u16, + uncompressed_size: Option, + compressed_size: Option, + header_start: Option, + // Excluded fields: + // u32: disk start number +} + +impl Zip64ExtraFieldBlock { + pub(crate) fn maybe_new( + large_file: bool, + uncompressed_size: u64, + compressed_size: u64, + header_start: u64, + ) -> Option { + let mut size: u16 = 0; + let uncompressed_size = if uncompressed_size >= ZIP64_BYTES_THR || large_file { + size += mem::size_of::() as u16; + Some(uncompressed_size) + } else { + None + }; + let compressed_size = if compressed_size >= ZIP64_BYTES_THR || large_file { + size += mem::size_of::() as u16; + Some(compressed_size) + } else { + None + }; + let header_start = if header_start >= ZIP64_BYTES_THR { + size += mem::size_of::() as u16; + Some(header_start) + } else { + None + }; + if size == 0 { + return None; + } + + Some(Zip64ExtraFieldBlock { + magic: spec::ExtraFieldMagic::ZIP64_EXTRA_FIELD_TAG, + size, + uncompressed_size, + compressed_size, + header_start, + }) + } +} + +impl Zip64ExtraFieldBlock { + pub fn full_size(&self) -> usize { + assert!(self.size > 0); + self.size as usize + mem::size_of::() + mem::size_of::() + } + + pub fn serialize(self) -> Box<[u8]> { + let Self { + magic, + size, + uncompressed_size, + compressed_size, + header_start, + } = self; + + let full_size = self.full_size(); + + let mut ret = Vec::with_capacity(full_size); + ret.extend(magic.to_le_bytes()); + ret.extend(u16::to_le_bytes(size)); + + if let Some(uncompressed_size) = uncompressed_size { + ret.extend(u64::to_le_bytes(uncompressed_size)); + } + if let Some(compressed_size) = compressed_size { + ret.extend(u64::to_le_bytes(compressed_size)); + } + if let Some(header_start) = header_start { + ret.extend(u64::to_le_bytes(header_start)); + } + debug_assert_eq!(ret.len(), full_size); + + ret.into_boxed_slice() + } +} + +#[derive(Copy, Clone, Debug)] +#[repr(packed, C)] +pub(crate) struct ZipDataDescriptorBlock { + magic: spec::Magic, + pub crc32: u32, + pub compressed_size: u32, + pub uncompressed_size: u32, +} + +unsafe impl Pod for ZipDataDescriptorBlock {} + +impl FixedSizeBlock for ZipDataDescriptorBlock { + const MAGIC: spec::Magic = spec::Magic::DATA_DESCRIPTOR_SIGNATURE; + + #[inline(always)] + fn magic(self) -> spec::Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("Invalid data descriptor header"); + + to_and_from_le![ + (magic, spec::Magic), + (crc32, u32), + (compressed_size, u32), + (uncompressed_size, u32), + ]; +} + +#[derive(Copy, Clone, Debug)] +#[repr(packed, C)] +pub(crate) struct Zip64DataDescriptorBlock { + magic: spec::Magic, + pub crc32: u32, + pub compressed_size: u64, + pub uncompressed_size: u64, +} + +unsafe impl Pod for Zip64DataDescriptorBlock {} + +impl FixedSizeBlock for Zip64DataDescriptorBlock { + const MAGIC: spec::Magic = spec::Magic::DATA_DESCRIPTOR_SIGNATURE; + + #[inline(always)] + fn magic(self) -> spec::Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("Invalid zip64 data descriptor header"); + + to_and_from_le![ + (magic, spec::Magic), + (crc32, u32), + (compressed_size, u64), + (uncompressed_size, u64), + ]; +} + +/// The encryption specification used to encrypt a file with AES. +/// +/// According to the [specification](https://www.winzip.com/win/en/aes_info.html#winzip11) AE-2 +/// does not make use of the CRC check. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +#[repr(u16)] +pub enum AesVendorVersion { + Ae1 = 0x0001, + Ae2 = 0x0002, +} + +/// AES variant used. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "_arbitrary", derive(arbitrary::Arbitrary))] +#[repr(u8)] +pub enum AesMode { + /// 128-bit AES encryption. + Aes128 = 0x01, + /// 192-bit AES encryption. + Aes192 = 0x02, + /// 256-bit AES encryption. + Aes256 = 0x03, +} + +#[cfg(feature = "aes-crypto")] +impl AesMode { + /// Length of the salt for the given AES mode. + pub const fn salt_length(&self) -> usize { + self.key_length() / 2 + } + + /// Length of the key for the given AES mode. + pub const fn key_length(&self) -> usize { + match self { + Self::Aes128 => 16, + Self::Aes192 => 24, + Self::Aes256 => 32, + } + } +} + +#[cfg(test)] +mod test { + #[test] + fn system() { + use super::System; + assert_eq!(u8::from(System::Dos), 0u8); + assert_eq!(System::Dos as u8, 0u8); + assert_eq!(System::Unix as u8, 3u8); + assert_eq!(u8::from(System::Unix), 3u8); + assert_eq!(System::from(0), System::Dos); + assert_eq!(System::from(3), System::Unix); + assert_eq!(u8::from(System::Unknown), 4u8); + assert_eq!(System::Unknown as u8, 4u8); + } + + #[test] + fn unix_mode_robustness() { + use super::{System, ZipFileData}; + use crate::types::ffi::S_IFLNK; + let mut data = ZipFileData { + system: System::Dos, + external_attributes: (S_IFLNK | 0o777) << 16, + ..ZipFileData::default() + }; + assert_eq!(data.unix_mode(), Some(S_IFLNK | 0o777)); + + data.system = System::Unknown; + assert_eq!(data.unix_mode(), Some(S_IFLNK | 0o777)); + + data.external_attributes = 0x10; // DOS directory bit + data.system = System::Dos; + assert_eq!( + data.unix_mode().unwrap() & 0o170000, + crate::types::ffi::S_IFDIR + ); + } + + #[test] + fn sanitize() { + let file_name = "/path/../../../../etc/./passwd\0/etc/shadow".to_string(); + let data = ZipFileData { + system: System::Dos, + version_made_by: 0, + flags: 0, + encrypted: false, + using_data_descriptor: false, + is_utf8: true, + compression_method: crate::compression::CompressionMethod::Stored, + compression_level: None, + last_modified_time: None, + crc32: 0, + compressed_size: 0, + uncompressed_size: 0, + file_name: file_name.clone().into_boxed_str(), + file_name_raw: file_name.into_bytes().into_boxed_slice(), + extra_field: None, + central_extra_field: None, + file_comment: String::with_capacity(0).into_boxed_str(), + header_start: 0, + extra_data_start: None, + data_start: OnceLock::new(), + central_header_start: 0, + external_attributes: 0, + large_file: false, + aes_mode: None, + aes_extra_data_start: 0, + extra_fields: Vec::new(), + }; + assert_eq!(data.file_name_sanitized(), PathBuf::from("path/etc/passwd")); + } + + #[test] + #[allow(clippy::unusual_byte_groupings)] + fn datetime_default() { + use super::DateTime; + let dt = DateTime::default(); + assert_eq!(dt.timepart(), 0); + assert_eq!(dt.datepart(), 0b0000000_0001_00001); + } + + #[test] + #[allow(clippy::unusual_byte_groupings)] + fn datetime_max() { + use super::DateTime; + let dt = DateTime::from_date_and_time(2107, 12, 31, 23, 59, 58).unwrap(); + assert_eq!(dt.timepart(), 0b10111_111011_11101); + assert_eq!(dt.datepart(), 0b1111111_1100_11111); + } + + #[test] + fn datetime_equality() { + use super::DateTime; + + let dt = DateTime::from_date_and_time(2018, 11, 17, 10, 38, 30).unwrap(); + assert_eq!( + dt, + DateTime::from_date_and_time(2018, 11, 17, 10, 38, 30).unwrap() + ); + assert_ne!(dt, DateTime::default()); + } + + #[test] + fn datetime_order() { + use std::cmp::Ordering; + + use super::DateTime; + + let dt = DateTime::from_date_and_time(2018, 11, 17, 10, 38, 30).unwrap(); + assert_eq!( + dt.cmp(&DateTime::from_date_and_time(2018, 11, 17, 10, 38, 30).unwrap()), + Ordering::Equal + ); + // year + assert!(dt < DateTime::from_date_and_time(2019, 11, 17, 10, 38, 30).unwrap()); + assert!(dt > DateTime::from_date_and_time(2017, 11, 17, 10, 38, 30).unwrap()); + // month + assert!(dt < DateTime::from_date_and_time(2018, 12, 17, 10, 38, 30).unwrap()); + assert!(dt > DateTime::from_date_and_time(2018, 10, 17, 10, 38, 30).unwrap()); + // day + assert!(dt < DateTime::from_date_and_time(2018, 11, 18, 10, 38, 30).unwrap()); + assert!(dt > DateTime::from_date_and_time(2018, 11, 16, 10, 38, 30).unwrap()); + // hour + assert!(dt < DateTime::from_date_and_time(2018, 11, 17, 11, 38, 30).unwrap()); + assert!(dt > DateTime::from_date_and_time(2018, 11, 17, 9, 38, 30).unwrap()); + // minute + assert!(dt < DateTime::from_date_and_time(2018, 11, 17, 10, 39, 30).unwrap()); + assert!(dt > DateTime::from_date_and_time(2018, 11, 17, 10, 37, 30).unwrap()); + // second + assert!(dt < DateTime::from_date_and_time(2018, 11, 17, 10, 38, 32).unwrap()); + assert_eq!( + dt.cmp(&DateTime::from_date_and_time(2018, 11, 17, 10, 38, 31).unwrap()), + Ordering::Equal + ); + assert!(dt > DateTime::from_date_and_time(2018, 11, 17, 10, 38, 29).unwrap()); + assert!(dt > DateTime::from_date_and_time(2018, 11, 17, 10, 38, 28).unwrap()); + } + + #[test] + fn datetime_display() { + use super::DateTime; + + assert_eq!(format!("{}", DateTime::default()), "1980-01-01 00:00:00"); + assert_eq!( + format!( + "{}", + DateTime::from_date_and_time(2018, 11, 17, 10, 38, 30).unwrap() + ), + "2018-11-17 10:38:30" + ); + assert_eq!( + format!( + "{}", + DateTime::from_date_and_time(2107, 12, 31, 23, 59, 58).unwrap() + ), + "2107-12-31 23:59:58" + ); + } + + #[test] + fn datetime_bounds() { + use super::DateTime; + + assert!(DateTime::from_date_and_time(2000, 1, 1, 23, 59, 60).is_ok()); + assert!(DateTime::from_date_and_time(2000, 1, 1, 24, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2000, 1, 1, 0, 60, 0).is_err()); + assert!(DateTime::from_date_and_time(2000, 1, 1, 0, 0, 61).is_err()); + + assert!(DateTime::from_date_and_time(2107, 12, 31, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(1980, 1, 1, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(1979, 1, 1, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(1980, 0, 1, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(1980, 1, 0, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2108, 12, 31, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2107, 13, 31, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2107, 12, 32, 0, 0, 0).is_err()); + + assert!(DateTime::from_date_and_time(2018, 1, 31, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 2, 28, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 2, 29, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2018, 3, 31, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 4, 30, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 4, 31, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2018, 5, 31, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 6, 30, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 6, 31, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2018, 7, 31, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 8, 31, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 9, 30, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 9, 31, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2018, 10, 31, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 11, 30, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 11, 31, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2018, 12, 31, 0, 0, 0).is_ok()); + + // leap year: divisible by 4 + assert!(DateTime::from_date_and_time(2024, 2, 29, 0, 0, 0).is_ok()); + // leap year: divisible by 100 and by 400 + assert!(DateTime::from_date_and_time(2000, 2, 29, 0, 0, 0).is_ok()); + // common year: divisible by 100 but not by 400 + assert!(DateTime::from_date_and_time(2100, 2, 29, 0, 0, 0).is_err()); + } + + use std::{path::PathBuf, sync::OnceLock}; + + #[cfg(feature = "time")] + use time::{format_description::well_known::Rfc3339, OffsetDateTime, PrimitiveDateTime}; + + use crate::types::{System, ZipFileData}; + + #[cfg(feature = "time")] + #[test] + fn datetime_try_from_offset_datetime() { + use time::macros::datetime; + + use super::DateTime; + + // 2018-11-17 10:38:30 + let dt = DateTime::try_from(datetime!(2018-11-17 10:38:30 UTC)).unwrap(); + assert_eq!(dt.year(), 2018); + assert_eq!(dt.month(), 11); + assert_eq!(dt.day(), 17); + assert_eq!(dt.hour(), 10); + assert_eq!(dt.minute(), 38); + assert_eq!(dt.second(), 30); + } + + #[cfg(feature = "time")] + #[test] + fn datetime_try_from_primitive_datetime() { + use time::macros::datetime; + + use super::DateTime; + + // 2018-11-17 10:38:30 + let dt = DateTime::try_from(datetime!(2018-11-17 10:38:30)).unwrap(); + assert_eq!(dt.year(), 2018); + assert_eq!(dt.month(), 11); + assert_eq!(dt.day(), 17); + assert_eq!(dt.hour(), 10); + assert_eq!(dt.minute(), 38); + assert_eq!(dt.second(), 30); + } + + #[cfg(feature = "time")] + #[test] + fn datetime_try_from_bounds() { + use super::DateTime; + use time::macros::datetime; + + // 1979-12-31 23:59:59 + assert!(DateTime::try_from(datetime!(1979-12-31 23:59:59)).is_err()); + + // 1980-01-01 00:00:00 + assert!(DateTime::try_from(datetime!(1980-01-01 00:00:00)).is_ok()); + + // 2107-12-31 23:59:59 + assert!(DateTime::try_from(datetime!(2107-12-31 23:59:59)).is_ok()); + + // 2108-01-01 00:00:00 + assert!(DateTime::try_from(datetime!(2108-01-01 00:00:00)).is_err()); + } + + #[cfg(feature = "time")] + #[test] + fn offset_datetime_try_from_datetime() { + use time::macros::datetime; + + use super::DateTime; + + // 2018-11-17 10:38:30 UTC + let dt = + OffsetDateTime::try_from(DateTime::try_from_msdos(0x4D71, 0x54CF).unwrap()).unwrap(); + assert_eq!(dt, datetime!(2018-11-17 10:38:30 UTC)); + } + + #[cfg(feature = "time")] + #[test] + fn primitive_datetime_try_from_datetime() { + use time::macros::datetime; + + use super::DateTime; + + // 2018-11-17 10:38:30 + let dt = + PrimitiveDateTime::try_from(DateTime::try_from_msdos(0x4D71, 0x54CF).unwrap()).unwrap(); + assert_eq!(dt, datetime!(2018-11-17 10:38:30)); + } + + #[cfg(feature = "time")] + #[test] + fn offset_datetime_try_from_bounds() { + use super::DateTime; + + // 1980-00-00 00:00:00 + assert!(OffsetDateTime::try_from(unsafe { + DateTime::from_msdos_unchecked(0x0000, 0x0000) + }) + .is_err()); + + // 2107-15-31 31:63:62 + assert!(OffsetDateTime::try_from(unsafe { + DateTime::from_msdos_unchecked(0xFFFF, 0xFFFF) + }) + .is_err()); + } + + #[cfg(feature = "time")] + #[test] + fn primitive_datetime_try_from_bounds() { + use super::DateTime; + + // 1980-00-00 00:00:00 + assert!(PrimitiveDateTime::try_from(unsafe { + DateTime::from_msdos_unchecked(0x0000, 0x0000) + }) + .is_err()); + + // 2107-15-31 31:63:62 + assert!(PrimitiveDateTime::try_from(unsafe { + DateTime::from_msdos_unchecked(0xFFFF, 0xFFFF) + }) + .is_err()); + } + + #[cfg(feature = "jiff-02")] + #[test] + fn datetime_try_from_civil_datetime() { + use jiff::civil; + + use super::DateTime; + + // 2018-11-17 10:38:30 + let dt = DateTime::try_from(civil::datetime(2018, 11, 17, 10, 38, 30, 0)).unwrap(); + assert_eq!(dt.year(), 2018); + assert_eq!(dt.month(), 11); + assert_eq!(dt.day(), 17); + assert_eq!(dt.hour(), 10); + assert_eq!(dt.minute(), 38); + assert_eq!(dt.second(), 30); + } + + #[cfg(feature = "jiff-02")] + #[test] + fn datetime_try_from_civil_datetime_bounds() { + use jiff::civil; + + use super::DateTime; + + // 1979-12-31 23:59:59 + assert!(DateTime::try_from(civil::datetime(1979, 12, 31, 23, 59, 59, 0)).is_err()); + + // 1980-01-01 00:00:00 + assert!(DateTime::try_from(civil::datetime(1980, 1, 1, 0, 0, 0, 0)).is_ok()); + + // 2107-12-31 23:59:59 + assert!(DateTime::try_from(civil::datetime(2107, 12, 31, 23, 59, 59, 0)).is_ok()); + + // 2108-01-01 00:00:00 + assert!(DateTime::try_from(civil::datetime(2108, 1, 1, 0, 0, 0, 0)).is_err()); + } + + #[cfg(feature = "jiff-02")] + #[test] + fn civil_datetime_try_from_datetime() { + use jiff::civil; + + use super::DateTime; + + // 2018-11-17 10:38:30 UTC + let dt = + civil::DateTime::try_from(DateTime::try_from_msdos(0x4D71, 0x54CF).unwrap()).unwrap(); + assert_eq!(dt, civil::datetime(2018, 11, 17, 10, 38, 30, 0)); + } + + #[cfg(feature = "jiff-02")] + #[test] + fn civil_datetime_try_from_datetime_bounds() { + use jiff::civil; + + use super::DateTime; + + // 1980-00-00 00:00:00 + assert!(civil::DateTime::try_from(unsafe { + DateTime::from_msdos_unchecked(0x0000, 0x0000) + }) + .is_err()); + + // 2107-15-31 31:63:62 + assert!(civil::DateTime::try_from(unsafe { + DateTime::from_msdos_unchecked(0xFFFF, 0xFFFF) + }) + .is_err()); + } + + #[test] + #[allow(deprecated)] + fn time_conversion() { + use super::DateTime; + let dt = DateTime::try_from_msdos(0x4D71, 0x54CF).unwrap(); + assert_eq!(dt.year(), 2018); + assert_eq!(dt.month(), 11); + assert_eq!(dt.day(), 17); + assert_eq!(dt.hour(), 10); + assert_eq!(dt.minute(), 38); + assert_eq!(dt.second(), 30); + + let dt = DateTime::try_from((0x4D71, 0x54CF)).unwrap(); + assert_eq!(dt.year(), 2018); + assert_eq!(dt.month(), 11); + assert_eq!(dt.day(), 17); + assert_eq!(dt.hour(), 10); + assert_eq!(dt.minute(), 38); + assert_eq!(dt.second(), 30); + + #[cfg(feature = "time")] + assert_eq!( + dt.to_time().unwrap().format(&Rfc3339).unwrap(), + "2018-11-17T10:38:30Z" + ); + + assert_eq!(<(u16, u16)>::from(dt), (0x4D71, 0x54CF)); + } + + #[test] + #[allow(deprecated)] + fn time_out_of_bounds() { + use super::DateTime; + let dt = unsafe { DateTime::from_msdos_unchecked(0xFFFF, 0xFFFF) }; + assert_eq!(dt.year(), 2107); + assert_eq!(dt.month(), 15); + assert_eq!(dt.day(), 31); + assert_eq!(dt.hour(), 31); + assert_eq!(dt.minute(), 63); + assert_eq!(dt.second(), 62); + + #[cfg(feature = "time")] + assert!(dt.to_time().is_err()); + + let dt = unsafe { DateTime::from_msdos_unchecked(0x0000, 0x0000) }; + assert_eq!(dt.year(), 1980); + assert_eq!(dt.month(), 0); + assert_eq!(dt.day(), 0); + assert_eq!(dt.hour(), 0); + assert_eq!(dt.minute(), 0); + assert_eq!(dt.second(), 0); + + #[cfg(feature = "time")] + assert!(dt.to_time().is_err()); + } + + #[cfg(feature = "time")] + #[test] + fn time_at_january() { + use super::DateTime; + + // 2020-01-01 00:00:00 + let clock = OffsetDateTime::from_unix_timestamp(1_577_836_800).unwrap(); + + assert!(DateTime::try_from(PrimitiveDateTime::new(clock.date(), clock.time())).is_ok()); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/unstable.rs b/v2/vendor/security/zip-7.2.0/src/unstable.rs new file mode 100644 index 0000000000..4bce5a6160 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/unstable.rs @@ -0,0 +1,121 @@ +#![allow(missing_docs)] + +use std::borrow::Cow; +use std::io; +use std::io::{Read, Write}; +use std::path::{Component, Path, MAIN_SEPARATOR}; + +/// Provides high level API for reading from a stream. +pub mod stream { + pub use crate::read::stream::*; +} +/// Types for creating ZIP archives. +pub mod write { + use crate::write::{FileOptionExtension, FileOptions}; + /// Unstable methods for [`FileOptions`]. + pub trait FileOptionsExt { + /// Write the file with the given password using the deprecated ZipCrypto algorithm. + /// + /// This is not recommended for new archives, as ZipCrypto is not secure. + fn with_deprecated_encryption(self, password: &[u8]) -> Self; + } + impl FileOptionsExt for FileOptions<'_, T> { + fn with_deprecated_encryption(self, password: &[u8]) -> FileOptions<'static, T> { + self.with_deprecated_encryption(password) + } + } +} + +/// Helper methods for writing unsigned integers in little-endian form. +pub trait LittleEndianWriteExt: Write { + fn write_u16_le(&mut self, input: u16) -> io::Result<()> { + self.write_all(&input.to_le_bytes()) + } + + fn write_u32_le(&mut self, input: u32) -> io::Result<()> { + self.write_all(&input.to_le_bytes()) + } + + fn write_u64_le(&mut self, input: u64) -> io::Result<()> { + self.write_all(&input.to_le_bytes()) + } + + fn write_u128_le(&mut self, input: u128) -> io::Result<()> { + self.write_all(&input.to_le_bytes()) + } +} + +impl LittleEndianWriteExt for W {} + +/// Helper methods for reading unsigned integers in little-endian form. +pub trait LittleEndianReadExt: Read { + fn read_u16_le(&mut self) -> io::Result { + let mut out = [0u8; 2]; + self.read_exact(&mut out)?; + Ok(u16::from_le_bytes(out)) + } + + fn read_u32_le(&mut self) -> io::Result { + let mut out = [0u8; 4]; + self.read_exact(&mut out)?; + Ok(u32::from_le_bytes(out)) + } + + fn read_u64_le(&mut self) -> io::Result { + let mut out = [0u8; 8]; + self.read_exact(&mut out)?; + Ok(u64::from_le_bytes(out)) + } +} + +impl LittleEndianReadExt for R {} + +/// Converts a path to the ZIP format (forward-slash-delimited and normalized). +pub fn path_to_string>(path: T) -> Box { + let mut maybe_original = None; + if let Some(original) = path.as_ref().to_str() { + if original.is_empty() || original == "." || original == ".." { + return String::new().into_boxed_str(); + } + if original.starts_with(MAIN_SEPARATOR) { + if original.len() == 1 { + return MAIN_SEPARATOR.to_string().into_boxed_str(); + } else if (MAIN_SEPARATOR == '/' || !original[1..].contains(MAIN_SEPARATOR)) + && !original.ends_with('.') + && !original.contains([MAIN_SEPARATOR, MAIN_SEPARATOR]) + && !original.contains([MAIN_SEPARATOR, '.', MAIN_SEPARATOR]) + && !original.contains([MAIN_SEPARATOR, '.', '.', MAIN_SEPARATOR]) + { + maybe_original = Some(&original[1..]); + } + } else if !original.contains(MAIN_SEPARATOR) { + return original.into(); + } + } + let mut recreate = maybe_original.is_none(); + let mut normalized_components = Vec::new(); + + for component in path.as_ref().components() { + match component { + Component::Normal(os_str) => match os_str.to_str() { + Some(valid_str) => normalized_components.push(Cow::Borrowed(valid_str)), + None => { + recreate = true; + normalized_components.push(os_str.to_string_lossy()); + } + }, + Component::ParentDir => { + recreate = true; + normalized_components.pop(); + } + _ => { + recreate = true; + } + } + } + if recreate { + normalized_components.join("/").into() + } else { + maybe_original.unwrap().into() + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/write.rs b/v2/vendor/security/zip-7.2.0/src/write.rs new file mode 100644 index 0000000000..ff4932b7c1 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/write.rs @@ -0,0 +1,4077 @@ +//! Types for creating ZIP archives + +#[cfg(feature = "aes-crypto")] +use crate::aes::AesWriter; +use crate::compression::CompressionMethod; +use crate::read::{parse_single_extra_field, Config, ZipArchive, ZipFile}; +use crate::result::{invalid, ZipError, ZipResult}; +use crate::spec::{self, FixedSizeBlock, Zip32CDEBlock}; +use crate::types::ffi::S_IFLNK; +#[cfg(feature = "aes-crypto")] +use crate::types::AesMode; +use crate::types::{ + ffi, AesVendorVersion, DateTime, Zip64ExtraFieldBlock, ZipFileData, ZipLocalEntryBlock, + ZipRawValues, MIN_VERSION, +}; +use core::default::Default; +use core::fmt::{Debug, Formatter}; +use core::marker::PhantomData; +use core::mem::{self, size_of}; +#[cfg(feature = "deflate-zopfli")] +use core::num::NonZeroU64; +use core::str::{from_utf8, Utf8Error}; +use crc32fast::Hasher; +use indexmap::IndexMap; +use std::borrow::ToOwned; +use std::io::{self, Read, Seek, Write}; +use std::io::{BufReader, SeekFrom}; +use std::io::{Cursor, ErrorKind}; +use std::sync::Arc; + +#[cfg(feature = "deflate-flate2")] +use flate2::{write::DeflateEncoder, Compression}; + +#[cfg(feature = "_bzip2_any")] +use bzip2::write::BzEncoder; + +#[cfg(feature = "deflate-zopfli")] +use zopfli::Options; + +#[cfg(feature = "deflate-zopfli")] +use std::io::BufWriter; +use std::path::Path; +#[cfg(feature = "zstd")] +use zstd::stream::write::Encoder as ZstdEncoder; +#[cfg(feature = "_bzip2_any")] +use GenericZipWriter::Bzip2; +#[cfg(feature = "deflate-flate2")] +use GenericZipWriter::Deflater; +#[cfg(feature = "ppmd")] +use GenericZipWriter::Ppmd; +#[cfg(feature = "xz")] +use GenericZipWriter::Xz; +#[cfg(feature = "zstd")] +use GenericZipWriter::Zstd; +#[cfg(feature = "deflate-zopfli")] +use GenericZipWriter::{BufferedZopfliDeflater, ZopfliDeflater}; + +// re-export from types +pub use crate::types::{FileOptions, SimpleFileOptions}; + +#[cfg_attr(feature = "aes-crypto", allow(clippy::large_enum_variant))] +enum MaybeEncrypted { + Unencrypted(W), + #[cfg(feature = "aes-crypto")] + Aes(AesWriter), + ZipCrypto(crate::zipcrypto::ZipCryptoWriter), +} + +impl Debug for MaybeEncrypted { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + // Don't print W, since it may be a huge Vec + f.write_str(match self { + MaybeEncrypted::Unencrypted(_) => "Unencrypted", + #[cfg(feature = "aes-crypto")] + MaybeEncrypted::Aes(_) => "AES", + MaybeEncrypted::ZipCrypto(_) => "ZipCrypto", + }) + } +} + +impl Write for MaybeEncrypted { + fn write(&mut self, buf: &[u8]) -> io::Result { + match self { + MaybeEncrypted::Unencrypted(w) => w.write(buf), + #[cfg(feature = "aes-crypto")] + MaybeEncrypted::Aes(w) => w.write(buf), + MaybeEncrypted::ZipCrypto(w) => w.write(buf), + } + } + fn flush(&mut self) -> io::Result<()> { + match self { + MaybeEncrypted::Unencrypted(w) => w.flush(), + #[cfg(feature = "aes-crypto")] + MaybeEncrypted::Aes(w) => w.flush(), + MaybeEncrypted::ZipCrypto(w) => w.flush(), + } + } +} + +enum GenericZipWriter { + Closed, + Storer(MaybeEncrypted), + #[cfg(feature = "deflate-flate2")] + Deflater(DeflateEncoder>), + #[cfg(feature = "deflate-zopfli")] + ZopfliDeflater(zopfli::DeflateEncoder>), + #[cfg(feature = "deflate-zopfli")] + BufferedZopfliDeflater(BufWriter>>), + #[cfg(feature = "_bzip2_any")] + Bzip2(BzEncoder>), + #[cfg(feature = "zstd")] + Zstd(ZstdEncoder<'static, MaybeEncrypted>), + #[cfg(feature = "xz")] + Xz(Box>>), + #[cfg(feature = "ppmd")] + Ppmd(Box>>), +} + +impl Debug for GenericZipWriter { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Closed => f.write_str("Closed"), + Storer(w) => f.write_fmt(format_args!("Storer({w:?})")), + #[cfg(feature = "deflate-flate2")] + Deflater(w) => f.write_fmt(format_args!("Deflater({:?})", w.get_ref())), + #[cfg(feature = "deflate-zopfli")] + ZopfliDeflater(_) => f.write_str("ZopfliDeflater"), + #[cfg(feature = "deflate-zopfli")] + BufferedZopfliDeflater(_) => f.write_str("BufferedZopfliDeflater"), + #[cfg(feature = "_bzip2_any")] + Bzip2(w) => f.write_fmt(format_args!("Bzip2({:?})", w.get_ref())), + #[cfg(feature = "zstd")] + Zstd(w) => f.write_fmt(format_args!("Zstd({:?})", w.get_ref())), + #[cfg(feature = "xz")] + Xz(w) => f.write_fmt(format_args!("Xz({:?})", w.inner())), + #[cfg(feature = "ppmd")] + Ppmd(_) => f.write_fmt(format_args!("Ppmd8Encoder")), + } + } +} + +// Put the struct declaration in a private module to convince rustdoc to display ZipWriter nicely +pub(crate) mod zip_writer { + use core::fmt::{Debug, Formatter}; + use std::io::{Seek, Write}; + + use indexmap::IndexMap; + + use crate::{ + types::ZipFileData, + write::{GenericZipWriter, ZipWriterStats}, + }; + + /// ZIP archive generator + /// + /// Handles the bookkeeping involved in building an archive, and provides an + /// API to edit its contents. + /// + /// ``` + /// # fn doit() -> zip::result::ZipResult<()> + /// # { + /// # use zip::ZipWriter; + /// use std::io::Write; + /// use zip::write::SimpleFileOptions; + /// + /// // We use a cursor + vec here, though you'd normally use a `File` + /// let mut cur = std::io::Cursor::new(Vec::new()); + /// let mut zip = ZipWriter::new(&mut cur); + /// + /// let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + /// zip.start_file("hello_world.txt", options)?; + /// zip.write(b"Hello, World!")?; + /// + /// // Apply the changes you've made. + /// // Dropping the `ZipWriter` will have the same effect, but may silently fail + /// zip.finish()?; + /// + /// // raw zip data is available as a Vec + /// let zip_bytes = cur.into_inner(); + /// + /// # Ok(()) + /// # } + /// # doit().unwrap(); + /// ``` + pub struct ZipWriter { + pub(super) inner: GenericZipWriter, + pub(super) files: IndexMap, ZipFileData>, + pub(super) stats: ZipWriterStats, + pub(super) writing_to_file: bool, + pub(super) writing_raw: bool, + pub(super) comment: Box<[u8]>, + pub(super) zip64_comment: Option>, + pub(super) flush_on_finish_file: bool, + pub(super) seek_possible: bool, + pub(crate) auto_large_file: bool, + } + + impl Debug for ZipWriter { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_fmt(format_args!( + "ZipWriter {{files: {:?}, stats: {:?}, writing_to_file: {}, writing_raw: {}, comment: {:?}, flush_on_finish_file: {}}}", + self.files, self.stats, self.writing_to_file, self.writing_raw, + self.comment, self.flush_on_finish_file)) + } + } +} +#[doc(inline)] +pub use self::sealed::FileOptionExtension; +use crate::result::ZipError::UnsupportedArchive; +use crate::unstable::path_to_string; +use crate::unstable::LittleEndianWriteExt; +use crate::write::GenericZipWriter::{Closed, Storer}; +use crate::zipcrypto::{EncryptWith, ZipCryptoKeys}; +use crate::CompressionMethod::Stored; +pub use zip_writer::ZipWriter; + +#[derive(Default, Debug)] +struct ZipWriterStats { + hasher: Hasher, + start: u64, + bytes_written: u64, +} + +mod sealed { + use std::sync::Arc; + + use super::ExtendedFileOptions; + + pub trait Sealed {} + /// File options Extensions + #[doc(hidden)] + pub trait FileOptionExtension: Default + Sealed { + /// Extra Data + fn extra_data(&self) -> Option<&Arc>>; + /// Central Extra Data + fn central_extra_data(&self) -> Option<&Arc>>; + } + impl Sealed for () {} + impl FileOptionExtension for () { + fn extra_data(&self) -> Option<&Arc>> { + None + } + fn central_extra_data(&self) -> Option<&Arc>> { + None + } + } + impl Sealed for ExtendedFileOptions {} + + impl FileOptionExtension for ExtendedFileOptions { + fn extra_data(&self) -> Option<&Arc>> { + Some(&self.extra_data) + } + fn central_extra_data(&self) -> Option<&Arc>> { + Some(&self.central_extra_data) + } + } +} + +/// Adds Extra Data and Central Extra Data. It does not implement copy. +pub type FullFileOptions<'k> = FileOptions<'k, ExtendedFileOptions>; +/// The Extension for Extra Data and Central Extra Data +#[cfg_attr(feature = "_arbitrary", derive(arbitrary::Arbitrary))] +#[derive(Clone, Default, Eq, PartialEq)] +pub struct ExtendedFileOptions { + extra_data: Arc>, + central_extra_data: Arc>, +} + +impl ExtendedFileOptions { + /// Adds an extra data field, unless we detect that it's invalid. + pub fn add_extra_data>( + &mut self, + header_id: u16, + data: D, + central_only: bool, + ) -> ZipResult<()> { + let data = data.as_ref(); + let len = data.len() + 4; + if self.extra_data.len() + self.central_extra_data.len() + len > u16::MAX as usize { + Err(invalid!("Extra data field would be longer than allowed")) + } else { + let field = if central_only { + &mut self.central_extra_data + } else { + &mut self.extra_data + }; + let vec = Arc::get_mut(field); + let vec = match vec { + Some(exclusive) => exclusive, + None => { + *field = Arc::new(field.to_vec()); + Arc::get_mut(field).unwrap() + } + }; + Self::add_extra_data_unchecked(vec, header_id, data)?; + Self::validate_extra_data(vec, true)?; + Ok(()) + } + } + + pub(crate) fn add_extra_data_unchecked( + vec: &mut Vec, + header_id: u16, + data: &[u8], + ) -> Result<(), ZipError> { + vec.reserve_exact(data.len() + 4); + vec.write_u16_le(header_id)?; + vec.write_u16_le(data.len() as u16)?; + vec.write_all(data)?; + Ok(()) + } + + fn validate_extra_data(data: &[u8], disallow_zip64: bool) -> ZipResult<()> { + let len = data.len() as u64; + if len == 0 { + return Ok(()); + } + if len > u16::MAX as u64 { + return Err(ZipError::Io(io::Error::other( + "Extra-data field can't exceed u16::MAX bytes", + ))); + } + let mut data = Cursor::new(data); + let mut pos = data.position(); + while pos < len { + if len - data.position() < 4 { + return Err(ZipError::Io(io::Error::other( + "Extra-data field doesn't have room for ID and length", + ))); + } + #[cfg(not(feature = "unreserved"))] + { + use crate::unstable::LittleEndianReadExt; + let header_id = data.read_u16_le()?; + if EXTRA_FIELD_MAPPING.contains(&header_id) { + return Err(ZipError::Io(io::Error::other( + format!( + "Extra data header ID {header_id:#06} requires crate feature \"unreserved\"", + ), + ))); + } + data.seek(SeekFrom::Current(-2))?; + } + parse_single_extra_field(&mut ZipFileData::default(), &mut data, pos, disallow_zip64)?; + pos = data.position(); + } + Ok(()) + } +} + +impl Debug for ExtendedFileOptions { + fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), std::fmt::Error> { + f.write_fmt(format_args!("ExtendedFileOptions {{extra_data: vec!{:?}.into(), central_extra_data: vec!{:?}.into()}}", + self.extra_data, self.central_extra_data)) + } +} + +#[cfg(feature = "_arbitrary")] +impl<'a> arbitrary::Arbitrary<'a> for FileOptions<'a, ExtendedFileOptions> { + fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { + let mut options = FullFileOptions { + compression_method: CompressionMethod::arbitrary(u)?, + compression_level: if bool::arbitrary(u)? { + Some(u.int_in_range(0..=24)?) + } else { + None + }, + last_modified_time: DateTime::arbitrary(u)?, + permissions: Option::::arbitrary(u)?, + large_file: bool::arbitrary(u)?, + encrypt_with: Option::::arbitrary(u)?, + alignment: u16::arbitrary(u)?, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: None, + ..Default::default() + }; + #[cfg(feature = "deflate-zopfli")] + if options.compression_method == CompressionMethod::Deflated && bool::arbitrary(u)? { + options.zopfli_buffer_size = + Some(if bool::arbitrary(u)? { 2 } else { 3 } << u.int_in_range(8..=20)?); + } + u.arbitrary_loop(Some(0), Some(10), |u| { + options + .add_extra_data( + u.int_in_range(2..=u16::MAX)?, + Box::<[u8]>::arbitrary(u)?, + bool::arbitrary(u)?, + ) + .map_err(|_| arbitrary::Error::IncorrectFormat)?; + Ok(core::ops::ControlFlow::Continue(())) + })?; + ZipWriter::new(Cursor::new(Vec::new())) + .start_file("", options.clone()) + .map_err(|_| arbitrary::Error::IncorrectFormat)?; + Ok(options) + } +} + +impl FileOptions<'_, T> { + pub(crate) fn normalize(&mut self) { + if !self.last_modified_time.is_valid() { + self.last_modified_time = FileOptions::::default().last_modified_time; + } + + *self.permissions.get_or_insert(0o644) |= ffi::S_IFREG; + } + + /// Indicates whether this file will be encrypted (whether with AES or ZipCrypto). + pub const fn has_encryption(&self) -> bool { + #[cfg(feature = "aes-crypto")] + { + self.encrypt_with.is_some() || self.aes_mode.is_some() + } + #[cfg(not(feature = "aes-crypto"))] + { + self.encrypt_with.is_some() + } + } + + /// Set the compression method for the new file + /// + /// The default is [`CompressionMethod::Deflated`] if it is enabled. If not, + /// [`CompressionMethod::Bzip2`] is the default if it is enabled. If neither `bzip2` nor `deflate` + /// is enabled, [`CompressionMethod::Stored`] becomes the default and files are written uncompressed. + #[must_use] + pub const fn compression_method(mut self, method: CompressionMethod) -> Self { + self.compression_method = method; + self + } + + /// Set the compression level for the new file + /// + /// `None` value specifies default compression level. + /// + /// Range of values depends on compression method: + /// * `Deflated`: 10 - 264 for Zopfli, 0 - 9 for other encoders. Default is 24 if Zopfli is the + /// only encoder, or 6 otherwise. + /// * `Bzip2`: 0 - 9. Default is 6 + /// * `Zstd`: -7 - 22, with zero being mapped to default level. Default is 3 + /// * others: only `None` is allowed + #[must_use] + pub const fn compression_level(mut self, level: Option) -> Self { + self.compression_level = level; + self + } + + /// Set the last modified time + /// + /// The default is the current timestamp if the 'time' feature is enabled, and 1980-01-01 + /// otherwise + #[must_use] + pub const fn last_modified_time(mut self, mod_time: DateTime) -> Self { + self.last_modified_time = mod_time; + self + } + + /// Set the permissions for the new file. + /// + /// The format is represented with unix-style permissions. + /// The default is `0o644`, which represents `rw-r--r--` for files, + /// and `0o755`, which represents `rwxr-xr-x` for directories. + /// + /// This method only preserves the file permissions bits (via a `& 0o777`) and discards + /// higher file mode bits. So it cannot be used to denote an entry as a directory, + /// symlink, or other special file type. + #[must_use] + pub const fn unix_permissions(mut self, mode: u32) -> Self { + self.permissions = Some(mode & 0o777); + self + } + + /// Set whether the new file's compressed and uncompressed size is less than 4 GiB. + /// + /// If set to `false` and the file exceeds the limit, an I/O error is thrown and the file is + /// aborted. If set to `true`, readers will require ZIP64 support and if the file does not + /// exceed the limit, 20 B are wasted. The default is `false`. + #[must_use] + pub const fn large_file(mut self, large: bool) -> Self { + self.large_file = large; + self + } + + pub(crate) fn with_deprecated_encryption(self, password: &[u8]) -> FileOptions<'static, T> { + FileOptions { + encrypt_with: Some(EncryptWith::ZipCrypto( + ZipCryptoKeys::derive(password), + PhantomData, + )), + ..self + } + } + + /// Set the AES encryption parameters. + #[cfg(feature = "aes-crypto")] + pub fn with_aes_encryption(self, mode: AesMode, password: &str) -> FileOptions<'_, T> { + FileOptions { + encrypt_with: Some(EncryptWith::Aes { mode, password }), + ..self + } + } + + /// Sets the size of the buffer used to hold the next block that Zopfli will compress. The + /// larger the buffer, the more effective the compression, but the more memory is required. + /// A value of `None` indicates no buffer, which is recommended only when all non-empty writes + /// are larger than about 32 KiB. + #[must_use] + #[cfg(feature = "deflate-zopfli")] + pub const fn with_zopfli_buffer(mut self, size: Option) -> Self { + self.zopfli_buffer_size = size; + self + } + + /// Returns the compression level currently set. + pub const fn get_compression_level(&self) -> Option { + self.compression_level + } + /// Sets the alignment to the given number of bytes. + #[must_use] + pub const fn with_alignment(mut self, alignment: u16) -> Self { + self.alignment = alignment; + self + } +} +impl FileOptions<'_, ExtendedFileOptions> { + /// Adds an extra data field. + pub fn add_extra_data>( + &mut self, + header_id: u16, + data: D, + central_only: bool, + ) -> ZipResult<()> { + self.extended_options + .add_extra_data(header_id, data, central_only) + } + + /// Removes the extra data fields. + #[must_use] + pub fn clear_extra_data(mut self) -> Self { + if !self.extended_options.extra_data.is_empty() { + self.extended_options.extra_data = Arc::new(vec![]); + } + if !self.extended_options.central_extra_data.is_empty() { + self.extended_options.central_extra_data = Arc::new(vec![]); + } + self + } +} +impl FileOptions<'static, ()> { + /// Constructs a const FileOptions object. + /// + /// Note: This value is different than the return value of [`FileOptions::default()`]: + /// + /// - The `last_modified_time` is [`DateTime::DEFAULT`]. This corresponds to 1980-01-01 00:00:00 + pub const DEFAULT: Self = Self { + compression_method: CompressionMethod::DEFAULT, + compression_level: None, + last_modified_time: DateTime::DEFAULT, + large_file: false, + permissions: None, + encrypt_with: None, + extended_options: (), + alignment: 1, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: Some(1 << 15), + #[cfg(feature = "aes-crypto")] + aes_mode: None, + }; +} + +impl Default for FileOptions<'_, T> { + /// Construct a new FileOptions object + fn default() -> Self { + Self { + compression_method: Default::default(), + compression_level: None, + last_modified_time: DateTime::default_for_write(), + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: T::default(), + alignment: 1, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: Some(1 << 15), + #[cfg(feature = "aes-crypto")] + aes_mode: None, + } + } +} + +impl Write for ZipWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + if !self.writing_to_file { + return Err(io::Error::other("No file has been started")); + } + if buf.is_empty() { + return Ok(0); + } + match self.inner.ref_mut() { + Some(ref mut w) => { + let write_result = w.write(buf); + if let Ok(count) = write_result { + self.stats.update(&buf[..count]); + if self.stats.bytes_written > spec::ZIP64_BYTES_THR + && !self.files.last_mut().unwrap().1.large_file + { + let _ = self.abort_file(); + return Err(io::Error::other("Large file option has not been set")); + } + } + write_result + } + None => Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "write(): ZipWriter was already closed", + )), + } + } + + fn flush(&mut self) -> io::Result<()> { + match self.inner.ref_mut() { + Some(ref mut w) => w.flush(), + None => Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "flush(): ZipWriter was already closed", + )), + } + } +} + +impl ZipWriterStats { + fn update(&mut self, buf: &[u8]) { + self.hasher.update(buf); + self.bytes_written += buf.len() as u64; + } +} + +impl ZipWriter { + /// Initializes the archive from an existing ZIP archive, making it ready for append. + /// + /// This uses a default configuration to initially read the archive. + pub fn new_append(readwriter: A) -> ZipResult> { + Self::new_append_with_config(Default::default(), readwriter) + } + + /// Initializes the archive from an existing ZIP archive, making it ready for append. + /// + /// This uses the given read configuration to initially read the archive. + pub fn new_append_with_config(config: Config, mut readwriter: A) -> ZipResult> { + readwriter.seek(SeekFrom::Start(0))?; + let shared = ZipArchive::get_metadata(config, &mut readwriter)?; + + Ok(ZipWriter { + inner: Storer(MaybeEncrypted::Unencrypted(readwriter)), + files: shared.files, + stats: Default::default(), + writing_to_file: false, + comment: shared.comment, + zip64_comment: shared.zip64_comment, + writing_raw: true, // avoid recomputing the last file's header + flush_on_finish_file: false, + seek_possible: true, + auto_large_file: false, + }) + } + + /// `flush_on_finish_file` is designed to support a streaming `inner` that may unload flushed + /// bytes. It flushes a file's header and body once it starts writing another file. A ZipWriter + /// will not try to seek back into where a previous file was written unless + /// either [`ZipWriter::abort_file`] is called while [`ZipWriter::is_writing_file`] returns + /// false, or [`ZipWriter::deep_copy_file`] is called. In the latter case, it will only need to + /// read previously-written files and not overwrite them. + /// + /// Note: when using an `inner` that cannot overwrite flushed bytes, do not wrap it in a + /// [BufWriter], because that has a [Seek::seek] method that implicitly calls + /// [BufWriter::flush], and ZipWriter needs to seek backward to update each file's header with + /// the size and checksum after writing the body. + /// + /// This setting is false by default. + pub fn set_flush_on_finish_file(&mut self, flush_on_finish_file: bool) { + self.flush_on_finish_file = flush_on_finish_file; + } +} + +impl ZipWriter { + /// Adds another copy of a file already in this archive. This will produce a larger but more + /// widely-compatible archive compared to [Self::shallow_copy_file]. Does not copy alignment. + pub fn deep_copy_file(&mut self, src_name: &str, dest_name: &str) -> ZipResult<()> { + self.finish_file()?; + if src_name == dest_name || self.files.contains_key(dest_name) { + return Err(invalid!("That file already exists")); + } + let write_position = self.inner.get_plain().stream_position()?; + let src_index = self.index_by_name(src_name)?; + let src_data = &mut self.files[src_index]; + let src_data_start = src_data.data_start(self.inner.get_plain())?; + debug_assert!(src_data_start <= write_position); + let mut compressed_size = src_data.compressed_size; + if compressed_size > (write_position - src_data_start) { + compressed_size = write_position - src_data_start; + src_data.compressed_size = compressed_size; + } + let mut reader = BufReader::new(self.inner.get_plain()); + reader.seek(SeekFrom::Start(src_data_start))?; + let mut copy = vec![0; compressed_size as usize]; + reader.take(compressed_size).read_exact(&mut copy)?; + self.inner + .get_plain() + .seek(SeekFrom::Start(write_position))?; + let mut new_data = src_data.clone(); + let dest_name_raw = dest_name.as_bytes(); + new_data.file_name = dest_name.into(); + new_data.file_name_raw = dest_name_raw.into(); + new_data.is_utf8 = !dest_name.is_ascii(); + new_data.header_start = write_position; + let extra_data_start = write_position + + size_of::() as u64 + + new_data.file_name_raw.len() as u64; + new_data.extra_data_start = Some(extra_data_start); + if let Some(extra) = &src_data.extra_field { + let stripped = strip_alignment_extra_field(extra); + if !stripped.is_empty() { + new_data.extra_field = Some(stripped.into()); + } else { + new_data.extra_field = None; + } + } + + let mut data_start = extra_data_start; + if let Some(extra) = &new_data.extra_field { + data_start += extra.len() as u64; + } + new_data.data_start.take(); + new_data.data_start.get_or_init(|| data_start); + new_data.central_header_start = 0; + let block = new_data.local_block()?; + let index = self.insert_file_data(new_data)?; + let new_data = &self.files[index]; + let result: io::Result<()> = { + let plain_writer = self.inner.get_plain(); + block.write(plain_writer)?; + plain_writer.write_all(&new_data.file_name_raw)?; + if let Some(data) = &new_data.extra_field { + plain_writer.write_all(data)?; + } + debug_assert_eq!(data_start, plain_writer.stream_position()?); + self.writing_to_file = true; + plain_writer.write_all(©)?; + if self.flush_on_finish_file { + plain_writer.flush()?; + } + Ok(()) + }; + self.ok_or_abort_file(result)?; + self.writing_to_file = false; + Ok(()) + } + + /// Like `deep_copy_file`, but uses Path arguments. + /// + /// This function ensures that the '/' path separator is used and normalizes `.` and `..`. It + /// ignores any `..` or Windows drive letter that would produce a path outside the ZIP file's + /// root. + pub fn deep_copy_file_from_path, U: AsRef>( + &mut self, + src_path: T, + dest_path: U, + ) -> ZipResult<()> { + let src = path_to_string(src_path); + let dest = path_to_string(dest_path); + self.deep_copy_file(&src, &dest) + } + + /// Write the zip file into the backing stream, then produce a readable archive of that data. + /// + /// This method avoids parsing the central directory records at the end of the stream for + /// a slight performance improvement over running [`ZipArchive::new()`] on the output of + /// [`Self::finish()`]. + /// + ///``` + /// # fn main() -> Result<(), zip::result::ZipError> { + /// # #[cfg(any(feature = "deflate-flate2", not(feature = "_deflate-any")))] + /// # { + /// use std::io::{Cursor, Read, Write}; + /// use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions}; + /// + /// let buf = Cursor::new(Vec::new()); + /// let mut zip = ZipWriter::new(buf); + /// let options = SimpleFileOptions::default(); + /// zip.start_file("a.txt", options)?; + /// zip.write_all(b"hello\n")?; + /// + /// let mut zip = zip.finish_into_readable()?; + /// let mut s: String = String::new(); + /// zip.by_name("a.txt")?.read_to_string(&mut s)?; + /// assert_eq!(s, "hello\n"); + /// # } + /// # Ok(()) + /// # } + ///``` + pub fn finish_into_readable(mut self) -> ZipResult> { + let central_start = self.finalize()?; + let inner = mem::replace(&mut self.inner, Closed).unwrap(); + let comment = mem::take(&mut self.comment); + let zip64_comment = mem::take(&mut self.zip64_comment); + let files = mem::take(&mut self.files); + + let archive = + ZipArchive::from_finalized_writer(files, comment, zip64_comment, inner, central_start)?; + Ok(archive) + } +} + +impl ZipWriter { + /// Initializes the archive. + /// + /// Before writing to this object, the [`ZipWriter::start_file`] function should be called. + /// After a successful write, the file remains open for writing. After a failed write, call + /// [`ZipWriter::is_writing_file`] to determine if the file remains open. + pub fn new(inner: W) -> ZipWriter { + ZipWriter { + inner: Storer(MaybeEncrypted::Unencrypted(inner)), + files: IndexMap::new(), + stats: Default::default(), + writing_to_file: false, + writing_raw: false, + comment: Box::new([]), + zip64_comment: None, + flush_on_finish_file: false, + seek_possible: true, + auto_large_file: false, + } + } + + /// Set automatically large file to true if needed + pub fn set_auto_large_file(mut self) -> Self { + self.auto_large_file = true; + self + } + + /// Returns true if a file is currently open for writing. + pub const fn is_writing_file(&self) -> bool { + self.writing_to_file && !self.inner.is_closed() + } + + /// Set ZIP archive comment. + pub fn set_comment(&mut self, comment: S) + where + S: Into>, + { + self.set_raw_comment(comment.into().into_boxed_bytes()) + } + + /// Set ZIP archive comment. + /// + /// This sets the raw bytes of the comment. The comment + /// is typically expected to be encoded in UTF-8. + pub fn set_raw_comment(&mut self, comment: Box<[u8]>) { + let max_comment_len = u16::MAX as usize; // 65,535 + if comment.len() > max_comment_len { + self.set_raw_zip64_comment(Some(comment)); + self.comment = Box::new([]); + } else { + self.comment = comment; + self.set_raw_zip64_comment(None); + } + } + + /// Get ZIP archive comment. + pub fn get_comment(&mut self) -> Result<&str, Utf8Error> { + from_utf8(self.get_raw_comment()) + } + + /// Get ZIP archive comment. + /// + /// This returns the raw bytes of the comment. The comment + /// is typically expected to be encoded in UTF-8. + pub const fn get_raw_comment(&self) -> &[u8] { + &self.comment + } + + /// Set ZIP64 archive comment. + pub fn set_zip64_comment(&mut self, comment: Option) + where + S: Into>, + { + self.set_raw_zip64_comment(comment.map(|v| v.into().into_boxed_bytes())) + } + + /// Set ZIP64 archive comment. + /// + /// This sets the raw bytes of the comment. The comment + /// is typically expected to be encoded in UTF-8. + pub fn set_raw_zip64_comment(&mut self, comment: Option>) { + self.zip64_comment = comment; + } + + /// Get ZIP64 archive comment. + pub fn get_zip64_comment(&mut self) -> Option> { + self.get_raw_zip64_comment().map(from_utf8) + } + + /// Get ZIP archive comment. + /// + /// This returns the raw bytes of the comment. The comment + /// is typically expected to be encoded in UTF-8. + pub fn get_raw_zip64_comment(&self) -> Option<&[u8]> { + self.zip64_comment.as_deref() + } + + /// Set the file length and crc32 manually. + /// + /// # Safety + /// + /// This overwrites the internal crc32 calculation. It should only be used in case + /// the underlying [Write] is written independently and you need to adjust the zip metadata. + pub unsafe fn set_file_metadata(&mut self, length: u64, crc32: u32) -> ZipResult<()> { + if !self.writing_to_file { + return Err(ZipError::Io(io::Error::other("No file has been started"))); + } + self.stats.hasher = Hasher::new_with_initial_len(crc32, length); + self.stats.bytes_written = length; + Ok(()) + } + + fn ok_or_abort_file>(&mut self, result: Result) -> ZipResult { + match result { + Err(e) => { + let _ = self.abort_file(); + Err(e.into()) + } + Ok(t) => Ok(t), + } + } + + /// Start a new file for with the requested options. + fn start_entry( + &mut self, + name: S, + mut options: FileOptions, + raw_values: Option, + ) -> ZipResult<()> { + self.finish_file()?; + + let header_start = self.inner.get_plain().stream_position()?; + let raw_values = raw_values.unwrap_or(ZipRawValues { + crc32: 0, + compressed_size: 0, + uncompressed_size: 0, + }); + + // Check if we're close to the 4GB boundary and force ZIP64 if needed + // This ensures we properly handle appending to files close to 4GB + if header_start > spec::ZIP64_BYTES_THR { + // Files that start on or past the 4GiB boundary are always ZIP64 + options.large_file = true; + } + + let mut extra_data = match options.extended_options.extra_data() { + Some(data) => data.to_vec(), + None => vec![], + }; + let central_extra_data = options.extended_options.central_extra_data(); + if let Some(zip64_block) = + Zip64ExtraFieldBlock::maybe_new(options.large_file, 0, 0, header_start) + { + let mut new_extra_data = zip64_block.serialize().into_vec(); + new_extra_data.append(&mut extra_data); + extra_data = new_extra_data; + } + + // Figure out the underlying compression_method and aes mode when using + // AES encryption. + let (compression_method, aes_mode) = match options.encrypt_with { + // Preserve AES method for raw copies without needing a password + #[cfg(feature = "aes-crypto")] + None if options.aes_mode.is_some() => (CompressionMethod::Aes, options.aes_mode), + #[cfg(feature = "aes-crypto")] + Some(EncryptWith::Aes { mode, .. }) => ( + CompressionMethod::Aes, + Some((mode, AesVendorVersion::Ae2, options.compression_method)), + ), + _ => (options.compression_method, None), + }; + + // Write AES encryption extra data. + #[allow(unused_mut)] + let mut aes_extra_data_start = 0; + #[cfg(feature = "aes-crypto")] + if let Some((mode, vendor, underlying)) = aes_mode { + // For raw copies of AES entries, write the correct AES extra data immediately + let mut body = [0; 7]; + [body[0], body[1]] = (vendor as u16).to_le_bytes(); // vendor version (1 or 2) + [body[2], body[3]] = *b"AE"; // vendor id + body[4] = mode as u8; // strength + [body[5], body[6]] = underlying.serialize_to_u16().to_le_bytes(); // real compression method + aes_extra_data_start = extra_data.len() as u64; + ExtendedFileOptions::add_extra_data_unchecked(&mut extra_data, 0x9901, &body)?; + } + let header_end = + header_start + size_of::() as u64 + name.to_string().len() as u64; + + if options.alignment > 1 { + let extra_data_end = header_end + extra_data.len() as u64; + let align = options.alignment as u64; + let unaligned_header_bytes = extra_data_end % align; + if unaligned_header_bytes != 0 { + let mut pad_length = (align - unaligned_header_bytes) as usize; + while pad_length < 6 { + pad_length += align as usize; + } + let new_len = extra_data.len() + pad_length; + if new_len > u16::MAX as usize { + // Alignment is impossible without exceeding extra field size limits. + // Skip alignment. + } else { + // Add an extra field to the extra_data, per APPNOTE 4.6.11 + let mut pad_body = vec![0; pad_length - 4]; + debug_assert!(pad_body.len() >= 2); + [pad_body[0], pad_body[1]] = options.alignment.to_le_bytes(); + ExtendedFileOptions::add_extra_data_unchecked( + &mut extra_data, + 0xa11e, + &pad_body, + )?; + debug_assert_eq!((extra_data.len() as u64 + header_end) % align, 0); + } + } + } + let extra_data_len = extra_data.len(); + if let Some(data) = central_extra_data { + if extra_data_len + data.len() > u16::MAX as usize { + return Err(invalid!( + "Extra data and central extra data must be less than 64KiB when combined" + )); + } + ExtendedFileOptions::validate_extra_data(data, true)?; + } + let mut file = ZipFileData::initialize_local_block( + name, + &options, + raw_values, + header_start, + None, + aes_extra_data_start, + compression_method, + aes_mode, + &extra_data, + ); + file.using_data_descriptor = + !self.seek_possible || matches!(options.encrypt_with, Some(EncryptWith::ZipCrypto(..))); + file.version_made_by = file.version_made_by.max(file.version_needed() as u8); + file.extra_data_start = Some(header_end); + let index = self.insert_file_data(file)?; + self.writing_to_file = true; + let result: ZipResult<()> = { + ExtendedFileOptions::validate_extra_data(&extra_data, false)?; + let file = &mut self.files[index]; + let block = file.local_block()?; + let writer = self.inner.get_plain(); + block.write(writer)?; + // file name + writer.write_all(&file.file_name_raw)?; + if extra_data_len > 0 { + writer.write_all(&extra_data)?; + file.extra_field = Some(extra_data.into()); + } + Ok(()) + }; + self.ok_or_abort_file(result)?; + let writer = self.inner.get_plain(); + self.stats.start = writer.stream_position()?; + match options.encrypt_with { + #[cfg(feature = "aes-crypto")] + Some(EncryptWith::Aes { mode, password }) => { + let aeswriter = AesWriter::new( + mem::replace(&mut self.inner, Closed).unwrap(), + mode, + password.as_bytes(), + )?; + self.inner = Storer(MaybeEncrypted::Aes(aeswriter)); + } + Some(EncryptWith::ZipCrypto(keys, ..)) => { + let file = &mut self.files[index]; + // With ZipCrypto, we _need_ to use a data descriptor so that + // we can initialize the stream properly. + let mut zipwriter = crate::zipcrypto::ZipCryptoWriter { + writer: mem::replace(&mut self.inner, Closed).unwrap(), + keys, + }; + self.stats.start = zipwriter.writer.stream_position()?; + // crypto_header is counted as part of the data + let mut crypto_header = [0u8; 12]; + // The last two bytes of the header should either be the CRC of + // the file _OR_ the last timepart of the last modified time if + // a data descriptor is used. + // + // However, this header is encrypted, and ZipCrypto is not a + // seekable encryption algorithm - an earlier plaintext byte + // will have an impact on the later values. + // + // This makes it impossible to write the file without first + // calculating its CRC. To avoid having to read the file twice + // or keeping the entire file in memory, we force all ZipCrypto + // files to use a data descriptor. This way, we can use the + // local_modified_time as a password check byte instead of the + // CRC. + crypto_header[10..=11].copy_from_slice( + &file + .last_modified_time + .unwrap_or_else(DateTime::default_for_write) + .timepart() + .to_le_bytes(), + ); + let result = zipwriter.write_all(&crypto_header); + self.ok_or_abort_file(result)?; + self.inner = Storer(MaybeEncrypted::ZipCrypto(zipwriter)); + } + None => {} + } + let file = &mut self.files[index]; + debug_assert!(file.data_start.get().is_none()); + file.data_start.get_or_init(|| self.stats.start); + self.stats.bytes_written = 0; + self.stats.hasher = Hasher::new(); + Ok(()) + } + + fn insert_file_data(&mut self, file: ZipFileData) -> ZipResult { + if self.files.contains_key(&file.file_name) { + return Err(invalid!("Duplicate filename: {}", file.file_name)); + } + let (index, _) = self.files.insert_full(file.file_name.clone(), file); + Ok(index) + } + + fn finish_file(&mut self) -> ZipResult<()> { + if !self.writing_to_file { + return Ok(()); + } + + let make_plain_writer = self.inner.prepare_next_writer( + Stored, + None, + #[cfg(feature = "deflate-zopfli")] + None, + )?; + self.inner.switch_to(make_plain_writer)?; + self.switch_to_non_encrypting_writer()?; + let writer = self.inner.get_plain(); + + if !self.writing_raw { + let file = match self.files.last_mut() { + None => return Ok(()), + Some((_, f)) => f, + }; + file.uncompressed_size = self.stats.bytes_written; + + let file_end = writer.stream_position()?; + debug_assert!(file_end >= self.stats.start); + file.compressed_size = file_end - self.stats.start; + + if !file.using_data_descriptor { + // Not using a data descriptor means the underlying writer + // supports seeking, so we can go back and update the AES Extra + // Data header to use AE1 for large files. + update_aes_extra_data(writer, file, self.stats.bytes_written)?; + } + + let crc = !matches!(file.aes_mode, Some((_, AesVendorVersion::Ae2, _))); + + file.crc32 = if crc { + self.stats.hasher.clone().finalize() + } else { + 0 + }; + + if file.using_data_descriptor { + file.write_data_descriptor(writer, self.auto_large_file)?; + } else { + update_local_file_header(writer, file)?; + writer.seek(SeekFrom::Start(file_end))?; + } + } + if self.flush_on_finish_file { + let result = writer.flush(); + self.ok_or_abort_file(result)?; + } + + self.writing_to_file = false; + Ok(()) + } + + fn switch_to_non_encrypting_writer(&mut self) -> Result<(), ZipError> { + match mem::replace(&mut self.inner, Closed) { + #[cfg(feature = "aes-crypto")] + Storer(MaybeEncrypted::Aes(writer)) => { + self.inner = Storer(MaybeEncrypted::Unencrypted(writer.finish()?)); + } + Storer(MaybeEncrypted::ZipCrypto(writer)) => { + self.inner = Storer(MaybeEncrypted::Unencrypted(writer.finish()?)) + } + Storer(MaybeEncrypted::Unencrypted(w)) => { + self.inner = Storer(MaybeEncrypted::Unencrypted(w)) + } + _ => unreachable!(), + } + Ok(()) + } + + /// Removes the file currently being written from the archive if there is one, or else removes + /// the file most recently written. + pub fn abort_file(&mut self) -> ZipResult<()> { + let (_, last_file) = self.files.pop().ok_or(ZipError::FileNotFound)?; + let make_plain_writer = self.inner.prepare_next_writer( + Stored, + None, + #[cfg(feature = "deflate-zopfli")] + None, + )?; + self.inner.switch_to(make_plain_writer)?; + self.switch_to_non_encrypting_writer()?; + // Make sure this is the last file, and that no shallow copies of it remain; otherwise we'd + // overwrite a valid file and corrupt the archive + let rewind_safe: bool = match last_file.data_start.get() { + None => self.files.is_empty(), + Some(last_file_start) => self.files.values().all(|file| { + file.data_start + .get() + .is_some_and(|start| start < last_file_start) + }), + }; + if rewind_safe { + self.inner + .get_plain() + .seek(SeekFrom::Start(last_file.header_start))?; + } + self.writing_to_file = false; + Ok(()) + } + + /// Create a file in the archive and start writing its' contents. The file must not have the + /// same name as a file already in the archive. + /// + /// The data should be written using the [`Write`] implementation on this [`ZipWriter`] + pub fn start_file( + &mut self, + name: S, + mut options: FileOptions, + ) -> ZipResult<()> { + options.normalize(); + let make_new_self = self.inner.prepare_next_writer( + options.compression_method, + options.compression_level, + #[cfg(feature = "deflate-zopfli")] + options.zopfli_buffer_size, + )?; + self.start_entry(name, options, None)?; + let result = self.inner.switch_to(make_new_self); + self.ok_or_abort_file(result)?; + self.writing_raw = false; + Ok(()) + } + + /* TODO: link to/use Self::finish_into_readable() from https://github.com/zip-rs/zip/pull/400 in + * this docstring. */ + /// Copy over the entire contents of another archive verbatim. + /// + /// This method extracts file metadata from the `source` archive, then simply performs a single + /// big [`io::copy()`](io::copy) to transfer all the actual file contents without any + /// decompression or decryption. This is more performant than the equivalent operation of + /// calling [`Self::raw_copy_file()`] for each entry from the `source` archive in sequence. + /// + ///``` + /// # fn main() -> Result<(), zip::result::ZipError> { + /// # #[cfg(any(feature = "deflate-flate2", not(feature = "_deflate-any")))] + /// # { + /// use std::io::{Cursor, Write, Read}; + /// use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions}; + /// + /// let buf = Cursor::new(Vec::new()); + /// let mut zip = ZipWriter::new(buf); + /// zip.start_file("a.txt", SimpleFileOptions::default())?; + /// zip.write_all(b"hello\n")?; + /// let src = ZipArchive::new(zip.finish()?)?; + /// + /// let buf = Cursor::new(Vec::new()); + /// let mut zip = ZipWriter::new(buf); + /// zip.start_file("b.txt", SimpleFileOptions::default())?; + /// zip.write_all(b"hey\n")?; + /// let src2 = ZipArchive::new(zip.finish()?)?; + /// + /// let buf = Cursor::new(Vec::new()); + /// + /// let mut zip = ZipWriter::new(buf); + /// zip.merge_archive(src)?; + /// zip.merge_archive(src2)?; + /// let mut result = ZipArchive::new(zip.finish()?)?; + /// + /// let mut s: String = String::new(); + /// result.by_name("a.txt")?.read_to_string(&mut s)?; + /// assert_eq!(s, "hello\n"); + /// s.clear(); + /// result.by_name("b.txt")?.read_to_string(&mut s)?; + /// assert_eq!(s, "hey\n"); + /// # } + /// # Ok(()) + /// # } + ///``` + pub fn merge_archive(&mut self, mut source: ZipArchive) -> ZipResult<()> + where + R: Read + Seek, + { + self.finish_file()?; + + /* Ensure we accept the file contents on faith (and avoid overwriting the data). + * See raw_copy_file_rename(). */ + self.writing_to_file = true; + self.writing_raw = true; + + let writer = self.inner.get_plain(); + /* Get the file entries from the source archive. */ + let new_files = source.merge_contents(writer)?; + + /* These file entries are now ours! */ + self.files.extend(new_files); + + Ok(()) + } + + /// Starts a file, taking a Path as argument. + /// + /// This function ensures that the '/' path separator is used and normalizes `.` and `..`. It + /// ignores any `..` or Windows drive letter that would produce a path outside the ZIP file's + /// root. + pub fn start_file_from_path>( + &mut self, + path: P, + options: FileOptions, + ) -> ZipResult<()> { + self.start_file(path_to_string(path), options) + } + + /// Add a new file using the already compressed data from a ZIP file being read and renames it, this + /// allows faster copies of the `ZipFile` since there is no need to decompress and compress it again. + /// Any `ZipFile` metadata is copied and not checked, for example the file CRC. + /// + /// ```no_run + /// use std::fs::File; + /// use std::io::{Read, Seek, Write}; + /// use zip::{ZipArchive, ZipWriter}; + /// + /// fn copy_rename( + /// src: &mut ZipArchive, + /// dst: &mut ZipWriter, + /// ) -> zip::result::ZipResult<()> + /// where + /// R: Read + Seek, + /// W: Write + Seek, + /// { + /// // Retrieve file entry by name + /// let file = src.by_name("src_file.txt")?; + /// + /// // Copy and rename the previously obtained file entry to the destination zip archive + /// dst.raw_copy_file_rename(file, "new_name.txt")?; + /// + /// Ok(()) + /// } + /// ``` + pub fn raw_copy_file_rename( + &mut self, + file: ZipFile, + name: S, + ) -> ZipResult<()> { + let options = file.options(); + self.raw_copy_file_rename_internal(file, name, options) + } + + fn raw_copy_file_rename_internal( + &mut self, + mut file: ZipFile, + name: S, + options: SimpleFileOptions, + ) -> ZipResult<()> { + let raw_values = ZipRawValues { + crc32: file.crc32(), + compressed_size: file.compressed_size(), + uncompressed_size: file.size(), + }; + + self.start_entry(name, options, Some(raw_values))?; + self.writing_to_file = true; + self.writing_raw = true; + + io::copy(&mut file.take_raw_reader()?, self)?; + self.finish_file() + } + + /// Like `raw_copy_file_to_path`, but uses Path arguments. + /// + /// This function ensures that the '/' path separator is used and normalizes `.` and `..`. It + /// ignores any `..` or Windows drive letter that would produce a path outside the ZIP file's + /// root. + pub fn raw_copy_file_to_path>( + &mut self, + file: ZipFile, + path: P, + ) -> ZipResult<()> { + self.raw_copy_file_rename(file, path_to_string(path)) + } + + /// Add a new file using the already compressed data from a ZIP file being read, this allows faster + /// copies of the `ZipFile` since there is no need to decompress and compress it again. Any `ZipFile` + /// metadata is copied and not checked, for example the file CRC. + /// + /// ```no_run + /// use std::fs::File; + /// use std::io::{Read, Seek, Write}; + /// use zip::{ZipArchive, ZipWriter}; + /// + /// fn copy(src: &mut ZipArchive, dst: &mut ZipWriter) -> zip::result::ZipResult<()> + /// where + /// R: Read + Seek, + /// W: Write + Seek, + /// { + /// // Retrieve file entry by name + /// let file = src.by_name("src_file.txt")?; + /// + /// // Copy the previously obtained file entry to the destination zip archive + /// dst.raw_copy_file(file)?; + /// + /// Ok(()) + /// } + /// ``` + pub fn raw_copy_file(&mut self, file: ZipFile) -> ZipResult<()> { + let name = file.name().to_owned(); + self.raw_copy_file_rename(file, name) + } + + /// Add a new file using the already compressed data from a ZIP file being read and set the last + /// modified date and unix mode. This allows faster copies of the `ZipFile` since there is no need + /// to decompress and compress it again. Any `ZipFile` metadata other than the last modified date + /// and the unix mode is copied and not checked, for example the file CRC. + /// + /// ```no_run + /// use std::io::{Read, Seek, Write}; + /// use zip::{DateTime, ZipArchive, ZipWriter}; + /// + /// fn copy(src: &mut ZipArchive, dst: &mut ZipWriter) -> zip::result::ZipResult<()> + /// where + /// R: Read + Seek, + /// W: Write + Seek, + /// { + /// // Retrieve file entry by name + /// let file = src.by_name("src_file.txt")?; + /// + /// // Copy the previously obtained file entry to the destination zip archive + /// dst.raw_copy_file_touch(file, DateTime::default(), Some(0o644))?; + /// + /// Ok(()) + /// } + /// ``` + pub fn raw_copy_file_touch( + &mut self, + file: ZipFile, + last_modified_time: DateTime, + unix_mode: Option, + ) -> ZipResult<()> { + let name = file.name().to_owned(); + + let mut options = file.options(); + + options = options.last_modified_time(last_modified_time); + + if let Some(perms) = unix_mode { + options = options.unix_permissions(perms); + } + + options.normalize(); + + self.raw_copy_file_rename_internal(file, name, options) + } + + /// Add a directory entry. + /// + /// As directories have no content, you must not call [`ZipWriter::write`] before adding a new file. + pub fn add_directory( + &mut self, + name: S, + mut options: FileOptions, + ) -> ZipResult<()> + where + S: Into, + { + if options.permissions.is_none() { + options.permissions = Some(0o755); + } + *options.permissions.as_mut().unwrap() |= 0o40000; + options.compression_method = Stored; + options.encrypt_with = None; + + let name_as_string = name.into(); + // Append a slash to the filename if it does not end with it. + let name_with_slash = match name_as_string.chars().last() { + Some('/') | Some('\\') => name_as_string, + _ => name_as_string + "/", + }; + + self.start_entry(name_with_slash, options, None)?; + self.writing_to_file = false; + self.switch_to_non_encrypting_writer()?; + Ok(()) + } + + /// Add a directory entry, taking a Path as argument. + /// + /// This function ensures that the '/' path separator is used and normalizes `.` and `..`. It + /// ignores any `..` or Windows drive letter that would produce a path outside the ZIP file's + /// root. + pub fn add_directory_from_path>( + &mut self, + path: P, + options: FileOptions, + ) -> ZipResult<()> { + self.add_directory(path_to_string(path), options) + } + + /// Finish the last file and write all other zip-structures + /// + /// This will return the writer, but one should normally not append any data to the end of the file. + /// Note that the zipfile will also be finished on drop. + pub fn finish(mut self) -> ZipResult { + let _central_start = self.finalize()?; + let inner = mem::replace(&mut self.inner, Closed); + Ok(inner.unwrap()) + } + + /// Add a symlink entry. + /// + /// The zip archive will contain an entry for path `name` which is a symlink to `target`. + /// + /// No validation or normalization of the paths is performed. For best results, + /// callers should normalize `\` to `/` and ensure symlinks are relative to other + /// paths within the zip archive. + /// + /// WARNING: not all zip implementations preserve symlinks on extract. Some zip + /// implementations may materialize a symlink as a regular file, possibly with the + /// content incorrectly set to the symlink target. For maximum portability, consider + /// storing a regular file instead. + pub fn add_symlink( + &mut self, + name: N, + target: T, + mut options: FileOptions, + ) -> ZipResult<()> { + if options.permissions.is_none() { + options.permissions = Some(0o777); + } + *options.permissions.as_mut().unwrap() |= S_IFLNK; + // The symlink target is stored as file content. And compressing the target path + // likely wastes space. So always store. + options.compression_method = Stored; + + self.start_entry(name, options, None)?; + self.writing_to_file = true; + let result = self.write_all(target.to_string().as_bytes()); + self.ok_or_abort_file(result)?; + self.writing_raw = false; + self.finish_file()?; + + Ok(()) + } + + /// Add a symlink entry, taking Paths to the location and target as arguments. + /// + /// This function ensures that the '/' path separator is used and normalizes `.` and `..`. It + /// ignores any `..` or Windows drive letter that would produce a path outside the ZIP file's + /// root. + pub fn add_symlink_from_path, T: AsRef, E: FileOptionExtension>( + &mut self, + path: P, + target: T, + options: FileOptions, + ) -> ZipResult<()> { + self.add_symlink(path_to_string(path), path_to_string(target), options) + } + + fn finalize(&mut self) -> ZipResult { + self.finish_file()?; + + let mut central_start = self.write_central_and_footer()?; + let writer = self.inner.get_plain(); + let footer_end = writer.stream_position()?; + let archive_end = writer.seek(SeekFrom::End(0))?; + if footer_end < archive_end { + // Data from an aborted file is past the end of the footer. + + // Overwrite the magic so the footer is no longer valid. + writer.seek(SeekFrom::Start(central_start))?; + writer.write_u32_le(0)?; + writer.seek(SeekFrom::Start( + footer_end - size_of::() as u64 - self.comment.len() as u64, + ))?; + writer.write_u32_le(0)?; + + // Rewrite the footer at the actual end. + let central_and_footer_size = footer_end - central_start; + writer.seek(SeekFrom::End(-(central_and_footer_size as i64)))?; + central_start = self.write_central_and_footer()?; + debug_assert!(self.inner.get_plain().stream_position()? == archive_end); + } + + Ok(central_start) + } + + fn write_central_and_footer(&mut self) -> Result { + let writer = self.inner.get_plain(); + + let mut version_needed = MIN_VERSION as u16; + let central_start = writer.stream_position()?; + for file in self.files.values() { + write_central_directory_header(writer, file)?; + version_needed = version_needed.max(file.version_needed()); + } + let central_size = writer.stream_position()? - central_start; + let is64 = self.files.len() > spec::ZIP64_ENTRY_THR + || central_size.max(central_start) > spec::ZIP64_BYTES_THR + || self.zip64_comment.is_some(); + + if is64 { + let comment = self.zip64_comment.clone().unwrap_or_default(); + + let zip64_footer = spec::Zip64CentralDirectoryEnd { + record_size: comment.len() as u64 + 44, + version_made_by: version_needed, + version_needed_to_extract: version_needed, + disk_number: 0, + disk_with_central_directory: 0, + number_of_files_on_this_disk: self.files.len() as u64, + number_of_files: self.files.len() as u64, + central_directory_size: central_size, + central_directory_offset: central_start, + extensible_data_sector: comment, + }; + + zip64_footer.write(writer)?; + + let zip64_footer = spec::Zip64CentralDirectoryEndLocator { + disk_with_central_directory: 0, + end_of_central_directory_offset: central_start + central_size, + number_of_disks: 1, + }; + + zip64_footer.write(writer)?; + } + + let number_of_files = self.files.len().min(spec::ZIP64_ENTRY_THR) as u16; + let footer = spec::Zip32CentralDirectoryEnd { + disk_number: 0, + disk_with_central_directory: 0, + zip_file_comment: self.comment.clone(), + number_of_files_on_this_disk: number_of_files, + number_of_files, + central_directory_size: central_size.min(spec::ZIP64_BYTES_THR) as u32, + central_directory_offset: central_start.min(spec::ZIP64_BYTES_THR) as u32, + }; + + footer.write(writer)?; + Ok(central_start) + } + + fn index_by_name(&self, name: &str) -> ZipResult { + self.files.get_index_of(name).ok_or(ZipError::FileNotFound) + } + + /// Adds another entry to the central directory referring to the same content as an existing + /// entry. The file's local-file header will still refer to it by its original name, so + /// unzipping the file will technically be unspecified behavior. [ZipArchive] ignores the + /// filename in the local-file header and treat the central directory as authoritative. However, + /// some other software (e.g. Minecraft) will refuse to extract a file copied this way. + pub fn shallow_copy_file(&mut self, src_name: &str, dest_name: &str) -> ZipResult<()> { + self.finish_file()?; + if src_name == dest_name { + return Err(invalid!("Trying to copy a file to itself")); + } + let src_index = self.index_by_name(src_name)?; + let mut dest_data = self.files[src_index].to_owned(); + dest_data.file_name = dest_name.to_string().into(); + dest_data.file_name_raw = dest_name.to_string().into_bytes().into(); + dest_data.central_header_start = 0; + self.insert_file_data(dest_data)?; + + Ok(()) + } + + /// Like `shallow_copy_file`, but uses Path arguments. + /// + /// This function ensures that the '/' path separator is used and normalizes `.` and `..`. It + /// ignores any `..` or Windows drive letter that would produce a path outside the ZIP file's + /// root. + pub fn shallow_copy_file_from_path, U: AsRef>( + &mut self, + src_path: T, + dest_path: U, + ) -> ZipResult<()> { + self.shallow_copy_file(&path_to_string(src_path), &path_to_string(dest_path)) + } +} + +impl ZipWriter> { + /// Creates a writer that doesn't require the inner writer to implement [Seek], but where + /// operations that would overwrite previously-written bytes or cause subsequent operations to + /// do so (such as `abort_file`) will always return an error. + pub fn new_stream(inner: W) -> ZipWriter> { + ZipWriter { + inner: Storer(MaybeEncrypted::Unencrypted(StreamWriter::new(inner))), + files: IndexMap::new(), + stats: Default::default(), + writing_to_file: false, + writing_raw: false, + comment: Box::new([]), + zip64_comment: None, + flush_on_finish_file: false, + seek_possible: false, + auto_large_file: false, + } + } +} + +impl Drop for ZipWriter { + fn drop(&mut self) { + if !self.inner.is_closed() { + if let Err(e) = self.finalize() { + let _ = write!( + io::stderr(), + "ZipWriter::drop: failed to finalize archive: {e:?}" + ); + } + } + } +} + +type SwitchWriterFunction = Box) -> ZipResult>>; + +impl GenericZipWriter { + fn prepare_next_writer( + &self, + compression: CompressionMethod, + compression_level: Option, + #[cfg(feature = "deflate-zopfli")] zopfli_buffer_size: Option, + ) -> ZipResult> { + if let Closed = self { + return Err( + io::Error::new(io::ErrorKind::BrokenPipe, "ZipWriter was already closed").into(), + ); + } + + match compression { + Stored => { + if compression_level.is_some() { + Err(UnsupportedArchive("Unsupported compression level")) + } else { + Ok(Box::new(|bare| Ok(Storer(bare)))) + } + } + #[allow(unreachable_code)] + #[cfg(feature = "_deflate-any")] + CompressionMethod::Deflated => { + let default = crate::cfg_if_expr! { + i64: + #[cfg(feature = "deflate-flate2")] => Compression::default().level() as i64, + #[cfg(feature = "deflate-zopfli")] => 24, + _ => compile_error!("could not calculate default: unknown deflate variant"), + }; + + let level = validate_value_in_range( + compression_level.unwrap_or(default), + deflate_compression_level_range(), + ) + .ok_or(UnsupportedArchive("Unsupported compression level"))? + as u32; + + #[cfg(feature = "deflate-zopfli")] + { + macro_rules! deflate_zopfli_and_return { + ($bare:expr, $best_non_zopfli:expr) => {{ + let options = Options { + iteration_count: NonZeroU64::try_from( + (level - $best_non_zopfli) as u64, + ) + .unwrap(), + ..Default::default() + }; + Ok(Box::new(move |bare| { + Ok(match zopfli_buffer_size { + Some(size) => BufferedZopfliDeflater(BufWriter::with_capacity( + size, + zopfli::DeflateEncoder::new( + options, + Default::default(), + bare, + ), + )), + None => ZopfliDeflater(zopfli::DeflateEncoder::new( + options, + Default::default(), + bare, + )), + }) + })) + }}; + } + + crate::cfg_if! { + if #[cfg(feature = "deflate-flate2")] { + let best_non_zopfli = Compression::best().level(); + if level > best_non_zopfli { + return deflate_zopfli_and_return!(bare, best_non_zopfli); + } + } else { + let best_non_zopfli = 9; + return deflate_zopfli_and_return!(bare, best_non_zopfli); + } + } + } + crate::cfg_if_expr! { + ZipResult>: + #[cfg(feature = "deflate-flate2")] => Ok(Box::new(move |bare| { + Ok(GenericZipWriter::Deflater(DeflateEncoder::new( + bare, + Compression::new(level), + ))) + })), + _ => unreachable!("deflate writer: have no fallback for this case") + } + } + #[cfg(feature = "deflate64")] + CompressionMethod::Deflate64 => { + Err(UnsupportedArchive("Compressing Deflate64 is not supported")) + } + #[cfg(feature = "_bzip2_any")] + CompressionMethod::Bzip2 => { + let level = validate_value_in_range( + compression_level.unwrap_or(bzip2::Compression::default().level() as i64), + bzip2_compression_level_range(), + ) + .ok_or(UnsupportedArchive("Unsupported compression level"))? + as u32; + Ok(Box::new(move |bare| { + Ok(Bzip2(BzEncoder::new(bare, bzip2::Compression::new(level)))) + })) + } + CompressionMethod::AES => Err(UnsupportedArchive( + "AES encryption is enabled through FileOptions::with_aes_encryption", + )), + #[cfg(feature = "zstd")] + CompressionMethod::Zstd => { + let level = validate_value_in_range( + compression_level.unwrap_or(zstd::DEFAULT_COMPRESSION_LEVEL as i64), + zstd::compression_level_range(), + ) + .ok_or(UnsupportedArchive("Unsupported compression level"))?; + Ok(Box::new(move |bare| { + Ok(Zstd( + ZstdEncoder::new(bare, level as i32).map_err(ZipError::Io)?, + )) + })) + } + #[cfg(feature = "legacy-zip")] + CompressionMethod::Shrink => Err(ZipError::UnsupportedArchive( + "Shrink compression unsupported", + )), + #[cfg(feature = "legacy-zip")] + CompressionMethod::Reduce(_) => Err(ZipError::UnsupportedArchive( + "Reduce compression unsupported", + )), + #[cfg(feature = "legacy-zip")] + CompressionMethod::Implode => Err(ZipError::UnsupportedArchive( + "Implode compression unsupported", + )), + #[cfg(feature = "lzma")] + CompressionMethod::Lzma => { + Err(UnsupportedArchive("LZMA isn't supported for compression")) + } + #[cfg(feature = "xz")] + CompressionMethod::Xz => { + let level = validate_value_in_range(compression_level.unwrap_or(6), 0..=9) + .ok_or(UnsupportedArchive("Unsupported compression level"))? + as u32; + Ok(Box::new(move |bare| { + Ok(Xz(Box::new( + lzma_rust2::XzWriter::new(bare, lzma_rust2::XzOptions::with_preset(level)) + .map_err(ZipError::Io)?, + ))) + })) + } + #[cfg(feature = "ppmd")] + CompressionMethod::Ppmd => { + const ORDERS: [u32; 10] = [0, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + + let level = validate_value_in_range(compression_level.unwrap_or(7), 1..=9) + .ok_or(UnsupportedArchive("Unsupported compression level"))? + as u32; + + let order = ORDERS[level as usize]; + let memory_size = 1 << (level + 19); + let memory_size_mb = memory_size / 1024 / 1024; + + Ok(Box::new(move |mut bare| { + let parameter: u16 = (order as u16 - 1) + + ((memory_size_mb - 1) << 4) as u16 + + ((ppmd_rust::RestoreMethod::Restart as u16) << 12); + + bare.write_all(¶meter.to_le_bytes()) + .map_err(ZipError::Io)?; + + let encoder = ppmd_rust::Ppmd8Encoder::new( + bare, + order, + memory_size, + ppmd_rust::RestoreMethod::Restart, + ) + .map_err(|error| match error { + ppmd_rust::Error::RangeDecoderInitialization => ZipError::InvalidArchive( + "PPMd range coder initialization failed".into(), + ), + ppmd_rust::Error::InvalidParameter => { + ZipError::InvalidArchive("Invalid PPMd parameter".into()) + } + ppmd_rust::Error::IoError(io_error) => ZipError::Io(io_error), + ppmd_rust::Error::MemoryAllocation => ZipError::Io(io::Error::new( + ErrorKind::OutOfMemory, + "PPMd could not allocate memory", + )), + })?; + + Ok(Ppmd(Box::new(encoder))) + })) + } + #[allow(deprecated)] + CompressionMethod::Unsupported(..) => { + Err(UnsupportedArchive("Unsupported compression")) + } + } + } + + fn switch_to(&mut self, make_new_self: SwitchWriterFunction) -> ZipResult<()> { + let bare = match mem::replace(self, Closed) { + Storer(w) => w, + #[cfg(feature = "deflate-flate2")] + Deflater(w) => w.finish()?, + #[cfg(feature = "deflate-zopfli")] + ZopfliDeflater(w) => w.finish()?, + #[cfg(feature = "deflate-zopfli")] + BufferedZopfliDeflater(w) => w + .into_inner() + .map_err(|e| ZipError::Io(e.into_error()))? + .finish()?, + #[cfg(feature = "_bzip2_any")] + Bzip2(w) => w.finish()?, + #[cfg(feature = "zstd")] + Zstd(w) => w.finish()?, + #[cfg(feature = "xz")] + Xz(w) => w.finish()?, + #[cfg(feature = "ppmd")] + Ppmd(w) => { + // ZIP needs to encode an end marker (7z for example doesn't encode one). + w.finish(true)? + } + Closed => { + return Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "ZipWriter was already closed", + ) + .into()); + } + }; + *self = make_new_self(bare)?; + Ok(()) + } + + fn ref_mut(&mut self) -> Option<&mut dyn Write> { + match self { + Storer(ref mut w) => Some(w as &mut dyn Write), + #[cfg(feature = "deflate-flate2")] + Deflater(ref mut w) => Some(w as &mut dyn Write), + #[cfg(feature = "deflate-zopfli")] + ZopfliDeflater(w) => Some(w as &mut dyn Write), + #[cfg(feature = "deflate-zopfli")] + BufferedZopfliDeflater(w) => Some(w as &mut dyn Write), + #[cfg(feature = "_bzip2_any")] + Bzip2(ref mut w) => Some(w as &mut dyn Write), + #[cfg(feature = "zstd")] + Zstd(ref mut w) => Some(w as &mut dyn Write), + #[cfg(feature = "xz")] + Xz(ref mut w) => Some(w as &mut dyn Write), + #[cfg(feature = "ppmd")] + Ppmd(ref mut w) => Some(w as &mut dyn Write), + Closed => None, + } + } + + const fn is_closed(&self) -> bool { + matches!(*self, Closed) + } + + fn get_plain(&mut self) -> &mut W { + match *self { + Storer(MaybeEncrypted::Unencrypted(ref mut w)) => w, + _ => panic!("Should have switched to stored and unencrypted beforehand"), + } + } + + fn unwrap(self) -> W { + match self { + Storer(MaybeEncrypted::Unencrypted(w)) => w, + _ => panic!("Should have switched to stored and unencrypted beforehand"), + } + } +} + +#[cfg(feature = "_deflate-any")] +fn deflate_compression_level_range() -> std::ops::RangeInclusive { + let min = crate::cfg_if_expr! { + i64: + #[cfg(feature = "deflate-flate2")] => Compression::fast().level() as i64, + #[cfg(feature = "deflate-zopfli")] => 1, + _ => compile_error!("min: unknown deflate variant"), + }; + + let max = crate::cfg_if_expr! { + i64: + #[cfg(feature = "deflate-zopfli")] => 264, + #[cfg(feature = "deflate-flate2")] => Compression::best().level() as i64, + _ => compile_error!("max: unknown deflate variant"), + }; + + min..=max +} + +#[cfg(feature = "_bzip2_any")] +fn bzip2_compression_level_range() -> std::ops::RangeInclusive { + let min = bzip2::Compression::fast().level() as i64; + let max = bzip2::Compression::best().level() as i64; + min..=max +} + +#[cfg(any( + feature = "_deflate-any", + feature = "_bzip2_any", + feature = "ppmd", + feature = "xz", + feature = "zstd", +))] +fn validate_value_in_range>( + value: T, + range: std::ops::RangeInclusive, +) -> Option { + if range.contains(&value.try_into().ok()?) { + Some(value) + } else { + None + } +} + +fn update_aes_extra_data( + writer: &mut W, + file: &mut ZipFileData, + bytes_written: u64, +) -> ZipResult<()> { + let Some((aes_mode, version, compression_method)) = &mut file.aes_mode else { + return Ok(()); + }; + + // We prefer using AE-1 which provides an extra CRC check, but for small files we + // switch to AE-2 to prevent being able to use the CRC value to to reconstruct the + // unencrypted contents. + // + // We can only do this when the underlying writer supports + // seek operations, so we gate this behind using_data_descriptor. + // + // C.f. https://www.winzip.com/en/support/aes-encryption/#crc-faq + *version = if bytes_written < 20 { + AesVendorVersion::Ae2 + } else { + AesVendorVersion::Ae1 + }; + + let extra_data_start = file.extra_data_start.unwrap(); + + writer.seek(SeekFrom::Start( + extra_data_start + file.aes_extra_data_start, + ))?; + + let mut buf = Vec::new(); + + /* TODO: implement this using the Block trait! */ + // Extra field header ID. + buf.write_u16_le(0x9901)?; + // Data size. + buf.write_u16_le(7)?; + // Integer version number. + buf.write_u16_le(*version as u16)?; + // Vendor ID. + buf.write_all(b"AE")?; + // AES encryption strength. + buf.write_all(&[*aes_mode as u8])?; + // Real compression method. + buf.write_u16_le(compression_method.serialize_to_u16())?; + + writer.write_all(&buf)?; + + let aes_extra_data_start = file.aes_extra_data_start as usize; + let extra_field = Arc::get_mut(file.extra_field.as_mut().unwrap()).unwrap(); + extra_field[aes_extra_data_start..aes_extra_data_start + buf.len()].copy_from_slice(&buf); + + Ok(()) +} + +fn update_local_file_header( + writer: &mut T, + file: &mut ZipFileData, +) -> ZipResult<()> { + const CRC32_OFFSET: u64 = 14; + writer.seek(SeekFrom::Start(file.header_start + CRC32_OFFSET))?; + writer.write_u32_le(file.crc32)?; + if file.large_file { + writer.write_u32_le(spec::ZIP64_BYTES_THR as u32)?; + writer.write_u32_le(spec::ZIP64_BYTES_THR as u32)?; + + update_local_zip64_extra_field(writer, file)?; + + file.compressed_size = spec::ZIP64_BYTES_THR; + file.uncompressed_size = spec::ZIP64_BYTES_THR; + } else { + // check compressed size as well as it can also be slightly larger than uncompressed size + if file.compressed_size > spec::ZIP64_BYTES_THR { + return Err(ZipError::Io(io::Error::other( + "Large file option has not been set", + ))); + } + writer.write_u32_le(file.compressed_size as u32)?; + // uncompressed size is already checked on write to catch it as soon as possible + writer.write_u32_le(file.uncompressed_size as u32)?; + } + Ok(()) +} + +fn write_central_directory_header(writer: &mut T, file: &ZipFileData) -> ZipResult<()> { + let mut block = file.block()?; + let stripped_extra = if let Some(extra) = &file.extra_field { + strip_alignment_extra_field(extra) + } else { + Vec::new() + }; + let central_len = file.central_extra_field_len(); + let zip64_len = if let Some(zip64) = file.zip64_extra_field_block() { + zip64.full_size() + } else { + 0 + }; + block.extra_field_length = (zip64_len + stripped_extra.len() + central_len) + .try_into() + .map_err(|_| invalid!("Extra field length in central directory exceeds 64KiB"))?; + + block.write(writer)?; + // file name + writer.write_all(&file.file_name_raw)?; + // extra field + if let Some(zip64_extra_field) = &file.zip64_extra_field_block() { + writer.write_all(&zip64_extra_field.serialize())?; + } + if !stripped_extra.is_empty() { + writer.write_all(&stripped_extra)?; + } + if let Some(central_extra_field) = &file.central_extra_field { + writer.write_all(central_extra_field)?; + } + // file comment + writer.write_all(file.file_comment.as_bytes())?; + + Ok(()) +} + +fn strip_alignment_extra_field(extra_field: &[u8]) -> Vec { + let mut new_extra = Vec::with_capacity(extra_field.len()); + let mut cursor = 0; + while cursor + 4 <= extra_field.len() { + let tag = u16::from_le_bytes([extra_field[cursor], extra_field[cursor + 1]]); + let len = u16::from_le_bytes([extra_field[cursor + 2], extra_field[cursor + 3]]) as usize; + if cursor + 4 + len > extra_field.len() { + new_extra.extend_from_slice(&extra_field[cursor..]); + break; + } + + if tag != 0xa11e { + new_extra.extend_from_slice(&extra_field[cursor..cursor + 4 + len]); + } + cursor += 4 + len; + } + if cursor < extra_field.len() { + new_extra.extend_from_slice(&extra_field[cursor..]); + } + new_extra +} + +fn update_local_zip64_extra_field( + writer: &mut T, + file: &mut ZipFileData, +) -> ZipResult<()> { + let block = file.zip64_extra_field_block().ok_or(invalid!( + "Attempted to update a nonexistent ZIP64 extra field" + ))?; + + let zip64_extra_field_start = file.header_start + + size_of::() as u64 + + file.file_name_raw.len() as u64; + + writer.seek(SeekFrom::Start(zip64_extra_field_start))?; + let block = block.serialize(); + writer.write_all(&block)?; + + let extra_field = Arc::get_mut(file.extra_field.as_mut().unwrap()).unwrap(); + extra_field[..block.len()].copy_from_slice(&block); + + Ok(()) +} + +/// Wrapper around a [Write] implementation that implements the [Seek] trait, but where seeking +/// returns an error unless it's a no-op. +pub struct StreamWriter { + inner: W, + bytes_written: u64, +} + +impl StreamWriter { + /// Creates an instance wrapping the provided inner writer. + pub fn new(inner: W) -> StreamWriter { + Self { + inner, + bytes_written: 0, + } + } + + /// Consumes this wrapper, returning the underlying writer. + pub fn into_inner(self) -> W { + self.inner + } +} + +impl Write for StreamWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + let bytes_written = self.inner.write(buf)?; + self.bytes_written += bytes_written as u64; + Ok(bytes_written) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +impl Seek for StreamWriter { + fn seek(&mut self, pos: SeekFrom) -> io::Result { + match pos { + SeekFrom::Current(0) | SeekFrom::End(0) => return Ok(self.bytes_written), + SeekFrom::Start(x) => { + if x == self.bytes_written { + return Ok(self.bytes_written); + } + } + _ => {} + } + Err(io::Error::new( + ErrorKind::Unsupported, + "seek is not supported", + )) + } +} + +#[cfg(not(feature = "unreserved"))] +const EXTRA_FIELD_MAPPING: [u16; 43] = [ + 0x0007, 0x0008, 0x0009, 0x000a, 0x000c, 0x000d, 0x000e, 0x000f, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x0020, 0x0021, 0x0022, 0x0023, 0x0065, 0x0066, 0x4690, 0x07c8, 0x2605, 0x2705, + 0x2805, 0x334d, 0x4341, 0x4453, 0x4704, 0x470f, 0x4b46, 0x4c41, 0x4d49, 0x4f4c, 0x5356, 0x554e, + 0x5855, 0x6542, 0x756e, 0x7855, 0xa220, 0xfd4a, 0x9902, +]; + +#[cfg(test)] +#[allow(unknown_lints)] // needless_update is new in clippy pre 1.29.0 +#[allow(clippy::needless_update)] // So we can use the same FileOptions decls with and without zopfli_buffer_size +#[allow(clippy::octal_escapes)] // many false positives in converted fuzz cases +mod test { + use super::{ExtendedFileOptions, FileOptions, FullFileOptions, ZipWriter}; + use crate::compression::CompressionMethod; + use crate::result::ZipResult; + use crate::types::{DateTime, System}; + use crate::write::EncryptWith::ZipCrypto; + use crate::write::SimpleFileOptions; + use crate::zipcrypto::ZipCryptoKeys; + use crate::CompressionMethod::Stored; + use crate::ZipArchive; + #[cfg(feature = "deflate-flate2")] + use std::io::Read; + use std::io::{Cursor, Write}; + use std::marker::PhantomData; + use std::path::PathBuf; + + const SYSTEM_BYTE: u8 = if cfg!(windows) { + System::Dos + } else { + System::Unix + } as u8; + + #[test] + fn write_empty_zip() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_comment("ZIP"); + let result = writer.finish().unwrap(); + assert_eq!(result.get_ref().len(), 25); + assert_eq!( + *result.get_ref(), + [80, 75, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 90, 73, 80] + ); + } + + #[test] + fn unix_permissions_bitmask() { + // unix_permissions() throws away upper bits. + let options = SimpleFileOptions::default().unix_permissions(0o120777); + assert_eq!(options.permissions, Some(0o777)); + } + + #[test] + fn write_zip_dir() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .add_directory( + "test", + SimpleFileOptions::default().last_modified_time( + DateTime::from_date_and_time(2018, 8, 15, 20, 45, 6).unwrap(), + ), + ) + .unwrap(); + assert!(writer + .write(b"writing to a directory is not allowed, and will not write any data") + .is_err()); + let result = writer.finish().unwrap(); + assert_eq!(result.get_ref().len(), 108); + const DOS_DIR_ATTRIBUTES_BYTE: u8 = if cfg!(windows) { 0x10 } else { 0 }; + #[rustfmt::skip] + assert_eq!( + *result.get_ref(), + &[ + 80u8, 75, 3, 4, 20, 0, 0, 0, 0, 0, 163, 165, 15, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 5, 0, 0, 0, 116, 101, 115, 116, 47, 80, 75, 1, 2, 20, SYSTEM_BYTE, 20, 0, 0, 0, 0, 0, + 163, 165, 15, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, + DOS_DIR_ATTRIBUTES_BYTE, 0, 237, 65, 0, 0, 0, 0, 116, 101, 115, 116, 47, 80, 75, 5, 6, 0, 0, 0, 0, 1, 0, + 1, 0, 51, 0, 0, 0, 35, 0, 0, 0, 0, 0, + ] as &[u8] + ); + } + + #[test] + fn write_symlink_simple() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .add_symlink( + "name", + "target", + SimpleFileOptions::default().last_modified_time( + DateTime::from_date_and_time(2018, 8, 15, 20, 45, 6).unwrap(), + ), + ) + .unwrap(); + assert!(writer + .write(b"writing to a symlink is not allowed and will not write any data") + .is_err()); + let result = writer.finish().unwrap(); + assert_eq!(result.get_ref().len(), 112); + #[rustfmt::skip] + assert_eq!( + *result.get_ref(), + &[ + 80u8, 75, 3, 4, 10, 0, 0, 0, 0, 0, 163, 165, 15, 77, 252, 47, 111, 70, 6, 0, 0, 0, + 6, 0, 0, 0, 4, 0, 0, 0, 110, 97, 109, 101, 116, 97, 114, 103, 101, 116, 80, 75, 1, + 2, 10, System::Unix as u8, 10, 0, 0, 0, 0, 0, 163, 165, 15, 77, 252, 47, 111, 70, 6, 0, 0, 0, 6, 0, + 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 161, 0, 0, 0, 0, 110, 97, 109, 101, + 80, 75, 5, 6, 0, 0, 0, 0, 1, 0, 1, 0, 50, 0, 0, 0, 40, 0, 0, 0, 0, 0 + ] as &[u8], + ); + } + + #[test] + fn test_path_normalization() { + let mut path = PathBuf::new(); + path.push("foo"); + path.push("bar"); + path.push(".."); + path.push("."); + path.push("example.txt"); + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file_from_path(path, SimpleFileOptions::default()) + .unwrap(); + let archive = writer.finish_into_readable().unwrap(); + assert_eq!(Some("foo/example.txt"), archive.name_for_index(0)); + } + + #[test] + fn write_symlink_wonky_paths() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .add_symlink( + "directory\\link", + "/absolute/symlink\\with\\mixed/slashes", + SimpleFileOptions::default().last_modified_time( + DateTime::from_date_and_time(2018, 8, 15, 20, 45, 6).unwrap(), + ), + ) + .unwrap(); + assert!(writer + .write(b"writing to a symlink is not allowed and will not write any data") + .is_err()); + let result = writer.finish().unwrap(); + assert_eq!(result.get_ref().len(), 162); + #[rustfmt::skip] + assert_eq!( + *result.get_ref(), + &[ + 80u8, 75, 3, 4, 10, 0, 0, 0, 0, 0, 163, 165, 15, 77, 95, 41, 81, 245, 36, 0, 0, 0, + 36, 0, 0, 0, 14, 0, 0, 0, 100, 105, 114, 101, 99, 116, 111, 114, 121, 92, 108, 105, + 110, 107, 47, 97, 98, 115, 111, 108, 117, 116, 101, 47, 115, 121, 109, 108, 105, + 110, 107, 92, 119, 105, 116, 104, 92, 109, 105, 120, 101, 100, 47, 115, 108, 97, + 115, 104, 101, 115, 80, 75, 1, 2, 10, System::Unix as u8, 10, 0, 0, 0, 0, 0, 163, 165, 15, 77, 95, + 41, 81, 245, 36, 0, 0, 0, 36, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, + 161, 0, 0, 0, 0, 100, 105, 114, 101, 99, 116, 111, 114, 121, 92, 108, 105, 110, + 107, 80, 75, 5, 6, 0, 0, 0, 0, 1, 0, 1, 0, 60, 0, 0, 0, 80, 0, 0, 0, 0, 0 + ] as &[u8], + ); + } + + #[test] + fn write_mimetype_zip() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::default(), + permissions: Some(33188), + large_file: false, + encrypt_with: None, + extended_options: (), + alignment: 1, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: None, + #[cfg(feature = "aes-crypto")] + aes_mode: None, + }; + writer.start_file("mimetype", options).unwrap(); + writer + .write_all(b"application/vnd.oasis.opendocument.text") + .unwrap(); + let result = writer.finish().unwrap(); + + assert_eq!(result.get_ref().len(), 153); + #[rustfmt::skip] + assert_eq!(result.get_ref(), &[80, 75, 3, 4, 10, 0, 0, 0, 0, 0, 0, 0, 33, 0, 94, 198, 50, + 12, 39, 0, 0, 0, 39, 0, 0, 0, 8, 0, 0, 0, 109, 105, 109, 101, 116, 121, 112, 101, 97, + 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 118, 110, 100, 46, 111, 97, 115, + 105, 115, 46, 111, 112, 101, 110, 100, 111, 99, 117, 109, 101, 110, 116, 46, 116, 101, + 120, 116, 80, 75, 1, 2, 10, SYSTEM_BYTE, 10, 0, 0, 0, 0, 0, 0, 0, 33, 0, 94, 198, 50, + 12, 39, 0, 0, 0, 39, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 129, 0, 0, 0, 0, + 109, 105, 109, 101, 116, 121, 112, 101, 80, 75, 5, 6, 0, 0, 0, 0, 1, 0, 1, 0, 54, 0, 0, + 0, 77, 0, 0, 0, 0, 0]); + } + + #[cfg(feature = "deflate-flate2")] + const RT_TEST_TEXT: &str = "And I can't stop thinking about the moments that I lost to you\ + And I can't stop thinking of things I used to do\ + And I can't stop making bad decisions\ + And I can't stop eating stuff you make me chew\ + I put on a smile like you wanna see\ + Another day goes by that I long to be like you"; + #[cfg(feature = "deflate-flate2")] + const RT_TEST_FILENAME: &str = "subfolder/sub-subfolder/can't_stop.txt"; + #[cfg(feature = "deflate-flate2")] + const SECOND_FILENAME: &str = "different_name.xyz"; + #[cfg(feature = "deflate-flate2")] + const THIRD_FILENAME: &str = "third_name.xyz"; + + #[test] + fn write_non_utf8() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::default(), + permissions: Some(33188), + large_file: false, + encrypt_with: None, + extended_options: (), + alignment: 1, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: None, + #[cfg(feature = "aes-crypto")] + aes_mode: None, + }; + + // GB18030 + // "中文" = [214, 208, 206, 196] + let filename = unsafe { String::from_utf8_unchecked(vec![214, 208, 206, 196]) }; + writer.start_file(filename, options).unwrap(); + writer.write_all(b"encoding GB18030").unwrap(); + + // SHIFT_JIS + // "日文" = [147, 250, 149, 182] + let filename = unsafe { String::from_utf8_unchecked(vec![147, 250, 149, 182]) }; + writer.start_file(filename, options).unwrap(); + writer.write_all(b"encoding SHIFT_JIS").unwrap(); + let result = writer.finish().unwrap(); + + assert_eq!(result.get_ref().len(), 224); + #[rustfmt::skip] + assert_eq!(result.get_ref(), &[80, 75, 3, 4, 10, 0, 0, 0, 0, 0, 0, 0, 33, 0, 29, 183, 125, + 75, 16, 0, 0, 0, 16, 0, 0, 0, 4, 0, 0, 0, 214, 208, 206, 196, 101, 110, 99, 111, 100, + 105, 110, 103, 32, 71, 66, 49, 56, 48, 51, 48, 80, 75, 3, 4, 10, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 75, 110, 14, 242, 18, 0, 0, 0, 18, 0, 0, 0, 4, 0, 0, 0, 147, 250, 149, 182, 101, + 110, 99, 111, 100, 105, 110, 103, 32, 83, 72, 73, 70, 84, 95, 74, 73, 83, 80, 75, 1, 2, + 10, SYSTEM_BYTE, 10, 0, 0, 0, 0, 0, 0, 0, 33, 0, 29, 183, 125, 75, 16, 0, 0, 0, 16, 0, + 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 129, 0, 0, 0, 0, 214, 208, 206, 196, 80, + 75, 1, 2, 10, SYSTEM_BYTE, 10, 0, 0, 0, 0, 0, 0, 0, 33, 0, 75, 110, 14, 242, 18, 0, 0, 0, 18, 0, + 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 129, 50, 0, 0, 0, 147, 250, 149, 182, 80, + 75, 5, 6, 0, 0, 0, 0, 2, 0, 2, 0, 100, 0, 0, 0, 102, 0, 0, 0, 0, 0]); + } + + #[test] + fn path_to_string() { + let mut path = PathBuf::new(); + #[cfg(windows)] + path.push(r"C:\"); + #[cfg(unix)] + path.push("/"); + path.push("windows"); + path.push(".."); + path.push("."); + path.push("system32"); + let path_str = super::path_to_string(&path); + assert_eq!(&*path_str, "system32"); + } + + #[test] + #[cfg(feature = "deflate-flate2")] + fn test_shallow_copy() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = FileOptions { + compression_method: CompressionMethod::default(), + compression_level: None, + last_modified_time: DateTime::default(), + permissions: Some(33188), + large_file: false, + encrypt_with: None, + extended_options: (), + alignment: 0, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: None, + #[cfg(feature = "aes-crypto")] + aes_mode: None, + }; + writer.start_file(RT_TEST_FILENAME, options).unwrap(); + writer.write_all(RT_TEST_TEXT.as_ref()).unwrap(); + writer + .shallow_copy_file(RT_TEST_FILENAME, SECOND_FILENAME) + .unwrap(); + writer + .shallow_copy_file(RT_TEST_FILENAME, SECOND_FILENAME) + .expect_err("Duplicate filename"); + let zip = writer.finish().unwrap(); + let mut writer = ZipWriter::new_append(zip).unwrap(); + writer + .shallow_copy_file(SECOND_FILENAME, SECOND_FILENAME) + .expect_err("Duplicate filename"); + let mut reader = writer.finish_into_readable().unwrap(); + let mut file_names: Vec<&str> = reader.file_names().collect(); + file_names.sort(); + let mut expected_file_names = vec![RT_TEST_FILENAME, SECOND_FILENAME]; + expected_file_names.sort(); + assert_eq!(file_names, expected_file_names); + let mut first_file_content = String::new(); + reader + .by_name(RT_TEST_FILENAME) + .unwrap() + .read_to_string(&mut first_file_content) + .unwrap(); + assert_eq!(first_file_content, RT_TEST_TEXT); + let mut second_file_content = String::new(); + reader + .by_name(SECOND_FILENAME) + .unwrap() + .read_to_string(&mut second_file_content) + .unwrap(); + assert_eq!(second_file_content, RT_TEST_TEXT); + } + + #[test] + #[cfg(feature = "deflate-flate2")] + fn test_deep_copy() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = FileOptions { + compression_method: CompressionMethod::default(), + compression_level: None, + last_modified_time: DateTime::default(), + permissions: Some(33188), + large_file: false, + encrypt_with: None, + extended_options: (), + alignment: 0, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: None, + #[cfg(feature = "aes-crypto")] + aes_mode: None, + }; + writer.start_file(RT_TEST_FILENAME, options).unwrap(); + writer.write_all(RT_TEST_TEXT.as_ref()).unwrap(); + writer + .deep_copy_file(RT_TEST_FILENAME, SECOND_FILENAME) + .unwrap(); + let zip = writer.finish().unwrap().into_inner(); + zip.iter().copied().for_each(|x| print!("{x:02x}")); + println!(); + let mut writer = ZipWriter::new_append(Cursor::new(zip)).unwrap(); + writer + .deep_copy_file(RT_TEST_FILENAME, THIRD_FILENAME) + .unwrap(); + let zip = writer.finish().unwrap(); + let mut reader = ZipArchive::new(zip).unwrap(); + let mut file_names: Vec<&str> = reader.file_names().collect(); + file_names.sort(); + let mut expected_file_names = vec![RT_TEST_FILENAME, SECOND_FILENAME, THIRD_FILENAME]; + expected_file_names.sort(); + assert_eq!(file_names, expected_file_names); + let mut first_file_content = String::new(); + reader + .by_name(RT_TEST_FILENAME) + .unwrap() + .read_to_string(&mut first_file_content) + .unwrap(); + assert_eq!(first_file_content, RT_TEST_TEXT); + let mut second_file_content = String::new(); + reader + .by_name(SECOND_FILENAME) + .unwrap() + .read_to_string(&mut second_file_content) + .unwrap(); + assert_eq!(second_file_content, RT_TEST_TEXT); + } + + #[test] + fn duplicate_filenames() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file("foo/bar/test", SimpleFileOptions::default()) + .unwrap(); + writer + .write_all("The quick brown 🦊 jumps over the lazy 🐕".as_bytes()) + .unwrap(); + writer + .start_file("foo/bar/test", SimpleFileOptions::default()) + .expect_err("Expected duplicate filename not to be allowed"); + } + + #[test] + fn test_filename_looks_like_zip64_locator() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file( + "PK\u{6}\u{7}\0\0\0\u{11}\0\0\0\0\0\0\0\0\0\0\0\0", + SimpleFileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + let _ = ZipArchive::new(zip).unwrap(); + } + + #[test] + fn test_filename_looks_like_zip64_locator_2() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file( + "PK\u{6}\u{6}\0\0\0\0\0\0\0\0\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", + SimpleFileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + let _ = ZipArchive::new(zip).unwrap(); + } + + #[test] + fn test_filename_looks_like_zip64_locator_2a() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file( + "PK\u{6}\u{6}PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", + SimpleFileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + let _ = ZipArchive::new(zip).unwrap(); + } + + #[test] + fn test_filename_looks_like_zip64_locator_3() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file("\0PK\u{6}\u{6}", SimpleFileOptions::default()) + .unwrap(); + writer + .start_file( + "\0\u{4}\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{3}", + SimpleFileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + let _ = ZipArchive::new(zip).unwrap(); + } + + #[test] + fn test_filename_looks_like_zip64_locator_4() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file("PK\u{6}\u{6}", SimpleFileOptions::default()) + .unwrap(); + writer + .start_file("\0\0\0\0\0\0", SimpleFileOptions::default()) + .unwrap(); + writer + .start_file("\0", SimpleFileOptions::default()) + .unwrap(); + writer.start_file("", SimpleFileOptions::default()).unwrap(); + writer + .start_file("\0\0", SimpleFileOptions::default()) + .unwrap(); + writer + .start_file( + "\0\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", + SimpleFileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + let _ = ZipArchive::new(zip).unwrap(); + } + + #[test] + fn test_filename_looks_like_zip64_locator_5() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .add_directory("", SimpleFileOptions::default().with_alignment(21)) + .unwrap(); + let mut writer = ZipWriter::new_append(writer.finish().unwrap()).unwrap(); + writer.shallow_copy_file("/", "").unwrap(); + writer.shallow_copy_file("", "\0").unwrap(); + writer.shallow_copy_file("\0", "PK\u{6}\u{6}").unwrap(); + let mut writer = ZipWriter::new_append(writer.finish().unwrap()).unwrap(); + writer + .start_file("\0\0\0\0\0\0", SimpleFileOptions::default()) + .unwrap(); + let mut writer = ZipWriter::new_append(writer.finish().unwrap()).unwrap(); + writer + .start_file( + "#PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", + SimpleFileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + let _ = ZipArchive::new(zip).unwrap(); + Ok(()) + } + + #[test] + #[cfg(feature = "deflate-flate2")] + fn remove_shallow_copy_keeps_original() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file("original", SimpleFileOptions::default()) + .unwrap(); + writer.write_all(RT_TEST_TEXT.as_bytes()).unwrap(); + writer + .shallow_copy_file("original", "shallow_copy") + .unwrap(); + writer.abort_file().unwrap(); + let mut zip = ZipArchive::new(writer.finish().unwrap()).unwrap(); + let mut file = zip.by_name("original").unwrap(); + let mut contents = Vec::new(); + file.read_to_end(&mut contents).unwrap(); + assert_eq!(RT_TEST_TEXT.as_bytes(), contents); + Ok(()) + } + + #[test] + fn remove_encrypted_file() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let first_file_options = SimpleFileOptions::default() + .with_alignment(65535) + .with_deprecated_encryption(b"Password"); + writer.start_file("", first_file_options).unwrap(); + writer.abort_file().unwrap(); + let zip = writer.finish().unwrap(); + let mut writer = ZipWriter::new(zip); + writer.start_file("", SimpleFileOptions::default()).unwrap(); + Ok(()) + } + + #[test] + fn remove_encrypted_aligned_symlink() -> ZipResult<()> { + let mut options = SimpleFileOptions::default(); + options = options.with_deprecated_encryption(b"Password"); + options.alignment = 65535; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.add_symlink("", "s\t\0\0ggggg\0\0", options).unwrap(); + writer.abort_file().unwrap(); + let zip = writer.finish().unwrap(); + let mut writer = ZipWriter::new_append(zip).unwrap(); + writer.start_file("", SimpleFileOptions::default()).unwrap(); + Ok(()) + } + + #[cfg(feature = "deflate-zopfli")] + #[test] + fn zopfli_empty_write() -> ZipResult<()> { + let mut options = SimpleFileOptions::default(); + options = options + .compression_method(CompressionMethod::default()) + .compression_level(Some(264)); + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.start_file("", options).unwrap(); + writer.write_all(&[]).unwrap(); + writer.write_all(&[]).unwrap(); + Ok(()) + } + + #[test] + fn crash_with_no_features() -> ZipResult<()> { + const ORIGINAL_FILE_NAME: &str = "PK\u{6}\u{6}\0\0\0\0\0\0\0\0\0\u{2}g\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\u{7}\0\t'"; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let mut options = SimpleFileOptions::default(); + options = options.with_alignment(3584).compression_method(Stored); + writer.start_file(ORIGINAL_FILE_NAME, options)?; + let archive = writer.finish()?; + let mut writer = ZipWriter::new_append(archive)?; + writer.shallow_copy_file(ORIGINAL_FILE_NAME, "\u{6}\\")?; + writer.finish()?; + Ok(()) + } + + #[test] + fn test_alignment() { + let page_size = 4096; + let options = SimpleFileOptions::default() + .compression_method(Stored) + .with_alignment(page_size); + let mut zip = ZipWriter::new(Cursor::new(Vec::new())); + let contents = b"sleeping"; + let () = zip.start_file("sleep", options).unwrap(); + let _count = zip.write(&contents[..]).unwrap(); + let mut zip = zip.finish_into_readable().unwrap(); + let file = zip.by_index(0).unwrap(); + assert_eq!(file.name(), "sleep"); + assert_eq!(file.data_start(), u64::from(page_size)); + } + + #[test] + fn test_alignment_2() { + let page_size = 4096; + let mut data = Vec::new(); + { + let options = SimpleFileOptions::default() + .compression_method(Stored) + .with_alignment(page_size); + let mut zip = ZipWriter::new(Cursor::new(&mut data)); + let contents = b"sleeping"; + let () = zip.start_file("sleep", options).unwrap(); + let _count = zip.write(&contents[..]).unwrap(); + } + assert_eq!(data[4096..4104], b"sleeping"[..]); + { + let mut zip = ZipArchive::new(Cursor::new(&mut data)).unwrap(); + let file = zip.by_index(0).unwrap(); + assert_eq!(file.name(), "sleep"); + assert_eq!(file.data_start(), u64::from(page_size)); + } + } + + #[test] + fn test_crash_short_read() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let comment = vec![ + 1, 80, 75, 5, 6, 237, 237, 237, 237, 237, 237, 237, 237, 44, 255, 191, 255, 255, 255, + 255, 255, 255, 255, 255, 16, + ] + .into_boxed_slice(); + writer.set_raw_comment(comment); + let options = SimpleFileOptions::default() + .compression_method(Stored) + .with_alignment(11823); + writer.start_file("", options).unwrap(); + writer.write_all(&[255, 255, 44, 255, 0]).unwrap(); + let written = writer.finish().unwrap(); + let _ = ZipWriter::new_append(written).unwrap(); + } + + #[cfg(all(feature = "_deflate-any", feature = "aes-crypto"))] + #[test] + fn test_fuzz_failure_2024_05_08() -> ZipResult<()> { + let mut first_writer = ZipWriter::new(Cursor::new(Vec::new())); + let mut second_writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = SimpleFileOptions::default() + .compression_method(Stored) + .with_alignment(46036); + second_writer.add_symlink("\0", "", options)?; + let second_archive = second_writer.finish_into_readable()?.into_inner(); + let mut second_writer = ZipWriter::new_append(second_archive)?; + let options = SimpleFileOptions::default() + .compression_method(CompressionMethod::Deflated) + .large_file(true) + .with_alignment(46036) + .with_aes_encryption(crate::AesMode::Aes128, "\0\0"); + second_writer.add_symlink("", "", options)?; + let second_archive = second_writer.finish_into_readable()?.into_inner(); + let mut second_writer = ZipWriter::new_append(second_archive)?; + let options = SimpleFileOptions::default().compression_method(Stored); + second_writer.start_file(" ", options)?; + let second_archive = second_writer.finish_into_readable()?; + first_writer.merge_archive(second_archive)?; + let _ = ZipArchive::new(first_writer.finish()?)?; + Ok(()) + } + + #[cfg(all(feature = "_bzip2_any", not(miri)))] + #[test] + fn test_fuzz_failure_2024_06_08() -> ZipResult<()> { + use crate::write::ExtendedFileOptions; + use CompressionMethod::Bzip2; + + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + const SYMLINK_PATH: &str = "PK\u{6}\u{6}K\u{6}\u{6}\u{6}\0\0\0\0\u{18}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\u{1}\0\0PK\u{1}\u{2},\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0PK\u{1}\u{2},\0\0\0\0\0\0\0\0\0\0l\0\0\0\0\0\0PK\u{6}\u{7}P\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0"; + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Bzip2, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 5, 20, 21, 0, 57)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 2048, + ..Default::default() + }; + writer.add_symlink_from_path(SYMLINK_PATH, "||\0\0\0\0", options)?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.deep_copy_file_from_path(SYMLINK_PATH, "")?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.abort_file()?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.deep_copy_file_from_path(SYMLINK_PATH, "foo")?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_short_extra_data() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![99, 0, 15, 0, 207].into(), + }, + ..Default::default() + }; + assert!(writer.start_file_from_path("", options).is_err()); + } + + #[test] + #[cfg(not(feature = "unreserved"))] + fn test_invalid_extra_data() -> ZipResult<()> { + use crate::write::ExtendedFileOptions; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 1, 4, 6, 54, 0)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![ + 7, 0, 15, 0, 207, 117, 177, 117, 112, 2, 0, 255, 255, 131, 255, 255, 255, 80, + 185, + ] + .into(), + }, + alignment: 32787, + ..Default::default() + }; + assert!(writer.start_file_from_path("", options).is_err()); + Ok(()) + } + + #[test] + #[cfg(not(feature = "unreserved"))] + fn test_invalid_extra_data_unreserved() { + use crate::write::ExtendedFileOptions; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2021, 8, 8, 1, 0, 29).unwrap(), + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![ + 1, 41, 4, 0, 1, 255, 245, 117, 117, 112, 5, 0, 80, 255, 149, 255, 247, + ] + .into(), + }, + alignment: 4103, + ..Default::default() + }; + assert!(writer.start_file_from_path("", options).is_err()); + } + + #[cfg(feature = "deflate64")] + #[test] + fn test_fuzz_crash_2024_06_13a() -> ZipResult<()> { + use crate::write::ExtendedFileOptions; + use CompressionMethod::Deflate64; + + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Deflate64, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2039, 4, 17, 6, 18, 19)?, + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 4, + ..Default::default() + }; + writer.add_directory_from_path("", options)?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_13b() -> ZipResult<()> { + use crate::write::ExtendedFileOptions; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 4, 14, 6, 11, 54)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 185, + ..Default::default() + }; + writer.add_symlink_from_path("", "", options)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer.deep_copy_file_from_path("", "_copy")?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_14() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FullFileOptions { + compression_method: Stored, + large_file: true, + alignment: 93, + ..Default::default() + }; + writer.start_file_from_path("\0", options)?; + writer = ZipWriter::new_append(writer.finish()?)?; + writer.deep_copy_file_from_path("\0", "")?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer.deep_copy_file_from_path("", "copy")?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_14a() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2083, 5, 30, 21, 45, 35)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 2565, + ..Default::default() + }; + writer.add_symlink_from_path("", "", options)?; + writer.abort_file()?; + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::default(), + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 0, + ..Default::default() + }; + writer.start_file_from_path("", options)?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[allow(deprecated)] + #[test] + fn test_fuzz_crash_2024_06_14b() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2078, 3, 6, 12, 48, 58)?, + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 65521, + ..Default::default() + }; + writer.start_file_from_path("\u{4}\0@\n//\u{c}", options)?; + writer = ZipWriter::new_append(writer.finish()?)?; + writer.abort_file()?; + let options = FileOptions { + compression_method: CompressionMethod::Unsupported(65535), + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2055, 10, 2, 11, 48, 49)?, + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![255, 255, 1, 0, 255, 0, 0, 0, 0].into(), + central_extra_data: vec![].into(), + }, + alignment: 65535, + ..Default::default() + }; + writer.add_directory_from_path("", options)?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_14c() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2060, 4, 6, 13, 13, 3)?, + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 0, + ..Default::default() + }; + writer.start_file_from_path("\0", options)?; + writer.write_all(&([]))?; + writer = ZipWriter::new_append(writer.finish()?)?; + writer.deep_copy_file_from_path("\0", "")?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer.deep_copy_file_from_path("", "_copy")?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[cfg(all(feature = "_deflate-any", feature = "aes-crypto"))] + #[test] + fn test_fuzz_crash_2024_06_14d() -> ZipResult<()> { + use crate::write::EncryptWith::Aes; + use crate::AesMode::Aes256; + use CompressionMethod::Deflated; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Deflated, + compression_level: Some(5), + last_modified_time: DateTime::from_date_and_time(2107, 4, 8, 15, 54, 19)?, + permissions: None, + large_file: true, + encrypt_with: Some(Aes { + mode: Aes256, + password: "", + }), + extended_options: ExtendedFileOptions { + extra_data: vec![2, 0, 1, 0, 0].into(), + central_extra_data: vec![ + 35, 229, 2, 0, 41, 41, 231, 44, 2, 0, 52, 233, 82, 201, 0, 0, 3, 0, 2, 0, 233, + 255, 3, 0, 2, 0, 26, 154, 38, 251, 0, 0, + ] + .into(), + }, + alignment: 65535, + ..Default::default() + }; + assert!(writer.add_directory_from_path("", options).is_err()); + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_14e() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1988, 1, 1, 1, 6, 26)?, + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![76, 0, 1, 0, 0, 2, 0, 0, 0].into(), + central_extra_data: vec![ + 1, 149, 1, 0, 255, 3, 0, 0, 0, 2, 255, 0, 0, 12, 65, 1, 0, 0, 67, 149, 0, 0, + 76, 149, 2, 0, 149, 149, 67, 149, 0, 0, + ] + .into(), + }, + alignment: 65535, + ..Default::default() + }; + assert!(writer.add_directory_from_path("", options).is_err()); + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[allow(deprecated)] + #[test] + fn test_fuzz_crash_2024_06_17() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = + ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: CompressionMethod::Unsupported( + 65535, + ), + compression_level: Some(5), + last_modified_time: DateTime::from_date_and_time( + 2107, 2, 8, 15, 0, 0, + )?, + permissions: None, + large_file: true, + encrypt_with: Some(ZipCrypto( + ZipCryptoKeys::of( + 0x63ff, 0xc62d3103, 0xfffe00ea, + ), + PhantomData, + )), + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 255, + ..Default::default() + }; + writer.add_symlink_from_path("1\0PK\u{6}\u{6}\u{b}\u{6}\u{6}\u{6}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\u{b}\0\0PK\u{1}\u{2},\0\0\0\0\0\0\0\0\0\0\0\u{10}\0\0\0K\u{6}\u{6}\0\0\0\0\0\0\0\0PK\u{2}\u{6}", "", options)?; + writer = ZipWriter::new_append( + writer.finish_into_readable()?.into_inner(), + )?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append( + writer.finish_into_readable()?.into_inner(), + )?; + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time( + 1992, 7, 3, 0, 0, 0, + )?, + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 43, + ..Default::default() + }; + writer.start_file_from_path( + "\0\0\0\u{3}\0\u{1a}\u{1a}\u{1a}\u{1a}\u{1a}\u{1a}", + options, + )?; + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time( + 2006, 3, 27, 2, 24, 26, + )?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 26, + ..Default::default() + }; + writer.start_file_from_path("\0K\u{6}\u{6}\0PK\u{6}\u{7}PK\u{6}\u{6}\0\0\0\0\0\0\0\0PK\u{2}\u{6}", options)?; + writer = ZipWriter::new_append( + writer.finish_into_readable()?.into_inner(), + )?; + let options = FileOptions { + compression_method: Stored, + compression_level: Some(17), + last_modified_time: DateTime::from_date_and_time( + 2103, 4, 10, 23, 15, 18, + )?, + permissions: Some(3284386755), + large_file: true, + encrypt_with: Some(ZipCrypto( + ZipCryptoKeys::of( + 0x8888c5bf, 0x88888888, 0xff888888, + ), + PhantomData, + )), + extended_options: ExtendedFileOptions { + extra_data: vec![3, 0, 1, 0, 255, 144, 136, 0, 0] + .into(), + central_extra_data: vec![].into(), + }, + alignment: 65535, + ..Default::default() + }; + writer.add_symlink_from_path("", "\nu", options)?; + writer = ZipWriter::new_append(writer.finish()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append( + writer.finish_into_readable()?.into_inner(), + )?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append(writer.finish()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = + ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.abort_file()?; + let options = FileOptions { + compression_method: CompressionMethod::Unsupported(49603), + compression_level: Some(20), + last_modified_time: DateTime::from_date_and_time( + 2047, 4, 14, 3, 15, 14, + )?, + permissions: Some(3284386755), + large_file: true, + encrypt_with: Some(ZipCrypto( + ZipCryptoKeys::of(0xc3, 0x0, 0x0), + PhantomData, + )), + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 0, + ..Default::default() + }; + writer.add_directory_from_path("", options)?; + writer.deep_copy_file_from_path("/", "")?; + writer.shallow_copy_file_from_path("", "copy")?; + assert!(writer.shallow_copy_file_from_path("", "copy").is_err()); + assert!(writer.shallow_copy_file_from_path("", "copy").is_err()); + assert!(writer.shallow_copy_file_from_path("", "copy").is_err()); + assert!(writer.shallow_copy_file_from_path("", "copy").is_err()); + assert!(writer.shallow_copy_file_from_path("", "copy").is_err()); + assert!(writer.shallow_copy_file_from_path("", "copy").is_err()); + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_17a() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + const PATH_1: &str = "\0I\01\0P\0\0\u{2}\0\0\u{1a}\u{1a}\u{1a}\u{1a}\u{1b}\u{1a}UT\u{5}\0\0\u{1a}\u{1a}\u{1a}\u{1a}UT\u{5}\0\u{1}\0\u{1a}\u{1a}\u{1a}UT\t\0uc\u{5}\0\0\0\0\u{7f}\u{7f}\u{7f}\u{7f}PK\u{6}"; + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1981, 1, 1, 0, 24, 21)?, + permissions: Some(16908288), + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 20555, + ..Default::default() + }; + writer.start_file_from_path( + "\0\u{7}\u{1}\0\0\0\0\0\0\0\0\u{1}\0\0PK\u{1}\u{2};", + options, + )?; + writer.write_all( + &([ + 255, 255, 255, 255, 253, 253, 253, 203, 203, 203, 253, 253, 253, 253, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 249, 191, 225, 225, + 241, 197, + ]), + )?; + writer.write_all( + &([ + 197, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 75, 0, + ]), + )?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 11, 14, 10, 46, 47)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 0, + ..Default::default() + }; + writer.start_file_from_path(PATH_1, options)?; + writer.deep_copy_file_from_path(PATH_1, "eee\u{6}\0\0\0\0\0\0\0\0\0\0\0$\0\0\0\0\0\0\u{7f}\u{7f}PK\u{6}\u{6}K\u{6}\u{6}\u{6}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\u{1}\0\0PK\u{1}\u{1e},\0\0\0\0\0\0\0\0\0\0\0\u{8}\0*\0\0\u{1}PK\u{6}\u{7}PK\u{6}\u{6}\0\0\0\0\0\0\0\0}K\u{2}\u{6}")?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.deep_copy_file_from_path(PATH_1, "")?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.shallow_copy_file_from_path("", "copy")?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + #[allow(clippy::octal_escapes)] + #[cfg(all(feature = "_bzip2_any", not(miri)))] + fn test_fuzz_crash_2024_06_17b() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time( + 1981, 1, 1, 0, 0, 21, + )?, + permissions: Some(16908288), + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 20555, + ..Default::default() + }; + writer.start_file_from_path("\0\u{7}\u{1}\0\0\0\0\0\0\0\0\u{1}\0\0PK\u{1}\u{2};\u{1a}\u{18}\u{1a}UT\t.........................\0u", options)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + let options = FileOptions { + compression_method: CompressionMethod::Bzip2, + compression_level: Some(5), + last_modified_time: DateTime::from_date_and_time( + 2055, 7, 7, 3, 6, 6, + )?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 0, + ..Default::default() + }; + writer.start_file_from_path("\0\0\0\0..\0\0\0\0\0\u{7f}\u{7f}PK\u{6}\u{6}K\u{6}\u{6}\u{6}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\u{1}\0\0PK\u{1}\u{1e},\0\0\0\0\0\0\0\0\0\0\0\u{8}\0*\0\0\u{1}PK\u{6}\u{7}PK\u{6}\u{6}\0\0\0\0\0\0\0\0}K\u{2}\u{6}", options)?; + writer = ZipWriter::new_append( + writer.finish_into_readable()?.into_inner(), + )?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append( + writer.finish_into_readable()?.into_inner(), + )?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = + ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = + ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_18() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_raw_comment(Box::<[u8]>::from([ + 80, 75, 5, 6, 255, 255, 255, 255, 255, 255, 80, 75, 5, 6, 255, 255, 255, 255, 255, 255, + 13, 0, 13, 13, 13, 13, 13, 255, 255, 255, 255, 255, 255, 255, 255, + ])); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + writer.set_raw_comment(Box::new([])); + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append(writer.finish()?)?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_18a() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + writer.set_raw_comment(Box::<[u8]>::from([])); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FullFileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2107, 4, 8, 14, 0, 19)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![ + 182, 180, 1, 0, 180, 182, 74, 0, 0, 200, 0, 0, 0, 2, 0, 0, 0, + ] + .into(), + central_extra_data: vec![].into(), + }, + alignment: 1542, + ..Default::default() + }; + writer.start_file_from_path("\0\0PK\u{6}\u{6}K\u{6}PK\u{3}\u{4}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\u{1}\u{1}\0PK\u{1}\u{2},\0\0\0\0\0\0\0\0\0\0\0P\u{7}\u{4}/.\0KP\0\0;\0\0\0\u{1e}\0\0\0\0\0\0\0\0\0\0\0\0\0", options)?; + let finished = writer.finish_into_readable()?; + assert_eq!(1, finished.file_names().count()); + writer = ZipWriter::new_append(finished.into_inner())?; + let options = FullFileOptions { + compression_method: Stored, + compression_level: Some(5), + last_modified_time: DateTime::from_date_and_time(2107, 4, 1, 0, 0, 0)?, + permissions: None, + large_file: false, + encrypt_with: Some(ZipCrypto( + ZipCryptoKeys::of(0x0, 0x62e4b50, 0x100), + PhantomData, + )), + ..Default::default() + }; + writer.add_symlink_from_path( + "\0K\u{6}\0PK\u{6}\u{7}PK\u{6}\u{6}\0\0\0\0\0\0\0\0PK\u{2}\u{6}", + "\u{8}\0\0\0\0/\0", + options, + )?; + let finished = writer.finish_into_readable()?; + assert_eq!(2, finished.file_names().count()); + writer = ZipWriter::new_append(finished.into_inner())?; + assert_eq!(2, writer.files.len()); + writer + }; + let finished = sub_writer.finish_into_readable()?; + assert_eq!(2, finished.file_names().count()); + writer.merge_archive(finished)?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[cfg(all(feature = "_bzip2_any", feature = "aes-crypto", not(miri)))] + #[test] + fn test_fuzz_crash_2024_06_18b() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(true); + writer.set_raw_comment([0].into()); + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + assert_eq!(writer.get_raw_comment()[0], 0); + let options = FileOptions { + compression_method: CompressionMethod::Bzip2, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2009, 6, 3, 13, 37, 39)?, + permissions: Some(2644352413), + large_file: true, + encrypt_with: Some(crate::write::EncryptWith::Aes { + mode: crate::AesMode::Aes256, + password: "", + }), + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 255, + ..Default::default() + }; + writer.add_symlink_from_path("", "", options)?; + writer.deep_copy_file_from_path("", "PK\u{5}\u{6}\0\0\0\0\0\0\0\0\0\0\0\0\0\u{4}\0\0\0")?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + assert_eq!(writer.get_raw_comment()[0], 0); + writer.deep_copy_file_from_path( + "PK\u{5}\u{6}\0\0\0\0\0\0\0\0\0\0\0\0\0\u{4}\0\0\0", + "\u{2}yy\u{5}qu\0", + )?; + let finished = writer.finish()?; + let archive = ZipArchive::new(finished.clone())?; + assert_eq!(archive.comment(), [0]); + writer = ZipWriter::new_append(finished)?; + assert_eq!(writer.get_raw_comment()[0], 0); + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_19() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 3, 1, 19, 55, 58)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 256, + ..Default::default() + }; + writer.start_file_from_path( + "\0\0\0PK\u{5}\u{6}\0\0\0\0\u{1}\0\u{12}\u{6}\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\0", + options, + )?; + writer.set_flush_on_finish_file(false); + writer.shallow_copy_file_from_path( + "\0\0\0PK\u{5}\u{6}\0\0\0\0\u{1}\0\u{12}\u{6}\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\0", + "", + )?; + writer.set_flush_on_finish_file(false); + writer.deep_copy_file_from_path("", "copy")?; + writer.abort_file()?; + writer.set_flush_on_finish_file(false); + writer.set_raw_comment([255, 0].into()); + writer.abort_file()?; + assert_eq!(writer.get_raw_comment(), [255, 0]); + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + assert_eq!(writer.get_raw_comment(), [255, 0]); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::default(), + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + ..Default::default() + }; + writer.start_file_from_path("", options)?; + assert_eq!(writer.get_raw_comment(), [255, 0]); + let archive = writer.finish_into_readable()?; + assert_eq!(archive.comment(), [255, 0]); + Ok(()) + } + + #[test] + fn fuzz_crash_2024_06_21() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FullFileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 2, 1, 0, 0, 0)?, + permissions: None, + large_file: false, + encrypt_with: None, + ..Default::default() + }; + const LONG_PATH: &str = "\0@PK\u{6}\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0@/\0\0\00ΝPK\u{5}\u{6}O\0\u{10}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0@PK\u{6}\u{7}\u{6}\0/@\0\0\0\0\0\0\0\0 \0\0"; + writer.start_file_from_path(LONG_PATH, options)?; + writer = ZipWriter::new_append(writer.finish()?)?; + writer.deep_copy_file_from_path(LONG_PATH, "oo\0\0\0")?; + writer.abort_file()?; + writer.set_raw_comment([33].into()); + let archive = writer.finish_into_readable()?; + writer = ZipWriter::new_append(archive.into_inner())?; + assert!(writer.get_raw_comment().starts_with(&[33])); + let archive = writer.finish_into_readable()?; + assert!(archive.comment().starts_with(&[33])); + Ok(()) + } + + #[test] + #[cfg(all(feature = "_bzip2_any", not(miri)))] + fn fuzz_crash_2024_07_17() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: CompressionMethod::Bzip2, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2095, 2, 16, 21, 0, 1)?, + permissions: Some(84238341), + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![117, 99, 6, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 2, 0, 0, 0].into(), + central_extra_data: vec![].into(), + }, + alignment: 65535, + ..Default::default() + }; + writer.start_file_from_path("", options)?; + //writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.deep_copy_file_from_path("", "copy")?; + let _ = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + Ok(()) + } + + #[test] + fn fuzz_crash_2024_07_19() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 6, 1, 0, 34, 47)?, + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 45232, + ..Default::default() + }; + writer.add_directory_from_path("", options)?; + writer.deep_copy_file_from_path("/", "")?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.deep_copy_file_from_path("", "copy")?; + let _ = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + Ok(()) + } + + #[test] + #[cfg(feature = "aes-crypto")] + fn fuzz_crash_2024_07_19a() -> ZipResult<()> { + use crate::write::EncryptWith::Aes; + use crate::AesMode::Aes128; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2107, 6, 5, 13, 0, 21)?, + permissions: None, + large_file: true, + encrypt_with: Some(Aes { + mode: Aes128, + password: "", + }), + extended_options: ExtendedFileOptions { + extra_data: vec![3, 0, 4, 0, 209, 53, 53, 8, 2, 61, 0, 0].into(), + central_extra_data: vec![].into(), + }, + alignment: 65535, + ..Default::default() + }; + writer.start_file_from_path("", options)?; + let _ = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + Ok(()) + } + + #[test] + fn fuzz_crash_2024_07_20() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(true); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2041, 8, 2, 19, 38, 0)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 0, + ..Default::default() + }; + writer.add_directory_from_path("\0\0\0\0\0\0\07黻", options)?; + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::default(), + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 4, + ..Default::default() + }; + writer.add_directory_from_path("\0\0\0黻", options)?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.abort_file()?; + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 1, 1, 0, 7, 0)?, + permissions: Some(2663103419), + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 32256, + ..Default::default() + }; + writer.add_directory_from_path("\0", options)?; + writer = ZipWriter::new_append(writer.finish()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + let _ = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + Ok(()) + } + + #[test] + fn fuzz_crash_2024_07_21() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.add_directory_from_path( + "", + FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2105, 8, 1, 15, 0, 0)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 0, + ..Default::default() + }, + )?; + writer.abort_file()?; + let mut writer = ZipWriter::new_append(writer.finish()?)?; + writer.add_directory_from_path( + "", + FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::default(), + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 16, + ..Default::default() + }, + )?; + ZipWriter::new_append(writer.finish()?)? + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + let writer = ZipWriter::new_append(writer.finish()?)?; + let _ = writer.finish_into_readable()?; + + Ok(()) + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/zipcrypto.rs b/v2/vendor/security/zip-7.2.0/src/zipcrypto.rs new file mode 100644 index 0000000000..ee37b74b85 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/zipcrypto.rs @@ -0,0 +1,280 @@ +//! Implementation of the ZipCrypto algorithm +//! +//! The following paper was used to implement the ZipCrypto algorithm: +//! [https://courses.cs.ut.ee/MTAT.07.022/2015_fall/uploads/Main/dmitri-report-f15-16.pdf](https://courses.cs.ut.ee/MTAT.07.022/2015_fall/uploads/Main/dmitri-report-f15-16.pdf) + +use core::fmt::{Debug, Formatter}; +use core::hash::Hash; +use core::marker::PhantomData; +use core::num::Wrapping; + +use crate::cfg_if_expr; +use crate::result::ZipError; +#[cfg(feature = "aes-crypto")] +use crate::AesMode; + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(crate) enum EncryptWith<'k> { + #[cfg(feature = "aes-crypto")] + Aes { + mode: AesMode, + password: &'k str, + }, + ZipCrypto(ZipCryptoKeys, PhantomData<&'k ()>), +} + +#[cfg(feature = "_arbitrary")] +impl<'a> arbitrary::Arbitrary<'a> for EncryptWith<'a> { + fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { + #[cfg(feature = "aes-crypto")] + if bool::arbitrary(u)? { + return Ok(EncryptWith::Aes { + mode: AesMode::arbitrary(u)?, + password: u.arbitrary::<&str>()?, + }); + } + + Ok(EncryptWith::ZipCrypto( + ZipCryptoKeys::arbitrary(u)?, + PhantomData, + )) + } +} + +/// A container to hold the current key state +#[cfg_attr(feature = "_arbitrary", derive(arbitrary::Arbitrary))] +#[derive(Clone, Copy, Hash, Ord, PartialOrd, Eq, PartialEq)] +pub(crate) struct ZipCryptoKeys { + key_0: Wrapping, + key_1: Wrapping, + key_2: Wrapping, +} + +impl Debug for ZipCryptoKeys { + #[allow(unreachable_code)] + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + cfg_if_expr! { + #[cfg(any(test, fuzzing))] => { + f.write_fmt(format_args!( + "ZipCryptoKeys::of({:#10x},{:#10x},{:#10x})", + self.key_0, self.key_1, self.key_2 + )) + }, + _ => { + use std::collections::hash_map::DefaultHasher; + use std::hash::Hasher; + let mut t = DefaultHasher::new(); + self.hash(&mut t); + f.write_fmt(format_args!("ZipCryptoKeys(hash {})", t.finish())) + } + } + } +} + +impl ZipCryptoKeys { + const fn new() -> ZipCryptoKeys { + ZipCryptoKeys { + key_0: Wrapping(0x12345678), + key_1: Wrapping(0x23456789), + key_2: Wrapping(0x34567890), + } + } + + #[allow(unused)] + pub const fn of(key_0: u32, key_1: u32, key_2: u32) -> ZipCryptoKeys { + ZipCryptoKeys { + key_0: Wrapping(key_0), + key_1: Wrapping(key_1), + key_2: Wrapping(key_2), + } + } + + fn update(&mut self, input: u8) { + self.key_0 = ZipCryptoKeys::crc32(self.key_0, input); + self.key_1 = + (self.key_1 + (self.key_0 & Wrapping(0xff))) * Wrapping(0x08088405) + Wrapping(1); + self.key_2 = ZipCryptoKeys::crc32(self.key_2, (self.key_1 >> 24).0 as u8); + } + + fn stream_byte(&mut self) -> u8 { + let temp: Wrapping = Wrapping(self.key_2.0 as u16) | Wrapping(3); + ((temp * (temp ^ Wrapping(1))) >> 8).0 as u8 + } + + fn decrypt_byte(&mut self, cipher_byte: u8) -> u8 { + let plain_byte: u8 = self.stream_byte() ^ cipher_byte; + self.update(plain_byte); + plain_byte + } + + #[allow(dead_code)] + fn encrypt_byte(&mut self, plain_byte: u8) -> u8 { + let cipher_byte: u8 = self.stream_byte() ^ plain_byte; + self.update(plain_byte); + cipher_byte + } + + fn crc32(crc: Wrapping, input: u8) -> Wrapping { + (crc >> 8) ^ Wrapping(CRCTABLE[((crc & Wrapping(0xff)).0 as u8 ^ input) as usize]) + } + pub(crate) fn derive(password: &[u8]) -> ZipCryptoKeys { + let mut keys = ZipCryptoKeys::new(); + for byte in password.iter() { + keys.update(*byte); + } + keys + } +} + +/// A ZipCrypto reader with unverified password +pub struct ZipCryptoReader { + file: R, + keys: ZipCryptoKeys, +} + +pub enum ZipCryptoValidator { + PkzipCrc32(u32), + InfoZipMsdosTime(u16), +} + +impl ZipCryptoReader { + /// Note: The password is `&[u8]` and not `&str` because the + /// [zip specification](https://pkware.cachefly.net/webdocs/APPNOTE/APPNOTE-6.3.3.TXT) + /// does not specify password encoding (see function `update_keys` in the specification). + /// Therefore, if `&str` was used, the password would be UTF-8 and it + /// would be impossible to decrypt files that were encrypted with a + /// password byte sequence that is unrepresentable in UTF-8. + pub fn new(file: R, password: &[u8]) -> ZipCryptoReader { + ZipCryptoReader { + file, + keys: ZipCryptoKeys::derive(password), + } + } + + /// Read the ZipCrypto header bytes and validate the password. + pub fn validate( + mut self, + validator: ZipCryptoValidator, + ) -> Result, ZipError> { + // ZipCrypto prefixes a file with a 12 byte header + let mut header_buf = [0u8; 12]; + self.file.read_exact(&mut header_buf)?; + for byte in header_buf.iter_mut() { + *byte = self.keys.decrypt_byte(*byte); + } + + match validator { + ZipCryptoValidator::PkzipCrc32(crc32_plaintext) => { + // PKZIP before 2.0 used 2 byte CRC check. + // PKZIP 2.0+ used 1 byte CRC check. It's more secure. + // We also use 1 byte CRC. + + if (crc32_plaintext >> 24) as u8 != header_buf[11] { + return Err(ZipError::InvalidPassword); + } + } + ZipCryptoValidator::InfoZipMsdosTime(last_mod_time) => { + // Info-ZIP modification to ZipCrypto format: + // If bit 3 of the general purpose bit flag is set + // (indicates that the file uses a data-descriptor section), + // it uses high byte of 16-bit File Time. + // Info-ZIP code probably writes 2 bytes of File Time. + // We check only 1 byte. + + if (last_mod_time >> 8) as u8 != header_buf[11] { + return Err(ZipError::InvalidPassword); + } + } + } + + Ok(ZipCryptoReaderValid { reader: self }) + } +} +#[allow(unused)] +pub(crate) struct ZipCryptoWriter { + pub(crate) writer: W, + pub(crate) keys: ZipCryptoKeys, +} +impl ZipCryptoWriter { + #[allow(unused)] + pub(crate) fn finish(mut self) -> std::io::Result { + Ok(self.writer) + } +} +impl std::io::Write for ZipCryptoWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + const CHUNK_SIZE: usize = 4096; + let mut temp_buf = [0u8; CHUNK_SIZE]; + for chunk in buf.chunks(CHUNK_SIZE) { + let encrypted_chunk = &mut temp_buf[..chunk.len()]; + for (i, &byte) in chunk.iter().enumerate() { + encrypted_chunk[i] = self.keys.encrypt_byte(byte); + } + self.writer.write_all(encrypted_chunk)?; + } + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + self.writer.flush() + } +} + +/// A ZipCrypto reader with verified password +pub struct ZipCryptoReaderValid { + reader: ZipCryptoReader, +} + +impl std::io::Read for ZipCryptoReaderValid { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + // Note: There might be potential for optimization. Inspiration can be found at: + // https://github.com/kornelski/7z/blob/master/CPP/7zip/Crypto/ZipCrypto.cpp + + let n = self.reader.file.read(buf)?; + for byte in buf.iter_mut().take(n) { + *byte = self.reader.keys.decrypt_byte(*byte); + } + Ok(n) + } +} + +impl ZipCryptoReaderValid { + /// Consumes this decoder, returning the underlying reader. + pub fn into_inner(self) -> R { + self.reader.file + } +} + +static CRCTABLE: [u32; 256] = [ + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, + 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, + 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, + 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, + 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, + 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, + 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, + 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, + 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, + 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, + 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, + 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, + 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, + 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, + 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, + 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, + 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, + 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d, +]; diff --git a/v2/vendor/security/zip-7.2.0/tests/ruview_0429_backport.rs b/v2/vendor/security/zip-7.2.0/tests/ruview_0429_backport.rs new file mode 100644 index 0000000000..747b45dc77 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/tests/ruview_0429_backport.rs @@ -0,0 +1,143 @@ +/// A symlink entry whose central directory declares an absurd uncompressed size +/// must be rejected, not pre-allocated. +/// +/// The declared size comes from the archive itself. Before the fix, `extract` +/// passed it straight to `Vec::with_capacity`, so a hostile entry could ask for +/// 2^62 bytes and abort the whole process ("memory allocation of ... failed"). +/// A symlink target is a path, so a size this large is malformed and the entry +/// must come back as an error. +#[cfg(not(miri))] +#[test] +fn extract_symlink_with_hostile_declared_size() -> zip::result::ZipResult<()> { + use std::io::Cursor; + use tempfile::TempDir; + use zip::ZipArchive; + + /// One entry, flagged as a symlink, whose Zip64 extra field claims + /// `declared_size` uncompressed bytes while storing only `payload`. + fn build(declared_size: u64, payload: &[u8]) -> Vec { + const S_IFLNK: u32 = 0o120_000; + const SENTINEL: u32 = 0xFFFF_FFFF; + let name = b"link"; + let mut o = Vec::new(); + + // local file header + o.extend_from_slice(&[0x50, 0x4b, 0x03, 0x04]); + o.extend_from_slice(&45u16.to_le_bytes()); // version needed: zip64 + o.extend_from_slice(&0u16.to_le_bytes()); // flags + o.extend_from_slice(&0u16.to_le_bytes()); // method: Stored + o.extend_from_slice(&0u16.to_le_bytes()); // mod time + o.extend_from_slice(&0x21u16.to_le_bytes()); // mod date + o.extend_from_slice(&0u32.to_le_bytes()); // crc32 + o.extend_from_slice(&SENTINEL.to_le_bytes()); + o.extend_from_slice(&SENTINEL.to_le_bytes()); + o.extend_from_slice(&(name.len() as u16).to_le_bytes()); + o.extend_from_slice(&20u16.to_le_bytes()); // zip64 extra field + o.extend_from_slice(name); + o.extend_from_slice(&0x0001u16.to_le_bytes()); // zip64 header id + o.extend_from_slice(&16u16.to_le_bytes()); + o.extend_from_slice(&declared_size.to_le_bytes()); + o.extend_from_slice(&(payload.len() as u64).to_le_bytes()); + o.extend_from_slice(payload); + + let cd_offset = o.len() as u32; + + // central directory header + o.extend_from_slice(&[0x50, 0x4b, 0x01, 0x02]); + o.extend_from_slice(&0x031Eu16.to_le_bytes()); // version made by: Unix + o.extend_from_slice(&45u16.to_le_bytes()); + o.extend_from_slice(&0u16.to_le_bytes()); + o.extend_from_slice(&0u16.to_le_bytes()); + o.extend_from_slice(&0u16.to_le_bytes()); + o.extend_from_slice(&0x21u16.to_le_bytes()); + o.extend_from_slice(&0u32.to_le_bytes()); // crc32 + o.extend_from_slice(&SENTINEL.to_le_bytes()); + o.extend_from_slice(&SENTINEL.to_le_bytes()); + o.extend_from_slice(&(name.len() as u16).to_le_bytes()); + o.extend_from_slice(&20u16.to_le_bytes()); + o.extend_from_slice(&0u16.to_le_bytes()); // comment len + o.extend_from_slice(&0u16.to_le_bytes()); // disk number start + o.extend_from_slice(&0u16.to_le_bytes()); // internal attrs + o.extend_from_slice(&((S_IFLNK | 0o777) << 16).to_le_bytes()); // symlink + o.extend_from_slice(&0u32.to_le_bytes()); // local header offset + o.extend_from_slice(name); + o.extend_from_slice(&0x0001u16.to_le_bytes()); + o.extend_from_slice(&16u16.to_le_bytes()); + o.extend_from_slice(&declared_size.to_le_bytes()); + o.extend_from_slice(&(payload.len() as u64).to_le_bytes()); + + let cd_size = o.len() as u32 - cd_offset; + + // end of central directory + o.extend_from_slice(&[0x50, 0x4b, 0x05, 0x06]); + o.extend_from_slice(&0u16.to_le_bytes()); + o.extend_from_slice(&0u16.to_le_bytes()); + o.extend_from_slice(&1u16.to_le_bytes()); + o.extend_from_slice(&1u16.to_le_bytes()); + o.extend_from_slice(&cd_size.to_le_bytes()); + o.extend_from_slice(&cd_offset.to_le_bytes()); + o.extend_from_slice(&0u16.to_le_bytes()); + o + } + + for declared_size in [4097, 1u64 << 40, 1u64 << 62, u64::MAX] { + let mut reader = ZipArchive::new(Cursor::new(build(declared_size, b"/etc/passwd")))?; + assert!(reader.by_index(0)?.is_symlink()); + let tempdir = TempDir::with_prefix("test_hostile_symlink_size")?; + + let err = reader + .extract(&tempdir) + .expect_err("oversized symlink target must be rejected"); + assert!( + matches!(err, zip::result::ZipError::InvalidArchive(_)), + "expected InvalidArchive for a {declared_size} byte symlink target, got {err:?}" + ); + } + + Ok(()) +} + + +#[test] +fn stream_does_not_expose_the_symlink_bit_backport() -> zip::result::ZipResult<()> { + use std::io::Cursor; + use zip::{ZipWriter, write::SimpleFileOptions}; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.add_symlink("link", "/target", SimpleFileOptions::default())?; + let mut cur = writer.finish()?; + cur.set_position(0); + let file = zip::read::read_zipfile_from_stream(&mut cur)?.unwrap(); + assert!(!file.is_symlink()); + Ok(()) +} + +#[test] +fn valid_regular_file_roundtrip() -> zip::result::ZipResult<()> { + use std::io::{Cursor, Write}; + use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions}; + let mut w = ZipWriter::new(Cursor::new(Vec::new())); + w.start_file("regular.txt", SimpleFileOptions::default())?; + w.write_all(b"ordinary archive payload")?; + let mut bytes = w.finish()?; bytes.set_position(0); + let dir = tempfile::TempDir::new()?; + ZipArchive::new(bytes)?.extract(dir.path())?; + assert_eq!(std::fs::read(dir.path().join("regular.txt"))?, b"ordinary archive payload"); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn valid_relative_symlink_roundtrip() -> zip::result::ZipResult<()> { + use std::io::{Cursor, Write}; + use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions}; + let mut w = ZipWriter::new(Cursor::new(Vec::new())); + w.start_file("target", SimpleFileOptions::default())?; + w.write_all(b"target payload")?; + w.add_symlink("link", "target", SimpleFileOptions::default())?; + let mut bytes = w.finish()?; bytes.set_position(0); + let dir = tempfile::TempDir::new()?; + ZipArchive::new(bytes)?.extract(dir.path())?; + assert_eq!(std::fs::read_link(dir.path().join("link"))?, std::path::PathBuf::from("target")); + assert_eq!(std::fs::read(dir.path().join("link"))?, b"target payload"); + Ok(()) +} From 42c3f2d434848e7a7bd10494a33ca028426a69d0 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Tue, 3 Nov 2026 15:11:36 +0100 Subject: [PATCH 048/100] docs(calibration): define qualified learner selection and Room Save contract --- ...R-420-managed-source-aware-room-learner.md | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/docs/adr/ADR-420-managed-source-aware-room-learner.md b/docs/adr/ADR-420-managed-source-aware-room-learner.md index 32b6d7ff31..460533a3a2 100644 --- a/docs/adr/ADR-420-managed-source-aware-room-learner.md +++ b/docs/adr/ADR-420-managed-source-aware-room-learner.md @@ -123,6 +123,73 @@ Corrupt evaluation evidence must be retained/quarantined without discarding a valid candidate or manufacturing a pass. Evaluation receipts authorize neither physical qualification nor adoption. +### Proposed November 3 extension: local qualification, explicit selection and Room Save + +The current app's legacy Room Save requires a background-model receipt and its +empty check. A separately trained learner cannot satisfy that contract merely +by completing capture or evaluation. Do not manufacture a legacy receipt, or +make the user repeat both workflows without explaining the missing seam. + +Introduce a distinct, versioned **local operational qualification**. The server +computes it from the immutable candidate, exact retained Train lineage, +validated role-disjoint HeldOut data, managed sealed-phase history, and the +existing immutable evaluation artifact. The policy is fixed before evaluation; +clients cannot submit scores, thresholds or a claim that qualification passed. +Reuse the targets proposed in issue #2418: at least four Train and four HeldOut +cohorts for each of Empty, OccupiedStatic, OccupiedMoving and AdjacentOnly; +unconditional sensitivity at least 90% for each occupied condition; false +positive rate at most 5% for each negative condition; evaluated cohort coverage +at least 90% in every condition. Missing results, abstentions and unavailable +rates cannot be counted as correct. At the minimum cohort count these gates +require all four observations in every condition to be correctly scored. + +These are operator-labelled representative-cohort checks for one installation. +They do not measure wall-time coverage, independent trial statistics, p95 +decision latency, coherent source clocks or general physical accuracy. Preserve +the original diagnostic receipts' false `physically_validated`, +`independence_verified` and `adoption_authorized` fields. The separate receipt +names its narrower authority, policy hash, exact model/evaluation/data/context +hashes, label provenance, original observation bounds and evaluation time. +Older trusted managed sealed history may establish that the existing +conditions-held gate was passed; it cannot acquire invented historical +confirmation times or full-duration coverage. + +Qualification never selects a model automatically. Explicit guarded selection +binds the verified owner, installation, room revision, ordered source/PHY +context, immutable model and qualification hashes. Candidate discovery, +training, evaluation, a new latest-model pointer, or consent completion cannot +switch the selected model. Keep previous selection history for explicit +rollback. A newer completed check that fails for the selected model withdraws +current authority without rewriting its historical passing receipt or silently +falling back to another detector. + +Persist model and selection independently of optional capture recovery. The +24-hour reduced-sample recovery deadline does not expire a trained model or +renew its evidence. After restart/logout the original selection and evidence +times remain historical. Current use additionally requires the actual published +room binding, exact artifact/policy verification, complete fresh source windows, +and the model's existing out-of-distribution checks. Missing sources, changed +room/PHY context, corruption, failed checks or unknown model state produce an +explicit unavailable/abstaining result. They must not be rendered as an empty +room or trigger an unlabelled heuristic fallback. + +Publish the selected result through a versioned binary-only evidence surface. +It grants no person count, movement class, location, pose, numeric Vitals, +HomeCore or cloud authority. Keep legacy output visibly attributable as a +comparison where compatibility requires it; selected-model clients exclusively +use the selected proof and its withdrawal state. Withhold downstream projections +whose contracts cannot preserve this binary evidence boundary. + +The app stores a discriminated learner installation receipt and its own saved +timestamp/history. It does not reuse the legacy baseline's receipt or timestamp. +Room Save validates a fresh exact selection proof around durable persistence, +and reports saved progress separately from qualified/selected/current state. +Attaching a model must not republish geometry: the existing companion publish +operation increments its revision and would invalidate the candidate's binding. +Paired clients receive only the bounded read-scoped proof; qualification and +selection mutations remain with the managed owner. Any final native delivery +still requires composed tests and actual device qualification. + ### Proposed November 2 extension: opt-in provisional capture recovery The operator can explicitly choose encrypted local recovery when configuring a From c8220171c8be4c2906423766a0190f288ec74869 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Tue, 3 Nov 2026 15:10:48 +0100 Subject: [PATCH 049/100] feat(learner): qualify and explicitly select durable room models --- .../src/room_learner/artifact.rs | 6 + .../src/room_learner/assessment.rs | 122 +++++ .../src/room_learner/assessment_store.rs | 104 ++++ .../src/room_learner/capture.rs | 2 + .../src/room_learner/evaluation_store.rs | 12 +- .../src/room_learner/lifecycle.rs | 35 +- .../lifecycle_assessment_tests.rs | 242 +++++++++ .../src/room_learner/lifecycle_capture.rs | 12 + .../src/room_learner/lifecycle_evaluation.rs | 2 +- .../src/room_learner/lifecycle_selection.rs | 489 ++++++++++++++++++ .../room_learner/lifecycle_selection_tests.rs | 399 ++++++++++++++ .../src/room_learner/mod.rs | 5 + .../src/room_learner/qualification.rs | 362 +++++++++++++ .../src/room_learner/selection.rs | 106 ++++ .../src/room_learner/selection_store.rs | 191 +++++++ 15 files changed, 2086 insertions(+), 3 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/assessment.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/assessment_store.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_assessment_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/selection.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/selection_store.rs diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs index 6743e852f6..a842f88b64 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs @@ -297,3 +297,9 @@ pub use catalog::{CandidateCatalog,SavedCandidate}; #[path="evaluation_store.rs"] mod evaluation_store; pub use evaluation_store::EvaluationReceipt; + +#[path="selection_store.rs"] +mod selection_store; + +#[path="assessment_store.rs"] +mod assessment_store; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/assessment.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/assessment.rs new file mode 100644 index 0000000000..348fb4bfc5 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/assessment.rs @@ -0,0 +1,122 @@ +//! Latest completed operator evaluation withdraws stale positive authority without erasing history. +use super::{ + artifact::EvaluationReceipt, + model::RoomModel, + observation::{CaptureProvenance, DataRole, MAX_COHORTS}, + qualification::{criteria, hash, ManagedEvidence, QualificationEnvelope, QualificationPolicy}, +}; +use serde::{Deserialize, Serialize}; +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct AssessmentReceipt { + pub schema: String, + pub owner_scope: String, + pub installation_id: String, + pub context_digest: String, + pub model_sha256: String, + pub evaluation_receipt_sha256: String, + pub policy_sha256: String, + pub evaluated_at_unix_ms: u64, + pub last_held_out_observed_ns: u64, + pub train_cohorts_per_condition: [usize; 4], + pub sealed_history_sha256: String, + pub qualification_sha256: Option, + pub failure_reason: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct AssessmentEnvelope { + pub receipt_sha256: String, + pub receipt: AssessmentReceipt, +} +impl AssessmentEnvelope { + pub(crate) fn issue( + model: &RoomModel, + evaluation: &EvaluationReceipt, + evidence: &ManagedEvidence, + qualified: Option<&QualificationEnvelope>, + ) -> Result { + let outcome = criteria(&evaluation.report, &evidence.counts); + match (&outcome, qualified) { + (Ok(_), None) | (Err(_), Some(_)) => { + return Err("qualification outcome mismatch".into()) + } + _ => {} + } + let c = model.context(); + let receipt = AssessmentReceipt { + schema: "ruview.local-operational-assessment.v1".into(), + owner_scope: c.owner_scope.clone(), + installation_id: c.installation_id.clone(), + context_digest: c.digest(), + model_sha256: evaluation.model_sha256.clone(), + evaluation_receipt_sha256: evaluation.sha256()?, + policy_sha256: hash(&QualificationPolicy::default())?, + evaluated_at_unix_ms: evaluation.evaluated_at_unix_ms, + last_held_out_observed_ns: evidence.last_held_out_observed_ns, + train_cohorts_per_condition: evidence.counts, + sealed_history_sha256: evidence.sealed_history_sha256.clone(), + qualification_sha256: qualified.map(|q| q.receipt_sha256.clone()), + failure_reason: outcome.err(), + }; + let a = Self { + receipt_sha256: hash(&receipt)?, + receipt, + }; + a.validate(model, evaluation)?; + Ok(a) + } + pub(crate) fn validate( + &self, + model: &RoomModel, + eval: &EvaluationReceipt, + ) -> Result<(), String> { + let r = &self.receipt; + let c = model.context(); + let hex = |s: &str| { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }; + if self.receipt_sha256 != hash(r)? + || r.schema != "ruview.local-operational-assessment.v1" + || r.owner_scope != c.owner_scope + || r.installation_id != c.installation_id + || r.context_digest != c.digest() + || r.model_sha256 != eval.model_sha256 + || r.evaluation_receipt_sha256 != eval.sha256()? + || r.policy_sha256 != hash(&QualificationPolicy::default())? + || r.evaluated_at_unix_ms != eval.evaluated_at_unix_ms + || r.last_held_out_observed_ns == 0 + || !hex(&r.sealed_history_sha256) + || eval.role != DataRole::HeldOut + || eval.provenance != [CaptureProvenance::LiveReceiverObserved] + || model.training_manifest().training_provenance + != [CaptureProvenance::LiveReceiverObserved] + || r.train_cohorts_per_condition + .iter() + .any(|n| *n < 4 || *n > MAX_COHORTS) + || r.train_cohorts_per_condition.iter().sum::() + != model.training_manifest().training_cohorts + || eval.report.per_condition.iter().any(|v| v.samples < 4) + { + return Err("assessment identity/lineage mismatch".into()); + } + match criteria(&eval.report, &r.train_cohorts_per_condition) { + Ok(_) => { + if r.failure_reason.is_some() || !r.qualification_sha256.as_deref().is_some_and(hex) + { + return Err("assessment pass invalid".into()); + } + } + Err(e) => { + if r.failure_reason.as_deref() != Some(e.as_str()) + || r.qualification_sha256.is_some() + { + return Err("assessment failure invalid".into()); + } + } + } + Ok(()) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/assessment_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/assessment_store.rs new file mode 100644 index 0000000000..3768136804 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/assessment_store.rs @@ -0,0 +1,104 @@ +use super::*; +use crate::room_learner::{assessment::AssessmentEnvelope, qualification::QualificationEnvelope}; +impl ArtifactStore { + fn assessment_dir(&self, model: &RoomModel, create: bool) -> Result, String> { + let Some(dir) = self.context_dir(model.context(), create)? else { + return Ok(None); + }; + let path = dir.join("assessments"); + match std::fs::symlink_metadata(&path) { + Err(e) if e.kind() == io::ErrorKind::NotFound && !create => return Ok(None), + Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e.to_string()), + _ => {} + } + private_dir(&path).map_err(|e| e.to_string())?; + Ok(Some(path)) + } + fn verify_assessment(&self, model: &RoomModel, a: &AssessmentEnvelope) -> Result<(), String> { + let e = self.load_exact_evaluation( + model, + crate::room_learner::observation::DataRole::HeldOut, + &a.receipt.evaluation_receipt_sha256, + )?; + a.validate(model, &e)?; + if let Some(sha) = &a.receipt.qualification_sha256 { + let q = self.load_qualification(model, sha)?; + if q.receipt.evaluation_receipt_sha256 != a.receipt.evaluation_receipt_sha256 + || q.receipt.sealed_history_sha256 != a.receipt.sealed_history_sha256 + { + return Err("assessment qualification differs".into()); + } + } + Ok(()) + } + pub(crate) fn load_latest_assessment( + &self, + model: &RoomModel, + ) -> Result, String> { + let Some(dir) = self.assessment_dir(model, false)? else { + return Ok(None); + }; + let path = dir.join(format!("{}.latest.json", digest(&model.export()?))); + if !regular_or_missing(&path)? { + return Ok(None); + } + let a: AssessmentEnvelope = + serde_json::from_slice(&private_read(&path).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + self.verify_assessment(model, &a)?; + let immutable = dir.join(format!("{}.json", a.receipt_sha256)); + if private_read(&immutable).map_err(|e| e.to_string())? != encode(&a)? { + return Err("assessment pointer differs from immutable history".into()); + } + Ok(Some(a)) + } + pub(crate) fn save_assessment( + &self, + model: &RoomModel, + a: &AssessmentEnvelope, + ) -> Result<(), String> { + self.verify_assessment(model, a)?; + if let Some(previous) = self.load_latest_assessment(model)? { + if previous == *a { + return Ok(()); + } + if a.receipt.evaluated_at_unix_ms < previous.receipt.evaluated_at_unix_ms + || a.receipt.last_held_out_observed_ns <= previous.receipt.last_held_out_observed_ns + { + return Err( + "older qualification evidence cannot replace a newer assessment".into(), + ); + } + } + let dir = self + .assessment_dir(model, true)? + .ok_or("assessment directory unavailable")?; + let bytes = encode(a)?; + evaluation_store::write_immutable(&dir.join(format!("{}.json", a.receipt_sha256)), &bytes)?; + atomic_private_write( + &dir.join(format!("{}.latest.json", a.receipt.model_sha256)), + &bytes, + ) + .map_err(|e| e.to_string())?; + if self.load_latest_assessment(model)?.as_ref() != Some(a) { + return Err("assessment durable readback mismatch".into()); + } + Ok(()) + } + pub(crate) fn current_qualification_assessment( + &self, + model: &RoomModel, + q: &QualificationEnvelope, + ) -> Result { + let a = self + .load_latest_assessment(model)? + .ok_or("latest completed qualification assessment unavailable")?; + if a.receipt.failure_reason.is_some() { + return Err("newer_qualification_failed".into()); + } + if a.receipt.qualification_sha256.as_deref() != Some(q.receipt_sha256.as_str()) { + return Err("new qualification requires explicit selection".into()); + } + Ok(a) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs index f7b1770da7..4d8c92206d 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs @@ -135,6 +135,8 @@ impl CaptureSession { trials: HashMap::new(), sessions: HashMap::new() }) } + /// A rejected selected input withdraws old live windows, not sealed training rows. + pub(crate) fn clear_live_windows(&mut self) { for source in self.sources.values_mut() { source.live.clear(); } } pub fn context(&self) -> &CaptureContext { &self.context } pub fn cohorts(&self) -> &[CaptureCohort] { &self.cohorts } pub fn history(&self) -> &[PhaseReceipt] { &self.history } diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_store.rs index 57173b1275..f8cda8e966 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_store.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_store.rs @@ -50,7 +50,7 @@ fn private_subdirectory( private_dir(&path).map_err(|e| e.to_string())?; Ok(Some(path)) } -fn write_immutable(path: &Path, bytes: &[u8]) -> Result<(), String> { +pub(super) fn write_immutable(path: &Path, bytes: &[u8]) -> Result<(), String> { if regular_or_missing(path)? { return if private_read(path).map_err(|e| e.to_string())? == bytes { Ok(()) @@ -263,3 +263,13 @@ impl ArtifactStore { #[cfg(test)] #[path = "evaluation_store_tests.rs"] mod tests; + +impl ArtifactStore { + pub(crate) fn load_exact_evaluation(&self, model: &RoomModel, role: DataRole, key: &str) -> Result { + let directory=self.evaluation_directory(model,role,false)?.ok_or("evaluation unavailable")?; + let receipt=self.read_evaluation(&directory,key,model,role)?; + let original=self.read_evidence_index(&directory,&receipt.dataset_sha256,model,role)?.ok_or("original evaluation index missing")?; + if original.sha256()?!=key {return Err("evaluation differs from original dataset index".into())} + Ok(receipt) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs index 04d93dc4dc..118b4e73c8 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs @@ -55,6 +55,9 @@ struct Inner { evaluation_warning: Option, input_error: Option, recovery: RecoveryState, + selection: selection_control::SelectionState, + qualification: Option, + qualification_error: Option, } pub struct RoomLearningRuntime { runtime_epoch: String, @@ -162,6 +165,9 @@ impl RoomLearningRuntime { evaluation_warning: None, input_error: None, recovery: RecoveryState::default(), + selection: selection_control::SelectionState::default(), + qualification: None, + qualification_error: None, }), }; { @@ -169,6 +175,7 @@ impl RoomLearningRuntime { .inner .lock() .map_err(|_| "learner state unavailable")?; + runtime.restore_selection(&mut state); runtime.inspect_recovery(&mut state, runtime.receiver_now_ns()); } Ok(runtime) @@ -263,6 +270,20 @@ impl RoomLearningRuntime { state.last_evaluation = restored_evaluation; state.evaluation_warning = evaluation_warning; state.input_error = None; + state.qualification = None; + state.qualification_error = None; + if let Some(model)=state.model.clone() { + match self.store.load_latest_qualification(&model) { + Ok(q)=>{ + state.qualification=q; + match self.store.load_latest_assessment(&model) { + Ok(Some(a))=>state.qualification_error=a.receipt.failure_reason, + Ok(None)=>{},Err(e)=>state.qualification_error=Some(bounded(e)), + } + }, + Err(e)=>state.qualification_error=Some(bounded(e)), + } + } self.checkpoint_recovery(&mut state, now_ns, true)?; Ok(()) } @@ -419,7 +440,8 @@ impl RoomLearningRuntime { "capture":state.capture.as_ref().map(|c|c.progress(now_ns)), "training_frozen":state.frozen_training,"training_defaults":FitConfig::for_source_count(state.capture.as_ref().map_or(0, |c| c.context().sources.len())),"training_report":state.job.report, "error":state.job.error,"input_error":state.input_error, - "candidate":state.receipt,"candidate_loaded":state.model.is_some(), + "candidate":state.receipt,"candidate_loaded":state.model.is_some(),"candidate_identity":self.candidate_identity(&state), + "model_selection":self.selection_status_locked(&state,now_ns,None), "last_evaluation":state.last_evaluation,"evaluation_warning":state.evaluation_warning,"physically_validated":false, "control_detector_changed":false,"raw_observations_persisted":state.recovery.derived_persisted(),"derived_samples_persisted":state.recovery.derived_persisted(),"raw_iq_persisted":false, "limitations":["receiver-observed magnitude path; no coherent phase claim", @@ -453,3 +475,14 @@ mod evaluation_control; #[cfg(test)] #[path = "lifecycle_evaluation_tests.rs"] mod evaluation_tests; + +#[path = "lifecycle_selection.rs"] +mod selection_control; + +#[cfg(test)] +#[path="lifecycle_selection_tests.rs"] +mod selection_tests; + +#[cfg(test)] +#[path="lifecycle_assessment_tests.rs"] +mod assessment_tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_assessment_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_assessment_tests.rs new file mode 100644 index 0000000000..a55d808d7d --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_assessment_tests.rs @@ -0,0 +1,242 @@ +//! Synthetic measured-path counterexamples; no hardware qualification claims. +use super::selection_tests::{feed, feed_sequence, fitted, phase_override}; +use super::tests::{context, finished, runtime, START}; +use super::*; +use crate::room_learner::selection::SelectedBinaryEvidence; + +#[tokio::test] +async fn newer_completed_failed_assessment_withdraws_persistent_selection_across_restart() { + let dir = tempfile::tempdir().unwrap(); + let rt = fitted(dir.path()).await; + let q = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + let selected = rt + .select_model( + &rt.mutation_guard().unwrap(), + &q.receipt.model_sha256, + &q.receipt_sha256, + ) + .unwrap(); + let model = rt.inner.lock().unwrap().model.as_ref().unwrap().clone(); + let original_assessment = rt.store.load_latest_assessment(&model).unwrap().unwrap(); + // Record deliberately mislabelled held-out empty evidence through actual commands. + let now = phase_override(&rt, DataRole::HeldOut, RoomLabel::Empty, 8, 7.0); + let error = rt.qualify(&rt.mutation_guard().unwrap()).unwrap_err(); + assert!(error.contains("false presence"), "{error}"); + let snapshot = rt.selection_snapshot(now, Some("synthetic-room")); + assert_eq!( + snapshot.selected.unwrap().selection_sha256, + selected.receipt_sha256 + ); + assert!(snapshot.qualification.is_none()); + assert!( + matches!(snapshot.current,SelectedBinaryEvidence::Unavailable{reason,..} if reason=="newer_qualification_failed") + ); + assert!(rt + .select_model( + &rt.mutation_guard().unwrap(), + &q.receipt.model_sha256, + &q.receipt_sha256 + ) + .is_err()); + assert!( + rt.store + .save_assessment(&model, &original_assessment) + .is_err(), + "older pass must not erase a later completed failure" + ); + let cold = runtime(dir.path()); + let now = feed(&cold, cold.receiver_now_ns(), &[1, 2], 2.0); + let snapshot = cold.selection_snapshot(now, Some("synthetic-room")); + assert_eq!( + snapshot.selected.unwrap().selection_sha256, + selected.receipt_sha256 + ); + assert!( + matches!(snapshot.current,SelectedBinaryEvidence::Unavailable{reason,..} if reason=="newer_qualification_failed") + ); + assert!(cold + .store + .load_qualification( + cold.inner.lock().unwrap().model.as_ref().unwrap(), + &q.receipt_sha256 + ) + .is_ok()); + // Restoring the explicit legacy control is permitted; history/model stay intact. + cold.deselect_model(&cold.mutation_guard().unwrap()) + .unwrap(); + assert!(matches!( + cold.selected_binary_evidence(now, Some("synthetic-room")), + SelectedBinaryEvidence::None + )); +} + +#[tokio::test] +async fn rejected_selected_input_withdraws_old_window_until_all_sources_are_fresh_again() { + let dir = tempfile::tempdir().unwrap(); + let rt = fitted(dir.path()).await; + let q = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + rt.select_model( + &rt.mutation_guard().unwrap(), + &q.receipt.model_sha256, + &q.receipt_sha256, + ) + .unwrap(); + let cold = runtime(dir.path()); + let at = cold.receiver_now_ns(); + let now = feed(&cold, at, &[1, 2], 2.0); + assert!(matches!( + cold.selected_binary_evidence(now, Some("synthetic-room")), + SelectedBinaryEvidence::Ready { .. } + )); + cold.record_native_rejection(99, "ht64-canonical8", "foreign source"); + cold.record_native_rejection(1, "other-grid", "unselected PHY"); + assert!(matches!( + cold.selected_binary_evidence(now, Some("synthetic-room")), + SelectedBinaryEvidence::Ready { .. } + )); + cold.record_native_rejection(1, "ht64-canonical8", "invalid native frame"); + assert!(matches!( + cold.selected_binary_evidence(now, Some("synthetic-room")), + SelectedBinaryEvidence::Unavailable { .. } + )); + let partial = feed_sequence(&cold, at + 200_000_000, &[1], 2.0, 20_000); + assert!(matches!( + cold.selected_binary_evidence(partial, Some("synthetic-room")), + SelectedBinaryEvidence::Unavailable { .. } + )); + let fresh = feed_sequence(&cold, at + 400_000_000, &[1, 2], 2.0, 30_000); + assert!(matches!( + cold.selected_binary_evidence(fresh, Some("synthetic-room")), + SelectedBinaryEvidence::Ready { .. } + )); + assert!(cold + .ingest( + RawSourceFrame { + source_id: 1, + grid_id: "ht64-canonical8", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: 30_007, + amplitudes: &[2.3; 8] + }, + fresh + 1 + ) + .is_err()); + assert!(matches!( + cold.selected_binary_evidence(fresh + 1, Some("synthetic-room")), + SelectedBinaryEvidence::Unavailable { .. } + )); +} + +#[tokio::test] +async fn fresh_selection_snapshot_rechecks_durable_artifacts_not_only_cached_model() { + let dir = tempfile::tempdir().unwrap(); + let rt = fitted(dir.path()).await; + let q = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + rt.select_model( + &rt.mutation_guard().unwrap(), + &q.receipt.model_sha256, + &q.receipt_sha256, + ) + .unwrap(); + let now = START + 103_000_000_000; + assert!(matches!( + rt.selection_snapshot(now, Some("synthetic-room")).current, + SelectedBinaryEvidence::Ready { .. } + )); + let path = dir + .path() + .join("a".repeat(64)) + .join("SensingServer/room-learner") + .join(context().digest()) + .join(format!("{}.json", q.receipt.model_sha256)); + let root = dir + .path() + .join("a".repeat(64)) + .join("SensingServer/room-learner"); + let qualification = root + .join(context().digest()) + .join("qualifications") + .join(format!("{}.json", q.receipt_sha256)); + let pointer = root.join("model-selection-v1.json"); + let assessment = root + .join(context().digest()) + .join("assessments") + .join(format!("{}.latest.json", q.receipt.model_sha256)); + for damaged in [path, qualification, pointer, assessment] { + let original = std::fs::read(&damaged).unwrap(); + std::fs::write(&damaged, b"corrupt durable evidence").unwrap(); + assert!( + matches!( + rt.selection_snapshot(now, Some("synthetic-room")).current, + SelectedBinaryEvidence::Unavailable { .. } + ), + "{}", + damaged.display() + ); + assert!(rt + .selection_snapshot(now, Some("synthetic-room")) + .integrity_error + .is_some()); + std::fs::write(&damaged, original).unwrap(); + assert!(matches!( + rt.selection_snapshot(now, Some("synthetic-room")).current, + SelectedBinaryEvidence::Ready { .. } + )); + } +} + +#[tokio::test] +async fn canonical_56_feature_context_qualifies_through_actual_managed_training_commands() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + let mut c = context(); + for s in &mut c.sources { + s.feature_count = 56; + s.grid_id = "ht64-canonical56".into(); + } + rt.configure( + &rt.mutation_guard().unwrap(), + c, + "canonical-width-fixture".into(), + true, + ) + .unwrap(); + let cases = [ + (RoomLabel::Empty, 2.0), + (RoomLabel::OccupiedStatic, 6.0), + (RoomLabel::OccupiedMoving, 7.0), + (RoomLabel::AdjacentOnly, 2.3), + ]; + for (i, (label, gain)) in cases.into_iter().enumerate() { + phase_override(&rt, DataRole::Train, label, i as u32, gain); + } + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 8, + learning_rate: 0.5, + hidden_dim: 2, + embedding_dim: 2, + temporal_tokens: 2, + ..FitConfig::default() + }, + ) + .unwrap(); + let training = finished(&rt).await; + assert_eq!(training["state"], "ready", "{training}"); + for (i, (label, gain)) in cases.into_iter().enumerate() { + phase_override(&rt, DataRole::HeldOut, label, i as u32 + 4, gain); + } + let q = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + rt.select_model( + &rt.mutation_guard().unwrap(), + &q.receipt.model_sha256, + &q.receipt_sha256, + ) + .unwrap(); + assert!(matches!( + rt.selected_binary_evidence(START + 103_000_000_000, Some("synthetic-room")), + SelectedBinaryEvidence::Ready { .. } + )); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs index 6896815fff..4844f20868 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs @@ -80,7 +80,10 @@ impl RoomLearningRuntime { }; let result = capture.ingest(frame, now_ns); if let Err(error) = &result { + capture.clear_live_windows(); state.input_error = Some(bounded(error.clone())); + } else if capture.live_cohort(now_ns).is_ok() { + state.input_error = None; } let saved = self.checkpoint_recovery(&mut state, now_ns, false); result.and(saved) @@ -95,6 +98,7 @@ impl RoomLearningRuntime { || packet.native_frame.provenance.class != ruview_unified::frame::ProvenanceClass::Measured { + self.record_native_rejection(packet.source_id,&packet.grid_id,"synthetic/replayed native provenance"); return Err("live managed capture rejects synthetic/replayed native provenance".into()); } let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; @@ -110,6 +114,10 @@ impl RoomLearningRuntime { if packet.native_frame.provenance.device_id != source.device_id || packet.native_frame.modality != source.signal_kind { + if packet.grid_id==source.grid_id && packet.frequency_hz==source.center_frequency_hz && packet.bandwidth_hz==source.bandwidth_hz { + capture.clear_live_windows(); + state.input_error=Some("native RF provenance does not match the selected source".into()); + } return Err("native RF provenance does not match the selected source".into()); } // Another PHY class from the same receiver is not this selected source @@ -131,7 +139,10 @@ impl RoomLearningRuntime { now_ns, ); if let Err(error) = &result { + capture.clear_live_windows(); state.input_error = Some(bounded(error.clone())); + } else if capture.live_cohort(now_ns).is_ok() { + state.input_error = None; } let saved = self.checkpoint_recovery(&mut state, now_ns, false); result.and(saved) @@ -146,6 +157,7 @@ impl RoomLearningRuntime { .iter() .any(|s| s.source_id == source_id && s.grid_id == grid_id) }) { + if let Some(capture)=state.capture.as_mut() {capture.clear_live_windows();} state.input_error = Some(bounded(format!( "Node {source_id} native input rejected: {reason}" ))); diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_evaluation.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_evaluation.rs index 266152cff6..f2d4e6b1b0 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_evaluation.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_evaluation.rs @@ -1,7 +1,7 @@ //! Durable diagnostics only. Historical lineage checks are not current identity. use super::*; use crate::room_learner::artifact::EvaluationReceipt; -fn presentation( +pub(super) fn presentation( receipt: &EvaluationReceipt, restored: bool, current_train_checked: bool, diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection.rs new file mode 100644 index 0000000000..88e52da081 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection.rs @@ -0,0 +1,489 @@ +//! Local operator qualification and explicit persisted selection; no automatic promotion. +use super::*; +use crate::room_learner::{ + qualification::{hash, QualificationEnvelope}, + selection::*, +}; +use sha2::{Digest, Sha256}; + +#[derive(Default)] +pub(super) struct SelectionState { + envelope: Option, + model: Option>, + qualification: Option, + error: Option, + assessment: Option, +} +impl RoomLearningRuntime { + pub(super) fn restore_selection(&self, state: &mut Inner) { + let restored = (|| -> Result<(), String> { + let Some(envelope) = self.store.load_selection()? else { + return Ok(()); + }; + state.selection.envelope = Some(envelope.clone()); + if let Some(target) = &envelope.receipt.selected { + let (model, q) = self.store.resolve_selection_target(target)?; + let model = Arc::new(model); + state.capture = Some(CaptureSession::new( + target.context.clone(), + format!("selected-{}", envelope.receipt.revision), + )?); + state.model = Some(model.clone()); + state.frozen_training = true; + state.job.phase = JobPhase::Ready; + state.qualification = self.store.load_latest_qualification(&model)?; + state.selection.assessment = self.store.load_latest_assessment(&model)?; + state.selection.model = Some(model); + state.selection.qualification = Some(q); + } + Ok(()) + })(); + if let Err(e) = restored { + state.selection.error = Some(bounded(e)); + } + } + pub fn qualify(&self, expected: &MutationGuard) -> Result { + let mut state = self.command_state(expected)?; + let result = (|| -> Result { + if busy(&state.job) || !state.frozen_training { + return Err("wait for the frozen candidate".into()); + } + let model = state.model.as_ref().ok_or("no candidate")?.clone(); + let capture = state.capture.as_ref().ok_or("no capture context")?; + if capture.phase_active() { + return Err("finish the current capture phase first".into()); + } + let _policy = crate::room_learner::qualification::QualificationPolicy::default(); + let dataset = capture.validated_dataset()?; + let evaluation = self + .store + .evaluate_candidate(&model, &dataset, DataRole::HeldOut)?; + // Incomplete data or missing lineage is not a completed failed assessment. + let evidence = crate::room_learner::qualification::managed_evidence( + &model, + &dataset, + capture.history(), + &evaluation, + )?; + let outcome = + QualificationEnvelope::issue(&model, &dataset, capture.history(), &evaluation); + let assessment = crate::room_learner::assessment::AssessmentEnvelope::issue( + &model, + &evaluation, + &evidence, + outcome.as_ref().ok(), + )?; + state.last_evaluation = Some(super::evaluation_control::presentation( + &evaluation, + false, + true, + )?); + state.evaluation_warning = None; + if let Ok(q) = &outcome { + self.store.save_qualification(&model, q)?; + } + if let Err(e) = self.store.save_assessment(&model, &assessment) { + if state + .selection + .envelope + .as_ref() + .and_then(|e| e.receipt.selected.as_ref()) + .is_some_and(|t| t.model_sha256 == evaluation.model_sha256) + { + state.selection.error=Some("Latest qualification persistence could not be verified; restart and inspect retained evidence.".into()); + } + return Err(e); + } + if state + .selection + .envelope + .as_ref() + .and_then(|e| e.receipt.selected.as_ref()) + .is_some_and(|t| t.model_sha256 == evaluation.model_sha256) + { + state.selection.assessment = Some(assessment); + } + outcome + })(); + match result { + Ok(q) => { + state.qualification = Some(q.clone()); + state.qualification_error = None; + Ok(q) + } + Err(e) => { + state.qualification = None; + state.qualification_error = Some(bounded(e.clone())); + Err(e) + } + } + } + fn publish_selection( + &self, + state: &mut Inner, + target: Option, + previous: Option, + ) -> Result { + let resolved = target + .as_ref() + .map(|t| self.store.resolve_selection_target(t)) + .transpose()?; + let assessment = resolved + .as_ref() + .map(|(model, q)| self.store.current_qualification_assessment(model, q)) + .transpose()?; + let old_revision = state + .selection + .envelope + .as_ref() + .map_or(0, |s| s.receipt.revision); + let receipt = SelectionReceipt { + schema: "ruview.room-learner.model-selection.v1".into(), + owner_scope: self.owner.clone(), + installation_id: self.installation.clone(), + revision: old_revision + .checked_add(1) + .ok_or("selection revision exhausted")?, + selected_at_unix_ms: self.receiver_now_ns() / 1_000_000, + selected: target, + previous, + }; + let envelope = SelectionEnvelope { + receipt_sha256: hash(&receipt)?, + receipt, + }; + if let Err(error) = self.store.save_selection(&envelope, old_revision) { + state.selection.error=Some("Selection persistence outcome is unavailable; restart and verify the saved selection before continuing.".into()); + return Err(error); + } + // Only publish in-memory authority after durable pointer and readback succeed. + state.selection = SelectionState { + envelope: Some(envelope.clone()), + model: resolved.as_ref().map(|(m, _)| Arc::new(m.clone())), + qualification: resolved.map(|(_, q)| q), + error: None, + assessment, + }; + Ok(envelope) + } + pub fn select_model( + &self, + expected: &MutationGuard, + model_sha256: &str, + qualification_sha256: &str, + ) -> Result { + let mut state = self.command_state(expected)?; + if busy(&state.job) || state.capture.as_ref().is_some_and(|c| c.phase_active()) { + return Err("finish active capture/training before selecting".into()); + } + let model = state + .model + .as_ref() + .ok_or("no current candidate to select")?; + let sha = format!("{:x}", Sha256::digest(model.export()?)); + if sha != model_sha256 { + return Err("candidate changed before selection".into()); + } + let qualification = self.store.load_qualification(model, qualification_sha256)?; + self.store + .current_qualification_assessment(model, &qualification)?; + let target = SelectionTarget { + context: model.context().clone(), + model_sha256: sha, + model_version: model.version, + qualification_sha256: qualification_sha256.into(), + }; + let previous = state + .selection + .envelope + .as_ref() + .and_then(|e| e.receipt.selected.clone()); + if previous.as_ref() == Some(&target) { + if state.selection.error.is_some() + || self.store.load_selection()?.as_ref() != state.selection.envelope.as_ref() + { + state.selection.error = Some( + "Saved selection differs from runtime; restart before selecting again.".into(), + ); + return Err("saved selection changed".into()); + } + return Ok(state + .selection + .envelope + .as_ref() + .ok_or("selection unavailable")? + .clone()); + } + self.publish_selection(&mut state, Some(target), previous) + } + pub fn deselect_model(&self, expected: &MutationGuard) -> Result { + let mut state = self.command_state(expected)?; + if busy(&state.job) || state.capture.as_ref().is_some_and(|c| c.phase_active()) { + return Err("finish active capture/training before changing selection".into()); + } + if let Some(e) = &state.selection.envelope { + if e.receipt.selected.is_none() && state.selection.error.is_none() { + if self.store.load_selection()?.as_ref() != Some(e) { + return Err("saved selection changed".into()); + } + return Ok(e.clone()); + } + } + let previous = state + .selection + .envelope + .as_ref() + .and_then(|e| e.receipt.selected.clone()); + self.publish_selection(&mut state, None, previous) + } + pub fn rollback_model(&self, expected: &MutationGuard) -> Result { + let mut state = self.command_state(expected)?; + if busy(&state.job) || state.capture.as_ref().is_some_and(|c| c.phase_active()) { + return Err("finish active capture/training before rollback".into()); + } + let previous = state + .selection + .envelope + .as_ref() + .and_then(|e| e.receipt.previous.clone()) + .ok_or("no prior selection")?; + let context = state + .capture + .as_ref() + .map(|c| c.context()) + .ok_or("no current context")?; + if *context != previous.context { + return Err("rollback context does not match current installation".into()); + } + let current = state + .selection + .envelope + .as_ref() + .and_then(|e| e.receipt.selected.clone()); + self.publish_selection(&mut state, Some(previous), current) + } + fn selected_identity(state: &Inner) -> Option { + let e = state.selection.envelope.as_ref()?; + let t = e.receipt.selected.as_ref()?; + let q = state.selection.qualification.as_ref()?; + Some(SelectedModelIdentity { + selection_sha256: e.receipt_sha256.clone(), + selection_revision: e.receipt.revision, + model_sha256: t.model_sha256.clone(), + model_version: t.model_version, + qualification_sha256: t.qualification_sha256.clone(), + policy_sha256: q.receipt.policy_sha256.clone(), + context_digest: t.context.digest(), + room_digest: t.context.room_digest.clone(), + source_node_ids: t.context.sources.iter().map(|s| s.source_id).collect(), + selected_at_unix_ms: e.receipt.selected_at_unix_ms, + evaluated_at_unix_ms: q.receipt.evaluated_at_unix_ms, + authority_scope: "local_operator_binary_presence".into(), + }) + } + fn selected_evidence_locked( + &self, + state: &Inner, + now_ns: u64, + expected_room_binding: Option<&str>, + ) -> SelectedBinaryEvidence { + let identity = Self::selected_identity(state); + let unavailable = |reason: String| SelectedBinaryEvidence::Unavailable { + identity: identity.clone(), + reason: bounded(reason), + }; + if let Some(e) = &state.selection.error { + return unavailable(e.clone()); + } + let Some(target) = state + .selection + .envelope + .as_ref() + .and_then(|e| e.receipt.selected.as_ref()) + else { + return SelectedBinaryEvidence::None; + }; + let result = (|| -> Result { + if expected_room_binding != Some(target.context.room_digest.as_str()) { + return Err("selected model does not match the current published room".into()); + } + let identity = identity + .clone() + .ok_or("selected qualification unavailable")?; + let model = state + .selection + .model + .as_ref() + .ok_or("selected model unavailable")?; + let q = state + .selection + .qualification + .as_ref() + .ok_or("selected qualification unavailable")?; + if q.receipt.policy + != crate::room_learner::qualification::QualificationPolicy::default() + { + return Err("selected policy is not current".into()); + } + let assessment = state + .selection + .assessment + .as_ref() + .ok_or("latest completed qualification assessment unavailable")?; + if assessment.receipt.failure_reason.is_some() { + return Err("newer_qualification_failed".into()); + } + if assessment.receipt.qualification_sha256.as_deref() != Some(q.receipt_sha256.as_str()) + { + return Err("new qualification requires explicit selection".into()); + } + if let Some(error) = &state.input_error { + return Err(format!("selected input is unverified: {error}")); + } + let capture = state + .capture + .as_ref() + .ok_or("awaiting current selected sources")?; + if capture.context() != &target.context { + return Err("selected context differs from current RF source contract".into()); + } + let cohort = capture.live_cohort(now_ns)?; + let prediction = model.infer(&cohort); + if prediction.decision == crate::room_learner::model::PresenceDecision::Abstain { + return Err(prediction + .reason + .unwrap_or_else(|| "selected model abstained".into())); + } + let probability = prediction + .occupied_probability + .filter(|x| x.is_finite() && (0.0..=1.0).contains(x)) + .ok_or("selected probability unavailable")?; + let observed = cohort + .views + .iter() + .map(|v| v.last_time_ns) + .min() + .ok_or("no selected source window")? + / 1_000_000; + // Source identities, not poll time/live-cohort id/age, define repeated observations. + let windows: Vec<_> = cohort + .views + .iter() + .map(|v| { + ( + v.source_id, + &v.frame_ids, + &v.sequence_ids, + v.first_time_ns, + v.last_time_ns, + ) + }) + .collect(); + let observation_digest = hash(&(target.context.digest(), windows))?; + Ok(SelectedBinaryEvidence::Ready { + identity, + presence: prediction.decision + == crate::room_learner::model::PresenceDecision::Occupied, + decision: prediction.decision, + occupied_probability: probability, + observed_at_unix_ms: observed, + observation_digest, + }) + })(); + result.unwrap_or_else(unavailable) + } + pub fn selected_binary_evidence( + &self, + now_ns: u64, + expected_room_binding: Option<&str>, + ) -> SelectedBinaryEvidence { + match self.inner.lock() { + Ok(state) => self.selected_evidence_locked(&state, now_ns, expected_room_binding), + Err(_) => SelectedBinaryEvidence::Unavailable { + identity: None, + reason: "learner state unavailable".into(), + }, + } + } + pub(super) fn selection_status_locked( + &self, + state: &Inner, + now_ns: u64, + binding: Option<&str>, + ) -> ModelSelectionStatus { + ModelSelectionStatus { + schema: "ruview.room-learner.model-selection-status.v1", + revision: state + .selection + .envelope + .as_ref() + .map_or(0, |e| e.receipt.revision), + selected: Self::selected_identity(state), + previous_available: state + .selection + .envelope + .as_ref() + .is_some_and(|e| e.receipt.previous.is_some()), + qualification: state.qualification.as_ref().map(QualificationSummary::from), + qualification_error: state.qualification_error.clone(), + integrity_error: state.selection.error.clone(), + current: self.selected_evidence_locked(state, now_ns, binding), + } + } + pub fn selection_snapshot( + &self, + now_ns: u64, + expected_room_binding: Option<&str>, + ) -> ModelSelectionStatus { + match self.inner.lock() { + Ok(state) => { + let mut snapshot = + self.selection_status_locked(&state, now_ns, expected_room_binding); + let verified = (|| -> Result<(), String> { + if self.store.load_selection()?.as_ref() != state.selection.envelope.as_ref() { + return Err("saved selection differs from current runtime".into()); + } + if let Some(target) = state + .selection + .envelope + .as_ref() + .and_then(|e| e.receipt.selected.as_ref()) + { + let (model, q) = self.store.resolve_selection_target(target)?; + self.store.current_qualification_assessment(&model, &q)?; + } + Ok(()) + })(); + if let Err(e) = verified { + snapshot.integrity_error = Some(bounded(e.clone())); + snapshot.current = SelectedBinaryEvidence::Unavailable { + identity: snapshot.selected.clone(), + reason: bounded(e), + }; + } + snapshot + } + Err(_) => ModelSelectionStatus { + schema: "ruview.room-learner.model-selection-status.v1", + revision: 0, + selected: None, + previous_available: false, + qualification: None, + qualification_error: None, + integrity_error: Some("learner state unavailable".into()), + current: SelectedBinaryEvidence::Unavailable { + identity: None, + reason: "learner state unavailable".into(), + }, + }, + } + } + pub(super) fn candidate_identity(&self, state: &Inner) -> Option { + let model = state.model.as_ref()?; + let sha = format!("{:x}", Sha256::digest(model.export().ok()?)); + Some( + json!({"model_sha256":sha,"model_version":model.version,"context_digest":model.context().digest(), + "room_digest":model.context().room_digest,"persisted":self.store.load_exact_candidate(model.context(),&sha).is_ok()}), + ) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection_tests.rs new file mode 100644 index 0000000000..ff4b0e6f0b --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection_tests.rs @@ -0,0 +1,399 @@ +//! SYNTHETIC managed packets exercise the actual qualification/selection lifecycle. +//! No hardware, physical accuracy or operator-ground-truth validation is claimed. +use super::tests::{context, finished, phase, runtime, train_phases, START}; +use super::*; +use crate::room_learner::{ + qualification::QualificationEnvelope, selection::SelectedBinaryEvidence, +}; + +pub(super) async fn fitted(root: &Path) -> Arc { + let rt = runtime(root); + rt.configure( + &rt.mutation_guard().unwrap(), + context(), + "selection-fixture".into(), + true, + ) + .unwrap(); + train_phases(&rt); + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 8, + learning_rate: 0.5, + hidden_dim: 2, + embedding_dim: 2, + temporal_tokens: 2, + ..FitConfig::default() + }, + ) + .unwrap(); + let status = finished(&rt).await; + assert_eq!(status["state"], "ready", "{status}"); + for (i, label) in [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ] + .into_iter() + .enumerate() + { + phase(&rt, DataRole::HeldOut, label, i as u32 + 4); + } + rt +} +pub(super) fn feed(rt: &RoomLearningRuntime, at: u64, sources: &[u8], gain: f64) -> u64 { + feed_sequence(rt, at, sources, gain, 10_000) +} +pub(super) fn feed_sequence( + rt: &RoomLearningRuntime, + at: u64, + sources: &[u8], + gain: f64, + sequence_base: u32, +) -> u64 { + for i in 0..8u32 { + for source in sources { + let values: Vec = (0..8) + .map(|bin| { + gain + f64::from(*source) * 0.3 + bin as f64 * 0.04 + f64::from(i) * 0.003 + }) + .collect(); + rt.ingest( + RawSourceFrame { + source_id: *source, + grid_id: "ht64-canonical8", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: sequence_base + i, + amplitudes: &values, + }, + at + u64::from(i) * 10_000_000 + u64::from(*source), + ) + .unwrap(); + } + } + at + 80_000_000 +} +fn ready(value: SelectedBinaryEvidence) -> crate::room_learner::selection::SelectedModelIdentity { + match value { + SelectedBinaryEvidence::Ready { identity, .. } => identity, + other => panic!("expected ready: {other:?}"), + } +} +#[tokio::test] +async fn qualification_selection_cold_restore_deselect_and_exact_rollback_compose() { + let dir = tempfile::tempdir().unwrap(); + let rt = fitted(dir.path()).await; + assert!(matches!( + rt.selected_binary_evidence(START + 104_000_000_000, Some("synthetic-room")), + SelectedBinaryEvidence::None + )); + let q = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + assert_eq!(q.receipt.label_provenance, "managed-sealed-history-v1"); + assert!(!q.receipt.physically_validated && !q.receipt.independence_verified); + assert_eq!(q.receipt.criteria.len(), 4); + let evaltime = q.receipt.evaluated_at_unix_ms; + let oldguard = rt.mutation_guard().unwrap(); + let selected = rt + .select_model(&oldguard, &q.receipt.model_sha256, &q.receipt_sha256) + .unwrap(); + assert!(rt.deselect_model(&oldguard).is_err()); + let first = ready(rt.selected_binary_evidence(START + 103_000_000_000, Some("synthetic-room"))); + assert_eq!(first.selection_sha256, selected.receipt_sha256); + assert!(matches!( + rt.selected_binary_evidence(START + 103_000_000_000, Some("wrong-room")), + SelectedBinaryEvidence::Unavailable { .. } + )); + assert!(matches!( + rt.selected_binary_evidence(START + 200_000_000_000, Some("synthetic-room")), + SelectedBinaryEvidence::Unavailable { .. } + )); + + let cold = runtime(dir.path()); + let snapshot = cold.selection_snapshot(cold.receiver_now_ns(), Some("synthetic-room")); + assert_eq!( + snapshot.selected.unwrap().selection_sha256, + selected.receipt_sha256 + ); + assert_eq!( + snapshot.qualification.unwrap().evaluated_at_unix_ms, + evaltime + ); + assert!(matches!( + snapshot.current, + SelectedBinaryEvidence::Unavailable { .. } + )); + // Stored selection survives cold load, but historical capture buffers do not. + let at = cold.receiver_now_ns(); + let now = feed(&cold, at, &[1, 2], 2.0); + let a = cold.selected_binary_evidence(now, Some("synthetic-room")); + let b = cold.selected_binary_evidence(now + 10_000_000, Some("synthetic-room")); + match (&a, &b) { + ( + SelectedBinaryEvidence::Ready { + observation_digest: x, + observed_at_unix_ms: t, + .. + }, + SelectedBinaryEvidence::Ready { + observation_digest: y, + observed_at_unix_ms: u, + .. + }, + ) => { + assert_eq!(x, y); + assert_eq!(t, u); + assert!(*t < now / 1_000_000) + } + _ => panic!("fresh repeated proof unavailable: {a:?}, {b:?}"), + } + assert_eq!(ready(a).model_sha256, q.receipt.model_sha256); + let deselected = cold + .deselect_model(&cold.mutation_guard().unwrap()) + .unwrap(); + assert!(matches!( + cold.selected_binary_evidence(now, Some("synthetic-room")), + SelectedBinaryEvidence::None + )); + let repeated = cold + .deselect_model(&cold.mutation_guard().unwrap()) + .unwrap(); + assert_eq!(repeated, deselected); + let rolled = cold + .rollback_model(&cold.mutation_guard().unwrap()) + .unwrap(); + assert_eq!( + rolled.receipt.selected.unwrap().model_sha256, + q.receipt.model_sha256 + ); + assert_eq!( + ready(cold.selected_binary_evidence(now, Some("synthetic-room"))).evaluated_at_unix_ms, + evaltime + ); + // Selection/model are not coupled to the 24-hour provisional-capture TTL. + let tomorrow = feed_sequence( + &cold, + now + 25 * 60 * 60 * 1_000_000_000, + &[1, 2], + 2.0, + 20_000, + ); + assert_eq!( + ready(cold.selected_binary_evidence(tomorrow, Some("synthetic-room"))).evaluated_at_unix_ms, + evaltime + ); + // Starting another candidate must never automatically replace the selected model. + cold.configure( + &cold.mutation_guard().unwrap(), + context(), + "new-candidate".into(), + true, + ) + .unwrap(); + assert_eq!( + cold.selection_snapshot(now, Some("synthetic-room")) + .selected + .unwrap() + .model_sha256, + q.receipt.model_sha256 + ); + assert!(cold.status(now)["candidate_identity"].is_null()); + assert!(matches!( + cold.selected_binary_evidence(now, Some("synthetic-room")), + SelectedBinaryEvidence::Unavailable { .. } + )); + let fresh = feed(&cold, tomorrow + 1_000_000_000, &[1, 2], 2.0); + assert!(cold.qualify(&cold.mutation_guard().unwrap()).is_err()); + assert!( + matches!( + cold.selected_binary_evidence(fresh, Some("synthetic-room")), + SelectedBinaryEvidence::Ready { .. } + ), + "incomplete candidate must not revoke the existing selected model" + ); +} + +#[tokio::test] +async fn qualification_requires_real_lineage_all_conditions_and_matching_sealed_history() { + let dir = tempfile::tempdir().unwrap(); + let rt = fitted(dir.path()).await; + let state = rt.inner.lock().unwrap(); + let model = state.model.as_ref().unwrap(); + let capture = state.capture.as_ref().unwrap(); + let dataset = capture.validated_dataset().unwrap(); + let evaluation = rt + .store + .evaluate_candidate(model, &dataset, DataRole::HeldOut) + .unwrap(); + let good = + QualificationEnvelope::issue(model, &dataset, capture.history(), &evaluation).unwrap(); + // Actual inference reports from deliberately wrong/abstaining model heads, + // not hand-edited success metrics, must fail the per-condition policy. + for (bias, reason) in [ + (-100.0, "occupied sensitivity"), + (100.0, "false presence"), + (0.0, "coverage"), + ] { + let mut wrong = (**model).clone(); + wrong.head_weights.fill(0.0); + *wrong.head_weights.last_mut().unwrap() = bias; + let report = wrong.evaluate(dataset.held_out()).unwrap(); + let error = crate::room_learner::qualification::criteria(&report, &[4; 4]).unwrap_err(); + assert!(error.contains(reason), "{error}"); + } + let incomplete: Vec<_> = dataset + .held_out() + .iter() + .filter(|c| c.label != RoomLabel::AdjacentOnly) + .cloned() + .collect(); + let report = model.evaluate(&incomplete).unwrap(); + assert!( + crate::room_learner::qualification::criteria(&report, &[4; 4]) + .unwrap_err() + .contains("AdjacentOnly") + ); + let mut missing = capture.history().to_vec(); + missing.retain(|h| h.label != RoomLabel::AdjacentOnly); + assert!(QualificationEnvelope::issue(model, &dataset, &missing, &evaluation).is_err()); + let no_train = crate::room_learner::observation::ValidatedDataset::new( + context(), + dataset.held_out().to_vec(), + ) + .unwrap(); + assert!( + QualificationEnvelope::issue(model, &no_train, capture.history(), &evaluation).is_err() + ); + let mut tampered = good.clone(); + tampered.receipt.policy.minimum_evaluated_cohort_coverage = 0.0; + tampered.receipt.policy_sha256 = + crate::room_learner::qualification::hash(&tampered.receipt.policy).unwrap(); + tampered.receipt_sha256 = crate::room_learner::qualification::hash(&tampered.receipt).unwrap(); + assert!(tampered.validate(model, &evaluation).is_err()); + let mut overflow = good.clone(); + overflow.receipt.criteria[0].train_cohorts = usize::MAX; + overflow.receipt_sha256 = crate::room_learner::qualification::hash(&overflow.receipt).unwrap(); + assert!(overflow.validate(model, &evaluation).is_err()); + let mut other = good; + other.receipt.model_sha256 = "b".repeat(64); + other.receipt_sha256 = crate::room_learner::qualification::hash(&other.receipt).unwrap(); + assert!(other.validate(model, &evaluation).is_err()); +} + +#[tokio::test] +async fn selection_corruption_owner_mismatch_missing_sources_and_ood_never_fall_back() { + let dir = tempfile::tempdir().unwrap(); + let rt = fitted(dir.path()).await; + let q = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + rt.select_model( + &rt.mutation_guard().unwrap(), + &q.receipt.model_sha256, + &q.receipt_sha256, + ) + .unwrap(); + let cold = runtime(dir.path()); + let at = cold.receiver_now_ns(); + let now = feed(&cold, at, &[1], 2.0); + assert!(matches!( + cold.selected_binary_evidence(now, Some("synthetic-room")), + SelectedBinaryEvidence::Unavailable { .. } + )); + let ood = runtime(dir.path()); + let now = feed(&ood, ood.receiver_now_ns(), &[1, 2], 1_000_000.0); + assert!(matches!( + ood.selected_binary_evidence(now, Some("synthetic-room")), + SelectedBinaryEvidence::Unavailable { .. } + )); + let path = dir + .path() + .join("a".repeat(64)) + .join("SensingServer/room-learner/model-selection-v1.json"); + let original = std::fs::read(&path).unwrap(); + let foreign_dir = dir.path().join("b".repeat(64)).join("SensingServer"); + let foreign = + RoomLearningRuntime::new(&foreign_dir, &"b".repeat(64), "synthetic-install").unwrap(); + drop(foreign); + std::fs::write( + foreign_dir.join("room-learner/model-selection-v1.json"), + &original, + ) + .unwrap(); + let foreign = + RoomLearningRuntime::new(&foreign_dir, &"b".repeat(64), "synthetic-install").unwrap(); + assert!(matches!( + foreign.selected_binary_evidence(now, Some("synthetic-room")), + SelectedBinaryEvidence::Unavailable { .. } + )); + std::fs::write(&path, b"broken").unwrap(); + let broken = runtime(dir.path()); + assert!(matches!( + broken.selected_binary_evidence(now, Some("synthetic-room")), + SelectedBinaryEvidence::Unavailable { .. } + )); + assert_eq!(std::fs::read(&path).unwrap(), b"broken"); + assert!(rt + .store + .load_exact_candidate(&context(), &q.receipt.model_sha256) + .is_ok()); +} + +/// Genuine managed capture path with synthetic, deliberately selectable signal gain. +pub(super) fn phase_override( + rt: &RoomLearningRuntime, + role: DataRole, + label: RoomLabel, + index: u32, + gain: f64, +) -> u64 { + let at = START + u64::from(index) * 13_000_000_000; + rt.begin_phase( + &rt.mutation_guard().unwrap(), + PhaseRequest { + trial_id: format!("selection-trial-{index}"), + session_id: format!("selection-session-{index}"), + role, + label, + duration_ms: 12_000, + lead_in_ms: 0, + conditions_confirmed: true, + }, + at, + ) + .unwrap(); + let context = rt.context().unwrap(); + for quarter in 0..4u32 { + for sample in 0..8u32 { + for source in &context.sources { + let values: Vec = (0..source.feature_count) + .map(|bin| { + gain + f64::from(source.source_id) * 0.3 + + bin as f64 * 0.04 + + f64::from(sample) * 0.003 + }) + .collect(); + let t = at + + u64::from(quarter) * 3_000_000_000 + + 1_000_000 + + u64::from(sample) * 10_000_000 + + u64::from(source.source_id); + rt.ingest( + RawSourceFrame { + source_id: source.source_id, + grid_id: &source.grid_id, + frequency_hz: source.center_frequency_hz, + bandwidth_hz: source.bandwidth_hz, + sequence: index * 32 + quarter * 8 + sample + 1, + amplitudes: &values, + }, + t, + ) + .unwrap(); + } + } + } + rt.finish_phase(&rt.mutation_guard().unwrap(), at + 12_000_000_000, true) + .unwrap(); + at + 12_000_000_000 +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs index 757f4fc665..3d1ef496e1 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs @@ -7,3 +7,8 @@ pub mod artifact; pub mod lifecycle; pub mod native; pub mod recovery_store; + +pub mod qualification; +pub mod selection; + +pub mod assessment; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification.rs new file mode 100644 index 0000000000..3840c10ec6 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification.rs @@ -0,0 +1,362 @@ +//! Narrow operator-labelled local qualification; never statistical independence. +use super::{ + capture::{PhaseReceipt, PhaseStatus}, + model::RoomModel, + observation::{CaptureProvenance, DataRole, RoomLabel, ValidatedDataset}, + training::EvaluationReport, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +pub const POLICY_ID: &str = "ruview.local-operational-policy.v1"; +pub(crate) fn hash(value: &T) -> Result { + Ok(format!( + "{:x}", + Sha256::digest(serde_json::to_vec(value).map_err(|e| e.to_string())?) + )) +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct QualificationPolicy { + pub id: String, + pub minimum_cohorts_per_condition: usize, + pub minimum_unconditional_sensitivity: f32, + pub maximum_false_positive_rate: f32, + pub minimum_evaluated_cohort_coverage: f32, + pub coverage_scope: String, +} +impl Default for QualificationPolicy { + fn default() -> Self { + Self { + id: POLICY_ID.into(), + minimum_cohorts_per_condition: 4, + minimum_unconditional_sensitivity: 0.9, + maximum_false_positive_rate: 0.05, + minimum_evaluated_cohort_coverage: 0.9, + coverage_scope: "representative_cohorts_not_wall_time".into(), + } + } +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct ConditionCriterion { + pub condition: RoomLabel, + pub train_cohorts: usize, + pub held_out_cohorts: usize, + pub evaluated_cohort_coverage: f32, + pub unconditional_sensitivity: Option, + pub false_positive_rate: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct QualificationReceipt { + pub schema: String, + pub owner_scope: String, + pub installation_id: String, + pub room_digest: String, + pub context_digest: String, + pub model_sha256: String, + pub model_version: u32, + pub training_data_sha256: String, + pub held_out_dataset_sha256: String, + pub evaluation_receipt_sha256: String, + pub evaluated_at_unix_ms: u64, + pub policy: QualificationPolicy, + pub policy_sha256: String, + pub label_provenance: String, + pub sealed_history_sha256: String, + pub first_observed_at_ns: u64, + pub last_observed_at_ns: u64, + pub criteria: Vec, + pub report: EvaluationReport, + pub independence_verified: bool, + pub physically_validated: bool, + pub authority_scope: String, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct QualificationEnvelope { + pub receipt_sha256: String, + pub receipt: QualificationReceipt, +} + +pub(crate) const CONDITIONS: [RoomLabel; 4] = [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, +]; + +pub(crate) fn criteria( + report: &EvaluationReport, + train_counts: &[usize; 4], +) -> Result, String> { + report.validate()?; + let policy = QualificationPolicy::default(); + CONDITIONS + .iter() + .enumerate() + .map(|(i, label)| { + let c = report + .per_condition + .iter() + .find(|c| c.condition == *label) + .ok_or("missing required held-out condition")?; + if train_counts[i] < policy.minimum_cohorts_per_condition + || c.samples < policy.minimum_cohorts_per_condition + { + return Err(format!( + "qualification requires four Train and HeldOut cohorts for {label:?}" + )); + } + let coverage = c + .rates + .evaluated_cohort_coverage + .ok_or("held-out coverage unavailable")?; + if !coverage.is_finite() || coverage < policy.minimum_evaluated_cohort_coverage { + return Err(format!("held-out coverage failed for {label:?}")); + } + let occupied = matches!(label, RoomLabel::OccupiedStatic | RoomLabel::OccupiedMoving); + let (sensitivity, false_positive) = if occupied { + let s = c + .rates + .unconditional_sensitivity + .ok_or("occupied sensitivity unavailable")?; + if !s.is_finite() || s < policy.minimum_unconditional_sensitivity { + return Err(format!("occupied sensitivity failed for {label:?}")); + } + (Some(s), None) + } else { + let f = c.confusion.false_occupied as f32 / c.samples as f32; + if !f.is_finite() || f > policy.maximum_false_positive_rate { + return Err(format!("false presence failed for {label:?}")); + } + (None, Some(f)) + }; + Ok(ConditionCriterion { + condition: *label, + train_cohorts: train_counts[i], + held_out_cohorts: c.samples, + evaluated_cohort_coverage: coverage, + unconditional_sensitivity: sensitivity, + false_positive_rate: false_positive, + }) + }) + .collect() +} + +#[derive(Debug, Clone)] +pub(crate) struct ManagedEvidence { + pub counts: [usize; 4], + pub sealed_history_sha256: String, + pub first_observed_at_ns: u64, + pub last_observed_at_ns: u64, + pub last_held_out_observed_ns: u64, +} +pub(crate) fn managed_evidence( + model: &RoomModel, + dataset: &ValidatedDataset, + history: &[PhaseReceipt], + evaluation: &super::artifact::EvaluationReceipt, +) -> Result { + if dataset.context() != model.context() + || dataset.train().is_empty() + || hash(&dataset.train())? != model.training_manifest().training_data_sha256 + { + return Err("qualification requires retained exact Train manifest lineage".into()); + } + let rows: Vec<_> = dataset.train().iter().chain(dataset.held_out()).collect(); + if rows.iter().any(|c| { + c.provenance != CaptureProvenance::LiveReceiverObserved + || c.views + .iter() + .any(|v| v.provenance != CaptureProvenance::LiveReceiverObserved) + }) { + return Err( + "qualification requires live receiver-observed Train and HeldOut sources".into(), + ); + } + if model.training_manifest().training_provenance != [CaptureProvenance::LiveReceiverObserved] { + return Err("candidate training provenance is not live receiver-observed".into()); + } + let mut relevant = Vec::new(); + for row in &rows { + let matching: Vec<_> = history + .iter() + .filter(|p| { + p.trial_id == row.trial_id + && p.session_id == row.session_id + && p.role == row.role + && p.label == row.label + && p.status == PhaseStatus::Sealed + }) + .collect(); + if matching.len() != 1 { + return Err("required managed sealed phase history is missing or ambiguous".into()); + } + let phase = matching[0]; + let count = rows + .iter() + .filter(|r| { + r.trial_id == row.trial_id + && r.session_id == row.session_id + && r.role == row.role + && r.label == row.label + }) + .count(); + if phase.cohort_count != count { + return Err("sealed phase history does not match exact cohorts".into()); + } + if !relevant.iter().any(|p: &&PhaseReceipt| { + p.trial_id == phase.trial_id && p.session_id == phase.session_id + }) { + relevant.push(phase); + } + } + let counts = + CONDITIONS.map(|label| dataset.train().iter().filter(|c| c.label == label).count()); + evaluation.report.validate()?; + for (i, label) in CONDITIONS.iter().enumerate() { + if counts[i] < 4 + || dataset + .held_out() + .iter() + .filter(|c| c.label == *label) + .count() + < 4 + { + return Err(format!( + "qualification requires four Train and HeldOut cohorts for {label:?}" + )); + } + } + let c = model.context(); + if evaluation.role != DataRole::HeldOut + || evaluation.context_digest != c.digest() + || evaluation.training_data_sha256 != model.training_manifest().training_data_sha256 + || evaluation.dataset_sha256 != hash(&dataset.held_out())? + { + return Err("qualification evaluation lineage mismatch".into()); + } + let first = rows + .iter() + .flat_map(|c| &c.views) + .map(|v| v.first_time_ns) + .min() + .ok_or("no source time")?; + let last = rows + .iter() + .flat_map(|c| &c.views) + .map(|v| v.last_time_ns) + .max() + .ok_or("no source time")?; + + Ok(ManagedEvidence { + counts, + sealed_history_sha256: hash(&relevant)?, + first_observed_at_ns: first, + last_observed_at_ns: last, + last_held_out_observed_ns: dataset + .held_out() + .iter() + .flat_map(|c| &c.views) + .map(|v| v.last_time_ns) + .max() + .ok_or("no held-out source time")?, + }) +} + +impl QualificationEnvelope { + pub(crate) fn issue( + model: &RoomModel, + dataset: &ValidatedDataset, + history: &[PhaseReceipt], + evaluation: &super::artifact::EvaluationReceipt, + ) -> Result { + let evidence = managed_evidence(model, dataset, history, evaluation)?; + let checks = criteria(&evaluation.report, &evidence.counts)?; + let c = model.context(); + let policy = QualificationPolicy::default(); + let receipt = QualificationReceipt { + schema: "ruview.local-operational-qualification.v1".into(), + owner_scope: c.owner_scope.clone(), + installation_id: c.installation_id.clone(), + room_digest: c.room_digest.clone(), + context_digest: c.digest(), + model_sha256: evaluation.model_sha256.clone(), + model_version: evaluation.model_version, + training_data_sha256: evaluation.training_data_sha256.clone(), + held_out_dataset_sha256: evaluation.dataset_sha256.clone(), + evaluation_receipt_sha256: evaluation.sha256()?, + evaluated_at_unix_ms: evaluation.evaluated_at_unix_ms, + policy_sha256: hash(&policy)?, + policy, + label_provenance: "managed-sealed-history-v1".into(), + sealed_history_sha256: evidence.sealed_history_sha256, + first_observed_at_ns: evidence.first_observed_at_ns, + last_observed_at_ns: evidence.last_observed_at_ns, + criteria: checks, + report: evaluation.report.clone(), + independence_verified: false, + physically_validated: false, + authority_scope: "local_operator_binary_presence".into(), + }; + let envelope = Self { + receipt_sha256: hash(&receipt)?, + receipt, + }; + envelope.validate(model, evaluation)?; + Ok(envelope) + } + pub(crate) fn validate( + &self, + model: &RoomModel, + eval: &super::artifact::EvaluationReceipt, + ) -> Result<(), String> { + let r = &self.receipt; + let c = model.context(); + let hex = |x: &str| { + x.len() == 64 + && x.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) + }; + if r.schema != "ruview.local-operational-qualification.v1" + || self.receipt_sha256 != hash(r)? + || r.owner_scope != c.owner_scope + || r.installation_id != c.installation_id + || r.room_digest != c.room_digest + || r.context_digest != c.digest() + || r.model_sha256 != eval.model_sha256 + || r.model_version != eval.model_version + || r.training_data_sha256 != model.training_manifest().training_data_sha256 + || r.held_out_dataset_sha256 != eval.dataset_sha256 + || r.evaluation_receipt_sha256 != eval.sha256()? + || r.evaluated_at_unix_ms != eval.evaluated_at_unix_ms + || r.report != eval.report + || eval.role != DataRole::HeldOut + || r.policy != QualificationPolicy::default() + || r.policy_sha256 != hash(&r.policy)? + || r.label_provenance != "managed-sealed-history-v1" + || !hex(&r.sealed_history_sha256) + || r.first_observed_at_ns == 0 + || r.last_observed_at_ns < r.first_observed_at_ns + || r.independence_verified + || r.physically_validated + || r.authority_scope != "local_operator_binary_presence" + || model.training_manifest().training_provenance + != [CaptureProvenance::LiveReceiverObserved] + || eval.provenance != [CaptureProvenance::LiveReceiverObserved] + || r.criteria.len() != 4 + { + return Err("local qualification identity, provenance or policy mismatch".into()); + } + let counts = std::array::from_fn(|i| r.criteria[i].train_cohorts); + if counts.iter().any(|n| *n > super::observation::MAX_COHORTS) + || counts.iter().sum::() != model.training_manifest().training_cohorts + || criteria(&r.report, &counts)? != r.criteria + { + return Err("local qualification criterion mismatch".into()); + } + Ok(()) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection.rs new file mode 100644 index 0000000000..8909302695 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection.rs @@ -0,0 +1,106 @@ +//! Explicit durable selection is distinct from candidate discovery and current usability. +use super::{ + model::PresenceDecision, observation::CaptureContext, qualification::QualificationEnvelope, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct SelectionTarget { + pub context: CaptureContext, + pub model_sha256: String, + pub model_version: u32, + pub qualification_sha256: String, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct SelectionReceipt { + pub schema: String, + pub owner_scope: String, + pub installation_id: String, + pub revision: u64, + pub selected_at_unix_ms: u64, + pub selected: Option, + pub previous: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct SelectionEnvelope { + pub receipt_sha256: String, + pub receipt: SelectionReceipt, +} + +/// Bounded identity suitable for read-scoped paired phones; no owner identifier/raw context. +#[derive(Debug, Clone, Serialize, PartialEq)] +pub struct SelectedModelIdentity { + pub selection_sha256: String, + pub selection_revision: u64, + pub model_sha256: String, + pub model_version: u32, + pub qualification_sha256: String, + pub policy_sha256: String, + pub context_digest: String, + pub room_digest: String, + pub source_node_ids: Vec, + pub selected_at_unix_ms: u64, + pub evaluated_at_unix_ms: u64, + pub authority_scope: String, +} +#[derive(Debug, Clone, Serialize)] +#[serde(tag = "status", rename_all = "snake_case")] +pub enum SelectedBinaryEvidence { + None, + Unavailable { + identity: Option, + reason: String, + }, + Ready { + identity: SelectedModelIdentity, + decision: PresenceDecision, + presence: bool, + occupied_probability: f32, + observed_at_unix_ms: u64, + observation_digest: String, + }, +} +#[derive(Debug, Clone, Serialize)] +pub struct QualificationSummary { + pub receipt_sha256: String, + pub model_sha256: String, + pub context_digest: String, + pub room_digest: String, + pub evaluated_at_unix_ms: u64, + pub policy: super::qualification::QualificationPolicy, + pub policy_sha256: String, + pub label_provenance: String, + pub criteria: Vec, + pub authority_scope: String, +} +impl From<&QualificationEnvelope> for QualificationSummary { + fn from(q: &QualificationEnvelope) -> Self { + let r = &q.receipt; + Self { + receipt_sha256: q.receipt_sha256.clone(), + model_sha256: r.model_sha256.clone(), + context_digest: r.context_digest.clone(), + room_digest: r.room_digest.clone(), + evaluated_at_unix_ms: r.evaluated_at_unix_ms, + policy: r.policy.clone(), + policy_sha256: r.policy_sha256.clone(), + label_provenance: r.label_provenance.clone(), + criteria: r.criteria.clone(), + authority_scope: r.authority_scope.clone(), + } + } +} +#[derive(Debug, Clone, Serialize)] +pub struct ModelSelectionStatus { + pub schema: &'static str, + pub revision: u64, + pub selected: Option, + pub previous_available: bool, + pub qualification: Option, + pub qualification_error: Option, + pub integrity_error: Option, + pub current: SelectedBinaryEvidence, +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection_store.rs new file mode 100644 index 0000000000..c762aaf20e --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection_store.rs @@ -0,0 +1,191 @@ +//! Owner-private immutable qualification and guarded durable selected-model pointer. +use super::*; +use crate::room_learner::{ + qualification::{hash, QualificationEnvelope}, + selection::{SelectionEnvelope, SelectionTarget}, +}; +const SELECTION: &str = "model-selection-v1.json"; +impl ArtifactStore { + fn qualification_dir( + &self, + context: &CaptureContext, + create: bool, + ) -> Result, String> { + let Some(dir) = self.context_dir(context, create)? else { + return Ok(None); + }; + let path = dir.join("qualifications"); + match std::fs::symlink_metadata(&path) { + Err(e) if e.kind() == io::ErrorKind::NotFound && !create => return Ok(None), + Err(e) if e.kind() != io::ErrorKind::NotFound => return Err(e.to_string()), + _ => {} + } + private_dir(&path).map_err(|e| e.to_string())?; + Ok(Some(path)) + } + pub(crate) fn save_qualification( + &self, + model: &RoomModel, + q: &QualificationEnvelope, + ) -> Result<(), String> { + self.load_exact_candidate(model.context(), &q.receipt.model_sha256)?; + let eval = self.load_exact_evaluation( + model, + crate::room_learner::observation::DataRole::HeldOut, + &q.receipt.evaluation_receipt_sha256, + )?; + q.validate(model, &eval)?; + let dir = self + .qualification_dir(model.context(), true)? + .ok_or("qualification directory unavailable")?; + evaluation_store::write_immutable( + &dir.join(format!("{}.json", q.receipt_sha256)), + &encode(q)?, + )?; + self.load_qualification(model, &q.receipt_sha256)?; + let pointer = serde_json::json!({"receipt_sha256":q.receipt_sha256,"model_sha256":q.receipt.model_sha256,"context_digest":model.context().digest()}); + atomic_private_write( + &dir.join(format!("{}.latest.json", q.receipt.model_sha256)), + &encode(&pointer)?, + ) + .map_err(|e| e.to_string())?; + Ok(()) + } + pub(crate) fn load_qualification( + &self, + model: &RoomModel, + sha: &str, + ) -> Result { + if !hex_digest(sha) { + return Err("invalid qualification digest".into()); + } + let dir = self + .qualification_dir(model.context(), false)? + .ok_or("qualification unavailable")?; + let q: QualificationEnvelope = serde_json::from_slice( + &private_read(&dir.join(format!("{sha}.json"))).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + if q.receipt_sha256 != sha { + return Err("qualification hash mismatch".into()); + } + let eval = self.load_exact_evaluation( + model, + crate::room_learner::observation::DataRole::HeldOut, + &q.receipt.evaluation_receipt_sha256, + )?; + q.validate(model, &eval)?; + Ok(q) + } + pub(crate) fn load_latest_qualification( + &self, + model: &RoomModel, + ) -> Result, String> { + let Some(dir) = self.qualification_dir(model.context(), false)? else { + return Ok(None); + }; + let sha = digest(&model.export()?); + let path = dir.join(format!("{sha}.latest.json")); + if !regular_or_missing(&path)? { + return Ok(None); + } + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct Pointer { + receipt_sha256: String, + model_sha256: String, + context_digest: String, + } + let p: Pointer = serde_json::from_slice(&private_read(&path).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + if p.model_sha256 != sha || p.context_digest != model.context().digest() { + return Err("qualification pointer identity mismatch".into()); + } + let q = self.load_qualification(model, &p.receipt_sha256)?; + let Some(a) = self.load_latest_assessment(model)? else { + return Ok(None); + }; + if a.receipt.failure_reason.is_some() { + return Ok(None); + } + if a.receipt.qualification_sha256.as_deref() != Some(q.receipt_sha256.as_str()) { + return Err("latest qualification pointer differs from completed assessment".into()); + } + Ok(Some(q)) + } + pub(crate) fn resolve_selection_target( + &self, + t: &SelectionTarget, + ) -> Result<(RoomModel, QualificationEnvelope), String> { + self.validate_context(&t.context)?; + let model = self.load_exact_candidate(&t.context, &t.model_sha256)?; + let q = self.load_qualification(&model, &t.qualification_sha256)?; + if model.version != t.model_version || q.receipt.model_sha256 != t.model_sha256 { + return Err("selection model identity mismatch".into()); + } + Ok((model, q)) + } + pub(crate) fn load_selection(&self) -> Result, String> { + self.verify_owner()?; + let path = self.root.join(SELECTION); + if !regular_or_missing(&path)? { + return Ok(None); + } + let envelope: SelectionEnvelope = + serde_json::from_slice(&private_read(&path).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + let r = &envelope.receipt; + if r.schema != "ruview.room-learner.model-selection.v1" + || envelope.receipt_sha256 != hash(r)? + || r.owner_scope != self.owner_scope + || r.installation_id != self.installation_id + || r.revision == 0 + || r.selected_at_unix_ms == 0 + || r.selected_at_unix_ms > 253_402_300_799_999 + { + return Err("selected model pointer identity mismatch".into()); + } + for t in [&r.selected, &r.previous].into_iter().flatten() { + self.validate_context(&t.context)?; + if !hex_digest(&t.model_sha256) || !hex_digest(&t.qualification_sha256) { + return Err("invalid selection target digest".into()); + } + } + Ok(Some(envelope)) + } + pub(crate) fn save_selection( + &self, + envelope: &SelectionEnvelope, + expected_revision: u64, + ) -> Result<(), String> { + let current = self.load_selection()?; + if current.as_ref().map_or(0, |e| e.receipt.revision) != expected_revision + || envelope.receipt.revision + != expected_revision + .checked_add(1) + .ok_or("selection revision exhausted")? + || envelope.receipt.owner_scope != self.owner_scope + || envelope.receipt.installation_id != self.installation_id + || envelope.receipt.schema != "ruview.room-learner.model-selection.v1" + || envelope.receipt_sha256 != hash(&envelope.receipt)? + { + return Err("selection pointer changed or invalid".into()); + } + if let Some(t) = &envelope.receipt.selected { + self.resolve_selection_target(t)?; + } + let bytes = encode(envelope)?; + // Save history before atomically replacing the sole authoritative pointer. + let history = self.root.join("selections"); + private_dir(&history).map_err(|e| e.to_string())?; + evaluation_store::write_immutable( + &history.join(format!("{}.json", envelope.receipt_sha256)), + &bytes, + )?; + atomic_private_write(&self.root.join(SELECTION), &bytes).map_err(|e| e.to_string())?; + if self.load_selection()?.as_ref() != Some(envelope) { + return Err("selection durable readback mismatch".into()); + } + Ok(()) + } +} From 16687c74348363e9f91552672a5825fe33924593 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Tue, 3 Nov 2026 15:30:32 +0100 Subject: [PATCH 050/100] feat(learner): publish guarded selected binary room evidence --- .../src/companion_integration.rs | 11 +- .../src/companion_room.rs | 29 ++ .../wifi-densepose-sensing-server/src/main.rs | 84 +++++- .../src/room_learner_api.rs | 14 +- .../src/room_learner_api_publication.rs | 181 ++++++++++++ .../src/room_learner_api_selection.rs | 145 +++++++++ ...om_learner_api_selection_composer_tests.rs | 276 ++++++++++++++++++ .../src/room_learner_api_selection_fixture.rs | 247 ++++++++++++++++ .../src/room_learner_api_selection_tests.rs | 106 +++++++ .../src/room_learner_api_tests.rs | 6 + 10 files changed, 1093 insertions(+), 6 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner_api_publication.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_fixture.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_tests.rs diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs index 7ef55af8e1..67084e8025 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs @@ -94,6 +94,7 @@ impl Integration { ) }; let room = companion_room::SharedRoomStore::load(&dir, &identity.installation_id)?; + state.write().await.current_room_store = Some(room.clone()); let control = Control { state: state.clone(), serial: Arc::new(tokio::sync::Mutex::new(())), @@ -229,7 +230,7 @@ fn bearer(request: &Request) -> Option<&str> { }; Some(token) } -async fn require_owner(State(auth): State, request: Request, next: Next) -> Response { +pub(super) async fn require_owner(State(auth): State, request: Request, next: Next) -> Response { let local = request .extensions() .get::>() @@ -256,6 +257,7 @@ fn policy(method: &axum::http::Method, path: &str) -> Option { | "/api/v1/pose/stats" | "/api/v1/pose/zones/summary" | "/api/v1/pose/activities" + | "/api/v1/room-learner/selection" | "/api/v1/calibration/status" | "/api/v1/pose/calibration/status" | "/ws/sensing" @@ -382,6 +384,13 @@ mod tests { assert_eq!(policy(&Method::POST, path), None, "{path}"); } assert_eq!(policy(&Method::GET, "/ws/train/progress"), None); + assert_eq!(policy(&Method::GET, "/api/v1/room-learner/selection"), Some(false)); + assert_eq!(policy(&Method::HEAD, "/api/v1/room-learner/selection"), Some(false)); + assert_eq!(policy(&Method::GET, "/api/v1/room-learner/status"), None); + for path in ["/api/v1/room-learner/qualify", "/api/v1/room-learner/select", "/api/v1/room-learner/deselect"] { + assert_eq!(policy(&Method::POST, path), None, "paired read/calibration grants cannot select: {path}"); + } + } #[tokio::test] async fn peer_composition_never_takes_the_calibration_lock_twice() { diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_room.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_room.rs index aa9a62d24a..12ec77f0d3 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/companion_room.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_room.rs @@ -259,6 +259,21 @@ impl SharedRoomStore { && ids.len() == node_ids.len() && ids == expected } + /// Read the current geometry binding without publishing or advancing its revision. + /// Model selection is separate metadata and must never mutate room geometry. + /// Hold the read binding stable across a guarded model-selection operation. + /// Lock order is app state -> shared room -> learner; no callback may publish + /// geometry or reacquire this store. + pub fn with_current_binding(&self, operation: impl FnOnce(Option<&str>) -> T) -> Result { + let room = self.current.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + Ok(operation(room.binding_digest.as_deref())) + } + + pub fn current_binding_digest(&self) -> Result, StatusCode> { + self.current.lock().map(|room| room.binding_digest.clone()) + .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR) + } + fn publish(&self, expected: u64, room: Value) -> Result { let room = validate(room).map_err(|_| StatusCode::BAD_REQUEST)?; let mut current = self @@ -365,6 +380,20 @@ mod tests { assert!(restored.matches_binding(e.binding_digest.as_ref().unwrap(), &[17])); assert!(SharedRoomStore::load(dir.path(), "other").is_err()); } + #[test] + fn reading_current_binding_never_publishes_a_new_geometry_revision() { + let dir = tempfile::tempdir().unwrap(); + let store = SharedRoomStore::load(dir.path(), "install-1").unwrap(); + assert_eq!(store.current_binding_digest().unwrap(), None); + let published = store.publish(0, room()).unwrap(); + let before = fs::read(&store.path).unwrap(); + for _ in 0..3 { + assert_eq!(store.current_binding_digest().unwrap(), published.binding_digest); + } + assert_eq!(store.current.lock().unwrap().revision, published.revision); + assert_eq!(fs::read(&store.path).unwrap(), before); + } + #[test] fn malformed_geometry_unknown_evidence_and_duplicate_slots_rejected() { for (key, value) in [ diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index 152205c8f5..8b667cf270 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -533,6 +533,12 @@ struct SensingUpdate { nodes: Vec, features: FeatureInfo, classification: ClassificationInfo, + /// Legacy classification is retained only as a labelled comparison while a + /// selected binary model owns presence. It does not acquire learner authority. + #[serde(default, skip_serializing_if = "Option::is_none")] + classification_source: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + selected_binary_evidence: Option, signal_field: SignalField, /// Vital sign estimates (breathing rate, heart rate, confidence). #[serde(skip_serializing_if = "Option::is_none")] @@ -2212,6 +2218,7 @@ struct AppStateInner { source_bank_shadow: Option, room_learning: Option>, room_learning_sources: HashMap, + current_room_store: Option>, /// Stable installation identity used only to bind local persisted state. installation_id: Option, /// Metadata for the privacy reduced empty room image, when available. @@ -3148,6 +3155,7 @@ impl AppStateInner { source_bank_shadow: None, room_learning: None, room_learning_sources: HashMap::new(), + current_room_store: None, installation_id: None, bootstrap_baseline: None, bootstrap_baseline_active: false, @@ -5553,6 +5561,8 @@ async fn wifi_task(state: SharedState, tick_ms: u64) { }], features, classification, + classification_source: None, + selected_binary_evidence: None, signal_field: generate_signal_field( first_rssi, motion_score, @@ -5600,6 +5610,7 @@ async fn wifi_task(state: SharedState, tick_ms: u64) { attach_field_positions(&mut update); assess_legacy_image_pose(&mut s, &update); + room_learner_api::project_live_selection(&s, &mut update); if let Ok(json) = serde_json::to_string(&update) { let _ = s.tx.send(json); } @@ -5724,6 +5735,8 @@ async fn windows_wifi_fallback_tick(state: &SharedState, seq: u32) { }], features, classification, + classification_source: None, + selected_binary_evidence: None, signal_field: generate_signal_field( rssi_dbm, motion_score, @@ -5767,6 +5780,7 @@ async fn windows_wifi_fallback_tick(state: &SharedState, seq: u32) { attach_field_positions(&mut update); assess_legacy_image_pose(&mut s, &update); + room_learner_api::project_live_selection(&s, &mut update); if let Ok(json) = serde_json::to_string(&update) { let _ = s.tx.send(json); } @@ -6285,6 +6299,7 @@ fn current_sensing_snapshot(state: &AppStateInner) -> Option { snapshot.persons = None; snapshot.pose_keypoints = None; } + room_learner_api::project_cached_selection(state, &mut snapshot); Some(snapshot) } @@ -6325,6 +6340,11 @@ async fn handle_ws_client(mut socket: WebSocket, state: SharedState) { } => { match msg { Ok(json) => { + let refreshed = { + let s = state.read().await; + room_learner_api::refresh_queued_update(&s, &json) + }; + let Some(json) = refreshed else { continue; }; let json = if privacy_mode { match privacy_filter::redact_json_str(&json) { Some(json) => json, @@ -6454,11 +6474,16 @@ async fn handle_ws_pose_client(mut socket: WebSocket, state: SharedState) { } => { match msg { Ok(json) => { + let refreshed = { + let s = state.read().await; + room_learner_api::refresh_queued_update(&s, &json) + }; + let Some(json) = refreshed else { continue; }; // Parse the sensing update and convert to pose format if let Ok(sensing) = serde_json::from_str::(&json) { if sensing.msg_type == "sensing_update" { let age_ms = chrono::Utc::now().timestamp_millis() as f64 - sensing.timestamp * 1000.0; - if sensing.source.ends_with(":offline") || age_ms > LATEST_UPDATE_STALE_AFTER.as_millis() as f64 { + if sensing.selected_binary_evidence.is_some() || sensing.source.ends_with(":offline") || age_ms > LATEST_UPDATE_STALE_AFTER.as_millis() as f64 { // Unavailable is not measured absence. Never reconstruct // a skeleton from a deliberately expired cached snapshot. let mut unavailable = serde_json::json!({ @@ -6687,6 +6712,8 @@ mod latest_update_staleness_tests { presence: false, confidence: 0.97, }, + classification_source: None, + selected_binary_evidence: None, signal_field: SignalField { grid_size: [0, 0, 0], values: vec![], @@ -7853,6 +7880,10 @@ fn derive_single_person_pose( /// ever *lowers* a Derived cycle from P5 to P4, both of which are already held /// edge-local, so it cannot leak. fn emit_rufield_event(s: &AppStateInner, update: &SensingUpdate, node_id: u8) { + // The legacy field bridge cannot encode a selected model/qualification or + // its source-set/window identity. Never relabel packet features and signal + // field peaks as qualified binary-model motion/count/location evidence. + if update.selected_binary_evidence.is_some() { return; } // No-presence ⇒ no phantom event. if !update.classification.presence { return; @@ -8040,6 +8071,7 @@ fn assess_legacy_image_pose(state: &mut AppStateInner, update: &SensingUpdate) { } fn derive_pose_from_sensing(update: &SensingUpdate) -> Vec { + if update.selected_binary_evidence.is_some() { return vec![]; } let cls = &update.classification; if !cls.presence { return vec![]; @@ -8319,6 +8351,12 @@ async fn pose_current( Query(query): Query, ) -> impl IntoResponse { let s = state.read().await; + if room_learner_api::selection_owns_output(&s) { + return (StatusCode::CONFLICT, Json(serde_json::json!({ + "code":"selected_binary_model_has_no_pose_authority", "persons":[], "total_persons":null, + "detail":"The selected model qualifies room presence only; no pose or person count is available." + }))); + } let persons = match &s.latest_update { Some(update) => update .persons @@ -8404,6 +8442,10 @@ async fn pose_physics_metrics(State(state): State) -> impl IntoResp async fn pose_stats(State(state): State) -> Json { let s = state.read().await; + if room_learner_api::selection_owns_output(&s) { + return Json(serde_json::json!({"authority":"withheld","reason":"selected_binary_model_has_no_pose_authority", + "total_detections":null,"average_confidence":null,"frames_processed":s.tick,"source":s.effective_source()})); + } Json(serde_json::json!({ "total_detections": s.total_detections, "average_confidence": 0.87, @@ -8414,6 +8456,9 @@ async fn pose_stats(State(state): State) -> Json async fn pose_zones_summary(State(state): State) -> Json { let s = state.read().await; + if room_learner_api::selection_owns_output(&s) { + return Json(serde_json::json!({"zones":{},"authority":"withheld","reason":"selected_binary_model_has_no_zone_count_authority"})); + } let presence = s .latest_update .as_ref() @@ -11566,6 +11611,14 @@ fn chrono_timestamp() -> u64 { async fn vital_signs_endpoint(State(state): State) -> Json { let s = state.read().await; + if room_learner_api::selection_owns_output(&s) { + return Json(serde_json::json!({"vital_signs":{ + "breathing_rate_bpm":null,"heart_rate_bpm":null,"breathing_confidence":null, + "heartbeat_confidence":null,"signal_quality":null},"authority":"abstained", + "abstention_reason":"selected_binary_model_has_no_vital_authority","buffer_status":null, + "buffer_source_node_id":null,"diagnostics":{"schema":vital_publication::DIAGNOSTICS_SCHEMA, + "privacy_redacted":s.privacy_mode,"nodes":[]},"source":s.effective_source(),"tick":s.tick})); + } let observed = chrono::Utc::now().timestamp_millis().max(0) as u64; let room_count = current_vital_room_count(&s, std::time::Instant::now()); // Candidate, room aggregate, model and diagnostic snapshot share one lock. @@ -11688,6 +11741,10 @@ async fn edge_registry_endpoint( /// GET /api/v1/edge-vitals — latest edge vitals from ESP32 (ADR-039). async fn edge_vitals_endpoint(State(state): State) -> Json { let s = state.read().await; + if room_learner_api::selection_owns_output(&s) { + return Json(serde_json::json!({"status":"abstained","edge_vitals":null, + "message":"The selected binary model does not grant vital-sign authority."})); + } let observed_at_unix_ms = chrono::Utc::now().timestamp_millis().max(0) as u64; let explicit_single_occupant = s.explicit_calibration_fresh_at(observed_at_unix_ms) && s.person_count_at(observed_at_unix_ms) == 1; @@ -12823,6 +12880,8 @@ fn mediatek_room_update( classification, // No spatial field: this heuristic has no per-subcarrier or // multi-antenna geometry to place a heat map from. + classification_source: None, + selected_binary_evidence: None, signal_field: SignalField { grid_size: [0, 0, 0], values: vec![], @@ -13467,13 +13526,14 @@ async fn udp_receiver_task( .observe(now, mean_amp as f64, peak_amp as f64); s.tick += 1; let tick = s.tick; - let update = mediatek_room_update( + let mut update = mediatek_room_update( &mut s, now, ESP32_OFFLINE_TIMEOUT, tick, source_label, ); + room_learner_api::project_live_selection(&s, &mut update); if let Ok(json) = serde_json::to_string(&update) { let _ = s.tx.send(json); } @@ -13732,8 +13792,10 @@ async fn udp_receiver_task( explicit_calibration_fresh, total_persons, ); - if let Ok(json) = serde_json::to_string(&edge_vitals_message) { - let _ = s.tx.send(json); + if !room_learner_api::selection_owns_output(&s) { + if let Ok(json) = serde_json::to_string(&edge_vitals_message) { + let _ = s.tx.send(json); + } } let calibrated_presence_evidence = s.calibrated_presence_evidence( @@ -13749,6 +13811,8 @@ async fn udp_receiver_task( nodes: active_nodes, features: fused_features.clone(), classification, + classification_source: None, + selected_binary_evidence: None, signal_field, vital_signs: published_vitals, vital_evidence, @@ -13794,6 +13858,7 @@ async fn udp_receiver_task( attach_field_positions(&mut update); assess_legacy_image_pose(&mut s, &update); + room_learner_api::project_live_selection(&s, &mut update); if let Ok(json) = serde_json::to_string(&update) { let _ = s.tx.send(json); } @@ -14242,6 +14307,8 @@ async fn udp_receiver_task( // The count uses this same room presence gate; only // `motion_score` remains local to the arriving packet. classification: room_classification, + classification_source: None, + selected_binary_evidence: None, signal_field: generate_signal_field( fused_features.mean_rssi, motion_score, @@ -14293,6 +14360,7 @@ async fn udp_receiver_task( attach_field_positions(&mut update); assess_legacy_image_pose(&mut s, &update); + room_learner_api::project_live_selection(&s, &mut update); if let Ok(json) = serde_json::to_string(&update) { let _ = s.tx.send(json); } @@ -14483,6 +14551,8 @@ async fn simulated_data_task(state: SharedState, tick_ms: u64) { }], features: features.clone(), classification, + classification_source: None, + selected_binary_evidence: None, signal_field: generate_signal_field( features.mean_rssi, motion_score, @@ -14543,6 +14613,7 @@ async fn simulated_data_task(state: SharedState, tick_ms: u64) { if update.classification.presence { s.total_detections += 1; } + room_learner_api::project_live_selection(&s, &mut update); if let Ok(json) = serde_json::to_string(&update) { let _ = s.tx.send(json); } @@ -15926,6 +15997,7 @@ async fn main() { source_bank_shadow: None, room_learning, room_learning_sources: HashMap::new(), + current_room_store: None, installation_id: args.installation_id.clone(), bootstrap_baseline: bootstrap_metadata.clone(), bootstrap_baseline_active, @@ -16345,6 +16417,8 @@ async fn main() { .route("/api/v1/calibration/status", get(calibration_status)) // Mutations remain behind the existing default-admin bearer policy. .route("/api/v1/room-learner/status", get(room_learner_api::status)) + .route("/api/v1/room-learner/selection", get(room_learner_api::selection)) + .merge(room_learner_api::owner_selection_routes(bearer_auth_state.clone())) .route("/api/v1/room-learner/configure", post(room_learner_api::configure)) .route("/api/v1/room-learner/phase/start", post(room_learner_api::begin_phase)) .route("/api/v1/room-learner/phase/finish", post(room_learner_api::finish_phase)) @@ -17258,6 +17332,8 @@ mod observatory_persons_field_position_tests { presence, confidence: 0.8, }, + classification_source: None, + selected_binary_evidence: None, signal_field, vital_signs: None, vital_evidence: None, diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs index 7449bf488d..2967bf0499 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs @@ -199,7 +199,10 @@ pub(super) async fn configure( } pub(super) async fn status(State(state): State) -> Json { match runtime(&state).await { - Ok(rt) => Json(rt.status(now_ns(&rt))), + Ok(rt) => { + let state = state.read().await; + Json(selection_routes::enriched_status(&state, &rt)) + } Err(e) => e, } } @@ -437,3 +440,12 @@ pub(super) async fn discard_capture( #[cfg(test)] #[path = "room_learner_api_tests.rs"] mod tests; + + +#[path = "room_learner_api_publication.rs"] +mod publication; +pub(super) use publication::{project_cached_selection, project_live_selection, refresh_queued_update, selection_owns_output}; + +#[path = "room_learner_api_selection.rs"] +mod selection_routes; +pub(super) use selection_routes::{owner_selection_routes, selection}; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_publication.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_publication.rs new file mode 100644 index 0000000000..626b074be9 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_publication.rs @@ -0,0 +1,181 @@ +//! One final binary-only projection for live publication and cached readback. +//! Legacy classification remains a labelled comparison, never a fallback for +//! an unavailable selected model. No count, pose, motion or Vitals authority is +//! inferred from the binary model. +use super::*; +use wifi_densepose_sensing_server::room_learner::{ + model::PresenceDecision, selection::SelectedBinaryEvidence, +}; + +const SCHEMA: &str = "ruview.room-learner.selected-binary-evidence.v1"; + +fn unavailable(evidence: SelectedBinaryEvidence, reason: &str) -> SelectedBinaryEvidence { + let identity = match evidence { + SelectedBinaryEvidence::None => return SelectedBinaryEvidence::None, + SelectedBinaryEvidence::Unavailable { identity, .. } => identity, + SelectedBinaryEvidence::Ready { identity, .. } => Some(identity), + }; + SelectedBinaryEvidence::Unavailable { + identity, + reason: reason.into(), + } +} + +pub(crate) fn selection_owns_output(state: &AppStateInner) -> bool { + state.room_learning.as_ref().is_some_and(|runtime| { + !matches!( + runtime.selected_binary_evidence(now_ns(runtime), None), + SelectedBinaryEvidence::None + ) + }) +} + +fn current(state: &AppStateInner) -> SelectedBinaryEvidence { + let Some(rt) = state.room_learning.as_ref() else { + return SelectedBinaryEvidence::None; + }; + let now = now_ns(rt); + let evidence = match state.current_room_store.as_ref() { + Some(room) => room + .with_current_binding(|binding| rt.selected_binary_evidence(now, binding)) + .unwrap_or_else(|_| rt.selected_binary_evidence(now, None)), + None => rt.selected_binary_evidence(now, None), + }; + if state.privacy_mode { + return match evidence { + SelectedBinaryEvidence::None => SelectedBinaryEvidence::None, + _ => SelectedBinaryEvidence::Unavailable { + identity: None, + reason: "selected_learning_unavailable_in_privacy_mode".into(), + }, + }; + } + match &evidence { + SelectedBinaryEvidence::Ready { + decision, + presence, + occupied_probability, + .. + } if !occupied_probability.is_finite() + || !(0.0..=1.0).contains(occupied_probability) + || !matches!( + (decision, presence), + (PresenceDecision::Empty, false) | (PresenceDecision::Occupied, true) + ) => + { + unavailable(evidence, "invalid_selected_binary_decision") + } + _ => evidence, + } +} + +fn envelope(state: &AppStateInner, evidence: SelectedBinaryEvidence) -> Option { + if matches!(evidence, SelectedBinaryEvidence::None) { + return None; + } + let mut value = + serde_json::to_value(evidence).expect("bounded selected binary proof serializes"); + let object = value + .as_object_mut() + .expect("tagged selected binary proof is an object"); + object.insert("schema".into(), serde_json::json!(SCHEMA)); + object.insert( + "boot_epoch".into(), + if state.privacy_mode { + serde_json::Value::Null + } else { + serde_json::json!(state.calibration_boot_epoch) + }, + ); + Some(value) +} + +fn apply(update: &mut SensingUpdate, proof: Option) { + update.selected_binary_evidence = proof; + if update.selected_binary_evidence.is_none() { + update.classification_source = None; + return; + } + update.classification_source = Some("legacy_control_comparison".into()); + update.calibrated_presence_evidence = None; + update.detector_diagnostics = None; + update.estimated_persons = None; + update.persons = None; + update.pose_keypoints = None; + update.vital_signs = None; + update.vital_evidence = None; + update.enhanced_breathing = None; + update.enhanced_motion = None; + update.posture = None; + // Node and room classification diagnostics retain their own legacy source; + // they are not selected binary claims. None carries a qualified count. +} + +pub(crate) fn project_live_selection(state: &AppStateInner, update: &mut SensingUpdate) { + let mut evidence = current(state); + if update.source != "esp32" { + evidence = unavailable(evidence, "selected_native_source_update_required"); + } + apply(update, envelope(state, evidence)); +} + +pub(crate) fn project_cached_selection(state: &AppStateInner, update: &mut SensingUpdate) { + let mut evidence = current(state); + if let SelectedBinaryEvidence::Ready { identity, .. } = &evidence { + let cached = update.selected_binary_evidence.as_ref(); + let now_ms = state + .room_learning + .as_ref() + .map(|rt| now_ns(rt) / 1_000_000); + let identity = + serde_json::to_value(identity).expect("bounded selected identity serializes"); + let matches = cached.is_some_and(|proof| { + let fresh = proof["observed_at_unix_ms"] + .as_u64() + .zip(now_ms) + .and_then(|(observed, now)| now.checked_sub(observed)) + .is_some_and(|age| { + age <= wifi_densepose_sensing_server::room_learner::observation::MAX_AGE_NS + / 1_000_000 + }); + proof["schema"] == SCHEMA + && proof["boot_epoch"] == state.calibration_boot_epoch + && proof["status"] == "ready" + && proof["identity"] == identity + && proof["observation_digest"].as_str().is_some_and(|digest| { + digest.len() == 64 && digest.bytes().all(|b| b.is_ascii_hexdigit()) + }) + && fresh + }); + if update.source != "esp32" || !matches { + evidence = unavailable(evidence, "selected_cached_observation_unavailable"); + } else { + // A cache read cannot manufacture a newly observed measurement or + // advance its timestamp. Keep the original exact proof. + let cached = cached.cloned(); + apply(update, cached); + return; + } + } + apply(update, envelope(state, evidence)); +} + +/// A queued frame may predate selection, rollback, a room change or source +/// withdrawal. Revalidate its authority at the subscriber boundary without +/// converting it into a new measurement. Non-sensing control messages remain +/// byte-identical; legacy edge-vitals messages are withheld while selected. +pub(crate) fn refresh_queued_update(state: &AppStateInner, json: &str) -> Option { + if !selection_owns_output(state) && !json.contains("\"selected_binary_evidence\"") { + return Some(json.to_owned()); + } + let value: serde_json::Value = serde_json::from_str(json).ok()?; + match value.get("type").and_then(serde_json::Value::as_str) { + Some("sensing_update") => { + let mut update: SensingUpdate = serde_json::from_value(value).ok()?; + project_cached_selection(state, &mut update); + serde_json::to_string(&update).ok() + } + Some("edge_vitals" | "edge_fused_vitals") if selection_owns_output(state) => None, + _ => Some(json.to_owned()), + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs new file mode 100644 index 0000000000..bfef0f7ab6 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs @@ -0,0 +1,145 @@ +//! Owner-only model-selection commands and bounded read projection. +use super::*; + +/// Selection mutations are restricted to the local owner, even if ordinary +/// API bearer authentication is disabled or a peer has calibration access. +pub(crate) fn owner_selection_routes( + auth: wifi_densepose_sensing_server::bearer_auth::AuthState, +) -> Router { + Router::new() + .route("/api/v1/room-learner/qualify", post(qualify)) + .route("/api/v1/room-learner/select", post(select_model)) + .route("/api/v1/room-learner/deselect", post(deselect_model)) + .route_layer(axum::middleware::from_fn_with_state( + auth, + companion_integration::require_owner, + )) +} + +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct SelectRequest { + expected: MutationGuard, + model_sha256: String, + qualification_sha256: String, +} + +pub(super) fn selection_snapshot( + state: &AppStateInner, + rt: &RoomLearningRuntime, + now: u64, +) -> wifi_densepose_sensing_server::room_learner::selection::ModelSelectionStatus { + let snapshot = match state.current_room_store.as_ref() { + Some(room) => room + .with_current_binding(|binding| rt.selection_snapshot(now, binding)) + .unwrap_or_else(|_| rt.selection_snapshot(now, None)), + None => rt.selection_snapshot(now, None), + }; + snapshot +} + +pub(super) fn enriched_status( + state: &AppStateInner, + rt: &RoomLearningRuntime, +) -> serde_json::Value { + let now = now_ns(rt); + let mut status = rt.status(now); + status["model_selection"] = serde_json::to_value(selection_snapshot(state, rt, now)) + .expect("bounded model selection status serializes"); + status +} + +fn with_current_room( + state: &AppStateInner, + rt: &RoomLearningRuntime, + operation: impl FnOnce() -> Result, +) -> Result { + let room = state + .current_room_store + .as_ref() + .ok_or("current published room unavailable")?; + room.with_current_binding(|binding| { + let context = rt.context().ok_or("learner context unavailable")?; + if binding != Some(context.room_digest.as_str()) + || !context + .sources + .iter() + .all(|source| source_ready(state, source, std::time::Instant::now())) + { + return Err("current room or trained source binding mismatch".into()); + } + operation() + }) + .map_err(|_| "current published room unavailable".to_string())? +} + +pub(crate) async fn qualify( + State(state): State, + Json(request): Json, +) -> Json { + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + let state = state.read().await; + match with_current_room(&state, &rt, || rt.qualify(&request.expected)) { + Ok(receipt) => Json( + serde_json::json!({"success":true,"qualification":receipt,"status":enriched_status(&state, &rt)}), + ), + Err(e) => failure("learner_qualification_rejected", e), + } +} + +pub(crate) async fn select_model( + State(state): State, + Json(request): Json, +) -> Json { + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + let state = state.read().await; + match with_current_room(&state, &rt, || { + rt.select_model( + &request.expected, + &request.model_sha256, + &request.qualification_sha256, + ) + }) { + Ok(receipt) => Json( + serde_json::json!({"success":true,"selection":receipt,"status":enriched_status(&state, &rt)}), + ), + Err(e) => failure("learner_selection_rejected", e), + } +} + +pub(crate) async fn deselect_model( + State(state): State, + Json(request): Json, +) -> Json { + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + let state = state.read().await; + match rt.deselect_model(&request.expected) { + Ok(receipt) => Json( + serde_json::json!({"success":true,"selection":receipt,"status":enriched_status(&state, &rt)}), + ), + Err(e) => failure("learner_deselection_rejected", e), + } +} + +/// Read-scoped phone projection: no owner scope, mutation guard, raw context, +/// training examples or model weights are returned. +pub(crate) async fn selection(State(state): State) -> Json { + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + let state = state.read().await; + Json( + serde_json::json!({"schema":"ruview.room-learner.selection.v1","success":true, + "model_selection":selection_snapshot(&state, &rt, now_ns(&rt))}), + ) +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs new file mode 100644 index 0000000000..efbd5cb03c --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs @@ -0,0 +1,276 @@ +//! SYNTHETIC public-runtime training and actual publication/API composition. +//! These tests grant no hardware accuracy or real-room qualification evidence. +use super::*; +use wifi_densepose_sensing_server::room_learner::{ + capture::{RawSourceFrame, MIN_PHASE_MS, WINDOW_PACKETS}, + observation::RoomLabel, +}; + +#[path = "room_learner_api_selection_fixture.rs"] +mod fixture_support; +use fixture_support::{feed, fitted, owner_post, update}; + +#[tokio::test] +async fn actual_selected_composer_withdraws_without_fallback_and_never_grants_auxiliary_authority() +{ + let (_dir, state, rt, binding) = fitted().await; + let mut out = update(); + { + let inner = state.read().await; + project_live_selection(&inner, &mut out); + } + let proof = out.selected_binary_evidence.as_ref().unwrap(); + assert_eq!(proof["status"], "ready", "{proof}"); + assert_eq!(proof["decision"], "occupied"); + assert_eq!(proof["presence"], true); + assert_eq!( + out.classification_source.as_deref(), + Some("legacy_control_comparison") + ); + assert!( + out.estimated_persons.is_none() && out.persons.is_none() && out.pose_keypoints.is_none() + ); + assert!( + out.vital_signs.is_none() + && out.vital_evidence.is_none() + && out.enhanced_breathing.is_none() + && out.posture.is_none() + ); + assert!(derive_pose_from_sensing(&out).is_empty()); + // Exercise the actual RuField emitter with a valid governed cycle: the + // legacy positive control emits, but the selected binary update cannot be + // relabelled as node motion/location/count evidence. + { + let mut inner = state.write().await; + let mut engine_nodes = HashMap::new(); + for id in [15u8, 16] { + let mut node = NodeState::new(); + node.frame_history = std::collections::VecDeque::from(vec![(0..56) + .map(|bin| 1.0 + f64::from(id) * 0.01 + bin as f64 * 0.1) + .collect()]); + node.last_frame_time = Some(std::time::Instant::now()); + engine_nodes.insert(id, node); + } + assert!(inner + .engine_bridge + .observe_cycle(&engine_nodes, 1000) + .is_some()); + emit_rufield_event(&inner, &update(), 15); + let emitted = inner.field_surface.read().await.recent().len(); + assert!( + emitted > 0, + "valid legacy positive control must reach RuField" + ); + emit_rufield_event(&inner, &out, 15); + assert_eq!(inner.field_surface.read().await.recent().len(), emitted); + } + let observed = proof["observed_at_unix_ms"].clone(); + { + let mut inner = state.write().await; + inner.latest_update = Some(out.clone()); + } + let cached = current_sensing_snapshot(&*state.read().await).unwrap(); + assert_eq!( + cached.selected_binary_evidence.as_ref().unwrap()["observed_at_unix_ms"], + observed + ); + // A fresh Empty model result overrides no legacy fields by implication: + // the authoritative binary proof is false while the explicitly labelled + // comparison still contains its previous positive heuristic. + let grid = rt.context().unwrap().sources[0].grid_id.clone(); + feed(&rt, &grid, RoomLabel::Empty, 2000, rt.receiver_now_ns()); + tokio::time::sleep(std::time::Duration::from_millis(12)).await; + let mut empty = update(); + project_live_selection(&*state.read().await, &mut empty); + let evidence = empty.selected_binary_evidence.as_ref().unwrap(); + assert_eq!(evidence["status"], "ready", "{evidence}"); + assert_eq!(evidence["decision"], "empty"); + assert_eq!(evidence["presence"], false); + assert!( + empty.classification.presence, + "legacy comparator stays source-labelled" + ); + let queued_original = serde_json::to_string(&out).unwrap(); + let retained = refresh_queued_update(&*state.read().await, &queued_original).unwrap(); + let retained: serde_json::Value = serde_json::from_str(&retained).unwrap(); + assert_eq!( + retained["selected_binary_evidence"]["observed_at_unix_ms"], + observed + ); + assert_eq!( + retained["selected_binary_evidence"]["decision"], "occupied", + "rolling cohort advancement must not relabel an earlier still-fresh measured decision" + ); + let mut expired: serde_json::Value = serde_json::from_str(&queued_original).unwrap(); + expired["selected_binary_evidence"]["observed_at_unix_ms"] = + serde_json::json!(rt.receiver_now_ns() / 1_000_000 - 6000); + let expired = refresh_queued_update(&*state.read().await, &expired.to_string()).unwrap(); + assert_eq!( + serde_json::from_str::(&expired).unwrap()["selected_binary_evidence"] + ["status"], + "unavailable" + ); + let boot = state.read().await.calibration_boot_epoch.clone(); + state.write().await.calibration_boot_epoch = "synthetic-other-boot".into(); + let withdrawn_boot = refresh_queued_update(&*state.read().await, &queued_original).unwrap(); + assert_eq!( + serde_json::from_str::(&withdrawn_boot).unwrap() + ["selected_binary_evidence"]["status"], + "unavailable" + ); + state.write().await.calibration_boot_epoch = boot; + state.write().await.latest_update = Some(empty); + + // Legacy preferred-grid bookkeeping is not the learner input authority. + // Its stale/missing node cache must not erase a valid native learner cohort. + let removed = state.write().await.node_states.remove(&16).unwrap(); + let last_seen = state + .write() + .await + .node_states + .get_mut(&15) + .unwrap() + .last_frame_time + .take(); + let cached = current_sensing_snapshot(&*state.read().await).unwrap(); + assert_eq!(cached.selected_binary_evidence.unwrap()["status"], "ready"); + state.write().await.node_states.insert(16, removed); + state + .write() + .await + .node_states + .get_mut(&15) + .unwrap() + .last_frame_time = last_seen; + let wrong_room = + companion_room::SharedRoomStore::load(&_dir.path().join("other-room"), "synthetic-install") + .unwrap(); + let original = state + .write() + .await + .current_room_store + .replace(wrong_room) + .unwrap(); + let mut withdrawn = update(); + project_live_selection(&*state.read().await, &mut withdrawn); + assert_eq!( + withdrawn.selected_binary_evidence.as_ref().unwrap()["status"], + "unavailable" + ); + state.write().await.current_room_store = Some(original); + assert_eq!( + rt.selection_snapshot(rt.receiver_now_ns(), Some(&binding)) + .selected + .unwrap() + .room_digest, + binding + ); + // Every read-only composer withholds auxiliary authority while selected. + let vitals = vital_signs_endpoint(State(state.clone())).await.0; + assert!(vitals["vital_signs"]["heart_rate_bpm"].is_null()); + assert_eq!(vitals["authority"], "abstained"); + let pose = pose_current( + State(state.clone()), + Query(PoseCurrentQuery { + view: pose_physics::PoseView::Raw, + }), + ) + .await + .into_response(); + assert_eq!(pose.status(), StatusCode::CONFLICT); + + assert!(edge_vitals_endpoint(State(state.clone())).await.0["edge_vitals"].is_null()); + assert_eq!( + pose_zones_summary(State(state.clone())).await.0["authority"], + "withheld" + ); + let read = selection(State(state.clone())).await.0; + let serialized = serde_json::to_string(&read).unwrap(); + for forbidden in [ + "owner_scope", + "mutation_guard", + "encoder_weights", + "training_samples", + "head_weights", + "geometry", + ] { + assert!( + !serialized.contains(forbidden), + "paired read leaked {forbidden}" + ); + } + let old_json = + serde_json::to_string(state.read().await.latest_update.as_ref().unwrap()).unwrap(); + { + let inner = state.read().await; + assert!( + refresh_queued_update(&inner, r#"{"type":"edge_vitals","heart_rate_bpm":62}"#) + .is_none() + ); + assert!(refresh_queued_update( + &inner, + r#"{"type":"edge_fused_vitals","heartrate_bpm":62}"# + ) + .is_none()); + } + state.write().await.privacy_mode = true; + let privacy = refresh_queued_update(&*state.read().await, &old_json).unwrap(); + let privacy: serde_json::Value = serde_json::from_str(&privacy).unwrap(); + let private_proof = &privacy["selected_binary_evidence"]; + assert_eq!(private_proof["status"], "unavailable"); + assert!(private_proof["identity"].is_null() && private_proof["boot_epoch"].is_null()); + assert!(private_proof.get("presence").is_none() && private_proof.get("decision").is_none()); + assert_eq!(selection(State(state.clone())).await.0["success"], false); + state.write().await.privacy_mode = false; + + // A selected native source failure clears the actual cohort. Refilling only + // one of the required inputs cannot borrow the old last-good other source. + rt.record_native_rejection(16, &grid, "synthetic selected source interruption"); + let at = rt.receiver_now_ns(); + for sample in 0..WINDOW_PACKETS { + let values: Vec<_> = (0..56) + .map(|bin| 2.3 + bin as f64 * 0.04 + sample as f64 * 0.003) + .collect(); + rt.ingest( + RawSourceFrame { + source_id: 15, + grid_id: &grid, + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: 4000 + sample as u32, + amplitudes: &values, + }, + at + sample as u64 * 1_000_000 + 15, + ) + .unwrap(); + } + tokio::time::sleep(std::time::Duration::from_millis(12)).await; + let mut missing = update(); + project_live_selection(&*state.read().await, &mut missing); + let missing = missing.selected_binary_evidence.unwrap(); + assert_eq!(missing["status"], "unavailable"); + assert!(missing.get("presence").is_none()); + assert!( + rt.selection_snapshot(rt.receiver_now_ns(), Some(&binding)) + .selected + .is_some(), + "withdrawal preserves selection history" + ); + + // Explicit owner deselection restores legacy control; it is never automatic. + let response = owner_post( + state.clone(), + "/api/v1/room-learner/deselect", + serde_json::json!({"expected":rt.mutation_guard().unwrap()}), + ) + .await; + assert_eq!(response["success"], true, "{response}"); + let mut legacy = update(); + project_live_selection(&*state.read().await, &mut legacy); + assert!(legacy.selected_binary_evidence.is_none()); + assert_eq!(legacy.estimated_persons, Some(3)); + let queued = refresh_queued_update(&*state.read().await, &old_json).unwrap(); + let queued: serde_json::Value = serde_json::from_str(&queued).unwrap(); + assert!(queued.get("selected_binary_evidence").is_none()); + assert!(queued.get("classification_source").is_none()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_fixture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_fixture.rs new file mode 100644 index 0000000000..5c81533ffb --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_fixture.rs @@ -0,0 +1,247 @@ +//! SYNTHETIC public API fixture; no manufactured model/qualification receipts. +use super::*; + +pub(super) fn update() -> SensingUpdate { + serde_json::from_value(serde_json::json!({ + "type":"sensing_update","timestamp":chrono::Utc::now().timestamp_millis() as f64/1000.0, + "source":"esp32","tick":321,"nodes":[], + "features":{"mean_rssi":-50.0,"variance":1.0,"motion_band_power":2.0, + "breathing_band_power":1.0,"dominant_freq_hz":0.2,"change_points":0,"spectral_power":3.0}, + "classification":{"motion_level":"present_moving","presence":true,"confidence":0.99}, + "signal_field":{"grid_size":[1,1,1],"values":[1.0]}, + "estimated_persons":3, + "vital_signs":{"breathing_rate_bpm":14.0,"heart_rate_bpm":62.0, + "breathing_confidence":0.8,"heartbeat_confidence":0.7,"signal_quality":0.9}, + "pose_keypoints":[[0.1,0.2,0.0,0.9]],"enhanced_motion":{"level":"moving"}, + "enhanced_breathing":{"rate_bpm":14.0},"posture":"standing" + })).unwrap() +} + +fn gain(label: RoomLabel) -> f64 { + match label { + RoomLabel::Empty => 2.0, + RoomLabel::OccupiedStatic => 6.0, + RoomLabel::OccupiedMoving => 7.0, + RoomLabel::AdjacentOnly => 2.3, + RoomLabel::Unknown => unreachable!(), + } +} +pub(super) fn feed(rt: &RoomLearningRuntime, grid: &str, label: RoomLabel, sequence: u32, at: u64) { + for sample in 0..WINDOW_PACKETS { + for source in [15u8, 16] { + let values: Vec<_> = (0..56) + .map(|bin| { + gain(label) + + f64::from(source - 14) * 0.3 + + bin as f64 * 0.04 + + sample as f64 * 0.003 + }) + .collect(); + rt.ingest( + RawSourceFrame { + source_id: source, + grid_id: grid, + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: sequence + sample as u32, + amplitudes: &values, + }, + at + sample as u64 * 1_000_000 + u64::from(source), + ) + .unwrap(); + } + } +} +fn phase( + rt: &RoomLearningRuntime, + grid: &str, + role: DataRole, + label: RoomLabel, + index: u32, + base: u64, +) { + let at = base + u64::from(index) * 13_000_000_000; + rt.begin_phase( + &rt.mutation_guard().unwrap(), + PhaseRequest { + trial_id: format!("synthetic-composer-trial-{index}"), + session_id: format!("synthetic-composer-session-{index}"), + role, + label, + duration_ms: MIN_PHASE_MS, + lead_in_ms: 0, + conditions_confirmed: true, + }, + at, + ) + .unwrap(); + for quarter in 0..4 { + feed( + rt, + grid, + label, + index * 32 + quarter * 8 + 1, + at + u64::from(quarter) * 3_000_000_000 + 1_000_000, + ); + } + rt.finish_phase( + &rt.mutation_guard().unwrap(), + at + MIN_PHASE_MS * 1_000_000, + true, + ) + .unwrap(); +} +pub(super) async fn owner_post( + state: SharedState, + path: &str, + body: serde_json::Value, +) -> serde_json::Value { + let app = owner_selection_routes( + wifi_densepose_sensing_server::bearer_auth::AuthState::from_token("synthetic-owner"), + ) + .with_state(state); + let mut request = Request::builder() + .method(Method::POST) + .uri(path) + .header("content-type", "application/json") + .header("authorization", "Bearer synthetic-owner") + .body(Body::from(serde_json::to_vec(&body).unwrap())) + .unwrap(); + request.extensions_mut().insert(axum::extract::ConnectInfo( + "127.0.0.1:41000".parse::().unwrap(), + )); + let response = app.oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + serde_json::from_slice( + &axum::body::to_bytes(response.into_body(), 65536) + .await + .unwrap(), + ) + .unwrap() +} + +pub(super) async fn fitted() -> ( + tempfile::TempDir, + SharedState, + Arc, + String, +) { + let (dir, state) = fixture(); + let room = + companion_room::SharedRoomStore::load(&dir.path().join("companion"), "synthetic-install") + .unwrap(); + let room_json = serde_json::json!({"schema":"ruview.companion.room.v1","id":"synthetic-room","name":"Test room", + "widthM":2.5,"lengthM":2.5,"heightM":2.8,"nodes":[ + {"nodeId":15,"slot":"north_west","xM":0,"yM":0,"role":"receiver","headingDeg":90,"orientationConfirmed":true,"confirmedAtUnixMs":1}, + {"nodeId":16,"slot":"north_east","xM":2.5,"yM":0,"role":"receiver","headingDeg":180,"orientationConfirmed":true,"confirmedAtUnixMs":1}],"accessPoint":{"slot":"south_center","xM":1.25,"yM":2.5,"provenance":"operator_entered","confirmedAtUnixMs":1}}); + let response = companion_room::owner_router(room.clone()) + .oneshot( + Request::builder() + .method(Method::PUT) + .uri("/api/v1/companion/room") + .header("content-type", "application/json") + .body(Body::from( + serde_json::to_vec( + &serde_json::json!({"expected_revision":0,"room":room_json}), + ) + .unwrap(), + )) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let binding = room.current_binding_digest().unwrap().unwrap(); + state.write().await.current_room_store = Some(room); + let configured = json_request( + app(state.clone()), + "/configure", + Method::POST, + serde_json::json!({ + "binding_digest":binding,"revision":1,"source_node_ids":[15,16],"new_training":true}), + ) + .await; + assert_eq!(configured["success"], true, "{configured}"); + let rt = state.read().await.room_learning.as_ref().unwrap().clone(); + let grid = rt.context().unwrap().sources[0].grid_id.clone(); + let base = rt.receiver_now_ns() - 200_000_000_000; + let labels = [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ]; + for (i, label) in labels.into_iter().enumerate() { + phase(&rt, &grid, DataRole::Train, label, i as u32, base) + } + // Small real optimizer fixture; production defaults remain unchanged. + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 8, + learning_rate: 0.5, + hidden_dim: 2, + embedding_dim: 2, + temporal_tokens: 2, + ..FitConfig::default() + }, + ) + .unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(90); + loop { + let status = rt.status(rt.receiver_now_ns()); + if status["state"] == "ready" { + break; + } + assert!(status["state"] != "failed", "{status}"); + assert!( + std::time::Instant::now() < deadline, + "bounded synthetic fit timed out" + ); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + for (i, label) in labels.into_iter().enumerate() { + phase(&rt, &grid, DataRole::HeldOut, label, i as u32 + 4, base) + } + rt.evaluate(&rt.mutation_guard().unwrap(), DataRole::HeldOut) + .unwrap(); + // Refresh the actual parser/readiness cache. Then admit labelled-value + // synthetic live frames with fresh sequence IDs through the public runtime. + let now = std::time::Instant::now(); + { + let mut inner = state.write().await; + for row in 0..120 { + for node in [15u8, 16] { + let packet = wire(node, 500 + row); + let frame = parse_esp32_frame(&packet).unwrap(); + let at = now - std::time::Duration::from_millis((120 - row) as u64 * 100); + observe(&mut inner, &frame, &packet, at); + } + } + for node in [15u8, 16] { + inner.node_states.get_mut(&node).unwrap().last_frame_time = + Some(std::time::Instant::now()); + } + inner.source = "esp32".into(); + inner.last_esp32_frame = Some(std::time::Instant::now()); + } + feed( + &rt, + &grid, + RoomLabel::OccupiedStatic, + 1000, + rt.receiver_now_ns(), + ); + tokio::time::sleep(std::time::Duration::from_millis(12)).await; + let q = owner_post( + state.clone(), + "/api/v1/room-learner/qualify", + serde_json::json!({"expected":rt.mutation_guard().unwrap()}), + ) + .await; + assert_eq!(q["success"], true, "{q}"); + let selected=owner_post(state.clone(),"/api/v1/room-learner/select",serde_json::json!({"expected":rt.mutation_guard().unwrap(), + "model_sha256":q["qualification"]["receipt"]["model_sha256"],"qualification_sha256":q["qualification"]["receipt_sha256"]})).await; + assert_eq!(selected["success"], true, "{selected}"); + (dir, state, rt, binding) +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_tests.rs new file mode 100644 index 0000000000..9afeb9972d --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_tests.rs @@ -0,0 +1,106 @@ +use super::*; + +#[tokio::test] +async fn selection_mutations_are_local_owner_only_even_without_outer_bearer_gate() { + let (_dir, state) = fixture(); + let routes = owner_selection_routes( + wifi_densepose_sensing_server::bearer_auth::AuthState::from_token("synthetic-owner"), + ) + .with_state(state.clone()); + for path in [ + "/api/v1/room-learner/qualify", + "/api/v1/room-learner/select", + "/api/v1/room-learner/deselect", + ] { + for (peer, token) in [ + ("127.0.0.1:41000", None), + ("127.0.0.1:41000", Some("synthetic-peer-read")), + ("192.0.2.1:41000", Some("synthetic-owner")), + ] { + let mut request = Request::builder() + .method(Method::POST) + .uri(path) + .header("content-type", "application/json"); + if let Some(token) = token { + request = request.header("authorization", format!("Bearer {token}")); + } + let mut request = request.body(Body::from("{}")).unwrap(); + request.extensions_mut().insert(axum::extract::ConnectInfo( + peer.parse::().unwrap(), + )); + let response = routes.clone().oneshot(request).await.unwrap(); + assert_eq!( + response.status(), + StatusCode::FORBIDDEN, + "{path} {peer} {token:?}" + ); + } + } + // A legitimate owner reaches the typed handler; no geometry is published + // and no data is erased by this negative missing-context operation. + let expected = state + .read() + .await + .room_learning + .as_ref() + .unwrap() + .mutation_guard() + .unwrap(); + let mut request = Request::builder() + .method(Method::POST) + .uri("/api/v1/room-learner/qualify") + .header("content-type", "application/json") + .header("authorization", "Bearer synthetic-owner") + .body(Body::from( + serde_json::to_vec(&serde_json::json!({"expected":expected})).unwrap(), + )) + .unwrap(); + request.extensions_mut().insert(axum::extract::ConnectInfo( + "127.0.0.1:41000".parse::().unwrap(), + )); + let response = routes.oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), 65536) + .await + .unwrap(); + let body: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(body["success"], false); + assert_eq!(body["error_code"], "learner_qualification_rejected"); +} + +#[tokio::test] +async fn selection_readback_does_not_expose_owner_or_mutation_authority() { + let (_dir, state) = fixture(); + let response = Router::new() + .route("/api/v1/room-learner/selection", get(selection)) + .with_state(state) + .oneshot( + Request::builder() + .uri("/api/v1/room-learner/selection") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let bytes = axum::body::to_bytes(response.into_body(), 65536) + .await + .unwrap(); + let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(body["schema"], "ruview.room-learner.selection.v1"); + assert_eq!(body["model_selection"]["current"]["status"], "none"); + let text = String::from_utf8(bytes.to_vec()).unwrap(); + for forbidden in [ + "owner_scope", + "mutation_guard", + "encoder_weights", + "training_samples", + "head_weights", + "runtime_epoch", + ] { + assert!( + !text.contains(forbidden), + "read-scoped selection leaked {forbidden}" + ); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs index ef9f5b0497..fda0d303da 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs @@ -535,3 +535,9 @@ async fn cache_pressure_preserves_selected_active_class_and_freshness() { #[cfg(unix)] #[path = "room_learner_api_recovery_tests.rs"] mod recovery_tests; + +#[path = "room_learner_api_selection_tests.rs"] +mod selection_tests; + +#[path = "room_learner_api_selection_composer_tests.rs"] +mod selection_composer_tests; From f542e618b77af2334edae55b2e7e40eede16f2c1 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Tue, 3 Nov 2026 15:40:54 +0100 Subject: [PATCH 051/100] test(learner): export actual selected-model producer contracts --- ...om_learner_api_selection_composer_tests.rs | 100 ++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs index efbd5cb03c..3e7f5e2bc7 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs @@ -274,3 +274,103 @@ async fn actual_selected_composer_withdraws_without_fallback_and_never_grants_au assert!(queued.get("selected_binary_evidence").is_none()); assert!(queued.get("classification_source").is_none()); } + +/// Explicit opt-in producer fixture for the actual TypeScript wire consumer. +/// All input, training and labels are SYNTHETIC; no device/user state is read. +#[tokio::test] +async fn export_actual_selected_contract_cases() { + async fn capture_case( + name: &str, + state: SharedState, + rt: &RoomLearningRuntime, + ) -> serde_json::Value { + let mut sensing = update(); + project_live_selection(&*state.read().await, &mut sensing); + let response = Router::new() + .route("/api/v1/room-learner/selection", get(selection)) + .with_state(state) + .oneshot( + Request::builder() + .uri("/api/v1/room-learner/selection") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), 65536) + .await + .unwrap(); + let paired_selection: serde_json::Value = serde_json::from_slice(&body).unwrap(); + serde_json::json!({ + "name": name, + "captured_at_unix_ms": rt.receiver_now_ns() / 1_000_000, + "sensing_update": sensing, + "paired_selection": paired_selection, + }) + } + let (_dir, state, rt, _binding) = fitted().await; + let grid = rt.context().unwrap().sources[0].grid_id.clone(); + let occupied = capture_case("ready_occupied", state.clone(), &rt).await; + assert_eq!( + occupied["sensing_update"]["selected_binary_evidence"]["decision"], + "occupied" + ); + feed(&rt, &grid, RoomLabel::Empty, 2000, rt.receiver_now_ns()); + tokio::time::sleep(std::time::Duration::from_millis(12)).await; + let empty = capture_case("ready_empty", state.clone(), &rt).await; + assert_eq!( + empty["sensing_update"]["selected_binary_evidence"]["decision"], + "empty" + ); + rt.record_native_rejection(16, &grid, "synthetic missing selected source"); + let unavailable = capture_case("unavailable_missing_selected_source", state.clone(), &rt).await; + assert_eq!( + unavailable["sensing_update"]["selected_binary_evidence"]["status"], + "unavailable" + ); + state.write().await.privacy_mode = true; + let privacy = capture_case("privacy_withheld", state.clone(), &rt).await; + assert_eq!(privacy["paired_selection"]["success"], false); + assert!(privacy["sensing_update"]["selected_binary_evidence"]["identity"].is_null()); + let producer_sha = std::env::var("RUVIEW_TEST_PRODUCER_SHA").ok(); + let fixture = serde_json::json!({ + "schema": "ruview.test.selected-contract-cases.v1", + "evidence": "SYNTHETIC_SOFTWARE_COMPOSITION_NOT_PHYSICAL_QUALIFICATION", + "producer_commit": producer_sha, + "producer_path": "real public runtime capture/train/evaluate/qualify/select; actual GET selection handler and final SensingUpdate projector", + "clock_contract": "Use each captured_at_unix_ms as an injected parser clock; stored fixtures are not fresh live observations.", + "authentication_scope": "owner and paired-read policy tested separately; this fixture exercises the real bounded GET response producer", + "cases": [occupied, empty, unavailable, privacy], + }); + if let Ok(output) = std::env::var("RUVIEW_TEST_SELECTED_CONTRACT_OUTPUT") { + use std::io::Write; + #[cfg(unix)] + use std::os::unix::fs::OpenOptionsExt; + let sha = producer_sha + .as_deref() + .expect("explicit fixture export requires producer commit"); + assert!(sha.len() == 40 && sha.bytes().all(|b| b.is_ascii_hexdigit())); + let path = std::path::PathBuf::from(output); + assert!(path.is_absolute()); + let parent = path.parent().unwrap().canonicalize().unwrap(); + let system_tmp = std::env::temp_dir().canonicalize().unwrap(); + assert!( + parent.starts_with(&system_tmp) + || parent == std::path::Path::new("/private/tmp") + || parent == std::path::Path::new("/tmp"), + "test export is confined to temporary storage" + ); + let mut options = std::fs::OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + options.mode(0o600); + let mut file = options + .open(&path) + .expect("never overwrite an existing fixture"); + serde_json::to_writer_pretty(&mut file, &fixture).unwrap(); + file.write_all(b"\n").unwrap(); + file.sync_all().unwrap(); + println!("RUVIEW_SELECTED_CONTRACT_FILE={}", path.display()); + } +} From 4fbd3fd5cb636c0716b34f395d2373620ea83254 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Tue, 3 Nov 2026 16:07:26 +0100 Subject: [PATCH 052/100] fix(learner): canonicalize qualification rates and held-out defaults --- .../src/room_learner/lifecycle.rs | 1 + .../room_learner/lifecycle_selection_tests.rs | 14 ++ .../src/room_learner/qualification.rs | 50 ++++-- .../qualification_boundary_tests.rs | 167 ++++++++++++++++++ 4 files changed, 215 insertions(+), 17 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification_boundary_tests.rs diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs index 118b4e73c8..0e57c7e93c 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs @@ -431,6 +431,7 @@ impl RoomLearningRuntime { "candidate_prediction":prediction,"training_manifest":state.model.as_ref().map(|m|m.training_manifest()), "phase_history":state.capture.as_ref().map(|c|c.history()), "phase_duration_defaults":{"Empty":600000,"OccupiedStatic":12000,"OccupiedMoving":12000,"AdjacentOnly":12000}, + "held_out_phase_duration_defaults":{"Empty":12000,"OccupiedStatic":12000,"OccupiedMoving":12000,"AdjacentOnly":12000}, "phase_lead_in_defaults":{"Empty":15000,"OccupiedStatic":5000,"OccupiedMoving":5000,"AdjacentOnly":15000}, "untrained_capture_progress_is_volatile":!state.recovery.enabled || state.recovery.error.is_some(),"recovery":self.recovery_status(&state, now_ns), "generation":state.generation,"state":state.job.phase,"context_digest":state.capture.as_ref().map(|c|c.context().digest()), diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection_tests.rs index ff4b0e6f0b..6da8312774 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection_tests.rs @@ -397,3 +397,17 @@ pub(super) fn phase_override( .unwrap(); at + 12_000_000_000 } + +#[test] +fn held_out_duration_defaults_are_short_and_leave_training_empty_unchanged() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + let status = rt.status(START); + assert_eq!(status["phase_duration_defaults"]["Empty"], 600_000); + for label in ["Empty", "OccupiedStatic", "OccupiedMoving", "AdjacentOnly"] { + assert_eq!(status["held_out_phase_duration_defaults"][label], 12_000); + } + assert_eq!(status["phase_duration_defaults"]["OccupiedStatic"], 12_000); + assert_eq!(crate::room_learner::capture::COHORTS_PER_PHASE, 4); + assert_eq!(crate::room_learner::capture::MIN_PHASE_MS, 12_000); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification.rs index 3840c10ec6..adc0b68898 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification.rs @@ -8,6 +8,10 @@ use super::{ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; +const PERCENT_SCALE: u128 = 100; +const MIN_SENSITIVITY_PERCENT: u128 = 90; +const MAX_FALSE_POSITIVE_PERCENT: u128 = 5; +const MIN_COVERAGE_PERCENT: u128 = 90; pub const POLICY_ID: &str = "ruview.local-operational-policy.v1"; pub(crate) fn hash(value: &T) -> Result { Ok(format!( @@ -20,9 +24,9 @@ pub(crate) fn hash(value: &T) -> Result { pub struct QualificationPolicy { pub id: String, pub minimum_cohorts_per_condition: usize, - pub minimum_unconditional_sensitivity: f32, - pub maximum_false_positive_rate: f32, - pub minimum_evaluated_cohort_coverage: f32, + pub minimum_unconditional_sensitivity: f64, + pub maximum_false_positive_rate: f64, + pub minimum_evaluated_cohort_coverage: f64, pub coverage_scope: String, } impl Default for QualificationPolicy { @@ -30,9 +34,10 @@ impl Default for QualificationPolicy { Self { id: POLICY_ID.into(), minimum_cohorts_per_condition: 4, - minimum_unconditional_sensitivity: 0.9, - maximum_false_positive_rate: 0.05, - minimum_evaluated_cohort_coverage: 0.9, + minimum_unconditional_sensitivity: MIN_SENSITIVITY_PERCENT as f64 + / PERCENT_SCALE as f64, + maximum_false_positive_rate: MAX_FALSE_POSITIVE_PERCENT as f64 / PERCENT_SCALE as f64, + minimum_evaluated_cohort_coverage: MIN_COVERAGE_PERCENT as f64 / PERCENT_SCALE as f64, coverage_scope: "representative_cohorts_not_wall_time".into(), } } @@ -43,9 +48,9 @@ pub struct ConditionCriterion { pub condition: RoomLabel, pub train_cohorts: usize, pub held_out_cohorts: usize, - pub evaluated_cohort_coverage: f32, - pub unconditional_sensitivity: Option, - pub false_positive_rate: Option, + pub evaluated_cohort_coverage: f64, + pub unconditional_sensitivity: Option, + pub false_positive_rate: Option, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] @@ -109,26 +114,33 @@ pub(crate) fn criteria( "qualification requires four Train and HeldOut cohorts for {label:?}" )); } - let coverage = c - .rates + // The diagnostic report remains f32. Operational public rates are + // computed from validated integer counts so JSON Value/JS widening + // cannot turn exact 90% or 5% boundaries into rejection. + c.rates .evaluated_cohort_coverage .ok_or("held-out coverage unavailable")?; - if !coverage.is_finite() || coverage < policy.minimum_evaluated_cohort_coverage { + let coverage = c.scored as f64 / c.samples as f64; + if (c.scored as u128) * PERCENT_SCALE < (c.samples as u128) * MIN_COVERAGE_PERCENT { return Err(format!("held-out coverage failed for {label:?}")); } let occupied = matches!(label, RoomLabel::OccupiedStatic | RoomLabel::OccupiedMoving); let (sensitivity, false_positive) = if occupied { - let s = c - .rates + c.rates .unconditional_sensitivity .ok_or("occupied sensitivity unavailable")?; - if !s.is_finite() || s < policy.minimum_unconditional_sensitivity { + let s = c.confusion.true_occupied as f64 / c.samples as f64; + if (c.confusion.true_occupied as u128) * PERCENT_SCALE + < (c.samples as u128) * MIN_SENSITIVITY_PERCENT + { return Err(format!("occupied sensitivity failed for {label:?}")); } (Some(s), None) } else { - let f = c.confusion.false_occupied as f32 / c.samples as f32; - if !f.is_finite() || f > policy.maximum_false_positive_rate { + let f = c.confusion.false_occupied as f64 / c.samples as f64; + if (c.confusion.false_occupied as u128) * PERCENT_SCALE + > (c.samples as u128) * MAX_FALSE_POSITIVE_PERCENT + { return Err(format!("false presence failed for {label:?}")); } (None, Some(f)) @@ -360,3 +372,7 @@ impl QualificationEnvelope { Ok(()) } } + +#[cfg(test)] +#[path = "qualification_boundary_tests.rs"] +mod boundary_tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification_boundary_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification_boundary_tests.rs new file mode 100644 index 0000000000..ab71b8fe14 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification_boundary_tests.rs @@ -0,0 +1,167 @@ +//! Exact count boundaries and Rust Value/JSON wire contracts; synthetic diagnostics only. +use super::*; +use crate::room_learner::training::{BinaryConfusion, ConditionEvaluation, EvaluationRates}; +fn rates(c: &BinaryConfusion) -> EvaluationRates { + let scored = c.true_empty + c.false_occupied + c.false_empty + c.true_occupied; + let n = scored + c.abstained_empty + c.abstained_occupied; + let ratio = |a: usize, b: usize| { + if b == 0 { + None + } else { + Some(a as f32 / b as f32) + } + }; + EvaluationRates { + evaluated_cohort_coverage: ratio(scored, n), + unconditional_accuracy: ratio(c.true_empty + c.true_occupied, n), + scored_accuracy: ratio(c.true_empty + c.true_occupied, scored), + unconditional_sensitivity: ratio( + c.true_occupied, + c.true_occupied + c.false_empty + c.abstained_occupied, + ), + scored_sensitivity: ratio(c.true_occupied, c.true_occupied + c.false_empty), + unconditional_specificity: ratio( + c.true_empty, + c.true_empty + c.false_occupied + c.abstained_empty, + ), + scored_specificity: ratio(c.true_empty, c.true_empty + c.false_occupied), + } +} +fn condition(label: RoomLabel, c: BinaryConfusion) -> ConditionEvaluation { + let scored = c.true_empty + c.false_occupied + c.false_empty + c.true_occupied; + let abstained = c.abstained_empty + c.abstained_occupied; + ConditionEvaluation { + condition: label, + samples: scored + abstained, + correct: c.true_empty + c.true_occupied, + abstained, + scored, + missing: c.missing_empty + c.missing_occupied, + binary_cross_entropy: Some(0.0), + binary_cross_entropy_samples: scored + abstained, + declared_trial_count: 1, + declared_session_count: 1, + rates: rates(&c), + confusion: c, + } +} +fn report( + false_positive: usize, + occupied_correct: usize, + occupied_abstained: usize, +) -> EvaluationReport { + let conditions = vec![ + condition( + RoomLabel::Empty, + BinaryConfusion { + true_empty: 40 - false_positive, + false_occupied: false_positive, + ..Default::default() + }, + ), + condition( + RoomLabel::OccupiedStatic, + BinaryConfusion { + true_occupied: occupied_correct, + false_empty: 10 - occupied_correct - occupied_abstained, + abstained_occupied: occupied_abstained, + ..Default::default() + }, + ), + condition( + RoomLabel::OccupiedMoving, + BinaryConfusion { + true_occupied: 4, + ..Default::default() + }, + ), + condition( + RoomLabel::AdjacentOnly, + BinaryConfusion { + true_empty: 4, + ..Default::default() + }, + ), + ]; + let mut sum = BinaryConfusion::default(); + for c in &conditions { + sum.true_empty += c.confusion.true_empty; + sum.false_occupied += c.confusion.false_occupied; + sum.true_occupied += c.confusion.true_occupied; + sum.false_empty += c.confusion.false_empty; + sum.abstained_occupied += c.confusion.abstained_occupied; + } + let report = EvaluationReport { + samples: conditions.iter().map(|c| c.samples).sum(), + correct: conditions.iter().map(|c| c.correct).sum(), + abstained: conditions.iter().map(|c| c.abstained).sum(), + scored: conditions.iter().map(|c| c.scored).sum(), + missing: 0, + binary_cross_entropy: Some(0.0), + binary_cross_entropy_samples: conditions + .iter() + .map(|c| c.binary_cross_entropy_samples) + .sum(), + declared_trial_count: 4, + declared_session_count: 4, + rates: rates(&sum), + confusion: sum, + per_condition: conditions, + independence_verified: false, + }; + report.validate().unwrap(); + report +} +#[test] +fn exact_ninety_percent_coverage_and_sensitivity_and_five_percent_false_positives_pass() { + let checks = criteria(&report(2, 9, 1), &[4; 4]).unwrap(); + assert_eq!(checks[0].false_positive_rate, Some(0.05)); + assert_eq!(checks[1].evaluated_cohort_coverage, 0.9); + assert_eq!(checks[1].unconditional_sensitivity, Some(0.9)); + let wire = serde_json::to_value(&checks).unwrap(); + assert_eq!(wire[0]["false_positive_rate"], serde_json::json!(0.05)); + assert_eq!(wire[1]["evaluated_cohort_coverage"], serde_json::json!(0.9)); + assert_eq!(wire[1]["unconditional_sensitivity"], serde_json::json!(0.9)); + let json = serde_json::to_string(&wire).unwrap(); + assert!(!json.contains("0.899999") && !json.contains("0.050000000")); +} +#[test] +fn below_threshold_coverage_sensitivity_and_excess_false_positives_fail_separately() { + assert!(criteria(&report(2, 8, 2), &[4; 4]) + .unwrap_err() + .contains("coverage")); + assert!(criteria(&report(2, 8, 0), &[4; 4]) + .unwrap_err() + .contains("sensitivity")); + assert!(criteria(&report(3, 9, 1), &[4; 4]) + .unwrap_err() + .contains("false presence")); +} +#[test] +fn policy_is_canonical_through_json_value_and_direct_hash_encoding() { + let p = QualificationPolicy::default(); + let value = serde_json::to_value(&p).unwrap(); + assert_eq!( + value["minimum_unconditional_sensitivity"], + serde_json::json!(0.9) + ); + assert_eq!( + value["minimum_evaluated_cohort_coverage"], + serde_json::json!(0.9) + ); + assert_eq!( + value["maximum_false_positive_rate"], + serde_json::json!(0.05) + ); + let encoded = serde_json::to_string(&p).unwrap(); + let expected = r#"{"id":"ruview.local-operational-policy.v1","minimum_cohorts_per_condition":4,"minimum_unconditional_sensitivity":0.9,"maximum_false_positive_rate":0.05,"minimum_evaluated_cohort_coverage":0.9,"coverage_scope":"representative_cohorts_not_wall_time"}"#; + assert_eq!(encoded, expected); + assert_eq!( + hash(&p).unwrap(), + format!("{:x}", Sha256::digest(expected.as_bytes())) + ); + assert_eq!( + serde_json::from_value::(value).unwrap(), + p + ); +} From 7aceef9e33e6894562735d83833b87d1a0e1391f Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Tue, 3 Nov 2026 16:14:20 +0100 Subject: [PATCH 053/100] fix(learner): withhold selected frames from legacy MQTT bridge --- .../wifi-densepose-sensing-server/src/main.rs | 134 ++++++++++-------- .../src/room_learner_api.rs | 2 + .../src/room_learner_api_publication.rs | 40 ++++++ .../src/room_learner_api_selection.rs | 9 +- ...om_learner_api_selection_composer_tests.rs | 4 +- .../src/room_learner_api_selection_fixture.rs | 13 +- 6 files changed, 138 insertions(+), 64 deletions(-) diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index 8b667cf270..7f8f56d6e7 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -15746,65 +15746,6 @@ async fn main() { ); } - // #872: actually start the MQTT publisher when `--mqtt` is set. The publisher - // (mqtt::) consumes a typed VitalsSnapshot stream; we bridge the existing JSON - // sensing broadcast into it with a defensive serde_json::Value mapping (absent - // fields default — never publish wrong values). Gated on the `mqtt` feature - // (the Docker image is built `--features mqtt`); without it `--mqtt` WARNs and - // no-ops, matching the documented contract. - if args.mqtt_opts.mqtt { - #[cfg(feature = "mqtt")] - { - use wifi_densepose_sensing_server::mqtt; - let mut mcfg = mqtt::config::MqttConfig::from_args_with_data_dir( - &args.mqtt_opts, - &data_dir, - ); - mcfg.privacy_mode = args.privacy_mode; - let mcfg = std::sync::Arc::new(mcfg); - match mcfg.validate() { - Ok(()) => { - let node_id = mcfg.client_id.clone(); - let builder = mqtt::publisher::OwnedDiscoveryBuilder { - discovery_prefix: mcfg.discovery_prefix.clone(), - node_id: node_id.clone(), - node_friendly_name: Some("RuView".to_string()), - sw_version: env!("CARGO_PKG_VERSION").to_string(), - model: "RuView WiFi Sensing".to_string(), - via_device: None, - }; - let (vtx, vrx) = broadcast::channel::(64); - let (host, port) = (mcfg.host.clone(), mcfg.port); - mqtt::publisher::spawn(mcfg, builder, vrx); - let mut jrx = tx.subscribe(); - tokio::spawn(async move { - // #898/#872/#2085: one snapshot per physical node from - // each `sensing_update`; `edge_vitals` feeds the fall - // detector; other message types are ignored. - let mut bridge = mqtt::bridge::SensingBridge::new(node_id); - let started = std::time::Instant::now(); - while let Ok(json) = jrx.recv().await { - let Ok(v) = serde_json::from_str::(&json) else { - continue; - }; - for snap in bridge.ingest(&v, started.elapsed()) { - let _ = vtx.send(snap); - } - } - }); - tracing::info!("MQTT publisher started -> {host}:{port}"); - } - Err(e) => tracing::error!("MQTT config invalid: {e}; publisher not started"), - } - } - #[cfg(not(feature = "mqtt"))] - tracing::warn!( - "--mqtt set but this binary was built without the `mqtt` feature; the publisher is a \ - no-op. Use the official Docker image (built `--features mqtt`) or rebuild with \ - `cargo build -p wifi-densepose-sensing-server --features mqtt`." - ); - } - // ADR-262 P3: build the live RuField surface (dedicated ed25519 signer from // WDP_RUFIELD_SIGNING_SEED, else a logged dev default). The same Arc is // stored in AppStateInner (so the sensing loop can `emit()` per cycle) and @@ -16040,6 +15981,81 @@ async fn main() { .expect("default pose physics configuration is valid"), })); + // #872: actually start the MQTT publisher when `--mqtt` is set. The publisher + // (mqtt::) consumes a typed VitalsSnapshot stream; we bridge the existing JSON + // sensing broadcast into it with a defensive serde_json::Value mapping (absent + // fields default — never publish wrong values). Gated on the `mqtt` feature + // (the Docker image is built `--features mqtt`); without it `--mqtt` WARNs and + // no-ops, matching the documented contract. + // Selected models have no MQTT authority in this version. This admission + // guard does not retract a snapshot already queued inside the publisher or + // a broker-retained legacy value; known nodes retain the existing 10-second + // stale threshold / 30-second availability heartbeat. Managed native helpers do + // not enable MQTT. Network-time withdrawal needs a separate sink contract. + if args.mqtt_opts.mqtt { + #[cfg(feature = "mqtt")] + { + use wifi_densepose_sensing_server::mqtt; + let mut mcfg = mqtt::config::MqttConfig::from_args_with_data_dir( + &args.mqtt_opts, + &data_dir, + ); + mcfg.privacy_mode = args.privacy_mode; + let mcfg = std::sync::Arc::new(mcfg); + match mcfg.validate() { + Ok(()) => { + let node_id = mcfg.client_id.clone(); + let builder = mqtt::publisher::OwnedDiscoveryBuilder { + discovery_prefix: mcfg.discovery_prefix.clone(), + node_id: node_id.clone(), + node_friendly_name: Some("RuView".to_string()), + sw_version: env!("CARGO_PKG_VERSION").to_string(), + model: "RuView WiFi Sensing".to_string(), + via_device: None, + }; + let (vtx, vrx) = broadcast::channel::(64); + let (host, port) = (mcfg.host.clone(), mcfg.port); + mqtt::publisher::spawn(mcfg, builder, vrx); + let mut jrx = state.read().await.tx.subscribe(); + let mqtt_state = Arc::downgrade(&state); + tokio::spawn(async move { + // #898/#872/#2085: one snapshot per physical node from + // each `sensing_update`; `edge_vitals` feeds the fall + // detector; other message types are ignored. + let mut bridge = mqtt::bridge::SensingBridge::new(node_id.clone()); + let started = std::time::Instant::now(); + while let Ok(json) = jrx.recv().await { + let Ok(v) = serde_json::from_str::(&json) else { + continue; + }; + let Some(live_state) = mqtt_state.upgrade() else { break; }; + let current = live_state.read().await; + if !room_learner_api::legacy_sink_frame_allowed( + room_learner_api::selection_owns_output(¤t), &v, + ) { + // Do not replay pending legacy fall/motion state when + // the operator later explicitly returns to control. + bridge = mqtt::bridge::SensingBridge::new(node_id.clone()); + continue; + } + for snap in bridge.ingest(&v, started.elapsed()) { + let _ = vtx.send(snap); + } + } + }); + tracing::info!("MQTT publisher started -> {host}:{port}"); + } + Err(e) => tracing::error!("MQTT config invalid: {e}; publisher not started"), + } + } + #[cfg(not(feature = "mqtt"))] + tracing::warn!( + "--mqtt set but this binary was built without the `mqtt` feature; the publisher is a \ + no-op. Use the official Docker image (built `--features mqtt`) or rebuild with \ + `cargo build -p wifi-densepose-sensing-server --features mqtt`." + ); + } + { let mut s = state.write().await; match s.restore_saved_room() { diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs index 2967bf0499..9698e563be 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs @@ -444,6 +444,8 @@ mod tests; #[path = "room_learner_api_publication.rs"] mod publication; +#[cfg(feature = "mqtt")] +pub(super) use publication::legacy_sink_frame_allowed; pub(super) use publication::{project_cached_selection, project_live_selection, refresh_queued_update, selection_owns_output}; #[path = "room_learner_api_selection.rs"] diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_publication.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_publication.rs index 626b074be9..15fdbfd403 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_publication.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_publication.rs @@ -179,3 +179,43 @@ pub(crate) fn refresh_queued_update(state: &AppStateInner, json: &str) -> Option _ => Some(json.to_owned()), } } + +/// Legacy MQTT/automation bridges cannot reinterpret selected-model frames, +/// even after deselection or when an older unmarked frame was queued before +/// selection. This is an admission guard, not new MQTT presence authority. +#[cfg(any(feature = "mqtt", test))] +pub(crate) fn legacy_sink_frame_allowed(selection_owned: bool, frame: &serde_json::Value) -> bool { + !selection_owned + && frame.get("selected_binary_evidence").is_none() + && frame.get("classification_source").is_none() +} + +#[cfg(test)] +mod legacy_sink_tests { + use super::*; + #[test] + fn selected_current_state_and_queued_markers_both_withhold_legacy_bridges() { + let legacy = + serde_json::json!({"type":"sensing_update","classification":{"presence":true}}); + assert!(legacy_sink_frame_allowed(false, &legacy)); + assert!( + !legacy_sink_frame_allowed(true, &legacy), + "pre-selection queued legacy data must be withheld after selection" + ); + for status in ["ready", "unavailable", "none"] { + let mut selected = legacy.clone(); + selected["selected_binary_evidence"] = serde_json::json!({"status":status}); + assert!( + !legacy_sink_frame_allowed(false, &selected), + "old selected frame must not regain legacy authority after deselection" + ); + assert!(!legacy_sink_frame_allowed(true, &selected)); + } + let mut malformed = legacy.clone(); + malformed["selected_binary_evidence"] = serde_json::Value::Null; + assert!(!legacy_sink_frame_allowed(false, &malformed)); + let mut comparison = legacy; + comparison["classification_source"] = serde_json::json!("legacy_control_comparison"); + assert!(!legacy_sink_frame_allowed(false, &comparison)); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs index bfef0f7ab6..49f41cd2e9 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs @@ -81,7 +81,8 @@ pub(crate) async fn qualify( Ok(rt) => rt, Err(e) => return e, }; - let state = state.read().await; + // Serialize authority transitions with live producers and legacy sink admission. + let state = state.write().await; match with_current_room(&state, &rt, || rt.qualify(&request.expected)) { Ok(receipt) => Json( serde_json::json!({"success":true,"qualification":receipt,"status":enriched_status(&state, &rt)}), @@ -98,7 +99,8 @@ pub(crate) async fn select_model( Ok(rt) => rt, Err(e) => return e, }; - let state = state.read().await; + // Serialize authority transitions with live producers and legacy sink admission. + let state = state.write().await; match with_current_room(&state, &rt, || { rt.select_model( &request.expected, @@ -121,7 +123,8 @@ pub(crate) async fn deselect_model( Ok(rt) => rt, Err(e) => return e, }; - let state = state.read().await; + // Serialize authority transitions with live producers and legacy sink admission. + let state = state.write().await; match rt.deselect_model(&request.expected) { Ok(receipt) => Json( serde_json::json!({"success":true,"selection":receipt,"status":enriched_status(&state, &rt)}), diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs index 3e7f5e2bc7..debcd64ea6 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs @@ -288,7 +288,7 @@ async fn export_actual_selected_contract_cases() { project_live_selection(&*state.read().await, &mut sensing); let response = Router::new() .route("/api/v1/room-learner/selection", get(selection)) - .with_state(state) + .with_state(state.clone()) .oneshot( Request::builder() .uri("/api/v1/room-learner/selection") @@ -302,7 +302,9 @@ async fn export_actual_selected_contract_cases() { .await .unwrap(); let paired_selection: serde_json::Value = serde_json::from_slice(&body).unwrap(); + let owner_status = status(State(state)).await.0; serde_json::json!({ + "owner_status": owner_status, "name": name, "captured_at_unix_ms": rt.receiver_now_ns() / 1_000_000, "sensing_update": sensing, diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_fixture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_fixture.rs index 5c81533ffb..fda3e8579e 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_fixture.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_fixture.rs @@ -151,6 +151,13 @@ pub(super) async fn fitted() -> ( .await .unwrap(); assert_eq!(response.status(), StatusCode::OK); + let published: serde_json::Value = serde_json::from_slice( + &axum::body::to_bytes(response.into_body(), 65536) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!(published["revision"], 1); let binding = room.current_binding_digest().unwrap().unwrap(); state.write().await.current_room_store = Some(room); let configured = json_request( @@ -158,10 +165,14 @@ pub(super) async fn fitted() -> ( "/configure", Method::POST, serde_json::json!({ - "binding_digest":binding,"revision":1,"source_node_ids":[15,16],"new_training":true}), + "binding_digest":binding,"revision":7001,"source_node_ids":[15,16],"new_training":true}), ) .await; assert_eq!(configured["success"], true, "{configured}"); + // Learner config generation and companion publication revision are separate + // domains. Only the full shared binding digest joins their authority. + assert_eq!(configured["status"]["revision"], 7001); + let rt = state.read().await.room_learning.as_ref().unwrap().clone(); let grid = rt.context().unwrap().sources[0].grid_id.clone(); let base = rt.receiver_now_ns() - 200_000_000_000; From 8c97f27b1df72c7da8dadf545968634ef8915443 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Tue, 3 Nov 2026 16:56:48 +0100 Subject: [PATCH 054/100] fix(learner): withhold selection when its pointer is lost --- .../lifecycle_assessment_tests.rs | 153 ++++++++++++++++++ .../src/room_learner/selection_store.rs | 41 +++++ 2 files changed, 194 insertions(+) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_assessment_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_assessment_tests.rs index a55d808d7d..be27c749c2 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_assessment_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_assessment_tests.rs @@ -240,3 +240,156 @@ async fn canonical_56_feature_context_qualifies_through_actual_managed_training_ SelectedBinaryEvidence::Ready { .. } )); } + +#[tokio::test] +async fn missing_authoritative_pointer_with_history_is_unknown_not_legacy_after_cold_load() { + let dir = tempfile::tempdir().unwrap(); + let rt = fitted(dir.path()).await; + let q = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + let selected = rt + .select_model( + &rt.mutation_guard().unwrap(), + &q.receipt.model_sha256, + &q.receipt_sha256, + ) + .unwrap(); + let root = dir + .path() + .join("a".repeat(64)) + .join("SensingServer/room-learner"); + let pointer = root.join("model-selection-v1.json"); + let original = std::fs::read(&pointer).unwrap(); + let history = root + .join("selections") + .join(format!("{}.json", selected.receipt_sha256)); + let history_bytes = std::fs::read(&history).unwrap(); + std::fs::rename(&pointer, root.join("selection-pointer-backup.json")).unwrap(); + let cold = runtime(dir.path()); + let status = cold.selection_snapshot(cold.receiver_now_ns(), Some("synthetic-room")); + assert!(status + .integrity_error + .as_deref() + .is_some_and(|s| s.contains("history"))); + assert!(matches!( + status.current, + SelectedBinaryEvidence::Unavailable { .. } + )); + assert!(!pointer.exists()); + assert_eq!(std::fs::read(&history).unwrap(), history_bytes); + assert_eq!( + std::fs::read(root.join("selection-pointer-backup.json")).unwrap(), + original + ); + // An actual explicit deselection pointer, unlike a missing pointer, proves legacy mode. + std::fs::rename(root.join("selection-pointer-backup.json"), &pointer).unwrap(); + let restored = runtime(dir.path()); + restored + .deselect_model(&restored.mutation_guard().unwrap()) + .unwrap(); + let deselected = runtime(dir.path()); + let status = + deselected.selection_snapshot(deselected.receiver_now_ns(), Some("synthetic-room")); + assert!(status.integrity_error.is_none()); + assert!(matches!(status.current, SelectedBinaryEvidence::None)); +} + +#[tokio::test] +async fn orphan_first_write_history_does_not_invent_a_selected_model_or_legacy_fallback() { + use crate::room_learner::{ + qualification::hash, + selection::{SelectionEnvelope, SelectionReceipt, SelectionTarget}, + }; + let dir = tempfile::tempdir().unwrap(); + let rt = fitted(dir.path()).await; + let q = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + let root = dir + .path() + .join("a".repeat(64)) + .join("SensingServer/room-learner"); + let history = root.join("selections"); + crate::companion::private_dir(&history).unwrap(); + // Simulate the persisted boundary after immutable history but before the + // first authoritative pointer. No successful selection is asserted. + let receipt = SelectionReceipt { + schema: "ruview.room-learner.model-selection.v1".into(), + owner_scope: "a".repeat(64), + installation_id: "synthetic-install".into(), + revision: 1, + selected_at_unix_ms: rt.receiver_now_ns() / 1_000_000, + selected: Some(SelectionTarget { + context: context(), + model_sha256: q.receipt.model_sha256.clone(), + model_version: q.receipt.model_version, + qualification_sha256: q.receipt_sha256.clone(), + }), + previous: None, + }; + let envelope = SelectionEnvelope { + receipt_sha256: hash(&receipt).unwrap(), + receipt, + }; + let bytes = serde_json::to_vec(&envelope).unwrap(); + let record = history.join(format!("{}.json", envelope.receipt_sha256)); + crate::companion::atomic_private_write(&record, &bytes).unwrap(); + let pointer = root.join("model-selection-v1.json"); + std::fs::create_dir(&pointer).unwrap(); + assert!(crate::companion::atomic_private_write(&pointer, &bytes).is_err()); + std::fs::remove_dir(&pointer).unwrap(); + let cold = runtime(dir.path()); + let status = cold.selection_snapshot(cold.receiver_now_ns(), Some("synthetic-room")); + assert!(status.selected.is_none()); + assert!(status.integrity_error.is_some()); + assert!(matches!( + status.current, + SelectedBinaryEvidence::Unavailable { .. } + )); + assert!(!pointer.exists()); + assert_eq!(std::fs::read(record).unwrap(), bytes); +} + +#[cfg(unix)] +#[test] +fn never_selected_empty_history_is_legacy_but_unsafe_history_is_unknown() { + use std::os::unix::fs::{symlink, PermissionsExt}; + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + assert!(matches!( + rt.selection_snapshot(START, Some("synthetic-room")).current, + SelectedBinaryEvidence::None + )); + let root = dir + .path() + .join("a".repeat(64)) + .join("SensingServer/room-learner"); + let history = root.join("selections"); + crate::companion::private_dir(&history).unwrap(); + assert!(matches!( + runtime(dir.path()) + .selection_snapshot(START, Some("synthetic-room")) + .current, + SelectedBinaryEvidence::None + )); + std::fs::set_permissions(&history, std::fs::Permissions::from_mode(0o000)).unwrap(); + assert!(matches!( + runtime(dir.path()) + .selection_snapshot(START, Some("synthetic-room")) + .current, + SelectedBinaryEvidence::Unavailable { .. } + )); + std::fs::set_permissions(&history, std::fs::Permissions::from_mode(0o700)).unwrap(); + std::fs::remove_dir(&history).unwrap(); + let external = dir.path().join("empty-history-target"); + crate::companion::private_dir(&external).unwrap(); + symlink(&external, &history).unwrap(); + assert!(matches!( + runtime(dir.path()) + .selection_snapshot(START, Some("synthetic-room")) + .current, + SelectedBinaryEvidence::Unavailable { .. } + )); + assert!(std::fs::symlink_metadata(&history) + .unwrap() + .file_type() + .is_symlink()); + assert_eq!(std::fs::read_dir(external).unwrap().count(), 0); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection_store.rs index c762aaf20e..9b0518d98b 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection_store.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection_store.rs @@ -125,10 +125,51 @@ impl ArtifactStore { } Ok((model, q)) } + fn verify_no_selection_history(&self) -> Result<(), String> { + let history = self.root.join("selections"); + let metadata = match std::fs::symlink_metadata(&history) { + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(_) => { + return Err( + "selection history cannot be inspected while its pointer is missing".into(), + ) + } + Ok(m) => m, + }; + if !metadata.is_dir() || metadata.file_type().is_symlink() { + return Err("selection history is unsafe while its pointer is missing".into()); + } + #[cfg(unix)] + { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + unsafe extern "C" { + fn geteuid() -> u32; + } + // SAFETY: POSIX geteuid has no arguments or memory side effects. + let uid = unsafe { geteuid() }; + if metadata.permissions().mode() & 0o777 != 0o700 || metadata.uid() != uid { + return Err( + "selection history is not privately readable while its pointer is missing" + .into(), + ); + } + } + let mut entries = std::fs::read_dir(&history) + .map_err(|_| "selection history cannot be read while its pointer is missing")?; + match entries.next() { + None=>Ok(()), + Some(Ok(_))=>Err("selection pointer is missing but retained selection history exists; explicit recovery is required".into()), + Some(Err(_))=>Err("selection history cannot be read while its pointer is missing".into()), + } + } pub(crate) fn load_selection(&self) -> Result, String> { self.verify_owner()?; let path = self.root.join(SELECTION); if !regular_or_missing(&path)? { + // Absence is only known when no selection commit was attempted. + // Immutable history may outlive a lost pointer or a failed first + // pointer write. Never guess a replacement from that history. + self.verify_no_selection_history()?; return Ok(None); } let envelope: SelectionEnvelope = From f160b0795647e56a0036865d786a2729f24e55f3 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Tue, 3 Nov 2026 16:39:55 +0100 Subject: [PATCH 055/100] test(learner): preserve offline owner recovery controls --- .../src/companion_integration.rs | 2 +- ...room_learner_api_offline_metadata_tests.rs | 205 ++++++++++++++++++ ...om_learner_api_selection_composer_tests.rs | 3 + .../src/room_learner_api_selection_fixture.rs | 2 +- 4 files changed, 210 insertions(+), 2 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner_api_offline_metadata_tests.rs diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs index 67084e8025..de2aacf3c1 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs @@ -387,7 +387,7 @@ mod tests { assert_eq!(policy(&Method::GET, "/api/v1/room-learner/selection"), Some(false)); assert_eq!(policy(&Method::HEAD, "/api/v1/room-learner/selection"), Some(false)); assert_eq!(policy(&Method::GET, "/api/v1/room-learner/status"), None); - for path in ["/api/v1/room-learner/qualify", "/api/v1/room-learner/select", "/api/v1/room-learner/deselect"] { + for path in ["/api/v1/room-learner/qualify", "/api/v1/room-learner/select", "/api/v1/room-learner/deselect", "/api/v1/room-learner/phase/cancel", "/api/v1/room-learner/train/cancel"] { assert_eq!(policy(&Method::POST, path), None, "paired read/calibration grants cannot select: {path}"); } diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_offline_metadata_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_offline_metadata_tests.rs new file mode 100644 index 0000000000..2126e54714 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_offline_metadata_tests.rs @@ -0,0 +1,205 @@ +//! Offline RF is not loss of authenticated owner metadata authority. +//! Synthetic temporary data only; no device or broker operations. +use super::*; + +async fn cancel_via_authenticated_router( + state: SharedState, + path: &str, + guard: MutationGuard, + token: &str, +) -> (StatusCode, serde_json::Value) { + let router = Router::new() + .route("/api/v1/room-learner/phase/cancel", post(cancel_phase)) + .route("/api/v1/room-learner/train/cancel", post(cancel_training)) + .with_state(state) + .layer(axum::middleware::from_fn_with_state( + wifi_densepose_sensing_server::bearer_auth::AuthState::from_token("synthetic-owner"), + wifi_densepose_sensing_server::bearer_auth::require_bearer, + )); + let response = router + .oneshot( + Request::builder() + .method(Method::POST) + .uri(path) + .header("content-type", "application/json") + .header("authorization", format!("Bearer {token}")) + .body(Body::from( + serde_json::to_vec(&serde_json::json!({"expected":guard})).unwrap(), + )) + .unwrap(), + ) + .await + .unwrap(); + let code = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), 65536) + .await + .unwrap(); + ( + code, + serde_json::from_slice(&bytes).unwrap_or(serde_json::Value::Null), + ) +} + +#[tokio::test] +async fn offline_owner_can_cancel_then_deselect_but_cannot_reset_corrupt_pointer() { + let (dir, state, rt, _binding) = fitted().await; + let started = begin_phase( + State(state.clone()), + Json(BeginRequest { + expected: rt.mutation_guard().unwrap(), + phase: PhaseRequest { + trial_id: "synthetic-offline-cancel".into(), + session_id: "synthetic-offline-session".into(), + role: DataRole::HeldOut, + label: RoomLabel::Empty, + duration_ms: 12000, + lead_in_ms: 0, + conditions_confirmed: true, + }, + }), + ) + .await + .0; + assert_eq!(started["success"], true, "{started}"); + assert!(rt.capture_active().unwrap()); + // Readiness requires a native arrival within two seconds. Let the actual + // observed metadata become stale; do not replace its clock or forge flags. + tokio::time::sleep(std::time::Duration::from_millis(2100)).await; + { + let mut inner = state.write().await; + inner.last_esp32_frame = None; + assert!(rt.context().unwrap().sources.iter().all(|s| !source_ready( + &inner, + s, + std::time::Instant::now() + ))); + } + let grid = rt.context().unwrap().sources[0].grid_id.clone(); + rt.record_native_rejection(16, &grid, "synthetic RF interruption"); + let metadata = status(State(state.clone())).await.0; + assert_eq!(metadata["enabled"], true); + assert!(metadata["mutation_guard"].is_object()); + assert!(metadata["model_selection"]["selected"].is_object()); + assert_eq!( + metadata["model_selection"]["current"]["status"], + "unavailable" + ); + let guard = rt.mutation_guard().unwrap(); + let (code, _) = cancel_via_authenticated_router( + state.clone(), + "/api/v1/room-learner/phase/cancel", + guard.clone(), + "synthetic-peer", + ) + .await; + assert_ne!( + code, + StatusCode::OK, + "loss of RF must not broaden actor rights" + ); + assert!(rt.capture_active().unwrap()); + let (code, cancelled) = cancel_via_authenticated_router( + state.clone(), + "/api/v1/room-learner/phase/cancel", + guard, + "synthetic-owner", + ) + .await; + assert_eq!(code, StatusCode::OK); + assert_eq!(cancelled["success"], true, "{cancelled}"); + assert!(!rt.capture_active().unwrap()); + let known = status(State(state.clone())).await.0; + let deselected = owner_post( + state.clone(), + "/api/v1/room-learner/deselect", + serde_json::json!({"expected":known["mutation_guard"]}), + ) + .await; + assert_eq!(deselected["success"], true, "{deselected}"); + assert!(deselected["selection"]["receipt"]["selected"].is_null()); + assert_eq!(deselected["selection"]["receipt"]["revision"], 2); + + // Unknown disk state is different from absent RF. Do not guess a pointer + // revision or overwrite the file merely to acknowledge a metadata action. + let pointer = dir + .path() + .join("a".repeat(64)) + .join("SensingServer/room-learner/model-selection-v1.json"); + assert!(pointer.is_file()); + let corrupt = b"{ synthetic corrupt selection pointer"; + std::fs::write(&pointer, corrupt).unwrap(); + let refused = owner_post( + state.clone(), + "/api/v1/room-learner/deselect", + serde_json::json!({"expected":rt.mutation_guard().unwrap()}), + ) + .await; + assert_eq!(refused["success"], false, "{refused}"); + assert_eq!(refused["error_code"], "learner_deselection_rejected"); + assert_eq!(std::fs::read(pointer).unwrap(), corrupt); +} + +#[tokio::test] +async fn authenticated_training_cancel_does_not_require_live_native_ingress() { + let (_dir, state) = fixture(); + let configured=json_request(app(state.clone()),"/configure",Method::POST,serde_json::json!({ + "binding_digest":"b".repeat(64),"revision":7001,"source_node_ids":[15,16],"new_training":true})).await; + assert_eq!(configured["success"], true, "{configured}"); + let rt = state.read().await.room_learning.as_ref().unwrap().clone(); + let grid = rt.context().unwrap().sources[0].grid_id.clone(); + let base = rt.receiver_now_ns() - 100_000_000_000; + for (i, label) in [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ] + .into_iter() + .enumerate() + { + fixture_support::phase(&rt, &grid, DataRole::Train, label, i as u32, base); + } + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 8, + learning_rate: 0.5, + hidden_dim: 2, + embedding_dim: 2, + temporal_tokens: 2, + ..FitConfig::default() + }, + ) + .unwrap(); + state.write().await.room_learning_sources.clear(); + let guard = rt.mutation_guard().unwrap(); + let (code, _) = cancel_via_authenticated_router( + state.clone(), + "/api/v1/room-learner/train/cancel", + guard.clone(), + "synthetic-peer", + ) + .await; + assert_ne!(code, StatusCode::OK); + let (code, body) = cancel_via_authenticated_router( + state.clone(), + "/api/v1/room-learner/train/cancel", + guard, + "synthetic-owner", + ) + .await; + assert_eq!(code, StatusCode::OK); + assert_eq!(body["success"], true, "{body}"); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + loop { + let snapshot = rt.status(rt.receiver_now_ns()); + if snapshot["state"] == "cancelled" { + break; + } + assert!( + std::time::Instant::now() < deadline, + "cancelled training did not settle: {snapshot}" + ); + tokio::time::sleep(std::time::Duration::from_millis(5)).await; + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs index debcd64ea6..6f148d3ee3 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs @@ -376,3 +376,6 @@ async fn export_actual_selected_contract_cases() { println!("RUVIEW_SELECTED_CONTRACT_FILE={}", path.display()); } } + +#[path = "room_learner_api_offline_metadata_tests.rs"] +mod offline_metadata_tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_fixture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_fixture.rs index fda3e8579e..ec75852de6 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_fixture.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_fixture.rs @@ -52,7 +52,7 @@ pub(super) fn feed(rt: &RoomLearningRuntime, grid: &str, label: RoomLabel, seque } } } -fn phase( +pub(super) fn phase( rt: &RoomLearningRuntime, grid: &str, role: DataRole, From 05efb454a814c14799f5a822580b1ca238515e79 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Tue, 3 Nov 2026 19:04:44 +0100 Subject: [PATCH 056/100] fix(calibration): retain verified expired reference diagnostics after restart --- .../src/bootstrap_baseline.rs | 62 ++++++++++++++++--- .../wifi-densepose-sensing-server/src/main.rs | 56 ++++++++++++++++- 2 files changed, 110 insertions(+), 8 deletions(-) diff --git a/v2/crates/wifi-densepose-sensing-server/src/bootstrap_baseline.rs b/v2/crates/wifi-densepose-sensing-server/src/bootstrap_baseline.rs index ec0450aba9..b22b789e35 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/bootstrap_baseline.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/bootstrap_baseline.rs @@ -315,6 +315,37 @@ pub fn load( installation_id: &str, current_unix_ms: u64, ) -> Result<(FieldModel, BootstrapBaselineMetadata), BootstrapBaselineError> { + let image = read_image(path, installation_id, current_unix_ms)?; + let model = FieldModel::from_snapshot( + image.payload.field_model.clone(), + current_unix_ms.saturating_mul(1_000), + )?; + Ok((model, metadata(&image))) +} + +/// Read historical expiry diagnostics without restoring an expired model. +/// Identity, size, schema and exact stored-byte integrity checks still apply. +/// This returns metadata only and must never confer inference authority. +pub fn load_expired_metadata( + path: &Path, + installation_id: &str, + current_unix_ms: u64, +) -> Result { + // Zero skips only the deadline check; all other payload checks are shared + // with the model loader. A newly replaced, unexpired file is not accepted + // through this diagnostics-only path. + let image = read_image(path, installation_id, 0)?; + if current_unix_ms <= image.payload.expires_at_unix_ms { + return Err(BootstrapBaselineError::Malformed("reference is not expired".into())); + } + Ok(metadata(&image)) +} + +fn read_image( + path: &Path, + installation_id: &str, + current_unix_ms: u64, +) -> Result { let file_metadata = fs::symlink_metadata(path)?; if file_metadata.file_type().is_symlink() || !file_metadata.is_file() { return Err(BootstrapBaselineError::InvalidPath); @@ -340,15 +371,10 @@ pub fn load( } let payload: BootstrapPayloadV2 = serde_json::from_str(raw_image.payload.get())?; validate_payload(&payload, installation_id, current_unix_ms)?; - let image = BootstrapImageV2 { + Ok(BootstrapImageV2 { payload, content_sha256: raw_image.content_sha256, - }; - let model = FieldModel::from_snapshot( - image.payload.field_model.clone(), - current_unix_ms.saturating_mul(1_000), - )?; - Ok((model, metadata(&image))) + }) } pub fn remove(path: &Path) -> Result { @@ -501,6 +527,28 @@ mod tests { )); } + #[test] + fn expired_metadata_is_historical_only_and_retains_original_lifetime() { + let dir = tempfile::tempdir().unwrap(); + let path = path_in(dir.path()); + let now_ms = 1_000_000; + let stored = store(&path, "home-a", &[5], TEST_GRID, "model", now_ms, + &completed_model(now_ms * 1_000)).unwrap(); + let original = fs::read(&path).unwrap(); + let expired_at = stored.expires_at_unix_ms + 1; + assert!(matches!(load(&path, "home-a", expired_at), Err(BootstrapBaselineError::Expired))); + assert_eq!(load_expired_metadata(&path, "home-a", expired_at).unwrap(), stored); + assert_eq!(fs::read(&path).unwrap(), original, "diagnostics must not renew or rewrite the reference"); + assert!(load_expired_metadata(&path, "home-a", stored.expires_at_unix_ms).is_err()); + assert!(matches!(load_expired_metadata(&path, "home-b", expired_at), + Err(BootstrapBaselineError::InstallationMismatch))); + let mut image: serde_json::Value = serde_json::from_slice(&original).unwrap(); + image["payload"]["expires_at_unix_ms"] = serde_json::json!(expired_at + 99_999); + fs::write(&path, serde_json::to_vec(&image).unwrap()).unwrap(); + assert!(matches!(load_expired_metadata(&path, "home-a", expired_at), + Err(BootstrapBaselineError::DigestMismatch))); + } + #[test] fn tamper_and_expiry_fail_closed() { let dir = tempfile::tempdir().unwrap(); diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index 7f8f56d6e7..a156bb000a 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -2576,6 +2576,9 @@ fn calibration_sequence_order(previous: Option, sequence: u32) -> Calibrati impl AppStateInner { fn restore_saved_room(&mut self) -> Result { + // Historical expiry metadata is retained for diagnostics after restart; + // it is not a restored model and cannot restore a full-room receipt. + if self.field_model.is_none() { return Ok(false); } let Some((installation, metadata)) = self.installation_id.as_deref().zip(self.bootstrap_baseline.as_ref()) else { return Ok(false); }; match room_calibration::load(&room_calibration::path_in(&self.data_dir), installation, metadata) { Ok(receipt) => { @@ -3426,6 +3429,29 @@ mod calibration_expiry_tests { assert!(!restarted.restore_saved_room().unwrap()); } + #[test] + fn expired_metadata_cannot_restore_a_full_room_without_a_model() { + let dir = tempfile::tempdir().unwrap(); + let old = (chrono::Utc::now().timestamp_millis().max(0) as u64) - 1_000_000_000; + let source = qualified_vital_state(old); + let receipt = source.calibration_model_receipt.as_ref().unwrap(); + let path = bootstrap_baseline::path_in(dir.path()); + let metadata = bootstrap_baseline::store(&path, "account-A-installation", &[5], + source.calibration_grid_binding.unwrap().grid.as_bootstrap_grid(), &receipt.model_id, old, + source.field_model.as_ref().unwrap()).unwrap(); + room_calibration::store(&room_calibration::path_in(dir.path()), "account-A-installation", &metadata, receipt).unwrap(); + let mut restarted = AppStateInner::minimal(); + restarted.data_dir = dir.path().to_path_buf(); + restarted.installation_id = Some("account-A-installation".into()); + restarted.bootstrap_baseline = Some(bootstrap_baseline::load_expired_metadata(&path, + "account-A-installation", metadata.expires_at_unix_ms + 1).unwrap()); + assert!(!restarted.restore_saved_room().unwrap()); + assert!(restarted.field_model.is_none()); + assert!(restarted.calibration_model_receipt.is_none()); + assert!(restarted.calibration_session_id.is_none()); + assert!(!restarted.bootstrap_baseline_active_at(metadata.expires_at_unix_ms + 1)); + } + #[test] fn restored_room_requires_new_boot_source_evidence_and_never_renews_completion() { let mut state = state_with_runtime_window(); @@ -3817,6 +3843,24 @@ mod calibration_expiry_tests { ); } + #[tokio::test] + async fn expired_metadata_after_restart_reports_original_deadline_without_authority() { + let mut state = state_with_model(true); + let expected_expiry = state.bootstrap_baseline.as_ref().unwrap().expires_at_unix_ms; + state.field_model = None; + state.bootstrap_baseline_active = false; + let state = Arc::new(RwLock::new(state)); + let Json(status) = calibration_status(State(state)).await; + assert_eq!(status["status"], "expired"); + assert_eq!(status["active"], false); + assert_eq!(status["binding_mode"], "none"); + assert_eq!(status["bootstrap_baseline"]["expires_at_unix_ms"], expected_expiry); + assert_eq!(status["bootstrap_baseline"]["active"], false); + assert_eq!(status["bootstrap_baseline"]["calibrated_evidence_authorized"], false); + assert_eq!(status["bootstrap_baseline"]["numeric_vitals_authorized"], false); + assert!(status["model_receipt"].is_null()); + } + #[tokio::test] async fn long_running_expiry_is_not_reported_active_for_runtime_or_bootstrap() { let runtime = Arc::new(RwLock::new(state_with_model(false))); @@ -10847,7 +10891,9 @@ async fn calibration_status(State(state): State) -> Json metadata.expires_at_unix_ms) + { "expired".to_string() } else { "none".to_string() }, |status| format!("{status:?}").to_lowercase(), ); // An expired calibration is inactive by definition, but "expired" is a @@ -15697,6 +15743,14 @@ async fn main() { { (None, None, false) } + Err(bootstrap_baseline::BootstrapBaselineError::Expired) => { + let metadata = bootstrap_baseline::load_expired_metadata( + &path, installation_id, chrono::Utc::now().timestamp_millis().max(0) as u64, + ).ok(); + // Keep original timestamps for honest status only. + // No FieldModel, active prior or authority is restored. + (None, metadata, false) + } Err(error) => { warn!(%error, "Ignored invalid empty room bootstrap prior"); (None, None, false) From cc760e84428a88ee0df72882d8710621b7e4450c Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Tue, 3 Nov 2026 19:23:52 +0100 Subject: [PATCH 057/100] fix(deps): preview ZIP allocation backport on main Wasmtime security base --- .github/workflows/security-scan.yml | 3 + v2/Cargo.lock | 50 - v2/Cargo.toml | 6 +- .../zip-7.2.0/.amazonq/rules/rules.md | 14 + .../security/zip-7.2.0/.cargo_vcs_info.json | 6 + v2/vendor/security/zip-7.2.0/.gitattributes | 8 + v2/vendor/security/zip-7.2.0/.gitignore | 6 + v2/vendor/security/zip-7.2.0/.whitesource | 14 + v2/vendor/security/zip-7.2.0/CHANGELOG.md | 860 ++++ .../security/zip-7.2.0/CODE_OF_CONDUCT.md | 76 + v2/vendor/security/zip-7.2.0/CONTRIBUTING.md | 1 + v2/vendor/security/zip-7.2.0/Cargo.lock | 1300 ++++++ v2/vendor/security/zip-7.2.0/Cargo.toml | 330 ++ v2/vendor/security/zip-7.2.0/Cargo.toml.orig | 139 + v2/vendor/security/zip-7.2.0/LICENSE | 24 + v2/vendor/security/zip-7.2.0/README.md | 145 + .../zip-7.2.0/RUSTSEC-2026-0429.patch | 59 + .../security/zip-7.2.0/RUVIEW-BACKPORT.md | 35 + v2/vendor/security/zip-7.2.0/SECURITY.md | 36 + .../zip-7.2.0/benches/merge_archive.rs | 130 + .../security/zip-7.2.0/benches/read_entry.rs | 41 + .../zip-7.2.0/benches/read_metadata.rs | 143 + v2/vendor/security/zip-7.2.0/cliff.toml | 91 + .../zip-7.2.0/pull_request_template.md | 38 + v2/vendor/security/zip-7.2.0/release-plz.toml | 27 + .../security/zip-7.2.0/ruview-provenance.json | 16 + v2/vendor/security/zip-7.2.0/src/aes.rs | 407 ++ v2/vendor/security/zip-7.2.0/src/aes_ctr.rs | 277 ++ .../security/zip-7.2.0/src/compression.rs | 557 +++ v2/vendor/security/zip-7.2.0/src/cp437.rs | 207 + v2/vendor/security/zip-7.2.0/src/crc32.rs | 131 + .../src/extra_fields/extended_timestamp.rs | 111 + .../zip-7.2.0/src/extra_fields/mod.rs | 35 + .../zip-7.2.0/src/extra_fields/ntfs.rs | 97 + .../src/extra_fields/zipinfo_utf8.rs | 40 + .../security/zip-7.2.0/src/legacy/huffman.rs | 260 ++ .../security/zip-7.2.0/src/legacy/implode.rs | 250 + .../security/zip-7.2.0/src/legacy/mod.rs | 4 + .../security/zip-7.2.0/src/legacy/reduce.rs | 313 ++ .../security/zip-7.2.0/src/legacy/shrink.rs | 373 ++ v2/vendor/security/zip-7.2.0/src/lib.rs | 76 + v2/vendor/security/zip-7.2.0/src/macros.rs | 162 + v2/vendor/security/zip-7.2.0/src/path.rs | 110 + v2/vendor/security/zip-7.2.0/src/read.rs | 2650 +++++++++++ .../security/zip-7.2.0/src/read/config.rs | 23 + .../zip-7.2.0/src/read/magic_finder.rs | 291 ++ .../security/zip-7.2.0/src/read/stream.rs | 415 ++ v2/vendor/security/zip-7.2.0/src/result.rs | 143 + v2/vendor/security/zip-7.2.0/src/spec.rs | 873 ++++ v2/vendor/security/zip-7.2.0/src/types.rs | 1768 +++++++ v2/vendor/security/zip-7.2.0/src/unstable.rs | 121 + v2/vendor/security/zip-7.2.0/src/write.rs | 4077 +++++++++++++++++ v2/vendor/security/zip-7.2.0/src/zipcrypto.rs | 280 ++ .../zip-7.2.0/tests/ruview_0429_backport.rs | 143 + 54 files changed, 17741 insertions(+), 51 deletions(-) create mode 100644 v2/vendor/security/zip-7.2.0/.amazonq/rules/rules.md create mode 100644 v2/vendor/security/zip-7.2.0/.cargo_vcs_info.json create mode 100644 v2/vendor/security/zip-7.2.0/.gitattributes create mode 100644 v2/vendor/security/zip-7.2.0/.gitignore create mode 100644 v2/vendor/security/zip-7.2.0/.whitesource create mode 100644 v2/vendor/security/zip-7.2.0/CHANGELOG.md create mode 100644 v2/vendor/security/zip-7.2.0/CODE_OF_CONDUCT.md create mode 100644 v2/vendor/security/zip-7.2.0/CONTRIBUTING.md create mode 100644 v2/vendor/security/zip-7.2.0/Cargo.lock create mode 100644 v2/vendor/security/zip-7.2.0/Cargo.toml create mode 100644 v2/vendor/security/zip-7.2.0/Cargo.toml.orig create mode 100644 v2/vendor/security/zip-7.2.0/LICENSE create mode 100644 v2/vendor/security/zip-7.2.0/README.md create mode 100644 v2/vendor/security/zip-7.2.0/RUSTSEC-2026-0429.patch create mode 100644 v2/vendor/security/zip-7.2.0/RUVIEW-BACKPORT.md create mode 100644 v2/vendor/security/zip-7.2.0/SECURITY.md create mode 100644 v2/vendor/security/zip-7.2.0/benches/merge_archive.rs create mode 100644 v2/vendor/security/zip-7.2.0/benches/read_entry.rs create mode 100644 v2/vendor/security/zip-7.2.0/benches/read_metadata.rs create mode 100644 v2/vendor/security/zip-7.2.0/cliff.toml create mode 100644 v2/vendor/security/zip-7.2.0/pull_request_template.md create mode 100644 v2/vendor/security/zip-7.2.0/release-plz.toml create mode 100644 v2/vendor/security/zip-7.2.0/ruview-provenance.json create mode 100644 v2/vendor/security/zip-7.2.0/src/aes.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/aes_ctr.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/compression.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/cp437.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/crc32.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/extra_fields/extended_timestamp.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/extra_fields/mod.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/extra_fields/ntfs.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/extra_fields/zipinfo_utf8.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/legacy/huffman.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/legacy/implode.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/legacy/mod.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/legacy/reduce.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/legacy/shrink.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/lib.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/macros.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/path.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/read.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/read/config.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/read/magic_finder.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/read/stream.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/result.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/spec.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/types.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/unstable.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/write.rs create mode 100644 v2/vendor/security/zip-7.2.0/src/zipcrypto.rs create mode 100644 v2/vendor/security/zip-7.2.0/tests/ruview_0429_backport.rs diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 4b406ed775..0921727c90 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -40,6 +40,9 @@ jobs: set -o pipefail cargo audit --file Cargo.lock --json | tee cargo-audit.json + - name: Exercise the vendored ZIP symlink allocation guard + run: cargo test --manifest-path v2/vendor/security/zip-7.2.0/Cargo.toml --locked --no-default-features --features deflate --test ruview_0429_backport + - name: Upload Rust advisory report uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 if: always() diff --git a/v2/Cargo.lock b/v2/Cargo.lock index 39888ea77a..e690ea8a7f 100644 --- a/v2/Cargo.lock +++ b/v2/Cargo.lock @@ -234,9 +234,6 @@ name = "arbitrary" version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" -dependencies = [ - "derive_arbitrary", -] [[package]] name = "arrayref" @@ -2763,17 +2760,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "derive_arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "derive_more" version = "0.99.20" @@ -3396,7 +3382,6 @@ dependencies = [ "crc32fast", "libz-sys", "miniz_oxide", - "zlib-rs", ] [[package]] @@ -6591,7 +6576,6 @@ dependencies = [ "num-complex", "num-traits", "py_literal", - "zip 6.0.0", ] [[package]] @@ -14708,25 +14692,9 @@ dependencies = [ "zstd", ] -[[package]] -name = "zip" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb2a05c7c36fde6c09b08576c9f7fb4cda705990f73b58fe011abf7dfb24168b" -dependencies = [ - "arbitrary", - "crc32fast", - "flate2", - "indexmap 2.13.0", - "memchr", - "zopfli", -] - [[package]] name = "zip" version = "7.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0" dependencies = [ "crc32fast", "indexmap 2.13.0", @@ -14734,30 +14702,12 @@ dependencies = [ "typed-path", ] -[[package]] -name = "zlib-rs" -version = "0.6.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be3d40e40a133f9c916ee3f9f4fa2d9d63435b5fbe1bfc6d9dae0aa0ada1513" - [[package]] name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" -[[package]] -name = "zopfli" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" -dependencies = [ - "bumpalo", - "crc32fast", - "log", - "simd-adler32", -] - [[package]] name = "zstd" version = "0.14.3" diff --git a/v2/Cargo.toml b/v2/Cargo.toml index e994169c5c..81924dd235 100644 --- a/v2/Cargo.toml +++ b/v2/Cargo.toml @@ -146,6 +146,9 @@ members = [ # excluded for the same reason. Build separately: # cargo build --target wasm32-unknown-unknown --release -p homecore-plugin-example exclude = [ + # ZIP registry archives omit large upstream unit fixtures; run its dedicated + # self-contained security regression target explicitly in CI. + "vendor/security/zip-7.2.0", "crates/wifi-densepose-wasm-edge", "crates/homecore-plugin-example", # ADR-368: napi-rs transport for ruview-kernel. Own workspace so napi never @@ -238,7 +241,7 @@ pcap = "1.1" petgraph = "0.6" # Data loading -ndarray-npy = "0.10" +ndarray-npy = { version = "0.10", default-features = false, features = ["num-complex-0_4"] } walkdir = "2.4" # Hashing (for proof) @@ -345,5 +348,6 @@ debug = true # unchanged; both parents use the fixed zstd 0.14.3 implementation. # See each directory's RUVIEW-PROVENANCE.md for archive digests and scope. [patch.crates-io] +zip-current = { package = "zip", path = "vendor/security/zip-7.2.0" } zip-legacy = { package = "zip", path = "vendor/security/zip-0.6.6" } wasmtime-internal-cache = { path = "vendor/security/wasmtime-internal-cache-36.0.20" } diff --git a/v2/vendor/security/zip-7.2.0/.amazonq/rules/rules.md b/v2/vendor/security/zip-7.2.0/.amazonq/rules/rules.md new file mode 100644 index 0000000000..a96b3eab20 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/.amazonq/rules/rules.md @@ -0,0 +1,14 @@ +- Conform to the ZIP file specification at https://support.pkware.com/pkzip/appnote. +- PR titles must conform to [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) and start + with one of the types specified by the [Angular convention](https://github.com/angular/angular/blob/22b96b9/CONTRIBUTING.md#type). + The type prefix will usually be `fix:`, `feat:`, `perf:`, `docs:` or `chore(deps):`. +- When adding a new compression or decompression method, add a feature flag for it. +- Make sure that if new code has a dependency that's gated on a feature flag, then so is the new code. +- Carefully consider the configurations of feature flags your change will affect, and test accordingly. +- Optimize your code's performance for archives that contain many files *and* archives that contain large files. +- Consider whether your change affects ZIP64 files. If so, test it on 4GiB files both with and without compression. + Generate these files by copying repeatedly to or from the same buffer, so that the test doesn't occupy or allocate + 4GiB of memory. +- Wait and see whether the tests for your PR pass in CI. Fix them if they fail. +- Rebase your PR whenever it has a merge conflict. If you can't fix the rebase conflicts, rewrite it from scratch. +- Always run `cargo fmt --all` and `cargo clippy --all-features --all-targets`. \ No newline at end of file diff --git a/v2/vendor/security/zip-7.2.0/.cargo_vcs_info.json b/v2/vendor/security/zip-7.2.0/.cargo_vcs_info.json new file mode 100644 index 0000000000..509f2182f7 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "46dc29cda6749736ddb1ec502aff06a65ffe54f8" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/v2/vendor/security/zip-7.2.0/.gitattributes b/v2/vendor/security/zip-7.2.0/.gitattributes new file mode 100644 index 0000000000..2b83c1e9d1 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/.gitattributes @@ -0,0 +1,8 @@ +tests/data/** binary=true +tests/data/**/LICENSE.*.txt binary=false +tests/data/folder/** binary=true +tests/data/legacy/** binary=true +fuzz/corpus/** binary=true + +tests/data/folder/first.txt text eol=lf +tests/data/legacy/*.out text eol=crlf \ No newline at end of file diff --git a/v2/vendor/security/zip-7.2.0/.gitignore b/v2/vendor/security/zip-7.2.0/.gitignore new file mode 100644 index 0000000000..eded420b2c --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/.gitignore @@ -0,0 +1,6 @@ +Cargo.lock +target +.DS_Store +\.idea/ +/fuzz_read/out/ +/fuzz_write/out/ diff --git a/v2/vendor/security/zip-7.2.0/.whitesource b/v2/vendor/security/zip-7.2.0/.whitesource new file mode 100644 index 0000000000..9c7ae90b4e --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/.whitesource @@ -0,0 +1,14 @@ +{ + "scanSettings": { + "baseBranches": [] + }, + "checkRunSettings": { + "vulnerableCheckRunConclusionLevel": "failure", + "displayMode": "diff", + "useMendCheckNames": true + }, + "issueSettings": { + "minSeverityLevel": "LOW", + "issueType": "DEPENDENCY" + } +} \ No newline at end of file diff --git a/v2/vendor/security/zip-7.2.0/CHANGELOG.md b/v2/vendor/security/zip-7.2.0/CHANGELOG.md new file mode 100644 index 0000000000..92c6d4d457 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/CHANGELOG.md @@ -0,0 +1,860 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [7.2.0](https://github.com/zip-rs/zip2/compare/v7.1.0...v7.2.0) - 2026-01-20 + +### 🚀 Features + +- add read_zipfile_from_stream_with_compressed_size ([#70](https://github.com/zip-rs/zip2/pull/70)) +- Allow choosing bzip2 rust backend ([#329](https://github.com/zip-rs/zip2/pull/329)) + +### 🐛 Bug Fixes + +- Need to include zip64 extra field in central directory (fix #353) ([#360](https://github.com/zip-rs/zip2/pull/360)) +- Fails to extract file which might or might not be malformed ([#376](https://github.com/zip-rs/zip2/pull/376)) ([#426](https://github.com/zip-rs/zip2/pull/426)) +- *(aes)* Allow AES encryption while streaming ([#463](https://github.com/zip-rs/zip2/pull/463)) +- Default "platform" field in zip files should be set to the local platform, rather than always "Unix" ([#470](https://github.com/zip-rs/zip2/pull/470)) ([#471](https://github.com/zip-rs/zip2/pull/471)) + +### 🚜 Refactor + +- Define cfg_if! and cfg_if_expr! internal macros ([#438](https://github.com/zip-rs/zip2/pull/438)) + +### ⚡ Performance + +- Change an assert to debug_assert when encrypting/decrypting AES, and eliminate a fallible operation ([#521](https://github.com/zip-rs/zip2/pull/521)) +- eliminate a String clone per new file added to archive, and other related refactors ([#522](https://github.com/zip-rs/zip2/pull/522)) + +### ⚙️ Miscellaneous Tasks + +- Fix another merge error, this one affecting only builds with flate2 and not zopfli +- Fix more merge issues +- Fix merge +- Fix write_dir build errors on specific feature configs +- Fix clippy warning +- Fix --all-features build error +- Fix merge + +## [7.1.0](https://github.com/zip-rs/zip2/compare/v7.0.0...v7.1.0) - 2026-01-14 + +### 🚀 Features + +- display the underlying error in Display impl for ZipError::Display ([#483](https://github.com/zip-rs/zip2/pull/483)) +- Enable creation of `ZipArchive` without reparsing ([#485](https://github.com/zip-rs/zip2/pull/485)) + +### 🐛 Bug Fixes + +- Return InvalidPassword rather than panic when AES key is the wrong length ([#457](https://github.com/zip-rs/zip2/pull/457)) +- bench with auto zip64 comment ([#505](https://github.com/zip-rs/zip2/pull/505)) +- add condition for `getrandom` dependency ([#504](https://github.com/zip-rs/zip2/pull/504)) +- *(zipcrypto)* Support streaming ZipCrypto encryption, don't store entire file in memory ([#462](https://github.com/zip-rs/zip2/pull/462)) + +### 🚜 Refactor + +- Clean up imports and move types ([#461](https://github.com/zip-rs/zip2/pull/461)) +- Replace handwritten `Ord` and `PartialOrd` for `DateTime` ([#484](https://github.com/zip-rs/zip2/pull/484)) + +### ⚙️ Miscellaneous Tasks + +- Lock `lzma-rust2` to at least 0.15.5 ([#491](https://github.com/zip-rs/zip2/pull/491)) + +## [7.0.0](https://github.com/zip-rs/zip2/compare/v6.0.0...v7.0.0) - 2025-12-05 + +### ⚠️ Breaking Changes + +- Removed the following features: `getrandom`, `hmac`, `pbkdf2`, `sha1`, `zeroize`. +- Removed `lzma-static` and `xz-static` feature flags, which were deprecated synonyms of `lzma` and `xz`. ([#405](https://github.com/zip-rs/zip2/pull/405), [#425](https://github.com/zip-rs/zip2/pull/425)) + +### 🚀 Features + +- *(`SimpleFileOptions`)* const DEFAULT implementation ([#474](https://github.com/zip-rs/zip2/pull/474)) +- ZipWriter `set_auto_large_file()` method to enable large-file data descriptor when necessary ([#468](https://github.com/zip-rs/zip2/pull/468)) + +### 🐛 Bug Fixes + +- print previous error when failing to search another cde ([#460](https://github.com/zip-rs/zip2/pull/460)) +- cargo doc warnings ([#472](https://github.com/zip-rs/zip2/pull/472)) +- Write ZIP64 data descriptors when large_file option is true ([#467](https://github.com/zip-rs/zip2/pull/467)) +- Pin generic-array to an old version to work around https://github.com/RustCrypto/traits/issues/2036 until next RustCrypto & aes-crypto releases ([#458](https://github.com/zip-rs/zip2/pull/458)) + +### ⚙️ Miscellaneous Tasks + +- Revert version bump so that release-plz will trigger +- expose more flate2 feature flags ([#476](https://github.com/zip-rs/zip2/pull/476)) +- Next release will be 7.0.0 +- release v6.0.0 ([#442](https://github.com/zip-rs/zip2/pull/442)) + +### Deps + +- Bump lzma-rust2 to v0.15 ([#465](https://github.com/zip-rs/zip2/pull/465)) + +## [6.0.0](https://github.com/zip-rs/zip2/compare/v5.1.1...v6.0.0) - 2025-10-09 + +### 🚀 Features + +- Add by_index_with_options(), which can be used to ignore encryption in a file's metadata ([#439](https://github.com/zip-rs/zip2/pull/439)) and may be used for other file-specific overrides in the future. + +### ⚙️ Miscellaneous Tasks + +- [**breaking**] `FileOptions::add_extra_data` is now generic and accepts any `AsRef<[u8]>`. (#435) + +## [5.1.1](https://github.com/zip-rs/zip2/compare/v5.1.0...v5.1.1) - 2025-09-11 + +### 🐛 Bug Fixes + +- panic when reading empty extended-timestamp field ([#404](https://github.com/zip-rs/zip2/pull/404)) ([#422](https://github.com/zip-rs/zip2/pull/422)) +- Restore original file timestamp when unzipping with `chrono` ([#46](https://github.com/zip-rs/zip2/pull/46)) + +### ⚙️ Miscellaneous Tasks + +- Configure Amazon Q rules ([#421](https://github.com/zip-rs/zip2/pull/421)) + +## [5.1.0](https://github.com/zip-rs/zip2/compare/v5.0.1...v5.1.0) - 2025-09-10 + +### 🚀 Features + +- Add legacy shrink/reduce/implode compression ([#303](https://github.com/zip-rs/zip2/pull/303)) + +## [5.0.1](https://github.com/zip-rs/zip2/compare/v5.0.0...v5.0.1) - 2025-09-09 + +### 🐛 Bug Fixes + +- AES metadata was not copied correctly in raw copy methods, which could corrupt the copied file. ([#417](https://github.com/zip-rs/zip2/pull/417)) + +## [5.0.0](https://github.com/zip-rs/zip2/compare/v4.6.1...v5.0.0) - 2025-09-05 + +### 🚀 Features + +- Implement by_path*() methods on ZipArchive ([#382](https://github.com/zip-rs/zip2/pull/382)) + +## [4.6.1](https://github.com/zip-rs/zip2/compare/v4.6.0...v4.6.1) - 2025-09-03 + +### 🐛 Bug Fixes + +- Fixes an issue introduced by the swap from `lzma-rs` to `liblzma` ([#407](https://github.com/zip-rs/zip2/pull/407)) + +## [4.6.0](https://github.com/zip-rs/zip2/compare/v4.5.0...v4.6.0) - 2025-08-30 + +### 🚀 Features + +- Allow to read zip files with unsupported extended timestamps ([#400](https://github.com/zip-rs/zip2/pull/400)) + +### 🐛 Bug Fixes + +- enable clamp_opt for ppmd and xz ([#401](https://github.com/zip-rs/zip2/pull/401)) + +## [4.5.0](https://github.com/zip-rs/zip2/compare/v4.4.0...v4.5.0) - 2025-08-21 + +### 🚀 Features + +- Allow reading ZIP files where the central directory comes *before* the files ([#384](https://github.com/zip-rs/zip2/pull/384)) ([#396](https://github.com/zip-rs/zip2/pull/396)) + +## [4.4.0](https://github.com/zip-rs/zip2/compare/v4.3.0...v4.4.0) - 2025-08-21 + +### 🚀 Features + +- Add `lzma-static` and `xz-static` features that enable `liblzma/static` ([#393](https://github.com/zip-rs/zip2/pull/393)) + +### ⚙️ Miscellaneous Tasks + +- Move deprecated annotations to fix a Clippy warning ([#391](https://github.com/zip-rs/zip2/pull/391)) + +## [4.3.0](https://github.com/zip-rs/zip2/compare/v4.2.0...v4.3.0) - 2025-07-09 + +### 🚀 Features + +- Add support for PPMd ([#370](https://github.com/zip-rs/zip2/pull/370)) + +## [4.2.0](https://github.com/zip-rs/zip2/compare/v4.1.0...v4.2.0) - 2025-06-21 + +### 🚀 Features + +- Write ZIP file to stream ([#246](https://github.com/zip-rs/zip2/pull/246)) + +## [4.1.0](https://github.com/zip-rs/zip2/compare/v4.0.0...v4.1.0) - 2025-06-14 + +### 🚀 Features + +- Add has_overlapping_files method + +## [4.0.0](https://github.com/zip-rs/zip2/compare/v3.0.0...v4.0.0) - 2025-05-21 + +### 🐛 Bug Fixes + +- Allow extraction of Zip64 where "Version needed to extract" is higher than "Version made by" ([#356](https://github.com/zip-rs/zip2/pull/356)) + +### ⚙️ Miscellaneous Tasks + +- Revert nt-time upgrade (would increase MSRV) +- Revert constant_time_eq update (would increase MSRV) +- Update fully-qualified names of liblzma imports + +## [3.0.0](https://github.com/zip-rs/zip2/compare/v2.6.1...v3.0.0) - 2025-05-14 + +### 🐛 Bug Fixes + +- return correct offset in SeekableTake::seek ([#342](https://github.com/zip-rs/zip2/pull/342)) +- When only zopfli is available, decompression of deflate should not be possible ([#348](https://github.com/zip-rs/zip2/pull/348)) +- Specify `flate2` dependency of the `deflate-flate2` feature. ([#345](https://github.com/zip-rs/zip2/pull/345)) + +### ⚙️ Miscellaneous Tasks + +- drop unused crossbeam-utils dependency ([#339](https://github.com/zip-rs/zip2/pull/339)) +- fix typo +- remove `deflate-flate2` dependency on specific backend +- [**breaking**] Drop deprecated `deflate-miniz` feature flag ([#351](https://github.com/zip-rs/zip2/pull/351)) + +## [2.6.1](https://github.com/zip-rs/zip2/compare/v2.6.0...v2.6.1) - 2025-04-03 + +### 🐛 Bug Fixes + +- avoid scanning through all local file headers while opening an archive ([#281](https://github.com/zip-rs/zip2/pull/281)) + +## [2.5.0](https://github.com/zip-rs/zip2/compare/v2.4.2...v2.5.0) - 2025-03-23 + +### 🚀 Features + +- Add support for `time::PrimitiveDateTime` ([#322](https://github.com/zip-rs/zip2/pull/322)) +- Add `jiff` integration ([#323](https://github.com/zip-rs/zip2/pull/323)) + +### 🐛 Bug Fixes + +- improve error message for duplicated file ([#277](https://github.com/zip-rs/zip2/pull/277)) + +## [2.4.2](https://github.com/zip-rs/zip2/compare/v2.4.1...v2.4.2) - 2025-03-18 + +### 🐛 Bug Fixes + +- `deep_copy_file` produced a mangled file header on big-endian platforms (#309) + +## [2.4.1](https://github.com/zip-rs/zip2/compare/v2.4.0...v2.4.1) - 2025-03-17 + +### 🐛 Bug Fixes + +- type issue in test +- double as_ref().canonicalize()? +- CI failures +- Create directory for extraction if necessary ([#314](https://github.com/zip-rs/zip2/pull/314)) + +## [2.4.0](https://github.com/zip-rs/zip2/compare/v2.3.0...v2.4.0) - 2025-03-17 + +### 🚀 Features + +- `ZipArchive::root_dir` and `ZipArchive::extract_unwrapped_root_dir` ([#304](https://github.com/zip-rs/zip2/pull/304)) + +### 🐛 Bug Fixes + +- wasm build failure due to a missing use statement ([#313](https://github.com/zip-rs/zip2/pull/313)) + +## [2.3.0](https://github.com/zip-rs/zip2/compare/v2.2.3...v2.3.0) - 2025-03-16 + +### 🚀 Features + +- Add support for NTFS extra field ([#279](https://github.com/zip-rs/zip2/pull/279)) + +### 🐛 Bug Fixes + +- *(test)* Conditionalize a zip64 doctest ([#308](https://github.com/zip-rs/zip2/pull/308)) +- fix failing tests, remove symlink loop check +- Canonicalize output path to avoid false negatives +- Symlink handling in stream extraction +- Canonicalize output paths and symlink targets, and ensure they descend from the destination + +### ⚙️ Miscellaneous Tasks + +- Fix clippy and cargo fmt warnings ([#310](https://github.com/zip-rs/zip2/pull/310)) + +## [2.2.3](https://github.com/zip-rs/zip2/compare/v2.2.2...v2.2.3) - 2025-02-26 + +### 🚜 Refactor + +- Change the inner structure of `DateTime` (#267) + +### ⚙️ Miscellaneous Tasks + +- cargo fix --edition + +## [2.2.2](https://github.com/zip-rs/zip2/compare/v2.2.1...v2.2.2) - 2024-12-16 + +### 🐛 Bug Fixes + +- rewrite the EOCD/EOCD64 detection to fix extreme performance regression (#247) + +## [2.2.1](https://github.com/zip-rs/zip2/compare/v2.2.0...v2.2.1) - 2024-11-20 + +### 🐛 Bug Fixes + +- remove executable bit ([#238](https://github.com/zip-rs/zip2/pull/238)) +- *(lzma)* fixed panic in case of invalid lzma stream ([#259](https://github.com/zip-rs/zip2/pull/259)) +- resolve new clippy warnings on nightly ([#262](https://github.com/zip-rs/zip2/pull/262)) +- resolve clippy warning in nightly ([#252](https://github.com/zip-rs/zip2/pull/252)) + +### ⚡ Performance + +- Faster cde rejection ([#255](https://github.com/zip-rs/zip2/pull/255)) + +## [2.2.0](https://github.com/zip-rs/zip2/compare/v2.1.6...v2.2.0) - 2024-08-11 + +### 🚀 Features +- Expose `ZipArchive::central_directory_start` ([#232](https://github.com/zip-rs/zip2/pull/232)) + +## [2.1.6](https://github.com/zip-rs/zip2/compare/v2.1.5...v2.1.6) - 2024-07-29 + +### 🐛 Bug Fixes +- ([#33](https://github.com/zip-rs/zip2/pull/33)) Rare combination of settings could lead to writing a corrupt archive with overlength extra data, and data_start locations when reading the archive back were also wrong ([#221](https://github.com/zip-rs/zip2/pull/221)) + +### 🚜 Refactor +- Eliminate some magic numbers and unnecessary path prefixes ([#225](https://github.com/zip-rs/zip2/pull/225)) + +## [2.1.5](https://github.com/zip-rs/zip2/compare/v2.1.4...v2.1.5) - 2024-07-20 + +### 🚜 Refactor +- change invalid_state() return type to io::Result + +## [2.1.4](https://github.com/zip-rs/zip2/compare/v2.1.3...v2.1.4) - 2024-07-18 + +### 🐛 Bug Fixes +- fix([#215](https://github.com/zip-rs/zip2/pull/215)): Upgrade to deflate64 0.1.9 +- Panic when reading a file truncated in the middle of an XZ block header +- Some archives with over u16::MAX files were handled incorrectly or slowly ([#189](https://github.com/zip-rs/zip2/pull/189)) +- Check number of files when deciding whether a CDE is the real one +- Could still select a fake CDE over a real one in some cases +- May have to consider multiple CDEs before filtering for validity +- We now keep searching for a real CDE header after read an invalid one from the file comment +- Always search for data start when opening an archive for append, and reject the header if data appears to start after central directory +- `deep_copy_file` no longer allows overwriting an existing file, to match the behavior of `shallow_copy_file` +- File start position was wrong when extra data was present +- Abort file if central extra data is too large +- Overflow panic when central directory extra data is too large +- ZIP64 header was being written twice when copying a file +- ZIP64 header was being written to central header twice +- Start position was incorrect when file had no extra data +- Allow all reserved headers we can create +- Fix a bug where alignment padding interacts with other extra-data fields +- Fix bugs involving alignment padding and Unicode extra fields +- Incorrect header when adding AES-encrypted files +- Parse the extra field and reject it if invalid +- Incorrect behavior following a rare combination of `merge_archive`, `abort_file` and `deep_copy_file`. As well, we now return an error when a file is being copied to itself. +- path_to_string now properly handles the case of an empty path +- Implement `Debug` for `ZipWriter` even when it's not implemented for the inner writer's type +- Fix an issue where the central directory could be incorrectly detected +- `finish_into_readable()` would corrupt the archive if the central directory had moved + +### 🚜 Refactor +- Verify with debug assertions that no FixedSizeBlock expects a multi-byte alignment ([#198](https://github.com/zip-rs/zip2/pull/198)) +- Use new do_or_abort_file method + +### ⚡ Performance +- Speed up CRC when encrypting small files +- Limit the number of extra fields +- Refactor extra-data validation +- Store extra data in plain vectors until after validation +- Only build one IndexMap after choosing among the possible valid headers +- Simplify validation of empty extra-data fields +- Validate automatic extra-data fields only once, even if several are present +- Remove redundant `validate_extra_data()` call +- Skip searching for the ZIP32 header if a valid ZIP64 header is present ([#189](https://github.com/zip-rs/zip2/pull/189)) + +### ⚙️ Miscellaneous Tasks +- Fix a bug introduced by c934c824 +- Fix a failing unit test +- Fix build errors on older Rust versions +- Fix build +- Fix another fuzz failure +- Switch to `ok_or_abort_file`, and inline when that fails borrow checker +- Switch to `ok_or_abort_file`, and inline when that fails borrow checker +- Fix a build error +- Fix boxed_local warning (can borrow instead) +- Partial debug +- Fix more errors when parsing multiple extra fields +- Fix an error when decoding AES header +- Fix an error caused by not allowing 0xa11e field +- Bug fix: crypto_header was being counted toward extra_data_end +- Bug fix: revert a change where crypto_header was incorrectly treated as an extra field +- Fix a bug where a modulo of 0 was used +- Fix a bug when ZipCrypto, alignment *and* a custom header are used +- Fix a bug when both ZipCrypto and alignment are used +- Fix another bug: header_end vs extra_data_end +- Fix use of a stale value in a `debug_assert_eq!` +- Fix: may still get an incorrect size if opening an invalid file for append +- Fix: may need the absolute start as tiebreaker to ensure deterministic behavior + +## [2.1.3](https://github.com/zip-rs/zip2/compare/v2.1.2...v2.1.3) - 2024-06-04 + +### 🐛 Bug Fixes +- Some date/time filters were previously unreliable (i.e. later-pass filters had no earliest-pass or latest-fail, and vice-versa) +- Decode Zip-Info UTF8 name and comment fields ([#159](https://github.com/zip-rs/zip2/pull/159)) + +### 🚜 Refactor +- Return extended timestamp fields copied rather than borrowed ([#183](https://github.com/zip-rs/zip2/pull/183)) + +### ⚙️ Miscellaneous Tasks +- Fix a new Clippy warning +- Fix a bug and inline `deserialize` for safety +- Add check for wrong-length blocks, and incorporate fixed-size requirement into the trait name +- Fix a fuzz failure by using checked_sub +- Add feature gate for new unit test + +## [2.1.1](https://github.com/zip-rs/zip2/compare/v2.1.0...v2.1.1) - 2024-05-28 + +### 🐛 Bug Fixes +- Derive `Debug` for `ZipWriter` +- lower default version to 4.5 and use the version-needed-to-extract where feasible. + +### 🚜 Refactor +- use a MIN_VERSION constant + +### ⚙️ Miscellaneous Tasks +- Bug fixes for debug implementation +- Bug fixes for debug implementation +- Update unit tests +- Remove unused import + +## [2.1.0](https://github.com/zip-rs/zip2/compare/v2.0.0...v2.1.0) - 2024-05-25 + +### 🚀 Features +- Support mutual conversion between `DateTime` and MS-DOS pair + +### 🐛 Bug Fixes +- version-needed-to-extract was incorrect in central header, and version-made-by could be lower than that ([#100](https://github.com/zip-rs/zip2/pull/100)) +- version-needed-to-extract was incorrect in central header, and version-made-by could be lower than that ([#100](https://github.com/zip-rs/zip2/pull/100)) + +### ⚙️ Miscellaneous Tasks +- Another tweak to ensure `version_needed` is applied +- Tweaks to make `version_needed` and `version_made_by` work with recently-merged changes + +## [2.0.0](https://github.com/zip-rs/zip2/compare/v1.3.1...v2.0.0) - 2024-05-24 + +### 🚀 Features +- Add `fmt::Display` for `DateTime` +- Implement more traits for `DateTime` + +### 🚜 Refactor +- Change type of `last_modified_time` to `Option` +- [**breaking**] Rename `from_msdos` to `from_msdos_unchecked`, make it unsafe, and add `try_from_msdos` ([#145](https://github.com/zip-rs/zip2/pull/145)) + +### ⚙️ Miscellaneous Tasks +- Continue to accept archives with invalid DateTime, and use `now_utc()` as default only when writing, not reading + +## [1.3.1](https://github.com/zip-rs/zip2/compare/v1.3.0...v1.3.1) - 2024-05-21 + +### 🚜 Refactor +- Make `deflate` enable both default implementations +- Merge the hidden deflate-flate2 flag into the public one +- Rename _deflate-non-zopfli to _deflate-flate2 +- Reject encrypted and using_data_descriptor files slightly faster in read_zipfile_from_stream +- Convert `impl TryInto for DateTime` to `impl TryFrom for NaiveDateTime` ([#136](https://github.com/zip-rs/zip2/pull/136)) + +### ⚡ Performance +- Change default compression implementation to `flate2/zlib-ng` + +### ⚙️ Miscellaneous Tasks +- chore([#132](https://github.com/zip-rs/zip2/pull/132)): Attribution for some copied test data +- chore([#133](https://github.com/zip-rs/zip2/pull/133)): chmod -x src/result.rs + +## [1.3.0](https://github.com/zip-rs/zip2/compare/v1.2.3...v1.3.0) - 2024-05-17 + +### 🚀 Features +- Add `is_symlink` method + +### 🐛 Bug Fixes +- Extract symlinks into symlinks on Unix and Windows, and fix a bug that affected making directories writable on MacOS + +### 🚜 Refactor +- Eliminate deprecation warning when `--all-features` implicitly enables the deprecated feature +- Check if archive contains a symlink's target, without borrowing both at the same time +- Eliminate a clone that's no longer necessary +- is_dir only needs to look at the filename +- Remove unnecessary #[cfg] attributes + +### ⚙️ Miscellaneous Tasks +- Fix borrow-of-moved-value +- Box doesn't directly convert to PathBuf, so convert back to String first +- partial revert - only &str has chars(), but Box should auto-deref +- contains_key needs a `Box`, so generify `is_dir` to accept one +- Add missing `ZipFileData::is_dir()` method +- Fix another Windows-specific error +- More bug fixes for Windows-specific symlink code +- More bug fixes for Windows-specific symlink code +- Bug fix: variable name change +- Bug fix: need both internal and output path to determine whether to symlink_dir +- Another bug fix +- Fix another error-type conversion error +- Fix error-type conversion on Windows +- Fix conditionally-unused import +- Fix continued issues, and factor out the Vec-to-OsString conversion (cc: [#125](https://github.com/zip-rs/zip2/pull/125)) +- Fix CI failure involving conversion to OsString for symlinks (see my comments on [#125](https://github.com/zip-rs/zip2/pull/125)) +- Move path join into platform-independent code + +## [1.2.3](https://github.com/zip-rs/zip2/compare/v1.2.2...v1.2.3) - 2024-05-10 + +### 🐛 Bug Fixes +- Remove a window when an extracted directory might be unexpectedly listable and/or `cd`able by non-owners +- Extract directory contents on Unix even if the directory doesn't have write permission (https://github.com/zip-rs/zip-old/issues/423) + +### ⚙️ Miscellaneous Tasks +- More conditionally-unused imports + +## [1.2.2](https://github.com/zip-rs/zip2/compare/v1.2.1...v1.2.2) - 2024-05-09 + +### 🐛 Bug Fixes +- Failed to clear "writing_raw" before finishing a symlink, leading to dropped extra fields + +### ⚡ Performance +- Use boxed slice for archive comment, since it can't be concatenated +- Optimize for the fact that false signatures can't overlap with real ones + +## [1.2.1](https://github.com/zip-rs/zip2/compare/v1.2.0...v1.2.1) - 2024-05-06 + +### 🐛 Bug Fixes +- Prevent panic when trying to read a file with an unsupported compression method +- Prevent panic after reading an invalid LZMA file +- Make `Stored` the default compression method if `Deflated` isn't available, so that zip files are readable by as much software as possible +- version_needed was wrong when e.g. cfg(bzip2) but current file wasn't bzip2 ([#100](https://github.com/zip-rs/zip2/pull/100)) +- file paths shouldn't start with slashes ([#102](https://github.com/zip-rs/zip2/pull/102)) + +### 🚜 Refactor +- Overhaul `impl Arbitrary for FileOptions` +- Remove unused `atomic` module + +## [1.2.0](https://github.com/zip-rs/zip2/compare/v1.1.4...v1.2.0) - 2024-05-06 + +### 🚀 Features +- Add method `decompressed_size()` so non-recursive ZIP bombs can be detected + +### 🚜 Refactor +- Make `ZipWriter::finish()` consume the `ZipWriter` + +### ⚙️ Miscellaneous Tasks +- Use panic! rather than abort to ensure the fuzz harness can process the failure +- Update fuzz_write to use replace_with +- Remove a drop that can no longer be explicit +- Add `#![allow(unexpected_cfgs)]` in nightly + +## [1.1.4](https://github.com/zip-rs/zip2/compare/v1.1.3...v1.1.4) - 2024-05-04 + +### 🐛 Bug Fixes +- Build was failing with bzip2 enabled +- use is_dir in more places where Windows paths might be handled incorrectly + +### ⚡ Performance +- Quick filter for paths that contain "/../" or "/./" or start with "./" or "../" +- Fast handling for separator-free paths +- Speed up logic if main separator isn't '/' +- Drop `normalized_components` slightly sooner when not using it +- Speed up `path_to_string` in cases where the path is already in the proper format + +### ⚙️ Miscellaneous Tasks +- Refactor: can short-circuit handling of paths that start with MAIN_SEPARATOR, no matter what MAIN_SEPARATOR is +- Bug fix: non-canonical path detection when MAIN_SEPARATOR is not slash or occurs twice in a row +- Bug fix: must recreate if . or .. is a path element +- Bug fix + +### ◀️ Revert +- [#58](https://github.com/zip-rs/zip2/pull/58) (partial): `bzip2-rs` can't replace `bzip2` because it's decompress-only + +## [1.1.3](https://github.com/zip-rs/zip2/compare/v1.1.2...v1.1.3) - 2024-04-30 + +### 🐛 Bug Fixes +- Rare bug where find_and_parse would give up prematurely on detecting a false end-of-CDR header + +## [1.1.2](https://github.com/Pr0methean/zip/compare/v1.1.1...v1.1.2) - 2024-04-28 + +### 🐛 Bug Fixes +- Alignment was previously handled incorrectly ([#33](https://github.com/Pr0methean/zip/pull/33)) + +### 🚜 Refactor +- deprecate `deflate-miniz` feature since it's now equivalent to `deflate` ([#35](https://github.com/Pr0methean/zip/pull/35)) + +## [1.1.1] + +### Added + +- `index_for_name`, `index_for_path`, `name_for_index`: get the index of a file given its path or vice-versa, without + initializing metadata from the local-file header or needing to mutably borrow the `ZipArchive`. +- `add_symlink_from_path`, `shallow_copy_file_from_path`, `deep_copy_file_from_path`, `raw_copy_file_to_path`: copy a + file or create a symlink using `AsRef` arguments + +### Changed + +- `add_directory_from_path` and `start_file_from_path` are no longer deprecated, and they now normalize `..` as well as + `.`. + +## [1.1.0] + +### Added + +- Support for decoding LZMA. + +### Changed + +- Eliminated a custom `AtomicU64` type by replacing it with `OnceLock` in the only place it's used. +- `FileOptions` now has the subtype `SimpleFileOptions` which implements `Copy` but has no extra data. + +## [1.0.1] + +### Changed + +- The published package on crates.io no longer includes the tests or examples. + +## [1.0.0] + +### Changed + +- Now uses boxed slices rather than `String` or `Vec` for metadata fields that aren't likely to grow. + +## [0.11.0] + +### Added + +- Support for `DEFLATE64` (decompression only). +- Support for Zopfli compression levels up to `i64::MAX`. + +### Changed + +- `InvalidPassword` is now a kind of `ZipError` to eliminate the need for nested `Result` structs. +- Updated dependencies. + +## [0.10.3] + +### Changed + +- Updated dependencies. +- MSRV increased to `1.67`. + +### Fixed + +- Fixed some rare bugs that could cause panics when trying to read an invalid ZIP file or using an incorrect password. + +## [0.10.2] + +### Changed + +- Where possible, methods are now `const`. This improves performance, especially when reading. + +## [0.10.1] + +### Changed + +- Date and time conversion methods now return `DateTimeRangeError` rather than `()` on error. + +## [0.10.0] + +### Changed + +- Replaces the `flush_on_finish_file` parameter of `ZipWriter::new` and `ZipWriter::Append` with + a `set_flush_on_finish_file` method. + +### Fixed + +- Fixes build errors that occur when all default features are disabled. +- Fixes more cases of a bug when ZIP64 magic bytes occur in filenames. + +## [0.9.2] + +### Added + +- `zlib-ng` for fast Deflate compression. This is now the default for compression levels 0-9. +- `chrono` to convert zip::DateTime to and from chrono::NaiveDateTime + +## [0.9.1] + +### Added + +- Zopfli for aggressive Deflate compression. + +## [0.9.0] + +### Added + + - `flush_on_finish_file` parameter for `ZipWriter`. + +## [0.8.3] + +### Changed + +- Uses the `aes::cipher::KeyInit` trait from `aes` 0.8.2 where appropriate. + +### Fixed + +- Calling `abort_file()` no longer corrupts the archive if called on a + shallow copy of a remaining file, or on an archive whose CDR entries are out + of sequence. However, it may leave an unused entry in the archive. +- Calling `abort_file()` while writing a ZipCrypto-encrypted file no longer + causes a crash. +- Calling `abort_file()` on the last file before `finish()` no longer produces + an invalid ZIP file or garbage in the comment. + +### Added + +- `ZipWriter` methods `get_comment()` and `get_raw_comment()`. + +## [0.8.2] + +### Fixed + +- Fixed an issue where code might spuriously fail during write fuzzing. + +### Added + +- New method `with_alignment` on `FileOptions`. + +## [0.8.1] + +### Fixed + +- `ZipWriter` now once again implements `Send` if the underlying writer does. + +## [0.8.0] + +### Deleted + +- Methods `start_file_aligned`, `start_file_with_extra_data`, `end_local_start_central_extra_data` and + `end_extra_data` (see below). + +### Changed + +- Alignment and extra-data fields are now attributes of [`zip::unstable::write::FileOptions`], allowing them to be + specified for `add_directory` and `add_symlink`. +- Extra-data fields are now formatted by the `FileOptions` method `add_extra_data`. +- Improved performance, especially for `shallow_copy_file` and `deep_copy_file` on files with extra data. + +### Fixed + +- Fixes a rare bug where the size of the extra-data field could overflow when `large_file` was set. +- Fixes more cases of a bug when ZIP64 magic bytes occur in filenames. + +## [0.7.5] + +### Fixed + +- Fixed a bug that occurs when ZIP64 magic bytes occur twice in a filename or across two filenames. + +## [0.7.4] + +### Added + +- Added experimental [`zip::unstable::write::FileOptions::with_deprecated_encryption`] API to enable encrypting + files with PKWARE encryption. + +## [0.7.3] + +### Fixed + +- Fixed a bug that occurs when a filename in a ZIP32 file includes the ZIP64 magic bytes. + +## [0.7.2] + +### Added + +- Method `abort_file` - removes the current or most recently-finished file from the archive. + +### Fixed + +- Fixed a bug where a file could remain open for writing after validations failed. + +## [0.7.1] + +### Changed + +- Bumped the version number in order to upload an updated README to crates.io. + +## [0.7.0] + +### Fixed + +- Calling `start_file` with invalid parameters no longer closes the `ZipWriter`. +- Attempting to write a 4GiB file without calling `FileOptions::large_file(true)` now removes the file from the archive + but does not close the `ZipWriter`. +- Attempting to write a file with an unrepresentable or invalid last-modified date will instead add it with a date of + 1980-01-01 00:00:00. + +### Added + +- Method `is_writing_file` - indicates whether a file is open for writing. + +## [0.6.13] + +### Fixed + +- Fixed a possible bug in deep_copy_file. + +## [0.6.12] + +### Fixed + +- Fixed a Clippy warning that was missed during the last release. + +## [0.6.11] + +### Fixed + +- Fixed a bug that could cause later writes to fail after a `deep_copy_file` call. + +## [0.6.10] + +### Changed + +- Updated dependency versions. + +## [0.6.9] + +### Fixed + +- Fixed an issue that prevented `ZipWriter` from implementing `Send`. + +## [0.6.8] + +### Added + +- Detects duplicate filenames. + +### Fixed + +- `deep_copy_file` could set incorrect Unix permissions. +- `deep_copy_file` could handle files incorrectly if their compressed size was u32::MAX bytes or less but their + uncompressed size was not. +- Documented that `deep_copy_file` does not copy a directory's contents. + +### Changed + +- Improved performance of `deep_copy_file` by using a HashMap and eliminating a redundant search. + +## [0.6.7] + +### Added + +- `deep_copy_file` method: more standards-compliant way to copy a file from within the ZipWriter + +## [0.6.6] + +### Fixed + +- Unused flag `#![feature(read_buf)]` was breaking compatibility with stable compiler. + +### Changed + +- Updated `aes` dependency to `0.8.2` (https://github.com/zip-rs/zip/pull/354) +- Updated other dependency versions. + +## [0.6.5] +### Changed + +- Added experimental [`zip::unstable::write::FileOptions::with_deprecated_encryption`] API to enable encrypting files with PKWARE encryption. + +### Added + +- `shallow_copy_file` method: copy a file from within the ZipWriter + + +## [0.6.4] + +### Changed + +- [#333](https://github.com/zip-rs/zip/pull/333): disabled the default features of the `time` dependency, and also `formatting` and `macros`, as they were enabled by mistake. +- Deprecated [`DateTime::from_time`](https://docs.rs/zip/0.6/zip/struct.DateTime.html#method.from_time) in favor of [`DateTime::try_from`](https://docs.rs/zip/0.6/zip/struct.DateTime.html#impl-TryFrom-for-DateTime) diff --git a/v2/vendor/security/zip-7.2.0/CODE_OF_CONDUCT.md b/v2/vendor/security/zip-7.2.0/CODE_OF_CONDUCT.md new file mode 100644 index 0000000000..2290ec2bf8 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/CODE_OF_CONDUCT.md @@ -0,0 +1,76 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to make participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, sex characteristics, gender identity and expression, +level of experience, education, socio-economic status, nationality, personal +appearance, race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. + +## Scope + +This Code of Conduct applies within all project spaces, and it also applies when +an individual is representing the project or its community in public spaces. +Examples of representing a project or community include using an official +project e-mail address, posting via an official social media account, or acting +as an appointed representative at an online or offline event. Representation of +a project may be further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at ryan.levick@gmail.com. All +complaints will be reviewed and investigated and will result in a response that +is deemed necessary and appropriate to the circumstances. The project team is +obligated to maintain confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html + +[homepage]: https://www.contributor-covenant.org + +For answers to common questions about this code of conduct, see +https://www.contributor-covenant.org/faq diff --git a/v2/vendor/security/zip-7.2.0/CONTRIBUTING.md b/v2/vendor/security/zip-7.2.0/CONTRIBUTING.md new file mode 100644 index 0000000000..8d638d9f7b --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/CONTRIBUTING.md @@ -0,0 +1 @@ +Pull requests are welcome, but they're subject to some requirements that a lot of them don't meet. See https://github.com/zip-rs/zip2/raw/master/pull_request_template.md for details. diff --git a/v2/vendor/security/zip-7.2.0/Cargo.lock b/v2/vendor/security/zip-7.2.0/Cargo.lock new file mode 100644 index 0000000000..342a95aa45 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/Cargo.lock @@ -0,0 +1,1300 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + +[[package]] +name = "anstream" +version = "0.6.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" + +[[package]] +name = "anstyle-parse" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" +dependencies = [ + "anstyle", + "once_cell_polyfill", + "windows-sys", +] + +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "async-trait" +version = "0.1.89" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "autocfg" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "bencher" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dfdb4953a096c551ce9ace855a604d702e6e62d77fac690575ae347571717f5" + +[[package]] +name = "bitflags" +version = "2.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" + +[[package]] +name = "bitstream-io" +version = "4.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60d4bd9d1db2c6bdf285e223a7fa369d5ce98ec767dec949c6ca62863ce61757" +dependencies = [ + "core2", +] + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "bumpalo" +version = "3.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dd9dc738b7a8311c7ade152424974d8115f2cdad61e8dab8dac9f2362298510" + +[[package]] +name = "bzip2" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" +dependencies = [ + "bzip2-sys", + "libbz2-rs-sys", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + +[[package]] +name = "cc" +version = "1.2.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "755d2fce177175ffca841e9a06afdb2c4ab0f593d53b4dee48147dfaade85932" +dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "chrono" +version = "0.4.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fac4744fb15ae8337dc853fee7fb3f4e48c0fbaa23d0afe49c447b4fab126118" +dependencies = [ + "num-traits", +] + +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + +[[package]] +name = "clap" +version = "4.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e578d6ec4194633722ccf9544794b71b1385c3c027efe0c55db226fc880865c" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4df4df40ec50c46000231c914968278b1eb05098cf8f1b3a518a95030e71d1c7" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" + +[[package]] +name = "cmake" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75443c44cd6b379beb8c5b45d85d0773baf31cce901fe7bb252f4eff3008ef7d" +dependencies = [ + "cc", +] + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + +[[package]] +name = "constant_time_eq" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c74b8349d32d297c9134b8c88677813a227df8f779daa29bfc29c183fe3dca6" + +[[package]] +name = "core2" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" +dependencies = [ + "memchr", +] + +[[package]] +name = "cpufeatures" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280" +dependencies = [ + "libc", +] + +[[package]] +name = "crc" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" +dependencies = [ + "crc-catalog", +] + +[[package]] +name = "crc-catalog" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" + +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crypto-common" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "deflate64" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26bf8fc351c5ed29b5c2f0cbbac1b209b74f60ecd62e675a998df72c49af5204" + +[[package]] +name = "deranged" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "either" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719" + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "find-msvc-tools" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db" + +[[package]] +name = "flate2" +version = "1.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b375d6465b98090a5f25b1c7703f3859783755aa9a80433b36e0379a3ec2f369" +dependencies = [ + "crc32fast", + "libz-ng-sys", + "libz-sys", + "miniz_oxide", + "zlib-rs", +] + +[[package]] +name = "futures-core" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" +dependencies = [ + "cfg-if", + "js-sys", + "libc", + "r-efi", + "wasip2", + "wasm-bindgen", +] + +[[package]] +name = "hashbrown" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "indexmap" +version = "2.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "generic-array", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" + +[[package]] +name = "itoa" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92ecc6618181def0457392ccd0ee51198e065e016d1d527a7ac1b6dc7c1f09d2" + +[[package]] +name = "jiff" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e67e8da4c49d6d9909fe03361f9b620f58898859f5c7aded68351e85e71ecf50" +dependencies = [ + "jiff-static", + "jiff-tzdb-platform", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", + "windows-sys", +] + +[[package]] +name = "jiff-static" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0c84ee7f197eca9a86c6fd6cb771e55eb991632f15f2bc3ca6ec838929e6e78" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "jiff-tzdb" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68971ebff725b9e2ca27a601c5eb38a4c5d64422c4cbab0c535f248087eda5c2" + +[[package]] +name = "jiff-tzdb-platform" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "875a5a69ac2bab1a891711cf5eccbec1ce0341ea805560dcd90b7a2e925132e8" +dependencies = [ + "jiff-tzdb", +] + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom", + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c942ebf8e95485ca0d52d97da7c5a2c387d0e7f0ba4c35e93bfcaee045955b3" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "libbz2-rs-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" + +[[package]] +name = "libc" +version = "0.2.180" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcc35a38544a891a5f7c865aca548a982ccb3b8650a5b06d0fd33a10283c56fc" + +[[package]] +name = "libm" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" + +[[package]] +name = "libz-ng-sys" +version = "1.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bf914b7dd154ca9193afec311d8e39345c1bd93b48b3faa77329f0db8f553c0" +dependencies = [ + "cmake", + "libc", +] + +[[package]] +name = "libz-sys" +version = "1.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15d118bbf3771060e7311cc7bb0545b01d08a8b4a7de949198dec1fa0ca1c0f7" +dependencies = [ + "cc", + "cmake", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "linux-raw-sys" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" + +[[package]] +name = "log" +version = "0.4.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" + +[[package]] +name = "lzma-rust2" +version = "0.15.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1670343e58806300d87950e3401e820b519b9384281bbabfb15e3636689ffd69" +dependencies = [ + "crc", + "sha2", +] + +[[package]] +name = "memchr" +version = "2.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" + +[[package]] +name = "minicov" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4869b6a491569605d66d3952bcdf03df789e5b536e5f0cf7758a7f08a55ae24d" +dependencies = [ + "cc", + "walkdir", +] + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + +[[package]] +name = "nt-time" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c367e8edaff1f8a871e56343eb5e03888f6e0d1c2861880ccf4b7dc830899ed" +dependencies = [ + "time", +] + +[[package]] +name = "nu-ansi-term" +version = "0.50.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + +[[package]] +name = "once_cell" +version = "1.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" + +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" +dependencies = [ + "digest", + "hmac", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "pkg-config" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" + +[[package]] +name = "portable-atomic" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f89776e4d69bb58bc6993e99ffa1d11f228b839984854c7daeb5d37f87cbe950" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppmd-rust" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d558c559f0450f16f2a27a1f017ef38468c1090c9ce63c8e51366232d53717b4" + +[[package]] +name = "proc-macro2" +version = "1.0.105" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "535d180e0ecab6268a3e718bb9fd44db66bbbc256257165fc699dadf70d16fe7" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc74d9a594b72ae6656596548f56f667211f8a97b3d4c3d467150794690dc40a" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "r-efi" +version = "5.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" + +[[package]] +name = "rayon" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "rustix" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "146c9e247ccc180c1f61615433868c99f3de3ae256a30a43b49f67c2d9171f34" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys", +] + +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "serde" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" +dependencies = [ + "serde_core", + "serde_derive", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.149" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +dependencies = [ + "itoa", + "memchr", + "serde", + "serde_core", + "zmij", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "simd-adler32" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e320a6c5ad31d271ad523dcf3ad13e2767ad8b1cb8f047f75a8aeaf8da139da2" + +[[package]] +name = "slab" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" + +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4d107df263a3013ef9b1879b0df87d706ff80f65a86ea879bd9c31f9b307c2a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tempfile" +version = "3.24.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "655da9c7eb6305c55742045d5a8d2037996d61d8de95806335c7c86ce0f82e9c" +dependencies = [ + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys", +] + +[[package]] +name = "time" +version = "0.3.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9e442fc33d7fdb45aa9bfeb312c095964abdf596f7567261062b2a7107aaabd" +dependencies = [ + "deranged", + "itoa", + "js-sys", + "num-conv", + "powerfmt", + "serde_core", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b36ee98fd31ec7426d599183e8fe26932a8dc1fb76ddb6214d05493377d34ca" + +[[package]] +name = "time-macros" +version = "0.2.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e552d1249bf61ac2a52db88179fd0673def1e1ad8243a00d9ec9ed71fee3dd" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "typed-path" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7922f2cdc51280d47b491af9eafc41eb0cdab85eabcb390c854412fcbf26dbe8" + +[[package]] +name = "typenum" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" + +[[package]] +name = "unicode-ident" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasip2" +version = "1.0.2+wasi-0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9517f9239f02c069db75e65f174b3da828fe5f5b945c4dd26bd25d89c03ebcf5" +dependencies = [ + "wit-bindgen", +] + +[[package]] +name = "wasm-bindgen" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64024a30ec1e37399cf85a7ffefebdb72205ca1c972291c51512360d90bd8566" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" +dependencies = [ + "cfg-if", + "futures-util", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "008b239d9c740232e71bd39e8ef6429d27097518b6b30bdf9086833bd5b6d608" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5256bae2d58f54820e6490f9839c49780dff84c65aeab9e772f15d5f0e913a55" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f01b580c9ac74c8d8f0c0e4afb04eeef2acf145458e52c03845ee9cd23e3d12" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "wasm-bindgen-test" +version = "0.3.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45649196a53b0b7a15101d845d44d2dda7374fc1b5b5e2bbf58b7577ff4b346d" +dependencies = [ + "async-trait", + "cast", + "js-sys", + "libm", + "minicov", + "nu-ansi-term", + "num-traits", + "oorandom", + "serde", + "serde_json", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test-macro", + "wasm-bindgen-test-shared", +] + +[[package]] +name = "wasm-bindgen-test-macro" +version = "0.3.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f579cdd0123ac74b94e1a4a72bd963cf30ebac343f2df347da0b8df24cdebed2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "wasm-bindgen-test-shared" +version = "0.2.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8145dd1593bf0fb137dbfa85b8be79ec560a447298955877804640e40c2d6ea" + +[[package]] +name = "web-sys" +version = "0.3.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + +[[package]] +name = "wit-bindgen" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zip" +version = "7.2.0" +dependencies = [ + "aes", + "arbitrary", + "bencher", + "bitstream-io", + "bzip2", + "chrono", + "clap", + "constant_time_eq", + "crc32fast", + "deflate64", + "flate2", + "generic-array", + "getrandom", + "hmac", + "indexmap", + "jiff", + "lzma-rust2", + "memchr", + "nt-time", + "pbkdf2", + "ppmd-rust", + "rayon", + "sha1", + "tempfile", + "time", + "typed-path", + "walkdir", + "wasm-bindgen-test", + "zeroize", + "zopfli", + "zstd", +] + +[[package]] +name = "zlib-rs" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40990edd51aae2c2b6907af74ffb635029d5788228222c4bb811e9351c0caad3" + +[[package]] +name = "zmij" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfcd145825aace48cff44a8844de64bf75feec3080e0aa5cdbde72961ae51a65" + +[[package]] +name = "zopfli" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249" +dependencies = [ + "bumpalo", + "crc32fast", + "log", + "simd-adler32", +] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/v2/vendor/security/zip-7.2.0/Cargo.toml b/v2/vendor/security/zip-7.2.0/Cargo.toml new file mode 100644 index 0000000000..91548c5cf5 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/Cargo.toml @@ -0,0 +1,330 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +rust-version = "1.83.0" +name = "zip" +version = "7.2.0" +authors = [ + "Mathijs van de Nes ", + "Marli Frost ", + "Ryan Levick ", + "Chris Hennick ", +] +build = false +exclude = [ + "tests/**", + "examples/**", + ".github/**", + "fuzz_read/**", + "fuzz_write/**", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = """ +Library to support the reading and writing of zip files. +""" +readme = "README.md" +keywords = [ + "zip", + "archive", + "compression", +] +categories = [ + "compression", + "filesystem", + "parser-implementations", +] +license = "MIT" +repository = "https://github.com/zip-rs/zip2.git" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = [ + "--cfg", + "docsrs", +] + +[features] +_all-features = [] +_arbitrary = ["dep:arbitrary"] +_bzip2_any = [] +_deflate-any = [] +aes-crypto = [ + "dep:aes", + "dep:constant_time_eq", + "dep:generic-array", + "getrandom/std", + "dep:hmac", + "dep:pbkdf2", + "dep:sha1", + "dep:zeroize", +] +bzip2 = [ + "dep:bzip2", + "bzip2/default", + "_bzip2_any", +] +bzip2-rs = [ + "dep:bzip2", + "bzip2/bzip2-sys", + "_bzip2_any", +] +chrono = ["dep:chrono"] +default = [ + "aes-crypto", + "bzip2", + "deflate64", + "deflate", + "lzma", + "ppmd", + "time", + "zstd", + "xz", +] +deflate = [ + "deflate-zopfli", + "deflate-flate2-zlib-rs", +] +deflate-flate2 = [ + "_deflate-any", + "dep:flate2", +] +deflate-flate2-zlib = [ + "deflate-flate2", + "flate2/zlib", +] +deflate-flate2-zlib-ng = [ + "deflate-flate2", + "flate2/zlib-ng", +] +deflate-flate2-zlib-ng-compat = [ + "deflate-flate2", + "flate2/zlib-ng-compat", +] +deflate-flate2-zlib-rs = [ + "deflate-flate2", + "flate2/zlib-rs", +] +deflate-zopfli = [ + "dep:zopfli", + "_deflate-any", +] +jiff-02 = ["dep:jiff"] +legacy-zip = ["bitstream-io"] +lzma = ["dep:lzma-rust2"] +nt-time = ["dep:nt-time"] +ppmd = ["dep:ppmd-rust"] +unreserved = [] +xz = ["dep:lzma-rust2"] + +[lib] +name = "zip" +path = "src/lib.rs" + +[[bench]] +name = "merge_archive" +path = "benches/merge_archive.rs" +harness = false + +[[bench]] +name = "read_entry" +path = "benches/read_entry.rs" +harness = false + +[[bench]] +name = "read_metadata" +path = "benches/read_metadata.rs" +harness = false + +[dependencies.aes] +version = "0.8" +optional = true + +[dependencies.arbitrary] +version = "1.4" +features = ["derive"] +optional = true + +[dependencies.bitstream-io] +version = "4.5" +optional = true + +[dependencies.bzip2] +version = "0.6" +optional = true +default-features = false + +[dependencies.chrono] +version = "^0.4.27" +features = ["std"] +optional = true +default-features = false + +[dependencies.constant_time_eq] +version = "0.3" +optional = true + +[dependencies.crc32fast] +version = "1.5" + +[dependencies.deflate64] +version = "0.1.10" +optional = true + +[dependencies.flate2] +version = "1.1" +optional = true +default-features = false + +[dependencies.generic-array] +version = "=0.14.7" +optional = true + +[dependencies.hmac] +version = "0.12" +features = ["reset"] +optional = true + +[dependencies.indexmap] +version = "2" + +[dependencies.jiff] +version = "0.2.4" +optional = true + +[dependencies.lzma-rust2] +version = "0.15" +features = [ + "std", + "encoder", + "optimization", + "xz", +] +optional = true +default-features = false + +[dependencies.memchr] +version = "2.7" + +[dependencies.nt-time] +version = "0.10.6" +optional = true +default-features = false + +[dependencies.pbkdf2] +version = "0.12" +optional = true + +[dependencies.ppmd-rust] +version = "1.3" +optional = true + +[dependencies.sha1] +version = "0.10" +optional = true + +[dependencies.time] +version = "0.3" +features = ["std"] +optional = true +default-features = false + +[dependencies.typed-path] +version = "0.12" + +[dependencies.zeroize] +version = "1.8" +features = ["zeroize_derive"] +optional = true + +[dependencies.zopfli] +version = "0.8" +optional = true + +[dependencies.zstd] +version = "^0.13.3" +optional = true +default-features = false + +[dev-dependencies.bencher] +version = "0.1" + +[dev-dependencies.clap] +version = "=4.4.18" +features = ["derive"] + +[dev-dependencies.getrandom] +version = "0.3" +features = [] +default-features = false + +[dev-dependencies.rayon] +version = "1.11" + +[dev-dependencies.tempfile] +version = "3.15" + +[dev-dependencies.time] +version = "0.3" +features = [ + "formatting", + "macros", +] + +[dev-dependencies.walkdir] +version = "2.5" + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies.getrandom] +version = "0.3" +optional = true +default-features = false + +[target.'cfg(not(target_arch = "wasm32"))'.dependencies.time] +version = "0.3" +features = ["wasm-bindgen"] +optional = true +default-features = false + +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies.getrandom] +version = "0.3" +default-features = false + +[target.'cfg(target_arch = "wasm32")'.dependencies.getrandom] +version = "0.3" +features = ["wasm_js"] +optional = true +default-features = false + +[target.'cfg(target_arch = "wasm32")'.dependencies.time] +version = "0.3" +features = [ + "std", + "wasm-bindgen", +] +optional = true +default-features = false + +[target.'cfg(target_arch = "wasm32")'.dev-dependencies.getrandom] +version = "0.3" +features = ["wasm_js"] +default-features = false + +[target.'cfg(target_arch = "wasm32")'.dev-dependencies.wasm-bindgen-test] +version = "^0.3.56" + +[[test]] +name = "ruview_0429_backport" +path = "tests/ruview_0429_backport.rs" diff --git a/v2/vendor/security/zip-7.2.0/Cargo.toml.orig b/v2/vendor/security/zip-7.2.0/Cargo.toml.orig new file mode 100644 index 0000000000..761bf24a7e --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/Cargo.toml.orig @@ -0,0 +1,139 @@ +[package] +name = "zip" +version = "7.2.0" +authors = [ + "Mathijs van de Nes ", + "Marli Frost ", + "Ryan Levick ", + "Chris Hennick ", +] +license = "MIT" +repository = "https://github.com/zip-rs/zip2.git" +keywords = ["zip", "archive", "compression"] +# Any change to rust-version must be reflected also in `README.md` and `.github/workflows/ci.yaml`. +# The MSRV policy is documented in `README.md`. +rust-version = "1.83.0" +categories = ["compression", "filesystem", "parser-implementations"] +description = """ +Library to support the reading and writing of zip files. +""" +edition = "2021" +exclude = ["tests/**", "examples/**", ".github/**", "fuzz_read/**", "fuzz_write/**"] + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] + +[dependencies] +arbitrary = { version = "1.4", features = ["derive"], optional = true } +aes = { version = "0.8", optional = true } +bzip2 = { version = "0.6", default-features = false, optional = true } +chrono = { version = "^0.4.27", optional = true, default-features = false, features = ["std"] } +constant_time_eq = { version = "0.3", optional = true } +crc32fast = "1.5" +flate2 = { version = "1.1", default-features = false, optional = true } +generic-array = { version = "=0.14.7", optional = true } +hmac = { version = "0.12", optional = true, features = ["reset"] } +indexmap = "2" +jiff = { version = "0.2.4", optional = true } +memchr = "2.7" +nt-time = { version = "0.10.6", default-features = false, optional = true } +ppmd-rust = { version = "1.3", optional = true } +pbkdf2 = { version = "0.12", optional = true } +sha1 = { version = "0.10", optional = true } +time = { version = "0.3", default-features = false, optional = true, features = [ + "std", +] } +zeroize = { version = "1.8", optional = true, features = ["zeroize_derive"] } +zstd = { version = "^0.13.3", optional = true, default-features = false } +zopfli = { version = "0.8", optional = true } +deflate64 = { version = "0.1.10", optional = true } +lzma-rust2 = { version = "0.15", optional = true, default-features = false, features = ["std", "encoder", "optimization", "xz"] } +bitstream-io = { version = "4.5", optional = true } +typed-path = "0.12" + +# native: +[target.'cfg(not(target_arch = "wasm32"))'.dependencies] +getrandom = { version = "0.3", default-features = false, optional = true } +time = { version = "0.3", default-features = false, optional = true, features = [ + "wasm-bindgen" +] } + +[target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] +getrandom = { version = "0.3", default-features = false } + +# wasm32 +[target.'cfg(target_arch = "wasm32")'.dependencies] +getrandom = { version = "0.3", default-features = false, features = ["wasm_js"], optional = true } +time = { version = "0.3", default-features = false, optional = true, features = [ + "std", + "wasm-bindgen" +] } + +[target.'cfg(target_arch = "wasm32")'.dev-dependencies] +getrandom = { version = "0.3", default-features = false, features = ["wasm_js"] } +wasm-bindgen-test = "^0.3.56" + +[dev-dependencies] +bencher = "0.1" +getrandom = { version = "0.3", default-features = false, features = [] } +walkdir = "2.5" +time = { version = "0.3", features = ["formatting", "macros"] } +clap = { version = "=4.4.18", features = ["derive"] } +tempfile = "3.15" +rayon = "1.11" + +[features] +_arbitrary = ["dep:arbitrary"] +aes-crypto = ["dep:aes", "dep:constant_time_eq", "dep:generic-array", "getrandom/std", "dep:hmac", "dep:pbkdf2", "dep:sha1", "dep:zeroize"] +chrono = ["dep:chrono"] +_deflate-any = [] +_all-features = [] # Detect when --all-features is used +deflate = ["deflate-zopfli", "deflate-flate2-zlib-rs"] +# Pull in flate2, but don't choose a backend; useful if you want to choose your own flate2 backend +deflate-flate2 = ["_deflate-any", "dep:flate2"] +# Pull in flate2 and the fast zlib-rs backend; this is what most users will want +deflate-flate2-zlib-rs = ["deflate-flate2", "flate2/zlib-rs"] +# Pull in zopfli (write-only DEFLATE, slower than flate2 with better compression ratios) +deflate-zopfli = ["dep:zopfli", "_deflate-any"] +jiff-02 = ["dep:jiff"] +nt-time = ["dep:nt-time"] +lzma = ["dep:lzma-rust2"] +ppmd = ["dep:ppmd-rust"] +unreserved = [] +xz = ["dep:lzma-rust2"] +_bzip2_any = [] +bzip2 = ["dep:bzip2", "bzip2/default", "_bzip2_any"] +legacy-zip = ["bitstream-io"] +default = [ + "aes-crypto", + "bzip2", + "deflate64", + "deflate", + "lzma", + "ppmd", + "time", + "zstd", + "xz", +] + +# The following features are not supported on wasm +bzip2-rs = ["dep:bzip2", "bzip2/bzip2-sys", "_bzip2_any"] +# Pull in flate2 and the zlib backend; only use this if you need a dynamically linked system zlib +deflate-flate2-zlib = ["deflate-flate2", "flate2/zlib"] +# Pull in flate2 and the zlib-ng backend; a modern fork of zlib +deflate-flate2-zlib-ng = ["deflate-flate2", "flate2/zlib-ng"] +# Pull in flate2 and the zlib-ng backend with a zlib-compatible API +deflate-flate2-zlib-ng-compat = ["deflate-flate2", "flate2/zlib-ng-compat"] + +[[bench]] +name = "read_entry" +harness = false + +[[bench]] +name = "read_metadata" +harness = false + +[[bench]] +name = "merge_archive" +harness = false diff --git a/v2/vendor/security/zip-7.2.0/LICENSE b/v2/vendor/security/zip-7.2.0/LICENSE new file mode 100644 index 0000000000..4a45976f88 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/LICENSE @@ -0,0 +1,24 @@ +The MIT License (MIT) + +Copyright (c) 2014 Mathijs van de Nes + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +Some files in the "tests/data" subdirectory of this repository are under other +licences; see files named LICENSE.*.txt for details. \ No newline at end of file diff --git a/v2/vendor/security/zip-7.2.0/README.md b/v2/vendor/security/zip-7.2.0/README.md new file mode 100644 index 0000000000..1459e9b852 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/README.md @@ -0,0 +1,145 @@ +zip +=== + +[![Build Status](https://github.com/zip-rs/zip2/actions/workflows/ci.yaml/badge.svg)](https://github.com/zip-rs/zip2/actions?query=branch%3Amaster+workflow%3ACI) +[![Crates.io version](https://img.shields.io/crates/v/zip.svg)](https://crates.io/crates/zip) + +[Documentation](https://docs.rs/zip/latest/zip/) + +# Info + +A zip library for rust which supports reading and writing of simple ZIP files. Formerly hosted at +https://github.com/zip-rs/zip2. + +Supported compression formats: + +* stored (i.e. none) +* deflate +* deflate64 (decompression only) +* bzip2 +* zstd +* lzma (decompression only) +* xz +* ppmd + +Currently unsupported zip extensions: + +* Multi-disk + +# Features + +The features available are: + +* `aes-crypto`: Enables decryption of files which were encrypted with AES. Supports AE-1 and AE-2 methods. +* `deflate`: Enables compressing and decompressing an unspecified implementation (that may change in future versions) of + the deflate compression algorithm, which is the default for zip files. Supports compression quality 1..=264. +* `deflate-flate2`: Combine this with any `flate2` feature flag that enables a back-end, to support deflate compression + at quality 1..=9. +* `deflate-zopfli`: Enables deflating files with the `zopfli` library (used when compression quality is 10..=264). This + is the most effective `deflate` implementation available, but also among the slowest. If `flate2` isn't also enabled, + only compression will be supported and not decompression. +* `deflate64`: Enables the deflate64 compression algorithm. Only decompression is supported. +* `lzma`: Enables the LZMA compression algorithm. Only decompression is supported. +* `bzip2`: Enables the BZip2 compression algorithm. +* `ppmd`: Enables the PPMd compression algorithm. +* `time`: Enables features using the [time](https://github.com/rust-lang-deprecated/time) crate. +* `chrono`: Enables converting last-modified `zip::DateTime` to and from `chrono::NaiveDateTime`. +* `jiff-02`: Enables converting last-modified `zip::DateTime` to and from `jiff::civil::DateTime`. +* `nt-time`: Enables returning timestamps stored in the NTFS extra field as `nt_time::FileTime`. +* `xz`: Enables the XZ compression algorithm. +* `zstd`: Enables the Zstandard compression algorithm. + +By default `aes-crypto`, `bzip2`, `deflate`, `deflate64`, `lzma`, `ppmd`, `time`, `xz` and `zstd` are enabled. + +# MSRV + +Our current Minimum Supported Rust Version is **1.83**. When adding features, +we will follow these guidelines: + +- We will always support a minor Rust version that has been stable for at least 6 months. +- Any change to the MSRV will be accompanied with a **minor** version bump. + +# Examples + +See the [examples directory](examples) for: + +* How to write a file to a zip. +* How to write a directory of files to a zip (using [walkdir](https://github.com/BurntSushi/walkdir)). +* How to extract a zip file. +* How to extract a single file from a zip. +* How to read a zip from the standard input. +* How to append a directory to an existing archive + +# Fuzzing + +Fuzzing support is through [`cargo afl`](https://rust-fuzz.github.io/book/afl/tutorial.html). To install `cargo afl`: + +```bash +cargo install cargo-afl +``` + +To start fuzzing zip extraction: + +```bash +mkdir -vp fuzz-read-out +cargo afl build --manifest-path=fuzz/Cargo.toml --all-features -p fuzz_read +# Curated input corpus: +cargo afl fuzz -i fuzz/read/in -o fuzz-read-out fuzz/target/debug/fuzz_read +# Test data files: +cargo afl fuzz -i tests/data -e zip -o fuzz-read-out fuzz/target/debug/fuzz_read +``` + +To start fuzzing zip creation: + +```bash +mkdir -vp fuzz-write-out +cargo afl build --manifest-path=fuzz/Cargo.toml --all-features -p fuzz_write +# Curated input corpus and dictionary schema: +cargo afl fuzz -x fuzz/write/fuzz.dict -i fuzz/write/in -o fuzz-write-out fuzz/target/debug/fuzz_write +``` + +## Fuzzing stdio + +The read and write fuzzers can also receive input over stdin for one-off validation. Note here that the fuzzers can be configured to build in support for DEFLATE, or not: +```bash +# Success, no output: +cargo run --manifest-path=fuzz/Cargo.toml --quiet --all-features -p fuzz_read MAX_SYMLINK_TARGET_LEN { ++ return Err(invalid!( ++ "symlink target declares {} bytes, more than the {} byte limit", ++ declared_len, ++ MAX_SYMLINK_TARGET_LEN ++ )); ++ } ++ let mut target = Vec::with_capacity(declared_len as usize); + file.read_to_end(&mut target)?; + drop(file); + make_symlink(&outpath, &target, &self.shared.files)?; +--- a/src/read/stream.rs ++++ b/src/read/stream.rs +@@ -1,4 +1,7 @@ ++use crate::result::invalid; ++ + use super::{ ++ MAX_SYMLINK_TARGET_LEN, + central_header_to_zip_file_inner, make_symlink, read_zipfile_from_stream, ZipCentralEntryBlock, + ZipFile, ZipFileData, ZipResult, + }; +@@ -67,7 +70,15 @@ + file.safe_prepare_path(&self.0, &mut outpath, None::<&(_, fn(&Path) -> bool)>)?; + + if file.is_symlink() { +- let mut target = Vec::with_capacity(file.size() as usize); ++ let declared_len = file.size(); ++ if declared_len > MAX_SYMLINK_TARGET_LEN { ++ return Err(invalid!( ++ "symlink target declares {} bytes, more than the {} byte limit", ++ declared_len, ++ MAX_SYMLINK_TARGET_LEN ++ )); ++ } ++ let mut target = Vec::with_capacity(declared_len as usize); + file.read_to_end(&mut target)?; + make_symlink(&outpath, &target, &self.1)?; + return Ok(()); diff --git a/v2/vendor/security/zip-7.2.0/RUVIEW-BACKPORT.md b/v2/vendor/security/zip-7.2.0/RUVIEW-BACKPORT.md new file mode 100644 index 0000000000..612944437a --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/RUVIEW-BACKPORT.md @@ -0,0 +1,35 @@ +# RuView ZIP 7.2.0 security backport + +This is the official crates.io ZIP 7.2.0 source package with the final upstream +[ZIP #984](https://github.com/zip-rs/zip2/pull/984) fix adapted to its archive and +stream readers. Version and licenses remain unchanged. Source/archive/fix hashes +are recorded in `ruview-provenance.json`; the production diff is +`RUSTSEC-2026-0429.patch` (two files). The only additional manifest entry enables +our self-contained public-API regression target. No Candle/framework snapshot or +RuView submodule revision changes are required. + +The patch rejects symlink target lengths over4096 before allocation. The exact +112-byte RUSTSEC-2026-0429 example aborts pristine7.2.0 and is rejected with +InvalidArchive here. Dedicated tests cover hostile lengths, the stream flag, +ordinary files and valid relative symlinks. A removed-guard mutation fails. + +Run from the repository root: + +```sh +cargo test --manifest-path v2/vendor/security/zip-7.2.0/Cargo.toml --locked --no-default-features --features deflate --test ruview_0429_backport +``` + +The registry package omits large upstream test archives. Therefore this package +is excluded from the parent workspace's unit-test membership; the explicit +regression target above is mandatory in security CI. Full upstream library and +selected integration tests were separately executed with fixtures from package +VCS46dc29cda6749736ddb1ec502aff06a65ffe54f8; they are not claimed as part of this +small committed fixture set. + +cargo-audit0.22.2 excludes path sources from registry advisory matching. A green +audit alone does not establish this patch's correctness; executable regressions, +provenance and review do. No advisory policy or exception was added or broadened. +This addresses0429 only; existing0367/0368/0369 reachability decisions remain +unchanged. Retire this backport when a reviewed compatible upstream upgrade is +available. Current Candle0.9 requires ZIP7, while moving the swarm submodule to +its new upstream workspace would be a separate integration change. diff --git a/v2/vendor/security/zip-7.2.0/SECURITY.md b/v2/vendor/security/zip-7.2.0/SECURITY.md new file mode 100644 index 0000000000..031b4cf473 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/SECURITY.md @@ -0,0 +1,36 @@ +# Security Policy + +## Supported Versions + +Use this section to tell people about which versions of your project are +currently being supported with security updates. + +| Version | Supported | +| ------- | ------------------ | +| 2.4.x | :white_check_mark: | +| 2.3.x | :white_check_mark: | +| < 2.0 | :x: | + +## Reporting a Vulnerability + +To report a vulnerability, please go to https://github.com/zip-rs/zip2/security/advisories/new. We'll attempt to: + +* Close the report within 7 days if it's invalid, or if a fix has already been released but some old versions needed to be yanked. +* Provide progress reports at least every 7 days to the original reporter. +* Fix vulnerabilities within 30 days of the initial report. + +## Disclosure + +A vulnerability will only be publicly disclosed once a fix is released. At that point, the delay before full public disclosure +will be determined as follows: + +* If the proof-of-concept is very simple, or an exploit is already in the wild (whether or not it specifically targets `zip`, + all details will be made public right away. +* If the vulnerability is specific to `zip` and cannot easily be reverse-engineered from the code history, then the + proof-of-concept and most of the details will be withheld until 14 days after the fix is released and all vulnerable + versions are yanked with `cargo yank`. +* If a potential victim requests more time to deploy a fix based on a credible risk, then the withholding of details can + be extended up to 30 days. This may be extended to 90 days if the victim is high-value (e.g. manages over US$1 billion + worth of financial assets or intellectual property, or has evidence that they're a target of nation-state attackers) + and there's a valid reason why they cannot deploy the fix as fast as most users (e.g. heavy reliance on an old version's + interface, or infrastructure damage in a war zone). diff --git a/v2/vendor/security/zip-7.2.0/benches/merge_archive.rs b/v2/vendor/security/zip-7.2.0/benches/merge_archive.rs new file mode 100644 index 0000000000..9e5598f7dd --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/benches/merge_archive.rs @@ -0,0 +1,130 @@ +use bencher::{benchmark_group, benchmark_main}; + +use std::io::{Cursor, Read, Seek, Write}; + +use bencher::Bencher; +use zip::{result::ZipResult, write::SimpleFileOptions, ZipArchive, ZipWriter}; + +fn generate_random_archive( + num_entries: usize, + entry_size: usize, + options: SimpleFileOptions, +) -> ZipResult<(usize, ZipArchive>>)> { + let buf = Cursor::new(Vec::new()); + let mut zip = ZipWriter::new(buf); + + let mut bytes = vec![0u8; entry_size]; + for i in 0..num_entries { + let name = format!("random{i}.dat"); + zip.start_file(name, options)?; + getrandom::fill(&mut bytes).unwrap(); + zip.write_all(&bytes)?; + } + + let buf = zip.finish()?.into_inner(); + let len = buf.len(); + + Ok((len, ZipArchive::new(Cursor::new(buf))?)) +} + +fn perform_merge( + src: ZipArchive, + mut target: ZipWriter, +) -> ZipResult> { + target.merge_archive(src)?; + Ok(target) +} + +fn perform_raw_copy_file( + mut src: ZipArchive, + mut target: ZipWriter, +) -> ZipResult> { + for i in 0..src.len() { + let entry = src.by_index(i)?; + target.raw_copy_file(entry)?; + } + Ok(target) +} + +const NUM_ENTRIES: usize = 100; +const ENTRY_SIZE: usize = 1024; + +fn merge_archive_stored(bench: &mut Bencher) { + let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + let (len, src) = generate_random_archive(NUM_ENTRIES, ENTRY_SIZE, options).unwrap(); + + bench.bytes = len as u64; + + bench.iter(|| { + let buf = Cursor::new(Vec::new()); + let zip = ZipWriter::new(buf); + let zip = perform_merge(src.clone(), zip).unwrap(); + let buf = zip.finish().unwrap().into_inner(); + assert_eq!(buf.len(), len); + }); +} + +#[cfg(feature = "_deflate-any")] +fn merge_archive_compressed(bench: &mut Bencher) { + let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); + let (len, src) = generate_random_archive(NUM_ENTRIES, ENTRY_SIZE, options).unwrap(); + + bench.bytes = len as u64; + + bench.iter(|| { + let buf = Cursor::new(Vec::new()); + let zip = ZipWriter::new(buf); + let zip = perform_merge(src.clone(), zip).unwrap(); + let buf = zip.finish().unwrap().into_inner(); + assert_eq!(buf.len(), len); + }); +} + +fn merge_archive_raw_copy_file_stored(bench: &mut Bencher) { + let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + let (len, src) = generate_random_archive(NUM_ENTRIES, ENTRY_SIZE, options).unwrap(); + + bench.bytes = len as u64; + + bench.iter(|| { + let buf = Cursor::new(Vec::new()); + let zip = ZipWriter::new(buf); + let zip = perform_raw_copy_file(src.clone(), zip).unwrap(); + let buf = zip.finish().unwrap().into_inner(); + assert_eq!(buf.len(), len); + }); +} + +#[cfg(feature = "_deflate-any")] +fn merge_archive_raw_copy_file_compressed(bench: &mut Bencher) { + let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Deflated); + let (len, src) = generate_random_archive(NUM_ENTRIES, ENTRY_SIZE, options).unwrap(); + + bench.bytes = len as u64; + + bench.iter(|| { + let buf = Cursor::new(Vec::new()); + let zip = ZipWriter::new(buf); + let zip = perform_raw_copy_file(src.clone(), zip).unwrap(); + let buf = zip.finish().unwrap().into_inner(); + assert_eq!(buf.len(), len); + }); +} + +#[cfg(feature = "_deflate-any")] +benchmark_group!( + benches, + merge_archive_stored, + merge_archive_compressed, + merge_archive_raw_copy_file_stored, + merge_archive_raw_copy_file_compressed, +); + +#[cfg(not(feature = "_deflate-any"))] +benchmark_group!( + benches, + merge_archive_stored, + merge_archive_raw_copy_file_stored, +); + +benchmark_main!(benches); diff --git a/v2/vendor/security/zip-7.2.0/benches/read_entry.rs b/v2/vendor/security/zip-7.2.0/benches/read_entry.rs new file mode 100644 index 0000000000..8fd4ef3a5c --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/benches/read_entry.rs @@ -0,0 +1,41 @@ +use bencher::{benchmark_group, benchmark_main}; + +use std::io::{Cursor, Read, Write}; + +use bencher::Bencher; +use zip::{write::SimpleFileOptions, ZipArchive, ZipWriter}; + +fn generate_random_archive(size: usize) -> Vec { + let data = Vec::new(); + let mut writer = ZipWriter::new(Cursor::new(data)); + let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + + writer.start_file("random.dat", options).unwrap(); + let mut bytes = vec![0u8; size]; + getrandom::fill(&mut bytes).unwrap(); + writer.write_all(&bytes).unwrap(); + + writer.finish().unwrap().into_inner() +} + +fn read_entry(bench: &mut Bencher) { + let size = 1024 * 1024; + let bytes = generate_random_archive(size); + let mut archive = ZipArchive::new(Cursor::new(bytes.as_slice())).unwrap(); + + bench.iter(|| { + let mut file = archive.by_name("random.dat").unwrap(); + let mut buf = [0u8; 1024]; + loop { + let n = file.read(&mut buf).unwrap(); + if n == 0 { + break; + } + } + }); + + bench.bytes = size as u64; +} + +benchmark_group!(benches, read_entry); +benchmark_main!(benches); diff --git a/v2/vendor/security/zip-7.2.0/benches/read_metadata.rs b/v2/vendor/security/zip-7.2.0/benches/read_metadata.rs new file mode 100644 index 0000000000..06b4aa397c --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/benches/read_metadata.rs @@ -0,0 +1,143 @@ +use bencher::{benchmark_group, benchmark_main}; + +use std::fs; +use std::io::{self, Cursor, Write}; + +use bencher::Bencher; +use tempfile::TempDir; +use zip::write::SimpleFileOptions; +use zip::{result::ZipResult, CompressionMethod, ZipArchive, ZipWriter}; + +const FILE_COUNT: usize = 15_000; +const FILE_SIZE: usize = 1024; + +fn generate_random_archive(count_files: usize, file_size: usize) -> ZipResult> { + let data = Vec::new(); + let mut writer = ZipWriter::new(Cursor::new(data)); + let options = SimpleFileOptions::default().compression_method(CompressionMethod::Stored); + + let mut bytes = vec![0u8; file_size]; + + for i in 0..count_files { + let name = format!("file_deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef_{i}.dat"); + writer.start_file(name, options)?; + getrandom::fill(&mut bytes).map_err(io::Error::from)?; + writer.write_all(&bytes)?; + } + + Ok(writer.finish()?.into_inner()) +} + +fn read_metadata(bench: &mut Bencher) { + let bytes = generate_random_archive(FILE_COUNT, FILE_SIZE).unwrap(); + + bench.iter(|| { + let archive = ZipArchive::new(Cursor::new(bytes.as_slice())).unwrap(); + archive.len() + }); + bench.bytes = bytes.len() as u64; +} + +const COMMENT_SIZE: usize = 50_000; + +fn generate_zip32_archive_with_random_comment(comment_length: usize) -> ZipResult> { + let data = Vec::new(); + let mut writer = ZipWriter::new(Cursor::new(data)); + let options = SimpleFileOptions::default().compression_method(CompressionMethod::Stored); + + let mut bytes = vec![0u8; comment_length]; + getrandom::fill(&mut bytes).unwrap(); + writer.set_raw_comment(bytes.into_boxed_slice()); + + writer.start_file("asdf.txt", options)?; + writer.write_all(b"asdf")?; + + Ok(writer.finish()?.into_inner()) +} + +fn parse_archive_with_comment(bench: &mut Bencher) { + let bytes = generate_zip32_archive_with_random_comment(COMMENT_SIZE).unwrap(); + + bench.bench_n(1, |_| { + let archive = ZipArchive::new(Cursor::new(bytes.as_slice())).unwrap(); + let _ = archive.comment().len(); + }); + bench.bytes = bytes.len() as u64; +} + +const COMMENT_SIZE_64: usize = 500_000; + +fn generate_zip64_archive_with_random_comment(comment_length: usize) -> ZipResult> { + let data = Vec::new(); + let mut writer = ZipWriter::new(Cursor::new(data)); + let options = SimpleFileOptions::default() + .compression_method(CompressionMethod::Stored) + .large_file(true); + + let mut bytes = vec![0u8; comment_length]; + getrandom::fill(&mut bytes).unwrap(); + // should use the line below but still works + // writer.set_raw_zip64_comment(Some(comment)); + writer.set_raw_comment(bytes.into_boxed_slice()); + + writer.start_file("asdf.txt", options)?; + writer.write_all(b"asdf")?; + + Ok(writer.finish()?.into_inner()) +} + +fn parse_zip64_archive_with_comment(bench: &mut Bencher) { + let bytes = generate_zip64_archive_with_random_comment(COMMENT_SIZE_64).unwrap(); + + bench.iter(|| { + let archive = ZipArchive::new(Cursor::new(bytes.as_slice())).unwrap(); + archive.comment().len() + }); + bench.bytes = bytes.len() as u64; +} + +fn parse_stream_archive(bench: &mut Bencher) { + const STREAM_ZIP_ENTRIES: usize = 5; + const STREAM_FILE_SIZE: usize = 5; + + let bytes = generate_random_archive(STREAM_ZIP_ENTRIES, STREAM_FILE_SIZE).unwrap(); + + /* Write to a temporary file path to incur some filesystem overhead from repeated reads */ + let dir = TempDir::with_prefix("stream-bench").unwrap(); + let out = dir.path().join("bench-out.zip"); + fs::write(&out, &bytes).unwrap(); + + bench.iter(|| { + let mut f = fs::File::open(&out).unwrap(); + while zip::read::read_zipfile_from_stream(&mut f) + .unwrap() + .is_some() + {} + }); + bench.bytes = bytes.len() as u64; +} + +fn parse_large_non_zip(bench: &mut Bencher) { + const FILE_SIZE: usize = 17_000_000; + + // Create a large file that doesn't have a zip header (generating random data _might_ make a zip magic + // number somewhere which is _not_ what we're trying to test). + let dir = TempDir::with_prefix("large-non-zip-bench").unwrap(); + let file = dir.path().join("zeros"); + let buf = vec![0u8; FILE_SIZE]; + fs::write(&file, &buf).unwrap(); + + bench.iter(|| { + assert!(zip::ZipArchive::new(std::fs::File::open(&file).unwrap()).is_err()); + }) +} + +benchmark_group!( + benches, + read_metadata, + parse_archive_with_comment, + parse_zip64_archive_with_comment, + parse_stream_archive, + parse_large_non_zip, +); +benchmark_main!(benches); diff --git a/v2/vendor/security/zip-7.2.0/cliff.toml b/v2/vendor/security/zip-7.2.0/cliff.toml new file mode 100644 index 0000000000..5c1ceed58a --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/cliff.toml @@ -0,0 +1,91 @@ +# git-cliff ~ default configuration file +# https://git-cliff.org/docs/configuration +# +# Lines starting with "#" are comments. +# Configuration options are organized into tables and keys. +# See documentation for more information on available options. + +[changelog] +# changelog header +header = """ +# Changelog\n +All notable changes to this project will be documented in this file.\n +""" +# template for the changelog body +# https://keats.github.io/tera/docs/#introduction +body = """ +{% if version %}\ + ## [{{ version | trim_start_matches(pat="v") }}] - {{ timestamp | date(format="%Y-%m-%d") }} +{% else %}\ + ## [unreleased] +{% endif %}\ +{% for group, commits in commits | group_by(attribute="group") %} + ### {{ group | striptags | trim | upper_first }} + {% for commit in commits %} + - {% if commit.scope %}*({{ commit.scope }})* {% endif %}\ + {% if commit.breaking %}[**breaking**] {% endif %}\ + {{ commit.message | upper_first }}\ + {% endfor %} +{% endfor %}\n +""" +# template for the changelog footer +footer = """ + +""" +# remove the leading and trailing s +trim = true +# postprocessors +postprocessors = [ + # { pattern = '', replace = "https://github.com/orhun/git-cliff" }, # replace repository URL +] + +[git] +# parse the commits based on https://www.conventionalcommits.org +conventional_commits = true +# filter out the commits that are not conventional +filter_unconventional = true +# process each line of a commit as an individual commit +split_commits = false +# regex for preprocessing the commit messages +commit_preprocessors = [ + # Replace issue numbers + #{ pattern = '\((\w+\s)?#([0-9]+)\)', replace = "([#${2}](/issues/${2}))"}, + # Check spelling of the commit with https://github.com/crate-ci/typos + # If the spelling is incorrect, it will be automatically fixed. + #{ pattern = '.*', replace_command = 'typos --write-changes -' }, +] +# regex for parsing and grouping commits +commit_parsers = [ + { message = "^feat", group = "🚀 Features" }, + { message = "^fix", group = "🐛 Bug Fixes" }, + { message = "^doc", skip = true }, + { message = "^perf", group = "⚡ Performance" }, + { message = "^refactor", group = "🚜 Refactor" }, + { message = "^style", skip = true }, + { message = "^test", skip = true }, + { message = "^build", skip = true }, + { message = "^ci", skip = true }, + { message = "^chore\\(release\\)", skip = true }, + { message = "^chore\\(deps.*\\)", skip = true }, + { message = "^chore\\(pr\\)", skip = true }, + { message = "^chore\\(pull\\)", skip = true }, + { message = "^chore", group = "⚙️ Miscellaneous Tasks" }, + { body = ".*security", group = "🛡️ Security" }, + { message = "^revert", group = "◀️ Revert" }, +] +# protect breaking changes from being skipped due to matching a skipping commit_parser +protect_breaking_commits = true +# filter out the commits that are not matched by commit parsers +filter_commits = false +# regex for matching git tags +# tag_pattern = "v[0-9].*" +# regex for skipping tags +# skip_tags = "" +# regex for ignoring tags +# ignore_tags = "" +# sort the tags topologically +topo_order = false +# sort the commits inside sections by oldest/newest order +sort_commits = "oldest" +# limit the number of commits included in the changelog. +# limit_commits = 42 diff --git a/v2/vendor/security/zip-7.2.0/pull_request_template.md b/v2/vendor/security/zip-7.2.0/pull_request_template.md new file mode 100644 index 0000000000..9b3e77c179 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/pull_request_template.md @@ -0,0 +1,38 @@ + diff --git a/v2/vendor/security/zip-7.2.0/release-plz.toml b/v2/vendor/security/zip-7.2.0/release-plz.toml new file mode 100644 index 0000000000..58e434cbb1 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/release-plz.toml @@ -0,0 +1,27 @@ +[workspace] +dependencies_update = true # update dependencies with `cargo update` +pr_labels = ["release"] # add the `release` label to the release Pull Request +release_commits = "^(feat|fix|perf|refactor):" # prepare release only if at least one commit matches a regex +git_release_type = "auto" +release_always = false + +[changelog] +protect_breaking_commits = true +commit_parsers = [ + { message = "^feat", group = "🚀 Features" }, + { message = "^fix", group = "🐛 Bug Fixes" }, + { message = "^doc", skip = true }, + { message = "^perf", group = "⚡ Performance" }, + { message = "^refactor", group = "🚜 Refactor" }, + { message = "^style", skip = true }, + { message = "^test", skip = true }, + { message = "^build", skip = true }, + { message = "^ci", skip = true }, + { message = "^chore\\(release\\)", skip = true }, + { message = "^chore\\(deps.*\\)", skip = true }, + { message = "^chore\\(pr\\)", skip = true }, + { message = "^chore\\(pull\\)", skip = true }, + { message = "^chore", group = "⚙️ Miscellaneous Tasks" }, + { body = ".*security", group = "🛡️ Security" }, + { message = "^revert", group = "◀️ Revert" }, +] \ No newline at end of file diff --git a/v2/vendor/security/zip-7.2.0/ruview-provenance.json b/v2/vendor/security/zip-7.2.0/ruview-provenance.json new file mode 100644 index 0000000000..c043d76c61 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/ruview-provenance.json @@ -0,0 +1,16 @@ +{ + "crate": "zip", + "version": "7.2.0", + "registrySHA256": "c42e33efc22a0650c311c2ef19115ce232583abbe80850bc8b66509ebef02de0", + "upstreamVCS": "46dc29cda6749736ddb1ec502aff06a65ffe54f8", + "fixPR": "https://github.com/zip-rs/zip2/pull/984", + "fixMerge": "a543a1fc8f3f3eef36f27bb137230295284cdb98", + "fixHead": "4d1c864bb62cc342147bf44c13a0a743fc844c4d", + "hostileSHA256": "0c68c3934f22651a6bd2726f7d58140124b1839196a1d0ae468864cdfda74676", + "licenseSHA256": "04dc52136d82400b11df5f474949863a70a5d05d2278dec45b99c479494450a6", + "securityPatchSHA256": "d7da70fb4223322f0915384589558fa732999eacc62f4eef09ca4cb58719280f", + "upstreamTestSourceCommit": "46dc29cda6749736ddb1ec502aff06a65ffe54f8", + "codeAdaptation": "final PR984 guards, read/mod.rs mapped to7.2 read.rs, stream invalid macro imported", + "tests": "original102lib+1symlink integration; upstream984 hostile test plus4097 boundary and stream-bit regression", + "manifestChanges": "only explicit ruview_0429_backport self-contained integration-test registration; version7.2.0 and licenses unchanged" +} diff --git a/v2/vendor/security/zip-7.2.0/src/aes.rs b/v2/vendor/security/zip-7.2.0/src/aes.rs new file mode 100644 index 0000000000..ec441f705c --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/aes.rs @@ -0,0 +1,407 @@ +//! Implementation of the AES decryption for zip files. +//! +//! This was implemented according to the [WinZip specification](https://www.winzip.com/win/en/aes_info.html). +//! Note that using CRC with AES depends on the used encryption specification, AE-1 or AE-2. +//! If the file is marked as encrypted with AE-2 the CRC field is ignored, even if it isn't set to 0. + +use crate::aes_ctr::AesCipher; +use crate::result::ZipResult; +use crate::types::AesMode; +use crate::{aes_ctr, result::ZipError}; +use constant_time_eq::constant_time_eq; +use hmac::{Hmac, Mac}; +use sha1::Sha1; +use std::io::{self, Error, ErrorKind, Read, Write}; +use zeroize::{Zeroize, Zeroizing}; + +/// The length of the password verification value in bytes +pub const PWD_VERIFY_LENGTH: usize = 2; +/// The length of the authentication code in bytes +const AUTH_CODE_LENGTH: usize = 10; +/// The number of iterations used with PBKDF2 +const ITERATION_COUNT: u32 = 1000; + +enum Cipher { + Aes128(Box>), + Aes192(Box>), + Aes256(Box>), +} + +impl Cipher { + /// Create a `Cipher` depending on the used `AesMode` and the given `key`. + /// + /// Returns `ZipError::InvalidPassword` if `key` doesn't have the correct size for the chosen + /// AES mode. + fn from_mode(aes_mode: AesMode, key: &[u8]) -> ZipResult { + Ok(match aes_mode { + AesMode::Aes128 => Cipher::Aes128(Box::new(aes_ctr::AesCtrZipKeyStream::< + aes_ctr::Aes128, + >::new(key)?)), + AesMode::Aes192 => Cipher::Aes192(Box::new(aes_ctr::AesCtrZipKeyStream::< + aes_ctr::Aes192, + >::new(key)?)), + AesMode::Aes256 => Cipher::Aes256(Box::new(aes_ctr::AesCtrZipKeyStream::< + aes_ctr::Aes256, + >::new(key)?)), + }) + } + + fn crypt_in_place(&mut self, target: &mut [u8]) { + match self { + Self::Aes128(cipher) => cipher.crypt_in_place(target), + Self::Aes192(cipher) => cipher.crypt_in_place(target), + Self::Aes256(cipher) => cipher.crypt_in_place(target), + } + } +} + +// An aes encrypted file starts with a salt, whose length depends on the used aes mode +// followed by a 2 byte password verification value +// then the variable length encrypted data +// and lastly a 10 byte authentication code +pub struct AesReader { + reader: R, + aes_mode: AesMode, + data_length: u64, +} + +impl AesReader { + pub const fn new(reader: R, aes_mode: AesMode, compressed_size: u64) -> AesReader { + let data_length = compressed_size + - (PWD_VERIFY_LENGTH + AUTH_CODE_LENGTH + aes_mode.salt_length()) as u64; + + Self { + reader, + aes_mode, + data_length, + } + } + + /// Read the AES header bytes and validate the password. + /// + /// Even if the validation succeeds, there is still a 1 in 65536 chance that an incorrect + /// password was provided. + /// It isn't possible to check the authentication code in this step. This will be done after + /// reading and decrypting the file. + pub fn validate(mut self, password: &[u8]) -> Result, ZipError> { + let salt_length = self.aes_mode.salt_length(); + let key_length = self.aes_mode.key_length(); + + let mut salt = vec![0; salt_length]; + self.reader.read_exact(&mut salt)?; + + // next are 2 bytes used for password verification + let mut pwd_verification_value = vec![0; PWD_VERIFY_LENGTH]; + self.reader.read_exact(&mut pwd_verification_value)?; + + // derive a key from the password and salt + // the length depends on the aes key length + let derived_key_len = 2 * key_length + PWD_VERIFY_LENGTH; + let mut derived_key: Box<[u8]> = vec![0; derived_key_len].into_boxed_slice(); + + // use PBKDF2 with HMAC-Sha1 to derive the key + pbkdf2::pbkdf2::>(password, &salt, ITERATION_COUNT, &mut derived_key) + .map_err(|e| Error::new(ErrorKind::InvalidInput, e))?; + let decrypt_key = &derived_key[0..key_length]; + let hmac_key = &derived_key[key_length..key_length * 2]; + let pwd_verify = &derived_key[derived_key_len - 2..]; + + // the last 2 bytes should equal the password verification value + if pwd_verification_value != pwd_verify { + // wrong password + return Err(ZipError::InvalidPassword); + } + + let cipher = Cipher::from_mode(self.aes_mode, decrypt_key)?; + let hmac = Hmac::::new_from_slice(hmac_key).unwrap(); + + Ok(AesReaderValid { + reader: self.reader, + data_remaining: self.data_length, + cipher, + hmac, + finalized: false, + }) + } + + /// Read the AES header bytes and returns the verification value and salt. + /// + /// # Returns + /// + /// the verification value and the salt + pub fn get_verification_value_and_salt( + mut self, + ) -> io::Result<([u8; PWD_VERIFY_LENGTH], Vec)> { + let salt_length = self.aes_mode.salt_length(); + + let mut salt = vec![0; salt_length]; + self.reader.read_exact(&mut salt)?; + + // next are 2 bytes used for password verification + let mut pwd_verification_value = [0; PWD_VERIFY_LENGTH]; + self.reader.read_exact(&mut pwd_verification_value)?; + Ok((pwd_verification_value, salt)) + } +} + +/// A reader for aes encrypted files, which has already passed the first password check. +/// +/// There is a 1 in 65536 chance that an invalid password passes that check. +/// After the data has been read and decrypted an HMAC will be checked and provide a final means +/// to check if either the password is invalid or if the data has been changed. +pub struct AesReaderValid { + reader: R, + data_remaining: u64, + cipher: Cipher, + hmac: Hmac, + finalized: bool, +} + +impl Read for AesReaderValid { + /// This implementation does not fulfill all requirements set in the trait documentation. + /// + /// ```txt + /// "If an error is returned then it must be guaranteed that no bytes were read." + /// ``` + /// + /// Whether this applies to errors that occur while reading the encrypted data depends on the + /// underlying reader. If the error occurs while verifying the HMAC, the reader might become + /// practically unusable, since its position after the error is not known. + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if self.data_remaining == 0 { + return Ok(0); + } + + // get the number of bytes to read, compare as u64 to make sure we can read more than + // 2^32 bytes even on 32 bit systems. + let bytes_to_read = self.data_remaining.min(buf.len() as u64) as usize; + let read = self.reader.read(&mut buf[0..bytes_to_read])?; + self.data_remaining -= read as u64; + + // Update the hmac with the encrypted data + self.hmac.update(&buf[0..read]); + + // decrypt the data + self.cipher.crypt_in_place(&mut buf[0..read]); + + // if there is no data left to read, check the integrity of the data + if self.data_remaining == 0 { + assert!( + !self.finalized, + "Tried to use an already finalized HMAC. This is a bug!" + ); + self.finalized = true; + + // Zip uses HMAC-Sha1-80, which only uses the first half of the hash + // see https://www.winzip.com/win/en/aes_info.html#auth-faq + let mut read_auth_code = [0; AUTH_CODE_LENGTH]; + self.reader.read_exact(&mut read_auth_code)?; + let computed_auth_code = &self.hmac.finalize_reset().into_bytes()[0..AUTH_CODE_LENGTH]; + + // use constant time comparison to mitigate timing attacks + if !constant_time_eq(computed_auth_code, &read_auth_code) { + return Err( + Error::new( + ErrorKind::InvalidData, + "Invalid authentication code, this could be due to an invalid password or errors in the data" + ) + ); + } + } + + Ok(read) + } +} + +impl AesReaderValid { + /// Consumes this decoder, returning the underlying reader. + pub fn into_inner(self) -> R { + self.reader + } +} + +pub struct AesWriter { + writer: W, + cipher: Cipher, + hmac: Hmac, + buffer: Zeroizing>, + encrypted_file_header: Option>, +} + +impl AesWriter { + pub fn new(writer: W, aes_mode: AesMode, password: &[u8]) -> ZipResult { + let salt_length = aes_mode.salt_length(); + let key_length = aes_mode.key_length(); + + let mut encrypted_file_header = Vec::with_capacity(salt_length + 2); + + let mut salt = vec![0; salt_length]; + getrandom::fill(&mut salt).map_err(|e| ZipError::Io(e.into()))?; + encrypted_file_header.write_all(&salt)?; + + // Derive a key from the password and salt. The length depends on the aes key length + let derived_key_len = 2 * key_length + PWD_VERIFY_LENGTH; + let mut derived_key: Zeroizing> = Zeroizing::new(vec![0; derived_key_len]); + + // Use PBKDF2 with HMAC-Sha1 to derive the key. + pbkdf2::pbkdf2::>(password, &salt, ITERATION_COUNT, &mut derived_key) + .map_err(|e| Error::new(ErrorKind::InvalidInput, e))?; + let encryption_key = &derived_key[0..key_length]; + let hmac_key = &derived_key[key_length..key_length * 2]; + + let pwd_verify = derived_key[derived_key_len - 2..].to_vec(); + encrypted_file_header.write_all(&pwd_verify)?; + + let cipher = Cipher::from_mode(aes_mode, encryption_key)?; + let hmac = Hmac::::new_from_slice(hmac_key).unwrap(); + + Ok(Self { + writer, + cipher, + hmac, + buffer: Default::default(), + encrypted_file_header: Some(encrypted_file_header), + }) + } + + pub fn finish(mut self) -> io::Result { + self.write_encrypted_file_header()?; + + // Zip uses HMAC-Sha1-80, which only uses the first half of the hash + // see https://www.winzip.com/win/en/aes_info.html#auth-faq + let computed_auth_code = &self.hmac.finalize_reset().into_bytes()[0..AUTH_CODE_LENGTH]; + self.writer.write_all(computed_auth_code)?; + + Ok(self.writer) + } + + /// The AES encryption specification requires some metadata being written at the start of the + /// file data section, but this can only be done once the extra data writing has been finished + /// so we can't do it when the writer is constructed. + fn write_encrypted_file_header(&mut self) -> io::Result<()> { + if let Some(header) = self.encrypted_file_header.take() { + self.writer.write_all(&header)?; + } + + Ok(()) + } +} + +impl Write for AesWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.write_encrypted_file_header()?; + + // Fill the internal buffer and encrypt it in-place. + self.buffer.extend_from_slice(buf); + self.cipher.crypt_in_place(&mut self.buffer[..]); + + // Update the hmac with the encrypted data. + self.hmac.update(&self.buffer[..]); + + // Write the encrypted buffer to the inner writer. We need to use `write_all` here as if + // we only write parts of the data we can't easily reverse the keystream in the cipher + // implementation. + self.writer.write_all(&self.buffer[..])?; + + // Zeroize the backing memory before clearing the buffer to prevent cleartext data from + // being left in memory. + self.buffer.zeroize(); + self.buffer.clear(); + + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + self.writer.flush() + } +} + +#[cfg(all(test, feature = "aes-crypto"))] +mod tests { + use std::io::{self, Read, Write}; + + use crate::{ + aes::{AesReader, AesWriter}, + result::ZipError, + types::AesMode, + }; + + /// Checks whether `AesReader` can successfully decrypt what `AesWriter` produces. + fn roundtrip(aes_mode: AesMode, password: &[u8], plaintext: &[u8]) -> Result { + let mut buf = io::Cursor::new(vec![]); + let mut read_buffer = vec![]; + + { + let mut writer = AesWriter::new(&mut buf, aes_mode, password)?; + writer.write_all(plaintext)?; + writer.finish()?; + } + + // Reset cursor position to the beginning. + buf.set_position(0); + + { + let compressed_length = buf.get_ref().len() as u64; + let mut reader = + AesReader::new(&mut buf, aes_mode, compressed_length).validate(password)?; + reader.read_to_end(&mut read_buffer)?; + } + + Ok(plaintext == read_buffer) + } + + #[test] + fn crypt_aes_256_0_byte() { + let plaintext = &[]; + let password = b"some super secret password"; + assert!(roundtrip(AesMode::Aes256, password, plaintext).expect("could encrypt and decrypt")); + } + + #[test] + fn crypt_aes_128_5_byte() { + let plaintext = b"asdf\n"; + let password = b"some super secret password"; + + assert!(roundtrip(AesMode::Aes128, password, plaintext).expect("could encrypt and decrypt")); + } + + #[test] + fn crypt_aes_192_5_byte() { + let plaintext = b"asdf\n"; + let password = b"some super secret password"; + + assert!(roundtrip(AesMode::Aes192, password, plaintext).expect("could encrypt and decrypt")); + } + + #[test] + fn crypt_aes_256_5_byte() { + let plaintext = b"asdf\n"; + let password = b"some super secret password"; + + assert!(roundtrip(AesMode::Aes256, password, plaintext).expect("could encrypt and decrypt")); + } + + #[test] + fn crypt_aes_128_40_byte() { + let plaintext = b"Lorem ipsum dolor sit amet, consectetur\n"; + let password = b"some super secret password"; + + assert!(roundtrip(AesMode::Aes128, password, plaintext).expect("could encrypt and decrypt")); + } + + #[test] + fn crypt_aes_192_40_byte() { + let plaintext = b"Lorem ipsum dolor sit amet, consectetur\n"; + let password = b"some super secret password"; + + assert!(roundtrip(AesMode::Aes192, password, plaintext).expect("could encrypt and decrypt")); + } + + #[test] + fn crypt_aes_256_40_byte() { + let plaintext = b"Lorem ipsum dolor sit amet, consectetur\n"; + let password = b"some super secret password"; + + assert!(roundtrip(AesMode::Aes256, password, plaintext).expect("could encrypt and decrypt")); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/aes_ctr.rs b/v2/vendor/security/zip-7.2.0/src/aes_ctr.rs new file mode 100644 index 0000000000..7b3a3c0c4d --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/aes_ctr.rs @@ -0,0 +1,277 @@ +//! A counter mode (CTR) for AES to work with the encryption used in zip files. +//! +//! This was implemented since the zip specification requires the mode to not use a nonce and uses a +//! different byte order (little endian) than NIST (big endian). +//! See [AesCtrZipKeyStream] for more information. + +use crate::result::{ZipError, ZipResult}; +use aes::cipher::{BlockEncrypt, KeyInit}; +use core::{any, fmt}; + +/// Internal block size of an AES cipher. +const AES_BLOCK_SIZE: usize = 16; + +/// AES-128. +#[derive(Debug)] +pub struct Aes128; +/// AES-192. +#[derive(Debug)] +pub struct Aes192; +/// AES-256. +#[derive(Debug)] +pub struct Aes256; + +/// An AES cipher kind. +pub trait AesKind { + /// Key type. + type Key: AsRef<[u8]>; + /// Cipher used to decrypt. + type Cipher: KeyInit; +} + +impl AesKind for Aes128 { + type Key = [u8; 16]; + type Cipher = aes::Aes128; +} + +impl AesKind for Aes192 { + type Key = [u8; 24]; + type Cipher = aes::Aes192; +} + +impl AesKind for Aes256 { + type Key = [u8; 32]; + type Cipher = aes::Aes256; +} + +/// An AES-CTR key stream generator. +/// +/// Implements the slightly non-standard AES-CTR variant used by WinZip AES encryption. +/// +/// Typical AES-CTR implementations combine a nonce with a 64 bit counter. WinZIP AES instead uses +/// no nonce and also uses a different byte order (little endian) than NIST (big endian). +/// +/// The stream implements the `Read` trait; encryption or decryption is performed by XOR-ing the +/// bytes from the key stream with the ciphertext/plaintext. +pub struct AesCtrZipKeyStream { + /// Current AES counter. + counter: u128, + /// AES cipher instance. + cipher: C::Cipher, + /// Stores the currently available keystream bytes. + buffer: [u8; AES_BLOCK_SIZE], + /// Number of bytes already used up from `buffer`. + pos: usize, +} + +impl fmt::Debug for AesCtrZipKeyStream +where + C: AesKind, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "AesCtrZipKeyStream<{}>(counter: {})", + any::type_name::(), + self.counter + ) + } +} + +impl AesCtrZipKeyStream +where + C: AesKind, + C::Cipher: KeyInit, +{ + /// Creates a new zip variant AES-CTR key stream. + /// + /// Returns `ZipError::InvalidPassword` if `key` doesn't have the correct size for cipher `C`. + pub fn new(key: &[u8]) -> ZipResult> { + Ok(AesCtrZipKeyStream { + counter: 1, + cipher: C::Cipher::new_from_slice(key).map_err(|_| ZipError::InvalidPassword)?, + buffer: [0u8; AES_BLOCK_SIZE], + pos: AES_BLOCK_SIZE, + }) + } +} + +impl AesCipher for AesCtrZipKeyStream +where + C: AesKind, + C::Cipher: BlockEncrypt, +{ + /// Decrypt or encrypt `target`. + #[inline] + fn crypt_in_place(&mut self, mut target: &mut [u8]) { + while !target.is_empty() { + if self.pos == AES_BLOCK_SIZE { + self.buffer = self.counter.to_le_bytes(); + self.cipher.encrypt_block(self.buffer.as_mut().into()); + self.counter += 1; + self.pos = 0; + } + + let target_len = target.len().min(AES_BLOCK_SIZE - self.pos); + + xor( + &mut target[0..target_len], + &self.buffer[self.pos..(self.pos + target_len)], + ); + target = &mut target[target_len..]; + self.pos += target_len; + } + } +} + +/// This trait allows using generic AES ciphers with different key sizes. +pub trait AesCipher { + fn crypt_in_place(&mut self, target: &mut [u8]); +} + +/// XORs a slice in place with another slice. +#[inline] +fn xor(dest: &mut [u8], src: &[u8]) { + debug_assert_eq!( + dest.len(), + src.len(), + "XOR operation requires destination and source slices to have equal length" + ); + + for (lhs, rhs) in dest.iter_mut().zip(src.iter()) { + *lhs ^= *rhs; + } +} + +#[cfg(test)] +mod tests { + use super::{Aes128, Aes192, Aes256, AesCipher, AesCtrZipKeyStream, AesKind}; + use aes::cipher::{BlockEncrypt, KeyInit}; + + /// Checks whether `crypt_in_place` produces the correct plaintext after one use and yields the + /// ciphertext again after applying it again. + fn roundtrip(key: &[u8], ciphertext: &[u8], expected_plaintext: &[u8]) + where + Aes: AesKind, + Aes::Cipher: KeyInit + BlockEncrypt, + { + let mut key_stream = AesCtrZipKeyStream::::new(key).unwrap(); + + let mut plaintext = ciphertext.to_vec().into_boxed_slice(); + key_stream.crypt_in_place(&mut plaintext); + assert_eq!(*plaintext, *expected_plaintext); + + // Round-tripping should yield the ciphertext again. + let mut key_stream = AesCtrZipKeyStream::::new(key).unwrap(); + key_stream.crypt_in_place(&mut plaintext); + assert_eq!(*plaintext, *ciphertext); + } + + #[test] + fn new_with_wrong_key_size() { + AesCtrZipKeyStream::::new(&[1, 2, 3, 4, 5]) + .expect_err("Wrong key size should fail"); + } + + // The data used in these tests was generated with p7zip without any compression. + // It's not possible to recreate the exact same data, since a random salt is used for encryption. + // `7z a -phelloworld -mem=AES256 -mx=0 aes256_40byte.zip 40byte_data.txt` + #[test] + fn crypt_aes_256_0_byte() { + let ciphertext = []; + let expected_plaintext = &[]; + let key = [ + 0x0b, 0xec, 0x2e, 0xf2, 0x46, 0xf0, 0x7e, 0x35, 0x16, 0x54, 0xe0, 0x98, 0x10, 0xb3, + 0x18, 0x55, 0x24, 0xa3, 0x9e, 0x0e, 0x40, 0xe7, 0x92, 0xad, 0xb2, 0x8a, 0x48, 0xf4, + 0x5c, 0xd0, 0xc0, 0x54, + ]; + + roundtrip::(&key, &ciphertext, expected_plaintext); + } + + #[test] + fn crypt_aes_128_5_byte() { + let ciphertext = [0x98, 0xa9, 0x8c, 0x26, 0x0e]; + let expected_plaintext = b"asdf\n"; + let key = [ + 0xe0, 0x25, 0x7b, 0x57, 0x97, 0x6a, 0xa4, 0x23, 0xab, 0x94, 0xaa, 0x44, 0xfd, 0x47, + 0x4f, 0xa5, + ]; + + roundtrip::(&key, &ciphertext, expected_plaintext); + } + + #[test] + fn crypt_aes_192_5_byte() { + let ciphertext = [0x36, 0x55, 0x5c, 0x61, 0x3c]; + let expected_plaintext = b"asdf\n"; + let key = [ + 0xe4, 0x4a, 0x88, 0x52, 0x8f, 0xf7, 0x0b, 0x81, 0x7b, 0x75, 0xf1, 0x74, 0x21, 0x37, + 0x8c, 0x90, 0xad, 0xbe, 0x4a, 0x65, 0xa8, 0x96, 0x0e, 0xcc, + ]; + + roundtrip::(&key, &ciphertext, expected_plaintext); + } + + #[test] + fn crypt_aes_256_5_byte() { + let ciphertext = [0xc2, 0x47, 0xc0, 0xdc, 0x56]; + let expected_plaintext = b"asdf\n"; + let key = [ + 0x79, 0x5e, 0x17, 0xf2, 0xc6, 0x3d, 0x28, 0x9b, 0x4b, 0x4b, 0xbb, 0xa9, 0xba, 0xc9, + 0xa5, 0xee, 0x3a, 0x4f, 0x0f, 0x4b, 0x29, 0xbd, 0xe9, 0xb8, 0x41, 0x9c, 0x41, 0xa5, + 0x15, 0xb2, 0x86, 0xab, + ]; + + roundtrip::(&key, &ciphertext, expected_plaintext); + } + + #[test] + fn crypt_aes_128_40_byte() { + let ciphertext = [ + 0xcf, 0x72, 0x6b, 0xa1, 0xb2, 0x0f, 0xdf, 0xaa, 0x10, 0xad, 0x9c, 0x7f, 0x6d, 0x1c, + 0x8d, 0xb5, 0x16, 0x7e, 0xbb, 0x11, 0x69, 0x52, 0x8c, 0x89, 0x80, 0x32, 0xaa, 0x76, + 0xa6, 0x18, 0x31, 0x98, 0xee, 0xdd, 0x22, 0x68, 0xb7, 0xe6, 0x77, 0xd2, + ]; + let expected_plaintext = b"Lorem ipsum dolor sit amet, consectetur\n"; + let key = [ + 0x43, 0x2b, 0x6d, 0xbe, 0x05, 0x76, 0x6c, 0x9e, 0xde, 0xca, 0x3b, 0xf8, 0xaf, 0x5d, + 0x81, 0xb6, + ]; + + roundtrip::(&key, &ciphertext, expected_plaintext); + } + + #[test] + fn crypt_aes_192_40_byte() { + let ciphertext = [ + 0xa6, 0xfc, 0x52, 0x79, 0x2c, 0x6c, 0xfe, 0x68, 0xb1, 0xa8, 0xb3, 0x07, 0x52, 0x8b, + 0x82, 0xa6, 0x87, 0x9c, 0x72, 0x42, 0x3a, 0xf8, 0xc6, 0xa9, 0xc9, 0xfb, 0x61, 0x19, + 0x37, 0xb9, 0x56, 0x62, 0xf4, 0xfc, 0x5e, 0x7a, 0xdd, 0x55, 0x0a, 0x48, + ]; + let expected_plaintext = b"Lorem ipsum dolor sit amet, consectetur\n"; + let key = [ + 0xac, 0x92, 0x41, 0xba, 0xde, 0xd9, 0x02, 0xfe, 0x40, 0x92, 0x20, 0xf6, 0x56, 0x03, + 0xfe, 0xae, 0x1b, 0xba, 0x01, 0x97, 0x97, 0x79, 0xbb, 0xa6, + ]; + + roundtrip::(&key, &ciphertext, expected_plaintext); + } + + #[test] + fn crypt_aes_256_40_byte() { + let ciphertext = [ + 0xa9, 0x99, 0xbd, 0xea, 0x82, 0x9b, 0x8f, 0x2f, 0xb7, 0x52, 0x2f, 0x6b, 0xd8, 0xf6, + 0xab, 0x0e, 0x24, 0x51, 0x9e, 0x18, 0x0f, 0xc0, 0x8f, 0x54, 0x15, 0x80, 0xae, 0xbc, + 0xa0, 0x5c, 0x8a, 0x11, 0x8d, 0x14, 0x7e, 0xc5, 0xb4, 0xae, 0xd3, 0x37, + ]; + let expected_plaintext = b"Lorem ipsum dolor sit amet, consectetur\n"; + let key = [ + 0x64, 0x7c, 0x7a, 0xde, 0xf0, 0xf2, 0x61, 0x49, 0x1c, 0xf1, 0xf1, 0xe3, 0x37, 0xfc, + 0xe1, 0x4d, 0x4a, 0x77, 0xd4, 0xeb, 0x9e, 0x3d, 0x75, 0xce, 0x9a, 0x3e, 0x10, 0x50, + 0xc2, 0x07, 0x36, 0xb6, + ]; + + roundtrip::(&key, &ciphertext, expected_plaintext); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/compression.rs b/v2/vendor/security/zip-7.2.0/src/compression.rs new file mode 100644 index 0000000000..2848f8749a --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/compression.rs @@ -0,0 +1,557 @@ +//! Possible ZIP compression methods. + +use crate::cfg_if_expr; +use core::fmt; +use std::io; + +#[allow(deprecated)] +/// Identifies the storage format used to compress a file within a ZIP archive. +/// +/// Each file's compression method is stored alongside it, allowing the +/// contents to be read without context. +/// +/// When creating ZIP files, you may choose the method to use with +/// [`crate::write::FileOptions::compression_method`] +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +#[cfg_attr(feature = "_arbitrary", derive(arbitrary::Arbitrary))] +#[non_exhaustive] +pub enum CompressionMethod { + /// Store the file as is + Stored, + /// Compress the file using Deflate + #[cfg(feature = "_deflate-any")] + Deflated, + /// Compress the file using Deflate64. + /// Decoding deflate64 is supported but encoding deflate64 is not supported. + #[cfg(feature = "deflate64")] + Deflate64, + /// Compress the file using BZIP2 + #[cfg(feature = "_bzip2_any")] + Bzip2, + /// Encrypted using AES. + /// + /// The actual compression method has to be taken from the AES extra data field + /// or from `ZipFileData`. + #[cfg(feature = "aes-crypto")] + Aes, + /// Compress the file using ZStandard + #[cfg(feature = "zstd")] + Zstd, + /// Compress the file using LZMA + #[cfg(feature = "lzma")] + Lzma, + #[cfg(feature = "legacy-zip")] + /// Method 1 Shrink + Shrink, + #[cfg(feature = "legacy-zip")] + /// Reduce (Method 2-5) + Reduce(u8), + #[cfg(feature = "legacy-zip")] + /// Method 6 Implode/explode + Implode, + /// Compress the file using XZ + #[cfg(feature = "xz")] + Xz, + /// Compress the file using PPMd + #[cfg(feature = "ppmd")] + Ppmd, + /// Unsupported compression method + #[cfg_attr( + not(any(fuzzing, feature = "_arbitrary")), + deprecated(since = "0.5.7", note = "use the constants instead") + )] + Unsupported(u16), +} +#[allow(deprecated, missing_docs)] +/// All compression methods defined for the ZIP format +impl CompressionMethod { + pub const STORE: Self = CompressionMethod::Stored; + #[cfg(feature = "legacy-zip")] + pub const SHRINK: Self = CompressionMethod::Shrink; + #[cfg(not(feature = "legacy-zip"))] + /// Legacy compression method (enable feature `legacy-zip` to get support) + pub const SHRINK: Self = CompressionMethod::Unsupported(1); + #[cfg(feature = "legacy-zip")] + /// Legacy compression method + pub const REDUCE_1: Self = CompressionMethod::Reduce(1); + #[cfg(not(feature = "legacy-zip"))] + /// Legacy compression method (enable feature `legacy-zip` to get support) + pub const REDUCE_1: Self = CompressionMethod::Unsupported(2); + #[cfg(feature = "legacy-zip")] + /// Legacy compression method + pub const REDUCE_2: Self = CompressionMethod::Reduce(2); + #[cfg(not(feature = "legacy-zip"))] + /// Legacy compression method (enable feature `legacy-zip` to get support) + pub const REDUCE_2: Self = CompressionMethod::Unsupported(3); + #[cfg(feature = "legacy-zip")] + /// Legacy compression method + pub const REDUCE_3: Self = CompressionMethod::Reduce(3); + #[cfg(not(feature = "legacy-zip"))] + /// Legacy compression method (enable feature `legacy-zip` to get support) + pub const REDUCE_3: Self = CompressionMethod::Unsupported(4); + #[cfg(feature = "legacy-zip")] + /// Legacy compression method + pub const REDUCE_4: Self = CompressionMethod::Reduce(4); + #[cfg(not(feature = "legacy-zip"))] + /// Legacy compression method (enable feature `legacy-zip` to get support) + pub const REDUCE_4: Self = CompressionMethod::Unsupported(5); + #[cfg(feature = "legacy-zip")] + /// Legacy compression method + pub const IMPLODE: Self = CompressionMethod::Implode; + #[cfg(not(feature = "legacy-zip"))] + /// Legacy compression method (enable feature `legacy-zip` to get support) + pub const IMPLODE: Self = CompressionMethod::Unsupported(6); + #[cfg(feature = "_deflate-any")] + pub const DEFLATE: Self = CompressionMethod::Deflated; + #[cfg(not(feature = "_deflate-any"))] + pub const DEFLATE: Self = CompressionMethod::Unsupported(8); + #[cfg(feature = "deflate64")] + pub const DEFLATE64: Self = CompressionMethod::Deflate64; + #[cfg(not(feature = "deflate64"))] + pub const DEFLATE64: Self = CompressionMethod::Unsupported(9); + pub const PKWARE_IMPLODE: Self = CompressionMethod::Unsupported(10); + #[cfg(feature = "_bzip2_any")] + pub const BZIP2: Self = CompressionMethod::Bzip2; + #[cfg(not(feature = "_bzip2_any"))] + pub const BZIP2: Self = CompressionMethod::Unsupported(12); + #[cfg(not(feature = "lzma"))] + pub const LZMA: Self = CompressionMethod::Unsupported(14); + #[cfg(feature = "lzma")] + pub const LZMA: Self = CompressionMethod::Lzma; + pub const IBM_ZOS_CMPSC: Self = CompressionMethod::Unsupported(16); + pub const IBM_TERSE: Self = CompressionMethod::Unsupported(18); + pub const ZSTD_DEPRECATED: Self = CompressionMethod::Unsupported(20); + #[cfg(feature = "zstd")] + pub const ZSTD: Self = CompressionMethod::Zstd; + #[cfg(not(feature = "zstd"))] + pub const ZSTD: Self = CompressionMethod::Unsupported(93); + pub const MP3: Self = CompressionMethod::Unsupported(94); + #[cfg(feature = "xz")] + pub const XZ: Self = CompressionMethod::Xz; + #[cfg(not(feature = "xz"))] + pub const XZ: Self = CompressionMethod::Unsupported(95); + pub const JPEG: Self = CompressionMethod::Unsupported(96); + pub const WAVPACK: Self = CompressionMethod::Unsupported(97); + #[cfg(feature = "ppmd")] + pub const PPMD: Self = CompressionMethod::Ppmd; + #[cfg(not(feature = "ppmd"))] + pub const PPMD: Self = CompressionMethod::Unsupported(98); + #[cfg(feature = "aes-crypto")] + pub const AES: Self = CompressionMethod::Aes; + #[cfg(not(feature = "aes-crypto"))] + pub const AES: Self = CompressionMethod::Unsupported(99); + + #[cfg(feature = "_deflate-any")] + pub const DEFAULT: Self = CompressionMethod::Deflated; + + #[cfg(all(not(feature = "_deflate-any"), feature = "_bzip2_any"))] + pub const DEFAULT: Self = CompressionMethod::Bzip2; + + #[cfg(all(not(feature = "_deflate-any"), not(feature = "_bzip2_any")))] + pub const DEFAULT: Self = CompressionMethod::Stored; +} +impl CompressionMethod { + pub(crate) const fn parse_from_u16(val: u16) -> Self { + match val { + 0 => CompressionMethod::Stored, + #[cfg(feature = "legacy-zip")] + 1 => CompressionMethod::Shrink, + #[cfg(feature = "legacy-zip")] + 2 => CompressionMethod::Reduce(1), + #[cfg(feature = "legacy-zip")] + 3 => CompressionMethod::Reduce(2), + #[cfg(feature = "legacy-zip")] + 4 => CompressionMethod::Reduce(3), + #[cfg(feature = "legacy-zip")] + 5 => CompressionMethod::Reduce(4), + #[cfg(feature = "legacy-zip")] + 6 => CompressionMethod::Implode, + #[cfg(feature = "_deflate-any")] + 8 => CompressionMethod::Deflated, + #[cfg(feature = "deflate64")] + 9 => CompressionMethod::Deflate64, + #[cfg(feature = "_bzip2_any")] + 12 => CompressionMethod::Bzip2, + #[cfg(feature = "lzma")] + 14 => CompressionMethod::Lzma, + #[cfg(feature = "xz")] + 95 => CompressionMethod::Xz, + #[cfg(feature = "zstd")] + 93 => CompressionMethod::Zstd, + #[cfg(feature = "ppmd")] + 98 => CompressionMethod::Ppmd, + #[cfg(feature = "aes-crypto")] + 99 => CompressionMethod::Aes, + #[allow(deprecated)] + v => CompressionMethod::Unsupported(v), + } + } + + /// Converts a u16 to its corresponding CompressionMethod + #[deprecated( + since = "0.5.7", + note = "use a constant to construct a compression method" + )] + pub const fn from_u16(val: u16) -> CompressionMethod { + Self::parse_from_u16(val) + } + + pub(crate) const fn serialize_to_u16(self) -> u16 { + match self { + CompressionMethod::Stored => 0, + #[cfg(feature = "legacy-zip")] + CompressionMethod::Shrink => 1, + #[cfg(feature = "legacy-zip")] + CompressionMethod::Reduce(n) => 1 + n as u16, + #[cfg(feature = "legacy-zip")] + CompressionMethod::Implode => 6, + + #[cfg(feature = "_deflate-any")] + CompressionMethod::Deflated => 8, + #[cfg(feature = "deflate64")] + CompressionMethod::Deflate64 => 9, + #[cfg(feature = "_bzip2_any")] + CompressionMethod::Bzip2 => 12, + #[cfg(feature = "aes-crypto")] + CompressionMethod::Aes => 99, + #[cfg(feature = "zstd")] + CompressionMethod::Zstd => 93, + #[cfg(feature = "lzma")] + CompressionMethod::Lzma => 14, + #[cfg(feature = "xz")] + CompressionMethod::Xz => 95, + #[cfg(feature = "ppmd")] + CompressionMethod::Ppmd => 98, + #[allow(deprecated)] + CompressionMethod::Unsupported(v) => v, + } + } + + /// Converts a CompressionMethod to a u16 + #[deprecated( + since = "0.5.7", + note = "to match on other compression methods, use a constant" + )] + pub const fn to_u16(self) -> u16 { + self.serialize_to_u16() + } +} + +impl Default for CompressionMethod { + fn default() -> Self { + cfg_if_expr! { + #[cfg(feature = "_deflate-any")] => CompressionMethod::Deflated, + _ => CompressionMethod::Stored + } + } +} + +impl fmt::Display for CompressionMethod { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + // Just duplicate what the Debug format looks like, i.e, the enum key: + write!(f, "{self:?}") + } +} + +/// The compression methods which have been implemented. +pub const SUPPORTED_COMPRESSION_METHODS: &[CompressionMethod] = &[ + CompressionMethod::Stored, + #[cfg(feature = "_deflate-any")] + CompressionMethod::Deflated, + #[cfg(feature = "deflate64")] + CompressionMethod::Deflate64, + #[cfg(feature = "_bzip2_any")] + CompressionMethod::Bzip2, + #[cfg(feature = "zstd")] + CompressionMethod::Zstd, + #[cfg(feature = "xz")] + CompressionMethod::Xz, + #[cfg(feature = "ppmd")] + CompressionMethod::Ppmd, +]; + +pub(crate) enum Decompressor { + Stored(R), + #[cfg(feature = "deflate-flate2")] + Deflated(flate2::bufread::DeflateDecoder), + #[cfg(feature = "deflate64")] + Deflate64(deflate64::Deflate64Decoder), + #[cfg(feature = "_bzip2_any")] + Bzip2(bzip2::bufread::BzDecoder), + #[cfg(feature = "zstd")] + Zstd(zstd::Decoder<'static, R>), + #[cfg(feature = "lzma")] + Lzma(Lzma), + #[cfg(feature = "legacy-zip")] + Shrink(crate::legacy::shrink::ShrinkDecoder), + #[cfg(feature = "legacy-zip")] + Reduce(crate::legacy::reduce::ReduceDecoder), + #[cfg(feature = "legacy-zip")] + Implode(crate::legacy::implode::ImplodeDecoder), + #[cfg(feature = "xz")] + Xz(Box>), + #[cfg(feature = "ppmd")] + Ppmd(Ppmd), +} + +#[cfg(feature = "lzma")] +pub(crate) enum Lzma { + Uninitialized { + reader: Option, + uncompressed_size: u64, + }, + Initialized(Box>), +} + +#[cfg(feature = "ppmd")] +pub(crate) enum Ppmd { + Uninitialized(Option), + Initialized(Box>), +} + +impl io::Read for Decompressor { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + Decompressor::Stored(r) => r.read(buf), + #[cfg(feature = "deflate-flate2")] + Decompressor::Deflated(r) => r.read(buf), + #[cfg(feature = "deflate64")] + Decompressor::Deflate64(r) => r.read(buf), + #[cfg(feature = "_bzip2_any")] + Decompressor::Bzip2(r) => r.read(buf), + #[cfg(feature = "zstd")] + Decompressor::Zstd(r) => r.read(buf), + #[cfg(feature = "lzma")] + Decompressor::Lzma(r) => match r { + Lzma::Uninitialized { + reader, + uncompressed_size, + } => { + let mut reader = reader.take().ok_or_else(|| { + io::Error::other("Reader was not set while reading LZMA data") + })?; + + // 5.8.8.1 LZMA Version Information & 5.8.8.2 LZMA Properties Size + let mut header = [0; 4]; + reader.read_exact(&mut header)?; + let _version_information = u16::from_le_bytes(header[0..2].try_into().unwrap()); + let properties_size = u16::from_le_bytes(header[2..4].try_into().unwrap()); + if properties_size != 5 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + format!("unexpected LZMA properties size of {properties_size}"), + )); + } + + let mut props_data = [0; 5]; + reader.read_exact(&mut props_data)?; + let props = props_data[0]; + let dict_size = u32::from_le_bytes(props_data[1..5].try_into().unwrap()); + + // We don't need to handle the end-of-stream marker here, since the LZMA reader + // stops at the end-of-stream marker OR when it has decoded uncompressed_size bytes, whichever comes first. + let mut decompressor = lzma_rust2::LzmaReader::new_with_props( + reader, + *uncompressed_size, + props, + dict_size, + None, + )?; + + let read = decompressor.read(buf)?; + + *r = Lzma::Initialized(Box::new(decompressor)); + + Ok(read) + } + Lzma::Initialized(decompressor) => decompressor.read(buf), + }, + #[cfg(feature = "xz")] + Decompressor::Xz(r) => r.read(buf), + #[cfg(feature = "ppmd")] + Decompressor::Ppmd(r) => match r { + Ppmd::Uninitialized(reader) => { + let mut reader = reader.take().ok_or_else(|| { + io::Error::other("Reader was not set while reading PPMd data") + })?; + + let mut buffer = [0; 2]; + reader.read_exact(&mut buffer)?; + let parameters = u16::from_le_bytes(buffer); + + let order = ((parameters & 0x0F) + 1) as u32; + let memory_size = 1024 * 1024 * (((parameters >> 4) & 0xFF) + 1) as u32; + let restoration_method = (parameters >> 12) & 0x0F; + + let mut decompressor = ppmd_rust::Ppmd8Decoder::new( + reader, + order, + memory_size, + restoration_method.into(), + ) + .map_err(|error| match error { + ppmd_rust::Error::RangeDecoderInitialization => io::Error::new( + io::ErrorKind::InvalidData, + "PPMd range coder initialization failed", + ), + ppmd_rust::Error::InvalidParameter => { + io::Error::new(io::ErrorKind::InvalidInput, "Invalid PPMd parameter") + } + ppmd_rust::Error::IoError(io_error) => io_error, + ppmd_rust::Error::MemoryAllocation => { + io::Error::new(io::ErrorKind::OutOfMemory, "Memory allocation failed") + } + })?; + + let read = decompressor.read(buf)?; + + *r = Ppmd::Initialized(Box::new(decompressor)); + + Ok(read) + } + Ppmd::Initialized(decompressor) => decompressor.read(buf), + }, + #[cfg(feature = "legacy-zip")] + Decompressor::Shrink(r) => r.read(buf), + #[cfg(feature = "legacy-zip")] + Decompressor::Reduce(r) => r.read(buf), + #[cfg(feature = "legacy-zip")] + Decompressor::Implode(r) => r.read(buf), + } + } +} + +impl Decompressor { + pub fn new( + reader: R, + compression_method: CompressionMethod, + #[cfg(any(feature = "lzma", feature = "legacy-zip"))] uncompressed_size: u64, + #[cfg(not(any(feature = "lzma", feature = "legacy-zip")))] _uncompressed_size: u64, + #[cfg(feature = "legacy-zip")] flags: u16, + #[cfg(not(feature = "legacy-zip"))] _flags: u16, + ) -> crate::result::ZipResult { + Ok(match compression_method { + CompressionMethod::Stored => Decompressor::Stored(reader), + #[cfg(feature = "deflate-flate2")] + CompressionMethod::Deflated => { + Decompressor::Deflated(flate2::bufread::DeflateDecoder::new(reader)) + } + #[cfg(feature = "deflate64")] + CompressionMethod::Deflate64 => { + Decompressor::Deflate64(deflate64::Deflate64Decoder::with_buffer(reader)) + } + #[cfg(feature = "_bzip2_any")] + CompressionMethod::Bzip2 => Decompressor::Bzip2(bzip2::bufread::BzDecoder::new(reader)), + #[cfg(feature = "zstd")] + CompressionMethod::Zstd => Decompressor::Zstd(zstd::Decoder::with_buffer(reader)?), + #[cfg(feature = "lzma")] + CompressionMethod::Lzma => Decompressor::Lzma(Lzma::Uninitialized { + reader: Some(reader), + uncompressed_size, + }), + #[cfg(feature = "xz")] + CompressionMethod::Xz => { + Decompressor::Xz(Box::new(lzma_rust2::XzReader::new(reader, false))) + } + #[cfg(feature = "ppmd")] + CompressionMethod::Ppmd => Decompressor::Ppmd(Ppmd::Uninitialized(Some(reader))), + #[cfg(feature = "legacy-zip")] + CompressionMethod::Shrink => Decompressor::Shrink( + crate::legacy::shrink::ShrinkDecoder::new(reader, uncompressed_size), + ), + #[cfg(feature = "legacy-zip")] + CompressionMethod::Reduce(n) => Decompressor::Reduce( + crate::legacy::reduce::ReduceDecoder::new(reader, uncompressed_size, n), + ), + #[cfg(feature = "legacy-zip")] + CompressionMethod::Implode => Decompressor::Implode( + crate::legacy::implode::ImplodeDecoder::new(reader, uncompressed_size, flags), + ), + _ => { + return Err(crate::result::ZipError::UnsupportedArchive( + "Compression method not supported", + )) + } + }) + } + + /// Consumes this decoder, returning the underlying reader. + #[allow(clippy::infallible_destructuring_match)] + pub fn into_inner(self) -> io::Result { + let inner = match self { + Decompressor::Stored(r) => r, + #[cfg(feature = "deflate-flate2")] + Decompressor::Deflated(r) => r.into_inner(), + #[cfg(feature = "deflate64")] + Decompressor::Deflate64(r) => r.into_inner(), + #[cfg(feature = "_bzip2_any")] + Decompressor::Bzip2(r) => r.into_inner(), + #[cfg(feature = "zstd")] + Decompressor::Zstd(r) => r.finish(), + #[cfg(feature = "lzma")] + Decompressor::Lzma(r) => match r { + Lzma::Uninitialized { mut reader, .. } => reader + .take() + .ok_or_else(|| io::Error::other("Reader was not set"))?, + Lzma::Initialized(decoder) => decoder.into_inner(), + }, + #[cfg(feature = "legacy-zip")] + Decompressor::Shrink(r) => r.into_inner(), + #[cfg(feature = "legacy-zip")] + Decompressor::Reduce(r) => r.into_inner(), + #[cfg(feature = "legacy-zip")] + Decompressor::Implode(r) => r.into_inner(), + #[cfg(feature = "xz")] + Decompressor::Xz(r) => r.into_inner(), + #[cfg(feature = "ppmd")] + Decompressor::Ppmd(r) => match r { + Ppmd::Uninitialized(mut reader) => reader + .take() + .ok_or_else(|| io::Error::other("Reader was not set"))?, + Ppmd::Initialized(decoder) => decoder.into_inner(), + }, + }; + Ok(inner) + } +} + +#[cfg(test)] +mod test { + use super::{CompressionMethod, SUPPORTED_COMPRESSION_METHODS}; + + #[test] + fn from_eq_to() { + for v in 0..(u16::MAX as u32 + 1) { + let from = CompressionMethod::parse_from_u16(v as u16); + let to = from.serialize_to_u16() as u32; + assert_eq!(v, to); + } + } + + #[test] + fn to_eq_from() { + fn check_match(method: CompressionMethod) { + let to = method.serialize_to_u16(); + let from = CompressionMethod::parse_from_u16(to); + let back = from.serialize_to_u16(); + assert_eq!(to, back); + } + + for &method in SUPPORTED_COMPRESSION_METHODS { + check_match(method); + } + } + + #[test] + fn to_display_fmt() { + fn check_match(method: CompressionMethod) { + let debug_str = format!("{method:?}"); + let display_str = format!("{method}"); + assert_eq!(debug_str, display_str); + } + + for &method in SUPPORTED_COMPRESSION_METHODS { + check_match(method); + } + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/cp437.rs b/v2/vendor/security/zip-7.2.0/src/cp437.rs new file mode 100644 index 0000000000..2c38063f32 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/cp437.rs @@ -0,0 +1,207 @@ +//! Convert a string in IBM codepage 437 to UTF-8 + +/// Trait to convert IBM codepage 437 to the target type +pub trait FromCp437 { + /// Target type + type Target; + + /// Function that does the conversion from cp437. + /// Generally allocations will be avoided if all data falls into the ASCII range. + #[allow(clippy::wrong_self_convention)] + fn from_cp437(self) -> Self::Target; +} + +impl<'a> FromCp437 for &'a [u8] { + type Target = ::std::borrow::Cow<'a, str>; + + fn from_cp437(self) -> Self::Target { + if self.iter().all(|c| *c < 0x80) { + ::std::str::from_utf8(self).unwrap().into() + } else { + self.iter().map(|c| to_char(*c)).collect::().into() + } + } +} + +impl FromCp437 for Box<[u8]> { + type Target = Box; + + fn from_cp437(self) -> Self::Target { + if self.iter().all(|c| *c < 0x80) { + String::from_utf8(self.into()).unwrap() + } else { + self.iter().copied().map(to_char).collect() + } + .into_boxed_str() + } +} + +fn to_char(input: u8) -> char { + let output = match input { + 0x00..=0x7f => input as u32, + 0x80 => 0x00c7, + 0x81 => 0x00fc, + 0x82 => 0x00e9, + 0x83 => 0x00e2, + 0x84 => 0x00e4, + 0x85 => 0x00e0, + 0x86 => 0x00e5, + 0x87 => 0x00e7, + 0x88 => 0x00ea, + 0x89 => 0x00eb, + 0x8a => 0x00e8, + 0x8b => 0x00ef, + 0x8c => 0x00ee, + 0x8d => 0x00ec, + 0x8e => 0x00c4, + 0x8f => 0x00c5, + 0x90 => 0x00c9, + 0x91 => 0x00e6, + 0x92 => 0x00c6, + 0x93 => 0x00f4, + 0x94 => 0x00f6, + 0x95 => 0x00f2, + 0x96 => 0x00fb, + 0x97 => 0x00f9, + 0x98 => 0x00ff, + 0x99 => 0x00d6, + 0x9a => 0x00dc, + 0x9b => 0x00a2, + 0x9c => 0x00a3, + 0x9d => 0x00a5, + 0x9e => 0x20a7, + 0x9f => 0x0192, + 0xa0 => 0x00e1, + 0xa1 => 0x00ed, + 0xa2 => 0x00f3, + 0xa3 => 0x00fa, + 0xa4 => 0x00f1, + 0xa5 => 0x00d1, + 0xa6 => 0x00aa, + 0xa7 => 0x00ba, + 0xa8 => 0x00bf, + 0xa9 => 0x2310, + 0xaa => 0x00ac, + 0xab => 0x00bd, + 0xac => 0x00bc, + 0xad => 0x00a1, + 0xae => 0x00ab, + 0xaf => 0x00bb, + 0xb0 => 0x2591, + 0xb1 => 0x2592, + 0xb2 => 0x2593, + 0xb3 => 0x2502, + 0xb4 => 0x2524, + 0xb5 => 0x2561, + 0xb6 => 0x2562, + 0xb7 => 0x2556, + 0xb8 => 0x2555, + 0xb9 => 0x2563, + 0xba => 0x2551, + 0xbb => 0x2557, + 0xbc => 0x255d, + 0xbd => 0x255c, + 0xbe => 0x255b, + 0xbf => 0x2510, + 0xc0 => 0x2514, + 0xc1 => 0x2534, + 0xc2 => 0x252c, + 0xc3 => 0x251c, + 0xc4 => 0x2500, + 0xc5 => 0x253c, + 0xc6 => 0x255e, + 0xc7 => 0x255f, + 0xc8 => 0x255a, + 0xc9 => 0x2554, + 0xca => 0x2569, + 0xcb => 0x2566, + 0xcc => 0x2560, + 0xcd => 0x2550, + 0xce => 0x256c, + 0xcf => 0x2567, + 0xd0 => 0x2568, + 0xd1 => 0x2564, + 0xd2 => 0x2565, + 0xd3 => 0x2559, + 0xd4 => 0x2558, + 0xd5 => 0x2552, + 0xd6 => 0x2553, + 0xd7 => 0x256b, + 0xd8 => 0x256a, + 0xd9 => 0x2518, + 0xda => 0x250c, + 0xdb => 0x2588, + 0xdc => 0x2584, + 0xdd => 0x258c, + 0xde => 0x2590, + 0xdf => 0x2580, + 0xe0 => 0x03b1, + 0xe1 => 0x00df, + 0xe2 => 0x0393, + 0xe3 => 0x03c0, + 0xe4 => 0x03a3, + 0xe5 => 0x03c3, + 0xe6 => 0x00b5, + 0xe7 => 0x03c4, + 0xe8 => 0x03a6, + 0xe9 => 0x0398, + 0xea => 0x03a9, + 0xeb => 0x03b4, + 0xec => 0x221e, + 0xed => 0x03c6, + 0xee => 0x03b5, + 0xef => 0x2229, + 0xf0 => 0x2261, + 0xf1 => 0x00b1, + 0xf2 => 0x2265, + 0xf3 => 0x2264, + 0xf4 => 0x2320, + 0xf5 => 0x2321, + 0xf6 => 0x00f7, + 0xf7 => 0x2248, + 0xf8 => 0x00b0, + 0xf9 => 0x2219, + 0xfa => 0x00b7, + 0xfb => 0x221a, + 0xfc => 0x207f, + 0xfd => 0x00b2, + 0xfe => 0x25a0, + 0xff => 0x00a0, + }; + ::std::char::from_u32(output).unwrap() +} + +#[cfg(test)] +mod test { + #[test] + fn to_char_valid() { + for i in 0x00_u32..0x100 { + super::to_char(i as u8); + } + } + + #[test] + fn ascii() { + for i in 0x00..0x80 { + assert_eq!(super::to_char(i), i as char); + } + } + + #[test] + #[allow(unknown_lints)] // invalid_from_utf8 was added in rust 1.72 + #[allow(invalid_from_utf8)] + fn example_slice() { + use super::FromCp437; + let data = b"Cura\x87ao"; + assert!(::std::str::from_utf8(data).is_err()); + assert_eq!(data.from_cp437(), "Curaçao"); + } + + #[test] + fn example_vec() { + use super::FromCp437; + let data = vec![0xCC, 0xCD, 0xCD, 0xB9]; + assert!(String::from_utf8(data.clone()).is_err()); + assert_eq!(&*data.from_cp437(), "╠══╣"); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/crc32.rs b/v2/vendor/security/zip-7.2.0/src/crc32.rs new file mode 100644 index 0000000000..4d1bd8e264 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/crc32.rs @@ -0,0 +1,131 @@ +//! Helper module to compute a CRC32 checksum + +use std::io::{self, Read}; + +use crc32fast::Hasher; + +/// Reader that validates the CRC32 when it reaches the EOF. +pub struct Crc32Reader { + inner: R, + hasher: Hasher, + check: u32, + /// Signals if `inner` stores aes encrypted data. + /// AE-2 encrypted data doesn't use crc and sets the value to 0. + enabled: bool, +} + +impl Crc32Reader { + /// Get a new Crc32Reader which checks the inner reader against checksum. + /// The check is disabled if `ae2_encrypted == true`. + pub(crate) fn new(inner: R, checksum: u32, ae2_encrypted: bool) -> Crc32Reader { + Crc32Reader { + inner, + hasher: Hasher::new(), + check: checksum, + enabled: !ae2_encrypted, + } + } + + fn check_matches(&self) -> bool { + self.check == self.hasher.clone().finalize() + } + + pub fn into_inner(self) -> R { + self.inner + } +} + +#[cold] +fn invalid_checksum() -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, "Invalid checksum") +} + +impl Read for Crc32Reader { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let count = self.inner.read(buf)?; + + if self.enabled { + if count == 0 && !buf.is_empty() && !self.check_matches() { + return Err(invalid_checksum()); + } + self.hasher.update(&buf[..count]); + } + Ok(count) + } + + fn read_to_end(&mut self, buf: &mut Vec) -> io::Result { + let start = buf.len(); + let n = self.inner.read_to_end(buf)?; + + if self.enabled { + self.hasher.update(&buf[start..]); + if !self.check_matches() { + return Err(invalid_checksum()); + } + } + + Ok(n) + } + + fn read_to_string(&mut self, buf: &mut String) -> io::Result { + let start = buf.len(); + let n = self.inner.read_to_string(buf)?; + + if self.enabled { + self.hasher.update(&buf.as_bytes()[start..]); + if !self.check_matches() { + return Err(invalid_checksum()); + } + } + + Ok(n) + } +} + +#[cfg(test)] +mod test { + use std::io::Read; + + use super::Crc32Reader; + + #[test] + fn test_empty_reader() { + let data: &[u8] = b""; + let mut buf = [0; 1]; + + let mut reader = Crc32Reader::new(data, 0, false); + assert_eq!(reader.read(&mut buf).unwrap(), 0); + + let mut reader = Crc32Reader::new(data, 1, false); + assert!(reader + .read(&mut buf) + .unwrap_err() + .to_string() + .contains("Invalid checksum")); + } + + #[test] + fn test_byte_by_byte() { + let data: &[u8] = b"1234"; + let mut buf = [0; 1]; + + let mut reader = Crc32Reader::new(data, 0x9be3e0a3, false); + assert_eq!(reader.read(&mut buf).unwrap(), 1); + assert_eq!(reader.read(&mut buf).unwrap(), 1); + assert_eq!(reader.read(&mut buf).unwrap(), 1); + assert_eq!(reader.read(&mut buf).unwrap(), 1); + assert_eq!(reader.read(&mut buf).unwrap(), 0); + // Can keep reading 0 bytes after the end + assert_eq!(reader.read(&mut buf).unwrap(), 0); + } + + #[test] + fn test_zero_read() { + let data: &[u8] = b"1234"; + let mut buf = [0; 5]; + + let mut reader = Crc32Reader::new(data, 0x9be3e0a3, false); + assert_eq!(reader.read(&mut buf[..0]).unwrap(), 0); + assert_eq!(reader.read(&mut buf).unwrap(), 4); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/extra_fields/extended_timestamp.rs b/v2/vendor/security/zip-7.2.0/src/extra_fields/extended_timestamp.rs new file mode 100644 index 0000000000..22ebacf23c --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/extra_fields/extended_timestamp.rs @@ -0,0 +1,111 @@ +use crate::result::invalid; +use crate::result::{ZipError, ZipResult}; +use crate::unstable::LittleEndianReadExt; +use std::io::Read; + +/// extended timestamp, as described in + +#[derive(Debug, Clone)] +pub struct ExtendedTimestamp { + mod_time: Option, + ac_time: Option, + cr_time: Option, +} + +impl ExtendedTimestamp { + /// creates an extended timestamp struct by reading the required bytes from the reader. + /// + /// This method assumes that the length has already been read, therefore + /// it must be passed as an argument + pub fn try_from_reader(reader: &mut R, len: u16) -> ZipResult + where + R: Read, + { + if len == 0 { + return Err(invalid!("Extended timestamp field is empty")); + } + let mut flags = [0u8]; + let mut bytes_to_read = len as usize; + reader.read_exact(&mut flags)?; + bytes_to_read -= flags.len(); + let flags = flags[0]; + + // the `flags` field refers to the local headers and might not correspond + // to the len field. If the length field is 1+4, we assume that only + // the modification time has been set + + // > Those times that are present will appear in the order indicated, but + // > any combination of times may be omitted. (Creation time may be + // > present without access time, for example.) TSize should equal + // > (1 + 4*(number of set bits in Flags)), as the block is currently + // > defined. + if len != 5 && len as u32 != 1 + 4 * flags.count_ones() { + //panic!("found len {len} and flags {flags:08b}"); + return Err(ZipError::UnsupportedArchive( + "flags and len don't match in extended timestamp field", + )); + } + + // allow unsupported/undocumented flags + + let mod_time = if (flags & 0b00000001u8 == 0b00000001u8) || len == 5 { + bytes_to_read -= size_of::(); + Some(reader.read_u32_le()?) + } else { + None + }; + + let ac_time = if flags & 0b00000010u8 == 0b00000010u8 && len > 5 { + bytes_to_read -= size_of::(); + Some(reader.read_u32_le()?) + } else { + None + }; + + let cr_time = if flags & 0b00000100u8 == 0b00000100u8 && len > 5 { + bytes_to_read -= size_of::(); + Some(reader.read_u32_le()?) + } else { + None + }; + + if bytes_to_read > 0 { + // ignore undocumented bytes + reader.read_exact(&mut vec![0; bytes_to_read])? + } + + Ok(Self { + mod_time, + ac_time, + cr_time, + }) + } + + /// returns the last modification timestamp, if defined, as UNIX epoch seconds + pub fn mod_time(&self) -> Option { + self.mod_time + } + + /// returns the last access timestamp, if defined, as UNIX epoch seconds + pub fn ac_time(&self) -> Option { + self.ac_time + } + + /// returns the creation timestamp, if defined, as UNIX epoch seconds + pub fn cr_time(&self) -> Option { + self.cr_time + } +} + +#[test] +/// Ensure we don't panic or read garbage data if the field body is empty +pub fn test_bad_extended_timestamp() -> ZipResult<()> { + use crate::ZipArchive; + use std::io::Cursor; + + assert!(ZipArchive::new(Cursor::new(include_bytes!( + "../../tests/data/extended_timestamp_bad.zip" + ))) + .is_err()); + Ok(()) +} diff --git a/v2/vendor/security/zip-7.2.0/src/extra_fields/mod.rs b/v2/vendor/security/zip-7.2.0/src/extra_fields/mod.rs new file mode 100644 index 0000000000..de6a3b6c72 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/extra_fields/mod.rs @@ -0,0 +1,35 @@ +//! types for extra fields + +/// marker trait to denote the place where this extra field has been stored +pub trait ExtraFieldVersion {} + +/// use this to mark extra fields specified in a local header + +#[derive(Debug, Clone)] +pub struct LocalHeaderVersion; + +/// use this to mark extra fields specified in the central header + +#[derive(Debug, Clone)] +pub struct CentralHeaderVersion; + +impl ExtraFieldVersion for LocalHeaderVersion {} +impl ExtraFieldVersion for CentralHeaderVersion {} + +mod extended_timestamp; +mod ntfs; +mod zipinfo_utf8; + +pub use extended_timestamp::*; +pub use ntfs::Ntfs; +pub use zipinfo_utf8::UnicodeExtraField; + +/// contains one extra field +#[derive(Debug, Clone)] +pub enum ExtraField { + /// NTFS extra field + Ntfs(Ntfs), + + /// extended timestamp, as described in + ExtendedTimestamp(ExtendedTimestamp), +} diff --git a/v2/vendor/security/zip-7.2.0/src/extra_fields/ntfs.rs b/v2/vendor/security/zip-7.2.0/src/extra_fields/ntfs.rs new file mode 100644 index 0000000000..d303735bac --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/extra_fields/ntfs.rs @@ -0,0 +1,97 @@ +use std::io::Read; + +use crate::{ + result::{ZipError, ZipResult}, + unstable::LittleEndianReadExt, +}; + +/// The NTFS extra field as described in [PKWARE's APPNOTE.TXT v6.3.9]. +/// +/// This field stores [Windows file times], which are 64-bit unsigned integer +/// values that represents the number of 100-nanosecond intervals that have +/// elapsed since "1601-01-01 00:00:00 UTC". +/// +/// [PKWARE's APPNOTE.TXT v6.3.9]: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT +/// [Windows file times]: https://docs.microsoft.com/en-us/windows/win32/sysinfo/file-times +#[derive(Clone, Debug)] +pub struct Ntfs { + mtime: u64, + atime: u64, + ctime: u64, +} + +impl Ntfs { + /// Creates a NTFS extra field struct by reading the required bytes from the + /// reader. + /// + /// This method assumes that the length has already been read, therefore it + /// must be passed as an argument. + pub fn try_from_reader(reader: &mut R, len: u16) -> ZipResult + where + R: Read, + { + if len != 32 { + return Err(ZipError::UnsupportedArchive( + "NTFS extra field has an unsupported length", + )); + } + + // Read reserved for future use. + let _ = reader.read_u32_le()?; + + let tag = reader.read_u16_le()?; + if tag != 0x0001 { + return Err(ZipError::UnsupportedArchive( + "NTFS extra field has an unsupported attribute tag", + )); + } + let size = reader.read_u16_le()?; + if size != 24 { + return Err(ZipError::UnsupportedArchive( + "NTFS extra field has an unsupported attribute size", + )); + } + + let mtime = reader.read_u64_le()?; + let atime = reader.read_u64_le()?; + let ctime = reader.read_u64_le()?; + Ok(Self { + mtime, + atime, + ctime, + }) + } + + /// Returns the file last modification time as a file time. + pub fn mtime(&self) -> u64 { + self.mtime + } + + /// Returns the file last modification time as a file time. + #[cfg(feature = "nt-time")] + pub fn modified_file_time(&self) -> nt_time::FileTime { + nt_time::FileTime::new(self.mtime) + } + + /// Returns the file last access time as a file time. + pub fn atime(&self) -> u64 { + self.atime + } + + /// Returns the file last access time as a file time. + #[cfg(feature = "nt-time")] + pub fn accessed_file_time(&self) -> nt_time::FileTime { + nt_time::FileTime::new(self.atime) + } + + /// Returns the file creation time as a file time. + pub fn ctime(&self) -> u64 { + self.ctime + } + + /// Returns the file creation time as a file time. + #[cfg(feature = "nt-time")] + pub fn created_file_time(&self) -> nt_time::FileTime { + nt_time::FileTime::new(self.ctime) + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/extra_fields/zipinfo_utf8.rs b/v2/vendor/security/zip-7.2.0/src/extra_fields/zipinfo_utf8.rs new file mode 100644 index 0000000000..c2c09e50cd --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/extra_fields/zipinfo_utf8.rs @@ -0,0 +1,40 @@ +use crate::result::{invalid, ZipResult}; +use crate::unstable::LittleEndianReadExt; +use core::mem::size_of; +use std::io::Read; + +/// Info-ZIP Unicode Path Extra Field (0x7075) or Unicode Comment Extra Field (0x6375), as +/// specified in APPNOTE 4.6.8 and 4.6.9 +#[derive(Clone, Debug)] +pub struct UnicodeExtraField { + crc32: u32, + content: Box<[u8]>, +} + +impl UnicodeExtraField { + /// Verifies the checksum and returns the content. + pub fn unwrap_valid(self, ascii_field: &[u8]) -> ZipResult> { + let mut crc32 = crc32fast::Hasher::new(); + crc32.update(ascii_field); + let actual_crc32 = crc32.finalize(); + if self.crc32 != actual_crc32 { + return Err(invalid!("CRC32 checksum failed on Unicode extra field")); + } + Ok(self.content) + } +} + +impl UnicodeExtraField { + pub(crate) fn try_from_reader(reader: &mut R, len: u16) -> ZipResult { + // Read and discard version byte + reader.read_exact(&mut [0u8])?; + + let crc32 = reader.read_u32_le()?; + let content_len = (len as usize) + .checked_sub(size_of::() + size_of::()) + .ok_or(invalid!("Unicode extra field is too small"))?; + let mut content = vec![0u8; content_len].into_boxed_slice(); + reader.read_exact(&mut content)?; + Ok(Self { crc32, content }) + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/legacy/huffman.rs b/v2/vendor/security/zip-7.2.0/src/legacy/huffman.rs new file mode 100644 index 0000000000..e7b2d3c77a --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/legacy/huffman.rs @@ -0,0 +1,260 @@ +use std::io::{self, Error, Seek}; + +use bitstream_io::{BitRead, BitReader, Endianness}; + +#[derive(Default, Clone, Copy)] +pub struct TableEntry { + /// Wide enough to fit the max symbol nbr. + pub sym: u16, + /// 0 means no symbol. + pub len: u8, +} + +/// Deflate uses max 288 symbols. +const MAX_HUFFMAN_SYMBOLS: usize = 288; +/// Implode uses max 16-bit codewords. +const MAX_HUFFMAN_BITS: usize = 16; +/// Seems a good trade-off. +const HUFFMAN_LOOKUP_TABLE_BITS: u8 = 8; + +pub struct HuffmanDecoder { + /// Lookup table for fast decoding of short codewords. + pub table: [TableEntry; 1 << HUFFMAN_LOOKUP_TABLE_BITS], + /// "Sentinel bits" value for each codeword length. + pub sentinel_bits: [u32; MAX_HUFFMAN_BITS + 1], + /// First symbol index minus first codeword mod 2**16 for each length. + pub offset_first_sym_idx: [u16; MAX_HUFFMAN_BITS + 1], + /// Map from symbol index to symbol. + pub syms: [u16; MAX_HUFFMAN_SYMBOLS], + // num_syms:usize +} + +impl Default for HuffmanDecoder { + fn default() -> Self { + Self { + table: [TableEntry::default(); 1 << HUFFMAN_LOOKUP_TABLE_BITS], + sentinel_bits: [0; MAX_HUFFMAN_BITS + 1], + offset_first_sym_idx: [0; MAX_HUFFMAN_BITS + 1], + syms: [0; MAX_HUFFMAN_SYMBOLS], + } + } +} + +/// Reverse the n least significant bits of x. +/// The (16 - n) most significant bits of the result will be zero. +pub fn reverse_lsb(x: u16, n: usize) -> u16 { + debug_assert!(n > 0); + debug_assert!(n <= 16); + x.reverse_bits() >> (16 - n) +} + +/// Initialize huffman decoder d for a code defined by the n codeword lengths. +/// Returns `Err` if the codeword lengths do not correspond to a valid prefix +/// code. +impl HuffmanDecoder { + pub fn init(&mut self, lengths: &[u8], n: usize) -> std::io::Result<()> { + let mut count = [0; MAX_HUFFMAN_BITS + 1]; + let mut code = [0; MAX_HUFFMAN_BITS + 1]; + let mut sym_idx = [0u16; MAX_HUFFMAN_BITS + 1]; + + // Count the number of codewords of each length. + for i in 0..n { + debug_assert!(lengths[i] as usize <= MAX_HUFFMAN_BITS); + count[lengths[i] as usize] += 1; + } + count[0] = 0; // Ignore zero-length codewords. + // Compute sentinel_bits and offset_first_sym_idx for each length. + code[0] = 0; + sym_idx[0] = 0; + for l in 1..=MAX_HUFFMAN_BITS { + // First canonical codeword of this length. + code[l] = (code[l - 1] + count[l - 1]) << 1; + + if count[l] != 0 && code[l] as u32 + count[l] as u32 - 1 > (1u32 << l) - 1 { + // The last codeword is longer than l bits. + return Err(Error::new( + io::ErrorKind::InvalidData, + "the last codeword is longer than len bits", + )); + } + + let s = (code[l] as u32 + count[l] as u32) << (MAX_HUFFMAN_BITS - l); + self.sentinel_bits[l] = s; + debug_assert!(self.sentinel_bits[l] >= code[l] as u32, "No overflow!"); + + sym_idx[l] = sym_idx[l - 1] + count[l - 1]; + self.offset_first_sym_idx[l] = sym_idx[l].wrapping_sub(code[l]); + } + + // Zero-initialize the lookup table. + self.table.fill(TableEntry::default()); + + // Build mapping from index to symbol and populate the lookup table. + lengths + .iter() + .enumerate() + .take(n) + .for_each(|(i, code_len)| { + let l = *code_len as usize; + if l == 0 { + return; + } + + self.syms[sym_idx[l] as usize] = i as u16; + sym_idx[l] += 1; + + if l <= HUFFMAN_LOOKUP_TABLE_BITS as usize { + self.table_insert(i, l, code[l]); + code[l] += 1; + } + }); + + Ok(()) + } + + pub fn table_insert(&mut self, sym: usize, len: usize, codeword: u16) { + debug_assert!(len <= HUFFMAN_LOOKUP_TABLE_BITS as usize); + + let codeword = reverse_lsb(codeword, len); // Make it LSB-first. + let pad_len = HUFFMAN_LOOKUP_TABLE_BITS as usize - len; + + // Pad the pad_len upper bits with all bit combinations. + for padding in 0..(1 << pad_len) { + let index = (codeword | (padding << len)) as usize; + debug_assert!(sym <= u16::MAX as usize); + self.table[index].sym = sym as u16; + debug_assert!(len <= u8::MAX as usize); + self.table[index].len = len as u8; + } + } + + /// Use the decoder d to decode a symbol from the LSB-first zero-padded bits. + /// Returns the decoded symbol number or an error if no symbol could be decoded. + /// *num_used_bits will be set to the number of bits used to decode the symbol, + /// or zero if no symbol could be decoded. + pub fn huffman_decode( + &mut self, + length: u64, + is: &mut BitReader, + ) -> std::io::Result { + // First try the lookup table. + let read_bits1 = (HUFFMAN_LOOKUP_TABLE_BITS as u64).min(length - is.position_in_bits()?); + let lookup_bits = !is.read_var::(read_bits1 as u32)? as usize; + debug_assert!(lookup_bits < self.table.len()); + if self.table[lookup_bits].len != 0 { + debug_assert!(self.table[lookup_bits].len <= HUFFMAN_LOOKUP_TABLE_BITS); + is.seek_bits(io::SeekFrom::Current( + -(read_bits1 as i64) + self.table[lookup_bits].len as i64, + ))?; + return Ok(self.table[lookup_bits].sym); + } + + // Then do canonical decoding with the bits in MSB-first order. + let read_bits2 = (HUFFMAN_LOOKUP_TABLE_BITS as u64).min(length - is.position_in_bits()?); + let mut bits = reverse_lsb( + (lookup_bits | ((!is.read_var::(read_bits2 as u32)? as usize) << read_bits1)) + as u16, + MAX_HUFFMAN_BITS, + ); + + for l in (HUFFMAN_LOOKUP_TABLE_BITS as usize + 1)..=MAX_HUFFMAN_BITS { + if (bits as u32) < self.sentinel_bits[l] { + bits >>= MAX_HUFFMAN_BITS - l; + let sym_idx = (self.offset_first_sym_idx[l] as usize + bits as usize) & 0xFFFF; + //assert(sym_idx < self.num_syms); + is.seek_bits(io::SeekFrom::Current( + -(read_bits1 as i64 + read_bits2 as i64) + l as i64, + ))?; + return Ok(self.syms[sym_idx]); + } + } + Err(Error::new( + io::ErrorKind::InvalidData, + "huffman decode failed", + )) + } +} + +#[cfg(test)] +mod tests { + use std::io::Cursor; + + use bitstream_io::{BitReader, LittleEndian}; + + use super::HuffmanDecoder; + #[test] + fn test_huffman_decode_basic() { + let lens = [ + 3, // sym 0: 000 + 3, // sym 1: 001 + 3, // sym 2: 010 + 3, // sym 3: 011 + 3, // sym 4: 100 + 3, // sym 5: 101 + 4, // sym 6: 1100 + 4, // sym 7: 1101 + 0, // sym 8: + 0, // sym 9: + 0, // sym 10: + 0, // sym 11: + 0, // sym 12: + 0, // sym 13: + 0, // sym 14: + 0, // sym 15: + 6, // sym 16: 111110 + 5, // sym 17: 11110 + 4, // sym 18: 1110 + ]; + + let mut d = HuffmanDecoder::default(); + d.init(&lens, lens.len()).unwrap(); + + // 000 (msb-first) -> 000 (lsb-first) + assert_eq!( + d.huffman_decode( + 8, + &mut BitReader::endian(&mut Cursor::new(&[!0x0]), LittleEndian) + ) + .unwrap(), + 0 + ); + + /* 011 (msb-first) -> 110 (lsb-first)*/ + assert_eq!( + d.huffman_decode( + 8, + &mut BitReader::endian(&mut Cursor::new(&[!0b110]), LittleEndian) + ) + .unwrap(), + 0b011 + ); + + /* 11110 (msb-first) -> 01111 (lsb-first)*/ + assert_eq!( + d.huffman_decode( + 8, + &mut BitReader::endian(&mut Cursor::new(&[!0b1111]), LittleEndian) + ) + .unwrap(), + 0b10001 + ); + + /* 111110 (msb-first) -> 011111 (lsb-first)*/ + assert_eq!( + d.huffman_decode( + 8, + &mut BitReader::endian(&mut Cursor::new(&[!0b11111]), LittleEndian) + ) + .unwrap(), + 0b10000 + ); + + /* 1111111 (msb-first) -> 1111111 (lsb-first)*/ + assert!(d + .huffman_decode( + 8, + &mut BitReader::endian(&mut Cursor::new(&[!0x7f]), LittleEndian) + ) + .is_err()); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/legacy/implode.rs b/v2/vendor/security/zip-7.2.0/src/legacy/implode.rs new file mode 100644 index 0000000000..bd35ed1a6f --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/legacy/implode.rs @@ -0,0 +1,250 @@ +use super::huffman::HuffmanDecoder; +use bitstream_io::{BitRead, BitReader, Endianness, LittleEndian}; +use std::io::{self, Cursor, Error, Read, Result}; + +/// Maximum symbol value in the length Huffman table (6 bits) +/// When this value is decoded, an additional byte is read for extended length +const MAX_LEN_SYMBOL: u16 = 63; + +/// Maximum number of symbols in a Huffman table (256 for literals, 64 for lengths/distances) +const MAX_HUFFMAN_SYMBOLS: usize = 1 << 8; // 256 + +/// Maximum code length in bits for Huffman codes (per ZIP specification) +const MAX_CODE_LENGTH: usize = 16; + +/// Size of the code length count array (lengths are actually 1..=16) +const CODE_LENGTH_COUNT_SIZE: usize = MAX_CODE_LENGTH; + +/// Initialize the Huffman decoder d with num_lens codeword lengths read from is. +/// Returns false if the input is invalid. +fn read_huffman_code( + is: &mut BitReader, + num_lens: usize, +) -> std::io::Result> { + let mut lens = [0; MAX_HUFFMAN_SYMBOLS]; + let mut len_count = [0; CODE_LENGTH_COUNT_SIZE]; + + // Number of bytes representing the Huffman code. + let byte = is.read::<8, u8>()?; + let num_bytes = (byte + 1) as usize; + + let mut codeword_idx = 0; + for _byte_idx in 0..num_bytes { + let byte = is.read::<8, u16>()?; + + let codeword_len = (byte & 0xf) + 1; /* Low four bits plus one. */ + let run_length = (byte >> 4) + 1; /* High four bits plus one. */ + + debug_assert!((1..=16).contains(&codeword_len)); + len_count[codeword_len as usize - 1] += run_length; + + if (codeword_idx + run_length) as usize > num_lens { + return Err(Error::new( + io::ErrorKind::InvalidData, + "too many codeword lengths", + )); + } + for _ in 0..run_length { + debug_assert!((codeword_idx as usize) < num_lens); + lens[codeword_idx as usize] = codeword_len as u8; + codeword_idx += 1; + } + } + + debug_assert!(codeword_idx as usize <= num_lens); + if (codeword_idx as usize) < num_lens { + return Err(Error::new( + io::ErrorKind::InvalidData, + "not enough codeword lengths", + )); + } + + // Check that the Huffman tree is full. + let mut avail_codewords = 1; + for i in 1..=16 { + debug_assert!(avail_codewords >= 0); + avail_codewords *= 2; + avail_codewords -= len_count[i - 1] as i32; + if avail_codewords < 0 { + return Err(Error::new( + io::ErrorKind::InvalidData, + "huffman tree is not full", + )); + } + } + if avail_codewords != 0 { + // Not all codewords were used. + return Err(Error::new( + io::ErrorKind::InvalidData, + "not all codewords were used", + )); + } + + let mut d = Box::new(HuffmanDecoder::default()); + d.init(&lens, num_lens)?; + Ok(d) +} + +fn hwexplode( + src: &[u8], + uncomp_len: usize, + large_wnd: bool, + lit_tree: bool, + pk101_bug_compat: bool, + dst: &mut Vec, +) -> std::io::Result<()> { + // Pre-allocate capacity + dst.reserve(uncomp_len); + + let bit_length = src.len() as u64 * 8; + let mut is = BitReader::endian(Cursor::new(&src), LittleEndian); + let mut lit_decoder_opt = if lit_tree { + Some(read_huffman_code(&mut is, 256)?) + } else { + None + }; + let mut len_decoder = read_huffman_code(&mut is, 64)?; + let mut dist_decoder = read_huffman_code(&mut is, 64)?; + let min_len = if (pk101_bug_compat && large_wnd) || (!pk101_bug_compat && lit_tree) { + 3 + } else { + 2 + }; + let dist_low_bits = if large_wnd { 7 } else { 6 }; + + while dst.len() < uncomp_len { + let is_literal = is.read_bit()?; + if is_literal { + // Literal. + let sym; + if let Some(lit_decoder) = &mut lit_decoder_opt { + sym = lit_decoder.huffman_decode(bit_length, &mut is)?; + } else { + sym = is.read::<8, u8>()? as u16; + } + debug_assert!(sym <= u8::MAX as u16); + dst.push(sym as u8); + continue; + } + + // Read the low dist bits. + let mut dist = is.read_var::(dist_low_bits)?; + // Read the Huffman-encoded high dist bits. + let sym = dist_decoder.huffman_decode(bit_length, &mut is)?; + dist |= (sym as u16) << dist_low_bits; + dist += 1; + + // Read the Huffman-encoded len. + let sym = len_decoder.huffman_decode(bit_length, &mut is)?; + let mut len = (sym + min_len) as usize; + + if sym == MAX_LEN_SYMBOL { + // Read an extra len byte. + len += is.read::<8, u16>()? as usize; + } + let len = len.min(uncomp_len - dst.len()); + + // Optimize the copy loop using extend_from_within when possible + if dist as usize <= dst.len() { + if dist as usize >= len { + // No overlap in copy, can use extend_from_within + let start = dst.len() - dist as usize; + dst.extend_from_within(start..start + len); + } else { + // Overlapping copy + for _ in 0..len { + let byte = dst[dst.len() - dist as usize]; + dst.push(byte); + } + } + } else { + // Copy with implicit zeros + for _ in 0..len { + if dist as usize > dst.len() { + dst.push(0); + } else { + let byte = dst[dst.len() - dist as usize]; + dst.push(byte); + } + } + } + } + Ok(()) +} + +#[derive(Debug)] +pub struct ImplodeDecoder { + compressed_reader: R, + uncompressed_size: u64, + stream_read: bool, + large_wnd: bool, + lit_tree: bool, + stream: Vec, + read_pos: usize, // Add read position tracker +} + +impl ImplodeDecoder { + pub fn new(inner: R, uncompressed_size: u64, flags: u16) -> Self { + let large_wnd = (flags & 2) != 0; + let lit_tree = (flags & 4) != 0; + ImplodeDecoder { + compressed_reader: inner, + uncompressed_size, + stream_read: false, + large_wnd, + lit_tree, + stream: Vec::new(), + read_pos: 0, + } + } + + pub fn into_inner(self) -> R { + self.compressed_reader + } +} + +impl Read for ImplodeDecoder { + fn read(&mut self, buf: &mut [u8]) -> Result { + if !self.stream_read { + self.stream_read = true; + let mut compressed_bytes = Vec::new(); + self.compressed_reader.read_to_end(&mut compressed_bytes)?; + + // Pre-allocate stream buffer + self.stream.reserve(self.uncompressed_size as usize); + + hwexplode( + &compressed_bytes, + self.uncompressed_size as usize, + self.large_wnd, + self.lit_tree, + false, + &mut self.stream, + )?; + } + + let available = self.stream.len() - self.read_pos; + let bytes_to_read = available.min(buf.len()); + buf[..bytes_to_read] + .copy_from_slice(&self.stream[self.read_pos..self.read_pos + bytes_to_read]); + self.read_pos += bytes_to_read; + Ok(bytes_to_read) + } +} + +#[cfg(test)] +mod tests { + use super::hwexplode; + + const HAMLET_256: &[u8; 249] = include_bytes!("../../tests/data/legacy/implode_hamlet_256.bin"); + const HAMLET_256_OUT: &[u8; 256] = + include_bytes!("../../tests/data/legacy/implode_hamlet_256.out"); + + #[test] + fn test_explode_hamlet_256() { + let mut dst = Vec::new(); + hwexplode(HAMLET_256, 256, false, false, false, &mut dst).unwrap(); + assert_eq!(dst.len(), 256); + assert_eq!(&dst, &HAMLET_256_OUT); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/legacy/mod.rs b/v2/vendor/security/zip-7.2.0/src/legacy/mod.rs new file mode 100644 index 0000000000..80858e2110 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/legacy/mod.rs @@ -0,0 +1,4 @@ +mod huffman; +pub(crate) mod implode; +pub(crate) mod reduce; +pub(crate) mod shrink; diff --git a/v2/vendor/security/zip-7.2.0/src/legacy/reduce.rs b/v2/vendor/security/zip-7.2.0/src/legacy/reduce.rs new file mode 100644 index 0000000000..e90d06a0ef --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/legacy/reduce.rs @@ -0,0 +1,313 @@ +use std::io::{self, Read, Result}; + +use bitstream_io::{BitRead, BitReader, Endianness, LittleEndian}; + +/// Number of bits used to represent indices in a follower set of size n. +const fn follower_idx_bitlen(n: u8) -> u8 { + debug_assert!(n <= 32); + match n { + 0 => 0, + 1 => 1, + _ => 8 - (n - 1).leading_zeros() as u8, + } +} + +#[derive(Default, Clone, Copy)] +struct FollowerSet { + followers: [u8; 32], + size: u8, + idx_bitlen: u8, +} + +/// Read the follower sets from is into fsets. Returns true on success. +type FollowerSetArray = [FollowerSet; u8::MAX as usize + 1]; + +fn read_follower_sets( + is: &mut BitReader, +) -> io::Result { + let mut fsets = [FollowerSet::default(); u8::MAX as usize + 1]; + for i in (0..=u8::MAX as usize).rev() { + let n = is.read::<6, u8>()?; + if n > 32 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid follower set", + )); + } + fsets[i].size = n; + fsets[i].idx_bitlen = follower_idx_bitlen(n); + + for j in 0..fsets[i].size as usize { + fsets[i].followers[j] = is.read::<8, u8>()?; + } + } + + Ok(fsets) +} + +/// Read the next byte from is, decoded based on prev_byte and the follower sets. +/// The byte is returned in *out_byte. +/// +/// # Returns +/// +/// * `Ok` with the byte if it was successfully read. +/// * `Err(io::Error)` on bad data or end of input. +fn read_next_byte( + is: &mut BitReader, + prev_byte: u8, + fsets: &mut FollowerSetArray, +) -> io::Result { + if fsets[prev_byte as usize].size == 0 // No followers + || is.read::<1, u8>()? == 1 + // Indicates next symbol is a literal byte + { + return is.read::<8, u8>(); + } + + // The bits represent the index of a follower byte. + let idx_bitlen = fsets[prev_byte as usize].idx_bitlen; + let follower_idx = is.read_var::(idx_bitlen as u32)? as usize; + if follower_idx >= fsets[prev_byte as usize].size as usize { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid follower index", + )); + } + Ok(fsets[prev_byte as usize].followers[follower_idx]) +} + +fn max_len(comp_factor: u8) -> usize { + debug_assert!((1..=4).contains(&comp_factor)); + let v_len_bits = (8 - comp_factor) as usize; + // Bits in V + extra len byte + implicit 3. + ((1 << v_len_bits) - 1) + u8::MAX as usize + 3 +} + +fn max_dist(comp_factor: u8) -> usize { + debug_assert!((1..=4).contains(&comp_factor)); + let v_dist_bits = comp_factor as usize; + // Bits in V * 256 + W byte + implicit 1. */ + 1 << (v_dist_bits + 8) +} + +const DLE_BYTE: u8 = 0x90; + +/// Get the n least significant bits of x. +fn lsb(x: u8, n: u8) -> u8 { + if n >= 8 { + return x; + } + x & ((1 << n) - 1) +} + +fn hwexpand(src: &[u8], uncomp_len: usize, comp_factor: u8, dst: &mut Vec) -> io::Result<()> { + debug_assert!((1..=4).contains(&comp_factor)); + + // Pre-allocate to avoid reallocations + dst.reserve(uncomp_len); + + let mut is = BitReader::endian(src, LittleEndian); + let mut fsets = read_follower_sets(&mut is)?; + + // Number of bits in V used for backref length. + let v_len_bits = 8 - comp_factor; + + let mut curr_byte = 0; // The first "previous byte" is implicitly zero. + + while dst.len() < uncomp_len { + // Read a literal byte or DLE marker. + curr_byte = read_next_byte(&mut is, curr_byte, &mut fsets)?; + if curr_byte != DLE_BYTE { + // Output a literal byte. + dst.push(curr_byte); + continue; + } + + // Read the V byte which determines the length. + curr_byte = read_next_byte(&mut is, curr_byte, &mut fsets)?; + if curr_byte == 0 { + // Output a literal DLE byte. + dst.push(DLE_BYTE); + continue; + } + let v = curr_byte; + let mut len = lsb(v, v_len_bits) as usize; + if len == (1 << v_len_bits) - 1 { + // Read an extra length byte. + curr_byte = read_next_byte(&mut is, curr_byte, &mut fsets)?; + len += curr_byte as usize; + } + len += 3; + + // Read the W byte, which together with V gives the distance. + curr_byte = read_next_byte(&mut is, curr_byte, &mut fsets)?; + let dist = (((v as usize) >> v_len_bits) << 8) + curr_byte as usize + 1; + + debug_assert!(len <= max_len(comp_factor)); + debug_assert!(dist <= max_dist(comp_factor)); + + // Output the back reference. + if dist <= dst.len() { + // Optimize for non-overlapping copies + if dist >= len { + // No overlap, can use extend_from_within + let start = dst.len() - dist; + dst.extend_from_within(start..start + len.min(uncomp_len - dst.len())); + } else { + // Overlapping copy + let copy_len = len.min(uncomp_len - dst.len()); + for _ in 0..copy_len { + let byte = dst[dst.len() - dist]; + dst.push(byte); + } + } + } else { + // Copy with implicit zeros + let copy_len = len.min(uncomp_len - dst.len()); + for _ in 0..copy_len { + if dist > dst.len() { + dst.push(0); + } else { + let byte = dst[dst.len() - dist]; + dst.push(byte); + } + } + } + } + Ok(()) +} + +#[derive(Debug)] +pub struct ReduceDecoder { + compressed_reader: R, + uncompressed_size: u64, + stream_read: bool, + comp_factor: u8, + stream: Vec, + read_pos: usize, // Add read position tracker +} + +impl ReduceDecoder { + pub fn new(inner: R, uncompressed_size: u64, comp_factor: u8) -> Self { + ReduceDecoder { + compressed_reader: inner, + uncompressed_size, + stream_read: false, + comp_factor, + stream: Vec::new(), + read_pos: 0, + } + } + + pub fn into_inner(self) -> R { + self.compressed_reader + } +} + +impl Read for ReduceDecoder { + fn read(&mut self, buf: &mut [u8]) -> Result { + if !self.stream_read { + self.stream_read = true; + let mut compressed_bytes = Vec::new(); + self.compressed_reader.read_to_end(&mut compressed_bytes)?; + hwexpand( + &compressed_bytes, + self.uncompressed_size as usize, + self.comp_factor, + &mut self.stream, + )?; + } + + let available = self.stream.len() - self.read_pos; + let bytes_to_read = available.min(buf.len()); + buf[..bytes_to_read] + .copy_from_slice(&self.stream[self.read_pos..self.read_pos + bytes_to_read]); + self.read_pos += bytes_to_read; + Ok(bytes_to_read) + } +} + +#[cfg(test)] +mod tests { + use super::hwexpand; + use crate::legacy::reduce::{follower_idx_bitlen, lsb, max_dist}; + const HAMLET_2048: &[u8; 1285] = + include_bytes!("../../tests/data/legacy/reduce_hamlet_2048.bin"); + const HAMLET_2048_OUT: &[u8; 2048] = + include_bytes!("../../tests/data/legacy/implode_hamlet_2048.out"); + + #[test] + fn test_lsb() { + assert_eq!(lsb(0xFF, 8), 0xFF); + for i in 0..7 { + assert_eq!(lsb(0xFF, i), (1 << i) - 1); + } + } + + #[test] + fn test_expand_hamlet2048() { + let mut dst = Vec::new(); + hwexpand(HAMLET_2048, 2048, 4, &mut dst).unwrap(); + assert_eq!(dst.len(), 2048); + assert_eq!(&dst, &HAMLET_2048_OUT); + } + + /* + Put some text first to make PKZIP actually use Reduce compression. + Target the code path which copies a zero when dist > current position. + + $ curl -O http://cd.textfiles.com/originalsw/25/pkz092.exe + $ dosbox -c "mount c ." -c "c:" -c "pkz092" -c "exit" + $ dd if=hamlet.txt bs=1 count=2048 > a + $ dd if=/dev/zero bs=1 count=1024 >> a + $ dosbox -c "mount c ." -c "c:" -c "pkzip -ea4 a.zip a" -c "exit" + $ xxd -i -s 31 -l $(expr $(find A.ZIP -printf %s) - 100) A.ZIP + */ + const ZEROS_REDUCED: &[u8; 1297] = + include_bytes!("../../tests/data/legacy/reduce_zero_reduced.bin"); + + #[test] + fn test_expand_zeros() { + let mut dst = Vec::new(); + hwexpand(ZEROS_REDUCED, 2048 + 1024, 4, &mut dst).unwrap(); + assert_eq!(dst.len(), 2048 + 1024); + for i in 0..(1 << 10) { + assert_eq!(dst[(1 << 11) + i], 0); + } + } + + fn orig_follower_idx_bitlen(n: u8) -> u8 { + if n > 16 { + return 5; + } + if n > 8 { + return 4; + } + if n > 4 { + return 3; + } + if n > 2 { + return 2; + } + if n > 0 { + return 1; + } + 0 + } + + #[test] + fn test_follower_idx_biten() { + for i in 0..=32 { + assert_eq!(orig_follower_idx_bitlen(i), follower_idx_bitlen(i)); + } + } + + #[test] + fn test_max_dist() { + for i in 1..=4 { + let v_dist_bits = i as usize; + let c = 1 << (v_dist_bits + 8); + assert_eq!(max_dist(i), c); + } + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/legacy/shrink.rs b/v2/vendor/security/zip-7.2.0/src/legacy/shrink.rs new file mode 100644 index 0000000000..c8615a920a --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/legacy/shrink.rs @@ -0,0 +1,373 @@ +use std::io::{self, Error, Read}; + +use bitstream_io::{BitRead, BitReader, Endianness, LittleEndian}; + +const MIN_CODE_SIZE: u8 = 9; +const MAX_CODE_SIZE: u8 = 13; + +const MAX_CODE: usize = (1 << MAX_CODE_SIZE) - 1; +const CONTROL_CODE: usize = 256; +const INC_CODE_SIZE: u16 = 1; +const PARTIAL_CLEAR: u16 = 2; + +/// Number of codes available for the LZW dictionary (excluding control codes) +/// These are the codes from CONTROL_CODE+1 to MAX_CODE +const FREE_CODE_QUEUE_SIZE: usize = MAX_CODE - CONTROL_CODE + 1; + +// const HASH_BITS: usize = MAX_CODE_SIZE + 1; /* For a load factor of 0.5. */ +// const HASHTAB_SIZE: usize = 1 << HASH_BITS; +const UNKNOWN_LEN: u16 = u16::MAX; + +struct CodeQueue { + next_idx: usize, + codes: [Option; FREE_CODE_QUEUE_SIZE], +} + +impl CodeQueue { + fn new() -> Self { + let mut codes = [None; FREE_CODE_QUEUE_SIZE]; + for (i, code) in ((CONTROL_CODE as u16 + 1)..=(MAX_CODE as u16 - 1)).enumerate() { + codes[i] = Some(code); + } + Self { next_idx: 0, codes } + } + + // Return the next code in the queue, or INVALID_CODE if the queue is empty. + fn next(&self) -> Option { + // assert(q->next_idx < sizeof(q->codes) / sizeof(q->codes[0])); + if let Some(Some(next)) = self.codes.get(self.next_idx) { + Some(*next) + } else { + None + } + } + + /// Return and remove the next code from the queue, or return INVALID_CODE if + /// the queue is empty. + fn remove_next(&mut self) -> Option { + let res = self.next(); + if res.is_some() { + self.next_idx += 1; + } + res + } +} + +#[derive(Clone, Debug, Copy, Default)] +struct Codetab { + last_dst_pos: usize, + prefix_code: Option, + len: u16, + ext_byte: u8, +} + +impl Codetab { + pub fn create_new() -> [Self; MAX_CODE + 1] { + let mut codetab = [Codetab::default(); MAX_CODE + 1]; + for (i, code) in codetab.iter_mut().enumerate().take(u8::MAX as usize + 1) { + *code = Codetab { + prefix_code: Some(i as u16), + ext_byte: i as u8, + len: 1, + last_dst_pos: 0, + }; + } + codetab + } +} + +fn unshrink_partial_clear(codetab: &mut [Codetab], queue: &mut CodeQueue) { + let mut is_prefix = [false; MAX_CODE + 1]; + + // Scan for codes that have been used as a prefix. + for code in codetab.iter().take(MAX_CODE + 1).skip(CONTROL_CODE + 1) { + if let Some(prefix_code) = code.prefix_code { + is_prefix[prefix_code as usize] = true; + } + } + + // Clear "non-prefix" codes in the table; populate the code queue. + let mut code_queue_size = 0; + for i in (CONTROL_CODE + 1)..MAX_CODE { + if !is_prefix[i] { + codetab[i].prefix_code = None; + queue.codes[code_queue_size] = Some(i as u16); + code_queue_size += 1; + } + } + queue.codes[code_queue_size] = None; // End-of-queue marker. + queue.next_idx = 0; +} + +/// Read the next code from the input stream and return it in next_code. Returns +/// `Ok(None)` if the end of the stream is reached. If the stream contains invalid +/// data, returns `Err`. +fn read_code( + is: &mut BitReader, + code_size: &mut u8, + codetab: &mut [Codetab], + queue: &mut CodeQueue, +) -> io::Result> { + // assert(sizeof(code) * CHAR_BIT >= *code_size); + let code = is.read_var::(*code_size as u32)?; + + // Handle regular codes (the common case). + if code != CONTROL_CODE as u16 { + return Ok(Some(code)); + } + + // Handle control codes. + if let Ok(control_code) = is.read_var::(*code_size as u32) { + match control_code { + INC_CODE_SIZE => { + if *code_size >= MAX_CODE_SIZE { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "tried to increase code size when already at maximum", + )); + } + *code_size += 1; + } + PARTIAL_CLEAR => { + unshrink_partial_clear(codetab, queue); + } + _ => { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!("Invalid control code {}", control_code), + )); + } + } + return read_code(is, code_size, codetab, queue); + } + Ok(None) +} + +/// Output the string represented by a code into dst at dst_pos. +/// +/// # Returns +/// new first byte of the string, or io::Error InvalidData on invalid prefix code. +fn output_code( + dst: &mut Vec, + code: u16, + prev_code: u16, + codetab: &mut [Codetab], + queue: &CodeQueue, + first_byte: u8, +) -> io::Result { + debug_assert!(code <= MAX_CODE as u16 && code != CONTROL_CODE as u16); + if code <= u8::MAX as u16 { + dst.push(code as u8); + return Ok(code as u8); + } + if codetab[code as usize].prefix_code.is_none() + || codetab[code as usize].prefix_code == Some(code) + { + // Reject invalid codes. Self-referential codes may exist in + // the table but cannot be used. + return Err(io::Error::new(io::ErrorKind::InvalidData, "invalid code")); + } + if codetab[code as usize].len != UNKNOWN_LEN { + // Output string with known length (the common case). + let ct = &codetab[code as usize]; + for i in ct.last_dst_pos..ct.last_dst_pos + ct.len as usize { + dst.push(dst[i]); + } + return Ok(dst[ct.last_dst_pos]); + } + + // Output a string of unknown length. This happens when the prefix + // was invalid (due to partial clearing) when the code was inserted into + // the table. The prefix can then become valid when it's added to the + // table at a later point. + let prefix_code = codetab[code as usize].prefix_code.unwrap(); + if cfg!(debug_assertions) { + let tab_entry = codetab[code as usize]; + assert!(tab_entry.len == UNKNOWN_LEN); + assert!(tab_entry.prefix_code.unwrap() as usize > CONTROL_CODE); + } + + if Some(prefix_code) == queue.next() { + /* The prefix code hasn't been added yet, but we were just + about to: the KwKwK case. Add the previous string extended + with its first byte. */ + codetab[prefix_code as usize] = Codetab { + prefix_code: Some(prev_code), + ext_byte: first_byte, + len: codetab[prev_code as usize].len + 1, + last_dst_pos: codetab[prev_code as usize].last_dst_pos, + }; + dst.push(first_byte); + } else if codetab[prefix_code as usize].prefix_code.is_none() { + // The prefix code is still invalid. + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "invalid prefix code", + )); + } + + // Output the prefix string, then the extension byte. + let ct = codetab[prefix_code as usize]; + let last_dst_pos = dst.len(); + let start = ct.last_dst_pos; + let len = ct.len as usize; + let first = dst[start]; + dst.extend_from_within(start..start + len); + dst.push(codetab[code as usize].ext_byte); + // Update the code table now that the string has a length and pos. + debug_assert!(prev_code != code); + + codetab[code as usize].len = (len + 1) as u16; + codetab[code as usize].last_dst_pos = last_dst_pos; + + Ok(first) +} + +fn hwunshrink(src: &[u8], uncompressed_size: usize, dst: &mut Vec) -> io::Result<()> { + dst.reserve(uncompressed_size); + let mut codetab = Codetab::create_new(); + let mut queue = CodeQueue::new(); + let mut is = BitReader::endian(src, LittleEndian); + let mut code_size = MIN_CODE_SIZE; + + // Handle the first code separately since there is no previous code. + let Some(curr_code) = read_code(&mut is, &mut code_size, &mut codetab, &mut queue)? else { + return Ok(()); + }; + + debug_assert!(curr_code != CONTROL_CODE as u16); + if curr_code > u8::MAX as u16 { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "the first code must be a literal", + )); + } + let mut first_byte = curr_code as u8; // make mutable + codetab[curr_code as usize].last_dst_pos = dst.len(); + dst.push(first_byte); + let mut prev_code = curr_code; + while dst.len() < uncompressed_size { + let curr_opt = match read_code(&mut is, &mut code_size, &mut codetab, &mut queue) { + Ok(c) => c, + Err(_) => break, // treat read error as end-of-stream for now; validated after loop + }; + + let Some(curr_code) = curr_opt else { + return Err(Error::new(io::ErrorKind::InvalidData, "invalid code")); + }; + + let dst_pos = dst.len(); + // Handle KwKwK: next code used before being added. + if Some(curr_code) == queue.next() { + if codetab[prev_code as usize].prefix_code.is_none() { + return Err(Error::new( + io::ErrorKind::InvalidData, + "previous code no longer valid", + )); + } + // Extend the previous code with its first byte. + debug_assert!(curr_code != prev_code); + codetab[curr_code as usize] = Codetab { + prefix_code: Some(prev_code), + ext_byte: first_byte, + len: codetab[prev_code as usize].len + 1, + last_dst_pos: codetab[prev_code as usize].last_dst_pos, + }; + } + // Output the string represented by the current code. + first_byte = output_code(dst, curr_code, prev_code, &mut codetab, &queue, first_byte)?; + if let Some(new_code) = queue.remove_next() { + let prev_entry = codetab[prev_code as usize]; + codetab[new_code as usize] = Codetab { + prefix_code: Some(prev_code), + ext_byte: first_byte, + last_dst_pos: prev_entry.last_dst_pos, + len: if prev_entry.prefix_code.is_none() { + // prev_code was invalidated in a partial + // clearing. Until that code is re-used, the + // string represented by new_code is + // indeterminate. + UNKNOWN_LEN + } else { + prev_entry.len + 1 + }, + }; + } + codetab[curr_code as usize].last_dst_pos = dst_pos; + prev_code = curr_code; + } + + if dst.len() != uncompressed_size { + return Err(io::Error::new( + io::ErrorKind::UnexpectedEof, + "unexpected end of compressed stream", + )); + } + + Ok(()) +} + +#[derive(Debug)] +pub struct ShrinkDecoder { + compressed_reader: R, + stream_read: bool, + uncompressed_size: u64, + stream: Vec, + read_pos: usize, +} + +impl ShrinkDecoder { + pub fn new(inner: R, uncompressed_size: u64) -> Self { + Self { + compressed_reader: inner, + uncompressed_size, + stream_read: false, + stream: Vec::new(), + read_pos: 0, + } + } + pub fn into_inner(self) -> R { + self.compressed_reader + } +} + +impl Read for ShrinkDecoder { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + if !self.stream_read { + self.stream_read = true; + let mut compressed_bytes = Vec::new(); + self.compressed_reader.read_to_end(&mut compressed_bytes)?; + self.stream.reserve(self.uncompressed_size as usize); + hwunshrink( + &compressed_bytes, + self.uncompressed_size as usize, + &mut self.stream, + )?; + } + let available = self.stream.len().saturating_sub(self.read_pos); + if available == 0 { + return Ok(0); + } + let n = available.min(buf.len()); + buf[..n].copy_from_slice(&self.stream[self.read_pos..self.read_pos + n]); + self.read_pos += n; + Ok(n) + } +} + +#[cfg(test)] +mod tests { + use crate::legacy::shrink::hwunshrink; + + const LZW_FIG5: &[u8; 17] = b"ababcbababaaaaaaa"; + const LZW_FIG5_SHRUNK: [u8; 12] = [ + 0x61, 0xc4, 0x04, 0x1c, 0x23, 0xb0, 0x60, 0x98, 0x83, 0x08, 0xc3, 0x00, + ]; + + #[test] + fn test_unshrink_lzw_fig5() { + let mut dst = Vec::with_capacity(LZW_FIG5.len()); + hwunshrink(&LZW_FIG5_SHRUNK, LZW_FIG5.len(), &mut dst).unwrap(); + assert_eq!(dst.as_slice(), LZW_FIG5); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/lib.rs b/v2/vendor/security/zip-7.2.0/src/lib.rs new file mode 100644 index 0000000000..adc69dc1cb --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/lib.rs @@ -0,0 +1,76 @@ +//! A library for reading and writing ZIP archives. +//! ZIP is a format designed for cross-platform file "archiving". +//! That is, storing a collection of files in a single datastream +//! to make them easier to share between computers. +//! Additionally, ZIP is able to compress and encrypt files in its +//! archives. +//! +//! The current implementation is based on [PKWARE's APPNOTE.TXT v6.3.9](https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT) +//! +//! --- +//! +//! [`zip`](`crate`) has support for the most common ZIP archives found in common use. +//! However, in special cases, +//! there are some zip archives that are difficult to read or write. +//! +//! This is a list of supported features: +//! +//! | | Reading | Writing | +//! | ------------------------------- | ------------------------------------- | -------------------------------------------------------- | +//! | Stored | ✅ | ✅ | +//! | Deflate | ✅ [->](`crate::ZipArchive::by_name`) | ✅ [->](`crate::write::FileOptions::compression_method`) | +//! | Deflate64 | ✅ | | +//! | Bzip2 | ✅ | ✅ | +//! | ZStandard | ✅ | ✅ | +//! | LZMA | ✅ | | +//! | XZ | ✅ | ✅ | +//! | PPMd | ✅ | ✅ | +//! | AES encryption | ✅ | ✅ | +//! | ZipCrypto deprecated encryption | ✅ | ✅ | +//! + +#![cfg_attr(docsrs, feature(doc_cfg))] +#![warn(missing_docs)] +#![allow(unexpected_cfgs)] // Needed for cfg(fuzzing) +pub use crate::compression::{CompressionMethod, SUPPORTED_COMPRESSION_METHODS}; +pub use crate::read::HasZipMetadata; +pub use crate::read::{ZipArchive, ZipReadOptions}; +pub use crate::spec::{ZIP64_BYTES_THR, ZIP64_ENTRY_THR}; +pub use crate::types::{AesMode, DateTime}; +pub use crate::write::ZipWriter; + +#[cfg(feature = "aes-crypto")] +mod aes; +#[cfg(feature = "aes-crypto")] +mod aes_ctr; +mod compression; +mod cp437; +mod crc32; +pub mod extra_fields; +mod path; +pub mod read; +pub mod result; +mod spec; +mod types; +pub mod write; +mod zipcrypto; +pub use extra_fields::ExtraField; +#[cfg(feature = "legacy-zip")] +mod legacy; + +#[doc = "Unstable APIs\n\ +\ +All APIs accessible by importing this module are unstable; They may be changed in patch \ +releases. You MUST use an exact version specifier in `Cargo.toml`, to indicate the version of this \ +API you're using:\n\ +\ +```toml\n +[dependencies]\n +zip = \"="] +#[doc=env!("CARGO_PKG_VERSION")] +#[doc = "\"\n\ +```"] +pub mod unstable; + +#[doc(hidden)] +pub mod macros; diff --git a/v2/vendor/security/zip-7.2.0/src/macros.rs b/v2/vendor/security/zip-7.2.0/src/macros.rs new file mode 100644 index 0000000000..3d07efd7b8 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/macros.rs @@ -0,0 +1,162 @@ +//! Macros used internally. +//! +//! These may technically be exported, but that's only to make them available to internal +//! project dependencies. The `#[doc(hidden)]` mark indicates that these are not stable or supported +//! APIs, and should not be relied upon by external dependees. + +/// The single macro export of the [`cfg-if`](https://docs.rs/cfg-if) crate. +/// +/// It is packaged here to avoid pulling in another dependency. The stdlib does the same[^1]. +/// +/// [^1]: https://github.com/rust-lang/rust/blob/a2db9280539229a3b8a084a09886670a57bc7e9c/library/compiler-builtins/libm/src/math/support/macros.rs#L1 +#[doc(hidden)] +#[macro_export] +macro_rules! cfg_if { + // match if/else chains with a final `else` + ( + $( + if #[cfg( $i_meta:meta )] { $( $i_tokens:tt )* } + ) else+ + else { $( $e_tokens:tt )* } + ) => { + $crate::cfg_if! { + @__items () ; + $( + (( $i_meta ) ( $( $i_tokens )* )) , + )+ + (() ( $( $e_tokens )* )) , + }; + }; + + // match if/else chains lacking a final `else` + ( + if #[cfg( $i_meta:meta )] { $( $i_tokens:tt )* } + $( + else if #[cfg( $e_meta:meta )] { $( $e_tokens:tt )* } + )* + ) => { + $crate::cfg_if! { + @__items () ; + (( $i_meta ) ( $( $i_tokens )* )) , + $( + (( $e_meta ) ( $( $e_tokens )* )) , + )* + }; + }; + + // Internal and recursive macro to emit all the items + // + // Collects all the previous cfgs in a list at the beginning, so they can be + // negated. After the semicolon are all the remaining items. + (@__items ( $( $_:meta , )* ) ; ) => {}; + ( + @__items ( $( $no:meta , )* ) ; + (( $( $yes:meta )? ) ( $( $tokens:tt )* )) , + $( $rest:tt , )* + ) => { + // Emit all items within one block, applying an appropriate #[cfg]. The + // #[cfg] will require all `$yes` matchers specified and must also negate + // all previous matchers. + #[cfg(all( + $( $yes , )? + not(any( $( $no ),* )) + ))] + $crate::cfg_if! { @__identity $( $tokens )* } + + // Recurse to emit all other items in `$rest`, and when we do so add all + // our `$yes` matchers to the list of `$no` matchers as future emissions + // will have to negate everything we just matched as well. + $crate::cfg_if! { + @__items ( $( $no , )* $( $yes , )? ) ; + $( $rest , )* + }; + }; + + // Internal macro to make __apply work out right for different match types, + // because of how macros match/expand stuff. + (@__identity $( $tokens:tt )* ) => { + $( $tokens )* + }; +} + +/// Similar to [`cfg_if`](cfg_if), but accepts a list of expressions, and generates an internal +/// closure to return each value. +/// +/// The main reason this is necessary is because attaching `#[cfg(...)]` annotations to certain +/// types of statements requires a nightly feature, or `cfg_if` would be enough on its own. This +/// macro's restricted interface allows it to generate a closure as a circumlocution that is legal +/// on stable rust. +/// +/// Note that any `return` operation within the expressions provided to this macro will apply to the +/// generated closure, not the enclosing scope--it cannot be used to interfere with external +/// control flow. +/// +/// The generated closure is non-[`const`](const@keyword), so cannot be used inside `const` methods. +#[doc(hidden)] +#[macro_export] +macro_rules! cfg_if_expr { + // Match =>, chains, maybe with a final _ => catchall clause. + ( + $( $ret_ty:ty : )? + $( + #[cfg( $i_meta:meta )] => $i_val:expr + ),+ , + _ => $rem_val:expr $(,)? + ) => { + (|| $( -> $ret_ty )? { + $crate::cfg_if_expr! { + @__items (); + $( + (( $i_meta ) ( + #[allow(unreachable_code)] + return $i_val ; + )) , + )+ + (() ( + #[allow(unreachable_code)] + return $rem_val ; + )) , + } + })() + }; + // Match =>, chains *without* any _ => clause. + ( + $( $ret_ty:ty : )? + $( + #[cfg( $i_meta:meta )] => $i_val:expr + ),+ $(,)? + ) => { + (|| $( -> $ret_ty )? { + $crate::cfg_if_expr! { + @__items (); + $( + (( $i_meta ) ( + #[allow(unreachable_code)] + return $i_val ; + )) , + )+ + } + })() + }; + + (@__items ( $( $_:meta , )* ) ; ) => {}; + ( + @__items ( $( $no:meta , )* ) ; + (( $( $yes:meta )? ) ( $( $tokens:tt )* )) , + $( $rest:tt , )* + ) => { + #[cfg(all( + $( $yes , )? + not(any( $( $no ),* )) + ))] + $crate::cfg_if_expr! { @__identity $( $tokens )* } + + $crate::cfg_if_expr! { + @__items ( $( $no , )* $( $yes , )? ) ; + $( $rest , )* + }; + }; + (@__identity $( $tokens:tt )* ) => { + $( $tokens )* + }; +} diff --git a/v2/vendor/security/zip-7.2.0/src/path.rs b/v2/vendor/security/zip-7.2.0/src/path.rs new file mode 100644 index 0000000000..8f0c698a03 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/path.rs @@ -0,0 +1,110 @@ +//! Path manipulation utilities + +use std::{ffi::OsStr, path::Path}; +use typed_path::{Utf8WindowsComponent, Utf8WindowsPath}; + +/// Simplify a path by removing the prefix and parent directories and only return normal components +pub(crate) fn simplified_components(input: &Path) -> Option> { + let mut out = Vec::new(); + let input_str = input.to_str()?; + for component in Utf8WindowsPath::new(input_str).components() { + match component { + // Skip prefix and root directory components instead of rejecting the entire path + // This allows extraction of ZIP files with absolute paths, similar to other ZIP tools + Utf8WindowsComponent::Prefix(_) | Utf8WindowsComponent::RootDir => (), + Utf8WindowsComponent::ParentDir => { + out.pop()?; + } + Utf8WindowsComponent::Normal(s) => out.push(OsStr::new(s)), + Utf8WindowsComponent::CurDir => (), + } + } + Some(out) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::path::Path; + + #[test] + fn test_simplified_components_relative_path() { + let path = Path::new("foo/bar/baz.txt"); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 3); + assert_eq!(components[0], "foo"); + assert_eq!(components[1], "bar"); + assert_eq!(components[2], "baz.txt"); + } + + #[test] + fn test_simplified_components_absolute_unix_path() { + let path = Path::new("/foo/bar/baz.txt"); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 3); + assert_eq!(components[0], "foo"); + assert_eq!(components[1], "bar"); + assert_eq!(components[2], "baz.txt"); + } + + #[test] + fn test_simplified_components_with_parent_dirs() { + let path = Path::new("foo/../bar/baz.txt"); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 2); + assert_eq!(components[0], "bar"); + assert_eq!(components[1], "baz.txt"); + } + + #[test] + fn test_simplified_components_too_many_parent_dirs() { + let path = Path::new("foo/../../bar"); + let result = simplified_components(path); + assert!(result.is_none()); // Should still fail for directory traversal attacks + } + + #[test] + fn test_simplified_components_with_current_dir() { + let path = Path::new("foo/./bar/baz.txt"); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 3); + assert_eq!(components[0], "foo"); + assert_eq!(components[1], "bar"); + assert_eq!(components[2], "baz.txt"); + } + + #[test] + fn test_simplified_components_empty_path() { + let path = Path::new(""); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 0); + } + + #[test] + fn test_simplified_components_root_only() { + let path = Path::new("/"); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 0); + } + + #[cfg(windows)] + #[test] + fn test_simplified_components_windows_absolute_path() { + let path = Path::new(r"C:\foo\bar\baz.txt"); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 3); + assert_eq!(components[0], "foo"); + assert_eq!(components[1], "bar"); + assert_eq!(components[2], "baz.txt"); + } + + #[cfg(windows)] + #[test] + fn test_simplified_components_windows_unc_path() { + let path = Path::new(r"\\server\share\foo\bar.txt"); + let components = simplified_components(path).unwrap(); + assert_eq!(components.len(), 2); + assert_eq!(components[0], "foo"); + assert_eq!(components[1], "bar.txt"); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/read.rs b/v2/vendor/security/zip-7.2.0/src/read.rs new file mode 100644 index 0000000000..be430435c0 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/read.rs @@ -0,0 +1,2650 @@ +//! Types for reading ZIP archives + +/// Upper bound on the uncompressed size of a symlink entry. +/// +/// A symlink target is a filesystem path, so its length is bounded by the +/// platform (`PATH_MAX` is 4096 on Linux) no matter what the archive says. The +/// uncompressed size stored in the central directory is attacker-controlled, +/// so entries above this bound are rejected rather than pre-allocated. +pub(crate) const MAX_SYMLINK_TARGET_LEN: u64 = 4096; + +#[cfg(feature = "aes-crypto")] +use crate::aes::{AesReader, AesReaderValid}; +use crate::cfg_if; +use crate::compression::{CompressionMethod, Decompressor}; +use crate::cp437::FromCp437; +use crate::crc32::Crc32Reader; +use crate::extra_fields::{ExtendedTimestamp, ExtraField, Ntfs}; +use crate::result::{invalid, ZipError, ZipResult}; +use crate::spec::{ + self, CentralDirectoryEndInfo, DataAndPosition, FixedSizeBlock, Pod, ZIP64_BYTES_THR, +}; +use crate::types::{ + AesMode, AesVendorVersion, DateTime, SimpleFileOptions, System, ZipCentralEntryBlock, + ZipFileData, ZipLocalEntryBlock, +}; +use crate::zipcrypto::{ZipCryptoReader, ZipCryptoReaderValid, ZipCryptoValidator}; +use core::mem::{replace, size_of}; +use core::ops::{Deref, Range}; +use indexmap::IndexMap; +use std::borrow::Cow; +use std::ffi::OsStr; +use std::io::{self, copy, sink, Read, Seek, SeekFrom, Write}; +use std::path::{Component, Path, PathBuf}; +use std::sync::{Arc, OnceLock}; + +mod config; + +pub use config::{ArchiveOffset, Config}; + +/// Provides high level API for reading from a stream. +pub(crate) mod stream; + +pub(crate) mod magic_finder; + +/// Immutable metadata about a `ZipArchive`. +#[derive(Debug)] +pub struct ZipArchiveMetadata { + pub(crate) files: IndexMap, ZipFileData>, + pub(crate) offset: u64, + pub(crate) dir_start: u64, + // This isn't yet used anywhere, but it is here for use cases in the future. + #[allow(dead_code)] + pub(crate) config: Config, + pub(crate) comment: Box<[u8]>, + pub(crate) zip64_comment: Option>, +} + +pub(crate) mod zip_archive { + use crate::read::ZipArchiveMetadata; + use indexmap::IndexMap; + use std::sync::Arc; + + #[derive(Debug)] + pub(crate) struct SharedBuilder { + pub(crate) files: Vec, + pub(super) offset: u64, + pub(super) dir_start: u64, + // This isn't yet used anywhere, but it is here for use cases in the future. + #[allow(dead_code)] + pub(super) config: super::Config, + } + + impl SharedBuilder { + pub fn build( + self, + comment: Box<[u8]>, + zip64_comment: Option>, + ) -> ZipArchiveMetadata { + let mut index_map = IndexMap::with_capacity(self.files.len()); + self.files.into_iter().for_each(|file| { + index_map.insert(file.file_name.clone(), file); + }); + ZipArchiveMetadata { + files: index_map, + offset: self.offset, + dir_start: self.dir_start, + config: self.config, + comment, + zip64_comment, + } + } + } + + /// ZIP archive reader + /// + /// At the moment, this type is cheap to clone if this is the case for the + /// reader it uses. However, this is not guaranteed by this crate and it may + /// change in the future. + /// + /// ```no_run + /// use std::io::{Read, Seek}; + /// fn list_zip_contents(reader: impl Read + Seek) -> zip::result::ZipResult<()> { + /// use zip::HasZipMetadata; + /// let mut zip = zip::ZipArchive::new(reader)?; + /// + /// for i in 0..zip.len() { + /// let mut file = zip.by_index(i)?; + /// println!("Filename: {}", file.name()); + /// std::io::copy(&mut file, &mut std::io::stdout())?; + /// } + /// + /// Ok(()) + /// } + /// ``` + #[derive(Clone, Debug)] + pub struct ZipArchive { + pub(super) reader: R, + pub(super) shared: Arc, + } +} + +#[cfg(feature = "aes-crypto")] +use crate::aes::PWD_VERIFY_LENGTH; +use crate::extra_fields::UnicodeExtraField; +use crate::result::ZipError::InvalidPassword; +use crate::spec::is_dir; +use crate::types::ffi::{S_IFLNK, S_IFREG}; +use crate::unstable::{path_to_string, LittleEndianReadExt}; +pub use zip_archive::ZipArchive; + +#[allow(clippy::large_enum_variant)] +pub(crate) enum CryptoReader<'a, R: Read + ?Sized> { + Plaintext(io::Take<&'a mut R>), + ZipCrypto(ZipCryptoReaderValid>), + #[cfg(feature = "aes-crypto")] + Aes { + reader: AesReaderValid>, + vendor_version: AesVendorVersion, + }, +} + +impl Read for CryptoReader<'_, R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + CryptoReader::Plaintext(r) => r.read(buf), + CryptoReader::ZipCrypto(r) => r.read(buf), + #[cfg(feature = "aes-crypto")] + CryptoReader::Aes { reader: r, .. } => r.read(buf), + } + } + + fn read_to_end(&mut self, buf: &mut Vec) -> io::Result { + match self { + CryptoReader::Plaintext(r) => r.read_to_end(buf), + CryptoReader::ZipCrypto(r) => r.read_to_end(buf), + #[cfg(feature = "aes-crypto")] + CryptoReader::Aes { reader: r, .. } => r.read_to_end(buf), + } + } + + fn read_to_string(&mut self, buf: &mut String) -> io::Result { + match self { + CryptoReader::Plaintext(r) => r.read_to_string(buf), + CryptoReader::ZipCrypto(r) => r.read_to_string(buf), + #[cfg(feature = "aes-crypto")] + CryptoReader::Aes { reader: r, .. } => r.read_to_string(buf), + } + } +} + +impl<'a, R: Read + ?Sized> CryptoReader<'a, R> { + /// Consumes this decoder, returning the underlying reader. + pub fn into_inner(self) -> io::Take<&'a mut R> { + match self { + CryptoReader::Plaintext(r) => r, + CryptoReader::ZipCrypto(r) => r.into_inner(), + #[cfg(feature = "aes-crypto")] + CryptoReader::Aes { reader: r, .. } => r.into_inner(), + } + } + + /// Returns `true` if the data is encrypted using AE2. + #[allow(clippy::needless_return)] // can't use cfg_if_expr! because const + pub const fn is_ae2_encrypted(&self) -> bool { + cfg_if! { + if #[cfg(feature = "aes-crypto")] { + return matches!( + self, + CryptoReader::Aes { + vendor_version: AesVendorVersion::Ae2, + .. + } + ); + } else { + return false; + } + } + } +} + +#[cold] +fn invalid_state() -> io::Result { + Err(io::Error::other("ZipFileReader was in an invalid state")) +} + +pub(crate) enum ZipFileReader<'a, R: Read + ?Sized> { + NoReader, + Raw(io::Take<&'a mut R>), + Compressed(Box>>>>), +} + +impl Read for ZipFileReader<'_, R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match self { + ZipFileReader::NoReader => invalid_state(), + ZipFileReader::Raw(r) => r.read(buf), + ZipFileReader::Compressed(r) => r.read(buf), + } + } + + fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> { + match self { + ZipFileReader::NoReader => invalid_state(), + ZipFileReader::Raw(r) => r.read_exact(buf), + ZipFileReader::Compressed(r) => r.read_exact(buf), + } + } + + fn read_to_end(&mut self, buf: &mut Vec) -> io::Result { + match self { + ZipFileReader::NoReader => invalid_state(), + ZipFileReader::Raw(r) => r.read_to_end(buf), + ZipFileReader::Compressed(r) => r.read_to_end(buf), + } + } + + fn read_to_string(&mut self, buf: &mut String) -> io::Result { + match self { + ZipFileReader::NoReader => invalid_state(), + ZipFileReader::Raw(r) => r.read_to_string(buf), + ZipFileReader::Compressed(r) => r.read_to_string(buf), + } + } +} + +impl<'a, R: Read + ?Sized> ZipFileReader<'a, R> { + fn into_inner(self) -> io::Result> { + match self { + ZipFileReader::NoReader => invalid_state(), + ZipFileReader::Raw(r) => Ok(r), + ZipFileReader::Compressed(r) => { + Ok(r.into_inner().into_inner()?.into_inner().into_inner()) + } + } + } +} + +/// A struct for reading a zip file +pub struct ZipFile<'a, R: Read + ?Sized> { + pub(crate) data: Cow<'a, ZipFileData>, + pub(crate) reader: ZipFileReader<'a, R>, +} + +/// A struct for reading and seeking a zip file +pub struct ZipFileSeek<'a, R> { + data: Cow<'a, ZipFileData>, + reader: ZipFileSeekReader<'a, R>, +} + +enum ZipFileSeekReader<'a, R: ?Sized> { + Raw(SeekableTake<'a, R>), +} + +struct SeekableTake<'a, R: ?Sized> { + inner: &'a mut R, + inner_starting_offset: u64, + length: u64, + current_offset: u64, +} + +impl<'a, R: Seek + ?Sized> SeekableTake<'a, R> { + pub fn new(inner: &'a mut R, length: u64) -> io::Result { + let inner_starting_offset = inner.stream_position()?; + Ok(Self { + inner, + inner_starting_offset, + length, + current_offset: 0, + }) + } +} + +impl Seek for SeekableTake<'_, R> { + fn seek(&mut self, pos: SeekFrom) -> io::Result { + let offset = match pos { + SeekFrom::Start(offset) => Some(offset), + SeekFrom::End(offset) => self.length.checked_add_signed(offset), + SeekFrom::Current(offset) => self.current_offset.checked_add_signed(offset), + }; + match offset { + None => Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid seek to a negative or overflowing position", + )), + Some(offset) => { + let clamped_offset = std::cmp::min(self.length, offset); + let new_inner_offset = self + .inner + .seek(SeekFrom::Start(self.inner_starting_offset + clamped_offset))?; + self.current_offset = new_inner_offset - self.inner_starting_offset; + Ok(self.current_offset) + } + } + } +} + +impl Read for SeekableTake<'_, R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + let written = self + .inner + .take(self.length - self.current_offset) + .read(buf)?; + self.current_offset += written as u64; + Ok(written) + } +} + +pub(crate) fn make_writable_dir_all>(outpath: T) -> Result<(), ZipError> { + use std::fs; + fs::create_dir_all(outpath.as_ref())?; + #[cfg(unix)] + { + // Dirs must be writable until all normal files are extracted + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions( + outpath.as_ref(), + std::fs::Permissions::from_mode( + 0o700 | std::fs::metadata(outpath.as_ref())?.permissions().mode(), + ), + )?; + } + Ok(()) +} + +pub(crate) fn find_content<'a, R: Read + Seek + ?Sized>( + data: &ZipFileData, + reader: &'a mut R, +) -> ZipResult> { + // TODO: use .get_or_try_init() once stabilized to provide a closure returning a Result! + let data_start = data.data_start(reader)?; + + reader.seek(SeekFrom::Start(data_start))?; + Ok(reader.take(data.compressed_size)) +} + +fn find_content_seek<'a, R: Read + Seek + ?Sized>( + data: &ZipFileData, + reader: &'a mut R, +) -> ZipResult> { + // Parse local header + let data_start = data.data_start(reader)?; + reader.seek(SeekFrom::Start(data_start))?; + + // Explicit Ok and ? are needed to convert io::Error to ZipError + Ok(SeekableTake::new(reader, data.compressed_size)?) +} + +pub(crate) fn find_data_start( + data: &ZipFileData, + reader: &mut (impl Read + Seek + ?Sized), +) -> Result { + // Go to start of data. + reader.seek(SeekFrom::Start(data.header_start))?; + + // Parse static-sized fields and check the magic value. + let block = ZipLocalEntryBlock::parse(reader)?; + + // Calculate the end of the local header from the fields we just parsed. + let variable_fields_len = + // Each of these fields must be converted to u64 before adding, as the result may + // easily overflow a u16. + block.file_name_length as u64 + block.extra_field_length as u64; + let data_start = + data.header_start + size_of::() as u64 + variable_fields_len; + + // Set the value so we don't have to read it again. + match data.data_start.set(data_start) { + Ok(()) => (), + // If the value was already set in the meantime, ensure it matches (this is probably + // unnecessary). + Err(_) => { + debug_assert_eq!(*data.data_start.get().unwrap(), data_start); + } + } + + Ok(data_start) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn make_crypto_reader<'a, R: Read + ?Sized>( + data: &ZipFileData, + reader: io::Take<&'a mut R>, + password: Option<&[u8]>, + aes_info: Option<(AesMode, AesVendorVersion, CompressionMethod)>, +) -> ZipResult> { + #[allow(deprecated)] + { + if let CompressionMethod::Unsupported(_) = data.compression_method { + return unsupported_zip_error("Compression method not supported"); + } + } + + let reader = match (password, aes_info) { + #[cfg(not(feature = "aes-crypto"))] + (Some(_), Some(_)) => { + return Err(ZipError::UnsupportedArchive( + "AES encrypted files cannot be decrypted without the aes-crypto feature.", + )) + } + #[cfg(feature = "aes-crypto")] + (Some(password), Some((aes_mode, vendor_version, _))) => CryptoReader::Aes { + reader: AesReader::new(reader, aes_mode, data.compressed_size).validate(password)?, + vendor_version, + }, + (Some(password), None) => { + let validator = if data.using_data_descriptor { + ZipCryptoValidator::InfoZipMsdosTime( + data.last_modified_time.map_or(0, |x| x.timepart()), + ) + } else { + ZipCryptoValidator::PkzipCrc32(data.crc32) + }; + CryptoReader::ZipCrypto(ZipCryptoReader::new(reader, password).validate(validator)?) + } + (None, Some(_)) => return Err(InvalidPassword), + (None, None) => CryptoReader::Plaintext(reader), + }; + Ok(reader) +} + +pub(crate) fn make_reader<'a, R: Read + ?Sized>( + compression_method: CompressionMethod, + uncompressed_size: u64, + crc32: u32, + reader: CryptoReader<'a, R>, + #[cfg(feature = "legacy-zip")] flags: u16, +) -> ZipResult> { + let ae2_encrypted = reader.is_ae2_encrypted(); + #[cfg(not(feature = "legacy-zip"))] + let flags = 0; + Ok(ZipFileReader::Compressed(Box::new(Crc32Reader::new( + Decompressor::new( + io::BufReader::new(reader), + compression_method, + uncompressed_size, + flags, + )?, + crc32, + ae2_encrypted, + )))) +} + +pub(crate) fn make_symlink( + outpath: &Path, + target: &[u8], + #[cfg_attr(not(windows), allow(unused))] existing_files: &IndexMap, T>, +) -> ZipResult<()> { + #[cfg_attr(not(any(unix, windows)), allow(unused))] + let Ok(target_str) = std::str::from_utf8(target) else { + return Err(invalid!("Invalid UTF-8 as symlink target")); + }; + + cfg_if! { + if #[cfg(unix)] { + std::os::unix::fs::symlink(Path::new(&target_str), outpath)?; + } else if #[cfg(windows)] { + let target = Path::new(OsStr::new(&target_str)); + let target_is_dir_from_archive = + existing_files.contains_key(target_str) && is_dir(target_str); + let target_is_dir = if target_is_dir_from_archive { + true + } else if let Ok(meta) = std::fs::metadata(target) { + meta.is_dir() + } else { + false + }; + if target_is_dir { + std::os::windows::fs::symlink_dir(target, outpath)?; + } else { + std::os::windows::fs::symlink_file(target, outpath)?; + } + } else { + use std::fs::File; + let output = File::create(outpath); + output?.write_all(target)?; + } + } + + Ok(()) +} + +#[derive(Debug)] +pub(crate) struct CentralDirectoryInfo { + pub(crate) archive_offset: u64, + pub(crate) directory_start: u64, + pub(crate) number_of_files: usize, + pub(crate) disk_number: u32, + pub(crate) disk_with_central_directory: u32, +} + +impl<'a> TryFrom<&'a CentralDirectoryEndInfo> for CentralDirectoryInfo { + type Error = ZipError; + + fn try_from(value: &'a CentralDirectoryEndInfo) -> Result { + let (relative_cd_offset, number_of_files, disk_number, disk_with_central_directory) = + match &value.eocd64 { + Some(DataAndPosition { data: eocd64, .. }) => { + if eocd64.number_of_files_on_this_disk > eocd64.number_of_files { + return Err(invalid!("ZIP64 footer indicates more files on this disk than in the whole archive")); + } + ( + eocd64.central_directory_offset, + eocd64.number_of_files as usize, + eocd64.disk_number, + eocd64.disk_with_central_directory, + ) + } + _ => ( + value.eocd.data.central_directory_offset as u64, + value.eocd.data.number_of_files_on_this_disk as usize, + value.eocd.data.disk_number as u32, + value.eocd.data.disk_with_central_directory as u32, + ), + }; + + let directory_start = relative_cd_offset + .checked_add(value.archive_offset) + .ok_or(invalid!("Invalid central directory size or offset"))?; + + Ok(Self { + archive_offset: value.archive_offset, + directory_start, + number_of_files, + disk_number, + disk_with_central_directory, + }) + } +} + +/// Store all entries which specify a numeric "mode" which is familiar to POSIX operating systems. +#[cfg(unix)] +#[derive(Default, Debug)] +struct UnixFileModes { + map: std::collections::BTreeMap, +} + +#[cfg(unix)] +impl UnixFileModes { + #[cfg_attr(not(debug_assertions), allow(unused))] + pub fn add_mode(&mut self, path: PathBuf, mode: u32) { + // We don't print a warning or consider it remotely out of the ordinary to receive two + // separate modes for the same path: just take the later one. + let old_entry = self.map.insert(path, mode); + debug_assert_eq!(old_entry, None); + } + + // Child nodes will be sorted later lexicographically, so reversing the order puts them first. + pub fn all_perms_with_children_first( + self, + ) -> impl IntoIterator { + use std::os::unix::fs::PermissionsExt; + self.map + .into_iter() + .rev() + .map(|(p, m)| (p, std::fs::Permissions::from_mode(m))) + } +} + +impl ZipArchive { + pub(crate) fn from_finalized_writer( + files: IndexMap, ZipFileData>, + comment: Box<[u8]>, + zip64_comment: Option>, + reader: R, + central_start: u64, + ) -> ZipResult { + let initial_offset = match files.first() { + Some((_, file)) => file.header_start, + None => central_start, + }; + let shared = Arc::new(ZipArchiveMetadata { + files, + offset: initial_offset, + dir_start: central_start, + config: Config { + archive_offset: ArchiveOffset::Known(initial_offset), + }, + comment, + zip64_comment, + }); + Ok(Self { reader, shared }) + } + + /// Total size of the files in the archive, if it can be known. Doesn't include directories or + /// metadata. + pub fn decompressed_size(&self) -> Option { + let mut total = 0u128; + for file in self.shared.files.values() { + if file.using_data_descriptor { + return None; + } + total = total.checked_add(file.uncompressed_size as u128)?; + } + Some(total) + } +} + +impl ZipArchive { + pub(crate) fn merge_contents( + &mut self, + mut w: W, + ) -> ZipResult, ZipFileData>> { + if self.shared.files.is_empty() { + return Ok(IndexMap::new()); + } + let mut new_files = self.shared.files.clone(); + /* The first file header will probably start at the beginning of the file, but zip doesn't + * enforce that, and executable zips like PEX files will have a shebang line so will + * definitely be greater than 0. + * + * assert_eq!(0, new_files[0].header_start); // Avoid this. + */ + + let first_new_file_header_start = w.stream_position()?; + + /* Push back file header starts for all entries in the covered files. */ + new_files.values_mut().try_for_each(|f| { + /* This is probably the only really important thing to change. */ + f.header_start = f + .header_start + .checked_add(first_new_file_header_start) + .ok_or(invalid!( + "new header start from merge would have been too large" + ))?; + /* This is only ever used internally to cache metadata lookups (it's not part of the + * zip spec), and 0 is the sentinel value. */ + f.central_header_start = 0; + /* This is an atomic variable so it can be updated from another thread in the + * implementation (which is good!). */ + if let Some(old_data_start) = f.data_start.take() { + let new_data_start = old_data_start + .checked_add(first_new_file_header_start) + .ok_or(invalid!( + "new data start from merge would have been too large" + ))?; + f.data_start.get_or_init(|| new_data_start); + } + Ok::<_, ZipError>(()) + })?; + + /* Rewind to the beginning of the file. + * + * NB: we *could* decide to start copying from new_files[0].header_start instead, which + * would avoid copying over e.g. any pex shebangs or other file contents that start before + * the first zip file entry. However, zip files actually shouldn't care about garbage data + * in *between* real entries, since the central directory header records the correct start + * location of each, and keeping track of that math is more complicated logic that will only + * rarely be used, since most zips that get merged together are likely to be produced + * specifically for that purpose (and therefore are unlikely to have a shebang or other + * preface). Finally, this preserves any data that might actually be useful. + */ + self.reader.rewind()?; + /* Find the end of the file data. */ + let length_to_read = self.shared.dir_start; + /* Produce a Read that reads bytes up until the start of the central directory header. + * This "as &mut dyn Read" trick is used elsewhere to avoid having to clone the underlying + * handle, which it really shouldn't need to anyway. */ + let mut limited_raw = (&mut self.reader as &mut dyn Read).take(length_to_read); + /* Copy over file data from source archive directly. */ + io::copy(&mut limited_raw, &mut w)?; + + /* Return the files we've just written to the data stream. */ + Ok(new_files) + } + + /// Get the directory start offset and number of files. This is done in a + /// separate function to ease the control flow design. + pub(crate) fn get_metadata(config: Config, reader: &mut R) -> ZipResult { + // End of the probed region, initially set to the end of the file + let file_len = reader.seek(io::SeekFrom::End(0))?; + let mut end_exclusive = file_len; + let mut last_err = None; + + loop { + // Find the EOCD and possibly EOCD64 entries and determine the archive offset. + let cde = match spec::find_central_directory( + reader, + config.archive_offset, + end_exclusive, + file_len, + ) { + Ok(cde) => cde, + Err(e) => { + // return the previous error first (if there is) + return Err(last_err.unwrap_or(e)); + } + }; + + // Turn EOCD into internal representation. + match CentralDirectoryInfo::try_from(&cde) + .and_then(|info| Self::read_central_header(info, config, reader)) + { + Ok(shared) => { + return Ok(shared.build( + cde.eocd.data.zip_file_comment, + cde.eocd64.map(|v| v.data.extensible_data_sector), + )); + } + Err(e) => { + last_err = Some(e); + } + }; + // Something went wrong while decoding the cde, try to find a new one + end_exclusive = cde.eocd.position; + continue; + } + } + + fn read_central_header( + dir_info: CentralDirectoryInfo, + config: Config, + reader: &mut R, + ) -> Result { + // If the parsed number of files is greater than the offset then + // something fishy is going on and we shouldn't trust number_of_files. + let file_capacity = if dir_info.number_of_files > dir_info.directory_start as usize { + 0 + } else { + dir_info.number_of_files + }; + + if dir_info.disk_number != dir_info.disk_with_central_directory { + return unsupported_zip_error("Support for multi-disk files is not implemented"); + } + + if file_capacity.saturating_mul(size_of::()) > isize::MAX as usize { + return unsupported_zip_error("Oversized central directory"); + } + + let mut files = Vec::with_capacity(file_capacity); + reader.seek(SeekFrom::Start(dir_info.directory_start))?; + for _ in 0..dir_info.number_of_files { + let file = central_header_to_zip_file(reader, &dir_info)?; + files.push(file); + } + + Ok(zip_archive::SharedBuilder { + files, + offset: dir_info.archive_offset, + dir_start: dir_info.directory_start, + config, + }) + } + + /// Returns the verification value and salt for the AES encryption of the file + /// + /// It fails if the file number is invalid. + /// + /// # Returns + /// + /// - None if the file is not encrypted with AES + #[cfg(feature = "aes-crypto")] + pub fn get_aes_verification_key_and_salt( + &mut self, + file_number: usize, + ) -> ZipResult> { + let (_, data) = self + .shared + .files + .get_index(file_number) + .ok_or(ZipError::FileNotFound)?; + + let limit_reader = find_content(data, &mut self.reader)?; + match data.aes_mode { + None => Ok(None), + Some((aes_mode, _, _)) => { + let (verification_value, salt) = + AesReader::new(limit_reader, aes_mode, data.compressed_size) + .get_verification_value_and_salt()?; + let aes_info = AesInfo { + aes_mode, + verification_value, + salt, + }; + Ok(Some(aes_info)) + } + } + } + + /// Read a ZIP archive, collecting the files it contains. + /// + /// This uses the central directory record of the ZIP file, and ignores local file headers. + /// + /// A default [`Config`] is used. + pub fn new(reader: R) -> ZipResult> { + Self::with_config(Default::default(), reader) + } + + /// Get the metadata associated with the ZIP archive. + /// + /// This can be used with [`Self::unsafe_new_with_metadata`] to create a new reader over the + /// same file without needing to reparse the metadata. + pub fn metadata(&self) -> Arc { + self.shared.clone() + } + + /// Read a ZIP archive using the given `metadata`. + /// + /// This is useful for creating multiple readers over the same file without + /// needing to reparse the metadata. + /// + /// # Safety + /// `unsafe` is used here to indicate that `reader` and `metadata` could + /// potentially be incompatible, and it is left to the user to ensure they are. + /// + /// # Example + /// + /// ```no_run + /// # use std::fs; + /// use rayon::prelude::*; + /// + /// const FILE_NAME: &str = "my_data.zip"; + /// + /// let file = fs::File::open(FILE_NAME).unwrap(); + /// let mut archive = zip::ZipArchive::new(file).unwrap(); + /// + /// let file_names = (0..archive.len()) + /// .into_par_iter() + /// .map_init({ + /// let metadata = archive.metadata().clone(); + /// move || { + /// let file = fs::File::open(FILE_NAME).unwrap(); + /// unsafe { zip::ZipArchive::unsafe_new_with_metadata(file, metadata.clone()) } + /// }}, + /// |archive, i| { + /// let mut file = archive.by_index(i).unwrap(); + /// file.enclosed_name() + /// } + /// ) + /// .filter_map(|name| name) + /// .collect::>(); + /// ``` + pub unsafe fn unsafe_new_with_metadata(reader: R, metadata: Arc) -> Self { + Self { + reader, + shared: metadata, + } + } + + /// Read a ZIP archive providing a read configuration, collecting the files it contains. + /// + /// This uses the central directory record of the ZIP file, and ignores local file headers. + pub fn with_config(config: Config, mut reader: R) -> ZipResult> { + let shared = Self::get_metadata(config, &mut reader)?; + + Ok(ZipArchive { + reader, + shared: shared.into(), + }) + } + + /// Extract a Zip archive into a directory, overwriting files if they + /// already exist. Paths are sanitized with [`ZipFile::enclosed_name`]. Symbolic links are only + /// created and followed if the target is within the destination directory (this is checked + /// conservatively using [`std::fs::canonicalize`]). + /// + /// Extraction is not atomic. If an error is encountered, some of the files + /// may be left on disk. However, on Unix targets, no newly-created directories with part but + /// not all of their contents extracted will be readable, writable or usable as process working + /// directories by any non-root user except you. + /// + /// On Unix and Windows, symbolic links are extracted correctly. On other platforms such as + /// WebAssembly, symbolic links aren't supported, so they're extracted as normal files + /// containing the target path in UTF-8. + pub fn extract>(&mut self, directory: P) -> ZipResult<()> { + self.extract_internal(directory, None:: bool>) + } + + /// Extracts a Zip archive into a directory in the same fashion as + /// [`ZipArchive::extract`], but detects a "root" directory in the archive + /// (a single top-level directory that contains the rest of the archive's + /// entries) and extracts its contents directly. + /// + /// For a sensible default `filter`, you can use [`root_dir_common_filter`]. + /// For a custom `filter`, see [`RootDirFilter`]. + /// + /// See [`ZipArchive::root_dir`] for more information on how the root + /// directory is detected and the meaning of the `filter` parameter. + /// + /// ## Example + /// + /// Imagine a Zip archive with the following structure: + /// + /// ```text + /// root/file1.txt + /// root/file2.txt + /// root/sub/file3.txt + /// root/sub/subsub/file4.txt + /// ``` + /// + /// If the archive is extracted to `foo` using [`ZipArchive::extract`], + /// the resulting directory structure will be: + /// + /// ```text + /// foo/root/file1.txt + /// foo/root/file2.txt + /// foo/root/sub/file3.txt + /// foo/root/sub/subsub/file4.txt + /// ``` + /// + /// If the archive is extracted to `foo` using + /// [`ZipArchive::extract_unwrapped_root_dir`], the resulting directory + /// structure will be: + /// + /// ```text + /// foo/file1.txt + /// foo/file2.txt + /// foo/sub/file3.txt + /// foo/sub/subsub/file4.txt + /// ``` + /// + /// ## Example - No Root Directory + /// + /// Imagine a Zip archive with the following structure: + /// + /// ```text + /// root/file1.txt + /// root/file2.txt + /// root/sub/file3.txt + /// root/sub/subsub/file4.txt + /// other/file5.txt + /// ``` + /// + /// Due to the presence of the `other` directory, + /// [`ZipArchive::extract_unwrapped_root_dir`] will extract this in the same + /// fashion as [`ZipArchive::extract`] as there is now no "root directory." + pub fn extract_unwrapped_root_dir>( + &mut self, + directory: P, + root_dir_filter: impl RootDirFilter, + ) -> ZipResult<()> { + self.extract_internal(directory, Some(root_dir_filter)) + } + + fn extract_internal>( + &mut self, + directory: P, + root_dir_filter: Option, + ) -> ZipResult<()> { + use std::fs; + + fs::create_dir_all(&directory)?; + let directory = directory.as_ref().canonicalize()?; + + let root_dir = root_dir_filter + .and_then(|filter| { + self.root_dir(&filter) + .transpose() + .map(|root_dir| root_dir.map(|root_dir| (root_dir, filter))) + }) + .transpose()?; + + // If we have a root dir, simplify the path components to be more + // appropriate for passing to `safe_prepare_path` + let root_dir = root_dir + .as_ref() + .map(|(root_dir, filter)| { + crate::path::simplified_components(root_dir) + .ok_or_else(|| { + // Should be unreachable + debug_assert!(false, "Invalid root dir path"); + + invalid!("Invalid root dir path") + }) + .map(|root_dir| (root_dir, filter)) + }) + .transpose()?; + + #[cfg(unix)] + let mut files_by_unix_mode = UnixFileModes::default(); + + for i in 0..self.len() { + let mut file = self.by_index(i)?; + + let mut outpath = directory.clone(); + /* TODO: the control flow of this method call and subsequent expectations about the + * values in this loop is extremely difficult to follow. It also appears to + * perform a nested loop upon extracting every single file entry? Why does it + * accept two arguments that point to the same directory path, one mutable? */ + file.safe_prepare_path(directory.as_ref(), &mut outpath, root_dir.as_ref())?; + + #[cfg(any(unix, windows))] + if file.is_symlink() { + let declared_len = file.size(); + if declared_len > MAX_SYMLINK_TARGET_LEN { + return Err(invalid!( + "symlink target declares {} bytes, more than the {} byte limit", + declared_len, + MAX_SYMLINK_TARGET_LEN + )); + } + let mut target = Vec::with_capacity(declared_len as usize); + file.read_to_end(&mut target)?; + drop(file); + make_symlink(&outpath, &target, &self.shared.files)?; + continue; + } else if file.is_dir() { + crate::read::make_writable_dir_all(&outpath)?; + continue; + } + let mut outfile = fs::File::create(&outpath)?; + io::copy(&mut file, &mut outfile)?; + + // Check for real permissions, which we'll set in a second pass. + #[cfg(unix)] + if let Some(mode) = file.unix_mode() { + files_by_unix_mode.add_mode(outpath, mode); + } + + // Set original timestamp. + #[cfg(feature = "chrono")] + if let Some(last_modified) = file.last_modified() { + if let Some(t) = datetime_to_systemtime(&last_modified) { + outfile.set_modified(t)?; + } + } + } + + // Ensure we update children's permissions before making a parent unwritable. + #[cfg(unix)] + for (path, perms) in files_by_unix_mode.all_perms_with_children_first() { + std::fs::set_permissions(path, perms)?; + } + + Ok(()) + } + + /// Number of files contained in this zip. + pub fn len(&self) -> usize { + self.shared.files.len() + } + + /// Get the starting offset of the zip central directory. + pub fn central_directory_start(&self) -> u64 { + self.shared.dir_start + } + + /// Whether this zip archive contains no files + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Get the offset from the beginning of the underlying reader that this zip begins at, in bytes. + /// + /// Normally this value is zero, but if the zip has arbitrary data prepended to it, then this value will be the size + /// of that prepended data. + pub fn offset(&self) -> u64 { + self.shared.offset + } + + /// Get the comment of the zip archive. + pub fn comment(&self) -> &[u8] { + &self.shared.comment + } + + /// Get the ZIP64 comment of the zip archive, if it is ZIP64. + pub fn zip64_comment(&self) -> Option<&[u8]> { + self.shared.zip64_comment.as_deref() + } + + /// Returns an iterator over all the file and directory names in this archive. + pub fn file_names(&self) -> impl Iterator { + self.shared.files.keys().map(|s| s.as_ref()) + } + + /// Returns Ok(true) if any compressed data in this archive belongs to more than one file. This + /// doesn't make the archive invalid, but some programs will refuse to decompress it because the + /// copies would take up space independently in the destination. + pub fn has_overlapping_files(&mut self) -> ZipResult { + let mut ranges = Vec::>::with_capacity(self.shared.files.len()); + for file in self.shared.files.values() { + if file.compressed_size == 0 { + continue; + } + let start = file.data_start(&mut self.reader)?; + let end = start + file.compressed_size; + if ranges + .iter() + .any(|range| range.start <= end && start <= range.end) + { + return Ok(true); + } + ranges.push(start..end); + } + Ok(false) + } + + /// Search for a file entry by name, decrypt with given password + /// + /// # Warning + /// + /// The implementation of the cryptographic algorithms has not + /// gone through a correctness review, and you should assume it is insecure: + /// passwords used with this API may be compromised. + /// + /// This function sometimes accepts wrong password. This is because the ZIP spec only allows us + /// to check for a 1/256 chance that the password is correct. + /// There are many passwords out there that will also pass the validity checks + /// we are able to perform. This is a weakness of the ZipCrypto algorithm, + /// due to its fairly primitive approach to cryptography. + pub fn by_name_decrypt(&mut self, name: &str, password: &[u8]) -> ZipResult> { + self.by_name_with_optional_password(name, Some(password)) + } + + /// Search for a file entry by name + pub fn by_name(&mut self, name: &str) -> ZipResult> { + self.by_name_with_optional_password(name, None) + } + + /// Get the index of a file entry by name, if it's present. + #[inline(always)] + pub fn index_for_name(&self, name: &str) -> Option { + self.shared.files.get_index_of(name) + } + + /// Search for a file entry by path, decrypt with given password + /// + /// # Warning + /// + /// The implementation of the cryptographic algorithms has not + /// gone through a correctness review, and you should assume it is insecure: + /// passwords used with this API may be compromised. + /// + /// This function sometimes accepts wrong password. This is because the ZIP spec only allows us + /// to check for a 1/256 chance that the password is correct. + /// There are many passwords out there that will also pass the validity checks + /// we are able to perform. This is a weakness of the ZipCrypto algorithm, + /// due to its fairly primitive approach to cryptography. + pub fn by_path_decrypt>( + &mut self, + path: T, + password: &[u8], + ) -> ZipResult> { + self.index_for_path(path) + .ok_or(ZipError::FileNotFound) + .and_then(|index| { + self.by_index_with_options(index, ZipReadOptions::new().password(Some(password))) + }) + } + + /// Search for a file entry by path + pub fn by_path>(&mut self, path: T) -> ZipResult> { + self.index_for_path(path) + .ok_or(ZipError::FileNotFound) + .and_then(|index| self.by_index_with_options(index, ZipReadOptions::new())) + } + + /// Get the index of a file entry by path, if it's present. + #[inline(always)] + pub fn index_for_path>(&self, path: T) -> Option { + self.index_for_name(&path_to_string(path)) + } + + /// Get the name of a file entry, if it's present. + #[inline(always)] + pub fn name_for_index(&self, index: usize) -> Option<&str> { + self.shared + .files + .get_index(index) + .map(|(name, _)| name.as_ref()) + } + + /// Search for a file entry by name and return a seekable object. + pub fn by_name_seek(&mut self, name: &str) -> ZipResult> { + self.by_index_seek(self.index_for_name(name).ok_or(ZipError::FileNotFound)?) + } + + /// Search for a file entry by index and return a seekable object. + pub fn by_index_seek(&mut self, index: usize) -> ZipResult> { + let reader = &mut self.reader; + self.shared + .files + .get_index(index) + .ok_or(ZipError::FileNotFound) + .and_then(move |(_, data)| { + let seek_reader = match data.compression_method { + CompressionMethod::Stored => { + ZipFileSeekReader::Raw(find_content_seek(data, reader)?) + } + _ => { + return Err(ZipError::UnsupportedArchive( + "Seekable compressed files are not yet supported", + )) + } + }; + Ok(ZipFileSeek { + reader: seek_reader, + data: Cow::Borrowed(data), + }) + }) + } + + fn by_name_with_optional_password<'a>( + &'a mut self, + name: &str, + password: Option<&[u8]>, + ) -> ZipResult> { + let Some(index) = self.shared.files.get_index_of(name) else { + return Err(ZipError::FileNotFound); + }; + self.by_index_with_options(index, ZipReadOptions::new().password(password)) + } + + /// Get a contained file by index, decrypt with given password + /// + /// # Warning + /// + /// The implementation of the cryptographic algorithms has not + /// gone through a correctness review, and you should assume it is insecure: + /// passwords used with this API may be compromised. + /// + /// This function sometimes accepts wrong password. This is because the ZIP spec only allows us + /// to check for a 1/256 chance that the password is correct. + /// There are many passwords out there that will also pass the validity checks + /// we are able to perform. This is a weakness of the ZipCrypto algorithm, + /// due to its fairly primitive approach to cryptography. + pub fn by_index_decrypt( + &mut self, + file_number: usize, + password: &[u8], + ) -> ZipResult> { + self.by_index_with_options(file_number, ZipReadOptions::new().password(Some(password))) + } + + /// Get a contained file by index + pub fn by_index(&mut self, file_number: usize) -> ZipResult> { + self.by_index_with_options(file_number, ZipReadOptions::new()) + } + + /// Get a contained file by index without decompressing it + pub fn by_index_raw(&mut self, file_number: usize) -> ZipResult> { + let reader = &mut self.reader; + let (_, data) = self + .shared + .files + .get_index(file_number) + .ok_or(ZipError::FileNotFound)?; + Ok(ZipFile { + reader: ZipFileReader::Raw(find_content(data, reader)?), + data: Cow::Borrowed(data), + }) + } + + /// Get a contained file by index with options. + pub fn by_index_with_options( + &mut self, + file_number: usize, + mut options: ZipReadOptions<'_>, + ) -> ZipResult> { + let (_, data) = self + .shared + .files + .get_index(file_number) + .ok_or(ZipError::FileNotFound)?; + + if options.ignore_encryption_flag { + // Always use no password when we're ignoring the encryption flag. + options.password = None; + } else { + // Require and use the password only if the file is encrypted. + match (options.password, data.encrypted) { + (None, true) => { + return Err(ZipError::UnsupportedArchive(ZipError::PASSWORD_REQUIRED)) + } + // Password supplied, but none needed! Discard. + (Some(_), false) => options.password = None, + _ => {} + } + } + let limit_reader = find_content(data, &mut self.reader)?; + + let crypto_reader = + make_crypto_reader(data, limit_reader, options.password, data.aes_mode)?; + + Ok(ZipFile { + data: Cow::Borrowed(data), + reader: make_reader( + data.compression_method, + data.uncompressed_size, + data.crc32, + crypto_reader, + #[cfg(feature = "legacy-zip")] + data.flags, + )?, + }) + } + + /// Find the "root directory" of an archive if it exists, filtering out + /// irrelevant entries when searching. + /// + /// Our definition of a "root directory" is a single top-level directory + /// that contains the rest of the archive's entries. This is useful for + /// extracting archives that contain a single top-level directory that + /// you want to "unwrap" and extract directly. + /// + /// For a sensible default filter, you can use [`root_dir_common_filter`]. + /// For a custom filter, see [`RootDirFilter`]. + pub fn root_dir(&self, filter: impl RootDirFilter) -> ZipResult> { + let mut root_dir: Option = None; + + for i in 0..self.len() { + let (_, file) = self + .shared + .files + .get_index(i) + .ok_or(ZipError::FileNotFound)?; + + let path = match file.enclosed_name() { + Some(path) => path, + None => return Ok(None), + }; + + if !filter(&path) { + continue; + } + + macro_rules! replace_root_dir { + ($path:ident) => { + match &mut root_dir { + Some(root_dir) => { + if *root_dir != $path { + // We've found multiple root directories, + // abort. + return Ok(None); + } else { + continue; + } + } + + None => { + root_dir = Some($path.into()); + continue; + } + } + }; + } + + // If this entry is located at the root of the archive... + if path.components().count() == 1 { + if file.is_dir() { + // If it's a directory, it could be the root directory. + replace_root_dir!(path); + } else { + // If it's anything else, this archive does not have a + // root directory. + return Ok(None); + } + } + + // Find the root directory for this entry. + let mut path = path.as_path(); + while let Some(parent) = path.parent().filter(|path| *path != Path::new("")) { + path = parent; + } + + replace_root_dir!(path); + } + + Ok(root_dir) + } + + /// Unwrap and return the inner reader object + /// + /// The position of the reader is undefined. + pub fn into_inner(self) -> R { + self.reader + } +} + +/// Holds the AES information of a file in the zip archive +#[derive(Debug)] +#[cfg(feature = "aes-crypto")] +pub struct AesInfo { + /// The AES encryption mode + pub aes_mode: AesMode, + /// The verification key + pub verification_value: [u8; PWD_VERIFY_LENGTH], + /// The salt + pub salt: Vec, +} + +const fn unsupported_zip_error(detail: &'static str) -> ZipResult { + Err(ZipError::UnsupportedArchive(detail)) +} + +/// Parse a central directory entry to collect the information for the file. +pub(crate) fn central_header_to_zip_file( + reader: &mut R, + central_directory: &CentralDirectoryInfo, +) -> ZipResult { + let central_header_start = reader.stream_position()?; + + // Parse central header + let block = ZipCentralEntryBlock::parse(reader)?; + + let file = central_header_to_zip_file_inner( + reader, + central_directory.archive_offset, + central_header_start, + block, + )?; + + let central_header_end = reader.stream_position()?; + + reader.seek(SeekFrom::Start(central_header_end))?; + Ok(file) +} + +#[inline] +fn read_variable_length_byte_field(reader: &mut R, len: usize) -> ZipResult> { + let mut data = vec![0; len].into_boxed_slice(); + if let Err(e) = reader.read_exact(&mut data) { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!( + "Variable-length field extends beyond file boundary" + )); + } + return Err(e.into()); + } + Ok(data) +} + +/// Parse a central directory entry to collect the information for the file. +fn central_header_to_zip_file_inner( + reader: &mut R, + archive_offset: u64, + central_header_start: u64, + block: ZipCentralEntryBlock, +) -> ZipResult { + let ZipCentralEntryBlock { + // magic, + version_made_by, + // version_to_extract, + flags, + compression_method, + last_mod_time, + last_mod_date, + crc32, + compressed_size, + uncompressed_size, + file_name_length, + extra_field_length, + file_comment_length, + // disk_number, + // internal_file_attributes, + external_file_attributes, + offset, + .. + } = block; + + let encrypted = flags & 1 == 1; + let is_utf8 = flags & (1 << 11) != 0; + let using_data_descriptor = flags & (1 << 3) != 0; + + let file_name_raw = read_variable_length_byte_field(reader, file_name_length as usize)?; + let extra_field = read_variable_length_byte_field(reader, extra_field_length as usize)?; + let file_comment_raw = read_variable_length_byte_field(reader, file_comment_length as usize)?; + let file_name: Box = match is_utf8 { + true => String::from_utf8_lossy(&file_name_raw).into(), + false => file_name_raw.clone().from_cp437(), + }; + let file_comment: Box = match is_utf8 { + true => String::from_utf8_lossy(&file_comment_raw).into(), + false => file_comment_raw.from_cp437(), + }; + + // Construct the result + let mut result = ZipFileData { + system: System::from((version_made_by >> 8) as u8), + /* NB: this strips the top 8 bits! */ + version_made_by: version_made_by as u8, + encrypted, + using_data_descriptor, + is_utf8, + compression_method: CompressionMethod::parse_from_u16(compression_method), + compression_level: None, + last_modified_time: DateTime::try_from_msdos(last_mod_date, last_mod_time).ok(), + crc32, + compressed_size: compressed_size.into(), + uncompressed_size: uncompressed_size.into(), + flags, + file_name, + file_name_raw, + extra_field: Some(Arc::new(extra_field.to_vec())), + central_extra_field: None, + file_comment, + header_start: offset.into(), + extra_data_start: None, + central_header_start, + data_start: OnceLock::new(), + external_attributes: external_file_attributes, + large_file: false, + aes_mode: None, + aes_extra_data_start: 0, + extra_fields: Vec::new(), + }; + parse_extra_field(&mut result)?; + + let aes_enabled = result.compression_method == CompressionMethod::AES; + if aes_enabled && result.aes_mode.is_none() { + return Err(invalid!("AES encryption without AES extra data field")); + } + + // Account for shifted zip offsets. + result.header_start = result + .header_start + .checked_add(archive_offset) + .ok_or(invalid!("Archive header is too large"))?; + + Ok(result) +} + +pub(crate) fn parse_extra_field(file: &mut ZipFileData) -> ZipResult<()> { + let mut extra_field = file.extra_field.clone(); + let mut central_extra_field = file.central_extra_field.clone(); + for field_group in [&mut extra_field, &mut central_extra_field] { + let Some(extra_field) = field_group else { + continue; + }; + let mut modified = false; + let mut processed_extra_field = vec![]; + let len = extra_field.len(); + let mut reader = io::Cursor::new(&**extra_field); + + let mut position = reader.position(); + while position < len as u64 { + let old_position = position; + let remove = parse_single_extra_field(file, &mut reader, position, false)?; + position = reader.position(); + if remove { + modified = true; + } else { + let field_len = (position - old_position) as usize; + let write_start = processed_extra_field.len(); + reader.seek(SeekFrom::Start(old_position))?; + processed_extra_field.extend_from_slice(&vec![0u8; field_len]); + if let Err(e) = reader + .read_exact(&mut processed_extra_field[write_start..(write_start + field_len)]) + { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!("Extra field content exceeds declared length")); + } + return Err(e.into()); + } + } + } + if modified { + *field_group = Some(Arc::new(processed_extra_field)); + } + } + file.extra_field = extra_field; + file.central_extra_field = central_extra_field; + Ok(()) +} + +pub(crate) fn parse_single_extra_field( + file: &mut ZipFileData, + reader: &mut R, + bytes_already_read: u64, + disallow_zip64: bool, +) -> ZipResult { + let kind = match reader.read_u16_le() { + Ok(kind) => kind, + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => return Ok(false), + Err(e) => return Err(e.into()), + }; + let len = match reader.read_u16_le() { + Ok(len) => len, + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => { + return Err(invalid!("Extra field header truncated")) + } + Err(e) => return Err(e.into()), + }; + match kind { + // Zip64 extended information extra field + 0x0001 => { + if disallow_zip64 { + return Err(invalid!("Can't write a custom field using the ZIP64 ID")); + } + file.large_file = true; + let mut consumed_len = 0; + if len >= 24 || file.uncompressed_size == spec::ZIP64_BYTES_THR { + file.uncompressed_size = match reader.read_u64_le() { + Ok(v) => v, + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => { + return Err(invalid!("ZIP64 extra field truncated")) + } + Err(e) => return Err(e.into()), + }; + consumed_len += size_of::(); + } + if len >= 24 || file.compressed_size == spec::ZIP64_BYTES_THR { + file.compressed_size = match reader.read_u64_le() { + Ok(v) => v, + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => { + return Err(invalid!("ZIP64 extra field truncated")) + } + Err(e) => return Err(e.into()), + }; + consumed_len += size_of::(); + } + if len >= 24 || file.header_start == spec::ZIP64_BYTES_THR { + file.header_start = match reader.read_u64_le() { + Ok(v) => v, + Err(e) if e.kind() == io::ErrorKind::UnexpectedEof => { + return Err(invalid!("ZIP64 extra field truncated")) + } + Err(e) => return Err(e.into()), + }; + consumed_len += size_of::(); + } + let Some(leftover_len) = (len as usize).checked_sub(consumed_len) else { + return Err(invalid!("ZIP64 extra-data field is the wrong length")); + }; + if let Err(e) = reader.read_exact(&mut vec![0u8; leftover_len]) { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!("ZIP64 extra field truncated")); + } + return Err(e.into()); + } + return Ok(true); + } + 0x000a => { + // NTFS extra field + file.extra_fields + .push(ExtraField::Ntfs(Ntfs::try_from_reader(reader, len)?)); + } + 0x9901 => { + // AES + if len != 7 { + return Err(ZipError::UnsupportedArchive( + "AES extra data field has an unsupported length", + )); + } + let vendor_version = reader.read_u16_le()?; + let vendor_id = reader.read_u16_le()?; + let mut out = [0u8]; + if let Err(e) = reader.read_exact(&mut out) { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!("AES extra field truncated")); + } + return Err(e.into()); + } + let aes_mode = out[0]; + let compression_method = CompressionMethod::parse_from_u16(reader.read_u16_le()?); + + if vendor_id != 0x4541 { + return Err(invalid!("Invalid AES vendor")); + } + let vendor_version = match vendor_version { + 0x0001 => AesVendorVersion::Ae1, + 0x0002 => AesVendorVersion::Ae2, + _ => return Err(invalid!("Invalid AES vendor version")), + }; + match aes_mode { + 0x01 => file.aes_mode = Some((AesMode::Aes128, vendor_version, compression_method)), + 0x02 => file.aes_mode = Some((AesMode::Aes192, vendor_version, compression_method)), + 0x03 => file.aes_mode = Some((AesMode::Aes256, vendor_version, compression_method)), + _ => return Err(invalid!("Invalid AES encryption strength")), + }; + file.compression_method = compression_method; + file.aes_extra_data_start = bytes_already_read; + } + 0x5455 => { + // extended timestamp + // https://libzip.org/specifications/extrafld.txt + + file.extra_fields.push(ExtraField::ExtendedTimestamp( + ExtendedTimestamp::try_from_reader(reader, len)?, + )); + } + 0x6375 => { + // Info-ZIP Unicode Comment Extra Field + // APPNOTE 4.6.8 and https://libzip.org/specifications/extrafld.txt + file.file_comment = String::from_utf8( + UnicodeExtraField::try_from_reader(reader, len)? + .unwrap_valid(file.file_comment.as_bytes())? + .into_vec(), + )? + .into(); + } + 0x7075 => { + // Info-ZIP Unicode Path Extra Field + // APPNOTE 4.6.9 and https://libzip.org/specifications/extrafld.txt + file.file_name_raw = UnicodeExtraField::try_from_reader(reader, len)? + .unwrap_valid(&file.file_name_raw)?; + file.file_name = + String::from_utf8(file.file_name_raw.clone().into_vec())?.into_boxed_str(); + file.is_utf8 = true; + } + _ => { + if let Err(e) = reader.read_exact(&mut vec![0u8; len as usize]) { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!("Extra field content truncated")); + } + return Err(e.into()); + } + // Other fields are ignored + } + } + Ok(false) +} + +/// A trait for exposing file metadata inside the zip. +pub trait HasZipMetadata { + /// Get the file metadata + fn get_metadata(&self) -> &ZipFileData; +} + +/// Options for reading a file from an archive. +#[derive(Default)] +pub struct ZipReadOptions<'a> { + /// The password to use when decrypting the file. This is ignored if not required. + password: Option<&'a [u8]>, + + /// Ignore the value of the encryption flag and proceed as if the file were plaintext. + ignore_encryption_flag: bool, +} + +impl<'a> ZipReadOptions<'a> { + /// Create a new set of options with the default values. + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Set the password, if any, to use. Return for chaining. + #[must_use] + pub fn password(mut self, password: Option<&'a [u8]>) -> Self { + self.password = password; + self + } + + /// Set the ignore encryption flag. Return for chaining. + #[must_use] + pub fn ignore_encryption_flag(mut self, ignore: bool) -> Self { + self.ignore_encryption_flag = ignore; + self + } +} + +/// Methods for retrieving information on zip files +impl<'a, R: Read + ?Sized> ZipFile<'a, R> { + pub(crate) fn take_raw_reader(&mut self) -> io::Result> { + replace(&mut self.reader, ZipFileReader::NoReader).into_inner() + } + + /// Get the version of the file + pub fn version_made_by(&self) -> (u8, u8) { + ( + self.get_metadata().version_made_by / 10, + self.get_metadata().version_made_by % 10, + ) + } + + /// Get the name of the file + /// + /// # Warnings + /// + /// It is dangerous to use this name directly when extracting an archive. + /// It may contain an absolute path (`/etc/shadow`), or break out of the + /// current directory (`../runtime`). Carelessly writing to these paths + /// allows an attacker to craft a ZIP archive that will overwrite critical + /// files. + /// + /// You can use the [`ZipFile::enclosed_name`] method to validate the name + /// as a safe path. + pub fn name(&self) -> &str { + &self.get_metadata().file_name + } + + /// Get the name of the file, in the raw (internal) byte representation. + /// + /// The encoding of this data is currently undefined. + pub fn name_raw(&self) -> &[u8] { + &self.get_metadata().file_name_raw + } + + /// Get the name of the file in a sanitized form. It truncates the name to the first NULL byte, + /// removes a leading '/' and removes '..' parts. + #[deprecated( + since = "0.5.7", + note = "by stripping `..`s from the path, the meaning of paths can change. + `mangled_name` can be used if this behaviour is desirable" + )] + pub fn sanitized_name(&self) -> PathBuf { + self.mangled_name() + } + + /// Rewrite the path, ignoring any path components with special meaning. + /// + /// - Absolute paths are made relative + /// - [`ParentDir`]s are ignored + /// - Truncates the filename at a NULL byte + /// + /// This is appropriate if you need to be able to extract *something* from + /// any archive, but will easily misrepresent trivial paths like + /// `foo/../bar` as `foo/bar` (instead of `bar`). Because of this, + /// [`ZipFile::enclosed_name`] is the better option in most scenarios. + /// + /// [`ParentDir`]: `Component::ParentDir` + pub fn mangled_name(&self) -> PathBuf { + self.get_metadata().file_name_sanitized() + } + + /// Ensure the file path is safe to use as a [`Path`]. + /// + /// - It can't contain NULL bytes + /// - It can't resolve to a path outside the current directory + /// > `foo/../bar` is fine, `foo/../../bar` is not. + /// - It can't be an absolute path + /// + /// This will read well-formed ZIP files correctly, and is resistant + /// to path-based exploits. It is recommended over + /// [`ZipFile::mangled_name`]. + pub fn enclosed_name(&self) -> Option { + self.get_metadata().enclosed_name() + } + + pub(crate) fn simplified_components(&self) -> Option> { + self.get_metadata().simplified_components() + } + + /// Prepare the path for extraction by creating necessary missing directories and checking for symlinks to be contained within the base path. + /// + /// `base_path` parameter is assumed to be canonicalized. + pub(crate) fn safe_prepare_path( + &self, + base_path: &Path, + outpath: &mut PathBuf, + root_dir: Option<&(Vec<&OsStr>, impl RootDirFilter)>, + ) -> ZipResult<()> { + let components = self + .simplified_components() + .ok_or(invalid!("Invalid file path"))?; + + let components = match root_dir { + Some((root_dir, filter)) => match components.strip_prefix(&**root_dir) { + Some(components) => components, + + // In this case, we expect that the file was not in the root + // directory, but was filtered out when searching for the + // root directory. + None => { + // We could technically find ourselves at this code + // path if the user provides an unstable or + // non-deterministic `filter` function. + // + // If debug assertions are on, we should panic here. + // Otherwise, the safest thing to do here is to just + // extract as-is. + debug_assert!( + !filter(&PathBuf::from_iter(components.iter())), + "Root directory filter should not match at this point" + ); + + // Extract as-is. + &components[..] + } + }, + + None => &components[..], + }; + + let components_len = components.len(); + + for (is_last, component) in components + .iter() + .copied() + .enumerate() + .map(|(i, c)| (i == components_len - 1, c)) + { + // we can skip the target directory itself because the base path is assumed to be "trusted" (if the user say extract to a symlink we can follow it) + outpath.push(component); + + // check if the path is a symlink, the target must be _inherently_ within the directory + for limit in (0..5u8).rev() { + let meta = match std::fs::symlink_metadata(&outpath) { + Ok(meta) => meta, + Err(e) if e.kind() == io::ErrorKind::NotFound => { + if !is_last { + crate::read::make_writable_dir_all(&outpath)?; + } + break; + } + Err(e) => return Err(e.into()), + }; + + if !meta.is_symlink() { + break; + } + + if limit == 0 { + return Err(invalid!("Extraction followed a symlink too deep")); + } + + // note that we cannot accept links that do not inherently resolve to a path inside the directory to prevent: + // - disclosure of unrelated path exists (no check for a path exist and then ../ out) + // - issues with file-system specific path resolution (case sensitivity, etc) + let target = std::fs::read_link(&outpath)?; + + if !crate::path::simplified_components(&target) + .ok_or(invalid!("Invalid symlink target path"))? + .starts_with( + &crate::path::simplified_components(base_path) + .ok_or(invalid!("Invalid base path"))?, + ) + { + let is_absolute_enclosed = base_path + .components() + .map(Some) + .chain(std::iter::once(None)) + .zip(target.components().map(Some).chain(std::iter::repeat(None))) + .all(|(a, b)| match (a, b) { + // both components are normal + (Some(Component::Normal(a)), Some(Component::Normal(b))) => a == b, + // both components consumed fully + (None, None) => true, + // target consumed fully but base path is not + (Some(_), None) => false, + // base path consumed fully but target is not (and normal) + (None, Some(Component::CurDir | Component::Normal(_))) => true, + _ => false, + }); + + if !is_absolute_enclosed { + return Err(invalid!("Symlink is not inherently safe")); + } + } + + outpath.push(target); + } + } + Ok(()) + } + + /// Get the comment of the file + pub fn comment(&self) -> &str { + &self.get_metadata().file_comment + } + + /// Get the compression method used to store the file + pub fn compression(&self) -> CompressionMethod { + self.get_metadata().compression_method + } + + /// Get if the files is encrypted or not + pub fn encrypted(&self) -> bool { + self.data.encrypted + } + + /// Get the size of the file, in bytes, in the archive + pub fn compressed_size(&self) -> u64 { + self.get_metadata().compressed_size + } + + /// Get the size of the file, in bytes, when uncompressed + pub fn size(&self) -> u64 { + self.get_metadata().uncompressed_size + } + + /// Get the time the file was last modified + pub fn last_modified(&self) -> Option { + self.data.last_modified_time + } + /// Returns whether the file is actually a directory + pub fn is_dir(&self) -> bool { + is_dir(self.name()) + } + + /// Returns whether the file is actually a symbolic link + pub fn is_symlink(&self) -> bool { + self.unix_mode() + .is_some_and(|mode| mode & S_IFLNK == S_IFLNK) + } + + /// Returns whether the file is a normal file (i.e. not a directory or symlink) + pub fn is_file(&self) -> bool { + !self.is_dir() && !self.is_symlink() + } + + /// Get unix mode for the file + pub fn unix_mode(&self) -> Option { + self.get_metadata().unix_mode() + } + + /// Get the CRC32 hash of the original file + pub fn crc32(&self) -> u32 { + self.get_metadata().crc32 + } + + /// Get the extra data of the zip header for this file + pub fn extra_data(&self) -> Option<&[u8]> { + self.get_metadata() + .extra_field + .as_ref() + .map(|v| v.deref().deref()) + } + + /// Get the starting offset of the data of the compressed file + pub fn data_start(&self) -> u64 { + *self.data.data_start.get().unwrap() + } + + /// Get the starting offset of the zip header for this file + pub fn header_start(&self) -> u64 { + self.get_metadata().header_start + } + /// Get the starting offset of the zip header in the central directory for this file + pub fn central_header_start(&self) -> u64 { + self.get_metadata().central_header_start + } + + /// Get the [`SimpleFileOptions`] that would be used to write this file to + /// a new zip archive. + pub fn options(&self) -> SimpleFileOptions { + let mut options = SimpleFileOptions::default() + .large_file(self.compressed_size().max(self.size()) > ZIP64_BYTES_THR) + .compression_method(self.compression()) + .unix_permissions(self.unix_mode().unwrap_or(0o644) | S_IFREG) + .last_modified_time( + self.last_modified() + .filter(|m| m.is_valid()) + .unwrap_or_else(DateTime::default_for_write), + ); + + options.normalize(); + #[cfg(feature = "aes-crypto")] + if let Some(aes) = self.get_metadata().aes_mode { + // Preserve AES metadata in options for downstream writers. + // This is metadata-only and does not trigger encryption. + options.aes_mode = Some(aes); + } + options + } +} + +/// Methods for retrieving information on zip files +impl ZipFile<'_, R> { + /// iterate through all extra fields + pub fn extra_data_fields(&self) -> impl Iterator { + self.data.extra_fields.iter() + } +} + +impl HasZipMetadata for ZipFile<'_, R> { + fn get_metadata(&self) -> &ZipFileData { + self.data.as_ref() + } +} + +impl Read for ZipFile<'_, R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.reader.read(buf) + } + + fn read_exact(&mut self, buf: &mut [u8]) -> io::Result<()> { + self.reader.read_exact(buf) + } + + fn read_to_end(&mut self, buf: &mut Vec) -> io::Result { + self.reader.read_to_end(buf) + } + + fn read_to_string(&mut self, buf: &mut String) -> io::Result { + self.reader.read_to_string(buf) + } +} + +impl Read for ZipFileSeek<'_, R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + match &mut self.reader { + ZipFileSeekReader::Raw(r) => r.read(buf), + } + } +} + +impl Seek for ZipFileSeek<'_, R> { + fn seek(&mut self, pos: SeekFrom) -> io::Result { + match &mut self.reader { + ZipFileSeekReader::Raw(r) => r.seek(pos), + } + } +} + +impl HasZipMetadata for ZipFileSeek<'_, R> { + fn get_metadata(&self) -> &ZipFileData { + self.data.as_ref() + } +} + +impl Drop for ZipFile<'_, R> { + fn drop(&mut self) { + // self.data is Owned, this reader is constructed by a streaming reader. + // In this case, we want to exhaust the reader so that the next file is accessible. + if let Cow::Owned(_) = self.data { + // Get the inner `Take` reader so all decryption, decompression and CRC calculation is skipped. + if let Ok(mut inner) = self.take_raw_reader() { + let _ = copy(&mut inner, &mut sink()); + } + } + } +} + +/// Read ZipFile structures from a non-seekable reader. +/// +/// This is an alternative method to read a zip file. If possible, use the ZipArchive functions +/// as some information will be missing when reading this manner. +/// +/// Reads a file header from the start of the stream. Will return `Ok(Some(..))` if a file is +/// present at the start of the stream. Returns `Ok(None)` if the start of the central directory +/// is encountered. No more files should be read after this. +/// +/// The Drop implementation of ZipFile ensures that the reader will be correctly positioned after +/// the structure is done. +/// +/// Missing fields are: +/// * `comment`: set to an empty string +/// * `data_start`: set to 0 +/// * `external_attributes`: `unix_mode()`: will return None +pub fn read_zipfile_from_stream(reader: &mut R) -> ZipResult>> { + // We can't use the typical ::parse() method, as we follow separate code paths depending on the + // "magic" value (since the magic value will be from the central directory header if we've + // finished iterating over all the actual files). + /* TODO: smallvec? */ + + let mut block = ZipLocalEntryBlock::zeroed(); + reader.read_exact(block.as_bytes_mut())?; + + match block.magic().from_le() { + spec::Magic::LOCAL_FILE_HEADER_SIGNATURE => (), + spec::Magic::CENTRAL_DIRECTORY_HEADER_SIGNATURE => return Ok(None), + _ => return Err(ZipLocalEntryBlock::WRONG_MAGIC_ERROR), + } + + let block = block.from_le(); + + let mut result = ZipFileData::from_local_block(block, reader)?; + + match parse_extra_field(&mut result) { + Ok(..) | Err(ZipError::Io(..)) => {} + Err(e) => return Err(e), + } + + let limit_reader = reader.take(result.compressed_size); + let crypto_reader = make_crypto_reader(&result, limit_reader, None, None)?; + let ZipFileData { + crc32, + uncompressed_size, + compression_method, + #[cfg(feature = "legacy-zip")] + flags, + .. + } = result; + + Ok(Some(ZipFile { + data: Cow::Owned(result), + reader: make_reader( + compression_method, + uncompressed_size, + crc32, + crypto_reader, + #[cfg(feature = "legacy-zip")] + flags, + )?, + })) +} + +/// A filter that determines whether an entry should be ignored when searching +/// for the root directory of a Zip archive. +/// +/// Returns `true` if the entry should be considered, and `false` if it should +/// be ignored. +/// +/// See [`root_dir_common_filter`] for a sensible default filter. +pub trait RootDirFilter: Fn(&Path) -> bool {} +impl bool> RootDirFilter for F {} + +/// Common filters when finding the root directory of a Zip archive. +/// +/// This filter is a sensible default for most use cases and filters out common +/// system files that are usually irrelevant to the contents of the archive. +/// +/// Currently, the filter ignores: +/// - `/__MACOSX/` +/// - `/.DS_Store` +/// - `/Thumbs.db` +/// +/// **This function is not guaranteed to be stable and may change in future versions.** +/// +/// # Example +/// +/// ```rust +/// # use std::path::Path; +/// assert!(zip::read::root_dir_common_filter(Path::new("foo.txt"))); +/// assert!(!zip::read::root_dir_common_filter(Path::new(".DS_Store"))); +/// assert!(!zip::read::root_dir_common_filter(Path::new("Thumbs.db"))); +/// assert!(!zip::read::root_dir_common_filter(Path::new("__MACOSX"))); +/// assert!(!zip::read::root_dir_common_filter(Path::new("__MACOSX/foo.txt"))); +/// ``` +pub fn root_dir_common_filter(path: &Path) -> bool { + const COMMON_FILTER_ROOT_FILES: &[&str] = &[".DS_Store", "Thumbs.db"]; + + if path.starts_with("__MACOSX") { + return false; + } + + if path.components().count() == 1 + && path.file_name().is_some_and(|file_name| { + COMMON_FILTER_ROOT_FILES + .iter() + .map(OsStr::new) + .any(|cmp| cmp == file_name) + }) + { + return false; + } + + true +} + +#[cfg(feature = "chrono")] +/// Generate a `SystemTime` from a `DateTime`. +fn datetime_to_systemtime(datetime: &DateTime) -> Option { + if let Some(t) = generate_chrono_datetime(datetime) { + let time = chrono::DateTime::::from_naive_utc_and_offset(t, chrono::Utc); + return Some(time.into()); + } + None +} + +#[cfg(feature = "chrono")] +/// Generate a `NaiveDateTime` from a `DateTime`. +fn generate_chrono_datetime(datetime: &DateTime) -> Option { + if let Some(d) = chrono::NaiveDate::from_ymd_opt( + datetime.year().into(), + datetime.month().into(), + datetime.day().into(), + ) { + if let Some(d) = d.and_hms_opt( + datetime.hour().into(), + datetime.minute().into(), + datetime.second().into(), + ) { + return Some(d); + } + } + None +} + +/// Read ZipFile from a non-seekable reader like [read_zipfile_from_stream] does, but assume the +/// given compressed size and don't read any further ahead than that. +pub fn read_zipfile_from_stream_with_compressed_size( + reader: &mut R, + compressed_size: u64, +) -> ZipResult>> { + let mut block = ZipLocalEntryBlock::zeroed(); + reader.read_exact(block.as_bytes_mut())?; + + match block.magic().from_le() { + spec::Magic::LOCAL_FILE_HEADER_SIGNATURE => (), + spec::Magic::CENTRAL_DIRECTORY_HEADER_SIGNATURE => return Ok(None), + _ => return Err(ZipLocalEntryBlock::WRONG_MAGIC_ERROR), + } + + let block = block.from_le(); + + let mut result = ZipFileData::from_local_block(block, reader)?; + result.compressed_size = compressed_size; + + if result.encrypted { + return unsupported_zip_error("Encrypted files are not supported"); + } + + let limit_reader = reader.take(result.compressed_size); + let crypto_reader = make_crypto_reader(&result, limit_reader, None, None)?; + let ZipFileData { + crc32, + compression_method, + uncompressed_size, + #[cfg(feature = "legacy-zip")] + flags, + .. + } = result; + + Ok(Some(ZipFile { + data: Cow::Owned(result), + reader: make_reader( + compression_method, + uncompressed_size, + crc32, + crypto_reader, + #[cfg(feature = "legacy-zip")] + flags, + )?, + })) +} + +#[cfg(test)] +mod test { + use crate::read::ZipReadOptions; + use crate::result::ZipResult; + use crate::types::SimpleFileOptions; + use crate::CompressionMethod::Stored; + use crate::{ZipArchive, ZipWriter}; + use std::io::{Cursor, Read, Write}; + use tempfile::TempDir; + + #[test] + fn invalid_offset() { + use super::ZipArchive; + + let reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/invalid_offset.zip" + ))); + assert!(reader.is_err()); + } + + #[test] + fn invalid_offset2() { + use super::ZipArchive; + + let reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/invalid_offset2.zip" + ))); + assert!(reader.is_err()); + } + + #[test] + fn zip64_with_leading_junk() { + use super::ZipArchive; + + let reader = + ZipArchive::new(Cursor::new(include_bytes!("../tests/data/zip64_demo.zip"))).unwrap(); + assert_eq!(reader.len(), 1); + } + + #[test] + fn zip_contents() { + use super::ZipArchive; + + let mut reader = + ZipArchive::new(Cursor::new(include_bytes!("../tests/data/mimetype.zip"))).unwrap(); + assert_eq!(reader.comment(), b""); + assert_eq!(reader.by_index(0).unwrap().central_header_start(), 77); + } + + #[test] + fn zip_read_streaming() { + use super::read_zipfile_from_stream; + + let mut reader = Cursor::new(include_bytes!("../tests/data/mimetype.zip")); + loop { + if read_zipfile_from_stream(&mut reader).unwrap().is_none() { + break; + } + } + } + + #[test] + fn zip_clone() { + use super::ZipArchive; + use std::io::Read; + + let mut reader1 = + ZipArchive::new(Cursor::new(include_bytes!("../tests/data/mimetype.zip"))).unwrap(); + let mut reader2 = reader1.clone(); + + let mut file1 = reader1.by_index(0).unwrap(); + let mut file2 = reader2.by_index(0).unwrap(); + + let t = file1.last_modified().unwrap(); + assert_eq!( + ( + t.year(), + t.month(), + t.day(), + t.hour(), + t.minute(), + t.second() + ), + (1980, 1, 1, 0, 0, 0) + ); + + let mut buf1 = [0; 5]; + let mut buf2 = [0; 5]; + let mut buf3 = [0; 5]; + let mut buf4 = [0; 5]; + + file1.read_exact(&mut buf1).unwrap(); + file2.read_exact(&mut buf2).unwrap(); + file1.read_exact(&mut buf3).unwrap(); + file2.read_exact(&mut buf4).unwrap(); + + assert_eq!(buf1, buf2); + assert_eq!(buf3, buf4); + assert_ne!(buf1, buf3); + } + + #[test] + fn file_and_dir_predicates() { + use super::ZipArchive; + + let mut zip = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/files_and_dirs.zip" + ))) + .unwrap(); + + for i in 0..zip.len() { + let zip_file = zip.by_index(i).unwrap(); + let full_name = zip_file.enclosed_name().unwrap(); + let file_name = full_name.file_name().unwrap().to_str().unwrap(); + assert!( + (file_name.starts_with("dir") && zip_file.is_dir()) + || (file_name.starts_with("file") && zip_file.is_file()) + ); + } + } + + #[test] + fn zip64_magic_in_filenames() { + let files = vec![ + include_bytes!("../tests/data/zip64_magic_in_filename_1.zip").to_vec(), + include_bytes!("../tests/data/zip64_magic_in_filename_2.zip").to_vec(), + include_bytes!("../tests/data/zip64_magic_in_filename_3.zip").to_vec(), + include_bytes!("../tests/data/zip64_magic_in_filename_4.zip").to_vec(), + include_bytes!("../tests/data/zip64_magic_in_filename_5.zip").to_vec(), + ]; + // Although we don't allow adding files whose names contain the ZIP64 CDB-end or + // CDB-end-locator signatures, we still read them when they aren't genuinely ambiguous. + for file in files { + ZipArchive::new(Cursor::new(file)).unwrap(); + } + } + + /// test case to ensure we don't preemptively over allocate based on the + /// declared number of files in the CDE of an invalid zip when the number of + /// files declared is more than the alleged offset in the CDE + #[test] + fn invalid_cde_number_of_files_allocation_smaller_offset() { + use super::ZipArchive; + + let reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/invalid_cde_number_of_files_allocation_smaller_offset.zip" + ))); + assert!(reader.is_err() || reader.unwrap().is_empty()); + } + + /// test case to ensure we don't preemptively over allocate based on the + /// declared number of files in the CDE of an invalid zip when the number of + /// files declared is less than the alleged offset in the CDE + #[test] + fn invalid_cde_number_of_files_allocation_greater_offset() { + use super::ZipArchive; + + let reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/invalid_cde_number_of_files_allocation_greater_offset.zip" + ))); + assert!(reader.is_err()); + } + + #[cfg(feature = "deflate64")] + #[test] + fn deflate64_index_out_of_bounds() -> std::io::Result<()> { + let mut reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/raw_deflate64_index_out_of_bounds.zip" + )))?; + std::io::copy(&mut reader.by_index(0)?, &mut std::io::sink()).expect_err("Invalid file"); + Ok(()) + } + + #[cfg(feature = "deflate64")] + #[test] + fn deflate64_not_enough_space() { + ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/deflate64_issue_25.zip" + ))) + .expect_err("Invalid file"); + } + + #[cfg(feature = "deflate-flate2")] + #[test] + fn test_read_with_data_descriptor() { + use std::io::Read; + + let mut reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/data_descriptor.zip" + ))) + .unwrap(); + let mut decompressed = [0u8; 16]; + let mut file = reader.by_index(0).unwrap(); + assert_eq!(file.read(&mut decompressed).unwrap(), 12); + } + + #[test] + fn test_is_symlink() -> std::io::Result<()> { + let mut reader = ZipArchive::new(Cursor::new(include_bytes!("../tests/data/symlink.zip")))?; + assert!(reader.by_index(0)?.is_symlink()); + let tempdir = TempDir::with_prefix("test_is_symlink")?; + reader.extract(&tempdir)?; + assert!(tempdir.path().join("bar").is_symlink()); + Ok(()) + } + + #[test] + #[cfg(feature = "deflate-flate2")] + fn test_utf8_extra_field() { + let mut reader = + ZipArchive::new(Cursor::new(include_bytes!("../tests/data/chinese.zip"))).unwrap(); + reader.by_name("七个房间.txt").unwrap(); + } + + #[test] + fn test_utf8() { + let mut reader = + ZipArchive::new(Cursor::new(include_bytes!("../tests/data/linux-7z.zip"))).unwrap(); + reader.by_name("你好.txt").unwrap(); + } + + #[test] + fn test_utf8_2() { + let mut reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/windows-7zip.zip" + ))) + .unwrap(); + reader.by_name("你好.txt").unwrap(); + } + + #[test] + fn test_64k_files() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = SimpleFileOptions { + compression_method: Stored, + ..Default::default() + }; + for i in 0..=u16::MAX { + let file_name = format!("{i}.txt"); + writer.start_file(&*file_name, options)?; + writer.write_all(i.to_string().as_bytes())?; + } + + let mut reader = ZipArchive::new(writer.finish()?)?; + for i in 0..=u16::MAX { + let expected_name = format!("{i}.txt"); + let expected_contents = i.to_string(); + let expected_contents = expected_contents.as_bytes(); + let mut file = reader.by_name(&expected_name)?; + let mut contents = Vec::with_capacity(expected_contents.len()); + file.read_to_end(&mut contents)?; + assert_eq!(contents, expected_contents); + drop(file); + contents.clear(); + let mut file = reader.by_index(i as usize)?; + file.read_to_end(&mut contents)?; + assert_eq!(contents, expected_contents); + } + Ok(()) + } + + /// Symlinks being extracted shouldn't be followed out of the destination directory. + #[test] + fn test_cannot_symlink_outside_destination() -> ZipResult<()> { + use std::fs::create_dir; + + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.add_symlink("symlink/", "../dest-sibling/", SimpleFileOptions::default())?; + writer.start_file("symlink/dest-file", SimpleFileOptions::default())?; + let mut reader = writer.finish_into_readable()?; + let dest_parent = TempDir::with_prefix("read__test_cannot_symlink_outside_destination")?; + let dest_sibling = dest_parent.path().join("dest-sibling"); + create_dir(&dest_sibling)?; + let dest = dest_parent.path().join("dest"); + create_dir(&dest)?; + assert!(reader.extract(dest).is_err()); + assert!(!dest_sibling.join("dest-file").exists()); + Ok(()) + } + + #[test] + fn test_can_create_destination() -> ZipResult<()> { + let mut reader = + ZipArchive::new(Cursor::new(include_bytes!("../tests/data/mimetype.zip")))?; + let dest = TempDir::with_prefix("read__test_can_create_destination")?; + reader.extract(&dest)?; + assert!(dest.path().join("mimetype").exists()); + Ok(()) + } + + #[test] + fn test_central_directory_not_at_end() -> ZipResult<()> { + let mut reader = ZipArchive::new(Cursor::new(include_bytes!("../tests/data/omni.ja")))?; + let mut file = reader.by_name("chrome.manifest")?; + let mut contents = String::new(); + file.read_to_string(&mut contents)?; // ensures valid UTF-8 + assert!(!contents.is_empty(), "chrome.manifest should not be empty"); + drop(file); + for i in 0..reader.len() { + let mut file = reader.by_index(i)?; + // Attempt to read a small portion or all of each file to ensure it's accessible + let mut buffer = Vec::new(); + file.read_to_end(&mut buffer)?; + assert_eq!( + buffer.len(), + file.size() as usize, + "File size mismatch for {}", + file.name() + ); + } + Ok(()) + } + + #[test] + fn test_ignore_encryption_flag() -> ZipResult<()> { + let mut reader = ZipArchive::new(Cursor::new(include_bytes!( + "../tests/data/ignore_encryption_flag.zip" + )))?; + + // Get the file entry by ignoring its encryption flag. + let mut file = + reader.by_index_with_options(0, ZipReadOptions::new().ignore_encryption_flag(true))?; + let mut contents = String::new(); + assert_eq!(file.name(), "plaintext.txt"); + + // The file claims it is encrypted, but it is not. + assert!(file.encrypted()); + file.read_to_string(&mut contents)?; // ensures valid UTF-8 + assert_eq!(contents, "This file is not encrypted.\n"); + Ok(()) + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/read/config.rs b/v2/vendor/security/zip-7.2.0/src/read/config.rs new file mode 100644 index 0000000000..b00c58a49a --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/read/config.rs @@ -0,0 +1,23 @@ +/// Configuration for reading ZIP archives. +#[repr(transparent)] +#[derive(Debug, Default, Clone, Copy)] +pub struct Config { + /// An offset into the reader to use to find the start of the archive. + pub archive_offset: ArchiveOffset, +} + +/// The offset of the start of the archive from the beginning of the reader. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash)] +pub enum ArchiveOffset { + /// Try to detect the archive offset automatically. + /// + /// This will look at the central directory specified by `FromCentralDirectory` for a header. + /// If missing, this will behave as if `None` were specified. + #[default] + Detect, + /// Use the central directory length and offset to determine the start of the archive. + #[deprecated(since = "2.3.0", note = "use `Detect` instead")] + FromCentralDirectory, + /// Specify a fixed archive offset. + Known(u64), +} diff --git a/v2/vendor/security/zip-7.2.0/src/read/magic_finder.rs b/v2/vendor/security/zip-7.2.0/src/read/magic_finder.rs new file mode 100644 index 0000000000..b6e8c7d8c9 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/read/magic_finder.rs @@ -0,0 +1,291 @@ +use std::io::{Read, Seek, SeekFrom}; + +use memchr::memmem::{Finder, FinderRev}; + +use crate::result::ZipResult; + +pub trait FinderDirection<'a> { + fn new(needle: &'a [u8]) -> Self; + fn reset_cursor(bounds: (u64, u64), window_size: usize) -> u64; + fn scope_window(window: &[u8], mid_window_offset: usize) -> (&[u8], usize); + + fn needle(&self) -> &[u8]; + fn find(&self, haystack: &[u8]) -> Option; + fn move_cursor(&self, cursor: u64, bounds: (u64, u64), window_size: usize) -> Option; + fn move_scope(&self, offset: usize) -> usize; +} + +pub struct Forward<'a>(Finder<'a>); +impl<'a> FinderDirection<'a> for Forward<'a> { + fn new(needle: &'a [u8]) -> Self { + Self(Finder::new(needle)) + } + + fn reset_cursor((start_inclusive, _): (u64, u64), _: usize) -> u64 { + start_inclusive + } + + fn scope_window(window: &[u8], mid_window_offset: usize) -> (&[u8], usize) { + (&window[mid_window_offset..], mid_window_offset) + } + + fn find(&self, haystack: &[u8]) -> Option { + self.0.find(haystack) + } + + fn needle(&self) -> &[u8] { + self.0.needle() + } + + fn move_cursor(&self, cursor: u64, bounds: (u64, u64), window_size: usize) -> Option { + let magic_overlap = self.needle().len().saturating_sub(1) as u64; + let next = cursor.saturating_add(window_size as u64 - magic_overlap); + + if next >= bounds.1 { + None + } else { + Some(next) + } + } + + fn move_scope(&self, offset: usize) -> usize { + offset + self.needle().len() + } +} + +pub struct Backwards<'a>(FinderRev<'a>); +impl<'a> FinderDirection<'a> for Backwards<'a> { + fn new(needle: &'a [u8]) -> Self { + Self(FinderRev::new(needle)) + } + + fn reset_cursor(bounds: (u64, u64), window_size: usize) -> u64 { + bounds + .1 + .saturating_sub(window_size as u64) + .clamp(bounds.0, bounds.1) + } + + fn scope_window(window: &[u8], mid_window_offset: usize) -> (&[u8], usize) { + (&window[..mid_window_offset], 0) + } + + fn find(&self, haystack: &[u8]) -> Option { + self.0.rfind(haystack) + } + + fn needle(&self) -> &[u8] { + self.0.needle() + } + + fn move_cursor(&self, cursor: u64, bounds: (u64, u64), window_size: usize) -> Option { + let magic_overlap = self.needle().len().saturating_sub(1) as u64; + + if cursor <= bounds.0 { + None + } else { + Some( + cursor + .saturating_add(magic_overlap) + .saturating_sub(window_size as u64) + .clamp(bounds.0, bounds.1), + ) + } + } + + fn move_scope(&self, offset: usize) -> usize { + offset + } +} + +/// A utility for finding magic symbols from the end of a seekable reader. +/// +/// Can be repurposed to recycle the internal buffer. +pub struct MagicFinder { + buffer: Box<[u8]>, + pub(self) finder: Direction, + cursor: u64, + mid_buffer_offset: Option, + bounds: (u64, u64), +} + +impl<'a, T: FinderDirection<'a>> MagicFinder { + /// Create a new magic bytes finder to look within specific bounds. + pub fn new(magic_bytes: &'a [u8], start_inclusive: u64, end_exclusive: u64) -> Self { + const BUFFER_SIZE: usize = 2048; + + // Smaller buffer size would be unable to locate bytes. + // Equal buffer size would stall (the window could not be moved). + debug_assert!(BUFFER_SIZE >= magic_bytes.len()); + + Self { + buffer: vec![0; BUFFER_SIZE].into_boxed_slice(), + finder: T::new(magic_bytes), + cursor: T::reset_cursor((start_inclusive, end_exclusive), BUFFER_SIZE), + mid_buffer_offset: None, + bounds: (start_inclusive, end_exclusive), + } + } + + /// Repurpose the finder for different bytes or bounds. + pub fn repurpose(&mut self, magic_bytes: &'a [u8], bounds: (u64, u64)) -> &mut Self { + debug_assert!(self.buffer.len() >= magic_bytes.len()); + + self.finder = T::new(magic_bytes); + self.cursor = T::reset_cursor(bounds, self.buffer.len()); + self.bounds = bounds; + + // Reset the mid-buffer offset, to invalidate buffer content. + self.mid_buffer_offset = None; + + self + } + + /// Find the next magic bytes in the direction specified in the type. + pub fn next(&mut self, reader: &mut R) -> ZipResult> { + loop { + if self.cursor < self.bounds.0 || self.cursor >= self.bounds.1 { + // The finder is consumed + break; + } + + /* Position the window and ensure correct length */ + let window_start = self.cursor; + let window_end = self + .cursor + .saturating_add(self.buffer.len() as u64) + .min(self.bounds.1); + + if window_end <= window_start { + // Short-circuit on zero-sized windows to prevent loop + break; + } + + let window = &mut self.buffer[..(window_end - window_start) as usize]; + + if self.mid_buffer_offset.is_none() { + reader.seek(SeekFrom::Start(window_start))?; + if let Err(e) = reader.read_exact(window) { + if e.kind() == std::io::ErrorKind::UnexpectedEof { + // If we can't read the whole window, it means the bounds or file length are wrong. + break; + } + return Err(e.into()); + } + } + + let (window, window_start_offset) = match self.mid_buffer_offset { + Some(mid_buffer_offset) => T::scope_window(window, mid_buffer_offset), + None => (&*window, 0usize), + }; + + if let Some(offset) = self.finder.find(window) { + let magic_pos = window_start + window_start_offset as u64 + offset as u64; + reader.seek(SeekFrom::Start(magic_pos))?; + + self.mid_buffer_offset = Some(self.finder.move_scope(window_start_offset + offset)); + + return Ok(Some(magic_pos)); + } + + self.mid_buffer_offset = None; + + match self + .finder + .move_cursor(self.cursor, self.bounds, self.buffer.len()) + { + Some(new_cursor) => { + self.cursor = new_cursor; + } + None => { + // Destroy the finder when we've reached the end of the bounds. + self.bounds.0 = self.bounds.1; + break; + } + } + } + + Ok(None) + } +} + +/// A magic bytes finder with an optimistic guess that is tried before +/// the inner finder begins searching from end. This enables much faster +/// lookup in files without appended junk, because the magic bytes will be +/// found directly. +/// +/// The guess can be marked as mandatory to produce an error. This is useful +/// if the ArchiveOffset is known and auto-detection is not desired. +pub struct OptimisticMagicFinder { + inner: MagicFinder, + initial_guess: Option<(u64, bool)>, +} + +/// This is a temporary restriction, to avoid heap allocation in [`Self::next_back`]. +/// +/// We only use magic bytes of size 4 at the moment. +const STACK_BUFFER_SIZE: usize = 8; + +impl<'a, Direction: FinderDirection<'a>> OptimisticMagicFinder { + /// Create a new empty optimistic magic bytes finder. + pub fn new_empty() -> Self { + Self { + inner: MagicFinder::new(&[], 0, 0), + initial_guess: None, + } + } + + /// Repurpose the finder for different bytes, bounds and initial guesses. + pub fn repurpose( + &mut self, + magic_bytes: &'a [u8], + bounds: (u64, u64), + initial_guess: Option<(u64, bool)>, + ) -> &mut Self { + debug_assert!(magic_bytes.len() <= STACK_BUFFER_SIZE); + + self.inner.repurpose(magic_bytes, bounds); + self.initial_guess = initial_guess; + + self + } + + /// Equivalent to `next_back`, with an optional initial guess attempted before + /// proceeding with reading from the back of the reader. + pub fn next(&mut self, reader: &mut R) -> ZipResult> { + if let Some((v, mandatory)) = self.initial_guess { + reader.seek(SeekFrom::Start(v))?; + + let mut buffer = [0; STACK_BUFFER_SIZE]; + let buffer = &mut buffer[..self.inner.finder.needle().len()]; + + // Attempt to match only if there's enough space for the needle + if v.saturating_add(buffer.len() as u64) <= self.inner.bounds.1 { + if let Err(e) = reader.read_exact(buffer) { + if e.kind() == std::io::ErrorKind::UnexpectedEof { + if mandatory { + return Ok(None); + } + } else { + return Err(e.into()); + } + } else if self.inner.finder.needle() == buffer { + // If a match is found, yield it. + self.initial_guess.take(); + reader.seek(SeekFrom::Start(v))?; + return Ok(Some(v)); + } + } + + // If a match is not found, but the initial guess was mandatory, return an error. + if mandatory { + return Ok(None); + } + + // If the initial guess was not mandatory, remove it, as it was not found. + self.initial_guess.take(); + } + + self.inner.next(reader) + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/read/stream.rs b/v2/vendor/security/zip-7.2.0/src/read/stream.rs new file mode 100644 index 0000000000..7b9dfada56 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/read/stream.rs @@ -0,0 +1,415 @@ +use crate::result::invalid; + +use super::{ + MAX_SYMLINK_TARGET_LEN, + central_header_to_zip_file_inner, make_symlink, read_zipfile_from_stream, ZipCentralEntryBlock, + ZipFile, ZipFileData, ZipResult, +}; +use crate::spec::FixedSizeBlock; +use indexmap::IndexMap; +use std::io::{self, Read}; +use std::path::{Path, PathBuf}; + +/// Stream decoder for zip. +#[derive(Debug)] +pub struct ZipStreamReader(R); + +impl ZipStreamReader { + /// Create a new ZipStreamReader + pub const fn new(reader: R) -> Self { + Self(reader) + } +} + +impl ZipStreamReader { + fn parse_central_directory(&mut self) -> ZipResult { + // Give archive_offset and central_header_start dummy value 0, since + // they are not used in the output. + let archive_offset = 0; + let central_header_start = 0; + + // Parse central header + let block = ZipCentralEntryBlock::parse(&mut self.0)?; + let file = central_header_to_zip_file_inner( + &mut self.0, + archive_offset, + central_header_start, + block, + )?; + Ok(ZipStreamFileMetadata(file)) + } + + /// Iterate over the stream and extract all file and their + /// metadata. + pub fn visit(mut self, visitor: &mut V) -> ZipResult<()> { + while let Some(mut file) = read_zipfile_from_stream(&mut self.0)? { + visitor.visit_file(&mut file)?; + } + + while let Ok(metadata) = self.parse_central_directory() { + visitor.visit_additional_metadata(&metadata)?; + } + + Ok(()) + } + + /// Extract a Zip archive into a directory, overwriting files if they + /// already exist. Paths are sanitized with [`ZipFile::enclosed_name`]. + /// + /// Extraction is not atomic; If an error is encountered, some of the files + /// may be left on disk. + pub fn extract>(self, directory: P) -> ZipResult<()> { + use std::fs; + fs::create_dir_all(&directory)?; + let directory = directory.as_ref().canonicalize()?; + struct Extractor(PathBuf, IndexMap, ()>); + impl ZipStreamVisitor for Extractor { + fn visit_file(&mut self, file: &mut ZipFile<'_, R>) -> ZipResult<()> { + self.1.insert(file.name().into(), ()); + let mut outpath = self.0.clone(); + file.safe_prepare_path(&self.0, &mut outpath, None::<&(_, fn(&Path) -> bool)>)?; + + if file.is_symlink() { + let declared_len = file.size(); + if declared_len > MAX_SYMLINK_TARGET_LEN { + return Err(invalid!( + "symlink target declares {} bytes, more than the {} byte limit", + declared_len, + MAX_SYMLINK_TARGET_LEN + )); + } + let mut target = Vec::with_capacity(declared_len as usize); + file.read_to_end(&mut target)?; + make_symlink(&outpath, &target, &self.1)?; + return Ok(()); + } + + if file.is_dir() { + fs::create_dir_all(&outpath)?; + } else { + let mut outfile = fs::File::create(&outpath)?; + io::copy(file, &mut outfile)?; + } + + Ok(()) + } + + #[allow(unused)] + fn visit_additional_metadata( + &mut self, + metadata: &ZipStreamFileMetadata, + ) -> ZipResult<()> { + #[cfg(unix)] + { + use super::ZipError; + let filepath = metadata + .enclosed_name() + .ok_or(crate::result::invalid!("Invalid file path"))?; + + let outpath = self.0.join(filepath); + + use std::os::unix::fs::PermissionsExt; + if let Some(mode) = metadata.unix_mode() { + fs::set_permissions(outpath, fs::Permissions::from_mode(mode))?; + } + } + + Ok(()) + } + } + + self.visit(&mut Extractor(directory, IndexMap::new())) + } +} + +/// Visitor for ZipStreamReader +pub trait ZipStreamVisitor { + /// * `file` - contains the content of the file and most of the metadata, + /// except: + /// - `comment`: set to an empty string + /// - `data_start`: set to 0 + /// - `external_attributes`: `unix_mode()`: will return None + fn visit_file(&mut self, file: &mut ZipFile<'_, R>) -> ZipResult<()>; + + /// This function is guaranteed to be called after all `visit_file`s. + /// + /// * `metadata` - Provides missing metadata in `visit_file`. + fn visit_additional_metadata(&mut self, metadata: &ZipStreamFileMetadata) -> ZipResult<()>; +} + +/// Additional metadata for the file. +#[derive(Debug)] +pub struct ZipStreamFileMetadata(ZipFileData); + +impl ZipStreamFileMetadata { + /// Get the name of the file + /// + /// # Warnings + /// + /// It is dangerous to use this name directly when extracting an archive. + /// It may contain an absolute path (`/etc/shadow`), or break out of the + /// current directory (`../runtime`). Carelessly writing to these paths + /// allows an attacker to craft a ZIP archive that will overwrite critical + /// files. + /// + /// You can use the [`ZipFile::enclosed_name`] method to validate the name + /// as a safe path. + pub fn name(&self) -> &str { + &self.0.file_name + } + + /// Get the name of the file, in the raw (internal) byte representation. + /// + /// The encoding of this data is currently undefined. + pub fn name_raw(&self) -> &[u8] { + &self.0.file_name_raw + } + + /// Rewrite the path, ignoring any path components with special meaning. + /// + /// - Absolute paths are made relative + /// - [std::path::Component::ParentDir]s are ignored + /// - Truncates the filename at a NULL byte + /// + /// This is appropriate if you need to be able to extract *something* from + /// any archive, but will easily misrepresent trivial paths like + /// `foo/../bar` as `foo/bar` (instead of `bar`). Because of this, + /// [`ZipFile::enclosed_name`] is the better option in most scenarios. + pub fn mangled_name(&self) -> PathBuf { + self.0.file_name_sanitized() + } + + /// Ensure the file path is safe to use as a [`Path`]. + /// + /// - It can't contain NULL bytes + /// - It can't resolve to a path outside the current directory + /// > `foo/../bar` is fine, `foo/../../bar` is not. + /// - It can't be an absolute path + /// + /// This will read well-formed ZIP files correctly, and is resistant + /// to path-based exploits. It is recommended over + /// [`ZipFile::mangled_name`]. + pub fn enclosed_name(&self) -> Option { + self.0.enclosed_name() + } + + /// Returns whether the file is actually a directory + pub fn is_dir(&self) -> bool { + self.name() + .chars() + .next_back() + .is_some_and(|c| c == '/' || c == '\\') + } + + /// Returns whether the file is a regular file + pub fn is_file(&self) -> bool { + !self.is_dir() + } + + /// Get the comment of the file + pub fn comment(&self) -> &str { + &self.0.file_comment + } + + /// Get unix mode for the file + pub const fn unix_mode(&self) -> Option { + self.0.unix_mode() + } +} + +#[cfg(test)] +mod test { + use tempfile::TempDir; + + use crate::read::stream::{ZipStreamFileMetadata, ZipStreamReader, ZipStreamVisitor}; + use crate::read::ZipFile; + use crate::result::ZipResult; + use crate::write::SimpleFileOptions; + use crate::ZipWriter; + use std::collections::BTreeSet; + use std::io::{Cursor, Read}; + + struct DummyVisitor; + impl ZipStreamVisitor for DummyVisitor { + fn visit_file(&mut self, _file: &mut ZipFile<'_, R>) -> ZipResult<()> { + Ok(()) + } + + fn visit_additional_metadata( + &mut self, + _metadata: &ZipStreamFileMetadata, + ) -> ZipResult<()> { + Ok(()) + } + } + + #[allow(dead_code)] + #[derive(Default, Debug, Eq, PartialEq)] + struct CounterVisitor(u64, u64); + impl ZipStreamVisitor for CounterVisitor { + fn visit_file(&mut self, _file: &mut ZipFile<'_, R>) -> ZipResult<()> { + self.0 += 1; + Ok(()) + } + + fn visit_additional_metadata( + &mut self, + _metadata: &ZipStreamFileMetadata, + ) -> ZipResult<()> { + self.1 += 1; + Ok(()) + } + } + + #[test] + fn invalid_offset() { + ZipStreamReader::new(Cursor::new(include_bytes!( + "../../tests/data/invalid_offset.zip" + ))) + .visit(&mut DummyVisitor) + .unwrap_err(); + } + + #[test] + fn invalid_offset2() { + ZipStreamReader::new(Cursor::new(include_bytes!( + "../../tests/data/invalid_offset2.zip" + ))) + .visit(&mut DummyVisitor) + .unwrap_err(); + } + + #[test] + fn zip_read_streaming() { + let reader = + ZipStreamReader::new(Cursor::new(include_bytes!("../../tests/data/mimetype.zip"))); + + #[derive(Default)] + struct V { + filenames: BTreeSet>, + } + impl ZipStreamVisitor for V { + fn visit_file(&mut self, file: &mut ZipFile<'_, R>) -> ZipResult<()> { + if file.is_file() { + self.filenames.insert(file.name().into()); + } + + Ok(()) + } + fn visit_additional_metadata( + &mut self, + metadata: &ZipStreamFileMetadata, + ) -> ZipResult<()> { + if metadata.is_file() { + assert!( + self.filenames.contains(metadata.name()), + "{} is missing its file content", + metadata.name() + ); + } + + Ok(()) + } + } + + reader.visit(&mut V::default()).unwrap(); + } + + #[test] + fn file_and_dir_predicates() { + let reader = ZipStreamReader::new(Cursor::new(include_bytes!( + "../../tests/data/files_and_dirs.zip" + ))); + + #[derive(Default)] + struct V { + filenames: BTreeSet>, + } + impl ZipStreamVisitor for V { + fn visit_file(&mut self, file: &mut ZipFile<'_, R>) -> ZipResult<()> { + let full_name = file.enclosed_name().unwrap(); + let file_name = full_name.file_name().unwrap().to_str().unwrap(); + assert!( + (file_name.starts_with("dir") && file.is_dir()) + || (file_name.starts_with("file") && file.is_file()) + ); + + if file.is_file() { + self.filenames.insert(file.name().into()); + } + + Ok(()) + } + fn visit_additional_metadata( + &mut self, + metadata: &ZipStreamFileMetadata, + ) -> ZipResult<()> { + if metadata.is_file() { + assert!( + self.filenames.contains(metadata.name()), + "{} is missing its file content", + metadata.name() + ); + } + + Ok(()) + } + } + + reader.visit(&mut V::default()).unwrap(); + } + + /// test case to ensure we don't preemptively over allocate based on the + /// declared number of files in the CDE of an invalid zip when the number of + /// files declared is more than the alleged offset in the CDE + #[test] + fn invalid_cde_number_of_files_allocation_smaller_offset() { + ZipStreamReader::new(Cursor::new(include_bytes!( + "../../tests/data/invalid_cde_number_of_files_allocation_smaller_offset.zip" + ))) + .visit(&mut DummyVisitor) + .unwrap_err(); + } + + /// test case to ensure we don't preemptively over allocate based on the + /// declared number of files in the CDE of an invalid zip when the number of + /// files declared is less than the alleged offset in the CDE + #[test] + fn invalid_cde_number_of_files_allocation_greater_offset() { + ZipStreamReader::new(Cursor::new(include_bytes!( + "../../tests/data/invalid_cde_number_of_files_allocation_greater_offset.zip" + ))) + .visit(&mut DummyVisitor) + .unwrap_err(); + } + + /// Symlinks being extracted shouldn't be followed out of the destination directory. + #[test] + fn test_cannot_symlink_outside_destination() -> ZipResult<()> { + use std::fs::create_dir; + + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.add_symlink("symlink/", "../dest-sibling/", SimpleFileOptions::default())?; + writer.start_file("symlink/dest-file", SimpleFileOptions::default())?; + let reader = ZipStreamReader::new(writer.finish()?); + let dest_parent = TempDir::with_prefix("stream__cannot_symlink_outside_destination")?; + let dest_sibling = dest_parent.path().join("dest-sibling"); + create_dir(&dest_sibling)?; + let dest = dest_parent.path().join("dest"); + create_dir(&dest)?; + assert!(reader.extract(dest).is_err()); + assert!(!dest_sibling.join("dest-file").exists()); + Ok(()) + } + + #[test] + fn test_can_create_destination() -> ZipResult<()> { + let mut v = Vec::new(); + v.extend_from_slice(include_bytes!("../../tests/data/mimetype.zip")); + let reader = ZipStreamReader::new(v.as_slice()); + let dest = TempDir::with_prefix("stream_test_can_create_destination").unwrap(); + reader.extract(&dest)?; + assert!(dest.path().join("mimetype").exists()); + Ok(()) + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/result.rs b/v2/vendor/security/zip-7.2.0/src/result.rs new file mode 100644 index 0000000000..2217f20eaa --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/result.rs @@ -0,0 +1,143 @@ +#![allow(unknown_lints)] // non_local_definitions isn't in Rust 1.70 +#![allow(non_local_definitions)] +//! Error types that can be emitted from this library + +use core::error::Error; +use core::fmt::{self, Display, Formatter}; +use core::num::TryFromIntError; +use std::borrow::Cow; +use std::io; + +/// Generic result type with ZipError as its error variant +pub type ZipResult = Result; + +/// Error type for Zip +#[derive(Debug)] +#[non_exhaustive] +pub enum ZipError { + /// i/o error + Io(io::Error), + + /// invalid Zip archive + InvalidArchive(Cow<'static, str>), + + /// unsupported Zip archive + UnsupportedArchive(&'static str), + + /// specified file not found in archive + FileNotFound, + + /// provided password is incorrect + InvalidPassword, +} + +impl ZipError { + /// The text used as an error when a password is required and not supplied + /// + /// ```rust,no_run + /// # use zip::result::ZipError; + /// # let mut archive = zip::ZipArchive::new(std::io::Cursor::new(&[])).unwrap(); + /// match archive.by_index(1) { + /// Err(ZipError::UnsupportedArchive(ZipError::PASSWORD_REQUIRED)) => eprintln!("a password is needed to unzip this file"), + /// _ => (), + /// } + /// # () + /// ``` + pub const PASSWORD_REQUIRED: &'static str = "Password required to decrypt file"; +} + +impl Display for ZipError { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + Self::Io(e) => write!(f, "i/o error: {e}"), + Self::InvalidArchive(e) => write!(f, "invalid Zip archive: {e}"), + Self::UnsupportedArchive(e) => write!(f, "unsupported Zip archive: {e}"), + Self::FileNotFound => f.write_str("specified file not found in archive"), + Self::InvalidPassword => f.write_str("provided password is incorrect"), + } + } +} + +impl Error for ZipError { + fn source(&self) -> Option<&(dyn Error + 'static)> { + match self { + Self::Io(e) => Some(e), + Self::InvalidArchive(_) + | Self::UnsupportedArchive(_) + | Self::FileNotFound + | Self::InvalidPassword => None, + } + } +} + +impl From for io::Error { + fn from(err: ZipError) -> io::Error { + let kind = match &err { + ZipError::Io(err) => err.kind(), + ZipError::InvalidArchive(_) => io::ErrorKind::InvalidData, + ZipError::UnsupportedArchive(_) => io::ErrorKind::Unsupported, + ZipError::FileNotFound => io::ErrorKind::NotFound, + ZipError::InvalidPassword => io::ErrorKind::InvalidInput, + }; + + io::Error::new(kind, err) + } +} + +impl From for ZipError { + fn from(value: io::Error) -> Self { + Self::Io(value) + } +} + +impl From for ZipError { + fn from(_: DateTimeRangeError) -> Self { + invalid!("Invalid date or time") + } +} + +impl From for ZipError { + fn from(_: std::string::FromUtf8Error) -> Self { + invalid!("Invalid UTF-8") + } +} + +/// Error type for time parsing +#[derive(Debug)] +pub struct DateTimeRangeError; + +// TryFromIntError is also an out-of-range error. +impl From for DateTimeRangeError { + fn from(_value: TryFromIntError) -> Self { + DateTimeRangeError + } +} + +impl fmt::Display for DateTimeRangeError { + fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result { + write!( + fmt, + "a date could not be represented within the bounds the MS-DOS date range (1980-2107)" + ) + } +} + +impl Error for DateTimeRangeError {} + +pub(crate) fn invalid_archive>>(message: M) -> ZipError { + ZipError::InvalidArchive(message.into()) +} + +pub(crate) const fn invalid_archive_const(message: &'static str) -> ZipError { + ZipError::InvalidArchive(Cow::Borrowed(message)) +} + +macro_rules! invalid { + ($message:literal) => { + crate::result::invalid_archive_const($message) + }; + ($($arg:tt)*) => { + crate::result::invalid_archive(format!($($arg)*)) + }; +} +pub(crate) use invalid; diff --git a/v2/vendor/security/zip-7.2.0/src/spec.rs b/v2/vendor/security/zip-7.2.0/src/spec.rs new file mode 100644 index 0000000000..61107d0f79 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/spec.rs @@ -0,0 +1,873 @@ +#![macro_use] + +use crate::read::magic_finder::{Backwards, Forward, MagicFinder, OptimisticMagicFinder}; +use crate::read::ArchiveOffset; +use crate::result::{invalid, ZipError, ZipResult}; +use core::any::type_name; +use core::mem; +use core::slice; +use std::io::{self, Read, Seek, Write}; + +/// "Magic" header values used in the zip spec to locate metadata records. +/// +/// These values currently always take up a fixed four bytes, so we can parse and wrap them in this +/// struct to enforce some small amount of type safety. +#[derive(Copy, Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash)] +#[repr(transparent)] +pub(crate) struct Magic(u32); + +impl Magic { + pub const fn literal(x: u32) -> Self { + Self(x) + } + + #[inline(always)] + #[allow(dead_code)] + pub const fn from_le_bytes(bytes: [u8; 4]) -> Self { + Self(u32::from_le_bytes(bytes)) + } + + #[inline(always)] + pub const fn to_le_bytes(self) -> [u8; 4] { + self.0.to_le_bytes() + } + + #[allow(clippy::wrong_self_convention)] + #[inline(always)] + pub fn from_le(self) -> Self { + Self(u32::from_le(self.0)) + } + + #[allow(clippy::wrong_self_convention)] + #[inline(always)] + pub fn to_le(self) -> Self { + Self(u32::to_le(self.0)) + } + + pub const LOCAL_FILE_HEADER_SIGNATURE: Self = Self::literal(0x04034b50); + pub const CENTRAL_DIRECTORY_HEADER_SIGNATURE: Self = Self::literal(0x02014b50); + pub const CENTRAL_DIRECTORY_END_SIGNATURE: Self = Self::literal(0x06054b50); + pub const ZIP64_CENTRAL_DIRECTORY_END_SIGNATURE: Self = Self::literal(0x06064b50); + pub const ZIP64_CENTRAL_DIRECTORY_END_LOCATOR_SIGNATURE: Self = Self::literal(0x07064b50); + pub const DATA_DESCRIPTOR_SIGNATURE: Self = Self::literal(0x08074b50); +} + +/// Similar to [`Magic`], but used for extra field tags as per section 4.5.3 of APPNOTE.TXT. +#[derive(Copy, Clone, Debug, PartialOrd, Ord, PartialEq, Eq, Hash)] +#[repr(transparent)] +pub(crate) struct ExtraFieldMagic(u16); + +/* TODO: maybe try to use this for parsing extra fields as well as writing them? */ +#[allow(dead_code)] +impl ExtraFieldMagic { + pub const fn literal(x: u16) -> Self { + Self(x) + } + + #[inline(always)] + pub const fn from_le_bytes(bytes: [u8; 2]) -> Self { + Self(u16::from_le_bytes(bytes)) + } + + #[inline(always)] + pub const fn to_le_bytes(self) -> [u8; 2] { + self.0.to_le_bytes() + } + + #[allow(clippy::wrong_self_convention)] + #[inline(always)] + pub fn from_le(self) -> Self { + Self(u16::from_le(self.0)) + } + + #[allow(clippy::wrong_self_convention)] + #[inline(always)] + pub fn to_le(self) -> Self { + Self(u16::to_le(self.0)) + } + + pub const ZIP64_EXTRA_FIELD_TAG: Self = Self::literal(0x0001); +} + +/// The file size at which a ZIP64 record becomes necessary. +/// +/// If a file larger than this threshold attempts to be written, compressed or uncompressed, and +/// [`FileOptions::large_file()`](crate::write::FileOptions::large_file) was not true, then [`crate::ZipWriter`] will +/// raise an [`io::Error`] with [`io::ErrorKind::Other`]. +/// +/// If the zip file itself is larger than this value, then a zip64 central directory record will be +/// written to the end of the file. +/// +///``` +/// # fn main() -> Result<(), zip::result::ZipError> { +/// # #[cfg(target_pointer_width = "64")] +/// # { +/// use std::io::{self, Cursor, Write}; +/// use std::error::Error; +/// use zip::{ZipWriter, write::SimpleFileOptions}; +/// +/// let mut zip = ZipWriter::new(Cursor::new(Vec::new())); +/// // Writing an extremely large file for this test is faster without compression. +/// +/// let big_len: usize = (zip::ZIP64_BYTES_THR as usize) + 1; +/// let big_buf = vec![0u8; big_len]; +/// { +/// let options = SimpleFileOptions::default() +/// .compression_method(zip::CompressionMethod::Stored); +/// zip.start_file("zero.dat", options)?; +/// // This is too big! +/// let res = zip.write_all(&big_buf[..]).err().unwrap(); +/// assert_eq!(res.kind(), io::ErrorKind::Other); +/// let description = format!("{}", &res); +/// assert_eq!(description, "Large file option has not been set"); +/// // Attempting to write anything further to the same zip will still succeed, but the previous +/// // failing entry has been removed. +/// zip.start_file("one.dat", options)?; +/// let zip = zip.finish_into_readable()?; +/// let names: Vec<_> = zip.file_names().collect(); +/// assert_eq!(&names, &["one.dat"]); +/// } +/// +/// // Create a new zip output. +/// let mut zip = ZipWriter::new(Cursor::new(Vec::new())); +/// // This time, create a zip64 record for the file. +/// let options = SimpleFileOptions::default() +/// .compression_method(zip::CompressionMethod::Stored) +/// .large_file(true); +/// zip.start_file("zero.dat", options)?; +/// // This succeeds because we specified that it could be a large file. +/// assert!(zip.write_all(&big_buf[..]).is_ok()); +/// # } +/// # Ok(()) +/// # } +///``` +pub const ZIP64_BYTES_THR: u64 = u32::MAX as u64; +/// The number of entries within a single zip necessary to allocate a zip64 central +/// directory record. +/// +/// If more than this number of entries is written to a [`crate::ZipWriter`], then [`crate::ZipWriter::finish()`] +/// will write out extra zip64 data to the end of the zip file. +pub const ZIP64_ENTRY_THR: usize = u16::MAX as usize; + +/// # Safety +/// +/// - No padding/uninit bytes +/// - All bytes patterns must be valid +/// - No cell, pointers +/// +/// See `bytemuck::Pod` for more details. +pub(crate) unsafe trait Pod: Copy + 'static { + #[inline] + fn zeroed() -> Self { + unsafe { mem::zeroed() } + } + + #[inline] + fn as_bytes(&self) -> &[u8] { + unsafe { slice::from_raw_parts(self as *const Self as *const u8, mem::size_of::()) } + } + + #[inline] + fn as_bytes_mut(&mut self) -> &mut [u8] { + unsafe { slice::from_raw_parts_mut(self as *mut Self as *mut u8, mem::size_of::()) } + } +} + +pub(crate) trait FixedSizeBlock: Pod { + const MAGIC: Magic; + + fn magic(self) -> Magic; + + const WRONG_MAGIC_ERROR: ZipError; + + #[allow(clippy::wrong_self_convention)] + fn from_le(self) -> Self; + + fn parse(reader: &mut R) -> ZipResult { + let mut block = Self::zeroed(); + if let Err(e) = reader.read_exact(block.as_bytes_mut()) { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!("Unexpected end of {}", type_name::())); + } + return Err(e.into()); + } + let block = Self::from_le(block); + + if block.magic() != Self::MAGIC { + return Err(Self::WRONG_MAGIC_ERROR); + } + Ok(block) + } + + fn to_le(self) -> Self; + + fn write(self, writer: &mut T) -> ZipResult<()> { + let block = self.to_le(); + writer.write_all(block.as_bytes())?; + Ok(()) + } +} + +/// Convert all the fields of a struct *from* little-endian representations. +macro_rules! from_le { + ($obj:ident, $field:ident, $type:ty) => { + $obj.$field = <$type>::from_le($obj.$field); + }; + ($obj:ident, [($field:ident, $type:ty) $(,)?]) => { + from_le![$obj, $field, $type]; + }; + ($obj:ident, [($field:ident, $type:ty), $($rest:tt),+ $(,)?]) => { + from_le![$obj, $field, $type]; + from_le!($obj, [$($rest),+]); + }; +} + +/// Convert all the fields of a struct *into* little-endian representations. +macro_rules! to_le { + ($obj:ident, $field:ident, $type:ty) => { + $obj.$field = <$type>::to_le($obj.$field); + }; + ($obj:ident, [($field:ident, $type:ty) $(,)?]) => { + to_le![$obj, $field, $type]; + }; + ($obj:ident, [($field:ident, $type:ty), $($rest:tt),+ $(,)?]) => { + to_le![$obj, $field, $type]; + to_le!($obj, [$($rest),+]); + }; +} + +/* TODO: derive macro to generate these fields? */ +/// Implement `from_le()` and `to_le()`, providing the field specification to both macros +/// and methods. +macro_rules! to_and_from_le { + ($($args:tt),+ $(,)?) => { + #[inline(always)] + fn from_le(mut self) -> Self { + from_le![self, [$($args),+]]; + self + } + #[inline(always)] + fn to_le(mut self) -> Self { + to_le![self, [$($args),+]]; + self + } + }; +} + +#[derive(Copy, Clone, Debug)] +#[repr(packed, C)] +pub(crate) struct Zip32CDEBlock { + magic: Magic, + pub disk_number: u16, + pub disk_with_central_directory: u16, + pub number_of_files_on_this_disk: u16, + pub number_of_files: u16, + pub central_directory_size: u32, + pub central_directory_offset: u32, + pub zip_file_comment_length: u16, +} + +unsafe impl Pod for Zip32CDEBlock {} + +impl FixedSizeBlock for Zip32CDEBlock { + const MAGIC: Magic = Magic::CENTRAL_DIRECTORY_END_SIGNATURE; + + #[inline(always)] + fn magic(self) -> Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("Invalid digital signature header"); + + to_and_from_le![ + (magic, Magic), + (disk_number, u16), + (disk_with_central_directory, u16), + (number_of_files_on_this_disk, u16), + (number_of_files, u16), + (central_directory_size, u32), + (central_directory_offset, u32), + (zip_file_comment_length, u16) + ]; +} + +#[derive(Debug)] +pub(crate) struct Zip32CentralDirectoryEnd { + pub disk_number: u16, + pub disk_with_central_directory: u16, + pub number_of_files_on_this_disk: u16, + pub number_of_files: u16, + pub central_directory_size: u32, + pub central_directory_offset: u32, + pub zip_file_comment: Box<[u8]>, +} + +impl Zip32CentralDirectoryEnd { + fn into_block_and_comment(self) -> (Zip32CDEBlock, Box<[u8]>) { + let Self { + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + zip_file_comment, + } = self; + let block = Zip32CDEBlock { + magic: Zip32CDEBlock::MAGIC, + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + zip_file_comment_length: zip_file_comment.len() as u16, + }; + + (block, zip_file_comment) + } + + pub fn parse(reader: &mut T) -> ZipResult { + let Zip32CDEBlock { + // magic, + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + zip_file_comment_length, + .. + } = Zip32CDEBlock::parse(reader)?; + + let mut zip_file_comment = vec![0u8; zip_file_comment_length as usize].into_boxed_slice(); + if let Err(e) = reader.read_exact(&mut zip_file_comment) { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!("EOCD comment exceeds file boundary")); + } + + return Err(e.into()); + } + + Ok(Zip32CentralDirectoryEnd { + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + zip_file_comment, + }) + } + + pub fn write(self, writer: &mut T) -> ZipResult<()> { + let (block, comment) = self.into_block_and_comment(); + + if comment.len() > u16::MAX as usize { + return Err(invalid!("EOCD comment length exceeds u16::MAX")); + } + + block.write(writer)?; + writer.write_all(&comment)?; + Ok(()) + } + + pub fn may_be_zip64(&self) -> bool { + self.number_of_files == u16::MAX || self.central_directory_offset == u32::MAX + } +} + +#[derive(Copy, Clone)] +#[repr(packed, C)] +pub(crate) struct Zip64CDELocatorBlock { + magic: Magic, + pub disk_with_central_directory: u32, + pub end_of_central_directory_offset: u64, + pub number_of_disks: u32, +} + +unsafe impl Pod for Zip64CDELocatorBlock {} + +impl FixedSizeBlock for Zip64CDELocatorBlock { + const MAGIC: Magic = Magic::ZIP64_CENTRAL_DIRECTORY_END_LOCATOR_SIGNATURE; + + #[inline(always)] + fn magic(self) -> Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("Invalid zip64 locator digital signature header"); + + to_and_from_le![ + (magic, Magic), + (disk_with_central_directory, u32), + (end_of_central_directory_offset, u64), + (number_of_disks, u32), + ]; +} + +pub(crate) struct Zip64CentralDirectoryEndLocator { + pub disk_with_central_directory: u32, + pub end_of_central_directory_offset: u64, + pub number_of_disks: u32, +} + +impl Zip64CentralDirectoryEndLocator { + pub fn parse(reader: &mut T) -> ZipResult { + let Zip64CDELocatorBlock { + // magic, + disk_with_central_directory, + end_of_central_directory_offset, + number_of_disks, + .. + } = Zip64CDELocatorBlock::parse(reader)?; + + Ok(Zip64CentralDirectoryEndLocator { + disk_with_central_directory, + end_of_central_directory_offset, + number_of_disks, + }) + } + + pub fn block(self) -> Zip64CDELocatorBlock { + let Self { + disk_with_central_directory, + end_of_central_directory_offset, + number_of_disks, + } = self; + Zip64CDELocatorBlock { + magic: Zip64CDELocatorBlock::MAGIC, + disk_with_central_directory, + end_of_central_directory_offset, + number_of_disks, + } + } + + pub fn write(self, writer: &mut T) -> ZipResult<()> { + self.block().write(writer) + } +} + +#[derive(Copy, Clone)] +#[repr(packed, C)] +pub(crate) struct Zip64CDEBlock { + magic: Magic, + pub record_size: u64, + pub version_made_by: u16, + pub version_needed_to_extract: u16, + pub disk_number: u32, + pub disk_with_central_directory: u32, + pub number_of_files_on_this_disk: u64, + pub number_of_files: u64, + pub central_directory_size: u64, + pub central_directory_offset: u64, +} + +unsafe impl Pod for Zip64CDEBlock {} + +impl FixedSizeBlock for Zip64CDEBlock { + const MAGIC: Magic = Magic::ZIP64_CENTRAL_DIRECTORY_END_SIGNATURE; + + fn magic(self) -> Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("Invalid digital signature header"); + + to_and_from_le![ + (magic, Magic), + (record_size, u64), + (version_made_by, u16), + (version_needed_to_extract, u16), + (disk_number, u32), + (disk_with_central_directory, u32), + (number_of_files_on_this_disk, u64), + (number_of_files, u64), + (central_directory_size, u64), + (central_directory_offset, u64), + ]; +} + +pub(crate) struct Zip64CentralDirectoryEnd { + pub record_size: u64, + pub version_made_by: u16, + pub version_needed_to_extract: u16, + pub disk_number: u32, + pub disk_with_central_directory: u32, + pub number_of_files_on_this_disk: u64, + pub number_of_files: u64, + pub central_directory_size: u64, + pub central_directory_offset: u64, + pub extensible_data_sector: Box<[u8]>, +} + +impl Zip64CentralDirectoryEnd { + pub fn parse( + reader: &mut T, + max_size: u64, + ) -> ZipResult { + let Zip64CDEBlock { + record_size, + version_made_by, + version_needed_to_extract, + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + .. + } = Zip64CDEBlock::parse(reader)?; + + if record_size < 44 { + return Err(invalid!("Low EOCD64 record size")); + } else if record_size.saturating_add(12) > max_size { + return Err(invalid!("EOCD64 extends beyond EOCD64 locator")); + } + + let mut zip_file_comment = vec![0u8; record_size as usize - 44].into_boxed_slice(); + if let Err(e) = reader.read_exact(&mut zip_file_comment) { + if e.kind() == io::ErrorKind::UnexpectedEof { + return Err(invalid!( + "EOCD64 extensible data sector exceeds file boundary" + )); + } + return Err(e.into()); + } + + Ok(Self { + record_size, + version_made_by, + version_needed_to_extract, + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + extensible_data_sector: zip_file_comment, + }) + } + + pub fn into_block_and_comment(self) -> (Zip64CDEBlock, Box<[u8]>) { + let Self { + record_size, + version_made_by, + version_needed_to_extract, + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + extensible_data_sector, + } = self; + + ( + Zip64CDEBlock { + magic: Zip64CDEBlock::MAGIC, + record_size, + version_made_by, + version_needed_to_extract, + disk_number, + disk_with_central_directory, + number_of_files_on_this_disk, + number_of_files, + central_directory_size, + central_directory_offset, + }, + extensible_data_sector, + ) + } + + pub fn write(self, writer: &mut T) -> ZipResult<()> { + let (block, comment) = self.into_block_and_comment(); + block.write(writer)?; + writer.write_all(&comment)?; + Ok(()) + } +} + +pub(crate) struct DataAndPosition { + pub data: T, + #[allow(dead_code)] + pub position: u64, +} + +impl From<(T, u64)> for DataAndPosition { + fn from(value: (T, u64)) -> Self { + Self { + data: value.0, + position: value.1, + } + } +} + +pub(crate) struct CentralDirectoryEndInfo { + pub eocd: DataAndPosition, + pub eocd64: Option>, + + pub archive_offset: u64, +} + +/// Finds the EOCD and possibly the EOCD64 block and determines the archive offset. +/// +/// In the best case scenario (no prepended junk), this function will not backtrack +/// in the reader. +pub(crate) fn find_central_directory( + reader: &mut R, + archive_offset: ArchiveOffset, + end_exclusive: u64, + file_len: u64, +) -> ZipResult { + const EOCD_SIG_BYTES: [u8; mem::size_of::()] = + Magic::CENTRAL_DIRECTORY_END_SIGNATURE.to_le_bytes(); + + const EOCD64_SIG_BYTES: [u8; mem::size_of::()] = + Magic::ZIP64_CENTRAL_DIRECTORY_END_SIGNATURE.to_le_bytes(); + + const CDFH_SIG_BYTES: [u8; mem::size_of::()] = + Magic::CENTRAL_DIRECTORY_HEADER_SIGNATURE.to_le_bytes(); + + // Instantiate the mandatory finder + let mut eocd_finder = MagicFinder::>::new(&EOCD_SIG_BYTES, 0, end_exclusive); + let mut subfinder: Option>> = None; + + // Keep the last errors for cases of improper EOCD instances. + let mut parsing_error = None; + + while let Some(eocd_offset) = eocd_finder.next(reader)? { + // Attempt to parse the EOCD block + let eocd = match Zip32CentralDirectoryEnd::parse(reader) { + Ok(eocd) => eocd, + Err(e) => { + if parsing_error.is_none() { + parsing_error = Some(e); + } + continue; + } + }; + + // ! Relaxed (inequality) due to garbage-after-comment Python files + // Consistency check: the EOCD comment must terminate before the end of file + if eocd.zip_file_comment.len() as u64 + eocd_offset + 22 > file_len { + parsing_error = Some(invalid!("Invalid EOCD comment length")); + continue; + } + + let zip64_metadata = if eocd.may_be_zip64() { + fn try_read_eocd64_locator( + reader: &mut (impl Read + Seek + ?Sized), + eocd_offset: u64, + ) -> ZipResult<(u64, Zip64CentralDirectoryEndLocator)> { + if eocd_offset < mem::size_of::() as u64 { + return Err(invalid!("EOCD64 Locator does not fit in file")); + } + + let locator64_offset = eocd_offset - mem::size_of::() as u64; + + reader.seek(io::SeekFrom::Start(locator64_offset))?; + let locator64 = Zip64CentralDirectoryEndLocator::parse(reader); + Ok((locator64_offset, locator64?)) + } + + try_read_eocd64_locator(reader, eocd_offset).ok() + } else { + None + }; + + let Some((locator64_offset, locator64)) = zip64_metadata else { + // Branch out for zip32 + let relative_cd_offset = eocd.central_directory_offset as u64; + + // If the archive is empty, there is nothing more to be checked, the archive is correct. + if eocd.number_of_files == 0 { + return Ok(CentralDirectoryEndInfo { + eocd: (eocd, eocd_offset).into(), + eocd64: None, + archive_offset: eocd_offset.saturating_sub(relative_cd_offset), + }); + } + + // Consistency check: the CD relative offset cannot be after the EOCD + if relative_cd_offset >= eocd_offset { + parsing_error = Some(invalid!("Invalid CDFH offset in EOCD")); + continue; + } + + // Attempt to find the first CDFH + let subfinder = subfinder + .get_or_insert_with(OptimisticMagicFinder::new_empty) + .repurpose( + &CDFH_SIG_BYTES, + // The CDFH must be before the EOCD and after the relative offset, + // because prepended junk can only move it forward. + (relative_cd_offset, eocd_offset), + match archive_offset { + ArchiveOffset::Known(n) => { + Some((relative_cd_offset.saturating_add(n).min(eocd_offset), true)) + } + _ => Some((relative_cd_offset, false)), + }, + ); + + // Consistency check: find the first CDFH + if let Some(cd_offset) = subfinder.next(reader)? { + // The first CDFH will define the archive offset + let archive_offset = cd_offset - relative_cd_offset; + + return Ok(CentralDirectoryEndInfo { + eocd: (eocd, eocd_offset).into(), + eocd64: None, + archive_offset, + }); + } + + parsing_error = Some(invalid!("No CDFH found")); + continue; + }; + + // Consistency check: the EOCD64 offset must be before EOCD64 Locator offset */ + if locator64.end_of_central_directory_offset >= locator64_offset { + parsing_error = Some(invalid!("Invalid EOCD64 Locator CD offset")); + continue; + } + + if locator64.number_of_disks > 1 { + parsing_error = Some(invalid!("Multi-disk ZIP files are not supported")); + continue; + } + + // This was hidden inside a function to collect errors in a single place. + // Once try blocks are stabilized, this can go away. + fn try_read_eocd64( + reader: &mut R, + locator64: &Zip64CentralDirectoryEndLocator, + expected_length: u64, + ) -> ZipResult { + let z64 = Zip64CentralDirectoryEnd::parse(reader, expected_length)?; + + // Consistency check: EOCD64 locator should agree with the EOCD64 + if z64.disk_with_central_directory != locator64.disk_with_central_directory { + return Err(invalid!("Invalid EOCD64: inconsistency with Locator data")); + } + + // Consistency check: the EOCD64 must have the expected length + if z64.record_size + 12 != expected_length { + return Err(invalid!("Invalid EOCD64: inconsistent length")); + } + + Ok(z64) + } + + // Attempt to find the EOCD64 with an initial guess + let subfinder = subfinder + .get_or_insert_with(OptimisticMagicFinder::new_empty) + .repurpose( + &EOCD64_SIG_BYTES, + (locator64.end_of_central_directory_offset, locator64_offset), + match archive_offset { + ArchiveOffset::Known(n) => Some(( + locator64 + .end_of_central_directory_offset + .saturating_add(n) + .min(locator64_offset), + true, + )), + _ => Some((locator64.end_of_central_directory_offset, false)), + }, + ); + + // Consistency check: Find the EOCD64 + let mut local_error = None; + while let Some(eocd64_offset) = subfinder.next(reader)? { + let archive_offset = eocd64_offset - locator64.end_of_central_directory_offset; + + match try_read_eocd64( + reader, + &locator64, + locator64_offset.saturating_sub(eocd64_offset), + ) { + Ok(eocd64) => { + if eocd64_offset + < eocd64 + .number_of_files + .saturating_mul( + mem::size_of::() as u64 + ) + .saturating_add(eocd64.central_directory_offset) + { + local_error = + Some(invalid!("Invalid EOCD64: inconsistent number of files")); + continue; + } + + return Ok(CentralDirectoryEndInfo { + eocd: (eocd, eocd_offset).into(), + eocd64: Some((eocd64, eocd64_offset).into()), + archive_offset, + }); + } + Err(e) => { + local_error = Some(e); + } + } + } + + parsing_error = local_error.or(Some(invalid!("Could not find EOCD64"))); + } + + Err(parsing_error.unwrap_or(invalid!("Could not find EOCD"))) +} + +pub(crate) fn is_dir(filename: &str) -> bool { + filename + .chars() + .next_back() + .is_some_and(|c| c == '/' || c == '\\') +} + +#[cfg(test)] +mod test { + use std::io::Cursor; + + use crate::{ + result::{invalid, ZipError}, + spec::{FixedSizeBlock, Magic, Pod}, + }; + + #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] + #[repr(packed, C)] + pub struct TestBlock { + magic: Magic, + pub file_name_length: u16, + } + + unsafe impl Pod for TestBlock {} + + impl FixedSizeBlock for TestBlock { + const MAGIC: Magic = Magic::literal(0x01111); + + fn magic(self) -> Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("unreachable"); + + to_and_from_le![(magic, Magic), (file_name_length, u16)]; + } + + /// Demonstrate that a block object can be safely written to memory and deserialized back out. + #[test] + fn block_serde() { + let block = TestBlock { + magic: TestBlock::MAGIC, + file_name_length: 3, + }; + let mut c = Cursor::new(Vec::new()); + block.write(&mut c).unwrap(); + c.set_position(0); + let block2 = TestBlock::parse(&mut c).unwrap(); + assert_eq!(block, block2); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/types.rs b/v2/vendor/security/zip-7.2.0/src/types.rs new file mode 100644 index 0000000000..cb906c1175 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/types.rs @@ -0,0 +1,1768 @@ +//! Types that specify what is contained in a ZIP. +use crate::cfg_if_expr; +use crate::cp437::FromCp437; +use crate::write::FileOptionExtension; +use crate::zipcrypto::EncryptWith; +use core::fmt::{self, Debug, Formatter}; +use core::mem; +use std::ffi::OsStr; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, OnceLock}; +use typed_path::{Utf8WindowsComponent, Utf8WindowsPath}; + +#[cfg(feature = "chrono")] +use chrono::{Datelike, NaiveDate, NaiveDateTime, NaiveTime, Timelike}; +#[cfg(feature = "jiff-02")] +use jiff::civil; + +use crate::result::{invalid, ZipError, ZipResult}; +use crate::spec::{self, FixedSizeBlock, Pod}; + +pub(crate) mod ffi { + pub const S_IFDIR: u32 = 0o0040000; + pub const S_IFREG: u32 = 0o0100000; + pub const S_IFLNK: u32 = 0o0120000; +} + +use crate::extra_fields::ExtraField; +use crate::read::find_data_start; +use crate::result::DateTimeRangeError; +use crate::spec::is_dir; +use crate::types::ffi::S_IFDIR; +use crate::{CompressionMethod, ZIP64_BYTES_THR}; +use std::io::{Read, Seek}; +#[cfg(feature = "time")] +use time::{error::ComponentRange, Date, Month, OffsetDateTime, PrimitiveDateTime, Time}; + +pub(crate) struct ZipRawValues { + pub(crate) crc32: u32, + pub(crate) compressed_size: u64, + pub(crate) uncompressed_size: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +#[repr(u8)] +pub enum System { + Dos = 0, + Unix = 3, + #[default] + Unknown, +} + +impl From for System { + fn from(system: u8) -> Self { + match system { + 0 => Self::Dos, + 3 => Self::Unix, + _ => Self::Unknown, + } + } +} + +impl From for u8 { + fn from(system: System) -> Self { + match system { + System::Dos => 0, + System::Unix => 3, + System::Unknown => 4, + } + } +} + +/// Metadata for a file to be written +#[derive(Clone, Debug, Copy, Eq, PartialEq)] +pub struct FileOptions<'k, T: FileOptionExtension> { + pub(crate) compression_method: CompressionMethod, + pub(crate) compression_level: Option, + pub(crate) last_modified_time: DateTime, + pub(crate) permissions: Option, + pub(crate) large_file: bool, + pub(crate) encrypt_with: Option>, + pub(crate) extended_options: T, + pub(crate) alignment: u16, + #[cfg(feature = "deflate-zopfli")] + pub(super) zopfli_buffer_size: Option, + #[cfg(feature = "aes-crypto")] + pub(crate) aes_mode: Option<(AesMode, AesVendorVersion, CompressionMethod)>, +} +/// Simple File Options. Can be copied and good for simple writing zip files +pub type SimpleFileOptions = FileOptions<'static, ()>; + +/// Representation of a moment in time. +/// +/// Zip files use an old format from DOS to store timestamps, +/// with its own set of peculiarities. +/// For example, it has a resolution of 2 seconds! +/// +/// A [`DateTime`] can be stored directly in a zipfile with [`FileOptions::last_modified_time`], +/// or read from one with [`ZipFile::last_modified`](crate::read::ZipFile::last_modified). +/// +/// # Warning +/// +/// Because there is no timezone associated with the [`DateTime`], they should ideally only +/// be used for user-facing descriptions. +/// +/// Modern zip files store more precise timestamps; see [`crate::extra_fields::ExtendedTimestamp`] +/// for details. +#[derive(Clone, Copy, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct DateTime { + datepart: u16, + timepart: u16, +} + +impl Debug for DateTime { + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + if *self == Self::default() { + return f.write_str("DateTime::default()"); + } + f.write_fmt(format_args!( + "DateTime::from_date_and_time({}, {}, {}, {}, {}, {})?", + self.year(), + self.month(), + self.day(), + self.hour(), + self.minute(), + self.second() + )) + } +} + +impl DateTime { + /// Constructs a default datetime of 1980-01-01 00:00:00. + pub const DEFAULT: Self = DateTime { + datepart: 0b0000000000100001, + timepart: 0, + }; + + /// Returns the current time if possible, otherwise the default of 1980-01-01. + #[cfg(feature = "time")] + pub fn default_for_write() -> Self { + let now = OffsetDateTime::now_utc(); + PrimitiveDateTime::new(now.date(), now.time()) + .try_into() + .unwrap_or_else(|_| DateTime::default()) + } + + /// Returns the current time if possible, otherwise the default of 1980-01-01. + #[cfg(not(feature = "time"))] + pub fn default_for_write() -> Self { + DateTime::default() + } +} + +#[cfg(feature = "_arbitrary")] +impl arbitrary::Arbitrary<'_> for DateTime { + fn arbitrary(u: &mut arbitrary::Unstructured) -> arbitrary::Result { + let year: u16 = u.int_in_range(1980..=2107)?; + let month: u16 = u.int_in_range(1..=12)?; + let day: u16 = u.int_in_range(1..=31)?; + let datepart = day | (month << 5) | ((year - 1980) << 9); + let hour: u16 = u.int_in_range(0..=23)?; + let minute: u16 = u.int_in_range(0..=59)?; + let second: u16 = u.int_in_range(0..=58)?; + let timepart = (second >> 1) | (minute << 5) | (hour << 11); + Ok(DateTime { datepart, timepart }) + } +} + +#[cfg(feature = "chrono")] +impl TryFrom for DateTime { + type Error = DateTimeRangeError; + + fn try_from(value: NaiveDateTime) -> Result { + DateTime::from_date_and_time( + value.year().try_into()?, + value.month().try_into()?, + value.day().try_into()?, + value.hour().try_into()?, + value.minute().try_into()?, + value.second().try_into()?, + ) + } +} + +#[cfg(feature = "chrono")] +impl TryFrom for NaiveDateTime { + type Error = DateTimeRangeError; + + fn try_from(value: DateTime) -> Result { + let date = NaiveDate::from_ymd_opt( + value.year().into(), + value.month().into(), + value.day().into(), + ) + .ok_or(DateTimeRangeError)?; + let time = NaiveTime::from_hms_opt( + value.hour().into(), + value.minute().into(), + value.second().into(), + ) + .ok_or(DateTimeRangeError)?; + Ok(NaiveDateTime::new(date, time)) + } +} + +#[cfg(feature = "jiff-02")] +impl TryFrom for DateTime { + type Error = DateTimeRangeError; + + fn try_from(value: civil::DateTime) -> Result { + Self::from_date_and_time( + value.year().try_into()?, + value.month() as u8, + value.day() as u8, + value.hour() as u8, + value.minute() as u8, + value.second() as u8, + ) + } +} + +#[cfg(feature = "jiff-02")] +impl TryFrom for civil::DateTime { + type Error = jiff::Error; + + fn try_from(value: DateTime) -> Result { + Self::new( + value.year() as i16, + value.month() as i8, + value.day() as i8, + value.hour() as i8, + value.minute() as i8, + value.second() as i8, + 0, + ) + } +} + +impl TryFrom<(u16, u16)> for DateTime { + type Error = DateTimeRangeError; + + #[inline] + fn try_from(values: (u16, u16)) -> Result { + Self::try_from_msdos(values.0, values.1) + } +} + +impl From for (u16, u16) { + #[inline] + fn from(dt: DateTime) -> Self { + (dt.datepart(), dt.timepart()) + } +} + +impl Default for DateTime { + /// Constructs an 'default' datetime of 1980-01-01 00:00:00 + fn default() -> DateTime { + DateTime::DEFAULT + } +} + +impl fmt::Display for DateTime { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{:04}-{:02}-{:02} {:02}:{:02}:{:02}", + self.year(), + self.month(), + self.day(), + self.hour(), + self.minute(), + self.second() + ) + } +} + +impl DateTime { + /// Converts an msdos (u16, u16) pair to a DateTime object + /// + /// # Safety + /// The caller must ensure the date and time are valid. + pub const unsafe fn from_msdos_unchecked(datepart: u16, timepart: u16) -> DateTime { + DateTime { datepart, timepart } + } + + /// Converts an msdos (u16, u16) pair to a DateTime object if it represents a valid date and + /// time. + pub fn try_from_msdos(datepart: u16, timepart: u16) -> Result { + let seconds = (timepart & 0b0000000000011111) << 1; + let minutes = (timepart & 0b0000011111100000) >> 5; + let hours = (timepart & 0b1111100000000000) >> 11; + let days = datepart & 0b0000000000011111; + let months = (datepart & 0b0000000111100000) >> 5; + let years = (datepart & 0b1111111000000000) >> 9; + Self::from_date_and_time( + years.checked_add(1980).ok_or(DateTimeRangeError)?, + months.try_into()?, + days.try_into()?, + hours.try_into()?, + minutes.try_into()?, + seconds.try_into()?, + ) + } + + /// Constructs a DateTime from a specific date and time + /// + /// The bounds are: + /// * year: [1980, 2107] + /// * month: [1, 12] + /// * day: [1, 28..=31] + /// * hour: [0, 23] + /// * minute: [0, 59] + /// * second: [0, 58] + pub fn from_date_and_time( + year: u16, + month: u8, + day: u8, + hour: u8, + minute: u8, + second: u8, + ) -> Result { + fn is_leap_year(year: u16) -> bool { + (year % 4 == 0) && ((year % 25 != 0) || (year % 16 == 0)) + } + + if (1980..=2107).contains(&year) + && (1..=12).contains(&month) + && (1..=31).contains(&day) + && hour <= 23 + && minute <= 59 + && second <= 60 + { + let second = second.min(58); // exFAT can't store leap seconds + let max_day = match month { + 1 | 3 | 5 | 7 | 8 | 10 | 12 => 31, + 4 | 6 | 9 | 11 => 30, + 2 if is_leap_year(year) => 29, + 2 => 28, + _ => unreachable!(), + }; + if day > max_day { + return Err(DateTimeRangeError); + } + let datepart = (day as u16) | ((month as u16) << 5) | ((year - 1980) << 9); + let timepart = ((second as u16) >> 1) | ((minute as u16) << 5) | ((hour as u16) << 11); + Ok(DateTime { datepart, timepart }) + } else { + Err(DateTimeRangeError) + } + } + + /// Indicates whether this date and time can be written to a zip archive. + pub fn is_valid(&self) -> bool { + Self::try_from_msdos(self.datepart, self.timepart).is_ok() + } + + #[cfg(feature = "time")] + /// Converts a OffsetDateTime object to a DateTime + /// + /// Returns `Err` when this object is out of bounds + #[deprecated(since = "0.6.4", note = "use `DateTime::try_from()` instead")] + pub fn from_time(dt: OffsetDateTime) -> Result { + dt.try_into() + } + + /// Gets the time portion of this datetime in the msdos representation + pub const fn timepart(&self) -> u16 { + self.timepart + } + + /// Gets the date portion of this datetime in the msdos representation + pub const fn datepart(&self) -> u16 { + self.datepart + } + + #[cfg(feature = "time")] + /// Converts the DateTime to a OffsetDateTime structure + #[deprecated(since = "1.3.1", note = "use `OffsetDateTime::try_from()` instead")] + pub fn to_time(&self) -> Result { + (*self).try_into() + } + + /// Get the year. There is no epoch, i.e. 2018 will be returned as 2018. + pub const fn year(&self) -> u16 { + (self.datepart >> 9) + 1980 + } + + /// Get the month, where 1 = january and 12 = december + /// + /// # Warning + /// + /// When read from a zip file, this may not be a reasonable value + pub const fn month(&self) -> u8 { + ((self.datepart & 0b0000000111100000) >> 5) as u8 + } + + /// Get the day + /// + /// # Warning + /// + /// When read from a zip file, this may not be a reasonable value + pub const fn day(&self) -> u8 { + (self.datepart & 0b0000000000011111) as u8 + } + + /// Get the hour + /// + /// # Warning + /// + /// When read from a zip file, this may not be a reasonable value + pub const fn hour(&self) -> u8 { + (self.timepart >> 11) as u8 + } + + /// Get the minute + /// + /// # Warning + /// + /// When read from a zip file, this may not be a reasonable value + pub const fn minute(&self) -> u8 { + ((self.timepart & 0b0000011111100000) >> 5) as u8 + } + + /// Get the second + /// + /// # Warning + /// + /// When read from a zip file, this may not be a reasonable value + pub const fn second(&self) -> u8 { + ((self.timepart & 0b0000000000011111) << 1) as u8 + } +} + +#[cfg(feature = "time")] +impl TryFrom for DateTime { + type Error = DateTimeRangeError; + + fn try_from(dt: OffsetDateTime) -> Result { + Self::try_from(PrimitiveDateTime::new(dt.date(), dt.time())) + } +} + +#[cfg(feature = "time")] +impl TryFrom for DateTime { + type Error = DateTimeRangeError; + + fn try_from(dt: PrimitiveDateTime) -> Result { + Self::from_date_and_time( + dt.year().try_into()?, + dt.month().into(), + dt.day(), + dt.hour(), + dt.minute(), + dt.second(), + ) + } +} + +#[cfg(feature = "time")] +impl TryFrom for OffsetDateTime { + type Error = ComponentRange; + + fn try_from(dt: DateTime) -> Result { + PrimitiveDateTime::try_from(dt).map(PrimitiveDateTime::assume_utc) + } +} + +#[cfg(feature = "time")] +impl TryFrom for PrimitiveDateTime { + type Error = ComponentRange; + + fn try_from(dt: DateTime) -> Result { + let date = + Date::from_calendar_date(dt.year() as i32, Month::try_from(dt.month())?, dt.day())?; + let time = Time::from_hms(dt.hour(), dt.minute(), dt.second())?; + Ok(PrimitiveDateTime::new(date, time)) + } +} + +pub const MIN_VERSION: u8 = 10; +pub const DEFAULT_VERSION: u8 = 45; + +/// Structure representing a ZIP file. +#[derive(Debug, Clone, Default)] +pub struct ZipFileData { + /// Compatibility of the file attribute information + pub system: System, + /// Specification version + pub version_made_by: u8, + /// ZIP flags + pub flags: u16, + /// True if the file is encrypted. + pub encrypted: bool, + /// True if file_name and file_comment are UTF8 + pub is_utf8: bool, + /// True if the file uses a data-descriptor section + pub using_data_descriptor: bool, + /// Compression method used to store the file + pub compression_method: crate::compression::CompressionMethod, + /// Compression level to store the file + pub compression_level: Option, + /// Last modified time. This will only have a 2 second precision. + pub last_modified_time: Option, + /// CRC32 checksum + pub crc32: u32, + /// Size of the file in the ZIP + pub compressed_size: u64, + /// Size of the file when extracted + pub uncompressed_size: u64, + /// Name of the file + pub file_name: Box, + /// Raw file name. To be used when file_name was incorrectly decoded. + pub file_name_raw: Box<[u8]>, + /// Extra field usually used for storage expansion + pub extra_field: Option>>, + /// Extra field only written to central directory + pub central_extra_field: Option>>, + /// File comment + pub file_comment: Box, + /// Specifies where the local header of the file starts + pub header_start: u64, + /// Specifies where the extra data of the file starts + pub extra_data_start: Option, + /// Specifies where the central header of the file starts + /// + /// Note that when this is not known, it is set to 0 + pub central_header_start: u64, + /// Specifies where the compressed data of the file starts + pub data_start: OnceLock, + /// External file attributes + pub external_attributes: u32, + /// Reserve local ZIP64 extra field + pub large_file: bool, + /// AES mode if applicable + pub aes_mode: Option<(AesMode, AesVendorVersion, CompressionMethod)>, + /// Specifies where in the extra data the AES metadata starts + pub aes_extra_data_start: u64, + + /// extra fields, see + pub extra_fields: Vec, +} + +impl ZipFileData { + /// Get the starting offset of the data of the compressed file + pub fn data_start(&self, reader: &mut (impl Read + Seek + ?Sized)) -> ZipResult { + match self.data_start.get() { + Some(data_start) => Ok(*data_start), + None => Ok(find_data_start(self, reader)?), + } + } + + #[allow(dead_code)] + pub fn is_dir(&self) -> bool { + is_dir(&self.file_name) + } + + pub fn file_name_sanitized(&self) -> PathBuf { + let no_null_filename = match self.file_name.find('\0') { + Some(index) => &self.file_name[0..index], + None => &self.file_name, + }; + + Utf8WindowsPath::new(no_null_filename) + .components() + .filter(|component| matches!(*component, Utf8WindowsComponent::Normal(..))) + .fold(PathBuf::new(), |mut path, cur| { + if let Utf8WindowsComponent::Normal(s) = cur { + path.push(s); + } + path + }) + } + + /// Simplify the file name by removing the prefix and parent directories and only return normal components + pub(crate) fn simplified_components(&self) -> Option> { + if self.file_name.contains('\0') { + return None; + } + let input = Path::new(OsStr::new(&*self.file_name)); + crate::path::simplified_components(input) + } + + pub(crate) fn enclosed_name(&self) -> Option { + if self.file_name.contains('\0') { + return None; + } + let mut depth = 0usize; + let mut out_path = PathBuf::new(); + for component in Utf8WindowsPath::new(&self.file_name).components() { + match component { + Utf8WindowsComponent::Prefix(_) | Utf8WindowsComponent::RootDir => { + if depth > 0 { + return None; + } + } + Utf8WindowsComponent::ParentDir => { + depth = depth.checked_sub(1)?; + out_path.pop(); + } + Utf8WindowsComponent::Normal(s) => { + depth += 1; + out_path.push(s); + } + Utf8WindowsComponent::CurDir => (), + } + } + Some(out_path) + } + + /// Get unix mode for the file + pub(crate) const fn unix_mode(&self) -> Option { + if self.external_attributes == 0 { + return None; + } + let unix_mode = self.external_attributes >> 16; + if unix_mode != 0 { + // If the high 16 bits are non-zero, they probably contain Unix permissions. + // This happens for archives created on Windows by this crate or other tools, + // and is the only way to identify symlinks in such archives. + return Some(unix_mode); + } + match self.system { + System::Unix => Some(unix_mode), + System::Dos => { + // Interpret MS-DOS directory bit + let mut mode = if 0x10 == (self.external_attributes & 0x10) { + ffi::S_IFDIR | 0o0775 + } else { + ffi::S_IFREG | 0o0664 + }; + if 0x01 == (self.external_attributes & 0x01) { + // Read-only bit; strip write permissions + mode &= !0o222; + } + Some(mode) + } + _ => None, + } + } + + /// PKZIP version needed to open this file (from APPNOTE 4.4.3.2). + pub fn version_needed(&self) -> u16 { + let compression_version: u16 = match self.compression_method { + CompressionMethod::Stored => MIN_VERSION.into(), + #[cfg(feature = "_deflate-any")] + CompressionMethod::Deflated => 20, + #[cfg(feature = "_bzip2_any")] + CompressionMethod::Bzip2 => 46, + #[cfg(feature = "deflate64")] + CompressionMethod::Deflate64 => 21, + #[cfg(feature = "lzma")] + CompressionMethod::Lzma => 63, + #[cfg(feature = "xz")] + CompressionMethod::Xz => 63, + // APPNOTE doesn't specify a version for Zstandard + _ => DEFAULT_VERSION as u16, + }; + let crypto_version: u16 = if self.aes_mode.is_some() { + 51 + } else if self.encrypted { + 20 + } else { + 10 + }; + let misc_feature_version: u16 = if self.large_file { + 45 + } else if self + .unix_mode() + .is_some_and(|mode| mode & S_IFDIR == S_IFDIR) + { + // file is directory + 20 + } else { + 10 + }; + compression_version + .max(crypto_version) + .max(misc_feature_version) + } + #[inline(always)] + pub(crate) fn extra_field_len(&self) -> usize { + self.extra_field + .as_ref() + .map(|v| v.len()) + .unwrap_or_default() + } + #[inline(always)] + pub(crate) fn central_extra_field_len(&self) -> usize { + self.central_extra_field + .as_ref() + .map(|v| v.len()) + .unwrap_or_default() + } + + #[allow(clippy::too_many_arguments)] + pub(crate) fn initialize_local_block( + name: S, + options: &FileOptions, + raw_values: ZipRawValues, + header_start: u64, + extra_data_start: Option, + aes_extra_data_start: u64, + compression_method: crate::compression::CompressionMethod, + aes_mode: Option<(AesMode, AesVendorVersion, CompressionMethod)>, + extra_field: &[u8], + ) -> Self + where + S: ToString, + { + let permissions = options.permissions.unwrap_or(0o100644); + let file_name: Box = name.to_string().into_boxed_str(); + let file_name_raw: Box<[u8]> = file_name.bytes().collect(); + let mut external_attributes = permissions << 16; + let system = if (permissions & ffi::S_IFLNK) == ffi::S_IFLNK { + System::Unix + } else if cfg!(windows) { + if is_dir(&file_name) { + // DOS directory bit + external_attributes |= 0x10; + } + if options + .permissions + .is_some_and(|permissions| permissions & 0o444 == 0) + { + // DOS read-only bit + external_attributes |= 0x01; + } + System::Dos + } else { + System::Unix + }; + let mut local_block = ZipFileData { + system, + version_made_by: DEFAULT_VERSION, + flags: 0, + encrypted: options.encrypt_with.is_some() + || cfg_if_expr! { + #[cfg(feature = "aes-crypto")] => options.aes_mode.is_some(), + _ => false + }, + using_data_descriptor: false, + is_utf8: !file_name.is_ascii(), + compression_method, + compression_level: options.compression_level, + last_modified_time: Some(options.last_modified_time), + crc32: raw_values.crc32, + compressed_size: raw_values.compressed_size, + uncompressed_size: raw_values.uncompressed_size, + file_name, // Never used for saving, but used as map key in insert_file_data() + file_name_raw, + extra_field: Some(extra_field.to_vec().into()), + central_extra_field: options.extended_options.central_extra_data().cloned(), + file_comment: String::with_capacity(0).into_boxed_str(), + header_start, + data_start: OnceLock::new(), + central_header_start: 0, + external_attributes, + large_file: options.large_file, + aes_mode, + extra_fields: Vec::new(), + extra_data_start, + aes_extra_data_start, + }; + local_block.version_made_by = local_block.version_needed() as u8; + local_block + } + + pub(crate) fn from_local_block( + block: ZipLocalEntryBlock, + reader: &mut R, + ) -> ZipResult { + let ZipLocalEntryBlock { + // magic, + version_made_by, + flags, + compression_method, + last_mod_time, + last_mod_date, + crc32, + compressed_size, + uncompressed_size, + file_name_length, + extra_field_length, + .. + } = block; + + let encrypted: bool = flags & 1 == 1; + if encrypted { + return Err(ZipError::UnsupportedArchive( + "Encrypted files are not supported", + )); + } + + /* FIXME: these were previously incorrect: add testing! */ + /* flags & (1 << 3) != 0 */ + let using_data_descriptor: bool = flags & (1 << 3) == 1 << 3; + if using_data_descriptor { + return Err(ZipError::UnsupportedArchive( + "The file length is not available in the local header", + )); + } + + /* flags & (1 << 1) != 0 */ + let is_utf8: bool = flags & (1 << 11) != 0; + let compression_method = crate::CompressionMethod::parse_from_u16(compression_method); + let file_name_length: usize = file_name_length.into(); + let extra_field_length: usize = extra_field_length.into(); + + let mut file_name_raw = vec![0u8; file_name_length]; + if let Err(e) = reader.read_exact(&mut file_name_raw) { + if e.kind() == std::io::ErrorKind::UnexpectedEof { + return Err(invalid!("File name extends beyond file boundary")); + } + return Err(e.into()); + } + let mut extra_field = vec![0u8; extra_field_length]; + if let Err(e) = reader.read_exact(&mut extra_field) { + if e.kind() == std::io::ErrorKind::UnexpectedEof { + return Err(invalid!("Extra field extends beyond file boundary")); + } + return Err(e.into()); + } + + let file_name: Box = match is_utf8 { + true => String::from_utf8_lossy(&file_name_raw).into(), + false => file_name_raw.clone().from_cp437().into(), + }; + + let system: u8 = (version_made_by >> 8).try_into().unwrap(); + Ok(ZipFileData { + system: System::from(system), + /* NB: this strips the top 8 bits! */ + version_made_by: version_made_by as u8, + flags, + encrypted, + using_data_descriptor, + is_utf8, + compression_method, + compression_level: None, + last_modified_time: DateTime::try_from_msdos(last_mod_date, last_mod_time).ok(), + crc32, + compressed_size: compressed_size.into(), + uncompressed_size: uncompressed_size.into(), + file_name, + file_name_raw: file_name_raw.into(), + extra_field: Some(Arc::new(extra_field)), + central_extra_field: None, + file_comment: String::with_capacity(0).into_boxed_str(), // file comment is only available in the central directory + // header_start and data start are not available, but also don't matter, since seeking is + // not available. + header_start: 0, + data_start: OnceLock::new(), + central_header_start: 0, + // The external_attributes field is only available in the central directory. + // We set this to zero, which should be valid as the docs state 'If input came + // from standard input, this field is set to zero.' + external_attributes: 0, + large_file: false, + aes_mode: None, + extra_fields: Vec::new(), + extra_data_start: None, + aes_extra_data_start: 0, + }) + } + + fn is_utf8(&self) -> bool { + std::str::from_utf8(&self.file_name_raw).is_ok() + } + + fn is_ascii(&self) -> bool { + self.file_name_raw.is_ascii() + } + + fn flags(&self) -> u16 { + let utf8_bit: u16 = if self.is_utf8() && !self.is_ascii() { + 1u16 << 11 + } else { + 0 + }; + + let using_data_descriptor_bit = if self.using_data_descriptor { + 1u16 << 3 + } else { + 0 + }; + + let encrypted_bit: u16 = if self.encrypted { 1u16 << 0 } else { 0 }; + + utf8_bit | using_data_descriptor_bit | encrypted_bit + } + + fn clamp_size_field(&self, field: u64) -> u32 { + if self.large_file { + spec::ZIP64_BYTES_THR as u32 + } else { + field.min(spec::ZIP64_BYTES_THR).try_into().unwrap() + } + } + + pub(crate) fn local_block(&self) -> ZipResult { + let (compressed_size, uncompressed_size) = if self.using_data_descriptor { + (0, 0) + } else { + ( + self.clamp_size_field(self.compressed_size), + self.clamp_size_field(self.uncompressed_size), + ) + }; + let extra_field_length: u16 = self + .extra_field_len() + .try_into() + .map_err(|_| invalid!("Extra data field is too large"))?; + + let last_modified_time = self + .last_modified_time + .unwrap_or_else(DateTime::default_for_write); + Ok(ZipLocalEntryBlock { + magic: ZipLocalEntryBlock::MAGIC, + version_made_by: self.version_needed(), + flags: self.flags(), + compression_method: self.compression_method.serialize_to_u16(), + last_mod_time: last_modified_time.timepart(), + last_mod_date: last_modified_time.datepart(), + crc32: self.crc32, + compressed_size, + uncompressed_size, + file_name_length: self.file_name_raw.len().try_into().unwrap(), + extra_field_length, + }) + } + + pub(crate) fn block(&self) -> ZipResult { + let extra_field_len: u16 = self.extra_field_len().try_into().unwrap(); + let central_extra_field_len: u16 = self.central_extra_field_len().try_into().unwrap(); + let last_modified_time = self + .last_modified_time + .unwrap_or_else(DateTime::default_for_write); + let version_to_extract = self.version_needed(); + let version_made_by = (self.version_made_by as u16).max(version_to_extract); + Ok(ZipCentralEntryBlock { + magic: ZipCentralEntryBlock::MAGIC, + version_made_by: ((self.system as u16) << 8) | version_made_by, + version_to_extract, + flags: self.flags(), + compression_method: self.compression_method.serialize_to_u16(), + last_mod_time: last_modified_time.timepart(), + last_mod_date: last_modified_time.datepart(), + crc32: self.crc32, + compressed_size: self + .compressed_size + .min(spec::ZIP64_BYTES_THR) + .try_into() + .unwrap(), + uncompressed_size: self + .uncompressed_size + .min(spec::ZIP64_BYTES_THR) + .try_into() + .unwrap(), + file_name_length: self.file_name_raw.len().try_into().unwrap(), + extra_field_length: extra_field_len.checked_add(central_extra_field_len).ok_or( + invalid!("Extra field length in central directory exceeds 64KiB"), + )?, + file_comment_length: self.file_comment.len().try_into().unwrap(), + disk_number: 0, + internal_file_attributes: 0, + external_file_attributes: self.external_attributes, + offset: self + .header_start + .min(spec::ZIP64_BYTES_THR) + .try_into() + .unwrap(), + }) + } + + pub(crate) fn zip64_extra_field_block(&self) -> Option { + Zip64ExtraFieldBlock::maybe_new( + self.large_file, + self.uncompressed_size, + self.compressed_size, + self.header_start, + ) + } + + pub(crate) fn write_data_descriptor( + &self, + writer: &mut W, + auto_large_file: bool, + ) -> Result<(), ZipError> { + if self.large_file { + return self.zip64_data_descriptor_block().write(writer); + } + if self.compressed_size > spec::ZIP64_BYTES_THR + || self.uncompressed_size > spec::ZIP64_BYTES_THR + { + if auto_large_file { + return self.zip64_data_descriptor_block().write(writer); + } + return Err(ZipError::Io(std::io::Error::other( + "Large file option has not been set - use .large_file(true) in options", + ))); + } + self.data_descriptor_block().write(writer) + } + + pub(crate) fn data_descriptor_block(&self) -> ZipDataDescriptorBlock { + ZipDataDescriptorBlock { + magic: ZipDataDescriptorBlock::MAGIC, + crc32: self.crc32, + compressed_size: self.compressed_size as u32, + uncompressed_size: self.uncompressed_size as u32, + } + } + + pub(crate) fn zip64_data_descriptor_block(&self) -> Zip64DataDescriptorBlock { + Zip64DataDescriptorBlock { + magic: Zip64DataDescriptorBlock::MAGIC, + crc32: self.crc32, + compressed_size: self.compressed_size, + uncompressed_size: self.uncompressed_size, + } + } +} + +#[derive(Copy, Clone, Debug)] +#[repr(packed, C)] +pub(crate) struct ZipCentralEntryBlock { + magic: spec::Magic, + pub version_made_by: u16, + pub version_to_extract: u16, + pub flags: u16, + pub compression_method: u16, + pub last_mod_time: u16, + pub last_mod_date: u16, + pub crc32: u32, + pub compressed_size: u32, + pub uncompressed_size: u32, + pub file_name_length: u16, + pub extra_field_length: u16, + pub file_comment_length: u16, + pub disk_number: u16, + pub internal_file_attributes: u16, + pub external_file_attributes: u32, + pub offset: u32, +} + +unsafe impl Pod for ZipCentralEntryBlock {} + +impl FixedSizeBlock for ZipCentralEntryBlock { + const MAGIC: spec::Magic = spec::Magic::CENTRAL_DIRECTORY_HEADER_SIGNATURE; + + #[inline(always)] + fn magic(self) -> spec::Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("Invalid Central Directory header"); + + to_and_from_le![ + (magic, spec::Magic), + (version_made_by, u16), + (version_to_extract, u16), + (flags, u16), + (compression_method, u16), + (last_mod_time, u16), + (last_mod_date, u16), + (crc32, u32), + (compressed_size, u32), + (uncompressed_size, u32), + (file_name_length, u16), + (extra_field_length, u16), + (file_comment_length, u16), + (disk_number, u16), + (internal_file_attributes, u16), + (external_file_attributes, u32), + (offset, u32), + ]; +} + +#[derive(Copy, Clone, Debug)] +#[repr(packed, C)] +pub(crate) struct ZipLocalEntryBlock { + magic: spec::Magic, + pub version_made_by: u16, + pub flags: u16, + pub compression_method: u16, + pub last_mod_time: u16, + pub last_mod_date: u16, + pub crc32: u32, + pub compressed_size: u32, + pub uncompressed_size: u32, + pub file_name_length: u16, + pub extra_field_length: u16, +} + +unsafe impl Pod for ZipLocalEntryBlock {} + +impl FixedSizeBlock for ZipLocalEntryBlock { + const MAGIC: spec::Magic = spec::Magic::LOCAL_FILE_HEADER_SIGNATURE; + + #[inline(always)] + fn magic(self) -> spec::Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("Invalid local file header"); + + to_and_from_le![ + (magic, spec::Magic), + (version_made_by, u16), + (flags, u16), + (compression_method, u16), + (last_mod_time, u16), + (last_mod_date, u16), + (crc32, u32), + (compressed_size, u32), + (uncompressed_size, u32), + (file_name_length, u16), + (extra_field_length, u16), + ]; +} + +#[derive(Copy, Clone, Debug)] +pub(crate) struct Zip64ExtraFieldBlock { + magic: spec::ExtraFieldMagic, + size: u16, + uncompressed_size: Option, + compressed_size: Option, + header_start: Option, + // Excluded fields: + // u32: disk start number +} + +impl Zip64ExtraFieldBlock { + pub(crate) fn maybe_new( + large_file: bool, + uncompressed_size: u64, + compressed_size: u64, + header_start: u64, + ) -> Option { + let mut size: u16 = 0; + let uncompressed_size = if uncompressed_size >= ZIP64_BYTES_THR || large_file { + size += mem::size_of::() as u16; + Some(uncompressed_size) + } else { + None + }; + let compressed_size = if compressed_size >= ZIP64_BYTES_THR || large_file { + size += mem::size_of::() as u16; + Some(compressed_size) + } else { + None + }; + let header_start = if header_start >= ZIP64_BYTES_THR { + size += mem::size_of::() as u16; + Some(header_start) + } else { + None + }; + if size == 0 { + return None; + } + + Some(Zip64ExtraFieldBlock { + magic: spec::ExtraFieldMagic::ZIP64_EXTRA_FIELD_TAG, + size, + uncompressed_size, + compressed_size, + header_start, + }) + } +} + +impl Zip64ExtraFieldBlock { + pub fn full_size(&self) -> usize { + assert!(self.size > 0); + self.size as usize + mem::size_of::() + mem::size_of::() + } + + pub fn serialize(self) -> Box<[u8]> { + let Self { + magic, + size, + uncompressed_size, + compressed_size, + header_start, + } = self; + + let full_size = self.full_size(); + + let mut ret = Vec::with_capacity(full_size); + ret.extend(magic.to_le_bytes()); + ret.extend(u16::to_le_bytes(size)); + + if let Some(uncompressed_size) = uncompressed_size { + ret.extend(u64::to_le_bytes(uncompressed_size)); + } + if let Some(compressed_size) = compressed_size { + ret.extend(u64::to_le_bytes(compressed_size)); + } + if let Some(header_start) = header_start { + ret.extend(u64::to_le_bytes(header_start)); + } + debug_assert_eq!(ret.len(), full_size); + + ret.into_boxed_slice() + } +} + +#[derive(Copy, Clone, Debug)] +#[repr(packed, C)] +pub(crate) struct ZipDataDescriptorBlock { + magic: spec::Magic, + pub crc32: u32, + pub compressed_size: u32, + pub uncompressed_size: u32, +} + +unsafe impl Pod for ZipDataDescriptorBlock {} + +impl FixedSizeBlock for ZipDataDescriptorBlock { + const MAGIC: spec::Magic = spec::Magic::DATA_DESCRIPTOR_SIGNATURE; + + #[inline(always)] + fn magic(self) -> spec::Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("Invalid data descriptor header"); + + to_and_from_le![ + (magic, spec::Magic), + (crc32, u32), + (compressed_size, u32), + (uncompressed_size, u32), + ]; +} + +#[derive(Copy, Clone, Debug)] +#[repr(packed, C)] +pub(crate) struct Zip64DataDescriptorBlock { + magic: spec::Magic, + pub crc32: u32, + pub compressed_size: u64, + pub uncompressed_size: u64, +} + +unsafe impl Pod for Zip64DataDescriptorBlock {} + +impl FixedSizeBlock for Zip64DataDescriptorBlock { + const MAGIC: spec::Magic = spec::Magic::DATA_DESCRIPTOR_SIGNATURE; + + #[inline(always)] + fn magic(self) -> spec::Magic { + self.magic + } + + const WRONG_MAGIC_ERROR: ZipError = invalid!("Invalid zip64 data descriptor header"); + + to_and_from_le![ + (magic, spec::Magic), + (crc32, u32), + (compressed_size, u64), + (uncompressed_size, u64), + ]; +} + +/// The encryption specification used to encrypt a file with AES. +/// +/// According to the [specification](https://www.winzip.com/win/en/aes_info.html#winzip11) AE-2 +/// does not make use of the CRC check. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +#[repr(u16)] +pub enum AesVendorVersion { + Ae1 = 0x0001, + Ae2 = 0x0002, +} + +/// AES variant used. +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +#[cfg_attr(feature = "_arbitrary", derive(arbitrary::Arbitrary))] +#[repr(u8)] +pub enum AesMode { + /// 128-bit AES encryption. + Aes128 = 0x01, + /// 192-bit AES encryption. + Aes192 = 0x02, + /// 256-bit AES encryption. + Aes256 = 0x03, +} + +#[cfg(feature = "aes-crypto")] +impl AesMode { + /// Length of the salt for the given AES mode. + pub const fn salt_length(&self) -> usize { + self.key_length() / 2 + } + + /// Length of the key for the given AES mode. + pub const fn key_length(&self) -> usize { + match self { + Self::Aes128 => 16, + Self::Aes192 => 24, + Self::Aes256 => 32, + } + } +} + +#[cfg(test)] +mod test { + #[test] + fn system() { + use super::System; + assert_eq!(u8::from(System::Dos), 0u8); + assert_eq!(System::Dos as u8, 0u8); + assert_eq!(System::Unix as u8, 3u8); + assert_eq!(u8::from(System::Unix), 3u8); + assert_eq!(System::from(0), System::Dos); + assert_eq!(System::from(3), System::Unix); + assert_eq!(u8::from(System::Unknown), 4u8); + assert_eq!(System::Unknown as u8, 4u8); + } + + #[test] + fn unix_mode_robustness() { + use super::{System, ZipFileData}; + use crate::types::ffi::S_IFLNK; + let mut data = ZipFileData { + system: System::Dos, + external_attributes: (S_IFLNK | 0o777) << 16, + ..ZipFileData::default() + }; + assert_eq!(data.unix_mode(), Some(S_IFLNK | 0o777)); + + data.system = System::Unknown; + assert_eq!(data.unix_mode(), Some(S_IFLNK | 0o777)); + + data.external_attributes = 0x10; // DOS directory bit + data.system = System::Dos; + assert_eq!( + data.unix_mode().unwrap() & 0o170000, + crate::types::ffi::S_IFDIR + ); + } + + #[test] + fn sanitize() { + let file_name = "/path/../../../../etc/./passwd\0/etc/shadow".to_string(); + let data = ZipFileData { + system: System::Dos, + version_made_by: 0, + flags: 0, + encrypted: false, + using_data_descriptor: false, + is_utf8: true, + compression_method: crate::compression::CompressionMethod::Stored, + compression_level: None, + last_modified_time: None, + crc32: 0, + compressed_size: 0, + uncompressed_size: 0, + file_name: file_name.clone().into_boxed_str(), + file_name_raw: file_name.into_bytes().into_boxed_slice(), + extra_field: None, + central_extra_field: None, + file_comment: String::with_capacity(0).into_boxed_str(), + header_start: 0, + extra_data_start: None, + data_start: OnceLock::new(), + central_header_start: 0, + external_attributes: 0, + large_file: false, + aes_mode: None, + aes_extra_data_start: 0, + extra_fields: Vec::new(), + }; + assert_eq!(data.file_name_sanitized(), PathBuf::from("path/etc/passwd")); + } + + #[test] + #[allow(clippy::unusual_byte_groupings)] + fn datetime_default() { + use super::DateTime; + let dt = DateTime::default(); + assert_eq!(dt.timepart(), 0); + assert_eq!(dt.datepart(), 0b0000000_0001_00001); + } + + #[test] + #[allow(clippy::unusual_byte_groupings)] + fn datetime_max() { + use super::DateTime; + let dt = DateTime::from_date_and_time(2107, 12, 31, 23, 59, 58).unwrap(); + assert_eq!(dt.timepart(), 0b10111_111011_11101); + assert_eq!(dt.datepart(), 0b1111111_1100_11111); + } + + #[test] + fn datetime_equality() { + use super::DateTime; + + let dt = DateTime::from_date_and_time(2018, 11, 17, 10, 38, 30).unwrap(); + assert_eq!( + dt, + DateTime::from_date_and_time(2018, 11, 17, 10, 38, 30).unwrap() + ); + assert_ne!(dt, DateTime::default()); + } + + #[test] + fn datetime_order() { + use std::cmp::Ordering; + + use super::DateTime; + + let dt = DateTime::from_date_and_time(2018, 11, 17, 10, 38, 30).unwrap(); + assert_eq!( + dt.cmp(&DateTime::from_date_and_time(2018, 11, 17, 10, 38, 30).unwrap()), + Ordering::Equal + ); + // year + assert!(dt < DateTime::from_date_and_time(2019, 11, 17, 10, 38, 30).unwrap()); + assert!(dt > DateTime::from_date_and_time(2017, 11, 17, 10, 38, 30).unwrap()); + // month + assert!(dt < DateTime::from_date_and_time(2018, 12, 17, 10, 38, 30).unwrap()); + assert!(dt > DateTime::from_date_and_time(2018, 10, 17, 10, 38, 30).unwrap()); + // day + assert!(dt < DateTime::from_date_and_time(2018, 11, 18, 10, 38, 30).unwrap()); + assert!(dt > DateTime::from_date_and_time(2018, 11, 16, 10, 38, 30).unwrap()); + // hour + assert!(dt < DateTime::from_date_and_time(2018, 11, 17, 11, 38, 30).unwrap()); + assert!(dt > DateTime::from_date_and_time(2018, 11, 17, 9, 38, 30).unwrap()); + // minute + assert!(dt < DateTime::from_date_and_time(2018, 11, 17, 10, 39, 30).unwrap()); + assert!(dt > DateTime::from_date_and_time(2018, 11, 17, 10, 37, 30).unwrap()); + // second + assert!(dt < DateTime::from_date_and_time(2018, 11, 17, 10, 38, 32).unwrap()); + assert_eq!( + dt.cmp(&DateTime::from_date_and_time(2018, 11, 17, 10, 38, 31).unwrap()), + Ordering::Equal + ); + assert!(dt > DateTime::from_date_and_time(2018, 11, 17, 10, 38, 29).unwrap()); + assert!(dt > DateTime::from_date_and_time(2018, 11, 17, 10, 38, 28).unwrap()); + } + + #[test] + fn datetime_display() { + use super::DateTime; + + assert_eq!(format!("{}", DateTime::default()), "1980-01-01 00:00:00"); + assert_eq!( + format!( + "{}", + DateTime::from_date_and_time(2018, 11, 17, 10, 38, 30).unwrap() + ), + "2018-11-17 10:38:30" + ); + assert_eq!( + format!( + "{}", + DateTime::from_date_and_time(2107, 12, 31, 23, 59, 58).unwrap() + ), + "2107-12-31 23:59:58" + ); + } + + #[test] + fn datetime_bounds() { + use super::DateTime; + + assert!(DateTime::from_date_and_time(2000, 1, 1, 23, 59, 60).is_ok()); + assert!(DateTime::from_date_and_time(2000, 1, 1, 24, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2000, 1, 1, 0, 60, 0).is_err()); + assert!(DateTime::from_date_and_time(2000, 1, 1, 0, 0, 61).is_err()); + + assert!(DateTime::from_date_and_time(2107, 12, 31, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(1980, 1, 1, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(1979, 1, 1, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(1980, 0, 1, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(1980, 1, 0, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2108, 12, 31, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2107, 13, 31, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2107, 12, 32, 0, 0, 0).is_err()); + + assert!(DateTime::from_date_and_time(2018, 1, 31, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 2, 28, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 2, 29, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2018, 3, 31, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 4, 30, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 4, 31, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2018, 5, 31, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 6, 30, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 6, 31, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2018, 7, 31, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 8, 31, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 9, 30, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 9, 31, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2018, 10, 31, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 11, 30, 0, 0, 0).is_ok()); + assert!(DateTime::from_date_and_time(2018, 11, 31, 0, 0, 0).is_err()); + assert!(DateTime::from_date_and_time(2018, 12, 31, 0, 0, 0).is_ok()); + + // leap year: divisible by 4 + assert!(DateTime::from_date_and_time(2024, 2, 29, 0, 0, 0).is_ok()); + // leap year: divisible by 100 and by 400 + assert!(DateTime::from_date_and_time(2000, 2, 29, 0, 0, 0).is_ok()); + // common year: divisible by 100 but not by 400 + assert!(DateTime::from_date_and_time(2100, 2, 29, 0, 0, 0).is_err()); + } + + use std::{path::PathBuf, sync::OnceLock}; + + #[cfg(feature = "time")] + use time::{format_description::well_known::Rfc3339, OffsetDateTime, PrimitiveDateTime}; + + use crate::types::{System, ZipFileData}; + + #[cfg(feature = "time")] + #[test] + fn datetime_try_from_offset_datetime() { + use time::macros::datetime; + + use super::DateTime; + + // 2018-11-17 10:38:30 + let dt = DateTime::try_from(datetime!(2018-11-17 10:38:30 UTC)).unwrap(); + assert_eq!(dt.year(), 2018); + assert_eq!(dt.month(), 11); + assert_eq!(dt.day(), 17); + assert_eq!(dt.hour(), 10); + assert_eq!(dt.minute(), 38); + assert_eq!(dt.second(), 30); + } + + #[cfg(feature = "time")] + #[test] + fn datetime_try_from_primitive_datetime() { + use time::macros::datetime; + + use super::DateTime; + + // 2018-11-17 10:38:30 + let dt = DateTime::try_from(datetime!(2018-11-17 10:38:30)).unwrap(); + assert_eq!(dt.year(), 2018); + assert_eq!(dt.month(), 11); + assert_eq!(dt.day(), 17); + assert_eq!(dt.hour(), 10); + assert_eq!(dt.minute(), 38); + assert_eq!(dt.second(), 30); + } + + #[cfg(feature = "time")] + #[test] + fn datetime_try_from_bounds() { + use super::DateTime; + use time::macros::datetime; + + // 1979-12-31 23:59:59 + assert!(DateTime::try_from(datetime!(1979-12-31 23:59:59)).is_err()); + + // 1980-01-01 00:00:00 + assert!(DateTime::try_from(datetime!(1980-01-01 00:00:00)).is_ok()); + + // 2107-12-31 23:59:59 + assert!(DateTime::try_from(datetime!(2107-12-31 23:59:59)).is_ok()); + + // 2108-01-01 00:00:00 + assert!(DateTime::try_from(datetime!(2108-01-01 00:00:00)).is_err()); + } + + #[cfg(feature = "time")] + #[test] + fn offset_datetime_try_from_datetime() { + use time::macros::datetime; + + use super::DateTime; + + // 2018-11-17 10:38:30 UTC + let dt = + OffsetDateTime::try_from(DateTime::try_from_msdos(0x4D71, 0x54CF).unwrap()).unwrap(); + assert_eq!(dt, datetime!(2018-11-17 10:38:30 UTC)); + } + + #[cfg(feature = "time")] + #[test] + fn primitive_datetime_try_from_datetime() { + use time::macros::datetime; + + use super::DateTime; + + // 2018-11-17 10:38:30 + let dt = + PrimitiveDateTime::try_from(DateTime::try_from_msdos(0x4D71, 0x54CF).unwrap()).unwrap(); + assert_eq!(dt, datetime!(2018-11-17 10:38:30)); + } + + #[cfg(feature = "time")] + #[test] + fn offset_datetime_try_from_bounds() { + use super::DateTime; + + // 1980-00-00 00:00:00 + assert!(OffsetDateTime::try_from(unsafe { + DateTime::from_msdos_unchecked(0x0000, 0x0000) + }) + .is_err()); + + // 2107-15-31 31:63:62 + assert!(OffsetDateTime::try_from(unsafe { + DateTime::from_msdos_unchecked(0xFFFF, 0xFFFF) + }) + .is_err()); + } + + #[cfg(feature = "time")] + #[test] + fn primitive_datetime_try_from_bounds() { + use super::DateTime; + + // 1980-00-00 00:00:00 + assert!(PrimitiveDateTime::try_from(unsafe { + DateTime::from_msdos_unchecked(0x0000, 0x0000) + }) + .is_err()); + + // 2107-15-31 31:63:62 + assert!(PrimitiveDateTime::try_from(unsafe { + DateTime::from_msdos_unchecked(0xFFFF, 0xFFFF) + }) + .is_err()); + } + + #[cfg(feature = "jiff-02")] + #[test] + fn datetime_try_from_civil_datetime() { + use jiff::civil; + + use super::DateTime; + + // 2018-11-17 10:38:30 + let dt = DateTime::try_from(civil::datetime(2018, 11, 17, 10, 38, 30, 0)).unwrap(); + assert_eq!(dt.year(), 2018); + assert_eq!(dt.month(), 11); + assert_eq!(dt.day(), 17); + assert_eq!(dt.hour(), 10); + assert_eq!(dt.minute(), 38); + assert_eq!(dt.second(), 30); + } + + #[cfg(feature = "jiff-02")] + #[test] + fn datetime_try_from_civil_datetime_bounds() { + use jiff::civil; + + use super::DateTime; + + // 1979-12-31 23:59:59 + assert!(DateTime::try_from(civil::datetime(1979, 12, 31, 23, 59, 59, 0)).is_err()); + + // 1980-01-01 00:00:00 + assert!(DateTime::try_from(civil::datetime(1980, 1, 1, 0, 0, 0, 0)).is_ok()); + + // 2107-12-31 23:59:59 + assert!(DateTime::try_from(civil::datetime(2107, 12, 31, 23, 59, 59, 0)).is_ok()); + + // 2108-01-01 00:00:00 + assert!(DateTime::try_from(civil::datetime(2108, 1, 1, 0, 0, 0, 0)).is_err()); + } + + #[cfg(feature = "jiff-02")] + #[test] + fn civil_datetime_try_from_datetime() { + use jiff::civil; + + use super::DateTime; + + // 2018-11-17 10:38:30 UTC + let dt = + civil::DateTime::try_from(DateTime::try_from_msdos(0x4D71, 0x54CF).unwrap()).unwrap(); + assert_eq!(dt, civil::datetime(2018, 11, 17, 10, 38, 30, 0)); + } + + #[cfg(feature = "jiff-02")] + #[test] + fn civil_datetime_try_from_datetime_bounds() { + use jiff::civil; + + use super::DateTime; + + // 1980-00-00 00:00:00 + assert!(civil::DateTime::try_from(unsafe { + DateTime::from_msdos_unchecked(0x0000, 0x0000) + }) + .is_err()); + + // 2107-15-31 31:63:62 + assert!(civil::DateTime::try_from(unsafe { + DateTime::from_msdos_unchecked(0xFFFF, 0xFFFF) + }) + .is_err()); + } + + #[test] + #[allow(deprecated)] + fn time_conversion() { + use super::DateTime; + let dt = DateTime::try_from_msdos(0x4D71, 0x54CF).unwrap(); + assert_eq!(dt.year(), 2018); + assert_eq!(dt.month(), 11); + assert_eq!(dt.day(), 17); + assert_eq!(dt.hour(), 10); + assert_eq!(dt.minute(), 38); + assert_eq!(dt.second(), 30); + + let dt = DateTime::try_from((0x4D71, 0x54CF)).unwrap(); + assert_eq!(dt.year(), 2018); + assert_eq!(dt.month(), 11); + assert_eq!(dt.day(), 17); + assert_eq!(dt.hour(), 10); + assert_eq!(dt.minute(), 38); + assert_eq!(dt.second(), 30); + + #[cfg(feature = "time")] + assert_eq!( + dt.to_time().unwrap().format(&Rfc3339).unwrap(), + "2018-11-17T10:38:30Z" + ); + + assert_eq!(<(u16, u16)>::from(dt), (0x4D71, 0x54CF)); + } + + #[test] + #[allow(deprecated)] + fn time_out_of_bounds() { + use super::DateTime; + let dt = unsafe { DateTime::from_msdos_unchecked(0xFFFF, 0xFFFF) }; + assert_eq!(dt.year(), 2107); + assert_eq!(dt.month(), 15); + assert_eq!(dt.day(), 31); + assert_eq!(dt.hour(), 31); + assert_eq!(dt.minute(), 63); + assert_eq!(dt.second(), 62); + + #[cfg(feature = "time")] + assert!(dt.to_time().is_err()); + + let dt = unsafe { DateTime::from_msdos_unchecked(0x0000, 0x0000) }; + assert_eq!(dt.year(), 1980); + assert_eq!(dt.month(), 0); + assert_eq!(dt.day(), 0); + assert_eq!(dt.hour(), 0); + assert_eq!(dt.minute(), 0); + assert_eq!(dt.second(), 0); + + #[cfg(feature = "time")] + assert!(dt.to_time().is_err()); + } + + #[cfg(feature = "time")] + #[test] + fn time_at_january() { + use super::DateTime; + + // 2020-01-01 00:00:00 + let clock = OffsetDateTime::from_unix_timestamp(1_577_836_800).unwrap(); + + assert!(DateTime::try_from(PrimitiveDateTime::new(clock.date(), clock.time())).is_ok()); + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/unstable.rs b/v2/vendor/security/zip-7.2.0/src/unstable.rs new file mode 100644 index 0000000000..4bce5a6160 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/unstable.rs @@ -0,0 +1,121 @@ +#![allow(missing_docs)] + +use std::borrow::Cow; +use std::io; +use std::io::{Read, Write}; +use std::path::{Component, Path, MAIN_SEPARATOR}; + +/// Provides high level API for reading from a stream. +pub mod stream { + pub use crate::read::stream::*; +} +/// Types for creating ZIP archives. +pub mod write { + use crate::write::{FileOptionExtension, FileOptions}; + /// Unstable methods for [`FileOptions`]. + pub trait FileOptionsExt { + /// Write the file with the given password using the deprecated ZipCrypto algorithm. + /// + /// This is not recommended for new archives, as ZipCrypto is not secure. + fn with_deprecated_encryption(self, password: &[u8]) -> Self; + } + impl FileOptionsExt for FileOptions<'_, T> { + fn with_deprecated_encryption(self, password: &[u8]) -> FileOptions<'static, T> { + self.with_deprecated_encryption(password) + } + } +} + +/// Helper methods for writing unsigned integers in little-endian form. +pub trait LittleEndianWriteExt: Write { + fn write_u16_le(&mut self, input: u16) -> io::Result<()> { + self.write_all(&input.to_le_bytes()) + } + + fn write_u32_le(&mut self, input: u32) -> io::Result<()> { + self.write_all(&input.to_le_bytes()) + } + + fn write_u64_le(&mut self, input: u64) -> io::Result<()> { + self.write_all(&input.to_le_bytes()) + } + + fn write_u128_le(&mut self, input: u128) -> io::Result<()> { + self.write_all(&input.to_le_bytes()) + } +} + +impl LittleEndianWriteExt for W {} + +/// Helper methods for reading unsigned integers in little-endian form. +pub trait LittleEndianReadExt: Read { + fn read_u16_le(&mut self) -> io::Result { + let mut out = [0u8; 2]; + self.read_exact(&mut out)?; + Ok(u16::from_le_bytes(out)) + } + + fn read_u32_le(&mut self) -> io::Result { + let mut out = [0u8; 4]; + self.read_exact(&mut out)?; + Ok(u32::from_le_bytes(out)) + } + + fn read_u64_le(&mut self) -> io::Result { + let mut out = [0u8; 8]; + self.read_exact(&mut out)?; + Ok(u64::from_le_bytes(out)) + } +} + +impl LittleEndianReadExt for R {} + +/// Converts a path to the ZIP format (forward-slash-delimited and normalized). +pub fn path_to_string>(path: T) -> Box { + let mut maybe_original = None; + if let Some(original) = path.as_ref().to_str() { + if original.is_empty() || original == "." || original == ".." { + return String::new().into_boxed_str(); + } + if original.starts_with(MAIN_SEPARATOR) { + if original.len() == 1 { + return MAIN_SEPARATOR.to_string().into_boxed_str(); + } else if (MAIN_SEPARATOR == '/' || !original[1..].contains(MAIN_SEPARATOR)) + && !original.ends_with('.') + && !original.contains([MAIN_SEPARATOR, MAIN_SEPARATOR]) + && !original.contains([MAIN_SEPARATOR, '.', MAIN_SEPARATOR]) + && !original.contains([MAIN_SEPARATOR, '.', '.', MAIN_SEPARATOR]) + { + maybe_original = Some(&original[1..]); + } + } else if !original.contains(MAIN_SEPARATOR) { + return original.into(); + } + } + let mut recreate = maybe_original.is_none(); + let mut normalized_components = Vec::new(); + + for component in path.as_ref().components() { + match component { + Component::Normal(os_str) => match os_str.to_str() { + Some(valid_str) => normalized_components.push(Cow::Borrowed(valid_str)), + None => { + recreate = true; + normalized_components.push(os_str.to_string_lossy()); + } + }, + Component::ParentDir => { + recreate = true; + normalized_components.pop(); + } + _ => { + recreate = true; + } + } + } + if recreate { + normalized_components.join("/").into() + } else { + maybe_original.unwrap().into() + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/write.rs b/v2/vendor/security/zip-7.2.0/src/write.rs new file mode 100644 index 0000000000..ff4932b7c1 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/write.rs @@ -0,0 +1,4077 @@ +//! Types for creating ZIP archives + +#[cfg(feature = "aes-crypto")] +use crate::aes::AesWriter; +use crate::compression::CompressionMethod; +use crate::read::{parse_single_extra_field, Config, ZipArchive, ZipFile}; +use crate::result::{invalid, ZipError, ZipResult}; +use crate::spec::{self, FixedSizeBlock, Zip32CDEBlock}; +use crate::types::ffi::S_IFLNK; +#[cfg(feature = "aes-crypto")] +use crate::types::AesMode; +use crate::types::{ + ffi, AesVendorVersion, DateTime, Zip64ExtraFieldBlock, ZipFileData, ZipLocalEntryBlock, + ZipRawValues, MIN_VERSION, +}; +use core::default::Default; +use core::fmt::{Debug, Formatter}; +use core::marker::PhantomData; +use core::mem::{self, size_of}; +#[cfg(feature = "deflate-zopfli")] +use core::num::NonZeroU64; +use core::str::{from_utf8, Utf8Error}; +use crc32fast::Hasher; +use indexmap::IndexMap; +use std::borrow::ToOwned; +use std::io::{self, Read, Seek, Write}; +use std::io::{BufReader, SeekFrom}; +use std::io::{Cursor, ErrorKind}; +use std::sync::Arc; + +#[cfg(feature = "deflate-flate2")] +use flate2::{write::DeflateEncoder, Compression}; + +#[cfg(feature = "_bzip2_any")] +use bzip2::write::BzEncoder; + +#[cfg(feature = "deflate-zopfli")] +use zopfli::Options; + +#[cfg(feature = "deflate-zopfli")] +use std::io::BufWriter; +use std::path::Path; +#[cfg(feature = "zstd")] +use zstd::stream::write::Encoder as ZstdEncoder; +#[cfg(feature = "_bzip2_any")] +use GenericZipWriter::Bzip2; +#[cfg(feature = "deflate-flate2")] +use GenericZipWriter::Deflater; +#[cfg(feature = "ppmd")] +use GenericZipWriter::Ppmd; +#[cfg(feature = "xz")] +use GenericZipWriter::Xz; +#[cfg(feature = "zstd")] +use GenericZipWriter::Zstd; +#[cfg(feature = "deflate-zopfli")] +use GenericZipWriter::{BufferedZopfliDeflater, ZopfliDeflater}; + +// re-export from types +pub use crate::types::{FileOptions, SimpleFileOptions}; + +#[cfg_attr(feature = "aes-crypto", allow(clippy::large_enum_variant))] +enum MaybeEncrypted { + Unencrypted(W), + #[cfg(feature = "aes-crypto")] + Aes(AesWriter), + ZipCrypto(crate::zipcrypto::ZipCryptoWriter), +} + +impl Debug for MaybeEncrypted { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + // Don't print W, since it may be a huge Vec + f.write_str(match self { + MaybeEncrypted::Unencrypted(_) => "Unencrypted", + #[cfg(feature = "aes-crypto")] + MaybeEncrypted::Aes(_) => "AES", + MaybeEncrypted::ZipCrypto(_) => "ZipCrypto", + }) + } +} + +impl Write for MaybeEncrypted { + fn write(&mut self, buf: &[u8]) -> io::Result { + match self { + MaybeEncrypted::Unencrypted(w) => w.write(buf), + #[cfg(feature = "aes-crypto")] + MaybeEncrypted::Aes(w) => w.write(buf), + MaybeEncrypted::ZipCrypto(w) => w.write(buf), + } + } + fn flush(&mut self) -> io::Result<()> { + match self { + MaybeEncrypted::Unencrypted(w) => w.flush(), + #[cfg(feature = "aes-crypto")] + MaybeEncrypted::Aes(w) => w.flush(), + MaybeEncrypted::ZipCrypto(w) => w.flush(), + } + } +} + +enum GenericZipWriter { + Closed, + Storer(MaybeEncrypted), + #[cfg(feature = "deflate-flate2")] + Deflater(DeflateEncoder>), + #[cfg(feature = "deflate-zopfli")] + ZopfliDeflater(zopfli::DeflateEncoder>), + #[cfg(feature = "deflate-zopfli")] + BufferedZopfliDeflater(BufWriter>>), + #[cfg(feature = "_bzip2_any")] + Bzip2(BzEncoder>), + #[cfg(feature = "zstd")] + Zstd(ZstdEncoder<'static, MaybeEncrypted>), + #[cfg(feature = "xz")] + Xz(Box>>), + #[cfg(feature = "ppmd")] + Ppmd(Box>>), +} + +impl Debug for GenericZipWriter { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Closed => f.write_str("Closed"), + Storer(w) => f.write_fmt(format_args!("Storer({w:?})")), + #[cfg(feature = "deflate-flate2")] + Deflater(w) => f.write_fmt(format_args!("Deflater({:?})", w.get_ref())), + #[cfg(feature = "deflate-zopfli")] + ZopfliDeflater(_) => f.write_str("ZopfliDeflater"), + #[cfg(feature = "deflate-zopfli")] + BufferedZopfliDeflater(_) => f.write_str("BufferedZopfliDeflater"), + #[cfg(feature = "_bzip2_any")] + Bzip2(w) => f.write_fmt(format_args!("Bzip2({:?})", w.get_ref())), + #[cfg(feature = "zstd")] + Zstd(w) => f.write_fmt(format_args!("Zstd({:?})", w.get_ref())), + #[cfg(feature = "xz")] + Xz(w) => f.write_fmt(format_args!("Xz({:?})", w.inner())), + #[cfg(feature = "ppmd")] + Ppmd(_) => f.write_fmt(format_args!("Ppmd8Encoder")), + } + } +} + +// Put the struct declaration in a private module to convince rustdoc to display ZipWriter nicely +pub(crate) mod zip_writer { + use core::fmt::{Debug, Formatter}; + use std::io::{Seek, Write}; + + use indexmap::IndexMap; + + use crate::{ + types::ZipFileData, + write::{GenericZipWriter, ZipWriterStats}, + }; + + /// ZIP archive generator + /// + /// Handles the bookkeeping involved in building an archive, and provides an + /// API to edit its contents. + /// + /// ``` + /// # fn doit() -> zip::result::ZipResult<()> + /// # { + /// # use zip::ZipWriter; + /// use std::io::Write; + /// use zip::write::SimpleFileOptions; + /// + /// // We use a cursor + vec here, though you'd normally use a `File` + /// let mut cur = std::io::Cursor::new(Vec::new()); + /// let mut zip = ZipWriter::new(&mut cur); + /// + /// let options = SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored); + /// zip.start_file("hello_world.txt", options)?; + /// zip.write(b"Hello, World!")?; + /// + /// // Apply the changes you've made. + /// // Dropping the `ZipWriter` will have the same effect, but may silently fail + /// zip.finish()?; + /// + /// // raw zip data is available as a Vec + /// let zip_bytes = cur.into_inner(); + /// + /// # Ok(()) + /// # } + /// # doit().unwrap(); + /// ``` + pub struct ZipWriter { + pub(super) inner: GenericZipWriter, + pub(super) files: IndexMap, ZipFileData>, + pub(super) stats: ZipWriterStats, + pub(super) writing_to_file: bool, + pub(super) writing_raw: bool, + pub(super) comment: Box<[u8]>, + pub(super) zip64_comment: Option>, + pub(super) flush_on_finish_file: bool, + pub(super) seek_possible: bool, + pub(crate) auto_large_file: bool, + } + + impl Debug for ZipWriter { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.write_fmt(format_args!( + "ZipWriter {{files: {:?}, stats: {:?}, writing_to_file: {}, writing_raw: {}, comment: {:?}, flush_on_finish_file: {}}}", + self.files, self.stats, self.writing_to_file, self.writing_raw, + self.comment, self.flush_on_finish_file)) + } + } +} +#[doc(inline)] +pub use self::sealed::FileOptionExtension; +use crate::result::ZipError::UnsupportedArchive; +use crate::unstable::path_to_string; +use crate::unstable::LittleEndianWriteExt; +use crate::write::GenericZipWriter::{Closed, Storer}; +use crate::zipcrypto::{EncryptWith, ZipCryptoKeys}; +use crate::CompressionMethod::Stored; +pub use zip_writer::ZipWriter; + +#[derive(Default, Debug)] +struct ZipWriterStats { + hasher: Hasher, + start: u64, + bytes_written: u64, +} + +mod sealed { + use std::sync::Arc; + + use super::ExtendedFileOptions; + + pub trait Sealed {} + /// File options Extensions + #[doc(hidden)] + pub trait FileOptionExtension: Default + Sealed { + /// Extra Data + fn extra_data(&self) -> Option<&Arc>>; + /// Central Extra Data + fn central_extra_data(&self) -> Option<&Arc>>; + } + impl Sealed for () {} + impl FileOptionExtension for () { + fn extra_data(&self) -> Option<&Arc>> { + None + } + fn central_extra_data(&self) -> Option<&Arc>> { + None + } + } + impl Sealed for ExtendedFileOptions {} + + impl FileOptionExtension for ExtendedFileOptions { + fn extra_data(&self) -> Option<&Arc>> { + Some(&self.extra_data) + } + fn central_extra_data(&self) -> Option<&Arc>> { + Some(&self.central_extra_data) + } + } +} + +/// Adds Extra Data and Central Extra Data. It does not implement copy. +pub type FullFileOptions<'k> = FileOptions<'k, ExtendedFileOptions>; +/// The Extension for Extra Data and Central Extra Data +#[cfg_attr(feature = "_arbitrary", derive(arbitrary::Arbitrary))] +#[derive(Clone, Default, Eq, PartialEq)] +pub struct ExtendedFileOptions { + extra_data: Arc>, + central_extra_data: Arc>, +} + +impl ExtendedFileOptions { + /// Adds an extra data field, unless we detect that it's invalid. + pub fn add_extra_data>( + &mut self, + header_id: u16, + data: D, + central_only: bool, + ) -> ZipResult<()> { + let data = data.as_ref(); + let len = data.len() + 4; + if self.extra_data.len() + self.central_extra_data.len() + len > u16::MAX as usize { + Err(invalid!("Extra data field would be longer than allowed")) + } else { + let field = if central_only { + &mut self.central_extra_data + } else { + &mut self.extra_data + }; + let vec = Arc::get_mut(field); + let vec = match vec { + Some(exclusive) => exclusive, + None => { + *field = Arc::new(field.to_vec()); + Arc::get_mut(field).unwrap() + } + }; + Self::add_extra_data_unchecked(vec, header_id, data)?; + Self::validate_extra_data(vec, true)?; + Ok(()) + } + } + + pub(crate) fn add_extra_data_unchecked( + vec: &mut Vec, + header_id: u16, + data: &[u8], + ) -> Result<(), ZipError> { + vec.reserve_exact(data.len() + 4); + vec.write_u16_le(header_id)?; + vec.write_u16_le(data.len() as u16)?; + vec.write_all(data)?; + Ok(()) + } + + fn validate_extra_data(data: &[u8], disallow_zip64: bool) -> ZipResult<()> { + let len = data.len() as u64; + if len == 0 { + return Ok(()); + } + if len > u16::MAX as u64 { + return Err(ZipError::Io(io::Error::other( + "Extra-data field can't exceed u16::MAX bytes", + ))); + } + let mut data = Cursor::new(data); + let mut pos = data.position(); + while pos < len { + if len - data.position() < 4 { + return Err(ZipError::Io(io::Error::other( + "Extra-data field doesn't have room for ID and length", + ))); + } + #[cfg(not(feature = "unreserved"))] + { + use crate::unstable::LittleEndianReadExt; + let header_id = data.read_u16_le()?; + if EXTRA_FIELD_MAPPING.contains(&header_id) { + return Err(ZipError::Io(io::Error::other( + format!( + "Extra data header ID {header_id:#06} requires crate feature \"unreserved\"", + ), + ))); + } + data.seek(SeekFrom::Current(-2))?; + } + parse_single_extra_field(&mut ZipFileData::default(), &mut data, pos, disallow_zip64)?; + pos = data.position(); + } + Ok(()) + } +} + +impl Debug for ExtendedFileOptions { + fn fmt(&self, f: &mut Formatter<'_>) -> Result<(), std::fmt::Error> { + f.write_fmt(format_args!("ExtendedFileOptions {{extra_data: vec!{:?}.into(), central_extra_data: vec!{:?}.into()}}", + self.extra_data, self.central_extra_data)) + } +} + +#[cfg(feature = "_arbitrary")] +impl<'a> arbitrary::Arbitrary<'a> for FileOptions<'a, ExtendedFileOptions> { + fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { + let mut options = FullFileOptions { + compression_method: CompressionMethod::arbitrary(u)?, + compression_level: if bool::arbitrary(u)? { + Some(u.int_in_range(0..=24)?) + } else { + None + }, + last_modified_time: DateTime::arbitrary(u)?, + permissions: Option::::arbitrary(u)?, + large_file: bool::arbitrary(u)?, + encrypt_with: Option::::arbitrary(u)?, + alignment: u16::arbitrary(u)?, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: None, + ..Default::default() + }; + #[cfg(feature = "deflate-zopfli")] + if options.compression_method == CompressionMethod::Deflated && bool::arbitrary(u)? { + options.zopfli_buffer_size = + Some(if bool::arbitrary(u)? { 2 } else { 3 } << u.int_in_range(8..=20)?); + } + u.arbitrary_loop(Some(0), Some(10), |u| { + options + .add_extra_data( + u.int_in_range(2..=u16::MAX)?, + Box::<[u8]>::arbitrary(u)?, + bool::arbitrary(u)?, + ) + .map_err(|_| arbitrary::Error::IncorrectFormat)?; + Ok(core::ops::ControlFlow::Continue(())) + })?; + ZipWriter::new(Cursor::new(Vec::new())) + .start_file("", options.clone()) + .map_err(|_| arbitrary::Error::IncorrectFormat)?; + Ok(options) + } +} + +impl FileOptions<'_, T> { + pub(crate) fn normalize(&mut self) { + if !self.last_modified_time.is_valid() { + self.last_modified_time = FileOptions::::default().last_modified_time; + } + + *self.permissions.get_or_insert(0o644) |= ffi::S_IFREG; + } + + /// Indicates whether this file will be encrypted (whether with AES or ZipCrypto). + pub const fn has_encryption(&self) -> bool { + #[cfg(feature = "aes-crypto")] + { + self.encrypt_with.is_some() || self.aes_mode.is_some() + } + #[cfg(not(feature = "aes-crypto"))] + { + self.encrypt_with.is_some() + } + } + + /// Set the compression method for the new file + /// + /// The default is [`CompressionMethod::Deflated`] if it is enabled. If not, + /// [`CompressionMethod::Bzip2`] is the default if it is enabled. If neither `bzip2` nor `deflate` + /// is enabled, [`CompressionMethod::Stored`] becomes the default and files are written uncompressed. + #[must_use] + pub const fn compression_method(mut self, method: CompressionMethod) -> Self { + self.compression_method = method; + self + } + + /// Set the compression level for the new file + /// + /// `None` value specifies default compression level. + /// + /// Range of values depends on compression method: + /// * `Deflated`: 10 - 264 for Zopfli, 0 - 9 for other encoders. Default is 24 if Zopfli is the + /// only encoder, or 6 otherwise. + /// * `Bzip2`: 0 - 9. Default is 6 + /// * `Zstd`: -7 - 22, with zero being mapped to default level. Default is 3 + /// * others: only `None` is allowed + #[must_use] + pub const fn compression_level(mut self, level: Option) -> Self { + self.compression_level = level; + self + } + + /// Set the last modified time + /// + /// The default is the current timestamp if the 'time' feature is enabled, and 1980-01-01 + /// otherwise + #[must_use] + pub const fn last_modified_time(mut self, mod_time: DateTime) -> Self { + self.last_modified_time = mod_time; + self + } + + /// Set the permissions for the new file. + /// + /// The format is represented with unix-style permissions. + /// The default is `0o644`, which represents `rw-r--r--` for files, + /// and `0o755`, which represents `rwxr-xr-x` for directories. + /// + /// This method only preserves the file permissions bits (via a `& 0o777`) and discards + /// higher file mode bits. So it cannot be used to denote an entry as a directory, + /// symlink, or other special file type. + #[must_use] + pub const fn unix_permissions(mut self, mode: u32) -> Self { + self.permissions = Some(mode & 0o777); + self + } + + /// Set whether the new file's compressed and uncompressed size is less than 4 GiB. + /// + /// If set to `false` and the file exceeds the limit, an I/O error is thrown and the file is + /// aborted. If set to `true`, readers will require ZIP64 support and if the file does not + /// exceed the limit, 20 B are wasted. The default is `false`. + #[must_use] + pub const fn large_file(mut self, large: bool) -> Self { + self.large_file = large; + self + } + + pub(crate) fn with_deprecated_encryption(self, password: &[u8]) -> FileOptions<'static, T> { + FileOptions { + encrypt_with: Some(EncryptWith::ZipCrypto( + ZipCryptoKeys::derive(password), + PhantomData, + )), + ..self + } + } + + /// Set the AES encryption parameters. + #[cfg(feature = "aes-crypto")] + pub fn with_aes_encryption(self, mode: AesMode, password: &str) -> FileOptions<'_, T> { + FileOptions { + encrypt_with: Some(EncryptWith::Aes { mode, password }), + ..self + } + } + + /// Sets the size of the buffer used to hold the next block that Zopfli will compress. The + /// larger the buffer, the more effective the compression, but the more memory is required. + /// A value of `None` indicates no buffer, which is recommended only when all non-empty writes + /// are larger than about 32 KiB. + #[must_use] + #[cfg(feature = "deflate-zopfli")] + pub const fn with_zopfli_buffer(mut self, size: Option) -> Self { + self.zopfli_buffer_size = size; + self + } + + /// Returns the compression level currently set. + pub const fn get_compression_level(&self) -> Option { + self.compression_level + } + /// Sets the alignment to the given number of bytes. + #[must_use] + pub const fn with_alignment(mut self, alignment: u16) -> Self { + self.alignment = alignment; + self + } +} +impl FileOptions<'_, ExtendedFileOptions> { + /// Adds an extra data field. + pub fn add_extra_data>( + &mut self, + header_id: u16, + data: D, + central_only: bool, + ) -> ZipResult<()> { + self.extended_options + .add_extra_data(header_id, data, central_only) + } + + /// Removes the extra data fields. + #[must_use] + pub fn clear_extra_data(mut self) -> Self { + if !self.extended_options.extra_data.is_empty() { + self.extended_options.extra_data = Arc::new(vec![]); + } + if !self.extended_options.central_extra_data.is_empty() { + self.extended_options.central_extra_data = Arc::new(vec![]); + } + self + } +} +impl FileOptions<'static, ()> { + /// Constructs a const FileOptions object. + /// + /// Note: This value is different than the return value of [`FileOptions::default()`]: + /// + /// - The `last_modified_time` is [`DateTime::DEFAULT`]. This corresponds to 1980-01-01 00:00:00 + pub const DEFAULT: Self = Self { + compression_method: CompressionMethod::DEFAULT, + compression_level: None, + last_modified_time: DateTime::DEFAULT, + large_file: false, + permissions: None, + encrypt_with: None, + extended_options: (), + alignment: 1, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: Some(1 << 15), + #[cfg(feature = "aes-crypto")] + aes_mode: None, + }; +} + +impl Default for FileOptions<'_, T> { + /// Construct a new FileOptions object + fn default() -> Self { + Self { + compression_method: Default::default(), + compression_level: None, + last_modified_time: DateTime::default_for_write(), + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: T::default(), + alignment: 1, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: Some(1 << 15), + #[cfg(feature = "aes-crypto")] + aes_mode: None, + } + } +} + +impl Write for ZipWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + if !self.writing_to_file { + return Err(io::Error::other("No file has been started")); + } + if buf.is_empty() { + return Ok(0); + } + match self.inner.ref_mut() { + Some(ref mut w) => { + let write_result = w.write(buf); + if let Ok(count) = write_result { + self.stats.update(&buf[..count]); + if self.stats.bytes_written > spec::ZIP64_BYTES_THR + && !self.files.last_mut().unwrap().1.large_file + { + let _ = self.abort_file(); + return Err(io::Error::other("Large file option has not been set")); + } + } + write_result + } + None => Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "write(): ZipWriter was already closed", + )), + } + } + + fn flush(&mut self) -> io::Result<()> { + match self.inner.ref_mut() { + Some(ref mut w) => w.flush(), + None => Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "flush(): ZipWriter was already closed", + )), + } + } +} + +impl ZipWriterStats { + fn update(&mut self, buf: &[u8]) { + self.hasher.update(buf); + self.bytes_written += buf.len() as u64; + } +} + +impl ZipWriter { + /// Initializes the archive from an existing ZIP archive, making it ready for append. + /// + /// This uses a default configuration to initially read the archive. + pub fn new_append(readwriter: A) -> ZipResult> { + Self::new_append_with_config(Default::default(), readwriter) + } + + /// Initializes the archive from an existing ZIP archive, making it ready for append. + /// + /// This uses the given read configuration to initially read the archive. + pub fn new_append_with_config(config: Config, mut readwriter: A) -> ZipResult> { + readwriter.seek(SeekFrom::Start(0))?; + let shared = ZipArchive::get_metadata(config, &mut readwriter)?; + + Ok(ZipWriter { + inner: Storer(MaybeEncrypted::Unencrypted(readwriter)), + files: shared.files, + stats: Default::default(), + writing_to_file: false, + comment: shared.comment, + zip64_comment: shared.zip64_comment, + writing_raw: true, // avoid recomputing the last file's header + flush_on_finish_file: false, + seek_possible: true, + auto_large_file: false, + }) + } + + /// `flush_on_finish_file` is designed to support a streaming `inner` that may unload flushed + /// bytes. It flushes a file's header and body once it starts writing another file. A ZipWriter + /// will not try to seek back into where a previous file was written unless + /// either [`ZipWriter::abort_file`] is called while [`ZipWriter::is_writing_file`] returns + /// false, or [`ZipWriter::deep_copy_file`] is called. In the latter case, it will only need to + /// read previously-written files and not overwrite them. + /// + /// Note: when using an `inner` that cannot overwrite flushed bytes, do not wrap it in a + /// [BufWriter], because that has a [Seek::seek] method that implicitly calls + /// [BufWriter::flush], and ZipWriter needs to seek backward to update each file's header with + /// the size and checksum after writing the body. + /// + /// This setting is false by default. + pub fn set_flush_on_finish_file(&mut self, flush_on_finish_file: bool) { + self.flush_on_finish_file = flush_on_finish_file; + } +} + +impl ZipWriter { + /// Adds another copy of a file already in this archive. This will produce a larger but more + /// widely-compatible archive compared to [Self::shallow_copy_file]. Does not copy alignment. + pub fn deep_copy_file(&mut self, src_name: &str, dest_name: &str) -> ZipResult<()> { + self.finish_file()?; + if src_name == dest_name || self.files.contains_key(dest_name) { + return Err(invalid!("That file already exists")); + } + let write_position = self.inner.get_plain().stream_position()?; + let src_index = self.index_by_name(src_name)?; + let src_data = &mut self.files[src_index]; + let src_data_start = src_data.data_start(self.inner.get_plain())?; + debug_assert!(src_data_start <= write_position); + let mut compressed_size = src_data.compressed_size; + if compressed_size > (write_position - src_data_start) { + compressed_size = write_position - src_data_start; + src_data.compressed_size = compressed_size; + } + let mut reader = BufReader::new(self.inner.get_plain()); + reader.seek(SeekFrom::Start(src_data_start))?; + let mut copy = vec![0; compressed_size as usize]; + reader.take(compressed_size).read_exact(&mut copy)?; + self.inner + .get_plain() + .seek(SeekFrom::Start(write_position))?; + let mut new_data = src_data.clone(); + let dest_name_raw = dest_name.as_bytes(); + new_data.file_name = dest_name.into(); + new_data.file_name_raw = dest_name_raw.into(); + new_data.is_utf8 = !dest_name.is_ascii(); + new_data.header_start = write_position; + let extra_data_start = write_position + + size_of::() as u64 + + new_data.file_name_raw.len() as u64; + new_data.extra_data_start = Some(extra_data_start); + if let Some(extra) = &src_data.extra_field { + let stripped = strip_alignment_extra_field(extra); + if !stripped.is_empty() { + new_data.extra_field = Some(stripped.into()); + } else { + new_data.extra_field = None; + } + } + + let mut data_start = extra_data_start; + if let Some(extra) = &new_data.extra_field { + data_start += extra.len() as u64; + } + new_data.data_start.take(); + new_data.data_start.get_or_init(|| data_start); + new_data.central_header_start = 0; + let block = new_data.local_block()?; + let index = self.insert_file_data(new_data)?; + let new_data = &self.files[index]; + let result: io::Result<()> = { + let plain_writer = self.inner.get_plain(); + block.write(plain_writer)?; + plain_writer.write_all(&new_data.file_name_raw)?; + if let Some(data) = &new_data.extra_field { + plain_writer.write_all(data)?; + } + debug_assert_eq!(data_start, plain_writer.stream_position()?); + self.writing_to_file = true; + plain_writer.write_all(©)?; + if self.flush_on_finish_file { + plain_writer.flush()?; + } + Ok(()) + }; + self.ok_or_abort_file(result)?; + self.writing_to_file = false; + Ok(()) + } + + /// Like `deep_copy_file`, but uses Path arguments. + /// + /// This function ensures that the '/' path separator is used and normalizes `.` and `..`. It + /// ignores any `..` or Windows drive letter that would produce a path outside the ZIP file's + /// root. + pub fn deep_copy_file_from_path, U: AsRef>( + &mut self, + src_path: T, + dest_path: U, + ) -> ZipResult<()> { + let src = path_to_string(src_path); + let dest = path_to_string(dest_path); + self.deep_copy_file(&src, &dest) + } + + /// Write the zip file into the backing stream, then produce a readable archive of that data. + /// + /// This method avoids parsing the central directory records at the end of the stream for + /// a slight performance improvement over running [`ZipArchive::new()`] on the output of + /// [`Self::finish()`]. + /// + ///``` + /// # fn main() -> Result<(), zip::result::ZipError> { + /// # #[cfg(any(feature = "deflate-flate2", not(feature = "_deflate-any")))] + /// # { + /// use std::io::{Cursor, Read, Write}; + /// use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions}; + /// + /// let buf = Cursor::new(Vec::new()); + /// let mut zip = ZipWriter::new(buf); + /// let options = SimpleFileOptions::default(); + /// zip.start_file("a.txt", options)?; + /// zip.write_all(b"hello\n")?; + /// + /// let mut zip = zip.finish_into_readable()?; + /// let mut s: String = String::new(); + /// zip.by_name("a.txt")?.read_to_string(&mut s)?; + /// assert_eq!(s, "hello\n"); + /// # } + /// # Ok(()) + /// # } + ///``` + pub fn finish_into_readable(mut self) -> ZipResult> { + let central_start = self.finalize()?; + let inner = mem::replace(&mut self.inner, Closed).unwrap(); + let comment = mem::take(&mut self.comment); + let zip64_comment = mem::take(&mut self.zip64_comment); + let files = mem::take(&mut self.files); + + let archive = + ZipArchive::from_finalized_writer(files, comment, zip64_comment, inner, central_start)?; + Ok(archive) + } +} + +impl ZipWriter { + /// Initializes the archive. + /// + /// Before writing to this object, the [`ZipWriter::start_file`] function should be called. + /// After a successful write, the file remains open for writing. After a failed write, call + /// [`ZipWriter::is_writing_file`] to determine if the file remains open. + pub fn new(inner: W) -> ZipWriter { + ZipWriter { + inner: Storer(MaybeEncrypted::Unencrypted(inner)), + files: IndexMap::new(), + stats: Default::default(), + writing_to_file: false, + writing_raw: false, + comment: Box::new([]), + zip64_comment: None, + flush_on_finish_file: false, + seek_possible: true, + auto_large_file: false, + } + } + + /// Set automatically large file to true if needed + pub fn set_auto_large_file(mut self) -> Self { + self.auto_large_file = true; + self + } + + /// Returns true if a file is currently open for writing. + pub const fn is_writing_file(&self) -> bool { + self.writing_to_file && !self.inner.is_closed() + } + + /// Set ZIP archive comment. + pub fn set_comment(&mut self, comment: S) + where + S: Into>, + { + self.set_raw_comment(comment.into().into_boxed_bytes()) + } + + /// Set ZIP archive comment. + /// + /// This sets the raw bytes of the comment. The comment + /// is typically expected to be encoded in UTF-8. + pub fn set_raw_comment(&mut self, comment: Box<[u8]>) { + let max_comment_len = u16::MAX as usize; // 65,535 + if comment.len() > max_comment_len { + self.set_raw_zip64_comment(Some(comment)); + self.comment = Box::new([]); + } else { + self.comment = comment; + self.set_raw_zip64_comment(None); + } + } + + /// Get ZIP archive comment. + pub fn get_comment(&mut self) -> Result<&str, Utf8Error> { + from_utf8(self.get_raw_comment()) + } + + /// Get ZIP archive comment. + /// + /// This returns the raw bytes of the comment. The comment + /// is typically expected to be encoded in UTF-8. + pub const fn get_raw_comment(&self) -> &[u8] { + &self.comment + } + + /// Set ZIP64 archive comment. + pub fn set_zip64_comment(&mut self, comment: Option) + where + S: Into>, + { + self.set_raw_zip64_comment(comment.map(|v| v.into().into_boxed_bytes())) + } + + /// Set ZIP64 archive comment. + /// + /// This sets the raw bytes of the comment. The comment + /// is typically expected to be encoded in UTF-8. + pub fn set_raw_zip64_comment(&mut self, comment: Option>) { + self.zip64_comment = comment; + } + + /// Get ZIP64 archive comment. + pub fn get_zip64_comment(&mut self) -> Option> { + self.get_raw_zip64_comment().map(from_utf8) + } + + /// Get ZIP archive comment. + /// + /// This returns the raw bytes of the comment. The comment + /// is typically expected to be encoded in UTF-8. + pub fn get_raw_zip64_comment(&self) -> Option<&[u8]> { + self.zip64_comment.as_deref() + } + + /// Set the file length and crc32 manually. + /// + /// # Safety + /// + /// This overwrites the internal crc32 calculation. It should only be used in case + /// the underlying [Write] is written independently and you need to adjust the zip metadata. + pub unsafe fn set_file_metadata(&mut self, length: u64, crc32: u32) -> ZipResult<()> { + if !self.writing_to_file { + return Err(ZipError::Io(io::Error::other("No file has been started"))); + } + self.stats.hasher = Hasher::new_with_initial_len(crc32, length); + self.stats.bytes_written = length; + Ok(()) + } + + fn ok_or_abort_file>(&mut self, result: Result) -> ZipResult { + match result { + Err(e) => { + let _ = self.abort_file(); + Err(e.into()) + } + Ok(t) => Ok(t), + } + } + + /// Start a new file for with the requested options. + fn start_entry( + &mut self, + name: S, + mut options: FileOptions, + raw_values: Option, + ) -> ZipResult<()> { + self.finish_file()?; + + let header_start = self.inner.get_plain().stream_position()?; + let raw_values = raw_values.unwrap_or(ZipRawValues { + crc32: 0, + compressed_size: 0, + uncompressed_size: 0, + }); + + // Check if we're close to the 4GB boundary and force ZIP64 if needed + // This ensures we properly handle appending to files close to 4GB + if header_start > spec::ZIP64_BYTES_THR { + // Files that start on or past the 4GiB boundary are always ZIP64 + options.large_file = true; + } + + let mut extra_data = match options.extended_options.extra_data() { + Some(data) => data.to_vec(), + None => vec![], + }; + let central_extra_data = options.extended_options.central_extra_data(); + if let Some(zip64_block) = + Zip64ExtraFieldBlock::maybe_new(options.large_file, 0, 0, header_start) + { + let mut new_extra_data = zip64_block.serialize().into_vec(); + new_extra_data.append(&mut extra_data); + extra_data = new_extra_data; + } + + // Figure out the underlying compression_method and aes mode when using + // AES encryption. + let (compression_method, aes_mode) = match options.encrypt_with { + // Preserve AES method for raw copies without needing a password + #[cfg(feature = "aes-crypto")] + None if options.aes_mode.is_some() => (CompressionMethod::Aes, options.aes_mode), + #[cfg(feature = "aes-crypto")] + Some(EncryptWith::Aes { mode, .. }) => ( + CompressionMethod::Aes, + Some((mode, AesVendorVersion::Ae2, options.compression_method)), + ), + _ => (options.compression_method, None), + }; + + // Write AES encryption extra data. + #[allow(unused_mut)] + let mut aes_extra_data_start = 0; + #[cfg(feature = "aes-crypto")] + if let Some((mode, vendor, underlying)) = aes_mode { + // For raw copies of AES entries, write the correct AES extra data immediately + let mut body = [0; 7]; + [body[0], body[1]] = (vendor as u16).to_le_bytes(); // vendor version (1 or 2) + [body[2], body[3]] = *b"AE"; // vendor id + body[4] = mode as u8; // strength + [body[5], body[6]] = underlying.serialize_to_u16().to_le_bytes(); // real compression method + aes_extra_data_start = extra_data.len() as u64; + ExtendedFileOptions::add_extra_data_unchecked(&mut extra_data, 0x9901, &body)?; + } + let header_end = + header_start + size_of::() as u64 + name.to_string().len() as u64; + + if options.alignment > 1 { + let extra_data_end = header_end + extra_data.len() as u64; + let align = options.alignment as u64; + let unaligned_header_bytes = extra_data_end % align; + if unaligned_header_bytes != 0 { + let mut pad_length = (align - unaligned_header_bytes) as usize; + while pad_length < 6 { + pad_length += align as usize; + } + let new_len = extra_data.len() + pad_length; + if new_len > u16::MAX as usize { + // Alignment is impossible without exceeding extra field size limits. + // Skip alignment. + } else { + // Add an extra field to the extra_data, per APPNOTE 4.6.11 + let mut pad_body = vec![0; pad_length - 4]; + debug_assert!(pad_body.len() >= 2); + [pad_body[0], pad_body[1]] = options.alignment.to_le_bytes(); + ExtendedFileOptions::add_extra_data_unchecked( + &mut extra_data, + 0xa11e, + &pad_body, + )?; + debug_assert_eq!((extra_data.len() as u64 + header_end) % align, 0); + } + } + } + let extra_data_len = extra_data.len(); + if let Some(data) = central_extra_data { + if extra_data_len + data.len() > u16::MAX as usize { + return Err(invalid!( + "Extra data and central extra data must be less than 64KiB when combined" + )); + } + ExtendedFileOptions::validate_extra_data(data, true)?; + } + let mut file = ZipFileData::initialize_local_block( + name, + &options, + raw_values, + header_start, + None, + aes_extra_data_start, + compression_method, + aes_mode, + &extra_data, + ); + file.using_data_descriptor = + !self.seek_possible || matches!(options.encrypt_with, Some(EncryptWith::ZipCrypto(..))); + file.version_made_by = file.version_made_by.max(file.version_needed() as u8); + file.extra_data_start = Some(header_end); + let index = self.insert_file_data(file)?; + self.writing_to_file = true; + let result: ZipResult<()> = { + ExtendedFileOptions::validate_extra_data(&extra_data, false)?; + let file = &mut self.files[index]; + let block = file.local_block()?; + let writer = self.inner.get_plain(); + block.write(writer)?; + // file name + writer.write_all(&file.file_name_raw)?; + if extra_data_len > 0 { + writer.write_all(&extra_data)?; + file.extra_field = Some(extra_data.into()); + } + Ok(()) + }; + self.ok_or_abort_file(result)?; + let writer = self.inner.get_plain(); + self.stats.start = writer.stream_position()?; + match options.encrypt_with { + #[cfg(feature = "aes-crypto")] + Some(EncryptWith::Aes { mode, password }) => { + let aeswriter = AesWriter::new( + mem::replace(&mut self.inner, Closed).unwrap(), + mode, + password.as_bytes(), + )?; + self.inner = Storer(MaybeEncrypted::Aes(aeswriter)); + } + Some(EncryptWith::ZipCrypto(keys, ..)) => { + let file = &mut self.files[index]; + // With ZipCrypto, we _need_ to use a data descriptor so that + // we can initialize the stream properly. + let mut zipwriter = crate::zipcrypto::ZipCryptoWriter { + writer: mem::replace(&mut self.inner, Closed).unwrap(), + keys, + }; + self.stats.start = zipwriter.writer.stream_position()?; + // crypto_header is counted as part of the data + let mut crypto_header = [0u8; 12]; + // The last two bytes of the header should either be the CRC of + // the file _OR_ the last timepart of the last modified time if + // a data descriptor is used. + // + // However, this header is encrypted, and ZipCrypto is not a + // seekable encryption algorithm - an earlier plaintext byte + // will have an impact on the later values. + // + // This makes it impossible to write the file without first + // calculating its CRC. To avoid having to read the file twice + // or keeping the entire file in memory, we force all ZipCrypto + // files to use a data descriptor. This way, we can use the + // local_modified_time as a password check byte instead of the + // CRC. + crypto_header[10..=11].copy_from_slice( + &file + .last_modified_time + .unwrap_or_else(DateTime::default_for_write) + .timepart() + .to_le_bytes(), + ); + let result = zipwriter.write_all(&crypto_header); + self.ok_or_abort_file(result)?; + self.inner = Storer(MaybeEncrypted::ZipCrypto(zipwriter)); + } + None => {} + } + let file = &mut self.files[index]; + debug_assert!(file.data_start.get().is_none()); + file.data_start.get_or_init(|| self.stats.start); + self.stats.bytes_written = 0; + self.stats.hasher = Hasher::new(); + Ok(()) + } + + fn insert_file_data(&mut self, file: ZipFileData) -> ZipResult { + if self.files.contains_key(&file.file_name) { + return Err(invalid!("Duplicate filename: {}", file.file_name)); + } + let (index, _) = self.files.insert_full(file.file_name.clone(), file); + Ok(index) + } + + fn finish_file(&mut self) -> ZipResult<()> { + if !self.writing_to_file { + return Ok(()); + } + + let make_plain_writer = self.inner.prepare_next_writer( + Stored, + None, + #[cfg(feature = "deflate-zopfli")] + None, + )?; + self.inner.switch_to(make_plain_writer)?; + self.switch_to_non_encrypting_writer()?; + let writer = self.inner.get_plain(); + + if !self.writing_raw { + let file = match self.files.last_mut() { + None => return Ok(()), + Some((_, f)) => f, + }; + file.uncompressed_size = self.stats.bytes_written; + + let file_end = writer.stream_position()?; + debug_assert!(file_end >= self.stats.start); + file.compressed_size = file_end - self.stats.start; + + if !file.using_data_descriptor { + // Not using a data descriptor means the underlying writer + // supports seeking, so we can go back and update the AES Extra + // Data header to use AE1 for large files. + update_aes_extra_data(writer, file, self.stats.bytes_written)?; + } + + let crc = !matches!(file.aes_mode, Some((_, AesVendorVersion::Ae2, _))); + + file.crc32 = if crc { + self.stats.hasher.clone().finalize() + } else { + 0 + }; + + if file.using_data_descriptor { + file.write_data_descriptor(writer, self.auto_large_file)?; + } else { + update_local_file_header(writer, file)?; + writer.seek(SeekFrom::Start(file_end))?; + } + } + if self.flush_on_finish_file { + let result = writer.flush(); + self.ok_or_abort_file(result)?; + } + + self.writing_to_file = false; + Ok(()) + } + + fn switch_to_non_encrypting_writer(&mut self) -> Result<(), ZipError> { + match mem::replace(&mut self.inner, Closed) { + #[cfg(feature = "aes-crypto")] + Storer(MaybeEncrypted::Aes(writer)) => { + self.inner = Storer(MaybeEncrypted::Unencrypted(writer.finish()?)); + } + Storer(MaybeEncrypted::ZipCrypto(writer)) => { + self.inner = Storer(MaybeEncrypted::Unencrypted(writer.finish()?)) + } + Storer(MaybeEncrypted::Unencrypted(w)) => { + self.inner = Storer(MaybeEncrypted::Unencrypted(w)) + } + _ => unreachable!(), + } + Ok(()) + } + + /// Removes the file currently being written from the archive if there is one, or else removes + /// the file most recently written. + pub fn abort_file(&mut self) -> ZipResult<()> { + let (_, last_file) = self.files.pop().ok_or(ZipError::FileNotFound)?; + let make_plain_writer = self.inner.prepare_next_writer( + Stored, + None, + #[cfg(feature = "deflate-zopfli")] + None, + )?; + self.inner.switch_to(make_plain_writer)?; + self.switch_to_non_encrypting_writer()?; + // Make sure this is the last file, and that no shallow copies of it remain; otherwise we'd + // overwrite a valid file and corrupt the archive + let rewind_safe: bool = match last_file.data_start.get() { + None => self.files.is_empty(), + Some(last_file_start) => self.files.values().all(|file| { + file.data_start + .get() + .is_some_and(|start| start < last_file_start) + }), + }; + if rewind_safe { + self.inner + .get_plain() + .seek(SeekFrom::Start(last_file.header_start))?; + } + self.writing_to_file = false; + Ok(()) + } + + /// Create a file in the archive and start writing its' contents. The file must not have the + /// same name as a file already in the archive. + /// + /// The data should be written using the [`Write`] implementation on this [`ZipWriter`] + pub fn start_file( + &mut self, + name: S, + mut options: FileOptions, + ) -> ZipResult<()> { + options.normalize(); + let make_new_self = self.inner.prepare_next_writer( + options.compression_method, + options.compression_level, + #[cfg(feature = "deflate-zopfli")] + options.zopfli_buffer_size, + )?; + self.start_entry(name, options, None)?; + let result = self.inner.switch_to(make_new_self); + self.ok_or_abort_file(result)?; + self.writing_raw = false; + Ok(()) + } + + /* TODO: link to/use Self::finish_into_readable() from https://github.com/zip-rs/zip/pull/400 in + * this docstring. */ + /// Copy over the entire contents of another archive verbatim. + /// + /// This method extracts file metadata from the `source` archive, then simply performs a single + /// big [`io::copy()`](io::copy) to transfer all the actual file contents without any + /// decompression or decryption. This is more performant than the equivalent operation of + /// calling [`Self::raw_copy_file()`] for each entry from the `source` archive in sequence. + /// + ///``` + /// # fn main() -> Result<(), zip::result::ZipError> { + /// # #[cfg(any(feature = "deflate-flate2", not(feature = "_deflate-any")))] + /// # { + /// use std::io::{Cursor, Write, Read}; + /// use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions}; + /// + /// let buf = Cursor::new(Vec::new()); + /// let mut zip = ZipWriter::new(buf); + /// zip.start_file("a.txt", SimpleFileOptions::default())?; + /// zip.write_all(b"hello\n")?; + /// let src = ZipArchive::new(zip.finish()?)?; + /// + /// let buf = Cursor::new(Vec::new()); + /// let mut zip = ZipWriter::new(buf); + /// zip.start_file("b.txt", SimpleFileOptions::default())?; + /// zip.write_all(b"hey\n")?; + /// let src2 = ZipArchive::new(zip.finish()?)?; + /// + /// let buf = Cursor::new(Vec::new()); + /// + /// let mut zip = ZipWriter::new(buf); + /// zip.merge_archive(src)?; + /// zip.merge_archive(src2)?; + /// let mut result = ZipArchive::new(zip.finish()?)?; + /// + /// let mut s: String = String::new(); + /// result.by_name("a.txt")?.read_to_string(&mut s)?; + /// assert_eq!(s, "hello\n"); + /// s.clear(); + /// result.by_name("b.txt")?.read_to_string(&mut s)?; + /// assert_eq!(s, "hey\n"); + /// # } + /// # Ok(()) + /// # } + ///``` + pub fn merge_archive(&mut self, mut source: ZipArchive) -> ZipResult<()> + where + R: Read + Seek, + { + self.finish_file()?; + + /* Ensure we accept the file contents on faith (and avoid overwriting the data). + * See raw_copy_file_rename(). */ + self.writing_to_file = true; + self.writing_raw = true; + + let writer = self.inner.get_plain(); + /* Get the file entries from the source archive. */ + let new_files = source.merge_contents(writer)?; + + /* These file entries are now ours! */ + self.files.extend(new_files); + + Ok(()) + } + + /// Starts a file, taking a Path as argument. + /// + /// This function ensures that the '/' path separator is used and normalizes `.` and `..`. It + /// ignores any `..` or Windows drive letter that would produce a path outside the ZIP file's + /// root. + pub fn start_file_from_path>( + &mut self, + path: P, + options: FileOptions, + ) -> ZipResult<()> { + self.start_file(path_to_string(path), options) + } + + /// Add a new file using the already compressed data from a ZIP file being read and renames it, this + /// allows faster copies of the `ZipFile` since there is no need to decompress and compress it again. + /// Any `ZipFile` metadata is copied and not checked, for example the file CRC. + /// + /// ```no_run + /// use std::fs::File; + /// use std::io::{Read, Seek, Write}; + /// use zip::{ZipArchive, ZipWriter}; + /// + /// fn copy_rename( + /// src: &mut ZipArchive, + /// dst: &mut ZipWriter, + /// ) -> zip::result::ZipResult<()> + /// where + /// R: Read + Seek, + /// W: Write + Seek, + /// { + /// // Retrieve file entry by name + /// let file = src.by_name("src_file.txt")?; + /// + /// // Copy and rename the previously obtained file entry to the destination zip archive + /// dst.raw_copy_file_rename(file, "new_name.txt")?; + /// + /// Ok(()) + /// } + /// ``` + pub fn raw_copy_file_rename( + &mut self, + file: ZipFile, + name: S, + ) -> ZipResult<()> { + let options = file.options(); + self.raw_copy_file_rename_internal(file, name, options) + } + + fn raw_copy_file_rename_internal( + &mut self, + mut file: ZipFile, + name: S, + options: SimpleFileOptions, + ) -> ZipResult<()> { + let raw_values = ZipRawValues { + crc32: file.crc32(), + compressed_size: file.compressed_size(), + uncompressed_size: file.size(), + }; + + self.start_entry(name, options, Some(raw_values))?; + self.writing_to_file = true; + self.writing_raw = true; + + io::copy(&mut file.take_raw_reader()?, self)?; + self.finish_file() + } + + /// Like `raw_copy_file_to_path`, but uses Path arguments. + /// + /// This function ensures that the '/' path separator is used and normalizes `.` and `..`. It + /// ignores any `..` or Windows drive letter that would produce a path outside the ZIP file's + /// root. + pub fn raw_copy_file_to_path>( + &mut self, + file: ZipFile, + path: P, + ) -> ZipResult<()> { + self.raw_copy_file_rename(file, path_to_string(path)) + } + + /// Add a new file using the already compressed data from a ZIP file being read, this allows faster + /// copies of the `ZipFile` since there is no need to decompress and compress it again. Any `ZipFile` + /// metadata is copied and not checked, for example the file CRC. + /// + /// ```no_run + /// use std::fs::File; + /// use std::io::{Read, Seek, Write}; + /// use zip::{ZipArchive, ZipWriter}; + /// + /// fn copy(src: &mut ZipArchive, dst: &mut ZipWriter) -> zip::result::ZipResult<()> + /// where + /// R: Read + Seek, + /// W: Write + Seek, + /// { + /// // Retrieve file entry by name + /// let file = src.by_name("src_file.txt")?; + /// + /// // Copy the previously obtained file entry to the destination zip archive + /// dst.raw_copy_file(file)?; + /// + /// Ok(()) + /// } + /// ``` + pub fn raw_copy_file(&mut self, file: ZipFile) -> ZipResult<()> { + let name = file.name().to_owned(); + self.raw_copy_file_rename(file, name) + } + + /// Add a new file using the already compressed data from a ZIP file being read and set the last + /// modified date and unix mode. This allows faster copies of the `ZipFile` since there is no need + /// to decompress and compress it again. Any `ZipFile` metadata other than the last modified date + /// and the unix mode is copied and not checked, for example the file CRC. + /// + /// ```no_run + /// use std::io::{Read, Seek, Write}; + /// use zip::{DateTime, ZipArchive, ZipWriter}; + /// + /// fn copy(src: &mut ZipArchive, dst: &mut ZipWriter) -> zip::result::ZipResult<()> + /// where + /// R: Read + Seek, + /// W: Write + Seek, + /// { + /// // Retrieve file entry by name + /// let file = src.by_name("src_file.txt")?; + /// + /// // Copy the previously obtained file entry to the destination zip archive + /// dst.raw_copy_file_touch(file, DateTime::default(), Some(0o644))?; + /// + /// Ok(()) + /// } + /// ``` + pub fn raw_copy_file_touch( + &mut self, + file: ZipFile, + last_modified_time: DateTime, + unix_mode: Option, + ) -> ZipResult<()> { + let name = file.name().to_owned(); + + let mut options = file.options(); + + options = options.last_modified_time(last_modified_time); + + if let Some(perms) = unix_mode { + options = options.unix_permissions(perms); + } + + options.normalize(); + + self.raw_copy_file_rename_internal(file, name, options) + } + + /// Add a directory entry. + /// + /// As directories have no content, you must not call [`ZipWriter::write`] before adding a new file. + pub fn add_directory( + &mut self, + name: S, + mut options: FileOptions, + ) -> ZipResult<()> + where + S: Into, + { + if options.permissions.is_none() { + options.permissions = Some(0o755); + } + *options.permissions.as_mut().unwrap() |= 0o40000; + options.compression_method = Stored; + options.encrypt_with = None; + + let name_as_string = name.into(); + // Append a slash to the filename if it does not end with it. + let name_with_slash = match name_as_string.chars().last() { + Some('/') | Some('\\') => name_as_string, + _ => name_as_string + "/", + }; + + self.start_entry(name_with_slash, options, None)?; + self.writing_to_file = false; + self.switch_to_non_encrypting_writer()?; + Ok(()) + } + + /// Add a directory entry, taking a Path as argument. + /// + /// This function ensures that the '/' path separator is used and normalizes `.` and `..`. It + /// ignores any `..` or Windows drive letter that would produce a path outside the ZIP file's + /// root. + pub fn add_directory_from_path>( + &mut self, + path: P, + options: FileOptions, + ) -> ZipResult<()> { + self.add_directory(path_to_string(path), options) + } + + /// Finish the last file and write all other zip-structures + /// + /// This will return the writer, but one should normally not append any data to the end of the file. + /// Note that the zipfile will also be finished on drop. + pub fn finish(mut self) -> ZipResult { + let _central_start = self.finalize()?; + let inner = mem::replace(&mut self.inner, Closed); + Ok(inner.unwrap()) + } + + /// Add a symlink entry. + /// + /// The zip archive will contain an entry for path `name` which is a symlink to `target`. + /// + /// No validation or normalization of the paths is performed. For best results, + /// callers should normalize `\` to `/` and ensure symlinks are relative to other + /// paths within the zip archive. + /// + /// WARNING: not all zip implementations preserve symlinks on extract. Some zip + /// implementations may materialize a symlink as a regular file, possibly with the + /// content incorrectly set to the symlink target. For maximum portability, consider + /// storing a regular file instead. + pub fn add_symlink( + &mut self, + name: N, + target: T, + mut options: FileOptions, + ) -> ZipResult<()> { + if options.permissions.is_none() { + options.permissions = Some(0o777); + } + *options.permissions.as_mut().unwrap() |= S_IFLNK; + // The symlink target is stored as file content. And compressing the target path + // likely wastes space. So always store. + options.compression_method = Stored; + + self.start_entry(name, options, None)?; + self.writing_to_file = true; + let result = self.write_all(target.to_string().as_bytes()); + self.ok_or_abort_file(result)?; + self.writing_raw = false; + self.finish_file()?; + + Ok(()) + } + + /// Add a symlink entry, taking Paths to the location and target as arguments. + /// + /// This function ensures that the '/' path separator is used and normalizes `.` and `..`. It + /// ignores any `..` or Windows drive letter that would produce a path outside the ZIP file's + /// root. + pub fn add_symlink_from_path, T: AsRef, E: FileOptionExtension>( + &mut self, + path: P, + target: T, + options: FileOptions, + ) -> ZipResult<()> { + self.add_symlink(path_to_string(path), path_to_string(target), options) + } + + fn finalize(&mut self) -> ZipResult { + self.finish_file()?; + + let mut central_start = self.write_central_and_footer()?; + let writer = self.inner.get_plain(); + let footer_end = writer.stream_position()?; + let archive_end = writer.seek(SeekFrom::End(0))?; + if footer_end < archive_end { + // Data from an aborted file is past the end of the footer. + + // Overwrite the magic so the footer is no longer valid. + writer.seek(SeekFrom::Start(central_start))?; + writer.write_u32_le(0)?; + writer.seek(SeekFrom::Start( + footer_end - size_of::() as u64 - self.comment.len() as u64, + ))?; + writer.write_u32_le(0)?; + + // Rewrite the footer at the actual end. + let central_and_footer_size = footer_end - central_start; + writer.seek(SeekFrom::End(-(central_and_footer_size as i64)))?; + central_start = self.write_central_and_footer()?; + debug_assert!(self.inner.get_plain().stream_position()? == archive_end); + } + + Ok(central_start) + } + + fn write_central_and_footer(&mut self) -> Result { + let writer = self.inner.get_plain(); + + let mut version_needed = MIN_VERSION as u16; + let central_start = writer.stream_position()?; + for file in self.files.values() { + write_central_directory_header(writer, file)?; + version_needed = version_needed.max(file.version_needed()); + } + let central_size = writer.stream_position()? - central_start; + let is64 = self.files.len() > spec::ZIP64_ENTRY_THR + || central_size.max(central_start) > spec::ZIP64_BYTES_THR + || self.zip64_comment.is_some(); + + if is64 { + let comment = self.zip64_comment.clone().unwrap_or_default(); + + let zip64_footer = spec::Zip64CentralDirectoryEnd { + record_size: comment.len() as u64 + 44, + version_made_by: version_needed, + version_needed_to_extract: version_needed, + disk_number: 0, + disk_with_central_directory: 0, + number_of_files_on_this_disk: self.files.len() as u64, + number_of_files: self.files.len() as u64, + central_directory_size: central_size, + central_directory_offset: central_start, + extensible_data_sector: comment, + }; + + zip64_footer.write(writer)?; + + let zip64_footer = spec::Zip64CentralDirectoryEndLocator { + disk_with_central_directory: 0, + end_of_central_directory_offset: central_start + central_size, + number_of_disks: 1, + }; + + zip64_footer.write(writer)?; + } + + let number_of_files = self.files.len().min(spec::ZIP64_ENTRY_THR) as u16; + let footer = spec::Zip32CentralDirectoryEnd { + disk_number: 0, + disk_with_central_directory: 0, + zip_file_comment: self.comment.clone(), + number_of_files_on_this_disk: number_of_files, + number_of_files, + central_directory_size: central_size.min(spec::ZIP64_BYTES_THR) as u32, + central_directory_offset: central_start.min(spec::ZIP64_BYTES_THR) as u32, + }; + + footer.write(writer)?; + Ok(central_start) + } + + fn index_by_name(&self, name: &str) -> ZipResult { + self.files.get_index_of(name).ok_or(ZipError::FileNotFound) + } + + /// Adds another entry to the central directory referring to the same content as an existing + /// entry. The file's local-file header will still refer to it by its original name, so + /// unzipping the file will technically be unspecified behavior. [ZipArchive] ignores the + /// filename in the local-file header and treat the central directory as authoritative. However, + /// some other software (e.g. Minecraft) will refuse to extract a file copied this way. + pub fn shallow_copy_file(&mut self, src_name: &str, dest_name: &str) -> ZipResult<()> { + self.finish_file()?; + if src_name == dest_name { + return Err(invalid!("Trying to copy a file to itself")); + } + let src_index = self.index_by_name(src_name)?; + let mut dest_data = self.files[src_index].to_owned(); + dest_data.file_name = dest_name.to_string().into(); + dest_data.file_name_raw = dest_name.to_string().into_bytes().into(); + dest_data.central_header_start = 0; + self.insert_file_data(dest_data)?; + + Ok(()) + } + + /// Like `shallow_copy_file`, but uses Path arguments. + /// + /// This function ensures that the '/' path separator is used and normalizes `.` and `..`. It + /// ignores any `..` or Windows drive letter that would produce a path outside the ZIP file's + /// root. + pub fn shallow_copy_file_from_path, U: AsRef>( + &mut self, + src_path: T, + dest_path: U, + ) -> ZipResult<()> { + self.shallow_copy_file(&path_to_string(src_path), &path_to_string(dest_path)) + } +} + +impl ZipWriter> { + /// Creates a writer that doesn't require the inner writer to implement [Seek], but where + /// operations that would overwrite previously-written bytes or cause subsequent operations to + /// do so (such as `abort_file`) will always return an error. + pub fn new_stream(inner: W) -> ZipWriter> { + ZipWriter { + inner: Storer(MaybeEncrypted::Unencrypted(StreamWriter::new(inner))), + files: IndexMap::new(), + stats: Default::default(), + writing_to_file: false, + writing_raw: false, + comment: Box::new([]), + zip64_comment: None, + flush_on_finish_file: false, + seek_possible: false, + auto_large_file: false, + } + } +} + +impl Drop for ZipWriter { + fn drop(&mut self) { + if !self.inner.is_closed() { + if let Err(e) = self.finalize() { + let _ = write!( + io::stderr(), + "ZipWriter::drop: failed to finalize archive: {e:?}" + ); + } + } + } +} + +type SwitchWriterFunction = Box) -> ZipResult>>; + +impl GenericZipWriter { + fn prepare_next_writer( + &self, + compression: CompressionMethod, + compression_level: Option, + #[cfg(feature = "deflate-zopfli")] zopfli_buffer_size: Option, + ) -> ZipResult> { + if let Closed = self { + return Err( + io::Error::new(io::ErrorKind::BrokenPipe, "ZipWriter was already closed").into(), + ); + } + + match compression { + Stored => { + if compression_level.is_some() { + Err(UnsupportedArchive("Unsupported compression level")) + } else { + Ok(Box::new(|bare| Ok(Storer(bare)))) + } + } + #[allow(unreachable_code)] + #[cfg(feature = "_deflate-any")] + CompressionMethod::Deflated => { + let default = crate::cfg_if_expr! { + i64: + #[cfg(feature = "deflate-flate2")] => Compression::default().level() as i64, + #[cfg(feature = "deflate-zopfli")] => 24, + _ => compile_error!("could not calculate default: unknown deflate variant"), + }; + + let level = validate_value_in_range( + compression_level.unwrap_or(default), + deflate_compression_level_range(), + ) + .ok_or(UnsupportedArchive("Unsupported compression level"))? + as u32; + + #[cfg(feature = "deflate-zopfli")] + { + macro_rules! deflate_zopfli_and_return { + ($bare:expr, $best_non_zopfli:expr) => {{ + let options = Options { + iteration_count: NonZeroU64::try_from( + (level - $best_non_zopfli) as u64, + ) + .unwrap(), + ..Default::default() + }; + Ok(Box::new(move |bare| { + Ok(match zopfli_buffer_size { + Some(size) => BufferedZopfliDeflater(BufWriter::with_capacity( + size, + zopfli::DeflateEncoder::new( + options, + Default::default(), + bare, + ), + )), + None => ZopfliDeflater(zopfli::DeflateEncoder::new( + options, + Default::default(), + bare, + )), + }) + })) + }}; + } + + crate::cfg_if! { + if #[cfg(feature = "deflate-flate2")] { + let best_non_zopfli = Compression::best().level(); + if level > best_non_zopfli { + return deflate_zopfli_and_return!(bare, best_non_zopfli); + } + } else { + let best_non_zopfli = 9; + return deflate_zopfli_and_return!(bare, best_non_zopfli); + } + } + } + crate::cfg_if_expr! { + ZipResult>: + #[cfg(feature = "deflate-flate2")] => Ok(Box::new(move |bare| { + Ok(GenericZipWriter::Deflater(DeflateEncoder::new( + bare, + Compression::new(level), + ))) + })), + _ => unreachable!("deflate writer: have no fallback for this case") + } + } + #[cfg(feature = "deflate64")] + CompressionMethod::Deflate64 => { + Err(UnsupportedArchive("Compressing Deflate64 is not supported")) + } + #[cfg(feature = "_bzip2_any")] + CompressionMethod::Bzip2 => { + let level = validate_value_in_range( + compression_level.unwrap_or(bzip2::Compression::default().level() as i64), + bzip2_compression_level_range(), + ) + .ok_or(UnsupportedArchive("Unsupported compression level"))? + as u32; + Ok(Box::new(move |bare| { + Ok(Bzip2(BzEncoder::new(bare, bzip2::Compression::new(level)))) + })) + } + CompressionMethod::AES => Err(UnsupportedArchive( + "AES encryption is enabled through FileOptions::with_aes_encryption", + )), + #[cfg(feature = "zstd")] + CompressionMethod::Zstd => { + let level = validate_value_in_range( + compression_level.unwrap_or(zstd::DEFAULT_COMPRESSION_LEVEL as i64), + zstd::compression_level_range(), + ) + .ok_or(UnsupportedArchive("Unsupported compression level"))?; + Ok(Box::new(move |bare| { + Ok(Zstd( + ZstdEncoder::new(bare, level as i32).map_err(ZipError::Io)?, + )) + })) + } + #[cfg(feature = "legacy-zip")] + CompressionMethod::Shrink => Err(ZipError::UnsupportedArchive( + "Shrink compression unsupported", + )), + #[cfg(feature = "legacy-zip")] + CompressionMethod::Reduce(_) => Err(ZipError::UnsupportedArchive( + "Reduce compression unsupported", + )), + #[cfg(feature = "legacy-zip")] + CompressionMethod::Implode => Err(ZipError::UnsupportedArchive( + "Implode compression unsupported", + )), + #[cfg(feature = "lzma")] + CompressionMethod::Lzma => { + Err(UnsupportedArchive("LZMA isn't supported for compression")) + } + #[cfg(feature = "xz")] + CompressionMethod::Xz => { + let level = validate_value_in_range(compression_level.unwrap_or(6), 0..=9) + .ok_or(UnsupportedArchive("Unsupported compression level"))? + as u32; + Ok(Box::new(move |bare| { + Ok(Xz(Box::new( + lzma_rust2::XzWriter::new(bare, lzma_rust2::XzOptions::with_preset(level)) + .map_err(ZipError::Io)?, + ))) + })) + } + #[cfg(feature = "ppmd")] + CompressionMethod::Ppmd => { + const ORDERS: [u32; 10] = [0, 4, 5, 6, 7, 8, 9, 10, 11, 12]; + + let level = validate_value_in_range(compression_level.unwrap_or(7), 1..=9) + .ok_or(UnsupportedArchive("Unsupported compression level"))? + as u32; + + let order = ORDERS[level as usize]; + let memory_size = 1 << (level + 19); + let memory_size_mb = memory_size / 1024 / 1024; + + Ok(Box::new(move |mut bare| { + let parameter: u16 = (order as u16 - 1) + + ((memory_size_mb - 1) << 4) as u16 + + ((ppmd_rust::RestoreMethod::Restart as u16) << 12); + + bare.write_all(¶meter.to_le_bytes()) + .map_err(ZipError::Io)?; + + let encoder = ppmd_rust::Ppmd8Encoder::new( + bare, + order, + memory_size, + ppmd_rust::RestoreMethod::Restart, + ) + .map_err(|error| match error { + ppmd_rust::Error::RangeDecoderInitialization => ZipError::InvalidArchive( + "PPMd range coder initialization failed".into(), + ), + ppmd_rust::Error::InvalidParameter => { + ZipError::InvalidArchive("Invalid PPMd parameter".into()) + } + ppmd_rust::Error::IoError(io_error) => ZipError::Io(io_error), + ppmd_rust::Error::MemoryAllocation => ZipError::Io(io::Error::new( + ErrorKind::OutOfMemory, + "PPMd could not allocate memory", + )), + })?; + + Ok(Ppmd(Box::new(encoder))) + })) + } + #[allow(deprecated)] + CompressionMethod::Unsupported(..) => { + Err(UnsupportedArchive("Unsupported compression")) + } + } + } + + fn switch_to(&mut self, make_new_self: SwitchWriterFunction) -> ZipResult<()> { + let bare = match mem::replace(self, Closed) { + Storer(w) => w, + #[cfg(feature = "deflate-flate2")] + Deflater(w) => w.finish()?, + #[cfg(feature = "deflate-zopfli")] + ZopfliDeflater(w) => w.finish()?, + #[cfg(feature = "deflate-zopfli")] + BufferedZopfliDeflater(w) => w + .into_inner() + .map_err(|e| ZipError::Io(e.into_error()))? + .finish()?, + #[cfg(feature = "_bzip2_any")] + Bzip2(w) => w.finish()?, + #[cfg(feature = "zstd")] + Zstd(w) => w.finish()?, + #[cfg(feature = "xz")] + Xz(w) => w.finish()?, + #[cfg(feature = "ppmd")] + Ppmd(w) => { + // ZIP needs to encode an end marker (7z for example doesn't encode one). + w.finish(true)? + } + Closed => { + return Err(io::Error::new( + io::ErrorKind::BrokenPipe, + "ZipWriter was already closed", + ) + .into()); + } + }; + *self = make_new_self(bare)?; + Ok(()) + } + + fn ref_mut(&mut self) -> Option<&mut dyn Write> { + match self { + Storer(ref mut w) => Some(w as &mut dyn Write), + #[cfg(feature = "deflate-flate2")] + Deflater(ref mut w) => Some(w as &mut dyn Write), + #[cfg(feature = "deflate-zopfli")] + ZopfliDeflater(w) => Some(w as &mut dyn Write), + #[cfg(feature = "deflate-zopfli")] + BufferedZopfliDeflater(w) => Some(w as &mut dyn Write), + #[cfg(feature = "_bzip2_any")] + Bzip2(ref mut w) => Some(w as &mut dyn Write), + #[cfg(feature = "zstd")] + Zstd(ref mut w) => Some(w as &mut dyn Write), + #[cfg(feature = "xz")] + Xz(ref mut w) => Some(w as &mut dyn Write), + #[cfg(feature = "ppmd")] + Ppmd(ref mut w) => Some(w as &mut dyn Write), + Closed => None, + } + } + + const fn is_closed(&self) -> bool { + matches!(*self, Closed) + } + + fn get_plain(&mut self) -> &mut W { + match *self { + Storer(MaybeEncrypted::Unencrypted(ref mut w)) => w, + _ => panic!("Should have switched to stored and unencrypted beforehand"), + } + } + + fn unwrap(self) -> W { + match self { + Storer(MaybeEncrypted::Unencrypted(w)) => w, + _ => panic!("Should have switched to stored and unencrypted beforehand"), + } + } +} + +#[cfg(feature = "_deflate-any")] +fn deflate_compression_level_range() -> std::ops::RangeInclusive { + let min = crate::cfg_if_expr! { + i64: + #[cfg(feature = "deflate-flate2")] => Compression::fast().level() as i64, + #[cfg(feature = "deflate-zopfli")] => 1, + _ => compile_error!("min: unknown deflate variant"), + }; + + let max = crate::cfg_if_expr! { + i64: + #[cfg(feature = "deflate-zopfli")] => 264, + #[cfg(feature = "deflate-flate2")] => Compression::best().level() as i64, + _ => compile_error!("max: unknown deflate variant"), + }; + + min..=max +} + +#[cfg(feature = "_bzip2_any")] +fn bzip2_compression_level_range() -> std::ops::RangeInclusive { + let min = bzip2::Compression::fast().level() as i64; + let max = bzip2::Compression::best().level() as i64; + min..=max +} + +#[cfg(any( + feature = "_deflate-any", + feature = "_bzip2_any", + feature = "ppmd", + feature = "xz", + feature = "zstd", +))] +fn validate_value_in_range>( + value: T, + range: std::ops::RangeInclusive, +) -> Option { + if range.contains(&value.try_into().ok()?) { + Some(value) + } else { + None + } +} + +fn update_aes_extra_data( + writer: &mut W, + file: &mut ZipFileData, + bytes_written: u64, +) -> ZipResult<()> { + let Some((aes_mode, version, compression_method)) = &mut file.aes_mode else { + return Ok(()); + }; + + // We prefer using AE-1 which provides an extra CRC check, but for small files we + // switch to AE-2 to prevent being able to use the CRC value to to reconstruct the + // unencrypted contents. + // + // We can only do this when the underlying writer supports + // seek operations, so we gate this behind using_data_descriptor. + // + // C.f. https://www.winzip.com/en/support/aes-encryption/#crc-faq + *version = if bytes_written < 20 { + AesVendorVersion::Ae2 + } else { + AesVendorVersion::Ae1 + }; + + let extra_data_start = file.extra_data_start.unwrap(); + + writer.seek(SeekFrom::Start( + extra_data_start + file.aes_extra_data_start, + ))?; + + let mut buf = Vec::new(); + + /* TODO: implement this using the Block trait! */ + // Extra field header ID. + buf.write_u16_le(0x9901)?; + // Data size. + buf.write_u16_le(7)?; + // Integer version number. + buf.write_u16_le(*version as u16)?; + // Vendor ID. + buf.write_all(b"AE")?; + // AES encryption strength. + buf.write_all(&[*aes_mode as u8])?; + // Real compression method. + buf.write_u16_le(compression_method.serialize_to_u16())?; + + writer.write_all(&buf)?; + + let aes_extra_data_start = file.aes_extra_data_start as usize; + let extra_field = Arc::get_mut(file.extra_field.as_mut().unwrap()).unwrap(); + extra_field[aes_extra_data_start..aes_extra_data_start + buf.len()].copy_from_slice(&buf); + + Ok(()) +} + +fn update_local_file_header( + writer: &mut T, + file: &mut ZipFileData, +) -> ZipResult<()> { + const CRC32_OFFSET: u64 = 14; + writer.seek(SeekFrom::Start(file.header_start + CRC32_OFFSET))?; + writer.write_u32_le(file.crc32)?; + if file.large_file { + writer.write_u32_le(spec::ZIP64_BYTES_THR as u32)?; + writer.write_u32_le(spec::ZIP64_BYTES_THR as u32)?; + + update_local_zip64_extra_field(writer, file)?; + + file.compressed_size = spec::ZIP64_BYTES_THR; + file.uncompressed_size = spec::ZIP64_BYTES_THR; + } else { + // check compressed size as well as it can also be slightly larger than uncompressed size + if file.compressed_size > spec::ZIP64_BYTES_THR { + return Err(ZipError::Io(io::Error::other( + "Large file option has not been set", + ))); + } + writer.write_u32_le(file.compressed_size as u32)?; + // uncompressed size is already checked on write to catch it as soon as possible + writer.write_u32_le(file.uncompressed_size as u32)?; + } + Ok(()) +} + +fn write_central_directory_header(writer: &mut T, file: &ZipFileData) -> ZipResult<()> { + let mut block = file.block()?; + let stripped_extra = if let Some(extra) = &file.extra_field { + strip_alignment_extra_field(extra) + } else { + Vec::new() + }; + let central_len = file.central_extra_field_len(); + let zip64_len = if let Some(zip64) = file.zip64_extra_field_block() { + zip64.full_size() + } else { + 0 + }; + block.extra_field_length = (zip64_len + stripped_extra.len() + central_len) + .try_into() + .map_err(|_| invalid!("Extra field length in central directory exceeds 64KiB"))?; + + block.write(writer)?; + // file name + writer.write_all(&file.file_name_raw)?; + // extra field + if let Some(zip64_extra_field) = &file.zip64_extra_field_block() { + writer.write_all(&zip64_extra_field.serialize())?; + } + if !stripped_extra.is_empty() { + writer.write_all(&stripped_extra)?; + } + if let Some(central_extra_field) = &file.central_extra_field { + writer.write_all(central_extra_field)?; + } + // file comment + writer.write_all(file.file_comment.as_bytes())?; + + Ok(()) +} + +fn strip_alignment_extra_field(extra_field: &[u8]) -> Vec { + let mut new_extra = Vec::with_capacity(extra_field.len()); + let mut cursor = 0; + while cursor + 4 <= extra_field.len() { + let tag = u16::from_le_bytes([extra_field[cursor], extra_field[cursor + 1]]); + let len = u16::from_le_bytes([extra_field[cursor + 2], extra_field[cursor + 3]]) as usize; + if cursor + 4 + len > extra_field.len() { + new_extra.extend_from_slice(&extra_field[cursor..]); + break; + } + + if tag != 0xa11e { + new_extra.extend_from_slice(&extra_field[cursor..cursor + 4 + len]); + } + cursor += 4 + len; + } + if cursor < extra_field.len() { + new_extra.extend_from_slice(&extra_field[cursor..]); + } + new_extra +} + +fn update_local_zip64_extra_field( + writer: &mut T, + file: &mut ZipFileData, +) -> ZipResult<()> { + let block = file.zip64_extra_field_block().ok_or(invalid!( + "Attempted to update a nonexistent ZIP64 extra field" + ))?; + + let zip64_extra_field_start = file.header_start + + size_of::() as u64 + + file.file_name_raw.len() as u64; + + writer.seek(SeekFrom::Start(zip64_extra_field_start))?; + let block = block.serialize(); + writer.write_all(&block)?; + + let extra_field = Arc::get_mut(file.extra_field.as_mut().unwrap()).unwrap(); + extra_field[..block.len()].copy_from_slice(&block); + + Ok(()) +} + +/// Wrapper around a [Write] implementation that implements the [Seek] trait, but where seeking +/// returns an error unless it's a no-op. +pub struct StreamWriter { + inner: W, + bytes_written: u64, +} + +impl StreamWriter { + /// Creates an instance wrapping the provided inner writer. + pub fn new(inner: W) -> StreamWriter { + Self { + inner, + bytes_written: 0, + } + } + + /// Consumes this wrapper, returning the underlying writer. + pub fn into_inner(self) -> W { + self.inner + } +} + +impl Write for StreamWriter { + fn write(&mut self, buf: &[u8]) -> io::Result { + let bytes_written = self.inner.write(buf)?; + self.bytes_written += bytes_written as u64; + Ok(bytes_written) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +impl Seek for StreamWriter { + fn seek(&mut self, pos: SeekFrom) -> io::Result { + match pos { + SeekFrom::Current(0) | SeekFrom::End(0) => return Ok(self.bytes_written), + SeekFrom::Start(x) => { + if x == self.bytes_written { + return Ok(self.bytes_written); + } + } + _ => {} + } + Err(io::Error::new( + ErrorKind::Unsupported, + "seek is not supported", + )) + } +} + +#[cfg(not(feature = "unreserved"))] +const EXTRA_FIELD_MAPPING: [u16; 43] = [ + 0x0007, 0x0008, 0x0009, 0x000a, 0x000c, 0x000d, 0x000e, 0x000f, 0x0014, 0x0015, 0x0016, 0x0017, + 0x0018, 0x0019, 0x0020, 0x0021, 0x0022, 0x0023, 0x0065, 0x0066, 0x4690, 0x07c8, 0x2605, 0x2705, + 0x2805, 0x334d, 0x4341, 0x4453, 0x4704, 0x470f, 0x4b46, 0x4c41, 0x4d49, 0x4f4c, 0x5356, 0x554e, + 0x5855, 0x6542, 0x756e, 0x7855, 0xa220, 0xfd4a, 0x9902, +]; + +#[cfg(test)] +#[allow(unknown_lints)] // needless_update is new in clippy pre 1.29.0 +#[allow(clippy::needless_update)] // So we can use the same FileOptions decls with and without zopfli_buffer_size +#[allow(clippy::octal_escapes)] // many false positives in converted fuzz cases +mod test { + use super::{ExtendedFileOptions, FileOptions, FullFileOptions, ZipWriter}; + use crate::compression::CompressionMethod; + use crate::result::ZipResult; + use crate::types::{DateTime, System}; + use crate::write::EncryptWith::ZipCrypto; + use crate::write::SimpleFileOptions; + use crate::zipcrypto::ZipCryptoKeys; + use crate::CompressionMethod::Stored; + use crate::ZipArchive; + #[cfg(feature = "deflate-flate2")] + use std::io::Read; + use std::io::{Cursor, Write}; + use std::marker::PhantomData; + use std::path::PathBuf; + + const SYSTEM_BYTE: u8 = if cfg!(windows) { + System::Dos + } else { + System::Unix + } as u8; + + #[test] + fn write_empty_zip() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_comment("ZIP"); + let result = writer.finish().unwrap(); + assert_eq!(result.get_ref().len(), 25); + assert_eq!( + *result.get_ref(), + [80, 75, 5, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 90, 73, 80] + ); + } + + #[test] + fn unix_permissions_bitmask() { + // unix_permissions() throws away upper bits. + let options = SimpleFileOptions::default().unix_permissions(0o120777); + assert_eq!(options.permissions, Some(0o777)); + } + + #[test] + fn write_zip_dir() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .add_directory( + "test", + SimpleFileOptions::default().last_modified_time( + DateTime::from_date_and_time(2018, 8, 15, 20, 45, 6).unwrap(), + ), + ) + .unwrap(); + assert!(writer + .write(b"writing to a directory is not allowed, and will not write any data") + .is_err()); + let result = writer.finish().unwrap(); + assert_eq!(result.get_ref().len(), 108); + const DOS_DIR_ATTRIBUTES_BYTE: u8 = if cfg!(windows) { 0x10 } else { 0 }; + #[rustfmt::skip] + assert_eq!( + *result.get_ref(), + &[ + 80u8, 75, 3, 4, 20, 0, 0, 0, 0, 0, 163, 165, 15, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 0, 0, 5, 0, 0, 0, 116, 101, 115, 116, 47, 80, 75, 1, 2, 20, SYSTEM_BYTE, 20, 0, 0, 0, 0, 0, + 163, 165, 15, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, + DOS_DIR_ATTRIBUTES_BYTE, 0, 237, 65, 0, 0, 0, 0, 116, 101, 115, 116, 47, 80, 75, 5, 6, 0, 0, 0, 0, 1, 0, + 1, 0, 51, 0, 0, 0, 35, 0, 0, 0, 0, 0, + ] as &[u8] + ); + } + + #[test] + fn write_symlink_simple() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .add_symlink( + "name", + "target", + SimpleFileOptions::default().last_modified_time( + DateTime::from_date_and_time(2018, 8, 15, 20, 45, 6).unwrap(), + ), + ) + .unwrap(); + assert!(writer + .write(b"writing to a symlink is not allowed and will not write any data") + .is_err()); + let result = writer.finish().unwrap(); + assert_eq!(result.get_ref().len(), 112); + #[rustfmt::skip] + assert_eq!( + *result.get_ref(), + &[ + 80u8, 75, 3, 4, 10, 0, 0, 0, 0, 0, 163, 165, 15, 77, 252, 47, 111, 70, 6, 0, 0, 0, + 6, 0, 0, 0, 4, 0, 0, 0, 110, 97, 109, 101, 116, 97, 114, 103, 101, 116, 80, 75, 1, + 2, 10, System::Unix as u8, 10, 0, 0, 0, 0, 0, 163, 165, 15, 77, 252, 47, 111, 70, 6, 0, 0, 0, 6, 0, + 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, 161, 0, 0, 0, 0, 110, 97, 109, 101, + 80, 75, 5, 6, 0, 0, 0, 0, 1, 0, 1, 0, 50, 0, 0, 0, 40, 0, 0, 0, 0, 0 + ] as &[u8], + ); + } + + #[test] + fn test_path_normalization() { + let mut path = PathBuf::new(); + path.push("foo"); + path.push("bar"); + path.push(".."); + path.push("."); + path.push("example.txt"); + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file_from_path(path, SimpleFileOptions::default()) + .unwrap(); + let archive = writer.finish_into_readable().unwrap(); + assert_eq!(Some("foo/example.txt"), archive.name_for_index(0)); + } + + #[test] + fn write_symlink_wonky_paths() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .add_symlink( + "directory\\link", + "/absolute/symlink\\with\\mixed/slashes", + SimpleFileOptions::default().last_modified_time( + DateTime::from_date_and_time(2018, 8, 15, 20, 45, 6).unwrap(), + ), + ) + .unwrap(); + assert!(writer + .write(b"writing to a symlink is not allowed and will not write any data") + .is_err()); + let result = writer.finish().unwrap(); + assert_eq!(result.get_ref().len(), 162); + #[rustfmt::skip] + assert_eq!( + *result.get_ref(), + &[ + 80u8, 75, 3, 4, 10, 0, 0, 0, 0, 0, 163, 165, 15, 77, 95, 41, 81, 245, 36, 0, 0, 0, + 36, 0, 0, 0, 14, 0, 0, 0, 100, 105, 114, 101, 99, 116, 111, 114, 121, 92, 108, 105, + 110, 107, 47, 97, 98, 115, 111, 108, 117, 116, 101, 47, 115, 121, 109, 108, 105, + 110, 107, 92, 119, 105, 116, 104, 92, 109, 105, 120, 101, 100, 47, 115, 108, 97, + 115, 104, 101, 115, 80, 75, 1, 2, 10, System::Unix as u8, 10, 0, 0, 0, 0, 0, 163, 165, 15, 77, 95, + 41, 81, 245, 36, 0, 0, 0, 36, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 255, + 161, 0, 0, 0, 0, 100, 105, 114, 101, 99, 116, 111, 114, 121, 92, 108, 105, 110, + 107, 80, 75, 5, 6, 0, 0, 0, 0, 1, 0, 1, 0, 60, 0, 0, 0, 80, 0, 0, 0, 0, 0 + ] as &[u8], + ); + } + + #[test] + fn write_mimetype_zip() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::default(), + permissions: Some(33188), + large_file: false, + encrypt_with: None, + extended_options: (), + alignment: 1, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: None, + #[cfg(feature = "aes-crypto")] + aes_mode: None, + }; + writer.start_file("mimetype", options).unwrap(); + writer + .write_all(b"application/vnd.oasis.opendocument.text") + .unwrap(); + let result = writer.finish().unwrap(); + + assert_eq!(result.get_ref().len(), 153); + #[rustfmt::skip] + assert_eq!(result.get_ref(), &[80, 75, 3, 4, 10, 0, 0, 0, 0, 0, 0, 0, 33, 0, 94, 198, 50, + 12, 39, 0, 0, 0, 39, 0, 0, 0, 8, 0, 0, 0, 109, 105, 109, 101, 116, 121, 112, 101, 97, + 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 118, 110, 100, 46, 111, 97, 115, + 105, 115, 46, 111, 112, 101, 110, 100, 111, 99, 117, 109, 101, 110, 116, 46, 116, 101, + 120, 116, 80, 75, 1, 2, 10, SYSTEM_BYTE, 10, 0, 0, 0, 0, 0, 0, 0, 33, 0, 94, 198, 50, + 12, 39, 0, 0, 0, 39, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 129, 0, 0, 0, 0, + 109, 105, 109, 101, 116, 121, 112, 101, 80, 75, 5, 6, 0, 0, 0, 0, 1, 0, 1, 0, 54, 0, 0, + 0, 77, 0, 0, 0, 0, 0]); + } + + #[cfg(feature = "deflate-flate2")] + const RT_TEST_TEXT: &str = "And I can't stop thinking about the moments that I lost to you\ + And I can't stop thinking of things I used to do\ + And I can't stop making bad decisions\ + And I can't stop eating stuff you make me chew\ + I put on a smile like you wanna see\ + Another day goes by that I long to be like you"; + #[cfg(feature = "deflate-flate2")] + const RT_TEST_FILENAME: &str = "subfolder/sub-subfolder/can't_stop.txt"; + #[cfg(feature = "deflate-flate2")] + const SECOND_FILENAME: &str = "different_name.xyz"; + #[cfg(feature = "deflate-flate2")] + const THIRD_FILENAME: &str = "third_name.xyz"; + + #[test] + fn write_non_utf8() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::default(), + permissions: Some(33188), + large_file: false, + encrypt_with: None, + extended_options: (), + alignment: 1, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: None, + #[cfg(feature = "aes-crypto")] + aes_mode: None, + }; + + // GB18030 + // "中文" = [214, 208, 206, 196] + let filename = unsafe { String::from_utf8_unchecked(vec![214, 208, 206, 196]) }; + writer.start_file(filename, options).unwrap(); + writer.write_all(b"encoding GB18030").unwrap(); + + // SHIFT_JIS + // "日文" = [147, 250, 149, 182] + let filename = unsafe { String::from_utf8_unchecked(vec![147, 250, 149, 182]) }; + writer.start_file(filename, options).unwrap(); + writer.write_all(b"encoding SHIFT_JIS").unwrap(); + let result = writer.finish().unwrap(); + + assert_eq!(result.get_ref().len(), 224); + #[rustfmt::skip] + assert_eq!(result.get_ref(), &[80, 75, 3, 4, 10, 0, 0, 0, 0, 0, 0, 0, 33, 0, 29, 183, 125, + 75, 16, 0, 0, 0, 16, 0, 0, 0, 4, 0, 0, 0, 214, 208, 206, 196, 101, 110, 99, 111, 100, + 105, 110, 103, 32, 71, 66, 49, 56, 48, 51, 48, 80, 75, 3, 4, 10, 0, 0, 0, 0, 0, 0, 0, + 33, 0, 75, 110, 14, 242, 18, 0, 0, 0, 18, 0, 0, 0, 4, 0, 0, 0, 147, 250, 149, 182, 101, + 110, 99, 111, 100, 105, 110, 103, 32, 83, 72, 73, 70, 84, 95, 74, 73, 83, 80, 75, 1, 2, + 10, SYSTEM_BYTE, 10, 0, 0, 0, 0, 0, 0, 0, 33, 0, 29, 183, 125, 75, 16, 0, 0, 0, 16, 0, + 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 129, 0, 0, 0, 0, 214, 208, 206, 196, 80, + 75, 1, 2, 10, SYSTEM_BYTE, 10, 0, 0, 0, 0, 0, 0, 0, 33, 0, 75, 110, 14, 242, 18, 0, 0, 0, 18, 0, + 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 164, 129, 50, 0, 0, 0, 147, 250, 149, 182, 80, + 75, 5, 6, 0, 0, 0, 0, 2, 0, 2, 0, 100, 0, 0, 0, 102, 0, 0, 0, 0, 0]); + } + + #[test] + fn path_to_string() { + let mut path = PathBuf::new(); + #[cfg(windows)] + path.push(r"C:\"); + #[cfg(unix)] + path.push("/"); + path.push("windows"); + path.push(".."); + path.push("."); + path.push("system32"); + let path_str = super::path_to_string(&path); + assert_eq!(&*path_str, "system32"); + } + + #[test] + #[cfg(feature = "deflate-flate2")] + fn test_shallow_copy() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = FileOptions { + compression_method: CompressionMethod::default(), + compression_level: None, + last_modified_time: DateTime::default(), + permissions: Some(33188), + large_file: false, + encrypt_with: None, + extended_options: (), + alignment: 0, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: None, + #[cfg(feature = "aes-crypto")] + aes_mode: None, + }; + writer.start_file(RT_TEST_FILENAME, options).unwrap(); + writer.write_all(RT_TEST_TEXT.as_ref()).unwrap(); + writer + .shallow_copy_file(RT_TEST_FILENAME, SECOND_FILENAME) + .unwrap(); + writer + .shallow_copy_file(RT_TEST_FILENAME, SECOND_FILENAME) + .expect_err("Duplicate filename"); + let zip = writer.finish().unwrap(); + let mut writer = ZipWriter::new_append(zip).unwrap(); + writer + .shallow_copy_file(SECOND_FILENAME, SECOND_FILENAME) + .expect_err("Duplicate filename"); + let mut reader = writer.finish_into_readable().unwrap(); + let mut file_names: Vec<&str> = reader.file_names().collect(); + file_names.sort(); + let mut expected_file_names = vec![RT_TEST_FILENAME, SECOND_FILENAME]; + expected_file_names.sort(); + assert_eq!(file_names, expected_file_names); + let mut first_file_content = String::new(); + reader + .by_name(RT_TEST_FILENAME) + .unwrap() + .read_to_string(&mut first_file_content) + .unwrap(); + assert_eq!(first_file_content, RT_TEST_TEXT); + let mut second_file_content = String::new(); + reader + .by_name(SECOND_FILENAME) + .unwrap() + .read_to_string(&mut second_file_content) + .unwrap(); + assert_eq!(second_file_content, RT_TEST_TEXT); + } + + #[test] + #[cfg(feature = "deflate-flate2")] + fn test_deep_copy() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = FileOptions { + compression_method: CompressionMethod::default(), + compression_level: None, + last_modified_time: DateTime::default(), + permissions: Some(33188), + large_file: false, + encrypt_with: None, + extended_options: (), + alignment: 0, + #[cfg(feature = "deflate-zopfli")] + zopfli_buffer_size: None, + #[cfg(feature = "aes-crypto")] + aes_mode: None, + }; + writer.start_file(RT_TEST_FILENAME, options).unwrap(); + writer.write_all(RT_TEST_TEXT.as_ref()).unwrap(); + writer + .deep_copy_file(RT_TEST_FILENAME, SECOND_FILENAME) + .unwrap(); + let zip = writer.finish().unwrap().into_inner(); + zip.iter().copied().for_each(|x| print!("{x:02x}")); + println!(); + let mut writer = ZipWriter::new_append(Cursor::new(zip)).unwrap(); + writer + .deep_copy_file(RT_TEST_FILENAME, THIRD_FILENAME) + .unwrap(); + let zip = writer.finish().unwrap(); + let mut reader = ZipArchive::new(zip).unwrap(); + let mut file_names: Vec<&str> = reader.file_names().collect(); + file_names.sort(); + let mut expected_file_names = vec![RT_TEST_FILENAME, SECOND_FILENAME, THIRD_FILENAME]; + expected_file_names.sort(); + assert_eq!(file_names, expected_file_names); + let mut first_file_content = String::new(); + reader + .by_name(RT_TEST_FILENAME) + .unwrap() + .read_to_string(&mut first_file_content) + .unwrap(); + assert_eq!(first_file_content, RT_TEST_TEXT); + let mut second_file_content = String::new(); + reader + .by_name(SECOND_FILENAME) + .unwrap() + .read_to_string(&mut second_file_content) + .unwrap(); + assert_eq!(second_file_content, RT_TEST_TEXT); + } + + #[test] + fn duplicate_filenames() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file("foo/bar/test", SimpleFileOptions::default()) + .unwrap(); + writer + .write_all("The quick brown 🦊 jumps over the lazy 🐕".as_bytes()) + .unwrap(); + writer + .start_file("foo/bar/test", SimpleFileOptions::default()) + .expect_err("Expected duplicate filename not to be allowed"); + } + + #[test] + fn test_filename_looks_like_zip64_locator() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file( + "PK\u{6}\u{7}\0\0\0\u{11}\0\0\0\0\0\0\0\0\0\0\0\0", + SimpleFileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + let _ = ZipArchive::new(zip).unwrap(); + } + + #[test] + fn test_filename_looks_like_zip64_locator_2() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file( + "PK\u{6}\u{6}\0\0\0\0\0\0\0\0\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", + SimpleFileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + let _ = ZipArchive::new(zip).unwrap(); + } + + #[test] + fn test_filename_looks_like_zip64_locator_2a() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file( + "PK\u{6}\u{6}PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", + SimpleFileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + let _ = ZipArchive::new(zip).unwrap(); + } + + #[test] + fn test_filename_looks_like_zip64_locator_3() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file("\0PK\u{6}\u{6}", SimpleFileOptions::default()) + .unwrap(); + writer + .start_file( + "\0\u{4}\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{3}", + SimpleFileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + let _ = ZipArchive::new(zip).unwrap(); + } + + #[test] + fn test_filename_looks_like_zip64_locator_4() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file("PK\u{6}\u{6}", SimpleFileOptions::default()) + .unwrap(); + writer + .start_file("\0\0\0\0\0\0", SimpleFileOptions::default()) + .unwrap(); + writer + .start_file("\0", SimpleFileOptions::default()) + .unwrap(); + writer.start_file("", SimpleFileOptions::default()).unwrap(); + writer + .start_file("\0\0", SimpleFileOptions::default()) + .unwrap(); + writer + .start_file( + "\0\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", + SimpleFileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + let _ = ZipArchive::new(zip).unwrap(); + } + + #[test] + fn test_filename_looks_like_zip64_locator_5() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .add_directory("", SimpleFileOptions::default().with_alignment(21)) + .unwrap(); + let mut writer = ZipWriter::new_append(writer.finish().unwrap()).unwrap(); + writer.shallow_copy_file("/", "").unwrap(); + writer.shallow_copy_file("", "\0").unwrap(); + writer.shallow_copy_file("\0", "PK\u{6}\u{6}").unwrap(); + let mut writer = ZipWriter::new_append(writer.finish().unwrap()).unwrap(); + writer + .start_file("\0\0\0\0\0\0", SimpleFileOptions::default()) + .unwrap(); + let mut writer = ZipWriter::new_append(writer.finish().unwrap()).unwrap(); + writer + .start_file( + "#PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0", + SimpleFileOptions::default(), + ) + .unwrap(); + let zip = writer.finish().unwrap(); + let _ = ZipArchive::new(zip).unwrap(); + Ok(()) + } + + #[test] + #[cfg(feature = "deflate-flate2")] + fn remove_shallow_copy_keeps_original() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer + .start_file("original", SimpleFileOptions::default()) + .unwrap(); + writer.write_all(RT_TEST_TEXT.as_bytes()).unwrap(); + writer + .shallow_copy_file("original", "shallow_copy") + .unwrap(); + writer.abort_file().unwrap(); + let mut zip = ZipArchive::new(writer.finish().unwrap()).unwrap(); + let mut file = zip.by_name("original").unwrap(); + let mut contents = Vec::new(); + file.read_to_end(&mut contents).unwrap(); + assert_eq!(RT_TEST_TEXT.as_bytes(), contents); + Ok(()) + } + + #[test] + fn remove_encrypted_file() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let first_file_options = SimpleFileOptions::default() + .with_alignment(65535) + .with_deprecated_encryption(b"Password"); + writer.start_file("", first_file_options).unwrap(); + writer.abort_file().unwrap(); + let zip = writer.finish().unwrap(); + let mut writer = ZipWriter::new(zip); + writer.start_file("", SimpleFileOptions::default()).unwrap(); + Ok(()) + } + + #[test] + fn remove_encrypted_aligned_symlink() -> ZipResult<()> { + let mut options = SimpleFileOptions::default(); + options = options.with_deprecated_encryption(b"Password"); + options.alignment = 65535; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.add_symlink("", "s\t\0\0ggggg\0\0", options).unwrap(); + writer.abort_file().unwrap(); + let zip = writer.finish().unwrap(); + let mut writer = ZipWriter::new_append(zip).unwrap(); + writer.start_file("", SimpleFileOptions::default()).unwrap(); + Ok(()) + } + + #[cfg(feature = "deflate-zopfli")] + #[test] + fn zopfli_empty_write() -> ZipResult<()> { + let mut options = SimpleFileOptions::default(); + options = options + .compression_method(CompressionMethod::default()) + .compression_level(Some(264)); + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.start_file("", options).unwrap(); + writer.write_all(&[]).unwrap(); + writer.write_all(&[]).unwrap(); + Ok(()) + } + + #[test] + fn crash_with_no_features() -> ZipResult<()> { + const ORIGINAL_FILE_NAME: &str = "PK\u{6}\u{6}\0\0\0\0\0\0\0\0\0\u{2}g\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\0\0PK\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\u{7}\0\t'"; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let mut options = SimpleFileOptions::default(); + options = options.with_alignment(3584).compression_method(Stored); + writer.start_file(ORIGINAL_FILE_NAME, options)?; + let archive = writer.finish()?; + let mut writer = ZipWriter::new_append(archive)?; + writer.shallow_copy_file(ORIGINAL_FILE_NAME, "\u{6}\\")?; + writer.finish()?; + Ok(()) + } + + #[test] + fn test_alignment() { + let page_size = 4096; + let options = SimpleFileOptions::default() + .compression_method(Stored) + .with_alignment(page_size); + let mut zip = ZipWriter::new(Cursor::new(Vec::new())); + let contents = b"sleeping"; + let () = zip.start_file("sleep", options).unwrap(); + let _count = zip.write(&contents[..]).unwrap(); + let mut zip = zip.finish_into_readable().unwrap(); + let file = zip.by_index(0).unwrap(); + assert_eq!(file.name(), "sleep"); + assert_eq!(file.data_start(), u64::from(page_size)); + } + + #[test] + fn test_alignment_2() { + let page_size = 4096; + let mut data = Vec::new(); + { + let options = SimpleFileOptions::default() + .compression_method(Stored) + .with_alignment(page_size); + let mut zip = ZipWriter::new(Cursor::new(&mut data)); + let contents = b"sleeping"; + let () = zip.start_file("sleep", options).unwrap(); + let _count = zip.write(&contents[..]).unwrap(); + } + assert_eq!(data[4096..4104], b"sleeping"[..]); + { + let mut zip = ZipArchive::new(Cursor::new(&mut data)).unwrap(); + let file = zip.by_index(0).unwrap(); + assert_eq!(file.name(), "sleep"); + assert_eq!(file.data_start(), u64::from(page_size)); + } + } + + #[test] + fn test_crash_short_read() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let comment = vec![ + 1, 80, 75, 5, 6, 237, 237, 237, 237, 237, 237, 237, 237, 44, 255, 191, 255, 255, 255, + 255, 255, 255, 255, 255, 16, + ] + .into_boxed_slice(); + writer.set_raw_comment(comment); + let options = SimpleFileOptions::default() + .compression_method(Stored) + .with_alignment(11823); + writer.start_file("", options).unwrap(); + writer.write_all(&[255, 255, 44, 255, 0]).unwrap(); + let written = writer.finish().unwrap(); + let _ = ZipWriter::new_append(written).unwrap(); + } + + #[cfg(all(feature = "_deflate-any", feature = "aes-crypto"))] + #[test] + fn test_fuzz_failure_2024_05_08() -> ZipResult<()> { + let mut first_writer = ZipWriter::new(Cursor::new(Vec::new())); + let mut second_writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = SimpleFileOptions::default() + .compression_method(Stored) + .with_alignment(46036); + second_writer.add_symlink("\0", "", options)?; + let second_archive = second_writer.finish_into_readable()?.into_inner(); + let mut second_writer = ZipWriter::new_append(second_archive)?; + let options = SimpleFileOptions::default() + .compression_method(CompressionMethod::Deflated) + .large_file(true) + .with_alignment(46036) + .with_aes_encryption(crate::AesMode::Aes128, "\0\0"); + second_writer.add_symlink("", "", options)?; + let second_archive = second_writer.finish_into_readable()?.into_inner(); + let mut second_writer = ZipWriter::new_append(second_archive)?; + let options = SimpleFileOptions::default().compression_method(Stored); + second_writer.start_file(" ", options)?; + let second_archive = second_writer.finish_into_readable()?; + first_writer.merge_archive(second_archive)?; + let _ = ZipArchive::new(first_writer.finish()?)?; + Ok(()) + } + + #[cfg(all(feature = "_bzip2_any", not(miri)))] + #[test] + fn test_fuzz_failure_2024_06_08() -> ZipResult<()> { + use crate::write::ExtendedFileOptions; + use CompressionMethod::Bzip2; + + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + const SYMLINK_PATH: &str = "PK\u{6}\u{6}K\u{6}\u{6}\u{6}\0\0\0\0\u{18}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\u{1}\0\0PK\u{1}\u{2},\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0PK\u{1}\u{2},\0\0\0\0\0\0\0\0\0\0l\0\0\0\0\0\0PK\u{6}\u{7}P\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0"; + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Bzip2, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 5, 20, 21, 0, 57)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 2048, + ..Default::default() + }; + writer.add_symlink_from_path(SYMLINK_PATH, "||\0\0\0\0", options)?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.deep_copy_file_from_path(SYMLINK_PATH, "")?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.abort_file()?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.deep_copy_file_from_path(SYMLINK_PATH, "foo")?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_short_extra_data() { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![99, 0, 15, 0, 207].into(), + }, + ..Default::default() + }; + assert!(writer.start_file_from_path("", options).is_err()); + } + + #[test] + #[cfg(not(feature = "unreserved"))] + fn test_invalid_extra_data() -> ZipResult<()> { + use crate::write::ExtendedFileOptions; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 1, 4, 6, 54, 0)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![ + 7, 0, 15, 0, 207, 117, 177, 117, 112, 2, 0, 255, 255, 131, 255, 255, 255, 80, + 185, + ] + .into(), + }, + alignment: 32787, + ..Default::default() + }; + assert!(writer.start_file_from_path("", options).is_err()); + Ok(()) + } + + #[test] + #[cfg(not(feature = "unreserved"))] + fn test_invalid_extra_data_unreserved() { + use crate::write::ExtendedFileOptions; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2021, 8, 8, 1, 0, 29).unwrap(), + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![ + 1, 41, 4, 0, 1, 255, 245, 117, 117, 112, 5, 0, 80, 255, 149, 255, 247, + ] + .into(), + }, + alignment: 4103, + ..Default::default() + }; + assert!(writer.start_file_from_path("", options).is_err()); + } + + #[cfg(feature = "deflate64")] + #[test] + fn test_fuzz_crash_2024_06_13a() -> ZipResult<()> { + use crate::write::ExtendedFileOptions; + use CompressionMethod::Deflate64; + + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Deflate64, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2039, 4, 17, 6, 18, 19)?, + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 4, + ..Default::default() + }; + writer.add_directory_from_path("", options)?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_13b() -> ZipResult<()> { + use crate::write::ExtendedFileOptions; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 4, 14, 6, 11, 54)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 185, + ..Default::default() + }; + writer.add_symlink_from_path("", "", options)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer.deep_copy_file_from_path("", "_copy")?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_14() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FullFileOptions { + compression_method: Stored, + large_file: true, + alignment: 93, + ..Default::default() + }; + writer.start_file_from_path("\0", options)?; + writer = ZipWriter::new_append(writer.finish()?)?; + writer.deep_copy_file_from_path("\0", "")?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer.deep_copy_file_from_path("", "copy")?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_14a() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2083, 5, 30, 21, 45, 35)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 2565, + ..Default::default() + }; + writer.add_symlink_from_path("", "", options)?; + writer.abort_file()?; + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::default(), + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 0, + ..Default::default() + }; + writer.start_file_from_path("", options)?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[allow(deprecated)] + #[test] + fn test_fuzz_crash_2024_06_14b() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2078, 3, 6, 12, 48, 58)?, + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 65521, + ..Default::default() + }; + writer.start_file_from_path("\u{4}\0@\n//\u{c}", options)?; + writer = ZipWriter::new_append(writer.finish()?)?; + writer.abort_file()?; + let options = FileOptions { + compression_method: CompressionMethod::Unsupported(65535), + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2055, 10, 2, 11, 48, 49)?, + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![255, 255, 1, 0, 255, 0, 0, 0, 0].into(), + central_extra_data: vec![].into(), + }, + alignment: 65535, + ..Default::default() + }; + writer.add_directory_from_path("", options)?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_14c() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2060, 4, 6, 13, 13, 3)?, + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 0, + ..Default::default() + }; + writer.start_file_from_path("\0", options)?; + writer.write_all(&([]))?; + writer = ZipWriter::new_append(writer.finish()?)?; + writer.deep_copy_file_from_path("\0", "")?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer.deep_copy_file_from_path("", "_copy")?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[cfg(all(feature = "_deflate-any", feature = "aes-crypto"))] + #[test] + fn test_fuzz_crash_2024_06_14d() -> ZipResult<()> { + use crate::write::EncryptWith::Aes; + use crate::AesMode::Aes256; + use CompressionMethod::Deflated; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Deflated, + compression_level: Some(5), + last_modified_time: DateTime::from_date_and_time(2107, 4, 8, 15, 54, 19)?, + permissions: None, + large_file: true, + encrypt_with: Some(Aes { + mode: Aes256, + password: "", + }), + extended_options: ExtendedFileOptions { + extra_data: vec![2, 0, 1, 0, 0].into(), + central_extra_data: vec![ + 35, 229, 2, 0, 41, 41, 231, 44, 2, 0, 52, 233, 82, 201, 0, 0, 3, 0, 2, 0, 233, + 255, 3, 0, 2, 0, 26, 154, 38, 251, 0, 0, + ] + .into(), + }, + alignment: 65535, + ..Default::default() + }; + assert!(writer.add_directory_from_path("", options).is_err()); + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_14e() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1988, 1, 1, 1, 6, 26)?, + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![76, 0, 1, 0, 0, 2, 0, 0, 0].into(), + central_extra_data: vec![ + 1, 149, 1, 0, 255, 3, 0, 0, 0, 2, 255, 0, 0, 12, 65, 1, 0, 0, 67, 149, 0, 0, + 76, 149, 2, 0, 149, 149, 67, 149, 0, 0, + ] + .into(), + }, + alignment: 65535, + ..Default::default() + }; + assert!(writer.add_directory_from_path("", options).is_err()); + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[allow(deprecated)] + #[test] + fn test_fuzz_crash_2024_06_17() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = + ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: CompressionMethod::Unsupported( + 65535, + ), + compression_level: Some(5), + last_modified_time: DateTime::from_date_and_time( + 2107, 2, 8, 15, 0, 0, + )?, + permissions: None, + large_file: true, + encrypt_with: Some(ZipCrypto( + ZipCryptoKeys::of( + 0x63ff, 0xc62d3103, 0xfffe00ea, + ), + PhantomData, + )), + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 255, + ..Default::default() + }; + writer.add_symlink_from_path("1\0PK\u{6}\u{6}\u{b}\u{6}\u{6}\u{6}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\u{b}\0\0PK\u{1}\u{2},\0\0\0\0\0\0\0\0\0\0\0\u{10}\0\0\0K\u{6}\u{6}\0\0\0\0\0\0\0\0PK\u{2}\u{6}", "", options)?; + writer = ZipWriter::new_append( + writer.finish_into_readable()?.into_inner(), + )?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append( + writer.finish_into_readable()?.into_inner(), + )?; + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time( + 1992, 7, 3, 0, 0, 0, + )?, + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 43, + ..Default::default() + }; + writer.start_file_from_path( + "\0\0\0\u{3}\0\u{1a}\u{1a}\u{1a}\u{1a}\u{1a}\u{1a}", + options, + )?; + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time( + 2006, 3, 27, 2, 24, 26, + )?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 26, + ..Default::default() + }; + writer.start_file_from_path("\0K\u{6}\u{6}\0PK\u{6}\u{7}PK\u{6}\u{6}\0\0\0\0\0\0\0\0PK\u{2}\u{6}", options)?; + writer = ZipWriter::new_append( + writer.finish_into_readable()?.into_inner(), + )?; + let options = FileOptions { + compression_method: Stored, + compression_level: Some(17), + last_modified_time: DateTime::from_date_and_time( + 2103, 4, 10, 23, 15, 18, + )?, + permissions: Some(3284386755), + large_file: true, + encrypt_with: Some(ZipCrypto( + ZipCryptoKeys::of( + 0x8888c5bf, 0x88888888, 0xff888888, + ), + PhantomData, + )), + extended_options: ExtendedFileOptions { + extra_data: vec![3, 0, 1, 0, 255, 144, 136, 0, 0] + .into(), + central_extra_data: vec![].into(), + }, + alignment: 65535, + ..Default::default() + }; + writer.add_symlink_from_path("", "\nu", options)?; + writer = ZipWriter::new_append(writer.finish()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append( + writer.finish_into_readable()?.into_inner(), + )?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append(writer.finish()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = + ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.abort_file()?; + let options = FileOptions { + compression_method: CompressionMethod::Unsupported(49603), + compression_level: Some(20), + last_modified_time: DateTime::from_date_and_time( + 2047, 4, 14, 3, 15, 14, + )?, + permissions: Some(3284386755), + large_file: true, + encrypt_with: Some(ZipCrypto( + ZipCryptoKeys::of(0xc3, 0x0, 0x0), + PhantomData, + )), + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 0, + ..Default::default() + }; + writer.add_directory_from_path("", options)?; + writer.deep_copy_file_from_path("/", "")?; + writer.shallow_copy_file_from_path("", "copy")?; + assert!(writer.shallow_copy_file_from_path("", "copy").is_err()); + assert!(writer.shallow_copy_file_from_path("", "copy").is_err()); + assert!(writer.shallow_copy_file_from_path("", "copy").is_err()); + assert!(writer.shallow_copy_file_from_path("", "copy").is_err()); + assert!(writer.shallow_copy_file_from_path("", "copy").is_err()); + assert!(writer.shallow_copy_file_from_path("", "copy").is_err()); + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_17a() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + const PATH_1: &str = "\0I\01\0P\0\0\u{2}\0\0\u{1a}\u{1a}\u{1a}\u{1a}\u{1b}\u{1a}UT\u{5}\0\0\u{1a}\u{1a}\u{1a}\u{1a}UT\u{5}\0\u{1}\0\u{1a}\u{1a}\u{1a}UT\t\0uc\u{5}\0\0\0\0\u{7f}\u{7f}\u{7f}\u{7f}PK\u{6}"; + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1981, 1, 1, 0, 24, 21)?, + permissions: Some(16908288), + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 20555, + ..Default::default() + }; + writer.start_file_from_path( + "\0\u{7}\u{1}\0\0\0\0\0\0\0\0\u{1}\0\0PK\u{1}\u{2};", + options, + )?; + writer.write_all( + &([ + 255, 255, 255, 255, 253, 253, 253, 203, 203, 203, 253, 253, 253, 253, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 249, 191, 225, 225, + 241, 197, + ]), + )?; + writer.write_all( + &([ + 197, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 75, 0, + ]), + )?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 11, 14, 10, 46, 47)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 0, + ..Default::default() + }; + writer.start_file_from_path(PATH_1, options)?; + writer.deep_copy_file_from_path(PATH_1, "eee\u{6}\0\0\0\0\0\0\0\0\0\0\0$\0\0\0\0\0\0\u{7f}\u{7f}PK\u{6}\u{6}K\u{6}\u{6}\u{6}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\u{1}\0\0PK\u{1}\u{1e},\0\0\0\0\0\0\0\0\0\0\0\u{8}\0*\0\0\u{1}PK\u{6}\u{7}PK\u{6}\u{6}\0\0\0\0\0\0\0\0}K\u{2}\u{6}")?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.deep_copy_file_from_path(PATH_1, "")?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.shallow_copy_file_from_path("", "copy")?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + #[allow(clippy::octal_escapes)] + #[cfg(all(feature = "_bzip2_any", not(miri)))] + fn test_fuzz_crash_2024_06_17b() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time( + 1981, 1, 1, 0, 0, 21, + )?, + permissions: Some(16908288), + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 20555, + ..Default::default() + }; + writer.start_file_from_path("\0\u{7}\u{1}\0\0\0\0\0\0\0\0\u{1}\0\0PK\u{1}\u{2};\u{1a}\u{18}\u{1a}UT\t.........................\0u", options)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + let options = FileOptions { + compression_method: CompressionMethod::Bzip2, + compression_level: Some(5), + last_modified_time: DateTime::from_date_and_time( + 2055, 7, 7, 3, 6, 6, + )?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 0, + ..Default::default() + }; + writer.start_file_from_path("\0\0\0\0..\0\0\0\0\0\u{7f}\u{7f}PK\u{6}\u{6}K\u{6}\u{6}\u{6}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\u{1}\0\0PK\u{1}\u{1e},\0\0\0\0\0\0\0\0\0\0\0\u{8}\0*\0\0\u{1}PK\u{6}\u{7}PK\u{6}\u{6}\0\0\0\0\0\0\0\0}K\u{2}\u{6}", options)?; + writer = ZipWriter::new_append( + writer.finish_into_readable()?.into_inner(), + )?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append( + writer.finish_into_readable()?.into_inner(), + )?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = + ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = + ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_18() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_raw_comment(Box::<[u8]>::from([ + 80, 75, 5, 6, 255, 255, 255, 255, 255, 255, 80, 75, 5, 6, 255, 255, 255, 255, 255, 255, + 13, 0, 13, 13, 13, 13, 13, 255, 255, 255, 255, 255, 255, 255, 255, + ])); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + writer.set_raw_comment(Box::new([])); + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer = ZipWriter::new_append(writer.finish()?)?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_18a() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + writer.set_raw_comment(Box::<[u8]>::from([])); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FullFileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2107, 4, 8, 14, 0, 19)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![ + 182, 180, 1, 0, 180, 182, 74, 0, 0, 200, 0, 0, 0, 2, 0, 0, 0, + ] + .into(), + central_extra_data: vec![].into(), + }, + alignment: 1542, + ..Default::default() + }; + writer.start_file_from_path("\0\0PK\u{6}\u{6}K\u{6}PK\u{3}\u{4}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\u{1}\u{1}\0PK\u{1}\u{2},\0\0\0\0\0\0\0\0\0\0\0P\u{7}\u{4}/.\0KP\0\0;\0\0\0\u{1e}\0\0\0\0\0\0\0\0\0\0\0\0\0", options)?; + let finished = writer.finish_into_readable()?; + assert_eq!(1, finished.file_names().count()); + writer = ZipWriter::new_append(finished.into_inner())?; + let options = FullFileOptions { + compression_method: Stored, + compression_level: Some(5), + last_modified_time: DateTime::from_date_and_time(2107, 4, 1, 0, 0, 0)?, + permissions: None, + large_file: false, + encrypt_with: Some(ZipCrypto( + ZipCryptoKeys::of(0x0, 0x62e4b50, 0x100), + PhantomData, + )), + ..Default::default() + }; + writer.add_symlink_from_path( + "\0K\u{6}\0PK\u{6}\u{7}PK\u{6}\u{6}\0\0\0\0\0\0\0\0PK\u{2}\u{6}", + "\u{8}\0\0\0\0/\0", + options, + )?; + let finished = writer.finish_into_readable()?; + assert_eq!(2, finished.file_names().count()); + writer = ZipWriter::new_append(finished.into_inner())?; + assert_eq!(2, writer.files.len()); + writer + }; + let finished = sub_writer.finish_into_readable()?; + assert_eq!(2, finished.file_names().count()); + writer.merge_archive(finished)?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[cfg(all(feature = "_bzip2_any", feature = "aes-crypto", not(miri)))] + #[test] + fn test_fuzz_crash_2024_06_18b() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(true); + writer.set_raw_comment([0].into()); + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + assert_eq!(writer.get_raw_comment()[0], 0); + let options = FileOptions { + compression_method: CompressionMethod::Bzip2, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2009, 6, 3, 13, 37, 39)?, + permissions: Some(2644352413), + large_file: true, + encrypt_with: Some(crate::write::EncryptWith::Aes { + mode: crate::AesMode::Aes256, + password: "", + }), + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 255, + ..Default::default() + }; + writer.add_symlink_from_path("", "", options)?; + writer.deep_copy_file_from_path("", "PK\u{5}\u{6}\0\0\0\0\0\0\0\0\0\0\0\0\0\u{4}\0\0\0")?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + assert_eq!(writer.get_raw_comment()[0], 0); + writer.deep_copy_file_from_path( + "PK\u{5}\u{6}\0\0\0\0\0\0\0\0\0\0\0\0\0\u{4}\0\0\0", + "\u{2}yy\u{5}qu\0", + )?; + let finished = writer.finish()?; + let archive = ZipArchive::new(finished.clone())?; + assert_eq!(archive.comment(), [0]); + writer = ZipWriter::new_append(finished)?; + assert_eq!(writer.get_raw_comment()[0], 0); + let _ = writer.finish_into_readable()?; + Ok(()) + } + + #[test] + fn test_fuzz_crash_2024_06_19() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 3, 1, 19, 55, 58)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 256, + ..Default::default() + }; + writer.start_file_from_path( + "\0\0\0PK\u{5}\u{6}\0\0\0\0\u{1}\0\u{12}\u{6}\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\0", + options, + )?; + writer.set_flush_on_finish_file(false); + writer.shallow_copy_file_from_path( + "\0\0\0PK\u{5}\u{6}\0\0\0\0\u{1}\0\u{12}\u{6}\0\0\0\0\0\u{1}\0\0\0\0\0\0\0\0\0", + "", + )?; + writer.set_flush_on_finish_file(false); + writer.deep_copy_file_from_path("", "copy")?; + writer.abort_file()?; + writer.set_flush_on_finish_file(false); + writer.set_raw_comment([255, 0].into()); + writer.abort_file()?; + assert_eq!(writer.get_raw_comment(), [255, 0]); + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + assert_eq!(writer.get_raw_comment(), [255, 0]); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::default(), + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + ..Default::default() + }; + writer.start_file_from_path("", options)?; + assert_eq!(writer.get_raw_comment(), [255, 0]); + let archive = writer.finish_into_readable()?; + assert_eq!(archive.comment(), [255, 0]); + Ok(()) + } + + #[test] + fn fuzz_crash_2024_06_21() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FullFileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 2, 1, 0, 0, 0)?, + permissions: None, + large_file: false, + encrypt_with: None, + ..Default::default() + }; + const LONG_PATH: &str = "\0@PK\u{6}\u{6}\u{7}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0@/\0\0\00ΝPK\u{5}\u{6}O\0\u{10}\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0@PK\u{6}\u{7}\u{6}\0/@\0\0\0\0\0\0\0\0 \0\0"; + writer.start_file_from_path(LONG_PATH, options)?; + writer = ZipWriter::new_append(writer.finish()?)?; + writer.deep_copy_file_from_path(LONG_PATH, "oo\0\0\0")?; + writer.abort_file()?; + writer.set_raw_comment([33].into()); + let archive = writer.finish_into_readable()?; + writer = ZipWriter::new_append(archive.into_inner())?; + assert!(writer.get_raw_comment().starts_with(&[33])); + let archive = writer.finish_into_readable()?; + assert!(archive.comment().starts_with(&[33])); + Ok(()) + } + + #[test] + #[cfg(all(feature = "_bzip2_any", not(miri)))] + fn fuzz_crash_2024_07_17() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: CompressionMethod::Bzip2, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2095, 2, 16, 21, 0, 1)?, + permissions: Some(84238341), + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![117, 99, 6, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 2, 0, 0, 0].into(), + central_extra_data: vec![].into(), + }, + alignment: 65535, + ..Default::default() + }; + writer.start_file_from_path("", options)?; + //writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.deep_copy_file_from_path("", "copy")?; + let _ = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + Ok(()) + } + + #[test] + fn fuzz_crash_2024_07_19() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 6, 1, 0, 34, 47)?, + permissions: None, + large_file: true, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 45232, + ..Default::default() + }; + writer.add_directory_from_path("", options)?; + writer.deep_copy_file_from_path("/", "")?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.deep_copy_file_from_path("", "copy")?; + let _ = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + Ok(()) + } + + #[test] + #[cfg(feature = "aes-crypto")] + fn fuzz_crash_2024_07_19a() -> ZipResult<()> { + use crate::write::EncryptWith::Aes; + use crate::AesMode::Aes128; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2107, 6, 5, 13, 0, 21)?, + permissions: None, + large_file: true, + encrypt_with: Some(Aes { + mode: Aes128, + password: "", + }), + extended_options: ExtendedFileOptions { + extra_data: vec![3, 0, 4, 0, 209, 53, 53, 8, 2, 61, 0, 0].into(), + central_extra_data: vec![].into(), + }, + alignment: 65535, + ..Default::default() + }; + writer.start_file_from_path("", options)?; + let _ = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + Ok(()) + } + + #[test] + fn fuzz_crash_2024_07_20() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(true); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2041, 8, 2, 19, 38, 0)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 0, + ..Default::default() + }; + writer.add_directory_from_path("\0\0\0\0\0\0\07黻", options)?; + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.set_flush_on_finish_file(false); + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::default(), + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 4, + ..Default::default() + }; + writer.add_directory_from_path("\0\0\0黻", options)?; + writer = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + writer.abort_file()?; + let options = FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(1980, 1, 1, 0, 7, 0)?, + permissions: Some(2663103419), + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 32256, + ..Default::default() + }; + writer.add_directory_from_path("\0", options)?; + writer = ZipWriter::new_append(writer.finish()?)?; + writer + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + let _ = ZipWriter::new_append(writer.finish_into_readable()?.into_inner())?; + Ok(()) + } + + #[test] + fn fuzz_crash_2024_07_21() -> ZipResult<()> { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + let sub_writer = { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.add_directory_from_path( + "", + FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::from_date_and_time(2105, 8, 1, 15, 0, 0)?, + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 0, + ..Default::default() + }, + )?; + writer.abort_file()?; + let mut writer = ZipWriter::new_append(writer.finish()?)?; + writer.add_directory_from_path( + "", + FileOptions { + compression_method: Stored, + compression_level: None, + last_modified_time: DateTime::default(), + permissions: None, + large_file: false, + encrypt_with: None, + extended_options: ExtendedFileOptions { + extra_data: vec![].into(), + central_extra_data: vec![].into(), + }, + alignment: 16, + ..Default::default() + }, + )?; + ZipWriter::new_append(writer.finish()?)? + }; + writer.merge_archive(sub_writer.finish_into_readable()?)?; + let writer = ZipWriter::new_append(writer.finish()?)?; + let _ = writer.finish_into_readable()?; + + Ok(()) + } +} diff --git a/v2/vendor/security/zip-7.2.0/src/zipcrypto.rs b/v2/vendor/security/zip-7.2.0/src/zipcrypto.rs new file mode 100644 index 0000000000..ee37b74b85 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/src/zipcrypto.rs @@ -0,0 +1,280 @@ +//! Implementation of the ZipCrypto algorithm +//! +//! The following paper was used to implement the ZipCrypto algorithm: +//! [https://courses.cs.ut.ee/MTAT.07.022/2015_fall/uploads/Main/dmitri-report-f15-16.pdf](https://courses.cs.ut.ee/MTAT.07.022/2015_fall/uploads/Main/dmitri-report-f15-16.pdf) + +use core::fmt::{Debug, Formatter}; +use core::hash::Hash; +use core::marker::PhantomData; +use core::num::Wrapping; + +use crate::cfg_if_expr; +use crate::result::ZipError; +#[cfg(feature = "aes-crypto")] +use crate::AesMode; + +#[derive(Copy, Clone, Debug, Eq, PartialEq)] +pub(crate) enum EncryptWith<'k> { + #[cfg(feature = "aes-crypto")] + Aes { + mode: AesMode, + password: &'k str, + }, + ZipCrypto(ZipCryptoKeys, PhantomData<&'k ()>), +} + +#[cfg(feature = "_arbitrary")] +impl<'a> arbitrary::Arbitrary<'a> for EncryptWith<'a> { + fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result { + #[cfg(feature = "aes-crypto")] + if bool::arbitrary(u)? { + return Ok(EncryptWith::Aes { + mode: AesMode::arbitrary(u)?, + password: u.arbitrary::<&str>()?, + }); + } + + Ok(EncryptWith::ZipCrypto( + ZipCryptoKeys::arbitrary(u)?, + PhantomData, + )) + } +} + +/// A container to hold the current key state +#[cfg_attr(feature = "_arbitrary", derive(arbitrary::Arbitrary))] +#[derive(Clone, Copy, Hash, Ord, PartialOrd, Eq, PartialEq)] +pub(crate) struct ZipCryptoKeys { + key_0: Wrapping, + key_1: Wrapping, + key_2: Wrapping, +} + +impl Debug for ZipCryptoKeys { + #[allow(unreachable_code)] + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + cfg_if_expr! { + #[cfg(any(test, fuzzing))] => { + f.write_fmt(format_args!( + "ZipCryptoKeys::of({:#10x},{:#10x},{:#10x})", + self.key_0, self.key_1, self.key_2 + )) + }, + _ => { + use std::collections::hash_map::DefaultHasher; + use std::hash::Hasher; + let mut t = DefaultHasher::new(); + self.hash(&mut t); + f.write_fmt(format_args!("ZipCryptoKeys(hash {})", t.finish())) + } + } + } +} + +impl ZipCryptoKeys { + const fn new() -> ZipCryptoKeys { + ZipCryptoKeys { + key_0: Wrapping(0x12345678), + key_1: Wrapping(0x23456789), + key_2: Wrapping(0x34567890), + } + } + + #[allow(unused)] + pub const fn of(key_0: u32, key_1: u32, key_2: u32) -> ZipCryptoKeys { + ZipCryptoKeys { + key_0: Wrapping(key_0), + key_1: Wrapping(key_1), + key_2: Wrapping(key_2), + } + } + + fn update(&mut self, input: u8) { + self.key_0 = ZipCryptoKeys::crc32(self.key_0, input); + self.key_1 = + (self.key_1 + (self.key_0 & Wrapping(0xff))) * Wrapping(0x08088405) + Wrapping(1); + self.key_2 = ZipCryptoKeys::crc32(self.key_2, (self.key_1 >> 24).0 as u8); + } + + fn stream_byte(&mut self) -> u8 { + let temp: Wrapping = Wrapping(self.key_2.0 as u16) | Wrapping(3); + ((temp * (temp ^ Wrapping(1))) >> 8).0 as u8 + } + + fn decrypt_byte(&mut self, cipher_byte: u8) -> u8 { + let plain_byte: u8 = self.stream_byte() ^ cipher_byte; + self.update(plain_byte); + plain_byte + } + + #[allow(dead_code)] + fn encrypt_byte(&mut self, plain_byte: u8) -> u8 { + let cipher_byte: u8 = self.stream_byte() ^ plain_byte; + self.update(plain_byte); + cipher_byte + } + + fn crc32(crc: Wrapping, input: u8) -> Wrapping { + (crc >> 8) ^ Wrapping(CRCTABLE[((crc & Wrapping(0xff)).0 as u8 ^ input) as usize]) + } + pub(crate) fn derive(password: &[u8]) -> ZipCryptoKeys { + let mut keys = ZipCryptoKeys::new(); + for byte in password.iter() { + keys.update(*byte); + } + keys + } +} + +/// A ZipCrypto reader with unverified password +pub struct ZipCryptoReader { + file: R, + keys: ZipCryptoKeys, +} + +pub enum ZipCryptoValidator { + PkzipCrc32(u32), + InfoZipMsdosTime(u16), +} + +impl ZipCryptoReader { + /// Note: The password is `&[u8]` and not `&str` because the + /// [zip specification](https://pkware.cachefly.net/webdocs/APPNOTE/APPNOTE-6.3.3.TXT) + /// does not specify password encoding (see function `update_keys` in the specification). + /// Therefore, if `&str` was used, the password would be UTF-8 and it + /// would be impossible to decrypt files that were encrypted with a + /// password byte sequence that is unrepresentable in UTF-8. + pub fn new(file: R, password: &[u8]) -> ZipCryptoReader { + ZipCryptoReader { + file, + keys: ZipCryptoKeys::derive(password), + } + } + + /// Read the ZipCrypto header bytes and validate the password. + pub fn validate( + mut self, + validator: ZipCryptoValidator, + ) -> Result, ZipError> { + // ZipCrypto prefixes a file with a 12 byte header + let mut header_buf = [0u8; 12]; + self.file.read_exact(&mut header_buf)?; + for byte in header_buf.iter_mut() { + *byte = self.keys.decrypt_byte(*byte); + } + + match validator { + ZipCryptoValidator::PkzipCrc32(crc32_plaintext) => { + // PKZIP before 2.0 used 2 byte CRC check. + // PKZIP 2.0+ used 1 byte CRC check. It's more secure. + // We also use 1 byte CRC. + + if (crc32_plaintext >> 24) as u8 != header_buf[11] { + return Err(ZipError::InvalidPassword); + } + } + ZipCryptoValidator::InfoZipMsdosTime(last_mod_time) => { + // Info-ZIP modification to ZipCrypto format: + // If bit 3 of the general purpose bit flag is set + // (indicates that the file uses a data-descriptor section), + // it uses high byte of 16-bit File Time. + // Info-ZIP code probably writes 2 bytes of File Time. + // We check only 1 byte. + + if (last_mod_time >> 8) as u8 != header_buf[11] { + return Err(ZipError::InvalidPassword); + } + } + } + + Ok(ZipCryptoReaderValid { reader: self }) + } +} +#[allow(unused)] +pub(crate) struct ZipCryptoWriter { + pub(crate) writer: W, + pub(crate) keys: ZipCryptoKeys, +} +impl ZipCryptoWriter { + #[allow(unused)] + pub(crate) fn finish(mut self) -> std::io::Result { + Ok(self.writer) + } +} +impl std::io::Write for ZipCryptoWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + const CHUNK_SIZE: usize = 4096; + let mut temp_buf = [0u8; CHUNK_SIZE]; + for chunk in buf.chunks(CHUNK_SIZE) { + let encrypted_chunk = &mut temp_buf[..chunk.len()]; + for (i, &byte) in chunk.iter().enumerate() { + encrypted_chunk[i] = self.keys.encrypt_byte(byte); + } + self.writer.write_all(encrypted_chunk)?; + } + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + self.writer.flush() + } +} + +/// A ZipCrypto reader with verified password +pub struct ZipCryptoReaderValid { + reader: ZipCryptoReader, +} + +impl std::io::Read for ZipCryptoReaderValid { + fn read(&mut self, buf: &mut [u8]) -> std::io::Result { + // Note: There might be potential for optimization. Inspiration can be found at: + // https://github.com/kornelski/7z/blob/master/CPP/7zip/Crypto/ZipCrypto.cpp + + let n = self.reader.file.read(buf)?; + for byte in buf.iter_mut().take(n) { + *byte = self.reader.keys.decrypt_byte(*byte); + } + Ok(n) + } +} + +impl ZipCryptoReaderValid { + /// Consumes this decoder, returning the underlying reader. + pub fn into_inner(self) -> R { + self.reader.file + } +} + +static CRCTABLE: [u32; 256] = [ + 0x00000000, 0x77073096, 0xee0e612c, 0x990951ba, 0x076dc419, 0x706af48f, 0xe963a535, 0x9e6495a3, + 0x0edb8832, 0x79dcb8a4, 0xe0d5e91e, 0x97d2d988, 0x09b64c2b, 0x7eb17cbd, 0xe7b82d07, 0x90bf1d91, + 0x1db71064, 0x6ab020f2, 0xf3b97148, 0x84be41de, 0x1adad47d, 0x6ddde4eb, 0xf4d4b551, 0x83d385c7, + 0x136c9856, 0x646ba8c0, 0xfd62f97a, 0x8a65c9ec, 0x14015c4f, 0x63066cd9, 0xfa0f3d63, 0x8d080df5, + 0x3b6e20c8, 0x4c69105e, 0xd56041e4, 0xa2677172, 0x3c03e4d1, 0x4b04d447, 0xd20d85fd, 0xa50ab56b, + 0x35b5a8fa, 0x42b2986c, 0xdbbbc9d6, 0xacbcf940, 0x32d86ce3, 0x45df5c75, 0xdcd60dcf, 0xabd13d59, + 0x26d930ac, 0x51de003a, 0xc8d75180, 0xbfd06116, 0x21b4f4b5, 0x56b3c423, 0xcfba9599, 0xb8bda50f, + 0x2802b89e, 0x5f058808, 0xc60cd9b2, 0xb10be924, 0x2f6f7c87, 0x58684c11, 0xc1611dab, 0xb6662d3d, + 0x76dc4190, 0x01db7106, 0x98d220bc, 0xefd5102a, 0x71b18589, 0x06b6b51f, 0x9fbfe4a5, 0xe8b8d433, + 0x7807c9a2, 0x0f00f934, 0x9609a88e, 0xe10e9818, 0x7f6a0dbb, 0x086d3d2d, 0x91646c97, 0xe6635c01, + 0x6b6b51f4, 0x1c6c6162, 0x856530d8, 0xf262004e, 0x6c0695ed, 0x1b01a57b, 0x8208f4c1, 0xf50fc457, + 0x65b0d9c6, 0x12b7e950, 0x8bbeb8ea, 0xfcb9887c, 0x62dd1ddf, 0x15da2d49, 0x8cd37cf3, 0xfbd44c65, + 0x4db26158, 0x3ab551ce, 0xa3bc0074, 0xd4bb30e2, 0x4adfa541, 0x3dd895d7, 0xa4d1c46d, 0xd3d6f4fb, + 0x4369e96a, 0x346ed9fc, 0xad678846, 0xda60b8d0, 0x44042d73, 0x33031de5, 0xaa0a4c5f, 0xdd0d7cc9, + 0x5005713c, 0x270241aa, 0xbe0b1010, 0xc90c2086, 0x5768b525, 0x206f85b3, 0xb966d409, 0xce61e49f, + 0x5edef90e, 0x29d9c998, 0xb0d09822, 0xc7d7a8b4, 0x59b33d17, 0x2eb40d81, 0xb7bd5c3b, 0xc0ba6cad, + 0xedb88320, 0x9abfb3b6, 0x03b6e20c, 0x74b1d29a, 0xead54739, 0x9dd277af, 0x04db2615, 0x73dc1683, + 0xe3630b12, 0x94643b84, 0x0d6d6a3e, 0x7a6a5aa8, 0xe40ecf0b, 0x9309ff9d, 0x0a00ae27, 0x7d079eb1, + 0xf00f9344, 0x8708a3d2, 0x1e01f268, 0x6906c2fe, 0xf762575d, 0x806567cb, 0x196c3671, 0x6e6b06e7, + 0xfed41b76, 0x89d32be0, 0x10da7a5a, 0x67dd4acc, 0xf9b9df6f, 0x8ebeeff9, 0x17b7be43, 0x60b08ed5, + 0xd6d6a3e8, 0xa1d1937e, 0x38d8c2c4, 0x4fdff252, 0xd1bb67f1, 0xa6bc5767, 0x3fb506dd, 0x48b2364b, + 0xd80d2bda, 0xaf0a1b4c, 0x36034af6, 0x41047a60, 0xdf60efc3, 0xa867df55, 0x316e8eef, 0x4669be79, + 0xcb61b38c, 0xbc66831a, 0x256fd2a0, 0x5268e236, 0xcc0c7795, 0xbb0b4703, 0x220216b9, 0x5505262f, + 0xc5ba3bbe, 0xb2bd0b28, 0x2bb45a92, 0x5cb36a04, 0xc2d7ffa7, 0xb5d0cf31, 0x2cd99e8b, 0x5bdeae1d, + 0x9b64c2b0, 0xec63f226, 0x756aa39c, 0x026d930a, 0x9c0906a9, 0xeb0e363f, 0x72076785, 0x05005713, + 0x95bf4a82, 0xe2b87a14, 0x7bb12bae, 0x0cb61b38, 0x92d28e9b, 0xe5d5be0d, 0x7cdcefb7, 0x0bdbdf21, + 0x86d3d2d4, 0xf1d4e242, 0x68ddb3f8, 0x1fda836e, 0x81be16cd, 0xf6b9265b, 0x6fb077e1, 0x18b74777, + 0x88085ae6, 0xff0f6a70, 0x66063bca, 0x11010b5c, 0x8f659eff, 0xf862ae69, 0x616bffd3, 0x166ccf45, + 0xa00ae278, 0xd70dd2ee, 0x4e048354, 0x3903b3c2, 0xa7672661, 0xd06016f7, 0x4969474d, 0x3e6e77db, + 0xaed16a4a, 0xd9d65adc, 0x40df0b66, 0x37d83bf0, 0xa9bcae53, 0xdebb9ec5, 0x47b2cf7f, 0x30b5ffe9, + 0xbdbdf21c, 0xcabac28a, 0x53b39330, 0x24b4a3a6, 0xbad03605, 0xcdd70693, 0x54de5729, 0x23d967bf, + 0xb3667a2e, 0xc4614ab8, 0x5d681b02, 0x2a6f2b94, 0xb40bbe37, 0xc30c8ea1, 0x5a05df1b, 0x2d02ef8d, +]; diff --git a/v2/vendor/security/zip-7.2.0/tests/ruview_0429_backport.rs b/v2/vendor/security/zip-7.2.0/tests/ruview_0429_backport.rs new file mode 100644 index 0000000000..747b45dc77 --- /dev/null +++ b/v2/vendor/security/zip-7.2.0/tests/ruview_0429_backport.rs @@ -0,0 +1,143 @@ +/// A symlink entry whose central directory declares an absurd uncompressed size +/// must be rejected, not pre-allocated. +/// +/// The declared size comes from the archive itself. Before the fix, `extract` +/// passed it straight to `Vec::with_capacity`, so a hostile entry could ask for +/// 2^62 bytes and abort the whole process ("memory allocation of ... failed"). +/// A symlink target is a path, so a size this large is malformed and the entry +/// must come back as an error. +#[cfg(not(miri))] +#[test] +fn extract_symlink_with_hostile_declared_size() -> zip::result::ZipResult<()> { + use std::io::Cursor; + use tempfile::TempDir; + use zip::ZipArchive; + + /// One entry, flagged as a symlink, whose Zip64 extra field claims + /// `declared_size` uncompressed bytes while storing only `payload`. + fn build(declared_size: u64, payload: &[u8]) -> Vec { + const S_IFLNK: u32 = 0o120_000; + const SENTINEL: u32 = 0xFFFF_FFFF; + let name = b"link"; + let mut o = Vec::new(); + + // local file header + o.extend_from_slice(&[0x50, 0x4b, 0x03, 0x04]); + o.extend_from_slice(&45u16.to_le_bytes()); // version needed: zip64 + o.extend_from_slice(&0u16.to_le_bytes()); // flags + o.extend_from_slice(&0u16.to_le_bytes()); // method: Stored + o.extend_from_slice(&0u16.to_le_bytes()); // mod time + o.extend_from_slice(&0x21u16.to_le_bytes()); // mod date + o.extend_from_slice(&0u32.to_le_bytes()); // crc32 + o.extend_from_slice(&SENTINEL.to_le_bytes()); + o.extend_from_slice(&SENTINEL.to_le_bytes()); + o.extend_from_slice(&(name.len() as u16).to_le_bytes()); + o.extend_from_slice(&20u16.to_le_bytes()); // zip64 extra field + o.extend_from_slice(name); + o.extend_from_slice(&0x0001u16.to_le_bytes()); // zip64 header id + o.extend_from_slice(&16u16.to_le_bytes()); + o.extend_from_slice(&declared_size.to_le_bytes()); + o.extend_from_slice(&(payload.len() as u64).to_le_bytes()); + o.extend_from_slice(payload); + + let cd_offset = o.len() as u32; + + // central directory header + o.extend_from_slice(&[0x50, 0x4b, 0x01, 0x02]); + o.extend_from_slice(&0x031Eu16.to_le_bytes()); // version made by: Unix + o.extend_from_slice(&45u16.to_le_bytes()); + o.extend_from_slice(&0u16.to_le_bytes()); + o.extend_from_slice(&0u16.to_le_bytes()); + o.extend_from_slice(&0u16.to_le_bytes()); + o.extend_from_slice(&0x21u16.to_le_bytes()); + o.extend_from_slice(&0u32.to_le_bytes()); // crc32 + o.extend_from_slice(&SENTINEL.to_le_bytes()); + o.extend_from_slice(&SENTINEL.to_le_bytes()); + o.extend_from_slice(&(name.len() as u16).to_le_bytes()); + o.extend_from_slice(&20u16.to_le_bytes()); + o.extend_from_slice(&0u16.to_le_bytes()); // comment len + o.extend_from_slice(&0u16.to_le_bytes()); // disk number start + o.extend_from_slice(&0u16.to_le_bytes()); // internal attrs + o.extend_from_slice(&((S_IFLNK | 0o777) << 16).to_le_bytes()); // symlink + o.extend_from_slice(&0u32.to_le_bytes()); // local header offset + o.extend_from_slice(name); + o.extend_from_slice(&0x0001u16.to_le_bytes()); + o.extend_from_slice(&16u16.to_le_bytes()); + o.extend_from_slice(&declared_size.to_le_bytes()); + o.extend_from_slice(&(payload.len() as u64).to_le_bytes()); + + let cd_size = o.len() as u32 - cd_offset; + + // end of central directory + o.extend_from_slice(&[0x50, 0x4b, 0x05, 0x06]); + o.extend_from_slice(&0u16.to_le_bytes()); + o.extend_from_slice(&0u16.to_le_bytes()); + o.extend_from_slice(&1u16.to_le_bytes()); + o.extend_from_slice(&1u16.to_le_bytes()); + o.extend_from_slice(&cd_size.to_le_bytes()); + o.extend_from_slice(&cd_offset.to_le_bytes()); + o.extend_from_slice(&0u16.to_le_bytes()); + o + } + + for declared_size in [4097, 1u64 << 40, 1u64 << 62, u64::MAX] { + let mut reader = ZipArchive::new(Cursor::new(build(declared_size, b"/etc/passwd")))?; + assert!(reader.by_index(0)?.is_symlink()); + let tempdir = TempDir::with_prefix("test_hostile_symlink_size")?; + + let err = reader + .extract(&tempdir) + .expect_err("oversized symlink target must be rejected"); + assert!( + matches!(err, zip::result::ZipError::InvalidArchive(_)), + "expected InvalidArchive for a {declared_size} byte symlink target, got {err:?}" + ); + } + + Ok(()) +} + + +#[test] +fn stream_does_not_expose_the_symlink_bit_backport() -> zip::result::ZipResult<()> { + use std::io::Cursor; + use zip::{ZipWriter, write::SimpleFileOptions}; + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + writer.add_symlink("link", "/target", SimpleFileOptions::default())?; + let mut cur = writer.finish()?; + cur.set_position(0); + let file = zip::read::read_zipfile_from_stream(&mut cur)?.unwrap(); + assert!(!file.is_symlink()); + Ok(()) +} + +#[test] +fn valid_regular_file_roundtrip() -> zip::result::ZipResult<()> { + use std::io::{Cursor, Write}; + use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions}; + let mut w = ZipWriter::new(Cursor::new(Vec::new())); + w.start_file("regular.txt", SimpleFileOptions::default())?; + w.write_all(b"ordinary archive payload")?; + let mut bytes = w.finish()?; bytes.set_position(0); + let dir = tempfile::TempDir::new()?; + ZipArchive::new(bytes)?.extract(dir.path())?; + assert_eq!(std::fs::read(dir.path().join("regular.txt"))?, b"ordinary archive payload"); + Ok(()) +} + +#[cfg(unix)] +#[test] +fn valid_relative_symlink_roundtrip() -> zip::result::ZipResult<()> { + use std::io::{Cursor, Write}; + use zip::{ZipArchive, ZipWriter, write::SimpleFileOptions}; + let mut w = ZipWriter::new(Cursor::new(Vec::new())); + w.start_file("target", SimpleFileOptions::default())?; + w.write_all(b"target payload")?; + w.add_symlink("link", "target", SimpleFileOptions::default())?; + let mut bytes = w.finish()?; bytes.set_position(0); + let dir = tempfile::TempDir::new()?; + ZipArchive::new(bytes)?.extract(dir.path())?; + assert_eq!(std::fs::read_link(dir.path().join("link"))?, std::path::PathBuf::from("target")); + assert_eq!(std::fs::read(dir.path().join("link"))?, b"target payload"); + Ok(()) +} From 069b079b39261dd3f366b6a64bc315f3c705c0ab Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Wed, 4 Nov 2026 09:11:40 +0100 Subject: [PATCH 058/100] fix(learner): expose protected source sequence admission diagnostics --- .../src/room_learner/capture.rs | 34 +++++- .../src/room_learner/capture_recovery.rs | 1 + .../room_learner/capture_recovery_tests.rs | 100 ++++++++++++++++++ .../src/room_learner/lifecycle_tests.rs | 42 ++++++++ ...om_learner_api_selection_composer_tests.rs | 4 + 5 files changed, 180 insertions(+), 1 deletion(-) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs index 4d8c92206d..f7253d7b37 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs @@ -11,6 +11,8 @@ pub const MIN_PHASE_MS: u64 = 12_000; pub const MAX_PHASE_MS: u64 = 600_000; pub const MAX_CAPTURE_BINS: usize = 128; const MAX_PHASE_HISTORY: usize = 128; +// Protected diagnostics are consumed by JavaScript clients as exact integers. +const MAX_SEQUENCE_DIAGNOSTIC_COUNT: u64 = (1u64 << 53) - 1; /// Borrowed actual packet metadata. The caller must reject unknown bandwidth /// and retain the known raw grid identity. Amplitudes may be length-only @@ -59,6 +61,11 @@ struct SourceBuffer { ordinal: u64, last_time_ns: u64, admitted_packets: u64, + // Transient diagnostics only. Never restore observations/rejections as + // fresh input, and never let them advance the admission watermark. + last_observed_sequence: Option, + sequence_rejection_count: u64, + sequence_rejection_reason: Option, live: VecDeque, } #[derive(Debug)] @@ -71,6 +78,12 @@ struct Phase { admitted: HashMap, } +/// A bounded diagnostic of the most recent sequence admission failure. +/// It does not establish a source reboot or authenticated source epoch. +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum SequenceRejectionReason { Duplicate, Reversed } + /// Counts and receiver freshness only. No learned/qualified state is implied. #[derive(Debug, Clone, Serialize)] pub struct CaptureProgress { @@ -98,6 +111,13 @@ pub struct SourceProgress { pub source_id: u8, pub grid_id: String, pub admitted_packets: u64, + /// Last admitted wire sequence, including an original recovery watermark. + pub last_accepted_sequence: Option, + /// Last sequence tested by this receiver after PHY/shape/time validation. + pub last_observed_sequence: Option, + /// Transient since this capture was created/restored; never persisted. + pub sequence_rejection_count: u64, + pub sequence_rejection_reason: Option, pub phase_admitted_packets: u64, pub live_buffer_frames: usize, pub phase_buffer_frames: usize, @@ -166,6 +186,10 @@ impl CaptureSession { else if source.live.len() != WINDOW_PACKETS { Some("incomplete_window") } else { None }; SourceProgress { source_id: spec.source_id, grid_id: spec.grid_id.clone(), admitted_packets: source.admitted_packets, + last_accepted_sequence: source.sequence, + last_observed_sequence: source.last_observed_sequence, + sequence_rejection_count: source.sequence_rejection_count, + sequence_rejection_reason: source.sequence_rejection_reason, phase_admitted_packets: self.phase.as_ref().and_then(|p| p.admitted.get(&spec.source_id)).copied().unwrap_or(0), live_buffer_frames: source.live.len(), phase_buffer_frames: self.phase.as_ref().and_then(|p| p.pending.get(&spec.source_id)).map_or(0, VecDeque::len), @@ -209,10 +233,17 @@ impl CaptureSession { } let source = self.sources.get_mut(&frame.source_id).ok_or("source buffer unavailable")?; if now_ns == 0 || now_ns < self.last_seen_ns || now_ns <= source.last_time_ns { return Err("nonmonotonic receiver time".into()); } + source.last_observed_sequence = Some(frame.sequence); let delta = match source.sequence { Some(last) => { let delta = frame.sequence.wrapping_sub(last); - if delta == 0 || delta >= (1u32 << 31) { return Err("replayed or reversed source sequence".into()); } + if delta == 0 || delta >= (1u32 << 31) { + source.sequence_rejection_count = source.sequence_rejection_count.saturating_add(1).min(MAX_SEQUENCE_DIAGNOSTIC_COUNT); + source.sequence_rejection_reason = Some(if delta == 0 { + SequenceRejectionReason::Duplicate + } else { SequenceRejectionReason::Reversed }); + return Err("replayed or reversed source sequence".into()); + } u64::from(delta) } None => 1, @@ -221,6 +252,7 @@ impl CaptureSession { let packet = Packet { values: frame.amplitudes.iter().map(|x| *x as f32).collect(), frame_id: (u128::from(self.prefix) << 64) | (u128::from(frame.source_id) << 56) | u128::from(ordinal), ordinal, received_ns: now_ns }; + source.sequence_rejection_reason = None; source.sequence = Some(frame.sequence); source.ordinal = ordinal; source.last_time_ns = now_ns; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery.rs index 6a0e9f876d..792ccc456c 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery.rs @@ -175,6 +175,7 @@ impl CaptureSession { last_time_ns: watermark.last_time_ns, admitted_packets: watermark.admitted_packets, live: VecDeque::new(), + ..SourceBuffer::default() }, ); } diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery_tests.rs index a6b9afb983..e819e40cca 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery_tests.rs @@ -330,3 +330,103 @@ fn unlabelled_live_windows_and_partial_packet_buffers_are_never_serialized() { .unwrap() .contains("\"samples\"")); } + +#[test] +fn recovered_sequence_diagnostics_explain_lower_input_without_rebasing_evidence() { + let mut s = session(); + let high = 1_000_000; + phase(&mut s, START, high, DataRole::Train, RoomLabel::Empty, "high", 2.0); + let at = START + MIN_PHASE_MS * 1_000_000; + let saved = s.checkpoint(at).unwrap(); + let original_rows = serde_json::to_vec(s.cohorts()).unwrap(); + let original_watermarks = serde_json::to_vec(&saved.sources).unwrap(); + let mut recovered = reopen(&saved, at + 1).unwrap().session; + let first = &recovered.progress(at + 1).sources[0]; + assert_eq!(first.last_accepted_sequence, Some(high + 31)); + assert_eq!(first.last_observed_sequence, None); + assert_eq!(first.sequence_rejection_count, 0); + assert_eq!(first.sequence_rejection_reason, None); + let values = vec![2.0; 64]; + for (index, seq) in [1, 2, high + 31].into_iter().enumerate() { + assert!(recovered.ingest(packet(1, seq, &values), at + 2 + index as u64).is_err()); + } + let progress = recovered.progress(at + 6); + let source = &progress.sources[0]; + assert_eq!(source.last_accepted_sequence, Some(high + 31)); + assert_eq!(source.last_observed_sequence, Some(high + 31)); + assert_eq!(source.sequence_rejection_count, 3); + assert_eq!(source.sequence_rejection_reason, Some(SequenceRejectionReason::Duplicate)); + assert_eq!(source.admitted_packets, 32); + assert_eq!(source.live_buffer_frames, 0); + assert!(!source.window_eligible); + assert_eq!(progress.sources[1].sequence_rejection_count, 0); + assert_eq!(serde_json::to_vec(&recovered.checkpoint(at + 6).unwrap().sources).unwrap(), original_watermarks); + assert_eq!(serde_json::to_vec(recovered.cohorts()).unwrap(), original_rows); + assert!(recovered.begin_phase(request(DataRole::HeldOut, RoomLabel::Empty, "high"), at + 7).is_err()); + + window(&mut recovered, at + 100_000_000, high + 32, 3.0); + let progress = recovered.progress(at + 180_000_000); + assert!(progress.sources.iter().all(|s| s.window_eligible && s.last_accepted_sequence == Some(high + 39))); + assert_eq!(progress.sources[0].last_observed_sequence, Some(high + 39)); + assert_eq!(progress.sources[0].sequence_rejection_count, 3); + assert_eq!(progress.sources[0].sequence_rejection_reason, None); + assert_eq!(progress.sources[0].admitted_packets, 40); + let live = recovered.live_cohort(at + 180_000_000).unwrap(); + assert!(live.views.iter().all(|v| v.sequence_ids == (33..=40).collect::>())); + assert_eq!(serde_json::to_vec(recovered.cohorts()).unwrap(), original_rows); + + let checkpoint = recovered.checkpoint(at + 200_000_000).unwrap(); + let serialized = serde_json::to_string(&checkpoint).unwrap(); + for transient in ["last_observed_sequence", "sequence_rejection_count", "sequence_rejection_reason"] { + assert!(!serialized.contains(transient), "diagnostics must not become retained evidence: {transient}"); + } + let restored = reopen(&checkpoint, at + 200_000_001).unwrap().session; + let progress = restored.progress(at + 200_000_001); + assert!(progress.sources.iter().all(|s| s.last_observed_sequence.is_none() && s.sequence_rejection_count == 0 && s.sequence_rejection_reason.is_none() && !s.window_eligible)); +} + +#[test] +fn reversed_sequence_reason_is_source_local_bounded_and_does_not_claim_reboot() { + let mut s = session(); + window(&mut s, START, 100, 2.0); + let values = vec![2.0; 64]; + s.sources.get_mut(&1).unwrap().sequence_rejection_count = MAX_SEQUENCE_DIAGNOSTIC_COUNT; + assert!(s.ingest(packet(1, 1, &values), START + 90_000_000).is_err()); + let progress = s.progress(START + 90_000_000); + let source = &progress.sources[0]; + assert_eq!(source.last_observed_sequence, Some(1)); + assert_eq!(source.last_accepted_sequence, Some(107)); + assert_eq!(source.sequence_rejection_count, MAX_SEQUENCE_DIAGNOSTIC_COUNT); + assert_eq!(source.sequence_rejection_reason, Some(SequenceRejectionReason::Reversed)); + assert_eq!(progress.sources[1].sequence_rejection_reason, None); + let diagnostic = serde_json::to_value(&progress).unwrap(); + assert_eq!(diagnostic["sources"][0]["sequence_rejection_reason"], "reversed"); + assert_eq!(diagnostic["sources"][0]["sequence_rejection_count"], 9_007_199_254_740_991_u64); + let json = serde_json::to_string(&progress).unwrap(); + for forbidden in ["amplitudes", "samples", "frame_ids", "ordinal", "receiver_epoch", "reboot"] { + assert!(!json.contains(forbidden), "unexpected diagnostic data/claim: {forbidden}"); + } +} + +#[test] +fn valid_sequence_wrap_remains_admitted_and_invalid_time_cannot_launder_diagnostics() { + let mut s = session(); + window(&mut s, START, u32::MAX - 3, 2.0); + let at = START + 80_000_000; + let checkpoint = s.checkpoint(at).unwrap(); + let mut recovered = reopen(&checkpoint, at + 1).unwrap().session; + window(&mut recovered, at + 100_000_000, 4, 3.0); + let source = &recovered.progress(at + 180_000_000).sources[0]; + assert_eq!(source.last_accepted_sequence, Some(11)); + assert_eq!(source.last_observed_sequence, Some(11)); + assert_eq!(source.sequence_rejection_count, 0); + assert_eq!(source.sequence_rejection_reason, None); + let values = vec![3.0; 64]; + assert!(recovered.ingest(packet(1, 2, &values), at + 190_000_000).is_err()); + assert!(recovered.ingest(packet(1, 999, &values), START).is_err()); + let source = &recovered.progress(at + 200_000_000).sources[0]; + assert_eq!(source.last_observed_sequence, Some(2)); + assert_eq!(source.last_accepted_sequence, Some(11)); + assert_eq!(source.sequence_rejection_count, 1); + assert_eq!(source.sequence_rejection_reason, Some(SequenceRejectionReason::Reversed)); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs index 9eac2f729c..fb67f28562 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs @@ -518,3 +518,45 @@ fn native_rejection_diagnostics_are_context_bound_bounded_and_not_admission() { assert_eq!(status["capture"]["sources"][0]["admitted_packets"],0); assert_eq!(status["physically_validated"],false); } + +#[test] +fn protected_progress_explains_rejections_without_advancing_admission_or_live_authority() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + rt.configure(&rt.mutation_guard().unwrap(), context(), "synthetic-sequence-diagnostics".into(), true).unwrap(); + let values = vec![2.0; 8]; + let input = |source_id, sequence| RawSourceFrame { + source_id, grid_id: "ht64-canonical8", frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, sequence, amplitudes: &values, + }; + for sample in 0..8 { + for source in 1..=2 { + rt.ingest(input(source, 100 + sample), START + 1 + u64::from(sample) * 10_000_000 + u64::from(source)).unwrap(); + } + } + assert!(rt.status(START + 80_000_000)["capture"]["sources"].as_array().unwrap().iter().all(|s| s["window_eligible"] == true)); + assert!(rt.ingest(input(1, 1), START + 90_000_000).is_err()); + let status = rt.status(START + 90_000_000); + let source = &status["capture"]["sources"][0]; + assert_eq!(source["last_accepted_sequence"], 107); + assert_eq!(source["last_observed_sequence"], 1); + assert_eq!(source["sequence_rejection_count"], 1); + assert_eq!(source["sequence_rejection_reason"], "reversed"); + assert_eq!(source["admitted_packets"], 8); + assert!(status["capture"]["sources"].as_array().unwrap().iter().all(|s| s["live_buffer_frames"] == 0 && s["window_eligible"] == false)); + assert_eq!(status["input_error"], "replayed or reversed source sequence"); + let json = serde_json::to_string(&status["capture"]).unwrap(); + for raw in ["samples", "amplitudes", "frame_ids", "sequence_ids"] { + assert!(!json.contains(raw)); + } + for sample in 0..8 { + for source in 1..=2 { + rt.ingest(input(source, 108 + sample), START + 100_000_000 + u64::from(sample) * 10_000_000 + u64::from(source)).unwrap(); + } + } + let fresh = rt.status(START + 180_000_000); + assert!(fresh["input_error"].is_null()); + assert_eq!(fresh["capture"]["sources"][0]["sequence_rejection_count"], 1); + assert!(fresh["capture"]["sources"][0]["sequence_rejection_reason"].is_null()); + assert!(fresh["capture"]["sources"].as_array().unwrap().iter().all(|s| s["last_accepted_sequence"] == 115 && s["last_observed_sequence"] == 115 && s["window_eligible"] == true)); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs index 6f148d3ee3..5ad6f1b575 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_composer_tests.rs @@ -193,6 +193,10 @@ async fn actual_selected_composer_withdraws_without_fallback_and_never_grants_au "training_samples", "head_weights", "geometry", + "last_accepted_sequence", + "last_observed_sequence", + "sequence_rejection_count", + "sequence_rejection_reason", ] { assert!( !serialized.contains(forbidden), From dbbc58cf97b1eaaf0f78478de9468a45b30ab7a1 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Wed, 4 Nov 2026 09:32:54 +0100 Subject: [PATCH 059/100] feat(worldgraph): publish bounded managed room topology --- ...9-worldgraph-environmental-digital-twin.md | 34 ++++ .../src/companion_integration.rs | 3 + .../src/companion_room.rs | 8 + .../wifi-densepose-sensing-server/src/main.rs | 2 + .../src/managed_worldgraph.rs | 154 ++++++++++++++++++ .../tests/fixtures/managed-worldgraph-v1.json | 51 ++++++ 6 files changed, 252 insertions(+) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/managed_worldgraph.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/tests/fixtures/managed-worldgraph-v1.json diff --git a/docs/adr/ADR-139-worldgraph-environmental-digital-twin.md b/docs/adr/ADR-139-worldgraph-environmental-digital-twin.md index d1a83624fa..c5277188cf 100644 --- a/docs/adr/ADR-139-worldgraph-environmental-digital-twin.md +++ b/docs/adr/ADR-139-worldgraph-environmental-digital-twin.md @@ -585,3 +585,37 @@ Add rows to `docs/WITNESS-LOG-028.md`: **Integration glue -- not yet on the live path:** feeding live fusion outputs and person tracks into nodes; the full `.rvf` bundle container (today it persists as JSON); and the live ADR-141 privacy-mode reducer. **Trust contribution:** the auditable map -- evidence and contradiction are first-class edges, and the privacy posture is *visible in the persisted graph* (an auditor can read what was suppressed). + + +## 2026-11-04 amendment: managed configured topology transport + +Status: implemented source slice; native delivery and physical geometry acceptance remain pending. + +The app-managed sensing server exposes authenticated `GET /api/v1/worldgraph/snapshot` +as protocol 2 when its authoritative companion room store is available. Protocol 1 +and genuine registered ENU WorldGraph snapshots remain a separate unchanged contract. +Protocol 2 is `ruview.worldgraph.managed-topology.v1`, with `managed_room_local` +coordinates and `x_right_y_down_metres` axes (room map origin at top left). Its only +geometry provenance is `operator_entered`; geographic registration and sensor +heights are explicitly null. Room dimensions may include the operator-entered +room height, but this never supplies sensor height or AR alignment. + +The projection takes one immutable published room envelope, preserving installation, +revision and binding digest. Canonical room/sensor and located-in vocabulary is +reused without injecting the engine's default room, person/semantic/vital records, +raw CSI or phone geometry. The configured AP remains part of the original room +binding but is not invented as a CSI receiver or RF link. This slice is not an ENU +registration, a spatial teacher, localization, occupancy model or model-selection +receipt. It requires no additional persistent graph store. + +An ephemeral random `wg-` epoch and safe-integer cursor belong to the loaded account/ +installation store. Equal canonical topology uses the same sequence; readiness, +binding or privacy changes advance it. Reads never publish geometry, renew a +baseline, train or select a model. Global server privacy mode conservatively returns +`privacy_denied` with no geometry. No published room yields `not_configured`; +missing managed support remains unsupported, not zero people. The same protected +owner and paired read paths apply, with `Cache-Control: private, no-store`. + +The managed client shares capability state between Zones, Settings and MAT, and +never converts this planar topology to a fabricated geographic or phone AR frame. +Cloud Cognitum Spaces/Zone projection remains separate under ADR-325. diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs index de2aacf3c1..7f6e1ff64c 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs @@ -258,6 +258,7 @@ fn policy(method: &axum::http::Method, path: &str) -> Option { | "/api/v1/pose/zones/summary" | "/api/v1/pose/activities" | "/api/v1/room-learner/selection" + | "/api/v1/worldgraph/snapshot" | "/api/v1/calibration/status" | "/api/v1/pose/calibration/status" | "/ws/sensing" @@ -385,6 +386,8 @@ mod tests { } assert_eq!(policy(&Method::GET, "/ws/train/progress"), None); assert_eq!(policy(&Method::GET, "/api/v1/room-learner/selection"), Some(false)); + assert_eq!(policy(&Method::GET, "/api/v1/worldgraph/snapshot"), Some(false)); + assert_eq!(policy(&Method::POST, "/api/v1/worldgraph/snapshot"), None); assert_eq!(policy(&Method::HEAD, "/api/v1/room-learner/selection"), Some(false)); assert_eq!(policy(&Method::GET, "/api/v1/room-learner/status"), None); for path in ["/api/v1/room-learner/qualify", "/api/v1/room-learner/select", "/api/v1/room-learner/deselect", "/api/v1/room-learner/phase/cancel", "/api/v1/room-learner/train/cancel"] { diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_room.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_room.rs index 12ec77f0d3..66f9d9d73a 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/companion_room.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_room.rs @@ -34,6 +34,7 @@ pub struct Envelope { pub struct SharedRoomStore { path: PathBuf, current: Mutex, + worldgraph_cursor: Mutex, } fn invalid() -> std::io::Error { std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid shared room state") @@ -240,8 +241,15 @@ impl SharedRoomStore { Ok(Arc::new(Self { path, current: Mutex::new(current), + worldgraph_cursor: Mutex::new(crate::managed_worldgraph::Cursor::default()), })) } + /// Lock publication then cursor: no mixed binding/geometry response under CAS. + pub fn worldgraph_snapshot(&self, privacy_denied: bool) -> Result { + let room = self.current.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; + self.worldgraph_cursor.lock().map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?.project(&room, privacy_denied) + } + pub fn snapshot(&self) -> Option { self.current.lock().ok().map(|v| v.clone()) } pub fn matches_binding(&self, binding: &str, node_ids: &[u8]) -> bool { let Ok(e) = self.current.lock() else { diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index a156bb000a..af912a183c 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -12,6 +12,7 @@ mod adaptive_classifier; mod companion_integration; mod companion_room; +mod managed_worldgraph; mod companion_spatial; pub mod cli; pub mod csi; @@ -16394,6 +16395,7 @@ async fn main() { .route("/api/v1/rf/vendors/:vendor/events", post(ingest_vendor_events)) // Per-node health endpoint .route("/api/v1/nodes", get(nodes_endpoint)) + .route("/api/v1/worldgraph/snapshot", get(managed_worldgraph::snapshot)) // ADR-110 iter 29 — per-node mesh sync state for HTTP clients. .route("/api/v1/nodes/:id/sync", get(node_sync_endpoint)) .route("/api/v1/mesh", get(mesh_endpoint)) diff --git a/v2/crates/wifi-densepose-sensing-server/src/managed_worldgraph.rs b/v2/crates/wifi-densepose-sensing-server/src/managed_worldgraph.rs new file mode 100644 index 0000000000..449d6827b8 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/managed_worldgraph.rs @@ -0,0 +1,154 @@ +//! Configured topology, not a geographic registration or perception result. +use axum::{extract::State, http::{header, StatusCode}, response::{IntoResponse, Response}, Json}; +use serde_json::{json, Value}; +use crate::{companion_room::Envelope, SharedState}; + +const MAX_SAFE: u64 = 9_007_199_254_740_991; + +/// Ephemeral per-authority cursor. Reads of identical content never freshen it. +pub struct Cursor { epoch: String, sequence: u64, content: Option } +impl Default for Cursor { + fn default() -> Self { + let random: [u8; 32] = rand::random(); + Self { epoch: format!("wg-{}", random.iter().map(|b| format!("{b:02x}")).collect::()), sequence: 0, content: None } + } +} +impl Cursor { + pub fn project(&mut self, envelope: &Envelope, privacy_denied: bool) -> Result { + let content = topology(envelope, privacy_denied)?; + if self.content.as_ref() != Some(&content) { + if self.sequence >= MAX_SAFE { return Err(StatusCode::SERVICE_UNAVAILABLE); } + self.sequence += 1; + self.content = Some(content.clone()); + } + Ok(json!({"protocol_version":2,"stream_epoch":self.epoch,"seq":self.sequence,"topology":content})) + } +} + +fn topology(envelope: &Envelope, privacy_denied: bool) -> Result { + let mut result = json!({"schema":"ruview.worldgraph.managed-topology.v1", + "coordinate_frame":"managed_room_local","axis_convention":"x_right_y_down_metres","provenance":"operator_entered", + "registration":null,"installation_id":envelope.installation_id, + "room_revision":envelope.revision,"binding_digest":null, + "status":if privacy_denied {"privacy_denied"} else {"not_configured"}, + "room":null,"access_point":null,"nodes":[],"edges":[]}); + if privacy_denied { return Ok(result); } + let Some(room) = &envelope.room else { return Ok(result); }; + let binding = envelope.binding_digest.as_deref().ok_or(StatusCode::INTERNAL_SERVER_ERROR)?; + let source_nodes = room["nodes"].as_array().ok_or(StatusCode::INTERNAL_SERVER_ERROR)?; + if envelope.revision == 0 || source_nodes.is_empty() || source_nodes.len() > 16 { + return Err(StatusCode::INTERNAL_SERVER_ERROR); + } + // The store validates/sorts this immutable publication before exposing it. + // Allowlist fields: never serialize live engine/default room, tracks, media, + // semantic prose, learned history, raw CSI, credentials or vital estimates. + let mut nodes: Vec = source_nodes.iter().map(|node| json!({ + "kind":"sensor","id":257 + node["nodeId"].as_u64().unwrap_or(256), + "node_id":node["nodeId"],"slot":node["slot"],"x_m":node["xM"],"y_m":node["yM"], + "height_m":null,"role":node["role"],"heading_deg":node["headingDeg"] + })).collect(); + nodes.sort_by_key(|node| node["id"].as_u64().unwrap_or(0)); + let edges: Vec = nodes.iter().map(|node| json!({"from":node["id"],"to":1,"rel":"located_in"})).collect(); + result["status"] = json!("configured"); + result["binding_digest"] = json!(binding); + result["room"] = json!({"kind":"room","id":1,"room_id":room["id"],"name":room["name"], + "width_m":room["widthM"],"length_m":room["lengthM"],"height_m":room["heightM"]}); + let ap = &room["accessPoint"]; + if !ap.is_null() { result["access_point"] = json!({"slot":ap["slot"],"x_m":ap["xM"],"y_m":ap["yM"],"provenance":"operator_entered"}); } + result["nodes"] = json!(nodes); + result["edges"] = json!(edges); + Ok(result) +} + +pub async fn snapshot(State(state): State) -> Result { + let state = state.read().await; + let store = state.current_room_store.as_ref().ok_or(StatusCode::NOT_IMPLEMENTED)?; + let value = store.worldgraph_snapshot(state.privacy_mode)?; + Ok(([(header::CACHE_CONTROL, "private, no-store")], Json(value)).into_response()) +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::{body::{Body, to_bytes}, http::Request, routing::get, Router}; + use tower::ServiceExt; + fn envelope() -> Envelope { + Envelope { installation_id: "installation-1".into(), revision: 7, binding_digest: Some("a".repeat(64)), room: Some(json!({ + "id":"room-1","name":"Configured room","widthM":2.5,"lengthM":3.0,"heightM":2.4, + "nodes":[{"nodeId":15,"slot":"north_west","xM":1.25,"yM":2.0,"role":"receiver","headingDeg":150}], + "raw_csi":"forbidden","person_track":"forbidden","accessPoint":{"slot":"north_east","xM":0,"yM":0} + })) } + } + #[test] + fn configured_projection_is_planar_and_allowlisted() { + let value = topology(&envelope(), false).unwrap(); + assert_eq!(value["room"]["width_m"], 2.5); + assert_eq!(value["nodes"][0]["x_m"], 1.25); + assert_eq!(value["nodes"][0]["id"], 272); + assert!(value["nodes"][0]["height_m"].is_null()); + assert!(value["registration"].is_null()); + assert_eq!(value["edges"][0], json!({"from":272,"to":1,"rel":"located_in"})); + let text = value.to_string(); + for forbidden in ["raw_csi","person_track","accessPoint","confidence","latitude"] { assert!(!text.contains(forbidden)); } + } + #[test] + fn cursors_cover_privacy_binding_readiness_and_restart() { + let mut cursor = Cursor::default(); let original = envelope(); + let a = cursor.project(&original, false).unwrap(); + assert_eq!(a, cursor.project(&original, false).unwrap()); + let denied = cursor.project(&original, true).unwrap(); + assert_eq!(denied["seq"], 2); + assert!(denied["topology"]["room"].is_null()); assert_eq!(denied["topology"]["nodes"], json!([])); + let mut changed = original.clone(); changed.revision += 1; changed.binding_digest = Some("b".repeat(64)); + assert_eq!(cursor.project(&changed, false).unwrap()["seq"], 3); + let mut empty = changed; empty.room = None; empty.binding_digest = None; + let absent = cursor.project(&empty, false).unwrap(); assert_eq!(absent["topology"]["status"], "not_configured"); + assert_ne!(a["stream_epoch"], Cursor::default().project(&original, false).unwrap()["stream_epoch"]); + assert_eq!(original.revision, 7); + } + #[test] + fn refuses_missing_binding_and_cursor_overflow() { + let mut missing = envelope(); missing.binding_digest = None; assert!(topology(&missing, false).is_err()); + let mut cursor = Cursor::default(); cursor.sequence = MAX_SAFE; assert!(cursor.project(&envelope(), false).is_err()); + } + #[test] + fn actual_projection_matches_cross_language_fixture() { + let mut cursor = Cursor { epoch: format!("wg-{}", "1".repeat(64)), sequence: 0, content: None }; + let wire = cursor.project(&envelope(), false).unwrap(); + let fixture: Value = serde_json::from_str(include_str!("../tests/fixtures/managed-worldgraph-v1.json")).unwrap(); + assert_eq!(wire, fixture); + } + #[tokio::test] + async fn real_route_requires_read_authority_and_never_publishes_or_selects() { + use wifi_densepose_sensing_server::bearer_auth::{AuthState, require_bearer}; + let directory = tempfile::tempdir().unwrap(); + let store = crate::companion_room::SharedRoomStore::load(directory.path(), "installation-1").unwrap(); + let publication = json!({"expected_revision":0,"room":{ + "schema":"ruview.companion.room.v1","id":"room-1","name":"Configured room","widthM":2.5,"lengthM":3,"heightM":2.4, + "nodes":[{"nodeId":15,"slot":"north_west","xM":1.25,"yM":2,"role":"receiver","headingDeg":150,"orientationConfirmed":true,"confirmedAtUnixMs":1}], + "accessPoint":{"slot":"south_east","xM":2,"yM":2,"provenance":"operator_entered","confirmedAtUnixMs":1} + }}); + let published = crate::companion_room::owner_router(store.clone()).oneshot(Request::builder().method("PUT") + .uri("/api/v1/companion/room").header("content-type","application/json").body(Body::from(publication.to_string())).unwrap()).await.unwrap(); + assert_eq!(published.status(), StatusCode::OK); + let original = store.snapshot().unwrap(); + let mut inner = crate::AppStateInner::minimal(); inner.current_room_store = Some(store.clone()); + let state = std::sync::Arc::new(tokio::sync::RwLock::new(inner)); + let app = Router::new().route("/api/v1/worldgraph/snapshot", get(snapshot)).with_state(state.clone()) + .layer(axum::middleware::from_fn_with_state(AuthState::from_token("owner-test-token"), require_bearer)); + let request = || Request::builder().uri("/api/v1/worldgraph/snapshot").header("authorization","Bearer owner-test-token").body(Body::empty()).unwrap(); + let denied = app.clone().oneshot(Request::builder().uri("/api/v1/worldgraph/snapshot").body(Body::empty()).unwrap()).await.unwrap(); + assert_eq!(denied.status(), StatusCode::UNAUTHORIZED); + let response = app.clone().oneshot(request()).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); assert_eq!(response.headers()[header::CACHE_CONTROL], "private, no-store"); + let first: Value = serde_json::from_slice(&to_bytes(response.into_body(), 16_384).await.unwrap()).unwrap(); + assert_eq!(first["topology"]["status"],"configured"); + assert_eq!(store.snapshot().unwrap().revision, original.revision); + assert_eq!(store.snapshot().unwrap().binding_digest, original.binding_digest); + state.write().await.privacy_mode = true; + let response = app.oneshot(request()).await.unwrap(); + let privacy: Value = serde_json::from_slice(&to_bytes(response.into_body(), 16_384).await.unwrap()).unwrap(); + assert_eq!(privacy["topology"]["status"],"privacy_denied"); assert!(privacy["topology"]["room"].is_null()); + assert_eq!(privacy["seq"].as_u64().unwrap(), first["seq"].as_u64().unwrap()+1); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/tests/fixtures/managed-worldgraph-v1.json b/v2/crates/wifi-densepose-sensing-server/tests/fixtures/managed-worldgraph-v1.json new file mode 100644 index 0000000000..a5e4fc87b7 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/tests/fixtures/managed-worldgraph-v1.json @@ -0,0 +1,51 @@ +{ + "protocol_version": 2, + "stream_epoch": "wg-1111111111111111111111111111111111111111111111111111111111111111", + "seq": 1, + "topology": { + "schema": "ruview.worldgraph.managed-topology.v1", + "coordinate_frame": "managed_room_local", + "axis_convention": "x_right_y_down_metres", + "provenance": "operator_entered", + "registration": null, + "installation_id": "installation-1", + "room_revision": 7, + "binding_digest": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "status": "configured", + "room": { + "kind": "room", + "id": 1, + "room_id": "room-1", + "name": "Configured room", + "width_m": 2.5, + "length_m": 3.0, + "height_m": 2.4 + }, + "nodes": [ + { + "kind": "sensor", + "id": 272, + "node_id": 15, + "x_m": 1.25, + "y_m": 2.0, + "height_m": null, + "role": "receiver", + "heading_deg": 150, + "slot": "north_west" + } + ], + "edges": [ + { + "from": 272, + "to": 1, + "rel": "located_in" + } + ], + "access_point": { + "slot": "north_east", + "x_m": 0, + "y_m": 0, + "provenance": "operator_entered" + } + } +} From 00d65f0782d649eecaa0c5cb55602c7fadb3d315 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Wed, 4 Nov 2026 10:48:42 +0100 Subject: [PATCH 060/100] fix(learning): preserve active RAM captures after recovery expiry --- .../src/room_learner/lifecycle_capture.rs | 2 + .../src/room_learner/lifecycle_recovery.rs | 63 ++++- .../room_learner/lifecycle_recovery_tests.rs | 226 ++++++++++++++++++ 3 files changed, 289 insertions(+), 2 deletions(-) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs index 4844f20868..67eec03d97 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs @@ -50,6 +50,7 @@ impl RoomLearningRuntime { now_ns: u64, ) -> Result<(), String> { let mut state = self.command_state(expected)?; + self.expire_loaded_recovery(&mut state, now_ns)?; self.require_recovery_ready(&state)?; if busy(&state.job) { return Err("training is active".into()); @@ -171,6 +172,7 @@ impl RoomLearningRuntime { conditions_held: bool, ) -> Result { let mut state = self.command_state(expected)?; + self.expire_loaded_recovery(&mut state, now_ns)?; let recovered_complete = state.recovery.disposition == Some( crate::room_learner::capture::RecoveryPhaseDisposition::CompletedNeedsConfirmation, diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs index 005e18675e..b4699a8105 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs @@ -17,6 +17,7 @@ pub(super) struct RecoveryState { pub(super) disposition: Option, last_now_ns: u64, last_attempt_ns: u64, + volatile_after_expiry: bool, } impl RecoveryState { pub(super) fn derived_persisted(&self) -> Option { @@ -82,10 +83,56 @@ impl RoomLearningRuntime { Ok(()) } } + /// Recovery availability is a disk deadline, not a lifetime for samples + /// already loaded into this active process. Authenticate the unchanged copy + /// before transitioning; other persistence failures never become RAM mode. + pub(super) fn expire_loaded_recovery( + &self, + state: &mut Inner, + now_ns: u64, + ) -> Result<(), String> { + state.recovery.last_now_ns = state.recovery.last_now_ns.max(now_ns); + let now_ns = state.recovery.last_now_ns; + if !state.recovery.enabled || state.capture.is_none() { + return Ok(()); + } + let Some(descriptor) = state.recovery.descriptor.as_ref() else { + return Ok(()); // Existing checkpoint validation reports missing state. + }; + if now_ns / 1_000_000 < descriptor.expires_at_ms { + return Ok(()); + } + if let Some(error) = &state.recovery.error { + return Err(error.clone()); + } + let verified = (|| -> Result<(), String> { + if state.capture.as_ref().map(|capture| capture.context().digest()) + .as_deref() != Some(descriptor.context_digest.as_str()) + { + return Err("recovery_context_mismatch".into()); + } + let retained = self.recovery_store.as_ref().ok_or("recovery_unavailable")? + .inspect()?.ok_or("recovery_missing")?; + if retained != *descriptor { + return Err("recovery_descriptor_changed".into()); + } + Ok(()) + })(); + if let Err(error) = verified { + state.recovery.error = Some(bounded(error.clone())); + return Err(error); + } + state.recovery.enabled = false; + state.recovery.marker = None; + state.recovery.volatile_after_expiry = true; + Ok(()) + } pub(super) fn checkpoint_now(&self, state: &Inner) -> u64 { self.receiver_now_ns().max(state.recovery.last_now_ns) } pub(super) fn inspect_recovery(&self, state: &mut Inner, now_ns: u64) { + state.recovery.last_now_ns = state.recovery.last_now_ns.max(now_ns); + let now_ns = state.recovery.last_now_ns; let Some(store) = &self.recovery_store else { return; }; @@ -168,6 +215,7 @@ impl RoomLearningRuntime { force: bool, ) -> Result<(), String> { state.recovery.last_now_ns = state.recovery.last_now_ns.max(now_ns); + self.expire_loaded_recovery(state, now_ns)?; if !state.recovery.enabled { return Ok(()); } @@ -238,7 +286,10 @@ impl RoomLearningRuntime { pub(super) fn recovery_status(&self, state: &Inner, now_ns: u64) -> Value { let supported = self.recovery_store.is_some() || state.recovery.supported; let available = self.recovery_store.as_ref().is_some_and(|s| s.available()); - let expired = state + // A request may have sampled receiver time before waiting for this + // mutex. Known expiry cannot be undone by that older observation. + let now_ns = now_ns.max(state.recovery.last_now_ns); + let expired = state.recovery.volatile_after_expiry || state .recovery .descriptor .as_ref() @@ -280,6 +331,10 @@ impl RoomLearningRuntime { "disabled" }; let error = state.recovery.error.clone().or_else(|| { + state.recovery.volatile_after_expiry.then(|| { + "Recovery expired. This loaded run continues in memory only; new progress is not saved. Closing the helper loses these samples. The expired encrypted copy remains until explicitly discarded.".into() + }) + }).or_else(|| { (supported && !available).then(|| { "recovery_unavailable: stable managed encryption key is unavailable".to_string() }) @@ -294,7 +349,9 @@ impl RoomLearningRuntime { recovery_id: &str, now_ns: u64, ) -> Result { - let _state = self.inner.lock().map_err(|_| "learner state unavailable")?; + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + state.recovery.last_now_ns = state.recovery.last_now_ns.max(now_ns); + let now_ns = state.recovery.last_now_ns; let loaded = self .recovery_store .as_ref() @@ -321,6 +378,8 @@ impl RoomLearningRuntime { now_ns: u64, ) -> Result<(), String> { let mut state = self.command_state(expected)?; + state.recovery.last_now_ns = state.recovery.last_now_ns.max(now_ns); + let now_ns = state.recovery.last_now_ns; if busy(&state.job) || state.capture.as_ref().is_some_and(|c| c.phase_active()) { return Err("finish or cancel the active operation first".into()); } diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_tests.rs index 2b3621a395..c57bd49639 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_tests.rs @@ -619,3 +619,229 @@ fn persisted_flags_confirmed_empty_discard_and_default_ram_are_false() { .unwrap(); assert_persisted_flags(&rt.status(BASE), json!(false)); } + +fn recovery_file(root: &Path) -> std::path::PathBuf { + root.join("a".repeat(64)).join("SensingServer/room-learner-recovery/capture-recovery-v1.bin") +} +fn expiry_request(role: DataRole, label: RoomLabel, index: u32) -> PhaseRequest { + PhaseRequest { + trial_id: format!("expiry-trial-{index}"), session_id: format!("expiry-session-{index}"), + role, label, duration_ms: 12_000, lead_in_ms: 0, conditions_confirmed: true, + } +} +fn expiry_phase(rt: &RoomLearningRuntime, at: u64, role: DataRole, label: RoomLabel, index: u32) { + rt.begin_phase(&rt.mutation_guard().unwrap(), expiry_request(role, label, index), at).unwrap(); + let gain = match label { RoomLabel::Empty => 2.0, RoomLabel::OccupiedStatic => 6.0, + RoomLabel::OccupiedMoving => 7.0, RoomLabel::AdjacentOnly => 2.3, RoomLabel::Unknown => panic!("label required") }; + for quarter in 0..4u32 { + for sample in 0..8u32 { + for source in 1..=2u8 { + let values: Vec<_> = (0..8).map(|bin| gain + f64::from(source)*0.3 + f64::from(bin)*0.04 + f64::from(sample)*0.003).collect(); + rt.ingest(RawSourceFrame {source_id: source, grid_id: "ht64-canonical8", + frequency_hz: 2_437_000_000, bandwidth_hz: 20_000_000, + sequence: index*32 + quarter*8 + sample + 1, amplitudes: &values}, + at + u64::from(quarter)*3_000_000_000 + 1_000_000 + u64::from(sample)*10_000_000 + u64::from(source)).unwrap(); + } + } + } + rt.finish_phase(&rt.mutation_guard().unwrap(), at + 12_000_000_000, true).unwrap(); +} +fn original_descriptor(root: &Path) -> crate::room_learner::recovery_store::RecoveryDescriptor { + let owner = "a".repeat(64); + RecoveryStore::new(&root.join(&owner).join("SensingServer"), &owner, "synthetic-install", Some(&[7;32])) + .unwrap().inspect().unwrap().unwrap() +} + +#[test] +fn loaded_recovery_expires_to_ram_at_exact_deadline_without_erasing_or_rebasing() { + let temp = tempfile::tempdir().unwrap(); + let rt = runtime(temp.path(), BASE, true); + rt.configure_for_recovery(&rt.mutation_guard().unwrap(), context(), "expiry-loaded".into(), true, true, BASE).unwrap(); + for index in 0..7 { expiry_phase(&rt, BASE + u64::from(index)*13_000_000_000, DataRole::Train, RoomLabel::Empty, index); } + let descriptor = original_descriptor(temp.path()); + drop(rt); + let recovered = runtime(temp.path(), BASE + 100_000_000_000, true); + recovered.restore_capture_recovery(&recovered.mutation_guard().unwrap(), &descriptor.id, &context(), BASE + 100_000_000_000).unwrap(); + let descriptor = original_descriptor(temp.path()); + let expires = descriptor.expires_at_ms*1_000_000; + let file = recovery_file(temp.path()); + let bytes = std::fs::read(&file).unwrap(); + let before = recovered.status(expires - 1); + assert_eq!(before["recovery"]["enabled"], true); + assert_eq!(before["capture"]["sealed_cohorts"], 28); + let (rows, history) = { + let state = recovered.inner.lock().unwrap();let capture = state.capture.as_ref().unwrap(); + (serde_json::to_vec(capture.cohorts()).unwrap(), serde_json::to_vec(capture.history()).unwrap()) + }; + let at = recovered.status(expires); + assert_eq!(at["recovery"]["state"], "expired");assert_eq!(at["recovery"]["enabled"], false); + assert!(at["recovery"]["saved"].is_null());assert_eq!(at["capture"]["sealed_cohorts"], 28); + assert_eq!(at["capture"]["sources"], before["capture"]["sources"]); + assert!(at["recovery"]["error"].as_str().unwrap().contains("memory only")); + assert!(recovered.inner.lock().unwrap().recovery.error.is_none(), "informational expiry must not become a persistence error"); + assert_eq!(original_descriptor(temp.path()), descriptor);assert_eq!(std::fs::read(&file).unwrap(), bytes); + let values = vec![6.0;8]; + assert!(recovered.ingest(RawSourceFrame{source_id:1,grid_id:"ht64-canonical8",frequency_hz:2_437_000_000, + bandwidth_hz:20_000_000,sequence:224,amplitudes:&values},expires+1).unwrap_err().contains("replayed")); + assert!(recovered.begin_phase(&recovered.mutation_guard().unwrap(), expiry_request(DataRole::Validation,RoomLabel::Empty,0), expires+2).is_err(), "frozen phase roles must survive expiry"); + expiry_phase(&recovered,expires+1_000_000,DataRole::Train,RoomLabel::Empty,7); + let after = recovered.status(expires+13_000_000_000); + assert_eq!(after["capture"]["sealed_cohorts"],32);assert!(after["capture"]["phase"].is_null()); + assert_eq!(after["capture"]["sources"][0]["last_accepted_sequence"],256); + { + let state=recovered.inner.lock().unwrap();let capture=state.capture.as_ref().unwrap(); + assert_eq!(serde_json::to_vec(&capture.cohorts()[..28]).unwrap(),rows); + assert_eq!(serde_json::to_vec(&capture.history()[..7]).unwrap(),history); + assert_eq!(capture.cohorts()[28].views[0].sequence_ids[0],225); + } + assert_eq!(original_descriptor(temp.path()),descriptor);assert_eq!(std::fs::read(&file).unwrap(),bytes); + drop(recovered); + let cold=runtime(temp.path(),expires+14_000_000_000,true); + assert!(cold.context().is_none()); + assert_eq!(cold.restore_capture_recovery(&cold.mutation_guard().unwrap(),&descriptor.id,&context(),expires+14_000_000_000).unwrap_err(),"recovery_expired"); + assert_eq!(std::fs::read(&file).unwrap(),bytes); +} + +#[test] +fn active_phase_can_cross_expiry_and_seal_without_a_false_failed_save() { + let temp=tempfile::tempdir().unwrap();let rt=runtime(temp.path(),BASE,true); + rt.configure_for_recovery(&rt.mutation_guard().unwrap(),context(),"crossing-phase".into(),true,true,BASE).unwrap(); + let descriptor=original_descriptor(temp.path());let expires=descriptor.expires_at_ms*1_000_000; + let start=expires-6_000_000_000; + rt.begin_phase(&rt.mutation_guard().unwrap(),request(12_000),start).unwrap(); + window(&rt,start+1_000_000,1);window(&rt,start+3_001_000_000,9); + let bytes=std::fs::read(recovery_file(temp.path())).unwrap();let final_saved=original_descriptor(temp.path()); + window(&rt,expires+1_000_000,17);window(&rt,expires+3_001_000_000,25); + assert_eq!(rt.status(expires+4_000_000_000)["capture"]["sealed_cohorts"],0,"expiry must not auto-confirm a phase"); + rt.finish_phase(&rt.mutation_guard().unwrap(),expires+6_000_000_000,true).unwrap(); + let status=rt.status(expires+7_000_000_000); + assert_eq!(status["capture"]["sealed_cohorts"],4);assert!(status["capture"]["phase"].is_null()); + assert_eq!(status["recovery"]["enabled"],false);assert_eq!(original_descriptor(temp.path()),final_saved); + assert_eq!(std::fs::read(recovery_file(temp.path())).unwrap(),bytes); +} + +#[test] +fn expiry_does_not_turn_io_tamper_missing_or_replacement_failures_into_ram_fallback() { + use std::os::unix::fs::PermissionsExt; + for case in ["latched_io", "tamper", "missing", "replacement"] { + let temp=tempfile::tempdir().unwrap();let rt=runtime(temp.path(),BASE,true); + rt.configure_for_recovery(&rt.mutation_guard().unwrap(),context(),format!("expiry-{case}"),true,true,BASE).unwrap(); + let original=original_descriptor(temp.path());let expires=original.expires_at_ms*1_000_000; + let file=recovery_file(temp.path()); + match case { + "latched_io"=>{ + std::fs::set_permissions(&file,std::fs::Permissions::from_mode(0o644)).unwrap(); + {let mut state=rt.inner.lock().unwrap();assert!(rt.checkpoint_recovery(&mut state,BASE+1_000_000,true).is_err());} + std::fs::set_permissions(&file,std::fs::Permissions::from_mode(0o600)).unwrap(); + }, + "tamper"=>{let mut bytes=std::fs::read(&file).unwrap();let last=bytes.len()-1;bytes[last]^=1;std::fs::write(&file,bytes).unwrap();}, + "missing"=>std::fs::remove_file(&file).unwrap(), + "replacement"=>{ + let owner="a".repeat(64);let store=RecoveryStore::new(&temp.path().join(&owner).join("SensingServer"),&owner,"synthetic-install",Some(&[7;32])).unwrap(); + store.discard(&original.id).unwrap();let mut other=original.clone();other.id="different-authenticated-copy".into();store.save(&other,b"SYNTHETIC REPLACEMENT",BASE/1_000_000).unwrap(); + },_=>unreachable!() + } + let bytes=std::fs::read(&file).ok(); + assert!(rt.begin_phase(&rt.mutation_guard().unwrap(),request(12_000),expires).is_err(),"{case}"); + let status=rt.status(expires+1); + assert!(status["capture"]["phase"].is_null(),"{case}: failed preflight must not start a phase"); + assert_eq!(status["recovery"]["enabled"],true,"{case}: generic error cannot disable recovery silently"); + assert!(!status["recovery"]["error"].as_str().unwrap().contains("memory only")); + assert_eq!(std::fs::read(&file).ok(),bytes,"{case}: expiry must not rewrite/delete a failed copy"); + } +} + +#[tokio::test] +async fn recovered_28_cohorts_can_finish_heldout_after_expiry_under_the_same_qualification_policy() { + let temp=tempfile::tempdir().unwrap();let rt=runtime(temp.path(),BASE,true); + rt.configure_for_recovery(&rt.mutation_guard().unwrap(),context(),"expiry-qualified".into(),true,true,BASE).unwrap(); + for (i,label) in [RoomLabel::Empty,RoomLabel::OccupiedStatic,RoomLabel::OccupiedMoving,RoomLabel::AdjacentOnly].into_iter().enumerate(){ + expiry_phase(&rt,BASE+i as u64*13_000_000_000,DataRole::Train,label,i as u32); + } + rt.start_training(&rt.mutation_guard().unwrap(),FitConfig{epochs:8,learning_rate:0.5,hidden_dim:2,embedding_dim:2,temporal_tokens:2,..FitConfig::default()}).unwrap(); + let wait_started=std::time::Instant::now(); + loop { + let ready=rt.status(BASE+51_000_000_000); + if ready["state"]=="ready" {break;} + assert!(ready["state"]=="running" && wait_started.elapsed() Date: Wed, 4 Nov 2026 16:30:37 +0100 Subject: [PATCH 061/100] feat(learner): retain managed training proof and sealed-phase replay fences --- ...R-420-managed-source-aware-room-learner.md | 96 ++ .../src/room_learner/artifact.rs | 6 + .../src/room_learner/capture.rs | 19 +- .../src/room_learner/capture_recovery.rs | 43 +- .../src/room_learner/evaluation_store.rs | 2 +- .../src/room_learner/lifecycle.rs | 25 +- .../src/room_learner/lifecycle_capture.rs | 19 +- .../src/room_learner/lifecycle_recovery.rs | 7 + .../src/room_learner/lifecycle_selection.rs | 24 +- .../src/room_learner/lifecycle_tests.rs | 3 +- .../src/room_learner/managed_capture.rs | 280 ++++++ .../managed_capture_fence_store.rs | 267 ++++++ .../room_learner/managed_capture_fences.rs | 114 +++ .../src/room_learner/managed_training.rs | 265 ++++++ .../managed_training_readiness.rs | 138 +++ .../room_learner/managed_training_store.rs | 78 ++ .../room_learner/managed_training_tests.rs | 898 ++++++++++++++++++ .../src/room_learner/mod.rs | 4 + .../src/room_learner/qualification.rs | 102 +- .../src/room_learner/selection.rs | 9 + .../src/room_learner/selection_store.rs | 14 + 21 files changed, 2365 insertions(+), 48 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_capture.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_capture_fence_store.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_capture_fences.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_readiness.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_store.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_tests.rs diff --git a/docs/adr/ADR-420-managed-source-aware-room-learner.md b/docs/adr/ADR-420-managed-source-aware-room-learner.md index 460533a3a2..c654d2676a 100644 --- a/docs/adr/ADR-420-managed-source-aware-room-learner.md +++ b/docs/adr/ADR-420-managed-source-aware-room-learner.md @@ -308,3 +308,99 @@ from source CI, packaged native verification and end-user validation. - [ ] Agree an adoption/rollback policy before enabling candidate authority. - [ ] Validate reference clock/coverage/registration and optional derived teachers. - [ ] Measure spatial, multi-source and reference benefit against frozen controls. + +## November 4 amendment: durable managed training and sealed-phase fences + +**Updated:** 2026-11-04. This extension is implemented in the source candidate; +the ADR remains Proposed. Software fixtures do not qualify a room, reference +sensor, delivered app or transmitter epoch. + +A v1 training manifest preserves the original Train SHA, total count and input +claims. It does not prove four managed, operator-sealed Train cohorts for every +condition. A stored v1 model without retained exact Train rows therefore remains +diagnostic: new HeldOut data alone cannot recover its missing training history. +Do not infer counts from the total, create acknowledgements from operator +recollections, or inspect expired encrypted samples to manufacture a proof. + +At successful managed fit, freeze a metadata-only seed while the actual native +Train rows and phase receipts are still available. Validate the exact original +Train SHA, four cohorts per required condition, unique sealed phase matches, +receiver-observed frame namespace, source ordinal/time ranges and original wire +counter watermarks. Bind this seed to the exact exported model SHA, complete +owner/installation/context contract, source/PHY contract, and domain-separated +trial/session role commitments. Store its immutable private proof and initial +sealed-phase fence, fsync and verify readback **before** publishing the candidate +pointer. Generic fitter/model-import persistence cannot issue this proof. +Retries reuse the first committed metadata, not later receive times. + +The proof contains no RF feature values, IQ, camera/depth frames or raw frame +lists. Metadata includes source counter/range bounds and hashed phase identities. +It uses the existing owner-private artifact directory, regular-file protections, +immutable hashes and durability primitives. This is integrity under the private +filesystem trust boundary, not a hardware signature or authenticated RF-source +attestation. A fully privileged local writer is outside that boundary. + +Fit-time fences alone are insufficient after later evaluations. Before a +post-fit phase becomes Sealed in RAM or its finish is acknowledged, commit a +bounded immutable metadata record and atomic verified ledger head. The record +extends the original model/proof identity, phase commitments and per-source +sequence/ordinal/receiver-time floors. Publish one newly sealed phase per link; +include no RF values. An I/O failure keeps the complete original phase retryable. +A retry can finish its exact pending publication but cannot replace its original +metadata with newer timestamps. A missing, torn, conflicting or unpublished +ledger blocks model resume/qualification; it cannot become an empty ledger. +This bounds previously sealed/labelled evidence, not every unused UDP packet. +Existing phase/cohort capacity limits also bound the ledger. + +Resume verifies the proof and complete ledger, restores the original capture +run/frame namespace, next phase index, frozen roles, original Train ranges and +maximum sealed source watermarks, and leaves all live/pending RF buffers empty. +It creates no dummy Train rows. Old sealed trial identities cannot be reused. +A current declared session may group multiple same-role conditions; a resumed +workflow needs a fresh group identity. Client IDs include the existing helper +runtime epoch and generation so a helper restart cannot collide with old groups. +Session declarations still do not establish statistical independence. + +Native ingress uses UTC-anchored monotonic receiver nanoseconds, not helper +uptime. The proof explicitly records that receiver clock domain. A lower new +helper wall anchor is incomparable and blocks capture; neither counter nor time +is silently rebased. Duplicate/reversed node counters remain rejected. Resetting +a node does not prove a new authenticated transmitter epoch. + +Qualification always recomputes the original Train SHA while rows remain. When +Train is empty, only a verified managed proof and a complete fresh disjoint +HeldOut set can provide that lineage basis. The per-condition minimums, coverage, +sensitivity, false-positive thresholds, OOD checks and explicit selection remain +unchanged. A failed latest assessment continues to withdraw positive authority; +restart cannot reuse its old labelled packets to clear the failure. + +Retained-row qualifications retain their v1 bytes/readers. Proof-backed receipts +use explicit `ruview.local-operational-qualification.v2`, bind the exact managed +training proof and sealed-phase fence SHA, and identify their lineage basis. +Their UTC nanoseconds are decimal strings, preserving precision beyond +JavaScript's safe integer range. Evaluation v1 still reports +`train_lineage_checked: false` at zero current Train rows. The bounded v2 public +summary carries only its version, lineage basis and opaque proof hash in addition +to the existing qualification summary; private phase identities/counters and +nanoseconds are not added to paired summaries. + +Owner status exposes a bounded qualification-readiness preflight before asking +for selection-oriented HeldOut captures: exact durable model, current room/source +binding, retained exact Train or verified managed proof, and comparable receiver +clock. Missing historical proof is a diagnostic limitation, not an invitation to +repeat unsaveable trials. Existing issued v1 qualification/selection readers are +preserved; no model is selected automatically and no general physical, +independence, count, pose or numeric Vitals authority is added. + +Managed lineage checkpoints use explicit capture-checkpoint v2 to retain these +metadata fences while v1 checkpoints remain readable. Optional reduced-sample +recovery keeps its original consent and expiry contract. Candidate weights and +bounded training/phase metadata survive that sample-recovery deadline in the +same owner-private model tree until explicitly erased under the model/account +lifecycle. They are not a retained sample set or a promise of automatic erasure. + +Verification covers synthetic packets through real managed fit, private writes, +cold resume, all-condition qualification, explicit selection, corrupted or pending +proof/ledger files, unchanged retained-row lineage, role/counter/time regression, +phase-ack ordering, and 2026-scale Rust-to-client wire fixtures. Physical +qualification and delivered-artifact acceptance remain separate work. diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs index a842f88b64..a1b1e69935 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs @@ -303,3 +303,9 @@ mod selection_store; #[path="assessment_store.rs"] mod assessment_store; + +#[path="managed_training_store.rs"] +mod managed_training_store; + +#[path="managed_capture_fence_store.rs"] +mod managed_capture_fence_store; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs index f7253d7b37..7362933299 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture.rs @@ -55,7 +55,7 @@ pub struct PhaseReceipt { #[derive(Debug, Clone)] struct Packet { values: Vec, frame_id: u128, ordinal: u64, received_ns: u64 } -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] struct SourceBuffer { sequence: Option, ordinal: u64, @@ -68,7 +68,7 @@ struct SourceBuffer { sequence_rejection_reason: Option, live: VecDeque, } -#[derive(Debug)] +#[derive(Debug,Clone)] struct Phase { request: PhaseRequest, started_ns: u64, @@ -127,7 +127,7 @@ pub struct SourceProgress { pub reason: Option, } -#[derive(Debug)] +#[derive(Debug,Clone)] pub struct CaptureSession { context: CaptureContext, run_id: String, @@ -139,6 +139,7 @@ pub struct CaptureSession { history: Vec, trials: HashMap, sessions: HashMap, + metadata_lineage: Option, } impl CaptureSession { @@ -152,7 +153,7 @@ impl CaptureSession { let prefix = u64::from_be_bytes(hash[..8].try_into().map_err(|_| "run digest")?); let sources = context.sources.iter().map(|s| (s.source_id, SourceBuffer::default())).collect(); Ok(Self { context, run_id, prefix, last_seen_ns: 0, sources, phase: None, cohorts: vec![], history: vec![], - trials: HashMap::new(), sessions: HashMap::new() }) + trials: HashMap::new(), sessions: HashMap::new(), metadata_lineage: None }) } /// A rejected selected input withdraws old live windows, not sealed training rows. @@ -207,14 +208,16 @@ impl CaptureSession { if !request.conditions_confirmed || request.role == DataRole::Inference || request.label == RoomLabel::Unknown || !(MIN_PHASE_MS..=MAX_PHASE_MS).contains(&request.duration_ms) || request.lead_in_ms > 120_000 || now_ns == 0 || now_ns < self.last_seen_ns || now_ns.checked_add((request.duration_ms + request.lead_in_ms) * 1_000_000).is_none() - || self.history.len() >= MAX_PHASE_HISTORY || self.cohorts.len() + COHORTS_PER_PHASE > MAX_COHORTS { + || self.phase_index() >= MAX_PHASE_HISTORY || self.sealed_cohort_capacity_used() + COHORTS_PER_PHASE > MAX_COHORTS { return Err("phase requires confirmed labelled conditions, bounded duration and capacity".into()); } + if let Some(lineage)=&self.metadata_lineage { lineage.check_role(&request.trial_id,&request.session_id,request.role)?; } for (roles, key) in [(&self.trials, &request.trial_id), (&self.sessions, &request.session_id)] { if roles.get(key).is_some_and(|role| *role != request.role) { return Err("trial/session role is frozen".into()); } } + if self.history.iter().any(|p|p.status==PhaseStatus::Sealed&&p.trial_id==request.trial_id) {return Err("phase identity was already sealed".into());} self.trials.insert(request.trial_id.clone(), request.role); self.sessions.insert(request.session_id.clone(), request.role); self.phase = Some(Phase { request, started_ns: now_ns, bucket: 0, pending: HashMap::new(), representatives: vec![], admitted: HashMap::new() }); @@ -259,6 +262,7 @@ impl CaptureSession { source.admitted_packets = source.admitted_packets.saturating_add(1); self.last_seen_ns = now_ns; push_bounded(&mut source.live, packet.clone()); + let phase_index=self.phase_index(); let Some(phase) = self.phase.as_mut() else { return Ok(()); }; if now_ns < phase.started_ns { return self.invalidate("receiver time precedes phase start"); } let capture_start = phase.started_ns + phase.request.lead_in_ms * 1_000_000; @@ -274,7 +278,7 @@ impl CaptureSession { push_bounded(phase.pending.entry(frame.source_id).or_default(), packet); if phase.pending.len() != self.context.sources.len() || phase.pending.values().any(|v| v.len() != WINDOW_PACKETS) { return Ok(()); } - let id = format!("{}-phase-{}-bucket-{bucket}", self.run_id, self.history.len()); + let id = format!("{}-phase-{}-bucket-{bucket}", self.run_id, phase_index); // Disjoint streams remain incomplete until a complete aligned window // arrives. No pooling, padding, or fabricated source-clock alignment. if let Ok(cohort) = assemble(&self.context, &phase.pending, now_ns, id, &phase.request) { @@ -371,3 +375,6 @@ mod tests; #[path = "capture_recovery.rs"] mod recovery; pub use recovery::{CaptureCheckpoint, CaptureCheckpointSummary, CaptureRestore, RecoveryPhaseDisposition, RecoveryPhaseSummary}; + +#[path = "managed_capture.rs"] +mod managed_capture; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery.rs index 792ccc456c..5e8ae19b08 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/capture_recovery.rs @@ -14,6 +14,8 @@ pub struct CaptureCheckpoint { cohorts: Vec, history: Vec, phase: Option, + #[serde(default, skip_serializing_if="Option::is_none")] + metadata_lineage: Option, } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -114,7 +116,7 @@ impl CaptureCheckpoint { impl CaptureSession { pub fn checkpoint(&self, now_ns: u64) -> Result { let snapshot = CaptureCheckpoint { - schema: SCHEMA.into(), + schema: if self.metadata_lineage.is_some() {"ruview.room-learner.capture-checkpoint.v2"} else {SCHEMA}.into(), context: self.context.clone(), run_id: self.run_id.clone(), saved_at_ns: now_ns, @@ -136,6 +138,7 @@ impl CaptureSession { .collect(), cohorts: self.cohorts.clone(), history: self.history.clone(), + metadata_lineage: self.metadata_lineage.clone(), phase: self.phase.as_ref().map(|p| CheckpointPhase { request: p.request.clone(), started_ns: p.started_ns, @@ -158,6 +161,7 @@ impl CaptureSession { session.last_seen_ns = snapshot.last_seen_ns; session.cohorts = snapshot.cohorts; session.history = snapshot.history; + session.metadata_lineage = snapshot.metadata_lineage; for receipt in &session.history { session .trials @@ -245,19 +249,28 @@ impl CaptureSession { impl CaptureCheckpoint { fn validate(&self, expected: &CaptureContext, now_ns: u64) -> Result<(), String> { self.context.validate()?; - if self.schema != SCHEMA + if let Some(seed)=&self.metadata_lineage {seed.validate()?;} + let lineage_offset=self.metadata_lineage.as_ref().map_or(0,|s|s.phase_claims.len()); + let expected_schema=if self.metadata_lineage.is_some() {"ruview.room-learner.capture-checkpoint.v2"} else {SCHEMA}; + if self.schema != expected_schema || &self.context != expected || self.saved_at_ns == 0 || self.saved_at_ns > now_ns || self.last_seen_ns > self.saved_at_ns || self.sources.len() != self.context.sources.len() - || self.history.len() > MAX_PHASE_HISTORY - || (self.phase.is_some() && self.history.len() >= MAX_PHASE_HISTORY) - || self.cohorts.len() > MAX_COHORTS + || self.history.len()+lineage_offset > MAX_PHASE_HISTORY + || (self.phase.is_some() && self.history.len()+lineage_offset >= MAX_PHASE_HISTORY) + || self.cohorts.len()+self.metadata_lineage.as_ref().map_or(0,|s|s.phase_claims.iter().map(|p|p.cohort_count).sum::()) > MAX_COHORTS { return Err("invalid recovery context, time or capacity".into()); } let session = CaptureSession::new(self.context.clone(), self.run_id.clone())?; + if let Some(seed)=&self.metadata_lineage { + seed.validate()?; + if seed.context!=self.context || seed.run_id!=self.run_id || seed.last_seen_ns>self.last_seen_ns { + return Err("recovery managed lineage context/namespace mismatch".into()); + } + } let mut latest = 0; for (w, spec) in self.sources.iter().zip(&self.context.sources) { if w.source_id != spec.source_id @@ -271,6 +284,13 @@ impl CaptureCheckpoint { { return Err("invalid source recovery watermark".into()); } + if let Some(seed)=&self.metadata_lineage { + let original=seed.sources.iter().find(|s|s.source_id==w.source_id).ok_or("missing original recovery source")?; + if w.ordinal Result<(), String> { identifier(trial)?; identifier(capture_session)?; + if let Some(seed)=&self.metadata_lineage {seed.check_role(trial,capture_session,role)?;} if role == DataRole::Inference || trials .insert(trial.clone(), role) @@ -326,7 +347,7 @@ impl CaptureCheckpoint { for c in &p.representatives { let bucket = (0..COHORTS_PER_PHASE) .find(|b| { - c.id == format!("{}-phase-{}-bucket-{b}", self.run_id, self.history.len()) + c.id == format!("{}-phase-{}-bucket-{b}", self.run_id, self.history.len()+lineage_offset) }) .ok_or("invalid provisional cohort identity")?; if previous_bucket.is_some_and(|b| b >= bucket) @@ -354,6 +375,7 @@ impl CaptureCheckpoint { .filter(|(_, r)| r.status == PhaseStatus::Sealed) { for bucket in 0..COHORTS_PER_PHASE { + let index=index+lineage_offset; let id = format!("{}-phase-{index}-bucket-{bucket}", self.run_id); let c = self .cohorts @@ -375,7 +397,8 @@ impl CaptureCheckpoint { } let mut frames = HashSet::new(); for c in &all { - if c.provenance != CaptureProvenance::LiveReceiverObserved || c.teacher.is_some() { + if c.provenance != CaptureProvenance::LiveReceiverObserved || c.teacher.is_some() + || (self.metadata_lineage.is_some() && c.role==DataRole::Train) { return Err("recovery requires original managed capture provenance".into()); } for v in &c.views { @@ -392,6 +415,12 @@ impl CaptureCheckpoint { { return Err("invalid reduced recovery source window".into()); } + if let Some(seed)=&self.metadata_lineage { + let old=seed.sources.iter().find(|s|s.source_id==v.source_id).ok_or("unknown original source")?; + if v.first_time_ns<=old.last_time_ns || v.sequence_ids.iter().any(|n|*n<=old.ordinal) { + return Err("recovered cohort overlaps original managed Train capture".into()); + } + } for (frame, ordinal) in v.frame_ids.iter().zip(&v.sequence_ids) { let expected = (u128::from(session.prefix) << 64) | (u128::from(v.source_id) << 56) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_store.rs index f8cda8e966..86983854b7 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_store.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/evaluation_store.rs @@ -33,7 +33,7 @@ fn role_directory(role: DataRole) -> Result<&'static str, String> { _ => Err("evaluation only accepts validation or held-out data".into()), } } -fn private_subdirectory( +pub(super) fn private_subdirectory( parent: &Path, name: &str, create: bool, diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs index 0e57c7e93c..c54191bac3 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs @@ -224,7 +224,7 @@ impl RoomLearningRuntime { return Err("learner context does not belong to this managed profile".into()); } FitConfig::for_source_count(context.sources.len()).validate_context(&context)?; - let session = CaptureSession::new(context.clone(), run_id)?; + let mut session = CaptureSession::new(context.clone(), run_id)?; let mut state = self.command_state(expected)?; if busy(&state.job) || state.capture.as_ref().is_some_and(|c| c.phase_active()) { return Err( @@ -237,6 +237,14 @@ impl RoomLearningRuntime { "no stored candidate for this exact context; start a new training capture".into(), ); } + if !new_training { + if let Some(model)=restored.as_ref() { + if let Some(proof)=self.store.load_managed_training_proof(model)? { + let fence=self.store.load_managed_capture_fence(model,&proof)?; + session=CaptureSession::resume_managed_fence(&proof,&fence)?; + } + } + } let (restored_evaluation, evaluation_warning) = if !new_training { restored .as_ref() @@ -295,7 +303,7 @@ impl RoomLearningRuntime { config: FitConfig, ) -> Result { config.validate()?; - let (generation, dataset, cancel) = { + let (generation, dataset, proof_seed, cancel) = { let mut state = self.command_state(expected)?; self.require_recovery_ready(&state)?; if busy(&state.job) @@ -323,6 +331,7 @@ impl RoomLearningRuntime { return Err("capture Train empty, occupied-static, occupied-moving and adjacent-only trials before training".into()); } } + let proof_seed = capture.managed_training_seed()?; let cancel = Arc::new(AtomicBool::new(false)); state.job = Job { phase: JobPhase::Running, @@ -337,7 +346,7 @@ impl RoomLearningRuntime { state.job.error = Some(bounded(error.clone())); return Err(error); } - (state.generation, dataset, cancel) + (state.generation, dataset, proof_seed, cancel) }; let runtime = self.clone(); tokio::task::spawn_blocking(move || { @@ -361,7 +370,7 @@ impl RoomLearningRuntime { return; } match outcome { - Ok((model, report)) => match runtime.store.save_candidate(&model) { + Ok((model, report)) => match runtime.store.save_managed_candidate(&model, proof_seed) { Ok(receipt) => { state.model = Some(Arc::new(model)); state.receipt = Some(receipt); @@ -442,6 +451,7 @@ impl RoomLearningRuntime { "training_frozen":state.frozen_training,"training_defaults":FitConfig::for_source_count(state.capture.as_ref().map_or(0, |c| c.context().sources.len())),"training_report":state.job.report, "error":state.job.error,"input_error":state.input_error, "candidate":state.receipt,"candidate_loaded":state.model.is_some(),"candidate_identity":self.candidate_identity(&state), + "qualification_readiness":self.qualification_readiness_locked(&state,now_ns), "model_selection":self.selection_status_locked(&state,now_ns,None), "last_evaluation":state.last_evaluation,"evaluation_warning":state.evaluation_warning,"physically_validated":false, "control_detector_changed":false,"raw_observations_persisted":state.recovery.derived_persisted(),"derived_samples_persisted":state.recovery.derived_persisted(),"raw_iq_persisted":false, @@ -487,3 +497,10 @@ mod selection_tests; #[cfg(test)] #[path="lifecycle_assessment_tests.rs"] mod assessment_tests; + +#[path="managed_training_readiness.rs"] +mod managed_training_readiness; + +#[cfg(test)] +#[path="managed_training_tests.rs"] +mod managed_training_tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs index 67eec03d97..ab1e563ccc 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs @@ -65,6 +65,12 @@ impl RoomLearningRuntime { "held-out capture starts only after a candidate is trained or restored".into(), ); } + if request.role==DataRole::HeldOut { + let readiness=self.qualification_readiness_locked(&state,now_ns); + if readiness["eligible_for_held_out"]!=true { + return Err(readiness["detail"].as_str().unwrap_or("managed training lineage unavailable before held-out capture").into()); + } + } state .capture .as_mut() @@ -191,7 +197,18 @@ impl RoomLearningRuntime { if recovered_complete { capture.live_cohort(now_ns)?; } - let receipt = capture.finish_phase(now_ns)?; + // A phase is not published as sealed until its metadata-only replay + // fence is durable. Failure leaves the complete original phase retryable. + let mut completed=capture.clone(); + let receipt=completed.finish_phase(now_ns)?; + if let Some(model)=state.model.as_ref() { + if let Some(proof)=self.store.load_managed_training_proof(model)? { + completed.checkpoint(now_ns)?; + let seed=completed.managed_fence_seed(&proof)?; + self.store.save_managed_capture_fence(model,&proof,seed)?; + } + } + state.capture=Some(completed); state.recovery.disposition = None; self.checkpoint_recovery(&mut state, now_ns, true)?; Ok(receipt) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs index b4699a8105..c1dc58c2c1 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs @@ -408,6 +408,13 @@ impl RoomLearningRuntime { .as_ref() .map(|sha| self.store.load_exact_candidate(context, sha)) .transpose()?; + if let Some(seed)=restored.session.metadata_lineage() { + let candidate=model.as_ref().ok_or("managed recovery has no exact candidate")?; + let proof=self.store.load_managed_training_proof(candidate)?.ok_or("managed recovery training proof missing")?; + seed.ensure_extends(&proof.seed)?; + let fence=self.store.load_managed_capture_fence(candidate,&proof)?; + restored.session.managed_fence_seed(&proof)?.ensure_extends(&fence.seed)?; + } state.generation = state .generation .checked_add(1) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection.rs index 88e52da081..61684ad246 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection.rs @@ -24,10 +24,10 @@ impl RoomLearningRuntime { if let Some(target) = &envelope.receipt.selected { let (model, q) = self.store.resolve_selection_target(target)?; let model = Arc::new(model); - state.capture = Some(CaptureSession::new( - target.context.clone(), - format!("selected-{}", envelope.receipt.revision), - )?); + state.capture = Some(if let Some(proof)=self.store.load_managed_training_proof(&model)? { + let fence=self.store.load_managed_capture_fence(&model,&proof)?; + CaptureSession::resume_managed_fence(&proof,&fence)? + } else {CaptureSession::new(target.context.clone(),format!("selected-{}",envelope.receipt.revision))?}); state.model = Some(model.clone()); state.frozen_training = true; state.job.phase = JobPhase::Ready; @@ -59,14 +59,24 @@ impl RoomLearningRuntime { .store .evaluate_candidate(&model, &dataset, DataRole::HeldOut)?; // Incomplete data or missing lineage is not a completed failed assessment. - let evidence = crate::room_learner::qualification::managed_evidence( + let proof=self.store.load_managed_training_proof(&model)?; + let fence=if let Some(proof)=&proof { + let fence=self.store.load_managed_capture_fence(&model,proof)?; + let current=capture.managed_fence_seed(proof)?; + current.ensure_extends(&fence.seed)?; + if current.phase_claims!=fence.seed.phase_claims {return Err("managed sealed phases are not durably published".into());} + Some(fence) + } else {None}; + let evidence = crate::room_learner::qualification::managed_evidence_with_fence( &model, &dataset, capture.history(), &evaluation, + proof.as_ref(), + fence.as_ref().map(|f|f.fence_sha256.as_str()), )?; let outcome = - QualificationEnvelope::issue(&model, &dataset, capture.history(), &evaluation); + QualificationEnvelope::issue_with_proof(&model, &dataset, capture.history(), &evaluation,proof.as_ref(),fence.as_ref().map(|f|f.fence_sha256.as_str())); let assessment = crate::room_learner::assessment::AssessmentEnvelope::issue( &model, &evaluation, @@ -76,7 +86,7 @@ impl RoomLearningRuntime { state.last_evaluation = Some(super::evaluation_control::presentation( &evaluation, false, - true, + !dataset.train().is_empty(), )?); state.evaluation_warning = None; if let Ok(q) = &outcome { diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs index fb67f28562..296dc34c61 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_tests.rs @@ -198,7 +198,8 @@ async fn real_capture_train_private_store_cold_load_and_unlabelled_inference_com grid_id: "ht64-canonical8", frequency_hz: 2_437_000_000, bandwidth_hz: 20_000_000, - sequence: 100 + row, + // Cold resume retains the original wire-sequence fence. + sequence: 1_000 + row, amplitudes: &values, }, START + 100_000_000_000 + u64::from(row) * 10_000_000 + u64::from(source), diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_capture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_capture.rs new file mode 100644 index 0000000000..4572cbed0a --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_capture.rs @@ -0,0 +1,280 @@ +//! Private native-session provenance; no imported rows or invented phase acknowledgements. +use super::*; +use crate::room_learner::{ + managed_training::{ + commitment, ManagedTrainingProof, PhaseClaim, SourceWatermark, TrainingProofSeed, + TrainingSourceRange, + }, + qualification::{hash, CONDITIONS}, +}; +impl CaptureSession { + pub(super) fn sealed_cohort_capacity_used(&self) -> usize { + self.metadata_lineage.as_ref().map_or(0, |s| { + s.phase_claims.iter().map(|p| p.cohort_count).sum::() + }) + self.cohorts.len() + } + pub(crate) fn minimum_receiver_time_ns(&self) -> u64 { + self.last_seen_ns + } + pub(crate) fn metadata_lineage(&self) -> Option<&TrainingProofSeed> { + self.metadata_lineage.as_ref() + } + pub(super) fn phase_index(&self) -> usize { + self.metadata_lineage + .as_ref() + .map_or(0, |s| s.phase_claims.len()) + + self.history.len() + } + pub(crate) fn managed_training_seed(&self) -> Result { + if self.phase_active() || self.metadata_lineage.is_some() { + return Err( + "managed training proof requires original completed native Train capture".into(), + ); + } + // Validate the actual capture's frame namespace, source counters, phase + // identity and sealed-row correspondence, not a caller's Live claim. + self.checkpoint(self.last_seen_ns)?; + let dataset = self.validated_dataset()?; + let rows = dataset.train(); + let counts = CONDITIONS.map(|label| rows.iter().filter(|r| r.label == label).count()); + if counts.iter().any(|n| *n < COHORTS_PER_PHASE) { + return Err("managed fit requires four sealed Train cohorts per condition".into()); + } + let mut phases = Vec::new(); + for row in rows { + let matches: Vec<_> = self + .history + .iter() + .filter(|p| { + p.role == DataRole::Train + && p.status == PhaseStatus::Sealed + && p.trial_id == row.trial_id + && p.session_id == row.session_id + && p.label == row.label + }) + .collect(); + if matches.len() != 1 + || rows + .iter() + .filter(|r| { + r.trial_id == row.trial_id + && r.session_id == row.session_id + && r.label == row.label + }) + .count() + != matches[0].cohort_count + { + return Err("Train rows lack unique managed sealed acknowledgement".into()); + } + if !phases.iter().any(|p: &&PhaseReceipt| { + p.trial_id == row.trial_id && p.session_id == row.session_id + }) { + phases.push(matches[0]); + } + } + // Hash identities, times and native metadata; RF sample values never + // enter the metadata artifact (the existing Train SHA remains exact). + let metadata: Vec<_> = rows + .iter() + .map(|r| { + ( + &r.id, + &r.trial_id, + &r.session_id, + r.role, + r.label, + r.views + .iter() + .map(|v| { + ( + v.source_id, + &v.frame_ids, + &v.sequence_ids, + v.first_time_ns, + v.last_time_ns, + v.provenance, + v.clock, + ) + }) + .collect::>(), + ) + }) + .collect(); + let seed = TrainingProofSeed { + context: self.context.clone(), + run_id: self.run_id.clone(), + training_data_sha256: hash(&rows)?, + training_metadata_sha256: hash(&metadata)?, + sealed_train_history_sha256: hash(&phases)?, + train_counts: counts, + phase_claims: self + .history + .iter() + .map(|p| { + Ok(PhaseClaim { + trial_sha256: commitment("trial", &p.trial_id)?, + session_sha256: commitment("session", &p.session_id)?, + role: p.role, + label: p.label, + status: p.status, + cohort_count: p.cohort_count, + }) + }) + .collect::, String>>()?, + sources: self + .context + .sources + .iter() + .map(|s| { + let w = &self.sources[&s.source_id]; + Ok(SourceWatermark { + source_id: s.source_id, + sequence: w.sequence.ok_or("missing original wire sequence")?, + ordinal: w.ordinal, + last_time_ns: w.last_time_ns, + admitted_packets: w.admitted_packets, + train_range: TrainingSourceRange { + first_ordinal: rows + .iter() + .flat_map(|c| &c.views) + .filter(|v| v.source_id == s.source_id) + .flat_map(|v| &v.sequence_ids) + .copied() + .min() + .ok_or("missing Train ordinal")?, + last_ordinal: rows + .iter() + .flat_map(|c| &c.views) + .filter(|v| v.source_id == s.source_id) + .flat_map(|v| &v.sequence_ids) + .copied() + .max() + .ok_or("missing Train ordinal")?, + first_time_ns: rows + .iter() + .flat_map(|c| &c.views) + .filter(|v| v.source_id == s.source_id) + .map(|v| v.first_time_ns) + .min() + .ok_or("missing Train time")?, + last_time_ns: rows + .iter() + .flat_map(|c| &c.views) + .filter(|v| v.source_id == s.source_id) + .map(|v| v.last_time_ns) + .max() + .ok_or("missing Train time")?, + }, + }) + }) + .collect::, String>>()?, + first_observed_ns: rows + .iter() + .flat_map(|c| &c.views) + .map(|v| v.first_time_ns) + .min() + .ok_or("no managed Train time")?, + last_seen_ns: self.last_seen_ns, + source_epoch_verified: false, + receiver_clock_domain: "utc_anchored_monotonic_v1".into(), + }; + seed.validate()?; + Ok(seed) + } + pub(crate) fn resume_managed_proof(proof: &ManagedTrainingProof) -> Result { + proof.seed.validate()?; + let seed = &proof.seed; + let mut session = Self::new(seed.context.clone(), seed.run_id.clone())?; + session.last_seen_ns = seed.last_seen_ns; + for w in &seed.sources { + session.sources.insert( + w.source_id, + SourceBuffer { + sequence: Some(w.sequence), + ordinal: w.ordinal, + last_time_ns: w.last_time_ns, + admitted_packets: w.admitted_packets, + ..SourceBuffer::default() + }, + ); + } + session.metadata_lineage = Some(seed.clone()); + Ok(session) + } +} +impl CaptureSession { + pub(crate) fn managed_fence_seed( + &self, + proof: &ManagedTrainingProof, + ) -> Result { + if self.context != proof.seed.context || self.run_id != proof.seed.run_id { + return Err("capture has a different managed namespace".into()); + } + let mut seed = proof.seed.clone(); + seed.phase_claims = self + .metadata_lineage + .as_ref() + .map_or_else(Vec::new, |m| m.phase_claims.clone()); + for p in &self.history { + seed.phase_claims.push(PhaseClaim { + trial_sha256: commitment("trial", &p.trial_id)?, + session_sha256: commitment("session", &p.session_id)?, + role: p.role, + label: p.label, + status: p.status, + cohort_count: p.cohort_count, + }); + } + seed.sources = self + .context + .sources + .iter() + .map(|s| { + let w = &self.sources[&s.source_id]; + Ok(SourceWatermark { + source_id: s.source_id, + sequence: w.sequence.ok_or("missing original source sequence")?, + ordinal: w.ordinal, + last_time_ns: w.last_time_ns, + admitted_packets: w.admitted_packets, + train_range: proof + .seed + .sources + .iter() + .find(|old| old.source_id == s.source_id) + .ok_or("unknown original Train range")? + .train_range + .clone(), + }) + }) + .collect::, String>>()?; + seed.last_seen_ns = self.last_seen_ns; + seed.ensure_extends(&proof.seed)?; + if let Some(original) = &self.metadata_lineage { + seed.ensure_extends(original)?; + } + Ok(seed) + } + pub(crate) fn resume_managed_fence( + proof: &ManagedTrainingProof, + fence: &crate::room_learner::managed_capture_fences::ManagedCaptureFence, + ) -> Result { + fence.validate(proof)?; + let mut session = Self::resume_managed_proof(proof)?; + session.last_seen_ns = fence.seed.last_seen_ns; + for w in &fence.seed.sources { + session.sources.insert( + w.source_id, + SourceBuffer { + sequence: Some(w.sequence), + ordinal: w.ordinal, + last_time_ns: w.last_time_ns, + admitted_packets: w.admitted_packets, + ..SourceBuffer::default() + }, + ); + } + session.metadata_lineage = Some(fence.seed.clone()); + Ok(session) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_capture_fence_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_capture_fence_store.rs new file mode 100644 index 0000000000..8c4a61a04f --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_capture_fence_store.rs @@ -0,0 +1,267 @@ +//! Private immutable phase-fence chain and atomically published verified head. +use super::*; +use crate::room_learner::{ + managed_capture_fences::ManagedCaptureFence, + managed_training::{ManagedTrainingProof, TrainingProofSeed}, +}; +const HEAD: &str = "managed-sealed-phase-head-v1.json"; +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct FenceHead { + schema: String, + model_sha256: String, + proof_sha256: String, + fence_sha256: String, + revision: u64, +} +impl ArtifactStore { + fn fence_dir(&self, model: &RoomModel, create: bool) -> Result, String> { + self.validate_context(model.context())?; + let Some(dir) = self.context_dir(model.context(), create)? else { + return Ok(None); + }; + evaluation_store::private_subdirectory( + &dir, + &format!("managed-fences-{}", digest(&model.export()?)), + create, + ) + } + fn read_fence_chain( + &self, + model: &RoomModel, + proof: &ManagedTrainingProof, + allow_pending: bool, + ) -> Result, String> { + proof.validate(model)?; + let dir = self + .fence_dir(model, false)? + .ok_or("managed sealed phase ledger missing")?; + let path = dir.join(HEAD); + if !regular_or_missing(&path)? { + return Err("managed sealed phase ledger head missing".into()); + } + let head: FenceHead = + serde_json::from_slice(&private_read(&path).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + if head.schema != "ruview.managed-sealed-phase-head.v1" + || head.model_sha256 != proof.model_sha256 + || head.proof_sha256 != proof.proof_sha256 + || head.revision >= 128 + || !hex_digest(&head.fence_sha256) + { + return Err("managed sealed phase ledger head mismatch".into()); + } + let mut result = Vec::new(); + let mut sha = head.fence_sha256; + let mut revision = head.revision; + loop { + if !hex_digest(&sha) { + return Err("invalid managed sealed phase ledger hash".into()); + } + let value: ManagedCaptureFence = serde_json::from_slice( + &private_read(&dir.join(format!("{sha}.json"))).map_err(|e| e.to_string())?, + ) + .map_err(|e| e.to_string())?; + value.validate(proof)?; + if value.fence_sha256 != sha || value.revision != revision { + return Err("managed sealed phase ledger chain changed".into()); + } + let previous = value.previous_sha256.clone(); + result.push(value); + if revision == 0 { + break; + } + sha = previous.ok_or("managed sealed phase ledger link missing")?; + revision -= 1; + } + result.reverse(); + for pair in result.windows(2) { + pair[1].seed.ensure_extends(&pair[0].seed)?; + let expected = ManagedCaptureFence::issue(proof, Some(&pair[0]), pair[1].seed.clone())?; + if expected != pair[1] { + return Err("managed phase ledger did not advance exact sealed history".into()); + } + } + let mut entries = 0; + for entry in std::fs::read_dir(&dir).map_err(|e| e.to_string())? { + entries += 1; + if entries > 130 { + return Err("managed sealed phase ledger exceeds capacity".into()); + } + let entry = entry.map_err(|e| e.to_string())?; + let name = entry + .file_name() + .into_string() + .map_err(|_| "invalid managed ledger path")?; + if name == HEAD { + continue; + } + let sha = name + .strip_suffix(".json") + .filter(|s| hex_digest(s)) + .ok_or("unknown pending managed phase artifact")?; + if !regular_or_missing(&entry.path())? { + return Err("managed ledger record disappeared".into()); + } + if !allow_pending && !result.iter().any(|f| f.fence_sha256 == sha) { + return Err("managed sealed phase publication is pending or conflicting".into()); + } + } + Ok(result) + } + pub(crate) fn load_managed_capture_fence( + &self, + model: &RoomModel, + proof: &ManagedTrainingProof, + ) -> Result { + self.read_fence_chain(model, proof, false)? + .pop() + .ok_or("managed capture fence unavailable".into()) + } + fn commit_fence( + &self, + model: &RoomModel, + proof: &ManagedTrainingProof, + fence: &ManagedCaptureFence, + ) -> Result<(), String> { + let dir = self + .fence_dir(model, true)? + .ok_or("managed fence directory unavailable")?; + evaluation_store::write_immutable( + &dir.join(format!("{}.json", fence.fence_sha256)), + &encode(fence)?, + )?; + let head = FenceHead { + schema: "ruview.managed-sealed-phase-head.v1".into(), + model_sha256: proof.model_sha256.clone(), + proof_sha256: proof.proof_sha256.clone(), + fence_sha256: fence.fence_sha256.clone(), + revision: fence.revision, + }; + let bytes = encode(&head)?; + atomic_private_write(&dir.join(HEAD), &bytes).map_err(|e| e.to_string())?; + if private_read(&dir.join(HEAD)).map_err(|e| e.to_string())? != bytes { + return Err("managed capture fence head readback mismatch".into()); + } + if self.load_managed_capture_fence(model, proof)? != *fence { + return Err("managed capture fence chain readback mismatch".into()); + } + Ok(()) + } + pub(crate) fn initialize_managed_capture_fence( + &self, + model: &RoomModel, + proof: &ManagedTrainingProof, + ) -> Result<(), String> { + if let Some(dir) = self.fence_dir(model, false)? { + if regular_or_missing(&dir.join(HEAD))? { + self.load_managed_capture_fence(model, proof)?; + return Ok(()); + } + // Retry only the exact initial inode before any model was installed. + // An existing candidate or another record means historical authority + // may have existed: never replace its missing head with an empty one. + let context = self + .context_dir(model.context(), false)? + .ok_or("managed context missing")?; + if regular_or_missing(&context.join(format!("{}.json", proof.model_sha256)))? { + return Err("existing candidate has an incomplete managed phase ledger".into()); + } + let initial = ManagedCaptureFence::issue(proof, None, proof.seed.clone())?; + let expected = format!("{}.json", initial.fence_sha256); + for entry in std::fs::read_dir(&dir).map_err(|e| e.to_string())? { + let entry = entry.map_err(|e| e.to_string())?; + if entry.file_name().to_str() != Some(expected.as_str()) + || private_read(&entry.path()).map_err(|e| e.to_string())? != encode(&initial)? + { + return Err("managed initial phase publication conflicts".into()); + } + } + } + let initial = ManagedCaptureFence::issue(proof, None, proof.seed.clone())?; + self.commit_fence(model, proof, &initial) + } + pub(crate) fn save_managed_capture_fence( + &self, + model: &RoomModel, + proof: &ManagedTrainingProof, + seed: TrainingProofSeed, + ) -> Result { + let chain = self.read_fence_chain(model, proof, true)?; + let previous = chain + .last() + .ok_or("managed capture fence unavailable")? + .clone(); + seed.ensure_extends(&previous.seed)?; + if seed.phase_claims == previous.seed.phase_claims { + // The same sealed phase may have committed before fsync reported + // failure. Retry its durability barrier, not its capture metadata. + self.load_managed_capture_fence(model, proof)?; + let dir = self + .fence_dir(model, false)? + .ok_or("managed fence directory unavailable")?; + std::fs::File::open(dir) + .and_then(|f| f.sync_all()) + .map_err(|e| e.to_string())?; + return Ok(previous); + } + let dir = self + .fence_dir(model, false)? + .ok_or("managed fence directory unavailable")?; + let mut pending = None; + for entry in std::fs::read_dir(&dir).map_err(|e| e.to_string())? { + let entry = entry.map_err(|e| e.to_string())?; + let name = entry + .file_name() + .into_string() + .map_err(|_| "invalid managed ledger path")?; + if name == HEAD { + continue; + } + let sha = name + .strip_suffix(".json") + .ok_or("invalid managed phase record")?; + if chain.iter().any(|f| f.fence_sha256 == sha) { + continue; + } + let record: ManagedCaptureFence = + serde_json::from_slice(&private_read(&entry.path()).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + record.validate(proof)?; + if record.fence_sha256 != sha + || pending.is_some() + || record.previous_sha256.as_deref() != Some(previous.fence_sha256.as_str()) + || record.revision != previous.revision + 1 + || record.seed.phase_claims != seed.phase_claims + { + return Err( + "pending managed phase publication conflicts with this exact phase".into(), + ); + } + seed.ensure_extends(&record.seed)?; + pending = Some(record); + } + let next = match pending { + Some(original) => original, + None => ManagedCaptureFence::issue(proof, Some(&previous), seed)?, + }; + self.commit_fence(model, proof, &next)?; + Ok(next) + } + pub(crate) fn require_managed_capture_fence( + &self, + model: &RoomModel, + proof: &ManagedTrainingProof, + sha: &str, + ) -> Result<(), String> { + if self + .read_fence_chain(model, proof, false)? + .iter() + .any(|f| f.fence_sha256 == sha) + { + Ok(()) + } else { + Err("qualification sealed phase fence is not retained".into()) + } + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_capture_fences.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_capture_fences.rs new file mode 100644 index 0000000000..a1147eb70e --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_capture_fences.rs @@ -0,0 +1,114 @@ +//! Bounded immutable metadata for sealed post-fit phases. No RF values. +use super::{ + managed_training::{ManagedTrainingProof, TrainingProofSeed}, + qualification::hash, +}; +use serde::{Deserialize, Serialize}; +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(crate) struct ManagedCaptureFence { + pub schema: String, + pub revision: u64, + pub previous_sha256: Option, + pub model_sha256: String, + pub managed_training_proof_sha256: String, + pub seed: TrainingProofSeed, + pub fence_sha256: String, +} +impl TrainingProofSeed { + pub(crate) fn ensure_extends(&self, original: &Self) -> Result<(), String> { + self.validate()?; + original.validate()?; + if self.context != original.context + || self.run_id != original.run_id + || self.training_data_sha256 != original.training_data_sha256 + || self.training_metadata_sha256 != original.training_metadata_sha256 + || self.sealed_train_history_sha256 != original.sealed_train_history_sha256 + || self.train_counts != original.train_counts + || self.first_observed_ns != original.first_observed_ns + || self.receiver_clock_domain != original.receiver_clock_domain + || self.phase_claims.len() < original.phase_claims.len() + || self.phase_claims[..original.phase_claims.len()] != original.phase_claims + || self.last_seen_ns < original.last_seen_ns + { + return Err( + "managed sealed capture lineage regressed or changed original training".into(), + ); + } + for (old, new) in original.sources.iter().zip(&self.sources) { + if new.source_id != old.source_id + || new.train_range != old.train_range + || new.ordinal < old.ordinal + || new.last_time_ns < old.last_time_ns + || new.admitted_packets < old.admitted_packets + || new.sequence.wrapping_sub(old.sequence) != (new.ordinal - old.ordinal) as u32 + { + return Err("managed sealed capture source counter/clock regressed".into()); + } + } + Ok(()) + } +} +impl ManagedCaptureFence { + pub(crate) fn issue( + proof: &ManagedTrainingProof, + previous: Option<&Self>, + seed: TrainingProofSeed, + ) -> Result { + seed.ensure_extends(&proof.seed)?; + if let Some(old) = previous { + old.validate(proof)?; + seed.ensure_extends(&old.seed)?; + let added = &seed.phase_claims[old.seed.phase_claims.len()..]; + if added + .iter() + .filter(|p| p.status == super::capture::PhaseStatus::Sealed) + .count() + != 1 + || added + .iter() + .any(|p| p.role == super::observation::DataRole::Train) + { + return Err( + "managed ledger requires exactly one newly sealed post-fit phase".into(), + ); + } + } + let mut value = Self { + schema: "ruview.managed-sealed-phase-fence.v1".into(), + revision: previous.map_or(0, |p| p.revision + 1), + previous_sha256: previous.map(|p| p.fence_sha256.clone()), + model_sha256: proof.model_sha256.clone(), + managed_training_proof_sha256: proof.proof_sha256.clone(), + seed, + fence_sha256: String::new(), + }; + value.fence_sha256 = value.digest()?; + value.validate(proof)?; + Ok(value) + } + fn digest(&self) -> Result { + hash(&( + &self.schema, + self.revision, + &self.previous_sha256, + &self.model_sha256, + &self.managed_training_proof_sha256, + &self.seed, + )) + } + pub(crate) fn validate(&self, proof: &ManagedTrainingProof) -> Result<(), String> { + self.seed.ensure_extends(&proof.seed)?; + if self.schema != "ruview.managed-sealed-phase-fence.v1" + || self.revision >= 128 + || self.model_sha256 != proof.model_sha256 + || self.managed_training_proof_sha256 != proof.proof_sha256 + || self.fence_sha256 != self.digest()? + || (self.revision == 0) != self.previous_sha256.is_none() + || (self.revision == 0 && self.seed != proof.seed) + { + return Err("managed sealed phase fence identity/integrity mismatch".into()); + } + Ok(()) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training.rs new file mode 100644 index 0000000000..0b94bfa293 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training.rs @@ -0,0 +1,265 @@ +//! Metadata-only provenance issued by the managed native capture path, not model imports. +use super::{ + capture::{PhaseStatus, COHORTS_PER_PHASE}, + model::RoomModel, + observation::{CaptureContext, CaptureProvenance, DataRole, RoomLabel, ValidatedDataset}, + qualification::{hash, CONDITIONS}, +}; +use serde::{Deserialize, Serialize}; + +pub(crate) const PROOF_SCHEMA: &str = "ruview.managed-training-proof.v1"; +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct TrainingSourceRange { + pub first_ordinal: u64, + pub last_ordinal: u64, + pub first_time_ns: u64, + pub last_time_ns: u64, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct SourceWatermark { + pub source_id: u8, + pub sequence: u32, + pub ordinal: u64, + pub last_time_ns: u64, + pub admitted_packets: u64, + pub train_range: TrainingSourceRange, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct PhaseClaim { + pub trial_sha256: String, + pub session_sha256: String, + pub role: DataRole, + pub label: RoomLabel, + pub status: PhaseStatus, + pub cohort_count: usize, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(crate) struct TrainingProofSeed { + pub context: CaptureContext, + pub run_id: String, + pub training_data_sha256: String, + pub training_metadata_sha256: String, + pub sealed_train_history_sha256: String, + pub train_counts: [usize; 4], + pub phase_claims: Vec, + pub sources: Vec, + pub first_observed_ns: u64, + pub last_seen_ns: u64, + /// Receiver observations never establish an authenticated transmitter epoch. + pub source_epoch_verified: bool, + pub receiver_clock_domain: String, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(crate) struct ManagedTrainingProof { + pub schema: String, + pub model_sha256: String, + pub seed: TrainingProofSeed, + pub proof_sha256: String, +} +pub(crate) fn commitment(domain: &str, value: &str) -> Result { + hash(&("ruview.managed-training-role.v1", domain, value)) +} +fn hex(s: &str) -> bool { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} +impl TrainingProofSeed { + pub fn validate(&self) -> Result<(), String> { + self.context.validate()?; + let _ = super::capture::CaptureSession::new(self.context.clone(), self.run_id.clone())?; + if !hex(&self.training_data_sha256) + || !hex(&self.training_metadata_sha256) + || !hex(&self.sealed_train_history_sha256) + || self.source_epoch_verified + || self.receiver_clock_domain != "utc_anchored_monotonic_v1" + || self + .train_counts + .iter() + .any(|n| *n < COHORTS_PER_PHASE || *n > 64) + || self.train_counts.iter().sum::() > 64 + || self.phase_claims.is_empty() + || self.phase_claims.len() > 128 + || self + .phase_claims + .iter() + .any(|p| p.cohort_count > COHORTS_PER_PHASE) + || self + .phase_claims + .iter() + .map(|p| p.cohort_count) + .sum::() + > super::observation::MAX_COHORTS + || self.sources.len() != self.context.sources.len() + || self.first_observed_ns == 0 + || self.last_seen_ns < self.first_observed_ns + { + return Err("invalid managed training metadata bounds".into()); + } + let mut trials = std::collections::HashMap::new(); + let mut sessions = std::collections::HashMap::new(); + for p in &self.phase_claims { + if !hex(&p.trial_sha256) + || !hex(&p.session_sha256) + || p.role == DataRole::Inference + || p.label == RoomLabel::Unknown + || p.cohort_count + != if p.status == PhaseStatus::Sealed { + COHORTS_PER_PHASE + } else { + 0 + } + || trials + .insert(&p.trial_sha256, p.role) + .is_some_and(|r| r != p.role) + || sessions + .insert(&p.session_sha256, p.role) + .is_some_and(|r| r != p.role) + { + return Err("managed training role/phase commitment mismatch".into()); + } + } + for (i, label) in CONDITIONS.iter().enumerate() { + let count: usize = self + .phase_claims + .iter() + .filter(|p| { + p.role == DataRole::Train + && p.label == *label + && p.status == PhaseStatus::Sealed + }) + .map(|p| p.cohort_count) + .sum(); + if count != self.train_counts[i] { + return Err("managed training condition counts lack sealed acknowledgement".into()); + } + } + for (s, w) in self.context.sources.iter().zip(&self.sources) { + if s.source_id != w.source_id + || w.ordinal == 0 + || w.ordinal >= 1u64 << 56 + || w.admitted_packets == 0 + || w.admitted_packets > w.ordinal + || w.last_time_ns == 0 + || w.last_time_ns > self.last_seen_ns + || w.train_range.first_ordinal == 0 + || w.train_range.last_ordinal < w.train_range.first_ordinal + || w.train_range.last_ordinal > w.ordinal + || w.train_range.first_time_ns == 0 + || w.train_range.last_time_ns <= w.train_range.first_time_ns + || w.train_range.last_time_ns > w.last_time_ns + { + return Err("invalid managed source watermark".into()); + } + } + if self.sources.iter().map(|s| s.last_time_ns).max() != Some(self.last_seen_ns) + || self + .sources + .iter() + .map(|s| s.train_range.first_time_ns) + .min() + != Some(self.first_observed_ns) + { + return Err("managed receiver watermark mismatch".into()); + } + Ok(()) + } + pub fn check_role(&self, trial: &str, session: &str, role: DataRole) -> Result<(), String> { + let trial = commitment("trial", trial)?; + let session = commitment("session", session)?; + if self.phase_claims.iter().any(|p| { + (p.trial_sha256 == trial || p.session_sha256 == session) + && (p.role != role || p.status == PhaseStatus::Sealed) + }) { + return Err("phase identity crosses original managed training roles".into()); + } + Ok(()) + } + pub fn prefix(&self) -> Result { + use sha2::{Digest, Sha256}; + let d = Sha256::digest(format!("{}:{}", self.context.digest(), self.run_id).as_bytes()); + Ok(u64::from_be_bytes( + d[..8].try_into().map_err(|_| "managed frame prefix")?, + )) + } +} +impl ManagedTrainingProof { + pub fn issue(model: &RoomModel, seed: TrainingProofSeed) -> Result { + let mut p = Self { + schema: PROOF_SCHEMA.into(), + model_sha256: String::new(), + seed, + proof_sha256: String::new(), + }; + // Model identity hashes its exact exported bytes, never a JSON Value roundtrip. + use sha2::{Digest, Sha256}; + p.model_sha256 = format!("{:x}", Sha256::digest(model.export()?)); + p.proof_sha256 = hash(&(&p.schema, &p.model_sha256, &p.seed))?; + p.validate(model)?; + Ok(p) + } + pub fn validate(&self, model: &RoomModel) -> Result<(), String> { + use sha2::{Digest, Sha256}; + self.seed.validate()?; + if self.schema != PROOF_SCHEMA + || self.proof_sha256 != hash(&(&self.schema, &self.model_sha256, &self.seed))? + || self.model_sha256 != format!("{:x}", Sha256::digest(model.export()?)) + || &self.seed.context != model.context() + || self.seed.training_data_sha256 != model.training_manifest().training_data_sha256 + || self.seed.train_counts.iter().sum::() + != model.training_manifest().training_cohorts + || model.training_manifest().training_provenance + != [CaptureProvenance::LiveReceiverObserved] + { + return Err("managed training proof model/context/Train identity mismatch".into()); + } + Ok(()) + } + pub fn check_held_out( + &self, + model: &RoomModel, + dataset: &ValidatedDataset, + ) -> Result<(), String> { + self.validate(model)?; + if dataset.context() != model.context() { + return Err("managed proof evaluation context mismatch".into()); + } + let prefix = self.seed.prefix()?; + for c in dataset.held_out() { + self.seed.check_role(&c.trial_id, &c.session_id, c.role)?; + if c.provenance != CaptureProvenance::LiveReceiverObserved || c.teacher.is_some() { + return Err("managed proof requires fresh native held-out capture".into()); + } + for v in &c.views { + let w = self + .seed + .sources + .iter() + .find(|w| w.source_id == v.source_id) + .ok_or("unknown managed held-out source")?; + if v.provenance != CaptureProvenance::LiveReceiverObserved + || v.first_time_ns <= w.last_time_ns + || v.sequence_ids + .iter() + .any(|n| *n <= w.ordinal || *n >= 1u64 << 56) + || v.frame_ids.iter().zip(&v.sequence_ids).any(|(f, n)| { + *f != ((u128::from(prefix) << 64) + | (u128::from(v.source_id) << 56) + | u128::from(*n)) + }) + { + return Err( + "held-out capture overlaps or replays original managed source lineage" + .into(), + ); + } + } + } + Ok(()) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_readiness.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_readiness.rs new file mode 100644 index 0000000000..d489b209db --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_readiness.rs @@ -0,0 +1,138 @@ +//! Qualification capture preflight, independent from a model's diagnostic availability. +use super::*; +use crate::room_learner::{ + capture::PhaseStatus, + qualification::{hash, CONDITIONS}, +}; +impl RoomLearningRuntime { + pub(super) fn qualification_readiness_locked(&self, state: &Inner, now_ns: u64) -> Value { + let retained = state + .capture + .as_ref() + .is_some_and(|c| c.cohorts().iter().any(|r| r.role == DataRole::Train)); + let result = (|| -> Result<(&str, [usize; 4], Option), (&str, String)> { + let model = state.model.as_ref().ok_or(( + "missing_candidate", + "Train or restore a candidate first.".into(), + ))?; + use sha2::{Digest, Sha256}; + let model_bytes = model.export().map_err(|e| ("missing_candidate", e))?; + self.store + .load_exact_candidate( + model.context(), + &format!("{:x}", Sha256::digest(&model_bytes)), + ) + .map_err(|e| { + ( + "missing_candidate", + format!("Frozen candidate artifact could not be verified: {e}"), + ) + })?; + let capture = state.capture.as_ref().ok_or(( + "context_mismatch", + "No current managed capture context.".into(), + ))?; + if capture.context() != model.context() { + return Err(( + "context_mismatch", + "Candidate does not match this room/source context.".into(), + )); + } + if now_ns < capture.minimum_receiver_time_ns() { + return Err(("receiver_clock_incomparable","The current receiver clock predates the original managed capture; no automatic clock/counter rebase is permitted.".into())); + } + let train: Vec<_> = capture + .cohorts() + .iter() + .filter(|c| c.role == DataRole::Train) + .collect(); + if !train.is_empty() { + capture + .validated_dataset() + .map_err(|e| ("invalid_managed_training_proof", e))?; + if hash(&train).map_err(|e| ("retained_train_mismatch", e))? + != model.training_manifest().training_data_sha256 + { + return Err(( + "retained_train_mismatch", + "Retained Train rows differ from this frozen model.".into(), + )); + } + let counts = + CONDITIONS.map(|label| train.iter().filter(|c| c.label == label).count()); + if counts.iter().any(|n| *n < 4) + || train.iter().any(|row| { + let phases: Vec<_> = capture + .history() + .iter() + .filter(|p| { + p.role == DataRole::Train + && p.status == PhaseStatus::Sealed + && p.trial_id == row.trial_id + && p.session_id == row.session_id + && p.label == row.label + }) + .collect(); + phases.len() != 1 + || phases[0].cohort_count + != train + .iter() + .filter(|r| { + r.trial_id == row.trial_id + && r.session_id == row.session_id + && r.label == row.label + }) + .count() + }) + { + return Err(( + "insufficient_managed_train", + "Four original managed sealed Train cohorts per condition are required." + .into(), + )); + } + if let Some(proof) = self + .store + .load_managed_training_proof(model) + .map_err(|e| ("invalid_managed_training_proof", e))? + { + let fence = self + .store + .load_managed_capture_fence(model, &proof) + .map_err(|e| ("invalid_managed_training_proof", e))?; + capture + .managed_fence_seed(&proof) + .and_then(|s| s.ensure_extends(&fence.seed)) + .map_err(|e| ("invalid_managed_training_proof", e))?; + } + return Ok(("retained_exact_train", counts, None)); + } + let proof=self.store.load_managed_training_proof(model).map_err(|e|("invalid_managed_training_proof",e))? + .ok_or(("missing_managed_training_proof","This diagnostic model has no retained Train rows or verified managed training proof; new held-out captures cannot qualify it.".into()))?; + let fence = self + .store + .load_managed_capture_fence(model, &proof) + .map_err(|e| ("invalid_managed_training_proof", e))?; + capture + .managed_fence_seed(&proof) + .and_then(|s| s.ensure_extends(&fence.seed)) + .map_err(|e| ("invalid_managed_training_proof", e))?; + proof + .validate(model) + .map_err(|e| ("invalid_managed_training_proof", e))?; + Ok(( + "managed_training_proof_v1", + proof.seed.train_counts, + Some(proof.proof_sha256), + )) + })(); + match result { + Ok((basis, counts, sha)) => { + json!({"schema":"ruview.qualification-readiness.v1","eligible_for_held_out":true,"training_lineage_basis":basis,"managed_training_proof_sha256":sha,"raw_train_rows_retained":retained,"train_cohorts_per_condition":counts,"reason":null,"detail":null}) + } + Err((reason, detail)) => { + json!({"schema":"ruview.qualification-readiness.v1","eligible_for_held_out":false,"training_lineage_basis":"unavailable","managed_training_proof_sha256":null,"raw_train_rows_retained":retained,"train_cohorts_per_condition":[0,0,0,0],"reason":reason,"detail":bounded(detail)}) + } + } + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_store.rs new file mode 100644 index 0000000000..b973a6b206 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_store.rs @@ -0,0 +1,78 @@ +//! Immutable managed provenance is committed before the candidate pointer. +use super::*; +use crate::room_learner::managed_training::{ManagedTrainingProof, TrainingProofSeed}; +impl ArtifactStore { + pub(crate) fn load_managed_training_proof( + &self, + model: &RoomModel, + ) -> Result, String> { + self.validate_context(model.context())?; + let Some(dir) = self.context_dir(model.context(), false)? else { + return Ok(None); + }; + let sha = digest(&model.export()?); + let path = dir.join(format!("managed-training-{sha}.json")); + if !regular_or_missing(&path)? { + return Ok(None); + } + let proof: ManagedTrainingProof = + serde_json::from_slice(&private_read(&path).map_err(|e| e.to_string())?) + .map_err(|e| e.to_string())?; + proof.validate(model)?; + Ok(Some(proof)) + } + pub(crate) fn save_managed_candidate( + &self, + model: &RoomModel, + seed: TrainingProofSeed, + ) -> Result { + self.validate_context(model.context())?; + seed.validate()?; + let proposed = ManagedTrainingProof::issue(model, seed)?; + let proof = if let Some(original) = self.load_managed_training_proof(model)? { + // Retries use the first committed proof, never re-attest its original + // source watermark with observations received during the fit/retry. + if original.seed.context != proposed.seed.context + || original.seed.run_id != proposed.seed.run_id + || original.seed.training_data_sha256 != proposed.seed.training_data_sha256 + || original.seed.training_metadata_sha256 != proposed.seed.training_metadata_sha256 + || original.seed.sealed_train_history_sha256 + != proposed.seed.sealed_train_history_sha256 + || original.seed.train_counts != proposed.seed.train_counts + || original.seed.phase_claims != proposed.seed.phase_claims + || original + .seed + .sources + .iter() + .zip(&proposed.seed.sources) + .any(|(a, b)| { + a.source_id != b.source_id + || a.ordinal > b.ordinal + || a.last_time_ns > b.last_time_ns + }) + { + return Err( + "existing managed training proof differs from the original fit lineage".into(), + ); + } + proposed.seed.ensure_extends(&original.seed)?; + original + } else { + proposed + }; + let dir = self + .context_dir(model.context(), true)? + .ok_or("missing managed model context")?; + let path = dir.join(format!("managed-training-{}.json", proof.model_sha256)); + let bytes = encode(&proof)?; + evaluation_store::write_immutable(&path, &bytes)?; + if private_read(&path).map_err(|e| e.to_string())? != bytes { + return Err("managed training proof readback mismatch".into()); + } + self.load_managed_training_proof(model)? + .ok_or("managed training proof was not retained")?; + self.initialize_managed_capture_fence(model, &proof)?; + // Generic save_candidate remains diagnostic-only for all import callers. + self.save_candidate(model) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_tests.rs new file mode 100644 index 0000000000..1dedc764b8 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_tests.rs @@ -0,0 +1,898 @@ +//! SYNTHETIC native packet fixtures; no hardware or general accuracy claim. +use super::tests::{context, phase, runtime, START}; +use super::*; +use crate::room_learner::{ + capture::CaptureSession, + managed_training::ManagedTrainingProof, + observation::ValidatedDataset, + qualification::{hash, managed_evidence_with_proof}, +}; +use std::path::PathBuf; +fn proof_path(root: &Path, model: &RoomModel) -> PathBuf { + use sha2::{Digest, Sha256}; + root.join("a".repeat(64)) + .join("SensingServer/room-learner") + .join(model.context().digest()) + .join(format!( + "managed-training-{:x}.json", + Sha256::digest(model.export().unwrap()) + )) +} +fn current(rt: &RoomLearningRuntime) -> (Arc, ManagedTrainingProof) { + let state = rt.inner.lock().unwrap(); + let model = state.model.as_ref().unwrap().clone(); + let proof = rt + .store + .load_managed_training_proof(&model) + .unwrap() + .unwrap(); + (model, proof) +} +fn restore(root: &Path) -> Arc { + let cold = runtime(root); + cold.configure( + &cold.mutation_guard().unwrap(), + context(), + "caller-new-run-must-not-replace-original".into(), + false, + ) + .unwrap(); + cold +} +#[tokio::test] +async fn metadata_proof_cold_resume_fresh_all_conditions_qualify_v2_without_raw_train() { + let dir = tempfile::tempdir().unwrap(); + let rt = super::selection_tests::fitted(dir.path()).await; + let (model, proof) = current(&rt); + let before = std::fs::read(proof_path(dir.path(), &model)).unwrap(); + let text = String::from_utf8(before.clone()).unwrap(); + for forbidden in [ + "\"samples\"", + "\"frame_ids\"", + "\"values\"", + "\"amplitudes\"", + ] { + assert!(!text.contains(forbidden)); + } + let legacy = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + assert_eq!( + legacy.receipt.schema, + "ruview.local-operational-qualification.v1" + ); + assert!(!serde_json::to_value(&legacy.receipt) + .unwrap() + .as_object() + .unwrap() + .contains_key("training_lineage_basis")); + let cold = restore(dir.path()); + let ready = cold.status(START + 200_000_000_000)["qualification_readiness"].clone(); + assert_eq!(ready["eligible_for_held_out"], true, "{ready}"); + assert_eq!(ready["raw_train_rows_retained"], false); + assert_eq!(ready["training_lineage_basis"], "managed_training_proof_v1"); + assert_eq!(ready["train_cohorts_per_condition"], json!([4, 4, 4, 4])); + assert_eq!(ready["managed_training_proof_sha256"], proof.proof_sha256); + for (i, label) in crate::room_learner::qualification::CONDITIONS + .into_iter() + .enumerate() + { + phase(&cold, DataRole::HeldOut, label, i as u32 + 8); + } + let q = cold.qualify(&cold.mutation_guard().unwrap()).unwrap(); + assert_eq!( + q.receipt.schema, + "ruview.local-operational-qualification.v2" + ); + assert_eq!( + q.receipt.managed_training_proof_sha256.as_deref(), + Some(proof.proof_sha256.as_str()) + ); + assert_eq!( + q.receipt.training_lineage_basis.as_deref(), + Some("managed_training_proof_v1") + ); + assert!(!q.receipt.report.independence_verified && !q.receipt.physically_validated); + assert_eq!( + cold.status(START + 200_000_000_000)["last_evaluation"]["train_lineage_checked"], + false + ); + assert_eq!( + std::fs::read(proof_path(dir.path(), &model)).unwrap(), + before + ); + let summary = crate::room_learner::selection::QualificationSummary::from(&q); + let public = serde_json::to_value(&summary).unwrap(); + assert_eq!( + public["schema"], + "ruview.local-operational-qualification.v2" + ); + assert!(public.get("first_observed_at_ns").is_none()); + cold.select_model( + &cold.mutation_guard().unwrap(), + &q.receipt.model_sha256, + &q.receipt_sha256, + ) + .unwrap(); + let selected = runtime(dir.path()); + let s = selected.status(START + 200_000_000_000); + assert_eq!( + s["model_selection"]["selected"]["qualification_sha256"], + q.receipt_sha256 + ); + assert_eq!(s["qualification_readiness"]["eligible_for_held_out"], true); +} +#[tokio::test] +async fn old_diagnostic_model_without_proof_loads_but_stops_selection_capture_before_effort() { + let dir = tempfile::tempdir().unwrap(); + let rt = super::selection_tests::fitted(dir.path()).await; + let (model, _) = current(&rt); + std::fs::remove_file(proof_path(dir.path(), &model)).unwrap(); + let cold = restore(dir.path()); + let status = cold.status(START + 200_000_000_000); + assert_eq!(status["candidate_loaded"], true); + assert_eq!( + status["qualification_readiness"]["reason"], + "missing_managed_training_proof" + ); + let result = cold.begin_phase( + &cold.mutation_guard().unwrap(), + PhaseRequest { + trial_id: "fresh-empty".into(), + session_id: "fresh-empty-session".into(), + role: DataRole::HeldOut, + label: RoomLabel::Empty, + duration_ms: 12000, + lead_in_ms: 0, + conditions_confirmed: true, + }, + START + 200_000_000_000, + ); + assert!(result.unwrap_err().contains("cannot qualify")); + assert!(!cold.capture_active().unwrap()); + assert!(cold.qualify(&cold.mutation_guard().unwrap()).is_err()); +} +#[tokio::test] +async fn proof_resume_preserves_counter_namespace_roles_and_rejects_regressed_receiver_clock() { + let dir = tempfile::tempdir().unwrap(); + let rt = super::selection_tests::fitted(dir.path()).await; + let (_, proof) = current(&rt); + let mut capture = CaptureSession::resume_managed_proof(&proof).unwrap(); + let w = &proof.seed.sources[0]; + let values = vec![2.0; 8]; + for sequence in [w.sequence, w.sequence.wrapping_sub(1), 0] { + assert!(capture + .ingest( + RawSourceFrame { + source_id: w.source_id, + grid_id: "ht64-canonical8", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence, + amplitudes: &values + }, + proof.seed.last_seen_ns + 1 + ) + .is_err()); + } + assert!(capture + .ingest( + RawSourceFrame { + source_id: w.source_id, + grid_id: "ht64-canonical8", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: w.sequence.wrapping_add(1), + amplitudes: &values + }, + proof.seed.last_seen_ns - 1 + ) + .is_err()); + capture + .ingest( + RawSourceFrame { + source_id: w.source_id, + grid_id: "ht64-canonical8", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: w.sequence.wrapping_add(1), + amplitudes: &values, + }, + proof.seed.last_seen_ns + 1, + ) + .unwrap(); + assert_eq!( + capture.progress(proof.seed.last_seen_ns + 2).sources[0].last_accepted_sequence, + Some(w.sequence + 1) + ); + let crossed = PhaseRequest { + trial_id: "synthetic-trial-0".into(), + session_id: "unused-session".into(), + role: DataRole::HeldOut, + label: RoomLabel::Empty, + duration_ms: 12000, + lead_in_ms: 0, + conditions_confirmed: true, + }; + assert!(capture + .begin_phase(crossed, proof.seed.last_seen_ns + 3) + .unwrap_err() + .contains("original managed training roles")); + let cold = restore(dir.path()); + assert_eq!( + cold.status(proof.seed.last_seen_ns - 1)["qualification_readiness"]["reason"], + "receiver_clock_incomparable" + ); +} +#[tokio::test] +async fn v2_capture_checkpoint_retains_metadata_fences_and_no_dummy_training_rows() { + let dir = tempfile::tempdir().unwrap(); + let rt = super::selection_tests::fitted(dir.path()).await; + let (_, proof) = current(&rt); + let capture = CaptureSession::resume_managed_proof(&proof).unwrap(); + let at = proof.seed.last_seen_ns + 10; + let snapshot = capture.checkpoint(at).unwrap(); + let json = serde_json::to_value(&snapshot).unwrap(); + assert_eq!(json["schema"], "ruview.room-learner.capture-checkpoint.v2"); + assert!(json["cohorts"].as_array().unwrap().is_empty()); + let restored = + CaptureSession::restore_checkpoint(snapshot, &proof.seed.context, at + 1).unwrap(); + assert_eq!(restored.session.metadata_lineage(), Some(&proof.seed)); + assert_eq!( + restored.session.progress(at + 2).sources[0].last_accepted_sequence, + Some(proof.seed.sources[0].sequence) + ); + let mut missing = json; + missing.as_object_mut().unwrap().remove("metadata_lineage"); + let snapshot = serde_json::from_value(missing).unwrap(); + assert!(CaptureSession::restore_checkpoint(snapshot, &proof.seed.context, at + 1).is_err()); +} +#[tokio::test] +async fn corrupt_foreign_and_torn_metadata_never_replace_exact_train_checks() { + let dir = tempfile::tempdir().unwrap(); + let rt = super::selection_tests::fitted(dir.path()).await; + let (model, proof) = current(&rt); + for mutate in [ + "owner", + "installation", + "context", + "model", + "train", + "source", + "count", + "epoch", + "clock", + "floor", + "ack", + "hash", + ] { + let mut p = proof.clone(); + match mutate { + "owner" => p.seed.context.owner_scope = "b".repeat(64), + "installation" => p.seed.context.installation_id = "foreign-installation".into(), + "context" => p.seed.context.room_digest = "foreign-room".into(), + "model" => p.model_sha256 = "b".repeat(64), + "train" => p.seed.training_data_sha256 = "b".repeat(64), + "source" => { + p.seed.context.sources[1].source_id = 99; + p.seed.sources[1].source_id = 99; + } + "count" => p.seed.train_counts[1] = 0, + "epoch" => p.seed.source_epoch_verified = true, + "clock" => p.seed.receiver_clock_domain = "unknown".into(), + "floor" => { + p.seed.sources[0].ordinal = p.seed.sources[0].train_range.last_ordinal - 1; + p.seed.sources[0].admitted_packets = p.seed.sources[0].ordinal; + } + "ack" => p.seed.phase_claims[0].cohort_count = 0, + _ => p.proof_sha256 = "b".repeat(64), + } + if mutate != "hash" { + p.proof_sha256 = hash(&(&p.schema, &p.model_sha256, &p.seed)).unwrap(); + } + assert!(p.validate(&model).is_err(), "{mutate}"); + } + let evaluation = rt + .store + .evaluate_candidate( + &model, + &rt.inner + .lock() + .unwrap() + .capture + .as_ref() + .unwrap() + .validated_dataset() + .unwrap(), + DataRole::HeldOut, + ) + .unwrap(); + let state = rt.inner.lock().unwrap(); + let capture = state.capture.as_ref().unwrap(); + let mut rows = capture.cohorts().to_vec(); + rows.iter_mut() + .find(|c| c.role == DataRole::Train) + .unwrap() + .views[0] + .samples[0][0] += 0.25; + let altered = ValidatedDataset::new(model.context().clone(), rows).unwrap(); + assert!(managed_evidence_with_proof( + &model, + &altered, + capture.history(), + &evaluation, + Some(&proof) + ) + .unwrap_err() + .contains("retained exact Train")); + drop(state); + crate::companion::atomic_private_write(&proof_path(dir.path(), &model), b"{").unwrap(); + assert!(rt.store.load_managed_training_proof(&model).is_err()); + let cold = runtime(dir.path()); + assert!(cold + .configure( + &cold.mutation_guard().unwrap(), + context(), + "torn".into(), + false + ) + .is_err()); +} +fn phase_at( + rt: &RoomLearningRuntime, + label: RoomLabel, + index: u32, + base: u64, + gain_override: Option, +) -> u64 { + phase_at_group(rt, label, index, base, gain_override, None) +} +fn phase_at_group( + rt: &RoomLearningRuntime, + label: RoomLabel, + index: u32, + base: u64, + gain_override: Option, + group: Option<&str>, +) -> u64 { + let end = prepare_phase_at(rt, label, index, base, gain_override, group); + rt.finish_phase(&rt.mutation_guard().unwrap(), end, true) + .unwrap(); + end +} +fn prepare_phase_at( + rt: &RoomLearningRuntime, + label: RoomLabel, + index: u32, + base: u64, + gain_override: Option, + group: Option<&str>, +) -> u64 { + let at = base + u64::from(index) * 13_000_000_000; + let role = if index < 4 { + DataRole::Train + } else { + DataRole::HeldOut + }; + rt.begin_phase( + &rt.mutation_guard().unwrap(), + PhaseRequest { + trial_id: format!("epoch-trial-{index}"), + session_id: group + .map(str::to_owned) + .unwrap_or_else(|| format!("epoch-session-{index}")), + role, + label, + duration_ms: 12000, + lead_in_ms: 0, + conditions_confirmed: true, + }, + at, + ) + .unwrap(); + let gain = gain_override.unwrap_or(match label { + RoomLabel::Empty => 2.0, + RoomLabel::OccupiedStatic => 6.0, + RoomLabel::OccupiedMoving => 7.0, + RoomLabel::AdjacentOnly => 2.3, + RoomLabel::Unknown => panic!("unlabelled"), + }); + let context = rt.context().unwrap(); + for quarter in 0..4u32 { + for sample in 0..8u32 { + for source in &context.sources { + let values: Vec<_> = (0..source.feature_count) + .map(|bin| { + gain + f64::from(source.source_id) * 0.3 + + bin as f64 * 0.04 + + f64::from(sample) * 0.003 + }) + .collect(); + rt.ingest( + RawSourceFrame { + source_id: source.source_id, + grid_id: &source.grid_id, + frequency_hz: source.center_frequency_hz, + bandwidth_hz: source.bandwidth_hz, + sequence: index * 32 + quarter * 8 + sample + 1, + amplitudes: &values, + }, + at + u64::from(quarter) * 3_000_000_000 + + 1_000_000 + + u64::from(sample) * 10_000_000 + + u64::from(source.source_id), + ) + .unwrap(); + } + } + } + at + 12_000_000_000 +} +#[tokio::test] +async fn failed_assessment_cold_resume_keeps_all_sealed_phase_fences_not_only_train() { + let dir = tempfile::tempdir().unwrap(); + let rt = super::selection_tests::fitted(dir.path()).await; + let (model, proof) = current(&rt); + rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + // Deliberately wrong synthetic Empty observations produce a failed policy. + phase_at(&rt, RoomLabel::Empty, 8, START, Some(7.0)); + assert!(rt.qualify(&rt.mutation_guard().unwrap()).is_err()); + let fence = rt.store.load_managed_capture_fence(&model, &proof).unwrap(); + assert_eq!(fence.revision, 5); + let cold = restore(dir.path()); + let status = cold.status(START + 200_000_000_000); + assert!(status["model_selection"]["qualification_error"] + .as_str() + .is_some()); + let state = cold.inner.lock().unwrap(); + let capture = state.capture.as_ref().unwrap(); + assert_eq!(capture.metadata_lineage(), Some(&fence.seed)); + let last = &fence.seed.sources[0]; + drop(state); + let values = vec![2.0; 8]; + assert!(cold + .ingest( + RawSourceFrame { + source_id: last.source_id, + grid_id: "ht64-canonical8", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: last.sequence, + amplitudes: &values + }, + START + 200_000_000_000 + ) + .is_err()); + assert!(cold + .begin_phase( + &cold.mutation_guard().unwrap(), + PhaseRequest { + trial_id: "epoch-trial-8".into(), + session_id: "new-session".into(), + role: DataRole::HeldOut, + label: RoomLabel::Empty, + duration_ms: 12000, + lead_in_ms: 0, + conditions_confirmed: true + }, + START + 200_000_000_000 + ) + .is_err()); + // New real conditions require new IDs AND advancing original source counters. + phase_at(&cold, RoomLabel::Empty, 9, START, None); + let next = cold + .store + .load_managed_capture_fence(&model, &proof) + .unwrap(); + assert_eq!(next.revision, 6); + assert!(next.seed.sources[0].ordinal > last.ordinal); +} +#[tokio::test] +async fn missing_torn_and_unpublished_sealed_ledger_never_reset_to_train_floor() { + let dir = tempfile::tempdir().unwrap(); + let rt = super::selection_tests::fitted(dir.path()).await; + let (model, proof) = current(&rt); + let directory = proof_path(dir.path(), &model) + .parent() + .unwrap() + .join(format!("managed-fences-{}", proof.model_sha256)); + let head = directory.join("managed-sealed-phase-head-v1.json"); + let retained = std::fs::read(&head).unwrap(); + std::fs::remove_file(&head).unwrap(); + assert!(rt.store.load_managed_capture_fence(&model, &proof).is_err()); + let cold = runtime(dir.path()); + assert!(cold + .configure( + &cold.mutation_guard().unwrap(), + context(), + "missing-ledger".into(), + false + ) + .is_err()); + crate::companion::atomic_private_write(&head, b"{").unwrap(); + assert!(rt.store.load_managed_capture_fence(&model, &proof).is_err()); + crate::companion::atomic_private_write(&head, &retained).unwrap(); + // A pending immutable child without published head blocks model resume. + let original = rt.store.load_managed_capture_fence(&model, &proof).unwrap(); + let mut next_seed = original.seed.clone(); + let mut claim = next_seed.phase_claims.last().unwrap().clone(); + claim.trial_sha256 = "c".repeat(64); + claim.session_sha256 = "d".repeat(64); + next_seed.phase_claims.push(claim); + let pending = crate::room_learner::managed_capture_fences::ManagedCaptureFence::issue( + &proof, + Some(&original), + next_seed.clone(), + ) + .unwrap(); + let bytes = serde_json::to_vec(&pending).unwrap(); + crate::companion::atomic_private_write( + &directory.join(format!("{}.json", pending.fence_sha256)), + &bytes, + ) + .unwrap(); + assert!(rt + .store + .load_managed_capture_fence(&model, &proof) + .unwrap_err() + .contains("pending")); + // The exact finishing caller can complete this pending publication, keeping + // its original identity instead of inventing a newer observation timestamp. + let saved = rt + .store + .save_managed_capture_fence(&model, &proof, next_seed) + .unwrap(); + assert_eq!(saved, pending); + assert_eq!( + rt.store.load_managed_capture_fence(&model, &proof).unwrap(), + pending + ); +} +#[tokio::test] +async fn unix_scale_v2_wire_fixture_uses_decimal_ns_and_bounded_summary_only() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + let mut context = context(); + context.room_digest = "c".repeat(64); + for source in &mut context.sources { + source.device_id = format!("esp32-node-{}", source.source_id); + source.link_id = format!("receiver-{}-transmitter-unverified", source.source_id); + source.grid_id = crate::room_learner::native::grid_id( + 1, + 64, + wifi_densepose_hardware::PpduType::HtLegacy, + ) + .unwrap(); + source.feature_count = 56; + } + rt.configure( + &rt.mutation_guard().unwrap(), + context.clone(), + "synthetic-unix-time-fixture".into(), + true, + ) + .unwrap(); + const BASE: u64 = 1_793_600_000_000_000_000; + for (i, label) in crate::room_learner::qualification::CONDITIONS + .into_iter() + .enumerate() + { + phase_at(&rt, label, i as u32, BASE, None); + } + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 8, + learning_rate: 0.5, + hidden_dim: 2, + embedding_dim: 2, + temporal_tokens: 2, + ..FitConfig::default() + }, + ) + .unwrap(); + assert_eq!(super::tests::finished(&rt).await["state"], "ready"); + for (i, label) in crate::room_learner::qualification::CONDITIONS + .into_iter() + .enumerate() + { + phase_at(&rt, label, i as u32 + 4, BASE, None); + } + let v1 = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + let cold = runtime(dir.path()); + cold.configure( + &cold.mutation_guard().unwrap(), + context.clone(), + "synthetic-fresh-runtime".into(), + false, + ) + .unwrap(); + let owner_status = cold.status(BASE + 300_000_000_000); + let readiness = owner_status["qualification_readiness"].clone(); + let clock_ineligible_status = cold.status(BASE - 1); + let diagnostic_dir = tempfile::tempdir().unwrap(); + let diagnostic = runtime(diagnostic_dir.path()); + let (diagnostic_model, _) = current(&rt); + diagnostic.store.save_candidate(&diagnostic_model).unwrap(); + diagnostic + .configure( + &diagnostic.mutation_guard().unwrap(), + context.clone(), + "v1-diagnostic-import".into(), + false, + ) + .unwrap(); + let legacy_ineligible_status = diagnostic.status(BASE + 300_000_000_000); + for (i, label) in crate::room_learner::qualification::CONDITIONS + .into_iter() + .enumerate() + { + phase_at(&cold, label, i as u32 + 8, BASE, None); + } + let q = cold.qualify(&cold.mutation_guard().unwrap()).unwrap(); + let receipt = serde_json::to_value(&q).unwrap(); + assert_eq!( + receipt["receipt"]["first_observed_at_ns"], + BASE.saturating_add(1_000_001).to_string() + ); + assert!(receipt["receipt"]["last_observed_at_ns"].is_string()); + let summary = serde_json::to_value(crate::room_learner::selection::QualificationSummary::from( + &q, + )) + .unwrap(); + assert!(summary["evaluated_at_unix_ms"].as_u64().unwrap() < (1u64 << 53)); + for key in [ + "first_observed_at_ns", + "last_observed_at_ns", + "phase_claims", + "sources", + "frame_ids", + "sequence_ids", + ] { + assert!(summary.get(key).is_none()); + } + let fixture = json!({"fixture_kind":"synthetic_native_producer_2026_utc","readiness":readiness,"owner_status":owner_status,"legacy_ineligible_status":legacy_ineligible_status,"clock_ineligible_status":clock_ineligible_status,"v1_summary":crate::room_learner::selection::QualificationSummary::from(&v1),"v2_summary":summary,"v2_envelope":receipt,"raw_train_lineage_checked":cold.status(BASE+300_000_000_000)["last_evaluation"]["train_lineage_checked"]}); + if let Ok(path) = std::env::var("RUVIEW_MANAGED_PROOF_FIXTURE_OUT") { + std::fs::write(path, serde_json::to_vec_pretty(&fixture).unwrap()).unwrap(); + } +} +#[tokio::test] +async fn shared_current_role_session_is_allowed_but_sealed_resume_group_cannot_be_reused() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path()); + rt.configure( + &rt.mutation_guard().unwrap(), + context(), + "shared-role-fixture".into(), + true, + ) + .unwrap(); + for (i, label) in crate::room_learner::qualification::CONDITIONS + .into_iter() + .enumerate() + { + phase_at_group( + &rt, + label, + i as u32, + START, + None, + Some("one-declared-train-session"), + ); + } + rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig { + epochs: 8, + learning_rate: 0.5, + hidden_dim: 2, + embedding_dim: 2, + temporal_tokens: 2, + ..FitConfig::default() + }, + ) + .unwrap(); + assert_eq!(super::tests::finished(&rt).await["state"], "ready"); + for (i, label) in crate::room_learner::qualification::CONDITIONS + .into_iter() + .enumerate() + { + phase_at_group( + &rt, + label, + i as u32 + 4, + START, + None, + Some("one-declared-heldout-session"), + ); + } + let q = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + assert_eq!(q.receipt.report.declared_session_count, 1); + assert!(!q.receipt.independence_verified); + let cold = restore(dir.path()); + assert!(cold + .begin_phase( + &cold.mutation_guard().unwrap(), + PhaseRequest { + trial_id: "actually-new-trial".into(), + session_id: "one-declared-heldout-session".into(), + role: DataRole::HeldOut, + label: RoomLabel::Empty, + duration_ms: 12000, + lead_in_ms: 0, + conditions_confirmed: true + }, + START + 200_000_000_000 + ) + .is_err()); + assert!(!cold.capture_active().unwrap()); +} +#[tokio::test] +async fn phase_seal_waits_for_durable_fence_and_pending_retry_keeps_original_metadata() { + let dir = tempfile::tempdir().unwrap(); + let rt = super::selection_tests::fitted(dir.path()).await; + let (model, proof) = current(&rt); + let directory = proof_path(dir.path(), &model) + .parent() + .unwrap() + .join(format!("managed-fences-{}", proof.model_sha256)); + let head = directory.join("managed-sealed-phase-head-v1.json"); + let old_head = std::fs::read(&head).unwrap(); + let old = rt.store.load_managed_capture_fence(&model, &proof).unwrap(); + let end = prepare_phase_at(&rt, RoomLabel::Empty, 8, START, None, None); + crate::companion::atomic_private_write(&head, b"{").unwrap(); + assert!(rt + .finish_phase(&rt.mutation_guard().unwrap(), end, true) + .is_err()); + assert!(rt.capture_active().unwrap()); + assert_eq!( + rt.inner + .lock() + .unwrap() + .capture + .as_ref() + .unwrap() + .history() + .len(), + 8 + ); + crate::companion::atomic_private_write(&head, &old_head).unwrap(); + let mut completed = rt.inner.lock().unwrap().capture.as_ref().unwrap().clone(); + completed.finish_phase(end).unwrap(); + let pending = crate::room_learner::managed_capture_fences::ManagedCaptureFence::issue( + &proof, + Some(&old), + completed.managed_fence_seed(&proof).unwrap(), + ) + .unwrap(); + crate::companion::atomic_private_write( + &directory.join(format!("{}.json", pending.fence_sha256)), + &serde_json::to_vec(&pending).unwrap(), + ) + .unwrap(); + // Unlabelled packets after a completed phase don't replace the first sealed + // phase record's timestamps when the finish acknowledgement is retried. + let values = vec![2.0; 8]; + let last = &pending.seed.sources[0]; + rt.ingest( + RawSourceFrame { + source_id: last.source_id, + grid_id: "ht64-canonical8", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: last.sequence + 1, + amplitudes: &values, + }, + end + 1, + ) + .unwrap(); + rt.finish_phase(&rt.mutation_guard().unwrap(), end + 2, true) + .unwrap(); + assert!(!rt.capture_active().unwrap()); + assert_eq!( + rt.store.load_managed_capture_fence(&model, &proof).unwrap(), + pending + ); +} +#[tokio::test] +async fn managed_proof_and_initial_fence_must_commit_before_candidate_pointer_and_retry_is_immutable( +) { + let source_dir = tempfile::tempdir().unwrap(); + let source = super::selection_tests::fitted(source_dir.path()).await; + let (model, original) = current(&source); + let target_dir = tempfile::tempdir().unwrap(); + let target = runtime(target_dir.path()); + let proof_file = proof_path(target_dir.path(), &model); + crate::companion::private_dir(proof_file.parent().unwrap()).unwrap(); + let fence_directory = proof_file + .parent() + .unwrap() + .join(format!("managed-fences-{}", original.model_sha256)); + crate::companion::atomic_private_write(&fence_directory, b"not a directory").unwrap(); + assert!(target + .store + .save_managed_candidate(&model, original.seed.clone()) + .is_err()); + assert!(target + .store + .load_candidate(model.context()) + .unwrap() + .is_none()); + assert!(!proof_file + .parent() + .unwrap() + .join("candidate-v1.json") + .exists()); + let first = std::fs::read(&proof_file).unwrap(); + std::fs::remove_file(&fence_directory).unwrap(); + let mut later = original.seed.clone(); + for source in &mut later.sources { + source.ordinal += 1; + source.sequence = source.sequence.wrapping_add(1); + source.last_time_ns += 1; + source.admitted_packets += 1; + } + later.last_seen_ns += 1; + target.store.save_managed_candidate(&model, later).unwrap(); + assert_eq!(std::fs::read(&proof_file).unwrap(), first); + assert_eq!( + target + .store + .load_managed_training_proof(&model) + .unwrap() + .unwrap(), + original + ); + let fence = target + .store + .load_managed_capture_fence(&model, &original) + .unwrap(); + assert_eq!(fence.revision, 0); + assert_eq!(fence.seed, original.seed); + assert!(target + .store + .load_candidate(model.context()) + .unwrap() + .is_some()); +} +#[tokio::test] +async fn malformed_metadata_checkpoint_fails_closed_without_integer_overflow() { + let dir = tempfile::tempdir().unwrap(); + let rt = super::selection_tests::fitted(dir.path()).await; + let (_, proof) = current(&rt); + let capture = CaptureSession::resume_managed_proof(&proof).unwrap(); + let at = proof.seed.last_seen_ns + 10; + let mut value = serde_json::to_value(capture.checkpoint(at).unwrap()).unwrap(); + value["metadata_lineage"]["phase_claims"][0]["cohort_count"] = json!(u64::MAX); + let snapshot = serde_json::from_value(value).unwrap(); + let result = std::panic::catch_unwind(|| { + CaptureSession::restore_checkpoint(snapshot, &proof.seed.context, at + 1) + }); + assert!( + result.is_ok(), + "malformed metadata panicked before bounded validation" + ); + assert!(result.unwrap().is_err()); +} +#[tokio::test] +async fn cold_resume_preserves_total_cohort_budget_and_refuses_before_operator_capture() { + let dir = tempfile::tempdir().unwrap(); + let rt = super::selection_tests::fitted(dir.path()).await; + for index in 8..32 { + phase_at(&rt, RoomLabel::Empty, index, START, None); + } + let cold = restore(dir.path()); + let result = cold.begin_phase( + &cold.mutation_guard().unwrap(), + PhaseRequest { + trial_id: "fresh-but-over-budget".into(), + session_id: "new-group".into(), + role: DataRole::HeldOut, + label: RoomLabel::Empty, + duration_ms: 12000, + lead_in_ms: 0, + conditions_confirmed: true, + }, + START + 500_000_000_000, + ); + assert!(result.unwrap_err().contains("capacity")); + assert!(!cold.capture_active().unwrap()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs index 3d1ef496e1..7ed686e92a 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs @@ -12,3 +12,7 @@ pub mod qualification; pub mod selection; pub mod assessment; + +pub(crate) mod managed_training; + +pub(crate) mod managed_capture_fences; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification.rs index adc0b68898..9d5145b06e 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification.rs @@ -52,6 +52,19 @@ pub struct ConditionCriterion { pub unconditional_sensitivity: Option, pub false_positive_rate: Option, } +/// v1 keeps historical numeric bytes; v2 uses exact decimal strings for UTC ns +/// beyond JavaScript's safe-integer range. Neither form asserts source time. +#[derive(Debug,Clone,Serialize,Deserialize,PartialEq)] +#[serde(untagged)] +pub enum ReceiptTimestamp { Legacy(u64), Decimal(String) } +impl ReceiptTimestamp { + fn new(value:u64,proof:bool)->Self {if proof {Self::Decimal(value.to_string())} else {Self::Legacy(value)}} + fn value(&self)->Result {match self { + Self::Legacy(n)=>Ok(*n), + Self::Decimal(s) if !s.is_empty()&&s.len()<=20&&s.bytes().all(|b|b.is_ascii_digit())&&(!s.starts_with('0')||s=="0")=>s.parse().map_err(|_|"invalid qualification timestamp".into()), + _=>Err("invalid qualification timestamp".into()), + }} +} #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct QualificationReceipt { @@ -70,13 +83,19 @@ pub struct QualificationReceipt { pub policy_sha256: String, pub label_provenance: String, pub sealed_history_sha256: String, - pub first_observed_at_ns: u64, - pub last_observed_at_ns: u64, + pub first_observed_at_ns: ReceiptTimestamp, + pub last_observed_at_ns: ReceiptTimestamp, pub criteria: Vec, pub report: EvaluationReport, pub independence_verified: bool, pub physically_validated: bool, pub authority_scope: String, + #[serde(default, skip_serializing_if="Option::is_none")] + pub training_lineage_basis: Option, + #[serde(default, skip_serializing_if="Option::is_none")] + pub managed_training_proof_sha256: Option, + #[serde(default, skip_serializing_if="Option::is_none")] + pub managed_sealed_phase_fence_sha256: Option, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] @@ -164,6 +183,8 @@ pub(crate) struct ManagedEvidence { pub first_observed_at_ns: u64, pub last_observed_at_ns: u64, pub last_held_out_observed_ns: u64, + pub training_proof_sha256: Option, + pub sealed_phase_fence_sha256: Option, } pub(crate) fn managed_evidence( model: &RoomModel, @@ -171,12 +192,24 @@ pub(crate) fn managed_evidence( history: &[PhaseReceipt], evaluation: &super::artifact::EvaluationReceipt, ) -> Result { - if dataset.context() != model.context() - || dataset.train().is_empty() - || hash(&dataset.train())? != model.training_manifest().training_data_sha256 - { - return Err("qualification requires retained exact Train manifest lineage".into()); - } + managed_evidence_with_proof(model,dataset,history,evaluation,None) +} +pub(crate) fn managed_evidence_with_proof( + model:&RoomModel, dataset:&ValidatedDataset, history:&[PhaseReceipt], + evaluation:&super::artifact::EvaluationReceipt, + proof:Option<&super::managed_training::ManagedTrainingProof>, +)->Result { + if dataset.context()!=model.context() {return Err("qualification context mismatch".into());} + let persisted_proof=if dataset.train().is_empty() { + let p=proof.ok_or("qualification requires retained exact Train rows or a verified managed training proof")?; + p.check_held_out(model,dataset)?; Some(p) + } else { + // Never hide changed retained rows behind a formerly valid proof. + if hash(&dataset.train())? != model.training_manifest().training_data_sha256 { + return Err("qualification requires retained exact Train manifest lineage".into()); + } + None + }; let rows: Vec<_> = dataset.train().iter().chain(dataset.held_out()).collect(); if rows.iter().any(|c| { c.provenance != CaptureProvenance::LiveReceiverObserved @@ -225,8 +258,9 @@ pub(crate) fn managed_evidence( relevant.push(phase); } } - let counts = - CONDITIONS.map(|label| dataset.train().iter().filter(|c| c.label == label).count()); + let counts = persisted_proof.map_or_else( + || CONDITIONS.map(|label|dataset.train().iter().filter(|c|c.label==label).count()), + |p|p.seed.train_counts); evaluation.report.validate()?; for (i, label) in CONDITIONS.iter().enumerate() { if counts[i] < 4 @@ -265,9 +299,11 @@ pub(crate) fn managed_evidence( Ok(ManagedEvidence { counts, - sealed_history_sha256: hash(&relevant)?, - first_observed_at_ns: first, + sealed_history_sha256: if let Some(p)=persisted_proof {hash(&("managed-training-proof-v1+sealed-heldout-v1",&p.proof_sha256,&p.seed.sealed_train_history_sha256,&relevant))?} else {hash(&relevant)?}, + first_observed_at_ns: persisted_proof.map_or(first,|p|p.seed.first_observed_ns.min(first)), last_observed_at_ns: last, + training_proof_sha256: persisted_proof.map(|p|p.proof_sha256.clone()), + sealed_phase_fence_sha256: None, last_held_out_observed_ns: dataset .held_out() .iter() @@ -278,6 +314,18 @@ pub(crate) fn managed_evidence( }) } +pub(crate) fn managed_evidence_with_fence( + model:&RoomModel,dataset:&ValidatedDataset,history:&[PhaseReceipt],evaluation:&super::artifact::EvaluationReceipt, + proof:Option<&super::managed_training::ManagedTrainingProof>,fence_sha:Option<&str>, +)->Result { + let mut evidence=managed_evidence_with_proof(model,dataset,history,evaluation,proof)?; + if evidence.training_proof_sha256.is_some() { + let sha=fence_sha.filter(|s|s.len()==64&&s.bytes().all(|b|b.is_ascii_digit()||(b'a'..=b'f').contains(&b))).ok_or("managed qualification requires its durable sealed phase fence")?; + evidence.sealed_history_sha256=hash(&("managed-phase-fence-v1",&evidence.sealed_history_sha256,sha))?; + evidence.sealed_phase_fence_sha256=Some(sha.into()); + } + Ok(evidence) +} impl QualificationEnvelope { pub(crate) fn issue( model: &RoomModel, @@ -285,12 +333,20 @@ impl QualificationEnvelope { history: &[PhaseReceipt], evaluation: &super::artifact::EvaluationReceipt, ) -> Result { - let evidence = managed_evidence(model, dataset, history, evaluation)?; + Self::issue_with_proof(model,dataset,history,evaluation,None,None) + } + pub(crate) fn issue_with_proof( + model:&RoomModel,dataset:&ValidatedDataset,history:&[PhaseReceipt], + evaluation:&super::artifact::EvaluationReceipt, + proof:Option<&super::managed_training::ManagedTrainingProof>, + fence_sha:Option<&str>, + )->Result { + let evidence = managed_evidence_with_fence(model,dataset,history,evaluation,proof,fence_sha)?; let checks = criteria(&evaluation.report, &evidence.counts)?; let c = model.context(); let policy = QualificationPolicy::default(); let receipt = QualificationReceipt { - schema: "ruview.local-operational-qualification.v1".into(), + schema: if evidence.training_proof_sha256.is_some() {"ruview.local-operational-qualification.v2"} else {"ruview.local-operational-qualification.v1"}.into(), owner_scope: c.owner_scope.clone(), installation_id: c.installation_id.clone(), room_digest: c.room_digest.clone(), @@ -303,15 +359,18 @@ impl QualificationEnvelope { evaluated_at_unix_ms: evaluation.evaluated_at_unix_ms, policy_sha256: hash(&policy)?, policy, - label_provenance: "managed-sealed-history-v1".into(), + label_provenance: if evidence.training_proof_sha256.is_some() {"managed-training-proof-v1+sealed-heldout-v1"} else {"managed-sealed-history-v1"}.into(), sealed_history_sha256: evidence.sealed_history_sha256, - first_observed_at_ns: evidence.first_observed_at_ns, - last_observed_at_ns: evidence.last_observed_at_ns, + first_observed_at_ns: ReceiptTimestamp::new(evidence.first_observed_at_ns,evidence.training_proof_sha256.is_some()), + last_observed_at_ns: ReceiptTimestamp::new(evidence.last_observed_at_ns,evidence.training_proof_sha256.is_some()), criteria: checks, report: evaluation.report.clone(), independence_verified: false, physically_validated: false, authority_scope: "local_operator_binary_presence".into(), + training_lineage_basis: evidence.training_proof_sha256.as_ref().map(|_|"managed_training_proof_v1".into()), + managed_training_proof_sha256: evidence.training_proof_sha256, + managed_sealed_phase_fence_sha256: evidence.sealed_phase_fence_sha256, }; let envelope = Self { receipt_sha256: hash(&receipt)?, @@ -332,7 +391,9 @@ impl QualificationEnvelope { && x.bytes() .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) }; - if r.schema != "ruview.local-operational-qualification.v1" + let v1=r.schema=="ruview.local-operational-qualification.v1" && r.training_lineage_basis.is_none() &&r.managed_training_proof_sha256.is_none()&&r.managed_sealed_phase_fence_sha256.is_none() &&r.label_provenance=="managed-sealed-history-v1" &&matches!(r.first_observed_at_ns,ReceiptTimestamp::Legacy(_))&&matches!(r.last_observed_at_ns,ReceiptTimestamp::Legacy(_)); + let v2=r.schema=="ruview.local-operational-qualification.v2" &&r.training_lineage_basis.as_deref()==Some("managed_training_proof_v1") &&r.managed_training_proof_sha256.as_deref().is_some_and(hex)&&r.managed_sealed_phase_fence_sha256.as_deref().is_some_and(hex) &&r.label_provenance=="managed-training-proof-v1+sealed-heldout-v1" && !eval.train_lineage_checked &&matches!(r.first_observed_at_ns,ReceiptTimestamp::Decimal(_))&&matches!(r.last_observed_at_ns,ReceiptTimestamp::Decimal(_)); + if !(v1||v2) || self.receipt_sha256 != hash(r)? || r.owner_scope != c.owner_scope || r.installation_id != c.installation_id @@ -348,10 +409,9 @@ impl QualificationEnvelope { || eval.role != DataRole::HeldOut || r.policy != QualificationPolicy::default() || r.policy_sha256 != hash(&r.policy)? - || r.label_provenance != "managed-sealed-history-v1" || !hex(&r.sealed_history_sha256) - || r.first_observed_at_ns == 0 - || r.last_observed_at_ns < r.first_observed_at_ns + || r.first_observed_at_ns.value()? == 0 + || r.last_observed_at_ns.value()? < r.first_observed_at_ns.value()? || r.independence_verified || r.physically_validated || r.authority_scope != "local_operator_binary_presence" diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection.rs index 8909302695..0837f383b1 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection.rs @@ -65,6 +65,12 @@ pub enum SelectedBinaryEvidence { } #[derive(Debug, Clone, Serialize)] pub struct QualificationSummary { + #[serde(skip_serializing_if="Option::is_none")] + pub schema: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub training_lineage_basis: Option, + #[serde(skip_serializing_if="Option::is_none")] + pub managed_training_proof_sha256: Option, pub receipt_sha256: String, pub model_sha256: String, pub context_digest: String, @@ -80,6 +86,9 @@ impl From<&QualificationEnvelope> for QualificationSummary { fn from(q: &QualificationEnvelope) -> Self { let r = &q.receipt; Self { + schema: (r.schema=="ruview.local-operational-qualification.v2").then(||r.schema.clone()), + training_lineage_basis:r.training_lineage_basis.clone(), + managed_training_proof_sha256:r.managed_training_proof_sha256.clone(), receipt_sha256: q.receipt_sha256.clone(), model_sha256: r.model_sha256.clone(), context_digest: r.context_digest.clone(), diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection_store.rs index 9b0518d98b..4983db4c38 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection_store.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/selection_store.rs @@ -35,6 +35,13 @@ impl ArtifactStore { &q.receipt.evaluation_receipt_sha256, )?; q.validate(model, &eval)?; + if let Some(proof)=self.load_managed_training_proof(model)? {self.load_managed_capture_fence(model,&proof)?;} + + if let Some(expected)=&q.receipt.managed_training_proof_sha256 { + let proof=self.load_managed_training_proof(model)?.ok_or("qualification managed training proof missing")?; + if &proof.proof_sha256!=expected {return Err("qualification managed training proof changed".into());} + self.require_managed_capture_fence(model,&proof,q.receipt.managed_sealed_phase_fence_sha256.as_deref().ok_or("qualification sealed phase fence missing")?)?; + } let dir = self .qualification_dir(model.context(), true)? .ok_or("qualification directory unavailable")?; @@ -75,6 +82,13 @@ impl ArtifactStore { &q.receipt.evaluation_receipt_sha256, )?; q.validate(model, &eval)?; + if let Some(proof)=self.load_managed_training_proof(model)? {self.load_managed_capture_fence(model,&proof)?;} + + if let Some(expected)=&q.receipt.managed_training_proof_sha256 { + let proof=self.load_managed_training_proof(model)?.ok_or("qualification managed training proof missing")?; + if &proof.proof_sha256!=expected {return Err("qualification managed training proof changed".into());} + self.require_managed_capture_fence(model,&proof,q.receipt.managed_sealed_phase_fence_sha256.as_deref().ok_or("qualification sealed phase fence missing")?)?; + } Ok(q) } pub(crate) fn load_latest_qualification( From 63e4f961a7e26c8ad1de0108e3c8e113ea084b28 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Wed, 4 Nov 2026 16:37:10 +0100 Subject: [PATCH 062/100] feat(companion): publish bounded read-only calibration progress --- docs/api/paired-calibration-progress-v1.md | 70 +++++++ .../src/bearer_auth.rs | 13 +- .../src/companion_integration.rs | 8 + .../wifi-densepose-sensing-server/src/main.rs | 1 + .../src/paired_progress_auth_tests.rs | 126 +++++++++++++ .../src/room_learner/lifecycle.rs | 4 + .../src/room_learner/paired_progress.rs | 154 ++++++++++++++++ .../src/room_learner/paired_progress_tests.rs | 131 +++++++++++++ .../src/room_learner_api.rs | 4 + .../src/room_learner_api_progress.rs | 40 ++++ .../src/room_learner_api_selection.rs | 1 + .../paired-calibration-progress-v1.json | 173 ++++++++++++++++++ 12 files changed, 720 insertions(+), 5 deletions(-) create mode 100644 docs/api/paired-calibration-progress-v1.md create mode 100644 v2/crates/wifi-densepose-sensing-server/src/paired_progress_auth_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/paired_progress.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/paired_progress_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner_api_progress.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/tests/fixtures/paired-calibration-progress-v1.json diff --git a/docs/api/paired-calibration-progress-v1.md b/docs/api/paired-calibration-progress-v1.md new file mode 100644 index 0000000000..9b62f94298 --- /dev/null +++ b/docs/api/paired-calibration-progress-v1.md @@ -0,0 +1,70 @@ +# Paired calibration progress v1 + +Implemented as a proposed read-only companion surface. It reports the managed +Mac's RF learner, not a second phone learner or legacy baseline controller. + +`GET /api/v1/room-learner/progress` uses existing sensing-read authentication and +the companion's installation-pinned, owner/account-bound read grant. Neither a +read nor legacy calibration grant permits POST to this route or owner-only +learner controls. The existing selection envelope advertises +`progress_schema: "ruview.room-learner.progress.v1"`. Consumers discover through +that authorized read first, without first parsing an unsupported inner +selection version or requiring a historical selection to match today's room. +An absent discovery field means selection-only; +old TLS policy may reject an unknown progress route with403 before reaching404. +A404 can also degrade to the selection-only view, but privacy/authentication +failures are never a fallback signal. Do not probe owner status. + +Success schema: `ruview.room-learner.progress.v1`. The explicit DTO contains: + +- Public installation ID, published room binding/revision, context match, + `runtime_epoch`, `observed_at_unix_ms` and `expires_at_unix_ms`. +- `runtime_epoch` is the existing public calibration process identity. It is + **not** the learner's private mutation-guard runtime epoch. +- Recorded Train/HeldOut counts for four conditions; these are sealed records + in the current capture session, not proof of passing accuracy or all history + in a durable model. Zero temporary records do not mean a model was lost. +- Bounded phase role/label, preparation/countdown/elapsed/target and representative + counts, plus aggregate source readiness. No source IDs, packet counters, + RF values or physiological values are included. +- Separate verified qualification and selection summaries, a bounded machine + blocker and a workflow stage. These do not start a capture, train, qualify, + select or enable another capability. + Qualification names the current candidate where available; selection names + the separate durable selected model. Their model hashes may differ while a + replacement candidate is being prepared. No candidate means qualification is + not reported, rather than a claim that an older selected model lost its proof. +- `save_status` is always `not_reported`: model selection does not prove that + the Mac app's separate durable Room Save transaction completed. + +The snapshot expires at most10seconds after observation, shortened by the +existing paired/account grant's expiry. A revoked or expired grant is refused +before response. Consumers must also fence account, transport, installation, +publication, request generation and runtime changes; reject delayed/stale data. +Published binding is held stable through projection. A concurrent learner +context change cannot borrow readiness from the prior source/grid context. +A published-room mismatch withholds counts, phase, source readiness and all +current qualification/selection claims rather than relabelling another room. +A historical selection for another room/context is not reported as this room's +selection and does not alone block setup for the newly published room. + +An empty/unconfigured room has null progress fields. An optional durable-proof +readiness field is not implemented by this base producer. The blocker +`training_lineage_not_reported` means only that this view does not report that +lineage; it is not a statement that a stored model or future managed proof is +invalid. Review readiness on the Mac before requesting physical trials. + +Privacy mode returns403 with only schema/success/error_code. No owner scope, +mutation guard, trial/session IDs, model weights, private training-proof data, +raw source counters, media or physiology is projected. Unknown/corrupt artifact +readback cannot produce a qualified/current badge. + +"Read-only" means no logical capture, model, geometry, checkpoint or Save +mutation. Existing trusted artifact readers still enforce private-directory +security idempotently (0700); this is not a claim of zero filesystem metadata +side effects. The progress method deliberately does not call owner status, +which can write capture checkpoints. + +Synthetic producer fixtures are generated by the real HTTP handler test +`companion_integration::paired_progress_auth_tests::export_actual_paired_progress_producer_cases`. +They are software evidence, not physical sensing qualification. diff --git a/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs b/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs index 81790cd018..71010d6d62 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/bearer_auth.rs @@ -1377,11 +1377,14 @@ mod oauth_tests { let workspace = make_token("10000000-0000-4000-8000-000000000001", "30000000-0000-4000-8000-000000000002"); for (credential, expected) in [(None, StatusCode::UNAUTHORIZED), (Some(b), StatusCode::FORBIDDEN), (Some(workspace), StatusCode::FORBIDDEN), (Some(a.clone()), StatusCode::OK), (Some(format!("{a}x")), StatusCode::UNAUTHORIZED)] { - let router = Router::new().route("/companion/v1/enroll", post(|| async { "accepted" })) - .layer(axum::middleware::from_fn_with_state(auth.clone(), crate::companion::require_account_ownership)); - let mut request = Request::builder().method("POST").uri("/companion/v1/enroll"); - if let Some(value) = credential { request = request.header("X-RuView-Account-Authorization", value); } - assert_eq!(router.oneshot(request.body(Body::empty()).unwrap()).await.unwrap().status(), expected); + for (method,path) in [("POST","/companion/v1/enroll"),("GET","/api/v1/room-learner/progress")] { + let router = Router::new().route("/companion/v1/enroll", post(|| async { "accepted" })) + .route("/api/v1/room-learner/progress", get(|| async { "accepted" })) + .layer(axum::middleware::from_fn_with_state(auth.clone(), crate::companion::require_account_ownership)); + let mut request = Request::builder().method(method).uri(path); + if let Some(value) = credential.as_ref() { request = request.header("X-RuView-Account-Authorization", value); } + assert_eq!(router.oneshot(request.body(Body::empty()).unwrap()).await.unwrap().status(), expected); + } } } diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs index de2aacf3c1..0473b3183d 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs @@ -258,6 +258,7 @@ fn policy(method: &axum::http::Method, path: &str) -> Option { | "/api/v1/pose/zones/summary" | "/api/v1/pose/activities" | "/api/v1/room-learner/selection" + | "/api/v1/room-learner/progress" | "/api/v1/calibration/status" | "/api/v1/pose/calibration/status" | "/ws/sensing" @@ -387,6 +388,9 @@ mod tests { assert_eq!(policy(&Method::GET, "/api/v1/room-learner/selection"), Some(false)); assert_eq!(policy(&Method::HEAD, "/api/v1/room-learner/selection"), Some(false)); assert_eq!(policy(&Method::GET, "/api/v1/room-learner/status"), None); + assert_eq!(policy(&Method::GET, "/api/v1/room-learner/progress"), Some(false)); + assert_eq!(policy(&Method::HEAD, "/api/v1/room-learner/progress"), Some(false)); + assert_eq!(policy(&Method::POST, "/api/v1/room-learner/progress"), None); for path in ["/api/v1/room-learner/qualify", "/api/v1/room-learner/select", "/api/v1/room-learner/deselect", "/api/v1/room-learner/phase/cancel", "/api/v1/room-learner/train/cancel"] { assert_eq!(policy(&Method::POST, path), None, "paired read/calibration grants cannot select: {path}"); } @@ -452,3 +456,7 @@ mod tests { } } + +#[cfg(test)] +#[path = "paired_progress_auth_tests.rs"] +mod paired_progress_auth_tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index a156bb000a..2666c074fe 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -16488,6 +16488,7 @@ async fn main() { // Mutations remain behind the existing default-admin bearer policy. .route("/api/v1/room-learner/status", get(room_learner_api::status)) .route("/api/v1/room-learner/selection", get(room_learner_api::selection)) + .route("/api/v1/room-learner/progress", get(room_learner_api::progress)) .merge(room_learner_api::owner_selection_routes(bearer_auth_state.clone())) .route("/api/v1/room-learner/configure", post(room_learner_api::configure)) .route("/api/v1/room-learner/phase/start", post(room_learner_api::begin_phase)) diff --git a/v2/crates/wifi-densepose-sensing-server/src/paired_progress_auth_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/paired_progress_auth_tests.rs new file mode 100644 index 0000000000..77efc18529 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/paired_progress_auth_tests.rs @@ -0,0 +1,126 @@ +//! Synthetic HTTP producer plus actual companion policy/credential boundaries. +use super::*; +use axum::{body::Body,http::{Method,Request},routing::get}; +use tower::ServiceExt; +use wifi_densepose_sensing_server::room_learner::{lifecycle::RoomLearningRuntime,observation::{CaptureContext,SourceSpec}}; +const PATH:&str="/api/v1/room-learner/progress"; +async fn json_call(router:Router,method:Method,path:&str,body:serde_json::Value)->(StatusCode,serde_json::Value){ + let r=router.oneshot(Request::builder().method(method).uri(path).header("content-type","application/json").body(Body::from(body.to_string())).unwrap()).await.unwrap(); + let status=r.status();let b=axum::body::to_bytes(r.into_body(),65536).await.unwrap();(status,serde_json::from_slice(&b).unwrap_or(serde_json::Value::Null)) +} +async fn pair(service:Arc,device:&str)->String { + let owner=companion::owner_router(service.clone());let public=companion::public_router(service); + let (s,invite)=json_call(owner.clone(),Method::POST,"/api/v1/companion/invites",serde_json::json!({})).await;assert_eq!(s,StatusCode::OK); + let proof=companion::secret(); + let (s,enrolled)=json_call(public.clone(),Method::POST,"/companion/v1/enroll",serde_json::json!({"installation_id":"synthetic-install","invite_id":invite["invite_id"],"invite_secret":invite["invite_secret"],"device_id":device,"device_name":"Synthetic phone","pairing_secret":proof})).await;assert_eq!(s,StatusCode::OK); + let (s,_)=json_call(owner,Method::POST,&format!("/api/v1/companion/requests/{}/approve",enrolled["request_id"].as_str().unwrap()),serde_json::json!({"comparison_code":enrolled["comparison_code"]})).await;assert_eq!(s,StatusCode::OK); + let (s,session)=json_call(public,Method::POST,"/companion/v1/session",serde_json::json!({"installation_id":"synthetic-install","device_id":device,"pairing_secret":proof})).await;assert_eq!(s,StatusCode::OK);session["access_token"].as_str().unwrap().into() +} +fn room()->serde_json::Value{serde_json::json!({"schema":"ruview.companion.room.v1","id":"synthetic-room","name":"Synthetic room","widthM":3,"lengthM":3,"heightM":2.8,"nodes":[{"nodeId":1,"slot":"north_west","xM":0,"yM":0,"role":"base_station","headingDeg":90,"orientationConfirmed":true,"confirmedAtUnixMs":1},{"nodeId":2,"slot":"south_east","xM":3,"yM":3,"role":"receiver","headingDeg":270,"orientationConfirmed":true,"confirmedAtUnixMs":1}],"accessPoint":null})} +async fn fixture()->(tempfile::TempDir,SharedState,Arc){ + let dir=tempfile::tempdir().unwrap();let owner="a".repeat(64); + let rt=Arc::new(RoomLearningRuntime::new(&dir.path().join(&owner).join("SensingServer"),&owner,"synthetic-install").unwrap()); + let publication=companion_room::SharedRoomStore::load(&dir.path().join("publication"),"synthetic-install").unwrap(); + let (status,envelope)=json_call(companion_room::owner_router(publication.clone()),Method::PUT,"/api/v1/companion/room",serde_json::json!({"expected_revision":0,"room":room()})).await;assert_eq!(status,StatusCode::OK); + let context=CaptureContext{owner_scope:owner,installation_id:"synthetic-install".into(),room_digest:envelope["binding_digest"].as_str().unwrap().into(),revision:77,sources:(1..=2).map(|id|SourceSpec{source_id:id,device_id:format!("esp32-node-{id}"),grid_id:"ant1-bins64-ppdu0-usable-drop2-linear-v2".into(),feature_count:56,signal_kind:ruview_unified::RfModality::WifiCsi,center_frequency_hz:2_437_000_000,bandwidth_hz:20_000_000,link_id:format!("synthetic-link-{id}"),link_index:0,geometry:None}).collect()}; + rt.configure(&rt.mutation_guard().unwrap(),context,"synthetic-actual-http-run".into(),true).unwrap(); + let mut inner=AppStateInner::minimal();inner.room_learning=Some(rt);inner.current_room_store=Some(publication.clone());inner.installation_id=Some("synthetic-install".into());inner.calibration_boot_epoch="cal-boot-synthetic".into(); + let now=std::time::Instant::now(); + for row in 0..120u32 {for node in 1..=2u8 { + let mut wire=Vec::new();wire.extend(0xC5110001u32.to_le_bytes());wire.extend([node,1]);wire.extend(64u16.to_le_bytes());wire.extend(2437u32.to_le_bytes());wire.extend((row+1).to_le_bytes());wire.extend([(-40i8)as u8,(-90i8)as u8,0,0x20]);for _ in 0..64 {wire.extend([3,4]);} + let frame=parse_esp32_frame(&wire).unwrap();room_learner_api::observe(&mut inner,&frame,&wire,now-std::time::Duration::from_millis(u64::from(120-row)*100)); + }} + (dir,Arc::new(RwLock::new(inner)),publication) +} +fn app(state:SharedState,service:Arc)->Router{ + Router::new().route(PATH,get(room_learner_api::progress)).route("/api/v1/room-learner/status",get(room_learner_api::status)) + .route("/api/v1/room-learner/phase/start",post(room_learner_api::begin_phase)) + .with_state(state).layer(axum::middleware::from_fn_with_state(service,require_companion)) +} +async fn fetch(router:Router,method:Method,installation:&str,token:&str,path:&str)->(StatusCode,serde_json::Value){ + let response=router.oneshot(Request::builder().method(method).uri(path).header("x-ruview-installation-id",installation).header("authorization",format!("Bearer {token}")).body(Body::empty()).unwrap()).await.unwrap(); + let status=response.status();let b=axum::body::to_bytes(response.into_body(),65536).await.unwrap();(status,serde_json::from_slice(&b).unwrap_or(serde_json::Value::Null)) +} +#[tokio::test] +async fn progress_http_read_scope_installation_revocation_and_no_mutation_grant_compose(){ + let (dir,state,_)=fixture().await;let service=Companion::load(&dir.path().join("pair"),"synthetic-install".into(),"f".repeat(64),"https://synthetic.local:3443".into()).unwrap();let token=pair(service.clone(),"phone").await;let router=app(state.clone(),service.clone()); + let rt=state.read().await.room_learning.clone().unwrap();let guard=rt.mutation_guard().unwrap(); + let (s,p)=fetch(router.clone(),Method::GET,"synthetic-install",&token,PATH).await;assert_eq!(s,StatusCode::OK);assert_eq!(p["schema"],"ruview.room-learner.progress.v1");assert_eq!(p["context_match"],true);assert_eq!(p["room"]["revision"],1);assert_eq!(p["sources"]["required"],2);assert_eq!(p["sources"]["ready"],2);assert_eq!(p["save_status"],"not_reported");assert_eq!(rt.mutation_guard().unwrap(),guard); + assert!(service.authorize("synthetic-install",&token,true).is_err(),"read grant must remain read-only"); + for (method,path,install,t) in [(Method::POST,PATH,"synthetic-install",token.as_str()),(Method::POST,"/api/v1/room-learner/phase/start","synthetic-install",token.as_str()),(Method::GET,"/api/v1/room-learner/status","synthetic-install",token.as_str()),(Method::GET,PATH,"foreign-install",token.as_str()),(Method::GET,PATH,"synthetic-install","invalid")] { + let status=fetch(router.clone(),method,install,t,path).await.0;assert!(matches!(status,StatusCode::UNAUTHORIZED|StatusCode::FORBIDDEN)); + } + assert_eq!(fetch(router.clone(),Method::GET,"synthetic-install",&token,&format!("{PATH}?token=ignored")).await.0,StatusCode::BAD_REQUEST); + assert_eq!(json_call(companion::owner_router(service.clone()),Method::POST,"/api/v1/companion/devices/phone/calibration",serde_json::json!({})).await.0,StatusCode::OK); + assert!(service.authorize("synthetic-install",&token,true).is_ok()); + assert_eq!(fetch(router.clone(),Method::POST,"synthetic-install",&token,"/api/v1/room-learner/phase/start").await.0,StatusCode::FORBIDDEN,"legacy calibration grant must not enable learner mutations"); + assert_eq!(fetch(router.clone(),Method::POST,"synthetic-install",&token,PATH).await.0,StatusCode::FORBIDDEN); + + // A separate account/installation service does not recognize this device proof. + let foreign=Companion::load(&dir.path().join("foreign-pair"),"synthetic-install".into(),"e".repeat(64),"https://foreign.local:3443".into()).unwrap(); + assert_eq!(fetch(app(state,foreign),Method::GET,"synthetic-install",&token,PATH).await.0,StatusCode::UNAUTHORIZED); + let (s,_)=json_call(companion::owner_router(service),Method::DELETE,"/api/v1/companion/devices/phone",serde_json::json!({})).await;assert_eq!(s,StatusCode::OK); + assert_eq!(fetch(router,Method::GET,"synthetic-install",&token,PATH).await.0,StatusCode::UNAUTHORIZED); +} +#[tokio::test] +async fn progress_http_private_projection_context_and_privacy_withhold_other_room_data(){ + let (dir,state,publication)=fixture().await;let service=Companion::load(&dir.path().join("pair"),"synthetic-install".into(),"f".repeat(64),"https://synthetic.local:3443".into()).unwrap();let token=pair(service.clone(),"phone").await;let router=app(state.clone(),service); + let (_,body)=fetch(router.clone(),Method::GET,"synthetic-install",&token,PATH).await; + let serialized=body.to_string();for secret in ["owner_scope","mutation_guard","trial_id","session_id","frame_ids","sequence_ids","last_accepted_sequence","training_data_sha256","encoder_weights","occupied_probability","heart_rate","amplitudes","geometry"]{assert!(!serialized.contains(secret),"leaked {secret}");} + let mut changed=room();changed["id"]=serde_json::json!("another-room");assert_eq!(json_call(companion_room::owner_router(publication.clone()),Method::PUT,"/api/v1/companion/room",serde_json::json!({"expected_revision":1,"room":changed})).await.0,StatusCode::OK); + let (_,mismatch)=fetch(router.clone(),Method::GET,"synthetic-install",&token,PATH).await;assert_eq!(mismatch["room"]["revision"],2);assert_eq!(mismatch["context_match"],false);assert!(mismatch["conditions"].is_null()&&mismatch["phase"].is_null()&&mismatch["sources"].is_null());assert_eq!(mismatch["blocker"],"context_mismatch"); + state.write().await.privacy_mode=true; + let (s,privacy)=fetch(router,Method::GET,"synthetic-install",&token,PATH).await;assert_eq!(s,StatusCode::FORBIDDEN);assert_eq!(privacy,serde_json::json!({"schema":"ruview.room-learner.progress.v1","success":false,"error_code":"privacy_redacted"})); +} +#[tokio::test] +async fn export_actual_paired_progress_producer_cases(){ + let (dir,state,publication)=fixture().await;let service=Companion::load(&dir.path().join("pair"),"synthetic-install".into(),"f".repeat(64),"https://synthetic.local:3443".into()).unwrap();let token=pair(service.clone(),"phone").await;let router=app(state.clone(),service); + let mut cases=vec![]; + let (s,p)=fetch(router.clone(),Method::GET,"synthetic-install",&token,PATH).await;assert_eq!(s,StatusCode::OK);cases.push(serde_json::json!({"name":"configured","payload":p})); + let rt=state.read().await.room_learning.clone().unwrap(); + rt.begin_phase(&rt.mutation_guard().unwrap(),wifi_densepose_sensing_server::room_learner::capture::PhaseRequest{trial_id:"private-operator-trial".into(),session_id:"private-operator-session".into(),role:wifi_densepose_sensing_server::room_learner::observation::DataRole::Train,label:wifi_densepose_sensing_server::room_learner::observation::RoomLabel::Empty,duration_ms:600_000,lead_in_ms:15_000,conditions_confirmed:true},rt.receiver_now_ns()).unwrap(); + let (_,p)=fetch(router.clone(),Method::GET,"synthetic-install",&token,PATH).await;assert_eq!(p["phase"]["stage"],"lead_in");cases.push(serde_json::json!({"name":"preparing_empty","payload":p}));rt.cancel_phase(&rt.mutation_guard().unwrap()).unwrap(); + let mut changed=room();changed["id"]=serde_json::json!("changed-room");assert_eq!(json_call(companion_room::owner_router(publication.clone()),Method::PUT,"/api/v1/companion/room",serde_json::json!({"expected_revision":1,"room":changed})).await.0,StatusCode::OK);let (_,p)=fetch(router.clone(),Method::GET,"synthetic-install",&token,PATH).await;cases.push(serde_json::json!({"name":"context_mismatch","payload":p})); + state.write().await.privacy_mode=true;let (_,p)=fetch(router,Method::GET,"synthetic-install",&token,PATH).await;cases.push(serde_json::json!({"name":"privacy_redacted","payload":p})); + if let Ok(path)=std::env::var("RUVIEW_TEST_PAIRED_PROGRESS_OUTPUT") { + use std::io::Write;use std::os::unix::fs::OpenOptionsExt; + let path=std::path::PathBuf::from(path);assert!(path.is_absolute()&&path.parent().unwrap().canonicalize().unwrap().starts_with(std::env::temp_dir().canonicalize().unwrap())); + let mut file=std::fs::OpenOptions::new().write(true).create_new(true).mode(0o600).open(path).unwrap(); + file.write_all(serde_json::to_string_pretty(&serde_json::json!({"schema":"ruview.test.paired-progress-producer.v1","evidence":"SYNTHETIC_ACTUAL_HTTP_PRODUCER_NOT_PHYSICAL_QUALIFICATION","cases":cases})).unwrap().as_bytes()).unwrap(); + } +} + +#[tokio::test] +async fn response_snapshot_cannot_outlive_or_resurrect_an_existing_peer_grant(){ + let (_dir,state,_)=fixture().await; + let (tx,rx)=tokio::sync::watch::channel(false); + let now=chrono::Utc::now().timestamp().max(0) as u64; + let grant=Grant{device_id:"private-phone-id".into(),expires_at:now+8,revoked:rx.clone()}; + let make=|grant:Grant|Router::new().route(PATH,get(room_learner_api::progress)).with_state(state.clone()).layer(axum::Extension(grant)); + let (s,body)=fetch(make(grant.clone()),Method::GET,"synthetic-install","unused",PATH).await; + assert_eq!(s,StatusCode::OK);assert!(body["expires_at_unix_ms"].as_u64().unwrap()<=grant.expires_at*1000);assert!(body["expires_at_unix_ms"].as_u64().unwrap()>,request:axum::extract::Request,next:Next)->Response { + if request.uri().path()==PATH {return StatusCode::FORBIDDEN.into_response();} + require_companion(State(service),request,next).await + } + let (dir,state,_)=fixture().await; + let service=Companion::load(&dir.path().join("pair"),"synthetic-install".into(),"f".repeat(64),"https://synthetic.local:3443".into()).unwrap();let token=pair(service.clone(),"phone").await; + let legacy=Router::new().route("/api/v1/room-learner/selection",get(|State(s):State|async move { + let Json(mut v)=room_learner_api::selection(State(s)).await; + v.as_object_mut().unwrap().remove("progress_schema");Json(v) + })).with_state(state.clone()).layer(axum::middleware::from_fn_with_state(service.clone(),old_policy)); + assert_eq!(fetch(legacy.clone(),Method::GET,"synthetic-install",&token,PATH).await.0,StatusCode::FORBIDDEN); + let (status,selection)=fetch(legacy,Method::GET,"synthetic-install",&token,"/api/v1/room-learner/selection").await; + assert_eq!(status,StatusCode::OK);assert!(selection.get("progress_schema").is_none());assert_eq!(selection["success"],true); + let new=Router::new().route("/api/v1/room-learner/selection",get(room_learner_api::selection)).with_state(state).layer(axum::middleware::from_fn_with_state(service,require_companion)); + assert_eq!(fetch(new,Method::GET,"synthetic-install",&token,"/api/v1/room-learner/selection").await.1["progress_schema"],"ruview.room-learner.progress.v1"); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs index 0e57c7e93c..bf78b82100 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs @@ -487,3 +487,7 @@ mod selection_tests; #[cfg(test)] #[path="lifecycle_assessment_tests.rs"] mod assessment_tests; + +#[path = "paired_progress.rs"] +mod paired_progress; +pub use paired_progress::{PairedCalibrationProgress, PublishedProgressRoom, PAIRED_PROGRESS_TTL_MS}; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/paired_progress.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/paired_progress.rs new file mode 100644 index 0000000000..084fcb2721 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/paired_progress.rs @@ -0,0 +1,154 @@ +//! Read-only phone companion projection. Counts record capture, never accuracy. +use super::*; +use crate::room_learner::{capture::PhaseStatus, + selection::{SelectedBinaryEvidence, ModelSelectionStatus}}; + +const MAX_SAFE: u64 = 9_007_199_254_740_991; +pub const PAIRED_PROGRESS_TTL_MS: u64 = 10_000; +#[derive(Debug, Clone, Serialize)] +pub struct PublishedProgressRoom { pub binding_digest: String, pub revision: u64 } +#[derive(Debug, Clone, Serialize)] +pub struct RecordedCondition { pub label: RoomLabel, pub train_recorded: usize, pub held_out_recorded: usize } +#[derive(Debug, Clone, Serialize)] +pub struct ProgressPhase { + pub role: DataRole, pub label: RoomLabel, pub stage: &'static str, + pub target_ms: u64, pub lead_in_remaining_ms: u64, pub capture_elapsed_ms: u64, + pub representative_cohorts: usize, pub required_cohorts: usize, +} +#[derive(Debug, Clone, Serialize)] +pub struct ProgressSources { pub required: usize, pub ready: usize } +#[derive(Debug, Clone, Serialize)] +pub struct ProgressQualification { pub state: &'static str, pub receipt_sha256: Option, pub model_sha256: Option } +#[derive(Debug, Clone, Serialize)] +pub struct ProgressSelection { pub state: &'static str, pub selection_sha256: Option, pub model_sha256: Option, pub current_usable: bool } +#[derive(Debug, Clone, Serialize)] +pub struct PairedCalibrationProgress { + pub schema: &'static str, pub success: bool, pub installation_id: String, + /// Existing public calibration process identity, never the owner mutation guard. + pub runtime_epoch: String, pub observed_at_unix_ms: u64, pub expires_at_unix_ms: u64, + pub room: Option, pub context_match: bool, + pub stage: &'static str, pub state: &'static str, + pub conditions: Option>, pub phase: Option, + pub sources: Option, pub qualification: ProgressQualification, + pub selection: ProgressSelection, pub save_status: &'static str, pub blocker: Option<&'static str>, +} +fn bounded_text(value: &str) -> bool { !value.is_empty() && value.encode_utf16().count() <= 128 && !value.chars().any(char::is_control) } +fn digest(value: &str) -> bool { value.len()==64 && value.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) } +impl RoomLearningRuntime { + /// Does not call owner status/checkpoint or advance workflow/model state. + /// Artifact readback reuses existing owner-private store enforcement. + /// Caller holds the published binding stable. + pub fn paired_progress_snapshot(&self, installation_id: &str, runtime_epoch: &str, + room: Option, externally_ready_sources: &[u8], ready_context_digest: Option<&str>, now_ns: u64, + ) -> Result { + let observed = now_ns / 1_000_000; + if !bounded_text(installation_id) || !bounded_text(runtime_epoch) || observed==0 || observed>MAX_SAFE-PAIRED_PROGRESS_TTL_MS + || room.as_ref().is_some_and(|r| !digest(&r.binding_digest) || r.revision==0 || r.revision>MAX_SAFE) + { return Err("progress identity/time unavailable".into()); } + let state=self.inner.lock().map_err(|_| "learner state unavailable")?; + let mut result=PairedCalibrationProgress { + schema:"ruview.room-learner.progress.v1",success:true,installation_id:installation_id.into(),runtime_epoch:runtime_epoch.into(), + observed_at_unix_ms:observed,expires_at_unix_ms:observed+PAIRED_PROGRESS_TTL_MS,room, + context_match:false,stage:"setup",state:"idle",conditions:None,phase:None,sources:None, + qualification:ProgressQualification{state:"not_reported",receipt_sha256:None,model_sha256:None}, + selection:ProgressSelection{state:"not_reported",selection_sha256:None,model_sha256:None,current_usable:false}, + save_status:"not_reported",blocker:Some("room_not_published"), + }; + let Some(published)=result.room.as_ref() else { return Ok(result); }; + let Some(capture)=state.capture.as_ref() else { result.blocker=Some("learner_not_configured");return Ok(result); }; + let context=capture.context(); + if context.installation_id!=installation_id || context.room_digest!=published.binding_digest { + result.stage="blocked";result.blocker=Some("context_mismatch");return Ok(result); + } + if ready_context_digest!=Some(context.digest().as_str()) {return Err("progress context changed".into());} + if context.sources.is_empty() || context.sources.len()>8 {return Err("invalid progress source count".into());} + if state.model.as_ref().is_some_and(|m|m.context()!=context) {return Err("invalid progress candidate".into());} + let candidate_sha=state.model.as_ref().map(|m|m.export().map(|bytes|{ + use sha2::{Digest,Sha256};format!("{:x}",Sha256::digest(bytes)) + })).transpose().map_err(|_|"invalid progress candidate")?; + result.context_match=true; + result.state=match state.job.phase {JobPhase::Idle=>"idle",JobPhase::Running=>"running",JobPhase::Cancelling=>"cancelling",JobPhase::Ready=>"ready",JobPhase::Failed=>"failed",JobPhase::Cancelled=>"cancelled"}; + let labels=[RoomLabel::Empty,RoomLabel::OccupiedStatic,RoomLabel::OccupiedMoving,RoomLabel::AdjacentOnly]; + let conditions:Vec<_>=labels.into_iter().map(|label| { + let count=|role| capture.history().iter().filter(|p| p.status==PhaseStatus::Sealed && p.role==role && p.label==label) + .map(|p|p.cohort_count).sum::().min(128); + RecordedCondition{label,train_recorded:count(DataRole::Train),held_out_recorded:count(DataRole::HeldOut)} + }).collect(); + let train_complete=conditions.iter().all(|c|c.train_recorded>=4); + let heldout_complete=conditions.iter().all(|c|c.held_out_recorded>=4); + result.conditions=Some(conditions); + let progress=capture.progress(now_ns); + result.sources=Some(ProgressSources{required:context.sources.len(),ready:progress.sources.iter().filter(|s| s.window_eligible && externally_ready_sources.contains(&s.source_id)).count()}); + result.phase=progress.phase.map(|p| ProgressPhase { + role:p.role,label:p.label, + stage:if p.lead_in_remaining_ms>0 {"lead_in"} else if p.capture_elapsed_ms Result<(),String> { + let stored=self.store.load_selection()?; + if stored.as_ref().map_or(0,|e|e.receipt.revision)!=snapshot.revision {return Err("selection changed".into());} + match (stored.as_ref().and_then(|e|e.receipt.selected.as_ref()),snapshot.selected.as_ref()) { + (Some(target),Some(identity))=>{ + if stored.as_ref().map(|e|e.receipt_sha256.as_str())!=Some(identity.selection_sha256.as_str()) + || target.model_sha256!=identity.model_sha256 || target.qualification_sha256!=identity.qualification_sha256 {return Err("selection changed".into());} + let (model,q)=self.store.resolve_selection_target(target)?;self.store.current_qualification_assessment(&model,&q)?; + }, + (None,None)=>{},_=>return Err("selection changed".into()), + } + Ok(()) + })(); + result.qualification.state=if candidate_sha.is_some() {"not_qualified"} else {"not_reported"}; + result.qualification.model_sha256=candidate_sha; + result.selection.state="not_selected"; + if snapshot.qualification_error.is_some() { result.qualification.state="failed"; } + else if let (Some(q),Some(model))=(state.qualification.as_ref(),state.model.as_ref()) { + let loaded=self.store.load_qualification(model,&q.receipt_sha256).and_then(|loaded| { + if loaded.receipt.room_digest!=published.binding_digest {return Err("qualification context changed".into());} + self.store.current_qualification_assessment(model,&loaded)?;Ok(loaded) + }); + match loaded { + Ok(loaded)=>result.qualification=ProgressQualification{state:"qualified",receipt_sha256:Some(loaded.receipt_sha256),model_sha256:Some(loaded.receipt.model_sha256)}, + Err(_)=>result.qualification.state="failed", + } + } + project_selection(&mut result,&snapshot,verified.is_ok(),&context.digest()); + let source_unready=result.sources.as_ref().is_none_or(|s|s.ready!=s.required); + if source_unready {result.selection.current_usable=false;} + if let Some(phase)=&result.phase { + result.stage=if phase.role==DataRole::Train {"train"} else {"held_out"}; + result.blocker=if phase.stage=="incomplete" {Some("phase_incomplete")} else if source_unready {Some("sources_unready")} else {None}; + } else if busy(&state.job) {result.stage="training";result.blocker=None;} + else if state.job.phase==JobPhase::Failed {result.stage="blocked";result.blocker=Some("training_failed");} + else if result.qualification.state=="failed" {result.stage="blocked";result.blocker=Some("qualification_failed");} + else if result.selection.state=="unavailable" {result.stage="blocked";result.blocker=Some("selection_unavailable");} + else if result.qualification.state=="qualified" && result.qualification.model_sha256!=result.selection.model_sha256 {result.stage="selection";result.blocker=Some("selection_required");} + else if result.selection.state=="selected" {result.stage="selected";result.blocker=if result.selection.current_usable {None} else if source_unready {Some("sources_unready")} else {Some("selection_unavailable")};} + else if result.qualification.state=="qualified" {result.stage="selection";result.blocker=Some("selection_required");} + // Zero temporary rows do not invalidate a durable managed proof. This + // base projection does not report that proof's readiness or counts. + else if state.model.is_some() && !train_complete {result.stage=if heldout_complete {"qualification"} else {"held_out"};result.blocker=Some("training_lineage_not_reported");} + else if state.model.is_some() {result.stage=if heldout_complete {"qualification"} else {"held_out"};result.blocker=if source_unready {Some("sources_unready")} else if heldout_complete {Some("qualification_required")} else {None};} + else {result.stage="train";result.blocker=if source_unready {Some("sources_unready")} else {None};} + Ok(result) + } +} +fn project_selection(result:&mut PairedCalibrationProgress,snapshot:&ModelSelectionStatus,verified:bool,context_digest:&str) { + if let Some(selected)=&snapshot.selected { + if Some(selected.room_digest.as_str())!=result.room.as_ref().map(|r|r.binding_digest.as_str()) || selected.context_digest!=context_digest { + result.selection.state="not_reported";return; + } + } + if !verified || snapshot.integrity_error.is_some() {result.selection.state="unavailable";return;} + if let Some(selected)=&snapshot.selected { + result.selection=ProgressSelection {state:"selected",selection_sha256:Some(selected.selection_sha256.clone()),model_sha256:Some(selected.model_sha256.clone()), + current_usable:matches!(snapshot.current,SelectedBinaryEvidence::Ready{..})}; + } +} + +#[cfg(test)] +#[path = "paired_progress_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/paired_progress_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/paired_progress_tests.rs new file mode 100644 index 0000000000..28aabd8ebc --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/paired_progress_tests.rs @@ -0,0 +1,131 @@ +//! SYNTHETIC current/read-only progress, not physical qualification. +use super::*; +use super::super::tests::{context,runtime,phase,train_phases,finished,START}; +use crate::room_learner::recovery_store::RecoveryStore; +fn binding()->String {"a".repeat(64)} +fn publication()->Option{Some(PublishedProgressRoom{binding_digest:binding(),revision:1})} +fn configured(root:&Path)->Arc{ + let rt=runtime(root);let mut ctx=context();ctx.room_digest=binding(); + rt.configure(&rt.mutation_guard().unwrap(),ctx,"paired-synthetic-run".into(),true).unwrap();rt +} +fn snapshot(rt:&RoomLearningRuntime,now:u64)->PairedCalibrationProgress { + rt.paired_progress_snapshot("synthetic-install","cal-boot-synthetic",publication(),&[1,2],rt.context().as_ref().map(|c|c.digest()).as_deref(),now).unwrap() +} +fn assert_private_free(value:&PairedCalibrationProgress) { + let s=serde_json::to_string(value).unwrap(); + for forbidden in ["owner_scope","mutation_guard","trial_id","session_id","frame_ids","sequence_ids","last_accepted_sequence","source_id","encoder_weights","head_weights","occupied_probability","heart_rate","amplitudes","\"samples\"","training_data_sha256","\"geometry\""] { + assert!(!s.contains(forbidden),"paired progress leaked {forbidden}"); + } + assert!(!s.contains("NaN")&&!s.contains("Infinity")); +} +#[test] +fn setup_and_foreign_publication_withhold_current_counts_and_private_identity() { + let dir=tempfile::tempdir().unwrap();let rt=configured(dir.path()); + let missing=rt.paired_progress_snapshot("synthetic-install","cal-boot-synthetic",None,&[1,2],None,START).unwrap(); + assert_eq!(missing.blocker,Some("room_not_published"));assert!(missing.conditions.is_none()); + let mut other=publication().unwrap();other.binding_digest="b".repeat(64); + let mismatch=rt.paired_progress_snapshot("synthetic-install","cal-boot-synthetic",Some(other),&[1,2],None,START).unwrap(); + assert_eq!(mismatch.blocker,Some("context_mismatch"));assert!(!mismatch.context_match); + assert!(mismatch.conditions.is_none()&&mismatch.phase.is_none()&&mismatch.sources.is_none()); + assert_eq!(mismatch.qualification.state,"not_reported");assert_eq!(mismatch.selection.state,"not_reported"); + let foreign=rt.paired_progress_snapshot("other-install","cal-boot-synthetic",publication(),&[1,2],None,START).unwrap(); + assert_eq!(foreign.blocker,Some("context_mismatch"));assert_private_free(&foreign); +} +#[test] +fn phases_are_bounded_and_recorded_counts_are_separate_from_qualification_and_save() { + let dir=tempfile::tempdir().unwrap();let rt=configured(dir.path()); + phase(&rt,DataRole::Train,RoomLabel::Empty,0); + let before=snapshot(&rt,START+12_000_000_000); + assert_eq!(before.conditions.as_ref().unwrap()[0].train_recorded,4); + assert_eq!(before.qualification.state,"not_reported");assert_eq!(before.save_status,"not_reported"); + rt.begin_phase(&rt.mutation_guard().unwrap(),PhaseRequest {trial_id:"private-trial-name".into(),session_id:"private-session-name".into(),role:DataRole::Train,label:RoomLabel::OccupiedStatic,duration_ms:12_000,lead_in_ms:5000,conditions_confirmed:true},START+13_000_000_000).unwrap(); + let guard=rt.mutation_guard().unwrap();let preparing=snapshot(&rt,START+14_000_000_000); + assert_eq!(preparing.phase.as_ref().unwrap().stage,"lead_in");assert_eq!(preparing.phase.as_ref().unwrap().lead_in_remaining_ms,4000); + assert_eq!(preparing.observed_at_unix_ms+10_000,preparing.expires_at_unix_ms); + let incomplete=snapshot(&rt,START+100_000_000_000); + let p=incomplete.phase.as_ref().unwrap();assert_eq!(p.stage,"incomplete");assert_eq!(p.capture_elapsed_ms,p.target_ms);assert_eq!(p.representative_cohorts,0); + assert_eq!(guard,rt.mutation_guard().unwrap(),"GET projection must not consume workflow revisions"); + let s=serde_json::to_string(&incomplete).unwrap();assert!(!s.contains("private-trial-name")&&!s.contains("private-session-name"));assert_private_free(&incomplete); +} +#[test] +fn progress_does_not_checkpoint_or_advance_expired_recovery() { + let dir=tempfile::tempdir().unwrap();let owner="a".repeat(64);let data=dir.path().join(&owner).join("SensingServer"); + let store=RecoveryStore::new(&data,&owner,"synthetic-install",Some(&[7;32])).unwrap(); + let rt=RoomLearningRuntime::new_with_recovery_at(&data,&owner,"synthetic-install",store,START).unwrap(); + let mut ctx=context();ctx.room_digest=binding();rt.configure_for_recovery(&rt.mutation_guard().unwrap(),ctx,"private-recovery-run".into(),true,true,START).unwrap(); + let file=data.join("room-learner-recovery/capture-recovery-v1.bin");let bytes=std::fs::read(&file).unwrap(); + let guard=rt.mutation_guard().unwrap(); + let view=snapshot(&rt,START+86_400_000_000_000); + assert_eq!(view.save_status,"not_reported");assert_eq!(rt.mutation_guard().unwrap(),guard); + assert!(rt.inner.lock().unwrap().recovery.enabled); + assert_eq!(std::fs::read(file).unwrap(),bytes); +} +#[test] +fn invalid_bounds_times_and_nonfinite_candidate_are_unavailable_without_raw_values() { + let dir=tempfile::tempdir().unwrap();let rt=configured(dir.path()); + assert!(rt.paired_progress_snapshot("synthetic-install","public\ninvalid",publication(),&[],None,START).is_err()); + assert!(rt.paired_progress_snapshot("synthetic-install","cal-boot-synthetic",publication(),&[],None,0).is_err()); + assert!(rt.paired_progress_snapshot("synthetic-install","cal-boot-synthetic",Some(PublishedProgressRoom{binding_digest:binding(),revision:MAX_SAFE+1}),&[],None,START).is_err()); + assert!(rt.paired_progress_snapshot("synthetic-install","cal-boot-synthetic",Some(PublishedProgressRoom{binding_digest:binding(),revision:0}),&[],None,START).is_err()); + assert!(rt.paired_progress_snapshot("synthetic-install","cal-boot-synthetic",publication(),&[1,2],Some(&"b".repeat(64)),START).is_err(),"ready-source class from a prior context must not supply current readiness"); + assert_private_free(&snapshot(&rt,START)); +} +async fn fitted(root:&Path)->Arc{ + let rt=configured(root);train_phases(&rt); + rt.start_training(&rt.mutation_guard().unwrap(),FitConfig{epochs:8,learning_rate:0.5,hidden_dim:2,embedding_dim:2,temporal_tokens:2,..FitConfig::default()}).unwrap(); + assert_eq!(finished(&rt).await["state"],"ready"); + for (i,l) in [RoomLabel::Empty,RoomLabel::OccupiedStatic,RoomLabel::OccupiedMoving,RoomLabel::AdjacentOnly].into_iter().enumerate(){phase(&rt,DataRole::HeldOut,l,i as u32+4);} + rt +} +#[tokio::test] +async fn verified_selection_is_not_mac_save_and_corrupt_artifacts_withhold_claims() { + let dir=tempfile::tempdir().unwrap();let rt=fitted(dir.path()).await; + let recorded=snapshot(&rt,START+104_000_000_000);assert!(recorded.conditions.as_ref().unwrap().iter().all(|c|c.train_recorded==4&&c.held_out_recorded==4));assert_eq!(recorded.qualification.state,"not_qualified"); + let q=rt.qualify(&rt.mutation_guard().unwrap()).unwrap();let qualified=snapshot(&rt,START+104_000_000_000); + assert_eq!(qualified.qualification.state,"qualified");assert_eq!(qualified.stage,"selection"); + rt.select_model(&rt.mutation_guard().unwrap(),&q.receipt.model_sha256,&q.receipt_sha256).unwrap(); + let now=super::super::selection_tests::feed(&rt,START+105_000_000_000,&[1,2],6.0); + let selected=snapshot(&rt,now);assert_eq!(selected.selection.state,"selected");assert!(selected.selection.current_usable);assert_eq!(selected.save_status,"not_reported");assert_private_free(&selected); + let context=rt.context().unwrap().digest();let unready=rt.paired_progress_snapshot("synthetic-install","cal-boot-synthetic",publication(),&[],Some(&context),now).unwrap(); + assert_eq!(unready.selection.state,"selected");assert!(!unready.selection.current_usable);assert_eq!(unready.blocker,Some("sources_unready")); + let path=dir.path().join("a".repeat(64)).join("SensingServer/room-learner").join(rt.context().unwrap().digest()).join("qualifications").join(format!("{}.json",q.receipt_sha256)); + let original=std::fs::read(&path).unwrap(); + std::fs::write(&path,b"{\"synthetic_corruption\":true}").unwrap(); + let corrupt=snapshot(&rt,now);assert_ne!(corrupt.qualification.state,"qualified");assert_eq!(corrupt.selection.state,"unavailable");assert!(!corrupt.selection.current_usable);assert_private_free(&corrupt); + #[cfg(unix)] { + let outside=dir.path().join("synthetic-other-file.json");std::fs::write(&outside,original).unwrap();std::fs::remove_file(&path).unwrap();std::os::unix::fs::symlink(outside,&path).unwrap(); + let alias=snapshot(&rt,now);assert_ne!(alias.qualification.state,"qualified");assert_eq!(alias.selection.state,"unavailable");assert!(!alias.selection.current_usable); + } +} +#[tokio::test] +async fn nonfinite_model_and_foreign_owner_witness_do_not_grant_current_progress() { + let dir=tempfile::tempdir().unwrap();let rt=fitted(dir.path()).await; + {let mut state=rt.inner.lock().unwrap();let model=Arc::make_mut(state.model.as_mut().unwrap());model.encoder_weights[0]=f32::NAN;} + assert!(rt.paired_progress_snapshot("synthetic-install","cal-boot-synthetic",publication(),&[1,2],rt.context().as_ref().map(|c|c.digest()).as_deref(),START+104_000_000_000).is_err()); + let dir2=tempfile::tempdir().unwrap();let good=fitted(dir2.path()).await;let q=good.qualify(&good.mutation_guard().unwrap()).unwrap();good.select_model(&good.mutation_guard().unwrap(),&q.receipt.model_sha256,&q.receipt_sha256).unwrap(); + std::fs::write(dir2.path().join("a".repeat(64)).join("SensingServer/account-owner-v1.json"),br#"{"schema":"ruview.account-owner.v1","scope_key":"foreign"}"#).unwrap(); + let denied=snapshot(&good,START+104_000_000_000);assert_eq!(denied.selection.state,"unavailable");assert!(!denied.selection.current_usable); +} + +#[tokio::test] +async fn restored_model_with_no_temporary_rows_is_not_reported_as_lost_or_invalid() { + let dir=tempfile::tempdir().unwrap();let rt=fitted(dir.path()).await;let ctx=rt.context().unwrap(); + rt.configure(&rt.mutation_guard().unwrap(),ctx,"new-progress-session".into(),false).unwrap(); + let progress=snapshot(&rt,START+104_000_000_000); + assert_eq!(progress.state,"ready");assert_eq!(progress.stage,"held_out"); + assert_eq!(progress.blocker,Some("training_lineage_not_reported")); + assert!(progress.conditions.as_ref().unwrap().iter().all(|c|c.train_recorded==0&&c.held_out_recorded==0)); + assert_ne!(progress.stage,"blocked");assert_eq!(progress.save_status,"not_reported"); +} + +#[tokio::test] +async fn another_rooms_selected_model_does_not_block_or_label_current_room_setup() { + let dir=tempfile::tempdir().unwrap();let rt=fitted(dir.path()).await; + let q=rt.qualify(&rt.mutation_guard().unwrap()).unwrap();rt.select_model(&rt.mutation_guard().unwrap(),&q.receipt.model_sha256,&q.receipt_sha256).unwrap(); + let mut next=rt.context().unwrap();next.room_digest="b".repeat(64);rt.configure(&rt.mutation_guard().unwrap(),next,"current-room-b".into(),true).unwrap(); + let context=rt.context().unwrap().digest(); + let view=rt.paired_progress_snapshot("synthetic-install","cal-boot-synthetic",Some(PublishedProgressRoom{binding_digest:"b".repeat(64),revision:2}),&[],Some(&context),START+104_000_000_000).unwrap(); + assert!(view.context_match);assert_eq!(view.stage,"train");assert_eq!(view.selection.state,"not_reported");assert!(view.selection.model_sha256.is_none()); + assert_eq!(view.qualification.state,"not_reported");assert!(view.qualification.model_sha256.is_none()); + assert!(view.conditions.as_ref().unwrap().iter().all(|c|c.train_recorded==0&&c.held_out_recorded==0));assert_private_free(&view); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs index 9698e563be..49716e24dc 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs @@ -451,3 +451,7 @@ pub(super) use publication::{project_cached_selection, project_live_selection, r #[path = "room_learner_api_selection.rs"] mod selection_routes; pub(super) use selection_routes::{owner_selection_routes, selection}; + +#[path = "room_learner_api_progress.rs"] +mod paired_progress; +pub(super) use paired_progress::progress; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_progress.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_progress.rs new file mode 100644 index 0000000000..eabc6b8bf9 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_progress.rs @@ -0,0 +1,40 @@ +//! Pair-read calibration progress only; never expose the owner's status object. +use super::*; +use axum::response::{IntoResponse, Response}; +use wifi_densepose_sensing_server::room_learner::lifecycle::PublishedProgressRoom; +const SCHEMA:&str="ruview.room-learner.progress.v1"; +fn denied(status:StatusCode,code:&'static str)->Response { + (status,Json(serde_json::json!({"schema":SCHEMA,"success":false,"error_code":code}))).into_response() +} +pub(crate) async fn progress(State(state):State, grant:Option>)->Response { + let state=state.read().await; + if state.privacy_mode {return denied(StatusCode::FORBIDDEN,"privacy_redacted");} + let Some(runtime)=state.room_learning.as_ref() else {return denied(StatusCode::SERVICE_UNAVAILABLE,"learner_unavailable");}; + let Some(store)=state.current_room_store.as_ref() else {return denied(StatusCode::SERVICE_UNAVAILABLE,"publication_unavailable");}; + let Some(publication)=store.snapshot() else {return denied(StatusCode::SERVICE_UNAVAILABLE,"publication_unavailable");}; + if state.installation_id.as_deref()!=Some(publication.installation_id.as_str()) {return denied(StatusCode::SERVICE_UNAVAILABLE,"publication_unavailable");} + let identity=publication.binding_digest.as_ref().map(|binding|PublishedProgressRoom{binding_digest:binding.clone(),revision:publication.revision}); + // The binding hashes installation, published revision and complete room. + // Hold its lock through projection; reject a publication changed since the + // initial envelope snapshot instead of mixing its identity with old counts. + match store.with_current_binding(|binding| { + if binding!=publication.binding_digest.as_deref() {return Err("publication_changed");} + let context=runtime.context(); + let ready:Vec<_>=context.as_ref().map(|c|c.sources.iter().filter(|s|source_ready(&state,s,std::time::Instant::now())).map(|s|s.source_id).collect()).unwrap_or_default(); + let ready_context=context.as_ref().map(|c|c.digest()); + runtime.paired_progress_snapshot(&publication.installation_id,&state.calibration_boot_epoch,identity,&ready,ready_context.as_deref(),runtime.receiver_now_ns()) + .map_err(|_|"progress_unavailable") + }) { + Ok(Ok(mut snapshot))=>{ + if let Some(axum::Extension(grant))=grant { + let Some(until)=grant.expires_at.checked_mul(1000) else {return StatusCode::UNAUTHORIZED.into_response();}; + let now=chrono::Utc::now().timestamp_millis().max(0) as u64; + if *grant.revoked.borrow() || until<=snapshot.observed_at_unix_ms || until<=now {return StatusCode::UNAUTHORIZED.into_response();} + snapshot.expires_at_unix_ms=snapshot.expires_at_unix_ms.min(until); + } + Json(snapshot).into_response() + }, + Ok(Err("publication_changed"))=>denied(StatusCode::CONFLICT,"publication_changed"), + _=>denied(StatusCode::SERVICE_UNAVAILABLE,"progress_unavailable"), + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs index 49f41cd2e9..21be2357be 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs @@ -143,6 +143,7 @@ pub(crate) async fn selection(State(state): State) -> Json Date: Wed, 4 Nov 2026 20:03:04 +0100 Subject: [PATCH 063/100] fix(calibration): preflight every bound legacy CSI grid --- .../ADR-110-esp32-c6-firmware-extension.md | 53 ++ .../src/legacy_grid_readiness.rs | 179 ++++++ .../src/legacy_grid_readiness_tests.rs | 539 ++++++++++++++++++ .../wifi-densepose-sensing-server/src/main.rs | 46 +- 4 files changed, 810 insertions(+), 7 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/legacy_grid_readiness.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/legacy_grid_readiness_tests.rs diff --git a/docs/adr/ADR-110-esp32-c6-firmware-extension.md b/docs/adr/ADR-110-esp32-c6-firmware-extension.md index 3905f167b2..03cd3f53f6 100644 --- a/docs/adr/ADR-110-esp32-c6-firmware-extension.md +++ b/docs/adr/ADR-110-esp32-c6-firmware-extension.md @@ -4,6 +4,7 @@ |-------|-------| | **Status** | Accepted — P1–P10 complete, firmware-side substrate closed at **v0.7.0-esp32** (2026-05-23) | | **Date** | 2026-05-22 (created) · 2026-05-23 (last revision — P10 + sprint summary) | +| **Updated** | 2026-11-04 — legacy calibration raw-grid compatibility boundary | | **Deciders** | ruv | | **Codename** | **C6-SOTA** | | **Relates to** | ADR-018 (CSI binary frame format), ADR-028 (ESP32 capability audit), ADR-029 (RuvSense multistatic), ADR-030 (RuvSense persistent field model), ADR-031 (RuView sensing-first), ADR-061 (QEMU CI), ADR-081 (adaptive CSI mesh kernel), ADR-097 (rvCSI adoption) | @@ -209,3 +210,55 @@ The firmware-side substrate for ADR-110 is now closed. Three categories remain, 3. **IDF upstream fixes**: - 802.15.4 RX path on C6 + IDF v5.4 — `c6_timesync` ships and initialises but never RXes a frame (D1, 5 hypotheses tested + rejected). ESP-NOW workaround (`c6_sync_espnow`) is the working primary mesh transport. The 802.15.4 source stays in for the day IDF fixes the driver. - Soft-AP HE/TWT-Responder advertise API — `c6_softap_he` ships as the in-place hook for when IDF v5.5+ exposes it. + + +## November 4, 2026: legacy calibration compatibility boundary + +The accepted firmware substrate above does not imply that HT-LTF and HE-LTF +raw vectors can be combined in one legacy FieldModel. That baseline has exactly +one author, one exact `(n_subcarriers, ppdu_type)` grid and one set of amplitude +offsets. Other placed nodes remain in the full room binding and must contribute +on that grid; they do not author or average their offsets into the model. +Equal bin counts with different PPDU types are not an interchangeable grid. + +Before any bootstrap/model/session/receipt mutation, legacy calibration start +checks every bound node's current raw-header evidence for the selected author's +exact grid. It searches **all** eligible grids per node, not only that node's +preferred or densest grid. Thus a C6 with a sufficiently sampled HT minority can +participate in an HT baseline authored by another node. Author selection itself +is unchanged; this does not automatically select a lower-density common grid. +A qualified nonmatching stream is refused before spending a room-capture window. +Missing, warming, weak or stale evidence is described separately and does not +become a hardware compatibility claim inferred from a user-set board label. + +The existing protected start/status responses gain optional bounded +`source_grid_readiness` diagnostics: the selected tuple and author, only the +bound node IDs, at most eight reported observed/eligible grids per node, +truncation flags and stable reasons. Reported eligible grids are a subset of +reported observations, and an observed selected tuple remains visible even +under truncation. The field is null without a valid active binding. No raw CSI, +RF values, device addresses, unbound inventory or new endpoint/grant is exposed. +Older clients may ignore this additive diagnostic. + +Current status recomputes these diagnostics without changing historical +contribution membership. For a **new unfinished** finalization only, a bound node +that now supplies a qualified different grid and lacks a current matching stream +blocks the new receipt while retaining the unfinished model and history. This +is not an additional blanket ten-second readiness gate on stop: unknown, +warming or stale diagnostics do not prove a hardware change, and existing +author freshness, contribution and sequence guards remain. Completed historical +receipts/metadata do not acquire this new finalization restriction. + +This rule applies only to the legacy single-author baseline. The separate +managed learner in Proposed ADR-420 preserves each source's own supported PHY/ +grid and projection; it must not inherit a global equality rule. Accepted +ADR-151's statistical specialist stages and Proposed ADR-008's distributed +coordination are not a claim that the legacy FieldModel is a joint heterogeneous +learner or that mixed-source sensing is physically qualified. + +The compatibility amendment is verified with synthetic raw-header fixtures +through the actual server start/status/stop handlers, including pre-mutation +preservation, PPDU mismatch, a shared minority grid, missing/warming/stale +sources, truncation/unbound exclusion and post-start drift. These software +results do not replace physical C5/C6/S3 or sensing-accuracy acceptance. No +firmware, production minimum duration, frame target or live node was changed. diff --git a/v2/crates/wifi-densepose-sensing-server/src/legacy_grid_readiness.rs b/v2/crates/wifi-densepose-sensing-server/src/legacy_grid_readiness.rs new file mode 100644 index 0000000000..5e55ebf155 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/legacy_grid_readiness.rs @@ -0,0 +1,179 @@ +//! Header-only admission for the legacy ONE-author exact-grid baseline. +//! This is not managed per-source learning or a hardware/accuracy claim. +use super::*; +use std::time::Instant; +const MAX_REPORTED_GRIDS: usize = 8; +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub(super) enum GridReadinessReason { + Compatible, + NodeMissing, + RawGridMissing, + GridEvidenceWarming, + GridEvidenceStale, + NonmatchingQualifiedGrid, + GridEvidenceUnqualified, +} +#[derive(Debug, Serialize)] +pub(super) struct BoundNodeGridReadiness { + pub node_id: u8, + pub ready: bool, + pub reason: GridReadinessReason, + pub observed_grids: Vec, + pub eligible_grids: Vec, + pub observed_grids_truncated: bool, + pub eligible_grids_truncated: bool, +} +#[derive(Debug, Serialize)] +pub(super) struct LegacyGridReadiness { + pub schema: &'static str, + pub author_source_node_id: u8, + pub selected_grid: CsiGridKey, + pub all_bound_nodes_ready: bool, + pub nodes: Vec, +} +/// Shared with the existing author selection: always inspect every candidate, +/// never infer compatibility from a node's preferred grid or board label. +pub(super) fn eligible_grid(grid: CsiGridKey, evidence: CalibrationGridEvidence) -> bool { + grid.n_subcarriers > 0 + && grid.ppdu_type <= 3 + && evidence.span_s >= CALIBRATION_GRID_MIN_SPAN_S + && evidence.rate_hz >= CALIBRATION_GRID_MIN_RATE_HZ + && evidence.max_gap_s < CALIBRATION_GRID_MAX_GAP_S + && evidence.latest_age_s < CALIBRATION_GRID_MAX_GAP_S +} +fn sorted_grids(mut values: Vec, selected: CsiGridKey) -> Vec { + values.retain(|g| g.n_subcarriers > 0); + values.sort_by_key(|g| { + ( + *g != selected, + std::cmp::Reverse(g.n_subcarriers), + g.ppdu_type, + ) + }); + values.dedup(); + values +} +fn node_readiness( + id: u8, + node: Option<&NodeState>, + selected: CsiGridKey, + now: Instant, +) -> BoundNodeGridReadiness { + let Some(node) = node else { + return BoundNodeGridReadiness { + node_id: id, + ready: false, + reason: GridReadinessReason::NodeMissing, + observed_grids: vec![], + eligible_grids: vec![], + observed_grids_truncated: false, + eligible_grids_truncated: false, + }; + }; + let candidates = node.calibration_grid_candidates(now); + let matching = candidates + .iter() + .find(|(g, _)| *g == selected) + .map(|(_, e)| *e); + let ready = matching.is_some_and(|e| eligible_grid(selected, e)); + let other_qualified = candidates + .iter() + .any(|(g, e)| *g != selected && eligible_grid(*g, *e)); + // A fresh but still warming/weak matching stream is not measured proof of a + // hardware change, even when a different grid is currently more frequent. + let reason = if ready { + GridReadinessReason::Compatible + } else if let Some(e) = matching.filter(|e| e.latest_age_s < CALIBRATION_GRID_MAX_GAP_S) { + if e.span_s < CALIBRATION_GRID_MIN_SPAN_S { + GridReadinessReason::GridEvidenceWarming + } else { + GridReadinessReason::GridEvidenceUnqualified + } + } else if other_qualified { + GridReadinessReason::NonmatchingQualifiedGrid + } else if node.raw_grid_observations.is_empty() { + GridReadinessReason::RawGridMissing + } else if candidates.is_empty() + || candidates + .iter() + .all(|(_, e)| e.latest_age_s >= CALIBRATION_GRID_MAX_GAP_S) + { + GridReadinessReason::GridEvidenceStale + } else if candidates.iter().any(|(g, e)| { + g.ppdu_type <= 3 && g.n_subcarriers > 0 && e.span_s < CALIBRATION_GRID_MIN_SPAN_S + }) { + GridReadinessReason::GridEvidenceWarming + } else { + GridReadinessReason::GridEvidenceUnqualified + }; + let all_observed = sorted_grids(candidates.iter().map(|(g, _)| *g).collect(), selected); + let all_eligible = sorted_grids( + candidates + .iter() + .filter(|(g, e)| eligible_grid(*g, *e)) + .map(|(g, _)| *g) + .collect(), + selected, + ); + // Joint projection preserves subset semantics under the cap. Reserve the + // selected observed-but-unqualified tuple rather than hiding why it blocks. + let selected_observed = all_observed.contains(&selected); + let reserve = usize::from(selected_observed && !all_eligible.contains(&selected)); + let eligible_grids: Vec<_> = all_eligible + .iter() + .copied() + .take(MAX_REPORTED_GRIDS - reserve) + .collect(); + let mut observed_grids = Vec::new(); + if selected_observed { + observed_grids.push(selected); + } + for g in eligible_grids.iter().chain(&all_observed) { + if observed_grids.len() == MAX_REPORTED_GRIDS { + break; + } + if !observed_grids.contains(g) { + observed_grids.push(*g); + } + } + let observed_grids_truncated = all_observed.len() > observed_grids.len(); + let eligible_grids_truncated = all_eligible.len() > eligible_grids.len(); + BoundNodeGridReadiness { + node_id: id, + ready, + reason, + observed_grids, + eligible_grids, + observed_grids_truncated, + eligible_grids_truncated, + } +} +impl LegacyGridReadiness { + pub(super) fn known_incompatibility(&self) -> bool { + self.nodes + .iter() + .any(|n| n.reason == GridReadinessReason::NonmatchingQualifiedGrid) + } +} +pub(super) fn inspect( + state: &AppStateInner, + ids: &[u8], + binding: CalibrationGridBinding, + now: Instant, +) -> Option { + if !valid_source_node_ids(ids) || ids.binary_search(&binding.source_node_id).is_err() { + return None; + } + let nodes: Vec<_> = ids + .iter() + .map(|id| node_readiness(*id, state.node_states.get(id), binding.grid, now)) + .collect(); + Some(LegacyGridReadiness { + schema: "ruview.legacy-calibration-grid-readiness.v1", + author_source_node_id: binding.source_node_id, + selected_grid: binding.grid, + all_bound_nodes_ready: nodes.iter().all(|n| n.ready), + nodes, + }) +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/legacy_grid_readiness_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/legacy_grid_readiness_tests.rs new file mode 100644 index 0000000000..16065c2a98 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/legacy_grid_readiness_tests.rs @@ -0,0 +1,539 @@ +//! SYNTHETIC header-grid fixtures through actual legacy calibration handlers. +use super::*; +use std::time::{Duration, Instant}; +fn grid(bins: u16, ppdu: u8) -> CsiGridKey { + CsiGridKey { + n_subcarriers: bins, + ppdu_type: ppdu, + } +} +fn ready_node(grids: &[CsiGridKey], now: Instant) -> NodeState { + let mut node = NodeState::new(); + for index in 0..=48 { + for grid in grids { + node.observe_raw_grid( + *grid, + now - Duration::from_secs(12) + Duration::from_millis(index * 250), + ); + } + } + node +} +fn request(ids: Vec) -> CalibrationBindingRequest { + CalibrationBindingRequest { + binding_digest: "a".repeat(64), + source_node_ids: ids, + } +} +#[tokio::test] +async fn mixed_he256_ht64_is_rejected_before_existing_baseline_or_receipt_mutation() { + let now = Instant::now(); + let mut inner = AppStateInner::minimal(); + inner + .node_states + .insert(3, ready_node(&[grid(256, 1)], now)); + inner.node_states.insert(5, ready_node(&[grid(64, 0)], now)); + inner + .node_states + .insert(99, ready_node(&[grid(192, 3)], now)); + inner.field_model = Some(FieldModel::new(field_bridge::single_link_config()).unwrap()); + inner.bootstrap_baseline_active = true; + inner.calibration_model_id = Some("old-model-must-stay".into()); + inner.calibration_session_id = Some("old-session-must-stay".into()); + inner.calibration_binding_digest = Some("b".repeat(64)); + inner.calibration_source_node_ids.extend([3, 5]); + inner.calibration_observed_source_node_ids.insert(3); + inner.calibration_model_receipt = Some(CalibrationModelReceipt { + schema: "ruview.room-calibration.model-receipt.v1", + boot_epoch: inner.calibration_boot_epoch.clone(), + session_id: "old-session-must-stay".into(), + model_id: "old-model-must-stay".into(), + binding_digest: "b".repeat(64), + source_node_ids: vec![3, 5], + frame_count: 12345, + variance_explained: 0.95, + baseline_eigenvalue_count: 2, + completed_at_unix_ms: 1793800000000, + }); + let before = serde_json::to_value(inner.calibration_model_receipt.clone()).unwrap(); + let state = Arc::new(RwLock::new(inner)); + let Json(result) = calibration_start( + State(state.clone()), + Query(CalibrationStartQuery { + source_node_id: Some(3), + }), + Json(request(vec![3, 5])), + ) + .await; + assert_eq!( + result["success"], false, + "heterogeneous raw grids must fail before the ten-minute run: {result}" + ); + assert_eq!(result["error_code"], "calibration_bound_grid_unavailable"); + let preserved = state.read().await; + assert!(preserved.field_model.is_some()); + assert!(preserved.bootstrap_baseline_active); + assert_eq!( + preserved.calibration_model_id.as_deref(), + Some("old-model-must-stay") + ); + assert_eq!( + preserved.calibration_session_id.as_deref(), + Some("old-session-must-stay") + ); + assert_eq!( + preserved.calibration_binding_digest.as_deref(), + Some("b".repeat(64).as_str()) + ); + assert_eq!( + preserved.calibration_observed_source_node_ids, + [3].into_iter().collect() + ); + assert_eq!( + serde_json::to_value(&preserved.calibration_model_receipt).unwrap(), + before + ); +} +async fn start_nodes( + nodes: Vec<(u8, NodeState)>, + ids: Vec, + author: u8, +) -> (SharedState, serde_json::Value) { + let mut inner = AppStateInner::minimal(); + inner.node_states.extend(nodes); + let state = Arc::new(RwLock::new(inner)); + let Json(result) = calibration_start( + State(state.clone()), + Query(CalibrationStartQuery { + source_node_id: Some(author), + }), + Json(request(ids)), + ) + .await; + (state, result) +} +#[tokio::test] +async fn same_bin_count_different_ppdu_is_not_a_compatible_legacy_grid() { + let now = Instant::now(); + let (state, result) = start_nodes( + vec![ + (3, ready_node(&[grid(256, 1)], now)), + (5, ready_node(&[grid(256, 2)], now)), + ], + vec![3, 5], + 3, + ) + .await; + assert_eq!(result["success"], false); + assert_eq!( + result["source_grid_readiness"]["nodes"][1]["reason"], + "nonmatching_qualified_grid" + ); + assert_eq!( + result["source_grid_readiness"]["nodes"][1]["eligible_grids"][0], + serde_json::json!({"n_subcarriers":256,"ppdu_type":2}) + ); + assert!(state.read().await.field_model.is_none()); +} +#[tokio::test] +async fn every_bound_node_can_supply_the_exact_grid_and_unbound_nodes_do_not_leak() { + let now = Instant::now(); + let selected = grid(64, 0); + let (state, result) = start_nodes( + vec![ + (3, ready_node(&[selected], now)), + (5, ready_node(&[selected], now)), + (99, ready_node(&[grid(256, 3)], now)), + ], + vec![3, 5], + 3, + ) + .await; + assert_eq!(result["success"], true); + assert_eq!( + result["source_grid_readiness"]["all_bound_nodes_ready"], + true + ); + let Json(status) = calibration_status(State(state)).await; + assert_eq!( + status["source_grid_readiness"]["nodes"] + .as_array() + .unwrap() + .iter() + .map(|n| n["node_id"].as_u64().unwrap()) + .collect::>(), + vec![3, 5] + ); + assert!(!status["source_grid_readiness"].to_string().contains("99")); +} +#[tokio::test] +async fn nonpreferred_but_eligible_shared_grid_passes_without_blending_model_authors() { + let now = Instant::now(); + let selected = grid(64, 0); + let dense = grid(256, 1); + let c6 = ready_node(&[selected, dense], now); + assert_eq!(c6.select_calibration_grid(now).unwrap().0, dense); + let (state, result) = start_nodes( + vec![(3, c6), (5, ready_node(&[selected], now))], + vec![3, 5], + 5, + ) + .await; + assert_eq!(result["success"], true, "{result}"); + assert_eq!( + result["source_grid"], + serde_json::json!({"n_subcarriers":64,"ppdu_type":0}) + ); + assert_eq!( + result["source_grid_readiness"]["nodes"][0]["eligible_grids"][0], + result["source_grid"] + ); + let mut inner = state.write().await; + let values = vec![2.0; 64]; + let before = inner + .field_model + .as_ref() + .unwrap() + .calibration_frame_count(); + assert!(inner.maybe_feed_calibration_frame(3, selected, 1, &values, Instant::now())); + assert_eq!( + inner + .field_model + .as_ref() + .unwrap() + .calibration_frame_count(), + before + ); + assert!(inner.calibration_observed_source_node_ids.contains(&3)); + assert!(inner.maybe_feed_calibration_frame(5, selected, 1, &values, Instant::now())); + assert_eq!( + inner + .field_model + .as_ref() + .unwrap() + .calibration_frame_count(), + before + 1 + ); +} +#[tokio::test] +async fn missing_edge_only_warming_and_stale_nodes_are_not_claimed_incompatible_hardware() { + let now = Instant::now(); + let selected = grid(64, 0); + let cases = [ + "node_missing", + "raw_grid_missing", + "grid_evidence_warming", + "grid_evidence_stale", + "grid_evidence_unqualified", + ]; + for reason in cases { + let mut nodes = vec![(3, ready_node(&[selected], now))]; + if reason != "node_missing" { + let mut other = if reason == "grid_evidence_stale" { + ready_node(&[selected], now - Duration::from_secs(6)) + } else { + NodeState::new() + }; + other.last_frame_time = Some(now); // Edge/other liveness is not raw grid proof. + if reason == "grid_evidence_warming" { + other.observe_raw_grid(selected, now); + } + if reason == "grid_evidence_unqualified" { + for i in 0..=12 { + other.observe_raw_grid( + selected, + now - Duration::from_secs(12) + Duration::from_secs(i), + ); + } + } + nodes.push((5, other)); + } + let (_, result) = start_nodes(nodes, vec![3, 5], 3).await; + assert_eq!(result["success"], false, "{reason}: {result}"); + assert_eq!( + result["source_grid_readiness"]["nodes"][1]["reason"], reason, + "{result}" + ); + assert_eq!(result["source_grid_readiness"]["nodes"][1]["ready"], false); + } +} +#[tokio::test] +async fn truncated_diagnostics_keep_selected_and_all_reported_eligible_tuples() { + let now = Instant::now(); + let selected = grid(64, 0); + let mut other = ready_node( + &(0..9).map(|n| grid(80 + n * 16, 0)).collect::>(), + now, + ); + other.observe_raw_grid(selected, now); + let (_, result) = start_nodes( + vec![(3, ready_node(&[selected], now)), (5, other)], + vec![3, 5], + 3, + ) + .await; + let node = &result["source_grid_readiness"]["nodes"][1]; + assert_eq!(node["reason"], "grid_evidence_warming"); + assert_eq!(node["observed_grids_truncated"], true); + assert_eq!(node["eligible_grids_truncated"], true); + let observed = node["observed_grids"].as_array().unwrap(); + let eligible = node["eligible_grids"].as_array().unwrap(); + assert!(observed.len() <= 8 && eligible.len() <= 8); + assert_eq!(observed[0], result["source_grid"]); + assert!(eligible.iter().all(|g| observed.contains(g))); +} +#[tokio::test] +async fn current_status_reports_grid_drift_without_rewriting_historical_contributions() { + let now = Instant::now(); + let selected = grid(64, 0); + let (state, result) = start_nodes( + vec![ + (3, ready_node(&[selected], now)), + (5, ready_node(&[selected], now)), + ], + vec![3, 5], + 3, + ) + .await; + assert_eq!(result["success"], true); + { + let mut inner = state.write().await; + inner.calibration_observed_source_node_ids.extend([3, 5]); + inner + .node_states + .insert(5, ready_node(&[grid(256, 1)], Instant::now())); + } + let Json(status) = calibration_status(State(state.clone())).await; + assert_eq!(status["missing_source_node_ids"], serde_json::json!([])); + assert_eq!( + status["source_grid_readiness"]["nodes"][1]["reason"], + "nonmatching_qualified_grid" + ); + assert_eq!( + status["source_grid_readiness"]["all_bound_nodes_ready"], + false + ); + assert_eq!( + state.read().await.calibration_observed_source_node_ids, + [3, 5].into_iter().collect() + ); +} +#[tokio::test] +async fn idle_status_does_not_expose_leftover_or_unbound_grid_diagnostics() { + let mut inner = AppStateInner::minimal(); + let now = Instant::now(); + inner + .node_states + .insert(99, ready_node(&[grid(256, 1)], now)); + inner.calibration_grid_binding = Some(CalibrationGridBinding { + source_node_id: 99, + grid: grid(256, 1), + evidence: None, + }); + let Json(status) = calibration_status(State(Arc::new(RwLock::new(inner)))).await; + assert!(status["source_grid_readiness"].is_null()); + assert_eq!(status["source_node_ids"], serde_json::json!([])); +} +fn current_identity(state: &AppStateInner) -> CalibrationIdentityRequest { + CalibrationIdentityRequest { + boot_epoch: state.calibration_boot_epoch.clone(), + session_id: state.calibration_session_id.clone(), + binding_digest: state.calibration_binding_digest.clone(), + source_node_ids: Some(state.calibration_source_node_ids.iter().copied().collect()), + } +} +#[tokio::test] +async fn post_start_known_grid_drift_blocks_only_new_finalization_and_preserves_capture() { + let now = Instant::now(); + let selected = grid(64, 0); + let different = grid(256, 1); + let (state, started) = start_nodes( + vec![ + (3, ready_node(&[selected], now)), + (5, ready_node(&[selected], now)), + ], + vec![3, 5], + 3, + ) + .await; + assert_eq!(started["success"], true); + let identity; + { + let mut inner = state.write().await; + // Short SYNTHETIC model fixture, not a shortened production capture. + let mut config = field_bridge::single_link_config(); + config.min_calibration_frames = 4; + config.min_calibration_duration_s = 0.0; + inner.field_model = Some(FieldModel::new(config).unwrap()); + for seq in 1..=4 { + let values: Vec<_> = (0..64) + .map(|bin| { + 2.0 + f64::from(seq) * 0.04 + (bin as f64 * 0.3 + f64::from(seq)).sin() * 0.01 + }) + .collect(); + assert!(inner.maybe_feed_calibration_frame(3, selected, seq, &values, Instant::now())); + } + assert!(inner.maybe_feed_calibration_frame(5, selected, 1, &[2.0; 64], Instant::now())); + inner.node_states.get_mut(&5).unwrap().raw_grid_observations = + ready_node(&[different], Instant::now()).raw_grid_observations; + identity = current_identity(&inner); + } + let Json(blocked) = calibration_stop(State(state.clone()), Json(identity)).await; + assert_eq!( + blocked["error_code"], "calibration_bound_grid_unavailable", + "{blocked}" + ); + { + let inner = state.read().await; + assert_eq!( + inner.field_model.as_ref().unwrap().status(), + CalibrationStatus::Collecting + ); + assert_eq!( + inner + .field_model + .as_ref() + .unwrap() + .calibration_frame_count(), + 4 + ); + assert!(inner.calibration_model_receipt.is_none()); + assert_eq!( + inner.calibration_observed_source_node_ids, + [3, 5].into_iter().collect() + ); + assert_eq!(inner.node_states[&3].field_model_history.len(), 4); + assert_eq!(inner.node_states[&5].field_model_history.len(), 1); + } + // A qualified common minority grid is sufficient at finalization too. + let identity = { + let mut inner = state.write().await; + inner.node_states.get_mut(&5).unwrap().raw_grid_observations = + ready_node(&[selected, different], Instant::now()).raw_grid_observations; + current_identity(&inner) + }; + let Json(finished) = calibration_stop(State(state.clone()), Json(identity)).await; + assert_eq!(finished["success"], true, "{finished}"); + assert_eq!( + state.read().await.field_model.as_ref().unwrap().status(), + CalibrationStatus::Fresh + ); + // Completed receipt/metadata calls do not acquire a new all-node readiness gate. + let identity = { + let mut inner = state.write().await; + inner.node_states.get_mut(&5).unwrap().raw_grid_observations = + ready_node(&[different], Instant::now()).raw_grid_observations; + current_identity(&inner) + }; + let Json(historical) = calibration_stop(State(state.clone()), Json(identity)).await; + assert_ne!( + historical["error_code"], + "calibration_bound_grid_unavailable" + ); + assert!(state.read().await.calibration_model_receipt.is_some()); +} +#[tokio::test] +async fn warming_matching_grid_is_not_a_new_stop_incompatibility_gate() { + let now = Instant::now(); + let selected = grid(64, 0); + let (state, _) = start_nodes( + vec![ + (3, ready_node(&[selected], now)), + (5, ready_node(&[selected], now)), + ], + vec![3, 5], + 3, + ) + .await; + let identity = { + let mut inner = state.write().await; + assert!(inner.maybe_feed_calibration_frame(3, selected, 1, &[2.0; 64], Instant::now())); + assert!(inner.maybe_feed_calibration_frame(5, selected, 1, &[2.0; 64], Instant::now())); + let other = inner.node_states.get_mut(&5).unwrap(); + other.raw_grid_observations.clear(); + other.observe_raw_grid(selected, Instant::now()); + current_identity(&inner) + }; + let Json(stopped) = calibration_stop(State(state), Json(identity)).await; + assert_ne!(stopped["error_code"], "calibration_bound_grid_unavailable"); + assert!(stopped["error"].as_str().unwrap().contains("not complete")); +} +#[tokio::test] +async fn export_actual_legacy_grid_readiness_contract_cases() { + let now = Instant::now(); + let selected = grid(64, 0); + let dense = grid(256, 1); + let (_, mixed) = start_nodes( + vec![ + (3, ready_node(&[dense], now)), + (5, ready_node(&[selected], now)), + ], + vec![3, 5], + 3, + ) + .await; + let (healthy_state, started) = start_nodes( + vec![ + (3, ready_node(&[dense, selected], now)), + (5, ready_node(&[selected], now)), + ], + vec![3, 5], + 5, + ) + .await; + assert_eq!(started["success"], true); + let Json(healthy_status) = calibration_status(State(healthy_state.clone())).await; + { + let mut inner = healthy_state.write().await; + inner.node_states.get_mut(&3).unwrap().raw_grid_observations = + ready_node(&[dense], now).raw_grid_observations; + } + let Json(drift_status) = calibration_status(State(healthy_state)).await; + for status in [&healthy_status, &drift_status] { + assert_eq!( + status["source_grid_readiness"]["author_source_node_id"], + status["grid_binding"]["source_node_id"] + ); + assert_eq!( + status["source_grid_readiness"]["selected_grid"], + status["grid_binding"]["grid"] + ); + } + + let mut warming = NodeState::new(); + warming.observe_raw_grid(selected, now); + let (_, warming_refusal) = start_nodes( + vec![(3, ready_node(&[selected], now)), (5, warming)], + vec![3, 5], + 3, + ) + .await; + let (_, stale_refusal) = start_nodes( + vec![ + (3, ready_node(&[selected], now)), + (5, ready_node(&[selected], now - Duration::from_secs(6))), + ], + vec![3, 5], + 3, + ) + .await; + let (_, missing_refusal) = + start_nodes(vec![(3, ready_node(&[selected], now))], vec![3, 5], 3).await; + let Json(idle_status) = + calibration_status(State(Arc::new(RwLock::new(AppStateInner::minimal())))).await; + let mut many = ready_node( + &(0..9).map(|n| grid(80 + n * 16, 0)).collect::>(), + now, + ); + many.observe_raw_grid(selected, now); + let (_, bounded_refusal) = start_nodes( + vec![(3, ready_node(&[selected], now)), (5, many)], + vec![3, 5], + 3, + ) + .await; + let cases = serde_json::json!({"fixture_kind":"synthetic_legacy_grid_handlers","requested_source_node_ids":[3,5],"mixed_start_refusal":mixed,"healthy_start":started,"healthy_status":healthy_status,"drift_status":drift_status,"warming_start_refusal":warming_refusal,"stale_start_refusal":stale_refusal,"missing_start_refusal":missing_refusal,"idle_status":idle_status,"bounded_start_refusal":bounded_refusal}); + if let Ok(path) = std::env::var("RUVIEW_GRID_READINESS_FIXTURE_OUT") { + std::fs::write(path, serde_json::to_vec_pretty(&cases).unwrap()).unwrap(); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index d35f8830fe..f6c892814e 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -1661,13 +1661,7 @@ impl NodeState { ) -> Option<(CsiGridKey, CalibrationGridEvidence)> { self.calibration_grid_candidates(now) .into_iter() - .filter(|(grid, evidence)| { - grid.ppdu_type <= 3 - && evidence.span_s >= CALIBRATION_GRID_MIN_SPAN_S - && evidence.rate_hz >= CALIBRATION_GRID_MIN_RATE_HZ - && evidence.max_gap_s < CALIBRATION_GRID_MAX_GAP_S - && evidence.latest_age_s < CALIBRATION_GRID_MAX_GAP_S - }) + .filter(|(grid, evidence)| legacy_grid_readiness::eligible_grid(*grid, *evidence)) // Density first. This was chosen to agree with `accept_grid` when // that gate locked each node onto the densest grid it had seen; // since #1894 the gate locks onto the most frequent grid instead, @@ -9561,6 +9555,18 @@ async fn calibration_start( }; *binding }; + let source_grid_readiness=legacy_grid_readiness::inspect(&s,&request.source_node_ids,selected_binding,now) + .expect("validated binding includes the selected author"); + if !source_grid_readiness.all_bound_nodes_ready { + return Json(serde_json::json!({ + "success":false, + "error_code":"calibration_bound_grid_unavailable", + "error":format!("Legacy baseline uses one author and exact {}-bin/PPDU{} data. Placed nodes {:?} are not grid-ready. No baseline changed. Review grid diagnostics; managed per-source learning is a separate workflow.",selected_binding.grid.n_subcarriers,selected_binding.grid.ppdu_type,source_grid_readiness.nodes.iter().filter(|n|!n.ready).map(|n|n.node_id).collect::>()), + "source_node_ids":request.source_node_ids, + "source_grid":selected_binding.grid, + "source_grid_readiness":source_grid_readiness, + })); + } if s.bootstrap_baseline_active { s.field_model = None; s.bootstrap_baseline_active = false; @@ -9612,6 +9618,7 @@ async fn calibration_start( "source_node_ids": s.calibration_source_node_ids, "source_grid": selected_binding.grid, "grid_evidence": selected_binding.evidence, + "source_grid_readiness":source_grid_readiness, })) } // ADR-080 #2: FieldModel init error chain stays server-side only. @@ -9687,6 +9694,19 @@ async fn calibration_stop( "latest_seen_ms": latest_seen_ms, })); } + let unfinished=s.field_model.as_ref().is_some_and(|fm|matches!(fm.status(),CalibrationStatus::Collecting|CalibrationStatus::Uncalibrated)); + if unfinished { + if let Some(readiness)=legacy_grid_readiness::inspect(&s,&source_node_ids,binding,std::time::Instant::now()) { + if readiness.known_incompatibility() { + return Json(serde_json::json!({ + "success":false,"error_code":"calibration_bound_grid_unavailable", + "error":"A placed node now has only a different qualified raw CSI grid from this legacy baseline. Restore the frozen grid before finalizing, or review the separate managed per-source learner. The unfinished baseline was preserved.", + "model_id":model_id,"source_node_ids":source_node_ids,"source_grid":source_grid, + "source_grid_readiness":readiness, + })); + } + } + } if let Some(ref mut fm) = s.field_model { // Guard: finalizing before enough empty-room frames have accumulated // is a client-side sequencing error, not a server fault. Return a @@ -9828,6 +9848,10 @@ async fn calibration_status(State(state): State) -> Json=s.calibration_source_node_ids.iter().copied().collect(); + legacy_grid_readiness::inspect(&s,&ids,binding,now) + })} else {None}; let mut response = serde_json::json!({ "active": active, "status": status, @@ -9879,6 +9903,7 @@ async fn calibration_status(State(state): State) -> Json Date: Wed, 4 Nov 2026 20:23:17 +0100 Subject: [PATCH 064/100] test: adapt grid readiness fixtures to combined companion handlers --- .../src/legacy_grid_readiness_tests.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/v2/crates/wifi-densepose-sensing-server/src/legacy_grid_readiness_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/legacy_grid_readiness_tests.rs index 16065c2a98..a41ac32455 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/legacy_grid_readiness_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/legacy_grid_readiness_tests.rs @@ -44,7 +44,7 @@ async fn mixed_he256_ht64_is_rejected_before_existing_baseline_or_receipt_mutati inner.calibration_source_node_ids.extend([3, 5]); inner.calibration_observed_source_node_ids.insert(3); inner.calibration_model_receipt = Some(CalibrationModelReceipt { - schema: "ruview.room-calibration.model-receipt.v1", + schema: "ruview.room-calibration.model-receipt.v1".into(), boot_epoch: inner.calibration_boot_epoch.clone(), session_id: "old-session-must-stay".into(), model_id: "old-model-must-stay".into(), @@ -62,6 +62,8 @@ async fn mixed_he256_ht64_is_rejected_before_existing_baseline_or_receipt_mutati Query(CalibrationStartQuery { source_node_id: Some(3), }), + None, + None, Json(request(vec![3, 5])), ) .await; @@ -107,6 +109,8 @@ async fn start_nodes( Query(CalibrationStartQuery { source_node_id: Some(author), }), + None, + None, Json(request(ids)), ) .await; From 59324e52cf6f652f23e26cd5e8014da1c61959dc Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Thu, 5 Nov 2026 17:08:44 +0100 Subject: [PATCH 065/100] feat(learner): checkpoint current new runs without resetting capture --- ...R-420-managed-source-aware-room-learner.md | 24 + .../src/companion_integration.rs | 2 +- .../src/room_learner/lifecycle.rs | 8 + .../src/room_learner/lifecycle_recovery.rs | 28 +- .../room_learner/lifecycle_recovery_enable.rs | 162 +++++++ .../lifecycle_recovery_enable_tests.rs | 412 ++++++++++++++++++ .../room_learner/lifecycle_recovery_tests.rs | 3 + .../src/room_learner_api.rs | 23 + .../src/room_learner_api_selection.rs | 3 +- .../src/room_learner_api_selection_tests.rs | 1 + .../src/room_learner_api_tests.rs | 14 + 11 files changed, 669 insertions(+), 11 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_enable.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_enable_tests.rs diff --git a/docs/adr/ADR-420-managed-source-aware-room-learner.md b/docs/adr/ADR-420-managed-source-aware-room-learner.md index c654d2676a..162eae24f5 100644 --- a/docs/adr/ADR-420-managed-source-aware-room-learner.md +++ b/docs/adr/ADR-420-managed-source-aware-room-learner.md @@ -404,3 +404,27 @@ cold resume, all-condition qualification, explicit selection, corrupted or pendi proof/ledger files, unchanged retained-row lineage, role/counter/time regression, phase-ack ordering, and 2026-scale Rust-to-client wire fixtures. Physical qualification and delivered-artifact acceptance remain separate work. + + +### Proposed November 5 extension: protect the current new RAM run in place + +An explicit local-owner-only `recovery/enable-current` command may checkpoint a +new RAM capture without configure/reset, training, selection or phase relabelling. +Its versioned request requires the current mutation guard, an opaque server-issued +run ticket and explicit reduced-sample consent. The ticket exists only for a new +training session configured in this process without recovery; restored captures +and resumed models cannot mint it. The fixed deadline is **24 hours from that +new run's creation**, not 24 hours from pressing Enable. Success consumes it; +discard, retry, delayed requests and clock regression cannot renew that deadline. + +The first version requires no active phase or training job. It saves and verifies +the exact existing snapshot through the existing encrypted atomic store before +publishing enabled status. RAM capture, counters, roles, model and selection remain +unchanged on failure. Persistence failures report unknown retained contents rather +than claiming no samples were saved. Only a partial write under the same opaque +ticket/context/original deadline can be retried. Any other retained slot, including +an expired copy, requires a separate explicit discard decision; it is never silently +replaced. Existing recovery descriptor/checkpoint versions and expiry semantics are +unchanged. This is proposed source work, not protection of a currently running +older helper: installing it would require stopping that helper, so live volatile +cohorts must not be sacrificed to obtain this capability. diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs index 399ad1b4c9..fa5183b5cd 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_integration.rs @@ -394,7 +394,7 @@ mod tests { assert_eq!(policy(&Method::GET, "/api/v1/room-learner/progress"), Some(false)); assert_eq!(policy(&Method::HEAD, "/api/v1/room-learner/progress"), Some(false)); assert_eq!(policy(&Method::POST, "/api/v1/room-learner/progress"), None); - for path in ["/api/v1/room-learner/qualify", "/api/v1/room-learner/select", "/api/v1/room-learner/deselect", "/api/v1/room-learner/phase/cancel", "/api/v1/room-learner/train/cancel"] { + for path in ["/api/v1/room-learner/recovery/enable-current", "/api/v1/room-learner/qualify", "/api/v1/room-learner/select", "/api/v1/room-learner/deselect", "/api/v1/room-learner/phase/cancel", "/api/v1/room-learner/train/cancel"] { assert_eq!(policy(&Method::POST, path), None, "paired read/calibration grants cannot select: {path}"); } diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs index c4504dd7ab..4d3f8191fe 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs @@ -55,6 +55,7 @@ struct Inner { evaluation_warning: Option, input_error: Option, recovery: RecoveryState, + recovery_enable: Option, selection: selection_control::SelectionState, qualification: Option, qualification_error: Option, @@ -165,6 +166,7 @@ impl RoomLearningRuntime { evaluation_warning: None, input_error: None, recovery: RecoveryState::default(), + recovery_enable: None, selection: selection_control::SelectionState::default(), qualification: None, qualification_error: None, @@ -263,6 +265,9 @@ impl RoomLearningRuntime { .generation .checked_add(1) .ok_or("learner generation exhausted")?; + state.recovery_enable = if new_training && !recovery_consent { + recovery_enable::CurrentNewRun::new(&session, now_ns) + } else { None }; state.capture = Some(session); state.model = if new_training { None @@ -507,3 +512,6 @@ mod managed_training_tests; #[path = "paired_progress.rs"] mod paired_progress; pub use paired_progress::{PairedCalibrationProgress, PublishedProgressRoom, PAIRED_PROGRESS_TTL_MS}; + +#[path = "lifecycle_recovery_enable.rs"] +mod recovery_enable; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs index c1dc58c2c1..0648f34753 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs @@ -20,6 +20,14 @@ pub(super) struct RecoveryState { volatile_after_expiry: bool, } impl RecoveryState { + pub(super) fn observe_clock(&mut self, now_ns: u64) -> u64 { + self.last_now_ns = self.last_now_ns.max(now_ns); + self.last_now_ns + } + pub(super) fn last_observed_ns(&self) -> u64 { self.last_now_ns } + pub(super) fn current_run_saved(descriptor: RecoveryDescriptor, summary: CaptureCheckpointSummary, now_ns: u64) -> Self { + Self { enabled:true, descriptor:Some(descriptor), summary:Some(summary), last_now_ns:now_ns, ..Default::default() } + } pub(super) fn derived_persisted(&self) -> Option { // Unreadable retained contents are unknown, never evidence of absence. // An error also invalidates any older cached summary for this claim. @@ -36,18 +44,18 @@ impl RecoveryState { } #[derive(Serialize, Deserialize)] #[serde(deny_unknown_fields)] -struct RuntimeCheckpoint { - schema: String, - capture: CaptureCheckpoint, - frozen_training: bool, - job_phase: JobPhase, - model_sha256: Option, +pub(super) struct RuntimeCheckpoint { + pub(super) schema: String, + pub(super) capture: CaptureCheckpoint, + pub(super) frozen_training: bool, + pub(super) job_phase: JobPhase, + pub(super) model_sha256: Option, } -const SCHEMA: &str = "ruview.room-learner.runtime-recovery.v1"; +pub(super) const SCHEMA: &str = "ruview.room-learner.runtime-recovery.v1"; pub(super) fn model_sha(model: &RoomModel) -> Result { Ok(format!("{:x}", Sha256::digest(model.export()?))) } -fn checkpoint_payload( +pub(super) fn checkpoint_payload( bytes: &[u8], d: &RecoveryDescriptor, owner: &str, @@ -342,7 +350,8 @@ impl RoomLearningRuntime { json!({"supported":supported,"enabled":state.recovery.enabled,"state":status,"error":error, "saved":saved,"discard_id":state.recovery.descriptor.as_ref().map(|d| &d.id), "restored_disposition":state.recovery.disposition,"raw_iq_persisted":false, - "derived_samples_persisted":state.recovery.derived_persisted(),"automatic_resume":false}) + "derived_samples_persisted":state.recovery.derived_persisted(),"automatic_resume":false, + "enable_current_run":self.recovery_enable_status(state, now_ns)}) } pub fn saved_recovery_context( &self, @@ -419,6 +428,7 @@ impl RoomLearningRuntime { .generation .checked_add(1) .ok_or("learner generation exhausted")?; + state.recovery_enable = None; state.capture = Some(restored.session); state.model = model.map(Arc::new); state.receipt = None; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_enable.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_enable.rs new file mode 100644 index 0000000000..83420131dd --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_enable.rs @@ -0,0 +1,162 @@ +//! Explicitly protect an existing NEW RAM run without reconfiguring it. +//! Eligibility is process-local and consumed after success: restored/expired +//! captures cannot mint a fresh recovery deadline, even after explicit discard. +use super::recovery_control::{checkpoint_payload, model_sha, RuntimeCheckpoint, SCHEMA}; +use super::*; +use crate::room_learner::recovery_store::{RecoveryDescriptor, RECOVERY_SCHEMA, RECOVERY_TTL_MS}; + +#[derive(Clone)] +pub(super) struct CurrentNewRun { + id: String, + capture_run: String, + context_digest: String, + created_at_ms: u64, + expires_at_ms: u64, +} +impl CurrentNewRun { + pub(super) fn new(capture: &CaptureSession, now_ns: u64) -> Option { + let created_at_ms = now_ns / 1_000_000; + if created_at_ms == 0 { + return None; + } + Some(Self { + id: crate::companion::secret(), + capture_run: capture.checkpoint(now_ns).ok()?.run_id().into(), + context_digest: capture.context().digest(), + created_at_ms, + expires_at_ms: created_at_ms.checked_add(RECOVERY_TTL_MS)?, + }) + } +} +impl RoomLearningRuntime { + pub(super) fn recovery_enable_status(&self, state: &Inner, now_ns: u64) -> Value { + let Some(run) = &state.recovery_enable else { + return json!({"schema":"ruview.room-learner.enable-current-recovery.v1","eligible":false,"run_id":null,"reason":"not_a_new_ram_run"}); + }; + let now_ms = now_ns.max(state.recovery.last_observed_ns()) / 1_000_000; + let reason = if now_ms >= run.expires_at_ms { + Some("new_run_recovery_deadline_expired") + } else if state.recovery.enabled { + Some("recovery_already_enabled") + } else if busy(&state.job) || state.capture.as_ref().is_some_and(|c| c.phase_active()) { + Some("finish_active_operation") + } else if self.recovery_store.as_ref().is_none_or(|s| !s.available()) { + Some("recovery_unavailable") + } else if state + .recovery + .descriptor + .as_ref() + .is_some_and(|d| d.id != run.id) + { + Some("learner_recovery_pending") + } else { + None + }; + json!({"schema":"ruview.room-learner.enable-current-recovery.v1","eligible":reason.is_none(),"run_id":run.id, + "created_at_ms":run.created_at_ms,"expires_at_ms":run.expires_at_ms,"reason":reason, + "consent_required":true,"deadline_basis":"new_run_created_at","reduced_samples_only":true}) + } + /// Saves the exact current capture under its original new-run deadline. + /// No reset, fit, role change, selection or renewal occurs. A failed write + /// may have installed this SAME ticket; retry never creates a second one. + pub fn enable_current_capture_recovery( + &self, + expected: &MutationGuard, + run_id: &str, + consent: bool, + now_ns: u64, + ) -> Result<(), String> { + if !consent { + return Err("recovery_consent_required".into()); + } + let mut state = self.command_state(expected)?; + let now_ns = state.recovery.observe_clock(now_ns); + let run = state.recovery_enable.clone().ok_or("not_a_new_ram_run")?; + if run.id != run_id { + return Err("recovery_run_identity_mismatch".into()); + } + if now_ns / 1_000_000 >= run.expires_at_ms { + return Err("new_run_recovery_deadline_expired".into()); + } + if state.recovery.enabled { + return Err("recovery_already_enabled".into()); + } + if busy(&state.job) || state.capture.as_ref().is_some_and(|c| c.phase_active()) { + return Err("finish_active_operation".into()); + } + let capture = state.capture.as_ref().ok_or("recovery_capture_missing")?; + if capture.context().digest() != run.context_digest { + return Err("recovery_context_mismatch".into()); + } + let snapshot = capture.checkpoint(now_ns)?; + if snapshot.run_id() != run.capture_run { + return Err("recovery_run_identity_mismatch".into()); + } + let summary = snapshot.summary(); + let payload = RuntimeCheckpoint { + schema: SCHEMA.into(), + capture: snapshot, + frozen_training: state.frozen_training, + job_phase: state.job.phase, + model_sha256: state.model.as_ref().map(|m| model_sha(m)).transpose()?, + }; + let bytes = serde_json::to_vec(&payload).map_err(|_| "recovery_payload_encode_failed")?; + let descriptor = RecoveryDescriptor { + schema: RECOVERY_SCHEMA.into(), + id: run.id, + owner_scope: self.owner.clone(), + installation_id: self.installation.clone(), + context_digest: run.context_digest, + created_at_ms: run.created_at_ms, + saved_at_ms: now_ns / 1_000_000, + expires_at_ms: run.expires_at_ms, + }; + let store = self + .recovery_store + .as_ref() + .filter(|s| s.available()) + .ok_or("recovery_unavailable")?; + let inspected = match store.inspect() { + Ok(value) => value, + Err(error) => { + state.recovery.error = Some(bounded(error.clone())); + return Err(error); + } + }; + if let Some(previous) = inspected { + // Only a partial durable write by this very action may be retried. + if previous.id != descriptor.id + || previous.owner_scope != descriptor.owner_scope + || previous.installation_id != descriptor.installation_id + || previous.context_digest != descriptor.context_digest + || previous.created_at_ms != descriptor.created_at_ms + || previous.expires_at_ms != descriptor.expires_at_ms + { + return Err("learner_recovery_pending".into()); + } + let loaded = store.load(now_ns / 1_000_000)?.ok_or("recovery_missing")?; + let prior = checkpoint_payload( + &loaded.plaintext, + &loaded.descriptor, + &self.owner, + &self.installation, + now_ns, + )?; + if prior.capture.run_id() != run.capture_run { + return Err("recovery_run_identity_mismatch".into()); + } + } + checkpoint_payload(&bytes, &descriptor, &self.owner, &self.installation, now_ns)?; + // save includes private atomic write, fsync and exact authenticated + // readback under the store lock; errors do not enable persistence. + if let Err(error) = store.save(&descriptor, &bytes, now_ns / 1_000_000) { + // A failed durability/readback barrier can still have installed + // ciphertext. Unknown retention must never be reported as absent. + state.recovery.error = Some(bounded(error.clone())); + return Err(error); + } + state.recovery = RecoveryState::current_run_saved(descriptor, summary, now_ns); + state.recovery_enable = None; + Ok(()) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_enable_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_enable_tests.rs new file mode 100644 index 0000000000..6ad1f4d2db --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_enable_tests.rs @@ -0,0 +1,412 @@ +//! SYNTHETIC guarded enable without a device, clock change or capture reset. +use super::*; +use crate::room_learner::recovery_store::RECOVERY_TTL_MS; +fn token(rt: &RoomLearningRuntime, now: u64) -> String { + rt.status(now)["recovery"]["enable_current_run"]["run_id"] + .as_str() + .unwrap() + .into() +} +fn ram_run(root: &Path) -> Arc { + let rt = runtime(root, BASE, true); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "new-ram".into(), + true, + false, + BASE, + ) + .unwrap(); + rt +} +fn completed(rt: &RoomLearningRuntime) -> u64 { + rt.begin_phase(&rt.mutation_guard().unwrap(), request(12_000), BASE) + .unwrap(); + for q in 0..4 { + window( + rt, + BASE + u64::from(q) * 3_000_000_000 + 1_000_000, + q * 8 + 1, + ); + } + let at = BASE + 12_000_000_000; + rt.finish_phase(&rt.mutation_guard().unwrap(), at, true) + .unwrap(); + at +} +#[test] +fn current_new_run_enable_preserves_exact_rows_history_watermarks_and_cold_restores() { + let dir = tempfile::tempdir().unwrap(); + let rt = ram_run(dir.path()); + let at = completed(&rt); + let id = token(&rt, at); + let before = rt + .inner + .lock() + .unwrap() + .capture + .as_ref() + .unwrap() + .checkpoint(at) + .unwrap(); + rt.enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, at) + .unwrap(); + let after = rt + .inner + .lock() + .unwrap() + .capture + .as_ref() + .unwrap() + .checkpoint(at) + .unwrap(); + assert_eq!( + serde_json::to_vec(&before).unwrap(), + serde_json::to_vec(&after).unwrap() + ); + let status = rt.status(at); + assert_eq!(status["capture"]["sealed_cohorts"], 4); + assert_eq!(status["recovery"]["enabled"], true); + assert!(status["candidate_loaded"] == false); + assert_eq!( + status["recovery"]["saved"]["created_at_ms"], + BASE / 1_000_000 + ); + assert_eq!( + status["recovery"]["saved"]["expires_at_ms"], + BASE / 1_000_000 + RECOVERY_TTL_MS + ); + let cold = runtime(dir.path(), at + 1, true); + cold.restore_capture_recovery(&cold.mutation_guard().unwrap(), &id, &context(), at + 1) + .unwrap(); + assert_eq!(cold.status(at + 1)["capture"]["sealed_cohorts"], 4); + assert!(cold + .enable_current_capture_recovery(&cold.mutation_guard().unwrap(), &id, true, at + 2) + .is_err()); +} +#[test] +fn wrong_ticket_consent_stale_guard_and_active_phase_do_not_enable_or_change_capture() { + let dir = tempfile::tempdir().unwrap(); + let rt = ram_run(dir.path()); + let id = token(&rt, BASE); + let stale = rt.mutation_guard().unwrap(); + assert!(rt + .enable_current_capture_recovery(&stale, &id, false, BASE) + .is_err()); + assert!(rt + .enable_current_capture_recovery(&stale, "wrong", true, BASE) + .is_err()); + assert!(rt + .enable_current_capture_recovery(&stale, &id, true, BASE) + .is_err()); + rt.begin_phase(&rt.mutation_guard().unwrap(), request(12_000), BASE) + .unwrap(); + let before = rt + .inner + .lock() + .unwrap() + .capture + .as_ref() + .unwrap() + .checkpoint(BASE) + .unwrap(); + assert!(rt + .enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, BASE) + .is_err()); + assert_eq!( + serde_json::to_vec(&before).unwrap(), + serde_json::to_vec( + &rt.inner + .lock() + .unwrap() + .capture + .as_ref() + .unwrap() + .checkpoint(BASE) + .unwrap() + ) + .unwrap() + ); + assert!(rt + .recovery_store + .as_ref() + .unwrap() + .inspect() + .unwrap() + .is_none()); +} +#[test] +fn original_new_run_deadline_never_renews_even_after_regressed_time_or_discard() { + let dir = tempfile::tempdir().unwrap(); + let rt = ram_run(dir.path()); + let id = token(&rt, BASE); + let deadline = BASE + RECOVERY_TTL_MS * 1_000_000; + assert!(rt + .enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, deadline) + .is_err()); + assert!(rt + .enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, deadline - 1) + .is_err()); + assert_eq!( + rt.status(deadline - 1)["recovery"]["enable_current_run"]["eligible"], + false + ); + let second = tempfile::tempdir().unwrap(); + let rt = ram_run(second.path()); + let id = token(&rt, BASE); + rt.enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, BASE) + .unwrap(); + rt.discard_capture_recovery(&rt.mutation_guard().unwrap(), &id) + .unwrap(); + assert!(rt + .enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, BASE + 1) + .is_err()); +} +#[test] +fn foreign_and_expired_slots_are_never_overwritten_and_new_run_stays_intact() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path(), BASE, true); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "old".into(), + true, + true, + BASE, + ) + .unwrap(); + let retained = rt + .recovery_store + .as_ref() + .unwrap() + .load(BASE / 1_000_000) + .unwrap() + .unwrap(); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "new".into(), + true, + false, + BASE + 1, + ) + .unwrap(); + let id = token(&rt, BASE + 1); + assert!(rt + .enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, BASE + 2) + .is_err()); + let again = rt + .recovery_store + .as_ref() + .unwrap() + .load(BASE / 1_000_000) + .unwrap() + .unwrap(); + assert_eq!(again.descriptor, retained.descriptor); + assert_eq!(again.plaintext, retained.plaintext); + let later = BASE + (RECOVERY_TTL_MS + 1) * 1_000_000; + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "later-new".into(), + true, + false, + later, + ) + .unwrap(); + let id = token(&rt, later); + assert!(rt + .enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, later) + .is_err()); + assert_eq!( + rt.recovery_store + .as_ref() + .unwrap() + .inspect() + .unwrap() + .unwrap(), + retained.descriptor + ); +} +#[test] +fn write_failure_keeps_ram_and_ticket_then_explicit_retry_can_succeed_without_reset() { + let dir = tempfile::tempdir().unwrap(); + let rt = ram_run(dir.path()); + let at = completed(&rt); + let id = token(&rt, at); + let path = dir + .path() + .join("a".repeat(64)) + .join("SensingServer") + .join("room-learner-recovery") + .join("capture-recovery-v1.bin"); + std::fs::create_dir(&path).unwrap(); + let before = serde_json::to_vec( + &rt.inner + .lock() + .unwrap() + .capture + .as_ref() + .unwrap() + .checkpoint(at) + .unwrap(), + ) + .unwrap(); + assert!(rt + .enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, at) + .is_err()); + assert!(!rt.inner.lock().unwrap().recovery.enabled); + assert!(rt.status(at)["recovery"]["derived_samples_persisted"].is_null()); + assert_eq!( + serde_json::to_vec( + &rt.inner + .lock() + .unwrap() + .capture + .as_ref() + .unwrap() + .checkpoint(at) + .unwrap() + ) + .unwrap(), + before + ); + std::fs::remove_dir(&path).unwrap(); + rt.enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, at + 1) + .unwrap(); + assert!(rt.inner.lock().unwrap().recovery.enabled); +} + +#[test] +fn concurrent_enable_consumes_one_guard_and_cannot_renew_the_ticket() { + let dir = tempfile::tempdir().unwrap(); + let rt = ram_run(dir.path()); + let id = token(&rt, BASE); + let guard = rt.mutation_guard().unwrap(); + let barrier = Arc::new(std::sync::Barrier::new(3)); + let handles: Vec<_> = (0..2) + .map(|_| { + let rt = rt.clone(); + let id = id.clone(); + let guard = guard.clone(); + let barrier = barrier.clone(); + std::thread::spawn(move || { + barrier.wait(); + rt.enable_current_capture_recovery(&guard, &id, true, BASE) + }) + }) + .collect(); + barrier.wait(); + let outcomes: Vec<_> = handles.into_iter().map(|h| h.join().unwrap()).collect(); + assert_eq!(outcomes.iter().filter(|r| r.is_ok()).count(), 1); + let saved = rt + .recovery_store + .as_ref() + .unwrap() + .load(BASE / 1_000_000) + .unwrap() + .unwrap(); + assert_eq!(saved.descriptor.id, id); + assert_eq!( + saved.descriptor.expires_at_ms, + BASE / 1_000_000 + RECOVERY_TTL_MS + ); + assert!(rt + .enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, BASE + 1) + .is_err()); +} +#[test] +fn partial_committed_same_ticket_is_retried_with_unchanged_original_deadline() { + use super::super::recovery_control::{RuntimeCheckpoint, SCHEMA}; + use crate::room_learner::recovery_store::{RecoveryDescriptor, RECOVERY_SCHEMA}; + let dir = tempfile::tempdir().unwrap(); + let rt = ram_run(dir.path()); + let at = completed(&rt); + let id = token(&rt, at); + let payload = RuntimeCheckpoint { + schema: SCHEMA.into(), + capture: rt + .inner + .lock() + .unwrap() + .capture + .as_ref() + .unwrap() + .checkpoint(at) + .unwrap(), + frozen_training: false, + job_phase: JobPhase::Idle, + model_sha256: None, + }; + let descriptor = RecoveryDescriptor { + schema: RECOVERY_SCHEMA.into(), + id: id.clone(), + owner_scope: "a".repeat(64), + installation_id: "synthetic-install".into(), + context_digest: context().digest(), + created_at_ms: BASE / 1_000_000, + saved_at_ms: at / 1_000_000, + expires_at_ms: BASE / 1_000_000 + RECOVERY_TTL_MS, + }; + // Model a prior call whose rename committed but whose final barrier failed: + // durable bytes exist, yet the runtime never claimed persistence enabled. + rt.recovery_store + .as_ref() + .unwrap() + .save( + &descriptor, + &serde_json::to_vec(&payload).unwrap(), + at / 1_000_000, + ) + .unwrap(); + assert!(!rt.inner.lock().unwrap().recovery.enabled); + rt.enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, at + 1_000_000) + .unwrap(); + let stored = rt + .recovery_store + .as_ref() + .unwrap() + .load((at + 1_000_000) / 1_000_000) + .unwrap() + .unwrap(); + assert_eq!(stored.descriptor.created_at_ms, descriptor.created_at_ms); + assert_eq!(stored.descriptor.expires_at_ms, descriptor.expires_at_ms); + assert_eq!(rt.status(at + 1_000_000)["capture"]["sealed_cohorts"], 4); +} +#[test] +fn unavailable_key_and_other_runtime_guard_cannot_enable() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path(), BASE, false); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + context(), + "ram-no-key".into(), + true, + false, + BASE, + ) + .unwrap(); + let id = token(&rt, BASE); + assert!(rt + .enable_current_capture_recovery(&rt.mutation_guard().unwrap(), &id, true, BASE) + .is_err()); + assert!(!rt.inner.lock().unwrap().recovery.enabled); + let otherdir = tempfile::tempdir().unwrap(); + let other = ram_run(otherdir.path()); + assert!(other + .enable_current_capture_recovery( + &rt.mutation_guard().unwrap(), + &token(&other, BASE), + true, + BASE + ) + .is_err()); + assert!(other + .recovery_store + .as_ref() + .unwrap() + .inspect() + .unwrap() + .is_none()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_tests.rs index c57bd49639..7ce070b54c 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_tests.rs @@ -845,3 +845,6 @@ fn cold_inspection_of_expired_copy_fences_earlier_queued_loads_without_creating_ assert!(status["recovery"]["saved"].is_null());assert!(status["capture"].is_null());assert!(!status["candidate_loaded"].as_bool().unwrap()); assert_eq!(original_descriptor(temp.path()),descriptor);assert_eq!(std::fs::read(recovery_file(temp.path())).unwrap(),bytes); } + +#[path = "lifecycle_recovery_enable_tests.rs"] +mod enable_tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs index 49716e24dc..18a8a922a3 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs @@ -417,6 +417,29 @@ pub(super) async fn restore_capture( Err(e) => failure("learner_recovery_restore_failed", e), } } +/// Owner-only: paired read/calibration grants do not include this mutation. +#[derive(serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct EnableCurrentRecoveryRequest { + schema: String, + expected: MutationGuard, + run_id: String, + consent: bool, +} +pub(super) async fn enable_current_recovery( + State(state): State, + Json(request): Json, +) -> Json { + if request.schema != "ruview.room-learner.enable-current-recovery.v1" || !request.consent { + return failure("learner_recovery_consent_required", "Explicit current-run recovery consent is required."); + } + let rt = match runtime(&state).await { Ok(rt) => rt, Err(e) => return e }; + match rt.enable_current_capture_recovery(&request.expected, &request.run_id, request.consent, now_ns(&rt)) { + Ok(()) => Json(serde_json::json!({"success":true,"status":rt.status(now_ns(&rt))})), + Err(e) => failure("learner_recovery_enable_failed", e), + } +} + #[derive(serde::Deserialize)] #[serde(deny_unknown_fields)] pub(super) struct DiscardCaptureRequest { diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs index 21be2357be..b91897b8ba 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs @@ -1,4 +1,4 @@ -//! Owner-only model-selection commands and bounded read projection. +//! Owner-only learner controls, model selection and bounded read projection. use super::*; /// Selection mutations are restricted to the local owner, even if ordinary @@ -7,6 +7,7 @@ pub(crate) fn owner_selection_routes( auth: wifi_densepose_sensing_server::bearer_auth::AuthState, ) -> Router { Router::new() + .route("/api/v1/room-learner/recovery/enable-current", post(enable_current_recovery)) .route("/api/v1/room-learner/qualify", post(qualify)) .route("/api/v1/room-learner/select", post(select_model)) .route("/api/v1/room-learner/deselect", post(deselect_model)) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_tests.rs index 9afeb9972d..e20c1641a4 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection_tests.rs @@ -8,6 +8,7 @@ async fn selection_mutations_are_local_owner_only_even_without_outer_bearer_gate ) .with_state(state.clone()); for path in [ + "/api/v1/room-learner/recovery/enable-current", "/api/v1/room-learner/qualify", "/api/v1/room-learner/select", "/api/v1/room-learner/deselect", diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs index fda0d303da..4d952587fd 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs @@ -54,6 +54,7 @@ fn app(state: SharedState) -> Router { .route("/status", get(status)) .route("/phase", post(begin_phase)) .route("/finish", post(finish_phase)) + .route("/enable-current", post(enable_current_recovery)) .with_state(state) } async fn json_request( @@ -199,6 +200,7 @@ fn every_new_mutation_requires_admin_and_default_cli_does_not_enable_capture() { "evaluate", "recovery/restore", "recovery/discard", + "recovery/enable-current", ] { assert_eq!( required_scope_for(&Method::POST, &format!("/api/v1/room-learner/{suffix}")), @@ -541,3 +543,15 @@ mod selection_tests; #[path = "room_learner_api_selection_composer_tests.rs"] mod selection_composer_tests; + +#[tokio::test] +async fn enable_current_recovery_requires_explicit_versioned_consent_and_does_not_configure() { + let (_dir,state)=fixture(); + let rt=state.read().await.room_learning.clone().unwrap(); + for body in [serde_json::json!({"schema":"wrong","run_id":"old","consent":true}),serde_json::json!({"schema":"ruview.room-learner.enable-current-recovery.v1","run_id":"old","consent":false})] { + let result=json_request(app(state.clone()),"/enable-current",Method::POST,body).await; + assert_eq!(result["success"],false); + assert_eq!(result["error_code"],"learner_recovery_consent_required"); + assert!(rt.status(rt.receiver_now_ns())["context_digest"].is_null()); + } +} From 1c986525128c6b58604758c4f5354f811d490c24 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Fri, 6 Nov 2026 08:41:47 +0100 Subject: [PATCH 066/100] feat(learner): add inactive native temporal input and durable-prefix storage --- .../src/room_learner/artifact.rs | 11 + .../src/room_learner/artifact_catalog.rs | 13 +- .../room_learner/managed_schedule_contract.rs | 548 +++++++ .../managed_schedule_contract_tests.rs | 334 +++++ .../src/room_learner/mod.rs | 12 + .../src/room_learner/recovery_store.rs | 4 + .../src/room_learner/temporal_acquisition.rs | 1131 +++++++++++++++ .../temporal_acquisition_tests.rs | 648 +++++++++ .../room_learner/temporal_artifact_store.rs | 363 +++++ .../temporal_artifact_store_tests.rs | 544 +++++++ .../src/room_learner/temporal_checkpoint.rs | 636 ++++++++ .../room_learner/temporal_checkpoint_tests.rs | 209 +++ .../src/room_learner/temporal_execution.rs | 769 ++++++++++ .../room_learner/temporal_execution_tests.rs | 707 +++++++++ .../src/room_learner/temporal_fit_summary.rs | 315 ++++ .../temporal_fit_summary_managed_tests.rs | 472 ++++++ .../src/room_learner/temporal_live_driver.rs | 487 +++++++ ...emporal_live_driver_prior_control_tests.rs | 337 +++++ .../temporal_live_driver_tests.rs | 646 +++++++++ .../src/room_learner/temporal_model_input.rs | 845 +++++++++++ .../temporal_model_input_managed.rs | 564 +++++++ .../temporal_model_input_managed_tests.rs | 485 +++++++ .../temporal_model_input_tests.rs | 439 ++++++ .../temporal_recovery_commitment.rs | 1291 +++++++++++++++++ .../temporal_recovery_commitment_tests.rs | 987 +++++++++++++ 25 files changed, 12792 insertions(+), 5 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_schedule_contract.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_schedule_contract_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_acquisition.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_acquisition_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_artifact_store.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_artifact_store_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_summary.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_summary_managed_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver_prior_control_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment_tests.rs diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs index a1b1e69935..d5a22429b3 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs @@ -309,3 +309,14 @@ mod managed_training_store; #[path="managed_capture_fence_store.rs"] mod managed_capture_fence_store; + +#[path = "temporal_artifact_store.rs"] +mod temporal_artifacts; +pub use temporal_artifacts::TemporalCandidateReceipt; + +#[path="temporal_fit_summary.rs"] +mod temporal_fit_summary; +pub(crate) use temporal_fit_summary::{TemporalFitDiagnostics,TemporalFitSaveResult}; + +#[cfg(test)] +pub(crate) use temporal_fit_summary::rehash_fixture as rehash_temporal_fit_fixture; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact_catalog.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact_catalog.rs index 1f511590b9..385c725118 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact_catalog.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact_catalog.rs @@ -27,16 +27,19 @@ impl ArtifactStore { self.verify_owner()?; let mut contexts = Vec::new(); let mut truncated = false; - for (index, entry) in std::fs::read_dir(&self.root) + let mut inspected = 0; + for entry in std::fs::read_dir(&self.root) .map_err(|e| e.to_string())? - .take(257) - .enumerate() + // One reserved non-v1 namespace cannot consume the v1 scan budget. + .take(258) { - if index == 256 { + let entry = entry.map_err(|e| e.to_string())?; + if entry.file_name() == temporal_artifacts::DIRECTORY { continue; } + if inspected == 256 { truncated = true; break; } - let entry = entry.map_err(|e| e.to_string())?; + inspected += 1; if contexts.len() >= 256 { truncated = true; break; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_schedule_contract.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_schedule_contract.rs new file mode 100644 index 0000000000..16d84dc991 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_schedule_contract.rs @@ -0,0 +1,548 @@ +//! Inactive engineering schedule contract for the EXISTING managed learner. +//! No capture/runtime/model/storage caller and no quality/adequacy authority. +use super::{ + observation::{CaptureContext, DataRole, RoomLabel, MAX_COHORTS, MAX_ROWS, MAX_TOTAL_VALUES}, + temporal_acquisition::{ + AcquisitionPlan, ExperimentalPlanId, ReceiverCadenceBand, ResourcePreflight, + SourceWindowPolicy, + }, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +const SCHEMA: &str = "ruview.room-learner.trial-schedule.v2"; +const PROFILE: &str = "native-2s-three-source-56bin-option-a-engineering-v1"; +const MAX_BYTES: usize = 65_536; +const MAX_PROFILE_ROWS: usize = 77; +const MAX_TRAIN_WINDOWS: usize = 64; + +/// Explicit engineering choice, never a user-facing or scientific default. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum LeadInPolicy { + Train5HeldOut15EngineeringV1, +} +impl LeadInPolicy { + fn milliseconds(self, role: DataRole) -> u64 { + match role { + DataRole::Train => 5_000, + DataRole::HeldOut => 15_000, + _ => unreachable!(), + } + } +} +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct WindowSlot { + pub index: usize, + pub target_offset_ms: u64, + pub deadline_offset_ms: u64, +} +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct Episode { + pub index: usize, + pub episode_key: String, + pub role: DataRole, + pub condition: RoomLabel, + pub target_active_recording_ms: u64, + pub lead_in_ms: u64, + pub requires_explicit_operator_start: bool, + pub requires_conditions_held_confirmation: bool, + pub separately_executed_episode_required: bool, + pub slots: Vec, + pub retained_nominal_window_ms: u64, + pub retained_maximum_window_ms: u64, +} +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct ScheduleCounts { + pub train_episodes: usize, + pub held_out_episodes: usize, + pub train_windows: usize, + pub held_out_windows: usize, + pub total_windows: usize, + pub train_active_recording_ms: u64, + pub held_out_active_recording_ms: u64, + pub active_recording_ms: u64, + pub train_lead_in_ms: u64, + pub held_out_lead_in_ms: u64, + pub total_lead_in_ms: u64, + pub nominal_recording_and_lead_in_ms: u64, + pub nominal_joint_window_ms: u64, + pub nominal_source_view_window_ms: u64, + pub verified_independent_trial_count: Option, +} +#[derive(Debug, Clone, Serialize)] +struct NativeInputPolicy { + id: &'static str, + original_frame_identity: &'static str, + projection: &'static str, + target_span_ms: u64, + maximum_span_ms: u64, + maximum_first_packet_delay_ms: u64, + maximum_pairwise_receiver_skew_ms: u64, + sources: Vec, +} +#[derive(Debug, Clone, Serialize)] +struct ContractClaims { + capture_ready: bool, + full_episode_quality_monitoring_implemented: bool, + independence_verified: bool, + scientific_adequacy_claimed: bool, + user_facing_default: bool, +} +#[derive(Debug, Clone, Serialize)] +struct Body { + profile_id: &'static str, + context: CaptureContext, + lead_in_policy: LeadInPolicy, + native_input: NativeInputPolicy, + episodes: Vec, + counts: ScheduleCounts, + resources: ResourcePreflight, + serialized_checkpoint_preflight_is_estimate_only: bool, + claims: ContractClaims, +} +#[derive(Serialize)] +struct Export<'a> { + schema: &'static str, + contract_sha256: &'a str, + body: &'a Body, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Import { + schema: String, + contract_sha256: String, + body: serde_json::Value, +} +/// Immutable metadata, not a preflight/checkpoint receipt or a capture controller. +#[derive(Debug, Clone)] +pub struct ManagedScheduleContract { + body: Body, + sha256: String, +} +fn digest(value: &[u8]) -> String { + format!("{:x}", Sha256::digest(value)) +} +fn hash(body: &Body) -> Result { + let bytes = serde_json::to_vec(body).map_err(|_| "managed_schedule_encoding")?; + let mut h = Sha256::new(); + h.update(SCHEMA); + h.update([0]); + h.update(bytes); + Ok(format!("{:x}", h.finalize())) +} +fn add(a: usize, b: usize) -> Result { + a.checked_add(b) + .ok_or("managed_schedule_capacity_overflow".into()) +} +fn mul(a: usize, b: usize) -> Result { + a.checked_mul(b) + .ok_or("managed_schedule_capacity_overflow".into()) +} +fn bounded_id(value: &str) -> bool { + !value.is_empty() + && value.len() <= 128 + && value.is_ascii() + && !value + .bytes() + .any(|b| b.is_ascii_control() || b.is_ascii_whitespace()) +} +fn sha(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} +fn capacity( + context: &CaptureContext, + sources: &[SourceWindowPolicy], + train: usize, + total: usize, +) -> Result<(usize, usize), String> { + if train > MAX_TRAIN_WINDOWS + || total > MAX_COHORTS + || train == 0 + || total <= train + || sources.len() != context.sources.len() + { + return Err("managed_schedule_window_capacity".into()); + } + let mut rows = 0; + let mut values = 0; + for (spec, source) in context.sources.iter().zip(sources) { + if source.source_id != spec.source_id + || source.max_rows > MAX_ROWS + || source.max_rows > MAX_PROFILE_ROWS + || source.min_rows < 2 + || source.min_rows > source.max_rows + { + return Err("managed_schedule_row_capacity".into()); + } + rows = add(rows, source.max_rows)?; + values = add(values, mul(source.max_rows, spec.feature_count)?)?; + } + let rows = mul(rows, total)?; + let values = mul(values, total)?; + if values > MAX_TOTAL_VALUES { + return Err("managed_schedule_numeric_capacity".into()); + } + Ok((rows, values)) +} +fn slots(duration: u64, count: usize, reserved_ms: u64) -> Result, String> { + if count < 2 + || count > 16 + || reserved_ms == 0 + || duration < reserved_ms + || duration + < reserved_ms + .checked_mul(count as u64) + .ok_or("managed_schedule_slot_overflow")? + { + return Err("managed_schedule_slot_bounds".into()); + } + let stride_span = duration - reserved_ms; + let result = (0..count) + .map(|index| { + let start = stride_span + .checked_mul(index as u64) + .ok_or("managed_schedule_slot_overflow")? + / (count - 1) as u64; + Ok(WindowSlot { + index, + target_offset_ms: start, + deadline_offset_ms: start + reserved_ms, + }) + }) + .collect::, String>>()?; + if result + .windows(2) + .any(|pair| pair[0].deadline_offset_ms > pair[1].target_offset_ms) + { + return Err("managed_schedule_slot_overlap".into()); + } + Ok(result) +} +impl ManagedScheduleContract { + pub fn build(context: CaptureContext, lead_in_policy: LeadInPolicy) -> Result { + context.validate()?; + // Reuse the existing actual-native 2s validators/capacity calculation, + // but NEVER its different schedule, serialized plan or plan hash. + let bands = context + .sources + .iter() + .map(|source| (source.source_id, ReceiverCadenceBand::Hz30To36)) + .collect::>(); + let native = AcquisitionPlan::build( + context.clone(), + ExperimentalPlanId::Native2000msSlots8, + &bands, + )?; + if context.sources.len() != 3 + || context + .sources + .iter() + .any(|source| source.feature_count != 56) + { + return Err("managed_schedule_profile_supports_three_native56_sources".into()); + } + let reserved = native.max_observed_span_ms() + native.max_first_packet_delay_ms(); + let mut episodes = Vec::new(); + for role in [DataRole::Train, DataRole::HeldOut] { + for condition in [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ] { + let empty_train = role == DataRole::Train && condition == RoomLabel::Empty; + for repeat in 0..if empty_train { 1 } else { 2 } { + let duration = if empty_train { 600_000 } else { 30_000 }; + let count = if empty_train { 16 } else { 8 }; + episodes.push(Episode { + index: episodes.len(), + episode_key: format!("{role:?}-{condition:?}-episode-{repeat}"), + role, + condition, + target_active_recording_ms: duration, + lead_in_ms: lead_in_policy.milliseconds(role), + requires_explicit_operator_start: true, + requires_conditions_held_confirmation: true, + separately_executed_episode_required: true, + slots: slots(duration, count, reserved)?, + retained_nominal_window_ms: count as u64 * native.target_span_ms(), + retained_maximum_window_ms: count as u64 * native.max_observed_span_ms(), + }); + } + } + } + let role_counts = |role| episodes.iter().filter(|e| e.role == role).count(); + let role_windows = |role| { + episodes + .iter() + .filter(|e| e.role == role) + .map(|e| e.slots.len()) + .sum::() + }; + let role_time = |role| { + episodes + .iter() + .filter(|e| e.role == role) + .map(|e| e.target_active_recording_ms) + .sum::() + }; + let role_lead = |role| { + episodes + .iter() + .filter(|e| e.role == role) + .map(|e| e.lead_in_ms) + .sum::() + }; + let train = role_windows(DataRole::Train); + let held = role_windows(DataRole::HeldOut); + let total = add(train, held)?; + let (rows, values) = capacity(&context, native.source_policies(), train, total)?; + let resources = native.resources().clone(); + if resources.cohorts != total + || resources.max_retained_rows != rows + || resources.max_numeric_values != values + { + return Err("managed_schedule_resource_contract_changed".into()); + } + let active = role_time(DataRole::Train) + role_time(DataRole::HeldOut); + let lead = role_lead(DataRole::Train) + role_lead(DataRole::HeldOut); + let counts = ScheduleCounts { + train_episodes: role_counts(DataRole::Train), + held_out_episodes: role_counts(DataRole::HeldOut), + train_windows: train, + held_out_windows: held, + total_windows: total, + train_active_recording_ms: role_time(DataRole::Train), + held_out_active_recording_ms: role_time(DataRole::HeldOut), + active_recording_ms: active, + train_lead_in_ms: role_lead(DataRole::Train), + held_out_lead_in_ms: role_lead(DataRole::HeldOut), + total_lead_in_ms: lead, + nominal_recording_and_lead_in_ms: active + lead, + nominal_joint_window_ms: total as u64 * native.target_span_ms(), + nominal_source_view_window_ms: total as u64 + * native.target_span_ms() + * context.sources.len() as u64, + verified_independent_trial_count: None, + }; + let body = Body { + profile_id: PROFILE, + context, + lead_in_policy, + native_input: NativeInputPolicy { + id: "existing-native-2s-56bin-receiver-30to36-v2", + original_frame_identity: "existing-esp32-receiver-ns-node-u32-sequence-v1", + projection: "native-esp32-drop2-linear56-ordered-magnitude-v2-experimental", + target_span_ms: native.target_span_ms(), + maximum_span_ms: native.max_observed_span_ms(), + maximum_first_packet_delay_ms: native.max_first_packet_delay_ms(), + maximum_pairwise_receiver_skew_ms: native.max_pairwise_receiver_skew_ms(), + sources: native.source_policies().to_vec(), + }, + episodes, + counts, + resources, + serialized_checkpoint_preflight_is_estimate_only: true, + claims: ContractClaims { + capture_ready: false, + full_episode_quality_monitoring_implemented: false, + independence_verified: false, + scientific_adequacy_claimed: false, + user_facing_default: false, + }, + }; + let result = Self { + sha256: hash(&body)?, + body, + }; + result.export()?; + Ok(result) + } + pub fn export(&self) -> Result, String> { + let bytes = serde_json::to_vec(&Export { + schema: SCHEMA, + contract_sha256: &self.sha256, + body: &self.body, + }) + .map_err(|_| "managed_schedule_encoding")?; + if bytes.len() > MAX_BYTES { + return Err("managed_schedule_size_limit".into()); + } + Ok(bytes) + } + pub fn import(bytes: &[u8]) -> Result { + if bytes.len() > MAX_BYTES { + return Err("managed_schedule_size_limit".into()); + } + let value: Import = + serde_json::from_slice(bytes).map_err(|_| "managed_schedule_invalid_schema")?; + if value.schema != SCHEMA || value.body["profile_id"] != PROFILE { + return Err("managed_schedule_unsupported_profile".into()); + } + let context = serde_json::from_value(value.body["context"].clone()) + .map_err(|_| "managed_schedule_invalid_context")?; + let lead = serde_json::from_value(value.body["lead_in_policy"].clone()) + .map_err(|_| "managed_schedule_invalid_lead_in")?; + let rebuilt = Self::build(context, lead)?; + if value.contract_sha256 != rebuilt.sha256 + || value.body + != serde_json::to_value(&rebuilt.body).map_err(|_| "managed_schedule_encoding")? + || rebuilt.export() != Ok(bytes.to_vec()) + { + return Err("managed_schedule_contract_mismatch".into()); + } + Ok(rebuilt) + } + pub(crate) fn native_window_policy(&self)->super::temporal_acquisition::WindowInputPolicy { + let p=&self.body.native_input;super::temporal_acquisition::WindowInputPolicy::from_contract_fields(self.body.context.clone(),p.sources.clone(),p.target_span_ms,p.maximum_span_ms,p.maximum_first_packet_delay_ms,p.maximum_pairwise_receiver_skew_ms) + } + pub(crate) fn bound_matches_run(&self,bound:&BoundEpisode,run:&str)->bool { + let model=bound.frozen_artifact_sha256.as_ref().and_then(|sha|FrozenModelIdentity::from_runtime_verified_artifact(self,sha.clone()).ok()); + self.bind_episode_start(run,bound.episode.index,bound.attempt_ordinal,bound.segment_ordinal,model.as_ref()).ok().is_some_and(|expected|serde_json::to_vec(&expected).ok()==serde_json::to_vec(bound).ok()) + } + pub fn sha256(&self) -> &str { + &self.sha256 + } + pub fn context(&self) -> &CaptureContext { + &self.body.context + } + pub fn episodes(&self) -> &[Episode] { + &self.body.episodes + } + pub fn counts(&self) -> &ScheduleCounts { + &self.body.counts + } + pub fn resources(&self) -> &ResourcePreflight { + &self.body.resources + } + /// Identity binding only. Future Runtime MUST verify owner/checkpoint/current + /// step/conditions and the actual frozen managed artifact before calling. + pub fn bind_episode_start( + &self, + run_id: &str, + index: usize, + attempt: u16, + segment: u16, + frozen: Option<&FrozenModelIdentity>, + ) -> Result { + if !bounded_id(run_id) { + return Err("managed_schedule_invalid_run".into()); + } + let episode = self + .body + .episodes + .get(index) + .ok_or("managed_schedule_episode_missing")?; + match episode.role { + DataRole::Train if frozen.is_some() => { + return Err("managed_schedule_training_is_frozen".into()) + } + DataRole::HeldOut => { + let model = frozen.ok_or("managed_schedule_frozen_model_required")?; + if model.contract_sha256 != self.sha256 + || model.context_digest != self.body.context.digest() + || !sha(&model.artifact_sha256) + { + return Err("managed_schedule_frozen_model_mismatch".into()); + } + } + DataRole::Train => {} + _ => return Err("managed_schedule_invalid_role".into()), + } + let identity = |kind: &str, parts: &[serde_json::Value]| -> Result { + let bytes = serde_json::to_vec(&( + SCHEMA, + kind, + &self.sha256, + run_id, + &episode.episode_key, + parts, + )) + .map_err(|_| "managed_schedule_encoding")?; + Ok(format!("{kind}-{}", digest(&bytes))) + }; + let trial = identity("trial", &[])?; + Ok(BoundEpisode { + contract_sha256: self.sha256.clone(), + episode: episode.clone(), + episode_id: identity("episode", &[])?, + trial_id: trial, + session_id: identity("session", &[attempt.into(), segment.into()])?, + attempt_id: identity("attempt", &[attempt.into()])?, + attempt_ordinal: attempt, + segment_ordinal: segment, + frozen_artifact_sha256: frozen.map(|m| m.artifact_sha256.clone()), + }) + } +} +/// Opaque, non-Deserialize binding supplied only by the future managed Runtime. +/// Hash validity is not proof of fitting, frozen state or filesystem custody. +#[derive(Debug, Clone)] +pub struct FrozenModelIdentity { + artifact_sha256: String, + contract_sha256: String, + context_digest: String, +} +impl FrozenModelIdentity { + pub(crate) fn from_runtime_verified_artifact( + plan: &ManagedScheduleContract, + artifact_sha256: String, + ) -> Result { + if !sha(&artifact_sha256) { + return Err("managed_schedule_invalid_artifact_identity".into()); + } + Ok(Self { + artifact_sha256, + contract_sha256: plan.sha256.clone(), + context_digest: plan.body.context.digest(), + }) + } +} +#[derive(Debug, Clone, Serialize)] +pub struct BoundEpisode { + contract_sha256: String, + episode: Episode, + episode_id: String, + trial_id: String, + session_id: String, + attempt_id: String, + attempt_ordinal: u16, + segment_ordinal: u16, + frozen_artifact_sha256: Option, +} +impl BoundEpisode { + pub fn contract_sha256(&self) -> &str { + &self.contract_sha256 + } + pub fn episode(&self) -> &Episode { + &self.episode + } + pub fn episode_id(&self) -> &str { + &self.episode_id + } + pub fn trial_id(&self) -> &str { + &self.trial_id + } + pub fn session_id(&self) -> &str { + &self.session_id + } + pub fn attempt_id(&self) -> &str { + &self.attempt_id + } + pub fn attempt_ordinal(&self) -> u16 { + self.attempt_ordinal + } + pub fn segment_ordinal(&self) -> u16 { + self.segment_ordinal + } + pub fn frozen_artifact_sha256(&self) -> Option<&str> { + self.frozen_artifact_sha256.as_deref() + } +} + +#[cfg(test)] +#[path = "managed_schedule_contract_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_schedule_contract_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_schedule_contract_tests.rs new file mode 100644 index 0000000000..d6ea664081 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_schedule_contract_tests.rs @@ -0,0 +1,334 @@ +//! SYNTHETIC metadata and resource oracles. No capture/fit/permission issued. +use super::*; +fn context(count: u8) -> CaptureContext { + CaptureContext { + owner_scope: "a".repeat(64), + installation_id: "synthetic-schedule".into(), + room_digest: "synthetic-room".into(), + revision: 1, + sources: (1..=count) + .map(|source_id| super::super::observation::SourceSpec { + source_id, + device_id: format!("esp32-node-{source_id}"), + grid_id: "ant1-bins64-ppdu0-usable-drop2-linear-v2".into(), + feature_count: 56, + signal_kind: ruview_unified::RfModality::WifiCsi, + center_frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + link_id: format!("native-{source_id}"), + link_index: 0, + geometry: None, + }) + .collect(), + } +} +fn plan() -> ManagedScheduleContract { + ManagedScheduleContract::build(context(3), LeadInPolicy::Train5HeldOut15EngineeringV1).unwrap() +} +#[test] +fn option_a_is_fifteen_real_episode_descriptors_not_ids_as_independence() { + let p = plan(); + assert_eq!(p.episodes().len(), 15); + let empty = p + .episodes() + .iter() + .filter(|e| e.role == DataRole::Train && e.condition == RoomLabel::Empty) + .collect::>(); + assert_eq!(empty.len(), 1); + assert_eq!(empty[0].target_active_recording_ms, 600_000); + assert_eq!(empty[0].slots.len(), 16); + assert_eq!(empty[0].retained_nominal_window_ms, 32_000); + assert_eq!( + empty[0].retained_nominal_window_ms * 10000 / empty[0].target_active_recording_ms, + 533 + ); + for condition in [ + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ] { + let episodes = p + .episodes() + .iter() + .filter(|e| e.role == DataRole::Train && e.condition == condition) + .collect::>(); + assert_eq!(episodes.len(), 2); + assert!(episodes + .iter() + .all(|e| e.target_active_recording_ms == 30_000 && e.slots.len() == 8)); + } + for condition in [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ] { + assert_eq!( + p.episodes() + .iter() + .filter(|e| e.role == DataRole::HeldOut + && e.condition == condition + && e.target_active_recording_ms == 30_000 + && e.slots.len() == 8) + .count(), + 2 + ); + } + assert!(p.episodes()[..7].iter().all(|e| e.role == DataRole::Train)); + assert!(p.episodes()[7..] + .iter() + .all(|e| e.role == DataRole::HeldOut)); + assert!(p + .episodes() + .iter() + .all(|e| e.requires_explicit_operator_start + && e.requires_conditions_held_confirmation + && e.separately_executed_episode_required)); + assert_eq!(p.counts().verified_independent_trial_count, None); + let value: serde_json::Value = serde_json::from_slice(&p.export().unwrap()).unwrap(); + for field in [ + "capture_ready", + "full_episode_quality_monitoring_implemented", + "independence_verified", + "scientific_adequacy_claimed", + "user_facing_default", + ] { + assert_eq!(value["body"]["claims"][field], false); + } +} +#[test] +fn timing_retention_and_resource_totals_are_derived_and_honest() { + let p = plan(); + let c = p.counts(); + assert_eq!( + ( + c.train_episodes, + c.held_out_episodes, + c.train_windows, + c.held_out_windows, + c.total_windows + ), + (7, 8, 64, 64, 128) + ); + assert_eq!( + ( + c.train_active_recording_ms, + c.held_out_active_recording_ms, + c.active_recording_ms + ), + (780_000, 240_000, 1_020_000) + ); + assert_eq!( + ( + c.train_lead_in_ms, + c.held_out_lead_in_ms, + c.total_lead_in_ms + ), + (35_000, 120_000, 155_000) + ); + assert_eq!(c.nominal_recording_and_lead_in_ms, 1_175_000); + assert_eq!(c.nominal_joint_window_ms, 256_000); + assert_eq!(c.nominal_source_view_window_ms, 768_000); + let r = p.resources(); + assert_eq!( + ( + r.cohorts, + r.source_views, + r.max_retained_rows, + r.max_numeric_values + ), + (128, 384, 29_568, 1_655_808) + ); + assert!(r.serialized_upper_bound_bytes <= 64 * 1024 * 1024); + assert!(p.body.serialized_checkpoint_preflight_is_estimate_only); +} +#[test] +fn windows_span_whole_episodes_without_overlap_or_shortened_horizon() { + let p = plan(); + for e in p.episodes() { + assert_eq!(e.slots.first().unwrap().target_offset_ms, 0); + assert_eq!( + e.slots.last().unwrap().deadline_offset_ms, + e.target_active_recording_ms + ); + for s in &e.slots { + assert_eq!(s.deadline_offset_ms - s.target_offset_ms, 2150); + assert!(s.deadline_offset_ms <= e.target_active_recording_ms); + } + assert!(e + .slots + .windows(2) + .all(|w| w[0].deadline_offset_ms <= w[1].target_offset_ms)); + } + assert!(slots(30_000, 16, 2150).is_err()); + assert!(slots(30_000, 1, 2150).is_err()); +} +#[test] +fn held_out_binding_requires_future_runtime_frozen_identity_without_role_relabeling() { + let p = plan(); + for i in 7..15 { + assert!(p + .bind_episode_start("synthetic-run", i, 0, 0, None) + .is_err()); + } + let model = FrozenModelIdentity::from_runtime_verified_artifact(&p, "b".repeat(64)).unwrap(); + for i in 7..15 { + let b = p + .bind_episode_start("synthetic-run", i, 0, 0, Some(&model)) + .unwrap(); + assert_eq!(b.episode.role, DataRole::HeldOut); + assert_eq!(b.frozen_artifact_sha256, Some("b".repeat(64))); + } + assert!(p + .bind_episode_start("synthetic-run", 0, 0, 0, Some(&model)) + .is_err()); + assert!(FrozenModelIdentity::from_runtime_verified_artifact( + &p, + "inner-v1-not-a-whole-hash".into() + ) + .is_err()); + let mut changed = context(3); + changed.revision += 1; + let other = ManagedScheduleContract::build(changed, LeadInPolicy::Train5HeldOut15EngineeringV1) + .unwrap(); + let foreign = + FrozenModelIdentity::from_runtime_verified_artifact(&other, "b".repeat(64)).unwrap(); + assert!(p + .bind_episode_start("synthetic-run", 7, 0, 0, Some(&foreign)) + .is_err()); +} +#[test] +fn attempts_and_sessions_do_not_create_new_trial_groups_or_unbounded_ids() { + let p = plan(); + let a = p + .bind_episode_start(&"r".repeat(128), 0, 0, 0, None) + .unwrap(); + let b = p + .bind_episode_start(&"r".repeat(128), 0, 1, 2, None) + .unwrap(); + assert_eq!(a.episode_id, b.episode_id); + assert_eq!(a.trial_id, b.trial_id); + assert_ne!(a.session_id, b.session_id); + assert_ne!(a.attempt_id, b.attempt_id); + for id in [&a.episode_id, &a.trial_id, &a.session_id, &a.attempt_id] { + assert!(bounded_id(id)); + } + let next = p + .bind_episode_start("synthetic-run", 1, 0, 0, None) + .unwrap(); + let distinct = p + .bind_episode_start("synthetic-run", 2, 0, 0, None) + .unwrap(); + assert_ne!(next.trial_id, distinct.trial_id); + assert!(p.bind_episode_start("", 0, 0, 0, None).is_err()); + assert!(p.bind_episode_start("bad run", 0, 0, 0, None).is_err()); + assert!(p + .bind_episode_start("synthetic-run", 15, 0, 0, None) + .is_err()); + assert_eq!(p.counts().train_episodes, 7); +} +#[test] +fn canonical_versioned_hash_does_not_relabel_seed_or_accept_unknown_contracts() { + let p = plan(); + let bytes = p.export().unwrap(); + assert_eq!( + ManagedScheduleContract::import(&bytes) + .unwrap() + .export() + .unwrap(), + bytes + ); + let seed = AcquisitionPlan::build( + context(3), + ExperimentalPlanId::Native2000msSlots8, + &[ + (1, ReceiverCadenceBand::Hz30To36), + (2, ReceiverCadenceBand::Hz30To36), + (3, ReceiverCadenceBand::Hz30To36), + ], + ) + .unwrap(); + assert_ne!(p.sha256(), seed.sha256()); + // Even canonical, self-consistently rehashed metadata cannot redefine this + // finite profile's schedule, roles, counts or claims. + for change in [ + "duration", + "role", + "slots", + "quality", + "independence", + "resource", + ] { + let mut body = p.body.clone(); + match change { + "duration" => body.episodes[0].target_active_recording_ms = 300_000, + "role" => body.episodes[7].role = DataRole::Train, + "slots" => { + body.episodes[0].slots.pop(); + } + "quality" => body.claims.full_episode_quality_monitoring_implemented = true, + "independence" => body.claims.independence_verified = true, + "resource" => body.resources.max_numeric_values = 1, + _ => unreachable!(), + } + let forged_hash = hash(&body).unwrap(); + let forged = serde_json::to_vec(&Export { + schema: SCHEMA, + contract_sha256: &forged_hash, + body: &body, + }) + .unwrap(); + assert!( + ManagedScheduleContract::import(&forged).is_err(), + "{change}" + ); + } + let mut unknown: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + unknown["body"]["automatic_start"] = true.into(); + assert!(ManagedScheduleContract::import(&serde_json::to_vec(&unknown).unwrap()).is_err()); + let wrong_schema = serde_json::to_vec(&Export { + schema: "ruview.experimental.native-temporal-acquisition.v2", + contract_sha256: p.sha256(), + body: &p.body, + }) + .unwrap(); + assert!(ManagedScheduleContract::import(&wrong_schema).is_err()); + assert!(ManagedScheduleContract::import(&vec![b' '; MAX_BYTES + 1]).is_err()); +} +#[test] +fn native_capacity_and_shape_reject_before_capture_without_fallback() { + assert!( + ManagedScheduleContract::build(context(8), LeadInPolicy::Train5HeldOut15EngineeringV1) + .is_err() + ); + assert!( + ManagedScheduleContract::build(context(2), LeadInPolicy::Train5HeldOut15EngineeringV1) + .is_err() + ); + let mut wrong = context(3); + wrong.sources[0].feature_count = 55; + assert!( + ManagedScheduleContract::build(wrong, LeadInPolicy::Train5HeldOut15EngineeringV1).is_err() + ); + let p = plan(); + let mut sources = p.body.native_input.sources.clone(); + sources[0].max_rows = MAX_ROWS + 1; + assert!(capacity(&context(3), &sources, 64, 128).is_err()); + assert!(capacity(&context(3), &p.body.native_input.sources, 65, 128).is_err()); + assert!(capacity(&context(3), &p.body.native_input.sources, 64, 129).is_err()); + let many = context(8); + let policies = many + .sources + .iter() + .map(|s| SourceWindowPolicy { + source_id: s.source_id, + receiver_cadence: ReceiverCadenceBand::Hz30To36, + min_rows: 61, + max_rows: 77, + min_packet_interval_ns: 1_000_000_000_000 / 36_000, + max_gap_ns: 50_000_000, + }) + .collect::>(); + assert!(capacity(&many, &policies, 64, 128).is_err()); + assert_eq!(p.counts().total_windows, 128); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs index 7ed686e92a..02bd0041d6 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs @@ -16,3 +16,15 @@ pub mod assessment; pub(crate) mod managed_training; pub(crate) mod managed_capture_fences; + +/// Experimental source-only temporal acquisition; not wired to v1 runtime. +pub mod temporal_acquisition; + +/// Experimental policy-bound adapter; no runtime or adoption wiring. +pub mod temporal_model_input; + +/// Default-inactive continuous temporal input diagnostic; no runtime wiring. +pub mod temporal_live_driver; + +/// Inactive engineering schedule contract; no managed capture authority. +pub mod managed_schedule_contract; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store.rs index c19a9c8687..cae2fc9ea0 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store.rs @@ -433,3 +433,7 @@ impl RecoveryStore { #[cfg(test)] #[path = "recovery_store_tests.rs"] mod tests; + +// Typed temporal consistency uses the SAME profile directory/key, not V1 files. +#[path = "temporal_recovery_commitment.rs"] +pub(crate) mod temporal_commitment; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_acquisition.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_acquisition.rs new file mode 100644 index 0000000000..23e8ad8b3a --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_acquisition.rs @@ -0,0 +1,1131 @@ +//! Source-only v2 acquisition seed. Not wired to capture, inference, artifacts or +//! qualification. All policy arms are experimental engineering choices. +use super::managed_schedule_contract::{BoundEpisode, ManagedScheduleContract}; +use super::{ + native::{self, CanonicalPacket}, + observation::*, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::{HashMap, HashSet}; + +const NS_PER_MS: u64 = 1_000_000; +const NS_PER_MILLI_HZ: u64 = 1_000_000_000_000; +const MAX_NATIVE_SPAN_NS: u64 = 4_000_000_000; +const SERIALIZED_LIMIT: usize = 64 * 1024 * 1024; +const SERIALIZED_METADATA_RESERVE: usize = 8 * 1024 * 1024; +const PREPROCESSING: &str = "native-esp32-drop2-linear56-ordered-magnitude-v2-experimental"; + +/// Finite, visible alternatives. Selecting a smaller variant is explicit; there +/// is no automatic truncation or silent fallback to fewer slots/shorter input. +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +pub enum ExperimentalPlanId { + Native1000msSlots8, + Native1000msSlots4, + Native1000msSlots2, + Native2000msSlots8, + Native2000msSlots4, + Native2000msSlots2, + Native3900msMax4000Slots8, + Native3900msMax4000Slots4, + Native3900msMax4000Slots2, +} +impl ExperimentalPlanId { + fn dimensions(self) -> (u64, usize) { + use ExperimentalPlanId::*; + match self { + Native1000msSlots8 => (1000, 8), + Native1000msSlots4 => (1000, 4), + Native1000msSlots2 => (1000, 2), + Native2000msSlots8 => (2000, 8), + Native2000msSlots4 => (2000, 4), + Native2000msSlots2 => (2000, 2), + Native3900msMax4000Slots8 => (3900, 8), + Native3900msMax4000Slots4 => (3900, 4), + Native3900msMax4000Slots2 => (3900, 2), + } + } +} +/// A frozen receiver-cadence study band, NOT a source measurement clock or +/// scientific adequacy claim. Actual packet intervals/rate must also fit it. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum ReceiverCadenceBand { + Hz30To36, + Hz45To55, +} +impl ReceiverCadenceBand { + /// Read-only frozen-rate bounds for versioned input validation. + pub fn rate_bounds_millihz(self) -> (u64, u64) { + let (lo, hi, _) = self.range(); + (lo, hi) + } + fn range(self) -> (u64, u64, u64) { + match self { + Self::Hz30To36 => (30_000, 36_000, 50), + Self::Hz45To55 => (45_000, 55_000, 35), + } + } +} +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +pub struct ResourcePreflight { + pub cohorts: usize, + pub source_views: usize, + pub max_retained_rows: usize, + pub max_numeric_values: usize, + pub serialized_upper_bound_bytes: usize, +} +#[derive(Debug, Clone, Serialize)] +pub struct TrialStep { + pub step_id: String, + pub role: DataRole, + pub label: RoomLabel, + pub duration_ms: u64, + pub lead_in_ms: u64, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SourceWindowPolicy { + pub source_id: u8, + pub receiver_cadence: ReceiverCadenceBand, + pub min_rows: usize, + pub max_rows: usize, + pub min_packet_interval_ns: u64, + pub max_gap_ns: u64, +} +#[derive(Debug, Clone, Serialize)] +struct HardLimits { + rows_per_source: usize, + cohorts: usize, + numeric_values: usize, + serialized_bytes: usize, + metadata_reserve_bytes: usize, + max_native_span_ns: u64, + max_first_packet_age_ns: u64, +} +#[derive(Debug, Clone, Serialize)] +struct PlanData { + schema: &'static str, + id: ExperimentalPlanId, + context_digest: String, + preprocessing_id: &'static str, + quality_policy_id: &'static str, + limits: HardLimits, + encoder_id: &'static str, + encoder: super::model::EncoderConfig, + target_span_ms: u64, + min_observed_span_ms: u64, + max_observed_span_ms: u64, + max_pairwise_receiver_skew_ms: u64, + max_first_packet_delay_ms: u64, + slots_per_trial: usize, + schedule: Vec, + sources: Vec, + resources: ResourcePreflight, +} +/// Private fields and no Deserialize: network input cannot supply thresholds or +/// issue a new policy by changing serialized bytes. Context/spec validators are +/// the existing learner's; this is not another model/dataset implementation. +#[derive(Debug, Clone)] +pub struct AcquisitionPlan { + context: CaptureContext, + data: PlanData, + sha256: String, +} +fn add(a: usize, b: usize) -> Result { + a.checked_add(b) + .ok_or_else(|| "acquisition_resource_overflow".into()) +} +fn mul(a: usize, b: usize) -> Result { + a.checked_mul(b) + .ok_or_else(|| "acquisition_resource_overflow".into()) +} +fn ceil_div(a: u64, b: u64) -> Result { + a.checked_add(b - 1) + .map(|v| v / b) + .ok_or_else(|| "acquisition_resource_overflow".into()) +} +impl AcquisitionPlan { + pub fn build( + context: CaptureContext, + id: ExperimentalPlanId, + cadences: &[(u8, ReceiverCadenceBand)], + ) -> Result { + context.validate()?; + if cadences.len() != context.sources.len() + || cadences + .iter() + .map(|s| s.0) + .ne(context.sources.iter().map(|s| s.source_id)) + { + return Err("acquisition_cadence_source_binding".into()); + } + let (target_ms, slots) = id.dimensions(); + let max_ms = if target_ms == 3900 { + 4000 + } else { + target_ms + 50 + }; + let target_ns = target_ms * NS_PER_MS; + let max_ns = max_ms * NS_PER_MS; + if max_ns > MAX_NATIVE_SPAN_NS { + return Err("acquisition_native_horizon_limit".into()); + } + let mut sources = vec![]; + for (spec, (_, band)) in context.sources.iter().zip(cadences) { + if spec.feature_count != native::CANONICAL_PACKET_BINS + || spec.signal_kind != ruview_unified::RfModality::WifiCsi + || spec.device_id != format!("esp32-node-{}", spec.source_id) + || spec.link_index != 0 + || !native::is_supported_frequency_hz(spec.center_frequency_hz) + || spec.bandwidth_hz != 20_000_000 + || !(0..=3).any(|ppdu| { + spec.grid_id + == format!( + "ant1-bins{}-ppdu{ppdu}-usable-drop2-linear-v2", + if ppdu == 0 { 64 } else { 256 } + ) + }) + { + return Err("acquisition_requires_existing_native56_source".into()); + } + let (lo, hi, gap_ms) = band.range(); + let max_rows = usize::try_from( + ceil_div( + max_ns + .checked_mul(hi) + .ok_or("acquisition_resource_overflow")?, + NS_PER_MILLI_HZ, + )? + 3, + ) + .map_err(|_| "acquisition_resource_overflow")?; + let min_rows = usize::try_from( + target_ns + .checked_mul(lo) + .ok_or("acquisition_resource_overflow")? + / NS_PER_MILLI_HZ + + 1, + ) + .map_err(|_| "acquisition_resource_overflow")?; + if max_rows > MAX_ROWS || min_rows < 2 || min_rows > max_rows { + return Err("acquisition_row_limit".into()); + } + sources.push(SourceWindowPolicy { + source_id: spec.source_id, + receiver_cadence: *band, + min_rows, + max_rows, + min_packet_interval_ns: NS_PER_MILLI_HZ / hi, + max_gap_ns: gap_ms * NS_PER_MS, + }); + } + let mut schedule = vec![]; + for (role_index, role) in [DataRole::Train, DataRole::HeldOut].into_iter().enumerate() { + for (condition, label) in [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ] + .into_iter() + .enumerate() + { + for trial in 0..2 { + // Whole-trial duration is distinct from the retained model + // horizon. Space each sparse slot so its window can finish. + let duration_ms = if label == RoomLabel::Empty { + 600_000 + } else { + (slots as u64 * 2 * (max_ms + 100)).div_ceil(1000).max(12) * 1000 + }; + schedule.push(TrialStep { + step_id: format!("v2-r{role_index}-c{condition}-t{trial}"), + role, + label, + duration_ms, + lead_in_ms: if matches!(label, RoomLabel::Empty | RoomLabel::AdjacentOnly) { + 15_000 + } else { + 5_000 + }, + }); + } + } + } + let cohorts = mul(schedule.len(), slots)?; + if cohorts > MAX_COHORTS { + return Err("acquisition_cohort_limit".into()); + } + let rows_per_cohort = sources.iter().try_fold(0, |sum, s| add(sum, s.max_rows))?; + let values_per_cohort = sources + .iter() + .zip(&context.sources) + .try_fold(0, |sum, (s, spec)| { + add(sum, mul(s.max_rows, spec.feature_count)?) + })?; + let rows = mul(cohorts, rows_per_cohort)?; + let values = mul(cohorts, values_per_cohort)?; + if values > MAX_TOTAL_VALUES { + return Err("acquisition_numeric_capacity".into()); + } + let views = mul(cohorts, sources.len())?; + // Conservative JSON upper bound: <=24-byte finite f32 text + comma; + // original u128 frame/u64 sequence/u64 row time + array delimiters; + // escaped bounded SourceView/identities, cohort wrappers, and reserved + // future policy/quality/checkpoint metadata. Uses MAX rows, never mean. + let serialized = add( + add(add(mul(values, 25)?, mul(rows, 86)?)?, mul(views, 4096)?)?, + add(mul(cohorts, 4096)?, SERIALIZED_METADATA_RESERVE)?, + )?; + if serialized > SERIALIZED_LIMIT { + return Err("acquisition_serialized_capacity".into()); + } + let resources = ResourcePreflight { + cohorts, + source_views: views, + max_retained_rows: rows, + max_numeric_values: values, + serialized_upper_bound_bytes: serialized, + }; + let data = PlanData { + schema: "ruview.experimental.native-temporal-acquisition.v2", + id, + context_digest: context.digest(), + preprocessing_id: PREPROCESSING, + quality_policy_id: "receiver-cadence-gap-skew-first-age-v1-experimental", + limits: HardLimits { + rows_per_source: MAX_ROWS, + cohorts: MAX_COHORTS, + numeric_values: MAX_TOTAL_VALUES, + serialized_bytes: SERIALIZED_LIMIT, + metadata_reserve_bytes: SERIALIZED_METADATA_RESERVE, + max_native_span_ns: MAX_NATIVE_SPAN_NS, + max_first_packet_age_ns: MAX_AGE_NS, + }, + encoder_id: "existing-aether-ordered-block-means-v1", + encoder: super::model::EncoderConfig { + source_count: context.sources.len(), + ..Default::default() + }, + target_span_ms: target_ms, + min_observed_span_ms: target_ms, + max_observed_span_ms: max_ms, + max_pairwise_receiver_skew_ms: 50, + max_first_packet_delay_ms: 100, + slots_per_trial: slots, + schedule, + sources, + resources, + }; + let sha256 = format!( + "{:x}", + Sha256::digest(serde_json::to_vec(&data).map_err(|_| "acquisition_plan_encoding")?) + ); + Ok(Self { + context, + data, + sha256, + }) + } + /// Read-only adapter seam; does not change serialized policy or its hash. + pub fn context(&self) -> &CaptureContext { + &self.context + } + pub fn id(&self) -> ExperimentalPlanId { + self.data.id + } + pub fn slots_per_trial(&self) -> usize { + self.data.slots_per_trial + } + pub fn max_pairwise_receiver_skew_ms(&self) -> u64 { + self.data.max_pairwise_receiver_skew_ms + } + pub fn max_first_packet_delay_ms(&self) -> u64 { + self.data.max_first_packet_delay_ms + } + pub fn sha256(&self) -> &str { + &self.sha256 + } + pub fn resources(&self) -> &ResourcePreflight { + &self.data.resources + } + pub fn schedule(&self) -> &[TrialStep] { + &self.data.schedule + } + pub fn source_policies(&self) -> &[SourceWindowPolicy] { + &self.data.sources + } + pub fn encoder_config(&self) -> &super::model::EncoderConfig { + &self.data.encoder + } + pub fn target_span_ms(&self) -> u64 { + self.data.target_span_ms + } + pub fn max_observed_span_ms(&self) -> u64 { + self.data.max_observed_span_ms + } +} + +struct PacketRow { + received_ns: u64, + frame_id: u128, + sequence: u64, + values: Vec, +} +struct ActiveWindow { + target_ns: u64, + end_ns: u64, + rows: HashMap>, + provenance: Option, + failed: bool, + ready: bool, +} +/// Read-only native window rules shared by seed-v2, managed-v3 and live input. +/// No experimental seed schedule/hash is returned for a managed contract. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WindowInputPolicy { + context: CaptureContext, + sources: Vec, + target_span_ms: u64, + max_observed_span_ms: u64, + max_first_packet_delay_ms: u64, + max_pairwise_receiver_skew_ms: u64, +} +impl WindowInputPolicy { + pub(crate) fn from_seed(plan: &AcquisitionPlan) -> Self { + Self { + context: plan.context.clone(), + sources: plan.data.sources.clone(), + target_span_ms: plan.data.target_span_ms, + max_observed_span_ms: plan.data.max_observed_span_ms, + max_first_packet_delay_ms: plan.data.max_first_packet_delay_ms, + max_pairwise_receiver_skew_ms: plan.data.max_pairwise_receiver_skew_ms, + } + } + pub(crate) fn from_managed(contract: &ManagedScheduleContract) -> Self { + contract.native_window_policy() + } + pub(crate) fn from_contract_fields( + context: CaptureContext, + sources: Vec, + target: u64, + max: u64, + delay: u64, + skew: u64, + ) -> Self { + Self { + context, + sources, + target_span_ms: target, + max_observed_span_ms: max, + max_first_packet_delay_ms: delay, + max_pairwise_receiver_skew_ms: skew, + } + } + pub fn context(&self) -> &CaptureContext { + &self.context + } + pub fn source_policies(&self) -> &[SourceWindowPolicy] { + &self.sources + } + pub fn target_span_ms(&self) -> u64 { + self.target_span_ms + } + pub fn max_observed_span_ms(&self) -> u64 { + self.max_observed_span_ms + } + pub fn max_first_packet_delay_ms(&self) -> u64 { + self.max_first_packet_delay_ms + } + pub fn max_pairwise_receiver_skew_ms(&self) -> u64 { + self.max_pairwise_receiver_skew_ms + } +} +/// Historical mapping, not operator confirmation, quality or managed proof. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ManagedWindowIdentity { + pub contract_sha256: String, + pub episode_index: usize, + pub slot_index: usize, + pub episode_id: String, + pub trial_id: String, + pub session_id: String, + pub attempt_id: String, + pub attempt_ordinal: u16, + pub segment_ordinal: u16, + pub role: DataRole, + pub label: RoomLabel, + pub collection_started_ns: String, + pub target_ns: String, + pub end_ns: String, + pub frozen_artifact_sha256: Option, + pub mapping: crate::room_learner::temporal_model_input::managed::execution::ObservedSegmentMap, +} +/// Only a future Runtime mapping (or explicit SYNTHETIC unit fixture) can issue +/// this non-Deserialize target. No client timestamps/roles are accepted here. +#[derive(Debug, Clone)] +pub struct ManagedWindowTarget { + run_id: String, + identity: ManagedWindowIdentity, +} +impl ManagedWindowTarget { + #[cfg(test)] + pub(crate) fn from_runtime_episode( + contract: &ManagedScheduleContract, + bound: &BoundEpisode, + run: &str, + slot: usize, + collection_started_ns: u64, + ) -> Result { + let mapping = crate::room_learner::temporal_model_input::managed::execution::ObservedSegmentMap::synthetic_uninterrupted(bound, collection_started_ns); + Self::from_observed_episode(contract, bound, run, slot, mapping) + } + pub(crate) fn from_observed_episode( + contract: &ManagedScheduleContract, + bound: &BoundEpisode, + run: &str, + slot: usize, + mapping: crate::room_learner::temporal_model_input::managed::execution::ObservedSegmentMap, + ) -> Result { + if !contract.bound_matches_run(bound, run) { + return Err("managed_window_episode_binding".into()); + } + let spec = bound + .episode() + .slots + .get(slot) + .ok_or("managed_window_slot_missing")?; + let (target, end) = mapping.slot_bounds( + bound, + spec.target_offset_ms * NS_PER_MS, + spec.deadline_offset_ms * NS_PER_MS, + )?; + Ok(Self { + run_id: run.into(), + identity: ManagedWindowIdentity { + contract_sha256: contract.sha256().into(), + episode_index: bound.episode().index, + slot_index: slot, + episode_id: bound.episode_id().into(), + trial_id: bound.trial_id().into(), + session_id: bound.session_id().into(), + attempt_id: bound.attempt_id().into(), + attempt_ordinal: bound.attempt_ordinal(), + segment_ordinal: bound.segment_ordinal(), + role: bound.episode().role, + label: bound.episode().condition, + collection_started_ns: mapping.receiver_start_ns.to_string(), + target_ns: target.to_string(), + end_ns: end.to_string(), + frozen_artifact_sha256: bound.frozen_artifact_sha256().map(str::to_owned), + mapping, + }, + }) + } + pub(crate) fn identity(&self) -> &ManagedWindowIdentity { + &self.identity + } + pub fn target_ns(&self) -> u64 { + self.identity + .target_ns + .parse() + .expect("internally bounded target") + } + pub fn end_ns(&self) -> u64 { + self.identity + .end_ns + .parse() + .expect("internally bounded target") + } +} +#[derive(Clone)] +enum WindowIdentityMode { + Seed(AcquisitionPlan), + Managed { + contract: ManagedScheduleContract, + binding_sha256: String, + pending: Option, + }, + Inference { + binding_sha256: String, + }, +} +/// Internal exact original row times are not a serialized/public progress DTO. +pub struct TemporalWindow { + pub policy_sha256: String, + pub cohort: CaptureCohort, + pub row_times_ns: Vec<(u8, Vec)>, + pub(crate) managed_identity: Option, +} +/// A bounded single-run seed. Runtime authorization, whole-trial quality/seal, +/// recovery and artifact version integration are deliberately not implemented. +/// Source watermarks survive successful windows/role transitions within this +/// object. A failure is terminal for this seed run: it cannot relabel/retry it. +pub struct NativeWindowAssembler { + policy: WindowInputPolicy, + identity: WindowIdentityMode, + run_id: String, + next: usize, + active: Option, + watermarks: HashMap, + frames: HashSet<(u8, u128)>, + last_end_ns: u64, + failed: bool, +} +#[derive(Debug, PartialEq, Eq)] +pub enum WindowAdmission { + Collecting, + Ready, +} +impl NativeWindowAssembler { + pub fn new(plan: AcquisitionPlan, run_id: String) -> Result { + if run_id.is_empty() + || run_id.len() > 128 + || !run_id.is_ascii() + || run_id + .bytes() + .any(|b| b.is_ascii_control() || b.is_ascii_whitespace()) + { + return Err("acquisition_run_id".into()); + } + Ok(Self { + policy: WindowInputPolicy::from_seed(&plan), + identity: WindowIdentityMode::Seed(plan), + run_id, + next: 0, + active: None, + watermarks: HashMap::new(), + frames: HashSet::new(), + last_end_ns: 0, + failed: false, + }) + } + pub(crate) fn managed( + contract: ManagedScheduleContract, + binding_sha256: String, + run_id: String, + ) -> Result { + let seed = AcquisitionPlan::build( + contract.context().clone(), + ExperimentalPlanId::Native2000msSlots8, + &contract + .context() + .sources + .iter() + .map(|s| (s.source_id, ReceiverCadenceBand::Hz30To36)) + .collect::>(), + )?; + let mut result = Self::new(seed, run_id)?; + result.policy = WindowInputPolicy::from_managed(&contract); + result.identity = WindowIdentityMode::Managed { + contract, + binding_sha256, + pending: None, + }; + Ok(result) + } + pub(crate) fn inference( + policy: WindowInputPolicy, + binding_sha256: String, + run_id: String, + ) -> Result { + // Run-id validation and native rules are identical; the seed is only an + // internal rules constructor and never supplies inference role/hash. + let bands = policy + .sources + .iter() + .map(|p| (p.source_id, p.receiver_cadence)) + .collect::>(); + let seed = AcquisitionPlan::build( + policy.context.clone(), + ExperimentalPlanId::Native2000msSlots8, + &bands, + )?; + let mut result = Self::new(seed, run_id)?; + result.policy = policy; + result.identity = WindowIdentityMode::Inference { binding_sha256 }; + Ok(result) + } + pub fn begin_managed_window(&mut self, target: &ManagedWindowTarget) -> Result<(), String> { + let WindowIdentityMode::Managed { + contract, pending, .. + } = &mut self.identity + else { + return Err("managed_window_mode_required".into()); + }; + if pending.is_some() + || target.run_id != self.run_id + || target.identity.contract_sha256 != contract.sha256() + { + return Err("managed_window_contract_changed".into()); + } + let expected = contract + .episodes() + .iter() + .take(target.identity.episode_index) + .map(|e| e.slots.len()) + .sum::() + + target.identity.slot_index; + if expected != self.next || expected >= contract.counts().total_windows { + return Err("managed_window_schedule_order".into()); + } + *pending = Some(target.identity.clone()); + if let Err(e) = self.begin_checked(target.target_ns(), target.end_ns()) { + if let WindowIdentityMode::Managed { pending, .. } = &mut self.identity { + *pending = None; + } + return Err(e); + } + Ok(()) + } + /// Target and end come from future server slot scheduling, not model scores. + /// First packets must begin near target, and the COMPLETE horizon must fit. + pub fn begin_window(&mut self, target_ns: u64, end_ns: u64) -> Result<(), String> { + if matches!(self.identity, WindowIdentityMode::Managed { .. }) { + return Err("managed_window_target_required".into()); + } + self.begin_checked(target_ns, end_ns) + } + fn begin_checked(&mut self, target_ns: u64, end_ns: u64) -> Result<(), String> { + let capacity = match &self.identity { + WindowIdentityMode::Seed(p) => p.resources().cohorts, + WindowIdentityMode::Managed { contract, .. } => contract.counts().total_windows, + WindowIdentityMode::Inference { .. } => 1, + }; + if self.failed || self.active.is_some() || self.next >= capacity { + return Err("acquisition_window_state".into()); + } + if target_ns == 0 + || target_ns <= self.last_end_ns + || end_ns <= target_ns + || end_ns - target_ns + < self.policy.max_observed_span_ms * NS_PER_MS + + self.policy.max_first_packet_delay_ms * NS_PER_MS + { + return Err("acquisition_slot_bounds".into()); + } + self.active = Some(ActiveWindow { + target_ns, + end_ns, + rows: HashMap::new(), + provenance: None, + failed: false, + ready: false, + }); + Ok(()) + } + pub fn ingest(&mut self, packet: &CanonicalPacket) -> Result { + let result = self.ingest_checked(packet); + if result.is_err() { + self.failed = true; + if let Some(a) = &mut self.active { + a.failed = true; + } + } + result + } + fn ingest_checked(&mut self, p: &CanonicalPacket) -> Result { + if self.failed { + return Err("acquisition_window_state".into()); + } + let a = self + .active + .as_mut() + .filter(|a| !a.failed && !a.ready) + .ok_or("acquisition_window_not_collecting")?; + let spec = self + .policy + .context + .sources + .iter() + .find(|s| s.source_id == p.source_id) + .ok_or("acquisition_unbound_source")?; + let policy = self + .policy + .sources + .iter() + .find(|s| s.source_id == p.source_id) + .ok_or("acquisition_unbound_source")?; + let t = p.native_frame.timestamp_ns; + if p.grid_id != spec.grid_id + || p.frequency_hz != spec.center_frequency_hz + || p.bandwidth_hz != spec.bandwidth_hz + || p.native_frame.modality != spec.signal_kind + || p.native_frame.provenance.device_id != spec.device_id + || p.native_frame.centre_frequency_hz != spec.center_frequency_hz as f64 + || p.native_frame.bandwidth_hz != spec.bandwidth_hz as f64 + || p.amplitudes.len() != spec.feature_count + || p.amplitudes + .iter() + .any(|x| !x.is_finite() || *x < 0. || *x > f32::MAX as f64) + || p.native_frame.frame_id == 0 + || p.clock.kind != ClockKind::ReceiverOnly + || p.clock.uncertainty_ns != u64::MAX + || p.phase_consumed + { + return Err("acquisition_native_source_changed".into()); + } + if let Some(g) = spec.geometry { + if p.native_frame.transmitter_pose.map(|p| p.position_m) != Some(g.tx_pos) + || p.native_frame.receiver_pose.map(|p| p.position_m) != Some(g.rx_pos) + { + return Err("acquisition_native_geometry_changed".into()); + } + } + let (lo, hi, _) = policy.receiver_cadence.range(); + if !p.receiver_sample_rate_hz.is_finite() + || p.receiver_sample_rate_hz < lo as f64 / 1000. + || p.receiver_sample_rate_hz > hi as f64 / 1000. + { + return Err("acquisition_cadence_changed".into()); + } + if t < a.target_ns || t >= a.end_ns { + return Err("acquisition_packet_outside_slot".into()); + } + if self + .watermarks + .get(&p.source_id) + .is_some_and(|(seq, last)| u64::from(p.sequence) <= *seq || t <= *last) + || self + .frames + .contains(&(p.source_id, p.native_frame.frame_id)) + { + return Err("acquisition_replayed_or_reversed".into()); + } + if a.provenance.is_some_and(|v| v != p.provenance) { + return Err("acquisition_provenance_changed".into()); + } + let rows = a.rows.entry(p.source_id).or_default(); + if rows.len() >= policy.max_rows { + return Err("acquisition_row_capacity".into()); + } + if let Some(last) = rows.last() { + let gap = t - last.received_ns; + if gap > policy.max_gap_ns || gap < policy.min_packet_interval_ns { + return Err("acquisition_packet_gap_or_rate".into()); + } + } else if t - a.target_ns > self.policy.max_first_packet_delay_ms * NS_PER_MS { + return Err("acquisition_late_first_packet".into()); + } + if let Some(first) = rows.first() { + if t - first.received_ns > self.policy.max_observed_span_ms * NS_PER_MS { + return Err("acquisition_window_span_limit".into()); + } + } + a.provenance = Some(p.provenance); + rows.push(PacketRow { + received_ns: t, + frame_id: p.native_frame.frame_id, + sequence: u64::from(p.sequence), + values: p.amplitudes.iter().map(|v| *v as f32).collect(), + }); + self.watermarks + .insert(p.source_id, (u64::from(p.sequence), t)); + self.frames.insert((p.source_id, p.native_frame.frame_id)); + a.ready = self.policy.sources.iter().all(|s| { + a.rows.get(&s.source_id).is_some_and(|r| { + r.len() >= s.min_rows + && r.last().unwrap().received_ns - r[0].received_ns + >= self.policy.target_span_ms * NS_PER_MS + }) + }); + if a.ready { + Ok(WindowAdmission::Ready) + } else { + Ok(WindowAdmission::Collecting) + } + } + pub fn finish_window(&mut self, now_ns: u64) -> Result { + let result = self.finish_checked(now_ns); + if result.is_err() { + self.failed = true; + } + result + } + fn finish_checked(&mut self, now_ns: u64) -> Result { + if self.failed { + return Err("acquisition_window_state".into()); + } + let a = self + .active + .as_ref() + .filter(|a| a.ready && !a.failed) + .ok_or("acquisition_horizon_incomplete")?; + if now_ns >= a.end_ns { + return Err("acquisition_slot_expired".into()); + } + + let mut views = vec![]; + let mut times = vec![]; + let mut first_min = u64::MAX; + let mut first_max = 0; + let mut last_min = u64::MAX; + let mut last_max = 0; + for (spec, policy) in self.policy.context.sources.iter().zip(&self.policy.sources) { + let rows = a + .rows + .get(&spec.source_id) + .ok_or("acquisition_missing_source")?; + let first = rows + .first() + .ok_or("acquisition_missing_source")? + .received_ns; + let last = rows.last().unwrap().received_ns; + let age = now_ns + .checked_sub(first) + .filter(|age| *age <= MAX_AGE_NS) + .ok_or("acquisition_first_packet_age")?; + if now_ns < last { + return Err("acquisition_future_packet".into()); + } + let span = last - first; + if span < self.policy.target_span_ms * NS_PER_MS + || span > self.policy.max_observed_span_ms * NS_PER_MS + || span > MAX_NATIVE_SPAN_NS + { + return Err("acquisition_observed_span".into()); + } + let (lo, hi, _) = policy.receiver_cadence.range(); + let scaled = (rows.len() - 1) as u128 * NS_PER_MILLI_HZ as u128; + if scaled < (lo as u128 * span as u128) || scaled > (hi as u128 * span as u128) { + return Err("acquisition_observed_cadence".into()); + } + first_min = first_min.min(first); + first_max = first_max.max(first); + last_min = last_min.min(last); + last_max = last_max.max(last); + let view = SourceView { + source_id: spec.source_id, + device_id: spec.device_id.clone(), + grid_id: spec.grid_id.clone(), + signal_kind: spec.signal_kind, + center_frequency_hz: spec.center_frequency_hz, + bandwidth_hz: spec.bandwidth_hz, + link_id: spec.link_id.clone(), + link_index: spec.link_index, + geometry: spec.geometry, + samples: rows.iter().map(|r| r.values.clone()).collect(), + frame_ids: rows.iter().map(|r| r.frame_id).collect(), + sequence_ids: rows.iter().map(|r| r.sequence).collect(), + first_time_ns: first, + last_time_ns: last, + clock: ClockInfo { + kind: ClockKind::ReceiverOnly, + uncertainty_ns: u64::MAX, + }, + sample_age_ns: age, + phase: PhaseAvailability::MagnitudeOnly, + provenance: a.provenance.ok_or("acquisition_provenance_missing")?, + }; + view.validate(spec)?; + views.push(view); + times.push((spec.source_id, rows.iter().map(|r| r.received_ns).collect())); + } + let skew = self.policy.max_pairwise_receiver_skew_ms * NS_PER_MS; + if first_max - first_min > skew || last_max - last_min > skew || first_max >= last_min { + return Err("acquisition_source_skew".into()); + } + let (id, trial_id, session_id, role, label, policy_sha256, managed_identity) = match &self + .identity + { + WindowIdentityMode::Seed(plan) => { + let step = &plan.data.schedule[self.next / plan.data.slots_per_trial]; + let slot = self.next % plan.data.slots_per_trial; + ( + format!("{}-{}-{slot}", self.run_id, step.step_id), + format!("{}-{}", self.run_id, step.step_id), + format!("{}-{}", self.run_id, step.step_id), + step.role, + step.label, + plan.sha256.clone(), + None, + ) + } + WindowIdentityMode::Managed { + binding_sha256, + pending, + .. + } => { + let p = pending.as_ref().ok_or("managed_window_target_missing")?; + let id = format!( + "window-{:x}", + Sha256::digest( + serde_json::to_vec(p).map_err(|_| "managed_window_identity_encoding")? + ) + ); + ( + id, + p.trial_id.clone(), + p.session_id.clone(), + p.role, + p.label, + binding_sha256.clone(), + Some(p.clone()), + ) + } + WindowIdentityMode::Inference { binding_sha256 } => ( + format!("{}-inference-0", self.run_id), + format!("{}-inference", self.run_id), + format!("{}-inference", self.run_id), + DataRole::Inference, + RoomLabel::Unknown, + binding_sha256.clone(), + None, + ), + }; + let cohort = CaptureCohort { + id, + context_digest: self.policy.context.digest(), + trial_id, + session_id, + role, + label, + views, + teacher: None, + provenance: a.provenance.unwrap(), + }; + validate_cohort(&self.policy.context, &cohort)?; + let result = TemporalWindow { + policy_sha256, + cohort, + row_times_ns: times, + managed_identity, + }; + if let WindowIdentityMode::Managed { pending, .. } = &mut self.identity { + *pending = None; + } + self.last_end_ns = last_max; + self.next += 1; + self.active = None; + Ok(result) + } +} +#[cfg(test)] +#[path = "temporal_acquisition_tests.rs"] +mod tests; + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct AssemblerCheckpoint { + next: usize, + last_end_ns: u64, + watermarks: Vec<(u8, u64, u64)>, + frames: Vec<(u8, String)>, +} +impl AssemblerCheckpoint { + pub(crate) fn for_prefix(mut self, n: usize) -> Result { + if n > self.next { + return Err("temporal_prefix_ahead".into()); + } + self.next = n; + Ok(self) + } +} +impl NativeWindowAssembler { + pub(crate) fn fence_checkpoint(&self) -> Result { + if self.failed { + return Err("temporal_failed_fences".into()); + } + let mut watermarks = self + .watermarks + .iter() + .map(|(id, (seq, t))| (*id, *seq, *t)) + .collect::>(); + watermarks.sort(); + let mut frames = self + .frames + .iter() + .map(|(id, f)| (*id, f.to_string())) + .collect::>(); + frames.sort(); + Ok(AssemblerCheckpoint { + next: self.next, + last_end_ns: self.last_end_ns, + watermarks, + frames, + }) + } + pub(crate) fn restore_prefix_fences( + &mut self, + s: AssemblerCheckpoint, + expected_next: usize, + ) -> Result<(), String> { + let max = self + .policy + .sources + .iter() + .map(|s| s.max_rows) + .sum::() + * 128; + if s.next != expected_next + || s.next > 128 + || s.frames.len() > max + || s.watermarks.len() > self.policy.sources.len() + { + return Err("temporal_checkpoint_fence_bounds".into()); + } + let mut watermarks = HashMap::new(); + let mut frames = HashSet::new(); + for (id, seq, t) in s.watermarks { + if !self + .policy + .context + .sources + .iter() + .any(|x| x.source_id == id) + || seq == 0 + || t == 0 + || watermarks.insert(id, (seq, t)).is_some() + { + return Err("temporal_checkpoint_fence_source".into()); + } + } + for (id, f) in s.frames { + let f = f.parse::().map_err(|_| "temporal_checkpoint_frame")?; + let (sequence, time) = watermarks + .get(&id) + .copied() + .ok_or("temporal_checkpoint_frame")?; + if ((f >> 32) & 255) != u128::from(id) + || (f >> 40) > u128::from(time) + || (f & u128::from(u32::MAX)) > u128::from(sequence) + || !frames.insert((id, f)) + { + return Err("temporal_checkpoint_frame".into()); + } + } + self.next = s.next; + self.last_end_ns = s.last_end_ns; + self.watermarks = watermarks; + self.frames = frames; + self.active = None; + self.failed = false; + if let WindowIdentityMode::Managed { pending, .. } = &mut self.identity { + *pending = None; + } + Ok(()) + } + pub(crate) fn fences_cover(&self, other: &AssemblerCheckpoint) -> bool { + other.watermarks.iter().all(|(id, seq, t)| { + self.watermarks + .get(id) + .is_some_and(|(s, n)| s >= seq && n >= t) + }) && other.frames.iter().all(|(id, f)| { + f.parse() + .ok() + .is_some_and(|f| self.frames.contains(&(*id, f))) + }) + } +} + +impl NativeWindowAssembler { + pub(crate) fn retained_fences_cover(&self, rows: &[(u8, u64, u64, u128)]) -> bool { + rows.iter().all(|(id, seq, t, f)| { + self.watermarks + .get(id) + .is_some_and(|(s, n)| s >= seq && n >= t) + && self.frames.contains(&(*id, *f)) + }) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_acquisition_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_acquisition_tests.rs new file mode 100644 index 0000000000..87111e709c --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_acquisition_tests.rs @@ -0,0 +1,648 @@ +//! SYNTHETIC original native IQ through the existing native adapter and SourceView +//! validators. No physical timing/accuracy or integrated v2 runtime claim. +use super::super::native::{canonicalize_esp32, NativePacketInput}; +use super::*; +use wifi_densepose_hardware::{Adr018Flags, PpduType}; + +fn context(count: u8) -> CaptureContext { + CaptureContext { + owner_scope: "a".repeat(64), + installation_id: "synthetic-install".into(), + room_digest: "synthetic-room".into(), + revision: 1, + sources: (1..=count) + .map(|source_id| SourceSpec { + source_id, + device_id: format!("esp32-node-{source_id}"), + grid_id: "ant1-bins64-ppdu0-usable-drop2-linear-v2".into(), + feature_count: 56, + signal_kind: ruview_unified::RfModality::WifiCsi, + center_frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + link_id: format!("native-{source_id}"), + link_index: 0, + geometry: None, + }) + .collect(), + } +} +fn plan( + count: u8, + id: ExperimentalPlanId, + band: ReceiverCadenceBand, +) -> Result { + AcquisitionPlan::build( + context(count), + id, + &(1..=count).map(|n| (n, band)).collect::>(), + ) +} +fn packet(source: u8, sequence: u32, at: u64, hz: f64) -> CanonicalPacket { + let bytes = (0..64).flat_map(|_| [3u8, 4]).collect::>(); + canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence, + received_at_ns: at, + receiver_sample_rate_hz: hz, + iq_bytes: &bytes, + flags: Adr018Flags::default(), + provenance: CaptureProvenance::Synthetic, + }) + .unwrap() +} +fn collect( + assembler: &mut NativeWindowAssembler, + target: u64, + hz: u64, + sequence: &mut u32, +) -> TemporalWindow { + let period = 1_000_000_000u64.div_ceil(hz); + let end = target + 6_000_000_000; + assembler.begin_window(target, end).unwrap(); + for index in 0..MAX_ROWS { + *sequence += 1; + for source in 1..=assembler.policy.context.sources.len() as u8 { + let last = target + index as u64 * period + u64::from(source) * 1000; + let result = assembler + .ingest(&packet(source, *sequence, last, hz as f64)) + .unwrap(); + if result == WindowAdmission::Ready { + return assembler.finish_window(last).unwrap(); + } + } + } + panic!("window did not reach its actual horizon"); +} +#[test] +fn immutable_plan_hash_and_actual_max_capacity_are_context_cadence_and_variant_bound() { + use ExperimentalPlanId::*; + use ReceiverCadenceBand::*; + let a = plan(3, Native2000msSlots8, Hz30To36).unwrap(); + let b = plan(3, Native2000msSlots8, Hz30To36).unwrap(); + assert_eq!(a.sha256(), b.sha256()); + assert_eq!(a.resources().cohorts, 128); + assert_eq!(a.source_policies()[0].max_rows, 77); + assert_eq!(a.resources().max_numeric_values, 128 * 3 * 56 * 77); + assert!(a.resources().serialized_upper_bound_bytes <= SERIALIZED_LIMIT); + assert_eq!( + a.schedule() + .iter() + .filter(|s| s.role == DataRole::Train) + .count(), + 8 + ); + assert_eq!( + a.schedule() + .iter() + .filter(|s| s.role == DataRole::HeldOut) + .count(), + 8 + ); + for label in [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ] { + for role in [DataRole::Train, DataRole::HeldOut] { + assert_eq!( + a.schedule() + .iter() + .filter(|s| s.role == role && s.label == label) + .count(), + 2 + ); + } + } + assert!(plan(2, Native2000msSlots8, Hz45To55).is_ok()); + assert_eq!( + plan(3, Native2000msSlots8, Hz45To55).unwrap_err(), + "acquisition_numeric_capacity" + ); + assert!(plan(3, Native2000msSlots4, Hz45To55).is_ok()); + assert!(plan(8, Native2000msSlots4, Hz30To36).is_err()); + let smaller = plan(8, Native2000msSlots2, Hz30To36).unwrap(); + assert_eq!(smaller.resources().cohorts, 32); + assert_ne!( + a.sha256(), + plan(2, Native2000msSlots8, Hz30To36).unwrap().sha256() + ); + assert_ne!( + a.sha256(), + plan(3, Native2000msSlots4, Hz30To36).unwrap().sha256() + ); + let mut ctx = context(2); + ctx.sources[1].source_id = 1; + assert!( + AcquisitionPlan::build(ctx, Native1000msSlots2, &[(1, Hz30To36), (1, Hz30To36)]).is_err() + ); + assert!(AcquisitionPlan::build( + context(2), + Native1000msSlots2, + &[(2, Hz30To36), (1, Hz30To36)] + ) + .is_err()); + let mut ctx = context(2); + ctx.sources[0].feature_count = 64; + assert!( + AcquisitionPlan::build(ctx, Native1000msSlots2, &[(1, Hz30To36), (2, Hz30To36)]).is_err() + ); +} +#[test] +fn actual_two_seconds_at_33_and_50_hz_preserve_all_original_rows_phy_and_frame_identity() { + for (hz, band, rows) in [ + (33, ReceiverCadenceBand::Hz30To36, 67), + (50, ReceiverCadenceBand::Hz45To55, 101), + ] { + let p = plan(2, ExperimentalPlanId::Native2000msSlots2, band).unwrap(); + let hash = p.sha256().to_string(); + let mut a = NativeWindowAssembler::new(p, "synthetic-run".into()).unwrap(); + let window = collect(&mut a, 10_000_000_000, hz, &mut 0); + assert_eq!(window.policy_sha256, hash); + assert_eq!(window.cohort.role, DataRole::Train); + assert_eq!(window.cohort.provenance, CaptureProvenance::Synthetic); + assert!(window.cohort.teacher.is_none()); + for (view, (source, times)) in window.cohort.views.iter().zip(&window.row_times_ns) { + assert_eq!(view.source_id, *source); + assert_eq!(view.samples.len(), rows); + assert_eq!( + view.last_time_ns - view.first_time_ns, + times.last().unwrap() - times[0] + ); + assert!(view.last_time_ns - view.first_time_ns >= 2_000_000_000); + assert_eq!(view.center_frequency_hz, 2_437_000_000); + assert_eq!(view.bandwidth_hz, 20_000_000); + assert_eq!(view.clock.kind, ClockKind::ReceiverOnly); + assert_eq!(view.clock.uncertainty_ns, u64::MAX); + for (index, ((sequence, frame), row)) in view + .sequence_ids + .iter() + .zip(&view.frame_ids) + .zip(&view.samples) + .enumerate() + { + let original = packet(*source, *sequence as u32, times[index], hz as f64); + assert_eq!(*frame, original.native_frame.frame_id); + assert_eq!( + row, + &original + .amplitudes + .iter() + .map(|n| *n as f32) + .collect::>() + ); + assert!(row.iter().all(|v| *v == 5.0)); + } + } + } +} +#[test] +fn fixed_64_rows_cannot_claim_two_seconds_and_legacy_eight_rows_are_not_upgraded() { + for count in [8, 64] { + for hz in [33, 50] { + let band = if hz == 33 { + ReceiverCadenceBand::Hz30To36 + } else { + ReceiverCadenceBand::Hz45To55 + }; + let p = plan(2, ExperimentalPlanId::Native2000msSlots2, band).unwrap(); + let mut a = NativeWindowAssembler::new(p, "short-control".into()).unwrap(); + let target = 10_000_000_000; + a.begin_window(target, target + 3_000_000_000).unwrap(); + let period = 1_000_000_000u64.div_ceil(hz); + for index in 0..count { + for source in 1..=2 { + assert_eq!( + a.ingest(&packet( + source, + index + 1, + target + u64::from(index) * period + u64::from(source) * 1000, + hz as f64 + )) + .unwrap(), + WindowAdmission::Collecting + ); + } + } + assert_eq!( + a.finish_window(target + u64::from(count) * period) + .err() + .as_deref(), + Some("acquisition_horizon_incomplete") + ); + } + } +} +#[test] +fn gaps_replay_wrong_source_grid_phy_age_and_skew_fail_without_downsampling() { + for failure in [ + "gap", + "sequence", + "frame", + "source", + "grid", + "phy", + "nonfinite", + "cadence", + "fast_actual", + "provenance", + ] { + let p = plan( + 2, + ExperimentalPlanId::Native1000msSlots2, + ReceiverCadenceBand::Hz30To36, + ) + .unwrap(); + let mut a = NativeWindowAssembler::new(p, "negative".into()).unwrap(); + let t = 10_000_000_000; + a.begin_window(t, t + 2_000_000_000).unwrap(); + let first = packet(1, 1, t + 1000, 33.); + let original = first.native_frame.frame_id; + a.ingest(&first).unwrap(); + let mut bad = packet(1, 2, t + 30_304_000, 33.); + match failure { + "gap" => bad = packet(1, 2, t + 100_000_000, 33.), + "sequence" => bad.sequence = 1, + "frame" => bad.native_frame.frame_id = original, + "source" => bad.source_id = 99, + "grid" => bad.grid_id = "fake".into(), + "phy" => bad.frequency_hz = 5_180_000_000, + "nonfinite" => bad.amplitudes[0] = f64::NAN, + "cadence" => bad.receiver_sample_rate_hz = 50., + "fast_actual" => bad = packet(1, 2, t + 20_000_000, 33.), + "provenance" => bad.provenance = CaptureProvenance::LiveReceiverObserved, + _ => unreachable!(), + } + assert!(a.ingest(&bad).is_err(), "{failure}"); + assert_eq!( + a.active.as_ref().unwrap().rows[&1].len(), + 1, + "{failure} replaced a row" + ); + assert!(a + .begin_window(t + 3_000_000_000, t + 5_000_000_000) + .is_err()); + } + let mut a = NativeWindowAssembler::new( + plan( + 2, + ExperimentalPlanId::Native3900msMax4000Slots2, + ReceiverCadenceBand::Hz30To36, + ) + .unwrap(), + "age".into(), + ) + .unwrap(); + let w = collect(&mut a, 10_000_000_000, 33, &mut 0); + assert!(w + .cohort + .views + .iter() + .all(|v| v.last_time_ns - v.first_time_ns >= 3_900_000_000 + && v.last_time_ns - v.first_time_ns <= 4_000_000_000)); + let mut a = NativeWindowAssembler::new( + plan( + 2, + ExperimentalPlanId::Native3900msMax4000Slots2, + ReceiverCadenceBand::Hz30To36, + ) + .unwrap(), + "stale".into(), + ) + .unwrap(); + let t = 10_000_000_000; + a.begin_window(t, t + 7_000_000_000).unwrap(); + let period = 1_000_000_000u64.div_ceil(33); + let mut last = t; + 'ready: for index in 0..MAX_ROWS { + for source in 1..=2 { + last = t + index as u64 * period + u64::from(source) * 1000; + if a.ingest(&packet(source, index as u32 + 1, last, 33.)) + .unwrap() + == WindowAdmission::Ready + { + break 'ready; + } + } + } + assert!(last - t <= 4_000_000_000); + assert_eq!( + a.finish_window(t + 5_000_000_001 + 1000).err().as_deref(), + Some("acquisition_first_packet_age") + ); +} +#[test] +fn schedule_roles_and_original_sequence_fences_survive_multiple_windows() { + let ctx = context(2); + let p = plan( + 2, + ExperimentalPlanId::Native1000msSlots2, + ReceiverCadenceBand::Hz30To36, + ) + .unwrap(); + let mut a = NativeWindowAssembler::new(p, "role-run".into()).unwrap(); + let mut rows = vec![]; + let mut sequence = 0; + for index in 0..32 { + let w = collect( + &mut a, + 10_000_000_000 + index * 3_000_000_000, + 33, + &mut sequence, + ); + assert_eq!( + w.cohort.role, + if index < 16 { + DataRole::Train + } else { + DataRole::HeldOut + } + ); + rows.push(w.cohort); + } + let valid = ValidatedDataset::new(ctx.clone(), rows.clone()).unwrap(); + assert_eq!(valid.train().len(), 16); + assert_eq!(valid.held_out().len(), 16); + assert!(!valid.measurement_clock_available()); + rows[16].views[0].frame_ids[0] = rows[0].views[0].frame_ids[0]; + assert!(ValidatedDataset::new(ctx, rows).is_err()); + assert!(a.begin_window(200_000_000_000, 202_000_000_000).is_err()); + let p = plan( + 2, + ExperimentalPlanId::Native1000msSlots2, + ReceiverCadenceBand::Hz30To36, + ) + .unwrap(); + let mut a = NativeWindowAssembler::new(p, "replay".into()).unwrap(); + collect(&mut a, 10_000_000_000, 33, &mut 0); + a.begin_window(20_000_000_000, 22_000_000_000).unwrap(); + assert_eq!( + a.ingest(&packet(1, 1, 20_000_001_000, 33.)) + .err() + .as_deref(), + Some("acquisition_replayed_or_reversed") + ); +} +#[test] +fn resource_upper_bound_includes_max_rows_and_actual_json_not_just_float_memory() { + let p = plan( + 3, + ExperimentalPlanId::Native2000msSlots8, + ReceiverCadenceBand::Hz30To36, + ) + .unwrap(); + let resources = p.resources(); + assert!( + resources.serialized_upper_bound_bytes + > resources.max_numeric_values * std::mem::size_of::() + ); + let mut a = NativeWindowAssembler::new(p.clone(), "serialization".into()).unwrap(); + let mut w = collect(&mut a, 10_000_000_000, 33, &mut 0); + for view in &mut w.cohort.views { + let max = p + .source_policies() + .iter() + .find(|s| s.source_id == view.source_id) + .unwrap() + .max_rows; + view.samples = vec![vec![1.2345678e-30f32; 56]; max]; + view.frame_ids = vec![u128::MAX; max]; + view.sequence_ids = vec![u64::MAX; max]; + } + let one = serde_json::to_vec(&w.cohort).unwrap().len(); + assert!( + one * resources.cohorts + SERIALIZED_METADATA_RESERVE + < resources.serialized_upper_bound_bytes + ); +} + +#[test] +fn individually_long_enough_sources_still_need_bounded_pairwise_receiver_skew() { + let p = plan( + 2, + ExperimentalPlanId::Native1000msSlots2, + ReceiverCadenceBand::Hz30To36, + ) + .unwrap(); + let mut a = NativeWindowAssembler::new(p, "skew".into()).unwrap(); + let t = 10_000_000_000; + a.begin_window(t, t + 2_000_000_000).unwrap(); + let period = 1_000_000_000u64.div_ceil(33); + let mut now = t; + 'ready: for index in 0..MAX_ROWS { + for source in 1..=2 { + now = t + index as u64 * period + if source == 1 { 1000 } else { 80_001_000 }; + if a.ingest(&packet(source, index as u32 + 1, now, 33.)) + .unwrap() + == WindowAdmission::Ready + { + break 'ready; + } + } + } + assert_eq!( + a.finish_window(now).err().as_deref(), + Some("acquisition_source_skew") + ); +} +#[test] +fn capacity_failure_is_terminal_and_does_not_evict_or_shorten_the_horizon() { + let p = plan( + 2, + ExperimentalPlanId::Native2000msSlots2, + ReceiverCadenceBand::Hz30To36, + ) + .unwrap(); + let capacity = p.source_policies()[0].max_rows; + let mut a = NativeWindowAssembler::new(p, "capacity".into()).unwrap(); + let t = 10_000_000_000; + a.begin_window(t, t + 3_000_000_000).unwrap(); + a.ingest(&packet(1, 1, t + 1000, 33.)).unwrap(); + // Boundary fault injection of a full buffer: the public admission operation + // must fail rather than pop old rows, regardless of any future caller bug. + let original = a.active.as_ref().unwrap().rows[&1][0].frame_id; + let buffer = a.active.as_mut().unwrap().rows.get_mut(&1).unwrap(); + while buffer.len() < capacity { + buffer.push(PacketRow { + received_ns: t + 1000, + frame_id: original, + sequence: 1, + values: vec![5.; 56], + }); + } + assert_eq!( + a.ingest(&packet(1, 2, t + 30_304_000, 33.)) + .err() + .as_deref(), + Some("acquisition_row_capacity") + ); + assert_eq!(a.active.as_ref().unwrap().rows[&1].len(), capacity); + assert_eq!(a.active.as_ref().unwrap().rows[&1][0].frame_id, original); + assert!(a.finish_window(t + 2_100_000_000).is_err()); +} + +#[test] +fn four_second_native_ceiling_is_independent_of_the_existing_five_second_age_guard() { + let p = plan( + 2, + ExperimentalPlanId::Native3900msMax4000Slots2, + ReceiverCadenceBand::Hz30To36, + ) + .unwrap(); + assert_eq!(p.max_observed_span_ms(), 4000); + let mut a = NativeWindowAssembler::new(p, "four-second-limit".into()).unwrap(); + let t = 10_000_000_000; + a.begin_window(t, t + 6_000_000_000).unwrap(); + let period = 1_000_000_000u64.div_ceil(33); + 'ready: for index in 0..MAX_ROWS { + for source in 1..=2 { + if a.ingest(&packet( + source, + index as u32 + 1, + t + index as u64 * period + u64::from(source) * 1000, + 33., + )) + .unwrap() + == WindowAdmission::Ready + { + break 'ready; + } + } + } + // Corrupted/overlong buffer negative control: the inherited SourceView + // validator permits up to60s, so v2's own <=4s guard must independently win. + let rows = a.active.as_mut().unwrap().rows.get_mut(&1).unwrap(); + let first = rows[0].received_ns; + rows.last_mut().unwrap().received_ns = first + 4_000_000_001; + assert_eq!( + a.finish_window(t + 4_100_000_000).err().as_deref(), + Some("acquisition_observed_span") + ); +} + +#[test] +fn approved_variants_preflight_every_supported_source_count_without_silent_fallback() { + use ExperimentalPlanId::*; + for count in 2..=16 { + for id in [ + Native1000msSlots8, + Native1000msSlots4, + Native1000msSlots2, + Native2000msSlots8, + Native2000msSlots4, + Native2000msSlots2, + Native3900msMax4000Slots8, + Native3900msMax4000Slots4, + Native3900msMax4000Slots2, + ] { + for band in [ReceiverCadenceBand::Hz30To36, ReceiverCadenceBand::Hz45To55] { + if let Ok(plan) = plan(count, id, band) { + assert_eq!(plan.data.id, id); + assert!(plan.resources().cohorts <= MAX_COHORTS); + assert!(plan.resources().max_numeric_values <= MAX_TOTAL_VALUES); + assert!(plan.resources().serialized_upper_bound_bytes <= SERIALIZED_LIMIT); + assert!(plan + .source_policies() + .iter() + .all(|s| s.max_rows <= MAX_ROWS)); + assert_eq!(plan.encoder_config().source_count, usize::from(count)); + assert_eq!(plan.encoder_config().temporal_tokens, 4); + } + } + } + } +} +#[test] +fn mixed_original_ht_and_he_phy_grids_remain_distinct_in_the_same_temporal_window() { + let mut ctx = context(2); + ctx.sources[1].grid_id = "ant1-bins256-ppdu1-usable-drop2-linear-v2".into(); + let p = AcquisitionPlan::build( + ctx, + ExperimentalPlanId::Native1000msSlots2, + &[ + (1, ReceiverCadenceBand::Hz30To36), + (2, ReceiverCadenceBand::Hz30To36), + ], + ) + .unwrap(); + let mut a = NativeWindowAssembler::new(p, "mixed-native".into()).unwrap(); + let t = 10_000_000_000; + a.begin_window(t, t + 2_000_000_000).unwrap(); + let period = 1_000_000_000u64.div_ceil(33); + for index in 0..MAX_ROWS { + for source in 1..=2 { + let bins = if source == 1 { 64 } else { 256 }; + let bytes = (0..bins).flat_map(|_| [3u8, 4]).collect::>(); + let packet = canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: bins, + frequency_hz: 2_437_000_000, + ppdu_type: if source == 1 { + PpduType::HtLegacy + } else { + PpduType::HeSu + }, + sequence: index as u32 + 1, + received_at_ns: t + index as u64 * period + u64::from(source) * 1000, + receiver_sample_rate_hz: 33., + iq_bytes: &bytes, + flags: Adr018Flags::default(), + provenance: CaptureProvenance::Synthetic, + }) + .unwrap(); + let now = packet.native_frame.timestamp_ns; + if a.ingest(&packet).unwrap() == WindowAdmission::Ready { + let w = a.finish_window(now).unwrap(); + assert!(w.cohort.views[0].grid_id.contains("bins64-ppdu0")); + assert!(w.cohort.views[1].grid_id.contains("bins256-ppdu1")); + assert!(w + .cohort + .views + .iter() + .all(|v| v.samples.len() == 34 && v.samples.iter().all(|r| r.len() == 56))); + return; + } + } + } + panic!("mixed source window incomplete"); +} + +#[test] +fn a_failed_finish_cannot_be_retried_with_an_earlier_clock_or_more_packets() { + let p = plan( + 2, + ExperimentalPlanId::Native1000msSlots2, + ReceiverCadenceBand::Hz30To36, + ) + .unwrap(); + let mut a = NativeWindowAssembler::new(p, "failed-finish".into()).unwrap(); + let t = 10_000_000_000; + a.begin_window(t, t + 7_000_000_000).unwrap(); + let period = 1_000_000_000u64.div_ceil(33); + let mut last = t; + 'ready: for index in 0..MAX_ROWS { + for source in 1..=2 { + last = t + index as u64 * period + u64::from(source) * 1000; + if a.ingest(&packet(source, index as u32 + 1, last, 33.)) + .unwrap() + == WindowAdmission::Ready + { + break 'ready; + } + } + } + assert_eq!( + a.finish_window(t + 6_000_000_000).err().as_deref(), + Some("acquisition_first_packet_age") + ); + assert_eq!( + a.finish_window(last).err().as_deref(), + Some("acquisition_window_state") + ); + assert!(a.ingest(&packet(1, 200, last + period, 33.)).is_err()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_artifact_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_artifact_store.rs new file mode 100644 index 0000000000..2f18433070 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_artifact_store.rs @@ -0,0 +1,363 @@ +//! Whole temporal-input diagnostics in the EXISTING owner-bound ArtifactStore. +//! No runtime caller, managed proof, qualification, selection or inner-v1 access. +use super::*; +use crate::room_learner::temporal_model_input::TemporalModel; +use std::{fs, io::Read}; + +// Stable kind namespace; codec kind/pointer are versioned inside it. +// This reader supports only the existing d4a v2 codec, with no upconversion. +pub(super) const DIRECTORY: &str = "temporal-models"; +const SCHEMA: &str = "ruview.room-learner.temporal-candidate.v1"; +const KIND: &str = "ruview.experimental.temporal-model-input.v2"; +const MANAGED_KIND: &str = "ruview.room-learner.managed-temporal-input.v3"; +const MANAGED_POINTER: &str = "candidate-managed-temporal-v3.json"; +const BLOB_LIMIT: usize = 4 * 1024 * 1024; +const POINTER: &str = "candidate-temporal-v2.json"; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalCandidateReceipt { + pub schema: String, + pub artifact_kind: String, + pub owner_scope: String, + pub installation_id: String, + pub context_digest: String, + pub room_digest: String, + pub artifact_sha256: String, + pub input_policy_sha256: String, + pub parameter_count: usize, + pub diagnostic_only: bool, + pub physically_validated: bool, + pub qualification_granted: bool, +} +fn metadata(meta: &fs::Metadata, directory: bool) -> Result<(), String> { + if meta.file_type().is_symlink() + || (directory && !meta.is_dir()) + || (!directory && !meta.is_file()) + { + return Err("temporal_artifact_unsafe_path".into()); + } + #[cfg(unix)] + { + use std::os::unix::fs::{MetadataExt, PermissionsExt}; + extern "C" { + fn geteuid() -> u32; + } + // SAFETY: geteuid has no arguments or memory side effects. + if meta.uid() != unsafe { geteuid() } + || meta.permissions().mode() & 0o7777 != if directory { 0o700 } else { 0o600 } + { + return Err("temporal_artifact_private_permissions".into()); + } + } + #[cfg(not(unix))] + { + return Err("temporal_artifact_private_permissions_unsupported".into()); + } + #[allow(unreachable_code)] + Ok(()) +} +fn directory(path: &Path, create: bool) -> Result { + match fs::symlink_metadata(path) { + Ok(meta) => { + metadata(&meta, true)?; + return Ok(true); + } + Err(error) if error.kind() == io::ErrorKind::NotFound && !create => return Ok(false), + Err(error) if error.kind() == io::ErrorKind::NotFound => {} + Err(_) => return Err("temporal_artifact_directory_unavailable".into()), + } + let mut builder = fs::DirBuilder::new(); + #[cfg(unix)] + { + use std::os::unix::fs::DirBuilderExt; + builder.mode(0o700); + } + match builder.create(path) { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => {} + Err(_) => return Err("temporal_artifact_directory_unavailable".into()), + } + metadata( + &fs::symlink_metadata(path).map_err(|_| "temporal_artifact_directory_unavailable")?, + true, + )?; + Ok(true) +} +// Only fixed callers select 64KiB pointer or 4MiB temporal-codec limits. This is +// not a replacement/global enlargement of companion::private_read. +pub(super) fn bounded_read(path: &Path, limit: usize) -> Result, String> { + let check = |meta: &fs::Metadata| -> Result<(), String> { + metadata(meta, false)?; + if meta.len() > limit as u64 { + return Err("temporal_artifact_size_limit".into()); + } + Ok(()) + }; + check(&fs::symlink_metadata(path).map_err(|_| "temporal_artifact_unavailable")?)?; + let mut options = fs::OpenOptions::new(); + options.read(true); + #[cfg(target_os = "macos")] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(0x100); + } + #[cfg(target_os = "linux")] + { + use std::os::unix::fs::OpenOptionsExt; + options.custom_flags(0x20000); + } + #[cfg(not(any(target_os = "macos", target_os = "linux")))] + { + return Err("temporal_artifact_nofollow_unsupported".into()); + } + let file = options + .open(path) + .map_err(|_| "temporal_artifact_unavailable")?; + check( + &file + .metadata() + .map_err(|_| "temporal_artifact_unavailable")?, + )?; + let mut bytes = Vec::new(); + file.take((limit + 1) as u64) + .read_to_end(&mut bytes) + .map_err(|_| "temporal_artifact_read_failed")?; + if bytes.len() > limit { + return Err("temporal_artifact_size_limit".into()); + } + Ok(bytes) +} +pub(super) fn immutable(path: &Path, bytes: &[u8]) -> Result<(), String> { + if bytes.len() > BLOB_LIMIT { + return Err("temporal_artifact_size_limit".into()); + } + if regular_or_missing(path)? { + return if bounded_read(path, BLOB_LIMIT)? == bytes { + Ok(()) + } else { + Err("temporal_artifact_immutable_mismatch".into()) + }; + } + let parent = path + .parent() + .ok_or("temporal_artifact_directory_unavailable")?; + let stage = parent.join(format!("temporal-stage-{}", crate::companion::secret())); + atomic_private_write(&stage, bytes).map_err(|_| "temporal_artifact_stage_failed")?; + let result = fs::hard_link(&stage, path); + let cleanup = fs::remove_file(&stage); + match result { + Ok(()) => {} + Err(error) if error.kind() == io::ErrorKind::AlreadyExists => { + if bounded_read(path, BLOB_LIMIT)? != bytes { + return Err("temporal_artifact_immutable_mismatch".into()); + } + } + Err(_) => return Err("temporal_artifact_install_failed".into()), + } + cleanup.map_err(|_| "temporal_artifact_stage_cleanup_failed")?; + fs::File::open(parent) + .and_then(|file| file.sync_all()) + .map_err(|_| "temporal_artifact_sync_failed")?; + Ok(()) +} +impl ArtifactStore { + pub(super) fn temporal_directory( + &self, + context: &CaptureContext, + create: bool, + ) -> Result, String> { + self.verify_owner()?; + self.validate_context(context)?; + let kind = self.root.join(DIRECTORY); + if !directory(&kind, create)? { + return Ok(None); + } + let path = kind.join(context.digest()); + if !directory(&path, create)? { + return Ok(None); + } + Ok(Some(path)) + } + fn temporal_receipt(&self, model: &TemporalModel, bytes: &[u8]) -> TemporalCandidateReceipt { + TemporalCandidateReceipt { + schema: SCHEMA.into(), + artifact_kind: model.artifact_kind().into(), + owner_scope: self.owner_scope.clone(), + installation_id: self.installation_id.clone(), + context_digest: model.context().digest(), + room_digest: model.context().room_digest.clone(), + artifact_sha256: digest(bytes), + input_policy_sha256: model.plan_sha256().into(), + parameter_count: model.parameter_count(), + diagnostic_only: true, + physically_validated: false, + qualification_granted: false, + } + } + fn validate_temporal_receipt( + &self, + receipt: &TemporalCandidateReceipt, + context: &CaptureContext, + ) -> Result<(), String> { + self.validate_context(context)?; + if receipt.schema != SCHEMA + || !matches!(receipt.artifact_kind.as_str(), KIND | MANAGED_KIND) + || receipt.owner_scope != self.owner_scope + || receipt.installation_id != self.installation_id + || receipt.context_digest != context.digest() + || receipt.room_digest != context.room_digest + || !hex_digest(&receipt.artifact_sha256) + || !hex_digest(&receipt.input_policy_sha256) + || !(1..=512).contains(&receipt.parameter_count) + || !receipt.diagnostic_only + || receipt.physically_validated + || receipt.qualification_granted + { + return Err("temporal_artifact_receipt_mismatch".into()); + } + Ok(()) + } + fn temporal_read_model( + &self, + path: &Path, + receipt: &TemporalCandidateReceipt, + context: &CaptureContext, + ) -> Result { + self.validate_temporal_receipt(receipt, context)?; + let bytes = bounded_read( + &path.join(format!("{}.json", receipt.artifact_sha256)), + BLOB_LIMIT, + )?; + if digest(&bytes) != receipt.artifact_sha256 { + return Err("temporal_artifact_hash_mismatch".into()); + } + let model = TemporalModel::import(&bytes)?; + if model.context() != context || self.temporal_receipt(&model, &model.export()?) != *receipt + { + return Err("temporal_artifact_context_or_policy_mismatch".into()); + } + Ok(model) + } + fn temporal_read_pointer( + &self, + path: &Path, + context: &CaptureContext, + ) -> Result, String> { + self.temporal_read_pointer_kind(path, context, KIND) + } + fn temporal_read_pointer_kind( + &self, + path: &Path, + context: &CaptureContext, + kind: &str, + ) -> Result, String> { + let pointer = path.join(match kind { + KIND => POINTER, + MANAGED_KIND => MANAGED_POINTER, + _ => return Err("temporal_artifact_kind_unsupported".into()), + }); + if !regular_or_missing(&pointer)? { + return Ok(None); + } + let receipt: TemporalCandidateReceipt = + serde_json::from_slice(&bounded_read(&pointer, LIMIT)?) + .map_err(|_| "temporal_artifact_pointer_invalid")?; + if receipt.artifact_kind != kind { + return Err("temporal_artifact_pointer_kind_changed".into()); + } + self.temporal_read_model(path, &receipt, context)?; + Ok(Some(receipt)) + } + /// Diagnostic persistence only; it cannot create managed proof or selection. + pub fn save_temporal_candidate( + &self, + model: &TemporalModel, + ) -> Result { + self.validate_context(model.context())?; + let bytes = model.export()?; + let receipt = self.temporal_receipt(model, &bytes); + self.validate_temporal_receipt(&receipt, model.context())?; + let pointer = encode(&receipt)?; + let path = self + .temporal_directory(model.context(), true)? + .ok_or("temporal_artifact_directory_missing")?; + self.temporal_read_pointer_kind(&path, model.context(), model.artifact_kind())?; + immutable( + &path.join(format!("{}.json", receipt.artifact_sha256)), + &bytes, + )?; + self.temporal_read_model(&path, &receipt, model.context())?; + let pointer_name = if model.artifact_kind() == KIND { + POINTER + } else if model.artifact_kind() == MANAGED_KIND { + MANAGED_POINTER + } else { + return Err("temporal_artifact_kind_unsupported".into()); + }; + atomic_private_write(&path.join(pointer_name), &pointer) + .map_err(|_| "temporal_artifact_pointer_write_failed")?; + if bounded_read(&path.join(pointer_name), LIMIT)? != pointer { + return Err("temporal_artifact_pointer_readback_failed".into()); + } + self.temporal_read_pointer_kind(&path, model.context(), model.artifact_kind())?; + Ok(receipt) + } + pub fn temporal_candidate_receipt( + &self, + context: &CaptureContext, + ) -> Result, String> { + let Some(path) = self.temporal_directory(context, false)? else { + return Ok(None); + }; + self.temporal_read_pointer(&path, context) + } + pub fn load_temporal_candidate( + &self, + context: &CaptureContext, + ) -> Result, String> { + let Some(path) = self.temporal_directory(context, false)? else { + return Ok(None); + }; + let Some(receipt) = self.temporal_read_pointer(&path, context)? else { + return Ok(None); + }; + self.temporal_read_model(&path, &receipt, context).map(Some) + } + pub fn load_managed_temporal_candidate( + &self, + context: &CaptureContext, + ) -> Result, String> { + let Some(path) = self.temporal_directory(context, false)? else { + return Ok(None); + }; + let Some(receipt) = self.temporal_read_pointer_kind(&path, context, MANAGED_KIND)? else { + return Ok(None); + }; + self.temporal_read_model(&path, &receipt, context).map(Some) + } + pub fn load_exact_temporal_candidate( + &self, + context: &CaptureContext, + artifact_sha256: &str, + ) -> Result { + if !hex_digest(artifact_sha256) { + return Err("temporal_artifact_invalid_hash".into()); + } + let path = self + .temporal_directory(context, false)? + .ok_or("temporal_artifact_directory_missing")?; + let bytes = bounded_read(&path.join(format!("{artifact_sha256}.json")), BLOB_LIMIT)?; + if digest(&bytes) != artifact_sha256 { + return Err("temporal_artifact_hash_mismatch".into()); + } + let model = TemporalModel::import(&bytes)?; + if model.context() != context { + return Err("temporal_artifact_context_mismatch".into()); + } + Ok(model) + } +} +#[cfg(all(test, unix))] +#[path = "temporal_artifact_store_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_artifact_store_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_artifact_store_tests.rs new file mode 100644 index 0000000000..3f1e9bbabc --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_artifact_store_tests.rs @@ -0,0 +1,544 @@ +//! SYNTHETIC native inputs + actual fitter/filesystem; no runtime or user data. +use super::*; +use crate::room_learner::{ + capture::{CaptureSession, PhaseRequest, RawSourceFrame}, + native::{canonicalize_esp32, CanonicalPacket, NativePacketInput}, + observation::*, + selection::{SelectionEnvelope, SelectionReceipt}, + temporal_acquisition::{ExperimentalPlanId, ReceiverCadenceBand, WindowAdmission}, + temporal_model_input::TemporalTrainingSession, + training::FitConfig, +}; +use std::{collections::BTreeMap, sync::OnceLock}; +use wifi_densepose_hardware::{Adr018Flags, PpduType}; +fn context() -> CaptureContext { + CaptureContext { + owner_scope: "a".repeat(64), + installation_id: "synthetic-temporal-store".into(), + room_digest: "synthetic-room".into(), + revision: 1, + sources: (1..=2) + .map(|source_id| SourceSpec { + source_id, + device_id: format!("esp32-node-{source_id}"), + grid_id: "ant1-bins64-ppdu0-usable-drop2-linear-v2".into(), + feature_count: 56, + signal_kind: ruview_unified::RfModality::WifiCsi, + center_frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + link_id: format!("native-{source_id}"), + link_index: 0, + geometry: None, + }) + .collect(), + } +} +fn packet(source: u8, sequence: u32, time: u64, label: RoomLabel, row: usize) -> CanonicalPacket { + let gain = match label { + RoomLabel::Empty => 3, + RoomLabel::AdjacentOnly => 4, + RoomLabel::OccupiedStatic => 8, + RoomLabel::OccupiedMoving => 10 + (row % 2) as u8, + RoomLabel::Unknown => 8, + }; + let bytes = (0..64) + .flat_map(|bin| [gain + source + (bin % 2) as u8, 4]) + .collect::>(); + canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence, + received_at_ns: time, + receiver_sample_rate_hz: 33., + iq_bytes: &bytes, + flags: Adr018Flags::default(), + provenance: CaptureProvenance::Synthetic, + }) + .unwrap() +} +fn fitted() -> TemporalModel { + static BYTES: OnceLock> = OnceLock::new(); + TemporalModel::import(BYTES.get_or_init(|| { + let mut session = TemporalTrainingSession::new( + context(), + ExperimentalPlanId::Native2000msSlots2, + &[ + (1, ReceiverCadenceBand::Hz30To36), + (2, ReceiverCadenceBand::Hz30To36), + ], + "synthetic-store-fit".into(), + FitConfig { + epochs: 2, + ..Default::default() + }, + ) + .unwrap(); + let labels = [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ]; + let mut sequence = 0; + for window in 0..16usize { + let target = 10_000_000_000 + window as u64 * 3_000_000_000; + session + .begin_window(target, target + 3_000_000_000) + .unwrap(); + let label = labels[window / 4]; + let mut complete = false; + 'rows: for row in 0..256usize { + sequence += 1; + for source in 1..=2 { + let at = target + + row as u64 * 1_000_000_000u64.div_ceil(33) + + u64::from(source) * 1000; + if session + .ingest(&packet(source, sequence, at, label, row)) + .unwrap() + == WindowAdmission::Ready + { + session.finish_window(at).unwrap(); + complete = true; + break 'rows; + } + } + } + assert!(complete); + } + let (model, report) = session.fit().unwrap(); + assert!(report.backbone_delta_l2 > 0.); + model.export().unwrap() + })) + .unwrap() +} +fn store(root: &Path) -> ArtifactStore { + let c = context(); + ArtifactStore::new( + &root.join(&c.owner_scope).join("SensingServer"), + &c.owner_scope, + &c.installation_id, + ) + .unwrap() +} +fn live(model: &TemporalModel) -> serde_json::Value { + let mut input = model.live_input("synthetic-fresh-live".into()).unwrap(); + let target = 1_000_000_000_000; + input.begin_window(target, target + 3_000_000_000).unwrap(); + for row in 0..256usize { + for source in 1..=2 { + let at = target + row as u64 * 1_000_000_000u64.div_ceil(33) + u64::from(source) * 1000; + if input + .ingest(&packet( + source, + 100_000 + row as u32, + at, + RoomLabel::Unknown, + row, + )) + .unwrap() + == WindowAdmission::Ready + { + return serde_json::to_value(model.predict(input.finish(at).unwrap(), at).unwrap()) + .unwrap(); + } + } + } + panic!("missing complete native horizon") +} +fn files(root: &Path) -> BTreeMap> { + fn visit(root: &Path, at: &Path, out: &mut BTreeMap>) { + for entry in fs::read_dir(at).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + visit(root, &path, out); + } else { + out.insert( + path.strip_prefix(root).unwrap().to_path_buf(), + fs::read(path).unwrap(), + ); + } + } + } + let mut result = BTreeMap::new(); + visit(root, root, &mut result); + result +} +fn blob_path(store: &ArtifactStore, receipt: &TemporalCandidateReceipt) -> PathBuf { + store + .root + .join(DIRECTORY) + .join(&receipt.context_digest) + .join(format!("{}.json", receipt.artifact_sha256)) +} +#[test] +fn actual_native_fit_whole_envelope_private_persist_coldload_same_inference() { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let model = fitted(); + let bytes = model.export().unwrap(); + assert!(bytes.len() > LIMIT); + assert!(bytes.len() < BLOB_LIMIT); + let before = serde_json::to_vec(&storage.catalog().unwrap()).unwrap(); + let receipt = storage.save_temporal_candidate(&model).unwrap(); + assert_eq!(receipt.artifact_sha256, digest(&bytes)); + assert_eq!(receipt.input_policy_sha256, model.plan_sha256()); + assert!(receipt.diagnostic_only); + assert!(!receipt.qualification_granted && !receipt.physically_validated); + let path = blob_path(&storage, &receipt); + assert_eq!(fs::read(&path).unwrap(), bytes); + assert!( + private_read(&path).is_err(), + "global64KiB reader was broadened" + ); + assert_eq!( + serde_json::to_vec(&storage.catalog().unwrap()).unwrap(), + before + ); + assert!(storage.load_candidate(&context()).unwrap().is_none()); + assert!(storage.load_selection().unwrap().is_none()); + let cold = store(root.path()); + let loaded = cold.load_temporal_candidate(&context()).unwrap().unwrap(); + assert_eq!(loaded.export().unwrap(), bytes); + assert_eq!(live(&model), live(&loaded)); + assert_eq!( + cold.temporal_candidate_receipt(&context()).unwrap(), + Some(receipt.clone()) + ); + assert_eq!( + cold.load_exact_temporal_candidate(&context(), &receipt.artifact_sha256) + .unwrap() + .export() + .unwrap(), + bytes + ); + // Whole codec includes private weights/policy AND per-row receive/frame/sequence + // lineage. It is NOT aggregate-only; raw IQ/CSI sample arrays are absent. + let value: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert!(value["payload"]["model_json"].is_string()); + assert!(value["payload"]["evidence"][0]["sources"][0]["times_ns"].is_array()); + assert!(value["payload"]["evidence"][0]["sources"][0]["frame_ids"].is_array()); + assert!(value["payload"]["evidence"][0]["sources"][0]["sequence_ids"].is_array()); + assert!(crate::room_learner::model::RoomModel::import(&bytes).is_err()); +} +fn altered(bytes: &[u8], case: &str) -> Vec { + let value: serde_json::Value = serde_json::from_slice(bytes).unwrap(); + let mut text = String::from_utf8(bytes.to_vec()).unwrap(); + match case { + "policy" => { + text = text.replacen("\"target_span_ms\":2000", "\"target_span_ms\":1000", 1); + } + "source" => { + text = text.replacen("\"bandwidth_hz\":20000000", "\"bandwidth_hz\":40000000", 1); + } + "frame" => { + let old = value["payload"]["evidence"][0]["sources"][0]["frame_ids"][0] + .as_str() + .unwrap(); + text = text.replacen( + &format!("\"frame_ids\":[\"{old}\""), + "\"frame_ids\":[\"0\"", + 1, + ); + let a = text.find("\"evidence\":").unwrap() + "\"evidence\":".len(); + let b = text.find(",\"model_sha256\"").unwrap(); + let hash = digest(text[a..b].as_bytes()); + text = text.replacen( + value["payload"]["input_evidence_sha256"].as_str().unwrap(), + &hash, + 1, + ); + } + _ => unreachable!(), + } + let start = text.find("\"payload\":").unwrap() + "\"payload\":".len(); + let hash = digest(text[start..text.len() - 1].as_bytes()); + text = text.replacen(value["payload_sha256"].as_str().unwrap(), &hash, 1); + text.into_bytes() +} +#[test] +fn policy_source_frame_and_expected_context_tampering_cannot_load() { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let model = fitted(); + let receipt = storage.save_temporal_candidate(&model).unwrap(); + let bytes = model.export().unwrap(); + let path = blob_path(&storage, &receipt); + let parent = path.parent().unwrap(); + for case in ["policy", "source", "frame"] { + let bad = altered(&bytes, case); + let hash = digest(&bad); + let target = parent.join(format!("{hash}.json")); + atomic_private_write(&target, &bad).unwrap(); + let error = storage + .load_exact_temporal_candidate(&context(), &hash) + .err() + .expect("tampered policy/source/frame accepted"); + assert!( + !error.contains("temporal_input_envelope_integrity"), + "negative was only noncanonical JSON: {error}" + ); + assert!( + !error.contains("temporal_artifact_hash_mismatch"), + "negative did not reach codec: {error}" + ); + } + let wrong_hash = "2".repeat(64); + atomic_private_write(&parent.join(format!("{wrong_hash}.json")), &bytes).unwrap(); + assert_eq!( + storage + .load_exact_temporal_candidate(&context(), &wrong_hash) + .err() + .unwrap(), + "temporal_artifact_hash_mismatch" + ); + let mut wrong = context(); + wrong.revision += 1; + assert!(storage + .load_exact_temporal_candidate(&wrong, &receipt.artifact_sha256) + .is_err()); + assert!(storage.load_temporal_candidate(&wrong).unwrap().is_none()); + assert_eq!( + storage.temporal_candidate_receipt(&context()).unwrap(), + Some(receipt) + ); + assert_eq!(fs::read(path).unwrap(), bytes); +} +#[test] +fn immutable_corruption_and_pointer_privilege_changes_are_preserved_and_rejected() { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let model = fitted(); + let receipt = storage.save_temporal_candidate(&model).unwrap(); + let path = blob_path(&storage, &receipt); + let original = fs::read(&path).unwrap(); + atomic_private_write(&path, b"synthetic corrupted immutable record").unwrap(); + let corrupt = fs::read(&path).unwrap(); + assert!(storage.save_temporal_candidate(&model).is_err()); + assert_eq!(fs::read(&path).unwrap(), corrupt); + assert!(storage.load_temporal_candidate(&context()).is_err()); + atomic_private_write(&path, &original).unwrap(); + let pointer = path.parent().unwrap().join(POINTER); + let mut forged = receipt.clone(); + forged.qualification_granted = true; + atomic_private_write(&pointer, &encode(&forged).unwrap()).unwrap(); + let pointer_bytes = fs::read(&pointer).unwrap(); + assert!(storage.load_temporal_candidate(&context()).is_err()); + assert!(storage.save_temporal_candidate(&model).is_err()); + assert_eq!(fs::read(&pointer).unwrap(), pointer_bytes); +} +#[test] +fn owner_installation_and_private_size_permissions_are_fail_closed() { + use std::os::unix::fs::PermissionsExt; + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let model = fitted(); + let receipt = storage.save_temporal_candidate(&model).unwrap(); + let wrong = ArtifactStore::new( + &root.path().join("b".repeat(64)).join("SensingServer"), + &"b".repeat(64), + &context().installation_id, + ) + .unwrap(); + assert!(wrong.save_temporal_candidate(&model).is_err()); + assert!(wrong + .load_exact_temporal_candidate(&context(), &receipt.artifact_sha256) + .is_err()); + let install = ArtifactStore::new( + &storage.data_dir, + &context().owner_scope, + "foreign-installation", + ) + .unwrap(); + assert!(install.save_temporal_candidate(&model).is_err()); + let path = blob_path(&storage, &receipt); + fs::set_permissions(&path, fs::Permissions::from_mode(0o644)).unwrap(); + assert!(storage.load_temporal_candidate(&context()).is_err()); + assert_eq!( + fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o644 + ); + fs::set_permissions(&path, fs::Permissions::from_mode(0o600)).unwrap(); + let oversized = path + .parent() + .unwrap() + .join(format!("{}.json", "1".repeat(64))); + let file = fs::File::create(&oversized).unwrap(); + fs::set_permissions(&oversized, fs::Permissions::from_mode(0o600)).unwrap(); + file.set_len((BLOB_LIMIT + 1) as u64).unwrap(); + assert_eq!( + storage + .load_exact_temporal_candidate(&context(), &"1".repeat(64)) + .err() + .unwrap(), + "temporal_artifact_size_limit" + ); + let pointer = path.parent().unwrap().join(POINTER); + let file = fs::File::create(&pointer).unwrap(); + fs::set_permissions(&pointer, fs::Permissions::from_mode(0o600)).unwrap(); + file.set_len((LIMIT + 1) as u64).unwrap(); + assert_eq!( + storage.load_temporal_candidate(&context()).err().unwrap(), + "temporal_artifact_size_limit" + ); +} +#[test] +fn blob_and_namespace_symlinks_never_follow_outside_owned_kind() { + use std::os::unix::fs::symlink; + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let model = fitted(); + let receipt = storage.save_temporal_candidate(&model).unwrap(); + let path = blob_path(&storage, &receipt); + let outside = root.path().join("external-synthetic-blob"); + fs::rename(&path, &outside).unwrap(); + symlink(&outside, &path).unwrap(); + assert!(storage.load_temporal_candidate(&context()).is_err()); + assert!(storage.save_temporal_candidate(&model).is_err()); + assert_eq!(fs::read(&outside).unwrap(), model.export().unwrap()); + fs::remove_file(&path).unwrap(); + fs::rename(&outside, &path).unwrap(); + let namespace = storage.root.join(DIRECTORY); + let moved = storage.root.join("outside-kind-test"); + fs::rename(&namespace, &moved).unwrap(); + symlink(&moved, &namespace).unwrap(); + assert!(storage.load_temporal_candidate(&context()).is_err()); + assert!(storage.save_temporal_candidate(&model).is_err()); +} +fn v1_model_and_seed() -> ( + crate::room_learner::model::RoomModel, + crate::room_learner::managed_training::TrainingProofSeed, +) { + let mut capture = CaptureSession::new(context(), "synthetic-v1-preserved".into()).unwrap(); + for (index, label) in [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ] + .into_iter() + .enumerate() + { + let start = 1_000_000_000 + index as u64 * 13_000_000_000; + capture + .begin_phase( + PhaseRequest { + trial_id: format!("v1-trial-{index}"), + session_id: format!("v1-session-{index}"), + role: DataRole::Train, + label, + duration_ms: 12000, + lead_in_ms: 0, + conditions_confirmed: true, + }, + start, + ) + .unwrap(); + for quarter in 0..4u64 { + for sample in 0..8u32 { + for source in 1..=2u8 { + let values = vec![ + match label { + RoomLabel::Empty => 2., + RoomLabel::AdjacentOnly => 2.3, + _ => 6., + }; + 56 + ]; + capture + .ingest( + RawSourceFrame { + source_id: source, + grid_id: "ant1-bins64-ppdu0-usable-drop2-linear-v2", + frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + sequence: index as u32 * 100 + quarter as u32 * 8 + sample + 1, + amplitudes: &values, + }, + start + + quarter * 3_000_000_000 + + 1_000_000 + + u64::from(sample) * 10_000_000 + + u64::from(source), + ) + .unwrap(); + } + } + } + capture.finish_phase(start + 12_000_000_000).unwrap(); + } + let seed = capture.managed_training_seed().unwrap(); + let (model, _) = crate::room_learner::model::RoomModel::fit( + &capture.validated_dataset().unwrap(), + FitConfig { + epochs: 2, + ..Default::default() + }, + ) + .unwrap(); + (model, seed) +} +#[test] +fn existing_v1_model_proof_fence_pointer_selection_and_catalog_bytes_do_not_change() { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let (v1, seed) = v1_model_and_seed(); + storage.save_managed_candidate(&v1, seed).unwrap(); + let receipt = SelectionReceipt { + schema: "ruview.room-learner.model-selection.v1".into(), + owner_scope: context().owner_scope, + installation_id: context().installation_id, + revision: 1, + selected_at_unix_ms: 1, + selected: None, + previous: None, + }; + let envelope = SelectionEnvelope { + receipt_sha256: crate::room_learner::qualification::hash(&receipt).unwrap(), + receipt, + }; + storage.save_selection(&envelope, 0).unwrap(); + let before = files(&storage.data_dir); + assert!(before + .keys() + .any(|p| p.to_string_lossy().contains("managed-training"))); + assert!(before.keys().any(|p| p.to_string_lossy().contains("fence"))); + let catalog = serde_json::to_vec(&storage.catalog().unwrap()).unwrap(); + let v1bytes = v1.export().unwrap(); + storage.save_temporal_candidate(&fitted()).unwrap(); + for (path, bytes) in before { + assert_eq!(fs::read(storage.data_dir.join(path)).unwrap(), bytes); + } + assert_eq!( + storage + .load_candidate(&context()) + .unwrap() + .unwrap() + .export() + .unwrap(), + v1bytes + ); + assert_eq!(storage.load_selection().unwrap(), Some(envelope)); + assert_eq!( + serde_json::to_vec(&storage.catalog().unwrap()).unwrap(), + catalog + ); +} +#[test] +fn reserved_temporal_kind_does_not_consume_v1_catalog_context_scan_budget() { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + for index in 0..256u32 { + let path = storage.root.join(format!("{index:064x}")); + private_dir(&path).unwrap(); + } + let before = serde_json::to_vec(&storage.catalog().unwrap()).unwrap(); + directory(&storage.root.join(DIRECTORY), true).unwrap(); + assert_eq!( + serde_json::to_vec(&storage.catalog().unwrap()).unwrap(), + before + ); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs new file mode 100644 index 0000000000..25cf3e386c --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs @@ -0,0 +1,636 @@ +//! Typed complete native-prefix payload INSIDE existing RecoveryStore AEAD. +//! This never issues operator confirmation, scientific quality, or fit authority. +use super::*; +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Prefix { + schema: String, + binding: ManagedBinding, + run: String, + descriptor: RecoveryDescriptor, + last_now_ns: u64, + rows: Vec, + evidence: Vec, + identities: Vec, + fences: AssemblerCheckpoint, + source_high: Vec<(u8, u64, u64)>, + ledger: Ledger, + mapping: Option, + episode: usize, + frozen_model: Option, + quality_issued: bool, + confirmation_issued: bool, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Envelope { + schema: String, + payload_sha256: String, + payload: Prefix, +} +const SCHEMA: &str = "ruview.room-learner.temporal-runtime-prefix.v1"; +pub(super) fn encode(v: &TemporalWorkflow) -> Result, String> { + if v.session.fitted + || v.session.failed + || v.session.active + || v.session.rows.iter().any(|r| r.role != DataRole::Train) + || v.session.rows.len() != v.ledger.spans.len() + { + return Err("temporal_checkpoint_complete_train_prefix_only".into()); + } + let m = v + .session + .managed + .as_ref() + .ok_or("managed_session_required")?; + let payload = Prefix { + schema: SCHEMA.into(), + binding: m.binding.clone(), + run: v.session.run_id.clone(), + descriptor: v.descriptor.clone(), + last_now_ns: v.last_now_ns, + rows: v.session.rows.clone(), + evidence: v.session.evidence.clone(), + identities: m.windows.clone(), + fences: v.session.assembler.fence_checkpoint()?, + source_high: v.source_high.clone(), + ledger: v.ledger.clone(), + mapping: v.mapping.clone(), + episode: v.episode, + frozen_model: None, + quality_issued: false, + confirmation_issued: false, + }; + let envelope = Envelope { + schema: SCHEMA.into(), + payload_sha256: hash(&payload)?, + payload, + }; + let bytes = serde_json::to_vec(&envelope).map_err(|_| "temporal_checkpoint_encoding")?; + if bytes.len() > crate::room_learner::recovery_store::MAX_RECOVERY_BYTES { + return Err("temporal_checkpoint_budget".into()); + } + Ok(bytes) +} +pub(super) fn decode( + bytes: &[u8], + descriptor: &RecoveryDescriptor, + now: u64, +) -> Result { + if bytes.len() > crate::room_learner::recovery_store::MAX_RECOVERY_BYTES { + return Err("temporal_checkpoint_budget".into()); + } + let e: Envelope = serde_json::from_slice(bytes).map_err(|_| "temporal_checkpoint_schema")?; + if e.schema != SCHEMA + || e.payload.schema != SCHEMA + || e.payload_sha256 != hash(&e.payload)? + || serde_json::to_vec(&e).map_err(|_| "temporal_checkpoint_encoding")? != bytes + { + return Err("temporal_checkpoint_integrity".into()); + } + let p = e.payload; + let schedule = p.binding.contract()?; + let context = schedule.context(); + if &p.descriptor != descriptor + || p.descriptor.owner_scope != context.owner_scope + || p.descriptor.installation_id != context.installation_id + || p.descriptor.context_digest != context.digest() + || p.last_now_ns / 1_000_000 != descriptor.saved_at_ms + || p.last_now_ns > now + || now / 1_000_000 >= descriptor.expires_at_ms + || descriptor + .expires_at_ms + .checked_sub(descriptor.created_at_ms) + != Some(RECOVERY_TTL_MS) + || p.frozen_model.is_some() + || p.quality_issued + || p.confirmation_issued + || p.episode >= schedule.episodes().len() + || p.rows.len() > schedule.counts().train_windows + || p.rows.len() != p.evidence.len() + || p.rows.len() != p.identities.len() + || p.rows.len() != p.ledger.spans.len() + || p.ledger.gaps.len() > 128 + || p.source_high.len() > context.sources.len() + || (!p.rows.is_empty() && p.source_high.len() != context.sources.len()) + { + return Err("temporal_checkpoint_binding_or_unsupported_state".into()); + } + if let Some(mapping) = &p.mapping { + if mapping.segment_ordinal >= 128 || mapping.receiver_start_ns > p.last_now_ns { + return Err("temporal_checkpoint_mapping".into()); + } + let bound = + schedule.bind_episode_start(&p.run, p.episode, 0, mapping.segment_ordinal, None)?; + let slot = schedule.episodes()[p.episode] + .slots + .iter() + .find(|s| s.target_offset_ms * 1_000_000 >= mapping.observed_start_ns) + .ok_or("temporal_checkpoint_mapping")?; + mapping.slot_bounds( + &bound, + slot.target_offset_ms * 1_000_000, + slot.deadline_offset_ms * 1_000_000, + )?; + let expected_start = p + .ledger + .gaps + .iter() + .rev() + .find(|g| g.episode == p.episode && g.new_mapping == *mapping) + .map_or(0, |g| g.accepted_observed_ns); + if mapping.observed_start_ns != expected_start { + return Err("temporal_checkpoint_mapping_start".into()); + } + } else if !p.rows.is_empty() { + return Err("temporal_checkpoint_mapping_missing".into()); + } + validate_records( + &schedule, + &p.run, + &p.evidence, + &p.identities, + &p.binding.validation_binding()?, + true, + )?; + let mut retained = Vec::new(); + let mut prior: Option = None; + for (i, ((row, evidence), span)) in p + .rows + .iter() + .zip(&p.evidence) + .zip(&p.ledger.spans) + .enumerate() + { + validate_cohort(context, row)?; + if row.views.len() != evidence.sources.len() { + return Err("temporal_checkpoint_row_evidence_shape".into()); + } + for (view, source) in row.views.iter().zip(&evidence.sources) { + if view.source_id != source.source_id + || view.samples.len() != source.times_ns.len() + || view + .frame_ids + .iter() + .map(u128::to_string) + .collect::>() + != source.frame_ids + || view.sequence_ids != source.sequence_ids + || source.times_ns.first() != Some(&view.first_time_ns.to_string()) + || source.times_ns.last() != Some(&view.last_time_ns.to_string()) + { + return Err("temporal_checkpoint_original_row_identity".into()); + } + } + let record = &p.identities[i]; + let episode = &schedule.episodes()[record.episode_index]; + let slot = &episode.slots[record.slot_index]; + if hash(row)? != evidence.cohort_sha256 + || row.id != evidence.cohort_id + || row.trial_id != evidence.trial_id + || row.session_id != evidence.session_id + || row.role != DataRole::Train + || row.label != evidence.label + || span.previous_sha256 != prior + || span.window_sha256 != hash(row)? + || span.mapping != record.mapping + || span.episode != record.episode_index + || span.slot != record.slot_index + || span.observed_end_ns != slot.deadline_offset_ms * 1_000_000 + || span.receiver_end_ns != integer::(&record.end_ns)? + || span.receiver_end_ns > p.last_now_ns + { + return Err("temporal_checkpoint_row_or_span_changed".into()); + } + let expected_start = if i > 0 && p.ledger.spans[i - 1].episode == span.episode { + p.ledger.spans[i - 1].observed_end_ns + } else { + 0 + }; + if span.observed_start_ns != expected_start + || span.observed_end_ns <= span.observed_start_ns + { + return Err("temporal_committed_span_overlap".into()); + } + for source in &evidence.sources { + for ((t, f), seq) in source + .times_ns + .iter() + .zip(&source.frame_ids) + .zip(&source.sequence_ids) + { + retained.push(( + source.source_id, + *seq, + integer::(t)?, + integer::(f)?, + )); + } + } + prior = Some(hash(span)?); + } + for (gap_index, gap) in p.ledger.gaps.iter().enumerate() { + if gap.episode >= schedule.episodes().len() { + return Err("temporal_gap_episode".into()); + } + let bound = schedule.bind_episode_start( + &p.run, + gap.episode, + 0, + gap.new_mapping.segment_ordinal, + None, + )?; + let next_slot = schedule.episodes()[gap.episode] + .slots + .iter() + .find(|s| s.target_offset_ms * 1_000_000 >= gap.accepted_observed_ns) + .ok_or("temporal_gap_mapping_changed")?; + gap.new_mapping.slot_bounds( + &bound, + next_slot.target_offset_ms * 1_000_000, + next_slot.deadline_offset_ms * 1_000_000, + )?; + if let Some(parent) = &gap.after_prefix_sha256 { + if gap.genesis.is_some() { + return Err("temporal_gap_conflicting_parent".into()); + } + let span = p + .ledger + .spans + .iter() + .find(|s| hash(s).ok().as_ref() == Some(parent)) + .ok_or("temporal_gap_prefix_missing")?; + if gap.episode != span.episode + || gap.accepted_observed_ns != span.observed_end_ns + || gap.previous_receiver_ns != span.receiver_end_ns + { + return Err("temporal_gap_mapping_changed".into()); + } + } else { + let genesis = gap.genesis.as_ref().ok_or("temporal_gap_genesis_missing")?; + let previous = &genesis.previous_mapping; + if gap.accepted_observed_ns != 0 + || previous.observed_start_ns != 0 + || genesis.sha256 != genesis.digest(descriptor, &p.run, gap.episode)? + || genesis.paused_receiver_ns < previous.receiver_start_ns + || gap.previous_receiver_ns != genesis.paused_receiver_ns + || gap.new_mapping.segment_ordinal + != previous + .segment_ordinal + .checked_add(1) + .ok_or("temporal_gap_segment_overflow")? + || p.ledger.spans.iter().any(|s| { + s.episode == gap.episode + && s.mapping.segment_ordinal <= previous.segment_ordinal + }) + { + return Err("temporal_gap_invalid_genesis".into()); + } + let prior_bound = schedule.bind_episode_start( + &p.run, + gap.episode, + 0, + previous.segment_ordinal, + None, + )?; + let first = &schedule.episodes()[gap.episode].slots[0]; + previous.slot_bounds( + &prior_bound, + first.target_offset_ms * 1_000_000, + first.deadline_offset_ms * 1_000_000, + )?; + if let Some(prior) = p.ledger.gaps[..gap_index] + .iter() + .rev() + .find(|g| g.episode == gap.episode) + { + if &prior.new_mapping != previous { + return Err("temporal_gap_genesis_history_changed".into()); + } + } else if previous.segment_ordinal != 0 { + return Err("temporal_gap_genesis_history_missing".into()); + } + if genesis.source_floors.len() > context.sources.len() { + return Err("temporal_gap_genesis_floors".into()); + } + let mut seen = HashSet::new(); + for (id, seq, t) in &genesis.source_floors { + if !seen.insert(*id) + || !context.sources.iter().any(|s| s.source_id == *id) + || *seq == 0 + || *t > genesis.paused_receiver_ns + || !p + .source_high + .iter() + .any(|(s, q, n)| s == id && q >= seq && n >= t) + { + return Err("temporal_gap_genesis_floors".into()); + } + } + } + if gap.resume_receiver_ns < gap.previous_receiver_ns + || gap.new_mapping.observed_start_ns != gap.accepted_observed_ns + || gap.new_mapping.receiver_start_ns != gap.resume_receiver_ns + || gap.new_mapping.clock_anchor_ns < gap.previous_receiver_ns + { + return Err("temporal_gap_mapping_changed".into()); + } + } + let mut session = + TemporalTrainingSession::managed(schedule.clone(), p.run, p.binding.fit.clone())?; + session.rows = p.rows; + session.evidence = p.evidence; + session.managed.as_mut().unwrap().windows = p.identities; + session + .assembler + .restore_prefix_fences(p.fences, session.rows.len())?; + if !session.assembler.retained_fences_cover(&retained) { + return Err("temporal_checkpoint_fences_regressed".into()); + } + let mut seen = HashSet::new(); + for (id, seq, t) in &p.source_high { + if !seen.insert(*id) + || !context.sources.iter().any(|s| s.source_id == *id) + || *seq == 0 + || *t > p.last_now_ns + || retained + .iter() + .any(|(s, q, n, _)| s == id && (*q > *seq || *n > *t)) + { + return Err("temporal_checkpoint_source_floor".into()); + } + } + let accepted = p + .ledger + .spans + .last() + .filter(|s| s.episode == p.episode) + .map_or(0, |s| s.observed_end_ns); + let durable_windows = session.rows.len(); + Ok(TemporalWorkflow { + session, + descriptor: p.descriptor, + ledger: p.ledger, + stage: Stage::PausedNeedsReconfirmation, + mapping: p.mapping, + episode: p.episode, + last_now_ns: p.last_now_ns, + live_observed_ns: accepted, + source_high: p.source_high, + pending_window: None, + durable: None, + durable_windows, + custody: SegmentCustody::default(), + commit_token: None, + pending_intent: None, + pending_prefix: None, + }) +} +#[cfg(test)] +#[path = "temporal_checkpoint_tests.rs"] +mod tests; + +/// Opaque, crate-local storage input; generic JSON cannot claim validity/proof. +#[derive(Clone)] +pub(crate) struct ValidatedTemporalPrefix { + bytes: Vec, + descriptor: RecoveryDescriptor, + run: String, + policy_sha256: String, + window_count: usize, + transition_limit: usize, + saved_receiver_ns: u64, +} +pub(crate) fn validated_prefix( + bytes: &[u8], + descriptor: &RecoveryDescriptor, + now_ns: u64, +) -> Result { + let v = decode(bytes, descriptor, now_ns)?; + let schedule = v.schedule()?; + let train_episodes = schedule + .episodes() + .iter() + .filter(|e| e.role == DataRole::Train) + .count(); + // Each64 native Train window +2x128 run-global pause/resume gap barriers +7 Train segment + // boundaries + genesis/terminal states. Identical retries consume none. + let transition_limit = schedule + .counts() + .train_windows + .checked_add(2 * 128) + .and_then(|n| n.checked_add(train_episodes)) + .and_then(|n| n.checked_add(2)) + .ok_or("temporal_transition_capacity")?; + if transition_limit > 512 { + return Err("temporal_transition_capacity".into()); + } + Ok(ValidatedTemporalPrefix { + bytes: bytes.to_vec(), + descriptor: descriptor.clone(), + run: v.session.run_id.clone(), + policy_sha256: v.session.plan_sha256().into(), + window_count: v.session.rows.len(), + transition_limit, + saved_receiver_ns: v.last_now_ns, + }) +} +pub(crate) fn validate_extension( + old: &ValidatedTemporalPrefix, + new: &ValidatedTemporalPrefix, +) -> Result<(), String> { + let a: Envelope = + serde_json::from_slice(&old.bytes).map_err(|_| "temporal_checkpoint_schema")?; + let b: Envelope = + serde_json::from_slice(&new.bytes).map_err(|_| "temporal_checkpoint_schema")?; + let a = a.payload; + let b = b.payload; + if old.run != new.run + || old.policy_sha256 != new.policy_sha256 + || old.transition_limit != new.transition_limit + || old.descriptor.id != new.descriptor.id + || old.descriptor.owner_scope != new.descriptor.owner_scope + || old.descriptor.installation_id != new.descriptor.installation_id + || old.descriptor.context_digest != new.descriptor.context_digest + || old.descriptor.created_at_ms != new.descriptor.created_at_ms + || old.descriptor.expires_at_ms != new.descriptor.expires_at_ms + || new.descriptor.saved_at_ms < old.descriptor.saved_at_ms + || b.last_now_ns < a.last_now_ns + { + return Err("temporal_commit_identity_or_time_changed".into()); + } + macro_rules! prefix { + ($old:expr,$new:expr) => { + if $new.len() < $old.len() + || serde_json::to_vec(&$new[..$old.len()]).map_err(|_| "temporal_encoding")? + != serde_json::to_vec(&$old).map_err(|_| "temporal_encoding")? + { + return Err("temporal_committed_prefix_changed".into()); + } + }; + } + prefix!(a.rows, b.rows); + prefix!(a.evidence, b.evidence); + prefix!(a.identities, b.identities); + prefix!(a.ledger.spans, b.ledger.spans); + prefix!(a.ledger.gaps, b.ledger.gaps); + for (id, seq, t) in &a.source_high { + if !b + .source_high + .iter() + .any(|(s, q, n)| s == id && q >= seq && n >= t) + { + return Err("temporal_commit_floor_regressed".into()); + } + } + if old.bytes == new.bytes { + return Ok(()); + } + let window = b.rows.len() == a.rows.len() + 1 + && b.ledger.gaps.len() == a.ledger.gaps.len() + && b.mapping == a.mapping; + let segment = b.rows.len() == a.rows.len() + && ((a.mapping.is_none() + && b.mapping.is_some() + && b.ledger.gaps.len() == a.ledger.gaps.len()) + || (b.ledger.gaps.len() == a.ledger.gaps.len() + 1)); + if !window && !segment { + return Err("temporal_transition_without_window_or_segment".into()); + } + Ok(()) +} + +impl ValidatedTemporalPrefix { + pub(crate) fn saved_receiver_ns(&self) -> u64 { + self.saved_receiver_ns + } + pub(crate) fn bytes(&self) -> &[u8] { + &self.bytes + } + pub(crate) fn descriptor(&self) -> &RecoveryDescriptor { + &self.descriptor + } + pub(crate) fn run(&self) -> &str { + &self.run + } + pub(crate) fn policy_sha256(&self) -> &str { + &self.policy_sha256 + } + pub(crate) fn window_count(&self) -> usize { + self.window_count + } + pub(crate) fn transition_limit(&self) -> usize { + self.transition_limit + } +} + +pub(crate) fn is_new_empty_run(p: &ValidatedTemporalPrefix) -> Result { + let e: Envelope = + serde_json::from_slice(p.bytes()).map_err(|_| "temporal_checkpoint_schema")?; + let q = e.payload; + let schedule = q.binding.contract()?; + let fresh = TemporalTrainingSession::managed(schedule, q.run.clone(), q.binding.fit.clone())?; + Ok(q.rows.is_empty() + && q.evidence.is_empty() + && q.identities.is_empty() + && q.ledger.spans.is_empty() + && q.ledger.gaps.is_empty() + && q.source_high.is_empty() + && q.mapping.is_none() + && q.episode == 0 + && serde_json::to_vec(&q.fences).map_err(|_| "temporal_encoding")? + == serde_json::to_vec(&fresh.assembler.fence_checkpoint()?) + .map_err(|_| "temporal_encoding")?) +} + +#[cfg(test)] +pub(crate) fn synthetic_storage_prefixes() -> Vec { + let (_dir, store, mut w) = super::tests::setup(); + let mut out = Vec::new(); + let add = |w: &TemporalWorkflow| { + let bytes = encode(w).unwrap(); + validated_prefix(&bytes, &w.descriptor, w.last_now_ns).unwrap() + }; + out.push(add(&w)); + w.persist_initial(&store).unwrap(); + super::tests::start(&mut w, &store, 2_000_000_000, "one"); + out.push(add(&w)); + let mut seq = 0; + let next = super::tests::commit_next(&mut w, &store, 2_000_000_000, &mut seq); + out.push(add(&w)); + super::tests::commit_next(&mut w, &store, next, &mut seq); + out.push(add(&w)); + out +} +#[cfg(test)] +pub(crate) fn synthetic_segment_fork(initial: &ValidatedTemporalPrefix) -> ValidatedTemporalPrefix { + let mut w = decode( + initial.bytes(), + initial.descriptor(), + initial.descriptor().saved_at_ms * 1_000_000, + ) + .unwrap(); + w.stage = Stage::DurableReady; + w.durable = Some(initial.bytes().to_vec()); + let p = w.synthetic_conditions(); + w.start_segment(&p, "different-segment", 2_000_000_000, 2_000_000_000) + .unwrap(); + w.descriptor.saved_at_ms = w.last_now_ns / 1_000_000; + let b = encode(&w).unwrap(); + validated_prefix(&b, &w.descriptor, w.last_now_ns).unwrap() +} +#[cfg(test)] +pub(crate) fn synthetic_changed_old_row( + input: &ValidatedTemporalPrefix, +) -> ValidatedTemporalPrefix { + let mut e: Envelope = serde_json::from_slice(input.bytes()).unwrap(); + e.payload.rows[0].views[0].samples[0][0] += 1.; + e.payload.evidence[0].cohort_sha256 = hash(&e.payload.rows[0]).unwrap(); + e.payload.ledger.spans[0].window_sha256 = hash(&e.payload.rows[0]).unwrap(); + for i in 1..e.payload.ledger.spans.len() { + e.payload.ledger.spans[i].previous_sha256 = + Some(hash(&e.payload.ledger.spans[i - 1]).unwrap()); + } + e.payload_sha256 = hash(&e.payload).unwrap(); + let b = serde_json::to_vec(&e).unwrap(); + validated_prefix(&b, &e.payload.descriptor, e.payload.last_now_ns).unwrap() +} +#[cfg(test)] +pub(crate) fn synthetic_new_empty_run( + input: &ValidatedTemporalPrefix, + created: u64, +) -> ValidatedTemporalPrefix { + let mut e: Envelope = serde_json::from_slice(input.bytes()).unwrap(); + assert!(e.payload.rows.is_empty()); + e.payload.run = "synthetic-next-run".into(); + e.payload.descriptor.id = "synthetic-new-recovery-id".into(); + e.payload.descriptor.created_at_ms = created; + e.payload.descriptor.saved_at_ms = created; + e.payload.descriptor.expires_at_ms = created + RECOVERY_TTL_MS; + e.payload.last_now_ns = created * 1_000_000; + e.payload_sha256 = hash(&e.payload).unwrap(); + let b = serde_json::to_vec(&e).unwrap(); + validated_prefix(&b, &e.payload.descriptor, e.payload.last_now_ns).unwrap() +} +#[cfg(test)] +pub(crate) fn synthetic_floor_regression( + input: &ValidatedTemporalPrefix, +) -> ValidatedTemporalPrefix { + let mut w = decode(input.bytes(), input.descriptor(), input.saved_receiver_ns()).unwrap(); + w.durable = Some(input.bytes().to_vec()); + let permit = w.synthetic_conditions(); + let start = input.saved_receiver_ns() + 10_000_000_000; + w.start_segment(&permit, "floor-regression", start, start) + .unwrap(); + w.descriptor.saved_at_ms = w.last_now_ns / 1_000_000; + let b = encode(&w).unwrap(); + let mut e: Envelope = serde_json::from_slice(&b).unwrap(); + let source = &e.payload.evidence.last().unwrap().sources[0]; + let minimum = *source.sequence_ids.last().unwrap(); + let old = e.payload.source_high[0].1; + assert!(old > minimum); + e.payload.source_high[0].1 = minimum; + e.payload_sha256 = hash(&e.payload).unwrap(); + let b = serde_json::to_vec(&e).unwrap(); + validated_prefix(&b, &e.payload.descriptor, e.payload.last_now_ns).unwrap() +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint_tests.rs new file mode 100644 index 0000000000..ef98c66ca9 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint_tests.rs @@ -0,0 +1,209 @@ +use super::super::tests::{commit_next, setup, start}; +use super::*; +#[test] +fn real_aead_initial_barrier_and_ambiguous_save_fail_closed_without_new_authority() { + let (_dir, store, mut w) = setup(); + assert!(w + .persist_with(&store, false, |_, _, _, _| Err( + "injected-before-save".into() + )) + .is_err()); + assert_eq!(w.stage, Stage::BlockedDurability); + assert!(store.load_temporal(w.last_now_ns).unwrap().is_none()); + assert!(w.begin_slot().is_err()); + let (_dir, store, mut w) = setup(); + let result = w.persist_with(&store, false, |s, d, b, n| { + s.publish_temporal(d, Some(b), n)?; + Err("injected-after-real-save-before-ack".into()) + }); + assert!(result.is_err()); + assert_eq!(w.stage, Stage::BlockedDurability); + assert!(w.begin_slot().is_err()); + let loaded = store + .load_temporal(w.last_now_ns) + .unwrap() + .unwrap() + .prefix + .unwrap(); + assert!(decode(loaded.bytes(), loaded.descriptor(), w.last_now_ns).is_ok()); + let c = crate::room_learner::temporal_model_input::managed::tests::context(); + let encrypted = std::fs::read( + _dir.path() + .join(&c.owner_scope) + .join("SensingServer/room-learner-recovery/temporal-prefix-commitment-v1/slot-0.bin"), + ) + .unwrap(); + assert!(encrypted.starts_with(b"RVTPAY01")); + assert!(serde_json::from_slice::(&encrypted).is_err()); + assert!(!encrypted + .windows(SCHEMA.len()) + .any(|w| w == SCHEMA.as_bytes())); +} +#[test] +fn rehashed_checkpoint_role_policy_epoch_fence_expiry_and_fake_receipt_tampering_fail() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "one"); + let mut seq = 0; + commit_next(&mut w, &store, 2_000_000_000, &mut seq); + let bytes = w.durable.as_ref().unwrap(); + for case in 0..9 { + let mut e: Envelope = serde_json::from_slice(bytes).unwrap(); + match case { + 0 => e.payload.rows[0].role = DataRole::HeldOut, + 1 => e.payload.binding.target_mapping = "old-uninterrupted".into(), + 2 => e.payload.identities[0].mapping.receiver_epoch = "".into(), + 3 => e.payload.source_high[0].1 = 1, + 4 => e.payload.descriptor.expires_at_ms += 1, + 5 => e.payload.frozen_model = Some("a".repeat(64)), + 6 => e.payload.quality_issued = true, + 7 => e.payload.ledger.spans[0].observed_start_ns = 1, + _ => { + e.payload.rows[0].views[0].frame_ids[0] += 1; + e.payload.evidence[0].cohort_sha256 = hash(&e.payload.rows[0]).unwrap(); + e.payload.ledger.spans[0].window_sha256 = hash(&e.payload.rows[0]).unwrap(); + } + }; + e.payload_sha256 = hash(&e.payload).unwrap(); + assert!( + decode( + &serde_json::to_vec(&e).unwrap(), + &w.descriptor, + w.last_now_ns + ) + .is_err(), + "case{case}" + ); + } + let expired = w.descriptor.expires_at_ms * 1_000_000; + assert!(decode(bytes, &w.descriptor, expired).is_err()); +} + +#[test] +fn actual_encrypted_file_corruption_during_readback_blocks_collection() { + let (dir, store, mut w) = setup(); + let c = crate::room_learner::temporal_model_input::managed::tests::context(); + let file = dir + .path() + .join(&c.owner_scope) + .join("SensingServer/room-learner-recovery/temporal-prefix-commitment-v1/slot-0.bin"); + assert!(w + .persist_with(&store, false, |s, d, b, n| { + let token = s.publish_temporal(d, Some(b), n)?; + let mut encrypted = std::fs::read(&file).unwrap(); + let i = encrypted.len() - 1; + encrypted[i] ^= 1; + std::fs::write(&file, encrypted).unwrap(); + Ok(token) + }) + .is_err()); + assert_eq!(w.stage, Stage::BlockedDurability); + assert!(w.begin_slot().is_err()); + assert!(store.load_temporal(w.last_now_ns).is_err()); +} + +#[test] +fn failed_completed_window_barrier_never_advances_promised_durable_prefix() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "one"); + let mut seq = 0; + let target = w.begin_slot().unwrap(); + super::super::tests::drive(&mut w, 2_000_000_000, target.end_ns(), &mut seq); + assert!(w + .persist_with(&store, true, |_, _, _, _| Err( + "injected-window-save-failure".into() + )) + .is_err()); + assert_eq!(w.accepted(), 0); + assert_eq!(w.stage, Stage::BlockedDurability); +} + +#[test] +fn credited_prefix_cannot_be_replaced_by_a_rehashed_genesis_parent() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "initial"); + let mut seq = 0; + let next = commit_next(&mut w, &store, 2_000_000_000, &mut seq); + w.pause().unwrap(); + start(&mut w, &store, next + 10_000_000_000, "resumed"); + let mut e: Envelope = serde_json::from_slice(w.durable.as_ref().unwrap()).unwrap(); + let old_mapping = e.payload.ledger.spans[0].mapping.clone(); + let paused = e.payload.last_now_ns; + let anchor = GenesisGap::new( + &e.payload.descriptor, + &e.payload.run, + 0, + old_mapping, + paused, + e.payload.source_high.clone(), + ) + .unwrap(); + let gap = &mut e.payload.ledger.gaps[0]; + gap.after_prefix_sha256 = None; + gap.genesis = Some(anchor); + gap.accepted_observed_ns = 0; + gap.previous_receiver_ns = paused; + gap.new_mapping.observed_start_ns = 0; + e.payload.mapping = Some(gap.new_mapping.clone()); + e.payload_sha256 = hash(&e.payload).unwrap(); + let err = decode( + &serde_json::to_vec(&e).unwrap(), + &w.descriptor, + w.last_now_ns, + ) + .err() + .expect("missing parent after credited prefix must reject"); + assert_eq!(err, "temporal_gap_invalid_genesis"); +} + +#[test] +fn zero_prefix_anchor_binds_original_run_clock_and_source_floor_history() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "initial"); + let target = w.begin_slot().unwrap(); + let mut seq = 0; + let next = super::super::tests::drive( + &mut w, + 2_000_000_000, + target.target_ns() + 1_000_000_000, + &mut seq, + ); + w.pause().unwrap(); + start(&mut w, &store, next + 10_000_000_000, "resumed"); + let bytes = w.durable.as_ref().unwrap(); + for case in 0..4 { + let mut e: Envelope = serde_json::from_slice(bytes).unwrap(); + match case { + 0 => e.payload.run = "foreign-run".into(), + 1 => { + e.payload.ledger.gaps[0] + .genesis + .as_mut() + .unwrap() + .paused_receiver_ns = 0 + } + 2 => { + e.payload.ledger.gaps[0] + .genesis + .as_mut() + .unwrap() + .source_floors[0] + .1 = 1 + } + _ => e.payload.ledger.gaps[0].genesis = None, + }; + e.payload_sha256 = hash(&e.payload).unwrap(); + assert!( + decode( + &serde_json::to_vec(&e).unwrap(), + &w.descriptor, + w.last_now_ns + ) + .is_err(), + "case{case}" + ); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs new file mode 100644 index 0000000000..ce3a8f5ba8 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs @@ -0,0 +1,769 @@ +//! Private future ONE Runtime Temporal workflow; no public constructor/API issuer. +//! Durability is not scientific qualification or an assertion of whole-trial quality. +use super::*; +use crate::room_learner::recovery_store::temporal_commitment::{ + TemporalAbortTicket, TemporalIntent, TemporalToken, +}; +use crate::room_learner::{ + managed_schedule_contract::BoundEpisode, + recovery_store::{RecoveryDescriptor, RecoveryStore, RECOVERY_TTL_MS}, + temporal_acquisition::{AssemblerCheckpoint, ManagedWindowTarget}, +}; +use serde::{Deserialize, Serialize}; +pub(crate) const EXECUTION_POLICY:&str="observed-complete-reservation-prefix-v1;paused-time;immutable-spans;partial-tail-drop;max128segments;persegment-native-firstarrival-skew-gaps;explicit-genesis-gap-anchor;quality-unimplemented;independence-false"; +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct ObservedSegmentMap { + pub(crate) segment_ordinal: u16, + pub(crate) session_id: String, + pub(crate) receiver_epoch: String, + pub(crate) clock_anchor_ns: u64, + pub(crate) receiver_start_ns: u64, + pub(crate) observed_start_ns: u64, +} +impl ObservedSegmentMap { + pub(crate) fn slot_bounds( + &self, + b: &BoundEpisode, + a: u64, + z: u64, + ) -> Result<(u64, u64), String> { + if self.segment_ordinal != b.segment_ordinal() + || self.session_id != b.session_id() + || self.receiver_epoch.is_empty() + || self.receiver_epoch.len() > 128 + || self.receiver_start_ns == 0 + || self.clock_anchor_ns == 0 + || self.receiver_start_ns < self.clock_anchor_ns + || a < self.observed_start_ns + || z <= a + || z > b.episode().target_active_recording_ms * 1_000_000 + { + return Err("managed_observed_segment_mapping".into()); + } + Ok(( + self.receiver_start_ns + .checked_add(a - self.observed_start_ns) + .ok_or("managed_mapping_overflow")?, + self.receiver_start_ns + .checked_add(z - self.observed_start_ns) + .ok_or("managed_mapping_overflow")?, + )) + } + #[cfg(test)] + pub(crate) fn synthetic_uninterrupted(b: &BoundEpisode, start: u64) -> Self { + Self { + segment_ordinal: b.segment_ordinal(), + session_id: b.session_id().into(), + receiver_epoch: "synthetic-uninterrupted".into(), + clock_anchor_ns: start, + receiver_start_ns: start, + observed_start_ns: 0, + } + } +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct CommittedSpan { + episode: usize, + slot: usize, + mapping: ObservedSegmentMap, + observed_start_ns: u64, + observed_end_ns: u64, + receiver_end_ns: u64, + previous_sha256: Option, + window_sha256: String, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct GenesisGap { + previous_mapping: ObservedSegmentMap, + paused_receiver_ns: u64, + source_floors: Vec<(u8, u64, u64)>, + sha256: String, +} +impl GenesisGap { + fn digest( + &self, + descriptor: &RecoveryDescriptor, + run: &str, + episode: usize, + ) -> Result { + hash(&( + "temporal-zero-prefix-genesis-v1", + EXECUTION_POLICY, + &descriptor.id, + &descriptor.owner_scope, + &descriptor.installation_id, + &descriptor.context_digest, + descriptor.created_at_ms, + descriptor.expires_at_ms, + run, + episode, + &self.previous_mapping, + self.paused_receiver_ns, + &self.source_floors, + )) + } + fn new( + descriptor: &RecoveryDescriptor, + run: &str, + episode: usize, + previous_mapping: ObservedSegmentMap, + paused_receiver_ns: u64, + source_floors: Vec<(u8, u64, u64)>, + ) -> Result { + let mut v = Self { + previous_mapping, + paused_receiver_ns, + source_floors, + sha256: String::new(), + }; + v.sha256 = v.digest(descriptor, run, episode)?; + Ok(v) + } +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct Gap { + episode: usize, + after_prefix_sha256: Option, + genesis: Option, + accepted_observed_ns: u64, + previous_receiver_ns: u64, + resume_receiver_ns: u64, + discarded_live_ns: u64, + new_mapping: ObservedSegmentMap, +} +#[derive(Clone, Default, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct Ledger { + spans: Vec, + gaps: Vec, +} +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum Stage { + Prepared, + DurableReady, + Observing, + PausedNeedsReconfirmation, + PreparedSegment, + BlockedDurability, +} +/// Neither JSON nor a generic saved checkpoint can construct an operator permit. +struct ConditionsPermit { + context: String, + run: String, + durable_sha256: Option, +} +/// Transient per-segment observation custody, distinct from persistent replay +/// high-watermarks. It is NEVER restored as active observation from JSON. +#[derive(Default)] +struct SegmentCustody { + first_last: Vec<(u8, u64, u64)>, + failed: bool, +} +/// Future value owned by existing RoomLearningRuntime::Inner's ONE tagged workflow. +/// No production constructor/permit issuer is exposed by this source slice. +struct TemporalWorkflow { + session: TemporalTrainingSession, + descriptor: RecoveryDescriptor, + ledger: Ledger, + stage: Stage, + mapping: Option, + episode: usize, + last_now_ns: u64, + live_observed_ns: u64, + source_high: Vec<(u8, u64, u64)>, + pending_window: Option, + durable: Option>, + durable_windows: usize, + custody: SegmentCustody, + commit_token: Option, + pending_intent: Option, + pending_prefix: Option, +} +impl TemporalWorkflow { + fn context(&self) -> &CaptureContext { + &self.session.binding.context + } + fn schedule(&self) -> Result { + self.session + .managed + .as_ref() + .ok_or("managed_session_required")? + .binding + .contract() + } + fn prefix_hash(&self) -> Result, String> { + self.ledger.spans.last().map(hash).transpose() + } + fn accepted(&self) -> u64 { + self.ledger + .spans + .iter() + .take(self.durable_windows) + .last() + .filter(|s| s.episode == self.episode) + .map_or(0, |s| s.observed_end_ns) + } + fn check_clock(&mut self, now: u64) -> Result<(), String> { + if now < self.last_now_ns || now / 1_000_000 >= self.descriptor.expires_at_ms { + return Err("temporal_receiver_clock_or_expiry".into()); + } + self.last_now_ns = now; + Ok(()) + } + #[cfg(test)] + fn prepare( + schedule: ManagedScheduleContract, + run: String, + config: FitConfig, + now: u64, + ) -> Result { + if now == 0 { + return Err("temporal_receiver_clock".into()); + } + let c = schedule.context().clone(); + let digest = c.digest(); + let session = TemporalTrainingSession::managed(schedule, run, config)?; + Ok(Self { + session, + descriptor: RecoveryDescriptor { + schema: crate::room_learner::recovery_store::RECOVERY_SCHEMA.into(), + id: crate::companion::secret(), + owner_scope: c.owner_scope, + installation_id: c.installation_id, + context_digest: digest, + created_at_ms: now / 1_000_000, + saved_at_ms: now / 1_000_000, + expires_at_ms: now / 1_000_000 + RECOVERY_TTL_MS, + }, + ledger: Ledger::default(), + stage: Stage::Prepared, + mapping: None, + episode: 0, + last_now_ns: now, + live_observed_ns: 0, + source_high: vec![], + pending_window: None, + durable: None, + durable_windows: 0, + custody: SegmentCustody::default(), + commit_token: None, + pending_intent: None, + pending_prefix: None, + }) + } + #[cfg(test)] + fn synthetic_conditions(&self) -> ConditionsPermit { + ConditionsPermit { + context: self.context().digest(), + run: self.session.run_id.clone(), + durable_sha256: self.durable.as_ref().map(|b| bytes_hash(b)), + } + } + fn start_segment( + &mut self, + permit: &ConditionsPermit, + epoch: &str, + anchor: u64, + start: u64, + ) -> Result<(), String> { + if !matches!( + self.stage, + Stage::DurableReady | Stage::PausedNeedsReconfirmation + ) || permit.context != self.context().digest() + || permit.run != self.session.run_id + || permit.durable_sha256 != self.durable.as_ref().map(|b| bytes_hash(b)) + || epoch.is_empty() + || epoch.len() > 128 + || anchor < self.last_now_ns + || start < anchor + || self.ledger.gaps.len() >= 128 + { + return Err("temporal_reconfirmation_or_clock_required".into()); + } + let paused_receiver_ns = self.last_now_ns; + self.check_clock(start)?; + let schedule = self.schedule()?; + let segment = self.mapping.as_ref().map_or(0, |m| m.segment_ordinal + 1); + let bound = + schedule.bind_episode_start(&self.session.run_id, self.episode, 0, segment, None)?; + let mapping = ObservedSegmentMap { + segment_ordinal: segment, + session_id: bound.session_id().into(), + receiver_epoch: epoch.into(), + clock_anchor_ns: anchor, + receiver_start_ns: start, + observed_start_ns: self.accepted(), + }; + if let Some(old) = &self.mapping { + let parent = self.prefix_hash()?; + let genesis = if parent.is_none() { + Some(GenesisGap::new( + &self.descriptor, + &self.session.run_id, + self.episode, + old.clone(), + paused_receiver_ns, + self.source_high.clone(), + )?) + } else { + None + }; + let prev = self + .ledger + .spans + .last() + .map_or(paused_receiver_ns, |s| s.receiver_end_ns); + self.ledger.gaps.push(Gap { + episode: self.episode, + after_prefix_sha256: parent, + genesis, + accepted_observed_ns: self.accepted(), + previous_receiver_ns: prev, + resume_receiver_ns: start, + discarded_live_ns: self.live_observed_ns.saturating_sub(self.accepted()), + new_mapping: mapping.clone(), + }); + } + self.custody = SegmentCustody::default(); + self.mapping = Some(mapping); + self.live_observed_ns = self.accepted(); + self.stage = Stage::PreparedSegment; + Ok(()) + } + + fn persist_segment(&mut self, store: &RecoveryStore) -> Result<(), String> { + if self.stage != Stage::PreparedSegment { + return Err("temporal_segment_not_prepared".into()); + } + self.persist_with(store, false, |s, intent, prefix, now_ns| { + s.publish_temporal(intent, Some(prefix), now_ns) + })?; + self.stage = Stage::Observing; + Ok(()) + } + fn begin_slot(&mut self) -> Result { + if self.stage != Stage::Observing || self.pending_window.is_some() { + return Err("temporal_slot_state".into()); + } + let schedule = self.schedule()?; + let mapping = self.mapping.clone().ok_or("temporal_segment_missing")?; + let bound = schedule.bind_episode_start( + &self.session.run_id, + self.episode, + 0, + mapping.segment_ordinal, + None, + )?; + let slot = self.session.rows.len() + - schedule + .episodes() + .iter() + .take(self.episode) + .map(|e| e.slots.len()) + .sum::(); + let target = ManagedWindowTarget::from_observed_episode( + &schedule, + &bound, + &self.session.run_id, + slot, + mapping, + )?; + self.session.begin_managed_window(&target)?; + self.pending_window = Some(target.clone()); + Ok(target) + } + /// Busy packets and trusted receiver ticks both withdraw custody when a + /// required source is absent/stale; neither can backfill missing intervals. + fn custody_at(&mut self, now: u64) -> Result<(), String> { + if self.stage != Stage::Observing || self.custody.failed { + return Err("temporal_segment_custody_unavailable".into()); + } + let mapping = self.mapping.as_ref().ok_or("temporal_segment_missing")?; + let native = &self + .session + .managed + .as_ref() + .ok_or("managed_session_required")? + .binding + .native; + let first_limit = native + .max_first_packet_delay_ms() + .checked_mul(1_000_000) + .ok_or("temporal_time_overflow")?; + if now < mapping.receiver_start_ns { + return Err("temporal_segment_clock".into()); + } + for source in native.source_policies() { + let invalid = match self + .custody + .first_last + .iter() + .find(|(id, _, _)| *id == source.source_id) + { + None => now - mapping.receiver_start_ns > first_limit, + Some((_, _, last)) => now < *last || now - *last > source.max_gap_ns, + }; + if invalid { + self.custody.failed = true; + return Err("temporal_missing_segment_source".into()); + } + } + Ok(()) + } + fn observe_tick(&mut self, now: u64) -> Result<(), String> { + self.check_clock(now)?; + self.custody_at(now) + } + fn observe(&mut self, p: &CanonicalPacket, now: u64) -> Result<(), String> { + if self.stage != Stage::Observing { + return Err("temporal_not_observing".into()); + } + self.check_clock(now)?; + self.custody_at(now)?; + let source_count = self.context().sources.len(); + let c = self.context(); + let spec = c + .sources + .iter() + .find(|s| s.source_id == p.source_id) + .ok_or("temporal_unbound_source")?; + let t = p.native_frame.timestamp_ns; + if t > now + || p.grid_id != spec.grid_id + || p.frequency_hz != spec.center_frequency_hz + || p.bandwidth_hz != spec.bandwidth_hz + || p.native_frame.provenance.device_id != spec.device_id + || p.amplitudes.len() != spec.feature_count + || p.amplitudes.iter().any(|v| !v.is_finite() || *v < 0.) + || self.source_high.iter().any(|(id, seq, last)| { + *id == p.source_id && (u64::from(p.sequence) <= *seq || t <= *last) + }) + { + return Err("temporal_observation_source_or_fence".into()); + } + let mapping = self.mapping.as_ref().ok_or("temporal_segment_missing")?; + if t < mapping.receiver_start_ns { + return Err("temporal_segment_replay".into()); + } + let native = &self + .session + .managed + .as_ref() + .ok_or("managed_session_required")? + .binding + .native; + let policy = native + .source_policies() + .iter() + .find(|s| s.source_id == p.source_id) + .ok_or("temporal_unbound_source")?; + if self.source_high.iter().any(|(id, _, last)| { + *id == p.source_id + && *last >= mapping.receiver_start_ns + && (t - *last > policy.max_gap_ns || t - *last < policy.min_packet_interval_ns) + }) { + return Err("temporal_observed_gap_or_rate".into()); + } + let first_limit = native.max_first_packet_delay_ms() * 1_000_000; + if let Some((_, _, last)) = self + .custody + .first_last + .iter_mut() + .find(|(id, _, _)| *id == p.source_id) + { + *last = t; + } else { + if t - mapping.receiver_start_ns > first_limit { + self.custody.failed = true; + return Err("temporal_late_segment_source".into()); + } + self.custody.first_last.push((p.source_id, t, t)); + } + let first_min = self + .custody + .first_last + .iter() + .map(|(_, first, _)| *first) + .min() + .unwrap(); + let first_max = self + .custody + .first_last + .iter() + .map(|(_, first, _)| *first) + .max() + .unwrap(); + if first_max - first_min > native.max_pairwise_receiver_skew_ms() * 1_000_000 { + self.custody.failed = true; + return Err("temporal_segment_first_source_skew".into()); + } + self.source_high.retain(|(id, _, _)| *id != p.source_id); + self.source_high + .push((p.source_id, u64::from(p.sequence), t)); + self.source_high.sort(); + if self.custody.first_last.len() == source_count { + let frontier = self + .custody + .first_last + .iter() + .map(|(_, _, t)| *t) + .min() + .unwrap(); + if frontier >= mapping.receiver_start_ns { + self.live_observed_ns = + mapping.observed_start_ns + frontier - mapping.receiver_start_ns; + } + } + if self.session.active { + let target = self + .pending_window + .as_ref() + .ok_or("temporal_target_missing")?; + if t >= target.target_ns() { + if self.session.ingest(p)? == WindowAdmission::Ready { + self.session.finish_window(t)?; + } + } + } + Ok(()) + } + fn pause(&mut self) -> Result<(), String> { + if self.stage != Stage::Observing { + return Err("temporal_pause_state".into()); + } + let bytes = self + .durable + .clone() + .ok_or("temporal_durable_prefix_missing")?; + let ram = self.session.assembler.fence_checkpoint()?; + let high = self.source_high.clone(); + let mapping = self.mapping.clone(); + let live = self.live_observed_ns; + let last = self.last_now_ns; + let token = self.commit_token.clone(); + let mut restored = checkpoint::decode(&bytes, &self.descriptor, last)?; + restored.commit_token = token; + restored.session.assembler.restore_prefix_fences( + ram.for_prefix(restored.session.rows.len())?, + restored.session.rows.len(), + )?; + restored.mapping = mapping; + restored.live_observed_ns = live; + restored.source_high = high; + restored.last_now_ns = last; + restored.stage = Stage::PausedNeedsReconfirmation; + restored.durable = Some(bytes); + *self = restored; + Ok(()) + } + fn persist_initial(&mut self, store: &RecoveryStore) -> Result<(), String> { + if self.stage != Stage::Prepared { + return Err("temporal_prepare_state".into()); + } + self.persist_with(store, false, |store, intent, prefix, now_ns| { + store.publish_temporal(intent, Some(prefix), now_ns) + })?; + self.stage = Stage::DurableReady; + Ok(()) + } + fn commit_slot(&mut self, store: &RecoveryStore, now: u64) -> Result<(), String> { + self.check_clock(now)?; + self.persist_with(store, true, |store, intent, prefix, now_ns| { + store.publish_temporal(intent, Some(prefix), now_ns) + })?; + self.last_now_ns = now.max(self.last_now_ns); + Ok(()) + } + fn persist_with( + &mut self, + store: &RecoveryStore, + slot: bool, + save: impl FnOnce( + &RecoveryStore, + &TemporalIntent, + &checkpoint::ValidatedTemporalPrefix, + u64, + ) -> Result, + ) -> Result<(), String> { + if slot { + self.custody_at(self.last_now_ns)?; + let t = self + .pending_window + .as_ref() + .ok_or("temporal_no_completed_window")?; + let schedule = self.schedule()?; + let e = &schedule.episodes()[self.episode]; + let s = &e.slots[t.identity().slot_index]; + if self.stage != Stage::Observing + || self.session.active + || self.session.rows.len() != self.ledger.spans.len() + 1 + || self.last_now_ns < t.end_ns() + || self.source_high.len() != self.context().sources.len() + || self.source_high.iter().any(|(_, _, n)| *n < t.end_ns()) + { + return Err("temporal_full_reservation_not_observed".into()); + } + let mapping = t.identity().mapping.clone(); + let accepted = self.accepted(); + let span = CommittedSpan { + episode: self.episode, + slot: s.index, + mapping, + observed_start_ns: accepted, + observed_end_ns: s.deadline_offset_ms * 1_000_000, + receiver_end_ns: t.end_ns(), + previous_sha256: self.prefix_hash()?, + window_sha256: hash(self.session.rows.last().unwrap())?, + }; + self.ledger.spans.push(span); + } + self.descriptor.saved_at_ms = self.last_now_ns / 1_000_000; + let bytes = checkpoint::encode(self)?; + let prefix = checkpoint::validated_prefix(&bytes, &self.descriptor, self.last_now_ns)?; + let intent = + match store.prepare_temporal(self.commit_token.as_ref(), &prefix, self.last_now_ns) { + Ok(i) => i, + Err(e) => { + self.stage = Stage::BlockedDurability; + return Err(e); + } + }; + self.pending_intent = Some(intent.clone()); + self.pending_prefix = Some(prefix.clone()); + let token = match save(store, &intent, &prefix, self.last_now_ns).and_then(|token| { + let loaded = store + .load_temporal(self.last_now_ns)? + .ok_or("temporal_readback_missing")?; + if loaded.token != token + || loaded.prefix.as_ref().map(|p| p.bytes()) != Some(prefix.bytes()) + { + return Err("temporal_readback_changed".into()); + } + Ok(token) + }) { + Ok(token) => token, + Err(e) => { + self.stage = Stage::BlockedDurability; + return Err(e); + } + }; + self.commit_token = Some(token); + self.pending_intent = None; + self.pending_prefix = None; + self.durable_windows = self.session.rows.len(); + self.durable = Some(bytes); + if slot { + self.pending_window = None; + } + Ok(()) + } + fn abort_pending_artifact( + &mut self, + store: &RecoveryStore, + ticket: &TemporalAbortTicket, + now_ns: u64, + ) -> Result<(), String> { + if self.stage != Stage::BlockedDurability + || Some(ticket.expected_token()) != self.commit_token.as_ref() + || now_ns < self.last_now_ns + { + return Err("temporal_abort_workflow_changed".into()); + } + let ram = self.session.assembler.fence_checkpoint()?; + let high = self.source_high.clone(); + let mapping = self.mapping.clone(); + let live = self.live_observed_ns; + let loaded = store.abort_temporal_pending(ticket, now_ns)?; + let prefix = loaded.prefix.ok_or("temporal_recovery_retired")?; + let mut restored = checkpoint::decode(prefix.bytes(), prefix.descriptor(), now_ns)?; + restored.session.assembler.restore_prefix_fences( + ram.for_prefix(restored.session.rows.len())?, + restored.session.rows.len(), + )?; + restored.source_high = high; + restored.mapping = mapping; + restored.live_observed_ns = live; + restored.commit_token = Some(loaded.token); + restored.durable = Some(prefix.bytes().to_vec()); + restored.stage = Stage::PausedNeedsReconfirmation; + restored.last_now_ns = now_ns; + *self = restored; + Ok(()) + } + fn repair_pending(&mut self, store: &RecoveryStore, now_ns: u64) -> Result<(), String> { + if self.stage != Stage::BlockedDurability || now_ns < self.last_now_ns { + return Err("temporal_pending_repair_state".into()); + } + let intent = self + .pending_intent + .as_ref() + .ok_or("temporal_pending_intent_missing")?; + let prefix = self + .pending_prefix + .as_ref() + .ok_or("temporal_pending_prefix_missing")?; + let token = store.repair_same_temporal(intent, Some(prefix), now_ns)?; + let mut restored = checkpoint::decode(prefix.bytes(), prefix.descriptor(), now_ns)?; + restored.commit_token = Some(token); + restored.durable = Some(prefix.bytes().to_vec()); + restored.stage = Stage::PausedNeedsReconfirmation; + restored.last_now_ns = now_ns; + *self = restored; + Ok(()) + } + fn cold_restore( + store: &RecoveryStore, + expected: &CaptureContext, + now: u64, + epoch: &str, + anchor: u64, + ) -> Result { + Self::cold_restore_with(store, expected, now, epoch, anchor, |s, t, n| { + s.recertify_temporal(t, n) + }) + } + fn cold_restore_with( + store: &RecoveryStore, + expected: &CaptureContext, + now: u64, + epoch: &str, + anchor: u64, + recertify: impl FnOnce( + &RecoveryStore, + &TemporalToken, + u64, + ) -> Result< + crate::room_learner::recovery_store::temporal_commitment::TemporalRestored, + String, + >, + ) -> Result { + let loaded = store + .load_temporal(now)? + .ok_or("temporal_recovery_missing")?; + if loaded.prefix.is_none() { + return Err("temporal_recovery_retired".into()); + } + // A read is only an authenticated observation. Cold restoration must + // re-establish actual barriers before promising a durable saved prefix. + let certified = recertify(store, &loaded.token, now)?; + let prefix = certified.prefix; + let mut v = checkpoint::decode(prefix.bytes(), prefix.descriptor(), now)?; + v.commit_token = Some(certified.token); + if v.context() != expected || epoch.is_empty() || anchor < v.last_now_ns || now < anchor { + return Err("temporal_restore_context_or_clock".into()); + } + v.stage = Stage::PausedNeedsReconfirmation; + v.last_now_ns = now; + v.durable = Some(prefix.bytes().to_vec()); + Ok(v) + } +} +#[path = "temporal_checkpoint.rs"] +pub(crate) mod checkpoint; +#[cfg(test)] +#[path = "temporal_execution_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution_tests.rs new file mode 100644 index 0000000000..6ab546e9e4 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution_tests.rs @@ -0,0 +1,707 @@ +//! SYNTHETIC simulated receiver clocks; no sleeps or physical sensing claims. +use super::super::tests::{context, packet}; +use super::*; +use crate::room_learner::managed_schedule_contract::LeadInPolicy; +pub(super) fn setup() -> (tempfile::TempDir, RecoveryStore, TemporalWorkflow) { + let dir = tempfile::tempdir().unwrap(); + let c = context(); + let data = dir.path().join(&c.owner_scope).join("SensingServer"); + let store = + RecoveryStore::new(&data, &c.owner_scope, &c.installation_id, Some(&[7; 32])).unwrap(); + let plan = + ManagedScheduleContract::build(c, LeadInPolicy::Train5HeldOut15EngineeringV1).unwrap(); + let w = TemporalWorkflow::prepare( + plan, + "synthetic-prefix".into(), + FitConfig { + epochs: 2, + ..Default::default() + }, + 1_000_000_000, + ) + .unwrap(); + (dir, store, w) +} +pub(super) fn start(w: &mut TemporalWorkflow, store: &RecoveryStore, time: u64, epoch: &str) { + let permit = w.synthetic_conditions(); + w.start_segment(&permit, epoch, time, time).unwrap(); + assert!( + w.begin_slot().is_err(), + "segment must be durable before collection" + ); + w.persist_segment(store).unwrap(); +} +pub(super) fn drive(w: &mut TemporalWorkflow, from: u64, to: u64, seq: &mut u32) -> u64 { + let period = 1_000_000_000u64.div_ceil(33); + let mut at = from; + while at <= to + period { + *seq += 1; + for source in 1..=3u8 { + let t = at + u64::from(source) * 1000; + w.observe(&packet(source, *seq, t, RoomLabel::Empty, 0), t) + .unwrap(); + } + at += period; + } + at +} +pub(super) fn commit_next( + w: &mut TemporalWorkflow, + store: &RecoveryStore, + from: u64, + seq: &mut u32, +) -> u64 { + let target = w.begin_slot().unwrap(); + let next = drive(w, from, target.end_ns(), seq); + w.commit_slot(store, w.last_now_ns).unwrap(); + next +} +#[test] +fn actual_600second_episode_interruption_keeps_only_durable_prefix_and_original_rows() { + let (_dir, store, mut w) = setup(); + assert!(w.begin_slot().is_err()); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "epoch-one"); + let mut seq = 0; + let mut next = 2_000_000_000; + for _ in 0..3 { + next = commit_next(&mut w, &store, next, &mut seq); + } + let saved_rows = serde_json::to_vec(&w.session.rows).unwrap(); + let old_ledger = w.ledger.clone(); + let deadline = w.descriptor.expires_at_ms; + let accepted = w.accepted(); + let original_trial = w.session.rows[0].trial_id.clone(); + let old_session = w.session.rows[0].session_id.clone(); + let partial = w.begin_slot().unwrap(); + let next = drive(&mut w, next, partial.target_ns() + 1_000_000_000, &mut seq); + assert!(w.session.active); + assert!(w.live_observed_ns > accepted); + let before_high = w.source_high.clone(); + w.pause().unwrap(); + assert_eq!(w.accepted(), accepted); + assert_eq!(w.source_high, before_high); + assert_eq!(serde_json::to_vec(&w.session.rows).unwrap(), saved_rows); + let resumed = next + 300_000_000_000; + start(&mut w, &store, resumed, "epoch-two"); + assert_eq!(w.ledger.gaps.len(), 1); + assert!(w.ledger.gaps[0].discarded_live_ns > 0); + assert_eq!(w.ledger.gaps[0].accepted_observed_ns, accepted); + assert_eq!(w.ledger.spans, old_ledger.spans); + assert_eq!(w.mapping.as_ref().unwrap().observed_start_ns, accepted); + let newtarget = w.begin_slot().unwrap(); + assert_eq!( + newtarget.target_ns(), + resumed + w.schedule().unwrap().episodes()[0].slots[3].target_offset_ms * 1_000_000 + - accepted + ); + assert_eq!(newtarget.identity().trial_id, original_trial); + assert_ne!(newtarget.identity().session_id, old_session); + let mut next = drive(&mut w, resumed, newtarget.end_ns(), &mut seq); + w.commit_slot(&store, w.last_now_ns).unwrap(); + while w.session.rows.len() < 16 { + next = commit_next(&mut w, &store, next, &mut seq); + } + assert_eq!(w.accepted(), 600_000_000_000); + assert_eq!(w.descriptor.expires_at_ms, deadline); + assert_eq!( + serde_json::to_vec(&w.session.rows[..3]).unwrap(), + saved_rows + ); + assert!(w.last_now_ns - resumed < 600_000_000_000); + assert!(w.session.fit().is_err()); + let mut cold = TemporalWorkflow::cold_restore( + &store, + &context(), + w.last_now_ns + 1_000_000, + "epoch-three", + w.last_now_ns + 1_000_000, + ) + .unwrap(); + assert_eq!(cold.accepted(), 600_000_000_000); + assert_eq!( + serde_json::to_vec(&cold.session.rows).unwrap(), + serde_json::to_vec(&w.session.rows).unwrap() + ); + assert_eq!(cold.descriptor.expires_at_ms, deadline); + assert_eq!(cold.stage, Stage::PausedNeedsReconfirmation); + assert!(cold.session.fit().is_err()); +} +#[test] +fn complete_native_ready_does_not_commit_unobserved_reservation_or_downtime() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "one"); + let target = w.begin_slot().unwrap(); + let mut seq = 0; + drive( + &mut w, + 2_000_000_000, + target.target_ns() + 2_100_000_000, + &mut seq, + ); + assert!(!w.session.active); + assert!(w.commit_slot(&store, w.last_now_ns).is_err()); + assert!(w.ledger.spans.is_empty()); + assert_eq!(w.accepted(), 0); +} +#[test] +fn explicit_pause_and_cold_restore_preserve_fences_reject_counter_epoch_and_clock_rescue() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "one"); + let mut seq = 0; + let next = commit_next(&mut w, &store, 2_000_000_000, &mut seq); + let last = w.last_now_ns; + assert!(TemporalWorkflow::cold_restore( + &store, + &context(), + last + 1_000_000, + "bad-clock", + last - 1 + ) + .is_err()); + w.pause().unwrap(); + let permit = w.synthetic_conditions(); + assert!(w.start_segment(&permit, "two", last - 1, last + 1).is_err()); + start(&mut w, &store, next + 1_000_000_000, "two"); + let t = w.begin_slot().unwrap(); + assert!(w + .observe( + &packet(1, 1, t.target_ns(), RoomLabel::Empty, 0), + t.target_ns() + ) + .is_err()); + let mut foreign = context(); + foreign.room_digest = "foreign-room".into(); + assert!(TemporalWorkflow::cold_restore( + &store, + &foreign, + w.last_now_ns + 1_000_000, + "three", + w.last_now_ns + 1_000_000 + ) + .is_err()); +} + +#[test] +fn missing_full_period_packets_and_stale_confirmation_cannot_turn_time_into_observation() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + let old = w.synthetic_conditions(); + start(&mut w, &store, 2_000_000_000, "one"); + let mut seq = 0; + let next = commit_next(&mut w, &store, 2_000_000_000, &mut seq); + let accepted = w.accepted(); + let bad_at = next + 10_000_000_000; + assert!(w + .observe(&packet(1, seq + 1, bad_at, RoomLabel::Empty, 0), bad_at) + .is_err()); + assert_eq!(w.accepted(), accepted); + w.pause().unwrap(); + assert!(w + .start_segment(&old, "two", bad_at + 1, bad_at + 1) + .is_err()); +} + +#[test] +fn two_successive_aead_cold_resumes_extend_prefix_without_renewing_descriptor() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "one"); + let mut seq = 0; + commit_next(&mut w, &store, 2_000_000_000, &mut seq); + let identity = ( + w.descriptor.id.clone(), + w.descriptor.created_at_ms, + w.descriptor.expires_at_ms, + ); + let original = serde_json::to_vec(&w.session.rows).unwrap(); + for epoch in ["two", "three"] { + let at = w.last_now_ns + 10_000_000_000; + w = TemporalWorkflow::cold_restore(&store, &context(), at, epoch, at).unwrap(); + start(&mut w, &store, at + 1_000_000_000, epoch); + commit_next(&mut w, &store, at + 1_000_000_000, &mut seq); + assert_eq!( + ( + w.descriptor.id.clone(), + w.descriptor.created_at_ms, + w.descriptor.expires_at_ms + ), + identity + ); + } + assert_eq!(w.session.rows.len(), 3); + assert_eq!(w.ledger.gaps.len(), 2); + assert_eq!(serde_json::to_vec(&w.session.rows[..1]).unwrap(), original); +} + +/// EXECUTED LIMITATION, not a security success: AEAD authenticates bytes but +/// published593 RecoveryStore has no independent latest-revision anchor. +#[test] +fn legacy_v1_authenticated_slot_remains_unchanged_and_has_no_new_freshness_claim() { + let (dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "one"); + let mut seq = 0; + let next = commit_next(&mut w, &store, 2_000_000_000, &mut seq); + let c = context(); + let file = dir + .path() + .join(&c.owner_scope) + .join("SensingServer/room-learner-recovery/capture-recovery-v1.bin"); + store + .save( + &w.descriptor, + w.durable.as_ref().unwrap(), + w.last_now_ns / 1_000_000, + ) + .unwrap(); + let old = std::fs::read(&file).unwrap(); + commit_next(&mut w, &store, next, &mut seq); + assert_eq!(w.session.rows.len(), 2); + store + .save( + &w.descriptor, + w.durable.as_ref().unwrap(), + w.last_now_ns / 1_000_000, + ) + .unwrap(); + std::fs::write(&file, &old).unwrap(); + let now = w.last_now_ns + 1_000_000; + let legacy = store.load(now / 1_000_000).unwrap().unwrap(); + let mut restored = checkpoint::decode(&legacy.plaintext, &legacy.descriptor, now).unwrap(); + assert_eq!(restored.session.rows.len(),1,"V1 legacy API bytes/meaning remain unchanged; typed temporal path uses independent commitments"); + assert!(restored.session.fit().is_err()); +} + +#[test] +fn independent_resumed_segment_cannot_credit_period_before_all_sources_arrive() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "initial"); + let mut seq = 0; + let next = commit_next(&mut w, &store, 2_000_000_000, &mut seq); + let original = w.accepted(); + w.pause().unwrap(); + let resume = next + 10_000_000_000; + start(&mut w, &store, resume, "resumed"); + let target = w.begin_slot().unwrap(); + assert!(target.target_ns() - resume > 1_000_000_000); + let period = 1_000_000_000u64.div_ceil(33); + let mut at = resume; + let mut node3_started = false; + while at <= target.end_ns() + period { + seq += 1; + for source in 1..=3u8 { + if source == 3 && at < target.target_ns() { + continue; + } + if source == 3 { + node3_started = true; + } + let t = at + u64::from(source) * 1000; + let result = w.observe(&packet(source, seq, t, RoomLabel::Empty, 0), t); + if result.is_err() { + assert_eq!(w.accepted(), original); + return; + } + } + at += period; + } + assert!(node3_started); + assert!(!w.session.active); + let result = w.commit_slot(&store, w.last_now_ns); + assert!( + result.is_err(), + "missing source before sparse window was credited as a fully observed reservation" + ); + assert_eq!(w.accepted(), original); +} + +#[test] +fn initial_segment_missing_source_is_rejected_at_native_first_arrival_limit() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + let start_ns = 2_000_000_000; + start(&mut w, &store, start_ns, "initial"); + let limit = w + .session + .managed + .as_ref() + .unwrap() + .binding + .native + .max_first_packet_delay_ms() + * 1_000_000; + assert!(w.observe_tick(start_ns + limit).is_ok()); + assert_eq!(w.live_observed_ns, 0); + assert_eq!(w.accepted(), 0); + assert!(w.observe_tick(start_ns + limit + 1).is_err()); + assert!(w.custody.failed); + assert_eq!(w.live_observed_ns, 0); + assert_eq!(w.accepted(), 0); + assert!(w + .observe( + &packet(3, 1, start_ns + limit + 2, RoomLabel::Empty, 0), + start_ns + limit + 2 + ) + .is_err()); +} + +#[test] +fn busy_sources_detect_dropout_between_sparse_windows_and_preserve_prefix_floors() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "initial"); + let mut seq = 0; + let next = commit_next(&mut w, &store, 2_000_000_000, &mut seq); + let accepted = w.accepted(); + let raw = w.live_observed_ns; + let third_floor = w + .source_high + .iter() + .find(|(id, _, _)| *id == 3) + .copied() + .unwrap(); + let saved = serde_json::to_vec(&w.session.rows).unwrap(); + let period = 1_000_000_000u64.div_ceil(33); + let mut rejected = false; + for i in 0..5 { + seq += 1; + for source in 1..=2u8 { + let at = next + i * period + u64::from(source) * 1000; + if w.observe(&packet(source, seq, at, RoomLabel::Empty, 0), at) + .is_err() + { + rejected = true; + break; + } + } + if rejected { + break; + } + } + assert!( + rejected, + "busy nodes must detect absent third node between retained windows" + ); + assert!(w.custody.failed); + assert_eq!(w.accepted(), accepted); + let known_frontier = w.mapping.as_ref().unwrap().observed_start_ns + third_floor.2 + - w.mapping.as_ref().unwrap().receiver_start_ns; + assert!( + w.live_observed_ns >= raw && w.live_observed_ns <= known_frontier, + "busy sources cannot advance beyond the absent source's last actual receiver timestamp" + ); + let failed_live = w.live_observed_ns; + assert_eq!( + w.source_high.iter().find(|(id, _, _)| *id == 3).copied(), + Some(third_floor) + ); + let at = w.last_now_ns + period; + assert!(w + .observe(&packet(3, seq + 1, at, RoomLabel::Empty, 0), at) + .is_err()); + assert_eq!(w.live_observed_ns, failed_live); + w.pause().unwrap(); + assert_eq!(w.accepted(), accepted); + assert_eq!(serde_json::to_vec(&w.session.rows).unwrap(), saved); + assert!(w + .source_high + .iter() + .find(|(id, _, _)| *id == 3) + .is_some_and(|(_, s, t)| *s >= third_floor.1 && *t >= third_floor.2)); +} + +#[test] +fn receiver_tick_withdraws_stale_segment_without_advancing_observed_time() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "initial"); + let mut seq = 0; + drive(&mut w, 2_000_000_000, 2_000_000_000, &mut seq); + let raw = w.live_observed_ns; + let gap = w + .session + .managed + .as_ref() + .unwrap() + .binding + .native + .source_policies()[0] + .max_gap_ns; + assert!(w.observe_tick(w.last_now_ns + gap + 1).is_err()); + assert!(w.custody.failed); + assert_eq!(w.live_observed_ns, raw); + assert_eq!(w.accepted(), 0); +} + +#[test] +fn initial_source_skew_uses_native_limit_even_within_first_arrival_deadline() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + let start_ns = 2_000_000_000; + start(&mut w, &store, start_ns, "initial"); + for row in 0..3u32 { + for source in 1..=2u8 { + let at = start_ns + u64::from(row) * 30_000_000 + u64::from(source) * 1000; + w.observe(&packet(source, row + 1, at, RoomLabel::Empty, 0), at) + .unwrap(); + } + } + let at = start_ns + 75_000_000; + assert!(w + .observe(&packet(3, 3, at, RoomLabel::Empty, 0), at) + .is_err()); + assert!(w.custody.failed); + assert_eq!(w.live_observed_ns, 0); + assert_eq!(w.accepted(), 0); + assert!(!w.source_high.iter().any(|(id, _, _)| *id == 3)); +} + +#[test] +fn independent_zero_prefix_partial_pause_then_saved_segment_cold_restores() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + let id = w.descriptor.id.clone(); + let expiry = w.descriptor.expires_at_ms; + start(&mut w, &store, 2_000_000_000, "zero-initial"); + let target = w.begin_slot().unwrap(); + let mut seq = 0; + let next = drive( + &mut w, + 2_000_000_000, + target.target_ns() + 1_000_000_000, + &mut seq, + ); + assert!(w.session.active); + assert!(w.session.rows.is_empty()); + let floors = w.source_high.clone(); + w.pause().unwrap(); + assert_eq!(w.accepted(), 0); + assert!(w.session.rows.is_empty()); + let resume = next + 10_000_000_000; + start(&mut w, &store, resume, "zero-resumed"); + assert_eq!(w.ledger.gaps.len(), 1); + assert!(w.ledger.gaps[0].after_prefix_sha256.is_none()); + let at = w.last_now_ns + 1_000_000; + let restored = TemporalWorkflow::cold_restore(&store, &context(), at, "zero-cold", at) + .expect("legitimate zero-prefix pause gap must cold-restore without inventing rows"); + assert!(restored.session.rows.is_empty()); + assert_eq!(restored.accepted(), 0); + assert_eq!(restored.descriptor.id, id); + assert_eq!(restored.descriptor.expires_at_ms, expiry); + for floor in floors { + assert!(restored + .source_high + .iter() + .any(|s| s.0 == floor.0 && s.1 >= floor.1 && s.2 >= floor.2)); + } + assert_eq!(restored.stage, Stage::PausedNeedsReconfirmation); +} + +#[test] +fn genesis_history_cold_restores_before_and_after_the_first_committed_window() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + let original = ( + w.descriptor.id.clone(), + w.descriptor.created_at_ms, + w.descriptor.expires_at_ms, + ); + start(&mut w, &store, 2_000_000_000, "initial"); + let mut seq = 0; + let target = w.begin_slot().unwrap(); + let next = drive( + &mut w, + 2_000_000_000, + target.target_ns() + 1_000_000_000, + &mut seq, + ); + let floors = w.source_high.clone(); + w.pause().unwrap(); + start(&mut w, &store, next + 10_000_000_000, "resume-one"); + let now = w.last_now_ns + 1_000_000; + w = TemporalWorkflow::cold_restore(&store, &context(), now, "cold-one", now).unwrap(); + assert_eq!(w.accepted(), 0); + assert_eq!(w.session.rows.len(), 0); + let resume = now + 1_000_000_000; + start(&mut w, &store, resume, "resume-two"); + assert_eq!(w.ledger.gaps.len(), 2); + assert!(w + .ledger + .gaps + .iter() + .all(|g| g.genesis.is_some() && g.after_prefix_sha256.is_none())); + commit_next(&mut w, &store, resume, &mut seq); + let committed = serde_json::to_vec(&w.session.rows).unwrap(); + let now = w.last_now_ns + 1_000_000; + let cold = TemporalWorkflow::cold_restore(&store, &context(), now, "cold-two", now).unwrap(); + assert_eq!(cold.session.rows.len(), 1); + assert_eq!(serde_json::to_vec(&cold.session.rows).unwrap(), committed); + assert_eq!(cold.ledger.gaps.len(), 2); + assert_eq!( + ( + cold.descriptor.id.clone(), + cold.descriptor.created_at_ms, + cold.descriptor.expires_at_ms + ), + original + ); + for (id, seq, t) in floors { + assert!(cold + .source_high + .iter() + .any(|(s, q, n)| *s == id && *q >= seq && *n >= t)); + } +} + +#[test] +fn reducer_acknowledges_only_explicit_same_intent_repair_then_requires_reconfirmation() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + let original_expiry = w.descriptor.expires_at_ms; + start(&mut w, &store, 2_000_000_000, "one"); + let mut seq = 0; + let target = w.begin_slot().unwrap(); + drive(&mut w, 2_000_000_000, target.end_ns(), &mut seq); + assert!(w + .persist_with(&store, true, |s, i, p, n| { + s.publish_temporal(i, Some(p), n)?; + Err("injected-ack-ambiguity".into()) + }) + .is_err()); + assert_eq!(w.accepted(), 0); + assert_eq!(w.stage, Stage::BlockedDurability); + assert!(w.begin_slot().is_err()); + let now = w.last_now_ns + 1_000_000; + w.repair_pending(&store, now).unwrap(); + assert_eq!(w.session.rows.len(), 1); + assert!(w.accepted() > 0); + assert_eq!(w.stage, Stage::PausedNeedsReconfirmation); + assert_eq!(w.descriptor.expires_at_ms, original_expiry); + assert!(w.begin_slot().is_err()); +} + +#[test] +fn crashed_truncated_stage_can_be_explicitly_aborted_then_resume_saved_run() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + let original = (w.descriptor.id.clone(), w.descriptor.expires_at_ms); + start(&mut w, &store, 2_000_000_000, "one"); + let mut seq = 0; + let next = commit_next(&mut w, &store, 2_000_000_000, &mut seq); + let saved = serde_json::to_vec(&w.session.rows).unwrap(); + let floors = w.source_high.clone(); + let target = w.begin_slot().unwrap(); + drive(&mut w, next, target.end_ns(), &mut seq); + assert!(w + .persist_with(&store, true, |s, i, p, n| s + .synthetic_partial_write(i, p, n)) + .is_err()); + let now = w.last_now_ns + 1_000_000; + drop(w); + let loaded = store.load_temporal(now).unwrap().unwrap(); + let token = loaded.token; + let tickets = store.inspect_abortable_temporal(&token, now).unwrap(); + assert!(!tickets.is_empty()); + store.abort_temporal_pending(&tickets[0], now).unwrap(); + let mut restored = + TemporalWorkflow::cold_restore(&store, &context(), now, "after-abort", now).unwrap(); + assert_eq!(restored.stage, Stage::PausedNeedsReconfirmation); + assert_eq!(serde_json::to_vec(&restored.session.rows).unwrap(), saved); + assert_eq!( + ( + restored.descriptor.id.clone(), + restored.descriptor.expires_at_ms + ), + original + ); + assert_eq!(restored.source_high, floors); + let start_ns = now + 1_000_000_000; + start(&mut restored, &store, start_ns, "resume"); + commit_next(&mut restored, &store, start_ns, &mut seq); + assert_eq!(restored.session.rows.len(), 2); + assert_eq!( + serde_json::to_vec(&restored.session.rows[..1]).unwrap(), + saved + ); +} + +#[test] +fn live_abort_retains_known_ram_highwater_while_dropping_only_unsaved_rows() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + start(&mut w, &store, 2_000_000_000, "one"); + let mut seq = 0; + let next = commit_next(&mut w, &store, 2_000_000_000, &mut seq); + let saved = serde_json::to_vec(&w.session.rows).unwrap(); + let target = w.begin_slot().unwrap(); + drive(&mut w, next, target.end_ns(), &mut seq); + let high = w.source_high.clone(); + assert!(w + .persist_with(&store, true, |s, i, p, n| s + .synthetic_partial_write(i, p, n)) + .is_err()); + let now = w.last_now_ns + 1_000_000; + let tickets = store + .inspect_abortable_temporal(w.commit_token.as_ref().unwrap(), now) + .unwrap(); + w.abort_pending_artifact(&store, &tickets[0], now).unwrap(); + assert_eq!(w.source_high, high); + assert_eq!(serde_json::to_vec(&w.session.rows).unwrap(), saved); + assert_eq!(w.stage, Stage::PausedNeedsReconfirmation); + let start_ns = now + 1_000_000_000; + start(&mut w, &store, start_ns, "resumed"); + commit_next(&mut w, &store, start_ns, &mut seq); + assert_eq!(w.session.rows.len(), 2); + assert_eq!(serde_json::to_vec(&w.session.rows[..1]).unwrap(), saved); +} + +#[test] +fn cold_restore_cannot_ack_visible_saved_rows_when_recertification_fails() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + let original = (w.descriptor.id.clone(), w.descriptor.expires_at_ms); + start(&mut w, &store, 2_000_000_000, "one"); + let mut seq = 0; + let target = w.begin_slot().unwrap(); + drive(&mut w, 2_000_000_000, target.end_ns(), &mut seq); + assert!(w + .persist_with(&store, true, |s, i, p, n| s + .synthetic_lost_witness_ack(i, p, n)) + .is_err()); + assert_eq!(w.accepted(), 0); + let expected_rows = serde_json::to_vec(&w.session.rows).unwrap(); + let now = w.last_now_ns + 1_000_000; + drop(w); + assert!(store.load_temporal(now).unwrap().unwrap().prefix.is_some()); + let mut called = false; + assert!(TemporalWorkflow::cold_restore_with( + &store, + &context(), + now, + "cold", + now, + |_, _, _| { + called = true; + Err("injected-restoration-durability-denied".into()) + } + ) + .is_err()); + assert!(called); + let restored = TemporalWorkflow::cold_restore(&store, &context(), now, "cold", now).unwrap(); + assert_eq!(restored.stage, Stage::PausedNeedsReconfirmation); + assert_eq!( + serde_json::to_vec(&restored.session.rows).unwrap(), + expected_rows + ); + assert_eq!( + ( + restored.descriptor.id.clone(), + restored.descriptor.expires_at_ms + ), + original + ); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_summary.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_summary.rs new file mode 100644 index 0000000000..d458c379d8 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_summary.rs @@ -0,0 +1,315 @@ +//! Optional ACTUAL aggregate fit diagnostics in the SAME owner ArtifactStore. +//! Reuses dca8's report projection and finite/config/count/delta validation. +//! Never reconstructs historical outcomes or establishes qualification authority. +use super::temporal_artifacts::{bounded_read, immutable}; +use super::*; +use crate::room_learner::{ + observation::CaptureContext, + temporal_model_input::{TemporalFitBinding, TemporalModel}, + training::{FitConfig, ObjectiveValues, TrainingReport}, +}; +const SCHEMA: &str = "ruview.room-learner.temporal-fit-summary.v1"; +const MAX_BYTES: usize = 16_384; +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct FitObjective { + pub total: f32, + pub contrastive: f32, + pub supervised: f32, +} +impl From for FitObjective { + fn from(v: ObjectiveValues) -> Self { + Self { + total: v.total, + contrastive: v.contrastive, + supervised: v.supervised, + } + } +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct FitMetrics { + pub initial: FitObjective, + pub final_objective: FitObjective, + pub completed_epochs: usize, + pub parameter_count: usize, + pub parameter_delta_l2: f32, + pub encoder_delta_l2: f32, + pub backbone_delta_l2: f32, + pub projection_delta_l2: f32, + pub head_delta_l2: f32, + pub objective_evaluations: usize, + pub training_cohorts: usize, + pub contrastive_trials: usize, + pub configured: FitConfig, + /// Actual TrainingReport has no elapsed measurement. Never copy its limit here. + pub elapsed_ms: Option, +} +impl FitMetrics { + fn from_report(r: &TrainingReport, c: &FitConfig) -> Self { + Self { + initial: r.initial.into(), + final_objective: r.final_objective.into(), + completed_epochs: r.completed_epochs, + parameter_count: r.parameter_count, + parameter_delta_l2: r.parameter_delta_l2, + encoder_delta_l2: r.encoder_delta_l2, + backbone_delta_l2: r.backbone_delta_l2, + projection_delta_l2: r.projection_delta_l2, + head_delta_l2: r.head_delta_l2, + objective_evaluations: r.objective_evaluations, + training_cohorts: r.training_cohorts, + contrastive_trials: r.contrastive_trials, + configured: c.clone(), + elapsed_ms: None, + } + } + fn validate(&self, b: &TemporalFitBinding, context: &CaptureContext) -> Result<(), String> { + let c = &self.configured; + c.validate_context(context)?; + let finite = |o: &FitObjective| { + [o.total, o.supervised, o.contrastive] + .iter() + .all(|v| v.is_finite() && *v >= 0.) + && (f64::from(o.total) + - f64::from(c.supervised_weight) * f64::from(o.supervised) + - f64::from(c.contrastive_weight) * f64::from(o.contrastive)) + .abs() + <= 1e-5 * (1. + f64::from(o.total).abs()) + }; + if self.training_cohorts != b.training_cohorts + || !(4..=64).contains(&self.training_cohorts) + || !(2..=self.training_cohorts).contains(&self.contrastive_trials) + || self.parameter_count != b.parameter_count + || self.parameter_count > c.max_parameters + || self.completed_epochs == 0 + || self.completed_epochs > c.epochs + || self.objective_evaluations < 2 + || self.objective_evaluations >= c.max_objective_evaluations + || c.hidden_dim != b.encoder.hidden_dim + || c.embedding_dim != b.encoder.embedding_dim + || c.temporal_tokens != b.encoder.temporal_tokens + || c.temperature != b.encoder.temperature + || self.elapsed_ms.is_some() + || !finite(&self.initial) + || !finite(&self.final_objective) + || self.final_objective.total >= self.initial.total + || ![ + self.parameter_delta_l2, + self.encoder_delta_l2, + self.backbone_delta_l2, + self.projection_delta_l2, + self.head_delta_l2, + ] + .iter() + .all(|v| v.is_finite() && *v >= 0.) + || self.parameter_delta_l2 <= 0. + || self.encoder_delta_l2 <= 0. + || self.backbone_delta_l2 <= 0. + || self.projection_delta_l2 <= 0. + { + return Err("temporal_fit_report_invalid".into()); + } + if let Some(bound) = &b.bound_fit_config { + if serde_json::to_vec(bound).map_err(|_| "fit_config_encoding")? + != serde_json::to_vec(c).map_err(|_| "fit_config_encoding")? + { + return Err("temporal_fit_configuration_changed".into()); + } + } + let norm = |whole: f32, a: f32, b: f32| { + let expected = f64::from(a).hypot(f64::from(b)); + (f64::from(whole) - expected).abs() <= 1e-4 * (1. + expected) + }; + if !norm( + self.parameter_delta_l2, + self.encoder_delta_l2, + self.head_delta_l2, + ) || !norm( + self.encoder_delta_l2, + self.backbone_delta_l2, + self.projection_delta_l2, + ) || f64::from(self.parameter_delta_l2) > 20_000. * (self.parameter_count as f64).sqrt() + { + return Err("temporal_fit_deltas_inconsistent".into()); + } + Ok(()) + } +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct TemporalFitSummary { + pub schema: String, + pub binding: TemporalFitBinding, + pub metrics: FitMetrics, + pub diagnostic_only: bool, + pub physically_validated: bool, + pub qualification_granted: bool, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Envelope { + summary: TemporalFitSummary, + summary_sha256: String, +} +fn small(value: &T) -> Result, String> { + let b = serde_json::to_vec(value).map_err(|_| "temporal_fit_encoding")?; + if b.len() > MAX_BYTES { + return Err("temporal_fit_summary_budget".into()); + } + Ok(b) +} +impl TemporalFitSummary { + fn actual( + model: &TemporalModel, + report: &TrainingReport, + configured: &FitConfig, + ) -> Result { + let v = Self { + schema: SCHEMA.into(), + binding: model.fit_diagnostic_binding()?, + metrics: FitMetrics::from_report(report, configured), + diagnostic_only: true, + physically_validated: false, + qualification_granted: false, + }; + v.validate(model)?; + Ok(v) + } + fn validate(&self, model: &TemporalModel) -> Result<(), String> { + let expected = model.fit_diagnostic_binding()?; + if self.schema != SCHEMA + || !self.diagnostic_only + || self.physically_validated + || self.qualification_granted + || small(&self.binding)? != small(&expected)? + { + return Err("temporal_fit_summary_binding_changed".into()); + } + self.metrics.validate(&expected, model.context()) + } +} +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub(crate) enum FitDiagnosticState { + Recorded, + NotRecorded, + Unavailable, +} +#[derive(Debug, Clone, Serialize)] +pub(crate) struct TemporalFitDiagnostics { + pub state: FitDiagnosticState, + pub summary: Option, + pub warning: Option<&'static str>, + pub diagnostic_only: bool, + pub physically_validated: bool, + pub qualification_granted: bool, +} +impl TemporalFitDiagnostics { + fn state( + state: FitDiagnosticState, + summary: Option, + warning: Option<&'static str>, + ) -> Self { + Self { + state, + summary, + warning, + diagnostic_only: true, + physically_validated: false, + qualification_granted: false, + } + } +} +#[derive(Debug, Serialize)] +pub(crate) struct TemporalFitSaveResult { + pub candidate: TemporalCandidateReceipt, + pub fit_diagnostics: TemporalFitDiagnostics, +} +impl ArtifactStore { + fn temporal_fit_summary_path(&self, model: &TemporalModel) -> Result { + let binding = model.fit_diagnostic_binding()?; + self.validate_context(model.context())?; + // Model must already exist as the WHOLE verified codec. Never call the old + // inner-v1 loader or create a loose summary from a supplied hash. + let loaded = + self.load_exact_temporal_candidate(model.context(), &binding.artifact_sha256)?; + if small(&loaded.fit_diagnostic_binding()?)? != small(&binding)? { + return Err("temporal_fit_artifact_changed".into()); + } + let directory = self + .temporal_directory(model.context(), false)? + .ok_or("temporal_fit_model_directory_missing")?; + Ok(directory.join(format!("{}.fit-summary-v1.json", binding.artifact_sha256))) + } + fn save_actual_temporal_fit_summary( + &self, + model: &TemporalModel, + report: &TrainingReport, + configured: &FitConfig, + ) -> Result { + let summary = TemporalFitSummary::actual(model, report, configured)?; + let envelope = Envelope { + summary_sha256: digest(&small(&summary)?), + summary, + }; + let bytes = small(&envelope)?; + let path = self.temporal_fit_summary_path(model)?; + immutable(&path, &bytes)?; + let loaded = self + .load_temporal_fit_summary(model)? + .ok_or("temporal_fit_readback_missing")?; + if small(&loaded)? != small(&envelope.summary)? { + return Err("temporal_fit_readback_changed".into()); + } + Ok(loaded) + } + pub(crate) fn load_temporal_fit_summary( + &self, + model: &TemporalModel, + ) -> Result, String> { + let path = self.temporal_fit_summary_path(model)?; + match std::fs::symlink_metadata(&path) { + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(None), + Err(_) => return Err("temporal_fit_summary_unavailable".into()), + Ok(_) => {} + } + let bytes = bounded_read(&path, MAX_BYTES)?; + let envelope: Envelope = + serde_json::from_slice(&bytes).map_err(|_| "temporal_fit_summary_malformed")?; + if digest(&small(&envelope.summary)?) != envelope.summary_sha256 { + return Err("temporal_fit_summary_integrity".into()); + } + envelope.summary.validate(model)?; + Ok(Some(envelope.summary)) + } + /// Coherent real-caller seam: a valid whole model is durable FIRST. A metrics + /// failure is explicit optional diagnostics failure, never failed model save. + pub(crate) fn save_temporal_candidate_with_fit_summary( + &self, + model: &TemporalModel, + report: &TrainingReport, + configured: &FitConfig, + ) -> Result { + let candidate = self.save_temporal_candidate(model)?; + let fit_diagnostics=match self.save_actual_temporal_fit_summary(model,report,configured){Ok(summary)=>TemporalFitDiagnostics::state(FitDiagnosticState::Recorded,Some(summary),None),Err(_)=>TemporalFitDiagnostics::state(FitDiagnosticState::NotRecorded,None,Some("The model was saved, but actual aggregate fit diagnostics were not recorded; existing files were retained."))}; + Ok(TemporalFitSaveResult { + candidate, + fit_diagnostics, + }) + } + /// Future OWNER-side status projection only. Missing history is unknown, never + /// zeros/defaults or an inferred TrainingReport; paired progress stays unchanged. + pub(crate) fn temporal_fit_diagnostics(&self, model: &TemporalModel) -> TemporalFitDiagnostics { + match self.load_temporal_fit_summary(model){Ok(Some(summary))=>TemporalFitDiagnostics::state(FitDiagnosticState::Recorded,Some(summary),None),Ok(None)=>TemporalFitDiagnostics::state(FitDiagnosticState::NotRecorded,None,None),Err(_)=>TemporalFitDiagnostics::state(FitDiagnosticState::Unavailable,None,Some("Saved fit diagnostics could not be verified; the model and existing files were retained."))} + } +} + +/// Typed rehash for tampering tests: exercise semantic validation AFTER integrity, +/// never merely hash a serde_json::Value with different key/float encoding. +#[cfg(test)] +pub(crate) fn rehash_fixture(bytes: &[u8]) -> Result, String> { + let mut envelope: Envelope = serde_json::from_slice(bytes).map_err(|_| "fixture_schema")?; + envelope.summary_sha256 = digest(&small(&envelope.summary)?); + small(&envelope) +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_summary_managed_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_summary_managed_tests.rs new file mode 100644 index 0000000000..946094909a --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_summary_managed_tests.rs @@ -0,0 +1,472 @@ +//! SYNTHETIC actual managed fitter -> combined SAME-store helper -> cold model +//! and actual aggregate report. No production issuer/runtime/paired API. +use super::*; +use crate::room_learner::{ + artifact::ArtifactStore, + managed_schedule_contract::{LeadInPolicy, ManagedScheduleContract}, + native::{canonicalize_esp32, NativePacketInput}, +}; +use serde_json::{json, Value}; +use std::{ + path::{Path, PathBuf}, + sync::OnceLock, +}; +use wifi_densepose_hardware::{Adr018Flags, PpduType}; +fn context() -> CaptureContext { + CaptureContext { + owner_scope: "a".repeat(64), + installation_id: "synthetic-fit-summary-v3".into(), + room_digest: "synthetic-room".into(), + revision: 1, + sources: (1..=3) + .map(|source_id| SourceSpec { + source_id, + device_id: format!("esp32-node-{source_id}"), + grid_id: "ant1-bins64-ppdu0-usable-drop2-linear-v2".into(), + feature_count: 56, + signal_kind: ruview_unified::RfModality::WifiCsi, + center_frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + link_id: format!("native-{source_id}"), + link_index: 0, + geometry: None, + }) + .collect(), + } +} +fn actual_fit() -> (TemporalModel, TrainingReport, FitConfig) { + let schedule = + ManagedScheduleContract::build(context(), LeadInPolicy::Train5HeldOut15EngineeringV1) + .unwrap(); + let configured = FitConfig { + epochs: 2, + ..Default::default() + }; + let mut session = TemporalTrainingSession::managed( + schedule.clone(), + "actual-fit-summary".into(), + configured.clone(), + ) + .unwrap(); + let mut start = 10_000_000_000; + let mut index = 0u32; + for episode in schedule + .episodes() + .iter() + .filter(|e| e.role == DataRole::Train) + { + let bound = schedule + .bind_episode_start("actual-fit-summary", episode.index, 0, 0, None) + .unwrap(); + for slot in 0..episode.slots.len() { + let target = ManagedWindowTarget::from_runtime_episode( + &schedule, + &bound, + "actual-fit-summary", + slot, + start, + ) + .unwrap(); + session.begin_managed_window(&target).unwrap(); + 'window: for row in 0..100 { + for source in 1..=3 { + let gain = match episode.condition { + RoomLabel::Empty => 3, + RoomLabel::AdjacentOnly => 4, + RoomLabel::OccupiedStatic => 8, + RoomLabel::OccupiedMoving => 10 + (row % 2) as u8, + _ => unreachable!(), + }; + let iq = (0..64) + .flat_map(|b| [gain + source + (b % 2) as u8, 4]) + .collect::>(); + let at = target.target_ns() + + row as u64 * 1_000_000_000u64.div_ceil(33) + + u64::from(source) * 1000; + let packet = canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence: index * 1000 + row as u32 + 1, + received_at_ns: at, + receiver_sample_rate_hz: 33., + iq_bytes: &iq, + flags: Adr018Flags::default(), + provenance: CaptureProvenance::Synthetic, + }) + .unwrap(); + if session.ingest(&packet).unwrap() == WindowAdmission::Ready { + session.finish_window(at).unwrap(); + break 'window; + } + } + } + index += 1; + } + start += episode.target_active_recording_ms * 1_000_000 + 10_000_000_000; + } + let permit = session.synthetic_test_fit_admission().unwrap(); + let (model, report) = session + .fit_managed_with_cancel(&permit, &AtomicBool::new(false)) + .unwrap(); + assert_eq!(report.training_cohorts, 64); + assert!(report.backbone_delta_l2 > 0.); + (model, report, configured) +} +fn fixture() -> (TemporalModel, TrainingReport, FitConfig) { + static FIT: OnceLock<(Vec, TrainingReport, FitConfig)> = OnceLock::new(); + let (bytes, report, c) = FIT.get_or_init(|| { + let (m, r, c) = actual_fit(); + (m.export().unwrap(), r, c) + }); + ( + TemporalModel::import(bytes).unwrap(), + report.clone(), + c.clone(), + ) +} +fn store(root: &Path) -> ArtifactStore { + ArtifactStore::new( + &root.join("a".repeat(64)).join("SensingServer"), + &"a".repeat(64), + "synthetic-fit-summary-v3", + ) + .unwrap() +} +fn sidecar(root: &Path, model: &TemporalModel) -> PathBuf { + let b = model.fit_diagnostic_binding().unwrap(); + root.join("a".repeat(64)) + .join("SensingServer/room-learner/temporal-models") + .join(b.context_digest) + .join(format!("{}.fit-summary-v1.json", b.artifact_sha256)) +} +fn json(v: &impl Serialize) -> Value { + serde_json::to_value(v).unwrap() +} +#[test] +fn actual_managed_fit_combined_model_first_save_and_cold_summary_preserve_the_returned_report() { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let (model, report, configured) = actual_fit(); + let before = model.export().unwrap(); + let result = storage + .save_temporal_candidate_with_fit_summary(&model, &report, &configured) + .unwrap(); + assert_eq!(json(&result.fit_diagnostics)["state"], "recorded"); + let summary = result.fit_diagnostics.summary.unwrap(); + assert_eq!( + json(&summary.metrics.initial), + json(&FitObjectiveValues(&report.initial)) + ); + assert_eq!(summary.metrics.completed_epochs, report.completed_epochs); + assert_eq!( + summary.metrics.objective_evaluations, + report.objective_evaluations + ); + assert_eq!(summary.metrics.training_cohorts, 64); + assert_eq!( + summary.metrics.contrastive_trials, + report.contrastive_trials + ); + assert_eq!( + summary.metrics.parameter_delta_l2, + report.parameter_delta_l2 + ); + assert_eq!(summary.metrics.backbone_delta_l2, report.backbone_delta_l2); + assert!(summary.metrics.elapsed_ms.is_none()); + assert_eq!( + summary.binding.artifact_sha256, + result.candidate.artifact_sha256 + ); + assert_eq!( + summary.binding.artifact_kind, + "ruview.room-learner.managed-temporal-input.v3" + ); + assert!( + summary.binding.schedule_sha256.is_some() + && summary.binding.execution_policy_sha256.is_some() + && summary.binding.identity_evidence_sha256.is_some() + ); + let cold = storage + .load_exact_temporal_candidate(model.context(), &result.candidate.artifact_sha256) + .unwrap(); + assert_eq!(cold.export().unwrap(), before); + assert_eq!(model.export().unwrap(), before); + let loaded = storage.temporal_fit_diagnostics(&cold); + assert_eq!(json(&loaded)["state"], "recorded"); + assert_eq!(json(&loaded.summary.unwrap()), json(&summary)); + assert!(!summary.physically_validated && !summary.qualification_granted); + assert!(sidecar(root.path(), &model).metadata().unwrap().len() < 16_384); +} +// Same names/types as ObjectiveValues serialization; no rounded/estimated loss. +struct FitObjectiveValues<'a>(&'a crate::room_learner::training::ObjectiveValues); +impl Serialize for FitObjectiveValues<'_> { + fn serialize(&self, s: S) -> Result { + self.0.serialize(s) + } +} +#[test] +fn missing_history_is_not_recorded_and_optional_write_failure_keeps_whole_model_available() { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let (model, report, c) = fixture(); + let receipt = storage.save_temporal_candidate(&model).unwrap(); + let before = model.export().unwrap(); + let missing = storage.temporal_fit_diagnostics(&model); + assert_eq!(json(&missing)["state"], "not_recorded"); + assert!(missing.summary.is_none()); + crate::companion::atomic_private_write( + &sidecar(root.path(), &model), + b"existing malformed diagnostics", + ) + .unwrap(); + let result = storage + .save_temporal_candidate_with_fit_summary(&model, &report, &c) + .unwrap(); + assert_eq!(json(&result.fit_diagnostics)["state"], "not_recorded"); + assert!(result.fit_diagnostics.warning.is_some()); + assert!(result.fit_diagnostics.summary.is_none()); + assert_eq!( + storage + .load_exact_temporal_candidate(model.context(), &receipt.artifact_sha256) + .unwrap() + .export() + .unwrap(), + before + ); + assert_eq!( + std::fs::read(sidecar(root.path(), &model)).unwrap(), + b"existing malformed diagnostics" + ); + assert_eq!( + json(&storage.temporal_fit_diagnostics(&model))["state"], + "unavailable" + ); +} +#[test] +fn exact_v3_bound_configuration_and_actual_numeric_report_are_required_without_invalidating_model() +{ + for mode in ["config", "objective", "delta", "count"] { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let (model, mut report, mut c) = fixture(); + match mode { + "config" => c.learning_rate = 0.3, + "objective" => report.final_objective.total = f32::NAN, + "delta" => report.parameter_delta_l2 = 0., + _ => report.training_cohorts = 4, + }; + let result = storage + .save_temporal_candidate_with_fit_summary(&model, &report, &c) + .unwrap(); + assert_eq!( + json(&result.fit_diagnostics)["state"], + "not_recorded", + "{mode}" + ); + assert!(result.fit_diagnostics.warning.is_some()); + assert!(storage + .load_exact_temporal_candidate(model.context(), &result.candidate.artifact_sha256) + .is_ok()); + } +} +#[test] +fn whole_identity_and_policy_tampering_even_with_recomputed_summary_hash_is_unavailable() { + for field in [ + "artifact_kind", + "artifact_sha256", + "context_digest", + "training_data_sha256", + "input_evidence_sha256", + "input_policy_sha256", + "execution_policy_sha256", + "schedule_sha256", + "identity_evidence_sha256", + ] { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let (model, r, c) = fixture(); + storage + .save_temporal_candidate_with_fit_summary(&model, &r, &c) + .unwrap(); + let path = sidecar(root.path(), &model); + let mut e: Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + e["summary"]["binding"][field] = json!("f".repeat(64)); + e["summary_sha256"] = json!(bytes_hash(&serde_json::to_vec(&e["summary"]).unwrap())); + crate::companion::atomic_private_write( + &path, + &crate::room_learner::artifact::rehash_temporal_fit_fixture( + &serde_json::to_vec(&e).unwrap(), + ) + .unwrap(), + ) + .unwrap(); + assert_eq!( + storage.load_temporal_fit_summary(&model).unwrap_err(), + "temporal_fit_summary_binding_changed", + "{field}" + ); + assert_eq!( + json(&storage.temporal_fit_diagnostics(&model))["state"], + "unavailable", + "{field}" + ); + assert!(storage.load_temporal_candidate(model.context()).is_ok()); + } +} +#[test] +fn aggregate_summary_contains_no_raw_rows_timestamps_credentials_or_authority() { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let (m, r, c) = fixture(); + let result = storage + .save_temporal_candidate_with_fit_summary(&m, &r, &c) + .unwrap(); + let v = json(&result.fit_diagnostics); + fn walk(v: &Value) { + match v { + Value::Object(o) => { + for (k, v) in o { + assert!( + ![ + "samples", + "frame_ids", + "sequence_ids", + "times_ns", + "captured_at", + "timestamp", + "secret", + "keys", + "raw_iq", + "teacher" + ] + .contains(&k.as_str()), + "{k}" + ); + walk(v) + } + } + Value::Array(a) => { + for v in a { + walk(v) + } + } + _ => {} + } + } + walk(&v); + assert_eq!(v["physically_validated"], false); + assert_eq!(v["qualification_granted"], false); +} + +#[test] +fn oversized_or_symlink_summary_is_optional_and_never_replaces_the_retained_bytes() { + for mode in ["oversize", "symlink"] { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let (m, r, c) = fixture(); + let receipt = storage.save_temporal_candidate(&m).unwrap(); + let path = sidecar(root.path(), &m); + if mode == "oversize" { + crate::companion::atomic_private_write(&path, &vec![b'x'; 16_385]).unwrap(); + } else { + #[cfg(unix)] + { + let target = root.path().join("unrelated.txt"); + std::fs::write(&target, b"keep unrelated").unwrap(); + std::os::unix::fs::symlink(&target, &path).unwrap(); + } + #[cfg(not(unix))] + { + continue; + } + } + let result = storage + .save_temporal_candidate_with_fit_summary(&m, &r, &c) + .unwrap(); + assert_eq!(json(&result.fit_diagnostics)["state"], "not_recorded"); + assert!(storage + .load_exact_temporal_candidate(m.context(), &receipt.artifact_sha256) + .is_ok()); + assert_eq!( + json(&storage.temporal_fit_diagnostics(&m))["state"], + "unavailable" + ); + if mode == "oversize" { + assert_eq!(std::fs::read(&path).unwrap().len(), 16_385); + } else { + assert_eq!( + std::fs::read(root.path().join("unrelated.txt")).unwrap(), + b"keep unrelated" + ); + assert!(std::fs::symlink_metadata(path) + .unwrap() + .file_type() + .is_symlink()); + } + } +} +#[test] +#[cfg(unix)] +fn insecure_summary_and_foreign_store_cannot_expose_actual_private_metrics() { + use std::os::unix::fs::PermissionsExt; + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let (m, r, c) = fixture(); + let result = storage + .save_temporal_candidate_with_fit_summary(&m, &r, &c) + .unwrap(); + let path = sidecar(root.path(), &m); + std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o644)).unwrap(); + assert_eq!( + json(&storage.temporal_fit_diagnostics(&m))["state"], + "unavailable" + ); + let wrong = ArtifactStore::new( + &root.path().join("b".repeat(64)).join("SensingServer"), + &"b".repeat(64), + "synthetic-fit-summary-v3", + ) + .unwrap(); + assert!(wrong + .save_temporal_candidate_with_fit_summary(&m, &r, &c) + .is_err()); + assert!(wrong.load_temporal_fit_summary(&m).is_err()); + assert!(storage + .load_exact_temporal_candidate(m.context(), &result.candidate.artifact_sha256) + .is_ok()); +} +#[test] +fn rehashed_false_objectives_cannot_be_reported_as_actual_fit_diagnostics() { + let root = tempfile::tempdir().unwrap(); + let storage = store(root.path()); + let (m, r, c) = fixture(); + storage + .save_temporal_candidate_with_fit_summary(&m, &r, &c) + .unwrap(); + let path = sidecar(root.path(), &m); + let mut envelope: Value = serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + envelope["summary"]["metrics"]["final_objective"]["total"] = json!(r.initial.total + 10.); + envelope["summary_sha256"] = json!(bytes_hash( + &serde_json::to_vec(&envelope["summary"]).unwrap() + )); + crate::companion::atomic_private_write( + &path, + &crate::room_learner::artifact::rehash_temporal_fit_fixture( + &serde_json::to_vec(&envelope).unwrap(), + ) + .unwrap(), + ) + .unwrap(); + assert_eq!( + storage.load_temporal_fit_summary(&m).unwrap_err(), + "temporal_fit_report_invalid" + ); + assert_eq!( + json(&storage.temporal_fit_diagnostics(&m))["state"], + "unavailable" + ); + assert!(storage.load_temporal_candidate(m.context()).is_ok()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver.rs new file mode 100644 index 0000000000..ac7eefdfdd --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver.rs @@ -0,0 +1,487 @@ +//! Default-inactive, bounded continuous INPUT diagnostic, not a runtime/selected +//! detector. Every window uses the same TemporalModel opaque builder/infer path. +use super::{ + model::RoomPrediction, + native::CanonicalPacket, + observation::MAX_AGE_NS, + temporal_acquisition::{WindowInputPolicy, WindowAdmission}, + temporal_model_input::{TemporalLiveInput, TemporalModel}, +}; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum LiveState { + Inactive, + Warming, + WindowComplete, + Withdrawn, +} +#[derive(Debug, Clone, Serialize)] +pub struct WindowTiming { + pub ordinal: u64, + pub target_ns: String, + pub completed_ns: String, + pub receiver_latency_ns: String, + pub min_observed_span_ns: String, + pub max_observed_span_ns: String, + pub interval_since_previous_ns: Option, + pub min_source_rows: usize, + pub max_source_rows: usize, +} +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ObservationAvailability { + Unavailable, + Available, +} +#[derive(Debug, Clone, Serialize)] +pub struct ObservationStatus { + pub availability: ObservationAvailability, + pub reason: Option<&'static str>, + pub digest: Option, + pub first_receiver_ns: Option, + pub completed_ns: Option, + pub expires_ns: Option, + pub age_ns: Option, + pub source_count: usize, +} +impl ObservationStatus { + fn unavailable(reason: &'static str) -> Self { + Self { + availability: ObservationAvailability::Unavailable, + reason: Some(reason), + digest: None, + first_receiver_ns: None, + completed_ns: None, + expires_ns: None, + age_ns: None, + source_count: 0, + } + } +} +#[derive(Clone, Serialize)] +struct SourceBounds { + source_id: u8, + first_ns: u64, + last_ns: u64, +} +struct CompletedObservation { + prediction: RoomPrediction, + model_sha256: String, + policy_sha256: String, + context_digest: String, + source_bounds: Vec, + first_ns: u64, + completed_ns: u64, + expires_ns: u64, + digest: String, +} +#[derive(Debug, Clone, Serialize)] +pub struct LiveDiagnostic { + /// Buffer progress only; observation availability is separate. + pub buffer_state: LiveState, + pub reason: Option<&'static str>, + pub model_sha256: String, + pub policy_sha256: String, + pub target_horizon_ms: u64, + pub cadence: &'static str, + pub buffer_sources_with_rows: usize, + pub required_sources: usize, + pub observation: ObservationStatus, + pub current_prediction: Option, + pub last_completed: Option, + pub diagnostic_only: bool, + pub physically_validated: bool, + pub adoption_authorized: bool, +} +struct Fence { + sequence: u32, + time_ns: u64, + frame_id: u128, +} +#[derive(Default)] +struct WindowCount { + first_ns: u64, + last_ns: u64, + rows: usize, +} +/// No data-set retention, training schedule cursor, new persistence or clocks. +/// At most one <=4s opaque buffer, <=8 source counters/fences and one aggregate +/// completion are retained. Error withdrawal preserves accepted replay fences. +pub struct ContinuousTemporalInput { + model: TemporalModel, + policy: WindowInputPolicy, + status: LiveDiagnostic, + active: bool, + input: Option, + target_ns: Option, + end_ns: Option, + ordinal: u64, + fences: HashMap, + continuity: HashMap, + counts: HashMap, + last_now: Option, + observation: Option, +} +impl ContinuousTemporalInput { + pub fn new(model: TemporalModel) -> Result { + let policy = model.window_input_policy()?; + let bytes = model.export()?; + let status = LiveDiagnostic { + buffer_state: LiveState::Inactive, + reason: Some("inactive"), + model_sha256: format!("{:x}", Sha256::digest(bytes)), + policy_sha256: model.plan_sha256().into(), + target_horizon_ms: policy.target_span_ms(), + cadence: "nonoverlap_full_native_horizon", + buffer_sources_with_rows: 0, + required_sources: policy.context().sources.len(), + observation: ObservationStatus::unavailable("inactive"), + current_prediction: None, + last_completed: None, + diagnostic_only: true, + physically_validated: false, + adoption_authorized: false, + }; + Ok(Self { + model, + policy, + status, + active: false, + input: None, + target_ns: None, + end_ns: None, + ordinal: 0, + fences: HashMap::new(), + continuity: HashMap::new(), + counts: HashMap::new(), + last_now: None, + observation: None, + }) + } + pub fn activate(&mut self) { + self.active = true; + self.withdraw("collecting_full_horizon"); + self.status.buffer_state = LiveState::Warming; + } + pub fn deactivate(&mut self) { + self.active = false; + self.withdraw("inactive"); + self.status.buffer_state = LiveState::Inactive; + } + fn withdraw(&mut self, reason: &'static str) { + self.input = None; + self.target_ns = None; + self.end_ns = None; + self.counts.clear(); + self.continuity.clear(); + self.observation = None; + self.status.observation = ObservationStatus::unavailable(reason); + self.status.current_prediction = None; + self.status.buffer_sources_with_rows = 0; + self.status.buffer_state = if self.active { + LiveState::Withdrawn + } else { + LiveState::Inactive + }; + self.status.reason = Some(reason); + } + fn refresh_observation(&mut self, now_ns: u64) { + let Some(observation) = &self.observation else { + self.status.current_prediction = None; + return; + }; + if !self.active + || observation.model_sha256 != self.status.model_sha256 + || observation.policy_sha256 != self.status.policy_sha256 + || observation.context_digest != self.policy.context().digest() + { + self.withdraw("observation_identity_changed"); + return; + } + // Required sources, not just whichever source most recently delivered. + if self.policy.source_policies().iter().any(|p| { + self.continuity + .get(&p.source_id) + .is_none_or(|last| now_ns < *last || now_ns - *last > p.max_gap_ns) + }) { + self.withdraw("source_gap"); + return; + } + let Some(age) = now_ns.checked_sub(observation.first_ns) else { + self.withdraw("receiver_clock_reversed"); + return; + }; + let available = now_ns <= observation.expires_ns; + self.status.current_prediction = if available { + Some(observation.prediction.clone()) + } else { + None + }; + self.status.observation = ObservationStatus { + availability: if available { + ObservationAvailability::Available + } else { + ObservationAvailability::Unavailable + }, + reason: if available { + None + } else { + Some("original_first_row_expired") + }, + digest: Some(observation.digest.clone()), + first_receiver_ns: Some(observation.first_ns.to_string()), + completed_ns: Some(observation.completed_ns.to_string()), + expires_ns: Some(observation.expires_ns.to_string()), + age_ns: Some(age.to_string()), + source_count: observation.source_bounds.len(), + }; + } + /// Explicit receiver-domain tick; a silent source cannot leave a last result + /// looking current. No missing input or failure is converted into absence. + pub fn poll(&mut self, now_ns: u64) -> LiveDiagnostic { + if !self.active { + return self.status.clone(); + } + if self.last_now.is_some_and(|t| now_ns < t) { + self.withdraw("receiver_clock_reversed"); + return self.status.clone(); + } + self.last_now = Some(now_ns); + if self.continuity.iter().any(|(id, last)| { + self.policy + .source_policies() + .iter() + .find(|p| p.source_id == *id) + .is_none_or(|p| now_ns.saturating_sub(*last) > p.max_gap_ns) + }) { + self.withdraw("source_gap"); + } else if self.target_ns.is_some_and(|target| { + self.counts.len() < self.policy.context().sources.len() + && now_ns.saturating_sub(target) + > self.policy.max_first_packet_delay_ms() * 1_000_000 + }) { + self.withdraw("missing_source_warmup"); + } else if self.end_ns.is_some_and(|end| now_ns >= end) { + self.withdraw("incomplete_horizon"); + } + self.refresh_observation(now_ns); + self.status.clone() + } + pub fn ingest(&mut self, p: &CanonicalPacket, now_ns: u64) -> LiveDiagnostic { + if !self.active { + return self.status.clone(); + } + if self.last_now.is_some_and(|t| now_ns < t) { + self.withdraw("receiver_clock_reversed"); + return self.status.clone(); + } + self.last_now = Some(now_ns); + let time = p.native_frame.timestamp_ns; + if time > now_ns || now_ns - time > MAX_AGE_NS { + self.withdraw("stale_or_future_packet"); + return self.status.clone(); + } + let Some(rule) = self + .policy + .source_policies() + .iter() + .find(|s| s.source_id == p.source_id) + else { + self.withdraw("unbound_source"); + return self.status.clone(); + }; + let max_gap = rule.max_gap_ns; + if self.fences.get(&p.source_id).is_some_and(|f| { + p.sequence <= f.sequence || time <= f.time_ns || p.native_frame.frame_id == f.frame_id + }) { + self.withdraw("replayed_or_reversed_packet"); + return self.status.clone(); + } + if self + .continuity + .get(&p.source_id) + .is_some_and(|last| time.saturating_sub(*last) > max_gap) + { + self.withdraw("source_gap"); + return self.status.clone(); + } + // Also check every OTHER source against the receiver tick. A busy source + // cannot hide a dropout while continuing to fill its own window. + if self.continuity.iter().any(|(id, last)| { + self.policy + .source_policies() + .iter() + .find(|s| s.source_id == *id) + .is_none_or(|s| now_ns.saturating_sub(*last) > s.max_gap_ns) + }) { + self.withdraw("source_gap"); + return self.status.clone(); + } + if self.input.is_none() { + let end = match time.checked_add( + self.policy.max_observed_span_ms() * 1_000_000 + + self.policy.max_first_packet_delay_ms() * 1_000_000 + + 1, + ) { + Some(end) => end, + None => { + self.withdraw("receiver_time_overflow"); + return self.status.clone(); + } + }; + let mut input = match self + .model + .live_input(format!("continuous-live-{}", self.ordinal)) + { + Ok(input) => input, + Err(_) => { + self.withdraw("model_input_unavailable"); + return self.status.clone(); + } + }; + if input.begin_window(time, end).is_err() { + self.withdraw("window_start_refused"); + return self.status.clone(); + } + self.input = Some(input); + self.target_ns = Some(time); + self.end_ns = Some(end); + self.counts.clear(); + } + self.status.buffer_state = LiveState::Warming; + self.status.reason = Some("collecting_full_horizon"); + let admission = match self.input.as_mut().unwrap().ingest(p) { + Ok(v) => v, + Err(_) => { + self.withdraw("source_or_window_invalid"); + return self.status.clone(); + } + }; + self.fences.insert( + p.source_id, + Fence { + sequence: p.sequence, + time_ns: time, + frame_id: p.native_frame.frame_id, + }, + ); + self.continuity.insert(p.source_id, time); + let count = self.counts.entry(p.source_id).or_default(); + if count.rows == 0 { + count.first_ns = time; + } + count.last_ns = time; + count.rows += 1; + self.status.buffer_sources_with_rows = self.counts.len(); + if admission == WindowAdmission::Ready { + let target = self.target_ns.unwrap(); + let input = self.input.take().unwrap(); + let observation = match input.finish(now_ns) { + Ok(v) => v, + Err(_) => { + self.withdraw("window_evidence_invalid"); + return self.status.clone(); + } + }; + let prediction = match self.model.predict(observation, now_ns) { + Ok(v) => v, + Err(_) => { + self.withdraw("prediction_input_refused"); + return self.status.clone(); + } + }; + if prediction.physically_validated { + self.withdraw("unexpected_authority"); + return self.status.clone(); + } + let timing = WindowTiming { + ordinal: self.ordinal, + target_ns: target.to_string(), + completed_ns: now_ns.to_string(), + receiver_latency_ns: (now_ns - target).to_string(), + min_observed_span_ns: self + .counts + .values() + .map(|c| c.last_ns - c.first_ns) + .min() + .unwrap() + .to_string(), + max_observed_span_ns: self + .counts + .values() + .map(|c| c.last_ns - c.first_ns) + .max() + .unwrap() + .to_string(), + interval_since_previous_ns: self + .status + .last_completed + .as_ref() + .and_then(|w| w.completed_ns.parse::().ok()) + .map(|last| now_ns.saturating_sub(last).to_string()), + min_source_rows: self.counts.values().map(|c| c.rows).min().unwrap(), + max_source_rows: self.counts.values().map(|c| c.rows).max().unwrap(), + }; + let Some(next) = self.ordinal.checked_add(1) else { + self.withdraw("window_counter_exhausted"); + return self.status.clone(); + }; + let mut source_bounds = Vec::with_capacity(self.policy.context().sources.len()); + let mut first = u64::MAX; + let mut expires = u64::MAX; + for spec in &self.policy.context().sources { + let count = self.counts.get(&spec.source_id).unwrap(); + let Some(deadline) = count.first_ns.checked_add(MAX_AGE_NS) else { + self.withdraw("observation_time_overflow"); + return self.status.clone(); + }; + first = first.min(count.first_ns); + expires = expires.min(deadline); + source_bounds.push(SourceBounds { + source_id: spec.source_id, + first_ns: count.first_ns, + last_ns: count.last_ns, + }); + } + let encoded = match serde_json::to_vec(&( + &self.status.model_sha256, + &self.status.policy_sha256, + &source_bounds, + &timing, + &prediction, + )) { + Ok(v) => v, + Err(_) => { + self.withdraw("observation_encoding_failed"); + return self.status.clone(); + } + }; + self.observation = Some(CompletedObservation { + prediction, + model_sha256: self.status.model_sha256.clone(), + policy_sha256: self.status.policy_sha256.clone(), + context_digest: self.policy.context().digest(), + source_bounds, + first_ns: first, + completed_ns: now_ns, + expires_ns: expires, + digest: format!("{:x}", Sha256::digest(encoded)), + }); + self.ordinal = next; + self.target_ns = None; + self.end_ns = None; + self.counts.clear(); + self.status.buffer_state = LiveState::WindowComplete; + self.status.reason = None; + self.status.last_completed = Some(timing); + } + self.refresh_observation(now_ns); + self.status.clone() + } +} +#[cfg(test)] +#[path = "temporal_live_driver_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver_prior_control_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver_prior_control_tests.rs new file mode 100644 index 0000000000..bac73f92f6 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver_prior_control_tests.rs @@ -0,0 +1,337 @@ +//! Default-inactive, bounded continuous INPUT diagnostic, not a runtime/selected +//! detector. Every window uses the same TemporalModel opaque builder/infer path. +use super::{ + model::RoomPrediction, + native::CanonicalPacket, + observation::MAX_AGE_NS, + temporal_acquisition::{AcquisitionPlan, WindowAdmission}, + temporal_model_input::{TemporalLiveInput, TemporalModel}, +}; +use serde::Serialize; +use sha2::{Digest, Sha256}; +use std::collections::HashMap; +#[derive(Debug, Clone, Copy, Serialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum LiveState { + Inactive, + Warming, + Current, + Withdrawn, +} +#[derive(Debug, Clone, Serialize)] +pub struct WindowTiming { + pub ordinal: u64, + pub target_ns: String, + pub completed_ns: String, + pub receiver_latency_ns: String, + pub min_observed_span_ns: String, + pub max_observed_span_ns: String, + pub interval_since_previous_ns: Option, + pub min_source_rows: usize, + pub max_source_rows: usize, +} +#[derive(Debug, Clone, Serialize)] +pub struct LiveDiagnostic { + pub state: LiveState, + pub reason: Option<&'static str>, + pub model_sha256: String, + pub policy_sha256: String, + pub target_horizon_ms: u64, + pub cadence: &'static str, + pub ready_sources: usize, + pub required_sources: usize, + pub current_prediction: Option, + pub last_completed: Option, + pub diagnostic_only: bool, + pub physically_validated: bool, + pub adoption_authorized: bool, +} +struct Fence { + sequence: u32, + time_ns: u64, + frame_id: u128, +} +#[derive(Default)] +struct WindowCount { + first_ns: u64, + last_ns: u64, + rows: usize, +} +/// No data-set retention, training schedule cursor, new persistence or clocks. +/// At most one <=4s opaque buffer, <=8 source counters/fences and one aggregate +/// completion are retained. Error withdrawal preserves accepted replay fences. +pub struct ContinuousTemporalInput { + model: TemporalModel, + policy: AcquisitionPlan, + status: LiveDiagnostic, + active: bool, + input: Option, + target_ns: Option, + end_ns: Option, + ordinal: u64, + fences: HashMap, + continuity: HashMap, + counts: HashMap, + last_now: Option, +} +impl ContinuousTemporalInput { + pub fn new(model: TemporalModel) -> Result { + let policy = model.input_policy()?; + let bytes = model.export()?; + let status = LiveDiagnostic { + state: LiveState::Inactive, + reason: Some("inactive"), + model_sha256: format!("{:x}", Sha256::digest(bytes)), + policy_sha256: model.plan_sha256().into(), + target_horizon_ms: policy.target_span_ms(), + cadence: "nonoverlap_full_native_horizon", + ready_sources: 0, + required_sources: policy.context().sources.len(), + current_prediction: None, + last_completed: None, + diagnostic_only: true, + physically_validated: false, + adoption_authorized: false, + }; + Ok(Self { + model, + policy, + status, + active: false, + input: None, + target_ns: None, + end_ns: None, + ordinal: 0, + fences: HashMap::new(), + continuity: HashMap::new(), + counts: HashMap::new(), + last_now: None, + }) + } + pub fn activate(&mut self) { + self.active = true; + self.withdraw("collecting_full_horizon"); + self.status.state = LiveState::Warming; + } + pub fn deactivate(&mut self) { + self.active = false; + self.withdraw("inactive"); + self.status.state = LiveState::Inactive; + } + fn withdraw(&mut self, reason: &'static str) { + self.input = None; + self.target_ns = None; + self.end_ns = None; + self.counts.clear(); + self.continuity.clear(); + self.status.current_prediction = None; + self.status.ready_sources = 0; + self.status.state = if self.active { + LiveState::Withdrawn + } else { + LiveState::Inactive + }; + self.status.reason = Some(reason); + } + /// Explicit receiver-domain tick; a silent source cannot leave a last result + /// looking current. No missing input or failure is converted into absence. + pub fn poll(&mut self, now_ns: u64) -> LiveDiagnostic { + if !self.active { + return self.status.clone(); + } + if self.last_now.is_some_and(|t| now_ns < t) { + self.withdraw("receiver_clock_reversed"); + return self.status.clone(); + } + self.last_now = Some(now_ns); + if self.continuity.iter().any(|(id, last)| { + self.policy + .source_policies() + .iter() + .find(|p| p.source_id == *id) + .is_none_or(|p| now_ns.saturating_sub(*last) > p.max_gap_ns) + }) { + self.withdraw("source_gap"); + } else if self.target_ns.is_some_and(|target| { + self.counts.len() < self.policy.context().sources.len() + && now_ns.saturating_sub(target) + > self.policy.max_first_packet_delay_ms() * 1_000_000 + }) { + self.withdraw("missing_source_warmup"); + } else if self.end_ns.is_some_and(|end| now_ns >= end) { + self.withdraw("incomplete_horizon"); + } + self.status.clone() + } + pub fn ingest(&mut self, p: &CanonicalPacket, now_ns: u64) -> LiveDiagnostic { + if !self.active { + return self.status.clone(); + } + if self.last_now.is_some_and(|t| now_ns < t) { + self.withdraw("receiver_clock_reversed"); + return self.status.clone(); + } + self.last_now = Some(now_ns); + let time = p.native_frame.timestamp_ns; + if time > now_ns || now_ns - time > MAX_AGE_NS { + self.withdraw("stale_or_future_packet"); + return self.status.clone(); + } + let Some(rule) = self + .policy + .source_policies() + .iter() + .find(|s| s.source_id == p.source_id) + else { + self.withdraw("unbound_source"); + return self.status.clone(); + }; + let max_gap = rule.max_gap_ns; + if self.fences.get(&p.source_id).is_some_and(|f| { + p.sequence <= f.sequence || time <= f.time_ns || p.native_frame.frame_id == f.frame_id + }) { + self.withdraw("replayed_or_reversed_packet"); + return self.status.clone(); + } + if self + .continuity + .get(&p.source_id) + .is_some_and(|last| time.saturating_sub(*last) > max_gap) + { + self.withdraw("source_gap"); + return self.status.clone(); + } + // Also check every OTHER source against the receiver tick. A busy source + // cannot hide a dropout while continuing to fill its own window. + if self.continuity.iter().any(|(id, last)| { + self.policy + .source_policies() + .iter() + .find(|s| s.source_id == *id) + .is_none_or(|s| now_ns.saturating_sub(*last) > s.max_gap_ns) + }) { + self.withdraw("source_gap"); + return self.status.clone(); + } + if self.input.is_none() { + let end = match time.checked_add( + self.policy.max_observed_span_ms() * 1_000_000 + + self.policy.max_first_packet_delay_ms() * 1_000_000 + + 1, + ) { + Some(end) => end, + None => { + self.withdraw("receiver_time_overflow"); + return self.status.clone(); + } + }; + let mut input = match self + .model + .live_input(format!("continuous-live-{}", self.ordinal)) + { + Ok(input) => input, + Err(_) => { + self.withdraw("model_input_unavailable"); + return self.status.clone(); + } + }; + if input.begin_window(time, end).is_err() { + self.withdraw("window_start_refused"); + return self.status.clone(); + } + self.input = Some(input); + self.target_ns = Some(time); + self.end_ns = Some(end); + self.counts.clear(); + } + self.status.state = LiveState::Warming; + self.status.reason = Some("collecting_full_horizon"); + self.status.current_prediction = None; + let admission = match self.input.as_mut().unwrap().ingest(p) { + Ok(v) => v, + Err(_) => { + self.withdraw("source_or_window_invalid"); + return self.status.clone(); + } + }; + self.fences.insert( + p.source_id, + Fence { + sequence: p.sequence, + time_ns: time, + frame_id: p.native_frame.frame_id, + }, + ); + self.continuity.insert(p.source_id, time); + let count = self.counts.entry(p.source_id).or_default(); + if count.rows == 0 { + count.first_ns = time; + } + count.last_ns = time; + count.rows += 1; + self.status.ready_sources = self.counts.len(); + if admission == WindowAdmission::Ready { + let target = self.target_ns.unwrap(); + let input = self.input.take().unwrap(); + let observation = match input.finish(now_ns) { + Ok(v) => v, + Err(_) => { + self.withdraw("window_evidence_invalid"); + return self.status.clone(); + } + }; + let prediction = match self.model.predict(observation, now_ns) { + Ok(v) => v, + Err(_) => { + self.withdraw("prediction_input_refused"); + return self.status.clone(); + } + }; + if prediction.physically_validated { + self.withdraw("unexpected_authority"); + return self.status.clone(); + } + let timing = WindowTiming { + ordinal: self.ordinal, + target_ns: target.to_string(), + completed_ns: now_ns.to_string(), + receiver_latency_ns: (now_ns - target).to_string(), + min_observed_span_ns: self + .counts + .values() + .map(|c| c.last_ns - c.first_ns) + .min() + .unwrap() + .to_string(), + max_observed_span_ns: self + .counts + .values() + .map(|c| c.last_ns - c.first_ns) + .max() + .unwrap() + .to_string(), + interval_since_previous_ns: self + .status + .last_completed + .as_ref() + .and_then(|w| w.completed_ns.parse::().ok()) + .map(|last| now_ns.saturating_sub(last).to_string()), + min_source_rows: self.counts.values().map(|c| c.rows).min().unwrap(), + max_source_rows: self.counts.values().map(|c| c.rows).max().unwrap(), + }; + let Some(next) = self.ordinal.checked_add(1) else { + self.withdraw("window_counter_exhausted"); + return self.status.clone(); + }; + self.ordinal = next; + self.target_ns = None; + self.end_ns = None; + self.counts.clear(); + self.status.state = LiveState::Current; + self.status.reason = None; + self.status.current_prediction = Some(prediction); + self.status.last_completed = Some(timing); + } + self.status.clone() + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver_tests.rs new file mode 100644 index 0000000000..33e3fae162 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver_tests.rs @@ -0,0 +1,646 @@ +//! SYNTHETIC native sources; actual fitter/store/cold loader/opaque infer path. +use crate::room_learner::temporal_acquisition::AcquisitionPlan; +use super::*; +use crate::room_learner::{ + artifact::ArtifactStore, + native::{canonicalize_esp32, NativePacketInput}, + observation::*, + temporal_acquisition::{ExperimentalPlanId, ReceiverCadenceBand}, + temporal_model_input::TemporalTrainingSession, + training::FitConfig, +}; +use std::sync::OnceLock; +use wifi_densepose_hardware::{Adr018Flags, PpduType}; +const PERIOD: u64 = 30_303_031; +const FRESH: u64 = 1_000_000_000_000; +fn context() -> CaptureContext { + CaptureContext { + owner_scope: "a".repeat(64), + installation_id: "synthetic-continuous".into(), + room_digest: "synthetic-room".into(), + revision: 1, + sources: (1..=2) + .map(|source_id| SourceSpec { + source_id, + device_id: format!("esp32-node-{source_id}"), + grid_id: "ant1-bins64-ppdu0-usable-drop2-linear-v2".into(), + feature_count: 56, + signal_kind: ruview_unified::RfModality::WifiCsi, + center_frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + link_id: format!("native-{source_id}"), + link_index: 0, + geometry: None, + }) + .collect(), + } +} +fn packet(source: u8, sequence: u32, time: u64, label: RoomLabel, row: usize) -> CanonicalPacket { + let gain = match label { + RoomLabel::Empty => 3, + RoomLabel::AdjacentOnly => 4, + RoomLabel::OccupiedStatic => 8, + RoomLabel::OccupiedMoving => 10 + (row % 2) as u8, + RoomLabel::Unknown => 8, + }; + let bytes = (0..64) + .flat_map(|bin| [gain + source + (bin % 2) as u8, 4]) + .collect::>(); + canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence, + received_at_ns: time, + receiver_sample_rate_hz: 33., + iq_bytes: &bytes, + flags: Adr018Flags::default(), + provenance: CaptureProvenance::Synthetic, + }) + .unwrap() +} +fn fitted() -> TemporalModel { + fitted_id(ExperimentalPlanId::Native2000msSlots2) +} +fn fitted_id(id: ExperimentalPlanId) -> TemporalModel { + static SHORT: OnceLock> = OnceLock::new(); + static LONG: OnceLock> = OnceLock::new(); + let bytes = if id == ExperimentalPlanId::Native2000msSlots2 { + &SHORT + } else { + &LONG + }; + TemporalModel::import(bytes.get_or_init(|| { + let mut s = TemporalTrainingSession::new( + context(), + id, + &[ + (1, ReceiverCadenceBand::Hz30To36), + (2, ReceiverCadenceBand::Hz30To36), + ], + "synthetic-continuous-fit".into(), + FitConfig { + epochs: 2, + ..Default::default() + }, + ) + .unwrap(); + let plan = AcquisitionPlan::build( + context(), + id, + &[ + (1, ReceiverCadenceBand::Hz30To36), + (2, ReceiverCadenceBand::Hz30To36), + ], + ) + .unwrap(); + let mut sequence = 0; + for index in 0..16 { + let at = 10_000_000_000 + index * 5_000_000_000; + let label = plan.schedule()[index as usize / 2].label; + s.begin_window(at, at + 5_000_000_000).unwrap(); + 'window: for row in 0..256 { + sequence += 1; + for source in 1..=2 { + let now = at + row as u64 * PERIOD + u64::from(source) * 1000; + if s.ingest(&packet(source, sequence, now, label, row)) + .unwrap() + == WindowAdmission::Ready + { + s.finish_window(now).unwrap(); + break 'window; + } + } + } + } + let (model, report) = s.fit().unwrap(); + assert!(report.backbone_delta_l2 > 0.); + assert_eq!(report.training_cohorts, 16); + model.export().unwrap() + })) + .unwrap() +} +fn feed( + d: &mut ContinuousTemporalInput, + start: u64, + rows: usize, + sequence: u32, +) -> Vec { + let mut completed = vec![]; + for row in 0..rows { + for source in 1..=2 { + let now = start + row as u64 * PERIOD + u64::from(source) * 1000; + let status = d.ingest( + &packet(source, sequence + row as u32, now, RoomLabel::Unknown, row), + now, + ); + if status.buffer_state == LiveState::WindowComplete { + completed.push(status); + } + } + } + completed +} +fn driver() -> ContinuousTemporalInput { + ContinuousTemporalInput::new(fitted()).unwrap() +} +#[test] +fn default_inactive_and_short_inputs_never_produce_absence_or_prediction() { + let mut d = driver(); + let p = packet(1, 100_000, FRESH, RoomLabel::Unknown, 0); + assert_eq!(d.ingest(&p, FRESH).buffer_state, LiveState::Inactive); + assert!(d.fences.is_empty()); + d.activate(); + assert!(feed(&mut d, FRESH, 8, 100_000).is_empty()); + let status = d.poll(FRESH + 7 * PERIOD + 2000); + assert_eq!(status.buffer_state, LiveState::Warming); + assert!(status.current_prediction.is_none()); + assert!(status.diagnostic_only && !status.physically_validated && !status.adoption_authorized); +} +#[test] +#[cfg(any(target_os = "macos", target_os = "linux"))] +fn actual_fit_store_cold_load_then_continuous_windows_use_the_same_infer_path() { + let dir = tempfile::tempdir().unwrap(); + let ctx = context(); + let store = ArtifactStore::new( + &dir.path().join(&ctx.owner_scope).join("SensingServer"), + &ctx.owner_scope, + &ctx.installation_id, + ) + .unwrap(); + let model = fitted(); + let receipt = store.save_temporal_candidate(&model).unwrap(); + let cold = store + .load_exact_temporal_candidate(&ctx, &receipt.artifact_sha256) + .unwrap(); + assert_eq!(cold.export().unwrap(), model.export().unwrap()); + let mut d = ContinuousTemporalInput::new(cold).unwrap(); + assert_eq!(d.status.model_sha256, receipt.artifact_sha256); + d.activate(); + let results = feed(&mut d, FRESH, 201, 100_000); + assert_eq!(results.len(), 3); + assert_eq!(d.ordinal, 3); + assert_eq!(d.fences.len(), 2); + assert!(d.input.is_none()); + assert!(d.counts.is_empty()); + for (i, status) in results.iter().enumerate() { + let t = status.last_completed.as_ref().unwrap(); + assert_eq!(t.ordinal, i as u64); + assert!(t.min_observed_span_ns.parse::().unwrap() >= 2_000_000_000); + assert_eq!(t.min_source_rows, 67); + assert_eq!(t.max_source_rows, 67); + assert!(status + .current_prediction + .as_ref() + .is_some_and(|p| !p.physically_validated)); + } + // Independently call the SAME opaque builder and model infer on the exact + // first source interval, without access to the inner v1 model. + let mut input = model.live_input("comparison-window".into()).unwrap(); + input + .begin_window(FRESH + 1000, FRESH + 3_000_000_000) + .unwrap(); + let mut now = FRESH; + 'ready: for row in 0..67 { + for source in 1..=2 { + now = FRESH + row as u64 * PERIOD + u64::from(source) * 1000; + if input + .ingest(&packet( + source, + 100_000 + row as u32, + now, + RoomLabel::Unknown, + row, + )) + .unwrap() + == WindowAdmission::Ready + { + break 'ready; + } + } + } + let expected = model.predict(input.finish(now).unwrap(), now).unwrap(); + assert_eq!( + serde_json::to_value(expected).unwrap(), + serde_json::to_value(results[0].current_prediction.as_ref().unwrap()).unwrap() + ); + assert!( + results[1] + .last_completed + .as_ref() + .unwrap() + .interval_since_previous_ns + .as_ref() + .unwrap() + .parse::() + .unwrap() + >= 2_000_000_000 + ); + let next = FRESH + 201 * PERIOD + 1000; + let status = d.ingest(&packet(1, 100_201, next, RoomLabel::Unknown, 201), next); + assert_eq!(status.buffer_state, LiveState::Warming); + assert!(status.current_prediction.is_some()); + assert_eq!( + status.observation.availability, + ObservationAvailability::Available + ); + assert_eq!(status.last_completed.as_ref().unwrap().ordinal, 2); +} +#[test] +fn replay_clock_and_sequence_fences_survive_window_rollover_and_reactivation() { + let mut d = driver(); + d.activate(); + assert_eq!(feed(&mut d, FRESH, 67, 100_000).len(), 1); + let now = FRESH + 67 * PERIOD + 1000; + let status = d.ingest(&packet(1, 100_067, now, RoomLabel::Unknown, 67), now); + assert_eq!(status.buffer_state, LiveState::Warming); + let old = packet( + 2, + 100_066, + FRESH + 66 * PERIOD + 2000, + RoomLabel::Unknown, + 66, + ); + let status = d.ingest(&old, now); + assert_eq!(status.buffer_state, LiveState::Withdrawn); + assert_eq!(status.reason, Some("replayed_or_reversed_packet")); + assert!(status.current_prediction.is_none()); + d.deactivate(); + d.activate(); + let status = d.ingest(&old, now + 1); + assert_eq!(status.reason, Some("replayed_or_reversed_packet")); + assert_eq!(d.fences.len(), 2); +} +#[test] +fn source_gap_and_polling_withdraw_without_renewing_source_freshness_then_fresh_warmup_recovers() { + let mut d = driver(); + d.activate(); + feed(&mut d, FRESH, 10, 100_000); + let last = FRESH + 9 * PERIOD + 2000; + let status = d.poll(last + 100_000_000); + assert_eq!(status.buffer_state, LiveState::Withdrawn); + assert_eq!(status.reason, Some("source_gap")); + assert!(status.current_prediction.is_none()); + let fences = d.fences.len(); + d.poll(last + 200_000_000); + assert_eq!(d.fences.len(), fences); + let fresh = last + 300_000_000; + let results = feed(&mut d, fresh, 67, 100_010); + assert_eq!(results.len(), 1); + let at = fresh + 66 * PERIOD + 2000; + let status = d.poll(at + 100_000_000); + assert_eq!(status.buffer_state, LiveState::Withdrawn); + assert!(status.current_prediction.is_none()); + assert_eq!(status.last_completed.as_ref().unwrap().ordinal, 0); +} +#[test] +fn cold_training_fences_invalid_phy_stale_and_reversed_receiver_clock_never_emit_prediction() { + let mut d = driver(); + d.activate(); + let old = packet(1, 1, FRESH + 1000, RoomLabel::Unknown, 0); + let status = d.ingest(&old, FRESH + 1000); + assert_eq!(status.reason, Some("source_or_window_invalid")); + assert!(d.fences.is_empty()); + for kind in ["grid", "frequency", "source", "stale", "clock"] { + let mut d = driver(); + d.activate(); + feed(&mut d, FRESH, 8, 100_000); + let last = FRESH + 7 * PERIOD + 2000; + let mut p = packet(1, 100_008, FRESH + 8 * PERIOD + 1000, RoomLabel::Unknown, 8); + let mut now = p.native_frame.timestamp_ns; + match kind { + "grid" => p.grid_id = "wrong-grid".into(), + "frequency" => p.frequency_hz = 5_180_000_000, + "source" => p.source_id = 99, + "stale" => now += 6_000_000_000, + "clock" => now = last - 1, + _ => unreachable!(), + } + let before = d.fences[&1].sequence; + let status = d.ingest(&p, now); + assert_eq!(status.buffer_state, LiveState::Withdrawn, "{kind}"); + assert!(status.current_prediction.is_none()); + assert_eq!( + d.fences[&1].sequence, before, + "invalid packet advanced fence: {kind}" + ); + } +} + +#[test] +fn repeated_poll_clock_reversal_cannot_lower_the_existing_receiver_floor() { + let mut d = driver(); + d.activate(); + feed(&mut d, FRESH, 8, 100_000); + let floor = FRESH + 7 * PERIOD + 2000; + for now in [floor - 1, floor - 100_000_000, floor - 2] { + let s = d.poll(now); + assert_eq!(s.reason, Some("receiver_clock_reversed")); + assert_eq!(d.last_now, Some(floor)); + assert!(s.current_prediction.is_none()); + } + d.deactivate(); + d.activate(); + assert_eq!(d.poll(floor - 1).reason, Some("receiver_clock_reversed")); + assert_eq!(d.last_now, Some(floor)); + let now = floor + PERIOD; + let s = d.ingest(&packet(1, 100_008, now, RoomLabel::Unknown, 8), now); + assert_eq!(s.buffer_state, LiveState::Warming); + assert!(s.current_prediction.is_none()); +} +#[test] +fn source_dropout_across_a_completed_window_boundary_withdraws_the_busy_source() { + let mut d = driver(); + d.activate(); + assert_eq!(feed(&mut d, FRESH, 67, 100_000).len(), 1); + let next = FRESH + 67 * PERIOD + 1000; + assert_eq!( + d.ingest(&packet(1, 100_067, next, RoomLabel::Unknown, 67), next) + .buffer_state, + LiveState::Warming + ); + let later = FRESH + 68 * PERIOD + 1000; + let s = d.ingest(&packet(1, 100_068, later, RoomLabel::Unknown, 68), later); + assert_eq!(s.buffer_state, LiveState::Withdrawn); + assert_eq!(s.reason, Some("source_gap")); + assert!(s.current_prediction.is_none()); + assert_eq!(s.last_completed.as_ref().unwrap().ordinal, 0); + assert_eq!(d.fences[&1].sequence, 100_067); + assert_eq!(d.fences[&2].sequence, 100_066); +} +#[test] +fn fresh_timestamps_do_not_authorize_a_counter_reset_after_rollover() { + let mut d = driver(); + d.activate(); + assert_eq!(feed(&mut d, FRESH, 67, 100_000).len(), 1); + let now = FRESH + 67 * PERIOD + 1000; + let reset = packet(1, 100_000, now, RoomLabel::Unknown, 67); + let s = d.ingest(&reset, now); + assert_eq!(s.reason, Some("replayed_or_reversed_packet")); + assert_eq!(s.buffer_state, LiveState::Withdrawn); + assert!(s.current_prediction.is_none()); + assert_eq!(d.fences[&1].sequence, 100_066); +} +#[test] +fn invalid_waveform_values_or_shape_withdraw_for_each_bound_source_without_advancing_fences() { + for source in 1..=2 { + for kind in ["nan", "shape"] { + let mut d = driver(); + d.activate(); + feed(&mut d, FRESH, 8, 100_000); + let now = FRESH + 8 * PERIOD + u64::from(source) * 1000; + let mut bad = packet(source, 100_008, now, RoomLabel::Unknown, 8); + if kind == "nan" { + bad.amplitudes[0] = f64::NAN; + } else { + bad.amplitudes.truncate(8); + } + let before = d.fences[&source].sequence; + let s = d.ingest(&bad, now); + assert_eq!(s.reason, Some("source_or_window_invalid")); + assert!(s.current_prediction.is_none()); + assert_eq!(d.fences[&source].sequence, before); + } + } +} + +#[test] +fn continuous_rollover_exceeds_finite_training_schedule_without_retaining_dataset_rows() { + let mut d = driver(); + let bound = d.model.input_policy().unwrap().resources().cohorts; + assert_eq!(bound, 32); + d.activate(); + let results = feed(&mut d, FRESH, (bound + 2) * 67, 100_000); + assert_eq!(results.len(), bound + 2); + assert_eq!(d.ordinal, (bound + 2) as u64); + assert_eq!(d.fences.len(), 2); + assert_eq!(d.continuity.len(), 2); + assert!(d.counts.is_empty()); + assert!(d.input.is_none()); + assert_eq!( + d.status.last_completed.as_ref().unwrap().ordinal, + (bound + 1) as u64 + ); + assert_eq!( + format!("{:x}", Sha256::digest(d.model.export().unwrap())), + d.status.model_sha256 + ); +} + +// The production portion of the frozen072 driver, verbatim, is test control +// only. It calls the SAME opaque model/infer; no parallel trainer is introduced. +use crate::room_learner::{model, native, observation, temporal_acquisition, temporal_model_input}; +#[allow(dead_code)] +#[path = "temporal_live_driver_prior_control_tests.rs"] +mod prior_control; + +#[test] +fn healthy_two_second_stream_retains_original_observation_during_collection_and_improves_duty_cycle( +) { + let model = fitted(); + let mut d = ContinuousTemporalInput::new(model.clone()).unwrap(); + let mut old = prior_control::ContinuousTemporalInput::new(model).unwrap(); + d.activate(); + old.activate(); + let mut previous = None; + let mut elapsed_new = 0u64; + let mut elapsed_old = 0u64; + let mut first_complete = None; + let mut digest = None; + let mut deadline = None; + for row in 0..201 { + for source in 1..=2 { + let now = FRESH + row as u64 * PERIOD + u64::from(source) * 1000; + if let Some((time, new_available, old_available)) = previous { + if new_available { + elapsed_new += now - time; + } + if old_available { + elapsed_old += now - time; + } + } + let p = packet(source, 100_000 + row as u32, now, RoomLabel::Unknown, row); + let new = d.ingest(&p, now); + let control = old.ingest(&p, now); + if row < 66 { + assert!(new.current_prediction.is_none()); + } + if row == 66 && source == 2 { + first_complete = Some(now); + digest = new.observation.digest.clone(); + deadline = new.observation.expires_ns.clone(); + } + if row == 67 { + assert_eq!(new.buffer_state, LiveState::Warming); + assert_eq!( + new.observation.availability, + ObservationAvailability::Available + ); + assert_eq!(new.observation.digest, digest); + assert_eq!(new.observation.expires_ns, deadline); + assert!(new.current_prediction.is_some()); + } + previous = Some(( + now, + new.current_prediction.is_some(), + control.current_prediction.is_some(), + )); + } + } + let end = FRESH + 200 * PERIOD + 2000; + assert_eq!(elapsed_new, end - first_complete.unwrap()); + assert!(elapsed_old * 20 < elapsed_new); + assert_eq!(d.ordinal, 3); +} +#[test] +fn first_row_deadline_is_fixed_exact_and_expiry_preserves_a_healthy_long_window() { + let model = fitted_id(ExperimentalPlanId::Native3900msMax4000Slots2); + let mut d = ContinuousTemporalInput::new(model).unwrap(); + d.activate(); + let completed = feed(&mut d, FRESH, 130, 100_000); + assert_eq!(completed.len(), 1); + let initial = completed[0].observation.clone(); + let expires = initial.expires_ns.as_ref().unwrap().parse::().unwrap(); + let first = initial + .first_receiver_ns + .as_ref() + .unwrap() + .parse::() + .unwrap(); + assert_eq!(expires, first + MAX_AGE_NS); + // Keep all sources healthy, collecting the next3.9s window, through the old + // result's5s FIRST-row deadline; its LAST-row+5s bound would be much later. + for row in 130..165 { + for source in 1..=2 { + let now = FRESH + row as u64 * PERIOD + u64::from(source) * 1000; + let s = d.ingest( + &packet(source, 100_000 + row as u32, now, RoomLabel::Unknown, row), + now, + ); + assert_eq!(s.observation.digest, initial.digest); + assert_eq!(s.observation.expires_ns, initial.expires_ns); + } + } + assert!(d.input.is_some()); + let count = d.counts.values().map(|c| c.rows).sum::(); + let at = d.poll(expires); + assert_eq!( + at.observation.availability, + ObservationAvailability::Available + ); + assert_eq!(at.observation.age_ns.as_deref(), Some("5000000000")); + let expired = d.poll(expires + 1); + assert_eq!(expired.buffer_state, LiveState::Warming); + assert_eq!( + expired.observation.availability, + ObservationAvailability::Unavailable + ); + assert_eq!( + expired.observation.reason, + Some("original_first_row_expired") + ); + assert!(expired.current_prediction.is_none()); + assert!(d.input.is_some()); + assert_eq!(d.counts.values().map(|c| c.rows).sum::(), count); + assert_eq!(d.ordinal, 1); + for _ in 0..3 { + let same = d.poll(expires + 1); + assert_eq!(same.observation.digest, initial.digest); + assert_eq!(same.observation.expires_ns, initial.expires_ns); + assert!(same.current_prediction.is_none()); + } + let next = FRESH + 165 * PERIOD + 1000; + let after_packet = d.ingest(&packet(1, 100_165, next, RoomLabel::Unknown, 165), next); + assert_eq!(after_packet.observation.digest, initial.digest); + assert_eq!(after_packet.observation.expires_ns, initial.expires_ns); + assert!(after_packet.current_prediction.is_none()); + assert!( + expires + < initial + .completed_ns + .as_ref() + .unwrap() + .parse::() + .unwrap() + + MAX_AGE_NS, + "causal first-vs-last boundary missing" + ); +} +#[test] +fn gap_invalidity_and_deactivation_cannot_resurrect_a_retained_observation() { + for mode in ["gap", "invalid", "deactivate"] { + let mut d = driver(); + d.activate(); + let complete = feed(&mut d, FRESH, 67, 100_000); + assert!(complete[0].current_prediction.is_some()); + let now = FRESH + 67 * PERIOD + 1000; + let s = match mode { + "gap" => d.poll(now + 100_000_000), + "invalid" => { + let mut bad = packet(1, 100_067, now, RoomLabel::Unknown, 67); + bad.amplitudes[0] = f64::NAN; + d.ingest(&bad, now) + } + _ => { + d.deactivate(); + d.poll(now) + } + }; + assert!(s.current_prediction.is_none()); + assert!(s.observation.digest.is_none()); + assert!(d.observation.is_none()); + d.activate(); + let reactivated = d.poll(now + 100_000_001); + assert!(reactivated.current_prediction.is_none()); + assert!(reactivated.observation.digest.is_none()); + } +} + +#[test] +fn repeated_available_reads_age_the_same_observation_without_changing_identity_or_deadline() { + let mut d = driver(); + d.activate(); + let completed = feed(&mut d, FRESH, 67, 100_000); + let initial = completed[0].observation.clone(); + let now = FRESH + 66 * PERIOD + 2000; + for offset in [1, 10, 1000, 1_000_000] { + let status = d.poll(now + offset); + assert_eq!( + status.observation.availability, + ObservationAvailability::Available + ); + assert_eq!(status.observation.digest, initial.digest); + assert_eq!( + status.observation.first_receiver_ns, + initial.first_receiver_ns + ); + assert_eq!(status.observation.completed_ns, initial.completed_ns); + assert_eq!(status.observation.expires_ns, initial.expires_ns); + assert_eq!( + status + .observation + .age_ns + .as_ref() + .unwrap() + .parse::() + .unwrap(), + now + offset + - initial + .first_receiver_ns + .as_ref() + .unwrap() + .parse::() + .unwrap() + ); + assert_eq!(d.ordinal, 1); + assert!(!status.physically_validated && !status.adoption_authorized); + } + // A model replacement is a new inactive owner-scoped driver; it cannot borrow + // a cached result merely because the cold bytes/model policy are identical. + let replacement = ContinuousTemporalInput::new(fitted()).unwrap(); + assert!(replacement.observation.is_none()); + assert!(replacement.status.current_prediction.is_none()); + assert_eq!(replacement.status.buffer_state, LiveState::Inactive); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input.rs new file mode 100644 index 0000000000..0ac8717e2d --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input.rs @@ -0,0 +1,845 @@ +//! Experimental input-policy adapter over the EXISTING RoomModel/fitter. +//! No API/runtime/storage/managed-proof/selection authority. Imported metadata +//! is integrity-checked historical input claims, not physical attestation. +use super::{ + model::{EncoderConfig, RoomModel, RoomPrediction}, + native::CanonicalPacket, + observation::*, + temporal_acquisition::{ + AcquisitionPlan, ExperimentalPlanId, NativeWindowAssembler, ReceiverCadenceBand, + TemporalWindow, WindowAdmission, + }, + training::{FitConfig, TrainingReport}, +}; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::collections::{HashMap, HashSet}; +pub(crate) const SCHEMA: &str = "ruview.experimental.temporal-model-input.v2"; +const MAX_BYTES: usize = 4 * 1024 * 1024; +fn hash(v: &T) -> Result { + Ok(format!( + "{:x}", + Sha256::digest(serde_json::to_vec(v).map_err(|_| "temporal_encoding")?) + )) +} +fn bytes_hash(v: &[u8]) -> String { + format!("{:x}", Sha256::digest(v)) +} +fn sha(s: &str) -> bool { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} +fn integer(s: &str) -> Result { + let n = s.parse::().map_err(|_| "temporal_integer")?; + if n.to_string() != s { + return Err("temporal_noncanonical_integer".into()); + } + Ok(n) +} +fn plan_id(s: &str) -> Result { + use ExperimentalPlanId::*; + Ok(match s { + "Native1000msSlots8" => Native1000msSlots8, + "Native1000msSlots4" => Native1000msSlots4, + "Native1000msSlots2" => Native1000msSlots2, + "Native2000msSlots8" => Native2000msSlots8, + "Native2000msSlots4" => Native2000msSlots4, + "Native2000msSlots2" => Native2000msSlots2, + "Native3900msMax4000Slots8" => Native3900msMax4000Slots8, + "Native3900msMax4000Slots4" => Native3900msMax4000Slots4, + "Native3900msMax4000Slots2" => Native3900msMax4000Slots2, + _ => return Err("temporal_plan_id".into()), + }) +} +fn band(s: &str) -> Result { + match s { + "Hz30To36" => Ok(ReceiverCadenceBand::Hz30To36), + "Hz45To55" => Ok(ReceiverCadenceBand::Hz45To55), + _ => Err("temporal_cadence_band".into()), + } +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +struct SourcePolicy { + source_id: u8, + band: String, + min_rows: usize, + max_rows: usize, + min_interval_ns: u64, + max_gap_ns: u64, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +struct Binding { + plan_id: String, + native_frame_identity: String, + plan_sha256: String, + context: CaptureContext, + encoder: EncoderConfig, + target_span_ms: u64, + max_span_ms: u64, + max_pairwise_skew_ms: u64, + max_first_delay_ms: u64, + sources: Vec, +} +impl Binding { + fn from_plan(plan: &AcquisitionPlan) -> Self { + Self { + plan_id: format!("{:?}", plan.id()), + native_frame_identity: "existing-esp32-receiver-ns-node-u32-sequence-v1".into(), + plan_sha256: plan.sha256().into(), + context: plan.context().clone(), + encoder: plan.encoder_config().clone(), + target_span_ms: plan.target_span_ms(), + max_span_ms: plan.max_observed_span_ms(), + max_pairwise_skew_ms: plan.max_pairwise_receiver_skew_ms(), + max_first_delay_ms: plan.max_first_packet_delay_ms(), + sources: plan + .source_policies() + .iter() + .map(|s| SourcePolicy { + source_id: s.source_id, + band: format!("{:?}", s.receiver_cadence), + min_rows: s.min_rows, + max_rows: s.max_rows, + min_interval_ns: s.min_packet_interval_ns, + max_gap_ns: s.max_gap_ns, + }) + .collect(), + } + } + fn rebuild(&self) -> Result { + let bands = self + .sources + .iter() + .map(|s| Ok((s.source_id, band(&s.band)?))) + .collect::, String>>()?; + let p = AcquisitionPlan::build(self.context.clone(), plan_id(&self.plan_id)?, &bands)?; + if *self != Self::from_plan(&p) { + return Err("temporal_policy_binding_changed".into()); + } + Ok(p) + } +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct SourceEvidence { + source_id: u8, + source_spec_sha256: String, + times_ns: Vec, + frame_ids: Vec, + sequence_ids: Vec, +} +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct WindowEvidence { + cohort_id: String, + trial_id: String, + session_id: String, + role: DataRole, + label: RoomLabel, + cohort_sha256: String, + target_ns: String, + sealed_at_ns: String, + provenance: CaptureProvenance, + sources: Vec, +} +impl WindowEvidence { + fn from_assembled( + plan: &AcquisitionPlan, + w: &TemporalWindow, + sealed_at: u64, + target_ns: u64, + ) -> Result { + Self::from_binding(&Binding::from_plan(plan), w, sealed_at, target_ns) + } + fn from_binding( + binding: &Binding, + w: &TemporalWindow, + sealed_at: u64, + target_ns: u64, + ) -> Result { + if w.policy_sha256 != binding.plan_sha256 { + return Err("temporal_window_policy_changed".into()); + } + validate_cohort(&binding.context, &w.cohort)?; + if w.row_times_ns.len() != w.cohort.views.len() || w.cohort.teacher.is_some() { + return Err("temporal_window_evidence_changed".into()); + } + let mut sources = vec![]; + for ((spec, view), (source, times)) in binding + .context + .sources + .iter() + .zip(&w.cohort.views) + .zip(&w.row_times_ns) + { + view.validate(spec)?; + if *source != spec.source_id + || view.source_id != *source + || times.len() != view.samples.len() + || times.first() != Some(&view.first_time_ns) + || times.last() != Some(&view.last_time_ns) + || view.clock.kind != ClockKind::ReceiverOnly + || view.clock.uncertainty_ns != u64::MAX + || view.phase != PhaseAvailability::MagnitudeOnly + { + return Err("temporal_source_evidence_changed".into()); + } + sources.push(SourceEvidence { + source_id: *source, + source_spec_sha256: hash(spec)?, + times_ns: times.iter().map(ToString::to_string).collect(), + frame_ids: view.frame_ids.iter().map(ToString::to_string).collect(), + sequence_ids: view.sequence_ids.clone(), + }); + } + let e = Self { + cohort_id: w.cohort.id.clone(), + trial_id: w.cohort.trial_id.clone(), + session_id: w.cohort.session_id.clone(), + role: w.cohort.role, + label: w.cohort.label, + cohort_sha256: hash(&w.cohort)?, + target_ns: target_ns.to_string(), + sealed_at_ns: sealed_at.to_string(), + provenance: w.cohort.provenance, + sources, + }; + e.validate_sources(binding)?; + Ok(e) + } + fn validate_sources(&self, b: &Binding) -> Result<(), String> { + if !sha(&self.cohort_sha256) || self.sources.len() != b.sources.len() { + return Err("temporal_evidence_shape".into()); + } + let sealed = integer::(&self.sealed_at_ns)?; + let target = integer::(&self.target_ns)?; + if target == 0 { + return Err("temporal_target_missing".into()); + } + let mut first_min = u64::MAX; + let mut first_max = 0; + let mut last_min = u64::MAX; + let mut last_max = 0; + for ((s, p), spec) in self.sources.iter().zip(&b.sources).zip(&b.context.sources) { + let n = s.times_ns.len(); + if s.source_id != p.source_id + || s.source_id != spec.source_id + || s.source_spec_sha256 != hash(spec)? + || n < p.min_rows + || n > p.max_rows + || s.frame_ids.len() != n + || s.sequence_ids.len() != n + { + return Err("temporal_source_shape_or_policy".into()); + } + let times = s + .times_ns + .iter() + .map(|s| integer::(s)) + .collect::, _>>()?; + let frames = s + .frame_ids + .iter() + .map(|s| integer::(s)) + .collect::, _>>()?; + if times.contains(&0) + || frames.contains(&0) + || frames.iter().collect::>().len() != n + || s.sequence_ids.iter().any(|s| *s > u32::MAX as u64) + || s.sequence_ids.windows(2).any(|w| w[1] <= w[0]) + { + return Err("temporal_frame_or_sequence".into()); + } + for w in times.windows(2) { + let gap = w[1].checked_sub(w[0]).ok_or("temporal_time_reversed")?; + if gap < p.min_interval_ns || gap > p.max_gap_ns { + return Err("temporal_gap_or_cadence".into()); + } + } + // Check the EXACT identity codec already used by native::canonicalize_esp32. + // This links original receiver timestamps/counters to their recorded IDs; + // it does not infer a source clock or mint replacement frame identities. + for ((time, frame), sequence) in times.iter().zip(&frames).zip(&s.sequence_ids) { + let expected = (u128::from(*time) << 40) + | (u128::from(s.source_id) << 32) + | u128::from(*sequence); + if *frame != expected { + return Err("temporal_native_frame_identity_changed".into()); + } + } + let first = times[0]; + let last = times[n - 1]; + let span = last - first; + if first < target || first - target > b.max_first_delay_ms * 1_000_000 { + return Err("temporal_first_arrival_changed".into()); + } + if span < b.target_span_ms * 1_000_000 + || span > b.max_span_ms * 1_000_000 + || sealed < last + || sealed - first > MAX_AGE_NS + { + return Err("temporal_horizon_or_age".into()); + } + let (lo, hi) = band(&p.band)?.rate_bounds_millihz(); + let (lo, hi) = (u128::from(lo), u128::from(hi)); + let rate = (n - 1) as u128 * 1_000_000_000_000u128; + if rate < lo * span as u128 || rate > hi * span as u128 { + return Err("temporal_observed_cadence".into()); + } + first_min = first_min.min(first); + first_max = first_max.max(first); + last_min = last_min.min(last); + last_max = last_max.max(last); + } + if first_max - first_min > b.max_pairwise_skew_ms * 1_000_000 + || last_max - last_min > b.max_pairwise_skew_ms * 1_000_000 + || first_max >= last_min + { + return Err("temporal_source_alignment".into()); + } + Ok(()) + } +} +#[derive(Clone, Default)] +struct Fences { + frames: HashSet<(u8, u128)>, + last: HashMap, +} +impl Fences { + fn add(&mut self, e: &WindowEvidence) -> Result<(), String> { + for s in &e.sources { + for ((t, f), seq) in s.times_ns.iter().zip(&s.frame_ids).zip(&s.sequence_ids) { + let t = integer::(t)?; + let f = integer::(f)?; + if self.frames.contains(&(s.source_id, f)) + || self + .last + .get(&s.source_id) + .is_some_and(|(last_seq, last_t)| *seq <= *last_seq || t <= *last_t) + { + return Err("temporal_cross_window_replay".into()); + } + self.frames.insert((s.source_id, f)); + self.last.insert(s.source_id, (*seq, t)); + } + } + Ok(()) + } + fn check_packet(&self, p: &CanonicalPacket) -> Result<(), String> { + if self + .frames + .contains(&(p.source_id, p.native_frame.frame_id)) + || self.last.get(&p.source_id).is_some_and(|(seq, time)| { + u64::from(p.sequence) <= *seq || p.native_frame.timestamp_ns <= *time + }) + { + return Err("temporal_training_frame_or_time_overlap".into()); + } + Ok(()) + } +} +/// Raw cohorts are private and assembled here; there is intentionally no API to +/// fit a caller-supplied temporal-evidence flag or caller-created window. +pub struct TemporalTrainingSession { + plan: AcquisitionPlan, + binding: Binding, + run_id: String, + assembler: NativeWindowAssembler, + rows: Vec, + evidence: Vec, + config: FitConfig, + fitted: bool, + failed: bool, + active: bool, + target_ns: Option, + managed: Option, +} +fn preflight( + context: &CaptureContext, + plan: &AcquisitionPlan, + config: &FitConfig, +) -> Result { + if !(2..=8).contains(&context.sources.len()) { + return Err("temporal_model_supports_2_to_8_sources".into()); + } + let encoder = plan.encoder_config(); + if config.hidden_dim != encoder.hidden_dim + || config.embedding_dim != encoder.embedding_dim + || config.temporal_tokens != encoder.temporal_tokens + || config.temperature != encoder.temperature + { + return Err("temporal_encoder_policy_changed".into()); + } + config.validate_context(context) +} +impl TemporalTrainingSession { + pub fn new( + context: CaptureContext, + id: ExperimentalPlanId, + cadences: &[(u8, ReceiverCadenceBand)], + run_id: String, + config: FitConfig, + ) -> Result { + if !(2..=8).contains(&context.sources.len()) { + return Err("temporal_model_supports_2_to_8_sources".into()); + } + let plan = AcquisitionPlan::build(context, id, cadences)?; + preflight(plan.context(), &plan, &config)?; + let binding = Binding::from_plan(&plan); + let assembler = NativeWindowAssembler::new(plan.clone(), run_id.clone())?; + Ok(Self { + plan, + binding, + run_id, + assembler, + rows: vec![], + evidence: vec![], + config, + fitted: false, + failed: false, + active: false, + target_ns: None, + managed: None, + }) + } + pub fn begin_window(&mut self, target_ns: u64, end_ns: u64) -> Result<(), String> { + if self.managed.is_some() { + return Err("managed_window_target_required".into()); + } + if self.failed { + return Err("temporal_session_failed".into()); + } + let train_count = self.plan.resources().cohorts / 2; + if self.rows.len() >= train_count && !self.fitted { + return Err("temporal_fit_before_heldout".into()); + } + self.assembler.begin_window(target_ns, end_ns)?; + self.active = true; + self.target_ns = Some(target_ns); + Ok(()) + } + pub fn ingest(&mut self, p: &CanonicalPacket) -> Result { + if self.failed { + return Err("temporal_session_failed".into()); + } + let result = self.assembler.ingest(p); + if result.is_err() { + self.failed = true; + } + result + } + pub fn finish_window(&mut self, now_ns: u64) -> Result<(), String> { + if self.failed { + return Err("temporal_session_failed".into()); + } + // The assembler consumes/advances its window before adapter evidence is + // checked. Any subsequent failure must terminate this wrapper run; + // previous valid windows remain available for diagnostics, never credit + // an unvalidated window or let its cursor be silently reused. + let result = (|| { + let window = self.assembler.finish_window(now_ns)?; + let evidence = if let Some(managed) = &self.managed { + WindowEvidence::from_binding( + &managed.binding.validation_binding()?, + &window, + now_ns, + self.target_ns.ok_or("temporal_target_missing")?, + )? + } else { + WindowEvidence::from_assembled( + &self.plan, + &window, + now_ns, + self.target_ns.ok_or("temporal_target_missing")?, + )? + }; + let managed_identity = window.managed_identity.clone(); + if self.managed.is_some() && managed_identity.is_none() { + return Err("managed_window_identity_missing".into()); + } + Ok::<_, String>((window, evidence, managed_identity)) + })(); + self.active = false; + self.target_ns = None; + match result { + Ok((window, evidence, identity)) => { + if let Some(managed) = &mut self.managed { + managed + .windows + .push(identity.ok_or("managed_window_identity_missing")?); + } + self.rows.push(window.cohort); + self.evidence.push(evidence); + Ok(()) + } + Err(error) => { + self.failed = true; + Err(error) + } + } + } + pub fn fit(&mut self) -> Result<(TemporalModel, TrainingReport), String> { + if self.managed.is_some() { + return Err("managed_fit_admission_required".into()); + } + if self.failed + || self.active + || self.fitted + || self.rows.len() != self.plan.resources().cohorts / 2 + || self.rows.iter().any(|r| r.role != DataRole::Train) + { + return Err("temporal_complete_train_schedule_required".into()); + } + preflight(self.plan.context(), &self.plan, &self.config)?; + let dataset = ValidatedDataset::new(self.plan.context().clone(), self.rows.clone())?; + let (model, report) = RoomModel::fit(&dataset, self.config.clone())?; + if model.encoder_config() != self.plan.encoder_config() { + self.failed = true; + return Err("temporal_fitter_encoder_changed".into()); + } + let input = TemporalModel { + binding: self.binding.clone(), + run_id: self.run_id.clone(), + model, + evidence: self.evidence.clone(), + managed: None, + }; + if let Err(error) = input.validate() { + self.failed = true; + return Err(error); + } + self.fitted = true; + Ok((input, report)) + } + pub fn dataset(&self) -> Result { + ValidatedDataset::new(self.plan.context().clone(), self.rows.clone()) + } + pub fn plan_sha256(&self) -> &str { + &self.binding.plan_sha256 + } +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Payload { + binding: Binding, + run_id: String, + training_data_sha256: String, + input_evidence_sha256: String, + evidence: Vec, + model_sha256: String, + model_json: String, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Envelope { + schema: String, + payload_sha256: String, + payload: Payload, +} +/// Crate-private aggregate diagnostic identity. Not provenance or an inner model +/// accessor; checked against the WHOLE validated temporal codec on every use. +#[derive(Debug,Clone,Serialize,Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct TemporalFitBinding { + pub artifact_kind:String,pub artifact_sha256:String,pub context_digest:String, + pub training_data_sha256:String,pub input_evidence_sha256:String, + pub identity_evidence_sha256:Option,pub input_policy_sha256:String, + pub execution_policy_sha256:Option,pub schedule_sha256:Option, + pub source_ids:Vec,pub parameter_count:usize,pub training_cohorts:usize, + pub encoder:EncoderConfig,pub bound_fit_config:Option, +} +/// Existing learned network, with a new, separate versioned INPUT envelope. +/// Neither a checksum nor historical imported metadata is managed provenance. +#[derive(Clone)] +pub struct TemporalModel { + managed: Option, + binding: Binding, + run_id: String, + model: RoomModel, + evidence: Vec, +} +impl TemporalModel { + fn validate(&self) -> Result { + if self.managed.is_some() { + return Err("managed_v3_requires_policy_dispatch".into()); + } + let p = self.binding.rebuild()?; + self.model.validate()?; + if !(2..=8).contains(&p.context().sources.len()) + || self.model.context() != p.context() + || self.model.encoder_config() != p.encoder_config() + || self.evidence.len() != p.resources().cohorts / 2 + || self.model.training_manifest().training_cohorts != self.evidence.len() + || self.model.training_manifest().clock_claims != [ClockKind::ReceiverOnly] + || self.model.training_manifest().measurement_clock_available + { + return Err("temporal_model_input_binding".into()); + } + let _ = NativeWindowAssembler::new(p.clone(), self.run_id.clone())?; + let mut provenance = self + .evidence + .iter() + .map(|e| e.provenance) + .collect::>(); + provenance.sort_by_key(|p| match p { + CaptureProvenance::Synthetic => 0, + CaptureProvenance::CapturedReplay => 1, + CaptureProvenance::LiveReceiverObserved => 2, + }); + provenance.dedup(); + if provenance != self.model.training_manifest().training_provenance { + return Err("temporal_training_provenance_changed".into()); + } + let mut fences = Fences::default(); + for (index, e) in self.evidence.iter().enumerate() { + let step = &p.schedule()[index / p.slots_per_trial()]; + let slot = index % p.slots_per_trial(); + let trial = format!("{}-{}", self.run_id, step.step_id); + if e.role != DataRole::Train + || e.role != step.role + || e.label != step.label + || e.trial_id != trial + || e.session_id != trial + || e.cohort_id != format!("{trial}-{slot}") + { + return Err("temporal_training_role_or_identity".into()); + } + e.validate_sources(&self.binding)?; + fences.add(e)?; + } + Ok(p) + } + pub fn export(&self) -> Result, String> { + if self.managed.is_some() { + return self.export_managed(); + } + self.validate()?; + let model = self.model.export()?; + let payload = Payload { + binding: self.binding.clone(), + run_id: self.run_id.clone(), + training_data_sha256: self.model.training_manifest().training_data_sha256.clone(), + input_evidence_sha256: hash(&self.evidence)?, + evidence: self.evidence.clone(), + model_sha256: bytes_hash(&model), + model_json: String::from_utf8(model).map_err(|_| "temporal_model_encoding")?, + }; + let envelope = Envelope { + schema: SCHEMA.into(), + payload_sha256: hash(&payload)?, + payload, + }; + let bytes = serde_json::to_vec(&envelope).map_err(|_| "temporal_encoding")?; + if bytes.len() > MAX_BYTES { + return Err("temporal_input_envelope_budget".into()); + } + Ok(bytes) + } + pub fn import(bytes: &[u8]) -> Result { + if bytes.len() > MAX_BYTES { + return Err("temporal_input_envelope_budget".into()); + } + let version: serde_json::Value = + serde_json::from_slice(bytes).map_err(|_| "temporal_input_envelope_schema")?; + if version["schema"] == managed::SCHEMA_V3 { + return Self::import_managed(bytes); + } + let envelope: Envelope = + serde_json::from_slice(bytes).map_err(|_| "temporal_input_envelope_schema")?; + if envelope.schema != SCHEMA + || envelope.payload_sha256 != hash(&envelope.payload)? + || serde_json::to_vec(&envelope).map_err(|_| "temporal_encoding")? != bytes + { + return Err("temporal_input_envelope_integrity".into()); + } + let payload = envelope.payload; + if payload.model_sha256 != bytes_hash(payload.model_json.as_bytes()) + || payload.input_evidence_sha256 != hash(&payload.evidence)? + { + return Err("temporal_model_or_evidence_hash".into()); + } + let model = RoomModel::import(payload.model_json.as_bytes())?; + if model.export()? != payload.model_json.as_bytes() + || model.training_manifest().training_data_sha256 != payload.training_data_sha256 + { + return Err("temporal_model_training_binding".into()); + } + let result = Self { + managed: None, + binding: payload.binding, + run_id: payload.run_id, + model, + evidence: payload.evidence, + }; + result.validate()?; + Ok(result) + } + pub fn live_input(&self, run_id: String) -> Result { + if self.managed.is_some() { + return self.managed_live_input(run_id); + } + let p = self.validate()?; + let mut fences = Fences::default(); + for e in &self.evidence { + fences.add(e)?; + } + Ok(TemporalLiveInput { + binding: self.binding.clone(), + assembler: NativeWindowAssembler::new(p, run_id)?, + fences, + failed: false, + target_ns: None, + }) + } + pub fn predict( + &self, + input: TemporalLiveObservation, + now_ns: u64, + ) -> Result { + if self.managed.is_some() { + return self.predict_managed(input, now_ns); + } + let plan = self.validate()?; + if input.window.cohort.role != DataRole::Inference + || input.window.cohort.label != RoomLabel::Unknown + || input.window.cohort.teacher.is_some() + { + return Err("temporal_live_must_be_unlabelled".into()); + } + let evidence = + WindowEvidence::from_assembled(&plan, &input.window, now_ns, input.target_ns)?; + let mut fences = Fences::default(); + for e in &self.evidence { + fences.add(e)?; + } + fences.add(&evidence)?; + if input.binding != self.binding || input.window.policy_sha256 != self.binding.plan_sha256 { + return Err("temporal_live_policy_changed".into()); + } + let cohort = input.window.cohort; + validate_cohort(&self.binding.context, &cohort)?; + for v in &cohort.views { + if now_ns < v.last_time_ns + || now_ns + .checked_sub(v.first_time_ns) + .filter(|a| *a <= MAX_AGE_NS) + .is_none() + { + return Err("temporal_live_input_stale".into()); + } + } + Ok(self.model.infer(&cohort)) + } + /// Read-only immutable input policy for a diagnostic window driver. + /// Does not expose the inner v1 network or issue managed provenance. + pub fn input_policy(&self) -> Result { + self.validate() + } + pub fn window_input_policy( + &self, + ) -> Result { + if let Some(managed) = &self.managed { + self.validate_managed()?; + Ok(managed.binding.native.clone()) + } else { + Ok(super::temporal_acquisition::WindowInputPolicy::from_seed( + &self.validate()?, + )) + } + } + pub fn artifact_kind(&self) -> &'static str { + if self.managed.is_some() { + managed::SCHEMA_V3 + } else { + SCHEMA + } + } + /// Read-only historical context for owner-scoped diagnostic persistence. + /// This does not expose the inner model or establish managed provenance. + pub fn context(&self) -> &CaptureContext { + &self.binding.context + } + pub(crate) fn fit_diagnostic_binding(&self)->Result{ + let bytes=self.export()?; + let (schedule,execution,identities,configured)=if self.managed.is_some(){ + let (s,e,i,c)=self.managed_fit_diagnostic_policy()?;(Some(s),Some(e),Some(i),Some(c)) + }else{(None,None,None,None)}; + Ok(TemporalFitBinding{artifact_kind:self.artifact_kind().into(),artifact_sha256:bytes_hash(&bytes),context_digest:self.context().digest(),training_data_sha256:self.model.training_manifest().training_data_sha256.clone(),input_evidence_sha256:hash(&self.evidence)?,identity_evidence_sha256:identities,input_policy_sha256:self.plan_sha256().into(),execution_policy_sha256:execution,schedule_sha256:schedule,source_ids:self.context().sources.iter().map(|s|s.source_id).collect(),parameter_count:self.parameter_count(),training_cohorts:self.model.training_manifest().training_cohorts,encoder:self.model.encoder_config().clone(),bound_fit_config:configured}) + } + pub fn parameter_count(&self) -> usize { + self.model.parameter_count() + } + pub fn plan_sha256(&self) -> &str { + &self.binding.plan_sha256 + } +} +/// Bounded one-window inference input seed. Runtime warmup/continuous replay +/// custody remains a future integration; this does not replace a live service. +pub struct TemporalLiveInput { + binding: Binding, + assembler: NativeWindowAssembler, + fences: Fences, + failed: bool, + target_ns: Option, +} +pub struct TemporalLiveObservation { + binding: Binding, + window: TemporalWindow, + target_ns: u64, +} +impl TemporalLiveInput { + pub fn begin_window(&mut self, target_ns: u64, end_ns: u64) -> Result<(), String> { + if self.failed { + return Err("temporal_live_failed".into()); + } + self.assembler.begin_window(target_ns, end_ns)?; + self.target_ns = Some(target_ns); + Ok(()) + } + pub fn ingest(&mut self, p: &CanonicalPacket) -> Result { + if self.failed { + return Err("temporal_live_failed".into()); + } + if let Err(e) = self.fences.check_packet(p) { + self.failed = true; + return Err(e); + } + let result = self.assembler.ingest(p); + if result.is_err() { + self.failed = true; + } + result + } + pub fn finish(self, now_ns: u64) -> Result { + if self.failed { + return Err("temporal_live_failed".into()); + } + let mut assembler = self.assembler; + let mut window = assembler.finish_window(now_ns)?; + // Fresh inference observations carry no operator schedule label. This + // never relabels retained Train/HeldOut rows or creates teacher truth. + window.cohort.role = DataRole::Inference; + window.cohort.label = RoomLabel::Unknown; + window.cohort.teacher = None; + WindowEvidence::from_binding( + &self.binding, + &window, + now_ns, + self.target_ns.ok_or("temporal_target_missing")?, + )?; + Ok(TemporalLiveObservation { + binding: self.binding, + window, + target_ns: self.target_ns.ok_or("temporal_target_missing")?, + }) + } +} +#[cfg(test)] +#[path = "temporal_model_input_tests.rs"] +mod tests; + +#[path = "temporal_model_input_managed.rs"] +pub(crate) mod managed; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs new file mode 100644 index 0000000000..3584a9626e --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs @@ -0,0 +1,564 @@ +//! Managed-v3 INPUT integration inside the SAME opaque temporal model/session. +//! No Runtime issuer, full-trial quality, confirmation or managed proof exists here. +use super::*; +use crate::room_learner::{ + managed_schedule_contract::{FrozenModelIdentity, ManagedScheduleContract}, + temporal_acquisition::{ManagedWindowIdentity, ManagedWindowTarget, WindowInputPolicy}, +}; +use std::sync::atomic::AtomicBool; +#[path = "temporal_execution.rs"] +pub(crate) mod execution; +pub(crate) const SCHEMA_V3: &str = "ruview.room-learner.managed-temporal-input.v3"; +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct LivePolicy { + id: String, + target_horizon_ms: u64, + maximum_first_row_age_ns: u64, + source_gap_policy: String, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct ManagedBinding { + schedule_json: String, + schedule_sha256: String, + pub(super) native: WindowInputPolicy, + encoder: EncoderConfig, + fit: FitConfig, + live: LivePolicy, + target_mapping: String, +} +impl ManagedBinding { + fn build(schedule: &ManagedScheduleContract, fit: FitConfig) -> Result { + fit.validate_context(schedule.context())?; + let encoder = EncoderConfig { + bins: 56, + source_count: schedule.context().sources.len(), + hidden_dim: fit.hidden_dim, + embedding_dim: fit.embedding_dim, + temporal_tokens: fit.temporal_tokens, + temperature: fit.temperature, + }; + Ok(Self { + schedule_json: String::from_utf8(schedule.export()?) + .map_err(|_| "managed_binding_encoding")?, + schedule_sha256: schedule.sha256().into(), + native: WindowInputPolicy::from_managed(schedule), + encoder, + fit, + live: LivePolicy { + id: "nonoverlap-native-oldest-first-v1".into(), + target_horizon_ms: 2000, + maximum_first_row_age_ns: MAX_AGE_NS, + source_gap_policy: "existing-native-profile-every-source-max-gap".into(), + }, + target_mapping: execution::EXECUTION_POLICY.into(), + }) + } + fn contract(&self) -> Result { + let schedule = ManagedScheduleContract::import(self.schedule_json.as_bytes())?; + let expected = Self::build(&schedule, self.fit.clone())?; + if serde_json::to_vec(self).map_err(|_| "managed_binding_encoding")? + != serde_json::to_vec(&expected).map_err(|_| "managed_binding_encoding")? + { + return Err("managed_binding_policy_changed".into()); + } + Ok(schedule) + } + pub(super) fn validation_binding(&self) -> Result { + self.contract()?; + Ok(Binding { + plan_id: "managed-v3-native-validation-only".into(), + native_frame_identity: "existing-esp32-receiver-ns-node-u32-sequence-v1".into(), + plan_sha256: hash(self)?, + context: self.native.context().clone(), + encoder: self.encoder.clone(), + target_span_ms: self.native.target_span_ms(), + max_span_ms: self.native.max_observed_span_ms(), + max_pairwise_skew_ms: self.native.max_pairwise_receiver_skew_ms(), + max_first_delay_ms: self.native.max_first_packet_delay_ms(), + sources: self + .native + .source_policies() + .iter() + .map(|p| SourcePolicy { + source_id: p.source_id, + band: format!("{:?}", p.receiver_cadence), + min_rows: p.min_rows, + max_rows: p.max_rows, + min_interval_ns: p.min_packet_interval_ns, + max_gap_ns: p.max_gap_ns, + }) + .collect(), + }) + } +} +pub(super) struct ManagedSession { + pub(super) binding: ManagedBinding, + pub(super) windows: Vec, + frozen_sha256: Option, +} +#[derive(Clone)] +pub(super) struct ManagedModel { + pub(super) binding: ManagedBinding, + windows: Vec, +} +/// No production issuer is implemented. A future owner Runtime must issue this +/// after its actual confirmed/quality-checked/durable Train episode evidence. +/// It cannot be uploaded/deserialized or synthesized from generic model metadata. +pub(crate) struct ManagedFitAdmission { + policy: String, + data: String, + evidence: String, + identities: String, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Claims { + diagnostic_only: bool, + managed_proof_issued: bool, + whole_episode_quality_issued: bool, + conditions_confirmed_by_runtime: bool, +} +impl Claims { + fn diagnostic() -> Self { + Self { + diagnostic_only: true, + managed_proof_issued: false, + whole_episode_quality_issued: false, + conditions_confirmed_by_runtime: false, + } + } +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PayloadV3 { + binding: ManagedBinding, + run_id: String, + training_data_sha256: String, + input_evidence_sha256: String, + identity_evidence_sha256: String, + evidence: Vec, + identities: Vec, + model_sha256: String, + model_json: String, + claims: Claims, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct EnvelopeV3 { + schema: String, + payload_sha256: String, + payload: PayloadV3, +} +fn identity( + schedule: &ManagedScheduleContract, + run: &str, + record: &ManagedWindowIdentity, +) -> Result { + let frozen = record + .frozen_artifact_sha256 + .as_ref() + .map(|sha| FrozenModelIdentity::from_runtime_verified_artifact(schedule, sha.clone())) + .transpose()?; + let bound = schedule.bind_episode_start( + run, + record.episode_index, + record.attempt_ordinal, + record.segment_ordinal, + frozen.as_ref(), + )?; + if record.collection_started_ns != record.mapping.receiver_start_ns.to_string() { + return Err("managed_segment_start_changed".into()); + } + let expected = ManagedWindowTarget::from_observed_episode( + schedule, + &bound, + run, + record.slot_index, + record.mapping.clone(), + )?; + if serde_json::to_vec(expected.identity()).map_err(|_| "managed_identity_encoding")? + != serde_json::to_vec(record).map_err(|_| "managed_identity_encoding")? + { + return Err("managed_identity_changed".into()); + } + Ok(expected) +} +fn expected_position( + schedule: &ManagedScheduleContract, + index: usize, +) -> Result<(usize, usize), String> { + let mut cursor = 0; + for episode in schedule.episodes() { + if index < cursor + episode.slots.len() { + return Ok((episode.index, index - cursor)); + } + cursor += episode.slots.len(); + } + Err("managed_window_schedule_exhausted".into()) +} +fn validate_records( + schedule: &ManagedScheduleContract, + run: &str, + evidence: &[WindowEvidence], + records: &[ManagedWindowIdentity], + binding: &Binding, + train_only: bool, +) -> Result<(), String> { + if evidence.len() != records.len() { + return Err("managed_identity_count_changed".into()); + } + let mut fences = Fences::default(); + let mut previous: Option<&ManagedWindowIdentity> = None; + for (index, (e, r)) in evidence.iter().zip(records).enumerate() { + let position = expected_position(schedule, index)?; + if (r.episode_index, r.slot_index) != position + || (train_only && (r.role != DataRole::Train || r.frozen_artifact_sha256.is_some())) + { + return Err("managed_window_role_or_order".into()); + } + let target = identity(schedule, run, r)?; + if e.role != r.role + || e.label != r.label + || e.trial_id != r.trial_id + || e.session_id != r.session_id + || e.target_ns != r.target_ns + { + return Err("managed_cohort_identity_changed".into()); + } + let expected_id = format!( + "window-{:x}", + Sha256::digest(serde_json::to_vec(r).map_err(|_| "managed_identity_encoding")?) + ); + if e.cohort_id != expected_id || integer::(&e.sealed_at_ns)? >= target.end_ns() { + return Err("managed_window_mapping_changed".into()); + } + if let Some(old) = previous { + if old.episode_index == r.episode_index { + if old.attempt_id != r.attempt_id { + return Err("managed_attempt_changed".into()); + } + if old.segment_ordinal == r.segment_ordinal { + if old.mapping != r.mapping || old.session_id != r.session_id { + return Err("managed_segment_changed".into()); + } + } else { + let last = schedule.episodes()[old.episode_index].slots[old.slot_index] + .deadline_offset_ms + * 1_000_000; + if r.segment_ordinal != old.segment_ordinal + 1 + || r.mapping.observed_start_ns != last + || r.mapping.receiver_start_ns < integer::(&old.end_ns)? + || r.mapping.clock_anchor_ns < integer::(&old.end_ns)? + { + return Err("managed_segment_overlap_or_clock".into()); + } + } + } else if r.mapping.observed_start_ns != 0 + || r.mapping.receiver_start_ns < integer::(&old.end_ns)? + { + return Err("managed_episode_time_overlap".into()); + } + } + e.validate_sources(binding)?; + fences.add(e)?; + previous = Some(r); + } + Ok(()) +} +impl TemporalTrainingSession { + pub fn managed( + schedule: ManagedScheduleContract, + run_id: String, + config: FitConfig, + ) -> Result { + let binding = ManagedBinding::build(&schedule, config.clone())?; + let policy = hash(&binding)?; + // Only the unchanged legacy fields' native-validation carrier is built; + // managed identity/hash/schedule NEVER comes from this seed. + let bands = schedule + .context() + .sources + .iter() + .map(|s| (s.source_id, ReceiverCadenceBand::Hz30To36)) + .collect::>(); + let plan = AcquisitionPlan::build( + schedule.context().clone(), + ExperimentalPlanId::Native2000msSlots8, + &bands, + )?; + let assembler = NativeWindowAssembler::managed(schedule, policy, run_id.clone())?; + Ok(Self { + plan, + binding: binding.validation_binding()?, + run_id, + assembler, + rows: vec![], + evidence: vec![], + config, + fitted: false, + failed: false, + active: false, + target_ns: None, + managed: Some(ManagedSession { + binding, + windows: vec![], + frozen_sha256: None, + }), + }) + } + pub fn begin_managed_window(&mut self, target: &ManagedWindowTarget) -> Result<(), String> { + let managed = self.managed.as_ref().ok_or("managed_session_required")?; + let r = target.identity(); + if self.failed || self.active { + return Err("temporal_session_failed_or_active".into()); + } + if r.role == DataRole::HeldOut + && (!self.fitted || r.frozen_artifact_sha256.as_ref() != managed.frozen_sha256.as_ref()) + { + return Err("managed_frozen_whole_model_required".into()); + } + if r.role == DataRole::Train && self.fitted { + return Err("managed_training_is_frozen".into()); + } + self.assembler.begin_managed_window(target)?; + self.active = true; + self.target_ns = Some(target.target_ns()); + Ok(()) + } + pub(crate) fn fit_managed_with_cancel( + &mut self, + permit: &ManagedFitAdmission, + cancel: &AtomicBool, + ) -> Result<(TemporalModel, TrainingReport), String> { + let managed = self.managed.as_ref().ok_or("managed_session_required")?; + let schedule = managed.binding.contract()?; + if self.failed + || self.active + || self.fitted + || self.rows.len() != schedule.counts().train_windows + || self.rows.iter().any(|r| r.role != DataRole::Train) + { + return Err("managed_complete_train_schedule_required".into()); + } + validate_records( + &schedule, + &self.run_id, + &self.evidence, + &managed.windows, + &self.binding, + true, + )?; + let dataset = ValidatedDataset::new(schedule.context().clone(), self.rows.clone())?; + if permit.policy != hash(&managed.binding)? + || permit.data != hash(&dataset.train())? + || permit.evidence != hash(&self.evidence)? + || permit.identities != hash(&managed.windows)? + { + return Err("managed_fit_admission_mismatch".into()); + } + let (model, report) = RoomModel::fit_with_cancel(&dataset, self.config.clone(), cancel)?; + if model.encoder_config() != &managed.binding.encoder { + return Err("managed_encoder_changed".into()); + } + let result = TemporalModel { + binding: self.binding.clone(), + run_id: self.run_id.clone(), + model, + evidence: self.evidence.clone(), + managed: Some(ManagedModel { + binding: managed.binding.clone(), + windows: managed.windows.clone(), + }), + }; + result.validate_managed()?; + let frozen = bytes_hash(&result.export()?); + self.managed.as_mut().unwrap().frozen_sha256 = Some(frozen); + self.fitted = true; + Ok((result, report)) + } + #[cfg(test)] + fn synthetic_test_fit_admission(&self) -> Result { + let managed = self.managed.as_ref().ok_or("managed_session_required")?; + let dataset = self.dataset()?; + Ok(ManagedFitAdmission { + policy: hash(&managed.binding)?, + data: hash(&dataset.train())?, + evidence: hash(&self.evidence)?, + identities: hash(&managed.windows)?, + }) + } +} +impl TemporalModel { + pub(super) fn validate_managed(&self) -> Result { + let m = self.managed.as_ref().ok_or("managed_model_required")?; + let schedule = m.binding.contract()?; + self.model.validate()?; + if self.model.context() != schedule.context() + || self.model.encoder_config() != &m.binding.encoder + || self.model.training_manifest().training_cohorts != schedule.counts().train_windows + || self.evidence.len() != schedule.counts().train_windows + || self.model.training_manifest().measurement_clock_available + || self.model.training_manifest().clock_claims != [ClockKind::ReceiverOnly] + { + return Err("managed_model_binding_changed".into()); + } + let expected = m.binding.validation_binding()?; + if serde_json::to_vec(&self.binding).map_err(|_| "managed_binding_encoding")? + != serde_json::to_vec(&expected).map_err(|_| "managed_binding_encoding")? + { + return Err("managed_native_binding_changed".into()); + } + let mut provenance = self + .evidence + .iter() + .map(|e| e.provenance) + .collect::>(); + provenance.sort_by_key(|p| match p { + CaptureProvenance::Synthetic => 0, + CaptureProvenance::CapturedReplay => 1, + CaptureProvenance::LiveReceiverObserved => 2, + }); + provenance.dedup(); + if provenance != self.model.training_manifest().training_provenance { + return Err("managed_provenance_changed".into()); + } + validate_records( + &schedule, + &self.run_id, + &self.evidence, + &m.windows, + &self.binding, + true, + )?; + Ok(schedule) + } + pub(super) fn managed_fit_diagnostic_policy(&self)->Result<(String,String,String,FitConfig),String>{ + self.validate_managed()?; + let model=self.managed.as_ref().ok_or("managed_model_required")?; + Ok((model.binding.schedule_sha256.clone(),hash(&model.binding.target_mapping)?,hash(&model.windows)?,model.binding.fit.clone())) + } + pub(super) fn export_managed(&self) -> Result, String> { + self.validate_managed()?; + let managed = self.managed.as_ref().unwrap(); + let bytes = self.model.export()?; + let payload = PayloadV3 { + binding: managed.binding.clone(), + run_id: self.run_id.clone(), + training_data_sha256: self.model.training_manifest().training_data_sha256.clone(), + input_evidence_sha256: hash(&self.evidence)?, + identity_evidence_sha256: hash(&managed.windows)?, + evidence: self.evidence.clone(), + identities: managed.windows.clone(), + model_sha256: bytes_hash(&bytes), + model_json: String::from_utf8(bytes).map_err(|_| "managed_model_encoding")?, + claims: Claims::diagnostic(), + }; + let e = EnvelopeV3 { + schema: SCHEMA_V3.into(), + payload_sha256: hash(&payload)?, + payload, + }; + let bytes = serde_json::to_vec(&e).map_err(|_| "managed_model_encoding")?; + if bytes.len() > MAX_BYTES { + return Err("temporal_input_envelope_budget".into()); + } + Ok(bytes) + } + pub(super) fn import_managed(bytes: &[u8]) -> Result { + let e: EnvelopeV3 = serde_json::from_slice(bytes).map_err(|_| "managed_model_schema")?; + if e.schema != SCHEMA_V3 + || e.payload_sha256 != hash(&e.payload)? + || serde_json::to_vec(&e).map_err(|_| "managed_model_encoding")? != bytes + { + return Err("managed_model_integrity".into()); + } + let p = e.payload; + if !p.claims.diagnostic_only + || p.claims.managed_proof_issued + || p.claims.whole_episode_quality_issued + || p.claims.conditions_confirmed_by_runtime + { + return Err("managed_input_is_not_proof".into()); + } + if p.input_evidence_sha256 != hash(&p.evidence)? + || p.identity_evidence_sha256 != hash(&p.identities)? + || p.model_sha256 != bytes_hash(p.model_json.as_bytes()) + { + return Err("managed_evidence_hash_changed".into()); + } + let model = RoomModel::import(p.model_json.as_bytes())?; + if model.export()? != p.model_json.as_bytes() + || model.training_manifest().training_data_sha256 != p.training_data_sha256 + { + return Err("managed_training_hash_changed".into()); + } + let binding = p.binding.validation_binding()?; + let result = Self { + binding, + run_id: p.run_id, + model, + evidence: p.evidence, + managed: Some(ManagedModel { + binding: p.binding, + windows: p.identities, + }), + }; + result.validate_managed()?; + Ok(result) + } + pub(super) fn managed_live_input(&self, run_id: String) -> Result { + self.validate_managed()?; + let m = self.managed.as_ref().unwrap(); + let mut fences = Fences::default(); + for e in &self.evidence { + fences.add(e)?; + } + Ok(TemporalLiveInput { + binding: self.binding.clone(), + assembler: NativeWindowAssembler::inference( + m.binding.native.clone(), + hash(&m.binding)?, + run_id, + )?, + fences, + failed: false, + target_ns: None, + }) + } + pub(super) fn predict_managed( + &self, + input: TemporalLiveObservation, + now_ns: u64, + ) -> Result { + self.validate_managed()?; + if input.window.cohort.role != DataRole::Inference + || input.window.cohort.label != RoomLabel::Unknown + || input.window.cohort.teacher.is_some() + || input.window.managed_identity.is_some() + { + return Err("temporal_live_must_be_unlabelled".into()); + } + let e = + WindowEvidence::from_binding(&self.binding, &input.window, now_ns, input.target_ns)?; + let mut fences = Fences::default(); + for old in &self.evidence { + fences.add(old)?; + } + fences.add(&e)?; + if serde_json::to_vec(&input.binding).map_err(|_| "managed_binding_encoding")? + != serde_json::to_vec(&self.binding).map_err(|_| "managed_binding_encoding")? + { + return Err("temporal_live_policy_changed".into()); + } + Ok(self.model.infer(&input.window.cohort)) + } +} +#[cfg(test)] +#[path = "temporal_model_input_managed_tests.rs"] +mod tests; + +#[cfg(test)] +#[path="temporal_fit_summary_managed_tests.rs"] +mod fit_summary_tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed_tests.rs new file mode 100644 index 0000000000..d2fe02a8e3 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed_tests.rs @@ -0,0 +1,485 @@ +//! SYNTHETIC source integration only: no whole-trial quality/confirmation proof. +use super::*; +use crate::room_learner::{ + artifact::ArtifactStore, + managed_schedule_contract::{LeadInPolicy, ManagedScheduleContract}, + native::{canonicalize_esp32, NativePacketInput}, + temporal_live_driver::{ContinuousTemporalInput, ObservationAvailability}, +}; +use std::sync::OnceLock; +use wifi_densepose_hardware::{Adr018Flags, PpduType}; +pub(super) fn context() -> CaptureContext { + CaptureContext { + owner_scope: "a".repeat(64), + installation_id: "synthetic-managed-v3".into(), + room_digest: "synthetic-room".into(), + revision: 1, + sources: (1..=3) + .map(|source_id| SourceSpec { + source_id, + device_id: format!("esp32-node-{source_id}"), + grid_id: "ant1-bins64-ppdu0-usable-drop2-linear-v2".into(), + feature_count: 56, + signal_kind: ruview_unified::RfModality::WifiCsi, + center_frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + link_id: format!("native-{source_id}"), + link_index: 0, + geometry: None, + }) + .collect(), + } +} +fn schedule() -> ManagedScheduleContract { + ManagedScheduleContract::build(context(), LeadInPolicy::Train5HeldOut15EngineeringV1).unwrap() +} +pub(super) fn packet( + source: u8, + seq: u32, + at: u64, + label: RoomLabel, + row: usize, +) -> CanonicalPacket { + let gain = match label { + RoomLabel::Empty => 3, + RoomLabel::AdjacentOnly => 4, + RoomLabel::OccupiedStatic => 8, + RoomLabel::OccupiedMoving => 10 + (row % 2) as u8, + RoomLabel::Unknown => 8, + }; + let iq = (0..64) + .flat_map(|bin| [gain + source + (bin % 2) as u8, 4]) + .collect::>(); + canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence: seq, + received_at_ns: at, + receiver_sample_rate_hz: 33., + iq_bytes: &iq, + flags: Adr018Flags::default(), + provenance: CaptureProvenance::Synthetic, + }) + .unwrap() +} +fn collect( + session: &mut TemporalTrainingSession, + target: &ManagedWindowTarget, + index: u32, + label: RoomLabel, +) { + session.begin_managed_window(target).unwrap(); + for row in 0..100usize { + for source in 1..=3u8 { + let at = target.target_ns() + + row as u64 * 1_000_000_000u64.div_ceil(33) + + u64::from(source) * 1000; + if session + .ingest(&packet( + source, + index * 1000 + row as u32 + 1, + at, + label, + row, + )) + .unwrap() + == WindowAdmission::Ready + { + session.finish_window(at).unwrap(); + return; + } + } + } + panic!("missing actual2s native window"); +} +fn train() -> (TemporalTrainingSession, TemporalModel) { + let plan = schedule(); + let mut session = TemporalTrainingSession::managed( + plan.clone(), + "synthetic-v3-run".into(), + FitConfig { + epochs: 2, + ..Default::default() + }, + ) + .unwrap(); + assert!(session.fit().is_err()); + let mut start = 10_000_000_000u64; + let mut index = 0; + for episode in plan.episodes().iter().filter(|e| e.role == DataRole::Train) { + let bound = plan + .bind_episode_start("synthetic-v3-run", episode.index, 0, 0, None) + .unwrap(); + for slot in 0..episode.slots.len() { + let target = ManagedWindowTarget::from_runtime_episode( + &plan, + &bound, + "synthetic-v3-run", + slot, + start, + ) + .unwrap(); + collect(&mut session, &target, index, episode.condition); + index += 1; + } + start += episode.target_active_recording_ms * 1_000_000 + 10_000_000_000; + } + assert_eq!(session.dataset().unwrap().train().len(), 64); + assert_eq!( + session + .dataset() + .unwrap() + .train() + .iter() + .map(|row| row.trial_id.as_str()) + .collect::>() + .len(), + 7 + ); + let permit = session.synthetic_test_fit_admission().unwrap(); + let (model, report) = session + .fit_managed_with_cancel(&permit, &AtomicBool::new(false)) + .unwrap(); + assert_eq!(report.training_cohorts, 64); + assert_eq!(report.completed_epochs, 2); + assert!(report.backbone_delta_l2 > 0.); + (session, model) +} +fn model() -> TemporalModel { + static BYTES: OnceLock> = OnceLock::new(); + TemporalModel::import(BYTES.get_or_init(|| train().1.export().unwrap())).unwrap() +} +fn live(model: TemporalModel) -> serde_json::Value { + let mut driver = ContinuousTemporalInput::new(model).unwrap(); + driver.activate(); + let target = 10_000_000_000_000; + for row in 0..100usize { + for source in 1..=3u8 { + let at = target + row as u64 * 1_000_000_000u64.div_ceil(33) + u64::from(source) * 1000; + let d = driver.ingest( + &packet(source, 1_000_000 + row as u32, at, RoomLabel::Unknown, row), + at, + ); + if d.observation.availability == ObservationAvailability::Available { + return serde_json::to_value(d.current_prediction).unwrap(); + } + } + } + panic!("continuous v3 input did not complete") +} +fn seed_v2_same_context() -> TemporalModel { + let mut session = TemporalTrainingSession::new( + context(), + ExperimentalPlanId::Native2000msSlots2, + &(1..=3) + .map(|s| (s, ReceiverCadenceBand::Hz30To36)) + .collect::>(), + "synthetic-seed-compatible".into(), + FitConfig { + epochs: 2, + ..Default::default() + }, + ) + .unwrap(); + for index in 0..16u32 { + let target = 10_000_000_000 + u64::from(index) * 3_000_000_000; + let label = + session.plan.schedule()[session.rows.len() / session.plan.slots_per_trial()].label; + session + .begin_window(target, target + 3_000_000_000) + .unwrap(); + 'window: for row in 0..100usize { + for source in 1..=3u8 { + let at = + target + row as u64 * 1_000_000_000u64.div_ceil(33) + u64::from(source) * 1000; + if session + .ingest(&packet( + source, + index * 1000 + row as u32 + 1, + at, + label, + row, + )) + .unwrap() + == WindowAdmission::Ready + { + session.finish_window(at).unwrap(); + break 'window; + } + } + } + } + session.fit().unwrap().0 +} +#[test] +fn actual_native_managed_schedule_same_fitter_whole_v3_store_cold_and_same_driver() { + let m = model(); + let bytes = m.export().unwrap(); + assert_eq!(m.artifact_kind(), SCHEMA_V3); + assert!( + m.input_policy().is_err(), + "v3 must not expose seed-v2 plan/hash" + ); + let value: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!(value["schema"], SCHEMA_V3); + assert_eq!(value["payload"]["evidence"].as_array().unwrap().len(), 64); + assert_eq!(value["payload"]["identities"].as_array().unwrap().len(), 64); + assert_eq!( + value["payload"]["claims"]["whole_episode_quality_issued"], + false + ); + assert_eq!( + value["payload"]["claims"]["conditions_confirmed_by_runtime"], + false + ); + let root = tempfile::tempdir().unwrap(); + let c = context(); + let path = root.path().join(&c.owner_scope).join("SensingServer"); + let store = ArtifactStore::new(&path, &c.owner_scope, &c.installation_id).unwrap(); + let seed = seed_v2_same_context(); + let seed_bytes = seed.export().unwrap(); + let seed_receipt = store.save_temporal_candidate(&seed).unwrap(); + let directory = path + .join("room-learner") + .join("temporal-models") + .join(&seed_receipt.context_digest); + let pointer = directory.join("candidate-temporal-v2.json"); + let before_pointer = std::fs::read(&pointer).unwrap(); + let receipt = store.save_temporal_candidate(&m).unwrap(); + assert_eq!(receipt.artifact_kind, SCHEMA_V3); + assert_eq!(receipt.artifact_sha256, bytes_hash(&bytes)); + assert!(receipt.diagnostic_only && !receipt.qualification_granted); + assert_eq!( + std::fs::read(&pointer).unwrap(), + before_pointer, + "V3 may not rewrite V2 pointer" + ); + assert_eq!( + store + .load_temporal_candidate(&c) + .unwrap() + .unwrap() + .export() + .unwrap(), + seed_bytes + ); + assert_eq!( + store + .load_exact_temporal_candidate(&c, &seed_receipt.artifact_sha256) + .unwrap() + .export() + .unwrap(), + seed_bytes + ); + let cold = ArtifactStore::new(&path, &c.owner_scope, &c.installation_id) + .unwrap() + .load_managed_temporal_candidate(&c) + .unwrap() + .unwrap(); + assert_eq!(cold.export().unwrap(), bytes); + assert_eq!(live(m), live(cold)); +} +#[test] +fn heldout_uses_frozen_whole_artifact_and_never_changes_train_codec_or_profiles() { + let (mut session, m) = train(); + let before = m.export().unwrap(); + let frozen = bytes_hash(&before); + let plan = schedule(); + let identity = + FrozenModelIdentity::from_runtime_verified_artifact(&plan, frozen.clone()).unwrap(); + let mut start = 2_000_000_000_000; + let mut index = 64; + for episode in plan + .episodes() + .iter() + .filter(|e| e.role == DataRole::HeldOut) + { + let bound = plan + .bind_episode_start("synthetic-v3-run", episode.index, 0, 0, Some(&identity)) + .unwrap(); + for slot in 0..episode.slots.len() { + let target = ManagedWindowTarget::from_runtime_episode( + &plan, + &bound, + "synthetic-v3-run", + slot, + start, + ) + .unwrap(); + collect(&mut session, &target, index, episode.condition); + index += 1; + } + start += episode.target_active_recording_ms * 1_000_000 + 20_000_000_000; + } + assert_eq!(session.dataset().unwrap().held_out().len(), 64); + assert_eq!(m.export().unwrap(), before); + let permit = session.synthetic_test_fit_admission().unwrap(); + assert!(session + .fit_managed_with_cancel(&permit, &AtomicBool::new(false)) + .is_err()); + let v: serde_json::Value = serde_json::from_slice(&before).unwrap(); + assert!(v["payload"]["identities"] + .as_array() + .unwrap() + .iter() + .all(|r| r["role"] == "Train")); +} +#[test] +fn managed_cannot_take_seed_labels_skip_slots_or_admit_wrong_frozen_model() { + let plan = schedule(); + let mut session = TemporalTrainingSession::managed( + plan.clone(), + "guard-run".into(), + FitConfig { + epochs: 2, + ..Default::default() + }, + ) + .unwrap(); + assert!(session.begin_window(10, 3_000_000_010).is_err()); + let bound = plan.bind_episode_start("guard-run", 0, 0, 0, None).unwrap(); + let target = + ManagedWindowTarget::from_runtime_episode(&plan, &bound, "guard-run", 1, 10_000_000_000) + .unwrap(); + assert!(session.begin_managed_window(&target).is_err()); + let foreign = plan + .bind_episode_start("foreign-run", 0, 0, 0, None) + .unwrap(); + let foreign_target = ManagedWindowTarget::from_runtime_episode( + &plan, + &foreign, + "foreign-run", + 0, + 10_000_000_000, + ) + .unwrap(); + assert!(session.begin_managed_window(&foreign_target).is_err()); + assert!(ManagedWindowTarget::from_runtime_episode( + &plan, + &bound, + "other-run", + 0, + 10_000_000_000 + ) + .is_err()); + let frozen = + FrozenModelIdentity::from_runtime_verified_artifact(&plan, "b".repeat(64)).unwrap(); + let held = plan + .bind_episode_start("guard-run", 7, 0, 0, Some(&frozen)) + .unwrap(); + let target = + ManagedWindowTarget::from_runtime_episode(&plan, &held, "guard-run", 0, 1_000_000_000_000) + .unwrap(); + assert!(session.begin_managed_window(&target).is_err()); +} +#[test] +fn canonical_v3_schedule_role_encoder_frame_provenance_and_authority_tampering_fail() { + let original = model().export().unwrap(); + for case in [ + "schedule", + "role", + "encoder", + "frame", + "provenance", + "authority", + ] { + let mut e: EnvelopeV3 = serde_json::from_slice(&original).unwrap(); + match case { + "schedule" => { + let mut v: serde_json::Value = + serde_json::from_str(&e.payload.binding.schedule_json).unwrap(); + v["body"]["episodes"][0]["target_active_recording_ms"] = 300000.into(); + e.payload.binding.schedule_json = v.to_string(); + } + "role" => { + e.payload.identities[0].role = DataRole::HeldOut; + e.payload.identity_evidence_sha256 = hash(&e.payload.identities).unwrap(); + } + "encoder" => e.payload.binding.encoder.temporal_tokens = 8, + "frame" => { + e.payload.evidence[0].sources[0].frame_ids[0] = "0".into(); + e.payload.input_evidence_sha256 = hash(&e.payload.evidence).unwrap(); + } + "provenance" => { + e.payload.evidence[0].provenance = CaptureProvenance::LiveReceiverObserved; + e.payload.input_evidence_sha256 = hash(&e.payload.evidence).unwrap(); + } + "authority" => e.payload.claims.whole_episode_quality_issued = true, + _ => unreachable!(), + } + e.payload_sha256 = hash(&e.payload).unwrap(); + let bad = serde_json::to_vec(&e).unwrap(); + assert!(TemporalModel::import(&bad).is_err(), "{case}"); + } +} +#[test] +fn v2_codec_remains_unwrapped_and_managed_fit_permit_cannot_be_omitted_or_forged() { + // The exact old V2 codec structs/field order are unchanged. Existing V2 native + // fit/cold/import golden suite is also run with this integration. + let plan = schedule(); + let mut s = TemporalTrainingSession::managed( + plan, + "permit-run".into(), + FitConfig { + epochs: 2, + ..Default::default() + }, + ) + .unwrap(); + assert!(s.fit().is_err()); + let fake = ManagedFitAdmission { + policy: "a".repeat(64), + data: "b".repeat(64), + evidence: "c".repeat(64), + identities: "d".repeat(64), + }; + assert!(s + .fit_managed_with_cancel(&fake, &AtomicBool::new(false)) + .is_err()); + assert!(RoomModel::import(&model().export().unwrap()).is_err()); +} + +#[test] +fn mode_correct_policy_identity_tracks_managed_targets_and_preserves_seed_v2_identity() { + let plan = schedule(); + let config = FitConfig { + epochs: 2, + ..Default::default() + }; + let expected = hash(&ManagedBinding::build(&plan, config.clone()).unwrap()).unwrap(); + let mut session = + TemporalTrainingSession::managed(plan.clone(), "identity-run".into(), config.clone()) + .unwrap(); + assert_eq!(session.plan_sha256(), expected, "managed session must expose the whole managed input binding, not its seed validation carrier"); + assert_ne!(session.plan_sha256(), session.plan.sha256()); + let bound = plan + .bind_episode_start("identity-run", 0, 0, 0, None) + .unwrap(); + let target = + ManagedWindowTarget::from_runtime_episode(&plan, &bound, "identity-run", 0, 10_000_000_000) + .unwrap(); + assert_eq!(target.identity().contract_sha256, plan.sha256()); + // Actual native target admission -> assembled window -> evidence conversion + // rejects any policy identity different from the session's binding. + collect(&mut session, &target, 0, RoomLabel::Empty); + assert_eq!(session.evidence.len(), 1); + assert_eq!(model().plan_sha256(), expected); + let bands = (1..=3) + .map(|id| (id, ReceiverCadenceBand::Hz30To36)) + .collect::>(); + let seed = TemporalTrainingSession::new( + context(), + ExperimentalPlanId::Native2000msSlots2, + &bands, + "seed-identity-run".into(), + config, + ) + .unwrap(); + let old = + AcquisitionPlan::build(context(), ExperimentalPlanId::Native2000msSlots2, &bands).unwrap(); + assert_eq!(seed.plan_sha256(), old.sha256()); + assert_eq!(seed.plan_sha256(), seed.binding.plan_sha256); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_tests.rs new file mode 100644 index 0000000000..d3dc29246e --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_tests.rs @@ -0,0 +1,439 @@ +//! SYNTHETIC complete native horizons through existing production fitter/infer. +use super::super::native::{canonicalize_esp32, NativePacketInput}; +use super::*; +use std::sync::OnceLock; +use wifi_densepose_hardware::{Adr018Flags, PpduType}; +fn context(count: u8) -> CaptureContext { + CaptureContext { + owner_scope: "a".repeat(64), + installation_id: "synthetic-temporal".into(), + room_digest: "synthetic-room".into(), + revision: 1, + sources: (1..=count) + .map(|source_id| SourceSpec { + source_id, + device_id: format!("esp32-node-{source_id}"), + grid_id: "ant1-bins64-ppdu0-usable-drop2-linear-v2".into(), + feature_count: 56, + signal_kind: ruview_unified::RfModality::WifiCsi, + center_frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + link_id: format!("native-{source_id}"), + link_index: 0, + geometry: None, + }) + .collect(), + } +} +fn bands(n: u8) -> Vec<(u8, ReceiverCadenceBand)> { + (1..=n) + .map(|n| (n, ReceiverCadenceBand::Hz30To36)) + .collect() +} +fn config() -> FitConfig { + FitConfig { + epochs: 2, + ..Default::default() + } +} +fn packet(source: u8, seq: u32, time: u64, label: RoomLabel, row: usize) -> CanonicalPacket { + let gain = match label { + RoomLabel::Empty => 3, + RoomLabel::AdjacentOnly => 4, + RoomLabel::OccupiedStatic => 8, + RoomLabel::OccupiedMoving => 10 + (row % 2) as u8, + RoomLabel::Unknown => 8, + }; + let bytes = (0..64) + .flat_map(|bin| [gain + source + (bin % 2) as u8, 4u8]) + .collect::>(); + canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence: seq, + received_at_ns: time, + receiver_sample_rate_hz: 33., + iq_bytes: &bytes, + flags: Adr018Flags::default(), + provenance: CaptureProvenance::Synthetic, + }) + .unwrap() +} +fn collect(session: &mut TemporalTrainingSession, target: u64, seq: &mut u32) { + session + .begin_window(target, target + 3_000_000_000) + .unwrap(); + let step = &session.plan.schedule()[session.rows.len() / session.plan.slots_per_trial()]; + let label = step.label; + let period = 1_000_000_000u64.div_ceil(33); + for row in 0..256 { + *seq += 1; + for source in 1..=2 { + let now = target + row as u64 * period + u64::from(source) * 1000; + if session + .ingest(&packet(source, *seq, now, label, row)) + .unwrap() + == WindowAdmission::Ready + { + session.finish_window(now).unwrap(); + return; + } + } + } + panic!("no native horizon"); +} +fn fitted_bytes() -> Vec { + static MODEL: OnceLock> = OnceLock::new(); + MODEL + .get_or_init(|| { + let mut s = TemporalTrainingSession::new( + context(2), + ExperimentalPlanId::Native2000msSlots2, + &bands(2), + "synthetic-fit".into(), + config(), + ) + .unwrap(); + let mut seq = 0; + assert!(s.fit().is_err()); + for index in 0..16 { + collect(&mut s, 10_000_000_000 + index * 3_000_000_000, &mut seq); + } + assert_eq!(s.dataset().unwrap().train().len(), 16); + assert!(s.begin_window(60_000_000_000, 63_000_000_000).is_err()); + let (model, report) = s.fit().unwrap(); + assert_eq!(report.training_cohorts, 16); + assert!(report.backbone_delta_l2 > 0.); + assert!(report.parameter_delta_l2 > 0.); + assert_eq!(model.model.encoder_config(), s.plan.encoder_config()); + // Frozen fit precedes new HeldOut windows. A later fit cannot consume them. + for index in 16..32 { + collect(&mut s, 10_000_000_000 + index * 3_000_000_000, &mut seq); + } + let d = s.dataset().unwrap(); + assert_eq!(d.held_out().len(), 16); + assert!(!d.measurement_clock_available()); + assert!(s.fit().is_err()); + model.export().unwrap() + }) + .clone() +} +fn model() -> TemporalModel { + TemporalModel::import(&fitted_bytes()).unwrap() +} +fn live( + model: &TemporalModel, + target: u64, + rows: usize, +) -> Result<(TemporalLiveObservation, u64), String> { + let mut input = model.live_input("fresh-live".into())?; + input.begin_window(target, target + 3_000_000_000)?; + let period = 1_000_000_000u64.div_ceil(33); + let mut last = target; + for row in 0..rows { + for source in 1..=2 { + last = target + row as u64 * period + u64::from(source) * 1000; + let state = input.ingest(&packet( + source, + 100_000 + row as u32, + last, + RoomLabel::Unknown, + row, + ))?; + if state == WindowAdmission::Ready { + return Ok((input.finish(last)?, last)); + } + } + } + Ok((input.finish(last)?, last)) +} +#[test] +fn parameter_source_and_frozen_encoder_preflight_run_before_collection_or_fit() { + for count in [1, 9] { + assert!(TemporalTrainingSession::new( + context(count), + ExperimentalPlanId::Native1000msSlots2, + &bands(count), + "bad".into(), + config() + ) + .is_err()); + } + let mut exact = config(); + let count = exact.validate_context(&context(2)).unwrap(); + exact.max_parameters = count - 1; + assert!(TemporalTrainingSession::new( + context(2), + ExperimentalPlanId::Native2000msSlots2, + &bands(2), + "actual-boundary".into(), + exact + ) + .is_err()); + let mut tiny = config(); + tiny.max_parameters = 1; + assert!(TemporalTrainingSession::new( + context(2), + ExperimentalPlanId::Native2000msSlots2, + &bands(2), + "budget".into(), + tiny + ) + .is_err()); + let mut changed = config(); + changed.temporal_tokens = 8; + assert!(TemporalTrainingSession::new( + context(2), + ExperimentalPlanId::Native2000msSlots2, + &bands(2), + "changed".into(), + changed + ) + .is_err()); +} +#[test] +fn actual_fitter_cold_roundtrip_and_live_forward_keep_old_room_model_bytes_unchanged() { + let m = model(); + let bytes = m.export().unwrap(); + assert_eq!(bytes, fitted_bytes()); + let cold = TemporalModel::import(&bytes).unwrap(); + assert_eq!(cold.export().unwrap(), bytes); + assert_eq!(cold.parameter_count(), m.parameter_count()); + let old = m.model.export().unwrap(); + assert_eq!(RoomModel::import(&old).unwrap().export().unwrap(), old); + assert!(RoomModel::import(&bytes).is_err()); + assert!(TemporalModel::import(&old).is_err()); + let (input, now) = live(&cold, 1_000_000_000_000, 100).unwrap(); + assert_eq!(input.window.cohort.role, DataRole::Inference); + assert_eq!(input.window.cohort.label, RoomLabel::Unknown); + assert_eq!(input.window.cohort.views[0].samples.len(), 67); + let plain = cold.model.infer(&input.window.cohort); + let prediction = cold.predict(input, now).unwrap(); + assert_eq!( + serde_json::to_value(plain).unwrap(), + serde_json::to_value(prediction).unwrap() + ); + let envelope: Envelope = serde_json::from_slice(&bytes).unwrap(); + assert!(!serde_json::to_string(&envelope.payload.evidence) + .unwrap() + .contains("samples")); + assert_eq!( + envelope.payload.training_data_sha256, + m.model.training_manifest().training_data_sha256 + ); +} +#[test] +fn live_eight_or_64_packets_cannot_substitute_for_two_seconds() { + let m = model(); + for rows in [8, 64] { + assert!(live(&m, 1_000_000_000_000, rows).is_err()); + } +} +#[test] +fn live_training_replay_foreign_grid_policy_times_and_labelled_roles_are_refused() { + let m = model(); + let mut input = m.live_input("replayed-live".into()).unwrap(); + input + .begin_window(1_000_000_000_000, 1_003_000_000_000) + .unwrap(); + assert!(input + .ingest(&packet(1, 1, 1_000_000_001_000, RoomLabel::Unknown, 0)) + .is_err()); + for mode in ["policy", "grid", "times", "role", "rows", "stale"] { + let (mut input, now) = live(&m, 1_000_000_000_000, 100).unwrap(); + match mode { + "policy" => input.binding.plan_sha256 = "b".repeat(64), + "grid" => input.window.cohort.views[0].grid_id = "wrong".into(), + "times" => input.window.row_times_ns[0].1[1] += 100_000_000, + "role" => { + input.window.cohort.role = DataRole::Train; + input.window.cohort.label = RoomLabel::Empty; + } + "rows" => { + input.window.cohort.views[0].samples.truncate(8); + input.window.cohort.views[0].frame_ids.truncate(8); + input.window.cohort.views[0].sequence_ids.truncate(8); + input.window.row_times_ns[0].1.truncate(8); + } + _ => {} + } + assert!( + m.predict( + input, + if mode == "stale" { + now + 6_000_000_000 + } else { + now + } + ) + .is_err(), + "{mode}" + ); + } +} +fn recode(mut e: Envelope) -> Vec { + e.payload.input_evidence_sha256 = hash(&e.payload.evidence).unwrap(); + e.payload_sha256 = hash(&e.payload).unwrap(); + serde_json::to_vec(&e).unwrap() +} + +#[test] +fn cold_import_rejects_hash_policy_timing_source_and_cross_role_tampering() { + let bytes = fitted_bytes(); + for mode in [ + "outer_hash", + "model_hash", + "plan_hash", + "plan_horizon", + "source", + "rows", + "gap", + "frame_overlap", + "role", + "provenance", + ] { + let mut e: Envelope = serde_json::from_slice(&bytes).unwrap(); + match mode { + "outer_hash" => e.payload_sha256 = "0".repeat(64), + "model_hash" => e.payload.model_sha256 = "0".repeat(64), + "plan_hash" => e.payload.binding.plan_sha256 = "0".repeat(64), + "plan_horizon" => { + let p = AcquisitionPlan::build( + context(2), + ExperimentalPlanId::Native1000msSlots2, + &bands(2), + ) + .unwrap(); + e.payload.binding = Binding::from_plan(&p); + } + "source" => e.payload.evidence[0].sources[0].source_spec_sha256 = "f".repeat(64), + "rows" => { + let s = &mut e.payload.evidence[0].sources[0]; + s.times_ns.truncate(8); + s.frame_ids.truncate(8); + s.sequence_ids.truncate(8); + } + "gap" => { + let s = &mut e.payload.evidence[0].sources[0]; + let t = integer::(&s.times_ns[1]).unwrap(); + s.times_ns[1] = (t + 500_000_000).to_string(); + } + "frame_overlap" => { + e.payload.evidence[1].sources[0].frame_ids[0] = + e.payload.evidence[0].sources[0].frame_ids[0].clone() + } + "role" => e.payload.evidence[0].role = DataRole::HeldOut, + "provenance" => { + e.payload.evidence[0].provenance = CaptureProvenance::LiveReceiverObserved + } + _ => unreachable!(), + } + // Even recomputing generic envelope checksums cannot make a structurally + // incompatible plan/role/time/source claim satisfy the real input policy. + let altered = if mode == "outer_hash" { + serde_json::to_vec(&e).unwrap() + } else { + recode(e) + }; + assert!(TemporalModel::import(&altered).is_err(), "{mode}"); + } + assert!(TemporalModel::import(&vec![b' '; MAX_BYTES + 1]).is_err()); +} +#[test] +fn caller_cannot_fit_incomplete_or_failed_acquisition_as_a_complete_policy() { + let mut s = TemporalTrainingSession::new( + context(2), + ExperimentalPlanId::Native2000msSlots2, + &bands(2), + "failure".into(), + config(), + ) + .unwrap(); + let t = 10_000_000_000; + s.begin_window(t, t + 3_000_000_000).unwrap(); + s.ingest(&packet(1, 1, t + 1000, RoomLabel::Empty, 0)) + .unwrap(); + assert!(s.finish_window(t + 1_000_000_000).is_err()); + assert!(s.fit().is_err()); + assert!(s + .begin_window(t + 4_000_000_000, t + 7_000_000_000) + .is_err()); +} + +#[test] +fn cold_consistent_rate_time_shift_cannot_detach_original_receiver_times_from_frame_ids() { + let mut e: Envelope = serde_json::from_slice(&fitted_bytes()).unwrap(); + // Keep duration, row count, gaps, average cadence and source skew valid. + // Recomputed envelope hashes cannot make the unchanged ORIGINAL frame IDs + // refer to the shifted receiver times or a new original wire counter. + for source in &mut e.payload.evidence[0].sources { + for time in &mut source.times_ns { + *time = (integer::(time).unwrap() + 1000).to_string(); + } + } + let seal = integer::(&e.payload.evidence[0].sealed_at_ns).unwrap(); + e.payload.evidence[0].sealed_at_ns = (seal + 1000).to_string(); + assert!(TemporalModel::import(&recode(e)).is_err()); +} + +#[test] +fn post_assembler_evidence_failure_is_terminal_and_preserves_previous_valid_windows() { + let mut session = TemporalTrainingSession::new( + context(2), + ExperimentalPlanId::Native2000msSlots2, + &bands(2), + "post-consume".into(), + config(), + ) + .unwrap(); + let mut sequence = 0; + collect(&mut session, 10_000_000_000, &mut sequence); + let prior = hash(&session.rows).unwrap(); + let target = 20_000_000_000; + session + .begin_window(target, target + 3_000_000_000) + .unwrap(); + let period = 1_000_000_000u64.div_ceil(33); + let mut now = target; + 'ready: for row in 0..256 { + sequence += 1; + for source in 1..=2 { + now = target + row as u64 * period + u64::from(source) * 1000; + let mut packet = packet(source, sequence, now, RoomLabel::Empty, row); + if row == 0 && source == 1 { + packet.native_frame.frame_id ^= 1; + assert_ne!(packet.native_frame.frame_id, 0); + } + if session.ingest(&packet).unwrap() == WindowAdmission::Ready { + break 'ready; + } + } + } + assert_eq!( + session.finish_window(now).err().as_deref(), + Some("temporal_native_frame_identity_changed") + ); + assert!(session.failed); + assert!(!session.active); + assert!(session.target_ns.is_none()); + assert_eq!(session.rows.len(), 1); + assert_eq!(session.evidence.len(), 1); + assert_eq!(hash(&session.rows).unwrap(), prior); + assert!(session + .begin_window(30_000_000_000, 33_000_000_000) + .is_err()); + assert!(session + .ingest(&packet( + 1, + sequence + 1, + 30_000_001_000, + RoomLabel::Empty, + 0 + )) + .is_err()); + assert!(session.finish_window(now).is_err()); + assert!(session.fit().is_err()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs new file mode 100644 index 0000000000..b97d4a2d0d --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs @@ -0,0 +1,1291 @@ +//! Bounded selective-file consistency for private temporal prefixes. +//! NOT a trusted monotonic anchor against coordinated whole-profile rollback. +//! Typed APIs take receiver nanoseconds; descriptor/retention fields stay ms. +use super::*; +use crate::room_learner::temporal_model_input::managed::execution::checkpoint::{ + is_new_empty_run, validate_extension, validated_prefix, ValidatedTemporalPrefix, +}; +use sha2::{Digest, Sha256}; +const KIND: &str = "temporal-prefix-commitment-v1"; +const LOCK: &str = "temporal-prefix.lock"; +const HEAD: &str = "temporal-head.bin"; +const MAGIC: &[u8; 8] = b"RVTCPR01"; +const SMALL: usize = 4096; +const MAX_TRANSITIONS: usize = 512; +const MAX_WITNESSES: usize = 2 * MAX_TRANSITIONS; +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct Meta { + schema: String, + generation: u64, + revision: usize, + descriptor: RecoveryDescriptor, + run: String, + policy_sha256: String, + transition_limit: usize, + window_count: usize, + receiver_ns: u64, + slot: Option, + payload_sha256: Option, + previous_sha256: Option, + retired_at_ms: Option, +} +fn hash(bytes: &[u8]) -> String { + format!("{:x}", Sha256::digest(bytes)) +} +impl Meta { + fn sha(&self) -> Result { + Ok(hash( + &serde_json::to_vec(self).map_err(|_| "temporal_metadata_encoding")?, + )) + } + fn token(&self) -> Result { + Ok(TemporalToken { + generation: self.generation, + revision: self.revision, + run: self.run.clone(), + id: self.descriptor.id.clone(), + commitment: self.sha()?, + }) + } + fn witness(&self) -> String { + format!("witness-{:016x}-{:04x}.bin", self.generation, self.revision) + } +} +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct TemporalToken { + generation: u64, + revision: usize, + run: String, + id: String, + commitment: String, +} +#[derive(Clone)] +pub(crate) struct TemporalIntent { + expected: Option, + meta: Meta, +} +/// Authenticated observation only; not a completed durability-barrier certificate. +pub(crate) struct TemporalLoaded { + pub(crate) token: TemporalToken, + pub(crate) prefix: Option, +} +struct Components { + head: Option, + latest: Option, + witnesses: Vec<(String, Meta)>, +} +struct Locked { + file: File, + directory: PathBuf, +} +impl Drop for Locked { + fn drop(&mut self) { + let _ = self.file.unlock(); + } +} +impl RecoveryStore { + fn temporal_dir(&self) -> PathBuf { + self.directory.join(KIND) + } + fn temporal_lock(&self, create: bool) -> Result { + self.verify_directory()?; + let path = self.directory.join(LOCK); + let mut options = OpenOptions::new(); + options.read(true).write(true).create(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + #[cfg(target_os = "macos")] + options.custom_flags(0x100); + #[cfg(target_os = "linux")] + options.custom_flags(0x20000); + } + let file = options.open(&path).map_err(|_| "temporal_lock_open")?; + private_metadata( + &file.metadata().map_err(|_| "temporal_lock_metadata")?, + false, + )?; + file.lock().map_err(|_| "temporal_lock_failed")?; + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let a = file.metadata().map_err(|_| "temporal_lock_metadata")?; + let b = fs::symlink_metadata(&path).map_err(|_| "temporal_lock_inode_changed")?; + if b.file_type().is_symlink() || a.ino() != b.ino() || a.dev() != b.dev() { + return Err("temporal_lock_inode_changed".into()); + } + } + let directory = self.temporal_dir(); + if create { + private_directory(&directory)?; + File::open(&self.directory) + .and_then(|f| f.sync_all()) + .map_err(|_| "temporal_namespace_parent_fsync")?; + } else if directory.exists() { + private_directory(&directory)?; + } + Ok(Locked { file, directory }) + } + fn temporal_read(&self, path: &Path, limit: usize) -> Result>, String> { + let meta = match fs::symlink_metadata(path) { + Ok(m) => m, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(_) => return Err("temporal_read_failed".into()), + }; + if meta.file_type().is_symlink() || !meta.is_file() || meta.len() > limit as u64 { + return Err("temporal_unsafe_file".into()); + } + private_metadata(&meta, false)?; + let mut options = OpenOptions::new(); + options.read(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + #[cfg(target_os = "macos")] + options.custom_flags(0x100); + #[cfg(target_os = "linux")] + options.custom_flags(0x20000); + } + let file = options.open(path).map_err(|_| "temporal_read_failed")?; + let opened = file.metadata().map_err(|_| "temporal_read_failed")?; + private_metadata(&opened, false)?; + if !opened.is_file() || opened.len() > limit as u64 { + return Err("temporal_unsafe_file".into()); + } + let mut bytes = Vec::new(); + file.take((limit + 1) as u64) + .read_to_end(&mut bytes) + .map_err(|_| "temporal_read_failed")?; + if bytes.len() > limit { + return Err("temporal_read_budget".into()); + } + Ok(Some(bytes)) + } + fn temporal_seal(&self, domain: &str, plain: &[u8]) -> Result, String> { + let key = self.key.as_ref().ok_or("recovery_unavailable")?; + let aad = serde_json::to_vec(&(KIND, domain, &self.owner, &self.installation)) + .map_err(|_| "temporal_aad")?; + let mut nonce = [0; 12]; + SystemRandom::new() + .fill(&mut nonce) + .map_err(|_| "recovery_random_unavailable")?; + let mut cipher = plain.to_vec(); + key.seal_in_place_append_tag( + aead::Nonce::assume_unique_for_key(nonce), + aead::Aad::from(&aad), + &mut cipher, + ) + .map_err(|_| "temporal_encrypt")?; + let mut out = MAGIC.to_vec(); + out.extend_from_slice(&nonce); + out.extend(cipher); + Ok(out) + } + fn temporal_open(&self, domain: &str, bytes: &[u8], limit: usize) -> Result, String> { + if bytes.len() < 36 || bytes.len() > limit + 36 || &bytes[..8] != MAGIC { + return Err("temporal_envelope".into()); + } + let key = self.key.as_ref().ok_or("recovery_unavailable")?; + let aad = serde_json::to_vec(&(KIND, domain, &self.owner, &self.installation)) + .map_err(|_| "temporal_aad")?; + let nonce = + aead::Nonce::try_assume_unique_for_key(&bytes[8..20]).map_err(|_| "temporal_nonce")?; + let mut cipher = bytes[20..].to_vec(); + let p = key + .open_in_place(nonce, aead::Aad::from(&aad), &mut cipher) + .map_err(|_| "temporal_authentication")?; + if p.len() > limit { + return Err("temporal_plaintext_budget".into()); + } + Ok(p.to_vec()) + } + fn temporal_meta(&self, path: &Path, domain: &str) -> Result, String> { + let Some(bytes) = self.temporal_read(path, SMALL + 36)? else { + return Ok(None); + }; + let p = self.temporal_open(domain, &bytes, SMALL)?; + let m: Meta = serde_json::from_slice(&p).map_err(|_| "temporal_metadata_schema")?; + if serde_json::to_vec(&m).map_err(|_| "temporal_metadata_encoding")? != p + || m.schema != KIND + || m.transition_limit > MAX_TRANSITIONS + || m.transition_limit < 2 + || m.revision >= m.transition_limit + || !id(&m.run) + || !hex(&m.policy_sha256) + || m.window_count > 64 + { + return Err("temporal_metadata_binding".into()); + } + self.descriptor(&m.descriptor, None)?; + if m.descriptor.expires_at_ms - m.descriptor.created_at_ms != RECOVERY_TTL_MS { + return Err("temporal_original_deadline".into()); + } + if m.receiver_ns / 1_000_000 < m.descriptor.saved_at_ms { + return Err("temporal_metadata_receiver_clock".into()); + } + match (m.slot, m.payload_sha256.as_ref(), m.retired_at_ms) { + (Some(slot), Some(sha), None) + if slot < 2 + && hex(sha) + && m.receiver_ns / 1_000_000 == m.descriptor.saved_at_ms => {} + (None, None, Some(t)) + if t >= m.descriptor.saved_at_ms && t == m.receiver_ns / 1_000_000 => {} + _ => return Err("temporal_metadata_state".into()), + }; + Ok(Some(m)) + } + fn temporal_components(&self, dir: &Path) -> Result { + if !dir.exists() { + return Ok(Components { + head: None, + latest: None, + witnesses: vec![], + }); + } + let mut witnesses = Vec::new(); + let mut count = 0; + for entry in fs::read_dir(dir).map_err(|_| "temporal_directory_read")? { + count += 1; + if count > MAX_WITNESSES + 6 { + return Err("temporal_directory_budget".into()); + } + let e = entry.map_err(|_| "temporal_directory_read")?; + let name = e + .file_name() + .into_string() + .map_err(|_| "temporal_unknown_path")?; + let entry_meta = + fs::symlink_metadata(e.path()).map_err(|_| "temporal_directory_read")?; + if entry_meta.file_type().is_symlink() + || !entry_meta.is_file() + || entry_meta.len() + > (if name.starts_with("witness-") + || [HEAD, "head.stage", "witness.stage"].contains(&name.as_str()) + { + SMALL + 36 + } else { + MAX_RECOVERY_BYTES + SMALL + 48 + }) as u64 + { + return Err("temporal_unsafe_directory_entry".into()); + } + private_metadata(&entry_meta, false)?; + if name.starts_with("witness-") && name.ends_with(".bin") { + let m = self + .temporal_meta(&e.path(), "witness")? + .ok_or("temporal_witness_missing")?; + if name != m.witness() { + return Err("temporal_witness_name".into()); + } + witnesses.push((name, m)); + } else if ![ + HEAD, + "slot-0.bin", + "slot-1.bin", + "payload.stage", + "head.stage", + "witness.stage", + ] + .contains(&name.as_str()) + { + return Err("temporal_unknown_path".into()); + } + } + witnesses.sort_by_key(|(_, m)| (m.generation, m.revision)); + let generations = witnesses + .iter() + .map(|(_, m)| m.generation) + .collect::>(); + if generations.len() > 2 { + return Err("temporal_generation_capacity".into()); + } + // GC may already have removed any prefix of an OBSOLETE generation. + // Authenticate every remaining record above, but require a COMPLETE + // contiguous chain for the highest committed-witness generation. The + // head/payload must match its exact tip before any caller gets authority. + let highest = witnesses.last().map(|(_, m)| m.generation); + let mut previous: Option<&Meta> = None; + for (_, m) in witnesses + .iter() + .filter(|(_, m)| Some(m.generation) == highest) + { + if let Some(p) = previous { + if m.revision != p.revision + 1 || m.previous_sha256.as_ref() != Some(&p.sha()?) { + return Err("temporal_witness_chain".into()); + } + } else if m.revision != 0 + || (m.generation == 0 && m.previous_sha256.is_some()) + || (m.generation > 0 && !m.previous_sha256.as_ref().is_some_and(|p| hex(p))) + { + return Err("temporal_witness_genesis".into()); + } + previous = Some(m); + } + let latest = witnesses.last().map(|(_, m)| m.clone()); + let head = self.temporal_meta(&dir.join(HEAD), "head")?; + Ok(Components { + head, + latest, + witnesses, + }) + } + fn temporal_current(&self, dir: &Path, now_ns: u64) -> Result, String> { + let c = self.temporal_components(dir)?; + match (c.head, c.latest) { + (None, None) => Ok(None), + (Some(h), Some(w)) if h == w => { + if now_ns < h.receiver_ns + || now_ns / 1_000_000 < h.descriptor.saved_at_ms + || h.retired_at_ms.is_some_and(|t| now_ns / 1_000_000 < t) + { + return Err("recovery_clock_regression".into()); + } + let token = h.token()?; + if h.retired_at_ms.is_some() { + return Ok(Some(TemporalLoaded { + token, + prefix: None, + })); + } + if now_ns / 1_000_000 >= h.descriptor.expires_at_ms { + return Err("recovery_expired".into()); + } + let prefix = self.temporal_payload(dir, &h, now_ns)?; + Ok(Some(TemporalLoaded { + token, + prefix: Some(prefix), + })) + } + _ => Err("temporal_head_witness_mismatch_or_pending".into()), + } + } + fn temporal_payload_seal( + &self, + m: &Meta, + p: &ValidatedTemporalPrefix, + ) -> Result, String> { + let metadata = serde_json::to_vec(m).map_err(|_| "temporal_metadata_encoding")?; + if metadata.len() > SMALL { + return Err("temporal_metadata_budget".into()); + } + let mut out = b"RVTPAY01".to_vec(); + out.extend_from_slice(&(metadata.len() as u32).to_be_bytes()); + out.extend(metadata); + out.extend(self.temporal_seal(&format!("payload:{}", m.sha()?), p.bytes())?); + Ok(out) + } + fn temporal_payload_any( + &self, + path: &Path, + now_ns: u64, + ) -> Result, String> { + let Some(bytes) = self.temporal_read(path, MAX_RECOVERY_BYTES + SMALL + 48)? else { + return Ok(None); + }; + if bytes.len() < 12 || &bytes[..8] != b"RVTPAY01" { + return Err("temporal_payload_header".into()); + } + let n = u32::from_be_bytes( + bytes[8..12] + .try_into() + .map_err(|_| "temporal_payload_header")?, + ) as usize; + if n == 0 || n > SMALL || bytes.len() < 12 + n + 36 { + return Err("temporal_payload_header".into()); + } + let m: Meta = + serde_json::from_slice(&bytes[12..12 + n]).map_err(|_| "temporal_payload_metadata")?; + if serde_json::to_vec(&m).map_err(|_| "temporal_metadata_encoding")? != bytes[12..12 + n] + || m.schema != KIND + || m.retired_at_ms.is_some() + { + return Err("temporal_payload_metadata".into()); + } + let plain = self.temporal_open( + &format!("payload:{}", m.sha()?), + &bytes[12 + n..], + MAX_RECOVERY_BYTES, + )?; + if Some(hash(&plain)).as_ref() != m.payload_sha256.as_ref() { + return Err("temporal_payload_hash".into()); + } + let p = validated_prefix(&plain, &m.descriptor, now_ns)?; + if p.run() != m.run + || p.policy_sha256() != m.policy_sha256 + || p.window_count() != m.window_count + || p.transition_limit() != m.transition_limit + || p.saved_receiver_ns() != m.receiver_ns + { + return Err("temporal_payload_metadata".into()); + } + Ok(Some((m, p))) + } + fn temporal_payload( + &self, + dir: &Path, + m: &Meta, + now_ns: u64, + ) -> Result { + let slot = m.slot.ok_or("temporal_retired")?; + let (actual, p) = self + .temporal_payload_any(&dir.join(format!("slot-{slot}.bin")), now_ns)? + .ok_or("temporal_payload_missing")?; + if actual != *m { + return Err("temporal_named_payload_changed".into()); + } + Ok(p) + } + pub(crate) fn load_temporal(&self, now_ns: u64) -> Result, String> { + let _serial = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + let locked = self.temporal_lock(false)?; + self.temporal_current(&locked.directory, now_ns) + } +} + +impl RecoveryStore { + pub(crate) fn prepare_temporal( + &self, + expected: Option<&TemporalToken>, + prefix: &ValidatedTemporalPrefix, + now_ns: u64, + ) -> Result { + let _serial = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + let locked = self.temporal_lock(true)?; + self.descriptor(prefix.descriptor(), Some(now_ns / 1_000_000))?; + if now_ns < prefix.saved_receiver_ns() { + return Err("recovery_receiver_clock_regression".into()); + } + let current = self.temporal_current(&locked.directory, now_ns)?; + let c = self.temporal_components(&locked.directory)?; + let (generation, revision, previous, slot) = match (current, expected) { + (None, None) => { + if fs::read_dir(&locked.directory) + .map_err(|_| "temporal_directory_read")? + .next() + .is_some() + || !is_new_empty_run(prefix)? + { + return Err("temporal_explicit_empty_genesis_required".into()); + } + (0, 0, None, 0) + } + (Some(loaded), Some(expected)) if &loaded.token == expected => { + let old = c.head.as_ref().ok_or("temporal_head_missing")?; + if let Some(previous_prefix) = loaded.prefix { + validate_extension(&previous_prefix, prefix)?; + if previous_prefix.bytes() == prefix.bytes() { + return Ok(TemporalIntent { + expected: Some(expected.clone()), + meta: old.clone(), + }); + } + if self.temporal_cleanup_pending(&locked.directory, &c, old)? { + return Err("temporal_cleanup_repair_required".into()); + } + ( + old.generation, + old.revision + .checked_add(1) + .ok_or("temporal_revision_overflow")?, + Some(old.sha()?), + 1 - old.slot.ok_or("temporal_retired")?, + ) + } else { + if self.temporal_cleanup_pending(&locked.directory, &c, old)? { + return Err("temporal_cleanup_repair_required".into()); + } + if !is_new_empty_run(prefix)? + || prefix.descriptor().id == old.descriptor.id + || prefix.run() == old.run + || prefix.descriptor().created_at_ms + < old.retired_at_ms.ok_or("temporal_retirement_missing")? + || prefix.saved_receiver_ns() < old.receiver_ns + { + return Err("temporal_explicit_new_empty_run_required".into()); + } + ( + old.generation + .checked_add(1) + .ok_or("temporal_generation_overflow")?, + 0, + Some(old.sha()?), + 0, + ) + } + } + _ => return Err("temporal_expected_token_changed".into()), + }; + if revision >= prefix.transition_limit() { + return Err("temporal_transition_budget".into()); + } + Ok(TemporalIntent { + expected: expected.cloned(), + meta: Meta { + schema: KIND.into(), + generation, + revision, + descriptor: prefix.descriptor().clone(), + run: prefix.run().into(), + policy_sha256: prefix.policy_sha256().into(), + transition_limit: prefix.transition_limit(), + window_count: prefix.window_count(), + receiver_ns: prefix.saved_receiver_ns(), + slot: Some(slot), + payload_sha256: Some(hash(prefix.bytes())), + previous_sha256: previous, + retired_at_ms: None, + }, + }) + } + pub(crate) fn prepare_retire_temporal( + &self, + expected: &TemporalToken, + now_ns: u64, + expiry_only: bool, + ) -> Result { + let _serial = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + let locked = self.temporal_lock(false)?; + let c = self.temporal_components(&locked.directory)?; + let mut meta = c + .head + .filter(|h| Some(h) == c.latest.as_ref()) + .ok_or("temporal_head_witness_mismatch_or_pending")?; + if meta.token()? != *expected + || now_ns < meta.receiver_ns + || now_ns / 1_000_000 < meta.descriptor.saved_at_ms + || meta.retired_at_ms.is_some_and(|t| now_ns / 1_000_000 < t) + || expiry_only && now_ns / 1_000_000 < meta.descriptor.expires_at_ms + { + return Err("temporal_retirement_identity_or_clock".into()); + } + if meta.retired_at_ms.is_none() { + meta.previous_sha256 = Some(meta.sha()?); + meta.revision = meta + .revision + .checked_add(1) + .ok_or("temporal_revision_overflow")?; + meta.slot = None; + meta.payload_sha256 = None; + meta.retired_at_ms = Some(now_ns / 1_000_000); + meta.receiver_ns = now_ns; + if meta.revision >= meta.transition_limit { + return Err("temporal_transition_budget".into()); + } + } + Ok(TemporalIntent { + expected: Some(expected.clone()), + meta, + }) + } + fn temporal_write_stage( + &self, + dir: &Path, + stage: &str, + bytes: &[u8], + hook: &mut impl FnMut(&str) -> Result<(), String>, + ) -> Result<(), String> { + hook(&format!("before:{stage}:write"))?; + let path = dir.join(stage); + if let Some(existing) = self.temporal_read(&path, MAX_RECOVERY_BYTES + SMALL + 48)? { + if existing != bytes { + return Err("temporal_conflicting_stage_requires_repair".into()); + } + let f = private_existing_file(&path, true)?; + f.sync_all().map_err(|_| "temporal_stage_fsync")?; + return Ok(()); + } + let mut options = OpenOptions::new(); + options.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(0o600); + } + let mut file = options.open(&path).map_err(|_| "temporal_stage_create")?; + file.write_all(bytes).map_err(|_| "temporal_stage_write")?; + hook(&format!("after:{stage}:write"))?; + hook(&format!("before:{stage}:fsync"))?; + file.sync_all().map_err(|_| "temporal_stage_fsync")?; + hook(&format!("after:{stage}:fsync"))?; + Ok(()) + } + fn temporal_replace( + &self, + dir: &Path, + stage: &str, + name: &str, + hook: &mut impl FnMut(&str) -> Result<(), String>, + ) -> Result<(), String> { + hook(&format!("before:{stage}:rename"))?; + fs::rename(dir.join(stage), dir.join(name)).map_err(|_| "temporal_stage_rename")?; + hook(&format!("after:{stage}:rename"))?; + hook(&format!("before:{stage}:directory_fsync"))?; + File::open(dir) + .and_then(|f| f.sync_all()) + .map_err(|_| "temporal_directory_fsync")?; + hook(&format!("after:{stage}:directory_fsync"))?; + Ok(()) + } + fn temporal_cleanup_pending( + &self, + dir: &Path, + c: &Components, + current: &Meta, + ) -> Result { + if c.witnesses + .iter() + .any(|(_, m)| m.generation < current.generation) + { + return Ok(true); + } + let mut names = vec!["payload.stage", "head.stage", "witness.stage"]; + if current.retired_at_ms.is_some() { + names.extend(["slot-0.bin", "slot-1.bin"]); + } + for name in names { + match fs::symlink_metadata(dir.join(name)) { + Ok(_) => return Ok(true), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(_) => return Err("temporal_cleanup_inspection".into()), + } + } + Ok(false) + } + fn temporal_cleanup(&self, dir: &Path, current: &Meta) -> Result<(), String> { + let c = self.temporal_components(dir)?; + for (name, m) in c.witnesses { + if m.generation < current.generation { + fs::remove_file(dir.join(name)).map_err(|_| "temporal_metadata_cleanup")?; + } + } + for name in ["payload.stage", "head.stage", "witness.stage"] { + if self + .temporal_read(&dir.join(name), MAX_RECOVERY_BYTES + SMALL + 48)? + .is_some() + { + fs::remove_file(dir.join(name)).map_err(|_| "temporal_stage_cleanup")?; + } + } + if current.retired_at_ms.is_some() { + for name in ["slot-0.bin", "slot-1.bin"] { + if self + .temporal_read(&dir.join(name), MAX_RECOVERY_BYTES + SMALL + 48)? + .is_some() + { + fs::remove_file(dir.join(name)).map_err(|_| "temporal_payload_cleanup")?; + } + } + } + File::open(dir) + .and_then(|f| f.sync_all()) + .map_err(|_| "temporal_cleanup_fsync")?; + Ok(()) + } + pub(crate) fn publish_temporal( + &self, + intent: &TemporalIntent, + prefix: Option<&ValidatedTemporalPrefix>, + now_ns: u64, + ) -> Result { + self.publish_temporal_with(intent, prefix, now_ns, false, &mut |_| Ok(())) + } + pub(crate) fn repair_same_temporal( + &self, + intent: &TemporalIntent, + prefix: Option<&ValidatedTemporalPrefix>, + now_ns: u64, + ) -> Result { + self.publish_temporal_with(intent, prefix, now_ns, true, &mut |_| Ok(())) + } + fn publish_temporal_with( + &self, + intent: &TemporalIntent, + prefix: Option<&ValidatedTemporalPrefix>, + now_ns: u64, + repair: bool, + hook: &mut impl FnMut(&str) -> Result<(), String>, + ) -> Result { + let _serial = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + let locked = self.temporal_lock(true)?; + let dir = &locked.directory; + let meta = &intent.meta; + if now_ns < meta.receiver_ns { + return Err("recovery_receiver_clock_regression".into()); + } + if let Some(p) = prefix { + self.descriptor(&meta.descriptor, Some(now_ns / 1_000_000))?; + if meta.retired_at_ms.is_some() + || p.descriptor() != &meta.descriptor + || p.run() != meta.run + || p.policy_sha256() != meta.policy_sha256 + || Some(hash(p.bytes())).as_ref() != meta.payload_sha256.as_ref() + { + return Err("temporal_intended_payload_changed".into()); + } + } else if meta.retired_at_ms.is_none() || now_ns / 1_000_000 < meta.retired_at_ms.unwrap() { + return Err("temporal_intended_retirement_changed".into()); + } + let c = self.temporal_components(dir)?; + let latest = c.latest.as_ref().map(Meta::token).transpose()?; + let target = meta.token()?; + let head = c.head.as_ref().map(Meta::token).transpose()?; + let already = latest.as_ref() == Some(&target); + if already { + if !repair && intent.expected.as_ref() != Some(&target) { + return Err("temporal_exact_repair_required".into()); + } + if head.as_ref() != Some(&target) && !repair { + return Err("temporal_head_witness_mismatch_or_pending".into()); + } + } else { + if latest != intent.expected { + return Err("temporal_expected_token_changed".into()); + } + if head != intent.expected && !(repair && head.as_ref() == Some(&target)) { + return Err("temporal_exact_pending_repair_required".into()); + } + } + if already && !repair && head.as_ref() == Some(&target) && meta.retired_at_ms.is_none() { + if self.temporal_cleanup_pending(dir, &c, meta)? { + return Err("temporal_cleanup_repair_required".into()); + } + let committed = self + .temporal_current(dir, now_ns)? + .ok_or("temporal_committed_readback")?; + if let (Some(a), Some(b)) = (committed.prefix, prefix) { + if a.bytes() != b.bytes() { + return Err("temporal_noop_conflict".into()); + } + } + return Ok(target); + } + if !already { + if let (Some(old), Some(p)) = (c.latest.as_ref(), prefix) { + if old.retired_at_ms.is_none() { + let previous = self.temporal_payload(dir, old, now_ns)?; + validate_extension(&previous, p)?; + } else if !is_new_empty_run(p)? { + return Err("temporal_new_run_not_empty".into()); + } + } + } + if !already { + if let Some(slot) = meta.slot { + if let Some((pending, _)) = + self.temporal_payload_any(&dir.join(format!("slot-{slot}.bin")), now_ns)? + { + let candidate = pending.previous_sha256.as_ref() + == meta.previous_sha256.as_ref() + && pending.generation == meta.generation + && pending.revision == meta.revision; + if candidate && pending != *meta { + return Err("temporal_conflicting_pending_payload".into()); + } + } + } + } + // A repair removes only known staging artifacts under the stable lock; its + // immutable SAME intended metadata/payload is re-established through barriers. + if repair { + for name in ["payload.stage", "head.stage", "witness.stage"] { + if self + .temporal_read(&dir.join(name), MAX_RECOVERY_BYTES + SMALL + 48)? + .is_some() + { + let matching = if name == "payload.stage" { + self.temporal_payload_any(&dir.join(name), now_ns)? + .is_some_and(|(m, _)| m == *meta) + } else { + self.temporal_meta( + &dir.join(name), + if name == "head.stage" { + "head" + } else { + "witness" + }, + )? + .as_ref() + == Some(meta) + }; + if !matching { + return Err("temporal_conflicting_pending_stage".into()); + } + fs::remove_file(dir.join(name)).map_err(|_| "temporal_repair_stage_cleanup")?; + } + } + File::open(dir) + .and_then(|f| f.sync_all()) + .map_err(|_| "temporal_repair_directory_fsync")?; + } + if let Some(p) = prefix { + let cipher = self.temporal_payload_seal(meta, p)?; + self.temporal_write_stage(dir, "payload.stage", &cipher, hook)?; + self.temporal_replace( + dir, + "payload.stage", + &format!("slot-{}.bin", meta.slot.ok_or("temporal_slot_missing")?), + hook, + )?; + hook("before:payload:readback")?; + let actual = self.temporal_payload(dir, meta, now_ns)?; + if actual.bytes() != p.bytes() { + return Err("temporal_payload_readback".into()); + } + hook("after:payload:readback")?; + } + let plain = serde_json::to_vec(meta).map_err(|_| "temporal_metadata_encoding")?; + if plain.len() > SMALL { + return Err("temporal_metadata_budget".into()); + } + let head_cipher = self.temporal_seal("head", &plain)?; + self.temporal_write_stage(dir, "head.stage", &head_cipher, hook)?; + self.temporal_replace(dir, "head.stage", HEAD, hook)?; + hook("before:head:readback")?; + if self.temporal_meta(&dir.join(HEAD), "head")?.as_ref() != Some(meta) { + return Err("temporal_head_readback".into()); + } + hook("after:head:readback")?; + let witness = dir.join(meta.witness()); + if let Some(existing) = self.temporal_meta(&witness, "witness")? { + if existing != *meta { + return Err("temporal_conflicting_committed_witness".into()); + } + private_existing_file(&witness, false)? + .sync_all() + .map_err(|_| "temporal_witness_fsync")?; + File::open(dir) + .and_then(|f| f.sync_all()) + .map_err(|_| "temporal_witness_directory_fsync")?; + } else { + let cipher = self.temporal_seal("witness", &plain)?; + self.temporal_write_stage(dir, "witness.stage", &cipher, hook)?; + hook("before:witness:install")?; + fs::hard_link(dir.join("witness.stage"), &witness) + .map_err(|_| "temporal_witness_install")?; + hook("after:witness:install")?; + File::open(dir) + .and_then(|f| f.sync_all()) + .map_err(|_| "temporal_witness_directory_fsync")?; + hook("after:witness:directory_fsync")?; + fs::remove_file(dir.join("witness.stage")) + .map_err(|_| "temporal_witness_stage_cleanup")?; + } + hook("before:committed:readback")?; + let loaded = self + .temporal_current(dir, now_ns)? + .ok_or("temporal_committed_readback")?; + if loaded.token != target { + return Err("temporal_committed_readback".into()); + } + hook("after:committed:readback")?; + hook("before:cleanup")?; + self.temporal_cleanup(dir, meta)?; + hook("after:cleanup")?; + Ok(target) + } +} + +#[cfg(test)] +#[path = "temporal_recovery_commitment_tests.rs"] +mod tests; + +fn private_existing_file(path: &Path, write: bool) -> Result { + let mut options = OpenOptions::new(); + options.read(true).write(write); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + #[cfg(target_os = "macos")] + options.custom_flags(0x100); + #[cfg(target_os = "linux")] + options.custom_flags(0x20000); + } + let file = options.open(path).map_err(|_| "temporal_private_open")?; + let m = file.metadata().map_err(|_| "temporal_private_metadata")?; + if !m.is_file() { + return Err("temporal_private_file_required".into()); + } + private_metadata(&m, false)?; + Ok(file) +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum AbortArtifact { + PayloadStage, + HeadStage, + WitnessStage, + InactiveSlot(u8), +} +impl AbortArtifact { + fn name(&self) -> String { + match self { + Self::PayloadStage => "payload.stage".into(), + Self::HeadStage => "head.stage".into(), + Self::WitnessStage => "witness.stage".into(), + Self::InactiveSlot(slot) => format!("slot-{slot}.bin"), + } + } +} +/// Private explicit-abort snapshot, never a caller-provided path or grant. +#[derive(Clone)] +pub(crate) struct TemporalAbortTicket { + expected: TemporalToken, + artifact: AbortArtifact, + digest: String, +} +impl RecoveryStore { + fn temporal_abortable( + &self, + dir: &Path, + c: &Components, + artifact: &AbortArtifact, + bytes: &[u8], + now_ns: u64, + ) -> Result { + let head = c.head.as_ref().ok_or("temporal_head_missing")?; + if head.retired_at_ms.is_some() { + return Ok(false); + } + if let AbortArtifact::InactiveSlot(slot) = artifact { + if *slot > 1 || Some(*slot) == head.slot { + return Ok(false); + } + } + let digest = hash(bytes); + // Both exact ciphertext copies/hardlinks and independently encrypted records + // naming a committed semantic identity are protected from ABORT. + if self + .temporal_read(&dir.join(HEAD), SMALL + 36)? + .is_some_and(|b| hash(&b) == digest) + { + return Ok(false); + } + for (name, m) in &c.witnesses { + if self + .temporal_read(&dir.join(name), SMALL + 36)? + .is_some_and(|b| hash(&b) == digest) + { + return Ok(false); + } + if let Some(slot) = m.slot { + let file = dir.join(format!("slot-{slot}.bin")); + if let Ok(Some((actual, _))) = self.temporal_payload_any(&file, now_ns) { + if actual.sha()? == m.sha()? + && self + .temporal_read(&file, MAX_RECOVERY_BYTES + SMALL + 48)? + .is_some_and(|b| hash(&b) == digest) + { + return Ok(false); + } + } + } + } + let identity = match artifact { + AbortArtifact::PayloadStage | AbortArtifact::InactiveSlot(_) => self + .temporal_payload_any(&dir.join(artifact.name()), now_ns) + .ok() + .flatten() + .map(|(m, _)| m), + AbortArtifact::HeadStage => self + .temporal_meta(&dir.join(artifact.name()), "head") + .ok() + .flatten(), + AbortArtifact::WitnessStage => self + .temporal_meta(&dir.join(artifact.name()), "witness") + .ok() + .flatten(), + }; + if let Some(m) = identity { + if c.witnesses + .iter() + .any(|(_, w)| w.sha().ok() == m.sha().ok()) + { + return Ok(false); + } + if matches!(artifact, AbortArtifact::InactiveSlot(_)) + && (m.previous_sha256.as_ref() != Some(&head.sha()?) + || m.generation != head.generation + || m.revision != head.revision + 1) + { + return Ok(false); + } + } else if matches!(artifact, AbortArtifact::InactiveSlot(_)) { + return Ok(false); + } + Ok(true) + } + pub(crate) fn inspect_abortable_temporal( + &self, + expected: &TemporalToken, + now_ns: u64, + ) -> Result, String> { + let _serial = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + let locked = self.temporal_lock(false)?; + let loaded = self + .temporal_current(&locked.directory, now_ns)? + .ok_or("temporal_committed_prefix_missing")?; + if loaded.token != *expected || loaded.prefix.is_none() { + return Err("temporal_abort_expected_changed".into()); + } + let c = self.temporal_components(&locked.directory)?; + let slot = c + .head + .as_ref() + .and_then(|m| m.slot) + .ok_or("temporal_committed_slot_missing")?; + let mut out = Vec::new(); + for artifact in [ + AbortArtifact::PayloadStage, + AbortArtifact::HeadStage, + AbortArtifact::WitnessStage, + AbortArtifact::InactiveSlot(1 - slot), + ] { + if let Some(bytes) = self.temporal_read( + &locked.directory.join(artifact.name()), + MAX_RECOVERY_BYTES + SMALL + 48, + )? { + if self.temporal_abortable(&locked.directory, &c, &artifact, &bytes, now_ns)? { + out.push(TemporalAbortTicket { + expected: expected.clone(), + artifact, + digest: hash(&bytes), + }); + } + } + } + Ok(out) + } + pub(crate) fn abort_temporal_pending( + &self, + ticket: &TemporalAbortTicket, + now_ns: u64, + ) -> Result { + let _serial = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + let locked = self.temporal_lock(false)?; + let before = self + .temporal_current(&locked.directory, now_ns)? + .ok_or("temporal_committed_prefix_missing")?; + if before.token != ticket.expected || before.prefix.is_none() { + return Err("temporal_abort_expected_changed".into()); + } + let c = self.temporal_components(&locked.directory)?; + let path = locked.directory.join(ticket.artifact.name()); + if let Some(bytes) = self.temporal_read(&path, MAX_RECOVERY_BYTES + SMALL + 48)? { + if hash(&bytes) != ticket.digest { + return Err("temporal_abort_artifact_changed".into()); + } + if !self.temporal_abortable(&locked.directory, &c, &ticket.artifact, &bytes, now_ns)? { + return Err("temporal_abort_committed_reference".into()); + } + fs::remove_file(&path).map_err(|_| "temporal_abort_remove")?; + } + File::open(&locked.directory) + .and_then(|f| f.sync_all()) + .map_err(|_| "temporal_abort_directory_fsync")?; + let after = self + .temporal_current(&locked.directory, now_ns)? + .ok_or("temporal_abort_committed_readback")?; + if after.token != before.token + || after.prefix.as_ref().map(|p| p.bytes()) != before.prefix.as_ref().map(|p| p.bytes()) + || path.exists() + { + return Err("temporal_abort_committed_changed".into()); + } + Ok(after) + } + #[cfg(test)] + pub(crate) fn synthetic_partial_write( + &self, + intent: &TemporalIntent, + prefix: &ValidatedTemporalPrefix, + now_ns: u64, + ) -> Result { + let path = self.temporal_dir().join("payload.stage"); + self.publish_temporal_with(intent, Some(prefix), now_ns, false, &mut |point| { + if point == "after:payload.stage:write" { + let mut bytes = fs::read(&path).unwrap(); + bytes.truncate(bytes.len() / 2); + fs::write(&path, bytes).unwrap(); + Err("injected-partial-native-prefix-write".into()) + } else { + Ok(()) + } + }) + } +} +impl TemporalAbortTicket { + pub(crate) fn expected_token(&self) -> &TemporalToken { + &self.expected + } +} + +/// Metadata-only erasure capability: deliberately not a TemporalToken accepted +/// by active commit/resume. Neither expired sample bytes nor fit authority escape. +pub(crate) struct TemporalRetirementInspection { + descriptor: RecoveryDescriptor, + run: String, + observed_ns: u64, + expired: bool, + token: TemporalToken, +} +impl TemporalRetirementInspection { + pub(crate) fn descriptor(&self) -> &RecoveryDescriptor { + &self.descriptor + } + pub(crate) fn expired(&self) -> bool { + self.expired + } + pub(crate) fn run(&self) -> &str { + &self.run + } +} +/// Only explicit recertification constructs this durability-bearing result. +pub(crate) struct TemporalRestored { + pub(crate) token: TemporalToken, + pub(crate) prefix: ValidatedTemporalPrefix, +} +impl RecoveryStore { + pub(crate) fn inspect_temporal_retirement( + &self, + now_ns: u64, + ) -> Result, String> { + let _serial = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + let locked = self.temporal_lock(false)?; + let c = self.temporal_components(&locked.directory)?; + match (c.head, c.latest) { + (None, None) => Ok(None), + (Some(head), Some(latest)) if head == latest => { + if now_ns < head.receiver_ns { + return Err("recovery_receiver_clock_regression".into()); + } + Ok(Some(TemporalRetirementInspection { + descriptor: head.descriptor.clone(), + run: head.run.clone(), + observed_ns: now_ns, + expired: now_ns / 1_000_000 >= head.descriptor.expires_at_ms, + token: head.token()?, + })) + } + _ => Err("temporal_head_witness_mismatch_or_pending".into()), + } + } + pub(crate) fn prepare_inspected_retirement( + &self, + inspection: &TemporalRetirementInspection, + now_ns: u64, + expiry_only: bool, + ) -> Result { + if now_ns < inspection.observed_ns { + return Err("recovery_receiver_clock_regression".into()); + } + self.prepare_retire_temporal(&inspection.token, now_ns, expiry_only) + } + pub(crate) fn recertify_temporal( + &self, + expected: &TemporalToken, + now_ns: u64, + ) -> Result { + self.recertify_temporal_with(expected, now_ns, &mut |_| Ok(())) + } + fn recertify_temporal_with( + &self, + expected: &TemporalToken, + now_ns: u64, + hook: &mut impl FnMut(&str) -> Result<(), String>, + ) -> Result { + let _serial = self + .serial + .lock() + .map_err(|_| "recovery_store_unavailable")?; + let locked = self.temporal_lock(false)?; + let before = self + .temporal_current(&locked.directory, now_ns)? + .ok_or("temporal_committed_prefix_missing")?; + if before.token != *expected || before.prefix.is_none() { + return Err("temporal_restore_expected_changed".into()); + } + let components = self.temporal_components(&locked.directory)?; + let head = components.head.as_ref().ok_or("temporal_head_missing")?; + let mut names = vec![ + HEAD.to_string(), + format!("slot-{}.bin", head.slot.ok_or("temporal_retired")?), + ]; + names.extend( + components + .witnesses + .iter() + .filter(|(_, m)| m.generation == head.generation) + .map(|(n, _)| n.clone()), + ); + for name in names { + hook("before:restore:file_fsync")?; + private_existing_file(&locked.directory.join(name), false)? + .sync_all() + .map_err(|_| "temporal_restore_file_fsync")?; + hook("after:restore:file_fsync")?; + } + hook("before:restore:directory_fsync")?; + File::open(&locked.directory) + .and_then(|f| f.sync_all()) + .map_err(|_| "temporal_restore_directory_fsync")?; + File::open(&self.directory) + .and_then(|f| f.sync_all()) + .map_err(|_| "temporal_restore_parent_fsync")?; + hook("after:restore:directory_fsync")?; + hook("before:restore:readback")?; + let after = self + .temporal_current(&locked.directory, now_ns)? + .ok_or("temporal_restore_readback")?; + if after.token != before.token + || after.prefix.as_ref().map(|p| p.bytes()) != before.prefix.as_ref().map(|p| p.bytes()) + { + return Err("temporal_restore_readback_changed".into()); + } + hook("after:restore:readback")?; + Ok(TemporalRestored { + token: after.token, + prefix: after.prefix.ok_or("temporal_retired")?, + }) + } +} + +impl RecoveryStore { + #[cfg(test)] + pub(crate) fn synthetic_lost_witness_ack( + &self, + intent: &TemporalIntent, + prefix: &ValidatedTemporalPrefix, + now_ns: u64, + ) -> Result { + self.publish_temporal_with(intent, Some(prefix), now_ns, false, &mut |point| { + if point == "after:witness:install" { + Err("injected-lost-witness-directory-ack".into()) + } else { + Ok(()) + } + }) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment_tests.rs new file mode 100644 index 0000000000..2161e7a4cf --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment_tests.rs @@ -0,0 +1,987 @@ +use super::*; +use crate::room_learner::temporal_model_input::managed::execution::checkpoint::{ + synthetic_changed_old_row, synthetic_segment_fork, synthetic_storage_prefixes, +}; +fn setup() -> ( + tempfile::TempDir, + RecoveryStore, + Vec, +) { + static PREFIXES: std::sync::OnceLock> = std::sync::OnceLock::new(); + let p = PREFIXES.get_or_init(synthetic_storage_prefixes).clone(); + let d = p[0].descriptor(); + let t = tempfile::tempdir().unwrap(); + let store = RecoveryStore::new( + &t.path().join(&d.owner_scope).join("SensingServer"), + &d.owner_scope, + &d.installation_id, + Some(&[7; 32]), + ) + .unwrap(); + (t, store, p) +} +fn now(p: &ValidatedTemporalPrefix) -> u64 { + p.saved_receiver_ns() +} +fn commit( + store: &RecoveryStore, + expected: Option<&TemporalToken>, + p: &ValidatedTemporalPrefix, +) -> TemporalToken { + let i = store.prepare_temporal(expected, p, now(p)).unwrap(); + store.publish_temporal(&i, Some(p), now(p)).unwrap() +} +#[test] +fn actual_aead_prefix_commits_two_slots_and_preserves_v1() { + let (_t, store, p) = setup(); + store + .save(p[0].descriptor(), p[0].bytes(), now(&p[0]) / 1_000_000) + .unwrap(); + let v1 = fs::read(store.directory.join(FILE)).unwrap(); + let mut token = None; + for value in &p { + let t = commit(&store, token.as_ref(), value); + let loaded = store.load_temporal(now(value)).unwrap().unwrap(); + assert_eq!(loaded.token, t); + assert_eq!(loaded.prefix.unwrap().bytes(), value.bytes()); + token = Some(t); + } + assert_eq!(fs::read(store.directory.join(FILE)).unwrap(), v1); + let files = fs::read_dir(store.temporal_dir()) + .unwrap() + .map(|e| e.unwrap().file_name().to_string_lossy().into_owned()) + .collect::>(); + assert_eq!(files.iter().filter(|n| n.starts_with("slot-")).count(), 2); + assert_eq!( + files.iter().filter(|n| n.starts_with("witness-")).count(), + p.len() + ); +} +#[test] +fn selective_head_payload_witness_replacement_fails_closed() { + for which in ["head", "payload", "witness"] { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let dir = store.temporal_dir(); + let old_head = fs::read(dir.join(HEAD)).unwrap(); + let old_payload = fs::read(dir.join("slot-0.bin")).unwrap(); + let old_witness = fs::read(dir.join(format!( + "witness-{:016x}-{:04x}.bin", + t0.generation, t0.revision + ))) + .unwrap(); + let t1 = commit(&store, Some(&t0), &p[1]); + match which { + "head" => fs::write(dir.join(HEAD), old_head).unwrap(), + "payload" => fs::write(dir.join("slot-1.bin"), old_payload).unwrap(), + _ => fs::write( + dir.join(format!( + "witness-{:016x}-{:04x}.bin", + t1.generation, t1.revision + )), + old_witness, + ) + .unwrap(), + }; + assert!(store.load_temporal(now(&p[1])).is_err(), "{which}"); + } +} +#[test] +fn stale_token_changed_old_row_and_same_revision_conflict_reject() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let a = store + .prepare_temporal(Some(&t0), &p[1], now(&p[1])) + .unwrap(); + let fork = synthetic_segment_fork(&p[0]); + let b = store + .prepare_temporal(Some(&t0), &fork, now(&fork)) + .unwrap(); + let t1 = store.publish_temporal(&a, Some(&p[1]), now(&p[1])).unwrap(); + assert!(store.publish_temporal(&b, Some(&fork), now(&fork)).is_err()); + assert!(store + .prepare_temporal(Some(&t0), &p[2], now(&p[2])) + .is_err()); + let t2 = commit(&store, Some(&t1), &p[2]); + let changed = synthetic_changed_old_row(&p[3]); + assert!(store + .prepare_temporal(Some(&t2), &changed, now(&changed)) + .is_err()); + let same = store + .prepare_temporal(Some(&t2), &p[2], now(&p[2])) + .unwrap(); + assert_eq!( + store + .publish_temporal(&same, Some(&p[2]), now(&p[2])) + .unwrap(), + t2 + ); +} +#[test] +fn known_pending_same_revision_repair_never_promotes_other_intent() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let intent = store + .prepare_temporal(Some(&t0), &p[1], now(&p[1])) + .unwrap(); + let mut hook = |point: &str| { + if point == "after:head.stage:directory_fsync" { + Err("injected-head-ambiguous".into()) + } else { + Ok(()) + } + }; + assert!(store + .publish_temporal_with(&intent, Some(&p[1]), now(&p[1]), false, &mut hook) + .is_err()); + assert!(store.load_temporal(now(&p[1])).is_err()); + let token = store + .repair_same_temporal(&intent, Some(&p[1]), now(&p[1])) + .unwrap(); + assert_eq!(token.revision, 1); + assert_eq!( + store + .load_temporal(now(&p[1])) + .unwrap() + .unwrap() + .prefix + .unwrap() + .bytes(), + p[1].bytes() + ); +} + +#[test] +fn fault_every_barrier_preserves_known_commit_or_blocks_then_exact_repair() { + let (_t, store, p) = setup(); + let token = commit(&store, None, &p[0]); + let intent = store + .prepare_temporal(Some(&token), &p[1], now(&p[1])) + .unwrap(); + let mut points = Vec::new(); + store + .publish_temporal_with(&intent, Some(&p[1]), now(&p[1]), false, &mut |s| { + points.push(s.to_owned()); + Ok(()) + }) + .unwrap(); + for point in points { + let (_t, store, p) = setup(); + let token = commit(&store, None, &p[0]); + let intent = store + .prepare_temporal(Some(&token), &p[1], now(&p[1])) + .unwrap(); + let mut hook = |s: &str| { + if s == point { + Err(format!("injected:{s}")) + } else { + Ok(()) + } + }; + assert!( + store + .publish_temporal_with(&intent, Some(&p[1]), now(&p[1]), false, &mut hook) + .is_err(), + "{point}" + ); + if let Ok(Some(loaded)) = store.load_temporal(now(&p[1])) { + let b = loaded.prefix.unwrap(); + assert!( + b.bytes() == p[0].bytes() || b.bytes() == p[1].bytes(), + "{point}" + ); + } + let repaired = store + .repair_same_temporal(&intent, Some(&p[1]), now(&p[1])) + .unwrap_or_else(|e| panic!("repair {point}: {e}")); + assert_eq!(repaired.revision, 1); + assert_eq!( + store + .load_temporal(now(&p[1])) + .unwrap() + .unwrap() + .prefix + .unwrap() + .bytes(), + p[1].bytes() + ); + assert!(!store.temporal_dir().join("payload.stage").exists()); + } +} +#[test] +fn simultaneous_independent_handles_allow_only_one_expected_revision() { + let (t, store, p) = setup(); + let token = commit(&store, None, &p[0]); + let a = store + .prepare_temporal(Some(&token), &p[1], now(&p[1])) + .unwrap(); + let fork = synthetic_segment_fork(&p[0]); + let b = store + .prepare_temporal(Some(&token), &fork, now(&fork)) + .unwrap(); + let d = p[0].descriptor(); + let data = t.path().join(&d.owner_scope).join("SensingServer"); + let other = + RecoveryStore::new(&data, &d.owner_scope, &d.installation_id, Some(&[7; 32])).unwrap(); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let c = barrier.clone(); + let p1 = p[1].clone(); + let first = std::thread::spawn(move || { + c.wait(); + store.publish_temporal(&a, Some(&p1), now(&p1)) + }); + let second = std::thread::spawn(move || { + barrier.wait(); + other.publish_temporal(&b, Some(&fork), now(&fork)) + }); + let outcomes = [ + first.join().unwrap().is_ok(), + second.join().unwrap().is_ok(), + ]; + assert_eq!(outcomes.into_iter().filter(|x| *x).count(), 1); +} +#[test] +fn discard_expiry_and_explicit_new_generation_keep_stale_tokens_fenced() { + use crate::room_learner::temporal_model_input::managed::execution::checkpoint::synthetic_new_empty_run; + let (_t, store, p) = setup(); + let token = commit(&store, None, &p[0]); + let lock = fs::metadata(store.directory.join(LOCK)).unwrap(); + assert!(store + .load_temporal(p[0].descriptor().expires_at_ms * 1_000_000) + .is_err()); + let retired = store + .prepare_retire_temporal(&token, p[0].descriptor().expires_at_ms * 1_000_000, true) + .unwrap(); + let tombstone = store + .publish_temporal(&retired, None, p[0].descriptor().expires_at_ms * 1_000_000) + .unwrap(); + assert!(store + .load_temporal(p[0].descriptor().expires_at_ms * 1_000_000) + .unwrap() + .unwrap() + .prefix + .is_none()); + assert!(!store.temporal_dir().join("slot-0.bin").exists()); + assert!(store + .prepare_temporal(Some(&token), &p[1], now(&p[1])) + .is_err()); + let fresh = synthetic_new_empty_run(&p[0], p[0].descriptor().expires_at_ms + 1); + assert!(store.prepare_temporal(None, &fresh, now(&fresh)).is_err()); + let next = store + .prepare_temporal(Some(&tombstone), &fresh, now(&fresh)) + .unwrap(); + let next_token = store + .publish_temporal(&next, Some(&fresh), now(&fresh)) + .unwrap(); + assert_eq!(next_token.generation, tombstone.generation + 1); + assert_eq!(next_token.revision, 0); + assert!(store + .prepare_temporal(Some(&tombstone), &fresh, now(&fresh)) + .is_err()); + assert_eq!( + fs::read_dir(store.temporal_dir()) + .unwrap() + .filter(|e| e + .as_ref() + .unwrap() + .file_name() + .to_string_lossy() + .starts_with("witness-")) + .count(), + 1 + ); + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + assert_eq!( + lock.ino(), + fs::metadata(store.directory.join(LOCK)).unwrap().ino() + ); + } +} +#[test] +fn coordinated_head_and_witness_rollback_is_documented_limit_not_safety_success() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let dir = store.temporal_dir(); + let old_head = fs::read(dir.join(HEAD)).unwrap(); + let t1 = commit(&store, Some(&t0), &p[1]); + fs::write(dir.join(HEAD), old_head).unwrap(); + fs::remove_file(dir.join(format!( + "witness-{:016x}-{:04x}.bin", + t1.generation, t1.revision + ))) + .unwrap(); + let old = store.load_temporal(now(&p[1])).unwrap().unwrap(); + assert_eq!( + old.token, t0, + "deliberately demonstrates excluded coordinated rollback scope" + ); +} +#[test] +fn clock_regression_unknown_entries_and_metadata_capacity_fail_closed() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + assert!(store.load_temporal(now(&p[0]) - 1).is_err()); + assert!(store + .prepare_temporal(Some(&t0), &p[1], now(&p[1]) - 1) + .is_err()); + assert_eq!(p[0].transition_limit(), 329); + assert!(p[0].transition_limit() <= MAX_TRANSITIONS); + let keep = store.temporal_dir().join("unrelated.db"); + fs::write(&keep, b"preserve").unwrap(); + assert!(store.load_temporal(now(&p[0])).is_err()); + assert!(keep.exists()); +} +#[test] +fn os_lock_is_shared_with_an_actual_child_process() { + let (t, store, p) = setup(); + let locked = store.temporal_lock(true).unwrap(); + let marker = t.path().join("child-acquired"); + let ready = t.path().join("child-ready"); + let d = p[0].descriptor(); + let child = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "room_learner::recovery_store::temporal_commitment::tests::os_lock_child_probe", + "--nocapture", + ]) + .env( + "RUVIEW_SYNTH_LOCK_DATA", + t.path().join(&d.owner_scope).join("SensingServer"), + ) + .env("RUVIEW_SYNTH_LOCK_OWNER", &d.owner_scope) + .env("RUVIEW_SYNTH_LOCK_INSTALL", &d.installation_id) + .env("RUVIEW_SYNTH_LOCK_READY", &ready) + .env("RUVIEW_SYNTH_LOCK_ACQUIRED", &marker) + .spawn() + .unwrap(); + let started = std::time::Instant::now(); + while !ready.exists() && started.elapsed() < std::time::Duration::from_secs(5) { + std::thread::yield_now(); + } + assert!(ready.exists()); + assert!( + !marker.exists(), + "child cannot acquire while parent holds same stable inode" + ); + drop(locked); + let output = child.wait_with_output().unwrap(); + assert!(output.status.success()); + assert!(marker.exists()); +} +#[test] +fn os_lock_child_probe() { + let Some(data) = std::env::var_os("RUVIEW_SYNTH_LOCK_DATA") else { + return; + }; + let owner = std::env::var("RUVIEW_SYNTH_LOCK_OWNER").unwrap(); + let installation = std::env::var("RUVIEW_SYNTH_LOCK_INSTALL").unwrap(); + let store = + RecoveryStore::new(Path::new(&data), &owner, &installation, Some(&[7; 32])).unwrap(); + fs::write( + std::env::var_os("RUVIEW_SYNTH_LOCK_READY").unwrap(), + b"ready", + ) + .unwrap(); + let _lock = store.temporal_lock(true).unwrap(); + fs::write( + std::env::var_os("RUVIEW_SYNTH_LOCK_ACQUIRED").unwrap(), + b"acquired", + ) + .unwrap(); +} + +#[test] +fn original_native_receiver_nanoseconds_are_never_rounded_for_cold_validation() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let t1 = commit(&store, Some(&t0), &p[1]); + let _t2 = commit(&store, Some(&t1), &p[2]); + let exact = p[2].saved_receiver_ns(); + let truncated = p[2].descriptor().saved_at_ms * 1_000_000; + assert!(exact > truncated); + assert!(store.load_temporal(truncated).is_err()); + let cold = store.load_temporal(exact).unwrap().unwrap().prefix.unwrap(); + assert_eq!(cold.saved_receiver_ns(), exact); + assert_eq!(cold.bytes(), p[2].bytes()); +} + +#[test] +fn valid_looking_lower_background_floor_cannot_extend_committed_prefix() { + use crate::room_learner::temporal_model_input::managed::execution::checkpoint::synthetic_floor_regression; + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let t1 = commit(&store, Some(&t0), &p[1]); + let t2 = commit(&store, Some(&t1), &p[2]); + let regressed = synthetic_floor_regression(&p[2]); + let err = store + .prepare_temporal(Some(&t2), ®ressed, now(®ressed)) + .err() + .expect("floor must not regress despite internal prefix validity"); + assert_eq!(err, "temporal_commit_floor_regressed"); +} +#[test] +fn conflicting_pending_payload_cannot_be_overwritten_by_other_same_revision_repair() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let a = store + .prepare_temporal(Some(&t0), &p[1], now(&p[1])) + .unwrap(); + let fork = synthetic_segment_fork(&p[0]); + let b = store + .prepare_temporal(Some(&t0), &fork, now(&fork)) + .unwrap(); + let mut hook = |s: &str| { + if s == "after:payload.stage:directory_fsync" { + Err("injected-before-head".into()) + } else { + Ok(()) + } + }; + assert!(store + .publish_temporal_with(&a, Some(&p[1]), now(&p[1]), false, &mut hook) + .is_err()); + let before = fs::read(store.temporal_dir().join("slot-1.bin")).unwrap(); + assert!(store + .repair_same_temporal(&b, Some(&fork), now(&fork)) + .is_err()); + assert_eq!( + fs::read(store.temporal_dir().join("slot-1.bin")).unwrap(), + before + ); + assert!(store + .repair_same_temporal(&a, Some(&p[1]), now(&p[1])) + .is_ok()); +} +#[test] +fn incomplete_unauthenticated_stage_blocks_repair_but_keeps_prior_commit_loadable() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let intent = store + .prepare_temporal(Some(&t0), &p[1], now(&p[1])) + .unwrap(); + let file = store.temporal_dir().join("payload.stage"); + let mut hook = |s: &str| { + if s == "after:payload.stage:write" { + let mut b = fs::read(&file).unwrap(); + b.truncate(b.len() / 2); + fs::write(&file, b).unwrap(); + Err("injected-partial-storage-write".into()) + } else { + Ok(()) + } + }; + assert!(store + .publish_temporal_with(&intent, Some(&p[1]), now(&p[1]), false, &mut hook) + .is_err()); + let old = store.load_temporal(now(&p[1])).unwrap().unwrap(); + assert_eq!(old.token, t0); + assert!( + store + .repair_same_temporal(&intent, Some(&p[1]), now(&p[1])) + .is_err(), + "unverifiable stage cannot be relabeled as same intent" + ); + let retire = store + .prepare_retire_temporal(&t0, now(&p[1]), false) + .unwrap(); + store.publish_temporal(&retire, None, now(&p[1])).unwrap(); + assert!(!file.exists()); +} + +#[test] +fn missing_head_or_witness_never_reinitializes_from_stale_ram() { + for target in ["head", "witness"] { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let path = if target == "head" { + store.temporal_dir().join(HEAD) + } else { + store.temporal_dir().join(format!( + "witness-{:016x}-{:04x}.bin", + t0.generation, t0.revision + )) + }; + fs::remove_file(path).unwrap(); + assert!(store.load_temporal(now(&p[0])).is_err()); + assert!(store.prepare_temporal(None, &p[0], now(&p[0])).is_err()); + assert!(store + .prepare_temporal(Some(&t0), &p[1], now(&p[1])) + .is_err()); + } +} +#[test] +fn no_op_commit_does_not_rewrite_payload_or_consume_revision() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let snapshot = |store: &RecoveryStore| { + let mut out = std::collections::BTreeMap::new(); + for e in fs::read_dir(store.temporal_dir()).unwrap() { + let e = e.unwrap(); + out.insert(e.file_name(), fs::read(e.path()).unwrap()); + } + out + }; + let before = snapshot(&store); + let same = store + .prepare_temporal(Some(&t0), &p[0], now(&p[0])) + .unwrap(); + assert_eq!( + store + .publish_temporal(&same, Some(&p[0]), now(&p[0])) + .unwrap(), + t0 + ); + assert_eq!(snapshot(&store), before); +} + +#[test] +fn cleanup_ambiguity_cannot_be_skipped_by_new_generation_or_noop_ack() { + use crate::room_learner::temporal_model_input::managed::execution::checkpoint::synthetic_new_empty_run; + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let retirement = store + .prepare_retire_temporal(&t0, now(&p[1]), false) + .unwrap(); + let mut hook = |point: &str| { + if point == "before:cleanup" { + Err("injected-cleanup-ambiguity".into()) + } else { + Ok(()) + } + }; + assert!(store + .publish_temporal_with(&retirement, None, now(&p[1]), false, &mut hook) + .is_err()); + assert!(store.temporal_dir().join("slot-0.bin").exists()); + let retired = store.load_temporal(now(&p[1])).unwrap().unwrap().token; + let fresh = synthetic_new_empty_run(&p[0], now(&p[1]) / 1_000_000 + 1); + assert!(store + .prepare_temporal(Some(&retired), &fresh, now(&fresh)) + .is_err()); + store + .repair_same_temporal(&retirement, None, now(&p[1])) + .unwrap(); + let new = store + .prepare_temporal(Some(&retired), &fresh, now(&fresh)) + .unwrap(); + let mut hook = |point: &str| { + if point == "before:cleanup" { + Err("injected-new-generation-cleanup".into()) + } else { + Ok(()) + } + }; + assert!(store + .publish_temporal_with(&new, Some(&fresh), now(&fresh), false, &mut hook) + .is_err()); + let loaded = store.load_temporal(now(&fresh)).unwrap().unwrap(); + let noop = store + .prepare_temporal(Some(&loaded.token), &fresh, now(&fresh)) + .unwrap(); + assert!(store + .publish_temporal(&noop, Some(&fresh), now(&fresh)) + .is_err()); + store + .repair_same_temporal(&new, Some(&fresh), now(&fresh)) + .unwrap(); + assert_eq!( + fs::read_dir(store.temporal_dir()) + .unwrap() + .filter(|e| e + .as_ref() + .unwrap() + .file_name() + .to_string_lossy() + .starts_with("witness-")) + .count(), + 1 + ); +} + +#[test] +fn explicit_abort_only_uncommitted_stage_preserves_exact_prefix_token_and_expiry() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let intent = store + .prepare_temporal(Some(&t0), &p[1], now(&p[1])) + .unwrap(); + assert!(store + .synthetic_partial_write(&intent, &p[1], now(&p[1])) + .is_err()); + let tickets = store.inspect_abortable_temporal(&t0, now(&p[1])).unwrap(); + let ticket = tickets + .iter() + .find(|t| t.artifact == AbortArtifact::PayloadStage) + .unwrap(); + let returned = store.abort_temporal_pending(ticket, now(&p[1])).unwrap(); + assert_eq!(returned.token, t0); + let restored = returned.prefix.unwrap(); + assert_eq!(restored.bytes(), p[0].bytes()); + assert_eq!( + restored.descriptor().expires_at_ms, + p[0].descriptor().expires_at_ms + ); + assert!(!store.temporal_dir().join("payload.stage").exists()); + let next = commit(&store, Some(&t0), &p[1]); + assert_eq!(next.revision, t0.revision + 1); +} +#[test] +fn abort_rejects_stale_digest_token_and_any_committed_reference() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let intent = store + .prepare_temporal(Some(&t0), &p[1], now(&p[1])) + .unwrap(); + assert!(store + .synthetic_partial_write(&intent, &p[1], now(&p[1])) + .is_err()); + let ticket = store + .inspect_abortable_temporal(&t0, now(&p[1])) + .unwrap() + .into_iter() + .find(|t| t.artifact == AbortArtifact::PayloadStage) + .unwrap(); + let file = store.temporal_dir().join("payload.stage"); + let mut changed = fs::read(&file).unwrap(); + changed.push(0); + fs::write(&file, &changed).unwrap(); + assert!(store.abort_temporal_pending(&ticket, now(&p[1])).is_err()); + assert_eq!(fs::read(&file).unwrap(), changed); + let fresh = store + .inspect_abortable_temporal(&t0, now(&p[1])) + .unwrap() + .into_iter() + .find(|t| t.artifact == AbortArtifact::PayloadStage) + .unwrap(); + store.abort_temporal_pending(&fresh, now(&p[1])).unwrap(); + let t1 = commit(&store, Some(&t0), &p[1]); + assert!(store.abort_temporal_pending(&fresh, now(&p[1])).is_err()); + let dir = store.temporal_dir(); + fs::copy(dir.join("slot-1.bin"), dir.join("payload.stage")).unwrap(); + assert!(!store + .inspect_abortable_temporal(&t1, now(&p[1])) + .unwrap() + .iter() + .any(|t| t.artifact == AbortArtifact::PayloadStage)); + let forged = TemporalAbortTicket { + expected: t1, + artifact: AbortArtifact::InactiveSlot(1), + digest: hash(&fs::read(dir.join("slot-1.bin")).unwrap()), + }; + assert!(store.abort_temporal_pending(&forged, now(&p[1])).is_err()); +} +#[test] +fn post_head_ambiguity_cannot_abort_to_a_guessed_prior_revision() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let intent = store + .prepare_temporal(Some(&t0), &p[1], now(&p[1])) + .unwrap(); + let mut hook = |s: &str| { + if s == "after:head.stage:directory_fsync" { + Err("injected-pending-head".into()) + } else { + Ok(()) + } + }; + assert!(store + .publish_temporal_with(&intent, Some(&p[1]), now(&p[1]), false, &mut hook) + .is_err()); + assert!(store.inspect_abortable_temporal(&t0, now(&p[1])).is_err()); + assert!(store + .repair_same_temporal(&intent, Some(&p[1]), now(&p[1])) + .is_ok()); +} + +#[test] +fn independent_interrupted_old_generation_gc_is_same_intent_repairable() { + use crate::room_learner::temporal_model_input::managed::execution::checkpoint::synthetic_new_empty_run; + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let retirement = store + .prepare_retire_temporal(&t0, now(&p[1]), false) + .unwrap(); + let retired = store + .publish_temporal(&retirement, None, now(&p[1])) + .unwrap(); + let fresh = synthetic_new_empty_run(&p[0], now(&p[1]) / 1_000_000 + 1); + let new = store + .prepare_temporal(Some(&retired), &fresh, now(&fresh)) + .unwrap(); + let mut hook = |point: &str| { + if point == "before:cleanup" { + Err("stop-before-gc".into()) + } else { + Ok(()) + } + }; + assert!(store + .publish_temporal_with(&new, Some(&fresh), now(&fresh), false, &mut hook) + .is_err()); + assert_eq!( + store + .load_temporal(now(&fresh)) + .unwrap() + .unwrap() + .prefix + .unwrap() + .bytes(), + fresh.bytes() + ); + fs::remove_file(store.temporal_dir().join(format!( + "witness-{:016x}-{:04x}.bin", + t0.generation, t0.revision + ))) + .unwrap(); + let repaired = store.repair_same_temporal(&new, Some(&fresh), now(&fresh)); + assert!( + repaired.is_ok(), + "same intent repair after one completed GC unlink: {:?}", + repaired.err() + ); +} + +#[test] +fn each_completed_old_gc_unlink_keeps_current_prefix_loadable_and_repairable() { + use crate::room_learner::temporal_model_input::managed::execution::checkpoint::synthetic_new_empty_run; + for cut in 0..=5 { + let (_t, store, p) = setup(); + let mut token = commit(&store, None, &p[0]); + for value in &p[1..] { + token = commit(&store, Some(&token), value); + } + let retired_intent = store + .prepare_retire_temporal(&token, now(&p[3]), false) + .unwrap(); + let retired = store + .publish_temporal(&retired_intent, None, now(&p[3])) + .unwrap(); + let original = synthetic_new_empty_run(&p[0], now(&p[3]) / 1_000_000 + 1); + let intent = store + .prepare_temporal(Some(&retired), &original, now(&original)) + .unwrap(); + let mut hook = |point: &str| { + if point == "before:cleanup" { + Err("injected-before-obsolete-unlinks".into()) + } else { + Ok(()) + } + }; + assert!(store + .publish_temporal_with(&intent, Some(&original), now(&original), false, &mut hook) + .is_err()); + let components = store.temporal_components(&store.temporal_dir()).unwrap(); + let old = components + .witnesses + .iter() + .filter(|(_, m)| m.generation == retired.generation) + .map(|(n, _)| n.clone()) + .collect::>(); + assert_eq!(old.len(), 5); + for name in old.iter().take(cut) { + fs::remove_file(store.temporal_dir().join(name)).unwrap(); + } + let current = store.load_temporal(now(&original)).unwrap().unwrap(); + assert_eq!( + current.prefix.as_ref().unwrap().bytes(), + original.bytes(), + "cut{cut}" + ); + let repaired = store + .repair_same_temporal(&intent, Some(&original), now(&original)) + .unwrap(); + assert_eq!(repaired, current.token); + let loaded = store + .load_temporal(now(&original)) + .unwrap() + .unwrap() + .prefix + .unwrap(); + assert_eq!(loaded.descriptor(), original.descriptor()); + assert_eq!(loaded.bytes(), original.bytes()); + assert_eq!( + store + .temporal_components(&store.temporal_dir()) + .unwrap() + .witnesses + .len(), + 1 + ); + } +} +#[test] +fn obsolete_gc_fragments_never_authorize_old_head_or_incomplete_current_chain() { + use crate::room_learner::temporal_model_input::managed::execution::checkpoint::synthetic_new_empty_run; + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let retire = store + .prepare_retire_temporal(&t0, now(&p[1]), false) + .unwrap(); + let retired = store.publish_temporal(&retire, None, now(&p[1])).unwrap(); + let old_head = fs::read(store.temporal_dir().join(HEAD)).unwrap(); + let fresh = synthetic_new_empty_run(&p[0], now(&p[1]) / 1_000_000 + 1); + let intent = store + .prepare_temporal(Some(&retired), &fresh, now(&fresh)) + .unwrap(); + assert!(store + .publish_temporal_with(&intent, Some(&fresh), now(&fresh), false, &mut |s| if s + == "before:cleanup" + { + Err("pause-gc".into()) + } else { + Ok(()) + }) + .is_err()); + fs::remove_file( + store + .temporal_dir() + .join(format!("witness-{:016x}-{:04x}.bin", t0.generation, 0)), + ) + .unwrap(); + fs::write(store.temporal_dir().join(HEAD), old_head).unwrap(); + assert!(store.load_temporal(now(&fresh)).is_err()); + assert!(store.inspect_temporal_retirement(now(&fresh)).is_err()); + store + .repair_same_temporal(&intent, Some(&fresh), now(&fresh)) + .unwrap(); + let current = store.load_temporal(now(&fresh)).unwrap().unwrap().token; + fs::remove_file( + store + .temporal_dir() + .join(format!("witness-{:016x}-{:04x}.bin", current.generation, 0)), + ) + .unwrap(); + assert!(store.load_temporal(now(&fresh)).is_err()); + assert!(store.inspect_temporal_retirement(now(&fresh)).is_err()); +} +#[test] +fn cold_expired_handle_can_inspect_only_metadata_retire_and_begin_new_empty_run() { + use crate::room_learner::temporal_model_input::managed::execution::checkpoint::synthetic_new_empty_run; + let (t, store, p) = setup(); + commit(&store, None, &p[0]); + let d = p[0].descriptor().clone(); + drop(store); + let cold = RecoveryStore::new( + &t.path().join(&d.owner_scope).join("SensingServer"), + &d.owner_scope, + &d.installation_id, + Some(&[7; 32]), + ) + .unwrap(); + let expired = d.expires_at_ms * 1_000_000; + assert!(cold.load_temporal(expired).is_err()); + let inspection = cold.inspect_temporal_retirement(expired).unwrap().unwrap(); + assert!(inspection.expired()); + assert_eq!(inspection.descriptor(), &d); + assert_eq!(inspection.run(), p[0].run()); + assert!(cold + .prepare_inspected_retirement(&inspection, expired - 1, true) + .is_err()); + let retire = cold + .prepare_inspected_retirement(&inspection, expired, true) + .unwrap(); + let retired = cold.publish_temporal(&retire, None, expired).unwrap(); + assert!(cold + .load_temporal(expired) + .unwrap() + .unwrap() + .prefix + .is_none()); + let fresh = synthetic_new_empty_run(&p[0], d.expires_at_ms + 1); + commit(&cold, Some(&retired), &fresh); + assert!(cold + .prepare_inspected_retirement(&inspection, now(&fresh), false) + .is_err()); +} +#[test] +fn metadata_inspection_never_blesses_missing_corrupt_or_foreign_head() { + for target in ["missing", "corrupt"] { + let (_t, store, p) = setup(); + commit(&store, None, &p[0]); + let file = store.temporal_dir().join(HEAD); + if target == "missing" { + fs::remove_file(file).unwrap(); + } else { + let mut b = fs::read(&file).unwrap(); + let i = b.len() - 1; + b[i] ^= 1; + fs::write(file, b).unwrap(); + } + assert!(store + .inspect_temporal_retirement(p[0].descriptor().expires_at_ms * 1_000_000) + .is_err()); + } +} +#[test] +fn visible_lost_ack_requires_real_recertification_before_durable_result() { + let (_t, store, p) = setup(); + let t0 = commit(&store, None, &p[0]); + let t1 = commit(&store, Some(&t0), &p[1]); + let intent = store + .prepare_temporal(Some(&t1), &p[2], now(&p[2])) + .unwrap(); + let mut hook = |point: &str| { + if point == "after:witness:install" { + Err("injected-lost-witness-directory-ack".into()) + } else { + Ok(()) + } + }; + assert!(store + .publish_temporal_with(&intent, Some(&p[2]), now(&p[2]), false, &mut hook) + .is_err()); + let observed = store.load_temporal(now(&p[2])).unwrap().unwrap(); + assert_eq!(observed.prefix.as_ref().unwrap().bytes(), p[2].bytes()); + for fail in [ + "before:restore:file_fsync", + "after:restore:file_fsync", + "before:restore:directory_fsync", + "after:restore:directory_fsync", + "before:restore:readback", + "after:restore:readback", + ] { + assert!( + store + .recertify_temporal_with(&observed.token, now(&p[2]), &mut |point| if point == fail + { + Err("injected-restore-barrier".into()) + } else { + Ok(()) + }) + .is_err(), + "{fail}" + ); + } + let certified = store + .recertify_temporal(&observed.token, now(&p[2])) + .unwrap(); + assert_eq!(certified.token, observed.token); + assert_eq!(certified.prefix.bytes(), p[2].bytes()); +} + +#[test] +fn metadata_only_cold_retirement_rejects_copied_foreign_owner_records() { + let (t, store, p) = setup(); + commit(&store, None, &p[0]); + let owner = "b".repeat(64); + let foreign = RecoveryStore::new( + &t.path().join(&owner).join("SensingServer"), + &owner, + &p[0].descriptor().installation_id, + Some(&[7; 32]), + ) + .unwrap(); + drop(foreign.temporal_lock(true).unwrap()); + for entry in fs::read_dir(store.temporal_dir()).unwrap() { + let entry = entry.unwrap(); + let name = entry.file_name(); + if name == HEAD || name.to_string_lossy().starts_with("witness-") { + fs::copy(entry.path(), foreign.temporal_dir().join(name)).unwrap(); + } + } + assert!(foreign + .inspect_temporal_retirement(p[0].descriptor().expires_at_ms * 1_000_000) + .is_err()); +} From 80d0430c369be628a3e3db950bc9df73ed5e63a4 Mon Sep 17 00:00:00 2001 From: Codex for Dragan Spiridonov Date: Fri, 6 Nov 2026 13:53:01 +0100 Subject: [PATCH 067/100] feat(learner): add inactive Runtime temporal Train bridge --- .../src/room_learner/command.rs | 15 +- .../src/room_learner/lifecycle.rs | 74 +- .../src/room_learner/lifecycle_capture.rs | 63 +- .../src/room_learner/lifecycle_evaluation.rs | 3 +- .../lifecycle_legacy_byte_probe.rs | 90 ++ .../src/room_learner/lifecycle_recovery.rs | 18 +- .../room_learner/lifecycle_recovery_enable.rs | 7 +- .../lifecycle_recovery_enable_tests.rs | 21 +- .../room_learner/lifecycle_recovery_tests.rs | 15 +- .../src/room_learner/lifecycle_selection.rs | 36 +- .../room_learner/lifecycle_selection_tests.rs | 2 +- .../src/room_learner/lifecycle_temporal.rs | 396 ++++++++ .../room_learner/lifecycle_temporal_tests.rs | 527 +++++++++++ .../managed_training_readiness.rs | 9 +- .../room_learner/managed_training_tests.rs | 12 +- .../src/room_learner/paired_progress.rs | 3 +- .../src/room_learner/temporal_acquisition.rs | 98 +- .../room_learner/temporal_capture_quality.rs | 884 ++++++++++++++++++ .../src/room_learner/temporal_checkpoint.rs | 134 ++- .../room_learner/temporal_checkpoint_tests.rs | 91 ++ .../src/room_learner/temporal_execution.rs | 17 +- .../room_learner/temporal_execution_tests.rs | 250 +++++ .../temporal_model_input_tests.rs | 7 + 23 files changed, 2598 insertions(+), 174 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_legacy_byte_probe.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/command.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/command.rs index eb6edb643c..93949f7485 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/command.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/command.rs @@ -22,15 +22,12 @@ impl RoomLearningRuntime { runtime_epoch: self.runtime_epoch.clone(), generation: state.generation, workflow_revision: state.workflow_revision, - context_digest: state.capture.as_ref().map(|c| c.context().digest()), - phase: state - .capture - .as_ref() - .and_then(|c| c.progress(0).phase) - .map(|p| PhaseIdentity { - trial_id: p.trial_id, - session_id: p.session_id, - }), + context_digest: state.capture_context().map(CaptureContext::digest), + phase: match state.capture.as_ref() { + Some(CaptureWorkflow::Legacy(c)) => c.progress(0).phase.map(|p| PhaseIdentity { trial_id: p.trial_id, session_id: p.session_id }), + Some(CaptureWorkflow::Temporal(c)) => c.phase_identity(), + None => None, + }, } } pub fn mutation_guard(&self) -> Result { diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs index 4d3f8191fe..54ce5833c2 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs @@ -43,10 +43,15 @@ impl Default for Job { } } } +/// Exactly one capture implementation. No public configure path constructs Temporal. +enum CaptureWorkflow { + Legacy(CaptureSession), + Temporal(Box), +} struct Inner { generation: u64, workflow_revision: u64, - capture: Option, + capture: Option, model: Option>, receipt: Option, frozen_training: bool, @@ -60,6 +65,25 @@ struct Inner { qualification: Option, qualification_error: Option, } +impl Inner { + fn legacy_capture(&self) -> Option<&CaptureSession> { + match self.capture.as_ref() { Some(CaptureWorkflow::Legacy(c)) => Some(c), _ => None } + } + fn legacy_capture_mut(&mut self) -> Option<&mut CaptureSession> { + match self.capture.as_mut() { Some(CaptureWorkflow::Legacy(c)) => Some(c), _ => None } + } + fn capture_context(&self) -> Option<&CaptureContext> { + self.capture.as_ref().map(|capture| match capture { + CaptureWorkflow::Legacy(c) => c.context(), + CaptureWorkflow::Temporal(c) => c.context(), + }) + } + fn require_legacy_kind(&self) -> Result<(), String> { + if matches!(self.capture, Some(CaptureWorkflow::Temporal(_))) { + Err("learner_temporal_capture_private_unsupported_command".into()) + } else { Ok(()) } + } +} pub struct RoomLearningRuntime { runtime_epoch: String, owner: String, @@ -192,6 +216,7 @@ impl RoomLearningRuntime { &self, room_digest: &str, ) -> Result { + self.inner.lock().map_err(|_| "learner state unavailable")?.require_legacy_kind()?; self.store.catalog_for_room(room_digest) } /// Explicit new capture versus resuming an exact persisted candidate. Neither @@ -228,7 +253,8 @@ impl RoomLearningRuntime { FitConfig::for_source_count(context.sources.len()).validate_context(&context)?; let mut session = CaptureSession::new(context.clone(), run_id)?; let mut state = self.command_state(expected)?; - if busy(&state.job) || state.capture.as_ref().is_some_and(|c| c.phase_active()) { + state.require_legacy_kind()?; + if busy(&state.job) || state.legacy_capture().is_some_and(|c| c.phase_active()) { return Err( "finish/cancel the active capture or training job before changing context".into(), ); @@ -268,7 +294,7 @@ impl RoomLearningRuntime { state.recovery_enable = if new_training && !recovery_consent { recovery_enable::CurrentNewRun::new(&session, now_ns) } else { None }; - state.capture = Some(session); + state.capture = Some(CaptureWorkflow::Legacy(session)); state.model = if new_training { None } else { @@ -310,6 +336,7 @@ impl RoomLearningRuntime { config.validate()?; let (generation, dataset, proof_seed, cancel) = { let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; self.require_recovery_ready(&state)?; if busy(&state.job) || (state.frozen_training @@ -317,9 +344,7 @@ impl RoomLearningRuntime { { return Err("this run is already training or frozen".into()); } - let capture = state - .capture - .as_ref() + let capture = state.legacy_capture() .ok_or("no learner capture configured")?; config.validate_context(capture.context())?; if capture.phase_active() { @@ -399,13 +424,12 @@ impl RoomLearningRuntime { } pub fn prediction(&self, now_ns: u64) -> Result<(MutationGuard, RoomPrediction), String> { let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + state.require_legacy_kind()?; let model = state .model .as_ref() .ok_or("no trained/restored candidate for this context")?; - let cohort = state - .capture - .as_ref() + let cohort = state.legacy_capture() .ok_or("no current capture context")? .live_cohort(now_ns)?; Ok((self.guard_for(&state), model.infer(&cohort))) @@ -414,10 +438,11 @@ impl RoomLearningRuntime { let Ok(mut state) = self.inner.lock() else { return json!({"enabled":true,"state":"unavailable"}); }; + if state.require_legacy_kind().is_err() { + return json!({"schema":"ruview.room-learner.status.v1","enabled":true,"success":false,"error_code":"learner_temporal_capture_private_unsupported_command"}); + } let _ = self.checkpoint_recovery(&mut state, now_ns, false); - let required_ids: Vec<_> = state - .capture - .as_ref() + let required_ids: Vec<_> = state.legacy_capture() .map(|c| c.context().sources.iter().map(|s| s.source_id).collect()) .unwrap_or_default(); let trained_ids: Vec<_> = state @@ -426,9 +451,7 @@ impl RoomLearningRuntime { .map(|m| m.context().sources.iter().map(|s| s.source_id).collect()) .unwrap_or_default(); let prediction = state.model.as_ref().and_then(|m| { - state - .capture - .as_ref() + state.legacy_capture() .and_then(|c| c.live_cohort(now_ns).ok()) .map(|c| m.infer(&c)) }); @@ -443,17 +466,17 @@ impl RoomLearningRuntime { json!({"schema":"ruview.room-learner.status.v1","enabled":true,"mutation_guard":self.guard_for(&state), "required_source_ids":required_ids,"trained_source_ids":trained_ids,"scored_source_ids":scored_ids, "candidate_prediction":prediction,"training_manifest":state.model.as_ref().map(|m|m.training_manifest()), - "phase_history":state.capture.as_ref().map(|c|c.history()), + "phase_history":state.legacy_capture().map(|c|c.history()), "phase_duration_defaults":{"Empty":600000,"OccupiedStatic":12000,"OccupiedMoving":12000,"AdjacentOnly":12000}, "held_out_phase_duration_defaults":{"Empty":12000,"OccupiedStatic":12000,"OccupiedMoving":12000,"AdjacentOnly":12000}, "phase_lead_in_defaults":{"Empty":15000,"OccupiedStatic":5000,"OccupiedMoving":5000,"AdjacentOnly":15000}, "untrained_capture_progress_is_volatile":!state.recovery.enabled || state.recovery.error.is_some(),"recovery":self.recovery_status(&state, now_ns), - "generation":state.generation,"state":state.job.phase,"context_digest":state.capture.as_ref().map(|c|c.context().digest()), - "room_digest":state.capture.as_ref().map(|c| &c.context().room_digest), - "revision":state.capture.as_ref().map(|c|c.context().revision), - "sources":state.capture.as_ref().map(|c| &c.context().sources), - "capture":state.capture.as_ref().map(|c|c.progress(now_ns)), - "training_frozen":state.frozen_training,"training_defaults":FitConfig::for_source_count(state.capture.as_ref().map_or(0, |c| c.context().sources.len())),"training_report":state.job.report, + "generation":state.generation,"state":state.job.phase,"context_digest":state.legacy_capture().map(|c|c.context().digest()), + "room_digest":state.legacy_capture().map(|c| &c.context().room_digest), + "revision":state.legacy_capture().map(|c|c.context().revision), + "sources":state.legacy_capture().map(|c| &c.context().sources), + "capture":state.legacy_capture().map(|c|c.progress(now_ns)), + "training_frozen":state.frozen_training,"training_defaults":FitConfig::for_source_count(state.legacy_capture().map_or(0, |c| c.context().sources.len())),"training_report":state.job.report, "error":state.job.error,"input_error":state.input_error, "candidate":state.receipt,"candidate_loaded":state.model.is_some(),"candidate_identity":self.candidate_identity(&state), "qualification_readiness":self.qualification_readiness_locked(&state,now_ns), @@ -515,3 +538,10 @@ pub use paired_progress::{PairedCalibrationProgress, PublishedProgressRoom, PAIR #[path = "lifecycle_recovery_enable.rs"] mod recovery_enable; + +#[path = "lifecycle_temporal.rs"] +mod temporal_control; + +#[cfg(test)] +#[path="lifecycle_legacy_byte_probe.rs"] +mod legacy_byte_probe; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs index ab1e563ccc..65f94f5d75 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs @@ -18,30 +18,18 @@ impl RoomLearningRuntime { Ok(context) } pub fn selects_grid(&self, source_id: u8, grid_id: &str) -> bool { - self.inner - .lock() - .ok() - .and_then(|s| { - s.capture.as_ref().map(|c| { - c.context() - .sources - .iter() - .any(|v| v.source_id == source_id && v.grid_id == grid_id) - }) - }) - .unwrap_or(false) + self.inner.lock().ok().and_then(|state| state.capture_context().map(|c| c.sources.iter().any(|s| s.source_id == source_id && s.grid_id == grid_id))).unwrap_or(false) } pub fn capture_active(&self) -> Result { let state = self.inner.lock().map_err(|_| "learner state unavailable")?; - Ok(state.capture.as_ref().is_some_and(|c| c.phase_active())) + Ok(match state.capture.as_ref() { + Some(CaptureWorkflow::Legacy(c)) => c.phase_active(), + Some(CaptureWorkflow::Temporal(c)) => c.active(), + None => false, + }) } pub fn context(&self) -> Option { - self.inner - .lock() - .ok()? - .capture - .as_ref() - .map(|c| c.context().clone()) + self.inner.lock().ok()?.capture_context().cloned() } pub fn begin_phase( &self, @@ -50,6 +38,7 @@ impl RoomLearningRuntime { now_ns: u64, ) -> Result<(), String> { let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; self.expire_loaded_recovery(&mut state, now_ns)?; self.require_recovery_ready(&state)?; if busy(&state.job) { @@ -71,9 +60,7 @@ impl RoomLearningRuntime { return Err(readiness["detail"].as_str().unwrap_or("managed training lineage unavailable before held-out capture").into()); } } - state - .capture - .as_mut() + state.legacy_capture_mut() .ok_or("configure the current room and source set first")? .begin_phase(request, now_ns)?; state.input_error = None; @@ -82,7 +69,8 @@ impl RoomLearningRuntime { } pub fn ingest(&self, frame: RawSourceFrame<'_>, now_ns: u64) -> Result<(), String> { let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; - let Some(capture) = state.capture.as_mut() else { + state.require_legacy_kind()?; + let Some(capture) = state.legacy_capture_mut() else { return Ok(()); }; let result = capture.ingest(frame, now_ns); @@ -109,7 +97,12 @@ impl RoomLearningRuntime { return Err("live managed capture rejects synthetic/replayed native provenance".into()); } let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; - let Some(capture) = state.capture.as_mut() else { + if let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() { + let result = capture.ingest(self.temporal_store()?, &self.runtime_epoch, packet, now_ns); + if let Err(error) = &result { state.input_error = Some(bounded(error.clone())); } + return result; + } + let Some(capture) = state.legacy_capture_mut() else { return Ok(()); }; let source = capture @@ -158,13 +151,18 @@ impl RoomLearningRuntime { /// workflow revision. Ignore sources/grids outside the current context. pub fn record_native_rejection(&self, source_id: u8, grid_id: &str, reason: &str) { if let Ok(mut state) = self.inner.lock() { - if state.capture.as_ref().is_some_and(|c| { + if let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() { + capture.reject(source_id, grid_id, reason); + return; + } + + if state.legacy_capture().is_some_and(|c| { c.context() .sources .iter() .any(|s| s.source_id == source_id && s.grid_id == grid_id) }) { - if let Some(capture)=state.capture.as_mut() {capture.clear_live_windows();} + if let Some(capture)=state.legacy_capture_mut() {capture.clear_live_windows();} state.input_error = Some(bounded(format!( "Node {source_id} native input rejected: {reason}" ))); @@ -178,14 +176,13 @@ impl RoomLearningRuntime { conditions_held: bool, ) -> Result { let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; self.expire_loaded_recovery(&mut state, now_ns)?; let recovered_complete = state.recovery.disposition == Some( crate::room_learner::capture::RecoveryPhaseDisposition::CompletedNeedsConfirmation, ); - let capture = state - .capture - .as_mut() + let capture = state.legacy_capture_mut() .ok_or("no learner capture configured")?; if !conditions_held { capture.cancel_phase(); @@ -208,16 +205,15 @@ impl RoomLearningRuntime { self.store.save_managed_capture_fence(model,&proof,seed)?; } } - state.capture=Some(completed); + state.capture=Some(CaptureWorkflow::Legacy(completed)); state.recovery.disposition = None; self.checkpoint_recovery(&mut state, now_ns, true)?; Ok(receipt) } pub fn cancel_phase(&self, expected: &MutationGuard) -> Result<(), String> { let mut state = self.command_state(expected)?; - state - .capture - .as_mut() + state.require_legacy_kind()?; + state.legacy_capture_mut() .ok_or("no learner capture configured")? .cancel_phase(); let now = self.checkpoint_now(&state); @@ -226,6 +222,7 @@ impl RoomLearningRuntime { } pub fn cancel_training(&self, expected: &MutationGuard) -> Result { let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; if busy(&state.job) { state.job.cancel.store(true, Ordering::Release); state.job.phase = JobPhase::Cancelling; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_evaluation.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_evaluation.rs index f2d4e6b1b0..a1650d2670 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_evaluation.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_evaluation.rs @@ -51,11 +51,12 @@ impl RoomLearningRuntime { return Err("evaluate only validation or held-out data".into()); } let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; if busy(&state.job) || !state.frozen_training { return Err("wait for the frozen candidate".into()); } let model = state.model.as_ref().ok_or("no frozen candidate")?.clone(); - let capture = state.capture.as_ref().ok_or("no capture context")?; + let capture = state.legacy_capture().ok_or("no capture context")?; if capture.phase_active() { return Err("finish the current phase before evaluation".into()); } diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_legacy_byte_probe.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_legacy_byte_probe.rs new file mode 100644 index 0000000000..17bf6ecf9f --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_legacy_byte_probe.rs @@ -0,0 +1,90 @@ +//! SYNTHETIC fixed-clock public Legacy Runtime status-byte compatibility probe. +use super::*; +use sha2::{Digest, Sha256}; +#[test] +fn legacy_public_status_byte_probe() { + let dir = tempfile::tempdir().unwrap(); + let c = tests::context(); + let path = dir.path().join(&c.owner_scope).join("SensingServer"); + let recovery = crate::room_learner::recovery_store::RecoveryStore::new( + &path, + &c.owner_scope, + &c.installation_id, + Some(&[7; 32]), + ) + .unwrap(); + let mut rt = RoomLearningRuntime::new_with_recovery_at( + &path, + &c.owner_scope, + &c.installation_id, + recovery, + tests::START, + ) + .unwrap(); + rt.runtime_epoch = "synthetic-legacy-golden-runtime".into(); + rt.configure_for_recovery( + &rt.mutation_guard().unwrap(), + c.clone(), + "synthetic-legacy-golden-run".into(), + true, + false, + tests::START, + ) + .unwrap(); + fn stable(mut value: Value) -> Value { + if let Some(cap) = value.pointer_mut("/recovery/enable_current_run") { + if cap["run_id"].is_string() { + cap["run_id"] = json!("synthetic-generated-ticket"); + } + } + value + } + let initial = stable(rt.status(tests::START)); + let first = tests::START; + rt.begin_phase( + &rt.mutation_guard().unwrap(), + PhaseRequest { + trial_id: "golden-trial".into(), + session_id: "golden-session".into(), + role: DataRole::Train, + label: crate::room_learner::observation::RoomLabel::Empty, + duration_ms: 12000, + lead_in_ms: 0, + conditions_confirmed: true, + }, + first, + ) + .unwrap(); + for bucket in 0..4u64 { + for row in 0..8u32 { + let at = first + bucket * 3_000_000_000 + u64::from(row + 1) * 10_000_000; + for source in &c.sources { + let amplitudes = vec![2.; source.feature_count]; + rt.ingest( + RawSourceFrame { + source_id: source.source_id, + grid_id: &source.grid_id, + frequency_hz: source.center_frequency_hz, + bandwidth_hz: source.bandwidth_hz, + sequence: bucket as u32 * 8 + row + 1, + amplitudes: &litudes, + }, + at, + ) + .unwrap(); + } + } + } + rt.finish_phase(&rt.mutation_guard().unwrap(), first + 12_000_000_000, true) + .unwrap(); + let sealed = stable(rt.status(first + 12_000_000_000)); + let bytes = serde_json::to_vec(&(initial, sealed)).unwrap(); + let digest = format!("{:x}", Sha256::digest(&bytes)); + // The receiver clock and runtime ID are fixed; ONLY the random new-run + // receipt handle is normalized above. All other public bytes remain exact. + assert_eq!( + digest, + "8e35280c09309b4f039e1e7cc646364bbd7a92934cfe2960df1c4a0dbeb91b5a" + ); + println!("LEGACY_PUBLIC_STATUS_SHA256={digest} BYTES={}", bytes.len()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs index 0648f34753..0e49580865 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery.rs @@ -99,6 +99,7 @@ impl RoomLearningRuntime { state: &mut Inner, now_ns: u64, ) -> Result<(), String> { + state.require_legacy_kind()?; state.recovery.last_now_ns = state.recovery.last_now_ns.max(now_ns); let now_ns = state.recovery.last_now_ns; if !state.recovery.enabled || state.capture.is_none() { @@ -114,7 +115,7 @@ impl RoomLearningRuntime { return Err(error.clone()); } let verified = (|| -> Result<(), String> { - if state.capture.as_ref().map(|capture| capture.context().digest()) + if state.legacy_capture().map(|capture| capture.context().digest()) .as_deref() != Some(descriptor.context_digest.as_str()) { return Err("recovery_context_mismatch".into()); @@ -222,13 +223,19 @@ impl RoomLearningRuntime { now_ns: u64, force: bool, ) -> Result<(), String> { + state.require_legacy_kind()?; state.recovery.last_now_ns = state.recovery.last_now_ns.max(now_ns); self.expire_loaded_recovery(state, now_ns)?; if !state.recovery.enabled { return Ok(()); } let result: Result = (|| { - let capture = state.capture.as_ref().ok_or("recovery_capture_missing")?; + // Borrow the capture field alone so recording an attempt still + // precedes checkpoint construction, exactly as on the legacy path. + let capture = match state.capture.as_ref() { + Some(CaptureWorkflow::Legacy(capture)) => capture, + _ => return Err("recovery_capture_missing".into()), + }; let marker = format!( "{}:{:?}:{}", capture.recovery_marker(now_ns), @@ -359,6 +366,7 @@ impl RoomLearningRuntime { now_ns: u64, ) -> Result { let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + state.require_legacy_kind()?; state.recovery.last_now_ns = state.recovery.last_now_ns.max(now_ns); let now_ns = state.recovery.last_now_ns; let loaded = self @@ -387,9 +395,10 @@ impl RoomLearningRuntime { now_ns: u64, ) -> Result<(), String> { let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; state.recovery.last_now_ns = state.recovery.last_now_ns.max(now_ns); let now_ns = state.recovery.last_now_ns; - if busy(&state.job) || state.capture.as_ref().is_some_and(|c| c.phase_active()) { + if busy(&state.job) || state.legacy_capture().is_some_and(|c| c.phase_active()) { return Err("finish or cancel the active operation first".into()); } let loaded = self @@ -429,7 +438,7 @@ impl RoomLearningRuntime { .checked_add(1) .ok_or("learner generation exhausted")?; state.recovery_enable = None; - state.capture = Some(restored.session); + state.capture = Some(CaptureWorkflow::Legacy(restored.session)); state.model = model.map(Arc::new); state.receipt = None; state.frozen_training = payload.frozen_training; @@ -465,6 +474,7 @@ impl RoomLearningRuntime { recovery_id: &str, ) -> Result<(), String> { let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; // Fence every later ingest/status/worker save before touching the file. state.recovery.enabled = false; state.recovery.marker = None; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_enable.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_enable.rs index 83420131dd..91ed81d4d9 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_enable.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_enable.rs @@ -38,7 +38,7 @@ impl RoomLearningRuntime { Some("new_run_recovery_deadline_expired") } else if state.recovery.enabled { Some("recovery_already_enabled") - } else if busy(&state.job) || state.capture.as_ref().is_some_and(|c| c.phase_active()) { + } else if busy(&state.job) || state.legacy_capture().is_some_and(|c| c.phase_active()) { Some("finish_active_operation") } else if self.recovery_store.as_ref().is_none_or(|s| !s.available()) { Some("recovery_unavailable") @@ -70,6 +70,7 @@ impl RoomLearningRuntime { return Err("recovery_consent_required".into()); } let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; let now_ns = state.recovery.observe_clock(now_ns); let run = state.recovery_enable.clone().ok_or("not_a_new_ram_run")?; if run.id != run_id { @@ -81,10 +82,10 @@ impl RoomLearningRuntime { if state.recovery.enabled { return Err("recovery_already_enabled".into()); } - if busy(&state.job) || state.capture.as_ref().is_some_and(|c| c.phase_active()) { + if busy(&state.job) || state.legacy_capture().is_some_and(|c| c.phase_active()) { return Err("finish_active_operation".into()); } - let capture = state.capture.as_ref().ok_or("recovery_capture_missing")?; + let capture = state.legacy_capture().ok_or("recovery_capture_missing")?; if capture.context().digest() != run.context_digest { return Err("recovery_context_mismatch".into()); } diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_enable_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_enable_tests.rs index 6ad1f4d2db..40525b5c99 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_enable_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_enable_tests.rs @@ -45,8 +45,7 @@ fn current_new_run_enable_preserves_exact_rows_history_watermarks_and_cold_resto .inner .lock() .unwrap() - .capture - .as_ref() + .legacy_capture() .unwrap() .checkpoint(at) .unwrap(); @@ -56,8 +55,7 @@ fn current_new_run_enable_preserves_exact_rows_history_watermarks_and_cold_resto .inner .lock() .unwrap() - .capture - .as_ref() + .legacy_capture() .unwrap() .checkpoint(at) .unwrap(); @@ -106,8 +104,7 @@ fn wrong_ticket_consent_stale_guard_and_active_phase_do_not_enable_or_change_cap .inner .lock() .unwrap() - .capture - .as_ref() + .legacy_capture() .unwrap() .checkpoint(BASE) .unwrap(); @@ -120,8 +117,7 @@ fn wrong_ticket_consent_stale_guard_and_active_phase_do_not_enable_or_change_cap &rt.inner .lock() .unwrap() - .capture - .as_ref() + .legacy_capture() .unwrap() .checkpoint(BASE) .unwrap() @@ -246,8 +242,7 @@ fn write_failure_keeps_ram_and_ticket_then_explicit_retry_can_succeed_without_re &rt.inner .lock() .unwrap() - .capture - .as_ref() + .legacy_capture() .unwrap() .checkpoint(at) .unwrap(), @@ -263,8 +258,7 @@ fn write_failure_keeps_ram_and_ticket_then_explicit_retry_can_succeed_without_re &rt.inner .lock() .unwrap() - .capture - .as_ref() + .legacy_capture() .unwrap() .checkpoint(at) .unwrap() @@ -330,8 +324,7 @@ fn partial_committed_same_ticket_is_retried_with_unchanged_original_deadline() { .inner .lock() .unwrap() - .capture - .as_ref() + .legacy_capture() .unwrap() .checkpoint(at) .unwrap(), diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_tests.rs index 7ce070b54c..deabc9cccb 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_recovery_tests.rs @@ -332,8 +332,7 @@ async fn trained_checkpoint_cold_restores_exact_candidate_without_live_buffers_o .inner .lock() .unwrap() - .capture - .as_ref() + .legacy_capture() .unwrap() .validated_dataset() .unwrap(); @@ -670,7 +669,7 @@ fn loaded_recovery_expires_to_ram_at_exact_deadline_without_erasing_or_rebasing( assert_eq!(before["recovery"]["enabled"], true); assert_eq!(before["capture"]["sealed_cohorts"], 28); let (rows, history) = { - let state = recovered.inner.lock().unwrap();let capture = state.capture.as_ref().unwrap(); + let state = recovered.inner.lock().unwrap();let capture = state.legacy_capture().unwrap(); (serde_json::to_vec(capture.cohorts()).unwrap(), serde_json::to_vec(capture.history()).unwrap()) }; let at = recovered.status(expires); @@ -689,7 +688,7 @@ fn loaded_recovery_expires_to_ram_at_exact_deadline_without_erasing_or_rebasing( assert_eq!(after["capture"]["sealed_cohorts"],32);assert!(after["capture"]["phase"].is_null()); assert_eq!(after["capture"]["sources"][0]["last_accepted_sequence"],256); { - let state=recovered.inner.lock().unwrap();let capture=state.capture.as_ref().unwrap(); + let state=recovered.inner.lock().unwrap();let capture=state.legacy_capture().unwrap(); assert_eq!(serde_json::to_vec(&capture.cohorts()[..28]).unwrap(),rows); assert_eq!(serde_json::to_vec(&capture.history()[..7]).unwrap(),history); assert_eq!(capture.cohorts()[28].views[0].sequence_ids[0],225); @@ -774,7 +773,7 @@ async fn recovered_28_cohorts_can_finish_heldout_after_expiry_under_the_same_qua recovered.restore_capture_recovery(&recovered.mutation_guard().unwrap(),&descriptor.id,&context(),BASE+100_000_000_000).unwrap(); let expires=descriptor.expires_at_ms*1_000_000;let bytes=std::fs::read(recovery_file(temp.path())).unwrap(); assert_eq!(recovered.status(expires)["capture"]["sealed_cohorts"],28); - let before={let state=recovered.inner.lock().unwrap();serde_json::to_vec(state.capture.as_ref().unwrap().cohorts()).unwrap()}; + let before={let state=recovered.inner.lock().unwrap();serde_json::to_vec(state.legacy_capture().unwrap().cohorts()).unwrap()}; assert!(recovered.qualify(&recovered.mutation_guard().unwrap()).is_err(),"expiry does not waive missing HeldOut Empty"); assert!(recovered.status(expires+1)["model_selection"]["selected"].is_null()); expiry_phase(&recovered,expires+1_000_000,DataRole::HeldOut,RoomLabel::Empty,7); @@ -785,7 +784,7 @@ async fn recovered_28_cohorts_can_finish_heldout_after_expiry_under_the_same_qua let status=recovered.status(expires+13_000_000_000); assert!(status["model_selection"]["selected"].is_null(),"qualification must not auto-select"); assert_eq!(status["control_detector_changed"],false);assert_eq!(status["recovery"]["enabled"],false); - {let state=recovered.inner.lock().unwrap();assert_eq!(serde_json::to_vec(&state.capture.as_ref().unwrap().cohorts()[..28]).unwrap(),before);} + {let state=recovered.inner.lock().unwrap();assert_eq!(serde_json::to_vec(&state.legacy_capture().unwrap().cohorts()[..28]).unwrap(),before);} assert_eq!(std::fs::read(recovery_file(temp.path())).unwrap(),bytes); assert_eq!(original_descriptor(temp.path()).expires_at_ms,descriptor.expires_at_ms); } @@ -813,7 +812,7 @@ fn known_expiry_cannot_be_reenabled_by_a_queued_restore_with_an_earlier_timestam expiry_phase(&rt,BASE,DataRole::Train,RoomLabel::Empty,0); let descriptor=original_descriptor(temp.path());let expires=descriptor.expires_at_ms*1_000_000; let bytes=std::fs::read(recovery_file(temp.path())).unwrap(); - let rows={let state=rt.inner.lock().unwrap();serde_json::to_vec(state.capture.as_ref().unwrap().cohorts()).unwrap()}; + let rows={let state=rt.inner.lock().unwrap();serde_json::to_vec(state.legacy_capture().unwrap().cohorts()).unwrap()}; if observe_by_status { rt.status(expires); } else { @@ -826,7 +825,7 @@ fn known_expiry_cannot_be_reenabled_by_a_queued_restore_with_an_earlier_timestam assert_eq!(status["recovery"]["state"],"expired");assert_eq!(status["recovery"]["enabled"],false); assert!(status["recovery"]["saved"].is_null());assert_eq!(status["capture"]["sealed_cohorts"],4); assert_eq!(original_descriptor(temp.path()),descriptor);assert_eq!(std::fs::read(recovery_file(temp.path())).unwrap(),bytes); - let state=rt.inner.lock().unwrap();assert_eq!(serde_json::to_vec(state.capture.as_ref().unwrap().cohorts()).unwrap(),rows); + let state=rt.inner.lock().unwrap();assert_eq!(serde_json::to_vec(state.legacy_capture().unwrap().cohorts()).unwrap(),rows); } } diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection.rs index 61684ad246..29269975c4 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection.rs @@ -14,6 +14,12 @@ pub(super) struct SelectionState { error: Option, assessment: Option, } +impl SelectionState { + pub(super) fn has_history(&self) -> bool { + self.envelope.is_some() || self.model.is_some() || self.qualification.is_some() + || self.assessment.is_some() || self.error.is_some() + } +} impl RoomLearningRuntime { pub(super) fn restore_selection(&self, state: &mut Inner) { let restored = (|| -> Result<(), String> { @@ -24,10 +30,10 @@ impl RoomLearningRuntime { if let Some(target) = &envelope.receipt.selected { let (model, q) = self.store.resolve_selection_target(target)?; let model = Arc::new(model); - state.capture = Some(if let Some(proof)=self.store.load_managed_training_proof(&model)? { + state.capture = Some(CaptureWorkflow::Legacy(if let Some(proof)=self.store.load_managed_training_proof(&model)? { let fence=self.store.load_managed_capture_fence(&model,&proof)?; CaptureSession::resume_managed_fence(&proof,&fence)? - } else {CaptureSession::new(target.context.clone(),format!("selected-{}",envelope.receipt.revision))?}); + } else {CaptureSession::new(target.context.clone(),format!("selected-{}",envelope.receipt.revision))?})); state.model = Some(model.clone()); state.frozen_training = true; state.job.phase = JobPhase::Ready; @@ -44,12 +50,13 @@ impl RoomLearningRuntime { } pub fn qualify(&self, expected: &MutationGuard) -> Result { let mut state = self.command_state(expected)?; + state.require_legacy_kind()?; let result = (|| -> Result { if busy(&state.job) || !state.frozen_training { return Err("wait for the frozen candidate".into()); } let model = state.model.as_ref().ok_or("no candidate")?.clone(); - let capture = state.capture.as_ref().ok_or("no capture context")?; + let capture = state.legacy_capture().ok_or("no capture context")?; if capture.phase_active() { return Err("finish the current capture phase first".into()); } @@ -183,7 +190,8 @@ impl RoomLearningRuntime { qualification_sha256: &str, ) -> Result { let mut state = self.command_state(expected)?; - if busy(&state.job) || state.capture.as_ref().is_some_and(|c| c.phase_active()) { + state.require_legacy_kind()?; + if busy(&state.job) || state.legacy_capture().is_some_and(|c| c.phase_active()) { return Err("finish active capture/training before selecting".into()); } let model = state @@ -228,7 +236,8 @@ impl RoomLearningRuntime { } pub fn deselect_model(&self, expected: &MutationGuard) -> Result { let mut state = self.command_state(expected)?; - if busy(&state.job) || state.capture.as_ref().is_some_and(|c| c.phase_active()) { + state.require_legacy_kind()?; + if busy(&state.job) || state.legacy_capture().is_some_and(|c| c.phase_active()) { return Err("finish active capture/training before changing selection".into()); } if let Some(e) = &state.selection.envelope { @@ -248,7 +257,8 @@ impl RoomLearningRuntime { } pub fn rollback_model(&self, expected: &MutationGuard) -> Result { let mut state = self.command_state(expected)?; - if busy(&state.job) || state.capture.as_ref().is_some_and(|c| c.phase_active()) { + state.require_legacy_kind()?; + if busy(&state.job) || state.legacy_capture().is_some_and(|c| c.phase_active()) { return Err("finish active capture/training before rollback".into()); } let previous = state @@ -257,9 +267,7 @@ impl RoomLearningRuntime { .as_ref() .and_then(|e| e.receipt.previous.clone()) .ok_or("no prior selection")?; - let context = state - .capture - .as_ref() + let context = state.legacy_capture() .map(|c| c.context()) .ok_or("no current context")?; if *context != previous.context { @@ -350,9 +358,7 @@ impl RoomLearningRuntime { if let Some(error) = &state.input_error { return Err(format!("selected input is unverified: {error}")); } - let capture = state - .capture - .as_ref() + let capture = state.legacy_capture() .ok_or("awaiting current selected sources")?; if capture.context() != &target.context { return Err("selected context differs from current RF source contract".into()); @@ -408,6 +414,7 @@ impl RoomLearningRuntime { expected_room_binding: Option<&str>, ) -> SelectedBinaryEvidence { match self.inner.lock() { + Ok(state) if state.require_legacy_kind().is_err() => SelectedBinaryEvidence::Unavailable { identity: None, reason: "learner_temporal_capture_private_unsupported_command".into() }, Ok(state) => self.selected_evidence_locked(&state, now_ns, expected_room_binding), Err(_) => SelectedBinaryEvidence::Unavailable { identity: None, @@ -449,6 +456,11 @@ impl RoomLearningRuntime { Ok(state) => { let mut snapshot = self.selection_status_locked(&state, now_ns, expected_room_binding); + if let Err(reason) = state.require_legacy_kind() { + snapshot.integrity_error = Some(reason.clone()); + snapshot.current = SelectedBinaryEvidence::Unavailable { identity: None, reason }; + return snapshot; + } let verified = (|| -> Result<(), String> { if self.store.load_selection()?.as_ref() != state.selection.envelope.as_ref() { return Err("saved selection differs from current runtime".into()); diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection_tests.rs index 6da8312774..f5f68ea985 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_selection_tests.rs @@ -221,7 +221,7 @@ async fn qualification_requires_real_lineage_all_conditions_and_matching_sealed_ let rt = fitted(dir.path()).await; let state = rt.inner.lock().unwrap(); let model = state.model.as_ref().unwrap(); - let capture = state.capture.as_ref().unwrap(); + let capture = state.legacy_capture().unwrap(); let dataset = capture.validated_dataset().unwrap(); let evaluation = rt .store diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs new file mode 100644 index 0000000000..0940236394 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs @@ -0,0 +1,396 @@ +//! Default-inactive native Train capture bridge. No HTTP constructor or fit authority. +use super::*; +use crate::room_learner::{ + managed_schedule_contract::{LeadInPolicy, ManagedScheduleContract}, + native::CanonicalPacket, + temporal_model_input::managed::execution::{TemporalCaptureStage, TemporalTrainCapture}, +}; + +const TRAIN_LEAD_IN_NS: u64 = 5_000_000_000; +const PRIVATE_KIND: &str = "learner_temporal_capture_private_unsupported_command"; + +pub(super) struct RuntimeTemporalCapture { + context: CaptureContext, + core: Option, + lead_in_until: Option, + failure: Option, + last_now_ns: u64, +} +impl RuntimeTemporalCapture { + pub(super) fn context(&self) -> &CaptureContext { + &self.context + } + pub(super) fn phase_identity(&self) -> Option { + self.core + .as_ref()? + .phase_identity() + .map(|(trial_id, session_id)| PhaseIdentity { + trial_id, + session_id, + }) + } + pub(super) fn active(&self) -> bool { + self.lead_in_until.is_some() + || self.core.as_ref().is_some_and(|c| { + matches!( + c.stage(), + TemporalCaptureStage::Observing | TemporalCaptureStage::NeedsConfirmation + ) + }) + } + fn check_time(&mut self, now: u64) -> Result<(), String> { + if now < self.last_now_ns { + self.lead_in_until = None; + self.failure = Some("temporal_receiver_clock_reversed".into()); + if let Some(core) = self.core.as_mut() { + core.invalidate("temporal_receiver_clock_reversed"); + } + return Err("temporal_receiver_clock_reversed".into()); + } + self.core + .as_mut() + .ok_or("temporal_initial_barrier_unconfirmed")? + .check_time(now)?; + self.last_now_ns = now; + Ok(()) + } + fn arm( + &mut self, + store: &super::super::recovery_store::RecoveryStore, + epoch: &str, + now: u64, + ) -> Result<(), String> { + if let Some(at) = self.lead_in_until { + if now >= at { + self.lead_in_until = None; + self.core + .as_mut() + .ok_or("temporal_initial_barrier_unconfirmed")? + .start_segment(epoch, at, store)?; + } + } + Ok(()) + } + pub(super) fn ingest( + &mut self, + store: &super::super::recovery_store::RecoveryStore, + epoch: &str, + packet: &CanonicalPacket, + now: u64, + ) -> Result<(), String> { + self.check_time(now)?; + self.arm(store, epoch, now)?; + if self.lead_in_until.is_some() { + return self + .core + .as_mut() + .ok_or("temporal_initial_barrier_unconfirmed")? + .prime_fence(packet, now); + } // Preparation never credits RF observation. + let core = self + .core + .as_mut() + .ok_or("temporal_initial_barrier_unconfirmed")?; + match core.stage() { + TemporalCaptureStage::Observing => core.ingest(packet, now, store), + TemporalCaptureStage::Failed | TemporalCaptureStage::Blocked => { + Err("temporal_capture_unavailable".into()) + } + TemporalCaptureStage::Ready | TemporalCaptureStage::Paused => { + core.prime_fence(packet, now) + } + _ => Ok(()), // No automatic start, confirmation or new phase from packet arrival. + } + } + pub(super) fn reject(&mut self, source_id: u8, grid_id: &str, reason: &str) { + if self + .context + .sources + .iter() + .any(|s| s.source_id == source_id && s.grid_id == grid_id) + { + self.lead_in_until = None; + self.failure = Some(bounded(reason.into())); + if let Some(core) = self.core.as_mut() { + core.invalidate(reason); + } + } + } + fn snapshot(&self, now: u64) -> Value { + let mut value = self.core.as_ref().map_or_else( + || json!({"stage":"blocked","error":self.failure}), + |c| c.snapshot(), + ); + value["lead_in_remaining_ms"] = json!(self + .lead_in_until + .map_or(0, |end| end.saturating_sub(now) / 1_000_000)); + value["runtime_state"] = json!(if self.lead_in_until.is_some() { + "lead_in" + } else { + "capture" + }); + value["runtime_error"] = json!(self.failure); + value["public_activation"] = json!(false); + value["fit_authority"] = json!(false); + value + } +} +impl RoomLearningRuntime { + pub(super) fn temporal_store( + &self, + ) -> Result<&super::super::recovery_store::RecoveryStore, String> { + self.recovery_store + .as_ref() + .ok_or("temporal_recovery_unavailable".into()) + } + /// Deliberately PRIVATE: current public configure always constructs Legacy. + /// Refuses to discard existing capture/candidate/selection/recovery state. + fn configure_temporal_capture( + &self, + expected: &MutationGuard, + context: CaptureContext, + run: String, + now: u64, + ) -> Result<(), String> { + self.configure_temporal_capture_with( + expected, + context, + run, + now, + TemporalTrainCapture::prepare_retained, + ) + } + fn configure_temporal_capture_with( + &self, + expected: &MutationGuard, + context: CaptureContext, + run: String, + now: u64, + prepare: impl FnOnce( + ManagedScheduleContract, + String, + FitConfig, + u64, + &super::super::recovery_store::RecoveryStore, + ) -> Result<(TemporalTrainCapture, Option), String>, + ) -> Result<(), String> { + context.validate()?; + if context.owner_scope != self.owner + || context.installation_id != self.installation + || now == 0 + { + return Err("temporal_context_owner_or_clock".into()); + } + let schedule = ManagedScheduleContract::build( + context.clone(), + LeadInPolicy::Train5HeldOut15EngineeringV1, + )?; + let fit = FitConfig::for_source_count(context.sources.len()); + fit.validate_context(&context)?; + let mut state = self.command_state(expected)?; + if state.capture.is_some() + || state.model.is_some() + || state.selection.has_history() + || state.job.phase != JobPhase::Idle + || state.recovery.descriptor.is_some() + { + return Err("temporal_setup_would_replace_existing_work".into()); + } + let store = self.temporal_store()?; + let next_generation = state + .generation + .checked_add(1) + .ok_or("learner generation exhausted")?; + let result = prepare(schedule, run, fit, now, store); + let (core, failure) = match result { + Ok((core, barrier_error)) => (Some(core), barrier_error), + Err(error) => (None, Some(error)), + }; + state.capture = Some(CaptureWorkflow::Temporal(Box::new( + RuntimeTemporalCapture { + context, + core, + lead_in_until: None, + failure: failure.clone(), + last_now_ns: now, + }, + ))); + state.generation = next_generation; + if let Some(error) = failure { + state.input_error = Some(bounded(error.clone())); + return Err(error); + } + Ok(()) + } + fn start_temporal_phase(&self, expected: &MutationGuard, now: u64) -> Result<(), String> { + let mut state = self.command_state(expected)?; + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { + return Err(PRIVATE_KIND.into()); + }; + capture.check_time(now)?; + let core = capture + .core + .as_ref() + .ok_or("temporal_initial_barrier_unconfirmed")?; + if capture.lead_in_until.is_some() + || !matches!( + core.stage(), + TemporalCaptureStage::Ready | TemporalCaptureStage::Paused + ) + { + return Err("temporal_phase_not_startable".into()); + } + // The leaf checks the original descriptor/floors when this preparation ends. + capture.lead_in_until = Some( + now.checked_add(TRAIN_LEAD_IN_NS) + .ok_or("temporal_time_overflow")?, + ); + Ok(()) + } + fn tick_temporal_capture(&self, expected: &MutationGuard, now: u64) -> Result<(), String> { + let store = self.temporal_store()?; + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if *expected != self.guard_for(&state) { + return Err("learner_state_changed".into()); + } + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { + return Err(PRIVATE_KIND.into()); + }; + capture.check_time(now)?; + capture.arm(store, &self.runtime_epoch, now)?; + if capture.lead_in_until.is_some() { + return Ok(()); + } + let core = capture + .core + .as_mut() + .ok_or("temporal_initial_barrier_unconfirmed")?; + if core.stage() == TemporalCaptureStage::Observing { + core.tick(now, store) + } else { + Ok(()) + } + } + fn temporal_confirmation_challenge(&self, expected: &MutationGuard) -> Result { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if *expected != self.guard_for(&state) { + return Err("learner_state_changed".into()); + } + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_ref() else { + return Err(PRIVATE_KIND.into()); + }; + capture + .core + .as_ref() + .ok_or("temporal_initial_barrier_unconfirmed")? + .confirmation_challenge() + } + fn confirm_temporal_phase( + &self, + expected: &MutationGuard, + challenge: &str, + now: u64, + ) -> Result<(), String> { + let store = self.temporal_store()?; + let mut state = self.command_state(expected)?; + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { + return Err(PRIVATE_KIND.into()); + }; + capture.check_time(now)?; + if capture.lead_in_until.is_some() { + return Err("temporal_phase_incomplete".into()); + } + capture + .core + .as_mut() + .ok_or("temporal_initial_barrier_unconfirmed")? + .confirm(challenge, store, now) + } + fn pause_temporal_capture(&self, expected: &MutationGuard, now: u64) -> Result<(), String> { + let store = self.temporal_store()?; + let mut state = self.command_state(expected)?; + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { + return Err(PRIVATE_KIND.into()); + }; + capture.check_time(now)?; + if capture.lead_in_until.is_some() { + return Err("temporal_preparation_has_no_observation_to_pause".into()); + } + capture + .core + .as_mut() + .ok_or("temporal_initial_barrier_unconfirmed")? + .pause(now, store) + } + fn restore_temporal_capture( + &self, + expected: &MutationGuard, + context: &CaptureContext, + now: u64, + ) -> Result<(), String> { + context.validate()?; + if context.owner_scope != self.owner || context.installation_id != self.installation { + return Err("temporal_context_owner_or_clock".into()); + } + let store = self.temporal_store()?; + let mut state = self.command_state(expected)?; + if state.model.is_some() + || state.selection.has_history() + || state.job.phase != JobPhase::Idle + || state.recovery.descriptor.is_some() + { + return Err("temporal_restore_would_replace_existing_work".into()); + } + // A same-process reload must never lower known RAM replay/priming + // floors. Resume Paused directly; repair Blocked by its exact intent. + // A cold restore is admitted only into a fresh, unconfigured Runtime. + if state.capture.is_some() { + return Err("temporal_restore_requires_fresh_runtime".into()); + } + let next_generation = state + .generation + .checked_add(1) + .ok_or("learner generation exhausted")?; + let core = + TemporalTrainCapture::cold_restore(store, context, now, &self.runtime_epoch, now)?; + state.capture = Some(CaptureWorkflow::Temporal(Box::new( + RuntimeTemporalCapture { + context: context.clone(), + core: Some(core), + lead_in_until: None, + failure: None, + last_now_ns: now, + }, + ))); + state.generation = next_generation; + Ok(()) + } + fn repair_temporal_capture(&self, expected: &MutationGuard, now: u64) -> Result<(), String> { + let store = self.temporal_store()?; + let mut state = self.command_state(expected)?; + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { + return Err(PRIVATE_KIND.into()); + }; + capture + .core + .as_mut() + .ok_or("temporal_initial_barrier_unconfirmed")? + .repair_pending(store, now)?; + capture.failure = None; + state.input_error = None; + Ok(()) + } + fn temporal_capture_snapshot(&self, now: u64) -> Result { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_ref() else { + return Err(PRIVATE_KIND.into()); + }; + Ok( + json!({"schema":"ruview.room-learner.private-temporal-capture.v1","mutation_guard":self.guard_for(&state),"capture":capture.snapshot(now),"fit_authority":false,"public_activation":false}), + ) + } +} + +#[cfg(test)] +#[path = "lifecycle_temporal_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs new file mode 100644 index 0000000000..1dd8a105ea --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs @@ -0,0 +1,527 @@ +//! SYNTHETIC raw native inputs through the real Runtime, simulated receiver clocks. +//! No real RF, owner credentials, helper process, selected model or fit authority. +use super::*; +use crate::room_learner::{ + native::{canonicalize_esp32, NativePacketInput}, + observation::{CaptureProvenance, SourceSpec}, + recovery_store::RecoveryStore, +}; +use wifi_densepose_hardware::{Adr018Flags, PpduType}; +const START: u64 = 1_000_000_000_000; +const PERIOD: u64 = 1_000_000_000u64.div_ceil(33); +fn context() -> CaptureContext { + CaptureContext { + owner_scope: "a".repeat(64), + installation_id: "synthetic-runtime-temporal".into(), + room_digest: "b".repeat(64), + revision: 1, + sources: (1..=3) + .map(|source_id| SourceSpec { + source_id, + device_id: format!("esp32-node-{source_id}"), + grid_id: "ant1-bins64-ppdu0-usable-drop2-linear-v2".into(), + feature_count: 56, + signal_kind: ruview_unified::RfModality::WifiCsi, + center_frequency_hz: 2_437_000_000, + bandwidth_hz: 20_000_000, + link_id: format!("native-{source_id}"), + link_index: 0, + geometry: None, + }) + .collect(), + } +} +fn runtime(root: &Path, now: u64) -> RoomLearningRuntime { + let c = context(); + let directory = root.join(&c.owner_scope).join("SensingServer"); + let store = RecoveryStore::new( + &directory, + &c.owner_scope, + &c.installation_id, + Some(&[7; 32]), + ) + .unwrap(); + RoomLearningRuntime::new_with_recovery_at( + &directory, + &c.owner_scope, + &c.installation_id, + store, + now, + ) + .unwrap() +} +fn configured(root: &Path) -> RoomLearningRuntime { + let rt = runtime(root, START); + rt.configure_temporal_capture( + &rt.mutation_guard().unwrap(), + context(), + "synthetic-runtime-train".into(), + START, + ) + .unwrap(); + rt +} +fn packet(source: u8, seq: u32, at: u64) -> CanonicalPacket { + let iq = vec![8; 128]; + canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence: seq, + received_at_ns: at, + receiver_sample_rate_hz: 33., + iq_bytes: &iq, + flags: Adr018Flags::default(), + provenance: CaptureProvenance::LiveReceiverObserved, + }) + .unwrap() +} +fn snapshot(rt: &RoomLearningRuntime, now: u64) -> Value { + rt.temporal_capture_snapshot(now).unwrap()["capture"].clone() +} +fn start(rt: &RoomLearningRuntime, at: u64) -> u64 { + rt.start_temporal_phase(&rt.mutation_guard().unwrap(), at) + .unwrap(); + at + TRAIN_LEAD_IN_NS +} +fn feed(rt: &RoomLearningRuntime, mut at: u64, to: u64, seq: &mut u32) -> u64 { + while at <= to { + *seq += 1; + for source in 1..=3 { + rt.ingest_native(&packet(source, *seq, at), at).unwrap(); + } + at += PERIOD; + } + at +} +#[test] +fn private_temporal_setup_uses_real_native_runtime_and_public_configure_cannot_replace_it() { + let dir = tempfile::tempdir().unwrap(); + let rt = configured(dir.path()); + assert_eq!(snapshot(&rt, START)["stage"], "ready"); + assert_eq!(rt.context().unwrap(), context()); + assert!(rt.selects_grid(1, &context().sources[0].grid_id)); + let before = snapshot(&rt, START); + let guard = rt.mutation_guard().unwrap(); + assert!(rt + .configure(&guard, context(), "other".into(), true) + .unwrap_err() + .contains("unsupported_command")); + assert_eq!(snapshot(&rt, START)["rows_sha256"], before["rows_sha256"]); + assert_eq!( + rt.status(START)["error_code"], + "learner_temporal_capture_private_unsupported_command" + ); + assert!(rt.saved_candidates(&context().room_digest).is_err()); + assert!(rt.prediction(START).is_err()); + assert!(rt.qualify(&rt.mutation_guard().unwrap()).is_err()); + let first = start(&rt, START + 1); + let mut seq = 0; + feed(&rt, first, first + 3_000_000_000, &mut seq); + assert!( + snapshot(&rt, first + 3_000_000_000)["durable_windows"] + .as_u64() + .unwrap() + > 0 + ); + assert!(rt + .temporal_confirmation_challenge(&rt.mutation_guard().unwrap()) + .is_err()); +} +#[test] +fn full_seven_train_phases_require_fresh_confirmation_and_never_issue_fit_authority() { + let dir = tempfile::tempdir().unwrap(); + let rt = configured(dir.path()); + let mut seq = 0; + let mut now = START + 1; + let original = snapshot(&rt, START)["descriptor"].clone(); + for episode in 0..7 { + let before = snapshot(&rt, now); + assert_eq!(before["episode_index"], episode); + let target = before["target_ms"].as_u64().unwrap() * 1_000_000; + let first = start(&rt, now); + now = feed(&rt, first, first + target + PERIOD, &mut seq); + assert_eq!(snapshot(&rt, now)["stage"], "needs_confirmation"); + assert!(rt + .confirm_temporal_phase( + &rt.mutation_guard().unwrap(), + "not-a-current-challenge", + now + ) + .is_err()); + assert_eq!(snapshot(&rt, now)["sealed_train_episodes"], episode); + let expected = rt.mutation_guard().unwrap(); + let challenge = rt.temporal_confirmation_challenge(&expected).unwrap(); + rt.confirm_temporal_phase(&expected, &challenge, now) + .unwrap(); + assert_eq!(snapshot(&rt, now)["sealed_train_episodes"], episode + 1); + assert!(rt + .confirm_temporal_phase(&expected, &challenge, now) + .is_err()); + now += 1; + } + let done = snapshot(&rt, now); + assert_eq!(done["stage"], "complete"); + assert_eq!(done["durable_windows"], 64); + assert_eq!(done["descriptor"]["id"], original["id"]); + assert_eq!( + done["descriptor"]["expires_at_ms"], + original["expires_at_ms"] + ); + assert_eq!(done["fit_admission_available"], false); + let rt = Arc::new(rt); + assert!(rt + .start_training( + &rt.mutation_guard().unwrap(), + FitConfig::for_source_count(3) + ) + .is_err()); +} +#[test] +fn a_missing_required_source_between_retained_strata_withdraws_runtime_quality_terminally() { + let dir = tempfile::tempdir().unwrap(); + let rt = configured(dir.path()); + let mut seq = 0; + let first = start(&rt, START + 1); + let mut now = feed(&rt, first, first + 3_000_000_000, &mut seq); + let prior = snapshot(&rt, now)["durable_windows"].clone(); + let mut failed = false; + for _ in 0..5 { + seq += 1; + for source in 1..=2 { + if rt.ingest_native(&packet(source, seq, now), now).is_err() { + failed = true; + } + } + now += PERIOD; + } + assert!(failed); + assert_eq!(snapshot(&rt, now)["stage"], "failed"); + assert_eq!(snapshot(&rt, now)["durable_windows"], prior); + assert!(rt + .temporal_confirmation_challenge(&rt.mutation_guard().unwrap()) + .is_err()); + assert!(rt + .start_temporal_phase(&rt.mutation_guard().unwrap(), now) + .is_err()); +} +#[test] +fn all_unsupported_legacy_commands_are_explicit_and_cannot_change_temporal_rows() { + let dir = tempfile::tempdir().unwrap(); + let rt = Arc::new(configured(dir.path())); + let before = snapshot(&rt, START); + let request = || PhaseRequest { + trial_id: "legacy-trial".into(), + session_id: "legacy-session".into(), + role: DataRole::Train, + label: crate::room_learner::observation::RoomLabel::Empty, + duration_ms: 600_000, + lead_in_ms: 5_000, + conditions_confirmed: true, + }; + macro_rules! denied { + ($call:expr) => { + assert!($call.unwrap_err().contains("unsupported_command")); + }; + } + denied!(rt.begin_phase(&rt.mutation_guard().unwrap(), request(), START)); + denied!(rt.finish_phase(&rt.mutation_guard().unwrap(), START, true)); + denied!(rt.cancel_phase(&rt.mutation_guard().unwrap())); + denied!(rt.cancel_training(&rt.mutation_guard().unwrap())); + denied!(rt.start_training( + &rt.mutation_guard().unwrap(), + FitConfig::for_source_count(3) + )); + denied!(rt.evaluate(&rt.mutation_guard().unwrap(), DataRole::HeldOut)); + denied!(rt.qualify(&rt.mutation_guard().unwrap())); + denied!(rt.select_model( + &rt.mutation_guard().unwrap(), + &"a".repeat(64), + &"b".repeat(64) + )); + denied!(rt.deselect_model(&rt.mutation_guard().unwrap())); + denied!(rt.rollback_model(&rt.mutation_guard().unwrap())); + denied!(rt.restore_capture_recovery( + &rt.mutation_guard().unwrap(), + "old-id", + &context(), + START + )); + denied!(rt.discard_capture_recovery(&rt.mutation_guard().unwrap(), "old-id")); + denied!(rt.saved_recovery_context("old-id", START)); + denied!(rt.enable_current_capture_recovery( + &rt.mutation_guard().unwrap(), + "old-id", + true, + START + )); + assert!(rt + .paired_progress_snapshot( + &context().installation_id, + "public-runtime", + None, + &[], + None, + START + ) + .unwrap_err() + .contains("unsupported_command")); + assert!(rt + .selection_snapshot(START, Some(&context().room_digest)) + .integrity_error + .unwrap() + .contains("unsupported_command")); + assert!(matches!( + rt.selected_binary_evidence(START, Some(&context().room_digest)), + crate::room_learner::selection::SelectedBinaryEvidence::Unavailable { .. } + )); + assert_eq!(snapshot(&rt, START)["rows_sha256"], before["rows_sha256"]); + assert_eq!(snapshot(&rt, START)["descriptor"], before["descriptor"]); + assert!(rt.inner.lock().unwrap().model.is_none()); +} +#[test] +fn preparation_replay_phy_and_receiver_clock_cannot_rescue_or_relabel_a_capture() { + for case in ["preparation_replay", "native_phy", "receiver_reverse"] { + let dir = tempfile::tempdir().unwrap(); + let rt = configured(dir.path()); + let first = start(&rt, START + 1); + if case == "preparation_replay" { + rt.ingest_native( + &packet(1, 100, START + 1_000_000_000), + START + 1_000_000_000, + ) + .unwrap(); + assert!(rt.ingest_native(&packet(1, 99, first), first).is_err()); + } else { + let mut seq = 0; + let now = feed(&rt, first, first + 3_000_000_000, &mut seq); + if case == "native_phy" { + let mut p = packet(1, seq + 1, now); + p.native_frame.centre_frequency_hz += 1.; + assert!(rt.ingest_native(&p, now).is_err()); + } else { + assert!(rt + .tick_temporal_capture(&rt.mutation_guard().unwrap(), first) + .is_err()); + } + } + assert_eq!( + snapshot(&rt, first + 4_000_000_000)["stage"], + "failed", + "{case}" + ); + assert!( + rt.restore_temporal_capture( + &rt.mutation_guard().unwrap(), + &context(), + first + 5_000_000_000 + ) + .is_err(), + "same process cannot rewind floors: {case}" + ); + } +} +#[test] +fn actual_runtime_two_cold_resumes_keep_original_rows_trial_and_expiry_without_downtime_credit() { + let dir = tempfile::tempdir().unwrap(); + let mut rt = configured(dir.path()); + let original = snapshot(&rt, START)["descriptor"].clone(); + let mut seq = 0; + let mut now = START + 1; + let mut trial = None; + for _ in 0..2 { + let first = start(&rt, now); + now = feed(&rt, first, first + 100_000_000_000, &mut seq); + let phase = rt.mutation_guard().unwrap().phase.unwrap(); + if let Some(prior) = &trial { + assert_eq!(prior, &phase.trial_id); + } else { + trial = Some(phase.trial_id); + } + rt.pause_temporal_capture(&rt.mutation_guard().unwrap(), now) + .unwrap(); + let paused = snapshot(&rt, now); + let accepted = paused["durable_observed_ns"] + .as_str() + .unwrap() + .parse::() + .unwrap(); + assert!(accepted < 600_000_000_000); + let old_guard = rt.mutation_guard().unwrap(); + drop(rt); + now += 50_000_000_000; + rt = runtime(dir.path(), now); + assert!(rt + .restore_temporal_capture(&old_guard, &context(), now) + .is_err()); + rt.restore_temporal_capture(&rt.mutation_guard().unwrap(), &context(), now) + .unwrap(); + let restored = snapshot(&rt, now); + assert_eq!(restored["stage"], "paused"); + assert_eq!(restored["rows_sha256"], paused["rows_sha256"]); + assert_eq!( + restored["durable_observed_ns"], + paused["durable_observed_ns"] + ); + assert_eq!(restored["descriptor"]["id"], original["id"]); + assert_eq!( + restored["descriptor"]["expires_at_ms"], + original["expires_at_ms"] + ); + assert!(rt + .temporal_confirmation_challenge(&rt.mutation_guard().unwrap()) + .is_err()); + now += 1; + } + let accepted = snapshot(&rt, now)["durable_observed_ns"] + .as_str() + .unwrap() + .parse::() + .unwrap(); + let first = start(&rt, now); + now = feed( + &rt, + first, + first + 600_000_000_000 - accepted + PERIOD, + &mut seq, + ); + assert_eq!(snapshot(&rt, now)["stage"], "needs_confirmation"); + let guard = rt.mutation_guard().unwrap(); + let challenge = rt.temporal_confirmation_challenge(&guard).unwrap(); + rt.confirm_temporal_phase(&guard, &challenge, now).unwrap(); + assert_eq!(snapshot(&rt, now)["sealed_train_episodes"], 1); + assert_eq!(snapshot(&rt, now)["durable_windows"], 16); +} +#[test] +fn temporal_private_setup_refuses_wrong_owner_stale_generation_and_existing_legacy_work() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path(), START); + let mut wrong = context(); + wrong.owner_scope = "c".repeat(64); + assert!(rt + .configure_temporal_capture( + &rt.mutation_guard().unwrap(), + wrong, + "wrong-owner".into(), + START + ) + .is_err()); + let stale = rt.mutation_guard().unwrap(); + rt.configure(&stale, context(), "legacy-saved-work".into(), true) + .unwrap(); + let before = rt.status(START); + assert!(rt + .configure_temporal_capture(&stale, context(), "stale".into(), START) + .is_err()); + assert!(rt + .configure_temporal_capture( + &rt.mutation_guard().unwrap(), + context(), + "would-replace".into(), + START + ) + .is_err()); + let after = rt.status(START); + assert_eq!(before["phase_history"], after["phase_history"]); + assert_eq!(before["capture"], after["capture"]); + assert_eq!(before["model_selection"], after["model_selection"]); +} + +#[test] +fn real_initial_publish_with_lost_ack_stays_blocked_until_exact_runtime_repair() { + let dir = tempfile::tempdir().unwrap(); + let rt = runtime(dir.path(), START); + assert!(rt + .configure_temporal_capture_with( + &rt.mutation_guard().unwrap(), + context(), + "synthetic-lost-ack".into(), + START, + TemporalTrainCapture::synthetic_prepare_lost_initial_ack + ) + .is_err()); + let blocked = snapshot(&rt, START); + assert_eq!(blocked["stage"], "blocked"); + assert_eq!(blocked["durable_windows"], 0); + assert!(rt + .start_temporal_phase(&rt.mutation_guard().unwrap(), START + 1) + .is_err()); + assert!(rt + .configure( + &rt.mutation_guard().unwrap(), + context(), + "cannot-replace".into(), + true + ) + .is_err()); + rt.repair_temporal_capture(&rt.mutation_guard().unwrap(), START + 2) + .unwrap(); + let repaired = snapshot(&rt, START + 2); + assert_eq!(repaired["stage"], "paused"); + assert_eq!(repaired["descriptor"]["id"], blocked["descriptor"]["id"]); + assert_eq!( + repaired["descriptor"]["expires_at_ms"], + blocked["descriptor"]["expires_at_ms"] + ); + let first = start(&rt, START + 3); + let mut seq = 0; + feed(&rt, first, first + 3_000_000_000, &mut seq); + assert_eq!(snapshot(&rt, first + 3_000_000_000)["durable_windows"], 1); +} +#[tokio::test] +async fn private_temporal_setup_cannot_replace_an_actual_selected_legacy_candidate_or_history() { + let dir = tempfile::tempdir().unwrap(); + let rt = super::super::selection_tests::fitted(dir.path()).await; + let q = rt.qualify(&rt.mutation_guard().unwrap()).unwrap(); + rt.select_model( + &rt.mutation_guard().unwrap(), + &q.receipt.model_sha256, + &q.receipt_sha256, + ) + .unwrap(); + let selected_before = serde_json::to_vec(&rt.store.load_selection().unwrap()).unwrap(); + let model_before = rt + .inner + .lock() + .unwrap() + .model + .as_ref() + .unwrap() + .export() + .unwrap(); + let history_before = + serde_json::to_vec(rt.inner.lock().unwrap().legacy_capture().unwrap().history()).unwrap(); + let mut proposed = context(); + proposed.installation_id = rt.installation.clone(); + assert_eq!( + rt.configure_temporal_capture( + &rt.mutation_guard().unwrap(), + proposed, + "would-replace-control".into(), + rt.receiver_now_ns() + ) + .unwrap_err(), + "temporal_setup_would_replace_existing_work" + ); + assert_eq!( + serde_json::to_vec(&rt.store.load_selection().unwrap()).unwrap(), + selected_before + ); + assert_eq!( + rt.inner + .lock() + .unwrap() + .model + .as_ref() + .unwrap() + .export() + .unwrap(), + model_before + ); + assert_eq!( + serde_json::to_vec(rt.inner.lock().unwrap().legacy_capture().unwrap().history()).unwrap(), + history_before + ); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_readiness.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_readiness.rs index d489b209db..391912c85f 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_readiness.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_readiness.rs @@ -6,9 +6,10 @@ use crate::room_learner::{ }; impl RoomLearningRuntime { pub(super) fn qualification_readiness_locked(&self, state: &Inner, now_ns: u64) -> Value { - let retained = state - .capture - .as_ref() + if state.require_legacy_kind().is_err() { + return json!({"schema":"ruview.qualification-readiness.v1","eligible_for_held_out":false,"training_lineage_basis":"unavailable","managed_training_proof_sha256":null,"raw_train_rows_retained":false,"train_cohorts_per_condition":[0,0,0,0],"reason":"unsupported_capture_kind","detail":"Private temporal capture has no fitting or qualification authority."}); + } + let retained = state.legacy_capture() .is_some_and(|c| c.cohorts().iter().any(|r| r.role == DataRole::Train)); let result = (|| -> Result<(&str, [usize; 4], Option), (&str, String)> { let model = state.model.as_ref().ok_or(( @@ -28,7 +29,7 @@ impl RoomLearningRuntime { format!("Frozen candidate artifact could not be verified: {e}"), ) })?; - let capture = state.capture.as_ref().ok_or(( + let capture = state.legacy_capture().ok_or(( "context_mismatch", "No current managed capture context.".into(), ))?; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_tests.rs index 1dedc764b8..b4c92a88ef 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_training_tests.rs @@ -297,8 +297,7 @@ async fn corrupt_foreign_and_torn_metadata_never_replace_exact_train_checks() { &rt.inner .lock() .unwrap() - .capture - .as_ref() + .legacy_capture() .unwrap() .validated_dataset() .unwrap(), @@ -306,7 +305,7 @@ async fn corrupt_foreign_and_torn_metadata_never_replace_exact_train_checks() { ) .unwrap(); let state = rt.inner.lock().unwrap(); - let capture = state.capture.as_ref().unwrap(); + let capture = state.legacy_capture().unwrap(); let mut rows = capture.cohorts().to_vec(); rows.iter_mut() .find(|c| c.role == DataRole::Train) @@ -443,7 +442,7 @@ async fn failed_assessment_cold_resume_keeps_all_sealed_phase_fences_not_only_tr .as_str() .is_some()); let state = cold.inner.lock().unwrap(); - let capture = state.capture.as_ref().unwrap(); + let capture = state.legacy_capture().unwrap(); assert_eq!(capture.metadata_lineage(), Some(&fence.seed)); let last = &fence.seed.sources[0]; drop(state); @@ -747,15 +746,14 @@ async fn phase_seal_waits_for_durable_fence_and_pending_retry_keeps_original_met rt.inner .lock() .unwrap() - .capture - .as_ref() + .legacy_capture() .unwrap() .history() .len(), 8 ); crate::companion::atomic_private_write(&head, &old_head).unwrap(); - let mut completed = rt.inner.lock().unwrap().capture.as_ref().unwrap().clone(); + let mut completed = rt.inner.lock().unwrap().legacy_capture().unwrap().clone(); completed.finish_phase(end).unwrap(); let pending = crate::room_learner::managed_capture_fences::ManagedCaptureFence::issue( &proof, diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/paired_progress.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/paired_progress.rs index 084fcb2721..492518aeb3 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/paired_progress.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/paired_progress.rs @@ -46,6 +46,7 @@ impl RoomLearningRuntime { || room.as_ref().is_some_and(|r| !digest(&r.binding_digest) || r.revision==0 || r.revision>MAX_SAFE) { return Err("progress identity/time unavailable".into()); } let state=self.inner.lock().map_err(|_| "learner state unavailable")?; + state.require_legacy_kind()?; let mut result=PairedCalibrationProgress { schema:"ruview.room-learner.progress.v1",success:true,installation_id:installation_id.into(),runtime_epoch:runtime_epoch.into(), observed_at_unix_ms:observed,expires_at_unix_ms:observed+PAIRED_PROGRESS_TTL_MS,room, @@ -55,7 +56,7 @@ impl RoomLearningRuntime { save_status:"not_reported",blocker:Some("room_not_published"), }; let Some(published)=result.room.as_ref() else { return Ok(result); }; - let Some(capture)=state.capture.as_ref() else { result.blocker=Some("learner_not_configured");return Ok(result); }; + let Some(capture)=state.legacy_capture() else { result.blocker=Some("learner_not_configured");return Ok(result); }; let context=capture.context(); if context.installation_id!=installation_id || context.room_digest!=published.binding_digest { result.stage="blocked";result.blocker=Some("context_mismatch");return Ok(result); diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_acquisition.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_acquisition.rs index 23e8ad8b3a..4535a34a42 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_acquisition.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_acquisition.rs @@ -422,6 +422,58 @@ impl WindowInputPolicy { max_pairwise_receiver_skew_ms: skew, } } + /// Same stateless native-source checks for retained windows and whole-period + /// observation. This does not admit a window, establish replay freshness or + /// create a measured clock/teacher claim. + pub(crate) fn validate_packet_source( + &self, + p: &CanonicalPacket, + ) -> Result<(&SourceSpec, &SourceWindowPolicy), String> { + let spec = self + .context + .sources + .iter() + .find(|s| s.source_id == p.source_id) + .ok_or("acquisition_unbound_source")?; + let policy = self + .sources + .iter() + .find(|s| s.source_id == p.source_id) + .ok_or("acquisition_unbound_source")?; + if p.grid_id != spec.grid_id + || p.frequency_hz != spec.center_frequency_hz + || p.bandwidth_hz != spec.bandwidth_hz + || p.native_frame.modality != spec.signal_kind + || p.native_frame.provenance.device_id != spec.device_id + || p.native_frame.centre_frequency_hz != spec.center_frequency_hz as f64 + || p.native_frame.bandwidth_hz != spec.bandwidth_hz as f64 + || p.amplitudes.len() != spec.feature_count + || p.amplitudes + .iter() + .any(|x| !x.is_finite() || *x < 0. || *x > f32::MAX as f64) + || p.native_frame.frame_id == 0 + || p.clock.kind != ClockKind::ReceiverOnly + || p.clock.uncertainty_ns != u64::MAX + || p.phase_consumed + { + return Err("acquisition_native_source_changed".into()); + } + if let Some(g) = spec.geometry { + if p.native_frame.transmitter_pose.map(|p| p.position_m) != Some(g.tx_pos) + || p.native_frame.receiver_pose.map(|p| p.position_m) != Some(g.rx_pos) + { + return Err("acquisition_native_geometry_changed".into()); + } + } + let (lo, hi, _) = policy.receiver_cadence.range(); + if !p.receiver_sample_rate_hz.is_finite() + || p.receiver_sample_rate_hz < lo as f64 / 1000. + || p.receiver_sample_rate_hz > hi as f64 / 1000. + { + return Err("acquisition_cadence_changed".into()); + } + Ok((spec, policy)) + } pub fn context(&self) -> &CaptureContext { &self.context } @@ -734,52 +786,8 @@ impl NativeWindowAssembler { .as_mut() .filter(|a| !a.failed && !a.ready) .ok_or("acquisition_window_not_collecting")?; - let spec = self - .policy - .context - .sources - .iter() - .find(|s| s.source_id == p.source_id) - .ok_or("acquisition_unbound_source")?; - let policy = self - .policy - .sources - .iter() - .find(|s| s.source_id == p.source_id) - .ok_or("acquisition_unbound_source")?; + let (_spec, policy) = self.policy.validate_packet_source(p)?; let t = p.native_frame.timestamp_ns; - if p.grid_id != spec.grid_id - || p.frequency_hz != spec.center_frequency_hz - || p.bandwidth_hz != spec.bandwidth_hz - || p.native_frame.modality != spec.signal_kind - || p.native_frame.provenance.device_id != spec.device_id - || p.native_frame.centre_frequency_hz != spec.center_frequency_hz as f64 - || p.native_frame.bandwidth_hz != spec.bandwidth_hz as f64 - || p.amplitudes.len() != spec.feature_count - || p.amplitudes - .iter() - .any(|x| !x.is_finite() || *x < 0. || *x > f32::MAX as f64) - || p.native_frame.frame_id == 0 - || p.clock.kind != ClockKind::ReceiverOnly - || p.clock.uncertainty_ns != u64::MAX - || p.phase_consumed - { - return Err("acquisition_native_source_changed".into()); - } - if let Some(g) = spec.geometry { - if p.native_frame.transmitter_pose.map(|p| p.position_m) != Some(g.tx_pos) - || p.native_frame.receiver_pose.map(|p| p.position_m) != Some(g.rx_pos) - { - return Err("acquisition_native_geometry_changed".into()); - } - } - let (lo, hi, _) = policy.receiver_cadence.range(); - if !p.receiver_sample_rate_hz.is_finite() - || p.receiver_sample_rate_hz < lo as f64 / 1000. - || p.receiver_sample_rate_hz > hi as f64 / 1000. - { - return Err("acquisition_cadence_changed".into()); - } if t < a.target_ns || t >= a.end_ns { return Err("acquisition_packet_outside_slot".into()); } diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs new file mode 100644 index 0000000000..e6fd351dba --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs @@ -0,0 +1,884 @@ +//! Private Runtime capture facade. Whole-period receiver quality is engineering +//! admission only: never source-clock precision, teacher truth or fit authority. +use super::*; +use crate::room_learner::temporal_acquisition::WindowInputPolicy; +use serde_json::{json, Value}; + +pub(super) const BRIDGE_POLICY: &str = "native-receiver-whole-train-period-v1;all-selected-packets;existing-cadence-gap-arrival-skew;confirmed-durable-seals;no-fit-authority"; +#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub(super) enum Disposition { + Ready, + Observing, + Paused, + NeedsConfirmation, + Complete, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct SourceQuality { + pub(super) source_id: u8, + pub(super) first_ns: u64, + pub(super) last_ns: u64, + pub(super) first_sequence: u64, + pub(super) last_sequence: u64, + pub(super) packets: u64, + pub(super) maximum_gap_ns: u64, + pub(super) minimum_interval_ns: u64, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct QualityPart { + pub(super) mapping: ObservedSegmentMap, + pub(super) observed_end_ns: u64, + pub(super) sources: Vec, +} +impl QualityPart { + fn observe(&mut self, p: &CanonicalPacket) -> Result<(), String> { + let t = p.native_frame.timestamp_ns; + if let Some(s) = self.sources.iter_mut().find(|s| s.source_id == p.source_id) { + let gap = t.checked_sub(s.last_ns).ok_or("temporal_quality_clock")?; + s.maximum_gap_ns = s.maximum_gap_ns.max(gap); + s.minimum_interval_ns = s.minimum_interval_ns.min(gap); + s.last_ns = t; + s.last_sequence = u64::from(p.sequence); + s.packets = s.packets.checked_add(1).ok_or("temporal_quality_count")?; + } else { + self.sources.push(SourceQuality { + source_id: p.source_id, + first_ns: t, + last_ns: t, + first_sequence: u64::from(p.sequence), + last_sequence: u64::from(p.sequence), + packets: 1, + maximum_gap_ns: 0, + minimum_interval_ns: u64::MAX, + }); + self.sources.sort_by_key(|s| s.source_id); + } + Ok(()) + } + fn validate( + &self, + native: &WindowInputPolicy, + bound: &BoundEpisode, + now: u64, + ) -> Result<(), String> { + let start = self.mapping.observed_start_ns; + if self.observed_end_ns < start + || self.observed_end_ns > bound.episode().target_active_recording_ms * 1_000_000 + || self.mapping.receiver_start_ns > now + || self.sources.len() > native.source_policies().len() + { + return Err("temporal_quality_part_bounds".into()); + } + if self.observed_end_ns == start && !self.sources.is_empty() { + return Err("temporal_quality_uncommitted_rows".into()); + } + let mut prior = None; + for s in &self.sources { + let policy = native + .source_policies() + .iter() + .find(|p| p.source_id == s.source_id) + .ok_or("temporal_quality_source")?; + if prior.is_some_and(|p| p >= s.source_id) + || s.packets == 0 + || s.packets > 1_000_000 + || s.first_ns < self.mapping.receiver_start_ns + || s.last_ns < s.first_ns + || s.last_ns > now + || s.first_ns - self.mapping.receiver_start_ns + > native.max_first_packet_delay_ms() * 1_000_000 + || s.last_sequence < s.first_sequence + || s.last_sequence - s.first_sequence + 1 < s.packets + || (s.packets == 1 + && (s.first_ns != s.last_ns + || s.maximum_gap_ns != 0 + || s.minimum_interval_ns != u64::MAX)) + || (s.packets > 1 + && (s.last_ns == s.first_ns + || s.minimum_interval_ns < policy.min_packet_interval_ns + || s.maximum_gap_ns > policy.max_gap_ns + || s.minimum_interval_ns > s.maximum_gap_ns)) + { + return Err("temporal_quality_source_bounds".into()); + } + if self.observed_end_ns > start { + let end = self + .mapping + .receiver_start_ns + .checked_add(self.observed_end_ns - start) + .ok_or("temporal_quality_time")?; + let (lo, hi) = policy.receiver_cadence.rate_bounds_millihz(); + let intervals = s.packets.saturating_sub(1) as u128; + let span = (s.last_ns - s.first_ns) as u128; + // The exact integer packet/rate bounds already declared by the + // native policy, including its integer interval representation. + if s.packets < 2 + || s.last_ns < end + || intervals < span * u128::from(lo) / 1_000_000_000_000u128 + || span < intervals * u128::from(1_000_000_000_000u64 / hi) + { + return Err("temporal_whole_period_cadence".into()); + } + } + prior = Some(s.source_id); + } + if self.observed_end_ns > start + && self + .sources + .iter() + .map(|s| s.source_id) + .ne(native.source_policies().iter().map(|s| s.source_id)) + { + return Err("temporal_quality_required_sources".into()); + } + if let (Some(lo), Some(hi)) = ( + self.sources.iter().map(|s| s.first_ns).min(), + self.sources.iter().map(|s| s.first_ns).max(), + ) { + if hi - lo > native.max_pairwise_receiver_skew_ms() * 1_000_000 { + return Err("temporal_quality_source_skew".into()); + } + } + Ok(()) + } +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct TrainSeal { + pub(super) episode: usize, + pub(super) trial_id: String, + pub(super) context_sha256: String, + pub(super) schedule_sha256: String, + pub(super) row_prefix_sha256: String, + pub(super) quality_sha256: String, + pub(super) parts: Vec, + pub(super) confirmed_at_ns: u64, + pub(super) previous_sha256: Option, + pub(super) sha256: String, +} +impl TrainSeal { + fn digest(&self) -> Result { + hash(&( + BRIDGE_POLICY, + self.episode, + &self.trial_id, + &self.context_sha256, + &self.schedule_sha256, + &self.row_prefix_sha256, + &self.quality_sha256, + &self.parts, + self.confirmed_at_ns, + &self.previous_sha256, + )) + } +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct BridgeState { + pub(super) policy: String, + pub(super) disposition: Disposition, + pub(super) seals: Vec, + pub(super) parts: Vec, +} +impl BridgeState { + pub(super) fn new() -> Self { + Self { + policy: BRIDGE_POLICY.into(), + disposition: Disposition::Ready, + seals: vec![], + parts: vec![], + } + } + pub(super) fn validate( + &self, + schedule: &ManagedScheduleContract, + run: &str, + rows: &[CaptureCohort], + evidence: &[WindowEvidence], + ledger: &Ledger, + episode: usize, + mapping: Option<&ObservedSegmentMap>, + now: u64, + ) -> Result<(), String> { + let train = schedule + .episodes() + .iter() + .take_while(|e| e.role == DataRole::Train) + .count(); + if self.policy != BRIDGE_POLICY + || self.seals.len() > train + || episode != self.seals.len() + || episode > train + || self.parts.len() > 128 + || self.seals.iter().map(|s| s.parts.len()).sum::() + self.parts.len() + > 128 + train + { + return Err("temporal_bridge_phase_order".into()); + } + let mut count = 0; + let mut previous = None; + let mut last_confirm = 0; + for (i, s) in self.seals.iter().enumerate() { + count += schedule.episodes()[i].slots.len(); + let bound = schedule.bind_episode_start(run, i, 0, 0, None)?; + if s.episode != i + || s.trial_id != bound.trial_id() + || s.context_sha256 != schedule.context().digest() + || s.schedule_sha256 != schedule.sha256() + || count > rows.len() + || s.confirmed_at_ns < last_confirm + || s.confirmed_at_ns > now + || s.previous_sha256 != previous + || s.sha256 != s.digest()? + || s.quality_sha256 != hash(&s.parts)? + || s.row_prefix_sha256 + != hash(&(&rows[..count], &evidence[..count], &ledger.spans[..count]))? + { + return Err("temporal_train_seal_binding".into()); + } + validate_parts( + &s.parts, + schedule, + run, + i, + Some(schedule.episodes()[i].target_active_recording_ms * 1_000_000), + s.confirmed_at_ns, + )?; + validate_part_lineage(&s.parts, i, ledger, rows, None)?; + previous = Some(s.sha256.clone()); + last_confirm = s.confirmed_at_ns; + } + if episode == train { + if self.disposition != Disposition::Complete + || !self.parts.is_empty() + || mapping.is_some() + || rows.len() != schedule.counts().train_windows + { + return Err("temporal_bridge_complete_state".into()); + } + } else { + if rows.len() < count || rows.len() > count + schedule.episodes()[episode].slots.len() { + return Err("temporal_bridge_row_phase".into()); + } + validate_parts(&self.parts, schedule, run, episode, None, now)?; + validate_part_lineage(&self.parts, episode, ledger, rows, mapping)?; + if let Some(m) = mapping { + if self.parts.last().map(|p| &p.mapping) != Some(m) { + return Err("temporal_quality_current_mapping".into()); + } + } else if !self.parts.is_empty() || rows.len() != count { + return Err("temporal_bridge_mapping_missing".into()); + } + let accepted = ledger + .spans + .last() + .filter(|s| s.episode == episode) + .map_or(0, |s| s.observed_end_ns); + if self.parts.last().map_or(0, |p| p.observed_end_ns) != accepted { + return Err("temporal_quality_durable_frontier".into()); + } + if self.disposition == Disposition::Ready + && (mapping.is_some() || !self.parts.is_empty() || rows.len() != count) + { + return Err("temporal_bridge_ready_with_active_data".into()); + } + if matches!( + self.disposition, + Disposition::Observing | Disposition::NeedsConfirmation | Disposition::Paused + ) && mapping.is_none() + { + return Err("temporal_bridge_phase_without_mapping".into()); + } + if self.disposition == Disposition::Complete { + return Err("temporal_bridge_early_complete".into()); + } + if self.disposition == Disposition::NeedsConfirmation + && accepted != schedule.episodes()[episode].target_active_recording_ms * 1_000_000 + { + return Err("temporal_bridge_confirmation_before_end".into()); + } + } + Ok(()) + } +} +fn validate_part_lineage( + parts: &[QualityPart], + episode: usize, + ledger: &Ledger, + rows: &[CaptureCohort], + current: Option<&ObservedSegmentMap>, +) -> Result<(), String> { + for part in parts { + let spans = ledger + .spans + .iter() + .enumerate() + .filter(|(_, s)| s.episode == episode && s.mapping == part.mapping) + .collect::>(); + let frontier = spans + .last() + .map_or(part.mapping.observed_start_ns, |(_, s)| s.observed_end_ns); + let known = !spans.is_empty() + || current == Some(&part.mapping) + || ledger.gaps.iter().any(|g| { + g.episode == episode + && (g.new_mapping == part.mapping + || g.genesis + .as_ref() + .is_some_and(|v| v.previous_mapping == part.mapping)) + }); + if !known || part.observed_end_ns != frontier { + return Err("temporal_quality_lineage_changed".into()); + } + for source in &part.sources { + let mut retained = 0u64; + for (index, _) in &spans { + let row = rows.get(*index).ok_or("temporal_quality_row_missing")?; + let view = row + .views + .iter() + .find(|v| v.source_id == source.source_id) + .ok_or("temporal_quality_row_source")?; + retained = retained + .checked_add(view.samples.len() as u64) + .ok_or("temporal_quality_count")?; + if source.first_ns > view.first_time_ns + || source.last_ns < view.last_time_ns + || view + .sequence_ids + .first() + .is_none_or(|v| *v < source.first_sequence) + || view + .sequence_ids + .last() + .is_none_or(|v| *v > source.last_sequence) + { + return Err("temporal_quality_retained_identity".into()); + } + } + if source.packets < retained { + return Err("temporal_quality_retained_count".into()); + } + } + } + Ok(()) +} +fn validate_parts( + parts: &[QualityPart], + schedule: &ManagedScheduleContract, + run: &str, + episode: usize, + complete: Option, + now: u64, +) -> Result<(), String> { + let native = schedule.native_window_policy(); + let mut frontier = 0; + let mut ordinal = None; + let mut receiver_end = 0; + for p in parts { + let bound = + schedule.bind_episode_start(run, episode, 0, p.mapping.segment_ordinal, None)?; + if p.mapping.session_id != bound.session_id() + || (ordinal.is_none() && p.mapping.segment_ordinal != 0) + || p.mapping.observed_start_ns != frontier + || ordinal.is_some_and(|n| p.mapping.segment_ordinal != n + 1) + || p.mapping.receiver_start_ns < receiver_end + { + return Err("temporal_quality_segment_order".into()); + } + p.validate(&native, &bound, now)?; + frontier = p.observed_end_ns; + ordinal = Some(p.mapping.segment_ordinal); + receiver_end = p + .mapping + .receiver_start_ns + .checked_add(p.observed_end_ns - p.mapping.observed_start_ns) + .ok_or("temporal_quality_time")?; + } + if complete.is_some_and(|v| v != frontier) || complete.is_some() && parts.is_empty() { + return Err("temporal_quality_period_incomplete".into()); + } + Ok(()) +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum TemporalCaptureStage { + Ready, + Observing, + NeedsConfirmation, + Complete, + Paused, + Blocked, + Failed, +} +impl TemporalCaptureStage { + pub(crate) fn as_str(self) -> &'static str { + match self { + Self::Ready => "ready", + Self::Observing => "observing", + Self::NeedsConfirmation => "needs_confirmation", + Self::Complete => "complete", + Self::Paused => "paused", + Self::Blocked => "blocked", + Self::Failed => "failed", + } + } +} +pub(crate) struct TemporalTrainCapture { + workflow: TemporalWorkflow, + // The exact immutable contract decoded from this workflow's validated + // binding. Packet handling must not reparse its full JSON on every tick. + schedule: ManagedScheduleContract, + failure: Option, + durable_seals: usize, +} +impl TemporalTrainCapture { + pub(crate) fn prepare( + schedule: ManagedScheduleContract, + run: String, + config: FitConfig, + now: u64, + store: &RecoveryStore, + ) -> Result { + let (capture, error) = Self::prepare_retained(schedule, run, config, now, store)?; + if let Some(error) = error { + Err(error) + } else { + Ok(capture) + } + } + pub(crate) fn prepare_retained( + schedule: ManagedScheduleContract, + run: String, + config: FitConfig, + now: u64, + store: &RecoveryStore, + ) -> Result<(Self, Option), String> { + Self::prepare_with(schedule, run, config, now, store, |w, store| { + w.persist_initial(store) + }) + } + fn prepare_with( + schedule: ManagedScheduleContract, + run: String, + config: FitConfig, + now: u64, + store: &RecoveryStore, + initial: impl FnOnce(&mut TemporalWorkflow, &RecoveryStore) -> Result<(), String>, + ) -> Result<(Self, Option), String> { + let mut workflow = TemporalWorkflow::prepare(schedule, run, config, now)?; + let schedule = workflow.schedule()?; + workflow.bridge = Some(BridgeState::new()); + let error = initial(&mut workflow, store).err(); + if error.is_some() { + workflow.stage = Stage::BlockedDurability; + } + Ok(( + Self { + workflow, + schedule, + failure: None, + durable_seals: 0, + }, + error, + )) + } + #[cfg(test)] + pub(crate) fn synthetic_prepare_lost_initial_ack( + schedule: ManagedScheduleContract, + run: String, + config: FitConfig, + now: u64, + store: &RecoveryStore, + ) -> Result<(Self, Option), String> { + Self::prepare_with(schedule, run, config, now, store, |w, store| { + w.persist_with(store, false, |s, i, p, n| { + s.publish_temporal(i, Some(p), n)?; + Err("synthetic-lost-initial-ack-after-real-publish".into()) + }) + }) + } + pub(crate) fn context(&self) -> &CaptureContext { + self.workflow.context() + } + pub(crate) fn check_time(&mut self, now: u64) -> Result<(), String> { + self.ensure_live()?; + if now < self.workflow.last_now_ns + || now / 1_000_000 >= self.workflow.descriptor.expires_at_ms + { + self.invalidate("temporal_receiver_clock_or_expiry"); + return Err("temporal_receiver_clock_or_expiry".into()); + } + Ok(()) + } + pub(crate) fn prime_fence(&mut self, p: &CanonicalPacket, now: u64) -> Result<(), String> { + self.ensure_live()?; + if !matches!( + self.stage(), + TemporalCaptureStage::Ready | TemporalCaptureStage::Paused + ) { + return Err("temporal_priming_state".into()); + } + let result: Result<(), String> = (|| { + let w = &mut self.workflow; + w.check_clock(now)?; + w.session + .managed + .as_ref() + .ok_or("managed_session_required")? + .binding + .native + .validate_packet_source(p)?; + let t = p.native_frame.timestamp_ns; + if p.provenance != CaptureProvenance::LiveReceiverObserved + || p.native_frame.provenance.class + != ruview_unified::frame::ProvenanceClass::Measured + || t > now + || w.source_high.iter().any(|(id, seq, last)| { + *id == p.source_id && (u64::from(p.sequence) <= *seq || t <= *last) + }) + { + return Err("temporal_priming_source_or_fence".into()); + } + w.source_high.retain(|(id, _, _)| *id != p.source_id); + w.source_high.push((p.source_id, u64::from(p.sequence), t)); + w.source_high.sort(); + Ok(()) + })(); + if let Err(e) = &result { + self.invalidate(e); + } + result + } + + pub(crate) fn stage(&self) -> TemporalCaptureStage { + if self.failure.is_some() { + return TemporalCaptureStage::Failed; + } + let w = &self.workflow; + if w.stage == Stage::BlockedDurability { + return TemporalCaptureStage::Blocked; + } + if w.bridge + .as_ref() + .is_some_and(|b| b.disposition == Disposition::Complete) + { + return TemporalCaptureStage::Complete; + } + if w.stage == Stage::PausedNeedsReconfirmation { + return TemporalCaptureStage::Paused; + } + if self.phase_ready() { + return TemporalCaptureStage::NeedsConfirmation; + } + if w.stage == Stage::Observing { + TemporalCaptureStage::Observing + } else { + TemporalCaptureStage::Ready + } + } + pub(crate) fn phase_identity(&self) -> Option<(String, String)> { + let w = &self.workflow; + let schedule = &self.schedule; + let e = schedule.episodes().get(w.episode)?; + if e.role != DataRole::Train { + return None; + } + let bound = schedule + .bind_episode_start( + &w.session.run_id, + w.episode, + 0, + w.mapping.as_ref().map_or(0, |m| m.segment_ordinal), + None, + ) + .ok()?; + Some((bound.trial_id().into(), bound.session_id().into())) + } + fn phase_ready(&self) -> bool { + let w = &self.workflow; + let s = &self.schedule; + let Some(e) = s.episodes().get(w.episode) else { + return false; + }; + e.role == DataRole::Train + && w.accepted() == e.target_active_recording_ms * 1_000_000 + && w.pending_window.is_none() + && w.session.rows.len() + == s.episodes() + .iter() + .take(w.episode + 1) + .map(|e| e.slots.len()) + .sum::() + } + pub(crate) fn start_segment( + &mut self, + epoch: &str, + now: u64, + store: &RecoveryStore, + ) -> Result<(), String> { + self.ensure_live()?; + if self.phase_ready() { + return Err("temporal_confirmation_required".into()); + } + let w = &mut self.workflow; + let schedule = w.schedule()?; + if schedule + .episodes() + .get(w.episode) + .is_none_or(|e| e.role != DataRole::Train) + { + return Err("temporal_train_complete".into()); + } + let permit = ConditionsPermit { + context: w.context().digest(), + run: w.session.run_id.clone(), + durable_sha256: w.durable.as_ref().map(|b| bytes_hash(b)), + }; + w.start_segment(&permit, epoch, now, now)?; + let mapping = w.mapping.clone().ok_or("temporal_segment_missing")?; + let b = w.bridge.as_mut().ok_or("temporal_bridge_required")?; + b.disposition = Disposition::Observing; + b.parts.push(QualityPart { + observed_end_ns: mapping.observed_start_ns, + mapping, + sources: vec![], + }); + w.persist_segment(store)?; + w.begin_slot()?; + Ok(()) + } + pub(crate) fn ingest( + &mut self, + p: &CanonicalPacket, + now: u64, + store: &RecoveryStore, + ) -> Result<(), String> { + self.ensure_live()?; + if self.phase_ready() { + return Ok(()); + } + let result = (|| { + let w = &mut self.workflow; + let native = &w + .session + .managed + .as_ref() + .ok_or("managed_session_required")? + .binding + .native; + native.validate_packet_source(p)?; + if p.provenance != CaptureProvenance::LiveReceiverObserved + || p.native_frame.provenance.class + != ruview_unified::frame::ProvenanceClass::Measured + { + return Err("temporal_measured_receiver_required".into()); + } + w.observe(p, now)?; + w.bridge + .as_mut() + .ok_or("temporal_bridge_required")? + .parts + .last_mut() + .ok_or("temporal_quality_segment_missing")? + .observe(p)?; + self.commit_if_ready(store) + })(); + if let Err(e) = &result { + if self.workflow.stage != Stage::BlockedDurability { + self.invalidate(e); + } + } + result + } + pub(crate) fn tick(&mut self, now: u64, store: &RecoveryStore) -> Result<(), String> { + self.ensure_live()?; + if self.phase_ready() { + return self.workflow.check_clock(now); + } + let result = self + .workflow + .observe_tick(now) + .and_then(|_| self.commit_if_ready(store)); + if let Err(e) = &result { + if self.workflow.stage != Stage::BlockedDurability { + self.invalidate(e); + } + } + result + } + fn commit_if_ready(&mut self, store: &RecoveryStore) -> Result<(), String> { + let w = &mut self.workflow; + let Some(target) = w.pending_window.as_ref() else { + return Ok(()); + }; + if w.session.active + || w.source_high.len() != w.context().sources.len() + || w.source_high.iter().any(|(_, _, t)| *t < target.end_ns()) + { + return Ok(()); + } + let observed = target.identity().slot_index; + let schedule = w.schedule()?; + let deadline = + schedule.episodes()[w.episode].slots[observed].deadline_offset_ms * 1_000_000; + let b = w.bridge.as_mut().ok_or("temporal_bridge_required")?; + b.parts + .last_mut() + .ok_or("temporal_quality_segment_missing")? + .observed_end_ns = deadline; + if deadline == schedule.episodes()[w.episode].target_active_recording_ms * 1_000_000 { + b.disposition = Disposition::NeedsConfirmation; + } + w.commit_slot(store, w.last_now_ns)?; + if !self.phase_ready() { + self.workflow.begin_slot()?; + } + Ok(()) + } + pub(crate) fn confirmation_challenge(&self) -> Result { + self.ensure_live()?; + if !self.phase_ready() { + return Err("temporal_phase_not_complete".into()); + } + let w = &self.workflow; + let b = w.bridge.as_ref().ok_or("temporal_bridge_required")?; + hash(&( + BRIDGE_POLICY, + &w.descriptor.id, + &w.session.run_id, + w.episode, + w.durable.as_ref().map(|v| bytes_hash(v)), + &b.parts, + &b.seals, + )) + } + pub(crate) fn confirm( + &mut self, + challenge: &str, + store: &RecoveryStore, + now: u64, + ) -> Result<(), String> { + self.ensure_live()?; + if challenge != self.confirmation_challenge()? { + return Err("temporal_stale_conditions_challenge".into()); + } + let w = &mut self.workflow; + w.check_clock(now)?; + let schedule = w.schedule()?; + let context_sha256 = w.context().digest(); + let row_prefix_sha256 = hash(&(&w.session.rows, &w.session.evidence, &w.ledger.spans))?; + let b = w.bridge.as_mut().ok_or("temporal_bridge_required")?; + validate_parts( + &b.parts, + &schedule, + &w.session.run_id, + w.episode, + Some(schedule.episodes()[w.episode].target_active_recording_ms * 1_000_000), + now, + )?; + let bound = schedule.bind_episode_start(&w.session.run_id, w.episode, 0, 0, None)?; + let mut seal = TrainSeal { + episode: w.episode, + trial_id: bound.trial_id().into(), + context_sha256, + schedule_sha256: schedule.sha256().into(), + row_prefix_sha256, + quality_sha256: hash(&b.parts)?, + parts: b.parts.clone(), + confirmed_at_ns: now, + previous_sha256: b.seals.last().map(|s| s.sha256.clone()), + sha256: String::new(), + }; + seal.sha256 = seal.digest()?; + b.seals.push(seal); + b.parts.clear(); + w.episode += 1; + w.mapping = None; + w.custody = SegmentCustody::default(); + w.live_observed_ns = 0; + b.disposition = if w.episode == schedule.counts().train_episodes { + Disposition::Complete + } else { + Disposition::Ready + }; + w.stage = Stage::DurableReady; + w.persist_with(store, false, |s, i, p, n| s.publish_temporal(i, Some(p), n))?; + self.durable_seals = w.bridge.as_ref().unwrap().seals.len(); + Ok(()) + } + pub(crate) fn pause(&mut self, now: u64, store: &RecoveryStore) -> Result<(), String> { + self.ensure_live()?; + self.workflow.check_clock(now)?; + self.workflow.pause()?; + self.workflow + .bridge + .as_mut() + .ok_or("temporal_bridge_required")? + .disposition = Disposition::Paused; + self.workflow + .persist_with(store, false, |s, i, p, n| s.publish_temporal(i, Some(p), n)) + } + pub(crate) fn cold_restore( + store: &RecoveryStore, + context: &CaptureContext, + now: u64, + epoch: &str, + anchor: u64, + ) -> Result { + let w = TemporalWorkflow::cold_restore(store, context, now, epoch, anchor)?; + let b = w + .bridge + .as_ref() + .ok_or("temporal_bridge_checkpoint_required")?; + let n = b.seals.len(); + let schedule = w.schedule()?; + Ok(Self { + workflow: w, + schedule, + failure: None, + durable_seals: n, + }) + } + pub(crate) fn repair_pending(&mut self, store: &RecoveryStore, now: u64) -> Result<(), String> { + self.workflow.repair_pending(store, now)?; + if self.workflow.schedule()?.sha256() != self.schedule.sha256() { + self.invalidate("temporal_repaired_schedule_changed"); + return Err("temporal_repaired_schedule_changed".into()); + } + self.durable_seals = self + .workflow + .bridge + .as_ref() + .ok_or("temporal_bridge_required")? + .seals + .len(); + Ok(()) + } + pub(crate) fn invalidate(&mut self, reason: &str) { + self.failure = Some( + reason + .chars() + .filter(|c| !c.is_control()) + .take(256) + .collect(), + ); + } + fn ensure_live(&self) -> Result<(), String> { + if self.failure.is_some() { + Err("temporal_capture_failed".into()) + } else if self.workflow.stage == Stage::BlockedDurability { + Err("temporal_capture_durability_blocked".into()) + } else { + Ok(()) + } + } + pub(crate) fn snapshot(&self) -> Value { + let w = &self.workflow; + let e = self.schedule.episodes().get(w.episode); + json!({"schema":"ruview.room-learner.private-temporal-capture.v1","stage":self.stage().as_str(),"episode_index":w.episode,"label":e.filter(|e|e.role==DataRole::Train).map(|e|e.condition),"role":"Train","sealed_train_episodes":self.durable_seals,"retained_rows":w.session.rows.len(),"durable_windows":w.durable_windows,"rows_sha256":hash(&w.session.rows).ok(),"observed_ns":w.live_observed_ns.to_string(),"durable_observed_ns":w.accepted().to_string(),"target_ms":e.filter(|e|e.role==DataRole::Train).map(|e|e.target_active_recording_ms),"descriptor":w.descriptor,"failure":self.failure,"fit_admission_available":false,"quality_is_engineering_only":true,"independence_verified":false,"source_clock_measured":false}) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs index 25cf3e386c..bdc51261e1 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs @@ -20,6 +20,8 @@ struct Prefix { frozen_model: Option, quality_issued: bool, confirmation_issued: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + bridge: Option, } #[derive(Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -29,6 +31,7 @@ struct Envelope { payload: Prefix, } const SCHEMA: &str = "ruview.room-learner.temporal-runtime-prefix.v1"; +const BRIDGE_SCHEMA: &str = "ruview.room-learner.temporal-runtime-prefix.v2"; pub(super) fn encode(v: &TemporalWorkflow) -> Result, String> { if v.session.fitted || v.session.failed @@ -43,8 +46,13 @@ pub(super) fn encode(v: &TemporalWorkflow) -> Result, String> { .managed .as_ref() .ok_or("managed_session_required")?; + let schema = if v.bridge.is_some() { + BRIDGE_SCHEMA + } else { + SCHEMA + }; let payload = Prefix { - schema: SCHEMA.into(), + schema: schema.into(), binding: m.binding.clone(), run: v.session.run_id.clone(), descriptor: v.descriptor.clone(), @@ -60,9 +68,10 @@ pub(super) fn encode(v: &TemporalWorkflow) -> Result, String> { frozen_model: None, quality_issued: false, confirmation_issued: false, + bridge: v.bridge.clone(), }; let envelope = Envelope { - schema: SCHEMA.into(), + schema: schema.into(), payload_sha256: hash(&payload)?, payload, }; @@ -81,8 +90,9 @@ pub(super) fn decode( return Err("temporal_checkpoint_budget".into()); } let e: Envelope = serde_json::from_slice(bytes).map_err(|_| "temporal_checkpoint_schema")?; - if e.schema != SCHEMA - || e.payload.schema != SCHEMA + if !matches!(e.schema.as_str(), SCHEMA | BRIDGE_SCHEMA) + || e.payload.schema != e.schema + || (e.schema == SCHEMA) != e.payload.bridge.is_none() || e.payload_sha256 != hash(&e.payload)? || serde_json::to_vec(&e).map_err(|_| "temporal_checkpoint_encoding")? != bytes { @@ -142,7 +152,7 @@ pub(super) fn decode( if mapping.observed_start_ns != expected_start { return Err("temporal_checkpoint_mapping_start".into()); } - } else if !p.rows.is_empty() { + } else if !p.rows.is_empty() && p.bridge.as_ref().is_none_or(|b| b.seals.len() != p.episode) { return Err("temporal_checkpoint_mapping_missing".into()); } validate_records( @@ -336,6 +346,38 @@ pub(super) fn decode( return Err("temporal_gap_mapping_changed".into()); } } + if let Some(bridge) = &p.bridge { + bridge.validate( + &schedule, + &p.run, + &p.rows, + &p.evidence, + &p.ledger, + p.episode, + p.mapping.as_ref(), + p.last_now_ns, + )?; + // Whole-period quality also observes the reservation tail and packets + // between retained strata. Those durably acknowledged observations + // must not disappear from the replay floors merely because they are + // not samples in a retained native window. + for part in bridge + .seals + .iter() + .flat_map(|seal| &seal.parts) + .chain(&bridge.parts) + { + for observed in &part.sources { + if !p.source_high.iter().any(|(id, sequence, time)| { + *id == observed.source_id + && *sequence >= observed.last_sequence + && *time >= observed.last_ns + }) { + return Err("temporal_quality_floors_regressed".into()); + } + } + } + } let mut session = TemporalTrainingSession::managed(schedule.clone(), p.run, p.binding.fit.clone())?; session.rows = p.rows; @@ -384,6 +426,7 @@ pub(super) fn decode( commit_token: None, pending_intent: None, pending_prefix: None, + bridge: p.bridge, }) } #[cfg(test)] @@ -429,7 +472,11 @@ pub(crate) fn validated_prefix( bytes: bytes.to_vec(), descriptor: descriptor.clone(), run: v.session.run_id.clone(), - policy_sha256: v.session.plan_sha256().into(), + policy_sha256: if v.bridge.is_some() { + hash(&(v.session.plan_sha256(), bridge::BRIDGE_POLICY))? + } else { + v.session.plan_sha256().into() + }, window_count: v.session.rows.len(), transition_limit, saved_receiver_ns: v.last_now_ns, @@ -494,8 +541,76 @@ pub(crate) fn validate_extension( && b.mapping.is_some() && b.ledger.gaps.len() == a.ledger.gaps.len()) || (b.ledger.gaps.len() == a.ledger.gaps.len() + 1)); - if !window && !segment { - return Err("temporal_transition_without_window_or_segment".into()); + match (&a.bridge, &b.bridge) { + (None, None) => { + if !window && !segment { + return Err("temporal_transition_without_window_or_segment".into()); + } + } + (Some(old), Some(new)) => { + prefix!(old.seals, new.seals); + let seal = b.rows.len() == a.rows.len() + && b.episode == a.episode + 1 + && b.ledger.gaps == a.ledger.gaps + && b.mapping.is_none() + && new.seals.len() == old.seals.len() + 1 + && new.parts.is_empty() + && new + .seals + .last() + .is_some_and(|s| s.parts == old.parts && s.confirmed_at_ns >= a.last_now_ns); + let pause = b.rows.len() == a.rows.len() + && b.episode == a.episode + && b.mapping == a.mapping + && b.ledger.gaps == a.ledger.gaps + && new.seals == old.seals + && new.parts == old.parts + && new.disposition == bridge::Disposition::Paused; + if seal { + return Ok(()); + } + if new.seals != old.seals || b.episode != a.episode { + return Err("temporal_bridge_seal_transition".into()); + } + if window { + if new.parts.len() != old.parts.len() || old.parts.is_empty() { + return Err("temporal_quality_part_transition".into()); + } + let n = old.parts.len() - 1; + if old.parts[..n] != new.parts[..n] + || old.parts[n].mapping != new.parts[n].mapping + || new.parts[n].observed_end_ns < old.parts[n].observed_end_ns + { + return Err("temporal_quality_prefix_changed".into()); + } + for source in &old.parts[n].sources { + if !new.parts[n].sources.iter().any(|s| { + s.source_id == source.source_id + && s.first_ns == source.first_ns + && s.first_sequence == source.first_sequence + && s.packets >= source.packets + && s.last_ns >= source.last_ns + && s.last_sequence >= source.last_sequence + && s.maximum_gap_ns >= source.maximum_gap_ns + && s.minimum_interval_ns <= source.minimum_interval_ns + }) { + return Err("temporal_quality_counts_regressed".into()); + } + } + } else if segment { + if new.parts.len() != old.parts.len() + 1 + || new.parts[..old.parts.len()] != old.parts + || new.parts.last().is_none_or(|p| { + !p.sources.is_empty() || p.observed_end_ns != p.mapping.observed_start_ns + }) + { + return Err("temporal_quality_segment_changed".into()); + } + } else if !pause { + return Err("temporal_bridge_transition_invalid".into()); + } + } + _ => return Err("temporal_bridge_kind_changed".into()), } Ok(()) } @@ -538,6 +653,9 @@ pub(crate) fn is_new_empty_run(p: &ValidatedTemporalPrefix) -> Result *view.sequence_ids.last().unwrap(), + "the actual reservation tail must include packets after the retained native horizon" + ); + let floor = envelope + .payload + .source_high + .iter_mut() + .find(|s| s.0 == view.source_id) + .unwrap(); + floor.1 = *view.sequence_ids.last().unwrap(); + floor.2 = view.last_time_ns; + envelope.payload_sha256 = hash(&envelope.payload).unwrap(); + let bytes = serde_json::to_vec(&envelope).unwrap(); + assert!( + validated_prefix(&bytes, prefix.descriptor(), now).is_err(), + "rehashed valid rows cannot hide replay-floor rollback below the observed quality tail" + ); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs index ce3a8f5ba8..f8f082b41e 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs @@ -10,6 +10,11 @@ use crate::room_learner::{ temporal_acquisition::{AssemblerCheckpoint, ManagedWindowTarget}, }; use serde::{Deserialize, Serialize}; +#[path = "temporal_capture_quality.rs"] +mod bridge; +use bridge::BridgeState; +pub(crate) use bridge::{TemporalCaptureStage, TemporalTrainCapture}; + pub(crate) const EXECUTION_POLICY:&str="observed-complete-reservation-prefix-v1;paused-time;immutable-spans;partial-tail-drop;max128segments;persegment-native-firstarrival-skew-gaps;explicit-genesis-gap-anchor;quality-unimplemented;independence-false"; #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] @@ -182,6 +187,7 @@ struct TemporalWorkflow { commit_token: Option, pending_intent: Option, pending_prefix: Option, + bridge: Option, } impl TemporalWorkflow { fn context(&self) -> &CaptureContext { @@ -214,7 +220,6 @@ impl TemporalWorkflow { self.last_now_ns = now; Ok(()) } - #[cfg(test)] fn prepare( schedule: ManagedScheduleContract, run: String, @@ -253,6 +258,7 @@ impl TemporalWorkflow { commit_token: None, pending_intent: None, pending_prefix: None, + bridge: None, }) } #[cfg(test)] @@ -299,7 +305,13 @@ impl TemporalWorkflow { observed_start_ns: self.accepted(), }; if let Some(old) = &self.mapping { - let parent = self.prefix_hash()?; + let parent = self + .ledger + .spans + .last() + .filter(|s| s.episode == self.episode) + .map(hash) + .transpose()?; let genesis = if parent.is_none() { Some(GenesisGap::new( &self.descriptor, @@ -316,6 +328,7 @@ impl TemporalWorkflow { .ledger .spans .last() + .filter(|s| s.episode == self.episode) .map_or(paused_receiver_ns, |s| s.receiver_end_ns); self.ledger.gaps.push(Gap { episode: self.episode, diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution_tests.rs index 6ab546e9e4..266e8c897d 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution_tests.rs @@ -705,3 +705,253 @@ fn cold_restore_cannot_ack_visible_saved_rows_when_recertification_fails() { original ); } + +pub(super) fn bridge_packet(source: u8, sequence: u32, time: u64) -> CanonicalPacket { + // Synthetic bytes go through the actual receiver canonicalizer. The live + // provenance arm is exercised as it would be by the real UDP dispatcher; + // this is not a physical observation or user-owned capture. + use crate::room_learner::native::{canonicalize_esp32, NativePacketInput}; + use wifi_densepose_hardware::{Adr018Flags, PpduType}; + canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence, + received_at_ns: time, + receiver_sample_rate_hz: 33., + iq_bytes: &[7; 128], + flags: Adr018Flags::default(), + provenance: CaptureProvenance::LiveReceiverObserved, + }) + .unwrap() +} +pub(super) fn bridge_setup() -> (tempfile::TempDir, RecoveryStore, TemporalTrainCapture) { + let d = tempfile::tempdir().unwrap(); + let c = context(); + let store = RecoveryStore::new( + &d.path().join(&c.owner_scope).join("SensingServer"), + &c.owner_scope, + &c.installation_id, + Some(&[7; 32]), + ) + .unwrap(); + let schedule = + ManagedScheduleContract::build(c, LeadInPolicy::Train5HeldOut15EngineeringV1).unwrap(); + let capture = TemporalTrainCapture::prepare( + schedule, + "synthetic-runtime-bridge".into(), + FitConfig { + epochs: 2, + ..Default::default() + }, + 1_000_000_000, + &store, + ) + .unwrap(); + (d, store, capture) +} +pub(super) fn drive_bridge( + c: &mut TemporalTrainCapture, + store: &RecoveryStore, + from: u64, + to: u64, + sequence: &mut u32, +) -> u64 { + let period = 1_000_000_000u64.div_ceil(33); + let mut at = from; + while at <= to + period && c.stage() == TemporalCaptureStage::Observing { + *sequence += 1; + for source in 1..=3 { + let t = at + u64::from(source) * 1000; + c.ingest(&bridge_packet(source, *sequence, t), t, store) + .unwrap(); + } + at += period; + } + at +} +#[test] +fn private_bridge_priming_preserves_replay_floors_without_observed_credit() { + let (_d, store, mut capture) = bridge_setup(); + capture + .prime_fence(&bridge_packet(1, 100, 2_000_000_000), 2_000_000_000) + .unwrap(); + assert_eq!(capture.snapshot()["durable_windows"], 0); + assert_eq!(capture.snapshot()["durable_observed_ns"], "0"); + capture + .start_segment("receiver-one", 7_000_000_000, &store) + .unwrap(); + let error = capture + .ingest(&bridge_packet(1, 99, 7_000_001_000), 7_000_001_000, &store) + .unwrap_err(); + assert!(error.contains("fence")); + assert_eq!(capture.stage(), TemporalCaptureStage::Failed); + assert!(capture.confirmation_challenge().is_err()); +} +#[test] +fn private_bridge_clock_validation_does_not_retime_exact_lead_in_start() { + let (_d, store, mut capture) = bridge_setup(); + capture.check_time(7_000_001_000).unwrap(); + capture + .start_segment("receiver-one", 7_000_000_000, &store) + .unwrap(); + assert!(capture.check_time(6_999_999_999).is_err()); + assert_eq!(capture.stage(), TemporalCaptureStage::Failed); +} +#[test] +fn private_bridge_detects_missing_source_between_sparse_slots_terminally() { + let (_d, store, mut capture) = bridge_setup(); + capture + .start_segment("receiver-one", 2_000_000_000, &store) + .unwrap(); + let mut sequence = 0; + let next = drive_bridge( + &mut capture, + &store, + 2_000_000_000, + 5_000_000_000, + &mut sequence, + ); + assert_eq!(capture.snapshot()["durable_windows"], 1); + assert!(capture.tick(next + 100_000_000, &store).is_err()); + assert_eq!(capture.stage(), TemporalCaptureStage::Failed); + assert!(capture.confirmation_challenge().is_err()); +} +#[test] +fn private_bridge_observation_reuses_native_validation_outside_retained_slots() { + let (_d, store, mut capture) = bridge_setup(); + capture + .start_segment("receiver-one", 2_000_000_000, &store) + .unwrap(); + let mut sequence = 0; + let next = drive_bridge( + &mut capture, + &store, + 2_000_000_000, + 5_000_000_000, + &mut sequence, + ); + let mut changed = bridge_packet(1, sequence + 1, next + 1000); + changed.native_frame.centre_frequency_hz += 1.; + assert!(capture.ingest(&changed, next + 1000, &store).is_err()); + assert_eq!(capture.stage(), TemporalCaptureStage::Failed); +} +#[test] +fn private_bridge_full_train_schedule_requires_fresh_seal_each_episode() { + let (_d, store, mut capture) = bridge_setup(); + let mut at = 2_000_000_000; + let mut sequence = 0; + let created = capture.snapshot()["descriptor"]["created_at_ms"].clone(); + let expires = capture.snapshot()["descriptor"]["expires_at_ms"].clone(); + let mut previous_challenge: Option = None; + for episode in 0..7 { + assert!(matches!( + capture.stage(), + TemporalCaptureStage::Ready | TemporalCaptureStage::Paused + )); + assert!(capture.confirmation_challenge().is_err()); + capture + .start_segment(&format!("receiver-{episode}"), at, &store) + .unwrap(); + let duration = if episode == 0 { + 600_000_000_000 + } else { + 30_000_000_000 + }; + at = drive_bridge(&mut capture, &store, at, at + duration, &mut sequence); + assert_eq!(capture.stage(), TemporalCaptureStage::NeedsConfirmation); + if episode == 0 { + let before = capture.snapshot(); + drop(capture); + capture = + TemporalTrainCapture::cold_restore(&store, &context(), at, "fresh-runtime", at) + .unwrap(); + assert_eq!(capture.stage(), TemporalCaptureStage::Paused); + assert_eq!(capture.snapshot()["rows_sha256"], before["rows_sha256"]); + assert_eq!(capture.snapshot()["sealed_train_episodes"], 0); + assert!(capture.start_segment("not-a-new-slot", at, &store).is_err()); + assert!(capture.confirmation_challenge().is_ok(), "full saved but unconfirmed phase needs a fresh owner confirmation, not recollection"); + } + if let Some(old) = previous_challenge.as_ref() { + assert!(capture.confirm(old, &store, at).is_err()); + } + let challenge = capture.confirmation_challenge().unwrap(); + capture.confirm(&challenge, &store, at).unwrap(); + previous_challenge = Some(challenge); + assert_eq!(capture.snapshot()["sealed_train_episodes"], episode + 1); + assert_eq!(capture.snapshot()["descriptor"]["created_at_ms"], created); + assert_eq!(capture.snapshot()["descriptor"]["expires_at_ms"], expires); + at += 5_000_000_000; + } + assert_eq!(capture.stage(), TemporalCaptureStage::Complete); + assert_eq!(capture.snapshot()["durable_windows"], 64); + assert_eq!(capture.snapshot()["fit_admission_available"], false); + assert!(capture.start_segment("no-held-out", at, &store).is_err()); +} + +#[test] +fn private_bridge_retains_initial_intent_until_exact_repair_after_real_lost_ack() { + let dir = tempfile::tempdir().unwrap(); + let c = context(); + let store = RecoveryStore::new( + &dir.path().join(&c.owner_scope).join("SensingServer"), + &c.owner_scope, + &c.installation_id, + Some(&[7; 32]), + ) + .unwrap(); + let schedule = + ManagedScheduleContract::build(c, LeadInPolicy::Train5HeldOut15EngineeringV1).unwrap(); + let (mut capture, error) = TemporalTrainCapture::synthetic_prepare_lost_initial_ack( + schedule, + "synthetic-retained-initial".into(), + FitConfig { + epochs: 2, + ..Default::default() + }, + 1_000_000_000, + &store, + ) + .unwrap(); + assert!(error.is_some()); + assert_eq!(capture.stage(), TemporalCaptureStage::Blocked); + assert!(capture.confirmation_challenge().is_err()); + assert!(capture + .start_segment("never", 2_000_000_000, &store) + .is_err()); + capture.repair_pending(&store, 2_000_000_000).unwrap(); + assert_eq!(capture.stage(), TemporalCaptureStage::Paused); + assert_eq!(capture.snapshot()["durable_windows"], 0); + assert_eq!(capture.snapshot()["sealed_train_episodes"], 0); + capture + .start_segment("explicit-after-repair", 3_000_000_000, &store) + .unwrap(); + assert_eq!(capture.stage(), TemporalCaptureStage::Observing); +} + +#[test] +fn private_bridge_preserves_native_first_sequence_zero_semantics() { + let (_dir, store, mut capture) = bridge_setup(); + let start = 2_000_000_000; + capture + .start_segment("zero-sequence", start, &store) + .unwrap(); + for source in 1..=3 { + let at = start + u64::from(source) * 1000; + capture + .ingest(&bridge_packet(source, 0, at), at, &store) + .unwrap(); + } + let mut sequence = 0; + drive_bridge( + &mut capture, + &store, + start + 1_000_000_000u64.div_ceil(33), + start + 3_000_000_000, + &mut sequence, + ); + assert_eq!(capture.snapshot()["durable_windows"], 1); + assert_eq!(capture.stage(), TemporalCaptureStage::Observing); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_tests.rs index d3dc29246e..afd56546ca 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_tests.rs @@ -437,3 +437,10 @@ fn post_assembler_evidence_failure_is_terminal_and_preserves_previous_valid_wind assert!(session.finish_window(now).is_err()); assert!(session.fit().is_err()); } + +#[test] +fn public_seed_v2_export_byte_probe(){ + let bytes=fitted_bytes();let digest=bytes_hash(&bytes); + assert_eq!(digest,"e46e9aff391b0e41a8fe26cc46789ee65d130f109df444726893303960b5735f"); + println!("PUBLIC_SEED_V2_SHA256={digest} BYTES={}",bytes.len()); +} From 780eb7ac203cde967ad60fa0ebf322e63cc6486f Mon Sep 17 00:00:00 2001 From: Codex for Dragan Spiridonov Date: Fri, 6 Nov 2026 15:22:35 +0100 Subject: [PATCH 068/100] test(learner): freeze portable seed V2 artifact compatibility --- .../temporal_model_input_tests.rs | 48 +++++++++++++++++-- .../temporal-seed-v2-synthetic-golden.json | 1 + 2 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/tests/fixtures/temporal-seed-v2-synthetic-golden.json diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_tests.rs index afd56546ca..84b44bfcd1 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_tests.rs @@ -108,6 +108,19 @@ fn fitted_bytes() -> Vec { assert_eq!(report.training_cohorts, 16); assert!(report.backbone_delta_l2 > 0.); assert!(report.parameter_delta_l2 > 0.); + assert_eq!(report.completed_epochs, config().epochs); + assert!(report.objective_evaluations > 0); + assert!(report.initial.total.is_finite()); + assert!(report.final_objective.total.is_finite()); + assert_eq!(model.model.training_manifest().training_cohorts, 16); + assert_eq!(model.evidence.len(), 16); + for (index, evidence) in model.evidence.iter().enumerate() { + assert_eq!(evidence.role, DataRole::Train); + assert_eq!(evidence.label, s.plan.schedule()[index / 2].label); + assert_eq!(evidence.target_ns, + (10_000_000_000u64 + index as u64 * 3_000_000_000).to_string()); + assert_eq!(evidence.sources.len(), 2); + } assert_eq!(model.model.encoder_config(), s.plan.encoder_config()); // Frozen fit precedes new HeldOut windows. A later fit cannot consume them. for index in 16..32 { @@ -438,9 +451,36 @@ fn post_assembler_evidence_failure_is_terminal_and_preserves_previous_valid_wind assert!(session.fit().is_err()); } +// Frozen SYNTHETIC pre-C1 artifact produced by the actual native-horizon fitter +// on base #2465. Compatibility means preserving its serialized bytes, not +// requiring newly trained floating-point weights to match across platforms. +const PUBLIC_SEED_V2_GOLDEN: &[u8] = + include_bytes!("../../tests/fixtures/temporal-seed-v2-synthetic-golden.json"); + +#[test] +fn public_seed_v2_frozen_artifact_import_reexports_exact_bytes() { + assert_eq!(PUBLIC_SEED_V2_GOLDEN.len(), 113_803); + assert_eq!( + bytes_hash(PUBLIC_SEED_V2_GOLDEN), + "e46e9aff391b0e41a8fe26cc46789ee65d130f109df444726893303960b5735f" + ); + let frozen = TemporalModel::import(PUBLIC_SEED_V2_GOLDEN).unwrap(); + assert_eq!(frozen.context(), &context(2)); + assert_eq!(frozen.model.training_manifest().training_cohorts, 16); + assert_eq!(frozen.model.training_manifest().training_provenance, + vec![CaptureProvenance::Synthetic]); + assert_eq!(frozen.evidence.len(), 16); + assert!(frozen.evidence.iter().all(|e| e.role == DataRole::Train)); + assert_eq!(frozen.export().unwrap(), PUBLIC_SEED_V2_GOLDEN); +} + #[test] -fn public_seed_v2_export_byte_probe(){ - let bytes=fitted_bytes();let digest=bytes_hash(&bytes); - assert_eq!(digest,"e46e9aff391b0e41a8fe26cc46789ee65d130f109df444726893303960b5735f"); - println!("PUBLIC_SEED_V2_SHA256={digest} BYTES={}",bytes.len()); +fn public_seed_v2_frozen_artifact_rehashed_role_tamper_is_rejected() { + let mut envelope: Envelope = serde_json::from_slice(PUBLIC_SEED_V2_GOLDEN).unwrap(); + envelope.payload.evidence[0].role = DataRole::HeldOut; + // Recompute the generic checksums: the real policy must still reject an + // artifact whose historical Train evidence was relabelled as HeldOut. + envelope.payload.input_evidence_sha256 = hash(&envelope.payload.evidence).unwrap(); + let altered = recode(envelope); + assert!(TemporalModel::import(&altered).is_err()); } diff --git a/v2/crates/wifi-densepose-sensing-server/tests/fixtures/temporal-seed-v2-synthetic-golden.json b/v2/crates/wifi-densepose-sensing-server/tests/fixtures/temporal-seed-v2-synthetic-golden.json new file mode 100644 index 0000000000..f417b81076 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/tests/fixtures/temporal-seed-v2-synthetic-golden.json @@ -0,0 +1 @@ +{"schema":"ruview.experimental.temporal-model-input.v2","payload_sha256":"040f7ca8174a6338f329ed7591c6c6a39441463fcb2a530e35d05605313cf405","payload":{"binding":{"plan_id":"Native2000msSlots2","native_frame_identity":"existing-esp32-receiver-ns-node-u32-sequence-v1","plan_sha256":"db70ffda37461a5ae5f46dfb4dd21cf200bb202d8d88e189e0716c6caa35b1c5","context":{"owner_scope":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","installation_id":"synthetic-temporal","room_digest":"synthetic-room","revision":1,"sources":[{"source_id":1,"device_id":"esp32-node-1","grid_id":"ant1-bins64-ppdu0-usable-drop2-linear-v2","feature_count":56,"signal_kind":"WifiCsi","center_frequency_hz":2437000000,"bandwidth_hz":20000000,"link_id":"native-1","link_index":0,"geometry":null},{"source_id":2,"device_id":"esp32-node-2","grid_id":"ant1-bins64-ppdu0-usable-drop2-linear-v2","feature_count":56,"signal_kind":"WifiCsi","center_frequency_hz":2437000000,"bandwidth_hz":20000000,"link_id":"native-2","link_index":0,"geometry":null}]},"encoder":{"bins":56,"source_count":2,"temporal_tokens":4,"hidden_dim":3,"embedding_dim":3,"temperature":0.3},"target_span_ms":2000,"max_span_ms":2050,"max_pairwise_skew_ms":50,"max_first_delay_ms":100,"sources":[{"source_id":1,"band":"Hz30To36","min_rows":61,"max_rows":77,"min_interval_ns":27777777,"max_gap_ns":50000000},{"source_id":2,"band":"Hz30To36","min_rows":61,"max_rows":77,"min_interval_ns":27777777,"max_gap_ns":50000000}]},"run_id":"synthetic-fit","training_data_sha256":"fe5579c3ff4e5853f1885dba20f1adc996e198d4c506c95a971e6afda6e6d9f0","input_evidence_sha256":"d8dcf49baea50b2f55b42576df3cf6333b6f33af191815d060381f023c4a7420","evidence":[{"cohort_id":"synthetic-fit-v2-r0-c0-t0-0","trial_id":"synthetic-fit-v2-r0-c0-t0","session_id":"synthetic-fit-v2-r0-c0-t0","role":"Train","label":"Empty","cohort_sha256":"74c046635820116696ad6c3262b74c115131f9bd33feee46a6caeecdd023e319","target_ns":"10000000000","sealed_at_ns":"12000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["10000001000","10030304031","10060607062","10090910093","10121213124","10151516155","10181819186","10212122217","10242425248","10272728279","10303031310","10333334341","10363637372","10393940403","10424243434","10454546465","10484849496","10515152527","10545455558","10575758589","10606061620","10636364651","10666667682","10696970713","10727273744","10757576775","10787879806","10818182837","10848485868","10878788899","10909091930","10939394961","10969697992","11000001023","11030304054","11060607085","11090910116","11121213147","11151516178","11181819209","11212122240","11242425271","11272728302","11303031333","11333334364","11363637395","11393940426","11424243457","11454546488","11484849519","11515152550","11545455581","11575758612","11606061643","11636364674","11666667705","11696970736","11727273767","11757576798","11787879829","11818182860","11848485891","11878788922","11909091953","11939394984","11969698015","12000001046"],"frame_ids":["10995117377275922743297","11028435912217279332354","11061754447158635921411","11095072982099992510468","11128391517041349099525","11161710051982705688582","11195028586924062277639","11228347121865418866696","11261665656806775455753","11294984191748132044810","11328302726689488633867","11361621261630845222924","11394939796572201811981","11428258331513558401038","11461576866454914990095","11494895401396271579152","11528213936337628168209","11561532471278984757266","11594851006220341346323","11628169541161697935380","11661488076103054524437","11694806611044411113494","11728125145985767702551","11761443680927124291608","11794762215868480880665","11828080750809837469722","11861399285751194058779","11894717820692550647836","11928036355633907236893","11961354890575263825950","11994673425516620415007","12027991960457977004064","12061310495399333593121","12094629030340690182178","12127947565282046771235","12161266100223403360292","12194584635164759949349","12227903170106116538406","12261221705047473127463","12294540239988829716520","12327858774930186305577","12361177309871542894634","12394495844812899483691","12427814379754256072748","12461132914695612661805","12494451449636969250862","12527769984578325839919","12561088519519682428976","12594407054461039018033","12627725589402395607090","12661044124343752196147","12694362659285108785204","12727681194226465374261","12760999729167821963318","12794318264109178552375","12827636799050535141432","12860955333991891730489","12894273868933248319546","12927592403874604908603","12960910938815961497660","12994229473757318086717","13027548008698674675774","13060866543640031264831","13094185078581387853888","13127503613522744442945","13160822148464101032002","13194140683405457621059"],"sequence_ids":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["10000002000","10030305031","10060608062","10090911093","10121214124","10151517155","10181820186","10212123217","10242426248","10272729279","10303032310","10333335341","10363638372","10393941403","10424244434","10454547465","10484850496","10515153527","10545456558","10575759589","10606062620","10636365651","10666668682","10696971713","10727274744","10757577775","10787880806","10818183837","10848486868","10878789899","10909092930","10939395961","10969698992","11000002023","11030305054","11060608085","11090911116","11121214147","11151517178","11181820209","11212123240","11242426271","11272729302","11303032333","11333335364","11363638395","11393941426","11424244457","11454547488","11484850519","11515153550","11545456581","11575759612","11606062643","11636365674","11666668705","11696971736","11727274767","11757577798","11787880829","11818183860","11848486891","11878789922","11909092953","11939395984","11969699015","12000002046"],"frame_ids":["10995118476791845486593","11028437011733202075650","11061755546674558664707","11095074081615915253764","11128392616557271842821","11161711151498628431878","11195029686439985020935","11228348221381341609992","11261666756322698199049","11294985291264054788106","11328303826205411377163","11361622361146767966220","11394940896088124555277","11428259431029481144334","11461577965970837733391","11494896500912194322448","11528215035853550911505","11561533570794907500562","11594852105736264089619","11628170640677620678676","11661489175618977267733","11694807710560333856790","11728126245501690445847","11761444780443047034904","11794763315384403623961","11828081850325760213018","11861400385267116802075","11894718920208473391132","11928037455149829980189","11961355990091186569246","11994674525032543158303","12027993059973899747360","12061311594915256336417","12094630129856612925474","12127948664797969514531","12161267199739326103588","12194585734680682692645","12227904269622039281702","12261222804563395870759","12294541339504752459816","12327859874446109048873","12361178409387465637930","12394496944328822226987","12427815479270178816044","12461134014211535405101","12494452549152891994158","12527771084094248583215","12561089619035605172272","12594408153976961761329","12627726688918318350386","12661045223859674939443","12694363758801031528500","12727682293742388117557","12761000828683744706614","12794319363625101295671","12827637898566457884728","12860956433507814473785","12894274968449171062842","12927593503390527651899","12960912038331884240956","12994230573273240830013","13027549108214597419070","13060867643155954008127","13094186178097310597184","13127504713038667186241","13160823247980023775298","13194141782921380364355"],"sequence_ids":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67]}]},{"cohort_id":"synthetic-fit-v2-r0-c0-t0-1","trial_id":"synthetic-fit-v2-r0-c0-t0","session_id":"synthetic-fit-v2-r0-c0-t0","role":"Train","label":"Empty","cohort_sha256":"af7cec0c758b8ece2556511a956eaa1223ff759854f1a45e81b50af4db7e4dc9","target_ns":"13000000000","sealed_at_ns":"15000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["13000001000","13030304031","13060607062","13090910093","13121213124","13151516155","13181819186","13212122217","13242425248","13272728279","13303031310","13333334341","13363637372","13393940403","13424243434","13454546465","13484849496","13515152527","13545455558","13575758589","13606061620","13636364651","13666667682","13696970713","13727273744","13757576775","13787879806","13818182837","13848485868","13878788899","13909091930","13939394961","13969697992","14000001023","14030304054","14060607085","14090910116","14121213147","14151516178","14181819209","14212122240","14242425271","14272728302","14303031333","14333334364","14363637395","14393940426","14424243457","14454546488","14484849519","14515152550","14545455581","14575758612","14606061643","14636364674","14666667705","14696970736","14727273767","14757576798","14787879829","14818182860","14848485891","14878788922","14909091953","14939394984","14969698015","15000001046"],"frame_ids":["14293652260603922743364","14326970795545279332421","14360289330486635921478","14393607865427992510535","14426926400369349099592","14460244935310705688649","14493563470252062277706","14526882005193418866763","14560200540134775455820","14593519075076132044877","14626837610017488633934","14660156144958845222991","14693474679900201812048","14726793214841558401105","14760111749782914990162","14793430284724271579219","14826748819665628168276","14860067354606984757333","14893385889548341346390","14926704424489697935447","14960022959431054524504","14993341494372411113561","15026660029313767702618","15059978564255124291675","15093297099196480880732","15126615634137837469789","15159934169079194058846","15193252704020550647903","15226571238961907236960","15259889773903263826017","15293208308844620415074","15326526843785977004131","15359845378727333593188","15393163913668690182245","15426482448610046771302","15459800983551403360359","15493119518492759949416","15526438053434116538473","15559756588375473127530","15593075123316829716587","15626393658258186305644","15659712193199542894701","15693030728140899483758","15726349263082256072815","15759667798023612661872","15792986332964969250929","15826304867906325839986","15859623402847682429043","15892941937789039018100","15926260472730395607157","15959579007671752196214","15992897542613108785271","16026216077554465374328","16059534612495821963385","16092853147437178552442","16126171682378535141499","16159490217319891730556","16192808752261248319613","16226127287202604908670","16259445822143961497727","16292764357085318086784","16326082892026674675841","16359401426968031264898","16392719961909387853955","16426038496850744443012","16459357031792101032069","16492675566733457621126"],"sequence_ids":[68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["13000002000","13030305031","13060608062","13090911093","13121214124","13151517155","13181820186","13212123217","13242426248","13272729279","13303032310","13333335341","13363638372","13393941403","13424244434","13454547465","13484850496","13515153527","13545456558","13575759589","13606062620","13636365651","13666668682","13696971713","13727274744","13757577775","13787880806","13818183837","13848486868","13878789899","13909092930","13939395961","13969698992","14000002023","14030305054","14060608085","14090911116","14121214147","14151517178","14181820209","14212123240","14242426271","14272729302","14303032333","14333335364","14363638395","14393941426","14424244457","14454547488","14484850519","14515153550","14545456581","14575759612","14606062643","14636365674","14666668705","14696971736","14727274767","14757577798","14787880829","14818183860","14848486891","14878789922","14909092953","14939395984","14969699015","15000002046"],"frame_ids":["14293653360119845486660","14326971895061202075717","14360290430002558664774","14393608964943915253831","14426927499885271842888","14460246034826628431945","14493564569767985021002","14526883104709341610059","14560201639650698199116","14593520174592054788173","14626838709533411377230","14660157244474767966287","14693475779416124555344","14726794314357481144401","14760112849298837733458","14793431384240194322515","14826749919181550911572","14860068454122907500629","14893386989064264089686","14926705524005620678743","14960024058946977267800","14993342593888333856857","15026661128829690445914","15059979663771047034971","15093298198712403624028","15126616733653760213085","15159935268595116802142","15193253803536473391199","15226572338477829980256","15259890873419186569313","15293209408360543158370","15326527943301899747427","15359846478243256336484","15393165013184612925541","15426483548125969514598","15459802083067326103655","15493120618008682692712","15526439152950039281769","15559757687891395870826","15593076222832752459883","15626394757774109048940","15659713292715465637997","15693031827656822227054","15726350362598178816111","15759668897539535405168","15792987432480891994225","15826305967422248583282","15859624502363605172339","15892943037304961761396","15926261572246318350453","15959580107187674939510","15992898642129031528567","16026217177070388117624","16059535712011744706681","16092854246953101295738","16126172781894457884795","16159491316835814473852","16192809851777171062909","16226128386718527651966","16259446921659884241023","16292765456601240830080","16326083991542597419137","16359402526483954008194","16392721061425310597251","16426039596366667186308","16459358131308023775365","16492676666249380364422"],"sequence_ids":[68,69,70,71,72,73,74,75,76,77,78,79,80,81,82,83,84,85,86,87,88,89,90,91,92,93,94,95,96,97,98,99,100,101,102,103,104,105,106,107,108,109,110,111,112,113,114,115,116,117,118,119,120,121,122,123,124,125,126,127,128,129,130,131,132,133,134]}]},{"cohort_id":"synthetic-fit-v2-r0-c0-t1-0","trial_id":"synthetic-fit-v2-r0-c0-t1","session_id":"synthetic-fit-v2-r0-c0-t1","role":"Train","label":"Empty","cohort_sha256":"74d05eb68924ac64637190cc64638fa9711205748b166c5d3c2d4e02f5745ac7","target_ns":"16000000000","sealed_at_ns":"18000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["16000001000","16030304031","16060607062","16090910093","16121213124","16151516155","16181819186","16212122217","16242425248","16272728279","16303031310","16333334341","16363637372","16393940403","16424243434","16454546465","16484849496","16515152527","16545455558","16575758589","16606061620","16636364651","16666667682","16696970713","16727273744","16757576775","16787879806","16818182837","16848485868","16878788899","16909091930","16939394961","16969697992","17000001023","17030304054","17060607085","17090910116","17121213147","17151516178","17181819209","17212122240","17242425271","17272728302","17303031333","17333334364","17363637395","17393940426","17424243457","17454546488","17484849519","17515152550","17545455581","17575758612","17606061643","17636364674","17666667705","17696970736","17727273767","17757576798","17787879829","17818182860","17848485891","17878788922","17909091953","17939394984","17969698015","18000001046"],"frame_ids":["17592187143931922743431","17625505678873279332488","17658824213814635921545","17692142748755992510602","17725461283697349099659","17758779818638705688716","17792098353580062277773","17825416888521418866830","17858735423462775455887","17892053958404132044944","17925372493345488634001","17958691028286845223058","17992009563228201812115","18025328098169558401172","18058646633110914990229","18091965168052271579286","18125283702993628168343","18158602237934984757400","18191920772876341346457","18225239307817697935514","18258557842759054524571","18291876377700411113628","18325194912641767702685","18358513447583124291742","18391831982524480880799","18425150517465837469856","18458469052407194058913","18491787587348550647970","18525106122289907237027","18558424657231263826084","18591743192172620415141","18625061727113977004198","18658380262055333593255","18691698796996690182312","18725017331938046771369","18758335866879403360426","18791654401820759949483","18824972936762116538540","18858291471703473127597","18891610006644829716654","18924928541586186305711","18958247076527542894768","18991565611468899483825","19024884146410256072882","19058202681351612661939","19091521216292969250996","19124839751234325840053","19158158286175682429110","19191476821117039018167","19224795356058395607224","19258113890999752196281","19291432425941108785338","19324750960882465374395","19358069495823821963452","19391388030765178552509","19424706565706535141566","19458025100647891730623","19491343635589248319680","19524662170530604908737","19557980705471961497794","19591299240413318086851","19624617775354674675908","19657936310296031264965","19691254845237387854022","19724573380178744443079","19757891915120101032136","19791210450061457621193"],"sequence_ids":[135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["16000002000","16030305031","16060608062","16090911093","16121214124","16151517155","16181820186","16212123217","16242426248","16272729279","16303032310","16333335341","16363638372","16393941403","16424244434","16454547465","16484850496","16515153527","16545456558","16575759589","16606062620","16636365651","16666668682","16696971713","16727274744","16757577775","16787880806","16818183837","16848486868","16878789899","16909092930","16939395961","16969698992","17000002023","17030305054","17060608085","17090911116","17121214147","17151517178","17181820209","17212123240","17242426271","17272729302","17303032333","17333335364","17363638395","17393941426","17424244457","17454547488","17484850519","17515153550","17545456581","17575759612","17606062643","17636365674","17666668705","17696971736","17727274767","17757577798","17787880829","17818183860","17848486891","17878789922","17909092953","17939395984","17969699015","18000002046"],"frame_ids":["17592188243447845486727","17625506778389202075784","17658825313330558664841","17692143848271915253898","17725462383213271842955","17758780918154628432012","17792099453095985021069","17825417988037341610126","17858736522978698199183","17892055057920054788240","17925373592861411377297","17958692127802767966354","17992010662744124555411","18025329197685481144468","18058647732626837733525","18091966267568194322582","18125284802509550911639","18158603337450907500696","18191921872392264089753","18225240407333620678810","18258558942274977267867","18291877477216333856924","18325196012157690445981","18358514547099047035038","18391833082040403624095","18425151616981760213152","18458470151923116802209","18491788686864473391266","18525107221805829980323","18558425756747186569380","18591744291688543158437","18625062826629899747494","18658381361571256336551","18691699896512612925608","18725018431453969514665","18758336966395326103722","18791655501336682692779","18824974036278039281836","18858292571219395870893","18891611106160752459950","18924929641102109049007","18958248176043465638064","18991566710984822227121","19024885245926178816178","19058203780867535405235","19091522315808891994292","19124840850750248583349","19158159385691605172406","19191477920632961761463","19224796455574318350520","19258114990515674939577","19291433525457031528634","19324752060398388117691","19358070595339744706748","19391389130281101295805","19424707665222457884862","19458026200163814473919","19491344735105171062976","19524663270046527652033","19557981804987884241090","19591300339929240830147","19624618874870597419204","19657937409811954008261","19691255944753310597318","19724574479694667186375","19757893014636023775432","19791211549577380364489"],"sequence_ids":[135,136,137,138,139,140,141,142,143,144,145,146,147,148,149,150,151,152,153,154,155,156,157,158,159,160,161,162,163,164,165,166,167,168,169,170,171,172,173,174,175,176,177,178,179,180,181,182,183,184,185,186,187,188,189,190,191,192,193,194,195,196,197,198,199,200,201]}]},{"cohort_id":"synthetic-fit-v2-r0-c0-t1-1","trial_id":"synthetic-fit-v2-r0-c0-t1","session_id":"synthetic-fit-v2-r0-c0-t1","role":"Train","label":"Empty","cohort_sha256":"2062593b0dda4a6ede2f5a0e6dd771596faa229472fe6d0656ec31c2877450fd","target_ns":"19000000000","sealed_at_ns":"21000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["19000001000","19030304031","19060607062","19090910093","19121213124","19151516155","19181819186","19212122217","19242425248","19272728279","19303031310","19333334341","19363637372","19393940403","19424243434","19454546465","19484849496","19515152527","19545455558","19575758589","19606061620","19636364651","19666667682","19696970713","19727273744","19757576775","19787879806","19818182837","19848485868","19878788899","19909091930","19939394961","19969697992","20000001023","20030304054","20060607085","20090910116","20121213147","20151516178","20181819209","20212122240","20242425271","20272728302","20303031333","20333334364","20363637395","20393940426","20424243457","20454546488","20484849519","20515152550","20545455581","20575758612","20606061643","20636364674","20666667705","20696970736","20727273767","20757576798","20787879829","20818182860","20848485891","20878788922","20909091953","20939394984","20969698015","21000001046"],"frame_ids":["20890722027259922743498","20924040562201279332555","20957359097142635921612","20990677632083992510669","21023996167025349099726","21057314701966705688783","21090633236908062277840","21123951771849418866897","21157270306790775455954","21190588841732132045011","21223907376673488634068","21257225911614845223125","21290544446556201812182","21323862981497558401239","21357181516438914990296","21390500051380271579353","21423818586321628168410","21457137121262984757467","21490455656204341346524","21523774191145697935581","21557092726087054524638","21590411261028411113695","21623729795969767702752","21657048330911124291809","21690366865852480880866","21723685400793837469923","21757003935735194058980","21790322470676550648037","21823641005617907237094","21856959540559263826151","21890278075500620415208","21923596610441977004265","21956915145383333593322","21990233680324690182379","22023552215266046771436","22056870750207403360493","22090189285148759949550","22123507820090116538607","22156826355031473127664","22190144889972829716721","22223463424914186305778","22256781959855542894835","22290100494796899483892","22323419029738256072949","22356737564679612662006","22390056099620969251063","22423374634562325840120","22456693169503682429177","22490011704445039018234","22523330239386395607291","22556648774327752196348","22589967309269108785405","22623285844210465374462","22656604379151821963519","22689922914093178552576","22723241449034535141633","22756559983975891730690","22789878518917248319747","22823197053858604908804","22856515588799961497861","22889834123741318086918","22923152658682674675975","22956471193624031265032","22989789728565387854089","23023108263506744443146","23056426798448101032203","23089745333389457621260"],"sequence_ids":[202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["19000002000","19030305031","19060608062","19090911093","19121214124","19151517155","19181820186","19212123217","19242426248","19272729279","19303032310","19333335341","19363638372","19393941403","19424244434","19454547465","19484850496","19515153527","19545456558","19575759589","19606062620","19636365651","19666668682","19696971713","19727274744","19757577775","19787880806","19818183837","19848486868","19878789899","19909092930","19939395961","19969698992","20000002023","20030305054","20060608085","20090911116","20121214147","20151517178","20181820209","20212123240","20242426271","20272729302","20303032333","20333335364","20363638395","20393941426","20424244457","20454547488","20484850519","20515153550","20545456581","20575759612","20606062643","20636365674","20666668705","20696971736","20727274767","20757577798","20787880829","20818183860","20848486891","20878789922","20909092953","20939395984","20969699015","21000002046"],"frame_ids":["20890723126775845486794","20924041661717202075851","20957360196658558664908","20990678731599915253965","21023997266541271843022","21057315801482628432079","21090634336423985021136","21123952871365341610193","21157271406306698199250","21190589941248054788307","21223908476189411377364","21257227011130767966421","21290545546072124555478","21323864081013481144535","21357182615954837733592","21390501150896194322649","21423819685837550911706","21457138220778907500763","21490456755720264089820","21523775290661620678877","21557093825602977267934","21590412360544333856991","21623730895485690446048","21657049430427047035105","21690367965368403624162","21723686500309760213219","21757005035251116802276","21790323570192473391333","21823642105133829980390","21856960640075186569447","21890279175016543158504","21923597709957899747561","21956916244899256336618","21990234779840612925675","22023553314781969514732","22056871849723326103789","22090190384664682692846","22123508919606039281903","22156827454547395870960","22190145989488752460017","22223464524430109049074","22256783059371465638131","22290101594312822227188","22323420129254178816245","22356738664195535405302","22390057199136891994359","22423375734078248583416","22456694269019605172473","22490012803960961761530","22523331338902318350587","22556649873843674939644","22589968408785031528701","22623286943726388117758","22656605478667744706815","22689924013609101295872","22723242548550457884929","22756561083491814473986","22789879618433171063043","22823198153374527652100","22856516688315884241157","22889835223257240830214","22923153758198597419271","22956472293139954008328","22989790828081310597385","23023109363022667186442","23056427897964023775499","23089746432905380364556"],"sequence_ids":[202,203,204,205,206,207,208,209,210,211,212,213,214,215,216,217,218,219,220,221,222,223,224,225,226,227,228,229,230,231,232,233,234,235,236,237,238,239,240,241,242,243,244,245,246,247,248,249,250,251,252,253,254,255,256,257,258,259,260,261,262,263,264,265,266,267,268]}]},{"cohort_id":"synthetic-fit-v2-r0-c1-t0-0","trial_id":"synthetic-fit-v2-r0-c1-t0","session_id":"synthetic-fit-v2-r0-c1-t0","role":"Train","label":"OccupiedStatic","cohort_sha256":"ea0572c8925666583d43d4e6efb4f1028a4c6041f403593b99d887faf0d5e196","target_ns":"22000000000","sealed_at_ns":"24000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["22000001000","22030304031","22060607062","22090910093","22121213124","22151516155","22181819186","22212122217","22242425248","22272728279","22303031310","22333334341","22363637372","22393940403","22424243434","22454546465","22484849496","22515152527","22545455558","22575758589","22606061620","22636364651","22666667682","22696970713","22727273744","22757576775","22787879806","22818182837","22848485868","22878788899","22909091930","22939394961","22969697992","23000001023","23030304054","23060607085","23090910116","23121213147","23151516178","23181819209","23212122240","23242425271","23272728302","23303031333","23333334364","23363637395","23393940426","23424243457","23454546488","23484849519","23515152550","23545455581","23575758612","23606061643","23636364674","23666667705","23696970736","23727273767","23757576798","23787879829","23818182860","23848485891","23878788922","23909091953","23939394984","23969698015","24000001046"],"frame_ids":["24189256910587922743565","24222575445529279332622","24255893980470635921679","24289212515411992510736","24322531050353349099793","24355849585294705688850","24389168120236062277907","24422486655177418866964","24455805190118775456021","24489123725060132045078","24522442260001488634135","24555760794942845223192","24589079329884201812249","24622397864825558401306","24655716399766914990363","24689034934708271579420","24722353469649628168477","24755672004590984757534","24788990539532341346591","24822309074473697935648","24855627609415054524705","24888946144356411113762","24922264679297767702819","24955583214239124291876","24988901749180480880933","25022220284121837469990","25055538819063194059047","25088857354004550648104","25122175888945907237161","25155494423887263826218","25188812958828620415275","25222131493769977004332","25255450028711333593389","25288768563652690182446","25322087098594046771503","25355405633535403360560","25388724168476759949617","25422042703418116538674","25455361238359473127731","25488679773300829716788","25521998308242186305845","25555316843183542894902","25588635378124899483959","25621953913066256073016","25655272448007612662073","25688590982948969251130","25721909517890325840187","25755228052831682429244","25788546587773039018301","25821865122714395607358","25855183657655752196415","25888502192597108785472","25921820727538465374529","25955139262479821963586","25988457797421178552643","26021776332362535141700","26055094867303891730757","26088413402245248319814","26121731937186604908871","26155050472127961497928","26188369007069318086985","26221687542010674676042","26255006076952031265099","26288324611893387854156","26321643146834744443213","26354961681776101032270","26388280216717457621327"],"sequence_ids":[269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["22000002000","22030305031","22060608062","22090911093","22121214124","22151517155","22181820186","22212123217","22242426248","22272729279","22303032310","22333335341","22363638372","22393941403","22424244434","22454547465","22484850496","22515153527","22545456558","22575759589","22606062620","22636365651","22666668682","22696971713","22727274744","22757577775","22787880806","22818183837","22848486868","22878789899","22909092930","22939395961","22969698992","23000002023","23030305054","23060608085","23090911116","23121214147","23151517178","23181820209","23212123240","23242426271","23272729302","23303032333","23333335364","23363638395","23393941426","23424244457","23454547488","23484850519","23515153550","23545456581","23575759612","23606062643","23636365674","23666668705","23696971736","23727274767","23757577798","23787880829","23818183860","23848486891","23878789922","23909092953","23939395984","23969699015","24000002046"],"frame_ids":["24189258010103845486861","24222576545045202075918","24255895079986558664975","24289213614927915254032","24322532149869271843089","24355850684810628432146","24389169219751985021203","24422487754693341610260","24455806289634698199317","24489124824576054788374","24522443359517411377431","24555761894458767966488","24589080429400124555545","24622398964341481144602","24655717499282837733659","24689036034224194322716","24722354569165550911773","24755673104106907500830","24788991639048264089887","24822310173989620678944","24855628708930977268001","24888947243872333857058","24922265778813690446115","24955584313755047035172","24988902848696403624229","25022221383637760213286","25055539918579116802343","25088858453520473391400","25122176988461829980457","25155495523403186569514","25188814058344543158571","25222132593285899747628","25255451128227256336685","25288769663168612925742","25322088198109969514799","25355406733051326103856","25388725267992682692913","25422043802934039281970","25455362337875395871027","25488680872816752460084","25521999407758109049141","25555317942699465638198","25588636477640822227255","25621955012582178816312","25655273547523535405369","25688592082464891994426","25721910617406248583483","25755229152347605172540","25788547687288961761597","25821866222230318350654","25855184757171674939711","25888503292113031528768","25921821827054388117825","25955140361995744706882","25988458896937101295939","26021777431878457884996","26055095966819814474053","26088414501761171063110","26121733036702527652167","26155051571643884241224","26188370106585240830281","26221688641526597419338","26255007176467954008395","26288325711409310597452","26321644246350667186509","26354962781292023775566","26388281316233380364623"],"sequence_ids":[269,270,271,272,273,274,275,276,277,278,279,280,281,282,283,284,285,286,287,288,289,290,291,292,293,294,295,296,297,298,299,300,301,302,303,304,305,306,307,308,309,310,311,312,313,314,315,316,317,318,319,320,321,322,323,324,325,326,327,328,329,330,331,332,333,334,335]}]},{"cohort_id":"synthetic-fit-v2-r0-c1-t0-1","trial_id":"synthetic-fit-v2-r0-c1-t0","session_id":"synthetic-fit-v2-r0-c1-t0","role":"Train","label":"OccupiedStatic","cohort_sha256":"de47ee5a29b1628776efa122c9b1f8bc1dfa506e5b406a93ccab04503ffaf6de","target_ns":"25000000000","sealed_at_ns":"27000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["25000001000","25030304031","25060607062","25090910093","25121213124","25151516155","25181819186","25212122217","25242425248","25272728279","25303031310","25333334341","25363637372","25393940403","25424243434","25454546465","25484849496","25515152527","25545455558","25575758589","25606061620","25636364651","25666667682","25696970713","25727273744","25757576775","25787879806","25818182837","25848485868","25878788899","25909091930","25939394961","25969697992","26000001023","26030304054","26060607085","26090910116","26121213147","26151516178","26181819209","26212122240","26242425271","26272728302","26303031333","26333334364","26363637395","26393940426","26424243457","26454546488","26484849519","26515152550","26545455581","26575758612","26606061643","26636364674","26666667705","26696970736","26727273767","26757576798","26787879829","26818182860","26848485891","26878788922","26909091953","26939394984","26969698015","27000001046"],"frame_ids":["27487791793915922743632","27521110328857279332689","27554428863798635921746","27587747398739992510803","27621065933681349099860","27654384468622705688917","27687703003564062277974","27721021538505418867031","27754340073446775456088","27787658608388132045145","27820977143329488634202","27854295678270845223259","27887614213212201812316","27920932748153558401373","27954251283094914990430","27987569818036271579487","28020888352977628168544","28054206887918984757601","28087525422860341346658","28120843957801697935715","28154162492743054524772","28187481027684411113829","28220799562625767702886","28254118097567124291943","28287436632508480881000","28320755167449837470057","28354073702391194059114","28387392237332550648171","28420710772273907237228","28454029307215263826285","28487347842156620415342","28520666377097977004399","28553984912039333593456","28587303446980690182513","28620621981922046771570","28653940516863403360627","28687259051804759949684","28720577586746116538741","28753896121687473127798","28787214656628829716855","28820533191570186305912","28853851726511542894969","28887170261452899484026","28920488796394256073083","28953807331335612662140","28987125866276969251197","29020444401218325840254","29053762936159682429311","29087081471101039018368","29120400006042395607425","29153718540983752196482","29187037075925108785539","29220355610866465374596","29253674145807821963653","29286992680749178552710","29320311215690535141767","29353629750631891730824","29386948285573248319881","29420266820514604908938","29453585355455961497995","29486903890397318087052","29520222425338674676109","29553540960280031265166","29586859495221387854223","29620178030162744443280","29653496565104101032337","29686815100045457621394"],"sequence_ids":[336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["25000002000","25030305031","25060608062","25090911093","25121214124","25151517155","25181820186","25212123217","25242426248","25272729279","25303032310","25333335341","25363638372","25393941403","25424244434","25454547465","25484850496","25515153527","25545456558","25575759589","25606062620","25636365651","25666668682","25696971713","25727274744","25757577775","25787880806","25818183837","25848486868","25878789899","25909092930","25939395961","25969698992","26000002023","26030305054","26060608085","26090911116","26121214147","26151517178","26181820209","26212123240","26242426271","26272729302","26303032333","26333335364","26363638395","26393941426","26424244457","26454547488","26484850519","26515153550","26545456581","26575759612","26606062643","26636365674","26666668705","26696971736","26727274767","26757577798","26787880829","26818183860","26848486891","26878789922","26909092953","26939395984","26969699015","27000002046"],"frame_ids":["27487792893431845486928","27521111428373202075985","27554429963314558665042","27587748498255915254099","27621067033197271843156","27654385568138628432213","27687704103079985021270","27721022638021341610327","27754341172962698199384","27787659707904054788441","27820978242845411377498","27854296777786767966555","27887615312728124555612","27920933847669481144669","27954252382610837733726","27987570917552194322783","28020889452493550911840","28054207987434907500897","28087526522376264089954","28120845057317620679011","28154163592258977268068","28187482127200333857125","28220800662141690446182","28254119197083047035239","28287437732024403624296","28320756266965760213353","28354074801907116802410","28387393336848473391467","28420711871789829980524","28454030406731186569581","28487348941672543158638","28520667476613899747695","28553986011555256336752","28587304546496612925809","28620623081437969514866","28653941616379326103923","28687260151320682692980","28720578686262039282037","28753897221203395871094","28787215756144752460151","28820534291086109049208","28853852826027465638265","28887171360968822227322","28920489895910178816379","28953808430851535405436","28987126965792891994493","29020445500734248583550","29053764035675605172607","29087082570616961761664","29120401105558318350721","29153719640499674939778","29187038175441031528835","29220356710382388117892","29253675245323744706949","29286993780265101296006","29320312315206457885063","29353630850147814474120","29386949385089171063177","29420267920030527652234","29453586454971884241291","29486904989913240830348","29520223524854597419405","29553542059795954008462","29586860594737310597519","29620179129678667186576","29653497664620023775633","29686816199561380364690"],"sequence_ids":[336,337,338,339,340,341,342,343,344,345,346,347,348,349,350,351,352,353,354,355,356,357,358,359,360,361,362,363,364,365,366,367,368,369,370,371,372,373,374,375,376,377,378,379,380,381,382,383,384,385,386,387,388,389,390,391,392,393,394,395,396,397,398,399,400,401,402]}]},{"cohort_id":"synthetic-fit-v2-r0-c1-t1-0","trial_id":"synthetic-fit-v2-r0-c1-t1","session_id":"synthetic-fit-v2-r0-c1-t1","role":"Train","label":"OccupiedStatic","cohort_sha256":"3565c70576c88230e846845cf90b17851a9ca34b850eb859b8bd3d6afb30c84d","target_ns":"28000000000","sealed_at_ns":"30000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["28000001000","28030304031","28060607062","28090910093","28121213124","28151516155","28181819186","28212122217","28242425248","28272728279","28303031310","28333334341","28363637372","28393940403","28424243434","28454546465","28484849496","28515152527","28545455558","28575758589","28606061620","28636364651","28666667682","28696970713","28727273744","28757576775","28787879806","28818182837","28848485868","28878788899","28909091930","28939394961","28969697992","29000001023","29030304054","29060607085","29090910116","29121213147","29151516178","29181819209","29212122240","29242425271","29272728302","29303031333","29333334364","29363637395","29393940426","29424243457","29454546488","29484849519","29515152550","29545455581","29575758612","29606061643","29636364674","29666667705","29696970736","29727273767","29757576798","29787879829","29818182860","29848485891","29878788922","29909091953","29939394984","29969698015","30000001046"],"frame_ids":["30786326677243922743699","30819645212185279332756","30852963747126635921813","30886282282067992510870","30919600817009349099927","30952919351950705688984","30986237886892062278041","31019556421833418867098","31052874956774775456155","31086193491716132045212","31119512026657488634269","31152830561598845223326","31186149096540201812383","31219467631481558401440","31252786166422914990497","31286104701364271579554","31319423236305628168611","31352741771246984757668","31386060306188341346725","31419378841129697935782","31452697376071054524839","31486015911012411113896","31519334445953767702953","31552652980895124292010","31585971515836480881067","31619290050777837470124","31652608585719194059181","31685927120660550648238","31719245655601907237295","31752564190543263826352","31785882725484620415409","31819201260425977004466","31852519795367333593523","31885838330308690182580","31919156865250046771637","31952475400191403360694","31985793935132759949751","32019112470074116538808","32052431005015473127865","32085749539956829716922","32119068074898186305979","32152386609839542895036","32185705144780899484093","32219023679722256073150","32252342214663612662207","32285660749604969251264","32318979284546325840321","32352297819487682429378","32385616354429039018435","32418934889370395607492","32452253424311752196549","32485571959253108785606","32518890494194465374663","32552209029135821963720","32585527564077178552777","32618846099018535141834","32652164633959891730891","32685483168901248319948","32718801703842604909005","32752120238783961498062","32785438773725318087119","32818757308666674676176","32852075843608031265233","32885394378549387854290","32918712913490744443347","32952031448432101032404","32985349983373457621461"],"sequence_ids":[403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["28000002000","28030305031","28060608062","28090911093","28121214124","28151517155","28181820186","28212123217","28242426248","28272729279","28303032310","28333335341","28363638372","28393941403","28424244434","28454547465","28484850496","28515153527","28545456558","28575759589","28606062620","28636365651","28666668682","28696971713","28727274744","28757577775","28787880806","28818183837","28848486868","28878789899","28909092930","28939395961","28969698992","29000002023","29030305054","29060608085","29090911116","29121214147","29151517178","29181820209","29212123240","29242426271","29272729302","29303032333","29333335364","29363638395","29393941426","29424244457","29454547488","29484850519","29515153550","29545456581","29575759612","29606062643","29636365674","29666668705","29696971736","29727274767","29757577798","29787880829","29818183860","29848486891","29878789922","29909092953","29939395984","29969699015","30000002046"],"frame_ids":["30786327776759845486995","30819646311701202076052","30852964846642558665109","30886283381583915254166","30919601916525271843223","30952920451466628432280","30986238986407985021337","31019557521349341610394","31052876056290698199451","31086194591232054788508","31119513126173411377565","31152831661114767966622","31186150196056124555679","31219468730997481144736","31252787265938837733793","31286105800880194322850","31319424335821550911907","31352742870762907500964","31386061405704264090021","31419379940645620679078","31452698475586977268135","31486017010528333857192","31519335545469690446249","31552654080411047035306","31585972615352403624363","31619291150293760213420","31652609685235116802477","31685928220176473391534","31719246755117829980591","31752565290059186569648","31785883825000543158705","31819202359941899747762","31852520894883256336819","31885839429824612925876","31919157964765969514933","31952476499707326103990","31985795034648682693047","32019113569590039282104","32052432104531395871161","32085750639472752460218","32119069174414109049275","32152387709355465638332","32185706244296822227389","32219024779238178816446","32252343314179535405503","32285661849120891994560","32318980384062248583617","32352298919003605172674","32385617453944961761731","32418935988886318350788","32452254523827674939845","32485573058769031528902","32518891593710388117959","32552210128651744707016","32585528663593101296073","32618847198534457885130","32652165733475814474187","32685484268417171063244","32718802803358527652301","32752121338299884241358","32785439873241240830415","32818758408182597419472","32852076943123954008529","32885395478065310597586","32918714013006667186643","32952032547948023775700","32985351082889380364757"],"sequence_ids":[403,404,405,406,407,408,409,410,411,412,413,414,415,416,417,418,419,420,421,422,423,424,425,426,427,428,429,430,431,432,433,434,435,436,437,438,439,440,441,442,443,444,445,446,447,448,449,450,451,452,453,454,455,456,457,458,459,460,461,462,463,464,465,466,467,468,469]}]},{"cohort_id":"synthetic-fit-v2-r0-c1-t1-1","trial_id":"synthetic-fit-v2-r0-c1-t1","session_id":"synthetic-fit-v2-r0-c1-t1","role":"Train","label":"OccupiedStatic","cohort_sha256":"e82fe125bb5ba2c70db70581f63cf3aab683bf25b3da0deae1f415b73d5e9974","target_ns":"31000000000","sealed_at_ns":"33000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["31000001000","31030304031","31060607062","31090910093","31121213124","31151516155","31181819186","31212122217","31242425248","31272728279","31303031310","31333334341","31363637372","31393940403","31424243434","31454546465","31484849496","31515152527","31545455558","31575758589","31606061620","31636364651","31666667682","31696970713","31727273744","31757576775","31787879806","31818182837","31848485868","31878788899","31909091930","31939394961","31969697992","32000001023","32030304054","32060607085","32090910116","32121213147","32151516178","32181819209","32212122240","32242425271","32272728302","32303031333","32333334364","32363637395","32393940426","32424243457","32454546488","32484849519","32515152550","32545455581","32575758612","32606061643","32636364674","32666667705","32696970736","32727273767","32757576798","32787879829","32818182860","32848485891","32878788922","32909091953","32939394984","32969698015","33000001046"],"frame_ids":["34084861560571922743766","34118180095513279332823","34151498630454635921880","34184817165395992510937","34218135700337349099994","34251454235278705689051","34284772770220062278108","34318091305161418867165","34351409840102775456222","34384728375044132045279","34418046909985488634336","34451365444926845223393","34484683979868201812450","34518002514809558401507","34551321049750914990564","34584639584692271579621","34617958119633628168678","34651276654574984757735","34684595189516341346792","34717913724457697935849","34751232259399054524906","34784550794340411113963","34817869329281767703020","34851187864223124292077","34884506399164480881134","34917824934105837470191","34951143469047194059248","34984462003988550648305","35017780538929907237362","35051099073871263826419","35084417608812620415476","35117736143753977004533","35151054678695333593590","35184373213636690182647","35217691748578046771704","35251010283519403360761","35284328818460759949818","35317647353402116538875","35350965888343473127932","35384284423284829716989","35417602958226186306046","35450921493167542895103","35484240028108899484160","35517558563050256073217","35550877097991612662274","35584195632932969251331","35617514167874325840388","35650832702815682429445","35684151237757039018502","35717469772698395607559","35750788307639752196616","35784106842581108785673","35817425377522465374730","35850743912463821963787","35884062447405178552844","35917380982346535141901","35950699517287891730958","35984018052229248320015","36017336587170604909072","36050655122111961498129","36083973657053318087186","36117292191994674676243","36150610726936031265300","36183929261877387854357","36217247796818744443414","36250566331760101032471","36283884866701457621528"],"sequence_ids":[470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["31000002000","31030305031","31060608062","31090911093","31121214124","31151517155","31181820186","31212123217","31242426248","31272729279","31303032310","31333335341","31363638372","31393941403","31424244434","31454547465","31484850496","31515153527","31545456558","31575759589","31606062620","31636365651","31666668682","31696971713","31727274744","31757577775","31787880806","31818183837","31848486868","31878789899","31909092930","31939395961","31969698992","32000002023","32030305054","32060608085","32090911116","32121214147","32151517178","32181820209","32212123240","32242426271","32272729302","32303032333","32333335364","32363638395","32393941426","32424244457","32454547488","32484850519","32515153550","32545456581","32575759612","32606062643","32636365674","32666668705","32696971736","32727274767","32757577798","32787880829","32818183860","32848486891","32878789922","32909092953","32939395984","32969699015","33000002046"],"frame_ids":["34084862660087845487062","34118181195029202076119","34151499729970558665176","34184818264911915254233","34218136799853271843290","34251455334794628432347","34284773869735985021404","34318092404677341610461","34351410939618698199518","34384729474560054788575","34418048009501411377632","34451366544442767966689","34484685079384124555746","34518003614325481144803","34551322149266837733860","34584640684208194322917","34617959219149550911974","34651277754090907501031","34684596289032264090088","34717914823973620679145","34751233358914977268202","34784551893856333857259","34817870428797690446316","34851188963739047035373","34884507498680403624430","34917826033621760213487","34951144568563116802544","34984463103504473391601","35017781638445829980658","35051100173387186569715","35084418708328543158772","35117737243269899747829","35151055778211256336886","35184374313152612925943","35217692848093969515000","35251011383035326104057","35284329917976682693114","35317648452918039282171","35350966987859395871228","35384285522800752460285","35417604057742109049342","35450922592683465638399","35484241127624822227456","35517559662566178816513","35550878197507535405570","35584196732448891994627","35617515267390248583684","35650833802331605172741","35684152337272961761798","35717470872214318350855","35750789407155674939912","35784107942097031528969","35817426477038388118026","35850745011979744707083","35884063546921101296140","35917382081862457885197","35950700616803814474254","35984019151745171063311","36017337686686527652368","36050656221627884241425","36083974756569240830482","36117293291510597419539","36150611826451954008596","36183930361393310597653","36217248896334667186710","36250567431276023775767","36283885966217380364824"],"sequence_ids":[470,471,472,473,474,475,476,477,478,479,480,481,482,483,484,485,486,487,488,489,490,491,492,493,494,495,496,497,498,499,500,501,502,503,504,505,506,507,508,509,510,511,512,513,514,515,516,517,518,519,520,521,522,523,524,525,526,527,528,529,530,531,532,533,534,535,536]}]},{"cohort_id":"synthetic-fit-v2-r0-c2-t0-0","trial_id":"synthetic-fit-v2-r0-c2-t0","session_id":"synthetic-fit-v2-r0-c2-t0","role":"Train","label":"OccupiedMoving","cohort_sha256":"c0fd00a4fb80c266d52fa8fb422be1e059488287061eadec8616af79350e0d5d","target_ns":"34000000000","sealed_at_ns":"36000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["34000001000","34030304031","34060607062","34090910093","34121213124","34151516155","34181819186","34212122217","34242425248","34272728279","34303031310","34333334341","34363637372","34393940403","34424243434","34454546465","34484849496","34515152527","34545455558","34575758589","34606061620","34636364651","34666667682","34696970713","34727273744","34757576775","34787879806","34818182837","34848485868","34878788899","34909091930","34939394961","34969697992","35000001023","35030304054","35060607085","35090910116","35121213147","35151516178","35181819209","35212122240","35242425271","35272728302","35303031333","35333334364","35363637395","35393940426","35424243457","35454546488","35484849519","35515152550","35545455581","35575758612","35606061643","35636364674","35666667705","35696970736","35727273767","35757576798","35787879829","35818182860","35848485891","35878788922","35909091953","35939394984","35969698015","36000001046"],"frame_ids":["37383396443899922743833","37416714978841279332890","37450033513782635921947","37483352048723992511004","37516670583665349100061","37549989118606705689118","37583307653548062278175","37616626188489418867232","37649944723430775456289","37683263258372132045346","37716581793313488634403","37749900328254845223460","37783218863196201812517","37816537398137558401574","37849855933078914990631","37883174468020271579688","37916493002961628168745","37949811537902984757802","37983130072844341346859","38016448607785697935916","38049767142727054524973","38083085677668411114030","38116404212609767703087","38149722747551124292144","38183041282492480881201","38216359817433837470258","38249678352375194059315","38282996887316550648372","38316315422257907237429","38349633957199263826486","38382952492140620415543","38416271027081977004600","38449589562023333593657","38482908096964690182714","38516226631906046771771","38549545166847403360828","38582863701788759949885","38616182236730116538942","38649500771671473127999","38682819306612829717056","38716137841554186306113","38749456376495542895170","38782774911436899484227","38816093446378256073284","38849411981319612662341","38882730516260969251398","38916049051202325840455","38949367586143682429512","38982686121085039018569","39016004656026395607626","39049323190967752196683","39082641725909108785740","39115960260850465374797","39149278795791821963854","39182597330733178552911","39215915865674535141968","39249234400615891731025","39282552935557248320082","39315871470498604909139","39349190005439961498196","39382508540381318087253","39415827075322674676310","39449145610264031265367","39482464145205387854424","39515782680146744443481","39549101215088101032538","39582419750029457621595"],"sequence_ids":[537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["34000002000","34030305031","34060608062","34090911093","34121214124","34151517155","34181820186","34212123217","34242426248","34272729279","34303032310","34333335341","34363638372","34393941403","34424244434","34454547465","34484850496","34515153527","34545456558","34575759589","34606062620","34636365651","34666668682","34696971713","34727274744","34757577775","34787880806","34818183837","34848486868","34878789899","34909092930","34939395961","34969698992","35000002023","35030305054","35060608085","35090911116","35121214147","35151517178","35181820209","35212123240","35242426271","35272729302","35303032333","35333335364","35363638395","35393941426","35424244457","35454547488","35484850519","35515153550","35545456581","35575759612","35606062643","35636365674","35666668705","35696971736","35727274767","35757577798","35787880829","35818183860","35848486891","35878789922","35909092953","35939395984","35969699015","36000002046"],"frame_ids":["37383397543415845487129","37416716078357202076186","37450034613298558665243","37483353148239915254300","37516671683181271843357","37549990218122628432414","37583308753063985021471","37616627288005341610528","37649945822946698199585","37683264357888054788642","37716582892829411377699","37749901427770767966756","37783219962712124555813","37816538497653481144870","37849857032594837733927","37883175567536194322984","37916494102477550912041","37949812637418907501098","37983131172360264090155","38016449707301620679212","38049768242242977268269","38083086777184333857326","38116405312125690446383","38149723847067047035440","38183042382008403624497","38216360916949760213554","38249679451891116802611","38282997986832473391668","38316316521773829980725","38349635056715186569782","38382953591656543158839","38416272126597899747896","38449590661539256336953","38482909196480612926010","38516227731421969515067","38549546266363326104124","38582864801304682693181","38616183336246039282238","38649501871187395871295","38682820406128752460352","38716138941070109049409","38749457476011465638466","38782776010952822227523","38816094545894178816580","38849413080835535405637","38882731615776891994694","38916050150718248583751","38949368685659605172808","38982687220600961761865","39016005755542318350922","39049324290483674939979","39082642825425031529036","39115961360366388118093","39149279895307744707150","39182598430249101296207","39215916965190457885264","39249235500131814474321","39282554035073171063378","39315872570014527652435","39349191104955884241492","39382509639897240830549","39415828174838597419606","39449146709779954008663","39482465244721310597720","39515783779662667186777","39549102314604023775834","39582420849545380364891"],"sequence_ids":[537,538,539,540,541,542,543,544,545,546,547,548,549,550,551,552,553,554,555,556,557,558,559,560,561,562,563,564,565,566,567,568,569,570,571,572,573,574,575,576,577,578,579,580,581,582,583,584,585,586,587,588,589,590,591,592,593,594,595,596,597,598,599,600,601,602,603]}]},{"cohort_id":"synthetic-fit-v2-r0-c2-t0-1","trial_id":"synthetic-fit-v2-r0-c2-t0","session_id":"synthetic-fit-v2-r0-c2-t0","role":"Train","label":"OccupiedMoving","cohort_sha256":"68e6653674d4c0a2768fe35fa743e03bfd561e593d24f7ff5e4529dce9c4f8a9","target_ns":"37000000000","sealed_at_ns":"39000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["37000001000","37030304031","37060607062","37090910093","37121213124","37151516155","37181819186","37212122217","37242425248","37272728279","37303031310","37333334341","37363637372","37393940403","37424243434","37454546465","37484849496","37515152527","37545455558","37575758589","37606061620","37636364651","37666667682","37696970713","37727273744","37757576775","37787879806","37818182837","37848485868","37878788899","37909091930","37939394961","37969697992","38000001023","38030304054","38060607085","38090910116","38121213147","38151516178","38181819209","38212122240","38242425271","38272728302","38303031333","38333334364","38363637395","38393940426","38424243457","38454546488","38484849519","38515152550","38545455581","38575758612","38606061643","38636364674","38666667705","38696970736","38727273767","38757576798","38787879829","38818182860","38848485891","38878788922","38909091953","38939394984","38969698015","39000001046"],"frame_ids":["40681931327227922743900","40715249862169279332957","40748568397110635922014","40781886932051992511071","40815205466993349100128","40848524001934705689185","40881842536876062278242","40915161071817418867299","40948479606758775456356","40981798141700132045413","41015116676641488634470","41048435211582845223527","41081753746524201812584","41115072281465558401641","41148390816406914990698","41181709351348271579755","41215027886289628168812","41248346421230984757869","41281664956172341346926","41314983491113697935983","41348302026055054525040","41381620560996411114097","41414939095937767703154","41448257630879124292211","41481576165820480881268","41514894700761837470325","41548213235703194059382","41581531770644550648439","41614850305585907237496","41648168840527263826553","41681487375468620415610","41714805910409977004667","41748124445351333593724","41781442980292690182781","41814761515234046771838","41848080050175403360895","41881398585116759949952","41914717120058116539009","41948035654999473128066","41981354189940829717123","42014672724882186306180","42047991259823542895237","42081309794764899484294","42114628329706256073351","42147946864647612662408","42181265399588969251465","42214583934530325840522","42247902469471682429579","42281221004413039018636","42314539539354395607693","42347858074295752196750","42381176609237108785807","42414495144178465374864","42447813679119821963921","42481132214061178552978","42514450749002535142035","42547769283943891731092","42581087818885248320149","42614406353826604909206","42647724888767961498263","42681043423709318087320","42714361958650674676377","42747680493592031265434","42780999028533387854491","42814317563474744443548","42847636098416101032605","42880954633357457621662"],"sequence_ids":[604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["37000002000","37030305031","37060608062","37090911093","37121214124","37151517155","37181820186","37212123217","37242426248","37272729279","37303032310","37333335341","37363638372","37393941403","37424244434","37454547465","37484850496","37515153527","37545456558","37575759589","37606062620","37636365651","37666668682","37696971713","37727274744","37757577775","37787880806","37818183837","37848486868","37878789899","37909092930","37939395961","37969698992","38000002023","38030305054","38060608085","38090911116","38121214147","38151517178","38181820209","38212123240","38242426271","38272729302","38303032333","38333335364","38363638395","38393941426","38424244457","38454547488","38484850519","38515153550","38545456581","38575759612","38606062643","38636365674","38666668705","38696971736","38727274767","38757577798","38787880829","38818183860","38848486891","38878789922","38909092953","38939395984","38969699015","39000002046"],"frame_ids":["40681932426743845487196","40715250961685202076253","40748569496626558665310","40781888031567915254367","40815206566509271843424","40848525101450628432481","40881843636391985021538","40915162171333341610595","40948480706274698199652","40981799241216054788709","41015117776157411377766","41048436311098767966823","41081754846040124555880","41115073380981481144937","41148391915922837733994","41181710450864194323051","41215028985805550912108","41248347520746907501165","41281666055688264090222","41314984590629620679279","41348303125570977268336","41381621660512333857393","41414940195453690446450","41448258730395047035507","41481577265336403624564","41514895800277760213621","41548214335219116802678","41581532870160473391735","41614851405101829980792","41648169940043186569849","41681488474984543158906","41714807009925899747963","41748125544867256337020","41781444079808612926077","41814762614749969515134","41848081149691326104191","41881399684632682693248","41914718219574039282305","41948036754515395871362","41981355289456752460419","42014673824398109049476","42047992359339465638533","42081310894280822227590","42114629429222178816647","42147947964163535405704","42181266499104891994761","42214585034046248583818","42247903568987605172875","42281222103928961761932","42314540638870318350989","42347859173811674940046","42381177708753031529103","42414496243694388118160","42447814778635744707217","42481133313577101296274","42514451848518457885331","42547770383459814474388","42581088918401171063445","42614407453342527652502","42647725988283884241559","42681044523225240830616","42714363058166597419673","42747681593107954008730","42781000128049310597787","42814318662990667186844","42847637197932023775901","42880955732873380364958"],"sequence_ids":[604,605,606,607,608,609,610,611,612,613,614,615,616,617,618,619,620,621,622,623,624,625,626,627,628,629,630,631,632,633,634,635,636,637,638,639,640,641,642,643,644,645,646,647,648,649,650,651,652,653,654,655,656,657,658,659,660,661,662,663,664,665,666,667,668,669,670]}]},{"cohort_id":"synthetic-fit-v2-r0-c2-t1-0","trial_id":"synthetic-fit-v2-r0-c2-t1","session_id":"synthetic-fit-v2-r0-c2-t1","role":"Train","label":"OccupiedMoving","cohort_sha256":"9416b1c5d2c266b3cef633a824cb8705a7c4b657086e3db0eea6b555e0ee19f5","target_ns":"40000000000","sealed_at_ns":"42000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["40000001000","40030304031","40060607062","40090910093","40121213124","40151516155","40181819186","40212122217","40242425248","40272728279","40303031310","40333334341","40363637372","40393940403","40424243434","40454546465","40484849496","40515152527","40545455558","40575758589","40606061620","40636364651","40666667682","40696970713","40727273744","40757576775","40787879806","40818182837","40848485868","40878788899","40909091930","40939394961","40969697992","41000001023","41030304054","41060607085","41090910116","41121213147","41151516178","41181819209","41212122240","41242425271","41272728302","41303031333","41333334364","41363637395","41393940426","41424243457","41454546488","41484849519","41515152550","41545455581","41575758612","41606061643","41636364674","41666667705","41696970736","41727273767","41757576798","41787879829","41818182860","41848485891","41878788922","41909091953","41939394984","41969698015","42000001046"],"frame_ids":["43980466210555922743967","44013784745497279333024","44047103280438635922081","44080421815379992511138","44113740350321349100195","44147058885262705689252","44180377420204062278309","44213695955145418867366","44247014490086775456423","44280333025028132045480","44313651559969488634537","44346970094910845223594","44380288629852201812651","44413607164793558401708","44446925699734914990765","44480244234676271579822","44513562769617628168879","44546881304558984757936","44580199839500341346993","44613518374441697936050","44646836909383054525107","44680155444324411114164","44713473979265767703221","44746792514207124292278","44780111049148480881335","44813429584089837470392","44846748119031194059449","44880066653972550648506","44913385188913907237563","44946703723855263826620","44980022258796620415677","45013340793737977004734","45046659328679333593791","45079977863620690182848","45113296398562046771905","45146614933503403360962","45179933468444759950019","45213252003386116539076","45246570538327473128133","45279889073268829717190","45313207608210186306247","45346526143151542895304","45379844678092899484361","45413163213034256073418","45446481747975612662475","45479800282916969251532","45513118817858325840589","45546437352799682429646","45579755887741039018703","45613074422682395607760","45646392957623752196817","45679711492565108785874","45713030027506465374931","45746348562447821963988","45779667097389178553045","45812985632330535142102","45846304167271891731159","45879622702213248320216","45912941237154604909273","45946259772095961498330","45979578307037318087387","46012896841978674676444","46046215376920031265501","46079533911861387854558","46112852446802744443615","46146170981744101032672","46179489516685457621729"],"sequence_ids":[671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["40000002000","40030305031","40060608062","40090911093","40121214124","40151517155","40181820186","40212123217","40242426248","40272729279","40303032310","40333335341","40363638372","40393941403","40424244434","40454547465","40484850496","40515153527","40545456558","40575759589","40606062620","40636365651","40666668682","40696971713","40727274744","40757577775","40787880806","40818183837","40848486868","40878789899","40909092930","40939395961","40969698992","41000002023","41030305054","41060608085","41090911116","41121214147","41151517178","41181820209","41212123240","41242426271","41272729302","41303032333","41333335364","41363638395","41393941426","41424244457","41454547488","41484850519","41515153550","41545456581","41575759612","41606062643","41636365674","41666668705","41696971736","41727274767","41757577798","41787880829","41818183860","41848486891","41878789922","41909092953","41939395984","41969699015","42000002046"],"frame_ids":["43980467310071845487263","44013785845013202076320","44047104379954558665377","44080422914895915254434","44113741449837271843491","44147059984778628432548","44180378519719985021605","44213697054661341610662","44247015589602698199719","44280334124544054788776","44313652659485411377833","44346971194426767966890","44380289729368124555947","44413608264309481145004","44446926799250837734061","44480245334192194323118","44513563869133550912175","44546882404074907501232","44580200939016264090289","44613519473957620679346","44646838008898977268403","44680156543840333857460","44713475078781690446517","44746793613723047035574","44780112148664403624631","44813430683605760213688","44846749218547116802745","44880067753488473391802","44913386288429829980859","44946704823371186569916","44980023358312543158973","45013341893253899748030","45046660428195256337087","45079978963136612926144","45113297498077969515201","45146616033019326104258","45179934567960682693315","45213253102902039282372","45246571637843395871429","45279890172784752460486","45313208707726109049543","45346527242667465638600","45379845777608822227657","45413164312550178816714","45446482847491535405771","45479801382432891994828","45513119917374248583885","45546438452315605172942","45579756987256961761999","45613075522198318351056","45646394057139674940113","45679712592081031529170","45713031127022388118227","45746349661963744707284","45779668196905101296341","45812986731846457885398","45846305266787814474455","45879623801729171063512","45912942336670527652569","45946260871611884241626","45979579406553240830683","46012897941494597419740","46046216476435954008797","46079535011377310597854","46112853546318667186911","46146172081260023775968","46179490616201380365025"],"sequence_ids":[671,672,673,674,675,676,677,678,679,680,681,682,683,684,685,686,687,688,689,690,691,692,693,694,695,696,697,698,699,700,701,702,703,704,705,706,707,708,709,710,711,712,713,714,715,716,717,718,719,720,721,722,723,724,725,726,727,728,729,730,731,732,733,734,735,736,737]}]},{"cohort_id":"synthetic-fit-v2-r0-c2-t1-1","trial_id":"synthetic-fit-v2-r0-c2-t1","session_id":"synthetic-fit-v2-r0-c2-t1","role":"Train","label":"OccupiedMoving","cohort_sha256":"2278fc454bc1504bc787c44cbee4e25957e32cc3659e4cda073dd9d93d735932","target_ns":"43000000000","sealed_at_ns":"45000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["43000001000","43030304031","43060607062","43090910093","43121213124","43151516155","43181819186","43212122217","43242425248","43272728279","43303031310","43333334341","43363637372","43393940403","43424243434","43454546465","43484849496","43515152527","43545455558","43575758589","43606061620","43636364651","43666667682","43696970713","43727273744","43757576775","43787879806","43818182837","43848485868","43878788899","43909091930","43939394961","43969697992","44000001023","44030304054","44060607085","44090910116","44121213147","44151516178","44181819209","44212122240","44242425271","44272728302","44303031333","44333334364","44363637395","44393940426","44424243457","44454546488","44484849519","44515152550","44545455581","44575758612","44606061643","44636364674","44666667705","44696970736","44727273767","44757576798","44787879829","44818182860","44848485891","44878788922","44909091953","44939394984","44969698015","45000001046"],"frame_ids":["47279001093883922744034","47312319628825279333091","47345638163766635922148","47378956698707992511205","47412275233649349100262","47445593768590705689319","47478912303532062278376","47512230838473418867433","47545549373414775456490","47578867908356132045547","47612186443297488634604","47645504978238845223661","47678823513180201812718","47712142048121558401775","47745460583062914990832","47778779118004271579889","47812097652945628168946","47845416187886984758003","47878734722828341347060","47912053257769697936117","47945371792711054525174","47978690327652411114231","48012008862593767703288","48045327397535124292345","48078645932476480881402","48111964467417837470459","48145283002359194059516","48178601537300550648573","48211920072241907237630","48245238607183263826687","48278557142124620415744","48311875677065977004801","48345194212007333593858","48378512746948690182915","48411831281890046771972","48445149816831403361029","48478468351772759950086","48511786886714116539143","48545105421655473128200","48578423956596829717257","48611742491538186306314","48645061026479542895371","48678379561420899484428","48711698096362256073485","48745016631303612662542","48778335166244969251599","48811653701186325840656","48844972236127682429713","48878290771069039018770","48911609306010395607827","48944927840951752196884","48978246375893108785941","49011564910834465374998","49044883445775821964055","49078201980717178553112","49111520515658535142169","49144839050599891731226","49178157585541248320283","49211476120482604909340","49244794655423961498397","49278113190365318087454","49311431725306674676511","49344750260248031265568","49378068795189387854625","49411387330130744443682","49444705865072101032739","49478024400013457621796"],"sequence_ids":[738,739,740,741,742,743,744,745,746,747,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800,801,802,803,804]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["43000002000","43030305031","43060608062","43090911093","43121214124","43151517155","43181820186","43212123217","43242426248","43272729279","43303032310","43333335341","43363638372","43393941403","43424244434","43454547465","43484850496","43515153527","43545456558","43575759589","43606062620","43636365651","43666668682","43696971713","43727274744","43757577775","43787880806","43818183837","43848486868","43878789899","43909092930","43939395961","43969698992","44000002023","44030305054","44060608085","44090911116","44121214147","44151517178","44181820209","44212123240","44242426271","44272729302","44303032333","44333335364","44363638395","44393941426","44424244457","44454547488","44484850519","44515153550","44545456581","44575759612","44606062643","44636365674","44666668705","44696971736","44727274767","44757577798","44787880829","44818183860","44848486891","44878789922","44909092953","44939395984","44969699015","45000002046"],"frame_ids":["47279002193399845487330","47312320728341202076387","47345639263282558665444","47378957798223915254501","47412276333165271843558","47445594868106628432615","47478913403047985021672","47512231937989341610729","47545550472930698199786","47578869007872054788843","47612187542813411377900","47645506077754767966957","47678824612696124556014","47712143147637481145071","47745461682578837734128","47778780217520194323185","47812098752461550912242","47845417287402907501299","47878735822344264090356","47912054357285620679413","47945372892226977268470","47978691427168333857527","48012009962109690446584","48045328497051047035641","48078647031992403624698","48111965566933760213755","48145284101875116802812","48178602636816473391869","48211921171757829980926","48245239706699186569983","48278558241640543159040","48311876776581899748097","48345195311523256337154","48378513846464612926211","48411832381405969515268","48445150916347326104325","48478469451288682693382","48511787986230039282439","48545106521171395871496","48578425056112752460553","48611743591054109049610","48645062125995465638667","48678380660936822227724","48711699195878178816781","48745017730819535405838","48778336265760891994895","48811654800702248583952","48844973335643605173009","48878291870584961762066","48911610405526318351123","48944928940467674940180","48978247475409031529237","49011566010350388118294","49044884545291744707351","49078203080233101296408","49111521615174457885465","49144840150115814474522","49178158685057171063579","49211477219998527652636","49244795754939884241693","49278114289881240830750","49311432824822597419807","49344751359763954008864","49378069894705310597921","49411388429646667186978","49444706964588023776035","49478025499529380365092"],"sequence_ids":[738,739,740,741,742,743,744,745,746,747,748,749,750,751,752,753,754,755,756,757,758,759,760,761,762,763,764,765,766,767,768,769,770,771,772,773,774,775,776,777,778,779,780,781,782,783,784,785,786,787,788,789,790,791,792,793,794,795,796,797,798,799,800,801,802,803,804]}]},{"cohort_id":"synthetic-fit-v2-r0-c3-t0-0","trial_id":"synthetic-fit-v2-r0-c3-t0","session_id":"synthetic-fit-v2-r0-c3-t0","role":"Train","label":"AdjacentOnly","cohort_sha256":"770cb4ac22965a33d9895af740220efe788c6e79724d2c3cd8d4184235de6204","target_ns":"46000000000","sealed_at_ns":"48000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["46000001000","46030304031","46060607062","46090910093","46121213124","46151516155","46181819186","46212122217","46242425248","46272728279","46303031310","46333334341","46363637372","46393940403","46424243434","46454546465","46484849496","46515152527","46545455558","46575758589","46606061620","46636364651","46666667682","46696970713","46727273744","46757576775","46787879806","46818182837","46848485868","46878788899","46909091930","46939394961","46969697992","47000001023","47030304054","47060607085","47090910116","47121213147","47151516178","47181819209","47212122240","47242425271","47272728302","47303031333","47333334364","47363637395","47393940426","47424243457","47454546488","47484849519","47515152550","47545455581","47575758612","47606061643","47636364674","47666667705","47696970736","47727273767","47757576798","47787879829","47818182860","47848485891","47878788922","47909091953","47939394984","47969698015","48000001046"],"frame_ids":["50577535977211922744101","50610854512153279333158","50644173047094635922215","50677491582035992511272","50710810116977349100329","50744128651918705689386","50777447186860062278443","50810765721801418867500","50844084256742775456557","50877402791684132045614","50910721326625488634671","50944039861566845223728","50977358396508201812785","51010676931449558401842","51043995466390914990899","51077314001332271579956","51110632536273628169013","51143951071214984758070","51177269606156341347127","51210588141097697936184","51243906676039054525241","51277225210980411114298","51310543745921767703355","51343862280863124292412","51377180815804480881469","51410499350745837470526","51443817885687194059583","51477136420628550648640","51510454955569907237697","51543773490511263826754","51577092025452620415811","51610410560393977004868","51643729095335333593925","51677047630276690182982","51710366165218046772039","51743684700159403361096","51777003235100759950153","51810321770042116539210","51843640304983473128267","51876958839924829717324","51910277374866186306381","51943595909807542895438","51976914444748899484495","52010232979690256073552","52043551514631612662609","52076870049572969251666","52110188584514325840723","52143507119455682429780","52176825654397039018837","52210144189338395607894","52243462724279752196951","52276781259221108786008","52310099794162465375065","52343418329103821964122","52376736864045178553179","52410055398986535142236","52443373933927891731293","52476692468869248320350","52510011003810604909407","52543329538751961498464","52576648073693318087521","52609966608634674676578","52643285143576031265635","52676603678517387854692","52709922213458744443749","52743240748400101032806","52776559283341457621863"],"sequence_ids":[805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["46000002000","46030305031","46060608062","46090911093","46121214124","46151517155","46181820186","46212123217","46242426248","46272729279","46303032310","46333335341","46363638372","46393941403","46424244434","46454547465","46484850496","46515153527","46545456558","46575759589","46606062620","46636365651","46666668682","46696971713","46727274744","46757577775","46787880806","46818183837","46848486868","46878789899","46909092930","46939395961","46969698992","47000002023","47030305054","47060608085","47090911116","47121214147","47151517178","47181820209","47212123240","47242426271","47272729302","47303032333","47333335364","47363638395","47393941426","47424244457","47454547488","47484850519","47515153550","47545456581","47575759612","47606062643","47636365674","47666668705","47696971736","47727274767","47757577798","47787880829","47818183860","47848486891","47878789922","47909092953","47939395984","47969699015","48000002046"],"frame_ids":["50577537076727845487397","50610855611669202076454","50644174146610558665511","50677492681551915254568","50710811216493271843625","50744129751434628432682","50777448286375985021739","50810766821317341610796","50844085356258698199853","50877403891200054788910","50910722426141411377967","50944040961082767967024","50977359496024124556081","51010678030965481145138","51043996565906837734195","51077315100848194323252","51110633635789550912309","51143952170730907501366","51177270705672264090423","51210589240613620679480","51243907775554977268537","51277226310496333857594","51310544845437690446651","51343863380379047035708","51377181915320403624765","51410500450261760213822","51443818985203116802879","51477137520144473391936","51510456055085829980993","51543774590027186570050","51577093124968543159107","51610411659909899748164","51643730194851256337221","51677048729792612926278","51710367264733969515335","51743685799675326104392","51777004334616682693449","51810322869558039282506","51843641404499395871563","51876959939440752460620","51910278474382109049677","51943597009323465638734","51976915544264822227791","52010234079206178816848","52043552614147535405905","52076871149088891994962","52110189684030248584019","52143508218971605173076","52176826753912961762133","52210145288854318351190","52243463823795674940247","52276782358737031529304","52310100893678388118361","52343419428619744707418","52376737963561101296475","52410056498502457885532","52443375033443814474589","52476693568385171063646","52510012103326527652703","52543330638267884241760","52576649173209240830817","52609967708150597419874","52643286243091954008931","52676604778033310597988","52709923312974667187045","52743241847916023776102","52776560382857380365159"],"sequence_ids":[805,806,807,808,809,810,811,812,813,814,815,816,817,818,819,820,821,822,823,824,825,826,827,828,829,830,831,832,833,834,835,836,837,838,839,840,841,842,843,844,845,846,847,848,849,850,851,852,853,854,855,856,857,858,859,860,861,862,863,864,865,866,867,868,869,870,871]}]},{"cohort_id":"synthetic-fit-v2-r0-c3-t0-1","trial_id":"synthetic-fit-v2-r0-c3-t0","session_id":"synthetic-fit-v2-r0-c3-t0","role":"Train","label":"AdjacentOnly","cohort_sha256":"760fc88fee72cff507da886a62ef718dd5f125ad9dd03ba30e672c17f5aad7c7","target_ns":"49000000000","sealed_at_ns":"51000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["49000001000","49030304031","49060607062","49090910093","49121213124","49151516155","49181819186","49212122217","49242425248","49272728279","49303031310","49333334341","49363637372","49393940403","49424243434","49454546465","49484849496","49515152527","49545455558","49575758589","49606061620","49636364651","49666667682","49696970713","49727273744","49757576775","49787879806","49818182837","49848485868","49878788899","49909091930","49939394961","49969697992","50000001023","50030304054","50060607085","50090910116","50121213147","50151516178","50181819209","50212122240","50242425271","50272728302","50303031333","50333334364","50363637395","50393940426","50424243457","50454546488","50484849519","50515152550","50545455581","50575758612","50606061643","50636364674","50666667705","50696970736","50727273767","50757576798","50787879829","50818182860","50848485891","50878788922","50909091953","50939394984","50969698015","51000001046"],"frame_ids":["53876070860539922744168","53909389395481279333225","53942707930422635922282","53976026465363992511339","54009345000305349100396","54042663535246705689453","54075982070188062278510","54109300605129418867567","54142619140070775456624","54175937675012132045681","54209256209953488634738","54242574744894845223795","54275893279836201812852","54309211814777558401909","54342530349718914990966","54375848884660271580023","54409167419601628169080","54442485954542984758137","54475804489484341347194","54509123024425697936251","54542441559367054525308","54575760094308411114365","54609078629249767703422","54642397164191124292479","54675715699132480881536","54709034234073837470593","54742352769015194059650","54775671303956550648707","54808989838897907237764","54842308373839263826821","54875626908780620415878","54908945443721977004935","54942263978663333593992","54975582513604690183049","55008901048546046772106","55042219583487403361163","55075538118428759950220","55108856653370116539277","55142175188311473128334","55175493723252829717391","55208812258194186306448","55242130793135542895505","55275449328076899484562","55308767863018256073619","55342086397959612662676","55375404932900969251733","55408723467842325840790","55442042002783682429847","55475360537725039018904","55508679072666395607961","55541997607607752197018","55575316142549108786075","55608634677490465375132","55641953212431821964189","55675271747373178553246","55708590282314535142303","55741908817255891731360","55775227352197248320417","55808545887138604909474","55841864422079961498531","55875182957021318087588","55908501491962674676645","55941820026904031265702","55975138561845387854759","56008457096786744443816","56041775631728101032873","56075094166669457621930"],"sequence_ids":[872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,894,895,896,897,898,899,900,901,902,903,904,905,906,907,908,909,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,930,931,932,933,934,935,936,937,938]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["49000002000","49030305031","49060608062","49090911093","49121214124","49151517155","49181820186","49212123217","49242426248","49272729279","49303032310","49333335341","49363638372","49393941403","49424244434","49454547465","49484850496","49515153527","49545456558","49575759589","49606062620","49636365651","49666668682","49696971713","49727274744","49757577775","49787880806","49818183837","49848486868","49878789899","49909092930","49939395961","49969698992","50000002023","50030305054","50060608085","50090911116","50121214147","50151517178","50181820209","50212123240","50242426271","50272729302","50303032333","50333335364","50363638395","50393941426","50424244457","50454547488","50484850519","50515153550","50545456581","50575759612","50606062643","50636365674","50666668705","50696971736","50727274767","50757577798","50787880829","50818183860","50848486891","50878789922","50909092953","50939395984","50969699015","51000002046"],"frame_ids":["53876071960055845487464","53909390494997202076521","53942709029938558665578","53976027564879915254635","54009346099821271843692","54042664634762628432749","54075983169703985021806","54109301704645341610863","54142620239586698199920","54175938774528054788977","54209257309469411378034","54242575844410767967091","54275894379352124556148","54309212914293481145205","54342531449234837734262","54375849984176194323319","54409168519117550912376","54442487054058907501433","54475805589000264090490","54509124123941620679547","54542442658882977268604","54575761193824333857661","54609079728765690446718","54642398263707047035775","54675716798648403624832","54709035333589760213889","54742353868531116802946","54775672403472473392003","54808990938413829981060","54842309473355186570117","54875628008296543159174","54908946543237899748231","54942265078179256337288","54975583613120612926345","55008902148061969515402","55042220683003326104459","55075539217944682693516","55108857752886039282573","55142176287827395871630","55175494822768752460687","55208813357710109049744","55242131892651465638801","55275450427592822227858","55308768962534178816915","55342087497475535405972","55375406032416891995029","55408724567358248584086","55442043102299605173143","55475361637240961762200","55508680172182318351257","55541998707123674940314","55575317242065031529371","55608635777006388118428","55641954311947744707485","55675272846889101296542","55708591381830457885599","55741909916771814474656","55775228451713171063713","55808546986654527652770","55841865521595884241827","55875184056537240830884","55908502591478597419941","55941821126419954008998","55975139661361310598055","56008458196302667187112","56041776731244023776169","56075095266185380365226"],"sequence_ids":[872,873,874,875,876,877,878,879,880,881,882,883,884,885,886,887,888,889,890,891,892,893,894,895,896,897,898,899,900,901,902,903,904,905,906,907,908,909,910,911,912,913,914,915,916,917,918,919,920,921,922,923,924,925,926,927,928,929,930,931,932,933,934,935,936,937,938]}]},{"cohort_id":"synthetic-fit-v2-r0-c3-t1-0","trial_id":"synthetic-fit-v2-r0-c3-t1","session_id":"synthetic-fit-v2-r0-c3-t1","role":"Train","label":"AdjacentOnly","cohort_sha256":"3f567fe2f7357e982b18a0eebe3958175e1443d7bc4590484f450e4ba57cb3c0","target_ns":"52000000000","sealed_at_ns":"54000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["52000001000","52030304031","52060607062","52090910093","52121213124","52151516155","52181819186","52212122217","52242425248","52272728279","52303031310","52333334341","52363637372","52393940403","52424243434","52454546465","52484849496","52515152527","52545455558","52575758589","52606061620","52636364651","52666667682","52696970713","52727273744","52757576775","52787879806","52818182837","52848485868","52878788899","52909091930","52939394961","52969697992","53000001023","53030304054","53060607085","53090910116","53121213147","53151516178","53181819209","53212122240","53242425271","53272728302","53303031333","53333334364","53363637395","53393940426","53424243457","53454546488","53484849519","53515152550","53545455581","53575758612","53606061643","53636364674","53666667705","53696970736","53727273767","53757576798","53787879829","53818182860","53848485891","53878788922","53909091953","53939394984","53969698015","54000001046"],"frame_ids":["57174605743867922744235","57207924278809279333292","57241242813750635922349","57274561348691992511406","57307879883633349100463","57341198418574705689520","57374516953516062278577","57407835488457418867634","57441154023398775456691","57474472558340132045748","57507791093281488634805","57541109628222845223862","57574428163164201812919","57607746698105558401976","57641065233046914991033","57674383767988271580090","57707702302929628169147","57741020837870984758204","57774339372812341347261","57807657907753697936318","57840976442695054525375","57874294977636411114432","57907613512577767703489","57940932047519124292546","57974250582460480881603","58007569117401837470660","58040887652343194059717","58074206187284550648774","58107524722225907237831","58140843257167263826888","58174161792108620415945","58207480327049977005002","58240798861991333594059","58274117396932690183116","58307435931874046772173","58340754466815403361230","58374073001756759950287","58407391536698116539344","58440710071639473128401","58474028606580829717458","58507347141522186306515","58540665676463542895572","58573984211404899484629","58607302746346256073686","58640621281287612662743","58673939816228969251800","58707258351170325840857","58740576886111682429914","58773895421053039018971","58807213955994395608028","58840532490935752197085","58873851025877108786142","58907169560818465375199","58940488095759821964256","58973806630701178553313","59007125165642535142370","59040443700583891731427","59073762235525248320484","59107080770466604909541","59140399305407961498598","59173717840349318087655","59207036375290674676712","59240354910232031265769","59273673445173387854826","59306991980114744443883","59340310515056101032940","59373629049997457621997"],"sequence_ids":[939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,964,965,966,967,968,969,970,971,972,973,974,975,976,977,978,979,980,981,982,983,984,985,986,987,988,989,990,991,992,993,994,995,996,997,998,999,1000,1001,1002,1003,1004,1005]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["52000002000","52030305031","52060608062","52090911093","52121214124","52151517155","52181820186","52212123217","52242426248","52272729279","52303032310","52333335341","52363638372","52393941403","52424244434","52454547465","52484850496","52515153527","52545456558","52575759589","52606062620","52636365651","52666668682","52696971713","52727274744","52757577775","52787880806","52818183837","52848486868","52878789899","52909092930","52939395961","52969698992","53000002023","53030305054","53060608085","53090911116","53121214147","53151517178","53181820209","53212123240","53242426271","53272729302","53303032333","53333335364","53363638395","53393941426","53424244457","53454547488","53484850519","53515153550","53545456581","53575759612","53606062643","53636365674","53666668705","53696971736","53727274767","53757577798","53787880829","53818183860","53848486891","53878789922","53909092953","53939395984","53969699015","54000002046"],"frame_ids":["57174606843383845487531","57207925378325202076588","57241243913266558665645","57274562448207915254702","57307880983149271843759","57341199518090628432816","57374518053031985021873","57407836587973341610930","57441155122914698199987","57474473657856054789044","57507792192797411378101","57541110727738767967158","57574429262680124556215","57607747797621481145272","57641066332562837734329","57674384867504194323386","57707703402445550912443","57741021937386907501500","57774340472328264090557","57807659007269620679614","57840977542210977268671","57874296077152333857728","57907614612093690446785","57940933147035047035842","57974251681976403624899","58007570216917760213956","58040888751859116803013","58074207286800473392070","58107525821741829981127","58140844356683186570184","58174162891624543159241","58207481426565899748298","58240799961507256337355","58274118496448612926412","58307437031389969515469","58340755566331326104526","58374074101272682693583","58407392636214039282640","58440711171155395871697","58474029706096752460754","58507348241038109049811","58540666775979465638868","58573985310920822227925","58607303845862178816982","58640622380803535406039","58673940915744891995096","58707259450686248584153","58740577985627605173210","58773896520568961762267","58807215055510318351324","58840533590451674940381","58873852125393031529438","58907170660334388118495","58940489195275744707552","58973807730217101296609","59007126265158457885666","59040444800099814474723","59073763335041171063780","59107081869982527652837","59140400404923884241894","59173718939865240830951","59207037474806597420008","59240356009747954009065","59273674544689310598122","59306993079630667187179","59340311614572023776236","59373630149513380365293"],"sequence_ids":[939,940,941,942,943,944,945,946,947,948,949,950,951,952,953,954,955,956,957,958,959,960,961,962,963,964,965,966,967,968,969,970,971,972,973,974,975,976,977,978,979,980,981,982,983,984,985,986,987,988,989,990,991,992,993,994,995,996,997,998,999,1000,1001,1002,1003,1004,1005]}]},{"cohort_id":"synthetic-fit-v2-r0-c3-t1-1","trial_id":"synthetic-fit-v2-r0-c3-t1","session_id":"synthetic-fit-v2-r0-c3-t1","role":"Train","label":"AdjacentOnly","cohort_sha256":"fcd7005b9f0f277ba32119af2d48939fa885104ffb4efa5768bc8313eea2a1c0","target_ns":"55000000000","sealed_at_ns":"57000002046","provenance":"Synthetic","sources":[{"source_id":1,"source_spec_sha256":"ea991b6de43cd497ae7a09091d30f7a4a181ce6ced05b217bca4dc3a396e8bdf","times_ns":["55000001000","55030304031","55060607062","55090910093","55121213124","55151516155","55181819186","55212122217","55242425248","55272728279","55303031310","55333334341","55363637372","55393940403","55424243434","55454546465","55484849496","55515152527","55545455558","55575758589","55606061620","55636364651","55666667682","55696970713","55727273744","55757576775","55787879806","55818182837","55848485868","55878788899","55909091930","55939394961","55969697992","56000001023","56030304054","56060607085","56090910116","56121213147","56151516178","56181819209","56212122240","56242425271","56272728302","56303031333","56333334364","56363637395","56393940426","56424243457","56454546488","56484849519","56515152550","56545455581","56575758612","56606061643","56636364674","56666667705","56696970736","56727273767","56757576798","56787879829","56818182860","56848485891","56878788922","56909091953","56939394984","56969698015","57000001046"],"frame_ids":["60473140627195922744302","60506459162137279333359","60539777697078635922416","60573096232019992511473","60606414766961349100530","60639733301902705689587","60673051836844062278644","60706370371785418867701","60739688906726775456758","60773007441668132045815","60806325976609488634872","60839644511550845223929","60872963046492201812986","60906281581433558402043","60939600116374914991100","60972918651316271580157","61006237186257628169214","61039555721198984758271","61072874256140341347328","61106192791081697936385","61139511326023054525442","61172829860964411114499","61206148395905767703556","61239466930847124292613","61272785465788480881670","61306104000729837470727","61339422535671194059784","61372741070612550648841","61406059605553907237898","61439378140495263826955","61472696675436620416012","61506015210377977005069","61539333745319333594126","61572652280260690183183","61605970815202046772240","61639289350143403361297","61672607885084759950354","61705926420026116539411","61739244954967473128468","61772563489908829717525","61805882024850186306582","61839200559791542895639","61872519094732899484696","61905837629674256073753","61939156164615612662810","61972474699556969251867","62005793234498325840924","62039111769439682429981","62072430304381039019038","62105748839322395608095","62139067374263752197152","62172385909205108786209","62205704444146465375266","62239022979087821964323","62272341514029178553380","62305660048970535142437","62338978583911891731494","62372297118853248320551","62405615653794604909608","62438934188735961498665","62472252723677318087722","62505571258618674676779","62538889793560031265836","62572208328501387854893","62605526863442744443950","62638845398384101033007","62672163933325457622064"],"sequence_ids":[1006,1007,1008,1009,1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065,1066,1067,1068,1069,1070,1071,1072]},{"source_id":2,"source_spec_sha256":"ec550d93545ee35041bd6e85e79292ebf40dddb726353c1701a5aeeb232761a5","times_ns":["55000002000","55030305031","55060608062","55090911093","55121214124","55151517155","55181820186","55212123217","55242426248","55272729279","55303032310","55333335341","55363638372","55393941403","55424244434","55454547465","55484850496","55515153527","55545456558","55575759589","55606062620","55636365651","55666668682","55696971713","55727274744","55757577775","55787880806","55818183837","55848486868","55878789899","55909092930","55939395961","55969698992","56000002023","56030305054","56060608085","56090911116","56121214147","56151517178","56181820209","56212123240","56242426271","56272729302","56303032333","56333335364","56363638395","56393941426","56424244457","56454547488","56484850519","56515153550","56545456581","56575759612","56606062643","56636365674","56666668705","56696971736","56727274767","56757577798","56787880829","56818183860","56848486891","56878789922","56909092953","56939395984","56969699015","57000002046"],"frame_ids":["60473141726711845487598","60506460261653202076655","60539778796594558665712","60573097331535915254769","60606415866477271843826","60639734401418628432883","60673052936359985021940","60706371471301341610997","60739690006242698200054","60773008541184054789111","60806327076125411378168","60839645611066767967225","60872964146008124556282","60906282680949481145339","60939601215890837734396","60972919750832194323453","61006238285773550912510","61039556820714907501567","61072875355656264090624","61106193890597620679681","61139512425538977268738","61172830960480333857795","61206149495421690446852","61239468030363047035909","61272786565304403624966","61306105100245760214023","61339423635187116803080","61372742170128473392137","61406060705069829981194","61439379240011186570251","61472697774952543159308","61506016309893899748365","61539334844835256337422","61572653379776612926479","61605971914717969515536","61639290449659326104593","61672608984600682693650","61705927519542039282707","61739246054483395871764","61772564589424752460821","61805883124366109049878","61839201659307465638935","61872520194248822227992","61905838729190178817049","61939157264131535406106","61972475799072891995163","62005794334014248584220","62039112868955605173277","62072431403896961762334","62105749938838318351391","62139068473779674940448","62172387008721031529505","62205705543662388118562","62239024078603744707619","62272342613545101296676","62305661148486457885733","62338979683427814474790","62372298218369171063847","62405616753310527652904","62438935288251884241961","62472253823193240831018","62505572358134597420075","62538890893075954009132","62572209428017310598189","62605527962958667187246","62638846497900023776303","62672165032841380365360"],"sequence_ids":[1006,1007,1008,1009,1010,1011,1012,1013,1014,1015,1016,1017,1018,1019,1020,1021,1022,1023,1024,1025,1026,1027,1028,1029,1030,1031,1032,1033,1034,1035,1036,1037,1038,1039,1040,1041,1042,1043,1044,1045,1046,1047,1048,1049,1050,1051,1052,1053,1054,1055,1056,1057,1058,1059,1060,1061,1062,1063,1064,1065,1066,1067,1068,1069,1070,1071,1072]}]}],"model_sha256":"0439ec86ebd7b3ae0285cf89a768e0d70bdc37f581a48ca7a5626240a8e06db0","model_json":"{\"version\":1,\"algorithm\":\"aether_multiview_infonce_room_bce_v1\",\"context\":{\"owner_scope\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"installation_id\":\"synthetic-temporal\",\"room_digest\":\"synthetic-room\",\"revision\":1,\"sources\":[{\"source_id\":1,\"device_id\":\"esp32-node-1\",\"grid_id\":\"ant1-bins64-ppdu0-usable-drop2-linear-v2\",\"feature_count\":56,\"signal_kind\":\"WifiCsi\",\"center_frequency_hz\":2437000000,\"bandwidth_hz\":20000000,\"link_id\":\"native-1\",\"link_index\":0,\"geometry\":null},{\"source_id\":2,\"device_id\":\"esp32-node-2\",\"grid_id\":\"ant1-bins64-ppdu0-usable-drop2-linear-v2\",\"feature_count\":56,\"signal_kind\":\"WifiCsi\",\"center_frequency_hz\":2437000000,\"bandwidth_hz\":20000000,\"link_id\":\"native-2\",\"link_index\":0,\"geometry\":null}]},\"training_manifest\":{\"schema\":\"ruview.room-learner.train-manifest.v1\",\"training_provenance\":[\"Synthetic\"],\"training_data_sha256\":\"fe5579c3ff4e5853f1885dba20f1adc996e198d4c506c95a971e6afda6e6d9f0\",\"consumed_channels\":[\"magnitude\"],\"clock_claims\":[\"ReceiverOnly\"],\"measurement_clock_available\":false,\"training_cohorts\":16},\"config\":{\"bins\":56,\"source_count\":2,\"temporal_tokens\":4,\"hidden_dim\":3,\"embedding_dim\":3,\"temperature\":0.3},\"profiles\":[{\"source\":{\"source_id\":1,\"device_id\":\"esp32-node-1\",\"grid_id\":\"ant1-bins64-ppdu0-usable-drop2-linear-v2\",\"feature_count\":56,\"signal_kind\":\"WifiCsi\",\"center_frequency_hz\":2437000000,\"bandwidth_hz\":20000000,\"link_id\":\"native-1\",\"link_index\":0,\"geometry\":null},\"mean\":[5.656854,6.321713,5.819677,6.1588902,5.982499,5.996068,6.145322,5.8332453,6.3081446,5.670423,6.3352814,5.8061085,6.172459,5.9689307,6.0096364,6.1317534,5.8468137,6.2945757,5.6839914,6.3488503,5.7925396,6.1860275,5.9553623,6.023205,6.1181846,5.8603826,6.2810073,5.69756,6.3624187,5.778971,6.199596,5.941794,6.0367737,6.104616,5.873951,6.267439,5.711128,6.375987,5.765403,6.213165,5.928225,6.050342,6.091048,5.8875194,6.25387,5.724697,6.3895555,5.751834,6.226733,5.9146566,6.0639105,6.0774794,5.901088,6.2403016,5.7382655,6.4031243],\"scale\":[0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1],\"max_normalized\":71.98347},{\"source\":{\"source_id\":2,\"device_id\":\"esp32-node-2\",\"grid_id\":\"ant1-bins64-ppdu0-usable-drop2-linear-v2\",\"feature_count\":56,\"signal_kind\":\"WifiCsi\",\"center_frequency_hz\":2437000000,\"bandwidth_hz\":20000000,\"link_id\":\"native-2\",\"link_index\":0,\"geometry\":null},\"mean\":[6.4031243,7.1229596,6.5794106,6.9466734,6.755697,6.770387,6.931983,6.594101,7.1082687,6.4178147,7.13765,6.5647197,6.961364,6.741006,6.7850776,6.917292,6.6087914,7.0935783,6.432505,7.1523404,6.5500293,6.976054,6.7263155,6.799768,6.9026017,6.6234818,7.078888,6.447196,7.167031,6.535339,6.9907446,6.711625,6.814459,6.8879113,6.6381726,7.0641975,6.4618864,7.1817217,6.5206485,7.0054355,6.6969347,6.8291492,6.873221,6.652863,7.0495067,6.476577,7.196412,6.5059576,7.020126,6.682244,6.8438396,6.85853,6.6675534,7.0348163,6.491267,7.2111025],\"scale\":[0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1],\"max_normalized\":73.49117}],\"encoder_weights\":[-0.31405702,-0.15605538,-0.019411659,-0.23722431,0.14105025,0.12448881,-0.17316124,-0.28230542,0.25161946,0.23627274,0.083663724,-0.11510868,0.16076146,-0.20444109,-0.28943694,-0.03411386,-0.05305876,0.1268511,-0.07930206,-0.16969015,-0.122171946,-0.18971951,0.2779942,-0.20771396,0.09614456,-0.27768067,-0.083600804,0.060154844,-0.27317116,-0.25799847,0.054317452,-0.31186154,-0.021532401,-0.24256697,-0.28336817,0.05801737,0.23311336,-0.08284997,0.18425474,0.3019741,0.0971509,-0.014180818,0.077958286,-0.21174139,-0.1348683,-0.18674442,-0.087604724,-0.13767663,0.10686318,0.17436363,0.1432364,0.08204346,-0.11314485,0.191478,-0.18946445,0.08444698,-1.0963722,-0.40854564,0.010921259,0.1592275,-0.24847019,0.28674957,0.22478792,0.17557,0.017283216,-0.1662348,-0.12553196,-0.04142716,-0.3048477,-0.05052951,0.30713764,0.016659763,-0.29538757,-0.22913143,0.12262335,-0.21247649,0.02428204,-0.18386027,0.26374215,-0.24371208,0.31186777,0.16386072,0.16449225,0.08720322,-0.121891394,-0.17309855,0.291033,-0.25434098,0.04707865,0.30687234,-0.23781548,0.15851216,-0.14798075,-0.18100414,-0.19928756,0.16521227,0.28165337,-0.008154298,-0.29637957,0.113215655,-0.2083508,0.11165612,-0.28445366,-0.20599249,0.083394624,-0.20772244,0.03947674,0.025858924,-0.07172223,-0.097277224,-0.23376308,0.23801543,0.12796475,-0.1848585,0.8503,0.9065268,-0.13296387,0.2758319,-0.2584599,0.19948332,-0.1988909,-0.1624675,0.16195628,-0.11784503,-0.10497568,0.19418798,-0.17294578,0.18471056,-0.08037034,0.0400655,0.0510028,-0.22936594,-0.16656943,0.16831462,0.12635878,0.044187985,0.21214493,0.17698123,0.0735129,-0.08216295,-0.15006073,0.22780012,0.1660651,-0.056969505,0.016330061,-0.027255945,0.28889793,-0.17832844,-0.17903326,0.12734558,0.2609079,-0.066520445,-0.047734052,-0.21986984,-0.2541291,0.050483312,-0.14191192,0.24874592,-0.16762812,-0.22350667,-0.13995315,0.11064909,-0.096272446,0.021436704,-0.2508471,-0.057291474,0.21689415,0.046441052,0.060559656,-0.3120285,-0.10967854,-0.21053293,-0.55092996,-0.5127955,-0.0028662088,-0.0010812389,-0.0024997608,-0.5477225,-0.064292245,-0.020785768,-0.50536597,-0.41897824,-0.5421137,0.20898235,-0.1542391,0.0101150265,-0.4036921,-0.24830106,0.27814552,-0.14486068,0.30726537,0.0039984505,-0.047804985,0.23023543,0.06093011,-0.37791353,-0.5216652,-0.3231506,0.19728532,-0.07173542,0.38454613,-0.40621033,0.4701021,-0.12535347,0.42216882,0.17630844,-0.1076794,0.16181,-0.1960641,-0.44001916,-0.53976184,-0.52484024,0.25335217,-0.17232236,0.13170429,0.08339477,0.5360642,-0.42585057,0.08468615,-0.13617964,0.5370887,-0.47148478,-0.3414682,0.3196109,-0.030125791,-0.33323315,0.33207354,-0.19529352,-1.0,0.3759377,-0.18503237,-0.04400152,0.19037545,0.85269415,-0.18120205,0.5656487,-0.96143025,0.0,0.0,0.0,-1.0,0.50096095,0.32058,-0.9772421,0.8576602,-0.2860003,-0.7461792,-0.14932656,0.6489322,0.0,0.0,0.0,-0.99967045,0.6257085,-0.6084689,0.93868876,-0.16036174,0.36021298,0.021588584,0.33143902,0.3636718,-0.0003348433,-0.00334344,-0.004175199,-0.99997467,0.7509599,0.24381536,-0.07059898,-0.47224766,-0.8430445,0.054828934,0.672361,0.9807283,0.000014342368,-0.0061520804,-0.009535632,1.0,0.0,0.0,0.0150192715,0.9982554,0.0026730986,0.024910392,-0.0037568703,1.0054708,1.0,0.993806,0.99038875,-0.99999994,0.004587412,0.47660053,-0.87525564,0.010010958,0.35218668,0.6689781,-0.722959,0.6814921,0.0,0.0,0.0,-1.2247448,0.61892134,-0.07202221,0.0],\"source_projection_weights\":[[-0.9999976,-0.8407852,-0.3105338,0.75500476,-0.67136806,-0.8269221,0.995204,-0.40957043,0.66540647,1.445065,1.3654729,0.49059448,-1.0003865,-0.9710343,0.5249519,-0.8282895,0.3677307,0.7756716,-0.27217376,0.10625149,0.312992,0.1267061,-0.009233973,-0.15559104],[-0.9999976,-0.5963865,-0.14052066,0.20321238,0.052331902,-0.2891694,-0.32647085,0.90668213,0.9648173,0.89604247,0.25865722,-1.6689024,-1.0002123,-0.720038,0.9145221,-0.1301688,-0.9965646,0.34389147,0.90929794,-0.57949734,0.051250495,-0.0006814752,-0.00033052405,-0.0029046456]],\"head_weights\":[0.003513919,-0.0003797346,0.0039039822,0.020262968,0.0005923073,0.004028996,0.0034028515,0.0000056478602,0.015872136,0.00059702375,-0.026364833]}"}} \ No newline at end of file From 0d98d443f9b44a9494ec894040efb074e3743044 Mon Sep 17 00:00:00 2001 From: Codex for Dragan Spiridonov Date: Fri, 6 Nov 2026 20:24:02 +0100 Subject: [PATCH 069/100] feat(learner): keep private temporal capture persistence off ingress lock --- .../c2_precision_pacer_test_support.rs | 96 ++ .../src/room_learner/lifecycle.rs | 18 +- .../src/room_learner/lifecycle_capture.rs | 8 +- .../src/room_learner/lifecycle_temporal.rs | 222 ++++- .../room_learner/lifecycle_temporal_tests.rs | 611 +++++++++++- .../room_learner/temporal_capture_quality.rs | 98 +- .../src/room_learner/temporal_checkpoint.rs | 71 +- .../room_learner/temporal_checkpoint_tests.rs | 18 + .../src/room_learner/temporal_durability.rs | 878 ++++++++++++++++++ .../src/room_learner/temporal_execution.rs | 31 +- .../src/room_learner/temporal_model_input.rs | 47 +- .../temporal_model_input_managed.rs | 7 +- 12 files changed, 2039 insertions(+), 66 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/c2_precision_pacer_test_support.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_durability.rs diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/c2_precision_pacer_test_support.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/c2_precision_pacer_test_support.rs new file mode 100644 index 0000000000..615c413d03 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/c2_precision_pacer_test_support.rs @@ -0,0 +1,96 @@ +//! PREPARED ONLY: std-only, no receiver/user-data/network access. +//! One baseline control + one absolute-deadline precision strategy; no tuning loop. +use std::{hint, time::{Duration, Instant}}; +const HZ: u64 = 33; +const NS: u64 = 1_000_000_000; +// Frozen r2 temporal_acquisition.rs217 and ingest_checked811–815; +// temporal_execution.rs484–489 also enforces this for whole-period input. +// This is ingestion compatibility, NOT an extra finish_checked predicate. +const MIN_INTERVAL: u64 = 1_000_000_000_000 / 36_000; +const MAX_GAP: u64 = 50_000_000; +const WINDOW_NS: u64 = 2_000_000_000; +const MAX_WINDOW_NS: u64 = 2_050_000_000; +const MIN_ROWS: usize = 61; +const LO: u128 = 30_000; +const HI: u128 = 36_000; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum Decision { Wait(u64), Emit(u64), MissedPeriod(u64) } +fn due_ns(index: usize) -> u64 { (index as u128 * u128::from(NS) / u128::from(HZ)) as u64 } +fn decide(index: usize, actual_ns: u64) -> Decision { + let due = due_ns(index); + if actual_ns < due { Decision::Wait(due - actual_ns) } + else if actual_ns >= due_ns(index + 1) { Decision::MissedPeriod(actual_ns - due) } + else { Decision::Emit(actual_ns - due) } +} +#[derive(Default, Clone, Copy)] +struct WaitCost { requested_sleep: u64, actual_sleep: u64, spin: u64 } +struct Wake { actual: u64, cost: WaitCost, missed: bool } +/// The emitted timestamp is ACTUAL monotonic wakeup, never due_ns(index). +/// Absolute deadlines never reset from actual wakeup. A missed whole period +/// stops the strategy rather than emitting catch-up/multiple batches. +fn wait_absolute(origin: Instant, index: usize, guard_ns: u64) -> Wake { + let due = origin.checked_add(Duration::from_nanos(due_ns(index))).expect("bounded study deadline"); + let mut cost = WaitCost::default(); + let now = Instant::now(); + if now < due { + let remaining = due.duration_since(now); + let guard = Duration::from_nanos(guard_ns); + if remaining > guard { + let sleep_for = remaining - guard; + cost.requested_sleep = sleep_for.as_nanos() as u64; + let slept = Instant::now(); std::thread::sleep(sleep_for); + cost.actual_sleep = slept.elapsed().as_nanos() as u64; + } + let spinning = Instant::now(); + while Instant::now() < due { hint::spin_loop(); } + cost.spin = spinning.elapsed().as_nanos() as u64; + } + let actual = origin.elapsed().as_nanos() as u64; + Wake { actual, cost, missed: matches!(decide(index, actual), Decision::MissedPeriod(_)) } +} +fn rate_ok(count: usize, span: u64) -> bool { + span > 0 && (count.saturating_sub(1) as u128) * 1_000_000_000_000 >= LO * u128::from(span) + && (count.saturating_sub(1) as u128) * 1_000_000_000_000 <= HI * u128::from(span) +} +// Mirrors retained-window readiness + finish span/mean cadence only. +fn first_window(times: &[u64]) -> Option<(usize, u64, bool)> { + let first = *times.first()?; + for count in MIN_ROWS..=times.len() { + let span = times[count - 1].checked_sub(first)?; + if span >= WINDOW_NS { return Some((count, span, span <= MAX_WINDOW_NS && rate_ok(count, span))); } + } + None +} +// Complete input compatibility combines REAL ingestion intervals with the +// separately checked retained-window mean/span/horizon. No invented gate. +fn qualifies(times: &[u64], requested: usize, missed: bool) -> bool { + if missed || times.len() != requested || times.len() < 2 { return false; } + if times.windows(2).any(|w| w[1] <= w[0] || w[1] - w[0] < MIN_INTERVAL || w[1] - w[0] > MAX_GAP) { return false; } + let span = times[times.len() - 1] - times[0]; + rate_ok(times.len(), span) && first_window(times).is_some_and(|(_,_,ok)| ok) +} +#[derive(Debug)] +pub(super) struct InputStudy { + pub qualified:bool, pub requested:usize, pub emitted:usize, + pub span_ns:u64, pub rate_millihz:u64, pub min_gap_ns:u64, pub max_gap_ns:u64, + pub missed_index:Option, pub spin_wall_ns:u128, pub sleep_overshoot_max_ns:u64, +} +pub(super) fn input_study(samples:usize,guard_ns:u64)->InputStudy { + assert!((70..=330).contains(&samples));assert!((500_000..=3_000_000).contains(&guard_ns)); + let origin=Instant::now();let mut times=Vec::with_capacity(samples);let mut missed=None;let mut spin=0u128;let mut overshoot=0; + for index in 0..samples { + let wake=wait_absolute(origin,index,guard_ns);spin+=u128::from(wake.cost.spin);overshoot=overshoot.max(wake.cost.actual_sleep.saturating_sub(wake.cost.requested_sleep)); + if wake.missed{missed=Some(index);break;}times.push(wake.actual); + } + let span=times.last().zip(times.first()).map_or(0,|(a,b)|a.saturating_sub(*b)); + let gaps=times.windows(2).map(|w|w[1].saturating_sub(w[0])).collect::>(); + InputStudy{qualified:qualifies(×,samples,missed.is_some()),requested:samples,emitted:times.len(),span_ns:span, + rate_millihz:if span==0{0}else{((times.len().saturating_sub(1)as u128)*1_000_000_000_000/u128::from(span))as u64}, + min_gap_ns:gaps.iter().min().copied().unwrap_or(0),max_gap_ns:gaps.iter().max().copied().unwrap_or(0),missed_index:missed,spin_wall_ns:spin,sleep_overshoot_max_ns:overshoot} +} +pub(super) fn next_actual_receipt(origin:Instant,index:usize,guard_ns:u64)->Result{ + let wake=wait_absolute(origin,index,guard_ns); + if wake.missed{return Err(format!("input_unqualified: whole period missed index={index} actual_ns={} planned_ns={}",wake.actual,due_ns(index)));} + Ok(wake.actual) +} +pub(super) fn actual_stream_qualifies(times:&[u64])->bool{qualifies(times,times.len(),false)} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs index 54ce5833c2..198f57511b 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs @@ -7,6 +7,7 @@ use super::{ observation::{CaptureContext, DataRole, RoomLabel}, training::{FitConfig, TrainingReport}, }; +use super::temporal_model_input::managed::execution::DeferredReclamation; use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; use std::{ @@ -89,11 +90,22 @@ pub struct RoomLearningRuntime { owner: String, installation: String, store: ArtifactStore, - recovery_store: Option, + recovery_store: Option>, inner: Mutex, + #[cfg(test)] + reclamation_probe:Mutex>>, + #[cfg(test)] + panic_after_reclamation_ack:AtomicBool, clock_origin: std::time::Instant, clock_epoch_ns: u64, } +impl RoomLearningRuntime { + fn deferred_reclamation(&self)->DeferredReclamation{ + let retired=DeferredReclamation::default(); + #[cfg(test)] let retired={let mut retired=retired;retired.probe=self.reclamation_probe.lock().unwrap_or_else(|e|e.into_inner()).clone();retired}; + retired + } +} fn busy(job: &Job) -> bool { matches!(job.phase, JobPhase::Running | JobPhase::Cancelling) } @@ -177,7 +189,9 @@ impl RoomLearningRuntime { owner: owner.into(), installation: installation.into(), store: ArtifactStore::new(directory, owner, installation)?, - recovery_store, + recovery_store: recovery_store.map(std::sync::Arc::new), + #[cfg(test)] reclamation_probe:Mutex::new(None), + #[cfg(test)] panic_after_reclamation_ack:AtomicBool::new(false), inner: Mutex::new(Inner { generation: 0, workflow_revision: 0, diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs index 65f94f5d75..88350e8e30 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs @@ -96,9 +96,15 @@ impl RoomLearningRuntime { self.record_native_rejection(packet.source_id,&packet.grid_id,"synthetic/replayed native provenance"); return Err("live managed capture rejects synthetic/replayed native provenance".into()); } + // Declared BEFORE the lock: every return/unwind releases inner before + // any accepted/rejected ACK payload or replaced raw state is reclaimed. + let mut retired=self.deferred_reclamation(); let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + let owner_generation=state.generation; if let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() { - let result = capture.ingest(self.temporal_store()?, &self.runtime_epoch, packet, now_ns); + if capture.owner_generation()!=owner_generation {capture.invalidate_owner();return Err("temporal_owner_generation_changed".into());} + let result = capture.ingest(self.temporal_store()?, &self.runtime_epoch, packet, now_ns,&mut retired); + #[cfg(test)] if retired.has_ack() && self.panic_after_reclamation_ack.swap(false,Ordering::AcqRel){panic!("synthetic panic after actual ACK admission");} if let Err(error) = &result { state.input_error = Some(bounded(error.clone())); } return result; } diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs index 0940236394..25e2d50444 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs @@ -11,12 +11,22 @@ const PRIVATE_KIND: &str = "learner_temporal_capture_private_unsupported_command pub(super) struct RuntimeTemporalCapture { context: CaptureContext, + owner_generation: u64, core: Option, lead_in_until: Option, failure: Option, last_now_ns: u64, } impl RuntimeTemporalCapture { + pub(super) fn owner_generation(&self) -> u64 { + self.owner_generation + } + pub(super) fn invalidate_owner(&mut self) { + self.lead_in_until = None; + if let Some(core) = self.core.as_mut() { + core.invalidate("temporal_owner_generation_changed"); + } + } pub(super) fn context(&self) -> &CaptureContext { &self.context } @@ -77,8 +87,23 @@ impl RuntimeTemporalCapture { epoch: &str, packet: &CanonicalPacket, now: u64, + retired: &mut DeferredReclamation, ) -> Result<(), String> { self.check_time(now)?; + let admission = self + .core + .as_mut() + .ok_or("temporal_initial_barrier_unconfirmed")? + .poll_async(now, retired)?; + let now = admission.map_or(now, |(_, at)| at); + self.last_now_ns = now; + if admission.is_some_and(|(start, _)| start) { + self.lead_in_until = Some( + now.checked_add(TRAIN_LEAD_IN_NS) + .ok_or("temporal_time_overflow")?, + ); + return self.core.as_mut().unwrap().prime_fence(packet, now); + } self.arm(store, epoch, now)?; if self.lead_in_until.is_some() { return self @@ -93,12 +118,11 @@ impl RuntimeTemporalCapture { .ok_or("temporal_initial_barrier_unconfirmed")?; match core.stage() { TemporalCaptureStage::Observing => core.ingest(packet, now, store), - TemporalCaptureStage::Failed | TemporalCaptureStage::Blocked => { - Err("temporal_capture_unavailable".into()) - } - TemporalCaptureStage::Ready | TemporalCaptureStage::Paused => { - core.prime_fence(packet, now) - } + TemporalCaptureStage::Failed => Err("temporal_capture_unavailable".into()), + TemporalCaptureStage::Blocked => core.prime_fence(packet, now), + TemporalCaptureStage::Ready + | TemporalCaptureStage::Paused + | TemporalCaptureStage::Preparing => core.prime_fence(packet, now), _ => Ok(()), // No automatic start, confirmation or new phase from packet arrival. } } @@ -140,7 +164,7 @@ impl RoomLearningRuntime { &self, ) -> Result<&super::super::recovery_store::RecoveryStore, String> { self.recovery_store - .as_ref() + .as_deref() .ok_or("temporal_recovery_unavailable".into()) } /// Deliberately PRIVATE: current public configure always constructs Legacy. @@ -152,12 +176,18 @@ impl RoomLearningRuntime { run: String, now: u64, ) -> Result<(), String> { + let store = self + .recovery_store + .clone() + .ok_or("temporal_recovery_unavailable")?; self.configure_temporal_capture_with( expected, context, run, now, - TemporalTrainCapture::prepare_retained, + move |schedule, run, fit, now, _| { + TemporalTrainCapture::prepare_async(schedule, run, fit, now, store) + }, ) } fn configure_temporal_capture_with( @@ -201,21 +231,37 @@ impl RoomLearningRuntime { .generation .checked_add(1) .ok_or("learner generation exhausted")?; - let result = prepare(schedule, run, fit, now, store); - let (core, failure) = match result { - Ok((core, barrier_error)) => (Some(core), barrier_error), - Err(error) => (None, Some(error)), - }; state.capture = Some(CaptureWorkflow::Temporal(Box::new( RuntimeTemporalCapture { - context, - core, + context: context.clone(), + owner_generation: next_generation, + core: None, lead_in_until: None, - failure: failure.clone(), + failure: None, last_now_ns: now, }, ))); state.generation = next_generation; + drop(state); + // Initial crypto/filesystem execution never holds the receiver lock. + let result = prepare(schedule, run, fit, now, store); + let (core, failure) = match result { + Ok((core, error)) => (Some(core), error), + Err(error) => (None, Some(error)), + }; + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if state.generation != next_generation || state.capture_context() != Some(&context) { + // A discarded asynchronous constructor may already have a queued + // initial transaction. Drop/drain it only outside the ingress lock. + drop(state); + drop(core); + return Err("learner_state_changed".into()); + } + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { + return Err(PRIVATE_KIND.into()); + }; + capture.core = core; + capture.failure = failure.clone(); if let Some(error) = failure { state.input_error = Some(bounded(error.clone())); return Err(error); @@ -224,13 +270,21 @@ impl RoomLearningRuntime { } fn start_temporal_phase(&self, expected: &MutationGuard, now: u64) -> Result<(), String> { let mut state = self.command_state(expected)?; + let owner_generation = state.generation; let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { return Err(PRIVATE_KIND.into()); }; + if capture.owner_generation != owner_generation { + capture + .core + .as_mut() + .map(|c| c.invalidate("temporal_owner_generation_changed")); + return Err("temporal_owner_generation_changed".into()); + } capture.check_time(now)?; let core = capture .core - .as_ref() + .as_mut() .ok_or("temporal_initial_barrier_unconfirmed")?; if capture.lead_in_until.is_some() || !matches!( @@ -240,7 +294,10 @@ impl RoomLearningRuntime { { return Err("temporal_phase_not_startable".into()); } - // The leaf checks the original descriptor/floors when this preparation ends. + if core.async_start(now)? { + return Ok(()); + } + // Synchronous reference path retains its original behavior. capture.lead_in_until = Some( now.checked_add(TRAIN_LEAD_IN_NS) .ok_or("temporal_time_overflow")?, @@ -249,14 +306,46 @@ impl RoomLearningRuntime { } fn tick_temporal_capture(&self, expected: &MutationGuard, now: u64) -> Result<(), String> { let store = self.temporal_store()?; + // Must precede MutexGuard for success, rejection and unwinding alike. + let mut retired = self.deferred_reclamation(); let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; if *expected != self.guard_for(&state) { return Err("learner_state_changed".into()); } + let owner_generation = state.generation; let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { return Err(PRIVATE_KIND.into()); }; + if capture.owner_generation != owner_generation { + capture + .core + .as_mut() + .map(|c| c.invalidate("temporal_owner_generation_changed")); + return Err("temporal_owner_generation_changed".into()); + } capture.check_time(now)?; + let admission = capture + .core + .as_mut() + .ok_or("temporal_initial_barrier_unconfirmed")? + .poll_async(now, &mut retired)?; + #[cfg(test)] + if retired.has_ack() + && self + .panic_after_reclamation_ack + .swap(false, Ordering::AcqRel) + { + panic!("synthetic panic after actual ACK admission"); + } + let now = admission.map_or(now, |(_, at)| at); + capture.last_now_ns = now; + if admission.is_some_and(|(start, _)| start) { + capture.lead_in_until = Some( + now.checked_add(TRAIN_LEAD_IN_NS) + .ok_or("temporal_time_overflow")?, + ); + return Ok(()); + } capture.arm(store, &self.runtime_epoch, now)?; if capture.lead_in_until.is_some() { return Ok(()); @@ -293,9 +382,17 @@ impl RoomLearningRuntime { ) -> Result<(), String> { let store = self.temporal_store()?; let mut state = self.command_state(expected)?; + let owner_generation = state.generation; let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { return Err(PRIVATE_KIND.into()); }; + if capture.owner_generation != owner_generation { + capture + .core + .as_mut() + .map(|c| c.invalidate("temporal_owner_generation_changed")); + return Err("temporal_owner_generation_changed".into()); + } capture.check_time(now)?; if capture.lead_in_until.is_some() { return Err("temporal_phase_incomplete".into()); @@ -309,9 +406,17 @@ impl RoomLearningRuntime { fn pause_temporal_capture(&self, expected: &MutationGuard, now: u64) -> Result<(), String> { let store = self.temporal_store()?; let mut state = self.command_state(expected)?; + let owner_generation = state.generation; let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { return Err(PRIVATE_KIND.into()); }; + if capture.owner_generation != owner_generation { + capture + .core + .as_mut() + .map(|c| c.invalidate("temporal_owner_generation_changed")); + return Err("temporal_owner_generation_changed".into()); + } capture.check_time(now)?; if capture.lead_in_until.is_some() { return Err("temporal_preparation_has_no_observation_to_pause".into()); @@ -351,11 +456,36 @@ impl RoomLearningRuntime { .generation .checked_add(1) .ok_or("learner generation exhausted")?; - let core = + state.capture = Some(CaptureWorkflow::Temporal(Box::new( + RuntimeTemporalCapture { + context: context.clone(), + owner_generation: next_generation, + core: None, + lead_in_until: None, + failure: None, + last_now_ns: now, + }, + ))); + state.generation = next_generation; + drop(state); + let mut core = TemporalTrainCapture::cold_restore(store, context, now, &self.runtime_epoch, now)?; + core.enable_async_restore( + self.recovery_store + .clone() + .ok_or("temporal_recovery_unavailable")?, + now, + )?; + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if state.generation != next_generation { + drop(state); + drop(core); + return Err("learner_state_changed".into()); + } state.capture = Some(CaptureWorkflow::Temporal(Box::new( RuntimeTemporalCapture { context: context.clone(), + owner_generation: next_generation, core: Some(core), lead_in_until: None, failure: None, @@ -368,9 +498,14 @@ impl RoomLearningRuntime { fn repair_temporal_capture(&self, expected: &MutationGuard, now: u64) -> Result<(), String> { let store = self.temporal_store()?; let mut state = self.command_state(expected)?; + let owner_generation = state.generation; let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { return Err(PRIVATE_KIND.into()); }; + if capture.owner_generation != owner_generation { + capture.invalidate_owner(); + return Err("temporal_owner_generation_changed".into()); + } capture .core .as_mut() @@ -380,11 +515,37 @@ impl RoomLearningRuntime { state.input_error = None; Ok(()) } + fn cancel_temporal_durability(&self, expected: &MutationGuard) -> Result<(), String> { + let writer = { + let mut state = self.command_state(expected)?; + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { + return Err(PRIVATE_KIND.into()); + }; + capture.lead_in_until = None; + let core = capture + .core + .as_mut() + .ok_or("temporal_initial_barrier_unconfirmed")?; + let writer = core.writer(); + core.invalidate("temporal_owner_write_cancelled"); + if let Some(writer) = &writer { + writer.cancel(); + } + writer + }; + if let Some(writer) = writer { + writer.drain()?; + } + Ok(()) + } fn temporal_capture_snapshot(&self, now: u64) -> Result { let state = self.inner.lock().map_err(|_| "learner state unavailable")?; let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_ref() else { return Err(PRIVATE_KIND.into()); }; + if capture.owner_generation != state.generation { + return Err("temporal_owner_generation_changed".into()); + } Ok( json!({"schema":"ruview.room-learner.private-temporal-capture.v1","mutation_guard":self.guard_for(&state),"capture":capture.snapshot(now),"fit_authority":false,"public_activation":false}), ) @@ -394,3 +555,26 @@ impl RoomLearningRuntime { #[cfg(test)] #[path = "lifecycle_temporal_tests.rs"] mod tests; + +impl Drop for RoomLearningRuntime { + fn drop(&mut self) { + // The last Runtime owner has exclusive access; get_mut takes no ingress + // lock. Join before owner/store release so an old writer cannot outlive + // Runtime shutdown and recreate an erased profile later. + let state = match self.inner.get_mut() { + Ok(state) => state, + Err(poisoned) => poisoned.into_inner(), + }; + let writer = match state.capture.as_mut() { + Some(CaptureWorkflow::Temporal(capture)) => capture.core.as_mut().and_then(|core| { + let writer = core.writer(); + core.invalidate("temporal_runtime_dropped"); + writer + }), + _ => None, + }; + if let Some(writer) = writer { + let _ = writer.drain(); + } + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs index 1dd8a105ea..98848b5279 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs @@ -52,11 +52,10 @@ fn runtime(root: &Path, now: u64) -> RoomLearningRuntime { } fn configured(root: &Path) -> RoomLearningRuntime { let rt = runtime(root, START); - rt.configure_temporal_capture( - &rt.mutation_guard().unwrap(), - context(), - "synthetic-runtime-train".into(), - START, + // Original synchronous C1 reducer remains a reference/golden adapter. + rt.configure_temporal_capture_with( + &rt.mutation_guard().unwrap(), context(), "synthetic-runtime-train".into(), START, + TemporalTrainCapture::prepare_retained, ) .unwrap(); rt @@ -525,3 +524,605 @@ async fn private_temporal_setup_cannot_replace_an_actual_selected_legacy_candida history_before ); } + +#[test] +fn temporal_storage_barrier_must_not_block_real_runtime_native_ingress() { + use std::{sync::{Arc, mpsc}, time::Duration}; + let dir = tempfile::tempdir().unwrap(); + let rt = Arc::new(runtime(dir.path(), START)); + let guard = rt.mutation_guard().unwrap(); + let (entered_tx, entered_rx) = mpsc::sync_channel(1); + let (release_tx, release_rx) = mpsc::sync_channel(1); + let writer_rt = rt.clone(); + let writer = std::thread::spawn(move || { + writer_rt.configure_temporal_capture_with(&guard, context(), + "synthetic-c2-stalled-storage".into(), START, + |schedule, run, fit, now, store| { + entered_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + TemporalTrainCapture::prepare_retained(schedule, run, fit, now, store) + }) + }); + entered_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + let (input_tx, input_rx) = mpsc::sync_channel(1); + let input_rt = rt.clone(); + let input = std::thread::spawn(move || { + input_tx.send(input_rt.ingest_native(&packet(1, 1, START), START)).unwrap(); + }); + let progressed_while_storage_blocked = input_rx.recv_timeout(Duration::from_millis(100)); + release_tx.send(()).unwrap(); + writer.join().unwrap().unwrap(); + input.join().unwrap(); + assert!(progressed_while_storage_blocked.is_ok(), + "real native ingress waited for a storage barrier held by Runtime.inner"); +} + +fn settle_async_boundary(rt: &RoomLearningRuntime, after: u64) -> u64 { + let started=std::time::Instant::now(); + loop { + let at=after+started.elapsed().as_nanos() as u64; + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),at).unwrap(); + if snapshot(rt,at)["durability_pending"]==false { + let admitted={let state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()};at.max(c.last_now_ns)}; + if started.elapsed()>std::time::Duration::from_secs(1){println!("SYNTHETIC_DURABLE_ACK_WAIT_NS={} ROWS={} SEALED={}",started.elapsed().as_nanos(),snapshot(rt,admitted)["durable_windows"],snapshot(rt,admitted)["sealed_train_episodes"]);} + return admitted; + } + assert!(started.elapsed()(RoomLearningRuntime,u64){ + let rt=runtime(root,START); + rt.configure_temporal_capture(&rt.mutation_guard().unwrap(),context(),"synthetic-c2-runtime".into(),START).unwrap(); + let at=settle_async_boundary(&rt,START); + assert_eq!(snapshot(&rt,at)["stage"],"ready"); + (rt,at) +} + +#[test] +fn c2_start_requires_exact_current_prefix_ack_before_any_lead_or_rows(){ + use std::{sync::mpsc,time::Duration}; + let dir=tempfile::tempdir().unwrap();let(rt,at)=configured_async(dir.path()); + let writer={let state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(capture))=&state.capture else{panic!()};capture.core.as_ref().unwrap().writer().unwrap()}; + let(entered_tx,entered_rx)=mpsc::sync_channel(1);let(release_tx,release_rx)=mpsc::sync_channel(1); + writer.hold_next(entered_tx,release_rx); + rt.start_temporal_phase(&rt.mutation_guard().unwrap(),at).unwrap(); + entered_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + let mut now=at+1_000_000; + for source in 1..=3 {rt.ingest_native(&packet(source,1,now),now).unwrap();} + let pending=snapshot(&rt,now); + assert_eq!(pending["stage"],"preparing");assert_eq!(pending["retained_rows"],0); + assert_eq!(pending["durable_windows"],0);assert_eq!(pending["lead_in_remaining_ms"],0); + assert!(rt.temporal_confirmation_challenge(&rt.mutation_guard().unwrap()).is_err()); + release_tx.send(()).unwrap(); + // Exact recertification chooses a fresh start coordinate only at admission. + let started=std::time::Instant::now(); + loop { + now=now.max(at+started.elapsed().as_nanos() as u64); + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap(); + if snapshot(&rt,now)["lead_in_remaining_ms"].as_u64().unwrap()>0 {break;} + assert!(started.elapsed()now); + drop(state); + let durable=rt.temporal_store().unwrap().load_temporal(now).unwrap().unwrap(); + let bytes=durable.prefix.unwrap();let v:Value=serde_json::from_slice(bytes.bytes()).unwrap(); + assert_eq!(v["payload"]["rows"].as_array().unwrap().len(),0); + assert!(v["payload"]["mapping"].is_null()); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); +} + +#[test] +fn c2_real_native_ingress_and_source_floors_progress_while_segment_writer_is_blocked(){ + use std::{sync::{Arc,mpsc},time::Duration}; + let dir=tempfile::tempdir().unwrap();let(rt,at)=configured_async(dir.path());let rt=Arc::new(rt); + rt.start_temporal_phase(&rt.mutation_guard().unwrap(),at).unwrap(); + let started=std::time::Instant::now();let end; + loop { + let now=at+started.elapsed().as_nanos() as u64; + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap(); + let state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()}; + if let Some(value)=c.lead_in_until {end=value;break;} + drop(state);assert!(started.elapsed()u64{ + rt.start_temporal_phase(&rt.mutation_guard().unwrap(),at).unwrap(); + let started=std::time::Instant::now(); + loop{ + let now=at+started.elapsed().as_nanos() as u64; + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap(); + let state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()}; + if let Some(end)=c.lead_in_until{return end;} + drop(state);assert!(started.elapsed()break 'lost,Err(e)=>panic!("unexpected {e}"),Ok(())=>{}}} + assert!(began.elapsed()u64{ + let wait=std::time::Instant::now();let mut now=after; + loop { + *seq+=1; + for source in 1..=3 {rt.ingest_native(&packet(source,*seq,now),now).unwrap();} + if snapshot(rt,now)["durability_pending"]==false {return now+PERIOD;} + assert!(wait.elapsed(){},Err(error)if error=="temporal_storage_busy_pause"=>{busy=now;break 'observe;},Err(error)=>panic!("unexpected {error}"), + } + } + assert!(now0,"busy restore did not retire the actual unconfirmed raw tail"); + assert_eq!(reclamation.blocked.load(Ordering::Acquire),0,"busy raw-tail destructor ran under Runtime.inner"); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); +} + +#[test] +fn c2_three_sources_keep_declared_real_cadence_during_250ms_worker_stall(){ + use std::{sync::mpsc,time::{Duration,Instant}}; + let dir=tempfile::tempdir().unwrap();let rt=runtime(dir.path(),START);let mut c=context(); + let template=c.sources[0].clone(); + c.sources=(1..=3).map(|id|{let mut source=template.clone();source.source_id=id;source.device_id=format!("esp32-node-{id}");source.link_id=format!("native-{id}");source}).collect(); + rt.configure_temporal_capture(&rt.mutation_guard().unwrap(),c,"synthetic-c2-three".into(),START).unwrap(); + let at=settle_async_boundary(&rt,START);let end=async_lead_end(&rt,at); + let writer={let state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()};c.core.as_ref().unwrap().writer().unwrap()}; + let(entered_tx,entered_rx)=mpsc::sync_channel(1);let(release_tx,release_rx)=mpsc::sync_channel(1);writer.hold_next(entered_tx,release_rx); + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),end).unwrap();entered_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + let clock=Instant::now();let mut max_ingest=Duration::ZERO;let mut last=end; + let result=std::panic::catch_unwind(std::panic::AssertUnwindSafe(||{ + let mut previous_batch=None; + for index in 0..9u32 { + if let Some(previous)=previous_batch { + let elapsed=Instant::now().duration_since(previous); + if let Some(wait)=Duration::from_nanos(PERIOD).checked_sub(elapsed){std::thread::sleep(wait);} + } + let arrival=Instant::now();previous_batch=Some(arrival); + // Three sources arrive together; record actual batch arrival once. + // Processing uses its own current clock. Never catch up missed + // deadlines by generating an artificially fast next batch. + let received=end+clock.elapsed().as_nanos() as u64; + for source in 1..=3 { + let packet=packet(source,index+1,received); + let now=end+clock.elapsed().as_nanos() as u64;let started=Instant::now(); + if let Err(error)=rt.ingest_native(&packet,now){panic!("paced index={index} source={source} max_ingest_ns={} error={error}",max_ingest.as_nanos());} + max_ingest=max_ingest.max(started.elapsed());last=now; + } + } + assert_eq!(snapshot(&rt,last)["stage"],"observing"); + assert_eq!(snapshot(&rt,last)["durable_windows"],0); + })); + release_tx.send(()).unwrap();rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); + if let Err(error)=result{std::panic::resume_unwind(error);} + println!("SYNTHETIC_3_SOURCE_PACED_WALL_NS={} MAX_INGEST_NS={}",clock.elapsed().as_nanos(),max_ingest.as_nanos()); + assert!(max_ingest{ + now=async_lead_end(&rt,now);fresh_starts+=1; + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap(); + } + "needs_confirmation"=>break, + "failed"=>panic!("phase{phase} failed: {state}"), + _=>{} + } + let pending=snapshot(&rt,now)["durability_pending"]==true; + seq+=1; + for source in 1..=3 { + match rt.ingest_native(&packet(source,seq,now),now){ + Ok(())=>{},Err(error) if error=="temporal_storage_busy_pause"=>break, + Err(error)=>panic!("phase{phase} input failed: {error}"), + } + } + now+=PERIOD; + if pending {std::thread::sleep(std::time::Duration::from_nanos(PERIOD));} + assert!(fresh_starts<120,"bounded interruption budget exhausted"); + } + let guard=rt.mutation_guard().unwrap();let challenge=rt.temporal_confirmation_challenge(&guard).unwrap(); + rt.confirm_temporal_phase(&guard,&challenge,now).unwrap(); + // Submission is not durable phase confirmation. + assert_eq!(snapshot(&rt,now)["sealed_train_episodes"],phase); + assert_eq!(snapshot(&rt,now)["durability_pending"],true); + now=settle_async_boundary(&rt,now); + assert_eq!(snapshot(&rt,now)["sealed_train_episodes"],phase+1); + println!("SYNTHETIC_ASYNC_PHASE_SEALED={} ROWS={} FRESH_STARTS={fresh_starts}",phase+1,snapshot(&rt,now)["durable_windows"]); + } + let complete=snapshot(&rt,now); + assert_eq!(complete["stage"],"complete");assert_eq!(complete["retained_rows"],64);assert_eq!(complete["durable_windows"],64); + assert_eq!(complete["fit_admission_available"],false); + assert_eq!(complete["descriptor"]["id"],original["id"]);assert_eq!(complete["descriptor"]["expires_at_ms"],original["expires_at_ms"]); + let inner=rt.inner.lock().unwrap();assert!(inner.model.is_none());assert!(!inner.selection.has_history());drop(inner); + println!("SYNTHETIC_ASYNC_TRAIN64_SEALS7_FRESH_STARTS={fresh_starts}"); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); +} + +#[derive(Default)] +struct ReclaimCounts { + unlocked:std::sync::atomic::AtomicUsize, + blocked:std::sync::atomic::AtomicUsize, + final_prefix_owner:std::sync::atomic::AtomicUsize, + raw_tail:std::sync::atomic::AtomicUsize, +} +fn install_reclaim_probe(rt:&std::sync::Arc)->std::sync::Arc{ + let counts=std::sync::Arc::new(ReclaimCounts::default());let observed=counts.clone();let weak=std::sync::Arc::downgrade(rt); + *rt.reclamation_probe.lock().unwrap()=Some(std::sync::Arc::new(move|stats|{ + if !stats.has_ack{return;} + // Never panic from a destructor, especially while testing unwinding. + let unlocked=weak.upgrade().is_some_and(|rt|match rt.inner.try_lock(){Ok(_)=>true,Err(std::sync::TryLockError::Poisoned(_))=>true,Err(std::sync::TryLockError::WouldBlock)=>false}); + if unlocked{observed.unlocked.fetch_add(1,Ordering::AcqRel);}else{observed.blocked.fetch_add(1,Ordering::AcqRel);} + if stats.old_prefix_bytes>0 && stats.old_prefix_last_owner{observed.final_prefix_owner.fetch_add(1,Ordering::AcqRel);} + if stats.raw_tail_rows>0{observed.raw_tail.fetch_add(1,Ordering::AcqRel);} + }));counts +} +#[test] +fn c2_actual_ack_retirement_unlocks_before_normal_rejection_and_unwind_destructors(){ + use std::{sync::{Arc,mpsc},time::{Duration,Instant}}; + for via_tick in [false,true]{for outcome in ["normal","rejected","unwind"]{ + let dir=tempfile::tempdir().unwrap();let(rt,at)=configured_async(dir.path());let rt=Arc::new(rt);let counts=install_reclaim_probe(&rt); + let writer={let state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()};c.core.as_ref().unwrap().writer().unwrap()}; + let(entered_tx,entered_rx)=mpsc::sync_channel(1);let(release_tx,release_rx)=mpsc::sync_channel(1);writer.hold_next(entered_tx,release_rx); + if outcome=="rejected"{writer.corrupt_next_ack_id();} + rt.start_temporal_phase(&rt.mutation_guard().unwrap(),at).unwrap();entered_rx.recv_timeout(Duration::from_secs(2)).unwrap();release_tx.send(()).unwrap(); + let wait=Instant::now();while !writer.ack_is_queued(){assert!(wait.elapsed()assert!(result.unwrap().is_ok()),"rejected"=>assert!(result.unwrap().is_err()),_=>assert!(result.is_err())} + assert!(counts.unlocked.load(Ordering::Acquire)+counts.blocked.load(Ordering::Acquire)>0,"destructor was never observed"); + assert_eq!(counts.blocked.load(Ordering::Acquire),0,"actual payload destructor ran while Runtime.inner was held"); + assert!(counts.unlocked.load(Ordering::Acquire)>0); + if outcome!="rejected"{assert!(counts.final_prefix_owner.load(Ordering::Acquire)>0,"did not retire last old-prefix Arc owner");} + writer.drain().unwrap(); + }} +} + +#[test] +#[ignore = "opt-in host/profile throughput acceptance; normal C2 tests retain growing-prefix custody, busy-pause and full-seven correctness"] +fn c2_real_growing_prefix_three_source_paced_seventeenth_row_throughput_acceptance(){ + use std::time::{Duration,Instant}; + let dir=tempfile::tempdir().unwrap();let(rt,mut now)=configured_async(dir.path());let mut seq=0; + // Prelude is synthetic receiver-clock collection, not a performance claim. + // Populate the SAME real Runtime/assembler/store with all16 Empty windows. + loop { + match snapshot(&rt,now)["stage"].as_str().unwrap(){ + "ready"|"paused"=>{now=async_lead_end(&rt,now);rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap();}, + "needs_confirmation"=>break, + "failed"=>panic!("growing-prefix prelude failed"),_=>{} + } + let pending=snapshot(&rt,now)["durability_pending"]==true;seq+=1; + for source in 1..=3{match rt.ingest_native(&packet(source,seq,now),now){Ok(())=>{},Err(e)if e=="temporal_storage_busy_pause"=>break,Err(e)=>panic!("prelude: {e}")}} + now+=PERIOD;if pending{std::thread::sleep(Duration::from_nanos(PERIOD));} + } + let guard=rt.mutation_guard().unwrap();let challenge=rt.temporal_confirmation_challenge(&guard).unwrap(); + rt.confirm_temporal_phase(&guard,&challenge,now).unwrap();now=settle_async_boundary(&rt,now); + assert_eq!(snapshot(&rt,now)["durable_windows"],16); + let old_prefix=rt.temporal_store().unwrap().load_temporal(now).unwrap().unwrap().prefix.unwrap().bytes().len(); + let base=async_lead_end(&rt,now);rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),base).unwrap(); + // Measure honest batch arrival and current processing clocks at real33Hz; + // no accelerated ticks, held-writer hook, replay queue, or deadline catch-up. + let clock=Instant::now();let mut previous_batch=None;let mut packet_wall=Vec::::new(); + let mut row_ready_at=None;let mut acked_at=None;let mut measured_now=base; + let mut batch_arrivals=Vec::::new();let mut sleep_samples=Vec::<(u64,u64)>::new(); + let mut source_arrivals:[Vec;3]=std::array::from_fn(|_|Vec::new()); + let mut snapshot_costs=Vec::::new();let mut failed_attempt:Option<(u8,u32,String)>=None; + let result:Result<(),String>=(||{ + loop { + if clock.elapsed()>Duration::from_secs(20){return Err("real growing-prefix ACK did not arrive within measurement window".into());} + if let Some(previous)=previous_batch{if let Some(wait)=Duration::from_nanos(PERIOD).checked_sub(Instant::now().duration_since(previous)){let slept=Instant::now();std::thread::sleep(wait);sleep_samples.push((wait.as_nanos()as u64,slept.elapsed().as_nanos()as u64));}} + previous_batch=Some(Instant::now());let received=base+clock.elapsed().as_nanos()as u64;seq+=1; + batch_arrivals.push(received-base); + for source in 1..=3{ + let before=Instant::now();let p=packet(source,seq,received);measured_now=base+clock.elapsed().as_nanos()as u64; + let outcome=rt.ingest_native(&p,measured_now); + let packet_elapsed=before.elapsed().as_nanos()as u64; + source_arrivals[usize::from(source)-1].push(p.native_frame.timestamp_ns-base); + packet_wall.push(packet_elapsed); + if let Err(error)=outcome{failed_attempt=Some((source,seq,error.clone()));return Err(error);} + } + let snapshot_started=Instant::now();let status=snapshot(&rt,measured_now);snapshot_costs.push(snapshot_started.elapsed().as_nanos()as u64); + if status["retained_rows"].as_u64().unwrap()>=17 && row_ready_at.is_none(){row_ready_at=Some(clock.elapsed());} + if status["durable_windows"].as_u64().unwrap()>=17 && acked_at.is_none(){acked_at=Some(clock.elapsed());} + if let Some(acked)=acked_at{if clock.elapsed()-acked>=Duration::from_millis(250){break;}} + } + if packet_wall.is_empty(){return Err("no measured packets".into());} + packet_wall.sort_unstable();let max=*packet_wall.last().unwrap();let p99=packet_wall[(packet_wall.len()-1)*99/100]; + if max>5_000_000 || p99>2_000_000{return Err(format!("packet wall budget exceeded: max_ns={max} p99_ns={p99}"));} + Ok(()) + })(); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); + let new_prefix=rt.temporal_store().unwrap().load_temporal(measured_now).unwrap().unwrap().prefix.unwrap().bytes().len(); + packet_wall.sort_unstable();let max=packet_wall.last().copied().unwrap_or(0);let p99=packet_wall.get(packet_wall.len().saturating_sub(1)*99/100).copied().unwrap_or(0); + println!("SYNTHETIC_REAL_3_SOURCE_GROWING_PREFIX status={} old_bytes={} new_bytes={} packets={} max_packet_wall_ns={} p99_packet_wall_ns={} row_ready_at_ns={:?} acked_at_ns={:?} row_ready_to_ack_ns={:?} debug_assertions={} error={:?}",if result.is_ok(){"PASS"}else{"UNQUALIFIED"},old_prefix,new_prefix,packet_wall.len(),max,p99,row_ready_at.map(|d|d.as_nanos()),acked_at.map(|d|d.as_nanos()),row_ready_at.zip(acked_at).map(|(q,a)|a.saturating_sub(q).as_nanos()),cfg!(debug_assertions),result.as_ref().err()); + let summaries=source_arrivals.iter().enumerate().map(|(index,times)|{ + let span=times.last().copied().unwrap_or(0).saturating_sub(times.first().copied().unwrap_or(0)); + let intervals=times.len().saturating_sub(1) as u128; + let gaps=times.windows(2).map(|w|w[1]-w[0]).collect::>(); + json!({"source":index+1,"attempts":times.len(),"first_relative_ns":times.first(),"last_relative_ns":times.last(),"span_ns":span, + "actual_rate_millihz":if span>0{Some(intervals*1_000_000_000_000u128/u128::from(span))}else{None}, + "min_gap_ns":gaps.iter().min(),"max_gap_ns":gaps.iter().max()}) + }).collect::>(); + let contract=ManagedScheduleContract::build(context(),LeadInPolicy::Train5HeldOut15EngineeringV1).unwrap();let policy=contract.native_window_policy();let slot=&contract.episodes()[1].slots[0]; + println!("SYNTHETIC_CADENCE_DIAGNOSTIC {}",json!({"batch_count":batch_arrivals.len(),"batch_first_relative_ns":batch_arrivals.first(),"batch_last_relative_ns":batch_arrivals.last(), + "sources":summaries,"failed_attempt":failed_attempt,"sleep_count":sleep_samples.len(),"requested_sleep_min_ns":sleep_samples.iter().map(|x|x.0).min(),"requested_sleep_max_ns":sleep_samples.iter().map(|x|x.0).max(), + "actual_sleep_min_ns":sleep_samples.iter().map(|x|x.1).min(),"actual_sleep_max_ns":sleep_samples.iter().map(|x|x.1).max(), + "sleep_overshoot_max_ns":sleep_samples.iter().map(|x|x.1.saturating_sub(x.0)).max(),"snapshot_max_ns":snapshot_costs.iter().max(), + "slot_target_ms":slot.target_offset_ms,"slot_deadline_ms":slot.deadline_offset_ms,"native_target_span_ms":policy.target_span_ms(), + "source_min_rows":policy.source_policies()[0].min_rows,"source_max_gap_ns":policy.source_policies()[0].max_gap_ns, + "rate_band_millihz":policy.source_policies()[0].receiver_cadence.rate_bounds_millihz()})); + assert!(result.is_ok(),"growing-prefix performance is UNQUALIFIED; no public activation: {:?}",result.err()); +} + +#[path = "c2_precision_pacer_test_support.rs"] +mod c2_precision_pacer_test_support; + +#[test] +#[ignore = "opt-in host/profile throughput acceptance; normal C2 tests retain growing-prefix custody, busy-pause and full-seven correctness"] +fn c2_real_growing_prefix_three_source_absolute_pacer_with_actual_input_precondition(){ + use std::time::{Duration,Instant}; + const SPIN_GUARD_NS:u64=3_000_000; + let precondition=c2_precision_pacer_test_support::input_study(100,SPIN_GUARD_NS); + println!("SYNTHETIC_INPUT_PRECONDITION {:?}",precondition); + assert!(precondition.qualified,"INPUT_UNQUALIFIED before backend exercise; no retry or public activation"); + let dir=tempfile::tempdir().unwrap();let(rt,mut now)=configured_async(dir.path());let mut seq=0; + // Prelude is synthetic receiver-clock collection, not a performance claim. + // Populate the SAME real Runtime/assembler/store with all16 Empty windows. + loop { + match snapshot(&rt,now)["stage"].as_str().unwrap(){ + "ready"|"paused"=>{now=async_lead_end(&rt,now);rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap();}, + "needs_confirmation"=>break, + "failed"=>panic!("growing-prefix prelude failed"),_=>{} + } + let pending=snapshot(&rt,now)["durability_pending"]==true;seq+=1; + for source in 1..=3{match rt.ingest_native(&packet(source,seq,now),now){Ok(())=>{},Err(e)if e=="temporal_storage_busy_pause"=>break,Err(e)=>panic!("prelude: {e}")}} + now+=PERIOD;if pending{std::thread::sleep(Duration::from_nanos(PERIOD));} + } + let guard=rt.mutation_guard().unwrap();let challenge=rt.temporal_confirmation_challenge(&guard).unwrap(); + rt.confirm_temporal_phase(&guard,&challenge,now).unwrap();now=settle_async_boundary(&rt,now); + assert_eq!(snapshot(&rt,now)["durable_windows"],16); + let old_prefix=rt.temporal_store().unwrap().load_temporal(now).unwrap().unwrap().prefix.unwrap().bytes().len(); + let base=async_lead_end(&rt,now);rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),base).unwrap(); + // Measure honest batch arrival and current processing clocks at real33Hz; + // no accelerated ticks, held-writer hook, replay queue, or deadline catch-up. + let clock=Instant::now();let mut absolute_index=0usize;let mut packet_wall=Vec::::new(); + let mut row_ready_at=None;let mut acked_at=None;let mut measured_now=base; + let mut batch_arrivals=Vec::::new(); + let mut source_arrivals:[Vec;3]=std::array::from_fn(|_|Vec::new()); + let mut snapshot_costs=Vec::::new();let mut failed_attempt:Option<(u8,u32,String)>=None; + let result:Result<(),String>=(||{ + loop { + if clock.elapsed()>Duration::from_secs(20){return Err("real growing-prefix ACK did not arrive within measurement window".into());} + let actual_relative=c2_precision_pacer_test_support::next_actual_receipt(clock,absolute_index,SPIN_GUARD_NS)?; + absolute_index+=1; + // ACTUAL wakeup, never nominal deadline. One batch only; an entire + // missed period returns UNQUALIFIED instead of catch-up emission. + let received=base+actual_relative;seq+=1; + batch_arrivals.push(received-base); + for source in 1..=3{ + let before=Instant::now();let p=packet(source,seq,received);measured_now=base+clock.elapsed().as_nanos()as u64; + let outcome=rt.ingest_native(&p,measured_now); + let packet_elapsed=before.elapsed().as_nanos()as u64; + source_arrivals[usize::from(source)-1].push(p.native_frame.timestamp_ns-base); + packet_wall.push(packet_elapsed); + if let Err(error)=outcome{failed_attempt=Some((source,seq,error.clone()));return Err(error);} + } + let snapshot_started=Instant::now();let status=snapshot(&rt,measured_now);snapshot_costs.push(snapshot_started.elapsed().as_nanos()as u64); + if status["retained_rows"].as_u64().unwrap()>=17 && row_ready_at.is_none(){row_ready_at=Some(clock.elapsed());} + if status["durable_windows"].as_u64().unwrap()>=17 && acked_at.is_none(){acked_at=Some(clock.elapsed());} + if let Some(acked)=acked_at{if clock.elapsed()-acked>=Duration::from_millis(250){break;}} + } + if !source_arrivals.iter().all(|times|c2_precision_pacer_test_support::actual_stream_qualifies(times)){return Err("INPUT_UNQUALIFIED during real backend measurement; planned deadlines are not evidence".into());} + if packet_wall.is_empty(){return Err("no measured packets".into());} + packet_wall.sort_unstable();let max=*packet_wall.last().unwrap();let p99=packet_wall[(packet_wall.len()-1)*99/100]; + if max>5_000_000 || p99>2_000_000{return Err(format!("packet wall budget exceeded: max_ns={max} p99_ns={p99}"));} + Ok(()) + })(); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); + let new_prefix=rt.temporal_store().unwrap().load_temporal(measured_now).unwrap().unwrap().prefix.unwrap().bytes().len(); + packet_wall.sort_unstable();let max=packet_wall.last().copied().unwrap_or(0);let p99=packet_wall.get(packet_wall.len().saturating_sub(1)*99/100).copied().unwrap_or(0); + println!("SYNTHETIC_REAL_3_SOURCE_GROWING_PREFIX status={} old_bytes={} new_bytes={} packets={} max_packet_wall_ns={} p99_packet_wall_ns={} row_ready_at_ns={:?} acked_at_ns={:?} row_ready_to_ack_ns={:?} debug_assertions={} error={:?}",if result.is_ok(){"PASS"}else{"UNQUALIFIED"},old_prefix,new_prefix,packet_wall.len(),max,p99,row_ready_at.map(|d|d.as_nanos()),acked_at.map(|d|d.as_nanos()),row_ready_at.zip(acked_at).map(|(q,a)|a.saturating_sub(q).as_nanos()),cfg!(debug_assertions),result.as_ref().err()); + let summaries=source_arrivals.iter().enumerate().map(|(index,times)|{ + let span=times.last().copied().unwrap_or(0).saturating_sub(times.first().copied().unwrap_or(0)); + let intervals=times.len().saturating_sub(1) as u128; + let gaps=times.windows(2).map(|w|w[1]-w[0]).collect::>(); + json!({"source":index+1,"attempts":times.len(),"first_relative_ns":times.first(),"last_relative_ns":times.last(),"span_ns":span, + "actual_rate_millihz":if span>0{Some(intervals*1_000_000_000_000u128/u128::from(span))}else{None}, + "min_gap_ns":gaps.iter().min(),"max_gap_ns":gaps.iter().max()}) + }).collect::>(); + let contract=ManagedScheduleContract::build(context(),LeadInPolicy::Train5HeldOut15EngineeringV1).unwrap();let policy=contract.native_window_policy();let slot=&contract.episodes()[1].slots[0]; + println!("SYNTHETIC_CADENCE_DIAGNOSTIC {}",json!({"batch_count":batch_arrivals.len(),"batch_first_relative_ns":batch_arrivals.first(),"batch_last_relative_ns":batch_arrivals.last(), + "sources":summaries,"failed_attempt":failed_attempt,"pacer_strategy":"absolute_actual_deadline_no_whole_period_catchup","spin_guard_ns":SPIN_GUARD_NS,"snapshot_max_ns":snapshot_costs.iter().max(), + "slot_target_ms":slot.target_offset_ms,"slot_deadline_ms":slot.deadline_offset_ms,"native_target_span_ms":policy.target_span_ms(), + "source_min_rows":policy.source_policies()[0].min_rows,"source_max_gap_ns":policy.source_policies()[0].max_gap_ns, + "rate_band_millihz":policy.source_policies()[0].receiver_cadence.rate_bounds_millihz()})); + assert!(result.is_ok(),"growing-prefix performance is UNQUALIFIED; no public activation: {:?}",result.err()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs index e6fd351dba..da1e26a1af 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs @@ -406,6 +406,7 @@ fn validate_parts( #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub(crate) enum TemporalCaptureStage { + Preparing, Ready, Observing, NeedsConfirmation, @@ -417,6 +418,7 @@ pub(crate) enum TemporalCaptureStage { impl TemporalCaptureStage { pub(crate) fn as_str(self) -> &'static str { match self { + Self::Preparing => "preparing", Self::Ready => "ready", Self::Observing => "observing", Self::NeedsConfirmation => "needs_confirmation", @@ -450,6 +452,43 @@ impl TemporalTrainCapture { Ok(capture) } } + pub(crate) fn prepare_async( + schedule: ManagedScheduleContract, run:String, config:FitConfig, now:u64, + store:std::sync::Arc, + ) -> Result<(Self,Option),String> { + let owned_store=store.clone(); + Self::prepare_with(schedule,run,config,now,&store,move |w,store|{ + let writer=std::sync::Arc::new(durability::Writer::new(owned_store,now)?); + w.async_state=Some(durability::AsyncState::new(writer)); + w.persist_initial(store) + }) + } + pub(crate) fn async_start(&mut self,now:u64)->Result{ + self.ensure_live()?; + if self.workflow.async_state.is_none(){return Ok(false);} + self.workflow.request_async_start(now)?;Ok(true) + } + pub(crate) fn poll_async(&mut self,now:u64,retired:&mut DeferredReclamation)->Result,String>{ + if self.failure.is_some(){return Err("temporal_capture_failed".into());} + let start=match self.workflow.poll_async(now,retired){ + Ok(start)=>start, + Err(error)=>{ + if matches!(error.as_str(),"temporal_writer_ack_clock_or_expiry"|"temporal_writer_ack_identity_changed"|"temporal_writer_ack_row_lineage_changed"|"temporal_writer_ack_binding_changed") {self.invalidate(&error);} + return Err(error); + } + }; + if let Some(state)=&self.workflow.async_state{self.durable_seals=state.accepted_seals;} + Ok(start) + } + pub(crate) fn enable_async_restore(&mut self,store:std::sync::Arc,now:u64)->Result<(),String>{ + if self.workflow.async_state.is_some(){return Err("temporal_writer_already_present".into());} + let mut state=durability::AsyncState::new(std::sync::Arc::new(durability::Writer::new(store,now)?)); + state.seed_restored(&self.workflow)?; + self.workflow.async_state=Some(state);Ok(()) + } + pub(crate) fn writer(&self)->Option>{ + self.workflow.async_state.as_ref().map(|a|a.writer.clone()) + } pub(crate) fn prepare_retained( schedule: ManagedScheduleContract, run: String, @@ -505,7 +544,7 @@ impl TemporalTrainCapture { self.workflow.context() } pub(crate) fn check_time(&mut self, now: u64) -> Result<(), String> { - self.ensure_live()?; + if self.workflow.async_state.is_none(){self.ensure_live()?;}else if self.failure.is_some(){return Err("temporal_capture_failed".into());} if now < self.workflow.last_now_ns || now / 1_000_000 >= self.workflow.descriptor.expires_at_ms { @@ -515,11 +554,11 @@ impl TemporalTrainCapture { Ok(()) } pub(crate) fn prime_fence(&mut self, p: &CanonicalPacket, now: u64) -> Result<(), String> { - self.ensure_live()?; + if self.workflow.async_state.is_none(){self.ensure_live()?;}else if self.failure.is_some(){return Err("temporal_capture_failed".into());} if !matches!( self.stage(), - TemporalCaptureStage::Ready | TemporalCaptureStage::Paused - ) { + TemporalCaptureStage::Ready | TemporalCaptureStage::Paused | TemporalCaptureStage::Preparing | TemporalCaptureStage::Blocked + ) && !(self.workflow.async_state.is_some() && self.logical_phase_complete()) { return Err("temporal_priming_state".into()); } let result: Result<(), String> = (|| { @@ -559,6 +598,9 @@ impl TemporalTrainCapture { return TemporalCaptureStage::Failed; } let w = &self.workflow; + if w.async_state.as_ref().is_some_and(|a| a.pending.as_ref().is_some_and(|p|p.start_ticket || !matches!(w.stage,Stage::Observing|Stage::BlockedDurability)) || w.durable.is_none()) { + return TemporalCaptureStage::Preparing; + } if w.stage == Stage::BlockedDurability { return TemporalCaptureStage::Blocked; } @@ -598,6 +640,12 @@ impl TemporalTrainCapture { .ok()?; Some((bound.trial_id().into(), bound.session_id().into())) } + fn logical_phase_complete(&self)->bool { + let w=&self.workflow; + self.schedule.episodes().get(w.episode).is_some_and(|e|e.role==DataRole::Train + && w.pending_window.is_none() + && w.session.rows.len()==self.schedule.episodes().iter().take(w.episode+1).map(|e|e.slots.len()).sum::()) + } fn phase_ready(&self) -> bool { let w = &self.workflow; let s = &self.schedule; @@ -658,7 +706,8 @@ impl TemporalTrainCapture { store: &RecoveryStore, ) -> Result<(), String> { self.ensure_live()?; - if self.phase_ready() { + if self.phase_ready() || (self.workflow.async_state.is_some() && self.logical_phase_complete()) { + if self.workflow.async_state.is_some(){return self.prime_fence(p,now);} return Ok(()); } let result = (|| { @@ -725,6 +774,18 @@ impl TemporalTrainCapture { let schedule = w.schedule()?; let deadline = schedule.episodes()[w.episode].slots[observed].deadline_offset_ms * 1_000_000; + if w.async_state.as_ref().is_some_and(|a|a.pending.is_some()) { + let next=schedule.episodes()[w.episode].slots.get(observed+1); + if let Some(next)=next { + let mapping=w.mapping.as_ref().ok_or("temporal_segment_missing")?; + let deadline=mapping.receiver_start_ns.checked_add(next.target_offset_ms*1_000_000-mapping.observed_start_ns).ok_or("temporal_time_overflow")?; + if w.last_now_ns>=deadline { + let state=w.async_state.as_mut().unwrap();state.busy_pause=true;state.error=Some("temporal_storage_busy_pause".into()); + w.stage=Stage::BlockedDurability;return Err("temporal_storage_busy_pause".into()); + } + } + return Ok(()); + } let b = w.bridge.as_mut().ok_or("temporal_bridge_required")?; b.parts .last_mut() @@ -734,14 +795,14 @@ impl TemporalTrainCapture { b.disposition = Disposition::NeedsConfirmation; } w.commit_slot(store, w.last_now_ns)?; - if !self.phase_ready() { + if !self.logical_phase_complete() { self.workflow.begin_slot()?; } Ok(()) } pub(crate) fn confirmation_challenge(&self) -> Result { self.ensure_live()?; - if !self.phase_ready() { + if !self.phase_ready() || self.workflow.async_state.as_ref().is_some_and(|a|a.pending.is_some()) { return Err("temporal_phase_not_complete".into()); } let w = &self.workflow; @@ -751,7 +812,7 @@ impl TemporalTrainCapture { &w.descriptor.id, &w.session.run_id, w.episode, - w.durable.as_ref().map(|v| bytes_hash(v)), + w.durable_digest(), &b.parts, &b.seals, )) @@ -770,7 +831,10 @@ impl TemporalTrainCapture { w.check_clock(now)?; let schedule = w.schedule()?; let context_sha256 = w.context().digest(); - let row_prefix_sha256 = hash(&(&w.session.rows, &w.session.evidence, &w.ledger.spans))?; + let row_prefix_sha256 = match &w.async_state { + Some(state)=>state.row_prefix_sha256.clone().ok_or("temporal_row_prefix_unconfirmed")?, + None=>hash(&(&w.session.rows, &w.session.evidence, &w.ledger.spans))?, + }; let b = w.bridge.as_mut().ok_or("temporal_bridge_required")?; validate_parts( &b.parts, @@ -807,11 +871,12 @@ impl TemporalTrainCapture { }; w.stage = Stage::DurableReady; w.persist_with(store, false, |s, i, p, n| s.publish_temporal(i, Some(p), n))?; - self.durable_seals = w.bridge.as_ref().unwrap().seals.len(); + if w.async_state.is_none(){self.durable_seals = w.bridge.as_ref().unwrap().seals.len();} Ok(()) } pub(crate) fn pause(&mut self, now: u64, store: &RecoveryStore) -> Result<(), String> { self.ensure_live()?; + if self.workflow.async_state.is_some(){return self.workflow.request_async_pause(now);} self.workflow.check_clock(now)?; self.workflow.pause()?; self.workflow @@ -844,6 +909,7 @@ impl TemporalTrainCapture { }) } pub(crate) fn repair_pending(&mut self, store: &RecoveryStore, now: u64) -> Result<(), String> { + if self.workflow.async_state.is_some(){return self.workflow.request_async_repair(now);} self.workflow.repair_pending(store, now)?; if self.workflow.schedule()?.sha256() != self.schedule.sha256() { self.invalidate("temporal_repaired_schedule_changed"); @@ -859,6 +925,7 @@ impl TemporalTrainCapture { Ok(()) } pub(crate) fn invalidate(&mut self, reason: &str) { + if let Some(state)=&self.workflow.async_state{state.writer.cancel();} self.failure = Some( reason .chars() @@ -879,6 +946,15 @@ impl TemporalTrainCapture { pub(crate) fn snapshot(&self) -> Value { let w = &self.workflow; let e = self.schedule.episodes().get(w.episode); - json!({"schema":"ruview.room-learner.private-temporal-capture.v1","stage":self.stage().as_str(),"episode_index":w.episode,"label":e.filter(|e|e.role==DataRole::Train).map(|e|e.condition),"role":"Train","sealed_train_episodes":self.durable_seals,"retained_rows":w.session.rows.len(),"durable_windows":w.durable_windows,"rows_sha256":hash(&w.session.rows).ok(),"observed_ns":w.live_observed_ns.to_string(),"durable_observed_ns":w.accepted().to_string(),"target_ms":e.filter(|e|e.role==DataRole::Train).map(|e|e.target_active_recording_ms),"descriptor":w.descriptor,"failure":self.failure,"fit_admission_available":false,"quality_is_engineering_only":true,"independence_verified":false,"source_clock_measured":false}) + json!({"schema":"ruview.room-learner.private-temporal-capture.v1","stage":self.stage().as_str(),"episode_index":w.episode,"label":e.filter(|e|e.role==DataRole::Train).map(|e|e.condition),"role":"Train","sealed_train_episodes":self.durable_seals,"retained_rows":w.session.rows.len(),"durable_windows":w.durable_windows,"rows_sha256":w.async_state.as_ref().map(|a|if w.session.rows.len()==w.durable_windows{a.rows_sha256.clone()}else{None}).unwrap_or_else(||hash(&w.session.rows).ok()),"durability_pending":w.async_state.as_ref().is_some_and(|a|a.pending.is_some()),"durability_error":w.async_state.as_ref().and_then(|a|a.error.clone()),"observed_ns":w.live_observed_ns.to_string(),"durable_observed_ns":w.accepted().to_string(),"target_ms":e.filter(|e|e.role==DataRole::Train).map(|e|e.target_active_recording_ms),"descriptor":w.descriptor,"failure":self.failure,"fit_admission_available":false,"quality_is_engineering_only":true,"independence_verified":false,"source_clock_measured":false}) + } +} + +impl Drop for TemporalTrainCapture { + fn drop(&mut self) { + // Runtime drains installed captures before dropping its inner state. + // Constructor failures/obsolete unpublished cores must also retain the + // owner store until their one writer has actually stopped. + if let Some(state)=&self.workflow.async_state {let _=state.writer.drain();} } } diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs index bdc51261e1..17d8f6edc4 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs @@ -3,14 +3,14 @@ use super::*; #[derive(Serialize, Deserialize)] #[serde(deny_unknown_fields)] -struct Prefix { +pub(super) struct Prefix { schema: String, binding: ManagedBinding, run: String, descriptor: RecoveryDescriptor, last_now_ns: u64, - rows: Vec, - evidence: Vec, + rows: Vec, + evidence: Vec, identities: Vec, fences: AssemblerCheckpoint, source_high: Vec<(u8, u64, u64)>, @@ -25,14 +25,56 @@ struct Prefix { } #[derive(Serialize, Deserialize)] #[serde(deny_unknown_fields)] -struct Envelope { +struct Envelope

{ schema: String, payload_sha256: String, - payload: Prefix, + payload: P, } const SCHEMA: &str = "ruview.room-learner.temporal-runtime-prefix.v1"; const BRIDGE_SCHEMA: &str = "ruview.room-learner.temporal-runtime-prefix.v2"; -pub(super) fn encode(v: &TemporalWorkflow) -> Result, String> { +pub(super) type Snapshot = Prefix, SharedRecord>; +#[derive(Clone)] +pub(super) struct SharedRecords { + rows: Vec>, + evidence: Vec>, +} +pub(super) fn snapshot_records(snapshot:&Snapshot)->SharedRecords { + SharedRecords{rows:snapshot.rows.clone(),evidence:snapshot.evidence.clone()} +} +pub(super) fn workflow_records(w:&TemporalWorkflow)->SharedRecords { + SharedRecords{rows:w.session.rows.clone(),evidence:w.session.evidence.clone()} +} +pub(super) fn workflow_record_prefix(w:&TemporalWorkflow,count:usize)->Result{ + if count>w.session.rows.len()||count>w.session.evidence.len(){return Err("temporal_record_prefix_missing".into());} + Ok(SharedRecords{rows:w.session.rows[..count].to_vec(),evidence:w.session.evidence[..count].to_vec()}) +} +fn reuse_snapshot_records(w:&mut TemporalWorkflow,records:&SharedRecords)->Result<(),String>{ + if w.session.rows.len()!=records.rows.len() || records.rows.len()!=records.evidence.len() + || hash(&w.session.evidence)?!=hash(&records.evidence)? { + return Err("temporal_writer_shared_records_changed".into()); + } + // decode() already validates every decoded row's exact canonical SHA + // against its WindowEvidence. Validate the SAME expected immutable rows + // against those evidence hashes once, then reuse their cached canonical + // hashes on later prefixes. This is not a new hash formula or authority. + for (row,evidence) in records.rows.iter().zip(&records.evidence) { + if row.canonical_sha256()?!=evidence.cohort_sha256 {return Err("temporal_writer_shared_records_changed".into());} + } + // This call executes on the writer. Free validation's decoded raw copies + // here; the receiver completion contains only refs to its SAME immutable + // authoritative rows/evidence plus bounded reducer metadata. + w.session.rows=records.rows.clone();w.session.evidence=records.evidence.clone();Ok(()) +} +pub(super) fn decode_shared( + bytes:&[u8], descriptor:&RecoveryDescriptor, now:u64, records:&SharedRecords, +)->Result{ + // Keep validation and immutable-ref reuse one private admission boundary. + // No caller-provided/mutated workflow can substitute for this decode. + let mut validated=decode(bytes,descriptor,now)?; + reuse_snapshot_records(&mut validated,records)?; + Ok(validated) +} +pub(super) fn snapshot(v: &TemporalWorkflow) -> Result { if v.session.fitted || v.session.failed || v.session.active @@ -70,8 +112,11 @@ pub(super) fn encode(v: &TemporalWorkflow) -> Result, String> { confirmation_issued: false, bridge: v.bridge.clone(), }; + Ok(payload) +} +pub(super) fn encode_snapshot(payload: Snapshot) -> Result, String> { let envelope = Envelope { - schema: schema.into(), + schema: payload.schema.clone(), payload_sha256: hash(&payload)?, payload, }; @@ -81,6 +126,9 @@ pub(super) fn encode(v: &TemporalWorkflow) -> Result, String> { } Ok(bytes) } +pub(super) fn encode(v: &TemporalWorkflow) -> Result, String> { + encode_snapshot(snapshot(v)?) +} pub(super) fn decode( bytes: &[u8], descriptor: &RecoveryDescriptor, @@ -380,8 +428,8 @@ pub(super) fn decode( } let mut session = TemporalTrainingSession::managed(schedule.clone(), p.run, p.binding.fit.clone())?; - session.rows = p.rows; - session.evidence = p.evidence; + session.rows = p.rows.into_iter().map(SharedRecord::new).collect(); + session.evidence = p.evidence.into_iter().map(SharedRecord::new).collect(); session.managed.as_mut().unwrap().windows = p.identities; session .assembler @@ -427,6 +475,7 @@ pub(super) fn decode( pending_intent: None, pending_prefix: None, bridge: p.bridge, + async_state: None, }) } #[cfg(test)] @@ -689,7 +738,7 @@ pub(crate) fn synthetic_segment_fork(initial: &ValidatedTemporalPrefix) -> Valid ) .unwrap(); w.stage = Stage::DurableReady; - w.durable = Some(initial.bytes().to_vec()); + w.durable = Some(std::sync::Arc::new(initial.bytes().to_vec())); let p = w.synthetic_conditions(); w.start_segment(&p, "different-segment", 2_000_000_000, 2_000_000_000) .unwrap(); @@ -735,7 +784,7 @@ pub(crate) fn synthetic_floor_regression( input: &ValidatedTemporalPrefix, ) -> ValidatedTemporalPrefix { let mut w = decode(input.bytes(), input.descriptor(), input.saved_receiver_ns()).unwrap(); - w.durable = Some(input.bytes().to_vec()); + w.durable = Some(std::sync::Arc::new(input.bytes().to_vec())); let permit = w.synthetic_conditions(); let start = input.saved_receiver_ns() + 10_000_000_000; w.start_segment(&permit, "floor-regression", start, start) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint_tests.rs index 599620c31b..1c07ba97f0 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint_tests.rs @@ -298,3 +298,21 @@ fn bridge_quality_observed_tail_must_remain_covered_by_durable_replay_floors() { "rehashed valid rows cannot hide replay-floor rollback below the observed quality tail" ); } + +#[test] +fn writer_readback_reuses_exact_authoritative_arc_records_and_rejects_changed_records(){ + let(_dir,store,mut workflow)=setup();workflow.persist_initial(&store).unwrap(); + start(&mut workflow,&store,2_000_000_000,"synthetic-shared-records");let mut seq=0; + commit_next(&mut workflow,&store,2_000_000_000,&mut seq); + let frozen=snapshot(&workflow).unwrap();let records=snapshot_records(&frozen); + let bytes=encode_snapshot(frozen).unwrap();let mut restored=decode(&bytes,&workflow.descriptor,workflow.last_now_ns).unwrap(); + assert!(!restored.session.rows[0].same_record(&workflow.session.rows[0])); + reuse_snapshot_records(&mut restored,&records).unwrap(); + assert!(restored.session.rows[0].same_record(&workflow.session.rows[0])); + assert!(restored.session.evidence[0].same_record(&workflow.session.evidence[0])); + assert_eq!(encode(&restored).unwrap(),bytes); + let mut changed=records.clone();let mut row=(*changed.rows[0]).clone();row.label=RoomLabel::Unknown; + changed.rows[0]=SharedRecord::new(row); + assert_eq!(reuse_snapshot_records(&mut restored,&changed).unwrap_err(),"temporal_writer_shared_records_changed"); + assert!(restored.session.rows[0].same_record(&workflow.session.rows[0])); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_durability.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_durability.rs new file mode 100644 index 0000000000..31870a13f0 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_durability.rs @@ -0,0 +1,878 @@ +//! Private, bounded serial persistence executor. No packet queue or authority issuer. +use super::*; +use std::sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + mpsc, Arc, Mutex, +}; +use std::time::Instant; + +pub(super) struct CommitRequest { + pub id: u64, + pub saved_receiver_ns: u64, + pub snapshot: checkpoint::Snapshot, + pub descriptor: RecoveryDescriptor, + pub expected: Option, + pub slot: bool, + pub rows: usize, + pub seals: usize, +} +pub(super) struct CommitAck { + pub id: u64, + pub slot: bool, + pub rows: usize, + pub seals: usize, + pub start_ticket: bool, + pub completed_at_ns: u64, + pub outcome: Result, +} +pub(super) struct Committed { + pub token: TemporalToken, + pub bytes: Arc>, + pub durable_sha256: String, + pub rows_sha256: String, + pub row_prefix_sha256: String, + pub restored: Option>, +} +#[cfg(test)] +#[derive(Clone, Copy, Default)] +pub(crate) struct ReclamationStats { + pub has_ack: bool, + pub old_prefix_bytes: usize, + pub old_prefix_last_owner: bool, + pub raw_tail_rows: usize, +} +#[derive(Default)] +pub(crate) struct DeferredReclamation { + ack: Option, + pending: Option, + previous_bytes: Option>>, + previous_workflow: Option>, + fences: Option, + #[cfg(test)] + pub probe: Option>, +} +impl DeferredReclamation { + pub(crate) fn has_ack(&self) -> bool { + self.ack.is_some() + } +} +impl Drop for DeferredReclamation { + fn drop(&mut self) { + // Callers declare this value BEFORE Runtime.inner's MutexGuard, so + // return/error/unwind unlocks before these owning payloads are freed. + #[cfg(test)] + if self.has_ack() { + if let Some(probe) = &self.probe { + probe(ReclamationStats { + has_ack: true, + old_prefix_bytes: self.previous_bytes.as_ref().map_or(0, |b| b.len()), + old_prefix_last_owner: self + .previous_bytes + .as_ref() + .is_some_and(|b| Arc::strong_count(b) == 1), + raw_tail_rows: self.previous_workflow.as_ref().map_or(0, |w| { + w.session.rows.len().saturating_sub(w.durable_windows) + }), + }); + } + } + drop(self.ack.take()); + drop(self.previous_workflow.take()); + drop(self.pending.take()); + drop(self.previous_bytes.take()); + drop(self.fences.take()); + } +} +enum Work { + Commit(CommitRequest), + Recertify { + id: u64, + expected: TemporalToken, + descriptor: RecoveryDescriptor, + rows: usize, + seals: usize, + start_ticket: bool, + records: checkpoint::SharedRecords, + }, + Repair { + id: u64, + }, + Stop, +} +struct Pending { + intent: TemporalIntent, + prefix: checkpoint::ValidatedTemporalPrefix, + id: u64, + slot: bool, + rows: usize, + seals: usize, + records: checkpoint::SharedRecords, +} +struct Shared { + cancelled: AtomicBool, + busy: AtomicBool, + now_ns: AtomicU64, + anchor_ns: u64, + origin: Instant, + #[cfg(test)] + lose_next_ack: AtomicBool, + #[cfg(test)] + ack_queued: AtomicBool, + #[cfg(test)] + corrupt_ack_id: AtomicBool, + #[cfg(test)] + hold: Mutex, mpsc::Receiver<()>)>>, +} +impl Shared { + fn now(&self) -> Result { + let elapsed: u64 = self + .origin + .elapsed() + .as_nanos() + .try_into() + .map_err(|_| "temporal_writer_clock_overflow")?; + Ok(self.now_ns.load(Ordering::Acquire).max( + self.anchor_ns + .checked_add(elapsed) + .ok_or("temporal_writer_clock_overflow")?, + )) + } + fn allowed(&self, descriptor: &RecoveryDescriptor) -> Result { + if self.cancelled.load(Ordering::Acquire) { + return Err("temporal_writer_cancelled".into()); + } + let now = self.now()?; + if now / 1_000_000 >= descriptor.expires_at_ms { + return Err("temporal_writer_original_deadline_expired".into()); + } + Ok(now) + } +} +pub(crate) struct Writer { + send: mpsc::SyncSender, + receive: Mutex>, + shared: Arc, + thread: Mutex>>, +} +impl Writer { + pub fn new(store: Arc, now: u64) -> Result { + let (send, requests) = mpsc::sync_channel(1); + let (replies, receive) = mpsc::sync_channel(1); + let shared = Arc::new(Shared { + cancelled: AtomicBool::new(false), + busy: AtomicBool::new(false), + now_ns: AtomicU64::new(now), + anchor_ns: now, + origin: Instant::now(), + #[cfg(test)] + lose_next_ack: AtomicBool::new(false), + #[cfg(test)] + ack_queued: AtomicBool::new(false), + #[cfg(test)] + corrupt_ack_id: AtomicBool::new(false), + #[cfg(test)] + hold: Mutex::new(None), + }); + let state = shared.clone(); + let thread = std::thread::Builder::new() + .name("ruview-temporal-durable".into()) + .spawn(move || { + let mut pending: Option = None; + while let Ok(work) = requests.recv() { + let (id, slot, rows, seals, start_ticket, result) = match work { + Work::Stop => break, + Work::Commit(request) => { + let meta = (request.id, request.slot, request.rows, request.seals); + let result = (|| { + state.allowed(&request.descriptor)?; + #[cfg(test)] + if let Some((entered, release)) = state + .hold + .lock() + .map_err(|_| "temporal_writer_test_lock")? + .take() + { + let _ = entered.send(()); + let _ = release.recv(); + } + state.allowed(&request.descriptor)?; + if pending.is_some() { + return Err("temporal_writer_exact_repair_required".into()); + } + let records = checkpoint::snapshot_records(&request.snapshot); + let bytes = checkpoint::encode_snapshot(request.snapshot)?; + let at = request.saved_receiver_ns; + let prefix = + checkpoint::validated_prefix(&bytes, &request.descriptor, at)?; + let intent = store.prepare_temporal( + request.expected.as_ref(), + &prefix, + state.allowed(&request.descriptor)?, + )?; + pending = Some(Pending { + intent, + prefix, + id: meta.0, + slot: meta.1, + rows: meta.2, + seals: meta.3, + records, + }); + let p = pending.as_ref().unwrap(); + let token = store.publish_temporal( + &p.intent, + Some(&p.prefix), + state.allowed(p.prefix.descriptor())?, + )?; + #[cfg(test)] + if state.lose_next_ack.swap(false, Ordering::AcqRel) { + return Err("synthetic_lost_publish_ack".into()); + } + readback(&store, &state, p, token) + })(); + if result.is_ok() { + pending = None; + } + (meta.0, meta.1, meta.2, meta.3, false, result) + } + Work::Recertify { + id, + expected, + descriptor, + rows, + seals, + start_ticket, + records, + } => { + let result = (|| { + state.allowed(&descriptor)?; + #[cfg(test)] + if let Some((entered, release)) = state + .hold + .lock() + .map_err(|_| "temporal_writer_test_lock")? + .take() + { + let _ = entered.send(()); + let _ = release.recv(); + } + if pending.is_some() { + return Err("temporal_writer_exact_repair_required".into()); + } + let recertified = store + .recertify_temporal(&expected, state.allowed(&descriptor)?)?; + let restored = checkpoint::decode_shared( + recertified.prefix.bytes(), + recertified.prefix.descriptor(), + state.allowed(&descriptor)?, + &records, + )?; + let rows_sha256 = hash(&restored.session.rows)?; + let row_prefix_sha256 = hash(&( + &restored.session.rows, + &restored.session.evidence, + &restored.ledger.spans, + ))?; + state.allowed(&descriptor)?; + Ok(Committed { + token: recertified.token, + durable_sha256: bytes_hash(recertified.prefix.bytes()), + bytes: Arc::new(recertified.prefix.bytes().to_vec()), + rows_sha256, + row_prefix_sha256, + restored: Some(Box::new(restored)), + }) + })(); + (id, false, rows, seals, start_ticket, result) + } + Work::Repair { id } => { + let Some(p) = pending.as_ref() else { + let _ = replies.send(CommitAck { + id, + slot: false, + rows: 0, + seals: 0, + start_ticket: false, + completed_at_ns: state.now().unwrap_or(0), + outcome: Err("temporal_writer_pending_intent_missing".into()), + }); + continue; + }; + let meta = (p.id, p.slot, p.rows, p.seals); + let result = (|| { + if id != p.id { + return Err("temporal_writer_repair_ticket_changed".into()); + } + let token = store.repair_same_temporal( + &p.intent, + Some(&p.prefix), + state.allowed(p.prefix.descriptor())?, + )?; + readback(&store, &state, p, token) + })(); + if result.is_ok() { + pending = None; + } + (meta.0, meta.1, meta.2, meta.3, false, result) + } + }; + // Capacity one and only one accepted transaction. The receiver + // must consume this result before another request is admitted. + #[cfg(test)] + let id = if state.corrupt_ack_id.swap(false, Ordering::AcqRel) { + id.wrapping_add(1) + } else { + id + }; + if replies + .send(CommitAck { + id, + slot, + rows, + seals, + start_ticket, + completed_at_ns: state.now().unwrap_or(0), + outcome: result, + }) + .is_err() + { + break; + } + #[cfg(test)] + state.ack_queued.store(true, Ordering::Release); + if state.cancelled.load(Ordering::Acquire) { + break; + } + } + }) + .map_err(|_| "temporal_writer_spawn_failed")?; + Ok(Self { + send, + receive: Mutex::new(receive), + shared, + thread: Mutex::new(Some(thread)), + }) + } + pub fn observe_now(&self, now: u64) { + self.shared.now_ns.fetch_max(now, Ordering::AcqRel); + } + pub fn submit(&self, request: CommitRequest) -> Result<(), String> { + if self.shared.cancelled.load(Ordering::Acquire) { + return Err("temporal_writer_cancelled".into()); + } + if self + .shared + .busy + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return Err("temporal_storage_busy".into()); + } + #[cfg(test)] + self.shared.ack_queued.store(false, Ordering::Release); + if self.send.try_send(Work::Commit(request)).is_err() { + self.shared.busy.store(false, Ordering::Release); + return Err("temporal_writer_unavailable".into()); + } + Ok(()) + } + fn recertify( + &self, + id: u64, + expected: TemporalToken, + descriptor: RecoveryDescriptor, + rows: usize, + seals: usize, + start_ticket: bool, + records: checkpoint::SharedRecords, + ) -> Result<(), String> { + if self.shared.cancelled.load(Ordering::Acquire) + || self + .shared + .busy + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return Err("temporal_storage_busy".into()); + } + #[cfg(test)] + self.shared.ack_queued.store(false, Ordering::Release); + if self + .send + .try_send(Work::Recertify { + id, + expected, + descriptor, + rows, + seals, + start_ticket, + records, + }) + .is_err() + { + self.shared.busy.store(false, Ordering::Release); + return Err("temporal_writer_unavailable".into()); + } + Ok(()) + } + pub fn poll(&self) -> Result, String> { + match self + .receive + .lock() + .map_err(|_| "temporal_writer_reply_lock")? + .try_recv() + { + Ok(ack) => { + self.shared.busy.store(false, Ordering::Release); + Ok(Some(ack)) + } + Err(mpsc::TryRecvError::Empty) => Ok(None), + Err(_) => Err("temporal_writer_disconnected".into()), + } + } + pub fn repair(&self, id: u64) -> Result<(), String> { + if self + .shared + .busy + .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire) + .is_err() + { + return Err("temporal_storage_busy".into()); + } + #[cfg(test)] + self.shared.ack_queued.store(false, Ordering::Release); + if self.send.try_send(Work::Repair { id }).is_err() { + self.shared.busy.store(false, Ordering::Release); + return Err("temporal_writer_unavailable".into()); + } + Ok(()) + } + pub fn cancel(&self) { + self.shared.cancelled.store(true, Ordering::Release); + let _ = self.send.try_send(Work::Stop); + } + /// Caller MUST be outside Runtime.inner. Finishing a publication cannot be + /// undone; this barrier prevents an old worker from publishing after erase. + pub fn drain(&self) -> Result<(), String> { + self.cancel(); + let joined = if let Some(thread) = self + .thread + .lock() + .map_err(|_| "temporal_writer_join_lock")? + .take() + { + thread + .join() + .map_err(|_| "temporal_writer_panicked".to_string()) + } else { + Ok(()) + }; + // A revoked/failed capture may never poll its final ACK. Release its + // large serialized/candidate buffers here, not when Core drops inner. + if let Ok(receiver) = self.receive.lock() { + while let Ok(ack) = receiver.try_recv() { + drop(ack); + } + } + joined + } + #[cfg(test)] + pub fn ack_is_queued(&self) -> bool { + self.shared.ack_queued.load(Ordering::Acquire) + } + #[cfg(test)] + pub fn corrupt_next_ack_id(&self) { + self.shared.corrupt_ack_id.store(true, Ordering::Release); + } + #[cfg(test)] + pub fn lose_next_ack(&self) { + self.shared.lose_next_ack.store(true, Ordering::Release); + } + #[cfg(test)] + pub fn hold_next(&self, entered: mpsc::SyncSender<()>, release: mpsc::Receiver<()>) { + *self.shared.hold.lock().unwrap() = Some((entered, release)); + } +} +impl Drop for Writer { + fn drop(&mut self) { + self.shared.cancelled.store(true, Ordering::Release); + let _ = self.send.try_send(Work::Stop); + } +} +fn readback( + store: &RecoveryStore, + state: &Shared, + p: &Pending, + token: TemporalToken, +) -> Result { + let now = state.allowed(p.prefix.descriptor())?; + let loaded = store + .load_temporal(now)? + .ok_or("temporal_readback_missing")?; + if loaded.token != token || loaded.prefix.as_ref().map(|x| x.bytes()) != Some(p.prefix.bytes()) + { + return Err("temporal_readback_changed".into()); + } + let restored = + checkpoint::decode_shared(p.prefix.bytes(), p.prefix.descriptor(), now, &p.records)?; + let rows_sha256 = hash(&restored.session.rows)?; + let row_prefix_sha256 = hash(&( + &restored.session.rows, + &restored.session.evidence, + &restored.ledger.spans, + ))?; + state.allowed(p.prefix.descriptor())?; + Ok(Committed { + token, + durable_sha256: bytes_hash(p.prefix.bytes()), + bytes: Arc::new(p.prefix.bytes().to_vec()), + rows_sha256, + row_prefix_sha256, + restored: Some(Box::new(restored)), + }) +} + +#[derive(Clone)] +pub(super) struct PendingMeta { + id: u64, + slot: bool, + rows: usize, + seals: usize, + pub start_ticket: bool, + predecessor: Option, + row_lineage: Vec>, + evidence_lineage: Vec>, +} +pub(super) struct AsyncState { + pub writer: Arc, + pub pending: Option, + next_id: u64, + pub busy_pause: bool, + repairing: bool, + durable_sha256: Option, + pub rows_sha256: Option, + pub row_prefix_sha256: Option, + pub error: Option, + pub accepted_seals: usize, +} +impl AsyncState { + pub fn new(writer: Arc) -> Self { + Self { + writer, + pending: None, + next_id: 0, + busy_pause: false, + repairing: false, + durable_sha256: None, + rows_sha256: None, + row_prefix_sha256: None, + error: None, + accepted_seals: 0, + } + } + pub fn seed_restored(&mut self, w: &TemporalWorkflow) -> Result<(), String> { + self.durable_sha256 = w.durable.as_ref().map(|b| bytes_hash(b)); + self.rows_sha256 = Some(hash(&w.session.rows)?); + self.row_prefix_sha256 = Some(hash(&( + &w.session.rows, + &w.session.evidence, + &w.ledger.spans, + ))?); + self.accepted_seals = w.bridge.as_ref().map_or(0, |b| b.seals.len()); + Ok(()) + } + fn id(&mut self) -> Result { + self.next_id = self + .next_id + .checked_add(1) + .ok_or("temporal_writer_revision_exhausted")?; + Ok(self.next_id) + } +} +impl TemporalWorkflow { + pub(super) fn durable_digest(&self) -> Option { + self.async_state + .as_ref() + .map(|a| a.durable_sha256.clone()) + .unwrap_or_else(|| self.durable.as_ref().map(|b| bytes_hash(b))) + } + pub(super) fn submit_async_commit(&mut self, slot: bool) -> Result<(), String> { + let snapshot = checkpoint::snapshot(self)?; + let state = self.async_state.as_mut().ok_or("temporal_async_required")?; + let id = state.id()?; + let seals = self.bridge.as_ref().map_or(0, |b| b.seals.len()); + let rows = self.session.rows.len(); + state.writer.observe_now(self.last_now_ns); + state.writer.submit(CommitRequest { + id, + saved_receiver_ns: self.last_now_ns, + snapshot, + descriptor: self.descriptor.clone(), + expected: self.commit_token.clone(), + slot, + rows, + seals, + })?; + state.pending = Some(PendingMeta { + id, + slot, + rows, + seals, + start_ticket: false, + predecessor: self.commit_token.clone(), + row_lineage: self.session.rows.clone(), + evidence_lineage: self.session.evidence.clone(), + }); + state.error = None; + if slot { + self.pending_window = None; + } + Ok(()) + } + pub(super) fn request_async_start(&mut self, now: u64) -> Result<(), String> { + self.check_clock(now)?; + if !matches!( + self.stage, + Stage::DurableReady | Stage::PausedNeedsReconfirmation + ) { + return Err("temporal_start_barrier_state".into()); + } + if self.session.rows.len() != self.durable_windows { + return Err("temporal_unconfirmed_rows".into()); + } + let records = checkpoint::workflow_records(self); + let state = self.async_state.as_mut().ok_or("temporal_async_required")?; + if state.pending.is_some() || state.busy_pause { + return Err("temporal_storage_busy".into()); + } + let token = self + .commit_token + .clone() + .ok_or("temporal_initial_barrier_unconfirmed")?; + let id = state.id()?; + let rows = self.durable_windows; + let seals = self.bridge.as_ref().map_or(0, |b| b.seals.len()); + state.writer.observe_now(now); + state.writer.recertify( + id, + token, + self.descriptor.clone(), + rows, + seals, + true, + records, + )?; + state.pending = Some(PendingMeta { + id, + slot: false, + rows, + seals, + start_ticket: true, + predecessor: self.commit_token.clone(), + row_lineage: self.session.rows.clone(), + evidence_lineage: self.session.evidence.clone(), + }); + state.error = None; + Ok(()) + } + pub(super) fn request_async_pause(&mut self, now: u64) -> Result<(), String> { + self.check_clock(now)?; + if self.stage != Stage::Observing { + return Err("temporal_pause_state".into()); + } + let records = checkpoint::workflow_record_prefix(self, self.durable_windows)?; + let state = self.async_state.as_mut().ok_or("temporal_async_required")?; + state.busy_pause = true; + state.error = Some("temporal_pause_pending_durability".into()); + if state.pending.is_none() { + let id = state.id()?; + let rows = self.durable_windows; + let seals = state.accepted_seals; + let token = self + .commit_token + .clone() + .ok_or("temporal_initial_barrier_unconfirmed")?; + state.writer.observe_now(now); + state.writer.recertify( + id, + token, + self.descriptor.clone(), + rows, + seals, + false, + records, + )?; + state.pending = Some(PendingMeta { + id, + slot: false, + rows, + seals, + start_ticket: false, + predecessor: self.commit_token.clone(), + row_lineage: self.session.rows.clone(), + evidence_lineage: self.session.evidence.clone(), + }); + } + self.stage = Stage::BlockedDurability; + Ok(()) + } + /// Only the exact current operation can admit a durable result. Live source + /// floors, current custody and uncheckpointed rows are never replaced by an + /// older normal ACK. Returns true only for a consumed one-use StartTicket. + pub(super) fn poll_async( + &mut self, + now: u64, + retired: &mut DeferredReclamation, + ) -> Result, String> { + let Some(state) = self.async_state.as_mut() else { + return Ok(None); + }; + if retired.has_ack() { + return Err("temporal_reclamation_already_occupied".into()); + } + state.writer.observe_now(now); + let Some(ack) = state.writer.poll()? else { + return Ok(None); + }; + // Own the WHOLE response before any fallible validation. Rejected or + // stale successful responses also contain a large plaintext buffer. + retired.ack = Some(ack); + let ack = retired.ack.as_mut().unwrap(); + let now = now.max(ack.completed_at_ns); + retired.pending = Some( + state + .pending + .clone() + .ok_or("temporal_obsolete_writer_ack")?, + ); + let pending = retired.pending.as_ref().unwrap(); + if ack.id != pending.id + || ack.slot != pending.slot + || ack.rows != pending.rows + || ack.seals != pending.seals + || ack.start_ticket != pending.start_ticket + || self.commit_token != pending.predecessor + { + return Err("temporal_writer_ack_identity_changed".into()); + } + if self.session.rows.len() < pending.rows + || self.session.evidence.len() < pending.rows + || self + .session + .rows + .iter() + .zip(&pending.row_lineage) + .any(|(a, b)| !a.same_record(b)) + || self + .session + .evidence + .iter() + .zip(&pending.evidence_lineage) + .any(|(a, b)| !a.same_record(b)) + { + return Err("temporal_writer_ack_row_lineage_changed".into()); + } + if now < self.last_now_ns || now / 1_000_000 >= self.descriptor.expires_at_ms { + return Err("temporal_writer_ack_clock_or_expiry".into()); + } + let committed = match &mut ack.outcome { + Ok(result) => result, + Err(error) => { + state.error = Some(error.clone()); + if pending.start_ticket { + state.pending = None; + self.stage = Stage::PausedNeedsReconfirmation; + } else { + self.stage = Stage::BlockedDurability; + } + return Err(error.clone()); + } + }; + let candidate = committed + .restored + .as_ref() + .ok_or("temporal_writer_restore_missing")?; + if candidate.context() != self.context() + || candidate.session.run_id != self.session.run_id + || candidate.descriptor.id != self.descriptor.id + || candidate.descriptor.expires_at_ms != self.descriptor.expires_at_ms + || candidate.session.rows.len() != pending.rows + { + return Err("temporal_writer_ack_binding_changed".into()); + } + let must_restore = self + .async_state + .as_ref() + .is_some_and(|s| s.busy_pause || s.repairing); + if must_restore { + retired.fences = Some(self.session.assembler.fence_checkpoint()?); + let restored = committed.restored.as_mut().unwrap(); + restored.session.assembler.restore_prefix_fences( + retired + .fences + .as_ref() + .unwrap() + .clone() + .for_prefix(pending.rows)?, + pending.rows, + )?; + restored.source_high = self.source_high.clone(); + restored.mapping = self.mapping.clone(); + restored.live_observed_ns = self.live_observed_ns; + restored.last_now_ns = now; + restored.stage = Stage::PausedNeedsReconfirmation; + restored + .bridge + .as_mut() + .ok_or("temporal_bridge_required")? + .disposition = bridge::Disposition::Paused; + } + // All fallible candidate checks finished while its ownership remained + // in retired. From here transfer old owners, never destroy them here. + let mut state = self.async_state.take().unwrap(); + state.pending = None; + state.error = None; + state.durable_sha256 = Some(committed.durable_sha256.clone()); + state.accepted_seals = pending.seals; + state.rows_sha256 = Some(committed.rows_sha256.clone()); + state.row_prefix_sha256 = Some(committed.row_prefix_sha256.clone()); + retired.previous_bytes = self.durable.take(); + self.commit_token = Some(committed.token.clone()); + self.durable = Some(committed.bytes.clone()); + self.durable_windows = pending.rows; + if must_restore { + let restored = committed.restored.as_mut().unwrap(); + restored.commit_token = self.commit_token.clone(); + restored.durable = self.durable.clone(); + let persist_pause = state.busy_pause; + state.busy_pause = false; + state.repairing = false; + restored.async_state = Some(state); + let replacement = *committed.restored.take().unwrap(); + retired.previous_workflow = Some(Box::new(std::mem::replace(self, replacement))); + if persist_pause { + self.descriptor.saved_at_ms = now / 1_000_000; + self.submit_async_commit(false)?; + } + } else { + self.async_state = Some(state); + } + self.last_now_ns = self.last_now_ns.max(now); + Ok(Some((pending.start_ticket, now))) + } + pub(super) fn request_async_repair(&mut self, now: u64) -> Result<(), String> { + self.check_clock(now)?; + if self.stage != Stage::BlockedDurability { + return Err("temporal_pending_repair_state".into()); + } + let state = self.async_state.as_mut().ok_or("temporal_async_required")?; + let pending = state + .pending + .as_ref() + .ok_or("temporal_pending_intent_missing")?; + state.writer.observe_now(now); + state.writer.repair(pending.id)?; + state.repairing = true; + Ok(()) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs index f8f082b41e..bc6fb5a8a2 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs @@ -10,6 +10,11 @@ use crate::room_learner::{ temporal_acquisition::{AssemblerCheckpoint, ManagedWindowTarget}, }; use serde::{Deserialize, Serialize}; +#[path = "temporal_durability.rs"] +mod durability; +use durability::AsyncState; +pub(crate) use durability::DeferredReclamation; +#[cfg(test)] pub(crate) use durability::ReclamationStats; #[path = "temporal_capture_quality.rs"] mod bridge; use bridge::BridgeState; @@ -181,13 +186,14 @@ struct TemporalWorkflow { live_observed_ns: u64, source_high: Vec<(u8, u64, u64)>, pending_window: Option, - durable: Option>, + durable: Option>>, durable_windows: usize, custody: SegmentCustody, commit_token: Option, pending_intent: Option, pending_prefix: Option, bridge: Option, + async_state: Option, } impl TemporalWorkflow { fn context(&self) -> &CaptureContext { @@ -259,6 +265,7 @@ impl TemporalWorkflow { pending_intent: None, pending_prefix: None, bridge: None, + async_state: None, }) } #[cfg(test)] @@ -266,7 +273,7 @@ impl TemporalWorkflow { ConditionsPermit { context: self.context().digest(), run: self.session.run_id.clone(), - durable_sha256: self.durable.as_ref().map(|b| bytes_hash(b)), + durable_sha256: self.durable_digest(), } } fn start_segment( @@ -281,7 +288,7 @@ impl TemporalWorkflow { Stage::DurableReady | Stage::PausedNeedsReconfirmation ) || permit.context != self.context().digest() || permit.run != self.session.run_id - || permit.durable_sha256 != self.durable.as_ref().map(|b| bytes_hash(b)) + || permit.durable_sha256 != self.durable_digest() || epoch.is_empty() || epoch.len() > 128 || anchor < self.last_now_ns @@ -602,6 +609,11 @@ impl TemporalWorkflow { u64, ) -> Result, ) -> Result<(), String> { + if self.async_state.as_ref().is_some_and(|a| a.pending.is_some()) { + self.async_state.as_mut().unwrap().busy_pause = true; + self.stage = Stage::BlockedDurability; + return Err("temporal_storage_busy_pause".into()); + } if slot { self.custody_at(self.last_now_ns)?; let t = self @@ -635,6 +647,11 @@ impl TemporalWorkflow { self.ledger.spans.push(span); } self.descriptor.saved_at_ms = self.last_now_ns / 1_000_000; + if self.async_state.is_some() { + let result=self.submit_async_commit(slot); + if result.is_err(){self.stage=Stage::BlockedDurability;} + return result; + } let bytes = checkpoint::encode(self)?; let prefix = checkpoint::validated_prefix(&bytes, &self.descriptor, self.last_now_ns)?; let intent = @@ -668,7 +685,7 @@ impl TemporalWorkflow { self.pending_intent = None; self.pending_prefix = None; self.durable_windows = self.session.rows.len(); - self.durable = Some(bytes); + self.durable = Some(std::sync::Arc::new(bytes)); if slot { self.pending_window = None; } @@ -701,7 +718,7 @@ impl TemporalWorkflow { restored.mapping = mapping; restored.live_observed_ns = live; restored.commit_token = Some(loaded.token); - restored.durable = Some(prefix.bytes().to_vec()); + restored.durable = Some(std::sync::Arc::new(prefix.bytes().to_vec())); restored.stage = Stage::PausedNeedsReconfirmation; restored.last_now_ns = now_ns; *self = restored; @@ -722,7 +739,7 @@ impl TemporalWorkflow { let token = store.repair_same_temporal(intent, Some(prefix), now_ns)?; let mut restored = checkpoint::decode(prefix.bytes(), prefix.descriptor(), now_ns)?; restored.commit_token = Some(token); - restored.durable = Some(prefix.bytes().to_vec()); + restored.durable = Some(std::sync::Arc::new(prefix.bytes().to_vec())); restored.stage = Stage::PausedNeedsReconfirmation; restored.last_now_ns = now_ns; *self = restored; @@ -771,7 +788,7 @@ impl TemporalWorkflow { } v.stage = Stage::PausedNeedsReconfirmation; v.last_now_ns = now; - v.durable = Some(prefix.bytes().to_vec()); + v.durable = Some(std::sync::Arc::new(prefix.bytes().to_vec())); Ok(v) } } diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input.rs index 0ac8717e2d..a04c0eff16 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input.rs @@ -341,6 +341,39 @@ impl Fences { Ok(()) } } +/// Immutable retained records shared with one bounded durability snapshot. +/// Serialization delegates exactly to the existing record; no new wire shape. +struct SharedRecordValue { + value:T, + // In-memory integrity cache only. Never serialized or accepted from JSON. + sha256:std::sync::OnceLock, +} +#[derive(Clone)] +struct SharedRecord(std::sync::Arc>); +impl SharedRecord { + fn new(value:T)->Self {Self(std::sync::Arc::new(SharedRecordValue{value,sha256:std::sync::OnceLock::new()}))} + fn same_record(&self,other:&Self)->bool {std::sync::Arc::ptr_eq(&self.0,&other.0)} +} +impl SharedRecord { + fn canonical_sha256(&self)->Result<&str,String>{ + if self.0.sha256.get().is_none(){let digest=hash(&self.0.value)?;let _=self.0.sha256.set(digest);} + self.0.sha256.get().map(String::as_str).ok_or("temporal_record_digest_unavailable".into()) + } +} +impl std::ops::Deref for SharedRecord { + type Target=T; + fn deref(&self)->&T {&self.0.value} +} +impl std::borrow::Borrow for SharedRecord { + fn borrow(&self)->&T {&self.0.value} +} +impl Serialize for SharedRecord { + fn serialize(&self,serializer:S)->Result{self.0.value.serialize(serializer)} +} +fn owned_records(records: &[SharedRecord]) -> Vec { + records.iter().map(|record| (**record).clone()).collect() +} + /// Raw cohorts are private and assembled here; there is intentionally no API to /// fit a caller-supplied temporal-evidence flag or caller-created window. pub struct TemporalTrainingSession { @@ -348,8 +381,8 @@ pub struct TemporalTrainingSession { binding: Binding, run_id: String, assembler: NativeWindowAssembler, - rows: Vec, - evidence: Vec, + rows: Vec>, + evidence: Vec>, config: FitConfig, fitted: bool, failed: bool, @@ -471,8 +504,8 @@ impl TemporalTrainingSession { .windows .push(identity.ok_or("managed_window_identity_missing")?); } - self.rows.push(window.cohort); - self.evidence.push(evidence); + self.rows.push(SharedRecord::new(window.cohort)); + self.evidence.push(SharedRecord::new(evidence)); Ok(()) } Err(error) => { @@ -494,7 +527,7 @@ impl TemporalTrainingSession { return Err("temporal_complete_train_schedule_required".into()); } preflight(self.plan.context(), &self.plan, &self.config)?; - let dataset = ValidatedDataset::new(self.plan.context().clone(), self.rows.clone())?; + let dataset = ValidatedDataset::new(self.plan.context().clone(), owned_records(&self.rows))?; let (model, report) = RoomModel::fit(&dataset, self.config.clone())?; if model.encoder_config() != self.plan.encoder_config() { self.failed = true; @@ -504,7 +537,7 @@ impl TemporalTrainingSession { binding: self.binding.clone(), run_id: self.run_id.clone(), model, - evidence: self.evidence.clone(), + evidence: owned_records(&self.evidence), managed: None, }; if let Err(error) = input.validate() { @@ -515,7 +548,7 @@ impl TemporalTrainingSession { Ok((input, report)) } pub fn dataset(&self) -> Result { - ValidatedDataset::new(self.plan.context().clone(), self.rows.clone()) + ValidatedDataset::new(self.plan.context().clone(), owned_records(&self.rows)) } pub fn plan_sha256(&self) -> &str { &self.binding.plan_sha256 diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs index 3584a9626e..041e5c9b11 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs @@ -201,7 +201,7 @@ fn expected_position( fn validate_records( schedule: &ManagedScheduleContract, run: &str, - evidence: &[WindowEvidence], + evidence: &[impl std::borrow::Borrow], records: &[ManagedWindowIdentity], binding: &Binding, train_only: bool, @@ -212,6 +212,7 @@ fn validate_records( let mut fences = Fences::default(); let mut previous: Option<&ManagedWindowIdentity> = None; for (index, (e, r)) in evidence.iter().zip(records).enumerate() { + let e = e.borrow(); let position = expected_position(schedule, index)?; if (r.episode_index, r.slot_index) != position || (train_only && (r.role != DataRole::Train || r.frozen_artifact_sha256.is_some())) @@ -350,7 +351,7 @@ impl TemporalTrainingSession { &self.binding, true, )?; - let dataset = ValidatedDataset::new(schedule.context().clone(), self.rows.clone())?; + let dataset = ValidatedDataset::new(schedule.context().clone(), owned_records(&self.rows))?; if permit.policy != hash(&managed.binding)? || permit.data != hash(&dataset.train())? || permit.evidence != hash(&self.evidence)? @@ -366,7 +367,7 @@ impl TemporalTrainingSession { binding: self.binding.clone(), run_id: self.run_id.clone(), model, - evidence: self.evidence.clone(), + evidence: owned_records(&self.evidence), managed: Some(ManagedModel { binding: managed.binding.clone(), windows: managed.windows.clone(), From 5c115cba7690d2e7a34407fa5f9d2f8c2bd86523 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Fri, 6 Nov 2026 20:37:20 +0100 Subject: [PATCH 070/100] feat(learner): fit confirmed private temporal Train captures --- .../src/room_learner/lifecycle.rs | 7 +- .../src/room_learner/lifecycle_temporal.rs | 48 +- .../room_learner/lifecycle_temporal_fit.rs | 449 +++++++++++ .../lifecycle_temporal_fit_tests.rs | 726 ++++++++++++++++++ .../room_learner/lifecycle_temporal_tests.rs | 14 +- .../room_learner/temporal_artifact_store.rs | 3 +- .../room_learner/temporal_capture_quality.rs | 18 +- .../src/room_learner/temporal_execution.rs | 3 + .../room_learner/temporal_fit_admission.rs | 246 ++++++ .../src/room_learner/temporal_fit_summary.rs | 16 + .../temporal_fit_summary_managed_tests.rs | 2 +- .../temporal_model_input_managed.rs | 15 +- .../temporal_model_input_managed_tests.rs | 6 +- 13 files changed, 1527 insertions(+), 26 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_admission.rs diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs index 198f57511b..2e048009c6 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs @@ -89,13 +89,15 @@ pub struct RoomLearningRuntime { runtime_epoch: String, owner: String, installation: String, - store: ArtifactStore, + store: Arc, recovery_store: Option>, inner: Mutex, #[cfg(test)] reclamation_probe:Mutex>>, #[cfg(test)] panic_after_reclamation_ack:AtomicBool, + #[cfg(test)] + temporal_fit_hook: Mutex>>, clock_origin: std::time::Instant, clock_epoch_ns: u64, } @@ -188,10 +190,11 @@ impl RoomLearningRuntime { runtime_epoch: crate::companion::secret(), owner: owner.into(), installation: installation.into(), - store: ArtifactStore::new(directory, owner, installation)?, + store: Arc::new(ArtifactStore::new(directory, owner, installation)?), recovery_store: recovery_store.map(std::sync::Arc::new), #[cfg(test)] reclamation_probe:Mutex::new(None), #[cfg(test)] panic_after_reclamation_ack:AtomicBool::new(false), + #[cfg(test)] temporal_fit_hook:Mutex::new(None), inner: Mutex::new(Inner { generation: 0, workflow_revision: 0, diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs index 25e2d50444..ecfd71253d 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs @@ -8,6 +8,8 @@ use crate::room_learner::{ const TRAIN_LEAD_IN_NS: u64 = 5_000_000_000; const PRIVATE_KIND: &str = "learner_temporal_capture_private_unsupported_command"; +#[path = "lifecycle_temporal_fit.rs"] +mod fit_control; pub(super) struct RuntimeTemporalCapture { context: CaptureContext, @@ -16,12 +18,15 @@ pub(super) struct RuntimeTemporalCapture { lead_in_until: Option, failure: Option, last_now_ns: u64, + fit_job: Option, + diagnostic_fit: Option, } impl RuntimeTemporalCapture { pub(super) fn owner_generation(&self) -> u64 { self.owner_generation } pub(super) fn invalidate_owner(&mut self) { + self.cancel_fit(); self.lead_in_until = None; if let Some(core) = self.core.as_mut() { core.invalidate("temporal_owner_generation_changed"); @@ -50,6 +55,7 @@ impl RuntimeTemporalCapture { } fn check_time(&mut self, now: u64) -> Result<(), String> { if now < self.last_now_ns { + self.cancel_fit(); self.lead_in_until = None; self.failure = Some("temporal_receiver_clock_reversed".into()); if let Some(core) = self.core.as_mut() { @@ -57,10 +63,15 @@ impl RuntimeTemporalCapture { } return Err("temporal_receiver_clock_reversed".into()); } - self.core + let checked = self.core .as_mut() .ok_or("temporal_initial_barrier_unconfirmed")? - .check_time(now)?; + .check_time(now); + if let Err(error) = checked { + self.cancel_fit(); + return Err(error); + } + if let Some(job) = &self.fit_job { job.observe_now(now); } self.last_now_ns = now; Ok(()) } @@ -133,6 +144,7 @@ impl RuntimeTemporalCapture { .iter() .any(|s| s.source_id == source_id && s.grid_id == grid_id) { + self.cancel_fit(); self.lead_in_until = None; self.failure = Some(bounded(reason.into())); if let Some(core) = self.core.as_mut() { @@ -239,6 +251,8 @@ impl RoomLearningRuntime { lead_in_until: None, failure: None, last_now_ns: now, + fit_job: None, + diagnostic_fit: None, }, ))); state.generation = next_generation; @@ -270,6 +284,7 @@ impl RoomLearningRuntime { } fn start_temporal_phase(&self, expected: &MutationGuard, now: u64) -> Result<(), String> { let mut state = self.command_state(expected)?; + if busy(&state.job) { return Err("temporal_fit_job_busy".into()); } let owner_generation = state.generation; let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { return Err(PRIVATE_KIND.into()); @@ -382,6 +397,7 @@ impl RoomLearningRuntime { ) -> Result<(), String> { let store = self.temporal_store()?; let mut state = self.command_state(expected)?; + if busy(&state.job) { return Err("temporal_fit_job_busy".into()); } let owner_generation = state.generation; let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { return Err(PRIVATE_KIND.into()); @@ -406,6 +422,7 @@ impl RoomLearningRuntime { fn pause_temporal_capture(&self, expected: &MutationGuard, now: u64) -> Result<(), String> { let store = self.temporal_store()?; let mut state = self.command_state(expected)?; + if busy(&state.job) { return Err("temporal_fit_job_busy".into()); } let owner_generation = state.generation; let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { return Err(PRIVATE_KIND.into()); @@ -464,6 +481,8 @@ impl RoomLearningRuntime { lead_in_until: None, failure: None, last_now_ns: now, + fit_job: None, + diagnostic_fit: None, }, ))); state.generation = next_generation; @@ -490,6 +509,8 @@ impl RoomLearningRuntime { lead_in_until: None, failure: None, last_now_ns: now, + fit_job: None, + diagnostic_fit: None, }, ))); state.generation = next_generation; @@ -498,6 +519,7 @@ impl RoomLearningRuntime { fn repair_temporal_capture(&self, expected: &MutationGuard, now: u64) -> Result<(), String> { let store = self.temporal_store()?; let mut state = self.command_state(expected)?; + if busy(&state.job) { return Err("temporal_fit_job_busy".into()); } let owner_generation = state.generation; let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { return Err(PRIVATE_KIND.into()); @@ -516,12 +538,14 @@ impl RoomLearningRuntime { Ok(()) } fn cancel_temporal_durability(&self, expected: &MutationGuard) -> Result<(), String> { - let writer = { + let (writer, fit) = { let mut state = self.command_state(expected)?; let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { return Err(PRIVATE_KIND.into()); }; capture.lead_in_until = None; + let fit = capture.fit_job.clone(); + capture.cancel_fit(); let core = capture .core .as_mut() @@ -531,11 +555,15 @@ impl RoomLearningRuntime { if let Some(writer) = &writer { writer.cancel(); } - writer + if busy(&state.job) { + state.job.cancel.store(true, Ordering::Release); + state.job.phase = JobPhase::Cancelling; + } + (writer, fit) }; - if let Some(writer) = writer { - writer.drain()?; - } + let writer_result = writer.map_or(Ok(()), |writer| writer.drain()); + if let Some(fit) = fit { fit.drain(); } + writer_result?; Ok(()) } fn temporal_capture_snapshot(&self, now: u64) -> Result { @@ -565,6 +593,11 @@ impl Drop for RoomLearningRuntime { Ok(state) => state, Err(poisoned) => poisoned.into_inner(), }; + let fit = match state.capture.as_ref() { + Some(CaptureWorkflow::Temporal(capture)) => capture.fit_job.clone(), + _ => None, + }; + if let Some(fit) = &fit { fit.cancel(); } let writer = match state.capture.as_mut() { Some(CaptureWorkflow::Temporal(capture)) => capture.core.as_mut().and_then(|core| { let writer = core.writer(); @@ -576,5 +609,6 @@ impl Drop for RoomLearningRuntime { if let Some(writer) = writer { let _ = writer.drain(); } + if let Some(fit) = fit { fit.drain(); } } } diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit.rs new file mode 100644 index 0000000000..14cde7d6b0 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit.rs @@ -0,0 +1,449 @@ +//! Private fit-only adapter into the EXISTING Job, fitter and ArtifactStore. +//! No public route, fitted recovery, HeldOut, proof, selection or live authority. +use super::*; +use crate::room_learner::{ + artifact::{TemporalCandidateReceipt, TemporalFitDiagnostics, TemporalFitSaveResult}, + recovery_store::RecoveryStore, + temporal_model_input::{ + managed::execution::{PendingTemporalFitInput, TemporalFitCaptureIdentity}, + TemporalModel, + }, +}; +use std::sync::{atomic::AtomicU64, Condvar, Weak}; +use std::time::Instant; + +#[derive(Clone, PartialEq, Eq)] +struct TemporalFitIdentity { + runtime_epoch: String, + generation: u64, + // The consumed start revision identifies THIS job. Later denied commands + // may consume global revisions without replacing this active operation. + attempt: u64, + capture: TemporalFitCaptureIdentity, +} +pub(super) struct TemporalDiagnosticFit { + model: Arc, + saved: TemporalFitSaveResult, +} +/// One lifetime barrier for the SAME existing Job/executor, not a new queue or +/// trainer. Cancellation is atomic; waiting/publication never holds inner. +struct FitLifetime { + cancel: Arc, + publication: Mutex<()>, + completed: Mutex, + completion: Condvar, + anchor: u64, + origin: Instant, + observed: AtomicU64, + expires_at_ms: u64, + #[cfg(test)] + hook: Option>, +} +impl FitLifetime { + fn now(&self) -> Result { + let elapsed: u64 = self + .origin + .elapsed() + .as_nanos() + .try_into() + .map_err(|_| "temporal_fit_clock_overflow")?; + Ok(self.observed.load(Ordering::Acquire).max( + self.anchor + .checked_add(elapsed) + .ok_or("temporal_fit_clock_overflow")?, + )) + } + fn allowed(&self) -> Result { + if self.cancel.load(Ordering::Acquire) { + return Err("temporal_fit_cancelled".into()); + } + let now = self.now()?; + if now / 1_000_000 >= self.expires_at_ms { + return Err("temporal_fit_original_deadline_expired".into()); + } + Ok(now) + } + fn cancel(&self) { + self.cancel.store(true, Ordering::Release); + } + fn done(&self) -> bool { + *self.completed.lock().unwrap_or_else(|e| e.into_inner()) + } + fn drain(&self) { + self.cancel(); + let mut complete = self.completed.lock().unwrap_or_else(|e| e.into_inner()); + while !*complete { + complete = self + .completion + .wait(complete) + .unwrap_or_else(|e| e.into_inner()); + } + } + fn at(&self, stage: &'static str) { + #[cfg(test)] + if let Some(hook) = &self.hook { + hook(stage); + } + #[cfg(not(test))] + let _ = stage; + } +} +#[derive(Clone)] +pub(super) struct TemporalFitJob { + identity: TemporalFitIdentity, + lifetime: Arc, +} +impl TemporalFitJob { + pub(super) fn observe_now(&self, now: u64) { + self.lifetime.observed.fetch_max(now, Ordering::AcqRel); + } + pub(super) fn cancel(&self) { + self.lifetime.cancel(); + } + pub(super) fn drain(&self) { + self.lifetime.drain(); + } + fn done(&self) -> bool { + self.lifetime.done() + } +} +/// Created before scheduling. Dropping an unstarted/discarded closure also +/// releases waiters. Worker outcome/raw ownership is destroyed before this. +struct FitCompletion { + runtime: Weak, + // An upgraded owner is held only AFTER all fit/storage work. The done + // signal precedes its release, or the last Arc could self-drain here. + owner: Option>, + identity: TemporalFitIdentity, + lifetime: Arc, +} +impl Drop for FitCompletion { + fn drop(&mut self) { + let fallback = if self.owner.is_none() { + self.runtime.upgrade() + } else { + None + }; + if let Some(runtime) = self.owner.as_ref().or(fallback.as_ref()) { + let mut state = runtime.inner.lock().unwrap_or_else(|e| e.into_inner()); + let current = Arc::ptr_eq(&state.job.cancel, &self.lifetime.cancel) + && matches!(state.capture.as_ref(),Some(CaptureWorkflow::Temporal(c)) + if c.fit_job.as_ref().is_some_and(|j|j.identity==self.identity&&Arc::ptr_eq(&j.lifetime,&self.lifetime))); + if current && busy(&state.job) { + state.job.phase = if self.lifetime.cancel.load(Ordering::Acquire) { + JobPhase::Cancelled + } else { + JobPhase::Failed + }; + state.job.error = Some( + if state.job.phase == JobPhase::Cancelled { + "temporal_fit_cancelled" + } else { + "temporal_fit_worker_unavailable" + } + .into(), + ); + } + } + *self + .lifetime + .completed + .lock() + .unwrap_or_else(|e| e.into_inner()) = true; + self.lifetime.completion.notify_all(); + // No fit/store operation follows this signal. A final Runtime Arc may + // now run Drop safely because its exact fit lifetime is complete. + drop(fallback); + } +} +impl RuntimeTemporalCapture { + pub(super) fn cancel_fit(&self) { + if let Some(job) = &self.fit_job { + job.cancel(); + } + } +} +impl RoomLearningRuntime { + /// Private cancellation of the SAME Job, preserving confirmed Train data. + /// Unlike an AtomicBool-only response, completion means publication drained. + pub(super) fn cancel_temporal_fit(&self, expected: &MutationGuard) -> Result { + let job = { + let mut state = self.command_state(expected)?; + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_ref() else { + return Err(PRIVATE_KIND.into()); + }; + let job = capture.fit_job.clone().ok_or("temporal_fit_job_missing")?; + job.cancel(); + if busy(&state.job) { + state.job.phase = JobPhase::Cancelling; + } + job + }; + job.drain(); + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + Ok(state.job.phase) + } + /// Private entry only. The existing public start_training remains Legacy. + pub(super) fn start_temporal_fit( + self: &Arc, + expected: &MutationGuard, + now: u64, + ) -> Result { + let executor = tokio::runtime::Handle::try_current() + .map_err(|_| "temporal_fit_executor_unavailable")?; + let store = self.store.clone(); + let recovery = self + .recovery_store + .clone() + .ok_or("temporal_recovery_unavailable")?; + let retired_job; + let retired_fit; + let (input, identity, lifetime) = { + let mut state = self.command_state(expected)?; + if busy(&state.job) || state.job.phase == JobPhase::Ready { + return Err("temporal_fit_job_busy_or_frozen".into()); + } + let generation = state.generation; + let attempt = state.workflow_revision; + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { + return Err(PRIVATE_KIND.into()); + }; + if capture.owner_generation != generation { + capture.invalidate_owner(); + return Err("temporal_owner_generation_changed".into()); + } + if capture.lead_in_until.is_some() + || capture.failure.is_some() + || capture.fit_job.as_ref().is_some_and(|j| !j.done()) + { + return Err("temporal_fit_capture_not_ready".into()); + } + capture.check_time(now)?; + let input = capture + .core + .as_ref() + .ok_or("temporal_initial_barrier_unconfirmed")? + .prepare_fit_input(now)?; + let binding = input.identity().clone(); + if binding.owner() != self.owner + || binding.installation() != self.installation + || binding.context() != capture.context.digest() + { + return Err("temporal_fit_owner_binding_changed".into()); + } + let cancel = Arc::new(AtomicBool::new(false)); + let identity = TemporalFitIdentity { + runtime_epoch: self.runtime_epoch.clone(), + generation, + attempt, + capture: binding.clone(), + }; + let lifetime = Arc::new(FitLifetime { + cancel: cancel.clone(), + publication: Mutex::new(()), + completed: Mutex::new(false), + completion: Condvar::new(), + anchor: now, + origin: Instant::now(), + observed: AtomicU64::new(now), + expires_at_ms: binding.expires_at_ms(), + #[cfg(test)] + hook: self + .temporal_fit_hook + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(), + }); + retired_fit = capture.fit_job.replace(TemporalFitJob { + identity: identity.clone(), + lifetime: lifetime.clone(), + }); + retired_job = Some(std::mem::replace( + &mut state.job, + Job { + phase: JobPhase::Running, + cancel, + report: None, + error: None, + }, + )); + state.frozen_training = true; + (input, identity, lifetime) + }; + drop(retired_fit); + drop(retired_job); + let attempt = identity.attempt; + let completion = FitCompletion { + runtime: Arc::downgrade(self), + owner: None, + identity: identity.clone(), + lifetime: lifetime.clone(), + }; + executor.spawn_blocking(move || { + // This guard outlives every outcome/session/prefix in the called + // body, including panic unwinding and rejected result admission. + let mut completion = completion; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + Self::compute_temporal_fit(&store, &recovery, input, &identity, &lifetime) + })) + .unwrap_or_else(|_| Err("temporal_fit_worker_failed_safely".into())); + completion.owner = completion.runtime.upgrade(); + lifetime.at("completion_owner_upgraded"); + if let Some(runtime) = &completion.owner { + runtime.admit_temporal_fit(&identity, &lifetime, result); + } else { + drop(result); + } + }); + Ok(attempt) + } + fn compute_temporal_fit( + store: &ArtifactStore, + recovery: &RecoveryStore, + input: PendingTemporalFitInput, + identity: &TemporalFitIdentity, + lifetime: &FitLifetime, + ) -> Result<(Arc, TrainingReport, TemporalFitSaveResult), String> { + lifetime.at("before_validate"); + let validated = input.validate(recovery, lifetime.allowed()?)?; + let (mut session, permit) = validated.into_session_and_admission()?; + lifetime.at("before_fit"); + lifetime.allowed()?; + let configured = session.bound_managed_fit_config()?; + let (model, report) = session.fit_managed_with_cancel(permit, &lifetime.cancel)?; + // Session owns raw refs; release it on the worker, never in admission. + drop(session); + lifetime.at("before_save"); + lifetime.allowed()?; + let saved = { + let _publication = lifetime + .publication + .lock() + .map_err(|_| "temporal_fit_publication_unavailable")?; + let now = lifetime.allowed()?; + identity.capture.recertify_current(recovery, now)?; + lifetime.at("publication_entered"); + lifetime.allowed()?; + #[cfg(not(test))] + let saved = + store.save_temporal_candidate_with_fit_summary(&model, &report, &configured)?; + #[cfg(test)] + let saved = store.save_temporal_candidate_with_fit_summary_probe( + &model, + &report, + &configured, + |_| lifetime.at("model_saved"), + )?; + lifetime.at("after_save"); + saved + }; + identity + .capture + .recertify_current(recovery, lifetime.allowed()?)?; + lifetime.at("before_admission"); + lifetime.allowed()?; + Ok((Arc::new(model), report, saved)) + } + fn admit_temporal_fit( + &self, + identity: &TemporalFitIdentity, + lifetime: &Arc, + outcome: Result<(Arc, TrainingReport, TemporalFitSaveResult), String>, + ) { + // Outcome and retired owners precede the guard: even rejection/panic + // releases inner before destroying model/evidence/report allocations. + let mut retired_diagnostic = None; + let mut retired_report = None; + let mut state = match self.inner.lock() { + Ok(state) => state, + Err(_) => return, + }; + let same = identity.runtime_epoch == self.runtime_epoch + && identity.generation == state.generation + && Arc::ptr_eq(&state.job.cancel, &lifetime.cancel) + && matches!(state.capture.as_ref(),Some(CaptureWorkflow::Temporal(c)) if c.owner_generation==state.generation + && c.context.digest()==identity.capture.context() + && c.context.owner_scope==self.owner && c.context.installation_id==self.installation + && c.fit_job.as_ref().is_some_and(|j|j.identity==*identity&&Arc::ptr_eq(&j.lifetime,lifetime))); + if !same { + return; + } + let now = lifetime.now(); + let capture_valid=now.as_ref().is_ok_and(|now|{ + identity.capture.check_deadline(*now).is_ok()&&matches!(state.capture.as_ref(),Some(CaptureWorkflow::Temporal(c)) + if c.failure.is_none() && c.core.as_ref().and_then(|core|core.fit_capture_identity(*now).ok()).as_ref()==Some(&identity.capture)) + }); + if lifetime.cancel.load(Ordering::Acquire) || !capture_valid { + state.job.phase = JobPhase::Cancelled; + state.job.error = Some("temporal_fit_cancelled_or_capture_changed".into()); + return; + } + state.workflow_revision = state.workflow_revision.saturating_add(1); + match &outcome { + Ok((model, report, saved)) => { + if let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() { + retired_diagnostic = capture.diagnostic_fit.take(); + capture.diagnostic_fit = Some(TemporalDiagnosticFit { + model: model.clone(), + saved: TemporalFitSaveResult { + candidate: saved.candidate.clone(), + fit_diagnostics: saved.fit_diagnostics.clone(), + }, + }); + } + retired_report = state.job.report.replace(report.clone()); + state.job.phase = JobPhase::Ready; + state.job.error = None; + } + Err(error) => { + state.job.phase = JobPhase::Failed; + state.job.error = Some(bounded(error.clone())); + } + } + drop(state); + drop(retired_report); + drop(retired_diagnostic); + drop(outcome); + } + pub(super) fn temporal_diagnostic_fit( + &self, + ) -> Result< + Option<( + Arc, + TemporalCandidateReceipt, + TemporalFitDiagnostics, + )>, + String, + > { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_ref() else { + return Err(PRIVATE_KIND.into()); + }; + if capture.owner_generation != state.generation { + return Err("temporal_owner_generation_changed".into()); + } + if capture.diagnostic_fit.is_some() + && (capture.failure.is_some() + || state.job.phase != JobPhase::Ready + || capture.context.owner_scope != self.owner + || capture.context.installation_id != self.installation + || capture.fit_job.as_ref().is_none_or(|job| { + job.lifetime.cancel.load(Ordering::Acquire) + || job.identity.capture.context() != capture.context.digest() + })) + { + return Err("temporal_fit_owner_or_context_unavailable".into()); + } + Ok(capture.diagnostic_fit.as_ref().map(|fit| { + ( + fit.model.clone(), + fit.saved.candidate.clone(), + fit.saved.fit_diagnostics.clone(), + ) + })) + } +} + +#[cfg(all(test, unix))] +#[path = "lifecycle_temporal_fit_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit_tests.rs new file mode 100644 index 0000000000..51624761e5 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit_tests.rs @@ -0,0 +1,726 @@ +//! SYNTHETIC packets/labels through the actual native/seal producer and fitter. +//! No test permit issuer, physical RF, teacher or qualification evidence. +use super::super::tests as capture_fixture; +use super::*; +use crate::room_learner::{ + native::{canonicalize_esp32, NativePacketInput}, + observation::CaptureProvenance, +}; +use std::sync::mpsc; +use std::time::Duration; +use wifi_densepose_hardware::{Adr018Flags, PpduType}; + +fn configured(root: &Path) -> (Arc, u64) { + let runtime = Arc::new(capture_fixture::runtime(root, capture_fixture::START)); + let store = runtime.recovery_store.clone().unwrap(); + runtime + .configure_temporal_capture_with( + &runtime.mutation_guard().unwrap(), + capture_fixture::context(), + "synthetic-c3-native-fit".into(), + capture_fixture::START, + move |schedule, run, mut config, now, _| { + // Bound when this capture is CREATED, never overridden by fit input. + config.epochs = 2; + TemporalTrainCapture::prepare_async(schedule, run, config, now, store) + }, + ) + .unwrap(); + let now = capture_fixture::settle_async_boundary(&runtime, capture_fixture::START); + (runtime, now) +} +fn packet(source: u8, sequence: u32, at: u64, label: RoomLabel) -> CanonicalPacket { + let gain = match label { + RoomLabel::Empty => 3, + RoomLabel::AdjacentOnly => 4, + RoomLabel::OccupiedStatic => 8, + RoomLabel::OccupiedMoving => 10 + (sequence % 2) as u8, + _ => unreachable!(), + }; + let iq = (0..64) + .flat_map(|bin| [gain + source + (bin % 2) as u8, 4]) + .collect::>(); + canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence, + received_at_ns: at, + receiver_sample_rate_hz: 33., + iq_bytes: &iq, + flags: Adr018Flags::default(), + provenance: CaptureProvenance::LiveReceiverObserved, + }) + .unwrap() +} +fn record_train(root: &Path) -> (Arc, u64) { + let (runtime, mut now) = configured(root); + let schedule = ManagedScheduleContract::build( + capture_fixture::context(), + LeadInPolicy::Train5HeldOut15EngineeringV1, + ) + .unwrap(); + let mut sequence = 0; + let mut starts = 0; + for episode in schedule + .episodes() + .iter() + .filter(|e| e.role == DataRole::Train) + { + loop { + let progress = capture_fixture::snapshot(&runtime, now); + match progress["stage"].as_str().unwrap() { + "ready" | "paused" => { + now = capture_fixture::async_lead_end(&runtime, now); + starts += 1; + runtime + .tick_temporal_capture(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + } + "needs_confirmation" => break, + "failed" => panic!("synthetic C3 phase{} failed: {progress}", episode.index), + _ => {} + } + let pending = capture_fixture::snapshot(&runtime, now)["durability_pending"] == true; + sequence += 1; + for source in 1..=3 { + match runtime.ingest_native(&packet(source, sequence, now, episode.condition), now) + { + Ok(()) => {} + Err(error) if error == "temporal_storage_busy_pause" => break, + Err(error) => panic!("synthetic C3 ingest: {error}"), + } + } + now += capture_fixture::PERIOD; + if pending { + std::thread::sleep(Duration::from_nanos(capture_fixture::PERIOD)); + } + assert!(starts < 120, "synthetic C3 interruption bound exhausted"); + } + let guard = runtime.mutation_guard().unwrap(); + let challenge = runtime.temporal_confirmation_challenge(&guard).unwrap(); + runtime + .confirm_temporal_phase(&guard, &challenge, now) + .unwrap(); + assert_eq!( + capture_fixture::snapshot(&runtime, now)["durability_pending"], + true + ); + now = capture_fixture::settle_async_boundary(&runtime, now); + } + let complete = capture_fixture::snapshot(&runtime, now); + assert_eq!(complete["stage"], "complete"); + assert_eq!( + complete["sealed_train_episodes"], + schedule.counts().train_episodes + ); + assert_eq!(complete["durable_windows"], schedule.counts().train_windows); + println!( + "SYNTHETIC_C3_REAL_TRAIN_CAPTURE: rows={} seals={} starts={starts}", + schedule.counts().train_windows, + schedule.counts().train_episodes + ); + (runtime, now) +} +// This process-local cache is created by the REAL seven-phase native/seal +// producer once, then strictly cold-restored for independent fault cases. It +// contains only disposable SYNTHETIC private test files, never user samples. +fn complete_train(root: &Path) -> (Arc, u64) { + static RECORDED: std::sync::OnceLock<(tempfile::TempDir, u64)> = std::sync::OnceLock::new(); + let (recorded, at) = RECORDED.get_or_init(|| { + let directory = tempfile::tempdir().unwrap(); + let (runtime, at) = record_train(directory.path()); + drop(runtime); + (directory, at) + }); + fn copy_private(from: &Path, to: &Path) { + use std::os::unix::fs::PermissionsExt; + assert!(!std::fs::symlink_metadata(from) + .unwrap() + .file_type() + .is_symlink()); + if from.is_dir() { + std::fs::create_dir_all(to).unwrap(); + std::fs::set_permissions(to, std::fs::Permissions::from_mode(0o700)).unwrap(); + for entry in std::fs::read_dir(from).unwrap() { + let entry = entry.unwrap(); + copy_private(&entry.path(), &to.join(entry.file_name())); + } + } else { + assert!(from.is_file()); + std::fs::copy(from, to).unwrap(); + std::fs::set_permissions(to, std::fs::Permissions::from_mode(0o600)).unwrap(); + } + } + copy_private(recorded.path(), root); + let now = at + 1_000_000_000; + let runtime = Arc::new(capture_fixture::runtime(root, now)); + runtime + .restore_temporal_capture( + &runtime.mutation_guard().unwrap(), + &capture_fixture::context(), + now, + ) + .unwrap(); + assert_eq!( + capture_fixture::snapshot(&runtime, now)["stage"], + "complete" + ); + (runtime, now) +} +struct ReleaseGate(Option>); +impl ReleaseGate { + fn release(&mut self) { + if let Some(send) = self.0.take() { + let _ = send.try_send(()); + } + } +} +impl Drop for ReleaseGate { + fn drop(&mut self) { + self.release(); + } +} +fn hold(runtime: &RoomLearningRuntime, stage: &'static str) -> (mpsc::Receiver<()>, ReleaseGate) { + hold_outcome(runtime, stage, false) +} +fn hold_outcome( + runtime: &RoomLearningRuntime, + stage: &'static str, + unwind: bool, +) -> (mpsc::Receiver<()>, ReleaseGate) { + let (entered_tx, entered_rx) = mpsc::sync_channel(1); + let (release_tx, release_rx) = mpsc::sync_channel(1); + let release = Mutex::new(release_rx); + *runtime.temporal_fit_hook.lock().unwrap() = Some(Arc::new(move |at| { + if at == stage { + entered_tx.send(()).unwrap(); + release + .lock() + .unwrap() + .recv_timeout(Duration::from_secs(30)) + .expect("synthetic fit gate not released"); + assert!(!unwind, "synthetic last-completion-owner unwind"); + } + })); + (entered_rx, ReleaseGate(Some(release_tx))) +} +async fn wait_fit(runtime: &RoomLearningRuntime) -> JobPhase { + let start = Instant::now(); + loop { + let phase = runtime.inner.lock().unwrap().job.phase; + if !matches!(phase, JobPhase::Running | JobPhase::Cancelling) { + return phase; + } + assert!( + start.elapsed() < Duration::from_secs(120), + "bounded synthetic fit did not finish" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn c3_incomplete_real_capture_cannot_issue_fit_or_weaken_public_legacy_guard() { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = configured(directory.path()); + assert!(runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap_err() + .contains("confirmed_durable_train")); + assert!(runtime + .start_training(&runtime.mutation_guard().unwrap(), FitConfig::default()) + .unwrap_err() + .contains("unsupported_command")); + assert_eq!(runtime.inner.lock().unwrap().job.phase, JobPhase::Idle); + assert!(runtime.temporal_diagnostic_fit().unwrap().is_none()); + runtime + .cancel_temporal_durability(&runtime.mutation_guard().unwrap()) + .unwrap(); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn c3_actual_native_train_fit_and_same_store_cold_report_keep_whole_model_diagnostic() { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = complete_train(directory.path()); + let before = capture_fixture::snapshot(&runtime, now); + let (entered, mut release) = hold(&runtime, "before_validate"); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + entered.recv_timeout(Duration::from_secs(10)).unwrap(); + // A denied unsupported command consumes a CAS revision, but does not + // replace the recorded fit operation or strand it in Running. + assert!(runtime + .start_training(&runtime.mutation_guard().unwrap(), FitConfig::default()) + .unwrap_err() + .contains("unsupported_command")); + assert_eq!(runtime.inner.lock().unwrap().job.phase, JobPhase::Running); + release.release(); + assert_eq!( + wait_fit(&runtime).await, + JobPhase::Ready, + "{:?}", + runtime.inner.lock().unwrap().job.error + ); + let (model, receipt, diagnostics) = runtime.temporal_diagnostic_fit().unwrap().unwrap(); + assert_eq!( + receipt.artifact_kind, + "ruview.room-learner.managed-temporal-input.v3" + ); + assert!(receipt.diagnostic_only); + assert!(!receipt.qualification_granted); + assert!(!receipt.physically_validated); + let report = runtime.inner.lock().unwrap().job.report.clone().unwrap(); + assert_eq!(report.training_cohorts, 64); + assert_eq!(report.completed_epochs, 2); + assert!(report.backbone_delta_l2 > 0.); + let summary = diagnostics.summary.unwrap(); + assert_eq!(summary.metrics.training_cohorts, report.training_cohorts); + assert_eq!( + summary.metrics.objective_evaluations, + report.objective_evaluations + ); + assert_eq!( + summary.metrics.parameter_delta_l2, + report.parameter_delta_l2 + ); + let cold = runtime + .store + .load_exact_temporal_candidate(&capture_fixture::context(), &receipt.artifact_sha256) + .unwrap(); + assert_eq!(cold.export().unwrap(), model.export().unwrap()); + let cold_report = runtime.store.temporal_fit_diagnostics(&cold); + assert_eq!( + serde_json::to_value(cold_report.summary).unwrap(), + serde_json::to_value(Some(summary)).unwrap() + ); + let state = runtime.inner.lock().unwrap(); + assert!(state.model.is_none()); + assert!(state.receipt.is_none()); + assert!(!state.selection.has_history()); + assert!(state.qualification.is_none()); + drop(state); + assert_eq!( + capture_fixture::snapshot(&runtime, now)["rows_sha256"], + before["rows_sha256"] + ); + assert!(runtime.qualify(&runtime.mutation_guard().unwrap()).is_err()); + runtime + .cancel_temporal_durability(&runtime.mutation_guard().unwrap()) + .unwrap(); + assert!( + runtime.temporal_diagnostic_fit().is_err(), + "cancelled owner retained a current diagnostic content read" + ); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn c3_cancel_before_validation_drains_same_job_without_candidate_or_legacy_changes() { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = complete_train(directory.path()); + let (entered, mut release) = hold(&runtime, "before_validate"); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + entered.recv_timeout(Duration::from_secs(10)).unwrap(); + let guard = runtime.mutation_guard().unwrap(); + let worker = runtime.clone(); + let (done_tx, done_rx) = mpsc::sync_channel(1); + let cancel = std::thread::spawn(move || { + done_tx.send(worker.cancel_temporal_fit(&guard)).unwrap(); + }); + assert!( + matches!( + done_rx.recv_timeout(Duration::from_millis(50)), + Err(mpsc::RecvTimeoutError::Timeout) + ), + "cancel falsely acknowledged an undrained worker" + ); + // Cancellation waits outside the actual ingress lock. + assert!(runtime.inner.try_lock().is_ok()); + release.release(); + assert_eq!( + done_rx + .recv_timeout(Duration::from_secs(10)) + .unwrap() + .unwrap(), + JobPhase::Cancelled + ); + cancel.join().unwrap(); + assert!(runtime.temporal_diagnostic_fit().unwrap().is_none()); + assert!(runtime + .store + .load_managed_temporal_candidate(&capture_fixture::context()) + .unwrap() + .is_none()); + assert_eq!( + capture_fixture::snapshot(&runtime, now)["stage"], + "complete" + ); + runtime + .cancel_temporal_durability(&runtime.mutation_guard().unwrap()) + .unwrap(); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn c3_exact_store_recertification_rejects_tampered_head_before_any_fit_save() { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = complete_train(directory.path()); + let (entered, mut release) = hold(&runtime, "before_validate"); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + entered.recv_timeout(Duration::from_secs(10)).unwrap(); + let head = directory + .path() + .join(capture_fixture::context().owner_scope) + .join( + "SensingServer/room-learner-recovery/temporal-prefix-commitment-v1/temporal-head.bin", + ); + let original = std::fs::read(&head).unwrap(); + std::fs::write(&head, b"synthetic-tampered-head").unwrap(); + release.release(); + assert_eq!(wait_fit(&runtime).await, JobPhase::Failed); + assert!(runtime.temporal_diagnostic_fit().unwrap().is_none()); + assert!(runtime + .store + .load_managed_temporal_candidate(&capture_fixture::context()) + .unwrap() + .is_none()); + std::fs::write(&head, &original).unwrap(); + runtime + .cancel_temporal_durability(&runtime.mutation_guard().unwrap()) + .unwrap(); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn c3_real_model_save_is_drained_before_synthetic_owner_erase_ack_and_never_recreated() { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = complete_train(directory.path()); + let (entered, mut release) = hold(&runtime, "model_saved"); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + entered.recv_timeout(Duration::from_secs(90)).unwrap(); + // The hook is INSIDE the shared real store body: an actual whole model is + // already saved, but optional report persistence has not executed yet. + assert!(runtime + .store + .load_managed_temporal_candidate(&capture_fixture::context()) + .unwrap() + .is_some()); + // Exercise the actual Runtime tick while genuine artifact persistence is + // held, not just an unrelated synthetic mutex. + runtime + .tick_temporal_capture(&runtime.mutation_guard().unwrap(), now + 1_000_000) + .unwrap(); + let worker = runtime.clone(); + let guard = runtime.mutation_guard().unwrap(); + let (done_tx, done_rx) = mpsc::sync_channel(1); + let cancel = std::thread::spawn(move || { + done_tx + .send(worker.cancel_temporal_durability(&guard)) + .unwrap(); + }); + assert!( + matches!( + done_rx.recv_timeout(Duration::from_millis(50)), + Err(mpsc::RecvTimeoutError::Timeout) + ), + "owner erase ACK escaped a real in-progress store body" + ); + assert!(runtime.inner.try_lock().is_ok()); + release.release(); + done_rx + .recv_timeout(Duration::from_secs(15)) + .unwrap() + .unwrap(); + cancel.join().unwrap(); + assert_eq!(runtime.inner.lock().unwrap().job.phase, JobPhase::Cancelled); + assert!(runtime.temporal_diagnostic_fit().unwrap().is_none()); + // Disposable, explicitly synthetic profile only. Never ask a store reader + // to recreate the erased owner namespace during this post-ACK oracle. + let owner = directory + .path() + .join(capture_fixture::context().owner_scope); + std::fs::remove_dir_all(&owner).unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!( + !owner.exists(), + "old fit recreated an erased profile after drain ACK" + ); + drop(runtime); + assert!(!owner.exists()); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn c3_last_external_owner_drop_drains_real_store_before_return_and_synthetic_erase() { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = complete_train(directory.path()); + let (entered, mut release) = hold(&runtime, "model_saved"); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + entered.recv_timeout(Duration::from_secs(90)).unwrap(); + assert_eq!( + Arc::strong_count(&runtime), + 1, + "worker retained the owner Runtime during storage" + ); + let weak = Arc::downgrade(&runtime); + let (done_tx, done_rx) = mpsc::sync_channel(1); + let dropper = std::thread::spawn(move || { + drop(runtime); + done_tx.send(()).unwrap(); + }); + assert!( + matches!( + done_rx.recv_timeout(Duration::from_millis(50)), + Err(mpsc::RecvTimeoutError::Timeout) + ), + "last owner Drop failed to drain actual storage" + ); + assert!( + weak.upgrade().is_none(), + "fit worker kept Runtime::Drop from starting" + ); + release.release(); + done_rx.recv_timeout(Duration::from_secs(15)).unwrap(); + dropper.join().unwrap(); + let owner = directory + .path() + .join(capture_fixture::context().owner_scope); + std::fs::remove_dir_all(&owner).unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(!owner.exists()); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn c3_last_completion_owner_releases_after_signal_on_normal_and_unwind_paths() { + for unwind in [false, true] { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = complete_train(directory.path()); + let (entered, mut release) = hold_outcome(&runtime, "completion_owner_upgraded", unwind); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + entered.recv_timeout(Duration::from_secs(90)).unwrap(); + let lifetime = { + let state = runtime.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_ref() else { + panic!() + }; + capture.fit_job.as_ref().unwrap().lifetime.clone() + }; + assert_eq!( + Arc::strong_count(&runtime), + 2, + "only post-I/O completion may transiently own Runtime" + ); + let weak = Arc::downgrade(&runtime); + drop(runtime); + release.release(); + let started = Instant::now(); + while !lifetime.done() { + assert!( + started.elapsed() < Duration::from_secs(10), + "last completion Arc self-drained before signaling" + ); + tokio::time::sleep(Duration::from_millis(1)).await; + } + while weak.strong_count() != 0 { + assert!(started.elapsed() < Duration::from_secs(10)); + tokio::time::sleep(Duration::from_millis(1)).await; + } + let owner = directory + .path() + .join(capture_fixture::context().owner_scope); + std::fs::remove_dir_all(&owner).unwrap(); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!owner.exists()); + } +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn c3_original_expiry_during_fitting_withdraws_publication_without_renewal() { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = complete_train(directory.path()); + let original = capture_fixture::snapshot(&runtime, now)["descriptor"].clone(); + let (entered, mut release) = hold(&runtime, "before_save"); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + entered.recv_timeout(Duration::from_secs(90)).unwrap(); + let expiry = original["expires_at_ms"].as_u64().unwrap() * 1_000_000; + assert!(runtime + .tick_temporal_capture(&runtime.mutation_guard().unwrap(), expiry) + .is_err()); + release.release(); + assert_eq!(wait_fit(&runtime).await, JobPhase::Cancelled); + assert!(runtime.temporal_diagnostic_fit().unwrap().is_none()); + assert!(runtime + .store + .load_managed_temporal_candidate(&capture_fixture::context()) + .unwrap() + .is_none()); + assert_eq!( + capture_fixture::snapshot(&runtime, expiry)["descriptor"], + original + ); + runtime + .cancel_temporal_durability(&runtime.mutation_guard().unwrap()) + .unwrap(); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn c3_stale_generation_after_real_save_cannot_adopt_or_strand_job() { + for change in ["generation", "room", "revision", "owner"] { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = complete_train(directory.path()); + let (entered, mut release) = hold(&runtime, "before_admission"); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + entered.recv_timeout(Duration::from_secs(90)).unwrap(); + { + let mut state = runtime.inner.lock().unwrap(); + if change == "generation" { + state.generation += 1; + } else { + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { + panic!() + }; + match change { + "room" => capture.context.room_digest = "changed-room".into(), + "revision" => capture.context.revision += 1, + "owner" => capture.context.owner_scope = "c".repeat(64), + _ => unreachable!(), + } + } + } + release.release(); + assert_eq!(wait_fit(&runtime).await, JobPhase::Failed, "{change}"); + if change == "generation" { + assert!(runtime.temporal_diagnostic_fit().is_err()); + } else { + assert!(runtime.temporal_diagnostic_fit().unwrap().is_none()); + } + assert!(runtime.inner.lock().unwrap().model.is_none()); + assert!( + runtime + .store + .load_managed_temporal_candidate(&capture_fixture::context()) + .unwrap() + .is_some(), + "completed immutable history should not be deleted to fake cancellation" + ); + runtime + .cancel_temporal_durability(&runtime.mutation_guard().unwrap()) + .unwrap(); + } +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn c3_old_completion_cannot_fail_a_replacement_job_with_different_cancel_identity() { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = complete_train(directory.path()); + let (entered, mut release) = hold(&runtime, "before_admission"); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + entered.recv_timeout(Duration::from_secs(90)).unwrap(); + let replacement = Arc::new(AtomicBool::new(false)); + let old_lifetime = { + let mut state = runtime.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_ref() else { + panic!() + }; + let lifetime = capture.fit_job.as_ref().unwrap().lifetime.clone(); + state.job = Job { + phase: JobPhase::Running, + cancel: replacement.clone(), + report: None, + error: None, + }; + lifetime + }; + release.release(); + let started = Instant::now(); + while !old_lifetime.done() { + assert!(started.elapsed() < Duration::from_secs(10)); + tokio::time::sleep(Duration::from_millis(1)).await; + } + { + let mut state = runtime.inner.lock().unwrap(); + assert_eq!(state.job.phase, JobPhase::Running); + assert!(Arc::ptr_eq(&state.job.cancel, &replacement)); + assert!(state.job.error.is_none()); + state.job.phase = JobPhase::Idle; + } + assert!(runtime.temporal_diagnostic_fit().unwrap().is_none()); + runtime + .cancel_temporal_durability(&runtime.mutation_guard().unwrap()) + .unwrap(); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn c3_optional_actual_summary_failure_keeps_durable_diagnostic_model() { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = complete_train(directory.path()); + let weak = Arc::downgrade(&runtime); + let root = directory.path().to_path_buf(); + *runtime.temporal_fit_hook.lock().unwrap() = Some(Arc::new(move |stage| { + if stage == "model_saved" { + let runtime = weak.upgrade().unwrap(); + let model = runtime + .store + .load_managed_temporal_candidate(&capture_fixture::context()) + .unwrap() + .unwrap(); + let binding = model.fit_diagnostic_binding().unwrap(); + let path = root + .join(capture_fixture::context().owner_scope) + .join("SensingServer/room-learner/temporal-models") + .join(binding.context_digest) + .join(format!("{}.fit-summary-v1.json", binding.artifact_sha256)); + std::fs::create_dir(path).unwrap(); + } + })); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + assert_eq!(wait_fit(&runtime).await, JobPhase::Ready); + let (model, receipt, diagnostics) = runtime.temporal_diagnostic_fit().unwrap().unwrap(); + assert_eq!( + serde_json::to_value(&diagnostics).unwrap()["state"], + "not_recorded" + ); + assert!(diagnostics.summary.is_none()); + assert_eq!( + runtime + .store + .load_exact_temporal_candidate(&capture_fixture::context(), &receipt.artifact_sha256) + .unwrap() + .export() + .unwrap(), + model.export().unwrap() + ); + runtime + .cancel_temporal_durability(&runtime.mutation_guard().unwrap()) + .unwrap(); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn c3_worker_unwind_keeps_saved_history_but_never_adopts_current_candidate() { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = complete_train(directory.path()); + *runtime.temporal_fit_hook.lock().unwrap() = Some(Arc::new(|stage| { + if stage == "before_admission" { + panic!("synthetic C3 worker failure") + } + })); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + assert_eq!(wait_fit(&runtime).await, JobPhase::Failed); + assert!(runtime.temporal_diagnostic_fit().unwrap().is_none()); + assert!(runtime + .store + .load_managed_temporal_candidate(&capture_fixture::context()) + .unwrap() + .is_some()); + runtime + .cancel_temporal_durability(&runtime.mutation_guard().unwrap()) + .unwrap(); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs index 98848b5279..7bf6e97441 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs @@ -7,9 +7,9 @@ use crate::room_learner::{ recovery_store::RecoveryStore, }; use wifi_densepose_hardware::{Adr018Flags, PpduType}; -const START: u64 = 1_000_000_000_000; -const PERIOD: u64 = 1_000_000_000u64.div_ceil(33); -fn context() -> CaptureContext { +pub(super) const START: u64 = 1_000_000_000_000; +pub(super) const PERIOD: u64 = 1_000_000_000u64.div_ceil(33); +pub(super) fn context() -> CaptureContext { CaptureContext { owner_scope: "a".repeat(64), installation_id: "synthetic-runtime-temporal".into(), @@ -31,7 +31,7 @@ fn context() -> CaptureContext { .collect(), } } -fn runtime(root: &Path, now: u64) -> RoomLearningRuntime { +pub(super) fn runtime(root: &Path, now: u64) -> RoomLearningRuntime { let c = context(); let directory = root.join(&c.owner_scope).join("SensingServer"); let store = RecoveryStore::new( @@ -77,7 +77,7 @@ fn packet(source: u8, seq: u32, at: u64) -> CanonicalPacket { }) .unwrap() } -fn snapshot(rt: &RoomLearningRuntime, now: u64) -> Value { +pub(super) fn snapshot(rt: &RoomLearningRuntime, now: u64) -> Value { rt.temporal_capture_snapshot(now).unwrap()["capture"].clone() } fn start(rt: &RoomLearningRuntime, at: u64) -> u64 { @@ -557,7 +557,7 @@ fn temporal_storage_barrier_must_not_block_real_runtime_native_ingress() { "real native ingress waited for a storage barrier held by Runtime.inner"); } -fn settle_async_boundary(rt: &RoomLearningRuntime, after: u64) -> u64 { +pub(super) fn settle_async_boundary(rt: &RoomLearningRuntime, after: u64) -> u64 { let started=std::time::Instant::now(); loop { let at=after+started.elapsed().as_nanos() as u64; @@ -645,7 +645,7 @@ fn c2_real_native_ingress_and_source_floors_progress_while_segment_writer_is_blo rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); } -fn async_lead_end(rt:&RoomLearningRuntime,at:u64)->u64{ +pub(super) fn async_lead_end(rt:&RoomLearningRuntime,at:u64)->u64{ rt.start_temporal_phase(&rt.mutation_guard().unwrap(),at).unwrap(); let started=std::time::Instant::now(); loop{ diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_artifact_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_artifact_store.rs index 2f18433070..bf9a784f7a 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_artifact_store.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_artifact_store.rs @@ -1,5 +1,6 @@ //! Whole temporal-input diagnostics in the EXISTING owner-bound ArtifactStore. -//! No runtime caller, managed proof, qualification, selection or inner-v1 access. +//! Private diagnostic Runtime callers only; no managed proof, qualification, +//! selection or inner-v1 access. use super::*; use crate::room_learner::temporal_model_input::TemporalModel; use std::{fs, io::Read}; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs index da1e26a1af..bf4c300a9e 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs @@ -1,5 +1,7 @@ //! Private Runtime capture facade. Whole-period receiver quality is engineering -//! admission only: never source-clock precision, teacher truth or fit authority. +//! admission only: never source-clock precision, teacher truth or qualification. +//! Seals alone grant no fit authority; the private owner must separately consume +//! the complete durable producer prefix into its opaque fit-only admission. use super::*; use crate::room_learner::temporal_acquisition::WindowInputPolicy; use serde_json::{json, Value}; @@ -438,6 +440,20 @@ pub(crate) struct TemporalTrainCapture { durable_seals: usize, } impl TemporalTrainCapture { + pub(crate) fn prepare_fit_input(&self, now: u64) -> Result { + self.ensure_live()?; + if self.stage() != TemporalCaptureStage::Complete { + return Err("temporal_fit_confirmed_durable_train_required".into()); + } + fit_admission::PendingTemporalFitInput::capture(&self.workflow, &self.schedule, now) + } + pub(crate) fn fit_capture_identity(&self, now: u64) -> Result { + self.ensure_live()?; + if self.stage() != TemporalCaptureStage::Complete { + return Err("temporal_fit_confirmed_durable_train_required".into()); + } + fit_admission::current_identity(&self.workflow, &self.schedule, now) + } pub(crate) fn prepare( schedule: ManagedScheduleContract, run: String, diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs index bc6fb5a8a2..99adb73892 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs @@ -19,6 +19,9 @@ pub(crate) use durability::DeferredReclamation; mod bridge; use bridge::BridgeState; pub(crate) use bridge::{TemporalCaptureStage, TemporalTrainCapture}; +#[path = "temporal_fit_admission.rs"] +mod fit_admission; +pub(crate) use fit_admission::{PendingTemporalFitInput, TemporalFitCaptureIdentity}; pub(crate) const EXECUTION_POLICY:&str="observed-complete-reservation-prefix-v1;paused-time;immutable-spans;partial-tail-drop;max128segments;persegment-native-firstarrival-skew-gaps;explicit-genesis-gap-anchor;quality-unimplemented;independence-false"; #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_admission.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_admission.rs new file mode 100644 index 0000000000..4265f4a388 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_admission.rs @@ -0,0 +1,246 @@ +//! Private producer-owned permission to COMPUTE one diagnostic candidate. +//! Existing durable Train seals are not physical/scientific qualification. +use super::*; +use std::sync::Arc; + +/// Fixed binding to the capture's accepted prefix, not a wire proof or a +/// caller-supplied permission. Only this producer module can construct it. +#[derive(Clone, Debug, PartialEq, Eq)] +pub(crate) struct TemporalFitCaptureIdentity { + owner: String, + installation: String, + context: String, + run: String, + schedule: String, + policy: String, + configuration: String, + descriptor: RecoveryDescriptor, + token: TemporalToken, + prefix: String, +} +impl TemporalFitCaptureIdentity { + pub(crate) fn owner(&self) -> &str { + &self.owner + } + pub(crate) fn installation(&self) -> &str { + &self.installation + } + pub(crate) fn context(&self) -> &str { + &self.context + } + pub(crate) fn expires_at_ms(&self) -> u64 { + self.descriptor.expires_at_ms + } + pub(crate) fn check_deadline(&self, now: u64) -> Result<(), String> { + if now == 0 || now / 1_000_000 >= self.descriptor.expires_at_ms { + Err("temporal_fit_original_deadline_expired".into()) + } else { + Ok(()) + } + } + /// Off-lock only. Same-store observation; never publishes or renews TTL. + pub(crate) fn recertify_current(&self, store: &RecoveryStore, now: u64) -> Result<(), String> { + self.check_deadline(now)?; + let current = store.recertify_temporal(&self.token, now)?; + if current.token != self.token + || current.prefix.descriptor() != &self.descriptor + || bytes_hash(current.prefix.bytes()) != self.prefix + { + return Err("temporal_fit_accepted_prefix_changed".into()); + } + Ok(()) + } +} + +/// A bounded ref snapshot of ACTUAL accepted Runtime data. No Clone or serde: +/// imported models, JSON claims and generic sessions cannot create this value. +pub(crate) struct PendingTemporalFitInput { + identity: TemporalFitCaptureIdentity, + accepted_bytes: Arc>, + records: checkpoint::SharedRecords, + not_before_ns: u64, +} +/// Constructed only after exact same-store recertification and strict decoding, +/// off the receiver lock. It owns a fit session, never the live capture owner. +pub(crate) struct ValidatedTemporalFitInput { + identity: TemporalFitCaptureIdentity, + session: TemporalTrainingSession, +} + +fn complete_train( + w: &TemporalWorkflow, + schedule: &ManagedScheduleContract, + now: u64, +) -> Result<(), String> { + let bridge = w + .bridge + .as_ref() + .ok_or("temporal_fit_actual_bridge_required")?; + let managed = w + .session + .managed + .as_ref() + .ok_or("managed_session_required")?; + let counts = schedule.counts(); + if now < w.last_now_ns || now / 1_000_000 >= w.descriptor.expires_at_ms { + return Err("temporal_fit_original_deadline_or_clock".into()); + } + if bridge.disposition != bridge::Disposition::Complete + || bridge.seals.len() != counts.train_episodes + || !bridge.parts.is_empty() + || w.episode != counts.train_episodes + || w.mapping.is_some() + || w.pending_window.is_some() + || w.session.active + || w.session.failed + || w.session.fitted + || w.session.rows.len() != counts.train_windows + || w.session.evidence.len() != counts.train_windows + || managed.windows.len() != counts.train_windows + || w.ledger.spans.len() != counts.train_windows + || w.durable_windows != counts.train_windows + || w.session.rows.iter().any(|row| row.role != DataRole::Train) + || w.pending_intent.is_some() + || w.pending_prefix.is_some() + || w.stage == Stage::BlockedDurability + { + return Err("temporal_fit_confirmed_durable_train_required".into()); + } + Ok(()) +} +fn identity( + w: &TemporalWorkflow, + schedule: &ManagedScheduleContract, + token: TemporalToken, + prefix: String, +) -> Result { + Ok(TemporalFitCaptureIdentity { + owner: w.descriptor.owner_scope.clone(), + installation: w.descriptor.installation_id.clone(), + context: w.context().digest(), + run: w.session.run_id.clone(), + schedule: schedule.sha256().into(), + policy: w.session.plan_sha256().into(), + configuration: hash(&w.session.config)?, + descriptor: w.descriptor.clone(), + token, + prefix, + }) +} +pub(super) fn current_identity( + w: &TemporalWorkflow, + schedule: &ManagedScheduleContract, + now: u64, +) -> Result { + complete_train(w, schedule, now)?; + // The async writer supplies a cached verified digest. Never hash a growing + // raw prefix under the receiver lock or silently fall back to a sync path. + let state = w + .async_state + .as_ref() + .ok_or("temporal_fit_async_durability_required")?; + if state.pending.is_some() + || state.busy_pause + || state.error.is_some() + || state.accepted_seals != schedule.counts().train_episodes + || w.durable.is_none() + { + return Err("temporal_fit_durability_unconfirmed".into()); + } + identity( + w, + schedule, + w.commit_token + .clone() + .ok_or("temporal_fit_commitment_required")?, + w.durable_digest().ok_or("temporal_fit_digest_required")?, + ) +} +impl PendingTemporalFitInput { + pub(super) fn capture( + w: &TemporalWorkflow, + schedule: &ManagedScheduleContract, + now: u64, + ) -> Result { + let identity = current_identity(w, schedule, now)?; + Ok(Self { + identity, + accepted_bytes: w + .durable + .as_ref() + .ok_or("temporal_fit_prefix_required")? + .clone(), + records: checkpoint::workflow_records(w), + not_before_ns: now, + }) + } + pub(crate) fn identity(&self) -> &TemporalFitCaptureIdentity { + &self.identity + } + pub(crate) fn validate( + self, + store: &RecoveryStore, + now: u64, + ) -> Result { + self.identity.check_deadline(now)?; + if now < self.not_before_ns { + return Err("temporal_fit_receiver_clock_reversed".into()); + } + let recertified = store.recertify_temporal(&self.identity.token, now)?; + if recertified.token != self.identity.token + || recertified.prefix.descriptor() != &self.identity.descriptor + || bytes_hash(recertified.prefix.bytes()) != self.identity.prefix + || recertified.prefix.bytes() != self.accepted_bytes.as_slice() + { + return Err("temporal_fit_accepted_prefix_changed".into()); + } + // The existing decoder validates all raw rows, temporal identities, + // source floors, native quality periods and the immutable Train seals. + let restored = checkpoint::decode_shared( + recertified.prefix.bytes(), + recertified.prefix.descriptor(), + now, + &self.records, + )?; + let schedule = restored.schedule()?; + complete_train(&restored, &schedule, now)?; + if identity( + &restored, + &schedule, + recertified.token, + self.identity.prefix.clone(), + )? != self.identity + { + return Err("temporal_fit_input_binding_changed".into()); + } + Ok(ValidatedTemporalFitInput { + identity: self.identity, + session: restored.session, + }) + } +} +impl ValidatedTemporalFitInput { + /// SAME existing permit/hash formulas and SAME fit session implementation. + /// Consumption prevents a cloned generic permit from being reused on a + /// second fit job. A retry requires fresh owner-command admission. + pub(crate) fn into_session_and_admission( + self, + ) -> Result<(TemporalTrainingSession, ManagedFitAdmission), String> { + let managed = self + .session + .managed + .as_ref() + .ok_or("managed_session_required")?; + let dataset = self.session.dataset()?; + let permit = ManagedFitAdmission { + policy: hash(&managed.binding)?, + data: hash(&dataset.train())?, + evidence: hash(&self.session.evidence)?, + identities: hash(&managed.windows)?, + }; + // Full validated data is intentionally dropped here, on the worker. + drop(dataset); + let _ = self.identity; + Ok((self.session, permit)) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_summary.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_summary.rs index d458c379d8..ccd208e566 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_summary.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_summary.rs @@ -291,7 +291,23 @@ impl ArtifactStore { report: &TrainingReport, configured: &FitConfig, ) -> Result { + self.save_temporal_candidate_with_fit_summary_using(model,report,configured, |_|{}) + } + #[cfg(test)] + pub(crate) fn save_temporal_candidate_with_fit_summary_probe( + &self,model:&TemporalModel,report:&TrainingReport,configured:&FitConfig, + model_saved:impl FnOnce(&TemporalCandidateReceipt), + )->Result{ + self.save_temporal_candidate_with_fit_summary_using(model,report,configured,model_saved) + } + /// One shared real store body. The production wrapper supplies a no-op; + /// the test seam can hold the genuine model-save/report-save boundary. + fn save_temporal_candidate_with_fit_summary_using( + &self,model:&TemporalModel,report:&TrainingReport,configured:&FitConfig, + model_saved:impl FnOnce(&TemporalCandidateReceipt), + )->Result{ let candidate = self.save_temporal_candidate(model)?; + model_saved(&candidate); let fit_diagnostics=match self.save_actual_temporal_fit_summary(model,report,configured){Ok(summary)=>TemporalFitDiagnostics::state(FitDiagnosticState::Recorded,Some(summary),None),Err(_)=>TemporalFitDiagnostics::state(FitDiagnosticState::NotRecorded,None,Some("The model was saved, but actual aggregate fit diagnostics were not recorded; existing files were retained."))}; Ok(TemporalFitSaveResult { candidate, diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_summary_managed_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_summary_managed_tests.rs index 946094909a..c75dc6fc7a 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_summary_managed_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_summary_managed_tests.rs @@ -109,7 +109,7 @@ fn actual_fit() -> (TemporalModel, TrainingReport, FitConfig) { } let permit = session.synthetic_test_fit_admission().unwrap(); let (model, report) = session - .fit_managed_with_cancel(&permit, &AtomicBool::new(false)) + .fit_managed_with_cancel(permit, &AtomicBool::new(false)) .unwrap(); assert_eq!(report.training_cohorts, 64); assert!(report.backbone_delta_l2 > 0.); diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs index 041e5c9b11..2ad95b7302 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs @@ -1,5 +1,6 @@ //! Managed-v3 INPUT integration inside the SAME opaque temporal model/session. -//! No Runtime issuer, full-trial quality, confirmation or managed proof exists here. +//! The private Runtime may issue fit-only admission from its exact durable Train +//! producer. Neither this INPUT codec nor its claims issue managed qualification. use super::*; use crate::room_learner::{ managed_schedule_contract::{FrozenModelIdentity, ManagedScheduleContract}, @@ -103,8 +104,9 @@ pub(super) struct ManagedModel { pub(super) binding: ManagedBinding, windows: Vec, } -/// No production issuer is implemented. A future owner Runtime must issue this -/// after its actual confirmed/quality-checked/durable Train episode evidence. +/// The private owner Runtime can obtain this only by consuming the producer's +/// exact confirmed/quality-checked/durable Train prefix. Permission to compute a +/// diagnostic candidate is not physical qualification or selection authority. /// It cannot be uploaded/deserialized or synthesized from generic model metadata. pub(crate) struct ManagedFitAdmission { policy: String, @@ -269,6 +271,11 @@ fn validate_records( Ok(()) } impl TemporalTrainingSession { + pub(crate) fn bound_managed_fit_config(&self) -> Result { + let managed=self.managed.as_ref().ok_or("managed_session_required")?; + managed.binding.contract()?; + Ok(managed.binding.fit.clone()) + } pub fn managed( schedule: ManagedScheduleContract, run_id: String, @@ -330,7 +337,7 @@ impl TemporalTrainingSession { } pub(crate) fn fit_managed_with_cancel( &mut self, - permit: &ManagedFitAdmission, + permit: ManagedFitAdmission, cancel: &AtomicBool, ) -> Result<(TemporalModel, TrainingReport), String> { let managed = self.managed.as_ref().ok_or("managed_session_required")?; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed_tests.rs index d2fe02a8e3..00cdf163eb 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed_tests.rs @@ -141,7 +141,7 @@ fn train() -> (TemporalTrainingSession, TemporalModel) { ); let permit = session.synthetic_test_fit_admission().unwrap(); let (model, report) = session - .fit_managed_with_cancel(&permit, &AtomicBool::new(false)) + .fit_managed_with_cancel(permit, &AtomicBool::new(false)) .unwrap(); assert_eq!(report.training_cohorts, 64); assert_eq!(report.completed_epochs, 2); @@ -318,7 +318,7 @@ fn heldout_uses_frozen_whole_artifact_and_never_changes_train_codec_or_profiles( assert_eq!(m.export().unwrap(), before); let permit = session.synthetic_test_fit_admission().unwrap(); assert!(session - .fit_managed_with_cancel(&permit, &AtomicBool::new(false)) + .fit_managed_with_cancel(permit, &AtomicBool::new(false)) .is_err()); let v: serde_json::Value = serde_json::from_slice(&before).unwrap(); assert!(v["payload"]["identities"] @@ -437,7 +437,7 @@ fn v2_codec_remains_unwrapped_and_managed_fit_permit_cannot_be_omitted_or_forged identities: "d".repeat(64), }; assert!(s - .fit_managed_with_cancel(&fake, &AtomicBool::new(false)) + .fit_managed_with_cancel(fake, &AtomicBool::new(false)) .is_err()); assert!(RoomModel::import(&model().export().unwrap()).is_err()); } From 85c9e9e2490874fa15127458216e6167d378a2ba Mon Sep 17 00:00:00 2001 From: Codex for Dragan Spiridonov Date: Fri, 6 Nov 2026 22:21:48 +0100 Subject: [PATCH 071/100] fix(learner): preserve setup guard and await async capture receipts --- .../src/room_learner/lifecycle_temporal.rs | 10 +-- .../room_learner/lifecycle_temporal_tests.rs | 61 ++++++++++++++++--- 2 files changed, 57 insertions(+), 14 deletions(-) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs index 25e2d50444..825ce8b72e 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs @@ -176,16 +176,16 @@ impl RoomLearningRuntime { run: String, now: u64, ) -> Result<(), String> { - let store = self - .recovery_store - .clone() - .ok_or("temporal_recovery_unavailable")?; self.configure_temporal_capture_with( expected, context, run, now, - move |schedule, run, fit, now, _| { + |schedule, run, fit, now, _| { + // The existing-work guard runs before this off-lock producer. + // Unavailable recovery must never mask refusal to replace a + // selected Legacy capture/model/history. + let store=self.recovery_store.clone().ok_or("temporal_recovery_unavailable")?; TemporalTrainCapture::prepare_async(schedule, run, fit, now, store) }, ) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs index 98848b5279..d3d299875b 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs @@ -9,6 +9,9 @@ use crate::room_learner::{ use wifi_densepose_hardware::{Adr018Flags, PpduType}; const START: u64 = 1_000_000_000_000; const PERIOD: u64 = 1_000_000_000u64.div_ceil(33); +// Test-only supervision for asynchronous storage/authorization ACKs, including +// final-row waits while ingress continues; not observation or shipping budgets. +const QUIESCENT_ACK_TEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(30); fn context() -> CaptureContext { CaptureContext { owner_scope: "a".repeat(64), @@ -325,14 +328,16 @@ fn preparation_replay_phy_and_receiver_clock_cannot_rescue_or_relabel_a_capture( #[test] fn actual_runtime_two_cold_resumes_keep_original_rows_trial_and_expiry_without_downtime_credit() { let dir = tempfile::tempdir().unwrap(); - let mut rt = configured(dir.path()); - let original = snapshot(&rt, START)["descriptor"].clone(); + let (mut rt,mut now) = configured_async(dir.path()); + let original = snapshot(&rt, now)["descriptor"].clone(); let mut seq = 0; - let mut now = START + 1; let mut trial = None; for _ in 0..2 { - let first = start(&rt, now); - now = feed(&rt, first, first + 100_000_000_000, &mut seq); + // Cold restore installs the actual asynchronous writer. Await the + // real start authorization before choosing any observation times. + let first = async_lead_end(&rt, now); + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),first).unwrap(); + now = feed_async_paced(&rt, first, first + 100_000_000_000, &mut seq); let phase = rt.mutation_guard().unwrap().phase.unwrap(); if let Some(prior) = &trial { assert_eq!(prior, &phase.trial_id); @@ -341,6 +346,7 @@ fn actual_runtime_two_cold_resumes_keep_original_rows_trial_and_expiry_without_d } rt.pause_temporal_capture(&rt.mutation_guard().unwrap(), now) .unwrap(); + now=settle_async_boundary(&rt,now); let paused = snapshot(&rt, now); let accepted = paused["durable_observed_ns"] .as_str() @@ -379,17 +385,26 @@ fn actual_runtime_two_cold_resumes_keep_original_rows_trial_and_expiry_without_d .unwrap() .parse::() .unwrap(); - let first = start(&rt, now); - now = feed( + let first = async_lead_end(&rt, now); + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),first).unwrap(); + now = feed_async_paced( &rt, first, first + 600_000_000_000 - accepted + PERIOD, &mut seq, ); + // Keep the declared receiver stream alive while the final complete + // reservation awaits its actual ACK. Do not treat submission as confirmed. + let waiting=std::time::Instant::now(); + while snapshot(&rt,now)["stage"]!="needs_confirmation" { + now=feed_async_paced(&rt,now,now,&mut seq); + assert!(waiting.elapsed()u64{ + // Same accelerated receiver cadence as the existing synthetic journey; + // give actual asynchronous transactions time while pending. No source + // timestamp backdating or production quality limits are changed. + while at<=to { + let pending=snapshot(rt,at)["durability_pending"]==true; + *seq+=1; + for source in 1..=3{rt.ingest_native(&packet(source,*seq,at),at).unwrap();} + at+=PERIOD; + if pending{std::thread::sleep(std::time::Duration::from_nanos(PERIOD));} + } + at +} fn settle_async_boundary(rt: &RoomLearningRuntime, after: u64) -> u64 { let started=std::time::Instant::now(); loop { @@ -567,7 +595,7 @@ fn settle_async_boundary(rt: &RoomLearningRuntime, after: u64) -> u64 { if started.elapsed()>std::time::Duration::from_secs(1){println!("SYNTHETIC_DURABLE_ACK_WAIT_NS={} ROWS={} SEALED={}",started.elapsed().as_nanos(),snapshot(rt,admitted)["durable_windows"],snapshot(rt,admitted)["sealed_train_episodes"]);} return admitted; } - assert!(started.elapsed()u64{ rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap(); let state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()}; if let Some(end)=c.lead_in_until{return end;} - drop(state);assert!(started.elapsed()=Duration::from_secs(6)); + let admitted=lead_end-TRAIN_LEAD_IN_NS;let status=snapshot(&rt,admitted); + assert_eq!(status["retained_rows"],0);assert_eq!(status["durable_windows"],0);assert_eq!(status["observed_ns"],"0"); + assert_eq!(status["lead_in_remaining_ms"],5000,"lead-in must start only at actual authorization ACK"); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); +} From 5124d7b9d64e435ac8d7601b21f3758757d82318 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Fri, 6 Nov 2026 23:41:00 +0100 Subject: [PATCH 072/100] feat(learner): recover privately frozen temporal fits Consume the actual current accepted fit into a durable Frozen transition and exact-model cold recovery, without activating public HeldOut or qualification flows. Prepared by Codex for Dragan's reviewed integration lane. --- .../src/room_learner/artifact.rs | 1 + .../src/room_learner/lifecycle.rs | 4 + .../src/room_learner/lifecycle_temporal.rs | 79 +- .../room_learner/lifecycle_temporal_fit.rs | 61 +- .../room_learner/lifecycle_temporal_frozen.rs | 71 ++ .../lifecycle_temporal_full_run_tests.rs | 938 ++++++++++++++++++ .../room_learner/lifecycle_temporal_tests.rs | 2 +- .../room_learner/temporal_capture_quality.rs | 46 +- .../src/room_learner/temporal_checkpoint.rs | 56 +- .../src/room_learner/temporal_durability.rs | 51 +- .../src/room_learner/temporal_execution.rs | 6 +- .../room_learner/temporal_fit_admission.rs | 70 +- .../src/room_learner/temporal_full_run.rs | 631 ++++++++++++ .../temporal_full_run_checkpoint.rs | 363 +++++++ .../temporal_recovery_commitment.rs | 26 +- 15 files changed, 2376 insertions(+), 29 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_frozen.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run_checkpoint.rs diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs index d5a22429b3..7e2f82b16c 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs @@ -317,6 +317,7 @@ pub use temporal_artifacts::TemporalCandidateReceipt; #[path="temporal_fit_summary.rs"] mod temporal_fit_summary; pub(crate) use temporal_fit_summary::{TemporalFitDiagnostics,TemporalFitSaveResult}; +pub(crate) use temporal_fit_summary::FitDiagnosticState; #[cfg(test)] pub(crate) use temporal_fit_summary::rehash_fixture as rehash_temporal_fit_fixture; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs index 2e048009c6..e49e6a1b6d 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs @@ -558,6 +558,10 @@ mod recovery_enable; #[path = "lifecycle_temporal.rs"] mod temporal_control; +pub(crate) use temporal_control::FullRunGenesisWitness; +pub(crate) use temporal_control::AcceptedTemporalFit; +pub(crate) use temporal_control::FrozenOwnerWitness; +pub(crate) use temporal_control::FullRunRecoveryWitness; #[cfg(test)] #[path="lifecycle_legacy_byte_probe.rs"] diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs index a3cba19cc7..85559c65f9 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs @@ -8,8 +8,42 @@ use crate::room_learner::{ const TRAIN_LEAD_IN_NS: u64 = 5_000_000_000; const PRIVATE_KIND: &str = "learner_temporal_capture_private_unsupported_command"; +/// Only the actual protected Runtime constructor can mint fresh full-run +/// custody. No serde/Clone or caller-supplied authority flag. +pub(crate) struct FullRunGenesisWitness { + runtime_epoch:String,generation:u64,context:CaptureContext,run:String, +} +impl FullRunGenesisWitness { + pub(crate) fn consume(self,context:&CaptureContext,run:&str)->Result<(String,u64),String>{ + if self.context!=*context||self.run!=run||self.runtime_epoch.is_empty()||self.generation==0{ + return Err("temporal_full_run_genesis_owner_changed".into()); + } + Ok((self.runtime_epoch,self.generation)) + } +} +pub(crate) struct FrozenOwnerWitness { + runtime_epoch:String,generation:u64,revision:u64,context:CaptureContext,run:String, +} +pub(crate) struct FullRunRecoveryWitness{runtime_epoch:String,generation:u64,context:CaptureContext} +impl FullRunRecoveryWitness{ + pub(crate) fn consume(self,context:&CaptureContext)->Result<(String,u64),String>{ + if self.context!=*context||self.runtime_epoch.is_empty()||self.generation==0{return Err("temporal_full_run_recovery_owner_changed".into());} + Ok((self.runtime_epoch,self.generation)) + } +} +impl FrozenOwnerWitness { + pub(crate) fn consume(self,context:&CaptureContext,run:&str)->Result<(String,u64,u64),String>{ + if self.context!=*context||self.run!=run||self.runtime_epoch.is_empty()||self.generation==0||self.revision==0{ + return Err("temporal_frozen_owner_operation_changed".into()); + } + Ok((self.runtime_epoch,self.generation,self.revision)) + } +} +#[path="lifecycle_temporal_frozen.rs"] +mod frozen_control; #[path = "lifecycle_temporal_fit.rs"] mod fit_control; +pub(crate) use fit_control::AcceptedTemporalFit; pub(super) struct RuntimeTemporalCapture { context: CaptureContext, @@ -134,6 +168,7 @@ impl RuntimeTemporalCapture { TemporalCaptureStage::Ready | TemporalCaptureStage::Paused | TemporalCaptureStage::Preparing => core.prime_fence(packet, now), + TemporalCaptureStage::Complete|TemporalCaptureStage::Frozen if core.is_full_run()=>core.prime_fence(packet,now), _ => Ok(()), // No automatic start, confirmation or new phase from packet arrival. } } @@ -194,14 +229,19 @@ impl RoomLearningRuntime { run, now, |schedule, run, fit, now, _| { - // The existing-work guard runs before this off-lock producer. - // Unavailable recovery must never mask refusal to replace a - // selected Legacy capture/model/history. + // Existing-work guard precedes this off-lock producer. let store=self.recovery_store.clone().ok_or("temporal_recovery_unavailable")?; TemporalTrainCapture::prepare_async(schedule, run, fit, now, store) }, ) } + /// Fresh genesis only; never upgrades a running old private capture. + fn configure_temporal_full_run(&self,expected:&MutationGuard,context:CaptureContext,run:String,now:u64)->Result<(),String>{ + self.configure_temporal_capture_with_custody(expected,context,run,now,|s,r,f,n,_,w|{ + let store=self.recovery_store.clone().ok_or("temporal_recovery_unavailable")?; + TemporalTrainCapture::prepare_full_run_async(s,r,f,n,store,w) + }) + } fn configure_temporal_capture_with( &self, expected: &MutationGuard, @@ -216,6 +256,12 @@ impl RoomLearningRuntime { &super::super::recovery_store::RecoveryStore, ) -> Result<(TemporalTrainCapture, Option), String>, ) -> Result<(), String> { + self.configure_temporal_capture_with_custody(expected,context,run,now,move|s,r,f,n,store,_|prepare(s,r,f,n,store)) + } + fn configure_temporal_capture_with_custody( + &self,expected:&MutationGuard,context:CaptureContext,run:String,now:u64, + prepare:impl FnOnce(ManagedScheduleContract,String,FitConfig,u64,&super::super::recovery_store::RecoveryStore,FullRunGenesisWitness)->Result<(TemporalTrainCapture,Option),String>, + )->Result<(),String>{ context.validate()?; if context.owner_scope != self.owner || context.installation_id != self.installation @@ -243,6 +289,7 @@ impl RoomLearningRuntime { .generation .checked_add(1) .ok_or("learner generation exhausted")?; + let witness=FullRunGenesisWitness{runtime_epoch:self.runtime_epoch.clone(),generation:next_generation,context:context.clone(),run:run.clone()}; state.capture = Some(CaptureWorkflow::Temporal(Box::new( RuntimeTemporalCapture { context: context.clone(), @@ -258,7 +305,7 @@ impl RoomLearningRuntime { state.generation = next_generation; drop(state); // Initial crypto/filesystem execution never holds the receiver lock. - let result = prepare(schedule, run, fit, now, store); + let result = prepare(schedule, run, fit, now, store,witness); let (core, failure) = match result { Ok((core, error)) => (Some(core), error), Err(error) => (None, Some(error)), @@ -450,6 +497,12 @@ impl RoomLearningRuntime { context: &CaptureContext, now: u64, ) -> Result<(), String> { + self.restore_temporal_capture_kind(expected,context,now,false) + } + fn restore_temporal_full_run(&self,expected:&MutationGuard,context:&CaptureContext,now:u64)->Result<(),String>{ + self.restore_temporal_capture_kind(expected,context,now,true) + } + fn restore_temporal_capture_kind(&self,expected:&MutationGuard,context:&CaptureContext,now:u64,full_run:bool)->Result<(),String>{ context.validate()?; if context.owner_scope != self.owner || context.installation_id != self.installation { return Err("temporal_context_owner_or_clock".into()); @@ -473,6 +526,7 @@ impl RoomLearningRuntime { .generation .checked_add(1) .ok_or("learner generation exhausted")?; + let full_witness=FullRunRecoveryWitness{runtime_epoch:self.runtime_epoch.clone(),generation:next_generation,context:context.clone()}; state.capture = Some(CaptureWorkflow::Temporal(Box::new( RuntimeTemporalCapture { context: context.clone(), @@ -487,8 +541,18 @@ impl RoomLearningRuntime { ))); state.generation = next_generation; drop(state); - let mut core = - TemporalTrainCapture::cold_restore(store, context, now, &self.runtime_epoch, now)?; + let recovered=if full_run{ + TemporalTrainCapture::cold_restore_full_run(store,self.store.clone(),context,now,&self.runtime_epoch,now,full_witness) + }else{ + TemporalTrainCapture::cold_restore(store, context, now, &self.runtime_epoch, now) + }; + let mut core=match recovered{ + Ok(core)=>core, + Err(error)=>{ + if full_run{let mut state=self.inner.lock().map_err(|_|"learner state unavailable")?;if state.generation==next_generation{if let Some(CaptureWorkflow::Temporal(c))=state.capture.as_mut(){c.failure=Some(bounded(error.clone()));}}} + return Err(error); + } + }; core.enable_async_restore( self.recovery_store .clone() @@ -583,6 +647,9 @@ impl RoomLearningRuntime { #[cfg(test)] #[path = "lifecycle_temporal_tests.rs"] mod tests; +#[cfg(all(test,unix))] +#[path="lifecycle_temporal_full_run_tests.rs"] +mod full_run_tests; impl Drop for RoomLearningRuntime { fn drop(&mut self) { diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit.rs index 14cde7d6b0..12b408283f 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit.rs @@ -5,7 +5,7 @@ use crate::room_learner::{ artifact::{TemporalCandidateReceipt, TemporalFitDiagnostics, TemporalFitSaveResult}, recovery_store::RecoveryStore, temporal_model_input::{ - managed::execution::{PendingTemporalFitInput, TemporalFitCaptureIdentity}, + managed::execution::{PendingTemporalFitInput, TemporalFitCaptureIdentity,VerifiedTemporalFitSource}, TemporalModel, }, }; @@ -24,6 +24,21 @@ struct TemporalFitIdentity { pub(super) struct TemporalDiagnosticFit { model: Arc, saved: TemporalFitSaveResult, + accepted: Option, +} +/// Minted ONLY in the actual current C3 result-admission branch. Not serde, +/// Clone, a candidate-import constructor, or a report/claims permission. +pub(crate) struct AcceptedTemporalFit { + runtime_epoch:String,generation:u64,fit_attempt:u64, + model:Arc,source:VerifiedTemporalFitSource, +} +impl AcceptedTemporalFit { + pub(crate) fn consume_for_owner(self,epoch:&str,generation:u64,binding:&TemporalFitCaptureIdentity)->Result<(Arc,VerifiedTemporalFitSource),String>{ + if self.runtime_epoch!=epoch||self.generation!=generation||self.fit_attempt==0||self.source.binding()!=binding{ + return Err("temporal_accepted_fit_owner_changed".into()); + } + Ok((self.model,self.source)) + } } /// One lifetime barrier for the SAME existing Job/executor, not a new queue or /// trainer. Cancellation is atomic; waiting/publication never holds inner. @@ -162,6 +177,15 @@ impl RuntimeTemporalCapture { job.cancel(); } } + pub(super) fn take_current_accepted_fit(&mut self,epoch:&str,generation:u64,now:u64)->Result{ + if self.owner_generation!=generation||self.failure.is_some(){return Err("temporal_accepted_fit_owner_changed".into());} + let job=self.fit_job.as_ref().ok_or("temporal_actual_current_fit_required")?; + if job.identity.runtime_epoch!=epoch||job.lifetime.cancel.load(Ordering::Acquire)||!job.done(){return Err("temporal_actual_current_fit_required".into());} + let current=self.core.as_ref().ok_or("temporal_initial_barrier_unconfirmed")?.fit_capture_identity(now)?; + let accepted=self.diagnostic_fit.as_ref().and_then(|d|d.accepted.as_ref()).ok_or("temporal_full_run_actual_accepted_fit_required")?; + if accepted.source.binding()!=¤t{return Err("temporal_accepted_fit_capture_changed".into());} + Ok(self.diagnostic_fit.as_mut().unwrap().accepted.take().unwrap()) + } } impl RoomLearningRuntime { /// Private cancellation of the SAME Job, preserving confirmed Train data. @@ -173,6 +197,22 @@ impl RoomLearningRuntime { return Err(PRIVATE_KIND.into()); }; let job = capture.fit_job.clone().ok_or("temporal_fit_job_missing")?; + // Ready is published only after actual fit/store validation. A + // late compute-only cancel cannot revoke that accepted result or + // its separately owned Frozen transaction, even before the final + // completion notification. Capture/owner cancellation still uses + // cancel_fit + drain and revokes the entire owner operation. + if state.job.phase == JobPhase::Ready + && job.identity.runtime_epoch == self.runtime_epoch + && job.identity.generation == state.generation + && Arc::ptr_eq(&state.job.cancel, &job.lifetime.cancel) + && capture.owner_generation == state.generation + && capture.failure.is_none() + && capture.core.as_ref().is_some_and(|c| c.is_full_run()) + && capture.diagnostic_fit.is_some() + { + return Ok(JobPhase::Ready); + } job.cancel(); if busy(&state.job) { state.job.phase = JobPhase::Cancelling; @@ -291,6 +331,8 @@ impl RoomLearningRuntime { lifetime.at("completion_owner_upgraded"); if let Some(runtime) = &completion.owner { runtime.admit_temporal_fit(&identity, &lifetime, result); + #[cfg(test)] + lifetime.at("accepted_ready_published"); } else { drop(result); } @@ -303,10 +345,10 @@ impl RoomLearningRuntime { input: PendingTemporalFitInput, identity: &TemporalFitIdentity, lifetime: &FitLifetime, - ) -> Result<(Arc, TrainingReport, TemporalFitSaveResult), String> { + ) -> Result<(Arc, TrainingReport, TemporalFitSaveResult,Option), String> { lifetime.at("before_validate"); let validated = input.validate(recovery, lifetime.allowed()?)?; - let (mut session, permit) = validated.into_session_and_admission()?; + let (mut session, permit,mut source) = validated.into_session_admission_and_source()?; lifetime.at("before_fit"); lifetime.allowed()?; let configured = session.bound_managed_fit_config()?; @@ -337,18 +379,19 @@ impl RoomLearningRuntime { lifetime.at("after_save"); saved }; + source.bind_saved_fit(&model,&saved)?; identity .capture .recertify_current(recovery, lifetime.allowed()?)?; lifetime.at("before_admission"); lifetime.allowed()?; - Ok((Arc::new(model), report, saved)) + Ok((Arc::new(model), report, saved,Some(source))) } fn admit_temporal_fit( &self, identity: &TemporalFitIdentity, lifetime: &Arc, - outcome: Result<(Arc, TrainingReport, TemporalFitSaveResult), String>, + mut outcome: Result<(Arc, TrainingReport, TemporalFitSaveResult,Option), String>, ) { // Outcome and retired owners precede the guard: even rejection/panic // releases inner before destroying model/evidence/report allocations. @@ -379,8 +422,8 @@ impl RoomLearningRuntime { return; } state.workflow_revision = state.workflow_revision.saturating_add(1); - match &outcome { - Ok((model, report, saved)) => { + match &mut outcome { + Ok((model, report, saved,source)) => { if let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() { retired_diagnostic = capture.diagnostic_fit.take(); capture.diagnostic_fit = Some(TemporalDiagnosticFit { @@ -389,6 +432,10 @@ impl RoomLearningRuntime { candidate: saved.candidate.clone(), fit_diagnostics: saved.fit_diagnostics.clone(), }, + accepted:if source.as_ref().is_some_and(|s|s.is_full_run()){ + Some(AcceptedTemporalFit{runtime_epoch:identity.runtime_epoch.clone(),generation:identity.generation, + fit_attempt:identity.attempt,model:model.clone(),source:source.take().unwrap()}) + }else{None}, }); } retired_report = state.job.report.replace(report.clone()); diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_frozen.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_frozen.rs new file mode 100644 index 0000000000..3148ba0f0f --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_frozen.rs @@ -0,0 +1,71 @@ +//! Private owner transition from actual C3 acceptance to a durable Frozen ACK. +//! No public fit/HeldOut/qualification/selection/Room Save activation. +use super::*; +impl RoomLearningRuntime { + pub(super) fn prepare_frozen_transition( + &self, + expected: &MutationGuard, + now: u64, + ) -> Result<(), String> { + let store = self.store.clone(); + let mut state = self.command_state(expected)?; + if state.job.phase != JobPhase::Ready { + return Err("temporal_actual_current_fit_required".into()); + } + let generation = state.generation; + let revision = state.workflow_revision; + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { + return Err(PRIVATE_KIND.into()); + }; + if capture.owner_generation != generation { + capture.invalidate_owner(); + return Err("temporal_owner_generation_changed".into()); + } + capture.check_time(now)?; + let core = capture + .core + .as_ref() + .ok_or("temporal_initial_barrier_unconfirmed")?; + if !core.is_full_run() { + return Err("temporal_full_run_no_upconversion".into()); + } + if core.stage() != TemporalCaptureStage::Complete { + return Err("temporal_full_run_train_complete_required".into()); + } + let accepted = capture.take_current_accepted_fit(&self.runtime_epoch, generation, now)?; + let run = capture + .core + .as_ref() + .unwrap() + .fit_capture_identity(now)? + .run_id() + .to_owned(); + let witness = FrozenOwnerWitness { + runtime_epoch: self.runtime_epoch.clone(), + generation, + revision, + context: capture.context.clone(), + run, + }; + capture + .core + .as_mut() + .unwrap() + .request_frozen(accepted, witness, store, now) + } + pub(super) fn current_verified_frozen_model( + &self, + ) -> Result>, String> { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_ref() else { + return Err(PRIVATE_KIND.into()); + }; + if capture.owner_generation != state.generation || capture.failure.is_some() { + return Err("temporal_frozen_owner_unavailable".into()); + } + Ok(capture + .core + .as_ref() + .and_then(|c| c.verified_frozen_model())) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs new file mode 100644 index 0000000000..90e27e4835 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs @@ -0,0 +1,938 @@ +//! SYNTHETIC private fresh-v3 genesis/Train-path checks; no physical evidence. +use super::tests as fixture; +use super::*; +use crate::room_learner::{ + native::{canonicalize_esp32, NativePacketInput}, + observation::CaptureProvenance, +}; +use std::path::Path; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use wifi_densepose_hardware::{Adr018Flags, PpduType}; +#[test] +fn c4a_fresh_genesis_is_preparing_until_actual_ack_in_same_store_and_raw_read_cannot_write() { + let directory = tempfile::tempdir().unwrap(); + let rt = fixture::runtime(directory.path(), fixture::START); + rt.configure_temporal_full_run( + &rt.mutation_guard().unwrap(), + fixture::context(), + "synthetic-c4a-fresh".into(), + fixture::START, + ) + .unwrap(); + assert_eq!(fixture::snapshot(&rt, fixture::START)["stage"], "preparing"); + std::thread::sleep(Duration::from_millis(30)); + assert_eq!( + fixture::snapshot(&rt, fixture::START)["stage"], + "preparing", + "queued storage result is not receiver ACK admission" + ); + let now = fixture::settle_async_boundary(&rt, fixture::START); + assert_eq!(fixture::snapshot(&rt, now)["stage"], "ready"); + let store = rt.temporal_store().unwrap(); + let loaded = store.load_temporal(now).unwrap().unwrap(); + let prefix = loaded.prefix.unwrap(); + assert_eq!(prefix.metadata_kind(), "temporal-prefix-commitment-v2"); + assert_eq!(prefix.transition_limit(), 433); + assert_eq!(prefix.window_count(), 0); + let data: serde_json::Value = serde_json::from_slice(prefix.bytes()).unwrap(); + assert_eq!( + data["schema"], + "ruview.room-learner.temporal-runtime-prefix.v3" + ); + assert_eq!(data["payload"]["policy"]["body"]["max_total_windows"], 128); + assert_eq!(data["payload"]["stage"]["kind"], "train"); + assert!(store + .prepare_temporal(Some(&loaded.token), &prefix, now) + .err() + .unwrap() + .contains("producer_write_grant_required")); + assert_eq!( + store.load_temporal(now).unwrap().unwrap().token, + loaded.token + ); + let base = directory + .path() + .join(fixture::context().owner_scope) + .join("SensingServer/room-learner-recovery"); + assert!(base + .join("temporal-prefix-commitment-v1/temporal-head.bin") + .is_file()); + assert!(!base.join("temporal-prefix-commitment-v2").exists()); + if let Some(destination) = std::env::var_os("RUVIEW_C4A_EXPORT_PROFILE") { + use std::os::unix::fs::PermissionsExt; + let destination = std::path::PathBuf::from(destination); + assert!(destination.starts_with("/tmp/ruview-c4a-kind2-current-fixture-20261106")); + assert!(!destination.exists()); + fn copy(from: &std::path::Path, to: &std::path::Path) { + use std::os::unix::fs::PermissionsExt; + assert!(!std::fs::symlink_metadata(from) + .unwrap() + .file_type() + .is_symlink()); + if from.is_dir() { + std::fs::create_dir_all(to).unwrap(); + std::fs::set_permissions(to, std::fs::Permissions::from_mode(0o700)).unwrap(); + for e in std::fs::read_dir(from).unwrap() { + let e = e.unwrap(); + copy(&e.path(), &to.join(e.file_name())); + } + } else { + std::fs::copy(from, to).unwrap(); + std::fs::set_permissions(to, std::fs::Permissions::from_mode(0o600)).unwrap(); + } + } + copy(directory.path(), &destination); + let marker = destination.join("synthetic-c4a-fixture.json"); + std::fs::write(&marker,serde_json::to_vec(&serde_json::json!({"schema":"ruview.test-only.synthetic-c4a-kind2-fixture.v1","receiver_now_ns":now,"real_user_data":false})).unwrap()).unwrap(); + std::fs::set_permissions(marker, std::fs::Permissions::from_mode(0o600)).unwrap(); + } + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()) + .unwrap(); +} +#[test] +fn c4a_new_full_run_reuses_actual_native_train_writer_without_changing_model_input_identity() { + let directory = tempfile::tempdir().unwrap(); + let rt = fixture::runtime(directory.path(), fixture::START); + rt.configure_temporal_full_run( + &rt.mutation_guard().unwrap(), + fixture::context(), + "synthetic-c4a-window".into(), + fixture::START, + ) + .unwrap(); + let mut now = fixture::settle_async_boundary(&rt, fixture::START); + let original = fixture::snapshot(&rt, now)["descriptor"].clone(); + now = fixture::async_lead_end(&rt, now); + let mut sequence = 0; + let end = now + 3_000_000_000; + while now <= end { + sequence += 1; + for source in 1..=3 { + rt.ingest_native(&fixture::packet(source, sequence, now), now) + .unwrap(); + } + now += fixture::PERIOD; + if fixture::snapshot(&rt, now)["durability_pending"] == true { + std::thread::sleep(Duration::from_nanos(fixture::PERIOD)); + } else { + std::thread::sleep(Duration::from_micros(500)); + } + } + let pending = fixture::snapshot(&rt, now)["durability_pending"] == true; + if pending { + now = fixture::settle_async_boundary(&rt, now); + } + assert!( + fixture::snapshot(&rt, now)["durable_windows"] + .as_u64() + .unwrap() + >= 1 + ); + let loaded = rt + .temporal_store() + .unwrap() + .load_temporal(now) + .unwrap() + .unwrap(); + let prefix = loaded.prefix.unwrap(); + assert_eq!(prefix.metadata_kind(), "temporal-prefix-commitment-v2"); + assert_eq!(prefix.transition_limit(), 433); + assert_eq!( + fixture::snapshot(&rt, now)["descriptor"]["id"], + original["id"] + ); + assert_eq!( + fixture::snapshot(&rt, now)["descriptor"]["expires_at_ms"], + original["expires_at_ms"] + ); + let data: serde_json::Value = serde_json::from_slice(prefix.bytes()).unwrap(); + let policy = &data["payload"]["policy"]["body"]; + assert_ne!( + policy["input_policy_sha256"], data["payload"]["policy"]["sha256"], + "continuation identity must not replace model input identity" + ); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()) + .unwrap(); +} +#[test] +fn c4a_full_run_rejects_unsupported_source_profile_before_genesis_write() { + let directory = tempfile::tempdir().unwrap(); + let rt = fixture::runtime(directory.path(), fixture::START); + let mut context = fixture::context(); + context.sources.pop(); + assert!(rt + .configure_temporal_full_run( + &rt.mutation_guard().unwrap(), + context, + "synthetic-c4a-unsupported".into(), + fixture::START + ) + .is_err()); + let path = directory.path().join(fixture::context().owner_scope).join( + "SensingServer/room-learner-recovery/temporal-prefix-commitment-v1/temporal-head.bin", + ); + assert!(!path.exists()); +} + +fn configured(root: &Path) -> (Arc, u64) { + let runtime = Arc::new(fixture::runtime(root, fixture::START)); + let store = runtime.recovery_store.clone().unwrap(); + runtime + .configure_temporal_capture_with_custody( + &runtime.mutation_guard().unwrap(), + fixture::context(), + "synthetic-c4a-native-fit".into(), + fixture::START, + move |schedule, run, mut config, now, _, witness| { + // Bound when this capture is CREATED, never overridden by fit input. + config.epochs = 2; + TemporalTrainCapture::prepare_full_run_async( + schedule, run, config, now, store, witness, + ) + }, + ) + .unwrap(); + let now = fixture::settle_async_boundary(&runtime, fixture::START); + (runtime, now) +} +fn packet(source: u8, sequence: u32, at: u64, label: RoomLabel) -> CanonicalPacket { + let gain = match label { + RoomLabel::Empty => 3, + RoomLabel::AdjacentOnly => 4, + RoomLabel::OccupiedStatic => 8, + RoomLabel::OccupiedMoving => 10 + (sequence % 2) as u8, + _ => unreachable!(), + }; + let iq = (0..64) + .flat_map(|bin| [gain + source + (bin % 2) as u8, 4]) + .collect::>(); + canonicalize_esp32(NativePacketInput { + node_id: source, + n_antennas: 1, + raw_bins: 64, + frequency_hz: 2_437_000_000, + ppdu_type: PpduType::HtLegacy, + sequence, + received_at_ns: at, + receiver_sample_rate_hz: 33., + iq_bytes: &iq, + flags: Adr018Flags::default(), + provenance: CaptureProvenance::LiveReceiverObserved, + }) + .unwrap() +} +fn record_train_uncached(root: &Path) -> (Arc, u64) { + let (runtime, mut now) = configured(root); + let schedule = ManagedScheduleContract::build( + fixture::context(), + LeadInPolicy::Train5HeldOut15EngineeringV1, + ) + .unwrap(); + let mut sequence = 0; + let mut starts = 0; + for episode in schedule + .episodes() + .iter() + .filter(|e| e.role == DataRole::Train) + { + loop { + let progress = fixture::snapshot(&runtime, now); + match progress["stage"].as_str().unwrap() { + "ready" | "paused" => { + now = fixture::async_lead_end(&runtime, now); + starts += 1; + runtime + .tick_temporal_capture(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + } + "needs_confirmation" => break, + "failed" => panic!("synthetic C4A phase{} failed: {progress}", episode.index), + _ => {} + } + let pending = fixture::snapshot(&runtime, now)["durability_pending"] == true; + sequence += 1; + for source in 1..=3 { + match runtime.ingest_native(&packet(source, sequence, now, episode.condition), now) + { + Ok(()) => {} + Err(error) if error == "temporal_storage_busy_pause" => break, + Err(error) => panic!("synthetic C4A ingest: {error}"), + } + } + now += fixture::PERIOD; + if pending { + std::thread::sleep(Duration::from_nanos(fixture::PERIOD)); + } + assert!(starts < 120, "synthetic C4A interruption bound exhausted"); + } + let guard = runtime.mutation_guard().unwrap(); + let challenge = runtime.temporal_confirmation_challenge(&guard).unwrap(); + runtime + .confirm_temporal_phase(&guard, &challenge, now) + .unwrap(); + assert_eq!(fixture::snapshot(&runtime, now)["durability_pending"], true); + now = fixture::settle_async_boundary(&runtime, now); + } + let complete = fixture::snapshot(&runtime, now); + assert_eq!(complete["stage"], "complete"); + assert_eq!( + complete["sealed_train_episodes"], + schedule.counts().train_episodes + ); + assert_eq!(complete["durable_windows"], schedule.counts().train_windows); + println!( + "SYNTHETIC_C4A_REAL_TRAIN_CAPTURE: rows={} seals={} starts={starts}", + schedule.counts().train_windows, + schedule.counts().train_episodes + ); + (runtime, now) +} + +fn copy_private(from: &Path, to: &Path) { + use std::os::unix::fs::PermissionsExt; + assert!(!std::fs::symlink_metadata(from) + .unwrap() + .file_type() + .is_symlink()); + if from.is_dir() { + std::fs::create_dir_all(to).unwrap(); + std::fs::set_permissions(to, std::fs::Permissions::from_mode(0o700)).unwrap(); + for e in std::fs::read_dir(from).unwrap() { + let e = e.unwrap(); + copy_private(&e.path(), &to.join(e.file_name())); + } + } else { + std::fs::copy(from, to).unwrap(); + std::fs::set_permissions(to, std::fs::Permissions::from_mode(0o600)).unwrap(); + } +} +// Cache only the actual seven-phase native producer output. Each independent +// fault case gets its own files and strictly current recertification/owner. +fn record_train(root: &Path) -> (Arc, u64) { + static TRAIN: std::sync::OnceLock<(tempfile::TempDir, u64)> = std::sync::OnceLock::new(); + let (saved, at) = TRAIN.get_or_init(|| { + let d = tempfile::tempdir().unwrap(); + let (rt, at) = record_train_uncached(d.path()); + drop(rt); + (d, at) + }); + copy_private(saved.path(), root); + let now = at + 1_000_000_000; + let rt = Arc::new(fixture::runtime(root, now)); + rt.restore_temporal_full_run(&rt.mutation_guard().unwrap(), &fixture::context(), now) + .unwrap(); + assert_eq!(fixture::snapshot(&rt, now)["stage"], "complete"); + (rt, now) +} +fn assert_original_lifetime(value: &serde_json::Value, original: &serde_json::Value) { + for field in [ + "schema", + "id", + "created_at_ms", + "expires_at_ms", + "owner_scope", + "installation_id", + "context_digest", + ] { + assert_eq!(value[field], original[field], "{field}"); + } +} +struct ReleaseGate(Option>); +impl ReleaseGate { + fn release(&mut self) { + if let Some(tx) = self.0.take() { + let _ = tx.send(()); + } + } +} +impl Drop for ReleaseGate { + fn drop(&mut self) { + self.release(); + } +} +fn floors(rt: &RoomLearningRuntime) -> (Vec<(u8, u64, u64)>, Option) { + let s = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &s.capture else { + panic!() + }; + c.core.as_ref().unwrap().test_full_run_floors() +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn c4a_actual_seven_train_fit_frozen_ack_and_exact_cold_model() { + let directory = tempfile::tempdir().unwrap(); + let (runtime, now) = record_train(directory.path()); + let original = fixture::snapshot(&runtime, now)["descriptor"].clone(); + let train_rows_sha = fixture::snapshot(&runtime, now)["rows_sha256"].clone(); + let loaded = runtime + .temporal_store() + .unwrap() + .load_temporal(now) + .unwrap() + .unwrap(); + assert_eq!( + loaded.token.data().revision, + 78, + "actual genesis +7 segments +64 windows +7 seals" + ); + let (compute_tx, compute_rx) = std::sync::mpsc::sync_channel(1); + let (compute_release_tx, compute_release_rx) = std::sync::mpsc::sync_channel(1); + let mut compute_gate = ReleaseGate(Some(compute_release_tx)); + let compute_release_rx = std::sync::Mutex::new(compute_release_rx); + let (ready_tx, ready_rx) = std::sync::mpsc::sync_channel(1); + let (done_tx, done_rx) = std::sync::mpsc::sync_channel(1); + let mut done_gate = ReleaseGate(Some(done_tx)); + let done_rx = std::sync::Mutex::new(done_rx); + *runtime.temporal_fit_hook.lock().unwrap() = Some(Arc::new(move |stage| { + if stage == "before_validate" { + compute_tx.send(()).unwrap(); + compute_release_rx + .lock() + .unwrap() + .recv_timeout(Duration::from_secs(30)) + .unwrap(); + } + if stage == "accepted_ready_published" { + ready_tx.send(()).unwrap(); + done_rx + .lock() + .unwrap() + .recv_timeout(Duration::from_secs(30)) + .unwrap(); + } + })); + let fit_started = Instant::now(); + runtime + .start_temporal_fit(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + compute_rx.recv_timeout(Duration::from_secs(10)).unwrap(); + assert_eq!(runtime.inner.lock().unwrap().job.phase, JobPhase::Running); + let during_fit = now + fit_started.elapsed().as_nanos() as u64; + for source in 1..=3 { + runtime + .ingest_native( + &packet(source, 500_000, during_fit, RoomLabel::Empty), + during_fit, + ) + .unwrap(); + } + assert_eq!( + fixture::snapshot(&runtime, during_fit)["rows_sha256"], + train_rows_sha + ); + compute_gate.release(); + loop { + let phase = runtime.inner.lock().unwrap().job.phase; + if phase == JobPhase::Ready { + break; + } + assert_eq!( + phase, + JobPhase::Running, + "{:?}", + runtime.inner.lock().unwrap().job.error + ); + assert!(fit_started.elapsed() < Duration::from_secs(120)); + tokio::time::sleep(Duration::from_millis(5)).await; + } + ready_rx.recv_timeout(Duration::from_secs(10)).unwrap(); + let (model, receipt, _) = runtime.temporal_diagnostic_fit().unwrap().unwrap(); + assert_eq!( + runtime + .cancel_temporal_fit(&runtime.mutation_guard().unwrap()) + .unwrap(), + JobPhase::Ready, + "late compute cancellation cannot revoke actual Accepted Ready" + ); + assert_eq!( + runtime + .inner + .lock() + .unwrap() + .job + .report + .as_ref() + .unwrap() + .completed_epochs, + 2 + ); + assert!(runtime.current_verified_frozen_model().unwrap().is_none()); + let before_notify = now + fit_started.elapsed().as_nanos() as u64; + assert_eq!( + runtime + .prepare_frozen_transition(&runtime.mutation_guard().unwrap(), before_notify) + .unwrap_err(), + "temporal_actual_current_fit_required" + ); + done_gate.release(); + // This real native ingress updates the live receiver floor without adding + // Train rows or modifying the exact accepted Train commitment. + let prime_at = now + fit_started.elapsed().as_nanos() as u64; + for source in 1..=3 { + runtime + .ingest_native( + &packet(source, 1_000_000, prime_at, RoomLabel::Empty), + prime_at, + ) + .unwrap(); + } + let prepare_floors = floors(&runtime).0; + let orphan = tempfile::tempdir().unwrap(); + copy_private(directory.path(), orphan.path()); + let writer = { + let state = runtime.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!() + }; + c.core.as_ref().unwrap().writer().unwrap() + }; + let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1); + let (release_tx, release_rx) = std::sync::mpsc::sync_channel(1); + let mut writer_gate = ReleaseGate(Some(release_tx)); + writer.hold_next_after_prepare(entered_tx, release_rx); + // Completion publication and drain signal are separate. Only the latter + // permits transfer of the actual non-serde accepted-fit capability. + let mut freeze_now; + loop { + freeze_now = now + u64::try_from(fit_started.elapsed().as_nanos()).unwrap(); + match runtime.prepare_frozen_transition(&runtime.mutation_guard().unwrap(), freeze_now) { + Ok(()) => break, + Err(e) if e == "temporal_actual_current_fit_required" => { + tokio::time::sleep(Duration::from_millis(1)).await + } + Err(e) => panic!("actual Frozen preparation failed: {e}"), + } + } + entered_rx.recv_timeout(Duration::from_secs(10)).unwrap(); + assert!( + runtime + .start_training(&runtime.mutation_guard().unwrap(), FitConfig::default()) + .unwrap_err() + .contains("unsupported_command"), + "a rejected command must not replace the accepted Frozen operation" + ); + assert_eq!( + fixture::snapshot(&runtime, freeze_now)["stage"], + "preparing" + ); + assert!(runtime.current_verified_frozen_model().unwrap().is_none()); + let advanced = freeze_now + fixture::PERIOD; + for source in 1..=3 { + runtime + .ingest_native( + &packet(source, 2_000_000, advanced, RoomLabel::Empty), + advanced, + ) + .unwrap(); + } + let live_floors = floors(&runtime).0; + assert!(live_floors + .iter() + .all(|(_, seq, t)| *seq == 2_000_000 && *t == advanced)); + writer_gate.release(); + let frozen_now = fixture::settle_async_boundary(&runtime, advanced); + assert_eq!(fixture::snapshot(&runtime, frozen_now)["stage"], "frozen"); + assert_eq!( + runtime + .current_verified_frozen_model() + .unwrap() + .unwrap() + .export() + .unwrap(), + model.export().unwrap() + ); + let loaded = runtime + .temporal_store() + .unwrap() + .load_temporal(frozen_now) + .unwrap() + .unwrap(); + assert_eq!( + loaded.token.data().revision, + 79, + "one actual durable Frozen transition" + ); + let data: serde_json::Value = serde_json::from_slice(loaded.prefix.unwrap().bytes()).unwrap(); + assert_eq!(data["payload"]["stage"]["kind"], "frozen"); + assert_eq!( + data["payload"]["stage"]["fit"]["body"]["artifact_sha256"], + receipt.artifact_sha256 + ); + assert_original_lifetime( + &fixture::snapshot(&runtime, frozen_now)["descriptor"], + &original, + ); + assert_eq!( + data["payload"]["stage"]["fit"]["body"]["freeze_prepare_receiver_ns"], + freeze_now + ); + assert_eq!( + fixture::snapshot(&runtime, frozen_now)["descriptor"]["saved_at_ms"], + freeze_now / 1_000_000 + ); + assert_eq!( + data["payload"]["stage"]["fit"]["body"]["source_floors"], + serde_json::to_value(&prepare_floors).unwrap() + ); + assert_eq!(floors(&runtime), (live_floors, Some(frozen_now))); + let loaded = runtime + .temporal_store() + .unwrap() + .load_temporal(frozen_now) + .unwrap() + .unwrap(); + assert!(runtime + .temporal_store() + .unwrap() + .prepare_temporal( + Some(&loaded.token), + loaded.prefix.as_ref().unwrap(), + frozen_now + ) + .err() + .unwrap() + .contains("producer_write_grant_required")); + + assert!(runtime.inner.lock().unwrap().model.is_none()); + assert!(runtime.inner.lock().unwrap().qualification.is_none()); + let post_ack = frozen_now + fixture::PERIOD; + for source in 1..=3 { + runtime + .ingest_native( + &packet(source, 3_000_000, post_ack, RoomLabel::Empty), + post_ack, + ) + .unwrap(); + } + assert!(floors(&runtime) + .0 + .iter() + .all(|(_, seq, t)| *seq == 3_000_000 && *t == post_ack)); + assert_eq!(floors(&runtime).1, Some(frozen_now)); + assert_eq!( + fixture::snapshot(&runtime, post_ack)["rows_sha256"], + train_rows_sha + ); + assert_eq!(fixture::snapshot(&runtime, post_ack)["retained_rows"], 64); + assert_eq!( + fixture::snapshot(&runtime, post_ack)["sealed_train_episodes"], + 7 + ); + assert_eq!( + runtime + .current_verified_frozen_model() + .unwrap() + .unwrap() + .export() + .unwrap(), + model.export().unwrap() + ); + assert_eq!( + runtime + .temporal_store() + .unwrap() + .load_temporal(post_ack) + .unwrap() + .unwrap() + .token + .data() + .revision, + 79 + ); + drop(runtime); + let cold_now = post_ack + 1_000_000_000; + let cold = fixture::runtime(directory.path(), cold_now); + cold.restore_temporal_full_run( + &cold.mutation_guard().unwrap(), + &fixture::context(), + cold_now, + ) + .unwrap(); + assert_eq!(fixture::snapshot(&cold, cold_now)["stage"], "frozen"); + assert_eq!( + floors(&cold), + (prepare_floors, Some(cold_now)), + "cold recertification retains only fsynced floors and a fresh local floor" + ); + assert_eq!( + cold.current_verified_frozen_model() + .unwrap() + .unwrap() + .export() + .unwrap(), + model.export().unwrap() + ); + assert_original_lifetime(&fixture::snapshot(&cold, cold_now)["descriptor"], &original); + assert!( + cold.start_temporal_phase(&cold.mutation_guard().unwrap(), cold_now) + .is_err(), + "private Frozen must not activate HeldOut" + ); + cold.cancel_temporal_durability(&cold.mutation_guard().unwrap()) + .unwrap(); + let orphan_rt = fixture::runtime(orphan.path(), cold_now); + orphan_rt + .restore_temporal_full_run( + &orphan_rt.mutation_guard().unwrap(), + &fixture::context(), + cold_now, + ) + .unwrap(); + assert_eq!(fixture::snapshot(&orphan_rt, cold_now)["stage"], "complete"); + assert!( + orphan_rt.current_verified_frozen_model().unwrap().is_none(), + "saved model before Frozen ACK is only a diagnostic orphan" + ); + assert!(orphan_rt + .store + .load_managed_temporal_candidate(&fixture::context()) + .unwrap() + .is_some()); + orphan_rt + .cancel_temporal_durability(&orphan_rt.mutation_guard().unwrap()) + .unwrap(); +} + +async fn fit_ready(rt: &Arc, now: u64) -> u64 { + let start = Instant::now(); + rt.start_temporal_fit(&rt.mutation_guard().unwrap(), now) + .unwrap(); + loop { + let phase = rt.inner.lock().unwrap().job.phase; + if phase == JobPhase::Ready { + break; + } + assert_eq!( + phase, + JobPhase::Running, + "{:?}", + rt.inner.lock().unwrap().job.error + ); + assert!(start.elapsed() < Duration::from_secs(120)); + tokio::time::sleep(Duration::from_millis(5)).await; + } + now + start.elapsed().as_nanos() as u64 +} +async fn prepare_frozen(rt: &Arc, at: u64) -> u64 { + let start = Instant::now(); + loop { + let now = at + start.elapsed().as_nanos() as u64; + match rt.prepare_frozen_transition(&rt.mutation_guard().unwrap(), now) { + Ok(()) => return now, + Err(e) if e == "temporal_actual_current_fit_required" => { + assert!(start.elapsed() < Duration::from_secs(10)); + tokio::time::sleep(Duration::from_millis(1)).await; + } + Err(e) => panic!("Frozen prepare {e}"), + } + } +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn c4a_actual_frozen_cold_missing_corrupt_wrong_kind_bytes_and_expiry_fail_closed() { + let directory = tempfile::tempdir().unwrap(); + let (rt, now) = record_train(directory.path()); + let at = fit_ready(&rt, now).await; + let (_, receipt, _) = rt.temporal_diagnostic_fit().unwrap().unwrap(); + let at = prepare_frozen(&rt, at).await; + let at = fixture::settle_async_boundary(&rt, at); + assert_eq!(fixture::snapshot(&rt, at)["stage"], "frozen"); + let descriptor = fixture::snapshot(&rt, at)["descriptor"].clone(); + drop(rt); + for corruption in ["missing", "corrupt", "wrong_kind_bytes", "head", "expired"] { + let copy = tempfile::tempdir().unwrap(); + copy_private(directory.path(), copy.path()); + let path = copy + .path() + .join(fixture::context().owner_scope) + .join("SensingServer/room-learner/temporal-models") + .join(fixture::context().digest()) + .join(format!("{}.json", receipt.artifact_sha256)); + match corruption{ + "missing"=>std::fs::remove_file(&path).unwrap(), + "corrupt"=>std::fs::write(&path,b"corrupt synthetic exact model").unwrap(), + "wrong_kind_bytes"=>std::fs::write(&path,b"{\"schema\":\"ruview.room-learner.temporal-input.v2\"}").unwrap(), + "head"=>std::fs::write(copy.path().join(fixture::context().owner_scope).join("SensingServer/room-learner-recovery/temporal-prefix-commitment-v1/temporal-head.bin"),b"corrupt synthetic current head").unwrap(), + _=>{}, + } + let now = if corruption == "expired" { + descriptor["expires_at_ms"].as_u64().unwrap() * 1_000_000 + } else { + at + 1_000_000_000 + }; + let cold = fixture::runtime(copy.path(), now); + assert!( + cold.restore_temporal_full_run( + &cold.mutation_guard().unwrap(), + &fixture::context(), + now + ) + .is_err(), + "{corruption}" + ); + assert!( + cold.current_verified_frozen_model().is_err(), + "{corruption}" + ); + assert!(cold.inner.lock().unwrap().model.is_none()); + assert!(!cold.inner.lock().unwrap().selection.has_history()); + // Failed recovery deliberately leaves a blocked placeholder with no + // core/writer; dropping it must not invent a cancellation target. + assert!(cold + .configure_temporal_capture( + &cold.mutation_guard().unwrap(), + fixture::context(), + "forbidden-fallback".into(), + now + ) + .is_err()); + drop(cold); + } +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn c4a_owner_cancel_drains_actual_prepared_frozen_writer_before_synthetic_erase() { + let directory = tempfile::tempdir().unwrap(); + let (rt, now) = record_train(directory.path()); + let at = fit_ready(&rt, now).await; + let writer = { + let s = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &s.capture else { + panic!() + }; + c.core.as_ref().unwrap().writer().unwrap() + }; + let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1); + let (release_tx, release_rx) = std::sync::mpsc::sync_channel(1); + let mut gate = ReleaseGate(Some(release_tx)); + writer.hold_next_after_prepare(entered_tx, release_rx); + let at = prepare_frozen(&rt, at).await; + entered_rx.recv_timeout(Duration::from_secs(10)).unwrap(); + assert!(rt.current_verified_frozen_model().unwrap().is_none()); + let (worker_tx, worker_rx) = std::sync::mpsc::sync_channel(1); + let worker = rt.clone(); + let guard = rt.mutation_guard().unwrap(); + let cancel = std::thread::spawn(move || { + worker_tx + .send(worker.cancel_temporal_durability(&guard)) + .unwrap() + }); + assert!( + worker_rx.recv_timeout(Duration::from_millis(50)).is_err(), + "owner cancellation acknowledged an active actual prepared Frozen write" + ); + assert!(rt.inner.try_lock().is_ok()); + gate.release(); + worker_rx + .recv_timeout(Duration::from_secs(10)) + .unwrap() + .unwrap(); + cancel.join().unwrap(); + assert!(rt + .current_verified_frozen_model() + .map_or(true, |m| m.is_none())); + let owner = directory.path().join(fixture::context().owner_scope); + std::fs::remove_dir_all(&owner).unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(!owner.exists()); + drop(rt); + drop(writer); + assert!(!owner.exists()); + println!("SYNTHETIC_C4A_OWNER_DRAIN_PREPARED_FROZEN_AT={at}"); +} +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn c4a_actual_frozen_lost_ack_same_intent_repair_installs_only_after_exact_ack() { + let directory = tempfile::tempdir().unwrap(); + let (rt, now) = record_train(directory.path()); + let at = fit_ready(&rt, now).await; + let writer = { + let s = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &s.capture else { + panic!() + }; + c.core.as_ref().unwrap().writer().unwrap() + }; + writer.lose_next_ack(); + let at = prepare_frozen(&rt, at).await; + let started = Instant::now(); + let mut now; + loop { + now = at + started.elapsed().as_nanos() as u64; + let _ = rt.tick_temporal_capture(&rt.mutation_guard().unwrap(), now); + if fixture::snapshot(&rt, now)["stage"] == "blocked" { + break; + } + assert!(started.elapsed() < Duration::from_secs(30)); + tokio::time::sleep(Duration::from_millis(5)).await; + } + assert!(rt.current_verified_frozen_model().unwrap().is_none()); + let token = rt + .temporal_store() + .unwrap() + .load_temporal(now) + .unwrap() + .unwrap() + .token; + assert_eq!(token.data().revision, 79); + rt.repair_temporal_capture(&rt.mutation_guard().unwrap(), now) + .unwrap(); + now = fixture::settle_async_boundary(&rt, now); + assert_eq!(fixture::snapshot(&rt, now)["stage"], "frozen"); + assert!(rt.current_verified_frozen_model().unwrap().is_some()); + assert_eq!( + rt.temporal_store() + .unwrap() + .load_temporal(now) + .unwrap() + .unwrap() + .token, + token, + "SAME-intent repair must not allocate another transition" + ); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()) + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn c4a_running_compute_cancel_cannot_produce_accepted_fit_or_frozen() { + let directory = tempfile::tempdir().unwrap(); + let (rt, now) = record_train(directory.path()); + let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1); + let (release_tx, release_rx) = std::sync::mpsc::sync_channel(1); + let mut gate = ReleaseGate(Some(release_tx)); + let release = std::sync::Mutex::new(release_rx); + *rt.temporal_fit_hook.lock().unwrap() = Some(Arc::new(move |stage| { + if stage == "before_validate" { + entered_tx.send(()).unwrap(); + release + .lock() + .unwrap() + .recv_timeout(Duration::from_secs(30)) + .unwrap(); + } + })); + rt.start_temporal_fit(&rt.mutation_guard().unwrap(), now) + .unwrap(); + entered_rx.recv_timeout(Duration::from_secs(10)).unwrap(); + let (done_tx, done_rx) = std::sync::mpsc::sync_channel(1); + let worker = rt.clone(); + let guard = rt.mutation_guard().unwrap(); + let cancel = + std::thread::spawn(move || done_tx.send(worker.cancel_temporal_fit(&guard)).unwrap()); + assert!(done_rx.recv_timeout(Duration::from_millis(50)).is_err()); + assert!(rt.inner.try_lock().is_ok()); + gate.release(); + assert_eq!( + done_rx + .recv_timeout(Duration::from_secs(10)) + .unwrap() + .unwrap(), + JobPhase::Cancelled + ); + cancel.join().unwrap(); + assert!(rt.temporal_diagnostic_fit().unwrap().is_none()); + assert!(rt + .prepare_frozen_transition(&rt.mutation_guard().unwrap(), now + 1_000_000_000) + .is_err()); + assert!(rt.current_verified_frozen_model().unwrap().is_none()); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()) + .unwrap(); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs index c2ecdb4f0d..0ea1589a20 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs @@ -63,7 +63,7 @@ fn configured(root: &Path) -> RoomLearningRuntime { .unwrap(); rt } -fn packet(source: u8, seq: u32, at: u64) -> CanonicalPacket { +pub(super) fn packet(source: u8, seq: u32, at: u64) -> CanonicalPacket { let iq = vec![8; 128]; canonicalize_esp32(NativePacketInput { node_id: source, diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs index bf4c300a9e..7acbf70465 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs @@ -413,6 +413,7 @@ pub(crate) enum TemporalCaptureStage { Observing, NeedsConfirmation, Complete, + Frozen, Paused, Blocked, Failed, @@ -425,6 +426,7 @@ impl TemporalCaptureStage { Self::Observing => "observing", Self::NeedsConfirmation => "needs_confirmation", Self::Complete => "complete", + Self::Frozen => "frozen", Self::Paused => "paused", Self::Blocked => "blocked", Self::Failed => "failed", @@ -440,6 +442,32 @@ pub(crate) struct TemporalTrainCapture { durable_seals: usize, } impl TemporalTrainCapture { + #[cfg(test)] + pub(crate) fn test_full_run_floors(&self)->(Vec<(u8,u64,u64)>,Option){ + (self.workflow.source_high.clone(),self.workflow.full_run.as_ref().and_then(|m|m.local_start_not_before_ns)) + } + pub(crate) fn is_full_run(&self)->bool{self.workflow.full_run.is_some()} + pub(crate) fn verified_frozen_model(&self)->Option>{if self.failure.is_some(){return None;}self.workflow.full_run.as_ref().and_then(|m|m.verified_model())} + pub(crate) fn request_frozen(&mut self,accepted:crate::room_learner::lifecycle::AcceptedTemporalFit,witness:crate::room_learner::lifecycle::FrozenOwnerWitness,store:std::sync::Arc,now:u64)->Result<(),String>{ + self.ensure_live()?; + if self.stage()!=TemporalCaptureStage::Complete{return Err("temporal_full_run_train_complete_required".into());} + self.workflow.request_frozen(accepted,witness,store,now) + } + pub(crate) fn prepare_full_run_async( + schedule:ManagedScheduleContract,run:String,config:FitConfig,now:u64, + store:std::sync::Arc,witness:crate::room_learner::lifecycle::FullRunGenesisWitness, + )->Result<(Self,Option),String>{ + // Fail unsupported full-run profiles before creating a writer/genesis. + full_run::ContinuationPolicy::build(&ManagedBinding::build(&schedule,config.clone())?)?; + let owned_store=store.clone(); + Self::prepare_with(schedule,run,config,now,&store,move|w,store|{ + let binding=&w.session.managed.as_ref().ok_or("managed_session_required")?.binding; + w.full_run=Some(full_run::FullRunState::fresh(binding,&w.session.run_id,witness)?); + let writer=std::sync::Arc::new(durability::Writer::new(owned_store,now)?); + w.async_state=Some(durability::AsyncState::new(writer)); + w.persist_initial(store) + }) + } pub(crate) fn prepare_fit_input(&self, now: u64) -> Result { self.ensure_live()?; if self.stage() != TemporalCaptureStage::Complete { @@ -571,10 +599,16 @@ impl TemporalTrainCapture { } pub(crate) fn prime_fence(&mut self, p: &CanonicalPacket, now: u64) -> Result<(), String> { if self.workflow.async_state.is_none(){self.ensure_live()?;}else if self.failure.is_some(){return Err("temporal_capture_failed".into());} + // A full-run owner keeps listening between Train completion and the + // future HeldOut stage. This admits only the existing replay/clock + // fence update below: never another row, seal, model or role credit. + let full_run_continuation_fence = self.is_full_run() + && matches!(self.stage(), TemporalCaptureStage::Complete | TemporalCaptureStage::Frozen); if !matches!( self.stage(), TemporalCaptureStage::Ready | TemporalCaptureStage::Paused | TemporalCaptureStage::Preparing | TemporalCaptureStage::Blocked - ) && !(self.workflow.async_state.is_some() && self.logical_phase_complete()) { + ) && !(self.workflow.async_state.is_some() && self.logical_phase_complete()) + && !full_run_continuation_fence { return Err("temporal_priming_state".into()); } let result: Result<(), String> = (|| { @@ -620,6 +654,7 @@ impl TemporalTrainCapture { if w.stage == Stage::BlockedDurability { return TemporalCaptureStage::Blocked; } + if w.full_run.as_ref().is_some_and(|m|m.is_frozen()){return TemporalCaptureStage::Frozen;} if w.bridge .as_ref() .is_some_and(|b| b.disposition == Disposition::Complete) @@ -911,6 +946,7 @@ impl TemporalTrainCapture { anchor: u64, ) -> Result { let w = TemporalWorkflow::cold_restore(store, context, now, epoch, anchor)?; + if w.full_run.is_some(){return Err("temporal_full_run_restore_requires_exact_model_consumer".into());} let b = w .bridge .as_ref() @@ -924,6 +960,14 @@ impl TemporalTrainCapture { durable_seals: n, }) } + pub(crate) fn cold_restore_full_run( + store:&RecoveryStore,artifact:std::sync::Arc,context:&CaptureContext, + now:u64,epoch:&str,anchor:u64,witness:crate::room_learner::lifecycle::FullRunRecoveryWitness, + )->Result{ + let w=full_run::recover_current(store,artifact,context,now,epoch,anchor,witness)?; + let n=w.bridge.as_ref().ok_or("temporal_bridge_checkpoint_required")?.seals.len();let schedule=w.schedule()?; + Ok(Self{workflow:w,schedule,failure:None,durable_seals:n}) + } pub(crate) fn repair_pending(&mut self, store: &RecoveryStore, now: u64) -> Result<(), String> { if self.workflow.async_state.is_some(){return self.workflow.request_async_repair(now);} self.workflow.repair_pending(store, now)?; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs index 17d8f6edc4..a61c6f67b7 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs @@ -32,14 +32,26 @@ struct Envelope

{ } const SCHEMA: &str = "ruview.room-learner.temporal-runtime-prefix.v1"; const BRIDGE_SCHEMA: &str = "ruview.room-learner.temporal-runtime-prefix.v2"; -pub(super) type Snapshot = Prefix, SharedRecord>; +pub(super) type OldSnapshot = Prefix, SharedRecord>; +#[path="temporal_full_run_checkpoint.rs"] +mod full_checkpoint; +pub(super) enum Snapshot { + Old(OldSnapshot), + Full(full_checkpoint::FullSnapshot), +} #[derive(Clone)] pub(super) struct SharedRecords { rows: Vec>, evidence: Vec>, } pub(super) fn snapshot_records(snapshot:&Snapshot)->SharedRecords { - SharedRecords{rows:snapshot.rows.clone(),evidence:snapshot.evidence.clone()} + match snapshot { + Snapshot::Old(snapshot)=>SharedRecords{rows:snapshot.rows.clone(),evidence:snapshot.evidence.clone()}, + Snapshot::Full(snapshot)=>snapshot.records(), + } +} +pub(super) fn snapshot_full_permit(snapshot:&mut Snapshot)->Option{ + match snapshot{Snapshot::Old(_)=>None,Snapshot::Full(snapshot)=>snapshot.permit.take()} } pub(super) fn workflow_records(w:&TemporalWorkflow)->SharedRecords { SharedRecords{rows:w.session.rows.clone(),evidence:w.session.evidence.clone()} @@ -75,6 +87,10 @@ pub(super) fn decode_shared( Ok(validated) } pub(super) fn snapshot(v: &TemporalWorkflow) -> Result { + if v.full_run.is_some(){return Ok(Snapshot::Full(full_checkpoint::FullSnapshot::from_workflow(v)?));} + Ok(Snapshot::Old(snapshot_old(v)?)) +} +fn snapshot_old(v: &TemporalWorkflow) -> Result { if v.session.fitted || v.session.failed || v.session.active @@ -115,6 +131,12 @@ pub(super) fn snapshot(v: &TemporalWorkflow) -> Result { Ok(payload) } pub(super) fn encode_snapshot(payload: Snapshot) -> Result, String> { + match payload { + Snapshot::Old(payload)=>encode_old_snapshot(payload), + Snapshot::Full(payload)=>payload.encode(), + } +} +fn encode_old_snapshot(payload: OldSnapshot) -> Result, String> { let envelope = Envelope { schema: payload.schema.clone(), payload_sha256: hash(&payload)?, @@ -134,6 +156,10 @@ pub(super) fn decode( descriptor: &RecoveryDescriptor, now: u64, ) -> Result { + if full_checkpoint::is_full(bytes)? {return full_checkpoint::decode_structural(bytes,descriptor,now);} + decode_old(bytes,descriptor,now) +} +fn decode_old(bytes:&[u8],descriptor:&RecoveryDescriptor,now:u64)->Result{ if bytes.len() > crate::room_learner::recovery_store::MAX_RECOVERY_BYTES { return Err("temporal_checkpoint_budget".into()); } @@ -458,6 +484,7 @@ pub(super) fn decode( .map_or(0, |s| s.observed_end_ns); let durable_windows = session.rows.len(); Ok(TemporalWorkflow { + full_run: None, session, descriptor: p.descriptor, ledger: p.ledger, @@ -492,12 +519,16 @@ pub(crate) struct ValidatedTemporalPrefix { window_count: usize, transition_limit: usize, saved_receiver_ns: u64, + metadata_kind: &'static str, + full_write_grant: Option, + full_frozen_predecessor: Option, } pub(crate) fn validated_prefix( bytes: &[u8], descriptor: &RecoveryDescriptor, now_ns: u64, ) -> Result { + if full_checkpoint::is_full(bytes)?{return full_checkpoint::validated_structural(bytes,descriptor,now_ns);} let v = decode(bytes, descriptor, now_ns)?; let schedule = v.schedule()?; let train_episodes = schedule @@ -529,12 +560,17 @@ pub(crate) fn validated_prefix( window_count: v.session.rows.len(), transition_limit, saved_receiver_ns: v.last_now_ns, + metadata_kind:"temporal-prefix-commitment-v1", + full_write_grant:None, + full_frozen_predecessor:None, }) } pub(crate) fn validate_extension( old: &ValidatedTemporalPrefix, new: &ValidatedTemporalPrefix, ) -> Result<(), String> { + if old.metadata_kind!=new.metadata_kind{return Err("temporal_prefix_generation_kind_changed".into());} + if old.metadata_kind==full_run::META_KIND{return full_checkpoint::validate_full_extension(old,new);} let a: Envelope = serde_json::from_slice(&old.bytes).map_err(|_| "temporal_checkpoint_schema")?; let b: Envelope = @@ -665,6 +701,21 @@ pub(crate) fn validate_extension( } impl ValidatedTemporalPrefix { + pub(crate) fn metadata_kind(&self)->&str{self.metadata_kind} + pub(crate) fn authorizes_write(&self,expected:Option<&TemporalToken>)->bool{ + self.metadata_kind!=full_run::META_KIND||(self.full_write_grant.as_ref().is_some_and(|g|g.permits(&self.bytes,expected)) + &&self.full_frozen_predecessor.as_ref().is_none_or(|p|expected.is_some_and(|e|e.data()==*p))) + } + pub(super) fn verify_full_artifact(&self)->Result<(),String>{ + if self.metadata_kind==full_run::META_KIND&&self.full_frozen_predecessor.is_some(){ + return self.full_write_grant.as_ref().ok_or("temporal_frozen_producer_model_required")?.verify_frozen_model(); + } + Ok(()) + } + pub(super) fn attach_full_grant(&mut self,grant:full_run::FullRunWriteGrant)->Result<(),String>{ + if self.metadata_kind!=full_run::META_KIND{return Err("temporal_full_run_grant_kind".into());} + self.full_write_grant=Some(grant);Ok(()) + } pub(crate) fn saved_receiver_ns(&self) -> u64 { self.saved_receiver_ns } @@ -689,6 +740,7 @@ impl ValidatedTemporalPrefix { } pub(crate) fn is_new_empty_run(p: &ValidatedTemporalPrefix) -> Result { + if p.metadata_kind==full_run::META_KIND{return full_checkpoint::is_new_full_run(p);} let e: Envelope = serde_json::from_slice(p.bytes()).map_err(|_| "temporal_checkpoint_schema")?; let q = e.payload; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_durability.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_durability.rs index 31870a13f0..453c840914 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_durability.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_durability.rs @@ -122,6 +122,8 @@ struct Shared { corrupt_ack_id: AtomicBool, #[cfg(test)] hold: Mutex, mpsc::Receiver<()>)>>, + #[cfg(test)] + hold_after_prepare: Mutex, mpsc::Receiver<()>)>>, } impl Shared { fn now(&self) -> Result { @@ -172,6 +174,8 @@ impl Writer { corrupt_ack_id: AtomicBool::new(false), #[cfg(test)] hold: Mutex::new(None), + #[cfg(test)] + hold_after_prepare: Mutex::new(None), }); let state = shared.clone(); let thread = std::thread::Builder::new() @@ -181,7 +185,7 @@ impl Writer { while let Ok(work) = requests.recv() { let (id, slot, rows, seals, start_ticket, result) = match work { Work::Stop => break, - Work::Commit(request) => { + Work::Commit(mut request) => { let meta = (request.id, request.slot, request.rows, request.seals); let result = (|| { state.allowed(&request.descriptor)?; @@ -200,10 +204,13 @@ impl Writer { return Err("temporal_writer_exact_repair_required".into()); } let records = checkpoint::snapshot_records(&request.snapshot); + let full_permit=checkpoint::snapshot_full_permit(&mut request.snapshot); let bytes = checkpoint::encode_snapshot(request.snapshot)?; let at = request.saved_receiver_ns; - let prefix = + let mut prefix = checkpoint::validated_prefix(&bytes, &request.descriptor, at)?; + if let Some(permit)=full_permit{prefix.attach_full_grant(permit.encoded(&bytes))?;} + prefix.verify_full_artifact()?; let intent = store.prepare_temporal( request.expected.as_ref(), &prefix, @@ -218,6 +225,11 @@ impl Writer { seals: meta.3, records, }); + #[cfg(test)] + if let Some((entered, release)) = state.hold_after_prepare.lock().map_err(|_|"temporal_writer_test_lock")?.take() { + let _ = entered.send(()); + let _ = release.recv(); + } let p = pending.as_ref().unwrap(); let token = store.publish_temporal( &p.intent, @@ -489,6 +501,10 @@ impl Writer { self.shared.lose_next_ack.store(true, Ordering::Release); } #[cfg(test)] + pub fn hold_next_after_prepare(&self, entered:mpsc::SyncSender<()>, release:mpsc::Receiver<()>) { + *self.shared.hold_after_prepare.lock().unwrap()=Some((entered,release)); + } + #[cfg(test)] pub fn hold_next(&self, entered: mpsc::SyncSender<()>, release: mpsc::Receiver<()>) { *self.shared.hold.lock().unwrap() = Some((entered, release)); } @@ -515,6 +531,7 @@ fn readback( } let restored = checkpoint::decode_shared(p.prefix.bytes(), p.prefix.descriptor(), now, &p.records)?; + p.prefix.verify_full_artifact()?; let rows_sha256 = hash(&restored.session.rows)?; let row_prefix_sha256 = hash(&( &restored.session.rows, @@ -554,6 +571,9 @@ pub(super) struct AsyncState { pub row_prefix_sha256: Option, pub error: Option, pub accepted_seals: usize, + // At most the same ONE completion, awaiting an actually fresh receiver + // timestamp for new full-run authority. Never a packet or work queue. + deferred_ack:Option, } impl AsyncState { pub fn new(writer: Arc) -> Self { @@ -568,6 +588,7 @@ impl AsyncState { row_prefix_sha256: None, error: None, accepted_seals: 0, + deferred_ack:None, } } pub fn seed_restored(&mut self, w: &TemporalWorkflow) -> Result<(), String> { @@ -724,6 +745,7 @@ impl TemporalWorkflow { now: u64, retired: &mut DeferredReclamation, ) -> Result, String> { + let full_run=self.full_run.is_some(); let Some(state) = self.async_state.as_mut() else { return Ok(None); }; @@ -731,14 +753,18 @@ impl TemporalWorkflow { return Err("temporal_reclamation_already_occupied".into()); } state.writer.observe_now(now); - let Some(ack) = state.writer.poll()? else { + let next=if state.deferred_ack.is_some(){state.deferred_ack.take()}else{state.writer.poll()?}; + let Some(ack) = next else { return Ok(None); }; + if full_run && nowfalse, + (Some(a),Some(b))=>a.policy.sha256()!=b.policy.sha256(), + _=>true, + } {return Err("temporal_writer_ack_continuation_changed".into());} + let frozen_transition=self.prepare_frozen_ack(candidate,now)?; let must_restore = self .async_state .as_ref() - .is_some_and(|s| s.busy_pause || s.repairing); + .is_some_and(|s| s.busy_pause || s.repairing) && frozen_transition.is_none(); if must_restore { retired.fences = Some(self.session.assembler.fence_checkpoint()?); let restored = committed.restored.as_mut().unwrap(); @@ -836,6 +868,7 @@ impl TemporalWorkflow { state.accepted_seals = pending.seals; state.rows_sha256 = Some(committed.rows_sha256.clone()); state.row_prefix_sha256 = Some(committed.row_prefix_sha256.clone()); + if frozen_transition.is_some(){state.busy_pause=false;state.repairing=false;} retired.previous_bytes = self.durable.take(); self.commit_token = Some(committed.token.clone()); self.durable = Some(committed.bytes.clone()); @@ -848,6 +881,13 @@ impl TemporalWorkflow { state.busy_pause = false; state.repairing = false; restored.async_state = Some(state); + if let Some(old)=self.full_run.as_mut(){ + // Exact continuation presence was already validated above; + // all fallible checks finished before moving live custody. + let recovered=restored.full_run.as_mut().unwrap(); + recovered.owner=old.owner.take(); + recovered.local_start_not_before_ns=old.local_start_not_before_ns; + } let replacement = *committed.restored.take().unwrap(); retired.previous_workflow = Some(Box::new(std::mem::replace(self, replacement))); if persist_pause { @@ -857,6 +897,7 @@ impl TemporalWorkflow { } else { self.async_state = Some(state); } + if let Some(transition)=frozen_transition{self.apply_frozen_ack(transition);} self.last_now_ns = self.last_now_ns.max(now); Ok(Some((pending.start_ticket, now))) } diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs index 99adb73892..620ad05387 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs @@ -21,7 +21,9 @@ use bridge::BridgeState; pub(crate) use bridge::{TemporalCaptureStage, TemporalTrainCapture}; #[path = "temporal_fit_admission.rs"] mod fit_admission; -pub(crate) use fit_admission::{PendingTemporalFitInput, TemporalFitCaptureIdentity}; +pub(crate) use fit_admission::{PendingTemporalFitInput, TemporalFitCaptureIdentity,VerifiedTemporalFitSource}; +#[path = "temporal_full_run.rs"] +pub(crate) mod full_run; pub(crate) const EXECUTION_POLICY:&str="observed-complete-reservation-prefix-v1;paused-time;immutable-spans;partial-tail-drop;max128segments;persegment-native-firstarrival-skew-gaps;explicit-genesis-gap-anchor;quality-unimplemented;independence-false"; #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] @@ -179,6 +181,7 @@ struct SegmentCustody { /// Future value owned by existing RoomLearningRuntime::Inner's ONE tagged workflow. /// No production constructor/permit issuer is exposed by this source slice. struct TemporalWorkflow { + full_run: Option, session: TemporalTrainingSession, descriptor: RecoveryDescriptor, ledger: Ledger, @@ -242,6 +245,7 @@ impl TemporalWorkflow { let digest = c.digest(); let session = TemporalTrainingSession::managed(schedule, run, config)?; Ok(Self { + full_run: None, session, descriptor: RecoveryDescriptor { schema: crate::room_learner::recovery_store::RECOVERY_SCHEMA.into(), diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_admission.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_admission.rs index 4265f4a388..8d5e1c2fd0 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_admission.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_admission.rs @@ -17,6 +17,9 @@ pub(crate) struct TemporalFitCaptureIdentity { descriptor: RecoveryDescriptor, token: TemporalToken, prefix: String, + prefix_schema: &'static str, + metadata_kind: &'static str, + continuation_policy: Option, } impl TemporalFitCaptureIdentity { pub(crate) fn owner(&self) -> &str { @@ -28,6 +31,7 @@ impl TemporalFitCaptureIdentity { pub(crate) fn context(&self) -> &str { &self.context } + pub(crate) fn run_id(&self)->&str{&self.run} pub(crate) fn expires_at_ms(&self) -> u64 { self.descriptor.expires_at_ms } @@ -65,6 +69,56 @@ pub(crate) struct PendingTemporalFitInput { pub(crate) struct ValidatedTemporalFitInput { identity: TemporalFitCaptureIdentity, session: TemporalTrainingSession, + train_seals_sha256:String, +} +/// Off-lock strict producer validation, later bound to an ACTUAL saved fit. +/// This is not serializable/importable, and has no public constructor. +pub(crate) struct VerifiedTemporalFitSource { + identity:TemporalFitCaptureIdentity, + train_data:String,evidence:String,identities:String,seals:String, + actual_candidate:Option, + actual_summary:full_run::FitSummaryReference, +} +impl VerifiedTemporalFitSource { + pub(crate) fn is_full_run(&self)->bool{self.identity.continuation_policy.is_some()} + pub(crate) fn binding(&self)->&TemporalFitCaptureIdentity{&self.identity} + pub(crate) fn bind_saved_fit(&mut self,model:&TemporalModel,saved:&crate::room_learner::artifact::TemporalFitSaveResult)->Result<(),String>{ + let b=model.fit_diagnostic_binding()?; + if b.artifact_kind!=full_run::MODEL_KIND||b.context_digest!=self.identity.context + ||b.training_data_sha256!=self.train_data||b.input_evidence_sha256!=self.evidence + ||b.identity_evidence_sha256.as_ref()!=Some(&self.identities)||b.schedule_sha256.as_ref()!=Some(&self.identity.schedule) + ||b.input_policy_sha256!=self.identity.policy + ||b.bound_fit_config.as_ref().map(hash).transpose()?.as_ref()!=Some(&self.identity.configuration) + ||saved.candidate.artifact_sha256!=b.artifact_sha256||saved.candidate.artifact_kind!=b.artifact_kind{ + return Err("temporal_actual_fit_source_binding_changed".into()); + } + self.actual_candidate=Some(saved.candidate.clone()); + self.actual_summary=match &saved.fit_diagnostics.summary{ + Some(summary)=>full_run::FitSummaryReference::Recorded{sha256:hash(summary)?}, + None=>match saved.fit_diagnostics.state{ + crate::room_learner::artifact::FitDiagnosticState::NotRecorded=>full_run::FitSummaryReference::NotRecorded, + _=>full_run::FitSummaryReference::Unavailable, + } + }; + Ok(()) + } + pub(super) fn record_data(self,now:u64,floors:Vec<(u8,u64,u64)>)->Result{ + self.identity.check_deadline(now)?; + let candidate=self.actual_candidate.ok_or("temporal_actual_saved_fit_required")?; + let continuation=self.identity.continuation_policy.ok_or("temporal_full_run_no_upconversion")?; + let body=full_run::VerifiedFitRecordBody{ + owner_scope:self.identity.owner,installation_id:self.identity.installation,context_digest:self.identity.context,run:self.identity.run, + prefix_schema:self.identity.prefix_schema.into(),metadata_kind:self.identity.metadata_kind.into(),continuation_policy_sha256:continuation, + artifact_kind:candidate.artifact_kind.clone(),artifact_sha256:candidate.artifact_sha256.clone(), + input_policy_sha256:self.identity.policy,execution_policy_sha256:hash(&EXECUTION_POLICY)?,schedule_sha256:self.identity.schedule, + configured_fit_sha256:self.identity.configuration,training_data_sha256:self.train_data,input_evidence_sha256:self.evidence, + identity_evidence_sha256:self.identities,train_seal_chain_sha256:self.seals,accepted_train_prefix_sha256:self.identity.prefix, + accepted_train_commitment:self.identity.token.data(),saved_candidate:candidate,actual_summary:self.actual_summary, + freeze_prepare_receiver_ns:now,source_floors:floors, + }; + let sha256=hash(&(full_run::FIT_RECORD_DOMAIN,&body))?; + Ok(full_run::VerifiedFitRecordData{schema:full_run::FIT_RECORD_DOMAIN.into(),body,sha256}) + } } fn complete_train( @@ -125,6 +179,9 @@ fn identity( descriptor: w.descriptor.clone(), token, prefix, + prefix_schema:if w.full_run.is_some(){full_run::PREFIX_SCHEMA}else{"ruview.room-learner.temporal-runtime-prefix.v2"}, + metadata_kind:if w.full_run.is_some(){full_run::META_KIND}else{"temporal-prefix-commitment-v1"}, + continuation_policy:w.full_run.as_ref().map(|m|m.policy.sha256().into()), }) } pub(super) fn current_identity( @@ -133,6 +190,7 @@ pub(super) fn current_identity( now: u64, ) -> Result { complete_train(w, schedule, now)?; + if w.full_run.as_ref().is_some_and(|m|m.owner.is_none()){return Err("temporal_full_run_owner_custody_required".into());} // The async writer supplies a cached verified digest. Never hash a growing // raw prefix under the receiver lock or silently fall back to a sync path. let state = w @@ -215,6 +273,7 @@ impl PendingTemporalFitInput { } Ok(ValidatedTemporalFitInput { identity: self.identity, + train_seals_sha256:hash(&restored.bridge.as_ref().ok_or("temporal_fit_actual_bridge_required")?.seals)?, session: restored.session, }) } @@ -226,6 +285,9 @@ impl ValidatedTemporalFitInput { pub(crate) fn into_session_and_admission( self, ) -> Result<(TemporalTrainingSession, ManagedFitAdmission), String> { + let(session,permit,_)=self.into_session_admission_and_source()?;Ok((session,permit)) + } + pub(crate) fn into_session_admission_and_source(self)->Result<(TemporalTrainingSession,ManagedFitAdmission,VerifiedTemporalFitSource),String>{ let managed = self .session .managed @@ -238,9 +300,13 @@ impl ValidatedTemporalFitInput { evidence: hash(&self.session.evidence)?, identities: hash(&managed.windows)?, }; + // Strict seal validation happened in the recertified producer decoder; + // recover its exact immutable chain from the accepted prefix again on + // the worker, never hash growing source data under Runtime.inner. + let source=VerifiedTemporalFitSource{identity:self.identity,train_data:permit.data.clone(),evidence:permit.evidence.clone(), + identities:permit.identities.clone(),seals:self.train_seals_sha256,actual_candidate:None,actual_summary:full_run::FitSummaryReference::Unavailable}; // Full validated data is intentionally dropped here, on the worker. drop(dataset); - let _ = self.identity; - Ok((self.session, permit)) + Ok((self.session, permit,source)) } } diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run.rs new file mode 100644 index 0000000000..2aac8abf60 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run.rs @@ -0,0 +1,631 @@ +//! Fresh-genesis full-run recovery DATA and private owner custody. +//! No public constructor, migration, HeldOut producer or qualification authority. +use super::*; +use crate::room_learner::artifact::ArtifactStore; +use crate::room_learner::temporal_acquisition::{ReceiverCadenceBand, WindowInputPolicy}; +use std::sync::Arc; + +pub(crate) const PREFIX_SCHEMA: &str = "ruview.room-learner.temporal-runtime-prefix.v3"; +pub(crate) const META_KIND: &str = "temporal-prefix-commitment-v2"; +pub(crate) const POLICY_DOMAIN: &str = "ruview.room-learner.temporal-full-run-policy.v1"; +pub(crate) const FIT_RECORD_DOMAIN: &str = "ruview.room-learner.temporal-verified-fit-record.v1"; +pub(crate) const MODEL_KIND: &str = "ruview.room-learner.managed-temporal-input.v3"; + +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct PolicyBody { + prefix_schema: String, + metadata_kind: String, + model_artifact_kind: String, + context_digest: String, + input_policy_sha256: String, + execution_policy_sha256: String, + schedule_sha256: String, + configured_fit_sha256: String, + native_profile: WindowInputPolicy, + train_episodes: usize, + held_out_episodes: usize, + train_windows: usize, + held_out_windows: usize, + total_windows: usize, + max_total_windows: usize, + max_numeric_values: usize, + max_recovery_bytes: usize, + max_global_gaps: usize, + max_quality_parts: usize, + transition_limit: usize, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(super) struct ContinuationPolicy { + schema: String, + body: PolicyBody, + sha256: String, +} +impl ContinuationPolicy { + pub(super) fn build(binding: &ManagedBinding) -> Result { + let schedule = binding.contract()?; + let c = schedule.counts(); + let context = schedule.context(); + if context.sources.len() != 3 + || context.sources.iter().any(|s| s.feature_count != 56) + || binding.native.target_span_ms() != 2000 + || binding.native.source_policies().len() != 3 + || binding + .native + .source_policies() + .iter() + .any(|s| s.receiver_cadence != ReceiverCadenceBand::Hz30To36 || s.max_rows != 77) + || ( + c.train_episodes, + c.held_out_episodes, + c.train_windows, + c.held_out_windows, + c.total_windows, + ) != (7, 8, 64, 64, 128) + || schedule.resources().max_numeric_values != 1_655_808 + || schedule.resources().max_numeric_values > 2_000_000 + { + return Err("temporal_full_run_engineering_profile_unsupported".into()); + } + let episodes = c + .train_episodes + .checked_add(c.held_out_episodes) + .ok_or("temporal_full_run_capacity")?; + let segments = episodes + .checked_add(128) + .ok_or("temporal_full_run_capacity")?; + // STATES, revision0..432: genesis; each initial/resumed segment; at + // most one accepted pause per segment (including terminal/deferred + // pauses); windows; separate confirmed seals; Frozen; reserved final + // completion; retirement. Recertification/SAME-intent retry add zero. + let transition_limit = 1usize + .checked_add(segments) + .and_then(|n| n.checked_add(segments)) + .and_then(|n| n.checked_add(c.total_windows)) + .and_then(|n| n.checked_add(episodes)) + .and_then(|n| n.checked_add(3)) + .ok_or("temporal_full_run_capacity")?; + if transition_limit != 433 || transition_limit > 512 { + return Err("temporal_full_run_transition_capacity".into()); + } + let body = PolicyBody { + prefix_schema: PREFIX_SCHEMA.into(), + metadata_kind: META_KIND.into(), + model_artifact_kind: MODEL_KIND.into(), + context_digest: context.digest(), + input_policy_sha256: hash(binding)?, + execution_policy_sha256: hash(&EXECUTION_POLICY)?, + schedule_sha256: schedule.sha256().into(), + configured_fit_sha256: hash(&binding.fit)?, + native_profile: binding.native.clone(), + train_episodes: c.train_episodes, + held_out_episodes: c.held_out_episodes, + train_windows: c.train_windows, + held_out_windows: c.held_out_windows, + total_windows: c.total_windows, + max_total_windows: 128, + max_numeric_values: 2_000_000, + max_recovery_bytes: crate::room_learner::recovery_store::MAX_RECOVERY_BYTES, + max_global_gaps: 128, + max_quality_parts: segments, + transition_limit, + }; + let sha256 = hash(&(POLICY_DOMAIN, &body))?; + Ok(Self { + schema: POLICY_DOMAIN.into(), + body, + sha256, + }) + } + pub(super) fn validate(&self, binding: &ManagedBinding) -> Result<(), String> { + let expected = Self::build(binding)?; + if serde_json::to_vec(self).map_err(|_| "temporal_full_run_policy_encoding")? + != serde_json::to_vec(&expected).map_err(|_| "temporal_full_run_policy_encoding")? + { + return Err("temporal_full_run_policy_changed".into()); + } + Ok(()) + } + pub(super) fn sha256(&self) -> &str { + &self.sha256 + } + pub(super) fn transition_limit(&self) -> usize { + self.body.transition_limit + } +} + +/// Non-serde custody is never reconstructed by a raw prefix decoder. +pub(super) struct FullRunOwnerCustody { + runtime_epoch: String, + generation: u64, +} +pub(super) struct FullRunState { + pub policy: ContinuationPolicy, + pub owner: Option, + // Only DATA after raw decode. Verified session installation is separate. + pub frozen: Option, + pub local_start_not_before_ns: Option, + pending: Option, + verified: Option, +} +impl FullRunState { + pub(super) fn fresh( + binding: &ManagedBinding, + run: &str, + witness: crate::room_learner::lifecycle::FullRunGenesisWitness, + ) -> Result { + let (epoch, generation) = witness.consume(binding.native.context(), run)?; + Ok(Self { + policy: ContinuationPolicy::build(binding)?, + owner: Some(FullRunOwnerCustody { + runtime_epoch: epoch, + generation, + }), + frozen: None, + local_start_not_before_ns: None, + pending: None, + verified: None, + }) + } + pub(super) fn structural( + policy: ContinuationPolicy, + frozen: Option, + ) -> Self { + Self { + policy, + owner: None, + frozen, + local_start_not_before_ns: None, + pending: None, + verified: None, + } + } + pub(super) fn snapshot_permit( + &self, + expected: Option, + ) -> Result { + let owner = self + .owner + .as_ref() + .ok_or("temporal_full_run_owner_custody_required")?; + if self.frozen.is_some() && self.pending.is_none() { + return Err("temporal_frozen_write_requires_current_producer".into()); + } + Ok(FullRunSnapshotPermit { + runtime_epoch: owner.runtime_epoch.clone(), + generation: owner.generation, + expected, + model: self.pending.as_ref().map(|p| p.model.clone()), + }) + } +} +/// Producer-only snapshot permission; serialized DATA never contains this. +pub(super) struct FullRunSnapshotPermit { + runtime_epoch: String, + generation: u64, + expected: Option, + model: Option, +} +#[derive(Clone)] +pub(crate) struct FullRunWriteGrant { + payload_sha256: String, + expected: Option, + runtime_epoch: String, + generation: u64, + model: Option, +} +impl FullRunSnapshotPermit { + pub(super) fn encoded(self, bytes: &[u8]) -> FullRunWriteGrant { + FullRunWriteGrant { + payload_sha256: bytes_hash(bytes), + expected: self.expected, + runtime_epoch: self.runtime_epoch, + generation: self.generation, + model: self.model, + } + } +} +impl FullRunWriteGrant { + pub(crate) fn permits(&self, bytes: &[u8], expected: Option<&TemporalToken>) -> bool { + !self.runtime_epoch.is_empty() + && self.generation > 0 + && self.expected.as_ref() == expected + && self.payload_sha256 == bytes_hash(bytes) + } + pub(crate) fn verify_frozen_model(&self) -> Result<(), String> { + match &self.model { + Some(m) => m.verify().map(|_| ()), + None => Ok(()), + } + } +} + +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(tag = "state", rename_all = "snake_case", deny_unknown_fields)] +pub(super) enum FitSummaryReference { + Recorded { sha256: String }, + NotRecorded, + Unavailable, +} +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct VerifiedFitRecordBody { + pub owner_scope: String, + pub installation_id: String, + pub context_digest: String, + pub run: String, + pub prefix_schema: String, + pub metadata_kind: String, + pub continuation_policy_sha256: String, + pub artifact_kind: String, + pub artifact_sha256: String, + pub input_policy_sha256: String, + pub execution_policy_sha256: String, + pub schedule_sha256: String, + pub configured_fit_sha256: String, + pub training_data_sha256: String, + pub input_evidence_sha256: String, + pub identity_evidence_sha256: String, + pub train_seal_chain_sha256: String, + pub accepted_train_prefix_sha256: String, + pub accepted_train_commitment: + crate::room_learner::recovery_store::temporal_commitment::TemporalCommitmentData, + pub saved_candidate: crate::room_learner::artifact::TemporalCandidateReceipt, + pub actual_summary: FitSummaryReference, + pub freeze_prepare_receiver_ns: u64, + pub source_floors: Vec<(u8, u64, u64)>, +} +/// Serializable DATA, not a permit. SHA binds data, never producer authority. +#[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct VerifiedFitRecordData { + pub schema: String, + pub body: VerifiedFitRecordBody, + pub sha256: String, +} +impl VerifiedFitRecordData { + pub(super) fn validate_data( + &self, + binding: &ManagedBinding, + policy: &ContinuationPolicy, + ) -> Result<(), String> { + policy.validate(binding)?; + let b = &self.body; + let p = &policy.body; + let c = binding.native.context(); + let sha = |s: &str| { + s.len() == 64 + && s.bytes() + .all(|x| x.is_ascii_digit() || (b'a'..=b'f').contains(&x)) + }; + if self.schema != FIT_RECORD_DOMAIN + || self.sha256 != hash(&(FIT_RECORD_DOMAIN, &self.body))? + || b.owner_scope != c.owner_scope + || b.installation_id != c.installation_id + || b.context_digest != c.digest() + || b.run.is_empty() + || b.run.len() > 256 + || !b.run.is_ascii() + || b.prefix_schema != PREFIX_SCHEMA + || b.metadata_kind != META_KIND + || b.continuation_policy_sha256 != policy.sha256 + || b.artifact_kind != MODEL_KIND + || b.input_policy_sha256 != p.input_policy_sha256 + || b.execution_policy_sha256 != p.execution_policy_sha256 + || b.schedule_sha256 != p.schedule_sha256 + || b.configured_fit_sha256 != p.configured_fit_sha256 + || [ + &b.artifact_sha256, + &b.training_data_sha256, + &b.input_evidence_sha256, + &b.identity_evidence_sha256, + &b.train_seal_chain_sha256, + &b.accepted_train_prefix_sha256, + ] + .iter() + .any(|s| !sha(s)) + || b.freeze_prepare_receiver_ns == 0 + || b.source_floors.len() != c.sources.len() + || b.saved_candidate.artifact_kind != MODEL_KIND + || b.saved_candidate.artifact_sha256 != b.artifact_sha256 + || b.saved_candidate.schema != "ruview.room-learner.temporal-candidate.v1" + || b.saved_candidate.owner_scope != b.owner_scope + || b.saved_candidate.installation_id != b.installation_id + || b.saved_candidate.context_digest != b.context_digest + || b.saved_candidate.room_digest != c.room_digest + || b.saved_candidate.input_policy_sha256 != b.input_policy_sha256 + || !(1..=512).contains(&b.saved_candidate.parameter_count) + || !b.saved_candidate.diagnostic_only + || b.saved_candidate.qualification_granted + || b.saved_candidate.physically_validated + { + return Err("temporal_verified_fit_record_binding".into()); + } + if matches!(&b.actual_summary,FitSummaryReference::Recorded{sha256}if !sha(sha256)) { + return Err("temporal_verified_fit_summary_binding".into()); + } + Ok(()) + } +} + +#[derive(Clone)] +struct FrozenModelVerifier { + store: Arc, + record: VerifiedFitRecordData, + context: CaptureContext, + actual: Arc, +} +impl FrozenModelVerifier { + fn verify(&self) -> Result, String> { + let loaded = self + .store + .load_exact_temporal_candidate(&self.context, &self.record.body.artifact_sha256)?; + verify_model_binding(&self.record, &loaded)?; + Ok(Arc::new(loaded)) + } +} +fn verify_model_binding( + record: &VerifiedFitRecordData, + model: &TemporalModel, +) -> Result<(), String> { + let b = model.fit_diagnostic_binding()?; + let r = &record.body; + if b.artifact_kind != MODEL_KIND + || b.artifact_sha256 != r.artifact_sha256 + || b.context_digest != r.context_digest + || b.input_policy_sha256 != r.input_policy_sha256 + || b.execution_policy_sha256.as_ref() != Some(&r.execution_policy_sha256) + || b.schedule_sha256.as_ref() != Some(&r.schedule_sha256) + || b.training_data_sha256 != r.training_data_sha256 + || b.input_evidence_sha256 != r.input_evidence_sha256 + || b.identity_evidence_sha256.as_ref() != Some(&r.identity_evidence_sha256) + || b.bound_fit_config.as_ref().map(hash).transpose()?.as_ref() + != Some(&r.configured_fit_sha256) + || b.parameter_count != r.saved_candidate.parameter_count + { + return Err("temporal_frozen_exact_model_kind_or_binding".into()); + } + Ok(()) +} +/// One current owner operation, separate from a completed fitting Job.cancel. +pub(crate) struct PendingFrozenTransition { + operation: (String, u64, u64), + record: VerifiedFitRecordData, + model: FrozenModelVerifier, + rows: Vec>, + evidence: Vec>, +} +/// Neither raw DATA nor latest model pointers can construct this install value. +pub(crate) struct DurableFrozenTransition { + install: VerifiedFrozenSession, + admitted_at_ns: u64, +} +pub(crate) struct RecoveredVerifiedFrozen { + install: VerifiedFrozenSession, + recertified_at_ns: u64, +} +struct VerifiedFrozenSession { + model: Arc, + record: VerifiedFitRecordData, + rows: Vec>, + evidence: Vec>, +} +impl VerifiedFrozenSession { + fn check(&self, session: &TemporalTrainingSession) -> Result<(), String> { + if session.active + || session.failed + || session.fitted + || session.rows.len() != 64 + || session.evidence.len() != 64 + || session + .rows + .iter() + .zip(&self.rows) + .any(|(a, b)| !a.same_record(b)) + || session + .evidence + .iter() + .zip(&self.evidence) + .any(|(a, b)| !a.same_record(b)) + || session.plan_sha256() != self.record.body.input_policy_sha256 + || session.run_id != self.record.body.run + || session.binding.context.digest() != self.record.body.context_digest + || hash(&session.config)? != self.record.body.configured_fit_sha256 + { + return Err("temporal_verified_frozen_session_changed".into()); + } + Ok(()) + } + fn apply(&self, session: &mut TemporalTrainingSession) { + session.managed.as_mut().unwrap().frozen_sha256 = + Some(self.record.body.artifact_sha256.clone()); + session.fitted = true; + } +} +impl FullRunState { + pub(super) fn is_frozen(&self) -> bool { + self.verified.is_some() + } + pub(super) fn is_freezing(&self) -> bool { + self.pending.is_some() + } + pub(super) fn pending_record(&self) -> Option<&VerifiedFitRecordData> { + self.pending.as_ref().map(|p| &p.record) + } + pub(super) fn verified_model(&self) -> Option> { + self.verified.as_ref().map(|v| v.model.clone()) + } +} +/// The ONLY cold Frozen authority producer calls the real CURRENT store and +/// recertifies its barriers itself; no raw decoded object is accepted as input. +pub(super) fn recover_current( + store: &RecoveryStore, + artifact: Arc, + expected: &CaptureContext, + now: u64, + epoch: &str, + anchor: u64, + witness: crate::room_learner::lifecycle::FullRunRecoveryWitness, +) -> Result { + let mut w = TemporalWorkflow::cold_restore(store, expected, now, epoch, anchor)?; + let (owner_epoch, generation) = witness.consume(expected)?; + let mode = w + .full_run + .as_ref() + .ok_or("temporal_full_run_no_upconversion")?; + mode.policy.validate( + &w.session + .managed + .as_ref() + .ok_or("managed_session_required")? + .binding, + )?; + let record = mode.frozen.clone(); + let verified = if let Some(record) = record { + if now < record.body.freeze_prepare_receiver_ns { + return Err("temporal_frozen_recovery_clock_reversed".into()); + } + let model = Arc::new( + artifact.load_exact_temporal_candidate(expected, &record.body.artifact_sha256)?, + ); + verify_model_binding(&record, &model)?; + let install = VerifiedFrozenSession { + model, + record, + rows: w.session.rows.clone(), + evidence: w.session.evidence.clone(), + }; + install.check(&w.session)?; + let recovered = RecoveredVerifiedFrozen { + install, + recertified_at_ns: now, + }; + recovered.install.apply(&mut w.session); + Some(recovered) + } else { + None + }; + let mode = w.full_run.as_mut().unwrap(); + mode.owner = Some(FullRunOwnerCustody { + runtime_epoch: owner_epoch, + generation, + }); + if let Some(recovered) = verified { + mode.local_start_not_before_ns = Some(recovered.recertified_at_ns); + mode.verified = Some(recovered.install); + } + Ok(w) +} +impl TemporalWorkflow { + pub(super) fn request_frozen( + &mut self, + accepted: crate::room_learner::lifecycle::AcceptedTemporalFit, + witness: crate::room_learner::lifecycle::FrozenOwnerWitness, + store: Arc, + now: u64, + ) -> Result<(), String> { + let identity = fit_admission::current_identity(self, &self.schedule()?, now)?; + let mode = self + .full_run + .as_ref() + .ok_or("temporal_full_run_no_upconversion")?; + let owner = mode + .owner + .as_ref() + .ok_or("temporal_full_run_owner_custody_required")?; + if mode.pending.is_some() || mode.frozen.is_some() || mode.verified.is_some() { + return Err("temporal_frozen_operation_already_present".into()); + } + let operation = witness.consume(self.context(), &self.session.run_id)?; + if operation.0 != owner.runtime_epoch || operation.1 != owner.generation { + return Err("temporal_frozen_owner_changed".into()); + } + let (model, source) = + accepted.consume_for_owner(&owner.runtime_epoch, owner.generation, &identity)?; + let record = source.record_data(now, self.source_high.clone())?; + record.validate_data( + &self.session.managed.as_ref().unwrap().binding, + &mode.policy, + )?; + if model.context() != self.context() { + return Err("temporal_frozen_exact_model_context".into()); + } + let verifier = FrozenModelVerifier { + store, + record: record.clone(), + context: self.context().clone(), + actual: model, + }; + let pending = PendingFrozenTransition { + operation, + record: record.clone(), + model: verifier, + rows: self.session.rows.clone(), + evidence: self.session.evidence.clone(), + }; + self.check_clock(now)?; + self.descriptor.saved_at_ms = now / 1_000_000; + let mode = self.full_run.as_mut().unwrap(); + mode.pending = Some(pending); + mode.frozen = Some(record); + self.submit_async_commit(false) + } + pub(super) fn prepare_frozen_ack( + &self, + candidate: &TemporalWorkflow, + now: u64, + ) -> Result, String> { + let Some(mode) = &self.full_run else { + return Ok(None); + }; + let Some(pending) = &mode.pending else { + return Ok(None); + }; + let owner = mode + .owner + .as_ref() + .ok_or("temporal_full_run_owner_custody_required")?; + let data = candidate + .full_run + .as_ref() + .and_then(|m| m.frozen.as_ref()) + .ok_or("temporal_frozen_ack_record_missing")?; + if data != &pending.record + || pending.operation.0 != owner.runtime_epoch + || pending.operation.1 != owner.generation + || now < data.body.freeze_prepare_receiver_ns + || now < self.last_now_ns + || now / 1_000_000 >= self.descriptor.expires_at_ms + || self.source_high.iter().any(|(id, seq, time)| { + data.body + .source_floors + .iter() + .any(|(s, q, t)| s == id && (seq < q || time < t)) + }) + { + return Err("temporal_frozen_ack_owner_cut_or_record".into()); + } + let install = VerifiedFrozenSession { + model: pending.model.actual.clone(), + record: data.clone(), + rows: pending.rows.clone(), + evidence: pending.evidence.clone(), + }; + // Model I/O must be done on the writer, not here. The exact same model + // Arc is carried by the pending capability after writer readback. + install.check(&self.session)?; + Ok(Some(DurableFrozenTransition { + install, + admitted_at_ns: now, + })) + } + pub(super) fn apply_frozen_ack(&mut self, transition: DurableFrozenTransition) { + transition.install.apply(&mut self.session); + let mode = self.full_run.as_mut().unwrap(); + mode.pending = None; + mode.local_start_not_before_ns = Some(transition.admitted_at_ns); + mode.verified = Some(transition.install); + self.stage = Stage::DurableReady; + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run_checkpoint.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run_checkpoint.rs new file mode 100644 index 0000000000..b5e67e7f63 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run_checkpoint.rs @@ -0,0 +1,363 @@ +//! Structural v3 codec. Raw JSON never installs owner/fitted/write authority. +use super::*; +use full_run::{ContinuationPolicy, FullRunSnapshotPermit, FullRunState, VerifiedFitRecordData}; +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Common { + binding: ManagedBinding, + run: String, + descriptor: RecoveryDescriptor, + last_now_ns: u64, + source_high: Vec<(u8, u64, u64)>, + ledger: Ledger, + mapping: Option, + episode: usize, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct Accepted { + rows: Vec, + evidence: Vec, + identities: Vec, + fences: AssemblerCheckpoint, +} +#[derive(Clone, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum FullStage { + Train { + bridge: BridgeState, + }, + Frozen { + fit: VerifiedFitRecordData, + train_seals: Vec, + }, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct FullData { + schema: String, + policy: ContinuationPolicy, + common: Common, + accepted: Accepted, + stage: FullStage, +} +#[derive(Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +struct FullEnvelope

{ + schema: String, + payload_sha256: String, + payload: P, +} +pub(super) struct FullSnapshot { + data: FullData, SharedRecord>, + pub permit: Option, +} +impl FullSnapshot { + pub(super) fn from_workflow(w: &TemporalWorkflow) -> Result { + let mode = w.full_run.as_ref().ok_or("temporal_full_run_required")?; + mode.policy.validate( + &w.session + .managed + .as_ref() + .ok_or("managed_session_required")? + .binding, + )?; + let permit = mode.snapshot_permit(w.commit_token.clone())?; + let old = super::snapshot_old(w)?; + let stage = match &mode.frozen { + None => FullStage::Train { + bridge: old + .bridge + .clone() + .ok_or("temporal_full_run_bridge_required")?, + }, + Some(fit) => { + if mode.pending_record() != Some(fit) { + return Err("temporal_frozen_current_producer_required".into()); + } + FullStage::Frozen { + fit: fit.clone(), + train_seals: old + .bridge + .as_ref() + .ok_or("temporal_full_run_bridge_required")? + .seals + .clone(), + } + } + }; + Ok(Self { + data: FullData { + schema: full_run::PREFIX_SCHEMA.into(), + policy: mode.policy.clone(), + common: Common { + binding: old.binding, + run: old.run, + descriptor: old.descriptor, + last_now_ns: old.last_now_ns, + source_high: old.source_high, + ledger: old.ledger, + mapping: old.mapping, + episode: old.episode, + }, + accepted: Accepted { + rows: old.rows, + evidence: old.evidence, + identities: old.identities, + fences: old.fences, + }, + stage, + }, + permit: Some(permit), + }) + } + pub(super) fn records(&self) -> SharedRecords { + SharedRecords { + rows: self.data.accepted.rows.clone(), + evidence: self.data.accepted.evidence.clone(), + } + } + pub(super) fn encode(self) -> Result, String> { + let envelope = FullEnvelope { + schema: full_run::PREFIX_SCHEMA.into(), + payload_sha256: hash(&self.data)?, + payload: self.data, + }; + let bytes = serde_json::to_vec(&envelope).map_err(|_| "temporal_full_run_encoding")?; + if bytes.len() > crate::room_learner::recovery_store::MAX_RECOVERY_BYTES { + return Err("temporal_checkpoint_budget".into()); + } + Ok(bytes) + } +} +pub(super) fn is_full(bytes: &[u8]) -> Result { + if bytes.len() > crate::room_learner::recovery_store::MAX_RECOVERY_BYTES { + return Err("temporal_checkpoint_budget".into()); + } + #[derive(Deserialize)] + struct Header { + schema: String, + } + // Header dispatch skips unknown fields without making a serde Value/raw + // copy. The selected strict decoder still checks every field/canonical byte. + let header: Header = serde_json::from_slice(bytes).map_err(|_| "temporal_checkpoint_schema")?; + Ok(header.schema == full_run::PREFIX_SCHEMA) +} +fn parse(bytes: &[u8]) -> Result { + if bytes.len() > crate::room_learner::recovery_store::MAX_RECOVERY_BYTES { + return Err("temporal_checkpoint_budget".into()); + } + let e: FullEnvelope = serde_json::from_slice(bytes).map_err(|_| "temporal_full_run_schema")?; + if e.schema != full_run::PREFIX_SCHEMA + || e.payload.schema != full_run::PREFIX_SCHEMA + || e.payload_sha256 != hash(&e.payload)? + || serde_json::to_vec(&e).map_err(|_| "temporal_full_run_encoding")? != bytes + { + return Err("temporal_full_run_integrity".into()); + } + e.payload.policy.validate(&e.payload.common.binding)?; + Ok(e) +} +fn old_projection( + data: FullData, +) -> Result<(Prefix, ContinuationPolicy, Option), String> { + let common = data.common; + let accepted = data.accepted; + let (bridge, frozen) = match data.stage { + FullStage::Train { bridge } => (bridge, None), + FullStage::Frozen { fit, train_seals } => { + fit.validate_data(&common.binding, &data.policy)?; + if common.run != fit.body.run + || common.last_now_ns != fit.body.freeze_prepare_receiver_ns + || common.source_high != fit.body.source_floors + || common.mapping.is_some() + { + return Err("temporal_frozen_record_cut_or_floors".into()); + } + let b = BridgeState { + policy: bridge::BRIDGE_POLICY.into(), + disposition: bridge::Disposition::Complete, + seals: train_seals, + parts: vec![], + }; + (b, Some(fit)) + } + }; + Ok(( + Prefix { + schema: BRIDGE_SCHEMA.into(), + binding: common.binding, + run: common.run, + descriptor: common.descriptor, + last_now_ns: common.last_now_ns, + rows: accepted.rows, + evidence: accepted.evidence, + identities: accepted.identities, + fences: accepted.fences, + source_high: common.source_high, + ledger: common.ledger, + mapping: common.mapping, + episode: common.episode, + frozen_model: None, + quality_issued: false, + confirmation_issued: false, + bridge: Some(bridge), + }, + data.policy, + frozen, + )) +} +fn old_bytes(prefix: Prefix) -> Result, String> { + serde_json::to_vec(&Envelope { + schema: BRIDGE_SCHEMA.into(), + payload_sha256: hash(&prefix)?, + payload: prefix, + }) + .map_err(|_| "temporal_full_run_projection".into()) +} +pub(super) fn decode_structural( + bytes: &[u8], + descriptor: &RecoveryDescriptor, + now: u64, +) -> Result { + let e = parse(bytes)?; + let (prefix, policy, frozen) = old_projection(e.payload)?; + // This slice reuses the exact existing Train/quality/native validator. No + // HeldOut/Complete variant is accepted before the actual C4B producer exists. + let mut w = super::decode_old(&old_bytes(prefix)?, descriptor, now)?; + if let Some(record) = &frozen { + let managed = w + .session + .managed + .as_ref() + .ok_or("managed_session_required")?; + let bridge = w + .bridge + .as_ref() + .ok_or("temporal_full_run_bridge_required")?; + let body = &record.body; + if w.session.rows.len() != 64 + || w.session.rows.iter().any(|r| r.role != DataRole::Train) + || bridge.disposition != bridge::Disposition::Complete + || bridge.seals.len() != 7 + || body.training_data_sha256 != hash(&w.session.rows)? + || body.input_evidence_sha256 != hash(&w.session.evidence)? + || body.identity_evidence_sha256 != hash(&managed.windows)? + || body.train_seal_chain_sha256 != hash(&bridge.seals)? + || body.accepted_train_commitment.run != w.session.run_id + || body.accepted_train_commitment.id != descriptor.id + { + return Err("temporal_frozen_train_binding".into()); + } + } + w.full_run = Some(FullRunState::structural(policy, frozen)); + // Deliberately remains unfitted. Only authenticated CURRENT recovery plus + // exact whole-model verification can mint the install witness. + Ok(w) +} +pub(super) fn validated_structural( + bytes: &[u8], + descriptor: &RecoveryDescriptor, + now: u64, +) -> Result { + let w = decode_structural(bytes, descriptor, now)?; + let mode = w.full_run.as_ref().ok_or("temporal_full_run_required")?; + Ok(ValidatedTemporalPrefix { + bytes: bytes.to_vec(), + descriptor: descriptor.clone(), + run: w.session.run_id.clone(), + policy_sha256: mode.policy.sha256().into(), + window_count: w.session.rows.len(), + transition_limit: mode.policy.transition_limit(), + saved_receiver_ns: w.last_now_ns, + metadata_kind: full_run::META_KIND, + full_write_grant: None, + full_frozen_predecessor: mode + .frozen + .as_ref() + .map(|r| r.body.accepted_train_commitment.clone()), + }) +} +pub(super) fn is_new_full_run(p: &ValidatedTemporalPrefix) -> Result { + let e = parse(p.bytes())?; + if !matches!(e.payload.stage, FullStage::Train { .. }) { + return Ok(false); + } + let (prefix, _, _) = old_projection(e.payload)?; + let bytes = old_bytes(prefix)?; + let old = super::validated_prefix(&bytes, p.descriptor(), p.saved_receiver_ns())?; + super::is_new_empty_run(&old) +} +pub(super) fn validate_full_extension( + old: &ValidatedTemporalPrefix, + new: &ValidatedTemporalPrefix, +) -> Result<(), String> { + if old.run != new.run + || old.policy_sha256 != new.policy_sha256 + || old.transition_limit != new.transition_limit + || old.descriptor.id != new.descriptor.id + || old.descriptor.owner_scope != new.descriptor.owner_scope + || old.descriptor.installation_id != new.descriptor.installation_id + || old.descriptor.context_digest != new.descriptor.context_digest + || old.descriptor.created_at_ms != new.descriptor.created_at_ms + || old.descriptor.expires_at_ms != new.descriptor.expires_at_ms + || new.descriptor.saved_at_ms < old.descriptor.saved_at_ms + || new.saved_receiver_ns < old.saved_receiver_ns + { + return Err("temporal_commit_identity_or_time_changed".into()); + } + let a = parse(old.bytes())?; + let b = parse(new.bytes())?; + if serde_json::to_vec(&a.payload.policy).map_err(|_| "temporal_encoding")? + != serde_json::to_vec(&b.payload.policy).map_err(|_| "temporal_encoding")? + { + return Err("temporal_full_run_policy_changed".into()); + } + if old.bytes() == new.bytes() { + return Ok(()); + } + match (&a.payload.stage, &b.payload.stage) { + (FullStage::Train { .. }, FullStage::Train { .. }) => { + let (ap, _, _) = old_projection(a.payload)?; + let (bp, _, _) = old_projection(b.payload)?; + let ab = old_bytes(ap)?; + let bb = old_bytes(bp)?; + let av = super::validated_prefix(&ab, old.descriptor(), old.saved_receiver_ns())?; + let bv = super::validated_prefix(&bb, new.descriptor(), new.saved_receiver_ns())?; + super::validate_extension(&av, &bv) + } + (FullStage::Train { bridge }, FullStage::Frozen { fit, train_seals }) => { + if bridge.disposition != bridge::Disposition::Complete + || bridge.seals != *train_seals + || fit.body.accepted_train_prefix_sha256 != bytes_hash(old.bytes()) + || serde_json::to_vec(&a.payload.accepted).map_err(|_| "temporal_encoding")? + != serde_json::to_vec(&b.payload.accepted).map_err(|_| "temporal_encoding")? + || a.payload.common.binding.schedule_sha256 + != b.payload.common.binding.schedule_sha256 + || a.payload.common.run != b.payload.common.run + || a.payload.common.ledger != b.payload.common.ledger + || a.payload.common.mapping.is_some() + || b.payload.common.mapping.is_some() + || a.payload.common.episode != b.payload.common.episode + || b.payload.common.last_now_ns < a.payload.common.last_now_ns + { + return Err("temporal_frozen_transition_binding".into()); + } + for (id, seq, time) in &a.payload.common.source_high { + if !b + .payload + .common + .source_high + .iter() + .any(|(s, q, t)| s == id && q >= seq && t >= time) + { + return Err("temporal_commit_floor_regressed".into()); + } + } + // General descriptor/policy identity is also enforced by store + // commitment preparation; the raw structure grants no write. + Ok(()) + } + _ => Err("temporal_full_run_transition_unsupported".into()), + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs index b97d4a2d0d..b9e70b6265 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs @@ -6,7 +6,10 @@ use crate::room_learner::temporal_model_input::managed::execution::checkpoint::{ is_new_empty_run, validate_extension, validated_prefix, ValidatedTemporalPrefix, }; use sha2::{Digest, Sha256}; +// Storage directory/AAD framing stay unchanged. The schema field below is +// authenticated version DATA; new kind2 never creates a parallel namespace. const KIND: &str = "temporal-prefix-commitment-v1"; +const FULL_KIND: &str = "temporal-prefix-commitment-v2"; const LOCK: &str = "temporal-prefix.lock"; const HEAD: &str = "temporal-head.bin"; const MAGIC: &[u8; 8] = b"RVTCPR01"; @@ -60,6 +63,18 @@ pub(crate) struct TemporalToken { id: String, commitment: String, } +/// Read-only commitment DATA. Deserializing this is never a TemporalToken or +/// store/producer permission. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(crate) struct TemporalCommitmentData { + pub generation:u64,pub revision:usize,pub run:String,pub id:String,pub commitment:String, +} +impl TemporalToken { + pub(crate) fn data(&self)->TemporalCommitmentData{ + TemporalCommitmentData{generation:self.generation,revision:self.revision,run:self.run.clone(),id:self.id.clone(),commitment:self.commitment.clone()} + } +} #[derive(Clone)] pub(crate) struct TemporalIntent { expected: Option, @@ -208,13 +223,13 @@ impl RecoveryStore { let p = self.temporal_open(domain, &bytes, SMALL)?; let m: Meta = serde_json::from_slice(&p).map_err(|_| "temporal_metadata_schema")?; if serde_json::to_vec(&m).map_err(|_| "temporal_metadata_encoding")? != p - || m.schema != KIND + || !matches!(m.schema.as_str(),KIND|FULL_KIND) || m.transition_limit > MAX_TRANSITIONS || m.transition_limit < 2 || m.revision >= m.transition_limit || !id(&m.run) || !hex(&m.policy_sha256) - || m.window_count > 64 + || m.window_count > if m.schema==FULL_KIND{128}else{64} { return Err("temporal_metadata_binding".into()); } @@ -398,7 +413,7 @@ impl RecoveryStore { let m: Meta = serde_json::from_slice(&bytes[12..12 + n]).map_err(|_| "temporal_payload_metadata")?; if serde_json::to_vec(&m).map_err(|_| "temporal_metadata_encoding")? != bytes[12..12 + n] - || m.schema != KIND + || !matches!(m.schema.as_str(),KIND|FULL_KIND) || m.retired_at_ms.is_some() { return Err("temporal_payload_metadata".into()); @@ -413,6 +428,7 @@ impl RecoveryStore { } let p = validated_prefix(&plain, &m.descriptor, now_ns)?; if p.run() != m.run + ||p.metadata_kind()!=m.schema || p.policy_sha256() != m.policy_sha256 || p.window_count() != m.window_count || p.transition_limit() != m.transition_limit @@ -454,6 +470,7 @@ impl RecoveryStore { prefix: &ValidatedTemporalPrefix, now_ns: u64, ) -> Result { + if !prefix.authorizes_write(expected){return Err("temporal_full_run_producer_write_grant_required".into());} let _serial = self .serial .lock() @@ -529,7 +546,7 @@ impl RecoveryStore { Ok(TemporalIntent { expected: expected.cloned(), meta: Meta { - schema: KIND.into(), + schema: prefix.metadata_kind().into(), generation, revision, descriptor: prefix.descriptor().clone(), @@ -731,6 +748,7 @@ impl RecoveryStore { if meta.retired_at_ms.is_some() || p.descriptor() != &meta.descriptor || p.run() != meta.run + ||p.metadata_kind()!=meta.schema || p.policy_sha256() != meta.policy_sha256 || Some(hash(p.bytes())).as_ref() != meta.payload_sha256.as_ref() { From ea46baf31751127e67c28ad84391754433a5abfe Mon Sep 17 00:00:00 2001 From: Codex for Dragan Spiridonov Date: Fri, 6 Nov 2026 23:49:27 +0100 Subject: [PATCH 073/100] fix(learner): preserve legacy budget identity and bound fresh Train runs Keep historical329 generations unchanged, admit only authenticated negative retirement329, and use an explicit new343 Train profile for the complete 128-gap schedule. Public activation and C4 Kind2 composition stay excluded. Authored by Codex for Dragan Spiridonov. --- .../src/room_learner/lifecycle_temporal.rs | 9 + .../room_learner/lifecycle_temporal_tests.rs | 220 ++++++++++++++++++ .../temporal_budget_profile_tests.rs | 41 ++++ .../room_learner/temporal_capture_quality.rs | 57 ++++- .../src/room_learner/temporal_checkpoint.rs | 30 +-- .../temporal_recovery_commitment.rs | 30 ++- .../temporal_recovery_commitment_tests.rs | 168 +++++++++++++ 7 files changed, 531 insertions(+), 24 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_budget_profile_tests.rs diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs index 825ce8b72e..708aaa52a1 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs @@ -190,6 +190,15 @@ impl RoomLearningRuntime { }, ) } + /// Explicit trusted new-Train entry point. No public route or profile flag. + /// Active old generations must retire first; only authenticated retired + /// metadata can chain a new343 generation in the SAME store. + fn configure_fresh_train_343_capture(&self,expected:&MutationGuard,context:CaptureContext,run:String,now:u64)->Result<(),String>{ + self.configure_temporal_capture_with(expected,context,run,now,|schedule,run,fit,now,_|{ + let store=self.recovery_store.clone().ok_or("temporal_recovery_unavailable")?; + TemporalTrainCapture::prepare_async_fresh_train_343(schedule,run,fit,now,store) + }) + } fn configure_temporal_capture_with( &self, expected: &MutationGuard, diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs index d3d299875b..130a83b631 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs @@ -1169,3 +1169,223 @@ fn c2_quiescent_start_waiter_handles_slow_recertification_without_observation_cr assert_eq!(status["lead_in_remaining_ms"],5000,"lead-in must start only at actual authorization ACK"); rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); } + +// READ-ONLY production behavior probe: no budget/policy/clock/threshold changes. +// Private token Debug is parsed ONLY in this synthetic test to read the actual +// authenticated header revision without adding a production metadata accessor. +fn budget_repro_store_observation(rt:&RoomLearningRuntime,now:u64)->(usize,usize,usize,usize,String){ + let loaded=rt.temporal_store().unwrap().load_temporal(now).unwrap().unwrap(); + let token= format!("{:?}",loaded.token); + let revision=token.split("revision: ").nth(1).unwrap().split(',').next().unwrap().parse::().unwrap(); + let prefix=loaded.prefix.unwrap();assert_eq!(prefix.transition_limit(),329); + let value:Value=serde_json::from_slice(prefix.bytes()).unwrap(); + let rows=value["payload"]["rows"].as_array().unwrap().len(); + let gaps=value["payload"]["ledger"]["gaps"].as_array().unwrap().len(); + let bridge=&value["payload"]["bridge"]; + let parts=bridge["parts"].as_array().unwrap().len()+bridge["seals"].as_array().unwrap().iter().map(|s|s["parts"].as_array().unwrap().len()).sum::(); + (revision,rows,gaps,parts,format!("{:x}",::digest(prefix.bytes()))) +} +#[test] +fn legacy_329_actual_runtime_saturation_retires_and_explicit_fresh_train_chains_343(){ + let dir=tempfile::tempdir().unwrap();let(rt,mut now)=configured_async(dir.path());let original=snapshot(&rt,now)["descriptor"].clone(); + assert_eq!(budget_repro_store_observation(&rt,now).0,0); + //127 zero-observation pauses followed by127 resumes: initialsegment0 is + //paused, segments1..126 paused, finalsegment127 will actually collectEmpty. + for cycle in 0..127usize{ + now=async_lead_end(&rt,now);rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap(); + rt.pause_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap(); + now=settle_async_boundary(&rt,now); + assert_eq!(snapshot(&rt,now)["stage"],"paused");assert_eq!(snapshot(&rt,now)["durable_windows"],0); + if cycle==0 || cycle==126{ + let actual=budget_repro_store_observation(&rt,now); + assert_eq!(actual.0,2*(cycle+1));assert_eq!(actual.2,cycle); + println!("SYNTHETIC_BUDGET_ZERO_PREFIX cycle={} authenticated_revision={} gaps={} parts={}",cycle+1,actual.0,actual.2,actual.3); + } + } + let mut seq=0;let mut before_failure=None; + for phase in 0..7u64{ + if phase==6{ + //Newlast-episodeinitialsegment has no newgap; pause it once so the + //following realresume consumes the128thGLOBALgap. No laterstart. + now=async_lead_end(&rt,now);rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap(); + rt.pause_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap();now=settle_async_boundary(&rt,now); + } + loop{ + let status=snapshot(&rt,now); + match status["stage"].as_str().unwrap(){ + "ready"|"paused"=>{now=async_lead_end(&rt,now);rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap();}, + "needs_confirmation"=>break, + "failed"=>panic!("STATIC_TRACE_REFUTED earlierphase failure phase={phase} status={status}"),_=>{} + } + if status["durable_windows"]==59 && before_failure.is_none(){ + let observed=budget_repro_store_observation(&rt,now); + assert_eq!((observed.0,observed.1,observed.2,observed.3),(328,59,128,135)); + println!("SYNTHETIC_BUDGET_LAST_HEAD revision={} windows={} gaps={} parts={}",observed.0,observed.1,observed.2,observed.3); + before_failure=Some(observed); + } + let pending=snapshot(&rt,now)["durability_pending"]==true;seq+=1; + for source in 1..=3{ + match rt.ingest_native(&packet(source,seq,now),now){ + Ok(())=>{},Err(error)if error=="temporal_storage_busy_pause"=>break, + Err(error)if error=="temporal_transition_budget"=>{ + assert_eq!(phase,6,"wrong phase hit budget");let before=before_failure.as_ref().expect("no observed last valid head"); + let after=budget_repro_store_observation(&rt,now);assert_eq!(&after,before,"failed operation changed authenticated durable prefix"); + let state=snapshot(&rt,now);assert_eq!(state["durable_windows"],59);assert!(state["retained_rows"].as_u64().unwrap()>=60); + assert_eq!(state["stage"],"blocked");assert_eq!(state["descriptor"]["id"],original["id"]);assert_eq!(state["descriptor"]["expires_at_ms"],original["expires_at_ms"]); + println!("SYNTHETIC_ACTUAL_TRANSITION_BUDGET_CONFIRMED phase_index=6 failed_window_in_episode=4 proposed_revision=329 accepted_states=329 last_authenticated_revision={} last_durable_windows={} global_gaps={} quality_parts={} current_stage={} last_prefix_sha256_unchanged=true",after.0,after.1,after.2,after.3,state["stage"]); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); + let store=rt.temporal_store().unwrap(); + let token=store.load_temporal(now).unwrap().unwrap().token; + let original_prefix=store.load_temporal(now).unwrap().unwrap().prefix.unwrap(); + // Same Runtime generated authenticated revision328. No + // synthetic metadata/counter injection creates the head. + assert!(store.fresh_train_genesis_predecessor(now).is_err()); + // Independent current-time branch of the SAME real + // authenticated saturated prefix. Writer already drained; + // copying only this synthetic TempDir creates no live data. + let live=tempfile::tempdir().unwrap();let live_root=live.path().join("fixture"); + copy_synthetic_budget_fixture(dir.path(),&live_root); + {let live_rt=runtime(&live_root,now);let live_store=live_rt.temporal_store().unwrap(); + let current=live_store.load_temporal(now).unwrap().unwrap(); + let live_intent=live_store.prepare_retire_temporal(¤t.token,now,false).unwrap(); + let terminal=live_store.publish_temporal(&live_intent,None,now).unwrap(); + assert!(format!("{terminal:?}").contains("revision: 329,")); + assert!(live_store.load_temporal(now).unwrap().unwrap().prefix.is_none()); + assert_eq!(live_store.repair_same_temporal(&live_intent,None,now).unwrap(),terminal); + } + let expired=original_prefix.descriptor().expires_at_ms*1_000_000; + assert_eq!(store.load_temporal(expired).err().as_deref(),Some("recovery_expired")); + let inspection=store.inspect_temporal_retirement(expired).unwrap().unwrap(); + assert!(inspection.expired()); + let intent=store.prepare_inspected_retirement(&inspection,expired,true).unwrap(); + let retired=store.publish_temporal(&intent,None,expired).unwrap(); + assert!(format!("{retired:?}").contains("generation: 0, revision: 329,")); + // A lost ACK can only repair the exact same terminal, + // never consume revision330 or restore positive rows. + assert_eq!(store.repair_same_temporal(&intent,None,expired).unwrap(),retired); + let same=store.prepare_retire_temporal(&retired,expired,false).unwrap();assert_eq!(store.publish_temporal(&same,None,expired).unwrap(),retired); + assert!(store.load_temporal(expired).unwrap().unwrap().prefix.is_none()); + assert_eq!(store.fresh_train_genesis_predecessor(expired).unwrap(),Some(retired.clone())); + assert!(store.prepare_retire_temporal(&token,expired,false).is_err()); + drop(rt); + let fresh_at=expired+1_000_000; + let fresh=runtime(dir.path(),fresh_at); + fresh.configure_fresh_train_343_capture(&fresh.mutation_guard().unwrap(),context(),"synthetic-fresh-train-343".into(),fresh_at).unwrap(); + let fresh_now=settle_async_boundary(&fresh,fresh_at); + let loaded=fresh.temporal_store().unwrap().load_temporal(fresh_now).unwrap().unwrap(); + assert_eq!(loaded.prefix.as_ref().unwrap().transition_limit(),343); + assert!(format!("{:?}",loaded.token).contains("generation: 1, revision: 0,")); + assert_ne!(loaded.prefix.as_ref().unwrap().descriptor().id,original_prefix.descriptor().id); + assert_eq!(snapshot(&fresh,fresh_now)["durable_windows"],0); + assert_eq!(snapshot(&fresh,fresh_now)["stage"],"ready"); + fresh.cancel_temporal_durability(&fresh.mutation_guard().unwrap()).unwrap(); + println!("SYNTHETIC_LEGACY_SATURATED_RETIRE_AND_FRESH_343_PASS expired_samples_unavailable=true terminal_revision=329 same_intent_repair=true fresh_rows=0");return; + }, + Err(error)=>panic!("STATIC_TRACE_REFUTED earlierguard phase={phase} error={error}"), + } + } + now+=PERIOD;if pending{std::thread::sleep(std::time::Duration::from_nanos(PERIOD));} + } + let guard=rt.mutation_guard().unwrap();let challenge=rt.temporal_confirmation_challenge(&guard).unwrap();rt.confirm_temporal_phase(&guard,&challenge,now).unwrap();now=settle_async_boundary(&rt,now); + let actual=budget_repro_store_observation(&rt,now);println!("SYNTHETIC_BUDGET_PHASE_SEAL phase={} authenticated_revision={} windows={} gaps={} parts={}",phase+1,actual.0,actual.1,actual.2,actual.3); + } + panic!("STATIC_TRACE_REFUTED complete path did not hit predicted old329 budget"); +} + +// Fresh-profile complete upper-bound journey: original 128 gap / 135 part +// schedule plus a legitimate deferred pause before EACH of the seven seals. +// All counters below come from authenticated store readback, not a mock limit. +#[test] +fn fresh_343_actual_runtime_all_128_gaps_seven_terminal_pauses_and_retirement_fit_budget(){ + let dir=tempfile::tempdir().unwrap();let rt=runtime(dir.path(),START); + rt.configure_fresh_train_343_capture(&rt.mutation_guard().unwrap(),context(),"synthetic-full-343".into(),START).unwrap(); + let mut now=settle_async_boundary(&rt,START);let original=snapshot(&rt,now)["descriptor"].clone(); + for _ in 0..127{ + now=async_lead_end(&rt,now);rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap(); + rt.pause_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap();now=settle_async_boundary(&rt,now); + assert_eq!(snapshot(&rt,now)["durable_windows"],0); + } + let mut seq=0;let mut terminal_pauses=0; + for phase in 0..7u64{ + if phase==6{ + now=async_lead_end(&rt,now);rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap(); + rt.pause_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap();now=settle_async_boundary(&rt,now); + } + let target=16+phase*8;let mut terminal_pause=false;let supervision=std::time::Instant::now(); + loop{ + assert!(supervision.elapsed(){now=async_lead_end(&rt,now);rt.tick_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap();}, + "needs_confirmation"=>break,"failed"|"blocked"=>panic!("fresh343 phase={phase} {status}"),_=>{} + } + let pending=snapshot(&rt,now)["durability_pending"]==true;seq+=1; + for source in 1..=3{ + match rt.ingest_native(&packet(source,seq,now),now){Ok(())=>{},Err(e)if e=="temporal_storage_busy_pause"=>break,Err(e)=>panic!("fresh343 phase={phase} ingress={e}")} + } + let after=snapshot(&rt,now); + if !terminal_pause && after["retained_rows"]==target && after["durability_pending"]==true{ + // Exact real pending final WINDOW. pause does not consume its + // ACK; no new raw data, target slide, or fake duration is used. + rt.pause_temporal_capture(&rt.mutation_guard().unwrap(),now).unwrap(); + terminal_pause=true;terminal_pauses+=1; + now=settle_async_boundary(&rt,now); + // Paused presentation precedes NeedsConfirmation; the real + // challenge below is the authority for complete-phase credit. + assert_eq!(snapshot(&rt,now)["durable_windows"],target);break; + }else{now+=PERIOD;if pending{std::thread::sleep(std::time::Duration::from_nanos(PERIOD));}} + } + assert!(terminal_pause,"did not exercise terminal deferred pause"); + let guard=rt.mutation_guard().unwrap();let challenge=rt.temporal_confirmation_challenge(&guard).unwrap(); + rt.confirm_temporal_phase(&guard,&challenge,now).unwrap();now=settle_async_boundary(&rt,now); + println!("SYNTHETIC_FRESH343_SEAL phase={} rows={} terminal_pauses={}",phase+1,target,terminal_pauses); + } + assert_eq!(snapshot(&rt,now)["sealed_train_episodes"],7);assert_eq!(snapshot(&rt,now)["durable_windows"],64); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); + let store=rt.temporal_store().unwrap();let loaded=store.load_temporal(now).unwrap().unwrap();let prefix=loaded.prefix.unwrap(); + assert_eq!(prefix.transition_limit(),343);assert_eq!(prefix.window_count(),64); + let value:Value=serde_json::from_slice(prefix.bytes()).unwrap(); + assert_eq!(value["payload"]["ledger"]["gaps"].as_array().unwrap().len(),128); + let bridge=&value["payload"]["bridge"]; + let parts=bridge["parts"].as_array().unwrap().len()+bridge["seals"].as_array().unwrap().iter().map(|seal|seal["parts"].as_array().unwrap().len()).sum::(); + assert_eq!(parts,135); + assert_eq!(prefix.descriptor().id,original["id"].as_str().unwrap()); + assert_eq!(prefix.descriptor().expires_at_ms,original["expires_at_ms"].as_u64().unwrap()); + let before=format!("{:?}",loaded.token);assert!(before.contains("revision: 341,"),"{before}"); + let intent=store.prepare_retire_temporal(&loaded.token,now,false).unwrap();let terminal=store.publish_temporal(&intent,None,now).unwrap(); + assert!(format!("{terminal:?}").contains("revision: 342,"));assert!(store.load_temporal(now).unwrap().unwrap().prefix.is_none()); + assert_eq!(store.repair_same_temporal(&intent,None,now).unwrap(),terminal); + println!("SYNTHETIC_FRESH343_COMPLETE authenticated_states=343 last_revision=342 rows=64 gaps=128 quality_parts=135 terminal_pauses=7 retired=true"); +} + +#[test] +fn fresh343_actual_genesis_fixture_for_frozen_old_runtime_consumer(){ + let dir=tempfile::tempdir().unwrap();let rt=runtime(dir.path(),START); + rt.configure_fresh_train_343_capture(&rt.mutation_guard().unwrap(),context(),"synthetic-old-reader-fixture".into(),START).unwrap(); + let now=settle_async_boundary(&rt,START);rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); + let loaded=rt.temporal_store().unwrap().load_temporal(now).unwrap().unwrap();assert_eq!(loaded.prefix.unwrap().transition_limit(),343); + if let Some(output)=std::env::var_os("RUVIEW_2473_SYNTHETIC_FIXTURE_OUTPUT"){ + fn copy_private(from:&Path,to:&Path){ + std::fs::create_dir(to).unwrap(); + #[cfg(unix)]{use std::os::unix::fs::PermissionsExt;std::fs::set_permissions(to,std::fs::Permissions::from_mode(0o700)).unwrap();} + for e in std::fs::read_dir(from).unwrap(){let e=e.unwrap();let m=std::fs::symlink_metadata(e.path()).unwrap();assert!(!m.file_type().is_symlink()); + if m.is_dir(){copy_private(&e.path(),&to.join(e.file_name()));}else{assert!(m.is_file());std::fs::copy(e.path(),to.join(e.file_name())).unwrap(); + #[cfg(unix)]{use std::os::unix::fs::PermissionsExt;std::fs::set_permissions(to.join(e.file_name()),std::fs::Permissions::from_mode(0o600)).unwrap();} + } + } + } + let output=std::path::PathBuf::from(output);std::fs::create_dir_all(&output).unwrap();copy_private(dir.path(),&output.join("fresh343")); + std::fs::write(output.join("fresh343.json"),serde_json::to_vec(&json!({"now_ns":now,"owner_scope":context().owner_scope,"installation_id":context().installation_id})).unwrap()).unwrap(); + } +} + +// Synthetic temp fixture only: no symbolic links, public state or user profile. +fn copy_synthetic_budget_fixture(from:&Path,to:&Path){ + std::fs::create_dir(to).unwrap(); + #[cfg(unix)]{use std::os::unix::fs::PermissionsExt;std::fs::set_permissions(to,std::fs::Permissions::from_mode(0o700)).unwrap();} + for entry in std::fs::read_dir(from).unwrap(){let entry=entry.unwrap();let m=std::fs::symlink_metadata(entry.path()).unwrap();assert!(!m.file_type().is_symlink()); + if m.is_dir(){copy_synthetic_budget_fixture(&entry.path(),&to.join(entry.file_name()));}else{assert!(m.is_file());std::fs::copy(entry.path(),to.join(entry.file_name())).unwrap(); + #[cfg(unix)]{use std::os::unix::fs::PermissionsExt;std::fs::set_permissions(to.join(entry.file_name()),std::fs::Permissions::from_mode(0o600)).unwrap();} + } + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_budget_profile_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_budget_profile_tests.rs new file mode 100644 index 0000000000..12e6565a2f --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_budget_profile_tests.rs @@ -0,0 +1,41 @@ +//! Synthetic profile/hash compatibility; the fresh constructor remains private. +use super::*; +use super::super::checkpoint; +const OLD_LITERAL:&str="native-receiver-whole-train-period-v1;all-selected-packets;existing-cadence-gap-arrival-skew;confirmed-durable-seals;no-fit-authority"; +#[test] +fn historical_constructor_and_seal_hash_keep_exact_bytes(){ + let old=BridgeState::new();assert_eq!(old.policy,OLD_LITERAL);assert_eq!(BRIDGE_POLICY,OLD_LITERAL); + let expected=format!("{{\"policy\":\"{OLD_LITERAL}\",\"disposition\":\"ready\",\"seals\":[],\"parts\":[]}}"); + assert_eq!(serde_json::to_vec(&old).unwrap(),expected.as_bytes()); + let seal=TrainSeal{episode:0,trial_id:"synthetic-trial".into(),context_sha256:"a".repeat(64),schedule_sha256:"b".repeat(64),row_prefix_sha256:"c".repeat(64),quality_sha256:"d".repeat(64),parts:vec![],confirmed_at_ns:1,previous_sha256:None,sha256:String::new()}; + // Independent frozen tuple golden, preserving the historical domain. + assert_eq!(seal.digest(TrainBudgetProfile::Historical329).unwrap(),"467e2740fd4a20ed6e09af36071a5b816522c20888936be1e976b3b8f4b4f4c1"); + assert_ne!(seal.digest(TrainBudgetProfile::FreshTrain343).unwrap(),seal.digest(TrainBudgetProfile::Historical329).unwrap()); +} +#[test] +fn generic_and_stored_historical_prefixes_remain329_and_byte_roundtrip(){ + let(_dir,_store,mut w)=super::super::tests::setup(); + for bridge in [None,Some(BridgeState::new())]{ + w.bridge=bridge; + let bytes=checkpoint::encode(&w).unwrap();let prefix=checkpoint::validated_prefix(&bytes,&w.descriptor,w.last_now_ns).unwrap(); + assert_eq!(prefix.transition_limit(),329); + let restored=checkpoint::decode(&bytes,&w.descriptor,w.last_now_ns).unwrap();assert_eq!(checkpoint::encode(&restored).unwrap(),bytes); + if w.bridge.is_some(){assert_eq!(prefix.policy_sha256(),hash(&(w.session.plan_sha256(),OLD_LITERAL)).unwrap());} + } +} +#[test] +fn fresh_train_profile343_is_explicit_and_cannot_upgrade_existing_generation(){ + let(_dir,_store,mut w)=super::super::tests::setup();w.bridge=Some(BridgeState::new()); + let old_bytes=checkpoint::encode(&w).unwrap();let old=checkpoint::validated_prefix(&old_bytes,&w.descriptor,w.last_now_ns).unwrap(); + w.bridge=Some(BridgeState::fresh_train_343()); + let new_bytes=checkpoint::encode(&w).unwrap();let fresh=checkpoint::validated_prefix(&new_bytes,&w.descriptor,w.last_now_ns).unwrap(); + assert_eq!(old.transition_limit(),329);assert_eq!(fresh.transition_limit(),343);assert_ne!(old.policy_sha256(),fresh.policy_sha256()); + assert!(checkpoint::validate_extension(&old,&fresh).is_err()); + assert_eq!(BridgeState::new().policy,OLD_LITERAL,"generic/C4 constructor must not flip"); +} +#[test] +fn unknown_stored_profile_and_injected_budget_never_choose_larger_capacity(){ + let(_dir,_store,mut w)=super::super::tests::setup();w.bridge=Some(BridgeState::new()); + w.bridge.as_mut().unwrap().policy="unknown-budget-profile".into();let bytes=checkpoint::encode(&w).unwrap();assert!(checkpoint::validated_prefix(&bytes,&w.descriptor,w.last_now_ns).is_err()); + w.bridge=Some(BridgeState::fresh_train_343());let bytes=checkpoint::encode(&w).unwrap();let mut v:serde_json::Value=serde_json::from_slice(&bytes).unwrap();v["payload"]["transition_limit"]=serde_json::json!(512);assert!(checkpoint::decode(&serde_json::to_vec(&v).unwrap(),&w.descriptor,w.last_now_ns).is_err()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs index da1e26a1af..9efe052176 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs @@ -5,6 +5,24 @@ use crate::room_learner::temporal_acquisition::WindowInputPolicy; use serde_json::{json, Value}; pub(super) const BRIDGE_POLICY: &str = "native-receiver-whole-train-period-v1;all-selected-packets;existing-cadence-gap-arrival-skew;confirmed-durable-seals;no-fit-authority"; +// Explicit NEW Train genesis only. The generic constructor and old literal +// remain historical329; restore never upgrades a stored generation. +pub(super) const FRESH_TRAIN_343_POLICY:&str="native-receiver-whole-train-period-v2;all-selected-packets;existing-cadence-gap-arrival-skew;confirmed-durable-seals;train-budget343;no-fit-authority"; +#[derive(Clone,Copy,Debug,PartialEq,Eq)] +pub(super) enum TrainBudgetProfile { Historical329, FreshTrain343 } +impl TrainBudgetProfile { + pub(super) fn from_stored(policy:&str)->Result{match policy{ + BRIDGE_POLICY=>Ok(Self::Historical329),FRESH_TRAIN_343_POLICY=>Ok(Self::FreshTrain343), + _=>Err("temporal_train_profile_unknown".into()), + }} + pub(super) fn policy(self)->&'static str{match self{Self::Historical329=>BRIDGE_POLICY,Self::FreshTrain343=>FRESH_TRAIN_343_POLICY}} + pub(super) fn transition_limit(self,schedule:&ManagedScheduleContract)->Result{ + let episodes=schedule.counts().train_episodes; + let old=schedule.counts().train_windows.checked_add(2*128).and_then(|n|n.checked_add(episodes)).and_then(|n|n.checked_add(2)).ok_or("temporal_transition_capacity")?; + match self{Self::Historical329=>Ok(old),Self::FreshTrain343=>old.checked_add(2*episodes).ok_or("temporal_transition_capacity".into())} + } +} + #[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq, Eq)] #[serde(rename_all = "snake_case")] pub(super) enum Disposition { @@ -160,9 +178,9 @@ pub(super) struct TrainSeal { pub(super) sha256: String, } impl TrainSeal { - fn digest(&self) -> Result { + fn digest(&self,profile:TrainBudgetProfile) -> Result { hash(&( - BRIDGE_POLICY, + profile.policy(), self.episode, &self.trial_id, &self.context_sha256, @@ -192,6 +210,10 @@ impl BridgeState { parts: vec![], } } + pub(super) fn fresh_train_343()->Self{ + Self{policy:FRESH_TRAIN_343_POLICY.into(),..Self::new()} + } + pub(super) fn profile(&self)->Result{TrainBudgetProfile::from_stored(&self.policy)} pub(super) fn validate( &self, schedule: &ManagedScheduleContract, @@ -208,8 +230,8 @@ impl BridgeState { .iter() .take_while(|e| e.role == DataRole::Train) .count(); - if self.policy != BRIDGE_POLICY - || self.seals.len() > train + let profile=self.profile()?; + if self.seals.len() > train || episode != self.seals.len() || episode > train || self.parts.len() > 128 @@ -232,7 +254,7 @@ impl BridgeState { || s.confirmed_at_ns < last_confirm || s.confirmed_at_ns > now || s.previous_sha256 != previous - || s.sha256 != s.digest()? + || s.sha256 != s.digest(profile)? || s.quality_sha256 != hash(&s.parts)? || s.row_prefix_sha256 != hash(&(&rows[..count], &evidence[..count], &ledger.spans[..count]))? @@ -463,6 +485,23 @@ impl TemporalTrainCapture { w.persist_initial(store) }) } + /// Explicit NEW Train genesis only. Existing generic/old constructors and + /// cold restore retain their stored profile; C4 never inherits this choice. + pub(crate) fn prepare_async_fresh_train_343( + schedule:ManagedScheduleContract,run:String,config:FitConfig,now:u64, + store:std::sync::Arc, + )->Result<(Self,Option),String>{ + let predecessor=store.fresh_train_genesis_predecessor(now)?; + let mut workflow=TemporalWorkflow::prepare(schedule,run,config,now)?; + let schedule=workflow.schedule()?; + workflow.bridge=Some(BridgeState::fresh_train_343()); + workflow.commit_token=predecessor; + let writer=std::sync::Arc::new(durability::Writer::new(store.clone(),now)?); + workflow.async_state=Some(durability::AsyncState::new(writer)); + let error=workflow.persist_initial(&store).err(); + if error.is_some(){workflow.stage=Stage::BlockedDurability;} + Ok((Self{workflow,schedule,failure:None,durable_seals:0},error)) + } pub(crate) fn async_start(&mut self,now:u64)->Result{ self.ensure_live()?; if self.workflow.async_state.is_none(){return Ok(false);} @@ -808,7 +847,7 @@ impl TemporalTrainCapture { let w = &self.workflow; let b = w.bridge.as_ref().ok_or("temporal_bridge_required")?; hash(&( - BRIDGE_POLICY, + b.profile()?.policy(), &w.descriptor.id, &w.session.run_id, w.episode, @@ -857,7 +896,7 @@ impl TemporalTrainCapture { previous_sha256: b.seals.last().map(|s| s.sha256.clone()), sha256: String::new(), }; - seal.sha256 = seal.digest()?; + seal.sha256 = seal.digest(b.profile()?)?; b.seals.push(seal); b.parts.clear(); w.episode += 1; @@ -958,3 +997,7 @@ impl Drop for TemporalTrainCapture { if let Some(state)=&self.workflow.async_state {let _=state.writer.drain();} } } + +#[cfg(test)] +#[path="temporal_budget_profile_tests.rs"] +mod budget_profile_tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs index 17d8f6edc4..d009cc361e 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs @@ -500,20 +500,10 @@ pub(crate) fn validated_prefix( ) -> Result { let v = decode(bytes, descriptor, now_ns)?; let schedule = v.schedule()?; - let train_episodes = schedule - .episodes() - .iter() - .filter(|e| e.role == DataRole::Train) - .count(); - // Each64 native Train window +2x128 run-global pause/resume gap barriers +7 Train segment - // boundaries + genesis/terminal states. Identical retries consume none. - let transition_limit = schedule - .counts() - .train_windows - .checked_add(2 * 128) - .and_then(|n| n.checked_add(train_episodes)) - .and_then(|n| n.checked_add(2)) - .ok_or("temporal_transition_capacity")?; + // Historical profiles retain their ORIGINAL limit/hash; only an explicit + // fresh-Train genesis can select the recognized343 budget profile. + let profile=match &v.bridge{Some(bridge)=>bridge.profile()?,None=>bridge::TrainBudgetProfile::Historical329}; + let transition_limit=profile.transition_limit(&schedule)?; if transition_limit > 512 { return Err("temporal_transition_capacity".into()); } @@ -522,7 +512,7 @@ pub(crate) fn validated_prefix( descriptor: descriptor.clone(), run: v.session.run_id.clone(), policy_sha256: if v.bridge.is_some() { - hash(&(v.session.plan_sha256(), bridge::BRIDGE_POLICY))? + hash(&(v.session.plan_sha256(), profile.policy()))? } else { v.session.plan_sha256().into() }, @@ -801,3 +791,13 @@ pub(crate) fn synthetic_floor_regression( let b = serde_json::to_vec(&e).unwrap(); validated_prefix(&b, &e.payload.descriptor, e.payload.last_now_ns).unwrap() } + +// Test-only typed fixture: create a NEW empty run and encode its explicit +// profile through the same strict codec; never upgrades an existing prefix. +#[cfg(test)] +pub(crate) fn synthetic_fresh_train343_empty_run(input:&ValidatedTemporalPrefix,created:u64)->ValidatedTemporalPrefix{ + let empty=synthetic_new_empty_run(input,created); + let mut workflow=decode(empty.bytes(),empty.descriptor(),empty.saved_receiver_ns()).unwrap(); + workflow.bridge=Some(super::bridge::BridgeState::fresh_train_343()); + let bytes=encode(&workflow).unwrap();validated_prefix(&bytes,&workflow.descriptor,workflow.last_now_ns).unwrap() +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs index b97d4a2d0d..722b2e3c95 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs @@ -70,6 +70,16 @@ pub(crate) struct TemporalLoaded { pub(crate) token: TemporalToken, pub(crate) prefix: Option, } +// NEGATIVE shape only, not an authentication primitive. Caller must first +// authenticate/canonical-validate metadata and enforce owner/clock/chain rules. +// Never accepts a positive payload or a generic limit+1 terminal. +fn legacy_saturated_terminal_shape(m:&Meta)->bool{ + m.schema==KIND && m.transition_limit==329 && m.revision==329 + && m.slot.is_none() && m.payload_sha256.is_none() + && m.previous_sha256.as_ref().is_some_and(|s|hex(s)) + && m.window_count<=64 + && m.retired_at_ms.is_some_and(|at|at>=m.descriptor.saved_at_ms && at==m.receiver_ns/1_000_000) +} struct Components { head: Option, latest: Option, @@ -211,7 +221,6 @@ impl RecoveryStore { || m.schema != KIND || m.transition_limit > MAX_TRANSITIONS || m.transition_limit < 2 - || m.revision >= m.transition_limit || !id(&m.run) || !hex(&m.policy_sha256) || m.window_count > 64 @@ -234,6 +243,10 @@ impl RecoveryStore { if t >= m.descriptor.saved_at_ms && t == m.receiver_ns / 1_000_000 => {} _ => return Err("temporal_metadata_state".into()), }; + // Only after AEAD/canonical metadata, known-kind/descriptor/owner and + // state/clock validation may the old-family NEGATIVE terminal occupy + // its one reserved boundary. Positive payloads stay strictly =m.transition_limit && !legacy_saturated_terminal_shape(&m){return Err("temporal_metadata_binding".into());} Ok(Some(m)) } fn temporal_components(&self, dir: &Path) -> Result { @@ -579,7 +592,7 @@ impl RecoveryStore { meta.payload_sha256 = None; meta.retired_at_ms = Some(now_ns / 1_000_000); meta.receiver_ns = now_ns; - if meta.revision >= meta.transition_limit { + if meta.revision >= meta.transition_limit && !legacy_saturated_terminal_shape(&meta) { return Err("temporal_transition_budget".into()); } } @@ -1203,6 +1216,19 @@ impl RecoveryStore { } self.prepare_retire_temporal(&inspection.token, now_ns, expiry_only) } + pub(crate) fn fresh_train_genesis_predecessor(&self,now_ns:u64)->Result,String>{ + let _serial=self.serial.lock().map_err(|_|"recovery_store_unavailable")?; + let locked=self.temporal_lock(false)?; + let c=self.temporal_components(&locked.directory)?; + match (c.head,c.latest){ + (None,None)=>Ok(None), + (Some(head),Some(latest)) if head==latest=>{ + if now_nsErr("temporal_head_witness_mismatch_or_pending".into()), + } + } pub(crate) fn recertify_temporal( &self, expected: &TemporalToken, diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment_tests.rs index 2161e7a4cf..0c47ea1413 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment_tests.rs @@ -985,3 +985,171 @@ fn metadata_only_cold_retirement_rejects_copied_foreign_owner_records() { .inspect_temporal_retirement(p[0].descriptor().expires_at_ms * 1_000_000) .is_err()); } + +#[test] +fn legacy_terminal_shape_is_known329_negative_only_and_not_a_loader_grant(){ + let descriptor=RecoveryDescriptor{schema:RECOVERY_SCHEMA.into(),id:"synthetic-terminal".into(),owner_scope:"a".repeat(64),installation_id:"synthetic-install".into(),context_digest:"b".repeat(64),created_at_ms:1,saved_at_ms:2,expires_at_ms:1+RECOVERY_TTL_MS}; + let terminal=Meta{schema:KIND.into(),generation:0,revision:329,descriptor,run:"synthetic-run".into(),policy_sha256:"c".repeat(64),transition_limit:329,window_count:59,receiver_ns:10_000_000,slot:None,payload_sha256:None,previous_sha256:Some("d".repeat(64)),retired_at_ms:Some(10)}; + assert!(legacy_saturated_terminal_shape(&terminal)); + for mode in 0..12{ + let mut bad=terminal.clone();match mode{ + 0=>bad.schema="unknown-kind".into(),1=>bad.transition_limit=343,2=>bad.transition_limit=512, + 3=>bad.revision=328,4=>bad.revision=330,5=>bad.slot=Some(0),6=>bad.payload_sha256=Some("e".repeat(64)), + 7=>bad.previous_sha256=None,8=>bad.previous_sha256=Some("invalid".into()),9=>bad.window_count=65, + 10=>bad.retired_at_ms=None,_=>bad.receiver_ns=11_000_000, + } + assert!(!legacy_saturated_terminal_shape(&bad),"shape mode{mode}"); + } + // Shape alone grants nothing. The wired loader/producer also enforce + // canonical AEAD metadata, owner, clock, exact head/token and witness chain. +} +// SYNTHETIC authenticated chain fixture for storage failpoints ONLY. Its +// counters do not establish positive sample admission; the independent actual +// Runtime128-gap companion suite is the positive reachability oracle. +fn synthetic_saturated_metadata_chain() -> (tempfile::TempDir,RecoveryStore,TemporalToken,u64){ + let(t,store,p)=setup();commit(&store,None,&p[0]); + let mut m=store.temporal_components(&store.temporal_dir()).unwrap().head.unwrap(); + assert_eq!(m.transition_limit,329); + for rev in 1..=328{ + m.previous_sha256=Some(m.sha().unwrap());m.revision=rev; + let plain=serde_json::to_vec(&m).unwrap(); + let path=store.temporal_dir().join(m.witness()); + let mut file=OpenOptions::new().write(true).create_new(true).open(&path).unwrap(); + #[cfg(unix)]{use std::os::unix::fs::PermissionsExt;file.set_permissions(fs::Permissions::from_mode(0o600)).unwrap();} + file.write_all(&store.temporal_seal("witness",&plain).unwrap()).unwrap();file.sync_all().unwrap(); + } + fs::write(store.temporal_dir().join(format!("slot-{}.bin",m.slot.unwrap())),store.temporal_payload_seal(&m,&p[0]).unwrap()).unwrap(); + fs::write(store.temporal_dir().join(HEAD),store.temporal_seal("head",&serde_json::to_vec(&m).unwrap()).unwrap()).unwrap(); + assert_eq!(store.temporal_components(&store.temporal_dir()).unwrap().head.as_ref(),Some(&m)); + let at=m.receiver_ns;(t,store,m.token().unwrap(),at) +} +#[test] +fn saturated_terminal_authenticated_writer_faults_require_exact_same_intent(){ + for point in ["before:head.stage:write","after:head.stage:write","after:head.stage:directory_fsync","after:witness:install","before:cleanup"]{ + let(_t,store,token,at)=synthetic_saturated_metadata_chain(); + let old=store.temporal_read(&store.temporal_dir().join(format!("witness-{:016x}-{:04x}.bin",token.generation,token.revision)),SMALL+36).unwrap().unwrap(); + let intent=store.prepare_retire_temporal(&token,at,false).unwrap();assert_eq!(intent.meta.revision,329);assert!(intent.meta.slot.is_none()); + let mut hook=|seen:&str|if seen==point{Err("synthetic-terminal-interrupted".into())}else{Ok(())}; + assert!(store.publish_temporal_with(&intent,None,at,false,&mut hook).is_err(),"{point}"); + let repaired=store.repair_same_temporal(&intent,None,at).unwrap();assert_eq!(repaired.revision,329); + assert!(store.load_temporal(at).unwrap().unwrap().prefix.is_none()); + assert_eq!(store.repair_same_temporal(&intent,None,at).unwrap(),repaired); + let same=store.prepare_retire_temporal(&repaired,at,false).unwrap();assert_eq!(same.meta.revision,329); + assert_eq!(store.temporal_read(&store.temporal_dir().join(format!("witness-{:016x}-{:04x}.bin",token.generation,token.revision)),SMALL+36).unwrap().unwrap(),old); + let mut forbidden=intent.clone();forbidden.meta.revision=330; + assert!(store.publish_temporal(&forbidden,None,at).is_err()); + } +} +#[test] +fn saturated_terminal_prehead_abort_and_corrupt_stage_never_become_acknowledgment(){ + let(_t,store,token,at)=synthetic_saturated_metadata_chain();let intent=store.prepare_retire_temporal(&token,at,false).unwrap(); + let mut hook=|point:&str|if point=="before:head.stage:rename"{Err("synthetic-pre-head-stop".into())}else{Ok(())}; + assert!(store.publish_temporal_with(&intent,None,at,false,&mut hook).is_err()); + let tickets=store.inspect_abortable_temporal(&token,at).unwrap();assert_eq!(tickets.len(),1); + assert_eq!(store.abort_temporal_pending(&tickets[0],at).unwrap().token,token); + // The abort only removes the uncommitted stage; authenticated tip is328. + assert_eq!(store.temporal_components(&store.temporal_dir()).unwrap().head.unwrap().revision,328); + assert!(store.publish_temporal_with(&intent,None,at,false,&mut hook).is_err()); + fs::write(store.temporal_dir().join("head.stage"),b"synthetic-corruption").unwrap(); + assert!(store.repair_same_temporal(&intent,None,at).is_err()); + assert!(store.fresh_train_genesis_predecessor(at).is_err()); + // Existing explicit negative abort may discard a corrupt UNCOMMITTED + // named stage. It is not a successful retirement/recovery ACK. The old + // stage ticket cannot erase changed bytes; a new hash-bound ticket is needed. + assert!(store.abort_temporal_pending(&tickets[0],at).is_err()); + let corrupt=store.inspect_abortable_temporal(&token,at).unwrap();assert_eq!(corrupt.len(),1); + let preserved=store.abort_temporal_pending(&corrupt[0],at).unwrap();assert_eq!(preserved.token,token); + assert!(preserved.prefix.is_some()); + assert_eq!(store.temporal_components(&store.temporal_dir()).unwrap().head.unwrap().revision,328); +} + +// Optional export is an independent OLD-binary compatibility test fixture, +// generated solely from this synthetic temp owner/key. No live profile reads. +#[test] +fn saturated_terminal_fixture_for_frozen_old_runtime_consumer(){ + let(t,store,token,at)=synthetic_saturated_metadata_chain(); + let intent=store.prepare_retire_temporal(&token,at,false).unwrap(); + let terminal=store.publish_temporal(&intent,None,at).unwrap();assert_eq!(terminal.revision,329); + assert!(store.load_temporal(at).unwrap().unwrap().prefix.is_none()); + if let Some(output)=std::env::var_os("RUVIEW_2473_SYNTHETIC_FIXTURE_OUTPUT"){ + fn copy_private(from:&Path,to:&Path){ + fs::create_dir(to).unwrap(); + #[cfg(unix)]{use std::os::unix::fs::PermissionsExt;fs::set_permissions(to,fs::Permissions::from_mode(0o700)).unwrap();} + for e in fs::read_dir(from).unwrap(){let e=e.unwrap();let m=fs::symlink_metadata(e.path()).unwrap();assert!(!m.file_type().is_symlink()); + if m.is_dir(){copy_private(&e.path(),&to.join(e.file_name()));}else{assert!(m.is_file());fs::copy(e.path(),to.join(e.file_name())).unwrap(); + #[cfg(unix)]{use std::os::unix::fs::PermissionsExt;fs::set_permissions(to.join(e.file_name()),fs::Permissions::from_mode(0o600)).unwrap();} + } + } + } + let output=PathBuf::from(output);fs::create_dir_all(&output).unwrap();let dest=output.join("terminal329");copy_private(t.path(),&dest); + fs::write(output.join("terminal329.json"),serde_json::to_vec(&serde_json::json!({"now_ns":at,"owner_scope":intent.meta.descriptor.owner_scope,"installation_id":intent.meta.descriptor.installation_id})).unwrap()).unwrap(); + } +} + +#[test] +fn authenticated_boundary_metadata_rejects_positive329_and_negative330(){ + for positive in [true,false]{ + let(_t,store,token,at)=synthetic_saturated_metadata_chain(); + let intent=store.prepare_retire_temporal(&token,at,false).unwrap(); + let terminal=store.publish_temporal(&intent,None,at).unwrap();assert_eq!(terminal.revision,329); + let mut inadmissible=intent.meta.clone(); + if positive{inadmissible.slot=Some(0);inadmissible.payload_sha256=Some("a".repeat(64));inadmissible.retired_at_ms=None;} + else{inadmissible.revision=330;inadmissible.previous_sha256=Some(intent.meta.sha().unwrap());} + // Actual canonical encryption under the synthetic owner key. This is + // not an outer-hash-only/corrupted-cipher rejection control. + let bytes=store.temporal_seal("head",&serde_json::to_vec(&inadmissible).unwrap()).unwrap(); + fs::write(store.temporal_dir().join(HEAD),bytes).unwrap(); + assert!(store.load_temporal(at).is_err());assert!(store.inspect_temporal_retirement(at).is_err()); + assert!(store.fresh_train_genesis_predecessor(at).is_err()); + } +} + +#[test] +fn boundary329_to_fresh343_interrupted_obsolete_gc_keeps_exact_new_chain(){ + use crate::room_learner::temporal_model_input::managed::execution::checkpoint::synthetic_fresh_train343_empty_run; + for cut in [1usize,329]{ + let(_t,store,token,at)=synthetic_saturated_metadata_chain(); + let initial=store.load_temporal(at).unwrap().unwrap().prefix.unwrap(); + let retirement=store.prepare_retire_temporal(&token,at,false).unwrap(); + let retired=store.publish_temporal(&retirement,None,at).unwrap();assert_eq!(retired.revision,329); + let old_head=fs::read(store.temporal_dir().join(HEAD)).unwrap(); + let fresh=synthetic_fresh_train343_empty_run(&initial,at/1_000_000+1);assert_eq!(fresh.transition_limit(),343); + let intent=store.prepare_temporal(Some(&retired),&fresh,now(&fresh)).unwrap(); + let mut hook=|point:&str|if point=="before:cleanup"{Err("synthetic-boundary-GC-interruption".into())}else{Ok(())}; + assert!(store.publish_temporal_with(&intent,Some(&fresh),now(&fresh),false,&mut hook).is_err()); + let components=store.temporal_components(&store.temporal_dir()).unwrap(); + let obsolete=components.witnesses.iter().filter(|(_,m)|m.generation==retired.generation).map(|(name,_)|name.clone()).collect::>(); + assert_eq!(obsolete.len(),330); + for name in obsolete.iter().take(cut){fs::remove_file(store.temporal_dir().join(name)).unwrap();} + // With cut329 the ONLY remaining obsolete record is negative329. + // It is authenticated and parsed but cannot outrank the complete new chain. + let loaded=store.load_temporal(now(&fresh)).unwrap().unwrap();assert_eq!(loaded.token.generation,1);assert_eq!(loaded.token.revision,0); + assert_eq!(loaded.prefix.as_ref().unwrap().bytes(),fresh.bytes()); + let repaired=store.repair_same_temporal(&intent,Some(&fresh),now(&fresh)).unwrap();assert_eq!(repaired,loaded.token); + assert_eq!(store.temporal_components(&store.temporal_dir()).unwrap().witnesses.len(),1); + // Selective replay of the old authentic terminal still fails closed. + fs::write(store.temporal_dir().join(HEAD),old_head).unwrap();assert!(store.load_temporal(now(&fresh)).is_err()); + assert_eq!(store.repair_same_temporal(&intent,Some(&fresh),now(&fresh)).unwrap(),repaired); + assert_eq!(store.load_temporal(now(&fresh)).unwrap().unwrap().prefix.unwrap().bytes(),fresh.bytes()); + } +} +#[test] +fn boundary329_to_fresh343_never_acknowledges_or_drops_malformed_or_old_stage(){ + use crate::room_learner::temporal_model_input::managed::execution::checkpoint::synthetic_fresh_train343_empty_run; + for corrupt in [false,true]{ + let(_t,store,token,at)=synthetic_saturated_metadata_chain();let initial=store.load_temporal(at).unwrap().unwrap().prefix.unwrap(); + let retirement=store.prepare_retire_temporal(&token,at,false).unwrap();let retired=store.publish_temporal(&retirement,None,at).unwrap(); + let old_head=fs::read(store.temporal_dir().join(HEAD)).unwrap();let fresh=synthetic_fresh_train343_empty_run(&initial,at/1_000_000+1); + let intent=store.prepare_temporal(Some(&retired),&fresh,now(&fresh)).unwrap(); + assert!(store.publish_temporal_with(&intent,Some(&fresh),now(&fresh),false,&mut |point|if point=="before:cleanup"{Err("synthetic-before-GC".into())}else{Ok(())}).is_err()); + let bytes=if corrupt{b"synthetic-corrupt-stage".to_vec()}else{old_head}; + let path=store.temporal_dir().join("head.stage"); + let mut file=OpenOptions::new().write(true).create_new(true).open(&path).unwrap(); + #[cfg(unix)]{use std::os::unix::fs::PermissionsExt;file.set_permissions(fs::Permissions::from_mode(0o600)).unwrap();} + file.write_all(&bytes).unwrap();file.sync_all().unwrap();drop(file); + assert!(store.repair_same_temporal(&intent,Some(&fresh),now(&fresh)).is_err()); + assert_eq!(fs::read(path).unwrap(),bytes,"failed repair removed untrusted pending bytes"); + let loaded=store.load_temporal(now(&fresh)).unwrap().unwrap();assert_eq!(loaded.token.generation,1);assert_eq!(loaded.token.revision,0); + assert_eq!(loaded.prefix.unwrap().bytes(),fresh.bytes()); + } +} From fe7f3491c1f4eed95fdaac9a9a2d9f60a619891c Mon Sep 17 00:00:00 2001 From: Codex for Dragan Spiridonov Date: Sat, 7 Nov 2026 01:26:23 +0100 Subject: [PATCH 074/100] test(learner): gate native persistence journeys by execution profile Keep Debug semantic and pressure coverage while explicitly executing and counting native persistence positive journeys in Release inside the same required Rust job. Preserve all runtime policy and original test cases. Authored by Codex for Dragan Spiridonov. --- .github/workflows/ci.yml | 21 ++++ scripts/ci/temporal_test_profiles.py | 114 ++++++++++++++++++ scripts/ci/test_temporal_test_profiles.py | 19 +++ .../room_learner/lifecycle_temporal_tests.rs | 2 + 4 files changed, 156 insertions(+) create mode 100644 scripts/ci/temporal_test_profiles.py create mode 100644 scripts/ci/test_temporal_test_profiles.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 91c0c411ff..60e78c2fd2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,6 +132,15 @@ jobs: # device" once the engine/benchmark crates landed; the same tree's local # debug target measured 151 GB). Debuginfo is useless in CI — tests either # pass or print their failure — so build without it; target shrinks ~5-10x. + - name: Verify temporal Debug/Release test classification + env: + CARGO_PROFILE_DEV_DEBUG: "0" + CARGO_PROFILE_TEST_DEBUG: "0" + CARGO_BUILD_JOBS: "2" + run: | + python3 scripts/ci/test_temporal_test_profiles.py + python3 scripts/ci/temporal_test_profiles.py debug-check --evidence-dir "${{ runner.temp }}/temporal-test-profiles" + - name: Run Rust tests working-directory: v2 env: @@ -139,6 +148,18 @@ jobs: CARGO_PROFILE_TEST_DEBUG: "0" run: cargo test --workspace --no-default-features + - name: Execute native temporal persistence journeys in Release + env: + CARGO_BUILD_JOBS: "2" + run: python3 scripts/ci/temporal_test_profiles.py release-run --evidence-dir "${{ runner.temp }}/temporal-test-profiles" + + - name: Retain temporal profile registration and execution evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 + with: + name: temporal-test-profiles + path: ${{ runner.temp }}/temporal-test-profiles + - name: Verify secure companion over real sockets env: CARGO_PROFILE_DEV_DEBUG: "0" diff --git a/scripts/ci/temporal_test_profiles.py b/scripts/ci/temporal_test_profiles.py new file mode 100644 index 0000000000..c32917c8cb --- /dev/null +++ b/scripts/ci/temporal_test_profiles.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Explicit Debug/Release classification; no production feature or timing changes.""" +import argparse, hashlib, json, os, pathlib, re, subprocess, sys + +PREFIX = 'room_learner::lifecycle::temporal_control::' +C2 = PREFIX + 'tests::c2_all_seven_train_phases_use_actual_async_runtime_and_durable_seals_without_fit' +BUDGET = [PREFIX + 'tests::legacy_329_actual_runtime_saturation_retires_and_explicit_fresh_train_chains_343', PREFIX + 'tests::fresh_343_actual_runtime_all_128_gaps_seven_terminal_pauses_and_retirement_fit_budget'] +# This initial policy is populated below from reviewed source call sites. +C3 = ['room_learner::lifecycle::temporal_control::fit_control::tests::c3_actual_native_train_fit_and_same_store_cold_report_keep_whole_model_diagnostic', + 'room_learner::lifecycle::temporal_control::fit_control::tests::c3_cancel_before_validation_drains_same_job_without_candidate_or_legacy_changes', + 'room_learner::lifecycle::temporal_control::fit_control::tests::c3_exact_store_recertification_rejects_tampered_head_before_any_fit_save', + 'room_learner::lifecycle::temporal_control::fit_control::tests::c3_real_model_save_is_drained_before_synthetic_owner_erase_ack_and_never_recreated', + 'room_learner::lifecycle::temporal_control::fit_control::tests::c3_last_external_owner_drop_drains_real_store_before_return_and_synthetic_erase', + 'room_learner::lifecycle::temporal_control::fit_control::tests::c3_last_completion_owner_releases_after_signal_on_normal_and_unwind_paths', + 'room_learner::lifecycle::temporal_control::fit_control::tests::c3_original_expiry_during_fitting_withdraws_publication_without_renewal', + 'room_learner::lifecycle::temporal_control::fit_control::tests::c3_stale_generation_after_real_save_cannot_adopt_or_strand_job', + 'room_learner::lifecycle::temporal_control::fit_control::tests::c3_old_completion_cannot_fail_a_replacement_job_with_different_cancel_identity', + 'room_learner::lifecycle::temporal_control::fit_control::tests::c3_optional_actual_summary_failure_keeps_durable_diagnostic_model', + 'room_learner::lifecycle::temporal_control::fit_control::tests::c3_worker_unwind_keeps_saved_history_but_never_adopts_current_candidate'] +C4 = ['room_learner::lifecycle::temporal_control::full_run_tests::c4a_actual_seven_train_fit_frozen_ack_and_exact_cold_model', + 'room_learner::lifecycle::temporal_control::full_run_tests::c4a_actual_frozen_cold_missing_corrupt_wrong_kind_bytes_and_expiry_fail_closed', + 'room_learner::lifecycle::temporal_control::full_run_tests::c4a_owner_cancel_drains_actual_prepared_frozen_writer_before_synthetic_erase', + 'room_learner::lifecycle::temporal_control::full_run_tests::c4a_actual_frozen_lost_ack_same_intent_repair_installs_only_after_exact_ack', + 'room_learner::lifecycle::temporal_control::full_run_tests::c4a_running_compute_cancel_cannot_produce_accepted_fit_or_frozen'] + +def policy(root): + source = root / 'v2/crates/wifi-densepose-sensing-server/src/room_learner' + components = [('C2', source / 'lifecycle_temporal_tests.rs', [C2], [(C2, True, 1)])] + components += [('C3', source / 'lifecycle_temporal_fit_tests.rs', C3, [(PREFIX + 'fit_control::tests::', False, 12)]), ('C4', source / 'lifecycle_temporal_full_run_tests.rs', C4, [(PREFIX + 'full_run_tests::', False, 8)]), ('budget343', source / 'temporal_budget_profile_tests.rs', BUDGET, [(name, True, 1) for name in BUDGET])] + active = [item for item in components if item[1].is_file()] + if not active or active[0][0] != 'C2': + raise ValueError('C2 classification/source must exist; zero selected tests is not coverage') + return active + +def validate(components, registered, ignored, mode): + batches = [] + for component, _, classified, selectors in components: + for name in classified: + if name not in registered: + raise ValueError('missing registered classified test: ' + name) + if (name in ignored) != (mode == 'debug-check'): + raise ValueError('wrong profile classification for ' + name) + for selector, exact, minimum in selectors: + names = sorted(n for n in registered if n == selector) if exact else sorted(n for n in registered if n.startswith(selector)) + if len(names) < minimum: + raise ValueError(f'{component}: expected >= {minimum} registered tests, got {len(names)}') + if mode == 'release-run' and any(n in ignored for n in names): + raise ValueError('an opt-in ignored test would enter a Release batch: ' + selector) + batches.append({'component': component, 'selector': selector, 'exact': exact, 'names': names}) + return batches + +def list_tests(binary, ignored=False): + result = subprocess.run([str(binary), '--list'] + (['--ignored'] if ignored else []), check=True, capture_output=True, text=True) + return set(re.findall(r'^(.+): test$', result.stdout, re.M)) + +def compile_binary(root, release): + command = ['cargo', 'test', '--locked', '-p', 'wifi-densepose-sensing-server', '--lib', '--no-default-features', '--no-run', '--message-format=json'] + if release: + command.append('--release') + result = subprocess.run(command, cwd=root/'v2', capture_output=True, text=True) + sys.stderr.write(result.stderr) + binaries = [] + for line in result.stdout.splitlines(): + try: item = json.loads(line) + except json.JSONDecodeError: continue + if item.get('reason') == 'compiler-message' and item.get('message', {}).get('rendered'): + sys.stderr.write(item['message']['rendered']) + if item.get('reason') == 'compiler-artifact' and item.get('profile', {}).get('test') and item.get('executable') and item.get('target', {}).get('name') == 'wifi_densepose_sensing_server': + binaries.append(item['executable']) + if result.returncode: + raise ValueError('Cargo lib-test compilation failed with exit ' + str(result.returncode)) + if len(set(binaries)) != 1: + raise ValueError('expected exactly one actual lib-test executable') + return pathlib.Path(binaries[0]) + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('mode', choices=['debug-check', 'release-run']) + parser.add_argument('--evidence-dir', type=pathlib.Path, required=True) + args = parser.parse_args() + root = pathlib.Path(__file__).resolve().parents[2] + args.evidence_dir.mkdir(parents=True, exist_ok=True) + head = os.environ.get('GITHUB_SHA') or subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=root, text=True).strip() + dirty = bool(subprocess.check_output(['git', 'status', '--porcelain', '--untracked-files=normal', '--ignore-submodules=all'], cwd=root, text=True).strip()) + receipt = {'mode': args.mode, 'head': head, 'worktree_dirty': dirty, 'profile_runner_sha256': hashlib.sha256(pathlib.Path(__file__).read_bytes()).hexdigest(), 'batches': [], 'status': 'started'} + destination = args.evidence_dir / (args.mode + '.json') + try: + receipt['rustc'] = subprocess.check_output(['rustc', '-Vv'], cwd=root/'v2', text=True) + receipt['cargo'] = subprocess.check_output(['cargo', '-V'], cwd=root/'v2', text=True) + binary = compile_binary(root, args.mode == 'release-run') + registered = list_tests(binary); ignored = list_tests(binary, True) + batches = validate(policy(root), registered, ignored, args.mode) + receipt.update(binary=str(binary), binary_sha256=hashlib.sha256(binary.read_bytes()).hexdigest(), registered_count=len(registered), ignored_names=sorted(ignored), batches=batches) + print(json.dumps({'mode': args.mode, 'selected_batches': batches}, indent=2), flush=True) + if args.mode == 'release-run': + for index, batch in enumerate(batches): + command = [str(binary), batch['selector'], '--nocapture', '--test-threads=1'] + (['--exact'] if batch['exact'] else []) + result = subprocess.run(command, cwd=root/'v2', capture_output=True, text=True) + output = result.stdout + result.stderr + (args.evidence_dir / f'{index}-{batch["component"]}.log').write_text(output) + print(output, flush=True) + counts = re.findall(r'^test result: ok\. (\d+) passed; (\d+) failed; (\d+) ignored;', output, re.M) + if result.returncode or not counts or tuple(map(int, counts[-1])) != (len(batch['names']), 0, 0): + raise ValueError('Release batch failed or nonzero/exact-count verification failed: ' + batch['selector']) + batch['passed'] = len(batch['names']) + receipt['status'] = 'passed' + except Exception as error: + receipt.update(status='failed', error=str(error)) + raise + finally: + destination.write_text(json.dumps(receipt, indent=2) + '\n') + +if __name__ == '__main__': + main() diff --git a/scripts/ci/test_temporal_test_profiles.py b/scripts/ci/test_temporal_test_profiles.py new file mode 100644 index 0000000000..274d12af6b --- /dev/null +++ b/scripts/ci/test_temporal_test_profiles.py @@ -0,0 +1,19 @@ +import importlib.util, pathlib, unittest +spec=importlib.util.spec_from_file_location('profiles',pathlib.Path(__file__).with_name('temporal_test_profiles.py'));profiles=importlib.util.module_from_spec(spec);spec.loader.exec_module(profiles) +class ClassificationTests(unittest.TestCase): + def component(self):return [('C2',None,[profiles.C2],[(profiles.C2,True,1)])] + def test_missing_registered_test_never_counts_as_coverage(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),set(),set(),'release-run') + def test_debug_requires_transparent_ignored_classification(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),{profiles.C2},set(),'debug-check') + def test_debug_classification_keeps_registered_name(self): + result=profiles.validate(self.component(),{profiles.C2},{profiles.C2},'debug-check');self.assertEqual(result[0]['names'],[profiles.C2]) + def test_release_case_is_present_active_and_nonzero(self): + result=profiles.validate(self.component(),{profiles.C2},set(),'release-run');self.assertEqual(len(result),1);self.assertTrue(result[0]['exact']) + def test_release_never_sweeps_ignored_optin_performance(self): + component=[('fixture',None,['module::positive'],[('module::',False,1)])] + with self.assertRaises(ValueError):profiles.validate(component,{'module::positive','module::manual_perf'},{'module::manual_perf'},'release-run') + def test_module_is_one_batch_to_preserve_fixture_cache(self): + component=[('fixture',None,['module::one'],[('module::',False,2)])] + result=profiles.validate(component,{'module::one','module::two'},set(),'release-run');self.assertEqual(len(result),1);self.assertEqual(result[0]['names'],['module::one','module::two']) +if __name__=='__main__':unittest.main() diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs index d3d299875b..5929c956a1 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs @@ -904,7 +904,9 @@ fn c2_reverse_clock_and_expiry_while_start_ack_pending_never_authorize_observati } #[test] +#[cfg_attr(debug_assertions, ignore = "native-persistence journey requires Release; CI explicitly executes and counts this test")] fn c2_all_seven_train_phases_use_actual_async_runtime_and_durable_seals_without_fit(){ + assert!(!cfg!(debug_assertions), "execute this native-persistence journey in the explicit Release batch"); // Synthetic receiver time accelerates only when no write is pending. While // a writer is pending we pace packets, preserving the declared cadence and // allowing explicit busy/pause+fresh-start rather than inventing credit. From cfa08bbe1821aed8800f00717ca23c06536f471d Mon Sep 17 00:00:00 2001 From: Codex for Dragan Spiridonov Date: Sat, 7 Nov 2026 01:54:12 +0100 Subject: [PATCH 075/100] fix(ci): bind temporal tests to verified Cargo artifacts Validate the actual Cargo package/library/profile and canonical executable custody, capture its admission hash, and recheck before and after registry execution and before batches. Preserve direct argv and all profile/count gates. Authored by Codex for Dragan Spiridonov. --- scripts/ci/temporal_test_profiles.py | 87 ++++++++++++++++++++--- scripts/ci/test_temporal_test_profiles.py | 62 ++++++++++++++++ 2 files changed, 138 insertions(+), 11 deletions(-) diff --git a/scripts/ci/temporal_test_profiles.py b/scripts/ci/temporal_test_profiles.py index c32917c8cb..847632fbf4 100644 --- a/scripts/ci/temporal_test_profiles.py +++ b/scripts/ci/temporal_test_profiles.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """Explicit Debug/Release classification; no production feature or timing changes.""" -import argparse, hashlib, json, os, pathlib, re, subprocess, sys +import argparse, hashlib, json, os, pathlib, re, stat, subprocess, sys PREFIX = 'room_learner::lifecycle::temporal_control::' C2 = PREFIX + 'tests::c2_all_seven_train_phases_use_actual_async_runtime_and_durable_seals_without_fit' @@ -53,25 +53,77 @@ def list_tests(binary, ignored=False): result = subprocess.run([str(binary), '--list'] + (['--ignored'] if ignored else []), check=True, capture_output=True, text=True) return set(re.findall(r'^(.+): test$', result.stdout, re.M)) +def validate_executable(path, target_directory, release): + raw = pathlib.Path(path) + target_raw = pathlib.Path(target_directory) + if not raw.is_absolute() or not target_raw.is_absolute(): + raise ValueError('Cargo artifact and target directory must be absolute') + target = target_raw.resolve(strict=True) + profile = 'release' if release else 'debug' + expected = target / profile / 'deps' + # The configured target root may be external/canonicalized by Cargo; below + # that root neither profile, deps nor executable may redirect through links. + for directory in [target_raw / profile, target_raw / profile / 'deps']: + if directory.is_symlink() or not directory.is_dir(): + raise ValueError('unsafe Cargo profile/deps directory') + if raw.parent not in (target_raw / profile / 'deps', expected): + raise ValueError('Cargo executable outside declared profile/deps') + info = raw.lstat() + if not stat.S_ISREG(info.st_mode) or raw.is_symlink() or not os.access(raw, os.X_OK): + raise ValueError('Cargo executable must be regular, executable and not a symlink') + if not re.fullmatch(r'wifi_densepose_sensing_server-[0-9a-f]{16}', raw.name): + raise ValueError('unexpected lib-test executable basename') + resolved = raw.resolve(strict=True) + if resolved.parent != expected.resolve(strict=True): + raise ValueError('Cargo executable escaped canonical target profile/deps') + return resolved + +def validate_cargo_artifacts(root, metadata, artifacts, release): + manifest = (root / 'v2/crates/wifi-densepose-sensing-server/Cargo.toml').resolve(strict=True) + source = (manifest.parent / 'src/lib.rs').resolve(strict=True) + packages = [p for p in metadata.get('packages', []) if p.get('name') == 'wifi-densepose-sensing-server' and pathlib.Path(p.get('manifest_path', '')).resolve() == manifest] + if len(packages) != 1: + raise ValueError('expected exactly one workspace sensing-server package') + package = packages[0] + libraries = [t for t in package.get('targets', []) if t.get('name') == 'wifi_densepose_sensing_server' and t.get('kind') == ['lib'] and pathlib.Path(t.get('src_path', '')).resolve() == source] + if len(libraries) != 1: + raise ValueError('expected exactly one actual workspace library source') + candidates = [a for a in artifacts if a.get('reason') == 'compiler-artifact' and a.get('executable') and a.get('package_id') == package['id'] and a.get('target', {}).get('name') == 'wifi_densepose_sensing_server' and a.get('profile', {}).get('test') is True] + if len(candidates) != 1: + raise ValueError('expected exactly one actual package/lib-test executable') + artifact = candidates[0] + target = artifact.get('target', {}) + profile = artifact.get('profile', {}) + if target.get('kind') != ['lib'] or pathlib.Path(target.get('src_path', '')).resolve() != source or profile.get('opt_level') != ('3' if release else '0') or profile.get('debug_assertions') is not (not release): + raise ValueError('Cargo executable library source or execution profile mismatch') + executable = validate_executable(artifact['executable'], metadata['target_directory'], release) + return {'path': executable, 'sha256': hashlib.sha256(executable.read_bytes()).hexdigest(), 'target_directory': pathlib.Path(metadata['target_directory']).resolve(strict=True), 'package_id': package['id'], 'lib_source': source, 'release': release} + +def verify_selected_executable(selected, expected_sha256): + binary = validate_executable(selected['path'], selected['target_directory'], selected['release']) + if hashlib.sha256(binary.read_bytes()).hexdigest() != expected_sha256: + raise ValueError('selected Cargo executable changed before invocation') + return binary + def compile_binary(root, release): + metadata_result = subprocess.run(['cargo', 'metadata', '--locked', '--no-deps', '--format-version=1', '--no-default-features'], cwd=root/'v2', check=True, capture_output=True, text=True) + metadata = json.loads(metadata_result.stdout) command = ['cargo', 'test', '--locked', '-p', 'wifi-densepose-sensing-server', '--lib', '--no-default-features', '--no-run', '--message-format=json'] if release: command.append('--release') result = subprocess.run(command, cwd=root/'v2', capture_output=True, text=True) sys.stderr.write(result.stderr) - binaries = [] + artifacts = [] for line in result.stdout.splitlines(): try: item = json.loads(line) except json.JSONDecodeError: continue if item.get('reason') == 'compiler-message' and item.get('message', {}).get('rendered'): sys.stderr.write(item['message']['rendered']) - if item.get('reason') == 'compiler-artifact' and item.get('profile', {}).get('test') and item.get('executable') and item.get('target', {}).get('name') == 'wifi_densepose_sensing_server': - binaries.append(item['executable']) + if item.get('reason') == 'compiler-artifact': + artifacts.append(item) if result.returncode: raise ValueError('Cargo lib-test compilation failed with exit ' + str(result.returncode)) - if len(set(binaries)) != 1: - raise ValueError('expected exactly one actual lib-test executable') - return pathlib.Path(binaries[0]) + return validate_cargo_artifacts(root, metadata, artifacts, release) def main(): parser = argparse.ArgumentParser() @@ -87,13 +139,26 @@ def main(): try: receipt['rustc'] = subprocess.check_output(['rustc', '-Vv'], cwd=root/'v2', text=True) receipt['cargo'] = subprocess.check_output(['cargo', '-V'], cwd=root/'v2', text=True) - binary = compile_binary(root, args.mode == 'release-run') - registered = list_tests(binary); ignored = list_tests(binary, True) - batches = validate(policy(root), registered, ignored, args.mode) - receipt.update(binary=str(binary), binary_sha256=hashlib.sha256(binary.read_bytes()).hexdigest(), registered_count=len(registered), ignored_names=sorted(ignored), batches=batches) + if args.mode == 'release-run': + selected = compile_binary(root, True) + else: + selected = compile_binary(root, False) + binary = selected['path'] + binary_sha256 = selected['sha256'] + verify_selected_executable(selected, binary_sha256) + registered = list_tests(binary) + verify_selected_executable(selected, binary_sha256) + ignored = list_tests(binary, True) + verify_selected_executable(selected, binary_sha256) + if args.mode == 'release-run': + batches = validate(policy(root), registered, ignored, 'release-run') + else: + batches = validate(policy(root), registered, ignored, 'debug-check') + receipt.update(binary=str(binary), binary_sha256=binary_sha256, cargo_artifact={'target_directory': str(selected['target_directory']), 'package_id': selected['package_id'], 'lib_source': str(selected['lib_source']), 'profile': 'release' if selected['release'] else 'debug'}, registered_count=len(registered), ignored_names=sorted(ignored), batches=batches) print(json.dumps({'mode': args.mode, 'selected_batches': batches}, indent=2), flush=True) if args.mode == 'release-run': for index, batch in enumerate(batches): + verify_selected_executable(selected, binary_sha256) command = [str(binary), batch['selector'], '--nocapture', '--test-threads=1'] + (['--exact'] if batch['exact'] else []) result = subprocess.run(command, cwd=root/'v2', capture_output=True, text=True) output = result.stdout + result.stderr diff --git a/scripts/ci/test_temporal_test_profiles.py b/scripts/ci/test_temporal_test_profiles.py index 274d12af6b..34e46f8a8d 100644 --- a/scripts/ci/test_temporal_test_profiles.py +++ b/scripts/ci/test_temporal_test_profiles.py @@ -16,4 +16,66 @@ def test_release_never_sweeps_ignored_optin_performance(self): def test_module_is_one_batch_to_preserve_fixture_cache(self): component=[('fixture',None,['module::one'],[('module::',False,2)])] result=profiles.validate(component,{'module::one','module::two'},set(),'release-run');self.assertEqual(len(result),1);self.assertEqual(result[0]['names'],['module::one','module::two']) + +class CargoExecutableCustodyTests(unittest.TestCase): + def setUp(self): + import tempfile, copy + self.temp=tempfile.TemporaryDirectory();self.addCleanup(self.temp.cleanup) + self.base=pathlib.Path(self.temp.name).resolve();self.root=self.base/'workspace';self.target=self.base/'external-warm-target' + self.manifest=self.root/'v2/crates/wifi-densepose-sensing-server/Cargo.toml';self.manifest.parent.mkdir(parents=True);self.manifest.write_text('[package]\nname="synthetic"\n') + self.source=self.manifest.parent/'src/lib.rs';self.source.parent.mkdir();self.source.write_text('// synthetic library source\n') + self.binary=self.target/'release/deps/wifi_densepose_sensing_server-0123456789abcdef';self.binary.parent.mkdir(parents=True);self.binary.write_text('#!/bin/sh\nexit 0\n');self.binary.chmod(0o700) + self.lib={'name':'wifi_densepose_sensing_server','kind':['lib'],'src_path':str(self.source)} + self.metadata={'target_directory':str(self.target),'packages':[{'name':'wifi-densepose-sensing-server','manifest_path':str(self.manifest),'id':'synthetic-workspace-package','targets':[self.lib]}]} + self.artifact={'reason':'compiler-artifact','package_id':'synthetic-workspace-package','target':copy.deepcopy(self.lib),'profile':{'test':True,'opt_level':'3','debug_assertions':False},'executable':str(self.binary)} + def valid(self,artifacts=None): + return profiles.validate_cargo_artifacts(self.root,self.metadata,artifacts if artifacts is not None else [self.artifact],True) + def test_actual_external_warm_target_is_preserved(self): + result=self.valid();self.assertEqual(result['path'],self.binary);self.assertEqual(result['target_directory'],self.target) + def test_executable_outside_reported_target_is_rejected(self): + outside=self.base/self.binary.name;outside.write_bytes(self.binary.read_bytes());outside.chmod(0o700);self.artifact['executable']=str(outside) + with self.assertRaises(ValueError):self.valid() + def test_symlink_executable_is_rejected(self): + original=self.binary.with_name('original');self.binary.rename(original);self.binary.symlink_to(original) + with self.assertRaises(ValueError):self.valid() + def test_wrong_profile_artifact_is_rejected(self): + self.artifact['profile']['opt_level']='0';self.artifact['profile']['debug_assertions']=True + with self.assertRaises(ValueError):self.valid() + def test_wrong_library_source_is_rejected(self): + wrong=self.source.with_name('wrong.rs');wrong.write_text('// wrong source\n');self.artifact['target']['src_path']=str(wrong) + with self.assertRaises(ValueError):self.valid() + def test_wrong_package_is_rejected(self): + self.artifact['package_id']='other-package' + with self.assertRaises(ValueError):self.valid() + def test_wrong_target_kind_is_rejected(self): + self.artifact['target']['kind']=['bin'] + with self.assertRaises(ValueError):self.valid() + def test_duplicate_test_artifacts_are_rejected(self): + with self.assertRaises(ValueError):self.valid([self.artifact,self.artifact]) + def test_nonexecutable_regular_file_is_rejected(self): + self.binary.chmod(0o600) + with self.assertRaises(ValueError):self.valid() + + + def test_swapped_bytes_after_admission_are_not_invoked(self): + import hashlib + selected=self.valid();digest=hashlib.sha256(self.binary.read_bytes()).hexdigest();self.binary.write_text('#!/bin/sh\necho changed\n') + with self.assertRaises(ValueError):profiles.verify_selected_executable(selected,digest) + def test_symlink_swap_after_admission_is_not_invoked(self): + import hashlib + selected=self.valid();digest=hashlib.sha256(self.binary.read_bytes()).hexdigest();other=self.base/'outside';other.write_bytes(self.binary.read_bytes());other.chmod(0o700);self.binary.unlink();self.binary.symlink_to(other) + with self.assertRaises(ValueError):profiles.verify_selected_executable(selected,digest) + + + def test_actual_main_rejects_replacement_before_any_registry_execution(self): + from unittest import mock + import sys + selected=self.valid();marker=self.base/'outside-executed';outside=self.base/'outside-libtest' + outside.write_text('#!/bin/sh\nprintf executed > '+str(marker)+'\n');outside.chmod(0o700) + def after_compile(root,release): + self.binary.unlink();self.binary.symlink_to(outside);return selected + with mock.patch.object(sys,'argv',['profiles','release-run','--evidence-dir',str(self.base/'receipt')]), mock.patch.object(profiles,'compile_binary',side_effect=after_compile), mock.patch.object(profiles.subprocess,'check_output',return_value='synthetic-provenance'): + with self.assertRaises(ValueError):profiles.main() + self.assertFalse(marker.exists(),'registry executed substituted out-of-target artifact') + if __name__=='__main__':unittest.main() From 916a47119cf08a79bcc1024e64a2a22174489ae6 Mon Sep 17 00:00:00 2001 From: Codex for Dragan Spiridonov Date: Sat, 7 Nov 2026 03:41:35 +0100 Subject: [PATCH 076/100] test: keep C4A sources live while window ACK is pending --- .../lifecycle_temporal_full_run_tests.rs | 164 +++++++++++------- .../room_learner/lifecycle_temporal_tests.rs | 8 +- 2 files changed, 110 insertions(+), 62 deletions(-) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs index 8a779c9d77..f3e3ddf951 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs @@ -92,68 +92,116 @@ fn c4a_fresh_genesis_is_preparing_until_actual_ack_in_same_store_and_raw_read_ca } #[test] fn c4a_new_full_run_reuses_actual_native_train_writer_without_changing_model_input_identity() { - let directory = tempfile::tempdir().unwrap(); - let rt = fixture::runtime(directory.path(), fixture::START); - rt.configure_temporal_full_run( - &rt.mutation_guard().unwrap(), - fixture::context(), - "synthetic-c4a-window".into(), - fixture::START, - ) - .unwrap(); - let mut now = fixture::settle_async_boundary(&rt, fixture::START); - let original = fixture::snapshot(&rt, now)["descriptor"].clone(); - now = fixture::async_lead_end(&rt, now); - let mut sequence = 0; - let end = now + 3_000_000_000; - while now <= end { - sequence += 1; - for source in 1..=3 { - rt.ingest_native(&fixture::packet(source, sequence, now), now) - .unwrap(); + struct ReleaseOnDrop(std::sync::mpsc::Sender<()>); + impl Drop for ReleaseOnDrop { + fn drop(&mut self) { let _ = self.0.send(()); } + } + // Isolated synthetic stores: the old quiet tail must fail the unchanged + // source-custody guard; continued input must survive the same delayed ACK. + for quiet_tail in [true, false] { + let directory = tempfile::tempdir().unwrap(); + let rt = fixture::runtime(directory.path(), fixture::START); + rt.configure_temporal_full_run( + &rt.mutation_guard().unwrap(), + fixture::context(), + "synthetic-c4a-window".into(), + fixture::START, + ) + .unwrap(); + let mut now = fixture::settle_async_boundary(&rt, fixture::START); + let original = fixture::snapshot(&rt, now)["descriptor"].clone(); + now = fixture::async_lead_end(&rt, now); + let mut sequence = 0; + // Admit the actual empty segment before holding the first WINDOW write. + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(), now).unwrap(); + now = fixture::settle_observing_with_sources(&rt, now, &mut sequence, Duration::from_secs(30)); + let writer = { + let state = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(capture)) = &state.capture else { panic!() }; + capture.core.as_ref().unwrap().writer().unwrap() + }; + let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + writer.hold_next(entered_tx, release_rx); + let release_on_unwind = ReleaseOnDrop(release_tx.clone()); + let end = now + 3_000_000_000; + while now <= end { + sequence += 1; + for source in 1..=3 { + rt.ingest_native(&fixture::packet(source, sequence, now), now) + .unwrap(); + } + now += fixture::PERIOD; + if fixture::snapshot(&rt, now)["durability_pending"] == true { + std::thread::sleep(Duration::from_nanos(fixture::PERIOD)); + } else { + std::thread::sleep(Duration::from_micros(500)); + } } - now += fixture::PERIOD; - if fixture::snapshot(&rt, now)["durability_pending"] == true { - std::thread::sleep(Duration::from_nanos(fixture::PERIOD)); - } else { - std::thread::sleep(Duration::from_micros(500)); + entered_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + assert_eq!(fixture::snapshot(&rt, now)["durability_pending"], true); + assert_eq!(fixture::snapshot(&rt, now)["retained_rows"], 1); + assert_eq!(fixture::snapshot(&rt, now)["durable_windows"], 0); + // Keep the actual WINDOW writer blocked for 150ms AFTER the old tail + // stopped feeding. A hold measured from row submission misses this seam. + if quiet_tail { + let wait = Instant::now(); + std::thread::sleep(Duration::from_millis(80)); + let error = rt.tick_temporal_capture( + &rt.mutation_guard().unwrap(), now + wait.elapsed().as_nanos() as u64, + ).unwrap_err(); + assert_eq!(error, "temporal_missing_segment_source"); + assert_eq!(fixture::snapshot(&rt, now)["durable_windows"], 0); + // Release only AFTER the negative observation; scheduler delay + // cannot let a valid ACK arrive before its zero-durable assertion. + drop(release_on_unwind); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); + continue; } - } - let pending = fixture::snapshot(&rt, now)["durability_pending"] == true; - if pending { + let release = std::thread::spawn(move || { + std::thread::sleep(Duration::from_millis(150)); + let _ = release_tx.send(()); + }); + // Observing requires fresh canonical source packets even while storage + // is pending. The supervision bound is test-only, not a receiver budget. + now = fixture::settle_observing_with_sources(&rt, now, &mut sequence, Duration::from_secs(30)); + release.join().unwrap(); + drop(release_on_unwind); + assert_eq!(fixture::snapshot(&rt, now)["stage"], "observing"); + rt.pause_temporal_capture(&rt.mutation_guard().unwrap(), now).unwrap(); now = fixture::settle_async_boundary(&rt, now); - } - assert!( - fixture::snapshot(&rt, now)["durable_windows"] - .as_u64() + assert!( + fixture::snapshot(&rt, now)["durable_windows"] + .as_u64() + .unwrap() + >= 1 + ); + let loaded = rt + .temporal_store() .unwrap() - >= 1 - ); - let loaded = rt - .temporal_store() - .unwrap() - .load_temporal(now) - .unwrap() - .unwrap(); - let prefix = loaded.prefix.unwrap(); - assert_eq!(prefix.metadata_kind(), "temporal-prefix-commitment-v2"); - assert_eq!(prefix.transition_limit(), 433); - assert_eq!( - fixture::snapshot(&rt, now)["descriptor"]["id"], - original["id"] - ); - assert_eq!( - fixture::snapshot(&rt, now)["descriptor"]["expires_at_ms"], - original["expires_at_ms"] - ); - let data: serde_json::Value = serde_json::from_slice(prefix.bytes()).unwrap(); - let policy = &data["payload"]["policy"]["body"]; - assert_ne!( - policy["input_policy_sha256"], data["payload"]["policy"]["sha256"], - "continuation identity must not replace model input identity" - ); - rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()) - .unwrap(); + .load_temporal(now) + .unwrap() + .unwrap(); + let prefix = loaded.prefix.unwrap(); + assert_eq!(prefix.metadata_kind(), "temporal-prefix-commitment-v2"); + assert_eq!(prefix.transition_limit(), 433); + assert_eq!( + fixture::snapshot(&rt, now)["descriptor"]["id"], + original["id"] + ); + assert_eq!( + fixture::snapshot(&rt, now)["descriptor"]["expires_at_ms"], + original["expires_at_ms"] + ); + let data: serde_json::Value = serde_json::from_slice(prefix.bytes()).unwrap(); + let policy = &data["payload"]["policy"]["body"]; + assert_ne!( + policy["input_policy_sha256"], data["payload"]["policy"]["sha256"], + "continuation identity must not replace model input identity" + ); + rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()) + .unwrap(); + } } #[test] fn c4a_full_run_rejects_unsupported_source_profile_before_genesis_write() { diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs index 104c91b1b3..7514b98da3 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs @@ -756,13 +756,13 @@ fn c2_real_publish_lost_ack_repairs_same_intent_to_paused_without_new_run_or_dea rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()).unwrap(); } -fn settle_observing_with_sources(rt:&RoomLearningRuntime,after:u64,seq:&mut u32)->u64{ +pub(super) fn settle_observing_with_sources(rt:&RoomLearningRuntime,after:u64,seq:&mut u32,timeout:std::time::Duration)->u64{ let wait=std::time::Instant::now();let mut now=after; loop { *seq+=1; for source in 1..=3 {rt.ingest_native(&packet(source,*seq,now),now).unwrap();} if snapshot(rt,now)["durability_pending"]==false {return now+PERIOD;} - assert!(wait.elapsed() Date: Sat, 7 Nov 2026 16:58:26 +0100 Subject: [PATCH 077/100] test: require Release execution of synchronous temporal reference --- scripts/ci/temporal_test_profiles.py | 2 ++ scripts/ci/test_temporal_test_profiles.py | 18 ++++++++++++++++++ .../room_learner/lifecycle_temporal_tests.rs | 1 + 3 files changed, 21 insertions(+) diff --git a/scripts/ci/temporal_test_profiles.py b/scripts/ci/temporal_test_profiles.py index af4397fb60..bf4b3a256b 100644 --- a/scripts/ci/temporal_test_profiles.py +++ b/scripts/ci/temporal_test_profiles.py @@ -3,6 +3,7 @@ import argparse, hashlib, json, os, pathlib, re, stat, subprocess, sys PREFIX = 'room_learner::lifecycle::temporal_control::' +C1 = PREFIX + 'tests::full_seven_train_phases_require_fresh_confirmation_and_never_issue_fit_authority' C2 = PREFIX + 'tests::c2_all_seven_train_phases_use_actual_async_runtime_and_durable_seals_without_fit' BUDGET = [PREFIX + 'tests::legacy_329_actual_runtime_saturation_retires_and_explicit_fresh_train_chains_343', PREFIX + 'tests::fresh_343_actual_runtime_all_128_gaps_seven_terminal_pauses_and_retirement_fit_budget'] # This initial policy is populated below from reviewed source call sites. @@ -30,6 +31,7 @@ def policy(root): source = root / 'v2/crates/wifi-densepose-sensing-server/src/room_learner' components = [('C2', source / 'lifecycle_temporal_tests.rs', [C2], [(C2, True, 1)])] + components += [('C1', source / 'lifecycle_temporal_tests.rs', [C1], [(C1, True, 1)])] components += [('C3', source / 'lifecycle_temporal_fit_tests.rs', C3, [(PREFIX + 'fit_control::tests::', False, 12)]), ('C4', source / 'lifecycle_temporal_full_run_tests.rs', C4, [(PREFIX + 'full_run_tests::', False, 8)]), ('budget343', source / 'temporal_budget_profile_tests.rs', BUDGET, [(name, True, 1) for name in BUDGET])] b_marker = source / 'lifecycle_temporal_held_out_tests.rs' if not b_marker.is_file(): diff --git a/scripts/ci/test_temporal_test_profiles.py b/scripts/ci/test_temporal_test_profiles.py index 96f4d4ee56..f20a8d824a 100644 --- a/scripts/ci/test_temporal_test_profiles.py +++ b/scripts/ci/test_temporal_test_profiles.py @@ -18,6 +18,24 @@ def test_module_is_one_batch_to_preserve_fixture_cache(self): result=profiles.validate(component,{'module::one','module::two'},set(),'release-run');self.assertEqual(len(result),1);self.assertEqual(result[0]['names'],['module::one','module::two']) +class C1ClassificationTests(unittest.TestCase): + def component(self): + return [next(item for item in profiles.policy(pathlib.Path(__file__).resolve().parents[2]) if item[0] == 'C1')] + def test_current_policy_requires_one_exact_release_reference(self): + result=profiles.validate(self.component(),{profiles.C1,profiles.C1+'_lookalike',profiles.C2},set(),'release-run') + self.assertEqual(result,[{'component':'C1','selector':profiles.C1,'exact':True,'names':[profiles.C1]}]) + def test_reference_stays_registered_but_transparently_ignored_in_debug(self): + result=profiles.validate(self.component(),{profiles.C1},{profiles.C1},'debug-check');self.assertEqual(result[0]['names'],[profiles.C1]) + def test_missing_reference_cannot_be_silently_omitted(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),set(),set(),'release-run') + def test_reference_cannot_run_unclassified_in_debug(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),{profiles.C1},set(),'debug-check') + def test_reference_cannot_be_ignored_in_required_release(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),{profiles.C1},{profiles.C1},'release-run') + def test_zero_or_lookalike_only_is_not_the_exact_reference(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),{profiles.C1+'_manual_perf'},set(),'release-run') + + class C4BClassificationTests(unittest.TestCase): def component(self):return [('C4B',None,profiles.C4B,[(profiles.C4B_PREFIX,False,11)])] def test_exact_eleven_release_is_one_batch_with_fixture_cache(self): diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs index 0beee3d157..04c0241d5c 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs @@ -133,6 +133,7 @@ fn private_temporal_setup_uses_real_native_runtime_and_public_configure_cannot_r .is_err()); } #[test] +#[cfg_attr(debug_assertions, ignore = "synchronous native-persistence reference requires Release; CI explicitly executes and counts this exact test")] fn full_seven_train_phases_require_fresh_confirmation_and_never_issue_fit_authority() { let dir = tempfile::tempdir().unwrap(); let rt = configured(dir.path()); From d398dd52dcaa083dee8472ec8542f829f99e584e Mon Sep 17 00:00:00 2001 From: Codex for Dragan Spiridonov Date: Sat, 7 Nov 2026 17:39:00 +0100 Subject: [PATCH 078/100] test: require exact Release C1 reference in qualification CI --- scripts/ci/temporal_test_profiles.py | 2 ++ scripts/ci/test_temporal_test_profiles.py | 18 ++++++++++++++++++ .../room_learner/lifecycle_temporal_tests.rs | 1 + 3 files changed, 21 insertions(+) diff --git a/scripts/ci/temporal_test_profiles.py b/scripts/ci/temporal_test_profiles.py index 847632fbf4..55fb5d3e75 100644 --- a/scripts/ci/temporal_test_profiles.py +++ b/scripts/ci/temporal_test_profiles.py @@ -3,6 +3,7 @@ import argparse, hashlib, json, os, pathlib, re, stat, subprocess, sys PREFIX = 'room_learner::lifecycle::temporal_control::' +C1 = PREFIX + 'tests::full_seven_train_phases_require_fresh_confirmation_and_never_issue_fit_authority' C2 = PREFIX + 'tests::c2_all_seven_train_phases_use_actual_async_runtime_and_durable_seals_without_fit' BUDGET = [PREFIX + 'tests::legacy_329_actual_runtime_saturation_retires_and_explicit_fresh_train_chains_343', PREFIX + 'tests::fresh_343_actual_runtime_all_128_gaps_seven_terminal_pauses_and_retirement_fit_budget'] # This initial policy is populated below from reviewed source call sites. @@ -26,6 +27,7 @@ def policy(root): source = root / 'v2/crates/wifi-densepose-sensing-server/src/room_learner' components = [('C2', source / 'lifecycle_temporal_tests.rs', [C2], [(C2, True, 1)])] + components += [('C1', source / 'lifecycle_temporal_tests.rs', [C1], [(C1, True, 1)])] components += [('C3', source / 'lifecycle_temporal_fit_tests.rs', C3, [(PREFIX + 'fit_control::tests::', False, 12)]), ('C4', source / 'lifecycle_temporal_full_run_tests.rs', C4, [(PREFIX + 'full_run_tests::', False, 8)]), ('budget343', source / 'temporal_budget_profile_tests.rs', BUDGET, [(name, True, 1) for name in BUDGET])] active = [item for item in components if item[1].is_file()] if not active or active[0][0] != 'C2': diff --git a/scripts/ci/test_temporal_test_profiles.py b/scripts/ci/test_temporal_test_profiles.py index 34e46f8a8d..651660a9cf 100644 --- a/scripts/ci/test_temporal_test_profiles.py +++ b/scripts/ci/test_temporal_test_profiles.py @@ -17,6 +17,24 @@ def test_module_is_one_batch_to_preserve_fixture_cache(self): component=[('fixture',None,['module::one'],[('module::',False,2)])] result=profiles.validate(component,{'module::one','module::two'},set(),'release-run');self.assertEqual(len(result),1);self.assertEqual(result[0]['names'],['module::one','module::two']) +class C1ClassificationTests(unittest.TestCase): + def component(self): + return [next(item for item in profiles.policy(pathlib.Path(__file__).resolve().parents[2]) if item[0] == 'C1')] + def test_current_policy_requires_one_exact_release_reference(self): + result=profiles.validate(self.component(),{profiles.C1,profiles.C1+'_lookalike',profiles.C2},set(),'release-run') + self.assertEqual(result,[{'component':'C1','selector':profiles.C1,'exact':True,'names':[profiles.C1]}]) + def test_reference_stays_registered_but_transparently_ignored_in_debug(self): + result=profiles.validate(self.component(),{profiles.C1},{profiles.C1},'debug-check');self.assertEqual(result[0]['names'],[profiles.C1]) + def test_missing_reference_cannot_be_silently_omitted(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),set(),set(),'release-run') + def test_reference_cannot_run_unclassified_in_debug(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),{profiles.C1},set(),'debug-check') + def test_reference_cannot_be_ignored_in_required_release(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),{profiles.C1},{profiles.C1},'release-run') + def test_zero_or_lookalike_only_is_not_the_exact_reference(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),{profiles.C1+'_manual_perf'},set(),'release-run') + + class CargoExecutableCustodyTests(unittest.TestCase): def setUp(self): import tempfile, copy diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs index 0beee3d157..04c0241d5c 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs @@ -133,6 +133,7 @@ fn private_temporal_setup_uses_real_native_runtime_and_public_configure_cannot_r .is_err()); } #[test] +#[cfg_attr(debug_assertions, ignore = "synchronous native-persistence reference requires Release; CI explicitly executes and counts this exact test")] fn full_seven_train_phases_require_fresh_confirmation_and_never_issue_fit_authority() { let dir = tempfile::tempdir().unwrap(); let rt = configured(dir.path()); From f6ae834044cb68259fbd2d835f01bf45158ecad0 Mon Sep 17 00:00:00 2001 From: Codex for Dragan Spiridonov Date: Sat, 7 Nov 2026 21:02:04 +0100 Subject: [PATCH 079/100] fix(discovery): refresh same service after interface recovery --- .../src/discovery.rs | 250 +++++++++++++++++- .../src/discovery_recovery_tests.rs | 223 ++++++++++++++++ 2 files changed, 461 insertions(+), 12 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/discovery_recovery_tests.rs diff --git a/v2/crates/wifi-densepose-sensing-server/src/discovery.rs b/v2/crates/wifi-densepose-sensing-server/src/discovery.rs index 61245c8205..99617b6a4a 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/discovery.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/discovery.rs @@ -4,9 +4,16 @@ //! sensor values, credentials, or personal data. Mobile clients treat the //! installation id as a routing hint only and still health-check the service. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; +use std::net::{IpAddr, Ipv4Addr}; +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, +}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; -use mdns_sd::{ServiceDaemon, ServiceInfo}; +use mdns_sd::{DaemonEvent, ServiceDaemon, ServiceInfo}; pub const SERVICE_TYPE: &str = "_ruview._tcp.local."; pub const SCHEMA: &str = "ruview.installation.v1"; @@ -14,10 +21,14 @@ pub const SCHEMA: &str = "ruview.installation.v1"; pub struct DiscoveryAdvertiser { daemon: ServiceDaemon, fullname: String, + recovery: RecoveryWorker, } impl Drop for DiscoveryAdvertiser { fn drop(&mut self) { + // Join BEFORE unregister: any already-dispatched refresh precedes the + // goodbye, and no worker can enqueue a registration after shutdown. + self.recovery.stop(); let _ = self.daemon.unregister(&self.fullname); let _ = self.daemon.shutdown(); } @@ -80,22 +91,237 @@ pub fn start_advertiser( hostname: &str, http_port: u16, ) -> Result { - let daemon = ServiceDaemon::new()?; - let service = build_service(instance_name, installation_id, hostname, http_port, false)?; - let fullname = service.get_fullname().to_owned(); - daemon.register(service)?; - Ok(DiscoveryAdvertiser { daemon, fullname }) + start_service(build_service( + instance_name, + installation_id, + hostname, + http_port, + false, + )?) } /// Advertise the QR-pinned companion TLS endpoint independently of legacy HTTP. -pub fn start_tls_advertiser(instance_name: &str, installation_id: &str, hostname: &str, port: u16) -> Result { +pub fn start_tls_advertiser( + instance_name: &str, + installation_id: &str, + hostname: &str, + port: u16, +) -> Result { + start_service(build_service( + instance_name, + installation_id, + hostname, + port, + true, + )?) +} + +// IpAdd is only a wakeup, never an address/owner/health authority. Re-registering +// this pristine, empty-address template lets mdns-sd perform its EXISTING live +// interface enumeration; no event address is copied into a service record. +fn start_service(template: ServiceInfo) -> Result { let daemon = ServiceDaemon::new()?; - let service = build_service(instance_name, installation_id, hostname, port, true)?; - let fullname = service.get_fullname().to_owned(); - daemon.register(service)?; - Ok(DiscoveryAdvertiser { daemon, fullname }) + let fullname = template.get_fullname().to_owned(); + let result = (|| { + let events = daemon.monitor()?; + daemon.register(template.clone())?; + let recovery = RecoveryWorker::start(daemon.clone(), events, template)?; + Ok(DiscoveryAdvertiser { + daemon: daemon.clone(), + fullname: fullname.clone(), + recovery, + }) + })(); + if result.is_err() { + let _ = daemon.unregister(&fullname); + let _ = daemon.shutdown(); + } + result } +const COALESCE_MS: u64 = 250; +const MIN_REFRESH_MS: u64 = 1_000; +const MAX_RECOVERED_IPS: usize = 8; +const MAX_RETRIES: u8 = 3; + +/// Per-advertiser bounded event reducer. It does not retain packets, names, +/// credentials, or an additional service/address authority. +#[derive(Default)] +struct RecoveryState { + // Pending wakeups only, not a persistent interface cache. + recovered: HashSet, + due: Option, + last_attempt: Option, + retries: u8, + stopped: bool, +} +impl RecoveryState { + fn event(&mut self, event: DaemonEvent, now: u64) { + if self.stopped { + return; + } + match event { + DaemonEvent::IpAdd(IpAddr::V4(ip)) if ip.is_private() => { + if !self.recovered.contains(&ip) && self.recovered.len() >= MAX_RECOVERED_IPS { + return; + } + self.recovered.insert(ip); + // Duplicate notifications coalesce while pending. Once settled, + // a fresh IpAdd is allowed to wake us even if its IpDel was + // dropped by the bounded daemon monitor. Rate limiting remains. + if self.due.is_none() { + self.retries = 0; + self.due = Some( + now.saturating_add(COALESCE_MS).max( + self.last_attempt + .map_or(0, |at| at.saturating_add(MIN_REFRESH_MS)), + ), + ); + } + } + DaemonEvent::IpDel(IpAddr::V4(ip)) => { + self.recovered.remove(&ip); + if self.recovered.is_empty() { + self.due = None; + } + } + // Announce is not a successful wire-send receipt in mdns-sd0.11.5. + _ => {} + } + } + fn stop(&mut self) { + self.stopped = true; + self.due = None; + self.recovered.clear(); + } + fn poll(&mut self, now: u64, register: impl FnOnce() -> Result<(), mdns_sd::Error>) { + if self.stopped || self.due.is_none_or(|due| now < due) { + return; + } + self.due = None; + self.last_attempt = Some(now); + match register() { + Ok(()) => { + self.retries = 0; + self.recovered.clear(); + } + Err(mdns_sd::Error::Again) if self.retries < MAX_RETRIES => { + self.retries += 1; + self.due = Some(now.saturating_add(MIN_REFRESH_MS << (self.retries - 1))); + } + Err(error) => { + self.recovered.clear(); + // No busyloop, implicit publisher replacement or health claim. + tracing::warn!(%error, "Discovery interface refresh was not accepted"); + } + } + } +} + +enum EventRead { + Event(DaemonEvent), + Idle, + Closed, +} +trait RecoveryEvents: Send + 'static { + fn read(&self, timeout: Duration) -> EventRead; +} +impl RecoveryEvents for mdns_sd::Receiver { + fn read(&self, timeout: Duration) -> EventRead { + match self.recv_timeout(timeout) { + Ok(event) => EventRead::Event(event), + Err(_) if self.is_disconnected() => EventRead::Closed, + Err(_) => EventRead::Idle, + } + } +} + +struct RecoveryWorker { + stop: Arc, + thread: Option>, +} +impl RecoveryWorker { + fn start( + daemon: ServiceDaemon, + events: mdns_sd::Receiver, + template: ServiceInfo, + ) -> Result { + Self::start_with(events, move || daemon.register(template.clone())) + } + fn start_with( + events: impl RecoveryEvents, + mut register: impl FnMut() -> Result<(), mdns_sd::Error> + Send + 'static, + ) -> Result { + let stop = Arc::new(AtomicBool::new(false)); + let cancelled = stop.clone(); + let thread = thread::Builder::new() + .name("ruview-mdns-recovery".into()) + .spawn(move || { + let origin = Instant::now(); + let now = || origin.elapsed().as_millis().min(u64::MAX as u128) as u64; + let mut state = RecoveryState::default(); + while !cancelled.load(Ordering::Acquire) { + match events.read(Duration::from_millis(100)) { + EventRead::Event(event) => state.event(event, now()), + EventRead::Closed => break, + EventRead::Idle => {} + } + let mut closed = false; + // Drain one bounded monitor batch before acting, including any + // queued IpDel. A continuous event producer cannot starve stop. + for _ in 0..100 { + if cancelled.load(Ordering::Acquire) { + break; + } + match events.read(Duration::ZERO) { + EventRead::Event(event) => state.event(event, now()), + EventRead::Closed => { + closed = true; + break; + } + EventRead::Idle => break, + } + } + if cancelled.load(Ordering::Acquire) || closed { + break; + } + state.poll(now(), || { + if cancelled.load(Ordering::Acquire) { + return Ok(()); + } + register() + }); + } + state.stop(); + }) + .map_err(|error| { + mdns_sd::Error::Msg(format!( + "Discovery recovery worker could not start: {error}" + )) + })?; + Ok(Self { + stop, + thread: Some(thread), + }) + } + fn stop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } +} + +impl Drop for RecoveryWorker { + fn drop(&mut self) { + self.stop(); + } +} + +#[cfg(test)] +#[path = "discovery_recovery_tests.rs"] +mod recovery_tests; + #[cfg(test)] mod tests { use super::*; diff --git a/v2/crates/wifi-densepose-sensing-server/src/discovery_recovery_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/discovery_recovery_tests.rs new file mode 100644 index 0000000000..43272fc592 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/discovery_recovery_tests.rs @@ -0,0 +1,223 @@ +//! No live daemon/socket/network: exercise the production reducer and worker +//! through synthetic event/registration ports. +use super::*; +use std::sync::{mpsc, Mutex}; + +fn add(ip: &str) -> DaemonEvent { + DaemonEvent::IpAdd(ip.parse().unwrap()) +} +fn del(ip: &str) -> DaemonEvent { + DaemonEvent::IpDel(ip.parse().unwrap()) +} + +#[test] +fn failed_bind_then_ip_add_refreshes_exact_pristine_service_without_event_address() { + let template = build_service( + "same companion", + "same-installation", + "ruview-same.local.", + 3443, + true, + ) + .unwrap(); + let original = format!("{template:?}"); + let mut state = RecoveryState::default(); + let mut sent = Vec::new(); + // A failed bind/Error itself is not an instruction to repeatedly announce. + state.event( + DaemonEvent::Error(mdns_sd::Error::Msg("synthetic failed bind".into())), + 0, + ); + state.poll(1000, || { + panic!("failure alone must not enqueue"); + }); + state.event(add("192.168.1.2"), 1000); + state.poll(1249, || { + panic!("coalescing boundary"); + }); + state.poll(1250, || { + sent.push(template.clone()); + Ok(()) + }); + assert_eq!(sent.len(), 1); + assert_eq!(format!("{:?}", sent[0]), original); + assert!( + sent[0].get_addresses().is_empty(), + "event address must never be injected" + ); + assert_eq!(sent[0].get_hostname(), "ruview-same.local."); + assert_eq!(sent[0].get_port(), 3443); + assert_eq!(sent[0].get_property_val_str("tls"), Some("1")); +} + +#[test] +fn duplicate_burst_and_irrelevant_events_are_bounded_without_postponing_refresh() { + let mut state = RecoveryState::default(); + for i in 1..=255 { + state.event(add(&format!("10.0.0.{i}")), i as u64); + } + for _ in 0..1000 { + state.event(add("10.0.0.1"), 249); + } + assert_eq!(state.recovered.len(), MAX_RECOVERED_IPS); + assert_eq!(state.due, Some(251)); + let mut calls = 0; + state.poll(251, || { + calls += 1; + Ok(()) + }); + state.event(add("127.0.0.1"), 1000); + state.event(add("8.8.8.8"), 1000); + state.event(add("169.254.1.2"), 1000); + state.event(add("::1"), 1000); + state.event( + DaemonEvent::Announce("foreign".into(), "not-health".into()), + 1000, + ); + state.poll(u64::MAX, || { + calls += 1; + Ok(()) + }); + assert_eq!(calls, 1); +} + +#[test] +fn same_ip_rebind_refreshes_with_or_without_a_del_notification() { + let mut state = RecoveryState::default(); + let mut calls = 0; + state.event(add("10.0.0.1"), 0); + state.poll(250, || { + calls += 1; + Ok(()) + }); + state.event(del("10.0.0.1"), 300); + state.event(add("10.0.0.1"), 301); + assert_eq!(state.due, Some(1250)); + state.poll(1250, || { + calls += 1; + Ok(()) + }); + // The monitor may drop IpDel when full: do not suppress this real rebind. + for _ in 0..1000 { + state.event(add("10.0.0.1"), 1251); + } + assert_eq!(state.due, Some(2250)); + state.poll(2249, || panic!("refresh storm")); + state.poll(2250, || { + calls += 1; + Ok(()) + }); + assert_eq!(calls, 3); +} + +#[test] +fn settled_wakeups_do_not_fill_a_permanent_interface_cache_when_ip_del_is_lost() { + let mut state = RecoveryState::default(); + let mut calls = 0; + for i in 1..=32 { + let now = i * 2000; + state.event(add(&format!("10.0.0.{i}")), now); + state.poll(now + COALESCE_MS, || { + calls += 1; + Ok(()) + }); + assert!(state.recovered.is_empty()); + } + assert_eq!(calls, 32); +} + +#[test] +fn queued_ip_del_and_stop_withdraw_pending_refresh() { + let mut state = RecoveryState::default(); + state.event(add("192.168.1.2"), 0); + state.event(del("192.168.1.2"), 249); + state.poll(250, || panic!("deleted wakeup must be withdrawn")); + state.event(add("192.168.1.2"), 251); + state.stop(); + state.event(add("10.0.0.1"), 1000); + state.poll(u64::MAX, || panic!("stopped generation must not register")); + assert!(state.recovered.is_empty()); +} + +#[test] +fn again_has_three_bounded_backoffs_then_stops_and_terminal_error_does_not_spin() { + let mut state = RecoveryState::default(); + state.event(add("10.0.0.1"), 0); + let mut calls = 0; + for (now, next) in [ + (250, Some(1250)), + (1250, Some(3250)), + (3250, Some(7250)), + (7250, None), + ] { + state.poll(now, || { + calls += 1; + Err(mdns_sd::Error::Again) + }); + assert_eq!(state.due, next); + state.poll(now, || panic!("no busyloop")); + } + assert_eq!(calls, 4); + state.event(add("10.0.0.2"), 8000); + state.poll(8250, || { + Err(mdns_sd::Error::Msg("synthetic closed daemon".into())) + }); + assert!(state.due.is_none()); +} + +struct TestEvents(mpsc::Receiver); +impl RecoveryEvents for TestEvents { + fn read(&self, timeout: Duration) -> EventRead { + match self.0.recv_timeout(timeout) { + Ok(event) => EventRead::Event(event), + Err(mpsc::RecvTimeoutError::Timeout) => EventRead::Idle, + Err(mpsc::RecvTimeoutError::Disconnected) => EventRead::Closed, + } + } +} + +#[test] +fn monitor_close_discards_pending_intent_and_worker_joins() { + let (tx, rx) = mpsc::channel(); + tx.send(add("10.0.0.1")).unwrap(); + drop(tx); + let mut worker = + RecoveryWorker::start_with(TestEvents(rx), || panic!("closed monitor cannot register")) + .unwrap(); + worker.thread.take().unwrap().join().unwrap(); +} + +#[test] +fn worker_stop_joins_inflight_registration_before_teardown_and_never_enqueues_again() { + let (tx, rx) = mpsc::channel(); + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + let (done_tx, done_rx) = mpsc::channel(); + let calls = Arc::new(Mutex::new(0)); + let seen = calls.clone(); + let worker = RecoveryWorker::start_with(TestEvents(rx), move || { + *seen.lock().unwrap() += 1; + entered_tx.send(()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + Ok(()) + }) + .unwrap(); + tx.send(add("10.0.0.1")).unwrap(); + entered_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + // Force stop publication before releasing the in-flight fake command. + worker.stop.store(true, Ordering::Release); + tx.send(del("10.0.0.1")).unwrap(); + tx.send(add("10.0.0.2")).unwrap(); + let stopper = thread::spawn(move || { + drop(worker); + done_tx.send(()).unwrap(); + }); + assert!( + done_rx.try_recv().is_err(), + "teardown must wait for dispatched registration" + ); + release_tx.send(()).unwrap(); + done_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + stopper.join().unwrap(); + assert_eq!(*calls.lock().unwrap(), 1); +} From e3961d4c480e08d2ddad5e6f502db5b9a137dd9a Mon Sep 17 00:00:00 2001 From: Codex for Dragan Spiridonov Date: Sun, 8 Nov 2026 10:05:24 +0100 Subject: [PATCH 080/100] test(ci): require exact Release C1 reference on original stack --- scripts/ci/temporal_test_profiles.py | 2 ++ scripts/ci/test_temporal_test_profiles.py | 18 ++++++++++++++++++ .../room_learner/lifecycle_temporal_tests.rs | 1 + 3 files changed, 21 insertions(+) diff --git a/scripts/ci/temporal_test_profiles.py b/scripts/ci/temporal_test_profiles.py index 847632fbf4..55fb5d3e75 100644 --- a/scripts/ci/temporal_test_profiles.py +++ b/scripts/ci/temporal_test_profiles.py @@ -3,6 +3,7 @@ import argparse, hashlib, json, os, pathlib, re, stat, subprocess, sys PREFIX = 'room_learner::lifecycle::temporal_control::' +C1 = PREFIX + 'tests::full_seven_train_phases_require_fresh_confirmation_and_never_issue_fit_authority' C2 = PREFIX + 'tests::c2_all_seven_train_phases_use_actual_async_runtime_and_durable_seals_without_fit' BUDGET = [PREFIX + 'tests::legacy_329_actual_runtime_saturation_retires_and_explicit_fresh_train_chains_343', PREFIX + 'tests::fresh_343_actual_runtime_all_128_gaps_seven_terminal_pauses_and_retirement_fit_budget'] # This initial policy is populated below from reviewed source call sites. @@ -26,6 +27,7 @@ def policy(root): source = root / 'v2/crates/wifi-densepose-sensing-server/src/room_learner' components = [('C2', source / 'lifecycle_temporal_tests.rs', [C2], [(C2, True, 1)])] + components += [('C1', source / 'lifecycle_temporal_tests.rs', [C1], [(C1, True, 1)])] components += [('C3', source / 'lifecycle_temporal_fit_tests.rs', C3, [(PREFIX + 'fit_control::tests::', False, 12)]), ('C4', source / 'lifecycle_temporal_full_run_tests.rs', C4, [(PREFIX + 'full_run_tests::', False, 8)]), ('budget343', source / 'temporal_budget_profile_tests.rs', BUDGET, [(name, True, 1) for name in BUDGET])] active = [item for item in components if item[1].is_file()] if not active or active[0][0] != 'C2': diff --git a/scripts/ci/test_temporal_test_profiles.py b/scripts/ci/test_temporal_test_profiles.py index 34e46f8a8d..651660a9cf 100644 --- a/scripts/ci/test_temporal_test_profiles.py +++ b/scripts/ci/test_temporal_test_profiles.py @@ -17,6 +17,24 @@ def test_module_is_one_batch_to_preserve_fixture_cache(self): component=[('fixture',None,['module::one'],[('module::',False,2)])] result=profiles.validate(component,{'module::one','module::two'},set(),'release-run');self.assertEqual(len(result),1);self.assertEqual(result[0]['names'],['module::one','module::two']) +class C1ClassificationTests(unittest.TestCase): + def component(self): + return [next(item for item in profiles.policy(pathlib.Path(__file__).resolve().parents[2]) if item[0] == 'C1')] + def test_current_policy_requires_one_exact_release_reference(self): + result=profiles.validate(self.component(),{profiles.C1,profiles.C1+'_lookalike',profiles.C2},set(),'release-run') + self.assertEqual(result,[{'component':'C1','selector':profiles.C1,'exact':True,'names':[profiles.C1]}]) + def test_reference_stays_registered_but_transparently_ignored_in_debug(self): + result=profiles.validate(self.component(),{profiles.C1},{profiles.C1},'debug-check');self.assertEqual(result[0]['names'],[profiles.C1]) + def test_missing_reference_cannot_be_silently_omitted(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),set(),set(),'release-run') + def test_reference_cannot_run_unclassified_in_debug(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),{profiles.C1},set(),'debug-check') + def test_reference_cannot_be_ignored_in_required_release(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),{profiles.C1},{profiles.C1},'release-run') + def test_zero_or_lookalike_only_is_not_the_exact_reference(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),{profiles.C1+'_manual_perf'},set(),'release-run') + + class CargoExecutableCustodyTests(unittest.TestCase): def setUp(self): import tempfile, copy diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs index 5929c956a1..df6968d461 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs @@ -133,6 +133,7 @@ fn private_temporal_setup_uses_real_native_runtime_and_public_configure_cannot_r .is_err()); } #[test] +#[cfg_attr(debug_assertions, ignore = "synchronous native-persistence reference requires Release; CI explicitly executes and counts this exact test")] fn full_seven_train_phases_require_fresh_confirmation_and_never_issue_fit_authority() { let dir = tempfile::tempdir().unwrap(); let rt = configured(dir.path()); From eb45d000f1d18a7f8f78dcc4c95c364ac1df20ac Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Sun, 8 Nov 2026 10:08:26 +0100 Subject: [PATCH 081/100] feat(learner): connect whole temporal evaluation and selected workflow --- scripts/ci/temporal_test_profiles.py | 6 + scripts/ci/test_temporal_test_profiles.py | 49 + .../src/companion.rs | 19 +- .../src/companion_atomic_write_tests.rs | 66 + .../wifi-densepose-sensing-server/src/main.rs | 24 +- .../src/room_learner/artifact.rs | 8 + .../src/room_learner/lifecycle.rs | 14 +- .../src/room_learner/lifecycle_capture.rs | 17 +- .../src/room_learner/lifecycle_selection.rs | 12 +- .../src/room_learner/lifecycle_temporal.rs | 115 +- .../lifecycle_temporal_evaluation.rs | 536 ++++++++ .../room_learner/lifecycle_temporal_fit.rs | 53 +- .../lifecycle_temporal_held_out_tests.rs | 14 +- .../room_learner/lifecycle_temporal_public.rs | 893 +++++++++++++ ...ecycle_temporal_public_acceptance_tests.rs | 1188 +++++++++++++++++ .../lifecycle_temporal_public_safety_tests.rs | 484 +++++++ .../lifecycle_temporal_selection.rs | 544 ++++++++ .../src/room_learner/mod.rs | 8 + .../src/room_learner/qualification.rs | 11 + .../src/room_learner/recovery_store.rs | 13 + .../room_learner/temporal_artifact_store.rs | 23 +- .../room_learner/temporal_capture_quality.rs | 80 +- .../src/room_learner/temporal_checkpoint.rs | 4 + .../src/room_learner/temporal_evaluation.rs | 571 ++++++++ .../temporal_evaluation_data_tests.rs | 439 ++++++ .../temporal_evaluation_projection.rs | 565 ++++++++ .../room_learner/temporal_evaluation_store.rs | 546 ++++++++ .../src/room_learner/temporal_full_run.rs | 9 + .../temporal_full_run_checkpoint.rs | 2 +- .../src/room_learner/temporal_live_driver.rs | 42 +- .../temporal_model_input_managed.rs | 26 + .../room_learner/temporal_qualification.rs | 348 +++++ .../temporal_recovery_commitment.rs | 49 +- .../room_learner/temporal_selection_store.rs | 523 ++++++++ .../src/room_learner/temporal_wire.rs | 540 ++++++++ .../src/room_learner/temporal_wire_tests.rs | 306 +++++ .../src/room_learner_api.rs | 6 +- .../src/room_learner_api_progress.rs | 49 + .../src/room_learner_api_publication.rs | 97 ++ .../src/room_learner_api_selection.rs | 8 +- .../src/room_learner_api_temporal.rs | 618 +++++++++ .../src/room_learner_api_temporal_tests.rs | 412 ++++++ .../src/room_learner_api_tests.rs | 3 + 43 files changed, 9239 insertions(+), 101 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/companion_atomic_write_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_evaluation.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public_acceptance_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public_safety_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_selection.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_data_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_projection.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_store.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_qualification.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_selection_store.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_wire.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_wire_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs diff --git a/scripts/ci/temporal_test_profiles.py b/scripts/ci/temporal_test_profiles.py index bf4b3a256b..0cd980ef92 100644 --- a/scripts/ci/temporal_test_profiles.py +++ b/scripts/ci/temporal_test_profiles.py @@ -5,6 +5,8 @@ PREFIX = 'room_learner::lifecycle::temporal_control::' C1 = PREFIX + 'tests::full_seven_train_phases_require_fresh_confirmation_and_never_issue_fit_authority' C2 = PREFIX + 'tests::c2_all_seven_train_phases_use_actual_async_runtime_and_durable_seals_without_fit' +# Exact C4C declaration; the author must verify this name in the actual Rust registry. +C4C = PREFIX + 'public_workflow::acceptance::c4c_default48_public_operator_workflow_native128_and_cold_selection' BUDGET = [PREFIX + 'tests::legacy_329_actual_runtime_saturation_retires_and_explicit_fresh_train_chains_343', PREFIX + 'tests::fresh_343_actual_runtime_all_128_gaps_seven_terminal_pauses_and_retirement_fit_budget'] # This initial policy is populated below from reviewed source call sites. C3 = ['room_learner::lifecycle::temporal_control::fit_control::tests::c3_actual_native_train_fit_and_same_store_cold_report_keep_whole_model_diagnostic', @@ -37,6 +39,10 @@ def policy(root): if not b_marker.is_file(): raise ValueError('C4B source marker is required; missing source is not coverage') components += [('C4B', b_marker, C4B, [(C4B_PREFIX, False, 11)])] + c_marker = source / 'lifecycle_temporal_public_acceptance_tests.rs' + if not c_marker.is_file(): + raise ValueError('C4C acceptance source marker is required; missing source is not coverage') + components += [('C4C', c_marker, [C4C], [(C4C, True, 1)])] active = [item for item in components if item[1].is_file()] if not active or active[0][0] != 'C2': raise ValueError('C2 classification/source must exist; zero selected tests is not coverage') diff --git a/scripts/ci/test_temporal_test_profiles.py b/scripts/ci/test_temporal_test_profiles.py index f20a8d824a..d3f11b0ad4 100644 --- a/scripts/ci/test_temporal_test_profiles.py +++ b/scripts/ci/test_temporal_test_profiles.py @@ -121,4 +121,53 @@ def after_compile(root,release): with self.assertRaises(ValueError):profiles.main() self.assertFalse(marker.exists(),'registry executed substituted out-of-target artifact') +class C4CClassificationTests(unittest.TestCase): + def component(self): + return [('C4C', None, [profiles.C4C], [(profiles.C4C, True, 1)])] + def test_exact_default48_case_is_one_required_release_case(self): + result = profiles.validate(self.component(), {profiles.C4C, profiles.C4C + '_lookalike'}, set(), 'release-run') + self.assertEqual(result, [{'component': 'C4C', 'selector': profiles.C4C, 'exact': True, 'names': [profiles.C4C]}]) + def test_debug_requires_the_same_case_registered_and_transparently_ignored(self): + result = profiles.validate(self.component(), {profiles.C4C}, {profiles.C4C}, 'debug-check') + self.assertEqual(result[0]['names'], [profiles.C4C]) + def test_default48_cannot_be_left_unclassified_in_debug(self): + with self.assertRaises(ValueError): + profiles.validate(self.component(), {profiles.C4C}, set(), 'debug-check') + def test_default48_cannot_be_ignored_in_required_release(self): + with self.assertRaises(ValueError): + profiles.validate(self.component(), {profiles.C4C}, {profiles.C4C}, 'release-run') + def test_zero_or_missing_case_is_not_release_coverage(self): + with self.assertRaises(ValueError): + profiles.validate(self.component(), set(), set(), 'release-run') + def test_same_leaf_in_a_different_module_cannot_satisfy_exact_registration(self): + wrong = profiles.C4C.replace('public_workflow::acceptance::', 'public_workflow::wrong_module::') + with self.assertRaises(ValueError): + profiles.validate(self.component(), {wrong}, set(), 'release-run') + def test_exact_selector_does_not_sweep_opt_in_siblings(self): + perf = profiles.C4C + '_manual_performance' + result = profiles.validate(self.component(), {profiles.C4C, perf}, {perf}, 'release-run') + self.assertEqual(result[0]['names'], [profiles.C4C]) + def test_missing_c4c_source_marker_cannot_silently_drop_release_batch(self): + import tempfile + with tempfile.TemporaryDirectory() as d: + root = pathlib.Path(d) + source = root / 'v2/crates/wifi-densepose-sensing-server/src/room_learner' + source.mkdir(parents=True) + for name in ['lifecycle_temporal_tests.rs', 'lifecycle_temporal_held_out_tests.rs']: + (source / name).write_text('// synthetic source marker only') + with self.assertRaisesRegex(ValueError, 'C4C acceptance source marker'): + profiles.policy(root) + def test_current_policy_registers_exact_c4c_and_preserves_existing_components(self): + components = profiles.policy(pathlib.Path(__file__).resolve().parents[2]) + self.assertEqual([c[0] for c in components], ['C2', 'C1', 'C3', 'C4', 'budget343', 'C4B', 'C4C']) + current = components[-1] + self.assertEqual(current[1].name, 'lifecycle_temporal_public_acceptance_tests.rs') + self.assertEqual(current[2:], ([profiles.C4C], [(profiles.C4C, True, 1)])) + self.assertEqual(components[0][3], [(profiles.C2, True, 1)]) + self.assertEqual(components[1][3], [(profiles.C1, True, 1)]) + self.assertEqual(components[2][3], [(profiles.PREFIX + 'fit_control::tests::', False, 12)]) + self.assertEqual(components[3][3], [(profiles.PREFIX + 'full_run_tests::', False, 8)]) + self.assertEqual(components[4][3], [(name, True, 1) for name in profiles.BUDGET]) + self.assertEqual(components[5][3], [(profiles.C4B_PREFIX, False, 11)]) + if __name__=='__main__':unittest.main() diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion.rs b/v2/crates/wifi-densepose-sensing-server/src/companion.rs index a43801a777..7a861f2cfb 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/companion.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/companion.rs @@ -90,7 +90,14 @@ pub(crate) fn private_read(path: &Path) -> io::Result> { } std::fs::read(path) } -pub(crate) fn atomic_private_write(path: &Path, data: &[u8]) -> io::Result<()> { +pub(crate) fn atomic_private_write(path:&Path,data:&[u8])->io::Result<()>{atomic_private_write_checked(path,data,&mut||Ok(()),&mut||Ok(()))} +/// One canonical writer body. Original callers use the no-op observer; C4C +/// instance-scoped test observers can exercise the actual post-rename boundary. +pub(crate) fn atomic_private_write_observed(path:&Path,data:&[u8],after_rename:&mut dyn FnMut()->io::Result<()>)->io::Result<()>{atomic_private_write_checked(path,data,&mut||Ok(()),after_rename)} +/// The optional admission callback runs after staging is fully synced and +/// immediately before the sole authoritative replacement. It cannot be delayed +/// by staging IO. A refusal removes only this invocation's own unique temp. +pub(crate) fn atomic_private_write_checked(path:&Path,data:&[u8],before_rename:&mut dyn FnMut()->io::Result<()>,after_rename:&mut dyn FnMut()->io::Result<()>)->io::Result<()>{ let temp = path.with_extension(format!("tmp-{}", secret())); let mut options = std::fs::OpenOptions::new(); options.create_new(true).write(true); @@ -102,7 +109,13 @@ pub(crate) fn atomic_private_write(path: &Path, data: &[u8]) -> io::Result<()> { let mut f = options.open(&temp)?; f.write_all(data)?; f.sync_all()?; + if let Err(error)=before_rename(){ + drop(f); + std::fs::remove_file(&temp)?; + return Err(error); + } std::fs::rename(&temp, path)?; + after_rename()?; std::fs::File::open(path.parent().ok_or_else(|| io::Error::other("parent"))?)?.sync_all()?; Ok(()) } @@ -967,3 +980,7 @@ mod tests { .all(|d| d["calibration_until"] == 0)); } } + +#[cfg(test)] +#[path="companion_atomic_write_tests.rs"] +mod atomic_write_tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/companion_atomic_write_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/companion_atomic_write_tests.rs new file mode 100644 index 0000000000..97d3f12af0 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/companion_atomic_write_tests.rs @@ -0,0 +1,66 @@ +//! Actual canonical writer boundary controls; no model/selection permission is +//! manufactured by these generic byte fixtures. +use super::*; +use std::cell::Cell; +#[test] +fn preinstall_refusal_after_synced_staging_preserves_pointer_and_cleans_own_temp() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pointer.json"); + atomic_private_write(&path, b"old-choice").unwrap(); + let observed = Cell::new(false); + let result = atomic_private_write_checked( + &path, + b"new-choice", + &mut || { + assert_eq!(std::fs::read(&path)?, b"old-choice"); + let staging = std::fs::read_dir(dir.path())? + .map(|e| e.unwrap().path()) + .filter(|p| p != &path) + .collect::>(); + assert_eq!(staging.len(), 1); + assert_eq!(std::fs::read(&staging[0])?, b"new-choice"); + observed.set(true); + Err(io::Error::other( + "synthetic-current-lease-expired-before-rename", + )) + }, + &mut || panic!("refused staging reached authoritative rename"), + ); + assert!(result.is_err()); + assert!(observed.get()); + assert_eq!(std::fs::read(&path).unwrap(), b"old-choice"); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); +} +#[test] +fn admitted_rename_then_ambiguous_sync_retains_exact_choice_for_cold_readback() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pointer.json"); + atomic_private_write(&path, b"old-choice").unwrap(); + let admitted = Cell::new(false); + let result = atomic_private_write_checked( + &path, + b"original-prepared-choice", + &mut || { + assert_eq!(std::fs::read(&path)?, b"old-choice"); + admitted.set(true); + Ok(()) + }, + &mut || { + assert!(admitted.get()); + assert_eq!(std::fs::read(&path)?, b"original-prepared-choice"); + Err(io::Error::other( + "synthetic-directory-sync-failure-after-valid-rename", + )) + }, + ); + assert!(result.is_err()); + assert_eq!(std::fs::read(&path).unwrap(), b"original-prepared-choice"); + assert_eq!(std::fs::read_dir(dir.path()).unwrap().count(), 1); +} +#[test] +fn original_noop_wrapper_keeps_private_exact_bytes() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("pointer.json"); + atomic_private_write(&path, b"unchanged-canonical-writer").unwrap(); + assert_eq!(private_read(&path).unwrap(), b"unchanged-canonical-writer"); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index aac71f77bd..885ffb276d 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -16488,20 +16488,20 @@ async fn main() { .route("/api/v1/calibration/cancel", post(calibration_cancel)) .route("/api/v1/calibration/status", get(calibration_status)) // Mutations remain behind the existing default-admin bearer policy. - .route("/api/v1/room-learner/status", get(room_learner_api::status)) - .route("/api/v1/room-learner/selection", get(room_learner_api::selection)) - .route("/api/v1/room-learner/progress", get(room_learner_api::progress)) + .route("/api/v1/room-learner/status", get(room_learner_api::status_dispatch)) + .route("/api/v1/room-learner/selection", get(room_learner_api::selection_dispatch)) + .route("/api/v1/room-learner/progress", get(room_learner_api::progress_dispatch)) .merge(room_learner_api::owner_selection_routes(bearer_auth_state.clone())) - .route("/api/v1/room-learner/configure", post(room_learner_api::configure)) - .route("/api/v1/room-learner/phase/start", post(room_learner_api::begin_phase)) - .route("/api/v1/room-learner/phase/finish", post(room_learner_api::finish_phase)) - .route("/api/v1/room-learner/phase/cancel", post(room_learner_api::cancel_phase)) - .route("/api/v1/room-learner/train", post(room_learner_api::train)) - .route("/api/v1/room-learner/train/cancel", post(room_learner_api::cancel_training)) - .route("/api/v1/room-learner/evaluate", post(room_learner_api::evaluate)) + .route("/api/v1/room-learner/configure", post(room_learner_api::configure_dispatch)) + .route("/api/v1/room-learner/phase/start", post(room_learner_api::begin_dispatch)) + .route("/api/v1/room-learner/phase/finish", post(room_learner_api::finish_dispatch)) + .route("/api/v1/room-learner/phase/cancel", post(room_learner_api::pause_dispatch)) + .route("/api/v1/room-learner/train", post(room_learner_api::train_dispatch)) + .route("/api/v1/room-learner/train/cancel", post(room_learner_api::cancel_operation_dispatch)) + .route("/api/v1/room-learner/evaluate", post(room_learner_api::evaluate_dispatch)) .route("/api/v1/room-learner/prediction", get(room_learner_api::prediction)) - .route("/api/v1/room-learner/recovery/restore", post(room_learner_api::restore_capture)) - .route("/api/v1/room-learner/recovery/discard", post(room_learner_api::discard_capture)) + .route("/api/v1/room-learner/recovery/restore", post(room_learner_api::restore_dispatch)) + .route("/api/v1/room-learner/recovery/discard", post(room_learner_api::discard_dispatch)) .route("/api/v1/calibration/reset", post(calibration_reset)) .route("/api/v1/calibration/bootstrap/recover", post(calibration_recover_bootstrap)) // ADR-044 §5.3: runtime-configurable dedup factor diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs index 7e2f82b16c..da22a95e47 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/artifact.rs @@ -321,3 +321,11 @@ pub(crate) use temporal_fit_summary::FitDiagnosticState; #[cfg(test)] pub(crate) use temporal_fit_summary::rehash_fixture as rehash_temporal_fit_fixture; + +#[path="temporal_evaluation_store.rs"] +mod temporal_evaluation_store; +pub(crate) use temporal_evaluation_store::{StoredTemporalEvaluation,StoredTemporalAssessment}; + +#[path="temporal_selection_store.rs"] +mod temporal_selection_store; +pub(crate) use temporal_selection_store::StoredTemporalSelection; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs index e49e6a1b6d..f15e68adfc 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs @@ -80,7 +80,7 @@ impl Inner { }) } fn require_legacy_kind(&self) -> Result<(), String> { - if matches!(self.capture, Some(CaptureWorkflow::Temporal(_))) { + if matches!(self.capture, Some(CaptureWorkflow::Temporal(_)))||self.selection.temporal.is_some() { Err("learner_temporal_capture_private_unsupported_command".into()) } else { Ok(()) } } @@ -98,6 +98,8 @@ pub struct RoomLearningRuntime { panic_after_reclamation_ack:AtomicBool, #[cfg(test)] temporal_fit_hook: Mutex>>, + #[cfg(test)] + temporal_evidence_io_hook: MutexResult<(),String> + Send + Sync>>>, clock_origin: std::time::Instant, clock_epoch_ns: u64, } @@ -195,6 +197,7 @@ impl RoomLearningRuntime { #[cfg(test)] reclamation_probe:Mutex::new(None), #[cfg(test)] panic_after_reclamation_ack:AtomicBool::new(false), #[cfg(test)] temporal_fit_hook:Mutex::new(None), + #[cfg(test)] temporal_evidence_io_hook:Mutex::new(None), inner: Mutex::new(Inner { generation: 0, workflow_revision: 0, @@ -213,12 +216,13 @@ impl RoomLearningRuntime { qualification_error: None, }), }; + let temporal_selected=runtime.store.load_temporal_selection().and_then(|stored|stored.map(temporal_control::selected_control::TemporalSelectedRuntime::new).transpose()); { let mut state = runtime .inner .lock() .map_err(|_| "learner state unavailable")?; - runtime.restore_selection(&mut state); + match temporal_selected{Ok(Some(selected))=>{state.selection.temporal=Some(Arc::new(selected));},Ok(None)=>runtime.restore_selection(&mut state),Err(error)=>state.selection.error=Some(bounded(error))} runtime.inspect_recovery(&mut state, runtime.receiver_now_ns()); } Ok(runtime) @@ -451,14 +455,16 @@ impl RoomLearningRuntime { .live_cohort(now_ns)?; Ok((self.guard_for(&state), model.infer(&cohort))) } - pub fn status(&self, now_ns: u64) -> Value { + pub fn status(&self,now_ns:u64)->Value{self.status_with_checkpoint(now_ns,true)} + pub(super) fn status_without_checkpoint(&self,now_ns:u64)->Value{self.status_with_checkpoint(now_ns,false)} + fn status_with_checkpoint(&self,now_ns:u64,checkpoint:bool)->Value{ let Ok(mut state) = self.inner.lock() else { return json!({"enabled":true,"state":"unavailable"}); }; if state.require_legacy_kind().is_err() { return json!({"schema":"ruview.room-learner.status.v1","enabled":true,"success":false,"error_code":"learner_temporal_capture_private_unsupported_command"}); } - let _ = self.checkpoint_recovery(&mut state, now_ns, false); + if checkpoint{let _ = self.checkpoint_recovery(&mut state, now_ns, false);} let required_ids: Vec<_> = state.legacy_capture() .map(|c| c.context().sources.iter().map(|s| s.source_id).collect()) .unwrap_or_default(); diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs index 88350e8e30..d43a70fc44 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs @@ -18,7 +18,7 @@ impl RoomLearningRuntime { Ok(context) } pub fn selects_grid(&self, source_id: u8, grid_id: &str) -> bool { - self.inner.lock().ok().and_then(|state| state.capture_context().map(|c| c.sources.iter().any(|s| s.source_id == source_id && s.grid_id == grid_id))).unwrap_or(false) + self.inner.lock().map(|state|state.capture_context().is_some_and(|c|c.sources.iter().any(|s|s.source_id==source_id&&s.grid_id==grid_id))||state.selection.temporal.as_ref().is_some_and(|selected|selected.stored.envelope().receipt.selected.context.sources.iter().any(|s|s.source_id==source_id&&s.grid_id==grid_id))).unwrap_or(false) } pub fn capture_active(&self) -> Result { let state = self.inner.lock().map_err(|_| "learner state unavailable")?; @@ -96,10 +96,25 @@ impl RoomLearningRuntime { self.record_native_rejection(packet.source_id,&packet.grid_id,"synthetic/replayed native provenance"); return Err("live managed capture rejects synthetic/replayed native provenance".into()); } + // Selected whole-model inference runs outside Runtime.inner. A stale + // Arc cannot publish evidence after selection is replaced; reads fence + // the exact current Arc again before returning a usable decision. + let selected={self.inner.lock().map_err(|_|"learner state unavailable")?.selection.temporal.clone()}; + let selected_present=selected.is_some(); + if let Some(selected)=selected{if selected.stored.envelope().receipt.selected.context.sources.iter().any(|s|s.source_id==packet.source_id&&s.grid_id==packet.grid_id&&s.center_frequency_hz==packet.frequency_hz&&s.bandwidth_hz==packet.bandwidth_hz){ + let mut driver=selected.driver.lock().map_err(|_|"selected_native_input_unavailable")?; + // Poll/inference may have advanced processing time while this + // packet waited for the driver. Measure the trusted processing + // clock HERE; preserve its original receive timestamp/identities. + let processing_now=self.receiver_now_ns(); + if processing_now, + pub(super) temporal:Option>, + pub(super) withdrawal:Option, model: Option>, qualification: Option, - error: Option, + pub(super) error: Option, assessment: Option, } impl SelectionState { + pub(super) fn known_revision(&self)->u64{self.temporal.as_ref().map(|s|s.stored.identity().selection_revision).or_else(||self.envelope.as_ref().map(|e|e.receipt.revision)).unwrap_or(0)} + pub(super) fn install_negative_temporal_tombstone(&mut self,envelope:SelectionEnvelope){self.envelope=Some(envelope);self.model=None;self.qualification=None;self.assessment=None;} + + pub(super) fn blocks_new_temporal_capture(&self)->bool{self.temporal.is_some()||self.envelope.as_ref().is_some_and(|e|e.receipt.selected.is_some())||self.error.is_some()} pub(super) fn has_history(&self) -> bool { - self.envelope.is_some() || self.model.is_some() || self.qualification.is_some() + self.temporal.is_some() || self.envelope.is_some() || self.model.is_some() || self.qualification.is_some() || self.assessment.is_some() || self.error.is_some() } } @@ -175,6 +181,8 @@ impl RoomLearningRuntime { } // Only publish in-memory authority after durable pointer and readback succeed. state.selection = SelectionState { + temporal:None, + withdrawal:None, envelope: Some(envelope.clone()), model: resolved.as_ref().map(|(m, _)| Arc::new(m.clone())), qualification: resolved.map(|(_, q)| q), diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs index e162e07741..f968b7c8d1 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs @@ -53,7 +53,12 @@ pub(super) struct RuntimeTemporalCapture { failure: Option, last_now_ns: u64, fit_job: Option, + evidence_job: Option, + training_proof: Option, + evaluation: Option>, + assessment: Option>, diagnostic_fit: Option, + public_train_requested: bool, } impl RuntimeTemporalCapture { pub(super) fn owner_generation(&self) -> u64 { @@ -109,6 +114,7 @@ impl RuntimeTemporalCapture { return Err(error); } if let Some(job) = &self.fit_job { job.observe_now(now); } + if let Some(job) = &self.evidence_job { job.observe_now(now); } self.last_now_ns = now; Ok(()) } @@ -274,6 +280,10 @@ impl RoomLearningRuntime { fn configure_temporal_capture_with_custody( &self,expected:&MutationGuard,context:CaptureContext,run:String,now:u64, prepare:impl FnOnce(ManagedScheduleContract,String,FitConfig,u64,&super::super::recovery_store::RecoveryStore,FullRunGenesisWitness)->Result<(TemporalTrainCapture,Option),String>, + )->Result<(),String>{self.configure_temporal_capture_with_custody_mode(expected,context,run,now,false,prepare)} + fn configure_temporal_capture_with_custody_mode( + &self,expected:&MutationGuard,context:CaptureContext,run:String,now:u64,replace_empty_legacy:bool, + prepare:impl FnOnce(ManagedScheduleContract,String,FitConfig,u64,&super::super::recovery_store::RecoveryStore,FullRunGenesisWitness)->Result<(TemporalTrainCapture,Option),String>, )->Result<(),String>{ context.validate()?; if context.owner_scope != self.owner @@ -289,10 +299,12 @@ impl RoomLearningRuntime { let fit = FitConfig::for_source_count(context.sources.len()); fit.validate_context(&context)?; let mut state = self.command_state(expected)?; - if state.capture.is_some() - || state.model.is_some() - || state.selection.has_history() - || state.job.phase != JobPhase::Idle + let empty_legacy=replace_empty_legacy&&matches!(state.capture.as_ref(),Some(CaptureWorkflow::Legacy(c))if !c.phase_active()&&c.cohorts().is_empty()); + let public_fresh=replace_empty_legacy&&(state.capture.is_none()||empty_legacy)&&!busy(&state.job)&&!state.selection.blocks_new_temporal_capture(); + if (state.capture.is_some()&&!empty_legacy) + || (state.model.is_some()&&!public_fresh) + || (state.selection.has_history()&&!public_fresh) + || (state.job.phase != JobPhase::Idle&&!public_fresh) || state.recovery.descriptor.is_some() { return Err("temporal_setup_would_replace_existing_work".into()); @@ -303,7 +315,10 @@ impl RoomLearningRuntime { .checked_add(1) .ok_or("learner generation exhausted")?; let witness=FullRunGenesisWitness{runtime_epoch:self.runtime_epoch.clone(),generation:next_generation,context:context.clone(),run:run.clone()}; - state.capture = Some(CaptureWorkflow::Temporal(Box::new( + // Historical immutable models remain in ArtifactStore. Retire only + // their RAM presentation when explicitly starting a new consented run. + let retired_legacy=if public_fresh{let old=(state.model.take(),state.receipt.take(),state.last_evaluation.take(),state.qualification.take());state.job=Job::default();state.frozen_training=false;state.qualification_error=None;Some(old)}else{None}; + let retired_capture=state.capture.replace(CaptureWorkflow::Temporal(Box::new( RuntimeTemporalCapture { context: context.clone(), owner_generation: next_generation, @@ -312,11 +327,16 @@ impl RoomLearningRuntime { failure: None, last_now_ns: now, fit_job: None, + evidence_job: None, + training_proof: None, + evaluation: None, + assessment: None, diagnostic_fit: None, + public_train_requested: false, }, ))); state.generation = next_generation; - drop(state); + drop(state);drop(retired_capture);drop(retired_legacy); // Initial crypto/filesystem execution never holds the receiver lock. let result = prepare(schedule, run, fit, now, store,witness); let (core, failure) = match result { @@ -342,7 +362,8 @@ impl RoomLearningRuntime { } Ok(()) } - fn start_temporal_phase(&self, expected: &MutationGuard, now: u64) -> Result<(), String> { + fn start_temporal_phase(&self, expected: &MutationGuard, now: u64) -> Result<(), String> {self.start_temporal_phase_checked(expected,now,|_|Ok(()))} + fn start_temporal_phase_checked(&self,expected:&MutationGuard,now:u64,check:impl FnOnce(&RuntimeTemporalCapture)->Result<(),String>)->Result<(),String>{ let mut state = self.command_state(expected)?; if busy(&state.job) { return Err("temporal_fit_job_busy".into()); } let owner_generation = state.generation; @@ -356,6 +377,7 @@ impl RoomLearningRuntime { .map(|c| c.invalidate("temporal_owner_generation_changed")); return Err("temporal_owner_generation_changed".into()); } + check(capture)?; capture.check_time(now)?; let core = capture .core @@ -447,12 +469,8 @@ impl RoomLearningRuntime { .ok_or("temporal_initial_barrier_unconfirmed")? .confirmation_challenge() } - fn confirm_temporal_phase( - &self, - expected: &MutationGuard, - challenge: &str, - now: u64, - ) -> Result<(), String> { + fn confirm_temporal_phase(&self,expected:&MutationGuard,challenge:&str,now:u64)->Result<(),String>{self.confirm_temporal_phase_checked(expected,challenge,now,|_|Ok(()))} + fn confirm_temporal_phase_checked(&self,expected:&MutationGuard,challenge:&str,now:u64,check:impl FnOnce(&RuntimeTemporalCapture)->Result<(),String>)->Result<(),String>{ let store = self.temporal_store()?; let mut state = self.command_state(expected)?; if busy(&state.job) { return Err("temporal_fit_job_busy".into()); } @@ -467,6 +485,7 @@ impl RoomLearningRuntime { .map(|c| c.invalidate("temporal_owner_generation_changed")); return Err("temporal_owner_generation_changed".into()); } + check(capture)?; capture.check_time(now)?; if capture.lead_in_until.is_some() { return Err("temporal_phase_incomplete".into()); @@ -547,7 +566,12 @@ impl RoomLearningRuntime { failure: None, last_now_ns: now, fit_job: None, + evidence_job: None, + training_proof: None, + evaluation: None, + assessment: None, diagnostic_fit: None, + public_train_requested: false, }, ))); state.generation = next_generation; @@ -585,7 +609,12 @@ impl RoomLearningRuntime { failure: None, last_now_ns: now, fit_job: None, + evidence_job: None, + training_proof: None, + evaluation: None, + assessment: None, diagnostic_fit: None, + public_train_requested: false, }, ))); state.generation = next_generation; @@ -613,33 +642,26 @@ impl RoomLearningRuntime { Ok(()) } fn cancel_temporal_durability(&self, expected: &MutationGuard) -> Result<(), String> { - let (writer, fit) = { - let mut state = self.command_state(expected)?; - let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() else { - return Err(PRIVATE_KIND.into()); - }; - capture.lead_in_until = None; - let fit = capture.fit_job.clone(); - capture.cancel_fit(); - let core = capture - .core - .as_mut() - .ok_or("temporal_initial_barrier_unconfirmed")?; - let writer = core.writer(); - core.invalidate("temporal_owner_write_cancelled"); - if let Some(writer) = &writer { - writer.cancel(); - } - if busy(&state.job) { - state.job.cancel.store(true, Ordering::Release); - state.job.phase = JobPhase::Cancelling; - } - (writer, fit) + let (writer,fit,evidence,withdrawal,retired_selected)={ + let mut state=self.command_state(expected)?; + let withdrawal=state.selection.withdrawal.clone(); + let has_selected=state.selection.temporal.is_some(); + if !matches!(state.capture,Some(CaptureWorkflow::Temporal(_)))&&withdrawal.is_none()&&!has_selected{return Err(PRIVATE_KIND.into());} + if let Some(job)=&withdrawal{job.cancel();} + let retired_selected=state.selection.temporal.take(); + let(writer,fit,evidence)=match state.capture.as_mut(){Some(CaptureWorkflow::Temporal(capture))=>{ + capture.lead_in_until=None;capture.public_train_requested=false; + let fit=capture.fit_job.clone();let evidence=capture.evidence_job.clone();capture.cancel_fit(); + let writer=capture.core.as_mut().and_then(|core|{let writer=core.writer();core.invalidate("temporal_owner_write_cancelled");writer}); + if let Some(writer)=&writer{writer.cancel();}(writer,fit,evidence) + },_=>(None,None,None)}; + if busy(&state.job){state.job.cancel.store(true,Ordering::Release);state.job.phase=JobPhase::Cancelling;} + (writer,fit,evidence,withdrawal,retired_selected) }; - let writer_result = writer.map_or(Ok(()), |writer| writer.drain()); - if let Some(fit) = fit { fit.drain(); } - writer_result?; - Ok(()) + let writer_result=writer.map_or(Ok(()),|writer|writer.drain()); + if let Some(fit)=fit{fit.drain();}if let Some(evidence)=evidence{evidence.drain();}if let Some(job)=withdrawal{job.drain();} + if let Some(selected)=retired_selected{selected.driver.lock().map_err(|_|"selected_driver_unavailable")?.deactivate();drop(selected);} + writer_result } fn temporal_capture_snapshot(&self, now: u64) -> Result { let state = self.inner.lock().map_err(|_| "learner state unavailable")?; @@ -675,7 +697,10 @@ impl Drop for RoomLearningRuntime { Some(CaptureWorkflow::Temporal(capture)) => capture.fit_job.clone(), _ => None, }; + let evidence=match state.capture.as_ref(){Some(CaptureWorkflow::Temporal(capture))=>capture.evidence_job.clone(),_=>None}; + let withdrawal=state.selection.withdrawal.clone();if let Some(job)=&withdrawal{job.cancel();} if let Some(fit) = &fit { fit.cancel(); } + if let Some(job)=&evidence{job.cancel();} let writer = match state.capture.as_mut() { Some(CaptureWorkflow::Temporal(capture)) => capture.core.as_mut().and_then(|core| { let writer = core.writer(); @@ -688,9 +713,21 @@ impl Drop for RoomLearningRuntime { let _ = writer.drain(); } if let Some(fit) = fit { fit.drain(); } + if let Some(job)=evidence{job.drain();} + if let Some(job)=withdrawal{job.drain();} + if let Some(selected)=state.selection.temporal.take(){if let Ok(mut driver)=selected.driver.lock(){driver.deactivate();}} } } #[cfg(test)] #[path="lifecycle_temporal_held_out_tests.rs"] mod held_out_tests; + +#[path="lifecycle_temporal_evaluation.rs"] +mod evidence_control; + +#[path="lifecycle_temporal_public.rs"] +mod public_workflow; + +#[path="lifecycle_temporal_selection.rs"] +pub(super) mod selected_control; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_evaluation.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_evaluation.rs new file mode 100644 index 0000000000..6776a0f496 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_evaluation.rs @@ -0,0 +1,536 @@ +//! C4C uses the SAME Runtime Job/executor/lifetime. No JSON authority issuer. +use super::fit_control::FitLifetime; +use super::*; +use crate::room_learner::artifact::StoredTemporalEvaluation; +use crate::room_learner::temporal_evaluation::TemporalTrainProofData; +use crate::room_learner::temporal_model_input::managed::execution::full_run::{ + PendingTemporalEvidenceSource, TemporalEvaluationBinding, +}; +use std::sync::{atomic::AtomicU64, Condvar, Weak}; +use std::time::Instant; +#[derive(Clone, Copy, PartialEq, Eq)] +pub(super) enum EvidencePurpose { + TrainProof, + HeldOutEvaluation, + Qualification, + Selection, + RestoreComplete, +} +#[derive(Clone, PartialEq)] +struct EvidenceIdentity { + epoch: String, + generation: u64, + attempt: u64, + purpose: EvidencePurpose, + binding: TemporalEvaluationBinding, +} +#[derive(Clone)] +pub(super) struct TemporalEvidenceJob { + identity: EvidenceIdentity, + lifetime: Arc, +} +impl TemporalEvidenceJob { + pub(super) fn uses_cancel(&self, cancel: &Arc) -> bool { + Arc::ptr_eq(cancel, &self.lifetime.cancel) + } + pub(super) fn observe_now(&self, now: u64) { + self.lifetime.observed.fetch_max(now, Ordering::AcqRel); + } + pub(super) fn cancel(&self) { + self.lifetime.cancel(); + } + pub(super) fn drain(&self) { + self.lifetime.drain(); + } + pub(super) fn done(&self) -> bool { + self.lifetime.done() + } + pub(super) fn operation_name(&self) -> &'static str { + match self.identity.purpose { + EvidencePurpose::TrainProof => "train_proof", + EvidencePurpose::HeldOutEvaluation => "evaluate", + EvidencePurpose::Qualification => "qualify", + EvidencePurpose::Selection => "select", + EvidencePurpose::RestoreComplete => "restore", + } + } +} +enum EvidenceOutcome { + Train(TemporalTrainProofData), + Evaluation(StoredTemporalEvaluation), + Assessment(crate::room_learner::artifact::StoredTemporalAssessment), + Selection(Arc), + Restored(TemporalTrainProofData, Option), +} +struct EvidenceCompletion { + runtime: Weak, + owner: Option>, + identity: EvidenceIdentity, + lifetime: Arc, +} +impl Drop for EvidenceCompletion { + fn drop(&mut self) { + let fallback = if self.owner.is_none() { + self.runtime.upgrade() + } else { + None + }; + if let Some(runtime) = self.owner.as_ref().or(fallback.as_ref()) { + let mut state = runtime.inner.lock().unwrap_or_else(|e| e.into_inner()); + let current = Arc::ptr_eq(&state.job.cancel, &self.lifetime.cancel) + && matches!(state.capture.as_ref(),Some(CaptureWorkflow::Temporal(c))if c.evidence_job.as_ref().is_some_and(|j|j.identity==self.identity&&Arc::ptr_eq(&j.lifetime,&self.lifetime))); + if current && busy(&state.job) { + state.job.phase = if self.lifetime.cancel.load(Ordering::Acquire) { + JobPhase::Cancelled + } else { + JobPhase::Failed + }; + state.job.error = Some("temporal_evidence_worker_unavailable".into()); + } + } + // Every result/raw snapshot and IO operation has completed before this + // signal. A last upgraded Runtime Arc may now safely enter its Drop. + *self + .lifetime + .completed + .lock() + .unwrap_or_else(|e| e.into_inner()) = true; + self.lifetime.completion.notify_all(); + drop(fallback); + } +} +impl RoomLearningRuntime { + pub(super) fn start_temporal_evidence( + self: &Arc, + expected: &MutationGuard, + now: u64, + purpose: EvidencePurpose, + ) -> Result { + let executor = tokio::runtime::Handle::try_current() + .map_err(|_| "temporal_evaluation_executor_unavailable")?; + let recovery = self + .recovery_store + .clone() + .ok_or("temporal_recovery_unavailable")?; + let store = self.store.clone(); + let (input, identity, lifetime) = self.reserve_temporal_evidence(expected, now, purpose)?; + let attempt = identity.attempt; + let completion = EvidenceCompletion { + runtime: Arc::downgrade(self), + owner: None, + identity: identity.clone(), + lifetime: lifetime.clone(), + }; + executor.spawn_blocking(move || { + let mut completion = completion; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + Self::compute_temporal_evidence(&store, &recovery, input, &identity, &lifetime) + })) + .unwrap_or_else(|_| Err("temporal_evaluation_worker_failed_safely".into())); + completion.owner = completion.runtime.upgrade(); + if let Some(runtime) = &completion.owner { + runtime.admit_temporal_evidence(&identity, &lifetime, result); + } else { + drop(result); + } + }); + Ok(attempt) + } + fn reserve_temporal_evidence( + &self, + expected: &MutationGuard, + now: u64, + purpose: EvidencePurpose, + ) -> Result< + ( + PendingTemporalEvidenceSource, + EvidenceIdentity, + Arc, + ), + String, + > { + let mut retired_job = None; + let mut retired_evidence = None; + let (input, identity, lifetime) = { + let mut state = self.command_state(expected)?; + if busy(&state.job) { + return Err("learner_job_busy".into()); + } + let generation = state.generation; + let attempt = state.workflow_revision; + let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_mut() else { + return Err(PRIVATE_KIND.into()); + }; + if c.owner_generation != generation + || c.failure.is_some() + || c.evidence_job.as_ref().is_some_and(|j| !j.done()) + || c.fit_job.as_ref().is_some_and(|j| !j.done()) + { + return Err("temporal_evaluation_owner_or_job_busy".into()); + } + c.check_time(now)?; + let input = c + .core + .as_ref() + .ok_or("temporal_initial_barrier_unconfirmed")? + .prepare_evaluation_source(now, purpose != EvidencePurpose::TrainProof)?; + if !input + .binding() + .owner_matches(&self.runtime_epoch, generation) + { + return Err("temporal_evaluation_owner_changed".into()); + } + let identity = EvidenceIdentity { + epoch: self.runtime_epoch.clone(), + generation, + attempt, + purpose, + binding: input.binding().clone(), + }; + let cancel = Arc::new(AtomicBool::new(false)); + let lifetime = Arc::new(FitLifetime { + cancel: cancel.clone(), + publication: Mutex::new(()), + completed: Mutex::new(false), + completion: Condvar::new(), + anchor: now, + origin: Instant::now(), + observed: AtomicU64::new(now), + expires_at_ms: input.expires_at_ms(), + #[cfg(test)] + hook: self + .temporal_fit_hook + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(), + #[cfg(test)] + io_hook: self + .temporal_evidence_io_hook + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(), + }); + retired_evidence = c.evidence_job.replace(TemporalEvidenceJob { + identity: identity.clone(), + lifetime: lifetime.clone(), + }); + retired_job = Some(std::mem::replace( + &mut state.job, + Job { + phase: JobPhase::Running, + cancel, + report: None, + error: None, + }, + )); + if purpose == EvidencePurpose::Selection { + state.selection.error=Some("Selection transition pending; current evidence is unavailable until durable reconciliation.".into()); + } + (input, identity, lifetime) + }; + drop(retired_evidence); + drop(retired_job); + Ok((input, identity, lifetime)) + } + pub(super) fn run_temporal_selection( + self: &Arc, + expected: &MutationGuard, + now: u64, + ) -> Result<(), String> { + let recovery = self + .recovery_store + .clone() + .ok_or("temporal_recovery_unavailable")?; + let store = self.store.clone(); + let (input, identity, lifetime) = + self.reserve_temporal_evidence(expected, now, EvidencePurpose::Selection)?; + let completion = EvidenceCompletion { + runtime: Arc::downgrade(self), + owner: Some(self.clone()), + identity: identity.clone(), + lifetime: lifetime.clone(), + }; + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + Self::compute_temporal_evidence(&store, &recovery, input, &identity, &lifetime) + })) + .unwrap_or_else(|_| Err("temporal_selection_worker_failed_safely".into())); + let failure = result.as_ref().err().cloned(); + self.admit_temporal_evidence(&identity, &lifetime, result); + // Finish/drain tracks the actual store operation, not an HTTP ACK. + drop(completion); + if let Some(error) = failure { + return Err(error); + } + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if state.job.phase != JobPhase::Ready || !Arc::ptr_eq(&state.job.cancel, &lifetime.cancel) { + return Err("temporal_selection_not_adopted".into()); + } + Ok(()) + } + fn compute_temporal_evidence( + store: &crate::room_learner::artifact::ArtifactStore, + recovery: &crate::room_learner::recovery_store::RecoveryStore, + input: PendingTemporalEvidenceSource, + identity: &EvidenceIdentity, + lifetime: &FitLifetime, + ) -> Result { + lifetime.at("evidence_before_validate"); + let now = lifetime.allowed()?; + let outcome: Result = match identity.purpose { + EvidencePurpose::TrainProof => { + let source = input.recertify_train(recovery, now, &lifetime.cancel)?; + let _publication = lifetime + .publication + .lock() + .map_err(|_| "temporal_evaluation_publication_unavailable")?; + lifetime.at("evidence_publication_entered"); + let proof = store.persist_current_temporal_train_proof( + &source, + &lifetime.cancel, + &mut || lifetime.allowed(), + &mut |stage| { + lifetime.allowed()?; + lifetime.io(stage) + }, + )?; + lifetime.allowed()?; + Ok(EvidenceOutcome::Train(proof)) + } + EvidencePurpose::RestoreComplete => { + let source = input.recertify_evaluation(recovery, now, &lifetime.cancel)?; + let _publication = lifetime + .publication + .lock() + .map_err(|_| "temporal_restore_publication_unavailable")?; + let (proof, evaluation) = store.reconcile_temporal_snapshot( + source, + &lifetime.cancel, + &mut || lifetime.allowed(), + &mut |stage| lifetime.io(stage), + )?; + Ok(EvidenceOutcome::Restored(proof, evaluation)) + } + EvidencePurpose::Selection => { + let source = input.recertify_evaluation(recovery, now, &lifetime.cancel)?; + let _publication = lifetime + .publication + .lock() + .map_err(|_| "temporal_selection_publication_unavailable")?; + let revision = store.selection_revision_any()?; + let assessment = store.qualify_temporal_snapshot( + &source, + &lifetime.cancel, + &mut || lifetime.allowed(), + &mut |stage| lifetime.io(stage), + )?; + let stored = store.save_current_temporal_selection( + &source, + &assessment, + revision, + lifetime.allowed()?, + &lifetime.cancel, + &mut || lifetime.allowed(), + &mut || { + lifetime.io("selection_before_pointer_install")?; + identity + .binding + .recertify_current(recovery, lifetime.allowed()?)?; + lifetime.allowed()?; + Ok(()) + }, + &mut || lifetime.io("selection_after_pointer_rename"), + )?; + Ok(EvidenceOutcome::Selection(Arc::new( + super::selected_control::TemporalSelectedRuntime::new(stored)?, + ))) + } + EvidencePurpose::Qualification => { + let source = input.recertify_evaluation(recovery, now, &lifetime.cancel)?; + let _publication = lifetime + .publication + .lock() + .map_err(|_| "temporal_evaluation_publication_unavailable")?; + lifetime.at("evidence_publication_entered"); + let receipt = store.qualify_temporal_snapshot( + &source, + &lifetime.cancel, + &mut || lifetime.allowed(), + &mut |stage| { + lifetime.allowed()?; + lifetime.io(stage) + }, + )?; + lifetime.allowed()?; + Ok(EvidenceOutcome::Assessment(receipt)) + } + EvidencePurpose::HeldOutEvaluation => { + let source = input.recertify_evaluation(recovery, now, &lifetime.cancel)?; + let _publication = lifetime + .publication + .lock() + .map_err(|_| "temporal_evaluation_publication_unavailable")?; + lifetime.at("evidence_publication_entered"); + let receipt = store.evaluate_temporal_snapshot( + source, + &lifetime.cancel, + &mut || lifetime.allowed(), + &mut |stage| { + lifetime.allowed()?; + lifetime.io(stage) + }, + )?; + lifetime.allowed()?; + Ok(EvidenceOutcome::Evaluation(receipt)) + } + }; + let outcome = outcome?; + identity + .binding + .recertify_current(recovery, lifetime.allowed()?)?; + let model = store.load_exact_temporal_candidate( + identity.binding.context(), + identity.binding.model_sha256(), + )?; + if model.artifact_kind() != crate::room_learner::temporal_model_input::managed::SCHEMA_V3 { + return Err("temporal_evaluation_model_kind_changed".into()); + } + lifetime.allowed()?; + Ok(outcome) + } + fn admit_temporal_evidence( + &self, + identity: &EvidenceIdentity, + lifetime: &Arc, + outcome: Result, + ) { + // Outcome and replaced metadata are destroyed after inner is released. + let mut retired_proof = None; + let mut retired_evaluation = None; + let mut retired_assessment = None; + let mut retired_selection = None; + let mut state = match self.inner.lock() { + Ok(s) => s, + Err(_) => return, + }; + let same = identity.epoch == self.runtime_epoch + && identity.generation == state.generation + && Arc::ptr_eq(&state.job.cancel, &lifetime.cancel) + && matches!(state.capture.as_ref(),Some(CaptureWorkflow::Temporal(c))if c.owner_generation==identity.generation&&c.failure.is_none()&&c.evidence_job.as_ref().is_some_and(|j|j.identity==*identity&&Arc::ptr_eq(&j.lifetime,lifetime))&&c.core.as_ref().is_some_and(|core|core.evaluation_binding_matches(&identity.binding))); + if !same { + return; + } + if lifetime.allowed().is_err() { + state.job.phase = JobPhase::Cancelled; + state.job.error = Some("temporal_evaluation_cancelled_or_expired".into()); + return; + } + state.workflow_revision = state.workflow_revision.saturating_add(1); + match outcome { + Ok(EvidenceOutcome::Train(proof)) => { + if let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_mut() { + retired_proof = c.training_proof.replace(proof); + } + state.job.phase = JobPhase::Ready; + state.job.error = None; + } + Ok(EvidenceOutcome::Evaluation(receipt)) => { + if receipt.binding() != &identity.binding { + state.job.phase = JobPhase::Failed; + state.job.error = Some("temporal_evaluation_result_identity_changed".into()); + return; + } + if let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_mut() { + retired_proof = c.training_proof.replace(receipt.train().clone()); + retired_evaluation = c.evaluation.replace(Arc::new(receipt)); + } + state.job.phase = JobPhase::Ready; + state.job.error = None; + } + Ok(EvidenceOutcome::Assessment(receipt)) => { + if receipt.binding() != &identity.binding { + state.job.phase = JobPhase::Failed; + state.job.error = Some("temporal_assessment_result_identity_changed".into()); + return; + } + if let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_mut() { + retired_assessment = c.assessment.replace(Arc::new(receipt)); + } + state.job.phase = JobPhase::Ready; + state.job.error = None; + } + Ok(EvidenceOutcome::Restored(proof, evaluation)) => { + if let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_mut() { + retired_proof = c.training_proof.replace(proof); + if let Some(evaluation) = evaluation { + retired_evaluation = c.evaluation.replace(Arc::new(evaluation)); + } + } + state.job.phase = JobPhase::Ready; + state.job.error = None; + } + Ok(EvidenceOutcome::Selection(selected)) => { + let floors = match state.capture.as_ref() { + Some(CaptureWorkflow::Temporal(c)) => { + c.core.as_ref().map(|core| core.current_source_floors()) + } + _ => None, + }; + if let Some((floors, receiver)) = floors { + let seeded = selected + .driver + .lock() + .map_err(|_| "selected driver unavailable".to_string()) + .and_then(|mut d| d.advance_selected_floors(&floors, receiver)); + if let Err(error) = seeded { + state.job.phase = JobPhase::Failed; + state.job.error = Some(error); + return; + } + } + retired_selection = state.selection.temporal.replace(selected); + state.selection.error = None; + state.job.phase = JobPhase::Ready; + state.job.error = None; + } + Err(error) => { + state.job.phase = if lifetime.cancel.load(Ordering::Acquire) { + JobPhase::Cancelled + } else { + JobPhase::Failed + }; + state.job.error = Some(bounded(error)); + } + } + } +} +impl RoomLearningRuntime { + pub(super) fn cancel_temporal_evidence( + &self, + expected: &MutationGuard, + ) -> Result { + let job = { + let mut state = self.command_state(expected)?; + let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_ref() else { + return Err(PRIVATE_KIND.into()); + }; + let job = c + .evidence_job + .clone() + .ok_or("temporal_evaluation_job_missing")?; + if state.job.phase == JobPhase::Ready + && Arc::ptr_eq(&state.job.cancel, &job.lifetime.cancel) + && c.failure.is_none() + { + return Ok(JobPhase::Ready); + } + job.cancel(); + if busy(&state.job) { + state.job.phase = JobPhase::Cancelling; + } + job + }; + job.drain(); + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + Ok(state.job.phase) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit.rs index 12b408283f..01c7fd79e7 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit.rs @@ -42,20 +42,27 @@ impl AcceptedTemporalFit { } /// One lifetime barrier for the SAME existing Job/executor, not a new queue or /// trainer. Cancellation is atomic; waiting/publication never holds inner. -struct FitLifetime { - cancel: Arc, - publication: Mutex<()>, - completed: Mutex, - completion: Condvar, - anchor: u64, - origin: Instant, - observed: AtomicU64, - expires_at_ms: u64, +pub(super) struct FitLifetime { + pub(super) cancel: Arc, + pub(super) publication: Mutex<()>, + pub(super) completed: Mutex, + pub(super) completion: Condvar, + pub(super) anchor: u64, + pub(super) origin: Instant, + pub(super) observed: AtomicU64, + pub(super) expires_at_ms: u64, #[cfg(test)] - hook: Option>, + pub(super) hook: Option>, + #[cfg(test)] + pub(super) io_hook: OptionResult<(),String>+Send+Sync>>, } impl FitLifetime { - fn now(&self) -> Result { + pub(super) fn io(&self,stage:&'static str)->Result<(),String>{ + #[cfg(test)] if let Some(hook)=&self.io_hook{return hook(stage);} + #[cfg(not(test))] let _=stage; + Ok(()) + } + pub(super) fn now(&self) -> Result { let elapsed: u64 = self .origin .elapsed() @@ -68,7 +75,7 @@ impl FitLifetime { .ok_or("temporal_fit_clock_overflow")?, )) } - fn allowed(&self) -> Result { + pub(super) fn allowed(&self) -> Result { if self.cancel.load(Ordering::Acquire) { return Err("temporal_fit_cancelled".into()); } @@ -78,13 +85,13 @@ impl FitLifetime { } Ok(now) } - fn cancel(&self) { + pub(super) fn cancel(&self) { self.cancel.store(true, Ordering::Release); } - fn done(&self) -> bool { + pub(super) fn done(&self) -> bool { *self.completed.lock().unwrap_or_else(|e| e.into_inner()) } - fn drain(&self) { + pub(super) fn drain(&self) { self.cancel(); let mut complete = self.completed.lock().unwrap_or_else(|e| e.into_inner()); while !*complete { @@ -94,7 +101,7 @@ impl FitLifetime { .unwrap_or_else(|e| e.into_inner()); } } - fn at(&self, stage: &'static str) { + pub(super) fn at(&self, stage: &'static str) { #[cfg(test)] if let Some(hook) = &self.hook { hook(stage); @@ -109,6 +116,9 @@ pub(super) struct TemporalFitJob { lifetime: Arc, } impl TemporalFitJob { + pub(super) fn uncancelled_completed(&self)->bool{self.done()&&!self.lifetime.cancel.load(Ordering::Acquire)} + pub(super) fn attempt_matches(&self,attempt:u64,cancel:&Arc)->bool{self.identity.attempt==attempt&&self.uses_cancel(cancel)&&!self.lifetime.cancel.load(Ordering::Acquire)} + pub(super) fn uses_cancel(&self,cancel:&Arc)->bool{Arc::ptr_eq(cancel,&self.lifetime.cancel)} pub(super) fn observe_now(&self, now: u64) { self.lifetime.observed.fetch_max(now, Ordering::AcqRel); } @@ -118,9 +128,7 @@ impl TemporalFitJob { pub(super) fn drain(&self) { self.lifetime.drain(); } - fn done(&self) -> bool { - self.lifetime.done() - } + pub(super) fn done(&self) -> bool { self.lifetime.done() } } /// Created before scheduling. Dropping an unstarted/discarded closure also /// releases waiters. Worker outcome/raw ownership is destroyed before this. @@ -173,9 +181,8 @@ impl Drop for FitCompletion { } impl RuntimeTemporalCapture { pub(super) fn cancel_fit(&self) { - if let Some(job) = &self.fit_job { - job.cancel(); - } + if let Some(job) = &self.fit_job { job.cancel(); } + if let Some(job) = &self.evidence_job { job.cancel(); } } pub(super) fn take_current_accepted_fit(&mut self,epoch:&str,generation:u64,now:u64)->Result{ if self.owner_generation!=generation||self.failure.is_some(){return Err("temporal_accepted_fit_owner_changed".into());} @@ -288,6 +295,8 @@ impl RoomLearningRuntime { observed: AtomicU64::new(now), expires_at_ms: binding.expires_at_ms(), #[cfg(test)] + io_hook:None, + #[cfg(test)] hook: self .temporal_fit_hook .lock() diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_held_out_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_held_out_tests.rs index 56fc6dcac5..8a63cc9804 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_held_out_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_held_out_tests.rs @@ -79,7 +79,7 @@ fn configured(root: &Path) -> (Arc, u64) { let now = fixture::settle_async_boundary(&runtime, fixture::START); (runtime, now) } -fn packet(source: u8, sequence: u32, at: u64, label: RoomLabel) -> CanonicalPacket { +pub(super) fn packet(source: u8, sequence: u32, at: u64, label: RoomLabel) -> CanonicalPacket { let gain = match label { RoomLabel::Empty => 3, RoomLabel::AdjacentOnly => 4, @@ -179,7 +179,7 @@ fn record_train_uncached(root: &Path) -> (Arc, u64) { (runtime, now) } -fn copy_private(from: &Path, to: &Path) { +pub(super) fn copy_private(from: &Path, to: &Path) { use std::os::unix::fs::PermissionsExt; assert!(!std::fs::symlink_metadata(from) .unwrap() @@ -215,7 +215,7 @@ fn record_train(root: &Path) -> (Arc, u64) { assert_eq!(fixture::snapshot(&rt, now)["stage"], "complete"); (rt, now) } -fn assert_original_lifetime(value: &serde_json::Value, original: &serde_json::Value) { +pub(super) fn assert_original_lifetime(value: &serde_json::Value, original: &serde_json::Value) { for field in [ "schema", "id", @@ -264,14 +264,14 @@ async fn prepare_frozen(rt: &Arc, at: u64) -> u64 { } } -fn phase_target(rt: &RoomLearningRuntime) -> Option<(u64, u64)> { +pub(super) fn phase_target(rt: &RoomLearningRuntime) -> Option<(u64, u64)> { let state = rt.inner.lock().unwrap(); let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { panic!() }; c.core.as_ref().unwrap().c4b_test_target() } -fn receiver_floor(rt: &RoomLearningRuntime) -> u32 { +pub(super) fn receiver_floor(rt: &RoomLearningRuntime) -> u32 { let state = rt.inner.lock().unwrap(); let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { panic!() @@ -286,7 +286,7 @@ fn receiver_floor(rt: &RoomLearningRuntime) -> u32 { .max() .unwrap() } -fn prefix_value(rt: &RoomLearningRuntime, now: u64) -> serde_json::Value { +pub(super) fn prefix_value(rt: &RoomLearningRuntime, now: u64) -> serde_json::Value { let loaded = rt .temporal_store() .unwrap() @@ -295,7 +295,7 @@ fn prefix_value(rt: &RoomLearningRuntime, now: u64) -> serde_json::Value { .unwrap(); serde_json::from_slice(loaded.prefix.unwrap().bytes()).unwrap() } -fn same_train_and_frozen(before: &serde_json::Value, after: &serde_json::Value) { +pub(super) fn same_train_and_frozen(before: &serde_json::Value, after: &serde_json::Value) { for field in ["rows", "evidence", "identities"] { assert_eq!( &before["payload"]["accepted"][field].as_array().unwrap()[..64], diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public.rs new file mode 100644 index 0000000000..36fcc8c14a --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public.rs @@ -0,0 +1,893 @@ +//! Versioned owner workflow over the existing tagged Runtime. No imported +//! JSON report/proof/Complete Boolean creates producer authority. +use super::evidence_control::EvidencePurpose; +use super::*; +use crate::room_learner::temporal_wire::*; +fn schema(actual: &str, expected: &str) -> Result<(), String> { + if actual != expected { + return Err("learner_unsupported_schema".into()); + } + Ok(()) +} +fn sha(s: &str) -> bool { + s.len() == 64 + && s.bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} +impl RoomLearningRuntime { + pub fn temporal_configure_v2( + &self, + expected: &MutationGuard, + context: CaptureContext, + run: String, + recovery_consent: bool, + now: u64, + ) -> Result<(), String> { + if !recovery_consent { + return Err("recovery_consent_required".into()); + } + if context.sources.len() != 3 { + return Err("learner_temporal_three_sources_required".into()); + } + let store = self + .recovery_store + .clone() + .filter(|s| s.available()) + .ok_or("recovery_unavailable")?; + if store.inspect()?.is_some() { + return Err("learner_recovery_pending".into()); + } + self.configure_temporal_capture_with_custody_mode( + expected, + context, + run, + now, + true, + move |s, r, f, n, _, w| { + TemporalTrainCapture::prepare_full_run_async(s, r, f, n, store, w) + }, + ) + } + pub fn temporal_begin_phase_v2( + &self, + request: TemporalPhaseStartRequestV2, + now: u64, + ) -> Result<(), String> { + schema(&request.schema, PHASE_START_REQUEST_SCHEMA)?; + if !request.conditions_confirmed { + return Err("learner_conditions_confirmation_required".into()); + } + self.start_temporal_phase_checked(&request.expected, now, |c| { + let core = c + .core + .as_ref() + .ok_or("temporal_initial_barrier_unconfirmed")?; + if !core.public_phase_matches( + &request.capture_generation_id, + &request.schedule_sha256, + request.episode_index as usize, + ) { + return Err("learner_phase_identity_changed".into()); + } + if core.stage() == TemporalCaptureStage::Frozen && c.training_proof.is_none() { + return Err("learner_training_proof_pending".into()); + } + Ok(()) + }) + } + pub fn temporal_finish_phase_v2( + &self, + request: TemporalPhaseFinishRequestV2, + now: u64, + ) -> Result<(), String> { + schema(&request.schema, PHASE_FINISH_REQUEST_SCHEMA)?; + if !request.conditions_confirmed { + return Err("learner_conditions_confirmation_required".into()); + } + self.confirm_temporal_phase_checked( + &request.expected, + &request.confirmation_challenge, + now, + |c| { + if c.core.as_ref().is_none_or(|core| { + !core.public_phase_matches( + &request.capture_generation_id, + &request.schedule_sha256, + request.episode_index as usize, + ) + }) { + return Err("learner_phase_identity_changed".into()); + } + Ok(()) + }, + ) + } + fn public_generation_matches(&self, id: &str) -> Result<(), String> { + if !sha(id) { + return Err("learner_capture_identity_invalid".into()); + } + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if !matches!(state.capture.as_ref(),Some(CaptureWorkflow::Temporal(c))if c.core.as_ref().is_some_and(|core|core.is_full_run()&&core.public_generation_id()==id)) + { + return Err("learner_capture_identity_changed".into()); + } + Ok(()) + } + pub fn temporal_pause_phase_v2( + &self, + request: TemporalCurrentActionRequestV2, + now: u64, + ) -> Result<(), String> { + schema(&request.schema, PHASE_CANCEL_REQUEST_SCHEMA)?; + self.public_generation_matches(&request.capture_generation_id)?; + self.pause_temporal_capture(&request.expected, now) + } + pub fn temporal_cancel_operation_v2( + &self, + request: TemporalCurrentActionRequestV2, + _now: u64, + ) -> Result<(), String> { + schema(&request.schema, TRAIN_CANCEL_REQUEST_SCHEMA)?; + let (fit, evidence) = { + let mut state = self.command_state(&request.expected)?; + let phase = state.job.phase; + let cancel = state.job.cancel.clone(); + let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_mut() else { + return Err(PRIVATE_KIND.into()); + }; + if c.core + .as_ref() + .is_none_or(|core| core.public_generation_id() != request.capture_generation_id) + { + return Err("learner_capture_identity_changed".into()); + } + if phase == JobPhase::Ready { + return Ok(()); + } + c.public_train_requested = false; + let fit = c + .fit_job + .as_ref() + .filter(|j| j.uses_cancel(&cancel)) + .cloned(); + let evidence = c + .evidence_job + .as_ref() + .filter(|j| j.uses_cancel(&cancel)) + .cloned(); + if let Some(j) = &fit { + j.cancel(); + } + if let Some(j) = &evidence { + j.cancel(); + } + if busy(&state.job) { + state.job.phase = JobPhase::Cancelling; + } + (fit, evidence) + }; + if let Some(j) = fit { + j.drain(); + } + if let Some(j) = evidence { + j.drain(); + } + Ok(()) + } + pub fn temporal_train_v2( + self: &Arc, + request: TemporalTrainRequestV2, + now: u64, + ) -> Result { + schema(&request.schema, TRAIN_REQUEST_SCHEMA)?; + self.public_generation_matches(&request.capture_generation_id)?; + let retry = { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + return Err(PRIVATE_KIND.into()); + }; + let core = c + .core + .as_ref() + .ok_or("temporal_initial_barrier_unconfirmed")?; + if core.public_candidate().is_some() { + if core.stage() == TemporalCaptureStage::FullComplete { + 4 + } else { + 3 + } + } else if c + .fit_job + .as_ref() + .is_some_and(|job| job.uncancelled_completed()) + && c.diagnostic_fit.is_some() + { + if core.stage() == TemporalCaptureStage::Blocked { + 2 + } else if core + .public_capture(&self.runtime_epoch, c.lead_in_until, now) + .durability_pending + { + 5 + } else { + 1 + } + } else { + 0 + } + }; + if retry >= 3 && retry <= 4 { + let attempt = self.start_temporal_evidence( + &request.expected, + now, + if retry == 4 { + EvidencePurpose::RestoreComplete + } else { + EvidencePurpose::TrainProof + }, + )?; + return Ok(attempt.to_string()); + } + if retry != 0 { + if retry == 2 { + self.repair_temporal_capture(&request.expected, now)?; + } else if retry == 5 { + self.tick_temporal_capture(&request.expected, now)?; + } else { + self.prepare_frozen_transition(&request.expected, now)?; + } + { + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_mut() { + if c.failure.is_none() + && c.fit_job + .as_ref() + .is_some_and(|j| j.uncancelled_completed()) + && c.core.as_ref().is_some_and(|core| { + core.public_generation_id() == request.capture_generation_id + }) + { + c.public_train_requested = true; + } + } + } + self.spawn_temporal_training_progress(request.capture_generation_id); + return Ok(request + .expected + .workflow_revision + .saturating_add(1) + .to_string()); + } + let attempt = self.start_temporal_fit(&request.expected, now)?; + { + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + let cancel = state.job.cancel.clone(); + if let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_mut() { + if c.core.as_ref().is_some_and(|core| { + core.public_generation_id() == request.capture_generation_id + }) && c + .fit_job + .as_ref() + .is_some_and(|j| j.attempt_matches(attempt, &cancel)) + { + c.public_train_requested = true; + } + } + } + self.spawn_temporal_training_progress(request.capture_generation_id); + Ok(attempt.to_string()) + } + fn spawn_temporal_training_progress(self: &Arc, id: String) { + let weak = Arc::downgrade(self); + tokio::spawn(async move { + loop { + let Some(runtime) = weak.upgrade() else { + return; + }; + let now = runtime.receiver_now_ns(); + let action = { + let state = match runtime.inner.lock() { + Ok(s) => s, + Err(_) => return, + }; + let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_ref() else { + return; + }; + if !c.public_train_requested + || c.failure.is_some() + || c.core + .as_ref() + .is_none_or(|core| core.public_generation_id() != id) + || matches!(state.job.phase, JobPhase::Failed | JobPhase::Cancelled) + { + return; + } + if c.training_proof.is_some() { + return; + } + if c.core + .as_ref() + .unwrap() + .public_capture(&runtime.runtime_epoch, c.lead_in_until, now) + .durability_pending + { + 0 + } else { + match c.core.as_ref().unwrap().stage() { + TemporalCaptureStage::Complete + if state.job.phase == JobPhase::Ready + && c.fit_job.as_ref().is_some_and(|j| j.done()) => + { + 1 + } + TemporalCaptureStage::FullComplete if !busy(&state.job) => 3, + TemporalCaptureStage::Frozen + | TemporalCaptureStage::Paused + | TemporalCaptureStage::Ready + if !busy(&state.job) + && c.core.as_ref().unwrap().public_candidate().is_some() => + { + 2 + } + _ => 0, + } + } + }; + let result = (|| -> Result<(), String> { + if action == 1 { + runtime.prepare_frozen_transition(&runtime.mutation_guard()?, now)?; + } else if action == 2 { + runtime.start_temporal_evidence( + &runtime.mutation_guard()?, + now, + EvidencePurpose::TrainProof, + )?; + } else if action == 3 { + runtime.start_temporal_evidence( + &runtime.mutation_guard()?, + now, + EvidencePurpose::RestoreComplete, + )?; + } + // Only already-authorized post-Train ACK progress; no + // capture/configuration/ground-truth is started by GET. + let _ = runtime.tick_temporal_capture(&runtime.mutation_guard()?, now); + Ok(()) + })(); + if let Err(error) = result { + if error != "learner_state_changed" && error != "learner_job_busy" { + if let Ok(mut state) = runtime.inner.lock() { + state.input_error = Some(bounded(error)); + } + return; + } + } + drop(runtime); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }); + } + pub fn temporal_evaluate_v2( + self: &Arc, + request: TemporalEvaluateRequestV2, + now: u64, + ) -> Result { + schema(&request.schema, EVALUATE_REQUEST_SCHEMA)?; + self.public_generation_matches(&request.capture_generation_id)?; + { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + return Err(PRIVATE_KIND.into()); + }; + let proof = c + .training_proof + .as_ref() + .ok_or("learner_training_proof_pending")?; + let refs = c + .core + .as_ref() + .and_then(|core| core.public_complete_refs(&proof.sha256)) + .ok_or("temporal_actual_complete_required")?; + if refs != request.refs { + return Err("learner_evidence_identity_changed".into()); + } + } + self.start_temporal_evidence(&request.expected, now, EvidencePurpose::HeldOutEvaluation) + .map(|attempt| attempt.to_string()) + } + pub fn temporal_status_v2(self: &Arc, now: u64) -> Result { + // ACKs may already be ready. This tick does not issue new consent, + // source samples, fit/proof/evaluation jobs or immutable metadata writes. + let temporal = { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + matches!(state.capture, Some(CaptureWorkflow::Temporal(_))) + }; + if temporal { + let _ = self.tick_temporal_capture(&self.mutation_guard()?, now); + } + let generation = { + self.inner + .lock() + .map_err(|_| "learner state unavailable")? + .generation + }; + let recovery = self + .recovery_store + .as_ref() + .and_then(|store| TemporalTrainCapture::recovery_discovery(store, now)); + let mut response = { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if state.generation != generation { + return Err("learner_state_changed".into()); + } + let capture = match state.capture.as_ref() { + Some(CaptureWorkflow::Temporal(c)) => Some(c), + _ => None, + }; + let context = state.capture_context().cloned(); + let mut profile = TemporalProfileWire::engineering(); + let mut wire_capture = None; + let mut candidate = None; + let mut train_proof = None; + let mut complete = None; + let mut evaluation = None; + let mut operation = None; + if let Some(c) = capture { + if let Some(core) = &c.core { + profile = core.public_profile(); + wire_capture = + Some(core.public_capture(&self.runtime_epoch, c.lead_in_until, now)); + candidate = core.public_candidate(); + train_proof = c.training_proof.as_ref().map(|p| p.sha256.clone()); + complete = train_proof + .as_ref() + .and_then(|sha| core.public_complete_refs(sha)); + } + if let Some(saved) = &c.evaluation { + evaluation = Some(evaluation_wire(saved)); + } + if let Some(job) = &c.evidence_job { + operation = Some(job.operation_name().into()); + } else if c.fit_job.is_some() { + operation = Some("train".into()); + } + } + TemporalLearnerStatusWire { + schema: STATUS_SCHEMA.into(), + enabled: true, + available_model_kinds: if self + .recovery_store + .as_ref() + .is_some_and(|s| s.available()) + { + vec![WholeTemporalKind::ManagedTemporalV3] + } else { + vec![] + }, + temporal_profile: profile, + workflow_kind: if capture.is_some() { + WorkflowKindWire::ManagedTemporalV3 + } else if state.capture.is_some() { + WorkflowKindWire::Legacy + } else { + WorkflowKindWire::Idle + }, + mutation_guard: self.guard_for(&state), + context_digest: context.as_ref().map(CaptureContext::digest), + room_digest: context.as_ref().map(|c| c.room_digest.clone()), + source_node_ids: context + .as_ref() + .map(|c| c.sources.iter().map(|s| s.source_id).collect()) + .unwrap_or_default(), + recovery, + capture: wire_capture, + candidate, + training_proof_sha256: train_proof, + complete, + evaluation, + assessment: capture.and_then(|c| { + c.assessment + .as_ref() + .map(|a| a.assessment.receipt.summary.clone()) + }), + model_selection: None, + job_state: if capture.is_some_and(|c| { + c.public_train_requested + && c.training_proof.is_none() + && c.failure.is_none() + && !matches!( + state.job.phase, + JobPhase::Failed | JobPhase::Cancelled | JobPhase::Cancelling + ) + }) { + "running".into() + } else { + serde_json::to_value(state.job.phase) + .unwrap() + .as_str() + .unwrap() + .into() + }, + job_operation: operation, + error_code: state + .job + .error + .as_ref() + .map(|_| "learner_operation_failed".into()), + error: state + .job + .error + .clone() + .or_else(|| state.input_error.clone()), + legacy_status: None, + } + }; + if response.workflow_kind != WorkflowKindWire::ManagedTemporalV3 { + response.legacy_status = Some(self.status_without_checkpoint(now)); + } + let selection = self.temporal_selection_v2(now, None, None)?; + response.model_selection = Some( + serde_json::from_value( + selection + .get("model_selection") + .cloned() + .ok_or("learner_selection_projection")?, + ) + .map_err(|_| "learner_selection_projection")?, + ); + serde_json::to_value(response).map_err(|_| "learner_status_encoding".into()) + } +} + +impl RoomLearningRuntime { + pub fn temporal_qualify_v2( + self: &Arc, + request: TemporalQualifyRequestV2, + now: u64, + ) -> Result { + schema(&request.schema, QUALIFY_REQUEST_SCHEMA)?; + self.public_generation_matches(&request.capture_generation_id)?; + { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + return Err(PRIVATE_KIND.into()); + }; + let saved = c + .evaluation + .as_ref() + .ok_or("temporal_evaluation_required")?; + if request.refs.evaluation_receipt_sha256 != saved.receipt().sha256 + || request.refs.held_out_evidence_sha256 != saved.held_out().sha256 + || request.refs.complete.training_proof_sha256 != saved.train().sha256 + || c.core + .as_ref() + .and_then(|core| core.public_complete_refs(&saved.train().sha256)) + .as_ref() + != Some(&request.refs.complete) + { + return Err("learner_evidence_identity_changed".into()); + } + } + self.start_temporal_evidence(&request.expected, now, EvidencePurpose::Qualification) + .map(|a| a.to_string()) + } + pub fn temporal_recovery_context_v2( + &self, + binding: &str, + revision: u64, + sources: &[u8], + now: u64, + ) -> Result { + let store = self.temporal_store()?; + let (context, _) = TemporalTrainCapture::saved_full_context(store, now)?; + let mut ids: Vec<_> = context.sources.iter().map(|s| s.source_id).collect(); + ids.sort(); + let mut supplied = sources.to_vec(); + supplied.sort(); + if context.owner_scope != self.owner + || context.installation_id != self.installation + || context.room_digest != binding + || context.revision != revision + || ids != supplied + { + return Err("recovery_context_mismatch".into()); + } + Ok(context) + } + pub fn temporal_restore_v2( + self: &Arc, + expected: &MutationGuard, + context: &CaptureContext, + capture_generation: &str, + now: u64, + ) -> Result<(), String> { + let (_, id) = TemporalTrainCapture::saved_full_context(self.temporal_store()?, now)?; + if id != capture_generation { + return Err("learner_capture_identity_changed".into()); + } + self.restore_temporal_full_run(expected, context, now)?; + let reconcile = { + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_mut() { + let needed = c + .core + .as_ref() + .is_some_and(|core| core.public_candidate().is_some()); + c.public_train_requested = needed; + needed + } else { + false + } + }; + if reconcile { + self.spawn_temporal_training_progress(capture_generation.into()); + } + Ok(()) + } + pub fn temporal_discard_v2( + &self, + expected: &MutationGuard, + capture_generation: &str, + now: u64, + ) -> Result<(), String> { + let store = self.temporal_store()?; + if TemporalTrainCapture::retirement_generation(store, now)? != capture_generation { + return Err("learner_capture_identity_changed".into()); + } + // Stop/drain actual accepted work before the retirement commit. No + // late accepted write may recreate the retired recovery head. + let (writer, fit, evidence, generation) = { + let mut state = self.command_state(expected)?; + if let Some(CaptureWorkflow::Temporal(c)) = state.capture.as_mut() { + c.public_train_requested = false; + c.invalidate_owner(); + let writer = c.core.as_ref().and_then(|c| c.writer()); + if let Some(w) = &writer { + w.cancel(); + } + ( + writer, + c.fit_job.clone(), + c.evidence_job.clone(), + state.generation, + ) + } else { + (None, None, None, state.generation) + } + }; + if let Some(w) = writer { + w.drain()?; + } + if let Some(j) = fit { + j.drain(); + } + if let Some(j) = evidence { + j.drain(); + } + let inspection = store + .inspect_temporal_retirement(now)? + .ok_or("recovery_missing")?; + let intent = store.prepare_inspected_retirement(&inspection, now, false)?; + store.publish_temporal(&intent, None, now)?; + let retired = { + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if state.generation != generation { + return Err("learner_state_changed".into()); + } + state.generation = state + .generation + .checked_add(1) + .ok_or("learner generation exhausted")?; + state.workflow_revision = state.workflow_revision.saturating_add(1); + state.job.phase = JobPhase::Idle; + state.input_error = None; + state.capture.take() + }; + drop(retired); + Ok(()) + } +} + +impl RoomLearningRuntime { + pub fn paired_temporal_progress_snapshot( + &self, + installation_id: &str, + boot_epoch: &str, + room: Option, + ready_ids: &[u8], + ready_context: Option<&str>, + now: u64, + ) -> Result { + const MAX_SAFE: u64 = 9_007_199_254_740_991; + let observed = now / 1_000_000; + if installation_id.is_empty() + || installation_id.len() > 128 + || installation_id.chars().any(char::is_control) + || boot_epoch.is_empty() + || boot_epoch.len() > 128 + || boot_epoch.chars().any(char::is_control) + || observed == 0 + || observed > MAX_SAFE - 10_000 + || room.as_ref().is_some_and(|r| { + !sha(&r.binding_digest) || r.revision == 0 || r.revision > MAX_SAFE + }) + { + return Err("progress_identity_or_time_unavailable".into()); + } + let mut result = TemporalPairedProgressWire { + schema: PAIRED_PROGRESS_SCHEMA.into(), + success: true, + installation_id: installation_id.into(), + runtime_epoch: boot_epoch.into(), + observed_at_unix_ms: observed, + expires_at_unix_ms: observed + 10_000, + room, + context_match: false, + stage: "setup".into(), + state: "idle".into(), + model_kind: None, + profile: None, + capture: None, + candidate: None, + evaluation: None, + qualification: None, + selection: None, + sources: None, + save_status: "not_reported".into(), + blocker: Some("room_not_published".into()), + }; + { + let state = self.inner.lock().map_err(|_| "learner state unavailable")?; + let context = state.capture_context().or_else(|| { + state + .selection + .temporal + .as_ref() + .map(|s| &s.stored.envelope().receipt.selected.context) + }); + if let (Some(published), Some(context)) = (result.room.as_ref(), context) { + if installation_id != self.installation + || context.installation_id != installation_id + || context.room_digest != published.binding_digest + || context.revision != published.revision + || state.selection.temporal.as_ref().is_some_and(|selected| { + let c = &selected.stored.envelope().receipt.selected.context; + c.room_digest != published.binding_digest + || c.revision != published.revision + || c.digest() != context.digest() + }) + { + result.stage = "blocked".into(); + result.blocker = Some("context_mismatch".into()); + } else { + result.context_match = true; + result.model_kind = Some(WholeTemporalKind::ManagedTemporalV3); + result.state = serde_json::to_value(state.job.phase) + .unwrap() + .as_str() + .unwrap() + .into(); + if ready_context != Some(context.digest().as_str()) { + return Err("progress_context_changed".into()); + } + result.sources = Some(TemporalPairedSourcesWire { + required: context + .sources + .len() + .try_into() + .map_err(|_| "progress_source_count")?, + ready: context + .sources + .iter() + .filter(|s| ready_ids.contains(&s.source_id)) + .count() as u8, + }); + result.blocker = None; + if let Some(CaptureWorkflow::Temporal(c)) = &state.capture { + if let Some(core) = &c.core { + let capture = + core.public_capture(&self.runtime_epoch, c.lead_in_until, now); + result.stage = capture.stage.clone(); + result.profile = Some(core.public_profile()); + result.candidate = core.public_candidate(); + result.capture = Some(TemporalPairedCaptureProgressWire { + capture_generation_id: capture.capture_generation_id, + schedule_sha256: capture.schedule_sha256, + stage: capture.stage, + current_episode: capture.current_episode, + role: capture.role, + label: capture.label, + target_ms: capture.target_ms, + lead_in_remaining_ms: capture.lead_in_remaining_ms, + observed_ms: capture.observed_ms, + retained_windows: capture.retained_windows, + durable_windows: capture.durable_windows, + sealed_train_episodes: capture.sealed_train_episodes, + sealed_held_out_episodes: capture.sealed_held_out_episodes, + durability_pending: capture.durability_pending, + awaiting_ground_truth: capture.awaiting_ground_truth, + original_deadline: capture.original_deadline, + }); + } else { + result.stage = "preparing".into(); + } + if let Some(saved) = &c.evaluation { + result.evaluation = Some(evaluation_wire(saved)); + } + if let Some(a) = &c.assessment { + result.qualification = + a.qualification.as_ref().map(|q| q.receipt.summary.clone()); + } + } else if state.capture.is_some() { + result.stage = "unsupported_mixed_capture".into(); + result.blocker = Some("legacy_capture_with_temporal_selection".into()); + } else { + result.stage = "selected".into(); + } + } + } + } + if !result.context_match { + result.room = None; + return Ok(result); + } + let selection = self.temporal_selection_v2( + now, + result.room.as_ref().map(|r| r.binding_digest.as_str()), + Some(boot_epoch), + )?; + let summary: TemporalSelectionStatusWire = serde_json::from_value( + selection + .get("model_selection") + .cloned() + .ok_or("progress_selection_projection")?, + ) + .map_err(|_| "progress_selection_projection")?; + if summary.selected.is_some() { + result.model_kind = Some(WholeTemporalKind::ManagedTemporalV3); + if result.qualification.is_none() { + result.qualification = summary.qualification.clone(); + } + } + result.selection = Some(summary); + Ok(result) + } +} + +fn evaluation_wire( + saved: &crate::room_learner::artifact::StoredTemporalEvaluation, +) -> TemporalEvaluationWire { + let identity = &saved.receipt().body.identity; + let refs = TemporalEvaluationRefsWire { + complete: TemporalCompleteRefsWire { + model: TemporalModelIdentityWire { + model_kind: WholeTemporalKind::ManagedTemporalV3, + model_sha256: identity.artifact_sha256.clone(), + verified_fit_record_sha256: identity.verified_fit_record_sha256.clone(), + }, + training_proof_sha256: saved.train().sha256.clone(), + complete_record_sha256: saved.held_out().body.complete_record_sha256.clone(), + complete_commitment_sha256: saved + .held_out() + .body + .complete_commitment + .commitment_sha256 + .clone(), + }, + held_out_evidence_sha256: saved.held_out().sha256.clone(), + evaluation_receipt_sha256: saved.receipt().sha256.clone(), + }; + TemporalEvaluationWire { + refs, + window_unit: "native_receiver_2s_window".into(), + evaluated_at_unix_ms: saved.receipt().body.evaluated_at_unix_ms, + report: saved.receipt().body.report.clone(), + durable: true, + independence_verified: false, + physically_validated: false, + } +} + +#[cfg(all(test, unix))] +#[path = "lifecycle_temporal_public_acceptance_tests.rs"] +mod acceptance; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public_acceptance_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public_acceptance_tests.rs new file mode 100644 index 0000000000..39cada3577 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public_acceptance_tests.rs @@ -0,0 +1,1188 @@ +//! SYNTHETIC production-default-profile software acceptance. No physical RF +//! or clock precision/generalization/cadence acceptance follows from this case. +use super::super::{held_out_tests as established, tests as fixture}; +use super::*; +use established::{ + assert_original_lifetime, copy_private, packet, phase_target, prefix_value, receiver_floor, + same_train_and_frozen, +}; +use std::{ + path::Path, + time::{Duration, Instant}, +}; +fn capture(rt: &RoomLearningRuntime, now: u64) -> TemporalCaptureWire { + let state = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!("actualTemporalcapturemissing") + }; + c.core + .as_ref() + .unwrap() + .public_capture(&rt.runtime_epoch, c.lead_in_until, now) +} +fn public_lead_end(rt: &Arc, at: u64) -> u64 { + let c = capture(rt, at); + rt.temporal_begin_phase_v2( + TemporalPhaseStartRequestV2 { + schema: PHASE_START_REQUEST_SCHEMA.into(), + expected: rt.mutation_guard().unwrap(), + capture_generation_id: c.capture_generation_id, + schedule_sha256: c.schedule_sha256, + episode_index: c.current_episode.unwrap(), + conditions_confirmed: true, + }, + at, + ) + .unwrap(); + let start = Instant::now(); + loop { + let now = at + start.elapsed().as_nanos() as u64; + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(), now) + .unwrap(); + let state = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!() + }; + if let Some(end) = c.lead_in_until { + return end; + } + drop(state); + assert!(start.elapsed() < Duration::from_secs(30)); + std::thread::sleep(Duration::from_millis(1)); + } +} +fn public_finish(rt: &Arc, guard: MutationGuard, challenge: String, now: u64) { + let c = capture(rt, now); + rt.temporal_finish_phase_v2( + TemporalPhaseFinishRequestV2 { + schema: PHASE_FINISH_REQUEST_SCHEMA.into(), + expected: guard, + capture_generation_id: c.capture_generation_id, + schedule_sha256: c.schedule_sha256, + episode_index: c.current_episode.unwrap(), + confirmation_challenge: challenge, + conditions_confirmed: true, + }, + now, + ) + .unwrap(); +} +async fn metadata_ready(rt: &RoomLearningRuntime) { + let start = Instant::now(); + loop { + let state = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!() + }; + if state.job.phase == JobPhase::Ready + && c.training_proof.is_some() + && c.evidence_job.as_ref().is_some_and(|j| j.done()) + { + return; + } + assert!( + !matches!(state.job.phase, JobPhase::Failed | JobPhase::Cancelled), + "actualmetadatajobfailed {:?}", + state.job.error + ); + drop(state); + assert!( + start.elapsed() < Duration::from_secs(180), + "actualmetadatajobwatchdog" + ); + tokio::time::sleep(Duration::from_millis(5)).await; + } +} +async fn restore_public(rt: &Arc, now: u64) { + let status = rt.temporal_status_v2(now).unwrap(); + let id = status["recovery"]["capture_generation_id"] + .as_str() + .unwrap(); + rt.temporal_restore_v2(&rt.mutation_guard().unwrap(), &fixture::context(), id, now) + .unwrap(); + if rt.current_verified_frozen_model().unwrap().is_some() { + metadata_ready(rt).await; + } +} +fn record_train_default48(root: &Path) -> (Arc, u64) { + let runtime = Arc::new(fixture::runtime(root, fixture::START)); + runtime + .temporal_configure_v2( + &runtime.mutation_guard().unwrap(), + fixture::context(), + "synthetic-c4c-production-default48".into(), + true, + fixture::START, + ) + .unwrap(); + let mut now = fixture::settle_async_boundary(&runtime, fixture::START); + let schedule = ManagedScheduleContract::build( + fixture::context(), + LeadInPolicy::Train5HeldOut15EngineeringV1, + ) + .unwrap(); + let mut sequence = 0; + let mut starts = 0; + for episode in schedule + .episodes() + .iter() + .filter(|e| e.role == DataRole::Train) + { + loop { + // A durable terminal pause can be confirmable without resuming. + if runtime + .temporal_confirmation_challenge(&runtime.mutation_guard().unwrap()) + .is_ok() + { + break; + } + let progress = fixture::snapshot(&runtime, now); + match progress["stage"].as_str().unwrap() { + "ready" | "paused" => { + now = public_lead_end(&runtime, now); + starts += 1; + runtime + .tick_temporal_capture(&runtime.mutation_guard().unwrap(), now) + .unwrap(); + } + "needs_confirmation" => break, + "failed" => panic!("synthetic C4B phase{} failed: {progress}", episode.index), + _ => {} + } + let pending = fixture::snapshot(&runtime, now)["durability_pending"] == true; + sequence += 1; + for source in 1..=3 { + match runtime.ingest_native(&packet(source, sequence, now, episode.condition), now) + { + Ok(()) => {} + Err(error) if error == "temporal_storage_busy_pause" => break, + Err(error) => panic!("synthetic C4B ingest: {error}"), + } + } + now += fixture::PERIOD; + if pending { + std::thread::sleep(Duration::from_nanos(fixture::PERIOD)); + } + assert!(starts < 120, "synthetic C4B interruption bound exhausted"); + } + let guard = runtime.mutation_guard().unwrap(); + let challenge = runtime.temporal_confirmation_challenge(&guard).unwrap(); + public_finish(&runtime, guard, challenge, now); + assert_eq!(fixture::snapshot(&runtime, now)["durability_pending"], true); + now = fixture::settle_async_boundary(&runtime, now); + } + let complete = fixture::snapshot(&runtime, now); + assert_eq!(complete["stage"], "complete"); + assert_eq!( + complete["sealed_train_episodes"], + schedule.counts().train_episodes + ); + assert_eq!(complete["durable_windows"], schedule.counts().train_windows); + println!( + "SYNTHETIC_C4B_REAL_TRAIN_CAPTURE: rows={} seals={} starts={starts}", + schedule.counts().train_windows, + schedule.counts().train_episodes + ); + (runtime, now) +} + +async fn produce_default48_complete(directory: &Path) -> u64 { + let (rt, mut now) = record_train_default48(directory); + // Fresh actual cold Runtime uses its authenticated producer floor as this + // explicitly synthetic clock-domain anchor; no live clock clamp/rescue. + drop(rt); + now += 1_000_000_000; + let mut rt = Arc::new(fixture::runtime(directory, now)); + restore_public(&rt, now).await; + let generation = capture(&rt, rt.receiver_now_ns()).capture_generation_id; + rt.temporal_train_v2( + TemporalTrainRequestV2 { + schema: TRAIN_REQUEST_SCHEMA.into(), + expected: rt.mutation_guard().unwrap(), + capture_generation_id: generation, + }, + rt.receiver_now_ns(), + ) + .unwrap(); + metadata_ready(&rt).await; + now = rt.receiver_now_ns(); + let fit = rt.temporal_diagnostic_fit().unwrap().unwrap().2; + assert_eq!( + serde_json::to_value(&fit.summary.as_ref().unwrap().metrics.configured).unwrap(), + serde_json::to_value(FitConfig::for_source_count(3)).unwrap() + ); + assert_eq!( + fit.summary.as_ref().unwrap().metrics.completed_epochs, + 48, + "production default profile was not executed" + ); + println!( + "SYNTHETIC_C4C_ACTUAL_DEFAULT48_FIT={}", + serde_json::to_string(&fit.summary.as_ref().unwrap().metrics).unwrap() + ); + now = fixture::settle_async_boundary(&rt, now); + assert_eq!(fixture::snapshot(&rt, now)["stage"], "frozen"); + drive_remaining_held_out(directory, rt, now).await +} +async fn drive_remaining_held_out( + directory: &Path, + mut rt: Arc, + mut now: u64, +) -> u64 { + let model = rt.current_verified_frozen_model().unwrap().unwrap(); + let model_bytes = model.export().unwrap(); + let model_sha = { + use sha2::Digest; + format!("{:x}", sha2::Sha256::digest(&model_bytes)) + }; + let existing = rt.store.temporal_fit_diagnostics(&model); + assert_eq!( + existing.summary.as_ref().unwrap().metrics.completed_epochs, + 48 + ); + assert_eq!( + serde_json::to_value(&existing.summary.as_ref().unwrap().metrics.configured).unwrap(), + serde_json::to_value(FitConfig::for_source_count(3)).unwrap() + ); + let report_before = serde_json::to_value(rt.store.temporal_fit_diagnostics(&model)).unwrap(); + now = fixture::settle_async_boundary(&rt, now); + let initial = capture(&rt, now); + assert!(matches!(initial.stage.as_str(), "frozen" | "paused")); + assert!((64..128).contains(&initial.durable_windows)); + let frozen = prefix_value(&rt, now); + assert_eq!( + frozen["payload"]["stage"]["fit"]["body"]["configured_fit_sha256"], + crate::room_learner::qualification::hash(&FitConfig::for_source_count(3)).unwrap() + ); + let original = frozen["payload"]["common"]["descriptor"].clone(); + let schedule = ManagedScheduleContract::build( + fixture::context(), + LeadInPolicy::Train5HeldOut15EngineeringV1, + ) + .unwrap(); + let mut sequence = receiver_floor(&rt); + let mut partial_cold = initial.durable_windows > 64; + let mut terminal_cold = false; + let mut starts = 0; + let original_proof = { + let state = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!() + }; + c.training_proof.as_ref().unwrap().sha256.clone() + }; + let mut observing_refused = false; + let mut pending_refused = false; + let mut train_overlap_refused = false; + if initial.durable_windows > 64 { + let before_rows = initial.durable_windows; + let forged = rt.temporal_begin_phase_v2( + TemporalPhaseStartRequestV2 { + schema: PHASE_START_REQUEST_SCHEMA.into(), + expected: rt.mutation_guard().unwrap(), + capture_generation_id: initial.capture_generation_id.clone(), + schedule_sha256: initial.schedule_sha256.clone(), + episode_index: 0, + conditions_confirmed: true, + }, + now, + ); + assert_eq!(forged.unwrap_err(), "learner_phase_identity_changed"); + assert_eq!(capture(&rt, now).durable_windows, before_rows); + train_overlap_refused = true; + println!("SYNTHETIC_C4C_COLD_CONTINUATION_BASELINE rows={} episode={} train_seals=7 fit_epochs=48 same_model={} proof={}",initial.durable_windows,initial.current_episode.unwrap(),model_sha,original_proof); + } + + for episode in schedule.episodes().iter().filter(|e| { + e.role == DataRole::HeldOut && e.index >= initial.current_episode.unwrap_or(15) as usize + }) { + loop { + let challenge = rt.temporal_confirmation_challenge(&rt.mutation_guard().unwrap()); + if challenge.is_ok() { + if episode.index == 14 && !terminal_cold { + let before = prefix_value(&rt, now); + let rows = fixture::snapshot(&rt, now)["durable_windows"].clone(); + drop(rt); + now += 1_000_000_000; + rt = Arc::new(fixture::runtime(directory, now)); + restore_public(&rt, now).await; + now = rt.receiver_now_ns(); + assert_eq!(fixture::snapshot(&rt, now)["durable_windows"], rows); + same_accepted_prefix(&frozen, &prefix_value(&rt, now)); + same_train_and_frozen(&frozen, &prefix_value(&rt, now)); + assert_eq!( + before["payload"]["accepted"], + prefix_value(&rt, now)["payload"]["accepted"] + ); + terminal_cold = true; + continue; + } + break; + } + let progress = fixture::snapshot(&rt, now); + assert_eq!( + progress["label"], + serde_json::to_value(episode.condition).unwrap() + ); + assert_eq!(progress["role"], "HeldOut"); + assert_eq!(progress["target_ms"], 30_000); + match progress["stage"].as_str().unwrap() { + "frozen" | "ready" | "paused" => { + now = public_lead_end(&rt, now); + starts += 1; + if episode.index == 7 && starts == 1 { + now += 50_000_000; + } + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(), now) + .unwrap(); + } + "failed" => panic!("HeldOut failed: {progress}"), + _ => {} + } + let pending = fixture::snapshot(&rt, now)["durability_pending"] == true; + if !observing_refused + && !pending + && fixture::snapshot(&rt, now)["stage"] == "observing" + && phase_target(&rt).is_some() + { + let state = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!() + }; + assert!( + c.core + .as_ref() + .unwrap() + .prepare_evaluation_source(now, false) + .is_err(), + "observingcannotmintstableTrainprojection" + ); + observing_refused = true; + } + sequence += 1; + for source in 1..=3 { + match rt.ingest_native(&packet(source, sequence, now, episode.condition), now) { + Ok(()) => {} + Err(e) if e == "temporal_storage_busy_pause" => break, + Err(e) => panic!("HeldOut ingress: {e}"), + } + } + now += fixture::PERIOD; + if pending { + std::thread::sleep(Duration::from_nanos(fixture::PERIOD)); + } + let progress = fixture::snapshot(&rt, now); + if !partial_cold + && episode.index == 7 + && progress["durable_windows"].as_u64().unwrap() >= 67 + && progress["durability_pending"] == false + && phase_target(&rt) + .is_some_and(|(start, end)| now >= start + 3 * fixture::PERIOD && now < end) + { + let durable = progress["durable_windows"].clone(); + let before = prefix_value(&rt, now); + drop(rt); + now += 1_000_000_000; + rt = Arc::new(fixture::runtime(directory, now)); + restore_public(&rt, now).await; + now = rt.receiver_now_ns(); + { + let state = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!() + }; + assert_eq!( + c.training_proof.as_ref().unwrap().sha256, + original_proof, + "partialHeldOutRestoremustreconcileSAMETrainproof" + ); + } + let current = capture(&rt, now); + let before_rejected_train = fixture::snapshot(&rt, now)["durable_windows"].clone(); + let forged = rt.temporal_begin_phase_v2( + TemporalPhaseStartRequestV2 { + schema: PHASE_START_REQUEST_SCHEMA.into(), + expected: rt.mutation_guard().unwrap(), + capture_generation_id: current.capture_generation_id, + schedule_sha256: current.schedule_sha256, + episode_index: 0, + conditions_confirmed: true, + }, + now, + ); + assert_eq!(forged.unwrap_err(), "learner_phase_identity_changed"); + assert_eq!( + fixture::snapshot(&rt, now)["durable_windows"], + before_rejected_train + ); + train_overlap_refused = true; + assert_eq!(fixture::snapshot(&rt, now)["durable_windows"], durable); + assert!(phase_target(&rt).is_none()); + assert_eq!( + before["payload"]["accepted"], + prefix_value(&rt, now)["payload"]["accepted"] + ); + same_accepted_prefix(&frozen, &prefix_value(&rt, now)); + same_train_and_frozen(&frozen, &prefix_value(&rt, now)); + partial_cold = true; + } + assert!( + starts < 300, + "bounded synthetic HeldOut interruption budget exhausted" + ); + } + let guard = rt.mutation_guard().unwrap(); + let challenge = rt.temporal_confirmation_challenge(&guard).unwrap(); + public_finish(&rt, guard, challenge, now); + { + let state = rt.inner.lock().unwrap(); + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!() + }; + assert!( + c.core + .as_ref() + .unwrap() + .public_capture(&rt.runtime_epoch, c.lead_in_until, now) + .durability_pending + ); + assert!( + c.core + .as_ref() + .unwrap() + .prepare_evaluation_source(now, false) + .is_err(), + "pendingdurabilitycannotmintstableTrainprojection" + ); + pending_refused = true; + } + now = fixture::settle_async_boundary(&rt, now); + assert_eq!( + fixture::snapshot(&rt, now)["sealed_held_out_episodes"], + episode.index - 6 + ); + let sealed = prefix_value(&rt, now); + same_accepted_prefix(&frozen, &sealed); + same_train_and_frozen(&frozen, &sealed); + println!( + "SYNTHETIC_C4B_HELDOUT_SEAL episode={} durable={} sealed={} starts={starts}", + episode.index, + fixture::snapshot(&rt, now)["durable_windows"], + fixture::snapshot(&rt, now)["sealed_held_out_episodes"] + ); + } + assert!( + partial_cold && terminal_cold, + "two actual cold interruptions must execute" + ); + assert!( + observing_refused && pending_refused && train_overlap_refused, + "actualprotectedsource/phaseadmissioncontrolsnotexecuted" + ); + println!("SYNTHETIC_C4C_ACTUAL_SOURCE_REFUSALS observing=true pending=true forged_train_episode=true same_stable_train_proof=true"); + + let status = fixture::snapshot(&rt, now); + assert_eq!(status["stage"], "full_complete"); + assert_eq!(status["durable_windows"], 128); + assert_eq!(status["sealed_train_episodes"], 7); + assert_eq!(status["sealed_held_out_episodes"], 8); + let complete = prefix_value(&rt, now); + same_accepted_prefix(&frozen, &complete); + same_train_and_frozen(&frozen, &complete); + assert_eq!(complete["payload"]["stage"]["kind"], "complete"); + assert_eq!( + rt.current_verified_frozen_model() + .unwrap() + .unwrap() + .export() + .unwrap(), + model_bytes + ); + assert_eq!( + serde_json::to_value(rt.store.temporal_fit_diagnostics(&model)).unwrap(), + report_before + ); + assert!(rt.inner.lock().unwrap().model.is_none()); + assert!(rt.inner.lock().unwrap().qualification.is_none()); + assert!(!rt.inner.lock().unwrap().selection.has_history()); + let loaded = rt + .temporal_store() + .unwrap() + .load_temporal(now) + .unwrap() + .unwrap(); + let final_prefix = loaded.prefix.unwrap(); + let bytes = final_prefix.bytes().len(); + let raw_values = complete["payload"]["accepted"]["rows"] + .as_array() + .unwrap() + .iter() + .map(|row| { + row["views"] + .as_array() + .unwrap() + .iter() + .map(|view| { + view["samples"] + .as_array() + .unwrap() + .iter() + .map(|sample| sample.as_array().unwrap().len()) + .sum::() + }) + .sum::() + }) + .sum::(); + assert!(raw_values <= 2_000_000); + fn slot_sizes(root: &Path, out: &mut Vec) { + for entry in std::fs::read_dir(root).unwrap() { + let path = entry.unwrap().path(); + if path.is_dir() { + slot_sizes(&path, out) + } else if path + .file_name() + .unwrap() + .to_string_lossy() + .starts_with("slot-") + && path.extension().is_some_and(|e| e == "bin") + { + out.push(std::fs::metadata(path).unwrap().len()); + } + } + } + let mut outer_slot_sizes = vec![]; + slot_sizes(directory, &mut outer_slot_sizes); + outer_slot_sizes.sort_unstable(); + println!( + "SYNTHETIC_C4B_RAW_NUMERIC_VALUES={raw_values} AEAD_SLOT_FILE_BYTES={outer_slot_sizes:?}" + ); + assert!(bytes <= crate::room_learner::recovery_store::MAX_RECOVERY_BYTES); + println!( + "SYNTHETIC_C4B_FULL128_ACTUAL_JSON_BYTES={bytes} STARTS={starts} GLOBAL_GAPS={}", + complete["payload"]["common"]["ledger"]["gaps"] + .as_array() + .unwrap() + .len() + ); + drop(rt); + now += 1_000_000_000; + let cold = Arc::new(fixture::runtime(directory, now)); + restore_public(&cold, now).await; + now = cold.receiver_now_ns(); + assert_eq!(fixture::snapshot(&cold, now)["stage"], "full_complete"); + assert_eq!( + cold.current_verified_frozen_model() + .unwrap() + .unwrap() + .export() + .unwrap(), + model_bytes + ); + assert_original_lifetime(&fixture::snapshot(&cold, now)["descriptor"], &original); + assert!(cold + .start_temporal_phase(&cold.mutation_guard().unwrap(), now) + .is_err()); + assert_eq!( + cold.store + .load_exact_temporal_candidate(&fixture::context(), &model_sha) + .unwrap() + .export() + .unwrap(), + model_bytes + ); + cold.cancel_temporal_durability(&cold.mutation_guard().unwrap()) + .unwrap(); + now +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +#[cfg_attr( + debug_assertions, + ignore = "actual native default-profile workflow runs in mandatory Temporal Release batch" +)] +async fn c4c_default48_public_operator_workflow_native128_and_cold_selection() { + let directory = tempfile::Builder::new() + .prefix("SYNTHETIC-c4c-default48-") + .tempdir() + .unwrap() + .keep(); + std::fs::write(directory.join("SYNTHETIC-OWNED-DEFAULT48-RUN.json"),br#"{"real_user_data":false,"physical_accuracy_claim":false,"profile":"production-default48","retention":"owned_test_evidence"}"#).unwrap(); + let configured = FitConfig::for_source_count(3); + assert_eq!(configured.epochs, 48); + println!( + "SYNTHETIC_C4C_ACTUAL_CONFIG={}", + serde_json::to_string(&configured).unwrap() + ); + let now = produce_default48_complete(&directory).await + 1_000_000_000; + finish_public_checks(&directory, now).await; +} +async fn finish_public_checks(directory: &Path, now: u64) { + let rt = Arc::new(fixture::runtime(directory, now)); + restore_public(&rt, now).await; + let status = rt.temporal_status_v2(rt.receiver_now_ns()).unwrap(); + let c: TemporalCaptureWire = serde_json::from_value(status["capture"].clone()).unwrap(); + let complete: TemporalCompleteRefsWire = + serde_json::from_value(status["complete"].clone()).unwrap(); + assert!(status["evaluation"].is_null()); + rt.temporal_evaluate_v2( + TemporalEvaluateRequestV2 { + schema: EVALUATE_REQUEST_SCHEMA.into(), + expected: rt.mutation_guard().unwrap(), + capture_generation_id: c.capture_generation_id.clone(), + refs: complete, + }, + rt.receiver_now_ns(), + ) + .unwrap(); + metadata_ready(&rt).await; + let status = rt.temporal_status_v2(rt.receiver_now_ns()).unwrap(); + let evaluation: TemporalEvaluationWire = + serde_json::from_value(status["evaluation"].clone()).unwrap(); + assert_eq!(evaluation.report.samples, 64); + assert_eq!(evaluation.report.declared_trial_count, 8); + assert!(evaluation + .report + .per_condition + .iter() + .all(|c| c.samples == 16 && c.declared_trial_count == 2)); + rt.temporal_qualify_v2( + TemporalQualifyRequestV2 { + schema: QUALIFY_REQUEST_SCHEMA.into(), + expected: rt.mutation_guard().unwrap(), + capture_generation_id: c.capture_generation_id.clone(), + refs: evaluation.refs.clone(), + }, + rt.receiver_now_ns(), + ) + .unwrap(); + metadata_ready(&rt).await; + let status = rt.temporal_status_v2(rt.receiver_now_ns()).unwrap(); + println!( + "SYNTHETIC_C4C_DEFAULT48_ACTUAL_EVALUATION_AND_ASSESSMENT={}", + serde_json::to_string( + &json!({"evaluation":status["evaluation"],"assessment":status["assessment"]}) + ) + .unwrap() + ); + let q = status["assessment"]["qualification_sha256"].as_str(); + if q.is_none() { + assert_eq!(status["assessment"]["outcome"], "not_qualified"); + assert!(!rt.inner.lock().unwrap().selection.has_history()); + let before = rt.store.load_selection().unwrap(); + let denied = rt.temporal_select_v2( + TemporalSelectRequestV2 { + schema: SELECT_REQUEST_SCHEMA.into(), + expected: rt.mutation_guard().unwrap(), + capture_generation_id: c.capture_generation_id.clone(), + refs: TemporalQualificationRefsWire { + evaluation: evaluation.refs.clone(), + qualification_sha256: "0".repeat(64), + }, + }, + rt.receiver_now_ns(), + ); + assert_eq!(denied.unwrap_err(), "temporal_candidate_not_qualified"); + assert_eq!( + rt.store.load_selection().unwrap(), + before, + "refusedSelectmustnotchangeprotectedpointerorhistory" + ); + assert!(!rt.inner.lock().unwrap().selection.has_history()); + panic!("production-default48 actual fixture refused qualification; positiveSelect remainsOPEN; preservedprofile={}",directory.display()); + } + let safety_preselect = tempfile::Builder::new() + .prefix("SYNTHETIC-c4c-safety-preselect-") + .tempdir() + .unwrap() + .keep(); + copy_private(directory, &safety_preselect); + let selection_request = TemporalSelectRequestV2 { + schema: SELECT_REQUEST_SCHEMA.into(), + expected: rt.mutation_guard().unwrap(), + capture_generation_id: c.capture_generation_id, + refs: TemporalQualificationRefsWire { + evaluation: evaluation.refs, + qualification_sha256: q.unwrap().into(), + }, + }; + let selected = rt + .temporal_select_v2(selection_request.clone(), rt.receiver_now_ns()) + .unwrap(); + assert_eq!(selected.as_object().unwrap().len(), 3); + assert_eq!( + selected["model_selection"]["schema"], + SELECTION_STATUS_SCHEMA + ); + let identity: TemporalSelectedIdentityWire = + serde_json::from_value(selected["model_selection"]["selected"].clone()).unwrap(); + assert_eq!(identity.model_kind, WholeTemporalKind::ManagedTemporalV3); + assert_ne!( + selected["model_selection"]["current"]["status"], "ready", + "historicalHeldOutisnotliveinput" + ); + assert!( + rt.deselect_model(&rt.mutation_guard().unwrap()).is_err(), + "cold/wholekindcannotuselegacywithdrawal" + ); + println!( + "SYNTHETIC_C4C_DEFAULT48_SELECTED={}", + serde_json::to_string(&identity).unwrap() + ); + exercise_live_and_cold_withdrawal(&rt, directory, &identity, &selection_request).await; + drop(rt); + safety::run(&safety_preselect, directory).await; +} + +#[derive(Debug, Default)] +struct LiveTiming { + packets: u64, + min_gap_ns: Option, + max_gap_ns: u64, + below_min: u64, + above_max: u64, + policy_min_ns: u64, + policy_max_ns: u64, +} +struct LiveFeed { + stop: Arc, + handle: Option>, + error: Arc>>, + timing: Arc>, +} +impl Drop for LiveFeed { + fn drop(&mut self) { + self.stop.store(true, Ordering::Release); + if let Some(handle) = self.handle.take() { + handle.join().unwrap(); + } + } +} +fn live_feed(rt: &Arc) -> LiveFeed { + let selected = rt.inner.lock().unwrap().selection.temporal.clone().unwrap(); + let mut sequence = selected + .driver + .lock() + .unwrap() + .c4c_replay_floor_probe() + .iter() + .map(|f| f.1) + .max() + .unwrap(); + let policy = selected.stored.model().window_input_policy().unwrap(); + let min_gap = policy + .source_policies() + .iter() + .map(|p| p.min_packet_interval_ns) + .max() + .unwrap(); + let max_gap = policy + .source_policies() + .iter() + .map(|p| p.max_gap_ns) + .min() + .unwrap(); + let previous = selected + .driver + .lock() + .unwrap() + .c4c_replay_floor_probe() + .iter() + .map(|f| f.2) + .max() + .unwrap(); + let timing = Arc::new(Mutex::new(LiveTiming { + policy_min_ns: min_gap, + policy_max_ns: max_gap, + ..LiveTiming::default() + })); + let stats = timing.clone(); + let stop = Arc::new(AtomicBool::new(false)); + let flag = stop.clone(); + let error = Arc::new(Mutex::new(None)); + let failure = error.clone(); + let weak = Arc::downgrade(rt); + let handle = std::thread::spawn(move || { + let origin = Instant::now(); + let mut ordinal = 0u32; + let mut previous = previous; + let mut next_minimum = origin + Duration::from_nanos(min_gap); + while !flag.load(Ordering::Acquire) { + // Schedule real emissions, never rewrite packet time. A missed + // nominal deadline cannot produce a catch-up burst below the + // unchanged source policy's minimum packet interval. + let target = (origin + Duration::from_nanos(u64::from(ordinal) * fixture::PERIOD)) + .max(next_minimum); + // This finite SOFTWARE stimulus owns one host thread. Sleeping + // timers may be coalesced beyond a30Hz receiver contract; wait + // against the actual monotonic deadline without forging time. + while Instant::now() < target && !flag.load(Ordering::Acquire) { + std::hint::spin_loop(); + } + if flag.load(Ordering::Acquire) { + break; + } + let Some(runtime) = weak.upgrade() else { break }; + sequence = sequence.checked_add(1).unwrap(); + let now = runtime.receiver_now_ns(); + next_minimum = Instant::now() + Duration::from_nanos(min_gap); + { + let gap = now - previous; + let mut st = stats.lock().unwrap(); + st.packets += 1; + st.min_gap_ns = Some(st.min_gap_ns.map_or(gap, |v| v.min(gap))); + st.max_gap_ns = st.max_gap_ns.max(gap); + st.below_min += u64::from(gap < min_gap); + st.above_max += u64::from(gap > max_gap); + } + previous = now; + for source in 1..=3 { + if !runtime.selects_grid( + source, + &fixture::context() + .sources + .iter() + .find(|s| s.source_id == source) + .unwrap() + .grid_id, + ) { + *failure.lock().unwrap() = Some("actualcoldselectedgridnotadmitted".into()); + return; + } + if let Err(e) = + runtime.ingest_native(&packet(source, sequence, now, RoomLabel::Empty), now) + { + *failure.lock().unwrap() = Some(e); + return; + } + } + drop(runtime); + ordinal = ordinal.checked_add(1).unwrap(); + } + }); + LiveFeed { + stop, + handle: Some(handle), + error, + timing, + } +} +async fn current_ready(rt: &RoomLearningRuntime, feed: &LiveFeed) -> Value { + let start = Instant::now(); + loop { + let error = feed.error.lock().unwrap().clone(); + assert!(error.is_none(), "actualsyntheticliveinputfailed {error:?}"); + let now = rt.receiver_now_ns(); + let frame = rt + .temporal_selected_binary_v2( + now, + Some(&fixture::context().room_digest), + Some("synthetic-c4c-public-boot"), + ) + .unwrap(); + if frame["status"] == "ready" { + println!( + "SYNTHETIC_C4C_ACTUAL_LIVE_EMISSION_TIMING={:?}", + feed.timing.lock().unwrap() + ); + return frame; + } + if start.elapsed() >= Duration::from_secs(12) { + let selected = rt.inner.lock().unwrap().selection.temporal.clone().unwrap(); + let error = selected + .driver + .lock() + .unwrap() + .c4c_last_input_failure() + .map(str::to_owned); + panic!("actualsyntheticselectedinputdidnotbecomeready input_error={error:?} timing={:?} {frame}",feed.timing.lock().unwrap()); + } + tokio::time::sleep(Duration::from_millis(10)).await; + } +} +async fn exercise_live_and_cold_withdrawal( + rt: &Arc, + directory: &Path, + identity: &TemporalSelectedIdentityWire, + selection_request: &TemporalSelectRequestV2, +) { + let feed = live_feed(rt); + let frame = current_ready(rt, &feed).await; + assert_eq!(frame["identity"], serde_json::to_value(identity).unwrap()); + assert_eq!(frame["schema"], SELECTED_BINARY_SCHEMA); + let first = frame["window_start_unix_ms"].as_u64().unwrap(); + let end = frame["window_end_unix_ms"].as_u64().unwrap(); + let expires = frame["valid_until_unix_ms"].as_u64().unwrap(); + assert!((2000..=2150).contains(&(end - first))); + assert_eq!(frame["observed_at_unix_ms"], end); + assert_eq!(expires, first + 5000); + let before_arc = rt.inner.lock().unwrap().selection.temporal.clone().unwrap(); + let before_floors = before_arc.driver.lock().unwrap().c4c_replay_floor_probe(); + let before_pointer = rt + .store + .load_temporal_selection() + .unwrap() + .unwrap() + .envelope() + .clone(); + let retried = rt + .temporal_select_v2( + TemporalSelectRequestV2 { + expected: rt.mutation_guard().unwrap(), + ..selection_request.clone() + }, + rt.receiver_now_ns(), + ) + .unwrap(); + assert_eq!( + retried["model_selection"]["selected"], + serde_json::to_value(identity).unwrap() + ); + let after_arc = rt.inner.lock().unwrap().selection.temporal.clone().unwrap(); + assert!( + Arc::ptr_eq(&before_arc, &after_arc), + "lostACKsamerateSelectresetdriver" + ); + assert_eq!( + *rt.store + .load_temporal_selection() + .unwrap() + .unwrap() + .envelope(), + before_pointer + ); + let after_floors = after_arc.driver.lock().unwrap().c4c_replay_floor_probe(); + for before in before_floors { + let after = after_floors.iter().find(|f| f.0 == before.0).unwrap(); + assert!(after.1 >= before.1 && after.2 >= before.2); + } + assert_eq!( + rt.temporal_selected_binary_v2( + rt.receiver_now_ns(), + Some(&fixture::context().room_digest), + Some("synthetic-c4c-public-boot") + ) + .unwrap()["status"], + "ready", + "idempotentSelectcaused2swarmup" + ); + let boot = frame["boot_epoch"].clone(); + let receiver = frame["receiver_epoch"].clone(); + for _ in 0..3 { + let proof = rt + .temporal_selection_v2( + rt.receiver_now_ns(), + Some(&fixture::context().room_digest), + Some("synthetic-c4c-public-boot"), + ) + .unwrap(); + assert_eq!(proof["model_selection"]["current"]["status"], "ready"); + assert_eq!( + proof["model_selection"]["current"]["identity"], + serde_json::to_value(identity).unwrap() + ); + assert_eq!(proof["model_selection"]["current"]["boot_epoch"], boot); + assert_eq!( + proof["model_selection"]["current"]["receiver_epoch"], + receiver + ); + } + let wrong = rt + .paired_temporal_progress_snapshot( + &fixture::context().installation_id, + "synthetic-c4c-public-boot", + Some(crate::room_learner::lifecycle::PublishedProgressRoom { + binding_digest: "c".repeat(64), + revision: fixture::context().revision, + }), + &[1, 2, 3], + Some(&fixture::context().digest()), + rt.receiver_now_ns(), + ) + .unwrap(); + let wrong = serde_json::to_value(wrong).unwrap(); + assert_eq!(wrong["context_match"], false); + for key in [ + "room", + "capture", + "candidate", + "profile", + "evaluation", + "qualification", + "selection", + "sources", + "model_kind", + ] { + assert!(wrong[key].is_null(), "pairedmismatchleaked{key}"); + } + println!( + "SYNTHETIC_C4C_ACTUAL_SELECTED_LIVE={}", + serde_json::to_string(&json!({"current":frame,"paired_mismatch":wrong})).unwrap() + ); + drop(feed); + let copy = tempfile::Builder::new() + .prefix("SYNTHETIC-c4c-selected-expired-") + .tempdir() + .unwrap() + .keep(); + copy_private(directory, ©); + // Already-selected cold operation intentionally has no capture. The + // protected admitted chain, not an expired raw reread, supplies its original + // retention deadline for this test's separate cold-copy clock domain. + let deadline = rt + .inner + .lock() + .unwrap() + .selection + .temporal + .as_ref() + .unwrap() + .stored + .held() + .body + .identity + .original_deadline + .expires_at_unix_ms; + fn deny_raw(root: &Path) { + use std::os::unix::fs::PermissionsExt; + for e in std::fs::read_dir(root).unwrap() { + let p = e.unwrap().path(); + if p.is_dir() { + deny_raw(&p) + } else if p + .file_name() + .unwrap() + .to_string_lossy() + .starts_with("slot-") + { + std::fs::set_permissions(p, std::fs::Permissions::from_mode(0o000)).unwrap(); + } + } + } + deny_raw(©); + let cold = Arc::new(fixture::runtime(©, deadline * 1_000_000 + 1)); + assert!(cold.inner.lock().unwrap().capture.is_none()); + assert!(cold.temporal_v2_state_present()); + let selection = cold + .temporal_selection_v2( + cold.receiver_now_ns(), + Some(&fixture::context().room_digest), + Some("synthetic-cold-boot"), + ) + .unwrap(); + assert_eq!( + selection["model_selection"]["selected"], + serde_json::to_value(identity).unwrap() + ); + assert_ne!(selection["model_selection"]["current"]["status"], "ready"); + assert_eq!(cold.temporal_progress_context(), Some(fixture::context())); + let cold_arc = cold + .inner + .lock() + .unwrap() + .selection + .temporal + .clone() + .unwrap(); + let cold_retry = cold + .temporal_select_v2( + TemporalSelectRequestV2 { + expected: cold.mutation_guard().unwrap(), + ..selection_request.clone() + }, + cold.receiver_now_ns(), + ) + .unwrap(); + assert_eq!( + cold_retry["model_selection"]["selected"], + serde_json::to_value(identity).unwrap() + ); + assert!(Arc::ptr_eq( + &cold_arc, + cold.inner + .lock() + .unwrap() + .selection + .temporal + .as_ref() + .unwrap() + )); + + // Actual fresh native input must reach the admitted model even when cold + // selected state has NO capture and the original raw slots are unreadable. + let cold_feed = live_feed(&cold); + let cold_frame = current_ready(&cold, &cold_feed).await; + assert_eq!( + cold_frame["identity"], + serde_json::to_value(identity).unwrap() + ); + drop(cold_feed); + tokio::time::sleep(Duration::from_millis(250)).await; + assert_ne!( + cold.temporal_selected_binary_v2( + cold.receiver_now_ns(), + Some(&fixture::context().room_digest), + Some("synthetic-cold-boot") + ) + .unwrap()["status"], + "ready" + ); + + assert!(cold + .deselect_model(&cold.mutation_guard().unwrap()) + .is_err()); + let off = cold + .temporal_deselect_v2( + TemporalDeselectRequestV2 { + schema: DESELECT_REQUEST_SCHEMA.into(), + model_kind: WholeTemporalKind::ManagedTemporalV3, + expected: cold.mutation_guard().unwrap(), + selection_sha256: identity.selection_sha256.clone(), + selection_revision: identity.selection_revision, + }, + cold.receiver_now_ns(), + ) + .unwrap(); + assert_eq!(off["model_selection"]["schema"], SELECTION_STATUS_SCHEMA); + assert_eq!( + off["model_selection"]["revision"], + identity.selection_revision + 1 + ); + assert!(off["model_selection"]["selected"].is_null()); + assert_eq!(off["model_selection"]["current"]["status"], "none"); + drop(cold); + let reread = fixture::runtime(©, deadline * 1_000_000 + 1_000_000_000); + assert!(!reread.temporal_v2_state_present()); + let pointer = reread.store.load_selection().unwrap().unwrap(); + assert!(pointer.receipt.selected.is_none()); + assert!(pointer.receipt.previous.is_none()); + assert_eq!(pointer.receipt.revision, identity.selection_revision + 1); + assert_eq!( + rt.store + .load_temporal_selection() + .unwrap() + .unwrap() + .identity(), + *identity, + "otherownedprofilewithdrawalmustnotchangeoriginalmodelselection" + ); +} + +fn same_accepted_prefix(before: &Value, after: &Value) { + let count = before["payload"]["accepted"]["rows"] + .as_array() + .unwrap() + .len(); + for field in ["rows", "evidence", "identities"] { + let a = &before["payload"]["accepted"][field].as_array().unwrap()[..count]; + let b = &after["payload"]["accepted"][field].as_array().unwrap()[..count]; + assert_eq!( + crate::room_learner::qualification::hash(&a).unwrap(), + crate::room_learner::qualification::hash(&b).unwrap(), + "originalacceptedprefixchanged{field}" + ); + } +} + +#[cfg(all(test, unix))] +#[path = "lifecycle_temporal_public_safety_tests.rs"] +mod safety; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public_safety_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public_safety_tests.rs new file mode 100644 index 0000000000..ffd72e965e --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public_safety_tests.rs @@ -0,0 +1,484 @@ +//! Reusable actual actor controls over file copies of a genuine producer's +//! Complete/qualified model. No decoded DATA is turned into a permission. +use super::*; +use std::sync::Condvar; + +fn branch(source: &Path, name: &str) -> std::path::PathBuf { + let dir = tempfile::Builder::new() + .prefix(&format!("SYNTHETIC-c4c-{name}-")) + .tempdir() + .unwrap() + .keep(); + copy_private(source, &dir); + dir +} +fn profile_root(root: &Path) -> std::path::PathBuf { + root.join(&fixture::context().owner_scope) + .join("SensingServer") +} +fn artifact_root(root: &Path) -> std::path::PathBuf { + profile_root(root).join("room-learner") +} +fn file_clock(root: &Path) -> u64 { + // Test clock candidate only. Actual owner/current AEAD cold restore and + // immutable metadata loaders must authenticate all data before admission. + fn visit(root: &Path, at: &mut u64) { + for e in std::fs::read_dir(root).unwrap() { + let p = e.unwrap().path(); + if p.is_dir() { + visit(&p, at) + } else { + let name = p.file_name().unwrap().to_string_lossy(); + if name.starts_with("slot-") && name.ends_with(".bin") { + let b = std::fs::read(&p).unwrap(); + let n = u32::from_be_bytes(b[8..12].try_into().unwrap()) as usize; + let v: Value = serde_json::from_slice(&b[12..12 + n]).unwrap(); + *at = (*at).max(v["receiver_ns"].as_u64().unwrap()); + } else if name.starts_with("evaluation-") { + let v: Value = serde_json::from_slice(&std::fs::read(&p).unwrap()).unwrap(); + *at = (*at).max( + v["body"]["evaluated_at_receiver_ns"] + .as_str() + .unwrap() + .parse() + .unwrap(), + ); + } else if name.starts_with("assessment-") { + let v: Value = serde_json::from_slice(&std::fs::read(&p).unwrap()).unwrap(); + *at = (*at).max( + v["receipt"]["summary"]["assessed_at_unix_ms"] + .as_u64() + .unwrap() + * 1_000_000, + ); + } else if name == "model-selection-v1.json" { + let v: Value = serde_json::from_slice(&std::fs::read(&p).unwrap()).unwrap(); + *at = (*at) + .max(v["receipt"]["selected_at_unix_ms"].as_u64().unwrap() * 1_000_000); + } + } + } + } + let mut at = 0; + visit(root, &mut at); + at + 1_000_000_000 +} +fn metadata_hashes(root: &Path) -> Vec<(String, String)> { + use sha2::Digest; + fn walk(base: &Path, path: &Path, out: &mut Vec<(String, String)>) { + for e in std::fs::read_dir(path).unwrap() { + let p = e.unwrap().path(); + if p.is_dir() { + walk(base, &p, out) + } else { + out.push(( + p.strip_prefix(base).unwrap().to_string_lossy().into(), + format!("{:x}", sha2::Sha256::digest(std::fs::read(p).unwrap())), + )); + } + } + } + let mut out = vec![]; + walk(root, root, &mut out); + out.sort(); + out +} +async fn terminal(rt: &RoomLearningRuntime) -> JobPhase { + let start = Instant::now(); + loop { + let state = rt.inner.lock().unwrap(); + let done = match state.capture.as_ref() { + Some(CaptureWorkflow::Temporal(c)) => c.evidence_job.as_ref().is_some_and(|j| j.done()), + _ => true, + }; + if done && !busy(&state.job) { + return state.job.phase; + } + drop(state); + assert!(start.elapsed() < Duration::from_secs(90)); + tokio::time::sleep(Duration::from_millis(2)).await; + } +} +async fn qualified(root: &Path) -> (Arc, TemporalSelectRequestV2) { + let now = file_clock(root); + let rt = Arc::new(fixture::runtime(root, now)); + restore_public(&rt, now).await; + let status = rt.temporal_status_v2(rt.receiver_now_ns()).unwrap(); + let c: TemporalCaptureWire = serde_json::from_value(status["capture"].clone()).unwrap(); + let e: TemporalEvaluationWire = serde_json::from_value(status["evaluation"].clone()).unwrap(); + rt.temporal_qualify_v2( + TemporalQualifyRequestV2 { + schema: QUALIFY_REQUEST_SCHEMA.into(), + expected: rt.mutation_guard().unwrap(), + capture_generation_id: c.capture_generation_id.clone(), + refs: e.refs.clone(), + }, + rt.receiver_now_ns(), + ) + .unwrap(); + assert_eq!(terminal(&rt).await, JobPhase::Ready); + let s = rt.temporal_status_v2(rt.receiver_now_ns()).unwrap(); + assert_eq!(s["assessment"]["outcome"], "qualified"); + let request = TemporalSelectRequestV2 { + schema: SELECT_REQUEST_SCHEMA.into(), + expected: rt.mutation_guard().unwrap(), + capture_generation_id: c.capture_generation_id, + refs: TemporalQualificationRefsWire { + evaluation: e.refs, + qualification_sha256: s["assessment"]["qualification_sha256"] + .as_str() + .unwrap() + .into(), + }, + }; + (rt, request) +} +fn selected(root: &Path) -> Arc { + let rt = Arc::new(fixture::runtime(root, file_clock(root))); + assert!(rt.inner.lock().unwrap().capture.is_none()); + assert!(rt.inner.lock().unwrap().selection.temporal.is_some()); + rt +} +fn deselect(rt: &RoomLearningRuntime) -> TemporalDeselectRequestV2 { + let identity = rt + .inner + .lock() + .unwrap() + .selection + .temporal + .as_ref() + .unwrap() + .stored + .identity(); + TemporalDeselectRequestV2 { + schema: DESELECT_REQUEST_SCHEMA.into(), + model_kind: WholeTemporalKind::ManagedTemporalV3, + expected: rt.mutation_guard().unwrap(), + selection_sha256: identity.selection_sha256, + selection_revision: identity.selection_revision, + } +} +struct Release { + state: Arc<(Mutex, Condvar)>, +} +impl Release { + fn new() -> Self { + Self { + state: Arc::new((Mutex::new(false), Condvar::new())), + } + } + fn open(&self) { + *self.state.0.lock().unwrap() = true; + self.state.1.notify_all(); + } +} +impl Drop for Release { + fn drop(&mut self) { + self.open(); + } +} + +pub(super) async fn run(preselect: &Path, selected_source: &Path) { + // Restore must never advertise future stored evaluation as Ready. The + // candidate clock is deliberately ONLY the older authenticated raw floor. + let root = branch(preselect, "future-evaluation"); + let mut raw = 0; + fn raw_clock(root: &Path, raw: &mut u64) { + for e in std::fs::read_dir(root).unwrap() { + let p = e.unwrap().path(); + if p.is_dir() { + raw_clock(&p, raw) + } else if p + .file_name() + .unwrap() + .to_string_lossy() + .starts_with("slot-") + { + let b = std::fs::read(p).unwrap(); + let n = u32::from_be_bytes(b[8..12].try_into().unwrap()) as usize; + let h: Value = serde_json::from_slice(&b[12..12 + n]).unwrap(); + *raw = (*raw).max(h["receiver_ns"].as_u64().unwrap()); + } + } + } + raw_clock(&root, &mut raw); + let rt = Arc::new(fixture::runtime(&root, raw + 1)); + let prior = metadata_hashes(&root); + let s = rt.temporal_status_v2(raw + 1).unwrap(); + let id = s["recovery"]["capture_generation_id"].as_str().unwrap(); + rt.temporal_restore_v2( + &rt.mutation_guard().unwrap(), + &fixture::context(), + id, + raw + 1, + ) + .unwrap(); + assert_eq!(terminal(&rt).await, JobPhase::Failed); + assert_eq!( + rt.inner.lock().unwrap().job.error.as_deref(), + Some("temporal_evaluation_stored_clock_reversed") + ); + assert_eq!(metadata_hashes(&root), prior); + drop(rt); + println!("C4C_ACTOR_SAFETY future_evaluation_refused=true original_bytes=true"); + + // The actual current source/job observes expiry AFTER staged file fsync, + // immediately before authoritative pointer rename. No pointer is installed. + for cancel in [false, true] { + let root = branch( + preselect, + if cancel { + "preinstall-cancel" + } else { + "preinstall-expiry" + }, + ); + let (rt, request) = qualified(&root).await; + let expiry = capture(&rt, rt.receiver_now_ns()) + .original_deadline + .expires_at_unix_ms + * 1_000_000; + let weak = Arc::downgrade(&rt); + let called = Arc::new(AtomicBool::new(false)); + let seen = called.clone(); + *rt.temporal_evidence_io_hook.lock().unwrap() = Some(Arc::new(move |stage| { + if stage == "selection_before_pointer_install" { + let owner = weak.upgrade().unwrap(); + let state = owner.inner.lock().unwrap(); + if cancel { + state.job.cancel.store(true, Ordering::Release) + } else { + let Some(CaptureWorkflow::Temporal(c)) = &state.capture else { + panic!() + }; + c.evidence_job.as_ref().unwrap().observe_now(expiry); + } + seen.store(true, Ordering::Release); + } + Ok(()) + })); + assert!(rt + .temporal_select_v2(request, rt.receiver_now_ns()) + .is_err()); + assert!(called.load(Ordering::Acquire)); + assert!(rt.inner.lock().unwrap().selection.temporal.is_none()); + assert!(!artifact_root(&root) + .join("model-selection-v1.json") + .exists()); + assert!(!std::fs::read_dir(artifact_root(&root)).unwrap().any(|e| e + .unwrap() + .file_name() + .to_string_lossy() + .starts_with("model-selection-v1.tmp-"))); + drop(rt); + } + println!("C4C_ACTOR_SAFETY staged_expiry_and_cancel_refused=true no_pointer_or_staging=true"); + + // A valid rename is the commit point even if the later directory barrier + // or ACK fails. RAM remains unavailable; cold trusted DAG reconciles it. + let root = branch(preselect, "select-after-rename"); + let (rt, request) = qualified(&root).await; + *rt.temporal_evidence_io_hook.lock().unwrap() = Some(Arc::new(|stage| { + if stage == "selection_after_pointer_rename" { + Err("synthetic_postrename_sync_failure".into()) + } else { + Ok(()) + } + })); + assert_eq!( + rt.temporal_select_v2(request, rt.receiver_now_ns()) + .unwrap_err(), + "temporal_selection_pointer_write_unknown" + ); + assert!(rt.inner.lock().unwrap().selection.temporal.is_none()); + assert!(rt.inner.lock().unwrap().selection.error.is_some()); + let committed = rt + .store + .load_temporal_selection() + .unwrap() + .unwrap() + .identity(); + drop(rt); + let cold = selected(&root); + assert_eq!( + cold.inner + .lock() + .unwrap() + .selection + .temporal + .as_ref() + .unwrap() + .stored + .identity(), + committed + ); + drop(cold); + println!("C4C_ACTOR_SAFETY select_postrename_unknown_cold_reconciled=true"); + + let root = branch(selected_source, "deselect-after-rename"); + let rt = selected(&root); + let identity = rt + .inner + .lock() + .unwrap() + .selection + .temporal + .as_ref() + .unwrap() + .stored + .identity(); + *rt.temporal_evidence_io_hook.lock().unwrap() = Some(Arc::new(|stage| { + if stage == "deselect_after_pointer_rename" { + Err("synthetic_postrename_sync_failure".into()) + } else { + Ok(()) + } + })); + assert_eq!( + rt.temporal_deselect_v2(deselect(&rt), rt.receiver_now_ns()) + .unwrap_err(), + "selection_pointer_write_unknown" + ); + assert!(rt.inner.lock().unwrap().selection.error.is_some()); + assert_ne!( + rt.temporal_selected_binary_v2( + rt.receiver_now_ns(), + Some(&fixture::context().room_digest), + Some("synthetic-safety-boot") + ) + .unwrap()["status"], + "ready" + ); + drop(rt); + let cold = Arc::new(fixture::runtime(&root, file_clock(&root))); + assert!(cold.inner.lock().unwrap().selection.temporal.is_none()); + assert!(cold + .store + .load_selection() + .unwrap() + .unwrap() + .receipt + .selected + .is_none()); + assert_eq!( + cold.store + .load_selection() + .unwrap() + .unwrap() + .receipt + .revision, + identity.selection_revision + 1 + ); + drop(cold); + println!("C4C_ACTOR_SAFETY deselect_postrename_unknown_cold_off=true"); + + // Real pending Select IO is drained before a simulated owner erasure ACK. + let root = branch(preselect, "owner-erase-drain"); + let (rt, request) = qualified(&root).await; + let release = Release::new(); + let held = release.state.clone(); + let (entered_tx, entered_rx) = std::sync::mpsc::sync_channel(1); + *rt.temporal_evidence_io_hook.lock().unwrap() = Some(Arc::new(move |stage| { + if stage == "selection_before_pointer_install" { + entered_tx.send(()).unwrap(); + let mut open = held.0.lock().unwrap(); + while !*open { + open = held.1.wait(open).unwrap(); + } + } + Ok(()) + })); + let worker = rt.clone(); + let operation = + std::thread::spawn(move || worker.temporal_select_v2(request, worker.receiver_now_ns())); + entered_rx.recv_timeout(Duration::from_secs(15)).unwrap(); + let owner = rt.clone(); + let (tx, rx) = std::sync::mpsc::sync_channel(1); + let cancellation = std::thread::spawn(move || { + let r = owner.cancel_temporal_durability(&owner.mutation_guard().unwrap()); + tx.send(r).unwrap(); + }); + assert!( + rx.recv_timeout(Duration::from_millis(100)).is_err(), + "eraseACKescapedpendingactualstore" + ); + release.open(); + assert!(operation.join().unwrap().is_err()); + rx.recv_timeout(Duration::from_secs(15)).unwrap().unwrap(); + cancellation.join().unwrap(); + drop(rt); + let protected = profile_root(&root); + std::fs::remove_dir_all(&protected).unwrap(); + tokio::time::sleep(Duration::from_millis(100)).await; + assert!(!protected.exists(), "latewriterrecreatederasedowner"); + println!("C4C_ACTOR_SAFETY owner_erase_waited_real_store_and_no_recreation=true"); + + // Learning of a corrupt current durable chain fences the SAME hot Arc; + // successful exact recertification restores only that committed choice. + let root = branch(selected_source, "hot-integrity"); + let rt = selected(&root); + let arc = rt.inner.lock().unwrap().selection.temporal.clone().unwrap(); + let identity = arc.stored.identity(); + let feed = live_feed(&rt); + current_ready(&rt, &feed).await; + fn model_file(root: &Path, sha: &str) -> Option { + for e in std::fs::read_dir(root).unwrap() { + let p = e.unwrap().path(); + if p.is_dir() { + if let Some(v) = model_file(&p, sha) { + return Some(v); + } + } else if p.file_name().unwrap().to_string_lossy() == format!("{sha}.json") { + return Some(p); + } + } + None + } + let file = model_file(&root, &identity.model_sha256).unwrap(); + let bytes = std::fs::read(&file).unwrap(); + std::fs::write(&file, b"corrupt-owned-model-control").unwrap(); + let snapshot = rt + .temporal_selection_v2( + rt.receiver_now_ns(), + Some(&fixture::context().room_digest), + Some("synthetic-safety-boot"), + ) + .unwrap(); + assert_eq!( + snapshot["model_selection"]["current"]["status"], + "unavailable" + ); + assert!(arc.invalid.load(Ordering::Acquire)); + assert_eq!( + rt.temporal_selected_binary_v2( + rt.receiver_now_ns(), + Some(&fixture::context().room_digest), + Some("synthetic-safety-boot") + ) + .unwrap()["status"], + "unavailable" + ); + std::fs::write(&file, bytes).unwrap(); + let snapshot = rt + .temporal_selection_v2( + rt.receiver_now_ns(), + Some(&fixture::context().room_digest), + Some("synthetic-safety-boot"), + ) + .unwrap(); + assert_eq!(snapshot["model_selection"]["current"]["status"], "ready"); + assert!(!arc.invalid.load(Ordering::Acquire)); + assert!(Arc::ptr_eq( + &arc, + rt.inner + .lock() + .unwrap() + .selection + .temporal + .as_ref() + .unwrap() + )); + drop(feed); + drop(rt); + println!("C4C_ACTOR_SAFETY deep_failure_fences_hot_arc_exact_recert_same_arc=true"); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_selection.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_selection.rs new file mode 100644 index 0000000000..6d8cadd949 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_selection.rs @@ -0,0 +1,544 @@ +//! Whole-kind selected state; same protected pointer/journal and live input. +use super::*; +use crate::room_learner::{ + artifact::StoredTemporalSelection, temporal_live_driver::ContinuousTemporalInput, + temporal_wire::*, +}; +pub(in crate::room_learner::lifecycle) struct TemporalSelectedRuntime { + pub(in crate::room_learner::lifecycle) stored: StoredTemporalSelection, + pub(in crate::room_learner::lifecycle) driver: Mutex, + pub(super) invalid: AtomicBool, +} +impl TemporalSelectedRuntime { + pub(in crate::room_learner::lifecycle) fn new( + stored: StoredTemporalSelection, + ) -> Result { + let driver = ContinuousTemporalInput::from_selected(&stored)?; + Ok(Self { + stored, + driver: Mutex::new(driver), + invalid: AtomicBool::new(false), + }) + } +} +impl RoomLearningRuntime { + pub fn temporal_progress_context(&self) -> Option { + let state = self.inner.lock().ok()?; + state.capture_context().cloned().or_else(|| { + state + .selection + .temporal + .as_ref() + .map(|s| s.stored.envelope().receipt.selected.context.clone()) + }) + } + pub fn temporal_v2_state_present(&self) -> bool { + self.inner.lock().map(|s|matches!(s.capture.as_ref(),Some(CaptureWorkflow::Temporal(c))if c.core.as_ref().is_some_and(|core|core.is_full_run()))||s.selection.temporal.is_some()).unwrap_or(true) + } + pub fn temporal_selection_v2( + &self, + now: u64, + expected_room_binding: Option<&str>, + boot_epoch: Option<&str>, + ) -> Result { + self.temporal_selection_snapshot(now, expected_room_binding, boot_epoch, true) + } + pub fn temporal_selected_binary_v2( + &self, + now: u64, + expected_room_binding: Option<&str>, + boot_epoch: Option<&str>, + ) -> Result { + let snapshot = + self.temporal_selection_snapshot(now, expected_room_binding, boot_epoch, false)?; + let mut current = snapshot + .pointer("/model_selection/current") + .cloned() + .ok_or("selected_projection_unavailable")?; + current + .as_object_mut() + .ok_or("selected_projection_unavailable")? + .insert( + "schema".into(), + Value::String(SELECTED_BINARY_SCHEMA.into()), + ); + Ok(current) + } + fn temporal_selection_snapshot( + &self, + now: u64, + expected_room_binding: Option<&str>, + boot_epoch: Option<&str>, + recertify: bool, + ) -> Result { + let (selected, assessment, integrity, revision) = { + let s = self.inner.lock().map_err(|_| "learner state unavailable")?; + ( + s.selection.temporal.clone(), + match s.capture.as_ref() { + Some(CaptureWorkflow::Temporal(c)) => c.assessment.clone(), + _ => None, + }, + s.selection.error.clone(), + s.selection.known_revision(), + ) + }; + let mut result = TemporalSelectionStatusWire { + schema: SELECTION_STATUS_SCHEMA.into(), + revision, + selected: None, + previous_available: false, + qualification: assessment + .as_ref() + .and_then(|a| a.qualification.as_ref().map(|q| q.receipt.summary.clone())), + qualification_error: assessment + .as_ref() + .filter(|a| a.qualification.is_none()) + .and_then(|a| a.assessment.receipt.summary.failure_reason.clone()), + integrity_error: integrity, + current: TemporalCurrentEvidenceWire::None, + }; + if let Some(selected) = selected { + let identity = selected.stored.identity(); + if expected_room_binding.is_some_and(|room| room != identity.room_digest) { + result.qualification = None; + result.qualification_error = Some("selected_room_context_mismatch".into()); + return Ok( + json!({"schema":SELECTION_SCHEMA,"success":true,"model_selection":result}), + ); + } + result.revision = identity.selection_revision; + result.selected = Some(identity.clone()); + result.qualification = Some(selected.stored.qualification().receipt.summary.clone()); + let unavailable = |reason: &str| TemporalCurrentEvidenceWire::Unavailable { + identity: Some(identity.clone()), + reason: bounded(reason.into()), + boot_epoch: boot_epoch.map(str::to_owned), + receiver_epoch: Some(self.runtime_epoch.clone()), + }; + let has_integrity_error = result.integrity_error.is_some(); + let check = (|| -> Result { + if has_integrity_error { + return Err("selected_integrity_unavailable".into()); + } + if expected_room_binding != Some(identity.room_digest.as_str()) { + return Err("selected_room_context_mismatch".into()); + } + let boot = boot_epoch + .filter(|s| !s.is_empty() && s.len() <= 128 && !s.chars().any(char::is_control)) + .ok_or("selected_boot_epoch_unavailable")?; + // Protected current pointer/journal and complete immutable DAG, + // not hashes returned by a status/JSON import. + if recertify { + let verified = self + .store + .load_temporal_selection() + .and_then(|current| { + current.ok_or("temporal_selection_pointer_missing".into()) + }) + .and_then(|current| { + if current.identity() == identity { + Ok(()) + } else { + Err("selected_pointer_changed".into()) + } + }); + match verified { + Ok(()) => { + let state = + self.inner.lock().map_err(|_| "learner state unavailable")?; + if state + .selection + .temporal + .as_ref() + .is_some_and(|s| Arc::ptr_eq(s, &selected)) + && state.selection.error.is_none() + && !assessment_blocks(&state, &identity) + { + selected.invalid.store(false, Ordering::Release); + } + } + Err(error) => { + let state = + self.inner.lock().map_err(|_| "learner state unavailable")?; + if state + .selection + .temporal + .as_ref() + .is_some_and(|s| Arc::ptr_eq(s, &selected)) + { + selected.invalid.store(true, Ordering::Release); + } + return Err(error); + } + } + } + if selected.invalid.load(Ordering::Acquire) { + return Err("selected_integrity_unavailable".into()); + } + if assessment.as_ref().is_some_and(|a| { + a.assessment.receipt.summary.model_sha256 == identity.model_sha256 + && (a.qualification.is_none() + || a.qualification + .as_ref() + .is_some_and(|q| q.receipt_sha256 != identity.qualification_sha256)) + }) { + return Err("newer_qualification_failed_or_requires_selection".into()); + } + let o = { + let mut driver = selected + .driver + .lock() + .map_err(|_| "selected_native_input_unavailable")?; + let current_now = self.receiver_now_ns(); + if current_now < now { + return Err("selected_receiver_clock_reversed".into()); + } + driver.selected_observation(current_now)? + }; + let probability = o + .prediction + .occupied_probability + .filter(|p| p.is_finite() && (0.0..=1.0).contains(p)) + .ok_or("selected_probability_unavailable")?; + let s = self.inner.lock().map_err(|_| "learner state unavailable")?; + if s.selection.error.is_some() + || selected.invalid.load(Ordering::Acquire) + || assessment_blocks(&s, &identity) + || !s + .selection + .temporal + .as_ref() + .is_some_and(|current| Arc::ptr_eq(current, &selected)) + { + return Err("selected_pointer_changed".into()); + } + if s.capture_context().is_some_and(|context| { + context != &selected.stored.envelope().receipt.selected.context + }) { + return Err("selected_source_context_changed".into()); + } + Ok(TemporalCurrentEvidenceWire::Ready { + identity: identity.clone(), + presence: o.prediction.decision + == crate::room_learner::model::PresenceDecision::Occupied, + decision: o.prediction.decision, + occupied_probability: probability, + observed_at_unix_ms: o.end_ns / 1_000_000, + observation_digest: o.digest, + boot_epoch: boot.into(), + receiver_epoch: self.runtime_epoch.clone(), + window_start_unix_ms: o.first_ns / 1_000_000, + window_end_unix_ms: o.end_ns / 1_000_000, + valid_until_unix_ms: o.expires_ns / 1_000_000, + source_observed_at_unix_ms: o.source_latest_ns / 1_000_000, + }) + })(); + result.current = check.unwrap_or_else(|error| unavailable(&error)); + if selected.invalid.load(Ordering::Acquire) && result.integrity_error.is_none() { + result.integrity_error = Some("selected_integrity_unavailable".into()); + } + } + Ok(json!({"schema":SELECTION_SCHEMA,"success":true,"model_selection":result})) + } + pub fn temporal_select_v2( + self: &Arc, + request: TemporalSelectRequestV2, + now: u64, + ) -> Result { + if request.schema != SELECT_REQUEST_SCHEMA { + return Err("learner_unsupported_schema".into()); + } + let prior = { + self.inner + .lock() + .map_err(|_| "learner state unavailable")? + .selection + .temporal + .clone() + }; + if let Some(selected) = prior { + let current = selected.stored.identity(); + let refs = &request.refs; + let complete = &refs.evaluation.complete; + let exact = current.model_kind == complete.model.model_kind + && current.model_sha256 == complete.model.model_sha256 + && current.verified_fit_record_sha256 == complete.model.verified_fit_record_sha256 + && current.training_proof_sha256 == complete.training_proof_sha256 + && current.complete_record_sha256 == complete.complete_record_sha256 + && selected + .stored + .held() + .body + .complete_commitment + .commitment_sha256 + == complete.complete_commitment_sha256 + && current.held_out_evidence_sha256 == refs.evaluation.held_out_evidence_sha256 + && current.evaluation_receipt_sha256 == refs.evaluation.evaluation_receipt_sha256 + && current.qualification_sha256 == refs.qualification_sha256; + if !exact { + return Err("selected_target_requires_explicit_deselect".into()); + } + if selected.stored.capture_generation_id()? != request.capture_generation_id { + return Err("learner_capture_identity_changed".into()); + } + { + let state = self.command_state(&request.expected)?; + if busy(&state.job) { + return Err("learner_job_busy".into()); + } + if !state + .selection + .temporal + .as_ref() + .is_some_and(|s| Arc::ptr_eq(s, &selected)) + { + return Err("selected_pointer_changed".into()); + } + if state.selection.error.is_some() + || selected.invalid.load(Ordering::Acquire) + || assessment_blocks(&state, ¤t) + { + return Err("selection_reconciliation_required".into()); + } + } + // Exact already-admitted choice, including a lost ACK or expired + // raw lease: no new adoption, IO, timestamp, revision or driver. + return self.temporal_selection_snapshot(self.receiver_now_ns(), None, None, false); + } + { + let s = self.inner.lock().map_err(|_| "learner state unavailable")?; + let Some(CaptureWorkflow::Temporal(c)) = &s.capture else { + return Err(PRIVATE_KIND.into()); + }; + if c.core + .as_ref() + .is_none_or(|core| core.public_generation_id() != request.capture_generation_id) + { + return Err("learner_capture_identity_changed".into()); + } + let a = c + .assessment + .as_ref() + .ok_or("temporal_qualification_required")?; + let q = a + .qualification + .as_ref() + .ok_or("temporal_candidate_not_qualified")?; + let e = &request.refs.evaluation; + if q.receipt_sha256 != request.refs.qualification_sha256 + || q.receipt.summary.evaluation_receipt_sha256 != e.evaluation_receipt_sha256 + || q.receipt.summary.held_out_evidence_sha256 != e.held_out_evidence_sha256 + || c.core + .as_ref() + .and_then(|core| { + core.public_complete_refs(&q.receipt.summary.training_proof_sha256) + }) + .as_ref() + != Some(&e.complete) + { + return Err("learner_qualification_identity_changed".into()); + } + } + self.run_temporal_selection(&request.expected, now)?; + // The actual operation already read back the durable DAG. Its drain + // signal has fired; ACK enrichment must not start another store read + // that could outlive an owner-erasure barrier. Poll/Save uses REST's + // separately deep-validated read path. + self.temporal_selection_snapshot(now, None, None, false) + } +} + +#[derive(Clone)] +pub(in crate::room_learner::lifecycle) struct TemporalWithdrawalJob { + lifetime: Arc, +} +impl TemporalWithdrawalJob { + pub(in crate::room_learner::lifecycle) fn cancel(&self) { + self.lifetime.cancel(); + } + pub(in crate::room_learner::lifecycle) fn drain(&self) { + self.lifetime.drain(); + } +} +struct WithdrawalCompletion { + runtime: std::sync::Weak, + job: TemporalWithdrawalJob, +} +impl Drop for WithdrawalCompletion { + fn drop(&mut self) { + let runtime = self.runtime.upgrade(); + if let Some(runtime) = &runtime { + let mut state = runtime.inner.lock().unwrap_or_else(|e| e.into_inner()); + if Arc::ptr_eq(&state.job.cancel, &self.job.lifetime.cancel) && busy(&state.job) { + state.job.phase = if self.job.lifetime.cancel.load(Ordering::Acquire) { + JobPhase::Cancelled + } else { + JobPhase::Failed + }; + state.job.error = Some("selection_withdrawal_unavailable".into()); + state.selection.error=Some("Selection withdrawal outcome unavailable; restart and verify the saved choice.".into()); + } + } + *self + .job + .lifetime + .completed + .lock() + .unwrap_or_else(|e| e.into_inner()) = true; + self.job.lifetime.completion.notify_all(); + drop(runtime); + } +} +impl RoomLearningRuntime { + pub fn temporal_deselect_v2( + self: &Arc, + request: TemporalDeselectRequestV2, + now: u64, + ) -> Result { + use std::sync::{atomic::AtomicU64, Condvar}; + use std::time::Instant; + if request.schema != DESELECT_REQUEST_SCHEMA { + return Err("learner_unsupported_schema".into()); + } + let retired_job; + let retired_withdrawal; + let (job, generation) = { + let mut state = self.command_state(&request.expected)?; + if busy(&state.job) { + return Err("learner_job_busy".into()); + } + let current = state + .selection + .temporal + .as_ref() + .ok_or("temporal_selection_missing")?; + let identity = current.stored.identity(); + if identity.selection_sha256 != request.selection_sha256 + || identity.selection_revision != request.selection_revision + { + return Err("selected_pointer_changed".into()); + } + let cancel = Arc::new(AtomicBool::new(false)); + let lifetime = Arc::new(super::fit_control::FitLifetime { + cancel: cancel.clone(), + publication: Mutex::new(()), + completed: Mutex::new(false), + completion: Condvar::new(), + anchor: now, + origin: Instant::now(), + observed: AtomicU64::new(now), + expires_at_ms: u64::MAX, + #[cfg(test)] + hook: self + .temporal_fit_hook + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(), + #[cfg(test)] + io_hook: self + .temporal_evidence_io_hook + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone(), + }); + let job = TemporalWithdrawalJob { lifetime }; + state.selection.error=Some("Selection withdrawal pending; current evidence is unavailable until durable reconciliation.".into()); + retired_withdrawal = state.selection.withdrawal.replace(job.clone()); + retired_job = std::mem::replace( + &mut state.job, + Job { + phase: JobPhase::Running, + cancel, + report: None, + error: None, + }, + ); + (job, state.generation) + }; + drop(retired_job); + drop(retired_withdrawal); + let completion = WithdrawalCompletion { + runtime: Arc::downgrade(self), + job: job.clone(), + }; + let result = (|| -> Result { + let _publication = job + .lifetime + .publication + .lock() + .map_err(|_| "selection_publication_unavailable")?; + job.lifetime.allowed()?; + let envelope = self.store.withdraw_temporal_selection( + &request.selection_sha256, + request.selection_revision, + now, + &mut || { + job.lifetime.allowed()?; + Ok(()) + }, + &mut || { + job.lifetime.io("deselect_before_pointer_install")?; + job.lifetime.allowed()?; + Ok(()) + }, + &mut || job.lifetime.io("deselect_after_pointer_rename"), + )?; + job.lifetime.allowed()?; + Ok(envelope) + })(); + let mut retired_selected = None; + { + let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; + if state.generation != generation + || !Arc::ptr_eq(&state.job.cancel, &job.lifetime.cancel) + { + return Err("selection_withdrawal_not_adopted".into()); + } + match &result { + Ok(envelope) => { + retired_selected = state.selection.temporal.take(); + state + .selection + .install_negative_temporal_tombstone(envelope.clone()); + state.selection.error = None; + state.job.phase = JobPhase::Ready; + state.job.error = None; + } + Err(error) => { + state.selection.error=Some("Selection withdrawal outcome unavailable; reread the saved choice before continuing.".into()); + state.job.phase = if job.lifetime.cancel.load(Ordering::Acquire) { + JobPhase::Cancelled + } else { + JobPhase::Failed + }; + state.job.error = Some(bounded(error.clone())); + } + } + } + if let Some(selected) = retired_selected { + selected + .driver + .lock() + .map_err(|_| "selected_driver_unavailable")? + .deactivate(); + drop(selected); + } + drop(completion); + result?; + self.temporal_selection_v2(self.receiver_now_ns(), None, None) + } +} + +fn assessment_blocks(state: &Inner, identity: &TemporalSelectedIdentityWire) -> bool { + match state.capture.as_ref() { + Some(CaptureWorkflow::Temporal(c)) => c.assessment.as_ref().is_some_and(|a| { + a.assessment.receipt.summary.model_sha256 == identity.model_sha256 + && (a.qualification.is_none() + || a.qualification + .as_ref() + .is_some_and(|q| q.receipt_sha256 != identity.qualification_sha256)) + }), + _ => false, + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs index 02bd0041d6..d303f72bfb 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/mod.rs @@ -28,3 +28,11 @@ pub mod temporal_live_driver; /// Inactive engineering schedule contract; no managed capture authority. pub mod managed_schedule_contract; + +/// Whole-model diagnostic proof/evaluation DATA; decoded bytes are not authority. +pub(crate) mod temporal_evaluation; + +/// Versioned public whole-kind DTOs; no decoder grants producer authority. +pub mod temporal_wire; + +pub(crate) mod temporal_qualification; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification.rs index 9d5145b06e..393c7b4787 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/qualification.rs @@ -436,3 +436,14 @@ impl QualificationEnvelope { #[cfg(test)] #[path = "qualification_boundary_tests.rs"] mod boundary_tests; + +/// Descriptive native-window failure flags use the SAME integer thresholds and +/// denominators as criteria. Legacy policy/behavior/serialization is unchanged. +pub(crate) fn native_condition_failure_codes(c:&super::training::ConditionEvaluation)->Vec{ + let mut codes=Vec::new(); + if (c.scored as u128)*PERCENT_SCALE<(c.samples as u128)*MIN_COVERAGE_PERCENT{codes.push("coverage_below_minimum".into());} + if matches!(c.condition,RoomLabel::OccupiedStatic|RoomLabel::OccupiedMoving){ + if(c.confusion.true_occupied as u128)*PERCENT_SCALE<(c.samples as u128)*MIN_SENSITIVITY_PERCENT{codes.push("sensitivity_below_minimum".into());} + }else if(c.confusion.false_occupied as u128)*PERCENT_SCALE>(c.samples as u128)*MAX_FALSE_POSITIVE_PERCENT{codes.push("false_positive_rate_above_maximum".into());} + codes +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store.rs index cae2fc9ea0..86dd811981 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store.rs @@ -229,6 +229,19 @@ impl RecoveryStore { } private_directory(&self.directory) } + /// Non-repairing owner validation for read-scoped status discovery. + /// Unlike prepare_directory this never creates directories or chmods them. + fn verify_directory_read_only(&self)->Result<(),String>{ + let owned=self.directory.parent().ok_or("recovery_owner_mismatch")?; + if !hex(&self.owner)||owned.file_name().and_then(|v|v.to_str())!=Some("SensingServer")||owned.parent().and_then(Path::file_name).and_then(|v|v.to_str())!=Some(self.owner.as_str())||owned.canonicalize().map_err(|_|"recovery_owner_mismatch")?!=owned{return Err("recovery_owner_mismatch".into());} + for(path,directory)in[(owned.to_path_buf(),true),(owned.join("account-owner-v1.json"),false),(self.directory.clone(),true)]{ + let m=fs::symlink_metadata(&path).map_err(|_|"recovery_owner_mismatch")?; + if m.file_type().is_symlink()||(directory&&!m.is_dir())||(!directory&&!m.is_file()){return Err("recovery_unsafe_owner_path".into());}private_metadata(&m,directory)?; + } + let witness=crate::companion::private_read(&owned.join("account-owner-v1.json")).map_err(|_|"recovery_owner_mismatch")?; + let value:serde_json::Value=serde_json::from_slice(&witness).map_err(|_|"recovery_owner_mismatch")?; + if value.get("schema").and_then(|v|v.as_str())!=Some("ruview.account-owner.v1")||value.get("scope_key").and_then(|v|v.as_str())!=Some(self.owner.as_str()){return Err("recovery_owner_mismatch".into());}Ok(()) + } fn descriptor(&self, d: &RecoveryDescriptor, now: Option) -> Result<(), String> { if d.schema != RECOVERY_SCHEMA || !id(&d.id) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_artifact_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_artifact_store.rs index bf9a784f7a..92374b5d68 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_artifact_store.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_artifact_store.rs @@ -58,7 +58,7 @@ fn metadata(meta: &fs::Metadata, directory: bool) -> Result<(), String> { #[allow(unreachable_code)] Ok(()) } -fn directory(path: &Path, create: bool) -> Result { +pub(super) fn directory(path: &Path, create: bool) -> Result { match fs::symlink_metadata(path) { Ok(meta) => { metadata(&meta, true)?; @@ -130,6 +130,11 @@ pub(super) fn bounded_read(path: &Path, limit: usize) -> Result, String> Ok(bytes) } pub(super) fn immutable(path: &Path, bytes: &[u8]) -> Result<(), String> { + immutable_with_io(path,bytes,&mut |_|Ok(())) +} +// Same body, with a bounded IO-stage seam used by the C4C current job. A +// callback cannot create source/result authority or choose paths/content. +pub(super) fn immutable_with_io(path:&Path,bytes:&[u8],io:&mut dyn FnMut(&'static str)->Result<(),String>)->Result<(),String>{ if bytes.len() > BLOB_LIMIT { return Err("temporal_artifact_size_limit".into()); } @@ -157,11 +162,27 @@ pub(super) fn immutable(path: &Path, bytes: &[u8]) -> Result<(), String> { Err(_) => return Err("temporal_artifact_install_failed".into()), } cleanup.map_err(|_| "temporal_artifact_stage_cleanup_failed")?; + io("before:directory_fsync")?; fs::File::open(parent) .and_then(|file| file.sync_all()) .map_err(|_| "temporal_artifact_sync_failed")?; Ok(()) } +/// Re-establish a failed/ambiguous immutable metadata durability barrier. Equal +/// existing bytes alone do not prove an earlier file/directory sync succeeded. +pub(super) fn recertify_immutable(path:&Path,expected:&[u8],limit:usize,io:&mut dyn FnMut(&'static str)->Result<(),String>)->Result<(),String>{ + if expected.len()>limit||bounded_read(path,limit)?!=expected{return Err("temporal_artifact_immutable_mismatch".into());} + let mut options=fs::OpenOptions::new();options.read(true); + #[cfg(target_os="macos")] {use std::os::unix::fs::OpenOptionsExt;options.custom_flags(0x100);} + #[cfg(target_os="linux")] {use std::os::unix::fs::OpenOptionsExt;options.custom_flags(0x20000);} + #[cfg(not(any(target_os="macos",target_os="linux")))] {return Err("temporal_artifact_nofollow_unsupported".into());} + let file=options.open(path).map_err(|_|"temporal_artifact_unavailable")?;metadata(&file.metadata().map_err(|_|"temporal_artifact_unavailable")?,false)?; + io("before:file_fsync")?;file.sync_all().map_err(|_|"temporal_artifact_sync_failed")?; + let parent=path.parent().ok_or("temporal_artifact_directory_unavailable")?;directory(parent,false)?; + io("before:directory_fsync")?;fs::File::open(parent).and_then(|f|f.sync_all()).map_err(|_|"temporal_artifact_sync_failed")?; + if bounded_read(path,limit)?!=expected{return Err("temporal_artifact_immutable_mismatch".into());}Ok(()) +} + impl ArtifactStore { pub(super) fn temporal_directory( &self, diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs index c8e0801a4c..aa70bc120a 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs @@ -479,6 +479,7 @@ impl TemporalCaptureStage { } pub(crate) struct TemporalTrainCapture { workflow: TemporalWorkflow, + public_generation_id: String, // The exact immutable contract decoded from this workflow's validated // binding. Packet handling must not reparse its full JSON on every tick. schedule: ManagedScheduleContract, @@ -513,9 +514,13 @@ impl TemporalTrainCapture { // Fail unsupported full-run profiles before creating a writer/genesis. full_run::ContinuationPolicy::build(&ManagedBinding::build(&schedule,config.clone())?)?; let owned_store=store.clone(); + // Reuse authenticated retired-head chaining. An active predecessor is + // refused; this never upgrades/replaces an existing capture in place. + let predecessor=store.fresh_train_genesis_predecessor(now)?; Self::prepare_with(schedule,run,config,now,&store,move|w,store|{ let binding=&w.session.managed.as_ref().ok_or("managed_session_required")?.binding; w.full_run=Some(full_run::FullRunState::fresh(binding,&w.session.run_id,witness)?); + w.commit_token=predecessor; let writer=std::sync::Arc::new(durability::Writer::new(owned_store,now)?); w.async_state=Some(durability::AsyncState::new(writer)); w.persist_initial(store) @@ -575,7 +580,7 @@ impl TemporalTrainCapture { workflow.async_state=Some(durability::AsyncState::new(writer)); let error=workflow.persist_initial(&store).err(); if error.is_some(){workflow.stage=Stage::BlockedDurability;} - Ok((Self{workflow,schedule,failure:None,durable_seals:0},error)) + Ok((Self{public_generation_id:public_generation_id(&workflow)?,workflow,schedule,failure:None,durable_seals:0},error)) } pub(crate) fn async_start(&mut self,now:u64)->Result{ self.ensure_live()?; @@ -634,6 +639,7 @@ impl TemporalTrainCapture { } Ok(( Self { + public_generation_id:public_generation_id(&workflow)?, workflow, schedule, failure: None, @@ -1037,6 +1043,7 @@ impl TemporalTrainCapture { let n = b.seals.len(); let schedule = w.schedule()?; Ok(Self { + public_generation_id:public_generation_id(&w)?, workflow: w, schedule, failure: None, @@ -1049,7 +1056,7 @@ impl TemporalTrainCapture { )->Result{ let w=full_run::recover_current(store,artifact,context,now,epoch,anchor,witness)?; let n=w.bridge.as_ref().ok_or("temporal_bridge_checkpoint_required")?.seals.len();let schedule=w.schedule()?; - Ok(Self{workflow:w,schedule,failure:None,durable_seals:n}) + Ok(Self{public_generation_id:public_generation_id(&w)?,workflow:w,schedule,failure:None,durable_seals:n}) } pub(crate) fn repair_pending(&mut self, store: &RecoveryStore, now: u64) -> Result<(), String> { if self.workflow.async_state.is_some(){return self.workflow.request_async_repair(now);} @@ -1108,3 +1115,72 @@ impl Drop for TemporalTrainCapture { #[cfg(test)] #[path="temporal_budget_profile_tests.rs"] mod budget_profile_tests; + +impl TemporalTrainCapture { + pub(crate) fn prepare_evaluation_source(&self,now:u64,held_out:bool)->Result{ + self.ensure_live()?;self.workflow.prepare_temporal_evidence_source(now,held_out,&self.schedule) + } + pub(crate) fn evaluation_binding_matches(&self,binding:&full_run::TemporalEvaluationBinding)->bool{ + self.failure.is_none()&&self.workflow.current_temporal_evidence_matches(binding) + } +} + +// Informational public generation binding only, never an owner/source permit. +// Computed once by off-lock construction/recovery, not on packet ingress. +fn public_generation_id(w:&TemporalWorkflow)->Result{hash(&("ruview.room-learner.capture-generation.v2",&w.descriptor.id,&w.session.run_id,w.context().digest()))} +impl TemporalTrainCapture { + pub(crate) fn public_generation_id(&self)->&str{&self.public_generation_id} + pub(crate) fn public_phase_matches(&self,generation:&str,schedule:&str,episode:usize)->bool{self.public_generation_id==generation&&self.schedule.sha256()==schedule&&self.workflow.episode==episode} + pub(crate) fn public_profile(&self)->crate::room_learner::temporal_wire::TemporalProfileWire{ + use crate::room_learner::temporal_wire::*; + TemporalProfileWire{profile_id:"native-2s-three-source-56bin-option-a-engineering-v1".into(),window_unit:"native_receiver_2s_window".into(),native_window_ms:2000,train_windows:64,held_out_windows:64,train_episodes:7,held_out_episodes:8,windows_per_condition:[16;4],train_episodes_per_condition:[1,2,2,2],held_out_episodes_per_condition:[2;4],episodes:self.schedule.episodes().iter().map(|e|TemporalEpisodeWire{index:e.index as u8,role:e.role,label:e.condition,target_ms:e.target_active_recording_ms,lead_in_ms:e.lead_in_ms,native_windows:e.slots.len()as u16}).collect()} + } + pub(crate) fn public_capture(&self,epoch:&str,lead_until:Option,now:u64)->crate::room_learner::temporal_wire::TemporalCaptureWire{ + use crate::room_learner::temporal_wire::*; + let w=&self.workflow;let episode=self.schedule.episodes().get(w.episode);let lead=lead_until.map_or(0,|end|end.saturating_sub(now)/1_000_000); + let challenge=if matches!(self.stage(),TemporalCaptureStage::NeedsConfirmation|TemporalCaptureStage::Paused){self.confirmation_challenge().ok()}else{None}; + TemporalCaptureWire{capture_generation_id:self.public_generation_id.clone(),schedule_sha256:self.schedule.sha256().into(),continuation_policy_sha256:w.full_run.as_ref().map(|m|m.policy.sha256().into()).unwrap_or_default(),durable_commitment_sha256:w.commit_token.as_ref().map(|t|t.data().commitment),original_deadline:TemporalOriginalDeadlineWire{created_at_unix_ms:w.descriptor.created_at_ms,expires_at_unix_ms:w.descriptor.expires_at_ms},receiver_epoch:epoch.into(),stage:if lead>0{"preparing".into()}else{self.stage().as_str().into()},current_episode:episode.map(|e|e.index as u8),role:episode.map(|e|e.role),label:episode.map(|e|e.condition),target_ms:episode.map(|e|e.target_active_recording_ms),lead_in_remaining_ms:lead,observed_ms:w.live_observed_ns/1_000_000,retained_windows:w.session.rows.len()as u16,durable_windows:w.durable_windows as u16,sealed_train_episodes:self.durable_seals as u8,sealed_held_out_episodes:w.async_state.as_ref().map_or(0,|a|a.accepted_held_out_seals as u8),durability_pending:w.async_state.as_ref().is_some_and(|a|a.pending.is_some()),awaiting_ground_truth:challenge.is_some(),confirmation_challenge:challenge,failure:self.failure.clone()} + } + pub(crate) fn public_candidate(&self)->Option{ + use crate::room_learner::temporal_wire::*; + let mode=self.workflow.full_run.as_ref()?;let record=mode.public_verified_fit()?; + Some(TemporalCandidateIdentityWire{model_kind:WholeTemporalKind::ManagedTemporalV3,model_sha256:record.body.artifact_sha256.clone(),verified_fit_record_sha256:record.sha256.clone(),context_digest:record.body.context_digest.clone(),room_digest:self.context().room_digest.clone(),persisted:true}) + } + pub(crate) fn public_complete_refs(&self,proof_sha:&str)->Option{ + use crate::room_learner::temporal_wire::*; + let mode=self.workflow.full_run.as_ref()?;let record=mode.public_complete(self.workflow.commit_token.as_ref())?;let fit=mode.public_verified_fit()?; + Some(TemporalCompleteRefsWire{model:TemporalModelIdentityWire{model_kind:WholeTemporalKind::ManagedTemporalV3,model_sha256:fit.body.artifact_sha256.clone(),verified_fit_record_sha256:fit.sha256.clone()},training_proof_sha256:proof_sha.into(),complete_record_sha256:record.sha256.clone(),complete_commitment_sha256:self.workflow.commit_token.as_ref()?.data().commitment}) + } +} + +impl TemporalTrainCapture { + /// Authenticated current recovery is decoded only to discover its context. + /// This DATA does not mint a continued/Frozen witness; restore does that. + pub(crate) fn saved_full_context(store:&RecoveryStore,now:u64)->Result<(CaptureContext,String),String>{ + let loaded=store.load_temporal(now)?.ok_or("recovery_missing")?; + let prefix=loaded.prefix.ok_or("temporal_retired")?; + if prefix.metadata_kind()!=full_run::META_KIND{return Err("temporal_full_run_no_upconversion".into());} + let w=checkpoint::decode(prefix.bytes(),prefix.descriptor(),now)?; + Ok((w.context().clone(),public_generation_id(&w)?)) + } + pub(crate) fn retirement_generation(store:&RecoveryStore,now:u64)->Result{ + let inspection=store.inspect_temporal_retirement(now)?.ok_or("recovery_missing")?; + if inspection.metadata_kind()!=full_run::META_KIND{return Err("temporal_full_run_no_upconversion".into());} + hash(&("ruview.room-learner.capture-generation.v2",&inspection.descriptor().id,inspection.run(),&inspection.descriptor().context_digest)) + } +} + +impl TemporalTrainCapture {pub(crate) fn current_source_floors(&self)->(Vec<(u8,u64,u64)>,u64){(self.workflow.source_high.clone(),self.workflow.last_now_ns)}} + +impl TemporalTrainCapture { + pub(crate) fn recovery_discovery(store:&RecoveryStore,now:u64)->Option{ + use crate::room_learner::temporal_wire::*; + let result=(||->Result,String>{ + let Some(inspection)=store.inspect_temporal_read_only(now)?else{return Ok(None)}; + if inspection.metadata_kind()!=full_run::META_KIND||inspection.retired(){return Ok(None)} + let descriptor=inspection.descriptor();let generation=hash(&("ruview.room-learner.capture-generation.v2",&descriptor.id,inspection.run(),&descriptor.context_digest))?; + Ok(Some(TemporalRecoveryStatusWire{schema:RECOVERY_STATUS_SCHEMA.into(),state:if inspection.expired(){"expired"}else{"available"}.into(),capture_generation_id:Some(generation),context_digest:Some(descriptor.context_digest.clone()),original_deadline:Some(TemporalOriginalDeadlineWire{created_at_unix_ms:descriptor.created_at_ms,expires_at_unix_ms:descriptor.expires_at_ms})})) + })(); + result.unwrap_or_else(|_|Some(TemporalRecoveryStatusWire{schema:RECOVERY_STATUS_SCHEMA.into(),state:"unavailable".into(),capture_generation_id:None,context_digest:None,original_deadline:None})) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs index 6e83cdd55d..a91e8e0420 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs @@ -920,3 +920,7 @@ pub(crate) fn c4b_reference_batch_and_stale_seal_probe(bytes:&[u8],descriptor:&R #[cfg(test)] pub(crate) fn c4b_complete_fault_matrix(bytes:&[u8],descriptor:&RecoveryDescriptor,now:u64)->Result{full_checkpoint::c4b_complete_fault_matrix(bytes,descriptor,now)} + +// Reuse the existing whole-prefix Train validator for C4C; this DATA check +// cannot construct a snapshot or owner/write/evaluation capability. +pub(super) fn validate_temporal_evaluation_train_prefix(w:&TemporalWorkflow,record:&full_run::VerifiedFitRecordData)->Result<(),String>{full_checkpoint::validate_immutable_train(w,record)} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation.rs new file mode 100644 index 0000000000..d56e4a93be --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation.rs @@ -0,0 +1,571 @@ +//! Whole Temporal evaluation DATA. No decoded producer or runtime/store permit, +//! model accessor, evaluator invocation or qualification/adoption authority. +//! Successful decoding validates only bounded metadata and its integrity links. +use super::{ + observation::{CaptureProvenance, ClockKind, DataRole}, + training::EvaluationReport, +}; +use serde::{de::DeserializeOwned, Deserialize, Serialize}; +use sha2::{Digest, Sha256}; + +pub(super) const TRAIN_SCHEMA: &str = "ruview.room-learner.temporal-training-proof.v1"; +pub(super) const HELD_OUT_SCHEMA: &str = "ruview.room-learner.temporal-heldout-evidence.v1"; +pub(super) const EVALUATION_SCHEMA: &str = "ruview.room-learner.temporal-evaluation.v1"; +const MODEL_KIND: &str = "ruview.room-learner.managed-temporal-input.v3"; +const PREFIX_SCHEMA: &str = "ruview.room-learner.temporal-runtime-prefix.v3"; +const METADATA_KIND: &str = "temporal-prefix-commitment-v2"; +pub(super) const MAX_METADATA_BYTES: usize = 64 * 1024; +const TTL_MS: u64 = 86_400_000; +fn invalid() -> Result { + Err("temporal_evaluation_data_invalid".into()) +} +fn sha(value: &str) -> bool { + value.len() == 64 + && value + .bytes() + .all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} +fn identifier(value: &str) -> bool { + !value.is_empty() + && value.len() <= 256 + && value.is_ascii() + && value.trim() == value + && !value.bytes().any(|b| b.is_ascii_control()) +} +fn decimal(value: &str) -> Result { + if value.is_empty() + || value.len() > 20 + || !value.bytes().all(|b| b.is_ascii_digit()) + || (value.starts_with('0') && value != "0") + { + return invalid(); + } + value + .parse() + .map_err(|_| "temporal_evaluation_data_integer".into()) +} +fn decimal_frame(value: &str) -> Result { + if value.is_empty() + || value.len() > 39 + || !value.bytes().all(|b| b.is_ascii_digit()) + || (value.starts_with('0') && value != "0") + { + return invalid(); + } + value + .parse() + .map_err(|_| "temporal_evaluation_data_frame_integer".into()) +} +fn data_sha(schema: &str, body: &T) -> Result { + let bytes = + serde_json::to_vec(&(schema, body)).map_err(|_| "temporal_evaluation_data_encoding")?; + Ok(format!("{:x}", Sha256::digest(bytes))) +} +fn decode_data(bytes: &[u8]) -> Result { + if bytes.is_empty() || bytes.len() > MAX_METADATA_BYTES { + return invalid(); + } + let value: T = serde_json::from_slice(bytes).map_err(|_| "temporal_evaluation_data_schema")?; + if serde_json::to_vec(&value).map_err(|_| "temporal_evaluation_data_encoding")? != bytes { + return invalid(); + } + Ok(value) +} + +/// Stable original identity ONLY; intentionally excludes mutable saved_at_ms. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct OriginalDeadlineData { + pub id: String, + pub created_at_unix_ms: u64, + pub expires_at_unix_ms: u64, +} +impl OriginalDeadlineData { + fn validate_data(&self) -> Result<(), String> { + if !identifier(&self.id) + || self.created_at_unix_ms == 0 + || self.expires_at_unix_ms > 253_402_300_799_999 + || self.created_at_unix_ms.checked_add(TTL_MS) != Some(self.expires_at_unix_ms) + { + return invalid(); + } + Ok(()) + } + fn contains_receiver_ns(&self, ns: u64) -> bool { + ns > 0 + && ns / 1_000_000 >= self.created_at_unix_ms + && ns / 1_000_000 < self.expires_at_unix_ms + } +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct TemporalEvaluationIdentityData { + pub owner_scope: String, + pub installation_id: String, + pub context_digest: String, + pub room_digest: String, + pub run: String, + pub original_deadline: OriginalDeadlineData, + pub artifact_kind: String, + pub artifact_sha256: String, + pub verified_fit_record_sha256: String, + pub prefix_schema: String, + pub metadata_kind: String, + pub continuation_policy_sha256: String, + pub input_policy_sha256: String, + pub execution_policy_sha256: String, + pub schedule_sha256: String, + pub configured_fit_sha256: String, + pub source_ids: [u8; 3], +} +impl TemporalEvaluationIdentityData { + fn validate_data(&self) -> Result<(), String> { + self.original_deadline.validate_data()?; + if [&self.owner_scope, &self.installation_id, &self.run] + .iter() + .any(|s| !identifier(s)) + || [ + &self.context_digest, + &self.room_digest, + &self.artifact_sha256, + &self.verified_fit_record_sha256, + &self.continuation_policy_sha256, + &self.input_policy_sha256, + &self.execution_policy_sha256, + &self.schedule_sha256, + &self.configured_fit_sha256, + ] + .iter() + .any(|s| !sha(s)) + || self.artifact_kind != MODEL_KIND + || self.prefix_schema != PREFIX_SCHEMA + || self.metadata_kind != METADATA_KIND + || self.source_ids.windows(2).any(|w| w[0] >= w[1]) + { + return invalid(); + } + Ok(()) + } +} +/// Projection, never a deserializable TemporalToken/storage permission. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct CommitmentData { + pub generation: u64, + pub revision: u16, + pub run: String, + pub id: String, + pub commitment_sha256: String, +} +impl CommitmentData { + fn validate_data(&self, identity: &TemporalEvaluationIdentityData) -> Result<(), String> { + if self.revision >= 433 + || self.run != identity.run + || self.id != identity.original_deadline.id + || !sha(&self.commitment_sha256) + { + return invalid(); + } + Ok(()) + } +} +/// Negative descriptive claims only; no field can grant authority. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct DiagnosticClaimsData { + pub independence_verified: bool, + pub physically_validated: bool, + pub qualification_granted: bool, + pub adoption_authorized: bool, + pub measurement_clock_available: bool, +} +impl DiagnosticClaimsData { + fn validate_data(&self) -> Result<(), String> { + if self.independence_verified + || self.physically_validated + || self.qualification_granted + || self.adoption_authorized + || self.measurement_clock_available + { + return invalid(); + } + Ok(()) + } +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct FrozenSourceFloorData { + pub source_id: u8, + pub sequence: u64, + pub receiver_ns: String, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct TemporalTrainProofBodyData { + pub identity: TemporalEvaluationIdentityData, + pub training_data_sha256: String, + pub training_evidence_sha256: String, + pub training_identities_sha256: String, + pub train_seal_sha256s: [String; 7], + pub train_seal_chain_sha256: String, + pub train_windows: u16, + pub windows_per_condition: [u16; 4], + pub declared_episodes_per_condition: [u16; 4], + pub accepted_train_commitment: CommitmentData, + pub freeze_prepare_receiver_ns: String, + pub source_floors: [FrozenSourceFloorData; 3], + pub claims: DiagnosticClaimsData, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct TemporalTrainProofData { + pub schema: String, + pub body: TemporalTrainProofBodyData, + pub sha256: String, +} +impl TemporalTrainProofData { + pub(super) fn decode_data(bytes: &[u8]) -> Result { + let value: Self = decode_data(bytes)?; + value.validate_data()?; + Ok(value) + } + pub(super) fn validate_data(&self) -> Result<(), String> { + let b = &self.body; + b.identity.validate_data()?; + b.claims.validate_data()?; + b.accepted_train_commitment.validate_data(&b.identity)?; + let cut = decimal(&b.freeze_prepare_receiver_ns)?; + if self.schema != TRAIN_SCHEMA + || !sha(&self.sha256) + || self.sha256 != data_sha(TRAIN_SCHEMA, b)? + || [ + &b.training_data_sha256, + &b.training_evidence_sha256, + &b.training_identities_sha256, + &b.train_seal_chain_sha256, + ] + .iter() + .any(|s| !sha(s)) + || b.train_seal_sha256s.iter().any(|s| !sha(s)) + || b.train_seal_sha256s + .iter() + .collect::>() + .len() + != 7 + || b.train_windows != 64 + || b.windows_per_condition != [16; 4] + || b.declared_episodes_per_condition != [1, 2, 2, 2] + || !b.identity.original_deadline.contains_receiver_ns(cut) + { + return invalid(); + } + for (floor, id) in b.source_floors.iter().zip(b.identity.source_ids) { + let ns = decimal(&floor.receiver_ns)?; + if floor.source_id != id || floor.sequence > u32::MAX as u64 || ns == 0 || ns > cut { + return invalid(); + } + } + Ok(()) + } +} +/// DATA from the acknowledged Complete prefix's whole-period source_high. +/// This is not a future ACK/RAM floor, hardware epoch or live continuation permit. +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct FinalSourceFenceData { + pub source_id: u8, + pub sequence: u64, + pub receiver_ns: String, + pub native_frame_id: String, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct TemporalHeldOutBodyData { + pub identity: TemporalEvaluationIdentityData, + pub training_proof_sha256: String, + pub held_out_data_sha256: String, + pub held_out_evidence_sha256: String, + pub held_out_identities_sha256: String, + pub complete_rows_sha256: String, + pub complete_evidence_sha256: String, + pub complete_identities_sha256: String, + pub complete_ledger_sha256: String, + pub held_out_seal_sha256s: [String; 8], + pub held_out_seal_chain_sha256: String, + pub complete_record_sha256: String, + pub complete_commitment: CommitmentData, + pub complete_prepare_receiver_ns: String, + pub complete_persisted_receiver_ns: String, + pub final_source_fences: [FinalSourceFenceData; 3], + pub held_out_windows: u16, + pub windows_per_condition: [u16; 4], + pub declared_episodes_per_condition: [u16; 4], + pub claims: DiagnosticClaimsData, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub(super) struct TemporalHeldOutEvidenceData { + pub schema: String, + pub body: TemporalHeldOutBodyData, + pub sha256: String, +} +impl TemporalHeldOutEvidenceData { + pub(super) fn decode_data(bytes: &[u8]) -> Result { + let value: Self = decode_data(bytes)?; + value.validate_data()?; + Ok(value) + } + pub(super) fn validate_data(&self) -> Result<(), String> { + let b = &self.body; + b.identity.validate_data()?; + b.claims.validate_data()?; + b.complete_commitment.validate_data(&b.identity)?; + if self.schema != HELD_OUT_SCHEMA + || !sha(&self.sha256) + || self.sha256 != data_sha(HELD_OUT_SCHEMA, b)? + || [ + &b.training_proof_sha256, + &b.held_out_data_sha256, + &b.held_out_evidence_sha256, + &b.held_out_identities_sha256, + &b.complete_rows_sha256, + &b.complete_evidence_sha256, + &b.complete_identities_sha256, + &b.complete_ledger_sha256, + &b.held_out_seal_chain_sha256, + &b.complete_record_sha256, + ] + .iter() + .any(|s| !sha(s)) + || b.held_out_seal_sha256s.iter().any(|s| !sha(s)) + || b.held_out_seal_sha256s + .iter() + .collect::>() + .len() + != 8 + || b.held_out_windows != 64 + || b.windows_per_condition != [16; 4] + || b.declared_episodes_per_condition != [2; 4] + || !b + .identity + .original_deadline + .contains_receiver_ns(decimal(&b.complete_prepare_receiver_ns)?) + { + return invalid(); + } + let persisted = decimal(&b.complete_persisted_receiver_ns)?; + if !b.identity.original_deadline.contains_receiver_ns(persisted) + || persisted != decimal(&b.complete_prepare_receiver_ns)? + { + return invalid(); + } + for (floor, id) in b.final_source_fences.iter().zip(b.identity.source_ids) { + let ns = decimal(&floor.receiver_ns)?; + let frame = decimal_frame(&floor.native_frame_id)?; + let expected = + (u128::from(ns) << 40) | (u128::from(id) << 32) | u128::from(floor.sequence); + if floor.source_id != id + || floor.sequence > u32::MAX as u64 + || ns == 0 + || ns > persisted + || frame != expected + { + return invalid(); + } + } + Ok(()) + } + /// Metadata equality/integrity only, NOT a complete capture or evaluation permit. + pub(super) fn validate_data_links(&self, train: &TemporalTrainProofData) -> Result<(), String> { + self.validate_data()?; + train.validate_data()?; + if self.body.identity != train.body.identity + || self.body.training_proof_sha256 != train.sha256 + || decimal(&self.body.complete_prepare_receiver_ns)? + < decimal(&train.body.freeze_prepare_receiver_ns)? + || self.body.complete_commitment.generation + != train.body.accepted_train_commitment.generation + || self.body.complete_commitment.revision + <= train.body.accepted_train_commitment.revision + { + return invalid(); + } + for (final_floor, train_floor) in self + .body + .final_source_fences + .iter() + .zip(&train.body.source_floors) + { + if final_floor.source_id != train_floor.source_id + || final_floor.sequence < train_floor.sequence + || decimal(&final_floor.receiver_ns)? < decimal(&train_floor.receiver_ns)? + { + return invalid(); + } + } + Ok(()) + } +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(super) struct TemporalEvaluationBodyData { + pub identity: TemporalEvaluationIdentityData, + pub training_proof_sha256: String, + pub held_out_evidence_sha256: String, + pub complete_record_sha256: String, + pub training_data_sha256: String, + pub held_out_data_sha256: String, + pub role: DataRole, + pub window_unit: String, + pub evaluated_at_receiver_ns: String, + pub evaluated_at_unix_ms: u64, + pub provenance: Vec, + pub clock_claims: Vec, + pub train_lineage_checked: bool, + pub report: EvaluationReport, + pub claims: DiagnosticClaimsData, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(super) struct TemporalEvaluationReceiptData { + pub schema: String, + pub body: TemporalEvaluationBodyData, + pub sha256: String, +} +impl TemporalEvaluationReceiptData { + pub(super) fn decode_data(bytes: &[u8]) -> Result { + let value: Self = decode_data(bytes)?; + value.validate_data()?; + Ok(value) + } + pub(super) fn validate_data(&self) -> Result<(), String> { + let b = &self.body; + b.identity.validate_data()?; + b.claims.validate_data()?; + b.report.validate()?; + let ns = decimal(&b.evaluated_at_receiver_ns)?; + if self.schema != EVALUATION_SCHEMA + || !sha(&self.sha256) + || self.sha256 != data_sha(EVALUATION_SCHEMA, b)? + || [ + &b.training_proof_sha256, + &b.held_out_evidence_sha256, + &b.complete_record_sha256, + &b.training_data_sha256, + &b.held_out_data_sha256, + ] + .iter() + .any(|s| !sha(s)) + || b.role != DataRole::HeldOut + || b.window_unit != "native_receiver_2s_window" + || !b.train_lineage_checked + || ns / 1_000_000 != b.evaluated_at_unix_ms + || !b.identity.original_deadline.contains_receiver_ns(ns) + || b.provenance != [CaptureProvenance::LiveReceiverObserved] + || b.clock_claims != [ClockKind::ReceiverOnly] + || b.report.samples != 64 + || b.report.declared_trial_count != 8 + || b.report + .per_condition + .iter() + .any(|c| c.samples != 16 || c.declared_trial_count != 2) + { + return invalid(); + } + Ok(()) + } + pub(super) fn validate_data_links( + &self, + train: &TemporalTrainProofData, + held: &TemporalHeldOutEvidenceData, + ) -> Result<(), String> { + self.validate_data()?; + held.validate_data_links(train)?; + let b = &self.body; + if b.identity != held.body.identity + || b.training_proof_sha256 != train.sha256 + || b.held_out_evidence_sha256 != held.sha256 + || b.complete_record_sha256 != held.body.complete_record_sha256 + || b.training_data_sha256 != train.body.training_data_sha256 + || b.held_out_data_sha256 != held.body.held_out_data_sha256 + || decimal(&b.evaluated_at_receiver_ns)? + < decimal(&held.body.complete_persisted_receiver_ns)? + { + return invalid(); + } + Ok(()) + } +} + +// These constructors compute bounded DATA integrity only. They do not issue a +// producer, current-store, evaluation, qualification or selection permission. +impl TemporalTrainProofData { + pub(super) fn from_body_data(body: TemporalTrainProofBodyData) -> Result { + let value = Self { + schema: TRAIN_SCHEMA.into(), + sha256: data_sha(TRAIN_SCHEMA, &body)?, + body, + }; + value.validate_data()?; + value.encode_data()?; + Ok(value) + } + pub(super) fn encode_data(&self) -> Result, String> { + self.validate_data()?; + bounded_encoding(self) + } +} +impl TemporalHeldOutEvidenceData { + pub(super) fn from_body_data(body: TemporalHeldOutBodyData) -> Result { + let value = Self { + schema: HELD_OUT_SCHEMA.into(), + sha256: data_sha(HELD_OUT_SCHEMA, &body)?, + body, + }; + value.validate_data()?; + value.encode_data()?; + Ok(value) + } + pub(super) fn encode_data(&self) -> Result, String> { + self.validate_data()?; + bounded_encoding(self) + } +} +impl TemporalEvaluationReceiptData { + pub(super) fn from_body_data(body: TemporalEvaluationBodyData) -> Result { + let value = Self { + schema: EVALUATION_SCHEMA.into(), + sha256: data_sha(EVALUATION_SCHEMA, &body)?, + body, + }; + value.validate_data()?; + value.encode_data()?; + Ok(value) + } + pub(super) fn encode_data(&self) -> Result, String> { + self.validate_data()?; + bounded_encoding(self) + } +} +impl DiagnosticClaimsData { + pub(super) fn diagnostic_only() -> Self { + Self { + independence_verified: false, + physically_validated: false, + qualification_granted: false, + adoption_authorized: false, + measurement_clock_available: false, + } + } +} +fn bounded_encoding(value: &T) -> Result, String> { + let bytes = serde_json::to_vec(value).map_err(|_| "temporal_evaluation_data_encoding")?; + if bytes.is_empty() || bytes.len() > MAX_METADATA_BYTES { + return invalid(); + } + Ok(bytes) +} + +#[cfg(test)] +#[path = "temporal_evaluation_data_tests.rs"] +mod tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_data_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_data_tests.rs new file mode 100644 index 0000000000..20d9e084f5 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_data_tests.rs @@ -0,0 +1,439 @@ +//! Synthetic metadata fixtures only. None creates a runtime/fit/Complete permit. +use super::*; +use serde_json::{json, Value}; +fn fixture_sha(n: u64) -> String { + format!("{n:064x}") +} +fn claims() -> DiagnosticClaimsData { + DiagnosticClaimsData { + independence_verified: false, + physically_validated: false, + qualification_granted: false, + adoption_authorized: false, + measurement_clock_available: false, + } +} +fn identity() -> TemporalEvaluationIdentityData { + TemporalEvaluationIdentityData { + owner_scope: "owner-a".into(), + installation_id: "installation-a".into(), + context_digest: fixture_sha(1), + room_digest: fixture_sha(2), + run: "run-a".into(), + original_deadline: OriginalDeadlineData { + id: "recovery-a".into(), + created_at_unix_ms: 1_800_000_000_000, + expires_at_unix_ms: 1_800_086_400_000, + }, + artifact_kind: MODEL_KIND.into(), + artifact_sha256: fixture_sha(3), + verified_fit_record_sha256: fixture_sha(4), + prefix_schema: PREFIX_SCHEMA.into(), + metadata_kind: METADATA_KIND.into(), + continuation_policy_sha256: fixture_sha(5), + input_policy_sha256: fixture_sha(6), + execution_policy_sha256: fixture_sha(7), + schedule_sha256: fixture_sha(8), + configured_fit_sha256: fixture_sha(9), + source_ids: [15, 16, 17], + } +} +fn token(id: &TemporalEvaluationIdentityData, revision: u16) -> CommitmentData { + CommitmentData { + generation: 0, + revision, + run: id.run.clone(), + id: id.original_deadline.id.clone(), + commitment_sha256: fixture_sha(u64::from(revision) + 100), + } +} +fn train() -> TemporalTrainProofData { + let id = identity(); + let cut = "1800000900000000000".to_owned(); + let body = TemporalTrainProofBodyData { + accepted_train_commitment: token(&id, 78), + source_floors: std::array::from_fn(|i| FrozenSourceFloorData { + source_id: id.source_ids[i], + sequence: 10_000, + receiver_ns: "1800000899999000000".into(), + }), + identity: id, + training_data_sha256: fixture_sha(20), + training_evidence_sha256: fixture_sha(21), + training_identities_sha256: fixture_sha(22), + train_seal_sha256s: std::array::from_fn(|i| fixture_sha(200 + i as u64)), + train_seal_chain_sha256: fixture_sha(23), + train_windows: 64, + windows_per_condition: [16; 4], + declared_episodes_per_condition: [1, 2, 2, 2], + freeze_prepare_receiver_ns: cut, + claims: claims(), + }; + TemporalTrainProofData { + schema: TRAIN_SCHEMA.into(), + sha256: data_sha(TRAIN_SCHEMA, &body).unwrap(), + body, + } +} +fn held(t: &TemporalTrainProofData) -> TemporalHeldOutEvidenceData { + let body = TemporalHeldOutBodyData { + identity: t.body.identity.clone(), + training_proof_sha256: t.sha256.clone(), + held_out_data_sha256: fixture_sha(30), + held_out_evidence_sha256: fixture_sha(31), + held_out_identities_sha256: fixture_sha(32), + complete_rows_sha256: fixture_sha(33), + complete_evidence_sha256: fixture_sha(34), + complete_identities_sha256: fixture_sha(35), + complete_ledger_sha256: fixture_sha(36), + held_out_seal_sha256s: std::array::from_fn(|i| fixture_sha(300 + i as u64)), + held_out_seal_chain_sha256: fixture_sha(37), + complete_record_sha256: fixture_sha(38), + complete_commitment: token(&t.body.identity, 160), + complete_prepare_receiver_ns: "1800001300000000000".into(), + complete_persisted_receiver_ns: "1800001300000000000".into(), + final_source_fences: std::array::from_fn(|i| { + let id = t.body.identity.source_ids[i]; + let ns = 1_800_001_299_999_000_000u64; + FinalSourceFenceData { + source_id: id, + sequence: 20_000, + receiver_ns: ns.to_string(), + native_frame_id: ((u128::from(ns) << 40) | (u128::from(id) << 32) | 20_000) + .to_string(), + } + }), + held_out_windows: 64, + windows_per_condition: [16; 4], + declared_episodes_per_condition: [2; 4], + claims: claims(), + }; + TemporalHeldOutEvidenceData { + schema: HELD_OUT_SCHEMA.into(), + sha256: data_sha(HELD_OUT_SCHEMA, &body).unwrap(), + body, + } +} +fn rates(sensitivity: Option, specificity: Option) -> Value { + json!({"evaluated_cohort_coverage":1.0,"unconditional_accuracy":1.0,"scored_accuracy":1.0,"unconditional_sensitivity":sensitivity,"scored_sensitivity":sensitivity,"unconditional_specificity":specificity,"scored_specificity":specificity}) +} +fn confusion(empty: usize, occupied: usize) -> Value { + json!({"true_empty":empty,"false_occupied":0,"false_empty":0,"true_occupied":occupied,"abstained_empty":0,"abstained_occupied":0,"missing_empty":0,"missing_occupied":0}) +} +fn report() -> EvaluationReport { + let labels = ["Empty", "OccupiedStatic", "OccupiedMoving", "AdjacentOnly"]; + let per:Vec=labels.into_iter().map(|label|{let occupied=label.starts_with("Occupied");json!({"condition":label,"samples":16,"correct":16,"abstained":0,"scored":16,"missing":0,"binary_cross_entropy":0.1,"binary_cross_entropy_samples":16,"declared_trial_count":2,"declared_session_count":2,"confusion":confusion(if occupied{0}else{16},if occupied{16}else{0}),"rates":rates(occupied.then_some(1.0),(!occupied).then_some(1.0))})}).collect(); + serde_json::from_value(json!({"samples":64,"correct":64,"abstained":0,"binary_cross_entropy":0.1,"binary_cross_entropy_samples":64,"scored":64,"missing":0,"declared_trial_count":8,"declared_session_count":8,"confusion":confusion(32,32),"rates":rates(Some(1.0),Some(1.0)),"per_condition":per,"independence_verified":false})).unwrap() +} +fn evaluation( + t: &TemporalTrainProofData, + h: &TemporalHeldOutEvidenceData, +) -> TemporalEvaluationReceiptData { + let body = TemporalEvaluationBodyData { + identity: t.body.identity.clone(), + training_proof_sha256: t.sha256.clone(), + held_out_evidence_sha256: h.sha256.clone(), + complete_record_sha256: h.body.complete_record_sha256.clone(), + training_data_sha256: t.body.training_data_sha256.clone(), + held_out_data_sha256: h.body.held_out_data_sha256.clone(), + role: DataRole::HeldOut, + window_unit: "native_receiver_2s_window".into(), + evaluated_at_receiver_ns: "1800001301000000000".into(), + evaluated_at_unix_ms: 1_800_001_301_000, + provenance: vec![CaptureProvenance::LiveReceiverObserved], + clock_claims: vec![ClockKind::ReceiverOnly], + train_lineage_checked: true, + report: report(), + claims: claims(), + }; + TemporalEvaluationReceiptData { + schema: EVALUATION_SCHEMA.into(), + sha256: data_sha(EVALUATION_SCHEMA, &body).unwrap(), + body, + } +} +fn rehash_train(t: &mut TemporalTrainProofData) { + t.sha256 = data_sha(TRAIN_SCHEMA, &t.body).unwrap(); +} +fn rehash_held(h: &mut TemporalHeldOutEvidenceData) { + h.sha256 = data_sha(HELD_OUT_SCHEMA, &h.body).unwrap(); +} +fn rehash_eval(e: &mut TemporalEvaluationReceiptData) { + e.sha256 = data_sha(EVALUATION_SCHEMA, &e.body).unwrap(); +} +#[test] +fn three_canonical_data_records_roundtrip_without_authority() { + let t = train(); + let h = held(&t); + let e = evaluation(&t, &h); + let tb = serde_json::to_vec(&t).unwrap(); + let hb = serde_json::to_vec(&h).unwrap(); + let eb = serde_json::to_vec(&e).unwrap(); + assert_eq!(TemporalTrainProofData::decode_data(&tb).unwrap(), t); + assert_eq!(TemporalHeldOutEvidenceData::decode_data(&hb).unwrap(), h); + assert_eq!(TemporalEvaluationReceiptData::decode_data(&eb).unwrap(), e); + e.validate_data_links(&t, &h).unwrap(); + assert!(!e.body.claims.adoption_authorized); + assert!(!e.body.claims.qualification_granted); +} +#[test] +fn unknown_inner_identity_and_mutable_descriptor_fields_are_rejected() { + let t = train(); + let h = held(&t); + let e = evaluation(&t, &h); + for key in ["model_sha256", "model_version", "adoption_authorized"] { + let mut value = serde_json::to_value(&e).unwrap(); + value["body"]["report"][key] = json!("untrusted-inner-identity"); + assert!( + TemporalEvaluationReceiptData::decode_data(&serde_json::to_vec(&value).unwrap()) + .is_err() + ); + } + let mut value = serde_json::to_value(&t).unwrap(); + value["body"]["identity"]["original_deadline"]["saved_at_ms"] = json!(1_800_000_500_000u64); + assert!(TemporalTrainProofData::decode_data(&serde_json::to_vec(&value).unwrap()).is_err()); +} +#[test] +fn canonical_encoding_size_digest_and_domain_near_misses_fail() { + let t = train(); + let bytes = serde_json::to_vec(&t).unwrap(); + assert!(TemporalTrainProofData::decode_data(&[bytes.clone(), vec![b' ']].concat()).is_err()); + assert!(TemporalTrainProofData::decode_data(&vec![b' '; MAX_METADATA_BYTES + 1]).is_err()); + let mut bad = t.clone(); + bad.sha256 = fixture_sha(999); + assert!(bad.validate_data().is_err()); + bad = t.clone(); + bad.schema = HELD_OUT_SCHEMA.into(); + bad.sha256 = data_sha(&bad.schema, &bad.body).unwrap(); + assert!(bad.validate_data().is_err()); +} +#[test] +fn rehashed_positive_claims_roles_fake_cohort_counts_and_wrong_kind_fail() { + let t = train(); + let h = held(&t); + let e = evaluation(&t, &h); + for i in 0..5 { + let mut bad = t.clone(); + match i { + 0 => bad.body.claims.adoption_authorized = true, + 1 => bad.body.claims.qualification_granted = true, + 2 => bad.body.claims.independence_verified = true, + 3 => bad.body.claims.physically_validated = true, + _ => bad.body.claims.measurement_clock_available = true, + }; + rehash_train(&mut bad); + assert!(bad.validate_data().is_err()); + } + let mut bad = t.clone(); + bad.body.windows_per_condition = [4; 4]; + rehash_train(&mut bad); + assert!(bad.validate_data().is_err()); + bad = t.clone(); + bad.body.identity.artifact_kind = "ruview.room-model.v1".into(); + rehash_train(&mut bad); + assert!(bad.validate_data().is_err()); + let mut badh = h.clone(); + badh.body.declared_episodes_per_condition = [16; 4]; + rehash_held(&mut badh); + assert!(badh.validate_data().is_err()); + let mut bade = e.clone(); + bade.body.role = DataRole::Inference; + rehash_eval(&mut bade); + assert!(bade.validate_data().is_err()); + bade = e.clone(); + bade.body.clock_claims = vec![ClockKind::SourceMeasurement]; + rehash_eval(&mut bade); + assert!(bade.validate_data().is_err()); + bade = e.clone(); + bade.body.report.declared_trial_count = 64; + rehash_eval(&mut bade); + assert!(bade.validate_data().is_err()); +} +#[test] +fn deadline_clock_floor_and_canonical_integer_checks_do_not_rebase() { + let t = train(); + let h = held(&t); + let e = evaluation(&t, &h); + for cut in [ + "01800000900000000000", + "+1800000900000000000", + "18446744073709551616", + "1800086400000000000", + "1799999999999999999", + ] { + let mut bad = t.clone(); + bad.body.freeze_prepare_receiver_ns = cut.into(); + rehash_train(&mut bad); + assert!(bad.validate_data().is_err(), "{cut}"); + } + let mut bad = t.clone(); + bad.body.identity.original_deadline.expires_at_unix_ms += 1; + rehash_train(&mut bad); + assert!(bad.validate_data().is_err()); + bad = t.clone(); + bad.body.source_floors[0].receiver_ns = "1800000900000000001".into(); + rehash_train(&mut bad); + assert!(bad.validate_data().is_err()); + let mut bade = e.clone(); + bade.body.evaluated_at_unix_ms += 1; + rehash_eval(&mut bade); + assert!(bade.validate_data().is_err()); + let mut badh = h.clone(); + badh.body.complete_prepare_receiver_ns = "1800000800000000000".into(); + rehash_held(&mut badh); + assert!(badh.validate_data_links(&t).is_err()); +} +#[test] +fn rehashed_cross_record_owner_model_policy_dataset_and_revision_mismatches_fail() { + let t = train(); + let h = held(&t); + let e = evaluation(&t, &h); + for i in 0..5 { + let mut bad = h.clone(); + match i { + 0 => bad.body.identity.owner_scope = "owner-b".into(), + 1 => bad.body.identity.artifact_sha256 = fixture_sha(999), + 2 => bad.body.identity.continuation_policy_sha256 = fixture_sha(998), + 3 => bad.body.training_proof_sha256 = fixture_sha(997), + _ => bad.body.complete_commitment.revision = 1, + }; + rehash_held(&mut bad); + assert!(bad.validate_data_links(&t).is_err()); + } + let mut bad = e.clone(); + bad.body.held_out_data_sha256 = fixture_sha(999); + rehash_eval(&mut bad); + assert!(bad.validate_data_links(&t, &h).is_err()); + bad = e.clone(); + bad.body.complete_record_sha256 = fixture_sha(998); + rehash_eval(&mut bad); + assert!(bad.validate_data_links(&t, &h).is_err()); +} +#[test] +fn train_data_identity_does_not_follow_heldout_progress_or_evaluation_time() { + let t = train(); + let original = serde_json::to_vec(&t).unwrap(); + let mut h = held(&t); + let mut e = evaluation(&t, &h); + h.body.complete_commitment.revision += 1; + h.body.complete_record_sha256 = fixture_sha(777); + rehash_held(&mut h); + e.body.held_out_evidence_sha256 = h.sha256.clone(); + e.body.complete_record_sha256 = h.body.complete_record_sha256.clone(); + e.body.evaluated_at_unix_ms += 1; + e.body.evaluated_at_receiver_ns = "1800001301001000000".into(); + rehash_eval(&mut e); + e.validate_data_links(&t, &h).unwrap(); + assert_eq!(original, serde_json::to_vec(&t).unwrap()); +} +#[test] +fn bounded_max_width_metadata_fixture_fits_without_relaxing_reader_limit() { + let mut t = train(); + let id = &mut t.body.identity; + let escaped = "\"".repeat(256); + id.owner_scope = escaped.clone(); + id.installation_id = escaped.clone(); + id.run = escaped.clone(); + id.original_deadline.id = escaped.clone(); + id.source_ids = [253, 254, 255]; + id.original_deadline.expires_at_unix_ms = u64::MAX / 1_000_000 + 1; + id.original_deadline.created_at_unix_ms = id.original_deadline.expires_at_unix_ms - TTL_MS; + t.body.freeze_prepare_receiver_ns = (u64::MAX - 1_000_000_000_000).to_string(); + t.body.accepted_train_commitment.run = escaped.clone(); + t.body.accepted_train_commitment.id = escaped; + t.body.accepted_train_commitment.generation = u64::MAX; + t.body.accepted_train_commitment.revision = 430; + for (floor, id) in t + .body + .source_floors + .iter_mut() + .zip(t.body.identity.source_ids) + { + floor.source_id = id; + floor.sequence = u32::MAX as u64 - 100_000; + floor.receiver_ns = (u64::MAX - 1_001_000_000_000).to_string(); + } + rehash_train(&mut t); + let mut h = held(&t); + h.body.complete_commitment.generation = u64::MAX; + h.body.complete_commitment.revision = 432; + h.body.complete_prepare_receiver_ns = (u64::MAX - 1_000_000_000).to_string(); + h.body.complete_persisted_receiver_ns = h.body.complete_prepare_receiver_ns.clone(); + for floor in &mut h.body.final_source_fences { + let ns = u64::MAX - 1_000_000_001; + floor.sequence = u32::MAX as u64; + floor.receiver_ns = ns.to_string(); + floor.native_frame_id = ((u128::from(ns) << 40) + | (u128::from(floor.source_id) << 32) + | u128::from(floor.sequence)) + .to_string(); + } + rehash_held(&mut h); + let mut e = evaluation(&t, &h); + e.body.evaluated_at_receiver_ns = (u64::MAX - 1).to_string(); + e.body.evaluated_at_unix_ms = (u64::MAX - 1) / 1_000_000; + for condition in &mut e.body.report.per_condition { + condition.binary_cross_entropy = Some(1.234_567_8e-30); + condition.declared_session_count = 16; + } + e.body.report.binary_cross_entropy = Some(1.234_567_8e-30); + e.body.report.declared_session_count = 64; + rehash_eval(&mut e); + e.validate_data_links(&t, &h).unwrap(); + let sizes = [ + serde_json::to_vec(&t).unwrap().len(), + serde_json::to_vec(&h).unwrap().len(), + serde_json::to_vec(&e).unwrap().len(), + ]; + assert!(sizes.iter().all(|n| *n < MAX_METADATA_BYTES)); + eprintln!("SYNTHETIC bounded max-width DATA bytes={sizes:?}; not a raw128 recovery-size or performance proof"); +} + +#[test] +fn final_complete_source_fences_are_exact_bounded_native_metadata_not_a_live_permit() { + let t = train(); + let h = held(&t); + h.validate_data_links(&t).unwrap(); + for case in 0..9 { + let mut bad = h.clone(); + match case { + 0 => bad.body.final_source_fences[0].source_id = 99, + 1 => bad.body.final_source_fences.swap(0, 1), + 2 => bad.body.final_source_fences[0].sequence = u32::MAX as u64 + 1, + 3 => bad.body.final_source_fences[0].receiver_ns = "01800001299999000000".into(), + 4 => bad.body.final_source_fences[0].native_frame_id = "01".into(), + 5 => { + bad.body.final_source_fences[0].native_frame_id = + "340282366920938463463374607431768211456".into() + } + 6 => bad.body.final_source_fences[0].native_frame_id = "999".into(), + 7 => bad.body.complete_persisted_receiver_ns = "1800001299000000000".into(), + _ => bad.body.complete_persisted_receiver_ns = "1800001300000000001".into(), + } + rehash_held(&mut bad); + assert!(bad.validate_data().is_err(), "case{case}"); + } + let mut lower = h.clone(); + let floor = &mut lower.body.final_source_fences[0]; + floor.sequence = t.body.source_floors[0].sequence - 1; + let ns = decimal(&floor.receiver_ns).unwrap(); + floor.native_frame_id = + ((u128::from(ns) << 40) | (u128::from(floor.source_id) << 32) | u128::from(floor.sequence)) + .to_string(); + rehash_held(&mut lower); + assert!(lower.validate_data_links(&t).is_err()); + let mut value = serde_json::to_value(&h).unwrap(); + value["body"]["final_source_fences"] + .as_array_mut() + .unwrap() + .pop(); + assert!( + TemporalHeldOutEvidenceData::decode_data(&serde_json::to_vec(&value).unwrap()).is_err() + ); + let mut value = serde_json::to_value(&h).unwrap(); + value["body"]["final_source_fences"][0]["sequence"] = json!("20000"); + assert!( + TemporalHeldOutEvidenceData::decode_data(&serde_json::to_vec(&value).unwrap()).is_err() + ); + assert!(!h.body.claims.adoption_authorized); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_projection.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_projection.rs new file mode 100644 index 0000000000..1af9c3fa40 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_projection.rs @@ -0,0 +1,565 @@ +//! Protected C4C projections from the SAME current FullRun producer. +//! Raw prefix/DATA decoding cannot create either projection or this pending +//! one-use source. All current-store IO, validation and hashing run off-lock. +use super::*; +use crate::room_learner::temporal_evaluation as data; +use std::sync::atomic::{AtomicBool, Ordering}; + +#[derive(Clone, PartialEq)] +pub(crate) struct TemporalEvaluationBinding { + epoch: String, + generation: u64, + context: CaptureContext, + context_sha256: String, + run: String, + token: TemporalToken, + fit: String, + model: String, + complete: Option, +} +impl TemporalEvaluationBinding { + pub(crate) fn owner_matches(&self, epoch: &str, generation: u64) -> bool { + self.epoch == epoch && self.generation == generation + } + pub(crate) fn context_digest(&self) -> &str { + &self.context_sha256 + } + pub(crate) fn context(&self) -> &CaptureContext { + &self.context + } + pub(crate) fn recertify_current(&self, store: &RecoveryStore, now: u64) -> Result<(), String> { + store.recertify_temporal(&self.token, now).map(|_| ()) + } + pub(crate) fn model_sha256(&self) -> &str { + &self.model + } + pub(crate) fn complete_sha256(&self) -> Option<&str> { + self.complete.as_deref() + } +} +/// No Clone/serde/import constructor. Minted below only from current owned +/// verified Frozen/Complete facts; consuming it authenticates the current store. +pub(crate) struct PendingTemporalEvidenceSource { + binding: TemporalEvaluationBinding, + model: Arc, + artifact: Arc, + context: CaptureContext, + start_floor: u64, + deadline: u64, + held_out: bool, +} +pub(crate) struct VerifiedTemporalTrainSnapshot { + model: Arc, + artifact: Arc, + proof: data::TemporalTrainProofData, +} +pub(crate) struct VerifiedTemporalEvaluationSnapshot { + train: VerifiedTemporalTrainSnapshot, + binding: TemporalEvaluationBinding, + held_out: data::TemporalHeldOutEvidenceData, + rows: Vec>, +} +impl VerifiedTemporalTrainSnapshot { + pub(crate) fn model(&self) -> &Arc { + &self.model + } + pub(crate) fn require_same_store(&self, store: &ArtifactStore) -> Result<(), String> { + if !std::ptr::eq(store, self.artifact.as_ref()) { + return Err("temporal_evaluation_same_store_required".into()); + } + Ok(()) + } + pub(crate) fn proof(&self) -> &data::TemporalTrainProofData { + &self.proof + } +} +impl VerifiedTemporalEvaluationSnapshot { + pub(crate) fn train_snapshot(&self) -> &VerifiedTemporalTrainSnapshot { + &self.train + } + pub(crate) fn proof(&self) -> &data::TemporalTrainProofData { + self.train.proof() + } + pub(crate) fn held_out(&self) -> &data::TemporalHeldOutEvidenceData { + &self.held_out + } + pub(crate) fn binding(&self) -> &TemporalEvaluationBinding { + &self.binding + } + pub(crate) fn model(&self) -> &Arc { + &self.train.model + } + pub(crate) fn require_same_store(&self, store: &ArtifactStore) -> Result<(), String> { + if !std::ptr::eq(store, self.train.artifact.as_ref()) { + return Err("temporal_evaluation_same_store_required".into()); + } + Ok(()) + } + pub(crate) fn evaluation_rows(&self) -> Vec { + self.rows.iter().map(|r| (**r).clone()).collect() + } + pub(crate) fn require_same_model(&self, model: &TemporalModel) -> Result<(), String> { + if !std::ptr::eq(model, self.train.model.as_ref()) { + return Err("temporal_evaluation_actual_model_required".into()); + } + Ok(()) + } +} +impl TemporalWorkflow { + pub(crate) fn prepare_temporal_evidence_source( + &self, + now: u64, + held_out: bool, + schedule: &ManagedScheduleContract, + ) -> Result { + let mode = self.full_run.as_ref().ok_or("temporal_full_run_required")?; + let owner = mode + .owner + .as_ref() + .ok_or("temporal_full_run_owner_custody_required")?; + let verified = mode + .verified + .as_ref() + .ok_or("temporal_verified_frozen_required")?; + let fit = mode + .frozen + .as_ref() + .ok_or("temporal_verified_fit_record_required")?; + let token = self + .commit_token + .as_ref() + .ok_or("temporal_current_token_required")?; + // A paused suffix mapping cannot change the already fitted/confirmed + // first64 Train rows. It is permitted ONLY for stable TrainProof, never + // for a HeldOut evaluation/qualification/selection capability. + let suffix_mapping = !held_out + && self.stage == Stage::PausedNeedsReconfirmation + && schedule.sha256() == fit.body.schedule_sha256 + && schedule + .episodes() + .get(self.episode) + .is_some_and(|e| e.role == DataRole::HeldOut) + && self.bridge.as_ref().is_some_and(|b| b.seals.len() == 7) + && self.session.rows.len() >= 64; + if self.mapping.is_some() && !suffix_mapping { + return Err("temporal_train_projection_mapping_overlaps_active_or_train".into()); + } + if self.session.failed + || self.session.active + || self.stage == Stage::BlockedDurability + || self.pending_intent.is_some() + || self.pending_prefix.is_some() + || self + .async_state + .as_ref() + .is_none_or(|a| a.pending.is_some() || a.busy_pause || a.error.is_some()) + || self.durable_windows != self.session.rows.len() + || fit != &verified.record + || now < self.last_now_ns + || now / 1_000_000 >= self.descriptor.expires_at_ms + || mode.pending.is_some() + || mode.pending_complete.is_some() + { + return Err("temporal_evidence_current_durable_owner_required".into()); + } + let complete = if held_out { + let fact = mode + .completed + .as_ref() + .ok_or("temporal_actual_complete_required")?; + let record = mode + .complete + .as_ref() + .ok_or("temporal_complete_record_required")?; + if fact.token != *token + || fact.record_sha256 != record.sha256 + || mode.held_out_seals() != 8 + || self.session.rows.len() != 128 + { + return Err("temporal_current_complete_changed".into()); + } + Some(record.sha256.clone()) + } else { + if !(64..=128).contains(&self.session.rows.len()) { + return Err("temporal_stable_train_projection_required".into()); + } + None + }; + // Arc refs and bounded identity only. Native tensors/records, SHA work, + // full model validation and disk access are deliberately not done here. + Ok(PendingTemporalEvidenceSource { + binding: TemporalEvaluationBinding { + epoch: owner.runtime_epoch.clone(), + generation: owner.generation, + context: self.context().clone(), + context_sha256: fit.body.context_digest.clone(), + run: self.session.run_id.clone(), + token: token.clone(), + fit: fit.sha256.clone(), + model: fit.body.artifact_sha256.clone(), + complete, + }, + model: verified.model.clone(), + artifact: verified.store.clone(), + context: self.context().clone(), + start_floor: now, + deadline: self.descriptor.expires_at_ms, + held_out, + }) + } + pub(crate) fn current_temporal_evidence_matches(&self, b: &TemporalEvaluationBinding) -> bool { + self.full_run.as_ref().is_some_and(|m| { + m.owner + .as_ref() + .is_some_and(|o| o.runtime_epoch == b.epoch && o.generation == b.generation) + && m.frozen + .as_ref() + .is_some_and(|f| f.sha256 == b.fit && f.body.artifact_sha256 == b.model) + && b.complete.as_ref().is_none_or(|sha| { + m.completed + .as_ref() + .is_some_and(|c| c.record_sha256 == *sha && c.token == b.token) + }) + }) && self.commit_token.as_ref() == Some(&b.token) + && self.session.run_id == b.run + && self.context() == &b.context + && !self.session.failed + } +} +fn commitment( + value: &crate::room_learner::recovery_store::temporal_commitment::TemporalCommitmentData, +) -> Result { + Ok(data::CommitmentData { + generation: value.generation, + revision: value + .revision + .try_into() + .map_err(|_| "temporal_commitment_revision")?, + run: value.run.clone(), + id: value.id.clone(), + commitment_sha256: value.commitment.clone(), + }) +} +impl PendingTemporalEvidenceSource { + pub(crate) fn binding(&self) -> &TemporalEvaluationBinding { + &self.binding + } + pub(crate) fn expires_at_ms(&self) -> u64 { + self.deadline + } + fn check(&self, now: u64, cancel: &AtomicBool) -> Result<(), String> { + if cancel.load(Ordering::Acquire) { + return Err("temporal_evaluation_cancelled".into()); + } + if now < self.start_floor || now / 1_000_000 >= self.deadline { + return Err("temporal_evaluation_clock_or_original_deadline".into()); + } + Ok(()) + } + /// One-use actual-current-store recertification; structural decoders have + /// no constructor for this source and cannot reach the authority return. + pub(crate) fn recertify_evaluation( + self, + store: &RecoveryStore, + now: u64, + cancel: &AtomicBool, + ) -> Result { + if !self.held_out { + return Err("temporal_complete_projection_required".into()); + } + let (binding, train, held, rows) = self.recertify(store, now, cancel)?; + Ok(VerifiedTemporalEvaluationSnapshot { + train, + binding, + held_out: held.ok_or("temporal_complete_evidence_required")?, + rows, + }) + } + pub(crate) fn recertify_train( + self, + store: &RecoveryStore, + now: u64, + cancel: &AtomicBool, + ) -> Result { + let (_, train, _, _) = self.recertify(store, now, cancel)?; + Ok(train) + } + fn recertify( + self, + store: &RecoveryStore, + now: u64, + cancel: &AtomicBool, + ) -> Result< + ( + TemporalEvaluationBinding, + VerifiedTemporalTrainSnapshot, + Option, + Vec>, + ), + String, + > { + self.check(now, cancel)?; + let current = store.recertify_temporal(&self.binding.token, now)?; + self.check(now, cancel)?; + let w = checkpoint::decode(current.prefix.bytes(), current.prefix.descriptor(), now)?; + let mode = w + .full_run + .as_ref() + .ok_or("temporal_full_run_no_upconversion")?; + let record = mode + .frozen + .as_ref() + .ok_or("temporal_verified_fit_record_required")?; + if w.context() != &self.context + || w.session.run_id != self.binding.run + || record.sha256 != self.binding.fit + || record.body.artifact_sha256 != self.binding.model + { + return Err("temporal_evaluation_current_source_changed".into()); + } + let loaded = self + .artifact + .load_exact_temporal_candidate(&self.context, &self.binding.model)?; + verify_model_binding(record, &loaded)?; + verify_model_binding(record, &self.model)?; + checkpoint::validate_temporal_evaluation_train_prefix(&w, record)?; + self.check(now, cancel)?; + let identity = data::TemporalEvaluationIdentityData { + owner_scope: w.descriptor.owner_scope.clone(), + installation_id: w.descriptor.installation_id.clone(), + context_digest: w.context().digest(), + room_digest: w.context().room_digest.clone(), + run: w.session.run_id.clone(), + original_deadline: data::OriginalDeadlineData { + id: w.descriptor.id.clone(), + created_at_unix_ms: w.descriptor.created_at_ms, + expires_at_unix_ms: w.descriptor.expires_at_ms, + }, + artifact_kind: MODEL_KIND.into(), + artifact_sha256: self.binding.model.clone(), + verified_fit_record_sha256: record.sha256.clone(), + prefix_schema: PREFIX_SCHEMA.into(), + metadata_kind: META_KIND.into(), + continuation_policy_sha256: record.body.continuation_policy_sha256.clone(), + input_policy_sha256: record.body.input_policy_sha256.clone(), + execution_policy_sha256: record.body.execution_policy_sha256.clone(), + schedule_sha256: record.body.schedule_sha256.clone(), + configured_fit_sha256: record.body.configured_fit_sha256.clone(), + source_ids: w + .context() + .sources + .iter() + .map(|s| s.source_id) + .collect::>() + .try_into() + .map_err(|_| "temporal_evaluation_three_sources_required")?, + }; + let seals = &w + .bridge + .as_ref() + .ok_or("temporal_train_seals_required")? + .seals; + let proof = + data::TemporalTrainProofData::from_body_data(data::TemporalTrainProofBodyData { + identity: identity.clone(), + training_data_sha256: record.body.training_data_sha256.clone(), + training_evidence_sha256: record.body.input_evidence_sha256.clone(), + training_identities_sha256: record.body.identity_evidence_sha256.clone(), + train_seal_sha256s: seals + .iter() + .map(|s| s.sha256.clone()) + .collect::>() + .try_into() + .map_err(|_| "temporal_train_seven_seals_required")?, + train_seal_chain_sha256: record.body.train_seal_chain_sha256.clone(), + train_windows: 64, + windows_per_condition: [16; 4], + declared_episodes_per_condition: [1, 2, 2, 2], + accepted_train_commitment: commitment(&record.body.accepted_train_commitment)?, + freeze_prepare_receiver_ns: record.body.freeze_prepare_receiver_ns.to_string(), + source_floors: record + .body + .source_floors + .iter() + .map(|(id, seq, ns)| data::FrozenSourceFloorData { + source_id: *id, + sequence: *seq, + receiver_ns: ns.to_string(), + }) + .collect::>() + .try_into() + .map_err(|_| "temporal_evaluation_three_sources_required")?, + claims: data::DiagnosticClaimsData::diagnostic_only(), + })?; + let held = if self.held_out { + let complete = mode + .complete + .as_ref() + .ok_or("temporal_actual_complete_required")?; + if Some(&complete.sha256) != self.binding.complete.as_ref() + || w.session.rows.len() != 128 + || current.token != self.binding.token + { + return Err("temporal_current_complete_changed".into()); + } + let managed = w + .session + .managed + .as_ref() + .ok_or("managed_session_required")?; + let hs = &mode + .held_out + .as_ref() + .ok_or("temporal_heldout_seals_required")? + .seals; + let evidence = + data::TemporalHeldOutEvidenceData::from_body_data(data::TemporalHeldOutBodyData { + identity, + training_proof_sha256: proof.sha256.clone(), + held_out_data_sha256: hash(&(&w.session.rows[64..]))?, + held_out_evidence_sha256: hash(&(&w.session.evidence[64..]))?, + held_out_identities_sha256: hash(&(&managed.windows[64..]))?, + complete_rows_sha256: complete.body.raw_sha256.clone(), + complete_evidence_sha256: complete.body.evidence_sha256.clone(), + complete_identities_sha256: complete.body.identity_sha256.clone(), + complete_ledger_sha256: complete.body.ledger_sha256.clone(), + held_out_seal_sha256s: hs + .iter() + .map(|s| s.sha256.clone()) + .collect::>() + .try_into() + .map_err(|_| "temporal_eight_heldout_seals_required")?, + held_out_seal_chain_sha256: complete.body.held_out_chain_sha256.clone(), + complete_record_sha256: complete.sha256.clone(), + complete_commitment: commitment(¤t.token.data())?, + complete_prepare_receiver_ns: complete + .body + .complete_prepare_receiver_ns + .to_string(), + complete_persisted_receiver_ns: w.last_now_ns.to_string(), + final_source_fences: w + .source_high + .iter() + .map(|(id, seq, ns)| data::FinalSourceFenceData { + source_id: *id, + sequence: *seq, + receiver_ns: ns.to_string(), + native_frame_id: ((u128::from(*ns) << 40) + | (u128::from(*id) << 32) + | u128::from(*seq)) + .to_string(), + }) + .collect::>() + .try_into() + .map_err(|_| "temporal_evaluation_three_sources_required")?, + held_out_windows: 64, + windows_per_condition: [16; 4], + declared_episodes_per_condition: [2; 4], + claims: data::DiagnosticClaimsData::diagnostic_only(), + })?; + evidence.validate_data_links(&proof)?; + Some(evidence) + } else { + None + }; + let rows = if self.held_out { + w.session.rows[64..].to_vec() + } else { + vec![] + }; + self.check(now, cancel)?; + Ok(( + self.binding, + VerifiedTemporalTrainSnapshot { + model: self.model, + artifact: self.artifact, + proof, + }, + held, + rows, + )) + } +} + +/// No serde/Clone/from-DATA constructor: computed only from the protected +/// snapshot and the SAME whole model's existing evaluator. +pub(crate) struct VerifiedTemporalEvaluationResult { + artifact: Arc, + binding: TemporalEvaluationBinding, + train: data::TemporalTrainProofData, + held: data::TemporalHeldOutEvidenceData, + receipt: data::TemporalEvaluationReceiptData, +} +impl VerifiedTemporalEvaluationResult { + pub(crate) fn require_same_store(&self, store: &ArtifactStore) -> Result<(), String> { + if !std::ptr::eq(store, self.artifact.as_ref()) { + return Err("temporal_evaluation_same_store_required".into()); + } + Ok(()) + } + pub(crate) fn binding(&self) -> &TemporalEvaluationBinding { + &self.binding + } + pub(crate) fn train(&self) -> &data::TemporalTrainProofData { + &self.train + } + pub(crate) fn held_out(&self) -> &data::TemporalHeldOutEvidenceData { + &self.held + } + pub(crate) fn receipt(&self) -> &data::TemporalEvaluationReceiptData { + &self.receipt + } +} +impl VerifiedTemporalEvaluationSnapshot { + pub(crate) fn compute_verified( + &self, + cancel: &AtomicBool, + clock: &mut dyn FnMut() -> Result, + ) -> Result { + let before = clock()?; + if cancel.load(Ordering::Acquire) { + return Err("temporal_evaluation_cancelled".into()); + } + let deadline = &self.proof().body.identity.original_deadline; + if before / 1_000_000 >= deadline.expires_at_unix_ms { + return Err("temporal_evaluation_original_deadline_expired".into()); + } + let report = self.model().evaluate_managed_held_out(self)?; + // Existing compute is bounded64 rows. Cancellation withdraws admission + // and drains the entire call; no per-row interruption is claimed. + let after = clock()?; + if cancel.load(Ordering::Acquire) + || after < before + || after / 1_000_000 >= deadline.expires_at_unix_ms + { + return Err("temporal_evaluation_cancelled_or_clock".into()); + } + let receipt = data::TemporalEvaluationReceiptData::from_body_data( + data::TemporalEvaluationBodyData { + identity: self.proof().body.identity.clone(), + training_proof_sha256: self.proof().sha256.clone(), + held_out_evidence_sha256: self.held_out().sha256.clone(), + complete_record_sha256: self.held_out().body.complete_record_sha256.clone(), + training_data_sha256: self.proof().body.training_data_sha256.clone(), + held_out_data_sha256: self.held_out().body.held_out_data_sha256.clone(), + role: DataRole::HeldOut, + window_unit: "native_receiver_2s_window".into(), + evaluated_at_receiver_ns: after.to_string(), + evaluated_at_unix_ms: after / 1_000_000, + provenance: vec![CaptureProvenance::LiveReceiverObserved], + clock_claims: vec![ClockKind::ReceiverOnly], + train_lineage_checked: true, + report, + claims: data::DiagnosticClaimsData::diagnostic_only(), + }, + )?; + receipt.validate_data_links(self.proof(), self.held_out())?; + Ok(VerifiedTemporalEvaluationResult { + artifact: self.train.artifact.clone(), + binding: self.binding.clone(), + train: self.proof().clone(), + held: self.held_out().clone(), + receipt, + }) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_store.rs new file mode 100644 index 0000000000..29c605c78b --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_store.rs @@ -0,0 +1,546 @@ +//! Immutable whole-Temporal metadata in the SAME ArtifactStore owner root. +//! Raw decoder returns DATA only. Protected source/result is required to write +//! or claim a current evaluation; no parallel DB/key/latest pointer is added. +use super::*; +use crate::room_learner::{ + observation::DataRole, + temporal_evaluation as data, + temporal_model_input::managed::execution::full_run::{ + TemporalEvaluationBinding, VerifiedTemporalEvaluationResult, + VerifiedTemporalEvaluationSnapshot, VerifiedTemporalTrainSnapshot, + }, +}; +use std::sync::atomic::{AtomicBool, Ordering}; +const CHILD: &str = "managed-evaluation"; +pub(crate) struct StoredTemporalEvaluation { + binding: TemporalEvaluationBinding, + train: data::TemporalTrainProofData, + held: data::TemporalHeldOutEvidenceData, + receipt: data::TemporalEvaluationReceiptData, + lookup_sha256: String, +} +impl StoredTemporalEvaluation { + pub(crate) fn binding(&self) -> &TemporalEvaluationBinding { + &self.binding + } + pub(crate) fn train(&self) -> &data::TemporalTrainProofData { + &self.train + } + pub(crate) fn held_out(&self) -> &data::TemporalHeldOutEvidenceData { + &self.held + } + pub(crate) fn receipt(&self) -> &data::TemporalEvaluationReceiptData { + &self.receipt + } + pub(crate) fn lookup_sha256(&self) -> &str { + &self.lookup_sha256 + } +} +fn lookup(source: &VerifiedTemporalEvaluationSnapshot) -> Result { + let identity = &source.proof().body.identity; + crate::room_learner::qualification::hash(&( + data::EVALUATION_SCHEMA, + "lookup", + &identity.artifact_kind, + &identity.artifact_sha256, + &source.proof().sha256, + &source.held_out().sha256, + DataRole::HeldOut, + )) +} +fn allowed( + cancel: &AtomicBool, + clock: &mut dyn FnMut() -> Result, +) -> Result { + if cancel.load(Ordering::Acquire) { + return Err("temporal_evaluation_cancelled".into()); + } + clock() +} +impl ArtifactStore { + pub(crate) fn persist_current_temporal_train_proof( + &self, + source: &VerifiedTemporalTrainSnapshot, + cancel: &AtomicBool, + clock: &mut dyn FnMut() -> Result, + io: &mut dyn FnMut(&'static str) -> Result<(), String>, + ) -> Result { + source.require_same_store(self)?; + allowed(cancel, clock)?; + let directory = self + .temporal_evaluation_directory(source.model().context(), true)? + .ok_or("temporal_evaluation_directory_unavailable")?; + let bytes = source.proof().encode_data()?; + let path = directory.join(format!("train-proof-{}.json", source.proof().sha256)); + temporal_artifacts::immutable_with_io(&path, &bytes, &mut |_| { + io("train_proof_install_before_directory_fsync") + })?; + allowed(cancel, clock)?; + let read = data::TemporalTrainProofData::decode_data(&temporal_artifacts::bounded_read( + &path, + data::MAX_METADATA_BYTES, + )?)?; + if read != *source.proof() { + return Err("temporal_train_proof_readback_changed".into()); + } + temporal_artifacts::recertify_immutable( + &path, + &bytes, + data::MAX_METADATA_BYTES, + &mut |stage| { + allowed(cancel, clock)?; + io(if stage == "before:file_fsync" { + "train_proof_recertify_before_file_fsync" + } else { + "train_proof_recertify_before_directory_fsync" + }) + }, + )?; + self.recertify_temporal_metadata_directories(&directory, cancel, clock, io)?; + Ok(read) + } + pub(super) fn temporal_evaluation_directory( + &self, + context: &CaptureContext, + create: bool, + ) -> Result, String> { + let Some(root) = self.temporal_directory(context, create)? else { + return Ok(None); + }; + let directory = root.join(CHILD); + if !temporal_artifacts::directory(&directory, create)? { + return Ok(None); + } + Ok(Some(directory)) + } + /// The current protected Complete snapshot supplies exact paths/identities. + /// A generic imported receipt/proof never reaches this return constructor. + pub(super) fn load_current_temporal_evaluation( + &self, + source: &VerifiedTemporalEvaluationSnapshot, + ) -> Result, String> { + source.require_same_store(self)?; + let Some(directory) = + self.temporal_evaluation_directory(source.model().context(), false)? + else { + return Ok(None); + }; + let key = lookup(source)?; + let path = directory.join(format!("evaluation-{key}.json")); + match std::fs::symlink_metadata(&path) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(_) => return Err("temporal_evaluation_file_unavailable".into()), + Ok(_) => {} + } + let receipt = data::TemporalEvaluationReceiptData::decode_data( + &temporal_artifacts::bounded_read(&path, data::MAX_METADATA_BYTES)?, + )?; + let train = data::TemporalTrainProofData::decode_data(&temporal_artifacts::bounded_read( + &directory.join(format!("train-proof-{}.json", source.proof().sha256)), + data::MAX_METADATA_BYTES, + )?)?; + let held = + data::TemporalHeldOutEvidenceData::decode_data(&temporal_artifacts::bounded_read( + &directory.join(format!( + "heldout-evidence-{}.json", + source.held_out().sha256 + )), + data::MAX_METADATA_BYTES, + )?)?; + receipt.validate_data_links(&train, &held)?; + if train != *source.proof() || held != *source.held_out() { + return Err("temporal_evaluation_current_source_changed".into()); + } + Ok(Some(StoredTemporalEvaluation { + binding: source.binding().clone(), + train, + held, + receipt, + lookup_sha256: key, + })) + } + /// Existing compute and immutable IO; exact retry loads the previous record + /// BEFORE inference, preserving the ORIGINAL evaluation timestamp/report. + pub(crate) fn evaluate_temporal_snapshot( + &self, + source: VerifiedTemporalEvaluationSnapshot, + cancel: &AtomicBool, + clock: &mut dyn FnMut() -> Result, + io: &mut dyn FnMut(&'static str) -> Result<(), String>, + ) -> Result { + allowed(cancel, clock)?; + source.require_same_store(self)?; + if let Some(existing) = self.load_current_temporal_evaluation(&source)? { + let now = allowed(cancel, clock)?; + let evaluated = existing + .receipt() + .body + .evaluated_at_receiver_ns + .parse::() + .map_err(|_| "temporal_evaluation_data_integer")?; + if now < evaluated { + return Err("temporal_evaluation_stored_clock_reversed".into()); + } + return self.recertify_current_temporal_evaluation(&source, existing, cancel, clock, io); + } + let key = lookup(&source)?; + let computed = source.compute_verified(cancel, clock)?; + self.persist_temporal_evaluation_result(&source, &computed, &key, cancel, clock, io) + } + fn persist_temporal_evaluation_result( + &self, + source: &VerifiedTemporalEvaluationSnapshot, + result: &VerifiedTemporalEvaluationResult, + key: &str, + cancel: &AtomicBool, + clock: &mut dyn FnMut() -> Result, + io: &mut dyn FnMut(&'static str) -> Result<(), String>, + ) -> Result { + source.require_same_store(self)?; + result.require_same_store(self)?; + if result.binding() != source.binding() { + return Err("temporal_evaluation_operation_changed".into()); + } + allowed(cancel, clock)?; + let directory = self + .temporal_evaluation_directory(source.model().context(), true)? + .ok_or("temporal_evaluation_directory_unavailable")?; + let writes = [ + ( + format!("train-proof-{}.json", result.train().sha256), + result.train().encode_data()?, + ), + ( + format!("heldout-evidence-{}.json", result.held_out().sha256), + result.held_out().encode_data()?, + ), + ( + format!("evaluation-{key}.json"), + result.receipt().encode_data()?, + ), + ]; + for (name, bytes) in writes { + allowed(cancel, clock)?; + // Same immutable private stage/link/fsync primitive as whole models. + let stage = if name.starts_with("evaluation-") { + "evaluation_receipt_install_before_directory_fsync" + } else if name.starts_with("train-proof-") { + "train_proof_install_before_directory_fsync" + } else { + "heldout_evidence_install_before_directory_fsync" + }; + temporal_artifacts::immutable_with_io(&directory.join(&name), &bytes, &mut |_| { + io(stage) + })?; + allowed(cancel, clock)?; + if temporal_artifacts::bounded_read(&directory.join(name), data::MAX_METADATA_BYTES)? + != bytes + { + return Err("temporal_evaluation_immutable_readback_changed".into()); + } + } + allowed(cancel, clock)?; + let stored = self + .load_current_temporal_evaluation(source)? + .ok_or("temporal_evaluation_readback_missing")?; + self.recertify_current_temporal_evaluation(source, stored, cancel, clock, io) + } + fn recertify_current_temporal_evaluation( + &self, + source: &VerifiedTemporalEvaluationSnapshot, + stored: StoredTemporalEvaluation, + cancel: &AtomicBool, + clock: &mut dyn FnMut() -> Result, + io: &mut dyn FnMut(&'static str) -> Result<(), String>, + ) -> Result { + source.require_same_store(self)?; + allowed(cancel, clock)?; + let directory = self + .temporal_evaluation_directory(source.model().context(), false)? + .ok_or("temporal_evaluation_directory_unavailable")?; + let records = [ + ( + format!("train-proof-{}.json", stored.train.sha256), + stored.train.encode_data()?, + "train", + ), + ( + format!("heldout-evidence-{}.json", stored.held.sha256), + stored.held.encode_data()?, + "held", + ), + ( + format!("evaluation-{}.json", stored.lookup_sha256), + stored.receipt.encode_data()?, + "evaluation", + ), + ]; + for (name, bytes, kind) in records { + temporal_artifacts::recertify_immutable( + &directory.join(name), + &bytes, + data::MAX_METADATA_BYTES, + &mut |stage| { + allowed(cancel, clock)?; + io(match (kind, stage) { + ("evaluation", "before:file_fsync") => { + "evaluation_receipt_recertify_before_file_fsync" + } + ("evaluation", _) => "evaluation_receipt_recertify_before_directory_fsync", + ("train", "before:file_fsync") => "train_proof_recertify_before_file_fsync", + ("train", _) => "train_proof_recertify_before_directory_fsync", + (_, "before:file_fsync") => "heldout_evidence_recertify_before_file_fsync", + _ => "heldout_evidence_recertify_before_directory_fsync", + }) + }, + )?; + } + self.recertify_temporal_metadata_directories(&directory, cancel, clock, io)?; + allowed(cancel, clock)?; + let read = self + .load_current_temporal_evaluation(source)? + .ok_or("temporal_evaluation_readback_missing")?; + if read.train != stored.train || read.held != stored.held || read.receipt != stored.receipt + { + return Err("temporal_evaluation_recertify_changed".into()); + } + Ok(read) + } + fn recertify_temporal_metadata_directories( + &self, + directory: &Path, + cancel: &AtomicBool, + clock: &mut dyn FnMut() -> Result, + io: &mut dyn FnMut(&'static str) -> Result<(), String>, + ) -> Result<(), String> { + let mut path = directory; + loop { + allowed(cancel, clock)?; + temporal_artifacts::directory(path, false)?; + io("metadata_directory_chain_before_fsync")?; + std::fs::File::open(path) + .and_then(|f| f.sync_all()) + .map_err(|_| "temporal_evaluation_directory_sync_failed")?; + if path == self.root { + break; + } + path = path + .parent() + .ok_or("temporal_evaluation_directory_outside_owner")?; + if !path.starts_with(&self.root) { + return Err("temporal_evaluation_directory_outside_owner".into()); + } + } + Ok(()) + } + /// Explicit inventory for privacy/reporting; contents include labels, + /// run/trial and receiver/frame/sequence lineage metadata, not aggregates only. + pub(crate) fn temporal_evaluation_inventory( + &self, + context: &CaptureContext, + ) -> Result, String> { + let Some(directory) = self.temporal_evaluation_directory(context, false)? else { + return Ok(vec![]); + }; + let mut files = vec![]; + for entry in + std::fs::read_dir(directory).map_err(|_| "temporal_evaluation_inventory_unavailable")? + { + if files.len() >= 512 { + return Err("temporal_evaluation_inventory_too_many_records".into()); + } + let entry = entry.map_err(|_| "temporal_evaluation_inventory_unavailable")?; + let name = entry.file_name().to_string_lossy().into_owned(); + if !(name.starts_with("train-proof-") + || name.starts_with("heldout-evidence-") + || name.starts_with("evaluation-") + || name.starts_with("latest-assessment-") + || name.starts_with("assessment-") + || name.starts_with("qualification-") + || name.starts_with("temporal-stage-")) + { + return Err("temporal_evaluation_inventory_unknown_file".into()); + } + let bytes = temporal_artifacts::bounded_read(&entry.path(), data::MAX_METADATA_BYTES)?; + files.push((name, bytes.len() as u64)); + } + files.sort(); + Ok(files) + } +} + +pub(crate) struct StoredTemporalAssessment { + binding: TemporalEvaluationBinding, + pub(crate) evaluation: StoredTemporalEvaluation, + pub(crate) qualification: + Option, + pub(crate) assessment: crate::room_learner::temporal_qualification::TemporalAssessmentEnvelope, +} +impl StoredTemporalAssessment { + pub(crate) fn binding(&self) -> &TemporalEvaluationBinding { + &self.binding + } +} +impl ArtifactStore { + pub(crate) fn qualify_temporal_snapshot( + &self, + source: &VerifiedTemporalEvaluationSnapshot, + cancel: &AtomicBool, + clock: &mut dyn FnMut() -> Result, + io: &mut dyn FnMut(&'static str) -> Result<(), String>, + ) -> Result { + use crate::room_learner::temporal_qualification::*; + source.require_same_store(self)?; + allowed(cancel, clock)?; + // A qualification operation consumes only an already durable evaluation; + // it does not invent or silently recompute an evaluation report. + let existing = self + .load_current_temporal_evaluation(&source)? + .ok_or("temporal_evaluation_required")?; + let evaluation = + self.recertify_current_temporal_evaluation(&source, existing, cancel, clock, io)?; + let directory = self + .temporal_evaluation_directory(source.model().context(), false)? + .ok_or("temporal_evaluation_directory_unavailable")?; + let key = crate::room_learner::qualification::hash(&( + crate::room_learner::temporal_wire::ASSESSMENT_SCHEMA, + &evaluation.receipt().sha256, + crate::room_learner::temporal_wire::TemporalQualificationPolicyWire::current(), + ))?; + let path = directory.join(format!("assessment-{key}.json")); + let (assessment, qualification) = match std::fs::symlink_metadata(&path) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + let computed = ComputedTemporalAssessment::issue_current( + &source, + &evaluation, + allowed(cancel, clock)?, + )?; + if let Some(q) = &computed.qualification { + let bytes = + serde_json::to_vec(q).map_err(|_| "temporal_qualification_encoding")?; + temporal_artifacts::immutable_with_io( + &directory.join(format!("qualification-{}.json", q.receipt_sha256)), + &bytes, + &mut |_| { + allowed(cancel, clock)?; + io("qualification_install_before_directory_fsync") + }, + )?; + } + let bytes = serde_json::to_vec(&computed.assessment) + .map_err(|_| "temporal_assessment_encoding")?; + temporal_artifacts::immutable_with_io(&path, &bytes, &mut |_| { + allowed(cancel, clock)?; + io("assessment_install_before_directory_fsync") + })?; + (computed.assessment, computed.qualification) + } + Err(_) => return Err("temporal_assessment_unavailable".into()), + Ok(_) => { + let assessment: TemporalAssessmentEnvelope = serde_json::from_slice( + &temporal_artifacts::bounded_read(&path, data::MAX_METADATA_BYTES)?, + ) + .map_err(|_| "temporal_assessment_encoding")?; + let qualification = match &assessment.receipt.summary.qualification_sha256 { + Some(sha) => { + if !hex_digest(sha) { + return Err("temporal_qualification_digest".into()); + } + Some( + serde_json::from_slice(&temporal_artifacts::bounded_read( + &directory.join(format!("qualification-{sha}.json")), + data::MAX_METADATA_BYTES, + )?) + .map_err(|_| "temporal_qualification_encoding")?, + ) + } + None => None, + }; + (assessment, qualification) + } + }; + assessment.validate(&evaluation, qualification.as_ref())?; + if allowed(cancel, clock)? / 1_000_000 < assessment.receipt.summary.assessed_at_unix_ms { + return Err("temporal_assessment_clock_reversed".into()); + } + let mut records = vec![( + path, + serde_json::to_vec(&assessment).map_err(|_| "temporal_assessment_encoding")?, + )]; + if let Some(q) = &qualification { + records.push(( + directory.join(format!("qualification-{}.json", q.receipt_sha256)), + serde_json::to_vec(q).map_err(|_| "temporal_qualification_encoding")?, + )); + } + for (path, bytes) in records { + temporal_artifacts::recertify_immutable( + &path, + &bytes, + data::MAX_METADATA_BYTES, + &mut |_| { + allowed(cancel, clock)?; + io("assessment_recertify_before_sync") + }, + )?; + } + self.recertify_temporal_metadata_directories(&directory, cancel, clock, io)?; + allowed(cancel, clock)?; + let latest = serde_json::json!({"schema":"ruview.room-learner.temporal-current-assessment.v1","model_sha256":source.binding().model_sha256(),"context_digest":source.binding().context_digest(),"assessment_sha256":assessment.receipt_sha256,"assessment_lookup_sha256":key}); + allowed(cancel, clock)?; + atomic_private_write( + &directory.join(format!( + "latest-assessment-{}.json", + source.binding().model_sha256() + )), + &serde_json::to_vec(&latest).map_err(|_| "temporal_assessment_encoding")?, + ) + .map_err(|_| "temporal_assessment_pointer_write")?; + allowed(cancel, clock)?; + Ok(StoredTemporalAssessment { + binding: source.binding().clone(), + evaluation, + qualification, + assessment, + }) + } +} + +impl ArtifactStore { + pub(crate) fn reconcile_temporal_snapshot( + &self, + source: VerifiedTemporalEvaluationSnapshot, + cancel: &AtomicBool, + clock: &mut dyn FnMut() -> Result, + io: &mut dyn FnMut(&'static str) -> Result<(), String>, + ) -> Result< + ( + data::TemporalTrainProofData, + Option, + ), + String, + > { + source.require_same_store(self)?; + let proof = + self.persist_current_temporal_train_proof(source.train_snapshot(), cancel, clock, io)?; + let existing = self.load_current_temporal_evaluation(&source)?; + if let Some(existing) = &existing { + let evaluated = existing + .receipt() + .body + .evaluated_at_receiver_ns + .parse::() + .map_err(|_| "temporal_evaluation_data_integer")?; + if allowed(cancel, clock)? < evaluated { + return Err("temporal_evaluation_stored_clock_reversed".into()); + } + } + let stored = existing + .map(|existing| { + self.recertify_current_temporal_evaluation(&source, existing, cancel, clock, io) + }) + .transpose()?; + allowed(cancel, clock)?; + Ok((proof, stored)) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run.rs index 8169ba0611..250e5ff2e9 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run.rs @@ -1116,3 +1116,12 @@ impl ContinuationPolicy { Ok(ReplayFenceBudget{max_frames,native_policy_sha256:hash(&binding.native)?}) } } + +#[path="temporal_evaluation_projection.rs"] +mod evaluation_projection; +pub(crate) use evaluation_projection::{PendingTemporalEvidenceSource, TemporalEvaluationBinding, VerifiedTemporalTrainSnapshot, VerifiedTemporalEvaluationSnapshot, VerifiedTemporalEvaluationResult}; + +impl FullRunState { + pub(super) fn public_verified_fit(&self)->Option<&VerifiedFitRecordData>{let verified=self.verified.as_ref()?;let data=self.frozen.as_ref()?;if self.owner.is_none()||data!=&verified.record{return None}Some(data)} + pub(super) fn public_complete(&self,current:Option<&TemporalToken>)->Option<&held_out::FullCompleteRecordData>{let fact=self.completed.as_ref()?;let data=self.complete.as_ref()?;if current!=Some(&fact.token)||data.sha256!=fact.record_sha256{return None}Some(data)} +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run_checkpoint.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run_checkpoint.rs index cc1d0845fc..aa1928878f 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run_checkpoint.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run_checkpoint.rs @@ -716,7 +716,7 @@ fn decode_continued( workflow.full_run = Some(mode); Ok(workflow) } -fn validate_immutable_train( +pub(super) fn validate_immutable_train( w: &TemporalWorkflow, record: &VerifiedFitRecordData, ) -> Result<(), String> { diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver.rs index ac7eefdfdd..7408618cb8 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_live_driver.rs @@ -124,6 +124,7 @@ pub struct ContinuousTemporalInput { counts: HashMap, last_now: Option, observation: Option, + #[cfg(test)] last_input_error:Option, } impl ContinuousTemporalInput { pub fn new(model: TemporalModel) -> Result { @@ -159,6 +160,7 @@ impl ContinuousTemporalInput { counts: HashMap::new(), last_now: None, observation: None, + #[cfg(test)] last_input_error:None, }) } pub fn activate(&mut self) { @@ -355,7 +357,9 @@ impl ContinuousTemporalInput { self.status.reason = Some("collecting_full_horizon"); let admission = match self.input.as_mut().unwrap().ingest(p) { Ok(v) => v, - Err(_) => { + Err(error) => { + #[cfg(test)]{self.last_input_error=Some(error.clone());} + #[cfg(not(test))]let _=error; self.withdraw("source_or_window_invalid"); return self.status.clone(); } @@ -485,3 +489,39 @@ impl ContinuousTemporalInput { #[cfg(test)] #[path = "temporal_live_driver_tests.rs"] mod tests; + +/// Bounded current observation projection. This never changes inference time. +pub(crate) struct SelectedTemporalObservation { + pub prediction:RoomPrediction,pub digest:String,pub first_ns:u64,pub end_ns:u64,pub expires_ns:u64,pub source_latest_ns:u64, +} +impl ContinuousTemporalInput { + pub(crate) fn from_selected(selected:&crate::room_learner::artifact::StoredTemporalSelection)->Result{ + let mut driver=Self::new(selected.model().clone())?; + let held=selected.held();held.validate_data()?; + for floor in &held.body.final_source_fences { + driver.fences.insert(floor.source_id,Fence{sequence:floor.sequence.try_into().map_err(|_|"selected_sequence_overflow")?,time_ns:floor.receiver_ns.parse().map_err(|_|"selected_receiver_clock")?,frame_id:floor.native_frame_id.parse().map_err(|_|"selected_native_frame")?}); + } + driver.last_now=Some(held.body.complete_persisted_receiver_ns.parse().map_err(|_|"selected_receiver_clock")?); + driver.activate();Ok(driver) + } + pub(crate) fn advance_selected_floors(&mut self,floors:&[(u8,u64,u64)],receiver_floor:u64)->Result<(),String>{ + if floors.len()!=3||floors.iter().any(|(id,seq,_)|*seq>u32::MAX as u64||!self.policy.context().sources.iter().any(|s|s.source_id==*id)){return Err("selected_source_floor_identity".into());} + for(id,seq,time)in floors{let frame=(u128::from(*time)<<40)|(u128::from(*id)<<32)|u128::from(*seq);if let Some(old)=self.fences.get(id){if *seqResult{ + self.poll(now); + if self.status.observation.availability!=ObservationAvailability::Available{return Err(self.status.observation.reason.unwrap_or("selected_native_input_unavailable").into());} + let o=self.observation.as_ref().ok_or("selected_native_input_unavailable")?; + if o.prediction.decision==super::model::PresenceDecision::Abstain{return Err("selected_model_abstained".into());} + let source_latest_ns=self.policy.context().sources.iter().map(|s|self.continuity.get(&s.source_id).copied().ok_or("selected_source_missing")).collect::,_>>()?.into_iter().min().ok_or("selected_source_missing")?; + if now1_000_000_000{return Err("selected_source_stale".into());} + Ok(SelectedTemporalObservation{prediction:o.prediction.clone(),digest:o.digest.clone(),first_ns:o.first_ns,end_ns:o.source_bounds.iter().map(|s|s.last_ns).max().ok_or("selected_source_missing")?,expires_ns:o.expires_ns,source_latest_ns}) + } +} + +#[cfg(test)] +impl ContinuousTemporalInput{pub(crate) fn c4c_replay_floor_probe(&self)->Vec<(u8,u32,u64,u128)>{let mut out=self.fences.iter().map(|(id,f)|(*id,f.sequence,f.time_ns,f.frame_id)).collect::>();out.sort_by_key(|v|v.0);out}} + +#[cfg(test)]impl ContinuousTemporalInput{pub(crate) fn c4c_last_input_failure(&self)->Option<&str>{self.last_input_error.as_deref()}} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs index 2ad95b7302..b4b35cf7a2 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs @@ -570,3 +570,29 @@ mod tests; #[cfg(test)] #[path="temporal_fit_summary_managed_tests.rs"] mod fit_summary_tests; + +impl TemporalModel { + /// Whole-model evaluation only. The protected snapshot has authenticated + /// original native/HeldOut lineage; no imported diagnostic or inner model + /// accessor can enter this boundary. Existing infer/tokens/math is reused. + pub(crate) fn evaluate_managed_held_out( + &self, + source:&execution::full_run::VerifiedTemporalEvaluationSnapshot, + )->Result{ + source.require_same_model(self)?; + self.validate_managed()?; + source.held_out().validate_data_links(source.proof())?; + let binding=self.fit_diagnostic_binding()?; + let identity=&source.proof().body.identity; + if binding.artifact_kind!=SCHEMA_V3||binding.artifact_sha256!=identity.artifact_sha256 + ||binding.context_digest!=identity.context_digest + ||binding.training_data_sha256!=source.proof().body.training_data_sha256 + ||binding.input_evidence_sha256!=source.proof().body.training_evidence_sha256 + ||binding.identity_evidence_sha256.as_ref()!=Some(&source.proof().body.training_identities_sha256) + {return Err("temporal_evaluation_whole_model_lineage_changed".into());} + // This bounded64-row temporary copy exists only off ingress lock and + // is not retained in another datastore or used for fitting/normalizing. + let rows=source.evaluation_rows(); + crate::room_learner::training::EvaluationReport::compute(&self.model,&rows) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_qualification.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_qualification.rs new file mode 100644 index 0000000000..a4738ee7bf --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_qualification.rs @@ -0,0 +1,348 @@ +//! Whole-kind LOCAL operational qualification. Existing numeric criteria are +//! reused; no four-cohort aliases, physical validity or automatic selection. +use super::{ + artifact::StoredTemporalEvaluation, + temporal_model_input::managed::execution::full_run::VerifiedTemporalEvaluationSnapshot, + temporal_wire::*, +}; +use serde::{Deserialize, Serialize}; +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(crate) struct TemporalQualificationEnvelope { + pub receipt_sha256: String, + pub receipt: TemporalQualificationReceipt, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(crate) struct TemporalQualificationReceipt { + pub schema: String, + pub owner_scope: String, + pub installation_id: String, + pub qualified_at_unix_ms: u64, + pub summary: TemporalQualificationSummaryWire, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(crate) struct TemporalAssessmentEnvelope { + pub receipt_sha256: String, + pub receipt: TemporalAssessmentReceipt, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(crate) struct TemporalAssessmentReceipt { + pub schema: String, + pub owner_scope: String, + pub installation_id: String, + pub summary: TemporalAssessmentWire, +} +/// Constructed only against the current protected Complete snapshot and exact +/// current-source verified evaluation. Imported DATA is not this result. +pub(crate) struct ComputedTemporalAssessment { + pub(super) qualification: Option, + pub(super) assessment: TemporalAssessmentEnvelope, +} +fn h(v: &T) -> Result { + super::qualification::hash(v) +} +impl ComputedTemporalAssessment { + pub(crate) fn issue_current( + source: &VerifiedTemporalEvaluationSnapshot, + evaluation: &StoredTemporalEvaluation, + now: u64, + ) -> Result { + if source.binding() != evaluation.binding() { + return Err("temporal_assessment_current_complete_changed".into()); + } + let proof = evaluation.train(); + let held = evaluation.held_out(); + let eval = evaluation.receipt(); + eval.validate_data_links(proof, held)?; + let id = &proof.body.identity; + if now / 1_000_000 >= id.original_deadline.expires_at_unix_ms + || now + < eval + .body + .evaluated_at_receiver_ns + .parse::() + .map_err(|_| "temporal_evaluation_data_integer")? + { + return Err("temporal_assessment_clock_or_original_deadline".into()); + } + if proof.body.windows_per_condition != [16; 4] + || held.body.windows_per_condition != [16; 4] + || eval.body.report.samples != 64 + || eval.body.report.per_condition.len() != 4 + || eval + .body + .report + .per_condition + .iter() + .any(|c| c.samples != 16 || c.declared_trial_count != 2) + { + return Err("temporal_assessment_exact_native_evidence_required".into()); + } + let policy = TemporalQualificationPolicyWire::current(); + let policy_sha = h(&policy)?; + let criteria = native_criteria(&eval.body.report); + // Authoritative pass/failure is the established integer criteria, + // with exact16 input shapes enforced above. No metric is replaced. + let outcome = super::qualification::criteria(&eval.body.report, &[16; 4]); + let qualification = if outcome.is_ok() { + let mut summary = TemporalQualificationSummaryWire { + schema: QUALIFICATION_SCHEMA.into(), + receipt_sha256: String::new(), + model_kind: WholeTemporalKind::ManagedTemporalV3, + model_sha256: id.artifact_sha256.clone(), + verified_fit_record_sha256: id.verified_fit_record_sha256.clone(), + training_lineage_basis: "managed_temporal_training_proof_v1".into(), + training_proof_sha256: proof.sha256.clone(), + held_out_evidence_sha256: held.sha256.clone(), + complete_record_sha256: held.body.complete_record_sha256.clone(), + evaluation_receipt_sha256: eval.sha256.clone(), + context_digest: id.context_digest.clone(), + room_digest: id.room_digest.clone(), + evaluated_at_unix_ms: eval.body.evaluated_at_unix_ms, + policy: policy.clone(), + policy_sha256: policy_sha.clone(), + label_provenance: "temporal-training-proof-v1+sealed-native-heldout-v1".into(), + criteria: criteria.clone(), + authority_scope: "local_operator_binary_presence".into(), + window_unit: "native_receiver_2s_window".into(), + train_windows: 64, + held_out_windows: 64, + train_episodes: 7, + held_out_episodes: 8, + independence_verified: false, + physically_validated: false, + measurement_clock_available: false, + }; + let receipt = TemporalQualificationReceipt { + schema: QUALIFICATION_SCHEMA.into(), + owner_scope: id.owner_scope.clone(), + installation_id: id.installation_id.clone(), + qualified_at_unix_ms: now / 1_000_000, + summary: summary.clone(), + }; + let sha = h(&receipt)?; + summary.receipt_sha256 = sha.clone(); + Some(TemporalQualificationEnvelope { + receipt_sha256: sha, + receipt: TemporalQualificationReceipt { summary, ..receipt }, + }) + } else { + None + }; + let condition_failures = eval + .body + .report + .per_condition + .iter() + .filter_map(|c| { + let codes = super::qualification::native_condition_failure_codes(c); + (!codes.is_empty()).then_some(TemporalConditionFailureWire { + condition: c.condition, + codes, + }) + }) + .collect(); + let summary = TemporalAssessmentWire { + schema: ASSESSMENT_SCHEMA.into(), + receipt_sha256: String::new(), + model_kind: WholeTemporalKind::ManagedTemporalV3, + model_sha256: id.artifact_sha256.clone(), + verified_fit_record_sha256: id.verified_fit_record_sha256.clone(), + training_proof_sha256: proof.sha256.clone(), + held_out_evidence_sha256: held.sha256.clone(), + complete_record_sha256: held.body.complete_record_sha256.clone(), + evaluation_receipt_sha256: eval.sha256.clone(), + context_digest: id.context_digest.clone(), + room_digest: id.room_digest.clone(), + evaluated_at_unix_ms: eval.body.evaluated_at_unix_ms, + assessed_at_unix_ms: now / 1_000_000, + policy, + policy_sha256: policy_sha, + window_unit: "native_receiver_2s_window".into(), + outcome: if qualification.is_some() { + "qualified" + } else { + "not_qualified" + } + .into(), + failure_reason: outcome.err(), + qualification_sha256: qualification.as_ref().map(|q| q.receipt_sha256.clone()), + criteria, + condition_failures, + }; + let mut receipt = TemporalAssessmentReceipt { + schema: ASSESSMENT_SCHEMA.into(), + owner_scope: id.owner_scope.clone(), + installation_id: id.installation_id.clone(), + summary, + }; + let sha = h(&receipt)?; + receipt.summary.receipt_sha256 = sha.clone(); + Ok(Self { + qualification, + assessment: TemporalAssessmentEnvelope { + receipt_sha256: sha, + receipt, + }, + }) + } +} + +// The public summary echoes the envelope digest; that echo is excluded from +// the canonical receipt hash to avoid a self-referential commitment. +impl TemporalQualificationEnvelope { + fn canonical_hash(&self) -> Result { + let mut receipt = self.receipt.clone(); + receipt.summary.receipt_sha256.clear(); + h(&receipt) + } + pub(crate) fn validate(&self, evaluation: &StoredTemporalEvaluation) -> Result<(), String> { + self.validate_links( + evaluation.train(), + evaluation.held_out(), + evaluation.receipt(), + ) + } + pub(crate) fn validate_links( + &self, + proof: &super::temporal_evaluation::TemporalTrainProofData, + held: &super::temporal_evaluation::TemporalHeldOutEvidenceData, + e: &super::temporal_evaluation::TemporalEvaluationReceiptData, + ) -> Result<(), String> { + e.validate_data_links(proof, held)?; + let q = &self.receipt.summary; + let id = &e.body.identity; + if self.receipt_sha256 != self.canonical_hash()? + || q.receipt_sha256 != self.receipt_sha256 + || self.receipt.schema != QUALIFICATION_SCHEMA + || q.schema != QUALIFICATION_SCHEMA + || self.receipt.owner_scope != id.owner_scope + || self.receipt.installation_id != id.installation_id + || q.model_sha256 != id.artifact_sha256 + || q.verified_fit_record_sha256 != id.verified_fit_record_sha256 + || q.training_proof_sha256 != proof.sha256 + || q.held_out_evidence_sha256 != held.sha256 + || q.complete_record_sha256 != held.body.complete_record_sha256 + || q.evaluation_receipt_sha256 != e.sha256 + || q.context_digest != id.context_digest + || q.room_digest != id.room_digest + || q.policy != TemporalQualificationPolicyWire::current() + || q.policy_sha256 != h(&q.policy)? + || q.criteria != native_criteria(&e.body.report) + || q.evaluated_at_unix_ms != e.body.evaluated_at_unix_ms + || self.receipt.qualified_at_unix_ms < e.body.evaluated_at_unix_ms + || self.receipt.qualified_at_unix_ms >= id.original_deadline.expires_at_unix_ms + { + return Err("temporal_qualification_identity_changed".into()); + } + super::qualification::criteria(&e.body.report, &[16; 4]).map(|_| ()) + } +} +impl TemporalAssessmentEnvelope { + fn canonical_hash(&self) -> Result { + let mut receipt = self.receipt.clone(); + receipt.summary.receipt_sha256.clear(); + h(&receipt) + } + pub(crate) fn validate( + &self, + evaluation: &StoredTemporalEvaluation, + qualification: Option<&TemporalQualificationEnvelope>, + ) -> Result<(), String> { + self.validate_links( + evaluation.train(), + evaluation.held_out(), + evaluation.receipt(), + qualification, + ) + } + pub(crate) fn validate_links( + &self, + proof: &super::temporal_evaluation::TemporalTrainProofData, + held: &super::temporal_evaluation::TemporalHeldOutEvidenceData, + e: &super::temporal_evaluation::TemporalEvaluationReceiptData, + qualification: Option<&TemporalQualificationEnvelope>, + ) -> Result<(), String> { + e.validate_data_links(proof, held)?; + let a = &self.receipt.summary; + let id = &e.body.identity; + if self.receipt_sha256 != self.canonical_hash()? + || a.receipt_sha256 != self.receipt_sha256 + || self.receipt.schema != ASSESSMENT_SCHEMA + || a.schema != ASSESSMENT_SCHEMA + || self.receipt.owner_scope != id.owner_scope + || self.receipt.installation_id != id.installation_id + || a.model_sha256 != id.artifact_sha256 + || a.verified_fit_record_sha256 != id.verified_fit_record_sha256 + || a.training_proof_sha256 != proof.sha256 + || a.held_out_evidence_sha256 != held.sha256 + || a.complete_record_sha256 != held.body.complete_record_sha256 + || a.evaluation_receipt_sha256 != e.sha256 + || a.policy != TemporalQualificationPolicyWire::current() + || a.policy_sha256 != h(&a.policy)? + || a.criteria != native_criteria(&e.body.report) + || a.evaluated_at_unix_ms != e.body.evaluated_at_unix_ms + || a.assessed_at_unix_ms < e.body.evaluated_at_unix_ms + || a.assessed_at_unix_ms >= id.original_deadline.expires_at_unix_ms + { + return Err("temporal_assessment_identity_changed".into()); + } + match qualification { + Some(q) => { + q.validate_links(proof, held, e)?; + if a.outcome != "qualified" + || a.qualification_sha256.as_deref() != Some(&q.receipt_sha256) + { + return Err("temporal_assessment_outcome_changed".into()); + } + } + None => { + if a.outcome != "not_qualified" + || a.qualification_sha256.is_some() + || super::qualification::criteria(&e.body.report, &[16; 4]).is_ok() + { + return Err("temporal_assessment_outcome_changed".into()); + } + } + } + Ok(()) + } +} + +fn native_criteria( + report: &super::training::EvaluationReport, +) -> Vec { + report + .per_condition + .iter() + .map(|c| { + let occupied = matches!( + c.condition, + super::observation::RoomLabel::OccupiedStatic + | super::observation::RoomLabel::OccupiedMoving + ); + TemporalConditionCriterionWire { + condition: c.condition, + train_windows: 16, + held_out_windows: c.samples as u16, + scored_windows: c.scored as u16, + abstained_windows: c.abstained as u16, + missing_windows: c.missing as u16, + declared_train_episodes: if c.condition == super::observation::RoomLabel::Empty { + 1 + } else { + 2 + }, + declared_held_out_episodes: 2, + evaluated_window_coverage: c.scored as f64 / c.samples as f64, + unconditional_sensitivity: occupied + .then_some(c.confusion.true_occupied as f64 / c.samples as f64), + false_positive_rate: (!occupied) + .then_some(c.confusion.false_occupied as f64 / c.samples as f64), + } + }) + .collect() +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs index 1959e1c8b2..b9af358003 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs @@ -113,11 +113,12 @@ impl RecoveryStore { fn temporal_dir(&self) -> PathBuf { self.directory.join(KIND) } - fn temporal_lock(&self, create: bool) -> Result { - self.verify_directory()?; + fn temporal_lock(&self,create:bool)->Result{self.temporal_lock_mode(create,true)} + fn temporal_lock_mode(&self,create:bool,create_lock:bool)->Result{ + if create_lock{self.verify_directory()?;}else{self.verify_directory_read_only()?;} let path = self.directory.join(LOCK); let mut options = OpenOptions::new(); - options.read(true).write(true).create(true); + options.read(true).write(true).create(create_lock); #[cfg(unix)] { use std::os::unix::fs::OpenOptionsExt; @@ -148,6 +149,9 @@ impl RecoveryStore { File::open(&self.directory) .and_then(|f| f.sync_all()) .map_err(|_| "temporal_namespace_parent_fsync")?; + } else if !create_lock { + let m=fs::symlink_metadata(&directory).map_err(|_|"temporal_directory_read")?; + if !m.is_dir()||m.file_type().is_symlink(){return Err("temporal_directory_unsafe".into());}private_metadata(&m,true)?; } else if directory.exists() { private_directory(&directory)?; } @@ -1173,6 +1177,8 @@ impl TemporalAbortTicket { /// Metadata-only erasure capability: deliberately not a TemporalToken accepted /// by active commit/resume. Neither expired sample bytes nor fit authority escape. pub(crate) struct TemporalRetirementInspection { + metadata_kind:String, + retired:bool, descriptor: RecoveryDescriptor, run: String, observed_ns: u64, @@ -1180,6 +1186,9 @@ pub(crate) struct TemporalRetirementInspection { token: TemporalToken, } impl TemporalRetirementInspection { + pub(crate) fn metadata_kind(&self)->&str{&self.metadata_kind} + pub(crate) fn retired(&self)->bool{self.retired} + pub(crate) fn descriptor(&self) -> &RecoveryDescriptor { &self.descriptor } @@ -1213,6 +1222,40 @@ impl RecoveryStore { return Err("recovery_receiver_clock_regression".into()); } Ok(Some(TemporalRetirementInspection { + metadata_kind:head.schema.clone(), + retired:head.retired_at_ms.is_some(), + descriptor: head.descriptor.clone(), + run: head.run.clone(), + observed_ns: now_ns, + expired: now_ns / 1_000_000 >= head.descriptor.expires_at_ms, + token: head.token()?, + })) + } + _ => Err("temporal_head_witness_mismatch_or_pending".into()), + } + } + /// Owner status discovery performs no mkdir/open-create/chmod or payload + /// read. Missing/corrupt metadata never creates a replacement lock/head. + pub(crate) fn inspect_temporal_read_only(&self,now_ns:u64)->Result,String>{ + let _serial=self.serial.lock().map_err(|_|"recovery_store_unavailable")?; + self.verify_directory_read_only()?; + match fs::symlink_metadata(self.temporal_dir()){ + Err(e)if e.kind()==std::io::ErrorKind::NotFound=>return Ok(None), + Err(_)=>return Err("temporal_directory_read".into()), + Ok(m)if !m.is_dir()||m.file_type().is_symlink()=>return Err("temporal_directory_unsafe".into()), + Ok(_)=>{} + } + let locked=self.temporal_lock_mode(false,false)?; + let c=self.temporal_components(&locked.directory)?; + match (c.head, c.latest) { + (None, None) => Ok(None), + (Some(head), Some(latest)) if head == latest => { + if now_ns < head.receiver_ns { + return Err("recovery_receiver_clock_regression".into()); + } + Ok(Some(TemporalRetirementInspection { + metadata_kind:head.schema.clone(), + retired:head.retired_at_ms.is_some(), descriptor: head.descriptor.clone(), run: head.run.clone(), observed_ns: now_ns, diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_selection_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_selection_store.rs new file mode 100644 index 0000000000..28f928f96b --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_selection_store.rs @@ -0,0 +1,523 @@ +//! Version2 of the SAME protected selection pointer and immutable journal. +//! Only current protected Complete assessment writes it. Cold loading trusts +//! that owner-private journal, then checks every exact immutable dependency; +//! it never creates a new recovery/Complete/evaluation authority after expiry. +use super::*; +use crate::room_learner::temporal_model_input::TemporalModel; +use crate::room_learner::{ + temporal_evaluation as data, temporal_qualification::*, temporal_wire::*, +}; +const POINTER: &str = "model-selection-v1.json"; +const SCHEMA: &str = "ruview.room-learner.model-selection.v2"; +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(crate) struct TemporalSelectionTarget { + pub context: CaptureContext, + pub model: TemporalModelIdentityWire, + pub training_proof_sha256: String, + pub held_out_evidence_sha256: String, + pub evaluation_lookup_sha256: String, + pub evaluation_receipt_sha256: String, + pub assessment_lookup_sha256: String, + pub assessment_sha256: String, + pub qualification_sha256: String, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(crate) struct TemporalSelectionReceipt { + pub schema: String, + pub owner_scope: String, + pub installation_id: String, + pub revision: u64, + pub selected_at_unix_ms: u64, + pub selected: TemporalSelectionTarget, +} +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub(crate) struct TemporalSelectionEnvelope { + pub receipt_sha256: String, + pub receipt: TemporalSelectionReceipt, +} +/// No serde/Clone constructor; obtained only through this store's protected +/// current pointer+same immutable journal validation or a current write. +pub(crate) struct StoredTemporalSelection { + pub(super) envelope: TemporalSelectionEnvelope, + model: TemporalModel, + pub(super) proof: data::TemporalTrainProofData, + pub(super) held: data::TemporalHeldOutEvidenceData, + pub(super) evaluation: data::TemporalEvaluationReceiptData, + pub(super) qualification: TemporalQualificationEnvelope, + pub(super) assessment: TemporalAssessmentEnvelope, +} +impl StoredTemporalSelection { + pub(crate) fn envelope(&self) -> &TemporalSelectionEnvelope { + &self.envelope + } + pub(crate) fn model(&self) -> &TemporalModel { + &self.model + } + pub(crate) fn held(&self) -> &data::TemporalHeldOutEvidenceData { + &self.held + } + pub(crate) fn qualification(&self) -> &TemporalQualificationEnvelope { + &self.qualification + } + pub(crate) fn capture_generation_id(&self) -> Result { + let id = &self.proof.body.identity; + crate::room_learner::qualification::hash(&( + "ruview.room-learner.capture-generation.v2", + &id.original_deadline.id, + &id.run, + &id.context_digest, + )) + } + pub(crate) fn identity(&self) -> TemporalSelectedIdentityWire { + let r = &self.envelope.receipt; + let q = &self.qualification.receipt.summary; + TemporalSelectedIdentityWire { + selection_sha256: self.envelope.receipt_sha256.clone(), + selection_revision: r.revision, + model_kind: WholeTemporalKind::ManagedTemporalV3, + model_sha256: q.model_sha256.clone(), + verified_fit_record_sha256: q.verified_fit_record_sha256.clone(), + training_proof_sha256: q.training_proof_sha256.clone(), + held_out_evidence_sha256: q.held_out_evidence_sha256.clone(), + complete_record_sha256: q.complete_record_sha256.clone(), + evaluation_receipt_sha256: q.evaluation_receipt_sha256.clone(), + qualification_sha256: q.receipt_sha256.clone(), + policy_sha256: q.policy_sha256.clone(), + context_digest: q.context_digest.clone(), + room_digest: q.room_digest.clone(), + source_node_ids: r + .selected + .context + .sources + .iter() + .map(|s| s.source_id) + .collect(), + selected_at_unix_ms: r.selected_at_unix_ms, + evaluated_at_unix_ms: q.evaluated_at_unix_ms, + authority_scope: q.authority_scope.clone(), + } + } +} +impl ArtifactStore { + pub(crate) fn temporal_selection_present(&self) -> Result { + self.verify_owner()?; + let path = self.root.join(POINTER); + if !regular_or_missing(&path)? { + return Ok(false); + } + let value: serde_json::Value = serde_json::from_slice( + &private_read(&path).map_err(|_| "selection pointer unavailable")?, + ) + .map_err(|_| "selection pointer invalid")?; + Ok(value + .pointer("/receipt/schema") + .and_then(serde_json::Value::as_str) + == Some(SCHEMA)) + } + pub(crate) fn selection_revision_any(&self) -> Result { + if self.temporal_selection_present()? { + let bytes = private_read(&self.root.join(POINTER)) + .map_err(|_| "temporal_selection_pointer_unavailable")?; + let e: TemporalSelectionEnvelope = + serde_json::from_slice(&bytes).map_err(|_| "temporal_selection_pointer_schema")?; + if e.receipt.schema != SCHEMA + || e.receipt.owner_scope != self.owner_scope + || e.receipt.installation_id != self.installation_id + || e.receipt.revision == 0 + || e.receipt_sha256 != crate::room_learner::qualification::hash(&e.receipt)? + || private_read( + &self + .root + .join("selections") + .join(format!("{}.json", e.receipt_sha256)), + ) + .map_err(|_| "temporal_selection_journal_unavailable")? + != bytes + { + return Err("temporal_selection_pointer_changed".into()); + } + // CAS identity only; a withdrawn old assessment must not prevent an + // explicitly authorized replacement from fresh protected evidence. + return Ok(e.receipt.revision); + } + Ok(self.load_selection()?.map_or(0, |e| e.receipt.revision)) + } + /// Final CAS for THIS already admitted write, not a history loader. + /// The first operation itself just created one immutable journal entry; + /// missing pointer+that exact entry is still revision0. Every other orphan + /// or concurrent history remains a conservative refusal. + fn revision_for_prepared_temporal_selection( + &self, + prepared: &TemporalSelectionEnvelope, + bytes: &[u8], + ) -> Result { + if regular_or_missing(&self.root.join(POINTER))? { + return self.selection_revision_any(); + } + if prepared.receipt.revision != 1 + || prepared.receipt.schema != SCHEMA + || prepared.receipt.owner_scope != self.owner_scope + || prepared.receipt.installation_id != self.installation_id + || prepared.receipt_sha256 + != crate::room_learner::qualification::hash(&prepared.receipt)? + || serde_json::to_vec(prepared).map_err(|_| "selection_encoding")? != bytes + { + return Err("selection_pointer_changed".into()); + } + let history = self.root.join("selections"); + if !temporal_artifacts::directory(&history, false)? { + return Err("selection_pending_history_missing".into()); + } + let mut entries = + std::fs::read_dir(&history).map_err(|_| "selection_pending_history_unavailable")?; + let only = entries + .next() + .ok_or("selection_pending_history_missing")? + .map_err(|_| "selection_pending_history_unavailable")?; + if only.path() != history.join(format!("{}.json", prepared.receipt_sha256)) + || entries.next().is_some() + || temporal_artifacts::bounded_read(&only.path(), LIMIT)? != bytes + { + return Err("selection_foreign_or_changed_history".into()); + } + Ok(0) + } + pub(crate) fn load_temporal_selection( + &self, + ) -> Result, String> { + if !self.temporal_selection_present()? { + return Ok(None); + } + let bytes = + private_read(&self.root.join(POINTER)).map_err(|_| "temporal_selection_unavailable")?; + let e: TemporalSelectionEnvelope = + serde_json::from_slice(&bytes).map_err(|_| "temporal_selection_schema")?; + let r = &e.receipt; + let t = &r.selected; + if r.schema != SCHEMA + || e.receipt_sha256 != crate::room_learner::qualification::hash(r)? + || r.owner_scope != self.owner_scope + || r.installation_id != self.installation_id + || r.revision == 0 + || r.selected_at_unix_ms == 0 + || r.selected_at_unix_ms > 253402300799999 + { + return Err("temporal_selection_identity".into()); + } + self.validate_context(&t.context)?; + if [ + &t.model.model_sha256, + &t.model.verified_fit_record_sha256, + &t.training_proof_sha256, + &t.held_out_evidence_sha256, + &t.evaluation_lookup_sha256, + &t.evaluation_receipt_sha256, + &t.assessment_lookup_sha256, + &t.assessment_sha256, + &t.qualification_sha256, + ] + .iter() + .any(|s| !hex_digest(s)) + { + return Err("temporal_selection_digest".into()); + } + if private_read( + &self + .root + .join("selections") + .join(format!("{}.json", e.receipt_sha256)), + ) + .map_err(|_| "temporal_selection_journal_missing")? + != bytes + { + return Err("temporal_selection_journal_changed".into()); + } + let model = self.load_exact_temporal_candidate(&t.context, &t.model.model_sha256)?; + if model.artifact_kind() != crate::room_learner::temporal_model_input::managed::SCHEMA_V3 { + return Err("temporal_selection_model_kind".into()); + } + let dir = self + .temporal_evaluation_directory(&t.context, false)? + .ok_or("temporal_selection_metadata_missing")?; + let proof = data::TemporalTrainProofData::decode_data(&temporal_artifacts::bounded_read( + &dir.join(format!("train-proof-{}.json", t.training_proof_sha256)), + data::MAX_METADATA_BYTES, + )?)?; + let held = + data::TemporalHeldOutEvidenceData::decode_data(&temporal_artifacts::bounded_read( + &dir.join(format!( + "heldout-evidence-{}.json", + t.held_out_evidence_sha256 + )), + data::MAX_METADATA_BYTES, + )?)?; + let evaluation = + data::TemporalEvaluationReceiptData::decode_data(&temporal_artifacts::bounded_read( + &dir.join(format!("evaluation-{}.json", t.evaluation_lookup_sha256)), + data::MAX_METADATA_BYTES, + )?)?; + evaluation.validate_data_links(&proof, &held)?; + let qualification: TemporalQualificationEnvelope = + serde_json::from_slice(&temporal_artifacts::bounded_read( + &dir.join(format!("qualification-{}.json", t.qualification_sha256)), + data::MAX_METADATA_BYTES, + )?) + .map_err(|_| "temporal_selection_qualification_schema")?; + let assessment: TemporalAssessmentEnvelope = + serde_json::from_slice(&temporal_artifacts::bounded_read( + &dir.join(format!("assessment-{}.json", t.assessment_lookup_sha256)), + data::MAX_METADATA_BYTES, + )?) + .map_err(|_| "temporal_selection_assessment_schema")?; + assessment.validate_links(&proof, &held, &evaluation, Some(&qualification))?; + #[derive(Deserialize)] + #[serde(deny_unknown_fields)] + struct CurrentAssessment { + schema: String, + model_sha256: String, + context_digest: String, + assessment_sha256: String, + assessment_lookup_sha256: String, + } + let current: CurrentAssessment = serde_json::from_slice(&temporal_artifacts::bounded_read( + &dir.join(format!("latest-assessment-{}.json", t.model.model_sha256)), + data::MAX_METADATA_BYTES, + )?) + .map_err(|_| "temporal_selection_latest_assessment_unavailable")?; + if current.schema != "ruview.room-learner.temporal-current-assessment.v1" + || current.model_sha256 != t.model.model_sha256 + || current.context_digest != t.context.digest() + || current.assessment_sha256 != t.assessment_sha256 + || current.assessment_lookup_sha256 != t.assessment_lookup_sha256 + { + return Err("newer_qualification_failed_or_requires_selection".into()); + } + + let id = &proof.body.identity; + if proof.sha256 != t.training_proof_sha256 + || held.sha256 != t.held_out_evidence_sha256 + || evaluation.sha256 != t.evaluation_receipt_sha256 + || qualification.receipt_sha256 != t.qualification_sha256 + || assessment.receipt_sha256 != t.assessment_sha256 + || id.artifact_sha256 != t.model.model_sha256 + || id.verified_fit_record_sha256 != t.model.verified_fit_record_sha256 + || id.context_digest != t.context.digest() + || id.owner_scope != self.owner_scope + || id.installation_id != self.installation_id + || r.selected_at_unix_ms < qualification.receipt.qualified_at_unix_ms + || r.selected_at_unix_ms >= id.original_deadline.expires_at_unix_ms + { + return Err("temporal_selection_chain_changed".into()); + } + Ok(Some(StoredTemporalSelection { + envelope: e, + model, + proof, + held, + evaluation, + qualification, + assessment, + })) + } + pub(crate) fn save_current_temporal_selection( + &self, + source:&crate::room_learner::temporal_model_input::managed::execution::full_run::VerifiedTemporalEvaluationSnapshot, + assessment: &StoredTemporalAssessment, + expected_revision: u64, + now: u64, + cancel: &std::sync::atomic::AtomicBool, + clock: &mut dyn FnMut() -> Result, + before_install: &mut dyn FnMut() -> Result<(), String>, + after_rename: &mut dyn FnMut() -> Result<(), String>, + ) -> Result { + use std::sync::atomic::Ordering; + source.require_same_store(self)?; + if source.binding() != assessment.binding() { + return Err("temporal_selection_current_complete_changed".into()); + } + let q = assessment + .qualification + .as_ref() + .ok_or("temporal_candidate_not_qualified")?; + assessment + .assessment + .validate(&assessment.evaluation, Some(q))?; + if now / 1_000_000 + >= source + .proof() + .body + .identity + .original_deadline + .expires_at_unix_ms + { + return Err("recovery_expired".into()); + } + if self.selection_revision_any()? != expected_revision { + return Err("selection pointer changed".into()); + } + let t = TemporalSelectionTarget { + context: source.binding().context().clone(), + model: TemporalModelIdentityWire { + model_kind: WholeTemporalKind::ManagedTemporalV3, + model_sha256: source.proof().body.identity.artifact_sha256.clone(), + verified_fit_record_sha256: source + .proof() + .body + .identity + .verified_fit_record_sha256 + .clone(), + }, + training_proof_sha256: source.proof().sha256.clone(), + held_out_evidence_sha256: source.held_out().sha256.clone(), + evaluation_lookup_sha256: assessment.evaluation.lookup_sha256().into(), + evaluation_receipt_sha256: assessment.evaluation.receipt().sha256.clone(), + assessment_lookup_sha256: crate::room_learner::qualification::hash(&( + crate::room_learner::temporal_wire::ASSESSMENT_SCHEMA, + &assessment.evaluation.receipt().sha256, + TemporalQualificationPolicyWire::current(), + ))?, + assessment_sha256: assessment.assessment.receipt_sha256.clone(), + qualification_sha256: q.receipt_sha256.clone(), + }; + let receipt = TemporalSelectionReceipt { + schema: SCHEMA.into(), + owner_scope: self.owner_scope.clone(), + installation_id: self.installation_id.clone(), + revision: expected_revision + .checked_add(1) + .ok_or("selection revision exhausted")?, + selected_at_unix_ms: now / 1_000_000, + selected: t, + }; + let envelope = TemporalSelectionEnvelope { + receipt_sha256: crate::room_learner::qualification::hash(&receipt)?, + receipt, + }; + let bytes = serde_json::to_vec(&envelope).map_err(|_| "temporal_selection_encoding")?; + if cancel.load(Ordering::Acquire) { + return Err("temporal_selection_cancelled".into()); + } + clock()?; + let history = self.root.join("selections"); + private_dir(&history).map_err(|_| "selection history unavailable")?; + let history_path = history.join(format!("{}.json", envelope.receipt_sha256)); + temporal_artifacts::immutable(&history_path, &bytes)?; + temporal_artifacts::recertify_immutable(&history_path, &bytes, LIMIT, &mut |_| { + if cancel.load(Ordering::Acquire) { + return Err("temporal_selection_cancelled".into()); + } + clock()?; + Ok(()) + })?; + if cancel.load(Ordering::Acquire) { + return Err("temporal_selection_cancelled".into()); + } + clock()?; + // The existing atomic pointer replacement is the linearization point. + // Revalidate the current Complete witness/owner/deadline after all + // preceding potentially slow IO, immediately before installing it. + crate::companion::atomic_private_write_checked( + &self.root.join(POINTER), + &bytes, + &mut || { + before_install().map_err(std::io::Error::other)?; + if self + .revision_for_prepared_temporal_selection(&envelope, &bytes) + .map_err(std::io::Error::other)? + != expected_revision + { + return Err(std::io::Error::other("selection pointer changed")); + } + if cancel.load(Ordering::Acquire) { + return Err(std::io::Error::other("temporal_selection_cancelled")); + } + clock().map_err(std::io::Error::other)?; + Ok(()) + }, + &mut || after_rename().map_err(std::io::Error::other), + ) + .map_err(|_| "temporal_selection_pointer_write_unknown")?; + let read = self + .load_temporal_selection()? + .ok_or("temporal_selection_readback_missing")?; + if read.envelope != envelope { + return Err("temporal_selection_readback_changed".into()); + } + Ok(read) + } +} + +impl ArtifactStore { + pub(crate) fn withdraw_temporal_selection( + &self, + sha: &str, + revision: u64, + now: u64, + allowed: &mut dyn FnMut() -> Result<(), String>, + before_install: &mut dyn FnMut() -> Result<(), String>, + after_rename: &mut dyn FnMut() -> Result<(), String>, + ) -> Result { + // Negative CAS needs only the authenticated pointer/journal identity; + // raw expiry, unavailable RF or a failed assessment cannot block Off. + self.verify_owner()?; + let bytes = private_read(&self.root.join(POINTER)) + .map_err(|_| "temporal_selection_pointer_unavailable")?; + let old: TemporalSelectionEnvelope = + serde_json::from_slice(&bytes).map_err(|_| "temporal_selection_pointer_schema")?; + if old.receipt_sha256 != sha + || old.receipt.revision != revision + || old.receipt.owner_scope != self.owner_scope + || old.receipt.installation_id != self.installation_id + || old.receipt.schema != SCHEMA + || old.receipt_sha256 != crate::room_learner::qualification::hash(&old.receipt)? + || self.selection_revision_any()? != revision + { + return Err("selected_pointer_changed".into()); + } + let receipt = crate::room_learner::selection::SelectionReceipt { + schema: "ruview.room-learner.model-selection.v1".into(), + owner_scope: self.owner_scope.clone(), + installation_id: self.installation_id.clone(), + revision: revision + .checked_add(1) + .ok_or("selection_revision_exhausted")?, + selected_at_unix_ms: now / 1_000_000, + selected: None, + previous: None, + }; + let envelope = crate::room_learner::selection::SelectionEnvelope { + receipt_sha256: crate::room_learner::qualification::hash(&receipt)?, + receipt, + }; + let bytes = serde_json::to_vec(&envelope).map_err(|_| "selection_encoding")?; + let history = self.root.join("selections"); + private_dir(&history).map_err(|_| "selection_history_unavailable")?; + let path = history.join(format!("{}.json", envelope.receipt_sha256)); + temporal_artifacts::immutable(&path, &bytes)?; + temporal_artifacts::recertify_immutable(&path, &bytes, LIMIT, &mut |_| allowed())?; + crate::companion::atomic_private_write_checked( + &self.root.join(POINTER), + &bytes, + &mut || { + before_install().map_err(std::io::Error::other)?; + if self + .selection_revision_any() + .map_err(std::io::Error::other)? + != revision + { + return Err(std::io::Error::other("selected_pointer_changed")); + } + allowed().map_err(std::io::Error::other) + }, + &mut || after_rename().map_err(std::io::Error::other), + ) + .map_err(|_| "selection_pointer_write_unknown")?; + if self.load_selection()?.as_ref() != Some(&envelope) { + return Err("selection_pointer_readback_changed".into()); + } + Ok(envelope) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_wire.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_wire.rs new file mode 100644 index 0000000000..555b4ac19b --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_wire.rs @@ -0,0 +1,540 @@ +//! Strict whole-Temporal public DTOs. These are read models/command inputs, +//! never Complete/evaluation/selection capabilities. V1 producers are unchanged. +use super::{ + lifecycle::MutationGuard, + observation::{DataRole, RoomLabel}, + training::EvaluationReport, +}; +use serde::{Deserialize, Serialize}; +pub const STATUS_SCHEMA: &str = "ruview.room-learner.status.v2"; +pub const SELECTION_SCHEMA: &str = "ruview.room-learner.selection.v2"; +pub const SELECTION_STATUS_SCHEMA: &str = "ruview.room-learner.model-selection-status.v2"; +pub const SELECTED_BINARY_SCHEMA: &str = "ruview.room-learner.selected-binary-evidence.v2"; +pub const QUALIFICATION_SCHEMA: &str = "ruview.local-operational-qualification.v3"; +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +pub enum WholeTemporalKind { + #[serde(rename = "ruview.room-learner.managed-temporal-input.v3")] + ManagedTemporalV3, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalModelIdentityWire { + pub model_kind: WholeTemporalKind, + pub model_sha256: String, + pub verified_fit_record_sha256: String, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalCompleteRefsWire { + pub model: TemporalModelIdentityWire, + pub training_proof_sha256: String, + pub complete_record_sha256: String, + pub complete_commitment_sha256: String, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalEvaluationRefsWire { + pub complete: TemporalCompleteRefsWire, + pub held_out_evidence_sha256: String, + pub evaluation_receipt_sha256: String, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalQualificationRefsWire { + pub evaluation: TemporalEvaluationRefsWire, + pub qualification_sha256: String, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalOriginalDeadlineWire { + pub created_at_unix_ms: u64, + pub expires_at_unix_ms: u64, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalEpisodeWire { + pub index: u8, + pub role: DataRole, + pub label: RoomLabel, + pub target_ms: u64, + pub lead_in_ms: u64, + pub native_windows: u16, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalProfileWire { + pub profile_id: String, + pub window_unit: String, + pub native_window_ms: u16, + pub train_windows: u16, + pub held_out_windows: u16, + pub train_episodes: u8, + pub held_out_episodes: u8, + pub windows_per_condition: [u16; 4], + pub train_episodes_per_condition: [u8; 4], + pub held_out_episodes_per_condition: [u8; 4], + pub episodes: Vec, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalCaptureWire { + pub capture_generation_id: String, + pub schedule_sha256: String, + pub continuation_policy_sha256: String, + pub durable_commitment_sha256: Option, + pub original_deadline: TemporalOriginalDeadlineWire, + pub receiver_epoch: String, + pub stage: String, + pub current_episode: Option, + pub role: Option, + pub label: Option, + pub target_ms: Option, + pub lead_in_remaining_ms: u64, + pub observed_ms: u64, + pub retained_windows: u16, + pub durable_windows: u16, + pub sealed_train_episodes: u8, + pub sealed_held_out_episodes: u8, + pub durability_pending: bool, + pub awaiting_ground_truth: bool, + pub confirmation_challenge: Option, + pub failure: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TemporalEvaluationWire { + pub refs: TemporalEvaluationRefsWire, + pub window_unit: String, + pub evaluated_at_unix_ms: u64, + pub report: EvaluationReport, + pub durable: bool, + pub independence_verified: bool, + pub physically_validated: bool, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum WorkflowKindWire { + Idle, + Legacy, + ManagedTemporalV3, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TemporalLearnerStatusWire { + pub schema: String, + pub enabled: bool, + pub available_model_kinds: Vec, + pub temporal_profile: TemporalProfileWire, + pub workflow_kind: WorkflowKindWire, + pub mutation_guard: MutationGuard, + pub context_digest: Option, + pub room_digest: Option, + pub source_node_ids: Vec, + pub recovery: Option, + pub capture: Option, + pub candidate: Option, + pub training_proof_sha256: Option, + pub complete: Option, + pub evaluation: Option, + pub assessment: Option, + pub model_selection: Option, + pub job_state: String, + pub job_operation: Option, + pub error_code: Option, + pub error: Option, + pub legacy_status: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalConfigureRequestV2 { + pub schema: String, + pub model_kind: WholeTemporalKind, + pub expected: MutationGuard, + pub binding_digest: String, + pub revision: u64, + pub source_node_ids: Vec, + pub new_training: bool, + pub recovery_consent: bool, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalPhaseStartRequestV2 { + pub schema: String, + pub expected: MutationGuard, + pub capture_generation_id: String, + pub schedule_sha256: String, + pub episode_index: u8, + pub conditions_confirmed: bool, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalPhaseFinishRequestV2 { + pub schema: String, + pub expected: MutationGuard, + pub capture_generation_id: String, + pub schedule_sha256: String, + pub episode_index: u8, + pub confirmation_challenge: String, + pub conditions_confirmed: bool, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalEvaluateRequestV2 { + pub schema: String, + pub expected: MutationGuard, + pub capture_generation_id: String, + pub refs: TemporalCompleteRefsWire, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalQualifyRequestV2 { + pub schema: String, + pub expected: MutationGuard, + pub capture_generation_id: String, + pub refs: TemporalEvaluationRefsWire, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalSelectRequestV2 { + pub schema: String, + pub expected: MutationGuard, + pub capture_generation_id: String, + pub refs: TemporalQualificationRefsWire, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalCandidateIdentityWire { + pub model_kind: WholeTemporalKind, + pub model_sha256: String, + pub verified_fit_record_sha256: String, + pub context_digest: String, + pub room_digest: String, + pub persisted: bool, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TemporalQualificationPolicyWire { + pub id: String, + pub window_unit: String, + pub minimum_windows_per_condition: u16, + pub minimum_unconditional_sensitivity: f64, + pub maximum_false_positive_rate: f64, + pub minimum_evaluated_window_coverage: f64, + pub coverage_scope: String, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TemporalConditionCriterionWire { + pub condition: RoomLabel, + pub train_windows: u16, + pub held_out_windows: u16, + pub scored_windows: u16, + pub abstained_windows: u16, + /// Subset of abstained, not an additional exclusion from its denominator. + pub missing_windows: u16, + pub declared_train_episodes: u8, + pub declared_held_out_episodes: u8, + pub evaluated_window_coverage: f64, + pub unconditional_sensitivity: Option, + pub false_positive_rate: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TemporalQualificationSummaryWire { + pub schema: String, + pub receipt_sha256: String, + pub model_kind: WholeTemporalKind, + pub model_sha256: String, + pub verified_fit_record_sha256: String, + pub training_lineage_basis: String, + pub training_proof_sha256: String, + pub held_out_evidence_sha256: String, + pub complete_record_sha256: String, + pub evaluation_receipt_sha256: String, + pub context_digest: String, + pub room_digest: String, + pub evaluated_at_unix_ms: u64, + pub policy: TemporalQualificationPolicyWire, + pub policy_sha256: String, + pub label_provenance: String, + pub criteria: Vec, + pub authority_scope: String, + pub window_unit: String, + pub train_windows: u16, + pub held_out_windows: u16, + pub train_episodes: u8, + pub held_out_episodes: u8, + pub independence_verified: bool, + pub physically_validated: bool, + pub measurement_clock_available: bool, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalSelectedIdentityWire { + pub selection_sha256: String, + pub selection_revision: u64, + pub model_kind: WholeTemporalKind, + pub model_sha256: String, + pub verified_fit_record_sha256: String, + pub training_proof_sha256: String, + pub held_out_evidence_sha256: String, + pub complete_record_sha256: String, + pub evaluation_receipt_sha256: String, + pub qualification_sha256: String, + pub policy_sha256: String, + pub context_digest: String, + pub room_digest: String, + pub source_node_ids: Vec, + pub selected_at_unix_ms: u64, + pub evaluated_at_unix_ms: u64, + pub authority_scope: String, +} + +pub const CONFIGURE_REQUEST_SCHEMA: &str = "ruview.room-learner.configure.v2"; +pub const PHASE_START_REQUEST_SCHEMA: &str = "ruview.room-learner.phase-start.v2"; +pub const PHASE_FINISH_REQUEST_SCHEMA: &str = "ruview.room-learner.phase-finish.v2"; +pub const TRAIN_REQUEST_SCHEMA: &str = "ruview.room-learner.train.v2"; +pub const EVALUATE_REQUEST_SCHEMA: &str = "ruview.room-learner.evaluate.v2"; +pub const QUALIFY_REQUEST_SCHEMA: &str = "ruview.room-learner.qualify.v2"; +pub const SELECT_REQUEST_SCHEMA: &str = "ruview.room-learner.select.v2"; +pub const RECOVERY_REQUEST_SCHEMA: &str = "ruview.room-learner.recovery.v2"; +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalTrainRequestV2 { + pub schema: String, + pub expected: MutationGuard, + pub capture_generation_id: String, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalRecoveryRequestV2 { + pub schema: String, + pub model_kind: WholeTemporalKind, + pub expected: MutationGuard, + pub binding_digest: String, + pub revision: u64, + pub source_node_ids: Vec, + pub capture_generation_id: String, +} + +pub const TEMPORAL_POLICY_ID: &str = "ruview.local-operational-temporal-policy.v1"; +impl TemporalQualificationPolicyWire { + pub(crate) fn current() -> Self { + let numeric = super::qualification::QualificationPolicy::default(); + Self { + id: TEMPORAL_POLICY_ID.into(), + window_unit: "native_receiver_2s_window".into(), + minimum_windows_per_condition: 16, + minimum_unconditional_sensitivity: numeric.minimum_unconditional_sensitivity, + maximum_false_positive_rate: numeric.maximum_false_positive_rate, + minimum_evaluated_window_coverage: numeric.minimum_evaluated_cohort_coverage, + coverage_scope: "declared_native_windows_not_wall_time".into(), + } + } +} +#[cfg(test)] +#[path = "temporal_wire_tests.rs"] +mod tests; + +pub const PHASE_CANCEL_REQUEST_SCHEMA: &str = "ruview.room-learner.phase-cancel.v2"; +pub const TRAIN_CANCEL_REQUEST_SCHEMA: &str = "ruview.room-learner.train-cancel.v2"; +pub const RECOVERY_RESTORE_REQUEST_SCHEMA: &str = "ruview.room-learner.recovery-restore.v2"; +pub const RECOVERY_DISCARD_REQUEST_SCHEMA: &str = "ruview.room-learner.recovery-discard.v2"; +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalCurrentActionRequestV2 { + pub schema: String, + pub expected: MutationGuard, + pub capture_generation_id: String, +} +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalDiscardRequestV2 { + pub schema: String, + pub model_kind: WholeTemporalKind, + pub expected: MutationGuard, + pub capture_generation_id: String, +} + +impl TemporalProfileWire { + pub(crate) fn engineering() -> Self { + let labels = [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ]; + let order = [0, 1, 1, 2, 2, 3, 3, 0, 0, 1, 1, 2, 2, 3, 3]; + Self { + profile_id: "native-2s-three-source-56bin-option-a-engineering-v1".into(), + window_unit: "native_receiver_2s_window".into(), + native_window_ms: 2000, + train_windows: 64, + held_out_windows: 64, + train_episodes: 7, + held_out_episodes: 8, + windows_per_condition: [16; 4], + train_episodes_per_condition: [1, 2, 2, 2], + held_out_episodes_per_condition: [2; 4], + episodes: order + .iter() + .enumerate() + .map(|(i, n)| TemporalEpisodeWire { + index: i as u8, + role: if i < 7 { + DataRole::Train + } else { + DataRole::HeldOut + }, + label: labels[*n], + target_ms: if i == 0 { 600000 } else { 30000 }, + lead_in_ms: if i < 7 { 5000 } else { 15000 }, + native_windows: if i == 0 { 16 } else { 8 }, + }) + .collect(), + } + } +} + +pub const ASSESSMENT_SCHEMA: &str = "ruview.local-operational-assessment.v2"; +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TemporalConditionFailureWire { + pub condition: RoomLabel, + pub codes: Vec, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TemporalAssessmentWire { + pub schema: String, + pub receipt_sha256: String, + pub model_kind: WholeTemporalKind, + pub model_sha256: String, + pub verified_fit_record_sha256: String, + pub training_proof_sha256: String, + pub held_out_evidence_sha256: String, + pub complete_record_sha256: String, + pub evaluation_receipt_sha256: String, + pub context_digest: String, + pub room_digest: String, + pub evaluated_at_unix_ms: u64, + pub assessed_at_unix_ms: u64, + pub policy: TemporalQualificationPolicyWire, + pub policy_sha256: String, + pub window_unit: String, + pub outcome: String, + pub failure_reason: Option, + pub qualification_sha256: Option, + pub criteria: Vec, + pub condition_failures: Vec, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)] +pub enum TemporalCurrentEvidenceWire { + None, + Unavailable { + identity: Option, + reason: String, + boot_epoch: Option, + receiver_epoch: Option, + }, + Ready { + identity: TemporalSelectedIdentityWire, + decision: super::model::PresenceDecision, + presence: bool, + occupied_probability: f32, + observed_at_unix_ms: u64, + observation_digest: String, + boot_epoch: String, + receiver_epoch: String, + window_start_unix_ms: u64, + window_end_unix_ms: u64, + valid_until_unix_ms: u64, + source_observed_at_unix_ms: u64, + }, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(deny_unknown_fields)] +pub struct TemporalSelectionStatusWire { + pub schema: String, + pub revision: u64, + pub selected: Option, + pub previous_available: bool, + pub qualification: Option, + pub qualification_error: Option, + pub integrity_error: Option, + pub current: TemporalCurrentEvidenceWire, +} + +pub const PAIRED_PROGRESS_SCHEMA: &str = "ruview.room-learner.progress.v2"; +#[derive(Debug, Clone, Serialize)] +pub struct TemporalPairedCaptureProgressWire { + pub capture_generation_id: String, + pub schedule_sha256: String, + pub stage: String, + pub current_episode: Option, + pub role: Option, + pub label: Option, + pub target_ms: Option, + pub lead_in_remaining_ms: u64, + pub observed_ms: u64, + pub retained_windows: u16, + pub durable_windows: u16, + pub sealed_train_episodes: u8, + pub sealed_held_out_episodes: u8, + pub durability_pending: bool, + pub awaiting_ground_truth: bool, + pub original_deadline: TemporalOriginalDeadlineWire, +} +#[derive(Debug, Clone, Serialize)] +pub struct TemporalPairedSourcesWire { + pub required: u8, + pub ready: u8, +} +#[derive(Debug, Clone, Serialize)] +pub struct TemporalPairedProgressWire { + pub schema: String, + pub success: bool, + pub installation_id: String, + /// Existing public calibration boot identity, never owner MutationGuard. + pub runtime_epoch: String, + pub observed_at_unix_ms: u64, + pub expires_at_unix_ms: u64, + pub room: Option, + pub context_match: bool, + pub stage: String, + pub state: String, + pub model_kind: Option, + pub profile: Option, + pub capture: Option, + pub candidate: Option, + pub evaluation: Option, + pub qualification: Option, + pub selection: Option, + pub sources: Option, + pub save_status: String, + pub blocker: Option, +} + +pub const DESELECT_REQUEST_SCHEMA: &str = "ruview.room-learner.deselect.v2"; +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalDeselectRequestV2 { + pub schema: String, + pub model_kind: WholeTemporalKind, + pub expected: MutationGuard, + pub selection_sha256: String, + pub selection_revision: u64, +} + +pub const RECOVERY_STATUS_SCHEMA: &str = "ruview.room-learner.recovery-status.v2"; +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct TemporalRecoveryStatusWire { + pub schema: String, + pub state: String, + pub capture_generation_id: Option, + pub context_digest: Option, + pub original_deadline: Option, +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_wire_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_wire_tests.rs new file mode 100644 index 0000000000..a6a3afc691 --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_wire_tests.rs @@ -0,0 +1,306 @@ +//! Structural wire examples only: fake digest strings do not grant authority. +use super::*; +fn h(c: char) -> String { + std::iter::repeat(c).take(64).collect() +} +fn guard() -> MutationGuard { + MutationGuard { + runtime_epoch: "synthetic-wire-epoch".into(), + generation: 0, + workflow_revision: 0, + context_digest: None, + phase: None, + } +} +fn model() -> TemporalModelIdentityWire { + TemporalModelIdentityWire { + model_kind: WholeTemporalKind::ManagedTemporalV3, + model_sha256: h('a'), + verified_fit_record_sha256: h('b'), + } +} +fn complete() -> TemporalCompleteRefsWire { + TemporalCompleteRefsWire { + model: model(), + training_proof_sha256: h('c'), + complete_record_sha256: h('d'), + complete_commitment_sha256: h('e'), + } +} +fn eval() -> TemporalEvaluationRefsWire { + TemporalEvaluationRefsWire { + complete: complete(), + held_out_evidence_sha256: h('f'), + evaluation_receipt_sha256: h('1'), + } +} +fn profile() -> TemporalProfileWire { + let conditions = [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ]; + let order = [0, 1, 1, 2, 2, 3, 3, 0, 0, 1, 1, 2, 2, 3, 3]; + TemporalProfileWire { + profile_id: "native-2s-three-source-56bin-option-a-engineering-v1".into(), + window_unit: "native_receiver_2s_window".into(), + native_window_ms: 2000, + train_windows: 64, + held_out_windows: 64, + train_episodes: 7, + held_out_episodes: 8, + windows_per_condition: [16; 4], + train_episodes_per_condition: [1, 2, 2, 2], + held_out_episodes_per_condition: [2; 4], + episodes: order + .iter() + .enumerate() + .map(|(i, c)| TemporalEpisodeWire { + index: i as u8, + role: if i < 7 { + DataRole::Train + } else { + DataRole::HeldOut + }, + label: conditions[*c], + target_ms: if i == 0 { 600000 } else { 30000 }, + lead_in_ms: if i < 7 { 5000 } else { 15000 }, + native_windows: if i == 0 { 16 } else { 8 }, + }) + .collect(), + } +} +#[test] +fn public_identity_kind_is_required_and_legacy_numeric_version_is_not_an_alias() { + let mut value = serde_json::to_value(model()).unwrap(); + assert_eq!( + value["model_kind"], + "ruview.room-learner.managed-temporal-input.v3" + ); + value.as_object_mut().unwrap().remove("model_kind"); + assert!(serde_json::from_value::(value.clone()).is_err()); + value["model_kind"] = serde_json::json!("unknown"); + assert!(serde_json::from_value::(value.clone()).is_err()); + value["model_kind"] = serde_json::json!("ruview.room-learner.managed-temporal-input.v3"); + value["model_version"] = serde_json::json!(1); + assert!(serde_json::from_value::(value).is_err()); +} +#[test] +fn command_inputs_reject_rows_reports_and_verified_booleans() { + let command = TemporalEvaluateRequestV2 { + schema: EVALUATE_REQUEST_SCHEMA.into(), + expected: guard(), + capture_generation_id: h('0'), + refs: complete(), + }; + let original = serde_json::to_value(command).unwrap(); + for key in ["rows", "report", "verified", "complete_witness"] { + let mut value = original.clone(); + value[key] = serde_json::json!(true); + assert!( + serde_json::from_value::(value).is_err(), + "{key}" + ); + } +} +#[test] +fn export_canonical_structural_wire_examples_without_runtime_authority() { + let selected = TemporalSelectedIdentityWire { + selection_sha256: h('2'), + selection_revision: 1, + model_kind: WholeTemporalKind::ManagedTemporalV3, + model_sha256: h('a'), + verified_fit_record_sha256: h('b'), + training_proof_sha256: h('c'), + held_out_evidence_sha256: h('f'), + complete_record_sha256: h('d'), + evaluation_receipt_sha256: h('1'), + qualification_sha256: h('3'), + policy_sha256: super::super::qualification::hash( + &TemporalQualificationPolicyWire::current(), + ) + .unwrap(), + context_digest: h('4'), + room_digest: h('5'), + source_node_ids: vec![15, 16, 17], + selected_at_unix_ms: 1_000_200, + evaluated_at_unix_ms: 1_000_100, + authority_scope: "local_operator_binary_presence".into(), + }; + let policy = TemporalQualificationPolicyWire::current(); + let criteria = [ + RoomLabel::Empty, + RoomLabel::OccupiedStatic, + RoomLabel::OccupiedMoving, + RoomLabel::AdjacentOnly, + ] + .iter() + .map(|label| { + let positive = matches!(label, RoomLabel::OccupiedStatic | RoomLabel::OccupiedMoving); + TemporalConditionCriterionWire { + condition: *label, + train_windows: 16, + held_out_windows: 16, + scored_windows: 16, + abstained_windows: 0, + missing_windows: 0, + declared_train_episodes: if *label == RoomLabel::Empty { 1 } else { 2 }, + declared_held_out_episodes: 2, + evaluated_window_coverage: 1., + unconditional_sensitivity: positive.then_some(1.), + false_positive_rate: (!positive).then_some(0.), + } + }) + .collect(); + let qualification = TemporalQualificationSummaryWire { + schema: QUALIFICATION_SCHEMA.into(), + receipt_sha256: h('3'), + model_kind: WholeTemporalKind::ManagedTemporalV3, + model_sha256: h('a'), + verified_fit_record_sha256: h('b'), + training_lineage_basis: "managed_temporal_training_proof_v1".into(), + training_proof_sha256: h('c'), + held_out_evidence_sha256: h('f'), + complete_record_sha256: h('d'), + evaluation_receipt_sha256: h('1'), + context_digest: h('4'), + room_digest: h('5'), + evaluated_at_unix_ms: 1_000_100, + policy_sha256: super::super::qualification::hash(&policy).unwrap(), + policy, + label_provenance: "temporal-training-proof-v1+sealed-native-heldout-v1".into(), + criteria, + authority_scope: "local_operator_binary_presence".into(), + window_unit: "native_receiver_2s_window".into(), + train_windows: 64, + held_out_windows: 64, + train_episodes: 7, + held_out_episodes: 8, + independence_verified: false, + physically_validated: false, + measurement_clock_available: false, + }; + let idle = TemporalLearnerStatusWire { + schema: STATUS_SCHEMA.into(), + enabled: true, + available_model_kinds: vec![WholeTemporalKind::ManagedTemporalV3], + temporal_profile: profile(), + workflow_kind: WorkflowKindWire::Idle, + mutation_guard: guard(), + context_digest: None, + room_digest: None, + source_node_ids: vec![], + recovery: None, + capture: None, + candidate: None, + training_proof_sha256: None, + complete: None, + evaluation: None, + assessment: None, + model_selection: None, + job_state: "idle".into(), + job_operation: None, + error_code: None, + error: None, + legacy_status: None, + }; + let selection = TemporalSelectionStatusWire { + schema: SELECTION_STATUS_SCHEMA.into(), + revision: 1, + selected: Some(selected.clone()), + previous_available: false, + qualification: Some(qualification.clone()), + qualification_error: None, + integrity_error: None, + current: TemporalCurrentEvidenceWire::Unavailable { + identity: Some(selected.clone()), + reason: "collecting_full_horizon".into(), + boot_epoch: Some("synthetic-boot".into()), + receiver_epoch: Some("synthetic-receiver".into()), + }, + }; + let matching = TemporalPairedProgressWire { + schema: PAIRED_PROGRESS_SCHEMA.into(), + success: true, + installation_id: "synthetic-installation".into(), + runtime_epoch: "synthetic-boot".into(), + observed_at_unix_ms: 1_001_000, + expires_at_unix_ms: 1_011_000, + room: Some(super::super::lifecycle::PublishedProgressRoom { + binding_digest: h('5'), + revision: 1, + }), + context_match: true, + stage: "full_complete".into(), + state: "ready".into(), + model_kind: Some(WholeTemporalKind::ManagedTemporalV3), + profile: Some(profile()), + capture: Some(TemporalPairedCaptureProgressWire { + capture_generation_id: h('0'), + schedule_sha256: h('6'), + stage: "full_complete".into(), + current_episode: None, + role: None, + label: None, + target_ms: None, + lead_in_remaining_ms: 0, + observed_ms: 0, + retained_windows: 128, + durable_windows: 128, + sealed_train_episodes: 7, + sealed_held_out_episodes: 8, + durability_pending: false, + awaiting_ground_truth: false, + original_deadline: TemporalOriginalDeadlineWire { + created_at_unix_ms: 1_000_000, + expires_at_unix_ms: 87_400_000, + }, + }), + candidate: Some(TemporalCandidateIdentityWire { + model_kind: WholeTemporalKind::ManagedTemporalV3, + model_sha256: h('a'), + verified_fit_record_sha256: h('b'), + context_digest: h('4'), + room_digest: h('5'), + persisted: true, + }), + evaluation: None, + qualification: Some(qualification.clone()), + selection: Some(selection.clone()), + sources: Some(TemporalPairedSourcesWire { + required: 3, + ready: 3, + }), + save_status: "not_reported".into(), + blocker: None, + }; + let mismatch = TemporalPairedProgressWire { + schema: PAIRED_PROGRESS_SCHEMA.into(), + success: true, + installation_id: "synthetic-installation".into(), + runtime_epoch: "synthetic-boot".into(), + observed_at_unix_ms: 1_001_000, + expires_at_unix_ms: 1_011_000, + room: None, + context_match: false, + stage: "blocked".into(), + state: "idle".into(), + model_kind: None, + profile: None, + capture: None, + candidate: None, + evaluation: None, + qualification: None, + selection: None, + sources: None, + save_status: "not_reported".into(), + blocker: Some("context_mismatch".into()), + }; + println!("C4C_CANONICAL_PROGRESS_EXAMPLES={}",serde_json::to_string(&serde_json::json!({"fixture_kind":"SYNTHETIC_STRUCTURAL_SERIALIZATION_ONLY_NOT_AUTHORITY","matching_complete":matching,"context_mismatch_redacted":mismatch,"selected_temporal_idle":{"schema":SELECTION_SCHEMA,"success":true,"progress_schema":PAIRED_PROGRESS_SCHEMA,"model_selection":selection}})).unwrap()); + let examples = serde_json::json!({"fixture_kind":"SYNTHETIC_STRUCTURAL_SERIALIZATION_ONLY_NOT_AUTHORITY","idle_response":{"success":true,"status":idle},"candidate_identity":TemporalCandidateIdentityWire{model_kind:WholeTemporalKind::ManagedTemporalV3,model_sha256:h('a'),verified_fit_record_sha256:h('b'),context_digest:h('4'),room_digest:h('5'),persisted:false},"selected_identity":selected,"qualification_summary":qualification,"configure_request":TemporalConfigureRequestV2{schema:CONFIGURE_REQUEST_SCHEMA.into(),model_kind:WholeTemporalKind::ManagedTemporalV3,expected:guard(),binding_digest:h('4'),revision:1,source_node_ids:vec![15,16,17],new_training:true,recovery_consent:true},"evaluate_request":TemporalEvaluateRequestV2{schema:EVALUATE_REQUEST_SCHEMA.into(),expected:guard(),capture_generation_id:h('0'),refs:complete()},"qualify_request":TemporalQualifyRequestV2{schema:QUALIFY_REQUEST_SCHEMA.into(),expected:guard(),capture_generation_id:h('0'),refs:eval()}}); + println!( + "C4C_CANONICAL_PUBLIC_WIRE_EXAMPLES={}", + serde_json::to_string(&examples).unwrap() + ); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs index 18a8a922a3..7db14a66c5 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs @@ -477,4 +477,8 @@ pub(super) use selection_routes::{owner_selection_routes, selection}; #[path = "room_learner_api_progress.rs"] mod paired_progress; -pub(super) use paired_progress::progress; +pub(super) use paired_progress::{progress, progress_dispatch}; + +#[path = "room_learner_api_temporal.rs"] +mod temporal_routes; +pub(super) use temporal_routes::{status_dispatch, selection_dispatch, configure_dispatch, begin_dispatch, finish_dispatch, train_dispatch, evaluate_dispatch, restore_dispatch, discard_dispatch, pause_dispatch, cancel_operation_dispatch}; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_progress.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_progress.rs index eabc6b8bf9..d4255edb40 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_progress.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_progress.rs @@ -38,3 +38,52 @@ pub(crate) async fn progress(State(state):State, grant:Optiondenied(StatusCode::SERVICE_UNAVAILABLE,"progress_unavailable"), } } + + +#[derive(Default, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct ProgressQuery {schema:Option} +fn temporal_denied(status:StatusCode,code:&'static str)->Response { + (status,Json(serde_json::json!({"schema":"ruview.room-learner.progress.v2","success":false,"error_code":code}))).into_response() +} +pub(crate) async fn progress_dispatch(State(state):State, Query(query):Query, grant:Option>)->Response { + if let Some(schema)=query.schema.as_deref() { + if schema!="ruview.room-learner.progress.v2" { return temporal_denied(StatusCode::BAD_REQUEST,"learner_schema_unsupported"); } + } else { + let actual_temporal=state.read().await.room_learning.as_ref().is_some_and(|rt|rt.temporal_v2_state_present()); + if !actual_temporal {return progress(State(state),grant).await;} + } + temporal_progress(State(state),grant).await +} +async fn temporal_progress(State(state):State, grant:Option>)->Response { + let state=state.read().await; + if state.privacy_mode {return temporal_denied(StatusCode::FORBIDDEN,"privacy_redacted");} + let Some(runtime)=state.room_learning.as_ref() else {return temporal_denied(StatusCode::SERVICE_UNAVAILABLE,"learner_unavailable");}; + let Some(store)=state.current_room_store.as_ref() else {return temporal_denied(StatusCode::SERVICE_UNAVAILABLE,"publication_unavailable");}; + let Some(publication)=store.snapshot() else {return temporal_denied(StatusCode::SERVICE_UNAVAILABLE,"publication_unavailable");}; + if state.installation_id.as_deref()!=Some(publication.installation_id.as_str()) {return temporal_denied(StatusCode::SERVICE_UNAVAILABLE,"publication_unavailable");} + let identity=publication.binding_digest.as_ref().map(|binding|PublishedProgressRoom{binding_digest:binding.clone(),revision:publication.revision}); + // The binding hashes installation, published revision and complete room. + // Hold its lock through projection; reject a publication changed since the + // initial envelope snapshot instead of mixing its identity with old counts. + match store.with_current_binding(|binding| { + if binding!=publication.binding_digest.as_deref() {return Err("publication_changed");} + let context=runtime.temporal_progress_context(); + let ready:Vec<_>=context.as_ref().map(|c|c.sources.iter().filter(|s|source_ready(&state,s,std::time::Instant::now())).map(|s|s.source_id).collect()).unwrap_or_default(); + let ready_context=context.as_ref().map(|c|c.digest()); + runtime.paired_temporal_progress_snapshot(&publication.installation_id,&state.calibration_boot_epoch,identity,&ready,ready_context.as_deref(),runtime.receiver_now_ns()) + .map_err(|_|"progress_unavailable") + }) { + Ok(Ok(mut snapshot))=>{ + if let Some(axum::Extension(grant))=grant { + let Some(until)=grant.expires_at.checked_mul(1000) else {return StatusCode::UNAUTHORIZED.into_response();}; + let now=chrono::Utc::now().timestamp_millis().max(0) as u64; + if *grant.revoked.borrow() || until<=snapshot.observed_at_unix_ms || until<=now {return StatusCode::UNAUTHORIZED.into_response();} + snapshot.expires_at_unix_ms=snapshot.expires_at_unix_ms.min(until); + } + Json(snapshot).into_response() + }, + Ok(Err("publication_changed"))=>temporal_denied(StatusCode::CONFLICT,"publication_changed"), + _=>temporal_denied(StatusCode::SERVICE_UNAVAILABLE,"progress_unavailable"), + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_publication.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_publication.rs index 15fdbfd403..ddc36f045b 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_publication.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_publication.rs @@ -21,7 +21,60 @@ fn unavailable(evidence: SelectedBinaryEvidence, reason: &str) -> SelectedBinary } } +fn temporal_current(state: &AppStateInner) -> Option { + let rt = state.room_learning.as_ref()?; + if !rt.temporal_v2_state_present() { + return None; + } + let unavailable = |reason: &str| serde_json::json!({"schema":"ruview.room-learner.selected-binary-evidence.v2","status":"unavailable","identity":null,"reason":reason,"boot_epoch":null,"receiver_epoch":null}); + if state.privacy_mode { + return Some(unavailable("selected_learning_unavailable_in_privacy_mode")); + } + let now = now_ns(rt); + let value = match state.current_room_store.as_ref() { + Some(room) => room + .with_current_binding(|binding| { + rt.temporal_selected_binary_v2(now, binding, Some(&state.calibration_boot_epoch)) + }) + .unwrap_or_else(|_| { + rt.temporal_selected_binary_v2(now, None, Some(&state.calibration_boot_epoch)) + }), + None => rt.temporal_selected_binary_v2(now, None, Some(&state.calibration_boot_epoch)), + } + .unwrap_or_else(|_| unavailable("selected_projection_unavailable")); + if value["status"] == "none" { + None + } else { + Some(value) + } +} +fn temporal_unavailable(mut value: serde_json::Value, reason: &str) -> serde_json::Value { + let identity = value + .get("identity") + .cloned() + .unwrap_or(serde_json::Value::Null); + let boot = value + .get("boot_epoch") + .cloned() + .unwrap_or(serde_json::Value::Null); + let receiver = value + .get("receiver_epoch") + .cloned() + .unwrap_or(serde_json::Value::Null); + value = serde_json::json!({"schema":"ruview.room-learner.selected-binary-evidence.v2","status":"unavailable","identity":identity,"reason":reason,"boot_epoch":boot,"receiver_epoch":receiver}); + value +} +fn temporal_dispatch(state: &AppStateInner) -> bool { + state + .room_learning + .as_ref() + .is_some_and(|rt| rt.temporal_v2_state_present()) +} + pub(crate) fn selection_owns_output(state: &AppStateInner) -> bool { + if temporal_dispatch(state) { + return temporal_current(state).is_some(); + } state.room_learning.as_ref().is_some_and(|runtime| { !matches!( runtime.selected_binary_evidence(now_ns(runtime), None), @@ -112,6 +165,17 @@ fn apply(update: &mut SensingUpdate, proof: Option) { } pub(crate) fn project_live_selection(state: &AppStateInner, update: &mut SensingUpdate) { + if temporal_dispatch(state) { + let proof = temporal_current(state).map(|value| { + if update.source == "esp32" { + value + } else { + temporal_unavailable(value, "selected_native_source_update_required") + } + }); + apply(update, proof); + return; + } let mut evidence = current(state); if update.source != "esp32" { evidence = unavailable(evidence, "selected_native_source_update_required"); @@ -120,6 +184,39 @@ pub(crate) fn project_live_selection(state: &AppStateInner, update: &mut Sensing } pub(crate) fn project_cached_selection(state: &AppStateInner, update: &mut SensingUpdate) { + if temporal_dispatch(state) { + let proof = temporal_current(state).map(|value| { + if value["status"] != "ready" { + return value; + } + let now = state + .room_learning + .as_ref() + .map(|rt| now_ns(rt) / 1_000_000); + let cached = update.selected_binary_evidence.as_ref(); + let valid = update.source == "esp32" + && cached == Some(&value) + && value["observed_at_unix_ms"] + .as_u64() + .zip(now) + .and_then(|(observed, now)| now.checked_sub(observed)) + .is_some_and(|age| { + age <= wifi_densepose_sensing_server::room_learner::observation::MAX_AGE_NS + / 1_000_000 + }) + && value["valid_until_unix_ms"] + .as_u64() + .zip(now) + .is_some_and(|(until, now)| now < until); + if valid { + cached.cloned().expect("matched admitted proof") + } else { + temporal_unavailable(value, "selected_cached_observation_unavailable") + } + }); + apply(update, proof); + return; + } let mut evidence = current(state); if let SelectedBinaryEvidence::Ready { identity, .. } = &evidence { let cached = update.selected_binary_evidence.as_ref(); diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs index b91897b8ba..5f0521f43d 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs @@ -8,9 +8,9 @@ pub(crate) fn owner_selection_routes( ) -> Router { Router::new() .route("/api/v1/room-learner/recovery/enable-current", post(enable_current_recovery)) - .route("/api/v1/room-learner/qualify", post(qualify)) - .route("/api/v1/room-learner/select", post(select_model)) - .route("/api/v1/room-learner/deselect", post(deselect_model)) + .route("/api/v1/room-learner/qualify", post(super::temporal_routes::qualify_dispatch)) + .route("/api/v1/room-learner/select", post(super::temporal_routes::select_dispatch)) + .route("/api/v1/room-learner/deselect", post(super::temporal_routes::deselect_dispatch)) .route_layer(axum::middleware::from_fn_with_state( auth, companion_integration::require_owner, @@ -50,7 +50,7 @@ pub(super) fn enriched_status( status } -fn with_current_room( +pub(super) fn with_current_room( state: &AppStateInner, rt: &RoomLearningRuntime, operation: impl FnOnce() -> Result, diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal.rs new file mode 100644 index 0000000000..b27546bb1e --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal.rs @@ -0,0 +1,618 @@ +//! Version-negotiated adapters. V1 handlers remain the default; malformed V2 +//! requests never enter the legacy workflow. Authority stays in Runtime. +use super::*; +use wifi_densepose_sensing_server::room_learner::temporal_wire::*; + +#[derive(Default, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct SchemaQuery { + schema: Option, +} +fn schema_matches(actual: &str, expected: &str) -> Result<(), Json> { + if actual == expected { + Ok(()) + } else { + Err(failure( + "learner_schema_unsupported", + "Unsupported learner request schema.", + )) + } +} +macro_rules! command { + ($name:ident, $legacy:ty, $temporal:ty) => { + #[derive(serde::Deserialize)] + #[serde(untagged)] + pub(crate) enum $name { + Temporal($temporal), + Legacy($legacy), + } + }; +} +command!( + ConfigureCommand, + ConfigureRequest, + TemporalConfigureRequestV2 +); +command!(StartCommand, BeginRequest, TemporalPhaseStartRequestV2); +command!(FinishCommand, FinishRequest, TemporalPhaseFinishRequestV2); +command!(TrainCommand, TrainRequest, TemporalTrainRequestV2); +command!( + EvaluateCommand, + EvaluationRequest, + TemporalEvaluateRequestV2 +); +command!(QualifyCommand, GuardRequest, TemporalQualifyRequestV2); +command!( + SelectCommand, + selection_routes::SelectRequest, + TemporalSelectRequestV2 +); +command!( + RestoreCommand, + RestoreCaptureRequest, + TemporalRecoveryRequestV2 +); +command!( + DiscardCommand, + DiscardCaptureRequest, + TemporalDiscardRequestV2 +); + +command!(DeselectCommand, GuardRequest, TemporalDeselectRequestV2); +command!(PauseCommand, GuardRequest, TemporalCurrentActionRequestV2); +command!( + CancelOperationCommand, + GuardRequest, + TemporalCurrentActionRequestV2 +); + +pub(crate) async fn status_dispatch( + State(state): State, + Query(query): Query, +) -> Json { + let Some(schema) = query.schema else { + return status(State(state)).await; + }; + if let Err(error) = schema_matches(&schema, STATUS_SCHEMA) { + return error; + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + temporal_status(&state, &rt).await +} +fn temporal_selection( + state: &AppStateInner, + rt: &RoomLearningRuntime, + now: u64, +) -> Result { + let projection = match state.current_room_store.as_ref() { + Some(room) => room + .with_current_binding(|binding| { + rt.temporal_selection_v2(now, binding, Some(&state.calibration_boot_epoch)) + }) + .unwrap_or_else(|_| { + rt.temporal_selection_v2(now, None, Some(&state.calibration_boot_epoch)) + }), + None => rt.temporal_selection_v2(now, None, Some(&state.calibration_boot_epoch)), + }?; + if projection["schema"] != SELECTION_SCHEMA + || projection["success"] != true + || projection["model_selection"]["schema"] != SELECTION_STATUS_SCHEMA + { + return Err("temporal_selection_projection_invalid".into()); + } + Ok(projection) +} +async fn temporal_status( + state: &SharedState, + rt: &Arc, +) -> Json { + let state = state.read().await; + if state.privacy_mode { + return failure( + "learner_privacy_mode", + "Room learning unavailable in privacy mode.", + ); + } + let now = now_ns(rt); + match rt.temporal_status_v2(now).and_then(|mut status| { + status["model_selection"] = temporal_selection(&state, rt, now)?["model_selection"].clone(); + Ok(status) + }) { + Ok(status) => Json(serde_json::json!({"success":true,"status":status})), + Err(e) => failure("learner_status_unavailable", e), + } +} +async fn operation_status( + state: &SharedState, + rt: &Arc, + id: String, + operation: &'static str, +) -> Json { + if id + .parse::() + .ok() + .is_none_or(|value| value.to_string() != id) + { + return failure( + "learner_operation_invalid", + "Operation identity is unavailable.", + ); + } + let mut result = temporal_status(state, rt).await; + if result.0["success"] == true { + result.0["operation_id"] = serde_json::json!(id); + result.0["operation"] = serde_json::json!(operation); + result.0["effect"] = serde_json::json!("started"); + } + result +} +fn field_capture_active(state: &AppStateInner) -> bool { + state.field_model.as_ref().is_some_and(|f| { + matches!(f.status(), CalibrationStatus::Collecting) + || (state.calibration_session_id.is_some() + && matches!(f.status(), CalibrationStatus::Uncalibrated)) + }) +} +fn require_live_context(state: &AppStateInner, rt: &RoomLearningRuntime) -> Result<(), String> { + let context = rt.context().ok_or("learner_source_unready")?; + if context + .sources + .iter() + .all(|spec| source_ready(state, spec, std::time::Instant::now())) + { + Ok(()) + } else { + Err("learner_source_unready".into()) + } +} + +pub(crate) async fn configure_dispatch( + State(state): State, + Json(command): Json, +) -> Json { + let ConfigureCommand::Temporal(request) = command else { + let ConfigureCommand::Legacy(request) = command else { + unreachable!() + }; + return configure(State(state), Json(request)).await; + }; + if let Err(e) = schema_matches(&request.schema, CONFIGURE_REQUEST_SCHEMA) { + return e; + } + if !request.new_training || !request.recovery_consent { + return failure( + "learner_recovery_consent_required", + "New Temporal training requires explicit encrypted recovery consent.", + ); + } + if !valid_binding_digest(&request.binding_digest) + || request.revision == 0 + || !valid_source_node_ids(&request.source_node_ids) + || request.source_node_ids.len() != 3 + { + return failure("learner_context_invalid", "Temporal engineering profile requires a current room revision and exactly three sorted unique sources."); + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + let s = state.read().await; + if field_capture_active(&s) { + return failure( + "learner_capture_conflict", + "Finish or cancel active field calibration first.", + ); + } + let now = std::time::Instant::now(); + let mut sources = Vec::new(); + for &id in &request.source_node_ids { + if !s.node_states.contains_key(&id) { + return failure( + "learner_source_missing", + "A selected node is not receiving.", + ); + } + let Some((key, meta)) = + native_candidates(&s, id, now) + .into_iter() + .max_by(|(ka, a), (kb, b)| { + a.receiver_rate_hz + .total_cmp(&b.receiver_rate_hz) + .then(ka.raw_bins.cmp(&kb.raw_bins)) + }) + else { + return failure( + "learner_source_unready", + "A selected native source/PHY stream is not ready.", + ); + }; + let grid_id = match native::grid_id( + key.n_antennas as usize, + key.raw_bins as usize, + wifi_densepose_hardware::PpduType::from_byte(key.ppdu_type), + ) { + Ok(grid) => grid, + Err(e) => return failure("learner_grid_unsupported", e), + }; + let Some(bandwidth_hz) = meta.bandwidth_hz else { + return failure("learner_source_unready", "Native bandwidth is unavailable."); + }; + sources.push(SourceSpec { + source_id: id, + device_id: format!("esp32-node-{id}"), + grid_id, + feature_count: 56, + signal_kind: RfModality::WifiCsi, + center_frequency_hz: meta.frequency_hz, + bandwidth_hz, + link_id: format!("receiver-{id}-transmitter-unverified"), + link_index: 0, + geometry: None, + }); + } + let context = match rt.bind_context(request.binding_digest, request.revision, sources) { + Ok(context) => context, + Err(e) => return failure("learner_context_invalid", e), + }; + let result = rt.temporal_configure_v2( + &request.expected, + context, + opaque_calibration_id("learner-temporal-run"), + request.recovery_consent, + now_ns(&rt), + ); + drop(s); + match result { + Ok(()) => temporal_status(&state, &rt).await, + Err(e) => failure("learner_configure_failed", e), + } +} + +pub(crate) async fn begin_dispatch( + State(state): State, + Json(command): Json, +) -> Json { + match command { + StartCommand::Legacy(request) => begin_phase(State(state), Json(request)).await, + StartCommand::Temporal(request) => { + if let Err(e) = schema_matches(&request.schema, PHASE_START_REQUEST_SCHEMA) { + return e; + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + let s = state.write().await; + if s.privacy_mode { + return failure( + "learner_privacy_mode", + "Room learning unavailable in privacy mode.", + ); + } + if field_capture_active(&s) { + return failure( + "legacy_empty_capture_active", + "Finish or cancel active field calibration first.", + ); + } + if let Err(e) = require_live_context(&s, &rt) { + return failure("learner_source_unready", e); + } + let result = rt.temporal_begin_phase_v2(request, now_ns(&rt)); + drop(s); + match result { + Ok(()) => temporal_status(&state, &rt).await, + Err(e) => failure("learner_phase_rejected", e), + } + } + } +} +pub(crate) async fn finish_dispatch( + State(state): State, + Json(command): Json, +) -> Json { + match command { + FinishCommand::Legacy(request) => finish_phase(State(state), Json(request)).await, + FinishCommand::Temporal(request) => { + if let Err(e) = schema_matches(&request.schema, PHASE_FINISH_REQUEST_SCHEMA) { + return e; + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + match rt.temporal_finish_phase_v2(request, now_ns(&rt)) { + Ok(()) => temporal_status(&state, &rt).await, + Err(e) => failure("learner_phase_incomplete", e), + } + } + } +} +macro_rules! operation_dispatch { + ($handler:ident, $command:ident, $legacy:ident, $schema:ident, $method:ident, $failure:literal) => { + pub(crate) async fn $handler( + State(state): State, + Json(command): Json<$command>, + ) -> Json { + match command { + $command::Legacy(request) => $legacy(State(state), Json(request)).await, + $command::Temporal(request) => { + if let Err(e) = schema_matches(&request.schema, $schema) { + return e; + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + match rt.$method(request, now_ns(&rt)) { + Ok(id) => operation_status(&state, &rt, id, stringify!($legacy)).await, + Err(e) => failure($failure, e), + } + } + } + } + }; +} +operation_dispatch!( + train_dispatch, + TrainCommand, + train, + TRAIN_REQUEST_SCHEMA, + temporal_train_v2, + "learner_train_rejected" +); +operation_dispatch!( + evaluate_dispatch, + EvaluateCommand, + evaluate, + EVALUATE_REQUEST_SCHEMA, + temporal_evaluate_v2, + "learner_evaluation_rejected" +); + +pub(crate) async fn qualify_dispatch( + State(state): State, + Json(command): Json, +) -> Json { + match command { + QualifyCommand::Legacy(request) => { + selection_routes::qualify(State(state), Json(request)).await + } + QualifyCommand::Temporal(request) => { + if let Err(e) = schema_matches(&request.schema, QUALIFY_REQUEST_SCHEMA) { + return e; + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + let s = state.write().await; + let result = selection_routes::with_current_room(&s, &rt, || { + rt.temporal_qualify_v2(request, now_ns(&rt)) + }); + drop(s); + match result { + Ok(id) => operation_status(&state, &rt, id, "qualify").await, + Err(e) => failure("learner_qualification_rejected", e), + } + } + } +} +pub(crate) async fn select_dispatch( + State(state): State, + Json(command): Json, +) -> Json { + match command { + SelectCommand::Legacy(request) => { + selection_routes::select_model(State(state), Json(request)).await + } + SelectCommand::Temporal(request) => { + if let Err(e) = schema_matches(&request.schema, SELECT_REQUEST_SCHEMA) { + return e; + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + let s = state.write().await; + let result = selection_routes::with_current_room(&s, &rt, || { + rt.temporal_select_v2(request, now_ns(&rt)) + }); + match result { + Ok(value) => Json(value), + Err(e) => failure("learner_selection_rejected", e), + } + } + } +} +pub(crate) async fn selection_dispatch( + State(state): State, + Query(query): Query, +) -> Json { + if let Some(schema) = query.schema.as_deref() { + if let Err(e) = schema_matches(schema, SELECTION_SCHEMA) { + return e; + } + } else { + let actual_temporal = state + .read() + .await + .room_learning + .as_ref() + .is_some_and(|rt| rt.temporal_v2_state_present()); + if !actual_temporal { + return selection_routes::selection(State(state)).await; + } + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + let s = state.read().await; + if s.privacy_mode { + return failure( + "learner_privacy_mode", + "Room learning unavailable in privacy mode.", + ); + } + match temporal_selection(&s, &rt, now_ns(&rt)) { + Ok(mut value) => { + value["progress_schema"] = serde_json::json!(PAIRED_PROGRESS_SCHEMA); + Json(value) + } + Err(e) => failure("learner_selection_unavailable", e), + } +} +pub(crate) async fn restore_dispatch( + State(state): State, + Json(command): Json, +) -> Json { + match command { + RestoreCommand::Legacy(request) => restore_capture(State(state), Json(request)).await, + RestoreCommand::Temporal(request) => { + if let Err(e) = schema_matches(&request.schema, RECOVERY_RESTORE_REQUEST_SCHEMA) { + return e; + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + let now = now_ns(&rt); + let context = match rt.temporal_recovery_context_v2( + &request.binding_digest, + request.revision, + &request.source_node_ids, + now, + ) { + Ok(context) => context, + Err(e) => return failure("learner_recovery_restore_failed", e), + }; + let s = state.read().await; + if field_capture_active(&s) { + return failure( + "learner_capture_conflict", + "Finish or cancel active field calibration first.", + ); + } + if context + .sources + .iter() + .any(|spec| !source_ready(&s, spec, std::time::Instant::now())) + { + return failure( + "learner_recovery_source_unready", + "Restore the exact native source/PHY streams before recovery.", + ); + } + let result = rt.temporal_restore_v2( + &request.expected, + &context, + &request.capture_generation_id, + now, + ); + drop(s); + match result { + Ok(()) => temporal_status(&state, &rt).await, + Err(e) => failure("learner_recovery_restore_failed", e), + } + } + } +} +pub(crate) async fn discard_dispatch( + State(state): State, + Json(command): Json, +) -> Json { + match command { + DiscardCommand::Legacy(request) => discard_capture(State(state), Json(request)).await, + DiscardCommand::Temporal(request) => { + if let Err(e) = schema_matches(&request.schema, RECOVERY_DISCARD_REQUEST_SCHEMA) { + return e; + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + match rt.temporal_discard_v2( + &request.expected, + &request.capture_generation_id, + now_ns(&rt), + ) { + Ok(()) => temporal_status(&state, &rt).await, + Err(e) => failure("learner_recovery_discard_failed", e), + } + } + } +} + +pub(crate) async fn pause_dispatch( + State(state): State, + Json(command): Json, +) -> Json { + match command { + PauseCommand::Legacy(request) => cancel_phase(State(state), Json(request)).await, + PauseCommand::Temporal(request) => { + if let Err(e) = schema_matches(&request.schema, PHASE_CANCEL_REQUEST_SCHEMA) { + return e; + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + match rt.temporal_pause_phase_v2(request, now_ns(&rt)) { + Ok(()) => temporal_status(&state, &rt).await, + Err(e) => failure("learner_phase_cancel_failed", e), + } + } + } +} +pub(crate) async fn cancel_operation_dispatch( + State(state): State, + Json(command): Json, +) -> Json { + match command { + CancelOperationCommand::Legacy(request) => { + cancel_training(State(state), Json(request)).await + } + CancelOperationCommand::Temporal(request) => { + if let Err(e) = schema_matches(&request.schema, TRAIN_CANCEL_REQUEST_SCHEMA) { + return e; + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + match rt.temporal_cancel_operation_v2(request, now_ns(&rt)) { + Ok(()) => temporal_status(&state, &rt).await, + Err(e) => failure("learner_cancel_failed", e), + } + } + } +} + +pub(crate) async fn deselect_dispatch( + State(state): State, + Json(command): Json, +) -> Json { + match command { + DeselectCommand::Legacy(request) => { + selection_routes::deselect_model(State(state), Json(request)).await + } + DeselectCommand::Temporal(request) => { + if let Err(e) = schema_matches(&request.schema, DESELECT_REQUEST_SCHEMA) { + return e; + } + let rt = match runtime(&state).await { + Ok(rt) => rt, + Err(e) => return e, + }; + let _state = state.write().await; + match rt.temporal_deselect_v2(request, now_ns(&rt)) { + Ok(value) => Json(value), + Err(e) => failure("learner_deselection_rejected", e), + } + } + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs new file mode 100644 index 0000000000..eaba84bfea --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs @@ -0,0 +1,412 @@ +//! Actual Axum extraction/dispatch controls; synthetic state only. +use super::super::temporal_routes::*; +use super::*; +use wifi_densepose_sensing_server::room_learner::temporal_wire::*; +fn temporal_app(state: SharedState) -> Router { + Router::new() + .route("/status", get(status_dispatch)) + .route("/selection", get(selection_dispatch)) + .route("/configure", post(configure_dispatch)) + .route("/phase/start", post(begin_dispatch)) + .route("/phase/finish", post(finish_dispatch)) + .route("/train", post(train_dispatch)) + .route("/evaluate", post(evaluate_dispatch)) + .route("/restore", post(restore_dispatch)) + .route("/discard", post(discard_dispatch)) + .route("/phase/cancel", post(pause_dispatch)) + .route("/train/cancel", post(cancel_operation_dispatch)) + .with_state(state) +} +async fn request_raw( + app: Router, + path: &str, + body: Option, +) -> (StatusCode, Vec) { + let mut request = Request::builder().uri(path); + let body = if let Some(body) = body { + request = request + .method(Method::POST) + .header("content-type", "application/json"); + Body::from(serde_json::to_vec(&body).unwrap()) + } else { + Body::empty() + }; + let response = app.oneshot(request.body(body).unwrap()).await.unwrap(); + let status = response.status(); + let bytes = axum::body::to_bytes(response.into_body(), 65536) + .await + .unwrap(); + (status, bytes.to_vec()) +} +async fn decoded(app: Router, path: &str, body: Option) -> serde_json::Value { + let (status, bytes) = request_raw(app, path, body).await; + assert_eq!( + status, + StatusCode::OK, + "{}", + String::from_utf8_lossy(&bytes) + ); + serde_json::from_slice(&bytes).unwrap() +} +fn configure_v2(expected: MutationGuard) -> serde_json::Value { + serde_json::json!({"schema":CONFIGURE_REQUEST_SCHEMA,"model_kind":"ruview.room-learner.managed-temporal-input.v3", + "expected":expected,"binding_digest":"b".repeat(64),"revision":1,"source_node_ids":[15,16,17], + "new_training":true,"recovery_consent":true}) +} +#[tokio::test] +async fn negotiated_idle_capability_preserves_default_status_and_selection() { + let (_dir, state) = fixture(); + let app = temporal_app(state); + let legacy = decoded(app.clone(), "/status", None).await; + assert!(legacy.get("mutation_guard").is_some()); + assert!(legacy.get("available_model_kinds").is_none()); + let v2 = decoded( + app.clone(), + &format!("/status?schema={STATUS_SCHEMA}"), + None, + ) + .await; + assert_eq!(v2["success"], true); + assert_eq!(v2["status"]["schema"], STATUS_SCHEMA); + assert_eq!(v2["status"]["workflow_kind"], "idle"); + assert_eq!( + v2["status"]["model_selection"]["schema"], + SELECTION_STATUS_SCHEMA + ); + assert!(v2["status"]["model_selection"].get("success").is_none()); + assert_eq!( + v2["status"]["available_model_kinds"], + serde_json::json!([]), + "Runtime without encrypted recovery must not advertise Temporal activation" + ); + let selection = decoded(app, "/selection", None).await; + assert_eq!(selection["schema"], "ruview.room-learner.selection.v1"); + assert_eq!( + selection["progress_schema"], + "ruview.room-learner.progress.v1" + ); +} +#[tokio::test] +async fn unknown_status_schema_fails_without_starting_or_reconfiguring() { + let (_dir, state) = fixture(); + let rt = state.read().await.room_learning.clone().unwrap(); + let before = rt.mutation_guard().unwrap(); + let result = decoded( + temporal_app(state), + "/status?schema=ruview.room-learner.status.v999", + None, + ) + .await; + assert_eq!(result["success"], false); + assert_eq!(result["error_code"], "learner_schema_unsupported"); + assert_eq!( + serde_json::to_value(rt.mutation_guard().unwrap()).unwrap(), + serde_json::to_value(before).unwrap() + ); + assert!(rt.context().is_none()); +} +#[tokio::test] +async fn temporal_configure_unknown_kind_or_injected_authority_cannot_fall_back_to_legacy() { + let (_dir, state) = fixture(); + let rt = state.read().await.room_learning.clone().unwrap(); + let app = temporal_app(state); + let valid = configure_v2(rt.mutation_guard().unwrap()); + for (key, value) in [ + ("model_kind", serde_json::json!("legacy")), + ("report", serde_json::json!({"passed":true})), + ("verified", serde_json::json!(true)), + ("fit_config", serde_json::json!({"epochs":1})), + ] { + let mut body = valid.clone(); + body[key] = value; + let (status, _) = request_raw(app.clone(), "/configure", Some(body)).await; + assert_eq!(status, StatusCode::UNPROCESSABLE_ENTITY, "{key}"); + assert!(rt.context().is_none()); + } + let mut unknown = valid; + unknown["schema"] = serde_json::json!("configure.v999"); + let result = decoded(app, "/configure", Some(unknown)).await; + assert_eq!(result["error_code"], "learner_schema_unsupported"); + assert!(rt.context().is_none()); +} +#[tokio::test] +async fn temporal_configure_requires_exact_profile_and_current_explicit_consent() { + let (_dir, state) = fixture(); + let rt = state.read().await.room_learning.clone().unwrap(); + let app = temporal_app(state); + for (key, value, code) in [ + ( + "recovery_consent", + serde_json::json!(false), + "learner_recovery_consent_required", + ), + ( + "new_training", + serde_json::json!(false), + "learner_recovery_consent_required", + ), + ( + "source_node_ids", + serde_json::json!([15, 16]), + "learner_context_invalid", + ), + ( + "source_node_ids", + serde_json::json!([17, 16, 15]), + "learner_context_invalid", + ), + ] { + let mut body = configure_v2(rt.mutation_guard().unwrap()); + body[key] = value; + assert_eq!( + decoded(app.clone(), "/configure", Some(body)).await["error_code"], + code + ); + assert!(rt.context().is_none()); + } +} +#[tokio::test] +async fn temporal_commands_reject_role_rows_scores_and_unknown_schema_without_legacy_fallback() { + let (_dir, state) = fixture(); + let rt = state.read().await.room_learning.clone().unwrap(); + let app = temporal_app(state); + let start = serde_json::json!({"schema":PHASE_START_REQUEST_SCHEMA,"expected":rt.mutation_guard().unwrap(), + "capture_generation_id":"c".repeat(64),"schedule_sha256":"d".repeat(64),"episode_index":0,"conditions_confirmed":true}); + for (key, value) in [ + ("role", serde_json::json!("HeldOut")), + ("label", serde_json::json!("Empty")), + ("rows", serde_json::json!([])), + ("verified", serde_json::json!(true)), + ] { + let mut body = start.clone(); + body[key] = value; + assert_eq!( + request_raw(app.clone(), "/phase/start", Some(body)).await.0, + StatusCode::UNPROCESSABLE_ENTITY + ); + } + let mut bad = start; + bad["schema"] = serde_json::json!("start.v999"); + assert_eq!( + decoded(app.clone(), "/phase/start", Some(bad)).await["error_code"], + "learner_schema_unsupported" + ); + let train = serde_json::json!({"schema":TRAIN_REQUEST_SCHEMA,"expected":rt.mutation_guard().unwrap(),"capture_generation_id":"c".repeat(64)}); + for key in ["config", "rows", "report", "score"] { + let mut bad = train.clone(); + bad[key] = serde_json::json!({}); + assert_eq!( + request_raw(app.clone(), "/train", Some(bad)).await.0, + StatusCode::UNPROCESSABLE_ENTITY + ); + } + assert!(rt.context().is_none()); +} +#[tokio::test] +async fn negotiated_status_still_obeys_privacy_and_disabled_runtime() { + let (_dir, state) = fixture(); + state.write().await.privacy_mode = true; + let path = format!("/status?schema={STATUS_SCHEMA}"); + let result = decoded(temporal_app(state.clone()), &path, None).await; + assert_eq!(result["error_code"], "learner_privacy_mode"); + assert!(result.get("status").is_none()); + let mut s = state.write().await; + s.privacy_mode = false; + s.room_learning = None; + drop(s); + assert_eq!( + decoded(temporal_app(state), &path, None).await["error_code"], + "learner_disabled" + ); +} + +#[tokio::test] +async fn temporal_selection_is_read_scoped_and_unknown_schema_does_not_use_v1() { + let (_dir, state) = fixture(); + let app = temporal_app(state); + let result = decoded( + app.clone(), + &format!("/selection?schema={SELECTION_SCHEMA}"), + None, + ) + .await; + assert_eq!(result["schema"], SELECTION_SCHEMA); + let text = serde_json::to_string(&result).unwrap(); + for forbidden in [ + "owner_scope", + "mutation_guard", + "runtime_epoch", + "encoder_weights", + "raw_frames", + "source_watermarks", + ] { + assert!( + !text.contains(forbidden), + "read projection leaked {forbidden}" + ); + } + let bad = decoded(app, "/selection?schema=selection.v999", None).await; + assert_eq!(bad["error_code"], "learner_schema_unsupported"); + assert!(bad.get("model_selection").is_none()); +} + +#[tokio::test] +async fn temporal_selection_mutations_keep_the_existing_local_owner_gate() { + let (_dir, state) = fixture(); + let app = owner_selection_routes( + wifi_densepose_sensing_server::bearer_auth::AuthState::from_token("synthetic-owner"), + ) + .with_state(state); + for path in [ + "/api/v1/room-learner/qualify", + "/api/v1/room-learner/select", + ] { + for (peer, token) in [ + ("127.0.0.1:41000", "synthetic-peer-read"), + ("192.0.2.1:41000", "synthetic-owner"), + ] { + let mut request = Request::builder() + .uri(path) + .method(Method::POST) + .header("authorization", format!("Bearer {token}")) + .header("content-type", "application/json") + .body(Body::from("{}")) + .unwrap(); + request.extensions_mut().insert(axum::extract::ConnectInfo( + peer.parse::().unwrap(), + )); + assert_eq!( + app.clone().oneshot(request).await.unwrap().status(), + StatusCode::FORBIDDEN + ); + } + } +} + +#[tokio::test] +async fn discard_and_pause_do_not_accept_ignored_context_or_new_phase_authority() { + let (_dir, state) = fixture(); + let rt = state.read().await.room_learning.clone().unwrap(); + let app = temporal_app(state); + let discard = serde_json::json!({"schema":RECOVERY_DISCARD_REQUEST_SCHEMA,"model_kind":"ruview.room-learner.managed-temporal-input.v3","expected":rt.mutation_guard().unwrap(),"capture_generation_id":"c".repeat(64)}); + for key in [ + "binding_digest", + "revision", + "source_node_ids", + "consent", + "rows", + ] { + let mut bad = discard.clone(); + bad[key] = serde_json::json!(null); + assert_eq!( + request_raw(app.clone(), "/discard", Some(bad)).await.0, + StatusCode::UNPROCESSABLE_ENTITY, + "discard:{key}" + ); + } + for (path, schema) in [ + ("/phase/cancel", PHASE_CANCEL_REQUEST_SCHEMA), + ("/train/cancel", TRAIN_CANCEL_REQUEST_SCHEMA), + ] { + let action = serde_json::json!({"schema":schema,"expected":rt.mutation_guard().unwrap(),"capture_generation_id":"c".repeat(64)}); + for key in ["phase", "role", "new_training", "consent"] { + let mut bad = action.clone(); + bad[key] = serde_json::json!(true); + assert_eq!( + request_raw(app.clone(), path, Some(bad)).await.0, + StatusCode::UNPROCESSABLE_ENTITY, + "{path}:{key}" + ); + } + } + assert!(rt.context().is_none()); +} + +#[tokio::test] +async fn actual_temporal_configure_dispatches_queryless_phone_reads_without_owner_status() { + use wifi_densepose_sensing_server::room_learner::recovery_store::RecoveryStore; + let (dir, state) = fixture(); + let owner = "a".repeat(64); + let path = dir.path().join(&owner).join("SensingServer"); + let store = RecoveryStore::new(&path, &owner, "synthetic-install", Some(&[7; 32])).unwrap(); + let rt = Arc::new( + RoomLearningRuntime::new_with_recovery(&path, &owner, "synthetic-install", store).unwrap(), + ); + state.write().await.room_learning = Some(rt.clone()); + let app = temporal_app(state.clone()); + let capable = decoded( + app.clone(), + &format!("/status?schema={STATUS_SCHEMA}"), + None, + ) + .await; + assert_eq!( + capable["status"]["available_model_kinds"], + serde_json::json!(["ruview.room-learner.managed-temporal-input.v3"]) + ); + let configured = decoded( + app.clone(), + "/configure", + Some(configure_v2(rt.mutation_guard().unwrap())), + ) + .await; + assert_eq!(configured["success"], true, "{configured}"); + for _ in 0..200 { + let _ = decoded( + app.clone(), + &format!("/status?schema={STATUS_SCHEMA}"), + None, + ) + .await; + if rt.temporal_v2_state_present() { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + rt.temporal_v2_state_present(), + "actual full run must be admitted" + ); + let selected = decoded(app.clone(), "/selection", None).await; + assert_eq!(selected["schema"], SELECTION_SCHEMA); + assert_eq!(selected["progress_schema"], PAIRED_PROGRESS_SCHEMA); + assert!(selected.get("status").is_none()); + assert!(!selected.to_string().contains("mutation_guard")); + let progress = Router::new() + .route( + "/progress", + get(super::super::paired_progress::progress_dispatch), + ) + .with_state(state); + let (status, body) = request_raw(progress, "/progress", None).await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); + let body: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!(body["schema"], PAIRED_PROGRESS_SCHEMA); + assert_eq!(body["error_code"], "publication_unavailable"); + assert!(body.get("mutation_guard").is_none()); +} + +#[tokio::test] +async fn queryless_idle_progress_remains_v1_and_unknown_query_cannot_activate() { + let (_dir, state) = fixture(); + let rt = state.read().await.room_learning.clone().unwrap(); + let app = Router::new() + .route( + "/progress", + get(super::super::paired_progress::progress_dispatch), + ) + .with_state(state); + let (status, body) = request_raw(app.clone(), "/progress", None).await; + assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + serde_json::from_slice::(&body).unwrap()["schema"], + "ruview.room-learner.progress.v1" + ); + let (status, body) = request_raw(app, "/progress?schema=progress.v999", None).await; + assert_eq!(status, StatusCode::BAD_REQUEST); + assert_eq!( + serde_json::from_slice::(&body).unwrap()["error_code"], + "learner_schema_unsupported" + ); + assert!(rt.context().is_none()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs index 4d952587fd..c6763579cb 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs @@ -555,3 +555,6 @@ async fn enable_current_recovery_requires_explicit_versioned_consent_and_does_no assert!(rt.status(rt.receiver_now_ns())["context_digest"].is_null()); } } + +#[path = "room_learner_api_temporal_tests.rs"] +mod temporal_contract_tests; From e21875f13aa6d7a02c2b815b4a5582932b2eabe0 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Sun, 8 Nov 2026 13:11:51 +0100 Subject: [PATCH 082/100] fix(learner): serialize owner status capture clock ticks --- ...ecycle_temporal_public_acceptance_tests.rs | 45 ++++++++++++++++++ .../src/room_learner_api_temporal.rs | 4 +- .../src/room_learner_api_temporal_tests.rs | 46 +++++++++++++++++++ 3 files changed, 94 insertions(+), 1 deletion(-) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public_acceptance_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public_acceptance_tests.rs index 39cada3577..2bb5924b52 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public_acceptance_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public_acceptance_tests.rs @@ -1186,3 +1186,48 @@ fn same_accepted_prefix(before: &Value, after: &Value) { #[cfg(all(test, unix))] #[path = "lifecycle_temporal_public_safety_tests.rs"] mod safety; + +#[test] +fn explicit_stale_native_processing_coordinate_still_fails_closed() { + let temp = tempfile::Builder::new().prefix("SYNTHETIC-fullrun-primer-order-").tempdir().unwrap(); + let rt = Arc::new(fixture::runtime(temp.path(), fixture::START)); + let now = rt.receiver_now_ns(); + rt.temporal_configure_v2(&rt.mutation_guard().unwrap(), fixture::context(), "synthetic-primer-order".into(), true, now).unwrap(); + let wait = Instant::now(); + loop { + let status=rt.temporal_status_v2(rt.receiver_now_ns()).unwrap(); + if status["capture"]["stage"]=="ready" {break;} + assert!(wait.elapsed(), ) -> Json { - let state = state.read().await; + // Polling a durable ACK advances the capture clock. Serialize before + // sampling receiver time, just like native ingress and owner mutations. + let state = state.write().await; if state.privacy_mode { return failure( "learner_privacy_mode", diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs index eaba84bfea..3e7b37fea7 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs @@ -410,3 +410,49 @@ async fn queryless_idle_progress_remains_v1_and_unknown_query_cannot_activate() ); assert!(rt.context().is_none()); } + +#[tokio::test] +async fn configured_owner_status_serializes_its_capture_clock_tick_before_sampling_time() { + use wifi_densepose_sensing_server::room_learner::recovery_store::RecoveryStore; + let (dir,state)=fixture(); + let owner="a".repeat(64); + let path=dir.path().join(&owner).join("SensingServer"); + let store=RecoveryStore::new(&path,&owner,"synthetic-install",Some(&[7;32])).unwrap(); + let rt=Arc::new(RoomLearningRuntime::new_with_recovery(&path,&owner,"synthetic-install",store).unwrap()); + state.write().await.room_learning=Some(rt.clone()); + let app=temporal_app(state.clone()); + let configured=decoded(app.clone(),"/configure",Some(configure_v2(rt.mutation_guard().unwrap()))).await; + assert_eq!(configured["success"],true,"{configured}"); + for _ in 0..100 { + let s=decoded(app.clone(),&format!("/status?schema={STATUS_SCHEMA}"),None).await; + if s["status"]["capture"]["stage"]=="ready"{break;} + tokio::time::sleep(std::time::Duration::from_millis(2)).await; + } + let shared_reader=state.read().await; + let task=tokio::spawn(async move{decoded(app,&format!("/status?schema={STATUS_SCHEMA}"),None).await}); + tokio::time::sleep(std::time::Duration::from_millis(20)).await; + let ran_concurrently=task.is_finished(); + drop(shared_reader); + let reply=tokio::time::timeout(std::time::Duration::from_secs(5),task).await.unwrap().unwrap(); + assert_eq!(reply["status"]["capture"]["stage"],"ready","{reply}"); + assert_eq!(reply["status"]["capture"]["retained_windows"],0); + assert_eq!(reply["status"]["capture"]["observed_ms"],0); + assert!(!ran_concurrently,"clock-mutating owner status must not sample/tick concurrently under a shared read lock"); +} + +#[tokio::test(flavor="multi_thread", worker_threads=2)] +async fn concurrent_actual_owner_status_requests_keep_unstarted_full_run_ready() { + use wifi_densepose_sensing_server::room_learner::recovery_store::RecoveryStore; + let (dir,state)=fixture();let owner="a".repeat(64);let path=dir.path().join(&owner).join("SensingServer"); + let store=RecoveryStore::new(&path,&owner,"synthetic-install",Some(&[7;32])).unwrap(); + let rt=Arc::new(RoomLearningRuntime::new_with_recovery(&path,&owner,"synthetic-install",store).unwrap()); + state.write().await.room_learning=Some(rt.clone());let app=temporal_app(state.clone()); + let configured=decoded(app.clone(),"/configure",Some(configure_v2(rt.mutation_guard().unwrap()))).await;assert_eq!(configured["success"],true,"{configured}"); + let mut ready=false;for _ in 0..100{let s=decoded(app.clone(),&format!("/status?schema={STATUS_SCHEMA}"),None).await;if s["status"]["capture"]["stage"]=="ready"{ready=true;break;}tokio::time::sleep(std::time::Duration::from_millis(2)).await;}assert!(ready); + let barrier=Arc::new(tokio::sync::Barrier::new(2));let mut tasks=Vec::new(); + for _ in 0..2 {let app=app.clone();let barrier=barrier.clone();tasks.push(tokio::spawn(async move{let mut failed=None;for _ in 0..200{barrier.wait().await;let s=decoded(app.clone(),&format!("/status?schema={STATUS_SCHEMA}"),None).await;if s["status"]["capture"]["stage"]!="ready"&&failed.is_none(){failed=Some(s);}}failed}));} + let mut errors=Vec::new();for task in tasks{if let Some(error)=tokio::time::timeout(std::time::Duration::from_secs(30),task).await.unwrap().unwrap(){errors.push(error);}} + if let Some(error)=errors.first(){println!("SYNTHETIC_ACTUAL_PARALLEL_OWNER_STATUS_FAILURE={error}");} + assert!(errors.is_empty(),"actual concurrent owner GETs failed an unstarted run: {errors:?}"); + let last=decoded(app,&format!("/status?schema={STATUS_SCHEMA}"),None).await;assert_eq!(last["status"]["capture"]["retained_windows"],0);assert_eq!(last["status"]["capture"]["durable_windows"],0);assert_eq!(last["status"]["capture"]["observed_ms"],0); +} From fd9e729ce945f00a955f866e1e0d07b917af043a Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Sun, 8 Nov 2026 13:54:17 +0100 Subject: [PATCH 083/100] fix(learner): match paired progress by full publication binding --- .../room_learner/lifecycle_temporal_public.rs | 4 +- .../src/room_learner_api_temporal_tests.rs | 56 ++++++++ .../paired_temporal_client_revision.json | 127 ++++++++++++++++++ 3 files changed, 185 insertions(+), 2 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/testdata/paired_temporal_client_revision.json diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public.rs index 36fcc8c14a..3399ceb849 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public.rs @@ -746,14 +746,14 @@ impl RoomLearningRuntime { .map(|s| &s.stored.envelope().receipt.selected.context) }); if let (Some(published), Some(context)) = (result.room.as_ref(), context) { + // The full binding digest includes the monotonic publication revision. + // Context.revision is a separate learner structural fingerprint. if installation_id != self.installation || context.installation_id != installation_id || context.room_digest != published.binding_digest - || context.revision != published.revision || state.selection.temporal.as_ref().is_some_and(|selected| { let c = &selected.stored.envelope().receipt.selected.context; c.room_digest != published.binding_digest - || c.revision != published.revision || c.digest() != context.digest() }) { diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs index 3e7b37fea7..e372c8a732 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs @@ -456,3 +456,59 @@ async fn concurrent_actual_owner_status_requests_keep_unstarted_full_run_ready() assert!(errors.is_empty(),"actual concurrent owner GETs failed an unstarted run: {errors:?}"); let last=decoded(app,&format!("/status?schema={STATUS_SCHEMA}"),None).await;assert_eq!(last["status"]["capture"]["retained_windows"],0);assert_eq!(last["status"]["capture"]["durable_windows"],0);assert_eq!(last["status"]["capture"]["observed_ms"],0); } + +#[tokio::test] +async fn actual_client_publication_and_learner_revision_domains_pair_without_rewriting_context() { + use wifi_densepose_sensing_server::room_learner::recovery_store::RecoveryStore; + let client:serde_json::Value=serde_json::from_str(include_str!("testdata/paired_temporal_client_revision.json")).unwrap(); + let (dir,state)=fixture();let owner="a".repeat(64);let installation=client["installation_id"].as_str().unwrap(); + let path=dir.path().join(&owner).join("SensingServer");let recovery=RecoveryStore::new(&path,&owner,installation,Some(&[7;32])).unwrap(); + let rt=Arc::new(RoomLearningRuntime::new_with_recovery(&path,&owner,installation,recovery).unwrap()); + let publication=crate::companion_room::SharedRoomStore::load(&dir.path().join("publication"),installation).unwrap(); + let publish=|body:serde_json::Value|{let router=crate::companion_room::owner_router(publication.clone());async move{ + let reply=router.oneshot(Request::builder().method(Method::PUT).uri("/api/v1/companion/room").header("content-type","application/json").body(Body::from(body.to_string())).unwrap()).await.unwrap();assert_eq!(reply.status(),StatusCode::OK);let bytes=axum::body::to_bytes(reply.into_body(),65536).await.unwrap();serde_json::from_slice::(&bytes).unwrap() + }}; + let initial=publish(client["publication_request"]["body"].clone()).await; + assert_eq!(initial["binding_digest"],client["binding_digest"]);assert_eq!(initial["revision"],1); + {let mut s=state.write().await;s.room_learning=Some(rt.clone());s.current_room_store=Some(publication.clone());s.installation_id=Some(installation.into());s.calibration_boot_epoch="synthetic-paired-domain-boot".into();} + let mut command=client["configure_request"]["body"].clone();command["expected"]=serde_json::to_value(rt.mutation_guard().unwrap()).unwrap(); + assert_eq!(command["revision"],211247076768791u64); + let app=temporal_app(state.clone());let configured=decoded(app.clone(),"/configure",Some(command)).await;assert_eq!(configured["success"],true,"{configured}"); + let mut ready=false;for _ in 0..100{let s=decoded(app.clone(),&format!("/status?schema={STATUS_SCHEMA}"),None).await;if s["status"]["capture"]["stage"]=="ready"{ready=true;break;}tokio::time::sleep(std::time::Duration::from_millis(2)).await;}assert!(ready); + let context=rt.context().unwrap();assert_eq!(context.revision,211247076768791);assert_eq!(context.room_digest,client["binding_digest"].as_str().unwrap()); + let before_guard=rt.mutation_guard().unwrap();let before=decoded(app.clone(),&format!("/status?schema={STATUS_SCHEMA}"),None).await; + let progress=Router::new().route("/progress",get(super::super::paired_progress::progress_dispatch)).with_state(state.clone()); + let current=decoded(progress.clone(),"/progress",None).await; + println!("SYNTHETIC_ACTUAL_CLIENT_PAIRED_REVISION_SEAM={current}"); + assert_eq!(current["context_match"],true,"{current}");assert_eq!(current["room"]["revision"],1);assert_eq!(current["room"]["binding_digest"],client["binding_digest"]); + assert_eq!(current["sources"]["ready"],3);assert_eq!(current["capture"]["retained_windows"],0);assert_eq!(current["capture"]["durable_windows"],0); + assert_eq!(rt.mutation_guard().unwrap(),before_guard);assert_eq!(rt.context().unwrap(),context); + let after=decoded(app.clone(),&format!("/status?schema={STATUS_SCHEMA}"),None).await; + assert_eq!(before["status"]["capture"]["capture_generation_id"],after["status"]["capture"]["capture_generation_id"]);assert_eq!(before["status"]["capture"]["original_deadline"],after["status"]["capture"]["original_deadline"]); + let mut old_learner_context=context.clone();old_learner_context.revision=1; + assert_ne!(old_learner_context.digest(),context.digest(),"exact selected/capture context digest must retain the learner revision binding"); + let wrong_ready=rt.paired_temporal_progress_snapshot(installation,"synthetic-paired-domain-boot",Some(wifi_densepose_sensing_server::room_learner::lifecycle::PublishedProgressRoom{binding_digest:context.room_digest.clone(),revision:1}),&[15,16,17],Some(&"f".repeat(64)),rt.receiver_now_ns()); + assert_eq!(wrong_ready.unwrap_err(),"progress_context_changed"); + {state.write().await.privacy_mode=true;} + assert_eq!(request_raw(progress.clone(),"/progress",None).await.0,StatusCode::FORBIDDEN); + {let mut s=state.write().await;s.privacy_mode=false;s.installation_id=Some("foreign-installation".into());} + assert_eq!(request_raw(progress.clone(),"/progress",None).await.0,StatusCode::SERVICE_UNAVAILABLE); + {state.write().await.installation_id=Some(installation.into());} + let source_cache={let mut s=state.write().await;let old=s.room_learning_sources.clone();s.room_learning_sources.retain(|key,_|key.node_id!=17);old}; + let missing_source=decoded(progress.clone(),"/progress",None).await;assert_eq!(missing_source["context_match"],true);assert_eq!(missing_source["sources"]["ready"],2); + {state.write().await.room_learning_sources=source_cache;} + let (revoked,rx)=tokio::sync::watch::channel(false);let expiry=chrono::Utc::now().timestamp().max(0)as u64+8; + let grant=wifi_densepose_sensing_server::companion::Grant{device_id:"synthetic-paired-reader".into(),expires_at:expiry,revoked:rx}; + let guarded=Router::new().route("/progress",get(super::super::paired_progress::progress_dispatch)).with_state(state.clone()).layer(axum::Extension(grant)); + let authorized=decoded(guarded.clone(),"/progress",None).await;assert!(authorized["expires_at_unix_ms"].as_u64().unwrap()<=expiry*1000); + revoked.send(true).unwrap();assert_eq!(request_raw(guarded,"/progress",None).await.0,StatusCode::UNAUTHORIZED); + let mut revision=1; + for change in ["same_geometry_new_publication","dimensions","nodes","access_point"] { + let mut room=client["canonical_room"].clone(); + match change{"dimensions"=>room["widthM"]=serde_json::json!(2.6),"nodes"=>room["nodes"][0]["xM"]=serde_json::json!(0.5),"access_point"=>room["accessPoint"]=serde_json::json!({"slot":"north_center","xM":1.25,"yM":0.4,"provenance":"operator_entered","confirmedAtUnixMs":1}),_=>{}} + let changed=publish(serde_json::json!({"expected_revision":revision,"room":room})).await;revision+=1;assert_ne!(changed["binding_digest"],client["binding_digest"]); + let blocked=decoded(progress.clone(),"/progress",None).await;assert_eq!(blocked["context_match"],false,"{change}:{blocked}");assert_eq!(blocked["blocker"],"context_mismatch"); + for key in ["room","model_kind","profile","capture","candidate","evaluation","qualification","selection","sources"]{assert!(blocked[key].is_null(),"{change}:{key}");} + assert_eq!(rt.context().unwrap(),context);assert_eq!(rt.mutation_guard().unwrap(),before_guard); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/testdata/paired_temporal_client_revision.json b/v2/crates/wifi-densepose-sensing-server/src/testdata/paired_temporal_client_revision.json new file mode 100644 index 0000000000..54a117db7d --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/testdata/paired_temporal_client_revision.json @@ -0,0 +1,127 @@ +{ + "schema": "ruview.test.actual-client-paired-revision-seam.v1", + "evidence": "SYNTHETIC_ACTUAL_PRODUCTION_CLIENT_FUNCTIONS_NO_RUNTIME_AUTHORITY", + "source_commit": "c3b89011a42154f8baef6c2ce08b113a3926584c", + "provenance": { + "publication": "actual companionService.publishRoom captured by API mock before network/ack/store import", + "binding": "actual sanitizeSharedRoom/sharedRoomDigest/createRoomCalibrationBinding; Node SHA256 at Expo boundary", + "configure": "actual roomLearnerService.temporalConfigure captured by API mock before network; expected guard is Rust structural idle fixture, NOT a live server permit" + }, + "installation_id": "synthetic-installation", + "published_revision": 1, + "binding_digest": "c020caf87c161f0bf595e1f603aef139045c9362f0d933341537d73225bca30d", + "computed_structural_revision": 211247076768791, + "canonical_room": { + "schema": "ruview.companion.room.v1", + "id": "synthetic-revision-room", + "name": "SYNTHETIC revision probe", + "widthM": 2.5, + "lengthM": 2.5, + "heightM": 2.8, + "nodes": [ + { + "nodeId": 15, + "slot": "north_west", + "xM": 0.4, + "yM": 0.4, + "role": "base_station", + "headingDeg": 45, + "orientationConfirmed": true, + "confirmedAtUnixMs": 1 + }, + { + "nodeId": 16, + "slot": "south_east", + "xM": 2, + "yM": 2, + "role": "receiver", + "headingDeg": 225, + "orientationConfirmed": true, + "confirmedAtUnixMs": 1 + }, + { + "nodeId": 17, + "slot": "south_west", + "xM": 0.4, + "yM": 2, + "role": "receiver", + "headingDeg": 315, + "orientationConfirmed": true, + "confirmedAtUnixMs": 1 + } + ], + "accessPoint": null + }, + "publication_request": { + "origin": "http://127.0.0.1:3000", + "path": "/api/v1/companion/room", + "body": { + "expected_revision": 0, + "room": { + "schema": "ruview.companion.room.v1", + "id": "synthetic-revision-room", + "name": "SYNTHETIC revision probe", + "widthM": 2.5, + "lengthM": 2.5, + "heightM": 2.8, + "nodes": [ + { + "nodeId": 15, + "slot": "north_west", + "xM": 0.4, + "yM": 0.4, + "role": "base_station", + "headingDeg": 45, + "orientationConfirmed": true, + "confirmedAtUnixMs": 1 + }, + { + "nodeId": 16, + "slot": "south_east", + "xM": 2, + "yM": 2, + "role": "receiver", + "headingDeg": 225, + "orientationConfirmed": true, + "confirmedAtUnixMs": 1 + }, + { + "nodeId": 17, + "slot": "south_west", + "xM": 0.4, + "yM": 2, + "role": "receiver", + "headingDeg": 315, + "orientationConfirmed": true, + "confirmedAtUnixMs": 1 + } + ], + "accessPoint": null + } + } + }, + "configure_request": { + "origin": "http://127.0.0.1:3000", + "path": "/api/v1/room-learner/configure", + "body": { + "schema": "ruview.room-learner.configure.v2", + "model_kind": "ruview.room-learner.managed-temporal-input.v3", + "expected": { + "runtime_epoch": "synthetic-wire-epoch", + "generation": 0, + "workflow_revision": 0, + "context_digest": null, + "phase": null + }, + "binding_digest": "c020caf87c161f0bf595e1f603aef139045c9362f0d933341537d73225bca30d", + "source_node_ids": [ + 15, + 16, + 17 + ], + "revision": 211247076768791, + "new_training": true, + "recovery_consent": true + } + } +} From fdf5b8658ef3f3ad606ddb4023f06364aba35e79 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Sun, 8 Nov 2026 14:47:37 +0100 Subject: [PATCH 084/100] fix(learner): keep idle cadence estimates from failing Full2 priming --- .../room_learner/lifecycle_temporal_tests.rs | 16 ++++ .../src/room_learner/temporal_acquisition.rs | 27 ++++-- .../room_learner/temporal_capture_quality.rs | 22 +++-- .../src/room_learner_api_temporal_tests.rs | 83 +++++++++++++++++++ 4 files changed, 136 insertions(+), 12 deletions(-) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs index 04c0241d5c..6f5e58f147 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs @@ -1396,3 +1396,19 @@ fn copy_synthetic_budget_fixture(from:&Path,to:&Path){ } } } + +#[test] +fn historical329_and_fresh343_priming_keep_the_original_cadence_band() { + for fresh in [false,true] { + let directory=tempfile::tempdir().unwrap();let rt=runtime(directory.path(),START); + if fresh {rt.configure_fresh_train_343_capture(&rt.mutation_guard().unwrap(),context(),"synthetic-old-profile-rate".into(),START).unwrap();} + else {rt.configure_temporal_capture(&rt.mutation_guard().unwrap(),context(),"synthetic-old-profile-rate".into(),START).unwrap();} + let now=settle_async_boundary(&rt,START); + let iq=vec![3u8;128]; + let packet=canonicalize_esp32(NativePacketInput{node_id:1,n_antennas:1,raw_bins:64,frequency_hz:2_437_000_000,ppdu_type:PpduType::HtLegacy,sequence:1,received_at_ns:now,receiver_sample_rate_hz:10.,iq_bytes:&iq,flags:Adr018Flags::default(),provenance:CaptureProvenance::LiveReceiverObserved}).unwrap(); + assert_eq!(rt.ingest_native(&packet,now),Err("acquisition_cadence_changed".into()),"fresh343={fresh}"); + assert_eq!(snapshot(&rt,now)["retained_rows"],0); + assert_eq!(snapshot(&rt,now)["durable_windows"],0); + assert_eq!(snapshot(&rt,now)["failure"],"acquisition_cadence_changed"); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_acquisition.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_acquisition.rs index 661b349d6c..e8344de80c 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_acquisition.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_acquisition.rs @@ -428,6 +428,27 @@ impl WindowInputPolicy { pub(crate) fn validate_packet_source( &self, p: &CanonicalPacket, + ) -> Result<(&SourceSpec, &SourceWindowPolicy), String> { + let (spec, policy) = self.validate_packet_source_identity(p)?; + let (lo, hi, _) = policy.receiver_cadence.range(); + if p.receiver_sample_rate_hz < lo as f64 / 1000. + || p.receiver_sample_rate_hz > hi as f64 / 1000. + { + return Err("acquisition_cadence_changed".into()); + } + Ok((spec, policy)) + } + /// Identity-only check for closed Full2 priming, never a window admission. + /// The owning capture derives and checks its non-observing stage first. + pub(super) fn validate_priming_packet_source_identity( + &self, + p: &CanonicalPacket, + ) -> Result<(&SourceSpec, &SourceWindowPolicy), String> { + self.validate_packet_source_identity(p) + } + fn validate_packet_source_identity( + &self, + p: &CanonicalPacket, ) -> Result<(&SourceSpec, &SourceWindowPolicy), String> { let spec = self .context @@ -465,11 +486,7 @@ impl WindowInputPolicy { return Err("acquisition_native_geometry_changed".into()); } } - let (lo, hi, _) = policy.receiver_cadence.range(); - if !p.receiver_sample_rate_hz.is_finite() - || p.receiver_sample_rate_hz < lo as f64 / 1000. - || p.receiver_sample_rate_hz > hi as f64 / 1000. - { + if !p.receiver_sample_rate_hz.is_finite() || p.receiver_sample_rate_hz <= 0.0 { return Err("acquisition_cadence_changed".into()); } Ok((spec, policy)) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs index aa70bc120a..24da3624de 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs @@ -690,16 +690,24 @@ impl TemporalTrainCapture { && !full_run_continuation_fence { return Err("temporal_priming_state".into()); } + // No recorded observation exists in these closed Full2 stages. A + // changing host-arrival estimate may prime identity/replay fences, + // while the frozen recording band remains mandatory in ingest(). + let full_run_idle = self.is_full_run() && matches!(self.stage(), + TemporalCaptureStage::Ready | TemporalCaptureStage::Paused | + TemporalCaptureStage::Preparing | TemporalCaptureStage::Blocked | + TemporalCaptureStage::Complete | TemporalCaptureStage::Frozen | + TemporalCaptureStage::FullComplete); let result: Result<(), String> = (|| { let w = &mut self.workflow; w.check_clock(now)?; - w.session - .managed - .as_ref() - .ok_or("managed_session_required")? - .binding - .native - .validate_packet_source(p)?; + let native = &w.session.managed.as_ref() + .ok_or("managed_session_required")?.binding.native; + if full_run_idle { + native.validate_priming_packet_source_identity(p)?; + } else { + native.validate_packet_source(p)?; + } let t = p.native_frame.timestamp_ns; if p.provenance != CaptureProvenance::LiveReceiverObserved || p.native_frame.provenance.class diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs index e372c8a732..bcb5550190 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs @@ -512,3 +512,86 @@ async fn actual_client_publication_and_learner_revision_domains_pair_without_rew assert_eq!(rt.context().unwrap(),context);assert_eq!(rt.mutation_guard().unwrap(),before_guard); } } + +async fn cadence_priming_fixture(cold_restore:bool)->(tempfile::TempDir,SharedState,Arc,Router,serde_json::Value) { + use wifi_densepose_sensing_server::room_learner::recovery_store::RecoveryStore; + let (dir,state)=fixture();let owner="a".repeat(64);let path=dir.path().join(&owner).join("SensingServer"); + let store=RecoveryStore::new(&path,&owner,"synthetic-install",Some(&[7;32])).unwrap(); + let rt=Arc::new(RoomLearningRuntime::new_with_recovery(&path,&owner,"synthetic-install",store).unwrap()); + state.write().await.room_learning=Some(rt.clone());let app=temporal_app(state.clone()); + let configured=decoded(app.clone(),"/configure",Some(configure_v2(rt.mutation_guard().unwrap()))).await;assert_eq!(configured["success"],true,"{configured}"); + let mut before=serde_json::Value::Null;for _ in 0..100{let s=decoded(app.clone(),&format!("/status?schema={STATUS_SCHEMA}"),None).await;if s["status"]["capture"]["stage"]=="ready"{before=s;break;}tokio::time::sleep(std::time::Duration::from_millis(2)).await;}assert!(before.is_object()); + if !cold_restore{return(dir,state,rt,app,before);} + let context=rt.context().unwrap();let capture_id=before["status"]["capture"]["capture_generation_id"].as_str().unwrap().to_string(); + state.write().await.room_learning=None;drop(rt); + let store=RecoveryStore::new(&path,&owner,"synthetic-install",Some(&[7;32])).unwrap(); + let restored=Arc::new(RoomLearningRuntime::new_with_recovery(&path,&owner,"synthetic-install",store).unwrap()); + restored.temporal_restore_v2(&restored.mutation_guard().unwrap(),&context,&capture_id,restored.receiver_now_ns()).unwrap(); + state.write().await.room_learning=Some(restored.clone()); + let after=decoded(app.clone(),&format!("/status?schema={STATUS_SCHEMA}"),None).await;assert_eq!(after["status"]["capture"]["stage"],"paused","{after}"); + assert_eq!(after["status"]["capture"]["capture_generation_id"],before["status"]["capture"]["capture_generation_id"]);assert_eq!(after["status"]["capture"]["original_deadline"],before["status"]["capture"]["original_deadline"]); + (dir,state,restored,app,after) +} +async fn actual_header_observe(state:&SharedState,sequence:u32) -> f64 { + let packet=wire(15,sequence);let frame=parse_esp32_frame(&packet).unwrap();let mut s=state.write().await; + observe(&mut s,&frame,&packet,std::time::Instant::now()); + s.room_learning_sources.iter().find(|(k,_)|k.node_id==15).unwrap().1.receiver_rate_hz +} +async fn assert_nonobserving_cadence_does_not_fail(cold:bool) { + let (_dir,state,rt,app,before)=cadence_priming_fixture(cold).await; + let rate=actual_header_observe(&state,121).await;assert!(rate>0.&&rate<30.,"fixture must expose a real moving-arrival estimate below recording band: {rate}"); + let after=decoded(app,&format!("/status?schema={STATUS_SCHEMA}"),None).await; + println!("SYNTHETIC_ACTUAL_INGRESS_PRIMING_CADENCE cold={cold} measured_receiver_hz={rate} capture={}",after["status"]["capture"]); + assert_eq!(after["status"]["capture"]["stage"],if cold{"paused"}else{"ready"},"{after}"); + for key in ["capture_generation_id","original_deadline","schedule_sha256","retained_windows","durable_windows","sealed_train_episodes","sealed_held_out_episodes","observed_ms","durable_commitment_sha256","continuation_policy_sha256"]{assert_eq!(after["status"]["capture"][key],before["status"]["capture"][key],"{key}");} + assert_eq!(rt.context().unwrap().revision,1); +} +#[tokio::test] +async fn actual_ingress_ready_zero_window_rate_estimate_does_not_record_or_fail(){assert_nonobserving_cadence_does_not_fail(false).await;} +#[tokio::test] +async fn actual_ingress_cold_paused_zero_window_rate_estimate_does_not_record_or_fail(){assert_nonobserving_cadence_does_not_fail(true).await;} + +#[tokio::test] +async fn actual_ingress_low_cadence_lead_in_has_zero_credit_then_observing_rejects() { + let (_dir,state,rt,app,before)=cadence_priming_fixture(false).await; + let capture=&before["status"]["capture"]; + let start=decoded(app.clone(),"/phase/start",Some(serde_json::json!({"schema":PHASE_START_REQUEST_SCHEMA,"expected":rt.mutation_guard().unwrap(),"capture_generation_id":capture["capture_generation_id"],"schedule_sha256":capture["schedule_sha256"],"episode_index":0,"conditions_confirmed":true}))).await; + assert_eq!(start["success"],true,"{start}"); + let wait=std::time::Instant::now();let mut saw_lead=false;let mut failed=false; + for sequence in 121..221 { + {let mut s=state.write().await;for node in [15,16,17]{let packet=wire(node,sequence);let frame=parse_esp32_frame(&packet).unwrap();observe(&mut s,&frame,&packet,std::time::Instant::now());}} + let current=decoded(app.clone(),&format!("/status?schema={STATUS_SCHEMA}"),None).await; + assert_eq!(current["status"]["capture"]["retained_windows"],0);assert_eq!(current["status"]["capture"]["durable_windows"],0); + if current["status"]["capture"]["stage"]=="failed" { + assert!(saw_lead,"must first prove non-observing lead-in accepted without credit"); + assert_eq!(current["status"]["capture"]["failure"],"acquisition_cadence_changed","{current}");failed=true;break; + } + if current["status"]["capture"]["lead_in_remaining_ms"].as_u64().unwrap()>0 { + saw_lead=true;assert_eq!(current["status"]["capture"]["observed_ms"],0); + } + assert!(wait.elapsed()native::CanonicalPacket { + let bytes=wire(15,sequence);let frame=parse_esp32_frame(&bytes).unwrap(); + native::canonicalize_esp32(native::NativePacketInput{node_id:15,n_antennas:1,raw_bins:64,frequency_hz:2_437_000_000,ppdu_type:frame.ppdu_type,sequence,received_at_ns:rt.receiver_now_ns(),receiver_sample_rate_hz:10.,iq_bytes:&bytes[20..],flags:frame.adr018_flags,provenance:wifi_densepose_sensing_server::room_learner::observation::CaptureProvenance::LiveReceiverObserved}).unwrap() +} +#[tokio::test] +async fn fullrun_priming_still_rejects_replay_and_changed_phy_identity() { + {let (_dir,state,_rt,app,_)=cadence_priming_fixture(false).await;actual_header_observe(&state,121).await;actual_header_observe(&state,121).await; + let s=decoded(app,&format!("/status?schema={STATUS_SCHEMA}"),None).await;assert_eq!(s["status"]["capture"]["failure"],"temporal_priming_source_or_fence");assert_eq!(s["status"]["capture"]["retained_windows"],0);} + {let (_dir,_state,rt,app,_)=cadence_priming_fixture(false).await;let mut p=low_rate_native_packet(&rt,121);p.frequency_hz=2_462_000_000; + assert_eq!(rt.ingest_native(&p,rt.receiver_now_ns()),Err("acquisition_native_source_changed".into())); + let s=decoded(app,&format!("/status?schema={STATUS_SCHEMA}"),None).await;assert_eq!(s["status"]["capture"]["retained_windows"],0);} +} +#[tokio::test] +async fn fullrun_priming_still_rejects_zero_nan_and_nonfinite_rate_metadata() { + for rate in [0.,-1.,f64::NAN,f64::INFINITY] { + let (_dir,_state,rt,app,_)=cadence_priming_fixture(false).await; + let mut p=low_rate_native_packet(&rt,121);p.receiver_sample_rate_hz=rate; + assert_eq!(rt.ingest_native(&p,rt.receiver_now_ns()),Err("acquisition_cadence_changed".into())); + let s=decoded(app,&format!("/status?schema={STATUS_SCHEMA}"),None).await;assert_eq!(s["status"]["capture"]["retained_windows"],0);assert_eq!(s["status"]["capture"]["observed_ms"],0); + } +} From 50d1422e6361df3ce3b825ccff376351ad8f834e Mon Sep 17 00:00:00 2001 From: Codex for Dragan Spiridonov Date: Sun, 8 Nov 2026 16:53:48 +0100 Subject: [PATCH 085/100] fix(ci): keep manual verification runs nonpublishing --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 60e78c2fd2..d874daf580 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -420,6 +420,9 @@ jobs: # workflow, so it doesn't gate the rest of the pipeline. docker-build: name: Docker Build & Test + # Manual CI is verification only. Use the separate canonical publisher + # for an explicitly authorized image publication. + if: github.event_name != 'workflow_dispatch' runs-on: ubuntu-latest needs: [code-quality, test, rust-tests] continue-on-error: true @@ -495,7 +498,7 @@ jobs: name: API Documentation runs-on: ubuntu-latest needs: [docker-build] - if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/main' && github.event_name != 'workflow_dispatch' permissions: contents: write # gh-pages deploy needs write (GITHUB_TOKEN is read-only by default -> 403) steps: @@ -540,7 +543,8 @@ jobs: name: Notify runs-on: ubuntu-latest needs: [code-quality, test, rust-tests, ui-tests, performance-test, docker-build, docs] - if: always() + # This job also creates releases and sends external notifications. + if: always() && github.event_name != 'workflow_dispatch' permissions: contents: write # required by softprops/action-gh-release # GitHub Actions does not allow `secrets.X` directly in step-level `if:` From 5509412eed9c44ffc93cd292621d68adf4cc9edf Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Mon, 9 Nov 2026 13:06:19 +0100 Subject: [PATCH 086/100] fix(learner): restore eight-step capture plan and safe retry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep the prior profile readable, reserve check captures outside fitting, and reconcile failed writer outcomes without restarting or extending recovery. Implemented by Codex in Dragan’s authorized local build session. --- scripts/ci/temporal_test_profiles.py | 5 +- scripts/ci/test_temporal_test_profiles.py | 20 +- .../src/room_learner/lifecycle.rs | 10 + .../src/room_learner/lifecycle_temporal.rs | 56 +++- .../lifecycle_temporal_evaluation.rs | 10 +- .../room_learner/lifecycle_temporal_fit.rs | 5 +- .../lifecycle_temporal_full_run_tests.rs | 261 +++++++++++++++++- .../room_learner/lifecycle_temporal_public.rs | 60 ++-- .../room_learner/lifecycle_temporal_tests.rs | 114 ++++++++ .../room_learner/managed_schedule_contract.rs | 60 +++- .../managed_schedule_contract_tests.rs | 57 ++++ .../src/room_learner/recovery_store.rs | 39 ++- .../src/room_learner/recovery_store_tests.rs | 21 ++ .../room_learner/temporal_capture_quality.rs | 109 ++++++-- .../src/room_learner/temporal_checkpoint.rs | 12 +- .../room_learner/temporal_checkpoint_tests.rs | 15 + .../src/room_learner/temporal_durability.rs | 81 ++++-- .../src/room_learner/temporal_evaluation.rs | 65 +++-- .../temporal_evaluation_data_tests.rs | 64 ++++- .../temporal_evaluation_projection.rs | 71 +++-- .../room_learner/temporal_evaluation_store.rs | 6 +- .../src/room_learner/temporal_execution.rs | 15 +- .../room_learner/temporal_execution_tests.rs | 69 +++++ .../room_learner/temporal_fit_admission.rs | 29 +- .../src/room_learner/temporal_full_run.rs | 65 +++-- .../temporal_full_run_checkpoint.rs | 97 +++++-- .../room_learner/temporal_held_out_capture.rs | 21 +- .../temporal_model_input_managed.rs | 33 ++- .../temporal_model_input_managed_tests.rs | 56 ++++ .../room_learner/temporal_qualification.rs | 53 ++-- .../temporal_recovery_commitment.rs | 7 + .../room_learner/temporal_selection_store.rs | 5 +- .../src/room_learner/temporal_wire.rs | 32 +++ .../src/room_learner/temporal_wire_tests.rs | 18 +- .../src/room_learner_api.rs | 2 +- .../src/room_learner_api_selection.rs | 6 + .../src/room_learner_api_temporal.rs | 19 +- .../src/room_learner_api_temporal_tests.rs | 64 +++++ .../src/room_learner_api_tests.rs | 1 + 39 files changed, 1471 insertions(+), 262 deletions(-) diff --git a/scripts/ci/temporal_test_profiles.py b/scripts/ci/temporal_test_profiles.py index 0cd980ef92..285c158cca 100644 --- a/scripts/ci/temporal_test_profiles.py +++ b/scripts/ci/temporal_test_profiles.py @@ -20,11 +20,12 @@ 'room_learner::lifecycle::temporal_control::fit_control::tests::c3_old_completion_cannot_fail_a_replacement_job_with_different_cancel_identity', 'room_learner::lifecycle::temporal_control::fit_control::tests::c3_optional_actual_summary_failure_keeps_durable_diagnostic_model', 'room_learner::lifecycle::temporal_control::fit_control::tests::c3_worker_unwind_keeps_saved_history_but_never_adopts_current_candidate'] +OPERATOR8 = PREFIX + 'full_run_tests::operator_eight_actual_reserved_checks_fit_complete_cold_and_evaluate' C4 = ['room_learner::lifecycle::temporal_control::full_run_tests::c4a_actual_seven_train_fit_frozen_ack_and_exact_cold_model', 'room_learner::lifecycle::temporal_control::full_run_tests::c4a_actual_frozen_cold_missing_corrupt_wrong_kind_bytes_and_expiry_fail_closed', 'room_learner::lifecycle::temporal_control::full_run_tests::c4a_owner_cancel_drains_actual_prepared_frozen_writer_before_synthetic_erase', 'room_learner::lifecycle::temporal_control::full_run_tests::c4a_actual_frozen_lost_ack_same_intent_repair_installs_only_after_exact_ack', - 'room_learner::lifecycle::temporal_control::full_run_tests::c4a_running_compute_cancel_cannot_produce_accepted_fit_or_frozen'] + 'room_learner::lifecycle::temporal_control::full_run_tests::c4a_running_compute_cancel_cannot_produce_accepted_fit_or_frozen', OPERATOR8] C4B_PREFIX = PREFIX + "held_out_tests::" C4B = ['room_learner::lifecycle::temporal_control::held_out_tests::c4b_actual_complete_cold_restore_requires_model_owner_context_and_original_deadline', 'room_learner::lifecycle::temporal_control::held_out_tests::c4b_actual_complete_owner_cancel_and_retirement_deny_reactivation', 'room_learner::lifecycle::temporal_control::held_out_tests::c4b_actual_complete_rehashed_role_phase_binding_floor_and_deadline_negatives', 'room_learner::lifecycle::temporal_control::held_out_tests::c4b_actual_native_train_frozen_eight_heldout_two_cold_interruptions_complete'] @@ -34,7 +35,7 @@ def policy(root): source = root / 'v2/crates/wifi-densepose-sensing-server/src/room_learner' components = [('C2', source / 'lifecycle_temporal_tests.rs', [C2], [(C2, True, 1)])] components += [('C1', source / 'lifecycle_temporal_tests.rs', [C1], [(C1, True, 1)])] - components += [('C3', source / 'lifecycle_temporal_fit_tests.rs', C3, [(PREFIX + 'fit_control::tests::', False, 12)]), ('C4', source / 'lifecycle_temporal_full_run_tests.rs', C4, [(PREFIX + 'full_run_tests::', False, 8)]), ('budget343', source / 'temporal_budget_profile_tests.rs', BUDGET, [(name, True, 1) for name in BUDGET])] + components += [('C3', source / 'lifecycle_temporal_fit_tests.rs', C3, [(PREFIX + 'fit_control::tests::', False, 12)]), ('C4', source / 'lifecycle_temporal_full_run_tests.rs', C4, [(PREFIX + 'full_run_tests::', False, 9)]), ('budget343', source / 'temporal_budget_profile_tests.rs', BUDGET, [(name, True, 1) for name in BUDGET])] b_marker = source / 'lifecycle_temporal_held_out_tests.rs' if not b_marker.is_file(): raise ValueError('C4B source marker is required; missing source is not coverage') diff --git a/scripts/ci/test_temporal_test_profiles.py b/scripts/ci/test_temporal_test_profiles.py index d3f11b0ad4..5290aba5ac 100644 --- a/scripts/ci/test_temporal_test_profiles.py +++ b/scripts/ci/test_temporal_test_profiles.py @@ -166,8 +166,26 @@ def test_current_policy_registers_exact_c4c_and_preserves_existing_components(se self.assertEqual(components[0][3], [(profiles.C2, True, 1)]) self.assertEqual(components[1][3], [(profiles.C1, True, 1)]) self.assertEqual(components[2][3], [(profiles.PREFIX + 'fit_control::tests::', False, 12)]) - self.assertEqual(components[3][3], [(profiles.PREFIX + 'full_run_tests::', False, 8)]) + self.assertEqual(components[3][3], [(profiles.PREFIX + 'full_run_tests::', False, 9)]) self.assertEqual(components[4][3], [(name, True, 1) for name in profiles.BUDGET]) self.assertEqual(components[5][3], [(profiles.C4B_PREFIX, False, 11)]) + + +class OperatorEightClassificationTests(unittest.TestCase): + def component(self):return [('operator8',None,[profiles.OPERATOR8],[(profiles.OPERATOR8,True,1)])] + def test_exact_operator_case_is_mandatory_active_in_release(self): + result=profiles.validate(self.component(),{profiles.OPERATOR8,profiles.OPERATOR8+'_lookalike'},set(),'release-run') + self.assertEqual(result[0]['names'],[profiles.OPERATOR8]) + def test_debug_registration_is_explicitly_ignored_not_passed(self): + result=profiles.validate(self.component(),{profiles.OPERATOR8},{profiles.OPERATOR8},'debug-check') + self.assertEqual(result[0]['names'],[profiles.OPERATOR8]) + with self.assertRaises(ValueError):profiles.validate(self.component(),{profiles.OPERATOR8},set(),'debug-check') + def test_missing_or_ignored_release_case_fails(self): + with self.assertRaises(ValueError):profiles.validate(self.component(),set(),set(),'release-run') + with self.assertRaises(ValueError):profiles.validate(self.component(),{profiles.OPERATOR8},{profiles.OPERATOR8},'release-run') + def test_actual_policy_carries_operator_case_in_cached_c4_batch(self): + c4=next(c for c in profiles.policy(pathlib.Path(__file__).resolve().parents[2]) if c[0]=='C4') + self.assertIn(profiles.OPERATOR8,c4[2]);self.assertEqual(c4[3],[(profiles.PREFIX+'full_run_tests::',False,9)]) + if __name__=='__main__':unittest.main() diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs index f15e68adfc..7a9b599230 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs @@ -91,6 +91,9 @@ pub struct RoomLearningRuntime { installation: String, store: Arc, recovery_store: Option>, + operator_recovery_store: Option>, + operator_profile_selected: AtomicBool, + operator_profile_initialized: AtomicBool, inner: Mutex, #[cfg(test)] reclamation_probe:Mutex>>, @@ -99,6 +102,8 @@ pub struct RoomLearningRuntime { #[cfg(test)] temporal_fit_hook: Mutex>>, #[cfg(test)] + temporal_status_profile_hook: Mutex>>, + #[cfg(test)] temporal_evidence_io_hook: MutexResult<(),String> + Send + Sync>>>, clock_origin: std::time::Instant, clock_epoch_ns: u64, @@ -186,6 +191,7 @@ impl RoomLearningRuntime { .as_nanos() .try_into() .map_err(|_| "receiver clock outside bounds")?; + let operator_recovery_store = recovery_store.as_ref().map(|store| store.operator_eight_scope()).transpose()?.map(Arc::new); let runtime = Self { clock_origin, clock_epoch_ns, @@ -194,9 +200,13 @@ impl RoomLearningRuntime { installation: installation.into(), store: Arc::new(ArtifactStore::new(directory, owner, installation)?), recovery_store: recovery_store.map(std::sync::Arc::new), + operator_recovery_store, + operator_profile_selected: AtomicBool::new(false), + operator_profile_initialized: AtomicBool::new(false), #[cfg(test)] reclamation_probe:Mutex::new(None), #[cfg(test)] panic_after_reclamation_ack:AtomicBool::new(false), #[cfg(test)] temporal_fit_hook:Mutex::new(None), + #[cfg(test)] temporal_status_profile_hook:Mutex::new(None), #[cfg(test)] temporal_evidence_io_hook:Mutex::new(None), inner: Mutex::new(Inner { generation: 0, diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs index f968b7c8d1..c62e5aeecd 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs @@ -220,10 +220,12 @@ impl RoomLearningRuntime { pub(super) fn temporal_store( &self, ) -> Result<&super::super::recovery_store::RecoveryStore, String> { - self.recovery_store - .as_deref() + if self.operator_profile_selected.load(Ordering::Acquire) { self.operator_recovery_store.as_deref() } else { self.recovery_store.as_deref() } .ok_or("temporal_recovery_unavailable".into()) } + fn temporal_store_arc(&self) -> Result, String> { + if self.operator_profile_selected.load(Ordering::Acquire) { self.operator_recovery_store.clone() } else { self.recovery_store.clone() }.ok_or("temporal_recovery_unavailable".into()) + } /// Deliberately PRIVATE: current public configure always constructs Legacy. /// Refuses to discard existing capture/candidate/selection/recovery state. fn configure_temporal_capture( @@ -240,7 +242,7 @@ impl RoomLearningRuntime { now, |schedule, run, fit, now, _| { // Existing-work guard precedes this off-lock producer. - let store=self.recovery_store.clone().ok_or("temporal_recovery_unavailable")?; + let store=self.temporal_store_arc()?; TemporalTrainCapture::prepare_async(schedule, run, fit, now, store) }, ) @@ -248,7 +250,7 @@ impl RoomLearningRuntime { /// Fresh genesis only; never upgrades a running old private capture. fn configure_temporal_full_run(&self,expected:&MutationGuard,context:CaptureContext,run:String,now:u64)->Result<(),String>{ self.configure_temporal_capture_with_custody(expected,context,run,now,|s,r,f,n,_,w|{ - let store=self.recovery_store.clone().ok_or("temporal_recovery_unavailable")?; + let store=self.temporal_store_arc()?; TemporalTrainCapture::prepare_full_run_async(s,r,f,n,store,w) }) } @@ -257,7 +259,7 @@ impl RoomLearningRuntime { /// metadata can chain a new343 generation in the SAME store. fn configure_fresh_train_343_capture(&self,expected:&MutationGuard,context:CaptureContext,run:String,now:u64)->Result<(),String>{ self.configure_temporal_capture_with(expected,context,run,now,|schedule,run,fit,now,_|{ - let store=self.recovery_store.clone().ok_or("temporal_recovery_unavailable")?; + let store=self.temporal_store_arc()?; TemporalTrainCapture::prepare_async_fresh_train_343(schedule,run,fit,now,store) }) } @@ -292,10 +294,7 @@ impl RoomLearningRuntime { { return Err("temporal_context_owner_or_clock".into()); } - let schedule = ManagedScheduleContract::build( - context.clone(), - LeadInPolicy::Train5HeldOut15EngineeringV1, - )?; + let schedule = ManagedScheduleContract::build(context.clone(), if self.operator_profile_selected.load(Ordering::Acquire) { LeadInPolicy::OperatorEightStepV1 } else { LeadInPolicy::Train5HeldOut15EngineeringV1 })?; let fit = FitConfig::for_source_count(context.sources.len()); fit.validate_context(&context)?; let mut state = self.command_state(expected)?; @@ -589,9 +588,7 @@ impl RoomLearningRuntime { } }; core.enable_async_restore( - self.recovery_store - .clone() - .ok_or("temporal_recovery_unavailable")?, + self.temporal_store_arc()?, now, )?; let mut state = self.inner.lock().map_err(|_| "learner state unavailable")?; @@ -731,3 +728,38 @@ mod public_workflow; #[path="lifecycle_temporal_selection.rs"] pub(super) mod selected_control; + +impl RoomLearningRuntime { + fn retry_failed_temporal_capture(&self, expected:&MutationGuard, capture_generation:&str, now:u64) -> Result<(),String> { + let (snapshot,writer,context,post_guard,generation)={ + let mut state=self.command_state(expected)?; + if busy(&state.job) {return Err("temporal_retry_operation_busy".into());} + let generation=state.generation; + let Some(CaptureWorkflow::Temporal(c))=state.capture.as_mut()else{return Err(PRIVATE_KIND.into())}; + if c.owner_generation!=generation || c.fit_job.is_some() || c.evidence_job.is_some() {return Err("temporal_retry_operation_busy".into());} + let core=c.core.as_mut().ok_or("temporal_initial_barrier_unconfirmed")?; + if core.public_generation_id()!=capture_generation {return Err("learner_capture_identity_changed".into());} + let snapshot=core.failed_retry_state()?; + let writer=core.writer().ok_or("temporal_writer_required")?; + let context=c.context.clone(); + (snapshot,writer,context,self.guard_for(&state),generation) + }; + // Never join/repair/recertify while holding the ingress mutex. Terminal + // owner drain fences captured Writer Arcs before account erasure. + writer.reconcile_failed(now)?; + let at=self.receiver_now_ns().max(now); + let witness=FullRunRecoveryWitness{runtime_epoch:self.runtime_epoch.clone(),generation,context:context.clone()}; + let mut core=TemporalTrainCapture::cold_restore_full_run(self.temporal_store()?,self.store.clone(),&context,at,&self.runtime_epoch,at,witness)?; + core.apply_failed_retry_state(snapshot,at)?; + core.enable_async_restore(self.temporal_store_arc()?,at)?; + let mut state=self.inner.lock().map_err(|_|"learner state unavailable")?; + if self.guard_for(&state)!=post_guard || state.generation!=generation || state.capture_context()!=Some(&context) { + drop(state);drop(core);return Err("learner_state_changed".into()); + } + let Some(CaptureWorkflow::Temporal(c))=state.capture.as_mut()else{drop(state);drop(core);return Err("learner_state_changed".into())}; + let retired=c.core.replace(core);c.lead_in_until=None;c.failure=None;c.last_now_ns=at;c.public_train_requested=false; + state.input_error=None; + drop(state);drop(retired); + Ok(()) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_evaluation.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_evaluation.rs index 6776a0f496..5e527362a0 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_evaluation.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_evaluation.rs @@ -108,10 +108,7 @@ impl RoomLearningRuntime { ) -> Result { let executor = tokio::runtime::Handle::try_current() .map_err(|_| "temporal_evaluation_executor_unavailable")?; - let recovery = self - .recovery_store - .clone() - .ok_or("temporal_recovery_unavailable")?; + let recovery = self.temporal_store_arc()?; let store = self.store.clone(); let (input, identity, lifetime) = self.reserve_temporal_evidence(expected, now, purpose)?; let attempt = identity.attempt; @@ -237,10 +234,7 @@ impl RoomLearningRuntime { expected: &MutationGuard, now: u64, ) -> Result<(), String> { - let recovery = self - .recovery_store - .clone() - .ok_or("temporal_recovery_unavailable")?; + let recovery = self.temporal_store_arc()?; let store = self.store.clone(); let (input, identity, lifetime) = self.reserve_temporal_evidence(expected, now, EvidencePurpose::Selection)?; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit.rs index 01c7fd79e7..a1a42b4319 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_fit.rs @@ -239,10 +239,7 @@ impl RoomLearningRuntime { let executor = tokio::runtime::Handle::try_current() .map_err(|_| "temporal_fit_executor_unavailable")?; let store = self.store.clone(); - let recovery = self - .recovery_store - .clone() - .ok_or("temporal_recovery_unavailable")?; + let recovery = self.temporal_store_arc()?; let retired_job; let retired_fit; let (input, identity, lifetime) = { diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs index 0314904b45..19d2f05fdd 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs @@ -280,6 +280,8 @@ fn record_train_uncached(root: &Path) -> (Arc, u64) { .unwrap(); let mut sequence = 0; let mut starts = 0; + let mut forced_pause = false; + let mut forced_resume = false; for episode in schedule .episodes() .iter() @@ -289,17 +291,69 @@ fn record_train_uncached(root: &Path) -> (Arc, u64) { let progress = fixture::snapshot(&runtime, now); match progress["stage"].as_str().unwrap() { "ready" | "paused" => { + let resume_revision = if progress["stage"] == "paused" + && forced_pause && !forced_resume + { + Some((runtime.temporal_store().unwrap().load_temporal(now) + .unwrap().unwrap().token.data().revision, + progress["durable_windows"].as_u64().unwrap())) + } else { None }; now = fixture::async_lead_end(&runtime, now); starts += 1; runtime .tick_temporal_capture(&runtime.mutation_guard().unwrap(), now) .unwrap(); + if let Some((before, durable_rows)) = resume_revision { + // Lead readiness is only recertification. Admit the + // actual SEGMENT ACK while every source remains live. + let wait = Instant::now(); + loop { + sequence += 1; + for source in 1..=3 { + runtime.ingest_native( + &packet(source, sequence, now, episode.condition), now, + ).unwrap(); + } + let pending = fixture::snapshot(&runtime, now)["durability_pending"] == true; + now += fixture::PERIOD; + if !pending { break; } + assert!(wait.elapsed() < Duration::from_secs(30)); + std::thread::sleep(Duration::from_nanos(fixture::PERIOD)); + } + let after = runtime.temporal_store().unwrap().load_temporal(now) + .unwrap().unwrap().token.data().revision; + assert_eq!(after, before.checked_add(1).unwrap(), + "the explicit resumed segment is exactly one durable transition"); + let resumed = fixture::snapshot(&runtime, now); + assert_eq!(resumed["stage"], "observing"); + assert_eq!(resumed["durable_windows"], durable_rows); + assert_eq!(resumed["retained_rows"], durable_rows, + "the SEGMENT ACK must not be confused with a new WINDOW ACK"); + forced_resume = true; + } } "needs_confirmation" => break, "failed" => panic!("synthetic C4A phase{} failed: {progress}", episode.index), _ => {} } let pending = fixture::snapshot(&runtime, now)["durability_pending"] == true; + if !forced_pause && episode.index == 0 && !pending + && progress["stage"] == "observing" + && progress["durable_windows"].as_u64().is_some_and(|n| n >= 1) + && progress["retained_rows"] == progress["durable_windows"] + { + let before = runtime.temporal_store().unwrap().load_temporal(now) + .unwrap().unwrap().token.data().revision; + runtime.pause_temporal_capture(&runtime.mutation_guard().unwrap(), now).unwrap(); + now = fixture::settle_async_boundary(&runtime, now); + assert_eq!(fixture::snapshot(&runtime, now)["stage"], "paused"); + let after = runtime.temporal_store().unwrap().load_temporal(now) + .unwrap().unwrap().token.data().revision; + assert_eq!(after, before.checked_add(1).unwrap(), + "the explicit legal pause is exactly one durable transition"); + forced_pause = true; + continue; + } sequence += 1; for source in 1..=3 { match runtime.ingest_native(&packet(source, sequence, now, episode.condition), now) @@ -324,6 +378,8 @@ fn record_train_uncached(root: &Path) -> (Arc, u64) { now = fixture::settle_async_boundary(&runtime, now); } let complete = fixture::snapshot(&runtime, now); + assert!(forced_pause && forced_resume, + "the native producer must exercise an actual durable pause and resume"); assert_eq!(complete["stage"], "complete"); assert_eq!( complete["sealed_train_episodes"], @@ -420,11 +476,9 @@ async fn c4a_actual_seven_train_fit_frozen_ack_and_exact_cold_model() { .load_temporal(now) .unwrap() .unwrap(); - assert_eq!( - loaded.token.data().revision, - 78, - "actual genesis +7 segments +64 windows +7 seals" - ); + let train_revision = loaded.token.data().revision; + assert!(train_revision >= 80, + "all initial segments, 64 windows and 7 seals plus the forced pause/resume must be durable; extra legal pauses add transitions"); let (compute_tx, compute_rx) = std::sync::mpsc::sync_channel(1); let (compute_release_tx, compute_release_rx) = std::sync::mpsc::sync_channel(1); let mut compute_gate = ReleaseGate(Some(compute_release_tx)); @@ -599,7 +653,7 @@ async fn c4a_actual_seven_train_fit_frozen_ack_and_exact_cold_model() { .unwrap(); assert_eq!( loaded.token.data().revision, - 79, + train_revision.checked_add(1).unwrap(), "one actual durable Frozen transition" ); let data: serde_json::Value = serde_json::from_slice(loaded.prefix.unwrap().bytes()).unwrap(); @@ -687,7 +741,7 @@ async fn c4a_actual_seven_train_fit_frozen_ack_and_exact_cold_model() { .token .data() .revision, - 79 + train_revision.checked_add(1).unwrap() ); drop(runtime); let cold_now = post_ack + 1_000_000_000; @@ -923,6 +977,8 @@ async fn c4a_owner_cancel_drains_actual_prepared_frozen_writer_before_synthetic_ async fn c4a_actual_frozen_lost_ack_same_intent_repair_installs_only_after_exact_ack() { let directory = tempfile::tempdir().unwrap(); let (rt, now) = record_train(directory.path()); + let train_revision = rt.temporal_store().unwrap().load_temporal(now) + .unwrap().unwrap().token.data().revision; let at = fit_ready(&rt, now).await; let writer = { let s = rt.inner.lock().unwrap(); @@ -952,7 +1008,8 @@ async fn c4a_actual_frozen_lost_ack_same_intent_repair_installs_only_after_exact .unwrap() .unwrap() .token; - assert_eq!(token.data().revision, 79); + assert_eq!(token.data().revision, train_revision.checked_add(1).unwrap(), + "Frozen adds exactly one transition to the actual completed Train prefix"); rt.repair_temporal_capture(&rt.mutation_guard().unwrap(), now) .unwrap(); now = fixture::settle_async_boundary(&rt, now); @@ -1018,3 +1075,191 @@ async fn c4a_running_compute_cancel_cannot_produce_accepted_fit_or_frozen() { rt.cancel_temporal_durability(&rt.mutation_guard().unwrap()) .unwrap(); } + +#[tokio::test(flavor="multi_thread",worker_threads=2)] +#[cfg_attr(debug_assertions,ignore="native-persistence operator journey requires Release; CI must explicitly execute this case")] +async fn operator_eight_actual_reserved_checks_fit_complete_cold_and_evaluate(){ + assert!(!cfg!(debug_assertions)); + let directory=tempfile::tempdir().unwrap(); + let mut runtime=Arc::new(fixture::runtime(directory.path(),fixture::START)); + runtime.temporal_select_profile_v2(Some("native-2s-three-source-56bin-operator-eight-step-v1")).unwrap(); + let store=runtime.temporal_store_arc().unwrap(); + runtime.configure_temporal_capture_with_custody(&runtime.mutation_guard().unwrap(),fixture::context(),"synthetic-eight-actual".into(),fixture::START, + move|schedule,run,mut config,now,_,witness|{config.epochs=2;TemporalTrainCapture::prepare_full_run_async(schedule,run,config,now,store,witness)}).unwrap(); + let mut now=fixture::settle_async_boundary(&runtime,fixture::START); + let original=fixture::snapshot(&runtime,now)["descriptor"].clone(); + let schedule=ManagedScheduleContract::build(fixture::context(),LeadInPolicy::OperatorEightStepV1).unwrap(); + let mut sequence=0u32; + // The same stopped-first-capture seam: segment write actually commits but + // its ACK is lost; a too-fast packet fails before any retained window. + now=fixture::async_lead_end(&runtime,now); + let (initial_enter_tx,initial_enter_rx)=std::sync::mpsc::sync_channel(1); + let (initial_release_tx,initial_release_rx)=std::sync::mpsc::sync_channel(1); + {let state=runtime.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()};let writer=c.core.as_ref().unwrap().writer().unwrap();writer.lose_next_ack();writer.hold_next_after_publish(initial_enter_tx,initial_release_rx);} + let mut initial_gate=ReleaseGate(Some(initial_release_tx)); + for source in 1..=3u8{let at=now+u64::from(source)*1000;runtime.ingest_native(&packet(source,1,at,RoomLabel::Empty),at).unwrap();} + initial_enter_rx.recv_timeout(Duration::from_secs(10)).unwrap(); + assert_eq!(fixture::snapshot(&runtime,now+3000)["durability_pending"],true); + let failed_at=now+1_001_000; + assert!(runtime.ingest_native(&packet(1,2,failed_at,RoomLabel::Empty),failed_at).unwrap_err().starts_with("temporal_observed_gap_or_rate:")); + initial_gate.release(); + let failed=runtime.temporal_status_v2(failed_at).unwrap(); + assert_eq!(failed["capture"]["stage"],"failed"); + let id=failed["capture"]["capture_generation_id"].as_str().unwrap().to_owned(); + let before=runtime.mutation_guard().unwrap();now=failed_at+1_000_000_000; + runtime.retry_failed_temporal_capture(&before,&id,now).unwrap(); + let retried=runtime.temporal_status_v2(now).unwrap(); + assert_eq!(retried["capture"]["capture_generation_id"],id); + assert_eq!(retried["capture"]["stage"],"paused"); + assert_eq!(retried["capture"]["durability_pending"],false); + assert_eq!(retried["capture"]["retained_windows"],0); + assert_eq!(retried["capture"]["durable_windows"],0); + assert!(retried["capture"]["failure"].is_null()); + assert_original_lifetime(&fixture::snapshot(&runtime,now)["descriptor"],&original); + assert_eq!(runtime.mutation_guard().unwrap().generation,before.generation); + assert!(runtime.mutation_guard().unwrap().workflow_revision>before.workflow_revision); + sequence=2; + for episode in schedule.episodes(){ + let supervised=Instant::now(); + loop{ + let progress=fixture::snapshot(&runtime,now); + match progress["stage"].as_str().unwrap(){ + "ready"|"paused"=>{now=fixture::async_lead_end(&runtime,now);runtime.tick_temporal_capture(&runtime.mutation_guard().unwrap(),now).unwrap();} + "needs_confirmation"=>break, + "failed"=>panic!("operator episode{} failed: {progress}",episode.index), + _=>{} + } + sequence+=1; + for source in 1..=3{match runtime.ingest_native(&packet(source,sequence,now,episode.condition),now){Ok(())=>{},Err(e)if e=="temporal_storage_busy_pause"=>break,Err(e)=>panic!("operator ingest: {e}")}} + now+=fixture::PERIOD; + if fixture::snapshot(&runtime,now)["durability_pending"]==true{std::thread::sleep(Duration::from_nanos(fixture::PERIOD));} + assert!(supervised.elapsed()serde_json::Value{ + use sha2::{Digest,Sha256}; + let loaded=rt.temporal_store().unwrap().load_temporal(at).unwrap().unwrap(); + let data:serde_json::Value=serde_json::from_slice(loaded.prefix.unwrap().bytes()).unwrap(); + let payload=&data["payload"]; + for key in ["rows","evidence","identities"]{assert_eq!(payload["accepted"][key].as_array().unwrap().len(),20);} + assert_eq!(payload["common"]["ledger"]["spans"].as_array().unwrap().len(),20); + assert_eq!(payload["stage"]["bridge"]["seals"].as_array().unwrap().len(),2); + let digest=|v:&serde_json::Value|format!("{:x}",Sha256::digest(serde_json::to_vec(v).unwrap())); + serde_json::json!({"rows":digest(&payload["accepted"]["rows"]),"evidence":digest(&payload["accepted"]["evidence"]), + "identities":digest(&payload["accepted"]["identities"]),"spans":digest(&payload["common"]["ledger"]["spans"]), + "seals":digest(&payload["stage"]["bridge"]["seals"])}) + } + let known_prefix=prefix20(&runtime,now); + let known_rows=fixture::snapshot(&runtime,now)["rows_sha256"].clone();assert!(known_rows.is_string()); + for committed_lost_ack in [true,false]{ + let lead=fixture::async_lead_end(&runtime,now); + let writer={let state=runtime.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()};c.core.as_ref().unwrap().writer().unwrap()}; + let gate=if committed_lost_ack{ + let(entered_tx,entered_rx)=std::sync::mpsc::sync_channel(1);let(release_tx,release_rx)=std::sync::mpsc::sync_channel(1); + writer.lose_next_ack();writer.hold_next_after_publish(entered_tx,release_rx);Some((entered_rx,ReleaseGate(Some(release_tx)))) + }else{writer.fail_next_publication();None}; + sequence+=1; + // One packet arms the pending segment in both variants. + runtime.ingest_native(&packet(1,sequence,lead+1000,RoomLabel::OccupiedStatic),lead+1000).unwrap(); + let stop=lead+1_001_000; + if let Some((entered,mut release))=gate{ + for source in 2..=3u8{let at=lead+u64::from(source)*1000;runtime.ingest_native(&packet(source,sequence,at,RoomLabel::OccupiedStatic),at).unwrap();} + entered.recv_timeout(Duration::from_secs(10)).unwrap(); + assert!(writer.has_failed_repair(),"the HEAD/WITNESS publication is real before the ACK is lost"); + sequence+=1; + assert!(runtime.ingest_native(&packet(1,sequence,stop,RoomLabel::OccupiedStatic),stop).unwrap_err().starts_with("temporal_observed_gap_or_rate:")); + release.release(); + }else{ + let wait=Instant::now();while !writer.ack_is_queued(){assert!(wait.elapsed()=seq&&*t>=time),"Retry must retain live replay fences");} + assert!(runtime.retry_failed_temporal_capture(&guard,&capture_id,now).is_err(),"consumed command cannot retry again"); + sequence+=1; + println!("SYNTHETIC_OPERATOR20_RETRY committed_lost_ack={committed_lost_ack} rows=20 seals=2 prefix_preserved=true"); + } + // Probe a real previously admitted CanonicalPacket while paused. + // A fail-closed replay may stop the run; explicitly Retry once more + // before continuing, never count that packet as new progress. + let replay_floors=floors(&runtime).0; + let(source,seq,time)=replay_floors[0];let replay_now=now+1_000_000; + assert!(runtime.ingest_native(&packet(source,seq as u32,time,RoomLabel::OccupiedStatic),replay_now).is_err()); + assert_eq!(fixture::snapshot(&runtime,replay_now)["retained_rows"],20); + assert_eq!(fixture::snapshot(&runtime,replay_now)["durable_windows"],20); + assert_eq!(floors(&runtime).0,replay_floors,"rejected replay adds no source credit"); + if matches!(fixture::snapshot(&runtime,replay_now)["stage"].as_str(),Some("failed"|"blocked")){ + let status=runtime.temporal_status_v2(replay_now).unwrap();let id=status["capture"]["capture_generation_id"].as_str().unwrap(); + now=replay_now+1_000_000_000;runtime.retry_failed_temporal_capture(&runtime.mutation_guard().unwrap(),id,now).unwrap(); + }else{now=replay_now;} + assert_eq!(prefix20(&runtime,now),known_prefix); + } + } + assert_eq!(fixture::snapshot(&runtime,now)["durable_windows"],80); + assert!(runtime.current_verified_frozen_model().unwrap().is_none()); + runtime.start_temporal_fit(&runtime.mutation_guard().unwrap(),now).unwrap(); + let fit_clock=Instant::now(); + loop{ + let phase=runtime.inner.lock().unwrap().job.phase; + if phase==JobPhase::Ready{break;} + assert_eq!(phase,JobPhase::Running,"{:?}",runtime.inner.lock().unwrap().job.error); + assert!(fit_clock.elapsed())->Result{ + let operator=match profile {Some(crate::room_learner::managed_schedule_contract::OPERATOR_EIGHT_PROFILE)=>true,Some("native-2s-three-source-56bin-option-a-engineering-v1")|None=>false,_=>return Err("learner_unsupported_profile".into())}; + if let Some(CaptureWorkflow::Temporal(c))=state.capture.as_ref(){ + if c.core.as_ref().is_none_or(|core|core.public_profile().profile_id.eq(crate::room_learner::managed_schedule_contract::OPERATOR_EIGHT_PROFILE)!=operator){return Err("temporal_previous_plan_loaded".into());} + } + Ok(operator) + } + pub fn temporal_select_profile_guarded_v2(&self,profile:Option<&str>,expected:&MutationGuard)->Result{ + let mut state=self.command_state(expected)?; + let operator=self.validate_profile_selection(&state,profile)?; + self.operator_profile_selected.store(operator,Ordering::Release); + self.operator_profile_initialized.store(true,Ordering::Release); + Ok(self.guard_for(&state)) + } + #[cfg(test)] + pub(super) fn temporal_select_profile_v2(&self,profile:Option<&str>)->Result<(),String>{ + let guard=self.mutation_guard()?;self.temporal_select_profile_guarded_v2(profile,&guard).map(|_|()) + } + pub fn temporal_default_operator_profile_v2(&self)->Result<(),String>{ + let mut state=self.inner.lock().map_err(|_|"learner state unavailable")?; + if state.capture.is_none()&&!self.operator_profile_initialized.load(Ordering::Acquire){ + state.workflow_revision=state.workflow_revision.checked_add(1).ok_or("learner command revision exhausted")?; + self.operator_profile_selected.store(true,Ordering::Release); + self.operator_profile_initialized.store(true,Ordering::Release); + } + Ok(()) + } + pub fn temporal_retry_v2(&self,request:TemporalRetryRequestV2,now:u64)->Result<(),String>{ + schema(&request.schema,RECOVERY_RETRY_REQUEST_SCHEMA)?; + if !request.retry_confirmed{return Err("temporal_retry_confirmation_required".into());} + if !sha(&request.capture_generation_id){return Err("learner_capture_identity_invalid".into());} + self.retry_failed_temporal_capture(&request.expected,&request.capture_generation_id,now) + } pub fn temporal_begin_phase_v2( &self, request: TemporalPhaseStartRequestV2, @@ -405,19 +435,17 @@ impl RoomLearningRuntime { if temporal { let _ = self.tick_temporal_capture(&self.mutation_guard()?, now); } - let generation = { - self.inner - .lock() - .map_err(|_| "learner state unavailable")? - .generation + let (generation,workflow_revision,operator_profile) = { + let state=self.inner.lock().map_err(|_| "learner state unavailable")?; + (state.generation,state.workflow_revision,self.operator_profile_selected.load(Ordering::Acquire)) }; - let recovery = self - .recovery_store - .as_ref() - .and_then(|store| TemporalTrainCapture::recovery_discovery(store, now)); + let recovery_store=if operator_profile{self.operator_recovery_store.as_ref()}else{self.recovery_store.as_ref()}; + let recovery=recovery_store.and_then(|store|TemporalTrainCapture::recovery_discovery(store,now)); + #[cfg(test)] + if let Some(hook)=self.temporal_status_profile_hook.lock().unwrap().take(){hook();} let mut response = { let state = self.inner.lock().map_err(|_| "learner state unavailable")?; - if state.generation != generation { + if state.generation != generation || state.workflow_revision!=workflow_revision || self.operator_profile_selected.load(Ordering::Acquire)!=operator_profile { return Err("learner_state_changed".into()); } let capture = match state.capture.as_ref() { @@ -425,7 +453,7 @@ impl RoomLearningRuntime { _ => None, }; let context = state.capture_context().cloned(); - let mut profile = TemporalProfileWire::engineering(); + let mut profile = if operator_profile{TemporalProfileWire::operator_eight_step()}else{TemporalProfileWire::engineering()}; let mut wire_capture = None; let mut candidate = None; let mut train_proof = None; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs index 6f5e58f147..f33bc3042c 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_tests.rs @@ -1412,3 +1412,117 @@ fn historical329_and_fresh343_priming_keep_the_original_cadence_band() { assert_eq!(snapshot(&rt,now)["failure"],"acquisition_cadence_changed"); } } + +fn failed_full_capture_for_retry(root:&Path,lost_ack:bool)->(Arc,u64){ + let rt=Arc::new(runtime(root,START)); + rt.configure_temporal_full_run(&rt.mutation_guard().unwrap(),context(),"synthetic-stopped-retry".into(),START).unwrap(); + let ready=settle_async_boundary(&rt,START); + let start=async_lead_end(&rt,ready); + let publication_gate=if lost_ack { + let state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()}; + let writer=c.core.as_ref().unwrap().writer().unwrap();writer.lose_next_ack(); + let(entered_tx,entered_rx)=std::sync::mpsc::sync_channel(1);let(release_tx,release_rx)=std::sync::mpsc::sync_channel(1); + writer.hold_next_after_publish(entered_tx,release_rx);Some((entered_rx,release_tx)) + }else{None}; + for source in 1..=3u8 { + let at=start+u64::from(source)*1000; + rt.ingest_native(&packet(source,1,at),at).unwrap(); + } + if let Some((entered,_))=&publication_gate{entered.recv_timeout(std::time::Duration::from_secs(10)).unwrap();} + let at=start+1_001_000; + assert!(rt.ingest_native(&packet(1,2,at),at).unwrap_err().starts_with("temporal_observed_gap_or_rate:")); + if let Some((_,release))=publication_gate{release.send(()).unwrap();} + assert_eq!(snapshot(&rt,at)["stage"],"failed"); + (rt,at+1_000_000) +} + +#[test] +fn explicit_failed_capture_retry_reconciles_actual_head_without_changing_identity_or_deadline(){ + for lost_ack in [false,true] { + let directory=tempfile::tempdir().unwrap();let(rt,at)=failed_full_capture_for_retry(directory.path(),lost_ack); + let before=rt.temporal_status_v2(at).unwrap(); + let guard=rt.mutation_guard().unwrap(); + let id=before["capture"]["capture_generation_id"].as_str().unwrap().to_owned(); + rt.retry_failed_temporal_capture(&guard,&id,at).unwrap(); + let after=rt.temporal_status_v2(at+1).unwrap(); + assert_eq!(after["capture"]["stage"],"paused"); + assert_eq!(after["capture"]["failure"],Value::Null); + assert_eq!(after["capture"]["durability_pending"],false); + for field in ["capture_generation_id","schedule_sha256","continuation_policy_sha256","original_deadline","retained_windows","durable_windows","sealed_train_episodes","sealed_held_out_episodes"] { + assert_eq!(after["capture"][field],before["capture"][field],"{field}"); + } + assert_eq!(after["mutation_guard"]["generation"],before["mutation_guard"]["generation"]); + assert_eq!(after["mutation_guard"]["runtime_epoch"],before["mutation_guard"]["runtime_epoch"]); + assert!(rt.retry_failed_temporal_capture(&guard,&id,at+2).is_err(),"old command is consumed"); + assert_eq!(after["capture"]["observed_ms"],0); + } +} + +#[test] +fn retry_cannot_reconcile_a_writer_after_terminal_owner_drain(){ + let directory=tempfile::tempdir().unwrap();let(rt,at)=failed_full_capture_for_retry(directory.path(),true); + let writer={let state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()};c.core.as_ref().unwrap().writer().unwrap()}; + assert!(writer.has_failed_repair(),"actual publication happened and its ACK was withheld/lost"); + writer.drain().unwrap(); + assert!(!writer.has_failed_repair(),"terminal drain releases captured plaintext repair buffers"); + fn files(root:&Path)->Vec<(std::path::PathBuf,Vec)>{let mut result=vec![];for e in std::fs::read_dir(root).unwrap(){let e=e.unwrap();if e.file_type().unwrap().is_dir(){result.extend(files(&e.path()));}else{result.push((e.path(),std::fs::read(e.path()).unwrap()));}}result.sort_by(|a,b|a.0.cmp(&b.0));result} + let before=files(directory.path()); + assert_eq!(writer.reconcile_failed(at).unwrap_err(),"temporal_writer_terminally_drained"); + assert_eq!(files(directory.path()),before); +} + +#[test] +fn profile_namespace_selection_is_guarded_and_default_reads_cannot_undo_explicit_choice(){ + let directory=tempfile::tempdir().unwrap();let rt=runtime(directory.path(),START); + let old_guard=rt.mutation_guard().unwrap(); + rt.temporal_default_operator_profile_v2().unwrap(); + assert!(rt.operator_profile_selected.load(Ordering::Acquire)); + assert!(rt.temporal_select_profile_guarded_v2(None,&old_guard).is_err()); + assert!(rt.operator_profile_selected.load(Ordering::Acquire)); + let guard=rt.mutation_guard().unwrap(); + let after=rt.temporal_select_profile_guarded_v2(None,&guard).unwrap(); + assert!(!rt.operator_profile_selected.load(Ordering::Acquire)); + rt.temporal_default_operator_profile_v2().unwrap(); + assert!(!rt.operator_profile_selected.load(Ordering::Acquire),"read must not race an explicit old-plan restore"); + assert_eq!(rt.mutation_guard().unwrap(),after); +} + +#[test] +fn failed_capture_retry_rejects_wrong_identity_stale_owner_and_expiry_without_success(){ + let directory=tempfile::tempdir().unwrap();let(rt,at)=failed_full_capture_for_retry(directory.path(),false); + let before=rt.temporal_status_v2(at).unwrap(); + let id=before["capture"]["capture_generation_id"].as_str().unwrap(); + let mut stale=rt.mutation_guard().unwrap();stale.runtime_epoch="another-runtime".into(); + assert!(rt.retry_failed_temporal_capture(&stale,id,at).is_err()); + assert!(rt.retry_failed_temporal_capture(&rt.mutation_guard().unwrap(),&"f".repeat(64),at).is_err()); + let expiry=before["capture"]["original_deadline"]["expires_at_unix_ms"].as_u64().unwrap()*1_000_000; + assert!(rt.retry_failed_temporal_capture(&rt.mutation_guard().unwrap(),id,expiry).is_err()); + let state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()}; + let capture=c.core.as_ref().unwrap().public_capture(&rt.runtime_epoch,None,at); + assert_eq!(capture.stage,"failed");assert_eq!(capture.retained_windows,0);assert_eq!(capture.durable_windows,0); + assert_eq!(capture.original_deadline.expires_at_unix_ms,expiry/1_000_000); +} + +#[test] +fn retry_rejects_count_equal_but_wrong_known_accepted_prefix_digest(){ + let directory=tempfile::tempdir().unwrap();let(rt,at)=failed_full_capture_for_retry(directory.path(),false); + let id={let mut state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=state.capture.as_mut()else{panic!()};let core=c.core.as_mut().unwrap();let id=core.public_generation_id().to_owned();core.test_corrupt_retry_prefix();id}; + assert_eq!(rt.retry_failed_temporal_capture(&rt.mutation_guard().unwrap(),&id,at).unwrap_err(),"temporal_retry_committed_prefix_changed"); + let state=rt.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()}; + assert_eq!(c.core.as_ref().unwrap().stage(),TemporalCaptureStage::Failed); +} + +#[test] +fn status_refuses_profile_and_recovery_snapshot_mixed_across_a_guarded_selection(){ + let directory=tempfile::tempdir().unwrap();let old=runtime(directory.path(),START); + old.configure_temporal_full_run(&old.mutation_guard().unwrap(),context(),"retained-old-plan".into(),START).unwrap(); + let settled=settle_async_boundary(&old,START);drop(old); + let rt=Arc::new(runtime(directory.path(),settled+1)); + rt.temporal_select_profile_v2(None).unwrap(); + let weak=Arc::downgrade(&rt); + *rt.temporal_status_profile_hook.lock().unwrap()=Some(Box::new(move||{let rt=weak.upgrade().unwrap();rt.temporal_select_profile_v2(Some(crate::room_learner::managed_schedule_contract::OPERATOR_EIGHT_PROFILE)).unwrap();})); + assert_eq!(rt.temporal_status_v2(settled+2).unwrap_err(),"learner_state_changed"); + let coherent=rt.temporal_status_v2(settled+3).unwrap(); + assert_eq!(coherent["temporal_profile"]["profile_id"],crate::room_learner::managed_schedule_contract::OPERATOR_EIGHT_PROFILE); + assert!(coherent["recovery"].is_null(),"new namespace cannot show the old plan recovery ID"); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_schedule_contract.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_schedule_contract.rs index 16d84dc991..1392863653 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_schedule_contract.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_schedule_contract.rs @@ -11,6 +11,7 @@ use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; const SCHEMA: &str = "ruview.room-learner.trial-schedule.v2"; const PROFILE: &str = "native-2s-three-source-56bin-option-a-engineering-v1"; +pub(crate) const OPERATOR_EIGHT_PROFILE: &str = "native-2s-three-source-56bin-operator-eight-step-v1"; const MAX_BYTES: usize = 65_536; const MAX_PROFILE_ROWS: usize = 77; const MAX_TRAIN_WINDOWS: usize = 64; @@ -20,9 +21,13 @@ const MAX_TRAIN_WINDOWS: usize = 64; #[serde(rename_all = "snake_case")] pub enum LeadInPolicy { Train5HeldOut15EngineeringV1, + OperatorEightStepV1, } impl LeadInPolicy { - fn milliseconds(self, role: DataRole) -> u64 { + fn milliseconds(self, role: DataRole, condition: RoomLabel) -> u64 { + if self == Self::OperatorEightStepV1 { + return if matches!(condition, RoomLabel::Empty | RoomLabel::AdjacentOnly) { 15_000 } else { 5_000 }; + } match role { DataRole::Train => 5_000, DataRole::HeldOut => 15_000, @@ -246,6 +251,28 @@ impl ManagedScheduleContract { } let reserved = native.max_observed_span_ms() + native.max_first_packet_delay_ms(); let mut episodes = Vec::new(); + if lead_in_policy == LeadInPolicy::OperatorEightStepV1 { + for condition in [RoomLabel::Empty, RoomLabel::OccupiedStatic, RoomLabel::OccupiedMoving, RoomLabel::AdjacentOnly] { + for role in [DataRole::Train, DataRole::Validation] { + let duration = if role == DataRole::Validation { 15_000 } else if condition == RoomLabel::Empty { 600_000 } else { 60_000 }; + let count = if role == DataRole::Train { 16 } else { 4 }; + episodes.push(Episode { + index: episodes.len(), + episode_key: format!("operator-{role:?}-{condition:?}-episode-0"), + role, + condition, + target_active_recording_ms: duration, + lead_in_ms: lead_in_policy.milliseconds(role, condition), + requires_explicit_operator_start: true, + requires_conditions_held_confirmation: true, + separately_executed_episode_required: true, + slots: slots(duration, count, reserved)?, + retained_nominal_window_ms: count as u64 * native.target_span_ms(), + retained_maximum_window_ms: count as u64 * native.max_observed_span_ms(), + }); + } + } + } else { for role in [DataRole::Train, DataRole::HeldOut] { for condition in [ RoomLabel::Empty, @@ -263,7 +290,7 @@ impl ManagedScheduleContract { role, condition, target_active_recording_ms: duration, - lead_in_ms: lead_in_policy.milliseconds(role), + lead_in_ms: lead_in_policy.milliseconds(role, condition), requires_explicit_operator_start: true, requires_conditions_held_confirmation: true, separately_executed_episode_required: true, @@ -274,6 +301,7 @@ impl ManagedScheduleContract { } } } + } let role_counts = |role| episodes.iter().filter(|e| e.role == role).count(); let role_windows = |role| { episodes @@ -297,29 +325,34 @@ impl ManagedScheduleContract { .sum::() }; let train = role_windows(DataRole::Train); - let held = role_windows(DataRole::HeldOut); + let check_role = if lead_in_policy == LeadInPolicy::OperatorEightStepV1 { DataRole::Validation } else { DataRole::HeldOut }; + let held = role_windows(check_role); let total = add(train, held)?; let (rows, values) = capacity(&context, native.source_policies(), train, total)?; - let resources = native.resources().clone(); + let resources = if lead_in_policy == LeadInPolicy::OperatorEightStepV1 { + let views = mul(total, context.sources.len())?; + let serialized = add(add(add(mul(values, 25)?, mul(rows, 86)?)?, mul(views, 4096)?)?, add(mul(total, 4096)?, 8 * 1024 * 1024)?)?; + ResourcePreflight { cohorts: total, source_views: views, max_retained_rows: rows, max_numeric_values: values, serialized_upper_bound_bytes: serialized } + } else { native.resources().clone() }; if resources.cohorts != total || resources.max_retained_rows != rows || resources.max_numeric_values != values { return Err("managed_schedule_resource_contract_changed".into()); } - let active = role_time(DataRole::Train) + role_time(DataRole::HeldOut); - let lead = role_lead(DataRole::Train) + role_lead(DataRole::HeldOut); + let active = role_time(DataRole::Train) + role_time(check_role); + let lead = role_lead(DataRole::Train) + role_lead(check_role); let counts = ScheduleCounts { train_episodes: role_counts(DataRole::Train), - held_out_episodes: role_counts(DataRole::HeldOut), + held_out_episodes: role_counts(check_role), train_windows: train, held_out_windows: held, total_windows: total, train_active_recording_ms: role_time(DataRole::Train), - held_out_active_recording_ms: role_time(DataRole::HeldOut), + held_out_active_recording_ms: role_time(check_role), active_recording_ms: active, train_lead_in_ms: role_lead(DataRole::Train), - held_out_lead_in_ms: role_lead(DataRole::HeldOut), + held_out_lead_in_ms: role_lead(check_role), total_lead_in_ms: lead, nominal_recording_and_lead_in_ms: active + lead, nominal_joint_window_ms: total as u64 * native.target_span_ms(), @@ -329,7 +362,7 @@ impl ManagedScheduleContract { verified_independent_trial_count: None, }; let body = Body { - profile_id: PROFILE, + profile_id: if lead_in_policy == LeadInPolicy::OperatorEightStepV1 { OPERATOR_EIGHT_PROFILE } else { PROFILE }, context, lead_in_policy, native_input: NativeInputPolicy { @@ -351,7 +384,7 @@ impl ManagedScheduleContract { full_episode_quality_monitoring_implemented: false, independence_verified: false, scientific_adequacy_claimed: false, - user_facing_default: false, + user_facing_default: lead_in_policy == LeadInPolicy::OperatorEightStepV1, }, }; let result = Self { @@ -379,7 +412,7 @@ impl ManagedScheduleContract { } let value: Import = serde_json::from_slice(bytes).map_err(|_| "managed_schedule_invalid_schema")?; - if value.schema != SCHEMA || value.body["profile_id"] != PROFILE { + if value.schema != SCHEMA || (value.body["profile_id"] != PROFILE && value.body["profile_id"] != OPERATOR_EIGHT_PROFILE) { return Err("managed_schedule_unsupported_profile".into()); } let context = serde_json::from_value(value.body["context"].clone()) @@ -403,6 +436,8 @@ impl ManagedScheduleContract { let model=bound.frozen_artifact_sha256.as_ref().and_then(|sha|FrozenModelIdentity::from_runtime_verified_artifact(self,sha.clone()).ok()); self.bind_episode_start(run,bound.episode.index,bound.attempt_ordinal,bound.segment_ordinal,model.as_ref()).ok().is_some_and(|expected|serde_json::to_vec(&expected).ok()==serde_json::to_vec(bound).ok()) } + pub fn is_operator_eight_step(&self) -> bool { self.body.profile_id == OPERATOR_EIGHT_PROFILE } + pub fn profile_id(&self) -> &str { self.body.profile_id } pub fn sha256(&self) -> &str { &self.sha256 } @@ -450,6 +485,7 @@ impl ManagedScheduleContract { } } DataRole::Train => {} + DataRole::Validation if self.is_operator_eight_step() && frozen.is_none() => {} _ => return Err("managed_schedule_invalid_role".into()), } let identity = |kind: &str, parts: &[serde_json::Value]| -> Result { diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_schedule_contract_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_schedule_contract_tests.rs index d6ea664081..20e63bb87f 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_schedule_contract_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/managed_schedule_contract_tests.rs @@ -332,3 +332,60 @@ fn native_capacity_and_shape_reject_before_capture_without_fallback() { assert!(capacity(&many, &policies, 64, 128).is_err()); assert_eq!(p.counts().total_windows, 128); } + +#[test] +fn operator_eight_step_is_exact_interleaved_user_sequence_with_bounded_reserved_checks() { + let old = plan().export().unwrap(); + let p=ManagedScheduleContract::build(context(3),LeadInPolicy::OperatorEightStepV1).unwrap(); + assert_eq!(p.profile_id(),OPERATOR_EIGHT_PROFILE); + let expected=[ + (DataRole::Train,RoomLabel::Empty,15_000,600_000,16), + (DataRole::Validation,RoomLabel::Empty,15_000,15_000,4), + (DataRole::Train,RoomLabel::OccupiedStatic,5_000,60_000,16), + (DataRole::Validation,RoomLabel::OccupiedStatic,5_000,15_000,4), + (DataRole::Train,RoomLabel::OccupiedMoving,5_000,60_000,16), + (DataRole::Validation,RoomLabel::OccupiedMoving,5_000,15_000,4), + (DataRole::Train,RoomLabel::AdjacentOnly,15_000,60_000,16), + (DataRole::Validation,RoomLabel::AdjacentOnly,15_000,15_000,4), + ]; + assert_eq!(p.episodes().len(),8); + for (episode,(role,label,lead,duration,windows)) in p.episodes().iter().zip(expected) { + assert_eq!((episode.role,episode.condition,episode.lead_in_ms,episode.target_active_recording_ms,episode.slots.len()),(role,label,lead,duration,windows)); + assert!(episode.requires_explicit_operator_start&&episode.requires_conditions_held_confirmation); + assert!(episode.slots.windows(2).all(|pair|pair[0].deadline_offset_ms<=pair[1].target_offset_ms)); + assert!(episode.slots.last().unwrap().deadline_offset_ms<=duration); + } + let c=p.counts(); + assert_eq!((c.train_episodes,c.held_out_episodes,c.train_windows,c.held_out_windows,c.total_windows),(4,4,64,16,80)); + assert_eq!(c.nominal_recording_and_lead_in_ms,920_000); + assert_eq!(p.resources().max_numeric_values,1_034_880); + assert_eq!(p.resources().max_retained_rows,18_480); + assert_eq!(ManagedScheduleContract::import(&p.export().unwrap()).unwrap().export().unwrap(),p.export().unwrap()); + assert_eq!(ManagedScheduleContract::import(&old).unwrap().export().unwrap(),old); + assert_ne!(p.sha256(),plan().sha256()); +} + +#[test] +fn operator_reserved_checks_bind_without_a_model_but_cannot_be_relabelled_as_training() { + let p=ManagedScheduleContract::build(context(3),LeadInPolicy::OperatorEightStepV1).unwrap(); + let check=p.bind_episode_start("operator-run",1,0,0,None).unwrap(); + assert_eq!(check.episode().role,DataRole::Validation); + let frozen=FrozenModelIdentity::from_runtime_verified_artifact(&p,"f".repeat(64)).unwrap(); + assert!(p.bind_episode_start("operator-run",1,0,0,Some(&frozen)).is_err()); + assert!(plan().bind_episode_start("old-run",7,0,0,None).is_err()); + let mut value:serde_json::Value=serde_json::from_slice(&p.export().unwrap()).unwrap(); + value["body"]["episodes"][1]["role"]=serde_json::json!("Train"); + assert!(ManagedScheduleContract::import(&serde_json::to_vec(&value).unwrap()).is_err()); +} + +#[test] +fn operator_default_is_explicit_and_rehashed_authority_claims_stay_rejected(){ + let p=ManagedScheduleContract::build(context(3),LeadInPolicy::OperatorEightStepV1).unwrap(); + assert!(p.body.claims.user_facing_default); + assert!(!p.body.claims.capture_ready&&!p.body.claims.scientific_adequacy_claimed&&!p.body.claims.independence_verified); + assert!(!plan().body.claims.user_facing_default); + let mut forged=p.clone();forged.body.claims.capture_ready=true;forged.sha256=hash(&forged.body).unwrap(); + assert!(ManagedScheduleContract::import(&forged.export().unwrap()).is_err()); + let mut forged=plan();forged.body.claims.user_facing_default=true;forged.sha256=hash(&forged.body).unwrap(); + assert!(ManagedScheduleContract::import(&forged.export().unwrap()).is_err()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store.rs index 86dd811981..7661bcfbb9 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store.rs @@ -36,9 +36,11 @@ pub struct LoadedRecovery { } pub struct RecoveryStore { directory: PathBuf, + owner_directory: PathBuf, owner: String, installation: String, - key: Option, + key: Option>, + operator_key: Option>, serial: Mutex<()>, } fn id(value: &str) -> bool { @@ -114,6 +116,15 @@ fn managed_seed(stable: Option<&str>, value: Option<&str>) -> Option<[u8; 32]> { Some(bytes) } impl RecoveryStore { + /// Same account-owned recovery root, isolated ciphertext/key/commitment + /// namespace. Creating its empty private directory grants no capture consent. + pub(crate) fn operator_eight_scope(&self) -> Result { + self.verify_directory()?; + if self.directory!=self.owner_directory.join("room-learner-recovery"){return Err("recovery_profile_scope_invalid".into());} + let directory = self.directory.join("operator-eight-step-v1"); + private_directory(&directory)?; + Ok(Self { directory, owner_directory:self.owner_directory.clone(), owner:self.owner.clone(), installation:self.installation.clone(), key:self.operator_key.clone(), operator_key:None, serial:Mutex::new(()) }) + } pub fn new( data_dir: &Path, owner: &str, @@ -171,12 +182,28 @@ impl RecoveryStore { material.fill(0); result }) - .transpose()?; + .transpose()?.map(std::sync::Arc::new); + // Purpose-separated key for the explicitly versioned operator plan; + // old ciphertext and its derivation remain unchanged/readable. + let operator_key = seed.map(|seed| { + let mut info = Vec::new(); + for item in [b"ruview.capture-recovery.operator-eight-step.aes256gcm.v1".as_slice(), owner.as_bytes(), installation.as_bytes()] { + info.extend_from_slice(&(item.len() as u32).to_be_bytes()); info.extend_from_slice(item); + } + let salt = hkdf::Salt::new(hkdf::HKDF_SHA256, b"ruview.capture-recovery.hkdf.v1"); + let prk = salt.extract(seed); let infos = [info.as_slice()]; + let okm = prk.expand(&infos, &aead::AES_256_GCM).map_err(|_| "recovery_key_unavailable")?; + let mut material = [0u8; 32]; okm.fill(&mut material).map_err(|_| "recovery_key_unavailable")?; + let result = aead::UnboundKey::new(&aead::AES_256_GCM, &material).map(aead::LessSafeKey::new).map(std::sync::Arc::new).map_err(|_| "recovery_key_unavailable"); + material.fill(0); result + }).transpose()?; Ok(Self { directory, + owner_directory:owned, owner: owner.into(), installation: installation.into(), key, + operator_key, serial: Mutex::new(()), }) } @@ -202,7 +229,7 @@ impl RecoveryStore { self.key.is_some() } fn verify_directory(&self) -> Result<(), String> { - let owned = self.directory.parent().ok_or("recovery_owner_mismatch")?; + let owned = self.owner_directory.as_path(); for (path, is_directory) in [ (owned.to_path_buf(), true), (owned.join("account-owner-v1.json"), false), @@ -227,14 +254,16 @@ impl RecoveryStore { if verified != owned { return Err("recovery_owner_mismatch".into()); } + private_directory(&owned.join("room-learner-recovery"))?; + if self.directory!=owned.join("room-learner-recovery")&&self.directory!=owned.join("room-learner-recovery/operator-eight-step-v1"){return Err("recovery_profile_scope_invalid".into());} private_directory(&self.directory) } /// Non-repairing owner validation for read-scoped status discovery. /// Unlike prepare_directory this never creates directories or chmods them. fn verify_directory_read_only(&self)->Result<(),String>{ - let owned=self.directory.parent().ok_or("recovery_owner_mismatch")?; + let owned=self.owner_directory.as_path(); if !hex(&self.owner)||owned.file_name().and_then(|v|v.to_str())!=Some("SensingServer")||owned.parent().and_then(Path::file_name).and_then(|v|v.to_str())!=Some(self.owner.as_str())||owned.canonicalize().map_err(|_|"recovery_owner_mismatch")?!=owned{return Err("recovery_owner_mismatch".into());} - for(path,directory)in[(owned.to_path_buf(),true),(owned.join("account-owner-v1.json"),false),(self.directory.clone(),true)]{ + for(path,directory)in[(owned.to_path_buf(),true),(owned.join("account-owner-v1.json"),false),(owned.join("room-learner-recovery"),true),(self.directory.clone(),true)]{ let m=fs::symlink_metadata(&path).map_err(|_|"recovery_owner_mismatch")?; if m.file_type().is_symlink()||(directory&&!m.is_dir())||(!directory&&!m.is_file()){return Err("recovery_unsafe_owner_path".into());}private_metadata(&m,directory)?; } diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store_tests.rs index db0360f71d..f96e72822a 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/recovery_store_tests.rs @@ -358,3 +358,24 @@ fn wrong_key_or_tampered_inspection_fails_without_altering_retained_evidence() { assert!(s.inspect().is_err()); assert_eq!(fs::read(&path).unwrap(), bad); } + +#[test] +fn operator_profile_scope_preserves_old_bytes_and_uses_a_separate_key_without_recording_consent() { + let (_t,old)=fixture(Some(&[7;32])); + old.save(&descriptor(),b"synthetic previous-plan evidence",NOW).unwrap(); + let bytes=fs::read(old.directory.join(FILE)).unwrap(); + let new=old.operator_eight_scope().unwrap(); + assert!(new.available()); + assert!(fs::read_dir(&new.directory).unwrap().next().is_none()); + assert!(new.inspect().unwrap().is_none()); + new.save(&descriptor(),b"synthetic eight-step evidence",NOW).unwrap(); + assert_eq!(new.load(NOW+1).unwrap().unwrap().plaintext,b"synthetic eight-step evidence"); + assert_eq!(fs::read(old.directory.join(FILE)).unwrap(),bytes); + assert_eq!(old.load(NOW+1).unwrap().unwrap().plaintext,b"synthetic previous-plan evidence"); + // The same owner's old ciphertext cannot authenticate in the new key domain. + let valid_new=fs::read(new.directory.join(FILE)).unwrap(); + fs::write(new.directory.join(FILE),&bytes).unwrap(); + assert!(new.load(NOW+1).is_err()); + fs::write(new.directory.join(FILE),valid_new).unwrap(); + assert!(new.operator_eight_scope().is_err()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs index 24da3624de..544de436a5 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs @@ -10,18 +10,19 @@ pub(super) const BRIDGE_POLICY: &str = "native-receiver-whole-train-period-v1;al // Explicit NEW Train genesis only. The generic constructor and old literal // remain historical329; restore never upgrades a stored generation. pub(super) const FRESH_TRAIN_343_POLICY:&str="native-receiver-whole-train-period-v2;all-selected-packets;existing-cadence-gap-arrival-skew;confirmed-durable-seals;train-budget343;no-fit-authority"; +pub(super) const OPERATOR_EIGHT_POLICY:&str="native-receiver-whole-operator-period-v1;reserved-validation-not-training;confirmed-durable-seals;no-fit-authority"; #[derive(Clone,Copy,Debug,PartialEq,Eq)] -pub(super) enum TrainBudgetProfile { Historical329, FreshTrain343 } +pub(super) enum TrainBudgetProfile { Historical329, FreshTrain343, OperatorEightStep } impl TrainBudgetProfile { pub(super) fn from_stored(policy:&str)->Result{match policy{ - BRIDGE_POLICY=>Ok(Self::Historical329),FRESH_TRAIN_343_POLICY=>Ok(Self::FreshTrain343), + BRIDGE_POLICY=>Ok(Self::Historical329),FRESH_TRAIN_343_POLICY=>Ok(Self::FreshTrain343),OPERATOR_EIGHT_POLICY=>Ok(Self::OperatorEightStep), _=>Err("temporal_train_profile_unknown".into()), }} - pub(super) fn policy(self)->&'static str{match self{Self::Historical329=>BRIDGE_POLICY,Self::FreshTrain343=>FRESH_TRAIN_343_POLICY}} + pub(super) fn policy(self)->&'static str{match self{Self::Historical329=>BRIDGE_POLICY,Self::FreshTrain343=>FRESH_TRAIN_343_POLICY,Self::OperatorEightStep=>OPERATOR_EIGHT_POLICY}} pub(super) fn transition_limit(self,schedule:&ManagedScheduleContract)->Result{ let episodes=schedule.counts().train_episodes; let old=schedule.counts().train_windows.checked_add(2*128).and_then(|n|n.checked_add(episodes)).and_then(|n|n.checked_add(2)).ok_or("temporal_transition_capacity")?; - match self{Self::Historical329=>Ok(old),Self::FreshTrain343=>old.checked_add(2*episodes).ok_or("temporal_transition_capacity".into())} + match self{Self::Historical329=>Ok(old),Self::FreshTrain343=>old.checked_add(2*episodes).ok_or("temporal_transition_capacity".into()),Self::OperatorEightStep=>Ok(schedule.counts().total_windows+2*(128+schedule.episodes().len())+schedule.episodes().len()+4)} } } @@ -212,6 +213,7 @@ impl BridgeState { parts: vec![], } } + pub(super) fn operator_eight_step()->Self{Self{policy:OPERATOR_EIGHT_POLICY.into(),..Self::new()}} pub(super) fn fresh_train_343()->Self{ Self{policy:FRESH_TRAIN_343_POLICY.into(),..Self::new()} } @@ -232,17 +234,15 @@ impl BridgeState { pub(super) fn validate_full_run_with_prefixes( &self,schedule:&ManagedScheduleContract,run:&str,rows:&[CaptureCohort],evidence:&[WindowEvidence],ledger:&Ledger,episode:usize,mapping:Option<&ObservedSegmentMap>,now:u64,digests:&prefix_digests::CanonicalPrefixDigests, )->Result<(),String>{ - if self.policy!=BRIDGE_POLICY{return Err("temporal_full_run_train_profile_changed".into());} + if self.policy!=if schedule.is_operator_eight_step(){OPERATOR_EIGHT_POLICY}else{BRIDGE_POLICY}{return Err("temporal_full_run_train_profile_changed".into());} self.validate_with_prefixes(schedule,run,rows,evidence,ledger,episode,mapping,now,Some(digests)) } fn validate_with_prefixes( &self,schedule:&ManagedScheduleContract,run:&str,rows:&[CaptureCohort],evidence:&[WindowEvidence],ledger:&Ledger,episode:usize,mapping:Option<&ObservedSegmentMap>,now:u64,digests:Option<&prefix_digests::CanonicalPrefixDigests>, )->Result<(),String>{ - let train = schedule - .episodes() - .iter() - .take_while(|e| e.role == DataRole::Train) - .count(); + let train = if schedule.is_operator_eight_step() { schedule.episodes().len() } else { + schedule.episodes().iter().take_while(|e| e.role == DataRole::Train).count() + }; let profile=self.profile()?; if self.seals.len() > train || episode != self.seals.len() @@ -292,7 +292,7 @@ impl BridgeState { if self.disposition != Disposition::Complete || !self.parts.is_empty() || mapping.is_some() - || rows.len() != schedule.counts().train_windows + || rows.len() != if schedule.is_operator_eight_step(){schedule.counts().total_windows}else{schedule.counts().train_windows} { return Err("temporal_bridge_complete_state".into()); } @@ -632,7 +632,7 @@ impl TemporalTrainCapture { ) -> Result<(Self, Option), String> { let mut workflow = TemporalWorkflow::prepare(schedule, run, config, now)?; let schedule = workflow.schedule()?; - workflow.bridge = Some(BridgeState::new()); + workflow.bridge = Some(if schedule.is_operator_eight_step(){BridgeState::operator_eight_step()}else{BridgeState::new()}); let error = initial(&mut workflow, store).err(); if error.is_some() { workflow.stage = Stage::BlockedDurability; @@ -765,7 +765,7 @@ impl TemporalTrainCapture { let w = &self.workflow; let schedule = &self.schedule; let e = schedule.episodes().get(w.episode)?; - if e.role != DataRole::Train && !w.full_run.as_ref().is_some_and(|m|m.is_frozen()) {return None;} + if e.role != DataRole::Train && !(schedule.is_operator_eight_step()&&e.role==DataRole::Validation) && !w.full_run.as_ref().is_some_and(|m|m.is_frozen()) {return None;} let frozen=w.frozen_identity().ok()?; let bound = schedule .bind_episode_start( @@ -776,7 +776,7 @@ impl TemporalTrainCapture { } fn logical_phase_complete(&self)->bool { let w=&self.workflow; - self.schedule.episodes().get(w.episode).is_some_and(|e|(e.role==DataRole::Train || w.full_run.as_ref().is_some_and(|m|m.held_out.is_some())) + self.schedule.episodes().get(w.episode).is_some_and(|e|(e.role==DataRole::Train || self.schedule.is_operator_eight_step()&&e.role==DataRole::Validation || w.full_run.as_ref().is_some_and(|m|m.held_out.is_some())) && w.pending_window.is_none() && w.session.rows.len()==self.schedule.episodes().iter().take(w.episode+1).map(|e|e.slots.len()).sum::()) } @@ -786,7 +786,7 @@ impl TemporalTrainCapture { let Some(e) = s.episodes().get(w.episode) else { return false; }; - (e.role == DataRole::Train || w.full_run.as_ref().is_some_and(|m|m.held_out.is_some())) + (e.role == DataRole::Train || s.is_operator_eight_step()&&e.role==DataRole::Validation || w.full_run.as_ref().is_some_and(|m|m.held_out.is_some())) && w.accepted() == e.target_active_recording_ms * 1_000_000 && w.pending_window.is_none() && w.session.rows.len() @@ -810,7 +810,7 @@ impl TemporalTrainCapture { let schedule = w.schedule()?; let role=schedule.episodes().get(w.episode).ok_or("temporal_capture_complete")?.role; if role==DataRole::HeldOut{w.begin_held_out_role(now)?;} - else if role!=DataRole::Train{return Err("temporal_train_complete".into());} + else if role!=DataRole::Train&&!(schedule.is_operator_eight_step()&&role==DataRole::Validation){return Err("temporal_train_complete".into());} else if w.full_run.is_some()&&w.mapping.is_none(){ let b=w.bridge.as_ref().ok_or("temporal_bridge_required")?; let expected=schedule.episodes().iter().take(w.episode).map(|e|e.slots.len()).sum::(); @@ -999,7 +999,7 @@ impl TemporalTrainCapture { w.mapping = None; w.custody = SegmentCustody::default(); w.live_observed_ns = 0; - b.disposition = if w.episode == schedule.counts().train_episodes { + b.disposition = if w.episode == if schedule.is_operator_eight_step(){schedule.episodes().len()}else{schedule.counts().train_episodes} { Disposition::Complete } else { Disposition::Ready @@ -1105,8 +1105,8 @@ impl TemporalTrainCapture { let w = &self.workflow; let e = self.schedule.episodes().get(w.episode); let full=w.full_run.is_some(); - let mut value=json!({"schema":"ruview.room-learner.private-temporal-capture.v1","stage":self.stage().as_str(),"episode_index":w.episode,"label":e.filter(|e|e.role==DataRole::Train||full).map(|e|e.condition),"role":if full&&e.is_some_and(|e|e.role==DataRole::HeldOut){"HeldOut"}else{"Train"},"sealed_train_episodes":self.durable_seals,"retained_rows":w.session.rows.len(),"durable_windows":w.durable_windows,"rows_sha256":w.async_state.as_ref().map(|a|if w.session.rows.len()==w.durable_windows{a.rows_sha256.clone()}else{None}).unwrap_or_else(||hash(&w.session.rows).ok()),"durability_pending":w.async_state.as_ref().is_some_and(|a|a.pending.is_some()),"durability_error":w.async_state.as_ref().and_then(|a|a.error.clone()),"observed_ns":w.live_observed_ns.to_string(),"durable_observed_ns":w.accepted().to_string(),"target_ms":e.filter(|e|e.role==DataRole::Train||full).map(|e|e.target_active_recording_ms),"descriptor":w.descriptor,"failure":self.failure,"fit_admission_available":false,"quality_is_engineering_only":true,"independence_verified":false,"source_clock_measured":false}); - if full{value["sealed_held_out_episodes"]=json!(w.async_state.as_ref().map_or(0,|s|s.accepted_held_out_seals));} + let mut value=json!({"schema":"ruview.room-learner.private-temporal-capture.v1","stage":self.stage().as_str(),"episode_index":w.episode,"label":e.filter(|e|e.role==DataRole::Train||full).map(|e|e.condition),"role":if self.schedule.is_operator_eight_step(){e.map(|e|match e.role{DataRole::Train=>"Train",DataRole::Validation=>"Validation",DataRole::HeldOut=>"HeldOut",DataRole::Inference=>"Inference"}).unwrap_or("Train")}else if full&&e.is_some_and(|e|e.role==DataRole::HeldOut){"HeldOut"}else{"Train"},"sealed_train_episodes":if self.schedule.is_operator_eight_step(){self.schedule.episodes().iter().take(self.durable_seals).filter(|e|e.role==DataRole::Train).count()}else{self.durable_seals},"retained_rows":w.session.rows.len(),"durable_windows":w.durable_windows,"rows_sha256":w.async_state.as_ref().map(|a|if w.session.rows.len()==w.durable_windows{a.rows_sha256.clone()}else{None}).unwrap_or_else(||hash(&w.session.rows).ok()),"durability_pending":w.async_state.as_ref().is_some_and(|a|a.pending.is_some()),"durability_error":w.async_state.as_ref().and_then(|a|a.error.clone()),"observed_ns":w.live_observed_ns.to_string(),"durable_observed_ns":w.accepted().to_string(),"target_ms":e.filter(|e|e.role==DataRole::Train||full).map(|e|e.target_active_recording_ms),"descriptor":w.descriptor,"failure":self.failure,"fit_admission_available":false,"quality_is_engineering_only":true,"independence_verified":false,"source_clock_measured":false}); + if full{value["sealed_held_out_episodes"]=json!(if self.schedule.is_operator_eight_step(){self.schedule.episodes().iter().take(self.durable_seals).filter(|e|e.role==DataRole::Validation).count()}else{w.async_state.as_ref().map_or(0,|s|s.accepted_held_out_seals)});} value } } @@ -1141,13 +1141,13 @@ impl TemporalTrainCapture { pub(crate) fn public_phase_matches(&self,generation:&str,schedule:&str,episode:usize)->bool{self.public_generation_id==generation&&self.schedule.sha256()==schedule&&self.workflow.episode==episode} pub(crate) fn public_profile(&self)->crate::room_learner::temporal_wire::TemporalProfileWire{ use crate::room_learner::temporal_wire::*; - TemporalProfileWire{profile_id:"native-2s-three-source-56bin-option-a-engineering-v1".into(),window_unit:"native_receiver_2s_window".into(),native_window_ms:2000,train_windows:64,held_out_windows:64,train_episodes:7,held_out_episodes:8,windows_per_condition:[16;4],train_episodes_per_condition:[1,2,2,2],held_out_episodes_per_condition:[2;4],episodes:self.schedule.episodes().iter().map(|e|TemporalEpisodeWire{index:e.index as u8,role:e.role,label:e.condition,target_ms:e.target_active_recording_ms,lead_in_ms:e.lead_in_ms,native_windows:e.slots.len()as u16}).collect()} + TemporalProfileWire{profile_id:self.schedule.profile_id().into(),window_unit:"native_receiver_2s_window".into(),native_window_ms:2000,train_windows:self.schedule.counts().train_windows as u16,held_out_windows:self.schedule.counts().held_out_windows as u16,train_episodes:self.schedule.counts().train_episodes as u8,held_out_episodes:self.schedule.counts().held_out_episodes as u8,windows_per_condition:[16;4],train_episodes_per_condition:if self.schedule.is_operator_eight_step(){[1;4]}else{[1,2,2,2]},held_out_episodes_per_condition:if self.schedule.is_operator_eight_step(){[1;4]}else{[2;4]},episodes:self.schedule.episodes().iter().map(|e|TemporalEpisodeWire{index:e.index as u8,role:e.role,label:e.condition,target_ms:e.target_active_recording_ms,lead_in_ms:e.lead_in_ms,native_windows:e.slots.len()as u16}).collect()} } pub(crate) fn public_capture(&self,epoch:&str,lead_until:Option,now:u64)->crate::room_learner::temporal_wire::TemporalCaptureWire{ use crate::room_learner::temporal_wire::*; let w=&self.workflow;let episode=self.schedule.episodes().get(w.episode);let lead=lead_until.map_or(0,|end|end.saturating_sub(now)/1_000_000); let challenge=if matches!(self.stage(),TemporalCaptureStage::NeedsConfirmation|TemporalCaptureStage::Paused){self.confirmation_challenge().ok()}else{None}; - TemporalCaptureWire{capture_generation_id:self.public_generation_id.clone(),schedule_sha256:self.schedule.sha256().into(),continuation_policy_sha256:w.full_run.as_ref().map(|m|m.policy.sha256().into()).unwrap_or_default(),durable_commitment_sha256:w.commit_token.as_ref().map(|t|t.data().commitment),original_deadline:TemporalOriginalDeadlineWire{created_at_unix_ms:w.descriptor.created_at_ms,expires_at_unix_ms:w.descriptor.expires_at_ms},receiver_epoch:epoch.into(),stage:if lead>0{"preparing".into()}else{self.stage().as_str().into()},current_episode:episode.map(|e|e.index as u8),role:episode.map(|e|e.role),label:episode.map(|e|e.condition),target_ms:episode.map(|e|e.target_active_recording_ms),lead_in_remaining_ms:lead,observed_ms:w.live_observed_ns/1_000_000,retained_windows:w.session.rows.len()as u16,durable_windows:w.durable_windows as u16,sealed_train_episodes:self.durable_seals as u8,sealed_held_out_episodes:w.async_state.as_ref().map_or(0,|a|a.accepted_held_out_seals as u8),durability_pending:w.async_state.as_ref().is_some_and(|a|a.pending.is_some()),awaiting_ground_truth:challenge.is_some(),confirmation_challenge:challenge,failure:self.failure.clone()} + TemporalCaptureWire{capture_generation_id:self.public_generation_id.clone(),schedule_sha256:self.schedule.sha256().into(),continuation_policy_sha256:w.full_run.as_ref().map(|m|m.policy.sha256().into()).unwrap_or_default(),durable_commitment_sha256:w.commit_token.as_ref().map(|t|t.data().commitment),original_deadline:TemporalOriginalDeadlineWire{created_at_unix_ms:w.descriptor.created_at_ms,expires_at_unix_ms:w.descriptor.expires_at_ms},receiver_epoch:epoch.into(),stage:if lead>0{"preparing".into()}else{self.stage().as_str().into()},current_episode:episode.map(|e|e.index as u8),role:episode.map(|e|e.role),label:episode.map(|e|e.condition),target_ms:episode.map(|e|e.target_active_recording_ms),lead_in_remaining_ms:lead,observed_ms:w.live_observed_ns/1_000_000,retained_windows:w.session.rows.len()as u16,durable_windows:w.durable_windows as u16,sealed_train_episodes:if self.schedule.is_operator_eight_step(){self.schedule.episodes().iter().take(self.durable_seals).filter(|e|e.role==DataRole::Train).count()as u8}else{self.durable_seals as u8},sealed_held_out_episodes:if self.schedule.is_operator_eight_step(){self.schedule.episodes().iter().take(self.durable_seals).filter(|e|e.role==DataRole::Validation).count()as u8}else{w.async_state.as_ref().map_or(0,|a|a.accepted_held_out_seals as u8)},durability_pending:w.async_state.as_ref().is_some_and(|a|a.pending.is_some()),awaiting_ground_truth:challenge.is_some(),confirmation_challenge:challenge,failure:self.failure.clone()} } pub(crate) fn public_candidate(&self)->Option{ use crate::room_learner::temporal_wire::*; @@ -1192,3 +1192,70 @@ impl TemporalTrainCapture { result.unwrap_or_else(|_|Some(TemporalRecoveryStatusWire{schema:RECOVERY_STATUS_SCHEMA.into(),state:"unavailable".into(),capture_generation_id:None,context_digest:None,original_deadline:None})) } } + +/// Opaque same-process Retry fence snapshot. It cannot be deserialized and +/// never grants recording or conditions authority. +pub(crate) struct FailedRetryState { + generation: String, + schedule: String, + created_at_ms: u64, + expires_at_ms: u64, + durable_windows: usize, + seals: usize, + accepted_prefix_sha256: Option, + accepted_seal_digests: Vec, + fences: AssemblerCheckpoint, + source_high: Vec<(u8, u64, u64)>, +} +impl TemporalTrainCapture { + pub(crate) fn failed_retry_state(&mut self) -> Result { + if !matches!(self.stage(), TemporalCaptureStage::Failed | TemporalCaptureStage::Blocked) + || !self.is_full_run() + || self.workflow.full_run.as_ref().is_some_and(|m| m.is_frozen() || m.is_complete()) + { return Err("temporal_retry_requires_stopped_capture".into()); } + let state = FailedRetryState { + generation:self.public_generation_id.clone(), schedule:self.schedule.sha256().into(), + created_at_ms:self.workflow.descriptor.created_at_ms, expires_at_ms:self.workflow.descriptor.expires_at_ms, + durable_windows:self.workflow.durable_windows, seals:self.durable_seals, + accepted_prefix_sha256:self.workflow.async_state.as_ref().and_then(|state|state.row_prefix_sha256.clone()), + accepted_seal_digests:self.workflow.bridge.as_ref().map_or_else(Vec::new,|b|b.seals.iter().take(self.durable_seals).map(|seal|seal.sha256.clone()).collect()), + fences:self.workflow.session.assembler.fence_checkpoint()?, source_high:self.workflow.source_high.clone(), + }; + if self.failure.is_none() { self.invalidate("temporal_retry_stopped_capture"); } + Ok(state) + } + pub(crate) fn apply_failed_retry_state(&mut self, old: FailedRetryState, now:u64) -> Result<(), String> { + if self.public_generation_id != old.generation || self.schedule.sha256()!=old.schedule + || self.workflow.descriptor.created_at_ms!=old.created_at_ms || self.workflow.descriptor.expires_at_ms!=old.expires_at_ms + || now/1_000_000>=old.expires_at_ms || self.workflow.durable_windows>()!=old.accepted_seal_digests){ + return Err("temporal_retry_confirmed_seals_changed".into()); + } + let n=self.workflow.session.rows.len(); + let budget=self.workflow.full_run.as_ref().ok_or("temporal_full_run_required")?.policy.replay_fence_budget(&self.workflow.session.managed.as_ref().unwrap().binding)?; + self.workflow.session.assembler.restore_full_run_prefix_fences(old.fences.for_prefix(n)?,n,&budget)?; + for (id,seq,time) in old.source_high { + if time>now {return Err("temporal_retry_receiver_clock_changed".into());} + if let Some((_,old_seq,old_time))=self.workflow.source_high.iter_mut().find(|(source,_,_)|*source==id) { + *old_seq=(*old_seq).max(seq); *old_time=(*old_time).max(time); + } else {self.workflow.source_high.push((id,seq,time));} + } + self.workflow.source_high.sort(); + self.workflow.stage=Stage::PausedNeedsReconfirmation; + self.workflow.last_now_ns=now; + self.failure=None; + Ok(()) + } +} + +#[cfg(test)] +impl TemporalTrainCapture { + pub(crate) fn test_corrupt_retry_prefix(&mut self){self.workflow.async_state.as_mut().unwrap().row_prefix_sha256=Some("f".repeat(64));} +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs index a91e8e0420..675573dc27 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint.rs @@ -91,10 +91,13 @@ pub(super) fn snapshot(v: &TemporalWorkflow) -> Result { Ok(Snapshot::Old(snapshot_old(v)?)) } fn snapshot_old(v: &TemporalWorkflow) -> Result { + // The Full operator codec reuses this structural carrier, with explicit + // reserved Validation rows. Old Train-only and old15 roles stay unchanged. + let operator_checks=v.full_run.is_some()&&v.schedule()?.is_operator_eight_step(); if v.session.fitted || v.session.failed || v.session.active - || v.session.rows.iter().any(|r| r.role != DataRole::Train) + || v.session.rows.iter().any(|r| r.role != DataRole::Train && !(operator_checks&&r.role==DataRole::Validation)) || v.session.rows.len() != v.ledger.spans.len() { return Err("temporal_checkpoint_complete_train_prefix_only".into()); @@ -194,7 +197,7 @@ pub(super) fn decode_full_post_frozen( } fn decode_prefix(p:Prefix,descriptor:&RecoveryDescriptor,now:u64,contract:StructuralDecode<'_>)->Result{ let schedule = p.binding.contract()?; - let full=matches!(&contract,StructuralDecode::FullPostFrozen{..}); + let full=matches!(&contract,StructuralDecode::FullPostFrozen{..})||schedule.is_operator_eight_step(); let frozen=match &contract{ StructuralDecode::OldTrain|StructuralDecode::FullTrain{..}=>None, StructuralDecode::FullPostFrozen{fit,..}=>Some(crate::room_learner::managed_schedule_contract::FrozenModelIdentity::from_runtime_verified_artifact(&schedule,fit.body.artifact_sha256.clone())?), @@ -457,9 +460,10 @@ fn decode_prefix(p:Prefix,descriptor:&RecoveryDescriptor,now:u64,contract:Struct } } if let Some(bridge) = &p.bridge { - if !full { + if !full || schedule.is_operator_eight_step() { if matches!(contract,StructuralDecode::FullTrain{..}){ - if bridge.seals.len()>schedule.counts().train_episodes{return Err("temporal_bridge_phase_order".into());} + let max_seals=if schedule.is_operator_eight_step(){schedule.episodes().len()}else{schedule.counts().train_episodes}; + if bridge.seals.len()>max_seals{return Err("temporal_bridge_phase_order".into());} let mut count=0;let cuts=bridge.seals.iter().enumerate().map(|(i,_)|{count+=schedule.episodes()[i].slots.len();count}).collect::>(); let digests=prefix_digests::canonical_tuple_prefixes(&p.rows,&p.evidence,&p.ledger.spans,&cuts)?; bridge.validate_full_run_with_prefixes(&schedule,&p.run,&p.rows,&p.evidence,&p.ledger,p.episode,p.mapping.as_ref(),p.last_now_ns,&digests)?; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint_tests.rs index 1c07ba97f0..5ca5f4d414 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_checkpoint_tests.rs @@ -316,3 +316,18 @@ fn writer_readback_reuses_exact_authoritative_arc_records_and_rejects_changed_re assert_eq!(reuse_snapshot_records(&mut restored,&changed).unwrap_err(),"temporal_writer_shared_records_changed"); assert!(restored.session.rows[0].same_record(&workflow.session.rows[0])); } + +#[test] +fn old_train_snapshot_still_refuses_validation_heldout_and_inference_rows(){ + let(_dir,store,mut workflow)=setup();workflow.persist_initial(&store).unwrap(); + start(&mut workflow,&store,2_000_000_000,"synthetic-old-role-boundary");let mut sequence=0; + commit_next(&mut workflow,&store,2_000_000_000,&mut sequence); + let original=workflow.session.rows[0].clone(); + let canonical=encode(&workflow).unwrap(); + for role in [DataRole::Validation,DataRole::HeldOut,DataRole::Inference]{ + let mut row=(*original).clone();row.role=role;workflow.session.rows[0]=SharedRecord::new(row); + assert!(matches!(snapshot_old(&workflow),Err(error)if error=="temporal_checkpoint_complete_train_prefix_only")); + } + workflow.session.rows[0]=original; + assert_eq!(encode(&workflow).unwrap(),canonical); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_durability.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_durability.rs index 92384351c9..ee172d333c 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_durability.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_durability.rs @@ -104,7 +104,7 @@ enum Work { } struct Pending { intent: TemporalIntent, - prefix: checkpoint::ValidatedTemporalPrefix, + prefix: Arc, id: u64, slot: bool, rows: usize, @@ -112,6 +112,8 @@ struct Pending { records: checkpoint::SharedRecords, } struct Shared { + failed_repair: Mutex)>>, + terminal_drained: AtomicBool, cancelled: AtomicBool, busy: AtomicBool, now_ns: AtomicU64, @@ -120,6 +122,8 @@ struct Shared { #[cfg(test)] lose_next_ack: AtomicBool, #[cfg(test)] + fail_next_publication: AtomicBool, + #[cfg(test)] ack_queued: AtomicBool, #[cfg(test)] corrupt_ack_id: AtomicBool, @@ -127,6 +131,8 @@ struct Shared { hold: Mutex, mpsc::Receiver<()>)>>, #[cfg(test)] hold_after_prepare: Mutex, mpsc::Receiver<()>)>>, + #[cfg(test)] + hold_after_publish: Mutex, mpsc::Receiver<()>)>>, } impl Shared { fn now(&self) -> Result { @@ -154,6 +160,7 @@ impl Shared { } } pub(crate) struct Writer { + store: Arc, send: mpsc::SyncSender, receive: Mutex>, shared: Arc, @@ -164,6 +171,8 @@ impl Writer { let (send, requests) = mpsc::sync_channel(1); let (replies, receive) = mpsc::sync_channel(1); let shared = Arc::new(Shared { + failed_repair: Mutex::new(None), + terminal_drained: AtomicBool::new(false), cancelled: AtomicBool::new(false), busy: AtomicBool::new(false), now_ns: AtomicU64::new(now), @@ -172,6 +181,8 @@ impl Writer { #[cfg(test)] lose_next_ack: AtomicBool::new(false), #[cfg(test)] + fail_next_publication: AtomicBool::new(false), + #[cfg(test)] ack_queued: AtomicBool::new(false), #[cfg(test)] corrupt_ack_id: AtomicBool::new(false), @@ -179,11 +190,15 @@ impl Writer { hold: Mutex::new(None), #[cfg(test)] hold_after_prepare: Mutex::new(None), + #[cfg(test)] + hold_after_publish: Mutex::new(None), }); let state = shared.clone(); + let worker_store = store.clone(); let thread = std::thread::Builder::new() .name("ruview-temporal-durable".into()) .spawn(move || { + let store = worker_store; let mut pending: Option = None; while let Ok(work) = requests.recv() { let (id, slot, rows, seals, start_ticket, result) = match work { @@ -219,6 +234,8 @@ impl Writer { &prefix, state.allowed(&request.descriptor)?, )?; + let prefix = Arc::new(prefix); + *state.failed_repair.lock().map_err(|_| "temporal_writer_repair_lock")? = Some((intent.clone(), prefix.clone())); pending = Some(Pending { intent, prefix, @@ -234,11 +251,14 @@ impl Writer { let _ = release.recv(); } let p = pending.as_ref().unwrap(); - let token = store.publish_temporal( - &p.intent, - Some(&p.prefix), - state.allowed(p.prefix.descriptor())?, - )?; + #[cfg(not(test))] + let token = store.publish_temporal(&p.intent, Some(&p.prefix), state.allowed(p.prefix.descriptor())?)?; + #[cfg(test)] + let token = if state.fail_next_publication.swap(false,Ordering::AcqRel){ + store.synthetic_fail_before_head_publication(&p.intent,&p.prefix,state.allowed(p.prefix.descriptor())?) + }else{store.publish_temporal(&p.intent,Some(&p.prefix),state.allowed(p.prefix.descriptor())?)}?; + #[cfg(test)] + if let Some((entered,release))=state.hold_after_publish.lock().map_err(|_|"temporal_writer_test_lock")?.take(){let _=entered.send(());let _=release.recv();} #[cfg(test)] if state.lose_next_ack.swap(false, Ordering::AcqRel) { return Err("synthetic_lost_publish_ack".into()); @@ -247,6 +267,7 @@ impl Writer { })(); if result.is_ok() { pending = None; + if let Ok(mut ticket) = state.failed_repair.lock() { *ticket = None; } } (meta.0, meta.1, meta.2, meta.3, false, result) } @@ -331,6 +352,7 @@ impl Writer { })(); if result.is_ok() { pending = None; + if let Ok(mut ticket) = state.failed_repair.lock() { *ticket = None; } } (meta.0, meta.1, meta.2, meta.3, false, result) } @@ -366,6 +388,7 @@ impl Writer { }) .map_err(|_| "temporal_writer_spawn_failed")?; Ok(Self { + store, send, receive: Mutex::new(receive), shared, @@ -467,6 +490,24 @@ impl Writer { } Ok(()) } + /// Explicit owner Retry only, outside Runtime.inner. The thread lock also + /// serializes erasure's drain, so no exact-intent repair can publish after + /// that drain has returned. Never infer a failed write from a lost ACK. + pub(crate) fn reconcile_failed(&self, now: u64) -> Result<(), String> { + self.observe_now(now); + self.cancel(); + let mut thread = self.thread.lock().map_err(|_| "temporal_writer_join_lock")?; + if self.shared.terminal_drained.load(Ordering::Acquire) { return Err("temporal_writer_terminally_drained".into()); } + if let Some(worker) = thread.take() { + worker.join().map_err(|_| "temporal_writer_panicked")?; + } + let mut repair = self.shared.failed_repair.lock().map_err(|_| "temporal_writer_repair_lock")?; + if let Some((intent, prefix)) = repair.as_ref() { + self.store.repair_same_temporal(intent, Some(prefix.as_ref()), self.shared.now()?)?; + *repair = None; + } + Ok(()) + } pub fn cancel(&self) { self.shared.cancelled.store(true, Ordering::Release); let _ = self.send.try_send(Work::Stop); @@ -475,18 +516,16 @@ impl Writer { /// undone; this barrier prevents an old worker from publishing after erase. pub fn drain(&self) -> Result<(), String> { self.cancel(); - let joined = if let Some(thread) = self - .thread - .lock() - .map_err(|_| "temporal_writer_join_lock")? - .take() - { - thread - .join() - .map_err(|_| "temporal_writer_panicked".to_string()) - } else { - Ok(()) - }; + let mut handle = self.thread.lock().map_err(|_| "temporal_writer_join_lock")?; + // Terminal owner invalidation/erase barrier. Unlike Retry's own join, + // no captured Writer Arc may reconcile or publish after this returns. + self.shared.terminal_drained.store(true, Ordering::Release); + let joined = if let Some(thread) = handle.take() { + thread.join().map_err(|_| "temporal_writer_panicked".to_string()) + } else { Ok(()) }; + let repair = self.shared.failed_repair.lock().map(|mut value|value.take()).unwrap_or_else(|error|error.into_inner().take()); + drop(repair); + drop(handle); // A revoked/failed capture may never poll its final ACK. Release its // large serialized/candidate buffers here, not when Core drops inner. if let Ok(receiver) = self.receive.lock() { @@ -497,6 +536,12 @@ impl Writer { joined } #[cfg(test)] + pub(crate) fn fail_next_publication(&self){self.shared.fail_next_publication.store(true,Ordering::Release);} + #[cfg(test)] + pub(crate) fn has_failed_repair(&self)->bool{self.shared.failed_repair.lock().unwrap().is_some()} + #[cfg(test)] + pub(crate) fn hold_next_after_publish(&self,entered:mpsc::SyncSender<()>,release:mpsc::Receiver<()>){*self.shared.hold_after_publish.lock().unwrap()=Some((entered,release));} + #[cfg(test)] pub fn ack_is_queued(&self) -> bool { self.shared.ack_queued.load(Ordering::Acquire) } diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation.rs index d56e4a93be..16c5c7661e 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation.rs @@ -11,6 +11,9 @@ use sha2::{Digest, Sha256}; pub(super) const TRAIN_SCHEMA: &str = "ruview.room-learner.temporal-training-proof.v1"; pub(super) const HELD_OUT_SCHEMA: &str = "ruview.room-learner.temporal-heldout-evidence.v1"; pub(super) const EVALUATION_SCHEMA: &str = "ruview.room-learner.temporal-evaluation.v1"; +pub(super) const OPERATOR_TRAIN_SCHEMA: &str = "ruview.room-learner.temporal-training-proof.operator-eight-step.v1"; +pub(super) const OPERATOR_CHECK_SCHEMA: &str = "ruview.room-learner.temporal-validation-evidence.operator-eight-step.v1"; +pub(super) const OPERATOR_EVALUATION_SCHEMA: &str = "ruview.room-learner.temporal-evaluation.operator-eight-step.v1"; const MODEL_KIND: &str = "ruview.room-learner.managed-temporal-input.v3"; const PREFIX_SCHEMA: &str = "ruview.room-learner.temporal-runtime-prefix.v3"; const METADATA_KIND: &str = "temporal-prefix-commitment-v2"; @@ -206,7 +209,7 @@ pub(super) struct TemporalTrainProofBodyData { pub training_data_sha256: String, pub training_evidence_sha256: String, pub training_identities_sha256: String, - pub train_seal_sha256s: [String; 7], + pub train_seal_sha256s: Vec, pub train_seal_chain_sha256: String, pub train_windows: u16, pub windows_per_condition: [u16; 4], @@ -224,6 +227,7 @@ pub(super) struct TemporalTrainProofData { pub sha256: String, } impl TemporalTrainProofData { + pub(super) fn is_operator_eight_step(&self) -> bool { self.schema == OPERATOR_TRAIN_SCHEMA } pub(super) fn decode_data(bytes: &[u8]) -> Result { let value: Self = decode_data(bytes)?; value.validate_data()?; @@ -235,9 +239,9 @@ impl TemporalTrainProofData { b.claims.validate_data()?; b.accepted_train_commitment.validate_data(&b.identity)?; let cut = decimal(&b.freeze_prepare_receiver_ns)?; - if self.schema != TRAIN_SCHEMA + if ![TRAIN_SCHEMA, OPERATOR_TRAIN_SCHEMA].contains(&self.schema.as_str()) || !sha(&self.sha256) - || self.sha256 != data_sha(TRAIN_SCHEMA, b)? + || self.sha256 != data_sha(&self.schema, b)? || [ &b.training_data_sha256, &b.training_evidence_sha256, @@ -251,10 +255,11 @@ impl TemporalTrainProofData { .iter() .collect::>() .len() - != 7 + != if self.is_operator_eight_step() { 4 } else { 7 } + || b.train_seal_sha256s.len() != if self.is_operator_eight_step() { 4 } else { 7 } || b.train_windows != 64 || b.windows_per_condition != [16; 4] - || b.declared_episodes_per_condition != [1, 2, 2, 2] + || b.declared_episodes_per_condition != if self.is_operator_eight_step() { [1; 4] } else { [1, 2, 2, 2] } || !b.identity.original_deadline.contains_receiver_ns(cut) { return invalid(); @@ -290,7 +295,7 @@ pub(super) struct TemporalHeldOutBodyData { pub complete_evidence_sha256: String, pub complete_identities_sha256: String, pub complete_ledger_sha256: String, - pub held_out_seal_sha256s: [String; 8], + pub held_out_seal_sha256s: Vec, pub held_out_seal_chain_sha256: String, pub complete_record_sha256: String, pub complete_commitment: CommitmentData, @@ -310,6 +315,7 @@ pub(super) struct TemporalHeldOutEvidenceData { pub sha256: String, } impl TemporalHeldOutEvidenceData { + pub(super) fn is_operator_eight_step(&self) -> bool { self.schema == OPERATOR_CHECK_SCHEMA } pub(super) fn decode_data(bytes: &[u8]) -> Result { let value: Self = decode_data(bytes)?; value.validate_data()?; @@ -320,9 +326,9 @@ impl TemporalHeldOutEvidenceData { b.identity.validate_data()?; b.claims.validate_data()?; b.complete_commitment.validate_data(&b.identity)?; - if self.schema != HELD_OUT_SCHEMA + if ![HELD_OUT_SCHEMA, OPERATOR_CHECK_SCHEMA].contains(&self.schema.as_str()) || !sha(&self.sha256) - || self.sha256 != data_sha(HELD_OUT_SCHEMA, b)? + || self.sha256 != data_sha(&self.schema, b)? || [ &b.training_proof_sha256, &b.held_out_data_sha256, @@ -342,10 +348,11 @@ impl TemporalHeldOutEvidenceData { .iter() .collect::>() .len() - != 8 - || b.held_out_windows != 64 - || b.windows_per_condition != [16; 4] - || b.declared_episodes_per_condition != [2; 4] + != if self.is_operator_eight_step() { 4 } else { 8 } + || b.held_out_seal_sha256s.len() != if self.is_operator_eight_step() { 4 } else { 8 } + || b.held_out_windows != if self.is_operator_eight_step() { 16 } else { 64 } + || b.windows_per_condition != if self.is_operator_eight_step() { [4; 4] } else { [16; 4] } + || b.declared_episodes_per_condition != if self.is_operator_eight_step() { [1; 4] } else { [2; 4] } || !b .identity .original_deadline @@ -379,7 +386,8 @@ impl TemporalHeldOutEvidenceData { pub(super) fn validate_data_links(&self, train: &TemporalTrainProofData) -> Result<(), String> { self.validate_data()?; train.validate_data()?; - if self.body.identity != train.body.identity + if self.is_operator_eight_step() != train.is_operator_eight_step() + || self.body.identity != train.body.identity || self.body.training_proof_sha256 != train.sha256 || decimal(&self.body.complete_prepare_receiver_ns)? < decimal(&train.body.freeze_prepare_receiver_ns)? @@ -433,6 +441,7 @@ pub(super) struct TemporalEvaluationReceiptData { pub sha256: String, } impl TemporalEvaluationReceiptData { + pub(super) fn is_operator_eight_step(&self) -> bool { self.schema == OPERATOR_EVALUATION_SCHEMA } pub(super) fn decode_data(bytes: &[u8]) -> Result { let value: Self = decode_data(bytes)?; value.validate_data()?; @@ -444,9 +453,9 @@ impl TemporalEvaluationReceiptData { b.claims.validate_data()?; b.report.validate()?; let ns = decimal(&b.evaluated_at_receiver_ns)?; - if self.schema != EVALUATION_SCHEMA + if ![EVALUATION_SCHEMA, OPERATOR_EVALUATION_SCHEMA].contains(&self.schema.as_str()) || !sha(&self.sha256) - || self.sha256 != data_sha(EVALUATION_SCHEMA, b)? + || self.sha256 != data_sha(&self.schema, b)? || [ &b.training_proof_sha256, &b.held_out_evidence_sha256, @@ -456,19 +465,19 @@ impl TemporalEvaluationReceiptData { ] .iter() .any(|s| !sha(s)) - || b.role != DataRole::HeldOut + || b.role != if self.is_operator_eight_step() { DataRole::Validation } else { DataRole::HeldOut } || b.window_unit != "native_receiver_2s_window" || !b.train_lineage_checked || ns / 1_000_000 != b.evaluated_at_unix_ms || !b.identity.original_deadline.contains_receiver_ns(ns) || b.provenance != [CaptureProvenance::LiveReceiverObserved] || b.clock_claims != [ClockKind::ReceiverOnly] - || b.report.samples != 64 - || b.report.declared_trial_count != 8 + || b.report.samples != if self.is_operator_eight_step() { 16 } else { 64 } + || b.report.declared_trial_count != if self.is_operator_eight_step() { 4 } else { 8 } || b.report .per_condition .iter() - .any(|c| c.samples != 16 || c.declared_trial_count != 2) + .any(|c| c.samples != if self.is_operator_eight_step() { 4 } else { 16 } || c.declared_trial_count != if self.is_operator_eight_step() { 1 } else { 2 }) { return invalid(); } @@ -482,7 +491,8 @@ impl TemporalEvaluationReceiptData { self.validate_data()?; held.validate_data_links(train)?; let b = &self.body; - if b.identity != held.body.identity + if self.is_operator_eight_step() != train.is_operator_eight_step() + || b.identity != held.body.identity || b.training_proof_sha256 != train.sha256 || b.held_out_evidence_sha256 != held.sha256 || b.complete_record_sha256 != held.body.complete_record_sha256 @@ -501,9 +511,10 @@ impl TemporalEvaluationReceiptData { // producer, current-store, evaluation, qualification or selection permission. impl TemporalTrainProofData { pub(super) fn from_body_data(body: TemporalTrainProofBodyData) -> Result { + let schema = if body.train_seal_sha256s.len() == 4 && body.declared_episodes_per_condition == [1; 4] { OPERATOR_TRAIN_SCHEMA } else { TRAIN_SCHEMA }; let value = Self { - schema: TRAIN_SCHEMA.into(), - sha256: data_sha(TRAIN_SCHEMA, &body)?, + schema: schema.into(), + sha256: data_sha(schema, &body)?, body, }; value.validate_data()?; @@ -517,9 +528,10 @@ impl TemporalTrainProofData { } impl TemporalHeldOutEvidenceData { pub(super) fn from_body_data(body: TemporalHeldOutBodyData) -> Result { + let schema = if body.held_out_windows == 16 && body.held_out_seal_sha256s.len() == 4 { OPERATOR_CHECK_SCHEMA } else { HELD_OUT_SCHEMA }; let value = Self { - schema: HELD_OUT_SCHEMA.into(), - sha256: data_sha(HELD_OUT_SCHEMA, &body)?, + schema: schema.into(), + sha256: data_sha(schema, &body)?, body, }; value.validate_data()?; @@ -533,9 +545,10 @@ impl TemporalHeldOutEvidenceData { } impl TemporalEvaluationReceiptData { pub(super) fn from_body_data(body: TemporalEvaluationBodyData) -> Result { + let schema = if body.role == DataRole::Validation { OPERATOR_EVALUATION_SCHEMA } else { EVALUATION_SCHEMA }; let value = Self { - schema: EVALUATION_SCHEMA.into(), - sha256: data_sha(EVALUATION_SCHEMA, &body)?, + schema: schema.into(), + sha256: data_sha(schema, &body)?, body, }; value.validate_data()?; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_data_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_data_tests.rs index 20d9e084f5..5a2cce8a0e 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_data_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_data_tests.rs @@ -61,7 +61,7 @@ fn train() -> TemporalTrainProofData { training_data_sha256: fixture_sha(20), training_evidence_sha256: fixture_sha(21), training_identities_sha256: fixture_sha(22), - train_seal_sha256s: std::array::from_fn(|i| fixture_sha(200 + i as u64)), + train_seal_sha256s: (0..7).map(|i| fixture_sha(200 + i)).collect(), train_seal_chain_sha256: fixture_sha(23), train_windows: 64, windows_per_condition: [16; 4], @@ -86,7 +86,7 @@ fn held(t: &TemporalTrainProofData) -> TemporalHeldOutEvidenceData { complete_evidence_sha256: fixture_sha(34), complete_identities_sha256: fixture_sha(35), complete_ledger_sha256: fixture_sha(36), - held_out_seal_sha256s: std::array::from_fn(|i| fixture_sha(300 + i as u64)), + held_out_seal_sha256s: (0..8).map(|i| fixture_sha(300 + i)).collect(), held_out_seal_chain_sha256: fixture_sha(37), complete_record_sha256: fixture_sha(38), complete_commitment: token(&t.body.identity, 160), @@ -437,3 +437,63 @@ fn final_complete_source_fences_are_exact_bounded_native_metadata_not_a_live_per ); assert!(!h.body.claims.adoption_authorized); } + + +fn operator_records() -> (TemporalTrainProofData, TemporalHeldOutEvidenceData, TemporalEvaluationReceiptData) { + let mut tb = train().body; + tb.train_seal_sha256s.truncate(4); + tb.declared_episodes_per_condition = [1; 4]; + let t = TemporalTrainProofData::from_body_data(tb).unwrap(); + let mut hb = held(&t).body; + hb.held_out_seal_sha256s.truncate(4); + hb.held_out_windows = 16; + hb.windows_per_condition = [4; 4]; + hb.declared_episodes_per_condition = [1; 4]; + let h = TemporalHeldOutEvidenceData::from_body_data(hb).unwrap(); + let mut eb = evaluation(&t, &h).body; + eb.role = DataRole::Validation; + let mut report = serde_json::to_value(&eb.report).unwrap(); + for key in ["samples", "correct", "scored", "binary_cross_entropy_samples"] { report[key] = json!(16); } + for key in ["declared_trial_count", "declared_session_count"] { report[key] = json!(4); } + report["confusion"] = confusion(8, 8); + for c in report["per_condition"].as_array_mut().unwrap() { + for key in ["samples", "correct", "scored", "binary_cross_entropy_samples"] { c[key] = json!(4); } + for key in ["declared_trial_count", "declared_session_count"] { c[key] = json!(1); } + let occupied = c["condition"].as_str().unwrap().starts_with("Occupied"); + c["confusion"] = confusion(if occupied { 0 } else { 4 }, if occupied { 4 } else { 0 }); + } + eb.report = serde_json::from_value(report).unwrap(); + let e = TemporalEvaluationReceiptData::from_body_data(eb).unwrap(); + (t, h, e) +} + +#[test] +fn operator_eight_step_validation_metadata_roundtrips_without_heldout_claims() { + let (t, h, e) = operator_records(); + assert_eq!(t.schema, OPERATOR_TRAIN_SCHEMA); + assert_eq!(h.schema, OPERATOR_CHECK_SCHEMA); + assert_eq!(e.schema, OPERATOR_EVALUATION_SCHEMA); + assert_eq!(e.body.role, DataRole::Validation); + assert_eq!(e.body.report.samples, 16); + assert!(!e.body.report.independence_verified); + e.validate_data_links(&t, &h).unwrap(); + assert_eq!(TemporalTrainProofData::decode_data(&t.encode_data().unwrap()).unwrap(), t); + assert_eq!(TemporalHeldOutEvidenceData::decode_data(&h.encode_data().unwrap()).unwrap(), h); + assert_eq!(TemporalEvaluationReceiptData::decode_data(&e.encode_data().unwrap()).unwrap(), e); +} + +#[test] +fn operator_and_legacy_metadata_cannot_be_relabelled_or_cross_linked_after_rehash() { + let (t, h, e) = operator_records(); + let mut old_name = t.clone(); old_name.schema = TRAIN_SCHEMA.into(); old_name.sha256 = data_sha(&old_name.schema, &old_name.body).unwrap(); + assert!(old_name.validate_data().is_err()); + let mut invented = train(); invented.schema = OPERATOR_TRAIN_SCHEMA.into(); invented.sha256 = data_sha(&invented.schema, &invented.body).unwrap(); + assert!(invented.validate_data().is_err()); + let mut wrong_role = e.clone(); wrong_role.body.role = DataRole::HeldOut; wrong_role.sha256 = data_sha(&wrong_role.schema, &wrong_role.body).unwrap(); + assert!(wrong_role.validate_data().is_err()); + let mut wrong_count = h.clone(); wrong_count.body.held_out_windows = 64; wrong_count.sha256 = data_sha(&wrong_count.schema, &wrong_count.body).unwrap(); + assert!(wrong_count.validate_data().is_err()); + let legacy = train(); + let mut crossed = h.clone(); crossed.body.training_proof_sha256 = legacy.sha256.clone(); crossed.sha256 = data_sha(&crossed.schema, &crossed.body).unwrap(); + assert!(crossed.validate_data_links(&legacy).is_err()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_projection.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_projection.rs index 1af9c3fa40..4c3ff453f2 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_projection.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_projection.rs @@ -112,6 +112,7 @@ impl TemporalWorkflow { held_out: bool, schedule: &ManagedScheduleContract, ) -> Result { + let operator = schedule.is_operator_eight_step(); let mode = self.full_run.as_ref().ok_or("temporal_full_run_required")?; let owner = mode .owner @@ -173,14 +174,19 @@ impl TemporalWorkflow { .ok_or("temporal_complete_record_required")?; if fact.token != *token || fact.record_sha256 != record.sha256 - || mode.held_out_seals() != 8 - || self.session.rows.len() != 128 + || (if operator { + self.bridge.as_ref().map_or(0, |b| b.seals.iter().filter(|s| + schedule.episodes().get(s.episode).is_some_and(|e| e.role == DataRole::Validation) + ).count()) + } else { mode.held_out_seals() }) != if operator { 4 } else { 8 } + || self.session.rows.len() != if operator { 80 } else { 128 } { return Err("temporal_current_complete_changed".into()); } Some(record.sha256.clone()) } else { - if !(64..=128).contains(&self.session.rows.len()) { + if (operator && self.session.rows.len() != 80) + || (!operator && !(64..=128).contains(&self.session.rows.len())) { return Err("temporal_stable_train_projection_required".into()); } None @@ -324,6 +330,16 @@ impl PendingTemporalEvidenceSource { verify_model_binding(record, &self.model)?; checkpoint::validate_temporal_evaluation_train_prefix(&w, record)?; self.check(now, cancel)?; + let schedule = w.schedule()?; + let operator = schedule.is_operator_eight_step(); + if loaded.managed_is_operator_eight_step()? != operator + || self.model.managed_is_operator_eight_step()? != operator { + return Err("temporal_evaluation_profile_changed".into()); + } + let check_role = if operator { DataRole::Validation } else { DataRole::HeldOut }; + let check_rows: Vec<_> = w.session.rows.iter().filter(|r| r.role == check_role).cloned().collect(); + let check_evidence: Vec<_> = w.session.evidence.iter().zip(&w.session.rows) + .filter(|(_, r)| r.role == check_role).map(|(e, _)| e.clone()).collect(); let identity = data::TemporalEvaluationIdentityData { owner_scope: w.descriptor.owner_scope.clone(), installation_id: w.descriptor.installation_id.clone(), @@ -367,14 +383,13 @@ impl PendingTemporalEvidenceSource { training_identities_sha256: record.body.identity_evidence_sha256.clone(), train_seal_sha256s: seals .iter() + .filter(|s| schedule.episodes().get(s.episode).is_some_and(|e| e.role == DataRole::Train)) .map(|s| s.sha256.clone()) - .collect::>() - .try_into() - .map_err(|_| "temporal_train_seven_seals_required")?, + .collect(), train_seal_chain_sha256: record.body.train_seal_chain_sha256.clone(), train_windows: 64, windows_per_condition: [16; 4], - declared_episodes_per_condition: [1, 2, 2, 2], + declared_episodes_per_condition: if operator { [1; 4] } else { [1, 2, 2, 2] }, accepted_train_commitment: commitment(&record.body.accepted_train_commitment)?, freeze_prepare_receiver_ns: record.body.freeze_prepare_receiver_ns.to_string(), source_floors: record @@ -397,7 +412,7 @@ impl PendingTemporalEvidenceSource { .as_ref() .ok_or("temporal_actual_complete_required")?; if Some(&complete.sha256) != self.binding.complete.as_ref() - || w.session.rows.len() != 128 + || w.session.rows.len() != if operator { 80 } else { 128 } || current.token != self.binding.token { return Err("temporal_current_complete_changed".into()); @@ -407,28 +422,30 @@ impl PendingTemporalEvidenceSource { .managed .as_ref() .ok_or("managed_session_required")?; - let hs = &mode - .held_out - .as_ref() - .ok_or("temporal_heldout_seals_required")? - .seals; + let check_windows: Vec<_> = managed.windows.iter().filter(|w| w.role == check_role).cloned().collect(); + if check_rows.len() != if operator { 16 } else { 64 } + || check_evidence.len() != check_rows.len() + || check_windows.len() != check_rows.len() { + return Err("temporal_evaluation_reserved_check_shape".into()); + } + let check_seals: Vec = if operator { + seals.iter().filter(|s| schedule.episodes().get(s.episode).is_some_and(|e| e.role == DataRole::Validation)) + .map(|s| s.sha256.clone()).collect() + } else { + mode.held_out.as_ref().ok_or("temporal_heldout_seals_required")?.seals.iter().map(|s| s.sha256.clone()).collect() + }; let evidence = data::TemporalHeldOutEvidenceData::from_body_data(data::TemporalHeldOutBodyData { identity, training_proof_sha256: proof.sha256.clone(), - held_out_data_sha256: hash(&(&w.session.rows[64..]))?, - held_out_evidence_sha256: hash(&(&w.session.evidence[64..]))?, - held_out_identities_sha256: hash(&(&managed.windows[64..]))?, + held_out_data_sha256: hash(&check_rows)?, + held_out_evidence_sha256: hash(&check_evidence)?, + held_out_identities_sha256: hash(&check_windows)?, complete_rows_sha256: complete.body.raw_sha256.clone(), complete_evidence_sha256: complete.body.evidence_sha256.clone(), complete_identities_sha256: complete.body.identity_sha256.clone(), complete_ledger_sha256: complete.body.ledger_sha256.clone(), - held_out_seal_sha256s: hs - .iter() - .map(|s| s.sha256.clone()) - .collect::>() - .try_into() - .map_err(|_| "temporal_eight_heldout_seals_required")?, + held_out_seal_sha256s: check_seals, held_out_seal_chain_sha256: complete.body.held_out_chain_sha256.clone(), complete_record_sha256: complete.sha256.clone(), complete_commitment: commitment(¤t.token.data())?, @@ -452,9 +469,9 @@ impl PendingTemporalEvidenceSource { .collect::>() .try_into() .map_err(|_| "temporal_evaluation_three_sources_required")?, - held_out_windows: 64, - windows_per_condition: [16; 4], - declared_episodes_per_condition: [2; 4], + held_out_windows: if operator { 16 } else { 64 }, + windows_per_condition: if operator { [4; 4] } else { [16; 4] }, + declared_episodes_per_condition: if operator { [1; 4] } else { [2; 4] }, claims: data::DiagnosticClaimsData::diagnostic_only(), })?; evidence.validate_data_links(&proof)?; @@ -463,7 +480,7 @@ impl PendingTemporalEvidenceSource { None }; let rows = if self.held_out { - w.session.rows[64..].to_vec() + check_rows } else { vec![] }; @@ -542,7 +559,7 @@ impl VerifiedTemporalEvaluationSnapshot { complete_record_sha256: self.held_out().body.complete_record_sha256.clone(), training_data_sha256: self.proof().body.training_data_sha256.clone(), held_out_data_sha256: self.held_out().body.held_out_data_sha256.clone(), - role: DataRole::HeldOut, + role: if self.proof().is_operator_eight_step() { DataRole::Validation } else { DataRole::HeldOut }, window_unit: "native_receiver_2s_window".into(), evaluated_at_receiver_ns: after.to_string(), evaluated_at_unix_ms: after / 1_000_000, diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_store.rs index 29c605c78b..483f360fce 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_store.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_evaluation_store.rs @@ -39,13 +39,13 @@ impl StoredTemporalEvaluation { fn lookup(source: &VerifiedTemporalEvaluationSnapshot) -> Result { let identity = &source.proof().body.identity; crate::room_learner::qualification::hash(&( - data::EVALUATION_SCHEMA, + if source.proof().is_operator_eight_step() { data::OPERATOR_EVALUATION_SCHEMA } else { data::EVALUATION_SCHEMA }, "lookup", &identity.artifact_kind, &identity.artifact_sha256, &source.proof().sha256, &source.held_out().sha256, - DataRole::HeldOut, + if source.proof().is_operator_eight_step() { DataRole::Validation } else { DataRole::HeldOut }, )) } fn allowed( @@ -405,7 +405,7 @@ impl ArtifactStore { let key = crate::room_learner::qualification::hash(&( crate::room_learner::temporal_wire::ASSESSMENT_SCHEMA, &evaluation.receipt().sha256, - crate::room_learner::temporal_wire::TemporalQualificationPolicyWire::current(), + crate::room_learner::temporal_wire::TemporalQualificationPolicyWire::for_operator_eight_step(source.proof().is_operator_eight_step()), ))?; let path = directory.join(format!("assessment-{key}.json")); let (assessment, qualification) = match std::fs::symlink_metadata(&path) { diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs index 52fb296f69..353354a467 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs @@ -489,12 +489,23 @@ impl TemporalWorkflow { .iter() .find(|s| s.source_id == p.source_id) .ok_or("temporal_unbound_source")?; - if self.source_high.iter().any(|(id, _, last)| { + if let Some((_, _, last)) = self.source_high.iter().find(|(id, _, last)| { *id == p.source_id && *last >= mapping.receiver_start_ns && (t - *last > policy.max_gap_ns || t - *last < policy.min_packet_interval_ns) }) { - return Err("temporal_observed_gap_or_rate".into()); + let interval_ns = t - *last; + let direction = if interval_ns < policy.min_packet_interval_ns { + "below_min" + } else { + "above_max" + }; + // Receiver-processing interval only: no RF values, absolute time or + // source-clock claim. Keep the existing failure prefix and guards. + return Err(format!( + "temporal_observed_gap_or_rate: source={} direction={} receiver_interval_ns={} min_ns={} max_ns={}", + p.source_id, direction, interval_ns, policy.min_packet_interval_ns, policy.max_gap_ns + )); } let first_limit = native.max_first_packet_delay_ms() * 1_000_000; if let Some((_, _, last)) = self diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution_tests.rs index 266e8c897d..cbe9a3a5e9 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution_tests.rs @@ -437,6 +437,75 @@ fn receiver_tick_withdraws_stale_segment_without_advancing_observed_time() { assert_eq!(w.accepted(), 0); } +#[test] +fn receiver_interval_diagnostics_preserve_exact_frozen_admission_boundaries() { + for boundary in ["below", "minimum", "maximum", "above"] { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + let base = 2_000_000_000; + start(&mut w, &store, base, "interval-boundaries"); + for source in 1..=3u8 { + let at = base + u64::from(source) * 1000; + w.observe(&packet(source, 1, at, RoomLabel::Empty, 0), at) + .unwrap(); + } + let policy = &w.session.managed.as_ref().unwrap().binding.native.source_policies()[0]; + let (minimum, maximum) = (policy.min_packet_interval_ns, policy.max_gap_ns); + assert_eq!((minimum, maximum), (27_777_777, 50_000_000)); + let interval = match boundary { + "below" => minimum - 1, + "minimum" => minimum, + "maximum" => maximum, + _ => maximum + 1, + }; + let before = (w.live_observed_ns, w.source_high.clone(), w.accepted()); + let at = base + 1000 + interval; + let result = w.observe(&packet(1, 2, at, RoomLabel::Empty, 0), at); + match boundary { + "minimum" | "maximum" => result.unwrap(), + "below" => { + let error = result.unwrap_err(); + assert_eq!(error, format!( + "temporal_observed_gap_or_rate: source=1 direction=below_min receiver_interval_ns={} min_ns={} max_ns={}", + interval, minimum, maximum + )); + assert!(error.len() <= 256); + assert_eq!((w.live_observed_ns, w.source_high.clone(), w.accepted()), before); + } + _ => { + // Custody still withdraws a too-late source before the interval + // diagnostic. Do not replace the earlier failure or admit it. + assert_eq!(result.unwrap_err(), "temporal_missing_segment_source"); + assert!(w.custody.failed); + assert_eq!((w.live_observed_ns, w.source_high.clone(), w.accepted()), before); + } + } + assert!(w.session.rows.is_empty()); + assert_eq!(w.durable_windows, 0); + } +} + +#[test] +fn synthetic_receiver_burst_reports_interval_without_advancing_observation() { + let (_dir, store, mut w) = setup(); + w.persist_initial(&store).unwrap(); + // A synthetic delay before the actual receiver start earns no observation. + let base = 12_000_000_000; + start(&mut w, &store, base, "synthetic-queued-drain"); + for source in 1..=3u8 { + let at = base + u64::from(source) * 1000; + w.observe(&packet(source, 1, at, RoomLabel::Empty, 0), at) + .unwrap(); + } + let before = (w.live_observed_ns, w.source_high.clone()); + let at = base + 1_001_000; + let error = w.observe(&packet(1, 2, at, RoomLabel::Empty, 0), at).unwrap_err(); + assert_eq!(error, "temporal_observed_gap_or_rate: source=1 direction=below_min receiver_interval_ns=1000000 min_ns=27777777 max_ns=50000000"); + assert_eq!((w.live_observed_ns, w.source_high.clone()), before); + assert_eq!(w.accepted(), 0); + assert!(w.session.rows.is_empty()); +} + #[test] fn initial_source_skew_uses_native_limit_even_within_first_arrival_deadline() { let (_dir, store, mut w) = setup(); diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_admission.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_admission.rs index 8d5e1c2fd0..a496bcb5fc 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_admission.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_fit_admission.rs @@ -20,6 +20,7 @@ pub(crate) struct TemporalFitCaptureIdentity { prefix_schema: &'static str, metadata_kind: &'static str, continuation_policy: Option, + operator_profile: Option, } impl TemporalFitCaptureIdentity { pub(crate) fn owner(&self) -> &str { @@ -114,7 +115,7 @@ impl VerifiedTemporalFitSource { configured_fit_sha256:self.identity.configuration,training_data_sha256:self.train_data,input_evidence_sha256:self.evidence, identity_evidence_sha256:self.identities,train_seal_chain_sha256:self.seals,accepted_train_prefix_sha256:self.identity.prefix, accepted_train_commitment:self.identity.token.data(),saved_candidate:candidate,actual_summary:self.actual_summary, - freeze_prepare_receiver_ns:now,source_floors:floors, + freeze_prepare_receiver_ns:now,source_floors:floors,operator_profile:self.identity.operator_profile, }; let sha256=hash(&(full_run::FIT_RECORD_DOMAIN,&body))?; Ok(full_run::VerifiedFitRecordData{schema:full_run::FIT_RECORD_DOMAIN.into(),body,sha256}) @@ -136,24 +137,26 @@ fn complete_train( .as_ref() .ok_or("managed_session_required")?; let counts = schedule.counts(); + let prefit_episodes=if schedule.is_operator_eight_step(){schedule.episodes().len()}else{counts.train_episodes}; + let prefit_windows=if schedule.is_operator_eight_step(){counts.total_windows}else{counts.train_windows}; if now < w.last_now_ns || now / 1_000_000 >= w.descriptor.expires_at_ms { return Err("temporal_fit_original_deadline_or_clock".into()); } if bridge.disposition != bridge::Disposition::Complete - || bridge.seals.len() != counts.train_episodes + || bridge.seals.len() != prefit_episodes || !bridge.parts.is_empty() - || w.episode != counts.train_episodes + || w.episode != prefit_episodes || w.mapping.is_some() || w.pending_window.is_some() || w.session.active || w.session.failed || w.session.fitted - || w.session.rows.len() != counts.train_windows - || w.session.evidence.len() != counts.train_windows - || managed.windows.len() != counts.train_windows - || w.ledger.spans.len() != counts.train_windows - || w.durable_windows != counts.train_windows - || w.session.rows.iter().any(|row| row.role != DataRole::Train) + || w.session.rows.len() != prefit_windows + || w.session.evidence.len() != prefit_windows + || managed.windows.len() != prefit_windows + || w.ledger.spans.len() != prefit_windows + || w.durable_windows != prefit_windows + || w.session.rows.iter().any(|row| row.role != DataRole::Train && !(schedule.is_operator_eight_step() && row.role == DataRole::Validation)) || w.pending_intent.is_some() || w.pending_prefix.is_some() || w.stage == Stage::BlockedDurability @@ -182,6 +185,7 @@ fn identity( prefix_schema:if w.full_run.is_some(){full_run::PREFIX_SCHEMA}else{"ruview.room-learner.temporal-runtime-prefix.v2"}, metadata_kind:if w.full_run.is_some(){full_run::META_KIND}else{"temporal-prefix-commitment-v1"}, continuation_policy:w.full_run.as_ref().map(|m|m.policy.sha256().into()), + operator_profile:schedule.is_operator_eight_step().then(||"native-2s-three-source-56bin-operator-eight-step-v1".into()), }) } pub(super) fn current_identity( @@ -200,7 +204,7 @@ pub(super) fn current_identity( if state.pending.is_some() || state.busy_pause || state.error.is_some() - || state.accepted_seals != schedule.counts().train_episodes + || state.accepted_seals != if schedule.is_operator_eight_step(){schedule.episodes().len()}else{schedule.counts().train_episodes} || w.durable.is_none() { return Err("temporal_fit_durability_unconfirmed".into()); @@ -273,7 +277,7 @@ impl PendingTemporalFitInput { } Ok(ValidatedTemporalFitInput { identity: self.identity, - train_seals_sha256:hash(&restored.bridge.as_ref().ok_or("temporal_fit_actual_bridge_required")?.seals)?, + train_seals_sha256:hash(&restored.bridge.as_ref().ok_or("temporal_fit_actual_bridge_required")?.seals.iter().filter(|seal|schedule.episodes().get(seal.episode).is_some_and(|e|e.role==DataRole::Train)).collect::>())?, session: restored.session, }) } @@ -287,7 +291,8 @@ impl ValidatedTemporalFitInput { ) -> Result<(TemporalTrainingSession, ManagedFitAdmission), String> { let(session,permit,_)=self.into_session_admission_and_source()?;Ok((session,permit)) } - pub(crate) fn into_session_admission_and_source(self)->Result<(TemporalTrainingSession,ManagedFitAdmission,VerifiedTemporalFitSource),String>{ + pub(crate) fn into_session_admission_and_source(mut self)->Result<(TemporalTrainingSession,ManagedFitAdmission,VerifiedTemporalFitSource),String>{ + self.session.retain_managed_train_only()?; let managed = self .session .managed diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run.rs index 250e5ff2e9..134d2c5ee3 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run.rs @@ -62,8 +62,8 @@ impl ContinuationPolicy { c.train_windows, c.held_out_windows, c.total_windows, - ) != (7, 8, 64, 64, 128) - || schedule.resources().max_numeric_values != 1_655_808 + ) != if schedule.is_operator_eight_step(){(4,4,64,16,80)}else{(7,8,64,64,128)} + || schedule.resources().max_numeric_values != if schedule.is_operator_eight_step(){1_034_880}else{1_655_808} || schedule.resources().max_numeric_values > 2_000_000 { return Err("temporal_full_run_engineering_profile_unsupported".into()); @@ -86,7 +86,7 @@ impl ContinuationPolicy { .and_then(|n| n.checked_add(episodes)) .and_then(|n| n.checked_add(3)) .ok_or("temporal_full_run_capacity")?; - if transition_limit != 433 || transition_limit > 512 { + if transition_limit != if schedule.is_operator_eight_step(){364}else{433} || transition_limit > 512 { return Err("temporal_full_run_transition_capacity".into()); } let body = PolicyBody { @@ -203,7 +203,7 @@ impl FullRunState { .ok_or("temporal_full_run_owner_custody_required")?; if self.frozen.is_some() && self.pending.is_none() - && (self.held_out.is_none() || self.verified.is_none() || self.completed.is_some()) + && ((!self.frozen.as_ref().is_some_and(|f|f.body.operator_profile.is_some()) && self.held_out.is_none()) || self.verified.is_none() || self.completed.is_some()) { return Err("temporal_frozen_write_requires_current_producer".into()); } @@ -304,6 +304,8 @@ pub(super) struct VerifiedFitRecordBody { pub actual_summary: FitSummaryReference, pub freeze_prepare_receiver_ns: u64, pub source_floors: Vec<(u8, u64, u64)>, + #[serde(default,skip_serializing_if="Option::is_none")] + pub operator_profile: Option, } /// Serializable DATA, not a permit. SHA binds data, never producer authority. #[derive(Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -333,6 +335,7 @@ impl VerifiedFitRecordData { || b.owner_scope != c.owner_scope || b.installation_id != c.installation_id || b.context_digest != c.digest() + || b.operator_profile.as_deref() != binding.contract()?.is_operator_eight_step().then_some("native-2s-three-source-56bin-operator-eight-step-v1") || b.run.is_empty() || b.run.len() > 256 || !b.run.is_ascii() @@ -400,7 +403,8 @@ fn verify_model_binding( ) -> Result<(), String> { let b = model.fit_diagnostic_binding()?; let r = &record.body; - if b.artifact_kind != MODEL_KIND + if record.body.operator_profile.is_some() != model.managed_is_operator_eight_step()? + || b.artifact_kind != MODEL_KIND || b.artifact_sha256 != r.artifact_sha256 || b.context_digest != r.context_digest || b.input_policy_sha256 != r.input_policy_sha256 @@ -451,15 +455,16 @@ impl VerifiedFrozenSession { } } fn check_current_train_prefix(&self, session: &TemporalTrainingSession) -> Result<(), String> { + let count=if self.record.body.operator_profile.is_some(){80}else{64}; if session.failed || !session.fitted - || session.rows.len() < 64 - || session.evidence.len() < 64 - || session.rows[..64] + || session.rows.len() < count + || session.evidence.len() < count + || session.rows[..count] .iter() .zip(&self.rows) .any(|(a, b)| !a.same_record(b)) - || session.evidence[..64] + || session.evidence[..count] .iter() .zip(&self.evidence) .any(|(a, b)| !a.same_record(b)) @@ -475,11 +480,12 @@ impl VerifiedFrozenSession { Ok(()) } fn check(&self, session: &TemporalTrainingSession) -> Result<(), String> { + let count=if self.record.body.operator_profile.is_some(){80}else{64}; if session.active || session.failed || session.fitted - || session.rows.len() != 64 - || session.evidence.len() != 64 + || session.rows.len() != count + || session.evidence.len() != count || session .rows .iter() @@ -816,8 +822,8 @@ impl VerifiedContinuedSession { store: self.store.clone(), model: self.model.clone(), record: self.record.clone(), - rows: self.rows[..64].to_vec(), - evidence: self.evidence[..64].to_vec(), + rows: self.rows[..if self.record.body.operator_profile.is_some(){80}else{64}].to_vec(), + evidence: self.evidence[..if self.record.body.operator_profile.is_some(){80}else{64}].to_vec(), }); Ok(()) } @@ -985,15 +991,18 @@ impl TemporalWorkflow { let Some(mode) = &self.full_run else { return Ok(false); }; - if mode.held_out_seals() != 8 || mode.complete.is_some() { + let operator=self.schedule()?.is_operator_eight_step(); + let operator_ready=operator && mode.verified.is_some() + && self.bridge.as_ref().is_some_and(|b|b.seals.len()==8); + if (!operator_ready && mode.held_out_seals() != 8) || mode.complete.is_some() { return Ok(false); } let state = self.async_state.as_ref().ok_or("temporal_async_required")?; if state.pending.is_some() - || state.accepted_held_out_seals != 8 - || self.durable_windows != 128 - || self.session.rows.len() != 128 - || self.episode != 15 + || if operator {state.accepted_seals != 8}else{state.accepted_held_out_seals != 8} + || self.durable_windows != if operator{80}else{128} + || self.session.rows.len() != if operator{80}else{128} + || self.episode != if operator{8}else{15} || self.mapping.is_some() { return Ok(false); @@ -1019,22 +1028,18 @@ impl TemporalWorkflow { verified_fit_sha256: fit.sha256.clone(), artifact_kind: fit.body.artifact_kind.clone(), artifact_sha256: fit.body.artifact_sha256.clone(), - held_out_chain_sha256: mode - .held_out - .as_ref() - .unwrap() - .seals - .last() - .unwrap() - .sha256 - .clone(), + held_out_chain_sha256: if operator { + let schedule=self.schedule()?; + hash(&self.bridge.as_ref().ok_or("operator_checks_seals_missing")?.seals.iter() + .filter(|seal|schedule.episodes()[seal.episode].role==DataRole::Validation).collect::>())? + } else {mode.held_out.as_ref().unwrap().seals.last().unwrap().sha256.clone()}, raw_sha256: hashes.raw.clone(), evidence_sha256: hashes.evidence.clone(), identity_sha256: hashes.identities.clone(), ledger_sha256: hashes.ledger.clone(), - windows: 128, - held_out_windows: 64, - held_out_seals: 8, + windows: if operator{80}else{128}, + held_out_windows: if operator{16}else{64}, + held_out_seals: if operator{4}else{8}, accepted_held_out_commitment: predecessor.data(), complete_prepare_receiver_ns: now, source_floors: self.source_high.clone(), diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run_checkpoint.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run_checkpoint.rs index aa1928878f..a712fbe2e4 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run_checkpoint.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_full_run_checkpoint.rs @@ -27,6 +27,8 @@ enum FullStage { Train { bridge: BridgeState, }, + OperatorFrozen { fit: VerifiedFitRecordData, seals: Vec }, + OperatorComplete { fit: VerifiedFitRecordData, seals: Vec, complete: held_out::FullCompleteRecordData }, Frozen { fit: VerifiedFitRecordData, train_seals: Vec, @@ -74,7 +76,8 @@ impl FullSnapshot { .binding, )?; let permit = mode.snapshot_permit(w.commit_token.clone())?; - let old = if mode.held_out.is_some() { + let operator=w.schedule()?.is_operator_eight_step(); + let old = if mode.held_out.is_some() || operator && mode.frozen.is_some() && mode.pending_record().is_none() { mode.check_current_train_prefix(&w.session)?; if w.session.failed || w.session.active || w.session.rows.len() != w.ledger.spans.len() { @@ -107,7 +110,12 @@ impl FullSnapshot { } else { super::snapshot_old(w)? }; - let stage = if let Some(held) = &mode.held_out { + let stage = if operator && mode.frozen.is_some() { + let fit=mode.frozen.as_ref().unwrap().clone(); + let seals=old.bridge.as_ref().ok_or("operator_bridge_required")?.seals.clone(); + if let Some(complete)=&mode.complete {FullStage::OperatorComplete{fit,seals,complete:complete.clone()}} + else {FullStage::OperatorFrozen{fit,seals}} + } else if let Some(held) = &mode.held_out { let fit = mode .frozen .as_ref() @@ -239,7 +247,8 @@ fn old_projection( // Full-v3 genesis and model identity retain their ORIGINAL Train // seal profile. Recognizing fresh343 for Kind1 must not widen this // separately versioned Kind2 continuation contract. - if bridge.policy != bridge::BRIDGE_POLICY { + let operator=common.binding.contract()?.is_operator_eight_step(); + if bridge.policy != if operator{bridge::OPERATOR_EIGHT_POLICY}else{bridge::BRIDGE_POLICY} { return Err("temporal_full_run_train_profile_changed".into()); } (bridge, None) @@ -301,6 +310,9 @@ pub(super) fn decode_structural( now: u64, ) -> Result { let e = parse(bytes)?; + if matches!(e.payload.stage,FullStage::OperatorFrozen{..}|FullStage::OperatorComplete{..}) { + return decode_operator(e.payload,descriptor,now); + } if matches!( e.payload.stage, FullStage::HeldOut { .. } | FullStage::Complete { .. } @@ -441,7 +453,8 @@ pub(super) fn validate_full_extension( )?; super::validate_extension(&av, &bv) } - (FullStage::Train { bridge }, FullStage::Frozen { fit, train_seals }) => { + (FullStage::Train { bridge }, FullStage::Frozen { fit, train_seals }) + | (FullStage::Train { bridge }, FullStage::OperatorFrozen { fit, seals:train_seals }) => { if bridge.disposition != bridge::Disposition::Complete || bridge.seals != *train_seals || fit.body.accepted_train_prefix_sha256 != bytes_hash(old.bytes()) @@ -473,10 +486,60 @@ pub(super) fn validate_full_extension( // commitment preparation; the raw structure grants no write. Ok(()) } + (FullStage::OperatorFrozen{fit:a_fit,seals:a_seals},FullStage::OperatorComplete{fit:b_fit,seals:b_seals,complete}) => { + if a_fit != b_fit || a_seals != b_seals + || serde_json::to_vec(&a.payload.accepted).map_err(|_|"temporal_encoding")? != serde_json::to_vec(&b.payload.accepted).map_err(|_|"temporal_encoding")? + || a.payload.common.binding.schedule_sha256 != b.payload.common.binding.schedule_sha256 + || a.payload.common.run != b.payload.common.run || a.payload.common.ledger != b.payload.common.ledger + || a.payload.common.mapping.is_some() || b.payload.common.mapping.is_some() + || a.payload.common.episode!=8 || b.payload.common.episode!=8 + || complete.body.accepted_held_out_commitment.run!=a.payload.common.run + || complete.body.complete_prepare_receiver_ns!=b.payload.common.last_now_ns { + return Err("operator_complete_transition_changed".into()); + } + for(id,seq,time)in &a.payload.common.source_high{if !b.payload.common.source_high.iter().any(|(s,q,t)|s==id&&q>=seq&&t>=time){return Err("temporal_commit_floor_regressed".into());}} + Ok(()) + } _ => Err("temporal_full_run_transition_unsupported".into()), } } +fn decode_operator(data:FullData,descriptor:&RecoveryDescriptor,now:u64)->Result{ + let (fit,seals,complete)=match data.stage{ + FullStage::OperatorFrozen{fit,seals}=>(fit,seals,None), + FullStage::OperatorComplete{fit,seals,complete}=>(fit,seals,Some(complete)), + _=>return Err("operator_frozen_stage_required".into()), + }; + let c=data.common;let a=data.accepted; + let schedule=c.binding.contract()?; + if !schedule.is_operator_eight_step() || c.episode!=8 || c.mapping.is_some() || a.rows.len()!=80 + || a.evidence.len()!=80 || a.identities.len()!=80 || seals.len()!=8 || c.run!=fit.body.run + || c.last_now_ns=seq&&t>=time)) { + return Err("operator_frozen_prefix_changed".into()); + } + fit.validate_data(&c.binding,&data.policy)?; + let prefix=Prefix{schema:BRIDGE_SCHEMA.into(),binding:c.binding,run:c.run,descriptor:c.descriptor,last_now_ns:c.last_now_ns, + rows:a.rows,evidence:a.evidence,identities:a.identities,fences:a.fences,source_high:c.source_high,ledger:c.ledger, + mapping:c.mapping,episode:c.episode,frozen_model:None,quality_issued:false,confirmation_issued:false, + bridge:Some(BridgeState{policy:bridge::OPERATOR_EIGHT_POLICY.into(),disposition:bridge::Disposition::Complete,seals,parts:vec![]})}; + let budget=data.policy.replay_fence_budget(&prefix.binding)?; + let mut w=super::decode_full_train_projection(&old_bytes(prefix)?,descriptor,now,&budget)?; + validate_immutable_train(&w,&fit)?; + if let Some(record)=&complete{ + record.validate_binding_data(&fit,descriptor,now)?; + let hashes=full_run::FullPrefixHashes::from_workflow(&w)?; + let checkseals=w.bridge.as_ref().unwrap().seals.iter().filter(|seal|schedule.episodes()[seal.episode].role==DataRole::Validation).collect::>(); + if record.body.raw_sha256!=hashes.raw || record.body.evidence_sha256!=hashes.evidence + || record.body.identity_sha256!=hashes.identities || record.body.ledger_sha256!=hashes.ledger + || record.body.held_out_chain_sha256!=hash(&checkseals)? { + return Err("operator_complete_actual_prefix_changed".into()); + } + } + let mut mode=FullRunState::structural(data.policy,Some(fit));mode.complete=complete;w.full_run=Some(mode); + Ok(w) +} + fn decode_continued( data: FullData, descriptor: &RecoveryDescriptor, @@ -730,21 +793,21 @@ pub(super) fn validate_immutable_train( .as_ref() .ok_or("temporal_full_run_bridge_required")?; let body = &record.body; - if w.session.rows.len() < 64 - || w.session.evidence.len() < 64 - || managed.windows.len() < 64 - || w.session.rows[..64] - .iter() - .any(|r| r.role != DataRole::Train) + let schedule=w.schedule()?; + let operator=schedule.is_operator_eight_step(); + let rows=w.session.rows.iter().filter(|r|r.role==DataRole::Train).collect::>(); + let evidence=w.session.evidence.iter().filter(|r|r.role==DataRole::Train).collect::>(); + let identities=managed.windows.iter().filter(|r|r.role==DataRole::Train).collect::>(); + let seals=bridge.seals.iter().filter(|seal|schedule.episodes()[seal.episode].role==DataRole::Train).collect::>(); + if rows.len()!=64 || evidence.len()!=64 || identities.len()!=64 || bridge.disposition != bridge::Disposition::Complete - || bridge.seals.len() != 7 - || body.training_data_sha256 != hash(&(&w.session.rows[..64]))? - || body.input_evidence_sha256 != hash(&(&w.session.evidence[..64]))? - || body.identity_evidence_sha256 != hash(&(&managed.windows[..64]))? - || body.train_seal_chain_sha256 != hash(&bridge.seals)? + || bridge.seals.len()!=if operator{8}else{7} + || body.training_data_sha256 != hash(&rows)? + || body.input_evidence_sha256 != hash(&evidence)? + || body.identity_evidence_sha256 != hash(&identities)? + || body.train_seal_chain_sha256 != hash(&seals)? || body.accepted_train_commitment.run != w.session.run_id - || body.accepted_train_commitment.id != w.descriptor.id - { + || body.accepted_train_commitment.id != w.descriptor.id { return Err("temporal_frozen_train_binding".into()); } Ok(()) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_held_out_capture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_held_out_capture.rs index dca223b514..ee3c126a99 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_held_out_capture.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_held_out_capture.rs @@ -13,6 +13,7 @@ pub(super) const HELD_OUT_SEAL_DOMAIN: &str = "ruview.room-learner.temporal-held pub(super) const HELD_OUT_CONDITIONS_DOMAIN: &str = "ruview.room-learner.temporal-heldout-conditions.v1"; pub(super) const FULL_COMPLETE_DOMAIN: &str = "ruview.room-learner.temporal-full-complete.v1"; +pub(super) const OPERATOR_COMPLETE_DOMAIN: &str = "ruview.room-learner.temporal-full-complete.operator-eight-step.v1"; const WHOLE_MODEL_KIND: &str = "ruview.room-learner.managed-temporal-input.v3"; const TRAIN_EPISODES: usize = 7; const TRAIN_WINDOWS: usize = 64; @@ -269,9 +270,10 @@ impl FullCompleteRecordData { /// Computes a DATA digest. Only actual last-SEAL ACK/current-store recovery /// may later supply the separate non-serde Complete write/install witness. pub(super) fn from_body(body: FullCompleteBody) -> Result { - let sha256 = hash(&(FULL_COMPLETE_DOMAIN, &body))?; + let domain=if (body.windows,body.held_out_windows,body.held_out_seals)==(80,16,4){OPERATOR_COMPLETE_DOMAIN}else{FULL_COMPLETE_DOMAIN}; + let sha256 = hash(&(domain, &body))?; Ok(Self { - schema: FULL_COMPLETE_DOMAIN.into(), + schema: domain.into(), body, sha256, }) @@ -283,6 +285,9 @@ impl FullCompleteRecordData { now: u64, ) -> Result<(), String> { let body = &self.body; + let operator=fit.body.operator_profile.as_deref()==Some("native-2s-three-source-56bin-operator-eight-step-v1"); + if fit.body.operator_profile.is_some() && !operator{return Err("temporal_full_complete_profile_changed".into());} + let domain=if operator{OPERATOR_COMPLETE_DOMAIN}else{FULL_COMPLETE_DOMAIN}; let hashes = [ &body.continuation_policy_sha256, &body.verified_fit_sha256, @@ -293,7 +298,7 @@ impl FullCompleteRecordData { &body.identity_sha256, &body.ledger_sha256, ]; - if self.schema != FULL_COMPLETE_DOMAIN + if self.schema != domain || hashes.iter().any(|value| !digest64(value)) || body.context_sha256 != descriptor.context_digest || body.context_sha256 != fit.body.context_digest @@ -307,19 +312,19 @@ impl FullCompleteRecordData { || body.artifact_kind != WHOLE_MODEL_KIND || body.artifact_kind != fit.body.artifact_kind || body.artifact_sha256 != fit.body.artifact_sha256 - || body.windows != FULL_WINDOWS - || body.held_out_windows != TRAIN_WINDOWS - || body.held_out_seals != HELD_OUT_EPISODES + || body.windows != if operator{80}else{FULL_WINDOWS} + || body.held_out_windows != if operator{16}else{TRAIN_WINDOWS} + || body.held_out_seals != if operator{4}else{HELD_OUT_EPISODES} || body.accepted_held_out_commitment.id != descriptor.id || body.accepted_held_out_commitment.run != body.run || !digest64(&body.accepted_held_out_commitment.commitment) - || body.accepted_held_out_commitment.revision > 430 + || body.accepted_held_out_commitment.revision > if operator{361}else{430} || body.complete_prepare_receiver_ns < fit.body.freeze_prepare_receiver_ns || body.complete_prepare_receiver_ns > now || now / 1_000_000 >= descriptor.expires_at_ms || body.source_floors.len() != fit.body.source_floors.len() || !digest64(&self.sha256) - || self.sha256 != hash(&(FULL_COMPLETE_DOMAIN, body))? + || self.sha256 != hash(&(domain, body))? { return Err("temporal_full_complete_data_binding".into()); } diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs index b4b35cf7a2..26635b07b4 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed.rs @@ -190,9 +190,11 @@ fn identity( fn expected_position( schedule: &ManagedScheduleContract, index: usize, + train_only: bool, ) -> Result<(usize, usize), String> { let mut cursor = 0; for episode in schedule.episodes() { + if train_only && episode.role != DataRole::Train { continue; } if index < cursor + episode.slots.len() { return Ok((episode.index, index - cursor)); } @@ -215,7 +217,7 @@ fn validate_records( let mut previous: Option<&ManagedWindowIdentity> = None; for (index, (e, r)) in evidence.iter().zip(records).enumerate() { let e = e.borrow(); - let position = expected_position(schedule, index)?; + let position = expected_position(schedule, index, train_only)?; if (r.episode_index, r.slot_index) != position || (train_only && (r.role != DataRole::Train || r.frozen_artifact_sha256.is_some())) { @@ -335,6 +337,32 @@ impl TemporalTrainingSession { self.target_ns = Some(target.target_ns()); Ok(()) } + /// Called only on the recertified fitting copy. Validation rows remain in + /// the durable capture and never enter training, normalization or its codec. + pub(crate) fn retain_managed_train_only(&mut self) -> Result<(), String> { + let managed = self.managed.as_mut().ok_or("managed_session_required")?; + let schedule = managed.binding.contract()?; + if !schedule.is_operator_eight_step() { return Ok(()); } + if self.active || self.fitted || self.failed || self.rows.len() != schedule.counts().total_windows + || self.rows.len() != self.evidence.len() || self.rows.len() != managed.windows.len() { + return Err("operator_check_train_filter_requires_complete_capture".into()); + } + validate_records(&schedule, &self.run_id, &self.evidence, &managed.windows, &self.binding, false)?; + for ((row,evidence),identity) in self.rows.iter().zip(&self.evidence).zip(&managed.windows) { + if row.role!=identity.role || row.label!=identity.label || row.id!=evidence.cohort_id + || row.canonical_sha256()?!=&evidence.cohort_sha256 { + return Err("operator_check_training_partition_changed".into()); + } + } + let keep = managed.windows.iter().map(|w| w.role == DataRole::Train).collect::>(); + let mut index=0;self.rows.retain(|_|{let k=keep[index];index+=1;k}); + let mut index=0;self.evidence.retain(|_|{let k=keep[index];index+=1;k}); + managed.windows.retain(|w|w.role==DataRole::Train); + if self.rows.len()!=schedule.counts().train_windows || self.rows.iter().any(|r|r.role!=DataRole::Train) { + return Err("operator_check_training_partition_changed".into()); + } + Ok(()) + } pub(crate) fn fit_managed_with_cancel( &mut self, permit: ManagedFitAdmission, @@ -399,6 +427,9 @@ impl TemporalTrainingSession { } } impl TemporalModel { + pub(crate) fn managed_is_operator_eight_step(&self) -> Result { + Ok(self.validate_managed()?.is_operator_eight_step()) + } pub(super) fn validate_managed(&self) -> Result { let m = self.managed.as_ref().ok_or("managed_model_required")?; let schedule = m.binding.contract()?; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed_tests.rs index 00cdf163eb..d0576b8b16 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_model_input_managed_tests.rs @@ -483,3 +483,59 @@ fn mode_correct_policy_identity_tracks_managed_targets_and_preserves_seed_v2_ide assert_eq!(seed.plan_sha256(), old.sha256()); assert_eq!(seed.plan_sha256(), seed.binding.plan_sha256); } + + +fn collect_operator(validation_changed:bool)->TemporalTrainingSession { + let plan=ManagedScheduleContract::build(context(),LeadInPolicy::OperatorEightStepV1).unwrap(); + let mut session=TemporalTrainingSession::managed(plan.clone(),"synthetic-eight-run".into(),FitConfig{epochs:2,..Default::default()}).unwrap(); + let mut start=10_000_000_000u64;let mut index=0; + for episode in plan.episodes(){ + let bound=plan.bind_episode_start("synthetic-eight-run",episode.index,0,0,None).unwrap(); + for slot in 0..episode.slots.len(){ + let target=ManagedWindowTarget::from_runtime_episode(&plan,&bound,"synthetic-eight-run",slot,start).unwrap(); + if validation_changed&&episode.role==DataRole::Validation { + session.begin_managed_window(&target).unwrap(); + 'window: for row in 0..100usize {for source in 1..=3u8 { + let at=target.target_ns()+row as u64*1_000_000_000u64.div_ceil(33)+u64::from(source)*1000; + let iq=vec![120u8;128]; + let p=canonicalize_esp32(NativePacketInput{node_id:source,n_antennas:1,raw_bins:64,frequency_hz:2_437_000_000, + ppdu_type:PpduType::HtLegacy,sequence:index*1000+row as u32+1,received_at_ns:at,receiver_sample_rate_hz:33., + iq_bytes:&iq,flags:Adr018Flags::default(),provenance:CaptureProvenance::Synthetic}).unwrap(); + if session.ingest(&p).unwrap()==WindowAdmission::Ready{session.finish_window(at).unwrap();break 'window;} + }} + } else {collect(&mut session,&target,index,episode.condition);} + index+=1; + } + start+=episode.target_active_recording_ms*1_000_000+10_000_000_000; + } + assert_eq!(session.rows.len(),80); + assert_eq!(session.rows.iter().filter(|r|r.role==DataRole::Validation).count(),16); + session +} +#[test] +fn operator_reserved_checks_never_change_actual_fitter_normalization_or_model_bytes(){ + let mut first=collect_operator(false);let mut changed=collect_operator(true); + assert_ne!(hash(&first.rows).unwrap(),hash(&changed.rows).unwrap()); + first.retain_managed_train_only().unwrap();changed.retain_managed_train_only().unwrap(); + assert_eq!(first.rows.len(),64);assert_eq!(hash(&first.rows).unwrap(),hash(&changed.rows).unwrap()); + assert_eq!(first.rows.iter().map(|r|&r.trial_id).collect::>().len(),4); + let a=first.synthetic_test_fit_admission().unwrap();let b=changed.synthetic_test_fit_admission().unwrap(); + let(ma,ra)=first.fit_managed_with_cancel(a,&AtomicBool::new(false)).unwrap(); + let(mb,rb)=changed.fit_managed_with_cancel(b,&AtomicBool::new(false)).unwrap(); + assert_eq!(ma.export().unwrap(),mb.export().unwrap()); + assert_eq!(serde_json::to_value(ra).unwrap(),serde_json::to_value(rb).unwrap()); + assert!(ma.managed_is_operator_eight_step().unwrap()); + assert_eq!(TemporalModel::import(&ma.export().unwrap()).unwrap().export().unwrap(),ma.export().unwrap()); +} +#[test] +fn operator_training_partition_rejects_relabelled_validation_before_fitting(){ + let mut session=collect_operator(false); + let index=session.rows.iter().position(|r|r.role==DataRole::Validation).unwrap(); + let original=(*session.rows[index]).clone(); + for label_or_role in [false,true] { + let mut forged=original.clone();if label_or_role{forged.role=DataRole::Train}else{forged.label=RoomLabel::OccupiedMoving} + session.rows[index]=SharedRecord::new(forged); + assert!(session.retain_managed_train_only().is_err()); + assert!(!session.fitted); + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_qualification.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_qualification.rs index a4738ee7bf..7f65f965be 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_qualification.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_qualification.rs @@ -57,6 +57,9 @@ impl ComputedTemporalAssessment { let held = evaluation.held_out(); let eval = evaluation.receipt(); eval.validate_data_links(proof, held)?; + let operator = proof.is_operator_eight_step(); + let check_windows = if operator { 4 } else { 16 }; + let check_episodes = if operator { 1 } else { 2 }; let id = &proof.body.identity; if now / 1_000_000 >= id.original_deadline.expires_at_unix_ms || now @@ -69,23 +72,24 @@ impl ComputedTemporalAssessment { return Err("temporal_assessment_clock_or_original_deadline".into()); } if proof.body.windows_per_condition != [16; 4] - || held.body.windows_per_condition != [16; 4] - || eval.body.report.samples != 64 + || held.body.windows_per_condition != [check_windows; 4] + || eval.body.report.samples != usize::from(check_windows) * 4 || eval.body.report.per_condition.len() != 4 || eval .body .report .per_condition .iter() - .any(|c| c.samples != 16 || c.declared_trial_count != 2) + .any(|c| c.samples != usize::from(check_windows) || c.declared_trial_count != check_episodes) { return Err("temporal_assessment_exact_native_evidence_required".into()); } - let policy = TemporalQualificationPolicyWire::current(); + let policy = TemporalQualificationPolicyWire::for_operator_eight_step(operator); let policy_sha = h(&policy)?; - let criteria = native_criteria(&eval.body.report); + let criteria = native_criteria(&eval.body.report, operator); // Authoritative pass/failure is the established integer criteria, - // with exact16 input shapes enforced above. No metric is replaced. + // with exact versioned 16- or 4-window input shapes enforced above. + // The smaller operator validation set is not equal-confidence evidence. let outcome = super::qualification::criteria(&eval.body.report, &[16; 4]); let qualification = if outcome.is_ok() { let mut summary = TemporalQualificationSummaryWire { @@ -94,7 +98,7 @@ impl ComputedTemporalAssessment { model_kind: WholeTemporalKind::ManagedTemporalV3, model_sha256: id.artifact_sha256.clone(), verified_fit_record_sha256: id.verified_fit_record_sha256.clone(), - training_lineage_basis: "managed_temporal_training_proof_v1".into(), + training_lineage_basis: if operator { "managed_temporal_training_proof_operator_eight_step_v1" } else { "managed_temporal_training_proof_v1" }.into(), training_proof_sha256: proof.sha256.clone(), held_out_evidence_sha256: held.sha256.clone(), complete_record_sha256: held.body.complete_record_sha256.clone(), @@ -104,14 +108,14 @@ impl ComputedTemporalAssessment { evaluated_at_unix_ms: eval.body.evaluated_at_unix_ms, policy: policy.clone(), policy_sha256: policy_sha.clone(), - label_provenance: "temporal-training-proof-v1+sealed-native-heldout-v1".into(), + label_provenance: if operator { "temporal-training-proof-operator-eight-step-v1+reserved-native-validation-v1" } else { "temporal-training-proof-v1+sealed-native-heldout-v1" }.into(), criteria: criteria.clone(), authority_scope: "local_operator_binary_presence".into(), window_unit: "native_receiver_2s_window".into(), train_windows: 64, - held_out_windows: 64, - train_episodes: 7, - held_out_episodes: 8, + held_out_windows: if operator { 16 } else { 64 }, + train_episodes: if operator { 4 } else { 7 }, + held_out_episodes: if operator { 4 } else { 8 }, independence_verified: false, physically_validated: false, measurement_clock_available: false, @@ -229,9 +233,19 @@ impl TemporalQualificationEnvelope { || q.evaluation_receipt_sha256 != e.sha256 || q.context_digest != id.context_digest || q.room_digest != id.room_digest - || q.policy != TemporalQualificationPolicyWire::current() + || q.policy != TemporalQualificationPolicyWire::for_operator_eight_step(proof.is_operator_eight_step()) + || q.train_windows != 64 + || q.held_out_windows != if proof.is_operator_eight_step() { 16 } else { 64 } + || q.train_episodes != if proof.is_operator_eight_step() { 4 } else { 7 } + || q.held_out_episodes != if proof.is_operator_eight_step() { 4 } else { 8 } + || q.training_lineage_basis != if proof.is_operator_eight_step() { "managed_temporal_training_proof_operator_eight_step_v1" } else { "managed_temporal_training_proof_v1" } + || q.label_provenance != if proof.is_operator_eight_step() { "temporal-training-proof-operator-eight-step-v1+reserved-native-validation-v1" } else { "temporal-training-proof-v1+sealed-native-heldout-v1" } + || q.independence_verified || q.physically_validated || q.measurement_clock_available + || q.model_kind != WholeTemporalKind::ManagedTemporalV3 + || q.authority_scope != "local_operator_binary_presence" + || q.window_unit != "native_receiver_2s_window" || q.policy_sha256 != h(&q.policy)? - || q.criteria != native_criteria(&e.body.report) + || q.criteria != native_criteria(&e.body.report, proof.is_operator_eight_step()) || q.evaluated_at_unix_ms != e.body.evaluated_at_unix_ms || self.receipt.qualified_at_unix_ms < e.body.evaluated_at_unix_ms || self.receipt.qualified_at_unix_ms >= id.original_deadline.expires_at_unix_ms @@ -281,9 +295,13 @@ impl TemporalAssessmentEnvelope { || a.held_out_evidence_sha256 != held.sha256 || a.complete_record_sha256 != held.body.complete_record_sha256 || a.evaluation_receipt_sha256 != e.sha256 - || a.policy != TemporalQualificationPolicyWire::current() + || a.context_digest != id.context_digest + || a.room_digest != id.room_digest + || a.model_kind != WholeTemporalKind::ManagedTemporalV3 + || a.window_unit != "native_receiver_2s_window" + || a.policy != TemporalQualificationPolicyWire::for_operator_eight_step(proof.is_operator_eight_step()) || a.policy_sha256 != h(&a.policy)? - || a.criteria != native_criteria(&e.body.report) + || a.criteria != native_criteria(&e.body.report, proof.is_operator_eight_step()) || a.evaluated_at_unix_ms != e.body.evaluated_at_unix_ms || a.assessed_at_unix_ms < e.body.evaluated_at_unix_ms || a.assessed_at_unix_ms >= id.original_deadline.expires_at_unix_ms @@ -314,6 +332,7 @@ impl TemporalAssessmentEnvelope { fn native_criteria( report: &super::training::EvaluationReport, + operator: bool, ) -> Vec { report .per_condition @@ -331,12 +350,12 @@ fn native_criteria( scored_windows: c.scored as u16, abstained_windows: c.abstained as u16, missing_windows: c.missing as u16, - declared_train_episodes: if c.condition == super::observation::RoomLabel::Empty { + declared_train_episodes: if operator || c.condition == super::observation::RoomLabel::Empty { 1 } else { 2 }, - declared_held_out_episodes: 2, + declared_held_out_episodes: if operator { 1 } else { 2 }, evaluated_window_coverage: c.scored as f64 / c.samples as f64, unconditional_sensitivity: occupied .then_some(c.confusion.true_occupied as f64 / c.samples as f64), diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs index b9af358003..cf6d301c78 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_recovery_commitment.rs @@ -1376,3 +1376,10 @@ impl RecoveryStore { }) } } + +#[cfg(test)] +impl RecoveryStore { + pub(crate) fn synthetic_fail_before_head_publication(&self,intent:&TemporalIntent,prefix:&ValidatedTemporalPrefix,now:u64)->Result{ + self.publish_temporal_with(intent,Some(prefix),now,false,&mut|point|{if point=="before:head.stage:rename"{Err("synthetic_failed_head_publication".into())}else{Ok(())}}) + } +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_selection_store.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_selection_store.rs index 28f928f96b..173d7f6ff4 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_selection_store.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_selection_store.rs @@ -297,7 +297,8 @@ impl ArtifactStore { } let id = &proof.body.identity; - if proof.sha256 != t.training_proof_sha256 + if model.managed_is_operator_eight_step()? != proof.is_operator_eight_step() + || proof.sha256 != t.training_proof_sha256 || held.sha256 != t.held_out_evidence_sha256 || evaluation.sha256 != t.evaluation_receipt_sha256 || qualification.receipt_sha256 != t.qualification_sha256 @@ -377,7 +378,7 @@ impl ArtifactStore { assessment_lookup_sha256: crate::room_learner::qualification::hash(&( crate::room_learner::temporal_wire::ASSESSMENT_SCHEMA, &assessment.evaluation.receipt().sha256, - TemporalQualificationPolicyWire::current(), + TemporalQualificationPolicyWire::for_operator_eight_step(source.proof().is_operator_eight_step()), ))?, assessment_sha256: assessment.assessment.receipt_sha256.clone(), qualification_sha256: q.receipt_sha256.clone(), diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_wire.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_wire.rs index 555b4ac19b..799411bfba 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_wire.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_wire.rs @@ -155,6 +155,19 @@ pub struct TemporalConfigureRequestV2 { pub source_node_ids: Vec, pub new_training: bool, pub recovery_consent: bool, + #[serde(default)] + pub profile_id: Option, +} + +pub const RECOVERY_RETRY_REQUEST_SCHEMA: &str = "ruview.room-learner.recovery-retry.v2"; +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TemporalRetryRequestV2 { + pub schema: String, + pub model_kind: WholeTemporalKind, + pub expected: MutationGuard, + pub capture_generation_id: String, + pub retry_confirmed: bool, } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields)] @@ -316,10 +329,21 @@ pub struct TemporalRecoveryRequestV2 { pub revision: u64, pub source_node_ids: Vec, pub capture_generation_id: String, + #[serde(default)] + pub profile_id: Option, } pub const TEMPORAL_POLICY_ID: &str = "ruview.local-operational-temporal-policy.v1"; +pub const OPERATOR_EIGHT_STEP_TEMPORAL_POLICY_ID: &str = "ruview.local-operational-temporal-policy.operator-eight-step.v1"; impl TemporalQualificationPolicyWire { + pub(crate) fn for_operator_eight_step(operator: bool) -> Self { + let mut policy = Self::current(); + if operator { + policy.id = OPERATOR_EIGHT_STEP_TEMPORAL_POLICY_ID.into(); + policy.minimum_windows_per_condition = 4; + } + policy + } pub(crate) fn current() -> Self { let numeric = super::qualification::QualificationPolicy::default(); Self { @@ -358,6 +382,14 @@ pub struct TemporalDiscardRequestV2 { } impl TemporalProfileWire { + pub(crate) fn operator_eight_step() -> Self { + let labels=[RoomLabel::Empty,RoomLabel::OccupiedStatic,RoomLabel::OccupiedMoving,RoomLabel::AdjacentOnly]; + Self {profile_id:super::managed_schedule_contract::OPERATOR_EIGHT_PROFILE.into(),window_unit:"native_receiver_2s_window".into(),native_window_ms:2000, + train_windows:64,held_out_windows:16,train_episodes:4,held_out_episodes:4,windows_per_condition:[16;4],train_episodes_per_condition:[1;4],held_out_episodes_per_condition:[1;4], + episodes:labels.into_iter().flat_map(|label| [DataRole::Train,DataRole::Validation].into_iter().map(move |role|(label,role))).enumerate().map(|(index,(label,role))|TemporalEpisodeWire{ + index:index as u8,role,label,target_ms:if role==DataRole::Validation{15_000}else if label==RoomLabel::Empty{600_000}else{60_000},lead_in_ms:if matches!(label,RoomLabel::Empty|RoomLabel::AdjacentOnly){15_000}else{5_000},native_windows:if role==DataRole::Train{16}else{4} + }).collect()} + } pub(crate) fn engineering() -> Self { let labels = [ RoomLabel::Empty, diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_wire_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_wire_tests.rs index a6a3afc691..a9643b53b6 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_wire_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_wire_tests.rs @@ -298,9 +298,25 @@ fn export_canonical_structural_wire_examples_without_runtime_authority() { blocker: Some("context_mismatch".into()), }; println!("C4C_CANONICAL_PROGRESS_EXAMPLES={}",serde_json::to_string(&serde_json::json!({"fixture_kind":"SYNTHETIC_STRUCTURAL_SERIALIZATION_ONLY_NOT_AUTHORITY","matching_complete":matching,"context_mismatch_redacted":mismatch,"selected_temporal_idle":{"schema":SELECTION_SCHEMA,"success":true,"progress_schema":PAIRED_PROGRESS_SCHEMA,"model_selection":selection}})).unwrap()); - let examples = serde_json::json!({"fixture_kind":"SYNTHETIC_STRUCTURAL_SERIALIZATION_ONLY_NOT_AUTHORITY","idle_response":{"success":true,"status":idle},"candidate_identity":TemporalCandidateIdentityWire{model_kind:WholeTemporalKind::ManagedTemporalV3,model_sha256:h('a'),verified_fit_record_sha256:h('b'),context_digest:h('4'),room_digest:h('5'),persisted:false},"selected_identity":selected,"qualification_summary":qualification,"configure_request":TemporalConfigureRequestV2{schema:CONFIGURE_REQUEST_SCHEMA.into(),model_kind:WholeTemporalKind::ManagedTemporalV3,expected:guard(),binding_digest:h('4'),revision:1,source_node_ids:vec![15,16,17],new_training:true,recovery_consent:true},"evaluate_request":TemporalEvaluateRequestV2{schema:EVALUATE_REQUEST_SCHEMA.into(),expected:guard(),capture_generation_id:h('0'),refs:complete()},"qualify_request":TemporalQualifyRequestV2{schema:QUALIFY_REQUEST_SCHEMA.into(),expected:guard(),capture_generation_id:h('0'),refs:eval()}}); + let examples = serde_json::json!({"fixture_kind":"SYNTHETIC_STRUCTURAL_SERIALIZATION_ONLY_NOT_AUTHORITY","idle_response":{"success":true,"status":idle},"candidate_identity":TemporalCandidateIdentityWire{model_kind:WholeTemporalKind::ManagedTemporalV3,model_sha256:h('a'),verified_fit_record_sha256:h('b'),context_digest:h('4'),room_digest:h('5'),persisted:false},"selected_identity":selected,"qualification_summary":qualification,"configure_request":TemporalConfigureRequestV2{schema:CONFIGURE_REQUEST_SCHEMA.into(),model_kind:WholeTemporalKind::ManagedTemporalV3,expected:guard(),binding_digest:h('4'),revision:1,source_node_ids:vec![15,16,17],new_training:true,recovery_consent:true,profile_id:None},"evaluate_request":TemporalEvaluateRequestV2{schema:EVALUATE_REQUEST_SCHEMA.into(),expected:guard(),capture_generation_id:h('0'),refs:complete()},"qualify_request":TemporalQualifyRequestV2{schema:QUALIFY_REQUEST_SCHEMA.into(),expected:guard(),capture_generation_id:h('0'),refs:eval()}}); println!( "C4C_CANONICAL_PUBLIC_WIRE_EXAMPLES={}", serde_json::to_string(&examples).unwrap() ); } + + +#[test] +fn operator_eight_step_policy_is_explicit_and_preserves_numeric_quality_thresholds() { + let old = TemporalQualificationPolicyWire::current(); + assert_eq!(TemporalQualificationPolicyWire::for_operator_eight_step(false), old); + let new = TemporalQualificationPolicyWire::for_operator_eight_step(true); + assert_eq!(new.id, OPERATOR_EIGHT_STEP_TEMPORAL_POLICY_ID); + assert_eq!(new.minimum_windows_per_condition, 4); + assert_eq!(old.minimum_windows_per_condition, 16); + assert_eq!(new.window_unit, old.window_unit); + assert_eq!(new.minimum_unconditional_sensitivity, old.minimum_unconditional_sensitivity); + assert_eq!(new.maximum_false_positive_rate, old.maximum_false_positive_rate); + assert_eq!(new.minimum_evaluated_window_coverage, old.minimum_evaluated_window_coverage); + assert_ne!(super::super::qualification::hash(&new).unwrap(), super::super::qualification::hash(&old).unwrap()); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs index 7db14a66c5..f3c9f78135 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs @@ -481,4 +481,4 @@ pub(super) use paired_progress::{progress, progress_dispatch}; #[path = "room_learner_api_temporal.rs"] mod temporal_routes; -pub(super) use temporal_routes::{status_dispatch, selection_dispatch, configure_dispatch, begin_dispatch, finish_dispatch, train_dispatch, evaluate_dispatch, restore_dispatch, discard_dispatch, pause_dispatch, cancel_operation_dispatch}; +pub(super) use temporal_routes::{status_dispatch, selection_dispatch, configure_dispatch, begin_dispatch, finish_dispatch, train_dispatch, evaluate_dispatch, restore_dispatch, retry_dispatch, discard_dispatch, pause_dispatch, cancel_operation_dispatch}; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs index 5f0521f43d..f42e6731a0 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_selection.rs @@ -8,6 +8,7 @@ pub(crate) fn owner_selection_routes( ) -> Router { Router::new() .route("/api/v1/room-learner/recovery/enable-current", post(enable_current_recovery)) + .route("/api/v1/room-learner/recovery/retry", post(super::temporal_routes::retry_dispatch).layer(axum::middleware::from_fn(retry_requires_unpaired_owner))) .route("/api/v1/room-learner/qualify", post(super::temporal_routes::qualify_dispatch)) .route("/api/v1/room-learner/select", post(super::temporal_routes::select_dispatch)) .route("/api/v1/room-learner/deselect", post(super::temporal_routes::deselect_dispatch)) @@ -148,3 +149,8 @@ pub(crate) async fn selection(State(state): State) -> Jsonaxum::response::Response{ + if request.extensions().get::().is_some(){return axum::response::IntoResponse::into_response(StatusCode::FORBIDDEN);} + next.run(request).await +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal.rs index f87fa0018d..3939d022ba 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal.rs @@ -80,6 +80,7 @@ pub(crate) async fn status_dispatch( Ok(rt) => rt, Err(e) => return e, }; + if let Err(e)=rt.temporal_default_operator_profile_v2(){return failure("learner_profile_unavailable",e);} temporal_status(&state, &rt).await } fn temporal_selection( @@ -175,7 +176,7 @@ pub(crate) async fn configure_dispatch( State(state): State, Json(command): Json, ) -> Json { - let ConfigureCommand::Temporal(request) = command else { + let ConfigureCommand::Temporal(mut request) = command else { let ConfigureCommand::Legacy(request) = command else { unreachable!() }; @@ -201,6 +202,7 @@ pub(crate) async fn configure_dispatch( Ok(rt) => rt, Err(e) => return e, }; + request.expected=match rt.temporal_select_profile_guarded_v2(request.profile_id.as_deref(),&request.expected){Ok(guard)=>guard,Err(e)=>return failure("learner_profile_rejected",e)}; let s = state.read().await; if field_capture_active(&s) { return failure( @@ -474,7 +476,7 @@ pub(crate) async fn restore_dispatch( ) -> Json { match command { RestoreCommand::Legacy(request) => restore_capture(State(state), Json(request)).await, - RestoreCommand::Temporal(request) => { + RestoreCommand::Temporal(mut request) => { if let Err(e) = schema_matches(&request.schema, RECOVERY_RESTORE_REQUEST_SCHEMA) { return e; } @@ -482,6 +484,7 @@ pub(crate) async fn restore_dispatch( Ok(rt) => rt, Err(e) => return e, }; + request.expected=match rt.temporal_select_profile_guarded_v2(request.profile_id.as_deref(),&request.expected){Ok(guard)=>guard,Err(e)=>return failure("learner_profile_rejected",e)}; let now = now_ns(&rt); let context = match rt.temporal_recovery_context_v2( &request.binding_digest, @@ -618,3 +621,15 @@ pub(crate) async fn deselect_dispatch( } } } + + +pub(crate) async fn retry_dispatch(State(state):State,Json(request):Json)->Json{ + if let Err(e)=schema_matches(&request.schema,RECOVERY_RETRY_REQUEST_SCHEMA){return e;} + let rt=match runtime(&state).await{Ok(rt)=>rt,Err(e)=>return e}; + {let s=state.read().await;if field_capture_active(&s){return failure("learner_capture_conflict","Finish active field calibration first.");}} + // Retry is metadata recovery, not capture. Its Runtime CAS still fences the + // owner/context/profile; never hold the UDP producer's SharedState lock + // across writer drain, filesystem barriers or cryptographic readback. + let result=rt.temporal_retry_v2(request,now_ns(&rt)); + match result{Ok(())=>temporal_status(&state,&rt).await,Err(e)=>failure("learner_recovery_retry_failed",e)} +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs index bcb5550190..f366233518 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs @@ -12,6 +12,7 @@ fn temporal_app(state: SharedState) -> Router { .route("/train", post(train_dispatch)) .route("/evaluate", post(evaluate_dispatch)) .route("/restore", post(restore_dispatch)) + .route("/retry", post(retry_dispatch)) .route("/discard", post(discard_dispatch)) .route("/phase/cancel", post(pause_dispatch)) .route("/train/cancel", post(cancel_operation_dispatch)) @@ -260,6 +261,7 @@ async fn temporal_selection_mutations_keep_the_existing_local_owner_gate() { for path in [ "/api/v1/room-learner/qualify", "/api/v1/room-learner/select", + "/api/v1/room-learner/recovery/retry", ] { for (peer, token) in [ ("127.0.0.1:41000", "synthetic-peer-read"), @@ -595,3 +597,65 @@ async fn fullrun_priming_still_rejects_zero_nan_and_nonfinite_rate_metadata() { let s=decoded(app,&format!("/status?schema={STATUS_SCHEMA}"),None).await;assert_eq!(s["status"]["capture"]["retained_windows"],0);assert_eq!(s["status"]["capture"]["observed_ms"],0); } } + +#[tokio::test] +async fn retry_owner_router_denies_anonymous_and_paired_grants_before_mutating_runtime(){ + let (_dir,state)=fixture();let rt=state.read().await.room_learning.as_ref().unwrap().clone(); + let auth=wifi_densepose_sensing_server::bearer_auth::AuthState::from_token("synthetic-owner"); + let app=owner_selection_routes(auth).with_state(state); + let guard=rt.mutation_guard().unwrap(); + let body=serde_json::json!({"schema":RECOVERY_RETRY_REQUEST_SCHEMA,"model_kind":"ruview.room-learner.managed-temporal-input.v3","expected":guard,"capture_generation_id":"c".repeat(64),"retry_confirmed":true}); + for (token,paired) in [(None,false),(Some("synthetic-peer-read"),false),(Some("synthetic-owner"),true)] { + let mut builder=Request::builder().method(Method::POST).uri("/api/v1/room-learner/recovery/retry").header("content-type","application/json"); + if let Some(token)=token{builder=builder.header("authorization",format!("Bearer {token}"));} + let mut request=builder.body(Body::from(serde_json::to_vec(&body).unwrap())).unwrap(); + request.extensions_mut().insert(axum::extract::ConnectInfo("127.0.0.1:41000".parse::().unwrap())); + if paired {let(_,rx)=tokio::sync::watch::channel(false);request.extensions_mut().insert(wifi_densepose_sensing_server::companion::Grant{device_id:"synthetic-paired-readonly".into(),expires_at:chrono::Utc::now().timestamp().max(0)as u64+30,revoked:rx});} + assert_eq!(app.clone().oneshot(request).await.unwrap().status(),StatusCode::FORBIDDEN); + assert_eq!(rt.mutation_guard().unwrap(),guard); + } +} + +#[tokio::test] +async fn retry_command_refuses_imported_rows_and_wrong_cas_at_the_actual_dispatch(){ + let (_dir,state)=fixture();let rt=state.read().await.room_learning.as_ref().unwrap().clone();let app=temporal_app(state); + let guard=rt.mutation_guard().unwrap(); + let body=serde_json::json!({"schema":RECOVERY_RETRY_REQUEST_SCHEMA,"model_kind":"ruview.room-learner.managed-temporal-input.v3","expected":guard,"capture_generation_id":"c".repeat(64),"retry_confirmed":true}); + for field in ["rows","recording_start","new_training","deadline","committed"]{ + let mut invalid=body.clone();invalid[field]=serde_json::json!(true); + assert_eq!(request_raw(app.clone(),"/retry",Some(invalid)).await.0,StatusCode::UNPROCESSABLE_ENTITY); + assert_eq!(rt.mutation_guard().unwrap(),guard); + } + let mut wrong=body;wrong["expected"]["runtime_epoch"]=serde_json::json!("wrong-runtime"); + let response=decoded(app,"/retry",Some(wrong)).await; + assert_eq!(response["success"],false);assert_eq!(rt.mutation_guard().unwrap(),guard); +} + +#[tokio::test] +async fn actual_owner_retry_http_returns_paused_same_capture_without_starting_recording(){ + use wifi_densepose_sensing_server::room_learner::{observation::SourceSpec,recovery_store::RecoveryStore}; + let(dir,state)=fixture();let owner="a".repeat(64);let path=dir.path().join(&owner).join("SensingServer"); + let store=RecoveryStore::new(&path,&owner,"synthetic-install",Some(&[7;32])).unwrap(); + let rt=Arc::new(RoomLearningRuntime::new_with_recovery(&path,&owner,"synthetic-install",store).unwrap()); + let profile="native-2s-three-source-56bin-operator-eight-step-v1"; + rt.temporal_select_profile_guarded_v2(Some(profile),&rt.mutation_guard().unwrap()).unwrap(); + let sources=(15..=17u8).map(|id|SourceSpec{source_id:id,device_id:format!("esp32-node-{id}"),grid_id:"ant1-bins64-ppdu0-usable-drop2-linear-v2".into(),feature_count:56,signal_kind:ruview_unified::RfModality::WifiCsi,center_frequency_hz:2_437_000_000,bandwidth_hz:20_000_000,link_id:format!("synthetic-{id}"),link_index:0,geometry:None}).collect(); + let context=rt.bind_context("b".repeat(64),1,sources).unwrap(); + rt.temporal_configure_v2(&rt.mutation_guard().unwrap(),context,"synthetic-http-retry".into(),true,rt.receiver_now_ns()).unwrap(); + let started=std::time::Instant::now(); + loop{let status=rt.temporal_status_v2(rt.receiver_now_ns()).unwrap();if status["capture"]["stage"]=="ready"{break;}assert!(started.elapsed()().unwrap())); + let response=router.oneshot(request).await.unwrap();assert_eq!(response.status(),StatusCode::OK); + let bytes=axum::body::to_bytes(response.into_body(),65536).await.unwrap();let reply:serde_json::Value=serde_json::from_slice(&bytes).unwrap(); + assert_eq!(reply["success"],true,"{reply}");let after=&reply["status"]["capture"]; + assert_eq!(after["stage"],"paused");assert_eq!(after["durability_pending"],false);assert!(after["failure"].is_null()); + assert_eq!(after["capture_generation_id"],before["capture"]["capture_generation_id"]);assert_eq!(after["original_deadline"],before["capture"]["original_deadline"]); + assert_eq!(after["retained_windows"],0);assert_eq!(after["durable_windows"],0);assert_eq!(after["observed_ms"],0); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs index c6763579cb..75d63b0a7a 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_tests.rs @@ -199,6 +199,7 @@ fn every_new_mutation_requires_admin_and_default_cli_does_not_enable_capture() { "train/cancel", "evaluate", "recovery/restore", + "recovery/retry", "recovery/discard", "recovery/enable-current", ] { From 54ffbc525be7445b966283117852b28240e87ca6 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Mon, 9 Nov 2026 15:17:09 +0100 Subject: [PATCH 087/100] fix(learner): keep recovery status off RF ingress lock --- .../wifi-densepose-sensing-server/src/main.rs | 14 +- .../src/room_learner/lifecycle.rs | 3 + .../src/room_learner/lifecycle_capture.rs | 51 +++++++- .../src/room_learner/lifecycle_temporal.rs | 4 + .../room_learner/lifecycle_temporal_public.rs | 16 +++ .../receiver_failure_annotation_tests.rs | 36 ++++++ .../receiver_pipeline_timing_tests.rs | 55 ++++++++ .../room_learner/temporal_capture_quality.rs | 11 ++ .../src/room_learner/temporal_execution.rs | 22 ++++ .../src/room_learner_api.rs | 2 +- .../src/room_learner_api_temporal.rs | 93 ++++++++++++-- .../src/room_learner_api_temporal_tests.rs | 120 ++++++++++++++++++ .../src/room_learner_ingress.rs | 81 +++++++++++- 13 files changed, 491 insertions(+), 17 deletions(-) create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/receiver_failure_annotation_tests.rs create mode 100644 v2/crates/wifi-densepose-sensing-server/src/room_learner/receiver_pipeline_timing_tests.rs diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index aa71cbb885..eedee37139 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -13438,6 +13438,8 @@ async fn udp_receiver_task( loop { match socket.recv_from(&mut buf).await { Ok((len, src)) => { + // Host dequeue coordinate only, not kernel or RF measurement time. + let receiver_dequeued_at = std::time::Instant::now(); UDP_INGRESS .datagrams .fetch_add(1, std::sync::atomic::Ordering::Relaxed); @@ -14032,13 +14034,18 @@ async fn udp_receiver_task( continue; } - if let Some(frame) = parse_esp32_frame(&buf[..len]) { + let receiver_parse_started_at = std::time::Instant::now(); + let csi_frame = parse_esp32_frame(&buf[..len]); + let receiver_parse_finished_at = std::time::Instant::now(); + if let Some(frame) = csi_frame { debug!( "ESP32 frame from {src}: node={}, subs={}, seq={}", frame.node_id, frame.n_subcarriers, frame.sequence ); + let receiver_lock_requested_at = std::time::Instant::now(); let mut s = state.write().await; + let receiver_lock_acquired_at = std::time::Instant::now(); s.source = "esp32".to_string(); let observed_at = std::time::Instant::now(); s.last_esp32_frame = Some(observed_at); @@ -14047,7 +14054,10 @@ async fn udp_receiver_task( .entry(frame.node_id) .or_insert_with(NodeState::new) .observe_raw_grid(grid_key, observed_at); - room_learner_api::observe(&mut s, &frame, &buf[..len], observed_at); + room_learner_api::observe_with_receiver_timing(&mut s, &frame, &buf[..len], observed_at, + Some(room_learner_api::ReceiverIngressTiming::new(receiver_dequeued_at, + receiver_parse_finished_at.checked_duration_since(receiver_parse_started_at), + receiver_lock_acquired_at.checked_duration_since(receiver_lock_requested_at)))); // The field model owns a separate, frozen grid path. Feed // it before the global inference grid gate so a stable diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs index 7a9b599230..2b47d522eb 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle.rs @@ -94,6 +94,7 @@ pub struct RoomLearningRuntime { operator_recovery_store: Option>, operator_profile_selected: AtomicBool, operator_profile_initialized: AtomicBool, + status_reader: Arc, inner: Mutex, #[cfg(test)] reclamation_probe:Mutex>>, @@ -203,6 +204,7 @@ impl RoomLearningRuntime { operator_recovery_store, operator_profile_selected: AtomicBool::new(false), operator_profile_initialized: AtomicBool::new(false), + status_reader: Arc::new(tokio::sync::Semaphore::new(1)), #[cfg(test)] reclamation_probe:Mutex::new(None), #[cfg(test)] panic_after_reclamation_ack:AtomicBool::new(false), #[cfg(test)] temporal_fit_hook:Mutex::new(None), @@ -528,6 +530,7 @@ mod tests; #[path = "lifecycle_capture.rs"] mod capture_control; +pub use capture_control::ReceiverPipelineTiming; #[path = "command.rs"] mod command; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs index d43a70fc44..c44f2d95d0 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_capture.rs @@ -1,5 +1,37 @@ //! Managed capture controls and source/provenance admission. use super::*; + +/// Transient metadata for one exact native packet. Deliberately not Serialize. +/// Relative receiver durations are not a source measurement clock. +/// +/// ```compile_fail +/// use wifi_densepose_sensing_server::room_learner::lifecycle::ReceiverPipelineTiming; +/// fn requires_serialize() {} +/// requires_serialize::>(); +/// ``` +#[derive(Clone, Copy)] +pub struct ReceiverPipelineTiming<'a> { + source_id: u8, + sequence: u32, + frame_id: u128, + grid_id: &'a str, + frequency_hz: u64, + bandwidth_hz: u64, + durations: [Option; 4], +} +impl<'a> ReceiverPipelineTiming<'a> { + pub fn for_packet(packet: &'a crate::room_learner::native::CanonicalPacket, + dequeue_interval_us: Option, parse_us: Option, state_lock_wait_us: Option, dequeue_to_stamp_us: Option) -> Self { + Self { source_id: packet.source_id, sequence: packet.sequence, frame_id: packet.native_frame.frame_id, + grid_id: &packet.grid_id, frequency_hz: packet.frequency_hz, bandwidth_hz: packet.bandwidth_hz, + durations: [dequeue_interval_us, parse_us, state_lock_wait_us, dequeue_to_stamp_us] } + } + fn failure_detail(&self, packet: &crate::room_learner::native::CanonicalPacket, primary: &str) -> Option { + if self.source_id != packet.source_id || self.sequence != packet.sequence || self.frame_id != packet.native_frame.frame_id + || self.grid_id != packet.grid_id || self.frequency_hz != packet.frequency_hz || self.bandwidth_hz != packet.bandwidth_hz { return None; } + crate::room_learner::temporal_model_input::managed::execution::receiver_pipeline_failure_detail(primary, self.durations) + } +} impl RoomLearningRuntime { pub fn bind_context( &self, @@ -87,6 +119,14 @@ impl RoomLearningRuntime { &self, packet: &crate::room_learner::native::CanonicalPacket, now_ns: u64, + ) -> Result<(), String> { + self.ingest_native_with_receiver_timing(packet, now_ns, None) + } + pub fn ingest_native_with_receiver_timing( + &self, + packet: &crate::room_learner::native::CanonicalPacket, + now_ns: u64, + receiver_timing: Option>, ) -> Result<(), String> { if packet.provenance != crate::room_learner::observation::CaptureProvenance::LiveReceiverObserved @@ -118,7 +158,12 @@ impl RoomLearningRuntime { let owner_generation=state.generation; if let Some(CaptureWorkflow::Temporal(capture)) = state.capture.as_mut() { if capture.owner_generation()!=owner_generation {capture.invalidate_owner();return Err("temporal_owner_generation_changed".into());} - let result = capture.ingest(self.temporal_store()?, &self.runtime_epoch, packet, now_ns,&mut retired); + let mut result = capture.ingest(self.temporal_store()?, &self.runtime_epoch, packet, now_ns,&mut retired); + if let (Err(primary), Some(timing)) = (&result, receiver_timing) { + if let Some(detail) = timing.failure_detail(packet, primary) { + if capture.annotate_receiver_failure(primary, detail.clone()) { result = Err(detail); } + } + } #[cfg(test)] if retired.has_ack() && self.panic_after_reclamation_ack.swap(false,Ordering::AcqRel){panic!("synthetic panic after actual ACK admission");} if let Err(error) = &result { state.input_error = Some(bounded(error.clone())); } return result; @@ -253,3 +298,7 @@ impl RoomLearningRuntime { Ok(state.job.phase) } } + +#[cfg(test)] +#[path = "receiver_pipeline_timing_tests.rs"] +mod receiver_timing_tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs index c62e5aeecd..aa9fb2e95d 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal.rs @@ -182,6 +182,10 @@ impl RuntimeTemporalCapture { _ => Ok(()), // No automatic start, confirmation or new phase from packet arrival. } } + /// Annotates only the already failed current capture; no failure transition is rerun. + pub(super) fn annotate_receiver_failure(&mut self, expected: &str, detail: String) -> bool { + self.core.as_mut().is_some_and(|core| core.annotate_receiver_failure(expected, detail)) + } pub(super) fn reject(&mut self, source_id: u8, grid_id: &str, reason: &str) { if self .context diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public.rs index b078c2ddc5..1e1d779a23 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_public.rs @@ -425,7 +425,18 @@ impl RoomLearningRuntime { self.start_temporal_evidence(&request.expected, now, EvidencePurpose::HeldOutEvaluation) .map(|attempt| attempt.to_string()) } + /// The permit must live inside the blocking read, not its cancellable HTTP + /// future: one stuck filesystem read cannot accumulate retry workers. + pub async fn acquire_temporal_status_reader(&self) -> Result { + self.status_reader.clone().acquire_owned().await.map_err(|_| "learner_status_reader_unavailable".into()) + } pub fn temporal_status_v2(self: &Arc, now: u64) -> Result { + self.temporal_poll_status_v2(now)?; + self.temporal_status_snapshot_v2(now) + } + /// Caller serializes this clock-advancing poll with live ingress. No disk + /// discovery runs here; it belongs in the separately fenced snapshot. + pub fn temporal_poll_status_v2(self: &Arc, now: u64) -> Result<(), String> { // ACKs may already be ready. This tick does not issue new consent, // source samples, fit/proof/evaluation jobs or immutable metadata writes. let temporal = { @@ -435,6 +446,11 @@ impl RoomLearningRuntime { if temporal { let _ = self.tick_temporal_capture(&self.mutation_guard()?, now); } + Ok(()) + } + /// Read-only recovery discovery may wait for a durable writer. Callers + /// must not hold the shared RF ingress lock across this operation. + pub fn temporal_status_snapshot_v2(self: &Arc, now: u64) -> Result { let (generation,workflow_revision,operator_profile) = { let state=self.inner.lock().map_err(|_| "learner state unavailable")?; (state.generation,state.workflow_revision,self.operator_profile_selected.load(Ordering::Acquire)) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/receiver_failure_annotation_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/receiver_failure_annotation_tests.rs new file mode 100644 index 0000000000..9731eec3ca --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/receiver_failure_annotation_tests.rs @@ -0,0 +1,36 @@ +//! SYNTHETIC private-core metadata update; no actual capture or durability claim. +use super::*; +use super::super::tests::bridge_setup; +const PRIMARY: &str = "temporal_observed_gap_or_rate: source=1 direction=below_min receiver_interval_ns=1000000 min_ns=27777777 max_ns=50000000"; +#[test] +fn failure_annotation_only_replaces_exact_failed_metadata_and_preserves_all_snapshot_facts() { + let (_dir, _store, mut c) = bridge_setup(); + let initial = c.snapshot(); + assert!(!c.annotate_receiver_failure(PRIMARY, format!("{PRIMARY} deq_us=30303"))); + assert_eq!(c.snapshot(), initial, "nonfailed capture must be untouched"); + c.invalidate(PRIMARY); + let before = c.snapshot(); + let floor = c.workflow.source_high.clone(); + let receiver_clock = c.workflow.last_now_ns; + let durable = c.workflow.durable.clone(); + let detail = format!("{PRIMARY} deq_us=30303 parse_us=10 lock_us=50000 stamp_age_us=50015"); + assert!(c.annotate_receiver_failure(PRIMARY, detail.clone())); + let mut after = c.snapshot(); + assert_eq!(after["failure"], detail); + after["failure"] = before["failure"].clone(); + assert_eq!(after, before, "no stage, time, pending ACK, retained rows, or counters may change"); + assert_eq!(c.workflow.source_high, floor); + assert_eq!(c.workflow.last_now_ns, receiver_clock); + assert_eq!(c.workflow.durable, durable); + let annotated = c.snapshot(); + assert!(!c.annotate_receiver_failure(PRIMARY, format!("{PRIMARY} stale_expected=1"))); + assert_eq!(c.snapshot(), annotated); +} +#[test] +fn failure_annotation_refuses_oversized_or_unrelated_metadata_without_cutting_primary() { + let (_dir, _store, mut c) = bridge_setup(); c.invalidate(PRIMARY); + let before = c.snapshot(); + assert!(!c.annotate_receiver_failure(PRIMARY, format!("{PRIMARY}{}", "x".repeat(256)))); + assert!(!c.annotate_receiver_failure(PRIMARY, "different failure".into())); + assert_eq!(c.snapshot(), before); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/receiver_pipeline_timing_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/receiver_pipeline_timing_tests.rs new file mode 100644 index 0000000000..cff186c38e --- /dev/null +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/receiver_pipeline_timing_tests.rs @@ -0,0 +1,55 @@ +//! SYNTHETIC relative receiver timing; no captured RF or physical clock claim. +use super::*; +use crate::room_learner::{native::{canonicalize_esp32, NativePacketInput, CanonicalPacket}, observation::CaptureProvenance}; +use wifi_densepose_hardware::{Adr018Flags, PpduType}; + +fn packet(source: u8, sequence: u32, stamp: u64, ppdu: PpduType, bins: usize) -> CanonicalPacket { + canonicalize_esp32(NativePacketInput { node_id: source, n_antennas: 1, raw_bins: bins, + frequency_hz: 2_437_000_000, ppdu_type: ppdu, sequence, received_at_ns: stamp, + receiver_sample_rate_hz: 33.0, iq_bytes: &vec![4u8; bins * 2], + flags: Adr018Flags::from_byte(0), provenance: CaptureProvenance::Synthetic }).unwrap() +} +const PRIMARY: &str = "temporal_observed_gap_or_rate: source=15 direction=below_min receiver_interval_ns=1000000 min_ns=27777777 max_ns=50000000"; + +#[test] +fn pipeline_failure_reports_relative_durations_without_changing_packet_identity_or_values() { + let p = packet(15, 12, 2_000_000_000, PpduType::HtLegacy, 64); + let before = serde_json::to_vec(&p.summary()).unwrap(); + let iq = p.native_frame.native_iq.clone(); + let magnitudes = p.amplitudes.clone(); + let id = p.native_frame.frame_id; + // Distinct synthetic dequeue spacing and processing interval are NOT clocks. + let timing = ReceiverPipelineTiming::for_packet(&p, Some(30_303), Some(42), Some(60_000), Some(60_050)); + let detail = timing.failure_detail(&p, PRIMARY).unwrap(); + assert_eq!(detail, format!("{PRIMARY} deq_us=30303 parse_us=42 lock_us=60000 stamp_age_us=60050")); + assert_eq!(serde_json::to_vec(&p.summary()).unwrap(), before); + assert_eq!(p.native_frame.native_iq, iq); + assert_eq!(p.amplitudes, magnitudes); + assert_eq!(p.native_frame.frame_id, id); + assert!(!detail.contains("owner") && !detail.contains("token") && !detail.contains("timestamp")); +} + +#[test] +fn receiver_timing_never_attaches_another_packet_or_phy_class() { + let p = packet(15, 12, 2_000_000_000, PpduType::HtLegacy, 64); + let timing = ReceiverPipelineTiming::for_packet(&p, Some(30_303), None, Some(12), None); + for other in [packet(16, 12, 2_000_000_000, PpduType::HtLegacy, 64), + packet(15, 13, 2_000_000_000, PpduType::HtLegacy, 64), + packet(15, 12, 2_000_000_001, PpduType::HtLegacy, 64), + packet(15, 12, 2_000_000_000, PpduType::HeSu, 256)] { + assert!(timing.failure_detail(&other, PRIMARY).is_none()); + } + assert!(timing.failure_detail(&p, "temporal_missing_segment_source").is_none()); +} + +#[test] +fn failure_primary_is_preserved_and_unknown_durations_are_not_fabricated() { + let p = packet(15, 12, 2_000_000_000, PpduType::HtLegacy, 64); + let timing = ReceiverPipelineTiming::for_packet(&p, None, Some(u32::MAX), None, Some(0)); + let detail = timing.failure_detail(&p, PRIMARY).unwrap(); + assert!(detail.starts_with(PRIMARY)); + assert!(detail.ends_with("deq_us=na parse_us=4294967295 lock_us=na stamp_age_us=0")); + assert!(detail.len() <= 256); + let long_primary = format!("{PRIMARY}{}", "x".repeat(256 - PRIMARY.len())); + assert!(timing.failure_detail(&p, &long_primary).is_none(), "never truncate the primary to make metadata fit"); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs index 544de436a5..db05dcddcd 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_capture_quality.rs @@ -1082,6 +1082,13 @@ impl TemporalTrainCapture { .len(); Ok(()) } + /// Metadata-only update after the exact packet has already failed admission. + pub(crate) fn annotate_receiver_failure(&mut self, expected: &str, detail: String) -> bool { + if self.stage() != TemporalCaptureStage::Failed || self.failure.as_deref() != Some(expected) + || detail.len() > 256 || !detail.starts_with(expected) { return false; } + self.failure = Some(detail); + true + } pub(crate) fn invalidate(&mut self, reason: &str) { if let Some(state)=&self.workflow.async_state{state.writer.cancel();} self.failure = Some( @@ -1259,3 +1266,7 @@ impl TemporalTrainCapture { impl TemporalTrainCapture { pub(crate) fn test_corrupt_retry_prefix(&mut self){self.workflow.async_state.as_mut().unwrap().row_prefix_sha256=Some("f".repeat(64));} } + +#[cfg(test)] +#[path = "receiver_failure_annotation_tests.rs"] +mod receiver_failure_annotation_tests; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs index 353354a467..a71522eedc 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/temporal_execution.rs @@ -837,3 +837,25 @@ impl TemporalWorkflow { Ok(()) } } + + +/// Failure-only receiver pipeline metadata. No clock value or RF row is exported. +/// Unknown/oversized durations stay unknown; the original failure is never cut. +pub(crate) fn receiver_pipeline_failure_detail( + primary: &str, + durations: [Option; 4], +) -> Option { + if primary.len() > 256 || !primary.starts_with("temporal_observed_gap_or_rate:") { return None; } + const KEYS: [&str; 4] = ["deq_us", "parse_us", "lock_us", "stamp_age_us"]; + let digits = |value: Option| -> usize { value.map_or(2, |n| if n == 0 { 1 } else { n.ilog10() as usize + 1 }) }; + let length = primary.len() + KEYS.iter().zip(durations).map(|(key, value)| key.len() + 2 + digits(value)).sum::(); + if length > 256 { return None; } + let mut detail = String::with_capacity(length); + detail.push_str(primary); + for (key, value) in KEYS.iter().zip(durations) { + detail.push(' '); detail.push_str(key); detail.push('='); + if let Some(value) = value { use std::fmt::Write; write!(&mut detail, "{value}").ok()?; } + else { detail.push_str("na"); } + } + Some(detail) +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs index f3c9f78135..ff95542388 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api.rs @@ -13,7 +13,7 @@ use wifi_densepose_sensing_server::room_learner::{ #[path = "room_learner_ingress.rs"] mod ingress; use ingress::native_candidates; -pub(super) use ingress::{observe, LiveSource, NativeSourceKey}; +pub(super) use ingress::{observe, observe_with_receiver_timing, ReceiverIngressTiming, LiveSource, NativeSourceKey}; fn failure(code: &str, error: impl ToString) -> Json { let message = error.to_string(); diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal.rs index 3939d022ba..409b43d11a 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal.rs @@ -98,6 +98,9 @@ fn temporal_selection( }), None => rt.temporal_selection_v2(now, None, Some(&state.calibration_boot_epoch)), }?; + validate_temporal_selection(projection) +} +fn validate_temporal_selection(projection: serde_json::Value) -> Result { if projection["schema"] != SELECTION_SCHEMA || projection["success"] != true || projection["model_selection"]["schema"] != SELECTION_STATUS_SCHEMA @@ -110,22 +113,88 @@ async fn temporal_status( state: &SharedState, rt: &Arc, ) -> Json { - // Polling a durable ACK advances the capture clock. Serialize before - // sampling receiver time, just like native ingress and owner mutations. - let state = state.write().await; - if state.privacy_mode { - return failure( - "learner_privacy_mode", - "Room learning unavailable in privacy mode.", - ); + temporal_status_with_reader(state, rt, |reader, now| reader.temporal_status_snapshot_v2(now)).await +} +// The reader seam keeps the blocking boundary explicit and allows a +// deterministic concurrency regression without a production runtime hook. +pub(super) async fn temporal_status_with_reader( + state: &SharedState, + rt: &Arc, + read_snapshot: F, +) -> Json +where + F: FnOnce(Arc, u64) -> Result + Send + 'static, +{ + // Concurrent polls wait as async futures, without owning an RF lock or + // spawning another blocking worker. Explicit cancellation removes a waiter. + let permit = match rt.acquire_temporal_status_reader().await { + Ok(permit) => permit, + Err(error) => return failure("learner_status_unavailable", error), + }; + // Clock advancement stays ordered with native ingress, but recovery disk + // discovery must never hold the shared RF lock while waiting on fsync. + let (now, expected, room_store, room_binding, boot_epoch) = { + let current = state.write().await; + if current.privacy_mode { + return failure("learner_privacy_mode", "Room learning unavailable in privacy mode."); + } + if !current.room_learning.as_ref().is_some_and(|live| Arc::ptr_eq(live, rt)) { + return failure("learner_state_changed", "learner_state_changed"); + } + let now = now_ns(rt); + if let Err(error) = rt.temporal_poll_status_v2(now) { + return failure("learner_status_unavailable", error); + } + let expected = match rt.mutation_guard() { + Ok(expected) => expected, + Err(error) => return failure("learner_status_unavailable", error), + }; + let room_binding = match current.current_room_store.as_ref().map(|room| room.current_binding_digest()).transpose() { + Ok(binding) => binding, + Err(_) => return failure("learner_state_changed", "current_room_binding_unavailable"), + }; + (now, expected, current.current_room_store.clone(), room_binding, current.calibration_boot_epoch.clone()) + }; + let reader = rt.clone(); + let read_binding = room_binding.clone(); + let read_epoch = boot_epoch.clone(); + let snapshot = tokio::task::spawn_blocking(move || { + // Cancellation of the awaiting HTTP request cannot drop this permit. + let _permit = permit; + let mut status = read_snapshot(reader.clone(), now)?; + let projection = reader.temporal_selection_v2(now, read_binding.as_ref().and_then(|binding| binding.as_deref()), Some(&read_epoch))?; + status["model_selection"] = validate_temporal_selection(projection)?["model_selection"].clone(); + Ok::<_, String>(status) + }).await; + let current = state.write().await; + if current.privacy_mode { + return failure("learner_privacy_mode", "Room learning unavailable in privacy mode."); } - let now = now_ns(rt); - match rt.temporal_status_v2(now).and_then(|mut status| { - status["model_selection"] = temporal_selection(&state, rt, now)?["model_selection"].clone(); + let same_room_store = match (&room_store, ¤t.current_room_store) { + (None, None) => true, + (Some(before), Some(after)) => Arc::ptr_eq(before, after), + _ => false, + }; + if !current.room_learning.as_ref().is_some_and(|live| Arc::ptr_eq(live, rt)) + || !same_room_store + || current.calibration_boot_epoch != boot_epoch + || current.current_room_store.as_ref().map(|room| room.current_binding_digest()).transpose().ok().as_ref() != Some(&room_binding) + || rt.mutation_guard().as_ref() != Ok(&expected) + { + return failure("learner_state_changed", "learner_state_changed"); + } + let result = match snapshot { + Ok(result) => result, + Err(_) => return failure("learner_status_unavailable", "learner_status_worker_unavailable"), + }; + match result.and_then(|status| { + if status["mutation_guard"] != serde_json::to_value(&expected).map_err(|_| "learner_status_guard_invalid")? { + return Err("learner_state_changed".into()); + } Ok(status) }) { Ok(status) => Json(serde_json::json!({"success":true,"status":status})), - Err(e) => failure("learner_status_unavailable", e), + Err(error) => failure("learner_status_unavailable", error), } } async fn operation_status( diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs index f366233518..2b777a0a0b 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_api_temporal_tests.rs @@ -659,3 +659,123 @@ async fn actual_owner_retry_http_returns_paused_same_capture_without_starting_re assert_eq!(after["capture_generation_id"],before["capture"]["capture_generation_id"]);assert_eq!(after["original_deadline"],before["capture"]["original_deadline"]); assert_eq!(after["retained_windows"],0);assert_eq!(after["durable_windows"],0);assert_eq!(after["observed_ms"],0); } + +/// SYNTHETIC real recovery file-lock contention through the production status +/// composer. The handshake runs inside the actual blocking reader; there is no +/// timing sleep or production Runtime hook standing in for contention. +#[cfg(unix)] +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn temporal_status_recovery_wait_releases_ingress_and_rechecks_authority() { + use std::os::unix::fs::{OpenOptionsExt, PermissionsExt}; + use wifi_densepose_sensing_server::room_learner::recovery_store::RecoveryStore; + for change in ["none", "privacy", "runtime", "profile", "epoch"] { + let (dir, state) = fixture(); + let owner = "a".repeat(64); + let path = dir.path().join(&owner).join("SensingServer"); + let recovery = RecoveryStore::new(&path, &owner, "synthetic-install", Some(&[7; 32])).unwrap(); + let rt = Arc::new(RoomLearningRuntime::new_with_recovery(&path, &owner, "synthetic-install", recovery).unwrap()); + state.write().await.room_learning = Some(rt.clone()); + let recovery_path = path.join("room-learner-recovery"); + let prefix_path = recovery_path.join("temporal-prefix-commitment-v1"); + std::fs::create_dir(&prefix_path).unwrap(); + std::fs::set_permissions(&prefix_path, std::fs::Permissions::from_mode(0o700)).unwrap(); + let held = std::fs::OpenOptions::new().read(true).write(true).create(true).truncate(false) + .mode(0o600).open(recovery_path.join("temporal-prefix.lock")).unwrap(); + held.lock().unwrap(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let reader_state = state.clone(); + let reader_runtime = rt.clone(); + let task = tokio::spawn(async move { + temporal_status_with_reader(&reader_state, &reader_runtime, move |reader, now| { + entered_tx.send(()).unwrap(); + reader.temporal_status_snapshot_v2(now) + }).await.0 + }); + tokio::time::timeout(std::time::Duration::from_secs(2), entered_rx).await.unwrap().unwrap(); + // This is exactly the shared write acquisition the live UDP path needs. + // Before the repair it could not complete until held was unlocked. + let mut ingress = tokio::time::timeout(std::time::Duration::from_secs(2), state.write()).await + .expect("recovery file I/O must not hold the RF ingress lock"); + assert!(!task.is_finished(), "the real recovery reader must still be blocked"); + match change { + "none" => {}, + "privacy" => ingress.privacy_mode = true, + "runtime" => ingress.room_learning = None, + "profile" => { rt.temporal_select_profile_guarded_v2(Some("native-2s-three-source-56bin-operator-eight-step-v1"), &rt.mutation_guard().unwrap()).unwrap(); }, + "epoch" => ingress.calibration_boot_epoch.push_str("-changed"), + _ => unreachable!(), + } + drop(ingress); + held.unlock().unwrap(); + let result = tokio::time::timeout(std::time::Duration::from_secs(2), task).await.unwrap().unwrap(); + if change == "none" { + assert_eq!(result["success"], true, "{result}"); + assert_eq!(result["status"]["mutation_guard"], serde_json::to_value(rt.mutation_guard().unwrap()).unwrap()); + } else { + assert_eq!(result["success"], false, "{change}: {result}"); + assert!(result.get("status").is_none(), "{change}: stale snapshot must not escape"); + assert_eq!(result["error_code"], if change == "privacy" {"learner_privacy_mode"} else {"learner_state_changed"}); + } + } +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn cancelled_status_retains_one_reader_permit_until_actual_work_finishes() { + let (_dir, state) = fixture(); + let rt = state.read().await.room_learning.clone().unwrap(); + let (entered_tx, entered_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let blocked_state = state.clone(); + let blocked_rt = rt.clone(); + let blocked = tokio::spawn(async move { + temporal_status_with_reader(&blocked_state, &blocked_rt, move |reader, now| { + entered_tx.send(()).unwrap(); + release_rx.recv().unwrap(); + reader.temporal_status_snapshot_v2(now) + }).await + }); + tokio::time::timeout(std::time::Duration::from_secs(2), entered_rx).await.unwrap().unwrap(); + blocked.abort(); + assert!(blocked.await.unwrap_err().is_cancelled()); + // Poll a real waiter once: the aborted HTTP task must not have released + // the permit still owned by its actual blocking reader. + let mut direct_wait = Box::pin(rt.acquire_temporal_status_reader()); + std::future::poll_fn(|cx| { + assert!(std::future::Future::poll(direct_wait.as_mut(), cx).is_pending()); + std::task::Poll::Ready(()) + }).await; + drop(direct_wait); + let invoked = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let mut waiters = Vec::new(); + for _ in 0..3 { + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let waiter_state = state.clone(); + let waiter_rt = rt.clone(); + let calls = invoked.clone(); + waiters.push(tokio::spawn(async move { + started_tx.send(()).unwrap(); + temporal_status_with_reader(&waiter_state, &waiter_rt, move |reader, now| { + calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + reader.temporal_status_snapshot_v2(now) + }).await + })); + tokio::time::timeout(std::time::Duration::from_secs(2), started_rx).await.unwrap().unwrap(); + } + assert_eq!(invoked.load(std::sync::atomic::Ordering::SeqCst), 0, + "concurrent async waiters must not spawn more blocking readers"); + for waiter in waiters { + assert!(!waiter.is_finished()); + waiter.abort(); + assert!(waiter.await.unwrap_err().is_cancelled()); + } + // Cancellation did not retain the RF ingress lock either. + drop(tokio::time::timeout(std::time::Duration::from_secs(2), state.write()).await.unwrap()); + release_tx.send(()).unwrap(); + let permit = tokio::time::timeout(std::time::Duration::from_secs(2), rt.acquire_temporal_status_reader()).await + .expect("permit must return only after real blocking work exits").unwrap(); + drop(permit); + assert_eq!(invoked.load(std::sync::atomic::Ordering::SeqCst), 0, + "explicitly canceled async waiters must never run later"); + let response = temporal_status_with_reader(&state, &rt, |reader, now| reader.temporal_status_snapshot_v2(now)).await.0; + assert_eq!(response["success"], true, "{response}"); +} diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner_ingress.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner_ingress.rs index f20845fedb..1796b722c5 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner_ingress.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner_ingress.rs @@ -1,6 +1,23 @@ //! Header-only readiness is keyed to the exact PHY class admitted by the learner. use super::*; use std::collections::VecDeque; + +/// Receiver-only pipeline coordinates. Deliberately not Serialize and never an admission clock. +#[derive(Clone, Copy)] +pub(crate) struct ReceiverIngressTiming { + dequeued_at: std::time::Instant, + parse_us: Option, + lock_us: Option, +} +fn diagnostic_micros(duration: Option) -> Option { + duration.and_then(|d| d.as_micros().try_into().ok()) +} +impl ReceiverIngressTiming { + pub(crate) fn new(dequeued_at: std::time::Instant, parse: Option, lock_wait: Option) -> Self { + Self { dequeued_at, parse_us: diagnostic_micros(parse), lock_us: diagnostic_micros(lock_wait) } + } +} + #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] pub(crate) struct NativeSourceKey { pub(super) node_id: u8, @@ -28,8 +45,14 @@ pub(crate) struct LiveSource { pub(super) bandwidth_hz: Option, pub(super) receiver_rate_hz: f64, arrivals: VecDeque, + // Separate from arrivals/readiness/model input; one relative diagnostic predecessor. + last_dequeued: Option, } impl LiveSource { + fn diagnostic_dequeue_interval(&mut self, at: std::time::Instant) -> Option { + let previous = self.last_dequeued.replace(at); + diagnostic_micros(previous.and_then(|before| at.checked_duration_since(before))) + } fn record(&mut self, now: std::time::Instant) { while self .arrivals @@ -102,6 +125,15 @@ pub(crate) fn observe( frame: &Esp32Frame, packet: &[u8], now: std::time::Instant, +) { + observe_with_receiver_timing(state, frame, packet, now, None); +} +pub(crate) fn observe_with_receiver_timing( + state: &mut AppStateInner, + frame: &Esp32Frame, + packet: &[u8], + now: std::time::Instant, + receiver_timing: Option, ) { let Some(runtime) = state.room_learning.clone() else { return; @@ -171,12 +203,15 @@ pub(crate) fn observe( bandwidth_hz: Some(20_000_000), receiver_rate_hz: 0.0, arrivals: VecDeque::new(), + last_dequeued: None, }); meta.record(now); let rate = meta.receiver_rate_hz; if !runtime.selects_grid(frame.node_id, &id) { return; } + let dequeue_interval_us = receiver_timing.and_then(|t| meta.diagnostic_dequeue_interval(t.dequeued_at)); + let native_stamp_at = receiver_timing.map(|_| std::time::Instant::now()); let ns = runtime.receiver_now_ns(); let projected = match native::canonicalize_esp32(NativePacketInput { node_id: frame.node_id, @@ -197,5 +232,49 @@ pub(crate) fn observe( return; } }; - let _ = runtime.ingest_native(&projected, ns); + let timing = receiver_timing.map(|t| { + wifi_densepose_sensing_server::room_learner::lifecycle::ReceiverPipelineTiming::for_packet( + &projected, dequeue_interval_us, t.parse_us, t.lock_us, + diagnostic_micros(native_stamp_at.and_then(|at| at.checked_duration_since(t.dequeued_at)))) + }); + let _ = runtime.ingest_native_with_receiver_timing(&projected, ns, timing); +} + + +#[cfg(test)] +mod receiver_timing_tests { + use super::*; + use std::time::{Duration, Instant}; + fn source() -> LiveSource { LiveSource { frequency_hz:2_437_000_000, bandwidth_hz:Some(20_000_000), receiver_rate_hz:33.0, arrivals:VecDeque::new(), last_dequeued:None } } + #[test] + fn diagnostic_cache_is_separate_from_readiness_and_exact_phy_keys_do_not_mix() { + let t = Instant::now(); + let ht_key = NativeSourceKey { node_id:15, n_antennas:1, raw_bins:64, ppdu_type:0, frequency_mhz:2437, bw40:false }; + let he_key = NativeSourceKey { raw_bins:256, ppdu_type:1, ..ht_key }; + let mut cache = std::collections::HashMap::from([(ht_key, source()), (he_key, source())]); + let ht = cache.get_mut(&ht_key).unwrap(); ht.record(t); + let prior = (ht.arrivals.len(), ht.receiver_rate_hz, ht.eligible(t)); + assert_eq!(ht.diagnostic_dequeue_interval(t), None); + assert_eq!(ht.diagnostic_dequeue_interval(t + Duration::from_micros(30_303)), Some(30_303)); + assert_eq!((ht.arrivals.len(), ht.receiver_rate_hz, ht.eligible(t)), prior); + let he = cache.get_mut(&he_key).unwrap(); + assert_eq!(he.diagnostic_dequeue_interval(t + Duration::from_micros(30_304)), None); + } + #[test] + fn invalid_or_oversized_durations_stay_unknown_instead_of_clipping() { + assert_eq!(diagnostic_micros(None), None); + assert_eq!(diagnostic_micros(Some(Duration::from_micros(u64::from(u32::MAX) + 1))), None); + assert_eq!(diagnostic_micros(Some(Duration::from_micros(12))), Some(12)); + let t = Instant::now(); let mut s = source(); + s.diagnostic_dequeue_interval(t + Duration::from_millis(1)); + assert_eq!(s.diagnostic_dequeue_interval(t), None); + } + #[test] + fn synthetic_post_dequeue_delay_does_not_become_a_radio_timestamp() { + let t = Instant::now(); + let timing = ReceiverIngressTiming::new(t, Some(Duration::from_micros(20)), Some(Duration::from_millis(60))); + assert_eq!(timing.parse_us, Some(20)); assert_eq!(timing.lock_us, Some(60_000)); + let before_native_stamp = t + Duration::from_micros(60_025); + assert_eq!(diagnostic_micros(before_native_stamp.checked_duration_since(timing.dequeued_at)), Some(60_025)); + } } From fd425a98cbfe6892c09257de6a94f0190b581f64 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Mon, 9 Nov 2026 15:27:23 +0100 Subject: [PATCH 088/100] ci: keep manual learner verification nonpublishing --- .github/workflows/ci.yml | 8 ++++++-- .github/workflows/security-scan.yml | 4 ++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 60e78c2fd2..d874daf580 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -420,6 +420,9 @@ jobs: # workflow, so it doesn't gate the rest of the pipeline. docker-build: name: Docker Build & Test + # Manual CI is verification only. Use the separate canonical publisher + # for an explicitly authorized image publication. + if: github.event_name != 'workflow_dispatch' runs-on: ubuntu-latest needs: [code-quality, test, rust-tests] continue-on-error: true @@ -495,7 +498,7 @@ jobs: name: API Documentation runs-on: ubuntu-latest needs: [docker-build] - if: github.ref == 'refs/heads/main' + if: github.ref == 'refs/heads/main' && github.event_name != 'workflow_dispatch' permissions: contents: write # gh-pages deploy needs write (GITHUB_TOKEN is read-only by default -> 403) steps: @@ -540,7 +543,8 @@ jobs: name: Notify runs-on: ubuntu-latest needs: [code-quality, test, rust-tests, ui-tests, performance-test, docker-build, docs] - if: always() + # This job also creates releases and sends external notifications. + if: always() && github.event_name != 'workflow_dispatch' permissions: contents: write # required by softprops/action-gh-release # GitHub Actions does not allow `secrets.X` directly in step-level `if:` diff --git a/.github/workflows/security-scan.yml b/.github/workflows/security-scan.yml index 0921727c90..134f7391f7 100644 --- a/.github/workflows/security-scan.yml +++ b/.github/workflows/security-scan.yml @@ -453,7 +453,7 @@ jobs: # job-level env block (added below). - name: Notify security team on critical findings continue-on-error: true - if: ${{ env.SECURITY_SLACK_WEBHOOK_URL != '' && (needs.sast.result == 'failure' || needs.dependency-scan.result == 'failure' || needs.container-scan.result == 'failure') }} + if: ${{ github.event_name != 'workflow_dispatch' && env.SECURITY_SLACK_WEBHOOK_URL != '' && (needs.sast.result == 'failure' || needs.dependency-scan.result == 'failure' || needs.container-scan.result == 'failure') }} uses: 8398a7/action-slack@77eaa4f1c608a7d68b38af4e3f739dcd8cba273e # v3 with: status: failure @@ -469,7 +469,7 @@ jobs: - name: Create security issue on critical findings continue-on-error: true - if: needs.sast.result == 'failure' || needs.dependency-scan.result == 'failure' + if: github.event_name != 'workflow_dispatch' && (needs.sast.result == 'failure' || needs.dependency-scan.result == 'failure') uses: actions/github-script@00f12e3e20659f42342b1c0226afda7f7c042325 # v6 with: script: | From 945d8f8528bfa5b61357d8a1be08830890b71283 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Mon, 9 Nov 2026 16:16:03 +0100 Subject: [PATCH 089/100] test(learner): bind Frozen revisions to the durable Train prefix --- .../lifecycle_temporal_full_run_tests.rs | 73 +++++++++++++++++-- 1 file changed, 65 insertions(+), 8 deletions(-) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs index f3e3ddf951..3d4a3c53b5 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs @@ -280,6 +280,8 @@ fn record_train_uncached(root: &Path) -> (Arc, u64) { .unwrap(); let mut sequence = 0; let mut starts = 0; + let mut forced_pause = false; + let mut forced_resume = false; for episode in schedule .episodes() .iter() @@ -289,17 +291,69 @@ fn record_train_uncached(root: &Path) -> (Arc, u64) { let progress = fixture::snapshot(&runtime, now); match progress["stage"].as_str().unwrap() { "ready" | "paused" => { + let resume_revision = if progress["stage"] == "paused" + && forced_pause && !forced_resume + { + Some((runtime.temporal_store().unwrap().load_temporal(now) + .unwrap().unwrap().token.data().revision, + progress["durable_windows"].as_u64().unwrap())) + } else { None }; now = fixture::async_lead_end(&runtime, now); starts += 1; runtime .tick_temporal_capture(&runtime.mutation_guard().unwrap(), now) .unwrap(); + if let Some((before, durable_rows)) = resume_revision { + // Lead readiness is only recertification. Admit the + // actual SEGMENT ACK while every source remains live. + let wait = Instant::now(); + loop { + sequence += 1; + for source in 1..=3 { + runtime.ingest_native( + &packet(source, sequence, now, episode.condition), now, + ).unwrap(); + } + let pending = fixture::snapshot(&runtime, now)["durability_pending"] == true; + now += fixture::PERIOD; + if !pending { break; } + assert!(wait.elapsed() < Duration::from_secs(30)); + std::thread::sleep(Duration::from_nanos(fixture::PERIOD)); + } + let after = runtime.temporal_store().unwrap().load_temporal(now) + .unwrap().unwrap().token.data().revision; + assert_eq!(after, before.checked_add(1).unwrap(), + "the explicit resumed segment is exactly one durable transition"); + let resumed = fixture::snapshot(&runtime, now); + assert_eq!(resumed["stage"], "observing"); + assert_eq!(resumed["durable_windows"], durable_rows); + assert_eq!(resumed["retained_rows"], durable_rows, + "the SEGMENT ACK must not be confused with a new WINDOW ACK"); + forced_resume = true; + } } "needs_confirmation" => break, "failed" => panic!("synthetic C4A phase{} failed: {progress}", episode.index), _ => {} } let pending = fixture::snapshot(&runtime, now)["durability_pending"] == true; + if !forced_pause && episode.index == 0 && !pending + && progress["stage"] == "observing" + && progress["durable_windows"].as_u64().is_some_and(|n| n >= 1) + && progress["retained_rows"] == progress["durable_windows"] + { + let before = runtime.temporal_store().unwrap().load_temporal(now) + .unwrap().unwrap().token.data().revision; + runtime.pause_temporal_capture(&runtime.mutation_guard().unwrap(), now).unwrap(); + now = fixture::settle_async_boundary(&runtime, now); + assert_eq!(fixture::snapshot(&runtime, now)["stage"], "paused"); + let after = runtime.temporal_store().unwrap().load_temporal(now) + .unwrap().unwrap().token.data().revision; + assert_eq!(after, before.checked_add(1).unwrap(), + "the explicit legal pause is exactly one durable transition"); + forced_pause = true; + continue; + } sequence += 1; for source in 1..=3 { match runtime.ingest_native(&packet(source, sequence, now, episode.condition), now) @@ -324,6 +378,8 @@ fn record_train_uncached(root: &Path) -> (Arc, u64) { now = fixture::settle_async_boundary(&runtime, now); } let complete = fixture::snapshot(&runtime, now); + assert!(forced_pause && forced_resume, + "the native producer must exercise an actual durable pause and resume"); assert_eq!(complete["stage"], "complete"); assert_eq!( complete["sealed_train_episodes"], @@ -420,11 +476,9 @@ async fn c4a_actual_seven_train_fit_frozen_ack_and_exact_cold_model() { .load_temporal(now) .unwrap() .unwrap(); - assert_eq!( - loaded.token.data().revision, - 78, - "actual genesis +7 segments +64 windows +7 seals" - ); + let train_revision = loaded.token.data().revision; + assert!(train_revision >= 80, + "all initial segments, 64 windows and 7 seals plus the forced pause/resume must be durable; extra legal pauses add transitions"); let (compute_tx, compute_rx) = std::sync::mpsc::sync_channel(1); let (compute_release_tx, compute_release_rx) = std::sync::mpsc::sync_channel(1); let mut compute_gate = ReleaseGate(Some(compute_release_tx)); @@ -599,7 +653,7 @@ async fn c4a_actual_seven_train_fit_frozen_ack_and_exact_cold_model() { .unwrap(); assert_eq!( loaded.token.data().revision, - 79, + train_revision.checked_add(1).unwrap(), "one actual durable Frozen transition" ); let data: serde_json::Value = serde_json::from_slice(loaded.prefix.unwrap().bytes()).unwrap(); @@ -687,7 +741,7 @@ async fn c4a_actual_seven_train_fit_frozen_ack_and_exact_cold_model() { .token .data() .revision, - 79 + train_revision.checked_add(1).unwrap() ); drop(runtime); let cold_now = post_ack + 1_000_000_000; @@ -895,6 +949,8 @@ async fn c4a_owner_cancel_drains_actual_prepared_frozen_writer_before_synthetic_ async fn c4a_actual_frozen_lost_ack_same_intent_repair_installs_only_after_exact_ack() { let directory = tempfile::tempdir().unwrap(); let (rt, now) = record_train(directory.path()); + let train_revision = rt.temporal_store().unwrap().load_temporal(now) + .unwrap().unwrap().token.data().revision; let at = fit_ready(&rt, now).await; let writer = { let s = rt.inner.lock().unwrap(); @@ -924,7 +980,8 @@ async fn c4a_actual_frozen_lost_ack_same_intent_repair_installs_only_after_exact .unwrap() .unwrap() .token; - assert_eq!(token.data().revision, 79); + assert_eq!(token.data().revision, train_revision.checked_add(1).unwrap(), + "Frozen adds exactly one transition to the actual completed Train prefix"); rt.repair_temporal_capture(&rt.mutation_guard().unwrap(), now) .unwrap(); now = fixture::settle_async_boundary(&rt, now); From 1c4454715adaf9448bfc04089f3472fe113ba388 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Mon, 9 Nov 2026 16:50:36 +0100 Subject: [PATCH 090/100] fix(calibration): abstain on stale or missing presence specialists --- .../src/multistatic.rs | 40 ++++++++++++++++++- .../wifi-densepose-calibration/src/runtime.rs | 40 +++++++++++++++++-- 2 files changed, 76 insertions(+), 4 deletions(-) diff --git a/v2/crates/wifi-densepose-calibration/src/multistatic.rs b/v2/crates/wifi-densepose-calibration/src/multistatic.rs index 7fbcb422d7..46115bbe60 100644 --- a/v2/crates/wifi-densepose-calibration/src/multistatic.rs +++ b/v2/crates/wifi-densepose-calibration/src/multistatic.rs @@ -293,7 +293,45 @@ mod tests { m.add_node(1, bank("b1"), "b2"); // trained on b1, now observing b2 → stale let mut per = BTreeMap::new(); per.insert(1u8, live(12.0, 0.2, 0.3, 0.9)); - assert!(m.infer(&per).stale); + let s = m.infer(&per); + assert!(s.stale); + assert!(s.presence.is_none(), "stale is unknown, not empty"); + assert!(s.breathing.is_none()); + assert!(s.heartbeat.is_none()); + assert!(s.posture.is_none()); + assert!(s.anomaly.is_none()); + assert!(s.restlessness.is_none()); + } + + #[test] + fn stale_node_cannot_supply_vitals_or_veto_fresh_node() { + let mut m = MultiNodeMixture::new(); + m.add_node(1, bank("old"), "new"); + m.add_node(2, bank("fresh"), "fresh"); + let mut per = BTreeMap::new(); + per.insert(1u8, live(9000.0, 500.0, 0.2, 0.99)); + per.insert(2u8, live(12.0, 0.2, 0.3, 0.95)); + let s = m.infer(&per); + assert!(s.stale, "existing conservative room flag is preserved"); + assert!(!s.vetoed, "stale anomaly must abstain"); + assert_eq!(s.presence.unwrap().value, 1.0); + assert!((s.breathing.unwrap().value - 18.0).abs() < 0.3); + } + + #[test] + fn incomplete_node_bank_does_not_invent_absent_presence_or_vitals() { + let mut m = MultiNodeMixture::new(); + let mut incomplete = bank("b1"); + incomplete.presence = None; + m.add_node(1, incomplete, "b1"); + let mut per = BTreeMap::new(); + per.insert(1u8, live(12.0, 0.2, 0.3, 0.95)); + let s = m.infer(&per); + assert!(!s.stale); + assert!(s.presence.is_none()); + assert!(s.breathing.is_none()); + assert!(s.heartbeat.is_none()); + assert!(s.posture.is_none()); } #[test] diff --git a/v2/crates/wifi-densepose-calibration/src/runtime.rs b/v2/crates/wifi-densepose-calibration/src/runtime.rs index 6b4b25fec5..2bebb0fe3e 100644 --- a/v2/crates/wifi-densepose-calibration/src/runtime.rs +++ b/v2/crates/wifi-densepose-calibration/src/runtime.rs @@ -6,7 +6,7 @@ //! suppresses positive vitals/posture rather than propagating a hallucination; //! - **presence = absent** short-circuits breathing/heartbeat/posture to `None` //! (you cannot have a respiration rate in an empty room); -//! - a **STALE** bank (baseline drift) flags every reading. +//! - a **STALE** bank (baseline drift) abstains instead of emitting readings. use serde::{Deserialize, Serialize}; @@ -63,6 +63,9 @@ impl MixtureOfSpecialists { stale: self.bank.is_stale(current_baseline_id), ..Default::default() }; + if state.stale { + return state; + } // Anomaly first — it can veto everything else. state.anomaly = self.bank.anomaly.as_ref().and_then(|a| a.infer(f)); @@ -79,8 +82,8 @@ impl MixtureOfSpecialists { .presence .as_ref() .map(|r| r.value > 0.5) - // No presence specialist → assume present so vitals still run. - .unwrap_or(true); + // An incomplete bank has no authority to unlock dependent output. + .unwrap_or(false); // Restlessness is reported regardless of presence (movement implies presence). state.restlessness = self.bank.restlessness.as_ref().and_then(|r| r.infer(f)); @@ -174,5 +177,36 @@ mod tests { let mix = MixtureOfSpecialists::new(bank()); let s = mix.infer(&live(10.0, 0.2, 0.3, 0.9), "base-2"); assert!(s.stale); + assert!(s.presence.is_none(), "stale presence must abstain"); + assert!(s.breathing.is_none(), "stale vitals must abstain"); + assert!(s.posture.is_none(), "stale posture must abstain"); + assert!(s.heartbeat.is_none()); + assert!(s.restlessness.is_none()); + assert!(s.anomaly.is_none()); + assert!(!s.vetoed, "abstention is not an observed anomaly veto"); + let json = serde_json::to_value(&s).unwrap(); + assert!(json["presence"].is_null(), "unknown must not serialize as absent"); + assert_eq!(json["stale"], true); + } + + #[test] + fn missing_presence_specialist_abstains_and_suppresses_vitals() { + let mut incomplete = bank(); + incomplete.presence = None; + let mix = MixtureOfSpecialists::new(incomplete); + + let mut features = live(10.0, 0.2, 0.3, 0.9); + features.heart_score = 0.95; + features.heart_hz = 1.2; + let s = mix.infer(&features, "base-1"); + + assert!(!s.stale); + assert!(s.anomaly.is_some(), "unaffected specialist still executes"); + assert!(s.restlessness.is_some(), "independent movement evidence is retained"); + assert!(serde_json::to_value(&s).unwrap()["presence"].is_null()); + assert!(s.presence.is_none()); + assert!(s.breathing.is_none()); + assert!(s.heartbeat.is_none()); + assert!(s.posture.is_none()); } } From b8af1dc360cf74ee9ec78acfb806280833900e49 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Mon, 9 Nov 2026 16:53:30 +0100 Subject: [PATCH 091/100] fix(sensing): measure DSP cadence from accepted frame history --- .../wifi-densepose-sensing-server/src/main.rs | 169 +++++++++++++++++- 1 file changed, 168 insertions(+), 1 deletion(-) diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index d35f8830fe..9a34410c5c 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -1172,6 +1172,13 @@ struct NodeState { csi_fps_ema: f64, /// Number of inter-frame deltas observed (need ≥5 before trusting EMA). csi_fps_samples: u32, + /// Start of the current cadence window for frames admitted to the active + /// feature history. This is separate from raw-arrival FPS because + /// grid-rejected packets must not affect DSP frequency bins. + accepted_csi_rate_window_started_at: Option, + accepted_csi_rate_window_frames: u32, + accepted_csi_rate_hz: f64, + accepted_csi_rate_windows: u32, /// Latest extracted features for cross-node fusion. latest_features: Option, // ── RuVector Phase 2: Temporal smoothing & coherence gating ── @@ -1372,7 +1379,158 @@ mod fps_ema_tests { } } +#[cfg(test)] +mod accepted_history_cadence_tests { + use super::{NodeState, GRID_VOTE_FRAMES, MAX_PHYSICAL_CSI_FPS}; + use std::time::{Duration, Instant}; + use wifi_densepose_hardware::PpduType; + + fn warm(node: &mut NodeState, start: Instant, period_ms: u64) -> Instant { + let count = 1000 / period_ms; + for i in 0..=count { + node.observe_accepted_csi_frame(i as u32, false, start + Duration::from_millis(i * period_ms)); + } + start + Duration::from_millis(count * period_ms) + } + + #[test] + fn rejected_grid_arrivals_do_not_clock_accepted_feature_history() { + let mut node = NodeState::new(); + let start = Instant::now(); + let active = (256, PpduType::HtLegacy); + let rejected = (64, PpduType::HtLegacy); + for _ in 0..GRID_VOTE_FRAMES { assert!(node.accept_grid(active)); } + for i in 0..=400 { + let now = start + Duration::from_millis(i * 25); + if i % 2 == 0 { + assert!(node.accept_grid(active)); + node.observe_accepted_csi_frame(i as u32, true, now); + } else { + assert!(!node.accept_grid(rejected)); + node.observe_csi_frame_arrival(now); + } + } + let now = start + Duration::from_secs(10); + assert!((node.measured_feature_sample_rate_hz(now).unwrap() - 20.0).abs() < 0.01); + assert!((node.measured_sample_rate_hz().unwrap() - 40.0).abs() < 2.0); + assert_eq!(node.latest_csi_sequence, Some(400)); + assert!(node.latest_csi_sync_valid); + assert_eq!(node.last_frame_time, Some(now)); + assert_eq!(node.latest_accepted_csi_at, Some(now)); + } + + #[test] + fn accepted_bursts_count_every_frame_and_keep_raw_clock() { + let mut node = NodeState::new(); + let start = Instant::now(); + for i in 0..=40 { + let at = start + Duration::from_millis(i * 50); + node.observe_accepted_csi_frame((i * 2) as u32, false, at); + node.observe_accepted_csi_frame((i * 2 + 1) as u32, false, at + Duration::from_micros(40)); + } + let now = start + Duration::from_secs(2) + Duration::from_micros(40); + assert!((node.measured_feature_sample_rate_hz(now).unwrap() - 40.0).abs() < 0.01); + assert!((node.measured_sample_rate_hz().unwrap() - 40.0).abs() < 3.0); + } + + #[test] + fn warmed_measurement_expires_then_restarts_without_stale_rate() { + let mut node = NodeState::new(); + let start = Instant::now(); + assert!(node.measured_feature_sample_rate_hz(start).is_none()); + let end = warm(&mut node, start, 25); + assert_eq!(node.measured_feature_sample_rate_hz(end), Some(40.0)); + let later = end + Duration::from_secs(4); + assert!(node.measured_feature_sample_rate_hz(later).is_none()); + assert_eq!(node.feature_sample_rate_hz(later), 20.0); + node.observe_accepted_csi_frame(99, false, later); + assert!(node.measured_feature_sample_rate_hz(later).is_none()); + assert_eq!(node.accepted_csi_rate_windows, 0); + let end = warm(&mut node, later + Duration::from_millis(50), 50); + assert_eq!(node.measured_feature_sample_rate_hz(end), Some(20.0)); + } + + #[test] + fn grid_switch_resets_only_accepted_clock() { + let mut node = NodeState::new(); + let active = (128, PpduType::HtLegacy); + for _ in 0..GRID_VOTE_FRAMES { assert!(node.accept_grid(active)); } + let end = warm(&mut node, Instant::now(), 25); + let raw_before = (node.csi_fps_ema, node.csi_fps_samples, node.last_frame_time); + assert!(!node.accept_grid((64, PpduType::HtLegacy))); + assert_eq!(node.measured_feature_sample_rate_hz(end), Some(40.0)); + for _ in 0..GRID_VOTE_FRAMES { node.accept_grid((64, PpduType::HtLegacy)); } + assert_eq!(node.active_grid, Some((64, PpduType::HtLegacy))); + assert!(node.measured_feature_sample_rate_hz(end).is_none()); + assert_eq!(raw_before, (node.csi_fps_ema, node.csi_fps_samples, node.last_frame_time)); + } + + #[test] + fn per_node_measurements_and_existing_consumption_bounds_are_preserved() { + let mut first = NodeState::new(); + let mut second = NodeState::new(); + let start = Instant::now(); + let end = warm(&mut first, start, 25); + warm(&mut second, start, 50); + assert_eq!(first.measured_feature_sample_rate_hz(end), Some(40.0)); + assert_eq!(second.measured_feature_sample_rate_hz(end), Some(20.0)); + first.accepted_csi_rate_hz = f64::NAN; + assert!(first.measured_feature_sample_rate_hz(end).is_none()); + first.accepted_csi_rate_hz = 9000.0; + assert_eq!(first.measured_feature_sample_rate_hz(end), Some(MAX_PHYSICAL_CSI_FPS)); + } +} + impl NodeState { + /// Track only frames admitted to this node's active-grid feature history. + /// One-to-three-second monotonic windows retain valid burst deliveries and + /// exclude rejected grids. Longer gaps restart warm-up instead of turning + /// a transport outage into a bogus low sample rate. + fn observe_accepted_feature_frame(&mut self, now: std::time::Instant) { + let Some(started_at) = self.accepted_csi_rate_window_started_at else { + self.accepted_csi_rate_window_started_at = Some(now); + self.accepted_csi_rate_window_frames = 1; + return; + }; + + let elapsed = now.saturating_duration_since(started_at); + if elapsed > std::time::Duration::from_secs(3) { + self.accepted_csi_rate_window_started_at = Some(now); + self.accepted_csi_rate_window_frames = 1; + self.accepted_csi_rate_hz = 20.0; + self.accepted_csi_rate_windows = 0; + return; + } + + self.accepted_csi_rate_window_frames = + self.accepted_csi_rate_window_frames.saturating_add(1); + if elapsed >= std::time::Duration::from_secs(1) { + let intervals = self.accepted_csi_rate_window_frames.saturating_sub(1); + let measured = intervals as f64 / elapsed.as_secs_f64(); + self.accepted_csi_rate_hz = measured; + self.accepted_csi_rate_windows = self.accepted_csi_rate_windows.saturating_add(1); + self.accepted_csi_rate_window_started_at = Some(now); + self.accepted_csi_rate_window_frames = 1; + } + } + + /// Accepted-history cadence is unknown before warm-up and after a stall. + /// Keep this optional: the DSP fallback is not an observed measurement. + fn measured_feature_sample_rate_hz(&self, now: std::time::Instant) -> Option { + let age = now.checked_duration_since(self.latest_accepted_csi_at?)?; + if age > std::time::Duration::from_secs(3) + || self.accepted_csi_rate_windows == 0 + || !self.accepted_csi_rate_hz.is_finite() + { + return None; + } + Some(self.accepted_csi_rate_hz.clamp(1.0, MAX_PHYSICAL_CSI_FPS)) + } + + fn feature_sample_rate_hz(&self, now: std::time::Instant) -> f64 { + self.measured_feature_sample_rate_hz(now).unwrap_or(20.0) + } + /// Measured sample rate only after enough inter-frame deltas have warmed /// the EMA. The physical ceiling is enforced again at the consumption /// boundary so restored or malformed state cannot overclock DSP math. @@ -1524,6 +1682,7 @@ impl NodeState { sync_valid: bool, now: std::time::Instant, ) -> bool { + self.observe_accepted_feature_frame(now); self.latest_accepted_csi_at = Some(now); self.latest_csi_sequence = Some(sequence); self.latest_csi_sync_valid = sync_valid; @@ -1560,6 +1719,10 @@ impl NodeState { latest_csi_sync_valid: false, csi_fps_ema: 20.0, csi_fps_samples: 0, + accepted_csi_rate_window_started_at: None, + accepted_csi_rate_window_frames: 0, + accepted_csi_rate_hz: 20.0, + accepted_csi_rate_windows: 0, latest_features: None, prev_keypoints: None, motion_energy_history: VecDeque::with_capacity(COHERENCE_WINDOW), @@ -1745,6 +1908,10 @@ impl NodeState { if 2 * votes(grid) > 3 * votes(active) { self.active_grid = Some(grid); self.frame_history.clear(); + self.accepted_csi_rate_window_started_at = None; + self.accepted_csi_rate_window_frames = 0; + self.accepted_csi_rate_hz = 20.0; + self.accepted_csi_rate_windows = 0; self.baseline_motion = 0.0; self.baseline_frames = 0; return true; @@ -12950,7 +13117,7 @@ async fn udp_receiver_task( ns.frame_history.pop_front(); } - let sample_rate_hz = ns.effective_sample_rate_hz(); + let sample_rate_hz = ns.feature_sample_rate_hz(std::time::Instant::now()); let ( features, mut classification, From 1008cc0d29ed14591c5dba3ad48132be578334e7 Mon Sep 17 00:00:00 2001 From: ruv Date: Mon, 9 Nov 2026 12:11:55 -0500 Subject: [PATCH 092/100] feat(firmware): support XIAO MR60BHA2 carrier --- firmware/esp32-csi-node/README.md | 19 ++++++++++++++++ .../esp32-csi-node/main/Kconfig.projbuild | 10 +++++++++ firmware/esp32-csi-node/main/mmwave_pins.h | 16 ++++++++++++++ firmware/esp32-csi-node/main/mmwave_sensor.c | 22 ++++++++++++++----- .../sdkconfig.defaults.xiao-mr60bha2 | 6 +++++ firmware/esp32-csi-node/test/Makefile | 3 ++- .../esp32-csi-node/test/test_mmwave_pins.c | 13 +++++++++++ 7 files changed, 83 insertions(+), 6 deletions(-) create mode 100644 firmware/esp32-csi-node/sdkconfig.defaults.xiao-mr60bha2 diff --git a/firmware/esp32-csi-node/README.md b/firmware/esp32-csi-node/README.md index d78fd4b5aa..81a359be56 100644 --- a/firmware/esp32-csi-node/README.md +++ b/firmware/esp32-csi-node/README.md @@ -304,6 +304,25 @@ The boot log must report the intended `internal ceramic` or `external U.FL` path before that measurement is labelled. The standard C6 build keeps the board default and emits no antenna-path claim. +#### Seeed XIAO ESP32-C6 MR60BHA2 carrier + +The assembled Seeed MR60BHA2 kit routes the radar UART to XIAO TX GPIO16 and +RX GPIO17. It also uses native USB Serial/JTAG for logs and onboarding, so the +radar UART does not overlap the console. Generic C6 boards keep the GPIO4 and +GPIO5 defaults. + +Build the dedicated carrier profile instead of flashing the generic C6 image: + +```bash +idf.py -D SDKCONFIG_DEFAULTS="sdkconfig.defaults;sdkconfig.defaults.esp32c6;sdkconfig.defaults.xiao-mr60bha2" \ + set-target esp32c6 +idf.py build +``` + +The boot log must identify `MR60BHA2` and the server must receive fresh +`edge_fused_vitals` packets before the radar is considered connected. Radar +reported heart and breathing values remain unvalidated, nonmedical evidence. + ### 3. Provision WiFi credentials (no reflash needed) ```bash diff --git a/firmware/esp32-csi-node/main/Kconfig.projbuild b/firmware/esp32-csi-node/main/Kconfig.projbuild index aed8bcc032..a9cb3c9c16 100644 --- a/firmware/esp32-csi-node/main/Kconfig.projbuild +++ b/firmware/esp32-csi-node/main/Kconfig.projbuild @@ -19,6 +19,16 @@ menu "CSI Node Configuration" external antenna. Leave this disabled for generic ESP32-C6 boards because GPIO3 and GPIO14 may have unrelated functions there. + config C6_XIAO_MR60BHA2 + bool "Use the Seeed XIAO ESP32-C6 MR60BHA2 carrier UART" + depends on IDF_TARGET_ESP32C6 + default n + help + Enable only for the assembled Seeed MR60BHA2 kit with a XIAO + ESP32-C6. The carrier routes the radar UART to XIAO TX GPIO16 and + RX GPIO17. The standard C6 profile keeps the generic GPIO4/GPIO5 + assignment so existing DevKit wiring is not changed. + choice C6_XIAO_ANTENNA_PATH prompt "Seeed XIAO ESP32-C6 antenna path" depends on C6_XIAO_ANTENNA_SELECT diff --git a/firmware/esp32-csi-node/main/mmwave_pins.h b/firmware/esp32-csi-node/main/mmwave_pins.h index 634265e92a..4605baf115 100644 --- a/firmware/esp32-csi-node/main/mmwave_pins.h +++ b/firmware/esp32-csi-node/main/mmwave_pins.h @@ -1,5 +1,21 @@ #pragma once #include +#include + +static inline void ruview_c6_mmwave_default_pins(bool xiao_mr60bha2, + int *tx, int *rx) +{ + if (tx != NULL) *tx = xiao_mr60bha2 ? 16 : 4; + if (rx != NULL) *rx = xiao_mr60bha2 ? 17 : 5; +} + +static inline bool ruview_mmwave_pins_overlap_console(int tx, int rx, + int console_tx, + int console_rx) +{ + return (console_tx >= 0 && (tx == console_tx || rx == console_tx)) + || (console_rx >= 0 && (tx == console_rx || rx == console_rx)); +} /* C5 SPI0/1 uses GPIO16..22. Never route a sensor onto the memory bus. * No generic C5 sensor pin assignment is safe without board wiring. */ diff --git a/firmware/esp32-csi-node/main/mmwave_sensor.c b/firmware/esp32-csi-node/main/mmwave_sensor.c index d63f3044ce..0742537664 100644 --- a/firmware/esp32-csi-node/main/mmwave_sensor.c +++ b/firmware/esp32-csi-node/main/mmwave_sensor.c @@ -517,13 +517,25 @@ esp_err_t mmwave_sensor_init(int uart_tx_pin, int uart_rx_pin) return ESP_ERR_NOT_SUPPORTED; } #elif defined(CONFIG_IDF_TARGET_ESP32C6) - if (uart_tx_pin < 0) uart_tx_pin = 4; - if (uart_rx_pin < 0) uart_rx_pin = 5; + int default_tx = 4, default_rx = 5; + ruview_c6_mmwave_default_pins( +#if defined(CONFIG_C6_XIAO_MR60BHA2) + true, +#else + false, +#endif + &default_tx, &default_rx); + if (uart_tx_pin < 0) uart_tx_pin = default_tx; + if (uart_rx_pin < 0) uart_rx_pin = default_rx; #else if (uart_tx_pin < 0) uart_tx_pin = 17; if (uart_rx_pin < 0) uart_rx_pin = 18; #endif -#if defined(CONFIG_ESP_CONSOLE_UART_CUSTOM) +#if defined(CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG) || defined(CONFIG_ESP_CONSOLE_USB_CDC) + /* Native USB has no GPIO console pins. This is the standard C6 profile + * and leaves the XIAO carrier's GPIO16/GPIO17 hardware UART available. */ + const int console_tx = -1, console_rx = -1; +#elif defined(CONFIG_ESP_CONSOLE_UART_CUSTOM) const int console_tx = CONFIG_ESP_CONSOLE_UART_TX_GPIO; const int console_rx = CONFIG_ESP_CONSOLE_UART_RX_GPIO; #elif defined(CONFIG_IDF_TARGET_ESP32C6) @@ -542,8 +554,8 @@ esp_err_t mmwave_sensor_init(int uart_tx_pin, int uart_rx_pin) return ESP_ERR_INVALID_ARG; } #endif - if (uart_tx_pin == console_tx || uart_tx_pin == console_rx || - uart_rx_pin == console_tx || uart_rx_pin == console_rx) { + if (ruview_mmwave_pins_overlap_console(uart_tx_pin, uart_rx_pin, + console_tx, console_rx)) { ESP_LOGW(TAG, "mmWave probe pins (TX=%d, RX=%d) overlap the console UART (TX=%d, RX=%d); skipping probe", uart_tx_pin, uart_rx_pin, console_tx, console_rx); return ESP_ERR_INVALID_ARG; diff --git a/firmware/esp32-csi-node/sdkconfig.defaults.xiao-mr60bha2 b/firmware/esp32-csi-node/sdkconfig.defaults.xiao-mr60bha2 new file mode 100644 index 0000000000..5a96434b41 --- /dev/null +++ b/firmware/esp32-csi-node/sdkconfig.defaults.xiao-mr60bha2 @@ -0,0 +1,6 @@ +# Seeed XIAO ESP32-C6 on the assembled MR60BHA2 carrier. +# The carrier uses UART TX GPIO16 and RX GPIO17. Native USB Serial/JTAG remains +# the onboarding console. The photographed kit uses its external U.FL antenna. +CONFIG_C6_XIAO_MR60BHA2=y +CONFIG_C6_XIAO_ANTENNA_SELECT=y +CONFIG_C6_XIAO_ANTENNA_EXTERNAL=y diff --git a/firmware/esp32-csi-node/test/Makefile b/firmware/esp32-csi-node/test/Makefile index 514fedf4e3..b674a4173d 100644 --- a/firmware/esp32-csi-node/test/Makefile +++ b/firmware/esp32-csi-node/test/Makefile @@ -58,11 +58,12 @@ FUZZ_JOBS ?= 1 test_thermal run_thermal test_csi_sanitize run_csi_sanitize \ test_c6_antenna run_c6_antenna test_serial_onboarding run_serial_onboarding \ test_config_rules run_config_rules \ + test_mmwave_pins run_mmwave_pins \ run_delivery_contract test_ota_health run_ota_health run_onboarding_identity run_onboarding_console all: fuzz_serialize fuzz_edge fuzz_nvs test_adr110 test_vitals test_mmwave_detect \ test_thermal test_csi_sanitize test_c6_antenna test_serial_onboarding \ - test_ota_health test_config_rules + test_ota_health test_config_rules test_mmwave_pins # --- XIAO ESP32-C6 RF switch selection --- # Host-side truth table for the GPIO14 antenna selector. GPIO3 is always driven diff --git a/firmware/esp32-csi-node/test/test_mmwave_pins.c b/firmware/esp32-csi-node/test/test_mmwave_pins.c index 92c8ce4b49..87948845f7 100644 --- a/firmware/esp32-csi-node/test/test_mmwave_pins.c +++ b/firmware/esp32-csi-node/test/test_mmwave_pins.c @@ -3,6 +3,19 @@ int main(void) { + int tx = -1, rx = -1; + ruview_c6_mmwave_default_pins(false, &tx, &rx); + assert(tx == 4 && rx == 5); + ruview_c6_mmwave_default_pins(true, &tx, &rx); + assert(tx == 16 && rx == 17); + + /* Native USB has no GPIO console assignment, so the assembled XIAO + * carrier UART remains usable. A UART console on those pins still fails + * closed. */ + assert(!ruview_mmwave_pins_overlap_console(16, 17, -1, -1)); + assert(ruview_mmwave_pins_overlap_console(16, 17, 16, 17)); + assert(ruview_mmwave_pins_overlap_console(4, 5, 5, 6)); + assert(!ruview_c5_mmwave_pins_safe(-1, -1, 11, 12)); assert(!ruview_c5_mmwave_pins_safe(17, 18, 11, 12)); for (int pin = 16; pin <= 22; pin++) { From f817562b7c68939dbd11eb556d726db617701049 Mon Sep 17 00:00:00 2001 From: ruv Date: Mon, 9 Nov 2026 13:01:37 -0500 Subject: [PATCH 093/100] fix(radar): publish fail closed MR60 presence --- firmware/esp32-csi-node/main/mmwave_detect.h | 8 +++ firmware/esp32-csi-node/main/mmwave_sensor.c | 10 ++++ .../esp32-csi-node/test/test_mmwave_detect.c | 2 + .../wifi-densepose-sensing-server/src/main.rs | 54 +++++++++++-------- 4 files changed, 52 insertions(+), 22 deletions(-) diff --git a/firmware/esp32-csi-node/main/mmwave_detect.h b/firmware/esp32-csi-node/main/mmwave_detect.h index 9fd9ade36d..f666497a97 100644 --- a/firmware/esp32-csi-node/main/mmwave_detect.h +++ b/firmware/esp32-csi-node/main/mmwave_detect.h @@ -25,6 +25,14 @@ static inline bool mmwave_mr60_identifying_type(uint16_t type) return type == 0x0A15 || type == 0x0A16 || type == 0x0F09; } +/** MR60BHA2 vital-sign firmware exposes one authoritative presence bit, not a + * dependable people-count frame. Map that bit to the single-target contract + * used by RuView's fused-vitals packet. */ +static inline uint8_t mmwave_mr60_target_count(bool person_present) +{ + return person_present ? 1u : 0u; +} + /** * True iff buf[i..] begins a *validated* LD2410 report frame within [0,len): * F4 F3 F2 F1 | len(LE,2) | data[len] | F8 F7 F6 F5 diff --git a/firmware/esp32-csi-node/main/mmwave_sensor.c b/firmware/esp32-csi-node/main/mmwave_sensor.c index 0742537664..97713e8911 100644 --- a/firmware/esp32-csi-node/main/mmwave_sensor.c +++ b/firmware/esp32-csi-node/main/mmwave_sensor.c @@ -147,6 +147,16 @@ static void mr60_process_frame(uint16_t type, const uint8_t *data, uint16_t len) case MR60_TYPE_PRESENCE: if (len >= 1) { s_state.person_present = (data[0] != 0); + /* MR60BHA2 exposes a single-target vital-sign presence flag rather + * than a reliable people-count frame. Keep the public count in + * lockstep with that authoritative flag so downstream clients do + * not report zero targets while fresh radar vitals are arriving. */ + s_state.target_count = mmwave_mr60_target_count(s_state.person_present); + if (!s_state.person_present) { + s_state.heart_rate_bpm = 0.0f; + s_state.breathing_rate = 0.0f; + s_state.distance_cm = 0.0f; + } } break; diff --git a/firmware/esp32-csi-node/test/test_mmwave_detect.c b/firmware/esp32-csi-node/test/test_mmwave_detect.c index 3646e530f9..bd3a993405 100644 --- a/firmware/esp32-csi-node/test/test_mmwave_detect.c +++ b/firmware/esp32-csi-node/test/test_mmwave_detect.c @@ -46,6 +46,8 @@ int main(void) CHECK(mmwave_mr60_identifying_type(0x0A15), "MR60BHA2 heart-rate type identifies sensor"); CHECK(mmwave_mr60_identifying_type(0x0A16), "MR60BHA2 distance type identifies sensor"); CHECK(mmwave_mr60_identifying_type(0x0F09), "MR60BHA2 presence type identifies sensor"); + CHECK(mmwave_mr60_target_count(false) == 0, "MR60BHA2 absence maps to zero targets"); + CHECK(mmwave_mr60_target_count(true) == 1, "MR60BHA2 presence maps to one target"); /* 1. A real basic-report frame (data len 13) validates. */ int n = make_frame(buf, 13); diff --git a/v2/crates/wifi-densepose-sensing-server/src/main.rs b/v2/crates/wifi-densepose-sensing-server/src/main.rs index eedee37139..d9cf3a0f11 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/main.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/main.rs @@ -4121,6 +4121,30 @@ fn parse_edge_fused_vitals(buf: &[u8]) -> Option { }) } +fn edge_fused_vitals_message(fused: &EdgeFusedVitalsPacket) -> serde_json::Value { + serde_json::json!({ + "type": "edge_fused_vitals", + "node_id": fused.node_id, + "breathing_rate_bpm": fused.breathing_rate_bpm, + "heartrate_bpm": fused.heartrate_bpm, + "n_persons": fused.n_persons, + "person_count_valid": fused.person_count_valid, + "fusion_confidence": fused.fusion_confidence, + "mmwave": { + "present": fused.flags & 0x08 != 0, + "hr_bpm": fused.mmwave_hr_bpm, + "br_bpm": fused.mmwave_br_bpm, + "distance_cm": fused.mmwave_distance_cm, + "targets": fused.mmwave_targets, + "confidence": fused.mmwave_confidence, + "type": fused.mmwave_type, + }, + "motion_energy": fused.motion_energy, + "presence_score": fused.presence_score, + "timestamp_ms": fused.timestamp_ms, + }) +} + #[cfg(test)] mod issue_928_magic_collision_tests { //! Issue #928 — `0xC511_0004` was being parsed as WASM output, eating the @@ -4140,7 +4164,7 @@ mod issue_928_magic_collision_tests { buf[8..12].copy_from_slice(&720_000u32.to_le_bytes()); // heartrate 72.0 BPM buf[12] = (-55i8) as u8; // rssi buf[13] = 1; // n_persons - buf[14] = 2; // mmwave_type + buf[14] = 1; // mmwave_type (MR60BHA2) buf[15] = 85; // fusion_confidence buf[16..20].copy_from_slice(&0.42f32.to_le_bytes()); // motion_energy buf[20..24].copy_from_slice(&0.95f32.to_le_bytes()); // presence_score @@ -4165,7 +4189,7 @@ mod issue_928_magic_collision_tests { assert_eq!(pkt.rssi, -55); assert_eq!(pkt.n_persons, 1); assert!(pkt.person_count_valid); - assert_eq!(pkt.mmwave_type, 2); + assert_eq!(pkt.mmwave_type, 1); assert_eq!(pkt.fusion_confidence, 85); assert!((pkt.motion_energy - 0.42).abs() < 1e-6); assert!((pkt.presence_score - 0.95).abs() < 1e-6); @@ -4175,6 +4199,11 @@ mod issue_928_magic_collision_tests { assert!((pkt.mmwave_distance_cm - 182.0).abs() < 1e-6); assert_eq!(pkt.mmwave_targets, 1); assert_eq!(pkt.mmwave_confidence, 90); + + let message = edge_fused_vitals_message(&pkt); + assert_eq!(message["mmwave"]["present"], true); + assert_eq!(message["mmwave"]["targets"], 1); + assert_eq!(message["mmwave"]["type"], 1); } #[test] @@ -13987,26 +14016,7 @@ async fn udp_receiver_task( fused.mmwave_targets, fused.fusion_confidence, ); let s = state.write().await; - if let Ok(json) = serde_json::to_string(&serde_json::json!({ - "type": "edge_fused_vitals", - "node_id": fused.node_id, - "breathing_rate_bpm": fused.breathing_rate_bpm, - "heartrate_bpm": fused.heartrate_bpm, - "n_persons": fused.n_persons, - "person_count_valid": fused.person_count_valid, - "fusion_confidence": fused.fusion_confidence, - "mmwave": { - "hr_bpm": fused.mmwave_hr_bpm, - "br_bpm": fused.mmwave_br_bpm, - "distance_cm": fused.mmwave_distance_cm, - "targets": fused.mmwave_targets, - "confidence": fused.mmwave_confidence, - "type": fused.mmwave_type, - }, - "motion_energy": fused.motion_energy, - "presence_score": fused.presence_score, - "timestamp_ms": fused.timestamp_ms, - })) { + if let Ok(json) = serde_json::to_string(&edge_fused_vitals_message(&fused)) { let _ = s.tx.send(json); } continue; From 3680487e76c7ac1b493f0ff4d4fec6b1a33d2a5a Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Tue, 10 Nov 2026 09:58:30 +0100 Subject: [PATCH 094/100] Retain existing QEMU runner repair from PR2510 --- .github/workflows/firmware-qemu.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/firmware-qemu.yml b/.github/workflows/firmware-qemu.yml index 1c295760c4..015304a5f4 100644 --- a/.github/workflows/firmware-qemu.yml +++ b/.github/workflows/firmware-qemu.yml @@ -30,17 +30,18 @@ env: jobs: build-qemu: name: Build Espressif QEMU - runs-on: ubuntu-latest + # QEMU's bundled dtc fails with GCC 15 on Ubuntu 26.04. Keep the + # Ubuntu 24.04 / GCC 13 build environment until that dependency is updated. + runs-on: ubuntu-24.04 steps: - name: Cache QEMU build id: cache-qemu uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 with: path: /opt/qemu-esp32 - # Include date component so cache refreshes monthly when branch updates - key: qemu-esp32s3-${{ env.QEMU_BRANCH }}-v5 - restore-keys: | - qemu-esp32s3-${{ env.QEMU_BRANCH }}- + # Scope native binaries to the build image; bump v1 when refreshing + # the floating upstream branch. Do not restore an unqualified cache. + key: qemu-esp32s3-ubuntu-24.04-${{ env.QEMU_BRANCH }}-v1 - name: Install QEMU build dependencies if: steps.cache-qemu.outputs.cache-hit != 'true' From b8b55643aa590a8f47adc2d01ea696e475b3fe3b Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Tue, 10 Nov 2026 13:50:02 +0100 Subject: [PATCH 095/100] perf(learner): prepare immutable training inputs once per fit --- .../src/room_learner/learning_tests.rs | 77 ++++++++++++++++ .../src/room_learner/optimization.rs | 4 +- .../src/room_learner/training.rs | 90 +++++++++++++++++-- 3 files changed, 163 insertions(+), 8 deletions(-) diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/learning_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/learning_tests.rs index 03415db175..7bde77a21f 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/learning_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/learning_tests.rs @@ -124,6 +124,83 @@ fn config() -> FitConfig { } } +#[test] +fn prepared_training_rows_match_uncached_representation_for_parameter_probes() { + use super::{represent_row, Row}; + for width in [8, 56, 128] { + let ds = dataset(width); + let (model, _) = RoomModel::fit( + &ds, + FitConfig { + epochs: 1, + hidden_dim: if width == 128 { 2 } else { 3 }, + embedding_dim: if width == 128 { 2 } else { 3 }, + ..config() + }, + ) + .unwrap(); + let parameters = model.parameters(); + for cohort in ds.train() { + let tokens = model.tokens(cohort, false).unwrap(); + let prepared = Row::prepare(&model, tokens.clone(), 0.0, true).unwrap(); + // Include the unchanged parameters and every positive/negative + // finite-difference probe, across backbone, projectors and head. + for probe in 0..=parameters.len() * 2 { + let mut params = parameters.clone(); + if probe > 0 { + let index = (probe - 1) / 2; + params[index] += if probe % 2 == 1 { 0.002 } else { -0.002 }; + } + let expected = model.represent(&tokens, ¶ms).unwrap(); + let mut encoder = extractor(&model.config); + encoder + .transformer + .unflatten_weights(¶ms[..model.encoder_weights.len()]) + .unwrap(); + let projectors = model.source_projectors(¶ms).unwrap(); + let actual = represent_row(&model, &prepared, &mut encoder, &projectors).unwrap(); + let bits = |values: &[f32]| values.iter().map(|x| x.to_bits()).collect::>(); + assert_eq!( + bits(&actual.0), + bits(&expected.0), + "width={width} probe={probe}" + ); + assert_eq!( + actual.1.iter().map(|x| bits(x)).collect::>(), + expected.1.iter().map(|x| bits(x)).collect::>(), + "width={width} probe={probe}" + ); + } + } + } +} + +#[test] +fn prepared_training_rows_reject_wrong_source_time_width_and_nonfinite_values() { + use super::Row; + let ds = dataset(8); + let (model, _) = RoomModel::fit( + &ds, + FitConfig { + epochs: 1, + ..config() + }, + ) + .unwrap(); + let valid = model.tokens(&ds.train()[0], false).unwrap(); + let mut wrong_source = valid.clone(); + wrong_source.pop(); + let mut wrong_time = valid.clone(); + wrong_time[0].pop(); + let mut wrong_width = valid.clone(); + wrong_width[0][0].pop(); + let mut nonfinite = valid.clone(); + nonfinite[0][0][0] = f32::NAN; + for invalid in [wrong_source, wrong_time, wrong_width, nonfinite] { + assert!(Row::prepare(&model, invalid, 0.0, true).is_err()); + } +} + #[test] fn actual_encoder_and_head_weights_reduce_declared_objective_and_learn_controls() { let ds = dataset(8); diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/optimization.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/optimization.rs index c27fa78868..02416ba602 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/optimization.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/optimization.rs @@ -1,6 +1,6 @@ //! Bounded RF descent followed by exact linear-task block descent. //! Both blocks minimize the same declared joint objective; no validation tuning. -use super::{objective, FitConfig, ObjectiveValues, Row}; +use super::{objective, represent_row, FitConfig, ObjectiveValues, Row}; use crate::{ room_learner::model::{extractor, sigmoid, RoomModel}, trainer::{clip_gradients, estimate_gradient, SgdOptimizer}, @@ -24,7 +24,7 @@ pub(super) fn head_step( let projectors = model.source_projectors(params)?; for row in rows { check()?; - let (features, _) = model.represent_with_encoder(&row.tokens, &mut encoder, &projectors)?; + let (features, _) = represent_row(model, row, &mut encoder, &projectors)?; let residual = (sigmoid(model.logit(&features, ¶ms[offset..])) - row.occupied) * config.supervised_weight / rows.len() as f32; diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/training.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/training.rs index 1d38a6d22d..737d4ae721 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/training.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/training.rs @@ -47,10 +47,88 @@ pub struct TrainingReport { pub evidence: String, } struct Row { + // Train-only inputs are immutable for the complete fit. Conditioning and + // task skips do not depend on any optimized parameter, so compute them once + // rather than allocating/rescanning raw tokens for every numerical probe. tokens: Vec>>, + skips: Vec<[f32; 2]>, occupied: f32, contrastive: bool, } +impl Row { + fn prepare( + model: &RoomModel, + tokens: Vec>>, + occupied: f32, + contrastive: bool, + ) -> Result { + if tokens.len() != model.profiles.len() + || tokens.iter().any(|source| { + source.len() != model.config.temporal_tokens + || source.iter().any(|row| { + row.len() != model.config.bins + model.config.source_count + || row.iter().any(|value| !value.is_finite()) + }) + }) + { + return Err("invalid immutable training representation shape or value".into()); + } + let mut conditioned = Vec::with_capacity(tokens.len()); + let mut skips = Vec::with_capacity(tokens.len()); + for (index, source) in tokens.into_iter().enumerate() { + // Keep the exact operation/order used by model::represent_with_encoder. + let mean = source + .iter() + .flat_map(|row| row[..model.config.bins].iter()) + .sum::() + / (model.config.temporal_tokens * model.config.bins) as f32; + let temporal = source + .windows(2) + .flat_map(|w| { + w[0][..model.config.bins] + .iter() + .zip(&w[1][..model.config.bins]) + }) + .map(|(a, b)| (a - b).abs()) + .sum::() + / ((model.config.temporal_tokens - 1) * model.config.bins) as f32; + skips.push([mean, temporal]); + let mut source = source; + for row in &mut source { + for value in &mut row[..model.config.bins] { + *value /= model.profiles[index].max_normalized; + } + } + conditioned.push(source); + } + Ok(Self { + tokens: conditioned, + skips, + occupied, + contrastive, + }) + } +} +fn represent_row( + model: &RoomModel, + row: &Row, + encoder: &mut crate::embedding::EmbeddingExtractor, + projectors: &[crate::embedding::ProjectionHead], +) -> Result<(Vec, Vec>), String> { + let mut features = Vec::with_capacity(model.profiles.len() * (model.config.embedding_dim + 2)); + let mut embeddings = Vec::with_capacity(row.tokens.len()); + for (index, source) in row.tokens.iter().enumerate() { + encoder.projection = projectors[index].clone(); + let embedding = encoder.extract(source); + features.extend_from_slice(&embedding); + features.extend_from_slice(&row.skips[index]); + embeddings.push(embedding); + } + if !features.iter().all(|value| value.is_finite()) { + return Err("non-finite representation".into()); + } + Ok((features, embeddings)) +} fn occupied(label: RoomLabel) -> f32 { if matches!(label, RoomLabel::OccupiedStatic | RoomLabel::OccupiedMoving) { 1.0 @@ -86,8 +164,7 @@ fn objective( .transformer .unflatten_weights(¶ms[..model.encoder_weights.len()])?; for row in rows { - let (features, embeddings) = - model.represent_with_encoder(&row.tokens, &mut encoder, &projectors)?; + let (features, embeddings) = represent_row(model, row, &mut encoder, &projectors)?; supervised += bce(model.logit(&features, head), row.occupied); // One event per independent trial per batch. Sibling/overlapping windows // in the same trial cannot accidentally become contrastive negatives. @@ -270,11 +347,12 @@ impl RoomModel { if contrastive { independent.push(c); } - rows.push(Row { - tokens: model.tokens(c, false)?, - occupied: occupied(c.label), + rows.push(Row::prepare( + &model, + model.tokens(c, false)?, + occupied(c.label), contrastive, - }); + )?); } let contrastive_trials = independent.len(); if contrastive_trials < 2 { From dc0661f47b9ff5b537e9d13da7cec9e91dbb0aad Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Wed, 11 Nov 2026 15:30:49 +0100 Subject: [PATCH 096/100] fix(ci): run MQTT feature tests before broker integration --- .github/workflows/mqtt-integration.yml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/mqtt-integration.yml b/.github/workflows/mqtt-integration.yml index 3a488dbe72..518fea91bd 100644 --- a/.github/workflows/mqtt-integration.yml +++ b/.github/workflows/mqtt-integration.yml @@ -90,16 +90,15 @@ jobs: python -m pip install --quiet pyyaml python scripts/validate-ha-blueprints.py - - name: Verify unit tests still pass under --features mqtt + - name: Verify MQTT-feature unit tests working-directory: v2 - # `cargo test` accepts a single TESTNAME filter, so we run the - # whole --lib suite here. That gives us the full 410-test green - # bar under --features mqtt (which is more reassuring than - # filtering anyway). + # Keep this broker workflow focused on the feature it changes. The full + # library/workspace and temporal Release suites run in ci.yml; repeating + # long learner training here exhausted this job before broker tests ran. run: >- - cargo test -p wifi-densepose-sensing-server + cargo test --locked -p wifi-densepose-sensing-server --features mqtt --no-default-features - --lib + --lib mqtt:: --no-fail-fast - name: Run integration tests against mosquitto From dc6149f67cd1a0e191b7632d4c5573c02a62050c Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Wed, 11 Nov 2026 19:26:43 +0100 Subject: [PATCH 097/100] fix(ci): remove failing mirror bootstrap from server validation --- .github/workflows/bench-regression.yml | 7 +-- .github/workflows/ci.yml | 5 +- .github/workflows/firmware-qemu.yml | 75 +++++++++++--------------- .github/workflows/mqtt-integration.yml | 43 +++++++-------- scripts/ci-install-ubuntu-deps.sh | 52 ++++++++++++++++++ 5 files changed, 108 insertions(+), 74 deletions(-) create mode 100755 scripts/ci-install-ubuntu-deps.sh diff --git a/.github/workflows/bench-regression.yml b/.github/workflows/bench-regression.yml index 3ad5a6cb7c..6299f24a85 100644 --- a/.github/workflows/bench-regression.yml +++ b/.github/workflows/bench-regression.yml @@ -42,6 +42,7 @@ on: - 'v2/Cargo.toml' - 'v2/Cargo.lock' - '.github/workflows/bench-regression.yml' + - 'scripts/ci-install-ubuntu-deps.sh' pull_request: paths: - 'v2/crates/**/benches/**' @@ -50,6 +51,7 @@ on: - 'v2/Cargo.toml' - 'v2/Cargo.lock' - '.github/workflows/bench-regression.yml' + - 'scripts/ci-install-ubuntu-deps.sh' workflow_dispatch: permissions: @@ -68,7 +70,7 @@ jobs: # ── HARD GATE: every bench must still compile + link ───────────────────── bench-compile: name: bench compile-verify (--no-run) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout (recursive — wifi-densepose-rufield path-deps vendor/rufield) uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 @@ -84,8 +86,7 @@ jobs: # the whole graph, so these are required here too. - name: Install Tauri / GTK / serial system dev libraries run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ + bash scripts/ci-install-ubuntu-deps.sh \ libglib2.0-dev \ libgtk-3-dev \ libsoup-3.0-dev \ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e7a0a379e..752a166963 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -79,7 +79,7 @@ jobs: # Rust Workspace Tests rust-tests: name: Rust Workspace Tests - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 steps: - name: Checkout code uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 @@ -96,8 +96,7 @@ jobs: # standard Tauri-on-Ubuntu set. - name: Install Tauri / GTK / serial system dev libraries run: | - sudo apt-get update - sudo apt-get install -y --no-install-recommends \ + bash scripts/ci-install-ubuntu-deps.sh \ libglib2.0-dev \ libgtk-3-dev \ libsoup-3.0-dev \ diff --git a/.github/workflows/firmware-qemu.yml b/.github/workflows/firmware-qemu.yml index 015304a5f4..09d382b5f0 100644 --- a/.github/workflows/firmware-qemu.yml +++ b/.github/workflows/firmware-qemu.yml @@ -22,50 +22,32 @@ on: - 'scripts/swarm_presets/**' - '.github/workflows/firmware-qemu.yml' +defaults: + run: + shell: bash + env: IDF_VERSION: "v5.4" - QEMU_REPO: "https://github.com/espressif/qemu.git" - QEMU_BRANCH: "esp-develop" jobs: build-qemu: name: Build Espressif QEMU - # QEMU's bundled dtc fails with GCC 15 on Ubuntu 26.04. Keep the - # Ubuntu 24.04 / GCC 13 build environment until that dependency is updated. runs-on: ubuntu-24.04 + # This SDK image installs Espressif's checksum-verified QEMU package and + # its runtime libraries. Match the image used by both consumers below. + # Avoid APT DNS and an unpinned floating QEMU source build on cold caches. + container: + image: espressif/idf:v5.4@sha256:f1e9f69dc052b9afc7801ca884e0ef40c17e014bb05ce73d9c09d29290bd17fb steps: - - name: Cache QEMU build - id: cache-qemu - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 - with: - path: /opt/qemu-esp32 - # Scope native binaries to the build image; bump v1 when refreshing - # the floating upstream branch. Do not restore an unqualified cache. - key: qemu-esp32s3-ubuntu-24.04-${{ env.QEMU_BRANCH }}-v1 - - - name: Install QEMU build dependencies - if: steps.cache-qemu.outputs.cache-hit != 'true' - run: | - sudo apt-get update - sudo apt-get install -y \ - git build-essential ninja-build pkg-config \ - libglib2.0-dev libpixman-1-dev libslirp-dev \ - libgcrypt20-dev \ - python3 python3-venv - - - name: Clone and build Espressif QEMU - if: steps.cache-qemu.outputs.cache-hit != 'true' + - name: Prepare SDK-provided Espressif QEMU run: | - git clone --depth 1 -b "$QEMU_BRANCH" "$QEMU_REPO" /tmp/qemu-esp - cd /tmp/qemu-esp - mkdir build && cd build - ../configure \ - --target-list=xtensa-softmmu \ - --prefix=/opt/qemu-esp32 \ - --enable-slirp \ - --disable-werror - ninja -j$(nproc) - ninja install + . "$IDF_PATH/export.sh" + QEMU_BIN="$(command -v qemu-system-xtensa)" + QEMU_ROOT="$(dirname "$(dirname "$QEMU_BIN")")" + "$QEMU_BIN" --version + "$QEMU_BIN" -machine help | grep -E '^esp32s3[[:space:]]' + mkdir -p /opt/qemu-esp32 + cp -a "$QEMU_ROOT/." /opt/qemu-esp32/ - name: Verify QEMU binary run: | @@ -85,7 +67,7 @@ jobs: needs: build-qemu runs-on: ubuntu-latest container: - image: espressif/idf:v5.4 + image: espressif/idf:v5.4@sha256:f1e9f69dc052b9afc7801ca884e0ef40c17e014bb05ce73d9c09d29290bd17fb strategy: fail-fast: false @@ -214,32 +196,35 @@ jobs: fuzz-test: name: Fuzz Testing (ADR-061 Layer 6) - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + env: + CC: clang-18 steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 with: submodules: recursive - - name: Install clang + - name: Verify hosted Clang + # Ubuntu 24.04 hosted runners already contain Clang 18 and libFuzzer. + # Do not fetch the redundant distro meta-package before testing. run: | - sudo apt-get update - sudo apt-get install -y clang + "$CC" --version - name: Build fuzz targets working-directory: firmware/esp32-csi-node/test - run: make all CC=clang + run: make all CC="$CC" - name: Run serialize fuzzer (60s) working-directory: firmware/esp32-csi-node/test - run: make run_serialize FUZZ_DURATION=60 || echo "FUZZER_CRASH=serialize" >> "$GITHUB_ENV" + run: make run_serialize CC="$CC" FUZZ_DURATION=60 || echo "FUZZER_CRASH=serialize" >> "$GITHUB_ENV" - name: Run edge enqueue fuzzer (60s) working-directory: firmware/esp32-csi-node/test - run: make run_edge FUZZ_DURATION=60 || echo "FUZZER_CRASH=edge" >> "$GITHUB_ENV" + run: make run_edge CC="$CC" FUZZ_DURATION=60 || echo "FUZZER_CRASH=edge" >> "$GITHUB_ENV" - name: Run NVS config fuzzer (60s) working-directory: firmware/esp32-csi-node/test - run: make run_nvs FUZZ_DURATION=60 || echo "FUZZER_CRASH=nvs" >> "$GITHUB_ENV" + run: make run_nvs CC="$CC" FUZZ_DURATION=60 || echo "FUZZER_CRASH=nvs" >> "$GITHUB_ENV" - name: Check for crashes working-directory: firmware/esp32-csi-node/test @@ -320,7 +305,7 @@ jobs: needs: [build-qemu] runs-on: ubuntu-latest container: - image: espressif/idf:v5.4 + image: espressif/idf:v5.4@sha256:f1e9f69dc052b9afc7801ca884e0ef40c17e014bb05ce73d9c09d29290bd17fb steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 diff --git a/.github/workflows/mqtt-integration.yml b/.github/workflows/mqtt-integration.yml index 518fea91bd..634ee21d73 100644 --- a/.github/workflows/mqtt-integration.yml +++ b/.github/workflows/mqtt-integration.yml @@ -33,11 +33,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 20 - # NB: we don't use a `services:` mosquitto container here because the - # eclipse-mosquitto:2.x image rejects anonymous connections by default - # and GH Actions `services` doesn't easily support mounting a custom - # config file. We start mosquitto manually in a step below with an - # inline `allow_anonymous true` config. + # Explicit vendor broker config; publish only the CI loopback port. + # No APT package install or production MQTT security policy is involved. env: RUVIEW_RUN_INTEGRATION: "1" @@ -50,29 +47,26 @@ jobs: with: submodules: recursive - - name: Install mosquitto + clients and start with allow_anonymous + - name: Start pinned Mosquitto broker with CI-only anonymous listener run: | - sudo apt-get update -qq - sudo apt-get install -y mosquitto mosquitto-clients - sudo systemctl stop mosquitto || true - # Inline config: anon listener on 11883 only — no TLS, no auth, - # OK for CI because we test the wire shape, not security. - # Production deployments enable mTLS per ADR-115 §3.9. - cat > /tmp/mosquitto-ci.conf <<'EOF' - listener 11883 + cat > "$RUNNER_TEMP/mosquitto-ci.conf" <<'CONFIG' + listener 1883 allow_anonymous true persistence false log_dest stdout - EOF - mosquitto -c /tmp/mosquitto-ci.conf -d + CONFIG + docker run --detach --user mosquitto --name ruview-mqtt-ci \ + --publish 127.0.0.1:11883:1883 \ + --mount "type=bind,source=$RUNNER_TEMP/mosquitto-ci.conf,target=/mosquitto/config/mosquitto.conf,readonly" \ + eclipse-mosquitto:2.0.18@sha256:d12c8f80dfc65b768bb9acecc7ef182b976f71fb681640b66358e5e0cf94e9e9 for i in {1..20}; do - if mosquitto_pub -h 127.0.0.1 -p 11883 -t healthcheck -m ok -q 0 2>/dev/null; then - echo "mosquitto reachable on 11883"; exit 0 + if docker exec ruview-mqtt-ci mosquitto_pub -h 127.0.0.1 -p 1883 -t healthcheck -m ok -q 0 \ + && python3 -c 'import socket; socket.create_connection(("127.0.0.1", 11883), timeout=3).close()'; then + exit 0 fi - sleep 2 + sleep 1 done - echo "mosquitto never became reachable" >&2 - tail -50 /var/log/mosquitto/*.log 2>/dev/null || true + docker logs ruview-mqtt-ci exit 1 - name: Install Rust toolchain @@ -113,5 +107,8 @@ jobs: - name: Dump broker logs on failure if: failure() run: | - docker ps -a - docker logs $(docker ps -aqf "ancestor=eclipse-mosquitto:2.0.18") || true + docker logs ruview-mqtt-ci || true + + - name: Stop the owned CI broker + if: always() + run: docker rm --force ruview-mqtt-ci diff --git a/scripts/ci-install-ubuntu-deps.sh b/scripts/ci-install-ubuntu-deps.sh new file mode 100755 index 0000000000..15324cb144 --- /dev/null +++ b/scripts/ci-install-ubuntu-deps.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Ephemeral Ubuntu 24.04 CI dependency installation. Keep Ubuntu signature/TLS +# verification, but bypass the hosted image's failing mirror+file indirection. +set -euo pipefail + +if (( $# == 0 )); then + echo 'Pass one or more package names.' >&2 + exit 2 +fi +for package in "$@"; do + if [[ ! "$package" =~ ^[a-z0-9][a-z0-9+.-]*(:amd64)?$ ]]; then + echo 'Invalid package argument; APT options are not accepted.' >&2 + exit 2 + fi +done +if [[ $(uname -s) != Linux ]] || [[ ! -r /etc/os-release ]]; then + echo 'This helper requires an Ubuntu 24.04 amd64 CI runner.' >&2 + exit 2 +fi +os_id=$(. /etc/os-release; printf '%s' "$ID") +os_version=$(. /etc/os-release; printf '%s' "$VERSION_ID") +if [[ "$os_id" != ubuntu || "$os_version" != 24.04 || $(dpkg --print-architecture) != amd64 ]]; then + echo 'This helper requires an Ubuntu 24.04 amd64 CI runner.' >&2 + exit 2 +fi +keyring=/usr/share/keyrings/ubuntu-archive-keyring.gpg +[[ -r "$keyring" ]] || { echo 'Ubuntu archive keyring is unavailable.' >&2; exit 1; } +# Fail quickly on actual repository DNS loss, rather than spending hours trying +# each package through broken mirror fallbacks. +for host in archive.ubuntu.com security.ubuntu.com; do + timeout 20 getent ahostsv4 "$host" >/dev/null +done +apt_dir=$(mktemp -d -t ruview-ci-apt.XXXXXX) +source_list="$apt_dir/ubuntu.list" +parts_dir="$apt_dir/empty" +trap 'rm -f -- "$source_list"; rmdir -- "$parts_dir" "$apt_dir"' EXIT +mkdir "$parts_dir" +chmod 755 "$apt_dir" "$parts_dir" +cat > "$source_list" < Date: Wed, 11 Nov 2026 21:16:52 +0100 Subject: [PATCH 098/100] perf(aether): reuse attention scratch buffers without numeric changes --- .../src/graph_transformer.rs | 73 ++++++-- .../graph_transformer_equivalence_tests.rs | 172 ++++++++++++++++++ 2 files changed, 228 insertions(+), 17 deletions(-) create mode 100644 v2/crates/wifi-densepose-aether/src/graph_transformer_equivalence_tests.rs diff --git a/v2/crates/wifi-densepose-aether/src/graph_transformer.rs b/v2/crates/wifi-densepose-aether/src/graph_transformer.rs index 395752b8f0..b31b650d81 100644 --- a/v2/crates/wifi-densepose-aether/src/graph_transformer.rs +++ b/v2/crates/wifi-densepose-aether/src/graph_transformer.rs @@ -130,6 +130,18 @@ impl Linear { } out } + // Same scalar accumulation as forward, writing into an existing buffer. + fn forward_into(&self, input: &[f32], out: &mut [f32]) { + assert_eq!(input.len(), self.in_features); + assert_eq!(out.len(), self.out_features); + for (i, row) in self.weights.iter().enumerate() { + let mut s = self.bias[i]; + for (w, x) in row.iter().zip(input) { + s += w * x; + } + out[i] = s; + } + } pub fn weights(&self) -> &[Vec] { &self.weights } @@ -393,38 +405,56 @@ impl CrossAttention { return vec![vec![0.0; self.d_model]; n_q]; } - let q_proj: Vec> = query.iter().map(|q| self.w_q.forward(q)).collect(); - let k_proj: Vec> = key.iter().map(|k| self.w_k.forward(k)).collect(); - let v_proj: Vec> = value.iter().map(|v| self.w_v.forward(v)).collect(); + let mut q_proj = vec![0.0; n_q * self.d_model]; + let mut k_proj = vec![0.0; n_kv * self.d_model]; + let mut v_proj = vec![0.0; value.len() * self.d_model]; + for (i, row) in query.iter().enumerate() { + self.w_q + .forward_into(row, &mut q_proj[i * self.d_model..(i + 1) * self.d_model]); + } + for (i, row) in key.iter().enumerate() { + self.w_k + .forward_into(row, &mut k_proj[i * self.d_model..(i + 1) * self.d_model]); + } + for (i, row) in value.iter().enumerate() { + self.w_v + .forward_into(row, &mut v_proj[i * self.d_model..(i + 1) * self.d_model]); + } let scale = (self.d_k as f32).sqrt(); let mut output = vec![vec![0.0f32; self.d_model]; n_q]; + // Scratch storage is reused across queries/heads. Keep the original + // scalar reduction order and concatenated width, including rejection of + // an inconsistent shape restored through unflatten_from. + let mut concat = vec![0.0; self.n_heads * self.d_k]; + let mut scores = vec![0.0; n_kv]; + let mut wts = vec![0.0; n_kv]; for qi in 0..n_q { - let mut concat = Vec::with_capacity(self.d_model); for h in 0..self.n_heads { let (start, end) = (h * self.d_k, (h + 1) * self.d_k); - let q_h = &q_proj[qi][start..end]; - let mut scores = vec![0.0f32; n_kv]; + let q_h = &q_proj[qi * self.d_model + start..qi * self.d_model + end]; for ki in 0..n_kv { let dot: f32 = q_h .iter() - .zip(&k_proj[ki][start..end]) + .zip(&k_proj[ki * self.d_model + start..ki * self.d_model + end]) .map(|(a, b)| a * b) .sum(); scores[ki] = dot / scale; } - let mut wts = vec![0.0f32; n_kv]; softmax(&scores, &mut wts); - let mut head_out = vec![0.0f32; self.d_k]; + let head_out = &mut concat[start..end]; + head_out.fill(0.0); for ki in 0..n_kv { - for (o, &v) in head_out.iter_mut().zip(&v_proj[ki][start..end]) { + for (o, &v) in head_out + .iter_mut() + .zip(&v_proj[ki * self.d_model + start..ki * self.d_model + end]) + { *o += wts[ki] * v; } } - concat.extend_from_slice(&head_out); } - output[qi] = self.w_o.forward(&concat); + self.w_o.forward_into(&concat, &mut output[qi]); } output } @@ -506,21 +536,26 @@ impl GraphMessagePassing { "expected 17 nodes, got {}", node_features.len() ); - let mut agg = vec![vec![0.0f32; self.in_features]; 17]; + // Rows are independent: reuse the aggregation buffer, clearing it + // before the same ascending-neighbor reduction used by the matrix path. + let mut agg = vec![0.0f32; self.in_features]; + let mut output = Vec::with_capacity(17); #[allow(clippy::needless_range_loop)] for i in 0..17 { + agg.fill(0.0); for j in 0..17 { let a = self.norm_adj[i][j]; if a.abs() > 1e-10 { - for (ag, &f) in agg[i].iter_mut().zip(&node_features[j]) { + for (ag, &f) in agg.iter_mut().zip(&node_features[j]) { *ag += a * f; } } } + let mut row = self.weight.forward(&agg); + row.iter_mut().for_each(|value| *value = relu(*value)); + output.push(row); } - agg.iter() - .map(|a| self.weight.forward(a).into_iter().map(relu).collect()) - .collect() + output } pub fn in_features(&self) -> usize { self.in_features @@ -1152,3 +1187,7 @@ mod tests { } } } + +#[cfg(test)] +#[path = "graph_transformer_equivalence_tests.rs"] +mod equivalence_tests; diff --git a/v2/crates/wifi-densepose-aether/src/graph_transformer_equivalence_tests.rs b/v2/crates/wifi-densepose-aether/src/graph_transformer_equivalence_tests.rs new file mode 100644 index 0000000000..fa76c1f177 --- /dev/null +++ b/v2/crates/wifi-densepose-aether/src/graph_transformer_equivalence_tests.rs @@ -0,0 +1,172 @@ +//! Frozen pre-optimization scalar oracles. These deliberately retain the +//! allocation-heavy paths so stale scratch data, reduction-order changes and +//! finite-difference perturbations cannot silently change trained artifacts. +use super::*; +fn bits(rows: Vec>) -> Vec> { + rows.into_iter() + .map(|row| row.into_iter().map(f32::to_bits).collect()) + .collect() +} +fn matrix(rows: usize, width: usize, seed: usize) -> Vec> { + (0..rows) + .map(|i| { + (0..width) + .map(|j| (((i * 17 + j * 7 + seed) % 31) as f32 - 15.0) / 8.0) + .collect() + }) + .collect() +} +#[allow(clippy::needless_range_loop)] +fn reference_attention( + layer: &CrossAttention, + query: &[Vec], + key: &[Vec], + value: &[Vec], +) -> Vec> { + let (n_q, n_kv) = (query.len(), key.len()); + if n_q == 0 || n_kv == 0 { + return vec![vec![0.0; layer.d_model]; n_q]; + } + + let q_proj: Vec> = query.iter().map(|q| layer.w_q.forward(q)).collect(); + let k_proj: Vec> = key.iter().map(|k| layer.w_k.forward(k)).collect(); + let v_proj: Vec> = value.iter().map(|v| layer.w_v.forward(v)).collect(); + + let scale = (layer.d_k as f32).sqrt(); + let mut output = vec![vec![0.0f32; layer.d_model]; n_q]; + + for qi in 0..n_q { + let mut concat = Vec::with_capacity(layer.d_model); + for h in 0..layer.n_heads { + let (start, end) = (h * layer.d_k, (h + 1) * layer.d_k); + let q_h = &q_proj[qi][start..end]; + let mut scores = vec![0.0f32; n_kv]; + for ki in 0..n_kv { + let dot: f32 = q_h + .iter() + .zip(&k_proj[ki][start..end]) + .map(|(a, b)| a * b) + .sum(); + scores[ki] = dot / scale; + } + let mut wts = vec![0.0f32; n_kv]; + softmax(&scores, &mut wts); + let mut head_out = vec![0.0f32; layer.d_k]; + for ki in 0..n_kv { + for (o, &v) in head_out.iter_mut().zip(&v_proj[ki][start..end]) { + *o += wts[ki] * v; + } + } + concat.extend_from_slice(&head_out); + } + output[qi] = layer.w_o.forward(&concat); + } + output +} +#[allow(clippy::needless_range_loop)] +fn reference_graph(layer: &GraphMessagePassing, node_features: &[Vec]) -> Vec> { + assert_eq!( + node_features.len(), + 17, + "expected 17 nodes, got {}", + node_features.len() + ); + let mut agg = vec![vec![0.0f32; layer.in_features]; 17]; + #[allow(clippy::needless_range_loop)] + for i in 0..17 { + for j in 0..17 { + let a = layer.norm_adj[i][j]; + if a.abs() > 1e-10 { + for (ag, &f) in agg[i].iter_mut().zip(&node_features[j]) { + *ag += a * f; + } + } + } + } + agg.iter() + .map(|a| layer.weight.forward(a).into_iter().map(relu).collect()) + .collect() +} +#[test] +fn attention_preserves_bits_across_shapes_and_parameter_probes() { + for (width, heads) in [(0, 1), (2, 1), (3, 1), (4, 2), (8, 4), (16, 4)] { + let layer = CrossAttention::new(width, heads); + let mut parameters = Vec::new(); + layer.flatten_into(&mut parameters); + for (queries, keys, values) in [(0, 4, 4), (17, 0, 0), (1, 1, 1), (17, 4, 4), (3, 7, 9)] { + let q = matrix(queries, width, 3); + let k = matrix(keys, width, 7); + let v = matrix(values, width, 19); + assert_eq!( + bits(layer.forward(&q, &k, &v)), + bits(reference_attention(&layer, &q, &k, &v)) + ); + for index in 0..parameters.len() { + for epsilon in [-0.002, 0.002] { + let mut changed = parameters.clone(); + changed[index] += epsilon; + let (probe, _) = CrossAttention::unflatten_from(&changed, width, heads); + assert_eq!( + bits(probe.forward(&q, &k, &v)), + bits(reference_attention(&probe, &q, &k, &v)), + "width={width} heads={heads} parameter={index} epsilon={epsilon}" + ); + } + } + } + } +} +#[test] +fn graph_aggregation_preserves_bits_across_widths_and_parameter_probes() { + for (input, output) in [(0, 3), (3, 3), (4, 2), (8, 6)] { + let layer = GraphMessagePassing::new(input, output, &BodyGraph::new()); + let features = matrix(17, input, 11); + let mut parameters = Vec::new(); + layer.weight.flatten_into(&mut parameters); + assert_eq!( + bits(layer.forward(&features)), + bits(reference_graph(&layer, &features)) + ); + for index in 0..parameters.len() { + for epsilon in [-0.002, 0.002] { + let mut changed = parameters.clone(); + changed[index] += epsilon; + let mut probe = layer.clone(); + probe.weight = Linear::unflatten_from(&changed, input, output).0; + assert_eq!( + bits(probe.forward(&features)), + bits(reference_graph(&probe, &features)) + ); + } + } + } +} +#[test] +fn linear_reuse_overwrites_every_output_and_matches_allocating_path() { + for (input, output) in [(0, 3), (3, 3), (4, 2), (8, 6)] { + let layer = Linear::with_seed(input, output, 29); + let mut scratch = vec![f32::NAN; output]; + for row in matrix(17, input, 13) { + layer.forward_into(&row, &mut scratch); + assert_eq!( + scratch.iter().map(|x| x.to_bits()).collect::>(), + layer + .forward(&row) + .iter() + .map(|x| x.to_bits()) + .collect::>() + ); + } + } +} +#[test] +fn inconsistent_restored_attention_shape_still_rejects() { + let original = CrossAttention::new(3, 1); + let mut parameters = Vec::new(); + original.flatten_into(&mut parameters); + let (invalid, _) = CrossAttention::unflatten_from(¶meters, 3, 2); + let q = matrix(17, 3, 1); + let k = matrix(4, 3, 2); + assert!(std::panic::catch_unwind(|| reference_attention(&invalid, &q, &k, &k)).is_err()); + assert!(std::panic::catch_unwind(|| invalid.forward(&q, &k, &k)).is_err()); +} From 68cd04927ac503ae68313edc249c6ea8627319c2 Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Wed, 11 Nov 2026 21:26:28 +0100 Subject: [PATCH 099/100] fix(aether): retain missing-value rejection for zero-width attention --- .../wifi-densepose-aether/src/graph_transformer.rs | 3 +++ .../src/graph_transformer_equivalence_tests.rs | 14 ++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/v2/crates/wifi-densepose-aether/src/graph_transformer.rs b/v2/crates/wifi-densepose-aether/src/graph_transformer.rs index b31b650d81..f68a186d90 100644 --- a/v2/crates/wifi-densepose-aether/src/graph_transformer.rs +++ b/v2/crates/wifi-densepose-aether/src/graph_transformer.rs @@ -421,6 +421,9 @@ impl CrossAttention { .forward_into(row, &mut v_proj[i * self.d_model..(i + 1) * self.d_model]); } + // The former row-indexed V projection rejected missing value rows even + // for zero-width models. Flat empty slices alone cannot enforce that. + assert!(value.len() >= n_kv, "missing attention value rows"); let scale = (self.d_k as f32).sqrt(); let mut output = vec![vec![0.0f32; self.d_model]; n_q]; diff --git a/v2/crates/wifi-densepose-aether/src/graph_transformer_equivalence_tests.rs b/v2/crates/wifi-densepose-aether/src/graph_transformer_equivalence_tests.rs index fa76c1f177..6b168decb5 100644 --- a/v2/crates/wifi-densepose-aether/src/graph_transformer_equivalence_tests.rs +++ b/v2/crates/wifi-densepose-aether/src/graph_transformer_equivalence_tests.rs @@ -170,3 +170,17 @@ fn inconsistent_restored_attention_shape_still_rejects() { assert!(std::panic::catch_unwind(|| reference_attention(&invalid, &q, &k, &k)).is_err()); assert!(std::panic::catch_unwind(|| invalid.forward(&q, &k, &k)).is_err()); } + +#[test] +fn missing_value_rows_still_reject_including_zero_width() { + for width in [0, 3] { + let layer = CrossAttention::new(width, 1); + let q = matrix(2, width, 1); + let k = matrix(4, width, 2); + for values in [0, 3] { + let v = matrix(values, width, 3); + assert!(std::panic::catch_unwind(|| reference_attention(&layer, &q, &k, &v)).is_err()); + assert!(std::panic::catch_unwind(|| layer.forward(&q, &k, &v)).is_err()); + } + } +} From 4f70c5eafe28a17395f1588cc15f5e64ba932d7d Mon Sep 17 00:00:00 2001 From: Dragan Spiridonov Date: Thu, 12 Nov 2026 19:25:21 +0100 Subject: [PATCH 100/100] test: await segment preparation before synthetic operator packets --- .github/workflows/ci.yml | 8 ++-- .../lifecycle_temporal_full_run_tests.rs | 47 ++++++++++++++++++- 2 files changed, 49 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 752a166963..a0ba7184ed 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -430,9 +430,9 @@ jobs: # workflow, so it doesn't gate the rest of the pipeline. docker-build: name: Docker Build & Test - # Manual CI is verification only. Use the separate canonical publisher - # for an explicitly authorized image publication. - if: github.event_name != 'workflow_dispatch' + # PR/manual verification must not publish images. The existing main-push + # publisher behavior is separate from these verification-only events. + if: github.event_name == 'push' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest needs: [code-quality, test, rust-tests] continue-on-error: true @@ -554,7 +554,7 @@ jobs: runs-on: ubuntu-latest needs: [code-quality, test, rust-tests, ui-tests, performance-test, docker-build, docs] # This job also creates releases and sends external notifications. - if: always() && github.event_name != 'workflow_dispatch' + if: always() && github.event_name == 'push' && github.ref == 'refs/heads/main' permissions: contents: write # required by softprops/action-gh-release # GitHub Actions does not allow `secrets.X` directly in step-level `if:` diff --git a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs index 19d2f05fdd..7956565ee4 100644 --- a/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs +++ b/v2/crates/wifi-densepose-sensing-server/src/room_learner/lifecycle_temporal_full_run_tests.rs @@ -270,6 +270,27 @@ fn packet(source: u8, sequence: u32, at: u64, label: RoomLabel) -> CanonicalPack }) .unwrap() } +// This functional fixture supplies a synthetic receiver timeline. Preparation +// I/O must finish before that timeline starts producing packets; otherwise host +// serialization latency repeatedly consumes the first Check window's buffer. +// Real-paced and storage-backpressure acceptance helpers remain separate. +fn wait_for_segment_commit(rt: &RoomLearningRuntime, at: u64, phase_started: Instant) { + loop { + rt.tick_temporal_capture(&rt.mutation_guard().unwrap(), at).unwrap(); + let progress = fixture::snapshot(rt, at); + if progress["durability_pending"] == false { + assert_eq!(progress["stage"], "observing"); + assert_eq!(progress["retained_rows"], progress["durable_windows"]); + assert_eq!(progress["observed_ns"], progress["durable_observed_ns"], + "preparation cannot credit a synthetic observation"); + return; + } + assert!(phase_started.elapsed() < Duration::from_secs(240), + "segment preparation did not settle within the original phase guard: {progress}"); + std::thread::sleep(Duration::from_millis(1)); + } +} + fn record_train_uncached(root: &Path) -> (Arc, u64) { assert!(!cfg!(debug_assertions), "complete native-persistence fixture requires Release; classify the caller"); let (runtime, mut now) = configured(root); @@ -1124,7 +1145,26 @@ async fn operator_eight_actual_reserved_checks_fit_complete_cold_and_evaluate(){ loop{ let progress=fixture::snapshot(&runtime,now); match progress["stage"].as_str().unwrap(){ - "ready"|"paused"=>{now=fixture::async_lead_end(&runtime,now);runtime.tick_temporal_capture(&runtime.mutation_guard().unwrap(),now).unwrap();} + "ready"|"paused"=>{ + now=fixture::async_lead_end(&runtime,now); + // Exercise preparation slower than the first Check slot's + // buffer horizon. No synthetic packet may precede its ACK. + let delayed = if episode.index == 7 { + let writer={let state=runtime.inner.lock().unwrap();let Some(CaptureWorkflow::Temporal(c))=&state.capture else{panic!()};c.core.as_ref().unwrap().writer().unwrap()}; + let(entered_tx,entered_rx)=std::sync::mpsc::sync_channel(1); + let(release_tx,release_rx)=std::sync::mpsc::sync_channel(1); + writer.hold_next_after_publish(entered_tx,release_rx); + let timer_release=release_tx.clone(); + let timer=std::thread::spawn(move||{ + entered_rx.recv_timeout(Duration::from_secs(30)).unwrap(); + std::thread::sleep(Duration::from_secs(5)); + let _=timer_release.send(()); + }); + Some((ReleaseGate(Some(release_tx)),timer)) + }else{None}; + wait_for_segment_commit(&runtime,now,supervised); + if let Some((mut gate,timer))=delayed{timer.join().unwrap();gate.release();} + } "needs_confirmation"=>break, "failed"=>panic!("operator episode{} failed: {progress}",episode.index), _=>{} @@ -1133,7 +1173,10 @@ async fn operator_eight_actual_reserved_checks_fit_complete_cold_and_evaluate(){ for source in 1..=3{match runtime.ingest_native(&packet(source,sequence,now,episode.condition),now){Ok(())=>{},Err(e)if e=="temporal_storage_busy_pause"=>break,Err(e)=>panic!("operator ingest: {e}")}} now+=fixture::PERIOD; if fixture::snapshot(&runtime,now)["durability_pending"]==true{std::thread::sleep(Duration::from_nanos(fixture::PERIOD));} - assert!(supervised.elapsed()